From 3fbe20129daf1ebff6e95e1643c326172198242c Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Sat, 1 Feb 2020 20:50:47 -0800
Subject: [PATCH 0001/1280] [SPARK-27686][DOC][SQL] Update migration guide for
 make Hive 2.3 dependency by default

### What changes were proposed in this pull request?

We have upgraded the built-in Hive from 1.2 to 2.3. This may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of your Hive metastore. Example:
```
--conf spark.sql.hive.metastore.version=1.2.1 --conf spark.sql.hive.metastore.jars=/root/hive-1.2.1-lib/*
```
Otherwise:
```
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to fetch table spark_27686. Invalid method name: 'get_table_req';
  at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:110)
  at org.apache.spark.sql.hive.HiveExternalCatalog.tableExists(HiveExternalCatalog.scala:841)
  at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.tableExists(ExternalCatalogWithListener.scala:146)
  at org.apache.spark.sql.catalyst.catalog.SessionCatalog.tableExists(SessionCatalog.scala:431)
  at org.apache.spark.sql.execution.command.CreateDataSourceTableCommand.run(createDataSourceTables.scala:52)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
  at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:226)
  at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3487)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$4(SQLExecution.scala:100)
  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:87)
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3485)
  at org.apache.spark.sql.Dataset.<init>(Dataset.scala:226)
  at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:96)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:607)
  ... 47 elided
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to fetch table spark_27686. Invalid method name: 'get_table_req'
  at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1282)
  at org.apache.spark.sql.hive.client.HiveClientImpl.getRawTableOption(HiveClientImpl.scala:422)
  at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$tableExists$1(HiveClientImpl.scala:436)
  at scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.java:23)
  at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:322)
  at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:256)
  at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:255)
  at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:305)
  at org.apache.spark.sql.hive.client.HiveClientImpl.tableExists(HiveClientImpl.scala:436)
  at org.apache.spark.sql.hive.HiveExternalCatalog.$anonfun$tableExists$1(HiveExternalCatalog.scala:841)
  at scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.java:23)
  at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:100)
  ... 63 more
Caused by: org.apache.thrift.TApplicationException: Invalid method name: 'get_table_req'
  at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:79)
  at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_table_req(ThriftHiveMetastore.java:1567)
  at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_table_req(ThriftHiveMetastore.java:1554)
  at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getTable(HiveMetaStoreClient.java:1350)
  at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.getTable(SessionHiveMetaStoreClient.java:127)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:498)
  at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
  at com.sun.proxy.$Proxy38.getTable(Unknown Source)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:498)
  at org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2336)
  at com.sun.proxy.$Proxy38.getTable(Unknown Source)
  at org.apache.hadoop.hive.ql.metadata.Hive.getTable(Hive.java:1274)
  ... 74 more
```

### Why are the changes needed?

Improve documentation.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?
```SKIP_API=1 jekyll build```:
![image](https://user-images.githubusercontent.com/5399861/73531432-67a50b80-4455-11ea-9401-5cad12fd3d14.png)

Closes #27161 from wangyum/SPARK-27686.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit cd5f03a3ba18ae455f93abc5e5d04f098fa8f046)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-migration-guide.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 533c96a0832de..e4d2358b5de09 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -330,6 +330,9 @@ license: |
 
   - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
 
+  - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3. This may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore.
+  For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
+
 ## Upgrading from Spark SQL 2.4.4 to 2.4.5
 
   - Since Spark 2.4.5, `TRUNCATE TABLE` command tries to set back original permission and ACLs during re-creating the table/partition paths. To restore the behaviour of earlier versions, set `spark.sql.truncateTable.ignorePermissionAcl.enabled` to `true`.

From 2f1fb4c01d0d4bfda17b3262e6f586f4f1a25bac Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Sun, 2 Feb 2020 00:44:25 -0800
Subject: [PATCH 0002/1280] [SPARK-30704][INFRA] Use jekyll-redirect-from
 0.15.0 instead of the latest

### What changes were proposed in this pull request?

This PR aims to pin the version of `jekyll-redirect-from` to 0.15.0. This is a release blocker for both Apache Spark 3.0.0 and 2.4.5.

### Why are the changes needed?

`jekyll-redirect-from` released 0.16.0 a few days ago and that requires Ruby 2.4.0.
- https://github.com/jekyll/jekyll-redirect-from/releases/tag/v0.16.0
```
$ cd dev/create-release/spark-rm/
$ docker build -t spark:test .
...
ERROR:  Error installing jekyll-redirect-from:
	jekyll-redirect-from requires Ruby version >= 2.4.0.
...
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Manually do the above command to build `spark-rm` Docker image.
```
...
Successfully installed jekyll-redirect-from-0.15.0
Parsing documentation for jekyll-redirect-from-0.15.0
Installing ri documentation for jekyll-redirect-from-0.15.0
Done installing documentation for jekyll-redirect-from after 0 seconds
1 gem installed
Successfully installed rouge-3.15.0
Parsing documentation for rouge-3.15.0
Installing ri documentation for rouge-3.15.0
Done installing documentation for rouge after 4 seconds
1 gem installed
Removing intermediate container e0ec7c77b69f
 ---> 32dec37291c6
```

Closes #27434 from dongjoon-hyun/SPARK-30704.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 1adf3520e3c753e6df8dccb752e8239de682a09a)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 dev/create-release/spark-rm/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 3ba8e97929613..63451687ee8c2 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -78,7 +78,7 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   # Install tools needed to build the documentation.
   $APT_INSTALL ruby2.3 ruby2.3-dev mkdocs && \
   gem install jekyll --no-rdoc --no-ri -v 3.8.6 && \
-  gem install jekyll-redirect-from && \
+  gem install jekyll-redirect-from -v 0.15.0 && \
   gem install rouge
 
 WORKDIR /opt/spark-rm/output

From 91f78aee718888fad5677445ba21024263d1037a Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Mon, 3 Feb 2020 14:08:59 +0800
Subject: [PATCH 0003/1280] [SPARK-30697][SQL] Handle database and namespace
 exceptions in catalog.isView

### What changes were proposed in this pull request?

Adds NoSuchDatabaseException and NoSuchNamespaceException to the `isView` method for SessionCatalog.

### Why are the changes needed?

This method prevents specialized resolutions from kicking in within Analysis when using V2 Catalogs if the identifier is a specialized identifier.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Added test to DataSourceV2SessionCatalogSuite

Closes #27423 from brkyvz/isViewF.

Authored-by: Burak Yavuz <brkyvz@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2eccfd8a73c4afa30a6aa97c2afd38661f29e24b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/catalog/SessionCatalog.scala |  2 ++
 ...SourceV2DataFrameSessionCatalogSuite.scala | 22 +++++++++++++++++++
 .../DataSourceV2SQLSessionCatalogSuite.scala  | 14 ++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 45f0ef6c97e70..12f9a61fc2b65 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -826,6 +826,8 @@ class SessionCatalog(
         getTempViewOrPermanentTableMetadata(ident).tableType == CatalogTableType.VIEW
       } catch {
         case _: NoSuchTableException => false
+        case _: NoSuchDatabaseException => false
+        case _: NoSuchNamespaceException => false
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
index 4c67888cbdc48..01caf8e2eb115 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
@@ -101,6 +101,13 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
     new InMemoryTable(name, schema, partitions, properties)
   }
 
+  override def loadTable(ident: Identifier): Table = {
+    val identToUse = Option(InMemoryTableSessionCatalog.customIdentifierResolution)
+      .map(_(ident))
+      .getOrElse(ident)
+    super.loadTable(identToUse)
+  }
+
   override def alterTable(ident: Identifier, changes: TableChange*): Table = {
     val fullIdent = fullIdentifier(ident)
     Option(tables.get(fullIdent)) match {
@@ -125,6 +132,21 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
   }
 }
 
+object InMemoryTableSessionCatalog {
+  private var customIdentifierResolution: Identifier => Identifier = _
+
+  def withCustomIdentifierResolver(
+      resolver: Identifier => Identifier)(
+      f: => Unit): Unit = {
+    try {
+      customIdentifierResolution = resolver
+      f
+    } finally {
+      customIdentifierResolution = null
+    }
+  }
+}
+
 private [connector] trait SessionCatalogTest[T <: Table, Catalog <: TestV2SessionCatalogBase[T]]
   extends QueryTest
   with SharedSparkSession
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
index 27725bcadbcd5..b6997445013e5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
@@ -49,4 +49,18 @@ class DataSourceV2SQLSessionCatalogSuite
     v2Catalog.asInstanceOf[TableCatalog]
       .loadTable(Identifier.of(Array.empty, nameParts.last))
   }
+
+  test("SPARK-30697: catalog.isView doesn't throw an error for specialized identifiers") {
+    val t1 = "tbl"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format")
+
+      def idResolver(id: Identifier): Identifier = Identifier.of(Array.empty, id.name())
+
+      InMemoryTableSessionCatalog.withCustomIdentifierResolver(idResolver) {
+        // The following should not throw AnalysisException.
+        sql(s"DESCRIBE TABLE ignored.$t1")
+      }
+    }
+  }
 }

From f9b86370cb04b72a4f00cbd4d60873960aa2792c Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sun, 2 Feb 2020 23:37:13 -0800
Subject: [PATCH 0004/1280] [SPARK-29543][SS][FOLLOWUP] Move
 `spark.sql.streaming.ui.*` configs to StaticSQLConf

### What changes were proposed in this pull request?
Put the configs below needed by Structured Streaming UI into StaticSQLConf:

- spark.sql.streaming.ui.enabled
- spark.sql.streaming.ui.retainedProgressUpdates
- spark.sql.streaming.ui.retainedQueries

### Why are the changes needed?
Make all SS UI configs consistent with other similar configs in usage and naming.

### Does this PR introduce any user-facing change?
Yes, add new static config `spark.sql.streaming.ui.retainedProgressUpdates`.

### How was this patch tested?
Existing UT.

Closes #27425 from xuanyuanking/SPARK-29543-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
(cherry picked from commit a4912cee615314e9578e6ab4eae25f147feacbd5)
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 16 ---------------
 .../spark/sql/internal/StaticSQLConf.scala    | 20 +++++++++++++++++++
 .../spark/sql/internal/SharedState.scala      | 15 +++++++-------
 .../ui/StreamingQueryStatusListener.scala     | 10 ++++++----
 .../sql/streaming/ui/StreamingQueryTab.scala  |  2 +-
 .../StreamingQueryStatusListenerSuite.scala   |  4 ++--
 6 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 04572c38be8dd..3ad3416256c7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1150,18 +1150,6 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
-  val STREAMING_UI_ENABLED =
-    buildConf("spark.sql.streaming.ui.enabled")
-      .doc("Whether to run the structured streaming UI for the Spark application.")
-      .booleanConf
-      .createWithDefault(true)
-
-  val STREAMING_UI_INACTIVE_QUERY_RETENTION =
-    buildConf("spark.sql.streaming.ui.numInactiveQueries")
-      .doc("The number of inactive queries to retain for structured streaming ui.")
-      .intConf
-      .createWithDefault(100)
-
   val VARIABLE_SUBSTITUTE_ENABLED =
     buildConf("spark.sql.variable.substitute")
       .doc("This enables substitution using syntax like ${var} ${system:var} and ${env:var}.")
@@ -2284,10 +2272,6 @@ class SQLConf extends Serializable with Logging {
 
   def isUnsupportedOperationCheckEnabled: Boolean = getConf(UNSUPPORTED_OPERATION_CHECK_ENABLED)
 
-  def isStreamingUIEnabled: Boolean = getConf(STREAMING_UI_ENABLED)
-
-  def streamingUIInactiveQueryRetention: Int = getConf(STREAMING_UI_INACTIVE_QUERY_RETENTION)
-
   def streamingFileCommitProtocolClass: String = getConf(STREAMING_FILE_COMMIT_PROTOCOL_CLASS)
 
   def fileSinkLogDeletion: Boolean = getConf(FILE_SINK_LOG_DELETION)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index 66ac9ddb21aaa..6bc752260a893 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -176,4 +176,24 @@ object StaticSQLConf {
       .internal()
       .booleanConf
       .createWithDefault(true)
+
+  val STREAMING_UI_ENABLED =
+    buildStaticConf("spark.sql.streaming.ui.enabled")
+      .doc("Whether to run the Structured Streaming Web UI for the Spark application when the " +
+        "Spark Web UI is enabled.")
+      .booleanConf
+      .createWithDefault(true)
+
+  val STREAMING_UI_RETAINED_PROGRESS_UPDATES =
+    buildStaticConf("spark.sql.streaming.ui.retainedProgressUpdates")
+      .doc("The number of progress updates to retain for a streaming query for Structured " +
+        "Streaming UI.")
+      .intConf
+      .createWithDefault(100)
+
+  val STREAMING_UI_RETAINED_QUERIES =
+    buildStaticConf("spark.sql.streaming.ui.retainedQueries")
+      .doc("The number of inactive queries to retain for Structured Streaming UI.")
+      .intConf
+      .createWithDefault(100)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index fefd72dcf1752..5347264d7c50a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -145,13 +145,14 @@ private[sql] class SharedState(
    * data to show.
    */
   lazy val streamingQueryStatusListener: Option[StreamingQueryStatusListener] = {
-    val sqlConf = SQLConf.get
-    if (sqlConf.isStreamingUIEnabled) {
-      val statusListener = new StreamingQueryStatusListener(sqlConf)
-      sparkContext.ui.foreach(new StreamingQueryTab(statusListener, _))
-      Some(statusListener)
-    } else {
-      None
+    sparkContext.ui.flatMap { ui =>
+      if (conf.get(STREAMING_UI_ENABLED)) {
+        val statusListener = new StreamingQueryStatusListener(conf)
+        new StreamingQueryTab(statusListener, ui)
+        Some(statusListener)
+      } else {
+        None
+      }
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
index db085dbe87ec4..91815110e0d39 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
@@ -24,8 +24,9 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryProgress}
 
 /**
@@ -33,7 +34,7 @@ import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryPro
  * UI data for both active and inactive query.
  * TODO: Add support for history server.
  */
-private[sql] class StreamingQueryStatusListener(sqlConf: SQLConf) extends StreamingQueryListener {
+private[sql] class StreamingQueryStatusListener(conf: SparkConf) extends StreamingQueryListener {
 
   private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
   timestampFormat.setTimeZone(DateTimeUtils.getTimeZone("UTC"))
@@ -45,8 +46,9 @@ private[sql] class StreamingQueryStatusListener(sqlConf: SQLConf) extends Stream
   private[ui] val activeQueryStatus = new ConcurrentHashMap[UUID, StreamingQueryUIData]()
   private[ui] val inactiveQueryStatus = new mutable.Queue[StreamingQueryUIData]()
 
-  private val streamingProgressRetention = sqlConf.streamingProgressRetention
-  private val inactiveQueryStatusRetention = sqlConf.streamingUIInactiveQueryRetention
+  private val streamingProgressRetention =
+    conf.get(StaticSQLConf.STREAMING_UI_RETAINED_PROGRESS_UPDATES)
+  private val inactiveQueryStatusRetention = conf.get(StaticSQLConf.STREAMING_UI_RETAINED_QUERIES)
 
   override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = {
     activeQueryStatus.putIfAbsent(event.runId,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
index f909cfd97514e..bb097ffc06912 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryTab.scala
@@ -34,6 +34,6 @@ private[sql] class StreamingQueryTab(
   parent.addStaticHandler(StreamingQueryTab.STATIC_RESOURCE_DIR, "/static/sql")
 }
 
-object StreamingQueryTab {
+private[sql] object StreamingQueryTab {
   private val STATIC_RESOURCE_DIR = "org/apache/spark/sql/execution/ui/static"
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
index bd74ed340b408..adbb501f9842e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.streaming
 class StreamingQueryStatusListenerSuite extends StreamTest {
 
   test("onQueryStarted, onQueryProgress, onQueryTerminated") {
-    val listener = new StreamingQueryStatusListener(spark.sqlContext.conf)
+    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf)
 
     // hanlde query started event
     val id = UUID.randomUUID()
@@ -73,7 +73,7 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
   }
 
   test("same query start multiple times") {
-    val listener = new StreamingQueryStatusListener(spark.sqlContext.conf)
+    val listener = new StreamingQueryStatusListener(spark.sparkContext.conf)
 
     // handle first time start
     val id = UUID.randomUUID()

From a496750cf3d6da210ebbdbaf8e66910798242f2a Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Mon, 3 Feb 2020 19:57:16 -0800
Subject: [PATCH 0005/1280] [SPARK-30718][BUILD] Exclude jdk.tools dependency
 from hadoop-yarn-api

### What changes were proposed in this pull request?

This PR removes the `jdk.tools:jdk.tools` transitive dependency from `hadoop-yarn-api`.
- This is only used in `hadoop-annotation` project in some `*Doclet.java`.

### Why are the changes needed?

Although this is not used in Apache Spark, this can cause a resolve failure in JDK11 environment.

<img width="530" alt="jdk tools" src="https://user-images.githubusercontent.com/9700541/73697745-2f3f4080-4694-11ea-95a7-228638e31cf7.png">

### Does this PR introduce any user-facing change?

No. This is a dev-only change.
From developers, this will remove the `Cannot resolve` error in IDE environment.

### How was this patch tested?

- Pass the Jenkins in JDK8
- Manually, import the project with JDK11.

Closes #27445 from dongjoon-hyun/SPARK-30718.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 41bdb7ad3949d05542abe5ab2b440a51c3a18bce)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 pom.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pom.xml b/pom.xml
index 7c23444054efc..a8d6ac932bac2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1200,6 +1200,10 @@
             <groupId>com.sun.jersey.contribs</groupId>
             <artifactId>*</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>jdk.tools</groupId>
+            <artifactId>jdk.tools</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>

From b4e6a7c74737e96895d723e6d0cf499f7012c9ea Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Tue, 4 Feb 2020 12:31:44 +0800
Subject: [PATCH 0006/1280] [SPARK-30717][SQL] AQE subquery map should cache
 `SubqueryExec` instead of `ExecSubqueryExpression`

### What changes were proposed in this pull request?
This PR is to fix a potential bug in AQE where an `ExecSubqueryExpression` could be mistakenly replaced with another `ExecSubqueryExpression` with the same `ListQuery` but a different `child` expression.
This is because a ListQuery's id can only identify the ListQuery itself, not the parent expression `InSubquery`, but right now the `subqueryMap` in `InsertAdaptiveSparkPlan` uses the `ListQuery`'s id as key and the corresponding `InSubqueryExec` for the `ListQuery`'s parent expression as value. So the fix uses the corresponding `SubqueryExec` for the `ListQuery` itself as the map's value.

### Why are the changes needed?
This logical bug could potentially cause a wrong query plan, which could throw an exception related to unresolved columns.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Passed existing UTs.

Closes #27446 from maryannxue/spark-30717.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6097b343baa8e4a8bc7159dc3d394f13b3c9959b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/InsertAdaptiveSparkPlan.scala    | 30 ++++++-------------
 .../adaptive/PlanAdaptiveSubqueries.scala     | 23 +++++++++-----
 2 files changed, 25 insertions(+), 28 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
index 04696209ce10e..9252827856af4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
@@ -20,11 +20,10 @@ package org.apache.spark.sql.execution.adaptive
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, DynamicPruningSubquery, ListQuery, Literal, SubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.{DynamicPruningSubquery, ListQuery, SubqueryExpression}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical.UnspecifiedDistribution
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.command.ExecutedCommandExec
 import org.apache.spark.sql.execution.exchange.Exchange
@@ -102,36 +101,25 @@ case class InsertAdaptiveSparkPlan(
    * For each sub-query, generate the adaptive execution plan for each sub-query by applying this
    * rule, or reuse the execution plan from another sub-query of the same semantics if possible.
    */
-  private def buildSubqueryMap(plan: SparkPlan): mutable.HashMap[Long, ExecSubqueryExpression] = {
-    val subqueryMap = mutable.HashMap.empty[Long, ExecSubqueryExpression]
+  private def buildSubqueryMap(plan: SparkPlan): Map[Long, SubqueryExec] = {
+    val subqueryMap = mutable.HashMap.empty[Long, SubqueryExec]
     plan.foreach(_.expressions.foreach(_.foreach {
       case expressions.ScalarSubquery(p, _, exprId)
           if !subqueryMap.contains(exprId.id) =>
         val executedPlan = compileSubquery(p)
         verifyAdaptivePlan(executedPlan, p)
-        val scalarSubquery = execution.ScalarSubquery(
-          SubqueryExec(s"subquery${exprId.id}", executedPlan), exprId)
-        subqueryMap.put(exprId.id, scalarSubquery)
-      case expressions.InSubquery(values, ListQuery(query, _, exprId, _))
+        val subquery = SubqueryExec(s"subquery${exprId.id}", executedPlan)
+        subqueryMap.put(exprId.id, subquery)
+      case expressions.InSubquery(_, ListQuery(query, _, exprId, _))
           if !subqueryMap.contains(exprId.id) =>
         val executedPlan = compileSubquery(query)
         verifyAdaptivePlan(executedPlan, query)
-        val expr = if (values.length == 1) {
-          values.head
-        } else {
-          CreateNamedStruct(
-            values.zipWithIndex.flatMap { case (v, index) =>
-              Seq(Literal(s"col_$index"), v)
-            }
-          )
-        }
-        val inSubquery = InSubqueryExec(expr,
-          SubqueryExec(s"subquery#${exprId.id}", executedPlan), exprId)
-        subqueryMap.put(exprId.id, inSubquery)
+        val subquery = SubqueryExec(s"subquery#${exprId.id}", executedPlan)
+        subqueryMap.put(exprId.id, subquery)
       case _ =>
     }))
 
-    subqueryMap
+    subqueryMap.toMap
   }
 
   def compileSubquery(plan: LogicalPlan): SparkPlan = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
index 91d4359224a6a..f845b6b16ee3a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/PlanAdaptiveSubqueries.scala
@@ -18,19 +18,28 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.catalyst.expressions.ListQuery
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, ListQuery, Literal}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{ExecSubqueryExpression, SparkPlan}
+import org.apache.spark.sql.execution
+import org.apache.spark.sql.execution.{InSubqueryExec, SparkPlan, SubqueryExec}
 
-case class PlanAdaptiveSubqueries(
-    subqueryMap: scala.collection.Map[Long, ExecSubqueryExpression]) extends Rule[SparkPlan] {
+case class PlanAdaptiveSubqueries(subqueryMap: Map[Long, SubqueryExec]) extends Rule[SparkPlan] {
 
   def apply(plan: SparkPlan): SparkPlan = {
     plan.transformAllExpressions {
       case expressions.ScalarSubquery(_, _, exprId) =>
-        subqueryMap(exprId.id)
-      case expressions.InSubquery(_, ListQuery(_, _, exprId, _)) =>
-        subqueryMap(exprId.id)
+        execution.ScalarSubquery(subqueryMap(exprId.id), exprId)
+      case expressions.InSubquery(values, ListQuery(_, _, exprId, _)) =>
+        val expr = if (values.length == 1) {
+          values.head
+        } else {
+          CreateNamedStruct(
+            values.zipWithIndex.flatMap { case (v, index) =>
+              Seq(Literal(s"col_$index"), v)
+            }
+          )
+        }
+        InSubqueryExec(expr, subqueryMap(exprId.id), exprId)
     }
   }
 }

From c8ffaa9c2d86b1b8e807ad6f0570efcd0d7cd3d9 Mon Sep 17 00:00:00 2001
From: zero323 <mszymkiewicz@gmail.com>
Date: Mon, 3 Feb 2020 11:09:25 +0900
Subject: [PATCH 0007/1280] [MINOR][SPARKR][DOCS] Remove duplicate @name tags
 from read.df and read.stream

### What changes were proposed in this pull request?

Remove duplicate `name` tags from `read.df` and `read.stream`.

### Why are the changes needed?

These tags are already present in

https://github.com/apache/spark/blob/1adf3520e3c753e6df8dccb752e8239de682a09a/R/pkg/R/SQLContext.R#L546

and

https://github.com/apache/spark/blob/1adf3520e3c753e6df8dccb752e8239de682a09a/R/pkg/R/SQLContext.R#L678

for `read.df` and `read.stream` respectively.

As only one `name` tag per block is allowed, this causes build warnings with recent `roxygen2` versions:

```
Warning: [/path/to/spark/R/pkg/R/SQLContext.R:559] name May only use one name per block
Warning: [/path/to/spark/R/pkg/R/SQLContext.R:690] name May only use one name per block
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #27437 from zero323/roxygen-warnings-names.

Authored-by: zero323 <mszymkiewicz@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/SQLContext.R | 2 --
 1 file changed, 2 deletions(-)

diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R
index f48a334ed6766..c6842912706af 100644
--- a/R/pkg/R/SQLContext.R
+++ b/R/pkg/R/SQLContext.R
@@ -556,7 +556,6 @@ tableToDF <- function(tableName) {
 #' stringSchema <- "name STRING, info MAP<STRING, DOUBLE>"
 #' df4 <- read.df(mapTypeJsonPath, "json", stringSchema, multiLine = TRUE)
 #' }
-#' @name read.df
 #' @note read.df since 1.4.0
 read.df <- function(path = NULL, source = NULL, schema = NULL, na.strings = "NA", ...) {
   if (!is.null(path) && !is.character(path)) {
@@ -687,7 +686,6 @@ read.jdbc <- function(url, tableName,
 #' stringSchema <- "name STRING, info MAP<STRING, DOUBLE>"
 #' df1 <- read.stream("json", path = jsonDir, schema = stringSchema, maxFilesPerTrigger = 1)
 #' }
-#' @name read.stream
 #' @note read.stream since 2.2.0
 #' @note experimental
 read.stream <- function(source = NULL, schema = NULL, ...) {

From 5fdea014873535457d6865594eb7ea2cc38fdc25 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Tue, 4 Feb 2020 16:33:34 +0900
Subject: [PATCH 0008/1280] [SPARK-26618][SQL][FOLLOWUP] Describe the behavior
 change of typed `TIMESTAMP`/`DATE` literals

### What changes were proposed in this pull request?
In the PR, I propose to update the SQL migration guide, and clarify behavior change of typed `TIMESTAMP` and `DATE` literals for input strings without time zone information - local timestamp and date strings.

### Why are the changes needed?
To inform users that the typed literals may change their behavior in Spark 3.0 because of different sources of the default time zone - JVM system time zone in Spark 2.4 and earlier, and `spark.sql.session.timeZone` in Spark 3.0.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
N/A

Closes #27435 from MaxGekk/timestamp-lit-migration-guide.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0202b675afca65c6615a06805a4d4d12f3f97bdb)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index e4d2358b5de09..a5ef1c2b1d045 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -77,7 +77,7 @@ license: |
 
     - Formatting of `TIMESTAMP` and `DATE` literals.
 
-    - Creating of typed `TIMESTAMP` and `DATE` literals from strings. Since Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. In Spark version 2.4 and earlier, the `java.sql.Timestamp.valueOf()` and `java.sql.Date.valueOf()` functions are used for the same purpose.
+    - Creating of typed `TIMESTAMP` and `DATE` literals from strings. Since Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. When the input string does not contain information about time zone, the time zone from the SQL config `spark.sql.session.timeZone` is used in that case. In Spark version 2.4 and earlier, the conversion is based on JVM system time zone. The different sources of the default time zone may change the behavior of typed `TIMESTAMP` and `DATE` literals.
 
   - In Spark version 2.4 and earlier, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. Since Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
 

From 0d1984269fc8f81de384529acca2b3f8584d2f1a Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Tue, 4 Feb 2020 21:17:05 +0800
Subject: [PATCH 0009/1280] [SPARK-30725][SQL] Make legacy SQL configs as
 internal configs

### What changes were proposed in this pull request?
All legacy SQL configs are marked as internal configs. In particular, the following configs are updated as internals:
- spark.sql.legacy.sizeOfNull
- spark.sql.legacy.replaceDatabricksSparkAvro.enabled
- spark.sql.legacy.typeCoercion.datetimeToString.enabled
- spark.sql.legacy.looseUpcast
- spark.sql.legacy.arrayExistsFollowsThreeValuedLogic

### Why are the changes needed?
In general case, users shouldn't change legacy configs, so, they can be marked as internals.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Should be tested by jenkins build and run tests.

Closes #27448 from MaxGekk/legacy-internal-sql-conf.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f2dd082544aeba5978d0c140d0194eedb969d132)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/internal/SQLConf.scala    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 3ad3416256c7d..b94ddbdc0fc9a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1916,6 +1916,7 @@ object SQLConf {
     .createWithDefault(Deflater.DEFAULT_COMPRESSION)
 
   val LEGACY_SIZE_OF_NULL = buildConf("spark.sql.legacy.sizeOfNull")
+    .internal()
     .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " +
       "The size function returns null for null input if the flag is disabled.")
     .booleanConf
@@ -1923,6 +1924,7 @@ object SQLConf {
 
   val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED =
     buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled")
+      .internal()
       .doc("If it is set to true, the data source provider com.databricks.spark.avro is mapped " +
         "to the built-in but external Avro data source module for backward compatibility.")
       .booleanConf
@@ -2048,10 +2050,11 @@ object SQLConf {
 
   val LEGACY_CAST_DATETIME_TO_STRING =
     buildConf("spark.sql.legacy.typeCoercion.datetimeToString.enabled")
+      .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
         "with String")
-    .booleanConf
-    .createWithDefault(false)
+      .booleanConf
+      .createWithDefault(false)
 
   val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog")
     .doc("Name of the default catalog. This will be the current catalog if users have not " +
@@ -2071,6 +2074,7 @@ object SQLConf {
       .createOptional
 
   val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.looseUpcast")
+    .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
     .booleanConf
     .createWithDefault(false)
@@ -2083,6 +2087,7 @@ object SQLConf {
 
   val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
     buildConf("spark.sql.legacy.arrayExistsFollowsThreeValuedLogic")
+      .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
       .booleanConf
       .createWithDefault(true)

From d3de7568f32e298442f07b0a28b2c906de72c797 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <liangchi@uber.com>
Date: Tue, 4 Feb 2020 17:22:23 -0800
Subject: [PATCH 0010/1280] [SPARK-25040][SQL][FOLLOWUP] Add legacy config for
 allowing empty strings for certain types in json parser

### What changes were proposed in this pull request?

This is a follow-up for #22787. In #22787 we disallowed empty strings for json parser except for string and binary types. This follow-up adds a legacy config for restoring previous behavior of allowing empty string.

### Why are the changes needed?

Adding a legacy config to make migration easy for Spark users.

### Does this PR introduce any user-facing change?

Yes. If set this legacy config to true, the users can restore previous behavior prior to Spark 3.0.0.

### How was this patch tested?

Unit test.

Closes #27456 from viirya/SPARK-25040-followup.

Lead-authored-by: Liang-Chi Hsieh <liangchi@uber.com>
Co-authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 7631275f974d2ecf68cd8394ed683e30be320e56)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-migration-guide.md                   |  2 +-
 .../sql/catalyst/json/JacksonParser.scala     | 14 ++++-
 .../apache/spark/sql/internal/SQLConf.scala   |  8 +++
 .../datasources/json/JsonSuite.scala          | 61 ++++++++++++++-----
 4 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index a5ef1c2b1d045..0c47370283736 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -37,7 +37,7 @@ license: |
 
   - Since Spark 3.0, the Dataset and DataFrame API `unionAll` is not deprecated any more. It is an alias for `union`.
 
-  - In Spark version 2.4 and earlier, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType` and `DoubleType`, it fails on empty strings and throws exceptions. Since Spark 3.0, we disallow empty strings and will throw exceptions for data types except for `StringType` and `BinaryType`.
+  - In Spark version 2.4 and earlier, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType`, `DoubleType`, `DateType` and `TimestampType`, it fails on empty strings and throws exceptions. Since Spark 3.0, we disallow empty strings and will throw exceptions for data types except for `StringType` and `BinaryType`. The previous behaviour of allowing empty string can be restored by setting `spark.sql.legacy.json.allowEmptyString.enabled` to `true`.
 
   - Since Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index c44025ca8bcfd..76efa574a99ff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -30,6 +30,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 import org.apache.spark.util.Utils
@@ -307,6 +308,8 @@ class JacksonParser(
     }
   }
 
+  private val allowEmptyString = SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_EMPTY_STRING_IN_JSON)
+
   /**
    * This function throws an exception for failed conversion. For empty string on data types
    * except for string and binary types, this also throws an exception.
@@ -315,7 +318,16 @@ class JacksonParser(
       parser: JsonParser,
       dataType: DataType): PartialFunction[JsonToken, R] = {
 
-    // SPARK-25040: Disallow empty strings for data types except for string and binary types.
+    // SPARK-25040: Disallows empty strings for data types except for string and binary types.
+    // But treats empty strings as null for certain types if the legacy config is enabled.
+    case VALUE_STRING if parser.getTextLength < 1 && allowEmptyString =>
+      dataType match {
+        case FloatType | DoubleType | TimestampType | DateType =>
+          throw new RuntimeException(
+            s"Failed to parse an empty string for data type ${dataType.catalogString}")
+        case _ => null
+      }
+
     case VALUE_STRING if parser.getTextLength < 1 =>
       throw new RuntimeException(
         s"Failed to parse an empty string for data type ${dataType.catalogString}")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b94ddbdc0fc9a..5ce5692123805 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1989,6 +1989,14 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val LEGACY_ALLOW_EMPTY_STRING_IN_JSON =
+    buildConf("spark.sql.legacy.json.allowEmptyString.enabled")
+      .internal()
+      .doc("When set to true, the parser of JSON data source treats empty strings as null for " +
+      "some data types such as `IntegerType`.")
+      .booleanConf
+      .createWithDefault(false)
+
   val TRUNCATE_TABLE_IGNORE_PERMISSION_ACL =
     buildConf("spark.sql.truncateTable.ignorePermissionAcl.enabled")
       .internal()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index d0e2e001034fb..b20da2266b0f3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2436,23 +2436,24 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     }
   }
 
-  test("SPARK-25040: empty strings should be disallowed") {
-    def failedOnEmptyString(dataType: DataType): Unit = {
-       val df = spark.read.schema(s"a ${dataType.catalogString}")
-        .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
-      val errMessage = intercept[SparkException] {
-        df.collect()
-      }.getMessage
-      assert(errMessage.contains(
-        s"Failed to parse an empty string for data type ${dataType.catalogString}"))
-    }
 
-    def emptyString(dataType: DataType, expected: Any): Unit = {
-      val df = spark.read.schema(s"a ${dataType.catalogString}")
-        .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
-      checkAnswer(df, Row(expected) :: Nil)
-    }
+  private def failedOnEmptyString(dataType: DataType): Unit = {
+    val df = spark.read.schema(s"a ${dataType.catalogString}")
+      .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
+    val errMessage = intercept[SparkException] {
+      df.collect()
+    }.getMessage
+    assert(errMessage.contains(
+      s"Failed to parse an empty string for data type ${dataType.catalogString}"))
+  }
 
+  private def emptyString(dataType: DataType, expected: Any): Unit = {
+    val df = spark.read.schema(s"a ${dataType.catalogString}")
+      .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
+    checkAnswer(df, Row(expected) :: Nil)
+  }
+
+  test("SPARK-25040: empty strings should be disallowed") {
     failedOnEmptyString(BooleanType)
     failedOnEmptyString(ByteType)
     failedOnEmptyString(ShortType)
@@ -2471,6 +2472,36 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     emptyString(BinaryType, "".getBytes(StandardCharsets.UTF_8))
   }
 
+  test("SPARK-25040: allowing empty strings when legacy config is enabled") {
+    def emptyStringAsNull(dataType: DataType): Unit = {
+      val df = spark.read.schema(s"a ${dataType.catalogString}")
+        .option("mode", "FAILFAST").json(Seq("""{"a":""}""").toDS)
+      checkAnswer(df, Row(null) :: Nil)
+    }
+
+    // Legacy mode prior to Spark 3.0.0
+    withSQLConf(SQLConf.LEGACY_ALLOW_EMPTY_STRING_IN_JSON.key -> "true") {
+      emptyStringAsNull(BooleanType)
+      emptyStringAsNull(ByteType)
+      emptyStringAsNull(ShortType)
+      emptyStringAsNull(IntegerType)
+      emptyStringAsNull(LongType)
+
+      failedOnEmptyString(FloatType)
+      failedOnEmptyString(DoubleType)
+      failedOnEmptyString(TimestampType)
+      failedOnEmptyString(DateType)
+
+      emptyStringAsNull(DecimalType.SYSTEM_DEFAULT)
+      emptyStringAsNull(ArrayType(IntegerType))
+      emptyStringAsNull(MapType(StringType, IntegerType, true))
+      emptyStringAsNull(StructType(StructField("f1", IntegerType, true) :: Nil))
+
+      emptyString(StringType, "")
+      emptyString(BinaryType, "".getBytes(StandardCharsets.UTF_8))
+    }
+  }
+
   test("return partial result for bad records") {
     val schema = "a double, b array<int>, c string, _corrupt_record string"
     val badRecords = Seq(

From d1991c8c1ee7948ab672d702aa89f4ce99d2f485 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Tue, 4 Feb 2020 17:26:46 -0800
Subject: [PATCH 0011/1280] Revert "[SPARK-28310][SQL] Support
 (FIRST_VALUE|LAST_VALUE)(expr[ (IGNORE|RESPECT) NULLS]?) syntax"

### What changes were proposed in this pull request?

This reverts commit b89c3de1a439ed7302dd8f44c49b89bb7da2eebe.

### Why are the changes needed?

`FIRST_VALUE` is used only for window expression. Please see the discussion on https://github.com/apache/spark/pull/25082 .

### Does this PR introduce any user-facing change?

Yes.

### How was this patch tested?

Pass the Jenkins.

Closes #27458 from dongjoon-hyun/SPARK-28310.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 898716980dce44a4cc09411e72d64c848698cad5)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-keywords.md                                  |  3 ---
 .../org/apache/spark/sql/catalyst/parser/SqlBase.g4   | 11 ++---------
 .../sql/catalyst/parser/ExpressionParserSuite.scala   |  9 ---------
 .../catalyst/parser/TableIdentifierParserSuite.scala  |  5 -----
 4 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/docs/sql-keywords.md b/docs/sql-keywords.md
index b18855366bb2b..9e4a3c54100c6 100644
--- a/docs/sql-keywords.md
+++ b/docs/sql-keywords.md
@@ -119,7 +119,6 @@ Below is a list of all the keywords in Spark SQL.
   <tr><td>FILTER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>FILEFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>FIRST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FIRST_VALUE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>FOLLOWING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>FOR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>FOREIGN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
@@ -153,7 +152,6 @@ Below is a list of all the keywords in Spark SQL.
   <tr><td>JOIN</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
   <tr><td>KEYS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>LAST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LAST_VALUE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>LATERAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>LAZY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>LEADING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
@@ -221,7 +219,6 @@ Below is a list of all the keywords in Spark SQL.
   <tr><td>REPAIR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>REPLACE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>RESET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RESPECT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>RESTRICT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>REVOKE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>RIGHT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 6f2bb7a9a7536..08d5ff53bf2e2 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -771,8 +771,8 @@ primaryExpression
     | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END                  #simpleCase
     | CAST '(' expression AS dataType ')'                                                      #cast
     | STRUCT '(' (argument+=namedExpression (',' argument+=namedExpression)*)? ')'             #struct
-    | (FIRST | FIRST_VALUE) '(' expression ((IGNORE | RESPECT) NULLS)? ')'                     #first
-    | (LAST | LAST_VALUE) '(' expression ((IGNORE | RESPECT) NULLS)? ')'                       #last
+    | FIRST '(' expression (IGNORE NULLS)? ')'                                                 #first
+    | LAST '(' expression (IGNORE NULLS)? ')'                                                  #last
     | POSITION '(' substr=valueExpression IN str=valueExpression ')'                           #position
     | constant                                                                                 #constantDefault
     | ASTERISK                                                                                 #star
@@ -1120,7 +1120,6 @@ ansiNonReserved
     | REPAIR
     | REPLACE
     | RESET
-    | RESPECT
     | RESTRICT
     | REVOKE
     | RLIKE
@@ -1280,7 +1279,6 @@ nonReserved
     | FIELDS
     | FILEFORMAT
     | FIRST
-    | FIRST_VALUE
     | FOLLOWING
     | FOR
     | FOREIGN
@@ -1310,7 +1308,6 @@ nonReserved
     | ITEMS
     | KEYS
     | LAST
-    | LAST_VALUE
     | LATERAL
     | LAZY
     | LEADING
@@ -1374,7 +1371,6 @@ nonReserved
     | REPAIR
     | REPLACE
     | RESET
-    | RESPECT
     | RESTRICT
     | REVOKE
     | RLIKE
@@ -1531,7 +1527,6 @@ FIELDS: 'FIELDS';
 FILTER: 'FILTER';
 FILEFORMAT: 'FILEFORMAT';
 FIRST: 'FIRST';
-FIRST_VALUE: 'FIRST_VALUE';
 FOLLOWING: 'FOLLOWING';
 FOR: 'FOR';
 FOREIGN: 'FOREIGN';
@@ -1565,7 +1560,6 @@ ITEMS: 'ITEMS';
 JOIN: 'JOIN';
 KEYS: 'KEYS';
 LAST: 'LAST';
-LAST_VALUE: 'LAST_VALUE';
 LATERAL: 'LATERAL';
 LAZY: 'LAZY';
 LEADING: 'LEADING';
@@ -1632,7 +1626,6 @@ RENAME: 'RENAME';
 REPAIR: 'REPAIR';
 REPLACE: 'REPLACE';
 RESET: 'RESET';
-RESPECT: 'RESPECT';
 RESTRICT: 'RESTRICT';
 REVOKE: 'REVOKE';
 RIGHT: 'RIGHT';
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 85efc6accf01f..df012ccf09620 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -771,15 +771,6 @@ class ExpressionParserSuite extends AnalysisTest {
     assertEqual("last(a)", Last('a, Literal(false)).toAggregateExpression())
   }
 
-  test("Support respect nulls keywords for first_value and last_value") {
-    assertEqual("first_value(a ignore nulls)", First('a, Literal(true)).toAggregateExpression())
-    assertEqual("first_value(a respect nulls)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("first_value(a)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("last_value(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression())
-    assertEqual("last_value(a respect nulls)", Last('a, Literal(false)).toAggregateExpression())
-    assertEqual("last_value(a)", Last('a, Literal(false)).toAggregateExpression())
-  }
-
   test("timestamp literals") {
     DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone.getID) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 43244b3c0a57d..053d57846ce8d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -369,7 +369,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "fields",
     "fileformat",
     "first",
-    "first_value",
     "following",
     "for",
     "foreign",
@@ -403,7 +402,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "join",
     "keys",
     "last",
-    "last_value",
     "lateral",
     "lazy",
     "leading",
@@ -467,7 +465,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "repair",
     "replace",
     "reset",
-    "respect",
     "restrict",
     "revoke",
     "right",
@@ -562,7 +559,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "except",
     "false",
     "fetch",
-    "first_value",
     "for",
     "foreign",
     "from",
@@ -577,7 +573,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "into",
     "join",
     "is",
-    "last_value",
     "leading",
     "left",
     "minute",

From d8bc72e89e8221cc7ec17470ae991878d9daecd7 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 5 Feb 2020 16:45:54 +0900
Subject: [PATCH 0012/1280] [SPARK-30733][R][HOTFIX] Fix SparkR tests per
 testthat and R version upgrade, and disable CRAN
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

There are currently the R test failures after upgrading `testthat` to 2.0.0, and R version 3.5.2 as of SPARK-23435. This PR targets to fix the tests and make the tests pass. See the explanations and causes below:

```
test_context.R:49: failure: Check masked functions
length(maskedCompletely) not equal to length(namesOfMaskedCompletely).
1/1 mismatches
[1] 6 - 4 == 2

test_context.R:53: failure: Check masked functions
sort(maskedCompletely, na.last = TRUE) not equal to sort(namesOfMaskedCompletely, na.last = TRUE).
5/6 mismatches
x[2]: "endsWith"
y[2]: "filter"

x[3]: "filter"
y[3]: "not"

x[4]: "not"
y[4]: "sample"

x[5]: "sample"
y[5]: NA

x[6]: "startsWith"
y[6]: NA
```

From my cursory look, R base and R's version are mismatched. I fixed accordingly and Jenkins will test it out.

```
test_includePackage.R:31: error: include inside function
package or namespace load failed for ���plyr���:
 package ���plyr��� was installed by an R version with different internals; it needs to be reinstalled for use with this R version
Seems it's a package installation issue. Looks like plyr has to be re-installed.
```

From my cursory look, previously installed `plyr` remains and it's not compatible with the new R version. I fixed accordingly and Jenkins will test it out.

```
test_sparkSQL.R:499: warning: SPARK-17811: can create DataFrame containing NA as date and time
Your system is mis-configured: ���/etc/localtime��� is not a symlink
```

Seems a env problem. I suppressed the warnings for now.

```
test_sparkSQL.R:499: warning: SPARK-17811: can create DataFrame containing NA as date and time
It is strongly recommended to set envionment variable TZ to ���America/Los_Angeles��� (or equivalent)
```

Seems a env problem. I suppressed the warnings for now.

```
test_sparkSQL.R:1814: error: string operators
unable to find an inherited method for function ���startsWith��� for signature ���"character"���
1: expect_true(startsWith("Hello World", "Hello")) at /home/jenkins/workspace/SparkPullRequestBuilder2/R/pkg/tests/fulltests/test_sparkSQL.R:1814
2: quasi_label(enquo(object), label)
3: eval_bare(get_expr(quo), get_env(quo))
4: startsWith("Hello World", "Hello")
5: (function (classes, fdef, mtable)
   {
       methods <- .findInheritedMethods(classes, fdef, mtable)
       if (length(methods) == 1L)
           return(methods[[1L]])
       else if (length(methods) == 0L) {
           cnames <- paste0("\"", vapply(classes, as.character, ""), "\"", collapse = ", ")
           stop(gettextf("unable to find an inherited method for function %s for signature %s",
               sQuote(fdefgeneric), sQuote(cnames)), domain = NA)
       }
       else stop("Internal error in finding inherited methods; didn't return a unique method",
           domain = NA)
   })(list("character"), new("nonstandardGenericFunction", .Data = function (x, prefix)
   {
       standardGeneric("startsWith")
   }, generic = structure("startsWith", package = "SparkR"), package = "SparkR", group = list(),
       valueClass = character(0), signature = c("x", "prefix"), default = NULL, skeleton = (function (x,
           prefix)
       stop("invalid call in method dispatch to 'startsWith' (no default method)", domain = NA))(x,
           prefix)), <environment>)
6: stop(gettextf("unable to find an inherited method for function %s for signature %s",
       sQuote(fdefgeneric), sQuote(cnames)), domain = NA)
```

From my cursory look, R base and R's version are mismatched. I fixed accordingly and Jenkins will test it out.

Also, this PR causes a CRAN check failure as below:

```
* creating vignettes ... ERROR
Error: processing vignette 'sparkr-vignettes.Rmd' failed with diagnostics:
package ���htmltools��� was installed by an R version with different internals; it needs to be reinstalled for use with this R version
```

This PR disables it for now.

### Why are the changes needed?

To unblock other PRs.

### Does this PR introduce any user-facing change?

No. Test only and dev only.

### How was this patch tested?

No. I am going to use Jenkins to test.

Closes #27460 from HyukjinKwon/r-test-failure.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e2d984aa1c79eb389cc8d333f656196b17af1c32)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/tests/fulltests/test_context.R        |  3 ++-
 R/pkg/tests/fulltests/test_includePackage.R |  8 ++++----
 R/pkg/tests/fulltests/test_sparkSQL.R       | 10 ++++++++--
 R/run-tests.sh                              |  7 ++++---
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/R/pkg/tests/fulltests/test_context.R b/R/pkg/tests/fulltests/test_context.R
index b9139154bc165..6be04b321e985 100644
--- a/R/pkg/tests/fulltests/test_context.R
+++ b/R/pkg/tests/fulltests/test_context.R
@@ -25,7 +25,8 @@ test_that("Check masked functions", {
   namesOfMasked <- c("describe", "cov", "filter", "lag", "na.omit", "predict", "sd", "var",
                      "colnames", "colnames<-", "intersect", "rank", "rbind", "sample", "subset",
                      "summary", "transform", "drop", "window", "as.data.frame", "union", "not")
-  if (as.numeric(R.version$major) >= 3 && as.numeric(R.version$minor) >= 3) {
+  version <- packageVersion("base")
+  if (as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3) {
     namesOfMasked <- c("endsWith", "startsWith", namesOfMasked)
   }
   masked <- conflicts(detail = TRUE)$`package:SparkR`
diff --git a/R/pkg/tests/fulltests/test_includePackage.R b/R/pkg/tests/fulltests/test_includePackage.R
index 916361ff4c797..1d16b260c4c52 100644
--- a/R/pkg/tests/fulltests/test_includePackage.R
+++ b/R/pkg/tests/fulltests/test_includePackage.R
@@ -27,8 +27,8 @@ rdd <- parallelize(sc, nums, 2L)
 
 test_that("include inside function", {
   # Only run the test if plyr is installed.
-  if ("plyr" %in% rownames(installed.packages())) {
-    suppressPackageStartupMessages(library(plyr))
+  if ("plyr" %in% rownames(installed.packages()) &&
+      suppressPackageStartupMessages(suppressWarnings(library(plyr, logical.return = TRUE)))) {
     generateData <- function(x) {
       suppressPackageStartupMessages(library(plyr))
       attach(airquality)
@@ -44,8 +44,8 @@ test_that("include inside function", {
 
 test_that("use include package", {
   # Only run the test if plyr is installed.
-  if ("plyr" %in% rownames(installed.packages())) {
-    suppressPackageStartupMessages(library(plyr))
+  if ("plyr" %in% rownames(installed.packages()) &&
+      suppressPackageStartupMessages(suppressWarnings(library(plyr, logical.return = TRUE)))) {
     generateData <- function(x) {
       attach(airquality)
       result <- transform(Ozone, logOzone = log(Ozone))
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 3b3768f7e2715..23fadc4373c3f 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -496,7 +496,12 @@ test_that("SPARK-17811: can create DataFrame containing NA as date and time", {
   expect_true(is.na(DF$date[2]))
   expect_equal(DF$date[1], as.Date("2016-10-01"))
   expect_true(is.na(DF$time[2]))
-  expect_equal(DF$time[1], as.POSIXlt("2016-01-10"))
+  # Warnings were suppressed due to Jenkins environment issues.
+  # It shows both warnings as below in Jenkins:
+  # - Your system is mis-configured: /etc/localtime is not a symlink
+  # - It is strongly recommended to set environment variable TZ to
+  #     America/Los_Angeles (or equivalent)
+  suppressWarnings(expect_equal(DF$time[1], as.POSIXlt("2016-01-10")))
 })
 
 test_that("create DataFrame with complex types", {
@@ -1810,7 +1815,8 @@ test_that("string operators", {
   expect_true(first(select(df, endsWith(df$name, "el")))[[1]])
   expect_equal(first(select(df, substr(df$name, 1, 2)))[[1]], "Mi")
   expect_equal(first(select(df, substr(df$name, 4, 6)))[[1]], "hae")
-  if (as.numeric(R.version$major) >= 3 && as.numeric(R.version$minor) >= 3) {
+  version <- packageVersion("base")
+  if (as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3) {
     expect_true(startsWith("Hello World", "Hello"))
     expect_false(endsWith("Hello World", "a"))
   }
diff --git a/R/run-tests.sh b/R/run-tests.sh
index 51ca7d600caf0..782b5f5baebaf 100755
--- a/R/run-tests.sh
+++ b/R/run-tests.sh
@@ -31,9 +31,10 @@ NUM_TEST_WARNING="$(grep -c -e 'Warnings ----------------' $LOGFILE)"
 # Also run the documentation tests for CRAN
 CRAN_CHECK_LOG_FILE=$FWDIR/cran-check.out
 rm -f $CRAN_CHECK_LOG_FILE
-
-NO_TESTS=1 NO_MANUAL=1 $FWDIR/check-cran.sh 2>&1 | tee -a $CRAN_CHECK_LOG_FILE
-FAILED=$((PIPESTATUS[0]||$FAILED))
+# TODO(SPARK-30737) reenable this once packages are correctly installed in Jenkins
+# NO_TESTS=1 NO_MANUAL=1 $FWDIR/check-cran.sh 2>&1 | tee -a $CRAN_CHECK_LOG_FILE
+# FAILED=$((PIPESTATUS[0]||$FAILED))
+touch $CRAN_CHECK_LOG_FILE
 
 NUM_CRAN_WARNING="$(grep -c WARNING$ $CRAN_CHECK_LOG_FILE)"
 NUM_CRAN_ERROR="$(grep -c ERROR$ $CRAN_CHECK_LOG_FILE)"

From 3854ad87c78f2a331f9c9c1a34f9ec281900f8fe Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 5 Feb 2020 16:15:44 +0800
Subject: [PATCH 0013/1280] [SPARK-30594][CORE] Do not post
 SparkListenerBlockUpdated when updateBlockInfo returns false

### What changes were proposed in this pull request?

If `updateBlockInfo` returns false, which means the `BlockManager` will re-register and report all blocks later. So, we may report two times for the same block, which causes `AppStatusListener` to count used memory for two times, too. As a result, the used memory can exceed the total memory.
So, this PR changes it to not post `SparkListenerBlockUpdated` when `updateBlockInfo` returns false. And, always clean up used memory whenever `AppStatusListener` receives `SparkListenerBlockManagerAdded`.

### Why are the changes needed?

This PR tries to fix negative memory usage in UI (https://user-images.githubusercontent.com/3488126/72131225-95e37e00-33b6-11ea-8708-6e5ed328d1ca.png, see #27144 ). Though, I'm not very sure this is the root cause for #27144 since known information is limited here.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added new tests by xuanyuanking

Closes #27306 from Ngone51/fix-possible-negative-memory.

Lead-authored-by: yi.wu <yi.wu@databricks.com>
Co-authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Co-authored-by: wuyi <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 30e418a6fe971b4a84c37ca0ae20f1a664b117d3)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/status/AppStatusListener.scala      |  9 +++++--
 .../org/apache/spark/status/LiveEntity.scala  |  2 +-
 .../storage/BlockManagerMasterEndpoint.scala  |  9 +++++--
 .../spark/status/AppStatusListenerSuite.scala | 24 +++++++++++++++++++
 .../spark/storage/BlockManagerSuite.scala     | 17 +++++++++++--
 5 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index a5850fc2ac4b9..c3f22f32993a8 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -71,7 +71,7 @@ private[spark] class AppStatusListener(
   // causing too many writes to the underlying store, and other expensive operations).
   private val liveStages = new ConcurrentHashMap[(Int, Int), LiveStage]()
   private val liveJobs = new HashMap[Int, LiveJob]()
-  private val liveExecutors = new HashMap[String, LiveExecutor]()
+  private[spark] val liveExecutors = new HashMap[String, LiveExecutor]()
   private val deadExecutors = new HashMap[String, LiveExecutor]()
   private val liveTasks = new HashMap[Long, LiveTask]()
   private val liveRDDs = new HashMap[Int, LiveRDD]()
@@ -772,6 +772,11 @@ private[spark] class AppStatusListener(
     event.maxOnHeapMem.foreach { _ =>
       exec.totalOnHeap = event.maxOnHeapMem.get
       exec.totalOffHeap = event.maxOffHeapMem.get
+      // SPARK-30594: whenever(first time or re-register) a BlockManager added, all blocks
+      // from this BlockManager will be reported to driver later. So, we should clean up
+      // used memory to avoid overlapped count.
+      exec.usedOnHeap = 0
+      exec.usedOffHeap = 0
     }
     exec.isActive = true
     exec.maxMemory = event.maxMem
@@ -1042,7 +1047,7 @@ private[spark] class AppStatusListener(
     }
   }
 
-  private def updateExecutorMemoryDiskInfo(
+  private[spark] def updateExecutorMemoryDiskInfo(
       exec: LiveExecutor,
       storageLevel: StorageLevel,
       memoryDelta: Long,
diff --git a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
index e3046dce34e67..2714f30de14f0 100644
--- a/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
+++ b/core/src/main/scala/org/apache/spark/status/LiveEntity.scala
@@ -245,7 +245,7 @@ private class LiveTask(
 
 }
 
-private class LiveExecutor(val executorId: String, _addTime: Long) extends LiveEntity {
+private[spark] class LiveExecutor(val executorId: String, _addTime: Long) extends LiveEntity {
 
   var hostPort: String = null
   var host: String = null
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index 41ef1909cd4c2..d7f7eedc7f33b 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -98,8 +98,13 @@ class BlockManagerMasterEndpoint(
 
     case _updateBlockInfo @
         UpdateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size) =>
-      context.reply(updateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size))
-      listenerBus.post(SparkListenerBlockUpdated(BlockUpdatedInfo(_updateBlockInfo)))
+      val isSuccess = updateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size)
+      context.reply(isSuccess)
+      // SPARK-30594: we should not post `SparkListenerBlockUpdated` when updateBlockInfo
+      // returns false since the block info would be updated again later.
+      if (isSuccess) {
+        listenerBus.post(SparkListenerBlockUpdated(BlockUpdatedInfo(_updateBlockInfo)))
+      }
 
     case GetLocations(blockId) =>
       context.reply(getLocations(blockId))
diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index e7eed7bf4c879..255f91866ef58 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -1657,6 +1657,30 @@ class AppStatusListenerSuite extends SparkFunSuite with BeforeAndAfter {
     }
   }
 
+  test("clean up used memory when BlockManager added") {
+    val listener = new AppStatusListener(store, conf, true)
+    // Add block manager at the first time
+    val driver = BlockManagerId(SparkContext.DRIVER_IDENTIFIER, "localhost", 42)
+    listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(
+      time, driver, 42L, Some(43L), Some(44L)))
+    // Update the memory metrics
+    listener.updateExecutorMemoryDiskInfo(
+      listener.liveExecutors(SparkContext.DRIVER_IDENTIFIER),
+      StorageLevel.MEMORY_AND_DISK,
+      10L,
+      10L
+    )
+    // Re-add the same block manager again
+    listener.onBlockManagerAdded(SparkListenerBlockManagerAdded(
+      time, driver, 42L, Some(43L), Some(44L)))
+
+    check[ExecutorSummaryWrapper](SparkContext.DRIVER_IDENTIFIER) { d =>
+      val memoryMetrics = d.info.memoryMetrics.get
+      assert(memoryMetrics.usedOffHeapStorageMemory == 0)
+      assert(memoryMetrics.usedOnHeapStorageMemory == 0)
+    }
+  }
+
 
   private def key(stage: StageInfo): Array[Int] = Array(stage.stageId, stage.attemptNumber)
 
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 89f00b5a9d902..8d06768a2b284 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -50,7 +50,7 @@ import org.apache.spark.network.server.{NoOpRpcHandler, TransportServer, Transpo
 import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, ExecutorDiskUtils, ExternalBlockStoreClient}
 import org.apache.spark.network.shuffle.protocol.{BlockTransferMessage, RegisterExecutor}
 import org.apache.spark.rpc.RpcEnv
-import org.apache.spark.scheduler.LiveListenerBus
+import org.apache.spark.scheduler.{LiveListenerBus, SparkListenerBlockUpdated}
 import org.apache.spark.security.{CryptoStreamUtils, EncryptionFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer, SerializerManager}
 import org.apache.spark.shuffle.sort.SortShuffleManager
@@ -71,6 +71,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
   val allStores = ArrayBuffer[BlockManager]()
   var rpcEnv: RpcEnv = null
   var master: BlockManagerMaster = null
+  var liveListenerBus: LiveListenerBus = null
   val securityMgr = new SecurityManager(new SparkConf(false))
   val bcastManager = new BroadcastManager(true, new SparkConf(false), securityMgr)
   val mapOutputTracker = new MapOutputTrackerMaster(new SparkConf(false), bcastManager, true)
@@ -145,9 +146,10 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     when(sc.conf).thenReturn(conf)
 
     val blockManagerInfo = new mutable.HashMap[BlockManagerId, BlockManagerInfo]()
+    liveListenerBus = spy(new LiveListenerBus(conf))
     master = spy(new BlockManagerMaster(rpcEnv.setupEndpoint("blockmanager",
       new BlockManagerMasterEndpoint(rpcEnv, true, conf,
-        new LiveListenerBus(conf), None, blockManagerInfo)),
+        liveListenerBus, None, blockManagerInfo)),
       rpcEnv.setupEndpoint("blockmanagerHeartbeat",
       new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true))
 
@@ -164,6 +166,7 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
       rpcEnv.awaitTermination()
       rpcEnv = null
       master = null
+      liveListenerBus = null
     } finally {
       super.afterEach()
     }
@@ -1693,6 +1696,16 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     assert(locs(blockIds(0)) == expectedLocs)
   }
 
+  test("SPARK-30594: Do not post SparkListenerBlockUpdated when updateBlockInfo returns false") {
+    // update block info for non-existent block manager
+    val updateInfo = UpdateBlockInfo(BlockManagerId("1", "host1", 100),
+      BlockId("test_1"), StorageLevel.MEMORY_ONLY, 1, 1)
+    val result = master.driverEndpoint.askSync[Boolean](updateInfo)
+
+    assert(!result)
+    verify(liveListenerBus, never()).post(SparkListenerBlockUpdated(BlockUpdatedInfo(updateInfo)))
+  }
+
   class MockBlockTransferService(val maxFailures: Int) extends BlockTransferService {
     var numCalls = 0
     var tempFileManager: DownloadFileManager = null

From 3ccebcdd45a0def4853c1175c18d8538602d571d Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 5 Feb 2020 17:16:38 +0800
Subject: [PATCH 0014/1280] [SPARK-30506][SQL][DOC] Document for generic file
 source options/configs

### What changes were proposed in this pull request?

Add a new document page named *Generic File Source Options* for *Data Sources* menu and added following sub items:

* spark.sql.files.ignoreCorruptFiles
* spark.sql.files.ignoreMissingFiles
* pathGlobFilter
* recursiveFileLookup

And here're snapshots of the generated document:
<img width="1080" alt="doc-1" src="https://user-images.githubusercontent.com/16397174/73816825-87a54800-4824-11ea-97da-e5c40c59a7d4.png">
<img width="1081" alt="doc-2" src="https://user-images.githubusercontent.com/16397174/73816827-8a07a200-4824-11ea-99ec-9c8b0286625e.png">
<img width="1080" alt="doc-3" src="https://user-images.githubusercontent.com/16397174/73816831-8c69fc00-4824-11ea-84f0-6c9e94c2f0e2.png">
<img width="1081" alt="doc-4" src="https://user-images.githubusercontent.com/16397174/73816834-8f64ec80-4824-11ea-9355-76ad45476634.png">

### Why are the changes needed?

Better guidance for end-user.

### Does this PR introduce any user-facing change?

No, added in Spark 3.0.

### How was this patch tested?

Pass Jenkins.

Closes #27302 from Ngone51/doc-generic-file-source-option.

Lead-authored-by: yi.wu <yi.wu@databricks.com>
Co-authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5983ad9cc4481e224a7e094de830ef2e816c1fe6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/_data/menu-sql.yaml                      |   2 +
 docs/sql-data-sources-avro.md                 |   2 +-
 docs/sql-data-sources-generic-options.md      | 121 ++++++++++++++++++
 docs/sql-data-sources-load-save-functions.md  |  21 ---
 docs/sql-data-sources.md                      |   5 +
 .../sql/JavaSQLDataSourceExample.java         |  48 ++++++-
 examples/src/main/python/sql/datasource.py    |  48 ++++++-
 examples/src/main/r/RSparkSQLExample.R        |  24 +++-
 .../main/resources/dir1/dir2/file2.parquet    | Bin 0 -> 520 bytes
 .../src/main/resources/dir1/file1.parquet     | Bin 0 -> 520 bytes
 examples/src/main/resources/dir1/file3.json   |   1 +
 .../do_not_read_this.txt                      |   1 -
 .../users.orc                                 | Bin 448 -> 0 bytes
 .../favorite_color=red/users.orc              | Bin 402 -> 0 bytes
 .../examples/sql/SQLDataSourceExample.scala   |  48 ++++++-
 15 files changed, 282 insertions(+), 39 deletions(-)
 create mode 100644 docs/sql-data-sources-generic-options.md
 create mode 100644 examples/src/main/resources/dir1/dir2/file2.parquet
 create mode 100644 examples/src/main/resources/dir1/file1.parquet
 create mode 100644 examples/src/main/resources/dir1/file3.json
 delete mode 100644 examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt
 delete mode 100644 examples/src/main/resources/partitioned_users.orc/favorite_color=__HIVE_DEFAULT_PARTITION__/users.orc
 delete mode 100644 examples/src/main/resources/partitioned_users.orc/favorite_color=red/users.orc

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 3e4db7107ec34..241ec399d7bd5 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -24,6 +24,8 @@
   subitems:
     - text: "Generic Load/Save Functions"
       url: sql-data-sources-load-save-functions.html
+    - text: "Generic File Source Options"
+      url: sql-data-sources-generic-options.html
     - text: Parquet Files
       url: sql-data-sources-parquet.html
     - text: ORC Files
diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index b0076878e02da..8e6a4079cd5de 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -230,7 +230,7 @@ Data source options of Avro can be set via:
   <tr>
     <td><code>ignoreExtension</code></td>
     <td>true</td>
-    <td>The option controls ignoring of files without <code>.avro</code> extensions in read.<br> If the option is enabled, all files (with and without <code>.avro</code> extension) are loaded.<br> The option has been deprecated, and it will be removed in the future releases. Please use the general data source option <code>pathGlobFilter</code> for filtering file names.</td>
+    <td>The option controls ignoring of files without <code>.avro</code> extensions in read.<br> If the option is enabled, all files (with and without <code>.avro</code> extension) are loaded.<br> The option has been deprecated, and it will be removed in the future releases. Please use the general data source option <a href="./sql-data-sources-generic-options.html#path-global-filter">pathGlobFilter</a> for filtering file names.</td>
     <td>read</td>
   </tr>
   <tr>
diff --git a/docs/sql-data-sources-generic-options.md b/docs/sql-data-sources-generic-options.md
new file mode 100644
index 0000000000000..0cfe2ed1aa891
--- /dev/null
+++ b/docs/sql-data-sources-generic-options.md
@@ -0,0 +1,121 @@
+---
+layout: global
+title: Generic File Source Options
+displayTitle: Generic File Source Options
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+* Table of contents
+{:toc}
+
+These generic options/configurations are effective only when using file-based sources: parquet, orc, avro, json, csv, text.
+
+Please note that the hierarchy of directories used in examples below are:
+
+{% highlight text %}
+
+dir1/
+ ├── dir2/
+ │    └── file2.parquet (schema: <file: string>, content: "file2.parquet")
+ └── file1.parquet (schema: <file, string>, content: "file1.parquet")
+ └── file3.json (schema: <file, string>, content: "{'file':'corrupt.json'}")
+
+{% endhighlight %}
+
+### Ignore Corrupt Files
+
+Spark allows you to use `spark.sql.files.ignoreCorruptFiles` to ignore corrupt files while reading data
+from files. When set to true, the Spark jobs will continue to run when encountering corrupted files and
+the contents that have been read will still be returned.
+
+To ignore corrupt files while reading data files, you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example ignore_corrupt_files scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example ignore_corrupt_files java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example ignore_corrupt_files python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example ignore_corrupt_files r/RSparkSQLExample.R %}
+</div>
+</div>
+
+### Ignore Missing Files
+
+Spark allows you to use `spark.sql.files.ignoreMissingFiles` to ignore missing files while reading data
+from files. Here, missing file really means the deleted file under directory after you construct the
+`DataFrame`. When set to true, the Spark jobs will continue to run when encountering missing files and
+the contents that have been read will still be returned.
+
+### Path Global Filter
+
+`pathGlobFilter` is used to only include files with file names matching the pattern.
+The syntax follows <code>org.apache.hadoop.fs.GlobFilter</code>.
+It does not change the behavior of partition discovery.
+
+To load files with paths matching a given glob pattern while keeping the behavior of partition discovery,
+you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example load_with_path_glob_filter scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example load_with_path_glob_filter java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example load_with_path_glob_filter python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example load_with_path_glob_filter r/RSparkSQLExample.R %}
+</div>
+</div>
+
+### Recursive File Lookup
+`recursiveFileLookup` is used to recursively load files and it disables partition inferring. Its default value is `false`.
+If data source explicitly specifies the `partitionSpec` when `recursiveFileLookup` is true, exception will be thrown.
+
+To load all files recursively, you can use:
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example recursive_file_lookup scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
+</div>
+
+<div data-lang="java"  markdown="1">
+{% include_example recursive_file_lookup java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
+</div>
+
+<div data-lang="python"  markdown="1">
+{% include_example recursive_file_lookup python/sql/datasource.py %}
+</div>
+
+<div data-lang="r"  markdown="1">
+{% include_example recursive_file_lookup r/RSparkSQLExample.R %}
+</div>
+</div>
\ No newline at end of file
diff --git a/docs/sql-data-sources-load-save-functions.md b/docs/sql-data-sources-load-save-functions.md
index 07482137a28a3..a7efb9347ac64 100644
--- a/docs/sql-data-sources-load-save-functions.md
+++ b/docs/sql-data-sources-load-save-functions.md
@@ -102,27 +102,6 @@ To load a CSV file you can use:
 </div>
 </div>
 
-To load files with paths matching a given glob pattern while keeping the behavior of partition discovery,
-you can use:
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-{% include_example load_with_path_glob_filter scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala %}
-</div>
-
-<div data-lang="java"  markdown="1">
-{% include_example load_with_path_glob_filter java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java %}
-</div>
-
-<div data-lang="python"  markdown="1">
-{% include_example load_with_path_glob_filter python/sql/datasource.py %}
-</div>
-
-<div data-lang="r"  markdown="1">
-{% include_example load_with_path_glob_filter r/RSparkSQLExample.R %}
-</div>
-</div>
-
 The extra options are also used during write operation.
 For example, you can control bloom filters and dictionary encodings for ORC data sources.
 The following ORC example will create bloom filter and use dictionary encoding only for `favorite_color`.
diff --git a/docs/sql-data-sources.md b/docs/sql-data-sources.md
index 079c54060d15d..9396846041709 100644
--- a/docs/sql-data-sources.md
+++ b/docs/sql-data-sources.md
@@ -33,6 +33,11 @@ goes into specific options that are available for the built-in data sources.
   * [Save Modes](sql-data-sources-load-save-functions.html#save-modes)
   * [Saving to Persistent Tables](sql-data-sources-load-save-functions.html#saving-to-persistent-tables)
   * [Bucketing, Sorting and Partitioning](sql-data-sources-load-save-functions.html#bucketing-sorting-and-partitioning)
+* [Generic File Source Options](sql-data-sources-generic-options.html)
+  * [Ignore Corrupt Files](sql-data-sources-generic-options.html#ignore-corrupt-iles)
+  * [Ignore Missing Files](sql-data-sources-generic-options.html#ignore-missing-iles)
+  * [Path Global Filter](sql-data-sources-generic-options.html#path-global-filter)
+  * [Recursive File Lookup](sql-data-sources-generic-options.html#recursive-file-lookup)
 * [Parquet Files](sql-data-sources-parquet.html)
   * [Loading Data Programmatically](sql-data-sources-parquet.html#loading-data-programmatically)
   * [Partition Discovery](sql-data-sources-parquet.html#partition-discovery)
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
index b2ce0bc08642a..2295225387a33 100644
--- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java
@@ -98,6 +98,7 @@ public static void main(String[] args) {
       .getOrCreate();
 
     runBasicDataSourceExample(spark);
+    runGenericFileSourceOptionsExample(spark);
     runBasicParquetExample(spark);
     runParquetSchemaMergingExample(spark);
     runJsonDatasetExample(spark);
@@ -106,6 +107,48 @@ public static void main(String[] args) {
     spark.stop();
   }
 
+  private static void runGenericFileSourceOptionsExample(SparkSession spark) {
+    // $example on:ignore_corrupt_files$
+    // enable ignore corrupt files
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=true");
+    // dir1/file3.json is corrupt from parquet's view
+    Dataset<Row> testCorruptDF = spark.read().parquet(
+            "examples/src/main/resources/dir1/",
+            "examples/src/main/resources/dir1/dir2/");
+    testCorruptDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:ignore_corrupt_files$
+    // $example on:recursive_file_lookup$
+    Dataset<Row> recursiveLoadedDF = spark.read().format("parquet")
+            .option("recursiveFileLookup", "true")
+            .load("examples/src/main/resources/dir1");
+    recursiveLoadedDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:recursive_file_lookup$
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=false");
+    // $example on:load_with_path_glob_filter$
+    Dataset<Row> testGlobFilterDF = spark.read().format("parquet")
+            .option("pathGlobFilter", "*.parquet") // json file should be filtered out
+            .load("examples/src/main/resources/dir1");
+    testGlobFilterDF.show();
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // +-------------+
+    // $example off:load_with_path_glob_filter$
+  }
+
   private static void runBasicDataSourceExample(SparkSession spark) {
     // $example on:generic_load_save_functions$
     Dataset<Row> usersDF = spark.read().load("examples/src/main/resources/users.parquet");
@@ -123,11 +166,6 @@ private static void runBasicDataSourceExample(SparkSession spark) {
       .option("header", "true")
       .load("examples/src/main/resources/people.csv");
     // $example off:manual_load_options_csv$
-    // $example on:load_with_path_glob_filter$
-    Dataset<Row> partitionedUsersDF = spark.read().format("orc")
-      .option("pathGlobFilter", "*.orc")
-      .load("examples/src/main/resources/partitioned_users.orc");
-    // $example off:load_with_path_glob_filter$
     // $example on:manual_save_options_orc$
     usersDF.write().format("orc")
       .option("orc.bloom.filter.columns", "favorite_color")
diff --git a/examples/src/main/python/sql/datasource.py b/examples/src/main/python/sql/datasource.py
index 0d78097ea975e..265f135e1e5f2 100644
--- a/examples/src/main/python/sql/datasource.py
+++ b/examples/src/main/python/sql/datasource.py
@@ -28,6 +28,48 @@
 # $example off:schema_merging$
 
 
+def generic_file_source_options_example(spark):
+    # $example on:ignore_corrupt_files$
+    # enable ignore corrupt files
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=true")
+    # dir1/file3.json is corrupt from parquet's view
+    test_corrupt_df = spark.read.parquet("examples/src/main/resources/dir1/",
+                                         "examples/src/main/resources/dir1/dir2/")
+    test_corrupt_df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # |file2.parquet|
+    # +-------------+
+    # $example off:ignore_corrupt_files$
+
+    # $example on:recursive_file_lookup$
+    recursive_loaded_df = spark.read.format("parquet")\
+        .option("recursiveFileLookup", "true")\
+        .load("examples/src/main/resources/dir1")
+    recursive_loaded_df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # |file2.parquet|
+    # +-------------+
+    # $example off:recursive_file_lookup$
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=false")
+
+    # $example on:load_with_path_glob_filter$
+    df = spark.read.load("examples/src/main/resources/dir1",
+                         format="parquet", pathGlobFilter="*.parquet")
+    df.show()
+    # +-------------+
+    # |         file|
+    # +-------------+
+    # |file1.parquet|
+    # +-------------+
+    # $example off:load_with_path_glob_filter$
+
+
 def basic_datasource_example(spark):
     # $example on:generic_load_save_functions$
     df = spark.read.load("examples/src/main/resources/users.parquet")
@@ -57,11 +99,6 @@ def basic_datasource_example(spark):
                          format="csv", sep=":", inferSchema="true", header="true")
     # $example off:manual_load_options_csv$
 
-    # $example on:load_with_path_glob_filter$
-    df = spark.read.load("examples/src/main/resources/partitioned_users.orc",
-                         format="orc", pathGlobFilter="*.orc")
-    # $example off:load_with_path_glob_filter$
-
     # $example on:manual_save_options_orc$
     df = spark.read.orc("examples/src/main/resources/users.orc")
     (df.write.format("orc")
@@ -233,6 +270,7 @@ def jdbc_dataset_example(spark):
         .getOrCreate()
 
     basic_datasource_example(spark)
+    generic_file_source_options_example(spark)
     parquet_example(spark)
     parquet_schema_merging_example(spark)
     json_dataset_example(spark)
diff --git a/examples/src/main/r/RSparkSQLExample.R b/examples/src/main/r/RSparkSQLExample.R
index fa083d5542fae..8685cfb5c05f2 100644
--- a/examples/src/main/r/RSparkSQLExample.R
+++ b/examples/src/main/r/RSparkSQLExample.R
@@ -99,6 +99,26 @@ createOrReplaceTempView(df, "table")
 df <- sql("SELECT * FROM table")
 # $example off:run_sql$
 
+# Ignore corrupt files
+# $example on:ignore_corrupt_files$
+# enable ignore corrupt files
+sql("set spark.sql.files.ignoreCorruptFiles=true")
+# dir1/file3.json is corrupt from parquet's view
+testCorruptDF <- read.parquet(c("examples/src/main/resources/dir1/", "examples/src/main/resources/dir1/dir2/"))
+head(testCorruptDF)
+#            file
+# 1 file1.parquet
+# 2 file2.parquet
+# $example off:ignore_corrupt_files$
+
+# $example on:recursive_file_lookup$
+recursiveLoadedDF <- read.df("examples/src/main/resources/dir1", "parquet", recursiveFileLookup = "true")
+head(recursiveLoadedDF)
+#            file
+# 1 file1.parquet
+# 2 file2.parquet
+# $example off:recursive_file_lookup$
+sql("set spark.sql.files.ignoreCorruptFiles=false")
 
 # $example on:generic_load_save_functions$
 df <- read.df("examples/src/main/resources/users.parquet")
@@ -119,7 +139,9 @@ namesAndAges <- select(df, "name", "age")
 # $example off:manual_load_options_csv$
 
 # $example on:load_with_path_glob_filter$
-df <- read.df("examples/src/main/resources/partitioned_users.orc", "orc", pathGlobFilter = "*.orc")
+df <- read.df("examples/src/main/resources/dir1", "parquet", pathGlobFilter = "*.parquet")
+#            file
+# 1 file1.parquet
 # $example off:load_with_path_glob_filter$
 
 # $example on:manual_save_options_orc$
diff --git a/examples/src/main/resources/dir1/dir2/file2.parquet b/examples/src/main/resources/dir1/dir2/file2.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..d1895bf29b75ce0d6c70009562975645b7699fb9
GIT binary patch
literal 520
zcmah{O-sW-5S>OXIi$xeyO2XzXlWrXNkh{nc=0Blibq9c(`;%mN!xrVQu;6aVg4ki
z+9Fm^*j@JVX5PG+-Pz@hOMnO>Y@*?%O>~oXk~C8zv6AJwQS}k*!r)IH05seutqz_)
zgowuME2Bc$r-y3(sB%d(AVyE4r@OcwKv!cXGyA$p3^s0q&b}CeMEAXgtFK=i**Sv$
zx??7G30N4bp`-@PrgT{@gj`AVBtGqlXH{0|vY=<4aD)SN_$#7XXNLiaa`_^1Rm)h`
zlHP83{kl>-<W*fZ^P1Z{NmUkCJiOcRLgqVa*Dn&^foZWoTr^oGBcwu1RW?6}@^G`=
z-fw|k@89G+FRi^t%N4F8Tx$?DX%<^@8pO#o2?B2v`6}^!&x;-3@dGt>=0P+GCQhWH
R*c~#{#2LJ`0A~0HJ^)`pl6wFE

literal 0
HcmV?d00001

diff --git a/examples/src/main/resources/dir1/file1.parquet b/examples/src/main/resources/dir1/file1.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..ad360b16fd898ea301cbbc00de07a2fd40bbbb44
GIT binary patch
literal 520
zcmah{O-sW-5S>OXIkd+uyRe6_&|)DjiJ@r|ym%8&#iJs!X*M;Oq-{PFDg77zFn^L$
zZ4oOd>@NFwGjHC^?)37;B|s#RHZkzhCMKudB+axdm#SPfx;}tCIQ^3c00Z|?tBdCk
zA!0E5s#Brt(?d0T)VU%CkWSyA$J@9*K+j;D@qx)nhmG1`vM+`p(R**<8kiRcb`Idb
z?ij}|0SjX_lr(_ZlrGA-R7-`P<flFPtjj806!gTCj&y(ye??TQY(F4bE?#7=YZa?n
z(c87CUzb{hqN>YgRtsAssm|g`gm-IEsC-Lp`$ghAFfHbYizdrdgj9&B+7^3J5w16z
z`wh_R-J6`{rM2^Dxzcr{Yn?_-n#Goy1aUG+g1|eEe4Y5d=f#fi_<<fdvmhD=V<*y4
Q><$^{ID@wqz!d+$2T@;=asU7T

literal 0
HcmV?d00001

diff --git a/examples/src/main/resources/dir1/file3.json b/examples/src/main/resources/dir1/file3.json
new file mode 100644
index 0000000000000..0490f92d7f317
--- /dev/null
+++ b/examples/src/main/resources/dir1/file3.json
@@ -0,0 +1 @@
+{"file":"corrupt.json"}
diff --git a/examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt b/examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt
deleted file mode 100644
index 9c19f2a0449eb..0000000000000
--- a/examples/src/main/resources/partitioned_users.orc/do_not_read_this.txt
+++ /dev/null
@@ -1 +0,0 @@
-do not read this
diff --git a/examples/src/main/resources/partitioned_users.orc/favorite_color=__HIVE_DEFAULT_PARTITION__/users.orc b/examples/src/main/resources/partitioned_users.orc/favorite_color=__HIVE_DEFAULT_PARTITION__/users.orc
deleted file mode 100644
index 890395a9281abb71a8444a3c9041155fae6c0f9f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 448
zcmZ9I&q~8U5XNVBo9(zw?Ydr;SjFZbMIi*dg&ryutq3;s6fbI&s73pSL<`=EPtdpU
zMZ_oa0o<ettqU{X4EqiIW_tb327utIgd~JhQ#OPJ`BNH+s{}uKe#^30{i?P@*fOq)
zV7aH5Ixzj_qPMZ6IC9z4S0PjZ7Q<zL6SygIy8`gVxSh-a!n(-4peq(4(R(ISEiLRa
zt<4XSM6Vg8M!AJREewfd8Wlw_LH{BJXhk9r1&nZ3u&Y?lk)xX6#66C<SYIZL@QeU5
z`AHmM<Wg_%o;QiRgSMiCI;BxOOdNS1Psi!tCAl8G4sVk*b19Zx3xy)glx)8t^f-O+
z;z@k>n0VPFPM^G~@eRf!WgU9H=ldim;DS}SrJF{zz-(+ldGL?0J-_Gzeh^9ZY$ja_
PcC+R4_ix5}{f_<tkfB2j

diff --git a/examples/src/main/resources/partitioned_users.orc/favorite_color=red/users.orc b/examples/src/main/resources/partitioned_users.orc/favorite_color=red/users.orc
deleted file mode 100644
index 150615a6f3b2401843a414932873bb41142b4363..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 402
zcmZvW!AiqG5Qb-Vo9(a-xGoizR;0#D@em@tNW2t#(31&0^(IlG7HL{i+jH^c(Yue}
zOZZSefRm`b=)(Rp!~Fd7CF9X9fL69an8F#iUF|{jtR)P5Ezk49R^n6&UEa5}thTlP
zi07T}nFy?KkU|rH-Ic-z>k!}tAl=uBlmXZBh|u*|g)<(VO4sHM8gz=Q*dq|0Nzq8h
zaaAOV-=FcZP}g^L`!CzXuYyvTD7|GlK_F?~v7@;p&h{?)7#xsq1&U6i$Yy!R+-2)|
zIep5ni`DEVFPDL!X5f*wD7lr#huY1{`!HK%w-0%^Tx8{AxMtsUURZAsMqw0TTvNsW
iQpG=1@s?`%dUXl(hDNx}#smWV{vcA%pHe2{k^2I!Wi|=`

diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
index d4c05e5ad9944..2c7abfcd335d1 100644
--- a/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala
@@ -32,6 +32,7 @@ object SQLDataSourceExample {
       .getOrCreate()
 
     runBasicDataSourceExample(spark)
+    runGenericFileSourceOptionsExample(spark)
     runBasicParquetExample(spark)
     runParquetSchemaMergingExample(spark)
     runJsonDatasetExample(spark)
@@ -40,6 +41,48 @@ object SQLDataSourceExample {
     spark.stop()
   }
 
+  private def runGenericFileSourceOptionsExample(spark: SparkSession): Unit = {
+    // $example on:ignore_corrupt_files$
+    // enable ignore corrupt files
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=true")
+    // dir1/file3.json is corrupt from parquet's view
+    val testCorruptDF = spark.read.parquet(
+      "examples/src/main/resources/dir1/",
+      "examples/src/main/resources/dir1/dir2/")
+    testCorruptDF.show()
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:ignore_corrupt_files$
+    // $example on:recursive_file_lookup$
+    val recursiveLoadedDF = spark.read.format("parquet")
+      .option("recursiveFileLookup", "true")
+      .load("examples/src/main/resources/dir1")
+    recursiveLoadedDF.show()
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // |file2.parquet|
+    // +-------------+
+    // $example off:recursive_file_lookup$
+    spark.sql("set spark.sql.files.ignoreCorruptFiles=false")
+    // $example on:load_with_path_glob_filter$
+    val testGlobFilterDF = spark.read.format("parquet")
+      .option("pathGlobFilter", "*.parquet") // json file should be filtered out
+      .load("examples/src/main/resources/dir1")
+    testGlobFilterDF.show()
+    // +-------------+
+    // |         file|
+    // +-------------+
+    // |file1.parquet|
+    // +-------------+
+    // $example off:load_with_path_glob_filter$
+  }
+
   private def runBasicDataSourceExample(spark: SparkSession): Unit = {
     // $example on:generic_load_save_functions$
     val usersDF = spark.read.load("examples/src/main/resources/users.parquet")
@@ -56,11 +99,6 @@ object SQLDataSourceExample {
       .option("header", "true")
       .load("examples/src/main/resources/people.csv")
     // $example off:manual_load_options_csv$
-    // $example on:load_with_path_glob_filter$
-    val partitionedUsersDF = spark.read.format("orc")
-      .option("pathGlobFilter", "*.orc")
-      .load("examples/src/main/resources/partitioned_users.orc")
-    // $example off:load_with_path_glob_filter$
     // $example on:manual_save_options_orc$
     usersDF.write.format("orc")
       .option("orc.bloom.filter.columns", "favorite_color")

From 92f57237871400ab9d499e1174af22a867c01988 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Wed, 5 Feb 2020 18:48:45 +0800
Subject: [PATCH 0015/1280] [SPARK-30668][SQL] Support `SimpleDateFormat`
 patterns in parsing timestamps/dates strings

### What changes were proposed in this pull request?
In the PR, I propose to partially revert the commit https://github.com/apache/spark/commit/51a6ba0181a013f2b62b47184785a8b6f6a78f12, and provide a legacy parser based on `FastDateFormat` which is compatible to `SimpleDateFormat`.

To enable the legacy parser, set `spark.sql.legacy.timeParser.enabled` to `true`.

### Why are the changes needed?
To allow users to restore old behavior in parsing timestamps/dates using `SimpleDateFormat` patterns. The main reason for restoring is `DateTimeFormatter`'s patterns are not fully compatible to `SimpleDateFormat` patterns, see https://issues.apache.org/jira/browse/SPARK-30668

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
- Added new test to `DateFunctionsSuite`
- Restored additional test cases in `JsonInferSchemaSuite`.

Closes #27441 from MaxGekk/support-simpledateformat.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 459e757ed40fd1cdd37911d3f57b48b54ca2fff7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |  4 +-
 .../sql/catalyst/util/DateFormatter.scala     | 35 +++++++-
 .../catalyst/util/TimestampFormatter.scala    | 38 +++++++--
 .../apache/spark/sql/internal/SQLConf.scala   | 10 +++
 .../catalyst/json/JsonInferSchemaSuite.scala  | 79 +++++++++++--------
 .../apache/spark/sql/DateFunctionsSuite.scala | 14 ++++
 6 files changed, 136 insertions(+), 44 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 0c47370283736..5a5e802f6a900 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -67,9 +67,7 @@ license: |
 
   - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
-    - CSV/JSON datasources use java.time API for parsing and generating CSV/JSON content. In Spark version 2.4 and earlier, java.text.SimpleDateFormat is used for the same purpose with fallbacks to the parsing mechanisms of Spark 2.0 and 1.x. For example, `2018-12-08 10:39:21.123` with the pattern `yyyy-MM-dd'T'HH:mm:ss.SSS` cannot be parsed since Spark 3.0 because the timestamp does not match to the pattern but it can be parsed by earlier Spark versions due to a fallback to `Timestamp.valueOf`. To parse the same timestamp since Spark 3.0, the pattern should be `yyyy-MM-dd HH:mm:ss.SSS`.
-
-    - The `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions. New implementation supports pattern formats as described here https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html and performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`.
+    - Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. Since Spark 3.0, the conversions are based on `java.time.format.DateTimeFormatter`, see https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and earlier, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html. The old behavior can be restored by setting `spark.sql.legacy.timeParser.enabled` to `true`.
 
     - The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 7f982b019c8d1..28189b65dee9a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -20,7 +20,10 @@ package org.apache.spark.sql.catalyst.util
 import java.time.{LocalDate, ZoneId}
 import java.util.Locale
 
-import DateTimeUtils.{convertSpecialDate, localDateToDays}
+import org.apache.commons.lang3.time.FastDateFormat
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{convertSpecialDate, localDateToDays}
+import org.apache.spark.sql.internal.SQLConf
 
 sealed trait DateFormatter extends Serializable {
   def parse(s: String): Int // returns days since epoch
@@ -48,17 +51,41 @@ class Iso8601DateFormatter(
   }
 }
 
+class LegacyDateFormatter(pattern: String, locale: Locale) extends DateFormatter {
+  @transient
+  private lazy val format = FastDateFormat.getInstance(pattern, locale)
+
+  override def parse(s: String): Int = {
+    val milliseconds = format.parse(s).getTime
+    DateTimeUtils.millisToDays(milliseconds)
+  }
+
+  override def format(days: Int): String = {
+    val date = DateTimeUtils.toJavaDate(days)
+    format.format(date)
+  }
+}
+
 object DateFormatter {
-  val defaultPattern: String = "uuuu-MM-dd"
   val defaultLocale: Locale = Locale.US
 
   def apply(format: String, zoneId: ZoneId, locale: Locale): DateFormatter = {
-    new Iso8601DateFormatter(format, zoneId, locale)
+    if (SQLConf.get.legacyTimeParserEnabled) {
+      new LegacyDateFormatter(format, locale)
+    } else {
+      new Iso8601DateFormatter(format, zoneId, locale)
+    }
   }
 
   def apply(format: String, zoneId: ZoneId): DateFormatter = {
     apply(format, zoneId, defaultLocale)
   }
 
-  def apply(zoneId: ZoneId): DateFormatter = apply(defaultPattern, zoneId)
+  def apply(zoneId: ZoneId): DateFormatter = {
+    if (SQLConf.get.legacyTimeParserEnabled) {
+      new LegacyDateFormatter("yyyy-MM-dd", defaultLocale)
+    } else {
+      new Iso8601DateFormatter("uuuu-MM-dd", zoneId, defaultLocale)
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 5be4807083fa3..fe1a4fe710c20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -22,10 +22,14 @@ import java.time._
 import java.time.format.DateTimeParseException
 import java.time.temporal.ChronoField.MICRO_OF_SECOND
 import java.time.temporal.TemporalQueries
-import java.util.Locale
+import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit.SECONDS
 
-import DateTimeUtils.convertSpecialTimestamp
+import org.apache.commons.lang3.time.FastDateFormat
+
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_MILLIS
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.convertSpecialTimestamp
+import org.apache.spark.sql.internal.SQLConf
 
 sealed trait TimestampFormatter extends Serializable {
   /**
@@ -86,12 +90,32 @@ class FractionTimestampFormatter(zoneId: ZoneId)
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
 }
 
+class LegacyTimestampFormatter(
+    pattern: String,
+    zoneId: ZoneId,
+    locale: Locale) extends TimestampFormatter {
+
+  @transient private lazy val format =
+    FastDateFormat.getInstance(pattern, TimeZone.getTimeZone(zoneId), locale)
+
+  protected def toMillis(s: String): Long = format.parse(s).getTime
+
+  override def parse(s: String): Long = toMillis(s) * MICROS_PER_MILLIS
+
+  override def format(us: Long): String = {
+    format.format(DateTimeUtils.toJavaTimestamp(us))
+  }
+}
+
 object TimestampFormatter {
-  val defaultPattern: String = "uuuu-MM-dd HH:mm:ss"
   val defaultLocale: Locale = Locale.US
 
   def apply(format: String, zoneId: ZoneId, locale: Locale): TimestampFormatter = {
-    new Iso8601TimestampFormatter(format, zoneId, locale)
+    if (SQLConf.get.legacyTimeParserEnabled) {
+      new LegacyTimestampFormatter(format, zoneId, locale)
+    } else {
+      new Iso8601TimestampFormatter(format, zoneId, locale)
+    }
   }
 
   def apply(format: String, zoneId: ZoneId): TimestampFormatter = {
@@ -99,7 +123,11 @@ object TimestampFormatter {
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
-    apply(defaultPattern, zoneId, defaultLocale)
+    if (SQLConf.get.legacyTimeParserEnabled) {
+      new LegacyTimestampFormatter("yyyy-MM-dd HH:mm:ss", zoneId, defaultLocale)
+    } else {
+      new Iso8601TimestampFormatter("uuuu-MM-dd HH:mm:ss", zoneId, defaultLocale)
+    }
   }
 
   def getFractionFormatter(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 5ce5692123805..acc0922e2cee7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2159,6 +2159,14 @@ object SQLConf {
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
 
+  val LEGACY_TIME_PARSER_ENABLED = buildConf("spark.sql.legacy.timeParser.enabled")
+    .internal()
+    .doc("When set to true, java.text.SimpleDateFormat is used for formatting and parsing " +
+      "dates/timestamps in a locale-sensitive manner. When set to false, classes from " +
+      "java.time.* packages are used for the same purpose.")
+    .booleanConf
+    .createWithDefault(false)
+
   /**
    * Holds information about keys that have been deprecated.
    *
@@ -2447,6 +2455,8 @@ class SQLConf extends Serializable with Logging {
   def legacyMsSqlServerNumericMappingEnabled: Boolean =
     getConf(LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED)
 
+  def legacyTimeParserEnabled: Boolean = getConf(SQLConf.LEGACY_TIME_PARSER_ENABLED)
+
   /**
    * Returns the [[Resolver]] for the current configuration, which can be used to determine if two
    * identifiers are equal.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
index a48e61861c158..c2e03bd2c3609 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
@@ -17,10 +17,9 @@
 
 package org.apache.spark.sql.catalyst.json
 
-import com.fasterxml.jackson.core.JsonFactory
-
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
@@ -41,45 +40,61 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("inferring timestamp type") {
-    checkTimestampType("yyyy", """{"a": "2018"}""")
-    checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
-    checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
-    checkTimestampType(
-      "yyyy-MM-dd'T'HH:mm:ss.SSS",
-      """{"a": "2018-12-02T21:04:00.123"}""")
-    checkTimestampType(
-      "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
-      """{"a": "2018-12-02T21:04:00.123567+01:00"}""")
+    Seq(true, false).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        checkTimestampType("yyyy", """{"a": "2018"}""")
+        checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
+        checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
+        checkTimestampType(
+          "yyyy-MM-dd'T'HH:mm:ss.SSS",
+          """{"a": "2018-12-02T21:04:00.123"}""")
+        checkTimestampType(
+          "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
+          """{"a": "2018-12-02T21:04:00.123567+01:00"}""")
+      }
+    }
   }
 
   test("prefer decimals over timestamps") {
-    checkType(
-      options = Map(
-        "prefersDecimal" -> "true",
-        "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
-      ),
-      json = """{"a": "20181202.210400123"}""",
-      dt = DecimalType(17, 9)
-    )
+    Seq(true, false).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        checkType(
+          options = Map(
+            "prefersDecimal" -> "true",
+            "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
+          ),
+          json = """{"a": "20181202.210400123"}""",
+          dt = DecimalType(17, 9)
+        )
+      }
+    }
   }
 
   test("skip decimal type inferring") {
-    checkType(
-      options = Map(
-        "prefersDecimal" -> "false",
-        "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
-      ),
-      json = """{"a": "20181202.210400123"}""",
-      dt = TimestampType
-    )
+    Seq(true, false).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        checkType(
+          options = Map(
+            "prefersDecimal" -> "false",
+            "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
+          ),
+          json = """{"a": "20181202.210400123"}""",
+          dt = TimestampType
+        )
+      }
+    }
   }
 
   test("fallback to string type") {
-    checkType(
-      options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
-      json = """{"a": "20181202.210400123"}""",
-      dt = StringType
-    )
+    Seq(true, false).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        checkType(
+          options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
+          json = """{"a": "20181202.210400123"}""",
+          dt = StringType
+        )
+      }
+    }
   }
 
   test("disable timestamp inferring") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index d7d8c2c52d12b..3b3d3cc3d7a17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -789,4 +789,18 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
         Row(Timestamp.valueOf("2015-07-24 07:00:00")),
         Row(Timestamp.valueOf("2015-07-24 22:00:00"))))
   }
+
+  test("SPARK-30668: use legacy timestamp parser in to_timestamp") {
+    def checkTimeZoneParsing(expected: Any): Unit = {
+      val df = Seq("2020-01-27T20:06:11.847-0800").toDF("ts")
+      checkAnswer(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")),
+        Row(expected))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> "true") {
+      checkTimeZoneParsing(Timestamp.valueOf("2020-01-27 20:06:11.847"))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> "false") {
+      checkTimeZoneParsing(null)
+    }
+  }
 }

From fb99ebc47d12e82a2814b6b4b3b97148fbaa8627 Mon Sep 17 00:00:00 2001
From: turbofei <fwang12@ebay.com>
Date: Wed, 5 Feb 2020 21:24:02 +0800
Subject: [PATCH 0016/1280] [SPARK-26218][SQL][FOLLOW UP] Fix the corner case
 when casting float to Integer

### What changes were proposed in this pull request?
When spark.sql.ansi.enabled is true, for the statement:
```
select cast(cast(2147483648 as Float) as Integer) //result is 2147483647
```
Its result is 2147483647 and does not throw `ArithmeticException`.

The root cause is that, the below code does not work for some corner cases.
https://github.com/apache/spark/blob/94fc0e3235162afc6038019eed6ec546e3d1983e/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala#L129-L141

For example:

![image](https://user-images.githubusercontent.com/6757692/72074911-badfde80-332d-11ea-963e-2db0e43c33e8.png)

In this PR, I fix it by comparing Math.floor(x) with Int.MaxValue directly.

### Why are the changes needed?
Result corrupt.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?

Added Unit test.

Closes #27151 from turboFei/SPARK-26218-follow-up-int-overflow.

Authored-by: turbofei <fwang12@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6d507b4a31feb965bf31d104f1a6a2c359b166dc)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/types/numerics.scala    | 16 ++++++++--------
 .../sql-tests/results/postgreSQL/float4.sql.out  |  5 +++--
 .../org/apache/spark/sql/SQLQuerySuite.scala     | 11 +++++++++++
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
index 1ac85360f944f..b5226213effc4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
@@ -121,10 +121,10 @@ object FloatExactNumeric extends FloatIsFractional {
   private def overflowException(x: Float, dataType: String) =
     throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
 
-  private val intUpperBound = Int.MaxValue.toFloat
-  private val intLowerBound = Int.MinValue.toFloat
-  private val longUpperBound = Long.MaxValue.toFloat
-  private val longLowerBound = Long.MinValue.toFloat
+  private val intUpperBound = Int.MaxValue
+  private val intLowerBound = Int.MinValue
+  private val longUpperBound = Long.MaxValue
+  private val longLowerBound = Long.MinValue
 
   override def toInt(x: Float): Int = {
     // When casting floating values to integral types, Spark uses the method `Numeric.toInt`
@@ -155,10 +155,10 @@ object DoubleExactNumeric extends DoubleIsFractional {
   private def overflowException(x: Double, dataType: String) =
     throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
 
-  private val intUpperBound = Int.MaxValue.toDouble
-  private val intLowerBound = Int.MinValue.toDouble
-  private val longUpperBound = Long.MaxValue.toDouble
-  private val longLowerBound = Long.MinValue.toDouble
+  private val intUpperBound = Int.MaxValue
+  private val intLowerBound = Int.MinValue
+  private val longUpperBound = Long.MaxValue
+  private val longLowerBound = Long.MinValue
 
   override def toInt(x: Double): Int = {
     if (Math.floor(x) <= intUpperBound && Math.ceil(x) >= intLowerBound) {
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
index ba913789d5623..fe8375c5eab8f 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/float4.sql.out
@@ -322,9 +322,10 @@ struct<CAST(CAST(2147483520 AS FLOAT) AS INT):int>
 -- !query
 SELECT int(float('2147483647'))
 -- !query schema
-struct<CAST(CAST(2147483647 AS FLOAT) AS INT):int>
+struct<>
 -- !query output
-2147483647
+java.lang.ArithmeticException
+Casting 2.14748365E9 to int causes overflow
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index a6dae9a269740..11f9724e587f2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3383,6 +3383,17 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       checkAnswer(df, Row(1))
     }
   }
+
+  test("SPARK-26218: Fix the corner case when casting float to Integer") {
+    withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+      intercept[ArithmeticException](
+        sql("SELECT CAST(CAST(2147483648 as FLOAT) as Integer)").collect()
+      )
+      intercept[ArithmeticException](
+        sql("SELECT CAST(CAST(2147483648 as DOUBLE) as Integer)").collect()
+      )
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From abc93b05272b2853862689b553eec1320d3f5f7f Mon Sep 17 00:00:00 2001
From: WeichenXu <weichen.xu@databricks.com>
Date: Wed, 5 Feb 2020 07:54:16 -0800
Subject: [PATCH 0017/1280] [MINOR][DOC] Add migration note for removing
 `org.apache.spark.ml.image.ImageSchema.readImages`

### What changes were proposed in this pull request?

Add migration note for removing `org.apache.spark.ml.image.ImageSchema.readImages`

### Why are the changes needed?

### Does this PR introduce any user-facing change?

### How was this patch tested?

Closes #27467 from WeichenXu123/SC-26286.

Authored-by: WeichenXu <weichen.xu@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit ec70e0708f953f3b22ec17d931ff388d007ac1f6)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/ml-migration-guide.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/ml-migration-guide.md b/docs/ml-migration-guide.md
index 49f701b2156b3..860c941e6b44b 100644
--- a/docs/ml-migration-guide.md
+++ b/docs/ml-migration-guide.md
@@ -32,6 +32,7 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 {:.no_toc}
 
 * `OneHotEncoder` which is deprecated in 2.3, is removed in 3.0 and `OneHotEncoderEstimator` is now renamed to `OneHotEncoder`.
+* `org.apache.spark.ml.image.ImageSchema.readImages` which is deprecated in 2.3, is removed in 3.0, use `spark.read.format('image')` instead.
 
 ### Changes of behavior
 {:.no_toc}

From 5561cead40041d2798fab3876ef75bcfbfdb854a Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 5 Feb 2020 11:01:53 -0800
Subject: [PATCH 0018/1280] [SPARK-30738][K8S] Use specific image version in
 "Launcher client dependencies" test

### What changes were proposed in this pull request?

This PR use a specific version of docker image instead of `latest`. As of today, when I run K8s integration test locally, this test case fails always.

Also, in this PR, I shows two consecutive failures with a dummy change.
- https://github.com/apache/spark/pull/27465#issuecomment-582326614
- https://github.com/apache/spark/pull/27465#issuecomment-582329114
```
- Launcher client dependencies *** FAILED ***
```

After that, I added the patch and K8s Integration test passed.
- https://github.com/apache/spark/pull/27465#issuecomment-582361696

### Why are the changes needed?

[SPARK-28465](https://github.com/apache/spark/pull/25222) switched from `v4.0.0-stable-4.0-master-centos-7-x86_64` to `latest` to catch up the API change. However, the API change seems to occur again. We had better use a specific version to prevent accidental failures.

```scala
- .withImage("ceph/daemon:v4.0.0-stable-4.0-master-centos-7-x86_64")
+ .withImage("ceph/daemon:latest")
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass `Launcher client dependencies` test in Jenkins K8s Integration Suite.
Or, run K8s Integration test locally.

Closes #27465 from dongjoon-hyun/SPARK-K8S-IT.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 9d90c8b898d0f043afbcebd901ec866c2883c6ca)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/deploy/k8s/integrationtest/DepsTestsSuite.scala       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index 7181774b9f17e..414126862ed69 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -63,7 +63,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     ).asJava
 
     new ContainerBuilder()
-      .withImage("ceph/daemon:latest")
+      .withImage("ceph/daemon:v4.0.3-stable-4.0-nautilus-centos-7-x86_64")
       .withImagePullPolicy("Always")
       .withName(cName)
       .withPorts(new ContainerPortBuilder()

From 3013d28d0a17e6fcea6e42e7caee23a43dd7e196 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Wed, 5 Feb 2020 11:19:42 -0800
Subject: [PATCH 0019/1280] [SPARK-29864][SQL][FOLLOWUP] Reference the config
 for the old behavior in error message

### What changes were proposed in this pull request?
Follow up work for SPARK-29864, reference the config  `spark.sql.legacy.fromDayTimeString.enabled` in error message.

### Why are the changes needed?
For better usability.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing tests.

Closes #27464 from xuanyuanking/SPARK-29864-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 4938905a1c047e367c066e39dce8232bfcff14f1)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/util/IntervalUtils.scala     |  9 +++++++--
 .../sql-tests/results/ansi/interval.sql.out   | 12 +++++------
 .../sql-tests/results/interval.sql.out        | 12 +++++------
 .../results/postgreSQL/interval.sql.out       | 20 +++++++++----------
 4 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 7692299a46ef5..2d98384363323 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -176,6 +176,9 @@ object IntervalUtils {
   private val dayTimePatternLegacy =
     "^([+|-])?((\\d+) )?((\\d+):)?(\\d+):(\\d+)(\\.(\\d+))?$".r
 
+  private val fallbackNotice = s"set ${SQLConf.LEGACY_FROM_DAYTIME_STRING.key} to true " +
+    "to restore the behavior before Spark 3.0."
+
   /**
    * Legacy method of parsing a string in a day-time format. It ignores the `from` bound,
    * and takes into account only the `to` bound by truncating the result. For example,
@@ -195,7 +198,8 @@ object IntervalUtils {
     require(input != null, "Interval day-time string must be not null")
     assert(input.length == input.trim.length)
     val m = dayTimePatternLegacy.pattern.matcher(input)
-    require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input")
+    require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input, " +
+      s"$fallbackNotice")
 
     try {
       val sign = if (m.group(1) != null && m.group(1) == "-") -1 else 1
@@ -296,7 +300,8 @@ object IntervalUtils {
     require(regexp.isDefined, s"Cannot support (interval '$input' $from to $to) expression")
     val pattern = regexp.get.pattern
     val m = pattern.matcher(input)
-    require(m.matches, s"Interval string must match day-time format of '$pattern': $input")
+    require(m.matches, s"Interval string must match day-time format of '$pattern': $input, " +
+      s"$fallbackNotice")
     var micros: Long = 0L
     var days: Int = 0
     unitsRange(to, from).foreach {
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index ab6130da869c4..f37049064d989 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -320,7 +320,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to hour
@@ -334,7 +334,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to minute
@@ -348,7 +348,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40:32.99899999' hour to minute
@@ -362,7 +362,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40.99899999' hour to second
@@ -376,7 +376,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40' hour to second
@@ -390,7 +390,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 40:32.99899999' minute to second
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 8f523a35f3c19..94b4f15815ca5 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -314,7 +314,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to hour
@@ -328,7 +328,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 20 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 15:40:32.99899999' day to minute
@@ -342,7 +342,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 15:40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40:32.99899999' hour to minute
@@ -356,7 +356,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40.99899999' hour to second
@@ -370,7 +370,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 15:40, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '15:40' hour to second
@@ -384,7 +384,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 20 40:32.99899999, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 select interval '20 40:32.99899999' minute to second
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out
index 4bd846d3ff923..62d47410aab65 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/interval.sql.out
@@ -105,7 +105,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' day to hour
@@ -119,7 +119,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' day to hour
@@ -141,7 +141,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' day to minute
@@ -155,7 +155,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' day to second
@@ -177,7 +177,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' hour to minute
@@ -191,7 +191,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2})$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' hour to minute
@@ -205,7 +205,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' hour to second
@@ -219,7 +219,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' hour to second
@@ -233,7 +233,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03' minute to second
@@ -247,7 +247,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04(line 1, pos 16)
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 1 2:03:04, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
 
 == SQL ==
 SELECT interval '1 2:03:04' minute to second

From d06a9dfd9098b704c75047161590bb4a32b25286 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 5 Feb 2020 12:36:51 -0800
Subject: [PATCH 0020/1280] [SPARK-30721][SQL][TESTS] Fix
 DataFrameAggregateSuite when enabling AQE

### What changes were proposed in this pull request?

update `DataFrameAggregateSuite` to make it pass with AQE

### Why are the changes needed?

We don't need to turn off AQE in `DataFrameAggregateSuite`

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

run `DataFrameAggregateSuite` locally with AQE on.

Closes #27451 from cloud-fan/aqe-test.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3b26f807a0eb0e59c5123c3f1e2262b712800c0f)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/DataFrameAggregateSuite.scala       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index dc1767a6852f6..d7df75fd0e2c3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -615,34 +615,33 @@ class DataFrameAggregateSuite extends QueryTest
          Seq((true, true), (true, false), (false, true), (false, false))) {
       withSQLConf(
         (SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, wholeStage.toString),
-        (SQLConf.USE_OBJECT_HASH_AGG.key, useObjectHashAgg.toString),
-        (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false")) {
-        // When enable AQE, the WholeStageCodegenExec is added during QueryStageExec.
+        (SQLConf.USE_OBJECT_HASH_AGG.key, useObjectHashAgg.toString)) {
 
         val df = Seq(("1", 1), ("1", 2), ("2", 3), ("2", 4)).toDF("x", "y")
 
         // test case for HashAggregate
         val hashAggDF = df.groupBy("x").agg(c, sum("y"))
+        hashAggDF.collect()
         val hashAggPlan = hashAggDF.queryExecution.executedPlan
         if (wholeStage) {
-          assert(hashAggPlan.find {
+          assert(find(hashAggPlan) {
             case WholeStageCodegenExec(_: HashAggregateExec) => true
             case _ => false
           }.isDefined)
         } else {
-          assert(hashAggPlan.isInstanceOf[HashAggregateExec])
+          assert(stripAQEPlan(hashAggPlan).isInstanceOf[HashAggregateExec])
         }
-        hashAggDF.collect()
 
         // test case for ObjectHashAggregate and SortAggregate
         val objHashAggOrSortAggDF = df.groupBy("x").agg(c, collect_list("y"))
-        val objHashAggOrSortAggPlan = objHashAggOrSortAggDF.queryExecution.executedPlan
+        objHashAggOrSortAggDF.collect()
+        val objHashAggOrSortAggPlan =
+          stripAQEPlan(objHashAggOrSortAggDF.queryExecution.executedPlan)
         if (useObjectHashAgg) {
           assert(objHashAggOrSortAggPlan.isInstanceOf[ObjectHashAggregateExec])
         } else {
           assert(objHashAggOrSortAggPlan.isInstanceOf[SortAggregateExec])
         }
-        objHashAggOrSortAggDF.collect()
       }
     }
   }

From da1caaede6aee381a74f9a19b7d850c459ba215d Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 6 Feb 2020 13:01:08 +0900
Subject: [PATCH 0021/1280] [SPARK-30737][SPARK-27262][R][BUILD] Reenable CRAN
 check with UTF-8 encoding to DESCRIPTION

### What changes were proposed in this pull request?

This PR proposes to reenable CRAN check disabled at https://github.com/apache/spark/pull/27460. Given the tests https://github.com/apache/spark/pull/27468, seems we should also port https://github.com/apache/spark/pull/23823 together.

### Why are the changes needed?
To check CRAN back.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
It was tested at https://github.com/apache/spark/pull/27468 and Jenkins should test it out.

Closes #27472 from HyukjinKwon/SPARK-30737.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b95ccb1d8b726b11435789cdb5882df6643430ed)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/DESCRIPTION                     | 1 +
 R/pkg/tests/fulltests/test_sparkSQL.R | 7 +------
 R/run-tests.sh                        | 7 +++----
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 95d3e52bef3a9..c8cb1c3a992ad 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -62,3 +62,4 @@ Collate:
 RoxygenNote: 5.0.1
 VignetteBuilder: knitr
 NeedsCompilation: no
+Encoding: UTF-8
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 23fadc4373c3f..c1d277ac84be1 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -496,12 +496,7 @@ test_that("SPARK-17811: can create DataFrame containing NA as date and time", {
   expect_true(is.na(DF$date[2]))
   expect_equal(DF$date[1], as.Date("2016-10-01"))
   expect_true(is.na(DF$time[2]))
-  # Warnings were suppressed due to Jenkins environment issues.
-  # It shows both warnings as below in Jenkins:
-  # - Your system is mis-configured: /etc/localtime is not a symlink
-  # - It is strongly recommended to set environment variable TZ to
-  #     America/Los_Angeles (or equivalent)
-  suppressWarnings(expect_equal(DF$time[1], as.POSIXlt("2016-01-10")))
+  expect_equal(DF$time[1], as.POSIXlt("2016-01-10"))
 })
 
 test_that("create DataFrame with complex types", {
diff --git a/R/run-tests.sh b/R/run-tests.sh
index 782b5f5baebaf..51ca7d600caf0 100755
--- a/R/run-tests.sh
+++ b/R/run-tests.sh
@@ -31,10 +31,9 @@ NUM_TEST_WARNING="$(grep -c -e 'Warnings ----------------' $LOGFILE)"
 # Also run the documentation tests for CRAN
 CRAN_CHECK_LOG_FILE=$FWDIR/cran-check.out
 rm -f $CRAN_CHECK_LOG_FILE
-# TODO(SPARK-30737) reenable this once packages are correctly installed in Jenkins
-# NO_TESTS=1 NO_MANUAL=1 $FWDIR/check-cran.sh 2>&1 | tee -a $CRAN_CHECK_LOG_FILE
-# FAILED=$((PIPESTATUS[0]||$FAILED))
-touch $CRAN_CHECK_LOG_FILE
+
+NO_TESTS=1 NO_MANUAL=1 $FWDIR/check-cran.sh 2>&1 | tee -a $CRAN_CHECK_LOG_FILE
+FAILED=$((PIPESTATUS[0]||$FAILED))
 
 NUM_CRAN_WARNING="$(grep -c WARNING$ $CRAN_CHECK_LOG_FILE)"
 NUM_CRAN_ERROR="$(grep -c ERROR$ $CRAN_CHECK_LOG_FILE)"

From baf1a07704a6404313ebf558652e26acf137c5b4 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 6 Feb 2020 12:48:27 +0800
Subject: [PATCH 0022/1280] [SPARK-30729][CORE] Eagerly filter out zombie
 TaskSetManager before offering resources

### What changes were proposed in this pull request?

Eagerly filter out zombie `TaskSetManager` before offering resources to reduce any overhead as possible.

And this PR also avoid doing `recomputeLocality` and `addPendingTask` when `TaskSetManager` is zombie.

### Why are the changes needed?

Zombie `TaskSetManager` could still exist in Pool's `schedulableQueue` when it has running tasks. Offering resources on a zombie `TaskSetManager` could bring unnecessary overhead and is meaningless.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #27455 from Ngone51/exclude-zombie-tsm.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit aebabf0bed712511eaa8844cab3a0c562219b2d0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala  | 2 +-
 .../scala/org/apache/spark/scheduler/TaskSetManager.scala     | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index f25a36c7af22a..6a1d460e6a9d9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -430,7 +430,7 @@ private[spark] class TaskSchedulerImpl(
     val tasks = shuffledOffers.map(o => new ArrayBuffer[TaskDescription](o.cores / CPUS_PER_TASK))
     val availableResources = shuffledOffers.map(_.resources).toArray
     val availableCpus = shuffledOffers.map(o => o.cores).toArray
-    val sortedTaskSets = rootPool.getSortedTaskSetQueue
+    val sortedTaskSets = rootPool.getSortedTaskSetQueue.filterNot(_.isZombie)
     for (taskSet <- sortedTaskSets) {
       logDebug("parentName: %s, name: %s, runningTasks: %s".format(
         taskSet.parent.name, taskSet.name, taskSet.runningTasks))
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
index 3b620ec69a9ab..2ce11347ade39 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSetManager.scala
@@ -229,6 +229,8 @@ private[spark] class TaskSetManager(
       index: Int,
       resolveRacks: Boolean = true,
       speculatable: Boolean = false): Unit = {
+    // A zombie TaskSetManager may reach here while handling failed task.
+    if (isZombie) return
     val pendingTaskSetToAddTo = if (speculatable) pendingSpeculatableTasks else pendingTasks
     for (loc <- tasks(index).preferredLocations) {
       loc match {
@@ -1082,6 +1084,8 @@ private[spark] class TaskSetManager(
   }
 
   def recomputeLocality(): Unit = {
+    // A zombie TaskSetManager may reach here while executorLost happens
+    if (isZombie) return
     val previousLocalityLevel = myLocalityLevels(currentLocalityIndex)
     myLocalityLevels = computeValidLocalityLevels()
     localityWaits = myLocalityLevels.map(getLocalityWait)

From fbb49664532135e86a7fb83d261ce1f500c041b9 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Thu, 6 Feb 2020 13:54:17 +0800
Subject: [PATCH 0023/1280] [SPARK-30612][SQL] Resolve qualified column name
 with v2 tables

### What changes were proposed in this pull request?

This PR fixes the issue where queries with qualified columns like `SELECT t.a FROM t` would fail to resolve for v2 tables.

This PR would allow qualified column names in query as following:
```SQL
SELECT testcat.ns1.ns2.tbl.foo FROM testcat.ns1.ns2.tbl
SELECT ns1.ns2.tbl.foo FROM testcat.ns1.ns2.tbl
SELECT ns2.tbl.foo FROM testcat.ns1.ns2.tbl
SELECT tbl.foo FROM testcat.ns1.ns2.tbl
```

### Why are the changes needed?

This is a bug because you cannot qualify column names in queries.

### Does this PR introduce any user-facing change?

Yes, now users can qualify column names for v2 tables.

### How was this patch tested?

Added new tests.

Closes #27391 from imback82/qualified_col.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c27a616450959a5e984d10bf93b12ac0ced6c94d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |   5 +-
 .../expressions/namedExpressions.scala        |   2 -
 .../sql/catalyst/expressions/package.scala    |  83 ++++++++++-
 .../spark/sql/catalyst/identifiers.scala      |  16 +-
 .../plans/logical/basicLogicalOperators.scala |  17 ++-
 .../spark/sql/catalyst/trees/TreeNode.scala   |   3 +-
 .../AttributeResolutionSuite.scala            | 137 ++++++++++++++++++
 .../sql/catalyst/trees/TreeNodeSuite.scala    |  10 +-
 .../sql-tests/results/group-by-filter.sql.out |  32 ++--
 .../invalid-correlation.sql.out               |   4 +-
 .../sql/connector/DataSourceV2SQLSuite.scala  |  38 +++++
 .../benchmark/TPCDSQueryBenchmark.scala       |   2 +-
 .../command/PlanResolutionSuite.scala         |  65 +++++----
 .../sql/hive/HiveMetastoreCatalogSuite.scala  |   2 +-
 14 files changed, 342 insertions(+), 74 deletions(-)
 create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 3fd5039a4f116..56cc2a274bb7a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -799,6 +799,7 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).resolveOperatorsUp {
       case u: UnresolvedRelation =>
         lookupV2Relation(u.multipartIdentifier)
+          .map(SubqueryAlias(u.multipartIdentifier, _))
           .getOrElse(u)
 
       case u @ UnresolvedTable(NonSessionCatalogAndIdentifier(catalog, ident)) =>
@@ -923,7 +924,9 @@ class Analyzer(
             case v1Table: V1Table =>
               v1SessionCatalog.getRelation(v1Table.v1Table)
             case table =>
-              DataSourceV2Relation.create(table, Some(catalog), Some(ident))
+              SubqueryAlias(
+                identifier,
+                DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
           }
           val key = catalog.name +: ident.namespace :+ ident.name
           Option(AnalysisContext.get.relationCache.getOrElseUpdate(key, loaded.orNull))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 3362353e2662a..02e90f8458c3d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -236,8 +236,6 @@ case class AttributeReference(
     val qualifier: Seq[String] = Seq.empty[String])
   extends Attribute with Unevaluable {
 
-  // currently can only handle qualifier of length 2
-  require(qualifier.length <= 2)
   /**
    * Returns true iff the expression id is the same for both attributes.
    */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
index 7164b6b82adbc..9f42e643e4cb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
@@ -23,7 +23,6 @@ import com.google.common.collect.Maps
 
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute}
-import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.types.{StructField, StructType}
 
 /**
@@ -153,13 +152,19 @@ package object expressions  {
       unique(grouped)
     }
 
-    /** Perform attribute resolution given a name and a resolver. */
-    def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = {
+    /** Returns true if all qualifiers in `attrs` have 2 or less parts. */
+    @transient private val hasTwoOrLessQualifierParts: Boolean =
+      attrs.forall(_.qualifier.length <= 2)
+
+    /** Match attributes for the case where all qualifiers in `attrs` have 2 or less parts. */
+    private def matchWithTwoOrLessQualifierParts(
+        nameParts: Seq[String],
+        resolver: Resolver): (Seq[Attribute], Seq[String]) = {
       // Collect matching attributes given a name and a lookup.
       def collectMatches(name: String, candidates: Option[Seq[Attribute]]): Seq[Attribute] = {
-        candidates.toSeq.flatMap(_.collect {
+        candidates.getOrElse(Nil).collect {
           case a if resolver(a.name, name) => a.withName(name)
-        })
+        }
       }
 
       // Find matches for the given name assuming that the 1st two parts are qualifier
@@ -204,13 +209,79 @@ package object expressions  {
 
       // If none of attributes match database.table.column pattern or
       // `table.column` pattern, we try to resolve it as a column.
-      val (candidates, nestedFields) = matches match {
+      matches match {
         case (Seq(), _) =>
           val name = nameParts.head
           val attributes = collectMatches(name, direct.get(name.toLowerCase(Locale.ROOT)))
           (attributes, nameParts.tail)
         case _ => matches
       }
+    }
+
+    /**
+     * Match attributes for the case where at least one qualifier in `attrs` has more than 2 parts.
+     */
+    private def matchWithThreeOrMoreQualifierParts(
+        nameParts: Seq[String],
+        resolver: Resolver): (Seq[Attribute], Seq[String]) = {
+      // Returns true if the `short` qualifier is a subset of the last elements of
+      // `long` qualifier. For example, Seq("a", "b") is a subset of Seq("a", "a", "b"),
+      // but not a subset of Seq("a", "b", "b").
+      def matchQualifier(short: Seq[String], long: Seq[String]): Boolean = {
+        (long.length >= short.length) &&
+          long.takeRight(short.length)
+            .zip(short)
+            .forall(x => resolver(x._1, x._2))
+      }
+
+      // Collect attributes that match the given name and qualifier.
+      // A match occurs if
+      //   1) the given name matches the attribute's name according to the resolver.
+      //   2) the given qualifier is a subset of the attribute's qualifier.
+      def collectMatches(
+          name: String,
+          qualifier: Seq[String],
+          candidates: Option[Seq[Attribute]]): Seq[Attribute] = {
+        candidates.getOrElse(Nil).collect {
+          case a if resolver(name, a.name) && matchQualifier(qualifier, a.qualifier) =>
+            a.withName(name)
+        }
+      }
+
+      // Iterate each string in `nameParts` in a reverse order and try to match the attributes
+      // considering the current string as the attribute name. For example, if `nameParts` is
+      // Seq("a", "b", "c"), the match will be performed in the following order:
+      // 1) name = "c", qualifier = Seq("a", "b")
+      // 2) name = "b", qualifier = Seq("a")
+      // 3) name = "a", qualifier = Seq()
+      // Note that the match is performed in the reverse order in order to match the longest
+      // qualifier as possible. If a match is found, the remaining portion of `nameParts`
+      // is also returned as nested fields.
+      var candidates: Seq[Attribute] = Nil
+      var nestedFields: Seq[String] = Nil
+      var i = nameParts.length - 1
+      while (i >= 0 && candidates.isEmpty) {
+        val name = nameParts(i)
+        candidates = collectMatches(
+          name,
+          nameParts.take(i),
+          direct.get(name.toLowerCase(Locale.ROOT)))
+        if (candidates.nonEmpty) {
+          nestedFields = nameParts.takeRight(nameParts.length - i - 1)
+        }
+        i -= 1
+      }
+
+      (candidates, nestedFields)
+    }
+
+    /** Perform attribute resolution given a name and a resolver. */
+    def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = {
+      val (candidates, nestedFields) = if (hasTwoOrLessQualifierParts) {
+        matchWithTwoOrLessQualifierParts(nameParts, resolver)
+      } else {
+        matchWithThreeOrMoreQualifierParts(nameParts, resolver)
+      }
 
       def name = UnresolvedAttribute(nameParts).name
       candidates match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
index deceec73dda30..c574a20da0b5b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/identifiers.scala
@@ -49,19 +49,21 @@ sealed trait IdentifierWithDatabase {
 
 /**
  * Encapsulates an identifier that is either a alias name or an identifier that has table
- * name and optionally a database name.
+ * name and a qualifier.
  * The SubqueryAlias node keeps track of the qualifier using the information in this structure
- * @param identifier - Is an alias name or a table name
- * @param database - Is a database name and is optional
+ * @param name - Is an alias name or a table name
+ * @param qualifier - Is a qualifier
  */
-case class AliasIdentifier(identifier: String, database: Option[String])
-  extends IdentifierWithDatabase {
+case class AliasIdentifier(name: String, qualifier: Seq[String]) {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
+
+  def this(identifier: String) = this(identifier, Seq())
 
-  def this(identifier: String) = this(identifier, None)
+  override def toString: String = (qualifier :+ name).quoted
 }
 
 object AliasIdentifier {
-  def apply(identifier: String): AliasIdentifier = new AliasIdentifier(identifier)
+  def apply(name: String): AliasIdentifier = new AliasIdentifier(name)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 40db8b6f49dc4..54e5ff7aeb754 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression,
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partitioning, RangePartitioning, RoundRobinPartitioning}
 import org.apache.spark.sql.catalyst.util.truncatedString
+import org.apache.spark.sql.connector.catalog.Identifier
 import org.apache.spark.sql.types._
 import org.apache.spark.util.random.RandomSampler
 
@@ -849,18 +850,18 @@ case class Tail(limitExpr: Expression, child: LogicalPlan) extends OrderPreservi
 /**
  * Aliased subquery.
  *
- * @param name the alias identifier for this subquery.
+ * @param identifier the alias identifier for this subquery.
  * @param child the logical plan of this subquery.
  */
 case class SubqueryAlias(
-    name: AliasIdentifier,
+    identifier: AliasIdentifier,
     child: LogicalPlan)
   extends OrderPreservingUnaryNode {
 
-  def alias: String = name.identifier
+  def alias: String = identifier.name
 
   override def output: Seq[Attribute] = {
-    val qualifierList = name.database.map(Seq(_, alias)).getOrElse(Seq(alias))
+    val qualifierList = identifier.qualifier :+ alias
     child.output.map(_.withQualifier(qualifierList))
   }
   override def doCanonicalize(): LogicalPlan = child.canonicalized
@@ -877,7 +878,13 @@ object SubqueryAlias {
       identifier: String,
       database: String,
       child: LogicalPlan): SubqueryAlias = {
-    SubqueryAlias(AliasIdentifier(identifier, Some(database)), child)
+    SubqueryAlias(AliasIdentifier(identifier, Seq(database)), child)
+  }
+
+  def apply(
+      multipartIdentifier: Seq[String],
+      child: LogicalPlan): SubqueryAlias = {
+    SubqueryAlias(AliasIdentifier(multipartIdentifier.last, multipartIdentifier.init), child)
   }
 }
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index ba1eeb38e247e..56a198763b4e5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -27,7 +27,7 @@ import org.json4s.JsonAST._
 import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
-import org.apache.spark.sql.catalyst.IdentifierWithDatabase
+import org.apache.spark.sql.catalyst.{AliasIdentifier, IdentifierWithDatabase}
 import org.apache.spark.sql.catalyst.ScalaReflection._
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, FunctionResource}
 import org.apache.spark.sql.catalyst.errors._
@@ -780,6 +780,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     case exprId: ExprId => true
     case field: StructField => true
     case id: IdentifierWithDatabase => true
+    case alias: AliasIdentifier => true
     case join: JoinType => true
     case spec: BucketSpec => true
     case catalog: CatalogTable => true
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
new file mode 100644
index 0000000000000..813a68f68451c
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/AttributeResolutionSuite.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution}
+import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+
+class AttributeResolutionSuite extends SparkFunSuite {
+  val resolver = caseInsensitiveResolution
+
+  test("basic attribute resolution with namespaces") {
+    val attrs = Seq(
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "ns2", "t1")),
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "ns2", "ns3", "t2")))
+
+    // Try to match attribute reference with name "a" with qualifier "ns1.ns2.t1".
+    Seq(Seq("t1", "a"), Seq("ns2", "t1", "a"), Seq("ns1", "ns2", "t1", "a")).foreach { nameParts =>
+      attrs.resolve(nameParts, resolver) match {
+        case Some(attr) => assert(attr.semanticEquals(attrs(0)))
+        case _ => fail()
+      }
+    }
+
+    // Non-matching cases
+    Seq(Seq("ns1", "ns2", "t1"), Seq("ns2", "a")).foreach { nameParts =>
+      assert(attrs.resolve(nameParts, resolver).isEmpty)
+    }
+  }
+
+  test("attribute resolution where table and attribute names are the same") {
+    val attrs = Seq(AttributeReference("t", IntegerType)(qualifier = Seq("ns1", "ns2", "t")))
+    // Matching cases
+    Seq(
+      Seq("t"), Seq("t", "t"), Seq("ns2", "t", "t"), Seq("ns1", "ns2", "t", "t")
+    ).foreach { nameParts =>
+      attrs.resolve(nameParts, resolver) match {
+        case Some(attr) => assert(attr.semanticEquals(attrs(0)))
+        case _ => fail()
+      }
+    }
+
+    // Non-matching case
+    assert(attrs.resolve(Seq("ns1", "ns2", "t"), resolver).isEmpty)
+  }
+
+  test("attribute resolution ambiguity at the attribute name level") {
+    val attrs = Seq(
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t1")),
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "ns2", "t2")))
+
+    val ex = intercept[AnalysisException] {
+      attrs.resolve(Seq("a"), resolver)
+    }
+    assert(ex.getMessage.contains(
+      "Reference 'a' is ambiguous, could be: ns1.t1.a, ns1.ns2.t2.a."))
+  }
+
+  test("attribute resolution ambiguity at the qualifier level") {
+    val attrs = Seq(
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t")),
+      AttributeReference("a", IntegerType)(qualifier = Seq("ns2", "ns1", "t")))
+
+    val ex = intercept[AnalysisException] {
+      attrs.resolve(Seq("ns1", "t", "a"), resolver)
+    }
+    assert(ex.getMessage.contains(
+      "Reference 'ns1.t.a' is ambiguous, could be: ns1.t.a, ns2.ns1.t.a."))
+  }
+
+  test("attribute resolution with nested fields") {
+    val attrType = StructType(Seq(StructField("aa", IntegerType), StructField("bb", IntegerType)))
+    val attrs = Seq(AttributeReference("a", attrType)(qualifier = Seq("ns1", "t")))
+
+    val resolved = attrs.resolve(Seq("ns1", "t", "a", "aa"), resolver)
+    resolved match {
+      case Some(Alias(_, name)) => assert(name == "aa")
+      case _ => fail()
+    }
+
+    val ex = intercept[AnalysisException] {
+      attrs.resolve(Seq("ns1", "t", "a", "cc"), resolver)
+    }
+    assert(ex.getMessage.contains("No such struct field cc in aa, bb"))
+  }
+
+  test("attribute resolution with case insensitive resolver") {
+    val attrs = Seq(AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t")))
+    attrs.resolve(Seq("Ns1", "T", "A"), caseInsensitiveResolution) match {
+      case Some(attr) => assert(attr.semanticEquals(attrs(0)) && attr.name == "A")
+      case _ => fail()
+    }
+  }
+
+  test("attribute resolution with case sensitive resolver") {
+    val attrs = Seq(AttributeReference("a", IntegerType)(qualifier = Seq("ns1", "t")))
+    assert(attrs.resolve(Seq("Ns1", "T", "A"), caseSensitiveResolution).isEmpty)
+    assert(attrs.resolve(Seq("ns1", "t", "A"), caseSensitiveResolution).isEmpty)
+    attrs.resolve(Seq("ns1", "t", "a"), caseSensitiveResolution) match {
+      case Some(attr) => assert(attr.semanticEquals(attrs(0)))
+      case _ => fail()
+    }
+  }
+
+  test("attribute resolution should try to match the longest qualifier") {
+    // We have two attributes:
+    // 1) "a.b" where "a" is the name and "b" is the nested field.
+    // 2) "a.b.a" where "b" is the name, left-side "a" is the qualifier and the right-side "a"
+    //    is the nested field.
+    // When "a.b" is resolved, "b" is tried first as the name, so it is resolved to #2 attribute.
+    val a1Type = StructType(Seq(StructField("b", IntegerType)))
+    val a2Type = StructType(Seq(StructField("a", IntegerType)))
+    val attrs = Seq(
+      AttributeReference("a", a1Type)(),
+      AttributeReference("b", a2Type)(qualifier = Seq("a")))
+    attrs.resolve(Seq("a", "b"), resolver) match {
+      case Some(attr) => assert(attr.semanticEquals(attrs(1)))
+      case _ => fail()
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index 0e094bc06b05f..e72b2e9b1b214 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -433,10 +433,11 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
 
     // Converts AliasIdentifier to JSON
     assertJSON(
-      AliasIdentifier("alias"),
+      AliasIdentifier("alias", Seq("ns1", "ns2")),
       JObject(
         "product-class" -> JString(classOf[AliasIdentifier].getName),
-          "identifier" -> "alias"))
+          "name" -> "alias",
+          "qualifier" -> "[ns1, ns2]"))
 
     // Converts SubqueryAlias to JSON
     assertJSON(
@@ -445,8 +446,9 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
         JObject(
           "class" -> classOf[SubqueryAlias].getName,
           "num-children" -> 1,
-          "name" -> JObject("product-class" -> JString(classOf[AliasIdentifier].getName),
-            "identifier" -> "t1"),
+          "identifier" -> JObject("product-class" -> JString(classOf[AliasIdentifier].getName),
+            "name" -> "t1",
+            "qualifier" -> JArray(Nil)),
           "child" -> 0),
         JObject(
           "class" -> classOf[JsonTestTreeNode].getName,
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
index a032678e90fe8..a4c7c2cf90cd7 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
@@ -369,13 +369,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, avg(salary#x) AS avg(salary)#x, avg(salary#x) FILTER (WHERE exists#x [dept_id#x]) AS avg(salary) FILTER (WHERE exists(dept_id))#x]
 :  +- Project [state#x]
 :     +- Filter (dept_id#x = outer(dept_id#x))
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
@@ -395,13 +395,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, sum(salary#x) AS sum(salary)#x, sum(salary#x) FILTER (WHERE NOT exists#x [dept_id#x]) AS sum(salary) FILTER (WHERE (NOT exists(dept_id)))#x]
 :  +- Project [state#x]
 :     +- Filter (dept_id#x = outer(dept_id#x))
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
@@ -420,13 +420,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, avg(salary#x) AS avg(salary)#x, avg(salary#x) FILTER (WHERE dept_id#x IN (list#x [])) AS avg(salary) FILTER (WHERE (dept_id IN (listquery())))#x]
 :  +- Distinct
 :     +- Project [dept_id#x]
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
@@ -445,13 +445,13 @@ org.apache.spark.sql.AnalysisException
 IN/EXISTS predicate sub-queries can only be used in Filter/Join and a few commands: Aggregate [dept_id#x], [dept_id#x, sum(salary#x) AS sum(salary)#x, sum(salary#x) FILTER (WHERE NOT dept_id#x IN (list#x [])) AS sum(salary) FILTER (WHERE (NOT (dept_id IN (listquery()))))#x]
 :  +- Distinct
 :     +- Project [dept_id#x]
-:        +- SubqueryAlias `dept`
+:        +- SubqueryAlias dept
 :           +- Project [dept_id#x, dept_name#x, state#x]
-:              +- SubqueryAlias `DEPT`
+:              +- SubqueryAlias DEPT
 :                 +- LocalRelation [dept_id#x, dept_name#x, state#x]
-+- SubqueryAlias `emp`
++- SubqueryAlias emp
    +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
-      +- SubqueryAlias `EMP`
+      +- SubqueryAlias EMP
          +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x]
 ;
 
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
index 1599634ff9efb..ec7ecf28754ef 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
@@ -110,8 +110,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 Expressions referencing the outer query are not supported outside of WHERE/HAVING clauses:
 Aggregate [min(outer(t2a#x)) AS min(outer())#x]
-+- SubqueryAlias `t3`
++- SubqueryAlias t3
    +- Project [t3a#x, t3b#x, t3c#x]
-      +- SubqueryAlias `t3`
+      +- SubqueryAlias t3
          +- LocalRelation [t3a#x, t3b#x, t3c#x]
 ;
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 2c8349a0e6a75..eabcb81c50646 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -679,6 +679,44 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("qualified column names for v2 tables") {
+    val t = "testcat.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint, point struct<x: bigint, y: bigint>) USING foo")
+      sql(s"INSERT INTO $t VALUES (1, (10, 20))")
+
+      checkAnswer(
+        sql(s"SELECT testcat.ns1.ns2.tbl.id, testcat.ns1.ns2.tbl.point.x FROM $t"),
+        Row(1, 10))
+      checkAnswer(sql(s"SELECT ns1.ns2.tbl.id, ns1.ns2.tbl.point.x FROM $t"), Row(1, 10))
+      checkAnswer(sql(s"SELECT ns2.tbl.id, ns2.tbl.point.x FROM $t"), Row(1, 10))
+      checkAnswer(sql(s"SELECT tbl.id, tbl.point.x FROM $t"), Row(1, 10))
+
+      val ex = intercept[AnalysisException] {
+        sql(s"SELECT ns1.ns2.ns3.tbl.id from $t")
+      }
+      assert(ex.getMessage.contains("cannot resolve '`ns1.ns2.ns3.tbl.id`"))
+    }
+  }
+
+  test("qualified column names for v1 tables") {
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+
+    withTable("t") {
+      sql("CREATE TABLE t USING json AS SELECT 1 AS i")
+      checkAnswer(sql("select default.t.i from spark_catalog.t"), Row(1))
+      checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
+      checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
+
+      // catalog name cannot be used for v1 tables.
+      val ex = intercept[AnalysisException] {
+        sql(s"select spark_catalog.default.t.i from spark_catalog.default.t")
+      }
+      assert(ex.getMessage.contains("cannot resolve '`spark_catalog.default.t.i`"))
+    }
+  }
+
   test("InsertInto: append - across catalog") {
     val t1 = "testcat.ns1.ns2.tbl"
     val t2 = "testcat2.db.tbl"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
index c93d27f02c686..ad3d79760adf0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
@@ -81,7 +81,7 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       val queryRelations = scala.collection.mutable.HashSet[String]()
       spark.sql(queryString).queryExecution.analyzed.foreach {
         case SubqueryAlias(alias, _: LogicalRelation) =>
-          queryRelations.add(alias.identifier)
+          queryRelations.add(alias.name)
         case LogicalRelation(_, _, Some(catalogTable), _) =>
           queryRelations.add(catalogTable.identifier.table)
         case HiveTableRelation(tableMeta, _, _, _, _) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index c0c3cd70fcc9e..88f30353cce94 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -885,33 +885,34 @@ class PlanResolutionSuite extends AnalysisTest {
       val parsed4 = parseAndResolve(sql4)
 
       parsed1 match {
-        case DeleteFromTable(_: DataSourceV2Relation, None) =>
-        case _ => fail("Expect DeleteFromTable, bug got:\n" + parsed1.treeString)
+        case DeleteFromTable(AsDataSourceV2Relation(_), None) =>
+        case _ => fail("Expect DeleteFromTable, but got:\n" + parsed1.treeString)
       }
 
       parsed2 match {
         case DeleteFromTable(
-          _: DataSourceV2Relation,
+          AsDataSourceV2Relation(_),
           Some(EqualTo(name: UnresolvedAttribute, StringLiteral("Robert")))) =>
           assert(name.name == "name")
-        case _ => fail("Expect DeleteFromTable, bug got:\n" + parsed2.treeString)
+        case _ => fail("Expect DeleteFromTable, but got:\n" + parsed2.treeString)
       }
 
       parsed3 match {
         case DeleteFromTable(
-          SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+          SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
           Some(EqualTo(name: UnresolvedAttribute, StringLiteral("Robert")))) =>
           assert(name.name == "t.name")
-        case _ => fail("Expect DeleteFromTable, bug got:\n" + parsed3.treeString)
+        case _ => fail("Expect DeleteFromTable, but got:\n" + parsed3.treeString)
       }
 
       parsed4 match {
-        case DeleteFromTable(SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+        case DeleteFromTable(
+            SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
             Some(InSubquery(values, query))) =>
           assert(values.size == 1 && values.head.isInstanceOf[UnresolvedAttribute])
           assert(values.head.asInstanceOf[UnresolvedAttribute].name == "t.name")
           query match {
-            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", None),
+            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", Seq()),
                 UnresolvedSubqueryColumnAliases(outputColumnNames, Project(_, _: OneRowRelation)))),
                 _, _, _) =>
               assert(projects.size == 1 && projects.head.name == "s.name")
@@ -944,7 +945,7 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parsed1 match {
         case UpdateTable(
-            _: DataSourceV2Relation,
+            AsDataSourceV2Relation(_),
             Seq(Assignment(name: UnresolvedAttribute, StringLiteral("Robert")),
               Assignment(age: UnresolvedAttribute, IntegerLiteral(32))),
             None) =>
@@ -956,7 +957,7 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parsed2 match {
         case UpdateTable(
-            SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+            SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
             Seq(Assignment(name: UnresolvedAttribute, StringLiteral("Robert")),
               Assignment(age: UnresolvedAttribute, IntegerLiteral(32))),
             None) =>
@@ -968,7 +969,7 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parsed3 match {
         case UpdateTable(
-            SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+            SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
             Seq(Assignment(name: UnresolvedAttribute, StringLiteral("Robert")),
               Assignment(age: UnresolvedAttribute, IntegerLiteral(32))),
             Some(EqualTo(p: UnresolvedAttribute, IntegerLiteral(1)))) =>
@@ -980,14 +981,14 @@ class PlanResolutionSuite extends AnalysisTest {
       }
 
       parsed4 match {
-        case UpdateTable(SubqueryAlias(AliasIdentifier("t", None), _: DataSourceV2Relation),
+        case UpdateTable(SubqueryAlias(AliasIdentifier("t", Seq()), AsDataSourceV2Relation(_)),
           Seq(Assignment(key: UnresolvedAttribute, IntegerLiteral(32))),
           Some(InSubquery(values, query))) =>
           assert(key.name == "t.age")
           assert(values.size == 1 && values.head.isInstanceOf[UnresolvedAttribute])
           assert(values.head.asInstanceOf[UnresolvedAttribute].name == "t.name")
           query match {
-            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", None),
+            case ListQuery(Project(projects, SubqueryAlias(AliasIdentifier("s", Seq()),
                 UnresolvedSubqueryColumnAliases(outputColumnNames, Project(_, _: OneRowRelation)))),
                 _, _, _) =>
               assert(projects.size == 1 && projects.head.name == "s.name")
@@ -1129,7 +1130,7 @@ class PlanResolutionSuite extends AnalysisTest {
         case AlterTable(_, _, r: DataSourceV2Relation, _) =>
           assert(r.catalog.exists(_ == catlogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
-        case Project(_, r: DataSourceV2Relation) =>
+        case Project(_, AsDataSourceV2Relation(r)) =>
           assert(r.catalog.exists(_ == catlogIdent))
           assert(r.identifier.exists(_.name() == tableIdent))
         case InsertIntoStatement(r: DataSourceV2Relation, _, _, _, _) =>
@@ -1206,8 +1207,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql1) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: DataSourceV2Relation),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference, StringLiteral("update"))),
@@ -1232,8 +1233,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql2) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: DataSourceV2Relation),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference,
@@ -1258,8 +1259,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql3) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: DataSourceV2Relation),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
               Seq(DeleteAction(None), UpdateAction(None, updateAssigns)),
               Seq(InsertAction(None, insertAssigns))) =>
@@ -1282,8 +1283,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql4) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: Project),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), source: Project),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference, StringLiteral("update"))),
@@ -1311,8 +1312,8 @@ class PlanResolutionSuite extends AnalysisTest {
            """.stripMargin
         parseAndResolve(sql5) match {
           case MergeIntoTable(
-              SubqueryAlias(AliasIdentifier("target", None), target: DataSourceV2Relation),
-              SubqueryAlias(AliasIdentifier("source", None), source: Project),
+              SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
+              SubqueryAlias(AliasIdentifier("source", Seq()), source: Project),
               mergeCondition,
               Seq(DeleteAction(Some(EqualTo(dl: AttributeReference, StringLiteral("delete")))),
                 UpdateAction(Some(EqualTo(ul: AttributeReference, StringLiteral("update"))),
@@ -1346,8 +1347,8 @@ class PlanResolutionSuite extends AnalysisTest {
 
       parseAndResolve(sql1) match {
         case MergeIntoTable(
-            target: DataSourceV2Relation,
-            source: DataSourceV2Relation,
+            AsDataSourceV2Relation(target),
+            AsDataSourceV2Relation(source),
             _,
             Seq(DeleteAction(None), UpdateAction(None, updateAssigns)),
             Seq(InsertAction(
@@ -1453,8 +1454,8 @@ class PlanResolutionSuite extends AnalysisTest {
 
     parseAndResolve(sql) match {
       case MergeIntoTable(
-          SubqueryAlias(AliasIdentifier("target", None), _: DataSourceV2Relation),
-          SubqueryAlias(AliasIdentifier("source", None), _: DataSourceV2Relation),
+          SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(_)),
+          SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(_)),
           EqualTo(l: UnresolvedAttribute, r: UnresolvedAttribute),
           Seq(
             DeleteAction(Some(EqualTo(dl: UnresolvedAttribute, StringLiteral("delete")))),
@@ -1481,3 +1482,11 @@ class PlanResolutionSuite extends AnalysisTest {
   }
   // TODO: add tests for more commands.
 }
+
+object AsDataSourceV2Relation {
+  def unapply(plan: LogicalPlan): Option[DataSourceV2Relation] = plan match {
+    case SubqueryAlias(_, r: DataSourceV2Relation) => Some(r)
+    case _ => None
+  }
+}
+
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 20bafd832d0da..b8ef44b096eed 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -62,7 +62,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
       spark.sql("create view vw1 as select 1 as id")
       val plan = spark.sql("select id from vw1").queryExecution.analyzed
       val aliases = plan.collect {
-        case x @ SubqueryAlias(AliasIdentifier("vw1", Some("default")), _) => x
+        case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("default")), _) => x
       }
       assert(aliases.size == 1)
     }

From 706c21763e94957efe4a309abd6e13601b8dcaf3 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 6 Feb 2020 15:24:26 +0900
Subject: [PATCH 0024/1280] [SPARK-29108][SQL][TESTS][FOLLOWUP] Comment out no
 use test case and add 'insert into' statement of window.sql (Part 2)

### What changes were proposed in this pull request?
When I running the `window_part2.sql` tests find it lack insert sql. Therefore, the output is empty.
I checked the postgresql and reference https://github.com/postgres/postgres/blob/master/src/test/regress/sql/window.sql
Although `window_part1.sql` and `window_part3.sql` exists the insert sql, I think should also add it into `window_part2.sql`.
Because only one case reference the table `empsalary` and it throws `AnalysisException`.
```
-- !query
select last(salary) over(order by salary range between 1000 preceding and 1000 following),
lag(salary) over(order by salary range between 1000 preceding and 1000 following),
salary from empsalary
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
Window Frame specifiedwindowframe(RangeFrame, -1000, 1000) must match the required frame specifiedwindowframe(RowFrame, -1, -1);
```

So we should do four work:
1. comment out the only one case and create a new ticket.
2. Add `INSERT INTO empsalary`.

Note: window_part4.sql not use the table `empsalary`.

### Why are the changes needed?
Supplementary test data.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
New test case

Closes #27439 from beliefer/add-insert-to-window.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../inputs/postgreSQL/window_part2.sql        | 19 ++++++++++--
 .../results/postgreSQL/window_part2.sql.out   | 29 ++++++++++++-------
 2 files changed, 34 insertions(+), 14 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
index 395149e48d5c8..ba1acc9f56b4a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part2.sql
@@ -15,6 +15,18 @@ CREATE TABLE empsalary (
     enroll_date date
 ) USING parquet;
 
+INSERT INTO empsalary VALUES
+  ('develop', 10, 5200, date '2007-08-01'),
+  ('sales', 1, 5000, date '2006-10-01'),
+  ('personnel', 5, 3500, date '2007-12-10'),
+  ('sales', 4, 4800, date '2007-08-08'),
+  ('personnel', 2, 3900, date '2006-12-23'),
+  ('develop', 7, 4200, date '2008-01-01'),
+  ('develop', 9, 4500, date '2008-01-01'),
+  ('sales', 3, 4800, date '2007-08-01'),
+  ('develop', 8, 6000, date '2006-10-01'),
+  ('develop', 11, 5200, date '2007-08-15');
+
 -- [SPARK-28429] SQL Datetime util function being casted to double instead of timestamp
 -- CREATE TEMP VIEW v_window AS
 -- SELECT i, min(i) over (order by i range between '1 day' preceding and '10 days' following) as min_i
@@ -99,9 +111,10 @@ FROM tenk1 WHERE unique1 < 10;
 -- nth_value(salary, 1) over(order by salary range between 1000 preceding and 1000 following),
 -- salary from empsalary;
 
-select last(salary) over(order by salary range between 1000 preceding and 1000 following),
-lag(salary) over(order by salary range between 1000 preceding and 1000 following),
-salary from empsalary;
+-- [SPARK-30734] AnalysisException that window RangeFrame not match RowFrame
+-- select last(salary) over(order by salary range between 1000 preceding and 1000 following),
+-- lag(salary) over(order by salary range between 1000 preceding and 1000 following),
+-- salary from empsalary;
 
 -- [SPARK-27951] ANSI SQL: NTH_VALUE function
 -- select first_value(salary) over(order by salary range between 1000 following and 3000 following
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
index 0015740a0638e..f41659a196ae1 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
@@ -15,6 +15,24 @@ struct<>
 
 
 
+-- !query
+INSERT INTO empsalary VALUES
+  ('develop', 10, 5200, date '2007-08-01'),
+  ('sales', 1, 5000, date '2006-10-01'),
+  ('personnel', 5, 3500, date '2007-12-10'),
+  ('sales', 4, 4800, date '2007-08-08'),
+  ('personnel', 2, 3900, date '2006-12-23'),
+  ('develop', 7, 4200, date '2008-01-01'),
+  ('develop', 9, 4500, date '2008-01-01'),
+  ('sales', 3, 4800, date '2007-08-01'),
+  ('develop', 8, 6000, date '2006-10-01'),
+  ('develop', 11, 5200, date '2007-08-15')
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 SELECT sum(unique1) over (order by four range between 2 preceding and 1 preceding),
 unique1, four
@@ -72,17 +90,6 @@ struct<sum(unique1) OVER (PARTITION BY four ORDER BY unique1 ASC NULLS FIRST RAN
 8	6	2
 
 
--- !query
-select last(salary) over(order by salary range between 1000 preceding and 1000 following),
-lag(salary) over(order by salary range between 1000 preceding and 1000 following),
-salary from empsalary
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Window Frame specifiedwindowframe(RangeFrame, -1000, 1000) must match the required frame specifiedwindowframe(RowFrame, -1, -1);
-
-
 -- !query
 select ss.id, ss.y,
        first(ss.y) over w,

From 659f8c8ef549fd040596978478898266d24e88ff Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 6 Feb 2020 20:34:29 +0800
Subject: [PATCH 0025/1280] [SPARK-27297][DOC][FOLLOW-UP] Improve documentation
 for various Scala functions

### What changes were proposed in this pull request?

Add examples and parameter description for these Scala functions:

* transform
* exists
* forall
* aggregate
* zip_with
* transform_keys
* transform_values
* map_filter
* map_zip_with

### Why are the changes needed?

Better documentation for UX.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #27449 from Ngone51/doc-funcs.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 368ee62a5dce83682ccaec92feeea8428af5a8cf)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/functions.scala      | 93 +++++++++++++++++--
 1 file changed, 83 insertions(+), 10 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index da26c5a2f4625..d125581857e0b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -3410,6 +3410,12 @@ object functions {
   /**
    * Returns an array of elements after applying a transformation to each element
    * in the input array.
+   * {{{
+   *   df.select(transform(col("i"), x => x + 1))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f col => transformed_col, the lambda function to transform the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3421,6 +3427,13 @@ object functions {
   /**
    * Returns an array of elements after applying a transformation to each element
    * in the input array.
+   * {{{
+   *   df.select(transform(col("i"), (x, i) => x + i))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f (col, index) => transformed_col, the lambda function to filter the input column
+   *           given the index. Indices start at 0.
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3431,6 +3444,12 @@ object functions {
 
   /**
    * Returns whether a predicate holds for one or more elements in the array.
+   * {{{
+   *   df.select(exists(col("i"), _ % 2 === 0))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f col => predicate, the Boolean predicate to check the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3441,6 +3460,12 @@ object functions {
 
   /**
    * Returns whether a predicate holds for every element in the array.
+   * {{{
+   *   df.select(forall(col("i"), x => x % 2 === 0))
+   * }}}
+   *
+   * @param column the input array column
+   * @param f col => predicate, the Boolean predicate to check the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3453,11 +3478,10 @@ object functions {
    * Returns an array of elements for which a predicate holds in a given array.
    * {{{
    *   df.select(filter(col("s"), x => x % 2 === 0))
-   *   df.selectExpr("filter(col, x -> x % 2 == 0)")
    * }}}
    *
-   * @param column: the input array column
-   * @param f: col => predicate, the Boolean predicate to filter the input column
+   * @param column the input array column
+   * @param f col => predicate, the Boolean predicate to filter the input column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3470,11 +3494,10 @@ object functions {
    * Returns an array of elements for which a predicate holds in a given array.
    * {{{
    *   df.select(filter(col("s"), (x, i) => i % 2 === 0))
-   *   df.selectExpr("filter(col, (x, i) -> i % 2 == 0)")
    * }}}
    *
-   * @param column: the input array column
-   * @param f: (col, index) => predicate, the Boolean predicate to filter the input column
+   * @param column the input array column
+   * @param f (col, index) => predicate, the Boolean predicate to filter the input column
    *           given the index. Indices start at 0.
    *
    * @group collection_funcs
@@ -3488,18 +3511,28 @@ object functions {
    * Applies a binary operator to an initial state and all elements in the array,
    * and reduces this to a single state. The final state is converted into the final result
    * by applying a finish function.
+   * {{{
+   *   df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10))
+   * }}}
+   *
+   * @param expr the input array column
+   * @param initialValue the initial value
+   * @param merge (combined_value, input_value) => combined_value, the merge function to merge
+   *              an input value to the combined_value
+   * @param finish combined_value => final_value, the lambda function to convert the combined value
+   *               of all inputs to final result
    *
    * @group collection_funcs
    * @since 3.0.0
    */
   def aggregate(
       expr: Column,
-      zero: Column,
+      initialValue: Column,
       merge: (Column, Column) => Column,
       finish: Column => Column): Column = withExpr {
     ArrayAggregate(
       expr.expr,
-      zero.expr,
+      initialValue.expr,
       createLambda(merge),
       createLambda(finish)
     )
@@ -3508,17 +3541,31 @@ object functions {
   /**
    * Applies a binary operator to an initial state and all elements in the array,
    * and reduces this to a single state.
+   * {{{
+   *   df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x))
+   * }}}
    *
+   * @param expr the input array column
+   * @param initialValue the initial value
+   * @param merge (combined_value, input_value) => combined_value, the merge function to merge
+   *              an input value to the combined_value
    * @group collection_funcs
    * @since 3.0.0
    */
-  def aggregate(expr: Column, zero: Column, merge: (Column, Column) => Column): Column =
-    aggregate(expr, zero, merge, c => c)
+  def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column =
+    aggregate(expr, initialValue, merge, c => c)
 
   /**
    * Merge two given arrays, element-wise, into a single array using a function.
    * If one array is shorter, nulls are appended at the end to match the length of the longer
    * array, before applying the function.
+   * {{{
+   *   df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y))
+   * }}}
+   *
+   * @param left the left input array column
+   * @param right the right input array column
+   * @param f (lCol, rCol) => col, the lambda function to merge two input columns into one column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3530,6 +3577,12 @@ object functions {
   /**
    * Applies a function to every key-value pair in a map and returns
    * a map with the results of those applications as the new keys for the pairs.
+   * {{{
+   *   df.select(transform_keys(col("i"), (k, v) => k + v))
+   * }}}
+   *
+   * @param expr the input map column
+   * @param f (key, value) => new_key, the lambda function to transform the key of input map column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3541,6 +3594,13 @@ object functions {
   /**
    * Applies a function to every key-value pair in a map and returns
    * a map with the results of those applications as the new values for the pairs.
+   * {{{
+   *   df.select(transform_values(col("i"), (k, v) => k + v))
+   * }}}
+   *
+   * @param expr the input map column
+   * @param f (key, value) => new_value, the lambda function to transform the value of input map
+   *          column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3551,6 +3611,12 @@ object functions {
 
   /**
    * Returns a map whose key-value pairs satisfy a predicate.
+   * {{{
+   *   df.select(map_filter(col("m"), (k, v) => k * 10 === v))
+   * }}}
+   *
+   * @param expr the input map column
+   * @param f (key, value) => predicate, the Boolean predicate to filter the input map column
    *
    * @group collection_funcs
    * @since 3.0.0
@@ -3561,6 +3627,13 @@ object functions {
 
   /**
    * Merge two given maps, key-wise into a single map using a function.
+   * {{{
+   *   df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2))
+   * }}}
+   *
+   * @param left the left input map column
+   * @param right the right input map column
+   * @param f (key, value1, value2) => new_value, the lambda function to merge the map values
    *
    * @group collection_funcs
    * @since 3.0.0

From 4546f128c17b73f8eaaef7524148d588c304a9d4 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Thu, 6 Feb 2020 20:53:44 +0800
Subject: [PATCH 0026/1280] [SPARK-26700][CORE][FOLLOWUP] Add config
 `spark.network.maxRemoteBlockSizeFetchToMem`

### What changes were proposed in this pull request?
Add new config `spark.network.maxRemoteBlockSizeFetchToMem` fallback to the old config `spark.maxRemoteBlockSizeFetchToMem`.

### Why are the changes needed?
For naming consistency.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing tests.

Closes #27463 from xuanyuanking/SPARK-26700-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d8613571bc1847775dd5c1945757279234cb388c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 core/src/main/scala/org/apache/spark/SparkConf.scala           | 3 ++-
 .../main/scala/org/apache/spark/internal/config/package.scala  | 2 +-
 docs/configuration.md                                          | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 0e0291d2407d1..40915e3904f7e 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -684,7 +684,8 @@ private[spark] object SparkConf extends Logging {
     "spark.yarn.jars" -> Seq(
       AlternateConfig("spark.yarn.jar", "2.0")),
     MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM.key -> Seq(
-      AlternateConfig("spark.reducer.maxReqSizeShuffleToMem", "2.3")),
+      AlternateConfig("spark.reducer.maxReqSizeShuffleToMem", "2.3"),
+      AlternateConfig("spark.maxRemoteBlockSizeFetchToMem", "3.0")),
     LISTENER_BUS_EVENT_QUEUE_CAPACITY.key -> Seq(
       AlternateConfig("spark.scheduler.listenerbus.eventqueue.size", "2.3")),
     DRIVER_MEMORY_OVERHEAD.key -> Seq(
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index f91f31be2f1ad..02acb6b530737 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -895,7 +895,7 @@ package object config {
       .createWithDefault(Int.MaxValue)
 
   private[spark] val MAX_REMOTE_BLOCK_SIZE_FETCH_TO_MEM =
-    ConfigBuilder("spark.maxRemoteBlockSizeFetchToMem")
+    ConfigBuilder("spark.network.maxRemoteBlockSizeFetchToMem")
       .doc("Remote block will be fetched to disk when size of the block is above this threshold " +
         "in bytes. This is to avoid a giant request takes too much memory. Note this " +
         "configuration will affect both shuffle fetch and block manager remote block fetch. " +
diff --git a/docs/configuration.md b/docs/configuration.md
index 2febfe9744d5c..5bd3f3e80cf71 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1810,7 +1810,7 @@ Apart from these, the following properties are also available, and may be useful
   </td>
 </tr>
 <tr>
-  <td><code>spark.maxRemoteBlockSizeFetchToMem</code></td>
+  <td><code>spark.network.maxRemoteBlockSizeFetchToMem</code></td>
   <td>200m</td>
   <td>
     Remote block will be fetched to disk when size of the block is above this threshold

From b29cb1a82b1a1facf1dd040025db93d998dad4cd Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 6 Feb 2020 09:16:14 -0800
Subject: [PATCH 0027/1280] [SPARK-30719][SQL] do not log warning if AQE is
 intentionally skipped and add a config to force apply

### What changes were proposed in this pull request?

Update `InsertAdaptiveSparkPlan` to not log warning if AQE is skipped intentionally.

This PR also add a config to not skip AQE.

### Why are the changes needed?

It's not a warning at all if we intentionally skip AQE.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

run `AdaptiveQueryExecSuite` locally and verify that there is no warning logs.

Closes #27452 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
(cherry picked from commit 8ce58627ebe4f0372fba9a30d8cd4213611acd9b)
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  9 ++
 .../adaptive/InsertAdaptiveSparkPlan.scala    | 83 +++++++++++--------
 .../adaptive/AdaptiveQueryExecSuite.scala     |  9 ++
 3 files changed, 65 insertions(+), 36 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index acc0922e2cee7..bed8410acaed7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -358,6 +358,15 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
+  val ADAPTIVE_EXECUTION_FORCE_APPLY = buildConf("spark.sql.adaptive.forceApply")
+    .internal()
+    .doc("Adaptive query execution is skipped when the query does not have exchanges or " +
+      "sub-queries. By setting this config to true (together with " +
+      s"'${ADAPTIVE_EXECUTION_ENABLED.key}' enabled), Spark will force apply adaptive query " +
+      "execution for all supported queries.")
+    .booleanConf
+    .createWithDefault(false)
+
   val REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED =
     buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled")
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables reducing " +
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
index 9252827856af4..621c063e5a7d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
@@ -40,49 +40,60 @@ case class InsertAdaptiveSparkPlan(
 
   private val conf = adaptiveExecutionContext.session.sessionState.conf
 
-  def containShuffle(plan: SparkPlan): Boolean = {
-    plan.find {
-      case _: Exchange => true
-      case s: SparkPlan => !s.requiredChildDistribution.forall(_ == UnspecifiedDistribution)
-    }.isDefined
-  }
-
-  def containSubQuery(plan: SparkPlan): Boolean = {
-    plan.find(_.expressions.exists(_.find {
-      case _: SubqueryExpression => true
-      case _ => false
-    }.isDefined)).isDefined
-  }
-
   override def apply(plan: SparkPlan): SparkPlan = applyInternal(plan, false)
 
   private def applyInternal(plan: SparkPlan, isSubquery: Boolean): SparkPlan = plan match {
+    case _ if !conf.adaptiveExecutionEnabled => plan
     case _: ExecutedCommandExec => plan
-    case _ if conf.adaptiveExecutionEnabled && supportAdaptive(plan)
-      && (isSubquery || containShuffle(plan) || containSubQuery(plan)) =>
-      try {
-        // Plan sub-queries recursively and pass in the shared stage cache for exchange reuse. Fall
-        // back to non-adaptive mode if adaptive execution is supported in any of the sub-queries.
-        val subqueryMap = buildSubqueryMap(plan)
-        val planSubqueriesRule = PlanAdaptiveSubqueries(subqueryMap)
-        val preprocessingRules = Seq(
-          planSubqueriesRule)
-        // Run pre-processing rules.
-        val newPlan = AdaptiveSparkPlanExec.applyPhysicalRules(plan, preprocessingRules)
-        logDebug(s"Adaptive execution enabled for plan: $plan")
-        AdaptiveSparkPlanExec(newPlan, adaptiveExecutionContext, preprocessingRules, isSubquery)
-      } catch {
-        case SubqueryAdaptiveNotSupportedException(subquery) =>
-          logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is enabled " +
-            s"but is not supported for sub-query: $subquery.")
-          plan
-      }
-    case _ =>
-      if (conf.adaptiveExecutionEnabled) {
+    case _ if shouldApplyAQE(plan, isSubquery) =>
+      if (supportAdaptive(plan)) {
+        try {
+          // Plan sub-queries recursively and pass in the shared stage cache for exchange reuse.
+          // Fall back to non-AQE mode if AQE is not supported in any of the sub-queries.
+          val subqueryMap = buildSubqueryMap(plan)
+          val planSubqueriesRule = PlanAdaptiveSubqueries(subqueryMap)
+          val preprocessingRules = Seq(
+            planSubqueriesRule)
+          // Run pre-processing rules.
+          val newPlan = AdaptiveSparkPlanExec.applyPhysicalRules(plan, preprocessingRules)
+          logDebug(s"Adaptive execution enabled for plan: $plan")
+          AdaptiveSparkPlanExec(newPlan, adaptiveExecutionContext, preprocessingRules, isSubquery)
+        } catch {
+          case SubqueryAdaptiveNotSupportedException(subquery) =>
+            logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is enabled " +
+              s"but is not supported for sub-query: $subquery.")
+            plan
+        }
+      } else {
         logWarning(s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is enabled " +
           s"but is not supported for query: $plan.")
+        plan
       }
-      plan
+
+    case _ => plan
+  }
+
+  // AQE is only useful when the query has exchanges or sub-queries. This method returns true if
+  // one of the following conditions is satisfied:
+  //   - The config ADAPTIVE_EXECUTION_FORCE_APPLY is true.
+  //   - The input query is from a sub-query. When this happens, it means we've already decided to
+  //     apply AQE for the main query and we must continue to do it.
+  //   - The query contains exchanges.
+  //   - The query may need to add exchanges. It's an overkill to run `EnsureRequirements` here, so
+  //     we just check `SparkPlan.requiredChildDistribution` and see if it's possible that the
+  //     the query needs to add exchanges later.
+  //   - The query contains sub-query.
+  private def shouldApplyAQE(plan: SparkPlan, isSubquery: Boolean): Boolean = {
+    conf.getConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY) || isSubquery || {
+      plan.find {
+        case _: Exchange => true
+        case p if !p.requiredChildDistribution.forall(_ == UnspecifiedDistribution) => true
+        case p => p.expressions.exists(_.find {
+          case _: SubqueryExpression => true
+          case _ => false
+        }.isDefined)
+      }.isDefined
+    }
   }
 
   private def supportAdaptive(plan: SparkPlan): Boolean = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 78a1183664749..96e977221e512 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -780,4 +780,13 @@ class AdaptiveQueryExecSuite
       )
     }
   }
+
+  test("force apply AQE") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      val plan = sql("SELECT * FROM testData").queryExecution.executedPlan
+      assert(plan.isInstanceOf[AdaptiveSparkPlanExec])
+    }
+  }
 }

From 1baee64750b4098ec37be6408906c70674579eb8 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 6 Feb 2020 13:33:39 -0800
Subject: [PATCH 0028/1280] [SPARK-27986][SQL][FOLLOWUP] window aggregate
 function with filter predicate is not supported

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/26656.

We don't support window aggregate function with filter predicate yet and we should fail explicitly.

Observable metrics has the same issue. This PR fixes it as well.

### Why are the changes needed?

If we simply ignore filter predicate when we don't support it, the result is wrong.

### Does this PR introduce any user-facing change?

yea, fix the query result.

### How was this patch tested?

new tests

Closes #27476 from cloud-fan/filter.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 5a4c70b4e2367441ce4260f02d39d3345078f411)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  4 ++++
 .../sql/catalyst/analysis/CheckAnalysis.scala |  3 +++
 .../analysis/AnalysisErrorSuite.scala         | 20 +++++++++++++++++--
 .../sql/catalyst/analysis/AnalysisSuite.scala | 10 +++++++++-
 .../resources/sql-tests/inputs/window.sql     |  5 +++++
 .../sql-tests/results/window.sql.out          | 13 +++++++++++-
 6 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 56cc2a274bb7a..75f1aa7185ef3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2428,6 +2428,10 @@ class Analyzer(
             }
             wsc.copy(partitionSpec = newPartitionSpec, orderSpec = newOrderSpec)
 
+          case WindowExpression(ae: AggregateExpression, _) if ae.filter.isDefined =>
+            failAnalysis(
+              "window aggregate function with filter predicate is not supported yet.")
+
           // Extract Windowed AggregateExpression
           case we @ WindowExpression(
               ae @ AggregateExpression(function, _, _, _, _),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 4ec737fd9b70d..e769e038c960f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -308,6 +308,9 @@ trait CheckAnalysis extends PredicateHelper {
                 case a: AggregateExpression if a.isDistinct =>
                   e.failAnalysis(
                     "distinct aggregates are not allowed in observed metrics, but found: " + s.sql)
+                case a: AggregateExpression if a.filter.isDefined =>
+                  e.failAnalysis("aggregates with filter predicate are not allowed in " +
+                    "observed metrics, but found: " + s.sql)
                 case _: Attribute if !seenAggregate =>
                   e.failAnalysis (s"attribute ${s.sql} can only be used as an argument to an " +
                     "aggregate function.")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 7023dbe2a3672..5cc0453135c07 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -164,6 +164,22 @@ class AnalysisErrorSuite extends AnalysisTest {
           UnspecifiedFrame)).as("window")),
     "Distinct window functions are not supported" :: Nil)
 
+  errorTest(
+    "window aggregate function with filter predicate",
+    testRelation2.select(
+      WindowExpression(
+        AggregateExpression(
+          Count(UnresolvedAttribute("b")),
+          Complete,
+          isDistinct = false,
+          filter = Some(UnresolvedAttribute("b") > 1)),
+        WindowSpecDefinition(
+          UnresolvedAttribute("a") :: Nil,
+          SortOrder(UnresolvedAttribute("b"), Ascending) :: Nil,
+          UnspecifiedFrame)).as("window")),
+    "window aggregate function with filter predicate is not supported" :: Nil
+  )
+
   errorTest(
     "distinct function",
     CatalystSqlParser.parsePlan("SELECT hex(DISTINCT a) FROM TaBlE"),
@@ -191,12 +207,12 @@ class AnalysisErrorSuite extends AnalysisTest {
     "FILTER predicate specified, but aggregate is not an aggregate function" :: Nil)
 
   errorTest(
-    "DISTINCT and FILTER cannot be used in aggregate functions at the same time",
+    "DISTINCT aggregate function with filter predicate",
     CatalystSqlParser.parsePlan("SELECT count(DISTINCT a) FILTER (WHERE c > 1) FROM TaBlE2"),
     "DISTINCT and FILTER cannot be used in aggregate functions at the same time" :: Nil)
 
   errorTest(
-    "FILTER expression is non-deterministic, it cannot be used in aggregate functions",
+    "non-deterministic filter predicate in aggregate functions",
     CatalystSqlParser.parsePlan("SELECT count(a) FILTER (WHERE rand(int(c)) > 1) FROM TaBlE2"),
     "FILTER expression is non-deterministic, it cannot be used in aggregate functions" :: Nil)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 5405009c9e208..c747d394b1bc2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.aggregate.{Count, Sum}
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Sum}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
 import org.apache.spark.sql.catalyst.plans.{Cross, Inner}
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -736,5 +736,13 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       b :: ScalarSubquery(subquery, Nil).as("sum") :: Nil,
       CollectMetrics("evt1", count :: Nil, tblB))
     assertAnalysisError(query, "Multiple definitions of observed metrics" :: "evt1" :: Nil)
+
+    // Aggregate with filter predicate - fail
+    val sumWithFilter = sum.transform {
+      case a: AggregateExpression => a.copy(filter = Some(true))
+    }.asInstanceOf[NamedExpression]
+    assertAnalysisError(
+      CollectMetrics("evt1", sumWithFilter :: Nil, testRelation),
+      "aggregates with filter predicate are not allowed" :: Nil)
   }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/window.sql b/sql/core/src/test/resources/sql-tests/inputs/window.sql
index e25a252418301..3d05dfda6c3fa 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/window.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/window.sql
@@ -120,3 +120,8 @@ SELECT cate, sum(val) OVER (w)
 FROM testData
 WHERE val is not null
 WINDOW w AS (PARTITION BY cate ORDER BY val);
+
+-- with filter predicate
+SELECT val, cate,
+count(val) FILTER (WHERE val > 1) OVER(PARTITION BY cate)
+FROM testData ORDER BY cate, val;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out
index f795374735f59..625088f90ced9 100644
--- a/sql/core/src/test/resources/sql-tests/results/window.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 23
+-- Number of queries: 24
 
 
 -- !query
@@ -380,3 +380,14 @@ a	4
 b	1
 b	3
 b	6
+
+
+-- !query
+SELECT val, cate,
+count(val) FILTER (WHERE val > 1) OVER(PARTITION BY cate)
+FROM testData ORDER BY cate, val
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+window aggregate function with filter predicate is not supported yet.;

From 6130deb911e6dc65530c5aa92cd27cb0afad92b2 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 6 Feb 2020 13:42:09 -0800
Subject: [PATCH 0029/1280] [MINOR][INFRA][3.0] Enable GitHub Action in
 branch-3.0

### What changes were proposed in this pull request?

This aims to enable `GitHub Action` in `branch-3.0`.

### Why are the changes needed?

Currently, it's not enabled.
<img width="731" alt="commitlog" src="https://user-images.githubusercontent.com/9700541/73978595-871eb700-48e1-11ea-87d4-f41ee5c8fe3a.png">

This will protect `branch-3.0` by monitoring every commits and PR against `branch-3.0`.

### Does this PR introduce any user-facing change?

No. This is a dev-only infra.

### How was this patch tested?

See the GitHub Action triggering in this PR.

Closes #27480 from dongjoon-hyun/GHA-branch-3.0.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/master.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index d53119ad75599..0f80c88eba2f2 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -3,10 +3,10 @@ name: master
 on:
   push:
     branches:
-    - master
+    - branch-3.0
   pull_request:
     branches:
-    - master
+    - branch-3.0
 
 jobs:
   build:

From 475a8f2e2639fde28787d58b50c1d38f9a9c05ed Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Thu, 6 Feb 2020 14:58:53 -0800
Subject: [PATCH 0030/1280] [MINOR][DOC] Fix document UI left menu broken

### What changes were proposed in this pull request?
Fix the left menu broken introduced in #25459.

### Why are the changes needed?
The `left-menu-wrapper` CSS reused for both ml-guide and sql-programming-guide, the before changes will break the UI.

Before:
![image](https://user-images.githubusercontent.com/4833765/73952563-1061d800-493a-11ea-8a75-d802a1534a44.png)
![image](https://user-images.githubusercontent.com/4833765/73952584-18217c80-493a-11ea-85a3-ce5f9875545f.png)
![image](https://user-images.githubusercontent.com/4833765/73952605-21124e00-493a-11ea-8d79-24f4dfec73d9.png)

After:
![image](https://user-images.githubusercontent.com/4833765/73952630-2a031f80-493a-11ea-80ff-4630801cfaf4.png)
![image](https://user-images.githubusercontent.com/4833765/73952652-30919700-493a-11ea-9db1-8bb4a3f913b4.png)
![image](https://user-images.githubusercontent.com/4833765/73952671-35eee180-493a-11ea-801b-d50c4397adf2.png)

### Does this PR introduce any user-facing change?
Document UI change only.

### How was this patch tested?
Local test, screenshot attached below.

Closes #27479 from xuanyuanking/doc-ui.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 4804445327f06ae3a26365d8f110f06ea07eb637)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/css/main.css | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/css/main.css b/docs/css/main.css
index e24dff8531f24..dc05d287996be 100755
--- a/docs/css/main.css
+++ b/docs/css/main.css
@@ -211,8 +211,6 @@ a.anchorjs-link:hover { text-decoration: none; }
   float: left;
   position: fixed;
   overflow-y: scroll;
-  top: 0;
-  bottom: 0;
 }
 
 .left-menu {

From 683a07d5bad9519df88d1528e1afc143e367a688 Mon Sep 17 00:00:00 2001
From: sharif ahmad <sharifahmad2061@gmail.com>
Date: Fri, 7 Feb 2020 18:42:16 +0900
Subject: [PATCH 0031/1280] [MINOR][DOCS] Fix typos at
 python/pyspark/sql/types.py

### What changes were proposed in this pull request?

This PR fixes some typos in `python/pyspark/sql/types.py` file.

### Why are the changes needed?

To deliver correct wording in documentation and codes.

### Does this PR introduce any user-facing change?

Yes, it fixes some typos in user-facing API documentation.

### How was this patch tested?

Locally tested the linter.

Closes #27475 from sharifahmad2061/master.

Lead-authored-by: sharif ahmad <sharifahmad2061@gmail.com>
Co-authored-by: Sharif ahmad <sharifahmad2061@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit dd2f4431f56e02cd06848b02b93b4cf34c97a5d5)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/types.py | 40 ++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 8afff77b723a8..a5302e7bfd5ab 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -76,7 +76,7 @@ def json(self):
 
     def needConversion(self):
         """
-        Does this type need to conversion between Python object and internal SQL object.
+        Does this type needs conversion between Python object and internal SQL object.
 
         This is used to avoid the unnecessary conversion for ArrayType/MapType/StructType.
         """
@@ -210,17 +210,17 @@ class DecimalType(FractionalType):
 
     The precision can be up to 38, the scale must be less or equal to precision.
 
-    When create a DecimalType, the default precision and scale is (10, 0). When infer
+    When creating a DecimalType, the default precision and scale is (10, 0). When inferring
     schema from decimal.Decimal objects, it will be DecimalType(38, 18).
 
-    :param precision: the maximum total number of digits (default: 10)
+    :param precision: the maximum (i.e. total) number of digits (default: 10)
     :param scale: the number of digits on right side of dot. (default: 0)
     """
 
     def __init__(self, precision=10, scale=0):
         self.precision = precision
         self.scale = scale
-        self.hasPrecisionInfo = True  # this is public API
+        self.hasPrecisionInfo = True  # this is a public API
 
     def simpleString(self):
         return "decimal(%d,%d)" % (self.precision, self.scale)
@@ -457,8 +457,8 @@ class StructType(DataType):
 
     This is the data type representing a :class:`Row`.
 
-    Iterating a :class:`StructType` will iterate its :class:`StructField`\\s.
-    A contained :class:`StructField` can be accessed by name or position.
+    Iterating a :class:`StructType` will iterate over its :class:`StructField`\\s.
+    A contained :class:`StructField` can be accessed by its name or position.
 
     >>> struct1 = StructType([StructField("f1", StringType(), True)])
     >>> struct1["f1"]
@@ -492,8 +492,8 @@ def __init__(self, fields=None):
 
     def add(self, field, data_type=None, nullable=True, metadata=None):
         """
-        Construct a StructType by adding new elements to it to define the schema. The method accepts
-        either:
+        Construct a StructType by adding new elements to it, to define the schema.
+        The method accepts either:
 
             a) A single parameter which is a StructField object.
             b) Between 2 and 4 parameters as (name, data_type, nullable (optional),
@@ -676,7 +676,7 @@ def needConversion(self):
     @classmethod
     def _cachedSqlType(cls):
         """
-        Cache the sqlType() into class, because it's heavy used in `toInternal`.
+        Cache the sqlType() into class, because it's heavily used in `toInternal`.
         """
         if not hasattr(cls, "_cached_sql_type"):
             cls._cached_sql_type = cls.sqlType()
@@ -693,7 +693,7 @@ def fromInternal(self, obj):
 
     def serialize(self, obj):
         """
-        Converts the a user-type object into a SQL datum.
+        Converts a user-type object into a SQL datum.
         """
         raise NotImplementedError("UDT must implement toInternal().")
 
@@ -760,7 +760,7 @@ def __eq__(self, other):
 def _parse_datatype_string(s):
     """
     Parses the given data type string to a :class:`DataType`. The data type string format equals
-    to :class:`DataType.simpleString`, except that top level struct type can omit
+    :class:`DataType.simpleString`, except that the top level struct type can omit
     the ``struct<>`` and atomic types use ``typeName()`` as their format, e.g. use ``byte`` instead
     of ``tinyint`` for :class:`ByteType`. We can also use ``int`` as a short name
     for :class:`IntegerType`. Since Spark 2.3, this also supports a schema in a DDL-formatted
@@ -921,7 +921,7 @@ def _parse_datatype_json_value(json_value):
 # We should be careful here. The size of these types in python depends on C
 # implementation. We need to make sure that this conversion does not lose any
 # precision. Also, JVM only support signed types, when converting unsigned types,
-# keep in mind that it required 1 more bit when stored as singed types.
+# keep in mind that it require 1 more bit when stored as signed types.
 #
 # Reference for C integer size, see:
 # ISO/IEC 9899:201x specification, chapter 5.2.4.2.1 Sizes of integer types <limits.h>.
@@ -959,7 +959,7 @@ def _int_size_to_type(size):
     if size <= 64:
         return LongType
 
-# The list of all supported array typecodes is stored here
+# The list of all supported array typecodes, is stored here
 _array_type_mappings = {
     # Warning: Actual properties for float and double in C is not specified in C.
     # On almost every system supported by both python and JVM, they are IEEE 754
@@ -995,9 +995,9 @@ def _int_size_to_type(size):
     _array_type_mappings['c'] = StringType
 
 # SPARK-21465:
-# In python2, array of 'L' happened to be mistakenly partially supported. To
+# In python2, array of 'L' happened to be mistakenly, just partially supported. To
 # avoid breaking user's code, we should keep this partial support. Below is a
-# dirty hacking to keep this partial support and make the unit test passes
+# dirty hacking to keep this partial support and pass the unit test.
 import platform
 if sys.version_info[0] < 3 and platform.python_implementation() != 'PyPy':
     if 'L' not in _array_type_mappings.keys():
@@ -1071,7 +1071,7 @@ def _infer_schema(row, names=None):
 
 
 def _has_nulltype(dt):
-    """ Return whether there is NullType in `dt` or not """
+    """ Return whether there is a NullType in `dt` or not """
     if isinstance(dt, StructType):
         return any(_has_nulltype(f.dataType) for f in dt.fields)
     elif isinstance(dt, ArrayType):
@@ -1211,7 +1211,7 @@ def _make_type_verifier(dataType, nullable=True, name=None):
 
     This verifier also checks the value of obj against datatype and raises a ValueError if it's not
     within the allowed range, e.g. using 128 as ByteType will overflow. Note that, Python float is
-    not checked, so it will become infinity when cast to Java float if it overflows.
+    not checked, so it will become infinity when cast to Java float, if it overflows.
 
     >>> _make_type_verifier(StructType([]))(None)
     >>> _make_type_verifier(StringType())("")
@@ -1433,7 +1433,7 @@ class Row(tuple):
     ``key in row`` will search through row keys.
 
     Row can be used to create a row object by using named arguments.
-    It is not allowed to omit a named argument to represent the value is
+    It is not allowed to omit a named argument to represent that the value is
     None or missing. This should be explicitly set to None in this case.
 
     NOTE: As of Spark 3.0.0, Rows created from named arguments no longer have
@@ -1524,9 +1524,9 @@ def __new__(cls, *args, **kwargs):
 
     def asDict(self, recursive=False):
         """
-        Return as an dict
+        Return as a dict
 
-        :param recursive: turns the nested Row as dict (default: False).
+        :param recursive: turns the nested Rows to dict (default: False).
 
         >>> Row(name="Alice", age=11).asDict() == {'name': 'Alice', 'age': 11}
         True

From a9d9d834c600a4b9888f8781cec04fcbef29972d Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Sat, 8 Feb 2020 02:32:07 +0800
Subject: [PATCH 0032/1280] [SPARK-30752][SQL] Fix `to_utc_timestamp` on
 daylight saving day

### What changes were proposed in this pull request?
- Rewrite the `convertTz` method of `DateTimeUtils` using Java 8 time API
- Change types of `convertTz` parameters from `TimeZone` to `ZoneId`. This allows to avoid unnecessary conversions `TimeZone` -> `ZoneId` and performance regressions as a consequence.

### Why are the changes needed?
- Fixes incorrect behavior of `to_utc_timestamp` on daylight saving day. For example:
```scala
scala> df.select(to_utc_timestamp(lit("2019-11-03T12:00:00"), "Asia/Hong_Kong").as("local UTC")).show
+-------------------+
|          local UTC|
+-------------------+
|2019-11-03 03:00:00|
+-------------------+
```
but the result must be 2019-11-03 04:00:00:
<img width="1013" alt="Screen Shot 2020-02-06 at 20 09 36" src="https://user-images.githubusercontent.com/1580697/73960846-a129bb00-491c-11ea-92f5-45831cb28a62.png">

- Simplifies the code, and make it more maintainable
- Switches `convertTz` on Proleptic Gregorian calendar used by Java 8 time classes by default. That makes the function consistent to other date-time functions.

### Does this PR introduce any user-facing change?
Yes, after the changes `to_utc_timestamp` returns the correct result `2019-11-03 04:00:00`.

### How was this patch tested?
- By existing test suite `DateTimeUtilsSuite`, `DateFunctionsSuite` and `DateExpressionsSuite`.
- Added `convert time zones on a daylight saving day` to DateFunctionsSuite

Closes #27474 from MaxGekk/port-convertTz-on-Java8-api.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a3e77773cfa03a18d31370acd9a10562ff5312bb)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/datetimeExpressions.scala     | 16 ++++------
 .../sql/catalyst/util/DateTimeUtils.scala     | 28 ++++--------------
 .../parquet/VectorizedColumnReader.java       |  9 +++---
 .../VectorizedParquetRecordReader.java        |  6 ++--
 .../parquet/ParquetFileFormat.scala           |  2 +-
 .../parquet/ParquetReadSupport.scala          |  5 ++--
 .../parquet/ParquetRecordMaterializer.scala   |  4 +--
 .../parquet/ParquetRowConverter.scala         |  9 +++---
 .../ParquetPartitionReaderFactory.scala       | 10 +++----
 .../apache/spark/sql/DateFunctionsSuite.scala | 29 +++++++++++++++++--
 .../ParquetInteroperabilitySuite.scala        |  5 ++--
 11 files changed, 64 insertions(+), 59 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 28f1d34267224..aa2bd5a1273e0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -1176,14 +1176,12 @@ case class FromUTCTimestamp(left: Expression, right: Expression)
            |long ${ev.value} = 0;
          """.stripMargin)
       } else {
-        val tzClass = classOf[TimeZone].getName
+        val tzClass = classOf[ZoneId].getName
         val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
         val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
         val tzTerm = ctx.addMutableState(tzClass, "tz",
-          v => s"""$v = $dtu.getTimeZone("$escapedTz");""")
-        val utcTerm = "tzUTC"
-        ctx.addImmutableStateIfNotExists(tzClass, utcTerm,
-          v => s"""$v = $dtu.getTimeZone("UTC");""")
+          v => s"""$v = $dtu.getZoneId("$escapedTz");""")
+        val utcTerm = "java.time.ZoneOffset.UTC"
         val eval = left.genCode(ctx)
         ev.copy(code = code"""
            |${eval.code}
@@ -1382,14 +1380,12 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
            |long ${ev.value} = 0;
          """.stripMargin)
       } else {
-        val tzClass = classOf[TimeZone].getName
+        val tzClass = classOf[ZoneId].getName
         val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
         val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
         val tzTerm = ctx.addMutableState(tzClass, "tz",
-          v => s"""$v = $dtu.getTimeZone("$escapedTz");""")
-        val utcTerm = "tzUTC"
-        ctx.addImmutableStateIfNotExists(tzClass, utcTerm,
-          v => s"""$v = $dtu.getTimeZone("UTC");""")
+          v => s"""$v = $dtu.getZoneId("$escapedTz");""")
+        val utcTerm = "java.time.ZoneOffset.UTC"
         val eval = left.genCode(ctx)
         ev.copy(code = code"""
            |${eval.code}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 6800abb2ae109..8eb560944d4cb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -801,27 +801,9 @@ object DateTimeUtils {
    * mapping, the conversion here may return wrong result, we should make the timestamp
    * timezone-aware.
    */
-  def convertTz(ts: SQLTimestamp, fromZone: TimeZone, toZone: TimeZone): SQLTimestamp = {
-    // We always use local timezone to parse or format a timestamp
-    val localZone = defaultTimeZone()
-    val utcTs = if (fromZone.getID == localZone.getID) {
-      ts
-    } else {
-      // get the human time using local time zone, that actually is in fromZone.
-      val localZoneOffsetMs = localZone.getOffset(MICROSECONDS.toMillis(ts))
-      val localTsUs = ts + MILLISECONDS.toMicros(localZoneOffsetMs)  // in fromZone
-      val offsetFromLocalMs = getOffsetFromLocalMillis(MICROSECONDS.toMillis(localTsUs), fromZone)
-      localTsUs - MILLISECONDS.toMicros(offsetFromLocalMs)
-    }
-    if (toZone.getID == localZone.getID) {
-      utcTs
-    } else {
-      val toZoneOffsetMs = toZone.getOffset(MICROSECONDS.toMillis(utcTs))
-      val localTsUs = utcTs + MILLISECONDS.toMicros(toZoneOffsetMs)  // in toZone
-      // treat it as local timezone, convert to UTC (we could get the expected human time back)
-      val offsetFromLocalMs = getOffsetFromLocalMillis(MICROSECONDS.toMillis(localTsUs), localZone)
-      localTsUs - MILLISECONDS.toMicros(offsetFromLocalMs)
-    }
+  def convertTz(ts: SQLTimestamp, fromZone: ZoneId, toZone: ZoneId): SQLTimestamp = {
+    val rebasedDateTime = microsToInstant(ts).atZone(toZone).toLocalDateTime.atZone(fromZone)
+    instantToMicros(rebasedDateTime.toInstant)
   }
 
   /**
@@ -829,7 +811,7 @@ object DateTimeUtils {
    * representation in their timezone.
    */
   def fromUTCTime(time: SQLTimestamp, timeZone: String): SQLTimestamp = {
-    convertTz(time, TimeZoneGMT, getTimeZone(timeZone))
+    convertTz(time, ZoneOffset.UTC, getZoneId(timeZone))
   }
 
   /**
@@ -837,7 +819,7 @@ object DateTimeUtils {
    * string representation in their timezone.
    */
   def toUTCTime(time: SQLTimestamp, timeZone: String): SQLTimestamp = {
-    convertTz(time, getTimeZone(timeZone), TimeZoneGMT)
+    convertTz(time, getZoneId(timeZone), ZoneOffset.UTC)
   }
 
   /**
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index ba26b57567e64..329465544979d 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -18,8 +18,9 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.util.Arrays;
-import java.util.TimeZone;
 
 import org.apache.parquet.bytes.ByteBufferInputStream;
 import org.apache.parquet.bytes.BytesInput;
@@ -98,14 +99,14 @@ public class VectorizedColumnReader {
   private final ColumnDescriptor descriptor;
   private final OriginalType originalType;
   // The timezone conversion to apply to int96 timestamps. Null if no conversion.
-  private final TimeZone convertTz;
-  private static final TimeZone UTC = DateTimeUtils.TimeZoneUTC();
+  private final ZoneId convertTz;
+  private static final ZoneId UTC = ZoneOffset.UTC;
 
   public VectorizedColumnReader(
       ColumnDescriptor descriptor,
       OriginalType originalType,
       PageReader pageReader,
-      TimeZone convertTz) throws IOException {
+      ZoneId convertTz) throws IOException {
     this.descriptor = descriptor;
     this.pageReader = pageReader;
     this.convertTz = convertTz;
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
index f02861355c404..7306709a79c34 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
@@ -18,9 +18,9 @@
 package org.apache.spark.sql.execution.datasources.parquet;
 
 import java.io.IOException;
+import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.List;
-import java.util.TimeZone;
 
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
@@ -86,7 +86,7 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    * The timezone that timestamp INT96 values should be converted to. Null if no conversion. Here to
    * workaround incompatibilities between different engines when writing timestamp values.
    */
-  private TimeZone convertTz = null;
+  private ZoneId convertTz = null;
 
   /**
    * columnBatch object that is used for batch decoding. This is created on first use and triggers
@@ -116,7 +116,7 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    */
   private final MemoryMode MEMORY_MODE;
 
-  public VectorizedParquetRecordReader(TimeZone convertTz, boolean useOffHeap, int capacity) {
+  public VectorizedParquetRecordReader(ZoneId convertTz, boolean useOffHeap, int capacity) {
     this.convertTz = convertTz;
     MEMORY_MODE = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
     this.capacity = capacity;
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index f52aaf0140e1d..29dbd8dfbca8f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -295,7 +295,7 @@ class ParquetFileFormat
 
       val convertTz =
         if (timestampConversion && !isCreatedByParquetMr) {
-          Some(DateTimeUtils.getTimeZone(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
+          Some(DateTimeUtils.getZoneId(sharedConf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
         } else {
           None
         }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
index 69c8bad5f1c83..c05ecf16311ab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.util.{Locale, Map => JMap, TimeZone}
+import java.time.ZoneId
+import java.util.{Locale, Map => JMap}
 
 import scala.collection.JavaConverters._
 
@@ -49,7 +50,7 @@ import org.apache.spark.sql.types._
  * Due to this reason, we no longer rely on [[ReadContext]] to pass requested schema from [[init()]]
  * to [[prepareForRead()]], but use a private `var` for simplicity.
  */
-class ParquetReadSupport(val convertTz: Option[TimeZone],
+class ParquetReadSupport(val convertTz: Option[ZoneId],
     enableVectorizedReader: Boolean)
   extends ReadSupport[InternalRow] with Logging {
   private var catalystRequestedSchema: StructType = _
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
index 3098a332d3027..5622169df1281 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.util.TimeZone
+import java.time.ZoneId
 
 import org.apache.parquet.io.api.{GroupConverter, RecordMaterializer}
 import org.apache.parquet.schema.MessageType
@@ -36,7 +36,7 @@ private[parquet] class ParquetRecordMaterializer(
     parquetSchema: MessageType,
     catalystSchema: StructType,
     schemaConverter: ParquetToSparkSchemaConverter,
-    convertTz: Option[TimeZone])
+    convertTz: Option[ZoneId])
   extends RecordMaterializer[InternalRow] {
 
   private val rootConverter =
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 98ac2ecd2955c..850adae8a6b95 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 
 import java.math.{BigDecimal, BigInteger}
 import java.nio.ByteOrder
-import java.util.TimeZone
+import java.time.{ZoneId, ZoneOffset}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
@@ -125,7 +125,7 @@ private[parquet] class ParquetRowConverter(
     schemaConverter: ParquetToSparkSchemaConverter,
     parquetType: GroupType,
     catalystType: StructType,
-    convertTz: Option[TimeZone],
+    convertTz: Option[ZoneId],
     updater: ParentContainerUpdater)
   extends ParquetGroupConverter(updater) with Logging {
 
@@ -154,8 +154,6 @@ private[parquet] class ParquetRowConverter(
        |${catalystType.prettyJson}
      """.stripMargin)
 
-  private[this] val UTC = DateTimeUtils.TimeZoneUTC
-
   /**
    * Updater used together with field converters within a [[ParquetRowConverter]].  It propagates
    * converted filed values to the `ordinal`-th cell in `currentRow`.
@@ -292,7 +290,8 @@ private[parquet] class ParquetRowConverter(
             val timeOfDayNanos = buf.getLong
             val julianDay = buf.getInt
             val rawTime = DateTimeUtils.fromJulianDay(julianDay, timeOfDayNanos)
-            val adjTime = convertTz.map(DateTimeUtils.convertTz(rawTime, _, UTC)).getOrElse(rawTime)
+            val adjTime = convertTz.map(DateTimeUtils.convertTz(rawTime, _, ZoneOffset.UTC))
+              .getOrElse(rawTime)
             updater.setLong(adjTime)
           }
         }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
index b2fc724057eba..047bc74a8d81e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.execution.datasources.v2.parquet
 
 import java.net.URI
-import java.util.TimeZone
+import java.time.ZoneId
 
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapreduce._
@@ -117,7 +117,7 @@ case class ParquetPartitionReaderFactory(
       file: PartitionedFile,
       buildReaderFunc: (
         ParquetInputSplit, InternalRow, TaskAttemptContextImpl, Option[FilterPredicate],
-          Option[TimeZone]) => RecordReader[Void, T]): RecordReader[Void, T] = {
+          Option[ZoneId]) => RecordReader[Void, T]): RecordReader[Void, T] = {
     val conf = broadcastedConf.value.value
 
     val filePath = new Path(new URI(file.filePath))
@@ -156,7 +156,7 @@ case class ParquetPartitionReaderFactory(
 
     val convertTz =
       if (timestampConversion && !isCreatedByParquetMr) {
-        Some(DateTimeUtils.getTimeZone(conf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
+        Some(DateTimeUtils.getZoneId(conf.get(SQLConf.SESSION_LOCAL_TIMEZONE.key)))
       } else {
         None
       }
@@ -184,7 +184,7 @@ case class ParquetPartitionReaderFactory(
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
-      convertTz: Option[TimeZone]): RecordReader[Void, InternalRow] = {
+      convertTz: Option[ZoneId]): RecordReader[Void, InternalRow] = {
     logDebug(s"Falling back to parquet-mr")
     val taskContext = Option(TaskContext.get())
     // ParquetRecordReader returns InternalRow
@@ -213,7 +213,7 @@ case class ParquetPartitionReaderFactory(
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
-      convertTz: Option[TimeZone]): VectorizedParquetRecordReader = {
+      convertTz: Option[ZoneId]): VectorizedParquetRecordReader = {
     val taskContext = Option(TaskContext.get())
     val vectorizedReader = new VectorizedParquetRecordReader(
       convertTz.orNull, enableOffHeapColumnVector && taskContext.isDefined, capacity)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 3b3d3cc3d7a17..bb8cdf3cb6de1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql
 
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
-import java.time.Instant
-import java.util.Locale
+import java.time.{Instant, LocalDateTime}
+import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
@@ -803,4 +803,29 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       checkTimeZoneParsing(null)
     }
   }
+
+  test("SPARK-30752: convert time zones on a daylight saving day") {
+    val systemTz = "PST"
+    val sessionTz = "UTC"
+    val fromTz = "Asia/Hong_Kong"
+    val fromTs = "2019-11-03T12:00:00" // daylight saving date in PST
+    val utsTs = "2019-11-03T04:00:00"
+    val defaultTz = TimeZone.getDefault
+    try {
+      TimeZone.setDefault(DateTimeUtils.getTimeZone(systemTz))
+      withSQLConf(
+        SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true",
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> sessionTz) {
+        val expected = LocalDateTime.parse(utsTs)
+          .atZone(DateTimeUtils.getZoneId(sessionTz))
+          .toInstant
+        val df = Seq(fromTs).toDF("localTs")
+        checkAnswer(
+          df.select(to_utc_timestamp(col("localTs"), fromTz)),
+          Row(expected))
+      }
+    } finally {
+      TimeZone.setDefault(defaultTz)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
index 1ded34f24e436..649a46f190580 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.parquet
 
 import java.io.File
+import java.time.ZoneOffset
 
 import org.apache.commons.io.FileUtils
 import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
@@ -145,8 +146,8 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
               impalaFileData.map { ts =>
                 DateTimeUtils.toJavaTimestamp(DateTimeUtils.convertTz(
                   DateTimeUtils.fromJavaTimestamp(ts),
-                  DateTimeUtils.TimeZoneUTC,
-                  DateTimeUtils.getTimeZone(conf.sessionLocalTimeZone)))
+                  ZoneOffset.UTC,
+                  DateTimeUtils.getZoneId(conf.sessionLocalTimeZone)))
               }
             }
             val fullExpectations = (ts ++ impalaExpectations).map(_.toString).sorted.toArray

From 73eba319f2ec548fd655a780ee3d240b24b6276d Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sat, 8 Feb 2020 14:10:28 -0800
Subject: [PATCH 0033/1280] [SPARK-28228][SQL] Change the default behavior for
 name conflict in nested WITH clause

### What changes were proposed in this pull request?
This is a follow-up for #25029, in this PR we throw an AnalysisException when name conflict is detected in nested WITH clause. In this way, the config `spark.sql.legacy.ctePrecedence.enabled` should be set explicitly for the expected behavior.

### Why are the changes needed?
The original change might risky to end-users, it changes behavior silently.

### Does this PR introduce any user-facing change?
Yes, change the config `spark.sql.legacy.ctePrecedence.enabled` as optional.

### How was this patch tested?
New UT.

Closes #27454 from xuanyuanking/SPARK-28228-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3db3e39f1122350f55f305bee049363621c5894d)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-migration-guide.md                   |   2 +-
 .../catalyst/analysis/CTESubstitution.scala   |  49 ++-
 .../apache/spark/sql/internal/SQLConf.scala   |   6 +-
 .../sql-tests/inputs/cte-nonlegacy.sql        |   2 +
 .../sql-tests/results/cte-nonlegacy.sql.out   | 343 ++++++++++++++++++
 .../resources/sql-tests/results/cte.sql.out   |  30 +-
 6 files changed, 415 insertions(+), 17 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 5a5e802f6a900..be0fe32ded99b 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -101,7 +101,7 @@ license: |
 
   - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
 
-  - Since Spark 3.0, substitution order of nested WITH clauses is changed and an inner CTE definition takes precedence over an outer. In version 2.4 and earlier, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `1` while in version 3.0 it returns `2`. The previous behaviour can be restored by setting `spark.sql.legacy.ctePrecedence.enabled` to `true`.
+  - Since Spark 3.0, Spark throws an AnalysisException if name conflict is detected in the nested WITH clause by default. It forces the users to choose the specific substitution order they wanted, which is controlled by `spark.sql.legacy.ctePrecedence.enabled`. If set to false (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `true`, the result is `1` which is the behavior in version 2.4 and earlier.
 
   - Since Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and earlier, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index 60e6bf8db06d7..d2be15d87d023 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, With}
 import org.apache.spark.sql.catalyst.rules.Rule
@@ -28,10 +29,54 @@ import org.apache.spark.sql.internal.SQLConf.LEGACY_CTE_PRECEDENCE_ENABLED
  */
 object CTESubstitution extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    if (SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_ENABLED)) {
+    val isLegacy = SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_ENABLED)
+    if (isLegacy.isEmpty) {
+      assertNoNameConflictsInCTE(plan, inTraverse = false)
+      traverseAndSubstituteCTE(plan, inTraverse = false)
+    } else if (isLegacy.get) {
       legacyTraverseAndSubstituteCTE(plan)
     } else {
-      traverseAndSubstituteCTE(plan, false)
+      traverseAndSubstituteCTE(plan, inTraverse = false)
+    }
+  }
+
+  /**
+   * Check the plan to be traversed has naming conflicts in nested CTE or not, traverse through
+   * child, innerChildren and subquery for the current plan.
+   */
+  private def assertNoNameConflictsInCTE(
+      plan: LogicalPlan,
+      inTraverse: Boolean,
+      cteNames: Set[String] = Set.empty): Unit = {
+    plan.foreach {
+      case w @ With(child, relations) =>
+        val newNames = relations.map {
+          case (cteName, _) =>
+            if (cteNames.contains(cteName)) {
+              throw new AnalysisException(s"Name $cteName is ambiguous in nested CTE. " +
+                s"Please set ${LEGACY_CTE_PRECEDENCE_ENABLED.key} to false so that name defined " +
+                "in inner CTE takes precedence. See more details in SPARK-28228.")
+            } else {
+              cteName
+            }
+        }.toSet
+        child.transformExpressions {
+          case e: SubqueryExpression =>
+            assertNoNameConflictsInCTE(e.plan, inTraverse = true, cteNames ++ newNames)
+            e
+        }
+        w.innerChildren.foreach { p =>
+          assertNoNameConflictsInCTE(p, inTraverse = true, cteNames ++ newNames)
+        }
+
+      case other if inTraverse =>
+        other.transformExpressions {
+          case e: SubqueryExpression =>
+            assertNoNameConflictsInCTE(e.plan, inTraverse = true, cteNames)
+            e
+        }
+
+      case _ =>
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index bed8410acaed7..a72bd53188f29 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2098,9 +2098,11 @@ object SQLConf {
 
   val LEGACY_CTE_PRECEDENCE_ENABLED = buildConf("spark.sql.legacy.ctePrecedence.enabled")
     .internal()
-    .doc("When true, outer CTE definitions takes precedence over inner definitions.")
+    .doc("When true, outer CTE definitions takes precedence over inner definitions. If set to " +
+      "false, inner CTE definitions take precedence. The default value is empty, " +
+      "AnalysisException is thrown while name conflict is detected in nested CTE.")
     .booleanConf
-    .createWithDefault(false)
+    .createOptional
 
   val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
     buildConf("spark.sql.legacy.arrayExistsFollowsThreeValuedLogic")
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
new file mode 100644
index 0000000000000..b711bf338ab08
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.ctePrecedence.enabled = false
+--IMPORT cte.sql
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
new file mode 100644
index 0000000000000..2d87781193c25
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
@@ -0,0 +1,343 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 27
+
+
+-- !query
+create temporary view t as select * from values 0, 1, 2 as t(id)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+create temporary view t2 as select * from values 0, 1 as t(id)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+WITH s AS (SELECT 1 FROM s) SELECT * FROM s
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table or view not found: s; line 1 pos 25
+
+
+-- !query
+WITH r AS (SELECT (SELECT * FROM r))
+SELECT * FROM r
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table or view not found: r; line 1 pos 33
+
+
+-- !query
+WITH t AS (SELECT 1 FROM t) SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+1
+1
+
+
+-- !query
+WITH s1 AS (SELECT 1 FROM s2), s2 AS (SELECT 1 FROM s1) SELECT * FROM s1, s2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table or view not found: s2; line 1 pos 26
+
+
+-- !query
+WITH t1 AS (SELECT * FROM t2), t2 AS (SELECT 2 FROM t1) SELECT * FROM t1 cross join t2
+-- !query schema
+struct<id:int,2:int>
+-- !query output
+0	2
+0	2
+1	2
+1	2
+
+
+-- !query
+WITH CTE1 AS (
+  SELECT b.id AS id
+  FROM   T2 a
+         CROSS JOIN (SELECT id AS id FROM T2) b
+)
+SELECT t1.id AS c1,
+       t2.id AS c2
+FROM   CTE1 t1
+       CROSS JOIN CTE1 t2
+-- !query schema
+struct<c1:int,c2:int>
+-- !query output
+0	0
+0	0
+0	0
+0	0
+0	1
+0	1
+0	1
+0	1
+1	0
+1	0
+1	0
+1	0
+1	1
+1	1
+1	1
+1	1
+
+
+-- !query
+WITH t(x) AS (SELECT 1)
+SELECT * FROM t WHERE x = 1
+-- !query schema
+struct<x:int>
+-- !query output
+1
+
+
+-- !query
+WITH t(x, y) AS (SELECT 1, 2)
+SELECT * FROM t WHERE x = 1 AND y = 2
+-- !query schema
+struct<x:int,y:int>
+-- !query output
+1	2
+
+
+-- !query
+WITH t(x, x) AS (SELECT 1, 2)
+SELECT * FROM t
+-- !query schema
+struct<x:int,x:int>
+-- !query output
+1	2
+
+
+-- !query
+WITH t() AS (SELECT 1)
+SELECT * FROM t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'WITH t()'(line 1, pos 7)
+
+== SQL ==
+WITH t() AS (SELECT 1)
+-------^^^
+SELECT * FROM t
+
+
+-- !query
+WITH
+  t(x) AS (SELECT 1),
+  t(x) AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+CTE definition can't have duplicate names: 't'.(line 1, pos 0)
+
+== SQL ==
+WITH
+^^^
+  t(x) AS (SELECT 1),
+  t(x) AS (SELECT 2)
+SELECT * FROM t
+
+
+-- !query
+WITH t as (
+  WITH t2 AS (SELECT 1)
+  SELECT * FROM t2
+)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+1
+
+
+-- !query
+SELECT (
+  WITH t AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+SELECT * FROM t2
+-- !query schema
+struct<2:int>
+-- !query output
+2
+
+
+-- !query
+WITH
+  t(c) AS (SELECT 1),
+  t2 AS (
+    SELECT (
+      SELECT max(c) FROM (
+        WITH t(c) AS (SELECT 2)
+        SELECT * FROM t
+      )
+    )
+  )
+SELECT * FROM t2
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2),
+    t2 AS (
+      WITH t AS (SELECT 3)
+      SELECT * FROM t
+    )
+    SELECT * FROM t2
+  )
+SELECT * FROM t2
+-- !query schema
+struct<3:int>
+-- !query output
+3
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+3
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  SELECT (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT (
+    WITH t AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+3
+
+
+-- !query
+DROP VIEW IF EXISTS t
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t2
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/cte.sql.out b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
index 2d87781193c25..1d50aa8f57505 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
@@ -204,9 +204,10 @@ WITH
   )
 SELECT * FROM t2
 -- !query schema
-struct<2:int>
+struct<>
 -- !query output
-2
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -222,9 +223,10 @@ WITH
   )
 SELECT * FROM t2
 -- !query schema
-struct<scalarsubquery():int>
+struct<>
 -- !query output
-2
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -240,9 +242,10 @@ WITH
   )
 SELECT * FROM t2
 -- !query schema
-struct<3:int>
+struct<>
 -- !query output
-3
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -293,9 +296,10 @@ SELECT (
   SELECT * FROM t
 )
 -- !query schema
-struct<scalarsubquery():int>
+struct<>
 -- !query output
-2
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -307,9 +311,10 @@ SELECT (
   )
 )
 -- !query schema
-struct<scalarsubquery():int>
+struct<>
 -- !query output
-2
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -322,9 +327,10 @@ SELECT (
   )
 )
 -- !query schema
-struct<scalarsubquery():int>
+struct<>
 -- !query output
-3
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
 
 
 -- !query

From 287d93fa1c5232fcbb5c4fe0ddbb4aeca39cd6b9 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sat, 8 Feb 2020 14:28:15 -0800
Subject: [PATCH 0034/1280] [SPARK-29587][DOC][FOLLOWUP] Add `SQL` tab in the
 `Data Types` page

### What changes were proposed in this pull request?
Add the new tab `SQL` in the `Data Types` page.

### Why are the changes needed?
New type added in SPARK-29587.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Locally test by Jekyll.
![image](https://user-images.githubusercontent.com/4833765/73908593-2e511d80-48e5-11ea-85a7-6ee451e6b727.png)

Closes #27447 from xuanyuanking/SPARK-29587-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit e1cd4d9dc25ac3abe33c07686fc2a7d1f2b5c122)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-ref-datatypes.md | 75 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 0add62b10ed6b..9700608fe8a34 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -631,4 +631,79 @@ from pyspark.sql.types import *
 </tr>
 </table>
 </div>
+
+<div data-lang="sql"  markdown="1">
+
+The following table shows the type names as well as aliases used in Spark SQL parser for each data type.
+
+<table class="table">
+<tr>
+  <th style="width:40%">Data type</th>
+  <th>SQL name</th></tr>
+<tr>
+  <td> <b>BooleanType</b> </td>
+  <td> BOOLEAN </td>
+</tr>
+<tr>
+  <td> <b>ByteType</b> </td>
+  <td> BYTE, TINYINT </td>
+</tr>
+<tr>
+  <td> <b>ShortType</b> </td>
+  <td> SHORT, SMALLINT </td>
+</tr>
+<tr>
+  <td> <b>IntegerType</b> </td>
+  <td> INT, INTEGER </td>
+</tr>
+<tr>
+  <td> <b>LongType</b> </td>
+  <td> LONG, BIGINT </td>
+</tr>
+<tr>
+  <td> <b>FloatType</b> </td>
+  <td> FLOAT, REAL </td>
+</tr>
+<tr>
+  <td> <b>DoubleType</b> </td>
+  <td> DOUBLE </td>
+</tr>
+<tr>
+  <td> <b>DateType</b> </td>
+  <td> DATE </td>
+</tr>
+<tr>
+  <td> <b>TimestampType</b> </td>
+  <td> TIMESTAMP </td>
+</tr>
+<tr>
+  <td> <b>StringType</b> </td>
+  <td> STRING </td>
+</tr>
+<tr>
+  <td> <b>BinaryType</b> </td>
+  <td> BINARY </td>
+</tr>
+<tr>
+  <td> <b>DecimalType</b> </td>
+  <td> DECIMAL, DEC, NUMERIC </td>
+</tr>
+<tr>
+  <td> <b>CalendarIntervalType</b> </td>
+  <td> INTERVAL </td>
+</tr>
+<tr>
+  <td> <b>ArrayType</b> </td>
+  <td> ARRAY&lt;element_type&gt; </td>
+</tr>
+<tr>
+  <td> <b>StructType</b> </td>
+  <td> STRUCT&lt;field1_name: field1_type, field2_name: field2_type, ...&gt; </td>
+</tr>
+<tr>
+  <td> <b>MapType</b> </td>
+  <td> MAP&lt;key_type, value_type&gt; </td>
+</tr>
+</table>
+</div>
 </div>

From d5865493ae71e6369e9f3350dd7e694afcf57298 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Sun, 9 Feb 2020 19:20:47 +0900
Subject: [PATCH 0035/1280] [SPARK-30510][SQL][DOCS] Publicly document Spark
 SQL configuration options

### What changes were proposed in this pull request?

This PR adds a doc builder for Spark SQL's configuration options.

Here's what the new Spark SQL config docs look like ([configuration.html.zip](https://github.com/apache/spark/files/4172109/configuration.html.zip)):

![Screen Shot 2020-02-07 at 12 13 23 PM](https://user-images.githubusercontent.com/1039369/74050007-425b5480-49a3-11ea-818c-42700c54d1fb.png)

Compare this to the [current docs](http://spark.apache.org/docs/3.0.0-preview2/configuration.html#spark-sql):

![Screen Shot 2020-02-04 at 4 55 10 PM](https://user-images.githubusercontent.com/1039369/73790828-24a5a980-476f-11ea-998c-12cd613883e8.png)

### Why are the changes needed?

There is no visibility into the various Spark SQL configs on [the config docs page](http://spark.apache.org/docs/3.0.0-preview2/configuration.html#spark-sql).

### Does this PR introduce any user-facing change?

No, apart from new documentation.

### How was this patch tested?

I tested this manually by building the docs and reviewing them in my browser.

Closes #27459 from nchammas/SPARK-30510-spark-sql-options.

Authored-by: Nicholas Chammas <nicholas.chammas@liveramp.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 339c0f9a623521acd4d66292b3fe3e6c4ec3b108)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/.gitignore                               |   1 +
 docs/configuration.md                         |  46 ++-----
 sql/README.md                                 |   2 +-
 .../apache/spark/sql/internal/SQLConf.scala   |  35 +++---
 .../spark/sql/api/python/PythonSQLUtils.scala |   7 ++
 sql/create-docs.sh                            |  14 ++-
 ...en-sql-markdown.py => gen-sql-api-docs.py} |   8 +-
 sql/gen-sql-config-docs.py                    | 117 ++++++++++++++++++
 8 files changed, 163 insertions(+), 67 deletions(-)
 create mode 100644 docs/.gitignore
 rename sql/{gen-sql-markdown.py => gen-sql-api-docs.py} (96%)
 create mode 100644 sql/gen-sql-config-docs.py

diff --git a/docs/.gitignore b/docs/.gitignore
new file mode 100644
index 0000000000000..2260493b46ab3
--- /dev/null
+++ b/docs/.gitignore
@@ -0,0 +1 @@
+sql-configs.html
diff --git a/docs/configuration.md b/docs/configuration.md
index 5bd3f3e80cf71..1343755f9d87f 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2399,47 +2399,15 @@ the driver or executor, or, in the absence of that value, the number of cores av
 Please refer to the [Security](security.html) page for available options on how to secure different
 Spark subsystems.
 
-### Spark SQL
-
-Running the <code>SET -v</code> command will show the entire list of the SQL configuration.
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
 
-{% highlight scala %}
-// spark is an existing SparkSession
-spark.sql("SET -v").show(numRows = 200, truncate = false)
-{% endhighlight %}
-
-</div>
-
-<div data-lang="java"  markdown="1">
-
-{% highlight java %}
-// spark is an existing SparkSession
-spark.sql("SET -v").show(200, false);
-{% endhighlight %}
-</div>
-
-<div data-lang="python"  markdown="1">
-
-{% highlight python %}
-# spark is an existing SparkSession
-spark.sql("SET -v").show(n=200, truncate=False)
-{% endhighlight %}
-
-</div>
-
-<div data-lang="r"  markdown="1">
-
-{% highlight r %}
-sparkR.session()
-properties <- sql("SET -v")
-showDF(properties, numRows = 200, truncate = FALSE)
-{% endhighlight %}
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'sql-configs.html' %}
+### Spark SQL
 
-</div>
-</div>
+        {% include_relative sql-configs.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
 
 
 ### Spark Streaming
diff --git a/sql/README.md b/sql/README.md
index 67e3225e2c275..ae5ebd1d75370 100644
--- a/sql/README.md
+++ b/sql/README.md
@@ -9,4 +9,4 @@ Spark SQL is broken up into four subprojects:
  - Hive Support (sql/hive) - Includes extensions that allow users to write queries using a subset of HiveQL and access data from a Hive Metastore using Hive SerDes. There are also wrappers that allow users to run queries that include Hive UDFs, UDAFs, and UDTFs.
  - HiveServer and CLI support (sql/hive-thriftserver) - Includes support for the SQL CLI (bin/spark-sql) and a HiveServer2 (for JDBC/ODBC) compatible server.
 
-Running `./sql/create-docs.sh` generates SQL documentation for built-in functions under `sql/site`.
+Running `./sql/create-docs.sh` generates SQL documentation for built-in functions under `sql/site`, and SQL configuration documentation that gets included as part of `configuration.md` in the main `docs` directory.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index a72bd53188f29..e38fe7606c4ee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -324,11 +324,11 @@ object SQLConf {
     .doc("Configures the maximum size in bytes for a table that will be broadcast to all worker " +
       "nodes when performing a join.  By setting this value to -1 broadcasting can be disabled. " +
       "Note that currently statistics are only supported for Hive Metastore tables where the " +
-      "command <code>ANALYZE TABLE &lt;tableName&gt; COMPUTE STATISTICS noscan</code> has been " +
+      "command `ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan` has been " +
       "run, and file-based data source tables where the statistics are computed directly on " +
       "the files of data.")
     .bytesConf(ByteUnit.BYTE)
-    .createWithDefault(10L * 1024 * 1024)
+    .createWithDefaultString("10MB")
 
   val LIMIT_SCALE_UP_FACTOR = buildConf("spark.sql.limit.scaleUpFactor")
     .internal()
@@ -402,7 +402,7 @@ object SQLConf {
         s"an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
         s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
       .bytesConf(ByteUnit.BYTE)
-      .createWithDefault(64 * 1024 * 1024)
+      .createWithDefaultString("64MB")
 
   val SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS =
     buildConf("spark.sql.adaptive.shuffle.maxNumPostShufflePartitions")
@@ -436,7 +436,7 @@ object SQLConf {
       .doc("Configures the minimum size in bytes for a partition that is considered as a skewed " +
         "partition in adaptive skewed join.")
       .bytesConf(ByteUnit.BYTE)
-      .createWithDefault(64 * 1024 * 1024)
+      .createWithDefaultString("64MB")
 
   val ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR =
     buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionFactor")
@@ -770,7 +770,7 @@ object SQLConf {
   val BROADCAST_TIMEOUT = buildConf("spark.sql.broadcastTimeout")
     .doc("Timeout in seconds for the broadcast wait time in broadcast joins.")
     .timeConf(TimeUnit.SECONDS)
-    .createWithDefault(5 * 60)
+    .createWithDefaultString(s"${5 * 60}")
 
   // This is only used for the thriftserver
   val THRIFTSERVER_POOL = buildConf("spark.sql.thriftserver.scheduler.pool")
@@ -830,7 +830,7 @@ object SQLConf {
     .createWithDefault(true)
 
   val BUCKETING_MAX_BUCKETS = buildConf("spark.sql.sources.bucketing.maxBuckets")
-    .doc("The maximum number of buckets allowed. Defaults to 100000")
+    .doc("The maximum number of buckets allowed.")
     .intConf
     .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
     .createWithDefault(100000)
@@ -1022,7 +1022,7 @@ object SQLConf {
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
     .bytesConf(ByteUnit.BYTE)
-    .createWithDefault(128 * 1024 * 1024) // parquet.block.size
+    .createWithDefaultString("128MB") // parquet.block.size
 
   val FILES_OPEN_COST_IN_BYTES = buildConf("spark.sql.files.openCostInBytes")
     .internal()
@@ -1161,7 +1161,8 @@ object SQLConf {
 
   val VARIABLE_SUBSTITUTE_ENABLED =
     buildConf("spark.sql.variable.substitute")
-      .doc("This enables substitution using syntax like ${var} ${system:var} and ${env:var}.")
+      .doc("This enables substitution using syntax like `${var}`, `${system:var}`, " +
+        "and `${env:var}`.")
       .booleanConf
       .createWithDefault(true)
 
@@ -1171,7 +1172,7 @@ object SQLConf {
       .doc("Enable two-level aggregate hash map. When enabled, records will first be " +
         "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " +
         "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " +
-        "When disabled, records go directly to the 2nd level. Defaults to true.")
+        "When disabled, records go directly to the 2nd level.")
       .booleanConf
       .createWithDefault(true)
 
@@ -1325,10 +1326,10 @@ object SQLConf {
 
   val STREAMING_STOP_TIMEOUT =
     buildConf("spark.sql.streaming.stopTimeout")
-      .doc("How long to wait for the streaming execution thread to stop when calling the " +
-        "streaming query's stop() method in milliseconds. 0 or negative values wait indefinitely.")
+      .doc("How long to wait in milliseconds for the streaming execution thread to stop when " +
+        "calling the streaming query's stop() method. 0 or negative values wait indefinitely.")
       .timeConf(TimeUnit.MILLISECONDS)
-      .createWithDefault(0L)
+      .createWithDefaultString("0")
 
   val STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL =
     buildConf("spark.sql.streaming.noDataProgressEventInterval")
@@ -1611,10 +1612,10 @@ object SQLConf {
   val PANDAS_UDF_BUFFER_SIZE =
     buildConf("spark.sql.pandas.udf.buffer.size")
       .doc(
-        s"Same as ${BUFFER_SIZE} but only applies to Pandas UDF executions. If it is not set, " +
-        s"the fallback is ${BUFFER_SIZE}. Note that Pandas execution requires more than 4 bytes. " +
-        "Lowering this value could make small Pandas UDF batch iterated and pipelined; however, " +
-        "it might degrade performance. See SPARK-27870.")
+        s"Same as `${BUFFER_SIZE.key}` but only applies to Pandas UDF executions. If it is not " +
+        s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
+        "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
+        "pipelined; however, it might degrade performance. See SPARK-27870.")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
@@ -2039,7 +2040,7 @@ object SQLConf {
     .checkValue(i => i >= 0 && i <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, "Invalid " +
       "value for 'spark.sql.maxPlanStringLength'.  Length must be a valid string length " +
       "(nonnegative and shorter than the maximum size).")
-    .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
+    .createWithDefaultString(s"${ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH}")
 
   val SET_COMMAND_REJECTS_SPARK_CORE_CONFS =
     buildConf("spark.sql.legacy.setCommandRejectsSparkCoreConfs")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index b232aa18c816e..bf3055d5e3e09 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.{ExplainMode, QueryExecution}
 import org.apache.spark.sql.execution.arrow.ArrowConverters
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.DataType
 
 private[sql] object PythonSQLUtils {
@@ -39,6 +40,12 @@ private[sql] object PythonSQLUtils {
     FunctionRegistry.functionSet.flatMap(f => FunctionRegistry.builtin.lookupFunction(f)).toArray
   }
 
+  def listSQLConfigs(): Array[(String, String, String)] = {
+    val conf = new SQLConf()
+    // Py4J doesn't seem to translate Seq well, so we convert to an Array.
+    conf.getAllDefinedConfs.toArray
+  }
+
   /**
    * Python callable function to read a file in Arrow stream format and create a [[RDD]]
    * using each serialized ArrowRecordBatch as a partition.
diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 4353708d22f7b..44aa877332fd5 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-# Script to create SQL API docs. This requires `mkdocs` and to build
+# Script to create SQL API and config docs. This requires `mkdocs` and to build
 # Spark first. After running this script the html docs can be found in
 # $SPARK_HOME/sql/site
 
@@ -39,14 +39,16 @@ fi
 
 pushd "$FWDIR" > /dev/null
 
-# Now create the markdown file
 rm -fr docs
 mkdir docs
-echo "Generating markdown files for SQL documentation."
-"$SPARK_HOME/bin/spark-submit" gen-sql-markdown.py
 
-# Now create the HTML files
-echo "Generating HTML files for SQL documentation."
+echo "Generating SQL API Markdown files."
+"$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py
+
+echo "Generating SQL configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
+
+echo "Generating HTML files for SQL API documentation."
 mkdocs build --clean
 rm -fr docs
 
diff --git a/sql/gen-sql-markdown.py b/sql/gen-sql-api-docs.py
similarity index 96%
rename from sql/gen-sql-markdown.py
rename to sql/gen-sql-api-docs.py
index e0529f8310613..4feee7ad52570 100644
--- a/sql/gen-sql-markdown.py
+++ b/sql/gen-sql-api-docs.py
@@ -15,10 +15,11 @@
 # limitations under the License.
 #
 
-import sys
 import os
 from collections import namedtuple
 
+from pyspark.java_gateway import launch_gateway
+
 ExpressionInfo = namedtuple(
     "ExpressionInfo", "className name usage arguments examples note since deprecated")
 
@@ -219,8 +220,7 @@ def generate_sql_markdown(jvm, path):
 
 
 if __name__ == "__main__":
-    from pyspark.java_gateway import launch_gateway
-
     jvm = launch_gateway().jvm
-    markdown_file_path = "%s/docs/index.md" % os.path.dirname(sys.argv[0])
+    spark_root_dir = os.path.dirname(os.path.dirname(__file__))
+    markdown_file_path = os.path.join(spark_root_dir, "sql/docs/index.md")
     generate_sql_markdown(jvm, markdown_file_path)
diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py
new file mode 100644
index 0000000000000..04f5a850c9980
--- /dev/null
+++ b/sql/gen-sql-config-docs.py
@@ -0,0 +1,117 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+from collections import namedtuple
+from textwrap import dedent
+
+# To avoid adding a new direct dependency, we import markdown from within mkdocs.
+from mkdocs.structure.pages import markdown
+from pyspark.java_gateway import launch_gateway
+
+SQLConfEntry = namedtuple(
+    "SQLConfEntry", ["name", "default", "description"])
+
+
+def get_public_sql_configs(jvm):
+    sql_configs = [
+        SQLConfEntry(
+            name=_sql_config._1(),
+            default=_sql_config._2(),
+            description=_sql_config._3(),
+        )
+        for _sql_config in jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
+    ]
+    return sql_configs
+
+
+def generate_sql_configs_table(sql_configs, path):
+    """
+    Generates an HTML table at `path` that lists all public SQL
+    configuration options.
+
+    The table will look something like this:
+
+    ```html
+    <table class="table">
+    <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+
+    <tr>
+        <td><code>spark.sql.adaptive.enabled</code></td>
+        <td>false</td>
+        <td><p>When true, enable adaptive query execution.</p></td>
+    </tr>
+
+    ...
+
+    </table>
+    ```
+    """
+    value_reference_pattern = re.compile(r"^<value of (\S*)>$")
+
+    with open(path, 'w') as f:
+        f.write(dedent(
+            """
+            <table class="table">
+            <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+            """
+        ))
+        for config in sorted(sql_configs, key=lambda x: x.name):
+            if config.default == "<undefined>":
+                default = "(none)"
+            elif config.default.startswith("<value of "):
+                referenced_config_name = value_reference_pattern.match(config.default).group(1)
+                default = "(value of <code>{}</code>)".format(referenced_config_name)
+            else:
+                default = config.default
+
+            if default.startswith("<"):
+                raise Exception(
+                    "Unhandled reference in SQL config docs. Config '{name}' "
+                    "has default '{default}' that looks like an HTML tag."
+                    .format(
+                        name=config.name,
+                        default=config.default,
+                    )
+                )
+
+            f.write(dedent(
+                """
+                <tr>
+                    <td><code>{name}</code></td>
+                    <td>{default}</td>
+                    <td>{description}</td>
+                </tr>
+                """
+                .format(
+                    name=config.name,
+                    default=default,
+                    description=markdown.markdown(config.description),
+                )
+            ))
+        f.write("</table>\n")
+
+
+if __name__ == "__main__":
+    jvm = launch_gateway().jvm
+    sql_configs = get_public_sql_configs(jvm)
+
+    spark_root_dir = os.path.dirname(os.path.dirname(__file__))
+    sql_configs_table_path = os.path.join(spark_root_dir, "docs/sql-configs.html")
+
+    generate_sql_configs_table(sql_configs, path=sql_configs_table_path)

From 00c761d15e2628c8c5e1a9dad5ae122f292eae28 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Sun, 9 Feb 2020 14:18:51 -0800
Subject: [PATCH 0036/1280] [SPARK-30684 ][WEBUI][FollowUp] A new approach for
 SPARK-30684

### What changes were proposed in this pull request?

Simplify the changes for adding metrics description for WholeStageCodegen in https://github.com/apache/spark/pull/27405

### Why are the changes needed?

In https://github.com/apache/spark/pull/27405, the UI changes can be made without using the function `adjustPositionOfOperationName` to adjust the position of operation name and mark as an operation-name class.

I suggest we make simpler changes so that it would be easier for future development.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Manual test with the queries provided in https://github.com/apache/spark/pull/27405
```
sc.parallelize(1 to 10).toDF.sort("value").filter("value > 1").selectExpr("value * 2").show
sc.parallelize(1 to 10).toDF.sort("value").filter("value > 1").selectExpr("value * 2").write.format("json").mode("overwrite").save("/tmp/test_output")
sc.parallelize(1 to 10).toDF.write.format("json").mode("append").save("/tmp/test_output")
```
![image](https://user-images.githubusercontent.com/1097932/74073629-e3f09f00-49bf-11ea-90dc-1edb5ca29e5e.png)

Closes #27490 from gengliangwang/wholeCodegenUI.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit b877aac14657832d1b896ea57e06b0d0fd15ee01)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../sql/execution/ui/static/spark-sql-viz.css |  8 ++---
 .../sql/execution/ui/static/spark-sql-viz.js  | 31 +------------------
 .../sql/execution/ui/SparkPlanGraph.scala     | 18 +++++------
 3 files changed, 11 insertions(+), 46 deletions(-)

diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index eff0142dc523f..20188387c9ba4 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -18,6 +18,7 @@
 #plan-viz-graph .label {
   font-weight: normal;
   text-shadow: none;
+  color: black;
 }
 
 #plan-viz-graph svg g.cluster rect {
@@ -32,13 +33,8 @@
   stroke-width: 1px;
 }
 
-/* This declaration is needed to define the width of rectangles */
-#plan-viz-graph svg text :first-child {
-  font-weight: bold;
-}
-
 /* Highlight the SparkPlan node name */
-#plan-viz-graph svg text .operator-name {
+#plan-viz-graph svg text :first-child {
   font-weight: bold;
 }
 
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index e6ce641a841b3..c8349149439c8 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -34,7 +34,6 @@ function renderPlanViz() {
   preprocessGraphLayout(g);
   var renderer = new dagreD3.render();
   renderer(graph, g);
-  adjustPositionOfOperationName();
 
   // Round corners on rectangles
   svg
@@ -82,7 +81,7 @@ function setupTooltipForSparkPlanNode(nodeId) {
  * and sizes of graph elements, e.g. padding, font style, shape.
  */
 function preprocessGraphLayout(g) {
-  g.graph().ranksep = "90";
+  g.graph().ranksep = "70";
   var nodes = g.nodes();
   for (var i = 0; i < nodes.length; i++) {
       var node = g.node(nodes[i]);
@@ -129,34 +128,6 @@ function resizeSvg(svg) {
      .attr("height", height);
 }
 
-
-/* Helper function to adjust the position of operation name and mark as a operation-name class. */
-function adjustPositionOfOperationName() {
-  $("#plan-viz-graph svg text")
-      .each(function() {
-        var tspans = $(this).find("tspan");
-
-        if (tspans[0].textContent.trim() !== "") {
-          var isOperationNameOnly =
-              $(tspans).filter(function(i, n) {
-                return i !== 0 && n.textContent.trim() !== "";
-              }).length === 0;
-
-          if (isOperationNameOnly) {
-            // If the only text in a node is operation name,
-            // vertically centering the position of operation name.
-            var operationName = tspans[0].textContent;
-            var half = parseInt(tspans.length / 2);
-            tspans[0].textContent = tspans[half].textContent;
-            tspans[half].textContent = operationName;
-            $(tspans[half]).addClass("operator-name");
-          } else {
-            tspans.first().addClass("operator-name");
-          }
-        }
-      });
-}
-
 /* Helper function to convert attributes to numeric values. */
 function toFloat(f) {
   if (f) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index 1e767c3c043c3..d31d77840b802 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -169,20 +169,18 @@ private[ui] class SparkPlanGraphNode(
       metric.name + ": " + value
     }
 
-    // If there are metrics, display each entry in a separate line.
-    // Note: whitespace between two "\n"s is to create an empty line between the name of
-    // SparkPlan and metrics. If removing it, it won't display the empty line in UI.
-    builder ++= "\n \n"
-
     if (values.nonEmpty) {
+      // If there are metrics, display each entry in a separate line.
+      // Note: whitespace between two "\n"s is to create an empty line between the name of
+      // SparkPlan and metrics. If removing it, it won't display the empty line in UI.
+      builder ++= "\n \n"
       builder ++= values.mkString("\n")
+      s"""  $id [label="${StringEscapeUtils.escapeJava(builder.toString())}"];"""
     } else {
-      // A certain level of height is needed for a rect as a node in a sub-graph
-      // to avoid layout collapse for sub-graphs.
-      builder ++= " "
+      // SPARK-30684: when there is no metrics, add empty lines to increase the height of the node,
+      // so that there won't be gaps between an edge and a small node.
+      s"""  $id [labelType="html" label="<br><b>$name</b><br><br>"];"""
     }
-
-    s"""  $id [label="${StringEscapeUtils.escapeJava(builder.toString())}"];"""
   }
 }
 

From 0c5403721cc085c6515a585c39b195f75ef6ac7d Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Sat, 8 Feb 2020 02:47:44 +0800
Subject: [PATCH 0037/1280] [SPARK-30614][SQL] The native ALTER COLUMN syntax
 should change one property at a time

### What changes were proposed in this pull request?

The current ALTER COLUMN syntax allows to change multiple properties at a time:
```
ALTER TABLE table=multipartIdentifier
  (ALTER | CHANGE) COLUMN? column=multipartIdentifier
  (TYPE dataType)?
  (COMMENT comment=STRING)?
  colPosition?
```
The SQL standard (section 11.12) only allows changing one property at a time. This is also true on other recent SQL systems like [snowflake](https://docs.snowflake.net/manuals/sql-reference/sql/alter-table-column.html) and [redshift](https://docs.aws.amazon.com/redshift/latest/dg/r_ALTER_TABLE.html). (credit to cloud-fan)

This PR proposes to change ALTER COLUMN to follow SQL standard, thus allows altering only one column property at a time.

Note that ALTER COLUMN syntax being changed here is newly added in Spark 3.0, so it doesn't affect Spark 2.4 behavior.

### Why are the changes needed?

To follow SQL standard (and other recent SQL systems) behavior.

### Does this PR introduce any user-facing change?

Yes, now the user can update the column properties only one at a time.

For example,
```
ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint COMMENT 'new comment'
```
should be broken into
```
ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint
ALTER TABLE table1 ALTER COLUMN a.b.c COMMENT 'new comment'
```

### How was this patch tested?

Updated existing tests.

Closes #27444 from imback82/alter_column_one_at_a_time.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/parser/SqlBase.g4      | 14 ++--
 .../sql/catalyst/parser/AstBuilder.scala      | 78 ++++++++++---------
 .../sql/catalyst/parser/DDLParserSuite.scala  | 23 +++---
 .../analysis/ResolveSessionCatalog.scala      | 24 +++---
 .../sql-tests/inputs/change-column.sql        | 21 +++--
 .../sql-tests/results/change-column.sql.out   | 46 ++++++++---
 .../spark/sql/connector/AlterTableTests.scala | 13 ----
 .../sql/execution/command/DDLSuite.scala      |  5 +-
 .../command/PlanResolutionSuite.scala         | 47 ++++++++---
 9 files changed, 165 insertions(+), 106 deletions(-)

diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 08d5ff53bf2e2..563ef69b3b8ae 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -158,12 +158,9 @@ statement
         SET TBLPROPERTIES tablePropertyList                            #setTableProperties
     | ALTER (TABLE | VIEW) multipartIdentifier
         UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList             #unsetTableProperties
-    | ALTER TABLE table=multipartIdentifier
+    |ALTER TABLE table=multipartIdentifier
         (ALTER | CHANGE) COLUMN? column=multipartIdentifier
-        (TYPE dataType)? commentSpec? colPosition?                     #alterTableColumn
-    | ALTER TABLE table=multipartIdentifier
-        ALTER COLUMN? column=multipartIdentifier
-        setOrDrop=(SET | DROP) NOT NULL                                #alterColumnNullability
+        alterColumnAction?                                             #alterTableAlterColumn
     | ALTER TABLE table=multipartIdentifier partitionSpec?
         CHANGE COLUMN?
         colName=multipartIdentifier colType colPosition?               #hiveChangeColumn
@@ -983,6 +980,13 @@ number
     | MINUS? BIGDECIMAL_LITERAL       #bigDecimalLiteral
     ;
 
+alterColumnAction
+    : TYPE dataType
+    | commentSpec
+    | colPosition
+    | setOrDrop=(SET | DROP) NOT NULL
+    ;
+
 // When `SQL_standard_keyword_behavior=true`, there are 2 kinds of keywords in Spark SQL.
 // - Reserved keywords:
 //     Keywords that are reserved and can't be used as identifiers for table, view, column,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index e9ad84472904d..6fc65e14868e0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2940,55 +2940,61 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   }
 
   /**
-   * Parse a [[AlterTableAlterColumnStatement]] command.
+   * Parse a [[AlterTableAlterColumnStatement]] command to alter a column's property.
    *
    * For example:
    * {{{
    *   ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint
-   *   ALTER TABLE table1 ALTER COLUMN a.b.c TYPE bigint COMMENT 'new comment'
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c SET NOT NULL
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c DROP NOT NULL
    *   ALTER TABLE table1 ALTER COLUMN a.b.c COMMENT 'new comment'
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c FIRST
+   *   ALTER TABLE table1 ALTER COLUMN a.b.c AFTER x
    * }}}
    */
-  override def visitAlterTableColumn(
-      ctx: AlterTableColumnContext): LogicalPlan = withOrigin(ctx) {
-    val verb = if (ctx.CHANGE != null) "CHANGE" else "ALTER"
-    if (ctx.dataType == null && ctx.commentSpec() == null && ctx.colPosition == null) {
+  override def visitAlterTableAlterColumn(
+      ctx: AlterTableAlterColumnContext): LogicalPlan = withOrigin(ctx) {
+    val action = ctx.alterColumnAction
+    if (action == null) {
+      val verb = if (ctx.CHANGE != null) "CHANGE" else "ALTER"
       operationNotAllowed(
-        s"ALTER TABLE table $verb COLUMN requires a TYPE or a COMMENT or a FIRST/AFTER", ctx)
+        s"ALTER TABLE table $verb COLUMN requires a TYPE, a SET/DROP, a COMMENT, or a FIRST/AFTER",
+        ctx)
+    }
+
+    val dataType = if (action.dataType != null) {
+      Some(typedVisit[DataType](action.dataType))
+    } else {
+      None
+    }
+    val nullable = if (action.setOrDrop != null) {
+      action.setOrDrop.getType match {
+        case SqlBaseParser.SET => Some(false)
+        case SqlBaseParser.DROP => Some(true)
+      }
+    } else {
+      None
+    }
+    val comment = if (action.commentSpec != null) {
+      Some(visitCommentSpec(action.commentSpec()))
+    } else {
+      None
+    }
+    val position = if (action.colPosition != null) {
+      Some(typedVisit[ColumnPosition](action.colPosition))
+    } else {
+      None
     }
 
+    assert(Seq(dataType, nullable, comment, position).count(_.nonEmpty) == 1)
+
     AlterTableAlterColumnStatement(
       visitMultipartIdentifier(ctx.table),
       typedVisit[Seq[String]](ctx.column),
-      dataType = Option(ctx.dataType).map(typedVisit[DataType]),
-      nullable = None,
-      comment = Option(ctx.commentSpec()).map(visitCommentSpec),
-      position = Option(ctx.colPosition).map(typedVisit[ColumnPosition]))
-  }
-
-  /**
-   * Parse a [[AlterTableAlterColumnStatement]] command to change column nullability.
-   *
-   * For example:
-   * {{{
-   *   ALTER TABLE table1 ALTER COLUMN a.b.c SET NOT NULL
-   *   ALTER TABLE table1 ALTER COLUMN a.b.c DROP NOT NULL
-   * }}}
-   */
-  override def visitAlterColumnNullability(ctx: AlterColumnNullabilityContext): LogicalPlan = {
-    withOrigin(ctx) {
-      val nullable = ctx.setOrDrop.getType match {
-        case SqlBaseParser.SET => false
-        case SqlBaseParser.DROP => true
-      }
-      AlterTableAlterColumnStatement(
-        visitMultipartIdentifier(ctx.table),
-        typedVisit[Seq[String]](ctx.column),
-        dataType = None,
-        nullable = Some(nullable),
-        comment = None,
-        position = None)
-    }
+      dataType = dataType,
+      nullable = nullable,
+      comment = comment,
+      position = position)
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 56d52571d1cc3..bc7b51f25b20d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -646,17 +646,18 @@ class DDLParserSuite extends AnalysisTest {
         Some(first())))
   }
 
-  test("alter table: update column type, comment and position") {
-    comparePlans(
-      parsePlan("ALTER TABLE table_name CHANGE COLUMN a.b.c " +
-        "TYPE bigint COMMENT 'new comment' AFTER d"),
-      AlterTableAlterColumnStatement(
-        Seq("table_name"),
-        Seq("a", "b", "c"),
-        Some(LongType),
-        None,
-        Some("new comment"),
-        Some(after("d"))))
+  test("alter table: mutiple property changes are not allowed") {
+    intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
+        "TYPE bigint COMMENT 'new comment'")}
+
+    intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
+        "TYPE bigint COMMENT AFTER d")}
+
+    intercept[ParseException] {
+      parsePlan("ALTER TABLE table_name ALTER COLUMN a.b.c " +
+        "TYPE bigint COMMENT 'new comment' AFTER d")}
   }
 
   test("alter table: SET/DROP NOT NULL") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 486e7f1f84b46..77d549c28aae5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -77,10 +77,6 @@ class ResolveSessionCatalog(
             throw new AnalysisException(
               "ALTER COLUMN with qualified column is only supported with v2 tables.")
           }
-          if (a.dataType.isEmpty) {
-            throw new AnalysisException(
-              "ALTER COLUMN with v1 tables must specify new data type.")
-          }
           if (a.nullable.isDefined) {
             throw new AnalysisException(
               "ALTER COLUMN with v1 tables cannot specify NOT NULL.")
@@ -92,17 +88,27 @@ class ResolveSessionCatalog(
           val builder = new MetadataBuilder
           // Add comment to metadata
           a.comment.map(c => builder.putString("comment", c))
+          val colName = a.column(0)
+          val dataType = a.dataType.getOrElse {
+            v1Table.schema.findNestedField(Seq(colName), resolver = conf.resolver)
+              .map(_._2.dataType)
+              .getOrElse {
+                throw new AnalysisException(
+                  s"ALTER COLUMN cannot find column ${quote(colName)} in v1 table. " +
+                    s"Available: ${v1Table.schema.fieldNames.mkString(", ")}")
+              }
+          }
           // Add Hive type string to metadata.
-          val cleanedDataType = HiveStringType.replaceCharType(a.dataType.get)
-          if (a.dataType.get != cleanedDataType) {
-            builder.putString(HIVE_TYPE_STRING, a.dataType.get.catalogString)
+          val cleanedDataType = HiveStringType.replaceCharType(dataType)
+          if (dataType != cleanedDataType) {
+            builder.putString(HIVE_TYPE_STRING, dataType.catalogString)
           }
           val newColumn = StructField(
-            a.column(0),
+            colName,
             cleanedDataType,
             nullable = true,
             builder.build())
-          AlterTableChangeColumnCommand(tbl.asTableIdentifier, a.column(0), newColumn)
+          AlterTableChangeColumnCommand(tbl.asTableIdentifier, colName, newColumn)
       }.getOrElse {
         val colName = a.column.toArray
         val typeChange = a.dataType.map { newDataType =>
diff --git a/sql/core/src/test/resources/sql-tests/inputs/change-column.sql b/sql/core/src/test/resources/sql-tests/inputs/change-column.sql
index dd2fc660b53e3..2b57891cfcbc5 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/change-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/change-column.sql
@@ -15,29 +15,34 @@ ALTER TABLE test_change CHANGE a TYPE STRING;
 DESC test_change;
 
 -- Change column position (not supported yet)
-ALTER TABLE test_change CHANGE a TYPE INT AFTER b;
-ALTER TABLE test_change CHANGE b TYPE STRING FIRST;
+ALTER TABLE test_change CHANGE a AFTER b;
+ALTER TABLE test_change CHANGE b FIRST;
 DESC test_change;
 
 -- Change column comment
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a';
-ALTER TABLE test_change CHANGE b TYPE STRING COMMENT '#*02?`';
-ALTER TABLE test_change CHANGE c TYPE INT COMMENT '';
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a';
+ALTER TABLE test_change CHANGE b COMMENT '#*02?`';
+ALTER TABLE test_change CHANGE c COMMENT '';
 DESC test_change;
 
 -- Don't change anything.
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a';
+ALTER TABLE test_change CHANGE a TYPE INT;
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a';
 DESC test_change;
 
 -- Change a invalid column
 ALTER TABLE test_change CHANGE invalid_col TYPE INT;
 DESC test_change;
 
+-- Check case insensitivity.
+ALTER TABLE test_change CHANGE A COMMENT 'case insensitivity';
+DESC test_change;
+
 -- Change column can't apply to a temporary/global_temporary view
 CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one";
-ALTER TABLE temp_view CHANGE a TYPE INT COMMENT 'this is column a';
+ALTER TABLE temp_view CHANGE a TYPE INT;
 CREATE GLOBAL TEMPORARY VIEW global_temp_view(a, b) AS SELECT 1, "one";
-ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT COMMENT 'this is column a';
+ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT;
 
 -- DROP TEST TABLE
 DROP TABLE test_change;
diff --git a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
index 5bb00e028c4b7..b1a32ad1f63e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 25
+-- Number of queries: 28
 
 
 -- !query
@@ -27,7 +27,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-Operation not allowed: ALTER TABLE table CHANGE COLUMN requires a TYPE or a COMMENT or a FIRST/AFTER(line 1, pos 0)
+Operation not allowed: ALTER TABLE table CHANGE COLUMN requires a TYPE, a SET/DROP, a COMMENT, or a FIRST/AFTER(line 1, pos 0)
 
 == SQL ==
 ALTER TABLE test_change CHANGE a
@@ -83,7 +83,7 @@ c                   	int
 
 
 -- !query
-ALTER TABLE test_change CHANGE a TYPE INT AFTER b
+ALTER TABLE test_change CHANGE a AFTER b
 -- !query schema
 struct<>
 -- !query output
@@ -92,7 +92,7 @@ ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables.;
 
 
 -- !query
-ALTER TABLE test_change CHANGE b TYPE STRING FIRST
+ALTER TABLE test_change CHANGE b FIRST
 -- !query schema
 struct<>
 -- !query output
@@ -111,7 +111,7 @@ c                   	int
 
 
 -- !query
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a'
 -- !query schema
 struct<>
 -- !query output
@@ -119,7 +119,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE test_change CHANGE b TYPE STRING COMMENT '#*02?`'
+ALTER TABLE test_change CHANGE b COMMENT '#*02?`'
 -- !query schema
 struct<>
 -- !query output
@@ -127,7 +127,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE test_change CHANGE c TYPE INT COMMENT ''
+ALTER TABLE test_change CHANGE c COMMENT ''
 -- !query schema
 struct<>
 -- !query output
@@ -145,7 +145,15 @@ c                   	int
 
 
 -- !query
-ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE test_change CHANGE a TYPE INT
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+ALTER TABLE test_change CHANGE a COMMENT 'this is column a'
 -- !query schema
 struct<>
 -- !query output
@@ -181,6 +189,24 @@ b                   	string              	#*02?`
 c                   	int
 
 
+-- !query
+ALTER TABLE test_change CHANGE A COMMENT 'case insensitivity'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DESC test_change
+-- !query schema
+struct<col_name:string,data_type:string,comment:string>
+-- !query output
+a                   	int                 	case insensitivity  
+b                   	string              	#*02?`              
+c                   	int
+
+
 -- !query
 CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one"
 -- !query schema
@@ -190,7 +216,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE temp_view CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE temp_view CHANGE a TYPE INT
 -- !query schema
 struct<>
 -- !query output
@@ -207,7 +233,7 @@ struct<>
 
 
 -- !query
-ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT COMMENT 'this is column a'
+ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT
 -- !query schema
 struct<>
 -- !query output
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
index 3cdac59c20fc9..420cb01d766a0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
@@ -651,19 +651,6 @@ trait AlterTableTests extends SharedSparkSession {
     }
   }
 
-  test("AlterTable: update column type and comment") {
-    val t = s"${catalogAndNamespace}table_name"
-    withTable(t) {
-      sql(s"CREATE TABLE $t (id int) USING $v2Format")
-      sql(s"ALTER TABLE $t ALTER COLUMN id TYPE bigint COMMENT 'doc'")
-
-      val table = getTableMetadata(t)
-
-      assert(table.name === fullTableName(t))
-      assert(table.schema === StructType(Seq(StructField("id", LongType).withComment("doc"))))
-    }
-  }
-
   test("AlterTable: update nested column comment") {
     val t = s"${catalogAndNamespace}table_name"
     withTable(t) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 913cd80a24c6e..31e00781ae6b4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -188,7 +188,7 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSparkSession {
     withTable("t") {
       sql("CREATE TABLE t(i INT) USING parquet")
       val e = intercept[AnalysisException] {
-        sql("ALTER TABLE t ALTER COLUMN i TYPE INT FIRST")
+        sql("ALTER TABLE t ALTER COLUMN i FIRST")
       }
       assert(e.message.contains("ALTER COLUMN ... FIRST | ALTER is only supported with v2 tables"))
     }
@@ -1786,7 +1786,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       column.map(_.metadata).getOrElse(Metadata.empty)
     }
     // Ensure that change column will preserve other metadata fields.
-    sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 TYPE INT COMMENT 'this is col1'")
+    sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 TYPE INT")
+    sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 COMMENT 'this is col1'")
     assert(getMetadata("col1").getString("key") == "value")
     assert(getMetadata("col1").getString("comment") == "this is col1")
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 88f30353cce94..d439e5b1cd651 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -1013,27 +1013,29 @@ class PlanResolutionSuite extends AnalysisTest {
     Seq("v1Table" -> true, "v2Table" -> false, "testcat.tab" -> false).foreach {
       case (tblName, useV1Command) =>
         val sql1 = s"ALTER TABLE $tblName ALTER COLUMN i TYPE bigint"
-        val sql2 = s"ALTER TABLE $tblName ALTER COLUMN i TYPE bigint COMMENT 'new comment'"
-        val sql3 = s"ALTER TABLE $tblName ALTER COLUMN i COMMENT 'new comment'"
+        val sql2 = s"ALTER TABLE $tblName ALTER COLUMN i COMMENT 'new comment'"
 
         val parsed1 = parseAndResolve(sql1)
         val parsed2 = parseAndResolve(sql2)
 
         val tableIdent = TableIdentifier(tblName, None)
         if (useV1Command) {
+          val oldColumn = StructField("i", IntegerType)
           val newColumn = StructField("i", LongType)
           val expected1 = AlterTableChangeColumnCommand(
             tableIdent, "i", newColumn)
           val expected2 = AlterTableChangeColumnCommand(
-            tableIdent, "i", newColumn.withComment("new comment"))
+            tableIdent, "i", oldColumn.withComment("new comment"))
 
           comparePlans(parsed1, expected1)
           comparePlans(parsed2, expected2)
 
+          val sql3 = s"ALTER TABLE $tblName ALTER COLUMN j COMMENT 'new comment'"
           val e1 = intercept[AnalysisException] {
             parseAndResolve(sql3)
           }
-          assert(e1.getMessage.contains("ALTER COLUMN with v1 tables must specify new data type"))
+          assert(e1.getMessage.contains(
+            "ALTER COLUMN cannot find column j in v1 table. Available: i, s"))
 
           val sql4 = s"ALTER TABLE $tblName ALTER COLUMN a.b.c TYPE bigint"
           val e2 = intercept[AnalysisException] {
@@ -1051,8 +1053,6 @@ class PlanResolutionSuite extends AnalysisTest {
           val parsed5 = parseAndResolve(sql5)
           comparePlans(parsed5, expected5)
         } else {
-          val parsed3 = parseAndResolve(sql3)
-
           parsed1 match {
             case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
               assert(changes == Seq(
@@ -1063,18 +1063,41 @@ class PlanResolutionSuite extends AnalysisTest {
           parsed2 match {
             case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
               assert(changes == Seq(
-                TableChange.updateColumnType(Array("i"), LongType),
                 TableChange.updateColumnComment(Array("i"), "new comment")))
             case _ => fail("expect AlterTable")
           }
+        }
+    }
+  }
 
-          parsed3 match {
-            case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
-              assert(changes == Seq(
-                TableChange.updateColumnComment(Array("i"), "new comment")))
-            case _ => fail("expect AlterTable")
+  test("alter table: alter column action is not specified") {
+    val e = intercept[AnalysisException] {
+      parseAndResolve("ALTER TABLE v1Table ALTER COLUMN i")
+    }
+    assert(e.getMessage.contains(
+      "ALTER TABLE table ALTER COLUMN requires a TYPE, a SET/DROP, a COMMENT, or a FIRST/AFTER"))
+  }
+
+  test("alter table: alter column case sensitivity for v1 table") {
+    val tblName = "v1Table"
+    Seq(true, false).foreach { caseSensitive =>
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        val sql = s"ALTER TABLE $tblName ALTER COLUMN I COMMENT 'new comment'"
+        if (caseSensitive) {
+          val e = intercept[AnalysisException] {
+            parseAndResolve(sql)
           }
+          assert(e.getMessage.contains(
+            "ALTER COLUMN cannot find column I in v1 table. Available: i, s"))
+        } else {
+          val actual = parseAndResolve(sql)
+          val expected = AlterTableChangeColumnCommand(
+            TableIdentifier(tblName, None),
+            "I",
+            StructField("I", IntegerType).withComment("new comment"))
+          comparePlans(actual, expected)
         }
+      }
     }
   }
 

From b012ff72b64f08e3fcb9e4fbcf04b874711cf5b6 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Mon, 10 Feb 2020 16:23:44 +0900
Subject: [PATCH 0038/1280] [SPARK-30592][SQL][FOLLOWUP] Add some round-trip
 test cases

### What changes were proposed in this pull request?

Add round-trip tests for CSV and JSON functions as  https://github.com/apache/spark/pull/27317#discussion_r376745135 asked.

### Why are the changes needed?

improve test coverage

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

add uts

Closes #27510 from yaooqinn/SPARK-30592-F.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 58b9ca1e6f7768b23e752dabc30468c06d0e1c57)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../resources/sql-tests/inputs/interval.sql   | 14 +++++--
 .../sql-tests/results/ansi/interval.sql.out   | 38 ++++++++-----------
 .../sql-tests/results/interval.sql.out        | 38 ++++++++-----------
 3 files changed, 40 insertions(+), 50 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index fb6c485f619ae..a4e621e9639d4 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -222,7 +222,13 @@ select a * 1.1 from values (interval '-2147483648 months', interval '2147483647
 select a / 0.5 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
 
 -- interval support for csv and json functions
-SELECT from_csv('1, 1 day', 'a INT, b interval');
-SELECT to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute));
-SELECT from_json('{"a":"1 days"}', 'a interval');
-SELECT to_json(map('a', interval 25 month 100 day 130 minute));
+SELECT
+  from_csv('1, 1 day', 'a INT, b interval'),
+  to_csv(from_csv('1, 1 day', 'a INT, b interval')),
+  to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)),
+  from_csv(to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)), 'a interval, b interval');
+SELECT
+  from_json('{"a":"1 days"}', 'a interval'),
+  to_json(from_json('{"a":"1 days"}', 'a interval')),
+  to_json(map('a', interval 25 month 100 day 130 minute)),
+  from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index f37049064d989..7fdb4c53d1dcb 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 101
+-- Number of queries: 99
 
 
 -- !query
@@ -988,32 +988,24 @@ integer overflow
 
 
 -- !query
-SELECT from_csv('1, 1 day', 'a INT, b interval')
--- !query schema
-struct<from_csv(1, 1 day):struct<a:int,b:interval>>
--- !query output
-{"a":1,"b":1 days}
-
-
--- !query
-SELECT to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute))
--- !query schema
-struct<to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string>
--- !query output
-2 years 8 months,1 hours 10 minutes
-
-
--- !query
-SELECT from_json('{"a":"1 days"}', 'a interval')
+SELECT
+  from_csv('1, 1 day', 'a INT, b interval'),
+  to_csv(from_csv('1, 1 day', 'a INT, b interval')),
+  to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)),
+  from_csv(to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)), 'a interval, b interval')
 -- !query schema
-struct<from_json({"a":"1 days"}):struct<a:interval>>
+struct<from_csv(1, 1 day):struct<a:int,b:interval>,to_csv(from_csv(1, 1 day)):string,to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string,from_csv(to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes'))):struct<a:interval,b:interval>>
 -- !query output
-{"a":1 days}
+{"a":1,"b":1 days}	1,1 days	2 years 8 months,1 hours 10 minutes	{"a":2 years 8 months,"b":1 hours 10 minutes}
 
 
 -- !query
-SELECT to_json(map('a', interval 25 month 100 day 130 minute))
+SELECT
+  from_json('{"a":"1 days"}', 'a interval'),
+  to_json(from_json('{"a":"1 days"}', 'a interval')),
+  to_json(map('a', interval 25 month 100 day 130 minute)),
+  from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval')
 -- !query schema
-struct<to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string>
+struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
 -- !query output
-{"a":"2 years 1 months 100 days 2 hours 10 minutes"}
+{"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 94b4f15815ca5..3c4b4301d0025 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 101
+-- Number of queries: 99
 
 
 -- !query
@@ -969,32 +969,24 @@ integer overflow
 
 
 -- !query
-SELECT from_csv('1, 1 day', 'a INT, b interval')
--- !query schema
-struct<from_csv(1, 1 day):struct<a:int,b:interval>>
--- !query output
-{"a":1,"b":1 days}
-
-
--- !query
-SELECT to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute))
--- !query schema
-struct<to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string>
--- !query output
-2 years 8 months,1 hours 10 minutes
-
-
--- !query
-SELECT from_json('{"a":"1 days"}', 'a interval')
+SELECT
+  from_csv('1, 1 day', 'a INT, b interval'),
+  to_csv(from_csv('1, 1 day', 'a INT, b interval')),
+  to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)),
+  from_csv(to_csv(named_struct('a', interval 32 month, 'b', interval 70 minute)), 'a interval, b interval')
 -- !query schema
-struct<from_json({"a":"1 days"}):struct<a:interval>>
+struct<from_csv(1, 1 day):struct<a:int,b:interval>,to_csv(from_csv(1, 1 day)):string,to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes')):string,from_csv(to_csv(named_struct(a, INTERVAL '2 years 8 months', b, INTERVAL '1 hours 10 minutes'))):struct<a:interval,b:interval>>
 -- !query output
-{"a":1 days}
+{"a":1,"b":1 days}	1,1 days	2 years 8 months,1 hours 10 minutes	{"a":2 years 8 months,"b":1 hours 10 minutes}
 
 
 -- !query
-SELECT to_json(map('a', interval 25 month 100 day 130 minute))
+SELECT
+  from_json('{"a":"1 days"}', 'a interval'),
+  to_json(from_json('{"a":"1 days"}', 'a interval')),
+  to_json(map('a', interval 25 month 100 day 130 minute)),
+  from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval')
 -- !query schema
-struct<to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string>
+struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
 -- !query output
-{"a":"2 years 1 months 100 days 2 hours 10 minutes"}
+{"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}

From dbf17f194d625c6bb8d8fcba5f4913e84000ab6f Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Mon, 10 Feb 2020 19:04:49 +0800
Subject: [PATCH 0039/1280] [SPARK-30757][SQL][DOC] Update the doc on
 TableCatalog.alterTable's behavior

### What changes were proposed in this pull request?

This PR updates the documentation on `TableCatalog.alterTable`s behavior on the order by which the requested changes are applied. It now explicitly mentions that the changes are applied in the order given.

### Why are the changes needed?

The current documentation on `TableCatalog.alterTable` doesn't mention which order the requested changes are applied. It will be useful to explicitly document this behavior so that the user can expect the behavior. For example, `REPLACE COLUMNS` needs to delete columns before adding new columns, and if the order is guaranteed by `alterTable`, it's much easier to work with the catalog API.

### Does this PR introduce any user-facing change?

Yes, document change.

### How was this patch tested?

Not added (doc changes).

Closes #27496 from imback82/catalog_table_alter_table.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 70e545a94d47afb2848c24e81c908d28d41016da)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/connector/catalog/TableCatalog.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
index a69b23bf84d0c..2f102348ec517 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
@@ -134,6 +134,8 @@ Table createTable(
    * Implementations may reject the requested changes. If any change is rejected, none of the
    * changes should be applied to the table.
    * <p>
+   * The requested changes must be applied in the order given.
+   * <p>
    * If the catalog supports views and contains a view for the identifier and not a table, this
    * must throw {@link NoSuchTableException}.
    *

From 7c10a6664e06fc00f9be9704f473135b2cf3e48b Mon Sep 17 00:00:00 2001
From: jiake <ke.a.jia@intel.com>
Date: Mon, 10 Feb 2020 21:48:00 +0800
Subject: [PATCH 0040/1280] [SPARK-30719][SQL] Add unit test to verify the log
 warning print when intentionally skip AQE

### What changes were proposed in this pull request?

This is a follow up in [#27452](https://github.com/apache/spark/pull/27452).
Add a unit test to verify whether the log warning is print when intentionally skip AQE.

### Why are the changes needed?

Add unit test

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

adding unit test

Closes #27515 from JkSelf/aqeLoggingWarningTest.

Authored-by: jiake <ke.a.jia@intel.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5a240603fd920e3cb5d9ef49c31d46df8a630d8c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/AdaptiveQueryExecSuite.scala         | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 96e977221e512..a2071903bea7e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -789,4 +789,19 @@ class AdaptiveQueryExecSuite
       assert(plan.isInstanceOf[AdaptiveSparkPlanExec])
     }
   }
+
+  test("SPARK-30719: do not log warning if intentionally skip AQE") {
+    val testAppender = new LogAppender("aqe logging warning test when skip")
+    withLogAppender(testAppender) {
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+        val plan = sql("SELECT * FROM testData").queryExecution.executedPlan
+        assert(!plan.isInstanceOf[AdaptiveSparkPlanExec])
+      }
+    }
+    assert(!testAppender.loggingEvents
+      .exists(msg => msg.getRenderedMessage.contains(
+        s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is" +
+        s" enabled but is not supported for")))
+  }
 }

From fd6d1b400630d7fee6d031e6de1fccfb4993778b Mon Sep 17 00:00:00 2001
From: Eric Wu <492960551@qq.com>
Date: Mon, 10 Feb 2020 23:41:39 +0800
Subject: [PATCH 0041/1280] [SPARK-30326][SQL] Raise exception if analyzer
 exceed max iterations

### What changes were proposed in this pull request?
Enhance RuleExecutor strategy to take different actions when exceeding max iterations. And raise exception if analyzer exceed max iterations.

### Why are the changes needed?
Currently, both analyzer and optimizer just log warning message if rule execution exceed max iterations. They should have different behavior. Analyzer should raise exception to indicates the plan is not fixed after max iterations, while optimizer just log warning to keep the current plan. This is more feasible after SPARK-30138 was introduced.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Add test in AnalysisSuite

Closes #26977 from Eric5553/EnhanceMaxIterations.

Authored-by: Eric Wu <492960551@qq.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b2011a295bd78b3693a516e049e90250366b8f52)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 10 ++++++-
 .../sql/catalyst/optimizer/Optimizer.scala    |  5 +++-
 .../sql/catalyst/rules/RuleExecutor.scala     | 27 ++++++++++++++++---
 .../sql/catalyst/analysis/AnalysisSuite.scala | 25 ++++++++++++++++-
 4 files changed, 60 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 75f1aa7185ef3..ce82b3b567b54 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -176,7 +176,15 @@ class Analyzer(
 
   def resolver: Resolver = conf.resolver
 
-  protected val fixedPoint = FixedPoint(maxIterations)
+  /**
+   * If the plan cannot be resolved within maxIterations, analyzer will throw exception to inform
+   * user to increase the value of SQLConf.ANALYZER_MAX_ITERATIONS.
+   */
+  protected val fixedPoint =
+    FixedPoint(
+      maxIterations,
+      errorOnExceed = true,
+      maxIterationsSetting = SQLConf.ANALYZER_MAX_ITERATIONS.key)
 
   /**
    * Override to provide additional rules for the "Resolution" batch.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 0fdf6b022d885..c90117b4fbbbc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -53,7 +53,10 @@ abstract class Optimizer(catalogManager: CatalogManager)
       "PartitionPruning",
       "Extract Python UDFs")
 
-  protected def fixedPoint = FixedPoint(SQLConf.get.optimizerMaxIterations)
+  protected def fixedPoint =
+    FixedPoint(
+      SQLConf.get.optimizerMaxIterations,
+      maxIterationsSetting = SQLConf.OPTIMIZER_MAX_ITERATIONS.key)
 
   /**
    * Defines the default rule batches in the Optimizer.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
index 287ae0e8e9f67..da5242bee28e1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
@@ -45,7 +45,17 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
    * An execution strategy for rules that indicates the maximum number of executions. If the
    * execution reaches fix point (i.e. converge) before maxIterations, it will stop.
    */
-  abstract class Strategy { def maxIterations: Int }
+  abstract class Strategy {
+
+    /** The maximum number of executions. */
+    def maxIterations: Int
+
+    /** Whether to throw exception when exceeding the maximum number. */
+    def errorOnExceed: Boolean = false
+
+    /** The key of SQLConf setting to tune maxIterations */
+    def maxIterationsSetting: String = null
+  }
 
   /** A strategy that is run once and idempotent. */
   case object Once extends Strategy { val maxIterations = 1 }
@@ -54,7 +64,10 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
    * A strategy that runs until fix point or maxIterations times, whichever comes first.
    * Especially, a FixedPoint(1) batch is supposed to run only once.
    */
-  case class FixedPoint(maxIterations: Int) extends Strategy
+  case class FixedPoint(
+    override val maxIterations: Int,
+    override val errorOnExceed: Boolean = false,
+    override val maxIterationsSetting: String = null) extends Strategy
 
   /** A batch of rules. */
   protected case class Batch(name: String, strategy: Strategy, rules: Rule[TreeType]*)
@@ -155,8 +168,14 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
         if (iteration > batch.strategy.maxIterations) {
           // Only log if this is a rule that is supposed to run more than once.
           if (iteration != 2) {
-            val message = s"Max iterations (${iteration - 1}) reached for batch ${batch.name}"
-            if (Utils.isTesting) {
+            val endingMsg = if (batch.strategy.maxIterationsSetting == null) {
+              "."
+            } else {
+              s", please set '${batch.strategy.maxIterationsSetting}' to a larger value."
+            }
+            val message = s"Max iterations (${iteration - 1}) reached for batch ${batch.name}" +
+              s"$endingMsg"
+            if (Utils.isTesting || batch.strategy.errorOnExceed) {
               throw new TreeNodeException(curPlan, message, null)
             } else {
               logWarning(message)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index c747d394b1bc2..d38513319388b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -25,9 +25,10 @@ import org.scalatest.Matchers
 
 import org.apache.spark.api.python.PythonEvalType
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
+import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Sum}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
@@ -745,4 +746,26 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       CollectMetrics("evt1", sumWithFilter :: Nil, testRelation),
       "aggregates with filter predicate are not allowed" :: Nil)
   }
+
+  test("Analysis exceed max iterations") {
+    // RuleExecutor only throw exception or log warning when the rule is supposed to run
+    // more than once.
+    val maxIterations = 2
+    val conf = new SQLConf().copy(SQLConf.ANALYZER_MAX_ITERATIONS -> maxIterations)
+    val testAnalyzer = new Analyzer(
+      new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+
+    val plan = testRelation2.select(
+      $"a" / Literal(2) as "div1",
+      $"a" / $"b" as "div2",
+      $"a" / $"c" as "div3",
+      $"a" / $"d" as "div4",
+      $"e" / $"e" as "div5")
+
+    val message = intercept[TreeNodeException[LogicalPlan]] {
+      testAnalyzer.execute(plan)
+    }.getMessage
+    assert(message.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+      s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+  }
 }

From ff395a39a5b10a7e71ef61813084bd3cf120280c Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Sun, 9 Feb 2020 19:45:16 -0800
Subject: [PATCH 0042/1280] Revert "[SPARK-29721][SQL] Prune unnecessary nested
 fields from Generate without Project

This reverts commit a0e63b61e7c5d55ae2a9213b95ab1e87ac7c203c.

### What changes were proposed in this pull request?

This reverts the patch at #26978 based on gatorsmile's suggestion.

### Why are the changes needed?

Original patch #26978 has not considered a corner case. We may need to put more time on ensuring we can cover all cases.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Unit test.

Closes #27504 from viirya/revert-SPARK-29721.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
---
 .../optimizer/NestedColumnAliasing.scala      | 47 -------------------
 .../sql/catalyst/optimizer/Optimizer.scala    | 43 ++++++++++-------
 .../datasources/SchemaPruningSuite.scala      | 32 -------------
 3 files changed, 25 insertions(+), 97 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index ea85014a37bd8..43a6006f9b5c0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -155,53 +155,6 @@ object NestedColumnAliasing {
     case MapType(keyType, valueType, _) => totalFieldNum(keyType) + totalFieldNum(valueType)
     case _ => 1 // UDT and others
   }
-}
-
-/**
- * This prunes unnessary nested columns from `Generate` and optional `Project` on top
- * of it.
- */
-object GeneratorNestedColumnAliasing {
-  def unapply(plan: LogicalPlan): Option[LogicalPlan] = plan match {
-    // Either `nestedPruningOnExpressions` or `nestedSchemaPruningEnabled` is enabled, we
-    // need to prune nested columns through Project and under Generate. The difference is
-    // when `nestedSchemaPruningEnabled` is on, nested columns will be pruned further at
-    // file format readers if it is supported.
-    case Project(projectList, g: Generate) if (SQLConf.get.nestedPruningOnExpressions ||
-        SQLConf.get.nestedSchemaPruningEnabled) && canPruneGenerator(g.generator) =>
-      // On top on `Generate`, a `Project` that might have nested column accessors.
-      // We try to get alias maps for both project list and generator's children expressions.
-      NestedColumnAliasing.getAliasSubMap(projectList ++ g.generator.children).map {
-        case (nestedFieldToAlias, attrToAliases) =>
-          val newChild = pruneGenerate(g, nestedFieldToAlias, attrToAliases)
-          Project(NestedColumnAliasing.getNewProjectList(projectList, nestedFieldToAlias), newChild)
-      }
-
-    case g: Generate if SQLConf.get.nestedSchemaPruningEnabled &&
-        canPruneGenerator(g.generator) =>
-      NestedColumnAliasing.getAliasSubMap(g.generator.children).map {
-        case (nestedFieldToAlias, attrToAliases) =>
-          pruneGenerate(g, nestedFieldToAlias, attrToAliases)
-      }
-
-    case _ =>
-      None
-  }
-
-  private def pruneGenerate(
-      g: Generate,
-      nestedFieldToAlias: Map[ExtractValue, Alias],
-      attrToAliases: Map[ExprId, Seq[Alias]]): LogicalPlan = {
-    val newGenerator = g.generator.transform {
-      case f: ExtractValue if nestedFieldToAlias.contains(f) =>
-        nestedFieldToAlias(f).toAttribute
-    }.asInstanceOf[Generator]
-
-    // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
-    val newGenerate = g.copy(generator = newGenerator)
-
-    NestedColumnAliasing.replaceChildrenWithAliases(newGenerate, attrToAliases)
-  }
 
   /**
    * This is a while-list for pruning nested fields at `Generator`.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index c90117b4fbbbc..08acac18f48bb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -601,24 +601,31 @@ object ColumnPruning extends Rule[LogicalPlan] {
       s.copy(child = prunedChild(child, s.references))
 
     // prune unrequired references
-    case p @ Project(_, g: Generate) =>
-      val currP = if (p.references != g.outputSet) {
-        val requiredAttrs = p.references -- g.producedAttributes ++ g.generator.references
-        val newChild = prunedChild(g.child, requiredAttrs)
-        val unrequired = g.generator.references -- p.references
-        val unrequiredIndices = newChild.output.zipWithIndex.filter(t => unrequired.contains(t._1))
-          .map(_._2)
-        p.copy(child = g.copy(child = newChild, unrequiredChildIndex = unrequiredIndices))
-      } else {
-        p
-      }
-      // If we can prune nested column on Project + Generate, do it now.
-      // Otherwise by transforming down to Generate, it could be pruned individually,
-      // and causes nested column on top Project unable to resolve.
-      GeneratorNestedColumnAliasing.unapply(currP).getOrElse(currP)
-
-    // prune unrequired nested fields from `Generate`.
-    case GeneratorNestedColumnAliasing(p) => p
+    case p @ Project(_, g: Generate) if p.references != g.outputSet =>
+      val requiredAttrs = p.references -- g.producedAttributes ++ g.generator.references
+      val newChild = prunedChild(g.child, requiredAttrs)
+      val unrequired = g.generator.references -- p.references
+      val unrequiredIndices = newChild.output.zipWithIndex.filter(t => unrequired.contains(t._1))
+        .map(_._2)
+      p.copy(child = g.copy(child = newChild, unrequiredChildIndex = unrequiredIndices))
+
+    // prune unrequired nested fields
+    case p @ Project(projectList, g: Generate) if SQLConf.get.nestedPruningOnExpressions &&
+        NestedColumnAliasing.canPruneGenerator(g.generator) =>
+      NestedColumnAliasing.getAliasSubMap(projectList ++ g.generator.children).map {
+        case (nestedFieldToAlias, attrToAliases) =>
+          val newGenerator = g.generator.transform {
+            case f: ExtractValue if nestedFieldToAlias.contains(f) =>
+              nestedFieldToAlias(f).toAttribute
+          }.asInstanceOf[Generator]
+
+          // Defer updating `Generate.unrequiredChildIndex` to next round of `ColumnPruning`.
+          val newGenerate = g.copy(generator = newGenerator)
+
+          val newChild = NestedColumnAliasing.replaceChildrenWithAliases(newGenerate, attrToAliases)
+
+          Project(NestedColumnAliasing.getNewProjectList(projectList, nestedFieldToAlias), newChild)
+      }.getOrElse(p)
 
     // Eliminate unneeded attributes from right side of a Left Existence Join.
     case j @ Join(_, right, LeftExistence(_), _, _) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
index 5977e867f788a..a3d4905e82cee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -301,38 +301,6 @@ abstract class SchemaPruningSuite
     checkAnswer(query, Row("Y.", 1) :: Row("X.", 1) :: Row(null, 2) :: Row(null, 2) :: Nil)
   }
 
-  testSchemaPruning("select explode of nested field of array of struct") {
-    // Config combinations
-    val configs = Seq((true, true), (true, false), (false, true), (false, false))
-
-    configs.foreach { case (nestedPruning, nestedPruningOnExpr) =>
-      withSQLConf(
-          SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key -> nestedPruning.toString,
-          SQLConf.NESTED_PRUNING_ON_EXPRESSIONS.key -> nestedPruningOnExpr.toString) {
-        val query1 = spark.table("contacts")
-          .select(explode(col("friends.first")))
-        if (nestedPruning) {
-          // If `NESTED_SCHEMA_PRUNING_ENABLED` is enabled,
-          // even disabling `NESTED_PRUNING_ON_EXPRESSIONS`,
-          // nested schema is still pruned at scan node.
-          checkScan(query1, "struct<friends:array<struct<first:string>>>")
-        } else {
-          checkScan(query1, "struct<friends:array<struct<first:string,middle:string,last:string>>>")
-        }
-        checkAnswer(query1, Row("Susan") :: Nil)
-
-        val query2 = spark.table("contacts")
-          .select(explode(col("friends.first")), col("friends.middle"))
-        if (nestedPruning) {
-          checkScan(query2, "struct<friends:array<struct<first:string,middle:string>>>")
-        } else {
-          checkScan(query2, "struct<friends:array<struct<first:string,middle:string,last:string>>>")
-        }
-        checkAnswer(query2, Row("Susan", Array("Z.")) :: Nil)
-      }
-    }
-  }
-
   protected def testSchemaPruning(testName: String)(testThunk: => Unit): Unit = {
     test(s"Spark vectorized reader - without partition data column - $testName") {
       withSQLConf(vectorizedReaderEnabledKey -> "true") {

From d5e4f2e262e78e4a22e6b881bb721c7a8c0a5823 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Mon, 10 Feb 2020 10:45:00 -0800
Subject: [PATCH 0043/1280] [SPARK-27946][SQL][FOLLOW-UP] Change doc and error
 message for SHOW CREATE TABLE

### What changes were proposed in this pull request?

This is a follow-up for #24938 to tweak error message and migration doc.

### Why are the changes needed?

Making user know workaround if SHOW CREATE TABLE doesn't work for some Hive tables.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Existing unit tests.

Closes #27505 from viirya/SPARK-27946-followup.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Liang-Chi Hsieh <liangchi@uber.com>
(cherry picked from commit acfdb46a60fc06dac0af55951492d74b7073f546)
Signed-off-by: Liang-Chi Hsieh <liangchi@uber.com>
---
 docs/sql-migration-guide.md                               | 2 +-
 .../org/apache/spark/sql/execution/command/tables.scala   | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index be0fe32ded99b..26eb5838892b4 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -326,7 +326,7 @@ license: |
 
   - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result.
 
-  - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
+  - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For generating Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
 
   - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3. This may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore.
   For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 468ca505cce1f..90dbdf5515d4d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -1076,7 +1076,9 @@ case class ShowCreateTableCommand(table: TableIdentifier)
             "Failed to execute SHOW CREATE TABLE against table " +
               s"${tableMetadata.identifier}, which is created by Hive and uses the " +
               "following unsupported feature(s)\n" +
-              tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n")
+              tableMetadata.unsupportedFeatures.map(" - " + _).mkString("\n") + ". " +
+              s"Please use `SHOW CREATE TABLE ${tableMetadata.identifier} AS SERDE` " +
+              "to show Hive DDL instead."
           )
         }
 
@@ -1086,7 +1088,9 @@ case class ShowCreateTableCommand(table: TableIdentifier)
 
         if ("true".equalsIgnoreCase(tableMetadata.properties.getOrElse("transactional", "false"))) {
           throw new AnalysisException(
-            "SHOW CREATE TABLE doesn't support transactional Hive table")
+            "SHOW CREATE TABLE doesn't support transactional Hive table. " +
+              s"Please use `SHOW CREATE TABLE ${tableMetadata.identifier} AS SERDE` " +
+              "to show Hive DDL instead.")
         }
 
         convertTableMetadata(tableMetadata)

From 3038a81ecdb526b01e80eeb34a7eacc6ac48d360 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Mon, 10 Feb 2020 22:16:25 +0100
Subject: [PATCH 0044/1280] [SPARK-30556][SQL][FOLLOWUP] Reset the status
 changed in SQLExecution.withThreadLocalCaptured

### What changes were proposed in this pull request?
Follow up for #27267, reset the status changed in SQLExecution.withThreadLocalCaptured.

### Why are the changes needed?
For code safety.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UT.

Closes #27516 from xuanyuanking/SPARK-30556-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit a6b91d2bf727e175d0e175295001db85647539b1)
Signed-off-by: herman <herman@databricks.com>
---
 .../apache/spark/sql/execution/SQLExecution.scala    | 12 +++++++++++-
 .../sql/internal/ExecutorSideSQLConfSuite.scala      | 10 ++++++----
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
index 995d94ef5eac7..9f177819f6ea7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
@@ -177,9 +177,19 @@ object SQLExecution {
     val sc = sparkSession.sparkContext
     val localProps = Utils.cloneProperties(sc.getLocalProperties)
     Future {
+      val originalSession = SparkSession.getActiveSession
+      val originalLocalProps = sc.getLocalProperties
       SparkSession.setActiveSession(activeSession)
       sc.setLocalProperties(localProps)
-      body
+      val res = body
+      // reset active session and local props.
+      sc.setLocalProperties(originalLocalProps)
+      if (originalSession.nonEmpty) {
+        SparkSession.setActiveSession(originalSession.get)
+      } else {
+        SparkSession.clearActiveSession()
+      }
+      res
     }(exec)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
index 0cc658c499615..46d0c64592a00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.internal
 
+import java.util.UUID
+
 import org.scalatest.Assertions._
 
 import org.apache.spark.{SparkException, SparkFunSuite, TaskContext}
@@ -144,16 +146,16 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
         }
 
         // set local configuration and assert
-        val confValue1 = "e"
+        val confValue1 = UUID.randomUUID().toString()
         createDataframe(confKey, confValue1).createOrReplaceTempView("m")
         spark.sparkContext.setLocalProperty(confKey, confValue1)
-        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM m)").collect.size == 1)
+        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM m)").collect().length == 1)
 
         // change the conf value and assert again
-        val confValue2 = "f"
+        val confValue2 = UUID.randomUUID().toString()
         createDataframe(confKey, confValue2).createOrReplaceTempView("n")
         spark.sparkContext.setLocalProperty(confKey, confValue2)
-        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM n)").collect().size == 1)
+        assert(sql("SELECT * FROM l WHERE EXISTS (SELECT * FROM n)").collect().length == 1)
       }
     }
   }

From 45d834cb8cc2c30f902d0dec1cdf561b993521d0 Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Mon, 10 Feb 2020 14:26:14 -0800
Subject: [PATCH 0045/1280] [SPARK-30779][SS] Fix some API issues found when
 reviewing Structured Streaming API docs

### What changes were proposed in this pull request?

- Fix the scope of `Logging.initializeForcefully` so that it doesn't appear in subclasses' public methods. Right now, `sc.initializeForcefully(false, false)` is allowed to called.
- Don't show classes under `org.apache.spark.internal` package in API docs.
- Add missing `since` annotation.
- Fix the scope of `ArrowUtils` to remove it from the API docs.

### Why are the changes needed?

Avoid leaking APIs unintentionally in Spark 3.0.0.

### Does this PR introduce any user-facing change?

No. All these changes are to avoid leaking APIs unintentionally in Spark 3.0.0.

### How was this patch tested?

Manually generated the API docs and verified the above issues have been fixed.

Closes #27528 from zsxwing/audit-ss-apis.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
---
 core/src/main/scala/org/apache/spark/internal/Logging.scala     | 2 +-
 project/SparkBuild.scala                                        | 1 +
 .../sql/connector/read/streaming/ContinuousPartitionReader.java | 2 ++
 .../read/streaming/ContinuousPartitionReaderFactory.java        | 2 ++
 .../spark/sql/connector/read/streaming/ContinuousStream.java    | 2 ++
 .../spark/sql/connector/read/streaming/MicroBatchStream.java    | 2 ++
 .../org/apache/spark/sql/connector/read/streaming/Offset.java   | 2 ++
 .../spark/sql/connector/read/streaming/PartitionOffset.java     | 2 ++
 .../apache/spark/sql/connector/read/streaming/ReadLimit.java    | 1 +
 .../spark/sql/connector/read/streaming/SparkDataStream.java     | 2 ++
 .../connector/write/streaming/StreamingDataWriterFactory.java   | 2 ++
 .../spark/sql/connector/write/streaming/StreamingWrite.java     | 2 ++
 .../src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala   | 2 +-
 13 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala
index 2e4846bec2db4..0c1d9635b6535 100644
--- a/core/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -117,7 +117,7 @@ trait Logging {
   }
 
   // For testing
-  def initializeForcefully(isInterpreter: Boolean, silent: Boolean): Unit = {
+  private[spark] def initializeForcefully(isInterpreter: Boolean, silent: Boolean): Unit = {
     initializeLogging(isInterpreter, silent)
   }
 
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 707c31d2248eb..9d0af3aa8c1b6 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -819,6 +819,7 @@ object Unidoc {
       .map(_.filterNot(_.getName.contains("$")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/deploy")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/examples")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/internal")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/memory")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/network")))
       .map(_.filterNot(f =>
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
index 8bd5273bb7d8e..c2ad9ec244a0d 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReader.java
@@ -22,6 +22,8 @@
 
 /**
  * A variation on {@link PartitionReader} for use with continuous streaming processing.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ContinuousPartitionReader<T> extends PartitionReader<T> {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java
index 962864da4aad8..385c6f655440f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousPartitionReaderFactory.java
@@ -27,6 +27,8 @@
 /**
  * A variation on {@link PartitionReaderFactory} that returns {@link ContinuousPartitionReader}
  * instead of {@link PartitionReader}. It's used for continuous streaming processing.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ContinuousPartitionReaderFactory extends PartitionReaderFactory {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
index ee01a2553ae7a..a84578fe461a3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ContinuousStream.java
@@ -23,6 +23,8 @@
 
 /**
  * A {@link SparkDataStream} for streaming queries with continuous mode.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ContinuousStream extends SparkDataStream {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java
index ceab0f75734d3..40ecbf0578ee5 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/MicroBatchStream.java
@@ -25,6 +25,8 @@
 
 /**
  * A {@link SparkDataStream} for streaming queries with micro-batch mode.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface MicroBatchStream extends SparkDataStream {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
index 400de2a659746..efb8ebb684f06 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/Offset.java
@@ -25,6 +25,8 @@
  * During execution, offsets provided by the data source implementation will be logged and used as
  * restart checkpoints. Each source should provide an offset implementation which the source can use
  * to reconstruct a position in the stream up to which data has been seen/processed.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public abstract class Offset {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
index 35ad3bbde5cbf..faee230467bea 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/PartitionOffset.java
@@ -26,6 +26,8 @@
  * provide a method to merge these into a global Offset.
  *
  * These offsets must be serializable.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PartitionOffset extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java
index 121ed1ad116f9..36f6e05e365d9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/ReadLimit.java
@@ -27,6 +27,7 @@
  * @see SupportsAdmissionControl#latestOffset(Offset, ReadLimit)
  * @see ReadAllAvailable
  * @see ReadMaxRows
+ * @since 3.0.0
  */
 @Evolving
 public interface ReadLimit {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
index 1ba0c25ef4466..95703e255ea4e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/streaming/SparkDataStream.java
@@ -25,6 +25,8 @@
  *
  * Data sources should implement concrete data stream interfaces:
  * {@link MicroBatchStream} and {@link ContinuousStream}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SparkDataStream {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
index 9946867e8ea65..0923d07e7e5a3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingDataWriterFactory.java
@@ -33,6 +33,8 @@
  * Note that, the writer factory will be serialized and sent to executors, then the data writer
  * will be created on executors and do the actual writing. So this interface must be
  * serializable and {@link DataWriter} doesn't need to be.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface StreamingDataWriterFactory extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
index 4f930e1c158e5..e3dec3b2ff55e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/streaming/StreamingWrite.java
@@ -40,6 +40,8 @@
  * do it manually in their Spark applications if they want to retry.
  *
  * Please refer to the documentation of commit/abort methods for detailed specifications.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface StreamingWrite {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
index 2da0d1a51cb29..003ce850c926e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/ArrowUtils.scala
@@ -27,7 +27,7 @@ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
-object ArrowUtils {
+private[sql] object ArrowUtils {
 
   val rootAllocator = new RootAllocator(Long.MaxValue)
 

From b2b7cca6dec575b578f093bc7caa80f1b9d7b170 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 11 Feb 2020 10:03:01 +0900
Subject: [PATCH 0046/1280] [SPARK-30777][PYTHON][TESTS] Fix test failures for
 Pandas >= 1.0.0

### What changes were proposed in this pull request?

Fix PySpark test failures for using Pandas >= 1.0.0.

### Why are the changes needed?

Pandas 1.0.0 has recently been released and has API changes that result in PySpark test failures, this PR fixes the broken tests.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Manually tested with Pandas 1.0.1 and PyArrow 0.16.0

Closes #27529 from BryanCutler/pandas-fix-tests-1.0-SPARK-30777.

Authored-by: Bryan Cutler <cutlerb@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 07a9885f2792be1353f4a923d649e90bc431cb38)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/tests/test_arrow.py                  | 4 ++--
 python/pyspark/sql/tests/test_pandas_grouped_map.py     | 6 +++---
 python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py | 8 ++++----
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 98f44dfd29da5..004c79f290213 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -297,9 +297,9 @@ def test_createDataFrame_does_not_modify_input(self):
         # Some series get converted for Spark to consume, this makes sure input is unchanged
         pdf = self.create_pandas_data_frame()
         # Use a nanosecond value to make sure it is not truncated
-        pdf.ix[0, '8_timestamp_t'] = pd.Timestamp(1)
+        pdf.iloc[0, 7] = pd.Timestamp(1)
         # Integers with nulls will get NaNs filled with 0 and will be casted
-        pdf.ix[1, '2_int_t'] = None
+        pdf.iloc[1, 1] = None
         pdf_copy = pdf.copy(deep=True)
         self.spark.createDataFrame(pdf, schema=self.schema)
         self.assertTrue(pdf.equals(pdf_copy))
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index 51dd07fd7d70c..ff53a0c6f2cf2 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -390,11 +390,11 @@ def rename_pdf(pdf, names):
         # Function returns a pdf with required column names, but order could be arbitrary using dict
         def change_col_order(pdf):
             # Constructing a DataFrame from a dict should result in the same order,
-            # but use from_items to ensure the pdf column order is different than schema
-            return pd.DataFrame.from_items([
+            # but use OrderedDict to ensure the pdf column order is different than schema
+            return pd.DataFrame.from_dict(OrderedDict([
                 ('id', pdf.id),
                 ('u', pdf.v * 2),
-                ('v', pdf.v)])
+                ('v', pdf.v)]))
 
         ordered_udf = pandas_udf(
             change_col_order,
diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
index 974ad560daebf..21679785a769e 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
@@ -357,7 +357,7 @@ def test_complex_expressions(self):
                         plus_one(sum_udf(col('v1'))),
                         sum_udf(plus_one(col('v2'))))
                    .sort(['id', '(v % 2)'])
-                   .toPandas().sort_index(by=['id', '(v % 2)']))
+                   .toPandas().sort_values(by=['id', '(v % 2)']))
 
         expected1 = (df.withColumn('v1', df.v + 1)
                      .withColumn('v2', df.v + 2)
@@ -368,7 +368,7 @@ def test_complex_expressions(self):
                           plus_one(sum(col('v1'))),
                           sum(plus_one(col('v2'))))
                      .sort(['id', '(v % 2)'])
-                     .toPandas().sort_index(by=['id', '(v % 2)']))
+                     .toPandas().sort_values(by=['id', '(v % 2)']))
 
         # Test complex expressions with sql expression, scala pandas UDF and
         # group aggregate pandas UDF
@@ -381,7 +381,7 @@ def test_complex_expressions(self):
                         plus_two(sum_udf(col('v1'))),
                         sum_udf(plus_two(col('v2'))))
                    .sort(['id', '(v % 2)'])
-                   .toPandas().sort_index(by=['id', '(v % 2)']))
+                   .toPandas().sort_values(by=['id', '(v % 2)']))
 
         expected2 = (df.withColumn('v1', df.v + 1)
                      .withColumn('v2', df.v + 2)
@@ -392,7 +392,7 @@ def test_complex_expressions(self):
                           plus_two(sum(col('v1'))),
                           sum(plus_two(col('v2'))))
                      .sort(['id', '(v % 2)'])
-                     .toPandas().sort_index(by=['id', '(v % 2)']))
+                     .toPandas().sort_values(by=['id', '(v % 2)']))
 
         # Test sequential groupby aggregate
         result3 = (df.groupby('id')

From 8efe367a4ee862b8a85aee8881b0335b34cbba70 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 11 Feb 2020 15:50:03 +0900
Subject: [PATCH 0047/1280] [SPARK-30756][SQL] Fix
 `ThriftServerWithSparkContextSuite` on
 spark-branch-3.0-test-sbt-hadoop-2.7-hive-2.3

### What changes were proposed in this pull request?

This PR tries #26710 (comment) way to fix the test.

### Why are the changes needed?

To make the tests pass.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Jenkins will test first, and then `on spark-branch-3.0-test-sbt-hadoop-2.7-hive-2.3` will test it out.

Closes #27513 from HyukjinKwon/test-SPARK-30756.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 project/SparkBuild.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 9d0af3aa8c1b6..1c5c36ea8eae2 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -478,7 +478,8 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite",
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
-    "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite"
+    "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite"
   )
 
   private val DEFAULT_TEST_GROUP = "default_test_group"

From 1e5766cbdd69080a7fc3881636406945fbd85752 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 11 Feb 2020 17:22:08 +0900
Subject: [PATCH 0048/1280] [SPARK-29462][SQL] The data type of "array()"
 should be array<null>

### What changes were proposed in this pull request?

This brings https://github.com/apache/spark/pull/26324 back. It was reverted basically because, firstly Hive compatibility, and the lack of investigations in other DBMSes and ANSI.

- In case of PostgreSQL seems coercing NULL literal to TEXT type.
- Presto seems coercing `array() + array(1)` -> array of int.
- Hive seems  `array() + array(1)` -> array of strings

 Given that, the design choices have been differently made for some reasons. If we pick one of both, seems coercing to array of int makes much more sense.

Another investigation was made offline internally. Seems ANSI SQL 2011, section 6.5 "<contextually typed value specification>" states:

> If ES is specified, then let ET be the element type determined by the context in which ES appears. The declared type DT of ES is Case:
>
> a) If ES simply contains ARRAY, then ET ARRAY[0].
>
> b) If ES simply contains MULTISET, then ET MULTISET.
>
> ES is effectively replaced by CAST ( ES AS DT )

From reading other related context, doing it to `NullType`. Given the investigation made, choosing to `null` seems correct, and we have a reference Presto now. Therefore, this PR proposes to bring it back.

### Why are the changes needed?
When empty array is created, it should be declared as array<null>.

### Does this PR introduce any user-facing change?
Yes, `array()` creates `array<null>`. Now `array(1) + array()` can correctly create `array(1)` instead of `array("1")`.

### How was this patch tested?
Tested manually

Closes #27521 from HyukjinKwon/SPARK-29462.

Lead-authored-by: HyukjinKwon <gurwls223@apache.org>
Co-authored-by: Aman Omer <amanomer1996@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0045be766b949dff23ed72bd559568f17f645ffe)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md                     |  2 ++
 .../expressions/complexTypeCreator.scala        | 11 ++++++++++-
 .../org/apache/spark/sql/internal/SQLConf.scala |  9 +++++++++
 .../spark/sql/DataFrameFunctionsSuite.scala     | 17 +++++++++++++----
 4 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 26eb5838892b4..f98fab5b4c56b 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -215,6 +215,8 @@ license: |
   For example `SELECT timestamp 'tomorrow';`.
 
   - Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
+  
+  - Since Spark 3.0, when the `array` function is called without any parameters, it returns an empty array of `NullType`. In Spark version 2.4 and earlier, it returns an empty array of string type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.arrayDefaultToStringType.enabled` to `true`.
 
   - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 9ce87a4922c01..7335e305bfe55 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -44,10 +45,18 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
     TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), s"function $prettyName")
   }
 
+  private val defaultElementType: DataType = {
+    if (SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_DEFAULT_TO_STRING)) {
+      StringType
+    } else {
+      NullType
+    }
+  }
+
   override def dataType: ArrayType = {
     ArrayType(
       TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(children.map(_.dataType))
-        .getOrElse(StringType),
+        .getOrElse(defaultElementType),
       containsNull = children.exists(_.nullable))
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e38fe7606c4ee..b79b767dbb22b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2007,6 +2007,15 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val LEGACY_ARRAY_DEFAULT_TO_STRING =
+    buildConf("spark.sql.legacy.arrayDefaultToStringType.enabled")
+      .internal()
+      .doc("When set to true, it returns an empty array of string type when the `array` " +
+        "function is called without any parameters. Otherwise, it returns an empty " +
+        "array of `NullType`")
+      .booleanConf
+      .createWithDefault(false)
+
   val TRUNCATE_TABLE_IGNORE_PERMISSION_ACL =
     buildConf("spark.sql.truncateTable.ignorePermissionAcl.enabled")
       .internal()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 7fce03658fc16..9e9d8c3e9a7c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -3499,12 +3499,9 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     ).foreach(assertValuesDoNotChangeAfterCoalesceOrUnion(_))
   }
 
-  test("SPARK-21281 use string types by default if array and map have no argument") {
+  test("SPARK-21281 use string types by default if map have no argument") {
     val ds = spark.range(1)
     var expectedSchema = new StructType()
-      .add("x", ArrayType(StringType, containsNull = false), nullable = false)
-    assert(ds.select(array().as("x")).schema == expectedSchema)
-    expectedSchema = new StructType()
       .add("x", MapType(StringType, StringType, valueContainsNull = false), nullable = false)
     assert(ds.select(map().as("x")).schema == expectedSchema)
   }
@@ -3577,6 +3574,18 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     }.getMessage
     assert(nonFoldableError.contains("The 'escape' parameter must be a string literal"))
   }
+
+  test("SPARK-29462: Empty array of NullType for array function with no arguments") {
+    Seq((true, StringType), (false, NullType)).foreach {
+      case (arrayDefaultToString, expectedType) =>
+        withSQLConf(SQLConf.LEGACY_ARRAY_DEFAULT_TO_STRING.key -> arrayDefaultToString.toString) {
+          val schema = spark.range(1).select(array()).schema
+          assert(schema.nonEmpty && schema.head.dataType.isInstanceOf[ArrayType])
+          val actualType = schema.head.dataType.asInstanceOf[ArrayType].elementType
+          assert(actualType === expectedType)
+        }
+    }
+  }
 }
 
 object DataFrameFunctionsSuite {

From 37edbab7803c35eecd664f72447418e79638024e Mon Sep 17 00:00:00 2001
From: root1 <raksonrakesh@gmail.com>
Date: Tue, 11 Feb 2020 20:42:02 +0800
Subject: [PATCH 0049/1280] [SPARK-27545][SQL][DOC] Update the Documentation
 for CACHE TABLE and UNCACHE TABLE

### What changes were proposed in this pull request?
Document updated for `CACHE TABLE` & `UNCACHE TABLE`

### Why are the changes needed?
Cache table creates a temp view while caching data using `CACHE TABLE name AS query`. `UNCACHE TABLE` does not remove this temp view.

These things were not mentioned in the existing doc for `CACHE TABLE` & `UNCACHE TABLE`.

### Does this PR introduce any user-facing change?
Document updated for `CACHE TABLE` & `UNCACHE TABLE` command.

### How was this patch tested?
Manually

Closes #27090 from iRakson/SPARK-27545.

Lead-authored-by: root1 <raksonrakesh@gmail.com>
Co-authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b20754d9ee033091e2ef4d5bfa2576f946c9df50)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-syntax-aux-cache-cache-table.md   | 3 ++-
 docs/sql-ref-syntax-aux-cache-uncache-table.md | 6 +++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index ed6ef973466dd..20ade1961ab0b 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -20,7 +20,8 @@ license: |
 ---
 
 ### Description
-`CACHE TABLE` statement caches contents of a table or output of a query with the given storage level. This reduces scanning of the original files in future queries.
+`CACHE TABLE` statement caches contents of a table or output of a query with the given storage level. If a query is cached, then a temp view will be created for this query.
+This reduces scanning of the original files in future queries. 
 
 ### Syntax
 {% highlight sql %}
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index e0581d0d213df..69e21c258a333 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -21,11 +21,13 @@ license: |
 
 ### Description
 `UNCACHE TABLE` removes the entries and associated data from the in-memory and/or on-disk cache for a given table or view. The
-underlying entries should already have been brought to cache by previous `CACHE TABLE` operation. `UNCACHE TABLE` on a non-existent table throws Exception if `IF EXISTS` is not specified.
+underlying entries should already have been brought to cache by previous `CACHE TABLE` operation. `UNCACHE TABLE` on a non-existent table throws an exception if `IF EXISTS` is not specified.
+
 ### Syntax
 {% highlight sql %}
 UNCACHE TABLE [ IF EXISTS ] table_identifier
 {% endhighlight %}
+
 ### Parameters
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
@@ -37,10 +39,12 @@ UNCACHE TABLE [ IF EXISTS ] table_identifier
       </code>
   </dd>
 </dl>
+
 ### Examples
 {% highlight sql %}
 UNCACHE TABLE t1;
 {% endhighlight %}
+
 ### Related Statements
  * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
  * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)

From e139bf3c1b541b235b9a8f2750a5245ccc902f8a Mon Sep 17 00:00:00 2001
From: fuwhu <bestwwg@163.com>
Date: Tue, 11 Feb 2020 22:16:44 +0800
Subject: [PATCH 0050/1280] [MINOR][DOC] Add class document for
 PruneFileSourcePartitions and PruneHiveTablePartitions

### What changes were proposed in this pull request?
Add class document for PruneFileSourcePartitions and PruneHiveTablePartitions.

### Why are the changes needed?
To describe these two classes.

### Does this PR introduce any user-facing change?
no

### How was this patch tested?
no

Closes #27535 from fuwhu/SPARK-15616-FOLLOW-UP.

Authored-by: fuwhu <bestwwg@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f1d0dce4848a53831268c80bf7e1e0f47a1f7612)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/PruneFileSourcePartitions.scala     | 13 +++++++++++++
 .../hive/execution/PruneHiveTablePartitions.scala   |  8 ++++++++
 2 files changed, 21 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
index 1ea19c187e51a..a7129fb14d1a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PruneFileSourcePartitions.scala
@@ -26,6 +26,19 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2ScanRelation, FileScan}
 import org.apache.spark.sql.types.StructType
 
+/**
+ * Prune the partitions of file source based table using partition filters. Currently, this rule
+ * is applied to [[HadoopFsRelation]] with [[CatalogFileIndex]] and [[DataSourceV2ScanRelation]]
+ * with [[FileScan]].
+ *
+ * For [[HadoopFsRelation]], the location will be replaced by pruned file index, and corresponding
+ * statistics will be updated. And the partition filters will be kept in the filters of returned
+ * logical plan.
+ *
+ * For [[DataSourceV2ScanRelation]], both partition filters and data filters will be added to
+ * its underlying [[FileScan]]. And the partition filters will be removed in the filters of
+ * returned logical plan.
+ */
 private[sql] object PruneFileSourcePartitions extends Rule[LogicalPlan] {
 
   private def getPartitionKeyFiltersAndDataFilters(
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
index a0349f627d107..da6e4c52cf3a7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/PruneHiveTablePartitions.scala
@@ -30,6 +30,14 @@ import org.apache.spark.sql.execution.datasources.DataSourceStrategy
 import org.apache.spark.sql.internal.SQLConf
 
 /**
+ * Prune hive table partitions using partition filters on [[HiveTableRelation]]. The pruned
+ * partitions will be kept in [[HiveTableRelation.prunedPartitions]], and the statistics of
+ * the hive table relation will be updated based on pruned partitions.
+ *
+ * This rule is executed in optimization phase, so the statistics can be updated before physical
+ * planning, which is useful for some spark strategy, eg.
+ * [[org.apache.spark.sql.execution.SparkStrategies.JoinSelection]].
+ *
  * TODO: merge this with PruneFileSourcePartitions after we completely make hive as a data source.
  */
 private[sql] class PruneHiveTablePartitions(session: SparkSession)

From 79c99d0ee02de79df9700123a13d34afaaf21602 Mon Sep 17 00:00:00 2001
From: Yin Huai <yhuai@databricks.com>
Date: Wed, 12 Feb 2020 00:12:45 +0800
Subject: [PATCH 0051/1280] [SPARK-30783] Exclude hive-service-rpc

### What changes were proposed in this pull request?
Exclude hive-service-rpc from build.

### Why are the changes needed?
hive-service-rpc 2.3.6 and spark sql's thrift server module have duplicate classes. Leaving hive-service-rpc 2.3.6 in the class path means that spark can pick up classes defined in hive instead of its thrift server module, which can cause hard to debug runtime errors due to class loading order and compilation errors for applications depend on spark.

 If you compare hive-service-rpc 2.3.6's jar (https://search.maven.org/remotecontent?filepath=org/apache/hive/hive-service-rpc/2.3.6/hive-service-rpc-2.3.6.jar) and spark thrift server's jar (e.g. https://repository.apache.org/content/groups/snapshots/org/apache/spark/spark-hive-thriftserver_2.12/3.0.0-SNAPSHOT/spark-hive-thriftserver_2.12-3.0.0-20200207.021914-364.jar), you will see that all of classes provided by hive-service-rpc-2.3.6.jar are covered by spark thrift server's jar. https://issues.apache.org/jira/browse/SPARK-30783 has output of jar tf for both jars.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing tests.

Closes #27533 from yhuai/SPARK-30783.

Authored-by: Yin Huai <yhuai@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ea626b6acf0de0ff3b0678372f30ba6f84ae2b09)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 |  1 -
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 |  1 -
 pom.xml                                 | 20 ++++++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 1b57250c1fb54..4f4d8b1d4a62a 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -87,7 +87,6 @@ hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
 hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
 hive-metastore/2.3.6//hive-metastore-2.3.6.jar
 hive-serde/2.3.6//hive-serde-2.3.6.jar
-hive-service-rpc/2.3.6//hive-service-rpc-2.3.6.jar
 hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
 hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
 hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index ffd2364a51317..18e4246d63996 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -86,7 +86,6 @@ hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
 hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
 hive-metastore/2.3.6//hive-metastore-2.3.6.jar
 hive-serde/2.3.6//hive-serde-2.3.6.jar
-hive-service-rpc/2.3.6//hive-service-rpc-2.3.6.jar
 hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
 hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
 hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
diff --git a/pom.xml b/pom.xml
index a8d6ac932bac2..925fa28a291a4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1452,6 +1452,11 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1508,6 +1513,11 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1761,6 +1771,11 @@
             <groupId>${hive.group}</groupId>
             <artifactId>hive-service</artifactId>
           </exclusion>
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>${hive.group}</groupId>
             <artifactId>hive-shims</artifactId>
@@ -1911,6 +1926,11 @@
             <artifactId>groovy-all</artifactId>
           </exclusion>
           <!-- Begin of Hive 2.3 exclusion -->
+          <exclusion>
+            <!-- All classes are covered by spark's hive-thriftserver module -->
+            <groupId>${hive.group}</groupId>
+            <artifactId>hive-service-rpc</artifactId>
+          </exclusion>
           <!-- parquet-hadoop-bundle:1.8.1 conflict with 1.10.1 -->
           <exclusion>
             <groupId>org.apache.parquet</groupId>

From a53969617e8770683e675ae6d39e8f8bfd787073 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 11 Feb 2020 09:55:02 -0800
Subject: [PATCH 0052/1280] [SPARK-29462][SQL][DOCS] Add some more context and
 details in 'spark.sql.defaultUrlStreamHandlerFactory.enabled' documentation

### What changes were proposed in this pull request?

This PR adds some more information and context to `spark.sql.defaultUrlStreamHandlerFactory.enabled`.

### Why are the changes needed?

It is a bit difficult to understand the documentation of `spark.sql.defaultUrlStreamHandlerFactory.enabled`.

### Does this PR introduce any user-facing change?

Nope, internal doc only fix.

### How was this patch tested?

Nope. I only tested linter.

Closes #27541 from HyukjinKwon/SPARK-29462-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 99bd59fe29a87bb70485db536b0ae676e7a9d42e)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../org/apache/spark/sql/internal/StaticSQLConf.scala     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index 6bc752260a893..563e51ed597b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -172,7 +172,13 @@ object StaticSQLConf {
 
   val DEFAULT_URL_STREAM_HANDLER_FACTORY_ENABLED =
     buildStaticConf("spark.sql.defaultUrlStreamHandlerFactory.enabled")
-      .doc("When true, set FsUrlStreamHandlerFactory to support ADD JAR against HDFS locations")
+      .doc(
+        "When true, register Hadoop's FsUrlStreamHandlerFactory to support " +
+        "ADD JAR against HDFS locations. " +
+        "It should be disabled when a different stream protocol handler should be registered " +
+        "to support a particular protocol type, or if Hadoop's FsUrlStreamHandlerFactory " +
+        "conflicts with other protocol types such as `http` or `https`. See also SPARK-25694 " +
+        "and HADOOP-14598.")
       .internal()
       .booleanConf
       .createWithDefault(true)

From 5199d2f9dcf044f759318457ce3c0a56e00d9537 Mon Sep 17 00:00:00 2001
From: herman <herman@databricks.com>
Date: Wed, 12 Feb 2020 10:48:29 +0900
Subject: [PATCH 0053/1280] [SPARK-30780][SQL] Empty LocalTableScan should use
 RDD without partitions

### What changes were proposed in this pull request?
This is a small follow-up for https://github.com/apache/spark/pull/27400. This PR makes an empty `LocalTableScanExec` return an `RDD` without partitions.

### Why are the changes needed?
It is a bit unexpected that the RDD contains partitions if there is not work to do. It also can save a bit of work when this is used in a more complex plan.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added test to `SparkPlanSuite`.

Closes #27530 from hvanhovell/SPARK-30780.

Authored-by: herman <herman@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b25359cca3190f6a34dce3c3e49c4d2a80e88bdc)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/LocalTableScanExec.scala     | 12 ++++++++----
 .../scala/org/apache/spark/sql/DataFrameSuite.scala  |  2 +-
 .../apache/spark/sql/execution/SparkPlanSuite.scala  |  4 ++++
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
index 1b5115f2e29a3..b452213cd6cc7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/LocalTableScanExec.scala
@@ -45,10 +45,14 @@ case class LocalTableScanExec(
     }
   }
 
-  private lazy val numParallelism: Int = math.min(math.max(unsafeRows.length, 1),
-    sqlContext.sparkContext.defaultParallelism)
-
-  private lazy val rdd = sqlContext.sparkContext.parallelize(unsafeRows, numParallelism)
+  @transient private lazy val rdd: RDD[InternalRow] = {
+    if (rows.isEmpty) {
+      sqlContext.sparkContext.emptyRDD
+    } else {
+      val numSlices = math.min(unsafeRows.length, sqlContext.sparkContext.defaultParallelism)
+      sqlContext.sparkContext.parallelize(unsafeRows, numSlices)
+    }
+  }
 
   protected override def doExecute(): RDD[InternalRow] = {
     val numOutputRows = longMetric("numOutputRows")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index d2d58a83ded5d..694e576fcded4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -330,7 +330,7 @@ class DataFrameSuite extends QueryTest
       testData.select("key").coalesce(1).select("key"),
       testData.select("key").collect().toSeq)
 
-    assert(spark.emptyDataFrame.coalesce(1).rdd.partitions.size === 1)
+    assert(spark.emptyDataFrame.coalesce(1).rdd.partitions.size === 0)
   }
 
   test("convert $\"attribute name\" into unresolved attribute") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
index e3bc414516c04..56fff1107ae39 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
@@ -84,4 +84,8 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-30780 empty LocalTableScan should use RDD without partitions") {
+    assert(LocalTableScanExec(Nil, Nil).execute().getNumPartitions == 0)
+  }
 }

From ed6193ad68a55a85f51f3ebda08f53cbcf023a24 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 12 Feb 2020 10:49:46 +0900
Subject: [PATCH 0054/1280] [SPARK-30722][PYTHON][DOCS] Update documentation
 for Pandas UDF with Python type hints

### What changes were proposed in this pull request?

This PR targets to document the Pandas UDF redesign with type hints introduced at SPARK-28264.
Mostly self-describing; however, there are few things to note for reviewers.

1. This PR replace the existing documentation of pandas UDFs to the newer redesign to promote the Python type hints. I added some words that Spark 3.0 still keeps the compatibility though.

2. This PR proposes to name non-pandas UDFs as "Pandas Function API"

3. SCALAR_ITER become two separate sections to reduce confusion:
  - `Iterator[pd.Series]` -> `Iterator[pd.Series]`
  - `Iterator[Tuple[pd.Series, ...]]` -> `Iterator[pd.Series]`

4. I removed some examples that look overkill to me.

5. I also removed some information in the doc, that seems duplicating or too much.

### Why are the changes needed?

To document new redesign in pandas UDF.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing tests should cover.

Closes #27466 from HyukjinKwon/SPARK-30722.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit aa6a60530e63ab3bb8b117f8738973d1b26a2cc7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/sparktestsupport/modules.py        |   1 -
 docs/sql-pyspark-pandas-with-arrow.md  | 233 +++++++----
 examples/src/main/python/sql/arrow.py  | 258 ++++++------
 python/pyspark/sql/pandas/functions.py | 538 +++++++++++--------------
 python/pyspark/sql/pandas/group_ops.py |  99 ++++-
 python/pyspark/sql/pandas/map_ops.py   |   6 +-
 python/pyspark/sql/udf.py              |  16 +-
 7 files changed, 609 insertions(+), 542 deletions(-)

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 40f2ca288d694..391e4bbe1b1f0 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -364,7 +364,6 @@ def __hash__(self):
         "pyspark.sql.avro.functions",
         "pyspark.sql.pandas.conversion",
         "pyspark.sql.pandas.map_ops",
-        "pyspark.sql.pandas.functions",
         "pyspark.sql.pandas.group_ops",
         "pyspark.sql.pandas.types",
         "pyspark.sql.pandas.serializers",
diff --git a/docs/sql-pyspark-pandas-with-arrow.md b/docs/sql-pyspark-pandas-with-arrow.md
index 7eb8a74547f70..92a515746b607 100644
--- a/docs/sql-pyspark-pandas-with-arrow.md
+++ b/docs/sql-pyspark-pandas-with-arrow.md
@@ -35,7 +35,7 @@ working with Arrow-enabled data.
 
 If you install PySpark using pip, then PyArrow can be brought in as an extra dependency of the
 SQL module with the command `pip install pyspark[sql]`. Otherwise, you must ensure that PyArrow
-is installed and available on all cluster nodes. The current supported version is 0.12.1.
+is installed and available on all cluster nodes. The current supported version is 0.15.1+.
 You can install using pip or conda from the conda-forge channel. See PyArrow
 [installation](https://arrow.apache.org/docs/python/install.html) for details.
 
@@ -65,132 +65,216 @@ Spark will fall back to create the DataFrame without Arrow.
 
 ## Pandas UDFs (a.k.a. Vectorized UDFs)
 
-Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer data and
-Pandas to work with the data. A Pandas UDF is defined using the keyword `pandas_udf` as a decorator
-or to wrap the function, no additional configuration is required. Currently, there are two types of
-Pandas UDF: Scalar and Grouped Map.
+Pandas UDFs are user defined functions that are executed by Spark using
+Arrow to transfer data and Pandas to work with the data, which allows vectorized operations. A Pandas
+UDF is defined using the `pandas_udf` as a decorator or to wrap the function, and no additional
+configuration is required. A Pandas UDF behaves as a regular PySpark function API in general.
 
-### Scalar
+Before Spark 3.0, Pandas UDFs used to be defined with `PandasUDFType`. From Spark 3.0
+with Python 3.6+, you can also use [Python type hints](https://www.python.org/dev/peps/pep-0484).
+Using Python type hints are preferred and using `PandasUDFType` will be deprecated in
+the future release.
 
-Scalar Pandas UDFs are used for vectorizing scalar operations. They can be used with functions such
-as `select` and `withColumn`. The Python function should take `pandas.Series` as inputs and return
-a `pandas.Series` of the same length. Internally, Spark will execute a Pandas UDF by splitting
-columns into batches and calling the function for each batch as a subset of the data, then
-concatenating the results together.
+Note that the type hint should use `pandas.Series` in all cases but there is one variant
+that `pandas.DataFrame` should be used for its input or output type hint instead when the input
+or output column is of `StructType`. The following example shows a Pandas UDF which takes long
+column, string column and struct column, and outputs a struct column. It requires the function to
+specify the type hints of `pandas.Series` and `pandas.DataFrame` as below:
 
-The following example shows how to create a scalar Pandas UDF that computes the product of 2 columns.
+<p>
+<div class="codetabs">
+<div data-lang="python" markdown="1">
+{% include_example ser_to_frame_pandas_udf python/sql/arrow.py %}
+</div>
+</div>
+</p>
+
+In the following sections, it describes the cominations of the supported type hints. For simplicity,
+`pandas.DataFrame` variant is omitted.
+
+### Series to Series
+
+The type hint can be expressed as `pandas.Series`, ... -> `pandas.Series`.
+
+By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF where the given
+function takes one or more `pandas.Series` and outputs one `pandas.Series`. The output of the function should
+always be of the same length as the input. Internally, PySpark will execute a Pandas UDF by splitting
+columns into batches and calling the function for each batch as a subset of the data, then concatenating
+the results together.
+
+The following example shows how to create this Pandas UDF that computes the product of 2 columns.
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example scalar_pandas_udf python/sql/arrow.py %}
+{% include_example ser_to_ser_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
-### Scalar Iterator
+For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
+
+### Iterator of Series to Iterator of Series
+
+The type hint can be expressed as `Iterator[pandas.Series]` -> `Iterator[pandas.Series]`.
+
+By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF where the given
+function takes an iterator of `pandas.Series` and outputs an iterator of `pandas.Series`. The output of each
+series from the function should always be of the same length as the input. In this case, the created
+Pandas UDF requires one input column when the Pandas UDF is called. To use multiple input columns,
+a different type hint is required. See Iterator of Multiple Series to Iterator of Series.
+
+It is useful when the UDF execution requires initializing some states although internally it works
+identically as Series to Series case. The pseudocode below illustrates the example.
+
+{% highlight python %}
+@pandas_udf("long")
+def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+    # Do some expensive initialization with a state
+    state = very_expensive_initialization()
+    for x in iterator:
+        # Use that state for whole iterator.
+        yield calculate_with_state(x, state)
 
-Scalar iterator (`SCALAR_ITER`) Pandas UDF is the same as scalar Pandas UDF above except that the
-underlying Python function takes an iterator of batches as input instead of a single batch and,
-instead of returning a single output batch, it yields output batches or returns an iterator of
-output batches.
-It is useful when the UDF execution requires initializing some states, e.g., loading an machine
-learning model file to apply inference to every input batch.
+df.select(calculate("value")).show()
+{% endhighlight %}
 
-The following example shows how to create scalar iterator Pandas UDFs:
+The following example shows how to create this Pandas UDF:
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example scalar_iter_pandas_udf python/sql/arrow.py %}
+{% include_example iter_ser_to_iter_ser_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
-### Grouped Map
-Grouped map Pandas UDFs are used with `groupBy().apply()` which implements the "split-apply-combine" pattern.
-Split-apply-combine consists of three steps:
-* Split the data into groups by using `DataFrame.groupBy`.
-* Apply a function on each group. The input and output of the function are both `pandas.DataFrame`. The
-  input data contains all the rows and columns for each group.
-* Combine the results into a new `DataFrame`.
+For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
 
-To use `groupBy().apply()`, the user needs to define the following:
-* A Python function that defines the computation for each group.
-* A `StructType` object or a string that defines the schema of the output `DataFrame`.
+### Iterator of Multiple Series to Iterator of Series
 
-The column labels of the returned `pandas.DataFrame` must either match the field names in the
-defined output schema if specified as strings, or match the field data types by position if not
-strings, e.g. integer indices. See [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame)
-on how to label columns when constructing a `pandas.DataFrame`.
+The type hint can be expressed as `Iterator[Tuple[pandas.Series, ...]]` -> `Iterator[pandas.Series]`.
 
-Note that all data for a group will be loaded into memory before the function is applied. This can
-lead to out of memory exceptions, especially if the group sizes are skewed. The configuration for
-[maxRecordsPerBatch](#setting-arrow-batch-size) is not applied on groups and it is up to the user
-to ensure that the grouped data will fit into the available memory.
+By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF where the
+given function takes an iterator of a tuple of multiple `pandas.Series` and outputs an iterator of `pandas.Series`.
+In this case, the created pandas UDF requires multiple input columns as many as the series in the tuple
+when the Pandas UDF is called. It works identically as Iterator of Series to Iterator of Series case except the parameter difference.
 
-The following example shows how to use `groupby().apply()` to subtract the mean from each value in the group.
+The following example shows how to create this Pandas UDF:
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example grouped_map_pandas_udf python/sql/arrow.py %}
+{% include_example iter_sers_to_iter_ser_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf) and
-[`pyspark.sql.GroupedData.apply`](api/python/pyspark.sql.html#pyspark.sql.GroupedData.apply).
+For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
+
+### Series to Scalar
 
-### Grouped Aggregate
+The type hint can be expressed as `pandas.Series`, ... -> `Any`.
 
-Grouped aggregate Pandas UDFs are similar to Spark aggregate functions. Grouped aggregate Pandas UDFs are used with `groupBy().agg()` and
-[`pyspark.sql.Window`](api/python/pyspark.sql.html#pyspark.sql.Window). It defines an aggregation from one or more `pandas.Series`
-to a scalar value, where each `pandas.Series` represents a column within the group or window.
+By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF similar
+to PySpark's aggregate functions. The given function takes `pandas.Series` and returns a scalar value.
+The return type should be a primitive data type, and the returned scalar can be either a python
+primitive type, e.g., `int` or `float` or a numpy data type, e.g., `numpy.int64` or `numpy.float64`.
+`Any` should ideally be a specific scalar type accordingly.
 
-Note that this type of UDF does not support partial aggregation and all data for a group or window will be loaded into memory. Also,
-only unbounded window is supported with Grouped aggregate Pandas UDFs currently.
+This UDF can be also used with `groupBy().agg()` and [`pyspark.sql.Window`](api/python/pyspark.sql.html#pyspark.sql.Window).
+It defines an aggregation from one or more `pandas.Series` to a scalar value, where each `pandas.Series`
+represents a column within the group or window.
 
-The following example shows how to use this type of UDF to compute mean with groupBy and window operations:
+Note that this type of UDF does not support partial aggregation and all data for a group or window
+will be loaded into memory. Also, only unbounded window is supported with Grouped aggregate Pandas
+UDFs currently. The following example shows how to use this type of UDF to compute mean with a group-by
+and window operations:
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example grouped_agg_pandas_udf python/sql/arrow.py %}
+{% include_example ser_to_scalar_pandas_udf python/sql/arrow.py %}
 </div>
 </div>
 
 For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf)
 
 
-### Map Iterator
+## Pandas Function APIs
+
+Pandas function APIs can directly apply a Python native function against the whole DataFrame by
+using Pandas instances. Internally it works similarly with Pandas UDFs by Spark using Arrow to transfer
+data and Pandas to work with the data, which allows vectorized operations. A Pandas function API behaves
+as a regular API under PySpark `DataFrame` in general.
+
+From Spark 3.0, Grouped map pandas UDF is now categorized as a separate Pandas Function API,
+`DataFrame.groupby().applyInPandas()`. It is still possible to use it with `PandasUDFType`
+and `DataFrame.groupby().apply()` as it was; however, it is preferred to use
+`DataFrame.groupby().applyInPandas()` directly. Using `PandasUDFType` will be deprecated
+in the future.
+
+### Grouped Map
+
+Grouped map operations with Pandas instances are supported by `DataFrame.groupby().applyInPandas()`
+which requires a Python function that takes a `pandas.DataFrame` and return another `pandas.DataFrame`.
+It maps each group to each `pandas.DataFrame` in the Python function.
+
+This API implements the "split-apply-combine" pattern which consists of three steps:
+* Split the data into groups by using `DataFrame.groupBy`.
+* Apply a function on each group. The input and output of the function are both `pandas.DataFrame`. The
+  input data contains all the rows and columns for each group.
+* Combine the results into a new PySpark `DataFrame`.
 
-Map iterator Pandas UDFs are used to transform data with an iterator of batches. Map iterator
-Pandas UDFs can be used with 
-[`pyspark.sql.DataFrame.mapInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
-It defines a map function that transforms an iterator of `pandas.DataFrame` to another.
+To use `groupBy().applyInPandas()`, the user needs to define the following:
+* A Python function that defines the computation for each group.
+* A `StructType` object or a string that defines the schema of the output PySpark `DataFrame`.
+
+The column labels of the returned `pandas.DataFrame` must either match the field names in the
+defined output schema if specified as strings, or match the field data types by position if not
+strings, e.g. integer indices. See [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html#pandas.DataFrame)
+on how to label columns when constructing a `pandas.DataFrame`.
 
-It can return the output of arbitrary length in contrast to the scalar Pandas UDF. It maps an iterator of `pandas.DataFrame`s,
-that represents the current `DataFrame`, using the map iterator UDF and returns the result as a `DataFrame`.
+Note that all data for a group will be loaded into memory before the function is applied. This can
+lead to out of memory exceptions, especially if the group sizes are skewed. The configuration for
+[maxRecordsPerBatch](#setting-arrow-batch-size) is not applied on groups and it is up to the user
+to ensure that the grouped data will fit into the available memory.
 
-The following example shows how to create map iterator Pandas UDFs:
+The following example shows how to use `groupby().applyInPandas()` to subtract the mean from each value
+in the group.
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example map_iter_pandas_udf python/sql/arrow.py %}
+{% include_example grouped_apply_in_pandas python/sql/arrow.py %}
 </div>
 </div>
 
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf) and
-[`pyspark.sql.DataFrame.mapsInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
+For detailed usage, please see [`pyspark.sql.GroupedData.applyInPandas`](api/python/pyspark.sql.html#pyspark.sql.GroupedData.applyInPandas).
 
+### Map
+
+Map operations with Pandas instances are supported by `DataFrame.mapInPandas()` which maps an iterator
+of `pandas.DataFrame`s to another iterator of `pandas.DataFrame`s that represents the current
+PySpark `DataFrame` and returns the result as a PySpark `DataFrame`. The functions takes and outputs
+an iterator of `pandas.DataFrame`. It can return the output of arbitrary length in contrast to some
+Pandas UDFs although internally it works similarly with Series to Series Pandas UDF.
+
+The following example shows how to use `mapInPandas()`:
+
+<div class="codetabs">
+<div data-lang="python" markdown="1">
+{% include_example map_in_pandas python/sql/arrow.py %}
+</div>
+</div>
 
-### Cogrouped Map
+For detailed usage, please see [`pyspark.sql.DataFrame.mapsInPandas`](api/python/pyspark.sql.html#pyspark.sql.DataFrame.mapInPandas).
 
-Cogrouped map Pandas UDFs allow two DataFrames to be cogrouped by a common key and then a python function applied to
-each cogroup.  They are used with `groupBy().cogroup().apply()` which consists of the following steps:
+### Co-grouped Map
 
+Co-grouped map operations with Pandas instances are supported by `DataFrame.groupby().cogroup().applyInPandas()` which
+allows two PySpark `DataFrame`s to be cogrouped by a common key and then a Python function applied to each
+cogroup. It consists of the following steps:
 * Shuffle the data such that the groups of each dataframe which share a key are cogrouped together.
-* Apply a function to each cogroup.  The input of the function is two `pandas.DataFrame` (with an optional Tuple
-representing the key).  The output of the function is a `pandas.DataFrame`.
-* Combine the pandas.DataFrames from all groups into a new `DataFrame`. 
+* Apply a function to each cogroup. The input of the function is two `pandas.DataFrame` (with an optional tuple
+representing the key). The output of the function is a `pandas.DataFrame`.
+* Combine the `pandas.DataFrame`s from all groups into a new PySpark `DataFrame`. 
 
-To use `groupBy().cogroup().apply()`, the user needs to define the following:
+To use `groupBy().cogroup().applyInPandas()`, the user needs to define the following:
 * A Python function that defines the computation for each cogroup.
-* A `StructType` object or a string that defines the schema of the output `DataFrame`.
+* A `StructType` object or a string that defines the schema of the output PySpark `DataFrame`.
 
 The column labels of the returned `pandas.DataFrame` must either match the field names in the
 defined output schema if specified as strings, or match the field data types by position if not
@@ -201,16 +285,15 @@ Note that all data for a cogroup will be loaded into memory before the function
 memory exceptions, especially if the group sizes are skewed. The configuration for [maxRecordsPerBatch](#setting-arrow-batch-size)
 is not applied and it is up to the user to ensure that the cogrouped data will fit into the available memory.
 
-The following example shows how to use `groupby().cogroup().apply()` to perform an asof join between two datasets.
+The following example shows how to use `groupby().cogroup().applyInPandas()` to perform an asof join between two datasets.
 
 <div class="codetabs">
 <div data-lang="python" markdown="1">
-{% include_example cogrouped_map_pandas_udf python/sql/arrow.py %}
+{% include_example cogrouped_apply_in_pandas python/sql/arrow.py %}
 </div>
 </div>
 
-For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/pyspark.sql.html#pyspark.sql.functions.pandas_udf) and
-[`pyspark.sql.CoGroupedData.apply`](api/python/pyspark.sql.html#pyspark.sql.CoGroupedData.apply).
+For detailed usage, please see [`pyspark.sql.PandasCogroupedOps.applyInPandas()`](api/python/pyspark.sql.html#pyspark.sql.PandasCogroupedOps.applyInPandas).
 
 
 ## Usage Notes
diff --git a/examples/src/main/python/sql/arrow.py b/examples/src/main/python/sql/arrow.py
index 1c983172d36ef..b7d8467172fab 100644
--- a/examples/src/main/python/sql/arrow.py
+++ b/examples/src/main/python/sql/arrow.py
@@ -23,12 +23,19 @@
 
 from __future__ import print_function
 
+import sys
+
 from pyspark.sql import SparkSession
 from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
 
 require_minimum_pandas_version()
 require_minimum_pyarrow_version()
 
+if sys.version_info < (3, 6):
+    raise Exception(
+        "Running this example file requires Python 3.6+; however, "
+        "your Python version was:\n %s" % sys.version)
+
 
 def dataframe_with_arrow_example(spark):
     # $example on:dataframe_with_arrow$
@@ -50,15 +57,45 @@ def dataframe_with_arrow_example(spark):
     print("Pandas DataFrame result statistics:\n%s\n" % str(result_pdf.describe()))
 
 
-def scalar_pandas_udf_example(spark):
-    # $example on:scalar_pandas_udf$
+def ser_to_frame_pandas_udf_example(spark):
+    # $example on:ser_to_frame_pandas_udf$
+    import pandas as pd
+
+    from pyspark.sql.functions import pandas_udf
+
+    @pandas_udf("col1 string, col2 long")
+    def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
+        s3['col2'] = s1 + s2.str.len()
+        return s3
+
+    # Create a Spark DataFrame that has three columns including a sturct column.
+    df = spark.createDataFrame(
+        [[1, "a string", ("a nested string",)]],
+        "long_col long, string_col string, struct_col struct<col1:string>")
+
+    df.printSchema()
+    # root
+    # |-- long_column: long (nullable = true)
+    # |-- string_column: string (nullable = true)
+    # |-- struct_column: struct (nullable = true)
+    # |    |-- col1: string (nullable = true)
+
+    df.select(func("long_col", "string_col", "struct_col")).printSchema()
+    # |-- func(long_col, string_col, struct_col): struct (nullable = true)
+    # |    |-- col1: string (nullable = true)
+    # |    |-- col2: long (nullable = true)
+    # $example off:ser_to_frame_pandas_udf$$
+
+
+def ser_to_ser_pandas_udf_example(spark):
+    # $example on:ser_to_ser_pandas_udf$
     import pandas as pd
 
     from pyspark.sql.functions import col, pandas_udf
     from pyspark.sql.types import LongType
 
     # Declare the function and create the UDF
-    def multiply_func(a, b):
+    def multiply_func(a: pd.Series, b: pd.Series) -> pd.Series:
         return a * b
 
     multiply = pandas_udf(multiply_func, returnType=LongType())
@@ -83,26 +120,27 @@ def multiply_func(a, b):
     # |                  4|
     # |                  9|
     # +-------------------+
-    # $example off:scalar_pandas_udf$
+    # $example off:ser_to_ser_pandas_udf$
 
 
-def scalar_iter_pandas_udf_example(spark):
-    # $example on:scalar_iter_pandas_udf$
+def iter_ser_to_iter_ser_pandas_udf_example(spark):
+    # $example on:iter_ser_to_iter_ser_pandas_udf$
+    from typing import Iterator
+
     import pandas as pd
 
-    from pyspark.sql.functions import col, pandas_udf, struct, PandasUDFType
+    from pyspark.sql.functions import pandas_udf
 
     pdf = pd.DataFrame([1, 2, 3], columns=["x"])
     df = spark.createDataFrame(pdf)
 
-    # When the UDF is called with a single column that is not StructType,
-    # the input to the underlying function is an iterator of pd.Series.
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def plus_one(batch_iter):
-        for x in batch_iter:
+    # Declare the function and create the UDF
+    @pandas_udf("long")
+    def plus_one(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+        for x in iterator:
             yield x + 1
 
-    df.select(plus_one(col("x"))).show()
+    df.select(plus_one("x")).show()
     # +-----------+
     # |plus_one(x)|
     # +-----------+
@@ -110,15 +148,28 @@ def plus_one(batch_iter):
     # |          3|
     # |          4|
     # +-----------+
+    # $example off:iter_ser_to_iter_ser_pandas_udf$
+
+
+def iter_sers_to_iter_ser_pandas_udf_example(spark):
+    # $example on:iter_sers_to_iter_ser_pandas_udf$
+    from typing import Iterator, Tuple
+
+    import pandas as pd
 
-    # When the UDF is called with more than one columns,
-    # the input to the underlying function is an iterator of pd.Series tuple.
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def multiply_two_cols(batch_iter):
-        for a, b in batch_iter:
+    from pyspark.sql.functions import pandas_udf
+
+    pdf = pd.DataFrame([1, 2, 3], columns=["x"])
+    df = spark.createDataFrame(pdf)
+
+    # Declare the function and create the UDF
+    @pandas_udf("long")
+    def multiply_two_cols(
+            iterator: Iterator[Tuple[pd.Series, pd.Series]]) -> Iterator[pd.Series]:
+        for a, b in iterator:
             yield a * b
 
-    df.select(multiply_two_cols(col("x"), col("x"))).show()
+    df.select(multiply_two_cols("x", "x")).show()
     # +-----------------------+
     # |multiply_two_cols(x, x)|
     # +-----------------------+
@@ -126,92 +177,32 @@ def multiply_two_cols(batch_iter):
     # |                      4|
     # |                      9|
     # +-----------------------+
+    # $example off:iter_sers_to_iter_ser_pandas_udf$
 
-    # When the UDF is called with a single column that is StructType,
-    # the input to the underlying function is an iterator of pd.DataFrame.
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def multiply_two_nested_cols(pdf_iter):
-        for pdf in pdf_iter:
-            yield pdf["a"] * pdf["b"]
-
-    df.select(
-        multiply_two_nested_cols(
-            struct(col("x").alias("a"), col("x").alias("b"))
-        ).alias("y")
-    ).show()
-    # +---+
-    # |  y|
-    # +---+
-    # |  1|
-    # |  4|
-    # |  9|
-    # +---+
-
-    # In the UDF, you can initialize some states before processing batches.
-    # Wrap your code with try/finally or use context managers to ensure
-    # the release of resources at the end.
-    y_bc = spark.sparkContext.broadcast(1)
-
-    @pandas_udf("long", PandasUDFType.SCALAR_ITER)
-    def plus_y(batch_iter):
-        y = y_bc.value  # initialize states
-        try:
-            for x in batch_iter:
-                yield x + y
-        finally:
-            pass  # release resources here, if any
-
-    df.select(plus_y(col("x"))).show()
-    # +---------+
-    # |plus_y(x)|
-    # +---------+
-    # |        2|
-    # |        3|
-    # |        4|
-    # +---------+
-    # $example off:scalar_iter_pandas_udf$
-
-
-def grouped_map_pandas_udf_example(spark):
-    # $example on:grouped_map_pandas_udf$
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
-
-    df = spark.createDataFrame(
-        [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-        ("id", "v"))
-
-    @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)
-    def subtract_mean(pdf):
-        # pdf is a pandas.DataFrame
-        v = pdf.v
-        return pdf.assign(v=v - v.mean())
-
-    df.groupby("id").apply(subtract_mean).show()
-    # +---+----+
-    # | id|   v|
-    # +---+----+
-    # |  1|-0.5|
-    # |  1| 0.5|
-    # |  2|-3.0|
-    # |  2|-1.0|
-    # |  2| 4.0|
-    # +---+----+
-    # $example off:grouped_map_pandas_udf$
 
+def ser_to_scalar_pandas_udf_example(spark):
+    # $example on:ser_to_scalar_pandas_udf$
+    import pandas as pd
 
-def grouped_agg_pandas_udf_example(spark):
-    # $example on:grouped_agg_pandas_udf$
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
+    from pyspark.sql.functions import pandas_udf
     from pyspark.sql import Window
 
     df = spark.createDataFrame(
         [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
         ("id", "v"))
 
-    @pandas_udf("double", PandasUDFType.GROUPED_AGG)
-    def mean_udf(v):
+    # Declare the function and create the UDF
+    @pandas_udf("double")
+    def mean_udf(v: pd.Series) -> float:
         return v.mean()
 
+    df.select(mean_udf(df['v'])).show()
+    # +-----------+
+    # |mean_udf(v)|
+    # +-----------+
+    # |        4.2|
+    # +-----------+
+
     df.groupby("id").agg(mean_udf(df['v'])).show()
     # +---+-----------+
     # | id|mean_udf(v)|
@@ -233,37 +224,54 @@ def mean_udf(v):
     # |  2| 5.0|   6.0|
     # |  2|10.0|   6.0|
     # +---+----+------+
-    # $example off:grouped_agg_pandas_udf$
+    # $example off:ser_to_scalar_pandas_udf$
 
 
-def map_iter_pandas_udf_example(spark):
-    # $example on:map_iter_pandas_udf$
-    import pandas as pd
+def grouped_apply_in_pandas_example(spark):
+    # $example on:grouped_apply_in_pandas$
+    df = spark.createDataFrame(
+        [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
+        ("id", "v"))
 
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
+    def subtract_mean(pdf):
+        # pdf is a pandas.DataFrame
+        v = pdf.v
+        return pdf.assign(v=v - v.mean())
+
+    df.groupby("id").applyInPandas(subtract_mean, schema="id long, v double").show()
+    # +---+----+
+    # | id|   v|
+    # +---+----+
+    # |  1|-0.5|
+    # |  1| 0.5|
+    # |  2|-3.0|
+    # |  2|-1.0|
+    # |  2| 4.0|
+    # +---+----+
+    # $example off:grouped_apply_in_pandas$
 
+
+def map_in_pandas_example(spark):
+    # $example on:map_in_pandas$
     df = spark.createDataFrame([(1, 21), (2, 30)], ("id", "age"))
 
-    @pandas_udf(df.schema, PandasUDFType.MAP_ITER)
-    def filter_func(batch_iter):
-        for pdf in batch_iter:
+    def filter_func(iterator):
+        for pdf in iterator:
             yield pdf[pdf.id == 1]
 
-    df.mapInPandas(filter_func).show()
+    df.mapInPandas(filter_func, schema=df.schema).show()
     # +---+---+
     # | id|age|
     # +---+---+
     # |  1| 21|
     # +---+---+
-    # $example off:map_iter_pandas_udf$
+    # $example off:map_in_pandas$
 
 
-def cogrouped_map_pandas_udf_example(spark):
-    # $example on:cogrouped_map_pandas_udf$
+def cogrouped_apply_in_pandas_example(spark):
+    # $example on:cogrouped_apply_in_pandas$
     import pandas as pd
 
-    from pyspark.sql.functions import pandas_udf, PandasUDFType
-
     df1 = spark.createDataFrame(
         [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)],
         ("time", "id", "v1"))
@@ -272,11 +280,11 @@ def cogrouped_map_pandas_udf_example(spark):
         [(20000101, 1, "x"), (20000101, 2, "y")],
         ("time", "id", "v2"))
 
-    @pandas_udf("time int, id int, v1 double, v2 string", PandasUDFType.COGROUPED_MAP)
     def asof_join(l, r):
         return pd.merge_asof(l, r, on="time", by="id")
 
-    df1.groupby("id").cogroup(df2.groupby("id")).apply(asof_join).show()
+    df1.groupby("id").cogroup(df2.groupby("id")).applyInPandas(
+        asof_join, schema="time int, id int, v1 double, v2 string").show()
     # +--------+---+---+---+
     # |    time| id| v1| v2|
     # +--------+---+---+---+
@@ -285,7 +293,7 @@ def asof_join(l, r):
     # |20000101|  2|2.0|  y|
     # |20000102|  2|4.0|  y|
     # +--------+---+---+---+
-    # $example off:cogrouped_map_pandas_udf$
+    # $example off:cogrouped_apply_in_pandas$
 
 
 if __name__ == "__main__":
@@ -296,17 +304,21 @@ def asof_join(l, r):
 
     print("Running Pandas to/from conversion example")
     dataframe_with_arrow_example(spark)
-    print("Running pandas_udf scalar example")
-    scalar_pandas_udf_example(spark)
-    print("Running pandas_udf scalar iterator example")
-    scalar_iter_pandas_udf_example(spark)
-    print("Running pandas_udf grouped map example")
-    grouped_map_pandas_udf_example(spark)
-    print("Running pandas_udf grouped agg example")
-    grouped_agg_pandas_udf_example(spark)
-    print("Running pandas_udf map iterator example")
-    map_iter_pandas_udf_example(spark)
-    print("Running pandas_udf cogrouped map example")
-    cogrouped_map_pandas_udf_example(spark)
+    print("Running pandas_udf example: Series to Frame")
+    ser_to_frame_pandas_udf_example(spark)
+    print("Running pandas_udf example: Series to Series")
+    ser_to_ser_pandas_udf_example(spark)
+    print("Running pandas_udf example: Iterator of Series to Iterator of Seires")
+    iter_ser_to_iter_ser_pandas_udf_example(spark)
+    print("Running pandas_udf example: Iterator of Multiple Series to Iterator of Series")
+    iter_sers_to_iter_ser_pandas_udf_example(spark)
+    print("Running pandas_udf example: Series to Scalar")
+    ser_to_scalar_pandas_udf_example(spark)
+    print("Running pandas function example: Grouped Map")
+    grouped_apply_in_pandas_example(spark)
+    print("Running pandas function example: Map")
+    map_in_pandas_example(spark)
+    print("Running pandas function example: Co-grouped Map")
+    cogrouped_apply_in_pandas_example(spark)
 
     spark.stop()
diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py
index 30602789a33a9..31aa321bf5826 100644
--- a/python/pyspark/sql/pandas/functions.py
+++ b/python/pyspark/sql/pandas/functions.py
@@ -43,303 +43,228 @@ class PandasUDFType(object):
 @since(2.3)
 def pandas_udf(f=None, returnType=None, functionType=None):
     """
-    Creates a vectorized user defined function (UDF).
+    Creates a pandas user defined function (a.k.a. vectorized user defined function).
+
+    Pandas UDFs are user defined functions that are executed by Spark using Arrow to transfer
+    data and Pandas to work with the data, which allows vectorized operations. A Pandas UDF
+    is defined using the `pandas_udf` as a decorator or to wrap the function, and no
+    additional configuration is required. A Pandas UDF behaves as a regular PySpark function
+    API in general.
 
     :param f: user-defined function. A python function if used as a standalone function
     :param returnType: the return type of the user-defined function. The value can be either a
         :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
     :param functionType: an enum value in :class:`pyspark.sql.functions.PandasUDFType`.
-                         Default: SCALAR.
-
-    .. seealso:: :meth:`pyspark.sql.DataFrame.mapInPandas`
-    .. seealso:: :meth:`pyspark.sql.GroupedData.applyInPandas`
-    .. seealso:: :meth:`pyspark.sql.PandasCogroupedOps.applyInPandas`
-
-    The function type of the UDF can be one of the following:
-
-    1. SCALAR
-
-       A scalar UDF defines a transformation: One or more `pandas.Series` -> A `pandas.Series`.
-       The length of the returned `pandas.Series` must be of the same as the input `pandas.Series`.
-       If the return type is :class:`StructType`, the returned value should be a `pandas.DataFrame`.
-
-       :class:`MapType`, nested :class:`StructType` are currently not supported as output types.
-
-       Scalar UDFs can be used with :meth:`pyspark.sql.DataFrame.withColumn` and
-       :meth:`pyspark.sql.DataFrame.select`.
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> from pyspark.sql.types import IntegerType, StringType
-       >>> slen = pandas_udf(lambda s: s.str.len(), IntegerType())  # doctest: +SKIP
-       >>> @pandas_udf(StringType())  # doctest: +SKIP
-       ... def to_upper(s):
-       ...     return s.str.upper()
-       ...
-       >>> @pandas_udf("integer", PandasUDFType.SCALAR)  # doctest: +SKIP
-       ... def add_one(x):
-       ...     return x + 1
-       ...
-       >>> df = spark.createDataFrame([(1, "John Doe", 21)],
-       ...                            ("id", "name", "age"))  # doctest: +SKIP
-       >>> df.select(slen("name").alias("slen(name)"), to_upper("name"), add_one("age")) \\
-       ...     .show()  # doctest: +SKIP
-       +----------+--------------+------------+
-       |slen(name)|to_upper(name)|add_one(age)|
-       +----------+--------------+------------+
-       |         8|      JOHN DOE|          22|
-       +----------+--------------+------------+
-       >>> @pandas_udf("first string, last string")  # doctest: +SKIP
-       ... def split_expand(n):
-       ...     return n.str.split(expand=True)
-       >>> df.select(split_expand("name")).show()  # doctest: +SKIP
-       +------------------+
-       |split_expand(name)|
-       +------------------+
-       |       [John, Doe]|
-       +------------------+
-
-       .. note:: The length of `pandas.Series` within a scalar UDF is not that of the whole input
-           column, but is the length of an internal batch used for each call to the function.
-           Therefore, this can be used, for example, to ensure the length of each returned
-           `pandas.Series`, and can not be used as the column length.
-
-    2. SCALAR_ITER
-
-       A scalar iterator UDF is semantically the same as the scalar Pandas UDF above except that the
-       wrapped Python function takes an iterator of batches as input instead of a single batch and,
-       instead of returning a single output batch, it yields output batches or explicitly returns an
-       generator or an iterator of output batches.
-       It is useful when the UDF execution requires initializing some state, e.g., loading a machine
-       learning model file to apply inference to every input batch.
-
-       .. note:: It is not guaranteed that one invocation of a scalar iterator UDF will process all
-           batches from one partition, although it is currently implemented this way.
-           Your code shall not rely on this behavior because it might change in the future for
-           further optimization, e.g., one invocation processes multiple partitions.
-
-       Scalar iterator UDFs are used with :meth:`pyspark.sql.DataFrame.withColumn` and
-       :meth:`pyspark.sql.DataFrame.select`.
-
-       >>> import pandas as pd  # doctest: +SKIP
-       >>> from pyspark.sql.functions import col, pandas_udf, struct, PandasUDFType
-       >>> pdf = pd.DataFrame([1, 2, 3], columns=["x"])  # doctest: +SKIP
-       >>> df = spark.createDataFrame(pdf)  # doctest: +SKIP
-
-       When the UDF is called with a single column that is not `StructType`, the input to the
-       underlying function is an iterator of `pd.Series`.
-
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def plus_one(batch_iter):
-       ...     for x in batch_iter:
-       ...         yield x + 1
-       ...
-       >>> df.select(plus_one(col("x"))).show()  # doctest: +SKIP
-       +-----------+
-       |plus_one(x)|
-       +-----------+
-       |          2|
-       |          3|
-       |          4|
-       +-----------+
-
-       When the UDF is called with more than one columns, the input to the underlying function is an
-       iterator of `pd.Series` tuple.
-
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def multiply_two_cols(batch_iter):
-       ...     for a, b in batch_iter:
-       ...         yield a * b
-       ...
-       >>> df.select(multiply_two_cols(col("x"), col("x"))).show()  # doctest: +SKIP
-       +-----------------------+
-       |multiply_two_cols(x, x)|
-       +-----------------------+
-       |                      1|
-       |                      4|
-       |                      9|
-       +-----------------------+
-
-       When the UDF is called with a single column that is `StructType`, the input to the underlying
-       function is an iterator of `pd.DataFrame`.
-
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def multiply_two_nested_cols(pdf_iter):
-       ...    for pdf in pdf_iter:
-       ...        yield pdf["a"] * pdf["b"]
-       ...
-       >>> df.select(
-       ...     multiply_two_nested_cols(
-       ...         struct(col("x").alias("a"), col("x").alias("b"))
-       ...     ).alias("y")
-       ... ).show()  # doctest: +SKIP
-       +---+
-       |  y|
-       +---+
-       |  1|
-       |  4|
-       |  9|
-       +---+
-
-       In the UDF, you can initialize some states before processing batches, wrap your code with
-       `try ... finally ...` or use context managers to ensure the release of resources at the end
-       or in case of early termination.
-
-       >>> y_bc = spark.sparkContext.broadcast(1)  # doctest: +SKIP
-       >>> @pandas_udf("long", PandasUDFType.SCALAR_ITER)  # doctest: +SKIP
-       ... def plus_y(batch_iter):
-       ...     y = y_bc.value  # initialize some state
-       ...     try:
-       ...         for x in batch_iter:
-       ...             yield x + y
-       ...     finally:
-       ...         pass  # release resources here, if any
-       ...
-       >>> df.select(plus_y(col("x"))).show()  # doctest: +SKIP
-       +---------+
-       |plus_y(x)|
-       +---------+
-       |        2|
-       |        3|
-       |        4|
-       +---------+
-
-    3. GROUPED_MAP
-
-       A grouped map UDF defines transformation: A `pandas.DataFrame` -> A `pandas.DataFrame`
-       The returnType should be a :class:`StructType` describing the schema of the returned
-       `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match
-       the field names in the defined returnType schema if specified as strings, or match the
-       field data types by position if not strings, e.g. integer indices.
-       The length of the returned `pandas.DataFrame` can be arbitrary.
-
-       Grouped map UDFs are used with :meth:`pyspark.sql.GroupedData.apply`.
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))  # doctest: +SKIP
-       >>> @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)  # doctest: +SKIP
-       ... def normalize(pdf):
-       ...     v = pdf.v
-       ...     return pdf.assign(v=(v - v.mean()) / v.std())
-       >>> df.groupby("id").apply(normalize).show()  # doctest: +SKIP
-       +---+-------------------+
-       | id|                  v|
-       +---+-------------------+
-       |  1|-0.7071067811865475|
-       |  1| 0.7071067811865475|
-       |  2|-0.8320502943378437|
-       |  2|-0.2773500981126146|
-       |  2| 1.1094003924504583|
-       +---+-------------------+
-
-       Alternatively, the user can define a function that takes two arguments.
-       In this case, the grouping key(s) will be passed as the first argument and the data will
-       be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy
-       data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in
-       as a `pandas.DataFrame` containing all columns from the original Spark DataFrame.
-       This is useful when the user does not want to hardcode grouping key(s) in the function.
-
-       >>> import pandas as pd  # doctest: +SKIP
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))  # doctest: +SKIP
-       >>> @pandas_udf("id long, v double", PandasUDFType.GROUPED_MAP)  # doctest: +SKIP
-       ... def mean_udf(key, pdf):
-       ...     # key is a tuple of one numpy.int64, which is the value
-       ...     # of 'id' for the current group
-       ...     return pd.DataFrame([key + (pdf.v.mean(),)])
-       >>> df.groupby('id').apply(mean_udf).show()  # doctest: +SKIP
-       +---+---+
-       | id|  v|
-       +---+---+
-       |  1|1.5|
-       |  2|6.0|
-       +---+---+
-       >>> @pandas_udf(
-       ...    "id long, `ceil(v / 2)` long, v double",
-       ...    PandasUDFType.GROUPED_MAP)  # doctest: +SKIP
-       >>> def sum_udf(key, pdf):
-       ...     # key is a tuple of two numpy.int64s, which is the values
-       ...     # of 'id' and 'ceil(df.v / 2)' for the current group
-       ...     return pd.DataFrame([key + (pdf.v.sum(),)])
-       >>> df.groupby(df.id, ceil(df.v / 2)).apply(sum_udf).show()  # doctest: +SKIP
-       +---+-----------+----+
-       | id|ceil(v / 2)|   v|
-       +---+-----------+----+
-       |  2|          5|10.0|
-       |  1|          1| 3.0|
-       |  2|          3| 5.0|
-       |  2|          2| 3.0|
-       +---+-----------+----+
-
-       .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
-           recommended to explicitly index the columns by name to ensure the positions are correct,
-           or alternatively use an `OrderedDict`.
-           For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
-           `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
-
-       .. seealso:: :meth:`pyspark.sql.GroupedData.apply`
-
-    4. GROUPED_AGG
-
-       A grouped aggregate UDF defines a transformation: One or more `pandas.Series` -> A scalar
-       The `returnType` should be a primitive data type, e.g., :class:`DoubleType`.
-       The returned scalar can be either a python primitive type, e.g., `int` or `float`
-       or a numpy data type, e.g., `numpy.int64` or `numpy.float64`.
-
-       :class:`MapType` and :class:`StructType` are currently not supported as output types.
-
-       Group aggregate UDFs are used with :meth:`pyspark.sql.GroupedData.agg` and
-       :class:`pyspark.sql.Window`
-
-       This example shows using grouped aggregated UDFs with groupby:
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))
-       >>> @pandas_udf("double", PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
-       ... def mean_udf(v):
-       ...     return v.mean()
-       >>> df.groupby("id").agg(mean_udf(df['v'])).show()  # doctest: +SKIP
-       +---+-----------+
-       | id|mean_udf(v)|
-       +---+-----------+
-       |  1|        1.5|
-       |  2|        6.0|
-       +---+-----------+
-
-       This example shows using grouped aggregated UDFs as window functions.
-
-       >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-       >>> from pyspark.sql import Window
-       >>> df = spark.createDataFrame(
-       ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-       ...     ("id", "v"))
-       >>> @pandas_udf("double", PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
-       ... def mean_udf(v):
-       ...     return v.mean()
-       >>> w = (Window.partitionBy('id')
-       ...            .orderBy('v')
-       ...            .rowsBetween(-1, 0))
-       >>> df.withColumn('mean_v', mean_udf(df['v']).over(w)).show()  # doctest: +SKIP
-       +---+----+------+
-       | id|   v|mean_v|
-       +---+----+------+
-       |  1| 1.0|   1.0|
-       |  1| 2.0|   1.5|
-       |  2| 3.0|   3.0|
-       |  2| 5.0|   4.0|
-       |  2|10.0|   7.5|
-       +---+----+------+
-
-       .. note:: For performance reasons, the input series to window functions are not copied.
+        Default: SCALAR.
+
+        .. note:: This parameter exists for compatibility. Using Python type hints is encouraged.
+
+    In order to use this API, customarily the below are imported:
+
+    >>> import pandas as pd
+    >>> from pyspark.sql.functions import pandas_udf
+
+    From Spark 3.0 with Python 3.6+, `Python type hints <https://www.python.org/dev/peps/pep-0484>`_
+    detect the function types as below:
+
+    >>> @pandas_udf(IntegerType())
+    ... def slen(s: pd.Series) -> pd.Series:
+    ...     return s.str.len()
+
+    Prior to Spark 3.0, the pandas UDF used `functionType` to decide the execution type as below:
+
+    >>> from pyspark.sql.functions import PandasUDFType
+    >>> from pyspark.sql.types import IntegerType
+    >>> @pandas_udf(IntegerType(), PandasUDFType.SCALAR)
+    ... def slen(s):
+    ...     return s.str.len()
+
+    It is preferred to specify type hints for the pandas UDF instead of specifying pandas UDF
+    type via `functionType` which will be deprecated in the future releases.
+
+    Note that the type hint should use `pandas.Series` in all cases but there is one variant
+    that `pandas.DataFrame` should be used for its input or output type hint instead when the input
+    or output column is of :class:`pyspark.sql.types.StructType`. The following example shows
+    a Pandas UDF which takes long column, string column and struct column, and outputs a struct
+    column. It requires the function to specify the type hints of `pandas.Series` and
+    `pandas.DataFrame` as below:
+
+    >>> @pandas_udf("col1 string, col2 long")
+    >>> def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
+    ...     s3['col2'] = s1 + s2.str.len()
+    ...     return s3
+    ...
+    >>> # Create a Spark DataFrame that has three columns including a sturct column.
+    ... df = spark.createDataFrame(
+    ...     [[1, "a string", ("a nested string",)]],
+    ...     "long_col long, string_col string, struct_col struct<col1:string>")
+    >>> df.printSchema()
+    root
+    |-- long_column: long (nullable = true)
+    |-- string_column: string (nullable = true)
+    |-- struct_column: struct (nullable = true)
+    |    |-- col1: string (nullable = true)
+    >>> df.select(func("long_col", "string_col", "struct_col")).printSchema()
+    |-- func(long_col, string_col, struct_col): struct (nullable = true)
+    |    |-- col1: string (nullable = true)
+    |    |-- col2: long (nullable = true)
+
+    In the following sections, it describes the cominations of the supported type hints. For
+    simplicity, `pandas.DataFrame` variant is omitted.
+
+    * Series to Series
+        `pandas.Series`, ... -> `pandas.Series`
+
+        The function takes one or more `pandas.Series` and outputs one `pandas.Series`.
+        The output of the function should always be of the same length as the input.
+
+        >>> @pandas_udf("string")
+        ... def to_upper(s: pd.Series) -> pd.Series:
+        ...     return s.str.upper()
+        ...
+        >>> df = spark.createDataFrame([("John Doe",)], ("name",))
+        >>> df.select(to_upper("name")).show()
+        +--------------+
+        |to_upper(name)|
+        +--------------+
+        |      JOHN DOE|
+        +--------------+
+
+        >>> @pandas_udf("first string, last string")
+        ... def split_expand(s: pd.Series) -> pd.DataFrame:
+        ...     return s.str.split(expand=True)
+        ...
+        >>> df = spark.createDataFrame([("John Doe",)], ("name",))
+        >>> df.select(split_expand("name")).show()
+        +------------------+
+        |split_expand(name)|
+        +------------------+
+        |       [John, Doe]|
+        +------------------+
+
+        .. note:: The length of the input is not that of the whole input column, but is the
+            length of an internal batch used for each call to the function.
+
+    * Iterator of Series to Iterator of Series
+        `Iterator[pandas.Series]` -> `Iterator[pandas.Series]`
+
+        The function takes an iterator of `pandas.Series` and outputs an iterator of
+        `pandas.Series`. In this case, the created pandas UDF instance requires one input
+        column when this is called as a PySpark column. The output of each series from
+        the function should always be of the same length as the input.
+
+        It is useful when the UDF execution
+        requires initializing some states although internally it works identically as
+        Series to Series case. The pseudocode below illustrates the example.
+
+        .. highlight:: python
+        .. code-block:: python
+
+            @pandas_udf("long")
+            def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+                # Do some expensive initialization with a state
+                state = very_expensive_initialization()
+                for x in iterator:
+                    # Use that state for whole iterator.
+                    yield calculate_with_state(x, state)
+
+            df.select(calculate("value")).show()
+
+        >>> from typing import Iterator
+        >>> @pandas_udf("long")
+        ... def plus_one(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
+        ...     for s in iterator:
+        ...         yield s + 1
+        ...
+        >>> df = spark.createDataFrame(pd.DataFrame([1, 2, 3], columns=["v"]))
+        >>> df.select(plus_one(df.v)).show()
+        +-----------+
+        |plus_one(v)|
+        +-----------+
+        |          2|
+        |          3|
+        |          4|
+        +-----------+
+
+        .. note:: The length of each series is the length of a batch internally used.
+
+    * Iterator of Multiple Series to Iterator of Series
+        `Iterator[Tuple[pandas.Series, ...]]` -> `Iterator[pandas.Series]`
+
+        The function takes an iterator of a tuple of multiple `pandas.Series` and outputs an
+        iterator of `pandas.Series`. In this case, the created pandas UDF instance requires
+        input columns as many as the series when this is called as a PySpark column.
+        It works identically as Iterator of Series to Iterator of Series case except
+        the parameter difference. The output of each series from the function should always
+        be of the same length as the input.
+
+        >>> from typing import Iterator, Tuple
+        >>> from pyspark.sql.functions import struct, col
+        >>> @pandas_udf("long")
+        ... def multiply(iterator: Iterator[Tuple[pd.Series, pd.DataFrame]]) -> Iterator[pd.Series]:
+        ...     for s1, df in iterator:
+        ...         yield s1 * df.v
+        ...
+        >>> df = spark.createDataFrame(pd.DataFrame([1, 2, 3], columns=["v"]))
+        >>> df.withColumn('output', multiply(col("v"), struct(col("v")))).show()
+        +---+------+
+        |  v|output|
+        +---+------+
+        |  1|     1|
+        |  2|     4|
+        |  3|     9|
+        +---+------+
+
+        .. note:: The length of each series is the length of a batch internally used.
+
+    * Series to Scalar
+        `pandas.Series`, ... -> `Any`
+
+        The function takes `pandas.Series` and returns a scalar value. The `returnType`
+        should be a primitive data type, and the returned scalar can be either a python primitive
+        type, e.g., int or float or a numpy data type, e.g., numpy.int64 or numpy.float64.
+        `Any` should ideally be a specific scalar type accordingly.
+
+        >>> @pandas_udf("double")
+        ... def mean_udf(v: pd.Series) -> float:
+        ...     return v.mean()
+        ...
+        >>> df = spark.createDataFrame(
+        ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], ("id", "v"))
+        >>> df.groupby("id").agg(mean_udf(df['v'])).show()
+        +---+-----------+
+        | id|mean_udf(v)|
+        +---+-----------+
+        |  1|        1.5|
+        |  2|        6.0|
+        +---+-----------+
+
+        This UDF can also be used as window functions as below:
+
+        >>> from pyspark.sql import Window
+        >>> @pandas_udf("double")
+        ... def mean_udf(v: pd.Series) -> float:
+        ...     return v.mean()
+        ...
+        >>> df = spark.createDataFrame(
+        ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)], ("id", "v"))
+        >>> w = Window.partitionBy('id').orderBy('v').rowsBetween(-1, 0)
+        >>> df.withColumn('mean_v', mean_udf("v").over(w)).show()
+        +---+----+------+
+        | id|   v|mean_v|
+        +---+----+------+
+        |  1| 1.0|   1.0|
+        |  1| 2.0|   1.5|
+        |  2| 3.0|   3.0|
+        |  2| 5.0|   4.0|
+        |  2|10.0|   7.5|
+        +---+----+------+
+
+        .. note:: For performance reasons, the input series to window functions are not copied.
             Therefore, mutating the input series is not allowed and will cause incorrect results.
             For the same reason, users should also not rely on the index of the input series.
 
-       .. seealso:: :meth:`pyspark.sql.GroupedData.agg` and :class:`pyspark.sql.Window`
+        .. seealso:: :meth:`pyspark.sql.GroupedData.agg` and :class:`pyspark.sql.Window`
 
     .. note:: The user-defined functions do not support conditional expressions or short circuiting
         in boolean expressions and it ends up with being executed all internally. If the functions
@@ -348,10 +273,21 @@ def pandas_udf(f=None, returnType=None, functionType=None):
     .. note:: The user-defined functions do not take keyword arguments on the calling side.
 
     .. note:: The data type of returned `pandas.Series` from the user-defined functions should be
-        matched with defined returnType (see :meth:`types.to_arrow_type` and
+        matched with defined `returnType` (see :meth:`types.to_arrow_type` and
         :meth:`types.from_arrow_type`). When there is mismatch between them, Spark might do
         conversion on returned data. The conversion is not guaranteed to be correct and results
         should be checked for accuracy by users.
+
+    .. note:: Currently,
+        :class:`pyspark.sql.types.MapType`,
+        :class:`pyspark.sql.types.ArrayType` of :class:`pyspark.sql.types.TimestampType` and
+        nested :class:`pyspark.sql.types.StructType`
+        are currently not supported as output types.
+
+    .. seealso:: :meth:`pyspark.sql.DataFrame.mapInPandas`
+    .. seealso:: :meth:`pyspark.sql.GroupedData.applyInPandas`
+    .. seealso:: :meth:`pyspark.sql.PandasCogroupedOps.applyInPandas`
+    .. seealso:: :meth:`pyspark.sql.UDFRegistration.register`
     """
 
     # The following table shows most of Pandas data and SQL type conversions in Pandas UDFs that
@@ -480,25 +416,3 @@ def _create_pandas_udf(f, returnType, evalType):
             "or three arguments (key, left, right).")
 
     return _create_udf(f, returnType, evalType)
-
-
-def _test():
-    import doctest
-    from pyspark.sql import SparkSession
-    import pyspark.sql.pandas.functions
-    globs = pyspark.sql.pandas.functions.__dict__.copy()
-    spark = SparkSession.builder\
-        .master("local[4]")\
-        .appName("sql.pandas.functions tests")\
-        .getOrCreate()
-    globs['spark'] = spark
-    (failure_count, test_count) = doctest.testmod(
-        pyspark.sql.pandas.functions, globs=globs,
-        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF)
-    spark.stop()
-    if failure_count:
-        sys.exit(-1)
-
-
-if __name__ == "__main__":
-    _test()
diff --git a/python/pyspark/sql/pandas/group_ops.py b/python/pyspark/sql/pandas/group_ops.py
index 3152271ba9df8..b93f0516cadb1 100644
--- a/python/pyspark/sql/pandas/group_ops.py
+++ b/python/pyspark/sql/pandas/group_ops.py
@@ -88,29 +88,27 @@ def applyInPandas(self, func, schema):
         to the user-function and the returned `pandas.DataFrame` are combined as a
         :class:`DataFrame`.
 
-        The returned `pandas.DataFrame` can be of arbitrary length and its schema must match the
-        returnType of the pandas udf.
-
-        .. note:: This function requires a full shuffle. All the data of a group will be loaded
-            into memory, so the user should be aware of the potential OOM risk if data is skewed
-            and certain groups are too large to fit in memory.
+        The `schema` should be a :class:`StructType` describing the schema of the returned
+        `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match
+        the field names in the defined schema if specified as strings, or match the
+        field data types by position if not strings, e.g. integer indices.
+        The length of the returned `pandas.DataFrame` can be arbitrary.
 
         :param func: a Python native function that takes a `pandas.DataFrame`, and outputs a
             `pandas.DataFrame`.
         :param schema: the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
-        .. note:: Experimental
-
-        >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
+        >>> import pandas as pd  # doctest: +SKIP
+        >>> from pyspark.sql.functions import pandas_udf, ceil
         >>> df = spark.createDataFrame(
         ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
-        ...     ("id", "v"))
+        ...     ("id", "v"))  # doctest: +SKIP
         >>> def normalize(pdf):
         ...     v = pdf.v
         ...     return pdf.assign(v=(v - v.mean()) / v.std())
-        >>> df.groupby("id").applyInPandas(normalize, schema="id long, v double").show()
-        ... # doctest: +SKIP
+        >>> df.groupby("id").applyInPandas(
+        ...     normalize, schema="id long, v double").show()  # doctest: +SKIP
         +---+-------------------+
         | id|                  v|
         +---+-------------------+
@@ -121,8 +119,56 @@ def applyInPandas(self, func, schema):
         |  2| 1.1094003924504583|
         +---+-------------------+
 
-        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
+        Alternatively, the user can pass a function that takes two arguments.
+        In this case, the grouping key(s) will be passed as the first argument and the data will
+        be passed as the second argument. The grouping key(s) will be passed as a tuple of numpy
+        data types, e.g., `numpy.int32` and `numpy.float64`. The data will still be passed in
+        as a `pandas.DataFrame` containing all columns from the original Spark DataFrame.
+        This is useful when the user does not want to hardcode grouping key(s) in the function.
+
+        >>> df = spark.createDataFrame(
+        ...     [(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
+        ...     ("id", "v"))  # doctest: +SKIP
+        >>> def mean_func(key, pdf):
+        ...     # key is a tuple of one numpy.int64, which is the value
+        ...     # of 'id' for the current group
+        ...     return pd.DataFrame([key + (pdf.v.mean(),)])
+        >>> df.groupby('id').applyInPandas(
+        ...     mean_func, schema="id long, v double").show()  # doctest: +SKIP
+        +---+---+
+        | id|  v|
+        +---+---+
+        |  1|1.5|
+        |  2|6.0|
+        +---+---+
+        >>> def sum_func(key, pdf):
+        ...     # key is a tuple of two numpy.int64s, which is the values
+        ...     # of 'id' and 'ceil(df.v / 2)' for the current group
+        ...     return pd.DataFrame([key + (pdf.v.sum(),)])
+        >>> df.groupby(df.id, ceil(df.v / 2)).applyInPandas(
+        ...     sum_func, schema="id long, `ceil(v / 2)` long, v double").show()  # doctest: +SKIP
+        +---+-----------+----+
+        | id|ceil(v / 2)|   v|
+        +---+-----------+----+
+        |  2|          5|10.0|
+        |  1|          1| 3.0|
+        |  2|          3| 5.0|
+        |  2|          2| 3.0|
+        +---+-----------+----+
+
+        .. note:: This function requires a full shuffle. All the data of a group will be loaded
+            into memory, so the user should be aware of the potential OOM risk if data is skewed
+            and certain groups are too large to fit in memory.
+
+        .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
+            recommended to explicitly index the columns by name to ensure the positions are correct,
+            or alternatively use an `OrderedDict`.
+            For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
+            `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
 
+        .. note:: Experimental
+
+        .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
         """
         from pyspark.sql import GroupedData
         from pyspark.sql.functions import pandas_udf, PandasUDFType
@@ -176,14 +222,11 @@ def applyInPandas(self, func, schema):
         `pandas.DataFrame` to the user-function and the returned `pandas.DataFrame` are combined as
         a :class:`DataFrame`.
 
-        The returned `pandas.DataFrame` can be of arbitrary length and its schema must match the
-        returnType of the pandas udf.
-
-        .. note:: This function requires a full shuffle. All the data of a cogroup will be loaded
-            into memory, so the user should be aware of the potential OOM risk if data is skewed
-            and certain groups are too large to fit in memory.
-
-        .. note:: Experimental
+        The `schema` should be a :class:`StructType` describing the schema of the returned
+        `pandas.DataFrame`. The column labels of the returned `pandas.DataFrame` must either match
+        the field names in the defined schema if specified as strings, or match the
+        field data types by position if not strings, e.g. integer indices.
+        The length of the returned `pandas.DataFrame` can be arbitrary.
 
         :param func: a Python native function that takes two `pandas.DataFrame`\\s, and
             outputs a `pandas.DataFrame`, or that takes one tuple (grouping keys) and two
@@ -191,7 +234,7 @@ def applyInPandas(self, func, schema):
         :param schema: the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
-        >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
+        >>> from pyspark.sql.functions import pandas_udf
         >>> df1 = spark.createDataFrame(
         ...     [(20000101, 1, 1.0), (20000101, 2, 2.0), (20000102, 1, 3.0), (20000102, 2, 4.0)],
         ...     ("time", "id", "v1"))
@@ -232,6 +275,18 @@ def applyInPandas(self, func, schema):
         |20000102|  1|3.0|  x|
         +--------+---+---+---+
 
+        .. note:: This function requires a full shuffle. All the data of a cogroup will be loaded
+            into memory, so the user should be aware of the potential OOM risk if data is skewed
+            and certain groups are too large to fit in memory.
+
+        .. note:: If returning a new `pandas.DataFrame` constructed with a dictionary, it is
+            recommended to explicitly index the columns by name to ensure the positions are correct,
+            or alternatively use an `OrderedDict`.
+            For example, `pd.DataFrame({'id': ids, 'a': data}, columns=['id', 'a'])` or
+            `pd.DataFrame(OrderedDict([('id', ids), ('a', data)]))`.
+
+        .. note:: Experimental
+
         .. seealso:: :meth:`pyspark.sql.functions.pandas_udf`
 
         """
diff --git a/python/pyspark/sql/pandas/map_ops.py b/python/pyspark/sql/pandas/map_ops.py
index 75cacd797f9dd..9835e88c6ac21 100644
--- a/python/pyspark/sql/pandas/map_ops.py
+++ b/python/pyspark/sql/pandas/map_ops.py
@@ -45,10 +45,10 @@ def mapInPandas(self, func, schema):
         :param schema: the return type of the `func` in PySpark. The value can be either a
             :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
 
-        >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
+        >>> from pyspark.sql.functions import pandas_udf
         >>> df = spark.createDataFrame([(1, 21), (2, 30)], ("id", "age"))
-        >>> def filter_func(batch_iter):
-        ...     for pdf in batch_iter:
+        >>> def filter_func(iterator):
+        ...     for pdf in iterator:
         ...         yield pdf[pdf.id == 1]
         >>> df.mapInPandas(filter_func, df.schema).show()  # doctest: +SKIP
         +---+---+
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 433c5fc845c59..10546ecacc57f 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -297,17 +297,18 @@ def register(self, name, f, returnType=None):
             >>> spark.sql("SELECT random_udf()").collect()  # doctest: +SKIP
             [Row(random_udf()=82)]
 
-            >>> from pyspark.sql.functions import pandas_udf, PandasUDFType
-            >>> @pandas_udf("integer", PandasUDFType.SCALAR)  # doctest: +SKIP
-            ... def add_one(x):
-            ...     return x + 1
+            >>> import pandas as pd  # doctest: +SKIP
+            >>> from pyspark.sql.functions import pandas_udf
+            >>> @pandas_udf("integer")  # doctest: +SKIP
+            ... def add_one(s: pd.Series) -> pd.Series:
+            ...     return s + 1
             ...
             >>> _ = spark.udf.register("add_one", add_one)  # doctest: +SKIP
             >>> spark.sql("SELECT add_one(id) FROM range(3)").collect()  # doctest: +SKIP
             [Row(add_one(id)=1), Row(add_one(id)=2), Row(add_one(id)=3)]
 
-            >>> @pandas_udf("integer", PandasUDFType.GROUPED_AGG)  # doctest: +SKIP
-            ... def sum_udf(v):
+            >>> @pandas_udf("integer")  # doctest: +SKIP
+            ... def sum_udf(v: pd.Series) -> int:
             ...     return v.sum()
             ...
             >>> _ = spark.udf.register("sum_udf", sum_udf)  # doctest: +SKIP
@@ -414,6 +415,9 @@ def _test():
         .appName("sql.udf tests")\
         .getOrCreate()
     globs['spark'] = spark
+    # Hack to skip the unit tests in register. These are currently being tested in proper tests.
+    # We should reenable this test once we completely drop Python 2.
+    del pyspark.sql.udf.UDFRegistration.register
     (failure_count, test_count) = doctest.testmod(
         pyspark.sql.udf, globs=globs,
         optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE)

From 6e1b6cc5c55c4d945f59da68d248cc3ef82569d3 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 10 Feb 2020 10:56:43 -0800
Subject: [PATCH 0055/1280] Revert "[SPARK-30245][SQL] Add cache for Like and
 RLike when pattern is not static"

### What changes were proposed in this pull request?

This reverts commit 8ce7962931680c204e84dd75783b1c943ea9c525. There's variable name conflicts with https://github.com/apache/spark/commit/8aebc80e0e67bcb1aa300b8c8b1a209159237632#diff-39298b470865a4cbc67398a4ea11e767.

This can be cleanly ported back to branch-3.0.

### Why are the changes needed?
Performance investigation were not made enough and it's not clear if it really beneficial or now.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Jenkins tests.

Closes #27514 from HyukjinKwon/revert-cache-PR.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
---
 .../expressions/regexpExpressions.scala       | 21 ++++++-------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index c9ddc70bf5bc6..f84c476ea5807 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -177,6 +177,8 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
         """)
       }
     } else {
+      val patternStr = ctx.freshName("patternStr")
+      val compiledPattern = ctx.freshName("compiledPattern")
       // We need double escape to avoid org.codehaus.commons.compiler.CompileException.
       // '\\' will cause exception 'Single quote must be backslash-escaped in character literal'.
       // '\"' will cause exception 'Line break in literal not allowed'.
@@ -185,17 +187,11 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
       } else {
         escapeChar
       }
-      val patternStr = ctx.freshName("patternStr")
-      val compiledPattern = ctx.addMutableState(patternClass, "compiledPattern")
-      val lastPatternStr = ctx.addMutableState(classOf[String].getName, "lastPatternStr")
-
       nullSafeCodeGen(ctx, ev, (eval1, eval2, _) => {
         s"""
           String $patternStr = $eval2.toString();
-          if (!$patternStr.equals($lastPatternStr)) {
-            $compiledPattern = $patternClass.compile($escapeFunc($patternStr, '$newEscapeChar'));
-            $lastPatternStr = $patternStr;
-          }
+          $patternClass $compiledPattern = $patternClass.compile(
+            $escapeFunc($patternStr, '$newEscapeChar'));
           ${ev.value} = $compiledPattern.matcher($eval1.toString()).matches();
         """
       })
@@ -278,16 +274,11 @@ case class RLike(left: Expression, right: Expression)
       }
     } else {
       val rightStr = ctx.freshName("rightStr")
-      val pattern = ctx.addMutableState(patternClass, "pattern")
-      val lastRightStr = ctx.addMutableState(classOf[String].getName, "lastRightStr")
-
+      val pattern = ctx.freshName("pattern")
       nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         s"""
           String $rightStr = $eval2.toString();
-          if (!$rightStr.equals($lastRightStr)) {
-            $pattern = $patternClass.compile($rightStr);
-            $lastRightStr = $rightStr;
-          }
+          $patternClass $pattern = $patternClass.compile($rightStr);
           ${ev.value} = $pattern.matcher($eval1.toString()).find(0);
         """
       })

From 9c739358487acf3cd2d5171cebdd053c098aeb8c Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Tue, 11 Feb 2020 10:15:34 -0800
Subject: [PATCH 0056/1280] Revert "[SPARK-30625][SQL] Support `escape` as
 third parameter of the `like` function

In the PR, I propose to revert the commit 8aebc80e0e67bcb1aa300b8c8b1a209159237632.

See the concerns https://github.com/apache/spark/pull/27355#issuecomment-584344438

No

By existing test suites.

Closes #27531 from MaxGekk/revert-like-3-args.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/dsl/package.scala      |  2 +-
 .../expressions/regexpExpressions.scala       | 85 ++++++-------------
 .../sql/catalyst/parser/AstBuilder.scala      |  4 +-
 .../spark/sql/DataFrameFunctionsSuite.scala   | 15 ----
 4 files changed, 31 insertions(+), 75 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index 40998080bc4e3..b4a8bafe22dfb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -99,7 +99,7 @@ package object dsl {
     }
 
     def like(other: Expression, escapeChar: Char = '\\'): Expression =
-      Like(expr, other, Literal(escapeChar.toString))
+      Like(expr, other, escapeChar)
     def rlike(other: Expression): Expression = RLike(expr, other)
     def contains(other: Expression): Expression = Contains(expr, other)
     def startsWith(other: Expression): Expression = StartsWith(expr, other)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index f84c476ea5807..32a653dba8fd4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -22,7 +22,6 @@ import java.util.regex.{MatchResult, Pattern}
 
 import org.apache.commons.text.StringEscapeUtils
 
-import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{GenericArrayData, StringUtils}
@@ -30,19 +29,17 @@ import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
 
-trait StringRegexExpression extends Expression
+abstract class StringRegexExpression extends BinaryExpression
   with ImplicitCastInputTypes with NullIntolerant {
 
-  def str: Expression
-  def pattern: Expression
-
   def escape(v: String): String
   def matches(regex: Pattern, str: String): Boolean
 
   override def dataType: DataType = BooleanType
+  override def inputTypes: Seq[DataType] = Seq(StringType, StringType)
 
   // try cache the pattern for Literal
-  private lazy val cache: Pattern = pattern match {
+  private lazy val cache: Pattern = right match {
     case Literal(value: String, StringType) => compile(value)
     case _ => null
   }
@@ -54,9 +51,10 @@ trait StringRegexExpression extends Expression
     Pattern.compile(escape(str))
   }
 
-  def nullSafeMatch(input1: Any, input2: Any): Any = {
-    val s = input2.asInstanceOf[UTF8String].toString
-    val regex = if (cache == null) compile(s) else cache
+  protected def pattern(str: String) = if (cache == null) compile(str) else cache
+
+  protected override def nullSafeEval(input1: Any, input2: Any): Any = {
+    val regex = pattern(input2.asInstanceOf[UTF8String].toString)
     if(regex == null) {
       null
     } else {
@@ -64,7 +62,7 @@ trait StringRegexExpression extends Expression
     }
   }
 
-  override def sql: String = s"${str.sql} ${prettyName.toUpperCase(Locale.ROOT)} ${pattern.sql}"
+  override def sql: String = s"${left.sql} ${prettyName.toUpperCase(Locale.ROOT)} ${right.sql}"
 }
 
 // scalastyle:off line.contains.tab
@@ -109,65 +107,46 @@ trait StringRegexExpression extends Expression
       true
       > SELECT '%SystemDrive%/Users/John' _FUNC_ '/%SystemDrive/%//Users%' ESCAPE '/';
       true
-      > SELECT _FUNC_('_Apache Spark_', '__%Spark__', '_');
-      true
   """,
   note = """
     Use RLIKE to match with standard regular expressions.
   """,
   since = "1.0.0")
 // scalastyle:on line.contains.tab
-case class Like(str: Expression, pattern: Expression, escape: Expression)
-  extends TernaryExpression with StringRegexExpression {
-
-  def this(str: Expression, pattern: Expression) = this(str, pattern, Literal("\\"))
-
-  override def inputTypes: Seq[DataType] = Seq(StringType, StringType, StringType)
-  override def children: Seq[Expression] = Seq(str, pattern, escape)
+case class Like(left: Expression, right: Expression, escapeChar: Char)
+  extends StringRegexExpression {
 
-  private lazy val escapeChar: Char = if (escape.foldable) {
-    escape.eval() match {
-      case s: UTF8String if s != null && s.numChars() == 1 => s.toString.charAt(0)
-      case s => throw new AnalysisException(
-        s"The 'escape' parameter must be a string literal of one char but it is $s.")
-    }
-  } else {
-    throw new AnalysisException("The 'escape' parameter must be a string literal.")
-  }
+  def this(left: Expression, right: Expression) = this(left, right, '\\')
 
   override def escape(v: String): String = StringUtils.escapeLikeRegex(v, escapeChar)
 
   override def matches(regex: Pattern, str: String): Boolean = regex.matcher(str).matches()
 
   override def toString: String = escapeChar match {
-    case '\\' => s"$str LIKE $pattern"
-    case c => s"$str LIKE $pattern ESCAPE '$c'"
-  }
-
-  protected override def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
-    nullSafeMatch(input1, input2)
+    case '\\' => s"$left LIKE $right"
+    case c => s"$left LIKE $right ESCAPE '$c'"
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val patternClass = classOf[Pattern].getName
     val escapeFunc = StringUtils.getClass.getName.stripSuffix("$") + ".escapeLikeRegex"
 
-    if (pattern.foldable) {
-      val patternVal = pattern.eval()
-      if (patternVal != null) {
+    if (right.foldable) {
+      val rVal = right.eval()
+      if (rVal != null) {
         val regexStr =
-          StringEscapeUtils.escapeJava(escape(patternVal.asInstanceOf[UTF8String].toString()))
-        val compiledPattern = ctx.addMutableState(patternClass, "compiledPattern",
+          StringEscapeUtils.escapeJava(escape(rVal.asInstanceOf[UTF8String].toString()))
+        val pattern = ctx.addMutableState(patternClass, "patternLike",
           v => s"""$v = $patternClass.compile("$regexStr");""")
 
         // We don't use nullSafeCodeGen here because we don't want to re-evaluate right again.
-        val eval = str.genCode(ctx)
+        val eval = left.genCode(ctx)
         ev.copy(code = code"""
           ${eval.code}
           boolean ${ev.isNull} = ${eval.isNull};
           ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
           if (!${ev.isNull}) {
-            ${ev.value} = $compiledPattern.matcher(${eval.value}.toString()).matches();
+            ${ev.value} = $pattern.matcher(${eval.value}.toString()).matches();
           }
         """)
       } else {
@@ -177,8 +156,8 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
         """)
       }
     } else {
-      val patternStr = ctx.freshName("patternStr")
-      val compiledPattern = ctx.freshName("compiledPattern")
+      val pattern = ctx.freshName("pattern")
+      val rightStr = ctx.freshName("rightStr")
       // We need double escape to avoid org.codehaus.commons.compiler.CompileException.
       // '\\' will cause exception 'Single quote must be backslash-escaped in character literal'.
       // '\"' will cause exception 'Line break in literal not allowed'.
@@ -187,12 +166,12 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
       } else {
         escapeChar
       }
-      nullSafeCodeGen(ctx, ev, (eval1, eval2, _) => {
+      nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         s"""
-          String $patternStr = $eval2.toString();
-          $patternClass $compiledPattern = $patternClass.compile(
-            $escapeFunc($patternStr, '$newEscapeChar'));
-          ${ev.value} = $compiledPattern.matcher($eval1.toString()).matches();
+          String $rightStr = $eval2.toString();
+          $patternClass $pattern = $patternClass.compile(
+            $escapeFunc($rightStr, '$newEscapeChar'));
+          ${ev.value} = $pattern.matcher($eval1.toString()).matches();
         """
       })
     }
@@ -231,20 +210,12 @@ case class Like(str: Expression, pattern: Expression, escape: Expression)
   """,
   since = "1.0.0")
 // scalastyle:on line.contains.tab
-case class RLike(left: Expression, right: Expression)
-  extends BinaryExpression with StringRegexExpression {
-
-  override def inputTypes: Seq[DataType] = Seq(StringType, StringType)
-
-  override def str: Expression = left
-  override def pattern: Expression = right
+case class RLike(left: Expression, right: Expression) extends StringRegexExpression {
 
   override def escape(v: String): String = v
   override def matches(regex: Pattern, str: String): Boolean = regex.matcher(str).find(0)
   override def toString: String = s"$left RLIKE $right"
 
-  protected override def nullSafeEval(input1: Any, input2: Any): Any = nullSafeMatch(input1, input2)
-
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val patternClass = classOf[Pattern].getName
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 6fc65e14868e0..62e568587fcc6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1392,9 +1392,9 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
             throw new ParseException("Invalid escape string." +
               "Escape string must contains only one character.", ctx)
           }
-          str
+          str.charAt(0)
         }.getOrElse('\\')
-        invertIfNotDefined(Like(e, expression(ctx.pattern), Literal(escapeChar)))
+        invertIfNotDefined(Like(e, expression(ctx.pattern), escapeChar))
       case SqlBaseParser.RLIKE =>
         invertIfNotDefined(RLike(e, expression(ctx.pattern)))
       case SqlBaseParser.NULL if ctx.NOT != null =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 9e9d8c3e9a7c5..6012678341ccc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -3560,21 +3560,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       Seq(Row(1)))
   }
 
-  test("the like function with the escape parameter") {
-    val df = Seq(("abc", "a_c", "!")).toDF("str", "pattern", "escape")
-    checkAnswer(df.selectExpr("like(str, pattern, '@')"), Row(true))
-
-    val longEscapeError = intercept[AnalysisException] {
-      df.selectExpr("like(str, pattern, '@%')").collect()
-    }.getMessage
-    assert(longEscapeError.contains("The 'escape' parameter must be a string literal of one char"))
-
-    val nonFoldableError = intercept[AnalysisException] {
-      df.selectExpr("like(str, pattern, escape)").collect()
-    }.getMessage
-    assert(nonFoldableError.contains("The 'escape' parameter must be a string literal"))
-  }
-
   test("SPARK-29462: Empty array of NullType for array function with no arguments") {
     Seq((true, StringType), (false, NullType)).foreach {
       case (arrayDefaultToString, expectedType) =>

From 0608361e5b14dcb08d631701894bd1cea7e39fdd Mon Sep 17 00:00:00 2001
From: Kris Mok <kris.mok@databricks.com>
Date: Wed, 12 Feb 2020 15:19:16 +0900
Subject: [PATCH 0057/1280] [SPARK-30795][SQL] Spark SQL codegen's code()
 interpolator should treat escapes like Scala's StringContext.s()

### What changes were proposed in this pull request?

This PR proposes to make the `code` string interpolator treat escapes the same way as Scala's builtin `StringContext.s()` string interpolator. This will remove the need for an ugly workaround in `Like` expression's codegen.

### Why are the changes needed?

The `code()` string interpolator in Spark SQL's code generator should treat escapes like Scala's builtin `StringContext.s()` interpolator, i.e. it should treat escapes in the code parts, and should not treat escapes in the input arguments.

For example,
```scala
val arg = "This is an argument."
val str = s"This is string part 1. $arg This is string part 2."
val code = code"This is string part 1. $arg This is string part 2."
assert(code.toString == str)
```
We should expect the `code()` interpolator to produce the same result as the `StringContext.s()` interpolator, where only escapes in the string parts should be treated, while the args should be kept verbatim.

But in the current implementation, due to the eager folding of code parts and literal input args, the escape treatment is incorrectly done on both code parts and literal args.
That causes a problem when an arg contains escape sequences and wants to preserve that in the final produced code string. For example, in `Like` expression's codegen, there's an ugly workaround for this bug:
```scala
      // We need double escape to avoid org.codehaus.commons.compiler.CompileException.
      // '\\' will cause exception 'Single quote must be backslash-escaped in character literal'.
      // '\"' will cause exception 'Line break in literal not allowed'.
      val newEscapeChar = if (escapeChar == '\"' || escapeChar == '\\') {
        s"""\\\\\\$escapeChar"""
      } else {
        escapeChar
      }
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added a new unit test case in `CodeBlockSuite`.

Closes #27544 from rednaxelafx/fix-code-string-interpolator.

Authored-by: Kris Mok <kris.mok@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/expressions/codegen/javaCode.scala | 13 +++++++++----
 .../catalyst/expressions/regexpExpressions.scala    | 13 ++++---------
 .../expressions/codegen/CodeBlockSuite.scala        | 12 ++++++++++++
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
index d9393b9df6bbd..dff258902a0b1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
@@ -223,6 +223,11 @@ object Block {
   implicit def blocksToBlock(blocks: Seq[Block]): Block = blocks.reduceLeft(_ + _)
 
   implicit class BlockHelper(val sc: StringContext) extends AnyVal {
+    /**
+     * A string interpolator that retains references to the `JavaCode` inputs, and behaves like
+     * the Scala builtin StringContext.s() interpolator otherwise, i.e. it will treat escapes in
+     * the code parts, and will not treat escapes in the input arguments.
+     */
     def code(args: Any*): Block = {
       sc.checkLengths(args)
       if (sc.parts.length == 0) {
@@ -250,7 +255,7 @@ object Block {
     val inputs = args.iterator
     val buf = new StringBuilder(Block.CODE_BLOCK_BUFFER_LENGTH)
 
-    buf.append(strings.next)
+    buf.append(StringContext.treatEscapes(strings.next))
     while (strings.hasNext) {
       val input = inputs.next
       input match {
@@ -262,7 +267,7 @@ object Block {
         case _ =>
           buf.append(input)
       }
-      buf.append(strings.next)
+      buf.append(StringContext.treatEscapes(strings.next))
     }
     codeParts += buf.toString
 
@@ -286,10 +291,10 @@ case class CodeBlock(codeParts: Seq[String], blockInputs: Seq[JavaCode]) extends
     val strings = codeParts.iterator
     val inputs = blockInputs.iterator
     val buf = new StringBuilder(Block.CODE_BLOCK_BUFFER_LENGTH)
-    buf.append(StringContext.treatEscapes(strings.next))
+    buf.append(strings.next)
     while (strings.hasNext) {
       buf.append(inputs.next)
-      buf.append(StringContext.treatEscapes(strings.next))
+      buf.append(strings.next)
     }
     buf.toString
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index 32a653dba8fd4..ac620b10cfd2e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -158,19 +158,14 @@ case class Like(left: Expression, right: Expression, escapeChar: Char)
     } else {
       val pattern = ctx.freshName("pattern")
       val rightStr = ctx.freshName("rightStr")
-      // We need double escape to avoid org.codehaus.commons.compiler.CompileException.
-      // '\\' will cause exception 'Single quote must be backslash-escaped in character literal'.
-      // '\"' will cause exception 'Line break in literal not allowed'.
-      val newEscapeChar = if (escapeChar == '\"' || escapeChar == '\\') {
-        s"""\\\\\\$escapeChar"""
-      } else {
-        escapeChar
-      }
+      // We need to escape the escapeChar to make sure the generated code is valid.
+      // Otherwise we'll hit org.codehaus.commons.compiler.CompileException.
+      val escapedEscapeChar = StringEscapeUtils.escapeJava(escapeChar.toString)
       nullSafeCodeGen(ctx, ev, (eval1, eval2) => {
         s"""
           String $rightStr = $eval2.toString();
           $patternClass $pattern = $patternClass.compile(
-            $escapeFunc($rightStr, '$newEscapeChar'));
+            $escapeFunc($rightStr, '$escapedEscapeChar'));
           ${ev.value} = $pattern.matcher($eval1.toString()).matches();
         """
       })
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
index 55569b6f2933e..67e3bc69543e8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeBlockSuite.scala
@@ -37,6 +37,18 @@ class CodeBlockSuite extends SparkFunSuite {
     assert(code.asInstanceOf[CodeBlock].blockInputs === Seq(value))
   }
 
+  test("Code parts should be treated for escapes, but string inputs shouldn't be") {
+    val strlit = raw"\\"
+    val code = code"""String s = "foo\\bar" + "$strlit";"""
+
+    val builtin = s"""String s = "foo\\bar" + "$strlit";"""
+
+    val expected = raw"""String s = "foo\bar" + "\\";"""
+
+    assert(builtin == expected)
+    assert(code.asInstanceOf[CodeBlock].toString == expected)
+  }
+
   test("Block.stripMargin") {
     val isNull = JavaCode.isNullVariable("expr1_isNull")
     val value = JavaCode.variable("expr1", IntegerType)

From 7c5d7d78ddc403a3e3701b2e8dc1f4b2885e1a84 Mon Sep 17 00:00:00 2001
From: turbofei <fwang12@ebay.com>
Date: Wed, 12 Feb 2020 20:21:52 +0900
Subject: [PATCH 0058/1280] [SPARK-29542][FOLLOW-UP] Keep the description of
 spark.sql.files.* in tuning guide be consistent with that in SQLConf

### What changes were proposed in this pull request?
This pr is a follow up of https://github.com/apache/spark/pull/26200.

In this PR, I modify the description of spark.sql.files.* in sql-performance-tuning.md to keep consistent with that in SQLConf.

### Why are the changes needed?

To keep consistent with the description in SQLConf.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existed UT.

Closes #27545 from turboFei/SPARK-29542-follow-up.

Authored-by: turbofei <fwang12@ebay.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8b1839728acaa5e61f542a7332505289726d3162)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-performance-tuning.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index e289854c7acc7..5a86c0cc31e12 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -67,6 +67,7 @@ that these options will be deprecated in future release as more optimizations ar
     <td>134217728 (128 MB)</td>
     <td>
       The maximum number of bytes to pack into a single partition when reading files.
+      This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.
     </td>
   </tr>
   <tr>
@@ -76,7 +77,8 @@ that these options will be deprecated in future release as more optimizations ar
       The estimated cost to open a file, measured by the number of bytes could be scanned in the same
       time. This is used when putting multiple files into a partition. It is better to over-estimated,
       then the partitions with small files will be faster than partitions with bigger files (which is
-      scheduled first).
+      scheduled first). This configuration is effective only when using file-based sources such as Parquet,
+      JSON and ORC.
     </td>
   </tr>
   <tr>

From 2a059e65bae93ddb61f7154d81da3fa0c2dcb669 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Wed, 12 Feb 2020 20:12:38 +0800
Subject: [PATCH 0059/1280] [SPARK-30788][SQL] Support `SimpleDateFormat` and
 `FastDateFormat` as legacy date/timestamp formatters

### What changes were proposed in this pull request?
In the PR, I propose to add legacy date/timestamp formatters based on `SimpleDateFormat` and `FastDateFormat`:
- `LegacyFastTimestampFormatter` - uses `FastDateFormat` and supports parsing/formatting in microsecond precision. The code was borrowed from Spark 2.4, see https://github.com/apache/spark/pull/26507 & https://github.com/apache/spark/pull/26582
- `LegacySimpleTimestampFormatter` uses `SimpleDateFormat`, and support the `lenient` mode. When the `lenient` parameter is set to `false`, the parser become much stronger in checking its input.

### Why are the changes needed?
Spark 2.4.x uses the following parsers for parsing/formatting date/timestamp strings:
- `DateTimeFormat` in CSV/JSON datasource
- `SimpleDateFormat` - is used in JDBC datasource, in partitions parsing.
- `SimpleDateFormat` in strong mode (`lenient = false`), see https://github.com/apache/spark/blob/branch-2.4/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala#L124. It is used by the `date_format`, `from_unixtime`, `unix_timestamp` and `to_unix_timestamp` functions.

The PR aims to make Spark 3.0 compatible with Spark 2.4.x in all those cases when `spark.sql.legacy.timeParser.enabled` is set to `true`.

### Does this PR introduce any user-facing change?
This shouldn't change behavior with default settings. If `spark.sql.legacy.timeParser.enabled` is set to `true`, users should observe behavior of Spark 2.4.

### How was this patch tested?
- Modified tests in `DateExpressionsSuite` to check the legacy parser - `SimpleDateFormat`.
- Added `CSVLegacyTimeParserSuite` and `JsonLegacyTimeParserSuite` to run `CSVSuite` and `JsonSuite` with the legacy parser - `FastDateFormat`.

Closes #27524 from MaxGekk/timestamp-formatter-legacy-fallback.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c1986204e59f1e8cc4b611d5a578cb248cb74c28)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/csv/CSVInferSchema.scala     |   4 +-
 .../spark/sql/catalyst/csv/CSVOptions.scala   |   4 +-
 .../sql/catalyst/csv/UnivocityGenerator.scala |   7 +-
 .../sql/catalyst/csv/UnivocityParser.scala    |   7 +-
 .../expressions/datetimeExpressions.scala     |  52 ++-
 .../spark/sql/catalyst/json/JSONOptions.scala |   4 +-
 .../sql/catalyst/json/JacksonGenerator.scala  |   7 +-
 .../sql/catalyst/json/JacksonParser.scala     |   7 +-
 .../sql/catalyst/json/JsonInferSchema.scala   |   4 +-
 .../sql/catalyst/util/DateFormatter.scala     |  66 ++-
 .../catalyst/util/TimestampFormatter.scala    | 132 +++++-
 .../org/apache/spark/sql/types/Decimal.scala  |   2 +-
 .../expressions/DateExpressionsSuite.scala    | 390 ++++++++++--------
 .../org/apache/spark/sql/functions.scala      |   7 +-
 .../resources/test-data/bad_after_good.csv    |   2 +-
 .../resources/test-data/value-malformed.csv   |   2 +-
 .../apache/spark/sql/DateFunctionsSuite.scala | 346 +++++++++-------
 .../execution/datasources/csv/CSVSuite.scala  |  23 +-
 .../datasources/json/JsonSuite.scala          |   7 +
 19 files changed, 654 insertions(+), 419 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index 03cc3cbdf790a..c6a03183ab45e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -24,6 +24,7 @@ import scala.util.control.Exception.allCatch
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.catalyst.util.TimestampFormatter
 import org.apache.spark.sql.types._
 
@@ -32,7 +33,8 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
   private val timestampParser = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
 
   private val decimalParser = if (options.locale == Locale.US) {
     // Special handling the default locale for backward compatibility
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
index 5e40d74e54f11..8892037e03a7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -146,10 +146,10 @@ class CSVOptions(
   // A language tag in IETF BCP 47 format
   val locale: Locale = parameters.get("locale").map(Locale.forLanguageTag).getOrElse(Locale.US)
 
-  val dateFormat: String = parameters.getOrElse("dateFormat", "uuuu-MM-dd")
+  val dateFormat: String = parameters.getOrElse("dateFormat", DateFormatter.defaultPattern)
 
   val timestampFormat: String =
-    parameters.getOrElse("timestampFormat", "uuuu-MM-dd'T'HH:mm:ss.SSSXXX")
+    parameters.getOrElse("timestampFormat", s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX")
 
   val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
index 05cb91d10868e..00e3d49787db1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
@@ -23,6 +23,7 @@ import com.univocity.parsers.csv.CsvWriter
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.types._
 
 class UnivocityGenerator(
@@ -44,11 +45,13 @@ class UnivocityGenerator(
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
 
   private def makeConverter(dataType: DataType): ValueConverter = dataType match {
     case DateType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index 5510953804025..cd69c21a01976 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -27,6 +27,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{ExprUtils, GenericInternalRow}
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -86,11 +87,13 @@ class UnivocityParser(
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
 
   private val csvFilters = new CSVFilters(filters, requiredSchema)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index aa2bd5a1273e0..1f4c8c041f8bf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -30,9 +30,10 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.SIMPLE_DATE_FORMAT
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -622,13 +623,15 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
 
   @transient private lazy val formatter: Option[TimestampFormatter] = {
     if (right.foldable) {
-      Option(right.eval()).map(format => TimestampFormatter(format.toString, zoneId))
+      Option(right.eval()).map { format =>
+        TimestampFormatter(format.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+      }
     } else None
   }
 
   override protected def nullSafeEval(timestamp: Any, format: Any): Any = {
     val tf = if (formatter.isEmpty) {
-      TimestampFormatter(format.toString, zoneId)
+      TimestampFormatter(format.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
     } else {
       formatter.get
     }
@@ -643,10 +646,14 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
       })
     }.getOrElse {
       val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+      val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
       val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
       defineCodeGen(ctx, ev, (timestamp, format) => {
-        s"""UTF8String.fromString($tf$$.MODULE$$.apply($format.toString(), $zid)
-          .format($timestamp))"""
+        s"""|UTF8String.fromString($tf$$.MODULE$$.apply(
+            |  $format.toString(),
+            |  $zid,
+            |  $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT())
+            |.format($timestamp))""".stripMargin
       })
     }
   }
@@ -688,7 +695,7 @@ case class ToUnixTimestamp(
     copy(timeZoneId = Option(timeZoneId))
 
   def this(time: Expression) = {
-    this(time, Literal("uuuu-MM-dd HH:mm:ss"))
+    this(time, Literal(TimestampFormatter.defaultPattern))
   }
 
   override def prettyName: String = "to_unix_timestamp"
@@ -732,7 +739,7 @@ case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Op
     copy(timeZoneId = Option(timeZoneId))
 
   def this(time: Expression) = {
-    this(time, Literal("uuuu-MM-dd HH:mm:ss"))
+    this(time, Literal(TimestampFormatter.defaultPattern))
   }
 
   def this() = {
@@ -758,7 +765,7 @@ abstract class ToTimestamp
   private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
   private lazy val formatter: TimestampFormatter =
     try {
-      TimestampFormatter(constFormat.toString, zoneId)
+      TimestampFormatter(constFormat.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
     } catch {
       case NonFatal(_) => null
     }
@@ -791,8 +798,8 @@ abstract class ToTimestamp
           } else {
             val formatString = f.asInstanceOf[UTF8String].toString
             try {
-              TimestampFormatter(formatString, zoneId).parse(
-                t.asInstanceOf[UTF8String].toString) / downScaleFactor
+              TimestampFormatter(formatString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+                .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
               case NonFatal(_) => null
             }
@@ -831,13 +838,16 @@ abstract class ToTimestamp
         }
       case StringType =>
         val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-        val locale = ctx.addReferenceObj("locale", Locale.US)
         val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+        val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
         nullSafeCodeGen(ctx, ev, (string, format) => {
           s"""
             try {
-              ${ev.value} = $tf$$.MODULE$$.apply($format.toString(), $zid, $locale)
-                .parse($string.toString()) / $downScaleFactor;
+              ${ev.value} = $tf$$.MODULE$$.apply(
+                $format.toString(),
+                $zid,
+                $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT())
+              .parse($string.toString()) / $downScaleFactor;
             } catch (java.lang.IllegalArgumentException e) {
               ${ev.isNull} = true;
             } catch (java.text.ParseException e) {
@@ -908,7 +918,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   override def prettyName: String = "from_unixtime"
 
   def this(unix: Expression) = {
-    this(unix, Literal("uuuu-MM-dd HH:mm:ss"))
+    this(unix, Literal(TimestampFormatter.defaultPattern))
   }
 
   override def dataType: DataType = StringType
@@ -922,7 +932,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
   private lazy val formatter: TimestampFormatter =
     try {
-      TimestampFormatter(constFormat.toString, zoneId)
+      TimestampFormatter(constFormat.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
     } catch {
       case NonFatal(_) => null
     }
@@ -948,8 +958,9 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
           null
         } else {
           try {
-            UTF8String.fromString(TimestampFormatter(f.toString, zoneId)
-              .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
+            UTF8String.fromString(
+              TimestampFormatter(f.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+                .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
             case NonFatal(_) => null
           }
@@ -980,13 +991,14 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
       }
     } else {
       val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-      val locale = ctx.addReferenceObj("locale", Locale.US)
       val tf = TimestampFormatter.getClass.getName.stripSuffix("$")
+      val ldf = LegacyDateFormats.getClass.getName.stripSuffix("$")
       nullSafeCodeGen(ctx, ev, (seconds, f) => {
         s"""
         try {
-          ${ev.value} = UTF8String.fromString($tf$$.MODULE$$.apply($f.toString(), $zid, $locale).
-            format($seconds * 1000000L));
+          ${ev.value} = UTF8String.fromString(
+            $tf$$.MODULE$$.apply($f.toString(), $zid, $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT())
+              .format($seconds * 1000000L));
         } catch (java.lang.IllegalArgumentException e) {
           ${ev.isNull} = true;
         }"""
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
index cdf4b4689e821..45c4edff47070 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
@@ -88,10 +88,10 @@ private[sql] class JSONOptions(
   val zoneId: ZoneId = DateTimeUtils.getZoneId(
     parameters.getOrElse(DateTimeUtils.TIMEZONE_OPTION, defaultTimeZoneId))
 
-  val dateFormat: String = parameters.getOrElse("dateFormat", "uuuu-MM-dd")
+  val dateFormat: String = parameters.getOrElse("dateFormat", DateFormatter.defaultPattern)
 
   val timestampFormat: String =
-    parameters.getOrElse("timestampFormat", "uuuu-MM-dd'T'HH:mm:ss.SSSXXX")
+    parameters.getOrElse("timestampFormat", s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX")
 
   val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index 9c63593ea1752..141360ff02117 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.core._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.types._
 
 /**
@@ -80,11 +81,13 @@ private[sql] class JacksonGenerator(
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
 
   private def makeWriter(dataType: DataType): ValueWriter = dataType match {
     case NullType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index 76efa574a99ff..1e408cdb126b8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -30,6 +30,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -58,11 +59,13 @@ class JacksonParser(
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
 
   /**
    * Create a converter which converts the JSON documents held by the `JsonParser`
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index f030955ee6e7f..82dd6d0da2632 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCoercion
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
 import org.apache.spark.sql.catalyst.json.JacksonUtils.nextUntil
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.FAST_DATE_FORMAT
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
@@ -40,7 +41,8 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
   private val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
-    options.locale)
+    options.locale,
+    legacyFormat = FAST_DATE_FORMAT)
 
   /**
    * Infer the type of a collection of json records in three stages:
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 28189b65dee9a..2cf82d1cfa177 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.text.SimpleDateFormat
 import java.time.{LocalDate, ZoneId}
-import java.util.Locale
+import java.util.{Date, Locale}
 
 import org.apache.commons.lang3.time.FastDateFormat
 
@@ -51,41 +52,76 @@ class Iso8601DateFormatter(
   }
 }
 
-class LegacyDateFormatter(pattern: String, locale: Locale) extends DateFormatter {
-  @transient
-  private lazy val format = FastDateFormat.getInstance(pattern, locale)
+trait LegacyDateFormatter extends DateFormatter {
+  def parseToDate(s: String): Date
+  def formatDate(d: Date): String
 
   override def parse(s: String): Int = {
-    val milliseconds = format.parse(s).getTime
+    val milliseconds = parseToDate(s).getTime
     DateTimeUtils.millisToDays(milliseconds)
   }
 
   override def format(days: Int): String = {
     val date = DateTimeUtils.toJavaDate(days)
-    format.format(date)
+    formatDate(date)
   }
 }
 
+class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
+  @transient
+  private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
+  override def parseToDate(s: String): Date = fdf.parse(s)
+  override def formatDate(d: Date): String = fdf.format(d)
+}
+
+class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
+  @transient
+  private lazy val sdf = new SimpleDateFormat(pattern, locale)
+  override def parseToDate(s: String): Date = sdf.parse(s)
+  override def formatDate(d: Date): String = sdf.format(d)
+}
+
 object DateFormatter {
+  import LegacyDateFormats._
+
   val defaultLocale: Locale = Locale.US
 
-  def apply(format: String, zoneId: ZoneId, locale: Locale): DateFormatter = {
+  def defaultPattern(): String = {
+    if (SQLConf.get.legacyTimeParserEnabled) "yyyy-MM-dd" else "uuuu-MM-dd"
+  }
+
+  private def getFormatter(
+    format: Option[String],
+    zoneId: ZoneId,
+    locale: Locale = defaultLocale,
+    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): DateFormatter = {
+
+    val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserEnabled) {
-      new LegacyDateFormatter(format, locale)
+      legacyFormat match {
+        case FAST_DATE_FORMAT =>
+          new LegacyFastDateFormatter(pattern, locale)
+        case SIMPLE_DATE_FORMAT | LENIENT_SIMPLE_DATE_FORMAT =>
+          new LegacySimpleDateFormatter(pattern, locale)
+      }
     } else {
-      new Iso8601DateFormatter(format, zoneId, locale)
+      new Iso8601DateFormatter(pattern, zoneId, locale)
     }
   }
 
+  def apply(
+    format: String,
+    zoneId: ZoneId,
+    locale: Locale,
+    legacyFormat: LegacyDateFormat): DateFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat)
+  }
+
   def apply(format: String, zoneId: ZoneId): DateFormatter = {
-    apply(format, zoneId, defaultLocale)
+    getFormatter(Some(format), zoneId)
   }
 
   def apply(zoneId: ZoneId): DateFormatter = {
-    if (SQLConf.get.legacyTimeParserEnabled) {
-      new LegacyDateFormatter("yyyy-MM-dd", defaultLocale)
-    } else {
-      new Iso8601DateFormatter("uuuu-MM-dd", zoneId, defaultLocale)
-    }
+    getFormatter(None, zoneId)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index fe1a4fe710c20..4893a7ec91cbb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -17,19 +17,20 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.text.ParseException
+import java.text.{ParseException, ParsePosition, SimpleDateFormat}
 import java.time._
 import java.time.format.DateTimeParseException
 import java.time.temporal.ChronoField.MICRO_OF_SECOND
 import java.time.temporal.TemporalQueries
-import java.util.{Locale, TimeZone}
+import java.util.{Calendar, GregorianCalendar, Locale, TimeZone}
 import java.util.concurrent.TimeUnit.SECONDS
 
 import org.apache.commons.lang3.time.FastDateFormat
 
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_MILLIS
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.convertSpecialTimestamp
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{ convertSpecialTimestamp, SQLTimestamp}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.Decimal
 
 sealed trait TimestampFormatter extends Serializable {
   /**
@@ -90,44 +91,139 @@ class FractionTimestampFormatter(zoneId: ZoneId)
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
 }
 
-class LegacyTimestampFormatter(
+/**
+ * The custom sub-class of `GregorianCalendar` is needed to get access to
+ * protected `fields` immediately after parsing. We cannot use
+ * the `get()` method because it performs normalization of the fraction
+ * part. Accordingly, the `MILLISECOND` field doesn't contain original value.
+ *
+ * Also this class allows to set raw value to the `MILLISECOND` field
+ * directly before formatting.
+ */
+class MicrosCalendar(tz: TimeZone, digitsInFraction: Int)
+  extends GregorianCalendar(tz, Locale.US) {
+  // Converts parsed `MILLISECOND` field to seconds fraction in microsecond precision.
+  // For example if the fraction pattern is `SSSS` then `digitsInFraction` = 4, and
+  // if the `MILLISECOND` field was parsed to `1234`.
+  def getMicros(): SQLTimestamp = {
+    // Append 6 zeros to the field: 1234 -> 1234000000
+    val d = fields(Calendar.MILLISECOND) * MICROS_PER_SECOND
+    // Take the first 6 digits from `d`: 1234000000 -> 123400
+    // The rest contains exactly `digitsInFraction`: `0000` = 10 ^ digitsInFraction
+    // So, the result is `(1234 * 1000000) / (10 ^ digitsInFraction)
+    d / Decimal.POW_10(digitsInFraction)
+  }
+
+  // Converts the seconds fraction in microsecond precision to a value
+  // that can be correctly formatted according to the specified fraction pattern.
+  // The method performs operations opposite to `getMicros()`.
+  def setMicros(micros: Long): Unit = {
+    val d = micros * Decimal.POW_10(digitsInFraction)
+    fields(Calendar.MILLISECOND) = (d / MICROS_PER_SECOND).toInt
+  }
+}
+
+class LegacyFastTimestampFormatter(
     pattern: String,
     zoneId: ZoneId,
     locale: Locale) extends TimestampFormatter {
 
-  @transient private lazy val format =
+  @transient private lazy val fastDateFormat =
     FastDateFormat.getInstance(pattern, TimeZone.getTimeZone(zoneId), locale)
+  @transient private lazy val cal = new MicrosCalendar(
+    fastDateFormat.getTimeZone,
+    fastDateFormat.getPattern.count(_ == 'S'))
+
+  def parse(s: String): SQLTimestamp = {
+    cal.clear() // Clear the calendar because it can be re-used many times
+    if (!fastDateFormat.parse(s, new ParsePosition(0), cal)) {
+      throw new IllegalArgumentException(s"'$s' is an invalid timestamp")
+    }
+    val micros = cal.getMicros()
+    cal.set(Calendar.MILLISECOND, 0)
+    cal.getTimeInMillis * MICROS_PER_MILLIS + micros
+  }
+
+  def format(timestamp: SQLTimestamp): String = {
+    cal.setTimeInMillis(Math.floorDiv(timestamp, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
+    cal.setMicros(Math.floorMod(timestamp, MICROS_PER_SECOND))
+    fastDateFormat.format(cal)
+  }
+}
 
-  protected def toMillis(s: String): Long = format.parse(s).getTime
+class LegacySimpleTimestampFormatter(
+    pattern: String,
+    zoneId: ZoneId,
+    locale: Locale,
+    lenient: Boolean = true) extends TimestampFormatter {
+  @transient private lazy val sdf = {
+    val formatter = new SimpleDateFormat(pattern, locale)
+    formatter.setTimeZone(TimeZone.getTimeZone(zoneId))
+    formatter.setLenient(lenient)
+    formatter
+  }
 
-  override def parse(s: String): Long = toMillis(s) * MICROS_PER_MILLIS
+  override def parse(s: String): Long = {
+    sdf.parse(s).getTime * MICROS_PER_MILLIS
+  }
 
   override def format(us: Long): String = {
-    format.format(DateTimeUtils.toJavaTimestamp(us))
+    val timestamp = DateTimeUtils.toJavaTimestamp(us)
+    sdf.format(timestamp)
   }
 }
 
+object LegacyDateFormats extends Enumeration {
+  type LegacyDateFormat = Value
+  val FAST_DATE_FORMAT, SIMPLE_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT = Value
+}
+
 object TimestampFormatter {
+  import LegacyDateFormats._
+
   val defaultLocale: Locale = Locale.US
 
-  def apply(format: String, zoneId: ZoneId, locale: Locale): TimestampFormatter = {
+  def defaultPattern(): String = s"${DateFormatter.defaultPattern()} HH:mm:ss"
+
+  private def getFormatter(
+    format: Option[String],
+    zoneId: ZoneId,
+    locale: Locale = defaultLocale,
+    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): TimestampFormatter = {
+
+    val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserEnabled) {
-      new LegacyTimestampFormatter(format, zoneId, locale)
+      legacyFormat match {
+        case FAST_DATE_FORMAT =>
+          new LegacyFastTimestampFormatter(pattern, zoneId, locale)
+        case SIMPLE_DATE_FORMAT =>
+          new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = false)
+        case LENIENT_SIMPLE_DATE_FORMAT =>
+          new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = true)
+      }
     } else {
-      new Iso8601TimestampFormatter(format, zoneId, locale)
+      new Iso8601TimestampFormatter(pattern, zoneId, locale)
     }
   }
 
+  def apply(
+    format: String,
+    zoneId: ZoneId,
+    locale: Locale,
+    legacyFormat: LegacyDateFormat): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat)
+  }
+
+  def apply(format: String, zoneId: ZoneId, legacyFormat: LegacyDateFormat): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat)
+  }
+
   def apply(format: String, zoneId: ZoneId): TimestampFormatter = {
-    apply(format, zoneId, defaultLocale)
+    getFormatter(Some(format), zoneId)
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
-    if (SQLConf.get.legacyTimeParserEnabled) {
-      new LegacyTimestampFormatter("yyyy-MM-dd HH:mm:ss", zoneId, defaultLocale)
-    } else {
-      new Iso8601TimestampFormatter("uuuu-MM-dd HH:mm:ss", zoneId, defaultLocale)
-    }
+    getFormatter(None, zoneId)
   }
 
   def getFractionFormatter(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
index 9ce64b09f7870..f32e48e1cc128 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala
@@ -541,7 +541,7 @@ object Decimal {
   /** Maximum number of decimal digits a Long can represent */
   val MAX_LONG_DIGITS = 18
 
-  private val POW_10 = Array.tabulate[Long](MAX_LONG_DIGITS + 1)(i => math.pow(10, i).toLong)
+  val POW_10 = Array.tabulate[Long](MAX_LONG_DIGITS + 1)(i => math.pow(10, i).toLong)
 
   private val BIG_DEC_ZERO = BigDecimal(0)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 274d0beebd300..f04149ab7eb29 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, Timesta
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneGMT
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -241,41 +242,45 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("DateFormat") {
-    checkEvaluation(
-      DateFormatClass(Literal.create(null, TimestampType), Literal("y"), gmtId),
-      null)
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-      Literal.create(null, StringType), gmtId), null)
-
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-      Literal("y"), gmtId), "2015")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), gmtId), "2013")
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-      Literal("H"), gmtId), "0")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), gmtId), "13")
-
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
-      Literal("y"), pstId), "2015")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), pstId), "2013")
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
-      Literal("H"), pstId), "0")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), pstId), "5")
-
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
-      Literal("y"), jstId), "2015")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), jstId), "2013")
-    checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
-      Literal("H"), jstId), "0")
-    checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), jstId), "22")
-
-    // SPARK-28072 The codegen path should work
-    checkEvaluation(
-      expression = DateFormatClass(
-        BoundReference(ordinal = 0, dataType = TimestampType, nullable = true),
-        BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-        jstId),
-      expected = "22",
-      inputRow = InternalRow(DateTimeUtils.fromJavaTimestamp(ts), UTF8String.fromString("H")))
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        checkEvaluation(
+          DateFormatClass(Literal.create(null, TimestampType), Literal("y"), gmtId),
+          null)
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
+          Literal.create(null, StringType), gmtId), null)
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
+          Literal("y"), gmtId), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), gmtId), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
+          Literal("H"), gmtId), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), gmtId), "13")
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
+          Literal("y"), pstId), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), pstId), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
+          Literal("H"), pstId), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), pstId), "5")
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
+          Literal("y"), jstId), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), jstId), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
+          Literal("H"), jstId), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), jstId), "22")
+
+        // SPARK-28072 The codegen path should work
+        checkEvaluation(
+          expression = DateFormatClass(
+            BoundReference(ordinal = 0, dataType = TimestampType, nullable = true),
+            BoundReference(ordinal = 1, dataType = StringType, nullable = true),
+            jstId),
+          expected = "22",
+          inputRow = InternalRow(DateTimeUtils.fromJavaTimestamp(ts), UTF8String.fromString("H")))
+      }
+    }
   }
 
   test("Hour") {
@@ -705,162 +710,189 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("from_unixtime") {
-    val fmt1 = "yyyy-MM-dd HH:mm:ss"
-    val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-      val timeZoneId = Option(tz.getID)
-      sdf1.setTimeZone(tz)
-      sdf2.setTimeZone(tz)
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val fmt1 = "yyyy-MM-dd HH:mm:ss"
+        val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+          val timeZoneId = Option(tz.getID)
+          sdf1.setTimeZone(tz)
+          sdf2.setTimeZone(tz)
 
-      checkEvaluation(
-        FromUnixTime(Literal(0L), Literal(fmt1), timeZoneId),
-        sdf1.format(new Timestamp(0)))
-      checkEvaluation(FromUnixTime(
-        Literal(1000L), Literal(fmt1), timeZoneId),
-        sdf1.format(new Timestamp(1000000)))
-      checkEvaluation(
-        FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId),
-        sdf2.format(new Timestamp(-1000000)))
-      checkEvaluation(
-        FromUnixTime(Literal.create(null, LongType), Literal.create(null, StringType), timeZoneId),
-        null)
-      checkEvaluation(
-        FromUnixTime(Literal.create(null, LongType), Literal(fmt1), timeZoneId),
-        null)
-      checkEvaluation(
-        FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId),
-        null)
-      checkEvaluation(
-        FromUnixTime(Literal(0L), Literal("not a valid format"), timeZoneId), null)
+          checkEvaluation(
+            FromUnixTime(Literal(0L), Literal(fmt1), timeZoneId),
+            sdf1.format(new Timestamp(0)))
+          checkEvaluation(FromUnixTime(
+            Literal(1000L), Literal(fmt1), timeZoneId),
+            sdf1.format(new Timestamp(1000000)))
+          checkEvaluation(
+            FromUnixTime(Literal(-1000L), Literal(fmt2), timeZoneId),
+            sdf2.format(new Timestamp(-1000000)))
+          checkEvaluation(
+            FromUnixTime(
+              Literal.create(null, LongType),
+              Literal.create(null, StringType), timeZoneId),
+            null)
+          checkEvaluation(
+            FromUnixTime(Literal.create(null, LongType), Literal(fmt1), timeZoneId),
+            null)
+          checkEvaluation(
+            FromUnixTime(Literal(1000L), Literal.create(null, StringType), timeZoneId),
+            null)
+          checkEvaluation(
+            FromUnixTime(Literal(0L), Literal("not a valid format"), timeZoneId), null)
 
-      // SPARK-28072 The codegen path for non-literal input should also work
-      checkEvaluation(
-        expression = FromUnixTime(
-          BoundReference(ordinal = 0, dataType = LongType, nullable = true),
-          BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-          timeZoneId),
-        expected = UTF8String.fromString(sdf1.format(new Timestamp(0))),
-        inputRow = InternalRow(0L, UTF8String.fromString(fmt1)))
+          // SPARK-28072 The codegen path for non-literal input should also work
+          checkEvaluation(
+            expression = FromUnixTime(
+              BoundReference(ordinal = 0, dataType = LongType, nullable = true),
+              BoundReference(ordinal = 1, dataType = StringType, nullable = true),
+              timeZoneId),
+            expected = UTF8String.fromString(sdf1.format(new Timestamp(0))),
+            inputRow = InternalRow(0L, UTF8String.fromString(fmt1)))
+        }
+      }
     }
   }
 
   test("unix_timestamp") {
-    val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    val fmt3 = "yy-MM-dd"
-    val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-    sdf3.setTimeZone(TimeZoneGMT)
-
-    withDefaultTimeZone(TimeZoneGMT) {
-      for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-        val timeZoneId = Option(tz.getID)
-        sdf1.setTimeZone(tz)
-        sdf2.setTimeZone(tz)
-
-        val date1 = Date.valueOf("2015-07-24")
-        checkEvaluation(UnixTimestamp(
-          Literal(sdf1.format(new Timestamp(0))), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId), 0L)
-        checkEvaluation(UnixTimestamp(
-          Literal(sdf1.format(new Timestamp(1000000))), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          1000L)
-        checkEvaluation(
-          UnixTimestamp(
-            Literal(new Timestamp(1000000)), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          1000L)
-        checkEvaluation(
-          UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2), timeZoneId),
-          -1000L)
-        checkEvaluation(UnixTimestamp(
-          Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-            DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
-        val t1 = UnixTimestamp(
-          CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
-        val t2 = UnixTimestamp(
-          CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
-        assert(t2 - t1 <= 1)
-        checkEvaluation(
-          UnixTimestamp(
-            Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId),
-          null)
-        checkEvaluation(
-          UnixTimestamp(Literal.create(null, DateType), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
-          null)
-        checkEvaluation(
-          UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        val fmt3 = "yy-MM-dd"
+        val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
+        sdf3.setTimeZone(TimeZoneGMT)
+
+        withDefaultTimeZone(TimeZoneGMT) {
+          for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+            val timeZoneId = Option(tz.getID)
+            sdf1.setTimeZone(tz)
+            sdf2.setTimeZone(tz)
+
+            val date1 = Date.valueOf("2015-07-24")
+            checkEvaluation(UnixTimestamp(
+              Literal(sdf1.format(new Timestamp(0))),
+              Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId), 0L)
+            checkEvaluation(UnixTimestamp(
+              Literal(sdf1.format(new Timestamp(1000000))),
+              Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              1000L)
+            checkEvaluation(
+              UnixTimestamp(
+                Literal(new Timestamp(1000000)), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              1000L)
+            checkEvaluation(
+              UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+            checkEvaluation(
+              UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))),
+                Literal(fmt2), timeZoneId),
+              -1000L)
+            checkEvaluation(UnixTimestamp(
+              Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
+              MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
+            val t1 = UnixTimestamp(
+              CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
+            val t2 = UnixTimestamp(
+              CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
+            assert(t2 - t1 <= 1)
+            checkEvaluation(
+              UnixTimestamp(
+                Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId),
+              null)
+            checkEvaluation(
+              UnixTimestamp(
+                Literal.create(null, DateType),
+                Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
+              null)
+            checkEvaluation(
+              UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+            checkEvaluation(
+              UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
+          }
+        }
       }
     }
   }
 
   test("to_unix_timestamp") {
-    val fmt1 = "yyyy-MM-dd HH:mm:ss"
-    val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    val fmt3 = "yy-MM-dd"
-    val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-    sdf3.setTimeZone(TimeZoneGMT)
-
-    withDefaultTimeZone(TimeZoneGMT) {
-      for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
-        val timeZoneId = Option(tz.getID)
-        sdf1.setTimeZone(tz)
-        sdf2.setTimeZone(tz)
-
-        val date1 = Date.valueOf("2015-07-24")
-        checkEvaluation(ToUnixTimestamp(
-          Literal(sdf1.format(new Timestamp(0))), Literal(fmt1), timeZoneId), 0L)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(sdf1.format(new Timestamp(1000000))), Literal(fmt1), timeZoneId),
-          1000L)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(new Timestamp(1000000)), Literal(fmt1)),
-          1000L)
-        checkEvaluation(
-          ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          ToUnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2), timeZoneId),
-          -1000L)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-            DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
-        val t1 = ToUnixTimestamp(
-          CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
-        val t2 = ToUnixTimestamp(
-          CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
-        assert(t2 - t1 <= 1)
-        checkEvaluation(ToUnixTimestamp(
-          Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId), null)
-        checkEvaluation(
-          ToUnixTimestamp(
-            Literal.create(null, DateType), Literal(fmt1), timeZoneId),
-          null)
-        checkEvaluation(ToUnixTimestamp(
-          Literal(date1), Literal.create(null, StringType), timeZoneId),
-          MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
-        checkEvaluation(
-          ToUnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val fmt1 = "yyyy-MM-dd HH:mm:ss"
+        val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        val fmt3 = "yy-MM-dd"
+        val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
+        sdf3.setTimeZone(TimeZoneGMT)
+
+        withDefaultTimeZone(TimeZoneGMT) {
+          for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+            val timeZoneId = Option(tz.getID)
+            sdf1.setTimeZone(tz)
+            sdf2.setTimeZone(tz)
+
+            val date1 = Date.valueOf("2015-07-24")
+            checkEvaluation(ToUnixTimestamp(
+              Literal(sdf1.format(new Timestamp(0))), Literal(fmt1), timeZoneId), 0L)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(sdf1.format(new Timestamp(1000000))), Literal(fmt1), timeZoneId),
+              1000L)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(new Timestamp(1000000)), Literal(fmt1)),
+              1000L)
+            checkEvaluation(
+              ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+            checkEvaluation(
+              ToUnixTimestamp(
+                Literal(sdf2.format(new Timestamp(-1000000))),
+                Literal(fmt2), timeZoneId),
+              -1000L)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
+              MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
+            val t1 = ToUnixTimestamp(
+              CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
+            val t2 = ToUnixTimestamp(
+              CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
+            assert(t2 - t1 <= 1)
+            checkEvaluation(ToUnixTimestamp(
+              Literal.create(null, DateType), Literal.create(null, StringType), timeZoneId), null)
+            checkEvaluation(
+              ToUnixTimestamp(
+                Literal.create(null, DateType), Literal(fmt1), timeZoneId),
+              null)
+            checkEvaluation(ToUnixTimestamp(
+              Literal(date1), Literal.create(null, StringType), timeZoneId),
+              MILLISECONDS.toSeconds(
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+            checkEvaluation(
+              ToUnixTimestamp(
+                Literal("2015-07-24"),
+                Literal("not a valid format"), timeZoneId), null)
 
-        // SPARK-28072 The codegen path for non-literal input should also work
-        checkEvaluation(
-          expression = ToUnixTimestamp(
-            BoundReference(ordinal = 0, dataType = StringType, nullable = true),
-            BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-            timeZoneId),
-          expected = 0L,
-          inputRow = InternalRow(
-            UTF8String.fromString(sdf1.format(new Timestamp(0))), UTF8String.fromString(fmt1)))
+            // SPARK-28072 The codegen path for non-literal input should also work
+            checkEvaluation(
+              expression = ToUnixTimestamp(
+                BoundReference(ordinal = 0, dataType = StringType, nullable = true),
+                BoundReference(ordinal = 1, dataType = StringType, nullable = true),
+                timeZoneId),
+              expected = 0L,
+              inputRow = InternalRow(
+                UTF8String.fromString(sdf1.format(new Timestamp(0))), UTF8String.fromString(fmt1)))
+          }
+        }
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index d125581857e0b..2d5504ac00ffa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, HintInfo, ResolvedHint}
+import org.apache.spark.sql.catalyst.util.TimestampFormatter
 import org.apache.spark.sql.execution.SparkSqlParser
 import org.apache.spark.sql.expressions.{Aggregator, SparkUserDefinedFunction, UserDefinedAggregator, UserDefinedFunction}
 import org.apache.spark.sql.internal.SQLConf
@@ -2881,7 +2882,7 @@ object functions {
    * @since 1.5.0
    */
   def from_unixtime(ut: Column): Column = withExpr {
-    FromUnixTime(ut.expr, Literal("uuuu-MM-dd HH:mm:ss"))
+    FromUnixTime(ut.expr, Literal(TimestampFormatter.defaultPattern))
   }
 
   /**
@@ -2913,7 +2914,7 @@ object functions {
    * @since 1.5.0
    */
   def unix_timestamp(): Column = withExpr {
-    UnixTimestamp(CurrentTimestamp(), Literal("uuuu-MM-dd HH:mm:ss"))
+    UnixTimestamp(CurrentTimestamp(), Literal(TimestampFormatter.defaultPattern))
   }
 
   /**
@@ -2927,7 +2928,7 @@ object functions {
    * @since 1.5.0
    */
   def unix_timestamp(s: Column): Column = withExpr {
-    UnixTimestamp(s.expr, Literal("uuuu-MM-dd HH:mm:ss"))
+    UnixTimestamp(s.expr, Literal(TimestampFormatter.defaultPattern))
   }
 
   /**
diff --git a/sql/core/src/test/resources/test-data/bad_after_good.csv b/sql/core/src/test/resources/test-data/bad_after_good.csv
index 4621a7d23714d..1a7c2651a11a7 100644
--- a/sql/core/src/test/resources/test-data/bad_after_good.csv
+++ b/sql/core/src/test/resources/test-data/bad_after_good.csv
@@ -1,2 +1,2 @@
 "good record",1999-08-01
-"bad record",1999-088-01
+"bad record",1999-088_01
diff --git a/sql/core/src/test/resources/test-data/value-malformed.csv b/sql/core/src/test/resources/test-data/value-malformed.csv
index 8945ed73d2e83..6e6f08fca6df8 100644
--- a/sql/core/src/test/resources/test-data/value-malformed.csv
+++ b/sql/core/src/test/resources/test-data/value-malformed.csv
@@ -1,2 +1,2 @@
-0,2013-111-11 12:13:14
+0,2013-111_11 12:13:14
 1,1983-08-04
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index bb8cdf3cb6de1..41d53c959ef99 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -96,15 +96,19 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("date format") {
-    val df = Seq((d, sdf.format(d), ts)).toDF("a", "b", "c")
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val df = Seq((d, sdf.format(d), ts)).toDF("a", "b", "c")
 
-    checkAnswer(
-      df.select(date_format($"a", "y"), date_format($"b", "y"), date_format($"c", "y")),
-      Row("2015", "2015", "2013"))
+        checkAnswer(
+          df.select(date_format($"a", "y"), date_format($"b", "y"), date_format($"c", "y")),
+          Row("2015", "2015", "2013"))
 
-    checkAnswer(
-      df.selectExpr("date_format(a, 'y')", "date_format(b, 'y')", "date_format(c, 'y')"),
-      Row("2015", "2015", "2013"))
+        checkAnswer(
+          df.selectExpr("date_format(a, 'y')", "date_format(b, 'y')", "date_format(c, 'y')"),
+          Row("2015", "2015", "2013"))
+      }
+    }
   }
 
   test("year") {
@@ -525,170 +529,194 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("from_unixtime") {
-    val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
-    val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-    val fmt3 = "yy-MM-dd HH-mm-ss"
-    val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-    val df = Seq((1000, "yyyy-MM-dd HH:mm:ss.SSS"), (-1000, "yy-MM-dd HH-mm-ss")).toDF("a", "b")
-    checkAnswer(
-      df.select(from_unixtime(col("a"))),
-      Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.select(from_unixtime(col("a"), fmt2)),
-      Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.select(from_unixtime(col("a"), fmt3)),
-      Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.selectExpr("from_unixtime(a)"),
-      Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.selectExpr(s"from_unixtime(a, '$fmt2')"),
-      Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
-    checkAnswer(
-      df.selectExpr(s"from_unixtime(a, '$fmt3')"),
-      Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
+        val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
+        val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
+        val fmt3 = "yy-MM-dd HH-mm-ss"
+        val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
+        val df = Seq((1000, "yyyy-MM-dd HH:mm:ss.SSS"), (-1000, "yy-MM-dd HH-mm-ss")).toDF("a", "b")
+        checkAnswer(
+          df.select(from_unixtime(col("a"))),
+          Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.select(from_unixtime(col("a"), fmt2)),
+          Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.select(from_unixtime(col("a"), fmt3)),
+          Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.selectExpr("from_unixtime(a)"),
+          Seq(Row(sdf1.format(new Timestamp(1000000))), Row(sdf1.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.selectExpr(s"from_unixtime(a, '$fmt2')"),
+          Seq(Row(sdf2.format(new Timestamp(1000000))), Row(sdf2.format(new Timestamp(-1000000)))))
+        checkAnswer(
+          df.selectExpr(s"from_unixtime(a, '$fmt3')"),
+          Seq(Row(sdf3.format(new Timestamp(1000000))), Row(sdf3.format(new Timestamp(-1000000)))))
+      }
+    }
   }
 
   private def secs(millis: Long): Long = TimeUnit.MILLISECONDS.toSeconds(millis)
 
   test("unix_timestamp") {
-    val date1 = Date.valueOf("2015-07-24")
-    val date2 = Date.valueOf("2015-07-25")
-    val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
-    val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
-    val s1 = "2015/07/24 10:00:00.5"
-    val s2 = "2015/07/25 02:02:02.6"
-    val ss1 = "2015-07-24 10:00:00"
-    val ss2 = "2015-07-25 02:02:02"
-    val fmt = "yyyy/MM/dd HH:mm:ss.S"
-    val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
-    checkAnswer(df.select(unix_timestamp(col("ts"))), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.select(unix_timestamp(col("ss"))), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.select(unix_timestamp(col("d"), fmt)), Seq(
-      Row(secs(date1.getTime)), Row(secs(date2.getTime))))
-    checkAnswer(df.select(unix_timestamp(col("s"), fmt)), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr("unix_timestamp(ts)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr("unix_timestamp(ss)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr(s"unix_timestamp(d, '$fmt')"), Seq(
-      Row(secs(date1.getTime)), Row(secs(date2.getTime))))
-    checkAnswer(df.selectExpr(s"unix_timestamp(s, '$fmt')"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-
-    val x1 = "2015-07-24 10:00:00"
-    val x2 = "2015-25-07 02:02:02"
-    val x3 = "2015-07-24 25:02:02"
-    val x4 = "2015-24-07 26:02:02"
-    val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
-    val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
-
-    val df1 = Seq(x1, x2, x3, x4).toDF("x")
-    checkAnswer(df1.select(unix_timestamp(col("x"))), Seq(
-      Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
-    checkAnswer(df1.selectExpr("unix_timestamp(x)"), Seq(
-      Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
-    checkAnswer(df1.select(unix_timestamp(col("x"), "yyyy-dd-MM HH:mm:ss")), Seq(
-      Row(null), Row(secs(ts2.getTime)), Row(null), Row(null)))
-    checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
-      Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
-
-    // invalid format
-    checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')"), Seq(
-      Row(null), Row(null), Row(null), Row(null)))
-
-    // february
-    val y1 = "2016-02-29"
-    val y2 = "2017-02-29"
-    val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
-    val df2 = Seq(y1, y2).toDF("y")
-    checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
-      Row(secs(ts5.getTime)), Row(null)))
-
-    val now = sql("select unix_timestamp()").collect().head.getLong(0)
-    checkAnswer(
-      sql(s"select cast ($now as timestamp)"),
-      Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now))))
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val date1 = Date.valueOf("2015-07-24")
+        val date2 = Date.valueOf("2015-07-25")
+        val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
+        val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
+        val s1 = "2015/07/24 10:00:00.5"
+        val s2 = "2015/07/25 02:02:02.6"
+        val ss1 = "2015-07-24 10:00:00"
+        val ss2 = "2015-07-25 02:02:02"
+        val fmt = "yyyy/MM/dd HH:mm:ss.S"
+        val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
+        checkAnswer(df.select(unix_timestamp(col("ts"))), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.select(unix_timestamp(col("ss"))), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.select(unix_timestamp(col("d"), fmt)), Seq(
+          Row(secs(date1.getTime)), Row(secs(date2.getTime))))
+        checkAnswer(df.select(unix_timestamp(col("s"), fmt)), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr("unix_timestamp(ts)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr("unix_timestamp(ss)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr(s"unix_timestamp(d, '$fmt')"), Seq(
+          Row(secs(date1.getTime)), Row(secs(date2.getTime))))
+        checkAnswer(df.selectExpr(s"unix_timestamp(s, '$fmt')"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+
+        val x1 = "2015-07-24 10:00:00"
+        val x2 = "2015-25-07 02:02:02"
+        val x3 = "2015-07-24 25:02:02"
+        val x4 = "2015-24-07 26:02:02"
+        val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
+        val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
+
+        val df1 = Seq(x1, x2, x3, x4).toDF("x")
+        checkAnswer(df1.select(unix_timestamp(col("x"))), Seq(
+          Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
+        checkAnswer(df1.selectExpr("unix_timestamp(x)"), Seq(
+          Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
+        checkAnswer(df1.select(unix_timestamp(col("x"), "yyyy-dd-MM HH:mm:ss")), Seq(
+          Row(null), Row(secs(ts2.getTime)), Row(null), Row(null)))
+        checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
+          Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
+
+        // invalid format
+        checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')"), Seq(
+          Row(null), Row(null), Row(null), Row(null)))
+
+        // february
+        val y1 = "2016-02-29"
+        val y2 = "2017-02-29"
+        val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
+        val df2 = Seq(y1, y2).toDF("y")
+        checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
+          Row(secs(ts5.getTime)), Row(null)))
+
+        val now = sql("select unix_timestamp()").collect().head.getLong(0)
+        checkAnswer(
+          sql(s"select cast ($now as timestamp)"),
+          Row(new java.util.Date(TimeUnit.SECONDS.toMillis(now))))
+      }
+    }
   }
 
   test("to_unix_timestamp") {
-    val date1 = Date.valueOf("2015-07-24")
-    val date2 = Date.valueOf("2015-07-25")
-    val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
-    val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
-    val s1 = "2015/07/24 10:00:00.5"
-    val s2 = "2015/07/25 02:02:02.6"
-    val ss1 = "2015-07-24 10:00:00"
-    val ss2 = "2015-07-25 02:02:02"
-    val fmt = "yyyy/MM/dd HH:mm:ss.S"
-    val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
-    checkAnswer(df.selectExpr("to_unix_timestamp(ts)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr("to_unix_timestamp(ss)"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-    checkAnswer(df.selectExpr(s"to_unix_timestamp(d, '$fmt')"), Seq(
-      Row(secs(date1.getTime)), Row(secs(date2.getTime))))
-    checkAnswer(df.selectExpr(s"to_unix_timestamp(s, '$fmt')"), Seq(
-      Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
-
-    val x1 = "2015-07-24 10:00:00"
-    val x2 = "2015-25-07 02:02:02"
-    val x3 = "2015-07-24 25:02:02"
-    val x4 = "2015-24-07 26:02:02"
-    val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
-    val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
-
-    val df1 = Seq(x1, x2, x3, x4).toDF("x")
-    checkAnswer(df1.selectExpr("to_unix_timestamp(x)"), Seq(
-      Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
-    checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
-      Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
-
-    // february
-    val y1 = "2016-02-29"
-    val y2 = "2017-02-29"
-    val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
-    val df2 = Seq(y1, y2).toDF("y")
-    checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
-      Row(secs(ts5.getTime)), Row(null)))
-
-    // invalid format
-    checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq(
-      Row(null), Row(null), Row(null), Row(null)))
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val date1 = Date.valueOf("2015-07-24")
+        val date2 = Date.valueOf("2015-07-25")
+        val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
+        val ts2 = Timestamp.valueOf("2015-07-25 02:02:02.2")
+        val s1 = "2015/07/24 10:00:00.5"
+        val s2 = "2015/07/25 02:02:02.6"
+        val ss1 = "2015-07-24 10:00:00"
+        val ss2 = "2015-07-25 02:02:02"
+        val fmt = "yyyy/MM/dd HH:mm:ss.S"
+        val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
+        checkAnswer(df.selectExpr("to_unix_timestamp(ts)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr("to_unix_timestamp(ss)"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+        checkAnswer(df.selectExpr(s"to_unix_timestamp(d, '$fmt')"), Seq(
+          Row(secs(date1.getTime)), Row(secs(date2.getTime))))
+        checkAnswer(df.selectExpr(s"to_unix_timestamp(s, '$fmt')"), Seq(
+          Row(secs(ts1.getTime)), Row(secs(ts2.getTime))))
+
+        val x1 = "2015-07-24 10:00:00"
+        val x2 = "2015-25-07 02:02:02"
+        val x3 = "2015-07-24 25:02:02"
+        val x4 = "2015-24-07 26:02:02"
+        val ts3 = Timestamp.valueOf("2015-07-24 02:25:02")
+        val ts4 = Timestamp.valueOf("2015-07-24 00:10:00")
+
+        val df1 = Seq(x1, x2, x3, x4).toDF("x")
+        checkAnswer(df1.selectExpr("to_unix_timestamp(x)"), Seq(
+          Row(secs(ts1.getTime)), Row(null), Row(null), Row(null)))
+        checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(
+          Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
+
+        // february
+        val y1 = "2016-02-29"
+        val y2 = "2017-02-29"
+        val ts5 = Timestamp.valueOf("2016-02-29 00:00:00")
+        val df2 = Seq(y1, y2).toDF("y")
+        checkAnswer(df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(
+          Row(secs(ts5.getTime)), Row(null)))
+
+        // invalid format
+        checkAnswer(df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')"), Seq(
+          Row(null), Row(null), Row(null), Row(null)))
+      }
+    }
   }
 
 
   test("to_timestamp") {
-    val date1 = Date.valueOf("2015-07-24")
-    val date2 = Date.valueOf("2015-07-25")
-    val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00")
-    val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00")
-    val ts1 = Timestamp.valueOf("2015-07-24 10:00:00")
-    val ts2 = Timestamp.valueOf("2015-07-25 02:02:02")
-    val s1 = "2015/07/24 10:00:00.5"
-    val s2 = "2015/07/25 02:02:02.6"
-    val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5")
-    val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6")
-    val ss1 = "2015-07-24 10:00:00"
-    val ss2 = "2015-07-25 02:02:02"
-    val fmt = "yyyy/MM/dd HH:mm:ss.S"
-    val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
-
-    checkAnswer(df.select(to_timestamp(col("ss"))),
-      df.select(unix_timestamp(col("ss")).cast("timestamp")))
-    checkAnswer(df.select(to_timestamp(col("ss"))), Seq(
-      Row(ts1), Row(ts2)))
-    checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(
-      Row(ts1m), Row(ts2m)))
-    checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(
-      Row(ts1), Row(ts2)))
-    checkAnswer(df.select(to_timestamp(col("d"), "yyyy-MM-dd")), Seq(
-      Row(ts_date1), Row(ts_date2)))
+    Seq(false, true).foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+        val date1 = Date.valueOf("2015-07-24")
+        val date2 = Date.valueOf("2015-07-25")
+        val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00")
+        val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00")
+        val ts1 = Timestamp.valueOf("2015-07-24 10:00:00")
+        val ts2 = Timestamp.valueOf("2015-07-25 02:02:02")
+        val s1 = "2015/07/24 10:00:00.5"
+        val s2 = "2015/07/25 02:02:02.6"
+        val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5")
+        val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6")
+        val ss1 = "2015-07-24 10:00:00"
+        val ss2 = "2015-07-25 02:02:02"
+        val fmt = "yyyy/MM/dd HH:mm:ss.S"
+        val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss")
+
+        checkAnswer(df.select(to_timestamp(col("ss"))),
+          df.select(unix_timestamp(col("ss")).cast("timestamp")))
+        checkAnswer(df.select(to_timestamp(col("ss"))), Seq(
+          Row(ts1), Row(ts2)))
+        if (legacyParser) {
+          // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off
+          // the fractional part of seconds. The behavior was changed by SPARK-27438.
+          val legacyFmt = "yyyy/MM/dd HH:mm:ss"
+          checkAnswer(df.select(to_timestamp(col("s"), legacyFmt)), Seq(
+            Row(ts1), Row(ts2)))
+        } else {
+          checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(
+            Row(ts1m), Row(ts2m)))
+        }
+        checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(
+          Row(ts1), Row(ts2)))
+        checkAnswer(df.select(to_timestamp(col("d"), "yyyy-MM-dd")), Seq(
+          Row(ts_date1), Row(ts_date2)))
+      }
+    }
   }
 
   test("datediff") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 97dfbbdb7fd2f..b1105b4a63bba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -1182,7 +1182,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .schema(schemaWithCorrField1)
         .csv(testFile(valueMalformedFile))
       checkAnswer(df2,
-        Row(0, null, "0,2013-111-11 12:13:14") ::
+        Row(0, null, "0,2013-111_11 12:13:14") ::
         Row(1, java.sql.Date.valueOf("1983-08-04"), null) ::
         Nil)
 
@@ -1199,7 +1199,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .schema(schemaWithCorrField2)
         .csv(testFile(valueMalformedFile))
       checkAnswer(df3,
-        Row(0, "0,2013-111-11 12:13:14", null) ::
+        Row(0, "0,2013-111_11 12:13:14", null) ::
         Row(1, null, java.sql.Date.valueOf("1983-08-04")) ::
         Nil)
 
@@ -1435,7 +1435,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     assert(df.filter($"_corrupt_record".isNull).count() == 1)
     checkAnswer(
       df.select(columnNameOfCorruptRecord),
-      Row("0,2013-111-11 12:13:14") :: Row(null) :: Nil
+      Row("0,2013-111_11 12:13:14") :: Row(null) :: Nil
     )
   }
 
@@ -2093,7 +2093,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     Seq("csv", "").foreach { reader =>
       withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> reader) {
         withTempPath { path =>
-          val df = Seq(("0", "2013-111-11")).toDF("a", "b")
+          val df = Seq(("0", "2013-111_11")).toDF("a", "b")
           df.write
             .option("header", "true")
             .csv(path.getAbsolutePath)
@@ -2109,7 +2109,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
             .option("columnNameOfCorruptRecord", columnNameOfCorruptRecord)
             .schema(schemaWithCorrField)
             .csv(path.getAbsoluteFile.toString)
-          checkAnswer(readDF, Row(0, null, "0,2013-111-11") :: Nil)
+          checkAnswer(readDF, Row(0, null, "0,2013-111_11") :: Nil)
         }
       }
     }
@@ -2216,7 +2216,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
                 val readback = spark.read
                   .option("mode", mode)
                   .option("header", true)
-                  .option("timestampFormat", "uuuu-MM-dd HH:mm:ss")
+                  .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
                   .option("multiLine", multiLine)
                   .schema("c0 string, c1 integer, c2 timestamp")
                   .csv(path.getAbsolutePath)
@@ -2235,7 +2235,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
   }
 
   test("filters push down - malformed input in PERMISSIVE mode") {
-    val invalidTs = "2019-123-14 20:35:30"
+    val invalidTs = "2019-123_14 20:35:30"
     val invalidRow = s"0,$invalidTs,999"
     val validTs = "2019-12-14 20:35:30"
     Seq(true, false).foreach { filterPushdown =>
@@ -2252,7 +2252,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
               .option("mode", "PERMISSIVE")
               .option("columnNameOfCorruptRecord", "c3")
               .option("header", true)
-              .option("timestampFormat", "uuuu-MM-dd HH:mm:ss")
+              .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
               .schema("c0 integer, c1 timestamp, c2 integer, c3 string")
               .csv(path.getAbsolutePath)
               .where(condition)
@@ -2309,3 +2309,10 @@ class CSVv2Suite extends CSVSuite {
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "")
 }
+
+class CSVLegacyTimeParserSuite extends CSVSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.LEGACY_TIME_PARSER_ENABLED, true)
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index b20da2266b0f3..7abe818a29d9f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2572,3 +2572,10 @@ class JsonV2Suite extends JsonSuite {
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "")
 }
+
+class JsonLegacyTimeParserSuite extends JsonSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.LEGACY_TIME_PARSER_ENABLED, true)
+}

From 258bfcfe4a87fe1d6a0bc27afb97e6b223e420e8 Mon Sep 17 00:00:00 2001
From: Eric Wu <492960551@qq.com>
Date: Thu, 13 Feb 2020 02:00:23 +0800
Subject: [PATCH 0060/1280] [SPARK-30651][SQL] Add detailed information for
 Aggregate operators in EXPLAIN FORMATTED

### What changes were proposed in this pull request?
Currently `EXPLAIN FORMATTED` only report input attributes of HashAggregate/ObjectHashAggregate/SortAggregate, while `EXPLAIN EXTENDED` provides more information of Keys, Functions, etc. This PR enhanced `EXPLAIN FORMATTED` to sync with original explain behavior.

### Why are the changes needed?
The newly added `EXPLAIN FORMATTED` got less information comparing to the original `EXPLAIN EXTENDED`

### Does this PR introduce any user-facing change?
Yes, taking HashAggregate explain result as example.

**SQL**
```
EXPLAIN FORMATTED
  SELECT
    COUNT(val) + SUM(key) as TOTAL,
    COUNT(key) FILTER (WHERE val > 1)
  FROM explain_temp1;
```

**EXPLAIN EXTENDED**
```
== Physical Plan ==
*(2) HashAggregate(keys=[], functions=[count(val#6), sum(cast(key#5 as bigint)), count(key#5)], output=[TOTAL#62L, count(key) FILTER (WHERE (val > 1))#71L])
+- Exchange SinglePartition, true, [id=#89]
   +- HashAggregate(keys=[], functions=[partial_count(val#6), partial_sum(cast(key#5 as bigint)), partial_count(key#5) FILTER (WHERE (val#6 > 1))], output=[count#75L, sum#76L, count#77L])
      +- *(1) ColumnarToRow
         +- FileScan parquet default.explain_temp1[key#5,val#6] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex[file:/Users/XXX/spark-dev/spark/spark-warehouse/explain_temp1], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<key:int,val:int>
```

**EXPLAIN FORMATTED - BEFORE**
```
== Physical Plan ==
* HashAggregate (5)
+- Exchange (4)
   +- HashAggregate (3)
      +- * ColumnarToRow (2)
         +- Scan parquet default.explain_temp1 (1)

...
...
(5) HashAggregate [codegen id : 2]
Input: [count#91L, sum#92L, count#93L]
...
...
```

**EXPLAIN FORMATTED - AFTER**
```
== Physical Plan ==
* HashAggregate (5)
+- Exchange (4)
   +- HashAggregate (3)
      +- * ColumnarToRow (2)
         +- Scan parquet default.explain_temp1 (1)

...
...
(5) HashAggregate [codegen id : 2]
Input: [count#91L, sum#92L, count#93L]
Keys: []
Functions: [count(val#6), sum(cast(key#5 as bigint)), count(key#5)]
Results: [(count(val#6)#84L + sum(cast(key#5 as bigint))#85L) AS TOTAL#78L, count(key#5)#86L AS count(key) FILTER (WHERE (val > 1))#87L]
Output: [TOTAL#78L, count(key) FILTER (WHERE (val > 1))#87L]
...
...
```

### How was this patch tested?
Three tests added in explain.sql for HashAggregate/ObjectHashAggregate/SortAggregate.

Closes #27368 from Eric5553/ExplainFormattedAgg.

Authored-by: Eric Wu <492960551@qq.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5919bd3b8d3ef3c3e957d8e3e245e00383b979bf)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../aggregate/BaseAggregateExec.scala         |  48 ++++
 .../aggregate/HashAggregateExec.scala         |   2 +-
 .../aggregate/ObjectHashAggregateExec.scala   |   2 +-
 .../aggregate/SortAggregateExec.scala         |   4 +-
 .../resources/sql-tests/inputs/explain.sql    |  22 +-
 .../sql-tests/results/explain.sql.out         | 232 +++++++++++++++++-
 6 files changed, 300 insertions(+), 10 deletions(-)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
new file mode 100644
index 0000000000000..0eaa0f53fdacd
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.aggregate
+
+import org.apache.spark.sql.catalyst.expressions.{Attribute, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.execution.{ExplainUtils, UnaryExecNode}
+
+/**
+ * Holds common logic for aggregate operators
+ */
+trait BaseAggregateExec extends UnaryExecNode {
+  def groupingExpressions: Seq[NamedExpression]
+  def aggregateExpressions: Seq[AggregateExpression]
+  def aggregateAttributes: Seq[Attribute]
+  def resultExpressions: Seq[NamedExpression]
+
+  override def verboseStringWithOperatorId(): String = {
+    val inputString = child.output.mkString("[", ", ", "]")
+    val keyString = groupingExpressions.mkString("[", ", ", "]")
+    val functionString = aggregateExpressions.mkString("[", ", ", "]")
+    val aggregateAttributeString = aggregateAttributes.mkString("[", ", ", "]")
+    val resultString = resultExpressions.mkString("[", ", ", "]")
+    s"""
+       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |Input: $inputString
+       |Keys: $keyString
+       |Functions: $functionString
+       |Aggregate Attributes: $aggregateAttributeString
+       |Results: $resultString
+     """.stripMargin
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index f73e214a6b41f..7a26fd7a8541a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -53,7 +53,7 @@ case class HashAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends UnaryExecNode with BlockingOperatorWithCodegen with AliasAwareOutputPartitioning {
+  extends BaseAggregateExec with BlockingOperatorWithCodegen with AliasAwareOutputPartitioning {
 
   private[this] val aggregateBufferAttributes = {
     aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
index 4376f6b6edd57..3fb58eb2cc8ba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
@@ -67,7 +67,7 @@ case class ObjectHashAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends UnaryExecNode with AliasAwareOutputPartitioning {
+  extends BaseAggregateExec with AliasAwareOutputPartitioning {
 
   private[this] val aggregateBufferAttributes = {
     aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
index b6e684e62ea5c..77ed469016fa3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.execution.{AliasAwareOutputPartitioning, SparkPlan, UnaryExecNode}
+import org.apache.spark.sql.execution.{AliasAwareOutputPartitioning, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetrics
 
 /**
@@ -38,7 +38,7 @@ case class SortAggregateExec(
     initialInputBufferOffset: Int,
     resultExpressions: Seq[NamedExpression],
     child: SparkPlan)
-  extends UnaryExecNode with AliasAwareOutputPartitioning {
+  extends BaseAggregateExec with AliasAwareOutputPartitioning {
 
   private[this] val aggregateBufferAttributes = {
     aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain.sql b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
index d5253e3daddb0..497b61c6134a2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/explain.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
@@ -5,6 +5,7 @@
 CREATE table  explain_temp1 (key int, val int) USING PARQUET;
 CREATE table  explain_temp2 (key int, val int) USING PARQUET;
 CREATE table  explain_temp3 (key int, val int) USING PARQUET;
+CREATE table  explain_temp4 (key int, val string) USING PARQUET;
 
 SET spark.sql.codegen.wholeStage = true;
 
@@ -61,7 +62,7 @@ EXPLAIN FORMATTED
                 FROM   explain_temp2 
                 WHERE  val > 0) 
          OR
-         key = (SELECT max(key) 
+         key = (SELECT avg(key)
                 FROM   explain_temp3
                 WHERE  val > 0);
 
@@ -93,6 +94,25 @@ EXPLAIN FORMATTED
   CREATE VIEW explain_view AS
     SELECT key, val FROM explain_temp1;
 
+-- HashAggregate
+EXPLAIN FORMATTED
+  SELECT
+    COUNT(val) + SUM(key) as TOTAL,
+    COUNT(key) FILTER (WHERE val > 1)
+  FROM explain_temp1;
+
+-- ObjectHashAggregate
+EXPLAIN FORMATTED
+  SELECT key, sort_array(collect_set(val))[0]
+  FROM explain_temp4
+  GROUP BY key;
+
+-- SortAggregate
+EXPLAIN FORMATTED
+  SELECT key, MIN(val)
+  FROM explain_temp4
+  GROUP BY key;
+
 -- cleanup
 DROP TABLE explain_temp1;
 DROP TABLE explain_temp2;
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index 756c14f28a657..bc28d7f87bf00 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 18
+-- Number of queries: 22
 
 
 -- !query
@@ -26,6 +26,14 @@ struct<>
 
 
 
+-- !query
+CREATE table  explain_temp4 (key int, val string) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 SET spark.sql.codegen.wholeStage = true
 -- !query schema
@@ -76,12 +84,20 @@ Input     : [key#x, val#x]
      
 (5) HashAggregate [codegen id : 1]
 Input: [key#x, val#x]
+Keys: [key#x]
+Functions: [partial_max(val#x)]
+Aggregate Attributes: [max#x]
+Results: [key#x, max#x]
      
 (6) Exchange 
 Input: [key#x, max#x]
      
 (7) HashAggregate [codegen id : 2]
 Input: [key#x, max#x]
+Keys: [key#x]
+Functions: [max(val#x)]
+Aggregate Attributes: [max(val#x)#x]
+Results: [key#x, max(val#x)#x AS max(val)#x]
      
 (8) Exchange 
 Input: [key#x, max(val)#x]
@@ -132,12 +148,20 @@ Input     : [key#x, val#x]
      
 (5) HashAggregate [codegen id : 1]
 Input: [key#x, val#x]
+Keys: [key#x]
+Functions: [partial_max(val#x)]
+Aggregate Attributes: [max#x]
+Results: [key#x, max#x]
      
 (6) Exchange 
 Input: [key#x, max#x]
      
 (7) HashAggregate [codegen id : 2]
 Input: [key#x, max#x]
+Keys: [key#x]
+Functions: [max(val#x)]
+Aggregate Attributes: [max(val#x)#x]
+Results: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
      
 (8) Filter [codegen id : 2]
 Input     : [key#x, max(val)#x, max(val#x)#x]
@@ -211,12 +235,20 @@ Input     : [key#x, val#x]
      
 (10) HashAggregate [codegen id : 3]
 Input: [key#x, val#x]
+Keys: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results: [key#x, val#x]
      
 (11) Exchange 
 Input: [key#x, val#x]
      
 (12) HashAggregate [codegen id : 4]
 Input: [key#x, val#x]
+Keys: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results: [key#x, val#x]
 
 
 -- !query
@@ -413,12 +445,20 @@ Input     : [key#x, val#x]
      
 (9) HashAggregate [codegen id : 1]
 Input: [key#x]
+Keys: []
+Functions: [partial_max(key#x)]
+Aggregate Attributes: [max#x]
+Results: [max#x]
      
 (10) Exchange 
 Input: [max#x]
      
 (11) HashAggregate [codegen id : 2]
 Input: [max#x]
+Keys: []
+Functions: [max(key#x)]
+Aggregate Attributes: [max(key#x)#x]
+Results: [max(key#x)#x AS max(key)#x]
      
 Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (18)
@@ -450,12 +490,20 @@ Input     : [key#x, val#x]
      
 (16) HashAggregate [codegen id : 1]
 Input: [key#x]
+Keys: []
+Functions: [partial_max(key#x)]
+Aggregate Attributes: [max#x]
+Results: [max#x]
      
 (17) Exchange 
 Input: [max#x]
      
 (18) HashAggregate [codegen id : 2]
 Input: [max#x]
+Keys: []
+Functions: [max(key#x)]
+Aggregate Attributes: [max(key#x)#x]
+Results: [max(key#x)#x AS max(key)#x]
 
 
 -- !query
@@ -466,7 +514,7 @@ EXPLAIN FORMATTED
                 FROM   explain_temp2 
                 WHERE  val > 0) 
          OR
-         key = (SELECT max(key) 
+         key = (SELECT avg(key)
                 FROM   explain_temp3
                 WHERE  val > 0)
 -- !query schema
@@ -489,7 +537,7 @@ Input: [key#x, val#x]
      
 (3) Filter [codegen id : 1]
 Input     : [key#x, val#x]
-Condition : ((key#x = Subquery scalar-subquery#x, [id=#x]) OR (key#x = Subquery scalar-subquery#x, [id=#x]))
+Condition : ((key#x = Subquery scalar-subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery scalar-subquery#x, [id=#x]))
      
 ===== Subqueries =====
 
@@ -523,12 +571,20 @@ Input     : [key#x, val#x]
      
 (8) HashAggregate [codegen id : 1]
 Input: [key#x]
+Keys: []
+Functions: [partial_max(key#x)]
+Aggregate Attributes: [max#x]
+Results: [max#x]
      
 (9) Exchange 
 Input: [max#x]
      
 (10) HashAggregate [codegen id : 2]
 Input: [max#x]
+Keys: []
+Functions: [max(key#x)]
+Aggregate Attributes: [max(key#x)#x]
+Results: [max(key#x)#x AS max(key)#x]
      
 Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (17)
@@ -560,12 +616,20 @@ Input     : [key#x, val#x]
      
 (15) HashAggregate [codegen id : 1]
 Input: [key#x]
+Keys: []
+Functions: [partial_avg(cast(key#x as bigint))]
+Aggregate Attributes: [sum#x, count#xL]
+Results: [sum#x, count#xL]
      
 (16) Exchange 
-Input: [max#x]
+Input: [sum#x, count#xL]
      
 (17) HashAggregate [codegen id : 2]
-Input: [max#x]
+Input: [sum#x, count#xL]
+Keys: []
+Functions: [avg(cast(key#x as bigint))]
+Aggregate Attributes: [avg(cast(key#x as bigint))#x]
+Results: [avg(cast(key#x as bigint))#x AS avg(key)#x]
 
 
 -- !query
@@ -615,12 +679,20 @@ Input: [key#x]
      
 (6) HashAggregate [codegen id : 1]
 Input: [key#x]
+Keys: []
+Functions: [partial_avg(cast(key#x as bigint))]
+Aggregate Attributes: [sum#x, count#xL]
+Results: [sum#x, count#xL]
      
 (7) Exchange 
 Input: [sum#x, count#xL]
      
 (8) HashAggregate [codegen id : 2]
 Input: [sum#x, count#xL]
+Keys: []
+Functions: [avg(cast(key#x as bigint))]
+Aggregate Attributes: [avg(cast(key#x as bigint))#x]
+Results: [avg(cast(key#x as bigint))#x AS avg(key)#x]
      
 Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#x, [id=#x]
 
@@ -740,18 +812,30 @@ Input     : [key#x, val#x]
      
 (5) HashAggregate [codegen id : 1]
 Input: [key#x, val#x]
+Keys: [key#x]
+Functions: [partial_max(val#x)]
+Aggregate Attributes: [max#x]
+Results: [key#x, max#x]
      
 (6) Exchange 
 Input: [key#x, max#x]
      
 (7) HashAggregate [codegen id : 4]
 Input: [key#x, max#x]
+Keys: [key#x]
+Functions: [max(val#x)]
+Aggregate Attributes: [max(val#x)#x]
+Results: [key#x, max(val#x)#x AS max(val)#x]
      
 (8) ReusedExchange  [Reuses operator id: 6]
 Output : ArrayBuffer(key#x, max#x)
      
 (9) HashAggregate [codegen id : 3]
 Input: [key#x, max#x]
+Keys: [key#x]
+Functions: [max(val#x)]
+Aggregate Attributes: [max(val#x)#x]
+Results: [key#x, max(val#x)#x AS max(val)#x]
      
 (10) BroadcastExchange 
 Input: [key#x, max(val)#x]
@@ -786,6 +870,144 @@ Output: []
 (4) Project
 
 
+-- !query
+EXPLAIN FORMATTED
+  SELECT
+    COUNT(val) + SUM(key) as TOTAL,
+    COUNT(key) FILTER (WHERE val > 1)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+* HashAggregate (5)
++- Exchange (4)
+   +- HashAggregate (3)
+      +- * ColumnarToRow (2)
+         +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+     
+(2) ColumnarToRow [codegen id : 1]
+Input: [key#x, val#x]
+     
+(3) HashAggregate 
+Input: [key#x, val#x]
+Keys: []
+Functions: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Aggregate Attributes: [count#xL, sum#xL, count#xL]
+Results: [count#xL, sum#xL, count#xL]
+     
+(4) Exchange 
+Input: [count#xL, sum#xL, count#xL]
+     
+(5) HashAggregate [codegen id : 2]
+Input: [count#xL, sum#xL, count#xL]
+Keys: []
+Functions: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
+Aggregate Attributes: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
+Results: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, sort_array(collect_set(val))[0]
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+ObjectHashAggregate (5)
++- Exchange (4)
+   +- ObjectHashAggregate (3)
+      +- * ColumnarToRow (2)
+         +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4 
+Output: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+     
+(2) ColumnarToRow [codegen id : 1]
+Input: [key#x, val#x]
+     
+(3) ObjectHashAggregate 
+Input: [key#x, val#x]
+Keys: [key#x]
+Functions: [partial_collect_set(val#x, 0, 0)]
+Aggregate Attributes: [buf#x]
+Results: [key#x, buf#x]
+     
+(4) Exchange 
+Input: [key#x, buf#x]
+     
+(5) ObjectHashAggregate 
+Input: [key#x, buf#x]
+Keys: [key#x]
+Functions: [collect_set(val#x, 0, 0)]
+Aggregate Attributes: [collect_set(val#x, 0, 0)#x]
+Results: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, MIN(val)
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+SortAggregate (7)
++- * Sort (6)
+   +- Exchange (5)
+      +- SortAggregate (4)
+         +- * Sort (3)
+            +- * ColumnarToRow (2)
+               +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4 
+Output: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+     
+(2) ColumnarToRow [codegen id : 1]
+Input: [key#x, val#x]
+     
+(3) Sort [codegen id : 1]
+Input: [key#x, val#x]
+     
+(4) SortAggregate 
+Input: [key#x, val#x]
+Keys: [key#x]
+Functions: [partial_min(val#x)]
+Aggregate Attributes: [min#x]
+Results: [key#x, min#x]
+     
+(5) Exchange 
+Input: [key#x, min#x]
+     
+(6) Sort [codegen id : 2]
+Input: [key#x, min#x]
+     
+(7) SortAggregate 
+Input: [key#x, min#x]
+Keys: [key#x]
+Functions: [min(val#x)]
+Aggregate Attributes: [min(val#x)#x]
+Results: [key#x, min(val#x)#x AS min(val)#x]
+
+
 -- !query
 DROP TABLE explain_temp1
 -- !query schema

From a5bf41fc7cbf6c9c3c613e87f37a1cbed64fa32f Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 13 Feb 2020 02:31:48 +0800
Subject: [PATCH 0061/1280] [SPARK-30760][SQL] Port `millisToDays` and
 `daysToMillis` on Java 8 time API

### What changes were proposed in this pull request?
In the PR, I propose to rewrite the `millisToDays` and `daysToMillis` of `DateTimeUtils` using Java 8 time API.

I removed `getOffsetFromLocalMillis` from `DateTimeUtils` because it is a private methods, and is not used anymore in Spark SQL.

### Why are the changes needed?
New implementation is based on Proleptic Gregorian calendar which has been already used by other date-time functions. This changes make `millisToDays` and `daysToMillis` consistent to rest Spark SQL API related to date & time operations.

### Does this PR introduce any user-facing change?
Yes, this might effect behavior for old dates before 1582 year.

### How was this patch tested?
By existing test suites `DateTimeUtilsSuite`, `DateFunctionsSuite`, DateExpressionsSuite`, `SQLQuerySuite` and `HiveResultSuite`.

Closes #27494 from MaxGekk/millis-2-days-java8-api.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit aa0d13683cdf9f38f04cc0e73dc8cf63eed29bf4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/datetimeExpressions.scala     |  8 +--
 .../sql/catalyst/util/DateTimeUtils.scala     | 58 +++++--------------
 .../catalyst/csv/UnivocityParserSuite.scala   |  3 +-
 .../expressions/DateExpressionsSuite.scala    | 19 +++---
 .../catalyst/util/DateTimeUtilsSuite.scala    | 34 ++++++-----
 .../spark/sql/execution/HiveResult.scala      |  5 ++
 .../sql-tests/results/postgreSQL/date.sql.out | 12 ++--
 .../apache/spark/sql/SQLQueryTestSuite.scala  |  1 +
 8 files changed, 62 insertions(+), 78 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 1f4c8c041f8bf..cf91489d8e6b7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -135,7 +135,7 @@ case class CurrentBatchTimestamp(
   def toLiteral: Literal = dataType match {
     case _: TimestampType =>
       Literal(DateTimeUtils.fromJavaTimestamp(new Timestamp(timestampMs)), TimestampType)
-    case _: DateType => Literal(DateTimeUtils.millisToDays(timestampMs, timeZone), DateType)
+    case _: DateType => Literal(DateTimeUtils.millisToDays(timestampMs, zoneId), DateType)
   }
 }
 
@@ -1332,14 +1332,14 @@ case class MonthsBetween(
 
   override def nullSafeEval(t1: Any, t2: Any, roundOff: Any): Any = {
     DateTimeUtils.monthsBetween(
-      t1.asInstanceOf[Long], t2.asInstanceOf[Long], roundOff.asInstanceOf[Boolean], timeZone)
+      t1.asInstanceOf[Long], t2.asInstanceOf[Long], roundOff.asInstanceOf[Boolean], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
     defineCodeGen(ctx, ev, (d1, d2, roundOff) => {
-      s"""$dtu.monthsBetween($d1, $d2, $roundOff, $tz)"""
+      s"""$dtu.monthsBetween($d1, $d2, $roundOff, $zid)"""
     })
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 8eb560944d4cb..01d36f19fc06f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -67,24 +67,22 @@ object DateTimeUtils {
 
   // we should use the exact day as Int, for example, (year, month, day) -> day
   def millisToDays(millisUtc: Long): SQLDate = {
-    millisToDays(millisUtc, defaultTimeZone())
+    millisToDays(millisUtc, defaultTimeZone().toZoneId)
   }
 
-  def millisToDays(millisUtc: Long, timeZone: TimeZone): SQLDate = {
-    // SPARK-6785: use Math.floorDiv so negative number of days (dates before 1970)
-    // will correctly work as input for function toJavaDate(Int)
-    val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
-    Math.floorDiv(millisLocal, MILLIS_PER_DAY).toInt
+  def millisToDays(millisUtc: Long, zoneId: ZoneId): SQLDate = {
+    val instant = microsToInstant(Math.multiplyExact(millisUtc, MICROS_PER_MILLIS))
+    localDateToDays(LocalDateTime.ofInstant(instant, zoneId).toLocalDate)
   }
 
   // reverse of millisToDays
   def daysToMillis(days: SQLDate): Long = {
-    daysToMillis(days, defaultTimeZone())
+    daysToMillis(days, defaultTimeZone().toZoneId)
   }
 
-  def daysToMillis(days: SQLDate, timeZone: TimeZone): Long = {
-    val millisLocal = days.toLong * MILLIS_PER_DAY
-    millisLocal - getOffsetFromLocalMillis(millisLocal, timeZone)
+  def daysToMillis(days: SQLDate, zoneId: ZoneId): Long = {
+    val instant = daysToLocalDate(days).atStartOfDay(zoneId).toInstant
+    instantToMicros(instant) / MICROS_PER_MILLIS
   }
 
   // Converts Timestamp to string according to Hive TimestampWritable convention.
@@ -587,11 +585,11 @@ object DateTimeUtils {
       time1: SQLTimestamp,
       time2: SQLTimestamp,
       roundOff: Boolean,
-      timeZone: TimeZone): Double = {
+      zoneId: ZoneId): Double = {
     val millis1 = MICROSECONDS.toMillis(time1)
     val millis2 = MICROSECONDS.toMillis(time2)
-    val date1 = millisToDays(millis1, timeZone)
-    val date2 = millisToDays(millis2, timeZone)
+    val date1 = millisToDays(millis1, zoneId)
+    val date2 = millisToDays(millis2, zoneId)
     val (year1, monthInYear1, dayInMonth1, daysToMonthEnd1) = splitDate(date1)
     val (year2, monthInYear2, dayInMonth2, daysToMonthEnd2) = splitDate(date2)
 
@@ -605,8 +603,8 @@ object DateTimeUtils {
     }
     // using milliseconds can cause precision loss with more than 8 digits
     // we follow Hive's implementation which uses seconds
-    val secondsInDay1 = MILLISECONDS.toSeconds(millis1 - daysToMillis(date1, timeZone))
-    val secondsInDay2 = MILLISECONDS.toSeconds(millis2 - daysToMillis(date2, timeZone))
+    val secondsInDay1 = MILLISECONDS.toSeconds(millis1 - daysToMillis(date1, zoneId))
+    val secondsInDay2 = MILLISECONDS.toSeconds(millis2 - daysToMillis(date2, zoneId))
     val secondsDiff = (dayInMonth1 - dayInMonth2) * SECONDS_PER_DAY + secondsInDay1 - secondsInDay2
     val secondsInMonth = DAYS.toSeconds(31)
     val diff = monthDiff + secondsDiff / secondsInMonth.toDouble
@@ -735,8 +733,8 @@ object DateTimeUtils {
         millis += offset
         millis - millis % MILLIS_PER_DAY - offset
       case _ => // Try to truncate date levels
-        val dDays = millisToDays(millis, timeZone)
-        daysToMillis(truncDate(dDays, level), timeZone)
+        val dDays = millisToDays(millis, timeZone.toZoneId)
+        daysToMillis(truncDate(dDays, level), timeZone.toZoneId)
     }
     truncated * MICROS_PER_MILLIS
   }
@@ -768,32 +766,6 @@ object DateTimeUtils {
     }
   }
 
-  /**
-   * Lookup the offset for given millis seconds since 1970-01-01 00:00:00 in given timezone.
-   * TODO: Improve handling of normalization differences.
-   * TODO: Replace with JSR-310 or similar system - see SPARK-16788
-   */
-  private[sql] def getOffsetFromLocalMillis(millisLocal: Long, tz: TimeZone): Long = {
-    var guess = tz.getRawOffset
-    // the actual offset should be calculated based on milliseconds in UTC
-    val offset = tz.getOffset(millisLocal - guess)
-    if (offset != guess) {
-      guess = tz.getOffset(millisLocal - offset)
-      if (guess != offset) {
-        // fallback to do the reverse lookup using java.time.LocalDateTime
-        // this should only happen near the start or end of DST
-        val localDate = LocalDate.ofEpochDay(MILLISECONDS.toDays(millisLocal))
-        val localTime = LocalTime.ofNanoOfDay(MILLISECONDS.toNanos(
-          Math.floorMod(millisLocal, MILLIS_PER_DAY)))
-        val localDateTime = LocalDateTime.of(localDate, localTime)
-        val millisEpoch = localDateTime.atZone(tz.toZoneId).toInstant.toEpochMilli
-
-        guess = (millisLocal - millisEpoch).toInt
-      }
-    }
-    guess
-  }
-
   /**
    * Convert the timestamp `ts` from one timezone to another.
    *
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index 77a2ca7e4a828..536c76f042d23 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.csv
 
 import java.math.BigDecimal
 import java.text.{DecimalFormat, DecimalFormatSymbols}
+import java.time.ZoneOffset
 import java.util.{Locale, TimeZone}
 
 import org.apache.commons.lang3.time.FastDateFormat
@@ -137,7 +138,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val expectedDate = format.parse(customDate).getTime
     val castedDate = parser.makeConverter("_1", DateType, nullable = true)
         .apply(customDate)
-    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, TimeZone.getTimeZone("GMT")))
+    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, ZoneOffset.UTC))
 
     val timestamp = "2015-01-01 00:00:00"
     timestampsOptions = new CSVOptions(Map(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index f04149ab7eb29..39b859af47ca9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -56,9 +56,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   val ts = new Timestamp(toMillis(time))
 
   test("datetime function current_date") {
-    val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), TimeZoneGMT)
+    val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), ZoneOffset.UTC)
     val cd = CurrentDate(gmtId).eval(EmptyRow).asInstanceOf[Int]
-    val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), TimeZoneGMT)
+    val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), ZoneOffset.UTC)
     assert(d0 <= cd && cd <= d1 && d1 - d0 <= 1)
 
     val cdjst = CurrentDate(jstId).eval(EmptyRow).asInstanceOf[Int]
@@ -499,7 +499,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Valid range of DateType is [0001-01-01, 9999-12-31]
     val maxMonthInterval = 10000 * 12
     checkEvaluation(
-      AddMonths(Literal(Date.valueOf("0001-01-01")), Literal(maxMonthInterval)), 2933261)
+      AddMonths(Literal(LocalDate.parse("0001-01-01")), Literal(maxMonthInterval)),
+      LocalDate.of(10001, 1, 1).toEpochDay.toInt)
     checkEvaluation(
       AddMonths(Literal(Date.valueOf("9999-12-31")), Literal(-1 * maxMonthInterval)), -719529)
     // Test evaluation results between Interpreted mode and Codegen mode
@@ -788,7 +789,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(
               UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
               MILLISECONDS.toSeconds(
-                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
             checkEvaluation(
               UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))),
                 Literal(fmt2), timeZoneId),
@@ -796,7 +797,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(UnixTimestamp(
               Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
               MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId)))
             val t1 = UnixTimestamp(
               CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
             val t2 = UnixTimestamp(
@@ -814,7 +815,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(
               UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
               MILLISECONDS.toSeconds(
-                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
             checkEvaluation(
               UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
           }
@@ -852,7 +853,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(
               ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
               MILLISECONDS.toSeconds(
-                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
             checkEvaluation(
               ToUnixTimestamp(
                 Literal(sdf2.format(new Timestamp(-1000000))),
@@ -861,7 +862,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(ToUnixTimestamp(
               Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
               MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId)))
             val t1 = ToUnixTimestamp(
               CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
             val t2 = ToUnixTimestamp(
@@ -876,7 +877,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(ToUnixTimestamp(
               Literal(date1), Literal.create(null, StringType), timeZoneId),
               MILLISECONDS.toSeconds(
-                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
             checkEvaluation(
               ToUnixTimestamp(
                 Literal("2015-07-24"),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index cabcd3007d1c0..cd0594c775a47 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -86,9 +86,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("SPARK-6785: java date conversion before and after epoch") {
+    def format(d: Date): String = {
+      TimestampFormatter("uuuu-MM-dd", defaultTimeZone().toZoneId)
+        .format(d.getTime * MICROS_PER_MILLIS)
+    }
     def checkFromToJavaDate(d1: Date): Unit = {
       val d2 = toJavaDate(fromJavaDate(d1))
-      assert(d2.toString === d1.toString)
+      assert(format(d2) === format(d1))
     }
 
     val df1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
@@ -413,22 +417,22 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   test("monthsBetween") {
     val date1 = date(1997, 2, 28, 10, 30, 0)
     var date2 = date(1996, 10, 30)
-    assert(monthsBetween(date1, date2, true, TimeZoneUTC) === 3.94959677)
-    assert(monthsBetween(date1, date2, false, TimeZoneUTC) === 3.9495967741935485)
+    assert(monthsBetween(date1, date2, true, ZoneOffset.UTC) === 3.94959677)
+    assert(monthsBetween(date1, date2, false, ZoneOffset.UTC) === 3.9495967741935485)
     Seq(true, false).foreach { roundOff =>
       date2 = date(2000, 2, 28)
-      assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === -36)
+      assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === -36)
       date2 = date(2000, 2, 29)
-      assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === -36)
+      assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === -36)
       date2 = date(1996, 3, 31)
-      assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === 11)
+      assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === 11)
     }
 
     val date3 = date(2000, 2, 28, 16, tz = TimeZonePST)
     val date4 = date(1997, 2, 28, 16, tz = TimeZonePST)
-    assert(monthsBetween(date3, date4, true, TimeZonePST) === 36.0)
-    assert(monthsBetween(date3, date4, true, TimeZoneGMT) === 35.90322581)
-    assert(monthsBetween(date3, date4, false, TimeZoneGMT) === 35.903225806451616)
+    assert(monthsBetween(date3, date4, true, TimeZonePST.toZoneId) === 36.0)
+    assert(monthsBetween(date3, date4, true, ZoneOffset.UTC) === 35.90322581)
+    assert(monthsBetween(date3, date4, false, ZoneOffset.UTC) === 35.903225806451616)
   }
 
   test("from UTC timestamp") {
@@ -571,15 +575,15 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
   test("daysToMillis and millisToDays") {
     val input = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, 16, tz = TimeZonePST))
-    assert(millisToDays(input, TimeZonePST) === 16800)
-    assert(millisToDays(input, TimeZoneGMT) === 16801)
-    assert(millisToDays(-1 * MILLIS_PER_DAY + 1, TimeZoneGMT) == -1)
+    assert(millisToDays(input, TimeZonePST.toZoneId) === 16800)
+    assert(millisToDays(input, ZoneOffset.UTC) === 16801)
+    assert(millisToDays(-1 * MILLIS_PER_DAY + 1, ZoneOffset.UTC) == -1)
 
     var expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZonePST))
-    assert(daysToMillis(16800, TimeZonePST) === expected)
+    assert(daysToMillis(16800, TimeZonePST.toZoneId) === expected)
 
     expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZoneGMT))
-    assert(daysToMillis(16800, TimeZoneGMT) === expected)
+    assert(daysToMillis(16800, ZoneOffset.UTC) === expected)
 
     // There are some days are skipped entirely in some timezone, skip them here.
     val skipped_days = Map[String, Set[Int]](
@@ -594,7 +598,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
       (-20000 to 20000).foreach { d =>
         if (!skipped.contains(d)) {
-          assert(millisToDays(daysToMillis(d, tz), tz) === d,
+          assert(millisToDays(daysToMillis(d, tz.toZoneId), tz.toZoneId) === d,
             s"Round trip of ${d} did not work in tz ${tz}")
         }
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index bbe47a63f4d61..5a2f16d8e1526 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate}
 
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
@@ -67,8 +68,12 @@ object HiveResult {
     case (null, _) => if (nested) "null" else "NULL"
     case (b, BooleanType) => b.toString
     case (d: Date, DateType) => dateFormatter.format(DateTimeUtils.fromJavaDate(d))
+    case (ld: LocalDate, DateType) =>
+      dateFormatter.format(DateTimeUtils.localDateToDays(ld))
     case (t: Timestamp, TimestampType) =>
       timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t))
+    case (i: Instant, TimestampType) =>
+      timestampFormatter.format(DateTimeUtils.instantToMicros(i))
     case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
     case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
     case (n, _: NumericType) => n.toString
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index fd5dc42632176..ed27317121623 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -800,7 +800,7 @@ SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000')
 -- !query schema
 struct<date_trunc(MILLENNIUM, TIMESTAMP '1970-03-20 04:30:00'):timestamp>
 -- !query output
-1001-01-01 00:07:02
+1001-01-01 00:00:00
 
 
 -- !query
@@ -808,7 +808,7 @@ SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20')
 -- !query schema
 struct<date_trunc(MILLENNIUM, CAST(DATE '1970-03-20' AS TIMESTAMP)):timestamp>
 -- !query output
-1001-01-01 00:07:02
+1001-01-01 00:00:00
 
 
 -- !query
@@ -840,7 +840,7 @@ SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04')
 -- !query schema
 struct<date_trunc(CENTURY, CAST(DATE '0002-02-04' AS TIMESTAMP)):timestamp>
 -- !query output
-0001-01-01 00:07:02
+0001-01-01 00:00:00
 
 
 -- !query
@@ -848,7 +848,7 @@ SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G'))
 -- !query schema
 struct<date_trunc(CENTURY, CAST(to_date('0055-08-10 BC', 'yyyy-MM-dd G') AS TIMESTAMP)):timestamp>
 -- !query output
--0099-01-01 00:07:02
+-0099-01-01 00:00:00
 
 
 -- !query
@@ -864,7 +864,7 @@ SELECT DATE_TRUNC('DECADE', DATE '0004-12-25')
 -- !query schema
 struct<date_trunc(DECADE, CAST(DATE '0004-12-25' AS TIMESTAMP)):timestamp>
 -- !query output
-0000-01-01 00:07:02
+0000-01-01 00:00:00
 
 
 -- !query
@@ -872,7 +872,7 @@ SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
 struct<date_trunc(DECADE, CAST(to_date('0002-12-31 BC', 'yyyy-MM-dd G') AS TIMESTAMP)):timestamp>
 -- !query output
--0010-01-01 00:07:02
+-0010-01-01 00:00:00
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 2e5a9e0b4d45d..6b9e5bbd3c961 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -337,6 +337,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
         localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true)
       case _ =>
     }
+    localSparkSession.conf.set(SQLConf.DATETIME_JAVA8API_ENABLED.key, true)
 
     if (configSet.nonEmpty) {
       // Execute the list of set operation in order to add the desired configs

From 82981737f762760c07ae82464b15dc866a2b64e5 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <liangchi@uber.com>
Date: Wed, 12 Feb 2020 14:27:18 -0800
Subject: [PATCH 0062/1280] [SPARK-30797][SQL] Set tradition user/group/other
 permission to ACL entries when setting up ACLs in truncate table

### What changes were proposed in this pull request?

This is a follow-up to the PR #26956. In #26956, the patch proposed to preserve path permission when truncating table. When setting up original ACLs, we need to set user/group/other permission as ACL entries too, otherwise if the path doesn't have default user/group/other ACL entries, ACL API will complain an error `Invalid ACL: the user, group and other entries are required.`.

 In short this change makes sure:

1. Permissions for user/group/other are always kept into ACLs to work with ACL API.
2. Other custom ACLs are still kept after TRUNCATE TABLE (#26956 did this).

### Why are the changes needed?

Without this fix, `TRUNCATE TABLE` will get an error when setting up ACLs if there is no default default user/group/other ACL entries.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Update unit test.

Manual test on dev Spark cluster.

Set ACLs for a table path without default user/group/other ACL entries:
```
hdfs dfs -setfacl --set 'user:liangchi:rwx,user::rwx,group::r--,other::r--' /user/hive/warehouse/test.db/test_truncate_table

hdfs dfs -getfacl /user/hive/warehouse/test.db/test_truncate_table
# file: /user/hive/warehouse/test.db/test_truncate_table
# owner: liangchi
# group: supergroup
user::rwx
user:liangchi:rwx
group::r--
mask::rwx
other::r--
```
Then run `sql("truncate table test.test_truncate_table")`, it works by normally truncating the table and preserve ACLs.

Closes #27548 from viirya/fix-truncate-table-permission.

Lead-authored-by: Liang-Chi Hsieh <liangchi@uber.com>
Co-authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 5b76367a9d0aaca53ce96ab7e555a596567e8335)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/execution/command/tables.scala  | 32 +++++++++++++++++--
 .../sql/execution/command/DDLSuite.scala      | 21 +++++++++++-
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 90dbdf5515d4d..61500b773cd7e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.command
 
 import java.net.{URI, URISyntaxException}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.util.Try
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.{FileContext, FsConstants, Path}
-import org.apache.hadoop.fs.permission.{AclEntry, FsPermission}
+import org.apache.hadoop.fs.permission.{AclEntry, AclEntryScope, AclEntryType, FsAction, FsPermission}
 
 import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.TableIdentifier
@@ -538,12 +539,27 @@ case class TruncateTableCommand(
               }
             }
             optAcls.foreach { acls =>
+              val aclEntries = acls.asScala.filter(_.getName != null).asJava
+
+              // If the path doesn't have default ACLs, `setAcl` API will throw an error
+              // as it expects user/group/other permissions must be in ACL entries.
+              // So we need to add tradition user/group/other permission
+              // in the form of ACL.
+              optPermission.map { permission =>
+                aclEntries.add(newAclEntry(AclEntryScope.ACCESS,
+                  AclEntryType.USER, permission.getUserAction()))
+                aclEntries.add(newAclEntry(AclEntryScope.ACCESS,
+                  AclEntryType.GROUP, permission.getGroupAction()))
+                aclEntries.add(newAclEntry(AclEntryScope.ACCESS,
+                  AclEntryType.OTHER, permission.getOtherAction()))
+              }
+
               try {
-                fs.setAcl(path, acls)
+                fs.setAcl(path, aclEntries)
               } catch {
                 case NonFatal(e) =>
                   throw new SecurityException(
-                    s"Failed to set original ACL $acls back to " +
+                    s"Failed to set original ACL $aclEntries back to " +
                       s"the created path: $path. Exception: ${e.getMessage}")
               }
             }
@@ -574,6 +590,16 @@ case class TruncateTableCommand(
     }
     Seq.empty[Row]
   }
+
+  private def newAclEntry(
+      scope: AclEntryScope,
+      aclType: AclEntryType,
+      permission: FsAction): AclEntry = {
+    new AclEntry.Builder()
+      .setScope(scope)
+      .setType(aclType)
+      .setPermission(permission).build()
+  }
 }
 
 abstract class DescribeCommandBase extends RunnableCommand {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 31e00781ae6b4..dbf4b09403423 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2042,6 +2042,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
           // Set ACL to table path.
           val customAcl = new java.util.ArrayList[AclEntry]()
           customAcl.add(new AclEntry.Builder()
+            .setName("test")
             .setType(AclEntryType.USER)
             .setScope(AclEntryScope.ACCESS)
             .setPermission(FsAction.READ).build())
@@ -2061,8 +2062,26 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
           if (ignore) {
             assert(aclEntries.size() == 0)
           } else {
-            assert(aclEntries.size() == 1)
+            assert(aclEntries.size() == 4)
             assert(aclEntries.get(0) == customAcl.get(0))
+
+            // Setting ACLs will also set user/group/other permissions
+            // as ACL entries.
+            val user = new AclEntry.Builder()
+              .setType(AclEntryType.USER)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            val group = new AclEntry.Builder()
+              .setType(AclEntryType.GROUP)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            val other = new AclEntry.Builder()
+              .setType(AclEntryType.OTHER)
+              .setScope(AclEntryScope.ACCESS)
+              .setPermission(FsAction.ALL).build()
+            assert(aclEntries.get(1) == user)
+            assert(aclEntries.get(2) == group)
+            assert(aclEntries.get(3) == other)
           }
         }
       }

From 8ab6ae3ede96adb093347470a5cbbf17fe8c04e9 Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Thu, 13 Feb 2020 12:23:40 +0800
Subject: [PATCH 0063/1280] [SPARK-30790] The dataType of map() should be
 map<null,null>

### What changes were proposed in this pull request?

`spark.sql("select map()")` returns {}.

After these changes it will return map<null,null>

### Why are the changes needed?
After changes introduced due to #27521, it is important to maintain consistency while using map().

### Does this PR introduce any user-facing change?
Yes. Now map() will give map<null,null> instead of {}.

### How was this patch tested?
UT added. Migration guide updated as well

Closes #27542 from iRakson/SPARK-30790.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 926e3a1efe9e142804fcbf52146b22700640ae1b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |  2 +-
 .../expressions/complexTypeCreator.scala      | 14 ++++++++---
 .../catalyst/util/ArrayBasedMapBuilder.scala  |  5 ++--
 .../apache/spark/sql/internal/SQLConf.scala   | 10 ++++----
 .../spark/sql/DataFrameFunctionsSuite.scala   | 25 +++++++++++++------
 5 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index f98fab5b4c56b..46b741687363f 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -216,7 +216,7 @@ license: |
 
   - Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
   
-  - Since Spark 3.0, when the `array` function is called without any parameters, it returns an empty array of `NullType`. In Spark version 2.4 and earlier, it returns an empty array of string type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.arrayDefaultToStringType.enabled` to `true`.
+  - Since Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and earlier, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
 
   - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 7335e305bfe55..4bd85d304ded2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -46,7 +46,7 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
   }
 
   private val defaultElementType: DataType = {
-    if (SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_DEFAULT_TO_STRING)) {
+    if (SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE)) {
       StringType
     } else {
       NullType
@@ -145,6 +145,14 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
   lazy val keys = children.indices.filter(_ % 2 == 0).map(children)
   lazy val values = children.indices.filter(_ % 2 != 0).map(children)
 
+  private val defaultElementType: DataType = {
+    if (SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE)) {
+      StringType
+    } else {
+      NullType
+    }
+  }
+
   override def foldable: Boolean = children.forall(_.foldable)
 
   override def checkInputDataTypes(): TypeCheckResult = {
@@ -167,9 +175,9 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
   override lazy val dataType: MapType = {
     MapType(
       keyType = TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(keys.map(_.dataType))
-        .getOrElse(StringType),
+        .getOrElse(defaultElementType),
       valueType = TypeCoercion.findCommonTypeDifferentOnlyInNullFlags(values.map(_.dataType))
-        .getOrElse(StringType),
+        .getOrElse(defaultElementType),
       valueContainsNull = values.exists(_.nullable))
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
index 98934368205ec..37d65309e2b89 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
@@ -29,12 +29,11 @@ import org.apache.spark.unsafe.array.ByteArrayMethods
  */
 class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Serializable {
   assert(!keyType.existsRecursively(_.isInstanceOf[MapType]), "key of map cannot be/contain map")
-  assert(keyType != NullType, "map key cannot be null type.")
 
   private lazy val keyToIndex = keyType match {
     // Binary type data is `byte[]`, which can't use `==` to check equality.
-    case _: AtomicType | _: CalendarIntervalType if !keyType.isInstanceOf[BinaryType] =>
-      new java.util.HashMap[Any, Int]()
+    case _: AtomicType | _: CalendarIntervalType | _: NullType
+      if !keyType.isInstanceOf[BinaryType] => new java.util.HashMap[Any, Int]()
     case _ =>
       // for complex types, use interpreted ordering to be able to compare unsafe data with safe
       // data, e.g. UnsafeRow vs GenericInternalRow.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b79b767dbb22b..442711db93f0e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2007,12 +2007,12 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_ARRAY_DEFAULT_TO_STRING =
-    buildConf("spark.sql.legacy.arrayDefaultToStringType.enabled")
+  val LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE =
+    buildConf("spark.sql.legacy.createEmptyCollectionUsingStringType")
       .internal()
-      .doc("When set to true, it returns an empty array of string type when the `array` " +
-        "function is called without any parameters. Otherwise, it returns an empty " +
-        "array of `NullType`")
+      .doc("When set to true, Spark returns an empty collection with `StringType` as element " +
+        "type if the `array`/`map` function is called without any parameters. Otherwise, Spark " +
+        "returns an empty collection with `NullType` as element type.")
       .booleanConf
       .createWithDefault(false)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 6012678341ccc..f7531ea446015 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -3499,13 +3499,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     ).foreach(assertValuesDoNotChangeAfterCoalesceOrUnion(_))
   }
 
-  test("SPARK-21281 use string types by default if map have no argument") {
-    val ds = spark.range(1)
-    var expectedSchema = new StructType()
-      .add("x", MapType(StringType, StringType, valueContainsNull = false), nullable = false)
-    assert(ds.select(map().as("x")).schema == expectedSchema)
-  }
-
   test("SPARK-21281 fails if functions have no argument") {
     val df = Seq(1).toDF("a")
 
@@ -3563,7 +3556,8 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
   test("SPARK-29462: Empty array of NullType for array function with no arguments") {
     Seq((true, StringType), (false, NullType)).foreach {
       case (arrayDefaultToString, expectedType) =>
-        withSQLConf(SQLConf.LEGACY_ARRAY_DEFAULT_TO_STRING.key -> arrayDefaultToString.toString) {
+        withSQLConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE.key ->
+          arrayDefaultToString.toString) {
           val schema = spark.range(1).select(array()).schema
           assert(schema.nonEmpty && schema.head.dataType.isInstanceOf[ArrayType])
           val actualType = schema.head.dataType.asInstanceOf[ArrayType].elementType
@@ -3571,6 +3565,21 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
         }
     }
   }
+
+  test("SPARK-30790: Empty map with NullType as key/value type for map function with no argument") {
+    Seq((true, StringType), (false, NullType)).foreach {
+      case (mapDefaultToString, expectedType) =>
+        withSQLConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE.key ->
+          mapDefaultToString.toString) {
+          val schema = spark.range(1).select(map()).schema
+          assert(schema.nonEmpty && schema.head.dataType.isInstanceOf[MapType])
+          val actualKeyType = schema.head.dataType.asInstanceOf[MapType].keyType
+          val actualValueType = schema.head.dataType.asInstanceOf[MapType].valueType
+          assert(actualKeyType === expectedType)
+          assert(actualValueType === expectedType)
+        }
+    }
+  }
 }
 
 object DataFrameFunctionsSuite {

From a2c46334d44355f022e0498b8e2de71d7c91a533 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 5 Feb 2020 16:55:45 -0800
Subject: [PATCH 0064/1280] [SPARK-30743][K8S][TESTS] Use JRE instead of JDK in
 K8S test docker image

### What changes were proposed in this pull request?

This PR aims to replace JDK to JRE in K8S integration test docker images.

### Why are the changes needed?

This will save some resources and make it sure that we only need JRE at runtime and testing.
- https://lists.apache.org/thread.html/3145150b711d7806a86bcd3ab43e18bcd0e4892ab5f11600689ba087%40%3Cdev.spark.apache.org%3E

### Does this PR introduce any user-facing change?

No. This is a dev-only test environment.

### How was this patch tested?

Pass the Jenkins K8s Integration Test.
- https://github.com/apache/spark/pull/27469#issuecomment-582681125

Closes #27469 from dongjoon-hyun/SPARK-30743.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 9d907bc84df2f6f7e1abdb810b761a65ac6ce064)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../kubernetes/docker/src/main/dockerfiles/spark/Dockerfile     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index b6eeff1cd18a9..a1fc63789bc61 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM openjdk:8-jdk-slim
+FROM openjdk:8-jre-slim
 
 ARG spark_uid=185
 

From 59a13c9b7bc3b3aa5b5bc30a60344f849c0f8012 Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Thu, 13 Feb 2020 19:32:38 +0800
Subject: [PATCH 0065/1280] [SPARK-30528][SQL] Turn off DPP subquery
 duplication by default

### What changes were proposed in this pull request?
This PR adds a config for Dynamic Partition Pruning subquery duplication and turns it off by default due to its potential performance regression.
When planning a DPP filter, it seeks to reuse the broadcast exchange relation if the corresponding join is a BHJ with the filter relation being on the build side, otherwise it will either opt out or plan the filter as an un-reusable subquery duplication based on the cost estimate. However, the cost estimate is not accurate and only takes into account the table scan overhead, thus adding an un-reusable subquery duplication DPP filter can sometimes cause perf regression.
This PR turns off the subquery duplication DPP filter by:
1. adding a config `spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly` and setting it `true` by default.
2. removing the existing meaningless config `spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcast` since we always want to reuse broadcast results if possible.

### Why are the changes needed?
This is to fix a potential performance regression caused by DPP.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Updated DynamicPartitionPruningSuite to test the new configuration.

Closes #27551 from maryannxue/spark-30528.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 453d5261b22ebcdd5886e65ab9d0d9857051e76a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  12 +-
 .../sql/dynamicpruning/PartitionPruning.scala |   4 +-
 .../PlanDynamicPruningFilters.scala           |   5 +-
 .../sql/DynamicPartitionPruningSuite.scala    | 183 +++++++-----------
 .../org/apache/spark/sql/ExplainSuite.scala   |   3 +-
 5 files changed, 82 insertions(+), 125 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 442711db93f0e..19c94e23e046e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -259,11 +259,11 @@ object SQLConf {
     .doubleConf
     .createWithDefault(0.5)
 
-  val DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST =
-    buildConf("spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcast")
+  val DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY =
+    buildConf("spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly")
       .internal()
-      .doc("When true, dynamic partition pruning will seek to reuse the broadcast results from " +
-        "a broadcast hash join operation.")
+      .doc("When true, dynamic partition pruning will only apply when the broadcast exchange of " +
+        "a broadcast hash join operation can be reused as the dynamic pruning filter.")
       .booleanConf
       .createWithDefault(true)
 
@@ -2303,8 +2303,8 @@ class SQLConf extends Serializable with Logging {
   def dynamicPartitionPruningFallbackFilterRatio: Double =
     getConf(DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO)
 
-  def dynamicPartitionPruningReuseBroadcast: Boolean =
-    getConf(DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST)
+  def dynamicPartitionPruningReuseBroadcastOnly: Boolean =
+    getConf(DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY)
 
   def stateStoreProviderClass: String = getConf(STATE_STORE_PROVIDER_CLASS)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
index 48ba8618f272e..28f8f49d2ce44 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
@@ -86,7 +86,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
       filteringPlan: LogicalPlan,
       joinKeys: Seq[Expression],
       hasBenefit: Boolean): LogicalPlan = {
-    val reuseEnabled = SQLConf.get.dynamicPartitionPruningReuseBroadcast
+    val reuseEnabled = SQLConf.get.exchangeReuseEnabled
     val index = joinKeys.indexOf(filteringKey)
     if (hasBenefit || reuseEnabled) {
       // insert a DynamicPruning wrapper to identify the subquery during query planning
@@ -96,7 +96,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
           filteringPlan,
           joinKeys,
           index,
-          !hasBenefit),
+          !hasBenefit || SQLConf.get.dynamicPartitionPruningReuseBroadcastOnly),
         pruningPlan)
     } else {
       // abort dynamic partition pruning
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
index 1398dc049dd99..be00f728aa3ca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
@@ -36,9 +36,6 @@ import org.apache.spark.sql.internal.SQLConf
 case class PlanDynamicPruningFilters(sparkSession: SparkSession)
     extends Rule[SparkPlan] with PredicateHelper {
 
-  private def reuseBroadcast: Boolean =
-    SQLConf.get.dynamicPartitionPruningReuseBroadcast && SQLConf.get.exchangeReuseEnabled
-
   /**
    * Identify the shape in which keys of a given plan are broadcasted.
    */
@@ -59,7 +56,7 @@ case class PlanDynamicPruningFilters(sparkSession: SparkSession)
           sparkSession, sparkSession.sessionState.planner, buildPlan)
         // Using `sparkPlan` is a little hacky as it is based on the assumption that this rule is
         // the first to be applied (apart from `InsertAdaptiveSparkPlan`).
-        val canReuseExchange = reuseBroadcast && buildKeys.nonEmpty &&
+        val canReuseExchange = SQLConf.get.exchangeReuseEnabled && buildKeys.nonEmpty &&
           plan.find {
             case BroadcastHashJoinExec(_, _, _, BuildLeft, _, left, _) =>
               left.sameResult(sparkPlan)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index e1f9bcc4e008d..f7b51d6f4c8ef 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -239,7 +239,8 @@ class DynamicPartitionPruningSuite
    */
   test("simple inner join triggers DPP with mock-up tables") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("df1", "df2") {
         spark.range(1000)
           .select(col("id"), col("id").as("k"))
@@ -271,7 +272,8 @@ class DynamicPartitionPruningSuite
    */
   test("self-join on a partitioned table should not trigger DPP") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact") {
         sql(
           s"""
@@ -302,7 +304,8 @@ class DynamicPartitionPruningSuite
    */
   test("static scan metrics") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact", "dim") {
         spark.range(10)
           .map { x => Tuple3(x, x + 1, 0) }
@@ -370,7 +373,8 @@ class DynamicPartitionPruningSuite
   test("DPP should not be rewritten as an existential join") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
       SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "1.5",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       val df = sql(
         s"""
            |SELECT * FROM product p WHERE p.store_id NOT IN
@@ -395,7 +399,7 @@ class DynamicPartitionPruningSuite
    */
   test("DPP triggers only for certain types of query") {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false") {
       Given("dynamic partition pruning disabled")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "false") {
         val df = sql(
@@ -433,7 +437,8 @@ class DynamicPartitionPruningSuite
       }
 
       Given("left-semi join with partition column on the left side")
-      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
         val df = sql(
           """
             |SELECT * FROM fact_sk f
@@ -457,7 +462,8 @@ class DynamicPartitionPruningSuite
       }
 
       Given("right outer join with partition column on the left side")
-      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+      withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
         val df = sql(
           """
             |SELECT * FROM fact_sk f RIGHT OUTER JOIN dim_store s
@@ -474,7 +480,8 @@ class DynamicPartitionPruningSuite
    */
   test("filtering ratio policy fallback") {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       Given("no stats and selective predicate")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
         SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true") {
@@ -543,7 +550,8 @@ class DynamicPartitionPruningSuite
    */
   test("filtering ratio policy with stats when the broadcast pruning is disabled") {
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       Given("disabling the use of stats in the DPP heuristic")
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
         SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false") {
@@ -613,10 +621,7 @@ class DynamicPartitionPruningSuite
 
   test("partition pruning in broadcast hash joins with non-deterministic probe part") {
     Given("alias with simple join condition, and non-deterministic query")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -630,10 +635,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias over multiple sub-queries with simple join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -651,10 +653,7 @@ class DynamicPartitionPruningSuite
 
   test("partition pruning in broadcast hash joins with aliases") {
     Given("alias with simple join condition, using attribute names only")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -674,10 +673,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias with expr as join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -697,10 +693,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias over multiple sub-queries with simple join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -722,10 +715,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("alias over multiple sub-queries with simple join condition")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid_d as pid, f.sid_d as sid FROM
@@ -754,10 +744,8 @@ class DynamicPartitionPruningSuite
   test("partition pruning in broadcast hash joins") {
     Given("disable broadcast pruning and disable subquery duplication")
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false",
-      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       val df = sql(
         """
           |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
@@ -777,9 +765,10 @@ class DynamicPartitionPruningSuite
 
     Given("disable reuse broadcast results and enable subquery duplication")
     withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false",
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0.5") {
+      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0.5",
+      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       val df = sql(
         """
           |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
@@ -798,52 +787,47 @@ class DynamicPartitionPruningSuite
     }
 
     Given("enable reuse broadcast results and disable query duplication")
-      withSQLConf(
-        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
-        val df = sql(
-          """
-            |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
-            |JOIN dim_stats s
-            |ON f.store_id = s.store_id WHERE s.country = 'DE'
-          """.stripMargin)
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
+      val df = sql(
+        """
+          |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+          |JOIN dim_stats s
+          |ON f.store_id = s.store_id WHERE s.country = 'DE'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, false, true)
+      checkPartitionPruningPredicate(df, false, true)
 
-        checkAnswer(df,
-          Row(1030, 2, 10, 3) ::
-          Row(1040, 2, 50, 3) ::
-          Row(1050, 2, 50, 3) ::
-          Row(1060, 2, 50, 3) :: Nil
-        )
+      checkAnswer(df,
+        Row(1030, 2, 10, 3) ::
+        Row(1040, 2, 50, 3) ::
+        Row(1050, 2, 50, 3) ::
+        Row(1060, 2, 50, 3) :: Nil
+      )
     }
 
     Given("disable broadcast hash join and disable query duplication")
-      withSQLConf(
-        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-        SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
-        val df = sql(
-          """
-            |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
-            |JOIN dim_stats s
-            |ON f.store_id = s.store_id WHERE s.country = 'DE'
-          """.stripMargin)
+    withSQLConf(
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      val df = sql(
+        """
+          |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+          |JOIN dim_stats s
+          |ON f.store_id = s.store_id WHERE s.country = 'DE'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, false, false)
+      checkPartitionPruningPredicate(df, false, false)
 
-        checkAnswer(df,
-          Row(1030, 2, 10, 3) ::
-          Row(1040, 2, 50, 3) ::
-          Row(1050, 2, 50, 3) ::
-          Row(1060, 2, 50, 3) :: Nil
-        )
+      checkAnswer(df,
+        Row(1030, 2, 10, 3) ::
+        Row(1040, 2, 50, 3) ::
+        Row(1050, 2, 50, 3) ::
+        Row(1060, 2, 50, 3) :: Nil
+      )
     }
 
     Given("disable broadcast hash join and enable query duplication")
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
       SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "true") {
       val df = sql(
@@ -865,9 +849,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("broadcast a single key in a HashedRelation") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -925,9 +907,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("broadcast multiple keys in a LongHashedRelation") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -962,9 +942,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("broadcast multiple keys in an UnsafeHashedRelation") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -999,9 +977,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("different broadcast subqueries with identical children") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(100).select(
           $"id",
@@ -1073,7 +1049,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("avoid reordering broadcast join keys to match input hash partitioning") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
       withTable("large", "dimTwo", "dimThree") {
         spark.range(100).select(
@@ -1123,9 +1099,7 @@ class DynamicPartitionPruningSuite
    * duplicated partitioning keys, also used to uniquely identify the dynamic pruning filters.
    */
   test("dynamic partition pruning ambiguity issue across nested joins") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("store", "date", "item") {
         spark.range(500)
           .select((($"id" + 30) % 50).as("ss_item_sk"),
@@ -1163,9 +1137,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("cleanup any DPP filter that isn't pushed down due to expression id clashes") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
         spark.range(1000).select($"id".as("A"), $"id".as("AA"))
           .write.partitionBy("A").format(tableFormat).mode("overwrite").saveAsTable("fact")
@@ -1186,10 +1158,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("cleanup any DPP filter that isn't pushed down due to non-determinism") {
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.pid, f.sid FROM
@@ -1204,9 +1173,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("join key with multiple references on the filtering plan") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0",
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
       // when enable AQE, the reusedExchange is inserted when executed.
       withTable("fact", "dim") {
@@ -1240,9 +1207,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("Make sure dynamic pruning works on uncorrelated queries") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT d.store_id,
@@ -1266,10 +1231,7 @@ class DynamicPartitionPruningSuite
 
   test("Plan broadcast pruning only when the broadcast can be reused") {
     Given("dynamic pruning filter on the build side")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT f.date_id, f.store_id, f.product_id, f.units_sold FROM fact_np f
@@ -1288,10 +1250,7 @@ class DynamicPartitionPruningSuite
     }
 
     Given("dynamic pruning filter on the probe side")
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_USE_STATS.key -> "false",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "0") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       val df = sql(
         """
           |SELECT /*+ BROADCAST(f)*/
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index d9f4d6d5132ae..b591705274110 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -239,7 +239,8 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
   test("explain formatted - check presence of subquery in case of DPP") {
     withTable("df1", "df2") {
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST.key -> "false") {
+        SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
+        SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
         withTable("df1", "df2") {
           spark.range(1000).select(col("id"), col("id").as("k"))
             .write

From f041aaaf55fb1e907e0e5b0876927ef328664664 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 13 Feb 2020 22:06:24 +0800
Subject: [PATCH 0066/1280] [SPARK-30758][SQL][TESTS] Improve bracketed
 comments tests

### What changes were proposed in this pull request?
Although Spark SQL support bracketed comments, but `SQLQueryTestSuite` can't treat bracketed comments well and lead to generated golden files can't display bracketed comments well.
This PR will improve the treatment of bracketed comments and add three test case in `PlanParserSuite`.
Spark SQL can't support nested bracketed comments and https://github.com/apache/spark/pull/27495 used to support it.

### Why are the changes needed?
Golden files can't display well.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
New UT.

Closes #27481 from beliefer/ansi-brancket-comments.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 04604b9899cc43a9726d671061ff305912fdb85f)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql-tests/inputs/postgreSQL/comments.sql  |   6 +-
 .../results/postgreSQL/comments.sql.out       | 137 ++++--------------
 .../apache/spark/sql/SQLQueryTestSuite.scala  |  51 ++++++-
 3 files changed, 78 insertions(+), 116 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql
index 6725ce45e72a5..1a454179ef79f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/comments.sql
@@ -11,17 +11,19 @@ SELECT /* embedded single line */ 'embedded' AS `second`;
 SELECT /* both embedded and trailing single line */ 'both' AS third; -- trailing single line
 
 SELECT 'before multi-line' AS fourth;
+--QUERY-DELIMITER-START
 -- [SPARK-28880] ANSI SQL: Bracketed comments
 /* This is an example of SQL which should not execute:
  * select 'multi-line';
  */
 SELECT 'after multi-line' AS fifth;
+--QUERY-DELIMITER-END
 
 -- [SPARK-28880] ANSI SQL: Bracketed comments
 --
 -- Nested comments
 --
-
+--QUERY-DELIMITER-START
 /*
 SELECT 'trailing' as x1; -- inside block comment
 */
@@ -44,5 +46,5 @@ Hoo boy. Still two deep...
 Now just one deep...
 */
 'deeply nested example' AS sixth;
-
+--QUERY-DELIMITER-END
 /* and this is the end of the file */
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out
index 4ea49013a62d1..637c5561bd940 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/comments.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 7
 
 
 -- !query
@@ -36,129 +36,32 @@ before multi-line
 
 -- !query
 /* This is an example of SQL which should not execute:
- * select 'multi-line'
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/* This is an example of SQL which should not execute:
-^^^
- * select 'multi-line'
-
-
--- !query
-*/
+ * select 'multi-line';
+ */
 SELECT 'after multi-line' AS fifth
 -- !query schema
-struct<>
+struct<fifth:string>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-extraneous input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-*/
-^^^
-SELECT 'after multi-line' AS fifth
+after multi-line
 
 
 -- !query
 /*
-SELECT 'trailing' as x1
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/*
-^^^
-SELECT 'trailing' as x1
-
-
--- !query
+SELECT 'trailing' as x1; -- inside block comment
 */
 
 /* This block comment surrounds a query which itself has a block comment...
-SELECT /* embedded single line */ 'embedded' AS x2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-*/
-^^^
-
-/* This block comment surrounds a query which itself has a block comment...
-SELECT /* embedded single line */ 'embedded' AS x2
-
-
--- !query
+SELECT /* embedded single line */ 'embedded' AS x2;
 */
 
 SELECT -- continued after the following block comments...
 /* Deeply nested comment.
    This includes a single apostrophe to make sure we aren't decoding this part as a string.
-SELECT 'deep nest' AS n1
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-extraneous input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-*/
-^^^
-
-SELECT -- continued after the following block comments...
-/* Deeply nested comment.
-   This includes a single apostrophe to make sure we aren't decoding this part as a string.
-SELECT 'deep nest' AS n1
-
-
--- !query
+SELECT 'deep nest' AS n1;
 /* Second level of nesting...
-SELECT 'deeper nest' as n2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/* Second level of nesting...
-^^^
-SELECT 'deeper nest' as n2
-
-
--- !query
+SELECT 'deeper nest' as n2;
 /* Third level of nesting...
-SELECT 'deepest nest' as n3
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
-
-== SQL ==
-/* Third level of nesting...
-^^^
-SELECT 'deepest nest' as n3
-
-
--- !query
+SELECT 'deepest nest' as n3;
 */
 Hoo boy. Still two deep...
 */
@@ -170,11 +73,27 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-mismatched input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)
+mismatched input ''embedded'' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 6, pos 34)
 
 == SQL ==
+/*
+SELECT 'trailing' as x1; -- inside block comment
+*/
+
+/* This block comment surrounds a query which itself has a block comment...
+SELECT /* embedded single line */ 'embedded' AS x2;
+----------------------------------^^^
+*/
+
+SELECT -- continued after the following block comments...
+/* Deeply nested comment.
+   This includes a single apostrophe to make sure we aren't decoding this part as a string.
+SELECT 'deep nest' AS n1;
+/* Second level of nesting...
+SELECT 'deeper nest' as n2;
+/* Third level of nesting...
+SELECT 'deepest nest' as n3;
 */
-^^^
 Hoo boy. Still two deep...
 */
 Now just one deep...
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 6b9e5bbd3c961..da4727f6a98cb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -19,7 +19,9 @@ package org.apache.spark.sql
 
 import java.io.File
 import java.util.{Locale, TimeZone}
+import java.util.regex.Pattern
 
+import scala.collection.mutable.{ArrayBuffer, HashMap}
 import scala.util.control.NonFatal
 
 import org.apache.spark.{SparkConf, SparkException}
@@ -62,7 +64,12 @@ import org.apache.spark.tags.ExtendedSQLTest
  * }}}
  *
  * The format for input files is simple:
- *  1. A list of SQL queries separated by semicolon.
+ *  1. A list of SQL queries separated by semicolons by default. If the semicolon cannot effectively
+ *     separate the SQL queries in the test file(e.g. bracketed comments), please use
+ *     --QUERY-DELIMITER-START and --QUERY-DELIMITER-END. Lines starting with
+ *     --QUERY-DELIMITER-START and --QUERY-DELIMITER-END represent the beginning and end of a query,
+ *     respectively. Code that is not surrounded by lines that begin with --QUERY-DELIMITER-START
+ *     and --QUERY-DELIMITER-END is still separated by semicolons.
  *  2. Lines starting with -- are treated as comments and ignored.
  *  3. Lines starting with --SET are used to specify the configs when running this testing file. You
  *     can set multiple configs in one --SET, using comma to separate them. Or you can use multiple
@@ -246,9 +253,15 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
 
   /** Run a test case. */
   protected def runTest(testCase: TestCase): Unit = {
+    def splitWithSemicolon(seq: Seq[String]) = {
+      seq.mkString("\n").split("(?<=[^\\\\]);")
+    }
     val input = fileToString(new File(testCase.inputFile))
 
-    val (comments, code) = input.split("\n").partition(_.trim.startsWith("--"))
+    val (comments, code) = input.split("\n").partition { line =>
+      val newLine = line.trim
+      newLine.startsWith("--") && !newLine.startsWith("--QUERY-DELIMITER")
+    }
 
     // If `--IMPORT` found, load code from another test case file, then insert them
     // into the head in this test.
@@ -261,10 +274,38 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       }
     }.flatten
 
+    val allCode = importedCode ++ code
+    val tempQueries = if (allCode.exists(_.trim.startsWith("--QUERY-DELIMITER"))) {
+      // Although the loop is heavy, only used for bracketed comments test.
+      val querys = new ArrayBuffer[String]
+      val otherCodes = new ArrayBuffer[String]
+      var tempStr = ""
+      var start = false
+      for (c <- allCode) {
+        if (c.trim.startsWith("--QUERY-DELIMITER-START")) {
+          start = true
+          querys ++= splitWithSemicolon(otherCodes.toSeq)
+          otherCodes.clear()
+        } else if (c.trim.startsWith("--QUERY-DELIMITER-END")) {
+          start = false
+          querys += s"\n${tempStr.stripSuffix(";")}"
+          tempStr = ""
+        } else if (start) {
+          tempStr += s"\n$c"
+        } else {
+          otherCodes += c
+        }
+      }
+      if (otherCodes.nonEmpty) {
+        querys ++= splitWithSemicolon(otherCodes.toSeq)
+      }
+      querys.toSeq
+    } else {
+      splitWithSemicolon(allCode).toSeq
+    }
+
     // List of SQL queries to run
-    // note: this is not a robust way to split queries using semicolon, but works for now.
-    val queries = (importedCode ++ code).mkString("\n").split("(?<=[^\\\\]);")
-      .map(_.trim).filter(_ != "").toSeq
+    val queries = tempQueries.map(_.trim).filter(_ != "").toSeq
       // Fix misplacement when comment is at the end of the query.
       .map(_.split("\n").filterNot(_.startsWith("--")).mkString("\n")).map(_.trim).filter(_ != "")
 

From 074712e329b347f769f8c009949c7845e95b3212 Mon Sep 17 00:00:00 2001
From: Liang Zhang <liang.zhang@databricks.com>
Date: Thu, 13 Feb 2020 23:55:13 +0800
Subject: [PATCH 0067/1280] [SPARK-30762] Add dtype=float32 support to
 vector_to_array UDF

### What changes were proposed in this pull request?
In this PR, we add a parameter in the python function vector_to_array(col) that allows converting to a column of arrays of Float (32bits) in scala, which would be mapped to a numpy array of dtype=float32.

### Why are the changes needed?
In the downstream ML training, using float32 instead of float64 (default) would allow a larger batch size, i.e., allow more data to fit in the memory.

### Does this PR introduce any user-facing change?
Yes.
Old: `vector_to_array()` only take one param
```
df.select(vector_to_array("colA"), ...)
```
New: `vector_to_array()` can take an additional optional param: `dtype` = "float32" (or "float64")
```
df.select(vector_to_array("colA", "float32"), ...)
```

### How was this patch tested?
Unit test in scala.
doctest in python.

Closes #27522 from liangz1/udf-float32.

Authored-by: Liang Zhang <liang.zhang@databricks.com>
Signed-off-by: WeichenXu <weichen.xu@databricks.com>
(cherry picked from commit 82d0aa37ae521231d8067e473c6ea79a118a115a)
Signed-off-by: WeichenXu <weichen.xu@databricks.com>
---
 .../scala/org/apache/spark/ml/functions.scala | 34 ++++++++++++++++---
 .../org/apache/spark/ml/FunctionsSuite.scala  | 33 +++++++++++++++---
 python/pyspark/ml/functions.py                | 27 ++++++++++++---
 3 files changed, 81 insertions(+), 13 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/functions.scala b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
index 1faf562c4d896..0f03231079866 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/functions.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml
 
 import org.apache.spark.annotation.Since
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{SparseVector, Vector}
 import org.apache.spark.mllib.linalg.{Vector => OldVector}
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.functions.udf
@@ -27,7 +27,6 @@ import org.apache.spark.sql.functions.udf
 @Since("3.0.0")
 object functions {
 // scalastyle:on
-
   private val vectorToArrayUdf = udf { vec: Any =>
     vec match {
       case v: Vector => v.toArray
@@ -39,10 +38,37 @@ object functions {
     }
   }.asNonNullable()
 
+  private val vectorToArrayFloatUdf = udf { vec: Any =>
+    vec match {
+      case v: SparseVector =>
+        val data = new Array[Float](v.size)
+        v.foreachActive { (index, value) => data(index) = value.toFloat }
+        data
+      case v: Vector => v.toArray.map(_.toFloat)
+      case v: OldVector => v.toArray.map(_.toFloat)
+      case v => throw new IllegalArgumentException(
+        "function vector_to_array requires a non-null input argument and input type must be " +
+        "`org.apache.spark.ml.linalg.Vector` or `org.apache.spark.mllib.linalg.Vector`, " +
+        s"but got ${ if (v == null) "null" else v.getClass.getName }.")
+    }
+  }.asNonNullable()
+
   /**
    * Converts a column of MLlib sparse/dense vectors into a column of dense arrays.
-   *
+   * @param v: the column of MLlib sparse/dense vectors
+   * @param dtype: the desired underlying data type in the returned array
+   * @return an array&lt;float&gt; if dtype is float32, or array&lt;double&gt; if dtype is float64
    * @since 3.0.0
    */
-  def vector_to_array(v: Column): Column = vectorToArrayUdf(v)
+  def vector_to_array(v: Column, dtype: String = "float64"): Column = {
+    if (dtype == "float64") {
+      vectorToArrayUdf(v)
+    } else if (dtype == "float32") {
+      vectorToArrayFloatUdf(v)
+    } else {
+      throw new IllegalArgumentException(
+        s"Unsupported dtype: $dtype. Valid values: float64, float32."
+      )
+    }
+  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
index 2f5062c689fc7..3dd9a7d8ec85d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
@@ -34,9 +34,8 @@ class FunctionsSuite extends MLTest {
       (Vectors.sparse(3, Seq((0, 2.0), (2, 3.0))), OldVectors.sparse(3, Seq((0, 20.0), (2, 30.0))))
     ).toDF("vec", "oldVec")
 
-      val result = df.select(vector_to_array('vec), vector_to_array('oldVec))
-      .as[(Seq[Double], Seq[Double])]
-      .collect().toSeq
+    val result = df.select(vector_to_array('vec), vector_to_array('oldVec))
+                   .as[(Seq[Double], Seq[Double])].collect().toSeq
 
     val expected = Seq(
       (Seq(1.0, 2.0, 3.0), Seq(10.0, 20.0, 30.0)),
@@ -50,7 +49,6 @@ class FunctionsSuite extends MLTest {
       (null, null, 0)
     ).toDF("vec", "oldVec", "label")
 
-
     for ((colName, valType) <- Seq(
         ("vec", "null"), ("oldVec", "null"), ("label", "java.lang.Integer"))) {
       val thrown1 = intercept[SparkException] {
@@ -61,5 +59,32 @@ class FunctionsSuite extends MLTest {
         "`org.apache.spark.ml.linalg.Vector` or `org.apache.spark.mllib.linalg.Vector`, " +
         s"but got ${valType}"))
     }
+
+    val df3 = Seq(
+      (Vectors.dense(1.0, 2.0, 3.0), OldVectors.dense(10.0, 20.0, 30.0)),
+      (Vectors.sparse(3, Seq((0, 2.0), (2, 3.0))), OldVectors.sparse(3, Seq((0, 20.0), (2, 30.0))))
+    ).toDF("vec", "oldVec")
+    val dfArrayFloat = df3.select(
+      vector_to_array('vec, dtype = "float32"), vector_to_array('oldVec, dtype = "float32"))
+
+    // Check values are correct
+    val result3 = dfArrayFloat.as[(Seq[Float], Seq[Float])].collect().toSeq
+
+    val expected3 = Seq(
+      (Seq(1.0, 2.0, 3.0), Seq(10.0, 20.0, 30.0)),
+      (Seq(2.0, 0.0, 3.0), Seq(20.0, 0.0, 30.0))
+    )
+    assert(result3 === expected3)
+
+    // Check data types are correct
+    assert(dfArrayFloat.schema.simpleString ===
+      "struct<UDF(vec):array<float>,UDF(oldVec):array<float>>")
+
+    val thrown2 = intercept[IllegalArgumentException] {
+      df3.select(
+        vector_to_array('vec, dtype = "float16"), vector_to_array('oldVec, dtype = "float16"))
+    }
+    assert(thrown2.getMessage.contains(
+      s"Unsupported dtype: float16. Valid values: float64, float32."))
   }
 }
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index 2b4d8ddcd00a8..ec164f34bc4db 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -19,10 +19,15 @@
 from pyspark.sql.column import Column, _to_java_column
 
 
-@since(3.0)
-def vector_to_array(col):
+@since("3.0.0")
+def vector_to_array(col, dtype="float64"):
     """
     Converts a column of MLlib sparse/dense vectors into a column of dense arrays.
+    :param col: A string of the column name or a Column
+    :param dtype: The data type of the output array. Valid values: "float64" or "float32".
+    :return: The converted column of dense arrays.
+
+    .. versionadded:: 3.0.0
 
     >>> from pyspark.ml.linalg import Vectors
     >>> from pyspark.ml.functions import vector_to_array
@@ -32,14 +37,26 @@ def vector_to_array(col):
     ...     (Vectors.sparse(3, [(0, 2.0), (2, 3.0)]),
     ...      OldVectors.sparse(3, [(0, 20.0), (2, 30.0)]))],
     ...     ["vec", "oldVec"])
-    >>> df.select(vector_to_array("vec").alias("vec"),
-    ...           vector_to_array("oldVec").alias("oldVec")).collect()
+    >>> df1 = df.select(vector_to_array("vec").alias("vec"),
+    ...                 vector_to_array("oldVec").alias("oldVec"))
+    >>> df1.collect()
+    [Row(vec=[1.0, 2.0, 3.0], oldVec=[10.0, 20.0, 30.0]),
+     Row(vec=[2.0, 0.0, 3.0], oldVec=[20.0, 0.0, 30.0])]
+    >>> df2 = df.select(vector_to_array("vec", "float32").alias("vec"),
+    ...                 vector_to_array("oldVec", "float32").alias("oldVec"))
+    >>> df2.collect()
     [Row(vec=[1.0, 2.0, 3.0], oldVec=[10.0, 20.0, 30.0]),
      Row(vec=[2.0, 0.0, 3.0], oldVec=[20.0, 0.0, 30.0])]
+    >>> df1.schema.fields
+    [StructField(vec,ArrayType(DoubleType,false),false),
+    StructField(oldVec,ArrayType(DoubleType,false),false)]
+    >>> df2.schema.fields
+    [StructField(vec,ArrayType(FloatType,false),false),
+    StructField(oldVec,ArrayType(FloatType,false),false)]
     """
     sc = SparkContext._active_spark_context
     return Column(
-        sc._jvm.org.apache.spark.ml.functions.vector_to_array(_to_java_column(col)))
+        sc._jvm.org.apache.spark.ml.functions.vector_to_array(_to_java_column(col), dtype))
 
 
 def _test():

From 82258aa4794d15bfe9cfd2b4bced790ef1d35e45 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 13 Feb 2020 10:53:55 -0800
Subject: [PATCH 0068/1280] [SPARK-30703][SQL][DOCS] Add a document for the
 ANSI mode

### What changes were proposed in this pull request?

This pr intends to add a document for the ANSI mode;

<img width="600" alt="Screen Shot 2020-02-13 at 8 08 52" src="https://user-images.githubusercontent.com/692303/74386041-5934f780-4e38-11ea-8162-26e524e11c65.png">
<img width="600" alt="Screen Shot 2020-02-13 at 8 09 13" src="https://user-images.githubusercontent.com/692303/74386040-589c6100-4e38-11ea-8a64-899788eaf55f.png">
<img width="600" alt="Screen Shot 2020-02-13 at 8 09 26" src="https://user-images.githubusercontent.com/692303/74386039-5803ca80-4e38-11ea-949f-049208d2203d.png">
<img width="600" alt="Screen Shot 2020-02-13 at 8 09 38" src="https://user-images.githubusercontent.com/692303/74386036-563a0700-4e38-11ea-9ec3-87a8f6771cf0.png">

### Why are the changes needed?

For better document coverage and usability.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A

Closes #27489 from maropu/SPARK-30703.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 3c4044ea77fe3b1268b52744cd4f1ae61f17a9a8)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 docs/_data/menu-sql.yaml                      |  11 +-
 ...keywords.md => sql-ref-ansi-compliance.md} | 125 +++++++++++++++++-
 docs/sql-ref-arithmetic-ops.md                |  22 ---
 3 files changed, 132 insertions(+), 26 deletions(-)
 rename docs/{sql-keywords.md => sql-ref-ansi-compliance.md} (82%)
 delete mode 100644 docs/sql-ref-arithmetic-ops.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 241ec399d7bd5..1e343f630f88e 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -80,6 +80,15 @@
       url: sql-ref-null-semantics.html
     - text: NaN Semantics
       url: sql-ref-nan-semantics.html
+    - text: ANSI Compliance
+      url: sql-ref-ansi-compliance.html
+      subitems:
+        - text: Arithmetic Operations
+          url: sql-ref-ansi-compliance.html#arithmetic-operations
+        - text: Type Conversion
+          url: sql-ref-ansi-compliance.html#type-conversion
+        - text: SQL Keywords
+          url: sql-ref-ansi-compliance.html#sql-keywords
     - text: SQL Syntax
       url: sql-ref-syntax.html
       subitems:
@@ -214,5 +223,3 @@
                   url: sql-ref-syntax-aux-resource-mgmt-list-file.html
                 - text: LIST JAR
                   url: sql-ref-syntax-aux-resource-mgmt-list-jar.html
-    - text: Arithmetic operations
-      url: sql-ref-arithmetic-ops.html
diff --git a/docs/sql-keywords.md b/docs/sql-ref-ansi-compliance.md
similarity index 82%
rename from docs/sql-keywords.md
rename to docs/sql-ref-ansi-compliance.md
index 9e4a3c54100c6..d02383518b967 100644
--- a/docs/sql-keywords.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Spark SQL Keywords
-displayTitle: Spark SQL Keywords
+title: ANSI Compliance
+displayTitle: ANSI Compliance
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -19,6 +19,127 @@ license: |
   limitations under the License.
 ---
 
+Spark SQL has two options to comply with the SQL standard: `spark.sql.ansi.enabled` and `spark.sql.storeAssignmentPolicy` (See a table below for details).
+When `spark.sql.ansi.enabled` is set to `true`, Spark SQL follows the standard in basic behaviours (e.g., arithmetic operations, type conversion, and SQL parsing).
+Moreover, Spark SQL has an independent option to control implicit casting behaviours when inserting rows in a table.
+The casting behaviours are defined as store assignment rules in the standard.
+When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with the ANSI store assignment rules.
+
+<table class="table">
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr>
+  <td><code>spark.sql.ansi.enabled</code></td>
+  <td>false</td>
+  <td>
+    When true, Spark tries to conform to the ANSI SQL specification:
+    1. Spark will throw a runtime exception if an overflow occurs in any operation on integral/decimal field.
+    2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in the SQL parser.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.sql.storeAssignmentPolicy</code></td>
+  <td>ANSI</td>
+  <td>
+    When inserting a value into a column with different data type, Spark will perform type coercion.
+    Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy,
+    Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL.
+    It disallows certain unreasonable type conversions such as converting string to int or double to boolean.
+    With legacy policy, Spark allows the type coercion as long as it is a valid Cast, which is very loose.
+    e.g. converting string to int or double to boolean is allowed.
+    It is also the only behavior in Spark 2.x and it is compatible with Hive.
+    With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion,
+    e.g. converting double to int or decimal to double is not allowed.
+  </td>
+</tr>
+</table>
+
+The following subsections present behaviour changes in arithmetic operations, type conversions, and SQL parsing when the ANSI mode enabled.
+
+### Arithmetic Operations
+
+In Spark SQL, arithmetic operations performed on numeric types (with the exception of decimal) are not checked for overflows by default.
+This means that in case an operation causes overflows, the result is the same that the same operation returns in a Java/Scala program (e.g., if the sum of 2 integers is higher than the maximum value representable, the result is a negative number).
+On the other hand, Spark SQL returns null for decimal overflows.
+When `spark.sql.ansi.enabled` is set to `true` and an overflow occurs in numeric and interval arithmetic operations, it throws an arithmetic exception at runtime.
+
+{% highlight sql %}
+-- `spark.sql.ansi.enabled=true`
+SELECT 2147483647 + 1;
+
+  java.lang.ArithmeticException: integer overflow
+
+-- `spark.sql.ansi.enabled=false`
+SELECT 2147483647 + 1;
+
+  +----------------+
+  |(2147483647 + 1)|
+  +----------------+
+  |     -2147483648|
+  +----------------+
+
+{% endhighlight %}
+
+### Type Conversion
+
+Spark SQL has three kinds of type conversions: explicit casting, type coercion, and store assignment casting.
+When `spark.sql.ansi.enabled` is set to `true`, explicit casting by `CAST` syntax throws a runtime exception for illegal cast patterns defined in the standard, e.g. casts from a string to an integer.
+On the other hand, `INSERT INTO` syntax throws an analysis exception when the ANSI mode enabled via `spark.sql.storeAssignmentPolicy=ANSI`.
+
+Currently, the ANSI mode affects explicit casting and assignment casting only.
+In future releases, the behaviour of type coercion might change along with the other two type conversion rules.
+
+{% highlight sql %}
+-- Examples of explicit casting
+
+-- `spark.sql.ansi.enabled=true`
+SELECT CAST('a' AS INT);
+
+  java.lang.NumberFormatException: invalid input syntax for type numeric: a
+
+SELECT CAST(2147483648L AS INT);
+
+  java.lang.ArithmeticException: Casting 2147483648 to int causes overflow
+
+-- `spark.sql.ansi.enabled=false` (This is a default behaviour)
+SELECT CAST('a' AS INT);
+
+  +--------------+
+  |CAST(a AS INT)|
+  +--------------+
+  |          null|
+  +--------------+
+
+SELECT CAST(2147483648L AS INT);
+
+  +-----------------------+
+  |CAST(2147483648 AS INT)|
+  +-----------------------+
+  |            -2147483648|
+  +-----------------------+
+
+-- Examples of store assignment rules
+CREATE TABLE t (v INT);
+
+-- `spark.sql.storeAssignmentPolicy=ANSI`
+INSERT INTO t VALUES ('1');
+
+  org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
+  - Cannot safely cast 'v': StringType to IntegerType;
+
+-- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
+INSERT INTO t VALUES ('1');
+SELECT * FROM t;
+
+  +---+
+  |  v|
+  +---+
+  |  1|
+  +---+
+
+{% endhighlight %}
+
+### SQL Keywords
+
 When `spark.sql.ansi.enabled` is true, Spark SQL will use the ANSI mode parser.
 In this mode, Spark SQL has two kinds of keywords:
 * Reserved keywords: Keywords that are reserved and can't be used as identifiers for table, view, column, function, alias, etc.
diff --git a/docs/sql-ref-arithmetic-ops.md b/docs/sql-ref-arithmetic-ops.md
deleted file mode 100644
index 7bc8ffe31c990..0000000000000
--- a/docs/sql-ref-arithmetic-ops.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Arithmetic Operations
-displayTitle: Arithmetic Operations
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Operations performed on numeric types (with the exception of decimal) are not checked for overflow. This means that in case an operation causes an overflow, the result is the same that the same operation returns in a Java/Scala program (eg. if the sum of 2 integers is higher than the maximum value representable, the result is a negative number).

From 78bd4b34ca7e0834b8b3878cd74b3f59b46b4f90 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 13 Feb 2020 20:09:24 +0100
Subject: [PATCH 0069/1280] [SPARK-30751][SQL] Combine the skewed readers into
 one in AQE skew join optimizations

<!--
Thanks for sending a pull request!  Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://spark.apache.org/contributing.html
  2. Ensure you have added or run the appropriate tests for your PR: https://spark.apache.org/developer-tools.html
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][SPARK-XXXX] Your PR title ...'.
  4. Be sure to keep the PR description updated to reflect all changes.
  5. Please write your PR title to summarize what this PR proposes.
  6. If possible, provide a concise example to reproduce the issue for a faster review.
-->

### What changes were proposed in this pull request?
<!--
Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
  1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
  2. If you fix some SQL features, you can provide some references of other DBMSes.
  3. If there is design documentation, please add the link.
  4. If there is a discussion in the mailing list, please add the link.
-->
This is a followup of https://github.com/apache/spark/pull/26434

This PR use one special shuffle reader for skew join, so that we only have one join after optimization. In order to do that, this PR
1. add a very general `CustomShuffledRowRDD` which support all kind of partition arrangement.
2. move the logic of coalescing shuffle partitions to a util function, and call it during skew join optimization, to totally decouple with the `ReduceNumShufflePartitions` rule. It's too complicated to interfere skew join with `ReduceNumShufflePartitions`, as you need to consider the size of split partitions which don't respect target size already.

### Why are the changes needed?
<!--
Please clarify why the changes are needed. For instance,
  1. If you propose a new API, clarify the use case for a new API.
  2. If you fix a bug, you can clarify why it is a bug.
-->
The current skew join optimization has a serious performance issue: the size of the query plan depends on the number and size of skewed partitions.

### Does this PR introduce any user-facing change?
<!--
If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
If no, write 'No'.
-->
no

### How was this patch tested?
<!--
If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
If tests were not added, please describe why they were not added and/or why it was difficult to add.
-->
existing tests

test UI manually:
![image](https://user-images.githubusercontent.com/3182036/74357390-cfb30480-4dfa-11ea-83f6-825d1b9379ca.png)

explain output
```
AdaptiveSparkPlan(isFinalPlan=true)
+- OverwriteByExpression org.apache.spark.sql.execution.datasources.noop.NoopTable$403a2ed5, [AlwaysTrue()], org.apache.spark.sql.util.CaseInsensitiveStringMap1f
   +- *(5) SortMergeJoin(skew=true) [key1#2L], [key2#6L], Inner
      :- *(3) Sort [key1#2L ASC NULLS FIRST], false, 0
      :  +- SkewJoinShuffleReader 2 skewed partitions with size(max=5 KB, min=5 KB, avg=5 KB)
      :     +- ShuffleQueryStage 0
      :        +- Exchange hashpartitioning(key1#2L, 200), true, [id=#53]
      :           +- *(1) Project [(id#0L % 2) AS key1#2L]
      :              +- *(1) Filter isnotnull((id#0L % 2))
      :                 +- *(1) Range (0, 100000, step=1, splits=6)
      +- *(4) Sort [key2#6L ASC NULLS FIRST], false, 0
         +- SkewJoinShuffleReader 2 skewed partitions with size(max=5 KB, min=5 KB, avg=5 KB)
            +- ShuffleQueryStage 1
               +- Exchange hashpartitioning(key2#6L, 200), true, [id=#64]
                  +- *(2) Project [((id#4L % 2) + 1) AS key2#6L]
                     +- *(2) Filter isnotnull(((id#4L % 2) + 1))
                        +- *(2) Range (0, 100000, step=1, splits=6)
```

Closes #27493 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit a4ceea6868002b88161517b14b94a2006be8af1b)
Signed-off-by: herman <herman@databricks.com>
---
 .../spark/sql/execution/ShuffledRowRDD.scala  |  23 +-
 .../adaptive/CustomShuffledRowRDD.scala       | 113 +++++++
 .../adaptive/OptimizeLocalShuffleReader.scala |   2 +-
 .../adaptive/OptimizeSkewedJoin.scala         | 276 +++++++++++-------
 .../adaptive/ReduceNumShufflePartitions.scala | 157 ++--------
 .../adaptive/ShufflePartitionsCoalescer.scala | 112 +++++++
 .../adaptive/SkewedShuffledRowRDD.scala       |  78 -----
 .../exchange/ShuffleExchangeExec.scala        |  21 +-
 .../execution/joins/SortMergeJoinExec.scala   |  13 +-
 .../ReduceNumShufflePartitionsSuite.scala     | 210 +------------
 .../ShufflePartitionsCoalescerSuite.scala     | 220 ++++++++++++++
 .../adaptive/AdaptiveQueryExecSuite.scala     | 219 +++++---------
 12 files changed, 741 insertions(+), 703 deletions(-)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
index efa493923ccc1..4c19f95796d04 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
@@ -116,7 +116,7 @@ class CoalescedPartitioner(val parent: Partitioner, val partitionStartIndices: A
 class ShuffledRowRDD(
     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
     metrics: Map[String, SQLMetric],
-    specifiedPartitionIndices: Option[Array[(Int, Int)]] = None)
+    specifiedPartitionStartIndices: Option[Array[Int]] = None)
   extends RDD[InternalRow](dependency.rdd.context, Nil) {
 
   if (SQLConf.get.fetchShuffleBlocksInBatchEnabled) {
@@ -126,8 +126,8 @@ class ShuffledRowRDD(
 
   private[this] val numPreShufflePartitions = dependency.partitioner.numPartitions
 
-  private[this] val partitionStartIndices: Array[Int] = specifiedPartitionIndices match {
-    case Some(indices) => indices.map(_._1)
+  private[this] val partitionStartIndices: Array[Int] = specifiedPartitionStartIndices match {
+    case Some(indices) => indices
     case None =>
       // When specifiedPartitionStartIndices is not defined, every post-shuffle partition
       // corresponds to a pre-shuffle partition.
@@ -142,15 +142,16 @@ class ShuffledRowRDD(
   override val partitioner: Option[Partitioner] = Some(part)
 
   override def getPartitions: Array[Partition] = {
-    specifiedPartitionIndices match {
-      case Some(indices) =>
-        Array.tabulate[Partition](indices.length) { i =>
-          new ShuffledRowRDDPartition(i, indices(i)._1, indices(i)._2)
-        }
-      case None =>
-        Array.tabulate[Partition](numPreShufflePartitions) { i =>
-          new ShuffledRowRDDPartition(i, i, i + 1)
+    assert(partitionStartIndices.length == part.numPartitions)
+    Array.tabulate[Partition](partitionStartIndices.length) { i =>
+      val startIndex = partitionStartIndices(i)
+      val endIndex =
+        if (i < partitionStartIndices.length - 1) {
+          partitionStartIndices(i + 1)
+        } else {
+          numPreShufflePartitions
         }
+      new ShuffledRowRDDPartition(i, startIndex, endIndex)
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala
new file mode 100644
index 0000000000000..5aba57443d632
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.{Dependency, MapOutputTrackerMaster, Partition, ShuffleDependency, SparkEnv, TaskContext}
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
+
+sealed trait ShufflePartitionSpec
+
+// A partition that reads data of one reducer.
+case class SinglePartitionSpec(reducerIndex: Int) extends ShufflePartitionSpec
+
+// A partition that reads data of multiple reducers, from `startReducerIndex` (inclusive) to
+// `endReducerIndex` (exclusive).
+case class CoalescedPartitionSpec(
+    startReducerIndex: Int, endReducerIndex: Int) extends ShufflePartitionSpec
+
+// A partition that reads partial data of one reducer, from `startMapIndex` (inclusive) to
+// `endMapIndex` (exclusive).
+case class PartialPartitionSpec(
+    reducerIndex: Int, startMapIndex: Int, endMapIndex: Int) extends ShufflePartitionSpec
+
+private final case class CustomShufflePartition(
+    index: Int, spec: ShufflePartitionSpec) extends Partition
+
+// TODO: merge this with `ShuffledRowRDD`, and replace `LocalShuffledRowRDD` with this RDD.
+class CustomShuffledRowRDD(
+    var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
+    metrics: Map[String, SQLMetric],
+    partitionSpecs: Array[ShufflePartitionSpec])
+  extends RDD[InternalRow](dependency.rdd.context, Nil) {
+
+  override def getDependencies: Seq[Dependency[_]] = List(dependency)
+
+  override def clearDependencies() {
+    super.clearDependencies()
+    dependency = null
+  }
+
+  override def getPartitions: Array[Partition] = {
+    Array.tabulate[Partition](partitionSpecs.length) { i =>
+      CustomShufflePartition(i, partitionSpecs(i))
+    }
+  }
+
+  override def getPreferredLocations(partition: Partition): Seq[String] = {
+    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
+    partition.asInstanceOf[CustomShufflePartition].spec match {
+      case SinglePartitionSpec(reducerIndex) =>
+        tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
+
+      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+        startReducerIndex.until(endReducerIndex).flatMap { reducerIndex =>
+          tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
+        }
+
+      case PartialPartitionSpec(_, startMapIndex, endMapIndex) =>
+        tracker.getMapLocation(dependency, startMapIndex, endMapIndex)
+    }
+  }
+
+  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
+    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
+    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
+    // as well as the `tempMetrics` for basic shuffle metrics.
+    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
+    val reader = split.asInstanceOf[CustomShufflePartition].spec match {
+      case SinglePartitionSpec(reducerIndex) =>
+        SparkEnv.get.shuffleManager.getReader(
+          dependency.shuffleHandle,
+          reducerIndex,
+          reducerIndex + 1,
+          context,
+          sqlMetricsReporter)
+
+      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+        SparkEnv.get.shuffleManager.getReader(
+          dependency.shuffleHandle,
+          startReducerIndex,
+          endReducerIndex,
+          context,
+          sqlMetricsReporter)
+
+      case PartialPartitionSpec(reducerIndex, startMapIndex, endMapIndex) =>
+        SparkEnv.get.shuffleManager.getReaderForRange(
+          dependency.shuffleHandle,
+          startMapIndex,
+          endMapIndex,
+          reducerIndex,
+          reducerIndex + 1,
+          context,
+          sqlMetricsReporter)
+    }
+    reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index a8d8f358ab660..e95441e28aafe 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -71,7 +71,7 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
     plan match {
       case c @ CoalescedShuffleReaderExec(s: ShuffleQueryStageExec, _) =>
         LocalShuffleReaderExec(
-          s, getPartitionStartIndices(s, Some(c.partitionIndices.length)))
+          s, getPartitionStartIndices(s, Some(c.partitionStartIndices.length)))
       case s: ShuffleQueryStageExec =>
         LocalShuffleReaderExec(s, getPartitionStartIndices(s, None))
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 74b7fbd317fc8..a716497c274b8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.execution.adaptive
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 
+import org.apache.commons.io.FileUtils
+
 import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -44,11 +46,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    * partition size * spark.sql.adaptive.skewedPartitionFactor and also larger than
    * spark.sql.adaptive.skewedPartitionSizeThreshold.
    */
-  private def isSkewed(
-      stats: MapOutputStatistics,
-      partitionId: Int,
-      medianSize: Long): Boolean = {
-    val size = stats.bytesByPartitionId(partitionId)
+  private def isSkewed(size: Long, medianSize: Long): Boolean = {
     size > medianSize * conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR) &&
       size > conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD)
   }
@@ -108,12 +106,12 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     stage.resultOption.get.asInstanceOf[MapOutputStatistics]
   }
 
-  private def supportSplitOnLeftPartition(joinType: JoinType) = {
+  private def canSplitLeftSide(joinType: JoinType) = {
     joinType == Inner || joinType == Cross || joinType == LeftSemi ||
       joinType == LeftAnti || joinType == LeftOuter
   }
 
-  private def supportSplitOnRightPartition(joinType: JoinType) = {
+  private def canSplitRightSide(joinType: JoinType) = {
     joinType == Inner || joinType == Cross || joinType == RightOuter
   }
 
@@ -130,17 +128,18 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    * 1. Check whether the shuffle partition is skewed based on the median size
    *    and the skewed partition threshold in origin smj.
    * 2. Assuming partition0 is skewed in left side, and it has 5 mappers (Map0, Map1...Map4).
-   *    And we will split the 5 Mappers into 3 mapper ranges [(Map0, Map1), (Map2, Map3), (Map4)]
+   *    And we may split the 5 Mappers into 3 mapper ranges [(Map0, Map1), (Map2, Map3), (Map4)]
    *    based on the map size and the max split number.
-   * 3. Create the 3 smjs with separately reading the above mapper ranges and then join with
-   *    the Partition0 in right side.
-   * 4. Finally union the above 3 split smjs and the origin smj.
+   * 3. Wrap the join left child with a special shuffle reader that reads each mapper range with one
+   *    task, so total 3 tasks.
+   * 4. Wrap the join right child with a special shuffle reader that reads partition0 3 times by
+   *    3 tasks separately.
    */
   def optimizeSkewJoin(plan: SparkPlan): SparkPlan = plan.transformUp {
-    case smj @ SortMergeJoinExec(leftKeys, rightKeys, joinType, condition,
+    case smj @ SortMergeJoinExec(_, _, joinType, _,
         s1 @ SortExec(_, _, left: ShuffleQueryStageExec, _),
         s2 @ SortExec(_, _, right: ShuffleQueryStageExec, _), _)
-      if (supportedJoinTypes.contains(joinType)) =>
+        if supportedJoinTypes.contains(joinType) =>
       val leftStats = getStatistics(left)
       val rightStats = getStatistics(right)
       val numPartitions = leftStats.bytesByPartitionId.length
@@ -155,61 +154,134 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
           |Right side partition size:
           |${getSizeInfo(rightMedSize, rightStats.bytesByPartitionId.max)}
         """.stripMargin)
+      val canSplitLeft = canSplitLeftSide(joinType)
+      val canSplitRight = canSplitRightSide(joinType)
+
+      val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
+      val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
+      // This is used to delay the creation of non-skew partitions so that we can potentially
+      // coalesce them like `ReduceNumShufflePartitions` does.
+      val nonSkewPartitionIndices = mutable.ArrayBuffer.empty[Int]
+      val leftSkewDesc = new SkewDesc
+      val rightSkewDesc = new SkewDesc
+      for (partitionIndex <- 0 until numPartitions) {
+        val leftSize = leftStats.bytesByPartitionId(partitionIndex)
+        val isLeftSkew = isSkewed(leftSize, leftMedSize) && canSplitLeft
+        val rightSize = rightStats.bytesByPartitionId(partitionIndex)
+        val isRightSkew = isSkewed(rightSize, rightMedSize) && canSplitRight
+        if (isLeftSkew || isRightSkew) {
+          if (nonSkewPartitionIndices.nonEmpty) {
+            // As soon as we see a skew, we'll "flush" out unhandled non-skew partitions.
+            createNonSkewPartitions(leftStats, rightStats, nonSkewPartitionIndices).foreach { p =>
+              leftSidePartitions += p
+              rightSidePartitions += p
+            }
+            nonSkewPartitionIndices.clear()
+          }
 
-      val skewedPartitions = mutable.HashSet[Int]()
-      val subJoins = mutable.ArrayBuffer[SparkPlan]()
-      for (partitionId <- 0 until numPartitions) {
-        val isLeftSkew = isSkewed(leftStats, partitionId, leftMedSize)
-        val isRightSkew = isSkewed(rightStats, partitionId, rightMedSize)
-        val leftMapIdStartIndices = if (isLeftSkew && supportSplitOnLeftPartition(joinType)) {
-          getMapStartIndices(left, partitionId)
-        } else {
-          Array(0)
-        }
-        val rightMapIdStartIndices = if (isRightSkew && supportSplitOnRightPartition(joinType)) {
-          getMapStartIndices(right, partitionId)
-        } else {
-          Array(0)
-        }
+          val leftParts = if (isLeftSkew) {
+            leftSkewDesc.addPartitionSize(leftSize)
+            createSkewPartitions(
+              partitionIndex,
+              getMapStartIndices(left, partitionIndex),
+              getNumMappers(left))
+          } else {
+            Seq(SinglePartitionSpec(partitionIndex))
+          }
 
-        if (leftMapIdStartIndices.length > 1 || rightMapIdStartIndices.length > 1) {
-          skewedPartitions += partitionId
-          for (i <- 0 until leftMapIdStartIndices.length;
-               j <- 0 until rightMapIdStartIndices.length) {
-            val leftEndMapId = if (i == leftMapIdStartIndices.length - 1) {
-              getNumMappers(left)
-            } else {
-              leftMapIdStartIndices(i + 1)
-            }
-            val rightEndMapId = if (j == rightMapIdStartIndices.length - 1) {
-              getNumMappers(right)
-            } else {
-              rightMapIdStartIndices(j + 1)
+          val rightParts = if (isRightSkew) {
+            rightSkewDesc.addPartitionSize(rightSize)
+            createSkewPartitions(
+              partitionIndex,
+              getMapStartIndices(right, partitionIndex),
+              getNumMappers(right))
+          } else {
+            Seq(SinglePartitionSpec(partitionIndex))
+          }
+
+          for {
+            leftSidePartition <- leftParts
+            rightSidePartition <- rightParts
+          } {
+            leftSidePartitions += leftSidePartition
+            rightSidePartitions += rightSidePartition
+          }
+        } else {
+          // Add to `nonSkewPartitionIndices` first, and add real partitions later, in case we can
+          // coalesce the non-skew partitions.
+          nonSkewPartitionIndices += partitionIndex
+          // If this is the last partition, add real partition immediately.
+          if (partitionIndex == numPartitions - 1) {
+            createNonSkewPartitions(leftStats, rightStats, nonSkewPartitionIndices).foreach { p =>
+              leftSidePartitions += p
+              rightSidePartitions += p
             }
-            // TODO: we may can optimize the sort merge join to broad cast join after
-            //       obtaining the raw data size of per partition,
-            val leftSkewedReader = SkewedPartitionReaderExec(
-              left, partitionId, leftMapIdStartIndices(i), leftEndMapId)
-            val rightSkewedReader = SkewedPartitionReaderExec(right, partitionId,
-              rightMapIdStartIndices(j), rightEndMapId)
-            subJoins += SortMergeJoinExec(leftKeys, rightKeys, joinType, condition,
-              s1.copy(child = leftSkewedReader), s2.copy(child = rightSkewedReader), true)
+            nonSkewPartitionIndices.clear()
           }
         }
       }
-      logDebug(s"number of skewed partitions is ${skewedPartitions.size}")
-      if (skewedPartitions.nonEmpty) {
-        val optimizedSmj = smj.copy(
-          left = s1.copy(child = PartialShuffleReaderExec(left, skewedPartitions.toSet)),
-          right = s2.copy(child = PartialShuffleReaderExec(right, skewedPartitions.toSet)),
-          isPartial = true)
-        subJoins += optimizedSmj
-        UnionExec(subJoins)
+
+      logDebug("number of skewed partitions: " +
+        s"left ${leftSkewDesc.numPartitions}, right ${rightSkewDesc.numPartitions}")
+      if (leftSkewDesc.numPartitions > 0 || rightSkewDesc.numPartitions > 0) {
+        val newLeft = SkewJoinShuffleReaderExec(
+          left, leftSidePartitions.toArray, leftSkewDesc.toString)
+        val newRight = SkewJoinShuffleReaderExec(
+          right, rightSidePartitions.toArray, rightSkewDesc.toString)
+        smj.copy(
+          left = s1.copy(child = newLeft), right = s2.copy(child = newRight), isSkewJoin = true)
       } else {
         smj
       }
   }
 
+  private def createNonSkewPartitions(
+      leftStats: MapOutputStatistics,
+      rightStats: MapOutputStatistics,
+      nonSkewPartitionIndices: Seq[Int]): Seq[ShufflePartitionSpec] = {
+    assert(nonSkewPartitionIndices.nonEmpty)
+    if (nonSkewPartitionIndices.length == 1) {
+      Seq(SinglePartitionSpec(nonSkewPartitionIndices.head))
+    } else {
+      val startIndices = ShufflePartitionsCoalescer.coalescePartitions(
+        Array(leftStats, rightStats),
+        firstPartitionIndex = nonSkewPartitionIndices.head,
+        // `lastPartitionIndex` is exclusive.
+        lastPartitionIndex = nonSkewPartitionIndices.last + 1,
+        advisoryTargetSize = conf.targetPostShuffleInputSize)
+      startIndices.indices.map { i =>
+        val startIndex = startIndices(i)
+        val endIndex = if (i == startIndices.length - 1) {
+          // `endIndex` is exclusive.
+          nonSkewPartitionIndices.last + 1
+        } else {
+          startIndices(i + 1)
+        }
+        // Do not create `CoalescedPartitionSpec` if only need to read a singe partition.
+        if (startIndex + 1 == endIndex) {
+          SinglePartitionSpec(startIndex)
+        } else {
+          CoalescedPartitionSpec(startIndex, endIndex)
+        }
+      }
+    }
+  }
+
+  private def createSkewPartitions(
+      reducerIndex: Int,
+      mapStartIndices: Array[Int],
+      numMappers: Int): Seq[PartialPartitionSpec] = {
+    mapStartIndices.indices.map { i =>
+      val startMapIndex = mapStartIndices(i)
+      val endMapIndex = if (i == mapStartIndices.length - 1) {
+        numMappers
+      } else {
+        mapStartIndices(i + 1)
+      }
+      PartialPartitionSpec(reducerIndex, startMapIndex, endMapIndex)
+    }
+  }
+
   override def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_JOIN_ENABLED)) {
       return plan
@@ -248,79 +320,69 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   }
 }
 
-/**
- * A wrapper of shuffle query stage, which submits one reduce task to read a single
- * shuffle partition 'partitionIndex' produced by the mappers in range [startMapIndex, endMapIndex).
- * This is used to increase the parallelism when reading skewed partitions.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange
- *              node during canonicalization.
- * @param partitionIndex The pre shuffle partition index.
- * @param startMapIndex The start map index.
- * @param endMapIndex The end map index.
- */
-case class SkewedPartitionReaderExec(
-    child: QueryStageExec,
-    partitionIndex: Int,
-    startMapIndex: Int,
-    endMapIndex: Int) extends LeafExecNode {
+private class SkewDesc {
+  private[this] var numSkewedPartitions: Int = 0
+  private[this] var totalSize: Long = 0
+  private[this] var maxSize: Long = 0
+  private[this] var minSize: Long = 0
 
-  override def output: Seq[Attribute] = child.output
+  def numPartitions: Int = numSkewedPartitions
 
-  override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(1)
+  def addPartitionSize(size: Long): Unit = {
+    if (numSkewedPartitions == 0) {
+      maxSize = size
+      minSize = size
+    }
+    numSkewedPartitions += 1
+    totalSize += size
+    if (size > maxSize) maxSize = size
+    if (size < minSize) minSize = size
   }
-  private var cachedSkewedShuffleRDD: SkewedShuffledRowRDD = null
 
-  override def doExecute(): RDD[InternalRow] = {
-    if (cachedSkewedShuffleRDD == null) {
-      cachedSkewedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createSkewedShuffleRDD(partitionIndex, startMapIndex, endMapIndex)
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
+  override def toString: String = {
+    if (numSkewedPartitions == 0) {
+      "no skewed partition"
+    } else {
+      val maxSizeStr = FileUtils.byteCountToDisplaySize(maxSize)
+      val minSizeStr = FileUtils.byteCountToDisplaySize(minSize)
+      val avgSizeStr = FileUtils.byteCountToDisplaySize(totalSize / numSkewedPartitions)
+      s"$numSkewedPartitions skewed partitions with " +
+        s"size(max=$maxSizeStr, min=$minSizeStr, avg=$avgSizeStr)"
     }
-    cachedSkewedShuffleRDD
   }
 }
 
 /**
- * A wrapper of shuffle query stage, which skips some partitions when reading the shuffle blocks.
+ * A wrapper of shuffle query stage, which follows the given partition arrangement.
  *
  * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
  *              canonicalization.
- * @param excludedPartitions The partitions to skip when reading.
+ * @param partitionSpecs The partition specs that defines the arrangement.
+ * @param skewDesc The description of the skewed partitions.
  */
-case class PartialShuffleReaderExec(
-    child: QueryStageExec,
-    excludedPartitions: Set[Int]) extends UnaryExecNode {
+case class SkewJoinShuffleReaderExec(
+    child: SparkPlan,
+    partitionSpecs: Array[ShufflePartitionSpec],
+    skewDesc: String) extends UnaryExecNode {
 
   override def output: Seq[Attribute] = child.output
 
   override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(1)
+    UnknownPartitioning(partitionSpecs.length)
   }
 
-  private def shuffleExchange(): ShuffleExchangeExec = child match {
-    case stage: ShuffleQueryStageExec => stage.shuffle
-    case _ =>
-      throw new IllegalStateException("operating on canonicalization plan")
-  }
-
-  private def getPartitionIndexRanges(): Array[(Int, Int)] = {
-    val length = shuffleExchange().shuffleDependency.partitioner.numPartitions
-    (0 until length).filterNot(excludedPartitions.contains).map(i => (i, i + 1)).toArray
-  }
+  override def stringArgs: Iterator[Any] = Iterator(skewDesc)
 
   private var cachedShuffleRDD: RDD[InternalRow] = null
 
-  override def doExecute(): RDD[InternalRow] = {
+  override protected def doExecute(): RDD[InternalRow] = {
     if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = if (excludedPartitions.isEmpty) {
-        child.execute()
-      } else {
-        shuffleExchange().createShuffledRDD(Some(getPartitionIndexRanges()))
+      cachedShuffleRDD = child match {
+        case stage: ShuffleQueryStageExec =>
+          new CustomShuffledRowRDD(
+            stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs)
+        case _ =>
+          throw new IllegalStateException("operating on canonicalization plan")
       }
     }
     cachedShuffleRDD
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
index 2c50b638b4d45..5bbcb14e008d8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import scala.collection.mutable.{ArrayBuffer, HashSet}
-
 import org.apache.spark.MapOutputStatistics
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -29,24 +27,8 @@ import org.apache.spark.sql.execution.{ShuffledRowRDD, SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
- * A rule to adjust the post shuffle partitions based on the map output statistics.
- *
- * The strategy used to determine the number of post-shuffle partitions is described as follows.
- * To determine the number of post-shuffle partitions, we have a target input size for a
- * post-shuffle partition. Once we have size statistics of all pre-shuffle partitions, we will do
- * a pass of those statistics and pack pre-shuffle partitions with continuous indices to a single
- * post-shuffle partition until adding another pre-shuffle partition would cause the size of a
- * post-shuffle partition to be greater than the target size.
- *
- * For example, we have two stages with the following pre-shuffle partition size statistics:
- * stage 1: [100 MiB, 20 MiB, 100 MiB, 10MiB, 30 MiB]
- * stage 2: [10 MiB,  10 MiB, 70 MiB,  5 MiB, 5 MiB]
- * assuming the target input size is 128 MiB, we will have four post-shuffle partitions,
- * which are:
- *  - post-shuffle partition 0: pre-shuffle partition 0 (size 110 MiB)
- *  - post-shuffle partition 1: pre-shuffle partition 1 (size 30 MiB)
- *  - post-shuffle partition 2: pre-shuffle partition 2 (size 170 MiB)
- *  - post-shuffle partition 3: pre-shuffle partition 3 and 4 (size 50 MiB)
+ * A rule to reduce the post shuffle partitions based on the map output statistics, which can
+ * avoid many small reduce tasks that hurt performance.
  */
 case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
 
@@ -54,28 +36,21 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
     if (!conf.reducePostShufflePartitionsEnabled) {
       return plan
     }
-    // 'SkewedShufflePartitionReader' is added by us, so it's safe to ignore it when changing
-    // number of reducers.
-    val leafNodes = plan.collectLeaves().filter(!_.isInstanceOf[SkewedPartitionReaderExec])
-    if (!leafNodes.forall(_.isInstanceOf[QueryStageExec])) {
+    if (!plan.collectLeaves().forall(_.isInstanceOf[QueryStageExec])) {
       // If not all leaf nodes are query stages, it's not safe to reduce the number of
       // shuffle partitions, because we may break the assumption that all children of a spark plan
       // have same number of output partitions.
       return plan
     }
 
-    def collectShuffles(plan: SparkPlan): Seq[SparkPlan] = plan match {
+    def collectShuffleStages(plan: SparkPlan): Seq[ShuffleQueryStageExec] = plan match {
       case _: LocalShuffleReaderExec => Nil
-      case p: PartialShuffleReaderExec => Seq(p)
+      case _: SkewJoinShuffleReaderExec => Nil
       case stage: ShuffleQueryStageExec => Seq(stage)
-      case _ => plan.children.flatMap(collectShuffles)
+      case _ => plan.children.flatMap(collectShuffleStages)
     }
 
-    val shuffles = collectShuffles(plan)
-    val shuffleStages = shuffles.map {
-      case PartialShuffleReaderExec(s: ShuffleQueryStageExec, _) => s
-      case s: ShuffleQueryStageExec => s
-    }
+    val shuffleStages = collectShuffleStages(plan)
     // ShuffleExchanges introduced by repartition do not support changing the number of partitions.
     // We change the number of partitions in the stage only if all the ShuffleExchanges support it.
     if (!shuffleStages.forall(_.shuffle.canChangeNumPartitions)) {
@@ -94,110 +69,27 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
       // partition) and a result of a SortMergeJoin (multiple partitions).
       val distinctNumPreShufflePartitions =
         validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
-      val distinctExcludedPartitions = shuffles.map {
-        case PartialShuffleReaderExec(_, excludedPartitions) => excludedPartitions
-        case _: ShuffleQueryStageExec => Set.empty[Int]
-      }.distinct
-      if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1
-        && distinctExcludedPartitions.length == 1) {
-        val excludedPartitions = distinctExcludedPartitions.head
-        val partitionIndices = estimatePartitionStartAndEndIndices(
-          validMetrics.toArray, excludedPartitions)
+      if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1) {
+        val partitionStartIndices = ShufflePartitionsCoalescer.coalescePartitions(
+          validMetrics.toArray,
+          firstPartitionIndex = 0,
+          lastPartitionIndex = distinctNumPreShufflePartitions.head,
+          advisoryTargetSize = conf.targetPostShuffleInputSize,
+          minNumPartitions = conf.minNumPostShufflePartitions)
         // This transformation adds new nodes, so we must use `transformUp` here.
-        // Even for shuffle exchange whose input RDD has 0 partition, we should still update its
-        // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
-        // number of output partitions.
-        val visitedStages = HashSet.empty[Int]
-        plan.transformDown {
-          // Replace `PartialShuffleReaderExec` with `CoalescedShuffleReaderExec`, which keeps the
-          // "excludedPartition" requirement and also merges some partitions.
-          case PartialShuffleReaderExec(stage: ShuffleQueryStageExec, _) =>
-            visitedStages.add(stage.id)
-            CoalescedShuffleReaderExec(stage, partitionIndices)
-
-          // We are doing `transformDown`, so the `ShuffleQueryStageExec` may already be optimized
-          // and wrapped by `CoalescedShuffleReaderExec`.
-          case stage: ShuffleQueryStageExec if !visitedStages.contains(stage.id) =>
-            visitedStages.add(stage.id)
-            CoalescedShuffleReaderExec(stage, partitionIndices)
+        val stageIds = shuffleStages.map(_.id).toSet
+        plan.transformUp {
+          // even for shuffle exchange whose input RDD has 0 partition, we should still update its
+          // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
+          // number of output partitions.
+          case stage: ShuffleQueryStageExec if stageIds.contains(stage.id) =>
+            CoalescedShuffleReaderExec(stage, partitionStartIndices)
         }
       } else {
         plan
       }
     }
   }
-
-  /**
-   * Estimates partition start and end indices for post-shuffle partitions based on
-   * mapOutputStatistics provided by all pre-shuffle stages and skip the omittedPartitions
-   * already handled in skewed partition optimization.
-   */
-  // visible for testing.
-  private[sql] def estimatePartitionStartAndEndIndices(
-      mapOutputStatistics: Array[MapOutputStatistics],
-      excludedPartitions: Set[Int] = Set.empty): Array[(Int, Int)] = {
-    val minNumPostShufflePartitions = conf.minNumPostShufflePartitions - excludedPartitions.size
-    val advisoryTargetPostShuffleInputSize = conf.targetPostShuffleInputSize
-    // If minNumPostShufflePartitions is defined, it is possible that we need to use a
-    // value less than advisoryTargetPostShuffleInputSize as the target input size of
-    // a post shuffle task.
-    val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum
-    // The max at here is to make sure that when we have an empty table, we
-    // only have a single post-shuffle partition.
-    // There is no particular reason that we pick 16. We just need a number to
-    // prevent maxPostShuffleInputSize from being set to 0.
-    val maxPostShuffleInputSize = math.max(
-      math.ceil(totalPostShuffleInputSize / minNumPostShufflePartitions.toDouble).toLong, 16)
-    val targetPostShuffleInputSize =
-      math.min(maxPostShuffleInputSize, advisoryTargetPostShuffleInputSize)
-
-    logInfo(
-      s"advisoryTargetPostShuffleInputSize: $advisoryTargetPostShuffleInputSize, " +
-        s"targetPostShuffleInputSize $targetPostShuffleInputSize.")
-
-    // Make sure we do get the same number of pre-shuffle partitions for those stages.
-    val distinctNumPreShufflePartitions =
-      mapOutputStatistics.map(stats => stats.bytesByPartitionId.length).distinct
-    // The reason that we are expecting a single value of the number of pre-shuffle partitions
-    // is that when we add Exchanges, we set the number of pre-shuffle partitions
-    // (i.e. map output partitions) using a static setting, which is the value of
-    // spark.sql.shuffle.partitions. Even if two input RDDs are having different
-    // number of partitions, they will have the same number of pre-shuffle partitions
-    // (i.e. map output partitions).
-    assert(
-      distinctNumPreShufflePartitions.length == 1,
-      "There should be only one distinct value of the number pre-shuffle partitions " +
-        "among registered Exchange operator.")
-
-    val partitionStartIndices = ArrayBuffer[Int]()
-    val partitionEndIndices = ArrayBuffer[Int]()
-    val numPartitions = distinctNumPreShufflePartitions.head
-    val includedPartitions = (0 until numPartitions).filter(!excludedPartitions.contains(_))
-    val firstStartIndex = includedPartitions(0)
-    partitionStartIndices += firstStartIndex
-    var postShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId(firstStartIndex)).sum
-    var i = firstStartIndex
-    includedPartitions.drop(1).foreach { nextPartitionIndex =>
-        val nextShuffleInputSize =
-          mapOutputStatistics.map(_.bytesByPartitionId(nextPartitionIndex)).sum
-        // If nextPartitionIndices is skewed and omitted, or including
-        // the nextShuffleInputSize would exceed the target partition size,
-        // then start a new partition.
-        if (nextPartitionIndex != i + 1 ||
-          (postShuffleInputSize + nextShuffleInputSize > targetPostShuffleInputSize)) {
-          partitionEndIndices += i + 1
-          partitionStartIndices += nextPartitionIndex
-          // reset postShuffleInputSize.
-          postShuffleInputSize = nextShuffleInputSize
-          i = nextPartitionIndex
-        } else {
-          postShuffleInputSize += nextShuffleInputSize
-          i += 1
-        }
-    }
-    partitionEndIndices += i + 1
-    partitionStartIndices.zip(partitionEndIndices).toArray
-  }
 }
 
 /**
@@ -206,15 +98,16 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
  *
  * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
  *              canonicalization.
+ * @param partitionStartIndices The start partition indices for the coalesced partitions.
  */
 case class CoalescedShuffleReaderExec(
     child: SparkPlan,
-    partitionIndices: Array[(Int, Int)]) extends UnaryExecNode {
+    partitionStartIndices: Array[Int]) extends UnaryExecNode {
 
   override def output: Seq[Attribute] = child.output
 
   override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(partitionIndices.length)
+    UnknownPartitioning(partitionStartIndices.length)
   }
 
   private var cachedShuffleRDD: ShuffledRowRDD = null
@@ -223,7 +116,7 @@ case class CoalescedShuffleReaderExec(
     if (cachedShuffleRDD == null) {
       cachedShuffleRDD = child match {
         case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createShuffledRDD(Some(partitionIndices))
+          stage.shuffle.createShuffledRDD(Some(partitionStartIndices))
         case _ =>
           throw new IllegalStateException("operating on canonicalization plan")
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
new file mode 100644
index 0000000000000..18f0585524aa2
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.MapOutputStatistics
+import org.apache.spark.internal.Logging
+
+object ShufflePartitionsCoalescer extends Logging {
+
+  /**
+   * Coalesce the same range of partitions (`firstPartitionIndex`` to `lastPartitionIndex`, the
+   * start is inclusive and the end is exclusive) from multiple shuffles. This method assumes that
+   * all the shuffles have the same number of partitions, and the partitions of same index will be
+   * read together by one task.
+   *
+   * The strategy used to determine the number of coalesced partitions is described as follows.
+   * To determine the number of coalesced partitions, we have a target size for a coalesced
+   * partition. Once we have size statistics of all shuffle partitions, we will do
+   * a pass of those statistics and pack shuffle partitions with continuous indices to a single
+   * coalesced partition until adding another shuffle partition would cause the size of a
+   * coalesced partition to be greater than the target size.
+   *
+   * For example, we have two shuffles with the following partition size statistics:
+   *  - shuffle 1 (5 partitions): [100 MiB, 20 MiB, 100 MiB, 10MiB, 30 MiB]
+   *  - shuffle 2 (5 partitions): [10 MiB,  10 MiB, 70 MiB,  5 MiB, 5 MiB]
+   * Assuming the target size is 128 MiB, we will have 4 coalesced partitions, which are:
+   *  - coalesced partition 0: shuffle partition 0 (size 110 MiB)
+   *  - coalesced partition 1: shuffle partition 1 (size 30 MiB)
+   *  - coalesced partition 2: shuffle partition 2 (size 170 MiB)
+   *  - coalesced partition 3: shuffle partition 3 and 4 (size 50 MiB)
+   *
+   *  @return An array of partition indices which represents the coalesced partitions. For example,
+   *          [0, 2, 3] means 3 coalesced partitions: [0, 2), [2, 3), [3, lastPartitionIndex]
+   */
+  def coalescePartitions(
+      mapOutputStatistics: Array[MapOutputStatistics],
+      firstPartitionIndex: Int,
+      lastPartitionIndex: Int,
+      advisoryTargetSize: Long,
+      minNumPartitions: Int = 1): Array[Int] = {
+    // If `minNumPartitions` is very large, it is possible that we need to use a value less than
+    // `advisoryTargetSize` as the target size of a coalesced task.
+    val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum
+    // The max at here is to make sure that when we have an empty table, we only have a single
+    // coalesced partition.
+    // There is no particular reason that we pick 16. We just need a number to prevent
+    // `maxTargetSize` from being set to 0.
+    val maxTargetSize = math.max(
+      math.ceil(totalPostShuffleInputSize / minNumPartitions.toDouble).toLong, 16)
+    val targetSize = math.min(maxTargetSize, advisoryTargetSize)
+
+    logInfo(s"advisory target size: $advisoryTargetSize, actual target size $targetSize.")
+
+    // Make sure these shuffles have the same number of partitions.
+    val distinctNumShufflePartitions =
+      mapOutputStatistics.map(stats => stats.bytesByPartitionId.length).distinct
+    // The reason that we are expecting a single value of the number of shuffle partitions
+    // is that when we add Exchanges, we set the number of shuffle partitions
+    // (i.e. map output partitions) using a static setting, which is the value of
+    // `spark.sql.shuffle.partitions`. Even if two input RDDs are having different
+    // number of partitions, they will have the same number of shuffle partitions
+    // (i.e. map output partitions).
+    assert(
+      distinctNumShufflePartitions.length == 1,
+      "There should be only one distinct value of the number of shuffle partitions " +
+        "among registered Exchange operators.")
+
+    val splitPoints = ArrayBuffer[Int]()
+    splitPoints += firstPartitionIndex
+    var coalescedSize = 0L
+    var i = firstPartitionIndex
+    while (i < lastPartitionIndex) {
+      // We calculate the total size of i-th shuffle partitions from all shuffles.
+      var totalSizeOfCurrentPartition = 0L
+      var j = 0
+      while (j < mapOutputStatistics.length) {
+        totalSizeOfCurrentPartition += mapOutputStatistics(j).bytesByPartitionId(i)
+        j += 1
+      }
+
+      // If including the `totalSizeOfCurrentPartition` would exceed the target size, then start a
+      // new coalesced partition.
+      if (i > firstPartitionIndex && coalescedSize + totalSizeOfCurrentPartition > targetSize) {
+        splitPoints += i
+        // reset postShuffleInputSize.
+        coalescedSize = totalSizeOfCurrentPartition
+      } else {
+        coalescedSize += totalSizeOfCurrentPartition
+      }
+      i += 1
+    }
+
+    splitPoints.toArray
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala
deleted file mode 100644
index 52f793b24aa17..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/SkewedShuffledRowRDD.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.adaptive
-
-import org.apache.spark._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
-
-/**
- * The [[Partition]] used by [[SkewedShuffledRowRDD]].
- */
-class SkewedShuffledRowRDDPartition(override val index: Int) extends Partition
-
-/**
- * This is a specialized version of [[org.apache.spark.sql.execution.ShuffledRowRDD]]. This is used
- * in Spark SQL adaptive execution to solve data skew issues. This RDD includes rearranged
- * partitions from mappers.
- *
- * This RDD takes a [[ShuffleDependency]] (`dependency`), a partitionIndex
- * and the range of startMapIndex to endMapIndex.
- */
-class SkewedShuffledRowRDD(
-     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
-     partitionIndex: Int,
-     startMapIndex: Int,
-     endMapIndex: Int,
-     metrics: Map[String, SQLMetric])
-  extends RDD[InternalRow](dependency.rdd.context, Nil) {
-
-  override def getDependencies: Seq[Dependency[_]] = List(dependency)
-
-  override def getPartitions: Array[Partition] = {
-    Array(new SkewedShuffledRowRDDPartition(0))
-  }
-
-  override def getPreferredLocations(partition: Partition): Seq[String] = {
-    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    tracker.getMapLocation(dependency, startMapIndex, endMapIndex)
-  }
-
-  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
-    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
-    // as well as the `tempMetrics` for basic shuffle metrics.
-    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-
-    val reader = SparkEnv.get.shuffleManager.getReaderForRange(
-      dependency.shuffleHandle,
-      startMapIndex,
-      endMapIndex,
-      partitionIndex,
-      partitionIndex + 1,
-      context,
-      sqlMetricsReporter)
-    reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
-  }
-
-  override def clearDependencies() {
-    super.clearDependencies()
-    dependency = null
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index ffcd6c7783354..4b08da043b83e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -30,11 +30,11 @@ import org.apache.spark.shuffle.{ShuffleWriteMetricsReporter, ShuffleWriteProces
 import org.apache.spark.shuffle.sort.SortShuffleManager
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.errors._
-import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Divide, Literal, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.{LocalShuffledRowRDD, SkewedShuffledRowRDD}
+import org.apache.spark.sql.execution.adaptive.LocalShuffledRowRDD
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
@@ -49,11 +49,9 @@ case class ShuffleExchangeExec(
     child: SparkPlan,
     canChangeNumPartitions: Boolean = true) extends Exchange {
 
-  // NOTE: coordinator can be null after serialization/deserialization,
-  //       e.g. it can be null on the Executor side
   private lazy val writeMetrics =
     SQLShuffleWriteMetricsReporter.createShuffleWriteMetrics(sparkContext)
-  private lazy val readMetrics =
+  private[sql] lazy val readMetrics =
     SQLShuffleReadMetricsReporter.createShuffleReadMetrics(sparkContext)
   override lazy val metrics = Map(
     "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size")
@@ -90,9 +88,8 @@ case class ShuffleExchangeExec(
       writeMetrics)
   }
 
-  def createShuffledRDD(
-      partitionRanges: Option[Array[(Int, Int)]]): ShuffledRowRDD = {
-    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionRanges)
+  def createShuffledRDD(partitionStartIndices: Option[Array[Int]]): ShuffledRowRDD = {
+    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionStartIndices)
   }
 
   def createLocalShuffleRDD(
@@ -100,14 +97,6 @@ case class ShuffleExchangeExec(
     new LocalShuffledRowRDD(shuffleDependency, readMetrics, partitionStartIndicesPerMapper)
   }
 
-  def createSkewedShuffleRDD(
-      partitionIndex: Int,
-      startMapIndex: Int,
-      endMapIndex: Int): SkewedShuffledRowRDD = {
-    new SkewedShuffledRowRDD(shuffleDependency,
-      partitionIndex, startMapIndex, endMapIndex, readMetrics)
-  }
-
   /**
    * Caches the created ShuffleRowRDD so we can reuse that.
    */
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
index 6384aed6a78e0..62eea611556ff 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.{PartialShuffleReaderExec, SkewedPartitionReaderExec}
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.util.collection.BitSet
 
@@ -42,11 +41,17 @@ case class SortMergeJoinExec(
     condition: Option[Expression],
     left: SparkPlan,
     right: SparkPlan,
-    isPartial: Boolean = false) extends BinaryExecNode with CodegenSupport {
+    isSkewJoin: Boolean = false) extends BinaryExecNode with CodegenSupport {
 
   override lazy val metrics = Map(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
 
+  override def nodeName: String = {
+    if (isSkewJoin) super.nodeName + "(skew=true)" else super.nodeName
+  }
+
+  override def stringArgs: Iterator[Any] = super.stringArgs.toSeq.dropRight(1).iterator
+
   override def simpleStringWithNodeId(): String = {
     val opId = ExplainUtils.getOpId(this)
     s"$nodeName $joinType ($opId)".trim
@@ -98,7 +103,9 @@ case class SortMergeJoinExec(
   }
 
   override def requiredChildDistribution: Seq[Distribution] = {
-    if (isPartial) {
+    if (isSkewJoin) {
+      // We re-arrange the shuffle partitions to deal with skew join, and the new children
+      // partitioning doesn't satisfy `HashClusteredDistribution`.
       UnspecifiedDistribution :: UnspecifiedDistribution :: Nil
     } else {
       HashClusteredDistribution(leftKeys) :: HashClusteredDistribution(rightKeys) :: Nil
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
index 04b4d4f29f850..5565a0dd01840 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
@@ -19,11 +19,11 @@ package org.apache.spark.sql.execution
 
 import org.scalatest.BeforeAndAfterAll
 
-import org.apache.spark.{MapOutputStatistics, SparkConf, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql._
 import org.apache.spark.sql.execution.adaptive._
-import org.apache.spark.sql.execution.adaptive.{CoalescedShuffleReaderExec, ReduceNumShufflePartitions}
+import org.apache.spark.sql.execution.adaptive.CoalescedShuffleReaderExec
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -52,212 +52,6 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
     }
   }
 
-  private def checkEstimation(
-      rule: ReduceNumShufflePartitions,
-      bytesByPartitionIdArray: Array[Array[Long]],
-      expectedPartitionStartIndices: Array[Int]): Unit = {
-    val mapOutputStatistics = bytesByPartitionIdArray.zipWithIndex.map {
-      case (bytesByPartitionId, index) =>
-        new MapOutputStatistics(index, bytesByPartitionId)
-    }
-    val estimatedPartitionStartIndices =
-      rule.estimatePartitionStartAndEndIndices(mapOutputStatistics).map(_._1)
-    assert(estimatedPartitionStartIndices === expectedPartitionStartIndices)
-  }
-
-  private def createReduceNumShufflePartitionsRule(
-      advisoryTargetPostShuffleInputSize: Long,
-      minNumPostShufflePartitions: Int = 1): ReduceNumShufflePartitions = {
-    val conf = new SQLConf().copy(
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE -> advisoryTargetPostShuffleInputSize,
-      SQLConf.SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS -> minNumPostShufflePartitions)
-    ReduceNumShufflePartitions(conf)
-  }
-
-  test("test estimatePartitionStartIndices - 1 Exchange") {
-    val rule = createReduceNumShufflePartitionsRule(100L)
-
-    {
-      // All bytes per partition are 0.
-      val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // Some bytes per partition are 0 and total size is less than the target size.
-      // 1 post-shuffle partition is needed.
-      val bytesByPartitionId = Array[Long](10, 0, 20, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // 2 post-shuffle partitions are needed.
-      val bytesByPartitionId = Array[Long](10, 0, 90, 20, 0)
-      val expectedPartitionStartIndices = Array[Int](0, 3)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // There are a few large pre-shuffle partitions.
-      val bytesByPartitionId = Array[Long](110, 10, 100, 110, 0)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // All pre-shuffle partitions are larger than the targeted size.
-      val bytesByPartitionId = Array[Long](100, 110, 100, 110, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-
-    {
-      // The last pre-shuffle partition is in a single post-shuffle partition.
-      val bytesByPartitionId = Array[Long](30, 30, 0, 40, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 4)
-      checkEstimation(rule, Array(bytesByPartitionId), expectedPartitionStartIndices)
-    }
-  }
-
-  test("test estimatePartitionStartIndices - 2 Exchanges") {
-    val rule = createReduceNumShufflePartitionsRule(100L)
-
-    {
-      // If there are multiple values of the number of pre-shuffle partitions,
-      // we should see an assertion error.
-      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
-      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0, 0)
-      val mapOutputStatistics =
-        Array(
-          new MapOutputStatistics(0, bytesByPartitionId1),
-          new MapOutputStatistics(1, bytesByPartitionId2))
-      intercept[AssertionError](rule.estimatePartitionStartAndEndIndices(
-        mapOutputStatistics))
-    }
-
-    {
-      // All bytes per partition are 0.
-      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
-      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // Some bytes per partition are 0.
-      // 1 post-shuffle partition is needed.
-      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 20, 0, 20)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // 2 post-shuffle partition are needed.
-      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 2, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // 4 post-shuffle partition are needed.
-      val bytesByPartitionId1 = Array[Long](0, 99, 0, 20, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // 2 post-shuffle partition are needed.
-      val bytesByPartitionId1 = Array[Long](0, 100, 0, 30, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // There are a few large pre-shuffle partitions.
-      val bytesByPartitionId1 = Array[Long](0, 100, 40, 30, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 60, 0, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // All pairs of pre-shuffle partitions are larger than the targeted size.
-      val bytesByPartitionId1 = Array[Long](100, 100, 40, 30, 0)
-      val bytesByPartitionId2 = Array[Long](30, 0, 60, 70, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-  }
-
-  test("test estimatePartitionStartIndices and enforce minimal number of reducers") {
-    val rule = createReduceNumShufflePartitionsRule(100L, 2)
-
-    {
-      // The minimal number of post-shuffle partitions is not enforced because
-      // the size of data is 0.
-      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
-      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // The minimal number of post-shuffle partitions is enforced.
-      val bytesByPartitionId1 = Array[Long](10, 5, 5, 0, 20)
-      val bytesByPartitionId2 = Array[Long](5, 10, 0, 10, 5)
-      val expectedPartitionStartIndices = Array[Int](0, 3)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-
-    {
-      // The number of post-shuffle partitions is determined by the coordinator.
-      val bytesByPartitionId1 = Array[Long](10, 50, 20, 80, 20)
-      val bytesByPartitionId2 = Array[Long](40, 10, 0, 10, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 3, 4)
-      checkEstimation(
-        rule,
-        Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices)
-    }
-  }
-
-  ///////////////////////////////////////////////////////////////////////////
-  // Query tests
-  ///////////////////////////////////////////////////////////////////////////
-
   val numInputPartitions: Int = 10
 
   def withSparkSession(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
new file mode 100644
index 0000000000000..fcfde83b2ffd5
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.{MapOutputStatistics, SparkFunSuite}
+import org.apache.spark.sql.execution.adaptive.ShufflePartitionsCoalescer
+
+class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
+
+  private def checkEstimation(
+      bytesByPartitionIdArray: Array[Array[Long]],
+      expectedPartitionStartIndices: Array[Int],
+      targetSize: Long,
+      minNumPartitions: Int = 1): Unit = {
+    val mapOutputStatistics = bytesByPartitionIdArray.zipWithIndex.map {
+      case (bytesByPartitionId, index) =>
+        new MapOutputStatistics(index, bytesByPartitionId)
+    }
+    val estimatedPartitionStartIndices = ShufflePartitionsCoalescer.coalescePartitions(
+      mapOutputStatistics,
+      0,
+      bytesByPartitionIdArray.head.length,
+      targetSize,
+      minNumPartitions)
+    assert(estimatedPartitionStartIndices === expectedPartitionStartIndices)
+  }
+
+  test("1 shuffle") {
+    val targetSize = 100
+
+    {
+      // All bytes per partition are 0.
+      val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
+      val expectedPartitionStartIndices = Array[Int](0)
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+    }
+
+    {
+      // Some bytes per partition are 0 and total size is less than the target size.
+      // 1 coalesced partition is expected.
+      val bytesByPartitionId = Array[Long](10, 0, 20, 0, 0)
+      val expectedPartitionStartIndices = Array[Int](0)
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+    }
+
+    {
+      // 2 coalesced partitions are expected.
+      val bytesByPartitionId = Array[Long](10, 0, 90, 20, 0)
+      val expectedPartitionStartIndices = Array[Int](0, 3)
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+    }
+
+    {
+      // There are a few large shuffle partitions.
+      val bytesByPartitionId = Array[Long](110, 10, 100, 110, 0)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+    }
+
+    {
+      // All shuffle partitions are larger than the targeted size.
+      val bytesByPartitionId = Array[Long](100, 110, 100, 110, 110)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+    }
+
+    {
+      // The last shuffle partition is in a single coalesced partition.
+      val bytesByPartitionId = Array[Long](30, 30, 0, 40, 110)
+      val expectedPartitionStartIndices = Array[Int](0, 4)
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+    }
+  }
+
+  test("2 shuffles") {
+    val targetSize = 100
+
+    {
+      // If there are multiple values of the number of shuffle partitions,
+      // we should see an assertion error.
+      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
+      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0, 0)
+      intercept[AssertionError] {
+        checkEstimation(Array(bytesByPartitionId1, bytesByPartitionId2), Array.empty, targetSize)
+      }
+    }
+
+    {
+      // All bytes per partition are 0.
+      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
+      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
+      val expectedPartitionStartIndices = Array[Int](0)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+
+    {
+      // Some bytes per partition are 0.
+      // 1 coalesced partition is expected.
+      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 20, 0, 20)
+      val expectedPartitionStartIndices = Array[Int](0)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+
+    {
+      // 2 coalesced partition are expected.
+      val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
+      val expectedPartitionStartIndices = Array[Int](0, 2, 4)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+
+    {
+      // 4 coalesced partition are expected.
+      val bytesByPartitionId1 = Array[Long](0, 99, 0, 20, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+
+    {
+      // 2 coalesced partition are needed.
+      val bytesByPartitionId1 = Array[Long](0, 100, 0, 30, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+
+    {
+      // There are a few large shuffle partitions.
+      val bytesByPartitionId1 = Array[Long](0, 100, 40, 30, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 60, 0, 110)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+
+    {
+      // All pairs of shuffle partitions are larger than the targeted size.
+      val bytesByPartitionId1 = Array[Long](100, 100, 40, 30, 0)
+      val bytesByPartitionId2 = Array[Long](30, 0, 60, 70, 110)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize)
+    }
+  }
+
+  test("enforce minimal number of coalesced partitions") {
+    val targetSize = 100
+    val minNumPartitions = 2
+
+    {
+      // The minimal number of coalesced partitions is not enforced because
+      // the size of data is 0.
+      val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
+      val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
+      val expectedPartitionStartIndices = Array[Int](0)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize, minNumPartitions)
+    }
+
+    {
+      // The minimal number of coalesced partitions is enforced.
+      val bytesByPartitionId1 = Array[Long](10, 5, 5, 0, 20)
+      val bytesByPartitionId2 = Array[Long](5, 10, 0, 10, 5)
+      val expectedPartitionStartIndices = Array[Int](0, 3)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize, minNumPartitions)
+    }
+
+    {
+      // The number of coalesced partitions is determined by the algorithm.
+      val bytesByPartitionId1 = Array[Long](10, 50, 20, 80, 20)
+      val bytesByPartitionId2 = Array[Long](40, 10, 0, 10, 30)
+      val expectedPartitionStartIndices = Array[Int](0, 1, 3, 4)
+      checkEstimation(
+        Array(bytesByPartitionId1, bytesByPartitionId2),
+        expectedPartitionStartIndices,
+        targetSize, minNumPartitions)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index a2071903bea7e..4edb35ea30fde 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -23,7 +23,7 @@ import java.net.URI
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan}
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight, SortMergeJoinExec}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
 import org.apache.spark.sql.internal.SQLConf
@@ -594,160 +594,84 @@ class AdaptiveQueryExecSuite
           .range(0, 1000, 1, 10)
           .selectExpr("id % 1 as key2", "id as value2")
           .createOrReplaceTempView("skewData2")
-        val (innerPlan, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
+        val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
           "SELECT key1 FROM skewData1 join skewData2 ON key1 = key2 group by key1")
-        val innerSmj = findTopLevelSortMergeJoin(innerPlan)
-        assert(innerSmj.size == 1)
         // Additional shuffle introduced, so disable the "OptimizeSkewedJoin" optimization
-        val innerSmjAfter = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        assert(innerSmjAfter.size == 1)
+        val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
+        assert(innerSmj.size == 1 && !innerSmj.head.isSkewJoin)
       }
     }
   }
 
+  // TODO: we need a way to customize data distribution after shuffle, to improve test coverage
+  //       of this case.
   test("SPARK-29544: adaptive skew join with different join types") {
-    Seq("false", "true").foreach { reducePostShufflePartitionsEnabled =>
-      withSQLConf(
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
-        SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-        SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "100",
-        SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key -> reducePostShufflePartitionsEnabled,
-        SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
-        withTempView("skewData1", "skewData2") {
-          spark
-            .range(0, 1000, 1, 10)
-            .selectExpr("id % 2 as key1", "id as value1")
-            .createOrReplaceTempView("skewData1")
-          spark
-            .range(0, 1000, 1, 10)
-            .selectExpr("id % 1 as key2", "id as value2")
-            .createOrReplaceTempView("skewData2")
-          // skewed inner join optimization
-          val (innerPlan, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
-            "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
-          val innerSmj = findTopLevelSortMergeJoin(innerPlan)
-          assert(innerSmj.size == 1)
-          // left stats: [3496, 0, 0, 0, 4014]
-          // right stats:[6292, 0, 0, 0, 0]
-          // the partition 0 in both left and right side are all skewed.
-          // And divide into 5 splits both in left and right (the max splits number).
-          // So there are 5 x 5 smjs for partition 0.
-          // Partition 4 in left side is skewed and is divided into 5 splits.
-          // The right side of partition 4 is not skewed.
-          // So there are 5 smjs for partition 4.
-          // So total (25 + 5 + 1) smjs.
-          // Union
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //             .
-          //             .
-          //             .
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-
-          val innerSmjAfter = findTopLevelSortMergeJoin(innerAdaptivePlan)
-          assert(innerSmjAfter.size == 31)
-
-          // skewed left outer join optimization
-          val (leftPlan, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
-            "SELECT * FROM skewData1 left outer join skewData2 ON key1 = key2")
-          val leftSmj = findTopLevelSortMergeJoin(leftPlan)
-          assert(leftSmj.size == 1)
-          // left stats: [3496, 0, 0, 0, 4014]
-          // right stats:[6292, 0, 0, 0, 0]
-          // The partition 0 in both left and right are all skewed.
-          // The partition 4 in left side is skewed.
-          // But for left outer join, we don't split the right partition even skewed.
-          // So the partition 0 in left side is divided into 5 splits(the max split number).
-          // the partition 4 in left side is divided into 5 splits(the max split number).
-          // So total (5 + 5 + 1) smjs.
-          // Union
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //             .
-          //             .
-          //             .
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-
-          val leftSmjAfter = findTopLevelSortMergeJoin(leftAdaptivePlan)
-          assert(leftSmjAfter.size == 11)
-
-          // skewed right outer join optimization
-          val (rightPlan, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
-            "SELECT * FROM skewData1 right outer join skewData2 ON key1 = key2")
-          val rightSmj = findTopLevelSortMergeJoin(rightPlan)
-          assert(rightSmj.size == 1)
-          // left stats: [3496, 0, 0, 0, 4014]
-          // right stats:[6292, 0, 0, 0, 0]
-          // The partition 0 in both left and right side are all skewed.
-          // And the partition 4 in left side is skewed.
-          // But for right outer join, we don't split the left partition even skewed.
-          // And divide right side into 5 splits(the max split number)
-          // So total 6 smjs.
-          // Union
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- CoalescedShuffleReader
-          //       +- ShuffleQueryStage
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //             .
-          //             .
-          //             .
-          // +- SortMergeJoin
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-          //   +- Sort
-          //     +- SkewedShuffleReader
-          //       +- ShuffleQueryStage
-
-          val rightSmjAfter = findTopLevelSortMergeJoin(rightAdaptivePlan)
-          assert(rightSmjAfter.size == 6)
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "100",
+      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
+      withTempView("skewData1", "skewData2") {
+        spark
+          .range(0, 1000, 1, 10)
+          .selectExpr("id % 2 as key1", "id as value1")
+          .createOrReplaceTempView("skewData1")
+        spark
+          .range(0, 1000, 1, 10)
+          .selectExpr("id % 1 as key2", "id as value2")
+          .createOrReplaceTempView("skewData2")
+
+        def checkSkewJoin(joins: Seq[SortMergeJoinExec], expectedNumPartitions: Int): Unit = {
+          assert(joins.size == 1 && joins.head.isSkewJoin)
+          assert(joins.head.left.collect {
+            case r: SkewJoinShuffleReaderExec => r
+          }.head.partitionSpecs.length == expectedNumPartitions)
+          assert(joins.head.right.collect {
+            case r: SkewJoinShuffleReaderExec => r
+          }.head.partitionSpecs.length == expectedNumPartitions)
         }
+
+        // skewed inner join optimization
+        val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
+        // left stats: [3496, 0, 0, 0, 4014]
+        // right stats:[6292, 0, 0, 0, 0]
+        // Partition 0: both left and right sides are skewed, and divide into 5 splits, so
+        //              5 x 5 sub-partitions.
+        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
+        // Partition 4: only left side is skewed, and divide into 5 splits, so
+        //              5 sub-partitions.
+        // So total (25 + 1 + 5) partitions.
+        val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
+        checkSkewJoin(innerSmj, 25 + 1 + 5)
+
+        // skewed left outer join optimization
+        val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT * FROM skewData1 left outer join skewData2 ON key1 = key2")
+        // left stats: [3496, 0, 0, 0, 4014]
+        // right stats:[6292, 0, 0, 0, 0]
+        // Partition 0: both left and right sides are skewed, but left join can't split right side,
+        //              so only left side is divided into 5 splits, and thus 5 sub-partitions.
+        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
+        // Partition 4: only left side is skewed, and divide into 5 splits, so
+        //              5 sub-partitions.
+        // So total (5 + 1 + 5) partitions.
+        val leftSmj = findTopLevelSortMergeJoin(leftAdaptivePlan)
+        checkSkewJoin(leftSmj, 5 + 1 + 5)
+
+        // skewed right outer join optimization
+        val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT * FROM skewData1 right outer join skewData2 ON key1 = key2")
+        // left stats: [3496, 0, 0, 0, 4014]
+        // right stats:[6292, 0, 0, 0, 0]
+        // Partition 0: both left and right sides are skewed, but right join can't split left side,
+        //              so only right side is divided into 5 splits, and thus 5 sub-partitions.
+        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
+        // Partition 4: only left side is skewed, but right join can't split left side, so just
+        //              1 partition.
+        // So total (5 + 1 + 1) partitions.
+        val rightSmj = findTopLevelSortMergeJoin(rightAdaptivePlan)
+        checkSkewJoin(rightSmj, 5 + 1 + 1)
       }
     }
   }
@@ -805,3 +729,4 @@ class AdaptiveQueryExecSuite
         s" enabled but is not supported for")))
   }
 }
+

From 5063cd93c8d76a6d9650d1243bf5e1cea8da1d94 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 13 Feb 2020 11:17:27 -0800
Subject: [PATCH 0070/1280] [SPARK-30807][K8S][TESTS] Support Java 11 in K8S
 integration tests

### What changes were proposed in this pull request?

This PR aims to support JDK11 test in K8S integration tests.
- This is an update in testing framework instead of individual tests.
- This will enable JDK11 runtime test when you didn't installed JDK11 on your local system.

### Why are the changes needed?

Apache Spark 3.0.0 adds JDK11 support, but K8s integration tests use JDK8 until now.

### Does this PR introduce any user-facing change?

No. This is a dev-only test-related PR.

### How was this patch tested?

This is irrelevant to Jenkins UT, but Jenkins K8S IT (JDK8) should pass.
- https://github.com/apache/spark/pull/27559#issuecomment-585903489 (JDK8 Passed)

And, manually do the following for JDK11 test.
```
$ NO_MANUAL=1 ./dev/make-distribution.sh --r --pip --tgz -Phadoop-3.2 -Pkubernetes
$ resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim --spark-tgz $PWD/spark-*.tgz
```

```
$ docker run -it --rm kubespark/spark:1318DD8A-2B15-4A00-BC69-D0E90CED235B /usr/local/openjdk-11/bin/java --version | tail -n1
OpenJDK 64-Bit Server VM 18.9 (build 11.0.6+10, mixed mode)
```

Closes #27559 from dongjoon-hyun/SPARK-30807.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 859699135cb63b57f5d844e762070065cedb4408)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../docker/src/main/dockerfiles/spark/Dockerfile  |  3 ++-
 .../kubernetes/integration-tests/README.md        | 15 +++++++++++++--
 .../dev/dev-run-integration-tests.sh              | 10 ++++++++++
 .../kubernetes/integration-tests/pom.xml          |  4 ++++
 .../scripts/setup-integration-test-env.sh         | 14 +++++++++++---
 5 files changed, 40 insertions(+), 6 deletions(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index a1fc63789bc61..6ed37fc637b31 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -14,8 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+ARG java_image_tag=8-jre-slim
 
-FROM openjdk:8-jre-slim
+FROM openjdk:${java_image_tag}
 
 ARG spark_uid=185
 
diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md
index d7ad35a175a61..18b91916208d6 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -6,13 +6,17 @@ title: Spark on Kubernetes Integration Tests
 # Running the Kubernetes Integration Tests
 
 Note that the integration test framework is currently being heavily revised and
-is subject to change. Note that currently the integration tests only run with Java 8.
+is subject to change.
 
 The simplest way to run the integration tests is to install and run Minikube, then run the following from this
 directory:
 
     ./dev/dev-run-integration-tests.sh
 
+To run tests with Java 11 instead of Java 8, use `--java-image-tag` to specify the base image.
+
+    ./dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim
+
 The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should
 run with a minimum of 4 CPUs and 6G of memory:
 
@@ -183,7 +187,14 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro
       A specific image tag to use, when set assumes images with those tags are already built and available in the 
       specified image repository.  When set to <code>N/A</code> (the default) fresh images will be built.
     </td>
-    <td><code>N/A</code>
+    <td><code>N/A</code></td>
+  </tr>
+  <tr>
+    <td><code>spark.kubernetes.test.javaImageTag</code></td>
+    <td>
+      A specific OpenJDK base image tag to use, when set uses it instead of 8-jre-slim.
+    </td>
+    <td><code>8-jre-slim</code></td>
   </tr>
   <tr>
     <td><code>spark.kubernetes.test.imageTagFile</code></td>
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 1f0a8035cea7b..76d6e1c1e8499 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -23,6 +23,7 @@ DEPLOY_MODE="minikube"
 IMAGE_REPO="docker.io/kubespark"
 SPARK_TGZ="N/A"
 IMAGE_TAG="N/A"
+JAVA_IMAGE_TAG=
 BASE_IMAGE_NAME=
 JVM_IMAGE_NAME=
 PYTHON_IMAGE_NAME=
@@ -52,6 +53,10 @@ while (( "$#" )); do
       IMAGE_TAG="$2"
       shift
       ;;
+    --java-image-tag)
+      JAVA_IMAGE_TAG="$2"
+      shift
+      ;;
     --deploy-mode)
       DEPLOY_MODE="$2"
       shift
@@ -120,6 +125,11 @@ properties=(
   -Dtest.include.tags=$INCLUDE_TAGS
 )
 
+if [ -n "$JAVA_IMAGE_TAG" ];
+then
+  properties=( ${properties[@]} -Dspark.kubernetes.test.javaImageTag=$JAVA_IMAGE_TAG )
+fi
+
 if [ -n $NAMESPACE ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE )
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 8e1043f77db6d..369dfd491826c 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -39,6 +39,7 @@
     <spark.kubernetes.test.sparkTgz></spark.kubernetes.test.sparkTgz>
     <spark.kubernetes.test.unpackSparkDir>${project.build.directory}/spark-dist-unpacked</spark.kubernetes.test.unpackSparkDir>
     <spark.kubernetes.test.imageTag>N/A</spark.kubernetes.test.imageTag>
+    <spark.kubernetes.test.javaImageTag>8-jre-slim</spark.kubernetes.test.javaImageTag>
     <spark.kubernetes.test.imageTagFile>${project.build.directory}/imageTag.txt</spark.kubernetes.test.imageTagFile>
     <spark.kubernetes.test.deployMode>minikube</spark.kubernetes.test.deployMode>
     <spark.kubernetes.test.imageRepo>docker.io/kubespark</spark.kubernetes.test.imageRepo>
@@ -109,6 +110,9 @@
                 <argument>--image-tag</argument>
                 <argument>${spark.kubernetes.test.imageTag}</argument>
 
+                <argument>--java-image-tag</argument>
+                <argument>${spark.kubernetes.test.javaImageTag}</argument>
+
                 <argument>--image-tag-output-file</argument>
                 <argument>${spark.kubernetes.test.imageTagFile}</argument>
 
diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index 9e04b963fc40e..ab906604fce06 100755
--- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -23,6 +23,7 @@ IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt"
 DEPLOY_MODE="minikube"
 IMAGE_REPO="docker.io/kubespark"
 IMAGE_TAG="N/A"
+JAVA_IMAGE_TAG="8-jre-slim"
 SPARK_TGZ="N/A"
 
 # Parse arguments
@@ -40,6 +41,10 @@ while (( "$#" )); do
       IMAGE_TAG="$2"
       shift
       ;;
+    --java-image-tag)
+      JAVA_IMAGE_TAG="$2"
+      shift
+      ;;
     --image-tag-output-file)
       IMAGE_TAG_OUTPUT_FILE="$2"
       shift
@@ -82,6 +87,9 @@ then
   IMAGE_TAG=$(uuidgen);
   cd $SPARK_INPUT_DIR
 
+  # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim)
+  JAVA_IMAGE_TAG_BUILD_ARG="-b java_image_tag=$JAVA_IMAGE_TAG"
+
   # Build PySpark image
   LANGUAGE_BINDING_BUILD_ARGS="-p $DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"
 
@@ -95,7 +103,7 @@ then
   case $DEPLOY_MODE in
     cloud)
       # Build images
-      $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+      $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
 
       # Push images appropriately
       if [[ $IMAGE_REPO == gcr.io* ]] ;
@@ -109,13 +117,13 @@ then
     docker-for-desktop)
        # Only need to build as this will place it in our local Docker repo which is all
        # we need for Docker for Desktop to work so no need to also push
-       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
        ;;
 
     minikube)
        # Only need to build and if we do this with the -m option for minikube we will
        # build the images directly using the minikube Docker daemon so no need to push
-       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG $LANGUAGE_BINDING_BUILD_ARGS build
+       $SPARK_INPUT_DIR/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG $JAVA_IMAGE_TAG_BUILD_ARG $LANGUAGE_BINDING_BUILD_ARGS build
        ;;
     *)
        echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1

From 72720ae144921efc405ef29c52f467c92ad0a3f3 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 13 Feb 2020 11:42:00 -0800
Subject: [PATCH 0071/1280] [SPARK-30816][K8S][TESTS] Fix
 dev-run-integration-tests.sh to ignore empty params

### What changes were proposed in this pull request?

This PR aims to fix `dev-run-integration-tests.sh` to ignore empty params correctly.

### Why are the changes needed?

The following script runs `mvn` integration test like the following.
```
$ resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
...
build/mvn integration-test
-f /Users/dongjoon/APACHE/spark/pom.xml
-pl resource-managers/kubernetes/integration-tests
-am
-Pscala-2.12
-Pkubernetes
-Pkubernetes-integration-tests
-Djava.version=8
-Dspark.kubernetes.test.sparkTgz=N/A
-Dspark.kubernetes.test.imageTag=N/A
-Dspark.kubernetes.test.imageRepo=docker.io/kubespark
-Dspark.kubernetes.test.deployMode=minikube
-Dtest.include.tags=k8s
-Dspark.kubernetes.test.namespace=
-Dspark.kubernetes.test.serviceAccountName=
-Dspark.kubernetes.test.kubeConfigContext=
-Dspark.kubernetes.test.master=
-Dtest.exclude.tags=
-Dspark.kubernetes.test.jvmImage=spark
-Dspark.kubernetes.test.pythonImage=spark-py
-Dspark.kubernetes.test.rImage=spark-r
```

After this PR, the empty parameters like the followings will be skipped like the original design.
```
-Dspark.kubernetes.test.namespace=
-Dspark.kubernetes.test.serviceAccountName=
-Dspark.kubernetes.test.kubeConfigContext=
-Dspark.kubernetes.test.master=
-Dtest.exclude.tags=
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins K8S integration test.

Closes #27566 from dongjoon-hyun/SPARK-30816.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 74cd46eb691be5dc1cb1c496eeeaa2614945bd98)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../integration-tests/dev/dev-run-integration-tests.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 76d6e1c1e8499..607bb243458a6 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -130,27 +130,27 @@ then
   properties=( ${properties[@]} -Dspark.kubernetes.test.javaImageTag=$JAVA_IMAGE_TAG )
 fi
 
-if [ -n $NAMESPACE ];
+if [ -n "$NAMESPACE" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE )
 fi
 
-if [ -n $SERVICE_ACCOUNT ];
+if [ -n "$SERVICE_ACCOUNT" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.serviceAccountName=$SERVICE_ACCOUNT )
 fi
 
-if [ -n $CONTEXT ];
+if [ -n "$CONTEXT" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.kubeConfigContext=$CONTEXT )
 fi
 
-if [ -n $SPARK_MASTER ];
+if [ -n "$SPARK_MASTER" ];
 then
   properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER )
 fi
 
-if [ -n $EXCLUDE_TAGS ];
+if [ -n "$EXCLUDE_TAGS" ];
 then
   properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS )
 fi

From 4db64ed37c601eb62aa3939d13f4f0e15bc1e4a9 Mon Sep 17 00:00:00 2001
From: Ali Afroozeh <ali.afroozeh@databricks.com>
Date: Thu, 13 Feb 2020 23:58:55 +0100
Subject: [PATCH 0072/1280] [SPARK-30798][SQL] Scope Session.active in
 QueryExecution

### What changes were proposed in this pull request?

This PR scopes `SparkSession.active` to prevent problems with processing queries with possibly different spark sessions (and different configs). A new method, `withActive` is introduced on `SparkSession` that restores the previous spark session after the block of code is executed.

### Why are the changes needed?
`SparkSession.active` is a thread local variable that points to the current thread's spark session. It is important to note that the `SQLConf.get` method depends on `SparkSession.active`. In the current implementation it is possible that `SparkSession.active` points to a different session which causes various problems. Most of these problems arise because part of the query processing is done using the configurations of a different session. For example, when creating a data frame using a new session, i.e., `session.sql("...")`, part of the data frame is constructed using the currently active spark session, which can be a different session from the one used later for processing the query.

### Does this PR introduce any user-facing change?
The `withActive` method is introduced on `SparkSession`.

### How was this patch tested?
Unit tests (to be added)

Closes #27387 from dbaliafroozeh/UseWithActiveSessionInQueryExecution.

Authored-by: Ali Afroozeh <ali.afroozeh@databricks.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit e2d3983de78f5c80fac066b7ee8bedd0987110dd)
Signed-off-by: herman <herman@databricks.com>
---
 .../apache/spark/sql/DataFrameWriter.scala    |  2 +-
 .../apache/spark/sql/DataFrameWriterV2.scala  |  2 +-
 .../scala/org/apache/spark/sql/Dataset.scala  | 36 ++++++++++---------
 .../spark/sql/KeyValueGroupedDataset.scala    |  5 +--
 .../org/apache/spark/sql/SparkSession.scala   | 30 +++++++++++-----
 .../spark/sql/execution/QueryExecution.scala  | 16 +++++----
 .../spark/sql/execution/SQLExecution.scala    |  4 +--
 .../streaming/MicroBatchExecution.scala       |  4 +--
 .../continuous/ContinuousExecution.scala      |  2 +-
 .../spark/sql/internal/CatalogImpl.scala      |  2 +-
 .../org/apache/spark/sql/DatasetSuite.scala   | 10 ++++++
 .../apache/spark/sql/SQLQueryTestSuite.scala  |  2 +-
 .../ui/SQLAppStatusListenerSuite.scala        |  2 +-
 .../SparkExecuteStatementOperation.scala      |  2 +-
 .../hive/thriftserver/SparkSQLDriver.scala    |  2 +-
 .../hive/execution/HiveComparisonTest.scala   |  3 +-
 .../apache/spark/sql/hive/test/TestHive.scala |  2 +-
 17 files changed, 74 insertions(+), 52 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 4557219abeb15..fff1f4b636dea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -896,7 +896,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   private def runCommand(session: SparkSession, name: String)(command: LogicalPlan): Unit = {
     val qe = session.sessionState.executePlan(command)
     // call `QueryExecution.toRDD` to trigger the execution of commands.
-    SQLExecution.withNewExecutionId(session, qe, Some(name))(qe.toRdd)
+    SQLExecution.withNewExecutionId(qe, Some(name))(qe.toRdd)
   }
 
   private def lookupV2Provider(): Option[TableProvider] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
index f5dd7613d4103..cf6bde5a2bcb9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
@@ -226,7 +226,7 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
   private def runCommand(name: String)(command: LogicalPlan): Unit = {
     val qe = sparkSession.sessionState.executePlan(command)
     // call `QueryExecution.toRDD` to trigger the execution of commands.
-    SQLExecution.withNewExecutionId(sparkSession, qe, Some(name))(qe.toRdd)
+    SQLExecution.withNewExecutionId(qe, Some(name))(qe.toRdd)
   }
 
   private def internalReplace(orCreate: Boolean): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index a1c33f92d17b4..42f35354e864f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -82,18 +82,19 @@ private[sql] object Dataset {
     dataset
   }
 
-  def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame = {
-    val qe = sparkSession.sessionState.executePlan(logicalPlan)
-    qe.assertAnalyzed()
-    new Dataset[Row](sparkSession, qe, RowEncoder(qe.analyzed.schema))
+  def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame =
+    sparkSession.withActive {
+      val qe = sparkSession.sessionState.executePlan(logicalPlan)
+      qe.assertAnalyzed()
+      new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
   }
 
   /** A variant of ofRows that allows passing in a tracker so we can track query parsing time. */
   def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan, tracker: QueryPlanningTracker)
-    : DataFrame = {
+    : DataFrame = sparkSession.withActive {
     val qe = new QueryExecution(sparkSession, logicalPlan, tracker)
     qe.assertAnalyzed()
-    new Dataset[Row](sparkSession, qe, RowEncoder(qe.analyzed.schema))
+    new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
   }
 }
 
@@ -185,13 +186,12 @@ private[sql] object Dataset {
  */
 @Stable
 class Dataset[T] private[sql](
-    @transient private val _sparkSession: SparkSession,
     @DeveloperApi @Unstable @transient val queryExecution: QueryExecution,
     @DeveloperApi @Unstable @transient val encoder: Encoder[T])
   extends Serializable {
 
   @transient lazy val sparkSession: SparkSession = {
-    if (_sparkSession == null) {
+    if (queryExecution == null || queryExecution.sparkSession == null) {
       throw new SparkException(
       "Dataset transformations and actions can only be invoked by the driver, not inside of" +
         " other Dataset transformations; for example, dataset1.map(x => dataset2.values.count()" +
@@ -199,7 +199,7 @@ class Dataset[T] private[sql](
         "performed inside of the dataset1.map transformation. For more information," +
         " see SPARK-28702.")
     }
-    _sparkSession
+    queryExecution.sparkSession
   }
 
   // A globally unique id of this Dataset.
@@ -211,7 +211,7 @@ class Dataset[T] private[sql](
   // you wrap it with `withNewExecutionId` if this actions doesn't call other action.
 
   def this(sparkSession: SparkSession, logicalPlan: LogicalPlan, encoder: Encoder[T]) = {
-    this(sparkSession, sparkSession.sessionState.executePlan(logicalPlan), encoder)
+    this(sparkSession.sessionState.executePlan(logicalPlan), encoder)
   }
 
   def this(sqlContext: SQLContext, logicalPlan: LogicalPlan, encoder: Encoder[T]) = {
@@ -445,7 +445,7 @@ class Dataset[T] private[sql](
    */
   // This is declared with parentheses to prevent the Scala compiler from treating
   // `ds.toDF("1")` as invoking this toDF and then apply on the returned DataFrame.
-  def toDF(): DataFrame = new Dataset[Row](sparkSession, queryExecution, RowEncoder(schema))
+  def toDF(): DataFrame = new Dataset[Row](queryExecution, RowEncoder(schema))
 
   /**
    * Returns a new Dataset where each record has been mapped on to the specified type. The
@@ -503,7 +503,9 @@ class Dataset[T] private[sql](
    * @group basic
    * @since 1.6.0
    */
-  def schema: StructType = queryExecution.analyzed.schema
+  def schema: StructType = sparkSession.withActive {
+    queryExecution.analyzed.schema
+  }
 
   /**
    * Prints the schema to the console in a nice tree format.
@@ -539,7 +541,7 @@ class Dataset[T] private[sql](
    * @group basic
    * @since 3.0.0
    */
-  def explain(mode: String): Unit = {
+  def explain(mode: String): Unit = sparkSession.withActive {
     // Because temporary views are resolved during analysis when we create a Dataset, and
     // `ExplainCommand` analyzes input query plan and resolves temporary views again. Using
     // `ExplainCommand` here will probably output different query plans, compared to the results
@@ -1502,7 +1504,7 @@ class Dataset[T] private[sql](
     val namedColumns =
       columns.map(_.withInputType(exprEnc, logicalPlan.output).named)
     val execution = new QueryExecution(sparkSession, Project(namedColumns, logicalPlan))
-    new Dataset(sparkSession, execution, ExpressionEncoder.tuple(encoders))
+    new Dataset(execution, ExpressionEncoder.tuple(encoders))
   }
 
   /**
@@ -3472,7 +3474,7 @@ class Dataset[T] private[sql](
    * an execution.
    */
   private def withNewExecutionId[U](body: => U): U = {
-    SQLExecution.withNewExecutionId(sparkSession, queryExecution)(body)
+    SQLExecution.withNewExecutionId(queryExecution)(body)
   }
 
   /**
@@ -3481,7 +3483,7 @@ class Dataset[T] private[sql](
    * reset.
    */
   private def withNewRDDExecutionId[U](body: => U): U = {
-    SQLExecution.withNewExecutionId(sparkSession, rddQueryExecution) {
+    SQLExecution.withNewExecutionId(rddQueryExecution) {
       rddQueryExecution.executedPlan.resetMetrics()
       body
     }
@@ -3492,7 +3494,7 @@ class Dataset[T] private[sql](
    * user-registered callback functions.
    */
   private def withAction[U](name: String, qe: QueryExecution)(action: SparkPlan => U) = {
-    SQLExecution.withNewExecutionId(sparkSession, qe, Some(name)) {
+    SQLExecution.withNewExecutionId(qe, Some(name)) {
       qe.executedPlan.resetMetrics()
       action(qe.executedPlan)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
index 89cc9735e4f6a..76ee297dfca79 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala
@@ -449,10 +449,7 @@ class KeyValueGroupedDataset[K, V] private[sql](
     val aggregate = Aggregate(groupingAttributes, keyColumn +: namedColumns, logicalPlan)
     val execution = new QueryExecution(sparkSession, aggregate)
 
-    new Dataset(
-      sparkSession,
-      execution,
-      ExpressionEncoder.tuple(kExprEnc +: encoders))
+    new Dataset(execution, ExpressionEncoder.tuple(kExprEnc +: encoders))
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index abefb348cafc7..1fb97fb4b4cf1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -293,8 +293,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = {
-    SparkSession.setActiveSession(this)
+  def createDataFrame[A <: Product : TypeTag](rdd: RDD[A]): DataFrame = withActive {
     val encoder = Encoders.product[A]
     Dataset.ofRows(self, ExternalRDD(rdd, self)(encoder))
   }
@@ -304,8 +303,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = {
-    SparkSession.setActiveSession(this)
+  def createDataFrame[A <: Product : TypeTag](data: Seq[A]): DataFrame = withActive {
     val schema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
     val attributeSeq = schema.toAttributes
     Dataset.ofRows(self, LocalRelation.fromProduct(attributeSeq, data))
@@ -343,7 +341,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   @DeveloperApi
-  def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = {
+  def createDataFrame(rowRDD: RDD[Row], schema: StructType): DataFrame = withActive {
     // TODO: use MutableProjection when rowRDD is another DataFrame and the applied
     // schema differs from the existing schema on any field data type.
     val encoder = RowEncoder(schema)
@@ -373,7 +371,7 @@ class SparkSession private(
    * @since 2.0.0
    */
   @DeveloperApi
-  def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = {
+  def createDataFrame(rows: java.util.List[Row], schema: StructType): DataFrame = withActive {
     Dataset.ofRows(self, LocalRelation.fromExternalRows(schema.toAttributes, rows.asScala))
   }
 
@@ -385,7 +383,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame = {
+  def createDataFrame(rdd: RDD[_], beanClass: Class[_]): DataFrame = withActive {
     val attributeSeq: Seq[AttributeReference] = getSchema(beanClass)
     val className = beanClass.getName
     val rowRdd = rdd.mapPartitions { iter =>
@@ -414,7 +412,7 @@ class SparkSession private(
    *          SELECT * queries will return the columns in an undefined order.
    * @since 1.6.0
    */
-  def createDataFrame(data: java.util.List[_], beanClass: Class[_]): DataFrame = {
+  def createDataFrame(data: java.util.List[_], beanClass: Class[_]): DataFrame = withActive {
     val attrSeq = getSchema(beanClass)
     val rows = SQLContext.beansToRows(data.asScala.iterator, beanClass, attrSeq)
     Dataset.ofRows(self, LocalRelation(attrSeq, rows.toSeq))
@@ -599,7 +597,7 @@ class SparkSession private(
    *
    * @since 2.0.0
    */
-  def sql(sqlText: String): DataFrame = {
+  def sql(sqlText: String): DataFrame = withActive {
     val tracker = new QueryPlanningTracker
     val plan = tracker.measurePhase(QueryPlanningTracker.PARSING) {
       sessionState.sqlParser.parsePlan(sqlText)
@@ -751,6 +749,20 @@ class SparkSession private(
     }
   }
 
+  /**
+   * Execute a block of code with the this session set as the active session, and restore the
+   * previous session on completion.
+   */
+  private[sql] def withActive[T](block: => T): T = {
+    // Use the active session thread local directly to make sure we get the session that is actually
+    // set and not the default session. This to prevent that we promote the default session to the
+    // active session once we are done.
+    val old = SparkSession.activeThreadSession.get()
+    SparkSession.setActiveSession(this)
+    try block finally {
+      SparkSession.setActiveSession(old)
+    }
+  }
 }
 
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 38ef66682c413..53b6b5d82c021 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -63,13 +63,12 @@ class QueryExecution(
     }
   }
 
-  lazy val analyzed: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.ANALYSIS) {
-    SparkSession.setActiveSession(sparkSession)
+  lazy val analyzed: LogicalPlan = executePhase(QueryPlanningTracker.ANALYSIS) {
     // We can't clone `logical` here, which will reset the `_analyzed` flag.
     sparkSession.sessionState.analyzer.executeAndCheck(logical, tracker)
   }
 
-  lazy val withCachedData: LogicalPlan = {
+  lazy val withCachedData: LogicalPlan = sparkSession.withActive {
     assertAnalyzed()
     assertSupported()
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
@@ -77,20 +76,20 @@ class QueryExecution(
     sparkSession.sharedState.cacheManager.useCachedData(analyzed.clone())
   }
 
-  lazy val optimizedPlan: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.OPTIMIZATION) {
+  lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
     // optimizing and planning.
     sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
   }
 
-  lazy val sparkPlan: SparkPlan = tracker.measurePhase(QueryPlanningTracker.PLANNING) {
+  lazy val sparkPlan: SparkPlan = executePhase(QueryPlanningTracker.PLANNING) {
     // Clone the logical plan here, in case the planner rules change the states of the logical plan.
     QueryExecution.createSparkPlan(sparkSession, planner, optimizedPlan.clone())
   }
 
   // executedPlan should not be used to initialize any SparkPlan. It should be
   // only used for execution.
-  lazy val executedPlan: SparkPlan = tracker.measurePhase(QueryPlanningTracker.PLANNING) {
+  lazy val executedPlan: SparkPlan = executePhase(QueryPlanningTracker.PLANNING) {
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
     // optimizing and planning.
     QueryExecution.prepareForExecution(preparations, sparkPlan.clone())
@@ -116,6 +115,10 @@ class QueryExecution(
     QueryExecution.preparations(sparkSession)
   }
 
+  private def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
+    tracker.measurePhase(phase)(block)
+  }
+
   def simpleString: String = simpleString(false)
 
   def simpleString(formatted: Boolean): String = withRedaction {
@@ -305,7 +308,6 @@ object QueryExecution {
       sparkSession: SparkSession,
       planner: SparkPlanner,
       plan: LogicalPlan): SparkPlan = {
-    SparkSession.setActiveSession(sparkSession)
     // TODO: We use next(), i.e. take the first plan returned by the planner, here for now,
     //       but we will implement to choose the best plan.
     planner.plan(ReturnAnswer(plan)).next()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
index 9f177819f6ea7..59c503e372535 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
@@ -62,9 +62,9 @@ object SQLExecution {
    * we can connect them with an execution.
    */
   def withNewExecutionId[T](
-      sparkSession: SparkSession,
       queryExecution: QueryExecution,
-      name: Option[String] = None)(body: => T): T = {
+      name: Option[String] = None)(body: => T): T = queryExecution.sparkSession.withActive {
+    val sparkSession = queryExecution.sparkSession
     val sc = sparkSession.sparkContext
     val oldExecutionId = sc.getLocalProperty(EXECUTION_ID_KEY)
     val executionId = SQLExecution.nextExecutionId
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index 83bc347e23ed4..45a2ce16183a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -563,11 +563,11 @@ class MicroBatchExecution(
     }
 
     val nextBatch =
-      new Dataset(sparkSessionToRunBatch, lastExecution, RowEncoder(lastExecution.analyzed.schema))
+      new Dataset(lastExecution, RowEncoder(lastExecution.analyzed.schema))
 
     val batchSinkProgress: Option[StreamWriterCommitProgress] =
       reportTimeTaken("addBatch") {
-      SQLExecution.withNewExecutionId(sparkSessionToRunBatch, lastExecution) {
+      SQLExecution.withNewExecutionId(lastExecution) {
         sink match {
           case s: Sink => s.addBatch(currentBatchId, nextBatch)
           case _: SupportsWrite =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
index a9b724a73a18e..a109c2171f3d2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousExecution.scala
@@ -252,7 +252,7 @@ class ContinuousExecution(
 
       updateStatusMessage("Running")
       reportTimeTaken("runContinuous") {
-        SQLExecution.withNewExecutionId(sparkSessionForQuery, lastExecution) {
+        SQLExecution.withNewExecutionId(lastExecution) {
           lastExecution.executedPlan.execute()
         }
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index 3740b56cb9cbb..d3ef03e9b3b74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -520,7 +520,7 @@ private[sql] object CatalogImpl {
     val encoded = data.map(d => enc.toRow(d).copy())
     val plan = new LocalRelation(enc.schema.toAttributes, encoded)
     val queryExecution = sparkSession.sessionState.executePlan(plan)
-    new Dataset[T](sparkSession, queryExecution, enc)
+    new Dataset[T](queryExecution, enc)
   }
 
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 233d67898f909..b0bd612e88d98 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1899,6 +1899,16 @@ class DatasetSuite extends QueryTest
     val e = intercept[AnalysisException](spark.range(1).tail(-1))
     e.getMessage.contains("tail expression must be equal to or greater than 0")
   }
+
+  test("SparkSession.active should be the same instance after dataset operations") {
+    val active = SparkSession.getActiveSession.get
+    val clone = active.cloneSession()
+    val ds = new Dataset(clone, spark.range(10).queryExecution.logical, Encoders.INT)
+
+    ds.queryExecution.analyzed
+
+    assert(active eq SparkSession.getActiveSession.get)
+  }
 }
 
 object AssertExecutionId {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index da4727f6a98cb..83285911b3948 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -511,7 +511,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     val df = session.sql(sql)
     val schema = df.schema.catalogString
     // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
-    val answer = SQLExecution.withNewExecutionId(session, df.queryExecution, Some(sql)) {
+    val answer = SQLExecution.withNewExecutionId(df.queryExecution, Some(sql)) {
       hiveResultString(df.queryExecution.executedPlan).map(replaceNotIncludedMsg)
     }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index 55b551d0af078..9f4a335309b63 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -506,7 +506,7 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       override lazy val executedPlan = physicalPlan
     }
 
-    SQLExecution.withNewExecutionId(spark, dummyQueryExecution) {
+    SQLExecution.withNewExecutionId(dummyQueryExecution) {
       physicalPlan.execute().collect()
     }
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 76d07848f79a9..cf0e5ebf3a2b1 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -295,7 +295,7 @@ private[hive] class SparkExecuteStatementOperation(
           resultList.get.iterator
         }
       }
-      dataTypes = result.queryExecution.analyzed.output.map(_.dataType).toArray
+      dataTypes = result.schema.fields.map(_.dataType)
     } catch {
       // Actually do need to catch Throwable as some failures don't inherit from Exception and
       // HiveServer will silently swallow them.
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 362ac362e9718..12fba0eae6dce 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -61,7 +61,7 @@ private[hive] class SparkSQLDriver(val context: SQLContext = SparkSQLEnv.sqlCont
     try {
       context.sparkContext.setJobDescription(command)
       val execution = context.sessionState.executePlan(context.sql(command).logicalPlan)
-      hiveResponse = SQLExecution.withNewExecutionId(context.sparkSession, execution) {
+      hiveResponse = SQLExecution.withNewExecutionId(execution) {
         hiveResultString(execution.executedPlan)
       }
       tableSchema = getResultSetSchema(execution)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 28e1db961f611..8b1f4c92755b9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -346,8 +346,7 @@ abstract class HiveComparisonTest
         val catalystResults = queryList.zip(hiveResults).map { case (queryString, hive) =>
           val query = new TestHiveQueryExecution(queryString.replace("../../data", testDataPath))
           def getResult(): Seq[String] = {
-            SQLExecution.withNewExecutionId(
-              query.sparkSession, query)(hiveResultString(query.executedPlan))
+            SQLExecution.withNewExecutionId(query)(hiveResultString(query.executedPlan))
           }
           try { (query, prepareAnswer(query, getResult())) } catch {
             case e: Throwable =>
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index cc4592a5caf68..222244a04f5f5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -501,7 +501,7 @@ private[hive] class TestHiveSparkSession(
       // has already set the execution id.
       if (sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) == null) {
         // We don't actually have a `QueryExecution` here, use a fake one instead.
-        SQLExecution.withNewExecutionId(this, new QueryExecution(this, OneRowRelation())) {
+        SQLExecution.withNewExecutionId(new QueryExecution(this, OneRowRelation())) {
           createCmds.foreach(_())
         }
       } else {

From 6001866cea1216da421c5acd71d6fc74228222ac Mon Sep 17 00:00:00 2001
From: sarthfrey-db <sarth.frey@databricks.com>
Date: Thu, 13 Feb 2020 16:15:00 -0800
Subject: [PATCH 0073/1280] [SPARK-30667][CORE] Add allGather method to
 BarrierTaskContext

### What changes were proposed in this pull request?

The `allGather` method is added to the `BarrierTaskContext`. This method contains the same functionality as the `BarrierTaskContext.barrier` method; it blocks the task until all tasks make the call, at which time they may continue execution. In addition, the `allGather` method takes an input message. Upon returning from the `allGather` the task receives a list of all the messages sent by all the tasks that made the `allGather` call.

### Why are the changes needed?

There are many situations where having the tasks communicate in a synchronized way is useful. One simple example is if each task needs to start a server to serve requests from one another; first the tasks must find a free port (the result of which is undetermined beforehand) and then start making requests, but to do so they each must know the port chosen by the other task. An `allGather` method would allow them to inform each other of the port they will run on.

### Does this PR introduce any user-facing change?

Yes, an `BarrierTaskContext.allGather` method will be available through the Scala, Java, and Python APIs.

### How was this patch tested?

Most of the code path is already covered by tests to the `barrier` method, since this PR includes a refactor so that much code is shared by the `barrier` and `allGather` methods. However, a test is added to assert that an all gather on each tasks partition ID will return a list of every partition ID.

An example through the Python API:
```python
>>> from pyspark import BarrierTaskContext
>>>
>>> def f(iterator):
...     context = BarrierTaskContext.get()
...     return [context.allGather('{}'.format(context.partitionId()))]
...
>>> sc.parallelize(range(4), 4).barrier().mapPartitions(f).collect()[0]
[u'3', u'1', u'0', u'2']
```

Closes #27395 from sarthfrey/master.

Lead-authored-by: sarthfrey-db <sarth.frey@databricks.com>
Co-authored-by: sarthfrey <sarth.frey@gmail.com>
Signed-off-by: Xiangrui Meng <meng@databricks.com>
(cherry picked from commit 57254c9719f9af9ad985596ed7fbbaafa4052002)
Signed-off-by: Xiangrui Meng <meng@databricks.com>
---
 .../org/apache/spark/BarrierCoordinator.scala | 113 +++++++++++--
 .../org/apache/spark/BarrierTaskContext.scala | 153 ++++++++++++------
 .../spark/api/python/PythonRunner.scala       |  51 ++++--
 .../scheduler/BarrierTaskContextSuite.scala   |  74 +++++++++
 python/pyspark/taskcontext.py                 |  49 +++++-
 python/pyspark/tests/test_taskcontext.py      |  20 +++
 6 files changed, 381 insertions(+), 79 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 4e417679ca663..042a2664a0e27 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -17,12 +17,17 @@
 
 package org.apache.spark
 
+import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
 import scala.collection.mutable.ArrayBuffer
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
@@ -99,10 +104,15 @@ private[spark] class BarrierCoordinator(
     // reset when a barrier() call fails due to timeout.
     private var barrierEpoch: Int = 0
 
-    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()
-    // call.
+    // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
+    // An Array of allGather messages for barrier tasks that have made a blocking runBarrier() call
+    private val allGatherMessages: ArrayBuffer[String] = new Array[String](numTasks).to[ArrayBuffer]
+
+    // The blocking requestMethod called by tasks to sync up for this stage attempt
+    private var requestMethodToSync: RequestMethod.Value = RequestMethod.BARRIER
+
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
 
@@ -130,9 +140,32 @@ private[spark] class BarrierCoordinator(
 
     // Process the global sync request. The barrier() call succeed if collected enough requests
     // within a configured time, otherwise fail all the pending requests.
-    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
+    def handleRequest(
+      requester: RpcCallContext,
+      request: RequestToSync
+    ): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
+      val requestMethod = request.requestMethod
+      val partitionId = request.partitionId
+      val allGatherMessage = request match {
+        case ag: AllGatherRequestToSync => ag.allGatherMessage
+        case _ => ""
+      }
+
+      if (requesters.size == 0) {
+        requestMethodToSync = requestMethod
+      }
+
+      if (requestMethodToSync != requestMethod) {
+        requesters.foreach(
+          _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
+            s"`$requestMethod` during barrier epoch $barrierEpoch, which does not match " +
+            s"the current synchronized requestMethod `$requestMethodToSync`"
+          ))
+        )
+        cleanupBarrierStage(barrierId)
+      }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
       require(request.numTasks == numTasks, s"Number of tasks of $barrierId is " +
@@ -153,6 +186,7 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
+        allGatherMessages(partitionId) = allGatherMessage
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
         if (maybeFinishAllRequesters(requesters, numTasks)) {
@@ -162,6 +196,7 @@ private[spark] class BarrierCoordinator(
             s"tasks, finished successfully.")
           barrierEpoch += 1
           requesters.clear()
+          allGatherMessages.clear()
           cancelTimerTask()
         }
       }
@@ -173,7 +208,13 @@ private[spark] class BarrierCoordinator(
         requesters: ArrayBuffer[RpcCallContext],
         numTasks: Int): Boolean = {
       if (requesters.size == numTasks) {
-        requesters.foreach(_.reply(()))
+        requestMethodToSync match {
+          case RequestMethod.BARRIER =>
+            requesters.foreach(_.reply(""))
+          case RequestMethod.ALL_GATHER =>
+            val json: String = compact(render(allGatherMessages))
+            requesters.foreach(_.reply(json))
+        }
         true
       } else {
         false
@@ -186,6 +227,7 @@ private[spark] class BarrierCoordinator(
       // messages come from current stage attempt shall fail.
       barrierEpoch = -1
       requesters.clear()
+      allGatherMessages.clear()
       cancelTimerTask()
     }
   }
@@ -199,11 +241,11 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>
+    case request: RequestToSync =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
-      val barrierId = ContextBarrierId(stageId, stageAttemptId)
+      val barrierId = ContextBarrierId(request.stageId, request.stageAttemptId)
       states.computeIfAbsent(barrierId,
-        (key: ContextBarrierId) => new ContextBarrierState(key, numTasks))
+        (key: ContextBarrierId) => new ContextBarrierState(key, request.numTasks))
       val barrierState = states.get(barrierId)
 
       barrierState.handleRequest(context, request)
@@ -216,6 +258,16 @@ private[spark] class BarrierCoordinator(
 
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
+private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
+  def numTasks: Int
+  def stageId: Int
+  def stageAttemptId: Int
+  def taskAttemptId: Long
+  def barrierEpoch: Int
+  def partitionId: Int
+  def requestMethod: RequestMethod.Value
+}
+
 /**
  * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
@@ -224,11 +276,44 @@ private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls.
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
  */
-private[spark] case class RequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptId: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int) extends BarrierCoordinatorMessage
+private[spark] case class BarrierRequestToSync(
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  requestMethod: RequestMethod.Value
+) extends RequestToSync
+
+/**
+ * A global sync request message from BarrierTaskContext, by `allGather()` call. Each request is
+ * identified by stageId + stageAttemptId + barrierEpoch.
+ *
+ * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
+ * @param stageId ID of current stage
+ * @param stageAttemptId ID of current stage attempt
+ * @param taskAttemptId Unique ID of current task
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
+ * @param allGatherMessage Message sent from the BarrierTaskContext if requestMethod is ALL_GATHER
+ */
+private[spark] case class AllGatherRequestToSync(
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  requestMethod: RequestMethod.Value,
+  allGatherMessage: String
+) extends RequestToSync
+
+private[spark] object RequestMethod extends Enumeration {
+  val BARRIER, ALL_GATHER = Value
+}
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 3d369802f3023..2263538a11676 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -17,11 +17,19 @@
 
 package org.apache.spark
 
+import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.parse
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -59,49 +67,31 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  /**
-   * :: Experimental ::
-   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
-   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
-   * stage have reached this routine.
-   *
-   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
-   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
-   * timeout. Some examples of '''misuses''' are listed below:
-   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
-   * shall lead to timeout of the function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       if (context.partitionId() == 0) {
-   *           // Do nothing.
-   *       } else {
-   *           context.barrier()
-   *       }
-   *       iter
-   *   }
-   * }}}
-   *
-   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
-   * second function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       try {
-   *           // Do something that might throw an Exception.
-   *           doSomething()
-   *           context.barrier()
-   *       } catch {
-   *           case e: Exception => logWarning("...", e)
-   *       }
-   *       context.barrier()
-   *       iter
-   *   }
-   * }}}
-   */
-  @Experimental
-  @Since("2.4.0")
-  def barrier(): Unit = {
+  private def getRequestToSync(
+    numTasks: Int,
+    stageId: Int,
+    stageAttemptNumber: Int,
+    taskAttemptId: Long,
+    barrierEpoch: Int,
+    partitionId: Int,
+    requestMethod: RequestMethod.Value,
+    allGatherMessage: String
+  ): RequestToSync = {
+    requestMethod match {
+      case RequestMethod.BARRIER =>
+        BarrierRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, requestMethod)
+      case RequestMethod.ALL_GATHER =>
+        AllGatherRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, requestMethod, allGatherMessage)
+    }
+  }
+
+  private def runBarrier(
+    requestMethod: RequestMethod.Value,
+    allGatherMessage: String = ""
+  ): String = {
+
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -118,10 +108,12 @@ class BarrierTaskContext private[spark] (
     // Log the update of global sync every 60 seconds.
     timer.schedule(timerTask, 60000, 60000)
 
+    var json: String = ""
+
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[Unit](
-        message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch),
+      val abortableRpcFuture = barrierCoordinator.askAbortable[String](
+        message = getRequestToSync(numTasks, stageId, stageAttemptNumber,
+          taskAttemptId, barrierEpoch, partitionId, requestMethod, allGatherMessage),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
@@ -133,7 +125,7 @@ class BarrierTaskContext private[spark] (
         while (!abortableRpcFuture.toFuture.isCompleted) {
           // wait RPC future for at most 1 second
           try {
-            ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
+            json = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
           } catch {
             case _: TimeoutException | _: InterruptedException =>
               // If `TimeoutException` thrown, waiting RPC future reach 1 second.
@@ -163,6 +155,73 @@ class BarrierTaskContext private[spark] (
       timerTask.cancel()
       timer.purge()
     }
+    json
+  }
+
+  /**
+   * :: Experimental ::
+   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
+   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
+   * stage have reached this routine.
+   *
+   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
+   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
+   * timeout. Some examples of '''misuses''' are listed below:
+   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
+   * shall lead to timeout of the function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       if (context.partitionId() == 0) {
+   *           // Do nothing.
+   *       } else {
+   *           context.barrier()
+   *       }
+   *       iter
+   *   }
+   * }}}
+   *
+   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
+   * second function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       try {
+   *           // Do something that might throw an Exception.
+   *           doSomething()
+   *           context.barrier()
+   *       } catch {
+   *           case e: Exception => logWarning("...", e)
+   *       }
+   *       context.barrier()
+   *       iter
+   *   }
+   * }}}
+   */
+  @Experimental
+  @Since("2.4.0")
+  def barrier(): Unit = {
+    runBarrier(RequestMethod.BARRIER)
+    ()
+  }
+
+  /**
+   * :: Experimental ::
+   * Blocks until all tasks in the same stage have reached this routine. Each task passes in
+   * a message and returns with a list of all the messages passed in by each of those tasks.
+   *
+   * CAUTION! The allGather method requires the same precautions as the barrier method
+   *
+   * The message is type String rather than Array[Byte] because it is more convenient for
+   * the user at the cost of worse performance.
+   */
+  @Experimental
+  @Since("3.0.0")
+  def allGather(message: String): ArrayBuffer[String] = {
+    val json = runBarrier(RequestMethod.ALL_GATHER, message)
+    val jsonArray = parse(json)
+    implicit val formats = DefaultFormats
+    ArrayBuffer(jsonArray.extract[Array[String]]: _*)
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 658e0d593a167..fa8bf0fc06358 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -24,8 +24,13 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -238,13 +243,18 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
                   sock.setSoTimeout(10000)
                   authHelper.authClient(sock)
                   val input = new DataInputStream(sock.getInputStream())
-                  input.readInt() match {
+                  val requestMethod = input.readInt()
+                  // The BarrierTaskContext function may wait infinitely, socket shall not timeout
+                  // before the function finishes.
+                  sock.setSoTimeout(0)
+                  requestMethod match {
                     case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-                      // The barrier() function may wait infinitely, socket shall not timeout
-                      // before the function finishes.
-                      sock.setSoTimeout(0)
-                      barrierAndServe(sock)
-
+                      barrierAndServe(requestMethod, sock)
+                    case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+                      val length = input.readInt()
+                      val message = new Array[Byte](length)
+                      input.readFully(message)
+                      barrierAndServe(requestMethod, sock, new String(message, UTF_8))
                     case _ =>
                       val out = new DataOutputStream(new BufferedOutputStream(
                         sock.getOutputStream))
@@ -395,15 +405,31 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     /**
-     * Gateway to call BarrierTaskContext.barrier().
+     * Gateway to call BarrierTaskContext methods.
      */
-    def barrierAndServe(sock: Socket): Unit = {
-      require(serverSocket.isDefined, "No available ServerSocket to redirect the barrier() call.")
-
+    def barrierAndServe(requestMethod: Int, sock: Socket, message: String = ""): Unit = {
+      require(
+        serverSocket.isDefined,
+        "No available ServerSocket to redirect the BarrierTaskContext method call."
+      )
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        context.asInstanceOf[BarrierTaskContext].barrier()
-        writeUTF(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS, out)
+        var result: String = ""
+        requestMethod match {
+          case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
+            context.asInstanceOf[BarrierTaskContext].barrier()
+            result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
+          case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+            val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
+              message
+            )
+            result = compact(render(JArray(
+              messages.map(
+                (message) => JString(message)
+              ).toList
+            )))
+        }
+        writeUTF(result, out)
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
@@ -638,6 +664,7 @@ private[spark] object SpecialLengths {
 
 private[spark] object BarrierTaskContextMessageProtocol {
   val BARRIER_FUNCTION = 1
+  val ALL_GATHER_FUNCTION = 2
   val BARRIER_RESULT_SUCCESS = "success"
   val ERROR_UNRECOGNIZED_FUNCTION = "Not recognized function call from python side."
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index fc8ac38479932..ed38b7f0ecac1 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
+import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
 import org.apache.spark._
@@ -52,6 +53,79 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     assert(times.max - times.min <= 1000)
   }
 
+  test("share messages with allGather() call") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message = context.partitionId().toString
+      val messages = context.allGather(message)
+      messages.toList.iterator
+    }
+    // Take a sorted list of all the partitionId messages
+    val messages = rdd2.collect().head
+    // All the task partitionIds are shared
+    for((x, i) <- messages.view.zipWithIndex) assert(x == i.toString)
+  }
+
+  test("throw exception if we attempt to synchronize with different blocking calls") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      val partitionId = context.partitionId
+      if (partitionId == 0) {
+        context.barrier()
+      } else {
+        context.allGather(partitionId.toString)
+      }
+      Seq(null).iterator
+    }
+    val error = intercept[SparkException] {
+      rdd2.collect()
+    }.getMessage
+    assert(error.contains("does not match the current synchronized requestMethod"))
+  }
+
+  test("successively sync with allGather and barrier") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      context.barrier()
+      val time1 = System.currentTimeMillis()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message = context.partitionId().toString
+      val messages = context.allGather(message)
+      val time2 = System.currentTimeMillis()
+      Seq((time1, time2)).iterator
+    }
+    val times = rdd2.collect()
+    // All the tasks shall finish the first round of global sync within a short time slot.
+    val times1 = times.map(_._1)
+    assert(times1.max - times1.min <= 1000)
+
+    // All the tasks shall finish the second round of global sync within a short time slot.
+    val times2 = times.map(_._2)
+    assert(times2.max - times2.min <= 1000)
+  }
+
   test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index d648f63338514..90bd2345ac525 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -16,9 +16,10 @@
 #
 
 from __future__ import print_function
+import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, UTF8Deserializer
+from pyspark.serializers import write_int, write_with_length, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -107,18 +108,28 @@ def resources(self):
 
 
 BARRIER_FUNCTION = 1
+ALL_GATHER_FUNCTION = 2
 
 
-def _load_from_socket(port, auth_secret):
+def _load_from_socket(port, auth_secret, function, all_gather_message=None):
     """
     Load data from a given socket, this is a blocking method thus only return when the socket
     connection has been closed.
     """
     (sockfile, sock) = local_connect_and_auth(port, auth_secret)
-    # The barrier() call may block forever, so no timeout
+
+    # The call may block forever, so no timeout
     sock.settimeout(None)
-    # Make a barrier() function call.
-    write_int(BARRIER_FUNCTION, sockfile)
+
+    if function == BARRIER_FUNCTION:
+        # Make a barrier() function call.
+        write_int(function, sockfile)
+    elif function == ALL_GATHER_FUNCTION:
+        # Make a all_gather() function call.
+        write_int(function, sockfile)
+        write_with_length(all_gather_message.encode("utf-8"), sockfile)
+    else:
+        raise ValueError("Unrecognized function type")
     sockfile.flush()
 
     # Collect result.
@@ -199,7 +210,33 @@ def barrier(self):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            _load_from_socket(self._port, self._secret)
+            _load_from_socket(self._port, self._secret, BARRIER_FUNCTION)
+
+    def allGather(self, message=""):
+        """
+        .. note:: Experimental
+
+        This function blocks until all tasks in the same stage have reached this routine.
+        Each task passes in a message and returns with a list of all the messages passed in
+        by each of those tasks.
+
+        .. warning:: In a barrier stage, each task much have the same number of `allGather()`
+            calls, in all possible code branches.
+            Otherwise, you may get the job hanging or a SparkException after timeout.
+        """
+        if not isinstance(message, str):
+            raise ValueError("Argument `message` must be of type `str`")
+        elif self._port is None or self._secret is None:
+            raise Exception("Not supported to call barrier() before initialize " +
+                            "BarrierTaskContext.")
+        else:
+            gathered_items = _load_from_socket(
+                self._port,
+                self._secret,
+                ALL_GATHER_FUNCTION,
+                message,
+            )
+            return [e for e in json.loads(gathered_items)]
 
     def getTaskInfos(self):
         """
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index 6095a384679af..f5dbd068387c2 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -134,6 +134,26 @@ def context_barrier(x):
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
         self.assertTrue(max(times) - min(times) < 1)
 
+    def test_all_gather(self):
+        """
+        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
+        within a stage and passes messages properly.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 10))
+            out = tc.allGather(str(context.partitionId()))
+            pids = [int(e) for e in out]
+            return [pids]
+
+        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
+        self.assertTrue(pids == [0, 1, 2, 3])
+
     def test_barrier_infos(self):
         """
         Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the

From eb37aa5595badd79becf4d3d332404cbcdb1b12d Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Thu, 13 Feb 2020 17:48:19 -0800
Subject: [PATCH 0074/1280] Revert "[SPARK-30667][CORE] Add allGather method to
 BarrierTaskContext"

This reverts commit 6001866cea1216da421c5acd71d6fc74228222ac.
---
 .../org/apache/spark/BarrierCoordinator.scala | 113 ++-----------
 .../org/apache/spark/BarrierTaskContext.scala | 153 ++++++------------
 .../spark/api/python/PythonRunner.scala       |  51 ++----
 .../scheduler/BarrierTaskContextSuite.scala   |  74 ---------
 python/pyspark/taskcontext.py                 |  49 +-----
 python/pyspark/tests/test_taskcontext.py      |  20 ---
 6 files changed, 79 insertions(+), 381 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 042a2664a0e27..4e417679ca663 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -17,17 +17,12 @@
 
 package org.apache.spark
 
-import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.{compact, render}
-
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
@@ -104,15 +99,10 @@ private[spark] class BarrierCoordinator(
     // reset when a barrier() call fails due to timeout.
     private var barrierEpoch: Int = 0
 
-    // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
+    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()
+    // call.
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
-    // An Array of allGather messages for barrier tasks that have made a blocking runBarrier() call
-    private val allGatherMessages: ArrayBuffer[String] = new Array[String](numTasks).to[ArrayBuffer]
-
-    // The blocking requestMethod called by tasks to sync up for this stage attempt
-    private var requestMethodToSync: RequestMethod.Value = RequestMethod.BARRIER
-
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
 
@@ -140,32 +130,9 @@ private[spark] class BarrierCoordinator(
 
     // Process the global sync request. The barrier() call succeed if collected enough requests
     // within a configured time, otherwise fail all the pending requests.
-    def handleRequest(
-      requester: RpcCallContext,
-      request: RequestToSync
-    ): Unit = synchronized {
+    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
-      val requestMethod = request.requestMethod
-      val partitionId = request.partitionId
-      val allGatherMessage = request match {
-        case ag: AllGatherRequestToSync => ag.allGatherMessage
-        case _ => ""
-      }
-
-      if (requesters.size == 0) {
-        requestMethodToSync = requestMethod
-      }
-
-      if (requestMethodToSync != requestMethod) {
-        requesters.foreach(
-          _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
-            s"`$requestMethod` during barrier epoch $barrierEpoch, which does not match " +
-            s"the current synchronized requestMethod `$requestMethodToSync`"
-          ))
-        )
-        cleanupBarrierStage(barrierId)
-      }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
       require(request.numTasks == numTasks, s"Number of tasks of $barrierId is " +
@@ -186,7 +153,6 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
-        allGatherMessages(partitionId) = allGatherMessage
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
         if (maybeFinishAllRequesters(requesters, numTasks)) {
@@ -196,7 +162,6 @@ private[spark] class BarrierCoordinator(
             s"tasks, finished successfully.")
           barrierEpoch += 1
           requesters.clear()
-          allGatherMessages.clear()
           cancelTimerTask()
         }
       }
@@ -208,13 +173,7 @@ private[spark] class BarrierCoordinator(
         requesters: ArrayBuffer[RpcCallContext],
         numTasks: Int): Boolean = {
       if (requesters.size == numTasks) {
-        requestMethodToSync match {
-          case RequestMethod.BARRIER =>
-            requesters.foreach(_.reply(""))
-          case RequestMethod.ALL_GATHER =>
-            val json: String = compact(render(allGatherMessages))
-            requesters.foreach(_.reply(json))
-        }
+        requesters.foreach(_.reply(()))
         true
       } else {
         false
@@ -227,7 +186,6 @@ private[spark] class BarrierCoordinator(
       // messages come from current stage attempt shall fail.
       barrierEpoch = -1
       requesters.clear()
-      allGatherMessages.clear()
       cancelTimerTask()
     }
   }
@@ -241,11 +199,11 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request: RequestToSync =>
+    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
-      val barrierId = ContextBarrierId(request.stageId, request.stageAttemptId)
+      val barrierId = ContextBarrierId(stageId, stageAttemptId)
       states.computeIfAbsent(barrierId,
-        (key: ContextBarrierId) => new ContextBarrierState(key, request.numTasks))
+        (key: ContextBarrierId) => new ContextBarrierState(key, numTasks))
       val barrierState = states.get(barrierId)
 
       barrierState.handleRequest(context, request)
@@ -258,16 +216,6 @@ private[spark] class BarrierCoordinator(
 
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
-private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
-  def numTasks: Int
-  def stageId: Int
-  def stageAttemptId: Int
-  def taskAttemptId: Long
-  def barrierEpoch: Int
-  def partitionId: Int
-  def requestMethod: RequestMethod.Value
-}
-
 /**
  * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
@@ -276,44 +224,11 @@ private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
- * @param partitionId ID of the current partition the task is assigned to
- * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls.
  */
-private[spark] case class BarrierRequestToSync(
-  numTasks: Int,
-  stageId: Int,
-  stageAttemptId: Int,
-  taskAttemptId: Long,
-  barrierEpoch: Int,
-  partitionId: Int,
-  requestMethod: RequestMethod.Value
-) extends RequestToSync
-
-/**
- * A global sync request message from BarrierTaskContext, by `allGather()` call. Each request is
- * identified by stageId + stageAttemptId + barrierEpoch.
- *
- * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
- * @param stageId ID of current stage
- * @param stageAttemptId ID of current stage attempt
- * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
- * @param partitionId ID of the current partition the task is assigned to
- * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
- * @param allGatherMessage Message sent from the BarrierTaskContext if requestMethod is ALL_GATHER
- */
-private[spark] case class AllGatherRequestToSync(
-  numTasks: Int,
-  stageId: Int,
-  stageAttemptId: Int,
-  taskAttemptId: Long,
-  barrierEpoch: Int,
-  partitionId: Int,
-  requestMethod: RequestMethod.Value,
-  allGatherMessage: String
-) extends RequestToSync
-
-private[spark] object RequestMethod extends Enumeration {
-  val BARRIER, ALL_GATHER = Value
-}
+private[spark] case class RequestToSync(
+    numTasks: Int,
+    stageId: Int,
+    stageAttemptId: Int,
+    taskAttemptId: Long,
+    barrierEpoch: Int) extends BarrierCoordinatorMessage
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 2263538a11676..3d369802f3023 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -17,19 +17,11 @@
 
 package org.apache.spark
 
-import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.json4s.DefaultFormats
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.parse
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -67,31 +59,49 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  private def getRequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptNumber: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int,
-    partitionId: Int,
-    requestMethod: RequestMethod.Value,
-    allGatherMessage: String
-  ): RequestToSync = {
-    requestMethod match {
-      case RequestMethod.BARRIER =>
-        BarrierRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch, partitionId, requestMethod)
-      case RequestMethod.ALL_GATHER =>
-        AllGatherRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch, partitionId, requestMethod, allGatherMessage)
-    }
-  }
-
-  private def runBarrier(
-    requestMethod: RequestMethod.Value,
-    allGatherMessage: String = ""
-  ): String = {
-
+  /**
+   * :: Experimental ::
+   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
+   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
+   * stage have reached this routine.
+   *
+   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
+   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
+   * timeout. Some examples of '''misuses''' are listed below:
+   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
+   * shall lead to timeout of the function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       if (context.partitionId() == 0) {
+   *           // Do nothing.
+   *       } else {
+   *           context.barrier()
+   *       }
+   *       iter
+   *   }
+   * }}}
+   *
+   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
+   * second function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       try {
+   *           // Do something that might throw an Exception.
+   *           doSomething()
+   *           context.barrier()
+   *       } catch {
+   *           case e: Exception => logWarning("...", e)
+   *       }
+   *       context.barrier()
+   *       iter
+   *   }
+   * }}}
+   */
+  @Experimental
+  @Since("2.4.0")
+  def barrier(): Unit = {
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -108,12 +118,10 @@ class BarrierTaskContext private[spark] (
     // Log the update of global sync every 60 seconds.
     timer.schedule(timerTask, 60000, 60000)
 
-    var json: String = ""
-
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[String](
-        message = getRequestToSync(numTasks, stageId, stageAttemptNumber,
-          taskAttemptId, barrierEpoch, partitionId, requestMethod, allGatherMessage),
+      val abortableRpcFuture = barrierCoordinator.askAbortable[Unit](
+        message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
@@ -125,7 +133,7 @@ class BarrierTaskContext private[spark] (
         while (!abortableRpcFuture.toFuture.isCompleted) {
           // wait RPC future for at most 1 second
           try {
-            json = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
+            ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
           } catch {
             case _: TimeoutException | _: InterruptedException =>
               // If `TimeoutException` thrown, waiting RPC future reach 1 second.
@@ -155,73 +163,6 @@ class BarrierTaskContext private[spark] (
       timerTask.cancel()
       timer.purge()
     }
-    json
-  }
-
-  /**
-   * :: Experimental ::
-   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
-   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
-   * stage have reached this routine.
-   *
-   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
-   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
-   * timeout. Some examples of '''misuses''' are listed below:
-   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
-   * shall lead to timeout of the function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       if (context.partitionId() == 0) {
-   *           // Do nothing.
-   *       } else {
-   *           context.barrier()
-   *       }
-   *       iter
-   *   }
-   * }}}
-   *
-   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
-   * second function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       try {
-   *           // Do something that might throw an Exception.
-   *           doSomething()
-   *           context.barrier()
-   *       } catch {
-   *           case e: Exception => logWarning("...", e)
-   *       }
-   *       context.barrier()
-   *       iter
-   *   }
-   * }}}
-   */
-  @Experimental
-  @Since("2.4.0")
-  def barrier(): Unit = {
-    runBarrier(RequestMethod.BARRIER)
-    ()
-  }
-
-  /**
-   * :: Experimental ::
-   * Blocks until all tasks in the same stage have reached this routine. Each task passes in
-   * a message and returns with a list of all the messages passed in by each of those tasks.
-   *
-   * CAUTION! The allGather method requires the same precautions as the barrier method
-   *
-   * The message is type String rather than Array[Byte] because it is more convenient for
-   * the user at the cost of worse performance.
-   */
-  @Experimental
-  @Since("3.0.0")
-  def allGather(message: String): ArrayBuffer[String] = {
-    val json = runBarrier(RequestMethod.ALL_GATHER, message)
-    val jsonArray = parse(json)
-    implicit val formats = DefaultFormats
-    ArrayBuffer(jsonArray.extract[Array[String]]: _*)
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index fa8bf0fc06358..658e0d593a167 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -24,13 +24,8 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.{compact, render}
-
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -243,18 +238,13 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
                   sock.setSoTimeout(10000)
                   authHelper.authClient(sock)
                   val input = new DataInputStream(sock.getInputStream())
-                  val requestMethod = input.readInt()
-                  // The BarrierTaskContext function may wait infinitely, socket shall not timeout
-                  // before the function finishes.
-                  sock.setSoTimeout(0)
-                  requestMethod match {
+                  input.readInt() match {
                     case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-                      barrierAndServe(requestMethod, sock)
-                    case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
-                      val length = input.readInt()
-                      val message = new Array[Byte](length)
-                      input.readFully(message)
-                      barrierAndServe(requestMethod, sock, new String(message, UTF_8))
+                      // The barrier() function may wait infinitely, socket shall not timeout
+                      // before the function finishes.
+                      sock.setSoTimeout(0)
+                      barrierAndServe(sock)
+
                     case _ =>
                       val out = new DataOutputStream(new BufferedOutputStream(
                         sock.getOutputStream))
@@ -405,31 +395,15 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     /**
-     * Gateway to call BarrierTaskContext methods.
+     * Gateway to call BarrierTaskContext.barrier().
      */
-    def barrierAndServe(requestMethod: Int, sock: Socket, message: String = ""): Unit = {
-      require(
-        serverSocket.isDefined,
-        "No available ServerSocket to redirect the BarrierTaskContext method call."
-      )
+    def barrierAndServe(sock: Socket): Unit = {
+      require(serverSocket.isDefined, "No available ServerSocket to redirect the barrier() call.")
+
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        var result: String = ""
-        requestMethod match {
-          case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-            context.asInstanceOf[BarrierTaskContext].barrier()
-            result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
-          case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
-            val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
-              message
-            )
-            result = compact(render(JArray(
-              messages.map(
-                (message) => JString(message)
-              ).toList
-            )))
-        }
-        writeUTF(result, out)
+        context.asInstanceOf[BarrierTaskContext].barrier()
+        writeUTF(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS, out)
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
@@ -664,7 +638,6 @@ private[spark] object SpecialLengths {
 
 private[spark] object BarrierTaskContextMessageProtocol {
   val BARRIER_FUNCTION = 1
-  val ALL_GATHER_FUNCTION = 2
   val BARRIER_RESULT_SUCCESS = "success"
   val ERROR_UNRECOGNIZED_FUNCTION = "Not recognized function call from python side."
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index ed38b7f0ecac1..fc8ac38479932 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
-import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
 import org.apache.spark._
@@ -53,79 +52,6 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     assert(times.max - times.min <= 1000)
   }
 
-  test("share messages with allGather() call") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
-    val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { it =>
-      val context = BarrierTaskContext.get()
-      // Sleep for a random time before global sync.
-      Thread.sleep(Random.nextInt(1000))
-      // Pass partitionId message in
-      val message = context.partitionId().toString
-      val messages = context.allGather(message)
-      messages.toList.iterator
-    }
-    // Take a sorted list of all the partitionId messages
-    val messages = rdd2.collect().head
-    // All the task partitionIds are shared
-    for((x, i) <- messages.view.zipWithIndex) assert(x == i.toString)
-  }
-
-  test("throw exception if we attempt to synchronize with different blocking calls") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
-    val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { it =>
-      val context = BarrierTaskContext.get()
-      val partitionId = context.partitionId
-      if (partitionId == 0) {
-        context.barrier()
-      } else {
-        context.allGather(partitionId.toString)
-      }
-      Seq(null).iterator
-    }
-    val error = intercept[SparkException] {
-      rdd2.collect()
-    }.getMessage
-    assert(error.contains("does not match the current synchronized requestMethod"))
-  }
-
-  test("successively sync with allGather and barrier") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
-    val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { it =>
-      val context = BarrierTaskContext.get()
-      // Sleep for a random time before global sync.
-      Thread.sleep(Random.nextInt(1000))
-      context.barrier()
-      val time1 = System.currentTimeMillis()
-      // Sleep for a random time before global sync.
-      Thread.sleep(Random.nextInt(1000))
-      // Pass partitionId message in
-      val message = context.partitionId().toString
-      val messages = context.allGather(message)
-      val time2 = System.currentTimeMillis()
-      Seq((time1, time2)).iterator
-    }
-    val times = rdd2.collect()
-    // All the tasks shall finish the first round of global sync within a short time slot.
-    val times1 = times.map(_._1)
-    assert(times1.max - times1.min <= 1000)
-
-    // All the tasks shall finish the second round of global sync within a short time slot.
-    val times2 = times.map(_._2)
-    assert(times2.max - times2.min <= 1000)
-  }
-
   test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index 90bd2345ac525..d648f63338514 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -16,10 +16,9 @@
 #
 
 from __future__ import print_function
-import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, write_with_length, UTF8Deserializer
+from pyspark.serializers import write_int, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -108,28 +107,18 @@ def resources(self):
 
 
 BARRIER_FUNCTION = 1
-ALL_GATHER_FUNCTION = 2
 
 
-def _load_from_socket(port, auth_secret, function, all_gather_message=None):
+def _load_from_socket(port, auth_secret):
     """
     Load data from a given socket, this is a blocking method thus only return when the socket
     connection has been closed.
     """
     (sockfile, sock) = local_connect_and_auth(port, auth_secret)
-
-    # The call may block forever, so no timeout
+    # The barrier() call may block forever, so no timeout
     sock.settimeout(None)
-
-    if function == BARRIER_FUNCTION:
-        # Make a barrier() function call.
-        write_int(function, sockfile)
-    elif function == ALL_GATHER_FUNCTION:
-        # Make a all_gather() function call.
-        write_int(function, sockfile)
-        write_with_length(all_gather_message.encode("utf-8"), sockfile)
-    else:
-        raise ValueError("Unrecognized function type")
+    # Make a barrier() function call.
+    write_int(BARRIER_FUNCTION, sockfile)
     sockfile.flush()
 
     # Collect result.
@@ -210,33 +199,7 @@ def barrier(self):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            _load_from_socket(self._port, self._secret, BARRIER_FUNCTION)
-
-    def allGather(self, message=""):
-        """
-        .. note:: Experimental
-
-        This function blocks until all tasks in the same stage have reached this routine.
-        Each task passes in a message and returns with a list of all the messages passed in
-        by each of those tasks.
-
-        .. warning:: In a barrier stage, each task much have the same number of `allGather()`
-            calls, in all possible code branches.
-            Otherwise, you may get the job hanging or a SparkException after timeout.
-        """
-        if not isinstance(message, str):
-            raise ValueError("Argument `message` must be of type `str`")
-        elif self._port is None or self._secret is None:
-            raise Exception("Not supported to call barrier() before initialize " +
-                            "BarrierTaskContext.")
-        else:
-            gathered_items = _load_from_socket(
-                self._port,
-                self._secret,
-                ALL_GATHER_FUNCTION,
-                message,
-            )
-            return [e for e in json.loads(gathered_items)]
+            _load_from_socket(self._port, self._secret)
 
     def getTaskInfos(self):
         """
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index f5dbd068387c2..6095a384679af 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -134,26 +134,6 @@ def context_barrier(x):
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
         self.assertTrue(max(times) - min(times) < 1)
 
-    def test_all_gather(self):
-        """
-        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
-        within a stage and passes messages properly.
-        """
-        rdd = self.sc.parallelize(range(10), 4)
-
-        def f(iterator):
-            yield sum(iterator)
-
-        def context_barrier(x):
-            tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
-            out = tc.allGather(str(context.partitionId()))
-            pids = [int(e) for e in out]
-            return [pids]
-
-        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
-        self.assertTrue(pids == [0, 1, 2, 3])
-
     def test_barrier_infos(self):
         """
         Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the

From 35539cad17fd2b425ba8f7a7e298e9805541aa73 Mon Sep 17 00:00:00 2001
From: David Toneian <david@toneian.com>
Date: Fri, 14 Feb 2020 11:00:35 +0900
Subject: [PATCH 0075/1280] [PYSPARK][DOCS][MINOR] Changed `:func:` to `:attr:`
 Sphinx roles, fixed links in documentation of
 `Data{Frame,Stream}{Reader,Writer}`

This commit is published into the public domain.

### What changes were proposed in this pull request?
This PR fixes the documentation of `DataFrameReader`, `DataFrameWriter`, `DataStreamReader`, and `DataStreamWriter`, where attributes of other classes were misrepresented as functions. Additionally, creation of hyperlinks across modules was fixed in these instances.

### Why are the changes needed?
The old state produced documentation that suggested invalid usage of PySpark objects (accessing attributes as though they were callable.)

### Does this PR introduce any user-facing change?
No, except for improved documentation.

### How was this patch tested?
No test added; documentation build runs through.

Closes #27553 from DavidToneian/docfix-DataFrameReader-DataFrameWriter-DataStreamReader-DataStreamWriter.

Authored-by: David Toneian <david@toneian.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 25db8c71a2100c167b8c2d7a6c540ebc61db9b73)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/readwriter.py |  4 ++--
 python/pyspark/sql/streaming.py  | 11 ++++++-----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 3d3280dbd9943..69660395ad823 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -48,7 +48,7 @@ def _set_opts(self, schema=None, **options):
 class DataFrameReader(OptionUtils):
     """
     Interface used to load a :class:`DataFrame` from external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`spark.read`
+    (e.g. file systems, key-value stores, etc). Use :attr:`SparkSession.read`
     to access this.
 
     .. versionadded:: 1.4
@@ -616,7 +616,7 @@ def jdbc(self, url, table, column=None, lowerBound=None, upperBound=None, numPar
 class DataFrameWriter(OptionUtils):
     """
     Interface used to write a :class:`DataFrame` to external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`DataFrame.write`
+    (e.g. file systems, key-value stores, etc). Use :attr:`DataFrame.write`
     to access this.
 
     .. versionadded:: 1.4
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index f17a52f6b3dc8..5fced8aca9bdf 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -276,9 +276,9 @@ def resetTerminated(self):
 
 class DataStreamReader(OptionUtils):
     """
-    Interface used to load a streaming :class:`DataFrame` from external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`spark.readStream`
-    to access this.
+    Interface used to load a streaming :class:`DataFrame <pyspark.sql.DataFrame>` from external
+    storage systems (e.g. file systems, key-value stores, etc).
+    Use :attr:`SparkSession.readStream <pyspark.sql.SparkSession.readStream>` to access this.
 
     .. note:: Evolving.
 
@@ -750,8 +750,9 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
 
 class DataStreamWriter(object):
     """
-    Interface used to write a streaming :class:`DataFrame` to external storage systems
-    (e.g. file systems, key-value stores, etc). Use :func:`DataFrame.writeStream`
+    Interface used to write a streaming :class:`DataFrame <pyspark.sql.DataFrame>` to external
+    storage systems (e.g. file systems, key-value stores, etc).
+    Use :attr:`DataFrame.writeStream <pyspark.sql.DataFrame.writeStream>`
     to access this.
 
     .. note:: Evolving.

From febe28517df715c7f3c3e6efdc45dc647f811411 Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Fri, 14 Feb 2020 11:20:55 +0800
Subject: [PATCH 0076/1280] [SPARK-30801][SQL] Subqueries should not be AQE-ed
 if main query is not

### What changes were proposed in this pull request?
This PR makes sure AQE is either enabled or disabled for the entire query, including the main query and all subqueries.
Currently there are unsupported queries by AQE, e.g., queries that contain DPP filters. We need to make sure that if the main query is unsupported, none of the sub-queries should apply AQE, otherwise it can lead to performance regressions due to missed opportunity of sub-query reuse.

### Why are the changes needed?
To get rid of potential perf regressions when AQE is turned on.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Updated DynamicPartitionPruningSuite:
1. Removed the existing workaround `withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")`
2. Added `DynamicPartitionPruningSuiteAEOn` and `DynamicPartitionPruningSuiteAEOff` to enable testing this suite with AQE on and off options
3. Added a check in `checkPartitionPruningPredicate` to verify that the subqueries are always in sync with the main query in terms of whether AQE is applied.

Closes #27554 from maryannxue/spark-30801.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 0aed77a0155b404e39bc5dbc0579e29e4c7bf887)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/QueryExecution.scala  | 19 ++++++++++---
 .../sql/DynamicPartitionPruningSuite.scala    | 27 ++++++++++++++++---
 2 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 53b6b5d82c021..9109c05e75853 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -274,13 +274,25 @@ object QueryExecution {
    * are correct, insert whole stage code gen, and try to reduce the work done by reusing exchanges
    * and subqueries.
    */
-  private[execution] def preparations(sparkSession: SparkSession): Seq[Rule[SparkPlan]] =
+  private[execution] def preparations(sparkSession: SparkSession): Seq[Rule[SparkPlan]] = {
+
+    val sparkSessionWithAdaptiveExecutionOff =
+    if (sparkSession.sessionState.conf.adaptiveExecutionEnabled) {
+      val session = sparkSession.cloneSession()
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      session
+    } else {
+      sparkSession
+    }
+
     Seq(
       // `AdaptiveSparkPlanExec` is a leaf node. If inserted, all the following rules will be no-op
       // as the original plan is hidden behind `AdaptiveSparkPlanExec`.
       InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession)),
-      PlanDynamicPruningFilters(sparkSession),
-      PlanSubqueries(sparkSession),
+      // If the following rules apply, it means the main query is not AQE-ed, so we make sure the
+      // subqueries are not AQE-ed either.
+      PlanDynamicPruningFilters(sparkSessionWithAdaptiveExecutionOff),
+      PlanSubqueries(sparkSessionWithAdaptiveExecutionOff),
       EnsureRequirements(sparkSession.sessionState.conf),
       ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
         sparkSession.sessionState.columnarRules),
@@ -288,6 +300,7 @@ object QueryExecution {
       ReuseExchange(sparkSession.sessionState.conf),
       ReuseSubquery(sparkSession.sessionState.conf)
     )
+  }
 
   /**
    * Prepares a planned [[SparkPlan]] for execution by inserting shuffle operations and internal
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index f7b51d6f4c8ef..baa9f5ecafc68 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -22,7 +22,7 @@ import org.scalatest.GivenWhenThen
 import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
 import org.apache.spark.sql.catalyst.plans.ExistenceJoin
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec}
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWrapper}
@@ -33,7 +33,7 @@ import org.apache.spark.sql.test.SharedSparkSession
 /**
  * Test suite for the filtering ratio policy used to trigger dynamic partition pruning (DPP).
  */
-class DynamicPartitionPruningSuite
+abstract class DynamicPartitionPruningSuiteBase
     extends QueryTest
     with SharedSparkSession
     with GivenWhenThen
@@ -43,9 +43,14 @@ class DynamicPartitionPruningSuite
 
   import testImplicits._
 
+  val adaptiveExecutionOn: Boolean
+
   override def beforeAll(): Unit = {
     super.beforeAll()
 
+    spark.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, adaptiveExecutionOn)
+    spark.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY, true)
+
     val factData = Seq[(Int, Int, Int, Int)](
       (1000, 1, 1, 10),
       (1010, 2, 1, 10),
@@ -153,6 +158,8 @@ class DynamicPartitionPruningSuite
       sql("DROP TABLE IF EXISTS fact_stats")
       sql("DROP TABLE IF EXISTS dim_stats")
     } finally {
+      spark.sessionState.conf.unsetConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)
+      spark.sessionState.conf.unsetConf(SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY)
       super.afterAll()
     }
   }
@@ -195,6 +202,11 @@ class DynamicPartitionPruningSuite
           fail(s"Invalid child node found in\n$s")
       }
     }
+
+    val isMainQueryAdaptive = plan.isInstanceOf[AdaptiveSparkPlanExec]
+    subqueriesAll(plan).filterNot(subqueryBroadcast.contains).foreach { s =>
+      assert(s.find(_.isInstanceOf[AdaptiveSparkPlanExec]).isDefined == isMainQueryAdaptive)
+    }
   }
 
   /**
@@ -1173,8 +1185,7 @@ class DynamicPartitionPruningSuite
   }
 
   test("join key with multiple references on the filtering plan") {
-    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
-      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       // when enable AQE, the reusedExchange is inserted when executed.
       withTable("fact", "dim") {
         spark.range(100).select(
@@ -1270,3 +1281,11 @@ class DynamicPartitionPruningSuite
     }
   }
 }
+
+class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase {
+  override val adaptiveExecutionOn: Boolean = false
+}
+
+class DynamicPartitionPruningSuiteAEOn extends DynamicPartitionPruningSuiteBase {
+  override val adaptiveExecutionOn: Boolean = true
+}

From 1a29f9fd0c2e304f93fcd9bce3cd038ee278c937 Mon Sep 17 00:00:00 2001
From: David Toneian <david@toneian.com>
Date: Fri, 14 Feb 2020 13:49:11 +0900
Subject: [PATCH 0077/1280] [SPARK-30823][PYTHON][DOCS] Set `%PYTHONPATH%` when
 building PySpark documentation on Windows
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit is published into the public domain.

### What changes were proposed in this pull request?
In analogy to `python/docs/Makefile`, which has
> export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.8.1-src.zip)

on line 10, this PR adds
> set PYTHONPATH=..;..\lib\py4j-0.10.8.1-src.zip

to `make2.bat`.

Since there is no `realpath` in default installations of Windows, I left the relative paths unresolved. Per the instructions on how to build docs, `make.bat` is supposed to be run from `python/docs` as the working directory, so this should probably not cause issues (`%BUILDDIR%` is a relative path as well.)

### Why are the changes needed?
When building the PySpark documentation on Windows, by changing directory to `python/docs` and running `make.bat` (which runs `make2.bat`), the majority of the documentation may not be built if pyspark is not in the default `%PYTHONPATH%`. Sphinx then reports that `pyspark` (and possibly dependencies) cannot be imported.

If `pyspark` is in the default `%PYTHONPATH%`, I suppose it is that version of `pyspark` – as opposed to the version found above the `python/docs` directory – that is considered when building the documentation, which may result in documentation that does not correspond to the development version one is trying to build.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Manual tests on my Windows 10 machine. Additional tests with other environments very welcome!

Closes #27569 from DavidToneian/SPARK-30823.

Authored-by: David Toneian <david@toneian.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b2134ee73cfad4d78aaf8f0a2898011ac0308e48)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/docs/make2.bat | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/docs/make2.bat b/python/docs/make2.bat
index 05d22eb5cdd23..742df373166da 100644
--- a/python/docs/make2.bat
+++ b/python/docs/make2.bat
@@ -2,6 +2,8 @@
 
 REM Command file for Sphinx documentation
 
+set PYTHONPATH=..;..\lib\py4j-0.10.8.1-src.zip
+
 
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build

From 0dcc4df0ca5dba8ae09388b95969080ca28cbe16 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 14 Feb 2020 16:52:28 +0800
Subject: [PATCH 0078/1280] [SPARK-25990][SQL] ScriptTransformation should
 handle different data types correctly

### What changes were proposed in this pull request?

We should convert Spark InternalRows to hive data via `HiveInspectors.wrapperFor`.

### Why are the changes needed?

We may hit below exception without this change:

```
[info]    org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 1 times, most recent failure: Lost task 0.0 in stage 1.0 (TID 1, 192.168.1.6, executor driver): java.lang.ClassCastException: org.apache.spark.sql.types.Decimal cannot be cast to org.apache.hadoop.hive.common.type.HiveDecimal
[info]   	at org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaHiveDecimalObjectInspector.getPrimitiveJavaObject(JavaHiveDecimalObjectInspector.java:55)
[info]   	at org.apache.hadoop.hive.serde2.lazy.LazyUtils.writePrimitiveUTF8(LazyUtils.java:321)
[info]   	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:292)
[info]   	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:247)
[info]   	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.doSerialize(LazySimpleSerDe.java:231)
[info]   	at org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.serialize(AbstractEncodingAwareSerDe.java:55)
[info]   	at org.apache.spark.sql.hive.execution.ScriptTransformationWriterThread.$anonfun$run$2(ScriptTransformationExec.scala:300)
[info]   	at org.apache.spark.sql.hive.execution.ScriptTransformationWriterThread.$anonfun$run$2$adapted(ScriptTransformationExec.scala:281)
[info]   	at scala.collection.Iterator.foreach(Iterator.scala:941)
[info]   	at scala.collection.Iterator.foreach$(Iterator.scala:941)
[info]   	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
[info]   	at org.apache.spark.sql.hive.execution.ScriptTransformationWriterThread.$anonfun$run$1(ScriptTransformationExec.scala:281)
[info]   	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[info]   	at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1932)
[info]   	at org.apache.spark.sql.hive.execution.ScriptTransformationWriterThread.run(ScriptTransformationExec.scala:270)
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added new test. But please note that this test returns different result between Hive1.2 and Hive2.3 due to `HiveDecimal` or `SerDe` difference(don't know the root cause yet).

Closes #27556 from Ngone51/script_transform.

Lead-authored-by: yi.wu <yi.wu@databricks.com>
Co-authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 99b8136a86030411e6bcbd312f40eb2a901ab0f0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../execution/ScriptTransformationExec.scala  | 32 ++++++++-----
 sql/hive/src/test/resources/test_script.py    | 21 +++++++++
 .../execution/ScriptTransformationSuite.scala | 46 ++++++++++++++++++-
 3 files changed, 85 insertions(+), 14 deletions(-)
 create mode 100644 sql/hive/src/test/resources/test_script.py

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
index e12f663304e7a..40f7b4e8db7c5 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
@@ -94,9 +94,8 @@ case class ScriptTransformationExec(
       // This new thread will consume the ScriptTransformation's input rows and write them to the
       // external process. That process's output will be read by this current thread.
       val writerThread = new ScriptTransformationWriterThread(
-        inputIterator,
+        inputIterator.map(outputProjection),
         input.map(_.dataType),
-        outputProjection,
         inputSerde,
         inputSoi,
         ioschema,
@@ -249,16 +248,15 @@ case class ScriptTransformationExec(
 private class ScriptTransformationWriterThread(
     iter: Iterator[InternalRow],
     inputSchema: Seq[DataType],
-    outputProjection: Projection,
     @Nullable inputSerde: AbstractSerDe,
-    @Nullable inputSoi: ObjectInspector,
+    @Nullable inputSoi: StructObjectInspector,
     ioschema: HiveScriptIOSchema,
     outputStream: OutputStream,
     proc: Process,
     stderrBuffer: CircularBuffer,
     taskContext: TaskContext,
     conf: Configuration
-  ) extends Thread("Thread-ScriptTransformation-Feed") with Logging {
+  ) extends Thread("Thread-ScriptTransformation-Feed") with HiveInspectors with Logging {
 
   setDaemon(true)
 
@@ -278,8 +276,8 @@ private class ScriptTransformationWriterThread(
     var threwException: Boolean = true
     val len = inputSchema.length
     try {
-      iter.map(outputProjection).foreach { row =>
-        if (inputSerde == null) {
+      if (inputSerde == null) {
+        iter.foreach { row =>
           val data = if (len == 0) {
             ioschema.inputRowFormatMap("TOK_TABLEROWFORMATLINES")
           } else {
@@ -295,10 +293,21 @@ private class ScriptTransformationWriterThread(
             sb.toString()
           }
           outputStream.write(data.getBytes(StandardCharsets.UTF_8))
-        } else {
-          val writable = inputSerde.serialize(
-            row.asInstanceOf[GenericInternalRow].values, inputSoi)
+        }
+      } else {
+        // Convert Spark InternalRows to hive data via `HiveInspectors.wrapperFor`.
+        val hiveData = new Array[Any](inputSchema.length)
+        val fieldOIs = inputSoi.getAllStructFieldRefs.asScala.map(_.getFieldObjectInspector).toArray
+        val wrappers = fieldOIs.zip(inputSchema).map { case (f, dt) => wrapperFor(f, dt) }
+
+        iter.foreach { row =>
+          var i = 0
+          while (i < fieldOIs.length) {
+            hiveData(i) = if (row.isNullAt(i)) null else wrappers(i)(row.get(i, inputSchema(i)))
+            i += 1
+          }
 
+          val writable = inputSerde.serialize(hiveData, inputSoi)
           if (scriptInputWriter != null) {
             scriptInputWriter.write(writable)
           } else {
@@ -374,14 +383,13 @@ case class HiveScriptIOSchema (
   val outputRowFormatMap = outputRowFormat.toMap.withDefault((k) => defaultFormat(k))
 
 
-  def initInputSerDe(input: Seq[Expression]): Option[(AbstractSerDe, ObjectInspector)] = {
+  def initInputSerDe(input: Seq[Expression]): Option[(AbstractSerDe, StructObjectInspector)] = {
     inputSerdeClass.map { serdeClass =>
       val (columns, columnTypes) = parseAttrs(input)
       val serde = initSerDe(serdeClass, columns, columnTypes, inputSerdeProps)
       val fieldObjectInspectors = columnTypes.map(toInspector)
       val objectInspector = ObjectInspectorFactory
         .getStandardStructObjectInspector(columns.asJava, fieldObjectInspectors.asJava)
-        .asInstanceOf[ObjectInspector]
       (serde, objectInspector)
     }
   }
diff --git a/sql/hive/src/test/resources/test_script.py b/sql/hive/src/test/resources/test_script.py
new file mode 100644
index 0000000000000..82ef7b38f0c1b
--- /dev/null
+++ b/sql/hive/src/test/resources/test_script.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+for line in sys.stdin:
+    (a, b, c, d, e) = line.split('\t')
+    sys.stdout.write('\t'.join([a, b, c, d, e]))
+    sys.stdout.flush()
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
index 80a50c18bcb93..7d01fc53a4099 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.hive.execution
 
+import java.sql.Timestamp
+
 import org.apache.hadoop.hive.serde2.`lazy`.LazySimpleSerDe
 import org.scalatest.Assertions._
 import org.scalatest.BeforeAndAfterEach
@@ -24,15 +26,18 @@ import org.scalatest.exceptions.TestFailedException
 
 import org.apache.spark.{SparkException, TaskContext, TestUtils}
 import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.{SparkPlan, SparkPlanTest, UnaryExecNode}
+import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types.StringType
 
-class ScriptTransformationSuite extends SparkPlanTest with TestHiveSingleton with
-  BeforeAndAfterEach {
+class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with TestHiveSingleton
+  with BeforeAndAfterEach {
   import spark.implicits._
 
   private val noSerdeIOSchema = HiveScriptIOSchema(
@@ -186,6 +191,43 @@ class ScriptTransformationSuite extends SparkPlanTest with TestHiveSingleton wit
       rowsDf.select("name").collect())
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
+
+  test("SPARK-25990: TRANSFORM should handle different data types correctly") {
+    assume(TestUtils.testCommandAvailable("python"))
+    val scriptFilePath = getTestResourcePath("test_script.py")
+
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      val query = sql(
+        s"""
+          |SELECT
+          |TRANSFORM(a, b, c, d, e)
+          |USING 'python $scriptFilePath' AS (a, b, c, d, e)
+          |FROM v
+        """.stripMargin)
+
+      // In Hive1.2, it does not do well on Decimal conversion. For example, in this case,
+      // it converts a decimal value's type from Decimal(38, 18) to Decimal(1, 0). So we need
+      // do extra cast here for Hive1.2. But in Hive2.3, it still keeps the original Decimal type.
+      val decimalToString: Column => Column = if (HiveUtils.isHive23) {
+        c => c.cast("string")
+      } else {
+        c => c.cast("decimal(1, 0)").cast("string")
+      }
+      checkAnswer(query, identity, df.select(
+        'a.cast("string"),
+        'b.cast("string"),
+        'c.cast("string"),
+        decimalToString('d),
+        'e.cast("string")).collect())
+    }
+  }
 }
 
 private case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {

From 79ce79234f02092e22fdd79e859d83f5a174ef95 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 14 Feb 2020 18:20:18 +0800
Subject: [PATCH 0079/1280] [SPARK-30810][SQL] Parses and convert a CSV Dataset
 having different column from 'value' in csv(dataset) API

### What changes were proposed in this pull request?

This PR fixes `DataFrameReader.csv(dataset: Dataset[String])` API to take a `Dataset[String]` originated from a column name different from `value`. This is a long-standing bug started from the very first place.

`CSVUtils.filterCommentAndEmpty` assumed the `Dataset[String]` to be originated with `value` column. This PR changes to use the first column name in the schema.

### Why are the changes needed?

For  `DataFrameReader.csv(dataset: Dataset[String])` to support any `Dataset[String]` as the signature indicates.

### Does this PR introduce any user-facing change?
Yes,

```scala
val ds = spark.range(2).selectExpr("concat('a,b,', id) AS text").as[String]
spark.read.option("header", true).option("inferSchema", true).csv(ds).show()
```

Before:

```
org.apache.spark.sql.AnalysisException: cannot resolve '`value`' given input columns: [text];;
'Filter (length(trim('value, None)) > 0)
+- Project [concat(a,b,, cast(id#0L as string)) AS text#2]
   +- Range (0, 2, step=1, splits=Some(2))
```

After:

```
+---+---+---+
|  a|  b|  0|
+---+---+---+
|  a|  b|  1|
+---+---+---+
```

### How was this patch tested?

Unittest was added.

Closes #27561 from HyukjinKwon/SPARK-30810.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2a270a731a3b1da9a0fb036d648dd522e5c4d5ad)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/datasources/csv/CSVUtils.scala     | 7 ++++---
 .../spark/sql/execution/datasources/csv/CSVSuite.scala     | 7 +++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
index 21fabac472f4b..d8b52c503ad34 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala
@@ -33,11 +33,12 @@ object CSVUtils {
     // with the one below, `filterCommentAndEmpty` but execution path is different. One of them
     // might have to be removed in the near future if possible.
     import lines.sqlContext.implicits._
-    val nonEmptyLines = lines.filter(length(trim($"value")) > 0)
+    val aliased = lines.toDF("value")
+    val nonEmptyLines = aliased.filter(length(trim($"value")) > 0)
     if (options.isCommentSet) {
-      nonEmptyLines.filter(!$"value".startsWith(options.comment.toString))
+      nonEmptyLines.filter(!$"value".startsWith(options.comment.toString)).as[String]
     } else {
-      nonEmptyLines
+      nonEmptyLines.as[String]
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index b1105b4a63bba..0be0e1e3da3dc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2294,6 +2294,13 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       }
     }
   }
+
+  test("SPARK-30810: parses and convert a CSV Dataset having different column from 'value'") {
+    val ds = spark.range(2).selectExpr("concat('a,b,', id) AS `a.text`").as[String]
+    val csv = spark.read.option("header", true).option("inferSchema", true).csv(ds)
+    assert(csv.schema.fieldNames === Seq("a", "b", "0"))
+    checkAnswer(csv, Row("a", "b", 1))
+  }
 }
 
 class CSVv1Suite extends CSVSuite {

From 0a8d7a35e24acbd7af57fe5169691afb8ccd8675 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 14 Feb 2020 22:16:57 +0800
Subject: [PATCH 0080/1280] [SPARK-30766][SQL] Fix the timestamp truncation to
 the `HOUR` and `DAY` levels

### What changes were proposed in this pull request?
In the PR, I propose to use Java 8 time API in timestamp truncations to the levels of `HOUR` and `DAY`. The problem is in the usage of `timeZone.getOffset(millis)` in days/hours truncations where the combined calendar (Julian + Gregorian) is used underneath.

### Why are the changes needed?
The change fix wrong truncations. For example, the following truncation to hours should print `0010-01-01 01:00:00` but it outputs wrong timestamp:
```scala
Seq("0010-01-01 01:02:03.123456").toDF()
    .select($"value".cast("timestamp").as("ts"))
    .select(date_trunc("HOUR", $"ts").cast("string"))
    .show(false)
+------------------------------------+
|CAST(date_trunc(HOUR, ts) AS STRING)|
+------------------------------------+
|0010-01-01 01:30:17                 |
+------------------------------------+
```

### Does this PR introduce any user-facing change?
Yes. After the changes, the result of the example above is:
```scala
+------------------------------------+
|CAST(date_trunc(HOUR, ts) AS STRING)|
+------------------------------------+
|0010-01-01 01:00:00                 |
+------------------------------------+
```

### How was this patch tested?
- Added new test to `DateFunctionsSuite`
- By `DateExpressionsSuite` and `DateTimeUtilsSuite`

Closes #27512 from MaxGekk/fix-trunc-old-timestamp.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 7137a6d065edeaab97bf5bf49ffaca3d060a14fe)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/datetimeExpressions.scala     |  6 +--
 .../sql/catalyst/util/DateTimeUtils.scala     | 44 ++++++++++---------
 .../catalyst/util/DateTimeUtilsSuite.scala    | 39 ++++++++--------
 .../apache/spark/sql/DateFunctionsSuite.scala | 13 ++++++
 4 files changed, 59 insertions(+), 43 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index cf91489d8e6b7..adf7251256041 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -1690,15 +1690,15 @@ case class TruncTimestamp(
 
   override def eval(input: InternalRow): Any = {
     evalHelper(input, minLevel = MIN_LEVEL_OF_TIMESTAMP_TRUNC) { (t: Any, level: Int) =>
-      DateTimeUtils.truncTimestamp(t.asInstanceOf[Long], level, timeZone)
+      DateTimeUtils.truncTimestamp(t.asInstanceOf[Long], level, zoneId)
     }
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     codeGenHelper(ctx, ev, minLevel = MIN_LEVEL_OF_TIMESTAMP_TRUNC, true) {
       (date: String, fmt: String) =>
-        s"truncTimestamp($date, $fmt, $tz);"
+        s"truncTimestamp($date, $fmt, $zid);"
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 01d36f19fc06f..ce0c138791f30 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -711,32 +711,34 @@ object DateTimeUtils {
     }
   }
 
+  private def truncToUnit(t: SQLTimestamp, zoneId: ZoneId, unit: ChronoUnit): SQLTimestamp = {
+    val truncated = microsToInstant(t).atZone(zoneId).truncatedTo(unit)
+    instantToMicros(truncated.toInstant)
+  }
+
   /**
    * Returns the trunc date time from original date time and trunc level.
    * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 12.
    */
-  def truncTimestamp(t: SQLTimestamp, level: Int, timeZone: TimeZone): SQLTimestamp = {
-    if (level == TRUNC_TO_MICROSECOND) return t
-    var millis = MICROSECONDS.toMillis(t)
-    val truncated = level match {
-      case TRUNC_TO_MILLISECOND => millis
-      case TRUNC_TO_SECOND =>
-        millis - millis % MILLIS_PER_SECOND
-      case TRUNC_TO_MINUTE =>
-        millis - millis % MILLIS_PER_MINUTE
-      case TRUNC_TO_HOUR =>
-        val offset = timeZone.getOffset(millis)
-        millis += offset
-        millis - millis % MILLIS_PER_HOUR - offset
-      case TRUNC_TO_DAY =>
-        val offset = timeZone.getOffset(millis)
-        millis += offset
-        millis - millis % MILLIS_PER_DAY - offset
-      case _ => // Try to truncate date levels
-        val dDays = millisToDays(millis, timeZone.toZoneId)
-        daysToMillis(truncDate(dDays, level), timeZone.toZoneId)
+  def truncTimestamp(t: SQLTimestamp, level: Int, zoneId: ZoneId): SQLTimestamp = {
+    level match {
+      case TRUNC_TO_MICROSECOND => t
+      case TRUNC_TO_HOUR => truncToUnit(t, zoneId, ChronoUnit.HOURS)
+      case TRUNC_TO_DAY => truncToUnit(t, zoneId, ChronoUnit.DAYS)
+      case _ =>
+        val millis = MICROSECONDS.toMillis(t)
+        val truncated = level match {
+          case TRUNC_TO_MILLISECOND => millis
+          case TRUNC_TO_SECOND =>
+            millis - millis % MILLIS_PER_SECOND
+          case TRUNC_TO_MINUTE =>
+            millis - millis % MILLIS_PER_MINUTE
+          case _ => // Try to truncate date levels
+            val dDays = millisToDays(millis, zoneId)
+            daysToMillis(truncDate(dDays, level), zoneId)
+        }
+        truncated * MICROS_PER_MILLIS
     }
-    truncated * MICROS_PER_MILLIS
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index cd0594c775a47..ff4d8a2457922 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -499,9 +499,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         level: Int,
         expected: String,
         inputTS: SQLTimestamp,
-        timezone: TimeZone = DateTimeUtils.defaultTimeZone()): Unit = {
+        zoneId: ZoneId = defaultZoneId): Unit = {
       val truncated =
-        DateTimeUtils.truncTimestamp(inputTS, level, timezone)
+        DateTimeUtils.truncTimestamp(inputTS, level, zoneId)
       val expectedTS = toTimestamp(expected, defaultZoneId)
       assert(truncated === expectedTS.get)
     }
@@ -539,6 +539,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
     for (tz <- ALL_TIMEZONES) {
       withDefaultTimeZone(tz) {
+        val zid = tz.toZoneId
         val inputTS = DateTimeUtils.stringToTimestamp(
           UTF8String.fromString("2015-03-05T09:32:05.359"), defaultZoneId)
         val inputTS1 = DateTimeUtils.stringToTimestamp(
@@ -552,23 +553,23 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         val inputTS5 = DateTimeUtils.stringToTimestamp(
           UTF8String.fromString("1999-03-29T01:02:03.456789"), defaultZoneId)
 
-        testTrunc(DateTimeUtils.TRUNC_TO_YEAR, "2015-01-01T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_MONTH, "2015-03-01T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_DAY, "2015-03-05T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_HOUR, "2015-03-05T09:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "2015-03-05T09:32:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "2015-03-05T09:32:05", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-02T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS1.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS2.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS3.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-23T00:00:00", inputTS4.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS1.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", inputTS2.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_DECADE, "1990-01-01", inputTS5.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "1901-01-01", inputTS5.get, tz)
-        testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", inputTS.get, tz)
+        testTrunc(DateTimeUtils.TRUNC_TO_YEAR, "2015-01-01T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MONTH, "2015-03-01T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_DAY, "2015-03-05T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_HOUR, "2015-03-05T09:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "2015-03-05T09:32:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "2015-03-05T09:32:05", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-02T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS1.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS2.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-30T00:00:00", inputTS3.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_WEEK, "2015-03-23T00:00:00", inputTS4.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS1.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", inputTS2.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_DECADE, "1990-01-01", inputTS5.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "1901-01-01", inputTS5.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", inputTS.get, zid)
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 41d53c959ef99..ba45b9f9b62df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -856,4 +856,17 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       TimeZone.setDefault(defaultTz)
     }
   }
+
+  test("SPARK-30766: date_trunc of old timestamps to hours and days") {
+    def checkTrunc(level: String, expected: String): Unit = {
+      val df = Seq("0010-01-01 01:02:03.123456")
+        .toDF()
+        .select($"value".cast("timestamp").as("ts"))
+        .select(date_trunc(level, $"ts").cast("string"))
+      checkAnswer(df, Row(expected))
+    }
+
+    checkTrunc("HOUR", "0010-01-01 01:00:00")
+    checkTrunc("DAY", "0010-01-01 00:00:00")
+  }
 }

From 1385fc02ce7d28e6570971e1687e74d245a5533f Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 14 Feb 2020 10:18:08 -0800
Subject: [PATCH 0081/1280] [SPARK-29748][DOCS][FOLLOW-UP] Add a note that the
 legacy environment variable to set in both executor and driver

### What changes were proposed in this pull request?

This PR address the comment at https://github.com/apache/spark/pull/26496#discussion_r379194091 and improves the migration guide to explicitly note that the legacy environment variable to set in both executor and driver.

### Why are the changes needed?

To clarify this env should be set both in driver and executors.

### Does this PR introduce any user-facing change?

Nope.

### How was this patch tested?

I checked it via md editor.

Closes #27573 from HyukjinKwon/SPARK-29748.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
(cherry picked from commit b343757b1bd5d0344b82f36aa4d65ed34f840606)
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
---
 docs/pyspark-migration-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index 8ea4fec75edf8..f7f20389aa694 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -87,7 +87,7 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
   - Since Spark 3.0, `Column.getItem` is fixed such that it does not call `Column.apply`. Consequently, if `Column` is used as an argument to `getItem`, the indexing operator should be used.
     For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
 
-  - As of Spark 3.0 `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to "true". For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
+  - As of Spark 3.0 `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to "true" for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
 
 ## Upgrading from PySpark 2.3 to 2.4
 

From 2824fec9fa57444b7c64edb8226cf75bb87a2e5d Mon Sep 17 00:00:00 2001
From: DB Tsai <d_tsai@apple.com>
Date: Fri, 14 Feb 2020 21:46:01 +0000
Subject: [PATCH 0082/1280] [SPARK-30289][SQL] Partitioned by Nested Column for
 `InMemoryTable`

### What changes were proposed in this pull request?
1. `InMemoryTable` was flatting the nested columns, and then the flatten columns was used to look up the indices which is not correct.

This PR implements partitioned by nested column for `InMemoryTable`.

### Why are the changes needed?

This PR implements partitioned by nested column for `InMemoryTable`, so we can test this features in DSv2

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing unit tests and new tests.

Closes #26929 from dbtsai/addTests.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit d0f961476031b62bda0d4d41f7248295d651ea92)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 .../spark/sql/connector/InMemoryTable.scala   | 35 ++++++--
 .../spark/sql/DataFrameWriterV2Suite.scala    | 86 ++++++++++++++++++-
 2 files changed, 114 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
index c9e4e0aad5704..0187ae31e2d1c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
@@ -26,7 +26,7 @@ import org.scalatest.Assertions._
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.catalog._
-import org.apache.spark.sql.connector.expressions.{IdentityTransform, Transform}
+import org.apache.spark.sql.connector.expressions.{IdentityTransform, NamedReference, Transform}
 import org.apache.spark.sql.connector.read._
 import org.apache.spark.sql.connector.write._
 import org.apache.spark.sql.sources.{And, EqualTo, Filter, IsNotNull}
@@ -59,10 +59,30 @@ class InMemoryTable(
 
   def rows: Seq[InternalRow] = dataMap.values.flatMap(_.rows).toSeq
 
-  private val partFieldNames = partitioning.flatMap(_.references).toSeq.flatMap(_.fieldNames)
-  private val partIndexes = partFieldNames.map(schema.fieldIndex)
+  private val partCols: Array[Array[String]] = partitioning.flatMap(_.references).map { ref =>
+    schema.findNestedField(ref.fieldNames(), includeCollections = false) match {
+      case Some(_) => ref.fieldNames()
+      case None => throw new IllegalArgumentException(s"${ref.describe()} does not exist.")
+    }
+  }
 
-  private def getKey(row: InternalRow): Seq[Any] = partIndexes.map(row.toSeq(schema)(_))
+  private def getKey(row: InternalRow): Seq[Any] = {
+    def extractor(fieldNames: Array[String], schema: StructType, row: InternalRow): Any = {
+      val index = schema.fieldIndex(fieldNames(0))
+      val value = row.toSeq(schema).apply(index)
+      if (fieldNames.length > 1) {
+        (value, schema(index).dataType) match {
+          case (row: InternalRow, nestedSchema: StructType) =>
+            extractor(fieldNames.drop(1), nestedSchema, row)
+          case (_, dataType) =>
+            throw new IllegalArgumentException(s"Unsupported type, ${dataType.simpleString}")
+        }
+      } else {
+        value
+      }
+    }
+    partCols.map(fieldNames => extractor(fieldNames, schema, row))
+  }
 
   def withData(data: Array[BufferedRows]): InMemoryTable = dataMap.synchronized {
     data.foreach(_.rows.foreach { row =>
@@ -146,8 +166,10 @@ class InMemoryTable(
   }
 
   private class Overwrite(filters: Array[Filter]) extends TestBatchWrite {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
     override def commit(messages: Array[WriterCommitMessage]): Unit = dataMap.synchronized {
-      val deleteKeys = InMemoryTable.filtersToKeys(dataMap.keys, partFieldNames, filters)
+      val deleteKeys = InMemoryTable.filtersToKeys(
+        dataMap.keys, partCols.map(_.toSeq.quoted), filters)
       dataMap --= deleteKeys
       withData(messages.map(_.asInstanceOf[BufferedRows]))
     }
@@ -161,7 +183,8 @@ class InMemoryTable(
   }
 
   override def deleteWhere(filters: Array[Filter]): Unit = dataMap.synchronized {
-    dataMap --= InMemoryTable.filtersToKeys(dataMap.keys, partFieldNames, filters)
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+    dataMap --= InMemoryTable.filtersToKeys(dataMap.keys, partCols.map(_.toSeq.quoted), filters)
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
index d49dc58e93ddb..cd157086a8b8e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
@@ -17,20 +17,24 @@
 
 package org.apache.spark.sql
 
+import java.sql.Timestamp
+
 import scala.collection.JavaConverters._
 
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic}
-import org.apache.spark.sql.connector.InMemoryTableCatalog
+import org.apache.spark.sql.connector.{InMemoryTable, InMemoryTableCatalog}
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 import org.apache.spark.sql.connector.expressions.{BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, YearsTransform}
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
+import org.apache.spark.sql.types.TimestampType
 import org.apache.spark.sql.util.QueryExecutionListener
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
 
 class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with BeforeAndAfter {
@@ -550,4 +554,84 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     assert(replaced.partitioning.isEmpty)
     assert(replaced.properties === defaultOwnership.asJava)
   }
+
+  test("SPARK-30289 Create: partitioned by nested column") {
+    val schema = new StructType().add("ts", new StructType()
+      .add("created", TimestampType)
+      .add("modified", TimestampType)
+      .add("timezone", StringType))
+
+    val data = Seq(
+      Row(Row(Timestamp.valueOf("2019-06-01 10:00:00"), Timestamp.valueOf("2019-09-02 07:00:00"),
+        "America/Los_Angeles")),
+      Row(Row(Timestamp.valueOf("2019-08-26 18:00:00"), Timestamp.valueOf("2019-09-26 18:00:00"),
+        "America/Los_Angeles")),
+      Row(Row(Timestamp.valueOf("2018-11-23 18:00:00"), Timestamp.valueOf("2018-12-22 18:00:00"),
+        "America/New_York")))
+    val df = spark.createDataFrame(spark.sparkContext.parallelize(data, 1), schema)
+
+    df.writeTo("testcat.table_name")
+      .partitionedBy($"ts.timezone")
+      .create()
+
+    val table = catalog("testcat").loadTable(Identifier.of(Array(), "table_name"))
+      .asInstanceOf[InMemoryTable]
+
+    assert(table.name === "testcat.table_name")
+    assert(table.partitioning === Seq(IdentityTransform(FieldReference(Array("ts", "timezone")))))
+    checkAnswer(spark.table(table.name), data)
+    assert(table.dataMap.toArray.length == 2)
+    assert(table.dataMap(Seq(UTF8String.fromString("America/Los_Angeles"))).rows.size == 2)
+    assert(table.dataMap(Seq(UTF8String.fromString("America/New_York"))).rows.size == 1)
+
+    // TODO: `DataSourceV2Strategy` can not translate nested fields into source filter yet
+    // so the following sql will fail.
+    // sql("DELETE FROM testcat.table_name WHERE ts.timezone = \"America/Los_Angeles\"")
+  }
+
+  test("SPARK-30289 Create: partitioned by multiple transforms on nested columns") {
+    spark.table("source")
+      .withColumn("ts", struct(
+        lit("2019-06-01 10:00:00.000000").cast("timestamp") as "created",
+        lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
+        lit("America/Los_Angeles") as "timezone"))
+      .writeTo("testcat.table_name")
+      .tableProperty("allow-unsupported-transforms", "true")
+      .partitionedBy(
+        years($"ts.created"), months($"ts.created"), days($"ts.created"), hours($"ts.created"),
+        years($"ts.modified"), months($"ts.modified"), days($"ts.modified"), hours($"ts.modified")
+      )
+      .create()
+
+    val table = catalog("testcat").loadTable(Identifier.of(Array(), "table_name"))
+
+    assert(table.name === "testcat.table_name")
+    assert(table.partitioning === Seq(
+      YearsTransform(FieldReference(Array("ts", "created"))),
+      MonthsTransform(FieldReference(Array("ts", "created"))),
+      DaysTransform(FieldReference(Array("ts", "created"))),
+      HoursTransform(FieldReference(Array("ts", "created"))),
+      YearsTransform(FieldReference(Array("ts", "modified"))),
+      MonthsTransform(FieldReference(Array("ts", "modified"))),
+      DaysTransform(FieldReference(Array("ts", "modified"))),
+      HoursTransform(FieldReference(Array("ts", "modified")))))
+  }
+
+  test("SPARK-30289 Create: partitioned by bucket(4, ts.timezone)") {
+    spark.table("source")
+      .withColumn("ts", struct(
+        lit("2019-06-01 10:00:00.000000").cast("timestamp") as "created",
+        lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
+        lit("America/Los_Angeles") as "timezone"))
+      .writeTo("testcat.table_name")
+      .tableProperty("allow-unsupported-transforms", "true")
+      .partitionedBy(bucket(4, $"ts.timezone"))
+      .create()
+
+    val table = catalog("testcat").loadTable(Identifier.of(Array(), "table_name"))
+
+    assert(table.name === "testcat.table_name")
+    assert(table.partitioning === Seq(BucketTransform(LiteralValue(4, IntegerType),
+      Seq(FieldReference(Seq("ts", "timezone"))))))
+  }
 }

From f7b38fe05e222ed0eed3aca89e90a5690416193a Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Sat, 15 Feb 2020 19:49:58 +0800
Subject: [PATCH 0083/1280] [SPARK-30826][SQL] Respect reference case in
 `StringStartsWith` pushed down to parquet

### What changes were proposed in this pull request?
In the PR, I propose to convert the attribute name of `StringStartsWith` pushed down to the Parquet datasource to column reference via the `nameToParquetField` map. Similar conversions are performed for other source filters pushed down to parquet.

### Why are the changes needed?
This fixes the bug described in [SPARK-30826](https://issues.apache.org/jira/browse/SPARK-30826). The query from an external table:
```sql
CREATE TABLE t1 (col STRING)
USING parquet
OPTIONS (path '$path')
```
created on top of written parquet files by `Seq("42").toDF("COL").write.parquet(path)` returns wrong empty result:
```scala
spark.sql("SELECT * FROM t1 WHERE col LIKE '4%'").show
+---+
|col|
+---+
+---+
```

### Does this PR introduce any user-facing change?
Yes. After the changes the result is correct for the example above:
```scala
spark.sql("SELECT * FROM t1 WHERE col LIKE '4%'").show
+---+
|col|
+---+
| 42|
+---+
```

### How was this patch tested?
Added a test to `ParquetFilterSuite`

Closes #27574 from MaxGekk/parquet-StringStartsWith-case-sens.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8b73b92aadd685b29ef3d9b33366f5e1fd3dae99)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/parquet/ParquetFilters.scala  |  2 +-
 .../parquet/ParquetFilterSuite.scala          | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index b9b86adb438e6..948a120e0d6e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -591,7 +591,7 @@ class ParquetFilters(
       case sources.StringStartsWith(name, prefix)
           if pushDownStartWith && canMakeFilterOn(name, prefix) =>
         Option(prefix).map { v =>
-          FilterApi.userDefined(binaryColumn(name),
+          FilterApi.userDefined(binaryColumn(nameToParquetField(name).fieldName),
             new UserDefinedPredicate[Binary] with Serializable {
               private val strToBinary = Binary.fromReusedByteArray(v.getBytes)
               private val size = strToBinary.length
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 286bb1e920266..4e0c1c2dbe601 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -1390,6 +1390,27 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       }
     }
   }
+
+  test("SPARK-30826: case insensitivity of StringStartsWith attribute") {
+    import testImplicits._
+    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+      withTable("t1") {
+        withTempPath { dir =>
+          val path = dir.toURI.toString
+          Seq("42").toDF("COL").write.parquet(path)
+          spark.sql(
+            s"""
+               |CREATE TABLE t1 (col STRING)
+               |USING parquet
+               |OPTIONS (path '$path')
+           """.stripMargin)
+          checkAnswer(
+            spark.sql("SELECT * FROM t1 WHERE col LIKE '4%'"),
+            Row("42"))
+        }
+      }
+    }
+  }
 }
 
 class ParquetV1FilterSuite extends ParquetFilterSuite {

From 8ed8baa74a6471d929fcc367bff282a87cead7a1 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 16 Feb 2020 09:53:12 -0600
Subject: [PATCH 0084/1280] [SPARK-30691][SQL][DOC][FOLLOW-UP] Make link names
 exactly the same as the side bar names

### What changes were proposed in this pull request?
Make link names exactly the same as the side bar names

### Why are the changes needed?
Make doc look better

### Does this PR introduce any user-facing change?
before:

![image](https://user-images.githubusercontent.com/13592258/74578603-ad300100-4f4a-11ea-8430-11fccf31eab4.png)

after:

![image](https://user-images.githubusercontent.com/13592258/74578670-eff1d900-4f4a-11ea-97d8-5908c0e50e95.png)

### How was this patch tested?
Manually build and check the docs

Closes #27591 from huaxingao/spark-doc-followup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 0a03e7e679771da8556fae72b35edf21ae71ac44)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 1e343f630f88e..38a5cf61245a6 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -157,12 +157,12 @@
         - text: Auxiliary Statements
           url: sql-ref-syntax-aux.html
           subitems:
-            - text: Analyze statement
+            - text: ANALYZE
               url: sql-ref-syntax-aux-analyze.html
               subitems: 
                 - text: ANALYZE TABLE
                   url: sql-ref-syntax-aux-analyze-table.html
-            - text: Caching statements 
+            - text: CACHE
               url: sql-ref-syntax-aux-cache.html
               subitems:
                 - text: CACHE TABLE
@@ -175,7 +175,7 @@
                   url: sql-ref-syntax-aux-refresh-table.html
                 - text: REFRESH
                   url: sql-ref-syntax-aux-cache-refresh.md
-            - text: Describe Commands
+            - text: DESCRIBE
               url: sql-ref-syntax-aux-describe.html
               subitems:
                 - text: DESCRIBE DATABASE
@@ -186,7 +186,7 @@
                   url: sql-ref-syntax-aux-describe-function.html
                 - text: DESCRIBE QUERY
                   url: sql-ref-syntax-aux-describe-query.html
-            - text: Show commands
+            - text: SHOW
               url: sql-ref-syntax-aux-show.html
               subitems:
                 - text: SHOW COLUMNS 
@@ -205,14 +205,14 @@
                   url: sql-ref-syntax-aux-show-partitions.html
                 - text: SHOW CREATE TABLE
                   url: sql-ref-syntax-aux-show-create-table.html
-            - text: Configuration Management Commands
+            - text: CONFIGURATION MANAGEMENT
               url: sql-ref-syntax-aux-conf-mgmt.html
               subitems:
                 - text: SET 
                   url: sql-ref-syntax-aux-conf-mgmt-set.html
                 - text: RESET
                   url: sql-ref-syntax-aux-conf-mgmt-reset.html
-            - text: Resource Management Commands
+            - text: RESOURCE MANAGEMENT
               url: sql-ref-syntax-aux-resource-mgmt.html
               subitems:
                 - text: ADD FILE

From 3c9231e20b70ffc6e3ea61944ae5000f774c613a Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sun, 16 Feb 2020 09:55:03 -0600
Subject: [PATCH 0085/1280] [SPARK-30803][DOCS] Fix the home page link for
 Scala API document

### What changes were proposed in this pull request?
Change the link to the Scala API document.

```
$ git grep "#org.apache.spark.package"
docs/_layouts/global.html:                                <li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
docs/index.md:* [Spark Scala API (Scaladoc)](api/scala/index.html#org.apache.spark.package)
docs/rdd-programming-guide.md:[Scala](api/scala/#org.apache.spark.package), [Java](api/java/), [Python](api/python/) and [R](api/R/).
```

### Why are the changes needed?
The home page link for Scala API document is incorrect after upgrade to 3.0

### Does this PR introduce any user-facing change?
Document UI change only.

### How was this patch tested?
Local test, attach screenshots below:
Before:
![image](https://user-images.githubusercontent.com/4833765/74335713-c2385300-4dd7-11ea-95d8-f5a3639d2578.png)
After:
![image](https://user-images.githubusercontent.com/4833765/74335727-cbc1bb00-4dd7-11ea-89d9-4dcc1310e679.png)

Closes #27549 from xuanyuanking/scala-doc.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 01cc852982cd065e08f9a652c14a0514f49fb662)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_layouts/global.html                     |  2 +-
 docs/configuration.md                         |  8 +-
 docs/graphx-programming-guide.md              | 68 +++++++-------
 docs/index.md                                 |  2 +-
 docs/ml-advanced.md                           | 10 +-
 docs/ml-classification-regression.md          | 40 ++++----
 docs/ml-clustering.md                         | 10 +-
 docs/ml-collaborative-filtering.md            |  2 +-
 docs/ml-datasource.md                         |  4 +-
 docs/ml-features.md                           | 92 +++++++++----------
 docs/ml-frequent-pattern-mining.md            |  4 +-
 docs/ml-migration-guide.md                    | 36 ++++----
 docs/ml-pipeline.md                           | 10 +-
 docs/ml-statistics.md                         |  8 +-
 docs/ml-tuning.md                             | 18 ++--
 docs/mllib-clustering.md                      | 26 +++---
 docs/mllib-collaborative-filtering.md         |  4 +-
 docs/mllib-data-types.md                      | 48 +++++-----
 docs/mllib-decision-tree.md                   | 10 +-
 docs/mllib-dimensionality-reduction.md        |  6 +-
 docs/mllib-ensembles.md                       | 10 +-
 docs/mllib-evaluation-metrics.md              |  8 +-
 docs/mllib-feature-extraction.md              | 34 +++----
 docs/mllib-frequent-pattern-mining.md         | 14 +--
 docs/mllib-isotonic-regression.md             |  2 +-
 docs/mllib-linear-methods.md                  | 22 ++---
 docs/mllib-naive-bayes.md                     |  8 +-
 docs/mllib-optimization.md                    | 14 +--
 docs/mllib-pmml-model-export.md               |  2 +-
 docs/mllib-statistics.md                      | 28 +++---
 docs/quick-start.md                           |  2 +-
 docs/rdd-programming-guide.md                 | 28 +++---
 docs/sql-data-sources-generic-options.md      |  2 +-
 docs/sql-data-sources-jdbc.md                 |  2 +-
 docs/sql-data-sources-json.md                 |  2 +-
 docs/sql-getting-started.md                   | 16 ++--
 docs/sql-migration-guide.md                   |  4 +-
 docs/sql-programming-guide.md                 |  2 +-
 docs/sql-ref-syntax-aux-analyze-table.md      |  2 +-
 docs/sql-ref-syntax-aux-cache-refresh.md      |  2 +-
 docs/sql-ref-syntax-aux-refresh-table.md      |  2 +-
 docs/sql-ref-syntax-aux-resource-mgmt.md      |  2 +-
 docs/sql-ref-syntax-aux-show-tables.md        |  2 +-
 docs/sql-ref-syntax-aux-show.md               |  2 +-
 docs/sql-ref-syntax-ddl-drop-database.md      |  2 +-
 docs/sql-ref-syntax-ddl-drop-function.md      |  2 +-
 ...tax-dml-insert-overwrite-directory-hive.md |  2 +-
 ...f-syntax-dml-insert-overwrite-directory.md |  2 +-
 docs/sql-ref-syntax-dml.md                    |  2 +-
 docs/sql-ref-syntax-qry-select-clusterby.md   |  2 +-
 ...sql-ref-syntax-qry-select-distribute-by.md |  2 +-
 docs/sql-ref-syntax-qry-select-sortby.md      |  2 +-
 docs/sql-ref-syntax-qry-select.md             |  2 +-
 docs/streaming-custom-receivers.md            |  2 +-
 docs/streaming-kafka-integration.md           |  2 +-
 docs/streaming-kinesis-integration.md         |  2 +-
 docs/streaming-programming-guide.md           | 42 ++++-----
 .../structured-streaming-programming-guide.md | 22 ++---
 docs/tuning.md                                |  2 +-
 59 files changed, 355 insertions(+), 355 deletions(-)

diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html
index d5fb18bfb06c0..d05ac6bbe129d 100755
--- a/docs/_layouts/global.html
+++ b/docs/_layouts/global.html
@@ -82,7 +82,7 @@
                         <li class="dropdown">
                             <a href="#" class="dropdown-toggle" data-toggle="dropdown">API Docs<b class="caret"></b></a>
                             <ul class="dropdown-menu">
-                                <li><a href="api/scala/index.html#org.apache.spark.package">Scala</a></li>
+                                <li><a href="api/scala/org/apache/spark/index.html">Scala</a></li>
                                 <li><a href="api/java/index.html">Java</a></li>
                                 <li><a href="api/python/index.html">Python</a></li>
                                 <li><a href="api/R/index.html">R</a></li>
diff --git a/docs/configuration.md b/docs/configuration.md
index 1343755f9d87f..9c9cf84151b20 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -24,7 +24,7 @@ license: |
 Spark provides three locations to configure the system:
 
 * [Spark properties](#spark-properties) control most application parameters and can be set by using
-  a [SparkConf](api/scala/index.html#org.apache.spark.SparkConf) object, or through Java
+  a [SparkConf](api/scala/org/apache/spark/SparkConf.html) object, or through Java
   system properties.
 * [Environment variables](#environment-variables) can be used to set per-machine settings, such as
   the IP address, through the `conf/spark-env.sh` script on each node.
@@ -34,7 +34,7 @@ Spark provides three locations to configure the system:
 
 Spark properties control most application settings and are configured separately for each
 application. These properties can be set directly on a
-[SparkConf](api/scala/index.html#org.apache.spark.SparkConf) passed to your
+[SparkConf](api/scala/org/apache/spark/SparkConf.html) passed to your
 `SparkContext`. `SparkConf` allows you to configure some of the common properties
 (e.g. master URL and application name), as well as arbitrary key-value pairs through the
 `set()` method. For example, we could initialize an application with two threads as follows:
@@ -1326,7 +1326,7 @@ Apart from these, the following properties are also available, and may be useful
     property is useful if you need to register your classes in a custom way, e.g. to specify a custom
     field serializer. Otherwise <code>spark.kryo.classesToRegister</code> is simpler. It should be
     set to classes that extend
-    <a href="api/scala/index.html#org.apache.spark.serializer.KryoRegistrator">
+    <a href="api/scala/org/apache/spark/serializer/KryoRegistrator.html">
     <code>KryoRegistrator</code></a>.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
@@ -1379,7 +1379,7 @@ Apart from these, the following properties are also available, and may be useful
     but is quite slow, so we recommend <a href="tuning.html">using
     <code>org.apache.spark.serializer.KryoSerializer</code> and configuring Kryo serialization</a>
     when speed is necessary. Can be any subclass of
-    <a href="api/scala/index.html#org.apache.spark.serializer.Serializer">
+    <a href="api/scala/org/apache/spark/serializer/Serializer.html">
     <code>org.apache.spark.Serializer</code></a>.
   </td>
 </tr>
diff --git a/docs/graphx-programming-guide.md b/docs/graphx-programming-guide.md
index 167c44aa1b2e9..50c9366a0999f 100644
--- a/docs/graphx-programming-guide.md
+++ b/docs/graphx-programming-guide.md
@@ -25,38 +25,38 @@ license: |
 
 <!-- All the documentation links  -->
 
-[EdgeRDD]: api/scala/index.html#org.apache.spark.graphx.EdgeRDD
-[VertexRDD]: api/scala/index.html#org.apache.spark.graphx.VertexRDD
-[Edge]: api/scala/index.html#org.apache.spark.graphx.Edge
-[EdgeTriplet]: api/scala/index.html#org.apache.spark.graphx.EdgeTriplet
-[Graph]: api/scala/index.html#org.apache.spark.graphx.Graph
-[GraphOps]: api/scala/index.html#org.apache.spark.graphx.GraphOps
-[Graph.mapVertices]: api/scala/index.html#org.apache.spark.graphx.Graph@mapVertices[VD2]((VertexId,VD)⇒VD2)(ClassTag[VD2]):Graph[VD2,ED]
-[Graph.reverse]: api/scala/index.html#org.apache.spark.graphx.Graph@reverse:Graph[VD,ED]
-[Graph.subgraph]: api/scala/index.html#org.apache.spark.graphx.Graph@subgraph((EdgeTriplet[VD,ED])⇒Boolean,(VertexId,VD)⇒Boolean):Graph[VD,ED]
-[Graph.mask]: api/scala/index.html#org.apache.spark.graphx.Graph@mask[VD2,ED2](Graph[VD2,ED2])(ClassTag[VD2],ClassTag[ED2]):Graph[VD,ED]
-[Graph.groupEdges]: api/scala/index.html#org.apache.spark.graphx.Graph@groupEdges((ED,ED)⇒ED):Graph[VD,ED]
-[GraphOps.joinVertices]: api/scala/index.html#org.apache.spark.graphx.GraphOps@joinVertices[U](RDD[(VertexId,U)])((VertexId,VD,U)⇒VD)(ClassTag[U]):Graph[VD,ED]
-[Graph.outerJoinVertices]: api/scala/index.html#org.apache.spark.graphx.Graph@outerJoinVertices[U,VD2](RDD[(VertexId,U)])((VertexId,VD,Option[U])⇒VD2)(ClassTag[U],ClassTag[VD2]):Graph[VD2,ED]
-[Graph.aggregateMessages]: api/scala/index.html#org.apache.spark.graphx.Graph@aggregateMessages[A]((EdgeContext[VD,ED,A])⇒Unit,(A,A)⇒A,TripletFields)(ClassTag[A]):VertexRDD[A]
-[EdgeContext]: api/scala/index.html#org.apache.spark.graphx.EdgeContext
-[GraphOps.collectNeighborIds]: api/scala/index.html#org.apache.spark.graphx.GraphOps@collectNeighborIds(EdgeDirection):VertexRDD[Array[VertexId]]
-[GraphOps.collectNeighbors]: api/scala/index.html#org.apache.spark.graphx.GraphOps@collectNeighbors(EdgeDirection):VertexRDD[Array[(VertexId,VD)]]
+[EdgeRDD]: api/scala/org/apache/spark/graphx/EdgeRDD.html
+[VertexRDD]: api/scala/org/apache/spark/graphx/VertexRDD.html
+[Edge]: api/scala/org/apache/spark/graphx/Edge.html
+[EdgeTriplet]: api/scala/org/apache/spark/graphx/EdgeTriplet.html
+[Graph]: api/scala/org/apache/spark/graphx/Graph$.html
+[GraphOps]: api/scala/org/apache/spark/graphx/GraphOps.html
+[Graph.mapVertices]: api/scala/org/apache/spark/graphx/Graph.html#mapVertices[VD2]((VertexId,VD)⇒VD2)(ClassTag[VD2]):Graph[VD2,ED]
+[Graph.reverse]: api/scala/org/apache/spark/graphx/Graph.html#reverse:Graph[VD,ED]
+[Graph.subgraph]: api/scala/org/apache/spark/graphx/Graph.html#subgraph((EdgeTriplet[VD,ED])⇒Boolean,(VertexId,VD)⇒Boolean):Graph[VD,ED]
+[Graph.mask]: api/scala/org/apache/spark/graphx/Graph.html#mask[VD2,ED2](Graph[VD2,ED2])(ClassTag[VD2],ClassTag[ED2]):Graph[VD,ED]
+[Graph.groupEdges]: api/scala/org/apache/spark/graphx/Graph.html#groupEdges((ED,ED)⇒ED):Graph[VD,ED]
+[GraphOps.joinVertices]: api/scala/org/apache/spark/graphx/GraphOps.html#joinVertices[U](RDD[(VertexId,U)])((VertexId,VD,U)⇒VD)(ClassTag[U]):Graph[VD,ED]
+[Graph.outerJoinVertices]: api/scala/org/apache/spark/graphx/Graph.html#outerJoinVertices[U,VD2](RDD[(VertexId,U)])((VertexId,VD,Option[U])⇒VD2)(ClassTag[U],ClassTag[VD2]):Graph[VD2,ED]
+[Graph.aggregateMessages]: api/scala/org/apache/spark/graphx/Graph.html#aggregateMessages[A]((EdgeContext[VD,ED,A])⇒Unit,(A,A)⇒A,TripletFields)(ClassTag[A]):VertexRDD[A]
+[EdgeContext]: api/scala/org/apache/spark/graphx/EdgeContext.html
+[GraphOps.collectNeighborIds]: api/scala/org/apache/spark/graphx/GraphOps.html#collectNeighborIds(EdgeDirection):VertexRDD[Array[VertexId]]
+[GraphOps.collectNeighbors]: api/scala/org/apache/spark/graphx/GraphOps.html#collectNeighbors(EdgeDirection):VertexRDD[Array[(VertexId,VD)]]
 [RDD Persistence]: rdd-programming-guide.html#rdd-persistence
-[Graph.cache]: api/scala/index.html#org.apache.spark.graphx.Graph@cache():Graph[VD,ED]
-[GraphOps.pregel]: api/scala/index.html#org.apache.spark.graphx.GraphOps@pregel[A](A,Int,EdgeDirection)((VertexId,VD,A)⇒VD,(EdgeTriplet[VD,ED])⇒Iterator[(VertexId,A)],(A,A)⇒A)(ClassTag[A]):Graph[VD,ED]
-[PartitionStrategy]: api/scala/index.html#org.apache.spark.graphx.PartitionStrategy$
-[GraphLoader.edgeListFile]: api/scala/index.html#org.apache.spark.graphx.GraphLoader$@edgeListFile(SparkContext,String,Boolean,Int):Graph[Int,Int]
-[Graph.apply]: api/scala/index.html#org.apache.spark.graphx.Graph$@apply[VD,ED](RDD[(VertexId,VD)],RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
-[Graph.fromEdgeTuples]: api/scala/index.html#org.apache.spark.graphx.Graph$@fromEdgeTuples[VD](RDD[(VertexId,VertexId)],VD,Option[PartitionStrategy])(ClassTag[VD]):Graph[VD,Int]
-[Graph.fromEdges]: api/scala/index.html#org.apache.spark.graphx.Graph$@fromEdges[VD,ED](RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
-[PartitionStrategy]: api/scala/index.html#org.apache.spark.graphx.PartitionStrategy
-[PageRank]: api/scala/index.html#org.apache.spark.graphx.lib.PageRank$
-[ConnectedComponents]: api/scala/index.html#org.apache.spark.graphx.lib.ConnectedComponents$
-[TriangleCount]: api/scala/index.html#org.apache.spark.graphx.lib.TriangleCount$
-[Graph.partitionBy]: api/scala/index.html#org.apache.spark.graphx.Graph@partitionBy(PartitionStrategy):Graph[VD,ED]
-[EdgeContext.sendToSrc]: api/scala/index.html#org.apache.spark.graphx.EdgeContext@sendToSrc(msg:A):Unit
-[EdgeContext.sendToDst]: api/scala/index.html#org.apache.spark.graphx.EdgeContext@sendToDst(msg:A):Unit
+[Graph.cache]: api/scala/org/apache/spark/graphx/Graph.html#cache():Graph[VD,ED]
+[GraphOps.pregel]: api/scala/org/apache/spark/graphx/GraphOps.html#pregel[A](A,Int,EdgeDirection)((VertexId,VD,A)⇒VD,(EdgeTriplet[VD,ED])⇒Iterator[(VertexId,A)],(A,A)⇒A)(ClassTag[A]):Graph[VD,ED]
+[PartitionStrategy]: api/scala/org/apache/spark/graphx/PartitionStrategy$.html
+[GraphLoader.edgeListFile]: api/scala/org/apache/spark/graphx/GraphLoader$.html#edgeListFile(SparkContext,String,Boolean,Int):Graph[Int,Int]
+[Graph.apply]: api/scala/org/apache/spark/graphx/Graph$.html#apply[VD,ED](RDD[(VertexId,VD)],RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
+[Graph.fromEdgeTuples]: api/scala/org/apache/spark/graphx/Graph$.html#fromEdgeTuples[VD](RDD[(VertexId,VertexId)],VD,Option[PartitionStrategy])(ClassTag[VD]):Graph[VD,Int]
+[Graph.fromEdges]: api/scala/org/apache/spark/graphx/Graph$.html#fromEdges[VD,ED](RDD[Edge[ED]],VD)(ClassTag[VD],ClassTag[ED]):Graph[VD,ED]
+[PartitionStrategy]: api/scala/org/apache/spark/graphx/PartitionStrategy$.html
+[PageRank]: api/scala/org/apache/spark/graphx/lib/PageRank$.html
+[ConnectedComponents]: api/scala/org/apache/spark/graphx/lib/ConnectedComponents$.html
+[TriangleCount]: api/scala/org/apache/spark/graphx/lib/TriangleCount$.html
+[Graph.partitionBy]: api/scala/org/apache/spark/graphx/Graph.html#partitionBy(PartitionStrategy):Graph[VD,ED]
+[EdgeContext.sendToSrc]: api/scala/org/apache/spark/graphx/EdgeContext.html#sendToSrc(msg:A):Unit
+[EdgeContext.sendToDst]: api/scala/org/apache/spark/graphx/EdgeContext.html#sendToDst(msg:A):Unit
 [TripletFields]: api/java/org/apache/spark/graphx/TripletFields.html
 [TripletFields.All]: api/java/org/apache/spark/graphx/TripletFields.html#All
 [TripletFields.None]: api/java/org/apache/spark/graphx/TripletFields.html#None
@@ -74,7 +74,7 @@ license: |
 # Overview
 
 GraphX is a new component in Spark for graphs and graph-parallel computation. At a high level,
-GraphX extends the Spark [RDD](api/scala/index.html#org.apache.spark.rdd.RDD) by introducing a
+GraphX extends the Spark [RDD](api/scala/org/apache/spark/rdd/RDD.html) by introducing a
 new [Graph](#property_graph) abstraction: a directed multigraph with properties
 attached to each vertex and edge.  To support graph computation, GraphX exposes a set of fundamental
 operators (e.g., [subgraph](#structural_operators), [joinVertices](#join_operators), and
@@ -99,7 +99,7 @@ getting started with Spark refer to the [Spark Quick Start Guide](quick-start.ht
 
 # The Property Graph
 
-The [property graph](api/scala/index.html#org.apache.spark.graphx.Graph) is a directed multigraph
+The [property graph](api/scala/org/apache/spark/graphx/Graph.html) is a directed multigraph
 with user defined objects attached to each vertex and edge.  A directed multigraph is a directed
 graph with potentially multiple parallel edges sharing the same source and destination vertex.  The
 ability to support parallel edges simplifies modeling scenarios where there can be multiple
@@ -175,7 +175,7 @@ val userGraph: Graph[(String, String), String]
 There are numerous ways to construct a property graph from raw files, RDDs, and even synthetic
 generators and these are discussed in more detail in the section on
 [graph builders](#graph_builders).  Probably the most general method is to use the
-[Graph object](api/scala/index.html#org.apache.spark.graphx.Graph$).  For example the following
+[Graph object](api/scala/org/apache/spark/graphx/Graph$.html).  For example the following
 code constructs a graph from a collection of RDDs:
 
 {% highlight scala %}
diff --git a/docs/index.md b/docs/index.md
index f6ec595231f9e..38f12dd4db77b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -118,7 +118,7 @@ options for deployment:
 
 **API Docs:**
 
-* [Spark Scala API (Scaladoc)](api/scala/index.html#org.apache.spark.package)
+* [Spark Scala API (Scaladoc)](api/scala/org/apache/spark/index.html)
 * [Spark Java API (Javadoc)](api/java/index.html)
 * [Spark Python API (Sphinx)](api/python/index.html)
 * [Spark R API (Roxygen2)](api/R/index.html)
diff --git a/docs/ml-advanced.md b/docs/ml-advanced.md
index 5787fe914ce7a..0e19bca92d19d 100644
--- a/docs/ml-advanced.md
+++ b/docs/ml-advanced.md
@@ -55,10 +55,10 @@ other first-order optimizations.
 Quasi-Newton](https://www.microsoft.com/en-us/research/wp-content/uploads/2007/01/andrew07scalable.pdf)
 (OWL-QN) is an extension of L-BFGS that can effectively handle L1 and elastic net regularization.
 
-L-BFGS is used as a solver for [LinearRegression](api/scala/index.html#org.apache.spark.ml.regression.LinearRegression),
-[LogisticRegression](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegression),
-[AFTSurvivalRegression](api/scala/index.html#org.apache.spark.ml.regression.AFTSurvivalRegression)
-and [MultilayerPerceptronClassifier](api/scala/index.html#org.apache.spark.ml.classification.MultilayerPerceptronClassifier).
+L-BFGS is used as a solver for [LinearRegression](api/scala/org/apache/spark/ml/regression/LinearRegression.html),
+[LogisticRegression](api/scala/org/apache/spark/ml/classification/LogisticRegression.html),
+[AFTSurvivalRegression](api/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.html)
+and [MultilayerPerceptronClassifier](api/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.html).
 
 MLlib L-BFGS solver calls the corresponding implementation in [breeze](https://github.com/scalanlp/breeze/blob/master/math/src/main/scala/breeze/optimize/LBFGS.scala).
 
@@ -108,4 +108,4 @@ It solves certain optimization problems iteratively through the following proced
 
 Since it involves solving a weighted least squares (WLS) problem by `WeightedLeastSquares` in each iteration,
 it also requires the number of features to be no more than 4096.
-Currently IRLS is used as the default solver of [GeneralizedLinearRegression](api/scala/index.html#org.apache.spark.ml.regression.GeneralizedLinearRegression).
+Currently IRLS is used as the default solver of [GeneralizedLinearRegression](api/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.html).
diff --git a/docs/ml-classification-regression.md b/docs/ml-classification-regression.md
index 630a15d8535f4..9d5388005e587 100644
--- a/docs/ml-classification-regression.md
+++ b/docs/ml-classification-regression.md
@@ -71,7 +71,7 @@ $\alpha$ and `regParam` corresponds to $\lambda$.
 
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegression).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/classification/LogisticRegression.html).
 
 {% include_example scala/org/apache/spark/examples/ml/LogisticRegressionWithElasticNetExample.scala %}
 </div>
@@ -109,12 +109,12 @@ only available on the driver.
 
 <div data-lang="scala" markdown="1">
 
-[`LogisticRegressionTrainingSummary`](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegressionTrainingSummary)
+[`LogisticRegressionTrainingSummary`](api/scala/org/apache/spark/ml/classification/LogisticRegressionTrainingSummary.html)
 provides a summary for a
-[`LogisticRegressionModel`](api/scala/index.html#org.apache.spark.ml.classification.LogisticRegressionModel).
+[`LogisticRegressionModel`](api/scala/org/apache/spark/ml/classification/LogisticRegressionModel.html).
 In the case of binary classification, certain additional metrics are
 available, e.g. ROC curve. The binary summary can be accessed via the
-`binarySummary` method. See [`BinaryLogisticRegressionTrainingSummary`](api/scala/index.html#org.apache.spark.ml.classification.BinaryLogisticRegressionTrainingSummary).
+`binarySummary` method. See [`BinaryLogisticRegressionTrainingSummary`](api/scala/org/apache/spark/ml/classification/BinaryLogisticRegressionTrainingSummary.html).
 
 Continuing the earlier example:
 
@@ -216,7 +216,7 @@ We use two feature transformers to prepare the data; these help index categories
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.classification.DecisionTreeClassifier).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.html).
 
 {% include_example scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala %}
 
@@ -261,7 +261,7 @@ We use two feature transformers to prepare the data; these help index categories
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.RandomForestClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/RandomForestClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala %}
 </div>
@@ -302,7 +302,7 @@ We use two feature transformers to prepare the data; these help index categories
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.GBTClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/GBTClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala %}
 </div>
@@ -358,7 +358,7 @@ MLPC employs backpropagation for learning the model. We use the logistic loss fu
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.MultilayerPerceptronClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala %}
 </div>
@@ -403,7 +403,7 @@ in Spark ML supports binary classification with linear SVM. Internally, it optim
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.LinearSVC) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/LinearSVC.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/LinearSVCExample.scala %}
 </div>
@@ -447,7 +447,7 @@ The example below demonstrates how to load the
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.OneVsRest) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/OneVsRest.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/OneVsRestExample.scala %}
 </div>
@@ -501,7 +501,7 @@ setting the parameter $\lambda$ (default to $1.0$).
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.NaiveBayes) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/NaiveBayes.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/NaiveBayesExample.scala %}
 </div>
@@ -544,7 +544,7 @@ We scale features to be between 0 and 1 to prevent the exploding gradient proble
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.classification.FMClassifier) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/classification/FMClassifier.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/FMClassifierExample.scala %}
 </div>
@@ -585,7 +585,7 @@ regression model and extracting model summary statistics.
 
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.regression.LinearRegression).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/regression/LinearRegression.html).
 
 {% include_example scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala %}
 </div>
@@ -726,7 +726,7 @@ function and extracting model summary statistics.
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.GeneralizedLinearRegression) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GeneralizedLinearRegressionExample.scala %}
 </div>
@@ -768,7 +768,7 @@ We use a feature transformer to index categorical features, adding metadata to t
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-More details on parameters can be found in the [Scala API documentation](api/scala/index.html#org.apache.spark.ml.regression.DecisionTreeRegressor).
+More details on parameters can be found in the [Scala API documentation](api/scala/org/apache/spark/ml/regression/DecisionTreeRegressor.html).
 
 {% include_example scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala %}
 </div>
@@ -810,7 +810,7 @@ We use a feature transformer to index categorical features, adding metadata to t
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.RandomForestRegressor) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/RandomForestRegressor.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala %}
 </div>
@@ -851,7 +851,7 @@ be true in general.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.GBTRegressor) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/GBTRegressor.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala %}
 </div>
@@ -945,7 +945,7 @@ The implementation matches the result from R's survival function
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.AFTSurvivalRegression) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala %}
 </div>
@@ -1025,7 +1025,7 @@ is treated as piecewise linear function. The rules for prediction therefore are:
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [`IsotonicRegression` Scala docs](api/scala/index.html#org.apache.spark.ml.regression.IsotonicRegression) for details on the API.
+Refer to the [`IsotonicRegression` Scala docs](api/scala/org/apache/spark/ml/regression/IsotonicRegression.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/IsotonicRegressionExample.scala %}
 </div>
@@ -1066,7 +1066,7 @@ We scale features to be between 0 and 1 to prevent the exploding gradient proble
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.regression.FMRegressor) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/regression/FMRegressor.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/FMRegressorExample.scala %}
 </div>
diff --git a/docs/ml-clustering.md b/docs/ml-clustering.md
index 2775d0421ccca..4574567fa9d50 100644
--- a/docs/ml-clustering.md
+++ b/docs/ml-clustering.md
@@ -85,7 +85,7 @@ called [kmeans||](http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf).
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.KMeans) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/KMeans.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/KMeansExample.scala %}
 </div>
@@ -123,7 +123,7 @@ and generates a `LDAModel` as the base model. Expert users may cast a `LDAModel`
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.LDA) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/LDA.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/LDAExample.scala %}
 </div>
@@ -166,7 +166,7 @@ Bisecting K-means can often be much faster than regular K-means, but it will gen
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.BisectingKMeans) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/BisectingKMeans.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/BisectingKMeansExample.scala %}
 </div>
@@ -255,7 +255,7 @@ model.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.GaussianMixture) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/GaussianMixture.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/GaussianMixtureExample.scala %}
 </div>
@@ -302,7 +302,7 @@ using truncated power iteration on a normalized pair-wise similarity matrix of t
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.clustering.PowerIterationClustering) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/clustering/PowerIterationClustering.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/PowerIterationClusteringExample.scala %}
 </div>
diff --git a/docs/ml-collaborative-filtering.md b/docs/ml-collaborative-filtering.md
index e6e596bed110b..6c41efd5cc306 100644
--- a/docs/ml-collaborative-filtering.md
+++ b/docs/ml-collaborative-filtering.md
@@ -115,7 +115,7 @@ explicit (`implicitPrefs` is `false`).
 We evaluate the recommendation model by measuring the root-mean-square error of
 rating prediction.
 
-Refer to the [`ALS` Scala docs](api/scala/index.html#org.apache.spark.ml.recommendation.ALS)
+Refer to the [`ALS` Scala docs](api/scala/org/apache/spark/ml/recommendation/ALS.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ALSExample.scala %}
diff --git a/docs/ml-datasource.md b/docs/ml-datasource.md
index 5dc2d057a9163..0f2f5f482ec50 100644
--- a/docs/ml-datasource.md
+++ b/docs/ml-datasource.md
@@ -42,7 +42,7 @@ The schema of the `image` column is:
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`ImageDataSource`](api/scala/index.html#org.apache.spark.ml.source.image.ImageDataSource)
+[`ImageDataSource`](api/scala/org/apache/spark/ml/source/image/ImageDataSource.html)
 implements a Spark SQL data source API for loading image data as a DataFrame.
 
 {% highlight scala %}
@@ -133,7 +133,7 @@ The schemas of the columns are:
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`LibSVMDataSource`](api/scala/index.html#org.apache.spark.ml.source.libsvm.LibSVMDataSource)
+[`LibSVMDataSource`](api/scala/org/apache/spark/ml/source/libsvm/LibSVMDataSource.html)
 implements a Spark SQL data source API for loading `LIBSVM` data as a DataFrame.
 
 {% highlight scala %}
diff --git a/docs/ml-features.md b/docs/ml-features.md
index 92d2f3d0b418d..9c05fd5fa1ce2 100644
--- a/docs/ml-features.md
+++ b/docs/ml-features.md
@@ -96,8 +96,8 @@ when using text as features.  Our feature vectors could then be passed to a lear
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [HashingTF Scala docs](api/scala/index.html#org.apache.spark.ml.feature.HashingTF) and
-the [IDF Scala docs](api/scala/index.html#org.apache.spark.ml.feature.IDF) for more details on the API.
+Refer to the [HashingTF Scala docs](api/scala/org/apache/spark/ml/feature/HashingTF.html) and
+the [IDF Scala docs](api/scala/org/apache/spark/ml/feature/IDF.html) for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/TfIdfExample.scala %}
 </div>
@@ -135,7 +135,7 @@ In the following code segment, we start with a set of documents, each of which i
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Word2Vec Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Word2Vec)
+Refer to the [Word2Vec Scala docs](api/scala/org/apache/spark/ml/feature/Word2Vec.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/Word2VecExample.scala %}
@@ -200,8 +200,8 @@ Each vector represents the token counts of the document over the vocabulary.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [CountVectorizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.CountVectorizer)
-and the [CountVectorizerModel Scala docs](api/scala/index.html#org.apache.spark.ml.feature.CountVectorizerModel)
+Refer to the [CountVectorizer Scala docs](api/scala/org/apache/spark/ml/feature/CountVectorizer.html)
+and the [CountVectorizerModel Scala docs](api/scala/org/apache/spark/ml/feature/CountVectorizerModel.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/CountVectorizerExample.scala %}
@@ -286,7 +286,7 @@ The resulting feature vectors could then be passed to a learning algorithm.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [FeatureHasher Scala docs](api/scala/index.html#org.apache.spark.ml.feature.FeatureHasher)
+Refer to the [FeatureHasher Scala docs](api/scala/org/apache/spark/ml/feature/FeatureHasher.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/FeatureHasherExample.scala %}
@@ -313,9 +313,9 @@ for more details on the API.
 
 ## Tokenizer
 
-[Tokenization](http://en.wikipedia.org/wiki/Lexical_analysis#Tokenization) is the process of taking text (such as a sentence) and breaking it into individual terms (usually words).  A simple [Tokenizer](api/scala/index.html#org.apache.spark.ml.feature.Tokenizer) class provides this functionality.  The example below shows how to split sentences into sequences of words.
+[Tokenization](http://en.wikipedia.org/wiki/Lexical_analysis#Tokenization) is the process of taking text (such as a sentence) and breaking it into individual terms (usually words).  A simple [Tokenizer](api/scala/org/apache/spark/ml/feature/Tokenizer.html) class provides this functionality.  The example below shows how to split sentences into sequences of words.
 
-[RegexTokenizer](api/scala/index.html#org.apache.spark.ml.feature.RegexTokenizer) allows more
+[RegexTokenizer](api/scala/org/apache/spark/ml/feature/RegexTokenizer.html) allows more
  advanced tokenization based on regular expression (regex) matching.
  By default, the parameter "pattern" (regex, default: `"\\s+"`) is used as delimiters to split the input text.
  Alternatively, users can set parameter "gaps" to false indicating the regex "pattern" denotes
@@ -326,8 +326,8 @@ for more details on the API.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Tokenizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Tokenizer)
-and the [RegexTokenizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RegexTokenizer)
+Refer to the [Tokenizer Scala docs](api/scala/org/apache/spark/ml/feature/Tokenizer.html)
+and the [RegexTokenizer Scala docs](api/scala/org/apache/spark/ml/feature/RegexTokenizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/TokenizerExample.scala %}
@@ -395,7 +395,7 @@ filtered out.
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [StopWordsRemover Scala docs](api/scala/index.html#org.apache.spark.ml.feature.StopWordsRemover)
+Refer to the [StopWordsRemover Scala docs](api/scala/org/apache/spark/ml/feature/StopWordsRemover.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala %}
@@ -430,7 +430,7 @@ An [n-gram](https://en.wikipedia.org/wiki/N-gram) is a sequence of $n$ tokens (t
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [NGram Scala docs](api/scala/index.html#org.apache.spark.ml.feature.NGram)
+Refer to the [NGram Scala docs](api/scala/org/apache/spark/ml/feature/NGram.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/NGramExample.scala %}
@@ -468,7 +468,7 @@ for `inputCol`.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Binarizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Binarizer)
+Refer to the [Binarizer Scala docs](api/scala/org/apache/spark/ml/feature/Binarizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/BinarizerExample.scala %}
@@ -493,14 +493,14 @@ for more details on the API.
 
 ## PCA
 
-[PCA](http://en.wikipedia.org/wiki/Principal_component_analysis) is a statistical procedure that uses an orthogonal transformation to convert a set of observations of possibly correlated variables into a set of values of linearly uncorrelated variables called principal components. A [PCA](api/scala/index.html#org.apache.spark.ml.feature.PCA) class trains a model to project vectors to a low-dimensional space using PCA. The example below shows how to project 5-dimensional feature vectors into 3-dimensional principal components.
+[PCA](http://en.wikipedia.org/wiki/Principal_component_analysis) is a statistical procedure that uses an orthogonal transformation to convert a set of observations of possibly correlated variables into a set of values of linearly uncorrelated variables called principal components. A [PCA](api/scala/org/apache/spark/ml/feature/PCA.html) class trains a model to project vectors to a low-dimensional space using PCA. The example below shows how to project 5-dimensional feature vectors into 3-dimensional principal components.
 
 **Examples**
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [PCA Scala docs](api/scala/index.html#org.apache.spark.ml.feature.PCA)
+Refer to the [PCA Scala docs](api/scala/org/apache/spark/ml/feature/PCA.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/PCAExample.scala %}
@@ -525,14 +525,14 @@ for more details on the API.
 
 ## PolynomialExpansion
 
-[Polynomial expansion](http://en.wikipedia.org/wiki/Polynomial_expansion) is the process of expanding your features into a polynomial space, which is formulated by an n-degree combination of original dimensions. A [PolynomialExpansion](api/scala/index.html#org.apache.spark.ml.feature.PolynomialExpansion) class provides this functionality.  The example below shows how to expand your features into a 3-degree polynomial space.
+[Polynomial expansion](http://en.wikipedia.org/wiki/Polynomial_expansion) is the process of expanding your features into a polynomial space, which is formulated by an n-degree combination of original dimensions. A [PolynomialExpansion](api/scala/org/apache/spark/ml/feature/PolynomialExpansion.html) class provides this functionality.  The example below shows how to expand your features into a 3-degree polynomial space.
 
 **Examples**
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [PolynomialExpansion Scala docs](api/scala/index.html#org.apache.spark.ml.feature.PolynomialExpansion)
+Refer to the [PolynomialExpansion Scala docs](api/scala/org/apache/spark/ml/feature/PolynomialExpansion.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala %}
@@ -561,7 +561,7 @@ The [Discrete Cosine
 Transform](https://en.wikipedia.org/wiki/Discrete_cosine_transform)
 transforms a length $N$ real-valued sequence in the time domain into
 another length $N$ real-valued sequence in the frequency domain. A
-[DCT](api/scala/index.html#org.apache.spark.ml.feature.DCT) class
+[DCT](api/scala/org/apache/spark/ml/feature/DCT.html) class
 provides this functionality, implementing the
 [DCT-II](https://en.wikipedia.org/wiki/Discrete_cosine_transform#DCT-II)
 and scaling the result by $1/\sqrt{2}$ such that the representing matrix
@@ -574,7 +574,7 @@ $0$th DCT coefficient and _not_ the $N/2$th).
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [DCT Scala docs](api/scala/index.html#org.apache.spark.ml.feature.DCT)
+Refer to the [DCT Scala docs](api/scala/org/apache/spark/ml/feature/DCT.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/DCTExample.scala %}
@@ -704,7 +704,7 @@ Notice that the rows containing "d" or "e" are mapped to index "3.0"
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [StringIndexer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.StringIndexer)
+Refer to the [StringIndexer Scala docs](api/scala/org/apache/spark/ml/feature/StringIndexer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/StringIndexerExample.scala %}
@@ -770,7 +770,7 @@ labels (they will be inferred from the columns' metadata):
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [IndexToString Scala docs](api/scala/index.html#org.apache.spark.ml.feature.IndexToString)
+Refer to the [IndexToString Scala docs](api/scala/org/apache/spark/ml/feature/IndexToString.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/IndexToStringExample.scala %}
@@ -809,7 +809,7 @@ for more details on the API.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [OneHotEncoder Scala docs](api/scala/index.html#org.apache.spark.ml.feature.OneHotEncoder) for more details on the API.
+Refer to the [OneHotEncoder Scala docs](api/scala/org/apache/spark/ml/feature/OneHotEncoder.html) for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala %}
 </div>
@@ -835,7 +835,7 @@ Refer to the [OneHotEncoder Python docs](api/python/pyspark.ml.html#pyspark.ml.f
 `VectorIndexer` helps index categorical features in datasets of `Vector`s.
 It can both automatically decide which features are categorical and convert original values to category indices.  Specifically, it does the following:
 
-1. Take an input column of type [Vector](api/scala/index.html#org.apache.spark.ml.linalg.Vector) and a parameter `maxCategories`.
+1. Take an input column of type [Vector](api/scala/org/apache/spark/ml/linalg/Vector.html) and a parameter `maxCategories`.
 2. Decide which features should be categorical based on the number of distinct values, where features with at most `maxCategories` are declared categorical.
 3. Compute 0-based category indices for each categorical feature.
 4. Index categorical features and transform original feature values to indices.
@@ -849,7 +849,7 @@ In the example below, we read in a dataset of labeled points and then use `Vecto
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorIndexer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorIndexer)
+Refer to the [VectorIndexer Scala docs](api/scala/org/apache/spark/ml/feature/VectorIndexer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorIndexerExample.scala %}
@@ -910,7 +910,7 @@ then `interactedCol` as the output column contains:
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Interaction Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Interaction)
+Refer to the [Interaction Scala docs](api/scala/org/apache/spark/ml/feature/Interaction.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/InteractionExample.scala %}
@@ -944,7 +944,7 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Normalizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Normalizer)
+Refer to the [Normalizer Scala docs](api/scala/org/apache/spark/ml/feature/Normalizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/NormalizerExample.scala %}
@@ -986,7 +986,7 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [StandardScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.StandardScaler)
+Refer to the [StandardScaler Scala docs](api/scala/org/apache/spark/ml/feature/StandardScaler.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/StandardScalerExample.scala %}
@@ -1030,7 +1030,7 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [RobustScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RobustScaler)
+Refer to the [RobustScaler Scala docs](api/scala/org/apache/spark/ml/feature/RobustScaler.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/RobustScalerExample.scala %}
@@ -1078,8 +1078,8 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [MinMaxScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinMaxScaler)
-and the [MinMaxScalerModel Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinMaxScalerModel)
+Refer to the [MinMaxScaler Scala docs](api/scala/org/apache/spark/ml/feature/MinMaxScaler.html)
+and the [MinMaxScalerModel Scala docs](api/scala/org/apache/spark/ml/feature/MinMaxScalerModel.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala %}
@@ -1121,8 +1121,8 @@ The following example demonstrates how to load a dataset in libsvm format and th
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [MaxAbsScaler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MaxAbsScaler)
-and the [MaxAbsScalerModel Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MaxAbsScalerModel)
+Refer to the [MaxAbsScaler Scala docs](api/scala/org/apache/spark/ml/feature/MaxAbsScaler.html)
+and the [MaxAbsScalerModel Scala docs](api/scala/org/apache/spark/ml/feature/MaxAbsScalerModel.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala %}
@@ -1157,7 +1157,7 @@ Note that if you have no idea of the upper and lower bounds of the targeted colu
 
 Note also that the splits that you provided have to be in strictly increasing order, i.e. `s0 < s1 < s2 < ... < sn`.
 
-More details can be found in the API docs for [Bucketizer](api/scala/index.html#org.apache.spark.ml.feature.Bucketizer).
+More details can be found in the API docs for [Bucketizer](api/scala/org/apache/spark/ml/feature/Bucketizer.html).
 
 **Examples**
 
@@ -1166,7 +1166,7 @@ The following example demonstrates how to bucketize a column of `Double`s into a
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Bucketizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Bucketizer)
+Refer to the [Bucketizer Scala docs](api/scala/org/apache/spark/ml/feature/Bucketizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/BucketizerExample.scala %}
@@ -1216,7 +1216,7 @@ This example below demonstrates how to transform vectors using a transforming ve
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [ElementwiseProduct Scala docs](api/scala/index.html#org.apache.spark.ml.feature.ElementwiseProduct)
+Refer to the [ElementwiseProduct Scala docs](api/scala/org/apache/spark/ml/feature/ElementwiseProduct.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala %}
@@ -1276,7 +1276,7 @@ This is the output of the `SQLTransformer` with statement `"SELECT *, (v1 + v2)
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [SQLTransformer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.SQLTransformer)
+Refer to the [SQLTransformer Scala docs](api/scala/org/apache/spark/ml/feature/SQLTransformer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/SQLTransformerExample.scala %}
@@ -1336,7 +1336,7 @@ output column to `features`, after transformation we should get the following Da
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorAssembler Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorAssembler)
+Refer to the [VectorAssembler Scala docs](api/scala/org/apache/spark/ml/feature/VectorAssembler.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala %}
@@ -1387,7 +1387,7 @@ to avoid this kind of inconsistent state.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorSizeHint Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorSizeHint)
+Refer to the [VectorSizeHint Scala docs](api/scala/org/apache/spark/ml/feature/VectorSizeHint.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorSizeHintExample.scala %}
@@ -1426,7 +1426,7 @@ NaN values, they will be handled specially and placed into their own bucket, for
 are used, then non-NaN data will be put into buckets[0-3], but NaNs will be counted in a special bucket[4].
 
 Algorithm: The bin ranges are chosen using an approximate algorithm (see the documentation for
-[approxQuantile](api/scala/index.html#org.apache.spark.sql.DataFrameStatFunctions) for a
+[approxQuantile](api/scala/org/apache/spark/sql/DataFrameStatFunctions.html) for a
 detailed description). The precision of the approximation can be controlled with the
 `relativeError` parameter. When set to zero, exact quantiles are calculated
 (**Note:** Computing exact quantiles is an expensive operation). The lower and upper bin bounds
@@ -1470,7 +1470,7 @@ a categorical one. Given `numBuckets = 3`, we should get the following DataFrame
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [QuantileDiscretizer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.QuantileDiscretizer)
+Refer to the [QuantileDiscretizer Scala docs](api/scala/org/apache/spark/ml/feature/QuantileDiscretizer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala %}
@@ -1539,7 +1539,7 @@ the relevant column.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [Imputer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.Imputer)
+Refer to the [Imputer Scala docs](api/scala/org/apache/spark/ml/feature/Imputer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ImputerExample.scala %}
@@ -1620,7 +1620,7 @@ Suppose also that we have potential input attributes for the `userFeatures`, i.e
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [VectorSlicer Scala docs](api/scala/index.html#org.apache.spark.ml.feature.VectorSlicer)
+Refer to the [VectorSlicer Scala docs](api/scala/org/apache/spark/ml/feature/VectorSlicer.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/VectorSlicerExample.scala %}
@@ -1706,7 +1706,7 @@ id | country | hour | clicked | features         | label
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [RFormula Scala docs](api/scala/index.html#org.apache.spark.ml.feature.RFormula)
+Refer to the [RFormula Scala docs](api/scala/org/apache/spark/ml/feature/RFormula.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/RFormulaExample.scala %}
@@ -1770,7 +1770,7 @@ id | features              | clicked | selectedFeatures
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [ChiSqSelector Scala docs](api/scala/index.html#org.apache.spark.ml.feature.ChiSqSelector)
+Refer to the [ChiSqSelector Scala docs](api/scala/org/apache/spark/ml/feature/ChiSqSelector.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala %}
@@ -1856,7 +1856,7 @@ Bucketed Random Projection accepts arbitrary vectors as input features, and supp
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [BucketedRandomProjectionLSH Scala docs](api/scala/index.html#org.apache.spark.ml.feature.BucketedRandomProjectionLSH)
+Refer to the [BucketedRandomProjectionLSH Scala docs](api/scala/org/apache/spark/ml/feature/BucketedRandomProjectionLSH.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala %}
@@ -1897,7 +1897,7 @@ The input sets for MinHash are represented as binary vectors, where the vector i
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [MinHashLSH Scala docs](api/scala/index.html#org.apache.spark.ml.feature.MinHashLSH)
+Refer to the [MinHashLSH Scala docs](api/scala/org/apache/spark/ml/feature/MinHashLSH.html)
 for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/MinHashLSHExample.scala %}
diff --git a/docs/ml-frequent-pattern-mining.md b/docs/ml-frequent-pattern-mining.md
index a243188603997..42d7e50357391 100644
--- a/docs/ml-frequent-pattern-mining.md
+++ b/docs/ml-frequent-pattern-mining.md
@@ -75,7 +75,7 @@ The `FPGrowthModel` provides:
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.fpm.FPGrowth) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/fpm/FPGrowth.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/FPGrowthExample.scala %}
 </div>
@@ -128,7 +128,7 @@ pattern mining problem.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [Scala API docs](api/scala/index.html#org.apache.spark.ml.fpm.PrefixSpan) for more details.
+Refer to the [Scala API docs](api/scala/org/apache/spark/ml/fpm/PrefixSpan.html) for more details.
 
 {% include_example scala/org/apache/spark/examples/ml/PrefixSpanExample.scala %}
 </div>
diff --git a/docs/ml-migration-guide.md b/docs/ml-migration-guide.md
index 860c941e6b44b..f3cd762b6f2ac 100644
--- a/docs/ml-migration-guide.md
+++ b/docs/ml-migration-guide.md
@@ -187,7 +187,7 @@ val mlVec: org.apache.spark.ml.linalg.Vector = mllibVec.asML
 val mlMat: org.apache.spark.ml.linalg.Matrix = mllibMat.asML
 {% endhighlight %}
 
-Refer to the [`MLUtils` Scala docs](api/scala/index.html#org.apache.spark.mllib.util.MLUtils$) for further detail.
+Refer to the [`MLUtils` Scala docs](api/scala/org/apache/spark/mllib/util/MLUtils$.html) for further detail.
 </div>
 
 <div data-lang="java" markdown="1">
@@ -341,9 +341,9 @@ In the `spark.ml` package, there exists one breaking API change and one behavior
 In the `spark.mllib` package, there were several breaking changes, but all in `DeveloperApi` or `Experimental` APIs:
 
 * Gradient-Boosted Trees
-    * *(Breaking change)* The signature of the [`Loss.gradient`](api/scala/index.html#org.apache.spark.mllib.tree.loss.Loss) method was changed.  This is only an issues for users who wrote their own losses for GBTs.
-    * *(Breaking change)* The `apply` and `copy` methods for the case class [`BoostingStrategy`](api/scala/index.html#org.apache.spark.mllib.tree.configuration.BoostingStrategy) have been changed because of a modification to the case class fields.  This could be an issue for users who use `BoostingStrategy` to set GBT parameters.
-* *(Breaking change)* The return value of [`LDA.run`](api/scala/index.html#org.apache.spark.mllib.clustering.LDA) has changed.  It now returns an abstract class `LDAModel` instead of the concrete class `DistributedLDAModel`.  The object of type `LDAModel` can still be cast to the appropriate concrete type, which depends on the optimization algorithm.
+    * *(Breaking change)* The signature of the [`Loss.gradient`](api/scala/org/apache/spark/mllib/tree/loss/Loss.html) method was changed.  This is only an issues for users who wrote their own losses for GBTs.
+    * *(Breaking change)* The `apply` and `copy` methods for the case class [`BoostingStrategy`](api/scala/org/apache/spark/mllib/tree/configuration/BoostingStrategy.html) have been changed because of a modification to the case class fields.  This could be an issue for users who use `BoostingStrategy` to set GBT parameters.
+* *(Breaking change)* The return value of [`LDA.run`](api/scala/org/apache/spark/mllib/clustering/LDA.html) has changed.  It now returns an abstract class `LDAModel` instead of the concrete class `DistributedLDAModel`.  The object of type `LDAModel` can still be cast to the appropriate concrete type, which depends on the optimization algorithm.
 
 In the `spark.ml` package, several major API changes occurred, including:
 
@@ -359,12 +359,12 @@ changes for future releases.
 
 In the `spark.mllib` package, there were several breaking changes.  The first change (in `ALS`) is the only one in a component not marked as Alpha or Experimental.
 
-* *(Breaking change)* In [`ALS`](api/scala/index.html#org.apache.spark.mllib.recommendation.ALS), the extraneous method `solveLeastSquares` has been removed.  The `DeveloperApi` method `analyzeBlocks` was also removed.
-* *(Breaking change)* [`StandardScalerModel`](api/scala/index.html#org.apache.spark.mllib.feature.StandardScalerModel) remains an Alpha component. In it, the `variance` method has been replaced with the `std` method.  To compute the column variance values returned by the original `variance` method, simply square the standard deviation values returned by `std`.
-* *(Breaking change)* [`StreamingLinearRegressionWithSGD`](api/scala/index.html#org.apache.spark.mllib.regression.StreamingLinearRegressionWithSGD) remains an Experimental component.  In it, there were two changes:
+* *(Breaking change)* In [`ALS`](api/scala/org/apache/spark/mllib/recommendation/ALS.html), the extraneous method `solveLeastSquares` has been removed.  The `DeveloperApi` method `analyzeBlocks` was also removed.
+* *(Breaking change)* [`StandardScalerModel`](api/scala/org/apache/spark/mllib/feature/StandardScalerModel.html) remains an Alpha component. In it, the `variance` method has been replaced with the `std` method.  To compute the column variance values returned by the original `variance` method, simply square the standard deviation values returned by `std`.
+* *(Breaking change)* [`StreamingLinearRegressionWithSGD`](api/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionWithSGD.html) remains an Experimental component.  In it, there were two changes:
     * The constructor taking arguments was removed in favor of a builder pattern using the default constructor plus parameter setter methods.
     * Variable `model` is no longer public.
-* *(Breaking change)* [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree) remains an Experimental component.  In it and its associated classes, there were several changes:
+* *(Breaking change)* [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html) remains an Experimental component.  In it and its associated classes, there were several changes:
     * In `DecisionTree`, the deprecated class method `train` has been removed.  (The object/static `train` methods remain.)
     * In `Strategy`, the `checkpointDir` parameter has been removed.  Checkpointing is still supported, but the checkpoint directory must be set before calling tree and tree ensemble training.
 * `PythonMLlibAPI` (the interface between Scala/Java and Python for MLlib) was a public API but is now private, declared `private[python]`.  This was never meant for external use.
@@ -373,31 +373,31 @@ In the `spark.mllib` package, there were several breaking changes.  The first ch
 
 In the `spark.ml` package, the main API changes are from Spark SQL.  We list the most important changes here:
 
-* The old [SchemaRDD](https://spark.apache.org/docs/1.2.1/api/scala/index.html#org.apache.spark.sql.SchemaRDD) has been replaced with [DataFrame](api/scala/index.html#org.apache.spark.sql.DataFrame) with a somewhat modified API.  All algorithms in `spark.ml` which used to use SchemaRDD now use DataFrame.
+* The old [SchemaRDD](https://spark.apache.org/docs/1.2.1/api/scala/index.html#org.apache.spark.sql.SchemaRDD) has been replaced with [DataFrame](api/scala/org/apache/spark/sql/DataFrame.html) with a somewhat modified API.  All algorithms in `spark.ml` which used to use SchemaRDD now use DataFrame.
 * In Spark 1.2, we used implicit conversions from `RDD`s of `LabeledPoint` into `SchemaRDD`s by calling `import sqlContext._` where `sqlContext` was an instance of `SQLContext`.  These implicits have been moved, so we now call `import sqlContext.implicits._`.
 * Java APIs for SQL have also changed accordingly.  Please see the examples above and the [Spark SQL Programming Guide](sql-programming-guide.html) for details.
 
 Other changes were in `LogisticRegression`:
 
 * The `scoreCol` output column (with default value "score") was renamed to be `probabilityCol` (with default value "probability").  The type was originally `Double` (for the probability of class 1.0), but it is now `Vector` (for the probability of each class, to support multiclass classification in the future).
-* In Spark 1.2, `LogisticRegressionModel` did not include an intercept.  In Spark 1.3, it includes an intercept; however, it will always be 0.0 since it uses the default settings for [spark.mllib.LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS).  The option to use an intercept will be added in the future.
+* In Spark 1.2, `LogisticRegressionModel` did not include an intercept.  In Spark 1.3, it includes an intercept; however, it will always be 0.0 since it uses the default settings for [spark.mllib.LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html).  The option to use an intercept will be added in the future.
 
 ## Upgrading from MLlib 1.1 to 1.2
 
 The only API changes in MLlib v1.2 are in
-[`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree),
+[`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html),
 which continues to be an experimental API in MLlib 1.2:
 
 1. *(Breaking change)* The Scala API for classification takes a named argument specifying the number
 of classes.  In MLlib v1.1, this argument was called `numClasses` in Python and
 `numClassesForClassification` in Scala.  In MLlib v1.2, the names are both set to `numClasses`.
 This `numClasses` parameter is specified either via
-[`Strategy`](api/scala/index.html#org.apache.spark.mllib.tree.configuration.Strategy)
-or via [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree)
+[`Strategy`](api/scala/org/apache/spark/mllib/tree/configuration/Strategy.html)
+or via [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html)
 static `trainClassifier` and `trainRegressor` methods.
 
 2. *(Breaking change)* The API for
-[`Node`](api/scala/index.html#org.apache.spark.mllib.tree.model.Node) has changed.
+[`Node`](api/scala/org/apache/spark/mllib/tree/model/Node.html) has changed.
 This should generally not affect user code, unless the user manually constructs decision trees
 (instead of using the `trainClassifier` or `trainRegressor` methods).
 The tree `Node` now includes more information, including the probability of the predicted label
@@ -411,7 +411,7 @@ Examples in the Spark distribution and examples in the
 ## Upgrading from MLlib 1.0 to 1.1
 
 The only API changes in MLlib v1.1 are in
-[`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree),
+[`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html),
 which continues to be an experimental API in MLlib 1.1:
 
 1. *(Breaking change)* The meaning of tree depth has been changed by 1 in order to match
@@ -421,12 +421,12 @@ and in [rpart](http://cran.r-project.org/web/packages/rpart/index.html).
 In MLlib v1.0, a depth-1 tree had 1 leaf node, and a depth-2 tree had 1 root node and 2 leaf nodes.
 In MLlib v1.1, a depth-0 tree has 1 leaf node, and a depth-1 tree has 1 root node and 2 leaf nodes.
 This depth is specified by the `maxDepth` parameter in
-[`Strategy`](api/scala/index.html#org.apache.spark.mllib.tree.configuration.Strategy)
-or via [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree)
+[`Strategy`](api/scala/org/apache/spark/mllib/tree/configuration/Strategy.html)
+or via [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html)
 static `trainClassifier` and `trainRegressor` methods.
 
 2. *(Non-breaking change)* We recommend using the newly added `trainClassifier` and `trainRegressor`
-methods to build a [`DecisionTree`](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree),
+methods to build a [`DecisionTree`](api/scala/org/apache/spark/mllib/tree/DecisionTree.html),
 rather than using the old parameter class `Strategy`.  These new training methods explicitly
 separate classification and regression, and they replace specialized parameter types with
 simple `String` types.
diff --git a/docs/ml-pipeline.md b/docs/ml-pipeline.md
index 993a428ab5489..0b581e1a09c97 100644
--- a/docs/ml-pipeline.md
+++ b/docs/ml-pipeline.md
@@ -238,7 +238,7 @@ notes, then it should be treated as a bug to be fixed.
 
 This section gives code examples illustrating the functionality discussed above.
 For more info, please refer to the API documentation
-([Scala](api/scala/index.html#org.apache.spark.ml.package),
+([Scala](api/scala/org/apache/spark/ml/package.html),
 [Java](api/java/org/apache/spark/ml/package-summary.html),
 and [Python](api/python/pyspark.ml.html)).
 
@@ -250,9 +250,9 @@ This example covers the concepts of `Estimator`, `Transformer`, and `Param`.
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`Estimator` Scala docs](api/scala/index.html#org.apache.spark.ml.Estimator),
-the [`Transformer` Scala docs](api/scala/index.html#org.apache.spark.ml.Transformer) and
-the [`Params` Scala docs](api/scala/index.html#org.apache.spark.ml.param.Params) for details on the API.
+Refer to the [`Estimator` Scala docs](api/scala/org/apache/spark/ml/Estimator.html),
+the [`Transformer` Scala docs](api/scala/org/apache/spark/ml/Transformer.html) and
+the [`Params` Scala docs](api/scala/org/apache/spark/ml/param/Params.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala %}
 </div>
@@ -285,7 +285,7 @@ This example follows the simple text document `Pipeline` illustrated in the figu
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`Pipeline` Scala docs](api/scala/index.html#org.apache.spark.ml.Pipeline) for details on the API.
+Refer to the [`Pipeline` Scala docs](api/scala/org/apache/spark/ml/Pipeline.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/PipelineExample.scala %}
 </div>
diff --git a/docs/ml-statistics.md b/docs/ml-statistics.md
index 997c57244b2ef..a3d57ff7d266b 100644
--- a/docs/ml-statistics.md
+++ b/docs/ml-statistics.md
@@ -50,7 +50,7 @@ correlation methods are currently Pearson's and Spearman's correlation.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Correlation`](api/scala/index.html#org.apache.spark.ml.stat.Correlation$)
+[`Correlation`](api/scala/org/apache/spark/ml/stat/Correlation$.html)
 computes the correlation matrix for the input Dataset of Vectors using the specified method.
 The output will be a DataFrame that contains the correlation matrix of the column of vectors.
 
@@ -87,7 +87,7 @@ the Chi-squared statistic is computed. All label and feature values must be cate
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`ChiSquareTest` Scala docs](api/scala/index.html#org.apache.spark.ml.stat.ChiSquareTest$) for details on the API.
+Refer to the [`ChiSquareTest` Scala docs](api/scala/org/apache/spark/ml/stat/ChiSquareTest$.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ChiSquareTestExample.scala %}
 </div>
@@ -114,7 +114,7 @@ as well as the total count.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-The following example demonstrates using [`Summarizer`](api/scala/index.html#org.apache.spark.ml.stat.Summarizer$)
+The following example demonstrates using [`Summarizer`](api/scala/org/apache/spark/ml/stat/Summarizer$.html)
 to compute the mean and variance for a vector column of the input dataframe, with and without a weight column.
 
 {% include_example scala/org/apache/spark/examples/ml/SummarizerExample.scala %}
@@ -133,4 +133,4 @@ Refer to the [`Summarizer` Python docs](api/python/index.html#pyspark.ml.stat.Su
 {% include_example python/ml/summarizer_example.py %}
 </div>
 
-</div>
\ No newline at end of file
+</div>
diff --git a/docs/ml-tuning.md b/docs/ml-tuning.md
index 0717cce538bf3..49e23684e5974 100644
--- a/docs/ml-tuning.md
+++ b/docs/ml-tuning.md
@@ -49,12 +49,12 @@ Built-in Cross-Validation and other tooling allow users to optimize hyperparamet
 An important task in ML is *model selection*, or using data to find the best model or parameters for a given task.  This is also called *tuning*.
 Tuning may be done for individual `Estimator`s such as `LogisticRegression`, or for entire `Pipeline`s which include multiple algorithms, featurization, and other steps.  Users can tune an entire `Pipeline` at once, rather than tuning each element in the `Pipeline` separately.
 
-MLlib supports model selection using tools such as [`CrossValidator`](api/scala/index.html#org.apache.spark.ml.tuning.CrossValidator) and [`TrainValidationSplit`](api/scala/index.html#org.apache.spark.ml.tuning.TrainValidationSplit).
+MLlib supports model selection using tools such as [`CrossValidator`](api/scala/org/apache/spark/ml/tuning/CrossValidator.html) and [`TrainValidationSplit`](api/scala/org/apache/spark/ml/tuning/TrainValidationSplit.html).
 These tools require the following items:
 
-* [`Estimator`](api/scala/index.html#org.apache.spark.ml.Estimator): algorithm or `Pipeline` to tune
+* [`Estimator`](api/scala/org/apache/spark/ml/Estimator.html): algorithm or `Pipeline` to tune
 * Set of `ParamMap`s: parameters to choose from, sometimes called a "parameter grid" to search over
-* [`Evaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.Evaluator): metric to measure how well a fitted `Model` does on held-out test data
+* [`Evaluator`](api/scala/org/apache/spark/ml/evaluation/Evaluator.html): metric to measure how well a fitted `Model` does on held-out test data
 
 At a high level, these model selection tools work as follows:
 
@@ -63,13 +63,13 @@ At a high level, these model selection tools work as follows:
   * For each `ParamMap`, they fit the `Estimator` using those parameters, get the fitted `Model`, and evaluate the `Model`'s performance using the `Evaluator`.
 * They select the `Model` produced by the best-performing set of parameters.
 
-The `Evaluator` can be a [`RegressionEvaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.RegressionEvaluator)
-for regression problems, a [`BinaryClassificationEvaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.BinaryClassificationEvaluator)
-for binary data, or a [`MulticlassClassificationEvaluator`](api/scala/index.html#org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator)
+The `Evaluator` can be a [`RegressionEvaluator`](api/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.html)
+for regression problems, a [`BinaryClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.html)
+for binary data, or a [`MulticlassClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.html)
 for multiclass problems. The default metric used to choose the best `ParamMap` can be overridden by the `setMetricName`
 method in each of these evaluators.
 
-To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/index.html#org.apache.spark.ml.tuning.ParamGridBuilder) utility.
+To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/org/apache/spark/ml/tuning/ParamGridBuilder.html) utility.
 By default, sets of parameters from the parameter grid are evaluated in serial. Parameter evaluation can be done in parallel by setting `parallelism` with a value of 2 or more (a value of 1 will be serial) before running model selection with `CrossValidator` or `TrainValidationSplit`.
 The value of `parallelism` should be chosen carefully to maximize parallelism without exceeding cluster resources, and larger values may not always lead to improved performance.  Generally speaking, a value up to 10 should be sufficient for most clusters.
 
@@ -93,7 +93,7 @@ However, it is also a well-established method for choosing parameters which is m
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`CrossValidator` Scala docs](api/scala/index.html#org.apache.spark.ml.tuning.CrossValidator) for details on the API.
+Refer to the [`CrossValidator` Scala docs](api/scala/org/apache/spark/ml/tuning/CrossValidator.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala %}
 </div>
@@ -133,7 +133,7 @@ Like `CrossValidator`, `TrainValidationSplit` finally fits the `Estimator` using
 
 <div data-lang="scala" markdown="1">
 
-Refer to the [`TrainValidationSplit` Scala docs](api/scala/index.html#org.apache.spark.ml.tuning.TrainValidationSplit) for details on the API.
+Refer to the [`TrainValidationSplit` Scala docs](api/scala/org/apache/spark/ml/tuning/TrainValidationSplit.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala %}
 </div>
diff --git a/docs/mllib-clustering.md b/docs/mllib-clustering.md
index 12c33a5e38049..4cb2e259ccfbc 100644
--- a/docs/mllib-clustering.md
+++ b/docs/mllib-clustering.md
@@ -55,12 +55,12 @@ initialization via k-means\|\|.
 The following code snippets can be executed in `spark-shell`.
 
 In the following example after loading and parsing data, we use the
-[`KMeans`](api/scala/index.html#org.apache.spark.mllib.clustering.KMeans) object to cluster the data
+[`KMeans`](api/scala/org/apache/spark/mllib/clustering/KMeans.html) object to cluster the data
 into two clusters. The number of desired clusters is passed to the algorithm. We then compute Within
 Set Sum of Squared Error (WSSSE). You can reduce this error measure by increasing *k*. In fact, the
 optimal *k* is usually one where there is an "elbow" in the WSSSE graph.
 
-Refer to the [`KMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.KMeans) and [`KMeansModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.KMeansModel) for details on the API.
+Refer to the [`KMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/KMeans.html) and [`KMeansModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/KMeansModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/KMeansExample.scala %}
 </div>
@@ -111,11 +111,11 @@ has the following parameters:
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 In the following example after loading and parsing data, we use a
-[GaussianMixture](api/scala/index.html#org.apache.spark.mllib.clustering.GaussianMixture)
+[GaussianMixture](api/scala/org/apache/spark/mllib/clustering/GaussianMixture.html)
 object to cluster the data into two clusters. The number of desired clusters is passed
 to the algorithm. We then output the parameters of the mixture model.
 
-Refer to the [`GaussianMixture` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.GaussianMixture) and [`GaussianMixtureModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.GaussianMixtureModel) for details on the API.
+Refer to the [`GaussianMixture` Scala docs](api/scala/org/apache/spark/mllib/clustering/GaussianMixture.html) and [`GaussianMixtureModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/GaussianMixtureExample.scala %}
 </div>
@@ -172,15 +172,15 @@ In the following, we show code snippets to demonstrate how to use PIC in `spark.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`PowerIterationClustering`](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClustering) 
+[`PowerIterationClustering`](api/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.html) 
 implements the PIC algorithm.
 It takes an `RDD` of `(srcId: Long, dstId: Long, similarity: Double)` tuples representing the
 affinity matrix.
 Calling `PowerIterationClustering.run` returns a
-[`PowerIterationClusteringModel`](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClusteringModel),
+[`PowerIterationClusteringModel`](api/scala/org/apache/spark/mllib/clustering/PowerIterationClusteringModel.html),
 which contains the computed clustering assignments.
 
-Refer to the [`PowerIterationClustering` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClustering) and [`PowerIterationClusteringModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.PowerIterationClusteringModel) for details on the API.
+Refer to the [`PowerIterationClustering` Scala docs](api/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.html) and [`PowerIterationClusteringModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/PowerIterationClusteringModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala %}
 </div>
@@ -278,9 +278,9 @@ separately.
 **Expectation Maximization**
 
 Implemented in
-[`EMLDAOptimizer`](api/scala/index.html#org.apache.spark.mllib.clustering.EMLDAOptimizer)
+[`EMLDAOptimizer`](api/scala/org/apache/spark/mllib/clustering/EMLDAOptimizer.html)
 and
-[`DistributedLDAModel`](api/scala/index.html#org.apache.spark.mllib.clustering.DistributedLDAModel).
+[`DistributedLDAModel`](api/scala/org/apache/spark/mllib/clustering/DistributedLDAModel.html).
 
 For the parameters provided to `LDA`:
 
@@ -350,13 +350,13 @@ perplexity of the provided `documents` given the inferred topics.
 **Examples**
 
 In the following example, we load word count vectors representing a corpus of documents.
-We then use [LDA](api/scala/index.html#org.apache.spark.mllib.clustering.LDA)
+We then use [LDA](api/scala/org/apache/spark/mllib/clustering/LDA.html)
 to infer three topics from the documents. The number of desired clusters is passed
 to the algorithm. We then output the topics, represented as probability distributions over words.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`LDA` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.LDA) and [`DistributedLDAModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.DistributedLDAModel) for details on the API.
+Refer to the [`LDA` Scala docs](api/scala/org/apache/spark/mllib/clustering/LDA.html) and [`DistributedLDAModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/DistributedLDAModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/LatentDirichletAllocationExample.scala %}
 </div>
@@ -398,7 +398,7 @@ The implementation in MLlib has the following parameters:
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`BisectingKMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.BisectingKMeans) and [`BisectingKMeansModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.BisectingKMeansModel) for details on the API.
+Refer to the [`BisectingKMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/BisectingKMeans.html) and [`BisectingKMeansModel` Scala docs](api/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/BisectingKMeansExample.scala %}
 </div>
@@ -451,7 +451,7 @@ This example shows how to estimate clusters on streaming data.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`StreamingKMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.StreamingKMeans) for details on the API.
+Refer to the [`StreamingKMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/StreamingKMeans.html) for details on the API.
 And Refer to [Spark Streaming Programming Guide](streaming-programming-guide.html#initializing) for details on StreamingContext.
 
 {% include_example scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala %}
diff --git a/docs/mllib-collaborative-filtering.md b/docs/mllib-collaborative-filtering.md
index 21546a63263f9..aaefa59c4a9c3 100644
--- a/docs/mllib-collaborative-filtering.md
+++ b/docs/mllib-collaborative-filtering.md
@@ -76,11 +76,11 @@ best parameter learned from a sampled subset to the full dataset and expect simi
 
 <div data-lang="scala" markdown="1">
 In the following example, we load rating data. Each row consists of a user, a product and a rating.
-We use the default [ALS.train()](api/scala/index.html#org.apache.spark.mllib.recommendation.ALS$) 
+We use the default [ALS.train()](api/scala/org/apache/spark/mllib/recommendation/ALS$.html)
 method which assumes ratings are explicit. We evaluate the
 recommendation model by measuring the Mean Squared Error of rating prediction.
 
-Refer to the [`ALS` Scala docs](api/scala/index.html#org.apache.spark.mllib.recommendation.ALS) for more details on the API.
+Refer to the [`ALS` Scala docs](api/scala/org/apache/spark/mllib/recommendation/ALS.html) for more details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RecommendationExample.scala %}
 
diff --git a/docs/mllib-data-types.md b/docs/mllib-data-types.md
index cdac46284b6be..6d3b1a599d48b 100644
--- a/docs/mllib-data-types.md
+++ b/docs/mllib-data-types.md
@@ -42,13 +42,13 @@ of the vector.
 <div data-lang="scala" markdown="1">
 
 The base class of local vectors is
-[`Vector`](api/scala/index.html#org.apache.spark.mllib.linalg.Vector), and we provide two
-implementations: [`DenseVector`](api/scala/index.html#org.apache.spark.mllib.linalg.DenseVector) and
-[`SparseVector`](api/scala/index.html#org.apache.spark.mllib.linalg.SparseVector).  We recommend
+[`Vector`](api/scala/org/apache/spark/mllib/linalg/Vector.html), and we provide two
+implementations: [`DenseVector`](api/scala/org/apache/spark/mllib/linalg/DenseVector.html) and
+[`SparseVector`](api/scala/org/apache/spark/mllib/linalg/SparseVector.html).  We recommend
 using the factory methods implemented in
-[`Vectors`](api/scala/index.html#org.apache.spark.mllib.linalg.Vectors$) to create local vectors.
+[`Vectors`](api/scala/org/apache/spark/mllib/linalg/Vectors$.html) to create local vectors.
 
-Refer to the [`Vector` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Vector) and [`Vectors` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Vectors$) for details on the API.
+Refer to the [`Vector` Scala docs](api/scala/org/apache/spark/mllib/linalg/Vector.html) and [`Vectors` Scala docs](api/scala/org/apache/spark/mllib/linalg/Vectors$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
@@ -138,9 +138,9 @@ For multiclass classification, labels should be class indices starting from zero
 <div data-lang="scala" markdown="1">
 
 A labeled point is represented by the case class
-[`LabeledPoint`](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint).
+[`LabeledPoint`](api/scala/org/apache/spark/mllib/regression/LabeledPoint.html).
 
-Refer to the [`LabeledPoint` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint) for details on the API.
+Refer to the [`LabeledPoint` Scala docs](api/scala/org/apache/spark/mllib/regression/LabeledPoint.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.Vectors
@@ -211,10 +211,10 @@ After loading, the feature indices are converted to zero-based.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`MLUtils.loadLibSVMFile`](api/scala/index.html#org.apache.spark.mllib.util.MLUtils$) reads training
+[`MLUtils.loadLibSVMFile`](api/scala/org/apache/spark/mllib/util/MLUtils$.html) reads training
 examples stored in LIBSVM format.
 
-Refer to the [`MLUtils` Scala docs](api/scala/index.html#org.apache.spark.mllib.util.MLUtils$) for details on the API.
+Refer to the [`MLUtils` Scala docs](api/scala/org/apache/spark/mllib/util/MLUtils$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.regression.LabeledPoint
@@ -272,14 +272,14 @@ is stored in a one-dimensional array `[1.0, 3.0, 5.0, 2.0, 4.0, 6.0]` with the m
 <div data-lang="scala" markdown="1">
 
 The base class of local matrices is
-[`Matrix`](api/scala/index.html#org.apache.spark.mllib.linalg.Matrix), and we provide two
-implementations: [`DenseMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.DenseMatrix),
-and [`SparseMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.SparseMatrix).
+[`Matrix`](api/scala/org/apache/spark/mllib/linalg/Matrix.html), and we provide two
+implementations: [`DenseMatrix`](api/scala/org/apache/spark/mllib/linalg/DenseMatrix.html),
+and [`SparseMatrix`](api/scala/org/apache/spark/mllib/linalg/SparseMatrix.html).
 We recommend using the factory methods implemented
-in [`Matrices`](api/scala/index.html#org.apache.spark.mllib.linalg.Matrices$) to create local
+in [`Matrices`](api/scala/org/apache/spark/mllib/linalg/Matrices$.html) to create local
 matrices. Remember, local matrices in MLlib are stored in column-major order.
 
-Refer to the [`Matrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Matrix) and [`Matrices` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Matrices$) for details on the API.
+Refer to the [`Matrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/Matrix.html) and [`Matrices` Scala docs](api/scala/org/apache/spark/mllib/linalg/Matrices$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.{Matrix, Matrices}
@@ -377,12 +377,12 @@ limited by the integer range but it should be much smaller in practice.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-A [`RowMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) can be
+A [`RowMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) can be
 created from an `RDD[Vector]` instance.  Then we can compute its column summary statistics and decompositions.
 [QR decomposition](https://en.wikipedia.org/wiki/QR_decomposition) is of the form A = QR where Q is an orthogonal matrix and R is an upper triangular matrix.
 For [singular value decomposition (SVD)](https://en.wikipedia.org/wiki/Singular_value_decomposition) and [principal component analysis (PCA)](https://en.wikipedia.org/wiki/Principal_component_analysis), please refer to [Dimensionality reduction](mllib-dimensionality-reduction.html).
 
-Refer to the [`RowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) for details on the API.
+Refer to the [`RowMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.Vector
@@ -463,13 +463,13 @@ vector.
 <div data-lang="scala" markdown="1">
 
 An
-[`IndexedRowMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.IndexedRowMatrix)
+[`IndexedRowMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.html)
 can be created from an `RDD[IndexedRow]` instance, where
-[`IndexedRow`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.IndexedRow) is a
+[`IndexedRow`](api/scala/org/apache/spark/mllib/linalg/distributed/IndexedRow.html) is a
 wrapper over `(Long, Vector)`.  An `IndexedRowMatrix` can be converted to a `RowMatrix` by dropping
 its row indices.
 
-Refer to the [`IndexedRowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.IndexedRowMatrix) for details on the API.
+Refer to the [`IndexedRowMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.distributed.{IndexedRow, IndexedRowMatrix, RowMatrix}
@@ -568,14 +568,14 @@ dimensions of the matrix are huge and the matrix is very sparse.
 <div data-lang="scala" markdown="1">
 
 A
-[`CoordinateMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.CoordinateMatrix)
+[`CoordinateMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/CoordinateMatrix.html)
 can be created from an `RDD[MatrixEntry]` instance, where
-[`MatrixEntry`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.MatrixEntry) is a
+[`MatrixEntry`](api/scala/org/apache/spark/mllib/linalg/distributed/MatrixEntry.html) is a
 wrapper over `(Long, Long, Double)`.  A `CoordinateMatrix` can be converted to an `IndexedRowMatrix`
 with sparse rows by calling `toIndexedRowMatrix`.  Other computations for 
 `CoordinateMatrix` are not currently supported.
 
-Refer to the [`CoordinateMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.CoordinateMatrix) for details on the API.
+Refer to the [`CoordinateMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/CoordinateMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.distributed.{CoordinateMatrix, MatrixEntry}
@@ -678,12 +678,12 @@ the sub-matrix at the given index with size `rowsPerBlock` x `colsPerBlock`.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-A [`BlockMatrix`](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.BlockMatrix) can be
+A [`BlockMatrix`](api/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.html) can be
 most easily created from an `IndexedRowMatrix` or `CoordinateMatrix` by calling `toBlockMatrix`.
 `toBlockMatrix` creates blocks of size 1024 x 1024 by default.
 Users may change the block size by supplying the values through `toBlockMatrix(rowsPerBlock, colsPerBlock)`.
 
-Refer to the [`BlockMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.BlockMatrix) for details on the API.
+Refer to the [`BlockMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.mllib.linalg.distributed.{BlockMatrix, CoordinateMatrix, MatrixEntry}
diff --git a/docs/mllib-decision-tree.md b/docs/mllib-decision-tree.md
index 045da744239b9..455649c8e686e 100644
--- a/docs/mllib-decision-tree.md
+++ b/docs/mllib-decision-tree.md
@@ -151,7 +151,7 @@ When tuning these parameters, be careful to validate on held-out test data to av
 
 * **`maxDepth`**: Maximum depth of a tree.  Deeper trees are more expressive (potentially allowing higher accuracy), but they are also more costly to train and are more likely to overfit.
 
-* **`minInstancesPerNode`**: For a node to be split further, each of its children must receive at least this number of training instances.  This is commonly used with [RandomForest](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) since those are often trained deeper than individual trees.
+* **`minInstancesPerNode`**: For a node to be split further, each of its children must receive at least this number of training instances.  This is commonly used with [RandomForest](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) since those are often trained deeper than individual trees.
 
 * **`minInfoGain`**: For a node to be split further, the split must improve at least this much (in terms of information gain).
 
@@ -167,13 +167,13 @@ These parameters may be tuned.  Be careful to validate on held-out test data whe
   * The default value is conservatively chosen to be 256 MiB to allow the decision algorithm to work in most scenarios.  Increasing `maxMemoryInMB` can lead to faster training (if the memory is available) by allowing fewer passes over the data.  However, there may be decreasing returns as `maxMemoryInMB` grows since the amount of communication on each iteration can be proportional to `maxMemoryInMB`.
   * *Implementation details*: For faster processing, the decision tree algorithm collects statistics about groups of nodes to split (rather than 1 node at a time).  The number of nodes which can be handled in one group is determined by the memory requirements (which vary per features).  The `maxMemoryInMB` parameter specifies the memory limit in terms of megabytes which each worker can use for these statistics.
 
-* **`subsamplingRate`**: Fraction of the training data used for learning the decision tree.  This parameter is most relevant for training ensembles of trees (using [`RandomForest`](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) and [`GradientBoostedTrees`](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees)), where it can be useful to subsample the original data.  For training a single decision tree, this parameter is less useful since the number of training instances is generally not the main constraint.
+* **`subsamplingRate`**: Fraction of the training data used for learning the decision tree.  This parameter is most relevant for training ensembles of trees (using [`RandomForest`](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) and [`GradientBoostedTrees`](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html)), where it can be useful to subsample the original data.  For training a single decision tree, this parameter is less useful since the number of training instances is generally not the main constraint.
 
 * **`impurity`**: Impurity measure (discussed above) used to choose between candidate splits.  This measure must match the `algo` parameter.
 
 ### Caching and checkpointing
 
-MLlib 1.2 adds several features for scaling up to larger (deeper) trees and tree ensembles.  When `maxDepth` is set to be large, it can be useful to turn on node ID caching and checkpointing.  These parameters are also useful for [RandomForest](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) when `numTrees` is set to be large.
+MLlib 1.2 adds several features for scaling up to larger (deeper) trees and tree ensembles.  When `maxDepth` is set to be large, it can be useful to turn on node ID caching and checkpointing.  These parameters are also useful for [RandomForest](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) when `numTrees` is set to be large.
 
 * **`useNodeIdCache`**: If this is set to true, the algorithm will avoid passing the current model (tree or trees) to executors on each iteration.
   * This can be useful with deep trees (speeding up computation on workers) and for large Random Forests (reducing communication on each iteration).
@@ -207,7 +207,7 @@ maximum tree depth of 5. The test error is calculated to measure the algorithm a
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`DecisionTree` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree) and [`DecisionTreeModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.DecisionTreeModel) for details on the API.
+Refer to the [`DecisionTree` Scala docs](api/scala/org/apache/spark/mllib/tree/DecisionTree.html) and [`DecisionTreeModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala %}
 </div>
@@ -238,7 +238,7 @@ depth of 5. The Mean Squared Error (MSE) is computed at the end to evaluate
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`DecisionTree` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.DecisionTree) and [`DecisionTreeModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.DecisionTreeModel) for details on the API.
+Refer to the [`DecisionTree` Scala docs](api/scala/org/apache/spark/mllib/tree/DecisionTree.html) and [`DecisionTreeModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala %}
 </div>
diff --git a/docs/mllib-dimensionality-reduction.md b/docs/mllib-dimensionality-reduction.md
index 5eb36b4228ca3..8818e403aabbc 100644
--- a/docs/mllib-dimensionality-reduction.md
+++ b/docs/mllib-dimensionality-reduction.md
@@ -77,7 +77,7 @@ passes, $O(n)$ storage on each executor, and $O(n k)$ storage on the driver.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`SingularValueDecomposition` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.SingularValueDecomposition) for details on the API.
+Refer to the [`SingularValueDecomposition` Scala docs](api/scala/org/apache/spark/mllib/linalg/SingularValueDecomposition.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SVDExample.scala %}
 
@@ -117,14 +117,14 @@ the rotation matrix are called principal components. PCA is used widely in dimen
 The following code demonstrates how to compute principal components on a `RowMatrix`
 and use them to project the vectors into a low-dimensional space.
 
-Refer to the [`RowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) for details on the API.
+Refer to the [`RowMatrix` Scala docs](api/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala %}
 
 The following code demonstrates how to compute principal components on source vectors
 and use them to project the vectors into a low-dimensional space while keeping associated labels:
 
-Refer to the [`PCA` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.PCA) for details on the API.
+Refer to the [`PCA` Scala docs](api/scala/org/apache/spark/mllib/feature/PCA.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala %}
 
diff --git a/docs/mllib-ensembles.md b/docs/mllib-ensembles.md
index 6149f458214e6..27a9fe67f3f6d 100644
--- a/docs/mllib-ensembles.md
+++ b/docs/mllib-ensembles.md
@@ -24,7 +24,7 @@ license: |
 
 An [ensemble method](http://en.wikipedia.org/wiki/Ensemble_learning)
 is a learning algorithm which creates a model composed of a set of other base models.
-`spark.mllib` supports two major ensemble algorithms: [`GradientBoostedTrees`](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees) and [`RandomForest`](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$).
+`spark.mllib` supports two major ensemble algorithms: [`GradientBoostedTrees`](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html) and [`RandomForest`](api/scala/org/apache/spark/mllib/tree/RandomForest$.html).
 Both use [decision trees](mllib-decision-tree.html) as their base models.
 
 ## Gradient-Boosted Trees vs. Random Forests
@@ -111,7 +111,7 @@ The test error is calculated to measure the algorithm accuracy.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`RandomForest` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) and [`RandomForestModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.RandomForestModel) for details on the API.
+Refer to the [`RandomForest` Scala docs](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) and [`RandomForestModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/RandomForestModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala %}
 </div>
@@ -142,7 +142,7 @@ The Mean Squared Error (MSE) is computed at the end to evaluate
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`RandomForest` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.RandomForest$) and [`RandomForestModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.RandomForestModel) for details on the API.
+Refer to the [`RandomForest` Scala docs](api/scala/org/apache/spark/mllib/tree/RandomForest$.html) and [`RandomForestModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/RandomForestModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala %}
 </div>
@@ -252,7 +252,7 @@ The test error is calculated to measure the algorithm accuracy.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`GradientBoostedTrees` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees) and [`GradientBoostedTreesModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.GradientBoostedTreesModel) for details on the API.
+Refer to the [`GradientBoostedTrees` Scala docs](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html) and [`GradientBoostedTreesModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/GradientBoostedTreesModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala %}
 </div>
@@ -283,7 +283,7 @@ The Mean Squared Error (MSE) is computed at the end to evaluate
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`GradientBoostedTrees` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.GradientBoostedTrees) and [`GradientBoostedTreesModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.tree.model.GradientBoostedTreesModel) for details on the API.
+Refer to the [`GradientBoostedTrees` Scala docs](api/scala/org/apache/spark/mllib/tree/GradientBoostedTrees.html) and [`GradientBoostedTreesModel` Scala docs](api/scala/org/apache/spark/mllib/tree/model/GradientBoostedTreesModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala %}
 </div>
diff --git a/docs/mllib-evaluation-metrics.md b/docs/mllib-evaluation-metrics.md
index fb2883de6810a..f9efa769fc140 100644
--- a/docs/mllib-evaluation-metrics.md
+++ b/docs/mllib-evaluation-metrics.md
@@ -117,7 +117,7 @@ The following code snippets illustrate how to load a sample dataset, train a bin
 data, and evaluate the performance of the algorithm by several binary evaluation metrics.
 
 <div data-lang="scala" markdown="1">
-Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS) and [`BinaryClassificationMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.BinaryClassificationMetrics) for details on the API.
+Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html) and [`BinaryClassificationMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala %}
 
@@ -243,7 +243,7 @@ The following code snippets illustrate how to load a sample dataset, train a mul
 the data, and evaluate the performance of the algorithm by several multiclass classification evaluation metrics.
 
 <div data-lang="scala" markdown="1">
-Refer to the [`MulticlassMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.MulticlassMetrics) for details on the API.
+Refer to the [`MulticlassMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/MulticlassMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala %}
 
@@ -393,7 +393,7 @@ True classes:
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`MultilabelMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.MultilabelMetrics) for details on the API.
+Refer to the [`MultilabelMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala %}
 
@@ -521,7 +521,7 @@ expanded world of non-positive weights are "the same as never having interacted
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`RegressionMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.RegressionMetrics) and [`RankingMetrics` Scala docs](api/scala/index.html#org.apache.spark.mllib.evaluation.RankingMetrics) for details on the API.
+Refer to the [`RegressionMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/RegressionMetrics.html) and [`RankingMetrics` Scala docs](api/scala/org/apache/spark/mllib/evaluation/RankingMetrics.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala %}
 
diff --git a/docs/mllib-feature-extraction.md b/docs/mllib-feature-extraction.md
index 33a223ad486af..8df9699150f6a 100644
--- a/docs/mllib-feature-extraction.md
+++ b/docs/mllib-feature-extraction.md
@@ -69,12 +69,12 @@ We refer users to the [Stanford NLP Group](http://nlp.stanford.edu/) and
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-TF and IDF are implemented in [HashingTF](api/scala/index.html#org.apache.spark.mllib.feature.HashingTF)
-and [IDF](api/scala/index.html#org.apache.spark.mllib.feature.IDF).
+TF and IDF are implemented in [HashingTF](api/scala/org/apache/spark/mllib/feature/HashingTF.html)
+and [IDF](api/scala/org/apache/spark/mllib/feature/IDF.html).
 `HashingTF` takes an `RDD[Iterable[_]]` as the input.
 Each record could be an iterable of strings or other types.
 
-Refer to the [`HashingTF` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.HashingTF) for details on the API.
+Refer to the [`HashingTF` Scala docs](api/scala/org/apache/spark/mllib/feature/HashingTF.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/TFIDFExample.scala %}
 </div>
@@ -135,7 +135,7 @@ Here we assume the extracted file is `text8` and in same directory as you run th
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`Word2Vec` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.Word2Vec) for details on the API.
+Refer to the [`Word2Vec` Scala docs](api/scala/org/apache/spark/mllib/feature/Word2Vec.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/Word2VecExample.scala %}
 </div>
@@ -159,19 +159,19 @@ against features with very large variances exerting an overly large influence du
 
 ### Model Fitting
 
-[`StandardScaler`](api/scala/index.html#org.apache.spark.mllib.feature.StandardScaler) has the
+[`StandardScaler`](api/scala/org/apache/spark/mllib/feature/StandardScaler.html) has the
 following parameters in the constructor:
 
 * `withMean` False by default. Centers the data with mean before scaling. It will build a dense
 output, so take care when applying to sparse input.
 * `withStd` True by default. Scales the data to unit standard deviation.
 
-We provide a [`fit`](api/scala/index.html#org.apache.spark.mllib.feature.StandardScaler) method in
+We provide a [`fit`](api/scala/org/apache/spark/mllib/feature/StandardScaler.html) method in
 `StandardScaler` which can take an input of `RDD[Vector]`, learn the summary statistics, and then
 return a model which can transform the input dataset into unit standard deviation and/or zero mean features
 depending how we configure the `StandardScaler`.
 
-This model implements [`VectorTransformer`](api/scala/index.html#org.apache.spark.mllib.feature.VectorTransformer)
+This model implements [`VectorTransformer`](api/scala/org/apache/spark/mllib/feature/VectorTransformer.html)
 which can apply the standardization on a `Vector` to produce a transformed `Vector` or on
 an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
 
@@ -185,7 +185,7 @@ so that the new features have unit standard deviation and/or zero mean.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`StandardScaler` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.StandardScaler) for details on the API.
+Refer to the [`StandardScaler` Scala docs](api/scala/org/apache/spark/mllib/feature/StandardScaler.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/StandardScalerExample.scala %}
 </div>
@@ -203,12 +203,12 @@ Normalizer scales individual samples to have unit $L^p$ norm. This is a common o
 classification or clustering. For example, the dot product of two $L^2$ normalized TF-IDF vectors
 is the cosine similarity of the vectors.
 
-[`Normalizer`](api/scala/index.html#org.apache.spark.mllib.feature.Normalizer) has the following
+[`Normalizer`](api/scala/org/apache/spark/mllib/feature/Normalizer.html) has the following
 parameter in the constructor:
 
 * `p` Normalization in $L^p$ space, $p = 2$ by default.
 
-`Normalizer` implements [`VectorTransformer`](api/scala/index.html#org.apache.spark.mllib.feature.VectorTransformer)
+`Normalizer` implements [`VectorTransformer`](api/scala/org/apache/spark/mllib/feature/VectorTransformer.html)
 which can apply the normalization on a `Vector` to produce a transformed `Vector` or on
 an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
 
@@ -221,7 +221,7 @@ with $L^2$ norm, and $L^\infty$ norm.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-Refer to the [`Normalizer` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.Normalizer) for details on the API.
+Refer to the [`Normalizer` Scala docs](api/scala/org/apache/spark/mllib/feature/Normalizer.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/NormalizerExample.scala %}
 </div>
@@ -239,7 +239,7 @@ Refer to the [`Normalizer` Python docs](api/python/pyspark.mllib.html#pyspark.ml
 features for use in model construction. It reduces the size of the feature space, which can improve
 both speed and statistical learning behavior.
 
-[`ChiSqSelector`](api/scala/index.html#org.apache.spark.mllib.feature.ChiSqSelector) implements
+[`ChiSqSelector`](api/scala/org/apache/spark/mllib/feature/ChiSqSelector.html) implements
 Chi-Squared feature selection. It operates on labeled data with categorical features. ChiSqSelector uses the
 [Chi-Squared test of independence](https://en.wikipedia.org/wiki/Chi-squared_test) to decide which
 features to choose. It supports five selection methods: `numTopFeatures`, `percentile`, `fpr`, `fdr`, `fwe`:
@@ -257,7 +257,7 @@ The number of features to select can be tuned using a held-out validation set.
 
 ### Model Fitting
 
-The [`fit`](api/scala/index.html#org.apache.spark.mllib.feature.ChiSqSelector) method takes
+The [`fit`](api/scala/org/apache/spark/mllib/feature/ChiSqSelector.html) method takes
 an input of `RDD[LabeledPoint]` with categorical features, learns the summary statistics, and then
 returns a `ChiSqSelectorModel` which can transform an input dataset into the reduced feature space.
 The `ChiSqSelectorModel` can be applied either to a `Vector` to produce a reduced `Vector`, or to
@@ -272,7 +272,7 @@ The following example shows the basic use of ChiSqSelector. The data set used ha
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [`ChiSqSelector` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.ChiSqSelector)
+Refer to the [`ChiSqSelector` Scala docs](api/scala/org/apache/spark/mllib/feature/ChiSqSelector.html)
 for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala %}
@@ -312,11 +312,11 @@ v_N
   \end{pmatrix}
 \]`
 
-[`ElementwiseProduct`](api/scala/index.html#org.apache.spark.mllib.feature.ElementwiseProduct) has the following parameter in the constructor:
+[`ElementwiseProduct`](api/scala/org/apache/spark/mllib/feature/ElementwiseProduct.html) has the following parameter in the constructor:
 
 * `scalingVec`: the transforming vector.
 
-`ElementwiseProduct` implements [`VectorTransformer`](api/scala/index.html#org.apache.spark.mllib.feature.VectorTransformer) which can apply the weighting on a `Vector` to produce a transformed `Vector` or on an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
+`ElementwiseProduct` implements [`VectorTransformer`](api/scala/org/apache/spark/mllib/feature/VectorTransformer.html) which can apply the weighting on a `Vector` to produce a transformed `Vector` or on an `RDD[Vector]` to produce a transformed `RDD[Vector]`.
 
 ### Example
 
@@ -325,7 +325,7 @@ This example below demonstrates how to transform vectors using a transforming ve
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-Refer to the [`ElementwiseProduct` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.ElementwiseProduct) for details on the API.
+Refer to the [`ElementwiseProduct` Scala docs](api/scala/org/apache/spark/mllib/feature/ElementwiseProduct.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala %}
 </div>
diff --git a/docs/mllib-frequent-pattern-mining.md b/docs/mllib-frequent-pattern-mining.md
index 8bc93ac2e8adf..709acde062d7e 100644
--- a/docs/mllib-frequent-pattern-mining.md
+++ b/docs/mllib-frequent-pattern-mining.md
@@ -54,18 +54,18 @@ We refer users to the papers for more details.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`FPGrowth`](api/scala/index.html#org.apache.spark.mllib.fpm.FPGrowth) implements the
+[`FPGrowth`](api/scala/org/apache/spark/mllib/fpm/FPGrowth.html) implements the
 FP-growth algorithm.
 It takes an `RDD` of transactions, where each transaction is an `Array` of items of a generic type.
 Calling `FPGrowth.run` with transactions returns an
-[`FPGrowthModel`](api/scala/index.html#org.apache.spark.mllib.fpm.FPGrowthModel)
+[`FPGrowthModel`](api/scala/org/apache/spark/mllib/fpm/FPGrowthModel.html)
 that stores the frequent itemsets with their frequencies.  The following
 example illustrates how to mine frequent itemsets and association rules
 (see [Association
 Rules](mllib-frequent-pattern-mining.html#association-rules) for
 details) from `transactions`.
 
-Refer to the [`FPGrowth` Scala docs](api/scala/index.html#org.apache.spark.mllib.fpm.FPGrowth) for details on the API.
+Refer to the [`FPGrowth` Scala docs](api/scala/org/apache/spark/mllib/fpm/FPGrowth.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala %}
 
@@ -111,7 +111,7 @@ Refer to the [`FPGrowth` Python docs](api/python/pyspark.mllib.html#pyspark.mlli
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[AssociationRules](api/scala/index.html#org.apache.spark.mllib.fpm.AssociationRules)
+[AssociationRules](api/scala/org/apache/spark/mllib/fpm/AssociationRules.html)
 implements a parallel rule generation algorithm for constructing rules
 that have a single item as the consequent.
 
@@ -168,13 +168,13 @@ The following example illustrates PrefixSpan running on the sequences
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`PrefixSpan`](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpan) implements the
+[`PrefixSpan`](api/scala/org/apache/spark/mllib/fpm/PrefixSpan.html) implements the
 PrefixSpan algorithm.
 Calling `PrefixSpan.run` returns a
-[`PrefixSpanModel`](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpanModel)
+[`PrefixSpanModel`](api/scala/org/apache/spark/mllib/fpm/PrefixSpanModel.html)
 that stores the frequent sequences with their frequencies.
 
-Refer to the [`PrefixSpan` Scala docs](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpan) and [`PrefixSpanModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.fpm.PrefixSpanModel) for details on the API.
+Refer to the [`PrefixSpan` Scala docs](api/scala/org/apache/spark/mllib/fpm/PrefixSpan.html) and [`PrefixSpanModel` Scala docs](api/scala/org/apache/spark/mllib/fpm/PrefixSpanModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala %}
 
diff --git a/docs/mllib-isotonic-regression.md b/docs/mllib-isotonic-regression.md
index d9cc775547bb1..94ffadaf65c55 100644
--- a/docs/mllib-isotonic-regression.md
+++ b/docs/mllib-isotonic-regression.md
@@ -74,7 +74,7 @@ i.e. 4710.28,500.00. The data are split to training and testing set.
 Model is created using the training set and a mean squared error is calculated from the predicted
 labels and real labels in the test set.
 
-Refer to the [`IsotonicRegression` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.IsotonicRegression) and [`IsotonicRegressionModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.IsotonicRegressionModel) for details on the API.
+Refer to the [`IsotonicRegression` Scala docs](api/scala/org/apache/spark/mllib/regression/IsotonicRegression.html) and [`IsotonicRegressionModel` Scala docs](api/scala/org/apache/spark/mllib/regression/IsotonicRegressionModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala %}
 </div>
diff --git a/docs/mllib-linear-methods.md b/docs/mllib-linear-methods.md
index 801876dbffa79..e7726271ccb72 100644
--- a/docs/mllib-linear-methods.md
+++ b/docs/mllib-linear-methods.md
@@ -184,7 +184,7 @@ training algorithm on this training data using a static method in the algorithm
 object, and make predictions with the resulting model to compute the training
 error.
 
-Refer to the [`SVMWithSGD` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.SVMWithSGD) and [`SVMModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.SVMModel) for details on the API.
+Refer to the [`SVMWithSGD` Scala docs](api/scala/org/apache/spark/mllib/classification/SVMWithSGD.html) and [`SVMModel` Scala docs](api/scala/org/apache/spark/mllib/classification/SVMModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SVMWithSGDExample.scala %}
 
@@ -305,11 +305,11 @@ We recommend L-BFGS over mini-batch gradient descent for faster convergence.
 <div data-lang="scala" markdown="1">
 The following code illustrates how to load a sample multiclass dataset, split it into train and
 test, and use
-[LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS)
+[LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html)
 to fit a logistic regression model.
 Then the model is evaluated against the test dataset and saved to disk.
 
-Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS) and [`LogisticRegressionModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionModel) for details on the API.
+Refer to the [`LogisticRegressionWithLBFGS` Scala docs](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html) and [`LogisticRegressionModel` Scala docs](api/scala/org/apache/spark/mllib/classification/LogisticRegressionModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/LogisticRegressionWithLBFGSExample.scala %}
 
@@ -438,8 +438,8 @@ regularization parameter (`regParam`) along with various parameters associated w
 gradient descent (`stepSize`, `numIterations`, `miniBatchFraction`).  For each of them, we support
 all three possible regularizations (none, L1 or L2).
 
-For Logistic Regression, [L-BFGS](api/scala/index.html#org.apache.spark.mllib.optimization.LBFGS)
-version is implemented under [LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS), and this
+For Logistic Regression, [L-BFGS](api/scala/org/apache/spark/mllib/optimization/LBFGS.html)
+version is implemented under [LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html), and this
 version supports both binary and multinomial Logistic Regression while SGD version only supports
 binary Logistic Regression. However, L-BFGS version doesn't support L1 regularization but SGD one
 supports L1 regularization. When L1 regularization is not required, L-BFGS version is strongly
@@ -448,10 +448,10 @@ inverse Hessian matrix using quasi-Newton method.
 
 Algorithms are all implemented in Scala:
 
-* [SVMWithSGD](api/scala/index.html#org.apache.spark.mllib.classification.SVMWithSGD)
-* [LogisticRegressionWithLBFGS](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithLBFGS)
-* [LogisticRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithSGD)
-* [LinearRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.regression.LinearRegressionWithSGD)
-* [RidgeRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.regression.RidgeRegressionWithSGD)
-* [LassoWithSGD](api/scala/index.html#org.apache.spark.mllib.regression.LassoWithSGD)
+* [SVMWithSGD](api/scala/org/apache/spark/mllib/classification/SVMWithSGD.html)
+* [LogisticRegressionWithLBFGS](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithLBFGS.html)
+* [LogisticRegressionWithSGD](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithSGD.html)
+* [LinearRegressionWithSGD](api/scala/org/apache/spark/mllib/regression/LinearRegressionWithSGD.html)
+* [RidgeRegressionWithSGD](api/scala/org/apache/spark/mllib/regression/RidgeRegressionWithSGD.html)
+* [LassoWithSGD](api/scala/org/apache/spark/mllib/regression/LassoWithSGD.html)
 
diff --git a/docs/mllib-naive-bayes.md b/docs/mllib-naive-bayes.md
index 09b15876a3914..a3602667b5835 100644
--- a/docs/mllib-naive-bayes.md
+++ b/docs/mllib-naive-bayes.md
@@ -46,14 +46,14 @@ sparsity. Since the training data is only used once, it is not necessary to cach
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[NaiveBayes](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayes$) implements
+[NaiveBayes](api/scala/org/apache/spark/mllib/classification/NaiveBayes$.html) implements
 multinomial naive Bayes. It takes an RDD of
-[LabeledPoint](api/scala/index.html#org.apache.spark.mllib.regression.LabeledPoint) and an optional
+[LabeledPoint](api/scala/org/apache/spark/mllib/regression/LabeledPoint.html) and an optional
 smoothing parameter `lambda` as input, an optional model type parameter (default is "multinomial"), and outputs a
-[NaiveBayesModel](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayesModel), which
+[NaiveBayesModel](api/scala/org/apache/spark/mllib/classification/NaiveBayesModel.html), which
 can be used for evaluation and prediction.
 
-Refer to the [`NaiveBayes` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayes) and [`NaiveBayesModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.classification.NaiveBayesModel) for details on the API.
+Refer to the [`NaiveBayes` Scala docs](api/scala/org/apache/spark/mllib/classification/NaiveBayes$.html) and [`NaiveBayesModel` Scala docs](api/scala/org/apache/spark/mllib/classification/NaiveBayesModel.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala %}
 </div>
diff --git a/docs/mllib-optimization.md b/docs/mllib-optimization.md
index f2e128ec215a1..f0b0c817ed3f6 100644
--- a/docs/mllib-optimization.md
+++ b/docs/mllib-optimization.md
@@ -111,12 +111,12 @@ As an alternative to just use the subgradient `$R'(\wv)$` of the regularizer in
 direction, an improved update for some cases can be obtained by using the proximal operator
 instead.
 For the L1-regularizer, the proximal operator is given by soft thresholding, as implemented in
-[L1Updater](api/scala/index.html#org.apache.spark.mllib.optimization.L1Updater).
+[L1Updater](api/scala/org/apache/spark/mllib/optimization/L1Updater.html).
 
 
 ### Update schemes for distributed SGD
 The SGD implementation in
-[GradientDescent](api/scala/index.html#org.apache.spark.mllib.optimization.GradientDescent) uses
+[GradientDescent](api/scala/org/apache/spark/mllib/optimization/GradientDescent.html) uses
 a simple (distributed) sampling of the data examples.
 We recall that the loss part of the optimization problem `$\eqref{eq:regPrimal}$` is
 `$\frac1n \sum_{i=1}^n L(\wv;\x_i,y_i)$`, and therefore `$\frac1n \sum_{i=1}^n L'_{\wv,i}$` would
@@ -169,7 +169,7 @@ are developed, see the
 section for example.
 
 The SGD class
-[GradientDescent](api/scala/index.html#org.apache.spark.mllib.optimization.GradientDescent)
+[GradientDescent](api/scala/org/apache/spark/mllib/optimization/GradientDescent.html)
 sets the following parameters:
 
 * `Gradient` is a class that computes the stochastic gradient of the function
@@ -195,15 +195,15 @@ each iteration, to compute the gradient direction.
 L-BFGS is currently only a low-level optimization primitive in `MLlib`. If you want to use L-BFGS in various 
 ML algorithms such as Linear Regression, and Logistic Regression, you have to pass the gradient of objective
 function, and updater into optimizer yourself instead of using the training APIs like 
-[LogisticRegressionWithSGD](api/scala/index.html#org.apache.spark.mllib.classification.LogisticRegressionWithSGD).
+[LogisticRegressionWithSGD](api/scala/org/apache/spark/mllib/classification/LogisticRegressionWithSGD.html).
 See the example below. It will be addressed in the next release. 
 
 The L1 regularization by using 
-[L1Updater](api/scala/index.html#org.apache.spark.mllib.optimization.L1Updater) will not work since the 
+[L1Updater](api/scala/org/apache/spark/mllib/optimization/L1Updater.html) will not work since the 
 soft-thresholding logic in L1Updater is designed for gradient descent. See the developer's note.
 
 The L-BFGS method
-[LBFGS.runLBFGS](api/scala/index.html#org.apache.spark.mllib.optimization.LBFGS)
+[LBFGS.runLBFGS](api/scala/org/apache/spark/mllib/optimization/LBFGS.html)
 has the following parameters:
 
 * `Gradient` is a class that computes the gradient of the objective function
@@ -233,7 +233,7 @@ L-BFGS optimizer.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-Refer to the [`LBFGS` Scala docs](api/scala/index.html#org.apache.spark.mllib.optimization.LBFGS) and [`SquaredL2Updater` Scala docs](api/scala/index.html#org.apache.spark.mllib.optimization.SquaredL2Updater) for details on the API.
+Refer to the [`LBFGS` Scala docs](api/scala/org/apache/spark/mllib/optimization/LBFGS.html) and [`SquaredL2Updater` Scala docs](api/scala/org/apache/spark/mllib/optimization/SquaredL2Updater.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/LBFGSExample.scala %}
 </div>
diff --git a/docs/mllib-pmml-model-export.md b/docs/mllib-pmml-model-export.md
index fbc14cefd14ec..d7eb51d69a1da 100644
--- a/docs/mllib-pmml-model-export.md
+++ b/docs/mllib-pmml-model-export.md
@@ -62,7 +62,7 @@ To export a supported `model` (see table above) to PMML, simply call `model.toPM
 
 As well as exporting the PMML model to a String (`model.toPMML` as in the example above), you can export the PMML model to other formats.
 
-Refer to the [`KMeans` Scala docs](api/scala/index.html#org.apache.spark.mllib.clustering.KMeans) and [`Vectors` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.Vectors$) for details on the API.
+Refer to the [`KMeans` Scala docs](api/scala/org/apache/spark/mllib/clustering/KMeans.html) and [`Vectors` Scala docs](api/scala/org/apache/spark/mllib/linalg/Vectors$.html) for details on the API.
 
 Here a complete example of building a KMeansModel and print it out in PMML format:
 {% include_example scala/org/apache/spark/examples/mllib/PMMLModelExportExample.scala %}
diff --git a/docs/mllib-statistics.md b/docs/mllib-statistics.md
index 4698d3e6347e7..7de214bb6b6a6 100644
--- a/docs/mllib-statistics.md
+++ b/docs/mllib-statistics.md
@@ -48,12 +48,12 @@ available in `Statistics`.
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-[`colStats()`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) returns an instance of
-[`MultivariateStatisticalSummary`](api/scala/index.html#org.apache.spark.mllib.stat.MultivariateStatisticalSummary),
+[`colStats()`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) returns an instance of
+[`MultivariateStatisticalSummary`](api/scala/org/apache/spark/mllib/stat/MultivariateStatisticalSummary.html),
 which contains the column-wise max, min, mean, variance, and number of nonzeros, as well as the
 total count.
 
-Refer to the [`MultivariateStatisticalSummary` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.MultivariateStatisticalSummary) for details on the API.
+Refer to the [`MultivariateStatisticalSummary` Scala docs](api/scala/org/apache/spark/mllib/stat/MultivariateStatisticalSummary.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/SummaryStatisticsExample.scala %}
 </div>
@@ -91,11 +91,11 @@ correlation methods are currently Pearson's and Spearman's correlation.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Statistics`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) provides methods to
 calculate correlations between series. Depending on the type of input, two `RDD[Double]`s or
 an `RDD[Vector]`, the output will be a `Double` or the correlation `Matrix` respectively.
 
-Refer to the [`Statistics` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) for details on the API.
+Refer to the [`Statistics` Scala docs](api/scala/org/apache/spark/mllib/stat/Statistics$.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/CorrelationsExample.scala %}
 </div>
@@ -137,7 +137,7 @@ python.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`sampleByKeyExact()`](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) allows users to
+[`sampleByKeyExact()`](api/scala/org/apache/spark/rdd/PairRDDFunctions.html) allows users to
 sample exactly $\lceil f_k \cdot n_k \rceil \, \forall k \in K$ items, where $f_k$ is the desired
 fraction for key $k$, $n_k$ is the number of key-value pairs for key $k$, and $K$ is the set of
 keys. Sampling without replacement requires one additional pass over the RDD to guarantee sample
@@ -181,7 +181,7 @@ independence tests.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Statistics`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) provides methods to
 run Pearson's chi-squared tests. The following example demonstrates how to run and interpret
 hypothesis tests.
 
@@ -221,11 +221,11 @@ message.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`Statistics`](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) provides methods to
+[`Statistics`](api/scala/org/apache/spark/mllib/stat/Statistics$.html) provides methods to
 run a 1-sample, 2-sided Kolmogorov-Smirnov test. The following example demonstrates how to run
 and interpret the hypothesis tests.
 
-Refer to the [`Statistics` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.Statistics$) for details on the API.
+Refer to the [`Statistics` Scala docs](api/scala/org/apache/spark/mllib/stat/Statistics$.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/HypothesisTestingKolmogorovSmirnovTestExample.scala %}
 </div>
@@ -269,7 +269,7 @@ all prior batches.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`StreamingTest`](api/scala/index.html#org.apache.spark.mllib.stat.test.StreamingTest)
+[`StreamingTest`](api/scala/org/apache/spark/mllib/stat/test/StreamingTest.html)
 provides streaming hypothesis testing.
 
 {% include_example scala/org/apache/spark/examples/mllib/StreamingTestExample.scala %}
@@ -292,12 +292,12 @@ uniform, standard normal, or Poisson.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
-[`RandomRDDs`](api/scala/index.html#org.apache.spark.mllib.random.RandomRDDs$) provides factory
+[`RandomRDDs`](api/scala/org/apache/spark/mllib/random/RandomRDDs$.html) provides factory
 methods to generate random double RDDs or vector RDDs.
 The following example generates a random double RDD, whose values follows the standard normal
 distribution `N(0, 1)`, and then map it to `N(1, 4)`.
 
-Refer to the [`RandomRDDs` Scala docs](api/scala/index.html#org.apache.spark.mllib.random.RandomRDDs$) for details on the API.
+Refer to the [`RandomRDDs` Scala docs](api/scala/org/apache/spark/mllib/random/RandomRDDs$.html) for details on the API.
 
 {% highlight scala %}
 import org.apache.spark.SparkContext
@@ -370,11 +370,11 @@ mean of PDFs of normal distributions centered around each of the samples.
 <div class="codetabs">
 
 <div data-lang="scala" markdown="1">
-[`KernelDensity`](api/scala/index.html#org.apache.spark.mllib.stat.KernelDensity) provides methods
+[`KernelDensity`](api/scala/org/apache/spark/mllib/stat/KernelDensity.html) provides methods
 to compute kernel density estimates from an RDD of samples. The following example demonstrates how
 to do so.
 
-Refer to the [`KernelDensity` Scala docs](api/scala/index.html#org.apache.spark.mllib.stat.KernelDensity) for details on the API.
+Refer to the [`KernelDensity` Scala docs](api/scala/org/apache/spark/mllib/stat/KernelDensity.html) for details on the API.
 
 {% include_example scala/org/apache/spark/examples/mllib/KernelDensityEstimationExample.scala %}
 </div>
diff --git a/docs/quick-start.md b/docs/quick-start.md
index 93abb25f20f47..86ba2c44b0e02 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -57,7 +57,7 @@ scala> val textFile = spark.read.textFile("README.md")
 textFile: org.apache.spark.sql.Dataset[String] = [value: string]
 {% endhighlight %}
 
-You can get values from Dataset directly, by calling some actions, or transform the Dataset to get a new one. For more details, please read the _[API doc](api/scala/index.html#org.apache.spark.sql.Dataset)_.
+You can get values from Dataset directly, by calling some actions, or transform the Dataset to get a new one. For more details, please read the _[API doc](api/scala/org/apache/spark/sql/Dataset.html)_.
 
 {% highlight scala %}
 scala> textFile.count() // Number of items in this Dataset
diff --git a/docs/rdd-programming-guide.md b/docs/rdd-programming-guide.md
index 5e55c93c4148e..ba99007aaf639 100644
--- a/docs/rdd-programming-guide.md
+++ b/docs/rdd-programming-guide.md
@@ -149,8 +149,8 @@ $ PYSPARK_PYTHON=/opt/pypy-2.5/bin/pypy bin/spark-submit examples/src/main/pytho
 
 <div data-lang="scala"  markdown="1">
 
-The first thing a Spark program must do is to create a [SparkContext](api/scala/index.html#org.apache.spark.SparkContext) object, which tells Spark
-how to access a cluster. To create a `SparkContext` you first need to build a [SparkConf](api/scala/index.html#org.apache.spark.SparkConf) object
+The first thing a Spark program must do is to create a [SparkContext](api/scala/org/apache/spark/SparkContext.html) object, which tells Spark
+how to access a cluster. To create a `SparkContext` you first need to build a [SparkConf](api/scala/org/apache/spark/SparkConf.html) object
 that contains information about your application.
 
 Only one SparkContext should be active per JVM. You must `stop()` the active SparkContext before creating a new one.
@@ -500,7 +500,7 @@ then this approach should work well for such cases.
 
 If you have custom serialized binary data (such as loading data from Cassandra / HBase), then you will first need to
 transform that data on the Scala/Java side to something which can be handled by Pyrolite's pickler.
-A [Converter](api/scala/index.html#org.apache.spark.api.python.Converter) trait is provided
+A [Converter](api/scala/org/apache/spark/api/python/Converter.html) trait is provided
 for this. Simply extend this trait and implement your transformation code in the ```convert```
 method. Remember to ensure that this class, along with any dependencies required to access your ```InputFormat```, are packaged into your Spark job jar and included on the PySpark
 classpath.
@@ -856,7 +856,7 @@ by a key.
 In Scala, these operations are automatically available on RDDs containing
 [Tuple2](http://www.scala-lang.org/api/{{site.SCALA_VERSION}}/index.html#scala.Tuple2) objects
 (the built-in tuples in the language, created by simply writing `(a, b)`). The key-value pair operations are available in the
-[PairRDDFunctions](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) class,
+[PairRDDFunctions](api/scala/org/apache/spark/rdd/PairRDDFunctions.html) class,
 which automatically wraps around an RDD of tuples.
 
 For example, the following code uses the `reduceByKey` operation on key-value pairs to count how
@@ -946,12 +946,12 @@ We could also use `counts.sortByKey()`, for example, to sort the pairs alphabeti
 
 The following table lists some of the common transformations supported by Spark. Refer to the
 RDD API doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.RDD),
+([Scala](api/scala/org/apache/spark/rdd/RDD.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaRDD.html),
  [Python](api/python/pyspark.html#pyspark.RDD),
  [R](api/R/index.html))
 and pair RDD functions doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions),
+([Scala](api/scala/org/apache/spark/rdd/PairRDDFunctions.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html))
 for details.
 
@@ -1060,13 +1060,13 @@ for details.
 
 The following table lists some of the common actions supported by Spark. Refer to the
 RDD API doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.RDD),
+([Scala](api/scala/org/apache/spark/rdd/RDD.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaRDD.html),
  [Python](api/python/pyspark.html#pyspark.RDD),
  [R](api/R/index.html))
 
 and pair RDD functions doc
-([Scala](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions),
+([Scala](api/scala/org/apache/spark/rdd/PairRDDFunctions.html),
  [Java](api/java/index.html?org/apache/spark/api/java/JavaPairRDD.html))
 for details.
 
@@ -1208,7 +1208,7 @@ In addition, each persisted RDD can be stored using a different *storage level*,
 to persist the dataset on disk, persist it in memory but as serialized Java objects (to save space),
 replicate it across nodes.
 These levels are set by passing a
-`StorageLevel` object ([Scala](api/scala/index.html#org.apache.spark.storage.StorageLevel),
+`StorageLevel` object ([Scala](api/scala/org/apache/spark/storage/StorageLevel.html),
 [Java](api/java/index.html?org/apache/spark/storage/StorageLevel.html),
 [Python](api/python/pyspark.html#pyspark.StorageLevel))
 to `persist()`. The `cache()` method is a shorthand for using the default storage level,
@@ -1404,11 +1404,11 @@ res2: Long = 10
 {% endhighlight %}
 
 While this code used the built-in support for accumulators of type Long, programmers can also
-create their own types by subclassing [AccumulatorV2](api/scala/index.html#org.apache.spark.util.AccumulatorV2).
+create their own types by subclassing [AccumulatorV2](api/scala/org/apache/spark/util/AccumulatorV2.html).
 The AccumulatorV2 abstract class has several methods which one has to override: `reset` for resetting
 the accumulator to zero, `add` for adding another value into the accumulator,
 `merge` for merging another same-type accumulator into this one. Other methods that must be overridden
-are contained in the [API documentation](api/scala/index.html#org.apache.spark.util.AccumulatorV2). For example, supposing we had a `MyVector` class
+are contained in the [API documentation](api/scala/org/apache/spark/util/AccumulatorV2.html). For example, supposing we had a `MyVector` class
 representing mathematical vectors, we could write:
 
 {% highlight scala %}
@@ -1457,11 +1457,11 @@ accum.value();
 {% endhighlight %}
 
 While this code used the built-in support for accumulators of type Long, programmers can also
-create their own types by subclassing [AccumulatorV2](api/scala/index.html#org.apache.spark.util.AccumulatorV2).
+create their own types by subclassing [AccumulatorV2](api/scala/org/apache/spark/util/AccumulatorV2.html).
 The AccumulatorV2 abstract class has several methods which one has to override: `reset` for resetting
 the accumulator to zero, `add` for adding another value into the accumulator,
 `merge` for merging another same-type accumulator into this one. Other methods that must be overridden
-are contained in the [API documentation](api/scala/index.html#org.apache.spark.util.AccumulatorV2). For example, supposing we had a `MyVector` class
+are contained in the [API documentation](api/scala/org/apache/spark/util/AccumulatorV2.html). For example, supposing we had a `MyVector` class
 representing mathematical vectors, we could write:
 
 {% highlight java %}
@@ -1620,4 +1620,4 @@ For help on deploying, the [cluster mode overview](cluster-overview.html) descri
 in distributed operation and supported cluster managers.
 
 Finally, full API documentation is available in
-[Scala](api/scala/#org.apache.spark.package), [Java](api/java/), [Python](api/python/) and [R](api/R/).
+[Scala](api/scala/org/apache/spark/), [Java](api/java/), [Python](api/python/) and [R](api/R/).
diff --git a/docs/sql-data-sources-generic-options.md b/docs/sql-data-sources-generic-options.md
index 0cfe2ed1aa891..6bcf48235bced 100644
--- a/docs/sql-data-sources-generic-options.md
+++ b/docs/sql-data-sources-generic-options.md
@@ -118,4 +118,4 @@ To load all files recursively, you can use:
 <div data-lang="r"  markdown="1">
 {% include_example recursive_file_lookup r/RSparkSQLExample.R %}
 </div>
-</div>
\ No newline at end of file
+</div>
diff --git a/docs/sql-data-sources-jdbc.md b/docs/sql-data-sources-jdbc.md
index b0d37b11c7117..3cdff42057ce1 100644
--- a/docs/sql-data-sources-jdbc.md
+++ b/docs/sql-data-sources-jdbc.md
@@ -23,7 +23,7 @@ license: |
 {:toc}
 
 Spark SQL also includes a data source that can read data from other databases using JDBC. This
-functionality should be preferred over using [JdbcRDD](api/scala/index.html#org.apache.spark.rdd.JdbcRDD).
+functionality should be preferred over using [JdbcRDD](api/scala/org/apache/spark/rdd/JdbcRDD.html).
 This is because the results are returned
 as a DataFrame and they can easily be processed in Spark SQL or joined with other data sources.
 The JDBC data source is also easier to use from Java or Python as it does not require the user to
diff --git a/docs/sql-data-sources-json.md b/docs/sql-data-sources-json.md
index 4ce4897189846..588f6cdaa0ad5 100644
--- a/docs/sql-data-sources-json.md
+++ b/docs/sql-data-sources-json.md
@@ -93,4 +93,4 @@ SELECT * FROM jsonTable
 
 </div>
 
-</div>
\ No newline at end of file
+</div>
diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index f637697a29ad7..9df0f768268d0 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -27,7 +27,7 @@ license: |
 <div class="codetabs">
 <div data-lang="scala"  markdown="1">
 
-The entry point into all functionality in Spark is the [`SparkSession`](api/scala/index.html#org.apache.spark.sql.SparkSession) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
+The entry point into all functionality in Spark is the [`SparkSession`](api/scala/org/apache/spark/sql/SparkSession.html) class. To create a basic `SparkSession`, just use `SparkSession.builder()`:
 
 {% include_example init_session scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
 </div>
@@ -104,7 +104,7 @@ As an example, the following creates a DataFrame based on the content of a JSON
 
 ## Untyped Dataset Operations (aka DataFrame Operations)
 
-DataFrames provide a domain-specific language for structured data manipulation in [Scala](api/scala/index.html#org.apache.spark.sql.Dataset), [Java](api/java/index.html?org/apache/spark/sql/Dataset.html), [Python](api/python/pyspark.sql.html#pyspark.sql.DataFrame) and [R](api/R/SparkDataFrame.html).
+DataFrames provide a domain-specific language for structured data manipulation in [Scala](api/scala/org/apache/spark/sql/Dataset.html), [Java](api/java/index.html?org/apache/spark/sql/Dataset.html), [Python](api/python/pyspark.sql.html#pyspark.sql.DataFrame) and [R](api/R/SparkDataFrame.html).
 
 As mentioned above, in Spark 2.0, DataFrames are just Dataset of `Row`s in Scala and Java API. These operations are also referred as "untyped transformations" in contrast to "typed transformations" come with strongly typed Scala/Java Datasets.
 
@@ -114,9 +114,9 @@ Here we include some basic examples of structured data processing using Datasets
 <div data-lang="scala"  markdown="1">
 {% include_example untyped_ops scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
 
-For a complete list of the types of operations that can be performed on a Dataset, refer to the [API Documentation](api/scala/index.html#org.apache.spark.sql.Dataset).
+For a complete list of the types of operations that can be performed on a Dataset, refer to the [API Documentation](api/scala/org/apache/spark/sql/Dataset.html).
 
-In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/scala/index.html#org.apache.spark.sql.functions$).
+In addition to simple column references and expressions, Datasets also have a rich library of functions including string manipulation, date arithmetic, common math operations and more. The complete list is available in the [DataFrame Function Reference](api/scala/org/apache/spark/sql/functions$.html).
 </div>
 
 <div data-lang="java" markdown="1">
@@ -222,7 +222,7 @@ SELECT * FROM global_temp.temp_view
 ## Creating Datasets
 
 Datasets are similar to RDDs, however, instead of using Java serialization or Kryo they use
-a specialized [Encoder](api/scala/index.html#org.apache.spark.sql.Encoder) to serialize the objects
+a specialized [Encoder](api/scala/org/apache/spark/sql/Encoder.html) to serialize the objects
 for processing or transmitting over the network. While both encoders and standard serialization are
 responsible for turning an object into bytes, encoders are code generated dynamically and use a format
 that allows Spark to perform many operations like filtering, sorting and hashing without deserializing
@@ -351,16 +351,16 @@ For example:
 
 ## Aggregations
 
-The [built-in DataFrames functions](api/scala/index.html#org.apache.spark.sql.functions$) provide common
+The [built-in DataFrames functions](api/scala/org/apache/spark/sql/functions$.html) provide common
 aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, etc.
 While those functions are designed for DataFrames, Spark SQL also has type-safe versions for some of them in
-[Scala](api/scala/index.html#org.apache.spark.sql.expressions.scalalang.typed$) and
+[Scala](api/scala/org/apache/spark/sql/expressions/scalalang/typed$.html) and
 [Java](api/java/org/apache/spark/sql/expressions/javalang/typed.html) to work with strongly typed Datasets.
 Moreover, users are not limited to the predefined aggregate functions and can create their own.
 
 ### Type-Safe User-Defined Aggregate Functions
 
-User-defined aggregations for strongly typed Datasets revolve around the [Aggregator](api/scala/index.html#org.apache.spark.sql.expressions.Aggregator) abstract class.
+User-defined aggregations for strongly typed Datasets revolve around the [Aggregator](api/scala/org/apache/spark/sql/expressions/Aggregator.html) abstract class.
 For example, a type-safe user-defined average can look like:
 
 <div class="codetabs">
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 46b741687363f..11afae14d0205 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -737,11 +737,11 @@ and writing data out (`DataFrame.write`),
 and deprecated the old APIs (e.g., `SQLContext.parquetFile`, `SQLContext.jsonFile`).
 
 See the API docs for `SQLContext.read` (
-  <a href="api/scala/index.html#org.apache.spark.sql.SQLContext@read:DataFrameReader">Scala</a>,
+  <a href="api/scala/org/apache/spark/sql/SQLContext.html#read:DataFrameReader">Scala</a>,
   <a href="api/java/org/apache/spark/sql/SQLContext.html#read()">Java</a>,
   <a href="api/python/pyspark.sql.html#pyspark.sql.SQLContext.read">Python</a>
 ) and `DataFrame.write` (
-  <a href="api/scala/index.html#org.apache.spark.sql.DataFrame@write:DataFrameWriter">Scala</a>,
+  <a href="api/scala/org/apache/spark/sql/DataFrame.html#write:DataFrameWriter">Scala</a>,
   <a href="api/java/org/apache/spark/sql/Dataset.html#write()">Java</a>,
   <a href="api/python/pyspark.sql.html#pyspark.sql.DataFrame.write">Python</a>
 ) more information.
diff --git a/docs/sql-programming-guide.md b/docs/sql-programming-guide.md
index 0a4d07ea37b69..06bf553e4f704 100644
--- a/docs/sql-programming-guide.md
+++ b/docs/sql-programming-guide.md
@@ -61,7 +61,7 @@ In Scala and Java, a DataFrame is represented by a Dataset of `Row`s.
 In [the Scala API][scala-datasets], `DataFrame` is simply a type alias of `Dataset[Row]`.
 While, in [Java API][java-datasets], users need to use `Dataset<Row>` to represent a `DataFrame`.
 
-[scala-datasets]: api/scala/index.html#org.apache.spark.sql.Dataset
+[scala-datasets]: api/scala/org/apache/spark/sql/Dataset.html
 [java-datasets]: api/java/index.html?org/apache/spark/sql/Dataset.html
 
 Throughout this document, we will often refer to Scala/Java Datasets of `Row`s as DataFrames.
diff --git a/docs/sql-ref-syntax-aux-analyze-table.md b/docs/sql-ref-syntax-aux-analyze-table.md
index e448ca38c849e..40513e836b026 100644
--- a/docs/sql-ref-syntax-aux-analyze-table.md
+++ b/docs/sql-ref-syntax-aux-analyze-table.md
@@ -106,4 +106,4 @@ ANALYZE TABLE table_identifier [ partition_spec ]
      max_col_len	13
      histogram	NULL
 
-{% endhighlight %}
\ No newline at end of file
+{% endhighlight %}
diff --git a/docs/sql-ref-syntax-aux-cache-refresh.md b/docs/sql-ref-syntax-aux-cache-refresh.md
index fea3100d96f01..4c56893aeca98 100644
--- a/docs/sql-ref-syntax-aux-cache-refresh.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh.md
@@ -51,4 +51,4 @@ REFRESH "hdfs://path/to/table";
 - [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 - [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
 - [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-- [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
\ No newline at end of file
+- [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-refresh-table.md
index 4b3fcdd65ba75..96d698a1d68b0 100644
--- a/docs/sql-ref-syntax-aux-refresh-table.md
+++ b/docs/sql-ref-syntax-aux-refresh-table.md
@@ -55,4 +55,4 @@ REFRESH TABLE tempDB.view1;
 ### Related Statements
 - [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 - [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
\ No newline at end of file
+- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt.md b/docs/sql-ref-syntax-aux-resource-mgmt.md
index 5eddb979388aa..0885f56bdb7cf 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt.md
@@ -22,4 +22,4 @@ license: |
 * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
 * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
 * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
-* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
\ No newline at end of file
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index 41af8e7d607d0..2a078abf911e7 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -104,4 +104,4 @@ SHOW TABLES  LIKE 'sam*|suj';
 - [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
 - [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
 - [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
\ No newline at end of file
+- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show.md b/docs/sql-ref-syntax-aux-show.md
index ad5c3954c23fa..f6d700acf9eef 100644
--- a/docs/sql-ref-syntax-aux-show.md
+++ b/docs/sql-ref-syntax-aux-show.md
@@ -25,4 +25,4 @@ license: |
 * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
 * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
 * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
-* [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
\ No newline at end of file
+* [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-database.md b/docs/sql-ref-syntax-ddl-drop-database.md
index 68244648c9a5b..0bdb98f2b129c 100644
--- a/docs/sql-ref-syntax-ddl-drop-database.md
+++ b/docs/sql-ref-syntax-ddl-drop-database.md
@@ -77,4 +77,4 @@ DROP DATABASE IF EXISTS inventory_db CASCADE;
 ### Related statements
 - [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
 - [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
\ No newline at end of file
+- [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-function.md b/docs/sql-ref-syntax-ddl-drop-function.md
index a059960920bef..16d08d1ae8e99 100644
--- a/docs/sql-ref-syntax-ddl-drop-function.md
+++ b/docs/sql-ref-syntax-ddl-drop-function.md
@@ -102,4 +102,4 @@ DROP TEMPORARY FUNCTION IF EXISTS test_avg;
 ### Related statements
 - [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
 - [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
\ No newline at end of file
+- [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
index 6d56cdeb051c6..3b0475aef1015 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
@@ -84,4 +84,4 @@ INSERT OVERWRITE [ LOCAL ] DIRECTORY directory_path
 ### Related Statements
   * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
   * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
\ No newline at end of file
+  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
index 06000b593bda4..6f19c62bc3d56 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
@@ -82,4 +82,4 @@ INSERT OVERWRITE DIRECTORY
 ### Related Statements
   * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
   * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
\ No newline at end of file
+  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml.md b/docs/sql-ref-syntax-dml.md
index af09e2054562e..b5dd45f8962c9 100644
--- a/docs/sql-ref-syntax-dml.md
+++ b/docs/sql-ref-syntax-dml.md
@@ -22,4 +22,4 @@ license: |
 Data Manipulation Statements are used to add, change, or delete data. Spark SQL supports the following Data Manipulation Statements:
 
 - [INSERT](sql-ref-syntax-dml-insert.html)
-- [LOAD](sql-ref-syntax-dml-load.html)
\ No newline at end of file
+- [LOAD](sql-ref-syntax-dml-load.html)
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index c96c4419212bb..bb60e8bfe25fc 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -96,4 +96,4 @@ SELECT age, name FROM person CLUSTER BY age;
 - [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
 - [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 - [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index e706ccf03917b..5ade9c16e4a0f 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -91,4 +91,4 @@ SELECT age, name FROM person DISTRIBUTE BY age;
 - [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
 - [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 - [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index c0a491b78ee83..1818a69f4a34b 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -184,4 +184,4 @@ SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
 - [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
 - [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 - [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 00bd719004fb8..80b930f335d1e 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -143,4 +143,4 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 - [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 - [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 - [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
\ No newline at end of file
+- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/streaming-custom-receivers.md b/docs/streaming-custom-receivers.md
index a37b961243f47..6ce73b27d11ba 100644
--- a/docs/streaming-custom-receivers.md
+++ b/docs/streaming-custom-receivers.md
@@ -28,7 +28,7 @@ in Scala or Java.
 ## Implementing a Custom Receiver
 
 This starts with implementing a **Receiver**
-([Scala doc](api/scala/index.html#org.apache.spark.streaming.receiver.Receiver),
+([Scala doc](api/scala/org/apache/spark/streaming/receiver/Receiver.html),
 [Java doc](api/java/org/apache/spark/streaming/receiver/Receiver.html)).
 A custom receiver must extend this abstract class by implementing two methods
 
diff --git a/docs/streaming-kafka-integration.md b/docs/streaming-kafka-integration.md
index 6fa363285f385..3282124b0be5d 100644
--- a/docs/streaming-kafka-integration.md
+++ b/docs/streaming-kafka-integration.md
@@ -23,4 +23,4 @@ replicated commit log service.  Please read the [Kafka documentation](https://ka
 thoroughly before starting an integration using Spark.
 
 At the moment, Spark requires Kafka 0.10 and higher. See 
-<a href="streaming-kafka-0-10-integration.html">Kafka 0.10 integration documentation</a> for details.
\ No newline at end of file
+<a href="streaming-kafka-0-10-integration.html">Kafka 0.10 integration documentation</a> for details.
diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index 3389d453c2cbd..c81597160ca09 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -59,7 +59,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                 .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
                 .build()
 
-	See the [API docs](api/scala/index.html#org.apache.spark.streaming.kinesis.KinesisInputDStream)
+	See the [API docs](api/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.html)
 	and the [example]({{site.SPARK_GITHUB_URL}}/tree/master/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala). Refer to the [Running the Example](#running-the-example) subsection for instructions on how to run the example.
 
 	</div>
diff --git a/docs/streaming-programming-guide.md b/docs/streaming-programming-guide.md
index f6b579fbf74d1..ac4aa9255ae68 100644
--- a/docs/streaming-programming-guide.md
+++ b/docs/streaming-programming-guide.md
@@ -57,7 +57,7 @@ Spark Streaming provides a high-level abstraction called *discretized stream* or
 which represents a continuous stream of data. DStreams can be created either from input data
 streams from sources such as Kafka, and Kinesis, or by applying high-level
 operations on other DStreams. Internally, a DStream is represented as a sequence of
-[RDDs](api/scala/index.html#org.apache.spark.rdd.RDD).
+[RDDs](api/scala/org/apache/spark/rdd/RDD.html).
 
 This guide shows you how to start writing Spark Streaming programs with DStreams. You can
 write Spark Streaming programs in Scala, Java or Python (introduced in Spark 1.2),
@@ -80,7 +80,7 @@ do is as follows.
 <div data-lang="scala"  markdown="1" >
 First, we import the names of the Spark Streaming classes and some implicit
 conversions from StreamingContext into our environment in order to add useful methods to
-other classes we need (like DStream). [StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) is the
+other classes we need (like DStream). [StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) is the
 main entry point for all streaming functionality. We create a local StreamingContext with two execution threads,  and a batch interval of 1 second.
 
 {% highlight scala %}
@@ -185,7 +185,7 @@ JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(x.split(" ")).itera
 generating multiple new records from each record in the source DStream. In this case,
 each line will be split into multiple words and the stream of words is represented as the
 `words` DStream. Note that we defined the transformation using a
-[FlatMapFunction](api/scala/index.html#org.apache.spark.api.java.function.FlatMapFunction) object.
+[FlatMapFunction](api/scala/org/apache/spark/api/java/function/FlatMapFunction.html) object.
 As we will discover along the way, there are a number of such convenience classes in the Java API
 that help defines DStream transformations.
 
@@ -201,9 +201,9 @@ wordCounts.print();
 {% endhighlight %}
 
 The `words` DStream is further mapped (one-to-one transformation) to a DStream of `(word,
-1)` pairs, using a [PairFunction](api/scala/index.html#org.apache.spark.api.java.function.PairFunction)
+1)` pairs, using a [PairFunction](api/scala/org/apache/spark/api/java/function/PairFunction.html)
 object. Then, it is reduced to get the frequency of words in each batch of data,
-using a [Function2](api/scala/index.html#org.apache.spark.api.java.function.Function2) object.
+using a [Function2](api/scala/org/apache/spark/api/java/function/Function2.html) object.
 Finally, `wordCounts.print()` will print a few of the counts generated every second.
 
 Note that when these lines are executed, Spark Streaming only sets up the computation it
@@ -435,7 +435,7 @@ To initialize a Spark Streaming program, a **StreamingContext** object has to be
 <div class="codetabs">
 <div data-lang="scala" markdown="1">
 
-A [StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) object can be created from a [SparkConf](api/scala/index.html#org.apache.spark.SparkConf) object.
+A [StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) object can be created from a [SparkConf](api/scala/org/apache/spark/SparkConf.html) object.
 
 {% highlight scala %}
 import org.apache.spark._
@@ -451,7 +451,7 @@ or a special __"local[\*]"__ string to run in local mode. In practice, when runn
 you will not want to hardcode `master` in the program,
 but rather [launch the application with `spark-submit`](submitting-applications.html) and
 receive it there. However, for local testing and unit tests, you can pass "local[\*]" to run Spark Streaming
-in-process (detects the number of cores in the local system). Note that this internally creates a [SparkContext](api/scala/index.html#org.apache.spark.SparkContext) (starting point of all Spark functionality) which can be accessed as `ssc.sparkContext`.
+in-process (detects the number of cores in the local system). Note that this internally creates a [SparkContext](api/scala/org/apache/spark/SparkContext.html) (starting point of all Spark functionality) which can be accessed as `ssc.sparkContext`.
 
 The batch interval must be set based on the latency requirements of your application
 and available cluster resources. See the [Performance Tuning](#setting-the-right-batch-interval)
@@ -584,7 +584,7 @@ Input DStreams are DStreams representing the stream of input data received from
 sources. In the [quick example](#a-quick-example), `lines` was an input DStream as it represented
 the stream of data received from the netcat server. Every input DStream
 (except file stream, discussed later in this section) is associated with a **Receiver**
-([Scala doc](api/scala/index.html#org.apache.spark.streaming.receiver.Receiver),
+([Scala doc](api/scala/org/apache/spark/streaming/receiver/Receiver.html),
 [Java doc](api/java/org/apache/spark/streaming/receiver/Receiver.html)) object which receives the
 data from a source and stores it in Spark's memory for processing.
 
@@ -739,7 +739,7 @@ DStreams can be created with data streams received through custom receivers. See
 For testing a Spark Streaming application with test data, one can also create a DStream based on a queue of RDDs, using `streamingContext.queueStream(queueOfRDDs)`. Each RDD pushed into the queue will be treated as a batch of data in the DStream, and processed like a stream.
 
 For more details on streams from sockets and files, see the API documentations of the relevant functions in
-[StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) for
+[StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) for
 Scala, [JavaStreamingContext](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html)
 for Java, and [StreamingContext](api/python/pyspark.streaming.html#pyspark.streaming.StreamingContext) for Python.
 
@@ -1219,8 +1219,8 @@ joinedStream = windowedStream.transform(lambda rdd: rdd.join(dataset))
 In fact, you can also dynamically change the dataset you want to join against. The function provided to `transform` is evaluated every batch interval and therefore will use the current dataset that `dataset` reference points to.
 
 The complete list of DStream transformations is available in the API documentation. For the Scala API,
-see [DStream](api/scala/index.html#org.apache.spark.streaming.dstream.DStream)
-and [PairDStreamFunctions](api/scala/index.html#org.apache.spark.streaming.dstream.PairDStreamFunctions).
+see [DStream](api/scala/org/apache/spark/streaming/dstream/DStream.html)
+and [PairDStreamFunctions](api/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.html).
 For the Java API, see [JavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaDStream.html)
 and [JavaPairDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaPairDStream.html).
 For the Python API, see [DStream](api/python/pyspark.streaming.html#pyspark.streaming.DStream).
@@ -2067,7 +2067,7 @@ for prime time, the old one be can be brought down. Note that this can be done f
 sending the data to two destinations (i.e., the earlier and upgraded applications).
 
 - The existing application is shutdown gracefully (see
-[`StreamingContext.stop(...)`](api/scala/index.html#org.apache.spark.streaming.StreamingContext)
+[`StreamingContext.stop(...)`](api/scala/org/apache/spark/streaming/StreamingContext.html)
 or [`JavaStreamingContext.stop(...)`](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html)
 for graceful shutdown options) which ensure data that has been received is completely
 processed before shutdown. Then the
@@ -2104,7 +2104,7 @@ In that case, consider
 [reducing](#reducing-the-batch-processing-times) the batch processing time.
 
 The progress of a Spark Streaming program can also be monitored using the
-[StreamingListener](api/scala/index.html#org.apache.spark.streaming.scheduler.StreamingListener) interface,
+[StreamingListener](api/scala/org/apache/spark/streaming/scheduler/StreamingListener.html) interface,
 which allows you to get receiver status and processing times. Note that this is a developer API
 and it is likely to be improved upon (i.e., more information reported) in the future.
 
@@ -2197,7 +2197,7 @@ computation is not high enough. For example, for distributed reduce operations l
 and `reduceByKeyAndWindow`, the default number of parallel tasks is controlled by
 the `spark.default.parallelism` [configuration property](configuration.html#spark-properties). You
 can pass the level of parallelism as an argument (see
-[`PairDStreamFunctions`](api/scala/index.html#org.apache.spark.streaming.dstream.PairDStreamFunctions)
+[`PairDStreamFunctions`](api/scala/org/apache/spark/streaming/dstream/PairDStreamFunctions.html)
 documentation), or set the `spark.default.parallelism`
 [configuration property](configuration.html#spark-properties) to change the default.
 
@@ -2205,9 +2205,9 @@ documentation), or set the `spark.default.parallelism`
 {:.no_toc}
 The overheads of data serialization can be reduced by tuning the serialization formats. In the case of streaming, there are two types of data that are being serialized.
 
-* **Input data**: By default, the input data received through Receivers is stored in the executors' memory with [StorageLevel.MEMORY_AND_DISK_SER_2](api/scala/index.html#org.apache.spark.storage.StorageLevel$). That is, the data is serialized into bytes to reduce GC overheads, and replicated for tolerating executor failures. Also, the data is kept first in memory, and spilled over to disk only if the memory is insufficient to hold all of the input data necessary for the streaming computation. This serialization obviously has overheads -- the receiver must deserialize the received data and re-serialize it using Spark's serialization format. 
+* **Input data**: By default, the input data received through Receivers is stored in the executors' memory with [StorageLevel.MEMORY_AND_DISK_SER_2](api/scala/org/apache/spark/storage/StorageLevel$.html). That is, the data is serialized into bytes to reduce GC overheads, and replicated for tolerating executor failures. Also, the data is kept first in memory, and spilled over to disk only if the memory is insufficient to hold all of the input data necessary for the streaming computation. This serialization obviously has overheads -- the receiver must deserialize the received data and re-serialize it using Spark's serialization format.
 
-* **Persisted RDDs generated by Streaming Operations**: RDDs generated by streaming computations may be persisted in memory. For example, window operations persist data in memory as they would be processed multiple times. However, unlike the Spark Core default of [StorageLevel.MEMORY_ONLY](api/scala/index.html#org.apache.spark.storage.StorageLevel$), persisted RDDs generated by streaming computations are persisted with [StorageLevel.MEMORY_ONLY_SER](api/scala/index.html#org.apache.spark.storage.StorageLevel$) (i.e. serialized) by default to minimize GC overheads.
+* **Persisted RDDs generated by Streaming Operations**: RDDs generated by streaming computations may be persisted in memory. For example, window operations persist data in memory as they would be processed multiple times. However, unlike the Spark Core default of [StorageLevel.MEMORY_ONLY](api/scala/org/apache/spark/storage/StorageLevel$.html), persisted RDDs generated by streaming computations are persisted with [StorageLevel.MEMORY_ONLY_SER](api/scala/org/apache/spark/storage/StorageLevel.html$) (i.e. serialized) by default to minimize GC overheads.
 
 In both cases, using Kryo serialization can reduce both CPU and memory overheads. See the [Spark Tuning Guide](tuning.html#data-serialization) for more details. For Kryo, consider registering custom classes, and disabling object reference tracking (see Kryo-related configurations in the [Configuration Guide](configuration.html#compression-and-serialization)).
 
@@ -2247,7 +2247,7 @@ A good approach to figure out the right batch size for your application is to te
 conservative batch interval (say, 5-10 seconds) and a low data rate. To verify whether the system
 is able to keep up with the data rate, you can check the value of the end-to-end delay experienced
 by each processed batch (either look for "Total delay" in Spark driver log4j logs, or use the
-[StreamingListener](api/scala/index.html#org.apache.spark.streaming.scheduler.StreamingListener)
+[StreamingListener](api/scala/org/apache/spark/streaming/scheduler/StreamingListener.html)
 interface).
 If the delay is maintained to be comparable to the batch size, then system is stable. Otherwise,
 if the delay is continuously increasing, it means that the system is unable to keep up and it
@@ -2485,10 +2485,10 @@ additional effort may be necessary to achieve exactly-once semantics. There are
 * Third-party DStream data sources can be found in [Third Party Projects](https://spark.apache.org/third-party-projects.html)
 * API documentation
   - Scala docs
-    * [StreamingContext](api/scala/index.html#org.apache.spark.streaming.StreamingContext) and
-  [DStream](api/scala/index.html#org.apache.spark.streaming.dstream.DStream)
-    * [KafkaUtils](api/scala/index.html#org.apache.spark.streaming.kafka.KafkaUtils$),
-    [KinesisUtils](api/scala/index.html#org.apache.spark.streaming.kinesis.KinesisInputDStream),
+    * [StreamingContext](api/scala/org/apache/spark/streaming/StreamingContext.html) and
+  [DStream](api/scala/org/apache/spark/streaming/dstream/DStream.html)
+    * [KafkaUtils](api/scala/org/apache/spark/streaming/kafka/KafkaUtils$.html),
+    [KinesisUtils](api/scala/org/apache/spark/streaming/kinesis/KinesisInputDStream.html),
   - Java docs
     * [JavaStreamingContext](api/java/index.html?org/apache/spark/streaming/api/java/JavaStreamingContext.html),
     [JavaDStream](api/java/index.html?org/apache/spark/streaming/api/java/JavaDStream.html) and
diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 727bda18a14fb..9f4edd73c2b65 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -498,13 +498,13 @@ to track the read position in the stream. The engine uses checkpointing and writ
 
 # API using Datasets and DataFrames
 Since Spark 2.0, DataFrames and Datasets can represent static, bounded data, as well as streaming, unbounded data. Similar to static Datasets/DataFrames, you can use the common entry point `SparkSession`
-([Scala](api/scala/index.html#org.apache.spark.sql.SparkSession)/[Java](api/java/org/apache/spark/sql/SparkSession.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.SparkSession)/[R](api/R/sparkR.session.html) docs)
+([Scala](api/scala/org/apache/spark/sql/SparkSession.html)/[Java](api/java/org/apache/spark/sql/SparkSession.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.SparkSession)/[R](api/R/sparkR.session.html) docs)
 to create streaming DataFrames/Datasets from streaming sources, and apply the same operations on them as static DataFrames/Datasets. If you are not familiar with Datasets/DataFrames, you are strongly advised to familiarize yourself with them using the
 [DataFrame/Dataset Programming Guide](sql-programming-guide.html).
 
 ## Creating streaming DataFrames and streaming Datasets
 Streaming DataFrames can be created through the `DataStreamReader` interface
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.DataStreamReader)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamReader.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader) docs)
+([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamReader.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamReader.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader) docs)
 returned by `SparkSession.readStream()`. In [R](api/R/read.stream.html), with the `read.stream()` method. Similar to the read interface for creating static DataFrame, you can specify the details of the source – data format, schema, options, etc.
 
 #### Input Sources
@@ -557,7 +557,7 @@ Here are the details of all the sources in Spark.
         NOTE 3: Both delete and move actions are best effort. Failing to delete or move files will not fail the streaming query. Spark may not clean up some source files in some circumstances - e.g. the application doesn't shut down gracefully, too many files are queued to clean up.
         <br/><br/>
         For file-format-specific options, see the related methods in <code>DataStreamReader</code>
-        (<a href="api/scala/index.html#org.apache.spark.sql.streaming.DataStreamReader">Scala</a>/<a href="api/java/org/apache/spark/sql/streaming/DataStreamReader.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader">Python</a>/<a
+        (<a href="api/scala/org/apache/spark/sql/streaming/DataStreamReader.html">Scala</a>/<a href="api/java/org/apache/spark/sql/streaming/DataStreamReader.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamReader">Python</a>/<a
         href="api/R/read.stream.html">R</a>).
         E.g. for "parquet" format options see <code>DataStreamReader.parquet()</code>.
         <br/><br/>
@@ -1622,7 +1622,7 @@ However, as a side effect, data from the slower streams will be aggressively dro
 this configuration judiciously.
 
 ### Arbitrary Stateful Operations
-Many usecases require more advanced stateful operations than aggregations. For example, in many usecases, you have to track sessions from data streams of events. For doing such sessionization, you will have to save arbitrary types of data as state, and perform arbitrary operations on the state using the data stream events in every trigger. Since Spark 2.2, this can be done using the operation `mapGroupsWithState` and the more powerful operation `flatMapGroupsWithState`. Both operations allow you to apply user-defined code on grouped Datasets to update user-defined state. For more concrete details, take a look at the API documentation ([Scala](api/scala/index.html#org.apache.spark.sql.streaming.GroupState)/[Java](api/java/org/apache/spark/sql/streaming/GroupState.html)) and the examples ([Scala]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala)/[Java]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredSessionization.java)).
+Many usecases require more advanced stateful operations than aggregations. For example, in many usecases, you have to track sessions from data streams of events. For doing such sessionization, you will have to save arbitrary types of data as state, and perform arbitrary operations on the state using the data stream events in every trigger. Since Spark 2.2, this can be done using the operation `mapGroupsWithState` and the more powerful operation `flatMapGroupsWithState`. Both operations allow you to apply user-defined code on grouped Datasets to update user-defined state. For more concrete details, take a look at the API documentation ([Scala](api/scala/org/apache/spark/sql/streaming/GroupState.html)/[Java](api/java/org/apache/spark/sql/streaming/GroupState.html)) and the examples ([Scala]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala)/[Java]({{site.SPARK_GITHUB_URL}}/blob/v{{site.SPARK_VERSION_SHORT}}/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredSessionization.java)).
 
 Though Spark cannot check and force it, the state function should be implemented with respect to the semantics of the output mode. For example, in Update mode Spark doesn't expect that the state function will emit rows which are older than current watermark plus allowed late record delay, whereas in Append mode the state function can emit these rows.
 
@@ -1679,7 +1679,7 @@ end-to-end exactly once per query. Ensuring end-to-end exactly once for the last
 
 ## Starting Streaming Queries
 Once you have defined the final result DataFrame/Dataset, all that is left is for you to start the streaming computation. To do that, you have to use the `DataStreamWriter`
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.DataStreamWriter)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamWriter) docs)
+([Scala](api/scala/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Java](api/java/org/apache/spark/sql/streaming/DataStreamWriter.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.DataStreamWriter) docs)
 returned through `Dataset.writeStream()`. You will have to specify one or more of the following in this interface.
 
 - *Details of the output sink:* Data format, location, etc.
@@ -1863,7 +1863,7 @@ Here are the details of all the sinks in Spark.
         <code>path</code>: path to the output directory, must be specified.
         <br/><br/>
         For file-format-specific options, see the related methods in DataFrameWriter
-        (<a href="api/scala/index.html#org.apache.spark.sql.DataFrameWriter">Scala</a>/<a href="api/java/org/apache/spark/sql/DataFrameWriter.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.DataFrameWriter">Python</a>/<a
+        (<a href="api/scala/org/apache/spark/sql/DataFrameWriter.html">Scala</a>/<a href="api/java/org/apache/spark/sql/DataFrameWriter.html">Java</a>/<a href="api/python/pyspark.sql.html#pyspark.sql.DataFrameWriter">Python</a>/<a
         href="api/R/write.stream.html">R</a>).
         E.g. for "parquet" format options see <code>DataFrameWriter.parquet()</code>
     </td>
@@ -2175,7 +2175,7 @@ Since Spark 2.4, `foreach` is available in Scala, Java and Python.
 <div class="codetabs">
 <div data-lang="scala"  markdown="1">
 
-In Scala, you have to extend the class `ForeachWriter` ([docs](api/scala/index.html#org.apache.spark.sql.ForeachWriter)).
+In Scala, you have to extend the class `ForeachWriter` ([docs](api/scala/org/apache/spark/sql/ForeachWriter.html)).
 
 {% highlight scala %}
 streamingDatasetOfString.writeStream.foreach(
@@ -2564,7 +2564,7 @@ lastProgress(query)       # the most recent progress update of this streaming qu
 </div>
 
 You can start any number of queries in a single SparkSession. They will all be running concurrently sharing the cluster resources. You can use `sparkSession.streams()` to get the `StreamingQueryManager`
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryManager)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryManager.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.StreamingQueryManager) docs)
+([Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryManager.html)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryManager.html)/[Python](api/python/pyspark.sql.html#pyspark.sql.streaming.StreamingQueryManager) docs)
 that can be used to manage the currently active queries.
 
 <div class="codetabs">
@@ -2624,7 +2624,7 @@ There are multiple ways to monitor active streaming queries. You can either push
 You can directly get the current status and metrics of an active query using 
 `streamingQuery.lastProgress()` and `streamingQuery.status()`. 
 `lastProgress()` returns a `StreamingQueryProgress` object 
-in [Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryProgress) 
+in [Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryProgress.html) 
 and [Java](api/java/org/apache/spark/sql/streaming/StreamingQueryProgress.html)
 and a dictionary with the same fields in Python. It has all the information about
 the progress made in the last trigger of the stream - what data was processed, 
@@ -2632,7 +2632,7 @@ what were the processing rates, latencies, etc. There is also
 `streamingQuery.recentProgress` which returns an array of last few progresses.  
 
 In addition, `streamingQuery.status()` returns a `StreamingQueryStatus` object 
-in [Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryStatus) 
+in [Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryStatus.html) 
 and [Java](api/java/org/apache/spark/sql/streaming/StreamingQueryStatus.html)
 and a dictionary with the same fields in Python. It gives information about
 what the query is immediately doing - is a trigger active, is data being processed, etc.
@@ -2853,7 +2853,7 @@ Will print something like the following.
 
 You can also asynchronously monitor all queries associated with a
 `SparkSession` by attaching a `StreamingQueryListener`
-([Scala](api/scala/index.html#org.apache.spark.sql.streaming.StreamingQueryListener)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryListener.html) docs).
+([Scala](api/scala/org/apache/spark/sql/streaming/StreamingQueryListener.html)/[Java](api/java/org/apache/spark/sql/streaming/StreamingQueryListener.html) docs).
 Once you attach your custom `StreamingQueryListener` object with
 `sparkSession.streams.attachListener()`, you will get callbacks when a query is started and
 stopped and when there is progress made in an active query. Here is an example,
diff --git a/docs/tuning.md b/docs/tuning.md
index 1faf7cfe0d68e..8e29e5d2e9e74 100644
--- a/docs/tuning.md
+++ b/docs/tuning.md
@@ -260,7 +260,7 @@ enough. Spark automatically sets the number of "map" tasks to run on each file a
 (though you can control it through optional parameters to `SparkContext.textFile`, etc), and for
 distributed "reduce" operations, such as `groupByKey` and `reduceByKey`, it uses the largest
 parent RDD's number of partitions. You can pass the level of parallelism as a second argument
-(see the [`spark.PairRDDFunctions`](api/scala/index.html#org.apache.spark.rdd.PairRDDFunctions) documentation),
+(see the [`spark.PairRDDFunctions`](api/scala/org/apache/spark/rdd/PairRDDFunctions.html) documentation),
 or set the config property `spark.default.parallelism` to change the default.
 In general, we recommend 2-3 tasks per CPU core in your cluster.
 

From 39a9e41753c6db606de501c53824b74d4927488f Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Mon, 17 Feb 2020 09:54:00 +0900
Subject: [PATCH 0086/1280] [SPARK-30703][SQL][DOCS][FOLLOWUP] Declare the ANSI
 SQL compliance options as experimental

### What changes were proposed in this pull request?

This is a follow-up of https://github.com/apache/spark/pull/27489.
It declares the ANSI SQL compliance options as experimental in the documentation.

### Why are the changes needed?

The options are experimental. There can be new features/behaviors in future releases.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Generating doc

Closes #27590 from gengliangwang/ExperimentalAnsi.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit da2ca85cee3960de7a86a21483de1d77767ca060)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-ref-ansi-compliance.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index d02383518b967..267184a1cb5fa 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -19,11 +19,13 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL has two options to comply with the SQL standard: `spark.sql.ansi.enabled` and `spark.sql.storeAssignmentPolicy` (See a table below for details).
+Since Spark 3.0, Spark SQL introduces two experimental options to comply with the SQL standard: `spark.sql.ansi.enabled` and `spark.sql.storeAssignmentPolicy` (See a table below for details).
+
 When `spark.sql.ansi.enabled` is set to `true`, Spark SQL follows the standard in basic behaviours (e.g., arithmetic operations, type conversion, and SQL parsing).
 Moreover, Spark SQL has an independent option to control implicit casting behaviours when inserting rows in a table.
 The casting behaviours are defined as store assignment rules in the standard.
-When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with the ANSI store assignment rules.
+
+When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with the ANSI store assignment rules. This is a separate configuration because its default value is `ANSI`, while the configuration `spark.sql.ansi.enabled` is disabled by default.
 
 <table class="table">
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
@@ -31,7 +33,7 @@ When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with
   <td><code>spark.sql.ansi.enabled</code></td>
   <td>false</td>
   <td>
-    When true, Spark tries to conform to the ANSI SQL specification:
+    (Experimental) When true, Spark tries to conform to the ANSI SQL specification:
     1. Spark will throw a runtime exception if an overflow occurs in any operation on integral/decimal field.
     2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in the SQL parser.
   </td>
@@ -40,7 +42,7 @@ When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with
   <td><code>spark.sql.storeAssignmentPolicy</code></td>
   <td>ANSI</td>
   <td>
-    When inserting a value into a column with different data type, Spark will perform type coercion.
+    (Experimental) When inserting a value into a column with different data type, Spark will perform type coercion.
     Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy,
     Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL.
     It disallows certain unreasonable type conversions such as converting string to int or double to boolean.

From fb2e7496006088bd6b98e9776ee51cedad1dfd6b Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 17 Feb 2020 11:06:51 +0900
Subject: [PATCH 0087/1280] [SPARK-30834][DOCS][PYTHON] Add note for
 recommended pandas and pyarrow versions

### What changes were proposed in this pull request?

Add doc for recommended pandas and pyarrow versions.

### Why are the changes needed?

The recommended versions are those that have been thoroughly tested by Spark CI. Other versions may be used at the discretion of the user.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

NA

Closes #27587 from BryanCutler/python-doc-rec-pandas-pyarrow-SPARK-30834-3.0.

Lead-authored-by: Bryan Cutler <cutlerb@gmail.com>
Co-authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit be3cb71e9cb34ad9054325c3122745e66e6f1ede)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-pyspark-pandas-with-arrow.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/sql-pyspark-pandas-with-arrow.md b/docs/sql-pyspark-pandas-with-arrow.md
index 92a515746b607..63ba0ba5e1b9f 100644
--- a/docs/sql-pyspark-pandas-with-arrow.md
+++ b/docs/sql-pyspark-pandas-with-arrow.md
@@ -33,9 +33,11 @@ working with Arrow-enabled data.
 
 ### Ensure PyArrow Installed
 
+To use Apache Arrow in PySpark, [the recommended version of PyArrow](#recommended-pandas-and-pyarrow-versions)
+should be installed.
 If you install PySpark using pip, then PyArrow can be brought in as an extra dependency of the
 SQL module with the command `pip install pyspark[sql]`. Otherwise, you must ensure that PyArrow
-is installed and available on all cluster nodes. The current supported version is 0.15.1+.
+is installed and available on all cluster nodes.
 You can install using pip or conda from the conda-forge channel. See PyArrow
 [installation](https://arrow.apache.org/docs/python/install.html) for details.
 
@@ -338,6 +340,12 @@ different than a Pandas timestamp. It is recommended to use Pandas time series f
 working with timestamps in `pandas_udf`s to get the best performance, see
 [here](https://pandas.pydata.org/pandas-docs/stable/timeseries.html) for details.
 
+### Recommended Pandas and PyArrow Versions
+
+For usage with pyspark.sql, the supported versions of Pandas is 0.24.2 and PyArrow is 0.15.1. Higher
+versions may be used, however, compatibility and data correctness can not be guaranteed and should
+be verified by the user.
+
 ### Compatibility Setting for PyArrow >= 0.15.0 and Spark 2.3.x, 2.4.x
 
 Since Arrow 0.15.0, a change in the binary IPC format requires an environment variable to be

From 44e314edb4b86ca3a8622124539073397dbe68de Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Mon, 17 Feb 2020 12:06:25 +0900
Subject: [PATCH 0088/1280] [MINOR][DOC] Fix 2 style issues in
 running-on-kubernetes doc

### What changes were proposed in this pull request?

fix style issue in the k8s document, please go to http://spark.apache.org/docs/3.0.0-preview2/running-on-kubernetes.html and search the keyword`spark.kubernetes.file.upload.path` to jump to the error context

### Why are the changes needed?

doc correctness

### Does this PR introduce any user-facing change?

Nah
### How was this patch tested?

Nah

Closes #27582 from yaooqinn/k8s-doc.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0353cbf092e15a09e8979070ecd5b653062b2cb5)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/running-on-kubernetes.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 61d6154ccb084..53b883b6bbf30 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1120,7 +1120,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>(none)</td>
   <td>
     Path to store files at the spark submit side in cluster mode. For example:
-    <code>spark.kubernetes.file.upload.path=s3a://<s3-bucket>/path</code>
+    <code>spark.kubernetes.file.upload.path=s3a://&lt;s3-bucket&gt;/path</code>
     File should specified as <code>file://path/to/file </code> or absolute path.
   </td>
 </tr>
@@ -1247,7 +1247,7 @@ The following affect the driver and executor containers. All other containers in
 </tr>
 <tr>
   <td>name</td>
-  <td>See description.</code></td>
+  <td>See description</td>
   <td>
     The container name will be assigned by spark ("spark-kubernetes-driver" for the driver container, and
     "executor" for each executor container) if not defined by the pod template. If the container is defined by the

From bda685d9f9cdd745e0a8f842034ea5d843c3f836 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 17 Feb 2020 12:07:46 +0800
Subject: [PATCH 0089/1280] [SPARK-30799][SQL] "spark_catalog.t" should not be
 resolved to temp view

### What changes were proposed in this pull request?

No v2 command supports temp views and the `ResolveCatalogs`/`ResolveSessionCatalog` framework is designed with this assumption.

However, `ResolveSessionCatalog` needs to fallback to v1 commands, which do support temp views (e.g. CACHE TABLE). To work around it, we add a hack in `CatalogAndIdentifier`, which does not expand the given identifier with current namespace if the catalog is session catalog.

This works fine in most cases, as temp views should take precedence over tables during lookup. So if `CatalogAndIdentifier` returns a single name "t", the v1 commands can still resolve it to temp views correctly, or resolve it to table "default.t" if temp view doesn't exist.

However, if users write `spark_catalog.t`, it shouldn't be resolved to temp views as temp views don't belong to any catalog. `CatalogAndIdentifier` can't distinguish between `spark_catalog.t` and `t`, so the caller side may mistakenly resolve `spark_catalog.t` to a temp view.

This PR proposes to fix this issue by
1. remove the hack in `CatalogAndIdentifier`, and clearly document that this shouldn't be used to resolve temp views.
2. update `ResolveSessionCatalog` to explicitly look up temp views first before calling `CatalogAndIdentifier`, for v1 commands that support temp views.

### Why are the changes needed?

To avoid releasing a behavior that we should not support.

Removing the hack also fixes the problem we hit in https://github.com/apache/spark/pull/27532/files#diff-57b3d87be744b7d79a9beacf8e5e5eb2R937

### Does this PR introduce any user-facing change?

yes, now it's not allowed to refer to a temp view with `spark_catalog` prefix.

### How was this patch tested?

new tests

Closes #27550 from cloud-fan/ns.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ab07c6300c884e772f88694f4b718659c45dbb33)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/catalog/SessionCatalog.scala |   4 +
 .../sql/connector/catalog/LookupCatalog.scala |  15 +-
 .../catalog/LookupCatalogSuite.scala          |   1 +
 .../analysis/ResolveSessionCatalog.scala      | 180 ++++++++++++------
 .../internal/BaseSessionStateBuilder.scala    |   3 +-
 .../sql-tests/results/describe.sql.out        |  12 +-
 .../sql/connector/DataSourceV2SQLSuite.scala  |  32 +++-
 .../command/PlanResolutionSuite.scala         |   3 +-
 .../sql/hive/HiveSessionStateBuilder.scala    |   3 +-
 9 files changed, 176 insertions(+), 77 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 12f9a61fc2b65..c80d9d2d7c183 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -1344,6 +1344,10 @@ class SessionCatalog(
       !hiveFunctions.contains(name.funcName.toLowerCase(Locale.ROOT))
   }
 
+  def isTempFunction(name: String): Boolean = {
+    isTemporaryFunction(FunctionIdentifier(name))
+  }
+
   /**
    * Return whether this function has been registered in the function registry of the current
    * session. If not existed, return false.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
index 080ddf1d027e9..b0b9d7b15cef3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
@@ -94,6 +94,10 @@ private[sql] trait LookupCatalog extends Logging {
    * Extract catalog and identifier from a multi-part name with the current catalog if needed.
    * Catalog name takes precedence over identifier, but for a single-part name, identifier takes
    * precedence over catalog name.
+   *
+   * Note that, this pattern is used to look up permanent catalog objects like table, view,
+   * function, etc. If you need to look up temp objects like temp view, please do it separately
+   * before calling this pattern, as temp objects don't belong to any catalog.
    */
   object CatalogAndIdentifier {
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
@@ -103,16 +107,7 @@ private[sql] trait LookupCatalog extends Logging {
     def unapply(nameParts: Seq[String]): Option[(CatalogPlugin, Identifier)] = {
       assert(nameParts.nonEmpty)
       if (nameParts.length == 1) {
-        // If the current catalog is session catalog, the current namespace is not used because
-        // the single-part name could be referencing a temp view, which doesn't belong to any
-        // namespaces. An empty namespace will be resolved inside the session catalog
-        // implementation when a relation is looked up.
-        val ns = if (CatalogV2Util.isSessionCatalog(currentCatalog)) {
-          Array.empty[String]
-        } else {
-          catalogManager.currentNamespace
-        }
-        Some((currentCatalog, Identifier.of(ns, nameParts.head)))
+        Some((currentCatalog, Identifier.of(catalogManager.currentNamespace, nameParts.head)))
       } else if (nameParts.head.equalsIgnoreCase(globalTempDB)) {
         // Conceptually global temp views are in a special reserved catalog. However, the v2 catalog
         // API does not support view yet, and we have to use v1 commands to deal with global temp
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
index b2f27e4740cbe..b36ded3d9d2ac 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/LookupCatalogSuite.scala
@@ -50,6 +50,7 @@ class LookupCatalogSuite extends SparkFunSuite with LookupCatalog with Inside {
     })
     when(manager.currentCatalog).thenReturn(sessionCatalog)
     when(manager.v2SessionCatalog).thenReturn(sessionCatalog)
+    when(manager.currentNamespace).thenReturn(Array.empty[String])
     manager
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 77d549c28aae5..62ebb1c1492ab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType, CatalogUtils}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, CatalogV2Util, Identifier, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange, V1Table}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogPlugin, CatalogV2Util, LookupCatalog, SupportsNamespaces, TableCatalog, TableChange, V1Table}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, RefreshTable}
@@ -39,7 +39,8 @@ import org.apache.spark.sql.types.{HIVE_TYPE_STRING, HiveStringType, MetadataBui
 class ResolveSessionCatalog(
     val catalogManager: CatalogManager,
     conf: SQLConf,
-    isView: Seq[String] => Boolean)
+    isTempView: Seq[String] => Boolean,
+    isTempFunction: String => Boolean)
   extends Rule[LogicalPlan] with LookupCatalog {
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
   import org.apache.spark.sql.connector.catalog.CatalogV2Util._
@@ -215,8 +216,9 @@ class ResolveSessionCatalog(
       }
       AlterDatabaseSetLocationCommand(ns.head, location)
 
-    case RenameTableStatement(SessionCatalogAndTable(_, oldName), newNameParts, isView) =>
-      AlterTableRenameCommand(oldName.asTableIdentifier, newNameParts.asTableIdentifier, isView)
+    // v1 RENAME TABLE supports temp view.
+    case RenameTableStatement(TempViewOrV1Table(oldName), newName, isView) =>
+      AlterTableRenameCommand(oldName.asTableIdentifier, newName.asTableIdentifier, isView)
 
     case DescribeRelation(ResolvedTable(_, ident, _: V1Table), partitionSpec, isExtended) =>
       DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended)
@@ -228,10 +230,12 @@ class ResolveSessionCatalog(
     case DescribeColumnStatement(
         SessionCatalogAndTable(catalog, tbl), colNameParts, isExtended) =>
       loadTable(catalog, tbl.asIdentifier).collect {
+        // `V1Table` also includes permanent views.
         case v1Table: V1Table =>
           DescribeColumnCommand(tbl.asTableIdentifier, colNameParts, isExtended)
       }.getOrElse {
-        if (isView(tbl)) {
+        if (isTempView(tbl)) {
+          // v1 DESCRIBE COLUMN supports temp view.
           DescribeColumnCommand(tbl.asTableIdentifier, colNameParts, isExtended)
         } else {
           throw new AnalysisException("Describing columns is not supported for v2 tables.")
@@ -279,8 +283,9 @@ class ResolveSessionCatalog(
           ignoreIfExists = c.ifNotExists)
       }
 
-    case RefreshTableStatement(SessionCatalogAndTable(_, tbl)) =>
-      RefreshTable(tbl.asTableIdentifier)
+    // v1 REFRESH TABLE supports temp view.
+    case RefreshTableStatement(TempViewOrV1Table(name)) =>
+      RefreshTable(name.asTableIdentifier)
 
     // For REPLACE TABLE [AS SELECT], we should fail if the catalog is resolved to the
     // session catalog and the table provider is not v2.
@@ -315,11 +320,13 @@ class ResolveSessionCatalog(
           orCreate = c.orCreate)
       }
 
-    case DropTableStatement(SessionCatalogAndTable(catalog, tbl), ifExists, purge) =>
-      DropTableCommand(tbl.asTableIdentifier, ifExists, isView = false, purge = purge)
+    // v1 DROP TABLE supports temp view.
+    case DropTableStatement(TempViewOrV1Table(name), ifExists, purge) =>
+      DropTableCommand(name.asTableIdentifier, ifExists, isView = false, purge = purge)
 
-    case DropViewStatement(SessionCatalogAndTable(catalog, viewName), ifExists) =>
-      DropTableCommand(viewName.asTableIdentifier, ifExists, isView = true, purge = false)
+    // v1 DROP TABLE supports temp view.
+    case DropViewStatement(TempViewOrV1Table(name), ifExists) =>
+      DropTableCommand(name.asTableIdentifier, ifExists, isView = true, purge = false)
 
     case c @ CreateNamespaceStatement(CatalogAndNamespace(catalog, ns), _, _)
         if isSessionCatalog(catalog) =>
@@ -393,12 +400,18 @@ class ResolveSessionCatalog(
       ShowCreateTableAsSerdeCommand(v1TableName.asTableIdentifier)
 
     case CacheTableStatement(tbl, plan, isLazy, options) =>
-      val v1TableName = parseV1Table(tbl, "CACHE TABLE")
-      CacheTableCommand(v1TableName.asTableIdentifier, plan, isLazy, options)
+      val name = if (plan.isDefined) {
+        // CACHE TABLE ... AS SELECT creates a temp view with the input query.
+        // Temp view doesn't belong to any catalog and we shouldn't resolve catalog in the name.
+        tbl
+      } else {
+        parseTempViewOrV1Table(tbl, "CACHE TABLE")
+      }
+      CacheTableCommand(name.asTableIdentifier, plan, isLazy, options)
 
     case UncacheTableStatement(tbl, ifExists) =>
-      val v1TableName = parseV1Table(tbl, "UNCACHE TABLE")
-      UncacheTableCommand(v1TableName.asTableIdentifier, ifExists)
+      val name = parseTempViewOrV1Table(tbl, "UNCACHE TABLE")
+      UncacheTableCommand(name.asTableIdentifier, ifExists)
 
     case TruncateTableStatement(tbl, partitionSpec) =>
       val v1TableName = parseV1Table(tbl, "TRUNCATE TABLE")
@@ -427,10 +440,6 @@ class ResolveSessionCatalog(
         throw new AnalysisException(
           s"Namespace name should have only one part if specified: ${ns.get.quoted}")
       }
-      if (tbl.length > 2) {
-        throw new AnalysisException(
-          s"Table name should have at most two parts: ${tbl.quoted}")
-      }
       ShowColumnsCommand(db, v1TableName)
 
     case AlterTableRecoverPartitionsStatement(tbl) =>
@@ -470,10 +479,10 @@ class ResolveSessionCatalog(
         serdeProperties,
         partitionSpec)
 
-    case AlterViewAsStatement(tbl, originalText, query) =>
-      val v1TableName = parseV1Table(tbl, "ALTER VIEW QUERY")
+    case AlterViewAsStatement(name, originalText, query) =>
+      val viewName = parseTempViewOrV1Table(name, "ALTER VIEW QUERY")
       AlterViewAsCommand(
-        v1TableName.asTableIdentifier,
+        viewName.asTableIdentifier,
         originalText,
         query)
 
@@ -481,7 +490,12 @@ class ResolveSessionCatalog(
       tbl, userSpecifiedColumns, comment, properties,
       originalText, child, allowExisting, replace, viewType) =>
 
-      val v1TableName = parseV1Table(tbl, "CREATE VIEW")
+      val v1TableName = if (viewType != PersistedView) {
+        // temp view doesn't belong to any catalog and we shouldn't resolve catalog in the name.
+        tbl
+      } else {
+        parseV1Table(tbl, "CREATE VIEW")
+      }
       CreateViewCommand(
         v1TableName.asTableIdentifier,
         userSpecifiedColumns,
@@ -496,56 +510,94 @@ class ResolveSessionCatalog(
     case ShowTableProperties(r: ResolvedTable, propertyKey) if isSessionCatalog(r.catalog) =>
       ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
 
-    case DescribeFunctionStatement(CatalogAndIdentifier(catalog, ident), extended) =>
+    case DescribeFunctionStatement(nameParts, extended) =>
       val functionIdent =
-        parseSessionCatalogFunctionIdentifier("DESCRIBE FUNCTION", catalog, ident)
+        parseSessionCatalogFunctionIdentifier(nameParts, "DESCRIBE FUNCTION")
       DescribeFunctionCommand(functionIdent, extended)
 
     case ShowFunctionsStatement(userScope, systemScope, pattern, fun) =>
       val (database, function) = fun match {
-        case Some(CatalogAndIdentifier(catalog, ident)) =>
+        case Some(nameParts) =>
           val FunctionIdentifier(fn, db) =
-            parseSessionCatalogFunctionIdentifier("SHOW FUNCTIONS", catalog, ident)
+            parseSessionCatalogFunctionIdentifier(nameParts, "SHOW FUNCTIONS")
           (db, Some(fn))
         case None => (None, pattern)
       }
       ShowFunctionsCommand(database, function, userScope, systemScope)
 
-    case DropFunctionStatement(CatalogAndIdentifier(catalog, ident), ifExists, isTemp) =>
+    case DropFunctionStatement(nameParts, ifExists, isTemp) =>
       val FunctionIdentifier(function, database) =
-        parseSessionCatalogFunctionIdentifier("DROP FUNCTION", catalog, ident)
+        parseSessionCatalogFunctionIdentifier(nameParts, "DROP FUNCTION")
       DropFunctionCommand(database, function, ifExists, isTemp)
 
-    case CreateFunctionStatement(CatalogAndIdentifier(catalog, ident),
+    case CreateFunctionStatement(nameParts,
       className, resources, isTemp, ignoreIfExists, replace) =>
-      val FunctionIdentifier(function, database) =
-        parseSessionCatalogFunctionIdentifier("CREATE FUNCTION", catalog, ident)
-      CreateFunctionCommand(database, function, className, resources, isTemp, ignoreIfExists,
-        replace)
+      if (isTemp) {
+        // temp func doesn't belong to any catalog and we shouldn't resolve catalog in the name.
+        val database = if (nameParts.length > 2) {
+          throw new AnalysisException(s"Unsupported function name '${nameParts.quoted}'")
+        } else if (nameParts.length == 2) {
+          Some(nameParts.head)
+        } else {
+          None
+        }
+        CreateFunctionCommand(
+          database,
+          nameParts.last,
+          className,
+          resources,
+          isTemp,
+          ignoreIfExists,
+          replace)
+      } else {
+        val FunctionIdentifier(function, database) =
+          parseSessionCatalogFunctionIdentifier(nameParts, "CREATE FUNCTION")
+        CreateFunctionCommand(database, function, className, resources, isTemp, ignoreIfExists,
+          replace)
+      }
   }
 
+  // TODO: move function related v2 statements to the new framework.
   private def parseSessionCatalogFunctionIdentifier(
-      sql: String,
-      catalog: CatalogPlugin,
-      functionIdent: Identifier): FunctionIdentifier = {
-    if (isSessionCatalog(catalog)) {
-      functionIdent.asMultipartIdentifier match {
-        case Seq(db, fn) => FunctionIdentifier(fn, Some(db))
-        case Seq(fn) => FunctionIdentifier(fn, None)
-        case _ =>
-          throw new AnalysisException(s"Unsupported function name '${functionIdent.quoted}'")
-      }
-    } else {
-      throw new AnalysisException(s"$sql is only supported in v1 catalog")
+      nameParts: Seq[String],
+      sql: String): FunctionIdentifier = {
+    if (nameParts.length == 1 && isTempFunction(nameParts.head)) {
+      return FunctionIdentifier(nameParts.head)
     }
-  }
 
-  private def parseV1Table(tableName: Seq[String], sql: String): Seq[String] = {
-    val CatalogAndIdentifier(catalog, ident) = tableName
-    if (!isSessionCatalog(catalog)) {
-      throw new AnalysisException(s"$sql is only supported with v1 tables.")
+    nameParts match {
+      case SessionCatalogAndIdentifier(_, ident) =>
+        if (nameParts.length == 1) {
+          // If there is only one name part, it means the current catalog is the session catalog.
+          // Here we don't fill the default database, to keep the error message unchanged for
+          // v1 commands.
+          FunctionIdentifier(nameParts.head, None)
+        } else {
+          ident.namespace match {
+            // For name parts like `spark_catalog.t`, we need to fill in the default database so
+            // that the caller side won't treat it as a temp function.
+            case Array() if nameParts.head == CatalogManager.SESSION_CATALOG_NAME =>
+              FunctionIdentifier(
+                ident.name, Some(catalogManager.v1SessionCatalog.getCurrentDatabase))
+            case Array(db) => FunctionIdentifier(ident.name, Some(db))
+            case _ =>
+              throw new AnalysisException(s"Unsupported function name '$ident'")
+          }
+        }
+
+      case _ => throw new AnalysisException(s"$sql is only supported in v1 catalog")
     }
-    ident.asMultipartIdentifier
+  }
+
+  private def parseV1Table(tableName: Seq[String], sql: String): Seq[String] = tableName match {
+    case SessionCatalogAndTable(_, tbl) => tbl
+    case _ => throw new AnalysisException(s"$sql is only supported with v1 tables.")
+  }
+
+  private def parseTempViewOrV1Table(
+      nameParts: Seq[String], sql: String): Seq[String] = nameParts match {
+    case TempViewOrV1Table(name) => name
+    case _ => throw new AnalysisException(s"$sql is only supported with temp views or v1 tables.")
   }
 
   private def buildCatalogTable(
@@ -584,7 +636,29 @@ class ResolveSessionCatalog(
   object SessionCatalogAndTable {
     def unapply(nameParts: Seq[String]): Option[(CatalogPlugin, Seq[String])] = nameParts match {
       case SessionCatalogAndIdentifier(catalog, ident) =>
-        Some(catalog -> ident.asMultipartIdentifier)
+        if (nameParts.length == 1) {
+          // If there is only one name part, it means the current catalog is the session catalog.
+          // Here we return the original name part, to keep the error message unchanged for
+          // v1 commands.
+          Some(catalog -> nameParts)
+        } else {
+          Some(catalog -> ident.asMultipartIdentifier)
+        }
+      case _ => None
+    }
+  }
+
+  object TempViewOrV1Table {
+    def unapply(nameParts: Seq[String]): Option[Seq[String]] = nameParts match {
+      case _ if isTempView(nameParts) => Some(nameParts)
+      case SessionCatalogAndTable(_, tbl) =>
+        if (nameParts.head == CatalogManager.SESSION_CATALOG_NAME && tbl.length == 1) {
+          // For name parts like `spark_catalog.t`, we need to fill in the default database so
+          // that the caller side won't treat it as a temp view.
+          Some(Seq(catalogManager.v1SessionCatalog.getCurrentDatabase, tbl.head))
+        } else {
+          Some(tbl)
+        }
       case _ => None
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index eb658e2d8850e..2137fe2d0a8ff 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -174,7 +174,8 @@ abstract class BaseSessionStateBuilder(
       new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
-        new ResolveSessionCatalog(catalogManager, conf, catalog.isView) +:
+        new ResolveSessionCatalog(
+          catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
         customResolutionRules
 
     override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index 697e006544acf..a7ef828e4d5d2 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -520,7 +520,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, false
+   +- DescribeTableCommand `default`.`t`, false
 
 
 -- !query
@@ -530,7 +530,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, true
+   +- DescribeTableCommand `default`.`t`, true
 
 
 -- !query
@@ -544,14 +544,14 @@ struct<plan:string>
 
 == Analyzed Logical Plan ==
 col_name: string, data_type: string, comment: string
-DescribeTableCommand `t`, false
+DescribeTableCommand `default`.`t`, false
 
 == Optimized Logical Plan ==
-DescribeTableCommand `t`, false
+DescribeTableCommand `default`.`t`, false
 
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, false
+   +- DescribeTableCommand `default`.`t`, false
 
 
 -- !query
@@ -571,7 +571,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeTableCommand
-   +- DescribeTableCommand `t`, Map(c -> Us, d -> 2), false
+   +- DescribeTableCommand `default`.`t`, Map(c -> Us, d -> 2), false
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index eabcb81c50646..2acd7e39fe949 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -1859,12 +1859,12 @@ class DataSourceV2SQLSuite
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-      testV1Command("CACHE TABLE", t)
+      testV1CommandSupportingTempView("CACHE TABLE", t)
 
       val e = intercept[AnalysisException] {
         sql(s"CACHE LAZY TABLE $t")
       }
-      assert(e.message.contains("CACHE TABLE is only supported with v1 tables"))
+      assert(e.message.contains("CACHE TABLE is only supported with temp views or v1 tables"))
     }
   }
 
@@ -1873,8 +1873,8 @@ class DataSourceV2SQLSuite
     withTable(t) {
       sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-      testV1Command("UNCACHE TABLE", t)
-      testV1Command("UNCACHE TABLE", s"IF EXISTS $t")
+      testV1CommandSupportingTempView("UNCACHE TABLE", t)
+      testV1CommandSupportingTempView("UNCACHE TABLE", s"IF EXISTS $t")
     }
   }
 
@@ -1954,7 +1954,7 @@ class DataSourceV2SQLSuite
     val e = intercept[AnalysisException] {
       sql(s"ALTER VIEW $v AS SELECT 1")
     }
-    assert(e.message.contains("ALTER VIEW QUERY is only supported with v1 tables"))
+    assert(e.message.contains("ALTER VIEW QUERY is only supported with temp views or v1 tables"))
   }
 
   test("CREATE VIEW") {
@@ -2252,6 +2252,21 @@ class DataSourceV2SQLSuite
       .head().getString(1) === expectedComment)
   }
 
+  test("SPARK-30799: temp view name can't contain catalog name") {
+    val sessionCatalogName = CatalogManager.SESSION_CATALOG_NAME
+    withTempView("v") {
+      spark.range(10).createTempView("v")
+      val e1 = intercept[AnalysisException](
+        sql(s"CACHE TABLE $sessionCatalogName.v")
+      )
+      assert(e1.message.contains("Table or view not found: default.v"))
+    }
+    val e2 = intercept[AnalysisException] {
+      sql(s"CREATE TEMP VIEW $sessionCatalogName.v AS SELECT 1")
+    }
+    assert(e2.message.contains("It is not allowed to add database prefix"))
+  }
+
   private def testV1Command(sqlCommand: String, sqlParams: String): Unit = {
     val e = intercept[AnalysisException] {
       sql(s"$sqlCommand $sqlParams")
@@ -2259,6 +2274,13 @@ class DataSourceV2SQLSuite
     assert(e.message.contains(s"$sqlCommand is only supported with v1 tables"))
   }
 
+  private def testV1CommandSupportingTempView(sqlCommand: String, sqlParams: String): Unit = {
+    val e = intercept[AnalysisException] {
+      sql(s"$sqlCommand $sqlParams")
+    }
+    assert(e.message.contains(s"$sqlCommand is only supported with temp views or v1 tables"))
+  }
+
   private def assertAnalysisError(sqlStatement: String, expectedError: String): Unit = {
     val errMsg = intercept[AnalysisException] {
       sql(sqlStatement)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index d439e5b1cd651..21c698c4a2c4d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -128,6 +128,7 @@ class PlanResolutionSuite extends AnalysisTest {
       }
     })
     when(manager.currentCatalog).thenReturn(v2SessionCatalog)
+    when(manager.currentNamespace).thenReturn(Array.empty[String])
     when(manager.v1SessionCatalog).thenReturn(v1SessionCatalog)
     manager
   }
@@ -145,7 +146,7 @@ class PlanResolutionSuite extends AnalysisTest {
       ResolveInlineTables(conf),
       analyzer.ResolveRelations,
       new ResolveCatalogs(catalogManager),
-      new ResolveSessionCatalog(catalogManager, conf, _ == Seq("v")),
+      new ResolveSessionCatalog(catalogManager, conf, _ == Seq("v"), _ => false),
       analyzer.ResolveTables,
       analyzer.ResolveReferences,
       analyzer.ResolveSubqueryColumnAliases,
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index b117c582a3e6e..8020492ff759d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -75,7 +75,8 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
         new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
-        new ResolveSessionCatalog(catalogManager, conf, catalog.isView) +:
+        new ResolveSessionCatalog(
+          catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
         customResolutionRules
 
     override val postHocResolutionRules: Seq[Rule[LogicalPlan]] =

From 518cc90f434b08c553bc6a23d650decd3449db9d Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 17 Feb 2020 13:59:21 +0800
Subject: [PATCH 0090/1280] [SPARK-30843][SQL] Fix getting of time components
 before 1582 year

### What changes were proposed in this pull request?

1. Rewrite DateTimeUtils methods `getHours()`, `getMinutes()`, `getSeconds()`, `getSecondsWithFraction()`, `getMilliseconds()` and `getMicroseconds()` using Java 8 time APIs. This will automatically switch the `Hour`, `Minute`, `Second` and `DatePart` expressions on Proleptic Gregorian calendar.
2. Remove unused methods and constant of DateTimeUtils - `to2001`, `YearZero `, `toYearZero` and `absoluteMicroSecond()`.
3. Remove unused value `timeZone` from `TimeZoneAwareExpression` since all expressions have been migrated to Java 8 time API, and legacy instance of `TimeZone` is not needed any more.
4. Change signatures of modified DateTimeUtils methods, and pass `ZoneId` instead of `TimeZone`. This will allow to avoid unnecessary conversions `TimeZone` -> `String` -> `ZoneId`.
5. Modify tests in `DateTimeUtilsSuite` and in `DateExpressionsSuite` to pass `ZoneId` instead of `TimeZone`. Correct the tests, to pass tested zone id instead of None.

### Why are the changes needed?
The changes fix the issue of wrong results returned by the `hour()`, `minute()`, `second()`, `date_part('millisecond', ...)` and `date_part('microsecond', ....)`, see example in [SPARK-30843](https://issues.apache.org/jira/browse/SPARK-30843).

### Does this PR introduce any user-facing change?
Yes. After the changes, the results of examples from SPARK-30843:
```sql
spark-sql> select hour(timestamp '0010-01-01 00:00:00');
0
spark-sql> select minute(timestamp '0010-01-01 00:00:00');
0
spark-sql> select second(timestamp '0010-01-01 00:00:00');
0
spark-sql> select date_part('milliseconds', timestamp '0010-01-01 00:00:00');
0.000
spark-sql> select date_part('microseconds', timestamp '0010-01-01 00:00:00');
0
```

### How was this patch tested?
- By existing test suites `DateTimeUtilsSuite`, `DateExpressionsSuite` and `DateFunctionsSuite`.
- Add new tests to `DateExpressionsSuite` and `DateTimeUtilsSuite` for 10 year, like:
```scala
  input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
  assert(getHours(input, zonePST) === 0)
```
- Re-run `DateTimeBenchmark` using Amazon EC2.

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ami-06f2f779464715dc5 (ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1) |
| Java | OpenJDK8/11 |

Closes #27596 from MaxGekk/localtimestamp-greg-cal.

Lead-authored-by: Maxim Gekk <max.gekk@gmail.com>
Co-authored-by: Max Gekk <max.gekk@gmail.com>
Co-authored-by: Ubuntu <ubuntu@ip-172-31-1-30.us-west-2.compute.internal>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9107f77f15cd0630dc981b6e8a9ca696b79e624f)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/datetimeExpressions.scala     |  37 +-
 .../sql/catalyst/util/DateTimeUtils.scala     |  44 +--
 .../expressions/DateExpressionsSuite.scala    |  39 ++-
 .../sql/catalyst/util/DateTimeTestUtils.scala |  18 +-
 .../catalyst/util/DateTimeUtilsSuite.scala    | 173 +++++-----
 .../DateTimeBenchmark-jdk11-results.txt       | 326 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 326 +++++++++---------
 7 files changed, 484 insertions(+), 479 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index adf7251256041..05074d97efa44 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -51,7 +51,6 @@ trait TimeZoneAwareExpression extends Expression {
   /** Returns a copy of this expression with the specified timeZoneId. */
   def withTimeZone(timeZoneId: String): TimeZoneAwareExpression
 
-  @transient lazy val timeZone: TimeZone = DateTimeUtils.getTimeZone(timeZoneId.get)
   @transient lazy val zoneId: ZoneId = DateTimeUtils.getZoneId(timeZoneId.get)
 }
 
@@ -229,13 +228,13 @@ case class Hour(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getHours(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getHours(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getHours($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getHours($c, $zid)")
   }
 }
 
@@ -260,13 +259,13 @@ case class Minute(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMinutes(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getMinutes(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMinutes($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getMinutes($c, $zid)")
   }
 }
 
@@ -291,13 +290,13 @@ case class Second(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getSeconds(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getSeconds(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getSeconds($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getSeconds($c, $zid)")
   }
 }
 
@@ -315,13 +314,13 @@ case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = No
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getSecondsWithFraction(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getSecondsWithFraction(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getSecondsWithFraction($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getSecondsWithFraction($c, $zid)")
   }
 }
 
@@ -338,13 +337,13 @@ case class Milliseconds(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMilliseconds(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getMilliseconds(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMilliseconds($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getMilliseconds($c, $zid)")
   }
 }
 
@@ -357,13 +356,13 @@ case class Microseconds(child: Expression, timeZoneId: Option[String] = None)
     copy(timeZoneId = Option(timeZoneId))
 
   override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMicroseconds(timestamp.asInstanceOf[Long], timeZone)
+    DateTimeUtils.getMicroseconds(timestamp.asInstanceOf[Long], zoneId)
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val tz = ctx.addReferenceObj("timeZone", timeZone)
+    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMicroseconds($c, $tz)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getMicroseconds($c, $zid)")
   }
 }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index ce0c138791f30..768dde0c72a89 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -47,12 +47,6 @@ object DateTimeUtils {
   // it's 2440587.5, rounding up to compatible with Hive
   final val JULIAN_DAY_OF_EPOCH = 2440588
 
-  // number of days between 1.1.1970 and 1.1.2001
-  final val to2001 = -11323
-
-  // this is year -17999, calculation: 50 * daysIn400Year
-  final val YearZero = -17999
-  final val toYearZero = to2001 + 7304850
   final val TimeZoneGMT = TimeZone.getTimeZone("GMT")
   final val TimeZoneUTC = TimeZone.getTimeZone("UTC")
 
@@ -411,63 +405,55 @@ object DateTimeUtils {
     }
   }
 
-  /**
-   * Returns the microseconds since year zero (-17999) from microseconds since epoch.
-   */
-  private def absoluteMicroSecond(microsec: SQLTimestamp): SQLTimestamp = {
-    microsec + toYearZero * MICROS_PER_DAY
-  }
-
-  private def localTimestamp(microsec: SQLTimestamp, timeZone: TimeZone): SQLTimestamp = {
-    val zoneOffsetUs = MILLISECONDS.toMicros(timeZone.getOffset(MICROSECONDS.toMillis(microsec)))
-    absoluteMicroSecond(microsec) + zoneOffsetUs
+  private def localTimestamp(microsec: SQLTimestamp, zoneId: ZoneId): LocalDateTime = {
+    microsToInstant(microsec).atZone(zoneId).toLocalDateTime
   }
 
   /**
    * Returns the hour value of a given timestamp value. The timestamp is expressed in microseconds.
    */
-  def getHours(microsec: SQLTimestamp, timeZone: TimeZone): Int = {
-    (MICROSECONDS.toHours(localTimestamp(microsec, timeZone)) % 24).toInt
+  def getHours(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
+    localTimestamp(microsec, zoneId).getHour
   }
 
   /**
    * Returns the minute value of a given timestamp value. The timestamp is expressed in
    * microseconds.
    */
-  def getMinutes(microsec: SQLTimestamp, timeZone: TimeZone): Int = {
-    (MICROSECONDS.toMinutes(localTimestamp(microsec, timeZone)) % 60).toInt
+  def getMinutes(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
+    localTimestamp(microsec, zoneId).getMinute
   }
 
   /**
    * Returns the second value of a given timestamp value. The timestamp is expressed in
    * microseconds.
    */
-  def getSeconds(microsec: SQLTimestamp, timeZone: TimeZone): Int = {
-    (MICROSECONDS.toSeconds(localTimestamp(microsec, timeZone)) % 60).toInt
+  def getSeconds(microsec: SQLTimestamp, zoneId: ZoneId): Int = {
+    localTimestamp(microsec, zoneId).getSecond
   }
 
   /**
    * Returns the seconds part and its fractional part with microseconds.
    */
-  def getSecondsWithFraction(microsec: SQLTimestamp, timeZone: TimeZone): Decimal = {
-    val secFrac = localTimestamp(microsec, timeZone) % (MILLIS_PER_MINUTE * MICROS_PER_MILLIS)
-    Decimal(secFrac, 8, 6)
+  def getSecondsWithFraction(microsec: SQLTimestamp, zoneId: ZoneId): Decimal = {
+    Decimal(getMicroseconds(microsec, zoneId), 8, 6)
   }
 
   /**
    * Returns seconds, including fractional parts, multiplied by 1000. The timestamp
    * is expressed in microseconds since the epoch.
    */
-  def getMilliseconds(timestamp: SQLTimestamp, timeZone: TimeZone): Decimal = {
-    Decimal(getMicroseconds(timestamp, timeZone), 8, 3)
+  def getMilliseconds(timestamp: SQLTimestamp, zoneId: ZoneId): Decimal = {
+    Decimal(getMicroseconds(timestamp, zoneId), 8, 3)
   }
 
   /**
    * Returns seconds, including fractional parts, multiplied by 1000000. The timestamp
    * is expressed in microseconds since the epoch.
    */
-  def getMicroseconds(timestamp: SQLTimestamp, timeZone: TimeZone): Int = {
-    Math.floorMod(localTimestamp(timestamp, timeZone), MICROS_PER_SECOND * 60).toInt
+  def getMicroseconds(timestamp: SQLTimestamp, zoneId: ZoneId): Int = {
+    val lt = localTimestamp(timestamp, zoneId)
+    (lt.getLong(ChronoField.MICRO_OF_SECOND) + lt.getSecond * MICROS_PER_SECOND).toInt
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 39b859af47ca9..a514c902f1d1d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -1060,22 +1060,31 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     outstandingTimezonesIds.foreach { timezone =>
       var timestamp = MakeTimestamp(Literal(2019), Literal(8), Literal(10),
         Literal(0), Literal(0), Literal(Decimal(BigDecimal(10.123456789), 8, 6)),
-        Some(Literal(timezone)))
+        Some(Literal(timezone)), Some(timezone))
+      def millis(ts: MakeTimestamp): Milliseconds = Milliseconds(timestamp, Some(timezone))
+      def micros(ts: MakeTimestamp): Microseconds = Microseconds(timestamp, Some(timezone))
 
-      checkEvaluation(Milliseconds(timestamp), Decimal(BigDecimal(10123.457), 8, 3))
-      checkEvaluation(Microseconds(timestamp), 10123457)
+      checkEvaluation(millis(timestamp), Decimal(BigDecimal(10123.457), 8, 3))
+      checkEvaluation(
+        millis(timestamp.copy(year = Literal(10))),
+        Decimal(BigDecimal(10123.457), 8, 3))
+
+      checkEvaluation(micros(timestamp), 10123457)
+      checkEvaluation(
+        micros(timestamp.copy(year = Literal(10))),
+        10123457)
 
       timestamp = timestamp.copy(sec = Literal(Decimal(0.0, 8, 6)))
-      checkEvaluation(Milliseconds(timestamp), Decimal(0, 8, 3))
-      checkEvaluation(Microseconds(timestamp), 0)
+      checkEvaluation(millis(timestamp), Decimal(0, 8, 3))
+      checkEvaluation(micros(timestamp), 0)
 
       timestamp = timestamp.copy(sec = Literal(Decimal(BigDecimal(59.999999), 8, 6)))
-      checkEvaluation(Milliseconds(timestamp), Decimal(BigDecimal(59999.999), 8, 3))
-      checkEvaluation(Microseconds(timestamp), 59999999)
+      checkEvaluation(millis(timestamp), Decimal(BigDecimal(59999.999), 8, 3))
+      checkEvaluation(micros(timestamp), 59999999)
 
       timestamp = timestamp.copy(sec = Literal(Decimal(BigDecimal(60.0), 8, 6)))
-      checkEvaluation(Milliseconds(timestamp), Decimal(0, 8, 3))
-      checkEvaluation(Microseconds(timestamp), 0)
+      checkEvaluation(millis(timestamp), Decimal(0, 8, 3))
+      checkEvaluation(micros(timestamp), 0)
     }
   }
 
@@ -1103,15 +1112,19 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     outstandingTimezonesIds.foreach { timezone =>
       val timestamp = MakeTimestamp(Literal(2019), Literal(8), Literal(10),
         Literal(0), Literal(0), Literal(Decimal(10.123456, 8, 6)),
-        Some(Literal(timezone)))
+        Some(Literal(timezone)), Some(timezone))
+      def secFrac(ts: MakeTimestamp): SecondWithFraction = SecondWithFraction(ts, Some(timezone))
 
-      checkEvaluation(SecondWithFraction(timestamp), Decimal(10.123456, 8, 6))
+      checkEvaluation(secFrac(timestamp), Decimal(10.123456, 8, 6))
       checkEvaluation(
-        SecondWithFraction(timestamp.copy(sec = Literal(Decimal(59000001, 8, 6)))),
+        secFrac(timestamp.copy(sec = Literal(Decimal(59000001, 8, 6)))),
         Decimal(59000001, 8, 6))
       checkEvaluation(
-        SecondWithFraction(timestamp.copy(sec = Literal(Decimal(1, 8, 6)))),
+        secFrac(timestamp.copy(sec = Literal(Decimal(1, 8, 6)))),
         Decimal(0.000001, 8, 6))
+      checkEvaluation(
+        secFrac(timestamp.copy(year = Literal(10))),
+        Decimal(10.123456, 8, 6))
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index 4dfeb85c74f93..8d16007a18f7f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -17,12 +17,10 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId}
+import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset}
 import java.util.TimeZone
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneUTC
-
 /**
  * Helper functions for testing date and time functionality.
  */
@@ -52,8 +50,8 @@ object DateTimeTestUtils {
     }
   }
 
-  def localDateTimeToMicros(localDateTime: LocalDateTime, tz: TimeZone): Long = {
-    val instant = localDateTime.atZone(tz.toZoneId).toInstant
+  def localDateTimeToMicros(localDateTime: LocalDateTime, zoneId: ZoneId): Long = {
+    val instant = localDateTime.atZone(zoneId).toInstant
     DateTimeUtils.instantToMicros(instant)
   }
 
@@ -66,10 +64,10 @@ object DateTimeTestUtils {
       minute: Byte = 0,
       sec: Byte = 0,
       micros: Int = 0,
-      tz: TimeZone = TimeZoneUTC): Long = {
+      zid: ZoneId = ZoneOffset.UTC): Long = {
     val nanos = TimeUnit.MICROSECONDS.toNanos(micros).toInt
     val localDateTime = LocalDateTime.of(year, month, day, hour, minute, sec, nanos)
-    localDateTimeToMicros(localDateTime, tz)
+    localDateTimeToMicros(localDateTime, zid)
   }
 
   // Returns number of days since epoch for the given date
@@ -90,11 +88,11 @@ object DateTimeTestUtils {
       minute: Byte = 0,
       sec: Byte = 0,
       micros: Int = 0,
-      tz: TimeZone = TimeZoneUTC): Long = {
+      zid: ZoneId = ZoneOffset.UTC): Long = {
     val nanos = TimeUnit.MICROSECONDS.toNanos(micros).toInt
-    val localDate = LocalDate.now(tz.toZoneId)
+    val localDate = LocalDate.now(zid)
     val localTime = LocalTime.of(hour, minute, sec, nanos)
     val localDateTime = LocalDateTime.of(localDate, localTime)
-    localDateTimeToMicros(localDateTime, tz)
+    localDateTimeToMicros(localDateTime, zid)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index ff4d8a2457922..6ed18ca1bedab 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -34,7 +34,9 @@ import org.apache.spark.unsafe.types.UTF8String
 
 class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
-  val TimeZonePST = TimeZone.getTimeZone("PST")
+  val zonePST = getZoneId("PST")
+  val zoneGMT = getZoneId("GMT")
+
   private def defaultZoneId = ZoneId.systemDefault()
 
   test("nanoseconds truncation") {
@@ -163,87 +165,88 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       def checkStringToTimestamp(str: String, expected: Option[Long]): Unit = {
         assert(toTimestamp(str, tz.toZoneId) === expected)
       }
+      val zid = tz.toZoneId
 
-      checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, tz = tz)))
-      checkStringToTimestamp("0001", Option(date(1, 1, 1, 0, tz = tz)))
-      checkStringToTimestamp("2015-03", Option(date(2015, 3, 1, tz = tz)))
+      checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, zid = zid)))
+      checkStringToTimestamp("0001", Option(date(1, 1, 1, 0, zid = zid)))
+      checkStringToTimestamp("2015-03", Option(date(2015, 3, 1, zid = zid)))
       Seq("2015-03-18", "2015-03-18 ", " 2015-03-18", " 2015-03-18 ", "2015-03-18T").foreach { s =>
-        checkStringToTimestamp(s, Option(date(2015, 3, 18, tz = tz)))
+        checkStringToTimestamp(s, Option(date(2015, 3, 18, zid = zid)))
       }
 
-      var expected = Option(date(2015, 3, 18, 12, 3, 17, tz = tz))
+      var expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zid))
       checkStringToTimestamp("2015-03-18 12:03:17", expected)
       checkStringToTimestamp("2015-03-18T12:03:17", expected)
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the tz parameter.
-      var timeZone = TimeZone.getTimeZone("GMT-13:53")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      var zoneId = getZoneId("GMT-13:53")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-13:53", expected)
 
-      timeZone = TimeZone.getTimeZone("UTC")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("UTC")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17Z", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT-01:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("GMT-01:00")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17-01:00", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:03")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, tz = timeZone))
+      zoneId = getZoneId("GMT+07:03")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:03", expected)
 
       // tests for the string including milliseconds.
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = tz))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zid))
       checkStringToTimestamp("2015-03-18 12:03:17.123", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123", expected)
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the tz parameter.
-      timeZone = TimeZone.getTimeZone("UTC")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, tz = timeZone))
+      zoneId = getZoneId("UTC")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.456Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17.456Z", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT-01:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = timeZone))
+      zoneId = getZoneId("GMT-01:00")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123-01:00", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123121, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123121, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123121+7:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.12312+7:30", expected)
 
-      expected = Option(time(18, 12, 15, tz = tz))
+      expected = Option(time(18, 12, 15, zid = zid))
       checkStringToTimestamp("18:12:15", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(time(18, 12, 15, 123120, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("T18:12:15.12312+7:30", expected)
 
-      timeZone = TimeZone.getTimeZone("GMT+07:30")
-      expected = Option(time(18, 12, 15, 123120, tz = timeZone))
+      zoneId = getZoneId("GMT+07:30")
+      expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("18:12:15.12312+7:30", expected)
 
-      expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, tz = tz))
+      expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, zid = zid))
       checkStringToTimestamp("2011-05-06 07:08:09.1000", expected)
 
       checkStringToTimestamp("238", None)
@@ -265,8 +268,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       checkStringToTimestamp("1999 08", None)
 
       // Truncating the fractional seconds
-      timeZone = TimeZone.getTimeZone("GMT+00:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, tz = timeZone))
+      zoneId = getZoneId("GMT+00:00")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = zoneId))
       checkStringToTimestamp(
         "2015-03-18T12:03:17.123456789+0:00", expected)
     }
@@ -290,32 +293,38 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("hours") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, TimeZonePST)
-    assert(getHours(input, TimeZonePST) === 13)
-    assert(getHours(input, TimeZoneGMT) === 20)
-    input = date(2015, 12, 8, 2, 7, 9, 0, TimeZonePST)
-    assert(getHours(input, TimeZonePST) === 2)
-    assert(getHours(input, TimeZoneGMT) === 10)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, zonePST)
+    assert(getHours(input, zonePST) === 13)
+    assert(getHours(input, zoneGMT) === 20)
+    input = date(2015, 12, 8, 2, 7, 9, 0, zonePST)
+    assert(getHours(input, zonePST) === 2)
+    assert(getHours(input, zoneGMT) === 10)
+    input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
+    assert(getHours(input, zonePST) === 0)
   }
 
   test("minutes") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, TimeZonePST)
-    assert(getMinutes(input, TimeZonePST) === 2)
-    assert(getMinutes(input, TimeZoneGMT) === 2)
-    assert(getMinutes(input, TimeZone.getTimeZone("Australia/North")) === 32)
-    input = date(2015, 3, 8, 2, 7, 9, 0, TimeZonePST)
-    assert(getMinutes(input, TimeZonePST) === 7)
-    assert(getMinutes(input, TimeZoneGMT) === 7)
-    assert(getMinutes(input, TimeZone.getTimeZone("Australia/North")) === 37)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, zonePST)
+    assert(getMinutes(input, zonePST) === 2)
+    assert(getMinutes(input, zoneGMT) === 2)
+    assert(getMinutes(input, getZoneId("Australia/North")) === 32)
+    input = date(2015, 3, 8, 2, 7, 9, 0, zonePST)
+    assert(getMinutes(input, zonePST) === 7)
+    assert(getMinutes(input, zoneGMT) === 7)
+    assert(getMinutes(input, getZoneId("Australia/North")) === 37)
+    input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
+    assert(getMinutes(input, zonePST) === 0)
   }
 
   test("seconds") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, TimeZonePST)
-    assert(getSeconds(input, TimeZonePST) === 11)
-    assert(getSeconds(input, TimeZoneGMT) === 11)
-    input = date(2015, 3, 8, 2, 7, 9, 0, TimeZonePST)
-    assert(getSeconds(input, TimeZonePST) === 9)
-    assert(getSeconds(input, TimeZoneGMT) === 9)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, zonePST)
+    assert(getSeconds(input, zonePST) === 11)
+    assert(getSeconds(input, zoneGMT) === 11)
+    input = date(2015, 3, 8, 2, 7, 9, 0, zonePST)
+    assert(getSeconds(input, zonePST) === 9)
+    assert(getSeconds(input, zoneGMT) === 9)
+    input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
+    assert(getSeconds(input, zonePST) === 0)
   }
 
   test("hours / minutes / seconds") {
@@ -381,37 +390,37 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     val ts2 = date(2000, 2, 28, 10, 30, 0, 123000)
     assert(timestampAddInterval(ts1, 36, 0, 123000, defaultZoneId) === ts2)
 
-    val ts3 = date(1997, 2, 27, 16, 0, 0, 0, TimeZonePST)
-    val ts4 = date(2000, 2, 27, 16, 0, 0, 123000, TimeZonePST)
-    val ts5 = date(2000, 2, 28, 0, 0, 0, 123000, TimeZoneGMT)
-    assert(timestampAddInterval(ts3, 36, 0, 123000, TimeZonePST.toZoneId) === ts4)
-    assert(timestampAddInterval(ts3, 36, 0, 123000, TimeZoneGMT.toZoneId) === ts5)
+    val ts3 = date(1997, 2, 27, 16, 0, 0, 0, zonePST)
+    val ts4 = date(2000, 2, 27, 16, 0, 0, 123000, zonePST)
+    val ts5 = date(2000, 2, 28, 0, 0, 0, 123000, zoneGMT)
+    assert(timestampAddInterval(ts3, 36, 0, 123000, zonePST) === ts4)
+    assert(timestampAddInterval(ts3, 36, 0, 123000, zoneGMT) === ts5)
   }
 
   test("timestamp add days") {
     // 2019-3-9 is the end of Pacific Standard Time
-    val ts1 = date(2019, 3, 9, 12, 0, 0, 123000, TimeZonePST)
+    val ts1 = date(2019, 3, 9, 12, 0, 0, 123000, zonePST)
     // 2019-3-10 is the start of Pacific Daylight Time
-    val ts2 = date(2019, 3, 10, 12, 0, 0, 123000, TimeZonePST)
-    val ts3 = date(2019, 5, 9, 12, 0, 0, 123000, TimeZonePST)
-    val ts4 = date(2019, 5, 10, 12, 0, 0, 123000, TimeZonePST)
+    val ts2 = date(2019, 3, 10, 12, 0, 0, 123000, zonePST)
+    val ts3 = date(2019, 5, 9, 12, 0, 0, 123000, zonePST)
+    val ts4 = date(2019, 5, 10, 12, 0, 0, 123000, zonePST)
     // 2019-11-2 is the end of Pacific Daylight Time
-    val ts5 = date(2019, 11, 2, 12, 0, 0, 123000, TimeZonePST)
+    val ts5 = date(2019, 11, 2, 12, 0, 0, 123000, zonePST)
     // 2019-11-3 is the start of Pacific Standard Time
-    val ts6 = date(2019, 11, 3, 12, 0, 0, 123000, TimeZonePST)
+    val ts6 = date(2019, 11, 3, 12, 0, 0, 123000, zonePST)
 
     // transit from Pacific Standard Time to Pacific Daylight Time
     assert(timestampAddInterval(
-      ts1, 0, 0, 23 * MICROS_PER_HOUR, TimeZonePST.toZoneId) === ts2)
-    assert(timestampAddInterval(ts1, 0, 1, 0, TimeZonePST.toZoneId) === ts2)
+      ts1, 0, 0, 23 * MICROS_PER_HOUR, zonePST) === ts2)
+    assert(timestampAddInterval(ts1, 0, 1, 0, zonePST) === ts2)
     // just a normal day
     assert(timestampAddInterval(
-      ts3, 0, 0, 24 * MICROS_PER_HOUR, TimeZonePST.toZoneId) === ts4)
-    assert(timestampAddInterval(ts3, 0, 1, 0, TimeZonePST.toZoneId) === ts4)
+      ts3, 0, 0, 24 * MICROS_PER_HOUR, zonePST) === ts4)
+    assert(timestampAddInterval(ts3, 0, 1, 0, zonePST) === ts4)
     // transit from Pacific Daylight Time to Pacific Standard Time
     assert(timestampAddInterval(
-      ts5, 0, 0, 25 * MICROS_PER_HOUR, TimeZonePST.toZoneId) === ts6)
-    assert(timestampAddInterval(ts5, 0, 1, 0, TimeZonePST.toZoneId) === ts6)
+      ts5, 0, 0, 25 * MICROS_PER_HOUR, zonePST) === ts6)
+    assert(timestampAddInterval(ts5, 0, 1, 0, zonePST) === ts6)
   }
 
   test("monthsBetween") {
@@ -428,9 +437,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === 11)
     }
 
-    val date3 = date(2000, 2, 28, 16, tz = TimeZonePST)
-    val date4 = date(1997, 2, 28, 16, tz = TimeZonePST)
-    assert(monthsBetween(date3, date4, true, TimeZonePST.toZoneId) === 36.0)
+    val date3 = date(2000, 2, 28, 16, zid = zonePST)
+    val date4 = date(1997, 2, 28, 16, zid = zonePST)
+    assert(monthsBetween(date3, date4, true, zonePST) === 36.0)
     assert(monthsBetween(date3, date4, true, ZoneOffset.UTC) === 35.90322581)
     assert(monthsBetween(date3, date4, false, ZoneOffset.UTC) === 35.903225806451616)
   }
@@ -575,15 +584,15 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("daysToMillis and millisToDays") {
-    val input = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, 16, tz = TimeZonePST))
-    assert(millisToDays(input, TimeZonePST.toZoneId) === 16800)
+    val input = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, 16, zid = zonePST))
+    assert(millisToDays(input, zonePST) === 16800)
     assert(millisToDays(input, ZoneOffset.UTC) === 16801)
     assert(millisToDays(-1 * MILLIS_PER_DAY + 1, ZoneOffset.UTC) == -1)
 
-    var expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZonePST))
-    assert(daysToMillis(16800, TimeZonePST.toZoneId) === expected)
+    var expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, zid = zonePST))
+    assert(daysToMillis(16800, zonePST) === expected)
 
-    expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZoneGMT))
+    expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, zid = zoneGMT))
     assert(daysToMillis(16800, ZoneOffset.UTC) === expected)
 
     // There are some days are skipped entirely in some timezone, skip them here.
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 729b112de5e75..7d9b147d8d053 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -2,428 +2,428 @@
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    546            572          36         18.3          54.6       1.0X
-cast to timestamp wholestage on                     412            438          16         24.3          41.2       1.3X
+cast to timestamp wholestage off                    408            445          53         24.5          40.8       1.0X
+cast to timestamp wholestage on                     401            453          63         24.9          40.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1240           1295          77          8.1         124.0       1.0X
-year of timestamp wholestage on                    1109           1130          24          9.0         110.9       1.1X
+year of timestamp wholestage off                   1197           1246          69          8.4         119.7       1.0X
+year of timestamp wholestage on                    1111           1123          10          9.0         111.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1572           1574           3          6.4         157.2       1.0X
-quarter of timestamp wholestage on                 1386           1405          18          7.2         138.6       1.1X
+quarter of timestamp wholestage off                1451           1462          16          6.9         145.1       1.0X
+quarter of timestamp wholestage on                 1409           1424          13          7.1         140.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1194           1196           2          8.4         119.4       1.0X
-month of timestamp wholestage on                   1057           1069          12          9.5         105.7       1.1X
+month of timestamp wholestage off                  1106           1109           3          9.0         110.6       1.0X
+month of timestamp wholestage on                   1092           1095           4          9.2         109.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             2070           2071           2          4.8         207.0       1.0X
-weekofyear of timestamp wholestage on              1549           1555           6          6.5         154.9       1.3X
+weekofyear of timestamp wholestage off             1675           1677           3          6.0         167.5       1.0X
+weekofyear of timestamp wholestage on              1616           1625           7          6.2         161.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1173           1186          18          8.5         117.3       1.0X
-day of timestamp wholestage on                     1056           1076          26          9.5         105.6       1.1X
+day of timestamp wholestage off                    1111           1115           6          9.0         111.1       1.0X
+day of timestamp wholestage on                     1090           1098          10          9.2         109.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1207           1211           6          8.3         120.7       1.0X
-dayofyear of timestamp wholestage on               1097           1108           9          9.1         109.7       1.1X
+dayofyear of timestamp wholestage off              1162           1167           7          8.6         116.2       1.0X
+dayofyear of timestamp wholestage on               1136           1142           6          8.8         113.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1184           1190           8          8.4         118.4       1.0X
-dayofmonth of timestamp wholestage on              1053           1060           9          9.5         105.3       1.1X
+dayofmonth of timestamp wholestage off             1102           1107           7          9.1         110.2       1.0X
+dayofmonth of timestamp wholestage on              1092           1111          16          9.2         109.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1343           1362          27          7.4         134.3       1.0X
-dayofweek of timestamp wholestage on               1228           1239           7          8.1         122.8       1.1X
+dayofweek of timestamp wholestage off              1254           1260           9          8.0         125.4       1.0X
+dayofweek of timestamp wholestage on               1264           1269           7          7.9         126.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1276           1278           3          7.8         127.6       1.0X
-weekday of timestamp wholestage on                 1160           1181          22          8.6         116.0       1.1X
+weekday of timestamp wholestage off                1211           1215           5          8.3         121.1       1.0X
+weekday of timestamp wholestage on                 1197           1206          12          8.4         119.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    854            862          12         11.7          85.4       1.0X
-hour of timestamp wholestage on                     741            748           6         13.5          74.1       1.2X
+hour of timestamp wholestage off                    899            904           6         11.1          89.9       1.0X
+hour of timestamp wholestage on                     823            827           3         12.2          82.3       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  853            854           1         11.7          85.3       1.0X
-minute of timestamp wholestage on                   730            737          11         13.7          73.0       1.2X
+minute of timestamp wholestage off                  906            923          24         11.0          90.6       1.0X
+minute of timestamp wholestage on                   821            830          13         12.2          82.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  726            728           2         13.8          72.6       1.0X
-second of timestamp wholestage on                   614            623           9         16.3          61.4       1.2X
+second of timestamp wholestage off                  902            908           7         11.1          90.2       1.0X
+second of timestamp wholestage on                   823            833           9         12.2          82.3       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         369            370           2         27.1          36.9       1.0X
-current_date wholestage on                          277            284           8         36.2          27.7       1.3X
+current_date wholestage off                         300            301           2         33.3          30.0       1.0X
+current_date wholestage on                          309            312           3         32.4          30.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    410            411           1         24.4          41.0       1.0X
-current_timestamp wholestage on                     283            418         259         35.4          28.3       1.5X
+current_timestamp wholestage off                    303            337          47         33.0          30.3       1.0X
+current_timestamp wholestage on                     320            448         160         31.3          32.0       0.9X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         987            992           7         10.1          98.7       1.0X
-cast to date wholestage on                          891            896           4         11.2          89.1       1.1X
+cast to date wholestage off                         910            912           2         11.0          91.0       1.0X
+cast to date wholestage on                          930            937           6         10.8          93.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1179           1179           1          8.5         117.9       1.0X
-last_day wholestage on                             1052           1080          44          9.5         105.2       1.1X
+last_day wholestage off                            1122           1123           1          8.9         112.2       1.0X
+last_day wholestage on                             1100           1109           6          9.1         110.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1073           1081          11          9.3         107.3       1.0X
-next_day wholestage on                              948            955          10         10.5          94.8       1.1X
+next_day wholestage off                             991            992           2         10.1          99.1       1.0X
+next_day wholestage on                              981            984           4         10.2          98.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1006           1009           5          9.9         100.6       1.0X
-date_add wholestage on                              867            870           4         11.5          86.7       1.2X
+date_add wholestage off                             927            927           1         10.8          92.7       1.0X
+date_add wholestage on                              908            915          12         11.0          90.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             980            988          11         10.2          98.0       1.0X
-date_sub wholestage on                              866            873           9         11.6          86.6       1.1X
+date_sub wholestage off                             921            921           0         10.9          92.1       1.0X
+date_sub wholestage on                              907            910           4         11.0          90.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1329           1332           4          7.5         132.9       1.0X
-add_months wholestage on                           1199           1206           8          8.3         119.9       1.1X
+add_months wholestage off                          1255           1257           3          8.0         125.5       1.0X
+add_months wholestage on                           1242           1259          13          8.1         124.2       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         4724           4736          18          2.1         472.4       1.0X
-format date wholestage on                          4550           4574          26          2.2         455.0       1.0X
+format date wholestage off                         4912           4968          79          2.0         491.2       1.0X
+format date wholestage on                          4734           4757          16          2.1         473.4       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7171           7183          17          1.4         717.1       1.0X
-from_unixtime wholestage on                        7114           7141          20          1.4         711.4       1.0X
+from_unixtime wholestage off                       8692           8701          13          1.2         869.2       1.0X
+from_unixtime wholestage on                        8717           8730          14          1.1         871.7       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1498           1504           8          6.7         149.8       1.0X
-from_utc_timestamp wholestage on                   1399           1405           5          7.1         139.9       1.1X
+from_utc_timestamp wholestage off                  1339           1348          12          7.5         133.9       1.0X
+from_utc_timestamp wholestage on                   1398           1404           6          7.2         139.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1505           1507           2          6.6         150.5       1.0X
-to_utc_timestamp wholestage on                     1396           1401           5          7.2         139.6       1.1X
+to_utc_timestamp wholestage off                    1936           1951          21          5.2         193.6       1.0X
+to_utc_timestamp wholestage on                     1783           1792           8          5.6         178.3       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        423            428           7         23.6          42.3       1.0X
-cast interval wholestage on                         300            302           2         33.3          30.0       1.4X
+cast interval wholestage off                        350            353           4         28.6          35.0       1.0X
+cast interval wholestage on                         348            355           4         28.7          34.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1626           1630           6          6.1         162.6       1.0X
-datediff wholestage on                             1467           1471           3          6.8         146.7       1.1X
+datediff wholestage off                            1679           1680           2          6.0         167.9       1.0X
+datediff wholestage on                             1674           1687          11          6.0         167.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      1988           1992           5          5.0         198.8       1.0X
-months_between wholestage on                       1812           1834          24          5.5         181.2       1.1X
+months_between wholestage off                      4116           4120           5          2.4         411.6       1.0X
+months_between wholestage on                       4092           4140          90          2.4         409.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2277           2334          80          0.4        2277.1       1.0X
-window wholestage on                              48996          49048          67          0.0       48996.0       0.0X
+window wholestage off                              2236           2396         227          0.4        2235.9       1.0X
+window wholestage on                              47208          47298         171          0.0       47208.3       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                      867            870           6         11.5          86.7       1.0X
-date_trunc YEAR wholestage on                       815            819           6         12.3          81.5       1.1X
+date_trunc YEAR wholestage off                     1939           1947          11          5.2         193.9       1.0X
+date_trunc YEAR wholestage on                      1890           1899           6          5.3         189.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                      866            875          13         11.5          86.6       1.0X
-date_trunc YYYY wholestage on                       811            813           2         12.3          81.1       1.1X
+date_trunc YYYY wholestage off                     1947           1950           5          5.1         194.7       1.0X
+date_trunc YYYY wholestage on                      1903           1909           7          5.3         190.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                        864            867           4         11.6          86.4       1.0X
-date_trunc YY wholestage on                         812            824          10         12.3          81.2       1.1X
+date_trunc YY wholestage off                       1941           1952          15          5.2         194.1       1.0X
+date_trunc YY wholestage on                        1895           1907          10          5.3         189.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                       881            884           4         11.3          88.1       1.0X
-date_trunc MON wholestage on                        820            826           7         12.2          82.0       1.1X
+date_trunc MON wholestage off                      1912           1934          30          5.2         191.2       1.0X
+date_trunc MON wholestage on                       1899           1923          20          5.3         189.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                     880            881           2         11.4          88.0       1.0X
-date_trunc MONTH wholestage on                      819            822           4         12.2          81.9       1.1X
+date_trunc MONTH wholestage off                    1923           1931          11          5.2         192.3       1.0X
+date_trunc MONTH wholestage on                     1897           1908          10          5.3         189.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                        889            904          21         11.2          88.9       1.0X
-date_trunc MM wholestage on                         818            828           8         12.2          81.8       1.1X
+date_trunc MM wholestage off                       1923           1924           2          5.2         192.3       1.0X
+date_trunc MM wholestage on                        1902           1911          10          5.3         190.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                       590            593           4         16.9          59.0       1.0X
-date_trunc DAY wholestage on                        510            514           4         19.6          51.0       1.2X
+date_trunc DAY wholestage off                      1713           1717           6          5.8         171.3       1.0X
+date_trunc DAY wholestage on                       1693           1699           4          5.9         169.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                        596            604          11         16.8          59.6       1.0X
-date_trunc DD wholestage on                         511            519           9         19.6          51.1       1.2X
+date_trunc DD wholestage off                       1711           1712           1          5.8         171.1       1.0X
+date_trunc DD wholestage on                        1691           1697          12          5.9         169.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                      586            592           9         17.1          58.6       1.0X
-date_trunc HOUR wholestage on                       507            513           5         19.7          50.7       1.2X
+date_trunc HOUR wholestage off                     1715           1715           0          5.8         171.5       1.0X
+date_trunc HOUR wholestage on                      1699           1705           4          5.9         169.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    561            562           1         17.8          56.1       1.0X
-date_trunc MINUTE wholestage on                     480            485           4         20.8          48.0       1.2X
+date_trunc MINUTE wholestage off                    520            524           7         19.2          52.0       1.0X
+date_trunc MINUTE wholestage on                     500            505           6         20.0          50.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    561            561           1         17.8          56.1       1.0X
-date_trunc SECOND wholestage on                     479            480           2         20.9          47.9       1.2X
+date_trunc SECOND wholestage off                    501            502           1         20.0          50.1       1.0X
+date_trunc SECOND wholestage on                     497            501           3         20.1          49.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                      725            727           2         13.8          72.5       1.0X
-date_trunc WEEK wholestage on                       674            684          11         14.8          67.4       1.1X
+date_trunc WEEK wholestage off                     1839           1840           1          5.4         183.9       1.0X
+date_trunc WEEK wholestage on                      1820           1827           8          5.5         182.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1653           1659          10          6.1         165.3       1.0X
-date_trunc QUARTER wholestage on                   1588           1601          12          6.3         158.8       1.0X
+date_trunc QUARTER wholestage off                  2624           2624           0          3.8         262.4       1.0X
+date_trunc QUARTER wholestage on                   2573           2583           7          3.9         257.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           391            393           2         25.6          39.1       1.0X
-trunc year wholestage on                            312            316           4         32.1          31.2       1.3X
+trunc year wholestage off                           315            317           3         31.8          31.5       1.0X
+trunc year wholestage on                            314            328          16         31.9          31.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           390            394           5         25.6          39.0       1.0X
-trunc yyyy wholestage on                            316            319           4         31.6          31.6       1.2X
+trunc yyyy wholestage off                           314            317           4         31.8          31.4       1.0X
+trunc yyyy wholestage on                            312            315           3         32.1          31.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             387            390           5         25.8          38.7       1.0X
-trunc yy wholestage on                              313            316           3         31.9          31.3       1.2X
+trunc yy wholestage off                             314            315           0         31.8          31.4       1.0X
+trunc yy wholestage on                              312            317           6         32.0          31.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            388            395          11         25.8          38.8       1.0X
-trunc mon wholestage on                             314            316           2         31.8          31.4       1.2X
+trunc mon wholestage off                            314            315           1         31.8          31.4       1.0X
+trunc mon wholestage on                             310            314           4         32.2          31.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          388            390           4         25.8          38.8       1.0X
-trunc month wholestage on                           315            318           2         31.7          31.5       1.2X
+trunc month wholestage off                          313            314           2         32.0          31.3       1.0X
+trunc month wholestage on                           312            316           6         32.1          31.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             384            388           4         26.0          38.4       1.0X
-trunc mm wholestage on                              314            321           9         31.9          31.4       1.2X
+trunc mm wholestage off                             312            313           1         32.0          31.2       1.0X
+trunc mm wholestage on                              309            312           3         32.3          30.9       1.0X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     178            191          18          5.6         178.5       1.0X
-to timestamp str wholestage on                      157            160           2          6.4         156.6       1.1X
+to timestamp str wholestage off                     170            173           5          5.9         170.0       1.0X
+to timestamp str wholestage on                      156            161           5          6.4         155.9       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1790           1795           7          0.6        1790.0       1.0X
-to_timestamp wholestage on                         1813           1820          10          0.6        1813.1       1.0X
+to_timestamp wholestage off                        1741           1746           8          0.6        1740.7       1.0X
+to_timestamp wholestage on                         1887           1896          11          0.5        1886.6       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1836           1837           1          0.5        1835.8       1.0X
-to_unix_timestamp wholestage on                    1786           1791           3          0.6        1785.8       1.0X
+to_unix_timestamp wholestage off                   1928           1934           9          0.5        1927.7       1.0X
+to_unix_timestamp wholestage on                    1857           1862           3          0.5        1857.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          169            169           1          5.9         168.8       1.0X
-to date str wholestage on                           151            153           2          6.6         151.2       1.1X
+to date str wholestage off                          161            162           0          6.2         161.5       1.0X
+to date str wholestage on                           151            156           3          6.6         151.4       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             2504           2512          10          0.4        2504.4       1.0X
-to_date wholestage on                              2522           2536          19          0.4        2522.3       1.0X
+to_date wholestage off                             2513           2519           8          0.4        2513.1       1.0X
+to_date wholestage on                              2484           2500          15          0.4        2484.2       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             346            367          35         14.5          69.2       1.0X
-Collect longs                                      2139           2329         289          2.3         427.7       0.2X
-Collect timestamps                                 1883           2086         303          2.7         376.5       0.2X
+From java.sql.Timestamp                             340            347           8         14.7          68.0       1.0X
+Collect longs                                      1170           1217          42          4.3         234.1       0.3X
+Collect timestamps                                 1771           2267         836          2.8         354.1       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 0c30534dd71ca..3567a60d1b0b8 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -2,428 +2,428 @@
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    425            447          30         23.5          42.5       1.0X
-cast to timestamp wholestage on                     368            401          29         27.2          36.8       1.2X
+cast to timestamp wholestage off                    447            462          21         22.4          44.7       1.0X
+cast to timestamp wholestage on                     390            426          54         25.7          39.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1158           1215          80          8.6         115.8       1.0X
-year of timestamp wholestage on                    1158           1179          31          8.6         115.8       1.0X
+year of timestamp wholestage off                   1189           1285         135          8.4         118.9       1.0X
+year of timestamp wholestage on                    1146           1156           9          8.7         114.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1285           1295          15          7.8         128.5       1.0X
-quarter of timestamp wholestage on                 1243           1257          11          8.0         124.3       1.0X
+quarter of timestamp wholestage off                1290           1293           4          7.8         129.0       1.0X
+quarter of timestamp wholestage on                 1237           1251          13          8.1         123.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1076           1082           8          9.3         107.6       1.0X
-month of timestamp wholestage on                   1088           1098           9          9.2         108.8       1.0X
+month of timestamp wholestage off                  1096           1101           7          9.1         109.6       1.0X
+month of timestamp wholestage on                   1088           1095           7          9.2         108.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1649           1659          14          6.1         164.9       1.0X
-weekofyear of timestamp wholestage on              1648           1656           8          6.1         164.8       1.0X
+weekofyear of timestamp wholestage off             1635           1636           1          6.1         163.5       1.0X
+weekofyear of timestamp wholestage on              1711           1714           4          5.8         171.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1083           1084           3          9.2         108.3       1.0X
-day of timestamp wholestage on                     1082           1089          13          9.2         108.2       1.0X
+day of timestamp wholestage off                    1094           1108          20          9.1         109.4       1.0X
+day of timestamp wholestage on                     1083           1092           8          9.2         108.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1102           1103           1          9.1         110.2       1.0X
-dayofyear of timestamp wholestage on               1123           1138          14          8.9         112.3       1.0X
+dayofyear of timestamp wholestage off              1145           1145           1          8.7         114.5       1.0X
+dayofyear of timestamp wholestage on               1131           1141          12          8.8         113.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1068           1073           7          9.4         106.8       1.0X
-dayofmonth of timestamp wholestage on              1082           1095          13          9.2         108.2       1.0X
+dayofmonth of timestamp wholestage off             1112           1124          17          9.0         111.2       1.0X
+dayofmonth of timestamp wholestage on              1082           1096          14          9.2         108.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1265           1294          41          7.9         126.5       1.0X
-dayofweek of timestamp wholestage on               1253           1262          11          8.0         125.3       1.0X
+dayofweek of timestamp wholestage off              1254           1255           2          8.0         125.4       1.0X
+dayofweek of timestamp wholestage on               1250           1260          10          8.0         125.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1189           1191           3          8.4         118.9       1.0X
-weekday of timestamp wholestage on                 1193           1199           6          8.4         119.3       1.0X
+weekday of timestamp wholestage off                1190           1196           8          8.4         119.0       1.0X
+weekday of timestamp wholestage on                 1204           1218          11          8.3         120.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    366            368           3         27.3          36.6       1.0X
-hour of timestamp wholestage on                     360            364           6         27.8          36.0       1.0X
+hour of timestamp wholestage off                    859            862           4         11.6          85.9       1.0X
+hour of timestamp wholestage on                     825            827           2         12.1          82.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  348            350           2         28.7          34.8       1.0X
-minute of timestamp wholestage on                   355            361           9         28.1          35.5       1.0X
+minute of timestamp wholestage off                  841            851          15         11.9          84.1       1.0X
+minute of timestamp wholestage on                   817            824           6         12.2          81.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  347            352           7         28.8          34.7       1.0X
-second of timestamp wholestage on                   351            359          10         28.5          35.1       1.0X
+second of timestamp wholestage off                  901            905           5         11.1          90.1       1.0X
+second of timestamp wholestage on                   830            844          12         12.1          83.0       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         284            287           4         35.2          28.4       1.0X
-current_date wholestage on                          312            318           6         32.1          31.2       0.9X
+current_date wholestage off                         290            292           2         34.4          29.0       1.0X
+current_date wholestage on                          301            309          10         33.3          30.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    291            292           2         34.4          29.1       1.0X
-current_timestamp wholestage on                     297            333          40         33.6          29.7       1.0X
+current_timestamp wholestage off                    300            301           0         33.3          30.0       1.0X
+current_timestamp wholestage on                     316            348          36         31.6          31.6       0.9X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         903            903           1         11.1          90.3       1.0X
-cast to date wholestage on                          897            900           7         11.2          89.7       1.0X
+cast to date wholestage off                         964            973          13         10.4          96.4       1.0X
+cast to date wholestage on                          900            905           4         11.1          90.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1082           1082           1          9.2         108.2       1.0X
-last_day wholestage on                             1107           1118          16          9.0         110.7       1.0X
+last_day wholestage off                            1125           1138          19          8.9         112.5       1.0X
+last_day wholestage on                             1111           1122          12          9.0         111.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             968            974           8         10.3          96.8       1.0X
-next_day wholestage on                              958            959           1         10.4          95.8       1.0X
+next_day wholestage off                             970            982          16         10.3          97.0       1.0X
+next_day wholestage on                              955            958           4         10.5          95.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             894            895           1         11.2          89.4       1.0X
-date_add wholestage on                              882            890           9         11.3          88.2       1.0X
+date_add wholestage off                             894            914          29         11.2          89.4       1.0X
+date_add wholestage on                              880            884           4         11.4          88.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             892            896           6         11.2          89.2       1.0X
-date_sub wholestage on                              881            888           7         11.3          88.1       1.0X
+date_sub wholestage off                             901            901           1         11.1          90.1       1.0X
+date_sub wholestage on                              883            892          16         11.3          88.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1221           1223           3          8.2         122.1       1.0X
-add_months wholestage on                           1212           1217           5          8.2         121.2       1.0X
+add_months wholestage off                          1212           1213           0          8.2         121.2       1.0X
+add_months wholestage on                           1212           1219          11          8.3         121.2       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         4989           5009          29          2.0         498.9       1.0X
-format date wholestage on                          5037           5055          26          2.0         503.7       1.0X
+format date wholestage off                         4973           5069         136          2.0         497.3       1.0X
+format date wholestage on                          5061           5075          18          2.0         506.1       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       9157           9164          10          1.1         915.7       1.0X
-from_unixtime wholestage on                        9101           9120          16          1.1         910.1       1.0X
+from_unixtime wholestage off                       8827           8835          11          1.1         882.7       1.0X
+from_unixtime wholestage on                        8840           8845           5          1.1         884.0       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   732            739          10         13.7          73.2       1.0X
-from_utc_timestamp wholestage on                    767            776           8         13.0          76.7       1.0X
+from_utc_timestamp wholestage off                  1143           1145           2          8.7         114.3       1.0X
+from_utc_timestamp wholestage on                   1174           1187          12          8.5         117.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                     802            805           3         12.5          80.2       1.0X
-to_utc_timestamp wholestage on                      776            781           5         12.9          77.6       1.0X
+to_utc_timestamp wholestage off                    1721           1725           7          5.8         172.1       1.0X
+to_utc_timestamp wholestage on                     1605           1613           5          6.2         160.5       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        328            330           3         30.5          32.8       1.0X
-cast interval wholestage on                         319            326           7         31.3          31.9       1.0X
+cast interval wholestage off                        342            345           5         29.3          34.2       1.0X
+cast interval wholestage on                         329            336          13         30.4          32.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1762           1764           3          5.7         176.2       1.0X
-datediff wholestage on                             1495           1502           7          6.7         149.5       1.2X
+datediff wholestage off                            1643           1645           3          6.1         164.3       1.0X
+datediff wholestage on                             1645           1657           8          6.1         164.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      1338           1339           1          7.5         133.8       1.0X
-months_between wholestage on                       1334           1339           5          7.5         133.4       1.0X
+months_between wholestage off                      3369           3375           8          3.0         336.9       1.0X
+months_between wholestage on                       3356           3362           5          3.0         335.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2023           2094         100          0.5        2023.2       1.0X
-window wholestage on                              43505          43551          33          0.0       43504.8       0.0X
+window wholestage off                              2017           2029          18          0.5        2016.8       1.0X
+window wholestage on                              44825          44909          56          0.0       44824.9       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                      660            661           1         15.1          66.0       1.0X
-date_trunc YEAR wholestage on                       589            599           7         17.0          58.9       1.1X
+date_trunc YEAR wholestage off                     1628           1629           0          6.1         162.8       1.0X
+date_trunc YEAR wholestage on                      1589           1599           9          6.3         158.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                      656            657           1         15.2          65.6       1.0X
-date_trunc YYYY wholestage on                       593            604          16         16.9          59.3       1.1X
+date_trunc YYYY wholestage off                     1634           1646          17          6.1         163.4       1.0X
+date_trunc YYYY wholestage on                      1596           1606          13          6.3         159.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                        666            669           4         15.0          66.6       1.0X
-date_trunc YY wholestage on                         591            603          19         16.9          59.1       1.1X
+date_trunc YY wholestage off                       1627           1627           0          6.1         162.7       1.0X
+date_trunc YY wholestage on                        1590           1598          10          6.3         159.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                       592            592           1         16.9          59.2       1.0X
-date_trunc MON wholestage on                        569            580           8         17.6          56.9       1.0X
+date_trunc MON wholestage off                      1633           1634           1          6.1         163.3       1.0X
+date_trunc MON wholestage on                       1602           1607           5          6.2         160.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                     593            594           2         16.9          59.3       1.0X
-date_trunc MONTH wholestage on                      575            579           4         17.4          57.5       1.0X
+date_trunc MONTH wholestage off                    1637           1644          11          6.1         163.7       1.0X
+date_trunc MONTH wholestage on                     1601           1609           5          6.2         160.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                        589            590           2         17.0          58.9       1.0X
-date_trunc MM wholestage on                         569            575           4         17.6          56.9       1.0X
+date_trunc MM wholestage off                       1639           1653          20          6.1         163.9       1.0X
+date_trunc MM wholestage on                        1602           1606           4          6.2         160.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                       438            442           5         22.8          43.8       1.0X
-date_trunc DAY wholestage on                        346            350           4         28.9          34.6       1.3X
+date_trunc DAY wholestage off                      1635           1643          11          6.1         163.5       1.0X
+date_trunc DAY wholestage on                       1511           1517           6          6.6         151.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                        438            439           2         22.8          43.8       1.0X
-date_trunc DD wholestage on                         347            354           7         28.8          34.7       1.3X
+date_trunc DD wholestage off                       1640           1646           8          6.1         164.0       1.0X
+date_trunc DD wholestage on                        1516           1519           4          6.6         151.6       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                      384            386           2         26.0          38.4       1.0X
-date_trunc HOUR wholestage on                       357            365           6         28.0          35.7       1.1X
+date_trunc HOUR wholestage off                     1533           1535           4          6.5         153.3       1.0X
+date_trunc HOUR wholestage on                      1532           1538           7          6.5         153.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    373            375           3         26.8          37.3       1.0X
-date_trunc MINUTE wholestage on                     327            331           5         30.6          32.7       1.1X
+date_trunc MINUTE wholestage off                    381            382           1         26.2          38.1       1.0X
+date_trunc MINUTE wholestage on                     337            343           9         29.7          33.7       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    361            363           3         27.7          36.1       1.0X
-date_trunc SECOND wholestage on                     335            341           8         29.9          33.5       1.1X
+date_trunc SECOND wholestage off                    376            376           1         26.6          37.6       1.0X
+date_trunc SECOND wholestage on                     341            344           6         29.4          34.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                      515            516           2         19.4          51.5       1.0X
-date_trunc WEEK wholestage on                       455            459           4         22.0          45.5       1.1X
+date_trunc WEEK wholestage off                     1556           1556           0          6.4         155.6       1.0X
+date_trunc WEEK wholestage on                      1498           1507           9          6.7         149.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1337           1341           6          7.5         133.7       1.0X
-date_trunc QUARTER wholestage on                   1328           1334          10          7.5         132.8       1.0X
+date_trunc QUARTER wholestage off                  2340           2346           9          4.3         234.0       1.0X
+date_trunc QUARTER wholestage on                   2309           2315           4          4.3         230.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           318            328          14         31.4          31.8       1.0X
-trunc year wholestage on                            297            308          17         33.6          29.7       1.1X
+trunc year wholestage off                           323            323           0         31.0          32.3       1.0X
+trunc year wholestage on                            304            307           4         32.9          30.4       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           318            319           2         31.5          31.8       1.0X
-trunc yyyy wholestage on                            296            302          10         33.8          29.6       1.1X
+trunc yyyy wholestage off                           323            324           2         31.0          32.3       1.0X
+trunc yyyy wholestage on                            303            313          12         33.0          30.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             321            345          35         31.2          32.1       1.0X
-trunc yy wholestage on                              297            319          45         33.6          29.7       1.1X
+trunc yy wholestage off                             325            327           2         30.8          32.5       1.0X
+trunc yy wholestage on                              304            308           3         32.9          30.4       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            318            318           0         31.5          31.8       1.0X
-trunc mon wholestage on                             299            306           6         33.4          29.9       1.1X
+trunc mon wholestage off                            327            351          34         30.5          32.7       1.0X
+trunc mon wholestage on                             301            302           2         33.3          30.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          316            318           3         31.6          31.6       1.0X
-trunc month wholestage on                           296            301           7         33.8          29.6       1.1X
+trunc month wholestage off                          319            319           1         31.4          31.9       1.0X
+trunc month wholestage on                           301            305           4         33.2          30.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             316            321           8         31.7          31.6       1.0X
-trunc mm wholestage on                              295            302           8         33.9          29.5       1.1X
+trunc mm wholestage off                             319            320           1         31.3          31.9       1.0X
+trunc mm wholestage on                              300            307           7         33.3          30.0       1.1X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     219            220           1          4.6         219.3       1.0X
-to timestamp str wholestage on                      212            214           2          4.7         212.3       1.0X
+to timestamp str wholestage off                     218            219           2          4.6         217.7       1.0X
+to timestamp str wholestage on                      212            217           5          4.7         212.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1852           1852           1          0.5        1851.9       1.0X
-to_timestamp wholestage on                         1862           1869           9          0.5        1861.6       1.0X
+to_timestamp wholestage off                        2063           2066           5          0.5        2062.5       1.0X
+to_timestamp wholestage on                         2000           2014           8          0.5        2000.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1839           1842           4          0.5        1839.1       1.0X
-to_unix_timestamp wholestage on                    1861           1866           4          0.5        1861.2       1.0X
+to_unix_timestamp wholestage off                   2038           2040           4          0.5        2037.7       1.0X
+to_unix_timestamp wholestage on                    2021           2029          11          0.5        2020.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          222            228           9          4.5         221.6       1.0X
-to date str wholestage on                           210            211           2          4.8         209.5       1.1X
+to date str wholestage off                          209            215           8          4.8         209.0       1.0X
+to date str wholestage on                           203            205           2          4.9         203.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             2386           2392           8          0.4        2386.3       1.0X
-to_date wholestage on                              2438           2457          18          0.4        2437.7       1.0X
+to_date wholestage off                             2191           2201          14          0.5        2191.3       1.0X
+to_date wholestage on                              2124           2137          16          0.5        2124.4       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             287            291           7         17.4          57.3       1.0X
-Collect longs                                      1903           2672         694          2.6         380.6       0.2X
-Collect timestamps                                 1544           1644          89          3.2         308.8       0.2X
+From java.sql.Timestamp                             284            289           5         17.6          56.9       1.0X
+Collect longs                                      1413           2057         930          3.5         282.6       0.2X
+Collect timestamps                                 1605           1712          93          3.1         321.0       0.2X
 
 

From 7c09b57355724c3330bab8e05c1c096ee487278d Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Mon, 17 Feb 2020 20:41:56 +0900
Subject: [PATCH 0091/1280] [SPARK-28869][DOCS][FOLLOWUP] Add direct
 relationship between configs for rolling event log

### What changes were proposed in this pull request?

This patch addresses the post-hoc review comment linked here - https://github.com/apache/spark/pull/25670#discussion_r373304076

### Why are the changes needed?

We would like to explicitly document the direct relationship before we finish up structuring of configurations.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A

Closes #27576 from HeartSaVioR/SPARK-28869-FOLLOWUP-doc.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 446b2d2653a8ae0c9e35799eb4fc1d9f4dcd7991)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/internal/config/package.scala     | 5 +++--
 docs/configuration.md                                        | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 02acb6b530737..652db2bdf90a9 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -191,14 +191,15 @@ package object config {
 
   private[spark] val EVENT_LOG_ENABLE_ROLLING =
     ConfigBuilder("spark.eventLog.rolling.enabled")
-      .doc("Whether rolling over event log files is enabled.  If set to true, it cuts down " +
+      .doc("Whether rolling over event log files is enabled. If set to true, it cuts down " +
         "each event log file to the configured size.")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ROLLING_MAX_FILE_SIZE =
     ConfigBuilder("spark.eventLog.rolling.maxFileSize")
-      .doc("The max size of event log file to be rolled over.")
+      .doc(s"When ${EVENT_LOG_ENABLE_ROLLING.key}=true, specifies the max size of event log file" +
+        " to be rolled over.")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
         "configured to be at least 10 MiB.")
diff --git a/docs/configuration.md b/docs/configuration.md
index 9c9cf84151b20..6aafc9110652b 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1033,7 +1033,7 @@ Apart from these, the following properties are also available, and may be useful
   <td><code>spark.eventLog.rolling.maxFileSize</code></td>
   <td>128m</td>
   <td>
-    The max size of event log file before it's rolled over.
+    When <code>spark.eventLog.rolling.enabled=true</code>, specifies the max size of event log file before it's rolled over.
   </td>
 </tr>
 <tr>

From cea5cbcc88d8e0109aa0967eee9fda157526d583 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Mon, 17 Feb 2020 20:45:24 +0900
Subject: [PATCH 0092/1280] [SPARK-30827][DOCS] Document direct relationship
 among configurations in "spark.history.*" namespace

### What changes were proposed in this pull request?

This patch adds direct relationship among configurations under "spark.history" namespace.

### Why are the changes needed?

Refer the discussion thread: https://lists.apache.org/thread.html/r43c4e57cace116aca1f0f099e8a577cf202859e3671a04077867b84a%40%3Cdev.spark.apache.org%3E

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Locally ran jekyll and confirmed. Screenshots for the modified spots:

<img width="1159" alt="Screen Shot 2020-02-15 at 8 20 14 PM" src="https://user-images.githubusercontent.com/1317309/74587003-d5922b00-5030-11ea-954b-ee37fc08470a.png">
<img width="1158" alt="Screen Shot 2020-02-15 at 8 20 44 PM" src="https://user-images.githubusercontent.com/1317309/74587005-d62ac180-5030-11ea-98fc-98b1c9d83ff4.png">
<img width="1149" alt="Screen Shot 2020-02-15 at 8 19 56 PM" src="https://user-images.githubusercontent.com/1317309/74587002-d1660d80-5030-11ea-84b5-dec3d7f5c97c.png">

Closes #27575 from HeartSaVioR/SPARK-30827.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 5445fe92887d04513d77af0890db572266136220)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/internal/config/History.scala |  2 +-
 docs/monitoring.md                             | 18 ++++++++----------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 14fb5ff075472..8f99908507ceb 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -162,7 +162,7 @@ private[spark] object History {
   val APPLY_CUSTOM_EXECUTOR_LOG_URL_TO_INCOMPLETE_APP =
     ConfigBuilder("spark.history.custom.executor.log.url.applyIncompleteApplication")
       .doc("Whether to apply custom executor log url, as specified by " +
-        "`spark.history.custom.executor.log.url`, to incomplete application as well. " +
+        s"${CUSTOM_EXECUTOR_LOG_URL.key}, to incomplete application as well. " +
         "Even if this is true, this still only affects the behavior of the history server, " +
         "not running spark applications.")
       .booleanConf
diff --git a/docs/monitoring.md b/docs/monitoring.md
index fc95f6a8d3191..3061c6761cdcf 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -159,23 +159,21 @@ Security options for the Spark History Server are covered more detail in the
     <td>false</td>
     <td>
       Indicates whether the history server should use kerberos to login. This is required
-      if the history server is accessing HDFS files on a secure Hadoop cluster. If this is
-      true, it uses the configs <code>spark.history.kerberos.principal</code> and
-      <code>spark.history.kerberos.keytab</code>.
+      if the history server is accessing HDFS files on a secure Hadoop cluster.
     </td>
   </tr>
   <tr>
     <td>spark.history.kerberos.principal</td>
     <td>(none)</td>
     <td>
-      Kerberos principal name for the History Server.
+      When <code>spark.history.kerberos.enabled=true</code>, specifies kerberos principal name for the History Server.
     </td>
   </tr>
   <tr>
     <td>spark.history.kerberos.keytab</td>
     <td>(none)</td>
     <td>
-      Location of the kerberos keytab file for the History Server.
+      When <code>spark.history.kerberos.enabled=true</code>, specifies location of the kerberos keytab file for the History Server.
     </td>
   </tr>
   <tr>
@@ -189,7 +187,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>spark.history.fs.cleaner.interval</td>
     <td>1d</td>
     <td>
-      How often the filesystem job history cleaner checks for files to delete.
+      When <code>spark.history.fs.cleaner.enabled=true</code>, specifies how often the filesystem job history cleaner checks for files to delete.
       Files are deleted if at least one of two conditions holds.
       First, they're deleted if they're older than <code>spark.history.fs.cleaner.maxAge</code>.
       They are also deleted if the number of files is more than
@@ -201,14 +199,14 @@ Security options for the Spark History Server are covered more detail in the
     <td>spark.history.fs.cleaner.maxAge</td>
     <td>7d</td>
     <td>
-      Job history files older than this will be deleted when the filesystem history cleaner runs.
+      When <code>spark.history.fs.cleaner.enabled=true</code>, job history files older than this will be deleted when the filesystem history cleaner runs.
     </td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.maxNum</td>
     <td>Int.MaxValue</td>
     <td>
-      The maximum number of files in the event log directory.
+      When <code>spark.history.fs.cleaner.enabled=true</code>, specifies the maximum number of files in the event log directory.
       Spark tries to clean up the completed attempt logs to maintain the log directory under this limit.
       This should be smaller than the underlying file system limit like
       `dfs.namenode.fs-limits.max-directory-items` in HDFS.
@@ -242,7 +240,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>spark.history.fs.driverlog.cleaner.interval</td>
     <td><code>spark.history.fs.cleaner.interval</code></td>
     <td>
-      How often the filesystem driver log cleaner checks for files to delete.
+      When <code>spark.history.fs.driverlog.cleaner.enabled=true</code>, specifies how often the filesystem driver log cleaner checks for files to delete.
       Files are only deleted if they are older than <code>spark.history.fs.driverlog.cleaner.maxAge</code>
     </td>
   </tr>
@@ -250,7 +248,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>spark.history.fs.driverlog.cleaner.maxAge</td>
     <td><code>spark.history.fs.cleaner.maxAge</code></td>
     <td>
-      Driver log files older than this will be deleted when the driver log cleaner runs.
+      When <code>spark.history.fs.driverlog.cleaner.enabled=true</code>, driver log files older than this will be deleted when the driver log cleaner runs.
     </td>
   </tr>
   <tr>

From 33329caa81827a245b84158b13234b88a4746e56 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Mon, 17 Feb 2020 22:06:58 +0800
Subject: [PATCH 0093/1280] [SPARK-25829][SQL] Add config
 `spark.sql.legacy.allowDuplicatedMapKeys` and change the default behavior

### What changes were proposed in this pull request?
This is a follow-up for #23124, add a new config `spark.sql.legacy.allowDuplicatedMapKeys` to control the behavior of removing duplicated map keys in build-in functions. With the default value `false`, Spark will throw a RuntimeException while duplicated keys are found.

### Why are the changes needed?
Prevent silent behavior changes.

### Does this PR introduce any user-facing change?
Yes, new config added and the default behavior for duplicated map keys changed to RuntimeException thrown.

### How was this patch tested?
Modify existing UT.

Closes #27478 from xuanyuanking/SPARK-25892-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ab186e36597e16d48f8eb16d57e7a39b6b67d4e4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |   2 +-
 python/pyspark/sql/functions.py               |   4 +-
 .../expressions/collectionOperations.scala    |   4 +-
 .../catalyst/util/ArrayBasedMapBuilder.scala  |  10 ++
 .../apache/spark/sql/internal/SQLConf.scala   |  10 ++
 .../CollectionExpressionsSuite.scala          |  18 ++-
 .../expressions/ComplexTypeSuite.scala        |  35 +++---
 .../HigherOrderFunctionsSuite.scala           |   6 +-
 .../util/ArrayBasedMapBuilderSuite.scala      | 111 ++++++++++--------
 .../spark/sql/DataFrameFunctionsSuite.scala   |  16 ++-
 10 files changed, 136 insertions(+), 80 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 11afae14d0205..1ea45fc1a8a40 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -49,7 +49,7 @@ license: |
 
   - In Spark version 2.4 and earlier, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys and join keys. Since Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and earlier.
 
-  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, these built-in functions will remove duplicated map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
+  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, new config `spark.sql.legacy.allowDuplicatedMapKeys` was added, with the default value `false`, Spark will throw RuntimeException while duplicated keys are found. If set to `true`, these built-in functions will remove duplicated map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
 
   - In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
 
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index e80d556cc89e3..b7ff7264aa183 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -2766,12 +2766,12 @@ def map_concat(*cols):
     :param cols: list of column names (string) or list of :class:`Column` expressions
 
     >>> from pyspark.sql.functions import map_concat
-    >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c', 1, 'd') as map2")
+    >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2")
     >>> df.select(map_concat("map1", "map2").alias("map3")).show(truncate=False)
     +------------------------+
     |map3                    |
     +------------------------+
-    |[1 -> d, 2 -> b, 3 -> c]|
+    |[1 -> a, 2 -> b, 3 -> c]|
     +------------------------+
     """
     sc = SparkContext._active_spark_context
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 6ed68e47ce7a1..2b123479a1909 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -516,8 +516,8 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
   usage = "_FUNC_(map, ...) - Returns the union of all the given maps",
   examples = """
     Examples:
-      > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd'));
-       {1:"a",2:"c",3:"d"}
+      > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(3, 'c'));
+       {1:"a",2:"b",3:"c"}
   """, since = "2.4.0")
 case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
index 37d65309e2b89..40e75b5b6cd25 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.catalyst.util
 import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.array.ByteArrayMethods
 
@@ -47,6 +49,9 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
   private lazy val keyGetter = InternalRow.getAccessor(keyType)
   private lazy val valueGetter = InternalRow.getAccessor(valueType)
 
+  private val allowDuplicatedMapKey =
+    SQLConf.get.getConf(LEGACY_ALLOW_DUPLICATED_MAP_KEY)
+
   def put(key: Any, value: Any): Unit = {
     if (key == null) {
       throw new RuntimeException("Cannot use null as map key.")
@@ -62,6 +67,11 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
       keys.append(key)
       values.append(value)
     } else {
+      if (!allowDuplicatedMapKey) {
+        throw new RuntimeException(s"Duplicate map key $key was founded, please check the input " +
+          "data. If you want to remove the duplicated keys with last-win policy, you can set " +
+          s"${LEGACY_ALLOW_DUPLICATED_MAP_KEY.key} to true.")
+      }
       // Overwrite the previous value, as the policy is last wins.
       values(index) = value
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 19c94e23e046e..c336f87d58cdd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2188,6 +2188,16 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
+  val LEGACY_ALLOW_DUPLICATED_MAP_KEY =
+    buildConf("spark.sql.legacy.allowDuplicatedMapKeys")
+      .doc("When true, use last wins policy to remove duplicated map keys in built-in functions, " +
+        "this config takes effect in below build-in functions: CreateMap, MapFromArrays, " +
+        "MapFromEntries, StringToMap, MapConcat and TransformKeys. Otherwise, if this is false, " +
+        "which is the default, Spark will throw an exception when duplicated map keys are " +
+        "detected.")
+      .booleanConf
+      .createWithDefault(false)
+
   /**
    * Holds information about keys that have been deprecated.
    *
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 9e98e146c7a0e..01df6675016d2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -139,8 +139,10 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
       MapType(IntegerType, IntegerType, valueContainsNull = true))
     val mNull = Literal.create(null, MapType(StringType, StringType))
 
-    // overlapping maps should remove duplicated map keys w.r.t. last win policy.
-    checkEvaluation(MapConcat(Seq(m0, m1)), create_map("a" -> "4", "b" -> "2", "c" -> "3"))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // overlapping maps should remove duplicated map keys w.r.t. last win policy.
+      checkEvaluation(MapConcat(Seq(m0, m1)), create_map("a" -> "4", "b" -> "2", "c" -> "3"))
+    }
 
     // maps with no overlap
     checkEvaluation(MapConcat(Seq(m0, m2)),
@@ -272,8 +274,10 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(MapFromEntries(ai1), create_map(1 -> null, 2 -> 20, 3 -> null))
     checkEvaluation(MapFromEntries(ai2), Map.empty)
     checkEvaluation(MapFromEntries(ai3), null)
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(MapFromEntries(ai4), create_map(1 -> 20))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(MapFromEntries(ai4), create_map(1 -> 20))
+    }
     // Map key can't be null
     checkExceptionInExpression[RuntimeException](
       MapFromEntries(ai5),
@@ -294,8 +298,10 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(MapFromEntries(as1), create_map("a" -> null, "b" -> "bb", "c" -> null))
     checkEvaluation(MapFromEntries(as2), Map.empty)
     checkEvaluation(MapFromEntries(as3), null)
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(MapFromEntries(as4), create_map("a" -> "bb"))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(MapFromEntries(as4), create_map("a" -> "bb"))
+    }
     // Map key can't be null
     checkExceptionInExpression[RuntimeException](
       MapFromEntries(as5),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 9039cd6451590..2c1e0c8460468 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedExtractValue}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -216,10 +217,12 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
       CreateMap(interlace(strWithNull, intSeq.map(Literal(_)))),
       "Cannot use null as map key")
 
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(
-      CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
-      create_map(1 -> 3))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(
+        CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
+        create_map(1 -> 3))
+    }
 
     // ArrayType map key and value
     val map = CreateMap(Seq(
@@ -281,12 +284,14 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
       MapFromArrays(intWithNullArray, strArray),
       "Cannot use null as map key")
 
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(
-      MapFromArrays(
-        Literal.create(Seq(1, 1), ArrayType(IntegerType)),
-        Literal.create(Seq(2, 3), ArrayType(IntegerType))),
-      create_map(1 -> 3))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(
+        MapFromArrays(
+          Literal.create(Seq(1, 1), ArrayType(IntegerType)),
+          Literal.create(Seq(2, 3), ArrayType(IntegerType))),
+        create_map(1 -> 3))
+    }
 
     // map key can't be map
     val arrayOfMap = Seq(create_map(1 -> "a", 2 -> "b"))
@@ -399,10 +404,12 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     val m5 = Map("a" -> null)
     checkEvaluation(new StringToMap(s5), m5)
 
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(
-      new StringToMap(Literal("a:1,b:2,a:3")),
-      create_map("a" -> "3", "b" -> "2"))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(
+        new StringToMap(Literal("a:1,b:2,a:3")),
+        create_map("a" -> "3", "b" -> "2"))
+    }
 
     // arguments checking
     assert(new StringToMap(Literal("a:1,b:2,c:3")).checkInputDataTypes().isSuccess)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
index e7b713840b884..b3438538a3367 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
@@ -465,8 +465,10 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(
       transformKeys(transformKeys(ai0, plusOne), plusValue),
       create_map(3 -> 1, 5 -> 2, 7 -> 3, 9 -> 4))
-    // Duplicated map keys will be removed w.r.t. the last wins policy.
-    checkEvaluation(transformKeys(ai0, modKey), create_map(1 -> 4, 2 -> 2, 0 -> 3))
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      // Duplicated map keys will be removed w.r.t. the last wins policy.
+      checkEvaluation(transformKeys(ai0, modKey), create_map(1 -> 4, 2 -> 2, 0 -> 3))
+    }
     checkEvaluation(transformKeys(ai1, plusOne), Map.empty[Int, Int])
     checkEvaluation(transformKeys(ai1, plusOne), Map.empty[Int, Int])
     checkEvaluation(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
index 8509bce177129..87bbdb7300e4c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
@@ -20,10 +20,12 @@ package org.apache.spark.sql.catalyst.util
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{UnsafeArrayData, UnsafeRow}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{ArrayType, BinaryType, IntegerType, StructType}
 import org.apache.spark.unsafe.Platform
 
-class ArrayBasedMapBuilderSuite extends SparkFunSuite {
+class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
 
   test("basic") {
     val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
@@ -42,64 +44,79 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite {
     assert(e.getMessage.contains("Cannot use null as map key"))
   }
 
-  test("remove duplicated keys with last wins policy") {
+  test("fail while duplicated keys detected") {
     val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
     builder.put(1, 1)
-    builder.put(2, 2)
-    builder.put(1, 2)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    assert(ArrayBasedMapData.toScalaMap(map) == Map(1 -> 2, 2 -> 2))
+    val e = intercept[RuntimeException](builder.put(1, 2))
+    assert(e.getMessage.contains("Duplicate map key 1 was founded"))
+  }
+
+  test("remove duplicated keys with last wins policy") {
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
+      builder.put(1, 1)
+      builder.put(2, 2)
+      builder.put(1, 2)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      assert(ArrayBasedMapData.toScalaMap(map) == Map(1 -> 2, 2 -> 2))
+    }
   }
 
   test("binary type key") {
-    val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
-    builder.put(Array(1.toByte), 1)
-    builder.put(Array(2.toByte), 2)
-    builder.put(Array(1.toByte), 3)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    val entries = ArrayBasedMapData.toScalaMap(map).iterator.toSeq
-    assert(entries(0)._1.asInstanceOf[Array[Byte]].toSeq == Seq(1))
-    assert(entries(0)._2 == 3)
-    assert(entries(1)._1.asInstanceOf[Array[Byte]].toSeq == Seq(2))
-    assert(entries(1)._2 == 2)
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
+      builder.put(Array(1.toByte), 1)
+      builder.put(Array(2.toByte), 2)
+      builder.put(Array(1.toByte), 3)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      val entries = ArrayBasedMapData.toScalaMap(map).iterator.toSeq
+      assert(entries(0)._1.asInstanceOf[Array[Byte]].toSeq == Seq(1))
+      assert(entries(0)._2 == 3)
+      assert(entries(1)._1.asInstanceOf[Array[Byte]].toSeq == Seq(2))
+      assert(entries(1)._2 == 2)
+    }
   }
 
   test("struct type key") {
-    val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
-    builder.put(InternalRow(1), 1)
-    builder.put(InternalRow(2), 2)
-    val unsafeRow = {
-      val row = new UnsafeRow(1)
-      val bytes = new Array[Byte](16)
-      row.pointTo(bytes, 16)
-      row.setInt(0, 1)
-      row
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
+      builder.put(InternalRow(1), 1)
+      builder.put(InternalRow(2), 2)
+      val unsafeRow = {
+        val row = new UnsafeRow(1)
+        val bytes = new Array[Byte](16)
+        row.pointTo(bytes, 16)
+        row.setInt(0, 1)
+        row
+      }
+      builder.put(unsafeRow, 3)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      assert(ArrayBasedMapData.toScalaMap(map) == Map(InternalRow(1) -> 3, InternalRow(2) -> 2))
     }
-    builder.put(unsafeRow, 3)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    assert(ArrayBasedMapData.toScalaMap(map) == Map(InternalRow(1) -> 3, InternalRow(2) -> 2))
   }
 
   test("array type key") {
-    val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
-    builder.put(new GenericArrayData(Seq(1, 1)), 1)
-    builder.put(new GenericArrayData(Seq(2, 2)), 2)
-    val unsafeArray = {
-      val array = new UnsafeArrayData()
-      val bytes = new Array[Byte](24)
-      Platform.putLong(bytes, Platform.BYTE_ARRAY_OFFSET, 2)
-      array.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET, 24)
-      array.setInt(0, 1)
-      array.setInt(1, 1)
-      array
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
+      builder.put(new GenericArrayData(Seq(1, 1)), 1)
+      builder.put(new GenericArrayData(Seq(2, 2)), 2)
+      val unsafeArray = {
+        val array = new UnsafeArrayData()
+        val bytes = new Array[Byte](24)
+        Platform.putLong(bytes, Platform.BYTE_ARRAY_OFFSET, 2)
+        array.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET, 24)
+        array.setInt(0, 1)
+        array.setInt(1, 1)
+        array
+      }
+      builder.put(unsafeArray, 3)
+      val map = builder.build()
+      assert(map.numElements() == 2)
+      assert(ArrayBasedMapData.toScalaMap(map) ==
+        Map(new GenericArrayData(Seq(1, 1)) -> 3, new GenericArrayData(Seq(2, 2)) -> 2))
     }
-    builder.put(unsafeArray, 3)
-    val map = builder.build()
-    assert(map.numElements() == 2)
-    assert(ArrayBasedMapData.toScalaMap(map) ==
-      Map(new GenericArrayData(Seq(1, 1)) -> 3, new GenericArrayData(Seq(2, 2)) -> 2))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index f7531ea446015..341b3255b12bf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -651,8 +651,10 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       Row(null)
     )
 
-    checkAnswer(df1.selectExpr("map_concat(map1, map2)"), expected1a)
-    checkAnswer(df1.select(map_concat($"map1", $"map2")), expected1a)
+    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      checkAnswer(df1.selectExpr("map_concat(map1, map2)"), expected1a)
+      checkAnswer(df1.select(map_concat($"map1", $"map2")), expected1a)
+    }
 
     val expected1b = Seq(
       Row(Map(1 -> 100, 2 -> 200)),
@@ -3068,11 +3070,13 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       checkAnswer(dfExample2.select(transform_keys(col("j"), (k, v) => k + v)),
         Seq(Row(Map(2.0 -> 1.0, 3.4 -> 1.4, 4.7 -> 1.7))))
 
-      checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)"),
-        Seq(Row(Map(true -> true, true -> false))))
+      withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+        checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)"),
+          Seq(Row(Map(true -> true, true -> false))))
 
-      checkAnswer(dfExample3.select(transform_keys(col("x"), (k, v) => k % 2 === 0 || v)),
-        Seq(Row(Map(true -> true, true -> false))))
+        checkAnswer(dfExample3.select(transform_keys(col("x"), (k, v) => k % 2 === 0 || v)),
+          Seq(Row(Map(true -> true, true -> false))))
+      }
 
       checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) -> if(v, 2 * k, 3 * k))"),
         Seq(Row(Map(50 -> true, 78 -> false))))

From d9383e1a895bdfd1e318974d8f418937cd08e118 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 17 Feb 2020 22:51:56 +0800
Subject: [PATCH 0094/1280] [SPARK-30793][SQL] Fix truncations of timestamps
 before the epoch to minutes and seconds

### What changes were proposed in this pull request?
In the PR, I propose to replace `%` by `Math.floorMod` in `DateTimeUtils.truncTimestamp` for the `SECOND` and `MINUTE` levels.

### Why are the changes needed?
This fixes the issue of incorrect truncation of timestamps before the epoch `1970-01-01T00:00:00.000000Z` to the `SECOND` and `MINUTE` levels. For example, timestamps after the epoch are truncated by cutting off the rest part of the timestamp:
```sql
spark-sql> select date_trunc('SECOND', '2020-02-11 00:01:02.123');
2020-02-11 00:01:02
```
but seconds in the truncated timestamp before the epoch are increased by 1:
```sql
spark-sql> select date_trunc('SECOND', '1960-02-11 00:01:02.123');
1960-02-11 00:01:03
```

### Does this PR introduce any user-facing change?
Yes. After the changes, the example above outputs correct result:
```sql
spark-sql> select date_trunc('SECOND', '1960-02-11 00:01:02.123');
1960-02-11 00:01:02
```

### How was this patch tested?
Added new tests to `DateFunctionsSuite`.

Closes #27543 from MaxGekk/fix-second-minute-truc.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 06217cfded8d32962e7c54c315f8e684eb9f0999)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/util/DateTimeUtils.scala     |  4 ++--
 .../org/apache/spark/sql/DateFunctionsSuite.scala   | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 768dde0c72a89..a1054a4ab9864 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -716,9 +716,9 @@ object DateTimeUtils {
         val truncated = level match {
           case TRUNC_TO_MILLISECOND => millis
           case TRUNC_TO_SECOND =>
-            millis - millis % MILLIS_PER_SECOND
+            millis - Math.floorMod(millis, MILLIS_PER_SECOND)
           case TRUNC_TO_MINUTE =>
-            millis - millis % MILLIS_PER_MINUTE
+            millis - Math.floorMod(millis, MILLIS_PER_MINUTE)
           case _ => // Try to truncate date levels
             val dDays = millisToDays(millis, zoneId)
             daysToMillis(truncDate(dDays, level), zoneId)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index ba45b9f9b62df..527dfe56c2289 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -869,4 +869,17 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     checkTrunc("HOUR", "0010-01-01 01:00:00")
     checkTrunc("DAY", "0010-01-01 00:00:00")
   }
+
+  test("SPARK-30793: truncate timestamps before the epoch to seconds and minutes") {
+    def checkTrunc(level: String, expected: String): Unit = {
+      val df = Seq("1961-04-12 00:01:02.345")
+        .toDF()
+        .select($"value".cast("timestamp").as("ts"))
+        .select(date_trunc(level, $"ts").cast("string"))
+      checkAnswer(df, Row(expected))
+    }
+
+    checkTrunc("SECOND", "1961-04-12 00:01:02")
+    checkTrunc("MINUTE", "1961-04-12 00:01:00")
+  }
 }

From 00943be81afbca6be13e1e72b24536cd98a788d6 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Tue, 18 Feb 2020 00:52:05 +0800
Subject: [PATCH 0095/1280] [SPARK-30829][SQL] Define LegacyBehaviorPolicy
 enumeration as the common value for result change configs

### What changes were proposed in this pull request?
Define a new enumeration `LegacyBehaviorPolicy` in SQLConf, it will be used as the common value for result change configs.

### Why are the changes needed?
During API auditing for the 3.0 release, we found several new approaches that will change the results silently. For these features, we need a common three-value config.

### Does this PR introduce any user-facing change?
Yes, original config `spark.sql.legacy.ctePrecedence.enabled` change to `spark.sql.legacy.ctePrecedencePolicy`.

### How was this patch tested?
Existing UT.

Closes #27579 from xuanyuanking/SPARK-30829.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit e4a541b278b47d375252b2d0a8482c053ec3dd3e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalyst/analysis/CTESubstitution.scala   | 23 ++++++++++---------
 .../apache/spark/sql/internal/SQLConf.scala   | 19 ++++++++++-----
 .../resources/sql-tests/inputs/cte-legacy.sql |  2 +-
 .../sql-tests/inputs/cte-nonlegacy.sql        |  2 +-
 .../sql-tests/results/cte-legacy.sql.out      |  4 ++--
 .../resources/sql-tests/results/cte.sql.out   | 12 +++++-----
 6 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index d2be15d87d023..c8078e10fd044 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -22,21 +22,21 @@ import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, With}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LEGACY_CTE_PRECEDENCE_ENABLED
+import org.apache.spark.sql.internal.SQLConf.{LEGACY_CTE_PRECEDENCE_POLICY, LegacyBehaviorPolicy}
 
 /**
  * Analyze WITH nodes and substitute child plan with CTE definitions.
  */
 object CTESubstitution extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    val isLegacy = SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_ENABLED)
-    if (isLegacy.isEmpty) {
-      assertNoNameConflictsInCTE(plan, inTraverse = false)
-      traverseAndSubstituteCTE(plan, inTraverse = false)
-    } else if (isLegacy.get) {
-      legacyTraverseAndSubstituteCTE(plan)
-    } else {
-      traverseAndSubstituteCTE(plan, inTraverse = false)
+    LegacyBehaviorPolicy.withName(SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_POLICY)) match {
+      case LegacyBehaviorPolicy.EXCEPTION =>
+        assertNoNameConflictsInCTE(plan, inTraverse = false)
+        traverseAndSubstituteCTE(plan, inTraverse = false)
+      case LegacyBehaviorPolicy.LEGACY =>
+        legacyTraverseAndSubstituteCTE(plan)
+      case LegacyBehaviorPolicy.CORRECTED =>
+        traverseAndSubstituteCTE(plan, inTraverse = false)
     }
   }
 
@@ -54,8 +54,9 @@ object CTESubstitution extends Rule[LogicalPlan] {
           case (cteName, _) =>
             if (cteNames.contains(cteName)) {
               throw new AnalysisException(s"Name $cteName is ambiguous in nested CTE. " +
-                s"Please set ${LEGACY_CTE_PRECEDENCE_ENABLED.key} to false so that name defined " +
-                "in inner CTE takes precedence. See more details in SPARK-28228.")
+                s"Please set ${LEGACY_CTE_PRECEDENCE_POLICY.key} to CORRECTED so that name " +
+                "defined in inner CTE takes precedence. If set it to LEGACY, outer CTE " +
+                "definitions will take precedence. See more details in SPARK-28228.")
             } else {
               cteName
             }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index c336f87d58cdd..ac3169d002713 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2106,13 +2106,20 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
-  val LEGACY_CTE_PRECEDENCE_ENABLED = buildConf("spark.sql.legacy.ctePrecedence.enabled")
+  object LegacyBehaviorPolicy extends Enumeration {
+    val EXCEPTION, LEGACY, CORRECTED = Value
+  }
+
+  val LEGACY_CTE_PRECEDENCE_POLICY = buildConf("spark.sql.legacy.ctePrecedencePolicy")
     .internal()
-    .doc("When true, outer CTE definitions takes precedence over inner definitions. If set to " +
-      "false, inner CTE definitions take precedence. The default value is empty, " +
-      "AnalysisException is thrown while name conflict is detected in nested CTE.")
-    .booleanConf
-    .createOptional
+    .doc("When LEGACY, outer CTE definitions takes precedence over inner definitions. If set to " +
+      "CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, " +
+      "AnalysisException is thrown while name conflict is detected in nested CTE. This config " +
+      "will be removed in future versions and CORRECTED will be the only behavior.")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+    .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
   val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
     buildConf("spark.sql.legacy.arrayExistsFollowsThreeValuedLogic")
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
index 2f2606d44d910..d8754d3023965 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
@@ -2,7 +2,7 @@ create temporary view t as select * from values 0, 1, 2 as t(id);
 create temporary view t2 as select * from values 0, 1 as t(id);
 
 -- CTE legacy substitution
-SET spark.sql.legacy.ctePrecedence.enabled=true;
+SET spark.sql.legacy.ctePrecedencePolicy=legacy;
 
 -- CTE in CTE definition
 WITH t as (
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
index b711bf338ab08..8b9ce8911b133 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
@@ -1,2 +1,2 @@
---SET spark.sql.legacy.ctePrecedence.enabled = false
+--SET spark.sql.legacy.ctePrecedencePolicy = corrected
 --IMPORT cte.sql
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
index a9709c4a79793..a20ae55b1b876 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
@@ -19,11 +19,11 @@ struct<>
 
 
 -- !query
-SET spark.sql.legacy.ctePrecedence.enabled=true
+SET spark.sql.legacy.ctePrecedencePolicy=legacy
 -- !query schema
 struct<key:string,value:string>
 -- !query output
-spark.sql.legacy.ctePrecedence.enabled	true
+spark.sql.legacy.ctePrecedencePolicy	legacy
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/cte.sql.out b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
index 1d50aa8f57505..6486fba18d20f 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
@@ -207,7 +207,7 @@ SELECT * FROM t2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -226,7 +226,7 @@ SELECT * FROM t2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -245,7 +245,7 @@ SELECT * FROM t2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -299,7 +299,7 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -314,7 +314,7 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
 
 
 -- !query
@@ -330,7 +330,7 @@ SELECT (
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedence.enabled to false so that name defined in inner CTE takes precedence. See more details in SPARK-28228.;
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
 
 
 -- !query

From 7fa7f7abf832fc211927468f59c9fc1a816efe51 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Mon, 17 Feb 2020 09:26:56 -0800
Subject: [PATCH 0096/1280] [SPARK-30755][SQL] Update migration guide and add
 actionable exception for HIVE-15167

### What changes were proposed in this pull request?
[HIVE-15167](https://issues.apache.org/jira/browse/HIVE-15167) removed the `SerDe` interface. This may break custom `SerDe` builds for Hive 1.2. This PR update the migration guide for this change.

### Why are the changes needed?

Otherwise:
```
2020-01-27 05:11:20.446 - stderr> 20/01/27 05:11:20 INFO DAGScheduler: ResultStage 2 (main at NativeMethodAccessorImpl.java:0) failed in 1.000 s due to Job aborted due to stage failure: Task 0 in stage 2.0 failed 4 times, most recent failure: Lost task 0.3 in stage 2.0 (TID 13, 10.110.21.210, executor 1): java.lang.NoClassDefFoundError: org/apache/hadoop/hive/serde2/SerDe
  2020-01-27 05:11:20.446 - stderr>  at java.lang.ClassLoader.defineClass1(Native Method)
  2020-01-27 05:11:20.446 - stderr>  at java.lang.ClassLoader.defineClass(ClassLoader.java:756)
  2020-01-27 05:11:20.446 - stderr>  at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
  2020-01-27 05:11:20.446 - stderr>  at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
  2020-01-27 05:11:20.446 - stderr>  at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
  2020-01-27 05:11:20.446 - stderr>  at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
  2020-01-27 05:11:20.446 - stderr>  at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
  2020-01-27 05:11:20.446 - stderr>  at java.security.AccessController.doPrivileged(Native Method)
  2020-01-27 05:11:20.446 - stderr>  at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
  2020-01-27 05:11:20.446 - stderr>  at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
  2020-01-27 05:11:20.446 - stderr>  at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
  2020-01-27 05:11:20.446 - stderr>  at java.lang.ClassLoader.loadClass(ClassLoader.java:405)
  2020-01-27 05:11:20.446 - stderr>  at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
  2020-01-27 05:11:20.446 - stderr>  at java.lang.Class.forName0(Native Method)
  2020-01-27 05:11:20.446 - stderr>  at java.lang.Class.forName(Class.java:348)
  2020-01-27 05:11:20.446 - stderr>  at org.apache.hadoop.hive.ql.plan.TableDesc.getDeserializerClass(TableDesc.java:76)
.....
```

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Manual test

Closes #27492 from wangyum/SPARK-30755.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 76ddb6d835e773d87d93fd2dac008ace1f756d3b)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-migration-guide.md                            |  2 ++
 .../apache/spark/sql/hive/client/HiveClientImpl.scala  | 10 +++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 1ea45fc1a8a40..54a1fbf2cac8e 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -332,6 +332,8 @@ license: |
 
   - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3. This may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore.
   For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
+  
+  - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3. You need to migrate your custom SerDes to Hive 2.3 or build your own Spark with `hive-1.2` profile. See HIVE-15167 for more details.
 
 ## Upgrading from Spark SQL 2.4.4 to 2.4.5
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index b5c5f0e9381bc..ab4db0cf36242 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -318,7 +318,15 @@ private[hive] class HiveClientImpl(
     // with the HiveConf in `state` to override the context class loader of the current
     // thread.
     shim.setCurrentSessionState(state)
-    val ret = try f finally {
+    val ret = try {
+      f
+    } catch {
+      case e: NoClassDefFoundError
+        if HiveUtils.isHive23 && e.getMessage.contains("org/apache/hadoop/hive/serde2/SerDe") =>
+        throw new ClassNotFoundException("The SerDe interface removed since Hive 2.3(HIVE-15167)." +
+          " Please migrate your custom SerDes to Hive 2.3 or build your own Spark with" +
+          " hive-1.2 profile. See HIVE-15167 for more details.", e)
+    } finally {
       state.getConf.setClassLoader(originalConfLoader)
       Thread.currentThread().setContextClassLoader(original)
       HiveCatalogMetrics.incrementHiveClientCalls(1)

From f50048a84f0c8240358f71dcb902193cb719f4ba Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Tue, 18 Feb 2020 01:44:14 +0800
Subject: [PATCH 0097/1280] [SPARK-11150][SQL][FOLLOWUP] Move
 sql/dynamicpruning to sql/execution/dynamicpruning

### What changes were proposed in this pull request?
Follow-up work for #25600. In this PR, we move `sql/dynamicpruning` to `sql/execution/dynamicpruning`.

### Why are the changes needed?
Fix the unexpected public APIs in 3.0.0 #27560.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UT.

Closes #27581 from xuanyuanking/SPARK-11150-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5ffc5ff55e86c180ad8a7fafb0e49e268bbfbca5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/execution/QueryExecution.scala   | 2 +-
 .../scala/org/apache/spark/sql/execution/SparkOptimizer.scala   | 2 +-
 .../dynamicpruning/CleanupDynamicPruningFilters.scala           | 2 +-
 .../sql/{ => execution}/dynamicpruning/PartitionPruning.scala   | 2 +-
 .../dynamicpruning/PlanDynamicPruningFilters.scala              | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
 rename sql/core/src/main/scala/org/apache/spark/sql/{ => execution}/dynamicpruning/CleanupDynamicPruningFilters.scala (97%)
 rename sql/core/src/main/scala/org/apache/spark/sql/{ => execution}/dynamicpruning/PartitionPruning.scala (99%)
 rename sql/core/src/main/scala/org/apache/spark/sql/{ => execution}/dynamicpruning/PlanDynamicPruningFilters.scala (98%)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 9109c05e75853..4571e69b302f6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -32,8 +32,8 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, ReturnAnswer}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
 import org.apache.spark.sql.catalyst.util.truncatedString
-import org.apache.spark.sql.dynamicpruning.PlanDynamicPruningFilters
 import org.apache.spark.sql.execution.adaptive.{AdaptiveExecutionContext, InsertAdaptiveSparkPlan}
+import org.apache.spark.sql.execution.dynamicpruning.PlanDynamicPruningFilters
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReuseExchange}
 import org.apache.spark.sql.execution.streaming.{IncrementalExecution, OffsetSeqMetadata}
 import org.apache.spark.sql.internal.SQLConf
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
index 013d94768a2a8..33b86a2b5340c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkOptimizer.scala
@@ -23,10 +23,10 @@ import org.apache.spark.sql.catalyst.optimizer._
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning}
 import org.apache.spark.sql.execution.datasources.PruneFileSourcePartitions
 import org.apache.spark.sql.execution.datasources.SchemaPruning
 import org.apache.spark.sql.execution.datasources.v2.V2ScanRelationPushDown
+import org.apache.spark.sql.execution.dynamicpruning.{CleanupDynamicPruningFilters, PartitionPruning}
 import org.apache.spark.sql.execution.python.{ExtractGroupingPythonUDFFromAggregate, ExtractPythonUDFFromAggregate, ExtractPythonUDFs}
 
 class SparkOptimizer(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/CleanupDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
similarity index 97%
rename from sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/CleanupDynamicPruningFilters.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
index 84be2c9cf5a21..1cf55a2f2954d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/CleanupDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/CleanupDynamicPruningFilters.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.dynamicpruning
+package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.catalyst.expressions.{DynamicPruning, PredicateHelper}
 import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
similarity index 99%
rename from sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
index 28f8f49d2ce44..43c6581632687 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PartitionPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.dynamicpruning
+package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
similarity index 98%
rename from sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
index be00f728aa3ca..eb091758910cd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/dynamicpruning/PlanDynamicPruningFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PlanDynamicPruningFilters.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.dynamicpruning
+package org.apache.spark.sql.execution.dynamicpruning
 
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.expressions

From 5f3786ce4afe25ddac77cf6466dd55beaaba0b9e Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 18 Feb 2020 01:52:23 +0800
Subject: [PATCH 0098/1280] [SPARK-15616][FOLLOW-UP][SQL] Sub Optimizer should
 include super.postHocOptimizationBatches

### What changes were proposed in this pull request?

Let sub optimizer's  `postHocOptimizationBatches` also includes super's `postHocOptimizationBatches`.

### Why are the changes needed?

It's necessary according to the design of catalyst optimizer.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass jenkins.

Closes #27607 from Ngone51/spark_15616_followup.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a1d536cb3e34b74b433d31c475058f38520ab399)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/hive/HiveSessionStateBuilder.scala  | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index 8020492ff759d..1858fbd910666 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -101,9 +101,8 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
    */
   override protected def optimizer: Optimizer = {
     new SparkOptimizer(catalogManager, catalog, experimentalMethods) {
-      override def postHocOptimizationBatches: Seq[Batch] = Seq(
-        Batch("Prune Hive Table Partitions", Once, new PruneHiveTablePartitions(session))
-      )
+      override def postHocOptimizationBatches: Seq[Batch] = super.postHocOptimizationBatches ++
+        Seq(Batch("Prune Hive Table Partitions", Once, new PruneHiveTablePartitions(session)))
 
       override def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] =
         super.extendedOperatorOptimizationRules ++ customOperatorOptimizationRules

From fbccc2ac757000ab260d0b04539216f8efbcefa0 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Tue, 18 Feb 2020 02:15:44 +0800
Subject: [PATCH 0099/1280] [SPARK-30808][SQL] Enable Java 8 time API in Thrift
 server

### What changes were proposed in this pull request?
- Set `spark.sql.datetime.java8API.enabled` to `true` in `hiveResultString()`, and restore it back at the end of the call.
- Convert collected `java.time.Instant` & `java.time.LocalDate` to `java.sql.Timestamp` and `java.sql.Date` for correct formatting.

### Why are the changes needed?
Because of textual representation of timestamps/dates before 1582 year is incorrect:
```shell
$ export TZ="America/Los_Angeles"
$ ./bin/spark-sql -S
```
```sql
spark-sql> set spark.sql.session.timeZone=America/Los_Angeles;
spark.sql.session.timeZone	America/Los_Angeles
spark-sql> SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20');
1001-01-01 00:07:02
```
It must be 1001-01-01 00:**00:00**.

### Does this PR introduce any user-facing change?
Yes. After the changes:
```shell
$ export TZ="America/Los_Angeles"
$ ./bin/spark-sql -S
```
```sql
spark-sql> set spark.sql.session.timeZone=America/Los_Angeles;
spark.sql.session.timeZone	America/Los_Angeles
spark-sql> SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20');
1001-01-01 00:00:00
```

### How was this patch tested?
By running hive-thiftserver tests. In particular:
```
./build/sbt -Phadoop-2.7 -Phive-2.3 -Phive-thriftserver "hive-thriftserver/test:testOnly *SparkThriftServerProtocolVersionsSuite"
```

Closes #27552 from MaxGekk/hive-thriftserver-java8-time-api.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit afaeb29599593f021c9ea47e52f8c70013a4afef)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/HiveResult.scala      | 61 ++++++++++++-------
 .../org/apache/spark/sql/SQLQuerySuite.scala  |  3 +-
 .../apache/spark/sql/SQLQueryTestSuite.scala  |  3 +-
 .../spark/sql/execution/HiveResultSuite.scala | 25 ++++----
 .../SparkExecuteStatementOperation.scala      | 10 ++-
 .../hive/thriftserver/SparkSQLDriver.scala    |  5 +-
 .../hive/execution/HiveComparisonTest.scala   |  4 +-
 7 files changed, 67 insertions(+), 44 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index 5a2f16d8e1526..b19184055268a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -21,9 +21,10 @@ import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 import java.time.{Instant, LocalDate}
 
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{Dataset, Row}
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
 import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand}
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -36,27 +37,43 @@ object HiveResult {
    * Returns the result as a hive compatible sequence of strings. This is used in tests and
    * `SparkSQLDriver` for CLI applications.
    */
-  def hiveResultString(executedPlan: SparkPlan): Seq[String] = executedPlan match {
-    case ExecutedCommandExec(_: DescribeCommandBase) =>
-      // If it is a describe command for a Hive table, we want to have the output format
-      // be similar with Hive.
-      executedPlan.executeCollectPublic().map {
-        case Row(name: String, dataType: String, comment) =>
-          Seq(name, dataType,
-            Option(comment.asInstanceOf[String]).getOrElse(""))
-            .map(s => String.format(s"%-20s", s))
-            .mkString("\t")
-      }
-    // SHOW TABLES in Hive only output table names, while ours output database, table name, isTemp.
-    case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
-      command.executeCollect().map(_.getString(1))
-    case other =>
-      val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
-      // We need the types so we can output struct field names
-      val types = executedPlan.output.map(_.dataType)
-      // Reformat to match hive tab delimited output.
-      result.map(_.zip(types).map(e => toHiveString(e)))
-        .map(_.mkString("\t"))
+  def hiveResultString(ds: Dataset[_]): Seq[String] = {
+    val executedPlan = ds.queryExecution.executedPlan
+    executedPlan match {
+      case ExecutedCommandExec(_: DescribeCommandBase) =>
+        // If it is a describe command for a Hive table, we want to have the output format
+        // be similar with Hive.
+        executedPlan.executeCollectPublic().map {
+          case Row(name: String, dataType: String, comment) =>
+            Seq(name, dataType,
+              Option(comment.asInstanceOf[String]).getOrElse(""))
+              .map(s => String.format(s"%-20s", s))
+              .mkString("\t")
+        }
+      // SHOW TABLES in Hive only output table names,
+      // while ours output database, table name, isTemp.
+      case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
+        command.executeCollect().map(_.getString(1))
+      case _ =>
+        val sessionWithJava8DatetimeEnabled = {
+          val cloned = ds.sparkSession.cloneSession()
+          cloned.conf.set(SQLConf.DATETIME_JAVA8API_ENABLED.key, true)
+          cloned
+        }
+        sessionWithJava8DatetimeEnabled.withActive {
+          // We cannot collect the original dataset because its encoders could be created
+          // with disabled Java 8 date-time API.
+          val result: Seq[Seq[Any]] = Dataset.ofRows(ds.sparkSession, ds.logicalPlan)
+            .queryExecution
+            .executedPlan
+            .executeCollectPublic().map(_.toSeq).toSeq
+          // We need the types so we can output struct field names
+          val types = executedPlan.output.map(_.dataType)
+          // Reformat to match hive tab delimited output.
+          result.map(_.zip(types).map(e => toHiveString(e)))
+            .map(_.mkString("\t"))
+        }
+    }
   }
 
   private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 11f9724e587f2..d6efb2f91ca0b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -189,8 +189,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
           example.split("  > ").toList.foreach(_ match {
             case exampleRe(sql, output) =>
               val df = clonedSpark.sql(sql)
-              val actual = unindentAndTrim(
-                hiveResultString(df.queryExecution.executedPlan).mkString("\n"))
+              val actual = unindentAndTrim(hiveResultString(df).mkString("\n"))
               val expected = unindentAndTrim(output)
               assert(actual === expected)
             case _ =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 83285911b3948..34829f18bfed2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -378,7 +378,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
         localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true)
       case _ =>
     }
-    localSparkSession.conf.set(SQLConf.DATETIME_JAVA8API_ENABLED.key, true)
 
     if (configSet.nonEmpty) {
       // Execute the list of set operation in order to add the desired configs
@@ -512,7 +511,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     val schema = df.schema.catalogString
     // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
     val answer = SQLExecution.withNewExecutionId(df.queryExecution, Some(sql)) {
-      hiveResultString(df.queryExecution.executedPlan).map(replaceNotIncludedMsg)
+      hiveResultString(df).map(replaceNotIncludedMsg)
     }
 
     // If the output is not pre-sorted, sort it.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index bb59b12e6f350..bddd15c6e25d6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -25,11 +25,10 @@ class HiveResultSuite extends SharedSparkSession {
   test("date formatting in hive result") {
     val dates = Seq("2018-12-28", "1582-10-13", "1582-10-14", "1582-10-15")
     val df = dates.toDF("a").selectExpr("cast(a as date) as b")
-    val executedPlan1 = df.queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan1)
+    val result = HiveResult.hiveResultString(df)
     assert(result == dates)
-    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
-    val result2 = HiveResult.hiveResultString(executedPlan2)
+    val df2 = df.selectExpr("array(b)")
+    val result2 = HiveResult.hiveResultString(df2)
     assert(result2 == dates.map(x => s"[$x]"))
   }
 
@@ -40,11 +39,10 @@ class HiveResultSuite extends SharedSparkSession {
       "1582-10-14 01:02:03",
       "1582-10-15 01:02:03")
     val df = timestamps.toDF("a").selectExpr("cast(a as timestamp) as b")
-    val executedPlan1 = df.queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan1)
+    val result = HiveResult.hiveResultString(df)
     assert(result == timestamps)
-    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
-    val result2 = HiveResult.hiveResultString(executedPlan2)
+    val df2 = df.selectExpr("array(b)")
+    val result2 = HiveResult.hiveResultString(df2)
     assert(result2 == timestamps.map(x => s"[$x]"))
   }
 
@@ -57,15 +55,14 @@ class HiveResultSuite extends SharedSparkSession {
   test("decimal formatting in hive result") {
     val df = Seq(new java.math.BigDecimal("1")).toDS()
     Seq(2, 6, 18).foreach { scala =>
-      val executedPlan =
-        df.selectExpr(s"CAST(value AS decimal(38, $scala))").queryExecution.executedPlan
-      val result = HiveResult.hiveResultString(executedPlan)
+      val decimalDf = df.selectExpr(s"CAST(value AS decimal(38, $scala))")
+      val result = HiveResult.hiveResultString(decimalDf)
       assert(result.head.split("\\.").last.length === scala)
     }
 
-    val executedPlan = Seq(java.math.BigDecimal.ZERO).toDS()
-      .selectExpr(s"CAST(value AS decimal(38, 8))").queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan)
+    val df2 = Seq(java.math.BigDecimal.ZERO).toDS()
+      .selectExpr(s"CAST(value AS decimal(38, 8))")
+    val result = HiveResult.hiveResultString(df2)
     assert(result.head === "0.00000000")
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index cf0e5ebf3a2b1..7bcd8032bd6af 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.security.PrivilegedExceptionAction
 import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate}
 import java.util.{Arrays, Map => JMap, UUID}
 import java.util.concurrent.RejectedExecutionException
 
@@ -178,7 +179,14 @@ private[hive] class SparkExecuteStatementOperation(
           }
           curCol += 1
         }
-        resultRowSet.addRow(row.toArray.asInstanceOf[Array[Object]])
+        // Convert date-time instances to types that are acceptable by Hive libs
+        // used in conversions to strings.
+        val resultRow = row.map {
+          case i: Instant => Timestamp.from(i)
+          case ld: LocalDate => Date.valueOf(ld)
+          case other => other
+        }.toArray.asInstanceOf[Array[Object]]
+        resultRowSet.addRow(resultRow)
         curRow += 1
         resultOffset += 1
       }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 12fba0eae6dce..64e91f405d613 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -60,9 +60,10 @@ private[hive] class SparkSQLDriver(val context: SQLContext = SparkSQLEnv.sqlCont
     // TODO unify the error code
     try {
       context.sparkContext.setJobDescription(command)
-      val execution = context.sessionState.executePlan(context.sql(command).logicalPlan)
+      val df = context.sql(command)
+      val execution = df.queryExecution
       hiveResponse = SQLExecution.withNewExecutionId(execution) {
-        hiveResultString(execution.executedPlan)
+        hiveResultString(df)
       }
       tableSchema = getResultSetSchema(execution)
       new CommandProcessorResponse(0)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 8b1f4c92755b9..82fe274515505 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -346,7 +346,9 @@ abstract class HiveComparisonTest
         val catalystResults = queryList.zip(hiveResults).map { case (queryString, hive) =>
           val query = new TestHiveQueryExecution(queryString.replace("../../data", testDataPath))
           def getResult(): Seq[String] = {
-            SQLExecution.withNewExecutionId(query)(hiveResultString(query.executedPlan))
+            SQLExecution.withNewExecutionId(query) {
+              hiveResultString(Dataset.ofRows(query.sparkSession, query.logical))
+            }
           }
           try { (query, prepareAnswer(query, getResult())) } catch {
             case e: Throwable =>

From 08131bbcfb7257caf146eba4af65ba0e407d34dd Mon Sep 17 00:00:00 2001
From: Ajith <ajith2489@gmail.com>
Date: Tue, 18 Feb 2020 02:26:52 +0800
Subject: [PATCH 0100/1280] [SPARK-22590][SQL] Copy
 sparkContext.localproperties to child thread in
 BroadcastExchangeExec.executionContext

### What changes were proposed in this pull request?
In `org.apache.spark.sql.execution.exchange.BroadcastExchangeExec#relationFuture` make a copy of `org.apache.spark.SparkContext#localProperties` and pass it to the broadcast execution thread in `org.apache.spark.sql.execution.exchange.BroadcastExchangeExec#executionContext`

### Why are the changes needed?
When executing `BroadcastExchangeExec`, the relationFuture is evaluated via a separate thread. The threads inherit the `localProperties` from `sparkContext` as they are the child threads.
These threads are created in the executionContext (thread pools). Each Thread pool has a default `keepAliveSeconds` of 60 seconds for idle threads.
Scenarios where the thread pool has threads which are idle and reused for a subsequent new query, the thread local properties will not be inherited from spark context (thread properties are inherited only on thread creation) hence end up having old or no properties set. This will cause taskset properties to be missing when properties are transferred by child thread via `sparkContext.runJob/submitJob`

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added UT

Closes #27266 from ajithme/broadcastlocalprop.

Authored-by: Ajith <ajith2489@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2854091d12d670b014c41713e72153856f4d3f6a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/util/ThreadUtils.scala   | 17 +++++++++++
 .../spark/sql/execution/SQLExecution.scala    | 10 +++----
 .../execution/basicPhysicalOperators.scala    |  5 ++--
 .../exchange/BroadcastExchangeExec.scala      | 16 +++--------
 .../internal/ExecutorSideSQLConfSuite.scala   | 28 +++++++++++++++++++
 5 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
index de39e4b410f25..e7872bb9cb6b0 100644
--- a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.util
 
 import java.util.concurrent._
+import java.util.concurrent.{Future => JFuture}
 import java.util.concurrent.locks.ReentrantLock
 
 import scala.concurrent.{Awaitable, ExecutionContext, ExecutionContextExecutor, Future}
@@ -304,6 +305,22 @@ private[spark] object ThreadUtils {
   }
   // scalastyle:on awaitresult
 
+  @throws(classOf[SparkException])
+  def awaitResult[T](future: JFuture[T], atMost: Duration): T = {
+    try {
+      atMost match {
+        case Duration.Inf => future.get()
+        case _ => future.get(atMost._1, atMost._2)
+      }
+    } catch {
+      case e: SparkFatalException =>
+        throw e.throwable
+      case NonFatal(t)
+        if !t.isInstanceOf[TimeoutException] && !t.isInstanceOf[RpcAbortException] =>
+        throw new SparkException("Exception thrown in awaitResult: ", t)
+    }
+  }
+
   // scalastyle:off awaitready
   /**
    * Preferred alternative to `Await.ready()`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
index 59c503e372535..5e4f30a5edaf1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
@@ -17,11 +17,9 @@
 
 package org.apache.spark.sql.execution
 
-import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.{ConcurrentHashMap, ExecutorService, Future => JFuture}
 import java.util.concurrent.atomic.AtomicLong
 
-import scala.concurrent.{ExecutionContext, Future}
-
 import org.apache.spark.SparkContext
 import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.sql.SparkSession
@@ -172,11 +170,11 @@ object SQLExecution {
    * SparkContext local properties are forwarded to execution thread
    */
   def withThreadLocalCaptured[T](
-      sparkSession: SparkSession, exec: ExecutionContext)(body: => T): Future[T] = {
+      sparkSession: SparkSession, exec: ExecutorService) (body: => T): JFuture[T] = {
     val activeSession = sparkSession
     val sc = sparkSession.sparkContext
     val localProps = Utils.cloneProperties(sc.getLocalProperties)
-    Future {
+    exec.submit(() => {
       val originalSession = SparkSession.getActiveSession
       val originalLocalProps = sc.getLocalProperties
       SparkSession.setActiveSession(activeSession)
@@ -190,6 +188,6 @@ object SQLExecution {
         SparkSession.clearActiveSession()
       }
       res
-    }(exec)
+    })
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index c35c48496e1c9..4e65bbf75282f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -17,10 +17,11 @@
 
 package org.apache.spark.sql.execution
 
+import java.util.concurrent.{Future => JFuture}
 import java.util.concurrent.TimeUnit._
 
 import scala.collection.mutable
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.{ExecutionContext}
 import scala.concurrent.duration.Duration
 
 import org.apache.spark.{InterruptibleIterator, Partition, SparkContext, TaskContext}
@@ -746,7 +747,7 @@ case class SubqueryExec(name: String, child: SparkPlan)
     "collectTime" -> SQLMetrics.createTimingMetric(sparkContext, "time to collect"))
 
   @transient
-  private lazy val relationFuture: Future[Array[InternalRow]] = {
+  private lazy val relationFuture: JFuture[Array[InternalRow]] = {
     // relationFuture is used in "doExecute". Therefore we can get the execution id correctly here.
     val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
     SQLExecution.withThreadLocalCaptured[Array[InternalRow]](
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
index 36f0d173cd0b0..65e6b7c2f0fba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
 import org.apache.spark.sql.execution.joins.HashedRelation
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
-import org.apache.spark.util.{SparkFatalException, ThreadUtils}
+import org.apache.spark.util.{SparkFatalException, ThreadUtils, Utils}
 
 /**
  * A [[BroadcastExchangeExec]] collects, transforms and finally broadcasts the result of
@@ -73,13 +73,8 @@ case class BroadcastExchangeExec(
 
   @transient
   private[sql] lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
-    // relationFuture is used in "doExecute". Therefore we can get the execution id correctly here.
-    val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
-    val task = new Callable[broadcast.Broadcast[Any]]() {
-      override def call(): broadcast.Broadcast[Any] = {
-        // This will run in another thread. Set the execution id so that we can connect these jobs
-        // with the correct execution.
-        SQLExecution.withExecutionId(sqlContext.sparkSession, executionId) {
+    SQLExecution.withThreadLocalCaptured[broadcast.Broadcast[Any]](
+      sqlContext.sparkSession, BroadcastExchangeExec.executionContext) {
           try {
             // Setup a job group here so later it may get cancelled by groupId if necessary.
             sparkContext.setJobGroup(runId.toString, s"broadcast exchange (runId $runId)",
@@ -121,7 +116,7 @@ case class BroadcastExchangeExec(
             val broadcasted = sparkContext.broadcast(relation)
             longMetric("broadcastTime") += NANOSECONDS.toMillis(
               System.nanoTime() - beforeBroadcast)
-
+            val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
             SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq)
             promise.success(broadcasted)
             broadcasted
@@ -146,10 +141,7 @@ case class BroadcastExchangeExec(
               promise.failure(e)
               throw e
           }
-        }
-      }
     }
-    BroadcastExchangeExec.executionContext.submit[broadcast.Broadcast[Any]](task)
   }
 
   override protected def doPrepare(): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
index 46d0c64592a00..888772c35d0ee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
@@ -159,6 +159,34 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-22590 propagate local properties to broadcast execution thread") {
+    withSQLConf(StaticSQLConf.BROADCAST_EXCHANGE_MAX_THREAD_THRESHOLD.key -> "1") {
+      val df1 = Seq(true).toDF()
+      val confKey = "spark.sql.y"
+      val confValue1 = UUID.randomUUID().toString()
+      val confValue2 = UUID.randomUUID().toString()
+
+      def generateBroadcastDataFrame(confKey: String, confValue: String): Dataset[Boolean] = {
+        val df = spark.range(1).mapPartitions { _ =>
+          Iterator(TaskContext.get.getLocalProperty(confKey) == confValue)
+        }
+        df.hint("broadcast")
+      }
+
+      // set local propert and assert
+      val df2 = generateBroadcastDataFrame(confKey, confValue1)
+      spark.sparkContext.setLocalProperty(confKey, confValue1)
+      val checks = df1.join(df2).collect()
+      assert(checks.forall(_.toSeq == Seq(true, true)))
+
+      // change local property and re-assert
+      val df3 = generateBroadcastDataFrame(confKey, confValue2)
+      spark.sparkContext.setLocalProperty(confKey, confValue2)
+      val checks2 = df1.join(df3).collect()
+      assert(checks2.forall(_.toSeq == Seq(true, true)))
+    }
+  }
 }
 
 case class SQLConfAssertPlan(confToCheck: Seq[(String, String)]) extends LeafExecNode {

From d6dcee487d4d353eea78ee567c256025b6bb0eff Mon Sep 17 00:00:00 2001
From: "wangguangxin.cn" <wangguangxin.cn@gmail.com>
Date: Mon, 17 Feb 2020 18:15:54 +0100
Subject: [PATCH 0101/1280] [SPARK-30806][SQL] Evaluate once per group in
 UnboundedWindowFunctionFrame
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
We only need to do aggregate evaluation once per group in `UnboundedWindowFunctionFrame`

### Why are the changes needed?
Currently, in `UnboundedWindowFunctionFrame.write`，it re-evaluate the processor for each row in a group, which is not necessary in fact which I'll address later. It hurts performance when the evaluation is time-consuming (for example, Percentile's eval need to sort its buffer and do some calculation). In our production, there is a percentile with window operation sql,  it costs more than 10 hours in SparkSQL while 10min in Hive.

In fact, `UnboundedWindowFunctionFrame` can be treated as `SlidingWindowFunctionFrame` with `lbound = UnboundedPreceding` and `ubound = UnboundedFollowing`, just as its comments. In that case, `SlidingWindowFunctionFrame` also only do evaluation once for each group.

The performance issue can be reproduced by running the follow scripts in local spark-shell
```
spark.range(100*100).map(i => (i, "India")).toDF("uv", "country").createOrReplaceTempView("test")
sql("select uv, country, percentile(uv, 0.95) over (partition by country) as ptc95 from test").collect.foreach(println)
```
Before this patch, the sql costs **128048 ms**.
With this patch,  the sql costs **3485 ms**.

If we increase the data size to 1000*1000 for example, then spark cannot even produce result without this patch(I'v waited for several hours).

### Does this PR introduce any user-facing change?
NO

### How was this patch tested?
Existing UT

Closes #27558 from WangGuangxin/windows.

Authored-by: wangguangxin.cn <wangguangxin.cn@gmail.com>
Signed-off-by: herman <herman@databricks.com>
---
 .../spark/sql/execution/window/WindowFunctionFrame.scala | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
index d5f2ffa5573a9..181ff42421c65 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
@@ -277,6 +277,8 @@ final class UnboundedWindowFunctionFrame(
       while (iterator.hasNext) {
         processor.update(iterator.next())
       }
+
+      processor.evaluate(target)
     }
 
     upperBound = rows.length
@@ -284,11 +286,8 @@ final class UnboundedWindowFunctionFrame(
 
   /** Write the frame columns for the current row to the given target row. */
   override def write(index: Int, current: InternalRow): Unit = {
-    // Unfortunately we cannot assume that evaluation is deterministic. So we need to re-evaluate
-    // for each row.
-    if (processor != null) {
-      processor.evaluate(target)
-    }
+    // The results are the same for each row in the partition, and have been evaluated in prepare.
+    // Don't need to recalculate here.
   }
 
   override def currentLowerBound(): Int = lowerBound

From 682018b3d2aed4a77b0b237ae9ff1e7a058599e7 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 18 Feb 2020 09:44:34 +0900
Subject: [PATCH 0102/1280] [SPARK-30825][SQL][DOC] Update documents
 information for window function

### What changes were proposed in this pull request?
I checked the all the window function and found all of them not add parameter information and version information to the document.
This PR will make a supplement.

### Why are the changes needed?
Documentation is missing and does not meet new standards.

### Does this PR introduce any user-facing change?
Yes. User will face the information of parameters and version.

### How was this patch tested?
Exists UT

Closes #27572 from beliefer/add_since_for_window_function.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit d8d3ce5c76732a8e3daf10a37621b5ec8a52dcec)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../expressions/windowExpressions.scala       | 82 ++++++++++++-------
 1 file changed, 52 insertions(+), 30 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index 74df1e7b8e8c3..9070a1ab0b059 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -410,10 +410,6 @@ abstract class OffsetWindowFunction
  * the window. Offsets start at 0, which is the current row. The offset must be constant
  * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
  * null is returned. If there is no such offset row, the `default` expression is evaluated.
- *
- * @param input expression to evaluate `offset` rows after the current row.
- * @param offset rows to jump ahead in the partition.
- * @param default to use when the offset is larger than the window. The default value is null.
  */
 @ExpressionDescription(
   usage = """
@@ -422,7 +418,15 @@ abstract class OffsetWindowFunction
       value of `default` is null. If the value of `input` at the `offset`th row is null,
       null is returned. If there is no such an offset row (e.g., when the offset is 1, the last
       row of the window does not have any subsequent row), `default` is returned.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * input - a string expression to evaluate `offset` rows after the current row.
+      * offset - an int expression which is rows to jump ahead in the partition.
+      * default - a string expression which is to use when the offset is larger than the window.
+          The default value is null.
+  """,
+  since = "2.0.0")
 case class Lead(input: Expression, offset: Expression, default: Expression)
     extends OffsetWindowFunction {
 
@@ -440,10 +444,6 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
  * the window. Offsets start at 0, which is the current row. The offset must be constant
  * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row,
  * null is returned. If there is no such offset row, the `default` expression is evaluated.
- *
- * @param input expression to evaluate `offset` rows before the current row.
- * @param offset rows to jump back in the partition.
- * @param default to use when the offset row does not exist.
  */
 @ExpressionDescription(
   usage = """
@@ -452,7 +452,14 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
       value of `default` is null. If the value of `input` at the `offset`th row is null,
       null is returned. If there is no such offset row (e.g., when the offset is 1, the first
       row of the window does not have any previous row), `default` is returned.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * input - a string expression to evaluate `offset` rows before the current row.
+      * offset - an int expression which is rows to jump back in the partition.
+      * default - a string expression which is to use when the offset row does not exist.
+  """,
+  since = "2.0.0")
 case class Lag(input: Expression, offset: Expression, default: Expression)
     extends OffsetWindowFunction {
 
@@ -509,7 +516,8 @@ object SizeBasedWindowFunction {
   usage = """
     _FUNC_() - Assigns a unique, sequential number to each row, starting with one,
       according to the ordering of rows within the window partition.
-  """)
+  """,
+  since = "2.0.0")
 case class RowNumber() extends RowNumberLike {
   override val evaluateExpression = rowNumber
   override def prettyName: String = "row_number"
@@ -526,7 +534,8 @@ case class RowNumber() extends RowNumberLike {
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the position of a value relative to all values in the partition.
-  """)
+  """,
+  since = "2.0.0")
 case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
   override def dataType: DataType = DoubleType
   // The frame for CUME_DIST is Range based instead of Row based, because CUME_DIST must
@@ -554,14 +563,18 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
  * threshold is increased by the bucket size (plus one extra if the current bucket is padded).
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param buckets number of buckets to divide the rows in. Default value is 1.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_(n) - Divides the rows for each window partition into `n` buckets ranging
       from 1 to at most `n`.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * buckets - an int expression which is number of buckets to divide the rows in.
+          Default value is 1.
+  """,
+  since = "2.0.0")
 case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction {
   def this() = this(Literal(1))
 
@@ -674,17 +687,20 @@ abstract class RankLike extends AggregateWindowFunction {
  * will produce gaps in the sequence.
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param children to base the rank on; a change in the value of one the children will trigger a
- *                 change in rank. This is an internal parameter and will be assigned by the
- *                 Analyser.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the number
       of rows preceding or equal to the current row in the ordering of the partition. The values
       will produce gaps in the sequence.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * children - this is to base the rank on; a change in the value of one the children will
+          trigger a change in rank. This is an internal parameter and will be assigned by the
+          Analyser.
+  """,
+  since = "2.0.0")
 case class Rank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): Rank = Rank(order)
@@ -696,17 +712,20 @@ case class Rank(children: Seq[Expression]) extends RankLike {
  * ranking sequence.
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param children to base the rank on; a change in the value of one the children will trigger a
- *                 change in rank. This is an internal parameter and will be assigned by the
- *                 Analyser.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the
       previously assigned rank value. Unlike the function rank, dense_rank will not produce gaps
       in the ranking sequence.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * children - this is to base the rank on; a change in the value of one the children will
+          trigger a change in rank. This is an internal parameter and will be assigned by the
+          Analyser.
+  """,
+  since = "2.0.0")
 case class DenseRank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): DenseRank = DenseRank(order)
@@ -726,15 +745,18 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
  * row counts in the its numerator.
  *
  * This documentation has been based upon similar documentation for the Hive and Presto projects.
- *
- * @param children to base the rank on; a change in the value of one of the children will trigger a
- *                 change in rank. This is an internal parameter and will be assigned by the
- *                 Analyser.
  */
 @ExpressionDescription(
   usage = """
     _FUNC_() - Computes the percentage ranking of a value in a group of values.
-  """)
+  """,
+  arguments = """
+    Arguments:
+      * children - this is to base the rank on; a change in the value of one the children will
+          trigger a change in rank. This is an internal parameter and will be assigned by the
+          Analyser.
+  """,
+  since = "2.0.0")
 case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): PercentRank = PercentRank(order)

From 99558f4315b3b865471776d67d341b1d49826b2c Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@gmail.com>
Date: Mon, 17 Feb 2020 19:19:28 -0600
Subject: [PATCH 0103/1280] [SPARK-30832][DOCS] SQL function doc headers should
 link to anchors

### Why are the changes needed?

In most of our docs, you can click on a heading to immediately get an anchor link to that specific section of the docs. This is very handy when you are reading the docs and want to share a link to a specific part.

The SQL function docs are lacking this. This PR adds this convenience to the SQL function docs.

Here's the impact on the generated HTML.

Before this PR:

```html
<h3 id="array_join">array_join</h3>
```

After this PR:

```html
<h3 id="array_join"><a class="toclink" href="#array_join">array_join</a></h3>
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

I built the docs manually and reviewed the results in my browser.

Closes #27585 from nchammas/SPARK-30832-sql-doc-headers.

Authored-by: Nicholas Chammas <nicholas.chammas@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 4ed9b88996dc0e8f096157659d537d9dd8eaed47)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 sql/mkdocs.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sql/mkdocs.yml b/sql/mkdocs.yml
index c34c891bb9e42..8aad985e30459 100644
--- a/sql/mkdocs.yml
+++ b/sql/mkdocs.yml
@@ -17,3 +17,6 @@ site_name: Spark SQL, Built-in Functions
 theme: readthedocs
 pages:
   - 'Functions': 'index.md'
+markdown_extensions:
+  - toc:
+      anchorlink: True

From ba9141592d0f0ce23c207efb21ae84ac7cc4670a Mon Sep 17 00:00:00 2001
From: Liang Zhang <liang.zhang@databricks.com>
Date: Tue, 18 Feb 2020 09:22:26 +0800
Subject: [PATCH 0104/1280] [SPARK-30791][SQL][PYTHON] Add 'sameSemantics' and
 'sementicHash' methods in Dataset

### What changes were proposed in this pull request?
This PR added two DeveloperApis to the Dataset[T] class. Both methods are just exposing lower-level methods to the Dataset[T] class.

### Why are the changes needed?
They are useful for checking whether two dataframes are the same when implementing dataframe caching in python, and also get a unique ID. It's easier to use if we wrap the lower-level APIs.

### Does this PR introduce any user-facing change?
```
scala> val df1 = Seq((1,2),(4,5)).toDF("col1", "col2")
df1: org.apache.spark.sql.DataFrame = [col1: int, col2: int]

scala> val df2 = Seq((1,2),(4,5)).toDF("col1", "col2")
df2: org.apache.spark.sql.DataFrame = [col1: int, col2: int]

scala> val df3 = Seq((0,2),(4,5)).toDF("col1", "col2")
df3: org.apache.spark.sql.DataFrame = [col1: int, col2: int]

scala> val df4 = Seq((0,2),(4,5)).toDF("col0", "col2")
df4: org.apache.spark.sql.DataFrame = [col0: int, col2: int]

scala> df1.semanticHash
res0: Int = 594427822

scala> df2.semanticHash
res1: Int = 594427822

scala> df1.sameSemantics(df2)
res2: Boolean = true

scala> df1.sameSemantics(df3)
res3: Boolean = false

scala> df3.semanticHash
res4: Int = -1592702048

scala> df4.semanticHash
res5: Int = -1592702048

scala> df4.sameSemantics(df3)
res6: Boolean = true
```

### How was this patch tested?
Unit test in scala and doctest in python.

Note: comments are copied from the corresponding lower-level APIs.
Note: There are some issues to be fixed that would improve the hash collision rate: https://github.com/apache/spark/pull/27565#discussion_r379881028

Closes #27565 from liangz1/df-same-result.

Authored-by: Liang Zhang <liang.zhang@databricks.com>
Signed-off-by: WeichenXu <weichen.xu@databricks.com>
(cherry picked from commit d8c0599e542976ef70b60bc673e7c9732fce49e5)
Signed-off-by: WeichenXu <weichen.xu@databricks.com>
---
 python/pyspark/sql/dataframe.py               | 46 +++++++++++++++++++
 python/pyspark/sql/tests/test_dataframe.py    |  5 ++
 .../scala/org/apache/spark/sql/Dataset.scala  | 28 +++++++++++
 .../org/apache/spark/sql/DatasetSuite.scala   | 15 ++++++
 4 files changed, 94 insertions(+)

diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 2432b8127840b..8325b6814f182 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -2153,6 +2153,52 @@ def transform(self, func):
                                               "should have been DataFrame." % type(result)
         return result
 
+    @since(3.1)
+    def sameSemantics(self, other):
+        """
+        Returns `True` when the logical query plans inside both :class:`DataFrame`\\s are equal and
+        therefore return same results.
+
+        .. note:: The equality comparison here is simplified by tolerating the cosmetic differences
+            such as attribute names.
+
+        .. note:: This API can compare both :class:`DataFrame`\\s very fast but can still return
+            `False` on the :class:`DataFrame` that return the same results, for instance, from
+            different plans. Such false negative semantic can be useful when caching as an example.
+
+        .. note:: DeveloperApi
+
+        >>> df1 = spark.range(10)
+        >>> df2 = spark.range(10)
+        >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col1", df2.id * 2))
+        True
+        >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col1", df2.id + 2))
+        False
+        >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col0", df2.id * 2))
+        True
+        """
+        if not isinstance(other, DataFrame):
+            raise ValueError("other parameter should be of DataFrame; however, got %s"
+                             % type(other))
+        return self._jdf.sameSemantics(other._jdf)
+
+    @since(3.1)
+    def semanticHash(self):
+        """
+        Returns a hash code of the logical query plan against this :class:`DataFrame`.
+
+        .. note:: Unlike the standard hash code, the hash is calculated against the query plan
+            simplified by tolerating the cosmetic differences such as attribute names.
+
+        .. note:: DeveloperApi
+
+        >>> spark.range(10).selectExpr("id as col0").semanticHash()  # doctest: +SKIP
+        1855039936
+        >>> spark.range(10).selectExpr("id as col1").semanticHash()  # doctest: +SKIP
+        1855039936
+        """
+        return self._jdf.semanticHash()
+
     where = copy_func(
         filter,
         sinceversion=1.3,
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index d738449799bda..942cd4b4b0ea3 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -782,6 +782,11 @@ def test_to_local_iterator_not_fully_consumed(self):
                     break
             self.assertEqual(df.take(8), result)
 
+    def test_same_semantics_error(self):
+        with QuietTest(self.sc):
+            with self.assertRaisesRegexp(ValueError, "should be of DataFrame.*int"):
+                self.spark.range(10).sameSemantics(1)
+
 
 class QueryExecutionListenerTests(unittest.TestCase, SQLTestUtils):
     # These tests are separate because it uses 'spark.sql.queryExecutionListeners' which is
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 42f35354e864f..5cd258341c183 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -3310,6 +3310,34 @@ class Dataset[T] private[sql](
     files.toSet.toArray
   }
 
+  /**
+   * Returns `true` when the logical query plans inside both [[Dataset]]s are equal and
+   * therefore return same results.
+   *
+   * @note The equality comparison here is simplified by tolerating the cosmetic differences
+   *       such as attribute names.
+   * @note This API can compare both [[Dataset]]s very fast but can still return `false` on
+   *       the [[Dataset]] that return the same results, for instance, from different plans. Such
+   *       false negative semantic can be useful when caching as an example.
+   * @since 3.1.0
+   */
+  @DeveloperApi
+  def sameSemantics(other: Dataset[T]): Boolean = {
+    queryExecution.analyzed.sameResult(other.queryExecution.analyzed)
+  }
+
+  /**
+   * Returns a `hashCode` of the logical query plan against this [[Dataset]].
+   *
+   * @note Unlike the standard `hashCode`, the hash is calculated against the query plan
+   *       simplified by tolerating the cosmetic differences such as attribute names.
+   * @since 3.1.0
+   */
+  @DeveloperApi
+  def semanticHash(): Int = {
+    queryExecution.analyzed.semanticHash()
+  }
+
   ////////////////////////////////////////////////////////////////////////////
   // For Python API
   ////////////////////////////////////////////////////////////////////////////
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index b0bd612e88d98..b4ed4ec28ceb0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1909,6 +1909,21 @@ class DatasetSuite extends QueryTest
 
     assert(active eq SparkSession.getActiveSession.get)
   }
+
+  test("SPARK-30791: sameSemantics and semanticHash work") {
+    val df1 = Seq((1, 2), (4, 5)).toDF("col1", "col2")
+    val df2 = Seq((1, 2), (4, 5)).toDF("col1", "col2")
+    val df3 = Seq((0, 2), (4, 5)).toDF("col1", "col2")
+    val df4 = Seq((0, 2), (4, 5)).toDF("col0", "col2")
+
+    assert(df1.sameSemantics(df2) === true)
+    assert(df1.sameSemantics(df3) === false)
+    assert(df3.sameSemantics(df4) === true)
+
+    assert(df1.semanticHash === df2.semanticHash)
+    assert(df1.semanticHash !== df3.semanticHash)
+    assert(df3.semanticHash === df4.semanticHash)
+  }
 }
 
 object AssertExecutionId {

From cb890d96bc38860988dba97efaf6d88cc8c09288 Mon Sep 17 00:00:00 2001
From: WeichenXu <weichen.xu@databricks.com>
Date: Tue, 18 Feb 2020 10:41:49 +0800
Subject: [PATCH 0105/1280] Revert "[SPARK-30791][SQL][PYTHON] Add
 'sameSemantics' and 'sementicHash' methods in Dataset"

This reverts commit ba9141592d0f0ce23c207efb21ae84ac7cc4670a.
---
 python/pyspark/sql/dataframe.py               | 46 -------------------
 python/pyspark/sql/tests/test_dataframe.py    |  5 --
 .../scala/org/apache/spark/sql/Dataset.scala  | 28 -----------
 .../org/apache/spark/sql/DatasetSuite.scala   | 15 ------
 4 files changed, 94 deletions(-)

diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 8325b6814f182..2432b8127840b 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -2153,52 +2153,6 @@ def transform(self, func):
                                               "should have been DataFrame." % type(result)
         return result
 
-    @since(3.1)
-    def sameSemantics(self, other):
-        """
-        Returns `True` when the logical query plans inside both :class:`DataFrame`\\s are equal and
-        therefore return same results.
-
-        .. note:: The equality comparison here is simplified by tolerating the cosmetic differences
-            such as attribute names.
-
-        .. note:: This API can compare both :class:`DataFrame`\\s very fast but can still return
-            `False` on the :class:`DataFrame` that return the same results, for instance, from
-            different plans. Such false negative semantic can be useful when caching as an example.
-
-        .. note:: DeveloperApi
-
-        >>> df1 = spark.range(10)
-        >>> df2 = spark.range(10)
-        >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col1", df2.id * 2))
-        True
-        >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col1", df2.id + 2))
-        False
-        >>> df1.withColumn("col1", df1.id * 2).sameSemantics(df2.withColumn("col0", df2.id * 2))
-        True
-        """
-        if not isinstance(other, DataFrame):
-            raise ValueError("other parameter should be of DataFrame; however, got %s"
-                             % type(other))
-        return self._jdf.sameSemantics(other._jdf)
-
-    @since(3.1)
-    def semanticHash(self):
-        """
-        Returns a hash code of the logical query plan against this :class:`DataFrame`.
-
-        .. note:: Unlike the standard hash code, the hash is calculated against the query plan
-            simplified by tolerating the cosmetic differences such as attribute names.
-
-        .. note:: DeveloperApi
-
-        >>> spark.range(10).selectExpr("id as col0").semanticHash()  # doctest: +SKIP
-        1855039936
-        >>> spark.range(10).selectExpr("id as col1").semanticHash()  # doctest: +SKIP
-        1855039936
-        """
-        return self._jdf.semanticHash()
-
     where = copy_func(
         filter,
         sinceversion=1.3,
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 942cd4b4b0ea3..d738449799bda 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -782,11 +782,6 @@ def test_to_local_iterator_not_fully_consumed(self):
                     break
             self.assertEqual(df.take(8), result)
 
-    def test_same_semantics_error(self):
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(ValueError, "should be of DataFrame.*int"):
-                self.spark.range(10).sameSemantics(1)
-
 
 class QueryExecutionListenerTests(unittest.TestCase, SQLTestUtils):
     # These tests are separate because it uses 'spark.sql.queryExecutionListeners' which is
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 5cd258341c183..42f35354e864f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -3310,34 +3310,6 @@ class Dataset[T] private[sql](
     files.toSet.toArray
   }
 
-  /**
-   * Returns `true` when the logical query plans inside both [[Dataset]]s are equal and
-   * therefore return same results.
-   *
-   * @note The equality comparison here is simplified by tolerating the cosmetic differences
-   *       such as attribute names.
-   * @note This API can compare both [[Dataset]]s very fast but can still return `false` on
-   *       the [[Dataset]] that return the same results, for instance, from different plans. Such
-   *       false negative semantic can be useful when caching as an example.
-   * @since 3.1.0
-   */
-  @DeveloperApi
-  def sameSemantics(other: Dataset[T]): Boolean = {
-    queryExecution.analyzed.sameResult(other.queryExecution.analyzed)
-  }
-
-  /**
-   * Returns a `hashCode` of the logical query plan against this [[Dataset]].
-   *
-   * @note Unlike the standard `hashCode`, the hash is calculated against the query plan
-   *       simplified by tolerating the cosmetic differences such as attribute names.
-   * @since 3.1.0
-   */
-  @DeveloperApi
-  def semanticHash(): Int = {
-    queryExecution.analyzed.semanticHash()
-  }
-
   ////////////////////////////////////////////////////////////////////////////
   // For Python API
   ////////////////////////////////////////////////////////////////////////////
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index b4ed4ec28ceb0..b0bd612e88d98 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1909,21 +1909,6 @@ class DatasetSuite extends QueryTest
 
     assert(active eq SparkSession.getActiveSession.get)
   }
-
-  test("SPARK-30791: sameSemantics and semanticHash work") {
-    val df1 = Seq((1, 2), (4, 5)).toDF("col1", "col2")
-    val df2 = Seq((1, 2), (4, 5)).toDF("col1", "col2")
-    val df3 = Seq((0, 2), (4, 5)).toDF("col1", "col2")
-    val df4 = Seq((0, 2), (4, 5)).toDF("col0", "col2")
-
-    assert(df1.sameSemantics(df2) === true)
-    assert(df1.sameSemantics(df3) === false)
-    assert(df3.sameSemantics(df4) === true)
-
-    assert(df1.semanticHash === df2.semanticHash)
-    assert(df1.semanticHash !== df3.semanticHash)
-    assert(df3.semanticHash === df4.semanticHash)
-  }
 }
 
 object AssertExecutionId {

From 949de25f0f65d5818028dc77381c596d69b23a48 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 18 Feb 2020 14:28:44 +0800
Subject: [PATCH 0106/1280] [SPARK-30847][SQL] Take productPrefix into account
 in MurmurHash3.productHash

### What changes were proposed in this pull request?

This PR proposes to port Scala's bugfix https://github.com/scala/scala/pull/7693 (Scala 2.13) to address https://github.com/scala/bug/issues/10495 issue.

In short, it is possible for different product instances having the same children to have the same hash. See:

```scala
scala> spark.range(1).selectExpr("id - 1").queryExecution.analyzed.semanticHash()
res0: Int = -565572825

scala> spark.range(1).selectExpr("id + 1").queryExecution.analyzed.semanticHash()
res1: Int = -565572825
```

### Why are the changes needed?

It was found during the review of https://github.com/apache/spark/pull/27565. We should better produce different hash for different objects.

### Does this PR introduce any user-facing change?

No, it's not identified. Possibly performance related issue.

### How was this patch tested?

Manually tested, and unittest was added.

Closes #27601 from HyukjinKwon/SPARK-30847.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9618806f44196aa2a69061794b34c31d264a4b3c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/trees/TreeNode.scala   | 25 ++++++++++++++++++-
 .../expressions/CanonicalizeSuite.scala       |  7 ++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index 56a198763b4e5..c4a106702a515 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -114,7 +114,30 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
 
   lazy val containsChild: Set[TreeNode[_]] = children.toSet
 
-  private lazy val _hashCode: Int = scala.util.hashing.MurmurHash3.productHash(this)
+  // Copied from Scala 2.13.1
+  // github.com/scala/scala/blob/v2.13.1/src/library/scala/util/hashing/MurmurHash3.scala#L56-L73
+  // to prevent the issue https://github.com/scala/bug/issues/10495
+  // TODO(SPARK-30848): Remove this once we drop Scala 2.12.
+  private final def productHash(x: Product, seed: Int, ignorePrefix: Boolean = false): Int = {
+    val arr = x.productArity
+    // Case objects have the hashCode inlined directly into the
+    // synthetic hashCode method, but this method should still give
+    // a correct result if passed a case object.
+    if (arr == 0) {
+      x.productPrefix.hashCode
+    } else {
+      var h = seed
+      if (!ignorePrefix) h = scala.util.hashing.MurmurHash3.mix(h, x.productPrefix.hashCode)
+      var i = 0
+      while (i < arr) {
+        h = scala.util.hashing.MurmurHash3.mix(h, x.productElement(i).##)
+        i += 1
+      }
+      scala.util.hashing.MurmurHash3.finalizeHash(h, arr)
+    }
+  }
+
+  private lazy val _hashCode: Int = productHash(this, scala.util.hashing.MurmurHash3.productSeed)
   override def hashCode(): Int = _hashCode
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
index 9802a6e5891b8..dd719437d618d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
@@ -79,4 +79,11 @@ class CanonicalizeSuite extends SparkFunSuite {
       0, Some("b2"))
     assert(fieldB1.semanticEquals(fieldB2))
   }
+
+  test("SPARK-30847: Take productPrefix into account in MurmurHash3.productHash") {
+    val range = Range(1, 1, 1, 1)
+    val addExpr = Add(range.output.head, Literal(1))
+    val subExpr = Subtract(range.output.head, Literal(1))
+    assert(addExpr.canonicalized.hashCode() != subExpr.canonicalized.hashCode())
+  }
 }

From d75b7a97a95e60c35e4dd7b3d9b391073ddbd28b Mon Sep 17 00:00:00 2001
From: David Toneian <david@toneian.com>
Date: Tue, 18 Feb 2020 16:46:45 +0900
Subject: [PATCH 0107/1280] [SPARK-30859][PYSPARK][DOCS][MINOR] Fixed docstring
 syntax issues preventing proper compilation of documentation

This commit is published into the public domain.

### What changes were proposed in this pull request?
Some syntax issues in docstrings have been fixed.

### Why are the changes needed?
In some places, the documentation did not render as intended, e.g. parameter documentations were not formatted as such.

### Does this PR introduce any user-facing change?
Slight improvements in documentation.

### How was this patch tested?
Manual testing. No new Sphinx warnings arise due to this change.

Closes #27613 from DavidToneian/SPARK-30859.

Authored-by: David Toneian <david@toneian.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 504b5135d09751c15ba1f515e82d3bb6fe87ea40)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/ml/functions.py   | 1 +
 python/pyspark/ml/util.py        | 2 ++
 python/pyspark/sql/column.py     | 3 ++-
 python/pyspark/sql/functions.py  | 3 +++
 python/pyspark/sql/readwriter.py | 2 +-
 python/pyspark/sql/streaming.py  | 4 +++-
 6 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index ec164f34bc4db..65b0558b282a4 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -23,6 +23,7 @@
 def vector_to_array(col, dtype="float64"):
     """
     Converts a column of MLlib sparse/dense vectors into a column of dense arrays.
+
     :param col: A string of the column name or a Column
     :param dtype: The data type of the output array. Valid values: "float64" or "float32".
     :return: The converted column of dense arrays.
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 81b785e71f302..fe0ed569f214d 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -390,6 +390,7 @@ def saveImpl(self, path):
     def saveMetadata(instance, path, sc, extraMetadata=None, paramMap=None):
         """
         Saves metadata + Params to: path + "/metadata"
+
         - class
         - timestamp
         - sparkVersion
@@ -397,6 +398,7 @@ def saveMetadata(instance, path, sc, extraMetadata=None, paramMap=None):
         - paramMap
         - defaultParamMap (since 2.4.0)
         - (optionally, extra metadata)
+
         :param extraMetadata:  Extra metadata to be saved at same level as uid, paramMap, etc.
         :param paramMap:  If given, this is saved in the "paramMap" field.
         """
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 59d1408e26ad5..9b728b393452f 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -540,7 +540,8 @@ def alias(self, *alias, **kwargs):
 
         :param alias: strings of desired column names (collects all positional arguments passed)
         :param metadata: a dict of information to be stored in ``metadata`` attribute of the
-            corresponding :class: `StructField` (optional, keyword only argument)
+            corresponding :class:`StructField <pyspark.sql.types.StructField>` (optional, keyword
+            only argument)
 
         .. versionchanged:: 2.2
            Added optional ``metadata`` argument.
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index b7ff7264aa183..5bf8165af6a44 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -236,12 +236,14 @@ def _options_to_str(options):
     'degrees': """
                Converts an angle measured in radians to an approximately equivalent angle
                measured in degrees.
+
                :param col: angle in radians
                :return: angle in degrees, as if computed by `java.lang.Math.toDegrees()`
                """,
     'radians': """
                Converts an angle measured in degrees to an approximately equivalent angle
                measured in radians.
+
                :param col: angle in degrees
                :return: angle in radians, as if computed by `java.lang.Math.toRadians()`
                """,
@@ -2116,6 +2118,7 @@ def array_remove(col, element):
 def array_distinct(col):
     """
     Collection function: removes duplicate values from the array.
+
     :param col: name of column or expression
 
     >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data'])
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 69660395ad823..2db25875621ff 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -133,7 +133,7 @@ def options(self, **options):
 
     @since(1.4)
     def load(self, path=None, format=None, schema=None, **options):
-        """Loads data from a data source and returns it as a :class`DataFrame`.
+        """Loads data from a data source and returns it as a :class:`DataFrame`.
 
         :param path: optional string or a list of string for file-system backed data sources.
         :param format: optional string for format of the data source. Default to 'parquet'.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 5fced8aca9bdf..f989cb3e133f1 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -125,6 +125,7 @@ def lastProgress(self):
         """
         Returns the most recent :class:`StreamingQueryProgress` update of this streaming query or
         None if there were no progress updates
+
         :return: a map
         """
         lastProgress = self._jsq.lastProgress()
@@ -368,7 +369,8 @@ def options(self, **options):
 
     @since(2.0)
     def load(self, path=None, format=None, schema=None, **options):
-        """Loads a data stream from a data source and returns it as a :class`DataFrame`.
+        """Loads a data stream from a data source and returns it as a
+        :class:`DataFrame <pyspark.sql.DataFrame>`.
 
         .. note:: Evolving.
 

From 17d02f1e5005508bd9f90438483a762db8fa79ea Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 18 Feb 2020 16:10:43 +0800
Subject: [PATCH 0108/1280] [SPARK-30863][SQL] Distinguish Cast and AnsiCast in
 toString

### What changes were proposed in this pull request?

Prefix by `ansi_`  in `toString` if it's a `AnsiCast` or ansi enabled `Cast`.

E.g. run `spark.sql("select cast('51' as int)").queryExecution.analyzed` under ansi mode.

Before this PR:
```
Project [cast(51 as int) AS CAST(51 AS INT)#0]
+- OneRowRelation
```

After this PR:
```
Project [ansi_cast(51 as int) AS CAST(51 AS INT)#0]
+- OneRowRelation
```

### Why are the changes needed?

This is useful while comparing `LogicalPlan`s literally.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #27608 from Ngone51/ansi_cast_tostring.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 643a480b115d19fdc26a1edb463cf896467f890a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/expressions/Cast.scala |   5 +-
 .../postgreSQL/udf-aggregates_part1.sql.out   |  62 +++++------
 .../postgreSQL/udf-aggregates_part2.sql.out   |  12 +-
 .../results/udf/postgreSQL/udf-case.sql.out   |  10 +-
 .../results/udf/postgreSQL/udf-join.sql.out   | 104 +++++++++---------
 .../udf/postgreSQL/udf-select_having.sql.out  |  10 +-
 .../postgreSQL/udf-select_implicit.sql.out    |  44 ++++----
 7 files changed, 125 insertions(+), 122 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 05b4fbef2b697..7c4316fe08433 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -251,7 +251,10 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   def dataType: DataType
 
-  override def toString: String = s"cast($child as ${dataType.simpleString})"
+  override def toString: String = {
+    val ansi = if (ansiEnabled) "ansi_" else ""
+    s"${ansi}cast($child as ${dataType.simpleString})"
+  }
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (Cast.canCast(child.dataType, dataType)) {
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
index d65c56774eafd..adf434b1bde95 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
@@ -77,7 +77,7 @@ struct<max_324_78:float>
 -- !query
 SELECT stddev_pop(udf(b)) FROM aggtest
 -- !query schema
-struct<stddev_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
+struct<stddev_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query output
 131.10703231895047
 
@@ -85,7 +85,7 @@ struct<stddev_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT udf(stddev_samp(b)) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(stddev_samp(cast(b as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(stddev_samp(ansi_cast(b as double)) as string)) AS DOUBLE):double>
 -- !query output
 151.38936080399804
 
@@ -93,7 +93,7 @@ struct<CAST(udf(cast(stddev_samp(cast(b as double)) as string)) AS DOUBLE):doubl
 -- !query
 SELECT var_pop(udf(b)) FROM aggtest
 -- !query schema
-struct<var_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
+struct<var_pop(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query output
 17189.053923482323
 
@@ -101,7 +101,7 @@ struct<var_pop(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE)):double>
 -- !query
 SELECT udf(var_samp(b)) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(var_samp(cast(b as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(var_samp(ansi_cast(b as double)) as string)) AS DOUBLE):double>
 -- !query output
 22918.738564643096
 
@@ -109,7 +109,7 @@ struct<CAST(udf(cast(var_samp(cast(b as double)) as string)) AS DOUBLE):double>
 -- !query
 SELECT udf(stddev_pop(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(stddev_pop(cast(cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(stddev_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
 -- !query output
 131.18117242958306
 
@@ -117,7 +117,7 @@ struct<CAST(udf(cast(stddev_pop(cast(cast(b as decimal(38,0)) as double)) as str
 -- !query
 SELECT stddev_samp(CAST(udf(b) AS Decimal(38,0))) FROM aggtest
 -- !query schema
-struct<stddev_samp(CAST(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_samp(CAST(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query output
 151.47497042966097
 
@@ -125,7 +125,7 @@ struct<stddev_samp(CAST(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DECIMAL(38
 -- !query
 SELECT udf(var_pop(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(var_pop(cast(cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(var_pop(ansi_cast(ansi_cast(b as decimal(38,0)) as double)) as string)) AS DOUBLE):double>
 -- !query output
 17208.5
 
@@ -133,7 +133,7 @@ struct<CAST(udf(cast(var_pop(cast(cast(b as decimal(38,0)) as double)) as string
 -- !query
 SELECT var_samp(udf(CAST(b AS Decimal(38,0)))) FROM aggtest
 -- !query schema
-struct<var_samp(CAST(CAST(udf(cast(cast(b as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<var_samp(CAST(CAST(udf(ansi_cast(ansi_cast(b as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query output
 22944.666666666668
 
@@ -141,7 +141,7 @@ struct<var_samp(CAST(CAST(udf(cast(cast(b as decimal(38,0)) as string)) AS DECIM
 -- !query
 SELECT udf(var_pop(1.0)), var_samp(udf(2.0))
 -- !query schema
-struct<CAST(udf(cast(var_pop(cast(1.0 as double)) as string)) AS DOUBLE):double,var_samp(CAST(CAST(udf(cast(2.0 as string)) AS DECIMAL(2,1)) AS DOUBLE)):double>
+struct<CAST(udf(ansi_cast(var_pop(ansi_cast(1.0 as double)) as string)) AS DOUBLE):double,var_samp(CAST(CAST(udf(ansi_cast(2.0 as string)) AS DECIMAL(2,1)) AS DOUBLE)):double>
 -- !query output
 0.0	NaN
 
@@ -149,7 +149,7 @@ struct<CAST(udf(cast(var_pop(cast(1.0 as double)) as string)) AS DOUBLE):double,
 -- !query
 SELECT stddev_pop(udf(CAST(3.0 AS Decimal(38,0)))), stddev_samp(CAST(udf(4.0) AS Decimal(38,0)))
 -- !query schema
-struct<stddev_pop(CAST(CAST(udf(cast(cast(3.0 as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double,stddev_samp(CAST(CAST(CAST(udf(cast(4.0 as string)) AS DECIMAL(2,1)) AS DECIMAL(38,0)) AS DOUBLE)):double>
+struct<stddev_pop(CAST(CAST(udf(ansi_cast(ansi_cast(3.0 as decimal(38,0)) as string)) AS DECIMAL(38,0)) AS DOUBLE)):double,stddev_samp(CAST(CAST(CAST(udf(ansi_cast(4.0 as string)) AS DECIMAL(2,1)) AS DECIMAL(38,0)) AS DOUBLE)):double>
 -- !query output
 0.0	NaN
 
@@ -157,7 +157,7 @@ struct<stddev_pop(CAST(CAST(udf(cast(cast(3.0 as decimal(38,0)) as string)) AS D
 -- !query
 select sum(udf(CAST(null AS int))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as int) as string)) AS INT)):bigint>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as int) as string)) AS INT)):bigint>
 -- !query output
 NULL
 
@@ -165,7 +165,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS long))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as bigint) as string)) AS BIGINT)):bigint>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as bigint) as string)) AS BIGINT)):bigint>
 -- !query output
 NULL
 
@@ -173,7 +173,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS Decimal(38,0)))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,0)>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,0)>
 -- !query output
 NULL
 
@@ -181,7 +181,7 @@ NULL
 -- !query
 select sum(udf(CAST(null AS DOUBLE))) from range(1,4)
 -- !query schema
-struct<sum(CAST(udf(cast(cast(null as double) as string)) AS DOUBLE)):double>
+struct<sum(CAST(udf(ansi_cast(ansi_cast(null as double) as string)) AS DOUBLE)):double>
 -- !query output
 NULL
 
@@ -189,7 +189,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS int))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as int) as string)) AS INT)):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as int) as string)) AS INT)):double>
 -- !query output
 NULL
 
@@ -197,7 +197,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS long))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as bigint) as string)) AS BIGINT)):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as bigint) as string)) AS BIGINT)):double>
 -- !query output
 NULL
 
@@ -205,7 +205,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS Decimal(38,0)))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,4)>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as decimal(38,0)) as string)) AS DECIMAL(38,0))):decimal(38,4)>
 -- !query output
 NULL
 
@@ -213,7 +213,7 @@ NULL
 -- !query
 select avg(udf(CAST(null AS DOUBLE))) from range(1,4)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(null as double) as string)) AS DOUBLE)):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(null as double) as string)) AS DOUBLE)):double>
 -- !query output
 NULL
 
@@ -221,7 +221,7 @@ NULL
 -- !query
 select sum(CAST(udf('NaN') AS DOUBLE)) from range(1,4)
 -- !query schema
-struct<sum(CAST(CAST(udf(cast(NaN as string)) AS STRING) AS DOUBLE)):double>
+struct<sum(CAST(CAST(udf(ansi_cast(NaN as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 NaN
 
@@ -229,7 +229,7 @@ NaN
 -- !query
 select avg(CAST(udf('NaN') AS DOUBLE)) from range(1,4)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(NaN as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(NaN as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 NaN
 
@@ -238,7 +238,7 @@ NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('Infinity'), ('1')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 Infinity	NaN
 
@@ -247,7 +247,7 @@ Infinity	NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('Infinity'), ('Infinity')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 Infinity	NaN
 
@@ -256,7 +256,7 @@ Infinity	NaN
 SELECT avg(CAST(udf(x) AS DOUBLE)), var_pop(CAST(udf(x) AS DOUBLE))
 FROM (VALUES ('-Infinity'), ('Infinity')) v(x)
 -- !query schema
-struct<avg(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(cast(x as string)) AS STRING) AS DOUBLE)):double>
+struct<avg(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double,var_pop(CAST(CAST(udf(ansi_cast(x as string)) AS STRING) AS DOUBLE)):double>
 -- !query output
 NaN	NaN
 
@@ -265,7 +265,7 @@ NaN	NaN
 SELECT avg(udf(CAST(x AS DOUBLE))), udf(var_pop(CAST(x AS DOUBLE)))
 FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(cast(var_pop(cast(x as double)) as string)) AS DOUBLE):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(ansi_cast(var_pop(ansi_cast(x as double)) as string)) AS DOUBLE):double>
 -- !query output
 1.00000005E8	2.5
 
@@ -274,7 +274,7 @@ struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(u
 SELECT avg(udf(CAST(x AS DOUBLE))), udf(var_pop(CAST(x AS DOUBLE)))
 FROM (VALUES (7000000000005), (7000000000007)) v(x)
 -- !query schema
-struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(cast(var_pop(cast(x as double)) as string)) AS DOUBLE):double>
+struct<avg(CAST(udf(ansi_cast(ansi_cast(x as double) as string)) AS DOUBLE)):double,CAST(udf(ansi_cast(var_pop(ansi_cast(x as double)) as string)) AS DOUBLE):double>
 -- !query output
 7.000000000006E12	1.0
 
@@ -282,7 +282,7 @@ struct<avg(CAST(udf(cast(cast(x as double) as string)) AS DOUBLE)):double,CAST(u
 -- !query
 SELECT udf(covar_pop(b, udf(a))), covar_samp(udf(b), a) FROM aggtest
 -- !query schema
-struct<CAST(udf(cast(covar_pop(cast(b as double), cast(cast(udf(cast(a as string)) as int) as double)) as string)) AS DOUBLE):double,covar_samp(CAST(CAST(udf(cast(b as string)) AS FLOAT) AS DOUBLE), CAST(a AS DOUBLE)):double>
+struct<CAST(udf(ansi_cast(covar_pop(ansi_cast(b as double), ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double)) as string)) AS DOUBLE):double,covar_samp(CAST(CAST(udf(ansi_cast(b as string)) AS FLOAT) AS DOUBLE), CAST(a AS DOUBLE)):double>
 -- !query output
 653.6289553875104	871.5052738500139
 
@@ -290,7 +290,7 @@ struct<CAST(udf(cast(covar_pop(cast(b as double), cast(cast(udf(cast(a as string
 -- !query
 SELECT corr(b, udf(a)) FROM aggtest
 -- !query schema
-struct<corr(CAST(b AS DOUBLE), CAST(CAST(udf(cast(a as string)) AS INT) AS DOUBLE)):double>
+struct<corr(CAST(b AS DOUBLE), CAST(CAST(udf(ansi_cast(a as string)) AS INT) AS DOUBLE)):double>
 -- !query output
 0.1396345165178734
 
@@ -315,7 +315,7 @@ struct<cnt_4:bigint>
 select ten, udf(count(*)), sum(udf(four)) from onek
 group by ten order by ten
 -- !query schema
-struct<ten:int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf(cast(four as string)) AS INT)):bigint>
+struct<ten:int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf(ansi_cast(four as string)) AS INT)):bigint>
 -- !query output
 0	100	100
 1	100	200
@@ -333,7 +333,7 @@ struct<ten:int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint,sum(CAST(udf
 select ten, count(udf(four)), udf(sum(DISTINCT four)) from onek
 group by ten order by ten
 -- !query schema
-struct<ten:int,count(CAST(udf(cast(four as string)) AS INT)):bigint,CAST(udf(cast(sum(distinct cast(four as bigint)) as string)) AS BIGINT):bigint>
+struct<ten:int,count(CAST(udf(ansi_cast(four as string)) AS INT)):bigint,CAST(udf(ansi_cast(sum(distinct ansi_cast(four as bigint)) as string)) AS BIGINT):bigint>
 -- !query output
 0	100	2
 1	100	4
@@ -352,7 +352,7 @@ select ten, udf(sum(distinct four)) from onek a
 group by ten
 having exists (select 1 from onek b where udf(sum(distinct a.four)) = b.four)
 -- !query schema
-struct<ten:int,CAST(udf(cast(sum(distinct cast(four as bigint)) as string)) AS BIGINT):bigint>
+struct<ten:int,CAST(udf(ansi_cast(sum(distinct ansi_cast(four as bigint)) as string)) AS BIGINT):bigint>
 -- !query output
 0	2
 2	2
@@ -372,7 +372,7 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) = CAST(CAST(udf(cast(four as string)) AS INT) AS BIGINT))]
+Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) = CAST(CAST(udf(ansi_cast(four as string)) AS INT) AS BIGINT))]
 Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))];
 
 
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
index c10fe9b51dd72..d4941d0a0b768 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part2.sql.out
@@ -161,7 +161,7 @@ true	true	false	NULL	false	true
 -- !query
 select min(udf(unique1)) from tenk1
 -- !query schema
-struct<min(CAST(udf(cast(unique1 as string)) AS INT)):int>
+struct<min(CAST(udf(ansi_cast(unique1 as string)) AS INT)):int>
 -- !query output
 0
 
@@ -169,7 +169,7 @@ struct<min(CAST(udf(cast(unique1 as string)) AS INT)):int>
 -- !query
 select udf(max(unique1)) from tenk1
 -- !query schema
-struct<CAST(udf(cast(max(unique1) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(max(unique1) as string)) AS INT):int>
 -- !query output
 9999
 
@@ -217,7 +217,7 @@ struct<min(tenthous):int>
 -- !query
 select distinct max(udf(unique2)) from tenk1
 -- !query schema
-struct<max(CAST(udf(cast(unique2 as string)) AS INT)):int>
+struct<max(CAST(udf(ansi_cast(unique2 as string)) AS INT)):int>
 -- !query output
 9999
 
@@ -241,7 +241,7 @@ struct<max(unique2):int>
 -- !query
 select udf(max(udf(unique2))) from tenk1 order by udf(max(unique2))+1
 -- !query schema
-struct<CAST(udf(cast(max(cast(udf(cast(unique2 as string)) as int)) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(unique2 as string)) as int)) as string)) AS INT):int>
 -- !query output
 9999
 
@@ -249,7 +249,7 @@ struct<CAST(udf(cast(max(cast(udf(cast(unique2 as string)) as int)) as string))
 -- !query
 select t1.max_unique2, udf(g) from (select max(udf(unique2)) as max_unique2 FROM tenk1) t1 LATERAL VIEW explode(array(1,2,3)) t2 AS g order by g desc
 -- !query schema
-struct<max_unique2:int,CAST(udf(cast(g as string)) AS INT):int>
+struct<max_unique2:int,CAST(udf(ansi_cast(g as string)) AS INT):int>
 -- !query output
 9999	3
 9999	2
@@ -259,6 +259,6 @@ struct<max_unique2:int,CAST(udf(cast(g as string)) AS INT):int>
 -- !query
 select udf(max(100)) from tenk1
 -- !query schema
-struct<CAST(udf(cast(max(100) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(max(100) as string)) AS INT):int>
 -- !query output
 100
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
index 04c4f54b02a3e..6c733e916d734 100755
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-case.sql.out
@@ -176,7 +176,7 @@ struct<None:string,NULL on no matches:int>
 -- !query
 SELECT CASE WHEN udf(1=0) THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN CAST(udf(cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<CASE WHEN CAST(udf(ansi_cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
 -- !query output
 1.0
 
@@ -184,7 +184,7 @@ struct<CASE WHEN CAST(udf(cast((1 = 0) as string)) AS BOOLEAN) THEN (CAST(1 AS D
 -- !query
 SELECT CASE 1 WHEN 0 THEN 1/udf(0) WHEN 1 THEN 1 ELSE 2/0 END
 -- !query schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(cast(0 as string)) AS INT) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
+struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(ansi_cast(0 as string)) AS INT) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
 -- !query output
 1.0
 
@@ -192,7 +192,7 @@ struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(cast(0 as strin
 -- !query
 SELECT CASE WHEN i > 100 THEN udf(1/0) ELSE udf(0) END FROM case_tbl
 -- !query schema
-struct<CASE WHEN (i > 100) THEN CAST(udf(cast((cast(1 as double) / cast(0 as double)) as string)) AS DOUBLE) ELSE CAST(CAST(udf(cast(0 as string)) AS INT) AS DOUBLE) END:double>
+struct<CASE WHEN (i > 100) THEN CAST(udf(ansi_cast((ansi_cast(1 as double) / ansi_cast(0 as double)) as string)) AS DOUBLE) ELSE CAST(CAST(udf(ansi_cast(0 as string)) AS INT) AS DOUBLE) END:double>
 -- !query output
 0.0
 0.0
@@ -203,7 +203,7 @@ struct<CASE WHEN (i > 100) THEN CAST(udf(cast((cast(1 as double) / cast(0 as dou
 -- !query
 SELECT CASE 'a' WHEN 'a' THEN udf(1) ELSE udf(2) END
 -- !query schema
-struct<CASE WHEN (a = a) THEN CAST(udf(cast(1 as string)) AS INT) ELSE CAST(udf(cast(2 as string)) AS INT) END:int>
+struct<CASE WHEN (a = a) THEN CAST(udf(ansi_cast(1 as string)) AS INT) ELSE CAST(udf(ansi_cast(2 as string)) AS INT) END:int>
 -- !query output
 1
 
@@ -294,7 +294,7 @@ struct<i:int,f:double>
 SELECT udf(COALESCE(a.f, b.i, b.j))
   FROM CASE_TBL a, CASE2_TBL b
 -- !query schema
-struct<CAST(udf(cast(coalesce(f, cast(i as double), cast(j as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast(coalesce(f, ansi_cast(i as double), ansi_cast(j as double)) as string)) AS DOUBLE):double>
 -- !query output
 -30.3
 -30.3
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
index f113aee6d3b51..3cc14ff04b1ba 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
@@ -243,7 +243,7 @@ struct<>
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t)
   FROM J1_TBL AS tx
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -262,7 +262,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf(udf('')) AS `xxx`, udf(udf(i)), udf(j), udf(t)
   FROM J1_TBL tx
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -281,7 +281,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, a, udf(udf(b)), c
   FROM J1_TBL AS t1 (a, b, c)
 -- !query schema
-struct<xxx:string,a:int,CAST(udf(cast(cast(udf(cast(b as string)) as int) as string)) AS INT):int,c:string>
+struct<xxx:string,a:int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string)) as int) as string)) AS INT):int,c:string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -300,7 +300,7 @@ struct<xxx:string,a:int,CAST(udf(cast(cast(udf(cast(b as string)) as int) as str
 SELECT udf('') AS `xxx`, udf(a), udf(b), udf(udf(c))
   FROM J1_TBL t1 (a, b, c)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(c as string)) as string) as string)) AS STRING):string>
+struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(c as string)) as string) as string)) AS STRING):string>
 -- !query output
 	0	NULL	zero
 	1	4	one
@@ -319,7 +319,7 @@ struct<xxx:string,CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(b as str
 SELECT udf('') AS `xxx`, udf(a), b, udf(c), udf(d), e
   FROM J1_TBL t1 (a, b, c), J2_TBL t2 (d, e)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(a as string)) AS INT):int,b:int,CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(d as string)) AS INT):int,e:int>
+struct<xxx:string,CAST(udf(ansi_cast(a as string)) AS INT):int,b:int,CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(d as string)) AS INT):int,e:int>
 -- !query output
 	0	NULL	zero	0	NULL
 	0	NULL	zero	1	-1
@@ -543,7 +543,7 @@ Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line
 SELECT udf('') AS `xxx`, udf(t1.i) AS i, udf(k), udf(t)
   FROM J1_TBL t1 CROSS JOIN J2_TBL t2
 -- !query schema
-struct<xxx:string,i:int,CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string>
+struct<xxx:string,i:int,CAST(udf(ansi_cast(k as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string>
 -- !query output
 	0	-1	zero
 	0	-3	zero
@@ -651,7 +651,7 @@ SELECT udf(udf('')) AS `xxx`, udf(udf(ii)) AS ii, udf(udf(tt)) AS tt, udf(udf(kk
   FROM (J1_TBL CROSS JOIN J2_TBL)
     AS tx (ii, jj, tt, ii2, kk)
 -- !query schema
-struct<xxx:string,ii:int,tt:string,CAST(udf(cast(cast(udf(cast(kk as string)) as int) as string)) AS INT):int>
+struct<xxx:string,ii:int,tt:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(kk as string)) as int) as string)) AS INT):int>
 -- !query output
 	0	zero	-1
 	0	zero	-3
@@ -758,7 +758,7 @@ struct<xxx:string,ii:int,tt:string,CAST(udf(cast(cast(udf(cast(kk as string)) as
 SELECT udf('') AS `xxx`, udf(udf(j1_tbl.i)), udf(j), udf(t), udf(a.i), udf(a.k), udf(b.i),  udf(b.k)
   FROM J1_TBL CROSS JOIN J2_TBL a CROSS JOIN J2_TBL b
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	0	NULL	0	NULL
 	0	NULL	zero	0	NULL	1	-1
@@ -1657,7 +1657,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(i) AS i, udf(j), udf(t) AS t, udf(k)
   FROM J1_TBL INNER JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,i:int,CAST(udf(cast(j as string)) AS INT):int,t:string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,i:int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1672,7 +1672,7 @@ struct<xxx:string,i:int,CAST(udf(cast(j as string)) AS INT):int,t:string,CAST(ud
 SELECT udf(udf('')) AS `xxx`, udf(i), udf(j) AS j, udf(t), udf(k) AS k
   FROM J1_TBL JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,j:int,CAST(udf(cast(t as string)) AS STRING):string,k:int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,j:int,CAST(udf(ansi_cast(t as string)) AS STRING):string,k:int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1703,7 +1703,7 @@ struct<xxx:string,a:int,b:int,c:string,d:int>
 SELECT udf(udf('')) AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL NATURAL JOIN J2_TBL
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1718,7 +1718,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(udf(udf(a))) AS a, udf(b), udf(c), udf(d)
   FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (a, d)
 -- !query schema
-struct<xxx:string,a:int,CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(d as string)) AS INT):int>
+struct<xxx:string,a:int,CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(d as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1733,7 +1733,7 @@ struct<xxx:string,a:int,CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c
 SELECT udf('') AS `xxx`, udf(udf(a)), udf(udf(b)), udf(udf(c)) AS c, udf(udf(udf(d))) AS d
   FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (d, a)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(b as string)) as int) as string)) AS INT):int,c:string,d:int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(b as string)) as int) as string)) AS INT):int,c:string,d:int>
 -- !query output
 	0	NULL	zero	NULL
 	2	3	two	2
@@ -1744,7 +1744,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(a as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(J1_TBL.i), udf(udf(J1_TBL.j)), udf(J1_TBL.t), udf(J2_TBL.i), udf(J2_TBL.k)
   FROM J1_TBL JOIN J2_TBL ON (udf(J1_TBL.i) = J2_TBL.i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(j as string)) as int) as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	0	NULL
 	1	4	one	1	-1
@@ -1759,7 +1759,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(cast(udf
 SELECT udf('') AS `xxx`, udf(udf(J1_TBL.i)), udf(udf(J1_TBL.j)), udf(udf(J1_TBL.t)), J2_TBL.i, J2_TBL.k
   FROM J1_TBL JOIN J2_TBL ON (J1_TBL.i = udf(J2_TBL.k))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(j as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(t as string)) as string) as string)) AS STRING):string,i:int,k:int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(t as string)) as string) as string)) AS STRING):string,i:int,k:int>
 -- !query output
 	0	NULL	zero	NULL	0
 	2	3	two	2	2
@@ -1770,7 +1770,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(J1_TBL.i), udf(J1_TBL.j), udf(J1_TBL.t), udf(J2_TBL.i), udf(J2_TBL.k)
   FROM J1_TBL JOIN J2_TBL ON (udf(J1_TBL.i) <= udf(udf(J2_TBL.k)))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	2	2
 	0	NULL	zero	2	4
@@ -1788,7 +1788,7 @@ SELECT udf(udf('')) AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT OUTER JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	NULL	NULL	null	NULL
 	NULL	0	zero	NULL
@@ -1810,7 +1810,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT JOIN J2_TBL USING (i)
   ORDER BY udf(i), udf(udf(k)), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	NULL	NULL	null	NULL
 	NULL	0	zero	NULL
@@ -1831,7 +1831,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(udf(i)), udf(j), udf(t), udf(k)
   FROM J1_TBL RIGHT OUTER JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(i as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1848,7 +1848,7 @@ struct<xxx:string,CAST(udf(cast(cast(udf(cast(i as string)) as int) as string))
 SELECT udf('') AS `xxx`, udf(i), udf(udf(j)), udf(t), udf(k)
   FROM J1_TBL RIGHT JOIN J2_TBL USING (i)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(j as string)) as int) as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(j as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	0	NULL	zero	NULL
 	1	4	one	-1
@@ -1866,7 +1866,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), udf(udf(t)), udf(k)
   FROM J1_TBL FULL OUTER JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(t)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(t as string)) as string) as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(t as string)) as string) as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	NULL	NULL	NULL	NULL
 	NULL	NULL	null	NULL
@@ -1890,7 +1890,7 @@ SELECT udf('') AS `xxx`, udf(i), udf(j), t, udf(udf(k))
   FROM J1_TBL FULL JOIN J2_TBL USING (i)
   ORDER BY udf(udf(i)), udf(k), udf(udf(t))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,t:string,CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,t:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(k as string)) as int) as string)) AS INT):int>
 -- !query output
 	NULL	NULL	NULL	NULL
 	NULL	NULL	null	NULL
@@ -1913,7 +1913,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(udf(k))
   FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (udf(k) = 1)
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(cast(udf(cast(k as string)) as int) as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(k as string)) as int) as string)) AS INT):int>
 -- !query output
 
 
@@ -1922,7 +1922,7 @@ struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as str
 SELECT udf('') AS `xxx`, udf(i), udf(j), udf(t), udf(k)
   FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (udf(udf(i)) = udf(1))
 -- !query schema
-struct<xxx:string,CAST(udf(cast(i as string)) AS INT):int,CAST(udf(cast(j as string)) AS INT):int,CAST(udf(cast(t as string)) AS STRING):string,CAST(udf(cast(k as string)) AS INT):int>
+struct<xxx:string,CAST(udf(ansi_cast(i as string)) AS INT):int,CAST(udf(ansi_cast(j as string)) AS INT):int,CAST(udf(ansi_cast(t as string)) AS STRING):string,CAST(udf(ansi_cast(k as string)) AS INT):int>
 -- !query output
 	1	4	one	-1
 
@@ -2052,7 +2052,7 @@ FULL JOIN
 (SELECT * FROM t3) s3
 USING (name)
 -- !query schema
-struct<CAST(udf(cast(name as string)) AS STRING):string,CAST(udf(cast(cast(udf(cast(n as string)) as int) as string)) AS INT):int,CAST(udf(cast(n as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(name as string)) AS STRING):string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(n as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(n as string)) AS INT):int>
 -- !query output
 bb	12	13
 cc	22	23
@@ -2138,7 +2138,7 @@ NATURAL FULL JOIN
     (SELECT name, udf(udf(n)) as s3_n FROM t3) as s3
   ) ss2
 -- !query schema
-struct<name:string,CAST(udf(cast(cast(udf(cast(s1_n as string)) as int) as string)) AS INT):int,CAST(udf(cast(s2_n as string)) AS INT):int,CAST(udf(cast(s3_n as string)) AS INT):int>
+struct<name:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(s1_n as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(s2_n as string)) AS INT):int,CAST(udf(ansi_cast(s3_n as string)) AS INT):int>
 -- !query output
 bb	11	12	13
 cc	NULL	22	23
@@ -2171,7 +2171,7 @@ FULL JOIN
   (SELECT name, 2 as s2_n FROM t2) as s2
 ON (udf(udf(s1_n)) = udf(s2_n))
 -- !query schema
-struct<name:string,CAST(udf(cast(s1_n as string)) AS INT):int,name:string,CAST(udf(cast(cast(udf(cast(s2_n as string)) as int) as string)) AS INT):int>
+struct<name:string,CAST(udf(ansi_cast(s1_n as string)) AS INT):int,name:string,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(s2_n as string)) as int) as string)) AS INT):int>
 -- !query output
 NULL	NULL	bb	2
 NULL	NULL	cc	2
@@ -2202,7 +2202,7 @@ struct<>
 -- !query
 select udf(udf(x1)), udf(x2) from x
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(x1 as string)) as int) as string)) AS INT):int,CAST(udf(cast(x2 as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(x1 as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(x2 as string)) AS INT):int>
 -- !query output
 1	11
 2	22
@@ -2214,7 +2214,7 @@ struct<CAST(udf(cast(cast(udf(cast(x1 as string)) as int) as string)) AS INT):in
 -- !query
 select udf(y1), udf(udf(y2)) from y
 -- !query schema
-struct<CAST(udf(cast(y1 as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(y2 as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(y1 as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(y2 as string)) as int) as string)) AS INT):int>
 -- !query output
 1	111
 2	222
@@ -2336,7 +2336,7 @@ select udf(udf(count(*))) from tenk1 a where udf(udf(unique1)) in
   (select udf(unique1) from tenk1 b join tenk1 c using (unique1)
    where udf(udf(b.unique2)) = udf(42))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
 -- !query output
 1
 
@@ -2347,7 +2347,7 @@ select udf(count(*)) from tenk1 x where
   udf(x.unique1) = 0 and
   udf(x.unique1) in (select aa.f1 from int4_tbl aa,float8_tbl bb where aa.f1=udf(udf(bb.f1)))
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1
 
@@ -2358,7 +2358,7 @@ select udf(udf(count(*))) from tenk1 x where
   udf(x.unique1) = 0 and
   udf(udf(x.unique1)) in (select udf(aa.f1) from int4_tbl aa,float8_tbl bb where udf(aa.f1)=udf(udf(bb.f1)))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(1) as string)) as bigint) as string)) AS BIGINT):bigint>
 -- !query output
 1
 
@@ -2388,7 +2388,7 @@ from
   tenk1 t5
 where udf(t4.thousand) = udf(t5.unique1) and udf(udf(ss.x1)) = t4.tenthous and udf(ss.x2) = udf(udf(t5.stringu1))
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1000
 
@@ -2400,7 +2400,7 @@ select udf(a.f1), udf(b.f1), udf(t.thousand), udf(t.tenthous) from
   (select udf(sum(udf(f1))) as f1 from int4_tbl i4b) b
 where b.f1 = udf(t.thousand) and udf(a.f1) = udf(b.f1) and udf((udf(a.f1)+udf(b.f1)+999)) = udf(udf(t.tenthous))
 -- !query schema
-struct<CAST(udf(cast(f1 as string)) AS BIGINT):bigint,CAST(udf(cast(f1 as string)) AS BIGINT):bigint,CAST(udf(cast(thousand as string)) AS INT):int,CAST(udf(cast(tenthous as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(f1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(f1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(thousand as string)) AS INT):int,CAST(udf(ansi_cast(tenthous as string)) AS INT):int>
 -- !query output
 
 
@@ -2441,7 +2441,7 @@ select udf(count(*)) from
   (select * from tenk1 y order by udf(y.unique2)) y
   on udf(x.thousand) = y.unique2 and x.twothousand = udf(y.hundred) and x.fivethous = y.unique2
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 10000
 
@@ -2530,7 +2530,7 @@ struct<tt1_id:int,joincol:int,tt2_id:int,joincol:int>
 select udf(count(*)) from tenk1 a, tenk1 b
   where udf(a.hundred) = b.thousand and udf(udf((b.fivethous % 10)) < 10)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 100000
 
@@ -2727,7 +2727,7 @@ from tenk1 a left join tenk1 b on a.unique2 = udf(b.tenthous)
 where udf(a.unique1) = 42 and
       ((udf(b.unique2) is null and udf(a.ten) = 2) or udf(udf(b.hundred)) = udf(udf(3)))
 -- !query schema
-struct<CAST(udf(cast(unique2 as string)) AS INT):int,CAST(udf(cast(ten as string)) AS INT):int,CAST(udf(cast(tenthous as string)) AS INT):int,CAST(udf(cast(unique2 as string)) AS INT):int,CAST(udf(cast(hundred as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(ten as string)) AS INT):int,CAST(udf(ansi_cast(tenthous as string)) AS INT):int,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(hundred as string)) AS INT):int>
 -- !query output
 
 
@@ -2761,7 +2761,7 @@ select udf(t1.q2), udf(count(t2.*))
 from int8_tbl t1 left join int8_tbl t2 on (udf(udf(t1.q2)) = t2.q1)
 group by udf(t1.q2) order by 1
 -- !query schema
-struct<CAST(udf(cast(q2 as string)) AS BIGINT):bigint,CAST(udf(cast(count(q1, q2) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(count(q1, q2) as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2774,7 +2774,7 @@ select udf(udf(t1.q2)), udf(count(t2.*))
 from int8_tbl t1 left join (select * from int8_tbl) t2 on (udf(udf(t1.q2)) = udf(t2.q1))
 group by udf(udf(t1.q2)) order by 1
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(q2 as string)) as bigint) as string)) AS BIGINT):bigint,CAST(udf(cast(count(q1, q2) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(q2 as string)) as bigint) as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(count(q1, q2) as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2789,7 +2789,7 @@ from int8_tbl t1 left join
   on (udf(t1.q2) = udf(t2.q1))
 group by t1.q2 order by 1
 -- !query schema
-struct<q2:bigint,CAST(udf(cast(cast(udf(cast(count(q1, q2) as string)) as bigint) as string)) AS BIGINT):bigint>
+struct<q2:bigint,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(count(q1, q2) as string)) as bigint) as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	0
 123	2
@@ -2838,7 +2838,7 @@ from c left join
   on (udf(udf(c.a)) = udf(ss.code))
 order by c.name
 -- !query schema
-struct<CAST(udf(cast(name as string)) AS STRING):string,CAST(udf(cast(code as string)) AS STRING):string,CAST(udf(cast(b_cnt as string)) AS BIGINT):bigint,CAST(udf(cast(const as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(name as string)) AS STRING):string,CAST(udf(ansi_cast(code as string)) AS STRING):string,CAST(udf(ansi_cast(b_cnt as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(const as string)) AS INT):int>
 -- !query output
 A	p	2	-1
 B	q	0	-1
@@ -2884,7 +2884,7 @@ LEFT JOIN
 ) sub2
 ON sub1.key1 = udf(udf(sub2.key3))
 -- !query schema
-struct<key1:int,key3:int,CAST(udf(cast(value2 as string)) AS INT):int,value3:int>
+struct<key1:int,key3:int,CAST(udf(ansi_cast(value2 as string)) AS INT):int,value3:int>
 -- !query output
 1	1	1	1
 
@@ -2898,7 +2898,7 @@ SELECT udf(qq), udf(udf(unique1))
   USING (qq)
   INNER JOIN tenk1 c ON udf(qq) = udf(unique2)
 -- !query schema
-struct<CAST(udf(cast(qq as string)) AS BIGINT):bigint,CAST(udf(cast(cast(udf(cast(unique1 as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(qq as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(unique1 as string)) as int) as string)) AS INT):int>
 -- !query output
 123	4596
 123	4596
@@ -2948,7 +2948,7 @@ from nt3 as nt3
     on udf(ss2.id) = nt3.nt2_id
 where udf(nt3.id) = 1 and udf(ss2.b3)
 -- !query schema
-struct<CAST(udf(cast(id as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(id as string)) AS INT):int>
 -- !query output
 1
 
@@ -3008,7 +3008,7 @@ select udf(count(*)) from
   left join tenk1 c on udf(a.unique2) = udf(b.unique1) and udf(c.thousand) = udf(udf(a.thousand))
   join int4_tbl on udf(b.thousand) = f1
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 10
 
@@ -3021,7 +3021,7 @@ select udf(b.unique1) from
   right join int4_tbl i2 on udf(udf(i2.f1)) = udf(b.tenthous)
   order by udf(1)
 -- !query schema
-struct<CAST(udf(cast(unique1 as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int>
 -- !query output
 NULL
 NULL
@@ -3039,7 +3039,7 @@ select * from
 where udf(fault) = udf(122)
 order by udf(fault)
 -- !query schema
-struct<CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(q1 as string)) AS BIGINT):bigint,fault:bigint>
+struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,fault:bigint>
 -- !query output
 NULL	123	122
 
@@ -3049,7 +3049,7 @@ select udf(q1), udf(unique2), udf(thousand), udf(hundred)
   from int8_tbl a left join tenk1 b on udf(q1) = udf(unique2)
   where udf(coalesce(thousand,123)) = udf(q1) and udf(q1) = udf(udf(coalesce(hundred,123)))
 -- !query schema
-struct<CAST(udf(cast(q1 as string)) AS BIGINT):bigint,CAST(udf(cast(unique2 as string)) AS INT):int,CAST(udf(cast(thousand as string)) AS INT):int,CAST(udf(cast(hundred as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CAST(udf(ansi_cast(thousand as string)) AS INT):int,CAST(udf(ansi_cast(hundred as string)) AS INT):int>
 -- !query output
 
 
@@ -3059,7 +3059,7 @@ select udf(f1), udf(unique2), case when udf(udf(unique2)) is null then udf(f1) e
   from int4_tbl a left join tenk1 b on udf(f1) = udf(udf(unique2))
   where (case when udf(unique2) is null then udf(f1) else 0 end) = 0
 -- !query schema
-struct<CAST(udf(cast(f1 as string)) AS INT):int,CAST(udf(cast(unique2 as string)) AS INT):int,CASE WHEN (CAST(udf(cast(cast(udf(cast(unique2 as string)) as int) as string)) AS INT) IS NULL) THEN CAST(udf(cast(f1 as string)) AS INT) ELSE 0 END:int>
+struct<CAST(udf(ansi_cast(f1 as string)) AS INT):int,CAST(udf(ansi_cast(unique2 as string)) AS INT):int,CASE WHEN (CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(unique2 as string)) as int) as string)) AS INT) IS NULL) THEN CAST(udf(ansi_cast(f1 as string)) AS INT) ELSE 0 END:int>
 -- !query output
 0	0	0
 
@@ -3069,7 +3069,7 @@ select udf(a.unique1), udf(b.unique1), udf(c.unique1), udf(coalesce(b.twothousan
   from tenk1 a left join tenk1 b on udf(b.thousand) = a.unique1                       left join tenk1 c on udf(c.unique2) = udf(coalesce(b.twothousand, a.twothousand))
   where a.unique2 < udf(10) and udf(udf(coalesce(b.twothousand, a.twothousand))) = udf(44)
 -- !query schema
-struct<CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(unique1 as string)) AS INT):int,CAST(udf(cast(coalesce(twothousand, twothousand) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(unique1 as string)) AS INT):int,CAST(udf(ansi_cast(coalesce(twothousand, twothousand) as string)) AS INT):int>
 -- !query output
 
 
@@ -3107,7 +3107,7 @@ select udf(a.q2), udf(b.q1)
   from int8_tbl a left join int8_tbl b on udf(a.q2) = coalesce(b.q1, 1)
   where udf(udf(coalesce(b.q1, 1)) > 0)
 -- !query schema
-struct<CAST(udf(cast(q2 as string)) AS BIGINT):bigint,CAST(udf(cast(q1 as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint>
 -- !query output
 -4567890123456789	NULL
 123	123
@@ -3237,7 +3237,7 @@ SELECT * FROM
      FROM int8_tbl LEFT JOIN innertab ON udf(udf(q2)) = id) ss2
   ON true
 -- !query schema
-struct<x:int,CAST(udf(cast(q1 as string)) AS BIGINT):bigint,CAST(udf(cast(q2 as string)) AS BIGINT):bigint,y:bigint>
+struct<x:int,CAST(udf(ansi_cast(q1 as string)) AS BIGINT):bigint,CAST(udf(ansi_cast(q2 as string)) AS BIGINT):bigint,y:bigint>
 -- !query output
 1	123	456	123
 1	123	4567890123456789	123
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
index 68113afdfae30..bb108a29f75ce 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
@@ -94,7 +94,7 @@ struct<>
 SELECT udf(b), udf(c) FROM test_having
 	GROUP BY b, c HAVING udf(count(*)) = 1 ORDER BY udf(b), udf(c)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS STRING):string>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string>
 -- !query output
 1	XXXX
 3	bbbb
@@ -104,7 +104,7 @@ struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS ST
 SELECT udf(b), udf(c) FROM test_having
 	GROUP BY b, c HAVING udf(b) = 3 ORDER BY udf(b), udf(c)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(c as string)) AS STRING):string>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(c as string)) AS STRING):string>
 -- !query output
 3	BBBB
 3	bbbb
@@ -115,7 +115,7 @@ SELECT udf(c), max(udf(a)) FROM test_having
 	GROUP BY c HAVING udf(count(*)) > 2 OR udf(min(a)) = udf(max(a))
 	ORDER BY c
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string,max(CAST(udf(cast(a as string)) AS INT)):int>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,max(CAST(udf(ansi_cast(a as string)) AS INT)):int>
 -- !query output
 XXXX	0
 bbbb	5
@@ -124,7 +124,7 @@ bbbb	5
 -- !query
 SELECT udf(udf(min(udf(a)))), udf(udf(max(udf(a)))) FROM test_having HAVING udf(udf(min(udf(a)))) = udf(udf(max(udf(a))))
 -- !query schema
-struct<CAST(udf(cast(cast(udf(cast(min(cast(udf(cast(a as string)) as int)) as string)) as int) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(max(cast(udf(cast(a as string)) as int)) as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) as int) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(max(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) as int) as string)) AS INT):int>
 -- !query output
 
 
@@ -132,7 +132,7 @@ struct<CAST(udf(cast(cast(udf(cast(min(cast(udf(cast(a as string)) as int)) as s
 -- !query
 SELECT udf(min(udf(a))), udf(udf(max(a))) FROM test_having HAVING udf(min(a)) < udf(max(udf(a)))
 -- !query schema
-struct<CAST(udf(cast(min(cast(udf(cast(a as string)) as int)) as string)) AS INT):int,CAST(udf(cast(cast(udf(cast(max(a) as string)) as int) as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(min(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) AS INT):int,CAST(udf(ansi_cast(ansi_cast(udf(ansi_cast(max(a) as string)) as int) as string)) AS INT):int>
 -- !query output
 0	9
 
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
index 11cb682ee1494..66e6c20a2f6f2 100755
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_implicit.sql.out
@@ -95,7 +95,7 @@ SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY
 udf(test_missing_target.c)
 ORDER BY udf(c)
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 ABAB	2
 BBBB	2
@@ -109,7 +109,7 @@ cccc	2
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(test_missing_target.c)
 ORDER BY udf(c)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 2
 2
@@ -125,13 +125,13 @@ SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(a) ORDER BY udf(b)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [CAST(udf(cast(count(1) as string)) AS BIGINT)]; line 1 pos 75
+cannot resolve '`b`' given input columns: [CAST(udf(ansi_cast(count(1) as string)) AS BIGINT)]; line 1 pos 75
 
 
 -- !query
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1
 2
@@ -143,7 +143,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 SELECT udf(test_missing_target.b), udf(count(*))
   FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1	1
 2	2
@@ -154,7 +154,7 @@ struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(count(1) as string)
 -- !query
 SELECT udf(c) FROM test_missing_target ORDER BY udf(a)
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string>
 -- !query output
 XXXX
 ABAB
@@ -171,7 +171,7 @@ CCCC
 -- !query
 SELECT udf(count(*)) FROM test_missing_target GROUP BY udf(b) ORDER BY udf(b) desc
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 4
 3
@@ -182,7 +182,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT udf(count(*)) FROM test_missing_target ORDER BY udf(1) desc
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 10
 
@@ -190,7 +190,7 @@ struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
 -- !query
 SELECT udf(c), udf(count(*)) FROM test_missing_target GROUP BY 1 ORDER BY 1
 -- !query schema
-struct<CAST(udf(cast(c as string)) AS STRING):string,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(c as string)) AS STRING):string,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 ABAB	2
 BBBB	2
@@ -224,7 +224,7 @@ Reference 'b' is ambiguous, could be: x.b, y.b.; line 3 pos 14
 SELECT udf(a), udf(a) FROM test_missing_target
 	ORDER BY udf(a)
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(a as string)) AS INT):int,CAST(udf(ansi_cast(a as string)) AS INT):int>
 -- !query output
 0	0
 1	1
@@ -242,7 +242,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int,CAST(udf(cast(a as string)) AS IN
 SELECT udf(udf(a)/2), udf(udf(a)/2) FROM test_missing_target
 	ORDER BY udf(udf(a)/2)
 -- !query schema
-struct<CAST(udf(cast((cast(cast(udf(cast(a as string)) as int) as double) / cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(cast((cast(cast(udf(cast(a as string)) as int) as double) / cast(2 as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast((ansi_cast(ansi_cast(udf(ansi_cast(a as string)) as int) as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -260,7 +260,7 @@ struct<CAST(udf(cast((cast(cast(udf(cast(a as string)) as int) as double) / cast
 SELECT udf(a/2), udf(a/2) FROM test_missing_target
 	GROUP BY udf(a/2) ORDER BY udf(a/2)
 -- !query schema
-struct<CAST(udf(cast((cast(a as double) / cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(cast((cast(a as double) / cast(2 as double)) as string)) AS DOUBLE):double>
+struct<CAST(udf(ansi_cast((ansi_cast(a as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast((ansi_cast(a as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double>
 -- !query output
 0.0	0.0
 0.5	0.5
@@ -279,7 +279,7 @@ SELECT udf(x.b), udf(count(*)) FROM test_missing_target x, test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b) ORDER BY udf(x.b)
 -- !query schema
-struct<CAST(udf(cast(b as string)) AS INT):int,CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(b as string)) AS INT):int,CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1	1
 2	2
@@ -292,7 +292,7 @@ SELECT udf(count(*)) FROM test_missing_target x, test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b) ORDER BY udf(x.b)
 -- !query schema
-struct<CAST(udf(cast(count(1) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(1) as string)) AS BIGINT):bigint>
 -- !query output
 1
 2
@@ -305,7 +305,7 @@ SELECT udf(a%2), udf(count(udf(b))) FROM test_missing_target
 GROUP BY udf(test_missing_target.a%2)
 ORDER BY udf(test_missing_target.a%2)
 -- !query schema
-struct<CAST(udf(cast((a % 2) as string)) AS INT):int,CAST(udf(cast(count(cast(udf(cast(b as string)) as int)) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast((a % 2) as string)) AS INT):int,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(b as string)) as int)) as string)) AS BIGINT):bigint>
 -- !query output
 0	5
 1	5
@@ -316,7 +316,7 @@ SELECT udf(count(c)) FROM test_missing_target
 GROUP BY udf(lower(test_missing_target.c))
 ORDER BY udf(lower(test_missing_target.c))
 -- !query schema
-struct<CAST(udf(cast(count(c) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(c) as string)) AS BIGINT):bigint>
 -- !query output
 2
 3
@@ -330,13 +330,13 @@ SELECT udf(count(udf(a))) FROM test_missing_target GROUP BY udf(a) ORDER BY udf(
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`b`' given input columns: [CAST(udf(cast(count(cast(udf(cast(a as string)) as int)) as string)) AS BIGINT)]; line 1 pos 80
+cannot resolve '`b`' given input columns: [CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(a as string)) as int)) as string)) AS BIGINT)]; line 1 pos 80
 
 
 -- !query
 SELECT udf(count(b)) FROM test_missing_target GROUP BY udf(b/2) ORDER BY udf(b/2)
 -- !query schema
-struct<CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
 -- !query output
 1
 2
@@ -348,7 +348,7 @@ struct<CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
 SELECT udf(lower(test_missing_target.c)), udf(count(udf(c)))
   FROM test_missing_target GROUP BY udf(lower(c)) ORDER BY udf(lower(c))
 -- !query schema
-struct<CAST(udf(cast(lower(c) as string)) AS STRING):string,CAST(udf(cast(count(cast(udf(cast(c as string)) as string)) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(lower(c) as string)) AS STRING):string,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(c as string)) as string)) as string)) AS BIGINT):bigint>
 -- !query output
 abab	2
 bbbb	3
@@ -359,7 +359,7 @@ xxxx	1
 -- !query
 SELECT udf(a) FROM test_missing_target ORDER BY udf(upper(udf(d)))
 -- !query schema
-struct<CAST(udf(cast(a as string)) AS INT):int>
+struct<CAST(udf(ansi_cast(a as string)) AS INT):int>
 -- !query output
 0
 1
@@ -377,7 +377,7 @@ struct<CAST(udf(cast(a as string)) AS INT):int>
 SELECT udf(count(b)) FROM test_missing_target
 	GROUP BY udf((b + 1) / 2) ORDER BY udf((b + 1) / 2) desc
 -- !query schema
-struct<CAST(udf(cast(count(b) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast(count(b) as string)) AS BIGINT):bigint>
 -- !query output
 4
 3
@@ -402,7 +402,7 @@ test_missing_target y
 	WHERE udf(x.a) = udf(y.a)
 	GROUP BY udf(x.b/2) ORDER BY udf(x.b/2)
 -- !query schema
-struct<CAST(udf(cast((cast(b as double) / cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(cast(count(cast(udf(cast(b as string)) as int)) as string)) AS BIGINT):bigint>
+struct<CAST(udf(ansi_cast((ansi_cast(b as double) / ansi_cast(2 as double)) as string)) AS DOUBLE):double,CAST(udf(ansi_cast(count(ansi_cast(udf(ansi_cast(b as string)) as int)) as string)) AS BIGINT):bigint>
 -- !query output
 0.5	1
 1.0	2

From b76bc0b1b8b2abd00a84f805af90ca4c5925faaa Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 18 Feb 2020 20:39:50 +0800
Subject: [PATCH 0109/1280] [SPARK-30812][SQL][CORE] Revise boolean config name
 to comply with new config naming policy

### What changes were proposed in this pull request?

Revise below config names to comply with [new config naming policy](http://apache-spark-developers-list.1001551.n3.nabble.com/DISCUSS-naming-policy-of-Spark-configs-td28875.html):

SQL:
* spark.sql.execution.subquery.reuse.enabled / [SPARK-27083](https://issues.apache.org/jira/browse/SPARK-27083)
* spark.sql.legacy.allowNegativeScaleOfDecimal.enabled / [SPARK-30252](https://issues.apache.org/jira/browse/SPARK-30252)
* spark.sql.adaptive.optimizeSkewedJoin.enabled / [SPARK-29544](https://issues.apache.org/jira/browse/SPARK-29544)
* spark.sql.legacy.property.nonReserved / [SPARK-30183](https://issues.apache.org/jira/browse/SPARK-30183)
* spark.sql.streaming.forceDeleteTempCheckpointLocation.enabled / [SPARK-26389](https://issues.apache.org/jira/browse/SPARK-26389)
* spark.sql.analyzer.failAmbiguousSelfJoin.enabled / [SPARK-28344](https://issues.apache.org/jira/browse/SPARK-28344)
* spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled / [SPARK-30074](https://issues.apache.org/jira/browse/SPARK-30074)
* spark.sql.execution.pandas.arrowSafeTypeConversion / [SPARK-25811](https://issues.apache.org/jira/browse/SPARK-25811)
* spark.sql.legacy.looseUpcast / [SPARK-24586](https://issues.apache.org/jira/browse/SPARK-24586)
* spark.sql.legacy.arrayExistsFollowsThreeValuedLogic / [SPARK-28052](https://issues.apache.org/jira/browse/SPARK-28052)
* spark.sql.sources.ignoreDataLocality.enabled / [SPARK-29189](https://issues.apache.org/jira/browse/SPARK-29189)
* spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch.enabled / [SPARK-9853](https://issues.apache.org/jira/browse/SPARK-9853)

CORE:
* spark.eventLog.erasureCoding.enabled / [SPARK-25855](https://issues.apache.org/jira/browse/SPARK-25855)
* spark.shuffle.readHostLocalDisk.enabled / [SPARK-30235](https://issues.apache.org/jira/browse/SPARK-30235)
* spark.scheduler.listenerbus.logSlowEvent.enabled / [SPARK-29001](https://issues.apache.org/jira/browse/SPARK-29001)
* spark.resources.coordinate.enable / [SPARK-27371](https://issues.apache.org/jira/browse/SPARK-27371)
* spark.eventLog.logStageExecutorMetrics.enabled / [SPARK-23429](https://issues.apache.org/jira/browse/SPARK-23429)

### Why are the changes needed?

To comply with the config naming policy.

### Does this PR introduce any user-facing change?

No. Configurations listed above are all newly added in Spark 3.0.

### How was this patch tested?

Pass Jenkins.

Closes #27563 from Ngone51/revise_boolean_conf_name.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 68d7edf9497bea2f73707d32ab55dd8e53088e7c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/internal/config/package.scala       | 30 +++++++++----------
 .../scheduler/EventLoggingListener.scala      |  2 +-
 .../application_1553914137147_0018            |  2 +-
 docs/configuration.md                         |  8 ++---
 docs/monitoring.md                            |  2 +-
 docs/pyspark-migration-guide.md               |  2 +-
 docs/spark-standalone.md                      |  8 ++---
 docs/sql-migration-guide.md                   |  8 ++---
 python/pyspark/sql/pandas/serializers.py      |  2 +-
 python/pyspark/sql/tests/test_pandas_udf.py   |  8 ++---
 python/pyspark/sql/tests/test_types.py        |  4 +--
 python/pyspark/worker.py                      |  2 +-
 .../apache/spark/sql/internal/SQLConf.scala   | 24 +++++++--------
 .../apache/spark/sql/types/DecimalType.scala  |  2 +-
 14 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 652db2bdf90a9..3f36e61fe1127 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -38,7 +38,7 @@ package object config {
   private[spark] val LISTENER_BUS_EVENT_QUEUE_PREFIX = "spark.scheduler.listenerbus.eventqueue"
 
   private[spark] val SPARK_RESOURCES_COORDINATE =
-    ConfigBuilder("spark.resources.coordinate.enable")
+    ConfigBuilder("spark.resources.coordinateResourcesInStandalone")
       .doc("Whether to coordinate resources automatically among workers/drivers(client only) " +
         "in Standalone. If false, the user is responsible for configuring different resources " +
         "for workers/drivers that run on the same host.")
@@ -159,7 +159,7 @@ package object config {
     .createWithDefaultString("100k")
 
   private[spark] val EVENT_LOG_STAGE_EXECUTOR_METRICS =
-    ConfigBuilder("spark.eventLog.logStageExecutorMetrics.enabled")
+    ConfigBuilder("spark.eventLog.logStageExecutorMetrics")
       .doc("Whether to write per-stage peaks of executor metrics (for each executor) " +
         "to the event log.")
       .booleanConf
@@ -632,7 +632,7 @@ package object config {
       .createWithDefault(128)
 
   private[spark] val LISTENER_BUS_LOG_SLOW_EVENT_ENABLED =
-    ConfigBuilder("spark.scheduler.listenerbus.logSlowEvent.enabled")
+    ConfigBuilder("spark.scheduler.listenerbus.logSlowEvent")
       .internal()
       .doc("When enabled, log the event that takes too much time to process. This helps us " +
         "discover the event types that cause performance bottlenecks. The time threshold is " +
@@ -644,7 +644,7 @@ package object config {
     ConfigBuilder("spark.scheduler.listenerbus.logSlowEvent.threshold")
       .internal()
       .doc("The time threshold of whether a event is considered to be taking too much time to " +
-        "process. Log the event if spark.scheduler.listenerbus.logSlowEvent.enabled is true.")
+        s"process. Log the event if ${LISTENER_BUS_LOG_SLOW_EVENT_ENABLED.key} is true.")
       .timeConf(TimeUnit.NANOSECONDS)
       .createWithDefaultString("1s")
 
@@ -1115,16 +1115,6 @@ package object config {
       .booleanConf
       .createWithDefault(false)
 
-  private[spark] val STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE =
-    ConfigBuilder("spark.storage.localDiskByExecutors.cacheSize")
-      .doc("The max number of executors for which the local dirs are stored. This size is " +
-        "both applied for the driver and both for the executors side to avoid having an " +
-        "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
-        "persisted RDD blocks or shuffle blocks (when `spark.shuffle.readHostLocalDisk.enabled` " +
-        "is set) from the same host.")
-      .intConf
-      .createWithDefault(1000)
-
   private[spark] val SHUFFLE_SYNC =
     ConfigBuilder("spark.shuffle.sync")
       .doc("Whether to force outstanding writes to disk.")
@@ -1161,13 +1151,23 @@ package object config {
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED =
-    ConfigBuilder("spark.shuffle.readHostLocalDisk.enabled")
+    ConfigBuilder("spark.shuffle.readHostLocalDisk")
       .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled), shuffle " +
         "blocks requested from those block managers which are running on the same host are read " +
         "from the disk directly instead of being fetched as remote blocks over the network.")
       .booleanConf
       .createWithDefault(true)
 
+  private[spark] val STORAGE_LOCAL_DISK_BY_EXECUTORS_CACHE_SIZE =
+    ConfigBuilder("spark.storage.localDiskByExecutors.cacheSize")
+      .doc("The max number of executors for which the local dirs are stored. This size is " +
+        "both applied for the driver and both for the executors side to avoid having an " +
+        "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
+        s"persisted RDD blocks or shuffle blocks " +
+        s"(when `${SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED.key}` is set) from the same host.")
+      .intConf
+      .createWithDefault(1000)
+
   private[spark] val MEMORY_MAP_LIMIT_FOR_TESTS =
     ConfigBuilder("spark.storage.memoryMapLimitForTests")
       .internal()
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 8c23388b37a3d..24e2a5e4d4a62 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -40,7 +40,7 @@ import org.apache.spark.util.{JsonProtocol, Utils}
  *   spark.eventLog.enabled - Whether event logging is enabled.
  *   spark.eventLog.dir - Path to the directory in which events are logged.
  *   spark.eventLog.logBlockUpdates.enabled - Whether to log block updates
- *   spark.eventLog.logStageExecutorMetrics.enabled - Whether to log stage executor metrics
+ *   spark.eventLog.logStageExecutorMetrics - Whether to log stage executor metrics
  *
  * Event log file writer maintains its own parameters: refer the doc of [[EventLogFileWriter]]
  * and its descendant for more details.
diff --git a/core/src/test/resources/spark-events/application_1553914137147_0018 b/core/src/test/resources/spark-events/application_1553914137147_0018
index 8c34e7265f8da..03ea3040de1db 100644
--- a/core/src/test/resources/spark-events/application_1553914137147_0018
+++ b/core/src/test/resources/spark-events/application_1553914137147_0018
@@ -1,6 +1,6 @@
 {"Event":"SparkListenerLogStart","Spark Version":"3.0.0-SNAPSHOT"}
 {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"test-1.vpc.company.com","Port":44768},"Maximum Memory":956615884,"Timestamp":1554755989747,"Maximum Onheap Memory":956615884,"Maximum Offheap Memory":0}
-{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/java/jdk1.8.0_144/jre","Java Version":"1.8.0_144 (Oracle Corporation)","Scala Version":"version 2.12.8"},"Spark Properties":{"spark.lineage.log.dir":"/var/log/spark2/lineage","spark.serializer":"org.apache.spark.serializer.KryoSerializer","spark.yarn.jars":"local:/opt/cloudera/parcels/SPARK2/lib/spark2/jars/*","spark.executor.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.driver.host":"test-1.vpc.company.com","spark.eventLog.enabled":"true","spark.executor.heartbeatInterval":"1000","spark.executor.memoryOverhead":"1024","spark.driver.port":"34194","spark.shuffle.service.enabled":"false","spark.driver.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.lineage.enabled":"false","spark.jars":"file:/tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar","spark.executor.metrics.pollingInterval":"100","spark.yarn.historyServer.address":"http://test-1.vpc.company.com:18089","spark.ui.enabled":"true","spark.app.name":"LargeBlocks","spark.ui.killEnabled":"true","spark.sql.hive.metastore.jars":"${env:HADOOP_COMMON_HOME}/../hive/lib/*:${env:HADOOP_COMMON_HOME}/client/*","spark.locality.wait.process":"0","spark.dynamicAllocation.schedulerBacklogTimeout":"1","spark.yarn.am.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.scheduler.mode":"FIFO","spark.eventLog.logStageExecutorMetrics.enabled":"true","spark.driver.memory":"2g","spark.executor.instances":"3","spark.submit.pyFiles":"","spark.yarn.config.gatewayPath":"/opt/cloudera/parcels","spark.executor.id":"driver","spark.yarn.config.replacementPath":"{{HADOOP_COMMON_HOME}}/../../..","spark.driver.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.eventLog.logStageExecutorProcessTreeMetrics.enabled":"true","spark.submit.deployMode":"client","spark.shuffle.service.port":"7337","spark.yarn.maxAppAttempts":"1","spark.master":"yarn","spark.authenticate":"false","spark.ui.filters":"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter","spark.network.crypto.enabled":"false","spark.executor.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.executor.memory":"7g","spark.io.encryption.enabled":"false","spark.eventLog.dir":"hdfs://test-1.vpc.company.com:8020/user/spark/spark2ApplicationHistory","spark.dynamicAllocation.enabled":"false","spark.sql.catalogImplementation":"hive","spark.executor.cores":"1","spark.driver.appUIAddress":"http://test-1.vpc.company.com:4040","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS":"test-1.vpc.company.com","spark.dynamicAllocation.minExecutors":"0","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES":"http://test-1.vpc.company.com:8088/proxy/application_1553914137147_0018","spark.dynamicAllocation.executorIdleTimeout":"60","spark.app.id":"application_1553914137147_0018","spark.sql.hive.metastore.version":"1.1.0"},"Hadoop Properties":{"yarn.resourcemanager.amlauncher.thread-count":"50","dfs.namenode.resource.check.interval":"5000","fs.s3a.connection.maximum":"100","mapreduce.jobtracker.jobhistory.task.numberprogresssplits":"12","dfs.data.transfer.server.tcpnodelay":"true","mapreduce.tasktracker.healthchecker.script.timeout":"600000","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","dfs.cachereport.intervalMsec":"10000","dfs.namenode.checkpoint.txns":"1000000","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","mapreduce.tasktracker.local.dir.minspacekill":"0","hadoop.security.groups.cache.background.reload.threads":"3","dfs.namenode.lease-recheck-interval-ms":"2000","fs.s3.block.size":"67108864","dfs.client.domain.socket.data.traffic":"false","dfs.ha.zkfc.nn.http.timeout.ms":"20000","hadoop.registry.secure":"false","hadoop.hdfs.configuration.version":"1","dfs.bytes-per-checksum":"512","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.jobtracker.persist.jobstatus.hours":"1","fs.s3a.s3guard.ddb.table.create":"false","dfs.datanode.slow.io.warning.threshold.ms":"300","dfs.namenode.handler.count":"10","dfs.namenode.list.reencryption.status.num.responses":"100","mapreduce.input.fileinputformat.split.minsize":"0","dfs.datanode.failed.volumes.tolerated":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.amliveliness-monitor.interval-ms":"1000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","mapreduce.tasktracker.http.threads":"40","dfs.namenode.retrycache.expirytime.millis":"600000","dfs.namenode.backup.address":"0.0.0.0:50100","dfs.datanode.data.dir":"file://${hadoop.tmp.dir}/dfs/data","dfs.datanode.shared.file.descriptor.paths":"/dev/shm,/tmp","dfs.replication":"3","mapreduce.jobtracker.jobhistory.block.size":"3145728","dfs.encrypt.data.transfer.cipher.key.bitlength":"128","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","dfs.secondary.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","mapreduce.task.profile.maps":"0-2","dfs.datanode.block-pinning.enabled":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","mapreduce.jobtracker.retiredjobs.cache.size":"1000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"test-1.vpc.company.com:19890","fs.trash.checkpoint.interval":"0","dfs.namenode.checkpoint.check.period":"60","yarn.nodemanager.container-monitor.interval-ms":"3000","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","hadoop.proxyuser.HTTP.groups":"*","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"10000","dfs.namenode.reencrypt.edek.threads":"10","yarn.resourcemanager.system-metrics-publisher.enabled":"false","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","dfs.namenode.fs-limits.max-component-length":"255","mapreduce.tasktracker.taskcontroller":"org.apache.hadoop.mapred.DefaultTaskController","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","dfs.namenode.top.window.num.buckets":"10","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","dfs.datanode.block.id.layout.upgrade.threads":"12","mapreduce.jobtracker.tasktracker.maxblacklists":"4","yarn.nodemanager.docker-container-executor.exec-name":"/usr/bin/docker","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"0.23.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","dfs.namenode.decommission.max.concurrent.tracked.nodes":"100","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","dfs.client.read.shortcircuit.skip.checksum":"false","mapreduce.task.profile.reduces":"0-2","dfs.datanode.address":"0.0.0.0:50010","dfs.https.server.keystore.resource":"ssl-server.xml","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"test-1.vpc.company.com:8030","mapreduce.task.timeout":"600000","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://test-1.vpc.company.com:8020","fs.har.impl.disable.cache":"true","io.compression.codec.bzip2.library":"system-native","dfs.namenode.audit.loggers":"default","dfs.block.access.key.update.interval":"600","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"test-1.vpc.company.com:8090","dfs.namenode.max.objects":"0","mapreduce.jobhistory.address":"test-1.vpc.company.com:10020","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","dfs.namenode.max.full.block.report.leases":"6","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","dfs.client.mmap.cache.timeout.ms":"3600000","dfs.mover.max-no-move-interval":"60000","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","dfs.client.datanode-restart.timeout":"30","dfs.datanode.drop.cache.behind.reads":"false","ipc.server.log.slow.rpc":"false","dfs.namenode.read-lock-reporting-threshold-ms":"5000","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","dfs.namenode.checkpoint.edits.dir":"${dfs.namenode.checkpoint.dir}","yarn.app.attempt.diagnostics.limit.kc":"64","dfs.balancer.block-move.timeout":"0","dfs.client.block.write.replace-datanode-on-failure.enable":"true","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"8192","io.compression.codecs":"org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","dfs.heartbeat.interval":"3","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","dfs.ha.tail-edits.period":"60","dfs.datanode.max.locked.memory":"0","dfs.datanode.scan.period.hours":"504","mapreduce.jobtracker.expire.trackers.interval":"600000","yarn.resourcemanager.nodemanager-connect-retries":"10","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","dfs.namenode.edits.noeditlogchannelflush":"false","mapreduce.task.io.sort.factor":"64","mapreduce.tasktracker.outofband.heartbeat":"false","ha.failover-controller.new-active.rpc-timeout.ms":"60000","dfs.webhdfs.ugi.expire.after.access":"600000","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","dfs.namenode.https-address":"test-1.vpc.company.com:20102","yarn.am.blacklisting.enabled":"true","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","dfs.image.transfer-bootstrap-standby.bandwidthPerSec":"0","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.proxyuser.hive.groups":"*","ha.zookeeper.session-timeout.ms":"5000","mapreduce.map.java.opts":"-Djava.net.preferIPv4Stack=true","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","dfs.ha.automatic-failover.enabled":"false","mapreduce.job.counters.groups.max":"50","dfs.namenode.decommission.interval":"30","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","dfs.datanode.handler.count":"10","dfs.namenode.xattrs.enabled":"true","dfs.namenode.safemode.threshold-pct":"0.999f","dfs.client.socket.send.buffer.size":"0","mapreduce.map.sort.spill.percent":"0.8","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","dfs.datanode.sync.behind.writes":"false","dfs.namenode.stale.datanode.interval":"30000","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","dfs.datanode.transfer.socket.send.buffer.size":"0","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","dfs.namenode.reencrypt.throttle.limit.handler.ratio":"1.0","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler","yarn.app.mapreduce.am.command-opts":"-Djava.net.preferIPv4Stack=true -Xmx825955249","dfs.journalnode.https-address":"0.0.0.0:8481","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","hadoop.proxyuser.hue.hosts":"*","io.mapfile.bloom.error.rate":"0.005","dfs.user.home.dir.prefix":"/user","hadoop.proxyuser.hue.groups":"*","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","mapreduce.jobtracker.persist.jobstatus.dir":"/jobtracker/jobsInfo","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","dfs.namenode.snapshot.capture.openfiles":"false","dfs.datanode.directoryscan.interval":"21600","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","dfs.client.read.shortcircuit.streams.cache.expiry.ms":"300000","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"10","dfs.namenode.write.stale.datanode.ratio":"0.5f","hadoop.registry.zk.session.timeout.ms":"60000","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","mapreduce.jobtracker.taskcache.levels":"2","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","mapreduce.job.emit-timeline-data":"false","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","dfs.domain.socket.path":"/var/run/hdfs-sockets/dn","yarn.resourcemanager.admin.client.thread-count":"1","mapreduce.jobtracker.persist.jobstatus.active":"true","dfs.namenode.reencrypt.sleep.interval":"1m","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"test-1.vpc.company.com:10033","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","mapreduce.tasktracker.report.address":"127.0.0.1:0","dfs.namenode.edit.log.autoroll.multiplier.threshold":"2.0","io.seqfile.lazydecompress":"true","ftp.blocksize":"67108864","dfs.namenode.backup.http-address":"0.0.0.0:50105","dfs.disk.balancer.max.disk.throughputInMBperSec":"10","mapreduce.jobtracker.instrumentation":"org.apache.hadoop.mapred.JobTrackerMetricsInst","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.delete.debug-delay-sec":"0","mapreduce.jobtracker.http.address":"0.0.0.0:50030","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","hadoop.security.groups.cache.secs":"300","yarn.resourcemanager.zk-retry-interval-ms":"1000","yarn.scheduler.increment-allocation-mb":"512","nfs.mountd.port":"4242","mapreduce.shuffle.max.threads":"0","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","mapreduce.jobhistory.http.policy":"HTTP_ONLY","dfs.datanode.dns.interface":"default","mapreduce.reduce.java.opts":"-Djava.net.preferIPv4Stack=true","s3native.replication":"3","hadoop.security.group.mapping.ldap.ssl":"false","dfs.namenode.fs-limits.max-xattrs-per-inode":"32","yarn.client.application-client-protocol.poll-interval-ms":"200","hadoop.proxyuser.flume.groups":"*","dfs.namenode.fs-limits.max-xattr-size":"16384","dfs.namenode.maintenance.replication.min":"1","dfs.client.write.exclude.nodes.cache.expiry.interval.millis":"600000","ha.zookeeper.parent-znode":"/hadoop-ha","dfs.namenode.safemode.extension":"30000","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","dfs.blocksize":"134217728","dfs.namenode.servicerpc-address":"test-1.vpc.company.com:8022","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"test-1.vpc.company.com:8032","ipc.client.ping":"true","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","dfs.namenode.resource.checked.volumes.minimum":"1","hadoop.proxyuser.HTTP.hosts":"*","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","dfs.namenode.quota.init-threads":"4","dfs.journalnode.http-address":"0.0.0.0:8480","dfs.block.scanner.volume.bytes.per.second":"1048576","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","dfs.storage.policy.enabled":"true","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","dfs.namenode.edits.dir":"${dfs.namenode.name.dir}","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","dfs.namenode.support.allow.format":"true","dfs.ha.tail-edits.rolledits.timeout":"60","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","mapreduce.cluster.temp.dir":"${hadoop.tmp.dir}/mapred/temp","s3.replication":"3","dfs.client.failover.connection.retries":"0","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","dfs.namenode.secondary.http-address":"0.0.0.0:50090","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","nfs.wtmax":"1048576","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","nfs.dump.dir":"/tmp/.hdfs-nfs","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","mapreduce.client.submit.file.replication":"3","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","dfs.namenode.service.handler.count":"10","dfs.datanode.data.dir.perm":"700","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","dfs.namenode.name.dir":"file:///dataroot/dataroot/dfs/nn","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","fs.ftp.host.port":"21","ipc.ping.interval":"60000","dfs.namenode.num.checkpoints.retained":"2","dfs.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","yarn.resourcemanager.admin.address":"test-1.vpc.company.com:8033","file.client-write-packet-size":"65536","hadoop.treat.subject.external":"false","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"false","dfs.disk.balancer.plan.threshold.percent":"10","mapreduce.local.clientfactory.class.name":"org.apache.hadoop.mapred.LocalClientFactory","dfs.client.use.legacy.blockreader":"false","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"256","yarn.nodemanager.localizer.client.thread-count":"5","dfs.namenode.checkpoint.max-retries":"3","dfs.namenode.reject-unresolved-dn-topology-mapping":"false","dfs.namenode.delegation.token.max-lifetime":"*********(redacted)","dfs.client.block.write.replace-datanode-on-failure.min-replication":"0","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"2000","dfs.namenode.num.extra.edits.retained":"1000000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.scheduler.increment-allocation-vcores":"1","fs.df.interval":"60000","fs.s3.sleepTimeSeconds":"10","fs.s3a.multiobjectdelete.enable":"true","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"true","io.file.buffer.size":"65536","hadoop.work.around.non.threadsafe.getpwuid":"false","dfs.permissions.superusergroup":"supergroup","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","mapreduce.tasktracker.dns.interface":"default","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","dfs.permissions.enabled":"true","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/user","mapreduce.reduce.shuffle.read.timeout":"180000","yarn.app.mapreduce.am.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","dfs.datanode.https.address":"0.0.0.0:50475","dfs.namenode.hosts.provider.classname":"org.apache.hadoop.hdfs.server.blockmanagement.HostFileManager","dfs.datanode.transfer.socket.recv.buffer.size":"0","fs.s3a.connection.establish.timeout":"5000","dfs.namenode.fslock.fair":"true","mapreduce.job.running.map.limit":"0","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","hadoop.fuse.connection.timeout":"300","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"5000","yarn.nodemanager.vmem-pmem-ratio":"2.1","dfs.client.slow.io.warning.threshold.ms":"30000","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","s3native.stream-buffer-size":"4096","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","fs.s3a.s3guard.cli.prune.age":"86400000","dfs.client.read.shortcircuit.streams.cache.size":"256","dfs.client.use.legacy.blockreader.local":"false","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3n.multipart.uploads.enabled":"false","dfs.namenode.path.based.cache.retry.interval.ms":"30000","hadoop.security.crypto.buffer.size":"8192","yarn.client.failover-retries-on-socket-timeouts":"0","dfs.balancer.keytab.enabled":"false","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","dfs.datanode.balance.bandwidthPerSec":"10485760","dfs.namenode.name.dir.restore":"false","hadoop.registry.jaas.context":"Client","dfs.client.failover.sleep.max.millis":"15000","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","s3.blocksize":"67108864","yarn.am.blacklisting.disable-failure-threshold":"0.8f","io.map.index.interval":"128","mapreduce.job.counters.max":"120","dfs.namenode.max-lock-hold-to-release-lease-ms":"25","dfs.namenode.datanode.registration.ip-hostname-check":"true","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","dfs.namenode.resource.du.reserved":"104857600","dfs.datanode.bp-ready.timeout":"20","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","dfs.namenode.full.block.report.lease.length.ms":"300000","mapreduce.tasktracker.instrumentation":"org.apache.hadoop.mapred.TaskTrackerMetricsInst","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","s3.stream-buffer-size":"4096","yarn.resourcemanager.nm.liveness-monitor.interval-ms":"1000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","dfs.namenode.secondary.https-address":"0.0.0.0:50091","s3native.bytes-per-checksum":"512","dfs.namenode.fs-limits.max-directory-items":"1048576","nfs.server.port":"2049","dfs.namenode.delegation.token.renew-interval":"*********(redacted)","mapreduce.jobtracker.address":"local","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","dfs.namenode.list.openfiles.num.responses":"1000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","dfs.namenode.blocks.per.postponedblocks.rescan":"10000","dfs.namenode.checkpoint.period":"3600","hadoop.security.groups.cache.background.reload":"false","yarn.resourcemanager.amlauncher.log.command":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","dfs.namenode.snapshot.skip.capture.accesstime-only-change":"false","dfs.webhdfs.user.provider.user.pattern":"^[A-Za-z_][A-Za-z0-9._-]*[$]?$","dfs.webhdfs.acl.provider.permission.pattern":"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"false","mapreduce.map.skip.maxrecords":"0","mapreduce.jobtracker.handler.count":"10","hadoop.http.authentication.type":"simple","mapreduce.job.jvm.numtasks":"1","hadoop.proxyuser.flume.hosts":"*","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","fs.s3n.block.size":"67108864","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","mapreduce.jobtracker.staging.root.dir":"${hadoop.tmp.dir}/mapred/staging","dfs.namenode.http-address":"test-1.vpc.company.com:20101","mapreduce.jobtracker.jobhistory.lru.cache.size":"5","dfs.datanode.directoryscan.threads":"1","dfs.datanode.fsdatasetcache.max.threads.per.volume":"4","dfs.namenode.fs-limits.max-blocks-per-file":"1048576","dfs.disk.balancer.enabled":"false","mapreduce.shuffle.listen.queue.size":"128","mapreduce.tasktracker.local.dir.minspacestart":"0","mapreduce.map.cpu.vcores":"1","hadoop.user.group.static.mapping.overrides":"dr.who=;","dfs.datanode.cache.revocation.timeout.ms":"900000","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","dfs.client.mmap.cache.size":"256","dfs.ha.log-roll.period":"120","dfs.client.failover.sleep.base.millis":"500","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","dfs.namenode.accesstime.precision":"3600000","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","hadoop.registry.zk.retry.interval.ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","dfs.client.context":"default","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","hadoop.ssl.server.conf":"ssl-server.xml","dfs.http.policy":"HTTP_ONLY","dfs.client.https.keystore.resource":"ssl-client.xml","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.acl.enable":"true","dfs.domain.socket.disable.interval.seconds":"1","dfs.image.transfer.chunksize":"65536","dfs.balancer.max-no-move-interval":"60000","mapreduce.tasktracker.map.tasks.maximum":"2","dfs.namenode.edits.journal-plugin.qjournal":"org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager","mapreduce.task.profile":"false","dfs.webhdfs.enabled":"true","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","dfs.namenode.list.encryption.zones.num.responses":"100","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","dfs.namenode.top.num.users":"10","dfs.disk.balancer.block.tolerance.percent":"10","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","io.seqfile.sorter.recordlimit":"1000000","hadoop.security.auth_to_local":"DEFAULT","dfs.blockreport.initialDelay":"0","fs.automatic.close":"true","dfs.client.block.write.replace-datanode-on-failure.best-effort":"false","dfs.namenode.replication.min":"1","dfs.balancer.address":"0.0.0.0:0","fs.s3n.multipart.copy.block.size":"5368709120","yarn.nodemanager.hostname":"0.0.0.0","nfs.rtmax":"1048576","yarn.resourcemanager.zk-timeout-ms":"10000","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","dfs.datanode.directoryscan.throttle.limit.ms.per.sec":"1000","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","dfs.namenode.replication.work.multiplier.per.iteration":"2","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","dfs.namenode.reencrypt.throttle.limit.updater.ratio":"1.0","dfs.namenode.avoid.write.stale.datanode":"false","dfs.short.circuit.shared.memory.watcher.interrupt.check.ms":"60000","dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction":"0.75f","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","dfs.client.mmap.enabled":"true","mapreduce.reduce.cpu.vcores":"1","hadoop.proxyuser.oozie.groups":"*","fs.client.resolve.remote.symlinks":"true","dfs.image.compression.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.jobtracker.restart.recover":"false","dfs.namenode.decommission.blocks.per.interval":"500000","mapreduce.tasktracker.reduce.tasks.maximum":"2","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","dfs.namenode.safemode.min.datanodes":"0","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.userlog.retain.hours":"24","yarn.scheduler.maximum-allocation-vcores":"4","yarn.nodemanager.log-aggregation.compression-type":"none","dfs.namenode.enable.retrycache":"true","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","dfs.namenode.startup.delay.block.deletion.sec":"0","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","dfs.datanode.readahead.bytes":"4194304","mapreduce.jobtracker.heartbeats.in.second":"100","mapreduce.job.running.reduce.limit":"0","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","mapreduce.task.tmp.dir":"./tmp","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","dfs.namenode.delegation.key.update-interval":"86400000","fs.s3a.max.total.tasks":"5","dfs.client.file-block-storage-locations.num-threads":"10","mapreduce.tasktracker.healthchecker.interval":"60000","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"false","fs.s3a.attempts.maximum":"20","dfs.namenode.avoid.read.stale.datanode":"false","hadoop.registry.zk.connection.timeout.ms":"15000","dfs.https.port":"20102","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","mapreduce.map.log.level":"INFO","mapreduce.output.fileoutputformat.compress.type":"BLOCK","hadoop.registry.rm.enabled":"false","mapreduce.ifile.readahead.bytes":"4194304","mapreduce.tasktracker.tasks.sleeptimebeforesigkill":"5000","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","dfs.namenode.posix.acl.inheritance.enabled":"false","dfs.blockreport.intervalMsec":"21600000","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","dfs.namenode.path.based.cache.refresh.interval.ms":"30000","dfs.namenode.edekcacheloader.interval.ms":"1000","file.stream-buffer-size":"4096","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.SnappyCodec","mapreduce.map.speculative":"false","dfs.disk.balancer.max.disk.errors":"5","dfs.datanode.use.datanode.hostname":"false","mapreduce.job.speculative.retry-after-speculate":"15000","hadoop.proxyuser.hdfs.hosts":"*","dfs.namenode.fs-limits.min-block-size":"1048576","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","mapreduce.job.reduce.slowstart.completedmaps":"0.8","dfs.client.read.shortcircuit":"false","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","hadoop.proxyuser.yarn.groups":"*","dfs.client.cached.conn.retry":"3","dfs.namenode.invalidate.work.pct.per.iteration":"0.32f","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.nodemanager.logaggregation.threadpool-size-max":"100","dfs.replication.max":"512","dfs.namenode.inotify.max.events.per.rpc":"1000","yarn.resourcemanager.hostname":"0.0.0.0","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"0","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","ipc.client.rpc-timeout.ms":"0","fs.s3.maxRetries":"4","dfs.default.chunk.view.size":"32768","mapreduce.input.lineinputformat.linespermap":"1","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","dfs.client.mmap.retry.timeout.ms":"300000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","dfs.namenode.list.cache.directives.num.responses":"100","fs.s3a.socket.recv.buffer":"8192","dfs.image.compress":"false","dfs.namenode.kerberos.principal.pattern":"*","yarn.application.classpath":"$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*","fs.s3n.multipart.uploads.block.size":"67108864","mapreduce.tasktracker.http.address":"0.0.0.0:50060","yarn.resourcemanager.resource-tracker.address":"test-1.vpc.company.com:8031","hadoop.fuse.timer.period":"5","mapreduce.job.heap.memory-mb.ratio":"0.8","dfs.datanode.hdfs-blocks-metadata.enabled":"true","dfs.namenode.checkpoint.dir":"file://${hadoop.tmp.dir}/dfs/namesecondary","dfs.datanode.max.transfer.threads":"4096","dfs.namenode.edits.asynclogging":"true","nfs.allow.insecure.ports":"true","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"0","s3native.client-write-packet-size":"65536","mapreduce.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","hadoop.proxyuser.mapred.hosts":"*","hadoop.proxyuser.oozie.hosts":"*","yarn.nodemanager.log.retain-seconds":"10800","hadoop.proxyuser.mapred.groups":"*","yarn.resourcemanager.keytab":"/etc/krb5.keytab","mapreduce.reduce.merge.inmem.threshold":"1000","dfs.client.https.need-auth":"false","dfs.blockreport.split.threshold":"1000000","dfs.client.block.write.replace-datanode-on-failure.policy":"DEFAULT","mapreduce.shuffle.ssl.enabled":"false","dfs.namenode.write-lock-reporting-threshold-ms":"5000","dfs.block.access.token.enable":"*********(redacted)","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","httpfs.buffer.size":"4096","dfs.client.file-block-storage-locations.timeout.millis":"1000","dfs.namenode.block-placement-policy.default.prefer-local-node":"true","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","dfs.namenode.replication.considerLoad":"true","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.proxyuser.hdfs.groups":"*","dfs.namenode.retrycache.heap.percent":"0.03f","dfs.datanode.cache.revocation.polling.ms":"500","mapreduce.jobhistory.webapp.address":"test-1.vpc.company.com:19888","dfs.namenode.path.based.cache.block.map.allocation.percent":"0.25","mapreduce.jobtracker.system.dir":"${hadoop.tmp.dir}/mapred/system","mapreduce.tasktracker.taskmemorymanager.monitoringinterval":"5000","dfs.journalnode.rpc-address":"0.0.0.0:8485","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"test-1.vpc.company.com:8088","mapreduce.jobhistory.recovery.enable":"false","dfs.client.short.circuit.replica.stale.threshold.ms":"1800000","mapreduce.reduce.shuffle.parallelcopies":"10","fs.trash.interval":"1","dfs.namenode.replication.interval":"3","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","dfs.namenode.top.enabled":"true","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","dfs.datanode.du.reserved":"0","yarn.app.mapreduce.am.resource.mb":"1024","mapreduce.input.fileinputformat.list-status.num-threads":"1","dfs.namenode.lazypersist.file.scrub.interval.sec":"300","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.nodemanager.resource.cpu-vcores":"8","mapreduce.job.reduces":"6","fs.s3a.multipart.size":"64M","yarn.scheduler.minimum-allocation-vcores":"1","dfs.namenode.reencrypt.batch.size":"1000","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","dfs.datanode.http.address":"0.0.0.0:50075","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","fs.s3a.metadatastore.authoritative":"false","ha.health-monitor.sleep-after-disconnect.ms":"1000","s3.bytes-per-checksum":"512","yarn.app.mapreduce.shuffle.log.limit.kb":"0","dfs.namenode.list.cache.pools.num.responses":"100","hadoop.security.group.mapping":"org.apache.hadoop.security.ShellBasedUnixGroupsMapping","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","dfs.encrypt.data.transfer":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","mapreduce.job.redacted-properties":"*********(redacted)","dfs.namenode.top.windows.minutes":"1,5,25","s3.client-write-packet-size":"65536","mapreduce.map.output.compress":"true","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.scheduler.maximum-allocation-mb":"12288","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.app.mapreduce.am.container.log.limit.kb":"0","s3native.blocksize":"67108864","ipc.client.connect.retry.interval":"1000","hadoop.proxyuser.httpfs.groups":"*","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","dfs.namenode.edit.log.autoroll.check.interval.ms":"300000","mapreduce.jobhistory.cleaner.enable":"true","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","hadoop.proxyuser.httpfs.hosts":"*","dfs.client.use.datanode.hostname":"false","dfs.stream-buffer-size":"4096","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","dfs.datanode.drop.cache.behind.writes":"false","mapreduce.tasktracker.dns.nameserver":"default","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","mapreduce.job.end-notification.retry.attempts":"0","hadoop.proxyuser.yarn.hosts":"*","yarn.resourcemanager.zk-num-retries":"1000","dfs.client.failover.max.attempts":"15","mapreduce.tasktracker.indexcache.mb":"10","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","dfs.namenode.snapshotdiff.allow.snap-root-descendant":"true","yarn.nodemanager.localizer.cache.target-size-mb":"10240","zlib.compress.level":"DEFAULT_COMPRESSION","ftp.client-write-packet-size":"65536","mapreduce.jobtracker.maxtasks.perjob":"-1","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.proxyuser.hive.hosts":"*","dfs.block.access.token.lifetime":"*********(redacted)","dfs.namenode.max.extra.edits.segments.retained":"10000","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","dfs.image.transfer.bandwidthPerSec":"0","io.native.lib.available":"true","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","ipc.server.listen.queue.size":"128","dfs.namenode.edekcacheloader.initial.delay.ms":"3000","map.sort.class":"org.apache.hadoop.util.QuickSort","dfs.namenode.acls.enabled":"false","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","dfs.datanode.ipc.address":"0.0.0.0:50020","yarn.nodemanager.vmem-check-enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","dfs.lock.suppress.warning.interval":"10s","dfs.client.block.write.retries":"3","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH","yarn.client.nodemanager-connect.retry-interval-ms":"10000","dfs.client-write-packet-size":"65536","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME","dfs.datanode.dns.nameserver":"default","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","dfs.image.transfer.timeout":"60000","yarn.resourcemanager.recovery.enabled":"false","dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold":"10737418240","dfs.client.failover.connection.retries.on.timeouts":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.security.egd":"file:///dev/urandom","java.vm.specification.version":"1.8","user.home":"/home/systest","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/java/jdk1.8.0_144/jre/lib/amd64","user.dir":"/tmp","java.library.path":":/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.144-b01","jetty.git.hash":"27208684755d94a92186989f695db2d7b21ebc51","java.endorsed.dirs":"/usr/java/jdk1.8.0_144/jre/lib/endorsed","java.runtime.version":"1.8.0_144-b01","java.vm.info":"mixed mode","java.ext.dirs":"/usr/java/jdk1.8.0_144/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/java/jdk1.8.0_144/jre/lib/resources.jar:/usr/java/jdk1.8.0_144/jre/lib/rt.jar:/usr/java/jdk1.8.0_144/jre/lib/sunrsasign.jar:/usr/java/jdk1.8.0_144/jre/lib/jsse.jar:/usr/java/jdk1.8.0_144/jre/lib/jce.jar:/usr/java/jdk1.8.0_144/jre/lib/charsets.jar:/usr/java/jdk1.8.0_144/jre/lib/jfr.jar:/usr/java/jdk1.8.0_144/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"3.10.0-514.26.2.el7.x86_64","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"systest","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master yarn --deploy-mode client --conf spark.driver.memory=2g --conf spark.executor.heartbeatInterval=1000 --conf spark.executor.metrics.pollingInterval=100 --conf spark.eventLog.logStageExecutorProcessTreeMetrics.enabled=true --conf spark.yarn.maxAppAttempts=1 --conf spark.locality.wait.process=0 --conf spark.executor.memoryOverhead=1024 --conf spark.executor.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --conf spark.eventLog.logStageExecutorMetrics.enabled=true --conf spark.driver.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --class com.company.spark.LargeBlocks --num-executors 3 --executor-memory 7g /tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar --targetBlockSizeGb 2.5 --taskSleepMillis 200 --doCache true --cacheOnDisk true --replicas 1 --concurrentReadJobs 2","java.home":"/usr/java/jdk1.8.0_144/jre","java.version":"1.8.0_144","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/opt/cloudera/parcels/CDH/jars/jackson-mapper-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/joni-2.1.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-external-blockcache-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-json-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/xz-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-annotations-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/cglib-2.2.1-v20090111.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-procedure-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/asm-3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xml-apis-1.3.04.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-jvm-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-streaming_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-xml-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/minlog-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hsqldb-1.8.0.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-rdbms-3.2.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pmml-model-1.4.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire-macros_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zstd-jni-1.3.2-2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpcore-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jta-1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-yarn_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-logging-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/activation-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xbean-asm7-shaded-4.12.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/NOTICE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-format-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpclient-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-3.9.9.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/htrace-core-3.1.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.ws.rs-api-2.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-core-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-nativetask-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-client-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-xc-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-configuration-1.6.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math3-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsp-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-auth-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-registry-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/JavaEWAH-0.3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-graphite-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/joda-time-2.9.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-gridmix-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-compress-1.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javolution-5.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-1.7.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/core-1.1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-runtime-3.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-net-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-framework-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-java-1.1.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-assembly_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-examples-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/shapeless_2.12-2.3.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xercesImpl-2.9.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-server-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang-2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jtransforms-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-0.2.jar":"System Classpath","/etc/spark2/conf/yarn-conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-cli-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.annotation-api-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0-2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javassist-3.18.1-GA.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-applicationhistoryservice-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka-clients-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guice-3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr4-runtime-4.7.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/core-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/opencsv-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-datajoin-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-common-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-webapp-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compiler-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-annotations-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/libthrift-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka_2.11-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-proxy-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-2.7.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jettison-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libfb303-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/zookeeper-3.4.5-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-resource-bundle-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jaxb-api-2.2.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-streaming-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop2-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pyrolite-4.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/activation-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aircompressor-0.10.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-ant-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jline-2.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-all-4.0.23.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-media-jaxb-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlets-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-xml_2.12-1.0.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-graphx_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-plugins-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-web-proxy-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-kvstore_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/objenesis-2.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-i18n-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stream-2.7.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-aws-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-scala_2.12-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/spymemcached-2.11.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-util-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-core-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/mockito-all-1.8.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jets3t-0.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-xc-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rsgroup-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-metastore-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/py4j-0.10.8.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-distributedshell-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/osgi-resource-locator-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-rumen-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsch-0.1.42.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/zkclient-0.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xmlenc-0.52.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-io-2.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-core-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jamon-runtime-2.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-common-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-common-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-3.10.5.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-app-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/machinist_2.12-0.6.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-shuffle_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill_2.12-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/findbugs-annotations-1.3.9-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rest-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ST4-4.0.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jruby-cloudera-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-hive_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-core_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-distcp-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-server-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core-3.2.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-mapreduce-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jcl-over-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-digester-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-http-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-jackson_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-math3-3.4.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-log4j12-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-jobclient-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-shell-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-server-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill-java-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-locator-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-parser-combinators_2.12-1.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/RoaringBitmap-0.5.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-api-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/log4j-1.2.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-compiler-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-security-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-shuffle-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-all-4.1.30.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze-macros_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/azure-data-lake-store-sdk-2.2.9.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-12.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-codec-1.10.jar":"System Classpath","/usr/java/jdk1.8.0_144/lib/tools.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-nodemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sql_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-exec-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jcodings-1.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-datalake-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-kerberos-codec-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-11.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-catalyst_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-io-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-unmanaged-am-launcher-2.6.0-cdh5.15.2.jar":"System Classpath","/etc/spark2/conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-sslengine-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-api-jdo-3.2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpclient-4.5.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-continuation-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-ast_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-jndi-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsr305-3.0.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/avro-1.7.6-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/univocity-parsers-2.7.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-annotations-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-asn1-api-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jettison-1.3.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-utils-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hppc-0.7.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/logredactor-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-vector-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-impl-2.2.3-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archives-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-plus-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-client-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hue-plugins-3.9.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/high-scale-lib-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-jaxrs-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-library-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/java-xmlbuilder-0.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apache-log4j-extras-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-format-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/disruptor-3.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib-local_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlet-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-annotations-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/log4j-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-repl_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/derby-10.12.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-shims-1.5.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-core-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-repackaged-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-api-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-daemon-1.0.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ivy-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hamcrest-core-1.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang3-3.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/spark-streaming-kafka-0-8_2.11-2.4.0.cloudera1-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-databind-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-reflect-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpcore-4.4.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-memory-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/LICENSE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compress-1.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-client-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/oro-2.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-thrift-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-ipc-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-httpclient-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okhttp-2.4.0.jar":"System Classpath","spark://test-1.vpc.company.com:34194/jars/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar":"Added By User","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-compiler-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-prefix-tree-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-api-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-core-1.8.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/lz4-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/janino-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libthrift-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-launcher_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-core_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/lz4-java-1.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-jaxrs-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-common_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-unsafe_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-sls-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/snappy-java-1.0.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-guava-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/validation-api-1.1.0.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/microsoft-windowsazure-storage-sdk-0.6.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/compress-lzf-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-openstack-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-client-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-it-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-extras-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-recipes-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-column-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/unused-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-crypto-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-databind-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core4-4.0.1-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guava-14.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/aws-java-sdk-bundle-1.11.134.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zookeeper-3.4.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-paranamer-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-protocol-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-runtime-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-scalap_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-resourcemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archive-logs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/paranamer-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-beanutils-1.9.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jul-to-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stringtemplate-3.2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/gson-2.2.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okio-1.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-api-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsr305-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sketch_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-el-1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-tags_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-core-3.2.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/protobuf-java-2.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-bundle-1.6.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xz-1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-client-2.6.0-cdh5.15.2.jar":"System Classpath"}}
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/java/jdk1.8.0_144/jre","Java Version":"1.8.0_144 (Oracle Corporation)","Scala Version":"version 2.12.8"},"Spark Properties":{"spark.lineage.log.dir":"/var/log/spark2/lineage","spark.serializer":"org.apache.spark.serializer.KryoSerializer","spark.yarn.jars":"local:/opt/cloudera/parcels/SPARK2/lib/spark2/jars/*","spark.executor.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.driver.host":"test-1.vpc.company.com","spark.eventLog.enabled":"true","spark.executor.heartbeatInterval":"1000","spark.executor.memoryOverhead":"1024","spark.driver.port":"34194","spark.shuffle.service.enabled":"false","spark.driver.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.lineage.enabled":"false","spark.jars":"file:/tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar","spark.executor.metrics.pollingInterval":"100","spark.yarn.historyServer.address":"http://test-1.vpc.company.com:18089","spark.ui.enabled":"true","spark.app.name":"LargeBlocks","spark.ui.killEnabled":"true","spark.sql.hive.metastore.jars":"${env:HADOOP_COMMON_HOME}/../hive/lib/*:${env:HADOOP_COMMON_HOME}/client/*","spark.locality.wait.process":"0","spark.dynamicAllocation.schedulerBacklogTimeout":"1","spark.yarn.am.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.scheduler.mode":"FIFO","spark.eventLog.logStageExecutorMetrics":"true","spark.driver.memory":"2g","spark.executor.instances":"3","spark.submit.pyFiles":"","spark.yarn.config.gatewayPath":"/opt/cloudera/parcels","spark.executor.id":"driver","spark.yarn.config.replacementPath":"{{HADOOP_COMMON_HOME}}/../../..","spark.driver.extraJavaOptions":"-Djava.security.egd=file:///dev/urandom","spark.eventLog.logStageExecutorProcessTreeMetrics.enabled":"true","spark.submit.deployMode":"client","spark.shuffle.service.port":"7337","spark.yarn.maxAppAttempts":"1","spark.master":"yarn","spark.authenticate":"false","spark.ui.filters":"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter","spark.network.crypto.enabled":"false","spark.executor.extraLibraryPath":"/opt/cloudera/parcels/CDH/lib/hadoop/lib/native","spark.executor.memory":"7g","spark.io.encryption.enabled":"false","spark.eventLog.dir":"hdfs://test-1.vpc.company.com:8020/user/spark/spark2ApplicationHistory","spark.dynamicAllocation.enabled":"false","spark.sql.catalogImplementation":"hive","spark.executor.cores":"1","spark.driver.appUIAddress":"http://test-1.vpc.company.com:4040","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS":"test-1.vpc.company.com","spark.dynamicAllocation.minExecutors":"0","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES":"http://test-1.vpc.company.com:8088/proxy/application_1553914137147_0018","spark.dynamicAllocation.executorIdleTimeout":"60","spark.app.id":"application_1553914137147_0018","spark.sql.hive.metastore.version":"1.1.0"},"Hadoop Properties":{"yarn.resourcemanager.amlauncher.thread-count":"50","dfs.namenode.resource.check.interval":"5000","fs.s3a.connection.maximum":"100","mapreduce.jobtracker.jobhistory.task.numberprogresssplits":"12","dfs.data.transfer.server.tcpnodelay":"true","mapreduce.tasktracker.healthchecker.script.timeout":"600000","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","mapreduce.framework.name":"yarn","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","dfs.cachereport.intervalMsec":"10000","dfs.namenode.checkpoint.txns":"1000000","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","mapreduce.tasktracker.local.dir.minspacekill":"0","hadoop.security.groups.cache.background.reload.threads":"3","dfs.namenode.lease-recheck-interval-ms":"2000","fs.s3.block.size":"67108864","dfs.client.domain.socket.data.traffic":"false","dfs.ha.zkfc.nn.http.timeout.ms":"20000","hadoop.registry.secure":"false","hadoop.hdfs.configuration.version":"1","dfs.bytes-per-checksum":"512","fs.s3.buffer.dir":"${hadoop.tmp.dir}/s3","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25","mapreduce.jobhistory.loadedjobs.cache.size":"5","mapreduce.jobtracker.persist.jobstatus.hours":"1","fs.s3a.s3guard.ddb.table.create":"false","dfs.datanode.slow.io.warning.threshold.ms":"300","dfs.namenode.handler.count":"10","dfs.namenode.list.reencryption.status.num.responses":"100","mapreduce.input.fileinputformat.split.minsize":"0","dfs.datanode.failed.volumes.tolerated":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.amliveliness-monitor.interval-ms":"1000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","mapreduce.tasktracker.http.threads":"40","dfs.namenode.retrycache.expirytime.millis":"600000","dfs.namenode.backup.address":"0.0.0.0:50100","dfs.datanode.data.dir":"file://${hadoop.tmp.dir}/dfs/data","dfs.datanode.shared.file.descriptor.paths":"/dev/shm,/tmp","dfs.replication":"3","mapreduce.jobtracker.jobhistory.block.size":"3145728","dfs.encrypt.data.transfer.cipher.key.bitlength":"128","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","dfs.secondary.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","mapreduce.task.profile.maps":"0-2","dfs.datanode.block-pinning.enabled":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","mapreduce.jobtracker.retiredjobs.cache.size":"1000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"test-1.vpc.company.com:19890","fs.trash.checkpoint.interval":"0","dfs.namenode.checkpoint.check.period":"60","yarn.nodemanager.container-monitor.interval-ms":"3000","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","hadoop.proxyuser.HTTP.groups":"*","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"10000","dfs.namenode.reencrypt.edek.threads":"10","yarn.resourcemanager.system-metrics-publisher.enabled":"false","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","dfs.namenode.fs-limits.max-component-length":"255","mapreduce.tasktracker.taskcontroller":"org.apache.hadoop.mapred.DefaultTaskController","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","dfs.namenode.top.window.num.buckets":"10","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","dfs.datanode.block.id.layout.upgrade.threads":"12","mapreduce.jobtracker.tasktracker.maxblacklists":"4","yarn.nodemanager.docker-container-executor.exec-name":"/usr/bin/docker","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"0.23.0","fs.s3a.s3guard.ddb.table.capacity.read":"500","yarn.nodemanager.remote-app-log-dir-suffix":"logs","dfs.namenode.decommission.max.concurrent.tracked.nodes":"100","file.blocksize":"67108864","hadoop.registry.zk.retry.ceiling.ms":"60000","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","dfs.client.read.shortcircuit.skip.checksum":"false","mapreduce.task.profile.reduces":"0-2","dfs.datanode.address":"0.0.0.0:50010","dfs.https.server.keystore.resource":"ssl-server.xml","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"test-1.vpc.company.com:8030","mapreduce.task.timeout":"600000","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"hdfs://test-1.vpc.company.com:8020","fs.har.impl.disable.cache":"true","io.compression.codec.bzip2.library":"system-native","dfs.namenode.audit.loggers":"default","dfs.block.access.key.update.interval":"600","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"test-1.vpc.company.com:8090","dfs.namenode.max.objects":"0","mapreduce.jobhistory.address":"test-1.vpc.company.com:10020","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","dfs.namenode.max.full.block.report.leases":"6","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","dfs.client.mmap.cache.timeout.ms":"3600000","dfs.mover.max-no-move-interval":"60000","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","dfs.client.datanode-restart.timeout":"30","dfs.datanode.drop.cache.behind.reads":"false","ipc.server.log.slow.rpc":"false","dfs.namenode.read-lock-reporting-threshold-ms":"5000","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","dfs.namenode.checkpoint.edits.dir":"${dfs.namenode.checkpoint.dir}","yarn.app.attempt.diagnostics.limit.kc":"64","dfs.balancer.block-move.timeout":"0","dfs.client.block.write.replace-datanode-on-failure.enable":"true","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"8192","io.compression.codecs":"org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec","fs.s3a.fast.upload.active.blocks":"4","hadoop.security.credential.clear-text-fallback":"true","dfs.heartbeat.interval":"3","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","dfs.ha.tail-edits.period":"60","dfs.datanode.max.locked.memory":"0","dfs.datanode.scan.period.hours":"504","mapreduce.jobtracker.expire.trackers.interval":"600000","yarn.resourcemanager.nodemanager-connect-retries":"10","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","dfs.namenode.edits.noeditlogchannelflush":"false","mapreduce.task.io.sort.factor":"64","mapreduce.tasktracker.outofband.heartbeat":"false","ha.failover-controller.new-active.rpc-timeout.ms":"60000","dfs.webhdfs.ugi.expire.after.access":"600000","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","dfs.namenode.https-address":"test-1.vpc.company.com:20102","yarn.am.blacklisting.enabled":"true","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","dfs.image.transfer-bootstrap-standby.bandwidthPerSec":"0","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.proxyuser.hive.groups":"*","ha.zookeeper.session-timeout.ms":"5000","mapreduce.map.java.opts":"-Djava.net.preferIPv4Stack=true","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"100","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","dfs.ha.automatic-failover.enabled":"false","mapreduce.job.counters.groups.max":"50","dfs.namenode.decommission.interval":"30","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","dfs.datanode.handler.count":"10","dfs.namenode.xattrs.enabled":"true","dfs.namenode.safemode.threshold-pct":"0.999f","dfs.client.socket.send.buffer.size":"0","mapreduce.map.sort.spill.percent":"0.8","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","dfs.datanode.sync.behind.writes":"false","dfs.namenode.stale.datanode.interval":"30000","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","dfs.datanode.transfer.socket.send.buffer.size":"0","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","dfs.namenode.reencrypt.throttle.limit.handler.ratio":"1.0","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler","yarn.app.mapreduce.am.command-opts":"-Djava.net.preferIPv4Stack=true -Xmx825955249","dfs.journalnode.https-address":"0.0.0.0:8481","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","hadoop.proxyuser.hue.hosts":"*","io.mapfile.bloom.error.rate":"0.005","dfs.user.home.dir.prefix":"/user","hadoop.proxyuser.hue.groups":"*","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","mapreduce.jobtracker.persist.jobstatus.dir":"/jobtracker/jobsInfo","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","dfs.namenode.snapshot.capture.openfiles":"false","dfs.datanode.directoryscan.interval":"21600","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","dfs.client.read.shortcircuit.streams.cache.expiry.ms":"300000","fs.s3a.connection.timeout":"200000","mapreduce.job.max.split.locations":"10","dfs.namenode.write.stale.datanode.ratio":"0.5f","hadoop.registry.zk.session.timeout.ms":"60000","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","mapreduce.jobtracker.taskcache.levels":"2","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","mapreduce.job.emit-timeline-data":"false","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","dfs.domain.socket.path":"/var/run/hdfs-sockets/dn","yarn.resourcemanager.admin.client.thread-count":"1","mapreduce.jobtracker.persist.jobstatus.active":"true","dfs.namenode.reencrypt.sleep.interval":"1m","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1,SSLv2Hello,TLSv1.1,TLSv1.2","mapreduce.jobhistory.admin.address":"test-1.vpc.company.com:10033","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","mapreduce.tasktracker.report.address":"127.0.0.1:0","dfs.namenode.edit.log.autoroll.multiplier.threshold":"2.0","io.seqfile.lazydecompress":"true","ftp.blocksize":"67108864","dfs.namenode.backup.http-address":"0.0.0.0:50105","dfs.disk.balancer.max.disk.throughputInMBperSec":"10","mapreduce.jobtracker.instrumentation":"org.apache.hadoop.mapred.JobTrackerMetricsInst","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.delete.debug-delay-sec":"0","mapreduce.jobtracker.http.address":"0.0.0.0:50030","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","hadoop.security.groups.cache.secs":"300","yarn.resourcemanager.zk-retry-interval-ms":"1000","yarn.scheduler.increment-allocation-mb":"512","nfs.mountd.port":"4242","mapreduce.shuffle.max.threads":"0","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","mapreduce.jobhistory.http.policy":"HTTP_ONLY","dfs.datanode.dns.interface":"default","mapreduce.reduce.java.opts":"-Djava.net.preferIPv4Stack=true","s3native.replication":"3","hadoop.security.group.mapping.ldap.ssl":"false","dfs.namenode.fs-limits.max-xattrs-per-inode":"32","yarn.client.application-client-protocol.poll-interval-ms":"200","hadoop.proxyuser.flume.groups":"*","dfs.namenode.fs-limits.max-xattr-size":"16384","dfs.namenode.maintenance.replication.min":"1","dfs.client.write.exclude.nodes.cache.expiry.interval.millis":"600000","ha.zookeeper.parent-znode":"/hadoop-ha","dfs.namenode.safemode.extension":"30000","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","dfs.blocksize":"134217728","dfs.namenode.servicerpc-address":"test-1.vpc.company.com:8022","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","yarn.resourcemanager.ha.automatic-failover.enabled":"true","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.resourcemanager.address":"test-1.vpc.company.com:8032","ipc.client.ping":"true","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","dfs.namenode.resource.checked.volumes.minimum":"1","hadoop.proxyuser.HTTP.hosts":"*","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","dfs.namenode.quota.init-threads":"4","dfs.journalnode.http-address":"0.0.0.0:8480","dfs.block.scanner.volume.bytes.per.second":"1048576","hadoop.ssl.enabled":"false","fs.s3a.multipart.purge":"false","dfs.storage.policy.enabled":"true","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","dfs.namenode.edits.dir":"${dfs.namenode.name.dir}","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","dfs.namenode.support.allow.format":"true","dfs.ha.tail-edits.rolledits.timeout":"60","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"10","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","mapreduce.cluster.temp.dir":"${hadoop.tmp.dir}/mapred/temp","s3.replication":"3","dfs.client.failover.connection.retries":"0","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","dfs.namenode.secondary.http-address":"0.0.0.0:50090","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","nfs.wtmax":"1048576","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","nfs.dump.dir":"/tmp/.hdfs-nfs","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","mapreduce.client.submit.file.replication":"3","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","dfs.namenode.service.handler.count":"10","dfs.datanode.data.dir.perm":"700","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","dfs.namenode.name.dir":"file:///dataroot/dataroot/dfs/nn","yarn.resourcemanager.zk-acl":"world:anyone:rwcda","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","fs.ftp.host.port":"21","ipc.ping.interval":"60000","dfs.namenode.num.checkpoints.retained":"2","dfs.namenode.kerberos.internal.spnego.principal":"${dfs.web.authentication.kerberos.principal}","yarn.resourcemanager.admin.address":"test-1.vpc.company.com:8033","file.client-write-packet-size":"65536","hadoop.treat.subject.external":"false","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"false","dfs.disk.balancer.plan.threshold.percent":"10","mapreduce.local.clientfactory.class.name":"org.apache.hadoop.mapred.LocalClientFactory","dfs.client.use.legacy.blockreader":"false","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"256","yarn.nodemanager.localizer.client.thread-count":"5","dfs.namenode.checkpoint.max-retries":"3","dfs.namenode.reject-unresolved-dn-topology-mapping":"false","dfs.namenode.delegation.token.max-lifetime":"*********(redacted)","dfs.client.block.write.replace-datanode-on-failure.min-replication":"0","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"2000","dfs.namenode.num.extra.edits.retained":"1000000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.scheduler.increment-allocation-vcores":"1","fs.df.interval":"60000","fs.s3.sleepTimeSeconds":"10","fs.s3a.multiobjectdelete.enable":"true","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"true","io.file.buffer.size":"65536","hadoop.work.around.non.threadsafe.getpwuid":"false","dfs.permissions.superusergroup":"supergroup","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","mapreduce.tasktracker.dns.interface":"default","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","dfs.permissions.enabled":"true","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/user","mapreduce.reduce.shuffle.read.timeout":"180000","yarn.app.mapreduce.am.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","dfs.datanode.https.address":"0.0.0.0:50475","dfs.namenode.hosts.provider.classname":"org.apache.hadoop.hdfs.server.blockmanagement.HostFileManager","dfs.datanode.transfer.socket.recv.buffer.size":"0","fs.s3a.connection.establish.timeout":"5000","dfs.namenode.fslock.fair":"true","mapreduce.job.running.map.limit":"0","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","hadoop.fuse.connection.timeout":"300","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"5000","yarn.nodemanager.vmem-pmem-ratio":"2.1","dfs.client.slow.io.warning.threshold.ms":"30000","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","s3native.stream-buffer-size":"4096","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","fs.s3a.s3guard.cli.prune.age":"86400000","dfs.client.read.shortcircuit.streams.cache.size":"256","dfs.client.use.legacy.blockreader.local":"false","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.s3n.multipart.uploads.enabled":"false","dfs.namenode.path.based.cache.retry.interval.ms":"30000","hadoop.security.crypto.buffer.size":"8192","yarn.client.failover-retries-on-socket-timeouts":"0","dfs.balancer.keytab.enabled":"false","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","dfs.datanode.balance.bandwidthPerSec":"10485760","dfs.namenode.name.dir.restore":"false","hadoop.registry.jaas.context":"Client","dfs.client.failover.sleep.max.millis":"15000","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","s3.blocksize":"67108864","yarn.am.blacklisting.disable-failure-threshold":"0.8f","io.map.index.interval":"128","mapreduce.job.counters.max":"120","dfs.namenode.max-lock-hold-to-release-lease-ms":"25","dfs.namenode.datanode.registration.ip-hostname-check":"true","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","dfs.namenode.resource.du.reserved":"104857600","dfs.datanode.bp-ready.timeout":"20","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","dfs.namenode.full.block.report.lease.length.ms":"300000","mapreduce.tasktracker.instrumentation":"org.apache.hadoop.mapred.TaskTrackerMetricsInst","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","s3.stream-buffer-size":"4096","yarn.resourcemanager.nm.liveness-monitor.interval-ms":"1000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","dfs.namenode.secondary.https-address":"0.0.0.0:50091","s3native.bytes-per-checksum":"512","dfs.namenode.fs-limits.max-directory-items":"1048576","nfs.server.port":"2049","dfs.namenode.delegation.token.renew-interval":"*********(redacted)","mapreduce.jobtracker.address":"local","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","dfs.namenode.list.openfiles.num.responses":"1000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","dfs.namenode.blocks.per.postponedblocks.rescan":"10000","dfs.namenode.checkpoint.period":"3600","hadoop.security.groups.cache.background.reload":"false","yarn.resourcemanager.amlauncher.log.command":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","dfs.namenode.snapshot.skip.capture.accesstime-only-change":"false","dfs.webhdfs.user.provider.user.pattern":"^[A-Za-z_][A-Za-z0-9._-]*[$]?$","dfs.webhdfs.acl.provider.permission.pattern":"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"false","mapreduce.map.skip.maxrecords":"0","mapreduce.jobtracker.handler.count":"10","hadoop.http.authentication.type":"simple","mapreduce.job.jvm.numtasks":"1","hadoop.proxyuser.flume.hosts":"*","mapreduce.task.userlog.limit.kb":"0","yarn.resourcemanager.scheduler.monitor.enable":"false","fs.s3n.block.size":"67108864","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","mapreduce.jobtracker.staging.root.dir":"${hadoop.tmp.dir}/mapred/staging","dfs.namenode.http-address":"test-1.vpc.company.com:20101","mapreduce.jobtracker.jobhistory.lru.cache.size":"5","dfs.datanode.directoryscan.threads":"1","dfs.datanode.fsdatasetcache.max.threads.per.volume":"4","dfs.namenode.fs-limits.max-blocks-per-file":"1048576","dfs.disk.balancer.enabled":"false","mapreduce.shuffle.listen.queue.size":"128","mapreduce.tasktracker.local.dir.minspacestart":"0","mapreduce.map.cpu.vcores":"1","hadoop.user.group.static.mapping.overrides":"dr.who=;","dfs.datanode.cache.revocation.timeout.ms":"900000","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","dfs.client.mmap.cache.size":"256","dfs.ha.log-roll.period":"120","dfs.client.failover.sleep.base.millis":"500","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","dfs.namenode.accesstime.precision":"3600000","yarn.app.mapreduce.client.job.max-retries":"3","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","hadoop.registry.zk.retry.interval.ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","dfs.client.context":"default","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","hadoop.ssl.server.conf":"ssl-server.xml","dfs.http.policy":"HTTP_ONLY","dfs.client.https.keystore.resource":"ssl-client.xml","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.acl.enable":"true","dfs.domain.socket.disable.interval.seconds":"1","dfs.image.transfer.chunksize":"65536","dfs.balancer.max-no-move-interval":"60000","mapreduce.tasktracker.map.tasks.maximum":"2","dfs.namenode.edits.journal-plugin.qjournal":"org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager","mapreduce.task.profile":"false","dfs.webhdfs.enabled":"true","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","dfs.namenode.list.encryption.zones.num.responses":"100","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","dfs.namenode.top.num.users":"10","dfs.disk.balancer.block.tolerance.percent":"10","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","io.seqfile.sorter.recordlimit":"1000000","hadoop.security.auth_to_local":"DEFAULT","dfs.blockreport.initialDelay":"0","fs.automatic.close":"true","dfs.client.block.write.replace-datanode-on-failure.best-effort":"false","dfs.namenode.replication.min":"1","dfs.balancer.address":"0.0.0.0:0","fs.s3n.multipart.copy.block.size":"5368709120","yarn.nodemanager.hostname":"0.0.0.0","nfs.rtmax":"1048576","yarn.resourcemanager.zk-timeout-ms":"10000","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","dfs.datanode.directoryscan.throttle.limit.ms.per.sec":"1000","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","dfs.namenode.replication.work.multiplier.per.iteration":"2","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","dfs.namenode.reencrypt.throttle.limit.updater.ratio":"1.0","dfs.namenode.avoid.write.stale.datanode":"false","dfs.short.circuit.shared.memory.watcher.interrupt.check.ms":"60000","dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction":"0.75f","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","dfs.client.mmap.enabled":"true","mapreduce.reduce.cpu.vcores":"1","hadoop.proxyuser.oozie.groups":"*","fs.client.resolve.remote.symlinks":"true","dfs.image.compression.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.jobtracker.restart.recover":"false","dfs.namenode.decommission.blocks.per.interval":"500000","mapreduce.tasktracker.reduce.tasks.maximum":"2","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.nodemanager.minimum.version":"NONE","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","dfs.namenode.safemode.min.datanodes":"0","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.userlog.retain.hours":"24","yarn.scheduler.maximum-allocation-vcores":"4","yarn.nodemanager.log-aggregation.compression-type":"none","dfs.namenode.enable.retrycache":"true","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","dfs.namenode.startup.delay.block.deletion.sec":"0","mapreduce.reduce.maxattempts":"4","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","dfs.datanode.readahead.bytes":"4194304","mapreduce.jobtracker.heartbeats.in.second":"100","mapreduce.job.running.reduce.limit":"0","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","mapreduce.task.tmp.dir":"./tmp","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","dfs.namenode.delegation.key.update-interval":"86400000","fs.s3a.max.total.tasks":"5","dfs.client.file-block-storage-locations.num-threads":"10","mapreduce.tasktracker.healthchecker.interval":"60000","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.fast.upload":"false","fs.s3a.attempts.maximum":"20","dfs.namenode.avoid.read.stale.datanode":"false","hadoop.registry.zk.connection.timeout.ms":"15000","dfs.https.port":"20102","yarn.nodemanager.health-checker.script.timeout-ms":"1200000","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","mapreduce.map.log.level":"INFO","mapreduce.output.fileoutputformat.compress.type":"BLOCK","hadoop.registry.rm.enabled":"false","mapreduce.ifile.readahead.bytes":"4194304","mapreduce.tasktracker.tasks.sleeptimebeforesigkill":"5000","yarn.resourcemanager.fs.state-store.retry-policy-spec":"2000, 500","dfs.namenode.posix.acl.inheritance.enabled":"false","dfs.blockreport.intervalMsec":"21600000","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","dfs.namenode.path.based.cache.refresh.interval.ms":"30000","dfs.namenode.edekcacheloader.interval.ms":"1000","file.stream-buffer-size":"4096","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.SnappyCodec","mapreduce.map.speculative":"false","dfs.disk.balancer.max.disk.errors":"5","dfs.datanode.use.datanode.hostname":"false","mapreduce.job.speculative.retry-after-speculate":"15000","hadoop.proxyuser.hdfs.hosts":"*","dfs.namenode.fs-limits.min-block-size":"1048576","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","mapreduce.job.reduce.slowstart.completedmaps":"0.8","dfs.client.read.shortcircuit":"false","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","hadoop.proxyuser.yarn.groups":"*","dfs.client.cached.conn.retry":"3","dfs.namenode.invalidate.work.pct.per.iteration":"0.32f","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.nodemanager.logaggregation.threadpool-size-max":"100","dfs.replication.max":"512","dfs.namenode.inotify.max.events.per.rpc":"1000","yarn.resourcemanager.hostname":"0.0.0.0","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"0","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","ipc.client.rpc-timeout.ms":"0","fs.s3.maxRetries":"4","dfs.default.chunk.view.size":"32768","mapreduce.input.lineinputformat.linespermap":"1","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","dfs.client.mmap.retry.timeout.ms":"300000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","dfs.namenode.list.cache.directives.num.responses":"100","fs.s3a.socket.recv.buffer":"8192","dfs.image.compress":"false","dfs.namenode.kerberos.principal.pattern":"*","yarn.application.classpath":"$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*","fs.s3n.multipart.uploads.block.size":"67108864","mapreduce.tasktracker.http.address":"0.0.0.0:50060","yarn.resourcemanager.resource-tracker.address":"test-1.vpc.company.com:8031","hadoop.fuse.timer.period":"5","mapreduce.job.heap.memory-mb.ratio":"0.8","dfs.datanode.hdfs-blocks-metadata.enabled":"true","dfs.namenode.checkpoint.dir":"file://${hadoop.tmp.dir}/dfs/namesecondary","dfs.datanode.max.transfer.threads":"4096","dfs.namenode.edits.asynclogging":"true","nfs.allow.insecure.ports":"true","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"0","s3native.client-write-packet-size":"65536","mapreduce.admin.user.env":"LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","hadoop.proxyuser.mapred.hosts":"*","hadoop.proxyuser.oozie.hosts":"*","yarn.nodemanager.log.retain-seconds":"10800","hadoop.proxyuser.mapred.groups":"*","yarn.resourcemanager.keytab":"/etc/krb5.keytab","mapreduce.reduce.merge.inmem.threshold":"1000","dfs.client.https.need-auth":"false","dfs.blockreport.split.threshold":"1000000","dfs.client.block.write.replace-datanode-on-failure.policy":"DEFAULT","mapreduce.shuffle.ssl.enabled":"false","dfs.namenode.write-lock-reporting-threshold-ms":"5000","dfs.block.access.token.enable":"*********(redacted)","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","httpfs.buffer.size":"4096","dfs.client.file-block-storage-locations.timeout.millis":"1000","dfs.namenode.block-placement-policy.default.prefer-local-node":"true","mapreduce.job.speculative.minimum-allowed-tasks":"10","yarn.log-aggregation.retain-seconds":"-1","dfs.namenode.replication.considerLoad":"true","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.proxyuser.hdfs.groups":"*","dfs.namenode.retrycache.heap.percent":"0.03f","dfs.datanode.cache.revocation.polling.ms":"500","mapreduce.jobhistory.webapp.address":"test-1.vpc.company.com:19888","dfs.namenode.path.based.cache.block.map.allocation.percent":"0.25","mapreduce.jobtracker.system.dir":"${hadoop.tmp.dir}/mapred/system","mapreduce.tasktracker.taskmemorymanager.monitoringinterval":"5000","dfs.journalnode.rpc-address":"0.0.0.0:8485","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"test-1.vpc.company.com:8088","mapreduce.jobhistory.recovery.enable":"false","dfs.client.short.circuit.replica.stale.threshold.ms":"1800000","mapreduce.reduce.shuffle.parallelcopies":"10","fs.trash.interval":"1","dfs.namenode.replication.interval":"3","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","dfs.namenode.top.enabled":"true","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","dfs.datanode.du.reserved":"0","yarn.app.mapreduce.am.resource.mb":"1024","mapreduce.input.fileinputformat.list-status.num-threads":"1","dfs.namenode.lazypersist.file.scrub.interval.sec":"300","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.nodemanager.resource.cpu-vcores":"8","mapreduce.job.reduces":"6","fs.s3a.multipart.size":"64M","yarn.scheduler.minimum-allocation-vcores":"1","dfs.namenode.reencrypt.batch.size":"1000","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","dfs.datanode.http.address":"0.0.0.0:50075","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","fs.s3a.metadatastore.authoritative":"false","ha.health-monitor.sleep-after-disconnect.ms":"1000","s3.bytes-per-checksum":"512","yarn.app.mapreduce.shuffle.log.limit.kb":"0","dfs.namenode.list.cache.pools.num.responses":"100","hadoop.security.group.mapping":"org.apache.hadoop.security.ShellBasedUnixGroupsMapping","mapreduce.jobhistory.jhist.format":"binary","yarn.resourcemanager.ha.enabled":"false","dfs.encrypt.data.transfer":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","mapreduce.job.redacted-properties":"*********(redacted)","dfs.namenode.top.windows.minutes":"1,5,25","s3.client-write-packet-size":"65536","mapreduce.map.output.compress":"true","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.scheduler.maximum-allocation-mb":"12288","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.app.mapreduce.am.container.log.limit.kb":"0","s3native.blocksize":"67108864","ipc.client.connect.retry.interval":"1000","hadoop.proxyuser.httpfs.groups":"*","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","dfs.namenode.edit.log.autoroll.check.interval.ms":"300000","mapreduce.jobhistory.cleaner.enable":"true","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","hadoop.proxyuser.httpfs.hosts":"*","dfs.client.use.datanode.hostname":"false","dfs.stream-buffer-size":"4096","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","dfs.datanode.drop.cache.behind.writes":"false","mapreduce.tasktracker.dns.nameserver":"default","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","mapreduce.job.end-notification.retry.attempts":"0","hadoop.proxyuser.yarn.hosts":"*","yarn.resourcemanager.zk-num-retries":"1000","dfs.client.failover.max.attempts":"15","mapreduce.tasktracker.indexcache.mb":"10","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","dfs.namenode.snapshotdiff.allow.snap-root-descendant":"true","yarn.nodemanager.localizer.cache.target-size-mb":"10240","zlib.compress.level":"DEFAULT_COMPRESSION","ftp.client-write-packet-size":"65536","mapreduce.jobtracker.maxtasks.perjob":"-1","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","hadoop.proxyuser.hive.hosts":"*","dfs.block.access.token.lifetime":"*********(redacted)","dfs.namenode.max.extra.edits.segments.retained":"10000","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","dfs.image.transfer.bandwidthPerSec":"0","io.native.lib.available":"true","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","ipc.server.listen.queue.size":"128","dfs.namenode.edekcacheloader.initial.delay.ms":"3000","map.sort.class":"org.apache.hadoop.util.QuickSort","dfs.namenode.acls.enabled":"false","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","dfs.datanode.ipc.address":"0.0.0.0:50020","yarn.nodemanager.vmem-check-enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","dfs.lock.suppress.warning.interval":"10s","dfs.client.block.write.retries":"3","mapreduce.job.ubertask.maxreduces":"1","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.application.classpath":"$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH","yarn.client.nodemanager-connect.retry-interval-ms":"10000","dfs.client-write-packet-size":"65536","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME","dfs.datanode.dns.nameserver":"default","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","dfs.image.transfer.timeout":"60000","yarn.resourcemanager.recovery.enabled":"false","dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold":"10737418240","dfs.client.failover.connection.retries.on.timeouts":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.version":"1.8","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Oracle Corporation","java.security.egd":"file:///dev/urandom","java.vm.specification.version":"1.8","user.home":"/home/systest","file.encoding.pkg":"sun.io","sun.nio.ch.bugLevel":"","sun.arch.data.model":"64","sun.boot.library.path":"/usr/java/jdk1.8.0_144/jre/lib/amd64","user.dir":"/tmp","java.library.path":":/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/opt/cloudera/parcels/CDH/lib/hadoop/lib/native:/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib","sun.cpu.isalist":"","os.arch":"amd64","java.vm.version":"25.144-b01","jetty.git.hash":"27208684755d94a92186989f695db2d7b21ebc51","java.endorsed.dirs":"/usr/java/jdk1.8.0_144/jre/lib/endorsed","java.runtime.version":"1.8.0_144-b01","java.vm.info":"mixed mode","java.ext.dirs":"/usr/java/jdk1.8.0_144/jre/lib/ext:/usr/java/packages/lib/ext","java.runtime.name":"Java(TM) SE Runtime Environment","file.separator":"/","java.class.version":"52.0","java.specification.name":"Java Platform API Specification","sun.boot.class.path":"/usr/java/jdk1.8.0_144/jre/lib/resources.jar:/usr/java/jdk1.8.0_144/jre/lib/rt.jar:/usr/java/jdk1.8.0_144/jre/lib/sunrsasign.jar:/usr/java/jdk1.8.0_144/jre/lib/jsse.jar:/usr/java/jdk1.8.0_144/jre/lib/jce.jar:/usr/java/jdk1.8.0_144/jre/lib/charsets.jar:/usr/java/jdk1.8.0_144/jre/lib/jfr.jar:/usr/java/jdk1.8.0_144/jre/classes","file.encoding":"UTF-8","user.timezone":"America/Los_Angeles","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","os.version":"3.10.0-514.26.2.el7.x86_64","sun.os.patch.level":"unknown","java.vm.specification.vendor":"Oracle Corporation","user.country":"US","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"http://java.oracle.com/","java.awt.printerjob":"sun.print.PSPrinterJob","java.awt.graphicsenv":"sun.awt.X11GraphicsEnvironment","awt.toolkit":"sun.awt.X11.XToolkit","os.name":"Linux","java.vm.vendor":"Oracle Corporation","java.vendor.url.bug":"http://bugreport.sun.com/bugreport/","user.name":"systest","java.vm.name":"Java HotSpot(TM) 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master yarn --deploy-mode client --conf spark.driver.memory=2g --conf spark.executor.heartbeatInterval=1000 --conf spark.executor.metrics.pollingInterval=100 --conf spark.eventLog.logStageExecutorProcessTreeMetrics.enabled=true --conf spark.yarn.maxAppAttempts=1 --conf spark.locality.wait.process=0 --conf spark.executor.memoryOverhead=1024 --conf spark.executor.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --conf spark.eventLog.logStageExecutorMetrics=true --conf spark.driver.extraJavaOptions=-Djava.security.egd=file:///dev/urandom --class com.company.spark.LargeBlocks --num-executors 3 --executor-memory 7g /tmp/__spark_test__/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar --targetBlockSizeGb 2.5 --taskSleepMillis 200 --doCache true --cacheOnDisk true --replicas 1 --concurrentReadJobs 2","java.home":"/usr/java/jdk1.8.0_144/jre","java.version":"1.8.0_144","sun.io.unicode.encoding":"UnicodeLittle"},"Classpath Entries":{"/opt/cloudera/parcels/CDH/jars/jackson-mapper-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/joni-2.1.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-external-blockcache-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-json-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/xz-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-annotations-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/cglib-2.2.1-v20090111.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-procedure-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-jackson-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/asm-3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xml-apis-1.3.04.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-jvm-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-dbcp-1.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-streaming_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-xml-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/minlog-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hsqldb-1.8.0.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-rdbms-3.2.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pmml-model-1.4.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire-macros_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zstd-jni-1.3.2-2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpcore-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jta-1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-yarn_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-logging-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/activation-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xbean-asm7-shaded-4.12.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/NOTICE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-format-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/httpclient-4.2.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-3.9.9.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/htrace-core-3.1.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.ws.rs-api-2.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-core-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-nativetask-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-client-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-xc-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-configuration-1.6.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math3-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsp-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-auth-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-registry-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/JavaEWAH-0.3.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-graphite-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/joda-time-2.9.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-1.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-gridmix-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-compress-1.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javolution-5.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-1.7.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/flatbuffers-java-1.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/core-1.1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-runtime-3.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-hdfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-net-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-framework-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-java-1.1.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-assembly_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-examples-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/shapeless_2.12-2.3.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xercesImpl-2.9.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-server-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang-2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jtransforms-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/snappy-0.2.jar":"System Classpath","/etc/spark2/conf/yarn-conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-cli-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.annotation-api-1.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0-2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javassist-3.18.1-GA.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-applicationhistoryservice-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka-clients-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guice-3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr4-runtime-4.7.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/core-3.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/opencsv-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-datajoin-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-common-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jdo-api-3.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-webapp-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compiler-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-annotations-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/libthrift-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/kafka_2.11-0.9.0-kafka-2.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-proxy-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/antlr-2.7.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jettison-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-core-asl-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libfb303-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/zookeeper-3.4.5-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-resource-bundle-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jaxb-api-2.2.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-streaming-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop2-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/pyrolite-4.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/activation-1.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aircompressor-0.10.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-ant-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jline-2.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-all-4.0.23.Final.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-media-jaxb-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlets-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-xml_2.12-1.0.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-graphx_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-hs-plugins-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-web-proxy-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-math-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-kvstore_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/objenesis-2.5.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-i18n-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stream-2.7.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-aws-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-scala_2.12-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/spymemcached-2.11.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-util-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-core-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/mockito-all-1.8.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jets3t-0.9.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-xc-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rsgroup-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-metastore-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/py4j-0.10.8.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-distributedshell-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/osgi-resource-locator-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-rumen-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsch-0.1.42.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/zkclient-0.7.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xmlenc-0.52.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-io-2.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/metrics-core-3.1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jamon-runtime-2.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-common-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-common-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/netty-3.10.5.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-app-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/machinist_2.12-0.6.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-util-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-shuffle_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill_2.12-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/findbugs-annotations-1.3.9-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-rest-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-hadoop-compat-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ST4-4.0.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jruby-cloudera-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-hive_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/leveldbjni-all-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-core_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-distcp-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-server-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core-3.2.0-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-mapreduce-1.5.5-nohive.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jcl-over-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-digester-1.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-http-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-jackson_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-2.2.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-math3-3.4.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-log4j12-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-jobclient-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-shell-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-server-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/chill-java-0.9.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-locator-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stax-api-1.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-parser-combinators_2.12-1.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/RoaringBitmap-0.5.11.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-api-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/log4j-1.2.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.inject-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-compiler-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-security-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-shuffle-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-nfs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/netty-all-4.1.30.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-api-2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze-macros_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jodd-core-3.5.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/azure-data-lake-store-sdk-2.2.9.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-12.0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-codec-1.10.jar":"System Classpath","/usr/java/jdk1.8.0_144/lib/tools.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-nodemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sql_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hive-exec-1.2.1.spark2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jcodings-1.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-azure-datalake-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apacheds-kerberos-codec-2.0.0-M15.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/guava-11.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-mapred-1.8.2-hadoop2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-catalyst_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-io-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-applications-unmanaged-am-launcher-2.6.0-cdh5.15.2.jar":"System Classpath","/etc/spark2/conf/":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-sslengine-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-api-jdo-3.2.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpclient-4.5.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-continuation-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-ast_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-jndi-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsr305-3.0.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/avro-1.7.6-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/univocity-parsers-2.7.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-annotations-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/api-asn1-api-1.0.0-M20.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jettison-1.3.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-utils-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hppc-0.7.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/logredactor-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-vector-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jaxb-impl-2.2.3-1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archives-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-plus-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-client-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hue-plugins-3.9.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/high-scale-lib-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-jaxrs-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-collections-3.2.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-library-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/java-xmlbuilder-0.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/apache-log4j-extras-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-format-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/paranamer-2.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spire_2.12-0.13.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/disruptor-3.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib-local_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-pool-1.5.4.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-servlet-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arpack_combined_all-0.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-annotations-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jsp-2.1-6.1.14.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/log4j-1.2.17.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-repl_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/derby-10.12.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/orc-shims-1.5.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-client-core-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/aopalliance-repackaged-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/hk2-api-2.4.0-b34.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-daemon-1.0.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/ivy-2.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hamcrest-core-1.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-lang3-3.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/spark-streaming-kafka-0-8_2.11-2.4.0.cloudera1-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-databind-2.2.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/kryo-shaded-4.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-reflect-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-mapreduce-examples-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/httpcore-4.4.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/arrow-memory-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/lib/hadoop/LICENSE.txt":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-compress-1.8.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-client-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/oro-2.0.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-thrift-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/avro-ipc-1.8.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-httpclient-3.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-encoding-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okhttp-2.4.0.jar":"System Classpath","spark://test-1.vpc.company.com:34194/jars/spark3-tests-0.1.0-cdh5.9.0-SNAPSHOT-jar-with-dependencies.jar":"Added By User","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/scala-compiler-2.12.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-prefix-tree-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-6.1.26.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-api-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.9.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-beanutils-core-1.8.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/kafka-0.9/lz4-1.3.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/janino-3.0.11.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/libthrift-0.12.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-launcher_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-core_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/lz4-java-1.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jackson-jaxrs-1.8.8.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-network-common_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/macro-compat_2.12-1.1.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/breeze_2.12-0.13.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-unsafe_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-sls-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/snappy-java-1.0.4.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-guava-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/validation-api-1.1.0.Final.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/microsoft-windowsazure-storage-sdk-0.6.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/compress-lzf-1.0.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-openstack-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-mllib_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jetty-client-9.4.12.v20180830.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-it-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-extras-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jersey-container-servlet-2.22.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/javax.servlet-api-3.1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/curator-recipes-2.7.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-column-1.10.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/unused-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/commons-crypto-1.0.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-databind-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-codec-1.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/htrace-core4-4.0.1-incubating.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/guava-14.0.1.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/aws-java-sdk-bundle-1.11.134.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/zookeeper-3.4.6.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jackson-module-paranamer-2.9.8.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hbase-protocol-1.2.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/jasper-runtime-5.5.23.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/json4s-scalap_2.12-3.5.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-logging-1.1.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-resourcemanager-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-archive-logs-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/paranamer-2.3.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/metrics-core-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-beanutils-1.9.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jul-to-slf4j-1.7.16.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/stringtemplate-3.2.1.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/gson-2.2.4.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/okio-1.4.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/slf4j-api-1.7.5.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/jsr305-3.0.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-sketch_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/commons-el-1.0.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/spark-tags_2.12-3.0.0-SNAPSHOT.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/datanucleus-core-3.2.10.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/protobuf-java-2.5.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-server-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/parquet-hadoop-bundle-1.6.0.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-common-2.6.0-cdh5.15.2.jar":"System Classpath","/opt/cloudera/parcels/SPARK2/lib/spark2/jars/xz-1.5.jar":"System Classpath","/opt/cloudera/parcels/CDH/jars/hadoop-yarn-client-2.6.0-cdh5.15.2.jar":"System Classpath"}}
 {"Event":"SparkListenerApplicationStart","App Name":"LargeBlocks","App ID":"application_1553914137147_0018","Timestamp":1554755984286,"User":"systest"}
 {"Event":"SparkListenerExecutorAdded","Timestamp":1554755994596,"Executor ID":"1","Executor Info":{"Host":"test-2.vpc.company.com","Total Cores":1,"Log Urls":{"stdout":"http://test-2.vpc.company.com:8042/node/containerlogs/container_1553914137147_0018_01_000002/systest/stdout?start=-4096","stderr":"http://test-2.vpc.company.com:8042/node/containerlogs/container_1553914137147_0018_01_000002/systest/stderr?start=-4096"},"Attributes":{"NM_HTTP_ADDRESS":"test-2.vpc.company.com:8042","USER":"systest","LOG_FILES":"stderr,stdout","NM_HTTP_PORT":"8042","CLUSTER_ID":"","NM_PORT":"8041","HTTP_SCHEME":"http://","NM_HOST":"test-2.vpc.company.com","CONTAINER_ID":"container_1553914137147_0018_01_000002"}}}
 {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"test-2.vpc.company.com","Port":43764},"Maximum Memory":3820172083,"Timestamp":1554755994649,"Maximum Onheap Memory":3820172083,"Maximum Offheap Memory":0}
diff --git a/docs/configuration.md b/docs/configuration.md
index 6aafc9110652b..df1cbe4c2ebea 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -194,7 +194,7 @@ of the most common options to set are:
   </td>
 </tr>
 <tr>
- <td><code>spark.resources.coordinate.enable</code></td>
+ <td><code>spark.resources.coordinateResourcesInStandalone</code></td>
   <td>true</td>
   <td>
     Whether to coordinate resources automatically among workers/drivers(client only) 
@@ -230,7 +230,7 @@ of the most common options to set are:
     write to STDOUT a JSON string in the format of the ResourceInformation class. This has a
     name and an array of addresses. For a client-submitted driver in Standalone, discovery
     script must assign different resource addresses to this driver comparing to workers' and
-    other drivers' when <code>spark.resources.coordinate.enable</code> is off.
+    other drivers' when <code>spark.resources.coordinateResourcesInStandalone</code> is off.
   </td>
 </tr>
 <tr>
@@ -1641,7 +1641,7 @@ Apart from these, the following properties are also available, and may be useful
 <table class="table">
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
 <tr>
-  <td><code>spark.eventLog.logStageExecutorMetrics.enabled</code></td>
+  <td><code>spark.eventLog.logStageExecutorMetrics</code></td>
   <td>false</td>
   <td>
     Whether to write per-stage peaks of executor metrics (for each executor) to the event log.
@@ -2755,5 +2755,5 @@ There are configurations available to request resources for the driver: <code>sp
 
 Spark will use the configurations specified to first request containers with the corresponding resources from the cluster manager. Once it gets the container, Spark launches an Executor in that container which will discover what resources the container has and the addresses associated with each resource. The Executor will register with the Driver and report back the resources available to that Executor. The Spark scheduler can then schedule tasks to each Executor and assign specific resource addresses based on the resource requirements the user specified. The user can see the resources assigned to a task using the <code>TaskContext.get().resources</code> api. On the driver, the user can see the resources assigned with the SparkContext <code>resources</code> call. It's then up to the user to use the assignedaddresses to do the processing they want or pass those into the ML/AI framework they are using.
 
-See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. If using local-cluster mode see the Spark Standalone documentation but be aware only a single worker resources file or discovery script can be specified the is shared by all the Workers so you should enable resource coordination (see <code>spark.resources.coordinate.enable</code>).
+See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. If using local-cluster mode see the Spark Standalone documentation but be aware only a single worker resources file or discovery script can be specified the is shared by all the Workers so you should enable resource coordination (see <code>spark.resources.coordinateResourcesInStandalone</code>).
 
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 3061c6761cdcf..c30aa99679399 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -659,7 +659,7 @@ A list of the available metrics, with a short description:
 
 Executor-level metrics are sent from each executor to the driver as part of the Heartbeat to describe the performance metrics of Executor itself like JVM heap memory, GC information.
 Executor metric values and their measured peak values per executor are exposed via the REST API at the end point `/applications/[app-id]/executors`.
-In addition, aggregated per-stage peak values of the executor metrics are written to the event log if `spark.eventLog.logStageExecutorMetrics.enabled` is true.
+In addition, aggregated per-stage peak values of the executor metrics are written to the event log if `spark.eventLog.logStageExecutorMetrics` is true.
 Executor metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
 A list of the available metrics, with a short description:
 
diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index f7f20389aa694..03e062f002680 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -34,7 +34,7 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
   - In PySpark, when creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. Since 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a `SparkSession`.
 
-  - In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. Setting `spark.sql.execution.pandas.arrowSafeTypeConversion` to true can enable it. The default setting is false. PySpark's behavior for Arrow versions is illustrated in the table below:
+  - In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. Setting `spark.sql.execution.pandas.convertToArrowArraySafely` to true can enable it. The default setting is false. PySpark's behavior for Arrow versions is illustrated in the table below:
     <table class="table">
         <tr>
           <th>
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 1264951a2f270..ff310edb5a633 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -256,7 +256,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Path to resource discovery script, which is used to find a particular resource while worker starting up.
     And the output of the script should be formatted like the <code>ResourceInformation</code> class.
-    When <code>spark.resources.coordinate.enable</code> is off, the discovery script must assign different
+    When <code>spark.resources.coordinateResourcesInStandalone</code> is off, the discovery script must assign different
     resources for workers and drivers in client mode that run on the same host to avoid resource conflict.
   </td>
 </tr>
@@ -267,7 +267,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Path to resources file which is used to find various resources while worker starting up.
     The content of resources file should be formatted like <code>
     [[{"id":{"componentName": "spark.worker","resourceName":"gpu"},"addresses":["0","1","2"]}]]</code>.
-    When <code>spark.resources.coordinate.enable</code> is off, resources file must assign different
+    When <code>spark.resources.coordinateResourcesInStandalone</code> is off, resources file must assign different
     resources for workers and drivers in client mode that run on the same host to avoid resource conflict.
     If a particular resource is not found in the resources file, the discovery script would be used to
     find that resource. If the discovery script also does not find the resources, the worker will fail
@@ -346,9 +346,9 @@ Please make sure to have read the Custom Resource Scheduling and Configuration O
 
 Spark Standalone has 2 parts, the first is configuring the resources for the Worker, the second is the resource allocation for a specific application.
 
-The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup. Please take note of <code>spark.resources.coordinate.enable</code> as it indicates whether Spark should handle coordinating resources or if the user has made sure each Worker has separate resources. Also note that if using the resources coordination <code>spark.resources.dir</code> can be used to specify the directory used to do that coordination.
+The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup. Please take note of <code>spark.resources.coordinateResourcesInStandalone</code> as it indicates whether Spark should handle coordinating resources or if the user has made sure each Worker has separate resources. Also note that if using the resources coordination <code>spark.resources.dir</code> can be used to specify the directory used to do that coordination.
 
-The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resources.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers or Workers there are 2 ways to make sure the they don't use the same resources. The user can either configure <code>spark.resources.coordinate.enable</code> on and give all the Driver/Workers the same set or resources and Spark will handle make sure each Driver/Worker has separate resources, or the user can make sure the resources file or discovery script only returns resources the do not conflict with other Drivers or Workers running on the same node.
+The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resources.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers or Workers there are 2 ways to make sure the they don't use the same resources. The user can either configure <code>spark.resources.coordinateResourcesInStandalone</code> on and give all the Driver/Workers the same set or resources and Spark will handle make sure each Driver/Worker has separate resources, or the user can make sure the resources file or discovery script only returns resources the do not conflict with other Drivers or Workers running on the same node.
 
 Note, the user does not need to specify a discovery script when submitting an application as the Worker will start each Executor with the resources it allocates to it.
 
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 54a1fbf2cac8e..76df66b9de50c 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -97,7 +97,7 @@ license: |
 
   - Since Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark will throw runtime NPE if any of the records contains null.
 
-  - Since Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, i.e., if the `predicate` returns any `null`s and no `true` is obtained, then `exists` will return `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` will be `null`. The previous behaviour can be restored by setting `spark.sql.legacy.arrayExistsFollowsThreeValuedLogic` to `false`.
+  - Since Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, i.e., if the `predicate` returns any `null`s and no `true` is obtained, then `exists` will return `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` will be `null`. The previous behaviour can be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
 
   - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
 
@@ -109,7 +109,7 @@ license: |
 
   - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
 
-  - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin.enabled` to `false`.
+  - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
 
   - Since Spark 3.0, `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in case insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behaviour change is illustrated in the table below:
     <table class="table">
@@ -258,13 +258,13 @@ license: |
 
   - Since Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
   
-  - Since Spark 3.0, negative scale of decimal is not allowed by default, e.g. data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and earlier, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal.enabled` to `true`.
+  - Since Spark 3.0, negative scale of decimal is not allowed by default, e.g. data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and earlier, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
 
   - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
 
   - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
 
-  - Since Spark 3.0, the properties listing below become reserved, commands will fail if we specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. We need their specific clauses to specify them, e.g. `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. We can set `spark.sql.legacy.property.nonReserved` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, e.g `SET DBPROTERTIES('location'='/tmp')` will affect nothing. In Spark version 2.4 and earlier, these properties are neither reserved nor have side effects, e.g. `SET DBPROTERTIES('location'='/tmp')` will not change the location of the database but only create a headless property just like `'a'='b'`.
+  - Since Spark 3.0, the properties listing below become reserved, commands will fail if we specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. We need their specific clauses to specify them, e.g. `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. We can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, e.g `SET DBPROTERTIES('location'='/tmp')` will affect nothing. In Spark version 2.4 and earlier, these properties are neither reserved nor have side effects, e.g. `SET DBPROTERTIES('location'='/tmp')` will not change the location of the database but only create a headless property just like `'a'='b'`.
     <table class="table">
         <tr>
           <th>
diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index 6f46e92f5a2a4..4dd15d14b9c53 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -161,7 +161,7 @@ def create_array(s, t):
                             "Array (%s). It can be caused by overflows or other unsafe " + \
                             "conversions warned by Arrow. Arrow safe type check can be " + \
                             "disabled by using SQL config " + \
-                            "`spark.sql.execution.pandas.arrowSafeTypeConversion`."
+                            "`spark.sql.execution.pandas.convertToArrowArraySafely`."
                 raise RuntimeError(error_msg % (s.dtype, t), e)
             return array
 
diff --git a/python/pyspark/sql/tests/test_pandas_udf.py b/python/pyspark/sql/tests/test_pandas_udf.py
index 94801a16ce7cd..4218f5cfc401f 100644
--- a/python/pyspark/sql/tests/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/test_pandas_udf.py
@@ -211,14 +211,14 @@ def udf(column):
 
         # Since 0.11.0, PyArrow supports the feature to raise an error for unsafe cast.
         with self.sql_conf({
-                "spark.sql.execution.pandas.arrowSafeTypeConversion": True}):
+                "spark.sql.execution.pandas.convertToArrowArraySafely": True}):
             with self.assertRaisesRegexp(Exception,
                                          "Exception thrown when converting pandas.Series"):
                 df.select(['A']).withColumn('udf', udf('A')).collect()
 
         # Disabling Arrow safe type check.
         with self.sql_conf({
-                "spark.sql.execution.pandas.arrowSafeTypeConversion": False}):
+                "spark.sql.execution.pandas.convertToArrowArraySafely": False}):
             df.select(['A']).withColumn('udf', udf('A')).collect()
 
     def test_pandas_udf_arrow_overflow(self):
@@ -232,13 +232,13 @@ def udf(column):
 
         # When enabling safe type check, Arrow 0.11.0+ disallows overflow cast.
         with self.sql_conf({
-                "spark.sql.execution.pandas.arrowSafeTypeConversion": True}):
+                "spark.sql.execution.pandas.convertToArrowArraySafely": True}):
             with self.assertRaisesRegexp(Exception,
                                          "Exception thrown when converting pandas.Series"):
                 df.withColumn('udf', udf('id')).collect()
 
         # Disabling safe type check, let Arrow do the cast anyway.
-        with self.sql_conf({"spark.sql.execution.pandas.arrowSafeTypeConversion": False}):
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
             df.withColumn('udf', udf('id')).collect()
 
 
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 37ce62b7f7159..81402f52af3b3 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -205,13 +205,13 @@ def test_create_dataframe_from_dict_respects_schema(self):
 
     def test_negative_decimal(self):
         try:
-            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal.enabled=true")
+            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal=true")
             df = self.spark.createDataFrame([(1, ), (11, )], ["value"])
             ret = df.select(col("value").cast(DecimalType(1, -1))).collect()
             actual = list(map(lambda r: int(r.value), ret))
             self.assertEqual(actual, [0, 10])
         finally:
-            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal.enabled=false")
+            self.spark.sql("set spark.sql.legacy.allowNegativeScaleOfDecimal=false")
 
     def test_create_dataframe_from_objects(self):
         data = [MyObject(1, "1"), MyObject(2, "2")]
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 5d498421e259d..7d62bf1783931 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -306,7 +306,7 @@ def read_udfs(pickleSer, infile, eval_type):
 
         # NOTE: if timezone is set here, that implies respectSessionTimeZone is True
         timezone = runner_conf.get("spark.sql.session.timeZone", None)
-        safecheck = runner_conf.get("spark.sql.execution.pandas.arrowSafeTypeConversion",
+        safecheck = runner_conf.get("spark.sql.execution.pandas.convertToArrowArraySafely",
                                     "false").lower() == 'true'
         # Used by SQL_GROUPED_MAP_PANDAS_UDF and SQL_SCALAR_PANDAS_UDF when returning StructType
         assign_cols_by_name = runner_conf.get(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index ac3169d002713..0783c49378aab 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -368,14 +368,14 @@ object SQLConf {
     .createWithDefault(false)
 
   val REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled")
+    buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions")
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables reducing " +
         "the number of post-shuffle partitions based on map output statistics.")
       .booleanConf
       .createWithDefault(true)
 
   val FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch.enabled")
+    buildConf("spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch")
       .doc("Whether to fetch the continuous shuffle blocks in batch. Instead of fetching blocks " +
         "one by one, fetching continuous shuffle blocks for the same map task in batch can " +
         "reduce IO and improve performance. Note, multiple continuous blocks exist in single " +
@@ -425,7 +425,7 @@ object SQLConf {
     .createWithDefault(true)
 
   val ADAPTIVE_EXECUTION_SKEWED_JOIN_ENABLED =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.enabled")
+    buildConf("spark.sql.adaptive.skewedJoinOptimization.enabled")
     .doc("When true and adaptive execution is enabled, a skewed join is automatically handled at " +
       "runtime.")
     .booleanConf
@@ -894,7 +894,7 @@ object SQLConf {
       .createWithDefault(10000)
 
   val IGNORE_DATA_LOCALITY =
-    buildConf("spark.sql.sources.ignoreDataLocality.enabled")
+    buildConf("spark.sql.sources.ignoreDataLocality")
       .doc("If true, Spark will not fetch the block locations for each file on " +
         "listing files. This speeds up file listing, but the scheduler cannot " +
         "schedule tasks to take advantage of data locality. It can be particularly " +
@@ -913,7 +913,7 @@ object SQLConf {
       .createWithDefault(true)
 
   val FAIL_AMBIGUOUS_SELF_JOIN_ENABLED =
-    buildConf("spark.sql.analyzer.failAmbiguousSelfJoin.enabled")
+    buildConf("spark.sql.analyzer.failAmbiguousSelfJoin")
       .doc("When true, fail the Dataset query if it contains ambiguous self-join.")
       .internal()
       .booleanConf
@@ -1062,7 +1062,7 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
-  val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.subquery.reuse.enabled")
+  val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.reuseSubquery")
     .internal()
     .doc("When true, the planner will try to find out duplicated subqueries and re-use them.")
     .booleanConf
@@ -1102,7 +1102,7 @@ object SQLConf {
     .createOptional
 
   val FORCE_DELETE_TEMP_CHECKPOINT_LOCATION =
-    buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation.enabled")
+    buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation")
       .doc("When true, enable temporary checkpoint locations force delete.")
       .booleanConf
       .createWithDefault(false)
@@ -1629,7 +1629,7 @@ object SQLConf {
       .createWithDefault(true)
 
   val PANDAS_ARROW_SAFE_TYPE_CONVERSION =
-    buildConf("spark.sql.execution.pandas.arrowSafeTypeConversion")
+    buildConf("spark.sql.execution.pandas.convertToArrowArraySafely")
       .internal()
       .doc("When true, Arrow will perform safe type conversion when converting " +
         "Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
@@ -1959,7 +1959,7 @@ object SQLConf {
       .createWithDefault(false)
 
   val LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED =
-    buildConf("spark.sql.legacy.allowNegativeScaleOfDecimal.enabled")
+    buildConf("spark.sql.legacy.allowNegativeScaleOfDecimal")
       .internal()
       .doc("When set to true, negative scale of Decimal type is allowed. For example, " +
         "the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is " +
@@ -2100,7 +2100,7 @@ object SQLConf {
       .stringConf
       .createOptional
 
-  val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.looseUpcast")
+  val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
     .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
     .booleanConf
@@ -2122,7 +2122,7 @@ object SQLConf {
     .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
   val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
-    buildConf("spark.sql.legacy.arrayExistsFollowsThreeValuedLogic")
+    buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
       .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
       .booleanConf
@@ -2149,7 +2149,7 @@ object SQLConf {
       .createWithDefault(false)
 
   val LEGACY_PROPERTY_NON_RESERVED =
-    buildConf("spark.sql.legacy.property.nonReserved")
+    buildConf("spark.sql.legacy.notReserveProperties")
       .internal()
       .doc("When true, all database and table properties are not reserved and available for " +
         "create/alter syntaxes. But please be aware that the reserved properties will be " +
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
index 05069e24e2329..360a14060c15d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
@@ -159,7 +159,7 @@ object DecimalType extends AbstractDataType {
   private[sql] def checkNegativeScale(scale: Int): Unit = {
     if (scale < 0 && !SQLConf.get.allowNegativeScaleOfDecimalEnabled) {
       throw new AnalysisException(s"Negative scale is not allowed: $scale. " +
-        s"You can use spark.sql.legacy.allowNegativeScaleOfDecimal.enabled=true " +
+        s"You can use spark.sql.legacy.allowNegativeScaleOfDecimal=true " +
         s"to enable legacy mode to allow it.")
     }
   }

From 9947f3d6b568685287c9a5f98bf0a3ecfcd1ae88 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 18 Feb 2020 21:19:57 +0100
Subject: [PATCH 0110/1280] revert SPARK-29663 and SPARK-29688

### What changes were proposed in this pull request?

This PR reverts https://github.com/apache/spark/pull/26325 and https://github.com/apache/spark/pull/26347

### Why are the changes needed?

When we do sum/avg, we need a wider type of input to hold the sum value, to reduce the possibility of overflow. For example, we use long to hold the sum of integral inputs, use double to hold the sum of float/double.

However, we don't have a wider type of interval. Also the semantic is unclear: what if the days field overflows but the months field doesn't? Currently the avg of `1 month` and `2 month` is `1 month 15 days`, which assumes 1 month has 30 days and we should avoid this assumption.

### Does this PR introduce any user-facing change?

yes, remove 2 features added in 3.0

### How was this patch tested?

N/A

Closes #27619 from cloud-fan/revert.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit 1b67d546bd96412943de0c7a3b4295cbde887bd2)
Signed-off-by: herman <herman@databricks.com>
---
 .../expressions/aggregate/Average.scala       |  15 +-
 .../catalyst/expressions/aggregate/Sum.scala  |  14 +-
 .../ExpressionTypeCheckingSuite.scala         |   4 +-
 .../resources/sql-tests/inputs/interval.sql   |  64 -------
 .../sql-tests/results/ansi/interval.sql.out   | 176 +-----------------
 .../sql-tests/results/interval.sql.out        | 176 +-----------------
 .../spark/sql/DataFrameAggregateSuite.scala   |  13 --
 7 files changed, 17 insertions(+), 445 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
index 996c548e1329c..17f906c698de2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
@@ -17,9 +17,10 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{DecimalPrecision, FunctionRegistry}
+import org.apache.spark.sql.catalyst.analysis.{DecimalPrecision, FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.types._
 
 @ExpressionDescription(
@@ -30,8 +31,6 @@ import org.apache.spark.sql.types._
        2.0
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (NULL) AS tab(col);
        1.5
-      > SELECT _FUNC_(cast(v as interval)) FROM VALUES ('-1 weeks'), ('2 seconds'), (null) t(v);
-       -3 days -11 hours -59 minutes -59 seconds
   """,
   since = "1.0.0")
 case class Average(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
@@ -40,7 +39,10 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
 
   override def children: Seq[Expression] = child :: Nil
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection.NumericAndInterval)
+  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
+
+  override def checkInputDataTypes(): TypeCheckResult =
+    TypeUtils.checkForNumericExpr(child.dataType, "function average")
 
   override def nullable: Boolean = true
 
@@ -50,13 +52,11 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
   private lazy val resultType = child.dataType match {
     case DecimalType.Fixed(p, s) =>
       DecimalType.bounded(p + 4, s + 4)
-    case interval: CalendarIntervalType => interval
     case _ => DoubleType
   }
 
   private lazy val sumDataType = child.dataType match {
     case _ @ DecimalType.Fixed(p, s) => DecimalType.bounded(p + 10, s)
-    case interval: CalendarIntervalType => interval
     case _ => DoubleType
   }
 
@@ -79,9 +79,6 @@ case class Average(child: Expression) extends DeclarativeAggregate with Implicit
   override lazy val evaluateExpression = child.dataType match {
     case _: DecimalType =>
       DecimalPrecision.decimalAndDecimal(sum / count.cast(DecimalType.LongDecimal)).cast(resultType)
-    case CalendarIntervalType =>
-      val newCount = If(EqualTo(count, Literal(0L)), Literal(null, LongType), count)
-      DivideInterval(sum.cast(resultType), newCount.cast(DoubleType))
     case _ =>
       sum.cast(resultType) / count.cast(resultType)
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
index 87f1a4f02e4fc..8bfd889ea0563 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
@@ -17,12 +17,13 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.TypeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
-// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.",
   examples = """
@@ -33,11 +34,8 @@ import org.apache.spark.sql.types._
        25
       > SELECT _FUNC_(col) FROM VALUES (NULL), (NULL) AS tab(col);
        NULL
-      > SELECT _FUNC_(cast(col as interval)) FROM VALUES ('1 seconds'), ('2 seconds'), (null) tab(col);
-       3 seconds
   """,
   since = "1.0.0")
-// scalastyle:on line.size.limit
 case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
 
   override def children: Seq[Expression] = child :: Nil
@@ -47,12 +45,14 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
   // Return data type.
   override def dataType: DataType = resultType
 
-  override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection.NumericAndInterval)
+  override def inputTypes: Seq[AbstractDataType] = Seq(NumericType)
+
+  override def checkInputDataTypes(): TypeCheckResult =
+    TypeUtils.checkForNumericExpr(child.dataType, "function sum")
 
   private lazy val resultType = child.dataType match {
     case DecimalType.Fixed(precision, scale) =>
       DecimalType.bounded(precision + 10, scale)
-    case _: CalendarIntervalType => CalendarIntervalType
     case _: IntegralType => LongType
     case _ => DoubleType
   }
@@ -61,7 +61,7 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
 
   private lazy val sum = AttributeReference("sum", sumDataType)()
 
-  private lazy val zero = Literal.default(resultType)
+  private lazy val zero = Literal.default(sumDataType)
 
   override lazy val aggBufferAttributes = sum :: Nil
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 86a1f1fb58a07..46634c93148b1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -158,8 +158,8 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
 
     assertError(Min(Symbol("mapField")), "min does not support ordering on type")
     assertError(Max(Symbol("mapField")), "max does not support ordering on type")
-    assertError(Sum(Symbol("booleanField")), "requires (numeric or interval) type")
-    assertError(Average(Symbol("booleanField")), "requires (numeric or interval) type")
+    assertError(Sum(Symbol("booleanField")), "function sum requires numeric type")
+    assertError(Average(Symbol("booleanField")), "function average requires numeric type")
   }
 
   test("check types for others") {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index a4e621e9639d4..facd6321a1bd4 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -84,70 +84,6 @@ select interval (-30) day;
 select interval (a + 1) day;
 select interval 30 day day day;
 
--- sum interval values
--- null
-select sum(cast(null as interval));
-
--- empty set
-select sum(cast(v as interval)) from VALUES ('1 seconds') t(v) where 1=0;
-
--- basic interval sum
-select sum(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v);
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v);
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v);
-select sum(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v);
-
--- group by
-select
-    i,
-    sum(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i;
-
--- having
-select
-    sum(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null;
-
--- window
-SELECT
-    i,
-    sum(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES(1, '1 seconds'), (1, '2 seconds'), (2, NULL), (2, NULL) t(i,v);
-
--- average with interval type
--- null
-select avg(cast(v as interval)) from VALUES (null) t(v);
-
--- empty set
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v) where 1=0;
-
--- basic interval avg
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v);
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v);
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v);
-select avg(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v);
-
--- group by
-select
-    i,
-    avg(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i;
-
--- having
-select
-    avg(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null;
-
--- window
-SELECT
-    i,
-    avg(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES (1,'1 seconds'), (1,'2 seconds'), (2,NULL), (2,NULL) t(i,v);
-
 -- Interval year-month arithmetic
 
 create temporary view interval_arithmetic as
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 7fdb4c53d1dcb..4b465406ee0ed 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 99
+-- Number of queries: 81
 
 
 -- !query
@@ -631,180 +631,6 @@ select interval 30 day day day
 -----------------------^^^
 
 
--- !query
-select sum(cast(null as interval))
--- !query schema
-struct<sum(CAST(NULL AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds') t(v) where 1=0
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--7 days 2 seconds
-
-
--- !query
-select
-    i,
-    sum(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-2 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    sum(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
--- !query output
--2 days 2 seconds
-
-
--- !query
-SELECT
-    i,
-    sum(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES(1, '1 seconds'), (1, '2 seconds'), (2, NULL), (2, NULL) t(i,v)
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	2 seconds
-1	3 seconds
-2	NULL
-2	NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v) where 1=0
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-0.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
--1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
--3 days -11 hours -59 minutes -59 seconds
-
-
--- !query
-select
-    i,
-    avg(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
--- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-1 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    avg(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
--- !query output
--15 hours -59 minutes -59.333333 seconds
-
-
--- !query
-SELECT
-    i,
-    avg(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES (1,'1 seconds'), (1,'2 seconds'), (2,NULL), (2,NULL) t(i,v)
--- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	1.5 seconds
-1	2 seconds
-2	NULL
-2	NULL
-
-
 -- !query
 create temporary view interval_arithmetic as
   select CAST(dateval AS date), CAST(tsval AS timestamp) from values
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 3c4b4301d0025..0509594ac13d3 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 99
+-- Number of queries: 81
 
 
 -- !query
@@ -615,180 +615,6 @@ select interval 30 day day day
 ---------------------------^^^
 
 
--- !query
-select sum(cast(null as interval))
--- !query schema
-struct<sum(CAST(NULL AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds') t(v) where 1=0
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--3 seconds
-
-
--- !query
-select sum(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<sum(CAST(v AS INTERVAL)):interval>
--- !query output
--7 days 2 seconds
-
-
--- !query
-select
-    i,
-    sum(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-2 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    sum(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
--- !query output
--2 days 2 seconds
-
-
--- !query
-SELECT
-    i,
-    sum(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES(1, '1 seconds'), (1, '2 seconds'), (2, NULL), (2, NULL) t(i,v)
--- !query schema
-struct<i:int,sum(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	2 seconds
-1	3 seconds
-2	NULL
-2	NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v) where 1=0
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-NULL
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
-0.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 seconds'), ('-2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
--1.5 seconds
-
-
--- !query
-select avg(cast(v as interval)) from VALUES ('-1 weeks'), ('2 seconds'), (null) t(v)
--- !query schema
-struct<avg(CAST(v AS INTERVAL)):interval>
--- !query output
--3 days -11 hours -59 minutes -59 seconds
-
-
--- !query
-select
-    i,
-    avg(cast(v as interval))
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-group by i
--- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)):interval>
--- !query output
-1	-1 days
-2	2 seconds
-3	NULL
-
-
--- !query
-select
-    avg(cast(v as interval)) as sv
-from VALUES (1, '-1 weeks'), (2, '2 seconds'), (3, null), (1, '5 days') t(i, v)
-having sv is not null
--- !query schema
-struct<sv:interval>
--- !query output
--15 hours -59 minutes -59.333333 seconds
-
-
--- !query
-SELECT
-    i,
-    avg(cast(v as interval)) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)
-FROM VALUES (1,'1 seconds'), (1,'2 seconds'), (2,NULL), (2,NULL) t(i,v)
--- !query schema
-struct<i:int,avg(CAST(v AS INTERVAL)) OVER (ORDER BY i ASC NULLS FIRST ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING):interval>
--- !query output
-1	1.5 seconds
-1	2 seconds
-2	NULL
-2	NULL
-
-
 -- !query
 create temporary view interval_arithmetic as
   select CAST(dateval AS date), CAST(tsval AS timestamp) from values
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index d7df75fd0e2c3..288f3dac36621 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -957,17 +957,4 @@ class DataFrameAggregateSuite extends QueryTest
       assert(error.message.contains("function count_if requires boolean type"))
     }
   }
-
-  test("calendar interval agg support hash aggregate") {
-    val df1 = Seq((1, "1 day"), (2, "2 day"), (3, "3 day"), (3, null)).toDF("a", "b")
-    val df2 = df1.select(avg($"b" cast CalendarIntervalType))
-    checkAnswer(df2, Row(new CalendarInterval(0, 2, 0)) :: Nil)
-    assert(find(df2.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
-    val df3 = df1.groupBy($"a").agg(avg($"b" cast CalendarIntervalType))
-    checkAnswer(df3,
-      Row(1, new CalendarInterval(0, 1, 0)) ::
-        Row(2, new CalendarInterval(0, 2, 0)) ::
-        Row(3, new CalendarInterval(0, 3, 0)) :: Nil)
-    assert(find(df3.queryExecution.executedPlan)(_.isInstanceOf[HashAggregateExec]).isDefined)
-  }
 }

From 2231a98b1bea8f384507a6856a25bd75591e8741 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 19 Feb 2020 11:17:47 +0900
Subject: [PATCH 0111/1280] [SPARK-30861][PYTHON][SQL] Deprecate constructor of
 SQLContext and getOrCreate in SQLContext at PySpark

### What changes were proposed in this pull request?

This PR proposes to deprecate the APIs at `SQLContext` removed in SPARK-25908. We should remove equivalent APIs; however, seems we missed to deprecate.

While I am here, I fix one more issue. After SPARK-25908, `sc._jvm.SQLContext.getOrCreate` dose not exist anymore. So,

```python
from pyspark.sql import SQLContext
from pyspark import SparkContext
sc = SparkContext.getOrCreate()
SQLContext.getOrCreate(sc).range(10).show()
```

throws an exception as below:

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/context.py", line 110, in getOrCreate
    jsqlContext = sc._jvm.SQLContext.getOrCreate(sc._jsc.sc())
  File "/.../spark/python/lib/py4j-0.10.8.1-src.zip/py4j/java_gateway.py", line 1516, in __getattr__
py4j.protocol.Py4JError: org.apache.spark.sql.SQLContext.getOrCreate does not exist in the JVM
```

After this PR:

```
/.../spark/python/pyspark/sql/context.py:113: DeprecationWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.
  DeprecationWarning)
+---+
| id|
+---+
|  0|
|  1|
|  2|
|  3|
|  4|
|  5|
|  6|
|  7|
|  8|
|  9|
+---+
```

In case of the constructor of `SQLContext`, after this PR:

```python
from pyspark.sql import SQLContext
sc = SparkContext.getOrCreate()
SQLContext(sc)
```

```
/.../spark/python/pyspark/sql/context.py:77: DeprecationWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.
  DeprecationWarning)
```

### Why are the changes needed?

To promote to use SparkSession, and keep the API party consistent with Scala side.

### Does this PR introduce any user-facing change?

Yes, it will show deprecation warning to users.

### How was this patch tested?

Manually tested as described above. Unittests were also added.

Closes #27614 from HyukjinKwon/SPARK-30861.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/context.py            | 15 ++++++++++++++-
 python/pyspark/sql/tests/test_context.py | 18 ++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 68d5ef4ca7b15..f203e1cb55588 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -56,6 +56,8 @@ class SQLContext(object):
     def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
         """Creates a new SQLContext.
 
+        .. note:: Deprecated in 3.0.0. Use :func:`SparkSession.builder.getOrCreate()` instead.
+
         >>> from datetime import datetime
         >>> sqlContext = SQLContext(sc)
         >>> allTypes = sc.parallelize([Row(i=1, s="string", d=1.0, l=1,
@@ -70,6 +72,10 @@ def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
         >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
         [(1, u'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
         """
+        warnings.warn(
+            "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
+            DeprecationWarning)
+
         self._sc = sparkContext
         self._jsc = self._sc._jsc
         self._jvm = self._sc._jvm
@@ -105,9 +111,16 @@ def getOrCreate(cls, sc):
         Get the existing SQLContext or create a new one with given SparkContext.
 
         :param sc: SparkContext
+
+        .. note:: Deprecated in 3.0.0. Use :func:`SparkSession.builder.getOrCreate()` instead.
         """
+        warnings.warn(
+            "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
+            DeprecationWarning)
+
         if cls._instantiatedContext is None:
-            jsqlContext = sc._jvm.SQLContext.getOrCreate(sc._jsc.sc())
+            jsqlContext = sc._jvm.SparkSession.builder().sparkContext(
+                sc._jsc.sc()).getOrCreate().sqlContext()
             sparkSession = SparkSession(sc, jsqlContext.sparkSession())
             cls(sc, sparkSession, jsqlContext)
         return cls._instantiatedContext
diff --git a/python/pyspark/sql/tests/test_context.py b/python/pyspark/sql/tests/test_context.py
index d57ebc48cf5e6..92e54340f8143 100644
--- a/python/pyspark/sql/tests/test_context.py
+++ b/python/pyspark/sql/tests/test_context.py
@@ -27,6 +27,7 @@
 
 import py4j
 
+from pyspark import SparkContext, SQLContext
 from pyspark.sql import Row, SparkSession
 from pyspark.sql.types import *
 from pyspark.sql.window import Window
@@ -259,6 +260,23 @@ def range_frame_match():
         reload(window)
 
 
+class SQLContextTests(unittest.TestCase):
+
+    def test_get_or_create(self):
+        sc = None
+        sql_context = None
+        try:
+            sc = SparkContext('local[4]', "SQLContextTests")
+            sql_context = SQLContext.getOrCreate(sc)
+            assert(isinstance(sql_context, SQLContext))
+        finally:
+            SQLContext._instantiatedContext = None
+            if sql_context is not None:
+                sql_context.sparkSession.stop()
+            if sc is not None:
+                sc.stop()
+
+
 if __name__ == "__main__":
     from pyspark.sql.tests.test_context import *
 

From 52c92ad31ea7460c2b0957e7ec54e41c6c6cf7d1 Mon Sep 17 00:00:00 2001
From: Liupengcheng <liupengcheng@xiaomi.com>
Date: Mon, 17 Feb 2020 20:16:31 +0800
Subject: [PATCH 0112/1280] [SPARK-30346][CORE] Improve logging when events
 dropped

### What changes were proposed in this pull request?

Make logging events dropping every 60s works fine, the orignal implementaion some times not working due to susequent events comming and updating the DroppedEventCounter

### Why are the changes needed?

Currenly, the logging may be skipped and delayed a long time under high concurrency, that make debugging hard. So This PR will try to fix it.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

NA

Closes #27002 from liupc/Improve-logging-dropped-events-and-logging-threadDump.

Authored-by: Liupengcheng <liupengcheng@xiaomi.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/scheduler/AsyncEventQueue.scala     | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
index 1bcddaceb3576..5164c30fce0a1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/AsyncEventQueue.scala
@@ -64,11 +64,14 @@ private class AsyncEventQueue(
   // processed (instead of just dequeued).
   private val eventCount = new AtomicLong()
 
-  /** A counter for dropped events. It will be reset every time we log it. */
+  /** A counter for dropped events. */
   private val droppedEventsCounter = new AtomicLong(0L)
 
+  /** A counter to keep number of dropped events last time it was logged */
+  @volatile private var lastDroppedEventsCounter: Long = 0L
+
   /** When `droppedEventsCounter` was logged last time in milliseconds. */
-  @volatile private var lastReportTimestamp = 0L
+  private val lastReportTimestamp = new AtomicLong(0L)
 
   private val logDroppedEvent = new AtomicBoolean(false)
 
@@ -167,21 +170,19 @@ private class AsyncEventQueue(
     }
     logTrace(s"Dropping event $event")
 
-    val droppedCount = droppedEventsCounter.get
-    if (droppedCount > 0) {
-      // Don't log too frequently
-      if (System.currentTimeMillis() - lastReportTimestamp >= 60 * 1000) {
-        // There may be multiple threads trying to decrease droppedEventsCounter.
-        // Use "compareAndSet" to make sure only one thread can win.
-        // And if another thread is increasing droppedEventsCounter, "compareAndSet" will fail and
-        // then that thread will update it.
-        if (droppedEventsCounter.compareAndSet(droppedCount, 0)) {
-          val prevLastReportTimestamp = lastReportTimestamp
-          lastReportTimestamp = System.currentTimeMillis()
-          val previous = new java.util.Date(prevLastReportTimestamp)
-          logWarning(s"Dropped $droppedCount events from $name since " +
-            s"${if (prevLastReportTimestamp == 0) "the application started" else s"$previous"}.")
-        }
+    val droppedEventsCount = droppedEventsCounter.get
+    val droppedCountIncreased = droppedEventsCount - lastDroppedEventsCounter
+    val lastReportTime = lastReportTimestamp.get
+    val curTime = System.currentTimeMillis()
+    // Don't log too frequently
+    if (droppedCountIncreased > 0 && curTime - lastReportTime >= LOGGING_INTERVAL) {
+      // There may be multiple threads trying to logging dropped events,
+      // Use 'compareAndSet' to make sure only one thread can win.
+      if (lastReportTimestamp.compareAndSet(lastReportTime, curTime)) {
+        val previous = new java.util.Date(lastReportTime)
+        lastDroppedEventsCounter = droppedEventsCount
+        logWarning(s"Dropped $droppedCountIncreased events from $name since " +
+          s"${if (lastReportTime == 0) "the application started" else s"$previous"}.")
       }
     }
   }
@@ -213,4 +214,5 @@ private object AsyncEventQueue {
 
   val POISON_PILL = new SparkListenerEvent() { }
 
+  val LOGGING_INTERVAL = 60 * 1000
 }

From ac82bb4f8ff189e3f0f26e6dc7297551a7e39c53 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Wed, 19 Feb 2020 13:41:00 +0800
Subject: [PATCH 0113/1280] [SPARK-30814][SQL][3.0] ALTER TABLE ... ADD COLUMN
 position should be able to reference columns being added

(Backport of #27584 + partial #27482)

### What changes were proposed in this pull request?

In ALTER TABLE, a column in ADD COLUMNS can depend on the position of a column that is just being added. For example, for a table with the following schema:
```
root:
  - a: string
  - b: long
```
, the following should work:
```
ALTER TABLE t ADD COLUMNS (x int AFTER a, y int AFTER x)
```
Currently, the above statement will throw an exception saying that AFTER x cannot be resolved, because x doesn't exist yet. This PR proposes to fix this issue.

### Why are the changes needed?

To fix a bug described above.

### Does this PR introduce any user-facing change?

Yes, now
```
ALTER TABLE t ADD COLUMNS (x int AFTER a, y int AFTER x)
```
works as expected.

### How was this patch tested?

Added new tests

Closes #27624 from imback82/backport_27584.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 46 +++++++++++--------
 .../sql/catalyst/analysis/CheckAnalysis.scala | 38 ++++++++++++---
 .../spark/sql/connector/AlterTableTests.scala | 36 +++++++++++++++
 .../V2CommandsCaseSensitivitySuite.scala      | 28 ++++++++++-
 4 files changed, 122 insertions(+), 26 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index ce82b3b567b54..aec71747fde68 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -3023,9 +3023,29 @@ class Analyzer(
   object ResolveAlterTableChanges extends Rule[LogicalPlan] {
     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       case a @ AlterTable(_, _, t: NamedRelation, changes) if t.resolved =>
+        // 'colsToAdd' keeps track of new columns being added. It stores a mapping from a
+        // normalized parent name of fields to field names that belong to the parent.
+        // For example, if we add columns "a.b.c", "a.b.d", and "a.c", 'colsToAdd' will become
+        // Map(Seq("a", "b") -> Seq("c", "d"), Seq("a") -> Seq("c")).
+        val colsToAdd = mutable.Map.empty[Seq[String], Seq[String]]
         val schema = t.schema
         val normalizedChanges = changes.flatMap {
           case add: AddColumn =>
+            def addColumn(
+                parentSchema: StructType,
+                parentName: String,
+                normalizedParentName: Seq[String]): TableChange = {
+              val fieldsAdded = colsToAdd.getOrElse(normalizedParentName, Nil)
+              val pos = findColumnPosition(add.position(), parentName, parentSchema, fieldsAdded)
+              val field = add.fieldNames().last
+              colsToAdd(normalizedParentName) = fieldsAdded :+ field
+              TableChange.addColumn(
+                (normalizedParentName :+ field).toArray,
+                add.dataType(),
+                add.isNullable,
+                add.comment,
+                pos)
+            }
             val parent = add.fieldNames().init
             if (parent.nonEmpty) {
               // Adding a nested field, need to normalize the parent column and position
@@ -3037,27 +3057,14 @@ class Analyzer(
                 val (normalizedName, sf) = target.get
                 sf.dataType match {
                   case struct: StructType =>
-                    val pos = findColumnPosition(add.position(), parent.quoted, struct)
-                    Some(TableChange.addColumn(
-                      (normalizedName ++ Seq(sf.name, add.fieldNames().last)).toArray,
-                      add.dataType(),
-                      add.isNullable,
-                      add.comment,
-                      pos))
-
+                    Some(addColumn(struct, parent.quoted, normalizedName :+ sf.name))
                   case other =>
                     Some(add)
                 }
               }
             } else {
               // Adding to the root. Just need to normalize position
-              val pos = findColumnPosition(add.position(), "root", schema)
-              Some(TableChange.addColumn(
-                add.fieldNames(),
-                add.dataType(),
-                add.isNullable,
-                add.comment,
-                pos))
+              Some(addColumn(schema, "root", Nil))
             }
 
           case typeChange: UpdateColumnType =>
@@ -3156,17 +3163,18 @@ class Analyzer(
 
     private def findColumnPosition(
         position: ColumnPosition,
-        field: String,
-        struct: StructType): ColumnPosition = {
+        parentName: String,
+        struct: StructType,
+        fieldsAdded: Seq[String]): ColumnPosition = {
       position match {
         case null => null
         case after: After =>
-          struct.fieldNames.find(n => conf.resolver(n, after.column())) match {
+          (struct.fieldNames ++ fieldsAdded).find(n => conf.resolver(n, after.column())) match {
             case Some(colName) =>
               ColumnPosition.after(colName)
             case None =>
               throw new AnalysisException("Couldn't find the reference column for " +
-                s"$after at $field")
+                s"$after at $parentName")
           }
         case other => other
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index e769e038c960f..066dc6db0227d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -440,12 +440,16 @@ trait CheckAnalysis extends PredicateHelper {
               }
               field.get._2
             }
-            def positionArgumentExists(position: ColumnPosition, struct: StructType): Unit = {
+            def positionArgumentExists(
+                position: ColumnPosition,
+                struct: StructType,
+                fieldsAdded: Seq[String]): Unit = {
               position match {
                 case after: After =>
-                  if (!struct.fieldNames.contains(after.column())) {
+                  val allFields = struct.fieldNames ++ fieldsAdded
+                  if (!allFields.contains(after.column())) {
                     alter.failAnalysis(s"Couldn't resolve positional argument $position amongst " +
-                      s"${struct.fieldNames.mkString("[", ", ", "]")}")
+                      s"${allFields.mkString("[", ", ", "]")}")
                   }
                 case _ =>
               }
@@ -473,12 +477,26 @@ trait CheckAnalysis extends PredicateHelper {
               }
             }
 
+            val colsToDelete = mutable.Set.empty[Seq[String]]
+            // 'colsToAdd' keeps track of new columns being added. It stores a mapping from a parent
+            // name of fields to field names that belong to the parent. For example, if we add
+            // columns "a.b.c", "a.b.d", and "a.c", 'colsToAdd' will become
+            // Map(Seq("a", "b") -> Seq("c", "d"), Seq("a") -> Seq("c")).
+            val colsToAdd = mutable.Map.empty[Seq[String], Seq[String]]
+
             alter.changes.foreach {
               case add: AddColumn =>
-                checkColumnNotExists("add", add.fieldNames(), table.schema)
+                // If a column to add is a part of columns to delete, we don't need to check
+                // if column already exists - applies to REPLACE COLUMNS scenario.
+                if (!colsToDelete.contains(add.fieldNames())) {
+                  checkColumnNotExists("add", add.fieldNames(), table.schema)
+                }
                 val parent = findParentStruct("add", add.fieldNames())
-                positionArgumentExists(add.position(), parent)
+                val parentName = add.fieldNames().init
+                val fieldsAdded = colsToAdd.getOrElse(parentName, Nil)
+                positionArgumentExists(add.position(), parent, fieldsAdded)
                 TypeUtils.failWithIntervalType(add.dataType())
+                colsToAdd(parentName) = fieldsAdded :+ add.fieldNames().last
               case update: UpdateColumnType =>
                 val field = findField("update", update.fieldNames)
                 val fieldName = update.fieldNames.quoted
@@ -517,7 +535,11 @@ trait CheckAnalysis extends PredicateHelper {
               case updatePos: UpdateColumnPosition =>
                 findField("update", updatePos.fieldNames)
                 val parent = findParentStruct("update", updatePos.fieldNames())
-                positionArgumentExists(updatePos.position(), parent)
+                val parentName = updatePos.fieldNames().init
+                positionArgumentExists(
+                  updatePos.position(),
+                  parent,
+                  colsToAdd.getOrElse(parentName, Nil))
               case rename: RenameColumn =>
                 findField("rename", rename.fieldNames)
                 checkColumnNotExists(
@@ -526,6 +548,10 @@ trait CheckAnalysis extends PredicateHelper {
                 findField("update", update.fieldNames)
               case delete: DeleteColumn =>
                 findField("delete", delete.fieldNames)
+                // REPLACE COLUMNS has deletes followed by adds. Remember the deleted columns
+                // so that add operations do not fail when the columns to add exist and they
+                // are to be deleted.
+                colsToDelete += delete.fieldNames
               case _ =>
               // no validation needed for set and remove property
             }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
index 420cb01d766a0..64188f71fb202 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
@@ -173,6 +173,42 @@ trait AlterTableTests extends SharedSparkSession {
     }
   }
 
+  test("SPARK-30814: add column with position referencing new columns being added") {
+    val t = s"${catalogAndNamespace}table_name"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (a string, b int, point struct<x: double, y: double>) USING $v2Format")
+      sql(s"ALTER TABLE $t ADD COLUMNS (x int AFTER a, y int AFTER x, z int AFTER y)")
+
+      assert(getTableMetadata(t).schema === new StructType()
+        .add("a", StringType)
+        .add("x", IntegerType)
+        .add("y", IntegerType)
+        .add("z", IntegerType)
+        .add("b", IntegerType)
+        .add("point", new StructType()
+          .add("x", DoubleType)
+          .add("y", DoubleType)))
+
+      sql(s"ALTER TABLE $t ADD COLUMNS (point.z double AFTER x, point.zz double AFTER z)")
+      assert(getTableMetadata(t).schema === new StructType()
+        .add("a", StringType)
+        .add("x", IntegerType)
+        .add("y", IntegerType)
+        .add("z", IntegerType)
+        .add("b", IntegerType)
+        .add("point", new StructType()
+          .add("x", DoubleType)
+          .add("z", DoubleType)
+          .add("zz", DoubleType)
+          .add("y", DoubleType)))
+
+      // The new column being referenced should come before being referenced.
+      val e = intercept[AnalysisException](
+        sql(s"ALTER TABLE $t ADD COLUMNS (yy int AFTER xx, xx int)"))
+      assert(e.getMessage().contains("Couldn't find the reference column for AFTER xx at root"))
+    }
+  }
+
   test("AlterTable: add multiple columns") {
     val t = s"${catalogAndNamespace}table_name"
     withTable(t) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
index 289f9dc427795..dd95ceb59bdc4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V2CommandsCaseSensitivitySuite.scala
@@ -151,6 +151,17 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
     }
   }
 
+  test("AlterTable: add column resolution - column position referencing new column") {
+    alterTableTest(
+      Seq(
+        TableChange.addColumn(
+          Array("x"), LongType, true, null, ColumnPosition.after("id")),
+        TableChange.addColumn(
+          Array("y"), LongType, true, null, ColumnPosition.after("X"))),
+      Seq("Couldn't find the reference column for AFTER X at root")
+    )
+  }
+
   test("AlterTable: add column resolution - nested positional") {
     Seq("X", "Y").foreach { ref =>
       alterTableTest(
@@ -161,6 +172,17 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
     }
   }
 
+  test("AlterTable: add column resolution - column position referencing new nested column") {
+    alterTableTest(
+      Seq(
+        TableChange.addColumn(
+          Array("point", "z"), LongType, true, null),
+        TableChange.addColumn(
+          Array("point", "zz"), LongType, true, null, ColumnPosition.after("Z"))),
+      Seq("Couldn't find the reference column for AFTER Z at point")
+    )
+  }
+
   test("AlterTable: drop column resolution") {
     Seq(Array("ID"), Array("point", "X"), Array("POINT", "X"), Array("POINT", "x")).foreach { ref =>
       alterTableTest(
@@ -207,13 +229,17 @@ class V2CommandsCaseSensitivitySuite extends SharedSparkSession with AnalysisTes
   }
 
   private def alterTableTest(change: TableChange, error: Seq[String]): Unit = {
+    alterTableTest(Seq(change), error)
+  }
+
+  private def alterTableTest(changes: Seq[TableChange], error: Seq[String]): Unit = {
     Seq(true, false).foreach { caseSensitive =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
         val plan = AlterTable(
           catalog,
           Identifier.of(Array(), "table_name"),
           TestRelation2,
-          Seq(change)
+          changes
         )
 
         if (caseSensitive) {

From cd69538cc7e6e2938d7a096df5e2821f986f6b3b Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Wed, 19 Feb 2020 14:26:27 +0800
Subject: [PATCH 0114/1280] [MINOR][SQL] Improve readability for window
 execution

### What changes were proposed in this pull request?
I read the comments of `WindowExec` and found some comment will cause confusion and another need to improve.

### Why are the changes needed?
This PR will enhance the readability and let developer works more easy

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
No need

Closes #27431 from beliefer/improve-window-readability.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 0894dbab2cd112b0f8f873cdb4dc510d92eac4fb)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/window/WindowExec.scala      | 18 ++++++++++++++----
 .../sql/execution/window/WindowExecBase.scala  |  6 +++---
 .../execution/window/WindowFunctionFrame.scala |  4 ++--
 3 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
index d191f3790ffa8..42fa07f4a6aee 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExec.scala
@@ -38,15 +38,25 @@ import org.apache.spark.sql.types.{CalendarIntervalType, DateType, IntegerType,
  * - Entire partition: The frame is the entire partition, i.e.
  *   UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING. For this case, window function will take all
  *   rows as inputs and be evaluated once.
- * - Growing frame: We only add new rows into the frame, i.e. UNBOUNDED PRECEDING AND ....
+ * - Growing frame: We only add new rows into the frame, Examples are:
+ *     1. UNBOUNDED PRECEDING AND 1 PRECEDING
+ *     2. UNBOUNDED PRECEDING AND CURRENT ROW
+ *     3. UNBOUNDED PRECEDING AND 1 FOLLOWING
  *   Every time we move to a new row to process, we add some rows to the frame. We do not remove
  *   rows from this frame.
- * - Shrinking frame: We only remove rows from the frame, i.e. ... AND UNBOUNDED FOLLOWING.
+ * - Shrinking frame: We only remove rows from the frame, Examples are:
+ *     1. 1 PRECEDING AND UNBOUNDED FOLLOWING
+ *     2. CURRENT ROW AND UNBOUNDED FOLLOWING
+ *     3. 1 FOLLOWING AND UNBOUNDED FOLLOWING
  *   Every time we move to a new row to process, we remove some rows from the frame. We do not add
  *   rows to this frame.
  * - Moving frame: Every time we move to a new row to process, we remove some rows from the frame
  *   and we add some rows to the frame. Examples are:
- *     1 PRECEDING AND CURRENT ROW and 1 FOLLOWING AND 2 FOLLOWING.
+ *     1. 2 PRECEDING AND 1 PRECEDING
+ *     2. 1 PRECEDING AND CURRENT ROW
+ *     3. CURRENT ROW AND 1 FOLLOWING
+ *     4. 1 PRECEDING AND 1 FOLLOWING
+ *     5. 1 FOLLOWING AND 2 FOLLOWING
  * - Offset frame: The frame consist of one row, which is an offset number of rows away from the
  *   current row. Only [[OffsetWindowFunction]]s can be processed in an offset frame.
  *
@@ -105,7 +115,7 @@ case class WindowExec(
   override def outputPartitioning: Partitioning = child.outputPartitioning
 
   protected override def doExecute(): RDD[InternalRow] = {
-    // Unwrap the expressions and factories from the map.
+    // Unwrap the window expressions and window frame factories from the map.
     val expressions = windowFrameExpressionFactoryPairs.flatMap(_._1)
     val factories = windowFrameExpressionFactoryPairs.map(_._2).toArray
     val inMemoryThreshold = sqlContext.conf.windowExecBufferInMemoryThreshold
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
index d5d11c45f8535..99c8917478865 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowExecBase.scala
@@ -114,7 +114,7 @@ abstract class WindowExecBase(
 
   /**
    * Collection containing an entry for each window frame to process. Each entry contains a frame's
-   * [[WindowExpression]]s and factory function for the WindowFrameFunction.
+   * [[WindowExpression]]s and factory function for the [[WindowFrameFunction]].
    */
   protected lazy val windowFrameExpressionFactoryPairs = {
     type FrameKey = (String, FrameType, Expression, Expression)
@@ -170,7 +170,7 @@ abstract class WindowExecBase(
               MutableProjection.create(expressions, schema))
         }
 
-        // Create the factory
+        // Create the factory to produce WindowFunctionFrame.
         val factory = key match {
           // Offset Frame
           case ("OFFSET", _, IntegerLiteral(offset), _) =>
@@ -223,7 +223,7 @@ abstract class WindowExecBase(
         // Keep track of the number of expressions. This is a side-effect in a map...
         numExpressions += expressions.size
 
-        // Create the Frame Expression - Factory pair.
+        // Create the Window Expression - Frame Factory pair.
         (expressions, factory)
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
index 181ff42421c65..dc1b919feefe4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/window/WindowFunctionFrame.scala
@@ -47,14 +47,14 @@ abstract class WindowFunctionFrame {
   /**
    * The current lower window bound in the row array (inclusive).
    *
-   * This should be called after the current row is updated via [[write]]
+   * This should be called after the current row is updated via `write`.
    */
   def currentLowerBound(): Int
 
   /**
    * The current row index of the upper window bound in the row array (exclusive)
    *
-   * This should be called after the current row is updated via [[write]]
+   * This should be called after the current row is updated via `write`.
    */
   def currentUpperBound(): Int
 }

From bf4b71a511d18377b77f3146c33ad00806bcf7bf Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Wed, 19 Feb 2020 15:05:34 +0800
Subject: [PATCH 0115/1280] [SPARK-30785][SQL] Create table like should keep
 tracksPartitionsInCatalog same with source table

### What changes were proposed in this pull request?
Table generated by `CREATE TABLE LIKE` a partitioned table is a partitioned table. But when run `ALTER TABLE ADD PARTITION`, it will throw `AnalysisException: ALTER TABLE ADD PARTITION is not allowed`. That's because the default value of `tracksPartitionsInCatalog` from `CREATE TABLE LIKE` always is false.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Add a unit test.

Closes #27538 from LantaoJin/SPARK-30785.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c0715221b23d5461b5498a9b832921291359b4b7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/command/tables.scala  |  3 ++-
 .../sql/hive/execution/HiveDDLSuite.scala     | 21 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 61500b773cd7e..6243261d3cd16 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -124,7 +124,8 @@ case class CreateTableLikeCommand(
         provider = newProvider,
         partitionColumnNames = sourceTableDesc.partitionColumnNames,
         bucketSpec = sourceTableDesc.bucketSpec,
-        properties = properties)
+        properties = properties,
+        tracksPartitionsInCatalog = sourceTableDesc.tracksPartitionsInCatalog)
 
     catalog.createTable(newTableDesc, ifNotExists)
     Seq.empty[Row]
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index ba48cfd4142f6..7adc68d548af2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -1580,6 +1580,12 @@ class HiveDDLSuite
         "source table/view path should be different from target table path")
     }
 
+    if (DDLUtils.isHiveTable(targetTable)) {
+      assert(targetTable.tracksPartitionsInCatalog)
+    } else {
+      assert(targetTable.tracksPartitionsInCatalog == sourceTable.tracksPartitionsInCatalog)
+    }
+
     // The source table contents should not been seen in the target table.
     assert(spark.table(sourceTable.identifier).count() != 0, "the source table should be nonempty")
     assert(spark.table(targetTable.identifier).count() == 0, "the target table should be empty")
@@ -2724,4 +2730,19 @@ class HiveDDLSuite
       }
     }
   }
+
+  test("SPARK-30785: create table like a partitioned table") {
+    val catalog = spark.sessionState.catalog
+    withTable("sc_part", "ta_part") {
+      sql("CREATE TABLE sc_part (key string, ts int) USING parquet PARTITIONED BY (ts)")
+      sql("CREATE TABLE ta_part like sc_part")
+      val sourceTable = catalog.getTableMetadata(TableIdentifier("sc_part", Some("default")))
+      val targetTable = catalog.getTableMetadata(TableIdentifier("ta_part", Some("default")))
+      assert(sourceTable.tracksPartitionsInCatalog)
+      assert(targetTable.tracksPartitionsInCatalog)
+      assert(targetTable.partitionColumnNames == Seq("ts"))
+      sql("ALTER TABLE ta_part ADD PARTITION (ts=10)") // no exception
+      checkAnswer(sql("SHOW PARTITIONS ta_part"), Row("ts=10") :: Nil)
+    }
+  }
 }

From 5b36cdbbfef147e93b35eaa4f8e0bea9690b6d06 Mon Sep 17 00:00:00 2001
From: jiake <ke.a.jia@intel.com>
Date: Wed, 19 Feb 2020 15:55:29 +0800
Subject: [PATCH 0116/1280] [SPARK-30812][SQL] update the skew join configs by
 adding the prefix "skewedJoinOptimization"

### What changes were proposed in this pull request?
This is a follow up in [PR#27563](https://github.com/apache/spark/pull/27563).
This PR adds the prefix of "skewedJoinOptimization" in the skew join related configs.

### Why are the changes needed?
address remaining address

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
only update config and no need new ut.

Closes #27630 from JkSelf/renameskewjoinconfig.

Authored-by: jiake <ke.a.jia@intel.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 10a4eafcfe0e0dcf4959894ac2e8f936693f5d7c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 0783c49378aab..612fa86599900 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -432,14 +432,14 @@ object SQLConf {
     .createWithDefault(true)
 
   val ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionSizeThreshold")
+    buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionSizeThreshold")
       .doc("Configures the minimum size in bytes for a partition that is considered as a skewed " +
         "partition in adaptive skewed join.")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("64MB")
 
   val ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionFactor")
+    buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionFactor")
       .doc("A partition is considered as a skewed partition if its size is larger than" +
         " this factor multiple the median partition size and also larger than " +
         s" ${ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key}")
@@ -447,7 +447,7 @@ object SQLConf {
       .createWithDefault(10)
 
   val ADAPTIVE_EXECUTION_SKEWED_PARTITION_MAX_SPLITS =
-    buildConf("spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionMaxSplits")
+    buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionMaxSplits")
       .doc("Configures the maximum number of task to handle a skewed partition in adaptive skewed" +
         "join.")
       .intConf

From 9b2c5ec721c17aa798d114d7ca5a5a16784ab45d Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Wed, 19 Feb 2020 17:28:58 +0900
Subject: [PATCH 0117/1280] [SPARK-30731] Update deprecated Mkdocs option

Split from #27534.

### What changes were proposed in this pull request?

This PR updates a deprecated Mkdocs option to use the new name.

### Why are the changes needed?

This change will prevent the docs from failing to build when we update to a version of Mkdocs that no longer supports the deprecated option.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

I built the docs locally and reviewed them in my browser.

Closes #27626 from nchammas/SPARK-30731-mkdocs-dep-opt.

Authored-by: Nicholas Chammas <nicholas.chammas@liveramp.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 2ab8d674ba98db4068c4cdefdc4c424dd39c071c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 sql/mkdocs.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/mkdocs.yml b/sql/mkdocs.yml
index 8aad985e30459..4463e11f17d1f 100644
--- a/sql/mkdocs.yml
+++ b/sql/mkdocs.yml
@@ -15,7 +15,7 @@
 
 site_name: Spark SQL, Built-in Functions
 theme: readthedocs
-pages:
+nav:
   - 'Functions': 'index.md'
 markdown_extensions:
   - toc:

From 23b4554dadc7807bc6da9ef4a89fb7cf29e155f1 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 19 Feb 2020 22:26:56 +0800
Subject: [PATCH 0118/1280] [SPARK-27528][FOLLOWUP] improve migration guide

### What changes were proposed in this pull request?

mention that `INT96` timestamp is still useful for interoperability.

### Why are the changes needed?

Give users more context of the behavior changes.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #27622 from cloud-fan/parquet.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c7bece354132eef3677004bd796f82ef72f85bd1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 76df66b9de50c..0690127a45b17 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -87,7 +87,7 @@ license: |
 
   - In Spark version 2.4, when a spark session is created via `cloneSession()`, the newly created spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent spark session. Since Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. The old behavior can be restored by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
 
-  - Since Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and earlier, `TIMESTAMP` columns are saved as `INT96` in parquet files. To set `INT96` to `spark.sql.parquet.outputTimestampType` restores the previous behavior.
+  - Since Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and earlier, `TIMESTAMP` columns are saved as `INT96` in parquet files. Note that, some SQL systems such as Hive 1.x and Impala 2.x can only read `INT96` timestamps, you can set `spark.sql.parquet.outputTimestampType` as `INT96` to restore the previous behavior and keep interoperability.
 
   - Since Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it will fallback to hive-site.xml present in the `Hadoop configuration` of `SparkContext`.
 

From a951d1376bec3411eaade1ea55141ddc1793c7f1 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 19 Feb 2020 10:47:59 -0600
Subject: [PATCH 0119/1280] [MINOR][DOCS] Fix fabric8 version in documentation

### What changes were proposed in this pull request?

fix kubernetes-client version doc

### Why are the changes needed?

correct doc

### Does this PR introduce any user-facing change?

nah
### How was this patch tested?

nah

Closes #27605 from yaooqinn/k8s-version-update.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 46019b6e6ce334db4649e269e0ad6e7e4d4ab6c2)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/running-on-kubernetes.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 53b883b6bbf30..a941b8c04cb54 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -77,8 +77,7 @@ logs and remains in "completed" state in the Kubernetes API until it's eventuall
 
 Note that in the completed state, the driver pod does *not* use any computational or memory resources.
 
-The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8, and we are
-currently running <code>kubernetes-client</code> version <code>4.1.0</code>. Make sure that when you are making infrastructure additions that you are aware of said version. It is possible to schedule the
+The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8. It is possible to schedule the
 driver and executor pods on a subset of available nodes through a [node selector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector)
 using the configuration property for it. It will be possible to use more advanced
 scheduling hints like [node/pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) in a future release.

From 3e884ec40a7b6e4f184d7c507c29e9f8eae0a721 Mon Sep 17 00:00:00 2001
From: herman <herman@databricks.com>
Date: Wed, 19 Feb 2020 10:17:46 -0800
Subject: [PATCH 0120/1280] [SPARK-30811][SQL] CTE should not cause stack
 overflow when it refers to non-existent table with same name

### Why are the changes needed?
This ports the tests introduced in https://github.com/apache/spark/commit/7285eea6839d40cbac15101c633a9a572eb3b603 to master to avoid future regressions.

### Background
A query with Common Table Expressions can cause a stack overflow when it contains a CTE that refers a non-existing table with the same name. The name of the table need to have a database qualifier. This is caused by a couple of things:

- CTESubstitution runs analysis on the CTE, but this does not throw an exception because the table has a database qualifier. The reason is that we don't fail is because we re-attempt to resolve the relation in a later rule;
- CTESubstitution replace logic does not check if the table it is replacing has a database, it shouldn't replace the relation if it does. So now we will happily replace nonexist.t with t;

Note that this not an issue for master or the spark-3.0 branch.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added regression test to `AnalysisErrorSuite` and `DataFrameSuite`.

Closes #27635 from hvanhovell/SPARK-30811-master.

Authored-by: herman <herman@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit c92d437c4639c3e5ae3bfc38bc70e7c38fe4cfad)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/analysis/AnalysisErrorSuite.scala   | 12 ++++++++++++
 .../scala/org/apache/spark/sql/DataFrameSuite.scala  |  8 ++++++++
 2 files changed, 20 insertions(+)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 5cc0453135c07..8f62b0b9b77a2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.analysis
 import org.scalatest.Assertions._
 
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
@@ -652,4 +653,15 @@ class AnalysisErrorSuite extends AnalysisTest {
     assertAnalysisError(plan,
       "Aggregate/Window/Generate expressions are not valid in where clause of the query" :: Nil)
   }
+
+  test("SPARK-30811: CTE should not cause stack overflow when " +
+    "it refers to non-existent table with same name") {
+    val plan = With(
+      UnresolvedRelation(TableIdentifier("t")),
+      Seq("t" -> SubqueryAlias("t",
+        Project(
+          Alias(Literal(1), "x")() :: Nil,
+          UnresolvedRelation(TableIdentifier("t", Option("nonexist")))))))
+    assertAnalysisError(plan, "Table or view not found:" :: Nil)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 694e576fcded4..42a90735368f2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2298,6 +2298,14 @@ class DataFrameSuite extends QueryTest
         fail("emptyDataFrame should be foldable")
     }
   }
+
+  test("SPARK-30811: CTE should not cause stack overflow when " +
+    "it refers to non-existent table with same name") {
+    val e = intercept[AnalysisException] {
+      sql("WITH t AS (SELECT 1 FROM nonexist.t) SELECT * FROM t")
+    }
+    assert(e.getMessage.contains("Table or view not found:"))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)

From f482187c127418d2ea538ac2551ae0fce1ddbc31 Mon Sep 17 00:00:00 2001
From: sarthfrey-db <sarth.frey@databricks.com>
Date: Thu, 13 Feb 2020 16:15:00 -0800
Subject: [PATCH 0121/1280] [SPARK-30667][CORE] Add allGather method to
 BarrierTaskContext

### What changes were proposed in this pull request?

The `allGather` method is added to the `BarrierTaskContext`. This method contains the same functionality as the `BarrierTaskContext.barrier` method; it blocks the task until all tasks make the call, at which time they may continue execution. In addition, the `allGather` method takes an input message. Upon returning from the `allGather` the task receives a list of all the messages sent by all the tasks that made the `allGather` call.

### Why are the changes needed?

There are many situations where having the tasks communicate in a synchronized way is useful. One simple example is if each task needs to start a server to serve requests from one another; first the tasks must find a free port (the result of which is undetermined beforehand) and then start making requests, but to do so they each must know the port chosen by the other task. An `allGather` method would allow them to inform each other of the port they will run on.

### Does this PR introduce any user-facing change?

Yes, an `BarrierTaskContext.allGather` method will be available through the Scala, Java, and Python APIs.

### How was this patch tested?

Most of the code path is already covered by tests to the `barrier` method, since this PR includes a refactor so that much code is shared by the `barrier` and `allGather` methods. However, a test is added to assert that an all gather on each tasks partition ID will return a list of every partition ID.

An example through the Python API:
```python
>>> from pyspark import BarrierTaskContext
>>>
>>> def f(iterator):
...     context = BarrierTaskContext.get()
...     return [context.allGather('{}'.format(context.partitionId()))]
...
>>> sc.parallelize(range(4), 4).barrier().mapPartitions(f).collect()[0]
[u'3', u'1', u'0', u'2']
```

Closes #27395 from sarthfrey/master.

Lead-authored-by: sarthfrey-db <sarth.frey@databricks.com>
Co-authored-by: sarthfrey <sarth.frey@gmail.com>
Signed-off-by: Xiangrui Meng <meng@databricks.com>
(cherry picked from commit 57254c9719f9af9ad985596ed7fbbaafa4052002)
Signed-off-by: Xiangrui Meng <meng@databricks.com>
---
 .../org/apache/spark/BarrierCoordinator.scala | 113 +++++++++++--
 .../org/apache/spark/BarrierTaskContext.scala | 153 ++++++++++++------
 .../spark/api/python/PythonRunner.scala       |  51 ++++--
 .../scheduler/BarrierTaskContextSuite.scala   |  74 +++++++++
 python/pyspark/taskcontext.py                 |  49 +++++-
 python/pyspark/tests/test_taskcontext.py      |  20 +++
 6 files changed, 381 insertions(+), 79 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 4e417679ca663..042a2664a0e27 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -17,12 +17,17 @@
 
 package org.apache.spark
 
+import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
 import scala.collection.mutable.ArrayBuffer
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
@@ -99,10 +104,15 @@ private[spark] class BarrierCoordinator(
     // reset when a barrier() call fails due to timeout.
     private var barrierEpoch: Int = 0
 
-    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()
-    // call.
+    // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
+    // An Array of allGather messages for barrier tasks that have made a blocking runBarrier() call
+    private val allGatherMessages: ArrayBuffer[String] = new Array[String](numTasks).to[ArrayBuffer]
+
+    // The blocking requestMethod called by tasks to sync up for this stage attempt
+    private var requestMethodToSync: RequestMethod.Value = RequestMethod.BARRIER
+
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
 
@@ -130,9 +140,32 @@ private[spark] class BarrierCoordinator(
 
     // Process the global sync request. The barrier() call succeed if collected enough requests
     // within a configured time, otherwise fail all the pending requests.
-    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
+    def handleRequest(
+      requester: RpcCallContext,
+      request: RequestToSync
+    ): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
+      val requestMethod = request.requestMethod
+      val partitionId = request.partitionId
+      val allGatherMessage = request match {
+        case ag: AllGatherRequestToSync => ag.allGatherMessage
+        case _ => ""
+      }
+
+      if (requesters.size == 0) {
+        requestMethodToSync = requestMethod
+      }
+
+      if (requestMethodToSync != requestMethod) {
+        requesters.foreach(
+          _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
+            s"`$requestMethod` during barrier epoch $barrierEpoch, which does not match " +
+            s"the current synchronized requestMethod `$requestMethodToSync`"
+          ))
+        )
+        cleanupBarrierStage(barrierId)
+      }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
       require(request.numTasks == numTasks, s"Number of tasks of $barrierId is " +
@@ -153,6 +186,7 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
+        allGatherMessages(partitionId) = allGatherMessage
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
         if (maybeFinishAllRequesters(requesters, numTasks)) {
@@ -162,6 +196,7 @@ private[spark] class BarrierCoordinator(
             s"tasks, finished successfully.")
           barrierEpoch += 1
           requesters.clear()
+          allGatherMessages.clear()
           cancelTimerTask()
         }
       }
@@ -173,7 +208,13 @@ private[spark] class BarrierCoordinator(
         requesters: ArrayBuffer[RpcCallContext],
         numTasks: Int): Boolean = {
       if (requesters.size == numTasks) {
-        requesters.foreach(_.reply(()))
+        requestMethodToSync match {
+          case RequestMethod.BARRIER =>
+            requesters.foreach(_.reply(""))
+          case RequestMethod.ALL_GATHER =>
+            val json: String = compact(render(allGatherMessages))
+            requesters.foreach(_.reply(json))
+        }
         true
       } else {
         false
@@ -186,6 +227,7 @@ private[spark] class BarrierCoordinator(
       // messages come from current stage attempt shall fail.
       barrierEpoch = -1
       requesters.clear()
+      allGatherMessages.clear()
       cancelTimerTask()
     }
   }
@@ -199,11 +241,11 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>
+    case request: RequestToSync =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
-      val barrierId = ContextBarrierId(stageId, stageAttemptId)
+      val barrierId = ContextBarrierId(request.stageId, request.stageAttemptId)
       states.computeIfAbsent(barrierId,
-        (key: ContextBarrierId) => new ContextBarrierState(key, numTasks))
+        (key: ContextBarrierId) => new ContextBarrierState(key, request.numTasks))
       val barrierState = states.get(barrierId)
 
       barrierState.handleRequest(context, request)
@@ -216,6 +258,16 @@ private[spark] class BarrierCoordinator(
 
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
+private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
+  def numTasks: Int
+  def stageId: Int
+  def stageAttemptId: Int
+  def taskAttemptId: Long
+  def barrierEpoch: Int
+  def partitionId: Int
+  def requestMethod: RequestMethod.Value
+}
+
 /**
  * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
@@ -224,11 +276,44 @@ private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls.
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
  */
-private[spark] case class RequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptId: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int) extends BarrierCoordinatorMessage
+private[spark] case class BarrierRequestToSync(
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  requestMethod: RequestMethod.Value
+) extends RequestToSync
+
+/**
+ * A global sync request message from BarrierTaskContext, by `allGather()` call. Each request is
+ * identified by stageId + stageAttemptId + barrierEpoch.
+ *
+ * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
+ * @param stageId ID of current stage
+ * @param stageAttemptId ID of current stage attempt
+ * @param taskAttemptId Unique ID of current task
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
+ * @param allGatherMessage Message sent from the BarrierTaskContext if requestMethod is ALL_GATHER
+ */
+private[spark] case class AllGatherRequestToSync(
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  requestMethod: RequestMethod.Value,
+  allGatherMessage: String
+) extends RequestToSync
+
+private[spark] object RequestMethod extends Enumeration {
+  val BARRIER, ALL_GATHER = Value
+}
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 3d369802f3023..2263538a11676 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -17,11 +17,19 @@
 
 package org.apache.spark
 
+import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.parse
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -59,49 +67,31 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  /**
-   * :: Experimental ::
-   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
-   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
-   * stage have reached this routine.
-   *
-   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
-   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
-   * timeout. Some examples of '''misuses''' are listed below:
-   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
-   * shall lead to timeout of the function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       if (context.partitionId() == 0) {
-   *           // Do nothing.
-   *       } else {
-   *           context.barrier()
-   *       }
-   *       iter
-   *   }
-   * }}}
-   *
-   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
-   * second function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       try {
-   *           // Do something that might throw an Exception.
-   *           doSomething()
-   *           context.barrier()
-   *       } catch {
-   *           case e: Exception => logWarning("...", e)
-   *       }
-   *       context.barrier()
-   *       iter
-   *   }
-   * }}}
-   */
-  @Experimental
-  @Since("2.4.0")
-  def barrier(): Unit = {
+  private def getRequestToSync(
+    numTasks: Int,
+    stageId: Int,
+    stageAttemptNumber: Int,
+    taskAttemptId: Long,
+    barrierEpoch: Int,
+    partitionId: Int,
+    requestMethod: RequestMethod.Value,
+    allGatherMessage: String
+  ): RequestToSync = {
+    requestMethod match {
+      case RequestMethod.BARRIER =>
+        BarrierRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, requestMethod)
+      case RequestMethod.ALL_GATHER =>
+        AllGatherRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, requestMethod, allGatherMessage)
+    }
+  }
+
+  private def runBarrier(
+    requestMethod: RequestMethod.Value,
+    allGatherMessage: String = ""
+  ): String = {
+
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -118,10 +108,12 @@ class BarrierTaskContext private[spark] (
     // Log the update of global sync every 60 seconds.
     timer.schedule(timerTask, 60000, 60000)
 
+    var json: String = ""
+
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[Unit](
-        message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch),
+      val abortableRpcFuture = barrierCoordinator.askAbortable[String](
+        message = getRequestToSync(numTasks, stageId, stageAttemptNumber,
+          taskAttemptId, barrierEpoch, partitionId, requestMethod, allGatherMessage),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
@@ -133,7 +125,7 @@ class BarrierTaskContext private[spark] (
         while (!abortableRpcFuture.toFuture.isCompleted) {
           // wait RPC future for at most 1 second
           try {
-            ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
+            json = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
           } catch {
             case _: TimeoutException | _: InterruptedException =>
               // If `TimeoutException` thrown, waiting RPC future reach 1 second.
@@ -163,6 +155,73 @@ class BarrierTaskContext private[spark] (
       timerTask.cancel()
       timer.purge()
     }
+    json
+  }
+
+  /**
+   * :: Experimental ::
+   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
+   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
+   * stage have reached this routine.
+   *
+   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
+   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
+   * timeout. Some examples of '''misuses''' are listed below:
+   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
+   * shall lead to timeout of the function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       if (context.partitionId() == 0) {
+   *           // Do nothing.
+   *       } else {
+   *           context.barrier()
+   *       }
+   *       iter
+   *   }
+   * }}}
+   *
+   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
+   * second function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       try {
+   *           // Do something that might throw an Exception.
+   *           doSomething()
+   *           context.barrier()
+   *       } catch {
+   *           case e: Exception => logWarning("...", e)
+   *       }
+   *       context.barrier()
+   *       iter
+   *   }
+   * }}}
+   */
+  @Experimental
+  @Since("2.4.0")
+  def barrier(): Unit = {
+    runBarrier(RequestMethod.BARRIER)
+    ()
+  }
+
+  /**
+   * :: Experimental ::
+   * Blocks until all tasks in the same stage have reached this routine. Each task passes in
+   * a message and returns with a list of all the messages passed in by each of those tasks.
+   *
+   * CAUTION! The allGather method requires the same precautions as the barrier method
+   *
+   * The message is type String rather than Array[Byte] because it is more convenient for
+   * the user at the cost of worse performance.
+   */
+  @Experimental
+  @Since("3.0.0")
+  def allGather(message: String): ArrayBuffer[String] = {
+    val json = runBarrier(RequestMethod.ALL_GATHER, message)
+    val jsonArray = parse(json)
+    implicit val formats = DefaultFormats
+    ArrayBuffer(jsonArray.extract[Array[String]]: _*)
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 658e0d593a167..fa8bf0fc06358 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -24,8 +24,13 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -238,13 +243,18 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
                   sock.setSoTimeout(10000)
                   authHelper.authClient(sock)
                   val input = new DataInputStream(sock.getInputStream())
-                  input.readInt() match {
+                  val requestMethod = input.readInt()
+                  // The BarrierTaskContext function may wait infinitely, socket shall not timeout
+                  // before the function finishes.
+                  sock.setSoTimeout(0)
+                  requestMethod match {
                     case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-                      // The barrier() function may wait infinitely, socket shall not timeout
-                      // before the function finishes.
-                      sock.setSoTimeout(0)
-                      barrierAndServe(sock)
-
+                      barrierAndServe(requestMethod, sock)
+                    case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+                      val length = input.readInt()
+                      val message = new Array[Byte](length)
+                      input.readFully(message)
+                      barrierAndServe(requestMethod, sock, new String(message, UTF_8))
                     case _ =>
                       val out = new DataOutputStream(new BufferedOutputStream(
                         sock.getOutputStream))
@@ -395,15 +405,31 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     /**
-     * Gateway to call BarrierTaskContext.barrier().
+     * Gateway to call BarrierTaskContext methods.
      */
-    def barrierAndServe(sock: Socket): Unit = {
-      require(serverSocket.isDefined, "No available ServerSocket to redirect the barrier() call.")
-
+    def barrierAndServe(requestMethod: Int, sock: Socket, message: String = ""): Unit = {
+      require(
+        serverSocket.isDefined,
+        "No available ServerSocket to redirect the BarrierTaskContext method call."
+      )
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        context.asInstanceOf[BarrierTaskContext].barrier()
-        writeUTF(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS, out)
+        var result: String = ""
+        requestMethod match {
+          case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
+            context.asInstanceOf[BarrierTaskContext].barrier()
+            result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
+          case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+            val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
+              message
+            )
+            result = compact(render(JArray(
+              messages.map(
+                (message) => JString(message)
+              ).toList
+            )))
+        }
+        writeUTF(result, out)
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
@@ -638,6 +664,7 @@ private[spark] object SpecialLengths {
 
 private[spark] object BarrierTaskContextMessageProtocol {
   val BARRIER_FUNCTION = 1
+  val ALL_GATHER_FUNCTION = 2
   val BARRIER_RESULT_SUCCESS = "success"
   val ERROR_UNRECOGNIZED_FUNCTION = "Not recognized function call from python side."
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index fc8ac38479932..ed38b7f0ecac1 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
+import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
 import org.apache.spark._
@@ -52,6 +53,79 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     assert(times.max - times.min <= 1000)
   }
 
+  test("share messages with allGather() call") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message = context.partitionId().toString
+      val messages = context.allGather(message)
+      messages.toList.iterator
+    }
+    // Take a sorted list of all the partitionId messages
+    val messages = rdd2.collect().head
+    // All the task partitionIds are shared
+    for((x, i) <- messages.view.zipWithIndex) assert(x == i.toString)
+  }
+
+  test("throw exception if we attempt to synchronize with different blocking calls") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      val partitionId = context.partitionId
+      if (partitionId == 0) {
+        context.barrier()
+      } else {
+        context.allGather(partitionId.toString)
+      }
+      Seq(null).iterator
+    }
+    val error = intercept[SparkException] {
+      rdd2.collect()
+    }.getMessage
+    assert(error.contains("does not match the current synchronized requestMethod"))
+  }
+
+  test("successively sync with allGather and barrier") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      context.barrier()
+      val time1 = System.currentTimeMillis()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message = context.partitionId().toString
+      val messages = context.allGather(message)
+      val time2 = System.currentTimeMillis()
+      Seq((time1, time2)).iterator
+    }
+    val times = rdd2.collect()
+    // All the tasks shall finish the first round of global sync within a short time slot.
+    val times1 = times.map(_._1)
+    assert(times1.max - times1.min <= 1000)
+
+    // All the tasks shall finish the second round of global sync within a short time slot.
+    val times2 = times.map(_._2)
+    assert(times2.max - times2.min <= 1000)
+  }
+
   test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index d648f63338514..90bd2345ac525 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -16,9 +16,10 @@
 #
 
 from __future__ import print_function
+import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, UTF8Deserializer
+from pyspark.serializers import write_int, write_with_length, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -107,18 +108,28 @@ def resources(self):
 
 
 BARRIER_FUNCTION = 1
+ALL_GATHER_FUNCTION = 2
 
 
-def _load_from_socket(port, auth_secret):
+def _load_from_socket(port, auth_secret, function, all_gather_message=None):
     """
     Load data from a given socket, this is a blocking method thus only return when the socket
     connection has been closed.
     """
     (sockfile, sock) = local_connect_and_auth(port, auth_secret)
-    # The barrier() call may block forever, so no timeout
+
+    # The call may block forever, so no timeout
     sock.settimeout(None)
-    # Make a barrier() function call.
-    write_int(BARRIER_FUNCTION, sockfile)
+
+    if function == BARRIER_FUNCTION:
+        # Make a barrier() function call.
+        write_int(function, sockfile)
+    elif function == ALL_GATHER_FUNCTION:
+        # Make a all_gather() function call.
+        write_int(function, sockfile)
+        write_with_length(all_gather_message.encode("utf-8"), sockfile)
+    else:
+        raise ValueError("Unrecognized function type")
     sockfile.flush()
 
     # Collect result.
@@ -199,7 +210,33 @@ def barrier(self):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            _load_from_socket(self._port, self._secret)
+            _load_from_socket(self._port, self._secret, BARRIER_FUNCTION)
+
+    def allGather(self, message=""):
+        """
+        .. note:: Experimental
+
+        This function blocks until all tasks in the same stage have reached this routine.
+        Each task passes in a message and returns with a list of all the messages passed in
+        by each of those tasks.
+
+        .. warning:: In a barrier stage, each task much have the same number of `allGather()`
+            calls, in all possible code branches.
+            Otherwise, you may get the job hanging or a SparkException after timeout.
+        """
+        if not isinstance(message, str):
+            raise ValueError("Argument `message` must be of type `str`")
+        elif self._port is None or self._secret is None:
+            raise Exception("Not supported to call barrier() before initialize " +
+                            "BarrierTaskContext.")
+        else:
+            gathered_items = _load_from_socket(
+                self._port,
+                self._secret,
+                ALL_GATHER_FUNCTION,
+                message,
+            )
+            return [e for e in json.loads(gathered_items)]
 
     def getTaskInfos(self):
         """
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index 6095a384679af..f5dbd068387c2 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -134,6 +134,26 @@ def context_barrier(x):
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
         self.assertTrue(max(times) - min(times) < 1)
 
+    def test_all_gather(self):
+        """
+        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
+        within a stage and passes messages properly.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 10))
+            out = tc.allGather(str(context.partitionId()))
+            pids = [int(e) for e in out]
+            return [pids]
+
+        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
+        self.assertTrue(pids == [0, 1, 2, 3])
+
     def test_barrier_infos(self):
         """
         Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the

From cadec3d239186e3abc104b67f7d1aa09d4d7516c Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Wed, 19 Feb 2020 17:06:20 -0800
Subject: [PATCH 0122/1280] Revert "[SPARK-30667][CORE] Add allGather method to
 BarrierTaskContext"

This reverts commit f482187c127418d2ea538ac2551ae0fce1ddbc31.
---
 .../org/apache/spark/BarrierCoordinator.scala | 113 ++-----------
 .../org/apache/spark/BarrierTaskContext.scala | 153 ++++++------------
 .../spark/api/python/PythonRunner.scala       |  51 ++----
 .../scheduler/BarrierTaskContextSuite.scala   |  74 ---------
 python/pyspark/taskcontext.py                 |  49 +-----
 python/pyspark/tests/test_taskcontext.py      |  20 ---
 6 files changed, 79 insertions(+), 381 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 042a2664a0e27..4e417679ca663 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -17,17 +17,12 @@
 
 package org.apache.spark
 
-import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.{compact, render}
-
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
@@ -104,15 +99,10 @@ private[spark] class BarrierCoordinator(
     // reset when a barrier() call fails due to timeout.
     private var barrierEpoch: Int = 0
 
-    // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
+    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()
+    // call.
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
-    // An Array of allGather messages for barrier tasks that have made a blocking runBarrier() call
-    private val allGatherMessages: ArrayBuffer[String] = new Array[String](numTasks).to[ArrayBuffer]
-
-    // The blocking requestMethod called by tasks to sync up for this stage attempt
-    private var requestMethodToSync: RequestMethod.Value = RequestMethod.BARRIER
-
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
 
@@ -140,32 +130,9 @@ private[spark] class BarrierCoordinator(
 
     // Process the global sync request. The barrier() call succeed if collected enough requests
     // within a configured time, otherwise fail all the pending requests.
-    def handleRequest(
-      requester: RpcCallContext,
-      request: RequestToSync
-    ): Unit = synchronized {
+    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
-      val requestMethod = request.requestMethod
-      val partitionId = request.partitionId
-      val allGatherMessage = request match {
-        case ag: AllGatherRequestToSync => ag.allGatherMessage
-        case _ => ""
-      }
-
-      if (requesters.size == 0) {
-        requestMethodToSync = requestMethod
-      }
-
-      if (requestMethodToSync != requestMethod) {
-        requesters.foreach(
-          _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
-            s"`$requestMethod` during barrier epoch $barrierEpoch, which does not match " +
-            s"the current synchronized requestMethod `$requestMethodToSync`"
-          ))
-        )
-        cleanupBarrierStage(barrierId)
-      }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
       require(request.numTasks == numTasks, s"Number of tasks of $barrierId is " +
@@ -186,7 +153,6 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
-        allGatherMessages(partitionId) = allGatherMessage
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
         if (maybeFinishAllRequesters(requesters, numTasks)) {
@@ -196,7 +162,6 @@ private[spark] class BarrierCoordinator(
             s"tasks, finished successfully.")
           barrierEpoch += 1
           requesters.clear()
-          allGatherMessages.clear()
           cancelTimerTask()
         }
       }
@@ -208,13 +173,7 @@ private[spark] class BarrierCoordinator(
         requesters: ArrayBuffer[RpcCallContext],
         numTasks: Int): Boolean = {
       if (requesters.size == numTasks) {
-        requestMethodToSync match {
-          case RequestMethod.BARRIER =>
-            requesters.foreach(_.reply(""))
-          case RequestMethod.ALL_GATHER =>
-            val json: String = compact(render(allGatherMessages))
-            requesters.foreach(_.reply(json))
-        }
+        requesters.foreach(_.reply(()))
         true
       } else {
         false
@@ -227,7 +186,6 @@ private[spark] class BarrierCoordinator(
       // messages come from current stage attempt shall fail.
       barrierEpoch = -1
       requesters.clear()
-      allGatherMessages.clear()
       cancelTimerTask()
     }
   }
@@ -241,11 +199,11 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request: RequestToSync =>
+    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
-      val barrierId = ContextBarrierId(request.stageId, request.stageAttemptId)
+      val barrierId = ContextBarrierId(stageId, stageAttemptId)
       states.computeIfAbsent(barrierId,
-        (key: ContextBarrierId) => new ContextBarrierState(key, request.numTasks))
+        (key: ContextBarrierId) => new ContextBarrierState(key, numTasks))
       val barrierState = states.get(barrierId)
 
       barrierState.handleRequest(context, request)
@@ -258,16 +216,6 @@ private[spark] class BarrierCoordinator(
 
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
-private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
-  def numTasks: Int
-  def stageId: Int
-  def stageAttemptId: Int
-  def taskAttemptId: Long
-  def barrierEpoch: Int
-  def partitionId: Int
-  def requestMethod: RequestMethod.Value
-}
-
 /**
  * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
@@ -276,44 +224,11 @@ private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
- * @param partitionId ID of the current partition the task is assigned to
- * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls.
  */
-private[spark] case class BarrierRequestToSync(
-  numTasks: Int,
-  stageId: Int,
-  stageAttemptId: Int,
-  taskAttemptId: Long,
-  barrierEpoch: Int,
-  partitionId: Int,
-  requestMethod: RequestMethod.Value
-) extends RequestToSync
-
-/**
- * A global sync request message from BarrierTaskContext, by `allGather()` call. Each request is
- * identified by stageId + stageAttemptId + barrierEpoch.
- *
- * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
- * @param stageId ID of current stage
- * @param stageAttemptId ID of current stage attempt
- * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
- * @param partitionId ID of the current partition the task is assigned to
- * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
- * @param allGatherMessage Message sent from the BarrierTaskContext if requestMethod is ALL_GATHER
- */
-private[spark] case class AllGatherRequestToSync(
-  numTasks: Int,
-  stageId: Int,
-  stageAttemptId: Int,
-  taskAttemptId: Long,
-  barrierEpoch: Int,
-  partitionId: Int,
-  requestMethod: RequestMethod.Value,
-  allGatherMessage: String
-) extends RequestToSync
-
-private[spark] object RequestMethod extends Enumeration {
-  val BARRIER, ALL_GATHER = Value
-}
+private[spark] case class RequestToSync(
+    numTasks: Int,
+    stageId: Int,
+    stageAttemptId: Int,
+    taskAttemptId: Long,
+    barrierEpoch: Int) extends BarrierCoordinatorMessage
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 2263538a11676..3d369802f3023 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -17,19 +17,11 @@
 
 package org.apache.spark
 
-import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
-import scala.language.postfixOps
-
-import org.json4s.DefaultFormats
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.parse
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -67,31 +59,49 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  private def getRequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptNumber: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int,
-    partitionId: Int,
-    requestMethod: RequestMethod.Value,
-    allGatherMessage: String
-  ): RequestToSync = {
-    requestMethod match {
-      case RequestMethod.BARRIER =>
-        BarrierRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch, partitionId, requestMethod)
-      case RequestMethod.ALL_GATHER =>
-        AllGatherRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch, partitionId, requestMethod, allGatherMessage)
-    }
-  }
-
-  private def runBarrier(
-    requestMethod: RequestMethod.Value,
-    allGatherMessage: String = ""
-  ): String = {
-
+  /**
+   * :: Experimental ::
+   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
+   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
+   * stage have reached this routine.
+   *
+   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
+   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
+   * timeout. Some examples of '''misuses''' are listed below:
+   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
+   * shall lead to timeout of the function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       if (context.partitionId() == 0) {
+   *           // Do nothing.
+   *       } else {
+   *           context.barrier()
+   *       }
+   *       iter
+   *   }
+   * }}}
+   *
+   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
+   * second function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       try {
+   *           // Do something that might throw an Exception.
+   *           doSomething()
+   *           context.barrier()
+   *       } catch {
+   *           case e: Exception => logWarning("...", e)
+   *       }
+   *       context.barrier()
+   *       iter
+   *   }
+   * }}}
+   */
+  @Experimental
+  @Since("2.4.0")
+  def barrier(): Unit = {
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -108,12 +118,10 @@ class BarrierTaskContext private[spark] (
     // Log the update of global sync every 60 seconds.
     timer.schedule(timerTask, 60000, 60000)
 
-    var json: String = ""
-
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[String](
-        message = getRequestToSync(numTasks, stageId, stageAttemptNumber,
-          taskAttemptId, barrierEpoch, partitionId, requestMethod, allGatherMessage),
+      val abortableRpcFuture = barrierCoordinator.askAbortable[Unit](
+        message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
@@ -125,7 +133,7 @@ class BarrierTaskContext private[spark] (
         while (!abortableRpcFuture.toFuture.isCompleted) {
           // wait RPC future for at most 1 second
           try {
-            json = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
+            ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
           } catch {
             case _: TimeoutException | _: InterruptedException =>
               // If `TimeoutException` thrown, waiting RPC future reach 1 second.
@@ -155,73 +163,6 @@ class BarrierTaskContext private[spark] (
       timerTask.cancel()
       timer.purge()
     }
-    json
-  }
-
-  /**
-   * :: Experimental ::
-   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
-   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
-   * stage have reached this routine.
-   *
-   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
-   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
-   * timeout. Some examples of '''misuses''' are listed below:
-   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
-   * shall lead to timeout of the function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       if (context.partitionId() == 0) {
-   *           // Do nothing.
-   *       } else {
-   *           context.barrier()
-   *       }
-   *       iter
-   *   }
-   * }}}
-   *
-   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
-   * second function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       try {
-   *           // Do something that might throw an Exception.
-   *           doSomething()
-   *           context.barrier()
-   *       } catch {
-   *           case e: Exception => logWarning("...", e)
-   *       }
-   *       context.barrier()
-   *       iter
-   *   }
-   * }}}
-   */
-  @Experimental
-  @Since("2.4.0")
-  def barrier(): Unit = {
-    runBarrier(RequestMethod.BARRIER)
-    ()
-  }
-
-  /**
-   * :: Experimental ::
-   * Blocks until all tasks in the same stage have reached this routine. Each task passes in
-   * a message and returns with a list of all the messages passed in by each of those tasks.
-   *
-   * CAUTION! The allGather method requires the same precautions as the barrier method
-   *
-   * The message is type String rather than Array[Byte] because it is more convenient for
-   * the user at the cost of worse performance.
-   */
-  @Experimental
-  @Since("3.0.0")
-  def allGather(message: String): ArrayBuffer[String] = {
-    val json = runBarrier(RequestMethod.ALL_GATHER, message)
-    val jsonArray = parse(json)
-    implicit val formats = DefaultFormats
-    ArrayBuffer(jsonArray.extract[Array[String]]: _*)
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index fa8bf0fc06358..658e0d593a167 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -24,13 +24,8 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.{compact, render}
-
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -243,18 +238,13 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
                   sock.setSoTimeout(10000)
                   authHelper.authClient(sock)
                   val input = new DataInputStream(sock.getInputStream())
-                  val requestMethod = input.readInt()
-                  // The BarrierTaskContext function may wait infinitely, socket shall not timeout
-                  // before the function finishes.
-                  sock.setSoTimeout(0)
-                  requestMethod match {
+                  input.readInt() match {
                     case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-                      barrierAndServe(requestMethod, sock)
-                    case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
-                      val length = input.readInt()
-                      val message = new Array[Byte](length)
-                      input.readFully(message)
-                      barrierAndServe(requestMethod, sock, new String(message, UTF_8))
+                      // The barrier() function may wait infinitely, socket shall not timeout
+                      // before the function finishes.
+                      sock.setSoTimeout(0)
+                      barrierAndServe(sock)
+
                     case _ =>
                       val out = new DataOutputStream(new BufferedOutputStream(
                         sock.getOutputStream))
@@ -405,31 +395,15 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     /**
-     * Gateway to call BarrierTaskContext methods.
+     * Gateway to call BarrierTaskContext.barrier().
      */
-    def barrierAndServe(requestMethod: Int, sock: Socket, message: String = ""): Unit = {
-      require(
-        serverSocket.isDefined,
-        "No available ServerSocket to redirect the BarrierTaskContext method call."
-      )
+    def barrierAndServe(sock: Socket): Unit = {
+      require(serverSocket.isDefined, "No available ServerSocket to redirect the barrier() call.")
+
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        var result: String = ""
-        requestMethod match {
-          case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-            context.asInstanceOf[BarrierTaskContext].barrier()
-            result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
-          case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
-            val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
-              message
-            )
-            result = compact(render(JArray(
-              messages.map(
-                (message) => JString(message)
-              ).toList
-            )))
-        }
-        writeUTF(result, out)
+        context.asInstanceOf[BarrierTaskContext].barrier()
+        writeUTF(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS, out)
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
@@ -664,7 +638,6 @@ private[spark] object SpecialLengths {
 
 private[spark] object BarrierTaskContextMessageProtocol {
   val BARRIER_FUNCTION = 1
-  val ALL_GATHER_FUNCTION = 2
   val BARRIER_RESULT_SUCCESS = "success"
   val ERROR_UNRECOGNIZED_FUNCTION = "Not recognized function call from python side."
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index ed38b7f0ecac1..fc8ac38479932 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
-import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
 import org.apache.spark._
@@ -53,79 +52,6 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     assert(times.max - times.min <= 1000)
   }
 
-  test("share messages with allGather() call") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
-    val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { it =>
-      val context = BarrierTaskContext.get()
-      // Sleep for a random time before global sync.
-      Thread.sleep(Random.nextInt(1000))
-      // Pass partitionId message in
-      val message = context.partitionId().toString
-      val messages = context.allGather(message)
-      messages.toList.iterator
-    }
-    // Take a sorted list of all the partitionId messages
-    val messages = rdd2.collect().head
-    // All the task partitionIds are shared
-    for((x, i) <- messages.view.zipWithIndex) assert(x == i.toString)
-  }
-
-  test("throw exception if we attempt to synchronize with different blocking calls") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
-    val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { it =>
-      val context = BarrierTaskContext.get()
-      val partitionId = context.partitionId
-      if (partitionId == 0) {
-        context.barrier()
-      } else {
-        context.allGather(partitionId.toString)
-      }
-      Seq(null).iterator
-    }
-    val error = intercept[SparkException] {
-      rdd2.collect()
-    }.getMessage
-    assert(error.contains("does not match the current synchronized requestMethod"))
-  }
-
-  test("successively sync with allGather and barrier") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
-    val rdd = sc.makeRDD(1 to 10, 4)
-    val rdd2 = rdd.barrier().mapPartitions { it =>
-      val context = BarrierTaskContext.get()
-      // Sleep for a random time before global sync.
-      Thread.sleep(Random.nextInt(1000))
-      context.barrier()
-      val time1 = System.currentTimeMillis()
-      // Sleep for a random time before global sync.
-      Thread.sleep(Random.nextInt(1000))
-      // Pass partitionId message in
-      val message = context.partitionId().toString
-      val messages = context.allGather(message)
-      val time2 = System.currentTimeMillis()
-      Seq((time1, time2)).iterator
-    }
-    val times = rdd2.collect()
-    // All the tasks shall finish the first round of global sync within a short time slot.
-    val times1 = times.map(_._1)
-    assert(times1.max - times1.min <= 1000)
-
-    // All the tasks shall finish the second round of global sync within a short time slot.
-    val times2 = times.map(_._2)
-    assert(times2.max - times2.min <= 1000)
-  }
-
   test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index 90bd2345ac525..d648f63338514 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -16,10 +16,9 @@
 #
 
 from __future__ import print_function
-import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, write_with_length, UTF8Deserializer
+from pyspark.serializers import write_int, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -108,28 +107,18 @@ def resources(self):
 
 
 BARRIER_FUNCTION = 1
-ALL_GATHER_FUNCTION = 2
 
 
-def _load_from_socket(port, auth_secret, function, all_gather_message=None):
+def _load_from_socket(port, auth_secret):
     """
     Load data from a given socket, this is a blocking method thus only return when the socket
     connection has been closed.
     """
     (sockfile, sock) = local_connect_and_auth(port, auth_secret)
-
-    # The call may block forever, so no timeout
+    # The barrier() call may block forever, so no timeout
     sock.settimeout(None)
-
-    if function == BARRIER_FUNCTION:
-        # Make a barrier() function call.
-        write_int(function, sockfile)
-    elif function == ALL_GATHER_FUNCTION:
-        # Make a all_gather() function call.
-        write_int(function, sockfile)
-        write_with_length(all_gather_message.encode("utf-8"), sockfile)
-    else:
-        raise ValueError("Unrecognized function type")
+    # Make a barrier() function call.
+    write_int(BARRIER_FUNCTION, sockfile)
     sockfile.flush()
 
     # Collect result.
@@ -210,33 +199,7 @@ def barrier(self):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            _load_from_socket(self._port, self._secret, BARRIER_FUNCTION)
-
-    def allGather(self, message=""):
-        """
-        .. note:: Experimental
-
-        This function blocks until all tasks in the same stage have reached this routine.
-        Each task passes in a message and returns with a list of all the messages passed in
-        by each of those tasks.
-
-        .. warning:: In a barrier stage, each task much have the same number of `allGather()`
-            calls, in all possible code branches.
-            Otherwise, you may get the job hanging or a SparkException after timeout.
-        """
-        if not isinstance(message, str):
-            raise ValueError("Argument `message` must be of type `str`")
-        elif self._port is None or self._secret is None:
-            raise Exception("Not supported to call barrier() before initialize " +
-                            "BarrierTaskContext.")
-        else:
-            gathered_items = _load_from_socket(
-                self._port,
-                self._secret,
-                ALL_GATHER_FUNCTION,
-                message,
-            )
-            return [e for e in json.loads(gathered_items)]
+            _load_from_socket(self._port, self._secret)
 
     def getTaskInfos(self):
         """
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index f5dbd068387c2..6095a384679af 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -134,26 +134,6 @@ def context_barrier(x):
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
         self.assertTrue(max(times) - min(times) < 1)
 
-    def test_all_gather(self):
-        """
-        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
-        within a stage and passes messages properly.
-        """
-        rdd = self.sc.parallelize(range(10), 4)
-
-        def f(iterator):
-            yield sum(iterator)
-
-        def context_barrier(x):
-            tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
-            out = tc.allGather(str(context.partitionId()))
-            pids = [int(e) for e in out]
-            return [pids]
-
-        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
-        self.assertTrue(pids == [0, 1, 2, 3])
-
     def test_barrier_infos(self):
         """
         Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the

From 74de94879d862cd404138fa24680a4e590a5f463 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 20 Feb 2020 13:07:52 +0800
Subject: [PATCH 0123/1280] [SPARK-30878][SQL][DOC] Improve the CREATE TABLE
 document

### What changes were proposed in this pull request?

Improve the CREATE TABLE document:
1. mention that some clauses can come in as any order.
2. refine the description for some parameters.
3. mention how data source table interacts with data source
4. make the examples consistent between data source and hive serde tables.

### Why are the changes needed?

improve doc

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #27638 from cloud-fan/doc.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ef90f1422fcbbf85253580a7ef1e97f5411314e1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 ...-ref-syntax-ddl-create-table-datasource.md | 49 ++++++++++++++-----
 ...-ref-syntax-ddl-create-table-hiveformat.md | 47 ++++++++++++------
 docs/sql-ref-syntax-ddl-create-table.md       |  4 +-
 3 files changed, 72 insertions(+), 28 deletions(-)

diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index 9b15c0865b38a..532377d7fcec3 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -27,7 +27,7 @@ The `CREATE TABLE` statement defines a new table using a Data Source.
 {% highlight sql %}
 CREATE TABLE [ IF NOT EXISTS ] table_identifier
   [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
-  USING data_source
+  [USING data_source]
   [ OPTIONS ( key1=val1, key2=val2, ... ) ]
   [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
   [ CLUSTERED BY ( col_name3, col_name4, ... ) 
@@ -39,6 +39,9 @@ CREATE TABLE [ IF NOT EXISTS ] table_identifier
   [ AS select_statement ]
 {% endhighlight %}
 
+Note that, the clauses between the USING clause and the AS SELECT clause can come in
+as any order. For example, you can write COMMENT table_comment after TBLPROPERTIES.
+
 ### Parameters
 
 <dl>
@@ -78,13 +81,12 @@ CREATE TABLE [ IF NOT EXISTS ] table_identifier
 
 <dl>
   <dt><code><em>COMMENT</em></code></dt>
-  <dd>Table comments are added.</dd>
+  <dd>A string literal to describe the table.</dd>
 </dl>
 
 <dl>
   <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
-  </dd>
+  <dd>A list of key-value pairs that is used to tag the table definition.</dd>
 </dl>
 
 <dl>
@@ -92,18 +94,43 @@ CREATE TABLE [ IF NOT EXISTS ] table_identifier
   <dd>The table is populated using the data from the select statement.</dd>
 </dl>
 
+### Data Source Interaction
+A Data Source table acts like a pointer to the underlying data source. For example, you can create
+a table "foo" in Spark which points to a table "bar" in MySQL using JDBC Data Source. When you
+read/write table "foo", you actually read/write table "bar".
+ 
+In general CREATE TABLE is creating a "pointer", and you need to make sure it points to something
+existing. An exception is file source such as parquet, json. If you don't specify the LOCATION,
+Spark will create a default table location for you.
+
+For CREATE TABLE AS SELECT, Spark will overwrite the underlying data source with the data of the
+input query, to make sure the table gets created contains exactly the same data as the input query.
+
 ### Examples
 {% highlight sql %}
 
---Using data source
-CREATE TABLE Student (Id INT,name STRING ,age INT) USING CSV;
+--Use data source
+CREATE TABLE student (id INT, name STRING, age INT) USING CSV;
+
+--Use data from another table
+CREATE TABLE student_copy USING CSV
+  AS SELECT * FROM student;
+  
+--Omit the USING clause, which uses the default data source (parquet by default)
+CREATE TABLE student (id INT, name STRING, age INT);
+
+--Specify table comment and properties
+CREATE TABLE student (id INT, name STRING, age INT) USING CSV
+  COMMENT 'this is a comment'
+  TBLPROPERTIES ('foo'='bar');
 
---Using data from another table
-CREATE TABLE StudentInfo
-  AS SELECT * FROM Student;
+--Specify table comment and properties with different clauses order
+CREATE TABLE student (id INT, name STRING, age INT) USING CSV
+  TBLPROPERTIES ('foo'='bar')
+  COMMENT 'this is a comment';
 
---Partitioned and bucketed
-CREATE TABLE Student (Id INT,name STRING ,age INT)
+--Create partitioned and bucketed table
+CREATE TABLE student (id INT, name STRING, age INT)
   USING CSV
   PARTITIONED BY (age)
   CLUSTERED BY (Id) INTO 4 buckets;
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 78b7d0581e985..0425bafd94398 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -37,6 +37,9 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
 
 {% endhighlight %}
 
+Note that, the clauses between the columns definition clause and the AS SELECT clause can come in
+as any order. For example, you can write COMMENT table_comment after TBLPROPERTIES.
+
 ### Parameters
 
 <dl>
@@ -77,14 +80,12 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
 
 <dl>
   <dt><code><em>COMMENT</em></code></dt>
-  <dd>Table comments are added.</dd>
+  <dd>A string literal to describe the table.</dd>
 </dl>
 
 <dl>
   <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>
-	Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
-  </dd>
+  <dd>A list of key-value pairs that is used to tag the table definition.</dd>
 </dl>
 
 <dl>
@@ -96,21 +97,37 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
 ### Examples
 {% highlight sql %}
 
---Using Comment and loading data from another table into the created table
-CREATE TABLE StudentInfo
-  COMMENT 'Table is created using existing data'
-  AS SELECT * FROM Student;
+--Use hive format
+CREATE TABLE student (id INT, name STRING, age INT) STORED AS ORC;
+
+--Use data from another table
+CREATE TABLE student_copy STORED AS ORC
+  AS SELECT * FROM student;
+
+--Specify table comment and properties
+CREATE TABLE student (id INT, name STRING, age INT)
+  COMMENT 'this is a comment'
+  STORED AS ORC
+  TBLPROPERTIES ('foo'='bar');  
+
+--Specify table comment and properties with different clauses order
+CREATE TABLE student (id INT, name STRING, age INT)
+  STORED AS ORC
+  TBLPROPERTIES ('foo'='bar')
+  COMMENT 'this is a comment';
 
---Partitioned table
-CREATE TABLE Student (Id INT,name STRING)
+--Create partitioned table
+CREATE TABLE student (id INT, name STRING)
   PARTITIONED BY (age INT)
-  TBLPROPERTIES ('owner'='xxxx');
+  STORED AS ORC;
 
-CREATE TABLE Student (Id INT,name STRING,age INT)
-  PARTITIONED BY (name,age);
+--Create partitioned table with different clauses order
+CREATE TABLE student (id INT, name STRING)
+  STORED AS ORC
+  PARTITIONED BY (age INT);
 
---Using Row Format and file format
-CREATE TABLE Student (Id INT,name STRING)
+--Use Row Format and file format
+CREATE TABLE student (id INT,name STRING)
   ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
   STORED AS TEXTFILE;
 
diff --git a/docs/sql-ref-syntax-ddl-create-table.md b/docs/sql-ref-syntax-ddl-create-table.md
index c0e58a41cf5cc..20aff6fb823cb 100644
--- a/docs/sql-ref-syntax-ddl-create-table.md
+++ b/docs/sql-ref-syntax-ddl-create-table.md
@@ -20,10 +20,10 @@ license: |
 ---
 
 ### Description
-`CREATE TABLE` statement is used to define a table in an exsisting database. 
+`CREATE TABLE` statement is used to define a table in an existing database. 
 
 The CREATE statements:
-* [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+* [CREATE TABLE USING DATA_SOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
 * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
 * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
 

From b38b237ac22c08ce6aadb041b2bf1b78f5f5db75 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Thu, 20 Feb 2020 00:59:22 -0800
Subject: [PATCH 0124/1280] [SPARK-30881][SQL][DOCS] Revise the doc of
 spark.sql.sources.parallelPartitionDiscovery.threshold

### What changes were proposed in this pull request?

Revise the doc of SQL configuration `spark.sql.sources.parallelPartitionDiscovery.threshold`.
### Why are the changes needed?

The doc of configuration "spark.sql.sources.parallelPartitionDiscovery.threshold" is not accurate on the part "This applies to Parquet, ORC, CSV, JSON and LibSVM data sources".

We should revise it as effective on all the file-based data sources.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

None. It's just doc.

Closes #27639 from gengliangwang/reviseParallelPartitionDiscovery.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 92d5d40c8efffd90eb04308b6dad77d3d1e1be14)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 612fa86599900..9cbaaeefd1978 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -878,8 +878,8 @@ object SQLConf {
     buildConf("spark.sql.sources.parallelPartitionDiscovery.threshold")
       .doc("The maximum number of paths allowed for listing files at driver side. If the number " +
         "of detected paths exceeds this value during partition discovery, it tries to list the " +
-        "files with another Spark distributed job. This applies to Parquet, ORC, CSV, JSON and " +
-        "LibSVM data sources.")
+        "files with another Spark distributed job. This configuration is effective only when " +
+        "using file-based sources such as Parquet, JSON and ORC.")
       .intConf
       .checkValue(parallel => parallel >= 0, "The maximum number of paths allowed for listing " +
         "files at driver side must not be negative")

From d90dd8a91f3763ae0c1cbc0a50c1a736db0c90d7 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 20 Feb 2020 21:26:20 +0800
Subject: [PATCH 0125/1280] [SPARK-30858][SQL] Make IntegralDivide's dataType
 independent from SQL config changes

### What changes were proposed in this pull request?
In the PR, I propose to add the `returnLong` parameter to `IntegralDivide`, and pass the value of `spark.sql.legacy.integralDivide.returnBigint` if `returnLong` is not provided on creation of `IntegralDivide`.

### Why are the changes needed?
This allows to avoid the issue when the configuration change between different phases of planning, and this can silently break a query plan which can lead to crashes or data corruption.

OptionsAttachments

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By `ArithmeticExpressionSuite`.

Closes #27628 from MaxGekk/integral-divide-conf.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 4248b7fbb9fd53b5d95bb60fbcec1248052d2879)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalyst/analysis/DecimalPrecision.scala  |  8 ++++---
 .../sql/catalyst/expressions/arithmetic.scala | 21 +++++++++++++++----
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
index a64befecb68d6..91489b3e46ff3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
@@ -175,10 +175,12 @@ object DecimalPrecision extends TypeCoercionRule {
         resultType, nullOnOverflow)
 
     case expr @ IntegralDivide(
-        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
+        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), returnLong) =>
       val widerType = widerDecimalType(p1, s1, p2, s2)
-      val promotedExpr =
-        IntegralDivide(promotePrecision(e1, widerType), promotePrecision(e2, widerType))
+      val promotedExpr = IntegralDivide(
+        promotePrecision(e1, widerType),
+        promotePrecision(e2, widerType),
+        returnLong)
       if (expr.dataType.isInstanceOf[DecimalType]) {
         // This follows division rule
         val intDig = p1 - s1 + s2
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index debd7c89adb9f..624891c619d41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -403,11 +403,18 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
   """,
   since = "3.0.0")
 // scalastyle:on line.size.limit
-case class IntegralDivide(left: Expression, right: Expression) extends DivModLike {
+case class IntegralDivide(
+    left: Expression,
+    right: Expression,
+    returnLong: Boolean) extends DivModLike {
+
+  def this(left: Expression, right: Expression) = {
+    this(left, right, SQLConf.get.integralDivideReturnLong)
+  }
 
   override def inputType: AbstractDataType = TypeCollection(IntegralType, DecimalType)
 
-  override def dataType: DataType = if (SQLConf.get.integralDivideReturnLong) {
+  override def dataType: DataType = if (returnLong) {
     LongType
   } else {
     left.dataType
@@ -416,7 +423,7 @@ case class IntegralDivide(left: Expression, right: Expression) extends DivModLik
   override def symbol: String = "/"
   override def decimalMethod: String = "quot"
   override def decimalToDataTypeCodeGen(decimalResult: String): String = {
-    if (SQLConf.get.integralDivideReturnLong) {
+    if (returnLong) {
       s"$decimalResult.toLong()"
     } else {
       decimalResult
@@ -433,7 +440,7 @@ case class IntegralDivide(left: Expression, right: Expression) extends DivModLik
         d.asIntegral.asInstanceOf[Integral[Any]]
     }
     val divide = integral.quot _
-    if (SQLConf.get.integralDivideReturnLong) {
+    if (returnLong) {
       val toLong = integral.asInstanceOf[Integral[Any]].toLong _
       (x, y) => {
         val res = divide(x, y)
@@ -451,6 +458,12 @@ case class IntegralDivide(left: Expression, right: Expression) extends DivModLik
   override def evalOperation(left: Any, right: Any): Any = div(left, right)
 }
 
+object IntegralDivide {
+  def apply(left: Expression, right: Expression): IntegralDivide = {
+    new IntegralDivide(left, right)
+  }
+}
+
 @ExpressionDescription(
   usage = "expr1 _FUNC_ expr2 - Returns the remainder after `expr1`/`expr2`.",
   examples = """

From c1000a6bdce53f171ff00ea03b515950aaff4f95 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 20 Feb 2020 22:10:04 +0800
Subject: [PATCH 0126/1280] [SPARK-26071][FOLLOWUP] Improve migration guide of
 disallowing map type map key

### What changes were proposed in this pull request?

mention the workaround if users do want to use map type as key, and add a test to demonstrate it.

### Why are the changes needed?

it's better to provide an alternative when we ban something.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #27621 from cloud-fan/map.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 704d249a56325fce4a8179a2a7a242b9469aa6ec)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                               | 2 +-
 .../org/apache/spark/sql/DataFrameFunctionsSuite.scala    | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 0690127a45b17..9b74b456b88e7 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -43,7 +43,7 @@ license: |
 
   - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
 
-  - In Spark version 2.4 and earlier, users can create map values with map type key via built-in function like `CreateMap`, `MapFromArrays`, etc. Since Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can still read map values with map type key from data source or Java/Scala collections, though they are not very useful.
+  - In Spark version 2.4 and earlier, users can create map values with map type key via built-in function such as `CreateMap`, `MapFromArrays`, etc. Since Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can use `map_entries` function to convert map to array<struct<key, value>> as a workaround. In addition, users can still read map values with map type key from data source or Java/Scala collections, though it is discouraged.
 
   - In Spark version 2.4 and earlier, `Dataset.groupByKey` results to a grouped dataset with key attribute wrongly named as "value", if the key is non-struct type, e.g. int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries weird. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behaviour is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 341b3255b12bf..b4b9a488b11c4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -3584,6 +3584,14 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
         }
     }
   }
+
+  test("SPARK-26071: convert map to array and use as map key") {
+    val df = Seq(Map(1 -> "a")).toDF("m")
+    intercept[AnalysisException](df.select(map($"m", lit(1))))
+    checkAnswer(
+      df.select(map(map_entries($"m"), lit(1))),
+      Row(Map(Seq(Row(1, "a")) -> 1)))
+  }
 }
 
 object DataFrameFunctionsSuite {

From 68d8ff1c5005122043084b94e0e44ecd6d91f4f0 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 21 Feb 2020 00:00:48 +0800
Subject: [PATCH 0127/1280] [SPARK-29930][SPARK-30416][SQL][FOLLOWUP] Move
 deprecated/removed config checks from RuntimeConfig to SQLConf

### What changes were proposed in this pull request?
- Output warnings for deprecated SQL configs in `SQLConf. setConfWithCheck()` and in `SQLConf. unsetConf()`
- Throw an exception for removed SQL configs in `SQLConf. setConfWithCheck()` when they set to non-default values
- Remove checking of deprecated and removed SQL configs from RuntimeConfig

### Why are the changes needed?
Currently, warnings/exceptions are printed only when a SQL config is set dynamically, for instance via `spark.conf.set()`. After the changes, removed/deprecated SQL configs will be checked when they set statically. For example:
```
$ bin/spark-shell --conf spark.sql.fromJsonForceNullableSchema=false
scala> spark.emptyDataFrame
java.lang.IllegalArgumentException: Error while instantiating 'org.apache.spark.sql.hive.HiveSessionStateBuilder':
...
Caused by: org.apache.spark.sql.AnalysisException: The SQL config 'spark.sql.fromJsonForceNullableSchema' was removed in the version 3.0.0. It was removed to prevent errors like SPARK-23173 for non-default value.
```
```
$ bin/spark-shell --conf spark.sql.hive.verifyPartitionPath=false
scala> spark.emptyDataFrame
20/02/20 02:10:26 WARN SQLConf: The SQL config 'spark.sql.hive.verifyPartitionPath' has been deprecated in Spark v3.0 and may be removed in the future. This config is replaced by 'spark.files.ignoreMissingFiles'.
```

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
By `SQLConfSuite`

Closes #27645 from MaxGekk/remove-sql-configs-followup-2.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a551715fd26779a0e02534d81d1c3f0804152fec)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 28 ++++++++++++++++++-
 .../org/apache/spark/sql/RuntimeConfig.scala  | 27 +-----------------
 2 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 9cbaaeefd1978..2d72344d7042b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -33,6 +33,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.{IGNORE_MISSING_FILES => SPARK_IGNORE_MISSING_FILES}
 import org.apache.spark.network.util.ByteUnit
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{HintErrorLogger, Resolver}
 import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
@@ -2899,16 +2900,41 @@ class SQLConf extends Serializable with Logging {
     settings.containsKey(key)
   }
 
+  /**
+   * Logs a warning message if the given config key is deprecated.
+   */
+  private def logDeprecationWarning(key: String): Unit = {
+    SQLConf.deprecatedSQLConfigs.get(key).foreach {
+      case DeprecatedConfig(configName, version, comment) =>
+        logWarning(
+          s"The SQL config '$configName' has been deprecated in Spark v$version " +
+          s"and may be removed in the future. $comment")
+    }
+  }
+
+  private def requireDefaultValueOfRemovedConf(key: String, value: String): Unit = {
+    SQLConf.removedSQLConfigs.get(key).foreach {
+      case RemovedConfig(configName, version, defaultValue, comment) =>
+        if (value != defaultValue) {
+          throw new AnalysisException(
+            s"The SQL config '$configName' was removed in the version $version. $comment")
+        }
+    }
+  }
+
   protected def setConfWithCheck(key: String, value: String): Unit = {
+    logDeprecationWarning(key)
+    requireDefaultValueOfRemovedConf(key, value)
     settings.put(key, value)
   }
 
   def unsetConf(key: String): Unit = {
+    logDeprecationWarning(key)
     settings.remove(key)
   }
 
   def unsetConf(entry: ConfigEntry[_]): Unit = {
-    settings.remove(entry.key)
+    unsetConf(entry.key)
   }
 
   def clear(): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
index e1b44b5918143..e9bc25d489718 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RuntimeConfig.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.internal.SQLConf.{DeprecatedConfig, RemovedConfig}
  * @since 2.0.0
  */
 @Stable
-class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging {
+class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
 
   /**
    * Sets the given Spark runtime configuration property.
@@ -40,8 +40,6 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
    */
   def set(key: String, value: String): Unit = {
     requireNonStaticConf(key)
-    requireDefaultValueOfRemovedConf(key, value)
-    logDeprecationWarning(key)
     sqlConf.setConfString(key, value)
   }
 
@@ -130,7 +128,6 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
    */
   def unset(key: String): Unit = {
     requireNonStaticConf(key)
-    logDeprecationWarning(key)
     sqlConf.unsetConf(key)
   }
 
@@ -161,26 +158,4 @@ class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) extends Logging
       throw new AnalysisException(s"Cannot modify the value of a Spark config: $key")
     }
   }
-
-  private def requireDefaultValueOfRemovedConf(key: String, value: String): Unit = {
-    SQLConf.removedSQLConfigs.get(key).foreach {
-      case RemovedConfig(configName, version, defaultValue, comment) =>
-        if (value != defaultValue) {
-          throw new AnalysisException(
-            s"The SQL config '$configName' was removed in the version $version. $comment")
-        }
-    }
-  }
-
-  /**
-   * Logs a warning message if the given config key is deprecated.
-   */
-  private def logDeprecationWarning(key: String): Unit = {
-    SQLConf.deprecatedSQLConfigs.get(key).foreach {
-      case DeprecatedConfig(configName, version, comment) =>
-        logWarning(
-          s"The SQL config '$configName' has been deprecated in Spark v$version " +
-          s"and may be removed in the future. $comment")
-    }
-  }
 }

From 45f2155654d1a219379737f7ee51311fa73a30d5 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 21 Feb 2020 00:44:09 +0800
Subject: [PATCH 0128/1280] [SPARK-30892][SQL] Exclude
 `spark.sql.variable.substitute.depth` from `removedSQLConfigs`

### What changes were proposed in this pull request?

Exclude the SQL config `spark.sql.variable.substitute.depth` from `SQLConf.removedSQLConfigs`

### Why are the changes needed?
By the #27169, the config was placed to `SQLConf.removedSQLConfigs`. And as a consequence of that when an user set it non-default value (1 for example),  he/she will get an exception. It is acceptable for SQL configs that could impact on the behavior but not for this particular config. Raising of such exception will just make migration to Spark 3.0 more difficult.

### Does this PR introduce any user-facing change?
Yes, before the changes users get an exception when he/she set `spark.sql.variable.substitute.depth` to a value different from `40`.

### How was this patch tested?
Run `spark.conf.set("spark.sql.variable.substitute.depth", 1)` in `spark-shell`.

Closes #27646 from MaxGekk/remove-substitute-depth-conf.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bb40ab09f4bcb0fd94b0649442cc0daa1162207d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 2d72344d7042b..bcec22f91924b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2264,8 +2264,6 @@ object SQLConf {
         "It was removed to prevent loosing of users data for non-default value."),
       RemovedConfig("spark.sql.legacy.compareDateTimestampInTimestamp", "3.0.0", "true",
         "It was removed to prevent errors like SPARK-23549 for non-default value."),
-      RemovedConfig("spark.sql.variable.substitute.depth", "3.0.0", "40",
-        "It was deprecated since Spark 2.1, and not used in Spark 2.4."),
       RemovedConfig("spark.sql.parquet.int64AsTimestampMillis", "3.0.0", "false",
         "The config was deprecated since Spark 2.3." +
         s"Use '${PARQUET_OUTPUT_TIMESTAMP_TYPE.key}' instead of it."),

From 30e0e46f40b88daa7a0711350ea3129ba29fedc4 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 20 Feb 2020 09:08:33 -0800
Subject: [PATCH 0129/1280] [SPARK-30884][PYSPARK] Upgrade to Py4J 0.10.9

This PR aims to upgrade Py4J to `0.10.9` for better Python 3.7 support in Apache Spark 3.0.0 (master/branch-3.0). This is not for `branch-2.4`.

- Apache Spark 3.0.0 is using `Py4J 0.10.8.1` (released on 2018-10-21) because `0.10.8.1` was the first official release to support Python 3.7.
    - https://www.py4j.org/changelog.html#py4j-0-10-8-and-py4j-0-10-8-1
- `Py4J 0.10.9` was released on January 25th 2020 with better Python 3.7 support and `magic_member` bug fix.
    - https://github.com/bartdag/py4j/releases/tag/0.10.9
    - https://www.py4j.org/changelog.html#py4j-0-10-9

No.

Pass the Jenkins with the existing tests.

Closes #27641 from dongjoon-hyun/SPARK-30884.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit fc4e56a54c15e20baf085e6061d3d83f5ce1185d)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 bin/pyspark                                     |   2 +-
 bin/pyspark2.cmd                                |   2 +-
 core/pom.xml                                    |   2 +-
 .../apache/spark/api/python/PythonUtils.scala   |   2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-1.2         |   2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3         |   2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3         |   2 +-
 python/README.md                                |   2 +-
 python/docs/Makefile                            |   2 +-
 python/docs/make2.bat                           |   2 +-
 python/lib/py4j-0.10.8.1-src.zip                | Bin 41255 -> 0 bytes
 python/lib/py4j-0.10.9-src.zip                  | Bin 0 -> 41587 bytes
 python/setup.py                                 |   2 +-
 sbin/spark-config.sh                            |   2 +-
 14 files changed, 12 insertions(+), 12 deletions(-)
 delete mode 100644 python/lib/py4j-0.10.8.1-src.zip
 create mode 100644 python/lib/py4j-0.10.9-src.zip

diff --git a/bin/pyspark b/bin/pyspark
index 44891aee2e0a3..ad4132fb59eb0 100755
--- a/bin/pyspark
+++ b/bin/pyspark
@@ -50,7 +50,7 @@ export PYSPARK_DRIVER_PYTHON_OPTS
 
 # Add the PySpark classes to the Python path:
 export PYTHONPATH="${SPARK_HOME}/python/:$PYTHONPATH"
-export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.8.1-src.zip:$PYTHONPATH"
+export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:$PYTHONPATH"
 
 # Load the PySpark shell.py script when ./pyspark is used interactively:
 export OLD_PYTHONSTARTUP="$PYTHONSTARTUP"
diff --git a/bin/pyspark2.cmd b/bin/pyspark2.cmd
index 479fd464c7d3e..dc34be1a41706 100644
--- a/bin/pyspark2.cmd
+++ b/bin/pyspark2.cmd
@@ -30,7 +30,7 @@ if "x%PYSPARK_DRIVER_PYTHON%"=="x" (
 )
 
 set PYTHONPATH=%SPARK_HOME%\python;%PYTHONPATH%
-set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.8.1-src.zip;%PYTHONPATH%
+set PYTHONPATH=%SPARK_HOME%\python\lib\py4j-0.10.9-src.zip;%PYTHONPATH%
 
 set OLD_PYTHONSTARTUP=%PYTHONSTARTUP%
 set PYTHONSTARTUP=%SPARK_HOME%\python\pyspark\shell.py
diff --git a/core/pom.xml b/core/pom.xml
index 9d54d21b95ba3..11faf6b1b0e38 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -414,7 +414,7 @@
     <dependency>
       <groupId>net.sf.py4j</groupId>
       <artifactId>py4j</artifactId>
-      <version>0.10.8.1</version>
+      <version>0.10.9</version>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 62d60475985b3..490b48719b6be 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkContext
 import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
 
 private[spark] object PythonUtils {
-  val PY4J_ZIP_NAME = "py4j-0.10.8.1-src.zip"
+  val PY4J_ZIP_NAME = "py4j-0.10.9-src.zip"
 
   /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */
   def sparkPythonPath: String = {
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 62d5772ffc94a..113b7d7ede7bd 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -174,7 +174,7 @@ parquet-hadoop-bundle/1.6.0//parquet-hadoop-bundle-1.6.0.jar
 parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
 parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.8.1//py4j-0.10.8.1.jar
+py4j/0.10.9//py4j-0.10.9.jar
 pyrolite/4.30//pyrolite-4.30.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
 scala-compiler/2.12.10//scala-compiler-2.12.10.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 4f4d8b1d4a62a..7b7423aa79217 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -188,7 +188,7 @@ parquet-format/2.4.0//parquet-format-2.4.0.jar
 parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
 parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.8.1//py4j-0.10.8.1.jar
+py4j/0.10.9//py4j-0.10.9.jar
 pyrolite/4.30//pyrolite-4.30.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
 scala-compiler/2.12.10//scala-compiler-2.12.10.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 18e4246d63996..4dddbba04405e 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -203,7 +203,7 @@ parquet-format/2.4.0//parquet-format-2.4.0.jar
 parquet-hadoop/1.10.1//parquet-hadoop-1.10.1.jar
 parquet-jackson/1.10.1//parquet-jackson-1.10.1.jar
 protobuf-java/2.5.0//protobuf-java-2.5.0.jar
-py4j/0.10.8.1//py4j-0.10.8.1.jar
+py4j/0.10.9//py4j-0.10.9.jar
 pyrolite/4.30//pyrolite-4.30.jar
 re2j/1.1//re2j-1.1.jar
 scala-collection-compat_2.12/2.1.1//scala-collection-compat_2.12-2.1.1.jar
diff --git a/python/README.md b/python/README.md
index 430efe5d724db..7e2cd1396aaf1 100644
--- a/python/README.md
+++ b/python/README.md
@@ -29,4 +29,4 @@ The Python packaging for Spark is not intended to replace all of the other use c
 
 ## Python Requirements
 
-At its core PySpark depends on Py4J (currently version 0.10.8.1), but some additional sub-packages have their own extra requirements for some features (including numpy, pandas, and pyarrow).
+At its core PySpark depends on Py4J, but some additional sub-packages have their own extra requirements for some features (including numpy, pandas, and pyarrow).
diff --git a/python/docs/Makefile b/python/docs/Makefile
index 66d3fc425daa3..b11e3589065c3 100644
--- a/python/docs/Makefile
+++ b/python/docs/Makefile
@@ -7,7 +7,7 @@ SPHINXBUILD   ?= sphinx-build
 PAPER         ?=
 BUILDDIR      ?= _build
 
-export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.8.1-src.zip)
+export PYTHONPATH=$(realpath ..):$(realpath ../lib/py4j-0.10.9-src.zip)
 
 # User-friendly check for sphinx-build
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
diff --git a/python/docs/make2.bat b/python/docs/make2.bat
index 742df373166da..2d7d929755a04 100644
--- a/python/docs/make2.bat
+++ b/python/docs/make2.bat
@@ -2,7 +2,7 @@
 
 REM Command file for Sphinx documentation
 
-set PYTHONPATH=..;..\lib\py4j-0.10.8.1-src.zip
+set PYTHONPATH=..;..\lib\py4j-0.10.9-src.zip
 
 
 if "%SPHINXBUILD%" == "" (
diff --git a/python/lib/py4j-0.10.8.1-src.zip b/python/lib/py4j-0.10.8.1-src.zip
deleted file mode 100644
index 1b5dede8f2d627760e571cee6ac08b472eb31d34..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 41255
zcmZs?Q*bXpv+n(mZQHhO+jg>I+qUhj*tTukHdbsW-@8xMKKoRC`(iHUX6m_^>7MT2
z(~2@6pr`-<01`kM+bl_H2(m8spPl`G79s!_;NZz>#b97yX=mwTU_kHSsiq1A0I9AO
z(AYNE&~f#E0RVwM0|5a4=L8jX$a2<uWk^ZcS$SE76KR^sIkkv*1sXakI_cPWjqz!^
zaRzE7Nm=UIIVw7-*zh}|$tS}`(+aVX%y&;+-TQk*a>X(_Sxp3~6c~Ww|CJ>zK_&MY
z5gu!T2>|d}0syH0Ez88l($vny+0@C+)ak#a;fkc{6#xIFUGrHxZ;2=EJ=2(7AON{r
zs&{YHY#OK5N>ePvxtUIoP28U}QbCf2BNHR;1LvFS|NHrTHZfylP)xBgzZKC$3kNmx
zGW*!!<14~AJRvZM@@yHO+KTk>|Nc1HyPFs=Wc1LXV-H2W)?1D8X`w%sPO~LpZWvFO
z(HcVV(3pblic)Jy2+f#y5t@7;MDd^FH#-dPw&Tzu+BJLZ>Q0$|-f8Wrl1@yosa|Ud
z@gIYLC?f+*;)F3B)@pUk$DV0X9hbf9sqacFO`RIxZ0z&qgMDdhn3CWGKr$9vG^X<I
zYsHRSR))o4$@`K%kii<r{G&CjFP-_PPG4V?b^WQ*f)5-!?*R-EfCtq?34N|sL)tbP
zPGV*17Zu(uh})m<)T`9r5E{rU-i?n-duc_s=+a@y(ITrdY^T;{ta1uTPUqAwqs~f_
zIFnTgkfdr;<U3PS8u!c|nsi+QqrVdTrmWo3$&dWmP<`~?a9WEqETyBhA+5dWxLG)*
z0jaKO>c}yD^#0x#GJ9>&t?a07^M5z}f-xlQnbCevuw~mn7VJMW$kuJW=q|?=ME|w8
zVZSc)qLf(952OH_mRKdZ4<o|G;JQP94D=edjSmFnOs3KDPkHXD1~&#N#a>$SJ7zZn
zcm-6_V)@&%YxQ78CSP0@y_&CGEKE$CJpHTmzPmYm$m+_X5YYGccg*eW>VB&ucz0d<
zeDnCY;qKx0<>~Q2pqq<}E88>3`76uttn5gBu^p4kZrh?SlhI!YjC)YIWV5sz3rg6O
zYud)FRxH2f`fSN_rC8elJGTXm9a(*Fi>@}!6Ha{F>i_=PTual>;l~FNS@X~PxmYVu
zu^47z+BdqmN}UEV+y#!o1M3Zy)LkY0Vyt>yGRztDG3@>95mrNdKMVo_Vmzu5Jv@tI
zZ2mJQmkcC}lp$M*r5|5oudk24<IN$rm!D67o9FxX!t~?d@PS}S2EZ%{_gYLfo;b?O
zAz3$Dy5t_>Z}ACjI1^oMIeghY9rfk^=Nkwe8ib_|fdZmACsa3x^3-&<)N*m$n9JBG
z=vCPF?ji#pST~LJ(W(hNVi%I+9}_N4{g7R`m&zg~`dfHccfy40bxdSl>ux8)xzXzO
z5Njt_?Ai*p_}x}u?@?)ZAw~(2%hxo3?{xr2i<NnopXff1rUI-b3O9)%j8P^>s5ySc
z^y<-z?Gx-LUG$R)d>NAD9=o~NWeq+}xK_teoBo)>cnA_&s3$gCc+z!oN=RggzdVe4
zBnh-q90?BfGn^a5u@>x=qzh_r*Np4ja$eln_M<%AaJ^r<x`1sRIzcZ4gfH!#ZXJwK
zhNBu<D|9Eu{z;JNh}-%P{VX~RWBiO!{<b0(E;qJ(z-q;!DSh+^QGv^Ge^FcU5JIAi
znsWx_bfp?z2q^NZrIM)<qMV9hxVoPe!dw;dR{BPD!i98grB`-PT;R{S;kwB7tVzA1
zmrAiw%R)aoRsfX{@aO*fWrxhxKmD}joh0tY*^jZ1^OAc`BAHr0K!c*qnfp`1dqz#)
z3nR1nu>4v4$c3^^)eMK^eyNZQtV?evH;>uXHX!3=_QHpyFZuEFsn%)}`7IpXB^ZRJ
zO=`}vCD2hri^si-qFz*qE}T^jj?^-2lPePh>5+`uPO2{z@0Amp3z0U)Mo}jogp^2u
zDQw-+1C@_mI;TMXn0X8JxbXYEZqN5sk5hfD8fDpuW0;s@Jdlx)1|f0gEUDyQZ7qA7
zdkB>|?sYA%>ItAHhfcn;yz+BK8}Uw@l<~91@gL?Ncv+gt#`UPoXyPY?#H{kH^4|nW
zUy)^O%LH)DY9Pr}lR7#$8VvaM8EmlB(XFVi9A6u#8%jV$+UOdZr0j!+#)j`2d4?7V
z^dF=SP~qgoSd-L{oQBF%IZ)POt6|(?1ir0})xsJc+`)aQq$8*t!5rxFX{$dBi!Ymz
zsnNX^^<It?L#j@RW!P5cJGsy-Ak}<*;TwdPse&S*wQN#MT(=n8b~YLDxHt!nF~+P{
zVEAf!zMy95&yAHr8`GJPEL?N!^&h&Sn3A?)VijoJKjR9Qz!3|B$f%g;=K2aEK?f$K
zDvdyitO9MW^p>VkR&S|=DFnORLm)tGQ~I{q>b0Yfa<M`?Y!<6+jQ8C{;H^f%1JqUO
zB5uBDER&EL@pjrI^Gd-67V(_AQx9Qxb=58|v1t|gVuumcCqgpZ-Q}T{sX|vRLVEw?
zp&B+UR}nM$&?Cf6X_lo?@lcN0xWGNfXKTI2(x*j;1Lq-4jyg!?=~{#<r7@-vu|y|=
z%&Aj!&*`b5N9Fknop{<xV?@!~9vgy^VSz;Yb;o~Cf@WeLE_}mWPC3F75VhWFH&J0!
zzX-T?A<QyhqL})V>5y{c-fb1VXjk!hv436@PU!f<p@{4nAAo45GIltR$HAuZrpl#D
zH9476=oSCuZDL1&H8xQxbw*!n=1gqa^rE5Nqt(NetHsdhwQLTf#$78RQvZthM9cG<
z@+>>Ero(}XlTq&5Y<M~FSeK>Xhq=Xe=_l79Qlqvl;7wbG%b2iG0zC8cMMCc;;Xy;5
zm+6fqHxA|FrW0VqzEh{AU(`T6&^4me{Jhtaa)@^(R(9BQdtLGaTNCmg(pqAj$UP?t
zbsDowfW}+GA?SQy#vh)n;wn(*dstCeY(|+8IeaGxNT&;}!cB!d0%`vZw_LTUsF;4}
zq6pDv{Hs$deE2JAYjF)0SbyY<-BIAM7|R)CvG{Dd@orML>vk-WCg#rPwxx0Vy!eUt
zlt@BP0^_YmD85j!D2E7MX|FH1Gq;av_S(mmh{Y0!mi<f?;n##X18QNwEDXae4^J6P
zi5KCT(PU#vDh+c~$SApm6k5R3Z6}A@YL6`IPb`ExKEo}dK~i!R;E0r!o8ck>Th`~F
zig>e!{e}5Y{^~raKe(rmr~|41rP$Yc5X@LfAu)L(0Ez@x^&{s@9E;ftMkra{b;oYH
z3`5KigZiyZJpkul+f=g_?!W4>ypWaIlb}txR(JGi*}!dhU69SP*?{>d01JI$DxXms
z3p~8IZjdC0JCnrV%bY?F;iL{S_zk<To|I}f3b#w6^TE1`n2~~GE|T;atp5_HW8VXS
zS68uK!KyC1M=WygL7Xdi`ltLQx)5k2-CZ&k2i>E%Fa=@L;JRB`V}}}QXNd-_2~Ha`
z$0<X7B++4x;%Hbg{&(}Jsq@3#zAjZwg|IcNH;T(-*IL387qo3#vJSKYJbMDIg%P{h
z&_R^bycZYA2MC+sR8r)rbXt*tj(g=zqo)|ED36&p;1b^*Elh-E*X$LMtTwR$7Vu?6
zj)Ah|uTw1;q_RTpwMZm_=>8Id&@;L$h5P43eS7}}PPowK9S-8c4jhWOsd_6`4U+h8
zL8xHRIcqtM-Sz>c&LI`L>;zEjcSOdLDR*Xg-B;#8L~YmK)b}J>YT~>sg|5VJAo0U=
zxV8~gn%W&X^u&{(eak>Mp_<kyl?a-R)@tE^laz!(4$WUDl{cZ8)=hx8rh8V8bI*co
z*WQ$kd`C#u0)(NJX@%Y?o9(UP;f(g|V|BpoN#XpoFoIAbivx?vg54T(ZSiZI8|A!_
z#9NgZPcO!AvB#L^Iewq}oaUHXseZ_jH>6{Smonj>TRKNTB4_rVUcb}CXcTgUqz!a%
z!UK6N*-ZYr8W}6v!h|9A%@pfcBh6#uMB1XfrWqKl!q@b>?Hx8`6yClI&sjzjs~B8X
zy|)-&NL!284cp9)XEAbI#fuS_utNadbFQ^?SH8C>&~85p4;N69d8l8pL-bWzJ<{<+
z=iS`vG%`Jy5po#yhQS#|Rv^xElxUgRxj82sK^qERNPD`Ity7P6^7tR$NVA!y`7KW(
zt#q#l$kC?=wgWvE7RIrc#NtE@X67-0H136GlCI^5xsZmMg1FBwh+H4<e!<xu3cJSu
zgnW^^>FdEDGb;AVa)3&Xhsr%7+)Ovc-<TJhuBHTS@2kcTJDgoN7qBbtB)l_FtV}jV
zw!bLR`n{=`?yX1FPoAV_v37x3AVJ6Nz~ArUo{z(5;2Dh~`cmL5+nT!j%lHrE;Dhj7
z+|^0$XeRC3vew8JTzT*&)7*g*X;bX?OuSc;n5&vkj;;vz-1&Ha8wRQO3D_cU0%bOe
zz;%~!+ws3E@j(nh4X$yptoLF3ImSbH-|;=#^$Nq;`t?GO;82UCb#+JMj?1B2nZD9B
zG3ymeqhnI`Khb!EjN}P2fd%nuB1NDHW=Xo2bS6yXep=Ia>0=qFfX$)Ki6RTegi(gu
z5lk3){(9v~O)?Kl9q#|=cA+WpYPUEdwA4Cnip>ztHvxsD3e2kJw4$4;f-NX*t#XCj
zGAoOPyc<3Ss9*^NC43Rm7hDzvvEXT2=Xj(cAt;{GX1UUr`3$b~A*SyW`>SR@+Uh0Z
zTv(WTHwi5MHnU}mwfI2wayo#Qp<VsumtC3{k4EUH)~>Q7k9+fc`vrR4Cu#R5s^zRw
zly-5qqW&bD)z#?Gh!%8dWNVGCbf%7NSc*15OR7ru-L9BObt0yxI|-Jl^YtLvb;pPv
z9$|H9^o{?seB==`nA=-b#H#g&UmKd}_&_rPPkxUiGCL1*$NeA*MM~3M+dlQW@Ro#v
zu7y6Bb3e4b-!KfD1cY+#5Gd$E?5}?_M@5%v&IMfgV4?1f@A|C?_tIm|>N;((7!Qb}
zY1faLBjD@H+#m_IM^|};3OuB;mHC)m=3>YA{*hinaE5Ny)&8(Y&Z4VIqZ_^p#)}R`
zH2F`Rlakb;ox3FK<jk=feycN=+F@wEBOCJgn=O1)JhH~YOEg9ov7riVj8;mbV(M%u
z=I$W!C8oJdlE$^I1V2|PMHzN8^N^2xN{zMCD`bZ|xS;p&3h(kT9mZh~1Kx>}bTWF$
zzeZt=`&Y#8Ornn?D0WwPV{_H++Dc+bx3M?YS%R2Rk?gg?EucwOO%Fa;#_#O0<sa^X
z@L!Ac57mEx-=RpID$F|HeJtB1>8j!nnS|jzt$*8t*G6B4FVa1#c1;J}vw}m|gvzj=
z!P`Umb`csrqq}Xit2>JIt(Y0iwSu*5TrN7dj}>oUuuQGBywu1~{?vc5SoaAzEs>QL
zwd+}Z=N^7VeLT`=vxP^YPz;P*F4S9brJt9U3r2kdAm~zd%+uBFw-q&uDjTnR06l?N
zCh2u$#&lyi@dm^T`18TRWHmy^W}b;{IvDlz<6JTFGJP-P!J+JYcyGmj+LzB_hGKKs
zGlZi!bOS1LLo3Wy6O87@^H<%}_f%`h^$p@YdyZlIh`|VaTjIZ;EY#5w#h<oqGtoVl
zq-js@8Z3I0D4{9RI5uU2MRA?5Bw`3s2njX|@yWuH7DY4?dW1|4(s{Ty<8m|J7rqKb
z$bSm`XdXHEz5eFrD%7<|F!y$H{z!Pa`f~AkUE(K9r8~YjK;s-%;P5ftgs5~KD@oor
zTr-#GiC8x3T#q5RwyM`keZ}Oj_!lGWWDThVa$~*_X`gW<k)E?Wqzo~Fi^{~oIi>DQ
zk>$SNE;k7kCy#ZuuOV434_Pg9+vI9ze=z5kC}XwdI(H+f7ngGn05kC$yuK-G!e6PY
znjb0*q}V^fe|!~iL)0RW<YK-^MX3oLe9+S1`rTc&{eI3ptZCfM8OoLg6Z=S;H;qZ$
z*7YgGGwk;AaSKSz#qY{XI2xVj1uX4~$*Hp=ERrrbw#g8j${QB$7HIMR{4I<t$^??W
z&>B<(rb@d{ZV5T(=Z})P+;;^BZcseygw<N7S>sQCFa=*_A2UIkyg8>WRrIDM@ZYdB
z=APIxe}%+tpxN}EmQ}fwl)vpjtUTQ>q;Vdcjg0#8DuYsMHt*jg;Bk!jn@8+%It`KU
zP2SZ=9;9_FBenGn9!E`GI^eQGy8<G#$DVFRb|VzK`&X5p_t3}<8QRg^SBQC@C=$Q5
z2vYs|Kq>{krz+G4{|gj+wX=-4VSLXX7m+`|O`&l{d?&l;@*=#50FR15WL$}^AJS&?
zWU3F&cy(yawQJ=F#!>lV`kbIsJeE6CW#$Zr*S-bJrX!o#hi}M($UJ-Nk0G~Hd=_+q
z5GPc;>Zka)stq4xYU+Ex3z4M0`Xr}IVW8_wU|TS<=tAdapBagseRWTuIv&QwYh=-h
zfx17w*Q*?Cul;(H=Yg4AISRUcY5rA9lr6G3#O`7bw~2gTPHfa=bTW+N@&g`AbGjJV
z2x0#+t~JAb5?0iG&^;_}dhys_{Q)gmP)rJw)F;xgZzm)ZPd&n&sS*Ar<B5{C)KarZ
zUSOZ_hiWZ3G%JmksIUev2gV1k$*})KtWV}-MP`#D!sEoGjf<0$5mPV6Tz$DY1ve^~
zumBV&Izw~UnkrSBRN-C6#B{<P0XhBN53cTxp^|+h5Y-Bx_@GJda7O)z+X|a}r7^Xd
zCaffP3ykJSf4MF4YVtcxFREj!9Xpb{<w4c=UbP+@ynkX}VO_l*$|2k&;190M!#q;p
z!EdI-e8bBY=j`I;n~>Hwj=jN%ATQ;Dk>KBvAyB?tJ%0Y4xxIh&_3(N!;l0TRD`dm&
zm#QY_gsOgUl$zNDRCkJED>oP{d)MPtGSuJ;9V|3-V#{{gj*6OCT#cvMdG9yes^l~f
zw++E^H+I22&$_~qb8jS{v?V@KRB&z(yKD`^57+<g;_CfSAi}4)xv}&0`Yj3{y<OYl
z_eD@Ky~iXHA1x^^*mH(D67CWZfcTfv*WE<7@Q)71D>Gc}BJ8HWmRtwiv?BbRg}xE?
zHO!_JT^xREF>-#yz~glWnScKb!f=y|ga&uU1S<esG&I?a)lK?QDlxu2$qThEZ=+A=
zm~8t(`JAuJmfQ8@<IBnfWa(auAH5pNfknJ})hl$;nu<*<?0keiKj>*%>z^8~w^>|y
z+j5tKc5+VlyNEv0+J}Lc;U3CAYt4{=uJxtc7(D8P)br^#oRUMcpB7p~^D>=XgNiM{
zAxAipT4AHa?}GEUr9L5VJF4PDX^ux6bi1fUuUW9;V0ujFklZolLldYcOSrbO$tLqz
z(dUpdgJl!`>T5nhAnQ4i<!>kR4$MNW@AZE&qAvW+9Ao3LFy=Ov?HuYg48<inXGqi1
zpXbZ;IH!pK{Y*ICm0hn>T(QFYHT=`$)%W6`0`K@4{J!7)q>}sQ8D@SGT#v<=+uErL
z-(kze1Xgt89eD5W3s%ayxzSQL8Xm#jV8x`~VoW4+5^EWr#rw}b&nbJ)oF~>`-slT?
z_AuQQHE*BPFE{v&H1FgQcgU~oPKGN)!5Q2(y$)iV1$V%l@JS1PUtfo86BcZX6Z=S8
z!X!70I*U)wUgH=xh^yRGyzL4fhr4zieC^*C^HxiXN^t;L(<;`9&^pgjYx`cGFKzsU
z1UeI&KJgUVD$uqgtLsJ+;uhNv^wb^4K$u&JW*P8C*=iIc!&>(0j=u6F-i5rxXzF5w
z=Qwd%7ovY-rsfxw%hMN+PqO{8RMiqngOSp&*>Y?*&Gm=|%!gU3ddpURlG1JKoul+u
z1P=m9)k#%ogSKF+*4FA@;qOStmq*DN^Ng>=z>)qd@hs&M?}C}0`}M)}n+E?s7Lj~q
zdkJKC>9Ox=4D1VN|A>mX39iZdL`EOu7W59plX|`b33@uPD(EkG4sqeg9>g_{$jH}S
zX0Ih4tm$?08*#hhbMPIf(E1r$HYT+XriOK%($UD=p$kwmQ3Rv5wD8`n`X$E{{w_B=
zwfGh0GtDWE30VA^d-)ya=~PlRKHc8QgKq&50=&0c;5w%n01M?JmQdeLIJ_n|J4BEG
zJUT47YV6@!?I$aiqtcZ{DrKL+Jlc$Mk(rOv-y6j44&#+dF}3Wh9LX|Rm!+Lpk#rCw
z#@`=*--S|V;l<xe&<{ufh2zxDR<!hywo-=!GQQ(d5@*51?TJKDczD<%|FnJ_xtWV@
z;4+fz1ugrru1rGSUZh#?BFp>g4^7|4^txc;v+uvh!q7)+TPe81dsD`*bkWyh2cmn<
zW$^hfub<jFbT?oWK8X|h;pN0S-~2+j^G14=?qJ>tt1$OCCU_8k9NhQqy?z2_P1iqc
zUprt`m^YxtXJKG-oo<W`s4jg?K6vCA3DVEwm><bX2J-mx%PJn*&}gO0v*i$i1r7{f
zAG5tvf&H$RYop2d1Uvr-3aFb+;`8b@dhZ`5f>jkuAvAU-?s%zPu4V|1@ZW+)`e#bx
z_E~+qb=cJV#6P=qh%kt3QkZRvi%X_LAkwovBed}n04+qF-swj33<dv!{13306Qh!Y
zQ-bq40|o%Fpa1~){|hYaZ5@nU{u@=O3#RJ)FBGLkUB+&U5pm}d6~vSTq85Fkl<}XK
zVzC}7rFOn|V3IhKN@R)@bt3&L%#$%*(ua0!GHN*n`upij_z*>W9bqyT1_F<T?EQ@|
zVwlmLk~$j;?+F<P9%}43Y`U-r5gKa{2ClQ@eSa<|5LK4BD9|3aa{{p4A$rF60lVE1
z;d(=uVoPurvd&wz-!C$RmO8ym!5=UwW!6%q4ATB8ZE6z~MX?M;JJsrC9m9$=QxnGf
z7{c>R-E(Xf78}<xUs-3x@vF?R1OZ$`P=#_<Jko>0g!(r4d)&f)u<5{d<9L@r1cxq|
z0DdlXK{Yk*Ne=u5f>0L#kmr?Lw&AQ=`n%N$Nj;lVS7}>A)06DTFvud3c3%Ju;=_1%
zO^VAyk3TIe)lzxv;VkbKt)%Z~BvW-76L+vI)#}bV=8%~aU-@n_I*r7n`IcQucnmK{
zXYa)>;)N?od#N{v>>tcT_6Sz{_S(PD_6%?S#Bv9g^zKpVO|i>b^3hlmp^=L%U8-CI
zWq))wu$;K>BM!Qp1J|NIy@TPn_j@ZZTz9-l-FU*gz9YQ-0+vHH$~%uYMT?sEim#>C
z)H_o4_|qO-=R&P2;?0M-D@~NoAZt)76qY6T%8+vwuC&C7a#aFYHl`3VXE_l#ev4Hk
ze2SC}qV5DjvwM}B;CF$c1;k!1$>o%Lm$&;s&)NKN+4wEqIhP)EKgTj(c5dGTEzQcu
z{WRy<`12big?E^EVK|m_=AG?{Gx-<S3r;elN@&9k@0pj6Tc7)wpL%8MHV|{7uF_Qv
z)tPGs=5z!~h=0#fmU49AP3*zWFh22jQcHYRkG}L2B{%yUVXZ;9hN7+n@-hntB!Q8(
zheeVuI+KR6lSmge(RJF%+4bh0TOPCb3<>NJ3`+Wu*ebIrVX_bq*Bax55f;f1Xl?Xj
z4eGlzqJ9`?g_m6tkmluLsk7W;!T_nAkz<nz*6`xyi@^##vWDMB4T^n`KmQNk`!DRM
z@=+AB1OfmoA_4$N|NUIdEbWYJEWQ2*?)?{c#Q$I5p2kk}mN=sSngNq8ffa^MG1)vZ
z=egdL5TPYe4}a>Rjtq&lbCq60SJY9+*X`f$1hS2XosP9|;X(>;*1W#=8Q2f-v^_c^
z4Y2fV*q`nmt)FAkiIEYp2XV9-!FseNMZ*a)Y(s}3GL$b8t8p+EaMmnwsiC!EZECq+
zj&055W5f(vNtn!H0jUiv@(Bj)%n{&}5DwX)c$l|?kynN!Qj?mRl-d0(OPzv#FNnTI
zIO)+?g^K0Mu$4+1+bYKvV0{j09SgEDWCt=%Ftb70fSOq3nh~{m$)J)N%6gg>7N1Y;
zlh=FGChvFp8MPh7j(fI_D!C-mf<+zsZwMO4_yO6PJtMm_E#N1`FbxNe0yl_lp*AF|
z8PVYyHqZby?{35BtKF^z%|h`^l-$1{AS}7Y(ypE@fs<)lQN>&+#xoC}o>?9QZD?Y!
zf$i~Xt-!5l(t|lM8O>&>InND(smq*@sh;Ni;<K*1(50X2<gSi)At&orx*364_S7)Q
zd&w)<z8|0v3>9LScs|zPxfK!KHni35@<ZM%)i<gp3ml>XU;Y2Y_XppjB6UOid-tv9
zCN2Yyt%RUUm1`&ksb!4nfzbcqi^xrGW22hFx;HKbn#qVT_;3d8%|J<4L3tn!b4mFA
z@jSt+2-CFgNUAt$$AIJZhsxvk&TQ*~&G|bMygS!B#<sYQsxKG}T}`R(4bikc1Hypb
zn}e2$vVo2D$*$ZV!-;4?HbIYmnOkj?wL&`Ov*&q$y~|M@UK!!bEy&3Cv}}s{g*UXb
zz=B8K{fh@lF8eRw!rCc5`33Q3B)Si(oqiK!+_HMbn-}~Rk-3ZwJjSOY=d<SYvC$4N
zCp<?tXFKvZ2@_s}7-aTG)j+5#QJ4=Uy;*5yn(2r`8yW#=^4eji$QYPbNIKcuQ-lXY
z!BBL<-Imk_d>}qy<8x|XO=_t`r(4GZBG}Rp`*mgLHY8=&DL7gwRXS=1Q&gMGZUm*E
z!S3oN!3v~|M8feRt_6<uKNVhW#1<s7K>MLZtq8%)*H!JElj7(3jObi9dF-zu7}Sr6
zhy>-D-ek@q?3fMv3%fvT6jb7Z%}rY7(P;Cks1rm;RQTh7v*%A@4sXIlzJWpf@$b20
z+~;Ks!ko#|Wm($f6JHrJtXxLF>3m%9hDA$(+JD+vlMFx9ehX!{m%iRty(4&B5AH9%
z*~33iXx_&`cm{R;?j5;qE%+Up6z7a1A!YUbC3&8crb~G+&48XJ`3iqDqaTFvBd8{b
zBhTRr0hJwRc1!L-S;NHKFg}M9%dFr3Cp!FBGL_On*U*6bk3~To0Kok3=wM~!W@KPu
zZ)0O>;$mrU=luVs)&DI>=-S$EaU%cR7z{Y~N$RXgc77fp+Mth{?Q}gQu}18Gq3;Nn
z*mA4W$nY!4AJV+^?M>f_73<?t$Zydqf{58VnQ`V|@#heg@R~6Ac?^(mu*%oLuWv0>
zVf+rK$+vA?=a??35o^*PGs~FrhTS&NLL5G6AA>R0uYF|=-I5|D6VEEMOSPv6Hxip*
zTpGvGwKf&;eA=+alj%?%+|lV6b11BPsMV7msW6!yv1LbL8IXzfEnVQA1!2rXKj=TE
z^!RL;FucCJa{mSC4N`gld7+t$?vNZdp7A@<AU<g*rkZ}R87Q=9rB?5VAV&YM&wsrd
z_E3sg%5dnoC$V=-ylGlBG!j)*+#zjCdKlX|<6BM*H2e$a(G=Nw!C2T~A5EgX=-ksf
zc)AZIG%4L+96I@$a(uJy?^A=o1iFm=8%aUfojnkD=^^^tneId3_yBkVZXL|PV@}-)
zV}oweivO^i)-fOZDRN&xTmj3hgoBO?7<tcuH4Ap<!^|7WbgRVP+^{jN;9x3cVJM(N
zWyLQ>pVk!X#WWWUD<j&mqL*iAgWuj5<1$Yl;emI_q<<TO_pi9jU9r^!7$;q9t!E=}
z*Xbd3nqf4hd^!zj86hZ~RL1p;dgvOS@SEa(;seF`4e?a4Q|kc`RDdCnx%8;qJ~?50
zO#Bn(yslW+8vH=ybWw*Pf$lqjVF9U>8wd?hXR|0xb+a0`DOU(}HKu$JKWwy3MTp?E
zuOaaDkliUUnUMM*sJc+rfU#7aXl=F_0SJCc|72iB{8D>Fp<vXXRTHqeg<>X7rBA*r
z(gw+yr+byn;y$)EQ}IE+5G3ppAEI@l%ft9uGLKPlESxdMBhuW+2`}Jif$V{10FW-r
z5ib+J3=YW=GsYTYF3Gi3;kwugme&g=thB#*6m_xWMiS<8HF6LYR7rh&#hp0^7dT|x
z8&I}D#tWxfqR~@=HWCgNIrRstvL>^%11VrS1(+yzz+ZM-9`t}{iac3u7I>lW-)nXE
zqfL9(Gz7rKFgCCN4yZ2!gF6lvGt}6|>L@thSlwAG3yl+FrCK$mL`<6Eveq5#=XL)P
znG-m^fk@bEdET1HnOL5!c|<R<&KD04qpb=%1vsTZnzd3=Wa_;XOrJP1r5my|)_^w$
zh!RRy@VPCTef*uSv?lD#8p1w6bZki}sb9aM1&}gRyqO%h6qOMKIqDp!SQRAt`Hcet
zq8RvhvPbmMy`TiAb5aQ8#?386!FZ?JPs%J{hrX-2=@s*uFb>r?T#wknOOGg0l0#U+
zVDuLyaEuvk0=y-X3M2YTdzrNtrTn8Gh)V?V1+M>1WJq1(jx$v!V>;nSqWUt8=0=eu
z2+lK-6+hdq>P~4882(i$|I_of4&=r2x6Y2>VH}JB_!yo)w7qL6{gUzNoN40=6kb@h
z(J}#>y1H`x`wOf1S1syB*yUNk%n=(==bQrAVm=HkszYn)FofP*ZLrR}S~_C#MGQ+U
zxG8#pV8MZLP!Gi-*!^~hp6E&Yg(x<W5NmaZ_LAXEqJCoTN`f4@pNts3OB)t4LY(mO
z$%3^$sh@HSng#u9Rdb+hNxVAX#V!@S5i*xUSw$gMpKeebeC4DN3AUUE2MH7nuJW8I
z_9WyC=baT%mCMzch2$+hYg?|<vMeZ7t}CN!AkaCx^&XvV+-~Lh3X^9Yjrtb6o6fP;
zs))R}O76BH&~CIJbY-<u)ZKO!sWJ_e%CH*_3fMk&Xo&mD@*Z7?H-Tqxaz2<#C3RIW
zPB8WyZ5(cd*snln!3mGCU>}D0MRimqaKb@iB)l+x@DFW$d%Sr=KuH`MU+J`ANtLmk
zvRgVni5K&k;soc)a)tu`0Z_1WE27Mz42$t-Ob)t91#DOYg%4S(G9zL&q*Xd#Y$EF(
z>cne`SAST{2hJ2DMZIHxQa9|0d`ckkz!Q2^lL3tMX;a&P0J{gQUULyjj2o@Z&TpuU
zK%<fKq><BS#HghU1|y9i|8J}khr{doTsS456+mhYYjExs0GfgNjlYo>Pga6NS*8YT
zdDl%}uG*Pu3lc)r%p)X2*?n2riHoBN&IA#dW+LmFEeA;@WYU@vl?_?uVWFO8?;@QA
zTf^S~(562Blt@pP@E0AkpPh}H^^cy7p9Q~q&&yq<-{}J&h1vV>NZLPcQx>%cLP7~M
zn!V07>!=*P8(rg)%`a74er;3o1r)?WV@RZv$V{ksk~mpI3ebNl_aka+=Kt0Osyri7
zC^BzJbN4|<{6SWjc7t3ECs=e`wi_kL&BaB)ANzttrM_a6QS<s}=$sH5es_173$ubi
zcc(Q9rsM>)RQ|_smI|l56@>);{9M;hthOEjZKafUAY{$^HZUx4ZX&A|yFPT(sf&Q$
z?Ya~uNV1O43aEe^pSA+p591+%rK%;oyjidVgKDKyw94ks0&?lU=84+J$yF6q(wq@S
zR%RGHQnQaupz`{M3hicVUU;lG+10=PW?&fk#W;#c&h(?Ic3Kg~6{M`WUawgX&^V3G
zrncVdD&YS0Lgf8gMbsr(MYksOoph*QDVRzt4?wQ3w2;-7lqa#lZ1s{AN1Yec4}^|K
z<a9O{9Z@0aSS3YcxY$UH@kbna7P57uX@A4xLepQSyjwZ}!*r^U$1aYkTVz%XiwSe8
zuw$VRgmD@`1~ZSYV6HWFLeNtjz^~8&+Tb2fJY88V#|j{h6}TzX5GDW&qf>07bh@!w
zYuY3YFhGsOmfYF}xNvj&y9j!D^&hB~m+hCGI=82Buk_?R-X7mL;!13<JaWE3y9qi@
zHAwA98`_w_Rn{6tT<-h~#NUdzp$t)?bf%bj{PyT(52lHl|7*Q@zBR9Oz_k)$+QqLS
z4JD9e1h*bo=S1khPNo{ntuPd-r7ooS#}7ea-4X=s#g=)bMiVzwwxqd(DWn#<RYtXx
znyX4SXJ)8hIQ#HME$=BjIPS3b;}vjIbnD9cnyQeCuj|p3l~H?J@;Y16cFEEW-YB(-
zG0m1rGc~||()N6%YcC%A?#Z4l)k(0`Bo~n2H&lwdi_B+IcZiGWn0)$?U;}LkU<9*c
za5e4&<d)abfeB_(ywBF@wQ29%R3cPE;aG#FOYe<)rPn?Swa{kdrl}FgZ#|C?ZM~{+
znQmjQj`mu2Bk`L9MMjNBNA>vGwPGC7W>x1n4+(W4WA5kk9YS7C@(QqKVZw^%LWCq~
zhlYop?(BYwhr*e6b&{4#8L$bf9oTE9^XvOgrpruw-0&c&DpNIVv`YI7{SApd)C+9r
z{M$-&v{b$3AdAy}l`cZl<NDslIbA2F%dZxf5{#Q>ZTjnb$w+s(P&?cO?j;3s-8N~z
z$-xd|U%A{$?|$DTHc>Ufp3<{vnJr`RWux(Pcy>U>=qMnJ6_aNMvLxG^J`{U$_Z@$w
zpb%C>Cv8I91o5&rciES>CNS&yy{3tpO#t(W9A;lmcGslm7N4iB4+kR&wL?GPmq2oD
zp|B9y0->O|)RoIagw4>BBVQ9xogw1&!8z{nb^7&Y(G0^{7_3wf=4TK=?l&o!ZqM9A
z;2fpA;L);1jx=1A1T(ZrxP#WFN!InYbIs-L-C0#+jgzHZ!PT2v9?hY(?A}ml<8U7*
zZ>bkUp)e|j@ESwV-}%SZ?@%7C1x`WepGe4OYWlO>;mu^*n7C&PrK1TTwJLWFtCOOk
z$AZ>Tm}yW5C&nj*G*Cn_O4Dp=aWVW%OS^$Zn=uWRhdoOJPo)WC*G!*nK}5llBTuTG
z;xZfmlC+bf=|iMygPKnJ&3#LEFIV8hOC9>hmr%pZ^ZL8o`$l(&BfTr@{{C)O9(Ipv
zs>GvzTN~um>F#sisa>VyqvW*n;NrpDCSg)7j^Sq3bHEOCCvI^DXjf-Z#?7)Io^vL-
ztrBguh=g2m4#^<LqfHNf{xY2_yI?oxh@H+=WGzN=SIWV%Yz5Ro2Rl*?vQiRU%uze5
z!qUVcuC9|brdFY4c<U};Cu{7{w$2~o#dNp9(oemcx=*Gq$mZT_S`l^Z^m>EPc`uQy
z_@u(KW7S5RK9VK>=y{soPv0Y755swk?J+@G=M%fpNyJ}mSn79AcebEw%Wp+iK2aUG
zN96<$CezxPv0!l&0eDD3DzvmFa|2P!`LO55?!{*`?##FlU&`JuTywTbtx)|06N8c^
zZ7f*92V*<21YZRf`bVmm(W}<7x_>if`tx)(TF*Io7mf~-v#v0k%Q!FID{9_fY^P)u
zw58tBz}Vm7X}$~YJi7I*0n}Ygu`opQ0|FH3VDzt2h$?%`?AwEqy6H}KLZsC71i8jm
z)6mUOY`irJdA;R@@T*>!K?10pZUgs8`{?1@Xff;{_)u_!`+Q^_HI4GeiiI!Xwaezu
zx!`^vX@EIR!C9MJ2){fbe%p^6h?;ph>A}@GG4{73(|JP(!>G#c&vYzvxF=7?_d<?C
zS0L3+$L)Btz8k(aAFcad!^)N-4}!eC(~h&nwv`+H+aK0;UhJNZ?$`5|czqk?gWvz@
zdYTrh%Beg*2X(6h09!6V|6|ksKZTmPk&CIjk>`IGY4mqfI{!<NcBO0kUpDQZSpy~+
zgOwn7(5bj4_bF=M1hP$~!nRaO9<DkVTx1bsbVx=3WoGun&&y@HCx|4qbdJo;5EoVC
ze_YyLUY*{#H@OQa4X-W=vQeZn=Dv<BK0U7nZ^cX3ZgWw8QmK?Mh-a+TCZ*_Zmb~br
z?j_hNXgsKk-{g0t?%C-sQ>TYBbEcPcBQ3RPL#}jdFaJZg_r(9s>S&cZ<y1&7J8MOY
zac8d?Crwz*k}Nxu^?G}gS8AtuDP~5i<(4;Qp_cpuWxeXq1*{#GkKd~5mocg;M|78q
zRgjMu7d(Jw?nxC>d&nA^#NeTJPcQ1E)_C_HDOLQbK?YKdUlH}ei^BN)^uKlaZ~|+h
z{yaoW_**4Qw|b~Kt4hBFU9ws*F?R2wRws=%Myy8*V`!LKMrev=PBqC~B}_D?mRMtj
z<{S<GnuDqZei*u;T54VBr(F^ElZ4iUHNELWM}5<(OEEI+q6sWeVyQ$`Y~F{KtaH-4
zass`J6l%q$pEe?@=C(a$T=Af&o2v_N1F<#EueBA=b{>H}H9?0`eMTs}Oe111i6HFg
z3N4!<|0=zkX0^$*<nhiF)S?-reGV=p(ch<h%7PZvQT<0Rok5r>Efx4B&<7Gb01x)v
zzj|5vnb&KHHC^k2L5&@&BTeObv)gH=263v8-jFS%`mIS~q1XD(qr>lgC!aQ&xj4SO
zXm`z7Z4cvwPN3l$K`2|%lL9$?+nY<pHAb8EkyJmZGW4lfHDArshoUL<(5)n^1g%_E
zxBA}+^k)ED${iu<3h>8Bq;L;RXeGH*#i8y_F#Kt!yY`5R))cM!G^Nj4p8ESL{5WV!
z1V%n6@)~G5%!;OORyo=1VE4mXzO*@^WtH~Y(~36&uZG3IhcE6WgyZ&w%e*X&WA9|6
zy%26DHHa-(;?M3SigRQZbc^IAz{L+c35OEp8~&FrP_`@?<gdpp-iT1JO`Qqk5U;`k
z3GXGb|BgOs@-XWol><g$bCLZM5-L${AV){~rCr^Ur|aX#^FDPi|NAY_lUK*@YoBWz
zL>+>?eE9vr{r$@R2R6QYUf<iC1;LN^Zi_twLv7gIzYz`t|L+#>x1+nQ=*PE*e}~}&
z-{=f@3q?f(Cye;HIdJrL`9l{63;SEqOGg7}aYZ^EN^$TC3I=%iz61scwY$K1GdJ@m
z*|R6{41LjceF7aZ@$~Wv5E4Vl(ay$Rm(lcX1o}JsKK?hgYAbMJ?y&+^A4;{VuPn4K
zCDNMi%Lh5WhPo>wPEdAlf>p4zNSs<fu(fm$5Wp`WIElM9K_r=`=fPfwSrJeYu+2Ia
zUi<rdC0L4@bm<#e>^TeLEx55EwB|JH!u>afYm(0EVh@qXBdQrdaW>Y3?4ZE?azXs1
z#7?D|r-O<S7V+!cN{p*}u7#qvzq_In-aZ4YYW!5{kdoAYGAefFmSkOTOtp{PSVAcl
zQO4A<6vx6#1sHFx+U7o{)|TmxTe<ES<F!P{U%^n`m1$f`U(aO!Kxt0Pn)OyDAM*b;
zZj3Y8w1+SR<_LP}tXv^vXeq7q-7Y6US6=6FR>0exfCfT_p+GM5v*f0ZSYhq$4(+Su
zyStyWj?)0Pxr_Ue2sg)(jhJl7$gvkq_JNHCal_c`Bgv=HEaA?mi1|Z>z#v6H(yiDs
zNl?4_J)NESj}alfK$g>vT?A2F){(K|FpcDZiX=Kh?hYC1g-%`94v=y|t|J+9KZ?ok
zS#Mt&L`wdhRO|Ycwo+Fuuj;dsI4CX2&QhPB?|ZtquyNwTA3hH7gn&x`s1G?bFgVil
zc(B=t6PBBsyJ>(ks-#B(Tm@6q440#Vz$yM&g9R0&3Adq+Byr?Yff?Ri@XrYOQ`0TM
zd=io}=@|@b1ZQ6Y9pWvy7yPj!9#m5=n}T%>YYxT=Kuh6D#GwH?9lOU%s#5F@OC_vf
zk{1*|^)e--F9sko$$s73{aN?^B4+y&a2b>raMItJum1J%amd=+%lYBQ2)jeVv7*R(
z>p2*<hirze&`dY%6yz<hTLuHROJ?C`J7M6CzAbq!Zs8M_W7fm9A{tmJHxD0lL*gOm
z@xr@#8y34k+*1l<Fo|3o-o;F)3ys=n49kD!n4jdui|x<OwfyjO8hVKd9f4(qA|}WQ
zq8-Z*e%KpKFG~&bI13MJEP&tL>Dk0If-LL>Z4Iv<456s~qdmeGB(H(K0;_a4_nEh_
z$C=i@N17kFFCpYC(?WT(4wqSec6OH6^=ku502HA$?lq<BrK3BD5rY%CI=+L$c$sJ5
zgYM<FP{d^|nYnSiP8nnXB*iPL?kygt-fu_#B#C>81z2!zoHmLC{x>EgcaI7YH89pC
z=96CesVET#P?I*Nq<D(q$RRtY1}y>ZCmKs$QI=b@ghs}y8yR;cC<w`iKmz&zzJSTG
z17l5jvG^vl>SjE81+#NJ#Zd;xLi2!|&2bi|@?NpbK<Jq6!A+rNv<QkdQ5HvvS8>l2
zszls-k9Tq;w{}K}w>3xPzHHyY5#*aJcqElTLc!DSDQ)|R)wrSqXDMxx1kv4GL)p+s
zV3?bSpN-wWA8ZKvoam2Im}7>Gu5r|9+vX5=04;%jxcz3jA861VA$rJI;c#K|?gPrR
zmFN-b)>He%ofL>Xow<>O5tG%Ro^Es72%L<;K3kmVfK*_0Kt&w&kE8rQT(FOuSLKa<
zqXj0J^(Vb<4$_P=iLI#01N9BoD*<JxQGLeB(A<TV=3pDHOC=`W!yvZ;^^=2`DiPuH
zB!&ie&`7+O*h9}wG}Dj5z{`<aJ1>3>1hF_$fM-IvrN0Jzfn?<zI0NL-dqweLSz8QJ
z+J%lC-^<+qhvdY1<3iY>7ekJyLh~;HrpV48Sl6*Qd(3>TZ(tZ45o~>Y;{mtLQD{)>
ze4b$?mFgZk=Fh*wN;9Sv5#hE;7BT0Vxz&@%tR8i$nMtWuFJEBlRct;qE3Dt3u)Mq<
zC=IgdF*imS#A<#)7I$W_OkdKxzdFso%*&I0W1+yQdsMiW`x}+ic?F~9St$T-00|*o
z=-zsTy#Z96fSZ3>iHRZ1P!2&Z*}lMQ^Imm^mH-U~-P$f<RCNuLbapF;<=;b^Hd%Lj
z1oq(vjoFXAxxzj3SvBn{0pr;|p%VYlRDtq6QN$&_V`ifzGJV`ELtTZTa)o?gwSc^y
znEJEE``Ft-Tg<EpHU3PT2C#NAysIO6okcIAtYW<_KM^XC07bm+-}*qRdxNA(OcbNM
z#jHfK(n=w5s~~SZ;Eb1m)>K>H@32bRj)MQp3wH5-r$^0v1{-y3$_Kgxc#g!6f5&0!
zu(ZWSOg8von9x-N#Tm(PlQ{xyWAzwC#<nX%9(QP45UZlt((q6<G%JNbSvwTG)>E5L
z#&!0h5Mve&YLf2Gv=r5?lxY(>1>BSz`h-U!O4Y7Mf`vOK$shx)3ffWgDO}AFmAGe9
z;MlfcBMn(Q83vYtuQqUEy>JhZ(?ub$X^AqN>)2vQZtm%_<x5Lz6De?csK6hRiR=E;
z2I9*N@RT%cXl`I9x2nfT524f%TRe6uMF$h+P7@=QFE!SyHVlA9suJ_nYzkp>5!0ff
zqrE|^+Q7|st}&FWcdylyu9km;EcThBVbRz?<~=}F6)97_uNa-TQ1T7qRu!jho^TWf
zQxmsFb|8TI!_LrdsolZ2Q}~|6tlObD)uC-zv(h|rCn`b_yt@(h3Ga5HHoA!08{5q3
z97;_y#ny>>O?N&2+Q>{#AVMGV;hA`AButZNI+zbPM5pO-UaMk1jG*oWhXoa9#I4p&
z6P>taERlg+5CB?Q$0;Cu#TM40THe2tQS2(uEBBc?d>_!PlVqH?DlswHI6RU8t=Opi
z?+eaIv%3Dkz7T~q%$bIbK|Nm#F|m`4M9RT!!3~qk9Z`9lk6AXRFn2EdOyQGqAL*l4
z>72Y+uWjZfkw}O^I!wFQB5x(yM4NafYPANrjS9MxC^c>C24`8HGyQay*${@A?P-H}
zJ~BNw8D`5=yQLfc$}qBXD9ePDp3&Mfo`_k8RkQV4A=@38Ni~z+n)X2YKG`*Oky~c+
z^l$_6I0zTglGaQmhZ6c4u4QS5%kmE2fY<VA)c>$JcE|*6o$X=<FHJSpO^cnTJ^-y9
z$M>qB(Gp{8X`_5*GGF~tmIX;G-`u`}&@@X4fk|m(!g4a@L$+;1F){!Q=N}p5R#hty
z*LKpKBkmu#_A%5p>H!e>M&KR_mIQTf2=F_^-Gy9u4@0CVCae0Uipc76e8ClV-6Pq5
zV0O;9#^S0bJF?2FX#pc)7*QLkmZ{J!45F6C{2v0nJU-r-{@#x49_;eYz}@ZgHw{B!
zDgU_%l7br5VI?g^BX8vwGVX$8hhtPA|9WY5w7!nY=(`(1rc~dbVRT&Y2w@!p_B8p}
zC5f~L*nn7l5VGSyXf+cN>jtowDLORGs7XccIC6_Rr(fc7FD6HQ6MtJyNeoV`w0N*7
zRQ|?UlvA`T&0gp@@sFDr1>p)xH9pCQa?`a2K;-MGJ3<a?cz{=B0a|I72G{IGv13i|
z_RVD;It>Mf(hR@y4}+Z9x&=~4^s@T<<pUZWfg!MWDs!s-lm@}NmEB8g)um4cQ@{Vm
z28J%f-Cxxq=s*0-jw!<Om-W6GmEq+dS{gU)_yd*9Uu0t7VQre{V2Ng94Wz{tyuU^4
zccUdcZ$Qm9rbXI2F{z$-2Mf4sX8=8xkPu5KLz~in`POn(mp+o*u6kJuX=Jl0y7d-Y
zB2N&d(V-BH0us%ck9H_Wt9W=zy=j)GMr?_wQNHLSfr$b;>LKN(%hqSb*B7<_nO^{e
zRl_;Eu?s<jeLnt}XEU;-c!gD{Kr~aUynw>pRl33U&<QsYS1u=&{E6^lDw}O?XTA2O
zK~FfywLKfFAfSvhV+A9~#_}WDeH<XGnbSSE@M9Eyyia?4;_a4Wy}lM|;tLo!LSAvR
z&!ksdGyCxc@e2Uu*cvk18KsrWuB4|Tyk(n&i^Q%o3yc#*(!>TxvYUaPFr-d^ZIvvx
z58EVFytFsN{G&r9mE%dNF=KVt+y*#=KTt=O;$YkEBf?iN_LQC0FwM8QYJM9fG&83@
z7@A0&1;_8ZW=*{-Vhl?3(9tdy%*hos-Rh*+1XCR1x4Lc$bK(!z$emt-<+itguOux4
zK~19Y9qzZD>maWxc2FI-1?v^EgSy+D*?B+X+B9pE&+~dLcYyLi_KG?p6KGGtQt}8A
zEv2)%=P^|krJJ)nK0H;-C9JHPz$T%mkmO&d9$B23!6FWz5VX_Az~9M+Q^E0BhEAbf
zn#iWWBf^7nrb?1{g|w>{I-)y+#cO+~&YuPR?P5u*8=`aqFtx2n-z5ZFu<y;zeg5n|
zB7~J$bAR>#%~TBzU^Fq1N)}{n^9LJxkM17@EmBByy4TB03Fo0CNAs-1VtOIbiOhTV
z8xc~-rS6Ulqw_e$)Y}TgCCa3Q-~m~N9Qs-(5#Wo>ZKj?Le&pJ}9fT8LULo^?xPT}8
zN~?`j=1}ULqtOF@aDSZA)UmkII-x{6Y$rQ3q{&~+^Kz>7qv;MFyl`cPN!SzFqY>Gl
z{Mi>cq$&k+fjlv5QKN8e<s_7W@<rQGFJG%M;lHz2;F42GC5s8*e1iq!i(<09QLfUc
zq*sYRr@?!q5+#}^Mj~1BIj*)lR>FNR%a+BLrx+=%igEl0+0KxW9ZYp&%k6IcG8cr$
zN904kRT<WGk4d)0V#IB0*~rTpvbS_Uhj&36H@d>BG|6!Hg@f3xm+R-t%*nT_pL42V
z53ZWMtCi*E9AQj<#ul|4x3s63YzKRH4*aaUS@a(J_}Zzt<560*BJVu2E-rqCalf&D
zwX#21AGv|d6Xaey9M|xe;J-xT`Y6nR@mFj@E(sT*9kaTnp^7UqjHh_dT5dD2gq_`<
zb=}Gvr{vp=xAgK0fp(FeJbSCfyl*$M4g!u3by)E-XDzrC9(|F~sW}NrX~Xh1M-rSN
z3Otrl(1<)n0<$Vo*K!RyQJ8xQ;}jLLH1LmJO0IwIxUP95^!j#_FuBX_SNtPHf9GEK
z>CDVKE(Gg=R{LQrR+xOI*G=%vx`oaI0oV22&9+T@WExs_T@G*6+Cn2`qyAZ2N7d!8
znEOh`mqu6j+HY5#z?koM;n0U(&Pr*~AUX^f`R;@2mkkAZVn1VQ9r@%s&Z}ToswWB~
zFLv1ePCnAfz!%ABw`BdX=rleMIa~TTbUdyS7}~j6j?xHC=4iCVE~bmM=l=&kK)}CK
zzA))y7MRk*I;u*Tt!A%@<Z{q&@^?(e@9GJ6)8pw;cR?VnC<x2W6y^;B&jc*lhARBB
zUOb>wMp4o#HWQ4CX`mZz8;254-nq;?p~`ZFf~AA_lG+)2qnttGLIEX4IeE)sb7Y%K
zpqc2~>Yu3{8)^@(7k0SNJnU*ugZ6Rv;}kaELKOk3*%|@}nQ={%>?G88q>r#uGcBX(
z>q^y@k$a6rZYF;mC;nB~#$88b_`qX%YZA$)WBGiLOcYOl${$6Gr}Ho|&^j72*oxR%
z-m&Sc9Tl-yRkgnu4XJMk*d&0Af7Rvgk9UiR(w^YP^cC!AxL&FoJb;^4CwiI})1Wz_
zhQ9kg)G-)rGlDA`zS1e`I~ip%pGCwnc}zD0S9L=Vb%#f?YI{xt+z_f?#F9wa4Qz2s
zT5S)TcGc8PQ{dRrNb4q5rzS;Ns%lx}2^CdjoBeIh$yXryjLo%TvdH|<8-HT3?WFY&
zd~0I=icV8=6XP>5w@X*D=4Ek@62T)N-W@(q9;RzwT)4h?VrfLwM<cU+v<=XXxsiE%
zB98fQ56HS!izOB9I87m`rC!3Um{oYS+MFh_3|1}$!2)B3>vHYo$MxtD%Sc-cQE;~v
zUGru#+&19Ll0*!o)D4jWxbbO0N5*=M#W45D0=~;)z5V90V9SbbjG+-y>tC}XE&6?N
z?MUE-GtI~Dm1z<9a~7z@MAQ;rbp7qkFupo|clBjUNB8qko(*yDT1gB$(L(?#klSx^
zU0=kfC+BYlR|MOrl)BI16{0S2bv>jj2QUE*+lq3Mj-4%<;;(lB+0-rZ&PlC|Lg<q}
zQ>@M5Z1C6|pF^A5iW;V;S+WRrqy|0oM;hGKKBHg&_4K&}@8}x_uvo~|7ow2<LaxUw
zHxii%bJN=8e;MlG!8J3T;nAf(r5xcu?C?$1%i<z0rc>C|p6l-{Yk08FannOL#~!>j
z#%tmL8UL}42$v!l|20kj8k*5VJ8aLa`ed#qx8AjDyY?YTb@NWy5q8k053B{#u^gGK
zc@ZWmy;FEu#{69mN;IYh((Cg9yl6na5|j_G_0}cDtN;v)I6P{!E%iRAYw~EEYrBtA
zZ*$JXyCK-p0@gEkOr_ym%+&dYjqYXC3+K=}-sHs7*T+-c6Yf{lg}$6&sWsKYUv+bv
zZ$H}K-+yjGJKR1iW!0qNN4&R|5QzUSc+qei`i<dr!`1QaM}r(bE$ML8Up(-lcWV0k
zD493@Htri*V<U6N>46w`yKU%ZJy*~j@);>(P1_c&Yr&I_Ht^JF(2i&CBo;c*i<h}R
z{tWJ;tSQFXb?iAwt7RtF>=>=iAM;bxO&p5;ZjQd#t#4})bj-Krx|@8lrob}{HT31F
zW^StNdpODi+c<=a+~^<-7x6r%ctYkx*-Yh_i)*X7mPwzgUVHkRJ>)q}vs#rpn#7T+
z#;G74p$o9}>Fc8C(g7Wpe>hviL;4Fe-tSkq5P05;l7JLI!c)CX4*5VWv4AJmWDOn(
zc9WUJ-I3V9LpOLz+XUvYyA<4z@{>%*kAzKBAM)ue{&J2N3!ZjcaKb8tcRYxao@BEr
z&bSd^+mohxR6xNM?HpPL{J<XRW4*!Sa6nxj;jeY3K{MWr1I;+Q=rp+3k6S03oQ7g*
z!t~_J?Dj8or8+af4Pl~bCQFlz86k<0%|O~jZhIDbkdF36C;(i#%7#}P=*#?BrvGX0
z+nd`sl05&{r|5w;!hjcorHR>%4c^eRG)2p_MN&sn_Bdn(&;*)dTOa_V0Z}}kzx!3b
zs<K`Ukdj|>i<q%UqTZF2m6?_K%Pt<y&JeU8Iy1w1(oq^<X#M=^;Q5PekRZ;(6+8Lj
z>x2DPTPS?Y2nK%t-Bj=(d?H&f6Kro|K}TOv*_Ia|WUY6OOhhd^TwtSuf2?KQf4+Bi
zw$-YdPXFarlI?QUveU!!9()Z0(+h1HP+Tp3t%pzKE+#dc<2yP-qi*w#;yO!p1t?b_
zskw_AAVz${?raIDPv6>^1bT{poBF%aa<zK5s{7HuTjg5ifA4O?;pxv{J=oAt=o_-|
zuE@eJ5Visn8ZO);!o31^&5m|u_}Wqh`jYLS<vG!o5gdX!cZHq+&gG|`0Xm!EpA}e)
zK)&i1%2z;Hg;30<+lR2*fIPGVC>DJrGBmlvSUm<W(7vZ?qZkj{z-qItDr|#AdohMX
zWx%B??BF-H<`tNa$7^utCM){QvPA5lnqX&=kayCO_4LQeq8Pv(=wCUWv<!9`p2%N>
z0q(s#|H@;4xg(kw3@F?rD;Gci(szsV^Dl$^BT%N>ei3*U!iVWY{FpRMUssb!=+)%-
z0(e^im;>&j;gDw6mv7a{&$ObKOuJI;dPDM;s#%ZGQA$?!<mLH`m*>N8PYx|#9snel
z8GC2Fx_tCsihW^yL+Qwqjf&Cb#@bwZ1s;sK|LN(;>G1U6{N?HK@aW*|Z134YYj+Qe
z^OL711#Cod2KEbTPhJA8lQsVNANi5u_r1Fi%|3xC1n%Tu>mbzzCr64~rj3AUCT&C@
zhE<WF$ez7`z3xTFa*2;wU_~-#h|zJ>kd0XFx1e#<I;nTd)LNi*W!HbjL^oDqk;6CT
z^1|%Kkg-l#REsn{_kOw)pL*{X_#{9*2LyRCv;d&-x%CcTAW1NwkP%GoR1N)+6Ls8l
zS}x&1W^EFc^X5ZE8Dv6?#uC{SxmycAOV(QIJO=*>EsgENc~~5w{f&V#@nyBdKuL1k
zn;Y%LSyvbCMIa&+WzWi5r1z~+(ps^Q+s(~(O^u2N{lloOjndh=z_r#YWAfwz@s1dF
zhKBzIYFS`lrjXgUv-u5Kq%hu9Adim<)G@((hV-a#ec49JR(K0=RbO9Mle$E!h+M{S
z0>?%oqb2gMNEqMr222<i6^Ly`SJ^fR#%xlLap(vJQ@vkI>v6qeE!8<v@*-T^+6N_x
zW>N5u{O&Mnzl;;6__UzB;f*0tmi;nQ1-B+YG>BZ>a*Lm)$#E-cG*{Gnzyp|1(ew8G
ztz%Uv$N}$ZD8}jit@Fx+3LxydK&tn3C9l{|1;HAxcMOmPk8CPqLFm|gvb>eUgLRMR
zt7@l!XhM#e;&3|gvOq80hVH;{G7Ja2lHDTj1}uxbKyaFiX?cAyDcy@CT2)|<^*gve
zT!gnK5hp}=5fid#PHsCmK0gH%+r#5?(f1FZwlIBjq;UgYNmH)_>BDwx!zdK7d{f;9
z7cYF6bKWrURLna8u?KBx!J!5<hT@B0$<+4VVysEjkO-5OcDxIdhi>7I14nR;d9A>N
zuE_jOue4aSVHg0rArENb3_+VCRIi3Z3ztmp+z^F8TFR#oWMKaJl*0|wOV@f~#^4VL
z0t=^?y@72utnO-?aswMI?1D-B0yGK`knx8Hn#Uw!S|t7@5MOmyL!eG*5dJuXh@|09
z7N()C<uFWUo0bW+UA1gY3$EP`K7n=a*z)B>?V+0mZzDoT?4;LvV1X#Y7I3uNv=D+-
zgbiIzwrNEnX1E5H>lxNnwGfSCh>-a!R14t8t|AYs-Spx8!I2o4(J{{3sGWWzo7TGP
z$#yuzo;`$Nu*Bw5C}F)4a1x<!VB~HXX1(#SQj~^x1PrH!>1;l)!RYUJU^$ct{_%8g
zJqCYq9~R$9FxVJo3CC+Cu5b+n>Ngh1#q_Lhc{Gv-t^EjYKJ}X!KfYZL;;r;X1CF2i
z_ZGD?S^3rLR^qg)G63XhrCxi&iY*kz*=I{((1|8w=XNqn3^XNF1zt3kep#k-Xr0};
zcG5Bw_Z^yR1S3%{6523?@Ny76^d-5$ft-YwBFUBW*=79(oGr^@q@3_ZEp#1iM_&+K
zcLQ|%K1QhRYE=Awp<G52utoSSE(*O6@*gNyf`BkpVDuY;AA72arkO$o0M_AWM~(-&
zHnQ6Cc6eRROns2}6OWMk^^|Kn0}TE_(UnL|@R28MS_yJe6rfupfiTGAl?1uF!1u~f
zaOsUf0irYD&nHWA^G5-1bawW9pXN70xzDRSB8P&jd9$+btRsTjp%70V6MEvI4brRL
zB?E7dUb486u2HQyP|N5ytNM~G5G#5}1$qf+0%Hh`N&~0}A~35(igAQCERTl~ue*B6
z*t%w7m~0{L%aI#F&LF{zfQH7BAqUk)Iu#6-1S^%&{Q+~K1Nl?X(PBEEzg;iL#koNc
zom=~2NKMx0_6CCYg}+3iyV;ns9&WV&;&79;F*66VRrMI1E1+A2Q39X!0*)H?cUxT5
z=<`9Zu8llsDz|I&vw7>vL?dg8DPrnenvF(VM|xH9m_|T8ZmS%S_r<N+Tfe(du@wjI
zTfA-q!QJZo(NKs%_=BNc@)wT_hTt6Vk#tFj8P)*l8QHSo<yttjq!UJ+vBXC)ECIzZ
zp)s)Q5|l*4tx=HxT^m~UEdWLt0j@n~s!?D!@NxD4X@-t7&8KmGdTM1Hn~XL|=y@fW
zro3wki^S_*nH$b-o0YlI8J6<gu*xGd@Vc?SLee%z9dXNEbFLROjwqkRdIdKMz37AK
zvPt<OsyQlVzauyi#I_BW^k`8I({(wi0zfYu60D}<tnp%I?huVGRKd})U{(#_mk_Fm
z*OPM6?RtS2){ci>ak`lv5D1@z;HuzrAywF}1(o8(@;#=^xIZr*Bb-me5K9=V52DK$
zU53lcm3pgykK@zgDY%-Oqv0p?u)aptl42*4JidIp?y#i+LkI9FD@1%je=>XtVWb#u
zY<QyyZj7J_GRjPd5+il=4UPhvT#vhQQ3gUJMcLbEd{@t2Gf>t3k|;zRZ&$I<R3T<l
zyB>S<3%cWNin(6`-Q1lI&`sTML0cK>hL$s5g$ko`z1#IDEt;M~1_*JabxgKM!a4Z*
z-#{StyZ&`#;xH*x@NO?L{380gpnk>M5^`}Y*+ycqFL!%T{kxkpQf(BQCFe#(Q3TFE
zJ0tf0UNZLYDxKQx+|lA6VR2RDeMdzK2~oB393^bg-L0;~4;vri38vmpF+oiOAW$F|
z!P85i#$)_3#(W7J2h?ksgC<FDKH(r#IiCv5v4i9)YsO-8h<tVk9wOfb;DC^g9cBcc
zhhBST&ECKWQ7%X~w}cSIS>0TvL`y9|9R^ZvFeT8e^n&_Fk5OH0aH1FPX2s;YNH@@m
zlU6=yJjIMfod;?mP!(g?R~fq6hQz(>F@&w9g#f%m=t%msFgLN(;6?+T7{QM~!6M~E
zuR1$w+00ITqz_uCKDJMetkgwT$v#JrUwc;nL#DJtQX%Suo>#lq5}4Ki_q8-?gHZ>~
z3fl&}v{Na%`Q2nR7zbw9V22zL)gnnXBerF^yQgZDv)f|irPv&?OH!y^ln$$a7l=TV
zEy}^nuv!3B@Vi6(>Y@tSAgp#MLz99n>OZ>K1|y3MnPz3`OJwn|_*neF^&EE4C$neS
zNDf<{2@orN3pjLhi!JOlo8qjZSRo|2x%J@UaO?WgYfQs9+XrmP@{2ykm*u!pxhW9e
zejcAFR~9lc$ulOV%oJi_@ZT{t?%Y(+Mf$@8dK3dm82h6M^!Bna-`bxuMfAm|iw{#5
z-N%&O;nw!}G`NUumfb?*zdR8;s4<<I$UaNM(Hsm0VP$aRVlOwFqqG1|2tb%yI`_9k
zlR;D1L5o;g?A|3!l_P<$ksvRXS9ohhCd&co=_UQ$2kJsk*Vos#n3dVrexi4$D3`93
zD#r+&z>iZXK$-Ki`y^X|(~h*aIvj#@vE0=85TPjT(-<%ihSD*=TtwCUrx54f@@!o$
zCmR-^Ux;&>JA>hikV~r+7$*5!7X-B%=?ks17RLbdSG|y>ytrN&uDko_NxFvaSihZi
zp;m9p#gqi!zDK+mDiba~dO-%mH}$(}h)L$OBUdoEHs{c2A`%H5ZYp*dsq&{}K|(_)
zTtR4cfcr=@S}`IzS=w_5lUuQLcuKI0kDBt5%&)pF&;!I+J)b?ARES;bT2ldyAO$>f
zz)H^8Pj+sb5m9L@*baNP`WY`^9Q@#1;fUC*iZ2FEyzuAu9bSHzQ4$}7#^#<h(%TLJ
z2YAnT;(G=ewy&=tujj;~|8p3p#t)Dtyxkc|f;kH93d^_@#k74FM$j<=1T%encCbPQ
zg`p`$ny;HCzieuk1d|$qBpX;k5LHkTpsWN|$r$`PPnV&^!hfg?v7{14miT4TR<gFK
zE$kn0r5%=yPzP3aE{Q30R;^I}^L9EG=E6zs*e2gr)k5WK5Ic5b!2|+z>vpW6v%_aY
zJxdU3SZMo;l4fXMLa#EHBh#M=w_vkk7M<ut-ABPjsYmq(JHBmaP<KO5x+~D)6MN`&
z7JqLpq1Y+FtAk|8<jnsv8588CB3bui>@uvWRQ;^xfZRPf!uXFlX*@1DXcN#A#bcSh
zZsf|r6xg08VQhd;jUZ!_508Pa&3|eW-4i`!E%B6WXqP-~y6V2q_?z?XA9&!y2mG57
zFH#bfzhsBsVme10ga{Q(J!q;YtJFJ_>QU3V1G#H=8wFk;0-G19+RygR55C*`US$3a
z4MZg2pRmY9)oq>*OpJ=ahlX=>aCBm3<A-BzF8}o4c^t)=tfo)`lizoKm{mW_g20;g
zXao4PgO=Pez`*;2xF+y7^2n}Mp(yE#roa<7Y?upWIj&YW^W|HP=TU}K_1^4ad2l7e
z&Gv+Qckm2TS#+bwjYA_hy-zqG$BhWwxSFeq#UUX-sY=B4f;IB`ie#@?vPtmgewvx}
z5`B2U)0@Vg)fO_!SF{;w{W)DgQp?yTj3)049?^OI4rn{DgeSmxWS$cLU=s`mJA^1b
zXp(gRHez07T}2Sg6t=c?fYBosxr(e^JNSG_paDsj1~28$weO8}Q8^PLj$v9~RwRw_
zOOm94!KV@dXj|3pO;oncv3)L`=KP6M5gh-m&gCFl?_Nwa8eq%&A`k^!G=rj~kd?H=
zWp%JX64pufG^U(rP>C5i>e82v93GD91uV&1Rx>W@S!sUJxf}!H3uq2g0<jXSfHYAl
z6MfLPpmRtGxmuGZin*J>h74{#oT87Qez>sj-=<Yj3I!ioJu_i4<`I(?Ga~4Uk)8(v
zT>0~{LD8;Xq}@w%ZDFJr=LW>9*W6{PmlmMD&@;-D?45rUKA=qy*#y;i*DLu-L-=T7
zX0%&gCpQ&mZo=m&dW}Jwf{fJq&$vFUBMm(wn34Q1)6OU6^y8C*V4na4QhB2fAg!5e
z2zP7go`y|h7hoFTI6Om}E=!Vrki%y+Wu0FJ1{w*E^XnDe`H}~R2?JJvV`XS(z<?{#
z<wwa|#pc_@C00qEEYUF7gpCDvE=b=V6<wG%mmoS(gNZ1nN6YkPO<B4U1`mqrYvvYv
z#iNQ5Z9bNw&Hoknr9z<6tTPL(p~lCw8QqjNGdi{0&q+2zT$$um4Ic&_9bT5Ja!P>D
zU1_ry_Zry>iNgg=hIU5y@RFs3kQOl_Wd87Ow+OAGh*%&^q*e+{F5&H>DytYJ)Aj&g
zGy4G?SYj=7+ny4eT`Vje?J%tjJ(3TR-F5N5eeY~9W~u#v>knSm;66(r8&YOEo8_tf
z^5gjkp~gy<z3KRNqkp#9d>LjYuwFi!C(t@#g>m5cPjF7!gn2$HEiy4TmU-)lDyq)B
z?Uew*)Fy|PMM<YUBXc+qC>m3DeN2awW20{$KXpC2Kp0>C;COD3Rh<=^BDFKFCX^@!
z^A)g-9swT7jmg^SUF_-wyv0|SmtYby4BxEP8z*)+n9xX2R$cB<W@gXu51UgOX-H%t
zmZzy#)J;7<D5thI;^r?mA0tc503nlHEFeyqNG<b5O)t#Tir-f#fYE*pck2+`J<I^#
zR9)}{F%^1SSTeR#aOL9#Bjp%+yD|))_IJDJnMeHNtjD;L93;-YG&|9Y<qezkTKREX
z>oRII-`2~O5mAey5>m@mgSH^Ol&+k#=l_8X^VZijWt4}frg!ri1X$GlTvxbgZRYqt
zps}~S9D?WvgiTkek(Z!7ysLEAeb;R7)#KOZe}S2nh|IP_|8%*1h4BiG2Ir>-dq>`z
zL1LVq`&L2&jNm#qw6Uo_?ID=!ZleWy0MJhiC7M89aV_=PBH_Zz^lArTsI4maZ)=Xf
zfrw-VMgY|K#6zI3`y+NjakK1)AQ`)AHT(r{!9XcIHMPBL1e&331Zb2?n4JhOB2d0A
z9784eAXJoiuSOPCWdx^Vlsx)rLmksFx82PixRe3JA_b_1AiWzTneKL$7a=B9?o}r)
zHPqlpY2|5KP$#5-5U;7TPvJ}+29su<;?WsQ!68JcA>>e^;B(!)fh4(r2!(n{77Z#4
z*@8>26F-A=*k!K!^W$H3etz;x&tjwHHm-dk%=mI}Tb^Cslru_!mQ-^)6u$;!?i}SX
z6<7_gfigN~hCj^A6n~iA9se!O_FBi_Rq@xvVqPY6o$Ejq=WI(ut7p`lAIYXy(xq$8
z(H;w54~spGj6?ZdIhFDP3B?(GP*|?YS)!4Ei!^g+2n!eU_0r7eUDXWItEF_&oEF#$
zN0wIIzC>iU2>&c|_7ZAJpOop8%~WAf1)NhEw3tshrgM17>Nz7+QKt6BgW#v}S#<a$
zf33NlkjdIS0E$hvd0RaM1ih8=G{2}W0eXQmC<wpcBnr%gF^@Y<Ot%$cJ*7qv0U}V7
zZ3TiB@&bap?0((nmQ>{WJ)c|L8;fr2KF=-cjYT)r`rLBY$J5}|R7T)-LhS8sRj(%l
z?ZOg?1smzXZn>3z3A{1Gogw|k)06Lxhw|XBIdzCC*#bY7qP%wMlM?N3jT*QAFRqXw
zhWgFbif(N1|3{ZST`~9a9^CQC2d1{S_w18p_EG8gcsi1FZPC+_!c_K7&>!c;=vRMg
zNFHsUT=xCh?I2c}$T<$Z)^-w}(uc)&n8yX#7}~ZlzYAVMnEa)dBrOEW1kNYsGP{Mq
zghnz7yT<6<tD(3Pb?<fUowxb?i}f{oq_>cld6C#ot)FSx?$&Ob&PcDZ7@cSuTJEQe
zrESd8iG~`!O0ms6ns7`Ldp-^NG{NFyBtO#M$;G1qF{MjW@X>n{ZvuAc^uwzsJFf$x
z_zL|u8iqaapDYEy5znHLPKa58`}B-*$pme2Z!F8Gcs2zL;V`LF^IO)7ZV&9)I^oHN
zjh7|G)9k$Fz~>mo7;1N9L#G};3f0n9{z`Of8&6X1>B0WN;kO6QNnhouj6L)P-Ur{r
zF(XR19#jL#KHBrW#hnM?h|-zN<~R02LuMp+HRv8sZs8zgeMwL_@K;HxsqPm{H)_6;
z%%DXfc2EH_cXQZ`)keuTrGce#s|?}k!%z|g*4f1d{4Z={^gtT`|6>v7<+tlPOdGsx
zz>5EeuuDqYaAnS#(?Js%huz5OPoXV#K9$Pbq)m?I4Tu$FC{DHE3iuUm2Ga?G*n!cV
z_Uper(;n@fDHBKEJjA=6zm%k(bz|{l4@toO?8WWV>fLA6YK#b>oT^8)f_YqAdMm@S
zkP%5hB70RqsmOHUO`;{<#-D3G|3iil`W(X`*&Tu>b1$e!brtw!0)ikPfB=jTv`74!
zAok?X5TX|U6+Ky&Z}8NaFQbQa6dc`qxG9rKKB^nERALeJvSgHgMyD5iDl!<CGl5cp
z{xdJ{dggR$rMj!yPYotEcSH**{*Cg7d=**b?Q2{Zx#4Z}ZTi_f)kD;Hx@xjoNON5&
z)pv8(rofAJJ)LOE`!pauIZ29-D0ph*`}WP$mFYpWyg%gRNIXXy9EA^~8P8nNJpAK*
z7aSA&9sEhOFECGb20za0z(L_vlxW9M7bc_E=Sl|gI$m(ymzt@5I{?`-v)Wqe0MHwh
zwpiy|TRYe%<d*SCn}!6eH{~g1a{=P3xGV}Jm*Y@4Ujs_6Vc%P6+CW7}pDH4QPnc_o
z9`IVTEv4?ypoq7r_eq4*Pdv@HWk&LP2pwgVz7L;=@W#ycEhY?#`9w{4X|!BFL5$(b
zl0enAk<VhU=a9XP21;hK0)`qQkhA5k04gb4x|2Lp1EV0V%JEe<Ci!$oF3^`1=t4UG
zh|~lD=)VIk1;n%s0Y(0QdUsP9nf|C7t_SJqC?rC(>nLBVto5kdQ7kC{X+RV&vLy?d
zxn##483EyOwz`mPqbej^Mr<`QE*Zgc1~5L(3pYRn{Pu{!Or9BiUm1xdTKL<eyswBP
z!1O>GNuzN7DB@RB1<<m#_0(kO^lYKX4DY1L3_Vo6O-e`(b#Ex!x?BAGOYjvI$9!qT
z1a!y?mw-%>!^M2Do|aihhOE*8pSRU*(%KWOh00<RFp`ZmIgw3*|F}KzB2bg9L<^dQ
zy!84JHj;I(kio?EYvpD!9`B|G)i)<%H<(E*l4kN0p_G`*VKg2?+dA79d{ViI%)Y3^
zL}oV`W<krd%(DVbG}m`}rolAB@Hv+zGw8)r^Md{UAEi0B(jTHV!+t=92+wd7#51Bn
z$O!!+kpvdVhmfHFmjop;0{4a1;ZgzF3_U-&E<D0>zVGx)iD}N}6*ObBP!`ZgE!p7l
zb+ai}H}hh#UM}F_I~>!Ko@{i{g~Nlk+vXoVgb)L?wHqXh*?7Lhj3Bt%^=uM2VsdLm
zA!0wl3&CW7qXfnN+AH>obE7?w1(VON<hG%VNH&!r<t@6dKncVtua=?RxF`hWt^oQ=
zSWWlL-qjcD74`Ta+BbUR@^6pGp3x1jsDwG83-J%|UEdR4eEky(bV+Io8-RYKjf}Y>
zy2yZGWtzh!NC;E@0vb+W&MX5N_Cud+hv9KEUpBBwW<{H^xh$6TJVe0;Os6sEk~Jj<
zoy^w=iEi|D%q-%elhaCv2vgoJyn=bXHp{0>)3JpD&A4<j1FqOlas*F<0zQaTP&YP_
zb(%SNW76zlQYS!xsE!B{v|&vgVGD^Ox*&*neGJUCV+!|*{~PbOm)Rv~KE>rf#@>G5
zzG|O^P{N81T6oS<zV5zm_Vn<yRW*~3EB)o+!SkmZlPmjM8C%rbddX{Y^6@pgP&(Tj
z(&dO~OV!TrIDIq{C#b^&o(P$VdC__?G3|BrAO_~ztyf)GvQYfG&qlhEM+fI$oou&@
z+wb5uyZ50xcjsOEI6Ee4Pi}L(!XQczTbxnF_!w?~efryQ7v8SNeqSE%dXAMl=Ur?C
zZ@Qn8f?i0k$VRrv=T7CG9h`5X)$Qn1z2SYB8R>4MYoFR($k_3TIfX}G9N1$EmqX~!
z`s(rPFo=~N4H(AU<@|<EY>WT3k4r2&$pFdO(-Y-9K)#=>Mu3vDuM&_WL{hQg_5&X@
z2%zoSWOP^^&a$Tr8Wx&t3MW<lv^FhL&CLr+101Mnc#|^&xoU+v96*VEFj9E+npHbO
z;$SsDLw(V&wyYM3dJn~SKuZ(cSVy50)stQO!mV{;Wi-Tu^^whRkJ4b_m!o1CSr94B
zy)qy+05CXa7l?)t0Ux-Lk;lToYt}lpq3;4lkOP=6!})>&h{gs)%v~-m?w|R*B-gpV
z*Bm=QsG924rp8B9vkB|*3f#;u9FFNqvh0YuNKPVtOylUsch@5ieZCZ(yI)j;Hv?<d
z8Hu~CsWAZD)4{Zyy&2Hv2B~Nh2Gq-+W}Cb$hNg@;@aDfCWL4Cj4I7PMv1`LHa<G75
zrCcb6u@^+x-(B1w`~pOH&HwmFWNUmhb5ByA`vd7xK0zec^1~$Sv-7?4!~N|AYj+_2
zr9^6EQ5j)6Jd_)SYLD=^-m9J0*@KKo(!f!Nc!wpzA)Zwn1n8JAT~`9=LQ(nd_Or>M
zoow*ax;8^*bXrOIHK1G5$n;P(hhpLr*y+KS&ky#`4^NJ_kcXi#io;*;eY-b&zIXg=
zxc_|b3{|<!kE|ZKZTAR&H^S0q4ZKcRjx9bfo{$G3xx+qw?F8Yom;3t>Tq$q)M<vF9
zEIO@nUJwKl^$QVn&XSB7vL*53*WVubfrBuBUhf`Oans<d4$FldJWnL+bwaJ6)mHm)
z=#7<?yOY_6Dq#Lj|32hs^|x0~R8g{`49~xRagfCZ+N7bAc5d@Pq4N#iATP|yAO6!y
z=0L+OhJiNsI@FHr@MC9o;vX=Jgxy?CgVcbQLGZ5s8}Jrqb=Wkxl;_;m#~(ghvjAUP
zz2~7H{QTrj2cNqQ-zfiFuB5}?*Lk#+f>Q1MkN4<SPF^-8$z=&DF9<Sg41B$=-^12l
zmCG>d4B4jT2C-<pjTh?hU)2qDxRoDj<R5o(e0F~Na{v6KUC~IY)W4da+}#w3`#Z`-
z44dvkCpi6P{&6pxYWbB>miN}H`TiV|VkDuM@Lza@H|GQb`2YbLr5ef>hE2imY>EOa
zT=l+$<~Z9G+!No{)r~}TzB4~l>z{xAc`-N32Y(r`t6mSZUbH%ynoeDzZ*8rm*^T+N
zRJCTM^{So@PVu`$L+V>e2ja53$sp@St~YUnW27+}y&Jf)X;Eum+)878HSkaIwfJ3G
zqrZZ*jl(X^2}QS@OyK|M-BC`-kXSXJy`aT5d|@(o-J*eUar?I)amGSTkkkYalJIZz
zv#p{2)>VTx^0guB0q+&*gHKpVfND1T$F80#mlGP}F;dnPB!#i1B(h-=K%1TaB}a>@
ztkpEMir6f*QZ)ljNPXb9xAQgDyvK9^8-kYg3B|a=6;m#4=6W||-xm4J!S}!x|1M-d
zUn+Y4yyF>-c_ngYWB%GR90NBcvnfUXb1%w-41bri%nK7{mdROwZHIqo8}Pl1zvg(o
zyB^k)5O?k4Sb`0yQS)u4M=oJ)W}F(tFp=Tm(+8Qd^utE;A@l54Y5qJgMEUw2y0y{b
z@(m`95$WQwPt?cJ+PCnGi;ybL9pVuJbI1|WNJ}7*5J`AS+d!1Ik8C!0dHl^W;8Sd}
z8_2NIrU8#|elzQV+KTY?maPP)kNt1<o*g9VMRP%(Ub~V?^V33YO9;A*0JZn@%Vvwx
z&gKAZ0QmBMAA|M!^nB;_Cbes`>b#|SjF+XS0;x&tNM0NC)d)am<~M>U>VbL%NS0on
zV=vp6yCu-B%Xf(Q_Pg{h!JB3Gl4GpRxnN(6o`$TGlc(;GW^*ugikp-Dgn2ssiIybI
z<nQ*ZCmk>k_KrP(RZO(*mSXqFzE#8mQmPx{R@E(|I5<5x-akMClsVh3l|kUl?JLa>
zi!*lh#WZSGT(u9z-T6$CFo@3Dv)R^yrtrbY;tvhN_bMx}V-Dqdc2zBFij4`{ai$e|
z(>g3}u*$T4TRCJ=&N2(sX((-4r4oEzA|Cvk_0))E2!JATV1_U9!tbi}3$jq*<=OzU
zty!KSsF3uSwp(<G#$LVdZM=)``Z(rF!5SVOe7Bt&PnOF^%6Hq;UOj#t#d`cnl;HMP
zGz6kiePxfEk*q<N+tP9EkLk~xC)<SC7^Z8sy)QpL*?Da!sZDguJTqn}>d9;S@c6q(
zo7$Ff2U5*QXi0JIe$t%(u%nK#*Ietv<Hr@avhNR8Taq#UDFv=!pCEmPoMnaijrX@C
zDyAzBYQE7sT+U@%FB<)};v5ivFt8jJ4?~|yp#Tc~1#8fe9LQd4&NOXPnIa-#-)Y-H
z;@XXhwR4;AV|o=ck4#h_#gYdim=}PV3$SE=s@PL1*Oe)p`^vMn@UrEsEh6?(a$m!5
zOS5~JYqq55dcr}ys8%->!cno{^zD(BZc7z3`)Lt1E%q^xr!_=kX<-gjA&S!FmMwUc
zZ5+W7%p{lKb6i8l8(La4sBG6Sx4lv@3j=KN1vMB9B{v9dRU8tL-E}ZBlDRjSGj40!
zae!gT_UrdbB!WI^_=v-r5sQp{)?ywA18CP}QW^|aJ>uYCimziU8ph$#N%Y4={+i}T
zNU2OHRBl_mtqn|@BWoF(%+F0%-QEc4)^oU_R%}VtoT_D=b^u>(Sx%})^UF(1<oF0R
ztKottt21Qwn6EflGNc<=H*i6>sBe+966$iSy*YM3K6U;}{sPY$_))%xIp|!<-=BYF
z?&lY$C;$9Dkysh8sG%@GCQj~wQ`pi&oBXXZtYbL@l)}_FVgzR%e$YcyAT=8<sy61H
zSt0gjUfyjvJub8FNYr+37+$)F;vGn+YsjENhIp1eg?wVr%<PQA0ePK{_PHLN?Zg<>
zxYA=v?MT6x&BYNT(6ZAg0bBwRI4iT1_1ZZBf76?~QG3}yYPAl53d&<dybjsT+!2$>
z5@h6NMI69mUq*`a27zAyu;G%@N%BjBBqx!RCZyrtk1=JmXVus;cXYn(h|CfqojcW9
zcJL0K{w4O`QRmV97-o;o(;~=l*)PMTXd&r9Z#$mr{3|!rQW0D;7o>EU8PT4!oFqMV
zG5BKd?4bD1zXh`34WYxEd8}tuGcFg^5G;a-=$Y9X1uCCPa@n?4xx<;jR>n-=AzR+W
zKba<Ybh~zXUd-(NRb^HRtUHn<9W!lFE>>^Ta$%f0xw^V>l>mB&1nZhU@6Zj`xI;&7
z>9*^}mA^wL$qO;tU##Wm0VLZZAt5A;1m3iaI7f3Sm*_n4s?~_ndZ=VKj9RnR`ucV3
z8b=e6`J|R!zKpfg$Q;eSE0pcc|F#=4yzHd|np5t(z}6Y<k*n+embqeFs+{Dg<@>I8
zuFtB~dJ$j+`Rzlpf?(z?vGvqmz*_CYaxmMFN<&<omOLT3b4d-mZ@ayywXNti(Q5K!
zbvHDrP19*I3M|zT2{Hv!ny_w?T$Ny(S+B@i7`(G&2WhD$GfYKwp$TLgiDjTA5K`V>
z0G+>>&!^p9f2$^PhK6|O^I~CoS#^ZiV6o0+&dN4x@t2R!PeBnnJU$<ueDU?c{`v6m
zX?9EOVX>Q%ID#pyEkyk1SP}nllQ$aZmzu2#ehgJ{rrHGzW?6&tRkY)9r(kjuWt%bD
zKKev4c8dA67iNPr|3gQ^$vaR*z=s80a%NLAiakt9flp89Z*)PK^&CKWNPo~899Qt4
zQFGshgXBesYyh8qJ~}*p@$!7Qe{%fo!Rfh5{j%vs@!Rr3{x(D@hA@B`j=JeRHMHL8
zJ4)f~C2Z+Q8r~R)PgD?_qBeFXc|pz&PR$zMxyN_lQz&opi3=k4biEt<0mZqa>Pe#5
z;nTZB%6Ku{$_S(@DwIWRQ(bBSGfro50|%|MTchJ%4>&IDyuPlE>glv@;+$!jL@0Fg
zNv@z&;xjuyeB(^qjyWK+plpx?FWJ;an@&D!Ua0Kd#dM<7F?Kb#28_`3AtsMQ(q2UY
z?L#IFbXi>c$%(>&`A5NVs~Cs(6xWi{Q%Nc!K3oX83&Ruh2D`n&dnPJ?p^XCuS3+W6
zcr(4Vb-Rq62{S<tgll5MkSJ_cNipAk<xeM*_Qbi}x9L0{X2N~#0<Eg`2H`FAM2O{8
zPtp!4cFS?Jr4mW@OOj|h?V(N;5gJJLF~yq^&ux}Bxmn>Pi*3t3t0f+_GmM~Bn)i0p
zaSXM=0U}FgH8-7q?d=hb#MfurL{lyEWgcR;+}GW<Ef1C;%L~D-1eX&CRq86)?TpvW
zYJLqDS0_k#hsy_V2c|B<$)Z4pKK!-ehLCWI+^!oVaJd52tXUgz&e5Xad9zFUbvZ#F
zQ8@Wn!M>}4ZFpLbS1vuX@?3`xJUGn=90eWb9R)l?zyWmHsLJt`yx(2VCu_`-g`yy`
z9vM1Gm+0&Up68_kD?m%_MP-WD^JOn)@vv|^#mR#G^C|d&7^ldsgW{>oYG$*Ad;7hf
z)jt753}r$AsKrU$tm>JWJ~Ti+dE%}`ULu~K8hpC=ggK*mqklGD4Opvm?bVZBg75D|
zK4V7MhHmIpwe*C3l!R=e+3n)G#p7wSe-y|)`=<v+R2zzqRBNrs3kpS?-_q3yMN#W>
z7I`|gIXA`8I7#_3cu3nIZlb$~S6_Kds=@Yrj)0gk%nq;=?tQ=bbYdjD89e_rIi`Sz
z1nVR1Lp;#PH8H}sD%Z*=Iz+B4r2^3De**9u7bC%Ui#Jj1nvY+Afg^pnaRAu^ZX#50
zfAy>D=UA_&(W@u@zrG%^Pop8P$$E^?J0q+J&Qa#;<WbYJ+rvfsyNlUj%wP1<umwIJ
zd40=TWaS`;E>yb)o<l)kVdSF>d>u#LP<lJ!8!OXP2`dD+snVdbc814%@ANdtyO=(R
zT#k<Uolm$U7mkXPlK7M$!-RW9ZMJ{GN>t!(+`6q3PxIe*e@)l=T%h+B3`A!>mE^Us
z!_J>6mB#kW)MzMM<Vu}7&p2o^e(Uv<lD-7wkuiTW@QZvtiVtKk&>c{7Mj*b8coI?e
z2}$ARZ7C+hY?7%kyQb$`^gEv3k~~2v;hN0lm64dk%?vZmZma%+$7(R=$Oh==|Hd|x
zfzdi4T=XT*%cYY`Elr?WWga~;3FSQTJ(8^xq?wVw?@X%yAJJk?t4#->!GksCmqlX&
zuX;lNy|Z!{>uCStG;V1ZbDQ3ua9yhKk3_V(8PBEjx1eR6-#mPeQ-kpplRqWJQ!eN~
z2f53xd>Ek{sXA*TZIv5BLg<S3K{e^ggAN%@$$Vh;uq7W%%YD+iV@bHf4}D%GDu~FX
zU%aiVMaJ9E7c^7cW2lZw3(o-Ks<r5<lWtT9l?zPL%P*_>qQd;8o*C9=>MFVza*gNQ
zQ$fg;b>v_S5>}4Xba$XBVJIm&phz5l@yzBT;eh|%it>3IdYrDC`dyehXq(j=gah4J
z!y7f0hILL_-x7?iEss(7D#r{Yv@s`_((8X~w;b$ZkZLgR!{Yn(WX@X3BWH_qi6JAK
zcOY&%eaEVEs1oJ$wY|kZH%H}Lx@yTF!Lu}#T?{KeTX0CFenYObOEp>my!7>qPQv-A
zHDWNU;%=9lg46dDtjbpt@n&a3BxT|{cRlps4du;vz!fy7BV%Q=;zM4~-p$|2K*%`(
zU#siu8v9w&H>>%)FuLU}&(z(=8nPkdVh<l}mZ2udE|ZuzT)TRKyO+zV>J>o@XV8(_
zvy>Gr1gilGK;%~1|97QPh8N`u%$@9&ZPv-~d9;2J*tWp72GJ>|!>9BOTt)~(yFiGN
zi}`wG5o^GcU=~veuZrn@#|v(?Rw}5d(4j6Vf%f=HcjYaM@NR*~og}=DX`VX;ygNSP
z->e7qWjzIGGRnJ&p$vJ8Rp(8$`r*?r=kt!Cjay}HIJz}|?88s$-!Do+{D@fi1;S#R
z5*QgQ;{z(ZE?46#Gp_PY3DiLkKeJU0nYrsXv(ofe`R_mHbE-s+87oPlg0UoWG~jKs
zL%$rGFS|c{+KE@m?LUgSL<6?brbfgeZJW2ZePljtSY*M~p+CqrF;M|7x9NoA2%t4g
z2;j8@5bBVSZ{qcuYc64=#IMw3pt_@1a2=Q0h<B$VnBMu@j^)h?)DS%_VK>my$eOik
z2@VhE#eF5jY1n2sLImQRKGA^-QjEQvwqxt#txvW+)XWwB8xO0>v;%BnTCQi~D{GPo
zq=}|a?#z$j5?Pu)u68?}KAn2|8!)2zV%6-vdhK7Ile!wtXGX-YZ4ik>^9&nvH?CKS
z5D;C$C|As{CZ*xsv-<y3L;0Th7IHxv3j7#c;QWdS*80VA4uakA&2qk8*q9dbS_8-H
zlwA277bl28lQ%s?8KOoh@=4&a7`MuisFEG!iv4>8IyF#Ve?H=94-f%#a7wM&>seJ5
zJC|mJJ0pJ=Q?2_YOb5Vh!IQwXJ)=NOkdcYK<DCv2&_i^@X$@simP>q^G$W=?Vu1n<
z3OQp?(cpwMi)u_~AF-uCt6p4H*YGAO<&l9$!Yk7x52A#3&p8VYfBPD~HQk;7Q*%N+
z`)HW&{EMY*=!QnI1yjJy=Cem|WlzfG#ORppY!7)hE=4fGafJ!l;O+1dZ}(L(YHpi>
z5l%1b_ak!Iz+9uViXL`xm|oXjOY7N0M?_eOoev*P+x}|rxXdmjR*%l@j!l37Y`1#9
zT9!7hXa(UMgp-18b;AcF*FB3~;4NtP(<QYY*@7c?m-vSjg#D2WYBcD@Jl?XCNR`6D
z3Hi&Z#CyZkLK!a*uguo*CQ4~&x1l^L#oHP-BU-V5d+k8O%Zfl>=o^<k7EAcIw>bfb
zXW3;A5hx8<15@znkjG-)3?|jPnYl3sE*1-;G#6kY@Y=Wp>&B#N#>;wv5M-sfSS!5S
zm&lpSZ-e4HSQmI&K}v*x8KU@A9XyfKX=)@pZmc~9%=)7*h-vfBwIK05g{1045(tf5
zU!w>LZeCWn@h?tkWr8;g79mHr>K%r#il=Wt!V#zp&@_k}qCAx|Qul&oX^Ji_cAvMY
z^-7Lz@m_IVzqz7Zk`!RLsFu@P(rhPz1k2vXP<Blidj;O3<{rK#0lJ<MI0!H!u&%=X
zW8SE6E5cdy(NCFft!aq{PQ28UoNq`hlu5A9yjYwA{|yoHzdAWO`0S#defA@UANuZk
zt_Vt(6w1sQiI5_-vmps5F)z=^CxILb=cjhwgd&6ZVk^@C*5?gpZz8&EP^FlHe>r6x
zV5uH{BQi=hC!yL#Hn=etA)P=;QtiiH!vP8}!~NBA`e^^Nv%_bH$LA0=`pJp}@8x>2
za_t<llHrrHVcktX;M<iic;Nz@bumOVHG+Jf^+(XxE-FV@DD6N#Q=h>w`V1`4eEsGs
z+8!ky20J1?p3pr+v4YM<@;Kd@<Z<fD1v3or0|k#|35s7t1B94+Ks9iqU=N7$ULvG_
zO>0aA4J0s%0xz{j0`Ka@ZmW_&!Hq15Db>K~1$}*U=Q#_g1Hte8NE--QQJQT!B_aNk
zRvGM1TWSrtFZ}9SsZc+AkuaLFyviIbI&`t1X8mCsl+iGPbIH}|C!-tK?f+S7Hxt23
zaWFHggw$#B!KC%0f?U;rd$~ZI2M-=REcW-FKmTHH|C{3M#limJmxud;UJ`heh5#mD
zZwX!uQJQEL<`%9QLD&2~t|+P;-Luh>?aZ$a?65Faos*JT&c!&f+UAWsb6KG{9@-4;
zf-xA@lJehbw;}aZu^?l*B$OeM;f|OR<U$+T4DNyyQBEdi$r@XH@9ERigR`?ch5^Ld
z<9sUG<9zCE3K)Nurd`+)mT*|!LS-bzYuI3x4MJQ~@mLmN0Vn{QvJzf&Xa6za9rCdh
z(M!wS;*ox1$bi<z8-L9PXbiyAD@2kluQneicKS|%|8h?FUp{2;pD0L~Vv08f!p337
z-1HW8xujpcX{PY7v@$yW7EP$q;qWs+RGJCy7B^M-c3E8xzBB)xRr;0YVex!!C|45U
zK{m!_4kEs}dTRs_1o%qefQWsmI3nJ_URVS`d4k{dd{Uv;w|JIX$AX3=tR<R!(e_|;
z`>BV{P2h#M*3djI*Pzb;rA?}GN<Q@if38!Q!ESL1GYOyC>t-mgf2N3^Zd4Zld2i^I
zvMimVOA@XGE}AMh9QuJw8Dsb*jsR9{OUw#@Sph&JfQhbW-QGa+j`DJd?!mk4e=>lm
zI73$GXralm@2DD*&YJ%PWJjb$t+?jspDuUPmp`AK?0+*nJ3l?xJJQPvKAwhJhTLa2
zZrPPiyR3MH2S%-%bhFb4u`U1hXtiFjyihOR{b#@UtN#}todlLhPMF`VSwr{<?DNU^
z9q_NN0M29R&N{TF7d2IySqc_?;);e|IPeG&NRQVIWYFpCu`d((FD)%Z{mR0E+gLky
z_jFRJ*gN9Z^7G?gc7A^HOV7`1lvr)A5q9UX=eKJHn8N|ofmTqiUsN6t!clp9Q3(nu
z#F8`=Ok(K2Drf6Ov0nVcr&TVyYfcmV(50ld6hUB)h}Zg4&URsrUE++hv_=}p7VOrl
zD#>Of+RQQt@Ot*QKfx^|Jkc8I$pwn7!wLTr+oD+~$MBw7gErc*N6^%W*s9p)AGbbf
zrKmMG6@9?-vyszrzX=_YeP9yIiyIxGb=m%cL76i!Fqj{@ULFQ~7noknn^nd)FEf9+
zo$V3;_sueUHwTn`(9ntMin|v@J5gd_QFu2(;FN)m5zt5=mkObc2F0FkDpaF1R3ZCE
zz~ID01#$iGmV{Wqk^TLC1`XY*ZjJlj(lwnND&+=`y$Ls7;I!?|<<N?n1+NY6npxVU
zQ;>HV(8hNEBgp{M{*0Y7x<wsLnB6ywwlj!|3e&v?m%d2_5>n#U^lzkIFB~)#rESG{
zj5Pj6pNk@=?0HmJ00Tqfhm2YA2>gqUZXM7d!ktKE`YR`3N9h-U3Qa%ZC00#{;etO3
z{gd_#wIc%nbNqj@*hXO+kK_=Z3&0%yAZkQhbX#WwA4(6G+R6e`Dank}wP`(va!&fi
zFfD}8cmw$(9Ba#baoKpb`TTsrXaK{A>!dfs*1JRR?Nd-1%+|xSE9UABZkFYO*KWhm
z!wryiuP0Ry3*wD;wpcsC*sa)g`L=>bj&5gp(MeZTcR&~jZ@<ZpIT|mHB;C^{#%U*W
zx#j8g1`?=`ASHI)@A-U+{u)F<`RTp_56cbgEj8A@>U4LTapp5T%IBwbZMMP&j_64R
z>P@~j`Ovmh-9jJOidT6TAd272xNk~)aU^shy)yy*rq?6h8f51!8$+9crFIjiD^e!5
z9;}kW`3jsb7D6Mr8_$DIaOaPUN6cYmN?v;jGSp!fv0^`Sj;$$18cOvLoBtAUUU^<i
zEyO?x&N^<VO{5pgu*x@|ob8;Ho(h1`vYRhuam<Dvp`<NNXqflqR7G@c><g1V`o=2|
z3c4Xz^687S-O7$hPRXq2Hhj{@G#8W_Y6J7d-IRf1&#d}1aBvJ?D&&<ur#OI74H9?|
z5|}e537CX_m1y6=P)?Shv!`ahYNr;<N+?lQBYWz$+TBnEfKR~XtR*hZqm8c5z_pM)
z3KszeZgm6~MTfrz{%x+6>U}j{lSjth6wxQ}MBIq6ZZ&7eh$`@7aZM~MfT}8u&I&Un
zkBOTqQ<xeS5REay$HGZOFEre}H5^Xy--i}sJpw*KOqhIf#!Lj0X)D}TF1QHW41InT
z$AUJMPnsnopd9#Y&l-|K`(611eAGP@#l(0MKtx2aWGw@U=dwmvlXm4cHa>f$_*i2O
z<6i2kaK#q$X0`RX4IgmTB}axgFAGInYQ(R$Rii-3fguL5B<&Blet<<Xy+?Fg%8M#W
z%84uYelSJRW2(*&vR+inZoCG3IS=eugKf$_(}=o(|N37AHu_-{u5-5sMVll1dj1Y^
zi=GS~4~lc6))brr&9mD{WFmQLOaolS`#wzCR_+XDo*X=4bY3xIh0$JAJc%ZV*7$(O
z#SJ*_@q~l&cegrgzSL@$OyBAITKoX`I`jd>BPv;g^xG>!mPU+6`A3Nbr`_)299}rq
zxy9~>S2SEWF<W$~EU~c+4++{F5Lv$0d$sfE$?K>IrY&y<Y4$D-I@?b!bGf6}Y9=t<
z)GaQ{dfHI1%WdH~fa_1V=94*}V)V|*Ys0)|eZ^Axnuqj@1kS#z3~)pmiCjMOiytY&
z<Gpucb2#`Z(dESMJ0z&GP9~7S^5-WhgDQO3sA^|V<L$-MG`qlaVbTPTW55}rzYZV}
zXkjLiz#5Vd83PjyTY43E9dM9h{~Zkslx~)>a$GJi7amUd<~&5bwdrkC+l@E9X`$=4
zKB#86ft~g0^3i{F66-`Uu;GNB?VWzH_w2yT<~byq+j?$ELP?e!vZNpYQap8~RrA(v
zJ@f2Q@l(K^9-O~CJsuvJpUn)nS)iQdNQ@f<NcD2|c80pKBWnBbD$q&k$rx5reF8y;
z<iLxLvKAsje9kYX@Ln{2LmEd(e%6`h7<U_A9bxWfiQyDnEP*4W&(%E^^x*XL<aF!h
zRPda8o*QlgrBiOFNIiv}1yCHz+OT)=;I0e7-GdX{A-KD1fIx7E;1FDc1qlRq3+_(v
z;O?-v`@gyWJ>She_miFKs@baDXM5hA>FMgXr=RdqnFmS)m{K9r+YPo;js&<%d3{TR
z;GacIoeg256t@vYVExR-C>7=d1tEB&Y1D%=rTmdJ;JQ3cv~_G&jSt^~^i7F;pTwk_
z;|v8p66VcWGcDh#LVXeIL$NjMrD6cy={!;A0%X!72AV%5TG-UjCJxg9#OhbtF#9sN
zi|6z-oZ#$X>3r1V0Ih9QLL-nVy-7Z=2CJy!sHWgXg}C`y8^)L6Lt4GLxnC9FMr&AJ
z^r5%9er8~o2I{!TrOaAA19r`1c%zZ4k$o$f7W2W(;xFlPzL0iE{3%c|X!Pb)9vb91
z>|f48-ji+u4FM!IS1jl~^|75wL3PX#sUhjl)AG`oj=xMh^xYax8FXr9eMl559_Toe
zZ3=uQZl9NJ<}WK-uHp%GTk?9`dU@}?YWQVvj4-vQNDb22q%H)BT_gKAh=DqN1o#}E
zW|Ns#oXKoTU<7=_NbrYzCQD^&&HUBmlLMIh6$02`=UrKFH6C%vnD#qf5;Q58PQ~r~
zBi)0;3@NudQgVvGn<UhJ2xq*aA=!JG_@V;{;5+y2pD6fnC+^d8{Al#g5;?ZVJ(g_2
zRf|7@)rOS#@1&>YGWJBQZwuDH+@>3@k#whIb{3TJvc3~c7?38yCcsI2Ul2g$Zm3D$
zT^NNkkX-~ePSTCnygeOft$fwf(byNp3<vJ*s2l!RDvJCqih*w$if;7CUvV@@B|UmC
zJlAXv<6c&W;76Mz_-9BUW~F}6Fajw%s|8jirKgCqza0_<tVhof8!}dxWXaxWy|$r0
z_7Ev3!L#KflQt9)nIBh+$25y%g3UtGs@ExedKvs||5NzrOkP_$KO9F_iV^~c*;|#o
zC<^=A{e6rU1-pT95VfBD#Y*4ZF|N?d#*2qWt6`k=Miu`ttC-h`sq+_O$xTG|pnft|
zFRX#oVQdPF)Jj_A&*6RO)xE=mJr(8*+TYB&Bv_D>h$MPqhbEWgB#VnD>P-XIVGE^D
zMyryNLphux2f&zUHkIt>SGbQYuA~FLj}wsj2B9xOjHhwHFgMbR42p+~ljh!pIb@`C
z#2#zD{Qc030c4%HXyeqnbsH|Hk;MazBlAUa>lj7ww3<&@I+xUTmV7ta#p0D3E{_(l
ztf9_Aw^~u!Ce|n$S}FH|#lOfXUvE#M!5-Dao4LUXD>cxr5<V`#Fi~Eh&j_O%Svk{B
zxE+7JZ?AjDUDlOOhKH=WBL)6DeE$P{_r-~!gGJ6io=!zNzm+5qL<XtO$5-cjTIl(s
zpFgd(k!1Ne-HN``3Z?0{B)=acGhAr)40k7|b#H8Mk^SXuhr10&y7KK6ql+)((Ql)W
zdE<Kcn$k>4t|knGm)HMV3D=laO&$fVU=bM;0GQzg0MP$ZmN<O;;QRriEct5{+z^^!
z-QUH<9SH!0xdH+JFxUV9ppM_pX+e5&^@?F<4lVSjcvZsm3a;A<{panV3>060C3lj1
zxCnB#I7=cWxU%T+<}izURJzM!-b=5@pF?07vORH$eV=mN(Uo-d(6BJZO`z00GL_V@
zCC)D;rFtD1U-<cWJ+6S^IRb}lSz<o~j8qTtW;l7dpXpGXfjS6qV4js#sui~rs3dsg
zzb#E-O3ix`2V=;|RsZ~0ZA>?8{b<%O=BC}55tOuOD<MkQFI+5@{1CrX^C;~9M&OK$
zzJapfE+o$RE2FkS>~VZBHolYp6m$;n8+V6*hDh3ce+j24^Oe5C@ZD|Sqj-3oSXQ}6
ztKN@B{IbFLR>~i#GiaH#vnEp+wF&L9()`4gu@}JsWm^lf(Uca!KL%)Bj7(HwZRI2O
z6sda%s75Gc4N3$Y(j7hD6DC6uQ8{Yx@z4RaV{=CW3_3OY1cP<c?EUREBN*~u9mcU1
z2<v#afA<hiVRJgI+QZNYAsI4GeWTMg!bpWh>T;bcm6e&5$*1u37^uENgU11{V3G_P
zpn|;`FLdBDVmo{*Cyicuvn^;TBAjcMvQ0JrT{*Q(3tfte9#eY4Jg)a49u}$#<Bc+w
zP`gr11ZI!$g?i=V9qa%W`emsbL&(XNntPZqg3WQhdtzX7cO;rYDi!~<KVXj!=6U1u
zB3dj?^&y?c`gV>P$JqY5c1+;L8^Dz?L0tt1@wF>_tirt>%AvJ6T*yR%;hp#`+B?K7
z{eDLSv=!t}>76H?R&Qx`x8~;~{n~D~@5Wf3>7*o0dC$esp2#hu`ZV*-te$b<`17z`
zE!0T(Osv4|pb@DL-bgz##8#|*v!;B<0E%Z@p2n-d^^#8P)#o$#mdEg0>+p1_*|pov
zyGuI<exUEC!QS_NJZO?B2)eEr>t=65@bU(E8Z#B_pY|WiP_SG<MuRVQr(CIbQ{2zf
zB=!A#KW$MT(SRv_ocF}QXRwp6LHiF!=BJX*<*T>;Cm&s(b4*)_(d@mw(VioYH{-`L
zUb~h$Ft7ZCNIU(!hWm{P8T<TREfs1i_y5THY<RGH812ONMeiZ7%C`>2B%wL-y<xjH
z+1>h{g+rO4{sv(I1+?mfqY>xx^lP+|FCP5_@4i<$TtTH~-Z?!r!n#V<9Pl3Va0lu>
zN%Q}_J#NZPbIn-nJQWH|!3id&-SuQG!9_`HJln0HPPvzd(xUE+@^e~uTDBzK9W_&n
zS`AFyd3I*~s`haKkl)}ReE0yF^~Z^KU3Qsk^hQslbn|v<KgjfD+OZtY2|da8f~NG&
zp6Al?jW=I*#*W(*n*&FBr^@N}>bSTc`5Ome^F|gVPO{ixSp49Ppj7g4e?0_TEh=2P
zoRSwJcq4b!HjKC{Bc2~W2s(Dl+`~(F16tz<p;TFMZ3H6Tq18|V`_*~6frYX;YFvcm
z^(AqBKJ)lGo9JLh)^E*Zi0!E8WF6p{<;Rcbi#$qRg{bPJ*bTY&(w`}e5@$ai{_1xE
zg$BPXkS~1|-FjZ}frvsEgrTATI+q8A|8jKTrHt(s2>-Mi`fSJWh9A@h;>@qyP)SvM
z>XbLN4Kd9Q1U6Z#fKzf*LX1wSz;*M>5~eE!I^AFdX=8uA7$e07U@y*iosd1OaY&;P
z%7v?Vj^03GnJI2EMv$cwXQ6moYV(<2pB1tl&$a9_NG0t9k&y}T+{jS?ph;an=y>hl
zeppD$^s{k&lv<U!XS0C!s`F&*kt<sd+M2f^LObo&oyRI)f^!6=FB~^aT<mjnfH+)E
zvm?WRj?DqR$=MKQ@Z9#~mJXAG@W=ugV)$(cm9{-m%~D><v&&*nQdJcE#huaZL7_L`
zi2?V7amzbl*lL-br34hf9%I^v#J}40xk_i@?G1tyQ)K9Ux~{dvkuSQj#j_E;#%!BD
zL_0JR5uG<gRU$S}l)~{S9YJiC5L>diUM~C2mAw`=DqG#lQB=Gma4%b|9+{e6(uY_A
z$K<wWaQX4tp5)X@zc)w9Lfui{!~xFjLKB4!nU7p6yq8Pu(~qS~C^4|Z$EeOtgtd^!
zbM_&ovOKiikQEz-_uD%X@so%NwD@1BNfsFqa&zU8Mg;Ku6Ulf`PlcVbfId6(%^Ip-
zolVmiH50uL;>5mwu1Wks-p%dQjR?-_^Xe11<7NpVWIbux0%5gykMVu1|H-jfVB#^k
z9a9d&?KDzPP<>oQ^W~NOBIN-1=#_fJ5*+hExp0agCu&Jcqun{*Pphc3ILe06x$iAS
z9iI{fgkir_+g#(p$KN#VthDZ`BiYw}SxTw2v}8vaoy<1cFIJ6u_mqR?5)T^D9?10y
zi9?o!SE0$=o}m7OxO(LDD`zLOTK}CG94Dt2DS}w`>CYVT{^;&=04LvlEY#33leVSf
zh!^nD2Qi;3rJm3-*Etl5Hz5etEKQ^;Q`3E`<r#I4QoL$<cND1HZ&D&`6r1hws!s8>
zl64Q1QrRaz$wNc}zW_hpKpnweR51fM(W@lv7QNE8$X*jPImuh+R-sGV0FlluDC(0`
zEY&miqegGsNTgrckYU_14tde?VhJXj)_M(WxQkZoL-k~=>u~C3d@#gOpTbrm&Z18F
zBPD`SIF>qAJm{!~YH{)bfz{+SgRHAZy{U(HM}JUq{x9g0GqCD}zV6wxBEk@+07!Q=
z{>*pgLrJYQ9Z`SMT6CfAdhzh$9*q0z?DG>HM2XD0mOi0!Gq(DR2z^kJeocwPr|t5?
zrYMi2N+T>TT3N?XAMQc-LypZspPm9rL~E5~AH2zEVNncPymSSl;^1S{AEQTP`y3Vn
z7O`xBDV1IWNUE-A&3u*|OP|VxU&xs7nU9li;s!4GFl2})T(7t92U%k52vETEnti%f
z0S+CHFTy-3Z0apQjZO@mE`aVBle^|%U+SmUd|G`sdaifgM<_+IW?Z=yYj+Emi3Ks$
zZ20^ksh5pR`SguUccIqkD*4x80V+3lO{1s<Y10Kf4i=c4r#ckp`W*D)<tn}u7D!6l
ztP?TkbcNnC>C>zVQI%GauB-bi7H<3JWk19Z?dq+X30ofw*syT|tWx0Cu!M-4mKxX`
zx|5LFd-{XEa=o=BKnoz01(6LzeT>BSq&!cZ?M8h_-dKC@sTe$|rfG25cGn5G{Us5^
zENBc_y<fWPG_1q5HZ9dTe;5s!=!C%C9M|`f(U3nh5w%P#10G~PdERdBTbLMgE{vcg
zJTfT@B4O?>(Q+BA*iyqE`yP8FyKxaiE2rzIFTzgd1Sk^bq`j&%HYc4r2ZSZ#Y?H9z
zV1ug}1$V70G{#xwIs=pX$n!r7ZxHlJ#6@O98^<~)CEhTuJ)kb)rRLib^s5U|cDZTb
zOIGC)a+6N?&O^ZxA#S)=#>e%xVP(4Dt`-Wi)nG6jf@M*|vC+sGnPAgCc3)O)($W}z
z0U(yCDA>i9v+6*H<$R_ZSQUuqvr+mwhuQ+MBIS{lgl?LZ;@+So0Ogy+Wz{0~S10+0
zP0{%SYDN6ivp4j%Xe%^yUqU^4F-NFUTZr|Clu?USgI-gQbx%>;v!aqIfDu*|@(=+o
zp^Fx3GctoRZ&>TGD#tpW+Y4*$(S0J^GLk(=oF-|H7pw(HzUzc>ph+mCl}|4o5%F~i
z8PFJY=!p3QtY2DD?&*5<Y^=j%OtkX}Yna-|aNFq3q82D1cLCvEtqTQO&A4P1+=xZA
zBz>Y_pG*Zv)?(z5r+_DG-G#U9tOiN;$Pi55qQi&Az~4wmg=UZlRSFXNsF07NYqwM&
z2wh36MzO07sB2Z5sO#2R$LP-M1d}R5>j0(KruX|owqvp}5_Ky{8?Xdv*<LZENL^?+
zV=<J&GGmw0${xPw{=QI+OPd(h!|&)77&=7RDGDVivGVQIhQX7L+S4pvR6G3>6eQoF
zJfJ5{A3`I_t6Kx&;!(w82m*N$YJ%1!B7%<f67=m5KdIKjVCP~?{iuIkuiVIK4RsRB
zyOBywy8kRT!v6q&ToLSjTF)3=Q{L=g%=9)Vx61N7ASj(Ntmo`Odu_Tc4wz>I@78nv
zc6htbaW%7z`@D^FjCdQJGQzSxrb7+hwF%$P?Rtf9>j6XZuyx#x=de}${#g6P$ERii
zhwn7|%#_!n87KYqF@qJ11Pislm4=|9E_4`@F@TC6HGvWNhIgeM=+Xa9ADaBDV!OBC
z!};Aj9@-*S8ISmEehCxefmH=XM&u$EW~HX7*9i643z2V*Cjf8xh2rV)aX99*T&u$U
z!}*hXR(s|uF5GJ@TiqhBY0L?`)2fSujNE$k<82b%qA)=rqki?z8=nv`FU`|UMT&8w
z3o;nn=juagp`LSCK-XXToo;+1<q|P>jmF+ijBnD%CidF%#U9gtQlm1&p<M1~8w;&_
z!=$$;g&jaj`U@ax???brv2)i8SvMG&f1#K)p`8I=ahL9~G?5nF+S_4xnj&3oaI&_z
z_o*$H&M;OW7Fi-ReI3du?l(P9cu%i-1S|NXNMQhQl82)^XeHQL-v@0!sz)wP5t}hr
zbEK86l$AiZ*i`qP-EgoeZ8iW!KI<)6Wf6bYQb)2ZNlGY{lZ5V#D`VtT>zk>d4t#~G
z@^~EqAI0Z_iZKoy8+~xaWlz!KS+Zq{8$WjHj=8)XVS^*(_d%5K=ArsC)#q{(bS<uO
zCK)K+Md1agzT_i0)>2_AXAjLb;mxD{w=CT9(j!}vl>Q8?F7sGU<RhG1j`fgbkKPgM
zATO|Pb@0(^;3t7nJdhMo&&U}tq3-Etx8~R#tJJB75Vq$D&dC;`$!tyDlds(h8U<9d
z<jrlm*`1PvSveDEbC>s1yy+V->8vo{phVYUtf<9~gsljmEXsBLr5;|Z2$%bzM<1y)
z;-H&64rl0{+;URM<hvTp4IR3yWMXo|6II8-5W_mAeJN^)xP;Ug#Hsi#e7ws_J8W^n
zpK4a5%Rbd*bB{tCuDV0IYHd-}R7Xv{Q?z8Tt~H=Gx@i-ex!_nHh3u2OW@I^$bK=5v
zY5RJTRlVu{LB@wh3f$4}elywcl|lr#cNTXTL-IL^lJCO=-lxA%zvn{M(1-?UM#eR)
zy%Yxw<CdFmwc8P}lO$n8<q5F?yS1vbAJ|oGc%-Bn_kdnLE2GCwhwj$c9xvr#vQwAH
zv4(j5*eVFLr^s{~8~ypheVuIT*CplK-?g>#7{TZG%53+CqO@f27rzA3h7ym6?d?Rw
z?gly|fZKY{x57&{EX8)1lWH<W4E0IPYy2OK+mB{Gh6Zm#eKWrqWEz0I5fJKG7PpSz
zt>kD3nI4zUs3+EL&dPr1j*r-BgWC;oh}~3u<4_CJcF)lyR;zCsy&0w&Lw_*~4>!{`
zQz8Dv|BH|od9Rr{L5;W?%=s1!h6nM?p=2lqj8%=uIFnWj+|&qnFnHnAP&YLk>lx>P
zm<MZAF=mZpc1p{^SAB3TyZ<b^WTSnBHHBj?P;e5HAG@PXD_ioZ4)**MXwf~|BpMWr
zPnjC=bU+vNoS$ERdN+$JXt6Dp?DzN+M>Rge!=gx26MVy1-+xA8*f6+r!!k#>fyA?W
z$ta`d6Zip7OxF3n!i>~!go4OuE|{y_QEhza{i~Ru_HDWbkLy@26F_@=6^t%o6;+=G
zD8*q9ced-j4HS}$OK0=$nmRPw&y>2#VwmQ7rS=Y}>#i950W)ymyz=MvbL)?DOt~<9
z?p7-Ss?3%=DclrI<qQM_mic;MjO1b@q%mnsyLzg)2fDxr@~-2~0Na3Ot#jz^eJq_I
z<!{Hdg>cl*(UEdB#l>c$dV{aIEM!DpQPP)S@1+h>_ZCRm-lb_oXasX1Xw29_*O3K~
z0id*QS4pbA1B0G&LnLQ2jjOv}W7n>_3R!b)9f`4}hv4n4M7I12$1}DOGGyI`Ax|-3
zr!<k0Y(W~=%fbT<yGszb7<6x}yYRd8EZIe5`__>u>HB{WWI*>VEJisyp|Nl9QK@!F
zm!;Tpvqf~gkxcSE$bwB<U0uI8S);j?hpxPx?x1F>V=a3!gz=km8;b$TwJjf&aKo~g
zO@ce$pc|VsG7albKwJmVD@N@-uL}h9CtLI;Mpc&#6!UWDYBu&SroQrqP!3m70W4nQ
z8OZCjM4aeywiuTR$Q6`d82)QislfDETI|PbyE<5i2p0nYK>KS{$;sLh;>rFm|1AO_
zQ}cKE+n`dJrhLpI7kcZVCi<X+4$lgSCoS^rIJtb4a2i1VBO(+FlQLHgW>LHv*Yk~!
zSRA%F{~&G|bF$3O*>vvi38(9g9oP~ViW%c>!4|)(4LnHmA>eiW-M&wO)om*pMba-W
zs`B$2h6muV1_|~F<KfndgOd(k@55#bpTlmD@;jYtiBzmtgV97g9G9;Uv}qDq8$3S*
zcZSmfI4kvPC2!u*^M@_4%}v1x)#Du(rdv-DN3Cr+FZq@ohMFu8DG2!)%r=GoY<%Cy
z7nVy)I8mJ?6qac_<f`UUM~6=`PknXAE+E;65s)Zi)VioWQP-?rXL5o0WK=}Y9&VLG
zyVBxw@o;=hwMaenp%!zF_8?k!c*SE}w)N$+L!fo`DN#c;YFPFDw=s!P99s~AXkdzt
zG{8!mK%@?5v)DKavWTFdr^`m9(oeYCveaGx&(Vv`7ZNx-2{B90as|s*$zA2_rb`F~
zwWhzUp$Y8g^VZPF9yRcDY%nX+WQIUQd3G@5>xi4(<AZ7ZML0*F=_P{1d`=J0w&q-D
z#$aSzy*&+mCD~nsM-?21Gl{@E*Vm-Wj!UZNABg-IpTYdC?MJ-Y()_^eOY#v<9*IjD
zv@n+DOtr})S1Kw&Q@AI6w`^ieIWR*>?NvtWRIAmkHLM<BH~cPlpYA+uR_fHX_nIPG
znU=|bUmM9!&HE}59_|Y>=d~9gZ;=t$Vap<Fchz$EORixjQ1|+53M5yS^a=QSC_|A_
zwoo#a>Fi^$*+9G0M$yYx31qEjMC)7y$dXVcbav7gJzru4JPT>QnLb$&dyEE%%$^A}
zNz?rby^2uIai<r?h(F@l>n`xj5V_jgbzy>I)jrE|w?s8P`Jz8m=b!uVke(hY#klzJ
zK!35R&{=>oIx^(|MUHRW`WDpsVT%C!#&nmmj+e%ZbVDX1@d$Q97)n(2@?Ir2#)y<W
zgH;?W^<W=Z=Eza=RN4x|n=J*4Gwkm@o&Zn+g0xxF1Bg~=CZeQxMGoZO4wXm3xA?KW
zvE(agQ6wyIJHI)jJ>2xrGRPsW&5iv9E|+iG`;v;~6;h$PDo&j_&CD3L$_)%6yvlk=
zDCGXZk%w^JERo1ttQrIKMoMWzq4MHKU;OyCaAVVp-<4QTc>h*sfjB6yKtNOOoE=wz
zlcs-Hs4ir*i=sH8uvhM+5Xgw+N+%Ly>qJ(+ED6sN<LR0<7rsfs$G|+sJX$^xiy^BG
zx9sFpSp35$TlIMt^_;Bkp5{9C`*`N!+htOzz9)&T)xtoUi)iZLL4(3)i`H?!!O~F<
zG~p_O*2n7d?4lWp2>PXYpy8f*g_5k?38&d;UM;!ii9+C9W$Ulym%bg^^ap6z1Tn>e
zrl9JI9tR<xq)eslcxQDN89ADJGAHiLr@cC#``fAS?_uM=`6_{;_R0@ieLV>TsZRN+
z+yvQoCY@ek^yIix*J^@y`Vz*O54VV2?S3_f&lr)V3-G<(*!%JRB9|ScC-Q}k`tt;<
z#K9v2=Q~#SHF$;MYmg~2e(I%kunIecejW1?3Wu#2Okid&gFRr?h3$h#LZ42tWK;jz
zelM|QyAM<5%qI2H$#aZ@@fsP;2L6{`U*?0IW=6FSmFKM+-cc(e{WVRg_WTUR`+Ut7
zY||w2E1K_FiV;k;^nRWMI78ody^|bLijX6e@=?N5OjdPR%urcGve2Jl=YC=k(cE=b
zP?<OLK0XI`#SgpnP;WTkH@ua9)GGD<v@kItfQr=w{bo;>{2{tSEj1CP3()F&5Cm3E
zhyM(~*BF<a_?(n*6z;yTwt1@4cfR#5TA`KNH#L*HqlTA{`K9n8X&Y6E8d}{oUGS=J
ztYxwK7FMQ3+xDB#?Ba6oRn}7Dh&_)-vi`i@(q+m`?~b8y#NN)K>nr`H;vpmo7*jrP
z-U+kZ3Hykv;s|uNScYfd@75m?k`Vv^mfxt$m&|&A2q62v%YU{0uI3+|Ao}yaQVQ{Y
zBf3k5^M6k%7#jZbbB2bb0;Dvo%xtX8Jj`q~h)QxmC=%%3SqB6FZB#A9=K?(^4|(?i
zQqUo?_1`Z@UIOqf;P;+`2Zs%dp`o?CwX>n&KYkJx09{%x^!J<1pHx{XI{AmN&?r*~
zOu!ld@dExyP5FyzYHMw7@9bp$5mM2=$RM$#zaPk-<lky|6lE08T_{LQUI758|3;RE
zgy|pkHT_`cVC?)CQ$zTFW&VzV>V00&6f^)Zi4Fi@{2S8^^iQURHRP6S?fI{&{vSd3
zPmBGZK}aI~C-xtz#J@Z3AtC-hD*d18#XsqS%zx7V6fr``zeS9H>nZ#h;{Qx#{YjP%
z{wMk05?c`Jf0NYullq@snLnvnkaOZ6f83uP8wm8j>D2sx=>K$u{0Ys-{3rC!>jeV+
beZBn0@x#LXKG|PEHY~_#@C^AX0Pz0+FY9IN

diff --git a/python/lib/py4j-0.10.9-src.zip b/python/lib/py4j-0.10.9-src.zip
new file mode 100644
index 0000000000000000000000000000000000000000..2c498361470305cd2f2cdecec81c2aa21a4a6f3d
GIT binary patch
literal 41587
zcmaf(V{<M5vt?u3c5-6dwr$(CZ5vN)CnvUT+b6a??@ZMlR87_W1O2Uc*Xp$uWkA8u
zfPjFYfV_#F<Ub=L;!LoBfcUI{fY5;OfE>M8tr<*gt<3CQUCf-_&7A2Sz0_2pfj~WJ
z;&r7k<n`S=VS#|bo`HdY{@0XiJ{y-U@#MW{8uN2RU=J(x?v2__<Fq>IKMV2hW)tKS
z_a{wMP-GD(BuM)p1!npeKcCMg=8O!AskRokBAVzCVCLTDA3J<}#h8aDga*-GE#p&L
zQJw+cA18Zv69a~fo;q~wVQANStI@tK^vBZacBIUW;~6qqLx`RlQ*d3;YAuOjnG-KU
zlMh6w0dxH3hY{WO99qP?=8s+7sq@b}tv%J!Ng1^@Yb~JxV~~*L<Uo_S;Y^2hS{?Io
zr&?6U<?njxyV6Qi{|s<9_IV4yzqB>XNO1$97)vah(s=i^;zlm3!sD^!{m353;S6Lh
zXbl_6W-ip}8;Y~9KUG=?K;q^-K_CP1p_?gT&eUqj+D0Qtt<C(SBf16g`U{+UmHHdQ
zf_TNd3Giqyt;rW%JFGZbWL1Xk)!K|z{y|aDIrqz`vyvvwWLE)6Qne}aovJB~d*uvG
zx~+lJUkQFwR&D7NMEz{2KKg7puf-dd(b3wH)tz_TEc~MZt*LG9$TfTP`Q8^Ye{IpN
z>ZoZ8csKikH6-ep(SA>~W7|I#>_0Wg(QQ5NuD}t*__e%Yzb^8olvpkZ`U5^Ku}XR$
zPK<}ib%*g7<UMSc5Cq1VLZcOs`rK6mVGLS^v$W)Y%x(_k9au$+9pJ#O)q@q4a(-F-
zYO!{{FfnoRbW!DVcXRlV-Ie`EKtCYB3DDcs{Z>!-?zZ;%=J|2M-NWz4)8mOqHy0mY
zzGsk|C(H1x>_l<C9SdN$Ytfg<>@NbrJE&T+UD}NUBWebiwK1!e$gjCQTk%{e)-}S-
zZNcC~)g0Vns7>=kkleNgynp_!qv_}H=Yx!@z3_Q1(F#&5ft{H4iz%sAr$LHvg=g@@
zenTVkP{}wSt67%}cL940e?NVM(-7YehlGS2k8Z+<$o?}n{}~G)2hAp9$WdbHC(zjI
z>*Mcua|HDA^9gYCeBYj%eH<J<5H868F-yX~mQal+jq-9x*3XtLd4vX7e!>{e#MD>~
zUv^JNe+8I+17pB|veYB~f&7~rrW;K8&uq8MYH{3{%h)LRRoL(DJd*%KH=Xs-x)~yJ
z7mD<P2@khn$iBi`Wswr&EuyPCal-97HY&e$w-fQqXmxvtwUaAuZ3RdCZY!wws4Sug
zvy|BNYZ{2}bpTh3m3fz+_&%Se61*iEFPR~nQ6^XDZ^DY%)uT7tC-_f>=qD4zG8E}O
z&fgN(HH38GIvpo%`r|*wLr^e6J#jh0lWvPsLLx)_72(_?$zWCD$na>N5!|3ob>Oe0
zUC@KO=3Hl1^Ww&K9~Bvf>;2j_g>36EiF%=+eCh9W>)?zs95panVLP!7PlCip+%~54
zvly_92{T3o+lttD+&J=qtCfpp^f4pEg|5f_#cd@+h)FVPE}4|mRciR5U?{6rN@hw(
zaw>)q>i*V<bJZwY85=c;=hAgm-Z{bXK|g1P>mu8;CJl<-DkVlO3;h__fmA{upZo8Z
z9Wq-N`svF%$=pq|A7i0srT3h~GIjnyjf%FX9#4tynYDc{jLa6p@}~(S=gPL#GaQop
zWkND=uDzYyJmy#1K$$PI=e{g`DUY8|b=I3GZxI--A)qvEQgc=<K~5T4JRV(t8bp=o
zB3RYn$*jURxw1f!AIYihrTWtFUpZm8kZ5CV6?GCo$%qx0!q=@l(fHV<a|`8<nYYl6
zi@x9M_xxV<IMv5$P?w!KhDkWagBXcu5R+z3lS?n^>e$;nLa8k9uIqSJPk?%I>Et^r
zsy=75k?zFF7(Z*BF0f1yWNE6J)}ymxNS+XrvMaJHeiJ4AM3!-^5+Sf^fF;vR>gnKV
zFcI2kaKO_>x1zgp{cNFcD1j=|N7v9LWgj#&HvHBoGPOuyOp!algi{vdOwvMg8>{}w
zfw7iY595^}@@;Lb7S;0L4emoHA3^5|=E77=TbnX0zHCON#q?G-csu<WQgu!$$Fa89
z0l=_;*6{U3Y!F?h35tZ(u}Lv;-C}Or+h!)<;T|}}8na%36R7F=fthDKH&qF3OlLu{
zaLutdeCUQ@N!p2tRibzQj4ND%L@o@XpkZNH=qrc>ADEP>Gyx~E3beV=TbV^$zoiwW
z67F&jfdaEl>D%R~*Nr~P#R=`OS+2G*-ggs2v>JsBP*<yqxci~AOhRcS*lUx{D}@+X
zCUELbJ%r!Y*SNaIrB@b+9Y)ri2+44FSA<!m30<`a>0QV}H*Q$1B4zPmM2efyEK8%|
zqaL$yL3oYN)_IR*OpA~N%|n?Sb&$!^wFp;9V@@Msi%tYvP^api(Nn{W%JUUD^R$)4
zilVnYHU_7_0gLqOj{lwn&%`~P`-Qvy;|Nbg(t4}gM1xiRBIMeIG|z;MX6jF&Lk7gZ
z+bMd}t`hL#{JbWf(D6q=6WcdE0Mkxo?r<KDgHPp8RY;d<ax$mVE1KqS;zWWsHB%{d
z#$0RWPHfrsqNCrVH^5h@#nR}tY!0KvUn?O||BCp=$n%=<EW5B~z=MgCQ|{Ytcsugg
zl&2DeyT^6ur_>@*qqQyIPg_OEn6OX+c@-3hgxyUdfQ3FU(;G`}9LmQ}C&G$-r%g-0
zsDXN7Xhf^|`>ZABlI%>Z?6B$fx)ubrCgwk+x5PP9cuf@PG-aCrkGDiX()q%UKRj8-
zSE9}Lu%fcqjxr;0_)QX$O&3{5m<f3X(OwL<T(zmFn0@G?3ejg?)T<Re<Vo6DUV{fU
z9J%0h6gn=(aRys1KAUa4o7C^RA4{Z*dGNV!Y1}?9e&RnRk<ydG`sfjfFH|ndAwg6*
z=nL-5?PHn0_OT^lvjm~%Jd;QIHzUn}SsE}4!!pYwP=-+AN4jM;+nSL{!yXkeN^T*C
z74mf3%i*;;pvVT02;q&-aEoY=mR<!qA!h?JT_xbk`vTICZuW4#urB1U&Vu_xdWwiU
zpbJq;{A>onjg=IVQYHeSN%2%aa!<vvnZ03!Qsmus?5E2y#SAg2-^$ekaWC4Y{%YY}
z)QshauFReUZ_2fLU`)#fZ6oM{ZkEpm&PM~W&?lww8MUz>Ac*S*OLBNHNesTsDfAFc
z>YzZ}unX%+spg<^yEeHTtgDC_DL4U;rBC4kmN=dI9)Ngt6&n<+>vMX<qUIjNxk9FY
zDqdoWfJZVsB;#-~JWGmF5jPF4yOlL|sG)b3XwaMCwXt%YGu20u9OwQV4J#)6ZXPvv
zet0<4r>UtBwPyE5bD8YgNO<9awQWn*gH=N0OrW<g;*=OVigH@?;vxG2;}HIn6nQF}
zR%D>#UU}2#DS<A|XXXvOB=A5F7h%~oe?=m%OKOAz`ZA)xL|qEdsSylTS)uS=Bo;yP
zcnL-98C{mbGd)q?-hY7?F0y@xhdj53fF^0K*@{zxBFQTZ6AV6Mt)Q{rKA_Y&q{5J$
z0B-$`%v>_#&Wfo2$~uUw>&i=ePo||N$<J2kO8N#CKg@t{8$qL~+mXXaItkvl3UU{!
zZJknyq}gb#5e_^_O&sLV{B>4&6Pjt=1QOTu$nJ6JS&;48o3fSf2+dxAG_*FW)ceO~
zcWZb!qdogr6L@=4G(RnjD3rwF$YQc!zs6iw@*3|>Id3HKRxQTUi}_pPIi`6=(C0Cy
zIi^;oA9~~i<<#M=Ok{dX=LD3*nX{+Y?>sRYjS?wo3loy~Kv73NQ?RZ^&WgS;VTf}x
z#X8nR^Vl?zz9_G04i2aAHT`aPhXWOjzwgR(n%T@M2A|#FBgPln)*^PpHnZbZf)Zc(
zV#Fov7)bXFu#xU6@DT;x?MLO|0#3FF^Dl9Xxk_(9KAz~jn|qx`p$9iY38&sLIK|8k
z!hMbwEjK^2;Dje^L*)x?&v3SL?y*T3H}#7$pJ|@o@*>vC@Q#EUeTrl|&~s&B9D7MB
zNy21i9ur9CUU(+$T8^9xZLBSf|NMdk`1<q<&i4GVe+)z{5V@Pa9t<|8Vy~(IQpxpH
zxkrMZ>Hd=!`(oSGoT%+{)f8%vyX)=>e#M=Pe+q`3#iq!XhZ>{bn}+4ldQ|h|MRpoz
zACwImeB2K5{VwkHIE)UF*(9Pb1;Mhdsk^^S@IV1Eh`_~Nlk9<R(!MQggJQ{*k6<#*
z9W;?X#eUDkdnJjrs`=#PhIr3ifS=bmNWD+U7IhOOvr!D8yM)(H@Lfd!Y6xa<jf-uw
z4;#QS9?JVp;MuNM6v5W77j^`XRxGWnI~spn0n^I#m7$5%pjZ|Yo4Wsr&Ld<bPnZQF
zh+i8e0z)`U+P$PRVIud_nz2hC$3O*Q0ewasRX8S$I^2$E!pM{74Un2-9+o=X|IzJ2
zSK`%faYAgVbKVr2A)0Rn4owr7Rn2Y1FjECzP}*AM3cY1k77Kkhd<;~<779-MBBC$6
zEDmPD*S5*^Oh-mk{70MZMqlnbxYCD|u}>19n)7I<mxOz6Y39=`u=x9zEk~@y7rK|z
z5uzOZDvw`wX<j@Av7cJI+KM9n&Fk$K_;sJOJ%G55vszKw)x(<llW0~~qeCM`(6x!J
zHKxjiI<9dk#sodNI>T?faw5%{gr4psM5f-)lXTYuGiG>%)wRhl!F2h^Gj<TrTV2em
zWy-G&LwtOo8Hq2yM;eurkG11*5REFO>7nhAc3pH!`iHKCK7?~Wti9hb9ETK?a_$f~
z_*^V6;4epImul`ge8pgq?v3C2tqJ$iWA5rYZHO2TsFPXOkGT`j*O!GsGF*?Y@(LA1
zXjLooF}uw9j`95?y@cQl-K?9#VUL_;SG7hrLKmzz9ja)GX}z<O)S|tIB<tkNu{%Ml
z3zynqSb-B8%J`cdLUaO(#=%PrW*3Q}3S6vKYLa5wY#G+>Aj&0{g-o)>wVeb%SJ|I(
zoWIONzVfNHHqNh59Uc&ZKEo@#%fob-hdm7VCrZ*O7^N3Y!W#FlNZ(n+A4kyaZV1K}
zs@-)}Bv9^SZ>+O~v7;h6YeieYldPJad~l54Ib+K|+=UUpmKh(a7eU`)$ek+8I^TUP
z+a?*R;tyFw5k0MW?ICNUFT>{<p4Gc%gC5x-p=?6sIL{F6VSKxYjh`{yw%RovCHmIP
z3>I1;TDGp|o!iHXw=dXcHd@|l6ep$)Uo19#Le5L%WyS4!*581`ujr3Q8f~_SXw*Lg
zBbN&e)?689Wfg+a-$0ObsXG=K>JHnAn#EO3*F8WzLD(i4_2tHNW4H+hBnt%dAtB_o
zLdWJ_No_iq4fNw&vGOu~FBBnR?0oodC8q7mr?EqE0QOAbXb#=LDnMAJ`D&uk+<3vN
zyZWALErq^8yjRaLTptNIp<heF_mibMdXo6lwp|v6*OD~tzq>}u9wkbcKj|EsvLT{)
z&e#&MgsDV?n?(fV;mM048i_qZCI=Zj+?(-$%=d+_A`yz8B7d4k4u0=E06?L>MS{7v
zlk-Qy+s%)Q&-;=faVo><#SsSguo9P#@g`KI>sU$hzVVv5R8PdJN#}YD(XCazPU<VR
zU?rdgaVL97C5RjAg;@KPBbn@s<so&55kgcZ9^N@^XNnx~g16i(RFX2**}jHswLE0K
z%xw$M&iP;llqzGl0bII~)k`Y42SAwk4PM`rH4(1VRV@w`22vfK5I(*NxgqNiNdZ_d
zQqgKc2OqRFc>Z^nZNHy$4{I8CbB40zAtb)i7R_T4xAlDr2@JcveB1(3a|yfh5>7^E
z`GHHjVsh&2h>K(kPHi%T|Ktseb_=!mfAWgri?e{G&$R{>L8#L2Q(8jL`1zw{F8AFa
zKpGWKJK?m}Y1a5N9?T$C*~d(fCvVPZ%M^WR2?I8)jJYSaEMB4T8fi9tre#$wCFO5B
zkgERe7ty#3&PGLld6z@0{k7=dB;;|5%*!Y7{5K6*;6u^XL=miYEF-n`4G~XGT{hsl
zLc0Pgw8x%dPJSa4x0|QR&wFU(jsoN4;U~mAPaH+iS`4NBd?1yI&{G{|MDPU$vD#Tq
z(m1~7fQQ6i(5BEdBfgVUe0d&GOo&fKC^D`@*AHd8c{0@pZ@fCR=GL`x1nZ=HK7B^m
zDIUk2r80Ah%j?hrZrhQ=?8`UgNo<j`WopRnoRAHZD8vb!p!z93u4>B%m6rD2?@BDG
zuRh7?S`_3q6Vw)hBD&DI*=J6w=TOrVq>hhy{u)($VxS&C;QcBG*Xyv}>~&x+SAmM*
zP*!l&5^aZK0lB-_!)+qpmm3#-8IuC*wETe2@;5^ae1xcf8PA5{J{dduKKLFEFQa5E
zPk%s577Xi;N!k-x__s5%iI*PH&eRBhv++b}TUwcU6fcNx#6yji9J;keOLTanw<F^N
z*JSvA66=#WMUmO$i0C*edE@-#WW>zd37{`Gr{GQn8y<)XO=oD~R$HxVn<l*Ll$1fV
zBOs^W`@z-SF;u#b460fQoDe+89l@v{d0S~)pfsjd+l-y;VTt)SDnM?FqK4v5)0^tp
zddHshZh27ky;rTr7XL!*E4-`MQ#q8I6ym{+d6-8EBIM1Cgl~A+@{C=)ViU^d#;G?1
z3G}5xFbd*3Diqp}tH(dU3(%XVuZQ20h2TRmSScHRzf?UjCsh4|tMr#mKy{}$u4;q9
zs&_p>B~uNd$k9?mC$4;_?Wnk!#m#t{o%epjy;@EKY1<GSu(1o_b=nny0=SWU(w6u{
zRl&VM>asJ8I9$Kj#nb!wgM^Uo?#|BF>%S;`^mc7W&=*O?^d6f;a<rtl;J_K?M6^pt
z2pUjEUw;$R!aq8kpv-W!i@2NdT6!IL(~9_W8umui*EpMAe17<?#mM;)i-6x5Z1Mdw
z2+K_&5*E@K8=?T@s-ekdtZveeT7~)TMNy=6c^mU@j>)bsjL+rDe7RjuKB2r^K$h;c
z<k7pa0z|~SSG`gvy}881(%x6-^MjtIwc)ApdYi?Kw+*lyyp#KHzl-=Iy?q#D8UCT-
zv(6kD_*!4Ojlr{CNWFl5!#O1^=V_ruG(XG5Ex5!I0&0XKxfL#2{4S(mTj~?)wxc>;
zl;(KEQMZd)^qK`H9=6A94%q`sJ}i-XvXpBphkUYt6=M!LD?~OiPhay15=GCMykI+p
zcVHHJeXswM5p5waYmAM@(wN&=wsWZ0Fbt3Qj3Hf1f1WSP^Nc1E>@)FrS9ZN#am5<{
z*U+@tyYD3+6~XB<<bA*UNd@rc6>f17(tynfXzf%*=&<8r0x!Pt3AzvP121FU+-RvE
zjfmuKv}RIoF(#HdiL;8x=Dl#pch1?f;E6MsH~K=EJ<M=J%ikyS2ZX$l<)1v_4f&Vf
z$#8`#xIoxt)I)Bw;0;(1J!v8A>+6tj!hvsb;v8v9m;l15v-$KKG>+kdxhl-W+pY+3
zx$D*;)-Jx7w^~|MN&?B7SFulo)_Im%+xPnXXcHzRFqqi%Nv6<Ofw!Gl-8Pz$w%C4P
zrtUZf!re<X%Rx5ESECsj)^b*N^pz*^&*ddX(-tGW#!1q<kOG>rG{2}^pT2l}QyiA1
zs+Uk3jg)@Pm*cu=u17RrKg`qATeb?4m2O+_oTR@Zc@RmfPpZQjwFO(XwpR0mzoVR9
z9wlciGQX07M*6SBvz1GI3TJxm*9S9h8Usu%qxj1A63Ov1;@;C4*cZ?*h>N)iugUvF
zMjzuB^bW+6d%lARdpfWy>Cbr%@!-iH#5In{Db`(QuO%LA=yeO4@VXLm37n?T`x#p{
zCbbWyhIO7Y(8)bu3emDq1*5mL@ZYTaCC3!<mj61p_!kv0%_)uvSpHgg`yb}(R8ciO
z-QLMVYylGi`E0eocTO_^EmVkDL4P~r@|xW25JLgs(_zci;0)JkKUuRJm8~pNDf<rQ
z(`HtP%zXU&y+P{kFkYz=Q_IQDl`MyIUD}Bg$pA%W{QU{=TPSl8Ui`fT`+yQqI8N(q
zMb8*%D|0*`=Q}PVbrD?Lo=6f!K!7U_Nbkp$n>p_WDJRWY&~g~-$|B<JMV|F3wz{wR
z(DZxEs1G4I{mwfUhB;c>O2r%An=*c-i@6p%5Z!YrM<{T8{nXZ>y8-{>n>3*xQ9+{f
z%`b#EZ=_e{0q&Ey3VV-hf)8ox=&|qM{S!EAw*F!F+5xA+ya7Ev3kwHuzA-kSy7V*o
z;E`t}%s7i@ek3m)$mcI8uY7Dnr<JbAkwXj-I52#D%<)MB@xNZKiy`L|>@*b=P&c0>
z;MHyN**{DIuP%{7Z0bzf@m9NB%@iErzXgj5$dbnEv;KDPux;>7cy{d&VG!B;W4<je
zE|~_2M9=n&*v3l;yb%5GPB(^UDC8GZ@&6GP@JjIBryxK;*w8>g2>)AHIM_KFx&Ak*
zqD5WCev1)l=MoLnj1;mCW1@`lLQJtl4~<g0z$YkKoJl1rRf;-^eiin~7(e+#yDkN-
z0u$r?-%P~NpM-j%6fR6e9!uH#8$YCQqdO&aHWuC!at?g7xN*1)VG&|<)?iFL7s>m6
z04Fe2wuLD09=A&(i2Wf(=J)}-{Snc6W4K~VNEeFETaEuO3Z#}gy-cAgxRf$$nNlWM
zf3-HX396!4rlP%S&9aVRWxAOO<9#gAS(fe@4hxH|Te+XCi{kiIR(PTS9uk;B1uH(;
zK~Z8u8^S$aQ9t-}P`h!0>mZ_I7i=Iu7lxpk8uugzK_g+9D-f{Pm0XVDtXjsq^$BSM
zn^IR<TVwN+?8q?aBC>X0APv&Pcz128>qAcfEgaQSMcm;m?-s43-)9t4O*#{Ih%D9W
z&N|kRxieqIZVCpC#H7WReQHE3FKB1)`7YAA8)<u)4~MKNRuX$8t3!KT9*hISTL6jN
zffc<+bVhUBvX*=d_C#3JVoR4Q*Fd?c&IYzK_kH9+mrKxE%%@KXJokQY)w$b_51Bhp
zMAvtukAL8Dm_|kC@uq0;-@TG+sWtVE)II+62e-K}>&gU+VeTpu<x{9yv`U3#$-Q!v
z+=VMGabkc<Aj`%SV%97tBFAruiiB^mvO)BnKv+(%ax=m%2#kQ(%O!=Ja_{nXAJ`e2
zKOP&u<vZungYM^8*2~WAdytiR#kjxbJR5&OqonW-6E7^slFqz~14$PD;(FmpR&*(C
zgyB8&@^R~PAM;bMZ2bmOZuC`#s-Ze_?ZBLlKq<-Z8R}B5E`o^z_$lTm!A@GK@9NQ)
zo}%Pte-oSyDA!Q*l|X)0(SRff^7gPu@_A?SFitYrq9%q;I|aMm+;hug&YmHm1EN7`
zKQdcY4kc_hBGOt@yfES-IU=pCUYtQgmqzpt1Fi6~Ya;TzTpV?_M{GDyT4&VQq=F5C
zxW!_KLXWKB_fex_AJh-f|GO?KAH|_dpg=&2NI*cy|FtgWR`y1=R^I=;+|$^J*%C(z
zSTkVKCA7xWDIuRH;XKou5+br9?%_{6)R7^zajDj8?20}L{kqNjP9)!W*y&h{5H9-T
z!<ygsJ_Gjwk-kSqtO1^p1NYP2qxEx4HZd|H_8^X4E7*YEtY|nvj$`OJM2`AJYCR6l
z0>PRsE;Y1PqD?LL%dxGwe2kPyD+!xbA|SPaO)<fMlQja88p<I%lmPp7F!IWfOlDGB
zn>xFnZKYG#?+w}41TQ@rr%<_E6~0nsYgg^m0;10$tz$`Ej^aqp32r_}8(15MQahqH
zFBx1~OW8ov!s7d>ee!y5*6j06Kclvz*m2L+Q7xBDR=B9+@C`}hlrSJ$yJuv7ss-|-
z7_Q;SQRoi2E!2i=JtI0?%LW#x=F@E$bG6&GpjjlIg$l?61!VykOS^fs1Wl%IMVD})
z8qYj@dS!bOwxNr`1+^!rwSu&wOAqG8X8tut%YAMXOk3uJO7pVd7oT<8g(>@7r*Lz+
z3q4u4*3AsccA$nu*-Kf$@%sRSWT+Iw!uPd-08~c$*wWT`$Pf9j)ZD0=EO3YleDz<5
z?+?C5N9l(3_wL)wO<V>YTMNOIDc4d8Qp*@M0ApMbh{#QD<Di+rc{D8rnahYU_;LpC
z%|J_6Lwh0(b4mD_dY#}`hHKh%Bv&4_W5RO>K<D%OWVLm{<>t+V?9TO$u`RBn=?jLy
z)KIGXKsImBfHGk8=Ax&eZs1^lvMcw;aw1uhPtapr0&0x1SIDM(_q+~pb~$Pysv`Zk
z1sVCCmd((<@P~F5Snw&jfAOIxWG@2GZJZNQUXXr9V)~%l={G^gt!h?$cp+|)n9JE9
zVtp%fKWqOzHrWHs3D42Z*^N9-!ba301)H0y8VGeI3G<<5{8gHnW;)`~hCxK0yms6v
zHU^;;l1}mQ65+vAFch8eup_gD7)VIm_?+5TlUgd(>DKXt46!oAd0iR04Ncv34vA4p
zlaAiO64fTRA3-f_w7<GZv<59Fm2f(bZ-Hn158>5CZb6|4v>#g5i4e|wUDe$=D}J8M
zh|YCW#QiFQL;skFNKmfnP3A7bjoGrlunV+CLnkfR-lS(8js9H~b%qR$j(8k!@%l;1
z<xQL@Ffd3s{yme7|GbPvoHKd4EKi?&;wwjilgsQkn~x9Kuxu$*`_DUTlHrHif1&*L
z($D9rcLbm7!Q;g*XZYs{-RC$M-=IFgqXXdHLeQc4=ZtYAw7j9eG~a8|Y$+f1FHldj
zd}RQ-(GTMI5p*-;k=O9KfXa>wyA}7KtYH!$oX_#ZD*N~UN(XsdE=Azuw6!I8ARs3R
zARw&&N(XBrcOwH62U}Y+6IUw-dzb$@tH<QD`v1QMfUbeV7AH!;YxzDG!{Vx2iaQ~K
zkP1<b+wG#vB9S{LWoMZ5lxvN8x^GeLu*Q`~Usj=H(#H{})24Y44yAPp%g}-2zcFm(
zNd%vdDHV*e6v~cmi(B#f{2x>u?j8F!mu&g;M#HSc7}lf(tX_pBxX2Wmq?7*s-8&kD
zz9Y?0uT%=!*p5`;CSJ4CYTt0WvSt<6mkmn{DRzy)1D(!(dxGYBW(%juKgNTjH0;PK
zy|It}<x5<XU~`4&$2|woK0n=KfZLmEckWML=wj2)D-OU33v$eR^!F@-*dVEtN~ZaF
zF8F|J76S`17;QGgBCW*~Suj-$QM_}~z2*xR;S{U>KN3Z>yjcVuu$3}z0)`I;Z@6&5
zLEa;y*->2>mIKPdS?5;sU>1LJ!FA0@u<SZR3`4w_y16kULMUFUK+ZW#=iiNHDV)il
zVDX#sjT08EZ!Nqy7C<dP&CBAQ_l3u+b@P_-?aDIF03QgGEeV<0b`ym_Y5&wU%SH20
zrB}0m@nv1{%Y!Oe@yRv8s-g)_EdID-51FG-^Y|*$6tM{|eN^p&obtI_snVfXkyy3P
zRN{Tm`1$hGKXVq7*F!>7Auzz@E=+VA=r>jdaEkq=O*U4odq;aj!=_5Mvf175?ifM9
z1?mgT=Vy~UheR+y-hRVoaK6B3x*IPrRO9pXtvWphLZ%zdn+co9t#mDMrd3Ki(>uP%
z3*O8`zv5xTxpXRf<@<bzs|~{jG3_^-BGnKWsYI6*xGxz@L%=Z5n$;dlwoVe6Jc2So
zwJs!rV(z_UcH{bqRE7pu^2HRc1~Av>(lPJ?0VAiaFxB-Dg?ku(lT2tFK(FGw1_Bx9
z4MzGjo%#g}L>oJADLJDd!!atYO@^KVLLpM*g8pEgUJ642>3xlY{)^ZJfJPq`7lyJ-
zDFKq6C>@VBUyv{qq6$<Dq*FpU5du2ERRAXT45x@QM2>Mo9OOia0`U7w=R9%zW1p5>
zN1PW437rGyG1U87U<2c^O>J@lV~H=Wgwh~3MC#X!&$P+GK8^a(+Qh=f%xVw5r?tu`
zG{#&Rp)^mJEqH1xtI&;3rHfLUqbANwb_cyOnf%^x3r7eh^&kAF(8(vAYGKJ|S{Qi`
zca83cGr@pOPD=(f6N*QiJs2QMjDDkKg+<TuFY3pv4=lOawH?j^bX~C@2UPBIh<^ZY
zt?{^@^4J9Rh!6~YX)~GJD&9qDMp{fuyF~Z6=u#x?m$b$U8V@kEzE8|&eb$nwMNp2j
z`89jR057v$SET_%*$NRNj&5biwf47PhAb!<v;XgXOb81QIRL)mCVc^dt54$ZNx!-E
z0em9bk1N(#VhWlz&}5nN34gzpsH`B^QI=qZn!Iw&Cn5wy0SHf;kJP|kSbCiU4J3T~
zRu*z^q?*fDvMgekox7IBEz_nDE!8kWSE$AFrm%5RO;~(y?9U(G2s4sHHfvNB3dWf#
zD_<JL=O9pLh`C#vK&OcCTKZEZ5)Q^Jithx?6}pVQB5@$x2M{Z~)}q=$320c__85?z
z`mTWM=6at1x6j1-Ij<Yyw%3ud9sFM1_g;|%sc%e8B2n@bh|0pRs_3111ECLt(ldHb
zjb9fWa)QQpdy+s1J1E8J8SP-y&YFr)1CL1QyUCXVxZr{F_cK36yjaIIFzwqRi9Q*q
zo@75s6Eey0_MEbBu3tEIuL*rdWKeqK#yz^>z(7N=CV2!2P*UjFrb<S;0m3+RUuLUe
z7|UCAE8>`aEmF4U<uf%TcIFCt+&IJjqGn_vMWy;^f)vScUY#ioAfdEoOMCOs=VA(<
z6#LCeq9!ZMW)4ox?J~P=G597OHUOQ8&zA7XbzmFuteV|&sB6nPAM3mvc4on*c82Ac
zIewwFR3gYIZa&eSI77QZM2~Q{5EA4uLL;-Qp}a;JTOz3<sW-T@2vY(8qrkHLB}?%x
z>=cGM8%j2uXHozS9G$`>$O3MG6$R<S(><7qMf03F|2%c4Od<in0*@NIIveNl4wm3p
z(}FXJtOrJXIx}g>kTz|A(S1BanF<_KNK~;4j5A&%HX`+suE~<%WC1-@Pf^<}n8w9w
z@sjmZIQ$e^)1(C{aUOH^<7fU1tlwAYkI;nZ_WC<~zOVk|SLoE&K3jxk8z|-=;{5De
z#dUkv)d?_6ARW*I&JJcL9T6~vSf9+xsfm;o*rZjOAP#ROEOnZ_nGUEC%ys+%3e0V1
z75x}QdhlFOvDul*jybBZ^n4~A`5|{d6~WoKbDX?XN*SFzZD2ag$IrpY$ie6S+4cGP
z!N%>t#=&_Y-}}<SX4xku(cs;VgUz`d0Jr3rW1kp+IC}My<!WQi%T<wA>^b6k!7Z>L
zjc90DJNSMuGOqX_svk3rHTXQ$i^eEQJWcacAEfe(G^WD5CC%Li17(UA=FJ0kIfQ6M
zx9mPj2*4wRCm8*LLYus1lu`5U?CrD<ihR6n=e4QAhq{p*gh+A$vQ{%$&{k(3A5@QJ
ztT6a`fKz^tt0)L=az%`Gy2ZRE?2TbkV<Uk~xOB$%bUZAB4-#+SLjx*dC*rH-3dFq$
zWh`kAt?82JLMK}<6RfoRu!NWgZo4H4bnsM02J>fwjgsmCO3=u|B35}?rb4^gTotu)
zqq+q#ISK#}JMUoWWe&HisAZLN|GSem@)v0^1g@LLWK&x$^N@4<`6lvwD<*Ci*VU{E
zdSV(H5Rah8Edr75uPS1)BIb%~DBQT<$4(Rh^#Z5H6W*V$K}J`IJFX(CtWjp-Nxnxc
zYTv8}ZwrnQNK`oxivtnR1B4J`UX%z;o-tFYn2lRfW?Znlfcz!G45g%P&7?fSq4=B!
zN54wyL{F>@!L!n*i^~S4q`VtL@FssE5VUn1<S(?*XbvCY#{NoI+}8ZChm`yi(hLb%
zG6spK9_9^lx$KLurd+<6;c}C=jURs$+2u_zray#0Y0t@CsRR9FS0l874(H^zz7=r4
zV6lcAi?VnkFmwusL8G^xe5Sgy+Vg&-iXJ~CtQguRAtQAIi5lCl&B#70LO*5-(^pQT
znOGftrnqyuZyKkrD9A8`t>3aZ^J|JyiA!2h<@mX5*14Ecb$pICEwgNep(hTVAcci}
zGiHD|@As4!|F8FPVyOq#K?il`x%p|g^Qm<Wu0BXM`5E3tt|j%v@bY2H+g;yQJnr3#
zJ5RckaEnbYFwuUf6K@ykx?xQTBDP)X@i5UA#1M!Pd`kafSRWWrQBMahnDwV4#-Pid
zt#e165DS%k6^dQWiS$ypa}|=#bl?d_>-hkti=WTX$DX=87FWpjYGV}lzy%|*B9gX#
z{nD(t2gzc=^duJ-VKYq-5%L;2!a(ESM9`|J3FE`|P?{<lfn_#g7a=bCYVP$#ash?k
z3Y1z{fTiAt<1>{3E#pzgrKqA*>9F?}^*y`?%4kd*_~?cEW_WBuv;GiM+)<MTq`0@u
zTN`brPRzP*6&@8N7o~~Wt;r=L-MuBreQOY4NzkYEA?GFTc`%2%C2mUZ%U1CL!_;PS
z*LYzyqxWs2=4)i`TgLFvKad$qU<RTr>x(u5cj};va3dcNQ35x0RMZ^dDgdzd%Ud6i
z`S@{*jCM%~`-u_mP(gmztmU4NudNRcToP)ZZn$SdJY$*QglvNdC@5>=^AKj0x8};i
z0=D80d3rKRyg$3%SU9l6wHyW)55xJ;BL=@W5%09k%hJnNNsS(@>{*Q>(ofU1T!lDq
zZGUIpHQ%`gEP4FdqIb0uQK4r0TqD&xbi;7V&{Sfly-$x>#EBhCFps+FW3B_mB)54)
z^{A?_^Mq;Q6m8e>(%K`lulMz_O;w9Q*@9H;xbVs=^FiB^2_jFYf#9?ZNkTkYZ-$aa
zj&W7e{@ACLqjndE6;C!A(|)Wnv3^<Jn69SiyrsJSV1fQL_1iJY6Mc2p4zOg?qws?L
zb<er%YKBa(DBa(KbT-Rg*-hv?nq~9#X{^O2M7W(EzRPQpjrN(YT`{7E2!Nvx*HNq5
zv#zrN(C-<Ah|;t83VDyZxYM1vr`jXj?I!Ct$vhlJst9;hYZDXDF#){e-@ev6uA2gT
zJ+qpzFP^#VRVr*c%|)H%gJpQ=MEdP2$<>J~NU+gHtk7lUa6{aDSICWA{j?>!?_W3K
z{CW4DIz0^c{^95Rvlyv<=z5|aho~Qe6Z0?G13W8#;%f<O1I{vRp3BTqG(oki0oh}7
zE|7mOjqN+k36!mD^hMOC<_-QSZU1s)u$<kr8n7)DNs<QbRyxOrJ+*LUDq0<J2mLEA
zDJrPV(n7^{Iqdmyf7v#YU_q{uSj^cYPH%R_uCBl{31qhO;(Z(T7@*U-U=m|EH4p_U
zwg8Vq*33!r-`lP+*4QiY@B@Wdtv>UPUq3AZkhmHswN=6fXK$@DH14hWob5p9#<2e~
zfwg%l)`Vhx_;4a0jMnOhs*c9VzaI(}HT7mEMok!#=cI4u;=A8E_{RQ+|2pTxZnoeA
z{y>8}ja;N2S;rjVrYt_<!Xc9I3;yY?s*ya`D1Ht?i`{$CfaHZF2j<g(&I{9#I0FF(
zalRyEQw$5AM3G0F3t9Ml6iOKJ<p<ooMA)bDJzY**&;5JLftP)s)srX--0-%1{=2O=
zrd}^%$H&<-={8pARKMZ9f3&cF==k(<I9r_4=iMkD_J4m||0g8Q2ZD7ZY5)OIcme;P
ziunJ6L<=KVGY=!L{{|4Rbag!bQxW$+tv@S#%_T;KO2@Zwk2pOOO1G4VvL{t?Zvx;o
zh_Ono69w**WTzaxU+ZM_0+*nX&r*Cg0HQ=ec2-ta)K(1CG%lmoKaCZpA2=s4NS#=`
zT3!v_O4e>)=N{Un(kfh%Ok1c=YA`NK^rOFCda_s0dC`@9X>LnAQ&QbXRh%gPgKa9t
z`u}K%eKK;qvn+PnjOt&-)1$iO)W|PNYeuNGXl<Be%vz3-FG^ANdbv>5=w$dRrbSam
zs%p%=D!oB*UUcgMZJbgqJgBQtv8ZRo($|Vqn|)i9wPL6DWJ)HzWDSoncql!yC<s+M
zu|X0fN|Rfn!4>(`S?<+{0JpclJ!`lV5aw#}!!kr&s+HPy!;QKXvJ~jjwGt@_MP4c#
zrO_lwHPLujmQ7$KHASpKT4icP$98Gua<wG(3rB!vUOAy|#+*slxD|!Ci}U~ZK+XP|
znYHdi`w@y+i!u@5uhcTJ(nP>_`W!#yYb`ylrvzHnzgEh}tg5T_46ni_L0y1uydC7$
zIRD0a0^3<+XIe9&Y=eF<lu|uh4>TXt@Cp^95qE9cmtvLIob1uaC(>#WP|G|#1RwCP
zbQYZ|QF>Fdn;t*naBU4*1L860JkXWl_;0aN9FF(XKPif(V}mMN7H7KCi!`s3Y#ris
zue>2^D6M;ogd)GS!+Yn)!!0MRe`bP&N)kQwr&ao=Cy@TeTZF;vg%7i2^c}Cxg%?mQ
zn&wWbfmPs7C8`B#TCRVV#GSg7>5^eosV8$fEgKLuMW7KulxhnM&}f#s1S_m5VA0L2
z;kh%<ccsv0%;=URnTqf90<4?z@Drh}k!uB^$?Bl#F{>IsI90%B2fH8E3ZyZJtg5xw
zo>qJic{MBtKK#(mA(fUdpXX(1oOVmjIf&v_(?IRO3xD|WkX;(HVOyjxfX}{oi8vHT
zKk$EO0^t&;0u}!r0}czsJl0uKjf%?ekO9w013!$ZPPW<aXzj2{o#hU11gyl`fjnAM
zaslg?J>6c<*1qU^`CqSC-aYHTpL^b8M;Z|v<tb-(Z*SJVe((vObAVrqcEn$w)iwtP
zhB}J7i}y7Hzvnccx1r;ln8&k+i^EUCFAN5}h2rA-6SC~w+&BjNf}!(+g}0oTrK5ZF
z_+p(7rFaAd1p@*CKTN%Z`dy&>>znzL%iR-1hTgckKAw))czSt-F^QCv7#Cyj(-{7?
zeBGUW-+&tj)fGh1fDi%e4<+`hU+DBQWpY|xYNt_o#;S{M%6>#b%e)|R5QeGzfi|;!
zq5eR_KZ{*6`;ui;+=Yeh6vl(dpwAfD1fFkgQZQ)QMUotUk7Ud1PF$%?J1h+mfrmou
zNGA+&C(NafU5p?&n`(pB!JvRRn7Ff|$Ft0{VMT~b<cv>62ebp$qELE%eNf3?@3EIu
zH)tJ@l2peTRdg6qRG(LwE1mkXhEUI=Ov>RZjzpFSaRXkqEWY1uE-;+e@H{ieYKT)l
zW21cQ(7BgBT_}`+uvk*G*xhpLo9A-ky2J1@fG84CB^}DMzkyk%sy|-yaFm5wzES_I
z4E5wPA{{si6XSS^$1poYpAF$a)I)gtM93f2fDG4~z~BK7#<5~xEUO!rG4YAhBmJQ@
z(tJ#}9Mnx(HO(SAR>4vWWMp-&%-%q0+<%_9OnHPJ<DAEBXeBEL%`P}fpLZNavOyr?
zn<g8^3<!c}uKq+wyu-6p2)$pHV2k;u6%dL~t%e!X^d@Sjrd^rWV=2C0^siD^b$;RC
z@$SspmlIp!^r#IAJ|0L_w7#C+o?hUY)n1gW(#pz9Ba|^mS}f2_Fm<KB0$mJVet8WB
zOoT4Xp6Vf(1HdI_cxRPoS|E^?ejetBl$cq^P*i<C^Ne*Lf8M3c%tCCGvMQBSG@Izo
z2-*<LOzPYw9r*4P7;#LER&QjMxQs<fL@Lw6jD)2O7KugS?c>Re&F`P2&D`%cpwR!&
z-&&yl_3?4Y+S$wbVb2S_P0Fz%$cyVe7`}&Mjx+O@PSiQvM_%<04fw9v{Lf}m-wkbh
z(tO<f8ywe+w@+0gh+0k|KHNvbKJdXtvtu9OU-Ou!O$c8qnM8t{$v{UcWywdKJFW!K
zwB@6Px8|9`^l}ac-7!JIdBq$?kVEWw>nA3pH~c3fWvWaMZ$qTd`?r&s%UY^*q$L$y
zmpd5Vz{Q6(>KkO&&Za7i$l!q8-_W1YMz9|W2zW37m~5RYHUBaD@lZYifzRb%3!672
z!D;DLo`1~S2Ppg|+lsE@H+z9vzh7S|^Ubh?Gg>l}(-dRk#9ugx!=R>amdBRhSLVVA
z!;Ju7BtM!dxO7XwZ_i39z@enm-H$!`4!Je{**tQA^f}x~DBwO^C9qCdl^O`oSx^hz
zI(-^7ar|0bR_~Qc8p<09xqu<DxBpySP{b;Mom%lI3_F0#F@|L?5yVh=or%$Jl&SXE
zDBqfYom#?Pv#h@cUbP(xG~050SqcXroG2i=+LPtFqV~6w-0N**9)9lum>VO`+$uP6
zzyASW{90pKyPUWH`-D$;{ctz8zt!2*$=%)4A1{v(5I2%>W9I4{Ze;1|_t>;FN|lVu
z7**)-o3BOa@1~3Z>L6uDDiH#S2kOQ$d2)c>g|aTWik93~j!FW?WR({5!}s(eGFxNV
zC*Ne&j~0S3Aeeiab(qhehrw0b;<El*^x6~)qs73t%}j+VO2@@Ee+jFDzW-PGUSnEX
zNP_j6^-q_5m#b}p=l)JYO%vlN-XZZ5)H-IjFx1?~sAJDD%wu;lkXw;`TaUq&#8Ehk
zAfM!t>#2r<A@s$yc*Dfu>zyCK^hG8&)l%19fQ<oYyM(MJl5(g)WIb;1j_Nl*Q&cY?
zy3=Hg18%;?KPar07@h&K(U9lb7(AG59{;d{ZuI~m>s$V?BE3m<c!*x=uedAiTy_%}
z{4R}Jm1*fFUqH}`HRzr+n*u)wC}P5oD4Nv38G$>o^2wh_2nT!OaEIaMzOYXQklHH`
zU*&=(o-h-9Uc0IXDr?^=Dd+}BMvz72x41m<m$bVNlh<-9XINXK%*XE~tglnROAb2A
zrNzmh@ezFC_@;KTtCdg6DpjxZnCt6{p$KijNzLy!>zCg@UGkO>(tlsLsvjDf?;#&@
zU`#tA^r-B5ha-j1qtC3dA5hea@ORA_C@;F~z4v<U@FFT3rp=|uTR?3kA%5FtGTIEH
zDSvIY94O78kg7sF1niBZzcBw$4~njjm6)4Gbyz1RWgqBcf|l_W*qLb<^c7Oe*i#xq
zuhYx?l#)FD7%H=OQ6(ZU*1sWl^!*uKoxUb1V&?6SxnrD`HENA#<M4hI(WX0w`&^-#
zKxvqNVvBK^JdI>z&nM5mTM0d!oYh2QA7*4iv>qS;Y-EW@`F)@gDHoYED|YK$=oP3F
z+5|O#bY?`S5L5V)6`K4>r-o3X4*^OH4_5ClUCu)mKj&0oN4RDtgIGH61mnRooI7yk
zKEBInC*z(k2$Y=bS;b0dA8WJKsLbvX2RL4pV4ci8)TE0SJb6O2k_L?|LL6p}4CwnB
zidNDlQxLjaD-rFQXoVh{(zAPkf~rDU(2)(J%4v#tnA((vNd$N`Yl76(q1Qma%%aIe
zxxo_#rLJ`|%cNoKQ=AsyaPLJcu=R_(;4&c;iHz!Qky_{|n*F5!ek~L<Z_dr3NQ*h3
zTdt@2PJ>pXU4c<BmeYyVW36MJ2f+ItGiK?V@wN}O6=GPFDVmve2n7`DRpNWo(KHhp
z@`gJJrd#-AUPqf@vlI<!!eJ0}UUuW%hn#kY$AZ-}<<o38jVxdHsZ3`m{smdm$mJz|
z(H_#KcKzDPE_v4FmHSK^z7PD%lWZKpAv`hJG(3_CtJtJ`af|@0%NsDXCu%W|Gh4gN
z>>wCJr0ZrQp1i+Ye9h=~Lsa1DYn_=HE|SkNw-K+{ui_fJ)GsIb>6QaSCK9ZlO4Z`K
zKv9D^_E$6;y;g^upN#S>U6q^P&y>jTMmM|KwI9RO`b>C4(84TSp58V?YUQ50p=aa*
z0g+6^DTr!@ZFJ6d!EmEh;QERQOw&wx*|VQ!L}67;`jVDvB_<7Z9GnYjQB$^sQ|WjU
z*Sg&2wq(G^_oZY3_ctnq9U(z&zgNn^p|8aHFL<+75IF0;y{nR_nG17kd9zYZlThud
zl_g1&@Z7GFkhDlKfiXr5%UY`VbJk6CAqpslZ9xXnS=}ZEfs@SO2=_RSW5T<GS}<g8
zDVMi`IZ>r+FkAy^cQGG+{~$F|TgAX^nR!#u5D0hl5Sx6d{yzN}aG4Anl012-E3~Gq
zS7D@HmU^Y&r*J9-MfLB|)(}U;0tcqWQF)t`xFy!w1fP3<(nECt4AWWkN<)eJUo74h
zzjHp$=r^ayz|Py*XWF0pN1WX(uqoB|2;3LR!4SrNV1JY6b&@bk@HMc42OM^MNX@%;
z;cgHcMZ%LRNj1UneFt6%x2zj{zLhb8OWseL8E_*rmF4)+3Efj_?F(r8RMemJT*L>>
zOd?3dWY!LqpxLb2ePGKBH9X-rbahKL79fCG^5;jKWN`}|0sFP2em=!~138vJMf<)e
z!mWatf5tFl18RZo%Bf&6BJ~j|t4lYW``TiV8?P~vvJJZFap0wSETC0=LQ4ui9JU7j
zVPy7SMh0$(*w$J5i4)!COWBly7aa95b7ZOk5GX5bkG~E~$ej>j1~#YqYL2j%B(05l
z+oZ5r#A!?7!p?&hX{2Jk)OI|1iD$0{hv&2jQ;CAsg)0^^(S*mMK4<NCUif(mU0y!N
z*-R<kJMiBt(?jl^Sk!y&!D8~NJ>VN&B!oZYhBIMRkbak3AG2Z<E^DRuR@#TM6X27r
ze4@SkS$Fec+cBh@v#%aePM3iAQ-cYz=Yonzml66b<Nf)3`#;>P%sB3k_pN{J<^@Q6
z#b?1D)7bxGg}xtu{^ZcIriO=Cszi{|xE#ac?B{7Zn;HREpevQaq$d+!jBIgF9V;*2
z)@TZb`?Tf}7XM0efkDLFHTrcD5A)D(E`Cgx>wFW&&HHkmvojOyQf$1u6l@R_d~Knu
zxYtX<thALA^agkbf^lpOnSZa+%H>qiQ(-gN;<pT6v}(b*7R0r0?CUf!kx~X!$`WkT
zr-kc5fRcHO6w2w7f+e!r^}mpn!xK3Mh}@p$6=mC7{so2HH%wavWP6MX8myW;M+hZP
zszSrH0>Ntd1+Q5#oQmi{{lI%PNTSzoiCgS>P;Njd4C5%mUMJ$+9L~};&%jK@)e5<h
zxd{e8h$b-h(R{>*x{=MNJGKq@6|sZeiqY7!+~L`-qcO1(@>WrZAO<ri<qk)zGKX*>
zDvm#qPi$LETb`e6%yJtAL;t`IV;a2WYPIl<8aj$~AO$1mGC2N>0@2g@3_Kg5g@)1n
z^SzS#L#NniEG~KuWHbkKigqy$UdWrL!nM>9ERiQCb7pOM`Wg#6e?a0D9Jys}=Q*zO
zH|O@>>5k>?MnZVGO;z9?j8N6~0Bf_*uq1K5pJ!+4MzGK-cxhC;>+OKw1;$%$X6W&N
zyCEx_6*j~WByYrky#4GeQ{zwp9;xkdg2{0*xsik*PLStX)^S2Ykp=wpqZ#R(l<)8N
zE8N9@P!c9={-3m(m&$A^|1}2v;wSajhfF<-+cXQRi<2+rV-#B&-2yKsd9tSODImql
z^GW?7q)2iQOyw=XGujdepSn})dRZz@h8GF6DXm{)QSTY1)o)mXu!f$K@^Xl3F2+S6
z@G%mXC(2hwE#OW0%*_`{+^#WRFT9q7%Ki|&bOtqT6mAns4KkQO%yaS}c}Js)9yM~0
z20x4jOn#mOLuJkP;%jKRnDaSiLDpQgoa$6=a^4$8c{VF2gv)-6JIB8DEBBQG20zob
z+;bpqtbaHC?p(BKqH6)kUtvtZqhyf^ALdS!>S*ZEq433?ZJ;{k{`~$%8^r(FTecMJ
zRUM_BD-=1ux78X3ue`4@w1BO@_!2sOQNPr3Cu6<^yw;+#y?-MUdu#mci-a3DzX#6%
zo1H1ZOyX`b@Cxfyf70@E)<6C7q-c>Cm&}+iIVeI0w=u=iG2tm9Pm1UBG90xQ!tQ8!
zv`sB)0fE>}YZNrbl`zFF^kAc2qu{lm5Hc9+BK3sN^1_gqj#Pps3nk6xI#1<u|8}7O
zxt?-b?g6UJngX{6TF^0twkeHjb6_~P=5cXL%A9%0s|(LiGJ9$r{52p_0BH4_pVh&B
z){J95^nU<1K*+ym;f~YHFjN%mxevM%rEST7<g~-PBemrwI#z7Hayql#ghigG!`^Rm
zSJ(fG?y5EUFYohS|L?y39QXB$4(oSuR@;sWTdWkAc*j%mvwJa#1cvEAai)yDaSyax
znY97KqhSbM<(g9J&c6}Biv&wf#B9YjskuZ}c;t}Iw1IBQIc~f{YrKf&jl%;iZ_41t
zBh1TF!d}OJUT)L%nn^yO`%V6i$@pD8;ck08J?ecBNGl4$vNMIL&VgqFmTXHEepxRb
zP%5J+Nfnt1M)`c88|@m0a%El_Z5Gp6e}#gjdlQn{8GEBNN9+Cv<#IWl&SG<9+e@IC
z7}*@RIn6iJ9$YW%aG`m)uRRSq!rhM#75LV$7@(SMAk>j5pE8r31o}?<5q4^4k~DpN
zcDQ5YVQZ1wVT<F$zq;JIn~Dq{cr5SCd--%MpAV7=qxnzSqiFGT9wr7_M?(&FBDRiq
zZ2M|QMXZ{t_7|fi^(_JS+tHXQUwePI3Mo?yZcJa94=0;K-C(_RQl02&Ud#vW3AOay
z_o0r#!7d}X@=jE!!a9}NA|#f{W4al*b5Qh9cUVDJ+sYYmL#TcsOCn)6u){5BwLNUx
zRa-aBnaqwxS~uz3X_l9zs){^|>6lA)+27@yd<CM<*jy_ni_A|F;7?2oIBER@-<m8%
zMaA#j#P|$MN7D_uWtp#0B3P8;UAgq+VY-gQh3ktoWK*I(oto{VYk+pltz_vFam;@!
zLe{lf6?BZs2^C2#^%AD}qQa~2roP8ASh-{g3yc}A%Z-=P*Q19lBW*E6!CfwO&D;5V
z+ko%35;2fcKR^oL#^*Uz_zhc&VXprT`0A1M_S?&XEi1a<hek-Pf6a<?=#Rv;BZ2n=
zH6Qn{Oozaqvp^jt!jAaD>u+u*(be(WtIu0Hx=xAmY>E56mBg?UJp`Zvx%=YX^+j}g
za{hXJMX-%dd-plKLewR$t|xRu1SX(iTVBqRTW5>rINrN2ZjM^<icqbLLg-qoIUXnC
zY_O1#D+8DI5D*jTELj9QQiC4)qZ;lercp3FROU(%uNV;nSUg(S_t%jAN^WjhZX_}j
z<|ehvR5R4WQbIGG$<d`hr3~Re?C@>XOQSr?=X2Q9p6Tx_Yk08FannN={T{sJ$7|vM
z8UJyW5SL<b``0A-YhXqX?65<#>a(SqeeiC{+qDl!s@qopkFbNT2eB4Rg;Fx!@FL7q
za;NZalleO=lxR#0q}LT2yl6na5|p{G_0}cDA_okMI6Q2%HI)UrN&<~@WA}0DZ7P4f
z3!NP;U_En%LoMfGrp_0lbT6Y`IEPO8gA>n47Yw^6+^?#0eV4{kYpaF7>gMw0ez?EC
z|ICDTxP4a2s!7X_cyBEs5dYoadCPI=H-^_OSI3uC4RZKcP`TS*Jn*7-YWjONsW<*E
z?ps>pCD;w82jaHh??N~0xd#0SSFIUq+O=qX3!ZeefpxrtZajM@OR9kZgUxh(IJl3p
zrWohGD)l(2id1gcF<PBJ=BKEeI28SzBYh8FU#=tQnC;AUxA|gCfoB+M=(}$1+*CR8
zaFhqOaR?W=QE3t%H+oF*giN{5LS>kX>#Dhy$%xM9_VhPfb~{hfT9rDQ#F462ofePK
z2iW?>t32;h$&|}KoNeGC{ly^f_e)#|tiYorAO(={RBw|LE^#Inu!d09;E`ZASxDR+
zi481+!n%1An8WUpb3@8cG95n>Hc@@>Kka>aa~nsN=l}W?)zm~7a6@QoqGw`)+w=@g
z(K6d2sU<199kK!_0!6Yc5P(qtDX!Pwee+$JFRub1$yaY9x-F8(`*``z-y?rH`sD=T
zKtA0G*a>?elm`<kJ@L-gFk?@^w!XjItKothd^vax=nvM(lFkh{D4NzeTw%$Y1|GOC
z2TDJ=cD&K^p95brcS1omAvFnmz#^*EtQnw!W}@|^T!=Pi2u%#F2GS~W!&z`AkC+y*
zdl#xzh%86km%wOs@N(`9K>DFOGpNTqN&*bEpI;rmc$tmq#Cf=4r(gX2@Ss%}3Li6q
zzTbZ*73&F~NY~2;w!5~VJ6}=R77GxP)~y3cQOXV%*ofdCYgrFo?4O@+w5q1lf4Y@q
zy<D}}>EeA4zJ`J6g*FW+suuedz$fx>lNv4*0o~J4wz;D)+)_OO%6my_?%@WA65o(J
zTLS9Sw{|9hoZ`Qx{%*89t=_BZe)Qi~c~<$~?R7Xj{VA*mD;f&@LKfZ=S=a@_R$xMd
zxiccXSA4bE(T)sXYpOtAvK_RT6KxrRjhJy)=&|6ul<Fy<=^6T2fyM~<t6rge8kAND
z*=%}!2x||HhxS>;qECc}CdVBs$KV6n^F(cA<6#?EY_?T}Y%pst%5bO*ID&;8{My#M
z0<+O*1upiaMZaH^2=!DGN>CD_P+GFF7g$*Y19$`dC&%NKK`+A-`I8X9{Z|)Xc?2+b
zMH7Poftz^c;-{Z`ZgGD4xt})zWxDMZfion0_<V@Rm4;R9=wlN4Upcw}eqn&UfxBoZ
zq}lbwTXpj@t>`7wPG5W8ko=`;RwH!jl9fGub@B4m#o(LMBTJY2fS_i^zO!0hJ^3$1
zzOcSI=}41}h|%H3+T3RZ&5f!5+1csY;Oy|?)!E75`0)IE|M_8ScaMvU(`TmzY(#Mm
z`U`4LEP>W99RK{^`H|xH-E9ud0bySR4*sC)AkhXVM}k|Xjeuw-bwt31Rgt2|-o1dl
z?nl>hfyOM*BAGkHU`A@lhOPEn(6DzqDZ6EAEl|6%=UXw+^VXV;Zp-U>G$28w6B!ik
ztrw?reEHBFj_6`F#gHenmxDn{*$m~$eXfC0ot?pe-*@~5s=L4Qs0P>N;?kUv0VC!z
z0~yJm&Z@f`(<1K|_#~iV`-Gb_urRVQHStJ7z~3+`m0_2*s)ml;i8^jNEv!L5vo?v!
zd8MX0ucX+F#u8~dxmycjOgd!hJVuZT^`Gs-d0ZT$R*=zF(cW8PKrJ~w&WudubiQ+E
zr-%-U(z0ds<Ofz&YISQ!gJ)*DCPwgt{_(_%=G1B6z%|*5h++tWxL6ENM03O(Wxo;L
z7YL4iJDuH=o($un1vvY-K*<+;dPuqp*OxW7Y#XpBY4z=GHLgq4)d-6KbYW!pL|P*M
z;fC=|?m@9~SphRybd<)Gu<*w92#1b{Le;zZq#o5vmYtm=Dlfw2gEezWSdhXHWoyW=
zokd6(2-58P8p}tbEc>;l3T}M-P)&3Bz%727K8_omM)PpK15k<C1YJ+xJvdT@)D>t?
zL$Pr09-OZzQ~(y-1>C)>E3vIV6C8A8IxvtLJld&@MKWZi&f-BX57s@IEvwxEqE0!s
zj>9g+_k+ngQS)Fj3@W{p-6Ee0EQ_>2(7uaFd3!l7ot+eQJut`m4jwg!1fY+IV{N?9
z6InFJO&*?HoB<T_(aD8qA&#E45Pfr{aRXjUQ*Q#E$aZYwQPg4ix_SuM0r)WGyl3>X
zn05l}6KeW`OAW#m1!}>PsqKBhpq;28VR7x*@t$!Wx&_e=9O*T3x<aJ7A)^4<j<IMX
zPSCO;k7(h{fi_pDUJeEpber7aA)JF`r%xeB1A;a<TtnI4R&z53e@GC3I=SlhZL?u@
zSKE~9+n{n6mE#wnQ2^VHKRnVjCJ`JYfkXjQt-Bh)>4XO1j{^wp8vJPCJla|g<Bc|H
znXvy=%U0px+TB6J+|HIQUyaosx>@ixBGkxkdae5wuq12&N4rT2A*e~%(A9X8Rur;|
zYhbaOVqH}WQCkL>cwvdc3jEkn$HU58J+O+nBL>QS!~-zGu;0kKweEPl9S*T;FJTyL
zvi=rI__qY?L?|s7;T?wKuf41k9U*=SgNY$J>+fqY`YkUk$ArN@o<=~A!C&0R#kUew
zH-cHh^;(L9UPFKmjRpKMJ*!t9Rrf(_KY;U5eLZE9;B7zNN^dmaj;eoe5r&hMU%zQR
zoOV_E0AH?zevetPg+e@rY$*)d(FfT%!%PzcO)*u0Z;w5{%+oo@*Un74j+qnpEh>=&
zXj2{%+Azd{b5Kt7B{}B-pM;Mh@s+dbRecREpk*;su7Sgra~*9*Phf%ff)ooqjPTpj
zsTF#Ka%4?lU*Wg7C}g+fKTyUEfx)Uk6*&O!`cxB5Gl2>Kz{JK-j;AuR^=$EA1nsG*
z4;+8OG^%a7T-&LI@DJ)-3D*Q4F|5-{kn5zNB^wTekxFj_48Z{;ROSSi-WU|BJ_AF2
zvY_AtWblUP=PwRuenaH@yvjpzbGVr`OKbfd5+oDF%JP`dKM*yeUiA*Cox5aT<3>70
zwB|rJBi}6RE7FTB$-E2aB{&lpgK0z>K+h0{SuIkGH??MYyhH=Kt7i<*Y$k>oJL10F
zaU*CVJTOBzLnFzvgkmGz3I^4Jl}gF}fX%T3S$NR>V=|k)UCqhyx<Rm|2W!WqCTn!Y
z1IGK(Um}s+Y@}Nkw_1RcxJlcXnS<G?dW0??(5=F<fzN6VR}K4nC~j)>L?K&nBW70R
z436GEZ(W&aWKA(a?4>KS(WvW4uPR>C2=K{ml|zHSd{BGqcNYq_;#U5E&uzfCTbVx^
z3K0l@Ftkhl;&H(c#05T*bPKWN8aR4}w`_R1<_=Zsl+lUo;iJf!;KcAjV|3sJ2#JPT
zqaXpYHni$%0E{pKRC~s!Mu8&1$LRy48M@9ipT>C)s+DnMGTJ0L&r1Qu<y}*lHD32h
z-EjWUEX|9~peh%JR34jw*NydHlD0V(id*)YbG@K)M0rZqOL$Sp&j1vvP0B-3%~28i
zT>;!6vTdlOcNXO`-In7j0Di+E!D>3n8ZTz%4bdo66<i$)B+~%S4Z*W`Jt^1Wju#$c
z^?2y3r<>`*0pYt4JQaK|qzdb`pi;b8z8{wv_vgiD2>aA96c>UX0P8YBN9OWssVpe)
zadcYz7nid+K-uW5FxCUgYEtY(lGm4C*Im{$U_b*J*g|9=^m0S{3?mA8W5X9s@P!0*
zlF^7#XhQJOFE}!6@;vUzLm4oUgi>J93|u|?%s^H9OQH}_ygfxiQ-zpJ?Rn(QFX*nf
zDdv6&WOH{vKsI&11#M+cH?*AjDij!%L*Slw(xT}(WQ7onTt~DEC!EWH@GCIHe%HSY
zPaI|z3%m&g13_3{2a2F*TY@i+IopUW_W5ov-^1Sev}SABW=Y&qQEY?r8qcu(zZH-D
zn{uaiJ9jnvN0@CD`2<ptLIP(kKSx=Aba%@e@eW7#0LpdX=P9A40a+>Fi@@{}i18Q|
zjge>qX9D$F=Ae(HH=l42s+@lc%(a6IJuAlSbI6T$2p)3dCE%!#jvZzM{*YdKX3ef)
zgvb}9n_EEO<g{*XQmmzBppF7AH<%JgR(e7GtH-D~4h~KS;e{x0Y5;le23m34$|sE{
zm@%vKNG$}4Vhj%~Ls#37c$Ynburap~fS<@Yl0Gd=P|P*B(Lg7L@gqZbB%J6~XZJCi
z9;}DwgBGfX?UQ>gb&(aa;i8P)UHuQ4(k^j@C=+_V_FhY%S_6FG(ufU495gFz8}M?M
zrOwT7#-qVFFii`)2??tfajF@%Elb@!QKOtb6hkjj>yRC#Lg}J(2nxJF1h#Fl7p8{P
z0@Q=w9m-diRY(S5wL>171Z)xi(fKzRS)|A`D^p(~iHF6<;)ky1u!A0{J<CQiA^Svt
zNa<Su^qW}>YA4wg=M_aT;gOqJzcLO2udlquG~Bl}U`wuO^nAW5N0mx&f=Lo)@r`n2
zA)S|)F){C|92N$N9#P}Yc?KP~Ka8PAHjtRH-y1`39}DxX{W()aKYY6QAob9FOx_({
zZI4ca<Jx-OEmZ!CiP%B4>HJpKEDcw)-|vT&!MTim?`)#m0z4rAs&C<(@)Au3O<@Nu
zY-zE#O|CO{1j0sw<Yr#sjRl!31til;`nwO5g`TZ$ZyzwXw=exfcBd#Kvy>`F2<X7Z
zDP*9`{n>k(t-!f!+FKnAfV)_%>wJKip4K!51cU*8%rBR57HQ8E;@n%Fugb-E%>wia
zaZYo0FtiA{++2ZTlD~DKS$mPZ&^l{z3^0Gy3t5WA^~O-$y(dr8HFU??cG`hj-3=F0
z(!~3I`l6^zxcKM==?|{!yJ~=$3bi9wFt|3C)o3CTpB-*0b{L8BXQV+wMJPN$sC9t%
zNE2F7B065$a|n}Hv2-X;u#8Wd@`}{2x-F0cL|HwXJ{eaC0_$2+VV1xJJaKSOPT5a-
zZktw9Ni5h7``h}NQD7WwaISDftX4&ffnzWH`F)FzAEqTmgV4yllZJcSB{&1mj3>Tl
zkf;6j7Lvn`ZI%cQF4d?8(uB7=L&;J|0cT+um*<+c@6s?jhJj$Bk7ftUWKeLN5~TUL
zY4Xdac1f_yA#}5W6$DlVIRVOEVTFwTFJrn4H5R_1GDNsaC|P35q^)FiQyb_%;!3;B
z8=(xW^js3@=(JiP|L5&=%*=(8+O<u-t*W`o>mhpV#zGtf;?~1RV~~etLp@7iYM5#J
zlj3G*UqY`kmuS_W3AbQVs}?(vMcs!JN3KWp2RpuLXHa)dPr56R;uCx5bryeb9--JN
zz^jADl8-b0%g2~dH`S4KKgKS@`jo1l)f|wzCr23nF~^O^CHrj*dZKvDv)7G0IhaS?
z^CXNF@Tn1Gbn@Xf(6#wbZLE8ur>rKPl2YxFr%gw__Zfe4e*FUveE5KWQ^F!8SOE)m
z_$?+gL}!Sw%9KG<nXFQFCe@=p=MH+W-E9<TeTWcWq-sCkzc~DM|2yIN*EA3v3IB#g
zE~;+xbhu<x1U@vJ<HO@qGaElhbba|}hcDvT(quJ-U6}lS=lf~({WJ*oX^%F5Pdk*#
zmH`IdCq!F;zmZ3FwG3HFpEU)VxMss#fXzv@gj^mRWuz3T%HHfEjBq8x&Gwjkckv8S
znRlYdjU&T1y-PSC$BhVXyPB(t5F#c&DN4lkf;IB`ig>S>vq{kBUYeM76McBV-Rs7l
zl@>C>SF{;w{W<SKQp(sQj3)049-Z_04$gL<2~R)<$vh?g!Dgupb_lV9P$lbNbws_&
zvWnoUDQt4<0Ha4Nd=+WCcF}xEPzFht1}`<#t?!LRxo}8@h+~-4R~2z%LDm~Il?ZIy
zvUWC6*)~V^xpbQHCvHV>{WrRo{b;?r@uAVMU)~o1E8wCT)JY0SNlRST4i-p)P060d
zCnp+IVn*(C=|@K{562n<!sP>t8JG36G{5MapTXe^&K#zWVj)(+d!kS#@}RFl=a3R|
zwI)dvb9{jg8Qgq0l_LQVabe%TO^c!wpgy#6W<q7mBPJ<kNPrqcJr9K4^5<hiwOzk>
zcCXB{g^^yILlU1}^OmJ7EpYlm&nS<xcm7dmKpP|83X1WrSMrsHpw`69Xt%sh?kmpl
zh2|-;#-L6?Mr!?MTp!kvhKvYiB>&5_^NG3rXmSul764`{@AU<wHFFK&ZY`W?*fe$t
zssWC}GiTGKo6-h3G^;7?{31}$NO+uIujtGcJUC3aunZh4Lp=iqT#+n4Oui~MFDovw
z%9dph4MSkq2zBRz^v!Y6foXFIIwxu{(TVBOGF`7In_EKRK~a6p%p$^gr(#H(kELky
ze*t=_aIrM$%mRL>@$uP=UP?Yq(sD1y*$i-Hl20{!7z%Z8RW8d3!BBUk&2HRlq$?x}
z7c?2_8R5fAmSRF$*oct$gWYZ(YDE!8L7GUd6sTOn*F{xUF-oTGK3X$-0Z3V5Ep*$S
z5Sv{rEFJAItqd8-2T1NZ_}`v)x98K;e!%kwmNj_K5_pT0!O*75YJdEAK5|fFA<MpW
z{JPOUn{2)eGZXGFpY;=H9kD`QaP%j*CvD6;UzHY~m>bL3I--iIGjDq(Krpq*p=MEX
zvQJ4J4hIyKse3-s#OaBVw@;qA9$g@e&wp?{H}I-Xi%pT*8CMfZw1Zg`SVxbrljO!^
z_4F=w^%89H)zuZKgbcwqYjy3|4hLQuG0LjTeahhNDgI$|O2Z8aFU0&b^@_5o=LhA)
z)<)d?<>h0%3jkFhKN}3~rlXdzQImyvQn7u70vPSr@U{-X-NOv<y6S)@h^f%sK$EeV
zfGZy_7%4~4+l68HwA<|>GmrSkS&wnWIY^v)X?CI;^BXofy7J?;+GUhzzNr^W!=e_)
zCFJO<`fWjaDOowG&;K1O=B=-5N+}Pfrn^}U3@plit}9%$Hg){(P}y5v4S@B7gH2DV
z;g_I2e5-WVJ=bja^=EI){{l5F9WvVv{nO$074j@N8eE(m?jL(LgTy#J_pO8k7=ehc
zX=7b~+Cwnc-C7NF9{{Ns;531};#%slM#6=c>D3OxP+LXtUsoM}4-v@>i~v~n2}7VS
z`y+Njalhz=AQ^jVHT(r{!9XcIF}1yH1e&3B1gMltsGSHeA`reV9YG~fBNUW)uZ9*?
zWeB%pn7sOFMIE1EZo8Aca47|dMG8<2L3%t$M&8+3T!u(lIjc@wYM>#T(!$fGP*ZpU
zLKLZ;0}5yIkfAj36pzke3J$kQ4IzhJ1)tmI8nO)oP8G@}S&XVMWD6d>o!AVr!!C23
zpFaC}_ot^ncP(ODZsXb)!i+Bm0_N%EO*tbJXihb^L$Ng=^X4drsX%LZ3nyd8%<%iE
znd0};?NRB{WUqA$UKM{$ED~oz*0~Ktan1oXuyRJtS(|LqCtVuSL#COp$Hl%z#-T*H
zoRfNv2gNCUkXf$EX`+#UiZnB52n!dp)xyl@u4?+})l#x(P6})VNlS}vUm{*ygvyq=
zdkHb6N5XVUXR0u$0?MfrT1=-L(K)(e@thH=dMHjdUSa?mpT(?C($|{X35l%DPN2wS
zo43_t!1Y@yPxH&_3ZNG_uY=GFPVK-%7(9D<V!Evm>nSyY2oQmqY%377kY^CwWB1FN
zTT+qh?`dvvHWuC3`<PpljYT)rYHr!qcp6wuWdv>~MBnaK^>*CXE-VpSu=YIIEw}tH
zK}u$*Gb9ChcKYqfKn(txQioWVE%0Nhlh=Owq(r-|QD+nQlPhEnLw$X-q!%0f|IuYn
zSIk-7gEu~DU~03yXHAw_qtb6-I+7%Ek?BYwJ$pCk_hT{o#Z3*#tL-0`1AlhAh*c(h
zj>FWoorJgaaq%r?mq9Xyx-HB)gO3oVq^Tu!41qF%^ND%P9w0EGkvzw)F|vC#6nCQD
zy^ekJ*57~8zGkoV2K+KF66>k8nU?i#?Yij<_ZqX&iKe0DUP@WoMl2m`sNt&=+00`c
z$7f>SKZE|6VD>TMAL;KJ43?(gV+tp10(R*1!|SKJZ-PVd75Z;9413_4ECs+3@1o&O
zh**OA^prB(1Z{C|%*&{FHU$jfkg`+rTh#MT7xdXW4a=I1mnFs9?7Zf{=NQHqN_V6~
zryf7bsim#_h0d*QJb7}@4i65GzBzPG`YL;7?4d94K4=riJT0BNU-cy?YS;G`cOHa8
zO0qJY-P;EZiIL#dpnE*Ig#(lIB|+i9UnS9}dS5UJs`*Otl@^8QK?Ppj&0#%O8>Ix5
z1{TV#G6c;JLrD-=XAc|jzp#ywfi_4-fJK~_-=6C*t?;q|EB+tCE-9JBjk#-122JEa
zb|a@hg}T_;L@IBR%sHAjAXbo~IMs$L;8(PHRmTWo2TFI^umAB>d$fC|OdNgl7~gvS
zQc{uDjm48aA_n^}FCU&&cYmptBSZ-01VXA6%r4`SZW)q=gh&Ds*{cdlMXC#L5-ssI
z{#^4}DKZD4$1x0&b0RRAdqG92tH3W45Cr)G1Ym?9J>u5{u_u3q5ViQP=*hCU##?8;
zj11|>I68Z{DU(TwsvESY#3Jftk5Sr;P8NJB(jS#mfl`6~GnRKfbGo%s-Bs<S1{140
zqJ<RyM)^a&iY)TxH7*U`@HYB3Z8lH!5H+5znyeO{xsH_Txw&gq;N_~GjP=R;G&p*4
zlN23M@YG26t<BVx=|Z%;Kjh>{JVzTGg%6_{&s?2(_{aM$xF+^H_>)Lqpq||6|1hfq
z2Zh&B`X6_?fU0)&y^=w^i5Fb=rDm#a2f$lqR$B`l0D6Pm7R!7qs|Wjp+%i6ClaPS)
zraYx=hC;N8%c4NaI}U~OC7|RQ_Pmv*4O9o|Q+3GT6J~%S172$msMPxz6!A9oebO`a
z5~lgKd`(^tp`(nF_u=yxY|O0R;)6jkpC}10jFjuAtua(t(z@C<qO?H?=_N`2dI6)+
za}0(WA{=K+T>(^5wsbdnr}{=fT9%`mPJHCkA-QnAEJGKb^N)C%AOQV0I7`7{+T;L5
z{(pLRQ+c9(uN$rh>B%U>LbS^$D9SOs9?EtUOA0_55XFmR$wFo>*s(`OKzN+3E+m_X
z3JI4HTaA=UhOwLijF0oe4G;mpIc6}E=SJRFN@9r?{^mIED<Ww*J&;C{D4aiv_|;Sa
zw5)ACH5s}+8wfIktt6R&hpM+u2+5)D4P;yQil2TCzQW>|#f^x74te1ckiT*;pUqd3
zGV92Y6<Xl)wt7fvdxEu4S!@DAvbH8CvPsY%w<lf%YO<ARL9>vTUOz%cvhEc!m{@<U
z+$=_;z0{z3=0@xVGl^N!Or9d-5_36>Mk8oj=OBYmDmRhY7nPXE>?Xr3XnvMyRv?My
z`cBU@m}VG0=aP2@y?A0?u;2d@nsY1tAyPB!2V{uw6i2@B(vr~2Efyi}Td){}%fedr
zC1T9sgywJG^j}}DmKEyv{P=Q&dY&z<%q$czlF2(EnN5hq9q{3Y?|R+??8{JCfF>C=
z0R2cC8Ie08#&GE}&8a=|7y^H3&1p90j=}hPp<l7Xp}3zd8h`>dE82_<!CTa`5Xl*E
ziAEp~)s%K~JX;~Gw~;w9JBEj_O)DLuBze2=8s_!JET2-Z#!@z_BI$GmH@lZS)t(e^
zAqnE@#-@l)A8>C>`iPI~1h5U&k;jlWtcjyUA=yC(9#gN6z9F5aaJTrs@qT;RNrL86
zl<H&b?Fa6ww!4R1LUhr>dzSLo_T`IbM`x|#g?wD;FOLpiJX@Q#*x$-nE8NygUK){)
zFO`HG#`>@%cZfC=PyCM4S0izQcKCoN!aibJw3?4idmTN9zIi(8We1ilWFg*XBVEbk
z!;7y@H`~R{cW{&4`_P@+de=VAjtT3Y+Z?Yjh@wOmDIaD+XT16I>2Jd}v|W$=eQCVw
zITlcyhpc5D>3vQzE8%%XGO|IYbSn4!aBcee&CaQ^Lw%SkWV@ql&BZS4>EzVh!s9Ov
z?X`u+A@s?7{n?u^5R(+&48`rTX~QShC;!sdCFY%EfaI{~v2q@$+>e&4Ma#X<){-$l
z$((chfgcqFfp%;{HLMPI*%N3C2~9SIld67PnHH($`URx{_SH1J$(e!Nw8HiEp~L|Q
z+Fre8*^ctpU(U`^=JBg7s(GT`W3hj0Zh{-zQAj&R^0ii4E!Qc<7p8)b^j`av$_c+5
z7K=y=MhWDV608A`yb+u68%DKz<VHqT_rBMxb<RTjJ4QhRAT2}EPHO;XgV;7eeO)Xp
zcAfdWBKNPJ*Bl%1RZV4isL>E;Heoedf~)kU!{S^@rWir*BS!~6E^qk5-R;mrP%lMp
z>=jl2x^GnzLva~3HTr<v*`Jit>ppF6kWMwB1ibojy3SKzV9J;aZ~ps1nlWviU#rN8
z&GUoENdShGaswE|mi(~4ZA|%o0XlfizwG?k8Xrw%k(B3tN3xWEAd+kOVVw2(#s0<7
z!RCy$yAc0WEH#p-3^N^G%C$_jcksC0>)kimi;QT<Kv4(yh9zA3pI02n=7=vnR{}~x
zk>&37vk8tJZ}8))HbZ8FRY_abr&rVP^iVa2#Ni*Xv%@c693EU8ot|vK4?|&OhyT9+
z&HmuU{>k&f!HfNK6y??%3ccgDnIU<*gQf2pcup`MTYO$TB`-R1>HO@CV}#FN9UMfE
zo4n;8^)Lp<VyBhQ3j!IUejy6X(4VT|&?m?cT3hRxAN&Rb=yv}gn%$3cI$5;Y{!o6X
zxaOaW-{$aIY47z@Rg`otgNyH89%j*S*2(iE%iGLH=x&1V%}cZLNB^*VJe>FzHA0(1
z8Ol<2__6yoQ5%>=Ld&kEK|({zAb8jR4JeV*QLGzW%2#Y-Vga9x35GA*z9)(3|MYY#
zGQr)3R=Gcx^W@;Sb(d^CL8<os<uAFBTa!(~aaDqV3~$Q{Lp^Wncd+$0<syu;Lb7S8
zLClD6;&(atXLY<BY~*<vd4!#woL`*1I=DD(7fX@?_%G%eww>7V{;pvW!KSy+v01;G
zN7$>TT6|?h>;2Vob})l95J~VM{PzN2WJ(NO1I%OOYA8PzHU+=4;tF(j)w>d!<NQwW
zPJB~W_Y%i=XMU#EKmYvmVrG^P{?cdXv@XcVsG~A9op3|n+FA><8?#%fYL!r{Wj*Pi
z;dhCK)VGoV!{sxRLDr2tZ{qsJ*jhAtH*lo^qt?E7kj8py;Gd#x`L3)dlJFkd#$gv1
zgd1Cq$MF9emcN{k+OcZ>@{$(Y(1r2Lb&KlA`NLm-z!}SFf;1k0350*6pKT5Gw~iXL
z5h%>D?(<%e^!SuT3aDoDUF_+paydC;JVweBf=6LwPZDW736jjt|BB-yRi0)VT18}*
zTB(}8K1e;Fw-2)w)_jlY02%@<>r;yAgeRt4*j(*y$iC+C>%;GWEdEW7{cNGm`{z5J
zikVj;r)1`@JvB2>Q&O^0tULFj6wX@m;e`ovspRy<w!`1G#`wKd#pZh5-45z;h&1+b
z^v33>QS)tzN-kkFY@7_k5Rt*rvqzb-^ut<ZBh&2HY5LqZhw{yKk~jt3?PUl)G?-Lf
zq>INMR1h~RjMq-Pgt2qsP<aqMLJoaDNd57fDGz{wp%diEczu!aKC+hK)ydZ<0N}7r
z%^<@{YZAP3_Nyry)OLlYVOA=bJ`TR#e}0%eUYcm~^x9=wnxE!@TVk|j1SssM&896f
zK5GxOzTnIMeFCcSvy0s~>!h^xlJ<tWH(r*miZmuJCHczGS3^K$ncWNIriXeJKs|bi
zfW35B?omOdF+Vxt{O<Cf1P0O06$c@kd%;(*t1;c=+N&q0Ss!hjBL1Xq;U|FpL`#x>
z6SlJ%N(W4`y=xD&X{<%P)a!uMUb*z+Np)k~sJbN-hi8W;2ZyMyGI!gx(ht0yy8(4P
zpHVOw<{7ixt+hsXW>d*hAR=zprXvfQLTi)-=;?>XFw5U#lHq!KQ!Q!=9tqlUniw*e
z9ToRjWm3Pb9P}oqVTIFaPTHhOWx%{b!1U|Y#Bgzl5F%`2hA#5L@2T|*(lp^)+yL~e
zS)Kt%oMfrnD>`(>Ucc$CeY&@`ytz`ahDV3rZssGDrSg%i-Zr(@pS_8KA^spTbGwNS
zky=z=+3RK)aS;7B^osj^I!ot|Z9>Zo(>2@PlOLb%zA-1MjnmB7Ff%9W=^Ja+{7sls
zt@_yFsAf2{Bs=$h+?;O*QrFmPuGOmfaT%_xxxxxpGR8k7!!_s;=FNc9kTAXR{+5J|
zbmc)SVB`arl-SmbM!zXL2ZuixSPra)u|cI!fEE6d1@4FsWS2On$+o#Z5wfr6g!3S?
z>smS7`Q-O7|A?7KChU!ZtpgU!i%`r(AhNSo?1q)=$`sCh<tb-)*>cJmVS6e2t)aK2
z**#3gT2lBq0fk;x%liuPnwW9==2&yLrHU&3w1}D(bsF%~8bh$OH~^{;g|TwW7G23U
zj_3hqk_&KXt|281EiLL_HtUz$UMd`dKDPLh8VrV#8-!Lu4(O0=I~W<s%p1%(x3%dw
zKo4d6^?Riwf(~zJ^I^$|S;ihqF^_}*w97In2_{+)PF1|&N{9^BtS-4?|0U#uuwmF>
z2T+KZB-aU0gx4;&6zp{mI1IE)G1#U*mf$$f7Zs8)modQWxKmGAJBbVqfZh_~vjdwN
zxe-b&B@NK!kqkx0qN;(?qN_CY2a%}^0rQ7SKD6dXNYzXoy3_34h(LinwED;K?80=_
z=??L_drnZ)iY=*{Q?;xk72vil%5n8%c6DWG1Dg7#HSk5G+lP!Hvn3~ChBOAN2Kbs5
zsW)QsLJ^pSUPndA&(ZgnnBywLNqh%$&^emFyZFkmH80Oj|M6WSVKct$19><2PIC{O
z8kfwO^0&%R+{FOU4^!iak)e6`K@U-Z)NHt@+L%OHeFNV!8O8q8OZP1_$t4sXiZaCO
z*~?N<+yUReg2XLk4PlO6PFIZ8nO%@@KwhWA1FlCDGciUruJo8v8&)t>baBkUx9sOj
zxSDWAoT^{SdhMKoi|~EjsI6`wquUN)8Hz<HUWe>n?}{R70g^|vBm&_*3WmUWgTOBU
zPjNw6EZN*6IZ@yuhTZGq!)ZyjpvzwGX&MdOnai$&<Qe#Cw;=kk`g~A)R@>8yY?wql
z-*$BH6Rf)<Lz+kQD24tdcI#3E#Qhki(azrv;MKEVhD*_M$pZQOWTq3*+*b>A1)7;a
z#>33$Hb~3K%fOx|U+kYB7XSIzfakv_n0zyj)wF6x<-8hz<`<z&Gh3t3^;6kC+tw;a
zE)pQqm=ZmFN7fO1J`+5;Jv%)w@cQ7UGAjiaNV3Ia$}vhx>up*t7FcJQSGP+E8e3$V
z*XMJKZn(x49l533tQ%K;i%ya+c(%V-n$-hHwnbb`crX&c+cM(ZnM?WA=MS%1jVQl~
z%KF3bJsYjBU&pR-G!dCkYU$<6SPK!E9oVIb(%$*sc0z)eeFi~7&V3i?w4*(8b=}`G
zPmIeMlpM8u-|_DCd9_^41JESDeaPAn%)AAlf!Yg5tMxYsW*eT;5MZb!VM^{^Qp4`s
zPB&_8qw|_*HF>i-YnluXiDp*7x}koJ^DG9-9@s9T!^&r!GOKlx2V3#%nU#(7$ART7
zJ4k!JGQ(8oGgOLg!rSyUuR;>(OSlm)XR}GC+uNv#oa!k)BYBoKy{tMQu|Hqsl8t4X
zwUE^(7iS>j9i3bZPQUp3;lah==vj74?6tC+lDMoXfiz5#z@bS#8DMs~_B3)VOgCJ`
zDevbnm_-e49np@%>x21Vl(z1NyIXXsW2heUYd1`wY5s?vs^dElqreFYeY&O-Gm3r8
z<AKIVbXdEjWP}b=Jv@KV8eD8}z*2MHgx2JljcmNAH7FjPynJ;rI5<7|=J4!7Wy@Lj
z_V{gi5vFT`VngJ_4DDTKVGs4W`ZZI~gbC|P@(i!VeQ?y#S5B_&PEx6yAD)>t-nz#-
za7q;~jl@%ub7t6!9m(Q!TlF|m?C9Axfmt48fYNX2iVBz#+f>hAK#bFg@4&<F{K3fQ
zw|x#nyr^%h<9afwn>fu}CU*?Ik&<31l`GB;5Ut*ccQl9n7L*P01}2+YtFFo?8Vtq4
zZPY$WaN}47t9%L7HlnUMA~{z?)gB}tL7&%^pU){An12-<wThvO&u}d%qn9X@`Ensn
zF$_;sGwe4F`ezgsL;Va4u7q63U`2nhbvq0<3Nt~Ulv^UhkXvn9Nikpd=1(V+DaOTZ
zv`I%EM92GT5L$)sHEdq!n-X)j9#0;g*bT?gmVG7JFY&DDOp!XfMyN_Tz^r(Nk++HQ
z<Yk4MEP6R>Xic~|WC%ek-|y|HqZn#~0|21RYOcHg+W#jSi7)xKv8I~m%b4aj+}FLf
zEf1C;OAA2*1&<RjRq83(+ZnBz<?I$7uALx9ATA%6MNC~p;6;WEgFtFS4IxJt`RzA`
z;c^A4X|poyoI8u6HRYIw+j5L?6L9mfg0-}RUVBoHmM){U3VnbN7~ZCY#e)$S?gTtX
zd<%5rsmjrfSafe^;}s?rLsk$;j|`o-OLTvP(1lWAW1s@~vNFZ%*`gbhcv!gI;&jdd
z9uz%8gj4v|e(_9ZHMI%Wz5QNI>mLD;h?2*c%LXVH%X(_24-Jr)ra17C-;n3I2%iob
zwWl<1bnwTg0c(}6y?WeD(Ea_$i_kFJ&<)wyOONSCPRQow-3&%ReBjo5%mLqXaCT@|
zwSjoewbqLK;*iDpHJ#s(6}4W7k?&iZyIUO2mE<o28{ryu6CHrO`pSn^4Yuo31cWGJ
za)709?|a3kW5eN1!Lr%phyq>`tdD0O0?LN2i6PqATr0!0U52tGp`~4CObW1;T#yaF
zE#5@2YrcK~<dXE|#sSA3U^=0K`>S7FTbjL|hOeLY{{796y<E-l8m~r(^fbhZ;4Wvr
zPF^)VyM0`=zqz0<hDk=h9b4e@q1U&pMV1eO@Km*H;7b-nrbb@tK-Y2P4P^u*y0J7(
zm9RpFKHCPBUNpSs`)6lCs><|5<Z^V)x4z--xNumdltiP03=`fJwb}jyD`BO3aqG5D
zJk3Av{gS~JIi!FM2#8J=EBSt5hn<I8D)aBTsnJk^&850^o^sG8N!RNqC4COaJI4G?
z-!Jm{Fust%KzBjR83Ow@<Vi#cJ0$C!x22drv`J<U?U<e)&=Y;~K>P${(Q7h~SB7H_
zH#1DmyQ%mKKFq<KBORchG9BAYM%L>bdeN6SFPDKYwKRcfl`)TGa`1WVd&FBOj%J4c
z-ukHie{>deQf)ed5lq{dk{6W;yy^-0cWdD=meKyn&$vCim=5;+53b9u{*ex?ZpL%+
z{WZ?AP6;1c?$ls>!Q>Bl;wgRhALF>ou6)=*H(YgAN7~9aga@G`ehSsND+V1>nv(j!
z>|sj^otF2cb;pwEnjd0_wU<BTwfVNH<{3vypV3TFkAb=>Ei?m^t5&0{Zn_a6RQfcO
z^3ib){MR7mtEYyw$<2xm#_Z!g_e2o#(j7V+jD(gWHQilkN(f5o98heQzj$VI@!){(
z;gV9IoAWqXHT7MXZE2I$>xTnfTf-YQmWFjsYTpustu6IaXq96E66%<fOX>B$u>%iw
z(N8s)_jd6;fHHS2<-4=Ny~H4*^*a!^oql80Jyf}cdRpJ&-<#v|Ej_iQkl<OG$}XxI
z%@!PFt8K`Y_7+DCfGNk>AMM4El-?SknniK9%T2)<j|vv$tFgp4up*L@mtD9X(1-te
z)aMGC(vh;VS@I>Xr+2frG7xf3Kx=h<TVp>9`er$s6}JGHKr?mkvBvz!1YN*Kt7WJO
z(#s?!4$rP$;Lg>es=7rG)f;rA_AI4E3(=0?1Rzi_?f+eAgyDI)1a&7noSStrbRM-|
zgzYXU!9l!|>F^nS1CJ4+9?ucu=5n@LTF4{tB$&phgjdCMzw3oNTP+n-RPbU~lyiJ^
zqr38!MQFDmGLI7;(=^ZB0_=`Y_&4kQeN|5Y+m2F|VsJ}dW3_W#Ex-T%(=TVU9dkGy
zl)B;g!ThlYKdFDeC<!wrqT!ba!){7ou(%8mj^S;&9Nn0KmDeR453>ABmo;QFudk=2
z>96t)Ko>KrM4lN-N!f!zHF7t=xVb~WoR}{=W)3^?I=K^oj4qu5+iFuIXqDE^+xI?F
zAU4dj;PTKPWTW_40T8_Dgo7lYHOxTawFH>`ki~H9^_pugp{B&I)MTK#JF(#SF0~Od
zQAM@g`5TYr%?i{IQ7@rb(9+N<wrT+k2<OFpB^YYhXShU!8lJw>ffG^;_MO&b>kF>$
zw!PL&7ybvYt4i$zgkn;zrlT9HlnMBWrbn*NpTRS-Fn!$Y?d<gE-rL`R6wT(#X7BYI
z{{bD>)nGO?Onz-6WF%B+(3sb8m5N!>C5(8*6l+`>>OHOhPc@M5nR1bz9}We644!bd
zV}ib(eE|p8i`i;!19r%74V<*)Pow(?P08%h3tXlkLQ(8{IwlO`;sk4rLD1Y`ReYp8
zw||Qe%=+r<&xahf!T}QQ;nbw|ysC=bD>K5~p}(K0*8LKu10cNMRp8p5QkX9A%S7t&
zeTNRxfg*55C_n_wlxD=#Nz7cpl_AFrDjM9CW?qem3LsD!NZ9k6>K5#iQXYwU#K`jL
z<Uy2R1D!Dw@z=k@x2D@uAa9PrXMYXzo&U&`4ZYUL!ayXR8SL~4Jm2GTF*fohyXZrT
zmn&gXa9l1-kUn5Xyuz1$Sqz(prf(S4tNPuL+&3^)>$D;xFAme|+G}Yw9qTv>%f0h8
zq-on<?R%Hmg~aO74c@i6H-G?F@0N?w#z!q72&Az<Qbi{;Cb{mJ4Fk_XyPqy6_|O&{
zx>v=&w!rugWl+O@H>UQMutZ`N4o;50+)8{wOfBU10#?gx9dDwPhISiD;ZwY=VKbr?
z3+UJ`G`y$?5{SNW$#=1Y|B#y-fI+AorVxPufi<9aFS4_kHT`jQH#Ki(-^G++jOl#o
zVrS`d1)7d=)r=PP9P#W*^Tby8$}f;Incw=wx3Df?X2FvPVK#K)mvvw+r`yy>a^YB#
z49NCJUl7ygA8SG47YmQ7V@@D&dUcEJD7buCHphQLrIj(hELen`+NwK@)D^Ga;0Q;M
zLvW^n@eo0(oRa7lEK5^#Xt8^|O|6e}^qzN%+xq$jnP3i-oL7s<14+5#fSF}aWGK5L
zz(x$Yo|zZ<mU!uUN|Dt-j6l;0`;RHh!mS8r(L<kQdbK7cDm(FcPg1@Ckx)K@J*LIt
zDEM!foZ+j}<HNsP*3-ZIfMJxr>z*rud?rO~b22PEk=pT)*p!%-XE>FBqlNoZJI_Ml
zL3{?5&j8lv4QJ0HdTda-n1HW2CA?t%9)6=^lx$8S(hYBLZyrLrfjp)l#2hYA_!u56
z7n3Ike>p#TesppH(XXE@i3MM*=1bSk5sMrCaW<s8=>>E<Kmou7HtS*z(ZsOyJ(eXw
zbi1tF!9s4QnFY65u!&%SW~=L)XnPc+80?7ndqVdRjSJ!%apv@9lJ}|47fdlM5M({(
zB`E3<l@X%)(L#cpk^Awkc(rog0OWLFPen$;r=J~=B=ZS)BZk{S*A^5aENbb+O!#4x
z@5Q|O41xw%&;S?1C{R>OMPS`8_F4rM3dUp}P>Cl_HtA`eAy3mlBnf`+MG{F!1k(iD
z2`cfQwAf;Q+E8-Iec@Nvf{pr_#2fQ}o6Zl~2<+4loT-kx*DlAs4qR9(2G~H3D%dBb
z+l+9Cs3sKl>T}KF&w?)J1|nSCo`+oHwVDXOiUYM#gr)wICZ5)h3gA_v?zR9ez`wc3
zqDPM&JuVLRU%dEY|KRK5{N>@n(U(UD0?-q<$c7j_pg#&spvV$6bHhs1j45q?A5|3M
zkN)PUns@r-M|M~U9KcEaEcs>RWUa(U!oR4Hu!Krby9$i=wjdN2?FXf7FLSaE2w5V&
zXG{DK@(B)AAlsmNl;iONwn>WbKYMm|cz(WRTth4Z&n6-Q&nDic0D)bYc414HO=G4J
zC8`)aV~w6Ui0(}Vm|3_BI04v{WmBU^<@bTLkq-`uOh?-VSpCXq39Z&Bf6e-+(ZFOx
zbR=8MP#-63d@IZYxhMQDUo!Yl6rWAu)azn^;|ObRdV{)L(y!h$6R?ymt9P4puS$o*
z&tR=-BDh!FSLNG9b=CjY{Ci$$v(w|^#mt;sNhAr`Oq&_VDdrWg5hD{2T7d#0ysM&!
z_?UZf9)MO0j^wj3qDfg-S?ej$5SzF_4K^xJCX;I7;nfp9=Y!=5N978HGB|1Ds+>@e
zK;NJ11ZJ>ToWV>&-+{Urh&j^~@zeF{0>HEmyi%5=Q@BjRH^PO71(!ox{FFwCO^bwf
zV_RZU0K^I~NC7W*HSKi!ns$^|1N3*^V~3hPg!!3cg`Ok&IQHDBhNR8rf5EY%qeZQ_
z=IEaechi$UpPwFlJvhHOJKR6k%L-1b=ClmB&raO3E1mYi@e22iY&z~_rxC)p{`JXn
zHD`XIp5OgvulRTWFFr5}ERozWzgx40@DrFDk^xEJFkS(Y$-v!pXiYC_sx*@nEc(<H
z4c&@B(h2y?+Xiw<?d-F27CB@sY|@6Mh26KogYNCwiL-W^Q!0KsKYjM|?oUsD?)sU&
z607Ys!v0D2{&vg&b5NH$atzA#ips-)I4&P9D*@Vt;FyM@W6b%l%IRuetmgmipH(iq
zXKoYx(4nNz6cS;sh}Zfv&i`VrUE+>2w?@Lu2ISUCJ;`Pye9jUGaO3v3KS3=d-qKnc
zW&*&Maj3B^nsssn@0r!gqauC?O$~{xijm>7RtsI~wC1LwFL-`7ay#B{LdT@!7zgv>
zMu(^(x4&SN=@hgi=7)}#nFNg+lbczy%s3ln<}a_aT>_xKSz_<zfV>ZCYf)Tr@1jU2
ziVZ9Z??ni_OwiK<8VUGPIcURvv9Fs7#VC!&$qpYdIMK5KPyrak5bioMBD|k&M0cuN
z<Ni1FwI>fzdBJ1%$h8kRZM*ZKw5(>qYlFLHmNx0-<UIzov7P@&62P>B=FU02qV7zX
z-8Y1Gryms+qI(NIisK46q{OS~zewF~IA|(L+lm1uX+V)47ez|h^-f^{1Pl*9<dBO;
z;6H5i>VO0h?nEm0WjO}DORoUnbNUG%v1&{N7aVNppR}v19T{Ns<NuSz)-v08B?n;h
z0G<A$s1flwZk-K$C}Co1D+^36CNoagru7`iJ?RyLG!sJO4fv06tS$4!W#ifAGgbzp
z0r)1alimzlZ;RgBr=TR5jfZJh%+>ARFUmQu-I}3?8zAdmkE=jw=#6(eUpdCut=Mh(
zwgO8{XJ>J_ldh`XfH1P(ev=<_G+rD@x~Fx7(@y4U!`tf(BoH5gOYFGc^BG3{HHc6K
z(tQOkqie`pYOFog>E0&e%;!9n&yTCxY=w=4(US_qn|y6@3T~;ofjqDgt@0i~WWSkk
z-<N1MCAcKLGr{>yuSdK!NX}a}hBgCB?ImtkBus2PSjC0&6F8gC<&5NBJP!-P-9IiK
z5r?HIdF$nXQI}cRii6BOwyGFuDAhx({!8GK<!LRo5F<Z0?YNyb9le-`Rlx@2JLtId
zR78*F-Fz;KgHZgCE3I)t#k|j_szcXCzAz1{ue<`Lpc7Ih|9o-&XW22yvYK^8hfn&N
z=AvvvX<#<rP8cZm&8klVf6dTRA+82F!2y(NkfVm6&73+(&@r@CqJ0NLK3SsZo|*Zo
zotiHyIf=3w*;}{K?uH@&d;%(GHF0SgZDf7M?1cnrxCk(CD<inDK>Rgu(sQj;@2b&?
z+)DN*2&aKJ;$D<>%NcujRDm;&YhqCW_Eu?RR+uTdecV@>!ql*!f{YM37j7cD(&6r{
z;Brbp0<;**5%39u+~ku}W+FOHTj92H!9&=l=<};M611*-QY{%F|G;N+(vW1@x1|s8
zQSVR`<l{{MAs#`KwFo$#s~T}y+J)O#`RtYAYmHgGyQ#0j6`RkR<;M3mG~lX7judZR
zqLsMRkgc{=qY&bOAO^uP?FL*wVkDWtCVDN!qKcey;>mqK+@;8vs*~WX=GCGTuK_>K
zeLL1*o3dsa5jXH(|EoYpKZxRwwlgR)cKKO#JG(<5rl<YS`o)D2YYI;D=jrVv)Kb!c
z_`SZbVbZp;HI937@CZ73!-N$^ds*=$njre*1B{DnAnfA_2j#cdI%~Srs+UZ^=?9wq
z06ac)WW_5giLtcp6){*N!lRtLM1#|=cX3WJ9K7A4_roU|9-NpgI+&SQS%#Mcbq)xs
z-|fEMee(28)C8X`Zw6`dE>1j~PcHMgqw8%Z(%#f9uF87SQ0&e_;d!y^Pq^me8Q)@b
z-pNbDyk<Q`QhJ((^om4^0aqFDl{5&ud}fzFP!iMk-ih_`?Pqi@$9CT#2b*;=fh4Uz
zJxwW8;loN*J9`>$H=d^11)i0i1yCK$w(mDiaCdi?;J$H!ySuvthv2@^;K70fcPF^J
zOK|t#791YA_xtY6dFP&U-<zqf>ZzLg^_reu-CeVK`VUVm=a>$?LGfTiyp^_Rj`r@H
z;y&&@%T$#>Wgs>rlJap!-shO<u^RGp8IAp|)ptvoVMUFcZPC^(7x`R_E>Axm=P<Y&
zE$UmnE)Do1TywM697QH>C)yYMd@aqdz(3O3^Ed7Ye@G@^DI^Kk<aec1yZT|ze|4;O
zQA?0TsNc%Vhf9$Dbqs+^D?4LPDiJ@p>c;xP4OCOjH~Zt9PE+x42uE4j2V6LDCnRdE
z9qUSjV4s=VkQq-ep;sxs)}w$28cnj4d~}t8btP~FN-_Al%_|m4fS->~VA+EQbM2sg
zUnaw!Fln9^O~Kf88jKYj>~Mmn-2;Jt>>Id`-aKyUBvd`6KVh{OWO=IedOdoh(y(aT
z%n0fG`n*Jb^~13O$8##URE|O5vLIpkx5qxJ?QAgKrk|)xFx8KDdXRb5N%nNIEVg(E
z&nlhZf^AsmaZ+~pQ-DE-fX*uh88(^<V11lKoNUcPvIY;W5saP$nd>=HFPZx!0D?^g
zb!4pCAGOA&{T$Vgk}UEz`dn9Ea<jX?**AYc4t@|!W$*BT+^o`@<FILP`NPQWF1@)g
zpkG%ZGb$M}F8R7Mt?BzyJMdlvNAyn2Fz72wM)y>$vG`dJwZObHK8Vfc6<op=f(SuN
z^C&m`%if_vCL}c~^)F}=&s(^T0uJxD4HRAYzABp(n=fHUudhcZ%OQ{GFc((`5K{Wm
zGfl3iuYM>2m=2Xdzpdt_O}kw*YP1~XL+kX*2ofQ$YOpn&37M=#+qnbtGNifvP)j!z
z4|dII1<gwUfF8!6SYA!ly3h$8ZSi?k(2r*CL9Yj6+QJk`MNNP(2ra3J_5Sii+ItK8
zE|B+33`Io5@NB@w-2uqjw5w)v#$R+t4Tax~zut#oc|Hzd3Lm!gfn15pDeo~qCq2}B
z%fUC`7N`9ffzxg-z=$(bY<f3)uqu0p2a%zxrCE8=(5?&!O=M17%8$~~oe~xrAV>0T
zfH5?m6`WOKT`!3o)0o6=WilTt%GMs{lXZV$my;9~DBel+fZodN-~q3O4<AW-G6?mV
z9Qf4Psp8*%06hiPz=+_!gGxWZBFjNgp3Ajt?)pxb2BtasXllCN6nO#Vur_C`N1v5I
zBO&H#EktPHTC_><RH3Yb=-0{$D;;`!um}n1b`o9%p;J#mA+=$qc0w@!DDB!_TlCc`
zz3;=~>s6gsBgI^=#c6X7!bwYPfORNNG@S|)DtN6>=(ovF#uQN)Sd<{vscfkR4<4e&
zdRt2PXIbD`2+ytr3ag}<(Hdos+Fa3cwbX4O|F_V+LFq`+u~xG8Ey2Qw>Jsg1{#o-#
zbBnP;ynIiW2gIS=eQ+de0Z}}XzxF4*`(`u|;nU&5W?ZCakZC{T*75R&8vY0^SybhA
zD21s{v5WjBAXu%UauaR8q?>eWi^ue3y@I-?vkz|EOUI^x(ndr%2oj3WpP}s)JQXf^
zBt+lkXfJ-?_G1ud0QUlP3l83$_~I;i6c+0&%I>}7b>yaf6wvmxaAzbQ1Fsw5lc$H9
z$45u*rn^(hed)Bs#+2<Ek`xEit)4;Fq*q#<5*uh}ZtC^to3o&+%2<66($T^7j;_Zm
zuBm@ZDRww0FeB`7<j8c4%UtagtWG|yQyJT$QWMp@h;Xlx<)}G9%$ti~^UvLVy-1zp
z5upo~5zqjDIUWE2<!=cX$Bzy!4sQ}LfA8$;K_8?0KS;mS^VvBsNlmZc(C;lEhmDI>
zC(N#4xv$Yh?FMHc`T{JtlH?+U5wgXY6Df|Xil1(evbe^ix;+=Yb&LJk1y<kAXHL<=
z8K*rR2{%u5b0e$-3Y`-Z33Xe-{4x?s;>h@-sMF2Q1@tc<Owv{H!_Z^+2C|VLXK#-S
zZSo6(PFzgL7bWEyh1~>7ac;S9E7NF_i(Z5ws4_A&;~#5`Xh*D`OdBWMwYoBbla_78
zMeqiIC6dXH@hi1YKz}xY3sSm9ih}#l&n{mXwDjXn<3liToc(7Ya(LK090M97Y4ZKW
zooCF}`i~;^cYROdVYK5|WFjs5J{s~#hu~N$eWc2uVbsc+PG!)-vqw+!6H~%iJ`O0~
zS(1*SFc0}YNaJc~tQ==62Rcxo>cyoTC70GO6>v;<^0L882FIs#Qs?ES1=EVl9SzX$
z(&!fq(MhxSx7PsC=f6Ap%u)c<{?t+EDVD<Od|rKksvb%-Y?R8T-95@ciB97BD_1fr
zGb@u<{_822+8Q-BJB++>GQl7v)Xij(Bd;Or5x<NSO4;qMpoK6n*ED69a<Nb;wOkWL
zl9CQhYRc?$-(x%!csDAW61q@_Vl4=*7kH&s^>hz4h>mhy=1w1awxjA14urKn&G$$Q
zYUznY)=#D6oAn1A@It<9MJ*%8Vb&bcs&DS*n6ghCZfeB_ZLtAvfVlOQ1aQP|Fmdt^
zx`;<sX3(Ki2?qCKcgV7ES$YFb`p9btf$3dmU6%aR`#Xz^k$&xWyY~~!FSL>pCOns7
z$j@XJ(ft~E7nUzr(0q9qZsw{)yvCNt9R#D24xixnJ`h^6^v|2{o&v~UY`L3mf;LOL
zFxFo#U|OFd?yMrx!RI&bw(qa)9QnX}1Bd!-{J4=NQebu5GB!>5L$ULQxSKNN?Vk^y
z$`R4s2n>f_?anz<?`ODPW{DaGc*kv#o{+&({5T#6!CsD?ef2xOJ25?%cCFs<`=5Pu
zd&x0rBSg0U^a=R|bh;fskwNTM=E$@*{w9Lz=RGoDgvZeD_im*~Luuf9R+Pcd{i7IX
z)-Sq`LDjzXkj4oukv0Y$TBP@z2j-6D26|g~1>_JaQ%;5)FSEqRrC&S;aAgat9B;r=
zGw+?Bo1okzYJc*aa&ragJWKJ7-<>w+rnzM-cby9brC^2-((HS&lwu*KHC^miQl&h|
zfooEAMf*8#I<Hy~?vI(OMz05@?!CCMd{zCp1jujn4>@|g1-)`$|0=)EHDuEjF5AAF
zISe*=opq{!c1B6^y`nC=x97gLVEe@THuG?wVRd9r?@~VBU7r;5BV%&}nl&-QbCAZ3
zK;eXJ1*eit`s>1CX;Na*=9IqT!x(y~w4;8$G35UK9k+A8+#{ltC!p;!9=Hk%mbCy#
z7P*!JY(R~>2dqdsN0k$gtfBO?pU)zW_BP5f1IxD-Qn(Hz6w=P)xz(qSm&@FW-bHU;
zI4~M>AEctl4HM@-9{m__CI}0WEs!gF7t?lG>3~nJLx8HTN1V$I$#*?A_*%|-2L|)J
z9`<5K&&Eg4PQa00wWXY@@Z2S5VjF6b9R${FrF@){qa12@PI+9vxGHY4R-oN;3@c^i
zuN!Nq&<NItIax1c4`meEWQcg>CYGZ+m{@Ltm5dr};mlDa)}GpO;n#18V8{JSdX+#q
z%>m!g80^x}NdTZh)iC7r%fI8Oh=y^zX>*K9g{gPDfaj*`Y~qPCTNlELrxB!;cIUxu
znJ>;Uir6246)q+k6%!x^ozvn(Kd5c}lg{{J7%gOBcX~&gQ64zDL<$%2w#rY-9=~=a
zFXhE`xi_gg8s_TW@a|{PC%`j3))~Xjr-Tv9)%VP$1Oe}3P5R;a*1MyswU_vBVI>(O
z!yeLgti(@zQH;!A4C&P8+w}@bAV)zldBc>YqJzaL>`zi4Leqq}(&fzxX<4`Tbx_gS
zYTiyFVx2(;*`f^yRCE$PgyNXScfCWaPrvMm&Mo!&awN^wob-$xq1~@E5NQ#3$uuMS
zI8_6`uUvzR9y@-F?%Iak2#vgaKg?L3hrA!UW=(IiyC)t$4VOTJ^M#6NnE^I8R}Owu
z0NX#2lpE<B=$r-Sv$xoyuJYBzB#l8M@zc-GqF<wG6Tg%7a5?wD9cT4>_Y2>1F^A%@
zoHg$dptpXS;QiPz&c0k=>^Z#~TLH=CJlcR;b6QRP<(>U9#o+PDJ2lV>G}F%tV2U6I
zQfX_G-6ik1WpvtSipH^p!q(!>z(fHc)R!9TU)V75x6ONNZTo8Q_H|!YQmQO0-Xo4p
zXB!@ts6@*?=ODYr6O3vN=6Z*IMv#V4rq0}*q6&mtKXLw%vzJ+;Co2lg!Qo8;E1G>i
zo+CC8({l;n;C+Y#A3kN&vTz#p27B^>%jZmKz_ZA82?OUz2!=9E6RytG@Yrd6K{}ui
ztDfB-15+9>E(IFKWqZD>S0Glj>IGLU5A>5bg2VL-@Z$;67VJY3)rS_jNy2E=Eo+bL
zGe(w?xN~U}y0#4v?%DyTI!i@Yxp;rl^a(2x{zrBNtaLfMoJd89IHPr2gE|KERU5{U
zS~B{t2&xvGV~3=Z6xLEPW;KfMDIf-5995iH@JTJ@^7JDti!m|1w3}yxiRY)zf#Br)
z9}s63$0}2LIu|bru)}Bq1Ul>S7rt{2rFB-c_yb8BF-1C?B_qoRkRHU@muK2=;+gfW
z{X!L{tPNKnJ%S{?+ET~B-HM~;XwQ=>Lv&6WX{RtBt|5;j_U$2`-U13ZE9GP#?CBVw
z2r3PBy1ZdY$SKnIu@llmcJo2=IM$$)D(^vf6*uG-UJLe>zzX0iDI*TkY4Yvo!7E<W
z4}?>0zjhynm}Bj55s&FK`gJS=96O(0f!xZhYOP@EU8vgK0G$a&4~?Pz)WEiU8a;P9
zPT5Z<h{e*ToVk@7_e<A_1+g`(IDDb0*G-K1bWM!+VOA*0`M<&glyC2w$B+urW(&9-
z&Cxi{waG8_*y+S7lzqv~;T3mTreZH?i$2Yz&$7rzS6N27tskzLyB}Vbe-}HlYp`s=
zYkSmZ#lQ@(Oo84&7b0w4X=HWmNrLa_9SHu)$#08`9DpZHKsp%xF%rj%;xcu<2k9|+
zYs1D%A!J%rL;t${z6)^oLp+#C(8x-QL8|*aywj~NE!8D|1Q`MU4A;X9%lDeWfG;c&
zsa!MzhQMt4vct@`C^7aD2&)Jjot7pbV(KZ?bRDYPQN@`kj60Fux{9Td(Q(ofeoyKQ
zC<b!S+*BEvk<45I!jmy~iC8f)j%yeM_pK_`Cs|~=f|B~l@}q!Txc%avBeNll;#`sv
zZy7cok(RMj^KEel)PyLy-PLg<s&nzUNM`#M!J+WswmhohKlinxXS!mo7YVY~qS7B7
zOCv>KAd@jLLZyA|xvt)(p*H#gfGby)w~Mb}(S``miJ}}_7XbBJD}G%-YF*Mc$RjNc
z+cqu3x<yU^lWP`}Rs|idPxFnKpzsCM3HzyKZ|UyPRI2N|hI#g(jZ&tz66y^rA(f~E
z6H`s}%#c5@Ad$)+!>-Hc!2w*umd#b?J`8<eV`)IIn&^D#D5|qZ@d3GKBzqD$Pt%+(
zSqTsoYKOBUi_51~%r2kc^L7jAQyX?_i~0m?URzQe=y>;TZ9-;Db?^e!O{_m~S?kUt
z704rWgF(OB6biDObImNc6$P~>1(Ls?P6bHRq2`gL98cGI0C(*yhlmbHVNLi^V8UWy
zZl$8bGKhq#1o3>7$tF>>S}S3NZp7E4->VF&Y1SC4>C{=p>MUx9kSIZDgGp`79`=Xs
z#%7}?>Qs?5q6^ZnzN1f(yi#{Tr>}rw!YHGWKC<B|T&lsMNeu7hbMg)f8>Z+I0hbV8
z`*v<k??p@HWg0J{l^zHV0%ukl)Rm$Or554QsU_g#R>7tZCh)@3AlMWK1)u6B=-I&q
zs?<SZ<f6`eZy;_^YT~d0Ka1npN~Iz>d=VYxdxSZy4Ec24zz|bg(c);t$e)v2ZE+b8
zoX!y5d-15XG28wbEYA?ez4wxTWVhdGJ+qzbvYlgsa2JIFWYG}YsS4xPjN|A2YYlJb
z5mn--ZPK0ls7>tQRO|KaH?}2A-t+7W6CU#x%yi;YdP_)gW-5P6bwL9ih;Vo#03{z%
z0t3P=&sqnV=YXsp1ld=Gj!%M*m-ma<$jj*E++y?jrHpVtEi1`0BA3z8sx(ZzN2w-W
z@qKf=0NAUq<j+q}BeCZdnw1_7m(OZh9hvJ`(8TDrI>p|zXj69Q)mJ|=avM-icZqb0
z!v%#52h^gr0%6gv&C*STOR!=JG8j4*8p3G6UviiUetj8mzV(fiNkrQ>oZz3D+@_05
z?6c>MJEaR$r8K~#SRG)U2&-pf)LoXu2p}Q(0T8iw!X;3)^Uw|5)E`}ZC7(B@nLEDW
zDm!3qCMmwNw?p+Zfxp?}U}^Q}S6wZeW2l5Hwt#K^I-F55VDeMmhEC-KO7MHJ{2<^g
z4^wBzQn0I`AHsf2mrRU2E@PqgL^E44D*<o0x!&fz!BBJBd;p?c7C&iKF<;h7XR<U=
zN*JZHxX!H`L*z^w+e~mLj(l}RytaUk!b?Hr1iQAi-f`u1Z}IX)vPFtJA4ck)nVbw>
zqZ38p5Mo5jaKnYlONB9tCT9iX2XLNc;1YO$@`(&f8IaP&Q==WYeR9ap%q1r^x+6j1
zPtW4Ii0(`_%E9T>@XZNAmQWjEiDjpgmrflgi6F%jUIFQXj2;dAfo^_h;k{F}8r3lD
z?jr6b=`sYVt+5T+#+{&HKn-)=!nV8JIZ?Q!3$7Md#Q-^5|DbVKrP&q*iZ(-K9Y!Qn
zWdKESuG<f_h&l!6T!&sg_%hJX9<tAv!?H4~Nu|@WwHjO6v{}i7WCmv{PD7yv^^Av-
zR2ca`NsI_M6uw1Fc3WzNFHiYX&I@<jr@C$*kc&arbV^lkEQ^?EtEzR0ln&Lm1=Phf
zZ(}eOoXR1R2FhtfR^YoNF8wO&*i5o)Fgg60;m|~mHCE_1mu;gMD!{e3yvGol&w-!(
z5H4Vo{z_%TiJ-0?L!c4)xpCvQBwz%q!fdC*4);A#5^8jw5Gz=ZW=;0vdlhSLNy(-I
zFmIo=u~X+G4=W7M*NSlInQMeN18jc`WmuYX1X}g2fqY<p7pvN@(u&<eEv-C;<4YVR
z)`uez8d8|69|CE^i6?~icEY0fgIyr7UEP;E;EFYKi5=Rs>W5<bhNPAaJ_n<Ylev#!
zA-mw;%x;Gm2cd2SgnC!StUx?f?2VzblTsNCgjy|G*^fQ(puKkJ{Q$?fw+#;*>mb`7
z*qcS`^i1Bg&s1XRuI6E&=i28g#lHA|5z-{<Gd07l6;p-0+<`>(B%C{v2t$RmtQDSQ
z)NF;G8RZH&UOG3>NsT~%!F(j-Mjum%-C&=e(RB1x8`{VoxX3Qu>R4k*VP6OmoJQlr
z=xo=_mUym*x;zIn?-^?r368;`NCiFrq>X;b&u=)tpT`n3-xW>vdm6`7i3fR_7i(x7
z-!e1|To4&F4(;7CFW_y#bMId>d{FfXa=;dqc6q2YCGi_2$2VLE;jC~{ogB7#7aQEM
zOWWxAD~{6`(9uy1sRLI{*{@EJ;&^~H-)&<J4*$Wmt7U&f4T5z%rM{{JvZX<>qZ9mB
zcP!4J>2c7aQdGyK)yD<KT*!V8%e4R%CJXKqF7oCIdR$zKeBEQz<Pv!J2`MzY2FlNm
zv_T-U?$fRS+kh6$ONgFBbnRfJZ>Kax&{QumkutR<C8lG#L&TisAB5jg(3N5wqz+N_
z6-e6Nr>TR~LpWj8=j<TrNdrg$;F@>qMAe00!Oyv&67!ixHQmG*b?a_IR-8L0qO9qm
z*avHotv@2LjjV+XSau=FQjFhI7)wgD!cXdEVH1pah~v8I_iSyt^11e|*nzTr>q!;$
z{2c`8QGAO^5HHTC?HhfRYaG+1$#>js;hb(Il6-$=L8Yy)Z(f~kP~XWxRNc*XQZd%E
zls_9l`YpIm#DdARubz~0K{1<7A9t~#7#TM)j_6JKJ-9%u8Fus%7YOK0x9U!fsjTQL
z<mE2ZZXI6DeB}wH7^$WNm=j~`%W1cQ&U83hjmiXM3M#G){#$nZw`SD%Lt1Dp6aerJ
z6#zi~`>T?(mBpLx`2VIST&^J(yUdBw_Naj}B(BfBM&w0<a5qUNR}D-9$bEzZXJ%C5
ztVJu1SLJ-U_32?&YDlo1`f2=S=Tn;}8-aiYv%P+mHkUFX=W)-Y2D5KYQeUmb#LMj1
z!n?hgmFlho<{H#wCv!6)!^PpCtxU7b6?}X7qrR<YCvCze=WZG*#z(Tzh+xwJL}$DQ
z$lx+`=~SJz#_kO<G=Drz{W^(TSvtP(CDw%*XrTt|)1vgM!x#z2kGysrKW=)+ju8vL
z3RT*yidrmj=o0oCS6hmWc)gIg)f6u&&5eu&Zhp2?H(;7E3C2g2!i4(Kl8LF;vdpMq
ztDQ^Cz=$0pRs$$?SZlxcalehXt3Bb;ez#Q2XV2)puT*;d!nPIN0M_HMGR!M7-2LHe
zR2n@F4@xLBbA23evW70lIwh(!lw%q^;Nf5{f?0$I|Dw?t4PvYAwbaZDrw$V_^YA-w
zrkN&O3tf~GdFphGRw1HklUZz_Q16|NQE<tJPHQS!S6kTz)bL>XC}dG;o6~RVb!;UB
zs7N<cr9yG0^O^jZ6pI)4KKHnbv#1>S8@#++3u7lI!p+-Fg~#ihFPaaF?p}V&&NXrF
zWWamz)Eh_X0~q9RmX=J_=@U0hv-WK0XFc}~LNuA{(qzvg9`8c$sq-1QCcrrQmD-HT
z4o77y$tmbW5}2o`)9O}(^-^^AlLr|APS5)I79i0m4(`9G-@EnGbH04vUJj=5c)O4`
zS(_dTLNJl`t5l+5G>gvGF5FO@Uu>_+YezUv)f487SUE{Vt-8um++OFGQ~|Fd+8vW;
zOG3}F0O9!yfh8%*A7M8jrF;)MAZq*x_d$<{uza@H<yA8lL~5CbLMvNX_3dcIjy7=W
z`FU)thdlY@`MKtNTfVCxdUbS$1%Mh|rB4*I#&wARdaQPif5=D>K)Ik8AN~dIL;_Y#
z@8!cYD9M6?3XL!YB_be@%@t!_V{n~919M%Jt&AIo%nJ*^>{NoR#ckU3!CLi5Wc#+(
zn|Ivo()pP3-noE$N5B&HZ0BrS>BqoTRjUY}r8sH%qF%b|6nbicS7@cmx>%sRHFomg
zSvo*O&THa%XcnGHjwbNQ9?LAk`heBnVQ;!I$J7DFSSJET?UR?oKt&Kd4WWTdOOWGi
zU7_`menGdE{Lsby#M@G<tMH!`Ru|KJ)_|+POqogYW#w?Y#-Xl}ysS(r*RAXDanMMu
zbFB@#X|S@I((Nok@<_2yw#v&s(j{r*1NE=C!nYq$->;HLc0FtDtQQ4QU&Tm-7&5E-
z)n*-bnEs$bfK;@VU>2&Rx3s<q&jq==^<sN1(kD)@cq3|E(UjB7wak!k+E%*Wc2x5W
zCBFcL9S??7QxtzC;hYXsmY|oUCD?y&{g#jZtk{%3;d{L#=<{?qQx86*EJPC{;WB@t
zFC>8c740rFp6@HJ%wcgb?3N}!*a>FK_X~=;D_+kqJ)LWNV-C$hs@U-`Zm-j&nj!=k
z4MlU%I}`S|rQ)yBi0BAJcL50{0s}Y@(B=ZlIf<iE8s(Ah5OT{XU4|AHApA~uy-}UX
zZ`H}ApHEpmyqpFW9SmeFZki%)TVBa^2|NZP*d?&4zeYA6GDXhemw&3{Hpp|IyvSwh
zwzKGauWK>S(7%*7^k{uTiRpwOGqNiX=KwHBU6jI<agyKuhWvAShz_!mk+#0g@P`_s
z%1j5^^<~DmR!M5qgkF|zs`Gu!f*3)}c5-0^ab%K$A|G$_fRTv1A+(PS#yV;$Ty2do
zPWF75{2)=iXD6(`5tJRqm?qC+R(K&|J5r@tnXJ4c`*qs2k=q(zV(*~tDQ{dujSu(*
zks)&Vx|qv%b2T}iS!!#{mi+bm+9^}T?Qy@E+xAfH@$lYOK}Rby=PR6w7?)n7^TDN>
znjQ`*&ei5fL;;GNn{%0lT5nU{$`<kOz8?<o`?!DW`xT@REV1A6ztey9{cdI-o!`Xd
z{|+cnQvZ)y@&*Qf-DhAxB0xgT!o<qL#6u1D-($<~hy{fq#Dd%#+u<$!SH!}=z{=jr
z#lYb20{%tBLPZ_6i2XttR!VkUURGgOntpgfEj(6%o{^SOIyzQkaFlV7nNCScmTr84
zmhnS$*g5I&wPB4}v3TE?drw{6iwi|6#X?3|P2>-W@Bl>_FmNIWj5jl20ly7<3=fMo
z!2tjStN?F9iEo*=1MDq*3;3OL^klbYHnFubvv+Ye`}p>p|6wW=0OGf)t{(6JFvy#?
zyCHw`{%<m(5~9~WH~@hA4gf&<2XggWG5<++uyZtW`Tt<5Jmdw>KmY*KC;$NJKQK4&
z|6-b3y^YsaUjNUg{vP?kc9Q025CH%uNx&QQ&q{b9{tNpb+L3S3C0jESm$x?YpA&+&
zirArK(Em3T@y}Y6f6~dB{-pn@O!>c)|2Yo!CmAgCFY-S`#s2pf_s{d7KdBn2e^LLO
z75zI?|CNLN302AZ3;L(Ky+MDw+rO#VfA*?Bp|yWQ|G~%p)$>C^|8DH>-cIzl#wGpr
H_tpOa3@;Xb

literal 0
HcmV?d00001

diff --git a/python/setup.py b/python/setup.py
index 40b49aaeeb27c..622e6077dc36e 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -206,7 +206,7 @@ def _supports_symlinks():
             'pyspark.examples.src.main.python': ['*.py', '*/*.py']},
         scripts=scripts,
         license='http://www.apache.org/licenses/LICENSE-2.0',
-        install_requires=['py4j==0.10.8.1'],
+        install_requires=['py4j==0.10.9'],
         extras_require={
             'ml': ['numpy>=1.7'],
             'mllib': ['numpy>=1.7'],
diff --git a/sbin/spark-config.sh b/sbin/spark-config.sh
index 0771e2a044757..b53442ec096a1 100755
--- a/sbin/spark-config.sh
+++ b/sbin/spark-config.sh
@@ -28,6 +28,6 @@ export SPARK_CONF_DIR="${SPARK_CONF_DIR:-"${SPARK_HOME}/conf"}"
 # Add the PySpark classes to the PYTHONPATH:
 if [ -z "${PYSPARK_PYTHONPATH_SET}" ]; then
   export PYTHONPATH="${SPARK_HOME}/python:${PYTHONPATH}"
-  export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.8.1-src.zip:${PYTHONPATH}"
+  export PYTHONPATH="${SPARK_HOME}/python/lib/py4j-0.10.9-src.zip:${PYTHONPATH}"
   export PYSPARK_PYTHONPATH_SET=1
 fi

From 5f6d158ed8c0d61b2e7749c3d0d2aaa1d9db49ff Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Fri, 21 Feb 2020 10:06:45 +0900
Subject: [PATCH 0130/1280] [SPARK-30907][DOCS] Revise the doc of
 spark.ui.retainedTasks

### What changes were proposed in this pull request?

Revise the documentation of `spark.ui.retainedTasks` to make it clear that the configuration is for one stage.

### Why are the changes needed?

There are configurations for the limitation of UI data.
`spark.ui.retainedJobs`, `spark.ui.retainedStages` and `spark.worker.ui.retainedExecutors` are the total max number for one application, while the configuration `spark.ui.retainedTasks` is the max number for one stage.
### Does this PR introduce any user-facing change?

No

### How was this patch tested?

None, just doc.

Closes #27660 from gengliangwang/reviseRetainTask.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 2a695e6d159fe49efe7601383fec4f3f172c5a97)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index df1cbe4c2ebea..2421e00f39068 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1101,7 +1101,7 @@ Apart from these, the following properties are also available, and may be useful
   <td><code>spark.ui.retainedTasks</code></td>
   <td>100000</td>
   <td>
-    How many tasks the Spark UI and status APIs remember before garbage collecting.
+    How many tasks in one stage the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
 </tr>

From 700be66ac48ec8bdac898ad1b136f819de074597 Mon Sep 17 00:00:00 2001
From: wuyi <yi.wu@databricks.com>
Date: Fri, 21 Feb 2020 12:55:32 +0800
Subject: [PATCH 0131/1280] Revert "[SPARK-28093][SQL] Fix TRIM/LTRIM/RTRIM
 function parameter order issue"

### What changes were proposed in this pull request?

This reverts commit bef5d9d6c348e390f99b2cd781a2471d635e55f8.

### Why are the changes needed?

Revert it according to https://github.com/apache/spark/pull/24902#issuecomment-584511167.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #27540 from Ngone51/revert_spark_28093.

Lead-authored-by: wuyi <yi.wu@databricks.com>
Co-authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/stringExpressions.scala       | 18 ++-----
 .../sql-tests/inputs/postgreSQL/strings.sql   |  3 +-
 .../sql-tests/inputs/string-functions.sql     | 18 +++----
 .../results/postgreSQL/strings.sql.out        | 10 +---
 .../results/string-functions.sql.out          | 48 +++++++++----------
 5 files changed, 39 insertions(+), 58 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 211ae3f02a0d8..cc09f601db9c4 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -719,8 +719,6 @@ object StringTrim {
 
     _FUNC_(TRAILING FROM str) - Removes the trailing space characters from `str`.
 
-    _FUNC_(str, trimStr) - Remove the leading and trailing `trimStr` characters from `str`.
-
     _FUNC_(trimStr FROM str) - Remove the leading and trailing `trimStr` characters from `str`.
 
     _FUNC_(BOTH trimStr FROM str) - Remove the leading and trailing `trimStr` characters from `str`.
@@ -750,8 +748,6 @@ object StringTrim {
        SparkSQL
       > SELECT _FUNC_(TRAILING FROM '    SparkSQL   ');
            SparkSQL
-      > SELECT _FUNC_('SSparkSQLS', 'SL');
-       parkSQ
       > SELECT _FUNC_('SL' FROM 'SSparkSQLS');
        parkSQ
       > SELECT _FUNC_(BOTH 'SL' FROM 'SSparkSQLS');
@@ -767,7 +763,7 @@ case class StringTrim(
     trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
-  def this(srcStr: Expression, trimStr: Expression) = this(srcStr, Option(trimStr))
+  def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
 
   def this(srcStr: Expression) = this(srcStr, None)
 
@@ -846,8 +842,6 @@ object StringTrimLeft {
 @ExpressionDescription(
   usage = """
     _FUNC_(str) - Removes the leading space characters from `str`.
-
-    _FUNC_(str, trimStr) - Removes the leading string contains the characters from the trim string
   """,
   arguments = """
     Arguments:
@@ -858,8 +852,6 @@ object StringTrimLeft {
     Examples:
       > SELECT _FUNC_('    SparkSQL   ');
        SparkSQL
-      > SELECT _FUNC_('SparkSQLS', 'Sp');
-       arkSQLS
   """,
   since = "1.5.0")
 case class StringTrimLeft(
@@ -867,7 +859,7 @@ case class StringTrimLeft(
     trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
-  def this(srcStr: Expression, trimStr: Expression) = this(srcStr, Option(trimStr))
+  def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
 
   def this(srcStr: Expression) = this(srcStr, None)
 
@@ -948,8 +940,6 @@ object StringTrimRight {
 @ExpressionDescription(
   usage = """
     _FUNC_(str) - Removes the trailing space characters from `str`.
-
-    _FUNC_(str, trimStr) - Removes the trailing string which contains the characters from the trim string from the `str`
   """,
   arguments = """
     Arguments:
@@ -960,8 +950,6 @@ object StringTrimRight {
     Examples:
       > SELECT _FUNC_('    SparkSQL   ');
        SparkSQL
-      > SELECT _FUNC_('SSparkSQLS', 'SQLS');
-       SSpark
   """,
   since = "1.5.0")
 // scalastyle:on line.size.limit
@@ -970,7 +958,7 @@ case class StringTrimRight(
     trimStr: Option[Expression] = None)
   extends String2TrimExpression {
 
-  def this(srcStr: Expression, trimStr: Expression) = this(srcStr, Option(trimStr))
+  def this(trimStr: Expression, srcStr: Expression) = this(srcStr, Option(trimStr))
 
   def this(srcStr: Expression) = this(srcStr, None)
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
index 541ff0bdad745..e2a94404395bc 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/strings.sql
@@ -630,7 +630,8 @@ SELECT rpad('hi', -5, 'xy');
 SELECT rpad('hello', 2);
 SELECT rpad('hi', 5, '');
 
-SELECT ltrim('zzzytrim', 'xyz');
+-- skip this test because PostgreSQL has different parameter order compares to SparkSQL
+-- SELECT ltrim('zzzytrim', 'xyz');
 
 SELECT translate('', '14', 'ax');
 SELECT translate('12345', '14', 'ax');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
index 59987b9792e25..fd6cc4d811045 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
@@ -39,12 +39,12 @@ SELECT substring('Spark SQL' from 5);
 SELECT substring('Spark SQL' from -3);
 SELECT substring('Spark SQL' from 5 for 1);
 
--- trim/ltrim/rtrim
-SELECT trim('yxTomxx', 'xyz'), trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx');
-SELECT trim('xxxbarxxx', 'x'), trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx');
-SELECT ltrim('zzzytest', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytest');
-SELECT ltrim('zzzytestxyz', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytestxyz');
-SELECT ltrim('xyxXxyLAST WORD', 'xy'), trim(LEADING 'xy' FROM 'xyxXxyLAST WORD');
-SELECT rtrim('testxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'testxxzx');
-SELECT rtrim('xyztestxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'xyztestxxzx');
-SELECT rtrim('TURNERyxXxy', 'xy'), trim(TRAILING 'xy' FROM 'TURNERyxXxy');
+-- trim
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx');
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx');
+SELECT trim(LEADING 'xyz' FROM 'zzzytest');
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz');
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD');
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx');
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx');
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy');
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
index c30eea8ab689d..3b26d561d20c3 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 122
+-- Number of queries: 121
 
 
 -- !query
@@ -910,14 +910,6 @@ struct<rpad(hi, 5, ):string>
 hi
 
 
--- !query
-SELECT ltrim('zzzytrim', 'xyz')
--- !query schema
-struct<ltrim(zzzytrim, xyz):string>
--- !query output
-trim
-
-
 -- !query
 SELECT translate('', '14', 'ax')
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 33d1b25aee483..708dbb404c285 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -205,64 +205,64 @@ k
 
 
 -- !query
-SELECT trim('yxTomxx', 'xyz'), trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
 -- !query schema
-struct<trim(yxTomxx, xyz):string,trim(yxTomxx, xyz):string,trim(yxTomxx, xyz):string>
+struct<trim(yxTomxx, xyz):string,trim(yxTomxx, xyz):string>
 -- !query output
-Tom	Tom	Tom
+Tom	Tom
 
 
 -- !query
-SELECT trim('xxxbarxxx', 'x'), trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
 -- !query schema
-struct<trim(xxxbarxxx, x):string,trim(xxxbarxxx, x):string,trim(xxxbarxxx, x):string>
+struct<trim(xxxbarxxx, x):string,trim(xxxbarxxx, x):string>
 -- !query output
-bar	bar	bar
+bar	bar
 
 
 -- !query
-SELECT ltrim('zzzytest', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytest')
+SELECT trim(LEADING 'xyz' FROM 'zzzytest')
 -- !query schema
-struct<ltrim(zzzytest, xyz):string,ltrim(zzzytest, xyz):string>
+struct<ltrim(zzzytest, xyz):string>
 -- !query output
-test	test
+test
 
 
 -- !query
-SELECT ltrim('zzzytestxyz', 'xyz'), trim(LEADING 'xyz' FROM 'zzzytestxyz')
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz')
 -- !query schema
-struct<ltrim(zzzytestxyz, xyz):string,ltrim(zzzytestxyz, xyz):string>
+struct<ltrim(zzzytestxyz, xyz):string>
 -- !query output
-testxyz	testxyz
+testxyz
 
 
 -- !query
-SELECT ltrim('xyxXxyLAST WORD', 'xy'), trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
 -- !query schema
-struct<ltrim(xyxXxyLAST WORD, xy):string,ltrim(xyxXxyLAST WORD, xy):string>
+struct<ltrim(xyxXxyLAST WORD, xy):string>
 -- !query output
-XxyLAST WORD	XxyLAST WORD
+XxyLAST WORD
 
 
 -- !query
-SELECT rtrim('testxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'testxxzx')
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx')
 -- !query schema
-struct<rtrim(testxxzx, xyz):string,rtrim(testxxzx, xyz):string>
+struct<rtrim(testxxzx, xyz):string>
 -- !query output
-test	test
+test
 
 
 -- !query
-SELECT rtrim('xyztestxxzx', 'xyz'), trim(TRAILING 'xyz' FROM 'xyztestxxzx')
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx')
 -- !query schema
-struct<rtrim(xyztestxxzx, xyz):string,rtrim(xyztestxxzx, xyz):string>
+struct<rtrim(xyztestxxzx, xyz):string>
 -- !query output
-xyztest	xyztest
+xyztest
 
 
 -- !query
-SELECT rtrim('TURNERyxXxy', 'xy'), trim(TRAILING 'xy' FROM 'TURNERyxXxy')
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
 -- !query schema
-struct<rtrim(TURNERyxXxy, xy):string,rtrim(TURNERyxXxy, xy):string>
+struct<rtrim(TURNERyxXxy, xy):string>
 -- !query output
-TURNERyxX	TURNERyxX
+TURNERyxX

From a415d07c90ad46c9d88d78e956cb5680b213ce71 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 21 Feb 2020 15:21:53 +0900
Subject: [PATCH 0132/1280] [MINOR][SQL] Fix error position of NOSCAN

### What changes were proposed in this pull request?

Point to correct position when miswrite `NOSCAN` detects.

### Why are the changes needed?

Before:

```
[info]   org.apache.spark.sql.catalyst.parser.ParseException: Expected `NOSCAN` instead of `SCAN`(line 1, pos 0)
[info]
[info] == SQL ==
[info] ANALYZE TABLE analyze_partition_with_null PARTITION (name) COMPUTE STATISTICS SCAN
[info] ^^^
```

After:

```
[info]   org.apache.spark.sql.catalyst.parser.ParseException: Expected `NOSCAN` instead of `SCAN`(line 1, pos 78)
[info]
[info] == SQL ==
[info] ANALYZE TABLE analyze_partition_with_null PARTITION (name) COMPUTE STATISTICS SCAN
[info] ------------------------------------------------------------------------------^^^
```

### Does this PR introduce any user-facing change?

Yes, user will see better error message.

### How was this patch tested?

Manually test.

Closes #27662 from Ngone51/fix_noscan_reference.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4d356554a61024c7d3dc450accec1b3639c37e19)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../org/apache/spark/sql/catalyst/parser/AstBuilder.scala      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 62e568587fcc6..36c16479e6be0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -3165,7 +3165,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     }
     if (ctx.identifier != null &&
         ctx.identifier.getText.toLowerCase(Locale.ROOT) != "noscan") {
-      throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`", ctx)
+      throw new ParseException(s"Expected `NOSCAN` instead of `${ctx.identifier.getText}`",
+        ctx.identifier())
     }
 
     val tableName = visitMultipartIdentifier(ctx.multipartIdentifier())

From bc30a07ce262840c99a752db4fbd3a423f652017 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 21 Feb 2020 14:46:54 +0800
Subject: [PATCH 0133/1280] [SPARK-26580][SQL][ML][FOLLOW-UP] Throw exception
 when use untyped UDF by default

### What changes were proposed in this pull request?

This PR proposes to throw exception by default when user use untyped UDF(a.k.a `org.apache.spark.sql.functions.udf(AnyRef, DataType)`).

And user could still use it by setting `spark.sql.legacy.useUnTypedUdf.enabled` to `true`.

### Why are the changes needed?

According to #23498, since Spark 3.0, the untyped UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will  return 0 in Spark 3.0 but null in Spark 2.4. And the behavior change is introduced due to Spark3.0 is built with Scala 2.12 by default.

As a result, this might change data silently and may cause correctness issue if user still expect `null` in some cases. Thus, we'd better to encourage user to use typed UDF to avoid this problem.

### Does this PR introduce any user-facing change?

Yeah. User will hit exception now when use untyped UDF.

### How was this patch tested?

Added test and updated some tests.

Closes #27488 from Ngone51/spark_26580_followup.

Lead-authored-by: yi.wu <yi.wu@databricks.com>
Co-authored-by: wuyi <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 82ce4753aafafc590f0cd6ba47d5db20a2d6137b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |  2 +-
 .../org/apache/spark/ml/Transformer.scala     |  5 +--
 .../org/apache/spark/ml/feature/LSH.scala     | 11 +++----
 .../org/apache/spark/ml/fpm/FPGrowth.scala    | 11 ++++---
 .../org/apache/spark/ml/feature/LSHTest.scala |  9 +++--
 project/MimaExcludes.scala                    |  3 ++
 .../apache/spark/sql/internal/SQLConf.scala   |  8 +++++
 .../sql/expressions/UserDefinedFunction.scala |  2 +-
 .../org/apache/spark/sql/functions.scala      |  9 +++++
 .../scala/org/apache/spark/sql/UDFSuite.scala | 33 ++++++++++++-------
 10 files changed, 62 insertions(+), 31 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 9b74b456b88e7..7e52e6924d9fc 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -63,7 +63,7 @@ license: |
 
   - Since Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inferring.
 
-  - In Spark version 2.4 and earlier, if `org.apache.spark.sql.functions.udf(Any, DataType)` gets a Scala closure with primitive-type argument, the returned UDF will return null if the input values is null. Since Spark 3.0, the UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will return null in Spark 2.4 and earlier if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
+  - Since Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. But please note that, in Spark version 2.4 and earlier, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF will return null if the input values is null. However, since Spark 3.0, the UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will return null in Spark 2.4 and earlier if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
 
   - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
index 7874fc29db6c8..1652131a9003a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.ml
 
 import scala.annotation.varargs
+import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.annotation.{DeveloperApi, Since}
 import org.apache.spark.internal.Logging
@@ -79,7 +80,7 @@ abstract class Transformer extends PipelineStage {
  * result as a new column.
  */
 @DeveloperApi
-abstract class UnaryTransformer[IN, OUT, T <: UnaryTransformer[IN, OUT, T]]
+abstract class UnaryTransformer[IN: TypeTag, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]
   extends Transformer with HasInputCol with HasOutputCol with Logging {
 
   /** @group setParam */
@@ -118,7 +119,7 @@ abstract class UnaryTransformer[IN, OUT, T <: UnaryTransformer[IN, OUT, T]]
 
   override def transform(dataset: Dataset[_]): DataFrame = {
     val outputSchema = transformSchema(dataset.schema, logging = true)
-    val transformUDF = udf(this.createTransformFunc, outputDataType)
+    val transformUDF = udf(this.createTransformFunc)
     dataset.withColumn($(outputCol), transformUDF(dataset($(inputCol))),
       outputSchema($(outputCol)).metadata)
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
index 01741019fb546..6d5c7c50dbacc 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/LSH.scala
@@ -98,7 +98,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
 
   override def transform(dataset: Dataset[_]): DataFrame = {
     transformSchema(dataset.schema, logging = true)
-    val transformUDF = udf(hashFunction(_: Vector), DataTypes.createArrayType(new VectorUDT))
+    val transformUDF = udf(hashFunction(_: Vector))
     dataset.withColumn($(outputCol), transformUDF(dataset($(inputCol))))
   }
 
@@ -128,14 +128,13 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
       }
 
       // In the origin dataset, find the hash value that hash the same bucket with the key
-      val sameBucketWithKeyUDF = udf((x: Seq[Vector]) =>
-        sameBucket(x, keyHash), DataTypes.BooleanType)
+      val sameBucketWithKeyUDF = udf((x: Seq[Vector]) => sameBucket(x, keyHash))
 
       modelDataset.filter(sameBucketWithKeyUDF(col($(outputCol))))
     } else {
       // In the origin dataset, find the hash value that is closest to the key
       // Limit the use of hashDist since it's controversial
-      val hashDistUDF = udf((x: Seq[Vector]) => hashDistance(x, keyHash), DataTypes.DoubleType)
+      val hashDistUDF = udf((x: Seq[Vector]) => hashDistance(x, keyHash))
       val hashDistCol = hashDistUDF(col($(outputCol)))
       val modelDatasetWithDist = modelDataset.withColumn(distCol, hashDistCol)
 
@@ -172,7 +171,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
     }
 
     // Get the top k nearest neighbor by their distance to the key
-    val keyDistUDF = udf((x: Vector) => keyDistance(x, key), DataTypes.DoubleType)
+    val keyDistUDF = udf((x: Vector) => keyDistance(x, key))
     val modelSubsetWithDistCol = modelSubset.withColumn(distCol, keyDistUDF(col($(inputCol))))
     modelSubsetWithDistCol.sort(distCol).limit(numNearestNeighbors)
   }
@@ -290,7 +289,7 @@ private[ml] abstract class LSHModel[T <: LSHModel[T]]
       .drop(explodeCols: _*).distinct()
 
     // Add a new column to store the distance of the two rows.
-    val distUDF = udf((x: Vector, y: Vector) => keyDistance(x, y), DataTypes.DoubleType)
+    val distUDF = udf((x: Vector, y: Vector) => keyDistance(x, y))
     val joinedDatasetWithDist = joinedDataset.select(col("*"),
       distUDF(col(s"$leftColName.${$(inputCol)}"), col(s"$rightColName.${$(inputCol)}")).as(distCol)
     )
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index 4d001c159eda0..e50d4255b1f37 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -29,10 +29,10 @@ import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared.HasPredictionCol
 import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
-import org.apache.spark.mllib.fpm.{AssociationRules => MLlibAssociationRules,
-  FPGrowth => MLlibFPGrowth}
+import org.apache.spark.mllib.fpm.{AssociationRules => MLlibAssociationRules, FPGrowth => MLlibFPGrowth}
 import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset
 import org.apache.spark.sql._
+import org.apache.spark.sql.expressions.SparkUserDefinedFunction
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 import org.apache.spark.storage.StorageLevel
@@ -286,14 +286,17 @@ class FPGrowthModel private[ml] (
 
     val dt = dataset.schema($(itemsCol)).dataType
     // For each rule, examine the input items and summarize the consequents
-    val predictUDF = udf((items: Seq[Any]) => {
+    val predictUDF = SparkUserDefinedFunction((items: Seq[Any]) => {
       if (items != null) {
         val itemset = items.toSet
         brRules.value.filter(_._1.forall(itemset.contains))
           .flatMap(_._2.filter(!itemset.contains(_))).distinct
       } else {
         Seq.empty
-      }}, dt)
+      }},
+      dt,
+      Nil
+    )
     dataset.withColumn($(predictionCol), predictUDF(col($(itemsCol))))
   }
 
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
index 76a4acd798e34..1d052fbebd92d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/LSHTest.scala
@@ -76,9 +76,8 @@ private[ml] object LSHTest {
 
     // Perform a cross join and label each pair of same_bucket and distance
     val pairs = transformedData.as("a").crossJoin(transformedData.as("b"))
-    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y), DataTypes.DoubleType)
-    val sameBucket = udf((x: Seq[Vector], y: Seq[Vector]) => model.hashDistance(x, y) == 0.0,
-      DataTypes.BooleanType)
+    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y))
+    val sameBucket = udf((x: Seq[Vector], y: Seq[Vector]) => model.hashDistance(x, y) == 0.0)
     val result = pairs
       .withColumn("same_bucket", sameBucket(col(s"a.$outputCol"), col(s"b.$outputCol")))
       .withColumn("distance", distUDF(col(s"a.$inputCol"), col(s"b.$inputCol")))
@@ -110,7 +109,7 @@ private[ml] object LSHTest {
     val model = lsh.fit(dataset)
 
     // Compute expected
-    val distUDF = udf((x: Vector) => model.keyDistance(x, key), DataTypes.DoubleType)
+    val distUDF = udf((x: Vector) => model.keyDistance(x, key))
     val expected = dataset.sort(distUDF(col(model.getInputCol))).limit(k)
 
     // Compute actual
@@ -148,7 +147,7 @@ private[ml] object LSHTest {
     val inputCol = model.getInputCol
 
     // Compute expected
-    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y), DataTypes.DoubleType)
+    val distUDF = udf((x: Vector, y: Vector) => model.keyDistance(x, y))
     val expected = datasetA.as("a").crossJoin(datasetB.as("b"))
       .filter(distUDF(col(s"a.$inputCol"), col(s"b.$inputCol")) < threshold)
 
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 65ffa228eddec..2a72284c011f5 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -74,6 +74,9 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.getRuns"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.setRuns"),
 
+    // [SPARK-26580][SQL][ML][FOLLOW-UP] Throw exception when use untyped UDF by default
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.UnaryTransformer.this"),
+
     // [SPARK-27090][CORE] Removing old LEGACY_DRIVER_IDENTIFIER ("<driver>")
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.SparkContext.LEGACY_DRIVER_IDENTIFIER"),
     // [SPARK-25838] Remove formatVersion from Saveable
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index bcec22f91924b..c73d8b6c1e7c5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2017,6 +2017,14 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val LEGACY_ALLOW_UNTYPED_SCALA_UDF =
+    buildConf("spark.sql.legacy.allowUntypedScalaUDF")
+      .internal()
+      .doc("When set to true, user is allowed to use org.apache.spark.sql.functions." +
+        "udf(f: AnyRef, dataType: DataType). Otherwise, exception will be throw.")
+      .booleanConf
+      .createWithDefault(false)
+
   val TRUNCATE_TABLE_IGNORE_PERMISSION_ACL =
     buildConf("spark.sql.truncateTable.ignorePermissionAcl.enabled")
       .internal()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index 85b2cd379ba24..c50168cf7ac13 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -90,7 +90,7 @@ sealed abstract class UserDefinedFunction {
   def asNondeterministic(): UserDefinedFunction
 }
 
-private[sql] case class SparkUserDefinedFunction(
+private[spark] case class SparkUserDefinedFunction(
     f: AnyRef,
     dataType: DataType,
     inputSchemas: Seq[Option[ScalaReflection.Schema]],
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 2d5504ac00ffa..c60df14f04817 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -4733,6 +4733,15 @@ object functions {
    * @since 2.0.0
    */
   def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = {
+    if (!SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF)) {
+      val errorMsg = "You're using untyped Scala UDF, which does not have the input type " +
+        "information. Spark may blindly pass null to the Scala closure with primitive-type " +
+        "argument, and the closure will see the default value of the Java type for the null " +
+        "argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. " +
+        "You could use other typed Scala UDF APIs to avoid this problem, or set " +
+        s"${SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key} to true and use this API with caution."
+      throw new AnalysisException(errorMsg)
+    }
     SparkUserDefinedFunction(f, dataType, inputSchemas = Nil)
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index cc3995516dcc2..cbe2e91a20d61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -134,10 +134,12 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     assert(df1.logicalPlan.asInstanceOf[Project].projectList.forall(!_.deterministic))
     assert(df1.head().getDouble(0) >= 0.0)
 
-    val bar = udf(() => Math.random(), DataTypes.DoubleType).asNondeterministic()
-    val df2 = testData.select(bar())
-    assert(df2.logicalPlan.asInstanceOf[Project].projectList.forall(!_.deterministic))
-    assert(df2.head().getDouble(0) >= 0.0)
+    withSQLConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key -> "true") {
+      val bar = udf(() => Math.random(), DataTypes.DoubleType).asNondeterministic()
+      val df2 = testData.select(bar())
+      assert(df2.logicalPlan.asInstanceOf[Project].projectList.forall(!_.deterministic))
+      assert(df2.head().getDouble(0) >= 0.0)
+    }
 
     val javaUdf = udf(new UDF0[Double] {
       override def call(): Double = Math.random()
@@ -441,16 +443,23 @@ class UDFSuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-25044 Verify null input handling for primitive types - with udf(Any, DataType)") {
-    val f = udf((x: Int) => x, IntegerType)
-    checkAnswer(
-      Seq(Integer.valueOf(1), null).toDF("x").select(f($"x")),
-      Row(1) :: Row(0) :: Nil)
+    withSQLConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key -> "true") {
+      val f = udf((x: Int) => x, IntegerType)
+      checkAnswer(
+        Seq(Integer.valueOf(1), null).toDF("x").select(f($"x")),
+        Row(1) :: Row(0) :: Nil)
+
+      val f2 = udf((x: Double) => x, DoubleType)
+      checkAnswer(
+        Seq(java.lang.Double.valueOf(1.1), null).toDF("x").select(f2($"x")),
+        Row(1.1) :: Row(0.0) :: Nil)
+    }
 
-    val f2 = udf((x: Double) => x, DoubleType)
-    checkAnswer(
-      Seq(java.lang.Double.valueOf(1.1), null).toDF("x").select(f2($"x")),
-      Row(1.1) :: Row(0.0) :: Nil)
+  }
 
+  test("use untyped Scala UDF should fail by default") {
+    val e = intercept[AnalysisException](udf((x: Int) => x, IntegerType))
+    assert(e.getMessage.contains("You're using untyped Scala UDF"))
   }
 
   test("SPARK-26308: udf with decimal") {

From db30c059344b54e81d9492a528860a300b58891f Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 21 Feb 2020 15:32:11 +0800
Subject: [PATCH 0134/1280] [SPARK-30894][SQL] Make Size's nullable independent
 from SQL config changes

### What changes were proposed in this pull request?
In the PR, I propose to add the `legacySizeOfNull ` parameter to the `Size` expression, and pass the value of `spark.sql.legacy.sizeOfNull` if `legacySizeOfNull` is not provided on creation of `Size`.

### Why are the changes needed?
This allows to avoid the issue when the configuration change between different phases of planning, and this can silently break a query plan which can lead to crashes or data corruption.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By `CollectionExpressionsSuite`.

Closes #27658 from MaxGekk/Size-SQLConf-get-deps.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit abe0821ee9414a441461e5a4a6cf5b781b46a2e8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/expressions/collectionOperations.scala  | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 2b123479a1909..cfa877b9c0892 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -90,9 +90,10 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
       > SELECT _FUNC_(NULL);
        NULL
   """)
-case class Size(child: Expression) extends UnaryExpression with ExpectsInputTypes {
+case class Size(child: Expression, legacySizeOfNull: Boolean)
+  extends UnaryExpression with ExpectsInputTypes {
 
-  val legacySizeOfNull = SQLConf.get.legacySizeOfNull
+  def this(child: Expression) = this(child, SQLConf.get.legacySizeOfNull)
 
   override def dataType: DataType = IntegerType
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(ArrayType, MapType))
@@ -124,6 +125,10 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType
   }
 }
 
+object Size {
+  def apply(child: Expression): Size = new Size(child)
+}
+
 /**
  * Returns an unordered array containing the keys of the map.
  */

From 483c789f3be8cb9df6fbfdff75980955ff5391c2 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Fri, 21 Feb 2020 17:03:22 +0800
Subject: [PATCH 0135/1280] [SPARK-30809][SQL] Review and fix issues in SQL API
 docs

### What changes were proposed in this pull request?
- Add missing `since` annotation.
- Don't show classes under `org.apache.spark.sql.dynamicpruning` package in API docs.
- Fix the scope of `xxxExactNumeric` to remove it from the API docs.

### Why are the changes needed?
Avoid leaking APIs unintentionally in Spark 3.0.0.

### Does this PR introduce any user-facing change?
No. All these changes are to avoid leaking APIs unintentionally in Spark 3.0.0.

### How was this patch tested?
Manually generated the API docs and verified the above issues have been fixed.

Closes #27560 from xuanyuanking/SPARK-30809.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a5efbb284e29b1d879490a4ee2c9fa08acec42b0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/rpc/RpcEndpointRef.scala |   2 +-
 project/SparkBuild.scala                      |   1 +
 .../sql/connector/ExternalCommandRunner.java  |   2 +
 .../connector/catalog/CatalogExtension.java   |   2 +
 .../sql/connector/catalog/CatalogPlugin.java  |   2 +
 .../spark/sql/connector/catalog/Catalogs.java | 127 ------------------
 .../catalog/DelegatingCatalogExtension.java   |   2 +
 .../sql/connector/catalog/Identifier.java     |   2 +
 .../connector/catalog/NamespaceChange.java    |   2 +
 .../catalog/SessionConfigSupport.java         |   2 +
 .../sql/connector/catalog/StagedTable.java    |   2 +
 .../catalog/StagingTableCatalog.java          |   2 +
 .../catalog/SupportsCatalogOptions.java       |   2 +
 .../sql/connector/catalog/SupportsDelete.java |   2 +
 .../connector/catalog/SupportsNamespaces.java |   2 +
 .../sql/connector/catalog/SupportsRead.java   |   2 +
 .../sql/connector/catalog/SupportsWrite.java  |   2 +
 .../spark/sql/connector/catalog/Table.java    |   2 +
 .../connector/catalog/TableCapability.java    |   2 +
 .../sql/connector/catalog/TableCatalog.java   |   2 +
 .../sql/connector/catalog/TableChange.java    |   2 +
 .../sql/connector/catalog/TableProvider.java  |   2 +
 .../sql/connector/expressions/Expression.java |   2 +
 .../connector/expressions/Expressions.java    |   2 +
 .../sql/connector/expressions/Literal.java    |   1 +
 .../connector/expressions/NamedReference.java |   2 +
 .../sql/connector/expressions/Transform.java  |   2 +
 .../spark/sql/connector/read/Batch.java       |   2 +
 .../sql/connector/read/InputPartition.java    |   2 +
 .../sql/connector/read/PartitionReader.java   |   2 +
 .../read/PartitionReaderFactory.java          |   2 +
 .../apache/spark/sql/connector/read/Scan.java |   2 +
 .../spark/sql/connector/read/ScanBuilder.java |   2 +
 .../spark/sql/connector/read/Statistics.java  |   2 +
 .../read/SupportsPushDownFilters.java         |   2 +
 .../read/SupportsPushDownRequiredColumns.java |   2 +
 .../read/SupportsReportPartitioning.java      |   2 +
 .../read/SupportsReportStatistics.java        |   2 +
 .../partitioning/ClusteredDistribution.java   |   2 +
 .../read/partitioning/Distribution.java       |   2 +
 .../read/partitioning/Partitioning.java       |   2 +
 .../spark/sql/connector/write/BatchWrite.java |   2 +
 .../spark/sql/connector/write/DataWriter.java |   2 +
 .../connector/write/DataWriterFactory.java    |   2 +
 .../sql/connector/write/LogicalWriteInfo.java |   2 +
 .../connector/write/PhysicalWriteInfo.java    |   2 +
 .../write/SupportsDynamicOverwrite.java       |   2 +
 .../connector/write/SupportsOverwrite.java    |   2 +
 .../sql/connector/write/SupportsTruncate.java |   2 +
 .../sql/connector/write/WriteBuilder.java     |   2 +
 .../connector/write/WriterCommitMessage.java  |   2 +
 .../sql/util/CaseInsensitiveStringMap.java    |   2 +
 .../expressions/bitwiseExpressions.scala      |   3 +-
 .../spark/sql/catalyst/expressions/hash.scala |   3 +-
 .../sql/catalyst/util/DataTypeJsonUtils.scala |  55 ++++++++
 .../sql/connector/catalog/Catalogs.scala      |  99 ++++++++++++++
 .../connector/expressions/expressions.scala   |   2 +-
 .../apache/spark/sql/sources/filters.scala    |   4 +
 .../org/apache/spark/sql/types/DataType.scala |  32 +----
 .../org/apache/spark/sql/types/numerics.scala |  14 +-
 .../apache/spark/sql/DataFrameWriterV2.scala  |   3 +
 61 files changed, 273 insertions(+), 169 deletions(-)
 delete mode 100644 sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java
 create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
 create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala

diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
index 49d58929a97a4..56f3d377f8e2a 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
@@ -108,7 +108,7 @@ private[spark] abstract class RpcEndpointRef(conf: SparkConf)
 /**
  * An exception thrown if the RPC is aborted.
  */
-class RpcAbortException(message: String) extends Exception(message)
+private[spark] class RpcAbortException(message: String) extends Exception(message)
 
 /**
  * A wrapper for [[Future]] but add abort method.
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 1c5c36ea8eae2..a07823cff2481 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -823,6 +823,7 @@ object Unidoc {
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/internal")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/memory")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/network")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/rpc")))
       .map(_.filterNot(f =>
         f.getCanonicalPath.contains("org/apache/spark/shuffle") &&
         !f.getCanonicalPath.contains("org/apache/spark/shuffle/api")))
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java
index 37534be31b6f6..a06483d8fa08b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/ExternalCommandRunner.java
@@ -27,6 +27,8 @@
  * cores for Solr and so on.
  * <p>
  * This interface will be instantiated when end users call `SparkSession#executeCommand`.
+ *
+ * @since 3.0.0
  */
 @Unstable
 public interface ExternalCommandRunner {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
index 65e0b6be00ef3..61cb83c4f15ad 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
@@ -26,6 +26,8 @@
  * some custom logic and call the built-in session catalog at the end. For example, they can
  * implement {@code createTable}, do something else before calling {@code createTable} of the
  * built-in session catalog.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface CatalogExtension extends TableCatalog, SupportsNamespaces {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
index 34fdc5dd4d0fb..2958538c21926 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
@@ -38,6 +38,8 @@
  * {@code spark.sql.catalog.catalog-name.(key)=(value)} will be passed in the case insensitive
  * string map of options in initialization with the prefix removed.
  * {@code name}, is also passed and is the catalog's name; in this case, "catalog-name".
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface CatalogPlugin {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java
deleted file mode 100644
index b6a896c52d535..0000000000000
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Catalogs.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.connector.catalog;
-
-import org.apache.spark.SparkException;
-import org.apache.spark.annotation.Private;
-import org.apache.spark.sql.internal.SQLConf;
-import org.apache.spark.sql.util.CaseInsensitiveStringMap;
-import org.apache.spark.util.Utils;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static scala.collection.JavaConverters.mapAsJavaMapConverter;
-
-@Private
-public class Catalogs {
-  private Catalogs() {
-  }
-
-  /**
-   * Load and configure a catalog by name.
-   * <p>
-   * This loads, instantiates, and initializes the catalog plugin for each call; it does not cache
-   * or reuse instances.
-   *
-   * @param name a String catalog name
-   * @param conf a SQLConf
-   * @return an initialized CatalogPlugin
-   * @throws CatalogNotFoundException if the plugin class cannot be found
-   * @throws SparkException if the plugin class cannot be instantiated
-   */
-  public static CatalogPlugin load(String name, SQLConf conf)
-      throws CatalogNotFoundException, SparkException {
-    String pluginClassName;
-    try {
-      pluginClassName = conf.getConfString("spark.sql.catalog." + name);
-    } catch (NoSuchElementException e){
-      throw new CatalogNotFoundException(String.format(
-          "Catalog '%s' plugin class not found: spark.sql.catalog.%s is not defined", name, name));
-    }
-
-    ClassLoader loader = Utils.getContextOrSparkClassLoader();
-
-    try {
-      Class<?> pluginClass = loader.loadClass(pluginClassName);
-
-      if (!CatalogPlugin.class.isAssignableFrom(pluginClass)) {
-        throw new SparkException(String.format(
-            "Plugin class for catalog '%s' does not implement CatalogPlugin: %s",
-            name, pluginClassName));
-      }
-
-      CatalogPlugin plugin =
-        CatalogPlugin.class.cast(pluginClass.getDeclaredConstructor().newInstance());
-
-      plugin.initialize(name, catalogOptions(name, conf));
-
-      return plugin;
-
-    } catch (ClassNotFoundException e) {
-      throw new SparkException(String.format(
-          "Cannot find catalog plugin class for catalog '%s': %s", name, pluginClassName));
-
-    } catch (NoSuchMethodException e) {
-      throw new SparkException(String.format(
-          "Failed to find public no-arg constructor for catalog '%s': %s", name, pluginClassName),
-          e);
-
-    } catch (IllegalAccessException e) {
-      throw new SparkException(String.format(
-          "Failed to call public no-arg constructor for catalog '%s': %s", name, pluginClassName),
-          e);
-
-    } catch (InstantiationException e) {
-      throw new SparkException(String.format(
-          "Cannot instantiate abstract catalog plugin class for catalog '%s': %s", name,
-          pluginClassName), e.getCause());
-
-    } catch (InvocationTargetException e) {
-      throw new SparkException(String.format(
-          "Failed during instantiating constructor for catalog '%s': %s", name, pluginClassName),
-          e.getCause());
-    }
-  }
-
-  /**
-   * Extracts a named catalog's configuration from a SQLConf.
-   *
-   * @param name a catalog name
-   * @param conf a SQLConf
-   * @return a case insensitive string map of options starting with spark.sql.catalog.(name).
-   */
-  private static CaseInsensitiveStringMap catalogOptions(String name, SQLConf conf) {
-    Map<String, String> allConfs = mapAsJavaMapConverter(conf.getAllConfs()).asJava();
-    Pattern prefix = Pattern.compile("^spark\\.sql\\.catalog\\." + name + "\\.(.+)");
-
-    HashMap<String, String> options = new HashMap<>();
-    for (Map.Entry<String, String> entry : allConfs.entrySet()) {
-      Matcher matcher = prefix.matcher(entry.getKey());
-      if (matcher.matches() && matcher.groupCount() > 0) {
-        options.put(matcher.group(1), entry.getValue());
-      }
-    }
-
-    return new CaseInsensitiveStringMap(options);
-  }
-}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
index cf0eef96d5a9c..5a5195934413e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
@@ -33,6 +33,8 @@
  * by calling the built-in session catalog directly. This is created for convenience, so that users
  * only need to override some methods where they want to apply custom logic. For example, they can
  * override {@code createTable}, do something else before calling {@code super.createTable}.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public abstract class DelegatingCatalogExtension implements CatalogExtension {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
index c3ac9c20db3b0..7b3a081aecdc0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
@@ -21,6 +21,8 @@
 
 /**
  * Identifies an object in a catalog.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface Identifier {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
index 1db6877c8d0be..f4eee84fee968 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
@@ -30,6 +30,8 @@
  *       removeProperty("other_prop")
  *     )
  * </pre>
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface NamespaceChange {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java
index fc076ee00a623..116efc1ada16a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SessionConfigSupport.java
@@ -23,6 +23,8 @@
  * A mix-in interface for {@link TableProvider}. Data sources can implement this interface to
  * propagate session configs with the specified key-prefix to all data source operations in this
  * session.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SessionConfigSupport extends TableProvider {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
index 84b24f204bf9c..c328f83f9a0c2 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
@@ -36,6 +36,8 @@
  * write will be committed. The job concludes with a call to {@link #commitStagedChanges()}, at
  * which point implementations are expected to commit the table's metadata into the metastore along
  * with the data that was written by the writes from the write builder this table created.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface StagedTable extends Table {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
index 1c8e9c5024376..9d4f6eafd820e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
@@ -48,6 +48,8 @@
  * {@link BatchWrite#commit(WriterCommitMessage[])} is called,
  * {@link StagedTable#commitStagedChanges()} is called, at which point the staged table can
  * complete both the data write and the metadata swap operation atomically.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface StagingTableCatalog extends TableCatalog {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java
index 5225b12788c49..e779d5234cfb7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsCatalogOptions.java
@@ -26,6 +26,8 @@
  * sources use the `DataFrameWriter.save(path)` method, the option `path` can translate to a
  * PathIdentifier. A catalog can then use this PathIdentifier to check the existence of a table, or
  * whether a table can be created at a given directory.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsCatalogOptions extends TableProvider {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
index 80aa57ca18775..92461882fc75f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
@@ -23,6 +23,8 @@
 /**
  * A mix-in interface for {@link Table} delete support. Data sources can implement this
  * interface to provide the ability to delete data from tables that matches filter expressions.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface SupportsDelete {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
index 190f1a14d7129..bd7e49ef6f08a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
@@ -35,6 +35,8 @@
  * and uses Java packages as namespaces is not required to support the methods to create, alter, or
  * drop a namespace. Implementations are allowed to discover the existence of objects or namespaces
  * without throwing {@link NoSuchNamespaceException} when no namespace is found.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface SupportsNamespaces extends CatalogPlugin {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
index cdff1a38ff05f..a4cb66b2d11e2 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
@@ -26,6 +26,8 @@
  * A mix-in interface of {@link Table}, to indicate that it's readable. This adds
  * {@link #newScanBuilder(CaseInsensitiveStringMap)} that is used to create a scan for batch,
  * micro-batch, or continuous processing.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface SupportsRead extends Table {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
index 90d79ed492ef3..19499d1061a9e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
@@ -26,6 +26,8 @@
  * A mix-in interface of {@link Table}, to indicate that it's writable. This adds
  * {@link #newWriteBuilder(LogicalWriteInfo)} that is used to create a
  * write for batch or streaming.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface SupportsWrite extends Table {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java
index a493736c77277..8f7a87404837c 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Table.java
@@ -36,6 +36,8 @@
  * The default implementation of {@link #partitioning()} returns an empty array of partitions, and
  * the default implementation of {@link #properties()} returns an empty map. These should be
  * overridden by implementations that support partitioning and table properties.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Table {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
index 9765118a8dbf7..6a64bf8441933 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
@@ -25,6 +25,8 @@
  * Tables use {@link Table#capabilities()} to return a set of capabilities. Each capability signals
  * to Spark that the table supports a feature identified by the capability. For example, returning
  * {@link #BATCH_READ} allows Spark to read from the table using a batch scan.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public enum TableCapability {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
index 2f102348ec517..357cd4b84f47a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
@@ -34,6 +34,8 @@
  * {@link #alterTable(Identifier, TableChange...)} will be normalized to match the case used in the
  * table schema when updating, renaming, or dropping existing columns when catalyst analysis is case
  * insensitive.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface TableCatalog extends CatalogPlugin {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
index 5ce020912f213..81e988a62a058 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
@@ -36,6 +36,8 @@
  *       deleteColumn("c")
  *     )
  * </pre>
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface TableChange {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
index 732c5352a15ac..82731e2c8e1e8 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableProvider.java
@@ -34,6 +34,8 @@
  * <p>
  * The major responsibility of this interface is to return a {@link Table} for read/write.
  * </p>
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface TableProvider {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
index 807731ac22948..a5b8e2224e711 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
@@ -21,6 +21,8 @@
 
 /**
  * Base class of the public logical expression API.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface Expression {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
index a06ac9f77c5b5..28fb71e59c320 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
@@ -26,6 +26,8 @@
 
 /**
  * Helper methods to create logical transforms to pass into Spark.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public class Expressions {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
index 942744bf6a5d3..6474eb4ea4d01 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
@@ -27,6 +27,7 @@
  * the literal's {@link DataType SQL data type}.
  *
  * @param <T> the JVM type of a value held by the literal
+ * @since 3.0.0
  */
 @Experimental
 public interface Literal<T> extends Expression {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
index e6e5fa59c2499..6dd46cc063145 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
@@ -21,6 +21,8 @@
 
 /**
  * Represents a field or column reference in the public logical expression API.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface NamedReference extends Expression {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
index edac6df8a4d3e..ad78f005d9218 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
@@ -24,6 +24,8 @@
  * <p>
  * For example, the transform date(ts) is used to derive a date value from a timestamp column. The
  * transform name is "date" and its argument is a reference to the "ts" column.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public interface Transform extends Expression {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
index 09592598a82f9..73aefa55ae5e1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Batch.java
@@ -23,6 +23,8 @@
  * A physical representation of a data source scan for batch queries. This interface is used to
  * provide physical information, like how many partitions the scanned data has, and how to read
  * records from the partitions.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Batch {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
index bedf30803f73f..03eec00168012 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/InputPartition.java
@@ -31,6 +31,8 @@
  * {@link PartitionReaderFactory#createColumnarReader(InputPartition)} on executors to do
  * the actual reading. So {@link InputPartition} must be serializable while {@link PartitionReader}
  * doesn't need to be.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface InputPartition extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
index 254ee56501b9a..23fbd95800e26 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReader.java
@@ -31,6 +31,8 @@
  * for normal data sources, or {@link org.apache.spark.sql.vectorized.ColumnarBatch} for columnar
  * data sources(whose {@link PartitionReaderFactory#supportColumnarReads(InputPartition)}
  * returns true).
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PartitionReader<T> extends Closeable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
index 7fef69ce2a5b8..9dded247e89f3 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/PartitionReaderFactory.java
@@ -29,6 +29,8 @@
  * If Spark fails to execute any methods in the implementations of this interface or in the returned
  * {@link PartitionReader} (by throwing an exception), corresponding Spark task would fail and
  * get retried until hitting the maximum retry times.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PartitionReaderFactory extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
index c1584a58c117f..4146f217985b4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Scan.java
@@ -34,6 +34,8 @@
  * implemented, if the {@link Table} that creates this {@link Scan} returns
  * {@link TableCapability#BATCH_READ} support in its {@link Table#capabilities()}.
  * </p>
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Scan {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java
index ca2cd59b2e442..cb3eea7680058 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/ScanBuilder.java
@@ -23,6 +23,8 @@
  * An interface for building the {@link Scan}. Implementations can mixin SupportsPushDownXYZ
  * interfaces to do operator pushdown, and keep the operator pushdown result in the returned
  * {@link Scan}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface ScanBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java
index 42778b4e28a1a..a4ef785da2c65 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/Statistics.java
@@ -24,6 +24,8 @@
 /**
  * An interface to represent statistics for a data source, which is returned by
  * {@link SupportsReportStatistics#estimateStatistics()}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Statistics {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
index bee9e5508ca66..6594af27734c7 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownFilters.java
@@ -23,6 +23,8 @@
 /**
  * A mix-in interface for {@link ScanBuilder}. Data sources can implement this interface to
  * push down filters to the data source and reduce the size of the data to be read.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsPushDownFilters extends ScanBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
index 97143686d3efc..4f7da3c2a38f4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsPushDownRequiredColumns.java
@@ -24,6 +24,8 @@
  * A mix-in interface for {@link ScanBuilder}. Data sources can implement this
  * interface to push down required columns to the data source and only read these columns during
  * scan to reduce the size of the data to be read.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsPushDownRequiredColumns extends ScanBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
index 5bbc191730391..e7a27e07498fb 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportPartitioning.java
@@ -26,6 +26,8 @@
  *
  * Note that, when a {@link Scan} implementation creates exactly one {@link InputPartition},
  * Spark may avoid adding a shuffle even if the reader does not implement this interface.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsReportPartitioning extends Scan {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
index 3cd5ef60dc533..b839fd5a4a726 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
@@ -26,6 +26,8 @@
  * As of Spark 2.4, statistics are reported to the optimizer before any operator is pushed to the
  * data source. Implementations that return more accurate statistics based on pushed operators will
  * not improve query performance until the planner can push operators before getting stats.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface SupportsReportStatistics extends Scan {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java
index 23d3c6b563d1c..ed0354484d7be 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/ClusteredDistribution.java
@@ -24,6 +24,8 @@
  * A concrete implementation of {@link Distribution}. Represents a distribution where records that
  * share the same values for the {@link #clusteredColumns} will be produced by the same
  * {@link PartitionReader}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public class ClusteredDistribution implements Distribution {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
index fd397c760f05a..264b268e247e1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Distribution.java
@@ -36,6 +36,8 @@
  * <ul>
  *   <li>{@link ClusteredDistribution}</li>
  * </ul>
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Distribution {}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
index f0b9410ee6e61..c2c474696f667 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/partitioning/Partitioning.java
@@ -26,6 +26,8 @@
  * {@link SupportsReportPartitioning#outputPartitioning()}. Note that this should work
  * like a snapshot. Once created, it should be deterministic and always report the same number of
  * partitions and the same "satisfy" result for a certain distribution.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface Partitioning {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
index 3e8b14172d6b2..c4b073458eac8 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/BatchWrite.java
@@ -36,6 +36,8 @@
  * do it manually in their Spark applications if they want to retry.
  *
  * Please refer to the documentation of commit/abort methods for detailed specifications.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface BatchWrite {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
index 59c69a18292d8..0022a71491738 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriter.java
@@ -56,6 +56,8 @@
  * Spark only accepts the commit message that arrives first and ignore others.
  *
  * Note that, Currently the type `T` can only be {@link org.apache.spark.sql.catalyst.InternalRow}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface DataWriter<T> extends Closeable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
index 310575df05d97..33b7ff3dd27b1 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/DataWriterFactory.java
@@ -31,6 +31,8 @@
  * Note that, the writer factory will be serialized and sent to executors, then the data writer
  * will be created on executors and do the actual writing. So this interface must be
  * serializable and {@link DataWriter} doesn't need to be.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface DataWriterFactory extends Serializable {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
index 831f4e5aac481..0638970df925e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/LogicalWriteInfo.java
@@ -24,6 +24,8 @@
 /**
  * This interface contains logical write information that data sources can use when generating a
  * {@link WriteBuilder}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface LogicalWriteInfo {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java
index 55a092e39970e..248a6e69ac11f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/PhysicalWriteInfo.java
@@ -23,6 +23,8 @@
 /**
  * This interface contains physical write information that data sources can use when
  * generating a {@link DataWriterFactory} or a {@link StreamingDataWriterFactory}.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface PhysicalWriteInfo {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
index 90e668d1935b1..bda0eed93c98a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
@@ -26,6 +26,8 @@
  * <p>
  * This is provided to implement SQL compatible with Hive table operations but is not recommended.
  * Instead, use the {@link SupportsOverwrite overwrite by filter API} to explicitly replace data.
+ *
+ * @since 3.0.0
  */
 public interface SupportsDynamicOverwrite extends WriteBuilder {
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
index 6063a155ee209..ffa106c0b72f0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
@@ -25,6 +25,8 @@
  * <p>
  * Overwriting data by filter will delete any data that matches the filter and replace it with data
  * that is committed in the write.
+ *
+ * @since 3.0.0
  */
 public interface SupportsOverwrite extends WriteBuilder, SupportsTruncate {
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
index 15a68375032d2..b89d409abea6f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
@@ -21,6 +21,8 @@
  * Write builder trait for tables that support truncation.
  * <p>
  * Truncation removes all data in a table and replaces it with data that is committed in the write.
+ *
+ * @since 3.0.0
  */
 public interface SupportsTruncate extends WriteBuilder {
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
index a8d99a8f04b1b..5398ca46e9777 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriteBuilder.java
@@ -28,6 +28,8 @@
  *
  * Unless modified by a mixin interface, the {@link BatchWrite} configured by this builder is to
  * append data without affecting existing data.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface WriteBuilder {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
index 823f955bffb53..f340d20c3549f 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/WriterCommitMessage.java
@@ -29,6 +29,8 @@
  *
  * This is an empty interface, data sources should define their own message class and use it when
  * generating messages at executor side and handling the messages at driver side.
+ *
+ * @since 3.0.0
  */
 @Evolving
 public interface WriterCommitMessage extends Serializable {}
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java b/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java
index c344a62be40c1..de7d98ba48c31 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/util/CaseInsensitiveStringMap.java
@@ -36,6 +36,8 @@
  * <p>
  * Methods that return keys in this map, like {@link #entrySet()} and {@link #keySet()}, return
  * keys converted to lower case. This map doesn't allow null key.
+ *
+ * @since 3.0.0
  */
 @Experimental
 public class CaseInsensitiveStringMap implements Map<String, String> {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
index 56f170cd77073..72a8f7e99729b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
@@ -162,7 +162,8 @@ case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInp
     Examples:
       > SELECT _FUNC_(0);
        0
-  """)
+  """,
+  since = "3.0.0")
 case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsInputTypes {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(IntegralType, BooleanType))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index 52429a63b306d..d8eed27df79df 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -601,7 +601,8 @@ object Murmur3HashFunction extends InterpretedHashFunction {
     Examples:
       > SELECT _FUNC_('Spark', array(123), 2);
        5602566077635097486
-  """)
+  """,
+  since = "3.0.0")
 case class XxHash64(children: Seq[Expression], seed: Long) extends HashExpression[Long] {
   def this(arguments: Seq[Expression]) = this(arguments, 42L)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
new file mode 100644
index 0000000000000..e75429c58cc7b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DataTypeJsonUtils.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import com.fasterxml.jackson.core.{JsonGenerator, JsonParser}
+import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer, JsonSerializer, SerializerProvider}
+import org.json4s.JsonAST.JValue
+import org.json4s.jackson.{JValueDeserializer, JValueSerializer}
+
+import org.apache.spark.sql.types.DataType
+
+object DataTypeJsonUtils {
+  /**
+   * Jackson serializer for [[DataType]]. Internally this delegates to json4s based serialization.
+   */
+  class DataTypeJsonSerializer extends JsonSerializer[DataType] {
+    private val delegate = new JValueSerializer
+    override def serialize(
+      value: DataType,
+      gen: JsonGenerator,
+      provider: SerializerProvider): Unit = {
+      delegate.serialize(value.jsonValue, gen, provider)
+    }
+  }
+
+  /**
+   * Jackson deserializer for [[DataType]]. Internally this delegates to json4s based
+   * deserialization.
+   */
+  class DataTypeJsonDeserializer extends JsonDeserializer[DataType] {
+    private val delegate = new JValueDeserializer(classOf[Any])
+
+    override def deserialize(
+      jsonParser: JsonParser,
+      deserializationContext: DeserializationContext): DataType = {
+      val json = delegate.deserialize(jsonParser, deserializationContext)
+      DataType.parseDataType(json.asInstanceOf[JValue])
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
new file mode 100644
index 0000000000000..cc4395e7bd62d
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/Catalogs.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.connector.catalog
+
+import java.lang.reflect.InvocationTargetException
+import java.util
+import java.util.NoSuchElementException
+import java.util.regex.Pattern
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+import org.apache.spark.util.Utils
+
+private[sql] object Catalogs {
+  /**
+   * Load and configure a catalog by name.
+   * <p>
+   * This loads, instantiates, and initializes the catalog plugin for each call; it does not cache
+   * or reuse instances.
+   *
+   * @param name a String catalog name
+   * @param conf a SQLConf
+   * @return an initialized CatalogPlugin
+   * @throws CatalogNotFoundException if the plugin class cannot be found
+   * @throws org.apache.spark.SparkException           if the plugin class cannot be instantiated
+   */
+  @throws[CatalogNotFoundException]
+  @throws[SparkException]
+  def load(name: String, conf: SQLConf): CatalogPlugin = {
+    val pluginClassName = try {
+      conf.getConfString("spark.sql.catalog." + name)
+    } catch {
+      case _: NoSuchElementException =>
+        throw new CatalogNotFoundException(
+          s"Catalog '$name' plugin class not found: spark.sql.catalog.$name is not defined")
+    }
+    val loader = Utils.getContextOrSparkClassLoader
+    try {
+      val pluginClass = loader.loadClass(pluginClassName)
+      if (!classOf[CatalogPlugin].isAssignableFrom(pluginClass)) {
+        throw new SparkException(
+          s"Plugin class for catalog '$name' does not implement CatalogPlugin: $pluginClassName")
+      }
+      val plugin = pluginClass.getDeclaredConstructor().newInstance().asInstanceOf[CatalogPlugin]
+      plugin.initialize(name, catalogOptions(name, conf))
+      plugin
+    } catch {
+      case _: ClassNotFoundException =>
+        throw new SparkException(
+          s"Cannot find catalog plugin class for catalog '$name': $pluginClassName")
+      case e: NoSuchMethodException =>
+        throw new SparkException(
+          s"Failed to find public no-arg constructor for catalog '$name': $pluginClassName)", e)
+      case e: IllegalAccessException =>
+        throw new SparkException(
+          s"Failed to call public no-arg constructor for catalog '$name': $pluginClassName)", e)
+      case e: InstantiationException =>
+        throw new SparkException("Cannot instantiate abstract catalog plugin class for " +
+          s"catalog '$name': $pluginClassName", e.getCause)
+      case e: InvocationTargetException =>
+        throw new SparkException("Failed during instantiating constructor for catalog " +
+          s"'$name': $pluginClassName", e.getCause)
+    }
+  }
+
+  /**
+   * Extracts a named catalog's configuration from a SQLConf.
+   *
+   * @param name a catalog name
+   * @param conf a SQLConf
+   * @return a case insensitive string map of options starting with spark.sql.catalog.(name).
+   */
+  private def catalogOptions(name: String, conf: SQLConf) = {
+    val prefix = Pattern.compile("^spark\\.sql\\.catalog\\." + name + "\\.(.+)")
+    val options = new util.HashMap[String, String]
+    conf.getAllConfs.foreach {
+      case (key, value) =>
+        val matcher = prefix.matcher(key)
+        if (matcher.matches && matcher.groupCount > 0) options.put(matcher.group(1), value)
+    }
+    new CaseInsensitiveStringMap(options)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
index 84adddf2671f5..ea26847ac3cef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/expressions/expressions.scala
@@ -64,7 +64,7 @@ private[sql] object LogicalExpressions {
 /**
  * Allows Spark to rewrite the given references of the transform during analysis.
  */
-sealed trait RewritableTransform extends Transform {
+private[sql] sealed trait RewritableTransform extends Transform {
   /** Creates a copy of this transform with the new analyzed references. */
   def withReferences(newReferences: Seq[NamedReference]): Transform
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
index fc7d33e823a99..020dd79f8f0d7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
@@ -221,6 +221,8 @@ case class StringContains(attribute: String, value: String) extends Filter {
 
 /**
  * A filter that always evaluates to `true`.
+ *
+ * @since 3.0.0
  */
 @Evolving
 case class AlwaysTrue() extends Filter {
@@ -233,6 +235,8 @@ object AlwaysTrue extends AlwaysTrue {
 
 /**
  * A filter that always evaluates to `false`.
+ *
+ * @since 3.0.0
  */
 @Evolving
 case class AlwaysFalse() extends Filter {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 8a8cea194bf2c..7449a28e069d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -21,20 +21,17 @@ import java.util.Locale
 
 import scala.util.control.NonFatal
 
-import com.fasterxml.jackson.core.{JsonGenerator, JsonParser}
-import com.fasterxml.jackson.databind.{DeserializationContext, JsonDeserializer, JsonSerializer, SerializerProvider}
-import com.fasterxml.jackson.databind.`type`.TypeFactory
 import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonSerialize}
 import org.json4s._
 import org.json4s.JsonAST.JValue
 import org.json4s.JsonDSL._
-import org.json4s.jackson.{JValueDeserializer, JValueSerializer}
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.annotation.Stable
 import org.apache.spark.sql.catalyst.analysis.Resolver
 import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.catalyst.util.DataTypeJsonUtils.{DataTypeJsonDeserializer, DataTypeJsonSerializer}
 import org.apache.spark.sql.catalyst.util.StringUtils.StringConcat
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
@@ -485,30 +482,3 @@ object DataType {
     }
   }
 }
-
-/**
- * Jackson serializer for [[DataType]]. Internally this delegates to json4s based serialization.
- */
-class DataTypeJsonSerializer extends JsonSerializer[DataType] {
-  private val delegate = new JValueSerializer
-  override def serialize(
-      value: DataType,
-      gen: JsonGenerator,
-      provider: SerializerProvider): Unit = {
-    delegate.serialize(value.jsonValue, gen, provider)
-  }
-}
-
-/**
- * Jackson deserializer for [[DataType]]. Internally this delegates to json4s based deserialization.
- */
-class DataTypeJsonDeserializer extends JsonDeserializer[DataType] {
-  private val delegate = new JValueDeserializer(classOf[Any])
-
-  override def deserialize(
-      jsonParser: JsonParser,
-      deserializationContext: DeserializationContext): DataType = {
-    val json = delegate.deserialize(jsonParser, deserializationContext)
-    DataType.parseDataType(json.asInstanceOf[JValue])
-  }
-}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
index b5226213effc4..3956629cf6a57 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
@@ -22,7 +22,7 @@ import scala.math.Ordering
 
 import org.apache.spark.sql.types.Decimal.DecimalIsConflicted
 
-object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
+private[sql] object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
   private def checkOverflow(res: Int, x: Byte, y: Byte, op: String): Unit = {
     if (res > Byte.MaxValue || res < Byte.MinValue) {
       throw new ArithmeticException(s"$x $op $y caused overflow.")
@@ -56,7 +56,7 @@ object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
 }
 
 
-object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
+private[sql] object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
   private def checkOverflow(res: Int, x: Short, y: Short, op: String): Unit = {
     if (res > Short.MaxValue || res < Short.MinValue) {
       throw new ArithmeticException(s"$x $op $y caused overflow.")
@@ -90,7 +90,7 @@ object ShortExactNumeric extends ShortIsIntegral with Ordering.ShortOrdering {
 }
 
 
-object IntegerExactNumeric extends IntIsIntegral with Ordering.IntOrdering {
+private[sql] object IntegerExactNumeric extends IntIsIntegral with Ordering.IntOrdering {
   override def plus(x: Int, y: Int): Int = Math.addExact(x, y)
 
   override def minus(x: Int, y: Int): Int = Math.subtractExact(x, y)
@@ -100,7 +100,7 @@ object IntegerExactNumeric extends IntIsIntegral with Ordering.IntOrdering {
   override def negate(x: Int): Int = Math.negateExact(x)
 }
 
-object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering {
+private[sql] object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering {
   override def plus(x: Long, y: Long): Long = Math.addExact(x, y)
 
   override def minus(x: Long, y: Long): Long = Math.subtractExact(x, y)
@@ -117,7 +117,7 @@ object LongExactNumeric extends LongIsIntegral with Ordering.LongOrdering {
     }
 }
 
-object FloatExactNumeric extends FloatIsFractional {
+private[sql] object FloatExactNumeric extends FloatIsFractional {
   private def overflowException(x: Float, dataType: String) =
     throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
 
@@ -151,7 +151,7 @@ object FloatExactNumeric extends FloatIsFractional {
   override def compare(x: Float, y: Float): Int = java.lang.Float.compare(x, y)
 }
 
-object DoubleExactNumeric extends DoubleIsFractional {
+private[sql] object DoubleExactNumeric extends DoubleIsFractional {
   private def overflowException(x: Double, dataType: String) =
     throw new ArithmeticException(s"Casting $x to $dataType causes overflow")
 
@@ -179,7 +179,7 @@ object DoubleExactNumeric extends DoubleIsFractional {
   override def compare(x: Double, y: Double): Int = java.lang.Double.compare(x, y)
 }
 
-object DecimalExactNumeric extends DecimalIsConflicted {
+private[sql] object DecimalExactNumeric extends DecimalIsConflicted {
   override def toInt(x: Decimal): Int = x.roundToInt()
 
   override def toLong(x: Decimal): Long = x.roundToLong()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
index cf6bde5a2bcb9..45a9b28154028 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
@@ -246,6 +246,7 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
 /**
  * Configuration methods common to create/replace operations and insert/overwrite operations.
  * @tparam R builder type to return
+ * @since 3.0.0
  */
 trait WriteConfigMethods[R] {
   /**
@@ -293,6 +294,8 @@ trait WriteConfigMethods[R] {
 
 /**
  * Trait to restrict calls to create and replace operations.
+ *
+ * @since 3.0.0
  */
 trait CreateTableWriter[T] extends WriteConfigMethods[CreateTableWriter[T]] {
   /**

From 36c59a473719fa7bb6a61048b50b717ff8f43c5f Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Fri, 21 Feb 2020 22:49:20 +0800
Subject: [PATCH 0136/1280] [SPARK-30906][SQL] Turning off AQE in CacheManager
 is not thread-safe

### What changes were proposed in this pull request?
This PR aims to fix the thread-safety issue in turning off AQE for CacheManager by cloning the current session and changing the AQE conf on the cloned session.
This PR also adds a utility function for cloning the session with AQE disabled conf value, which can be shared by another caller.

### Why are the changes needed?
To fix the potential thread-unsafe problem.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Manually tested CachedTableSuite with AQE settings enabled.

Closes #27659 from maryannxue/spark-30906.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6058ce97b989606ffedddf32778ce38f322cfb9e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/CacheManager.scala    | 28 +++++++++----------
 .../spark/sql/execution/QueryExecution.scala  | 27 +++++++++++-------
 .../adaptive/AdaptiveSparkPlanExec.scala      |  1 -
 3 files changed, 31 insertions(+), 25 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index 413bd7b29cf45..ad33ce5ef60ff 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -79,20 +79,16 @@ class CacheManager extends Logging {
     if (lookupCachedData(planToCache).nonEmpty) {
       logWarning("Asked to cache already cached data.")
     } else {
-      val sparkSession = query.sparkSession
-      val qe = sparkSession.sessionState.executePlan(planToCache)
-      val originalValue = sparkSession.sessionState.conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)
-      val inMemoryRelation = try {
-        // Avoiding changing the output partitioning, here disable AQE.
-        sparkSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
+      val sessionWithAqeOff = QueryExecution.getOrCloneSessionWithAqeOff(query.sparkSession)
+      val inMemoryRelation = sessionWithAqeOff.withActive {
+        val qe = sessionWithAqeOff.sessionState.executePlan(planToCache)
         InMemoryRelation(
-          sparkSession.sessionState.conf.useCompression,
-          sparkSession.sessionState.conf.columnBatchSize, storageLevel,
+          sessionWithAqeOff.sessionState.conf.useCompression,
+          sessionWithAqeOff.sessionState.conf.columnBatchSize, storageLevel,
           qe.executedPlan,
           tableName,
           optimizedPlan = qe.optimizedPlan)
-      } finally {
-        sparkSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, originalValue)
       }
 
       this.synchronized {
@@ -194,10 +190,14 @@ class CacheManager extends Logging {
     }
     needToRecache.map { cd =>
       cd.cachedRepresentation.cacheBuilder.clearCache()
-      val qe = spark.sessionState.executePlan(cd.plan)
-      val newCache = InMemoryRelation(
-        cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = qe.executedPlan),
-        optimizedPlan = qe.optimizedPlan)
+      // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
+      val sessionWithAqeOff = QueryExecution.getOrCloneSessionWithAqeOff(spark)
+      val newCache = sessionWithAqeOff.withActive {
+        val qe = sessionWithAqeOff.sessionState.executePlan(cd.plan)
+        InMemoryRelation(
+          cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = qe.executedPlan),
+          optimizedPlan = qe.optimizedPlan)
+      }
       val recomputedPlan = cd.copy(cachedRepresentation = newCache)
       this.synchronized {
         if (lookupCachedData(recomputedPlan.plan).nonEmpty) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 4571e69b302f6..20bc289c65586 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -276,14 +276,7 @@ object QueryExecution {
    */
   private[execution] def preparations(sparkSession: SparkSession): Seq[Rule[SparkPlan]] = {
 
-    val sparkSessionWithAdaptiveExecutionOff =
-    if (sparkSession.sessionState.conf.adaptiveExecutionEnabled) {
-      val session = sparkSession.cloneSession()
-      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
-      session
-    } else {
-      sparkSession
-    }
+    val sparkSessionWithAqeOff = getOrCloneSessionWithAqeOff(sparkSession)
 
     Seq(
       // `AdaptiveSparkPlanExec` is a leaf node. If inserted, all the following rules will be no-op
@@ -291,8 +284,8 @@ object QueryExecution {
       InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession)),
       // If the following rules apply, it means the main query is not AQE-ed, so we make sure the
       // subqueries are not AQE-ed either.
-      PlanDynamicPruningFilters(sparkSessionWithAdaptiveExecutionOff),
-      PlanSubqueries(sparkSessionWithAdaptiveExecutionOff),
+      PlanDynamicPruningFilters(sparkSessionWithAqeOff),
+      PlanSubqueries(sparkSessionWithAqeOff),
       EnsureRequirements(sparkSession.sessionState.conf),
       ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
         sparkSession.sessionState.columnarRules),
@@ -341,4 +334,18 @@ object QueryExecution {
     val sparkPlan = createSparkPlan(spark, spark.sessionState.planner, plan.clone())
     prepareExecutedPlan(spark, sparkPlan)
   }
+
+  /**
+   * Returns a cloned [[SparkSession]] with adaptive execution disabled, or the original
+   * [[SparkSession]] if its adaptive execution is already disabled.
+   */
+  def getOrCloneSessionWithAqeOff[T](session: SparkSession): SparkSession = {
+    if (!session.sessionState.conf.adaptiveExecutionEnabled) {
+      session
+    } else {
+      val newSession = session.cloneSession()
+      newSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      newSession
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 3f20b59361988..4036424bbacaa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -467,7 +467,6 @@ case class AdaptiveSparkPlanExec(
   private def reOptimize(logicalPlan: LogicalPlan): (SparkPlan, LogicalPlan) = {
     logicalPlan.invalidateStatsCache()
     val optimized = optimizer.execute(logicalPlan)
-    SparkSession.setActiveSession(context.session)
     val sparkPlan = context.session.sessionState.planner.plan(ReturnAnswer(optimized)).next()
     val newPlan = applyPhysicalRules(sparkPlan, preprocessingRules ++ queryStagePreparationRules)
     (newPlan, optimized)

From 27321a9e76de3b12d94b20a5d31217a0a6d408e9 Mon Sep 17 00:00:00 2001
From: Eric Wu <492960551@qq.com>
Date: Fri, 21 Feb 2020 23:36:14 +0800
Subject: [PATCH 0137/1280] [SPARK-30764][SQL] Improve the readability of
 EXPLAIN FORMATTED style
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
The style of `EXPLAIN FORMATTED` output needs to be improved. We’ve already got some observations/ideas in
https://github.com/apache/spark/pull/27368#discussion_r376694496
https://github.com/apache/spark/pull/27368#discussion_r376927143

Observations/Ideas:
1. Using comma as the separator is not clear, especially commas are used inside the expressions too.
2. Show the column counts first? For example, `Results [4]: …`
3. Currently the attribute names are automatically generated, this need to refined.
4. Add arguments field in common implementations as `EXPLAIN EXTENDED` did by calling `argString` in `TreeNode.simpleString`. This will eliminate most existing minor differences between
`EXPLAIN EXTENDED` and `EXPLAIN FORMATTED`.
5. Another improvement we can do is: the generated alias shouldn't include attribute id. collect_set(val, 0, 0)#123 looks clearer than collect_set(val#456, 0, 0)#123

This PR is currently addressing comments 2 & 4, and open for more discussions on improving readability.

### Why are the changes needed?
The readability of `EXPLAIN FORMATTED` need to be improved, which will help user better understand the query plan.

### Does this PR introduce any user-facing change?
Yes, `EXPLAIN FORMATTED` output style changed.

### How was this patch tested?
Update expect results of test cases in explain.sql

Closes #27509 from Eric5553/ExplainFormattedRefine.

Authored-by: Eric Wu <492960551@qq.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1f0300fb16ad8b2670a6194b5ba2d5f4d275e987)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 python/pyspark/sql/dataframe.py               |   3 +-
 .../spark/sql/catalyst/plans/QueryPlan.scala  |  17 +-
 .../sql/execution/DataSourceScanExec.scala    |   4 +-
 .../spark/sql/execution/ExplainUtils.scala    |  14 +-
 .../spark/sql/execution/SparkPlan.scala       |  65 +-
 .../aggregate/BaseAggregateExec.scala         |  15 +-
 .../execution/basicPhysicalOperators.scala    |   6 +-
 .../sql/execution/exchange/Exchange.scala     |   2 +-
 .../joins/CartesianProductExec.scala          |   2 +-
 .../spark/sql/execution/joins/HashJoin.scala  |   6 +-
 .../execution/joins/SortMergeJoinExec.scala   |   6 +-
 .../sql-tests/results/explain.sql.out         | 577 +++++++++---------
 12 files changed, 397 insertions(+), 320 deletions(-)

diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 2432b8127840b..44cb264714482 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -284,7 +284,8 @@ def explain(self, extended=None, mode=None):
         == Physical Plan ==
         * Scan ExistingRDD (1)
         (1) Scan ExistingRDD [codegen id : 1]
-        Output: [age#0, name#1]
+        Output [2]: [age#0, name#1]
+        ...
 
         .. versionchanged:: 3.0.0
            Added optional argument `mode` to specify the expected output format of plans.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index c5abb6378ff7c..12482667efa0d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.catalyst.plans
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode, TreeNodeTag}
-import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, StructType}
 
@@ -189,9 +188,19 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
     val codegenIdStr =
       getTagValue(QueryPlan.CODEGEN_ID_TAG).map(id => s"[codegen id : $id]").getOrElse("")
     val operatorId = getTagValue(QueryPlan.OP_ID_TAG).map(id => s"$id").getOrElse("unknown")
-    s"""
-       |($operatorId) $nodeName $codegenIdStr
-     """.stripMargin
+    val baseStr = s"($operatorId) $nodeName $codegenIdStr"
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$baseStr
+         |Arguments: $argumentString
+      """.stripMargin
+    } else {
+      s"""
+         |$baseStr
+      """.stripMargin
+    }
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 0d759085a7e2c..c1f7f0a195491 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -76,7 +76,7 @@ trait DataSourceScanExec extends LeafExecNode {
 
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output: ${producedAttributes.mkString("[", ", ", "]")}
+       |${ExplainUtils.generateFieldString("Output", producedAttributes)}
        |${metadataStr.mkString("\n")}
      """.stripMargin
   }
@@ -377,7 +377,7 @@ case class FileSourceScanExec(
 
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output: ${producedAttributes.mkString("[", ", ", "]")}
+       |${ExplainUtils.generateFieldString("Output", producedAttributes)}
        |${metadataStr.mkString("\n")}
      """.stripMargin
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index d4fe272f8c95f..5d4309357895b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -23,7 +23,6 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.{Expression, PlanExpression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
-import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 
 object ExplainUtils {
   /**
@@ -171,7 +170,7 @@ object ExplainUtils {
     var currentCodegenId = -1
     plan.foreach {
       case p: WholeStageCodegenExec => currentCodegenId = p.codegenStageId
-      case p: InputAdapter => currentCodegenId = -1
+      case _: InputAdapter => currentCodegenId = -1
       case other: QueryPlan[_] =>
         if (currentCodegenId != -1) {
           other.setTagValue(QueryPlan.CODEGEN_ID_TAG, currentCodegenId)
@@ -182,6 +181,17 @@ object ExplainUtils {
     }
   }
 
+  /**
+   * Generate detailed field string with different format based on type of input value
+   */
+  def generateFieldString(fieldName: String, values: Any): String = values match {
+    case iter: Iterable[_] if (iter.size == 0) => s"${fieldName}: []"
+    case iter: Iterable[_] => s"${fieldName} [${iter.size}]: ${iter.mkString("[", ", ", "]")}"
+    case str: String if (str == null || str.isEmpty) => s"${fieldName}: None"
+    case str: String => s"${fieldName}: ${str}"
+    case _ => throw new IllegalArgumentException(s"Unsupported type for argument values: $values")
+  }
+
   /**
    * Given a input plan, returns an array of tuples comprising of :
    *  1. Hosting opeator id.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index 3301e9b5ab180..f5bb554682eab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.vectorized.ColumnarBatch
 
 object SparkPlan {
@@ -512,10 +513,22 @@ trait LeafExecNode extends SparkPlan {
   override final def children: Seq[SparkPlan] = Nil
   override def producedAttributes: AttributeSet = outputSet
   override def verboseStringWithOperatorId(): String = {
-    s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output: ${producedAttributes.mkString("[", ", ", "]")}
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
+    val outputStr = s"${ExplainUtils.generateFieldString("Output", producedAttributes)}"
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$baseStr
+         |$outputStr
+         |Arguments: $argumentString
+      """.stripMargin
+    } else {
+      s"""
+         |$baseStr
+         |$outputStr
+      """.stripMargin
+    }
   }
 }
 
@@ -531,10 +544,22 @@ trait UnaryExecNode extends SparkPlan {
 
   override final def children: Seq[SparkPlan] = child :: Nil
   override def verboseStringWithOperatorId(): String = {
-    s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Input: ${child.output.mkString("[", ", ", "]")}
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
+    val inputStr = s"${ExplainUtils.generateFieldString("Input", child.output)}"
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$baseStr
+         |$inputStr
+         |Arguments: $argumentString
+      """.stripMargin
+    } else {
+      s"""
+         |$baseStr
+         |$inputStr
+      """.stripMargin
+    }
   }
 }
 
@@ -544,10 +569,24 @@ trait BinaryExecNode extends SparkPlan {
 
   override final def children: Seq[SparkPlan] = Seq(left, right)
   override def verboseStringWithOperatorId(): String = {
-    s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Left output: ${left.output.mkString("[", ", ", "]")}
-       |Right output: ${right.output.mkString("[", ", ", "]")}
-     """.stripMargin
+    val argumentString = argString(SQLConf.get.maxToStringFields)
+    val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
+    val leftOutputStr = s"${ExplainUtils.generateFieldString("Left output", left.output)}"
+    val rightOutputStr = s"${ExplainUtils.generateFieldString("Right output", right.output)}"
+
+    if (argumentString.nonEmpty) {
+      s"""
+         |$baseStr
+         |$leftOutputStr
+         |$rightOutputStr
+         |Arguments: $argumentString
+      """.stripMargin
+    } else {
+      s"""
+         |$baseStr
+         |$leftOutputStr
+         |$rightOutputStr
+      """.stripMargin
+    }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
index 0eaa0f53fdacd..19d7263feb2d1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
@@ -31,18 +31,13 @@ trait BaseAggregateExec extends UnaryExecNode {
   def resultExpressions: Seq[NamedExpression]
 
   override def verboseStringWithOperatorId(): String = {
-    val inputString = child.output.mkString("[", ", ", "]")
-    val keyString = groupingExpressions.mkString("[", ", ", "]")
-    val functionString = aggregateExpressions.mkString("[", ", ", "]")
-    val aggregateAttributeString = aggregateAttributes.mkString("[", ", ", "]")
-    val resultString = resultExpressions.mkString("[", ", ", "]")
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Input: $inputString
-       |Keys: $keyString
-       |Functions: $functionString
-       |Aggregate Attributes: $aggregateAttributeString
-       |Results: $resultString
+       |${ExplainUtils.generateFieldString("Input", child.output)}
+       |${ExplainUtils.generateFieldString("Keys", groupingExpressions)}
+       |${ExplainUtils.generateFieldString("Functions", aggregateExpressions)}
+       |${ExplainUtils.generateFieldString("Aggregate Attributes", aggregateAttributes)}
+       |${ExplainUtils.generateFieldString("Results", resultExpressions)}
      """.stripMargin
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index 4e65bbf75282f..c3e259d196ba2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -87,8 +87,8 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
   override def verboseStringWithOperatorId(): String = {
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Output    : ${projectList.mkString("[", ", ", "]")}
-       |Input     : ${child.output.mkString("[", ", ", "]")}
+       |${ExplainUtils.generateFieldString("Output", projectList)}
+       |${ExplainUtils.generateFieldString("Input", child.output)}
      """.stripMargin
   }
 }
@@ -244,7 +244,7 @@ case class FilterExec(condition: Expression, child: SparkPlan)
   override def verboseStringWithOperatorId(): String = {
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Input     : ${child.output.mkString("[", ", ", "]")}
+       |${ExplainUtils.generateFieldString("Input", child.output)}
        |Condition : ${condition}
      """.stripMargin
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
index 849ff384c130a..dda9a637194fa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
@@ -92,7 +92,7 @@ case class ReusedExchangeExec(override val output: Seq[Attribute], child: Exchan
     val reuse_op_str = ExplainUtils.getOpId(child)
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${cdgen} [Reuses operator id: $reuse_op_str]
-       |Output : ${output}
+       |${ExplainUtils.generateFieldString("Output", output)}
      """.stripMargin
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
index 29645a736548c..7e2f487fdcc5d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
@@ -73,7 +73,7 @@ case class CartesianProductExec(
 
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Join condition: ${joinCondStr}
+       |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
      """.stripMargin
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 137f0b87a2f3d..99cf60273bf08 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -49,9 +49,9 @@ trait HashJoin {
 
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Left keys: ${leftKeys}
-       |Right keys: ${rightKeys}
-       |Join condition: ${joinCondStr}
+       |${ExplainUtils.generateFieldString("Left keys", leftKeys)}
+       |${ExplainUtils.generateFieldString("Right keys", rightKeys)}
+       |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
      """.stripMargin
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
index 62eea611556ff..7a08dd1afd3a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
@@ -63,9 +63,9 @@ case class SortMergeJoinExec(
     } else "None"
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |Left keys : ${leftKeys}
-       |Right keys: ${rightKeys}
-       |Join condition : ${joinCondStr}
+       |${ExplainUtils.generateFieldString("Left keys", leftKeys)}
+       |${ExplainUtils.generateFieldString("Right keys", rightKeys)}
+       |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
      """.stripMargin
   }
 
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index bc28d7f87bf00..40b26e6fb64ce 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -65,45 +65,48 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
      
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (5) HashAggregate [codegen id : 1]
-Input: [key#x, val#x]
-Keys: [key#x]
-Functions: [partial_max(val#x)]
-Aggregate Attributes: [max#x]
-Results: [key#x, max#x]
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
      
 (6) Exchange 
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
 (7) HashAggregate [codegen id : 2]
-Input: [key#x, max#x]
-Keys: [key#x]
-Functions: [max(val#x)]
-Aggregate Attributes: [max(val#x)#x]
-Results: [key#x, max(val#x)#x AS max(val)#x]
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
      
 (8) Exchange 
-Input: [key#x, max(val)#x]
-     
+Input [2]: [key#x, max(val)#x]
+Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
+      
 (9) Sort [codegen id : 3]
-Input: [key#x, max(val)#x]
+Input [2]: [key#x, max(val)#x]
+Arguments: [key#x ASC NULLS FIRST], true, 0
 
 
 -- !query
@@ -129,47 +132,48 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
      
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (5) HashAggregate [codegen id : 1]
-Input: [key#x, val#x]
-Keys: [key#x]
-Functions: [partial_max(val#x)]
-Aggregate Attributes: [max#x]
-Results: [key#x, max#x]
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
      
 (6) Exchange 
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
 (7) HashAggregate [codegen id : 2]
-Input: [key#x, max#x]
-Keys: [key#x]
-Functions: [max(val#x)]
-Aggregate Attributes: [max(val#x)#x]
-Results: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
      
 (8) Filter [codegen id : 2]
-Input     : [key#x, max(val)#x, max(val#x)#x]
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
 Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
      
 (9) Project [codegen id : 2]
-Output    : [key#x, max(val)#x]
-Input     : [key#x, max(val)#x, max(val#x)#x]
+Output [2]: [key#x, max(val)#x]
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
 
 
 -- !query
@@ -196,59 +200,60 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
      
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (5) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
      
 (6) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (7) Filter [codegen id : 2]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
      
 (8) Project [codegen id : 2]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (9) Union 
-     
+      
 (10) HashAggregate [codegen id : 3]
-Input: [key#x, val#x]
-Keys: [key#x, val#x]
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
-Results: [key#x, val#x]
+Results [2]: [key#x, val#x]
      
 (11) Exchange 
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
+      
 (12) HashAggregate [codegen id : 4]
-Input: [key#x, val#x]
-Keys: [key#x, val#x]
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
-Results: [key#x, val#x]
+Results [2]: [key#x, val#x]
 
 
 -- !query
@@ -274,47 +279,48 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 2]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
      
 (4) Project [codegen id : 2]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (5) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
      
 (6) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (7) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
      
 (8) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (9) BroadcastExchange 
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
 (10) BroadcastHashJoin [codegen id : 2]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -339,38 +345,39 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
      
 (4) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (5) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
      
 (6) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (7) BroadcastExchange 
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
 (8) BroadcastHashJoin [codegen id : 2]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -396,22 +403,22 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), IsNotNull(val), GreaterThan(val,3)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x > 3))
      
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 ===== Subqueries =====
 
@@ -426,39 +433,40 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
 
 
 (5) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key), IsNotNull(val), EqualTo(val,2)]
 ReadSchema: struct<key:int,val:int>
      
 (6) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (7) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x = 2))
      
 (8) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
      
 (9) HashAggregate [codegen id : 1]
-Input: [key#x]
+Input [1]: [key#x]
 Keys: []
-Functions: [partial_max(key#x)]
-Aggregate Attributes: [max#x]
-Results: [max#x]
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
      
 (10) Exchange 
-Input: [max#x]
-     
+Input [1]: [max#x]
+Arguments: SinglePartition, true, [id=#x]
+      
 (11) HashAggregate [codegen id : 2]
-Input: [max#x]
+Input [1]: [max#x]
 Keys: []
-Functions: [max(key#x)]
-Aggregate Attributes: [max(key#x)#x]
-Results: [max(key#x)#x AS max(key)#x]
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
      
 Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (18)
@@ -471,39 +479,40 @@ Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery
 
 
 (12) Scan parquet default.explain_temp3 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
      
 (13) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (14) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
      
 (15) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
      
 (16) HashAggregate [codegen id : 1]
-Input: [key#x]
+Input [1]: [key#x]
 Keys: []
-Functions: [partial_max(key#x)]
-Aggregate Attributes: [max#x]
-Results: [max#x]
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
      
 (17) Exchange 
-Input: [max#x]
-     
+Input [1]: [max#x]
+Arguments: SinglePartition, true, [id=#x]
+      
 (18) HashAggregate [codegen id : 2]
-Input: [max#x]
+Input [1]: [max#x]
 Keys: []
-Functions: [max(key#x)]
-Aggregate Attributes: [max(key#x)#x]
-Results: [max(key#x)#x AS max(key)#x]
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
 
 
 -- !query
@@ -527,16 +536,16 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : ((key#x = Subquery scalar-subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery scalar-subquery#x, [id=#x]))
      
 ===== Subqueries =====
@@ -552,39 +561,40 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
 
 
 (4) Scan parquet default.explain_temp2 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
      
 (5) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (6) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
      
 (7) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
      
 (8) HashAggregate [codegen id : 1]
-Input: [key#x]
+Input [1]: [key#x]
 Keys: []
-Functions: [partial_max(key#x)]
-Aggregate Attributes: [max#x]
-Results: [max#x]
+Functions [1]: [partial_max(key#x)]
+Aggregate Attributes [1]: [max#x]
+Results [1]: [max#x]
      
 (9) Exchange 
-Input: [max#x]
-     
+Input [1]: [max#x]
+Arguments: SinglePartition, true, [id=#x]
+      
 (10) HashAggregate [codegen id : 2]
-Input: [max#x]
+Input [1]: [max#x]
 Keys: []
-Functions: [max(key#x)]
-Aggregate Attributes: [max(key#x)#x]
-Results: [max(key#x)#x AS max(key)#x]
+Functions [1]: [max(key#x)]
+Aggregate Attributes [1]: [max(key#x)#x]
+Results [1]: [max(key#x)#x AS max(key)#x]
      
 Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (17)
@@ -597,39 +607,40 @@ Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
 
 
 (11) Scan parquet default.explain_temp3 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
      
 (12) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (13) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
      
 (14) Project [codegen id : 1]
-Output    : [key#x]
-Input     : [key#x, val#x]
+Output [1]: [key#x]
+Input [2]: [key#x, val#x]
      
 (15) HashAggregate [codegen id : 1]
-Input: [key#x]
+Input [1]: [key#x]
 Keys: []
-Functions: [partial_avg(cast(key#x as bigint))]
-Aggregate Attributes: [sum#x, count#xL]
-Results: [sum#x, count#xL]
+Functions [1]: [partial_avg(cast(key#x as bigint))]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
      
 (16) Exchange 
-Input: [sum#x, count#xL]
-     
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+      
 (17) HashAggregate [codegen id : 2]
-Input: [sum#x, count#xL]
+Input [2]: [sum#x, count#xL]
 Keys: []
-Functions: [avg(cast(key#x as bigint))]
-Aggregate Attributes: [avg(cast(key#x as bigint))#x]
-Results: [avg(cast(key#x as bigint))#x AS avg(key)#x]
+Functions [1]: [avg(cast(key#x as bigint))]
+Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
+Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
 
 
 -- !query
@@ -653,10 +664,10 @@ ReadSchema: struct<>
      
 (2) ColumnarToRow [codegen id : 1]
 Input: []
-     
+      
 (3) Project [codegen id : 1]
-Output    : [(Subquery scalar-subquery#x, [id=#x] + ReusedSubquery Subquery scalar-subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
-Input     : []
+Output [1]: [(Subquery scalar-subquery#x, [id=#x] + ReusedSubquery Subquery scalar-subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
+Input: []
      
 ===== Subqueries =====
 
@@ -669,30 +680,31 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
 
 
 (4) Scan parquet default.explain_temp1 
-Output: [key#x]
+Output [1]: [key#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int>
      
 (5) ColumnarToRow [codegen id : 1]
-Input: [key#x]
-     
+Input [1]: [key#x]
+      
 (6) HashAggregate [codegen id : 1]
-Input: [key#x]
+Input [1]: [key#x]
 Keys: []
-Functions: [partial_avg(cast(key#x as bigint))]
-Aggregate Attributes: [sum#x, count#xL]
-Results: [sum#x, count#xL]
+Functions [1]: [partial_avg(cast(key#x as bigint))]
+Aggregate Attributes [2]: [sum#x, count#xL]
+Results [2]: [sum#x, count#xL]
      
 (7) Exchange 
-Input: [sum#x, count#xL]
-     
+Input [2]: [sum#x, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+      
 (8) HashAggregate [codegen id : 2]
-Input: [sum#x, count#xL]
+Input [2]: [sum#x, count#xL]
 Keys: []
-Functions: [avg(cast(key#x as bigint))]
-Aggregate Attributes: [avg(cast(key#x as bigint))#x]
-Results: [avg(cast(key#x as bigint))#x AS avg(key)#x]
+Functions [1]: [avg(cast(key#x as bigint))]
+Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
+Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
      
 Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#x, [id=#x]
 
@@ -722,47 +734,48 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 2]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 2]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
      
 (4) Project [codegen id : 2]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (5) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
      
 (6) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (7) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
      
 (8) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (9) BroadcastExchange 
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
 (10) BroadcastHashJoin [codegen id : 2]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -793,56 +806,58 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Filter [codegen id : 1]
-Input     : [key#x, val#x]
+Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
      
 (4) Project [codegen id : 1]
-Output    : [key#x, val#x]
-Input     : [key#x, val#x]
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
      
 (5) HashAggregate [codegen id : 1]
-Input: [key#x, val#x]
-Keys: [key#x]
-Functions: [partial_max(val#x)]
-Aggregate Attributes: [max#x]
-Results: [key#x, max#x]
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
      
 (6) Exchange 
-Input: [key#x, max#x]
-     
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
 (7) HashAggregate [codegen id : 4]
-Input: [key#x, max#x]
-Keys: [key#x]
-Functions: [max(val#x)]
-Aggregate Attributes: [max(val#x)#x]
-Results: [key#x, max(val#x)#x AS max(val)#x]
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
      
 (8) ReusedExchange  [Reuses operator id: 6]
-Output : ArrayBuffer(key#x, max#x)
+Output [2]: [key#x, max#x]
      
 (9) HashAggregate [codegen id : 3]
-Input: [key#x, max#x]
-Keys: [key#x]
-Functions: [max(val#x)]
-Aggregate Attributes: [max(val#x)#x]
-Results: [key#x, max(val#x)#x AS max(val)#x]
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
      
 (10) BroadcastExchange 
-Input: [key#x, max(val)#x]
-     
+Input [2]: [key#x, max(val)#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
 (11) BroadcastHashJoin [codegen id : 4]
-Left keys: List(key#x)
-Right keys: List(key#x)
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
 Join condition: None
 
 
@@ -862,12 +877,15 @@ Execute CreateViewCommand (1)
 
 (1) Execute CreateViewCommand 
 Output: []
-     
+      
 (2) CreateViewCommand 
-     
+Arguments: `explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
+      
 (3) UnresolvedRelation 
-     
-(4) Project
+Arguments: [explain_temp1]
+      
+(4) Project 
+Arguments: ['key, 'val]
 
 
 -- !query
@@ -888,30 +906,31 @@ struct<plan:string>
 
 
 (1) Scan parquet default.explain_temp1 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) HashAggregate 
-Input: [key#x, val#x]
+Input [2]: [key#x, val#x]
 Keys: []
-Functions: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
-Aggregate Attributes: [count#xL, sum#xL, count#xL]
-Results: [count#xL, sum#xL, count#xL]
+Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
+Results [3]: [count#xL, sum#xL, count#xL]
      
 (4) Exchange 
-Input: [count#xL, sum#xL, count#xL]
-     
+Input [3]: [count#xL, sum#xL, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+      
 (5) HashAggregate [codegen id : 2]
-Input: [count#xL, sum#xL, count#xL]
+Input [3]: [count#xL, sum#xL, count#xL]
 Keys: []
-Functions: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
-Aggregate Attributes: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
-Results: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
+Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
+Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
 
 
 -- !query
@@ -931,30 +950,31 @@ ObjectHashAggregate (5)
 
 
 (1) Scan parquet default.explain_temp4 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp4]
 ReadSchema: struct<key:int,val:string>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) ObjectHashAggregate 
-Input: [key#x, val#x]
-Keys: [key#x]
-Functions: [partial_collect_set(val#x, 0, 0)]
-Aggregate Attributes: [buf#x]
-Results: [key#x, buf#x]
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [buf#x]
+Results [2]: [key#x, buf#x]
      
 (4) Exchange 
-Input: [key#x, buf#x]
-     
+Input [2]: [key#x, buf#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
 (5) ObjectHashAggregate 
-Input: [key#x, buf#x]
-Keys: [key#x]
-Functions: [collect_set(val#x, 0, 0)]
-Aggregate Attributes: [collect_set(val#x, 0, 0)#x]
-Results: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
+Input [2]: [key#x, buf#x]
+Keys [1]: [key#x]
+Functions [1]: [collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [collect_set(val#x, 0, 0)#x]
+Results [2]: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
 
 
 -- !query
@@ -976,36 +996,39 @@ SortAggregate (7)
 
 
 (1) Scan parquet default.explain_temp4 
-Output: [key#x, val#x]
+Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp4]
 ReadSchema: struct<key:int,val:string>
      
 (2) ColumnarToRow [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+      
 (3) Sort [codegen id : 1]
-Input: [key#x, val#x]
-     
+Input [2]: [key#x, val#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+      
 (4) SortAggregate 
-Input: [key#x, val#x]
-Keys: [key#x]
-Functions: [partial_min(val#x)]
-Aggregate Attributes: [min#x]
-Results: [key#x, min#x]
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_min(val#x)]
+Aggregate Attributes [1]: [min#x]
+Results [2]: [key#x, min#x]
      
 (5) Exchange 
-Input: [key#x, min#x]
-     
+Input [2]: [key#x, min#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
 (6) Sort [codegen id : 2]
-Input: [key#x, min#x]
-     
+Input [2]: [key#x, min#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+      
 (7) SortAggregate 
-Input: [key#x, min#x]
-Keys: [key#x]
-Functions: [min(val#x)]
-Aggregate Attributes: [min(val#x)#x]
-Results: [key#x, min(val#x)#x AS min(val)#x]
+Input [2]: [key#x, min#x]
+Keys [1]: [key#x]
+Functions [1]: [min(val#x)]
+Aggregate Attributes [1]: [min(val#x)#x]
+Results [2]: [key#x, min(val#x)#x AS min(val)#x]
 
 
 -- !query

From a5b3377f795a76fe218e7c9c0dafc0cbcf9a80a6 Mon Sep 17 00:00:00 2001
From: sarthfrey-db <sarth.frey@databricks.com>
Date: Fri, 21 Feb 2020 11:40:28 -0800
Subject: [PATCH 0138/1280] [SPARK-30667][CORE] Add all gather method to
 BarrierTaskContext

Fix for #27395

### What changes were proposed in this pull request?

The `allGather` method is added to the `BarrierTaskContext`. This method contains the same functionality as the `BarrierTaskContext.barrier` method; it blocks the task until all tasks make the call, at which time they may continue execution. In addition, the `allGather` method takes an input message. Upon returning from the `allGather` the task receives a list of all the messages sent by all the tasks that made the `allGather` call.

### Why are the changes needed?

There are many situations where having the tasks communicate in a synchronized way is useful. One simple example is if each task needs to start a server to serve requests from one another; first the tasks must find a free port (the result of which is undetermined beforehand) and then start making requests, but to do so they each must know the port chosen by the other task. An `allGather` method would allow them to inform each other of the port they will run on.

### Does this PR introduce any user-facing change?

Yes, an `BarrierTaskContext.allGather` method will be available through the Scala, Java, and Python APIs.

### How was this patch tested?

Most of the code path is already covered by tests to the `barrier` method, since this PR includes a refactor so that much code is shared by the `barrier` and `allGather` methods. However, a test is added to assert that an all gather on each tasks partition ID will return a list of every partition ID.

An example through the Python API:
```python
>>> from pyspark import BarrierTaskContext
>>>
>>> def f(iterator):
...     context = BarrierTaskContext.get()
...     return [context.allGather('{}'.format(context.partitionId()))]
...
>>> sc.parallelize(range(4), 4).barrier().mapPartitions(f).collect()[0]
[u'3', u'1', u'0', u'2']
```

Closes #27640 from sarthfrey/master.

Lead-authored-by: sarthfrey-db <sarth.frey@databricks.com>
Co-authored-by: sarthfrey <sarth.frey@gmail.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit 274b328f57a626bb770c9831ae81034d060a7e1e)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../org/apache/spark/BarrierCoordinator.scala | 111 +++++++++++--
 .../org/apache/spark/BarrierTaskContext.scala | 153 ++++++++++++------
 .../spark/api/python/PythonRunner.scala       |  51 ++++--
 .../scheduler/BarrierTaskContextSuite.scala   |  77 +++++++++
 project/MimaExcludes.scala                    |   5 +-
 python/pyspark/taskcontext.py                 |  51 +++++-
 python/pyspark/tests/test_taskcontext.py      |  20 +++
 7 files changed, 388 insertions(+), 80 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 4e417679ca663..be5036e82e4b2 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -17,12 +17,17 @@
 
 package org.apache.spark
 
+import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
 import scala.collection.mutable.ArrayBuffer
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
@@ -99,10 +104,15 @@ private[spark] class BarrierCoordinator(
     // reset when a barrier() call fails due to timeout.
     private var barrierEpoch: Int = 0
 
-    // An array of RPCCallContexts for barrier tasks that are waiting for reply of a barrier()
-    // call.
+    // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
+    // An Array of allGather messages for barrier tasks that have made a blocking runBarrier() call
+    private val allGatherMessages: ArrayBuffer[String] = new Array[String](numTasks).to[ArrayBuffer]
+
+    // The blocking requestMethod called by tasks to sync up for this stage attempt
+    private var requestMethodToSync: RequestMethod.Value = RequestMethod.BARRIER
+
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
 
@@ -130,9 +140,32 @@ private[spark] class BarrierCoordinator(
 
     // Process the global sync request. The barrier() call succeed if collected enough requests
     // within a configured time, otherwise fail all the pending requests.
-    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
+    def handleRequest(
+      requester: RpcCallContext,
+      request: RequestToSync
+    ): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
+      val requestMethod = request.requestMethod
+      val partitionId = request.partitionId
+      val allGatherMessage = request match {
+        case ag: AllGatherRequestToSync => ag.allGatherMessage
+        case _ => ""
+      }
+
+      if (requesters.size == 0) {
+        requestMethodToSync = requestMethod
+      }
+
+      if (requestMethodToSync != requestMethod) {
+        requesters.foreach(
+          _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
+            s"`$requestMethod` during barrier epoch $barrierEpoch, which does not match " +
+            s"the current synchronized requestMethod `$requestMethodToSync`"
+          ))
+        )
+        cleanupBarrierStage(barrierId)
+      }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
       require(request.numTasks == numTasks, s"Number of tasks of $barrierId is " +
@@ -153,6 +186,7 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
+        allGatherMessages(partitionId) = allGatherMessage
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
         if (maybeFinishAllRequesters(requesters, numTasks)) {
@@ -173,7 +207,13 @@ private[spark] class BarrierCoordinator(
         requesters: ArrayBuffer[RpcCallContext],
         numTasks: Int): Boolean = {
       if (requesters.size == numTasks) {
-        requesters.foreach(_.reply(()))
+        requestMethodToSync match {
+          case RequestMethod.BARRIER =>
+            requesters.foreach(_.reply(""))
+          case RequestMethod.ALL_GATHER =>
+            val json: String = compact(render(allGatherMessages))
+            requesters.foreach(_.reply(json))
+        }
         true
       } else {
         false
@@ -199,11 +239,11 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _) =>
+    case request: RequestToSync =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
-      val barrierId = ContextBarrierId(stageId, stageAttemptId)
+      val barrierId = ContextBarrierId(request.stageId, request.stageAttemptId)
       states.computeIfAbsent(barrierId,
-        (key: ContextBarrierId) => new ContextBarrierState(key, numTasks))
+        (key: ContextBarrierId) => new ContextBarrierState(key, request.numTasks))
       val barrierState = states.get(barrierId)
 
       barrierState.handleRequest(context, request)
@@ -216,6 +256,16 @@ private[spark] class BarrierCoordinator(
 
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
+private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
+  def numTasks: Int
+  def stageId: Int
+  def stageAttemptId: Int
+  def taskAttemptId: Long
+  def barrierEpoch: Int
+  def partitionId: Int
+  def requestMethod: RequestMethod.Value
+}
+
 /**
  * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
@@ -224,11 +274,44 @@ private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls.
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
  */
-private[spark] case class RequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptId: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int) extends BarrierCoordinatorMessage
+private[spark] case class BarrierRequestToSync(
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  requestMethod: RequestMethod.Value
+) extends RequestToSync
+
+/**
+ * A global sync request message from BarrierTaskContext, by `allGather()` call. Each request is
+ * identified by stageId + stageAttemptId + barrierEpoch.
+ *
+ * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
+ * @param stageId ID of current stage
+ * @param stageAttemptId ID of current stage attempt
+ * @param taskAttemptId Unique ID of current task
+ * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param partitionId ID of the current partition the task is assigned to
+ * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
+ * @param allGatherMessage Message sent from the BarrierTaskContext if requestMethod is ALL_GATHER
+ */
+private[spark] case class AllGatherRequestToSync(
+  numTasks: Int,
+  stageId: Int,
+  stageAttemptId: Int,
+  taskAttemptId: Long,
+  barrierEpoch: Int,
+  partitionId: Int,
+  requestMethod: RequestMethod.Value,
+  allGatherMessage: String
+) extends RequestToSync
+
+private[spark] object RequestMethod extends Enumeration {
+  val BARRIER, ALL_GATHER = Value
+}
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 3d369802f3023..2263538a11676 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -17,11 +17,19 @@
 
 package org.apache.spark
 
+import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
+import scala.language.postfixOps
+
+import org.json4s.DefaultFormats
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.parse
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -59,49 +67,31 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  /**
-   * :: Experimental ::
-   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
-   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
-   * stage have reached this routine.
-   *
-   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
-   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
-   * timeout. Some examples of '''misuses''' are listed below:
-   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
-   * shall lead to timeout of the function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       if (context.partitionId() == 0) {
-   *           // Do nothing.
-   *       } else {
-   *           context.barrier()
-   *       }
-   *       iter
-   *   }
-   * }}}
-   *
-   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
-   * second function call.
-   * {{{
-   *   rdd.barrier().mapPartitions { iter =>
-   *       val context = BarrierTaskContext.get()
-   *       try {
-   *           // Do something that might throw an Exception.
-   *           doSomething()
-   *           context.barrier()
-   *       } catch {
-   *           case e: Exception => logWarning("...", e)
-   *       }
-   *       context.barrier()
-   *       iter
-   *   }
-   * }}}
-   */
-  @Experimental
-  @Since("2.4.0")
-  def barrier(): Unit = {
+  private def getRequestToSync(
+    numTasks: Int,
+    stageId: Int,
+    stageAttemptNumber: Int,
+    taskAttemptId: Long,
+    barrierEpoch: Int,
+    partitionId: Int,
+    requestMethod: RequestMethod.Value,
+    allGatherMessage: String
+  ): RequestToSync = {
+    requestMethod match {
+      case RequestMethod.BARRIER =>
+        BarrierRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, requestMethod)
+      case RequestMethod.ALL_GATHER =>
+        AllGatherRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, requestMethod, allGatherMessage)
+    }
+  }
+
+  private def runBarrier(
+    requestMethod: RequestMethod.Value,
+    allGatherMessage: String = ""
+  ): String = {
+
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -118,10 +108,12 @@ class BarrierTaskContext private[spark] (
     // Log the update of global sync every 60 seconds.
     timer.schedule(timerTask, 60000, 60000)
 
+    var json: String = ""
+
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[Unit](
-        message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch),
+      val abortableRpcFuture = barrierCoordinator.askAbortable[String](
+        message = getRequestToSync(numTasks, stageId, stageAttemptNumber,
+          taskAttemptId, barrierEpoch, partitionId, requestMethod, allGatherMessage),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
@@ -133,7 +125,7 @@ class BarrierTaskContext private[spark] (
         while (!abortableRpcFuture.toFuture.isCompleted) {
           // wait RPC future for at most 1 second
           try {
-            ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
+            json = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
           } catch {
             case _: TimeoutException | _: InterruptedException =>
               // If `TimeoutException` thrown, waiting RPC future reach 1 second.
@@ -163,6 +155,73 @@ class BarrierTaskContext private[spark] (
       timerTask.cancel()
       timer.purge()
     }
+    json
+  }
+
+  /**
+   * :: Experimental ::
+   * Sets a global barrier and waits until all tasks in this stage hit this barrier. Similar to
+   * MPI_Barrier function in MPI, the barrier() function call blocks until all tasks in the same
+   * stage have reached this routine.
+   *
+   * CAUTION! In a barrier stage, each task must have the same number of barrier() calls, in all
+   * possible code branches. Otherwise, you may get the job hanging or a SparkException after
+   * timeout. Some examples of '''misuses''' are listed below:
+   * 1. Only call barrier() function on a subset of all the tasks in the same barrier stage, it
+   * shall lead to timeout of the function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       if (context.partitionId() == 0) {
+   *           // Do nothing.
+   *       } else {
+   *           context.barrier()
+   *       }
+   *       iter
+   *   }
+   * }}}
+   *
+   * 2. Include barrier() function in a try-catch code block, this may lead to timeout of the
+   * second function call.
+   * {{{
+   *   rdd.barrier().mapPartitions { iter =>
+   *       val context = BarrierTaskContext.get()
+   *       try {
+   *           // Do something that might throw an Exception.
+   *           doSomething()
+   *           context.barrier()
+   *       } catch {
+   *           case e: Exception => logWarning("...", e)
+   *       }
+   *       context.barrier()
+   *       iter
+   *   }
+   * }}}
+   */
+  @Experimental
+  @Since("2.4.0")
+  def barrier(): Unit = {
+    runBarrier(RequestMethod.BARRIER)
+    ()
+  }
+
+  /**
+   * :: Experimental ::
+   * Blocks until all tasks in the same stage have reached this routine. Each task passes in
+   * a message and returns with a list of all the messages passed in by each of those tasks.
+   *
+   * CAUTION! The allGather method requires the same precautions as the barrier method
+   *
+   * The message is type String rather than Array[Byte] because it is more convenient for
+   * the user at the cost of worse performance.
+   */
+  @Experimental
+  @Since("3.0.0")
+  def allGather(message: String): ArrayBuffer[String] = {
+    val json = runBarrier(RequestMethod.ALL_GATHER, message)
+    val jsonArray = parse(json)
+    implicit val formats = DefaultFormats
+    ArrayBuffer(jsonArray.extract[Array[String]]: _*)
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 658e0d593a167..fa8bf0fc06358 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -24,8 +24,13 @@ import java.nio.charset.StandardCharsets.UTF_8
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
+import org.json4s.JsonAST._
+import org.json4s.JsonDSL._
+import org.json4s.jackson.JsonMethods.{compact, render}
+
 import org.apache.spark._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{BUFFER_SIZE, EXECUTOR_CORES}
@@ -238,13 +243,18 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
                   sock.setSoTimeout(10000)
                   authHelper.authClient(sock)
                   val input = new DataInputStream(sock.getInputStream())
-                  input.readInt() match {
+                  val requestMethod = input.readInt()
+                  // The BarrierTaskContext function may wait infinitely, socket shall not timeout
+                  // before the function finishes.
+                  sock.setSoTimeout(0)
+                  requestMethod match {
                     case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
-                      // The barrier() function may wait infinitely, socket shall not timeout
-                      // before the function finishes.
-                      sock.setSoTimeout(0)
-                      barrierAndServe(sock)
-
+                      barrierAndServe(requestMethod, sock)
+                    case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+                      val length = input.readInt()
+                      val message = new Array[Byte](length)
+                      input.readFully(message)
+                      barrierAndServe(requestMethod, sock, new String(message, UTF_8))
                     case _ =>
                       val out = new DataOutputStream(new BufferedOutputStream(
                         sock.getOutputStream))
@@ -395,15 +405,31 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
     }
 
     /**
-     * Gateway to call BarrierTaskContext.barrier().
+     * Gateway to call BarrierTaskContext methods.
      */
-    def barrierAndServe(sock: Socket): Unit = {
-      require(serverSocket.isDefined, "No available ServerSocket to redirect the barrier() call.")
-
+    def barrierAndServe(requestMethod: Int, sock: Socket, message: String = ""): Unit = {
+      require(
+        serverSocket.isDefined,
+        "No available ServerSocket to redirect the BarrierTaskContext method call."
+      )
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        context.asInstanceOf[BarrierTaskContext].barrier()
-        writeUTF(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS, out)
+        var result: String = ""
+        requestMethod match {
+          case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
+            context.asInstanceOf[BarrierTaskContext].barrier()
+            result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
+          case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
+            val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
+              message
+            )
+            result = compact(render(JArray(
+              messages.map(
+                (message) => JString(message)
+              ).toList
+            )))
+        }
+        writeUTF(result, out)
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
@@ -638,6 +664,7 @@ private[spark] object SpecialLengths {
 
 private[spark] object BarrierTaskContextMessageProtocol {
   val BARRIER_FUNCTION = 1
+  val ALL_GATHER_FUNCTION = 2
   val BARRIER_RESULT_SUCCESS = "success"
   val ERROR_UNRECOGNIZED_FUNCTION = "Not recognized function call from python side."
 }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index fc8ac38479932..33594c0a50d14 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
+import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
 import org.apache.spark._
@@ -52,6 +53,82 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     assert(times.max - times.min <= 1000)
   }
 
+  test("share messages with allGather() call") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message: String = context.partitionId().toString
+      val messages: ArrayBuffer[String] = context.allGather(message)
+      messages.toList.iterator
+    }
+    // Take a sorted list of all the partitionId messages
+    val messages = rdd2.collect().head
+    // All the task partitionIds are shared
+    for((x, i) <- messages.view.zipWithIndex) assert(x.toString == i.toString)
+  }
+
+  test("throw exception if we attempt to synchronize with different blocking calls") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      val partitionId = context.partitionId
+      if (partitionId == 0) {
+        context.barrier()
+      } else {
+        context.allGather(partitionId.toString)
+      }
+      Seq(null).iterator
+    }
+    val error = intercept[SparkException] {
+      rdd2.collect()
+    }.getMessage
+    assert(
+      error.contains("does not match the current synchronized requestMethod") ||
+      error.contains("not properly killed")
+    )
+  }
+
+  test("successively sync with allGather and barrier") {
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
+    val rdd = sc.makeRDD(1 to 10, 4)
+    val rdd2 = rdd.barrier().mapPartitions { it =>
+      val context = BarrierTaskContext.get()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      context.barrier()
+      val time1 = System.currentTimeMillis()
+      // Sleep for a random time before global sync.
+      Thread.sleep(Random.nextInt(1000))
+      // Pass partitionId message in
+      val message = context.partitionId().toString
+      val messages = context.allGather(message)
+      val time2 = System.currentTimeMillis()
+      Seq((time1, time2)).iterator
+    }
+    val times = rdd2.collect()
+    // All the tasks shall finish the first round of global sync within a short time slot.
+    val times1 = times.map(_._1)
+    assert(times1.max - times1.min <= 1000)
+
+    // All the tasks shall finish the second round of global sync within a short time slot.
+    val times2 = times.map(_._2)
+    assert(times2.max - times2.min <= 1000)
+  }
+
   test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 2a72284c011f5..289ccb2cf3664 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -495,7 +495,10 @@ object MimaExcludes {
     ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.AFTSurvivalRegression.setPredictionCol"),
 
     // [SPARK-29543][SS][UI] Init structured streaming ui
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryListener#QueryStartedEvent.this")
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryListener#QueryStartedEvent.this"),
+
+    // [SPARK-30667][CORE] Add allGather method to BarrierTaskContext
+    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.RequestToSync")
   )
 
   // Exclude rules for 2.4.x
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index d648f63338514..e4fc64b732ba7 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -16,9 +16,10 @@
 #
 
 from __future__ import print_function
+import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, UTF8Deserializer
+from pyspark.serializers import write_int, write_with_length, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -107,18 +108,28 @@ def resources(self):
 
 
 BARRIER_FUNCTION = 1
+ALL_GATHER_FUNCTION = 2
 
 
-def _load_from_socket(port, auth_secret):
+def _load_from_socket(port, auth_secret, function, all_gather_message=None):
     """
     Load data from a given socket, this is a blocking method thus only return when the socket
     connection has been closed.
     """
     (sockfile, sock) = local_connect_and_auth(port, auth_secret)
-    # The barrier() call may block forever, so no timeout
+
+    # The call may block forever, so no timeout
     sock.settimeout(None)
-    # Make a barrier() function call.
-    write_int(BARRIER_FUNCTION, sockfile)
+
+    if function == BARRIER_FUNCTION:
+        # Make a barrier() function call.
+        write_int(function, sockfile)
+    elif function == ALL_GATHER_FUNCTION:
+        # Make a all_gather() function call.
+        write_int(function, sockfile)
+        write_with_length(all_gather_message.encode("utf-8"), sockfile)
+    else:
+        raise ValueError("Unrecognized function type")
     sockfile.flush()
 
     # Collect result.
@@ -199,7 +210,35 @@ def barrier(self):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            _load_from_socket(self._port, self._secret)
+            _load_from_socket(self._port, self._secret, BARRIER_FUNCTION)
+
+    def allGather(self, message=""):
+        """
+        .. note:: Experimental
+
+        This function blocks until all tasks in the same stage have reached this routine.
+        Each task passes in a message and returns with a list of all the messages passed in
+        by each of those tasks.
+
+        .. warning:: In a barrier stage, each task much have the same number of `allGather()`
+            calls, in all possible code branches.
+            Otherwise, you may get the job hanging or a SparkException after timeout.
+
+        .. versionadded:: 3.0.0
+        """
+        if not isinstance(message, str):
+            raise ValueError("Argument `message` must be of type `str`")
+        elif self._port is None or self._secret is None:
+            raise Exception("Not supported to call barrier() before initialize " +
+                            "BarrierTaskContext.")
+        else:
+            gathered_items = _load_from_socket(
+                self._port,
+                self._secret,
+                ALL_GATHER_FUNCTION,
+                message,
+            )
+            return [e for e in json.loads(gathered_items)]
 
     def getTaskInfos(self):
         """
diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index 6095a384679af..57c880ad8043d 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -134,6 +134,26 @@ def context_barrier(x):
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
         self.assertTrue(max(times) - min(times) < 1)
 
+    def test_all_gather(self):
+        """
+        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
+        within a stage and passes messages properly.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 10))
+            out = tc.allGather(str(tc.partitionId()))
+            pids = [int(e) for e in out]
+            return pids
+
+        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
+        self.assertEqual(pids, [0, 1, 2, 3])
+
     def test_barrier_infos(self):
         """
         Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the

From f50d4f04aa11b6a61c7eb4f0ece46dc8ab99ba9c Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Sat, 22 Feb 2020 09:48:10 +0900
Subject: [PATCH 0139/1280] [MINOR][SQL] Add a comment for `removedSQLConfigs`

### What changes were proposed in this pull request?
In the PR, I propose to explain in the description of `removedSQLConfigs` when removed SQL configs should NOT be placed to the map.

### Why are the changes needed?
To make the cases when SQL configs should be added to `removedSQLConfigs` more clear. Recently, `spark.sql.variable.substitute.depth` was removed from the map by #27646 because it contradicts to the condition described by the PR.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By `./dev/scalastyle`

Closes #27653 from MaxGekk/removedSQLConfigs-comment.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 310c14ac8d581b4d5864a846f4f5cbcde43e71f1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index c73d8b6c1e7c5..72480e0915408 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2262,6 +2262,10 @@ object SQLConf {
    * The map contains info about removed SQL configs. Keys are SQL config names,
    * map values contain extra information like the version in which the config was removed,
    * config's default value and a comment.
+   *
+   * Please, add a removed SQL configuration property here only when it affects behaviours.
+   * For example, `spark.sql.variable.substitute.depth` was not added as it virtually
+   * became no-op later. By this, it makes migrations to new Spark versions painless.
    */
   val removedSQLConfigs: Map[String, RemovedConfig] = {
     val configs = Seq(

From f38554f33fb61245c869347962b182de1dcdd9ee Mon Sep 17 00:00:00 2001
From: roland-ondeviceresearch <roland@ondeviceresearch.com>
Date: Sat, 22 Feb 2020 09:52:45 +0900
Subject: [PATCH 0140/1280] [MINOR][DOCS] Fix ForEachWriter Java example

### What changes were proposed in this pull request?
Structured streaming documentation example fix

### Why are the changes needed?
Currently the java example uses incorrect syntax

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
In IDE

Closes #27671 from roland1982/foreachwriter_java_example_fix.

Authored-by: roland-ondeviceresearch <roland@ondeviceresearch.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 9a2cec9b1ee3ba095461d9c07c9932107228d42e)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/structured-streaming-programming-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 9f4edd73c2b65..47d899431888b 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -2202,7 +2202,7 @@ streamingDatasetOfString.writeStream.foreach(
 In Java, you have to extend the class `ForeachWriter` ([docs](api/java/org/apache/spark/sql/ForeachWriter.html)).
 {% highlight java %}
 streamingDatasetOfString.writeStream().foreach(
-  new ForeachWriter[String] {
+  new ForeachWriter<String>() {
 
     @Override public boolean open(long partitionId, long version) {
       // Open connection

From ba4212660305c6555ae16b10c6bbaf6114c4d830 Mon Sep 17 00:00:00 2001
From: Christian Stuart <christian.stuart@databricks.com>
Date: Fri, 21 Feb 2020 21:39:35 -0700
Subject: [PATCH 0141/1280] [SPARK-30904][SQL] Thrift RowBasedSet serialization
 throws NullPointerException on NULL BigDecimal

### What changes were proposed in this pull request?

This PR fixes SPARK-30904 by adding a null check.

### Why are the changes needed?

For HIVE_CLI_SERVICE_PROTOCOL_V5 and below, serialization fails on NULL-containing decimal columns, caused by a call to  `value.toPlainString()`, where `value` might be null. This null check fixes it.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

A test was added for serialization of NULL decimals for all HIVE_CLI_SERVICE_PROTOCOL versions.

Closes #27654 from CJStuart/SPARK-30904.

Authored-by: Christian Stuart <christian.stuart@databricks.com>
Signed-off-by: Yuming Wang <wgyumg@gmail.com>
(cherry picked from commit bcce1b10408b5c516c936e8770a598ef415d71c3)
Signed-off-by: Yuming Wang <wgyumg@gmail.com>
---
 .../SparkThriftServerProtocolVersionsSuite.scala             | 5 +++++
 .../main/java/org/apache/hive/service/cli/ColumnValue.java   | 3 ++-
 .../main/java/org/apache/hive/service/cli/ColumnValue.java   | 3 ++-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
index a63b5dac0aac3..fd45e7a48c0eb 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkThriftServerProtocolVersionsSuite.scala
@@ -192,6 +192,11 @@ class SparkThriftServerProtocolVersionsSuite extends HiveThriftJdbcTest {
         assert(rs.next())
         assert(rs.getBigDecimal(1) === new java.math.BigDecimal("1.00"))
       }
+      testExecuteStatementWithProtocolVersion(version,
+        "SELECT cast(null as decimal) ") { rs =>
+        assert(rs.next())
+        assert(rs.getBigDecimal(1) === null)
+      }
     }
 
     test(s"$version get string type") {
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index 547c6056b4fe8..a770bea9c2aa6 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -186,7 +186,8 @@ public static TColumnValue toTColumnValue(Type type, Object value) {
     case INTERVAL_DAY_TIME_TYPE:
       return stringValue((HiveIntervalDayTime) value);
     case DECIMAL_TYPE:
-      return stringValue(((BigDecimal)value).toPlainString());
+      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
+      return stringValue(plainStr);
     case BINARY_TYPE:
       String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
       return stringValue(strVal);
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index f881ab159ea67..53f0465a056d8 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -189,7 +189,8 @@ public static TColumnValue toTColumnValue(TypeDescriptor typeDescriptor, Object
     case INTERVAL_DAY_TIME_TYPE:
       return stringValue((HiveIntervalDayTime) value);
     case DECIMAL_TYPE:
-      return stringValue(((BigDecimal)value).toPlainString());
+      String plainStr = value == null ? null : ((BigDecimal)value).toPlainString();
+      return stringValue(plainStr);
     case BINARY_TYPE:
       String strVal = value == null ? null : UTF8String.fromBytes((byte[])value).toString();
       return stringValue(strVal);

From 4a82ead147c944c8d4828bbbd4a7e3ec3d3e1135 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Sun, 23 Feb 2020 09:52:54 +0900
Subject: [PATCH 0142/1280] [SPARK-30903][SQL] Fail fast on duplicate columns
 when analyze columns

<!--
Thanks for sending a pull request!  Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://spark.apache.org/contributing.html
  2. Ensure you have added or run the appropriate tests for your PR: https://spark.apache.org/developer-tools.html
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][SPARK-XXXX] Your PR title ...'.
  4. Be sure to keep the PR description updated to reflect all changes.
  5. Please write your PR title to summarize what this PR proposes.
  6. If possible, provide a concise example to reproduce the issue for a faster review.
  7. If you want to add a new configuration, please read the guideline first for naming configurations in
     'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
-->

### What changes were proposed in this pull request?
<!--
Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
  1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
  2. If you fix some SQL features, you can provide some references of other DBMSes.
  3. If there is design documentation, please add the link.
  4. If there is a discussion in the mailing list, please add the link.
-->

Add new `CommandCheck` rule and fail fast when detects duplicate columns in `AnalyzeColumnCommand`.

### Why are the changes needed?
<!--
Please clarify why the changes are needed. For instance,
  1. If you propose a new API, clarify the use case for a new API.
  2. If you fix a bug, you can clarify why it is a bug.
-->

To avoid duplicate statistics computation for the same column in `AnalyzeColumnCommand`.

### Does this PR introduce any user-facing change?
<!--
If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
If no, write 'No'.
-->

Yes. User now get exception when input duplicate columns.

### How was this patch tested?
<!--
If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
If tests were not added, please describe why they were not added and/or why it was difficult to add.
-->

Added new test.

Closes #27651 from Ngone51/fail_on_dup_cols.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 25f5bfaa6e624da7f491e770a2383038fc6009e1)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/execution/command/CommandCheck.scala  | 38 +++++++++++++++++++
 .../internal/BaseSessionStateBuilder.scala    |  2 +
 .../spark/sql/StatisticsCollectionSuite.scala | 17 +++++++++
 .../sql/hive/HiveSessionStateBuilder.scala    |  2 +
 4 files changed, 59 insertions(+)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
new file mode 100644
index 0000000000000..dedace4af4d14
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandCheck.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.util.SchemaUtils
+
+/**
+ * Checks legitimization of various execution commands.
+ */
+case class CommandCheck(conf: SQLConf) extends (LogicalPlan => Unit) {
+
+  override def apply(plan: LogicalPlan): Unit = {
+    plan.foreach {
+      case AnalyzeColumnCommand(_, colsOpt, allColumns) if !allColumns =>
+        colsOpt.foreach(SchemaUtils.checkColumnNameDuplication(
+          _, "in analyze columns.", conf.caseSensitiveAnalysis))
+
+      case _ =>
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 2137fe2d0a8ff..20e1b5694e952 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlanner, SparkSqlParser}
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
+import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.v2.{TableCapabilityCheck, V2SessionCatalog}
 import org.apache.spark.sql.streaming.StreamingQueryManager
@@ -190,6 +191,7 @@ abstract class BaseSessionStateBuilder(
         PreReadCheck +:
         HiveOnlyCheck +:
         TableCapabilityCheck +:
+        CommandCheck(conf) +:
         customCheckRules
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index e9ceab6724659..30b15a8710d8c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -650,4 +650,21 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
       }
     }
   }
+
+  Seq(true, false).foreach { caseSensitive =>
+    test(s"SPARK-30903: Fail fast on duplicate columns when analyze columns " +
+      s"- caseSensitive=$caseSensitive") {
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+        val table = "test_table"
+        withTable(table) {
+          sql(s"CREATE TABLE $table (value string, name string) USING PARQUET")
+          val dupCol = if (caseSensitive) "value" else "VaLuE"
+          val errorMsg = intercept[AnalysisException] {
+            sql(s"ANALYZE TABLE $table COMPUTE STATISTICS FOR COLUMNS value, name, $dupCol")
+          }.getMessage
+          assert(errorMsg.contains("Found duplicate column(s)"))
+        }
+      }
+    }
+  }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index 1858fbd910666..7ec52ba31c029 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{SparkOptimizer, SparkPlanner}
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
+import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.execution.datasources.v2.TableCapabilityCheck
 import org.apache.spark.sql.hive.client.HiveClient
@@ -93,6 +94,7 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
       PreWriteCheck +:
         PreReadCheck +:
         TableCapabilityCheck +:
+        CommandCheck(conf) +:
         customCheckRules
   }
 

From f30f50a76f4b9fb5e652620563fb9055c5f30521 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Sun, 23 Feb 2020 17:46:19 +0900
Subject: [PATCH 0143/1280] [SPARK-30844][SQL] Static partition should also
 follow StoreAssignmentPolicy when insert into table

### What changes were proposed in this pull request?

Make static partition also follows `StoreAssignmentPolicy` when insert into table:

if `StoreAssignmentPolicy=LEGACY`, using `Cast`;
if `StoreAssignmentPolicy=ANSI | STRIC`, using `AnsiCast`;

E.g., for the table `t` created by:

```
create table t(a int, b string) using parquet partitioned by (a)
```
and insert values with `StoreAssignmentPolicy=ANSI` using:
```
insert into t partition(a='ansi') values('ansi')
```

Before this PR:

```
+----+----+
|   b|   a|
+----+----+
|ansi|null|
+----+----+
```

After this PR, insert will fail by:
```
java.lang.NumberFormatException: invalid input syntax for type numeric: ansi
```

(It should be better if we could use `TableOutputResolver.checkField` to fully follow `StoreAssignmentPolicy`. But since we lost the data type of static partition's value at first place, it's hard to use `TableOutputResolver.checkField`.)

### Why are the changes needed?

I think we should follow `StoreAssignmentPolicy` when insert into table for any columns, including static partition.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added new test.

Closes #27597 from Ngone51/fix-static-partition.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 9c2eadc7268844d49ec41da818002c99bb56addf)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../datasources/DataSourceStrategy.scala      | 13 +++++++++++-
 .../sql/sources/DataSourceAnalysisSuite.scala | 10 +++++++--
 .../spark/sql/sources/InsertSuite.scala       | 21 +++++++++++++++++++
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index e3a0a0a6c34e5..2d902b5c43c44 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{RowDataSourceScanExec, SparkPlan}
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -104,7 +105,17 @@ case class DataSourceAnalysis(conf: SQLConf) extends Rule[LogicalPlan] with Cast
         None
       } else if (potentialSpecs.size == 1) {
         val partValue = potentialSpecs.head._2
-        Some(Alias(cast(Literal(partValue), field.dataType), field.name)())
+        conf.storeAssignmentPolicy match {
+          // SPARK-30844: try our best to follow StoreAssignmentPolicy for static partition
+          // values but not completely follow because we can't do static type checking due to
+          // the reason that the parser has erased the type info of static partition values
+          // and converted them to string.
+          case StoreAssignmentPolicy.ANSI | StoreAssignmentPolicy.STRICT =>
+            Some(Alias(AnsiCast(Literal(partValue), field.dataType,
+              Option(conf.sessionLocalTimeZone)), field.name)())
+          case _ =>
+            Some(Alias(cast(Literal(partValue), field.dataType), field.name)())
+        }
       } else {
         throw new AnalysisException(
           s"Partition column ${field.name} have multiple values specified, " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
index e1022e377132c..a6c50904d395b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/DataSourceAnalysisSuite.scala
@@ -22,9 +22,10 @@ import org.scalatest.BeforeAndAfterAll
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, Attribute, Cast, Expression, Literal}
 import org.apache.spark.sql.execution.datasources.DataSourceAnalysis
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.sql.types.{DataType, IntegerType, StructType}
 
 class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
@@ -52,7 +53,12 @@ class DataSourceAnalysisSuite extends SparkFunSuite with BeforeAndAfterAll {
   Seq(true, false).foreach { caseSensitive =>
     val conf = new SQLConf().copy(SQLConf.CASE_SENSITIVE -> caseSensitive)
     def cast(e: Expression, dt: DataType): Expression = {
-      Cast(e, dt, Option(conf.sessionLocalTimeZone))
+      conf.storeAssignmentPolicy match {
+        case StoreAssignmentPolicy.ANSI | StoreAssignmentPolicy.STRICT =>
+          AnsiCast(e, dt, Option(conf.sessionLocalTimeZone))
+        case _ =>
+          Cast(e, dt, Option(conf.sessionLocalTimeZone))
+      }
     }
     val rule = DataSourceAnalysis(conf)
     test(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index bcff30a51c3f5..2d666371da21b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -753,6 +753,27 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-30844: static partition should also follow StoreAssignmentPolicy") {
+    SQLConf.StoreAssignmentPolicy.values.foreach { policy =>
+      withSQLConf(
+        SQLConf.STORE_ASSIGNMENT_POLICY.key -> policy.toString) {
+        withTable("t") {
+          sql("create table t(a int, b string) using parquet partitioned by (a)")
+          policy match {
+            case SQLConf.StoreAssignmentPolicy.ANSI | SQLConf.StoreAssignmentPolicy.STRICT =>
+              val errorMsg = intercept[NumberFormatException] {
+                sql("insert into t partition(a='ansi') values('ansi')")
+              }.getMessage
+              assert(errorMsg.contains("invalid input syntax for type numeric: ansi"))
+            case SQLConf.StoreAssignmentPolicy.LEGACY =>
+              sql("insert into t partition(a='ansi') values('ansi')")
+              checkAnswer(sql("select * from t"), Row("ansi", null) :: Nil)
+          }
+        }
+      }
+    }
+  }
+
   test("SPARK-24860: dynamic partition overwrite specified per source without catalog table") {
     withTempPath { path =>
       Seq((1, 1), (2, 2)).toDF("i", "part")

From 2c36abf8c4d9079901f79b188d9e951256e925b2 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 24 Feb 2020 14:06:25 +0800
Subject: [PATCH 0144/1280] [SPARK-30925][SQL] Prevent overflow/round errors in
 conversions of milliseconds to/from microseconds

### What changes were proposed in this pull request?
- Use `Math.multiplyExact()` in `DateTimeUtils.fromMillis()` to prevent silent overflow in conversion milliseconds to microseconds.
- Use `DateTimeUtils.fromMillis()` in all places where milliseconds are converted to microseconds
- Use `DateTimeUtils.toMillis()` in all places where microseconds are converted to milliseconds

### Why are the changes needed?

1. To prevent silent arithmetic overflow while multiplying by 1000 in `fromMillis()`. Instead of it, `new ArithmeticException("long overflow")` will be thrown, and handled accordantly.
2. To correctly round microseconds in conversion to milliseconds. For example, `1965-01-01 10:11:12.123456` is represented as `-157700927876544` in micro precision. In milliseconds precision the above needs to be represented as `-157700927877` or `1965-01-01 10:11:12.123`.

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
By `TimestampFormatterSuite`, `CastSuite`, `DateExpressionsSuite`, `IntervalExpressionsSuite`, `ExpressionParserSuite`, `ExpressionParserSuite`, `DateTimeUtilsSuite`, `IntervalUtilsSuite`

Closes #27676 from MaxGekk/millis-2-micros-overflow.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c41ef3981953a9d59030ea5f29386804b2d560a6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/util/DateTimeUtils.scala    | 14 +++++++-------
 .../spark/sql/catalyst/util/IntervalUtils.scala    |  5 ++---
 .../sql/catalyst/util/TimestampFormatter.scala     |  6 +++---
 .../spark/sql/catalyst/expressions/CastSuite.scala |  5 ++---
 .../expressions/DateExpressionsSuite.scala         |  3 +--
 .../expressions/IntervalExpressionsSuite.scala     |  3 ++-
 .../catalyst/parser/ExpressionParserSuite.scala    |  5 ++---
 .../sql/catalyst/util/DateTimeTestUtils.scala      |  9 +++++++++
 .../sql/catalyst/util/DateTimeUtilsSuite.scala     |  9 ++++-----
 .../sql/catalyst/util/IntervalUtilsSuite.scala     |  5 +++--
 .../streaming/EventTimeWatermarkExec.scala         |  4 ++--
 .../spark/sql/execution/streaming/Triggers.scala   |  4 +++-
 .../spark/sql/StatisticsCollectionTestBase.scala   |  6 +++---
 13 files changed, 43 insertions(+), 35 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index a1054a4ab9864..f5f0309dd6b7d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -65,7 +65,7 @@ object DateTimeUtils {
   }
 
   def millisToDays(millisUtc: Long, zoneId: ZoneId): SQLDate = {
-    val instant = microsToInstant(Math.multiplyExact(millisUtc, MICROS_PER_MILLIS))
+    val instant = microsToInstant(fromMillis(millisUtc))
     localDateToDays(LocalDateTime.ofInstant(instant, zoneId).toLocalDate)
   }
 
@@ -76,7 +76,7 @@ object DateTimeUtils {
 
   def daysToMillis(days: SQLDate, zoneId: ZoneId): Long = {
     val instant = daysToLocalDate(days).atStartOfDay(zoneId).toInstant
-    instantToMicros(instant) / MICROS_PER_MILLIS
+    toMillis(instantToMicros(instant))
   }
 
   // Converts Timestamp to string according to Hive TimestampWritable convention.
@@ -149,7 +149,7 @@ object DateTimeUtils {
    * Converts milliseconds since epoch to SQLTimestamp.
    */
   def fromMillis(millis: Long): SQLTimestamp = {
-    MILLISECONDS.toMicros(millis)
+    Math.multiplyExact(millis, MICROS_PER_MILLIS)
   }
 
   def microsToEpochDays(epochMicros: SQLTimestamp, zoneId: ZoneId): SQLDate = {
@@ -572,8 +572,8 @@ object DateTimeUtils {
       time2: SQLTimestamp,
       roundOff: Boolean,
       zoneId: ZoneId): Double = {
-    val millis1 = MICROSECONDS.toMillis(time1)
-    val millis2 = MICROSECONDS.toMillis(time2)
+    val millis1 = toMillis(time1)
+    val millis2 = toMillis(time2)
     val date1 = millisToDays(millis1, zoneId)
     val date2 = millisToDays(millis2, zoneId)
     val (year1, monthInYear1, dayInMonth1, daysToMonthEnd1) = splitDate(date1)
@@ -712,7 +712,7 @@ object DateTimeUtils {
       case TRUNC_TO_HOUR => truncToUnit(t, zoneId, ChronoUnit.HOURS)
       case TRUNC_TO_DAY => truncToUnit(t, zoneId, ChronoUnit.DAYS)
       case _ =>
-        val millis = MICROSECONDS.toMillis(t)
+        val millis = toMillis(t)
         val truncated = level match {
           case TRUNC_TO_MILLISECOND => millis
           case TRUNC_TO_SECOND =>
@@ -723,7 +723,7 @@ object DateTimeUtils {
             val dDays = millisToDays(millis, zoneId)
             daysToMillis(truncDate(dDays, level), zoneId)
         }
-        truncated * MICROS_PER_MILLIS
+        fromMillis(truncated)
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 2d98384363323..2790f8e998358 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit
 import scala.util.control.NonFatal
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.fromMillis
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -704,9 +705,7 @@ object IntervalUtils {
                   microseconds = Math.addExact(microseconds, minutesUs)
                   i += minuteStr.numBytes()
                 } else if (s.matchAt(millisStr, i)) {
-                  val millisUs = Math.multiplyExact(
-                    currentValue,
-                    MICROS_PER_MILLIS)
+                  val millisUs = fromMillis(currentValue)
                   microseconds = Math.addExact(microseconds, millisUs)
                   i += millisStr.numBytes()
                 } else if (s.matchAt(microsStr, i)) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 4893a7ec91cbb..fd77326d6b016 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -28,7 +28,7 @@ import java.util.concurrent.TimeUnit.SECONDS
 import org.apache.commons.lang3.time.FastDateFormat
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{ convertSpecialTimestamp, SQLTimestamp}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal
 
@@ -141,7 +141,7 @@ class LegacyFastTimestampFormatter(
     }
     val micros = cal.getMicros()
     cal.set(Calendar.MILLISECOND, 0)
-    cal.getTimeInMillis * MICROS_PER_MILLIS + micros
+    Math.addExact(fromMillis(cal.getTimeInMillis), micros)
   }
 
   def format(timestamp: SQLTimestamp): String = {
@@ -164,7 +164,7 @@ class LegacySimpleTimestampFormatter(
   }
 
   override def parse(s: String): Long = {
-    sdf.parse(s).getTime * MICROS_PER_MILLIS
+    fromMillis(sdf.parse(s).getTime)
   }
 
   override def format(us: Long): String = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index ad66873c02518..bde95f0e3add5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
 import java.util.{Calendar, TimeZone}
-import java.util.concurrent.TimeUnit._
 
 import scala.collection.parallel.immutable.ParVector
 
@@ -272,13 +271,13 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       checkEvaluation(
         cast(cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
           TimestampType, timeZoneId),
-        MILLISECONDS.toMicros(c.getTimeInMillis))
+        fromMillis(c.getTimeInMillis))
       c = Calendar.getInstance(TimeZoneGMT)
       c.set(2015, 10, 1, 2, 30, 0)
       checkEvaluation(
         cast(cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId),
           TimestampType, timeZoneId),
-        MILLISECONDS.toMicros(c.getTimeInMillis))
+        fromMillis(c.getTimeInMillis))
     }
 
     val gmtId = Option("GMT")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index a514c902f1d1d..f1ad75321776d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -21,7 +21,6 @@ import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
 import java.time.{Instant, LocalDate, LocalDateTime, ZoneId, ZoneOffset}
 import java.util.{Calendar, Locale, TimeZone}
-import java.util.concurrent.TimeUnit
 import java.util.concurrent.TimeUnit._
 
 import org.apache.spark.SparkFunSuite
@@ -48,7 +47,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   def toMillis(timestamp: String): Long = {
     val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
-    TimeUnit.MICROSECONDS.toMillis(tf.parse(timestamp))
+    DateTimeUtils.toMillis(tf.parse(timestamp))
   }
   val date = "2015-04-08 13:10:15"
   val d = new Date(toMillis(date))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index d31a0e210552d..bc5743811c975 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -21,6 +21,7 @@ import scala.language.implicitConversions
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils.{safeStringToInterval, stringToInterval}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.Decimal
@@ -260,7 +261,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         seconds: Int = 0,
         millis: Int = 0,
         micros: Int = 0): Unit = {
-      val secFrac = seconds * MICROS_PER_SECOND + millis * MICROS_PER_MILLIS + micros
+      val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
       val intervalExpr = MakeInterval(Literal(years), Literal(months), Literal(weeks),
         Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, 8, 6)))
       val totalMonths = years * MONTHS_PER_YEAR + months
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index df012ccf09620..e8beb612b5134 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, _}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{First, Last}
 import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, IntervalUtils}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -681,13 +680,13 @@ class ExpressionParserSuite extends AnalysisTest {
       Literal(new CalendarInterval(
         0,
         0,
-        -13 * MICROS_PER_SECOND - 123 * MICROS_PER_MILLIS - 456)))
+        DateTimeTestUtils.secFrac(-13, -123, -456))))
     checkIntervals(
       "13.123456 second",
       Literal(new CalendarInterval(
         0,
         0,
-        13 * MICROS_PER_SECOND + 123 * MICROS_PER_MILLIS + 456)))
+        DateTimeTestUtils.secFrac(13, 123, 456))))
     checkIntervals("1.001 second",
       Literal(IntervalUtils.stringToInterval("1 second 1 millisecond")))
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index 8d16007a18f7f..5f1428f96e2b0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -21,6 +21,8 @@ import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset}
 import java.util.TimeZone
 import java.util.concurrent.TimeUnit
 
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+
 /**
  * Helper functions for testing date and time functionality.
  */
@@ -95,4 +97,11 @@ object DateTimeTestUtils {
     val localDateTime = LocalDateTime.of(localDate, localTime)
     localDateTimeToMicros(localDateTime, zid)
   }
+
+  def secFrac(seconds: Int, milliseconds: Int, microseconds: Int): Long = {
+    var result: Long = microseconds
+    result = Math.addExact(result, Math.multiplyExact(milliseconds, MICROS_PER_MILLIS))
+    result = Math.addExact(result, Math.multiplyExact(seconds, MICROS_PER_SECOND))
+    result
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 6ed18ca1bedab..2e689be01f338 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -89,8 +89,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
   test("SPARK-6785: java date conversion before and after epoch") {
     def format(d: Date): String = {
-      TimestampFormatter("uuuu-MM-dd", defaultTimeZone().toZoneId)
-        .format(d.getTime * MICROS_PER_MILLIS)
+      TimestampFormatter("uuuu-MM-dd", defaultTimeZone().toZoneId).format(fromMillis(d.getTime))
     }
     def checkFromToJavaDate(d1: Date): Unit = {
       val d2 = toJavaDate(fromJavaDate(d1))
@@ -584,15 +583,15 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("daysToMillis and millisToDays") {
-    val input = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, 16, zid = zonePST))
+    val input = toMillis(date(2015, 12, 31, 16, zid = zonePST))
     assert(millisToDays(input, zonePST) === 16800)
     assert(millisToDays(input, ZoneOffset.UTC) === 16801)
     assert(millisToDays(-1 * MILLIS_PER_DAY + 1, ZoneOffset.UTC) == -1)
 
-    var expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, zid = zonePST))
+    var expected = toMillis(date(2015, 12, 31, zid = zonePST))
     assert(daysToMillis(16800, zonePST) === expected)
 
-    expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, zid = zoneGMT))
+    expected = toMillis(date(2015, 12, 31, zid = zoneGMT))
     assert(daysToMillis(16800, ZoneOffset.UTC) === expected)
 
     // There are some days are skipped entirely in some timezone, skip them here.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 514804cbda16c..d0d79acd73883 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.fromMillis
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
 import org.apache.spark.sql.internal.SQLConf
@@ -76,7 +77,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     testSingleUnit("HouR", 3, 0, 0, 3 * MICROS_PER_HOUR)
     testSingleUnit("MiNuTe", 3, 0, 0, 3 * MICROS_PER_MINUTE)
     testSingleUnit("Second", 3, 0, 0, 3 * MICROS_PER_SECOND)
-    testSingleUnit("MilliSecond", 3, 0, 0, 3 * MICROS_PER_MILLIS)
+    testSingleUnit("MilliSecond", 3, 0, 0, fromMillis(3))
     testSingleUnit("MicroSecond", 3, 0, 0, 3)
 
     checkFromInvalidString(null, "cannot be null")
@@ -175,7 +176,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
         new CalendarInterval(
           0,
           10,
-          12 * MICROS_PER_MINUTE + 888 * MICROS_PER_MILLIS))
+          12 * MICROS_PER_MINUTE + fromMillis(888)))
       assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3, 0L))
 
       try {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
index eac5246904ffd..7dd52c1feabf2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/EventTimeWatermarkExec.scala
@@ -21,7 +21,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_MILLIS
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.toMillis
 import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
 import org.apache.spark.sql.types.MetadataBuilder
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -100,7 +100,7 @@ case class EventTimeWatermarkExec(
     child.execute().mapPartitions { iter =>
       val getEventTime = UnsafeProjection.create(eventTime :: Nil, child.output)
       iter.map { row =>
-        eventTimeStats.add(getEventTime(row).getLong(0) / MICROS_PER_MILLIS)
+        eventTimeStats.add(toMillis(getEventTime(row).getLong(0)))
         row
       }
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
index 1a27fe61d9602..d40208f02e7b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit
 import scala.concurrent.duration.Duration
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.toMillis
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.streaming.Trigger
 import org.apache.spark.unsafe.types.UTF8String
@@ -36,7 +37,8 @@ private object Triggers {
     if (cal.months != 0) {
       throw new IllegalArgumentException(s"Doesn't support month or year interval: $interval")
     }
-    TimeUnit.MICROSECONDS.toMillis(cal.microseconds + cal.days * MICROS_PER_DAY)
+    val microsInDays = Math.multiplyExact(cal.days, MICROS_PER_DAY)
+    toMillis(Math.addExact(cal.microseconds, microsInDays))
   }
 
   def convert(interval: Duration): Long = interval.toMillis
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
index fde8ddf491bd1..b6ea26ab95549 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionTestBase.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql
 import java.{lang => jl}
 import java.io.File
 import java.sql.{Date, Timestamp}
-import java.util.concurrent.TimeUnit
 
 import scala.collection.mutable
 import scala.util.Random
@@ -30,6 +29,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatisti
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, Histogram, HistogramBin, HistogramSerializer, LogicalPlan}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.test.SQLTestUtils
@@ -51,10 +51,10 @@ abstract class StatisticsCollectionTestBase extends QueryTest with SQLTestUtils
   private val d2 = Date.valueOf(d2Str)
   private val t1Str = "2016-05-08 00:00:01.000000"
   private val t1Internal = date(2016, 5, 8, 0, 0, 1)
-  private val t1 = new Timestamp(TimeUnit.MICROSECONDS.toMillis(t1Internal))
+  private val t1 = new Timestamp(DateTimeUtils.toMillis(t1Internal))
   private val t2Str = "2016-05-09 00:00:02.000000"
   private val t2Internal = date(2016, 5, 9, 0, 0, 2)
-  private val t2 = new Timestamp(TimeUnit.MICROSECONDS.toMillis(t2Internal))
+  private val t2 = new Timestamp(DateTimeUtils.toMillis(t2Internal))
 
   /**
    * Define a very simple 3 row table used for testing column serialization.

From c9dc06fb45161622b3e8ffd8d57a1e3c14f09c92 Mon Sep 17 00:00:00 2001
From: jiake <ke.a.jia@intel.com>
Date: Mon, 24 Feb 2020 14:29:25 +0800
Subject: [PATCH 0145/1280] [SPARK-30922][SQL] remove the max splits config in
 skewed join

### What changes were proposed in this pull request?
When skewed join optimization split more skewed readers, the plan may be very large and can not be shown in ui quickly. The config `spark.sql.adaptive.skewedJoinOptimization.skewedPartitionMaxSplits`  is to resolve the above ui shown issue. And after [PR#27493](https://github.com/apache/spark/pull/27493) combined the skewed readers into one, we not need this config.

### Why are the changes needed?
remove the unnecessary config

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
existing test

Closes #27673 from JkSelf/removeMaxSplitNum.

Authored-by: jiake <ke.a.jia@intel.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f4696ba252faf2ed32dca015abffefc8abad41bb)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  8 -----
 .../adaptive/OptimizeSkewedJoin.scala         |  8 ++---
 .../adaptive/AdaptiveQueryExecSuite.scala     | 31 ++++++++++---------
 3 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 72480e0915408..529749747b205 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -447,14 +447,6 @@ object SQLConf {
       .intConf
       .createWithDefault(10)
 
-  val ADAPTIVE_EXECUTION_SKEWED_PARTITION_MAX_SPLITS =
-    buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionMaxSplits")
-      .doc("Configures the maximum number of task to handle a skewed partition in adaptive skewed" +
-        "join.")
-      .intConf
-      .checkValue( _ >= 1, "The split size at least be 1")
-      .createWithDefault(5)
-
   val NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN =
     buildConf("spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin")
       .doc("The relation with a non-empty partition ratio lower than this config will not be " +
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index a716497c274b8..578d2d744f85c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -75,9 +75,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   private def getMapStartIndices(stage: ShuffleQueryStageExec, partitionId: Int): Array[Int] = {
     val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
     val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
-    val maxSplits = math.min(conf.getConf(
-      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_MAX_SPLITS), mapPartitionSizes.length)
-    val avgPartitionSize = mapPartitionSizes.sum / maxSplits
+    val avgPartitionSize = mapPartitionSizes.sum / mapPartitionSizes.length
     val advisoryPartitionSize = math.max(avgPartitionSize,
       conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD))
     val partitionStartIndices = ArrayBuffer[Int]()
@@ -95,9 +93,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       i += 1
     }
 
-    if (partitionStartIndices.size > maxSplits) {
-      partitionStartIndices.take(maxSplits).toArray
-    } else partitionStartIndices.toArray
+    partitionStartIndices.toArray
   }
 
   private def getStatistics(stage: ShuffleQueryStageExec): MapOutputStatistics = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 4edb35ea30fde..379e17f01d6be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -609,7 +609,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "100",
+      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "2000",
       SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
       withTempView("skewData1", "skewData2") {
         spark
@@ -636,14 +636,15 @@ class AdaptiveQueryExecSuite
           "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
         // left stats: [3496, 0, 0, 0, 4014]
         // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, and divide into 5 splits, so
-        //              5 x 5 sub-partitions.
+        // Partition 0: both left and right sides are skewed, left side is divided
+        //              into 2 splits and right side is divided into 4 splits, so
+        //              2 x 4 sub-partitions.
         // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, and divide into 5 splits, so
-        //              5 sub-partitions.
-        // So total (25 + 1 + 5) partitions.
+        // Partition 4: only left side is skewed, and divide into 3 splits, so
+        //              3 sub-partitions.
+        // So total (8 + 1 + 3) partitions.
         val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        checkSkewJoin(innerSmj, 25 + 1 + 5)
+        checkSkewJoin(innerSmj, 8 + 1 + 3)
 
         // skewed left outer join optimization
         val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
@@ -651,13 +652,13 @@ class AdaptiveQueryExecSuite
         // left stats: [3496, 0, 0, 0, 4014]
         // right stats:[6292, 0, 0, 0, 0]
         // Partition 0: both left and right sides are skewed, but left join can't split right side,
-        //              so only left side is divided into 5 splits, and thus 5 sub-partitions.
+        //              so only left side is divided into 2 splits, and thus 2 sub-partitions.
         // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, and divide into 5 splits, so
-        //              5 sub-partitions.
-        // So total (5 + 1 + 5) partitions.
+        // Partition 4: only left side is skewed, and divide into 3 splits, so
+        //              3 sub-partitions.
+        // So total (2 + 1 + 3) partitions.
         val leftSmj = findTopLevelSortMergeJoin(leftAdaptivePlan)
-        checkSkewJoin(leftSmj, 5 + 1 + 5)
+        checkSkewJoin(leftSmj, 2 + 1 + 3)
 
         // skewed right outer join optimization
         val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
@@ -665,13 +666,13 @@ class AdaptiveQueryExecSuite
         // left stats: [3496, 0, 0, 0, 4014]
         // right stats:[6292, 0, 0, 0, 0]
         // Partition 0: both left and right sides are skewed, but right join can't split left side,
-        //              so only right side is divided into 5 splits, and thus 5 sub-partitions.
+        //              so only right side is divided into 4 splits, and thus 4 sub-partitions.
         // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
         // Partition 4: only left side is skewed, but right join can't split left side, so just
         //              1 partition.
-        // So total (5 + 1 + 1) partitions.
+        // So total (4 + 1 + 1) partitions.
         val rightSmj = findTopLevelSortMergeJoin(rightAdaptivePlan)
-        checkSkewJoin(rightSmj, 5 + 1 + 1)
+        checkSkewJoin(rightSmj, 4 + 1 + 1)
       }
     }
   }

From d164ee3d44669ed717497b402776d567636d1f71 Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Mon, 24 Feb 2020 15:16:37 +0800
Subject: [PATCH 0146/1280] [SPARK-30924][SQL][3.0] Add additional checks to
 Merge Into

### What changes were proposed in this pull request?

Merge Into is currently missing additional validation around:

 1. The lack of any WHEN statements
 2. The first WHEN MATCHED statement needs to have a condition if there are two WHEN MATCHED statements.
 3. Single use of UPDATE/DELETE

This PR introduces these validations.
(1) is required, because otherwise the MERGE statement is useless.
(2) is required, because otherwise the second WHEN MATCHED condition becomes dead code
(3) is up for debate, but the idea there is that a single expression should be sufficient to specify when you would like to update or delete your records. We restrict it for now to reduce surface area and ambiguity.

### Why are the changes needed?

To ease DataSource developers when building implementations for MERGE

### Does this PR introduce any user-facing change?

Adds additional validation checks

### How was this patch tested?

Unit tests

Closes #27677 from brkyvz/mergeChecks.

Authored-by: Burak Yavuz <brkyvz@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 4ff2718d5448f212c0ede885c4072790c43bf826)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/parser/AstBuilder.scala      | 12 +++++
 .../sql/catalyst/parser/DDLParserSuite.scala  | 50 +++++++++++++++++++
 .../command/PlanResolutionSuite.scala         | 10 ++--
 3 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 36c16479e6be0..2806fd49fbc5e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -460,6 +460,18 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         }
       }
     }
+    if (matchedActions.isEmpty && notMatchedActions.isEmpty) {
+      throw new ParseException("There must be at least one WHEN clause in a MERGE statement", ctx)
+    }
+    // children being empty means that the condition is not set
+    if (matchedActions.length == 2 && matchedActions.head.children.isEmpty) {
+      throw new ParseException("When there are 2 MATCHED clauses in a MERGE statement, " +
+        "the first MATCHED clause must have a condition", ctx)
+    }
+    if (matchedActions.groupBy(_.getClass).mapValues(_.size).exists(_._2 > 1)) {
+      throw new ParseException(
+        "UPDATE and DELETE can appear at most once in MATCHED clauses in a MERGE statement", ctx)
+    }
 
     MergeIntoTable(
       aliasedTarget,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index bc7b51f25b20d..4a636bd57f42f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -1144,6 +1144,56 @@ class DDLParserSuite extends AnalysisTest {
     assert(exc.getMessage.contains("There should be at most 1 'WHEN NOT MATCHED' clause."))
   }
 
+  test("merge into table: the first matched clause must have a condition if there's a second") {
+    val exc = intercept[ParseException] {
+      parsePlan(
+        """
+          |MERGE INTO testcat1.ns1.ns2.tbl AS target
+          |USING testcat2.ns1.ns2.tbl AS source
+          |ON target.col1 = source.col1
+          |WHEN MATCHED THEN DELETE
+          |WHEN MATCHED THEN UPDATE SET target.col2 = source.col2
+          |WHEN NOT MATCHED AND (target.col2='insert')
+          |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
+        """.stripMargin)
+    }
+
+    assert(exc.getMessage.contains("the first MATCHED clause must have a condition"))
+  }
+
+  test("merge into table: there must be a when (not) matched condition") {
+    val exc = intercept[ParseException] {
+      parsePlan(
+        """
+          |MERGE INTO testcat1.ns1.ns2.tbl AS target
+          |USING testcat2.ns1.ns2.tbl AS source
+          |ON target.col1 = source.col1
+        """.stripMargin)
+    }
+
+    assert(exc.getMessage.contains("There must be at least one WHEN clause in a MERGE statement"))
+  }
+
+  test("merge into table: there can be only a single use DELETE or UPDATE") {
+    Seq("UPDATE SET *", "DELETE").foreach { op =>
+      val exc = intercept[ParseException] {
+        parsePlan(
+          s"""
+             |MERGE INTO testcat1.ns1.ns2.tbl AS target
+             |USING testcat2.ns1.ns2.tbl AS source
+             |ON target.col1 = source.col1
+             |WHEN MATCHED AND (target.col2='delete') THEN $op
+             |WHEN MATCHED THEN $op
+             |WHEN NOT MATCHED AND (target.col2='insert')
+             |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
+           """.stripMargin)
+      }
+
+      assert(exc.getMessage.contains(
+        "UPDATE and DELETE can appear at most once in MATCHED clauses"))
+    }
+  }
+
   test("show tables") {
     comparePlans(
       parsePlan("SHOW TABLES"),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 21c698c4a2c4d..01d16ce0bf285 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -1277,7 +1277,7 @@ class PlanResolutionSuite extends AnalysisTest {
              |MERGE INTO $target AS target
              |USING $source AS source
              |ON target.i = source.i
-             |WHEN MATCHED THEN DELETE
+             |WHEN MATCHED AND (target.s='delete') THEN DELETE
              |WHEN MATCHED THEN UPDATE SET target.s = source.s
              |WHEN NOT MATCHED THEN INSERT (target.i, target.s) values (source.i, source.s)
            """.stripMargin
@@ -1286,7 +1286,7 @@ class PlanResolutionSuite extends AnalysisTest {
               SubqueryAlias(AliasIdentifier("target", Seq()), AsDataSourceV2Relation(target)),
               SubqueryAlias(AliasIdentifier("source", Seq()), AsDataSourceV2Relation(source)),
               mergeCondition,
-              Seq(DeleteAction(None), UpdateAction(None, updateAssigns)),
+              Seq(DeleteAction(Some(_)), UpdateAction(None, updateAssigns)),
               Seq(InsertAction(None, insertAssigns))) =>
             checkResolution(target, source, mergeCondition, None, None, None,
               updateAssigns, insertAssigns)
@@ -1364,7 +1364,7 @@ class PlanResolutionSuite extends AnalysisTest {
            |MERGE INTO $target
            |USING $source
            |ON 1 = 1
-           |WHEN MATCHED THEN DELETE
+           |WHEN MATCHED AND (${target}.s='delete') THEN DELETE
            |WHEN MATCHED THEN UPDATE SET s = 1
            |WHEN NOT MATCHED AND (s = 'a') THEN INSERT (i) values (i)
          """.stripMargin
@@ -1374,7 +1374,7 @@ class PlanResolutionSuite extends AnalysisTest {
             AsDataSourceV2Relation(target),
             AsDataSourceV2Relation(source),
             _,
-            Seq(DeleteAction(None), UpdateAction(None, updateAssigns)),
+            Seq(DeleteAction(Some(_)), UpdateAction(None, updateAssigns)),
             Seq(InsertAction(
               Some(EqualTo(il: AttributeReference, StringLiteral("a"))),
               insertAssigns))) =>
@@ -1451,7 +1451,7 @@ class PlanResolutionSuite extends AnalysisTest {
          |MERGE INTO non_exist_target
          |USING non_exist_source
          |ON target.i = source.i
-         |WHEN MATCHED THEN DELETE
+         |WHEN MATCHED AND (non_exist_target.s='delete') THEN DELETE
          |WHEN MATCHED THEN UPDATE SET *
          |WHEN NOT MATCHED THEN INSERT *
        """.stripMargin

From 089ef4f91db30be2466a620f9c478a379163e5a2 Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Mon, 24 Feb 2020 16:47:08 +0900
Subject: [PATCH 0147/1280] [SPARK-30897][SQL] The behavior of ArrayExists
 should not depend on SQLConf.get

### What changes were proposed in this pull request?
This PR adds a new `followThreeValuedLogic` parameter to `ArrayExists` so as to avoid its value depending on `SQLConf.get` and change during planning.

### Why are the changes needed?
This allows to avoid the issue when the configuration change between different phases of planning, and this can silently break a query plan which can lead to crashes or data corruption.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UTs.

Closes #27655 from peter-toth/SPARK-30897.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 612f63f39ef91a06b71d40e20ad0216cb88776a1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../expressions/higherOrderFunctions.scala      | 17 ++++++++++++++---
 .../ReplaceNullWithFalseInPredicate.scala       |  3 +--
 .../ReplaceNullWithFalseInPredicateSuite.scala  |  2 +-
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index f8142d6b993cb..9dd4263fdeae3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -522,11 +522,16 @@ case class ArrayFilter(
   since = "2.4.0")
 case class ArrayExists(
     argument: Expression,
-    function: Expression)
+    function: Expression,
+    followThreeValuedLogic: Boolean)
   extends ArrayBasedSimpleHigherOrderFunction with CodegenFallback {
 
-  private val followThreeValuedLogic =
-    SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC)
+  def this(argument: Expression, function: Expression) = {
+    this(
+      argument,
+      function,
+      SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC))
+  }
 
   override def nullable: Boolean =
     if (followThreeValuedLogic) {
@@ -574,6 +579,12 @@ case class ArrayExists(
   override def prettyName: String = "exists"
 }
 
+object ArrayExists {
+  def apply(argument: Expression, function: Expression): ArrayExists = {
+    new ArrayExists(argument, function)
+  }
+}
+
 /**
  * Tests whether a predicate holds for all elements in the array.
  */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
index b8edf985dbda3..33b398e11cde9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicate.scala
@@ -64,8 +64,7 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
       case af @ ArrayFilter(_, lf @ LambdaFunction(func, _, _)) =>
         val newLambda = lf.copy(function = replaceNullWithFalse(func))
         af.copy(function = newLambda)
-      case ae @ ArrayExists(_, lf @ LambdaFunction(func, _, _))
-          if !SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC) =>
+      case ae @ ArrayExists(_, lf @ LambdaFunction(func, _, _), false) =>
         val newLambda = lf.copy(function = replaceNullWithFalse(func))
         ae.copy(function = newLambda)
       case mf @ MapFilter(_, lf @ LambdaFunction(func, _, _)) =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
index b692c3fee53c7..c7f42f08406e7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
@@ -324,7 +324,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
       testProjection(originalExpr = expr, expectedExpr = expr)
     }
     withSQLConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC.key -> "false") {
-      testHigherOrderFunc('a, ArrayExists, Seq(lv('e)))
+      testHigherOrderFunc('a, ArrayExists.apply, Seq(lv('e)))
     }
   }
 

From cfbcc58639d9ba1e5eb96642bc8d335dfa415dfb Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Mon, 24 Feb 2020 16:48:48 +0900
Subject: [PATCH 0148/1280] [SPARK-30898][SQL] The behavior of MakeDecimal
 should not depend on SQLConf.get

### What changes were proposed in this pull request?
This PR adds a new `nullOnOverflow` parameter to `MakeDecimal` so as to avoid its value depending on `SQLConf.get` and change during planning.

### Why are the changes needed?
This allows to avoid the issue when the configuration change between different phases of planning, and this can silently break a query plan which can lead to crashes or data corruption.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UTs.

Closes #27656 from peter-toth/SPARK-30898.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a372f76cbdfe1fb6b87f0ad7b1ced22604406f1a)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../expressions/decimalExpressions.scala         | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
index 7b2489e682661..9014ebfe2f96a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/decimalExpressions.scala
@@ -45,9 +45,15 @@ case class UnscaledValue(child: Expression) extends UnaryExpression {
  * Note: this expression is internal and created only by the optimizer,
  * we don't need to do type check for it.
  */
-case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends UnaryExpression {
+case class MakeDecimal(
+    child: Expression,
+    precision: Int,
+    scale: Int,
+    nullOnOverflow: Boolean) extends UnaryExpression {
 
-  private val nullOnOverflow = !SQLConf.get.ansiEnabled
+  def this(child: Expression, precision: Int, scale: Int) = {
+    this(child, precision, scale, !SQLConf.get.ansiEnabled)
+  }
 
   override def dataType: DataType = DecimalType(precision, scale)
   override def nullable: Boolean = child.nullable || nullOnOverflow
@@ -83,6 +89,12 @@ case class MakeDecimal(child: Expression, precision: Int, scale: Int) extends Un
   }
 }
 
+object MakeDecimal {
+  def apply(child: Expression, precision: Int, scale: Int): MakeDecimal = {
+    new MakeDecimal(child, precision, scale)
+  }
+}
+
 /**
  * An expression used to wrap the children when promote the precision of DecimalType to avoid
  * promote multiple times.

From d87febf5945b46eb70330325c2b9a91b43177cac Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Mon, 24 Feb 2020 18:48:19 +0900
Subject: [PATCH 0149/1280] [SPARK-30927][SS] StreamingQueryManager should
 avoid keeping reference to terminated StreamingQuery

### What changes were proposed in this pull request?

Right now `StreamingQueryManager` will keep the last terminated query until `resetTerminated` is called. When the last terminated query has lots of states (a large sql plan, cached RDDs, etc.), it will keep a lot of memory unnecessarily. Actually, what `StreamingQueryManager` really needs is just the exception of the last failed query.

This PR changes the internal field `lastTerminatedQuery` in `StreamingQueryManager` to remember the last exception rather than the query to save the memory.

### Why are the changes needed?

Avoid keeping memory unnecessarily.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

This PR doesn't change any public behaviors. The existing tests have covered the touched codes.

Closes #27678 from zsxwing/SPARK-30927.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 293e5364e570e8a6569e47e58918db2f1d45ddff)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/streaming/StreamingQueryManager.scala | 31 ++++++++++++-------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
index 4d0d8ffd959c6..0fe2d0be966d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryManager.scala
@@ -58,8 +58,17 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
   private val activeQueriesSharedLock = sparkSession.sharedState.activeQueriesLock
   private val awaitTerminationLock = new Object
 
+  /**
+   * Track the last terminated query and remember the last failure since the creation of the
+   * context, or since `resetTerminated()` was called. There are three possible values:
+   *
+   * - null: no query has been been terminated.
+   * - None: some queries have been terminated and no one has failed.
+   * - Some(StreamingQueryException): Some queries have been terminated and at least one query has
+   *   failed. The exception is the exception of the last failed query.
+   */
   @GuardedBy("awaitTerminationLock")
-  private var lastTerminatedQuery: StreamingQuery = null
+  private var lastTerminatedQueryException: Option[StreamingQueryException] = null
 
   try {
     sparkSession.sparkContext.conf.get(STREAMING_QUERY_LISTENERS).foreach { classNames =>
@@ -125,11 +134,11 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
   @throws[StreamingQueryException]
   def awaitAnyTermination(): Unit = {
     awaitTerminationLock.synchronized {
-      while (lastTerminatedQuery == null) {
+      while (lastTerminatedQueryException == null) {
         awaitTerminationLock.wait(10)
       }
-      if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
-        throw lastTerminatedQuery.exception.get
+      if (lastTerminatedQueryException != null && lastTerminatedQueryException.nonEmpty) {
+        throw lastTerminatedQueryException.get
       }
     }
   }
@@ -164,13 +173,13 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
     }
 
     awaitTerminationLock.synchronized {
-      while (!isTimedout && lastTerminatedQuery == null) {
+      while (!isTimedout && lastTerminatedQueryException == null) {
         awaitTerminationLock.wait(10)
       }
-      if (lastTerminatedQuery != null && lastTerminatedQuery.exception.nonEmpty) {
-        throw lastTerminatedQuery.exception.get
+      if (lastTerminatedQueryException != null && lastTerminatedQueryException.nonEmpty) {
+        throw lastTerminatedQueryException.get
       }
-      lastTerminatedQuery != null
+      lastTerminatedQueryException != null
     }
   }
 
@@ -182,7 +191,7 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
    */
   def resetTerminated(): Unit = {
     awaitTerminationLock.synchronized {
-      lastTerminatedQuery = null
+      lastTerminatedQueryException = null
     }
   }
 
@@ -422,8 +431,8 @@ class StreamingQueryManager private[sql] (sparkSession: SparkSession) extends Lo
   private[sql] def notifyQueryTermination(terminatedQuery: StreamingQuery): Unit = {
     unregisterTerminatedStream(terminatedQuery)
     awaitTerminationLock.synchronized {
-      if (lastTerminatedQuery == null || terminatedQuery.exception.nonEmpty) {
-        lastTerminatedQuery = terminatedQuery
+      if (lastTerminatedQueryException == null || terminatedQuery.exception.nonEmpty) {
+        lastTerminatedQueryException = terminatedQuery.exception
       }
       awaitTerminationLock.notifyAll()
     }

From 1fedafdd1de9234a91b532f1e3cbeef4e8878f24 Mon Sep 17 00:00:00 2001
From: lijunqing <lijunqing@baidu.com>
Date: Mon, 24 Feb 2020 15:42:45 +0800
Subject: [PATCH 0150/1280] [SPARK-30868][SQL] Throw Exception if runHive(sql)
 failed

### Why are the changes needed?
At present, HiveClientImpl.runHive will not throw an exception when it runs incorrectly, which will cause it to fail to feedback error information normally.
Example
```scala
spark.sql("add jar file:///tmp/not_exists.jar")
spark.sql("show databases").show()
```
/tmp/not_exists.jar doesn't exist, thus add jar is failed. However this code will run completely without causing application failure.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
add new suite tests

Closes #27644 from stczwd/SPARK-30868.

Authored-by: lijunqing <lijunqing@baidu.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/hive/client/HiveClientImpl.scala      |  7 +++++-
 .../sql/hive/HiveExternalCatalogSuite.scala   | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index ab4db0cf36242..f2c516ec0a8dc 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -839,7 +839,12 @@ private[hive] class HiveClientImpl(
             state.out.println(tokens(0) + " " + cmd_1)
             // scalastyle:on println
           }
-          Seq(proc.run(cmd_1).getResponseCode.toString)
+          val response: CommandProcessorResponse = proc.run(cmd_1)
+          // Throw an exception if there is an error in query processing.
+          if (response.getResponseCode != 0) {
+            throw new QueryExecutionException(response.getErrorMessage)
+          }
+          Seq(response.getResponseCode.toString)
       }
     } catch {
       case e: Exception =>
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
index 79e569b51ca1d..0a8889885df7b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog._
+import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.types.StructType
 
@@ -154,4 +155,27 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite {
     catalog.createTable(hiveTable, ignoreIfExists = false)
     assert(catalog.getTable("db1", "spark_29498").owner === owner)
   }
+
+  test("SPARK-30868 throw an exception if HiveClient#runSqlHive fails") {
+    val client = externalCatalog.client
+    // test add jars which doesn't exists
+    val jarPath = "file:///tmp/not_exists.jar"
+    assertThrows[QueryExecutionException](client.runSqlHive(s"ADD JAR $jarPath"))
+
+    // test change to the database which doesn't exists
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      s"use db_not_exists"))
+
+    // test create hive table failed with unsupported into type
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      s"CREATE TABLE t(n into)"))
+
+    // test desc table failed with wrong `FORMATED` keyword
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      s"DESC FORMATED t"))
+
+    // test wrong insert query
+    assertThrows[QueryExecutionException](client.runSqlHive(
+      "INSERT overwrite directory \"fs://localhost/tmp\" select 1 as a"))
+  }
 }

From 476b02b4df063569ffe24e7e7fdf75759a3b0677 Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Mon, 24 Feb 2020 13:46:21 -0800
Subject: [PATCH 0151/1280] [SPARK-30870][SQL] Column pruning shouldn't alias a
 nested column if it means the whole structure

### What changes were proposed in this pull request?
This PR fixes a bug in nested column aliasing by taking the data type of the referenced nested fields into account when calculating the number of extracted columns. After this PR this query runs without issues:
```
SELECT explodedvalue.*
FROM VALUES array(named_struct('nested', named_struct('a', 1, 'b', 2))) AS (value)
LATERAL VIEW explode(value) AS explodedvalue
```
This is a regression from Spark 2.4.

### Why are the changes needed?
To fix a bug.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Added new UT.

Closes #27675 from peter-toth/SPARK-30870.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 1a4e2423b284d8382fcf9547d4351b20d7a2be56)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/optimizer/NestedColumnAliasing.scala |  4 +++-
 .../optimizer/NestedColumnAliasingSuite.scala         |  7 +------
 .../scala/org/apache/spark/sql/SQLQuerySuite.scala    | 11 +++++++++++
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index 43a6006f9b5c0..46e07f3843925 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -129,7 +129,9 @@ object NestedColumnAliasing {
         // If all nested fields of `attr` are used, we don't need to introduce new aliases.
         // By default, ColumnPruning rule uses `attr` already.
         if (nestedFieldToAlias.nonEmpty &&
-            nestedFieldToAlias.length < totalFieldNum(attr.dataType)) {
+            nestedFieldToAlias
+              .map { case (nestedField, _) => totalFieldNum(nestedField.dataType) }
+              .sum < totalFieldNum(attr.dataType)) {
           Some(attr.exprId -> nestedFieldToAlias)
         } else {
           None
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
index 2351d8321c5f3..a9267379e12fd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasingSuite.scala
@@ -215,12 +215,7 @@ class NestedColumnAliasingSuite extends SchemaPruningTest {
 
     val optimized = Optimize.execute(query)
 
-    val expected = nestedRelation
-      .select(GetStructField('a, 0, Some("b")))
-      .limit(5)
-      .analyze
-
-    comparePlans(optimized, expected)
+    comparePlans(optimized, query)
   }
 
   test("nested field pruning for getting struct field in array of struct") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index d6efb2f91ca0b..c87e6a4e39b10 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3393,6 +3393,17 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       )
     }
   }
+
+  test("SPARK-30870: Column pruning shouldn't alias a nested column if it means the whole " +
+    "structure") {
+    val df = sql(
+      """
+        |SELECT explodedvalue.field
+        |FROM VALUES array(named_struct('field', named_struct('a', 1, 'b', 2))) AS (value)
+        |LATERAL VIEW explode(value) AS explodedvalue
+      """.stripMargin)
+    checkAnswer(df, Row(Row(1, 2)) :: Nil)
+  }
 }
 
 case class Foo(bar: Option[String])

From 9666cf10aac873ad17e95074da5e1a78c8b64b47 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Wed, 12 Feb 2020 14:49:22 +0800
Subject: [PATCH 0152/1280] [SPARK-30763][SQL] Fix
 java.lang.IndexOutOfBoundsException No group 1 for regexp_extract

### What changes were proposed in this pull request?
The current implement of `regexp_extract` will throws a unprocessed exception show below:

`SELECT regexp_extract('1a 2b 14m', 'd+')`
```
java.lang.IndexOutOfBoundsException: No group 1
[info] at java.util.regex.Matcher.group(Matcher.java:538)
[info] at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
[info] at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
[info] at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:729)
```
I think should treat this exception well.

### Why are the changes needed?
Fix a bug `java.lang.IndexOutOfBoundsException No group 1 `

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
New UT

Closes #27508 from beliefer/fix-regexp_extract-bug.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/regexpExpressions.scala       | 15 +++-
 .../expressions/RegexpExpressionsSuite.scala  | 12 ++++
 .../sql-tests/inputs/regexp-functions.sql     |  9 +++
 .../results/regexp-functions.sql.out          | 69 +++++++++++++++++++
 4 files changed, 104 insertions(+), 1 deletion(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index ac620b10cfd2e..7891c2868e366 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -409,6 +409,15 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
   }
 }
 
+object RegExpExtract {
+  def checkGroupIndex(groupCount: Int, groupIndex: Int): Unit = {
+    if (groupCount < groupIndex) {
+      throw new IllegalArgumentException(
+        s"Regex group count is $groupCount, but the specified group index is $groupIndex")
+    }
+  }
+}
+
 /**
  * Extract a specific(idx) group identified by a Java regex.
  *
@@ -440,7 +449,9 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
     val m = pattern.matcher(s.toString)
     if (m.find) {
       val mr: MatchResult = m.toMatchResult
-      val group = mr.group(r.asInstanceOf[Int])
+      val index = r.asInstanceOf[Int]
+      RegExpExtract.checkGroupIndex(mr.groupCount, index)
+      val group = mr.group(index)
       if (group == null) { // Pattern matched, but not optional group
         UTF8String.EMPTY_UTF8
       } else {
@@ -458,6 +469,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val classNamePattern = classOf[Pattern].getCanonicalName
+    val classNameRegExpExtract = classOf[RegExpExtract].getCanonicalName
     val matcher = ctx.freshName("matcher")
     val matchResult = ctx.freshName("matchResult")
 
@@ -481,6 +493,7 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
         $termPattern.matcher($subject.toString());
       if ($matcher.find()) {
         java.util.regex.MatchResult $matchResult = $matcher.toMatchResult();
+        $classNameRegExpExtract.checkGroupIndex($matchResult.groupCount(), $idx);
         if ($matchResult.group($idx) == null) {
           ${ev.value} = UTF8String.EMPTY_UTF8;
         } else {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 2c8794f083dbb..86da62bc74940 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -293,6 +293,18 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     val nonNullExpr = RegExpExtract(Literal("100-200"), Literal("(\\d+)-(\\d+)"), Literal(1))
     checkEvaluation(nonNullExpr, "100", row1)
+
+    // invalid group index
+    val row8 = create_row("100-200", "(\\d+)-(\\d+)", 3)
+    val row9 = create_row("100-200", "(\\d+).*", 2)
+    val row10 = create_row("100-200", "\\d+", 1)
+
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row8, "Regex group count is 2, but the specified group index is 3")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row9, "Regex group count is 1, but the specified group index is 2")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row10, "Regex group count is 0, but the specified group index is 1")
   }
 
   test("SPLIT") {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
new file mode 100644
index 0000000000000..c0827a3cba39b
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
@@ -0,0 +1,9 @@
+-- regexp_extract
+SELECT regexp_extract('1a 2b 14m', '\\d+');
+SELECT regexp_extract('1a 2b 14m', '\\d+', 0);
+SELECT regexp_extract('1a 2b 14m', '\\d+', 1);
+SELECT regexp_extract('1a 2b 14m', '\\d+', 2);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)');
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 0);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 1);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2);
diff --git a/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
new file mode 100644
index 0000000000000..c92c1ddca774f
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
@@ -0,0 +1,69 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 8
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', 0)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, \d+, 0):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', 1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', 2)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Regex group count is 0, but the specified group index is 2
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)')
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 1):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 0)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 0):string>
+-- !query output
+1a
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 1)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 1):string>
+-- !query output
+1
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2)
+-- !query schema
+struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 2):string>
+-- !query output
+a

From 1b9dbcf697c0d49534a200b729ffd1d088407fc6 Mon Sep 17 00:00:00 2001
From: XU Duo <Duo.XU@canal-plus.com>
Date: Mon, 24 Feb 2020 20:16:00 -0600
Subject: [PATCH 0153/1280] [SPARK-30901][DOCS] Fix doc exemple with deprecated
 codes

### What changes were proposed in this pull request?

Previous exemple given for spark-streaming-kinesis was true for Apache Spark < 2.3.0. After that the method used in exemple became deprecated:
deprecated("use initialPosition(initialPosition: KinesisInitialPosition)", "2.3.0")
def initialPositionInStream(initialPosition: InitialPositionInStream)

This PR updates the doc on rewriting exemple in Scala/Java (remain unchanged in Python) to adapt Apache Spark 2.4.0 + releases.

### Why are the changes needed?

It introduces some confusion for developers to test their spark-streaming-kinesis exemple.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

In my opinion, the change is only about the documentation level, so I did not add any special test.

Closes #27652 from supaggregator/SPARK-30901.

Authored-by: XU Duo <Duo.XU@canal-plus.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 10fa71321f607cb35c33d9b574595b27efcb7633)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/streaming-kinesis-integration.md | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index c81597160ca09..e68d51321cae3 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -46,14 +46,14 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
             import org.apache.spark.storage.StorageLevel
             import org.apache.spark.streaming.kinesis.KinesisInputDStream
             import org.apache.spark.streaming.{Seconds, StreamingContext}
-            import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream
+            import org.apache.spark.streaming.kinesis.KinesisInitialPositions
 
             val kinesisStream = KinesisInputDStream.builder
                 .streamingContext(streamingContext)
                 .endpointUrl([endpoint URL])
                 .regionName([region name])
                 .streamName([streamName])
-                .initialPositionInStream([initial position])
+                .initialPosition([initial position])
                 .checkpointAppName([Kinesis app name])
                 .checkpointInterval([checkpoint interval])
                 .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -68,14 +68,14 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
             import org.apache.spark.streaming.kinesis.KinesisInputDStream;
             import org.apache.spark.streaming.Seconds;
             import org.apache.spark.streaming.StreamingContext;
-            import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
+            import org.apache.spark.streaming.kinesis.KinesisInitialPositions;
 
             KinesisInputDStream<byte[]> kinesisStream = KinesisInputDStream.builder()
                 .streamingContext(streamingContext)
                 .endpointUrl([endpoint URL])
                 .regionName([region name])
                 .streamName([streamName])
-                .initialPositionInStream([initial position])
+                .initialPosition([initial position])
                 .checkpointAppName([Kinesis app name])
                 .checkpointInterval([checkpoint interval])
                 .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -110,7 +110,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                 import org.apache.spark.storage.StorageLevel
                 import org.apache.spark.streaming.kinesis.KinesisInputDStream
                 import org.apache.spark.streaming.{Seconds, StreamingContext}
-                import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream
+                import org.apache.spark.streaming.kinesis.KinesisInitialPositions
                 import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration
                 import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel
 
@@ -119,7 +119,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                     .endpointUrl([endpoint URL])
                     .regionName([region name])
                     .streamName([streamName])
-                    .initialPositionInStream([initial position])
+                    .initialPosition([initial position])
                     .checkpointAppName([Kinesis app name])
                     .checkpointInterval([checkpoint interval])
                     .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -133,7 +133,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                 import org.apache.spark.streaming.kinesis.KinesisInputDStream;
                 import org.apache.spark.streaming.Seconds;
                 import org.apache.spark.streaming.StreamingContext;
-                import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
+                import org.apache.spark.streaming.kinesis.KinesisInitialPositions;
                 import com.amazonaws.services.kinesis.clientlibrary.lib.worker.KinesisClientLibConfiguration;
                 import com.amazonaws.services.kinesis.metrics.interfaces.MetricsLevel;
                 import scala.collection.JavaConverters;
@@ -143,7 +143,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
                     .endpointUrl([endpoint URL])
                     .regionName([region name])
                     .streamName([streamName])
-                    .initialPositionInStream([initial position])
+                    .initialPosition([initial position])
                     .checkpointAppName([Kinesis app name])
                     .checkpointInterval([checkpoint interval])
                     .storageLevel(StorageLevel.MEMORY_AND_DISK_2)
@@ -170,7 +170,7 @@ A Kinesis stream can be set up at one of the valid Kinesis endpoints with 1 or m
 
 	- `[checkpoint interval]`: The interval (e.g., Duration(2000) = 2 seconds) at which the Kinesis Client Library saves its position in the stream.  For starters, set it to the same as the batch interval of the streaming application.
 
-	- `[initial position]`: Can be either `InitialPositionInStream.TRIM_HORIZON` or `InitialPositionInStream.LATEST` (see [`Kinesis Checkpointing`](#kinesis-checkpointing) section and [`Amazon Kinesis API documentation`](http://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-sdk.html) for more details).
+	- `[initial position]`: Can be either `KinesisInitialPositions.TrimHorizon` or `KinesisInitialPositions.Latest` or `KinesisInitialPositions.AtTimestamp` (see [`Kinesis Checkpointing`](#kinesis-checkpointing) section and [`Amazon Kinesis API documentation`](http://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-sdk.html) for more details).
 
 	- `[message handler]`: A function that takes a Kinesis `Record` and outputs generic `T`.
 
@@ -272,9 +272,9 @@ de-aggregate records during consumption.
 
 - Checkpointing too frequently will cause excess load on the AWS checkpoint storage layer and may lead to AWS throttling.  The provided example handles this throttling with a random-backoff-retry strategy.
 
-- If no Kinesis checkpoint info exists when the input DStream starts, it will start either from the oldest record available (`InitialPositionInStream.TRIM_HORIZON`) or from the latest tip (`InitialPositionInStream.LATEST`).  This is configurable.
-  - `InitialPositionInStream.LATEST` could lead to missed records if data is added to the stream while no input DStreams are running (and no checkpoint info is being stored).
-  - `InitialPositionInStream.TRIM_HORIZON` may lead to duplicate processing of records where the impact is dependent on checkpoint frequency and processing idempotency.
+- If no Kinesis checkpoint info exists when the input DStream starts, it will start either from the oldest record available (`KinesisInitialPositions.TrimHorizon`), or from the latest tip (`KinesisInitialPositions.Latest`), or (except Python) from the position denoted by the provided UTC timestamp (`KinesisInitialPositions.AtTimestamp(Date timestamp)`).  This is configurable.
+  - `KinesisInitialPositions.Latest` could lead to missed records if data is added to the stream while no input DStreams are running (and no checkpoint info is being stored).
+  - `KinesisInitialPositions.TrimHorizon` may lead to duplicate processing of records where the impact is dependent on checkpoint frequency and processing idempotency.
 
 #### Kinesis retry configuration
  - `spark.streaming.kinesis.retry.waitTime` : Wait time between Kinesis retries as a duration string. When reading from Amazon Kinesis, users may hit `ProvisionedThroughputExceededException`'s, when consuming faster than 5 transactions/second or, exceeding the maximum read rate of 2 MiB/second. This configuration can be tweaked to increase the sleep between fetches when a fetch fails to reduce these exceptions. Default is "100ms".

From 52363d4d975d0e49212245193234447ed2d38152 Mon Sep 17 00:00:00 2001
From: Sean Owen <srowen@gmail.com>
Date: Mon, 24 Feb 2020 20:18:10 -0600
Subject: [PATCH 0154/1280] [SPARK-30939][ML] Correctly set output col when
 StringIndexer.setOutputCols is used

### What changes were proposed in this pull request?

Set the supplied output col name as intended when StringIndexer transforms an input after setOutputCols is used.

### Why are the changes needed?

The output col names are wrong otherwise and downstream pipeline components fail.

### Does this PR introduce any user-facing change?

Yes in the sense that it fixes incorrect behavior, otherwise no.

### How was this patch tested?

Existing tests plus new direct tests of the schema.

Closes #27684 from srowen/SPARK-30939.

Authored-by: Sean Owen <srowen@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit cc8d356e4faf1b581d330119c42ef73df466e828)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../apache/spark/ml/feature/StringIndexer.scala |  2 +-
 .../spark/ml/feature/StringIndexerSuite.scala   | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index 9f9f097a26ead..be32f44287b6a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -107,7 +107,7 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
         s"but got $inputDataType.")
     require(schema.fields.forall(_.name != outputColName),
       s"Output column $outputColName already exists.")
-    NominalAttribute.defaultAttr.withName($(outputCol)).toStructField()
+    NominalAttribute.defaultAttr.withName(outputColName).toStructField()
   }
 
   /** Validates and transforms the input schema. */
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
index b5ce2bace98f6..948140897d8cc 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
@@ -96,6 +96,23 @@ class StringIndexerSuite extends MLTest with DefaultReadWriteTest {
     }
   }
 
+  test("StringIndexer.transformSchema)") {
+    val idxToStr = new StringIndexer().setInputCol("input").setOutputCol("output")
+    val inSchema = StructType(Seq(StructField("input", StringType)))
+    val outSchema = idxToStr.transformSchema(inSchema)
+    assert(outSchema("output").dataType === DoubleType)
+  }
+
+  test("StringIndexer.transformSchema multi col") {
+    val idxToStr = new StringIndexer().setInputCols(Array("input", "input2")).
+      setOutputCols(Array("output", "output2"))
+    val inSchema = StructType(Seq(StructField("input", StringType),
+      StructField("input2", StringType)))
+    val outSchema = idxToStr.transformSchema(inSchema)
+    assert(outSchema("output").dataType === DoubleType)
+    assert(outSchema("output2").dataType === DoubleType)
+  }
+
   test("StringIndexerUnseen") {
     val data = Seq((0, "a"), (1, "b"), (4, "b"))
     val data2 = Seq((0, "a"), (1, "b"), (2, "c"), (3, "d"))

From e19f47849559ca92ae42544b0d1b0e21a6f55d39 Mon Sep 17 00:00:00 2001
From: zhengruifeng <ruifengz@foxmail.com>
Date: Sat, 8 Feb 2020 08:46:16 +0800
Subject: [PATCH 0155/1280] Revert
 "[SPARK-30642][SPARK-30659][SPARK-30660][SPARK-30662]"

### What changes were proposed in this pull request?
Revert
#27360
#27396
#27374
#27389

### Why are the changes needed?
BLAS need more performace tests, specially on sparse datasets.
Perfermance test of LogisticRegression (https://github.com/apache/spark/pull/27374) on sparse dataset shows that blockify vectors to matrices and use BLAS will cause performance regression.
LinearSVC and LinearRegression were also updated in the same way as LogisticRegression, so we need to revert them to make sure no regression.

### Does this PR introduce any user-facing change?
remove newly added param blockSize

### How was this patch tested?
reverted testsuites

Closes #27487 from zhengruifeng/revert_blockify_ii.

Authored-by: zhengruifeng <ruifengz@foxmail.com>
Signed-off-by: zhengruifeng <ruifengz@foxmail.com>
---
 .../spark/serializer/KryoSerializer.scala     |   1 -
 .../org/apache/spark/ml/linalg/BLAS.scala     |   1 +
 .../spark/ml/classification/LinearSVC.scala   |  51 ++--
 .../classification/LogisticRegression.scala   |  58 ++--
 .../MultilayerPerceptronClassifier.scala      |  22 +-
 .../apache/spark/ml/feature/Instance.scala    | 130 +--------
 .../ml/optim/aggregator/HingeAggregator.scala | 102 +------
 .../ml/optim/aggregator/HuberAggregator.scala | 103 ++-----
 .../aggregator/LeastSquaresAggregator.scala   |  74 ++---
 .../optim/aggregator/LogisticAggregator.scala | 252 ++----------------
 .../ml/param/shared/SharedParamsCodeGen.scala |   6 +-
 .../spark/ml/param/shared/sharedParams.scala  |  23 +-
 .../apache/spark/ml/recommendation/ALS.scala  |  46 +---
 .../ml/regression/LinearRegression.scala      |  49 ++--
 .../classification/LogisticRegression.scala   |   4 +-
 .../ml/classification/LinearSVCSuite.scala    |   2 +-
 .../LogisticRegressionSuite.scala             |   4 +-
 .../spark/ml/feature/InstanceSuite.scala      |  31 ---
 .../aggregator/HingeAggregatorSuite.scala     |  52 +---
 .../aggregator/HuberAggregatorSuite.scala     |  61 +----
 .../LeastSquaresAggregatorSuite.scala         |  62 +----
 .../aggregator/LogisticAggregatorSuite.scala  |  56 +---
 python/pyspark/ml/classification.py           |  67 ++---
 .../ml/param/_shared_params_code_gen.py       |   5 +-
 python/pyspark/ml/param/shared.py             |  18 --
 python/pyspark/ml/recommendation.py           |  29 +-
 python/pyspark/ml/regression.py               |  22 +-
 27 files changed, 260 insertions(+), 1071 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index 55ac2c410953b..cdaab599e2a0b 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -502,7 +502,6 @@ private[serializer] object KryoSerializer {
       "org.apache.spark.ml.attribute.NumericAttribute",
 
       "org.apache.spark.ml.feature.Instance",
-      "org.apache.spark.ml.feature.InstanceBlock",
       "org.apache.spark.ml.feature.LabeledPoint",
       "org.apache.spark.ml.feature.OffsetInstance",
       "org.apache.spark.ml.linalg.DenseMatrix",
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
index 00e5b61dbdc18..e054a15fc9b75 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
@@ -682,6 +682,7 @@ private[spark] object BLAS extends Serializable {
 
           val xTemp = xValues(k) * alpha
           while (i < indEnd) {
+            val rowIndex = Arows(i)
             yValues(Arows(i)) += Avals(i) * xTemp
             i += 1
           }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index f16648d2abee6..905789090d625 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.optim.aggregator.HingeAggregator
 import org.apache.spark.ml.optim.loss.{L2Regularization, RDDLossFunction}
@@ -41,7 +41,7 @@ import org.apache.spark.storage.StorageLevel
 /** Params for linear SVM Classifier. */
 private[classification] trait LinearSVCParams extends ClassifierParams with HasRegParam
   with HasMaxIter with HasFitIntercept with HasTol with HasStandardization with HasWeightCol
-  with HasAggregationDepth with HasThreshold with HasBlockSize {
+  with HasAggregationDepth with HasThreshold {
 
   /**
    * Param for threshold in binary classification prediction.
@@ -155,26 +155,19 @@ class LinearSVC @Since("2.2.0") (
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
   setDefault(aggregationDepth -> 2)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 1024.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-
   @Since("2.2.0")
   override def copy(extra: ParamMap): LinearSVC = defaultCopy(extra)
 
   override protected def train(dataset: Dataset[_]): LinearSVCModel = instrumented { instr =>
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+
+    val instances = extractInstances(dataset)
+    if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
+
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, rawPredictionCol,
-      regParam, maxIter, fitIntercept, tol, standardization, threshold, aggregationDepth, blockSize)
-
-    val sc = dataset.sparkSession.sparkContext
-    val instances = extractInstances(dataset)
+      regParam, maxIter, fitIntercept, tol, standardization, threshold, aggregationDepth)
 
     val (summarizer, labelSummarizer) = instances.treeAggregate(
       (Summarizer.createSummarizerBuffer("mean", "std", "count"), new MultiClassSummarizer))(
@@ -215,33 +208,20 @@ class LinearSVC @Since("2.2.0") (
         throw new SparkException(msg)
       }
 
-      val featuresStd = summarizer.std.compressed
-      val bcFeaturesStd = sc.broadcast(featuresStd)
+      val featuresStd = summarizer.std.toArray
+      val getFeaturesStd = (j: Int) => featuresStd(j)
       val regParamL2 = $(regParam)
+      val bcFeaturesStd = instances.context.broadcast(featuresStd)
       val regularization = if (regParamL2 != 0.0) {
         val shouldApply = (idx: Int) => idx >= 0 && idx < numFeatures
         Some(new L2Regularization(regParamL2, shouldApply,
-          if ($(standardization)) None else Some(featuresStd.apply)))
+          if ($(standardization)) None else Some(getFeaturesStd)))
       } else {
         None
       }
 
-      val standardized = instances.map {
-        case Instance(label, weight, features) =>
-          val featuresStd = bcFeaturesStd.value
-          val array = Array.ofDim[Double](numFeatures)
-          features.foreachNonZero { (i, v) =>
-            val std = featuresStd(i)
-            if (std != 0) array(i) = v / std
-          }
-          Instance(label, weight, Vectors.dense(array))
-      }
-      val blocks = InstanceBlock.blokify(standardized, $(blockSize))
-        .persist(StorageLevel.MEMORY_AND_DISK)
-        .setName(s"training dataset (blockSize=${$(blockSize)})")
-
-      val getAggregatorFunc = new HingeAggregator(numFeatures, $(fitIntercept))(_)
-      val costFun = new RDDLossFunction(blocks, getAggregatorFunc, regularization,
+      val getAggregatorFunc = new HingeAggregator(bcFeaturesStd, $(fitIntercept))(_)
+      val costFun = new RDDLossFunction(instances, getAggregatorFunc, regularization,
         $(aggregationDepth))
 
       def regParamL1Fun = (index: Int) => 0D
@@ -258,7 +238,6 @@ class LinearSVC @Since("2.2.0") (
         scaledObjectiveHistory += state.adjustedValue
       }
 
-      blocks.unpersist()
       bcFeaturesStd.destroy()
       if (state == null) {
         val msg = s"${optimizer.getClass.getName} failed."
@@ -289,6 +268,8 @@ class LinearSVC @Since("2.2.0") (
       (Vectors.dense(coefficientArray), intercept, scaledObjectiveHistory.result())
     }
 
+    if (handlePersistence) instances.unpersist()
+
     copyValues(new LinearSVCModel(uid, coefficientVector, interceptVector))
   }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 9b5b36257a584..50c14d086957f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.optim.aggregator.LogisticAggregator
 import org.apache.spark.ml.optim.loss.{L2Regularization, RDDLossFunction}
@@ -50,8 +50,7 @@ import org.apache.spark.util.VersionUtils
  */
 private[classification] trait LogisticRegressionParams extends ProbabilisticClassifierParams
   with HasRegParam with HasElasticNetParam with HasMaxIter with HasFitIntercept with HasTol
-  with HasStandardization with HasWeightCol with HasThreshold with HasAggregationDepth
-  with HasBlockSize  {
+  with HasStandardization with HasWeightCol with HasThreshold with HasAggregationDepth {
 
   import org.apache.spark.ml.classification.LogisticRegression.supportedFamilyNames
 
@@ -431,15 +430,6 @@ class LogisticRegression @Since("1.2.0") (
   @Since("2.2.0")
   def setUpperBoundsOnIntercepts(value: Vector): this.type = set(upperBoundsOnIntercepts, value)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 1024.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-
   private def assertBoundConstrainedOptimizationParamsValid(
       numCoefficientSets: Int,
       numFeatures: Int): Unit = {
@@ -492,17 +482,24 @@ class LogisticRegression @Since("1.2.0") (
     this
   }
 
-  override protected[spark] def train(
-      dataset: Dataset[_]): LogisticRegressionModel = instrumented { instr =>
+  override protected[spark] def train(dataset: Dataset[_]): LogisticRegressionModel = {
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    train(dataset, handlePersistence)
+  }
+
+  protected[spark] def train(
+      dataset: Dataset[_],
+      handlePersistence: Boolean): LogisticRegressionModel = instrumented { instr =>
+    val instances = extractInstances(dataset)
+
+    if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
+
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, rawPredictionCol,
       probabilityCol, regParam, elasticNetParam, standardization, threshold, maxIter, tol,
       fitIntercept)
 
-    val sc = dataset.sparkSession.sparkContext
-    val instances = extractInstances(dataset)
-
     val (summarizer, labelSummarizer) = instances.treeAggregate(
       (Summarizer.createSummarizerBuffer("mean", "std", "count"), new MultiClassSummarizer))(
       seqOp = (c: (SummarizerBuffer, MultiClassSummarizer), instance: Instance) =>
@@ -585,9 +582,8 @@ class LogisticRegression @Since("1.2.0") (
             s"dangerous ground, so the algorithm may not converge.")
         }
 
-        val featuresMean = summarizer.mean.compressed
-        val featuresStd = summarizer.std.compressed
-        val bcFeaturesStd = sc.broadcast(featuresStd)
+        val featuresMean = summarizer.mean.toArray
+        val featuresStd = summarizer.std.toArray
 
         if (!$(fitIntercept) && (0 until numFeatures).exists { i =>
           featuresStd(i) == 0.0 && featuresMean(i) != 0.0 }) {
@@ -599,7 +595,8 @@ class LogisticRegression @Since("1.2.0") (
         val regParamL1 = $(elasticNetParam) * $(regParam)
         val regParamL2 = (1.0 - $(elasticNetParam)) * $(regParam)
 
-        val getAggregatorFunc = new LogisticAggregator(numFeatures, numClasses, $(fitIntercept),
+        val bcFeaturesStd = instances.context.broadcast(featuresStd)
+        val getAggregatorFunc = new LogisticAggregator(bcFeaturesStd, numClasses, $(fitIntercept),
           multinomial = isMultinomial)(_)
         val getFeaturesStd = (j: Int) => if (j >= 0 && j < numCoefficientSets * numFeatures) {
           featuresStd(j / numCoefficientSets)
@@ -615,21 +612,7 @@ class LogisticRegression @Since("1.2.0") (
           None
         }
 
-        val standardized = instances.map {
-          case Instance(label, weight, features) =>
-            val featuresStd = bcFeaturesStd.value
-            val array = Array.ofDim[Double](numFeatures)
-            features.foreachNonZero { (i, v) =>
-              val std = featuresStd(i)
-              if (std != 0) array(i) = v / std
-            }
-            Instance(label, weight, Vectors.dense(array))
-        }
-        val blocks = InstanceBlock.blokify(standardized, $(blockSize))
-          .persist(StorageLevel.MEMORY_AND_DISK)
-          .setName(s"training dataset (blockSize=${$(blockSize)})")
-
-        val costFun = new RDDLossFunction(blocks, getAggregatorFunc, regularization,
+        val costFun = new RDDLossFunction(instances, getAggregatorFunc, regularization,
           $(aggregationDepth))
 
         val numCoeffsPlusIntercepts = numFeaturesPlusIntercept * numCoefficientSets
@@ -823,7 +806,6 @@ class LogisticRegression @Since("1.2.0") (
           state = states.next()
           arrayBuilder += state.adjustedValue
         }
-        blocks.unpersist()
         bcFeaturesStd.destroy()
 
         if (state == null) {
@@ -893,6 +875,8 @@ class LogisticRegression @Since("1.2.0") (
       }
     }
 
+    if (handlePersistence) instances.unpersist()
+
     val model = copyValues(new LogisticRegressionModel(uid, coefficientMatrix, interceptVector,
       numClasses, isMultinomial))
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
index 6e8f92b9b1e64..c7a8237849b5b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
@@ -34,7 +34,7 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
 
 /** Params for Multilayer Perceptron. */
 private[classification] trait MultilayerPerceptronParams extends ProbabilisticClassifierParams
-  with HasSeed with HasMaxIter with HasTol with HasStepSize with HasSolver with HasBlockSize {
+  with HasSeed with HasMaxIter with HasTol with HasStepSize with HasSolver {
 
   import MultilayerPerceptronClassifier._
 
@@ -54,6 +54,26 @@ private[classification] trait MultilayerPerceptronParams extends ProbabilisticCl
   @Since("1.5.0")
   final def getLayers: Array[Int] = $(layers)
 
+  /**
+   * Block size for stacking input data in matrices to speed up the computation.
+   * Data is stacked within partitions. If block size is more than remaining data in
+   * a partition then it is adjusted to the size of this data.
+   * Recommended size is between 10 and 1000.
+   * Default: 128
+   *
+   * @group expertParam
+   */
+  @Since("1.5.0")
+  final val blockSize: IntParam = new IntParam(this, "blockSize",
+    "Block size for stacking input data in matrices. Data is stacked within partitions." +
+      " If block size is more than remaining data in a partition then " +
+      "it is adjusted to the size of this data. Recommended size is between 10 and 1000",
+    ParamValidators.gt(0))
+
+  /** @group expertGetParam */
+  @Since("1.5.0")
+  final def getBlockSize: Int = $(blockSize)
+
   /**
    * The solver algorithm for optimization.
    * Supported options: "gd" (minibatch gradient descent) or "l-bfgs".
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
index 5476a86eb9d76..11d0c4689cbba 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Instance.scala
@@ -17,10 +17,7 @@
 
 package org.apache.spark.ml.feature
 
-import scala.collection.mutable
-
-import org.apache.spark.ml.linalg._
-import org.apache.spark.rdd.RDD
+import org.apache.spark.ml.linalg.Vector
 
 /**
  * Class that represents an instance of weighted data point with label and features.
@@ -31,131 +28,6 @@ import org.apache.spark.rdd.RDD
  */
 private[spark] case class Instance(label: Double, weight: Double, features: Vector)
 
-
-/**
- * Class that represents an block of instance.
- * If all weights are 1, then an empty array is stored.
- */
-private[spark] case class InstanceBlock(
-    labels: Array[Double],
-    weights: Array[Double],
-    matrix: Matrix) {
-  require(labels.length == matrix.numRows)
-  require(matrix.isTransposed)
-  if (weights.nonEmpty) {
-    require(labels.length == weights.length)
-  }
-
-  def size: Int = labels.length
-
-  def numFeatures: Int = matrix.numCols
-
-  def instanceIterator: Iterator[Instance] = {
-    if (weights.nonEmpty) {
-      labels.iterator.zip(weights.iterator).zip(matrix.rowIter)
-        .map { case ((label, weight), vec) => Instance(label, weight, vec) }
-    } else {
-      labels.iterator.zip(matrix.rowIter)
-        .map { case (label, vec) => Instance(label, 1.0, vec) }
-    }
-  }
-
-  def getLabel(i: Int): Double = labels(i)
-
-  def labelIter: Iterator[Double] = labels.iterator
-
-  @transient lazy val getWeight: Int => Double = {
-    if (weights.nonEmpty) {
-      (i: Int) => weights(i)
-    } else {
-      (i: Int) => 1.0
-    }
-  }
-
-  def weightIter: Iterator[Double] = {
-    if (weights.nonEmpty) {
-      weights.iterator
-    } else {
-      Iterator.fill(size)(1.0)
-    }
-  }
-
-  // directly get the non-zero iterator of i-th row vector without array copy or slice
-  @transient lazy val getNonZeroIter: Int => Iterator[(Int, Double)] = {
-    matrix match {
-      case dm: DenseMatrix =>
-        (i: Int) =>
-          val start = numFeatures * i
-          Iterator.tabulate(numFeatures)(j =>
-            (j, dm.values(start + j))
-          ).filter(_._2 != 0)
-      case sm: SparseMatrix =>
-        (i: Int) =>
-          val start = sm.colPtrs(i)
-          val end = sm.colPtrs(i + 1)
-          Iterator.tabulate(end - start)(j =>
-            (sm.rowIndices(start + j), sm.values(start + j))
-          ).filter(_._2 != 0)
-    }
-  }
-}
-
-private[spark] object InstanceBlock {
-
-  def fromInstances(instances: Seq[Instance]): InstanceBlock = {
-    val labels = instances.map(_.label).toArray
-    val weights = if (instances.exists(_.weight != 1)) {
-      instances.map(_.weight).toArray
-    } else {
-      Array.emptyDoubleArray
-    }
-    val numRows = instances.length
-    val numCols = instances.head.features.size
-    val denseSize = Matrices.getDenseSize(numCols, numRows)
-    val nnz = instances.iterator.map(_.features.numNonzeros).sum
-    val sparseSize = Matrices.getSparseSize(nnz, numRows + 1)
-    val matrix = if (denseSize < sparseSize) {
-      val values = Array.ofDim[Double](numRows * numCols)
-      var offset = 0
-      var j = 0
-      while (j < numRows) {
-        instances(j).features.foreachNonZero { (i, v) =>
-          values(offset + i) = v
-        }
-        offset += numCols
-        j += 1
-      }
-      new DenseMatrix(numRows, numCols, values, true)
-    } else {
-      val colIndices = mutable.ArrayBuilder.make[Int]
-      val values = mutable.ArrayBuilder.make[Double]
-      val rowPtrs = mutable.ArrayBuilder.make[Int]
-      var rowPtr = 0
-      rowPtrs += 0
-      var j = 0
-      while (j < numRows) {
-        var nnz = 0
-        instances(j).features.foreachNonZero { (i, v) =>
-          colIndices += i
-          values += v
-          nnz += 1
-        }
-        rowPtr += nnz
-        rowPtrs += rowPtr
-        j += 1
-      }
-      new SparseMatrix(numRows, numCols, rowPtrs.result(),
-        colIndices.result(), values.result(), true)
-    }
-    InstanceBlock(labels, weights, matrix)
-  }
-
-  def blokify(instances: RDD[Instance], blockSize: Int): RDD[InstanceBlock] = {
-    instances.mapPartitions(_.grouped(blockSize).map(InstanceBlock.fromInstances))
-  }
-}
-
-
 /**
  * Case class that represents an instance of data point with
  * label, weight, offset and features.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
index 292187b3e146e..b0906f1b06511 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HingeAggregator.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg._
 
 /**
@@ -32,28 +32,21 @@ import org.apache.spark.ml.linalg._
  *
  * @param bcCoefficients The coefficients corresponding to the features.
  * @param fitIntercept Whether to fit an intercept term.
+ * @param bcFeaturesStd The standard deviation values of the features.
  */
 private[ml] class HingeAggregator(
-    numFeatures: Int,
+    bcFeaturesStd: Broadcast[Array[Double]],
     fitIntercept: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, HingeAggregator] {
+  extends DifferentiableLossAggregator[Instance, HingeAggregator] {
 
+  private val numFeatures: Int = bcFeaturesStd.value.length
   private val numFeaturesPlusIntercept: Int = if (fitIntercept) numFeatures + 1 else numFeatures
-  protected override val dim: Int = numFeaturesPlusIntercept
   @transient private lazy val coefficientsArray = bcCoefficients.value match {
     case DenseVector(values) => values
     case _ => throw new IllegalArgumentException(s"coefficients only supports dense vector" +
       s" but got type ${bcCoefficients.value.getClass}.")
   }
-
-  @transient private lazy val linear = {
-    if (fitIntercept) {
-      new DenseVector(coefficientsArray.take(numFeatures))
-    } else {
-      new DenseVector(coefficientsArray)
-    }
-  }
-
+  protected override val dim: Int = numFeaturesPlusIntercept
 
   /**
    * Add a new training instance to this HingeAggregator, and update the loss and gradient
@@ -69,13 +62,16 @@ private[ml] class HingeAggregator(
       require(weight >= 0.0, s"instance weight, $weight has to be >= 0.0")
 
       if (weight == 0.0) return this
+      val localFeaturesStd = bcFeaturesStd.value
       val localCoefficients = coefficientsArray
       val localGradientSumArray = gradientSumArray
 
       val dotProduct = {
         var sum = 0.0
         features.foreachNonZero { (index, value) =>
-          sum += localCoefficients(index) * value
+          if (localFeaturesStd(index) != 0.0) {
+            sum += localCoefficients(index) * value / localFeaturesStd(index)
+          }
         }
         if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
         sum
@@ -92,7 +88,9 @@ private[ml] class HingeAggregator(
       if (1.0 > labelScaled * dotProduct) {
         val gradientScale = -labelScaled * weight
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) += value * gradientScale
+          if (localFeaturesStd(index) != 0.0) {
+            localGradientSumArray(index) += value * gradientScale / localFeaturesStd(index)
+          }
         }
         if (fitIntercept) {
           localGradientSumArray(localGradientSumArray.length - 1) += gradientScale
@@ -104,78 +102,4 @@ private[ml] class HingeAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this HingeAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param block The InstanceBlock to be added.
-   * @return This HingeAggregator object.
-   */
-  def add(block: InstanceBlock): this.type = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // vec here represents dotProducts
-    val vec = if (fitIntercept && coefficientsArray.last != 0) {
-      val intercept = coefficientsArray.last
-      new DenseVector(Array.fill(size)(intercept))
-    } else {
-      new DenseVector(Array.ofDim[Double](size))
-    }
-
-    if (fitIntercept) {
-      BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
-    } else {
-      BLAS.gemv(1.0, block.matrix, linear, 0.0, vec)
-    }
-
-    // in-place convert dotProducts to gradient scales
-    // then, vec represents gradient scales
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        // Our loss function with {0, 1} labels is max(0, 1 - (2y - 1) (f_w(x)))
-        // Therefore the gradient is -(2y - 1)*x
-        val label = block.getLabel(i)
-        val labelScaled = 2 * label - 1.0
-        val loss = (1.0 - labelScaled * vec(i)) * weight
-        if (loss > 0) {
-          lossSum += loss
-          val gradScale = -labelScaled * weight
-          vec.values(i) = gradScale
-        } else {
-          vec.values(i) = 0.0
-        }
-      } else {
-        vec.values(i) = 0.0
-      }
-      i += 1
-    }
-
-    // predictions are all correct, no gradient signal
-    if (vec.values.forall(_ == 0)) return this
-
-    if (fitIntercept) {
-      // localGradientSumArray is of size numFeatures+1, so can not
-      // be directly used as the output of BLAS.gemv
-      val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 0.0, linearGradSumVec)
-      linearGradSumVec.foreachNonZero { (i, v) => localGradientSumArray(i) += v }
-      localGradientSumArray(numFeatures) += vec.values.sum
-    } else {
-      val gradSumVec = new DenseVector(localGradientSumArray)
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
-    }
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
index f83621506500e..8a1a41b2950c1 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/HuberAggregator.scala
@@ -17,8 +17,8 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.linalg.Vector
 
 /**
  * HuberAggregator computes the gradient and loss for a huber loss function,
@@ -62,17 +62,19 @@ import org.apache.spark.ml.linalg._
  *
  * @param fitIntercept Whether to fit an intercept term.
  * @param epsilon The shape parameter to control the amount of robustness.
+ * @param bcFeaturesStd The broadcast standard deviation values of the features.
  * @param bcParameters including three parts: the regression coefficients corresponding
  *                     to the features, the intercept (if fitIntercept is ture)
  *                     and the scale parameter (sigma).
  */
 private[ml] class HuberAggregator(
-    numFeatures: Int,
     fitIntercept: Boolean,
-    epsilon: Double)(bcParameters: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, HuberAggregator] {
+    epsilon: Double,
+    bcFeaturesStd: Broadcast[Array[Double]])(bcParameters: Broadcast[Vector])
+  extends DifferentiableLossAggregator[Instance, HuberAggregator] {
 
   protected override val dim: Int = bcParameters.value.size
+  private val numFeatures: Int = if (fitIntercept) dim - 2 else dim - 1
   private val sigma: Double = bcParameters.value(dim - 1)
   private val intercept: Double = if (fitIntercept) {
     bcParameters.value(dim - 2)
@@ -80,8 +82,7 @@ private[ml] class HuberAggregator(
     0.0
   }
   // make transient so we do not serialize between aggregation stages
-  @transient private lazy val linear =
-    new DenseVector(bcParameters.value.toArray.take(numFeatures))
+  @transient private lazy val coefficients = bcParameters.value.toArray.slice(0, numFeatures)
 
   /**
    * Add a new training instance to this HuberAggregator, and update the loss and gradient
@@ -97,13 +98,16 @@ private[ml] class HuberAggregator(
       require(weight >= 0.0, s"instance weight, $weight has to be >= 0.0")
 
       if (weight == 0.0) return this
-      val localCoefficients = linear.values
+      val localFeaturesStd = bcFeaturesStd.value
+      val localCoefficients = coefficients
       val localGradientSumArray = gradientSumArray
 
       val margin = {
         var sum = 0.0
         features.foreachNonZero { (index, value) =>
-            sum += localCoefficients(index) * value
+          if (localFeaturesStd(index) != 0.0) {
+            sum += localCoefficients(index) * (value / localFeaturesStd(index))
+          }
         }
         if (fitIntercept) sum += intercept
         sum
@@ -115,7 +119,10 @@ private[ml] class HuberAggregator(
         val linearLossDivSigma = linearLoss / sigma
 
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) -= weight * linearLossDivSigma * value
+          if (localFeaturesStd(index) != 0.0) {
+            localGradientSumArray(index) +=
+              -1.0 * weight * linearLossDivSigma * (value / localFeaturesStd(index))
+          }
         }
         if (fitIntercept) {
           localGradientSumArray(dim - 2) += -1.0 * weight * linearLossDivSigma
@@ -127,7 +134,10 @@ private[ml] class HuberAggregator(
           (sigma + 2.0 * epsilon * math.abs(linearLoss) - sigma * epsilon * epsilon)
 
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) += weight * sign * epsilon * value
+          if (localFeaturesStd(index) != 0.0) {
+            localGradientSumArray(index) +=
+              weight * sign * epsilon * (value / localFeaturesStd(index))
+          }
         }
         if (fitIntercept) {
           localGradientSumArray(dim - 2) += weight * sign * epsilon
@@ -139,75 +149,4 @@ private[ml] class HuberAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this HuberAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This HuberAggregator object.
-   */
-  def add(block: InstanceBlock): HuberAggregator = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // vec here represents margins or dotProducts
-    val vec = if (fitIntercept && intercept != 0) {
-      new DenseVector(Array.fill(size)(intercept))
-    } else {
-      new DenseVector(Array.ofDim[Double](size))
-    }
-
-    if (fitIntercept) {
-      BLAS.gemv(1.0, block.matrix, linear, 1.0, vec)
-    } else {
-      BLAS.gemv(1.0, block.matrix, linear, 0.0, vec)
-    }
-
-    // in-place convert margins to multipliers
-    // then, vec represents multipliers
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-        val margin = vec(i)
-        val linearLoss = label - margin
-
-        if (math.abs(linearLoss) <= sigma * epsilon) {
-          lossSum += 0.5 * weight * (sigma + math.pow(linearLoss, 2.0) / sigma)
-          val linearLossDivSigma = linearLoss / sigma
-          val multiplier = -1.0 * weight * linearLossDivSigma
-          vec.values(i) = multiplier
-          localGradientSumArray(dim - 1) += 0.5 * weight * (1.0 - math.pow(linearLossDivSigma, 2.0))
-        } else {
-          lossSum += 0.5 * weight *
-            (sigma + 2.0 * epsilon * math.abs(linearLoss) - sigma * epsilon * epsilon)
-          val sign = if (linearLoss >= 0) -1.0 else 1.0
-          val multiplier = weight * sign * epsilon
-          vec.values(i) = multiplier
-          localGradientSumArray(dim - 1) += 0.5 * weight * (1.0 - epsilon * epsilon)
-        }
-      } else {
-        vec.values(i) = 0.0
-      }
-      i += 1
-    }
-
-    val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
-    BLAS.gemv(1.0, block.matrix.transpose, vec, 0.0, linearGradSumVec)
-    linearGradSumVec.foreachNonZero { (i, v) => localGradientSumArray(i) += v }
-    if (fitIntercept) {
-      localGradientSumArray(dim - 2) += vec.values.sum
-    }
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
index a8bda9ca5d245..7a5806dc24aee 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregator.scala
@@ -17,8 +17,8 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 
 /**
  * LeastSquaresAggregator computes the gradient and loss for a Least-squared loss function,
@@ -157,25 +157,26 @@ private[ml] class LeastSquaresAggregator(
     labelStd: Double,
     labelMean: Double,
     fitIntercept: Boolean,
-    bcFeaturesStd: Broadcast[Vector],
-    bcFeaturesMean: Broadcast[Vector])(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, LeastSquaresAggregator] {
+    bcFeaturesStd: Broadcast[Array[Double]],
+    bcFeaturesMean: Broadcast[Array[Double]])(bcCoefficients: Broadcast[Vector])
+  extends DifferentiableLossAggregator[Instance, LeastSquaresAggregator] {
   require(labelStd > 0.0, s"${this.getClass.getName} requires the label standard " +
     s"deviation to be positive.")
 
-  private val numFeatures = bcFeaturesStd.value.size
+  private val numFeatures = bcFeaturesStd.value.length
   protected override val dim: Int = numFeatures
   // make transient so we do not serialize between aggregation stages
+  @transient private lazy val featuresStd = bcFeaturesStd.value
   @transient private lazy val effectiveCoefAndOffset = {
     val coefficientsArray = bcCoefficients.value.toArray.clone()
     val featuresMean = bcFeaturesMean.value
-    val featuresStd = bcFeaturesStd.value
     var sum = 0.0
     var i = 0
     val len = coefficientsArray.length
     while (i < len) {
       if (featuresStd(i) != 0.0) {
-        sum += coefficientsArray(i) / featuresStd(i) * featuresMean(i)
+        coefficientsArray(i) /=  featuresStd(i)
+        sum += coefficientsArray(i) * featuresMean(i)
       } else {
         coefficientsArray(i) = 0.0
       }
@@ -185,7 +186,7 @@ private[ml] class LeastSquaresAggregator(
     (Vectors.dense(coefficientsArray), offset)
   }
   // do not use tuple assignment above because it will circumvent the @transient tag
-  @transient private lazy val effectiveCoefficientsVec = effectiveCoefAndOffset._1
+  @transient private lazy val effectiveCoefficientsVector = effectiveCoefAndOffset._1
   @transient private lazy val offset = effectiveCoefAndOffset._2
 
   /**
@@ -203,20 +204,16 @@ private[ml] class LeastSquaresAggregator(
 
       if (weight == 0.0) return this
 
-      val localEffectiveCoefficientsVec = effectiveCoefficientsVec
-
-      val diff = {
-        var dot = 0.0
-        features.foreachNonZero { (index, value) =>
-          dot += localEffectiveCoefficientsVec(index) * value
-        }
-        dot - label / labelStd + offset
-      }
+      val diff = BLAS.dot(features, effectiveCoefficientsVector) - label / labelStd + offset
 
       if (diff != 0) {
         val localGradientSumArray = gradientSumArray
+        val localFeaturesStd = featuresStd
         features.foreachNonZero { (index, value) =>
-          localGradientSumArray(index) += weight * diff * value
+          val fStd = localFeaturesStd(index)
+          if (fStd != 0.0) {
+            localGradientSumArray(index) += weight * diff * value / fStd
+          }
         }
         lossSum += weight * diff * diff / 2.0
       }
@@ -224,43 +221,4 @@ private[ml] class LeastSquaresAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this LeastSquaresAggregator, and update the loss
-   * and gradient of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This LeastSquaresAggregator object.
-   */
-  def add(block: InstanceBlock): LeastSquaresAggregator = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-    val size = block.size
-
-    // vec here represents diffs
-    val vec = new DenseVector(Array.tabulate(size)(i => offset - block.getLabel(i) / labelStd))
-    BLAS.gemv(1.0, block.matrix, effectiveCoefficientsVec, 1.0, vec)
-
-    // in-place convert diffs to multipliers
-    // then, vec represents multipliers
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      val diff = vec(i)
-      lossSum += weight * diff * diff / 2
-      weightSum += weight
-      val multiplier = weight * diff
-      vec.values(i) = multiplier
-      i += 1
-    }
-
-    val gradSumVec = new DenseVector(gradientSumArray)
-    BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
index 76d21995a2c50..f2b3566f8f09e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregator.scala
@@ -18,8 +18,8 @@ package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
-import org.apache.spark.ml.linalg._
+import org.apache.spark.ml.feature.Instance
+import org.apache.spark.ml.linalg.{DenseVector, Vector}
 import org.apache.spark.mllib.util.MLUtils
 
 /**
@@ -171,6 +171,7 @@ import org.apache.spark.mllib.util.MLUtils
  *
  *
  * @param bcCoefficients The broadcast coefficients corresponding to the features.
+ * @param bcFeaturesStd The broadcast standard deviation values of the features.
  * @param numClasses the number of possible outcomes for k classes classification problem in
  *                   Multinomial Logistic Regression.
  * @param fitIntercept Whether to fit an intercept term.
@@ -182,12 +183,13 @@ import org.apache.spark.mllib.util.MLUtils
  * since this form is optimal for the matrix operations used for prediction.
  */
 private[ml] class LogisticAggregator(
-    numFeatures: Int,
+    bcFeaturesStd: Broadcast[Array[Double]],
     numClasses: Int,
     fitIntercept: Boolean,
     multinomial: Boolean)(bcCoefficients: Broadcast[Vector])
-  extends DifferentiableLossAggregator[InstanceBlock, LogisticAggregator] with Logging {
+  extends DifferentiableLossAggregator[Instance, LogisticAggregator] with Logging {
 
+  private val numFeatures = bcFeaturesStd.value.length
   private val numFeaturesPlusIntercept = if (fitIntercept) numFeatures + 1 else numFeatures
   private val coefficientSize = bcCoefficients.value.size
   protected override val dim: Int = coefficientSize
@@ -207,31 +209,6 @@ private[ml] class LogisticAggregator(
       s"got type ${bcCoefficients.value.getClass}.)")
   }
 
-  @transient private lazy val binaryLinear = {
-    if (!multinomial) {
-      if (fitIntercept) {
-        new DenseVector(coefficientsArray.take(numFeatures))
-      } else {
-        new DenseVector(coefficientsArray)
-      }
-    } else {
-      null
-    }
-  }
-
-  @transient private lazy val multinomialLinear = {
-    if (multinomial) {
-      if (fitIntercept) {
-        new DenseMatrix(numClasses, numFeatures, coefficientsArray.take(numClasses * numFeatures))
-      } else {
-        new DenseMatrix(numClasses, numFeatures, coefficientsArray)
-      }
-    } else {
-      null
-    }
-  }
-
-
   if (multinomial && numClasses <= 2) {
     logInfo(s"Multinomial logistic regression for binary classification yields separate " +
       s"coefficients for positive and negative classes. When no regularization is applied, the" +
@@ -242,12 +219,15 @@ private[ml] class LogisticAggregator(
   /** Update gradient and loss using binary loss function. */
   private def binaryUpdateInPlace(features: Vector, weight: Double, label: Double): Unit = {
 
+    val localFeaturesStd = bcFeaturesStd.value
     val localCoefficients = coefficientsArray
     val localGradientArray = gradientSumArray
     val margin = - {
       var sum = 0.0
       features.foreachNonZero { (index, value) =>
-        sum += localCoefficients(index) * value
+        if (localFeaturesStd(index) != 0.0) {
+          sum += localCoefficients(index) * value / localFeaturesStd(index)
+        }
       }
       if (fitIntercept) sum += localCoefficients(numFeaturesPlusIntercept - 1)
       sum
@@ -256,7 +236,9 @@ private[ml] class LogisticAggregator(
     val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
 
     features.foreachNonZero { (index, value) =>
-      localGradientArray(index) += multiplier * value
+      if (localFeaturesStd(index) != 0.0) {
+        localGradientArray(index) += multiplier * value / localFeaturesStd(index)
+      }
     }
 
     if (fitIntercept) {
@@ -271,61 +253,6 @@ private[ml] class LogisticAggregator(
     }
   }
 
-  /** Update gradient and loss using binary loss function. */
-  private def binaryUpdateInPlace(block: InstanceBlock): Unit = {
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // vec here represents margins or negative dotProducts
-    val vec = if (fitIntercept && coefficientsArray.last != 0) {
-      val intercept = coefficientsArray.last
-      new DenseVector(Array.fill(size)(intercept))
-    } else {
-      new DenseVector(Array.ofDim[Double](size))
-    }
-
-    if (fitIntercept) {
-      BLAS.gemv(-1.0, block.matrix, binaryLinear, -1.0, vec)
-    } else {
-      BLAS.gemv(-1.0, block.matrix, binaryLinear, 0.0, vec)
-    }
-
-    // in-place convert margins to multiplier
-    // then, vec represents multiplier
-    var i = 0
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-        val margin = vec(i)
-        if (label > 0) {
-          // The following is equivalent to log(1 + exp(margin)) but more numerically stable.
-          lossSum += weight * MLUtils.log1pExp(margin)
-        } else {
-          lossSum += weight * (MLUtils.log1pExp(margin) - margin)
-        }
-        val multiplier = weight * (1.0 / (1.0 + math.exp(margin)) - label)
-        vec.values(i) = multiplier
-      } else {
-        vec.values(i) = 0.0
-      }
-      i += 1
-    }
-
-    if (fitIntercept) {
-      // localGradientSumArray is of size numFeatures+1, so can not
-      // be directly used as the output of BLAS.gemv
-      val linearGradSumVec = new DenseVector(Array.ofDim[Double](numFeatures))
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 0.0, linearGradSumVec)
-      linearGradSumVec.foreachNonZero { (i, v) => localGradientSumArray(i) += v }
-      localGradientSumArray(numFeatures) += vec.values.sum
-    } else {
-      val gradSumVec = new DenseVector(localGradientSumArray)
-      BLAS.gemv(1.0, block.matrix.transpose, vec, 1.0, gradSumVec)
-    }
-  }
-
   /** Update gradient and loss using multinomial (softmax) loss function. */
   private def multinomialUpdateInPlace(features: Vector, weight: Double, label: Double): Unit = {
     // TODO: use level 2 BLAS operations
@@ -333,6 +260,7 @@ private[ml] class LogisticAggregator(
       Note: this can still be used when numClasses = 2 for binary
       logistic regression without pivoting.
      */
+    val localFeaturesStd = bcFeaturesStd.value
     val localCoefficients = coefficientsArray
     val localGradientArray = gradientSumArray
 
@@ -342,10 +270,13 @@ private[ml] class LogisticAggregator(
 
     val margins = new Array[Double](numClasses)
     features.foreachNonZero { (index, value) =>
-      var j = 0
-      while (j < numClasses) {
-        margins(j) += localCoefficients(index * numClasses + j) * value
-        j += 1
+      if (localFeaturesStd(index) != 0.0) {
+        val stdValue = value / localFeaturesStd(index)
+        var j = 0
+        while (j < numClasses) {
+          margins(j) += localCoefficients(index * numClasses + j) * stdValue
+          j += 1
+        }
       }
     }
     var i = 0
@@ -383,10 +314,13 @@ private[ml] class LogisticAggregator(
       multipliers(i) = multipliers(i) / sum - (if (label == i) 1.0 else 0.0)
     }
     features.foreachNonZero { (index, value) =>
-      var j = 0
-      while (j < numClasses) {
-        localGradientArray(index * numClasses + j) += weight * multipliers(j) * value
-        j += 1
+      if (localFeaturesStd(index) != 0.0) {
+        val stdValue = value / localFeaturesStd(index)
+        var j = 0
+        while (j < numClasses) {
+          localGradientArray(index * numClasses + j) += weight * multipliers(j) * stdValue
+          j += 1
+        }
       }
     }
     if (fitIntercept) {
@@ -405,112 +339,6 @@ private[ml] class LogisticAggregator(
     lossSum += weight * loss
   }
 
-  /** Update gradient and loss using multinomial (softmax) loss function. */
-  private def multinomialUpdateInPlace(block: InstanceBlock): Unit = {
-    val size = block.size
-    val localGradientSumArray = gradientSumArray
-
-    // mat here represents margins, shape: S X C
-    val mat = new DenseMatrix(size, numClasses, Array.ofDim[Double](size * numClasses))
-
-    if (fitIntercept) {
-      val intercept = coefficientsArray.takeRight(numClasses)
-      var i = 0
-      while (i < size) {
-        var j = 0
-        while (j < numClasses) {
-          mat.update(i, j, intercept(j))
-          j += 1
-        }
-        i += 1
-      }
-      BLAS.gemm(1.0, block.matrix, multinomialLinear.transpose, 1.0, mat)
-    } else {
-      BLAS.gemm(1.0, block.matrix, multinomialLinear.transpose, 0.0, mat)
-    }
-
-    // in-place convert margins to multipliers
-    // then, mat represents multipliers
-    var i = 0
-    val tmp = Array.ofDim[Double](numClasses)
-    while (i < size) {
-      val weight = block.getWeight(i)
-      if (weight > 0) {
-        weightSum += weight
-        val label = block.getLabel(i)
-
-        var maxMargin = Double.NegativeInfinity
-        var j = 0
-        while (j < numClasses) {
-          tmp(j) = mat(i, j)
-          maxMargin = math.max(maxMargin, tmp(j))
-          j += 1
-        }
-
-        // marginOfLabel is margins(label) in the formula
-        val marginOfLabel = tmp(label.toInt)
-
-        var sum = 0.0
-        j = 0
-        while (j < numClasses) {
-          if (maxMargin > 0) tmp(j) -= maxMargin
-          val exp = math.exp(tmp(j))
-          sum += exp
-          tmp(j) = exp
-          j += 1
-        }
-
-        j = 0
-        while (j < numClasses) {
-          val multiplier = weight * (tmp(j) / sum - (if (label == j) 1.0 else 0.0))
-          mat.update(i, j, multiplier)
-          j += 1
-        }
-
-        if (maxMargin > 0) {
-          lossSum += weight * (math.log(sum) - marginOfLabel + maxMargin)
-        } else {
-          lossSum += weight * (math.log(sum) - marginOfLabel)
-        }
-      } else {
-        var j = 0
-        while (j < numClasses) {
-          mat.update(i, j, 0.0)
-          j += 1
-        }
-      }
-      i += 1
-    }
-
-    // block.matrix:                  S X F, unknown type
-    // mat (multipliers):             S X C, dense
-    // gradSumMat(gradientSumArray):  C X FPI (numFeaturesPlusIntercept), dense
-    block.matrix match {
-      case dm: DenseMatrix if !fitIntercept =>
-        // If fitIntercept==false, gradientSumArray += mat.T X matrix
-        // GEMM requires block.matrix is dense
-        val gradSumMat = new DenseMatrix(numClasses, numFeatures, localGradientSumArray)
-        BLAS.gemm(1.0, mat.transpose, dm, 1.0, gradSumMat)
-
-      case _ =>
-        // Otherwise, use linearGradSumMat (F X C) as a temp matrix:
-        // linearGradSumMat = matrix.T X mat
-        val linearGradSumMat = new DenseMatrix(numFeatures, numClasses,
-          Array.ofDim[Double](numFeatures * numClasses))
-        BLAS.gemm(1.0, block.matrix.transpose, mat, 0.0, linearGradSumMat)
-        linearGradSumMat.foreachActive { (i, j, v) =>
-          if (v != 0) localGradientSumArray(i * numClasses + j) += v
-        }
-
-        if (fitIntercept) {
-          val start = numClasses * numFeatures
-          mat.foreachActive { (i, j, v) =>
-            if (v != 0) localGradientSumArray(start + j) += v
-          }
-        }
-    }
-  }
-
   /**
    * Add a new training instance to this LogisticAggregator, and update the loss and gradient
    * of the objective function.
@@ -535,28 +363,4 @@ private[ml] class LogisticAggregator(
       this
     }
   }
-
-  /**
-   * Add a new training instance block to this LogisticAggregator, and update the loss and gradient
-   * of the objective function.
-   *
-   * @param block The instance block of data point to be added.
-   * @return This LogisticAggregator object.
-   */
-  def add(block: InstanceBlock): this.type = {
-    require(numFeatures == block.numFeatures, s"Dimensions mismatch when adding new " +
-      s"instance. Expecting $numFeatures but got ${block.numFeatures}.")
-    require(block.weightIter.forall(_ >= 0),
-      s"instance weights ${block.weightIter.mkString("[", ",", "]")} has to be >= 0.0")
-
-    if (block.weightIter.forall(_ == 0)) return this
-
-    if (multinomial) {
-      multinomialUpdateInPlace(block)
-    } else {
-      binaryUpdateInPlace(block)
-    }
-
-    this
-  }
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
index 3d1fab8692af7..7ac680ec1183a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
@@ -104,11 +104,7 @@ private[shared] object SharedParamsCodeGen {
         isValid = "ParamValidators.inArray(Array(\"euclidean\", \"cosine\"))"),
       ParamDesc[String]("validationIndicatorCol", "name of the column that indicates whether " +
         "each row is for training or for validation. False indicates training; true indicates " +
-        "validation"),
-      ParamDesc[Int]("blockSize", "block size for stacking input data in matrices. Data is " +
-        "stacked within partitions. If block size is more than remaining data in a partition " +
-        "then it is adjusted to the size of this data", Some("1024"),
-        isValid = "ParamValidators.gt(0)", isExpertParam = true)
+        "validation.")
     )
 
     val code = genSharedParams(params)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
index 7fe8ccd973a72..44c993eeafddc 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
@@ -570,31 +570,12 @@ trait HasDistanceMeasure extends Params {
 trait HasValidationIndicatorCol extends Params {
 
   /**
-   * Param for name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation.
+   * Param for name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation..
    * @group param
    */
-  final val validationIndicatorCol: Param[String] = new Param[String](this, "validationIndicatorCol", "name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation")
+  final val validationIndicatorCol: Param[String] = new Param[String](this, "validationIndicatorCol", "name of the column that indicates whether each row is for training or for validation. False indicates training; true indicates validation.")
 
   /** @group getParam */
   final def getValidationIndicatorCol: String = $(validationIndicatorCol)
 }
-
-/**
- * Trait for shared param blockSize (default: 1024). This trait may be changed or
- * removed between minor versions.
- */
-@DeveloperApi
-trait HasBlockSize extends Params {
-
-  /**
-   * Param for block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.
-   * @group expertParam
-   */
-  final val blockSize: IntParam = new IntParam(this, "blockSize", "block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data", ParamValidators.gt(0))
-
-  setDefault(blockSize, 1024)
-
-  /** @group expertGetParam */
-  final def getBlockSize: Int = $(blockSize)
-}
 // scalastyle:on
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 002146f89e79a..2fb9a276be887 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -54,8 +54,7 @@ import org.apache.spark.util.random.XORShiftRandom
 /**
  * Common params for ALS and ALSModel.
  */
-private[recommendation] trait ALSModelParams extends Params with HasPredictionCol
-  with HasBlockSize {
+private[recommendation] trait ALSModelParams extends Params with HasPredictionCol {
   /**
    * Param for the column name for user ids. Ids must be integers. Other
    * numeric types are supported for this column, but will be cast to integers as long as they
@@ -126,8 +125,6 @@ private[recommendation] trait ALSModelParams extends Params with HasPredictionCo
 
   /** @group expertGetParam */
   def getColdStartStrategy: String = $(coldStartStrategy).toLowerCase(Locale.ROOT)
-
-  setDefault(blockSize -> 4096)
 }
 
 /**
@@ -291,15 +288,6 @@ class ALSModel private[ml] (
   @Since("2.2.0")
   def setColdStartStrategy(value: String): this.type = set(coldStartStrategy, value)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 4096.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-
   private val predict = udf { (featuresA: Seq[Float], featuresB: Seq[Float]) =>
     if (featuresA != null && featuresB != null) {
       var dotProduct = 0.0f
@@ -363,7 +351,7 @@ class ALSModel private[ml] (
    */
   @Since("2.2.0")
   def recommendForAllUsers(numItems: Int): DataFrame = {
-    recommendForAll(userFactors, itemFactors, $(userCol), $(itemCol), numItems, $(blockSize))
+    recommendForAll(userFactors, itemFactors, $(userCol), $(itemCol), numItems)
   }
 
   /**
@@ -378,7 +366,7 @@ class ALSModel private[ml] (
   @Since("2.3.0")
   def recommendForUserSubset(dataset: Dataset[_], numItems: Int): DataFrame = {
     val srcFactorSubset = getSourceFactorSubset(dataset, userFactors, $(userCol))
-    recommendForAll(srcFactorSubset, itemFactors, $(userCol), $(itemCol), numItems, $(blockSize))
+    recommendForAll(srcFactorSubset, itemFactors, $(userCol), $(itemCol), numItems)
   }
 
   /**
@@ -389,7 +377,7 @@ class ALSModel private[ml] (
    */
   @Since("2.2.0")
   def recommendForAllItems(numUsers: Int): DataFrame = {
-    recommendForAll(itemFactors, userFactors, $(itemCol), $(userCol), numUsers, $(blockSize))
+    recommendForAll(itemFactors, userFactors, $(itemCol), $(userCol), numUsers)
   }
 
   /**
@@ -404,7 +392,7 @@ class ALSModel private[ml] (
   @Since("2.3.0")
   def recommendForItemSubset(dataset: Dataset[_], numUsers: Int): DataFrame = {
     val srcFactorSubset = getSourceFactorSubset(dataset, itemFactors, $(itemCol))
-    recommendForAll(srcFactorSubset, userFactors, $(itemCol), $(userCol), numUsers, $(blockSize))
+    recommendForAll(srcFactorSubset, userFactors, $(itemCol), $(userCol), numUsers)
   }
 
   /**
@@ -453,12 +441,11 @@ class ALSModel private[ml] (
       dstFactors: DataFrame,
       srcOutputColumn: String,
       dstOutputColumn: String,
-      num: Int,
-      blockSize: Int): DataFrame = {
+      num: Int): DataFrame = {
     import srcFactors.sparkSession.implicits._
 
-    val srcFactorsBlocked = blockify(srcFactors.as[(Int, Array[Float])], blockSize)
-    val dstFactorsBlocked = blockify(dstFactors.as[(Int, Array[Float])], blockSize)
+    val srcFactorsBlocked = blockify(srcFactors.as[(Int, Array[Float])])
+    val dstFactorsBlocked = blockify(dstFactors.as[(Int, Array[Float])])
     val ratings = srcFactorsBlocked.crossJoin(dstFactorsBlocked)
       .as[(Seq[(Int, Array[Float])], Seq[(Int, Array[Float])])]
       .flatMap { case (srcIter, dstIter) =>
@@ -496,10 +483,11 @@ class ALSModel private[ml] (
 
   /**
    * Blockifies factors to improve the efficiency of cross join
+   * TODO: SPARK-20443 - expose blockSize as a param?
    */
   private def blockify(
       factors: Dataset[(Int, Array[Float])],
-      blockSize: Int): Dataset[Seq[(Int, Array[Float])]] = {
+      blockSize: Int = 4096): Dataset[Seq[(Int, Array[Float])]] = {
     import factors.sparkSession.implicits._
     factors.mapPartitions(_.grouped(blockSize))
   }
@@ -666,15 +654,6 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
   @Since("2.2.0")
   def setColdStartStrategy(value: String): this.type = set(coldStartStrategy, value)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 4096.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-
   /**
    * Sets both numUserBlocks and numItemBlocks to the specific value.
    *
@@ -704,7 +683,7 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
     instr.logDataset(dataset)
     instr.logParams(this, rank, numUserBlocks, numItemBlocks, implicitPrefs, alpha, userCol,
       itemCol, ratingCol, predictionCol, maxIter, regParam, nonnegative, checkpointInterval,
-      seed, intermediateStorageLevel, finalStorageLevel, blockSize)
+      seed, intermediateStorageLevel, finalStorageLevel)
 
     val (userFactors, itemFactors) = ALS.train(ratings, rank = $(rank),
       numUserBlocks = $(numUserBlocks), numItemBlocks = $(numItemBlocks),
@@ -715,8 +694,7 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
       checkpointInterval = $(checkpointInterval), seed = $(seed))
     val userDF = userFactors.toDF("id", "features")
     val itemDF = itemFactors.toDF("id", "features")
-    val model = new ALSModel(uid, $(rank), userDF, itemDF).setBlockSize($(blockSize))
-      .setParent(this)
+    val model = new ALSModel(uid, $(rank), userDF, itemDF).setParent(this)
     copyValues(model)
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index fc59da8a9c165..64e5e191ffd17 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -28,7 +28,7 @@ import org.apache.spark.SparkException
 import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{PipelineStage, PredictorParams}
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.linalg.BLAS._
 import org.apache.spark.ml.optim.WeightedLeastSquares
@@ -55,7 +55,7 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
 private[regression] trait LinearRegressionParams extends PredictorParams
     with HasRegParam with HasElasticNetParam with HasMaxIter with HasTol
     with HasFitIntercept with HasStandardization with HasWeightCol with HasSolver
-    with HasAggregationDepth with HasLoss with HasBlockSize {
+    with HasAggregationDepth with HasLoss {
 
   import LinearRegression._
 
@@ -316,15 +316,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
   def setEpsilon(value: Double): this.type = set(epsilon, value)
   setDefault(epsilon -> 1.35)
 
-  /**
-   * Set block size for stacking input data in matrices.
-   * Default is 1024.
-   *
-   * @group expertSetParam
-   */
-  @Since("3.0.0")
-  def setBlockSize(value: Int): this.type = set(blockSize, value)
-
   override protected def train(dataset: Dataset[_]): LinearRegressionModel = instrumented { instr =>
     // Extract the number of features before deciding optimization solver.
     val numFeatures = MetadataUtils.getNumFeatures(dataset, $(featuresCol))
@@ -363,6 +354,9 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
       return lrModel.setSummary(Some(trainingSummary))
     }
 
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
+
     val (featuresSummarizer, ySummarizer) = instances.treeAggregate(
       (Summarizer.createSummarizerBuffer("mean", "std"),
         Summarizer.createSummarizerBuffer("mean", "std", "count")))(
@@ -398,6 +392,7 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
             s"will be zeros and the intercept will be the mean of the label; as a result, " +
             s"training is not needed.")
         }
+        if (handlePersistence) instances.unpersist()
         val coefficients = Vectors.sparse(numFeatures, Seq.empty)
         val intercept = yMean
 
@@ -426,8 +421,8 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     // if y is constant (rawYStd is zero), then y cannot be scaled. In this case
     // setting yStd=abs(yMean) ensures that y is not scaled anymore in l-bfgs algorithm.
     val yStd = if (rawYStd > 0) rawYStd else math.abs(yMean)
-    val featuresMean = featuresSummarizer.mean.compressed
-    val featuresStd = featuresSummarizer.std.compressed
+    val featuresMean = featuresSummarizer.mean.toArray
+    val featuresStd = featuresSummarizer.std.toArray
     val bcFeaturesMean = instances.context.broadcast(featuresMean)
     val bcFeaturesStd = instances.context.broadcast(featuresStd)
 
@@ -447,36 +442,23 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
     val effectiveL1RegParam = $(elasticNetParam) * effectiveRegParam
     val effectiveL2RegParam = (1.0 - $(elasticNetParam)) * effectiveRegParam
 
+    val getFeaturesStd = (j: Int) => if (j >= 0 && j < numFeatures) featuresStd(j) else 0.0
     val regularization = if (effectiveL2RegParam != 0.0) {
       val shouldApply = (idx: Int) => idx >= 0 && idx < numFeatures
       Some(new L2Regularization(effectiveL2RegParam, shouldApply,
-        if ($(standardization)) None else Some(featuresStd.apply)))
+        if ($(standardization)) None else Some(getFeaturesStd)))
     } else {
       None
     }
 
-    val standardized = instances.map {
-      case Instance(label, weight, features) =>
-        val featuresStd = bcFeaturesStd.value
-        val array = Array.ofDim[Double](numFeatures)
-        features.foreachNonZero { (i, v) =>
-          val std = featuresStd(i)
-          if (std != 0) array(i) = v / std
-        }
-        Instance(label, weight, Vectors.dense(array))
-    }
-    val blocks = InstanceBlock.blokify(standardized, $(blockSize))
-      .persist(StorageLevel.MEMORY_AND_DISK)
-      .setName(s"training dataset (blockSize=${$(blockSize)})")
-
     val costFun = $(loss) match {
       case SquaredError =>
         val getAggregatorFunc = new LeastSquaresAggregator(yStd, yMean, $(fitIntercept),
           bcFeaturesStd, bcFeaturesMean)(_)
-        new RDDLossFunction(blocks, getAggregatorFunc, regularization, $(aggregationDepth))
+        new RDDLossFunction(instances, getAggregatorFunc, regularization, $(aggregationDepth))
       case Huber =>
-        val getAggregatorFunc = new HuberAggregator(numFeatures, $(fitIntercept), $(epsilon))(_)
-        new RDDLossFunction(blocks, getAggregatorFunc, regularization, $(aggregationDepth))
+        val getAggregatorFunc = new HuberAggregator($(fitIntercept), $(epsilon), bcFeaturesStd)(_)
+        new RDDLossFunction(instances, getAggregatorFunc, regularization, $(aggregationDepth))
     }
 
     val optimizer = $(loss) match {
@@ -542,7 +524,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
         throw new SparkException(msg)
       }
 
-      blocks.unpersist()
       bcFeaturesMean.destroy()
       bcFeaturesStd.destroy()
 
@@ -576,7 +557,7 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
             after the coefficients are converged. See the following discussion for detail.
             http://stats.stackexchange.com/questions/13617/how-is-the-intercept-computed-in-glmnet
             */
-            yMean - dot(Vectors.dense(rawCoefficients), featuresMean)
+            yMean - dot(Vectors.dense(rawCoefficients), Vectors.dense(featuresMean))
           case Huber => parameters(numFeatures)
         }
       } else {
@@ -591,6 +572,8 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
       (Vectors.dense(rawCoefficients).compressed, interceptValue, scaleValue, arrayBuilder.result())
     }
 
+    if (handlePersistence) instances.unpersist()
+
     val model = copyValues(new LinearRegressionModel(uid, coefficients, intercept, scale))
     // Handle possible missing or invalid prediction columns
     val (summaryModel, predictionColName) = model.findSummaryModelAndPredictionCol()
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
index f88f3fce61b33..21eb17dfaacb3 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
@@ -339,8 +339,10 @@ class LogisticRegressionWithLBFGS
         // Convert our input into a DataFrame
         val spark = SparkSession.builder().sparkContext(input.context).getOrCreate()
         val df = spark.createDataFrame(input.map(_.asML))
+        // Determine if we should cache the DF
+        val handlePersistence = input.getStorageLevel == StorageLevel.NONE
         // Train our model
-        val mlLogisticRegressionModel = lr.train(df)
+        val mlLogisticRegressionModel = lr.train(df, handlePersistence)
         // convert the model
         val weights = Vectors.dense(mlLogisticRegressionModel.coefficients.toArray)
         createModel(weights, mlLogisticRegressionModel.intercept)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
index 2b63dc259a14f..c2072cea11859 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
@@ -179,7 +179,7 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
   test("sparse coefficients in HingeAggregator") {
     val bcCoefficients = spark.sparkContext.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
     val bcFeaturesStd = spark.sparkContext.broadcast(Array(1.0))
-    val agg = new HingeAggregator(1, true)(bcCoefficients)
+    val agg = new HingeAggregator(bcFeaturesStd, true)(bcCoefficients)
     val thrown = withClue("LinearSVCAggregator cannot handle sparse coefficients") {
       intercept[IllegalArgumentException] {
         agg.add(Instance(1.0, 1.0, Vectors.dense(1.0)))
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 9e359ba098bfb..6d31e6efc7e1c 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -542,7 +542,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
   test("sparse coefficients in LogisticAggregator") {
     val bcCoefficientsBinary = spark.sparkContext.broadcast(Vectors.sparse(2, Array(0), Array(1.0)))
     val bcFeaturesStd = spark.sparkContext.broadcast(Array(1.0))
-    val binaryAgg = new LogisticAggregator(1, 2,
+    val binaryAgg = new LogisticAggregator(bcFeaturesStd, 2,
       fitIntercept = true, multinomial = false)(bcCoefficientsBinary)
     val thrownBinary = withClue("binary logistic aggregator cannot handle sparse coefficients") {
       intercept[IllegalArgumentException] {
@@ -552,7 +552,7 @@ class LogisticRegressionSuite extends MLTest with DefaultReadWriteTest {
     assert(thrownBinary.getMessage.contains("coefficients only supports dense"))
 
     val bcCoefficientsMulti = spark.sparkContext.broadcast(Vectors.sparse(6, Array(0), Array(1.0)))
-    val multinomialAgg = new LogisticAggregator(1, 3,
+    val multinomialAgg = new LogisticAggregator(bcFeaturesStd, 3,
       fitIntercept = true, multinomial = true)(bcCoefficientsMulti)
     val thrown = withClue("multinomial logistic aggregator cannot handle sparse coefficients") {
       intercept[IllegalArgumentException] {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
index d780bdf5f5dc8..5a74490058398 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/InstanceSuite.scala
@@ -42,36 +42,5 @@ class InstanceSuite extends SparkFunSuite{
       val o2 = ser.deserialize[OffsetInstance](ser.serialize(o))
       assert(o === o2)
     }
-
-    val block1 = InstanceBlock.fromInstances(Seq(instance1))
-    val block2 = InstanceBlock.fromInstances(Seq(instance1, instance2))
-    Seq(block1, block2).foreach { o =>
-      val o2 = ser.deserialize[InstanceBlock](ser.serialize(o))
-      assert(o.labels === o2.labels)
-      assert(o.weights === o2.weights)
-      assert(o.matrix === o2.matrix)
-    }
-  }
-
-  test("InstanceBlock: check correctness") {
-    val instance1 = Instance(19.0, 2.0, Vectors.dense(1.0, 7.0))
-    val instance2 = Instance(17.0, 1.0, Vectors.dense(0.0, 5.0).toSparse)
-    val instances = Seq(instance1, instance2)
-
-    val block = InstanceBlock.fromInstances(instances)
-    assert(block.size === 2)
-    assert(block.numFeatures === 2)
-    block.instanceIterator.zipWithIndex.foreach {
-      case (instance, i) =>
-        assert(instance.label === instances(i).label)
-        assert(instance.weight === instances(i).weight)
-        assert(instance.features.toArray === instances(i).features.toArray)
-    }
-    Seq(0, 1).foreach { i =>
-      val nzIter = block.getNonZeroIter(i)
-      val vec = Vectors.sparse(2, nzIter.toSeq)
-      assert(vec.toArray === instances(i).features.toArray)
-    }
   }
-
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
index c02a0a5e5e7d0..61b48ffa10944 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HingeAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(0.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(1.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantFeatureFiltered = standardize(Array(
+    )
+    instancesConstantFeatureFiltered = Array(
       Instance(0.0, 0.1, Vectors.dense(2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0)),
       Instance(2.0, 0.3, Vectors.dense(0.5))
-    ))
+    )
   }
 
    /** Get summary statistics for some data and create a new HingeAggregator. */
@@ -54,23 +54,12 @@ class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
       instances: Array[Instance],
       coefficients: Vector,
       fitIntercept: Boolean): HingeAggregator = {
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new HingeAggregator(instances.head.features.size, fitIntercept)(bcCoefficients)
-  }
-
-  private def standardize(instances: Array[Instance]): Array[Instance] = {
-    val (featuresSummarizer, _) =
+    val (featuresSummarizer, ySummarizer) =
       DifferentiableLossAggregatorSuite.getClassificationSummarizers(instances)
-    val stdArray = featuresSummarizer.variance.toArray.map(math.sqrt)
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
+    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
+    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
+    new HingeAggregator(bcFeaturesStd, fitIntercept)(bcCoefficients)
   }
 
   test("aggregator add method input size") {
@@ -171,21 +160,4 @@ class HingeAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(aggConstantFeatureBinary.gradient(1) == aggConstantFeatureBinaryFiltered.gradient(0))
   }
 
-  test("add instance block") {
-    val coefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-
-    val agg = getNewAggregator(instances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true)
-    instances.foreach(agg.add)
-
-    val agg2 = getNewAggregator(instances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true)
-    val block = InstanceBlock.fromInstances(instances)
-    agg2.add(block)
-
-    assert(agg.loss ~== agg2.loss relTol 1e-8)
-    assert(agg.gradient ~== agg2.gradient relTol 1e-8)
-  }
-
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala
index 7c544e99f88b5..718ffa230a749 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/HuberAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantFeatureFiltered = standardize(Array(
+    )
+    instancesConstantFeatureFiltered = Array(
       Instance(0.0, 0.1, Vectors.dense(2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0)),
       Instance(2.0, 0.3, Vectors.dense(0.5))
-    ))
+    )
   }
 
   /** Get summary statistics for some data and create a new HuberAggregator. */
@@ -56,28 +56,10 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
       fitIntercept: Boolean,
       epsilon: Double): HuberAggregator = {
     val (featuresSummarizer, _) = getRegressionSummarizers(instances)
-    val numFeatures = featuresSummarizer.variance.size
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
+    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
     val bcParameters = spark.sparkContext.broadcast(parameters)
-    new HuberAggregator(numFeatures, fitIntercept, epsilon)(bcParameters)
-  }
-
-  private def standardize(
-      instances: Array[Instance],
-      std: Array[Double] = null): Array[Instance] = {
-    val stdArray = if (std == null) {
-      getRegressionSummarizers(instances)._1.variance.toArray.map(math.sqrt)
-    } else {
-      std
-    }
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
+    new HuberAggregator(fitIntercept, epsilon, bcFeaturesStd)(bcParameters)
   }
 
   test("aggregator add method should check input size") {
@@ -173,15 +155,9 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
     val parametersFiltered = Vectors.dense(2.0, 3.0, 4.0)
     val aggConstantFeature = getNewAggregator(instancesConstantFeature, parameters,
       fitIntercept = true, epsilon = 1.35)
-    // std of instancesConstantFeature
-    val stdConstantFeature = getRegressionSummarizers(instancesConstantFeature)
-      ._1.variance.toArray.map(math.sqrt)
-    // Since 3.0.0, we start to standardize input outside of gradient computation,
-    // so here we use std of instancesConstantFeature to standardize instances
-    standardize(instances, stdConstantFeature).foreach(aggConstantFeature.add)
-
     val aggConstantFeatureFiltered = getNewAggregator(instancesConstantFeatureFiltered,
       parametersFiltered, fitIntercept = true, epsilon = 1.35)
+    instances.foreach(aggConstantFeature.add)
     instancesConstantFeatureFiltered.foreach(aggConstantFeatureFiltered.add)
     // constant features should not affect gradient
     def validateGradient(grad: Vector, gradFiltered: Vector): Unit = {
@@ -191,19 +167,4 @@ class HuberAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
     validateGradient(aggConstantFeature.gradient, aggConstantFeatureFiltered.gradient)
   }
-
-  test("add instance block") {
-    val paramWithIntercept = Vectors.dense(1.0, 2.0, 3.0, 4.0)
-    val agg1 = getNewAggregator(instances, paramWithIntercept,
-      fitIntercept = true, epsilon = 1.35)
-    instances.foreach(agg1.add)
-
-    val agg2 = getNewAggregator(instances, paramWithIntercept,
-      fitIntercept = true, epsilon = 1.35)
-    val block = InstanceBlock.fromInstances(instances)
-    agg2.add(block)
-
-    assert(agg1.loss ~== agg2.loss relTol 1e-8)
-    assert(agg1.gradient ~== agg2.gradient relTol 1e-8)
-  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala
index 5eb4e41c58262..35b6944624707 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LeastSquaresAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantLabel = standardize(Array(
+    )
+    instancesConstantLabel = Array(
       Instance(1.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(1.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
+    )
   }
 
   /** Get summary statistics for some data and create a new LeastSquaresAggregator. */
@@ -57,34 +57,15 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
     val (featuresSummarizer, ySummarizer) = getRegressionSummarizers(instances)
     val yStd = math.sqrt(ySummarizer.variance(0))
     val yMean = ySummarizer.mean(0)
-    val featuresStd = Vectors.dense(featuresSummarizer.variance.toArray.map(math.sqrt))
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
     val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
-    val featuresMean = featuresSummarizer.mean.asML
-    val bcFeaturesMean = spark.sparkContext.broadcast(featuresMean.compressed)
-    val bcCoefficients = spark.sparkContext.broadcast(coefficients.compressed)
+    val featuresMean = featuresSummarizer.mean
+    val bcFeaturesMean = spark.sparkContext.broadcast(featuresMean.toArray)
+    val bcCoefficients = spark.sparkContext.broadcast(coefficients)
     new LeastSquaresAggregator(yStd, yMean, fitIntercept, bcFeaturesStd,
       bcFeaturesMean)(bcCoefficients)
   }
 
-  private def standardize(
-      instances: Array[Instance],
-      std: Array[Double] = null): Array[Instance] = {
-    val stdArray = if (std == null) {
-      getRegressionSummarizers(instances)._1.variance.toArray.map(math.sqrt)
-    } else {
-      std
-    }
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
-  }
-
   test("aggregator add method input size") {
     val coefficients = Vectors.dense(1.0, 2.0)
     val agg = getNewAggregator(instances, coefficients, fitIntercept = true)
@@ -164,15 +145,9 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
 
   test("check with zero standard deviation") {
     val coefficients = Vectors.dense(1.0, 2.0)
-    // aggConstantFeature contains std of instancesConstantFeature, and the std of dim=0 is 0
     val aggConstantFeature = getNewAggregator(instancesConstantFeature, coefficients,
       fitIntercept = true)
-    // std of instancesConstantFeature
-    val stdConstantFeature = getRegressionSummarizers(instancesConstantFeature)
-      ._1.variance.toArray.map(math.sqrt)
-    // Since 3.0.0, we start to standardize input outside of gradient computation,
-    // so here we use std of instancesConstantFeature to standardize instances
-    standardize(instances, stdConstantFeature).foreach(aggConstantFeature.add)
+    instances.foreach(aggConstantFeature.add)
     // constant features should not affect gradient
     assert(aggConstantFeature.gradient(0) === 0.0)
 
@@ -182,17 +157,4 @@ class LeastSquaresAggregatorSuite extends SparkFunSuite with MLlibTestSparkConte
       }
     }
   }
-
-  test("add instance block") {
-    val coefficients = Vectors.dense(1.0, 2.0)
-    val agg1 = getNewAggregator(instances, coefficients, fitIntercept = true)
-    instances.foreach(agg1.add)
-
-    val agg2 = getNewAggregator(instances, coefficients, fitIntercept = true)
-    val block = InstanceBlock.fromInstances(instances)
-    agg2.add(block)
-
-    assert(agg1.loss ~== agg2.loss relTol 1e-8)
-    assert(agg1.gradient ~== agg2.gradient relTol 1e-8)
-  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
index 83718076dde7b..e699adcc14c03 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/optim/aggregator/LogisticAggregatorSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.ml.optim.aggregator
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.ml.feature.{Instance, InstanceBlock}
+import org.apache.spark.ml.feature.Instance
 import org.apache.spark.ml.linalg.{BLAS, Matrices, Vector, Vectors}
 import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.util.MLlibTestSparkContext
@@ -32,21 +32,21 @@ class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    instances = standardize(Array(
+    instances = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.5, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(4.0, 0.5))
-    ))
-    instancesConstantFeature = standardize(Array(
+    )
+    instancesConstantFeature = Array(
       Instance(0.0, 0.1, Vectors.dense(1.0, 2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0, 1.0)),
       Instance(2.0, 0.3, Vectors.dense(1.0, 0.5))
-    ))
-    instancesConstantFeatureFiltered = standardize(Array(
+    )
+    instancesConstantFeatureFiltered = Array(
       Instance(0.0, 0.1, Vectors.dense(2.0)),
       Instance(1.0, 0.5, Vectors.dense(1.0)),
       Instance(2.0, 0.3, Vectors.dense(0.5))
-    ))
+    )
   }
 
   /** Get summary statistics for some data and create a new LogisticAggregator. */
@@ -55,27 +55,13 @@ class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
       coefficients: Vector,
       fitIntercept: Boolean,
       isMultinomial: Boolean): LogisticAggregator = {
-    val (_, ySummarizer) =
+    val (featuresSummarizer, ySummarizer) =
       DifferentiableLossAggregatorSuite.getClassificationSummarizers(instances)
     val numClasses = ySummarizer.histogram.length
-    val numFeatures = instances.head.features.size
+    val featuresStd = featuresSummarizer.variance.toArray.map(math.sqrt)
+    val bcFeaturesStd = spark.sparkContext.broadcast(featuresStd)
     val bcCoefficients = spark.sparkContext.broadcast(coefficients)
-    new LogisticAggregator(numFeatures, numClasses, fitIntercept, isMultinomial)(bcCoefficients)
-  }
-
-  private def standardize(instances: Array[Instance]): Array[Instance] = {
-    val (featuresSummarizer, _) =
-      DifferentiableLossAggregatorSuite.getClassificationSummarizers(instances)
-    val stdArray = featuresSummarizer.variance.toArray.map(math.sqrt)
-    val numFeatures = stdArray.length
-    instances.map { case Instance(label, weight, features) =>
-      val standardized = Array.ofDim[Double](numFeatures)
-      features.foreachNonZero { (i, v) =>
-        val std = stdArray(i)
-        if (std != 0) standardized(i) = v / std
-      }
-      Instance(label, weight, Vectors.dense(standardized).compressed)
-    }
+    new LogisticAggregator(bcFeaturesStd, numClasses, fitIntercept, isMultinomial)(bcCoefficients)
   }
 
   test("aggregator add method input size") {
@@ -291,24 +277,4 @@ class LogisticAggregatorSuite extends SparkFunSuite with MLlibTestSparkContext {
     validateGradient(aggConstantFeatureBinary.gradient,
       aggConstantFeatureBinaryFiltered.gradient, 1)
   }
-
-  test("add instance block") {
-    val binaryInstances = instances.map { instance =>
-      if (instance.label <= 1.0) instance else Instance(0.0, instance.weight, instance.features)
-    }
-    val coefArray = Array(1.0, 2.0)
-    val intercept = 1.0
-
-    val agg = getNewAggregator(binaryInstances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true, isMultinomial = false)
-    binaryInstances.foreach(agg.add)
-
-    val agg2 = getNewAggregator(binaryInstances, Vectors.dense(coefArray ++ Array(intercept)),
-      fitIntercept = true, isMultinomial = false)
-    val block = InstanceBlock.fromInstances(binaryInstances)
-    agg2.add(block)
-
-    assert(agg.loss ~== agg2.loss relTol 1e-8)
-    assert(agg.gradient ~== agg2.gradient relTol 1e-8)
-  }
 }
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 5ee42318afd45..5ab8e606bda03 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -165,8 +165,7 @@ def predictProbability(self, value):
 
 
 class _LinearSVCParams(_JavaClassifierParams, HasRegParam, HasMaxIter, HasFitIntercept, HasTol,
-                       HasStandardization, HasWeightCol, HasAggregationDepth, HasThreshold,
-                       HasBlockSize):
+                       HasStandardization, HasWeightCol, HasAggregationDepth, HasThreshold):
     """
     Params for :py:class:`LinearSVC` and :py:class:`LinearSVCModel`.
 
@@ -215,8 +214,6 @@ class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable
     LinearSVCModel...
     >>> model.getThreshold()
     0.5
-    >>> model.getBlockSize()
-    1024
     >>> model.coefficients
     DenseVector([0.0, -0.2792, -0.1833])
     >>> model.intercept
@@ -255,19 +252,18 @@ class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable
     def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction",
                  fitIntercept=True, standardization=True, threshold=0.0, weightCol=None,
-                 aggregationDepth=2, blockSize=1024):
+                 aggregationDepth=2):
         """
         __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \
                  fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \
-                 aggregationDepth=2, blockSize=1024):
+                 aggregationDepth=2):
         """
         super(LinearSVC, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LinearSVC", self.uid)
         self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
-                         standardization=True, threshold=0.0, aggregationDepth=2,
-                         blockSize=1024)
+                         standardization=True, threshold=0.0, aggregationDepth=2)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -276,12 +272,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
     def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction",
                   fitIntercept=True, standardization=True, threshold=0.0, weightCol=None,
-                  aggregationDepth=2, blockSize=1024):
+                  aggregationDepth=2):
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, tol=1e-6, rawPredictionCol="rawPrediction", \
                   fitIntercept=True, standardization=True, threshold=0.0, weightCol=None, \
-                  aggregationDepth=2, blockSize=1024):
+                  aggregationDepth=2):
         Sets params for Linear SVM Classifier.
         """
         kwargs = self._input_kwargs
@@ -346,13 +342,6 @@ def setAggregationDepth(self, value):
         """
         return self._set(aggregationDepth=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class LinearSVCModel(JavaClassificationModel, _LinearSVCParams, JavaMLWritable, JavaMLReadable):
     """
@@ -388,7 +377,7 @@ def intercept(self):
 class _LogisticRegressionParams(_JavaProbabilisticClassifierParams, HasRegParam,
                                 HasElasticNetParam, HasMaxIter, HasFitIntercept, HasTol,
                                 HasStandardization, HasWeightCol, HasAggregationDepth,
-                                HasThreshold, HasBlockSize):
+                                HasThreshold):
     """
     Params for :py:class:`LogisticRegression` and :py:class:`LogisticRegressionModel`.
 
@@ -570,8 +559,6 @@ class LogisticRegression(JavaProbabilisticClassifier, _LogisticRegressionParams,
     10
     >>> blor.clear(blor.maxIter)
     >>> blorModel = blor.fit(bdf)
-    >>> blorModel.getBlockSize()
-    1024
     >>> blorModel.setFeaturesCol("features")
     LogisticRegressionModel...
     >>> blorModel.setProbabilityCol("newProbability")
@@ -640,7 +627,7 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
                  rawPredictionCol="rawPrediction", standardization=True, weightCol=None,
                  aggregationDepth=2, family="auto",
                  lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None,
-                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
 
         """
         __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
@@ -649,14 +636,13 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
                  rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \
                  aggregationDepth=2, family="auto", \
                  lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \
-                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                 lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
         If the threshold and thresholds Params are both set, they must be equivalent.
         """
         super(LogisticRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LogisticRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto",
-                         blockSize=1024)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
         self._checkThresholdConsistency()
@@ -669,7 +655,7 @@ def setParams(self, featuresCol="features", labelCol="label", predictionCol="pre
                   rawPredictionCol="rawPrediction", standardization=True, weightCol=None,
                   aggregationDepth=2, family="auto",
                   lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None,
-                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
@@ -677,7 +663,7 @@ def setParams(self, featuresCol="features", labelCol="label", predictionCol="pre
                   rawPredictionCol="rawPrediction", standardization=True, weightCol=None, \
                   aggregationDepth=2, family="auto", \
                   lowerBoundsOnCoefficients=None, upperBoundsOnCoefficients=None, \
-                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, blockSize=1024):
+                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None):
         Sets params for logistic regression.
         If the threshold and thresholds Params are both set, they must be equivalent.
         """
@@ -772,13 +758,6 @@ def setAggregationDepth(self, value):
         """
         return self._set(aggregationDepth=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class LogisticRegressionModel(JavaProbabilisticClassificationModel, _LogisticRegressionParams,
                               JavaMLWritable, JavaMLReadable, HasTrainingSummary):
@@ -2174,7 +2153,7 @@ def sigma(self):
 
 
 class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, HasMaxIter,
-                                  HasTol, HasStepSize, HasSolver, HasBlockSize):
+                                  HasTol, HasStepSize, HasSolver):
     """
     Params for :py:class:`MultilayerPerceptronClassifier`.
 
@@ -2185,6 +2164,11 @@ class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, H
                    "E.g., Array(780, 100, 10) means 780 inputs, one hidden layer with 100 " +
                    "neurons and output layer of 10 neurons.",
                    typeConverter=TypeConverters.toListInt)
+    blockSize = Param(Params._dummy(), "blockSize", "Block size for stacking input data in " +
+                      "matrices. Data is stacked within partitions. If block size is more than " +
+                      "remaining data in a partition then it is adjusted to the size of this " +
+                      "data. Recommended size is between 10 and 1000, default is 128.",
+                      typeConverter=TypeConverters.toInt)
     solver = Param(Params._dummy(), "solver", "The solver algorithm for optimization. Supported " +
                    "options: l-bfgs, gd.", typeConverter=TypeConverters.toString)
     initialWeights = Param(Params._dummy(), "initialWeights", "The initial weights of the model.",
@@ -2197,6 +2181,13 @@ def getLayers(self):
         """
         return self.getOrDefault(self.layers)
 
+    @since("1.6.0")
+    def getBlockSize(self):
+        """
+        Gets the value of blockSize or its default value.
+        """
+        return self.getOrDefault(self.blockSize)
+
     @since("2.0.0")
     def getInitialWeights(self):
         """
@@ -2220,17 +2211,11 @@ class MultilayerPerceptronClassifier(JavaProbabilisticClassifier, _MultilayerPer
     ...     (1.0, Vectors.dense([0.0, 1.0])),
     ...     (1.0, Vectors.dense([1.0, 0.0])),
     ...     (0.0, Vectors.dense([1.0, 1.0]))], ["label", "features"])
-    >>> mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
+    >>> mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], blockSize=1, seed=123)
     >>> mlp.setMaxIter(100)
     MultilayerPerceptronClassifier...
     >>> mlp.getMaxIter()
     100
-    >>> mlp.getBlockSize()
-    128
-    >>> mlp.setBlockSize(1)
-    MultilayerPerceptronClassifier...
-    >>> mlp.getBlockSize()
-    1
     >>> model = mlp.fit(df)
     >>> model.setFeaturesCol("features")
     MultilayerPerceptronClassificationModel...
diff --git a/python/pyspark/ml/param/_shared_params_code_gen.py b/python/pyspark/ml/param/_shared_params_code_gen.py
index fb4d55d57a2df..ded3ca84b30f2 100644
--- a/python/pyspark/ml/param/_shared_params_code_gen.py
+++ b/python/pyspark/ml/param/_shared_params_code_gen.py
@@ -164,10 +164,7 @@ def get$Name(self):
          "'euclidean'", "TypeConverters.toString"),
         ("validationIndicatorCol", "name of the column that indicates whether each row is for " +
          "training or for validation. False indicates training; true indicates validation.",
-         None, "TypeConverters.toString"),
-        ("blockSize", "block size for stacking input data in matrices. Data is stacked within "
-         "partitions. If block size is more than remaining data in a partition then it is "
-         "adjusted to the size of this data.", "1024", "TypeConverters.toInt")]
+         None, "TypeConverters.toString")]
 
     code = []
     for name, doc, defaultValueStr, typeConverter in shared:
diff --git a/python/pyspark/ml/param/shared.py b/python/pyspark/ml/param/shared.py
index 456463580878c..8fc115691f1ab 100644
--- a/python/pyspark/ml/param/shared.py
+++ b/python/pyspark/ml/param/shared.py
@@ -580,21 +580,3 @@ def getValidationIndicatorCol(self):
         Gets the value of validationIndicatorCol or its default value.
         """
         return self.getOrDefault(self.validationIndicatorCol)
-
-
-class HasBlockSize(Params):
-    """
-    Mixin for param blockSize: block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.
-    """
-
-    blockSize = Param(Params._dummy(), "blockSize", "block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.", typeConverter=TypeConverters.toInt)
-
-    def __init__(self):
-        super(HasBlockSize, self).__init__()
-        self._setDefault(blockSize=1024)
-
-    def getBlockSize(self):
-        """
-        Gets the value of blockSize or its default value.
-        """
-        return self.getOrDefault(self.blockSize)
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index fe571e25c05f5..ee276962c898b 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -28,7 +28,7 @@
 
 
 @inherit_doc
-class _ALSModelParams(HasPredictionCol, HasBlockSize):
+class _ALSModelParams(HasPredictionCol):
     """
     Params for :py:class:`ALS` and :py:class:`ALSModel`.
 
@@ -223,8 +223,6 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     0.1
     >>> als.clear(als.regParam)
     >>> model = als.fit(df)
-    >>> model.getBlockSize()
-    4096
     >>> model.getUserCol()
     'user'
     >>> model.setUserCol("user")
@@ -284,13 +282,13 @@ def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemB
                  implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None,
                  ratingCol="rating", nonnegative=False, checkpointInterval=10,
                  intermediateStorageLevel="MEMORY_AND_DISK",
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096):
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan"):
         """
         __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10, \
                  implicitPrefs=false, alpha=1.0, userCol="user", itemCol="item", seed=None, \
                  ratingCol="rating", nonnegative=false, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", lockSize=4096)
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
         """
         super(ALS, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid)
@@ -298,8 +296,7 @@ def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemB
                          implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
                          ratingCol="rating", nonnegative=False, checkpointInterval=10,
                          intermediateStorageLevel="MEMORY_AND_DISK",
-                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan",
-                         blockSize=4096)
+                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -309,13 +306,13 @@ def setParams(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItem
                   implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None,
                   ratingCol="rating", nonnegative=False, checkpointInterval=10,
                   intermediateStorageLevel="MEMORY_AND_DISK",
-                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096):
+                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan"):
         """
         setParams(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10, \
                  implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None, \
                  ratingCol="rating", nonnegative=False, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096)
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
         Sets params for ALS.
         """
         kwargs = self._input_kwargs
@@ -446,13 +443,6 @@ def setSeed(self, value):
         """
         return self._set(seed=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class ALSModel(JavaModel, _ALSModelParams, JavaMLWritable, JavaMLReadable):
     """
@@ -489,13 +479,6 @@ def setPredictionCol(self, value):
         """
         return self._set(predictionCol=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
     @property
     @since("1.4.0")
     def rank(self):
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index a74ba3485bcd9..a4c97827d55de 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -62,7 +62,7 @@ class JavaRegressionModel(JavaPredictionModel, _JavaPredictorParams):
 
 class _LinearRegressionParams(_JavaPredictorParams, HasRegParam, HasElasticNetParam, HasMaxIter,
                               HasTol, HasFitIntercept, HasStandardization, HasWeightCol, HasSolver,
-                              HasAggregationDepth, HasLoss, HasBlockSize):
+                              HasAggregationDepth, HasLoss):
     """
     Params for :py:class:`LinearRegression` and :py:class:`LinearRegressionModel`.
 
@@ -124,8 +124,6 @@ class LinearRegression(JavaRegressor, _LinearRegressionParams, JavaMLWritable, J
     >>> lr.setRegParam(0.0)
     LinearRegression...
     >>> model = lr.fit(df)
-    >>> model.getBlockSize()
-    1024
     >>> model.setFeaturesCol("features")
     LinearRegressionModel...
     >>> model.setPredictionCol("newPrediction")
@@ -171,18 +169,17 @@ class LinearRegression(JavaRegressor, _LinearRegressionParams, JavaMLWritable, J
     def __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                  standardization=True, solver="auto", weightCol=None, aggregationDepth=2,
-                 loss="squaredError", epsilon=1.35, blockSize=1024):
+                 loss="squaredError", epsilon=1.35):
         """
         __init__(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                  standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \
-                 loss="squaredError", epsilon=1.35, blockSize=1024)
+                 loss="squaredError", epsilon=1.35)
         """
         super(LinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.LinearRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35,
-                         blockSize=1024)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -191,12 +188,12 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
     def setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction",
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True,
                   standardization=True, solver="auto", weightCol=None, aggregationDepth=2,
-                  loss="squaredError", epsilon=1.35, blockSize=1024):
+                  loss="squaredError", epsilon=1.35):
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \
                   standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \
-                  loss="squaredError", epsilon=1.35, blockSize=1024)
+                  loss="squaredError", epsilon=1.35)
         Sets params for linear regression.
         """
         kwargs = self._input_kwargs
@@ -272,13 +269,6 @@ def setLoss(self, value):
         """
         return self._set(lossType=value)
 
-    @since("3.0.0")
-    def setBlockSize(self, value):
-        """
-        Sets the value of :py:attr:`blockSize`.
-        """
-        return self._set(blockSize=value)
-
 
 class LinearRegressionModel(JavaRegressionModel, _LinearRegressionParams, GeneralJavaMLWritable,
                             JavaMLReadable, HasTrainingSummary):

From b37b0855be0d7a4905ef3cb78e16a421e32c8ba0 Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Tue, 25 Feb 2020 12:28:31 +0800
Subject: [PATCH 0156/1280] [SPARK-30936][CORE] Set FAIL_ON_UNKNOWN_PROPERTIES
 to false by default to parse Spark events

### What changes were proposed in this pull request?

Set `FAIL_ON_UNKNOWN_PROPERTIES` to `false` in `JsonProtocol` to allow ignore unknown fields in a Spark event. After this change, if we add new fields to a Spark event parsed by `ObjectMapper`, the event json string generated by a new Spark version can still be read by an old Spark History Server.

Since Spark History Server is an extra service, it usually takes time to upgrade, and it's possible that a Spark application is upgraded before SHS. Forwards-compatibility will allow an old SHS to support new Spark applications (may lose some new features but most of functions should still work).

### Why are the changes needed?

`JsonProtocol` is supposed to provide strong backwards-compatibility and forwards-compatibility guarantees: any version of Spark should be able to read JSON output written by any other version, including newer versions.

However, the forwards-compatibility guarantee is broken for events parsed by `ObjectMapper`. If a new field is added to an event parsed by `ObjectMapper` (e.g., https://github.com/apache/spark/commit/6dc5921e66d56885b95c07e56e687f9f6c1eaca7#diff-dc5c7a41fbb7479cef48b67eb41ad254R33), the event json string generated by a new Spark version cannot be parsed by an old version of SHS right now.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

The new added tests.

Closes #27680 from zsxwing/SPARK-30936.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3126557b0763998ff893bf382089467f2a0d6ae5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/util/JsonProtocol.scala  |  3 +-
 .../apache/spark/util/JsonProtocolSuite.scala | 24 +++++++++++++++
 .../StreamingQueryListenerSuite.scala         | 29 ++++++++++---------
 3 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 4d89c4f079f29..aee986230d4b5 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -22,7 +22,7 @@ import java.util.{Properties, UUID}
 import scala.collection.JavaConverters._
 import scala.collection.Map
 
-import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import org.json4s.DefaultFormats
 import org.json4s.JsonAST._
@@ -59,6 +59,7 @@ private[spark] object JsonProtocol {
   private implicit val format = DefaultFormats
 
   private val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
 
   /** ------------------------------------------------- *
    * JSON serialization methods for SparkListenerEvents |
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index a2a4b3aa974fc..f2b9f97d6cf7d 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -481,6 +481,28 @@ class JsonProtocolSuite extends SparkFunSuite {
     testAccumValue(Some("anything"), blocks, JString(blocks.toString))
     testAccumValue(Some("anything"), 123, JString("123"))
   }
+
+  test("SPARK-30936: forwards compatibility - ignore unknown fields") {
+    val expected = TestListenerEvent("foo", 123)
+    val unknownFieldsJson =
+      """{
+        |  "Event" : "org.apache.spark.util.TestListenerEvent",
+        |  "foo" : "foo",
+        |  "bar" : 123,
+        |  "unknown" : "unknown"
+        |}""".stripMargin
+    assert(JsonProtocol.sparkEventFromJson(parse(unknownFieldsJson)) === expected)
+  }
+
+  test("SPARK-30936: backwards compatibility - set default values for missing fields") {
+    val expected = TestListenerEvent("foo", 0)
+    val unknownFieldsJson =
+      """{
+        |  "Event" : "org.apache.spark.util.TestListenerEvent",
+        |  "foo" : "foo"
+        |}""".stripMargin
+    assert(JsonProtocol.sparkEventFromJson(parse(unknownFieldsJson)) === expected)
+  }
 }
 
 
@@ -2310,3 +2332,5 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |}
     """.stripMargin
 }
+
+case class TestListenerEvent(foo: String, bar: Int) extends SparkListenerEvent
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index 9d0f829ac9684..6bb1646becf22 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -382,28 +382,27 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
   }
 
-  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2.0.0") {
+  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_0") {
     // query-event-logs-version-2.0.0.txt has all types of events generated by
-    // Structured Streaming in Spark 2.0.0.
+    // Structured Streaming in Spark 2.0.0. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.0.txt")
+    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.0.txt", 1)
   }
 
-  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2.0.1") {
+  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_1") {
     // query-event-logs-version-2.0.1.txt has all types of events generated by
-    // Structured Streaming in Spark 2.0.1.
+    // Structured Streaming in Spark 2.0.1. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.1.txt")
+    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.1.txt", 1)
   }
 
-  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2.0.2") {
+  testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_2") {
     // query-event-logs-version-2.0.2.txt has all types of events generated by
-    // Structured Streaming in Spark 2.0.2.
-    // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
-    // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.2.txt")
+    // Structured Streaming in Spark 2.0.2. SPARK-18516 refactored Structured Streaming query events
+    // in 2.1.0. This test is to verify we are able to load events generated by Spark 2.0.2.
+    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.2.txt", 5)
   }
 
   test("listener propagates observable metrics") {
@@ -463,7 +462,9 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
   }
 
-  private def testReplayListenerBusWithBorkenEventJsons(fileName: String): Unit = {
+  private def testReplayListenerBusWithBorkenEventJsons(
+      fileName: String,
+      expectedEventSize: Int): Unit = {
     val input = getClass.getResourceAsStream(s"/structured-streaming/$fileName")
     val events = mutable.ArrayBuffer[SparkListenerEvent]()
     try {
@@ -479,8 +480,8 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
       replayer.addListener(new SparkListener {})
       replayer.replay(input, fileName)
       // SparkListenerApplicationEnd is the only valid event
-      assert(events.size === 1)
-      assert(events(0).isInstanceOf[SparkListenerApplicationEnd])
+      assert(events.size === expectedEventSize)
+      assert(events.last.isInstanceOf[SparkListenerApplicationEnd])
     } finally {
       input.close()
     }

From d0811c79fa47583e64662a09f2729d446c8f621d Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Tue, 25 Feb 2020 13:04:28 +0800
Subject: [PATCH 0157/1280] [SPARK-30885][SQL] V1 table name should be fully
 qualified if catalog name is provided

For the following:
```
CREATE TABLE t USING json AS SELECT 1 AS i
SELECT * FROM spark_catalog.t
```
`spark_catalog.t` is resolved to `spark_catalog.default.t` assuming the current namespace is `default`. However, this is not consistent with V2 behavior where the namespace must be specified if the catalog name is provided. This PR proposes to fix this inconsistency.

To be consistent with V2 table naming scheme in SQL commands.

Yes, now the user has to specify the namespace if the catalog name is provided. For example,
```
SELECT * FROM spark_catalog.t # Will throw AnalysisException with 'Session catalog cannot have an empty namespace: spark_catalog.t'
SELECT * FROM spark_catalog.default.t # OK
```

Added new tests

Closes #27642 from imback82/disallow_spark_catalog_wihtout_db.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 0fd4fa70c8e0595dac423c203b1c61ddcc70565e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../analysis/ResolveSessionCatalog.scala      |  51 ++--
 .../datasources/v2/V2SessionCatalog.scala     |   7 +-
 .../sql-tests/results/describe.sql.out        |   2 +-
 .../sql-tests/results/explain.sql.out         |   2 +-
 .../results/postgreSQL/create_view.sql.out    |  24 +-
 .../results/show-create-table.sql.out         |  24 +-
 .../spark/sql/ShowCreateTableSuite.scala      |   2 +-
 .../spark/sql/connector/AlterTableTests.scala | 232 +++++++++++-------
 ...SourceV2DataFrameSessionCatalogSuite.scala |   7 +-
 .../DataSourceV2SQLSessionCatalogSuite.scala  |   4 +-
 .../sql/connector/DataSourceV2SQLSuite.scala  |  83 ++++---
 .../SupportsCatalogOptionsSuite.scala         |  16 +-
 .../connector/TestV2SessionCatalogBase.scala  |  23 +-
 .../spark/sql/execution/SQLViewSuite.scala    |   6 +-
 .../sql/execution/command/DDLSuite.scala      |  18 +-
 .../spark/sql/hive/CachedTableSuite.scala     |  13 +-
 .../sql/hive/HiveShowCreateTableSuite.scala   |   2 +-
 .../sql/hive/execution/HiveSQLViewSuite.scala |   4 +-
 18 files changed, 286 insertions(+), 234 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 62ebb1c1492ab..9941f5666b8e2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -227,20 +227,9 @@ class ResolveSessionCatalog(
     case DescribeRelation(ResolvedView(ident), partitionSpec, isExtended) =>
       DescribeTableCommand(ident.asTableIdentifier, partitionSpec, isExtended)
 
-    case DescribeColumnStatement(
-        SessionCatalogAndTable(catalog, tbl), colNameParts, isExtended) =>
-      loadTable(catalog, tbl.asIdentifier).collect {
-        // `V1Table` also includes permanent views.
-        case v1Table: V1Table =>
-          DescribeColumnCommand(tbl.asTableIdentifier, colNameParts, isExtended)
-      }.getOrElse {
-        if (isTempView(tbl)) {
-          // v1 DESCRIBE COLUMN supports temp view.
-          DescribeColumnCommand(tbl.asTableIdentifier, colNameParts, isExtended)
-        } else {
-          throw new AnalysisException("Describing columns is not supported for v2 tables.")
-        }
-      }
+    case DescribeColumnStatement(tbl, colNameParts, isExtended) =>
+      val name = parseTempViewOrV1Table(tbl, "Describing columns")
+      DescribeColumnCommand(name.asTableIdentifier, colNameParts, isExtended)
 
     // For CREATE TABLE [AS SELECT], we should use the v1 command if the catalog is resolved to the
     // session catalog and the table provider is not v2.
@@ -373,7 +362,7 @@ class ResolveSessionCatalog(
       }
 
     case AnalyzeColumnStatement(tbl, columnNames, allColumns) =>
-      val v1TableName = parseV1Table(tbl, "ANALYZE TABLE")
+      val v1TableName = parseTempViewOrV1Table(tbl, "ANALYZE TABLE")
       AnalyzeColumnCommand(v1TableName.asTableIdentifier, columnNames, allColumns)
 
     case RepairTableStatement(tbl) =>
@@ -392,8 +381,8 @@ class ResolveSessionCatalog(
         partition)
 
     case ShowCreateTableStatement(tbl, asSerde) if !asSerde =>
-      val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE")
-      ShowCreateTableCommand(v1TableName.asTableIdentifier)
+      val name = parseTempViewOrV1Table(tbl, "SHOW CREATE TABLE")
+      ShowCreateTableCommand(name.asTableIdentifier)
 
     case ShowCreateTableStatement(tbl, asSerde) if asSerde =>
       val v1TableName = parseV1Table(tbl, "SHOW CREATE TABLE AS SERDE")
@@ -426,20 +415,27 @@ class ResolveSessionCatalog(
         partitionSpec)
 
     case ShowColumnsStatement(tbl, ns) =>
+      if (ns.isDefined && ns.get.length > 1) {
+        throw new AnalysisException(
+          s"Namespace name should have only one part if specified: ${ns.get.quoted}")
+      }
+      // Use namespace only if table name doesn't specify it. If namespace is already specified
+      // in the table name, it's checked against the given namespace below.
+      val nameParts = if (ns.isDefined && tbl.length == 1) {
+        ns.get ++ tbl
+      } else {
+        tbl
+      }
       val sql = "SHOW COLUMNS"
-      val v1TableName = parseV1Table(tbl, sql).asTableIdentifier
+      val v1TableName = parseTempViewOrV1Table(nameParts, sql).asTableIdentifier
       val resolver = conf.resolver
       val db = ns match {
-        case Some(db) if (v1TableName.database.exists(!resolver(_, db.head))) =>
+        case Some(db) if v1TableName.database.exists(!resolver(_, db.head)) =>
           throw new AnalysisException(
             s"SHOW COLUMNS with conflicting databases: " +
               s"'${db.head}' != '${v1TableName.database.get}'")
         case _ => ns.map(_.head)
       }
-      if (ns.isDefined && ns.get.length > 1) {
-        throw new AnalysisException(
-          s"Namespace name should have only one part if specified: ${ns.get.quoted}")
-      }
       ShowColumnsCommand(db, v1TableName)
 
     case AlterTableRecoverPartitionsStatement(tbl) =>
@@ -636,14 +632,7 @@ class ResolveSessionCatalog(
   object SessionCatalogAndTable {
     def unapply(nameParts: Seq[String]): Option[(CatalogPlugin, Seq[String])] = nameParts match {
       case SessionCatalogAndIdentifier(catalog, ident) =>
-        if (nameParts.length == 1) {
-          // If there is only one name part, it means the current catalog is the session catalog.
-          // Here we return the original name part, to keep the error message unchanged for
-          // v1 commands.
-          Some(catalog -> nameParts)
-        } else {
-          Some(catalog -> ident.asMultipartIdentifier)
-        }
+        Some(catalog -> ident.asMultipartIdentifier)
       case _ => None
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
index cef9b5f675889..2ed33b867183b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala
@@ -167,14 +167,15 @@ class V2SessionCatalog(catalog: SessionCatalog, conf: SQLConf)
   }
 
   implicit class TableIdentifierHelper(ident: Identifier) {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
+
     def asTableIdentifier: TableIdentifier = {
       ident.namespace match {
         case Array(db) =>
           TableIdentifier(ident.name, Some(db))
-        case Array() =>
-          TableIdentifier(ident.name, Some(catalog.getCurrentDatabase))
         case _ =>
-          throw new NoSuchTableException(ident)
+          throw new NoSuchTableException(
+            s"V2 session catalog requires a single-part namespace: ${ident.quoted}")
       }
     }
   }
diff --git a/sql/core/src/test/resources/sql-tests/results/describe.sql.out b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
index a7ef828e4d5d2..a7de033e3a1ac 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe.sql.out
@@ -561,7 +561,7 @@ struct<plan:string>
 -- !query output
 == Physical Plan ==
 Execute DescribeColumnCommand
-   +- DescribeColumnCommand `t`, [b], false
+   +- DescribeColumnCommand `default`.`t`, [b], false
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index 40b26e6fb64ce..06226f1274863 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -879,7 +879,7 @@ Execute CreateViewCommand (1)
 Output: []
       
 (2) CreateViewCommand 
-Arguments: `explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
+Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
       
 (3) UnresolvedRelation 
 Arguments: [explain_temp1]
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
index 436b33ce43980..85ce978657844 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
@@ -266,7 +266,7 @@ CREATE VIEW v1_temp AS SELECT * FROM temp_table
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v1_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v1_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -371,7 +371,7 @@ CREATE VIEW v4_temp AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v4_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v4_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -383,7 +383,7 @@ CREATE VIEW v5_temp AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v5_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v5_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -542,7 +542,7 @@ CREATE VIEW v6_temp AS SELECT * FROM base_table WHERE id IN (SELECT id FROM temp
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v6_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v6_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -551,7 +551,7 @@ CREATE VIEW v7_temp AS SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM tem
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v7_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v7_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -560,7 +560,7 @@ CREATE VIEW v8_temp AS SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM temp
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v8_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v8_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -569,7 +569,7 @@ CREATE VIEW v9_temp AS SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `v9_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `temp_view_test`.`v9_temp` by referencing a temporary view temp_table. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -678,7 +678,7 @@ CREATE VIEW temporal1 AS SELECT * FROM t1 CROSS JOIN tt
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal1` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal1` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -719,7 +719,7 @@ CREATE VIEW temporal2 AS SELECT * FROM t1 INNER JOIN tt ON t1.num = tt.num2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal2` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal2` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -760,7 +760,7 @@ CREATE VIEW temporal3 AS SELECT * FROM t1 LEFT JOIN tt ON t1.num = tt.num2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal3` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal3` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -801,7 +801,7 @@ CREATE VIEW temporal4 AS SELECT * FROM t1 LEFT JOIN tt ON t1.num = tt.num2 AND t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal4` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal4` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
@@ -810,7 +810,7 @@ CREATE VIEW temporal5 AS SELECT * FROM t1 WHERE num IN (SELECT num FROM t1 WHERE
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Not allowed to create a permanent view `temporal5` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
+Not allowed to create a permanent view `testviewschm2`.`temporal5` by referencing a temporary view tt. Please create a temp view instead by CREATE TEMP VIEW;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
index e8ee07171651d..f5ca1eff9f0c3 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
@@ -15,7 +15,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -44,7 +44,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -75,7 +75,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -105,7 +105,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -135,7 +135,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `b` STRING,
   `c` INT,
   `a` INT)
@@ -165,7 +165,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -197,7 +197,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -227,7 +227,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` INT,
   `b` STRING,
   `c` INT)
@@ -257,7 +257,7 @@ SHOW CREATE TABLE tbl
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE TABLE `tbl` (
+CREATE TABLE `default`.`tbl` (
   `a` FLOAT,
   `b` DECIMAL(10,0),
   `c` DECIMAL(10,0),
@@ -295,7 +295,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE VIEW `view_SPARK_30302`(
+CREATE VIEW `default`.`view_SPARK_30302`(
   `aaa`,
   `bbb`)
 AS SELECT a, b FROM tbl
@@ -324,7 +324,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE VIEW `view_SPARK_30302`(
+CREATE VIEW `default`.`view_SPARK_30302`(
   `aaa` COMMENT 'comment with \'quoted text\' for aaa',
   `bbb`)
 COMMENT 'This is a comment with \'quoted text\' for view'
@@ -354,7 +354,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE
 -- !query schema
 struct<createtab_stmt:string>
 -- !query output
-CREATE VIEW `view_SPARK_30302`(
+CREATE VIEW `default`.`view_SPARK_30302`(
   `aaa`,
   `bbb`)
 TBLPROPERTIES (
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
index b3b94f8be0d17..04257642fac81 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
@@ -173,7 +173,7 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
       val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>)"
       sql(s"$createTable USING json")
       val shownDDL = getShowDDL("SHOW CREATE TABLE t1")
-      assert(shownDDL == createTable)
+      assert(shownDDL == "CREATE TABLE `default`.`t1` (`a` STRUCT<`b`: STRING>)")
 
       checkCreateTable("t1")
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
index 64188f71fb202..436862920f53f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/AlterTableTests.scala
@@ -67,9 +67,10 @@ trait AlterTableTests extends SharedSparkSession {
       assert(exc.getMessage.contains("Unsupported table change"))
       assert(exc.getMessage.contains("Cannot drop all fields")) // from the implementation
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", IntegerType))
     }
   }
@@ -80,9 +81,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN data string")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", IntegerType).add("data", StringType))
     }
   }
@@ -93,9 +95,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN data string NOT NULL")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(
         StructField("id", IntegerType),
         StructField("data", StringType, nullable = false))))
@@ -108,9 +111,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN data string COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(
         StructField("id", IntegerType),
         StructField("data", StringType).withComment("doc"))))
@@ -136,12 +140,13 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (point struct<x: int>) USING $v2Format")
 
       sql(s"ALTER TABLE $t ADD COLUMN a string FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      val tableName = fullTableName(t)
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType().add("x", IntegerType)))
 
       sql(s"ALTER TABLE $t ADD COLUMN b string AFTER point")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType().add("x", IntegerType))
         .add("b", StringType))
@@ -151,7 +156,7 @@ trait AlterTableTests extends SharedSparkSession {
       assert(e1.getMessage().contains("Couldn't find the reference column"))
 
       sql(s"ALTER TABLE $t ADD COLUMN point.y int FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType()
           .add("y", IntegerType)
@@ -159,7 +164,7 @@ trait AlterTableTests extends SharedSparkSession {
         .add("b", StringType))
 
       sql(s"ALTER TABLE $t ADD COLUMN point.z int AFTER x")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", StringType)
         .add("point", new StructType()
           .add("y", IntegerType)
@@ -179,7 +184,8 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (a string, b int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMNS (x int AFTER a, y int AFTER x, z int AFTER y)")
 
-      assert(getTableMetadata(t).schema === new StructType()
+      val tableName = fullTableName(t)
+      assert(getTableMetadata(tableName).schema === new StructType()
         .add("a", StringType)
         .add("x", IntegerType)
         .add("y", IntegerType)
@@ -190,7 +196,7 @@ trait AlterTableTests extends SharedSparkSession {
           .add("y", DoubleType)))
 
       sql(s"ALTER TABLE $t ADD COLUMNS (point.z double AFTER x, point.zz double AFTER z)")
-      assert(getTableMetadata(t).schema === new StructType()
+      assert(getTableMetadata(tableName).schema === new StructType()
         .add("a", StringType)
         .add("x", IntegerType)
         .add("y", IntegerType)
@@ -215,9 +221,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMNS data string COMMENT 'doc', ts timestamp")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(
         StructField("id", IntegerType),
         StructField("data", StringType).withComment("doc"),
@@ -231,9 +238,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN point.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -250,9 +258,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.key.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StructType(Seq(
@@ -269,9 +278,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.value.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -287,9 +297,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.element.z double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -305,9 +316,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points array<struct<x: double, y: double>>")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -322,9 +334,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ADD COLUMN points.element.z double COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -378,8 +391,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN id TYPE bigint")
 
-      val table = getTableMetadata(t)
-      assert(table.name === fullTableName(t))
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
+
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", LongType))
     }
   }
@@ -399,13 +414,14 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id bigint NOT NULL) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN id SET NOT NULL")
 
-      val table = getTableMetadata(t)
-      assert(table.name === fullTableName(t))
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", LongType, nullable = false))
 
       sql(s"ALTER TABLE $t ALTER COLUMN id DROP NOT NULL")
-      val table2 = getTableMetadata(t)
-      assert(table2.name === fullTableName(t))
+      val table2 = getTableMetadata(tableName)
+      assert(table2.name === tableName)
       assert(table2.schema === new StructType().add("id", LongType))
 
       val e = intercept[AnalysisException] {
@@ -421,8 +437,9 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: float, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN point.x TYPE double")
 
-      val table = getTableMetadata(t)
-      assert(table.name === fullTableName(t))
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -443,9 +460,10 @@ trait AlterTableTests extends SharedSparkSession {
       assert(exc.getMessage.contains("point"))
       assert(exc.getMessage.contains("update a struct by updating its fields"))
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -465,9 +483,10 @@ trait AlterTableTests extends SharedSparkSession {
 
       assert(exc.getMessage.contains("update the element by updating points.element"))
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(IntegerType)))
@@ -480,9 +499,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<int>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.element TYPE long")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(LongType)))
@@ -500,9 +520,10 @@ trait AlterTableTests extends SharedSparkSession {
 
       assert(exc.getMessage.contains("update a map by updating m.key or m.value"))
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("m", MapType(StringType, IntegerType)))
@@ -515,9 +536,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, m map<string, int>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN m.value TYPE long")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("m", MapType(StringType, LongType)))
@@ -531,9 +553,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.key.x TYPE double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StructType(Seq(
@@ -549,9 +572,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.value.x TYPE double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -566,9 +590,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: float, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.element.x TYPE double")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -625,9 +650,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN id COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === StructType(Seq(StructField("id", IntegerType).withComment("doc"))))
     }
   }
@@ -638,7 +664,8 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (a int, b int, point struct<x: int, y: int, z: int>) USING $v2Format")
 
       sql(s"ALTER TABLE $t ALTER COLUMN b FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      val tableName = fullTableName(t)
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("b", IntegerType)
         .add("a", IntegerType)
         .add("point", new StructType()
@@ -647,7 +674,7 @@ trait AlterTableTests extends SharedSparkSession {
           .add("z", IntegerType)))
 
       sql(s"ALTER TABLE $t ALTER COLUMN b AFTER point")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", IntegerType)
         .add("point", new StructType()
           .add("x", IntegerType)
@@ -660,7 +687,7 @@ trait AlterTableTests extends SharedSparkSession {
       assert(e1.getMessage.contains("Couldn't resolve positional argument"))
 
       sql(s"ALTER TABLE $t ALTER COLUMN point.y FIRST")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", IntegerType)
         .add("point", new StructType()
           .add("y", IntegerType)
@@ -669,7 +696,7 @@ trait AlterTableTests extends SharedSparkSession {
         .add("b", IntegerType))
 
       sql(s"ALTER TABLE $t ALTER COLUMN point.y AFTER z")
-      assert(getTableMetadata(t).schema == new StructType()
+      assert(getTableMetadata(tableName).schema == new StructType()
         .add("a", IntegerType)
         .add("point", new StructType()
           .add("x", IntegerType)
@@ -693,9 +720,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN point.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -711,9 +739,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.key.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StructType(Seq(
@@ -729,9 +758,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.value.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -746,9 +776,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t ALTER COLUMN points.element.y COMMENT 'doc'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -791,9 +822,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN id TO user_id")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("user_id", IntegerType))
     }
   }
@@ -804,9 +836,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, point struct<x: double, y: double>) USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN point.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -822,9 +855,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN point.key.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", MapType(StructType(Seq(
@@ -840,9 +874,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN points.value.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -857,9 +892,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t RENAME COLUMN points.element.y TO t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -933,9 +969,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, data string) USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN data")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType().add("id", IntegerType))
     }
   }
@@ -947,9 +984,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN point.t")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", StructType(Seq(
@@ -965,9 +1003,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN point.key.y")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("point", MapType(StructType(Seq(
@@ -982,9 +1021,10 @@ trait AlterTableTests extends SharedSparkSession {
         s"USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN points.value.y")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", MapType(StringType, StructType(Seq(
@@ -998,9 +1038,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int, points array<struct<x: double, y: double>>) USING $v2Format")
       sql(s"ALTER TABLE $t DROP COLUMN points.element.y")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.schema === new StructType()
         .add("id", IntegerType)
         .add("points", ArrayType(StructType(Seq(
@@ -1042,9 +1083,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t SET LOCATION 's3://bucket/path'")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.properties ===
         withDefaultOwnership(Map("provider" -> v2Format, "location" -> "s3://bucket/path")).asJava)
     }
@@ -1069,9 +1111,10 @@ trait AlterTableTests extends SharedSparkSession {
       sql(s"CREATE TABLE $t (id int) USING $v2Format")
       sql(s"ALTER TABLE $t SET TBLPROPERTIES ('test'='34')")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.properties ===
         withDefaultOwnership(Map("provider" -> v2Format, "test" -> "34")).asJava)
     }
@@ -1082,17 +1125,18 @@ trait AlterTableTests extends SharedSparkSession {
     withTable(t) {
       sql(s"CREATE TABLE $t (id int) USING $v2Format TBLPROPERTIES('test' = '34')")
 
-      val table = getTableMetadata(t)
+      val tableName = fullTableName(t)
+      val table = getTableMetadata(tableName)
 
-      assert(table.name === fullTableName(t))
+      assert(table.name === tableName)
       assert(table.properties ===
         withDefaultOwnership(Map("provider" -> v2Format, "test" -> "34")).asJava)
 
       sql(s"ALTER TABLE $t UNSET TBLPROPERTIES ('test')")
 
-      val updated = getTableMetadata(t)
+      val updated = getTableMetadata(tableName)
 
-      assert(updated.name === fullTableName(t))
+      assert(updated.name === tableName)
       assert(updated.properties === withDefaultOwnership(Map("provider" -> v2Format)).asJava)
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
index 01caf8e2eb115..6b25d7c61663c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSessionCatalogSuite.scala
@@ -85,7 +85,7 @@ class DataSourceV2DataFrameSessionCatalogSuite
     withTable(t1) {
       spark.range(20).write.format(v2Format).option("path", "abc").saveAsTable(t1)
       val cat = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
-      val tableInfo = cat.loadTable(Identifier.of(Array.empty, t1))
+      val tableInfo = cat.loadTable(Identifier.of(Array("default"), t1))
       assert(tableInfo.properties().get("location") === "abc")
       assert(tableInfo.properties().get("provider") === v2Format)
     }
@@ -109,8 +109,7 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
   }
 
   override def alterTable(ident: Identifier, changes: TableChange*): Table = {
-    val fullIdent = fullIdentifier(ident)
-    Option(tables.get(fullIdent)) match {
+    Option(tables.get(ident)) match {
       case Some(table) =>
         val properties = CatalogV2Util.applyPropertiesChanges(table.properties, changes)
         val schema = CatalogV2Util.applySchemaChanges(table.schema, changes)
@@ -123,7 +122,7 @@ class InMemoryTableSessionCatalog extends TestV2SessionCatalogBase[InMemoryTable
         val newTable = new InMemoryTable(table.name, schema, table.partitioning, properties)
           .withData(table.data)
 
-        tables.put(fullIdent, newTable)
+        tables.put(ident, newTable)
 
         newTable
       case _ =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
index b6997445013e5..249b27c28b072 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
@@ -47,7 +47,7 @@ class DataSourceV2SQLSessionCatalogSuite
     val v2Catalog = spark.sessionState.catalogManager.currentCatalog
     val nameParts = spark.sessionState.sqlParser.parseMultipartIdentifier(tableName)
     v2Catalog.asInstanceOf[TableCatalog]
-      .loadTable(Identifier.of(Array.empty, nameParts.last))
+      .loadTable(Identifier.of(nameParts.init.toArray, nameParts.last))
   }
 
   test("SPARK-30697: catalog.isView doesn't throw an error for specialized identifiers") {
@@ -55,7 +55,7 @@ class DataSourceV2SQLSessionCatalogSuite
     withTable(t1) {
       sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format")
 
-      def idResolver(id: Identifier): Identifier = Identifier.of(Array.empty, id.name())
+      def idResolver(id: Identifier): Identifier = Identifier.of(Array("default"), id.name())
 
       InMemoryTableSessionCatalog.withCustomIdentifierResolver(idResolver) {
         // The following should not throw AnalysisException.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 2acd7e39fe949..f642114983293 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -172,7 +172,7 @@ class DataSourceV2SQLSuite
     spark.sql(s"CREATE TABLE table_name (id bigint, data string) USING $v2Source")
 
     val testCatalog = catalog(SESSION_CATALOG_NAME).asTableCatalog
-    val table = testCatalog.loadTable(Identifier.of(Array(), "table_name"))
+    val table = testCatalog.loadTable(Identifier.of(Array("default"), "table_name"))
 
     assert(table.name == "default.table_name")
     assert(table.partitioning.isEmpty)
@@ -453,7 +453,7 @@ class DataSourceV2SQLSuite
     spark.sql(s"CREATE TABLE table_name USING $v2Source AS SELECT id, data FROM source")
 
     val testCatalog = catalog(SESSION_CATALOG_NAME).asTableCatalog
-    val table = testCatalog.loadTable(Identifier.of(Array(), "table_name"))
+    val table = testCatalog.loadTable(Identifier.of(Array("default"), "table_name"))
 
     assert(table.name == "default.table_name")
     assert(table.partitioning.isEmpty)
@@ -565,7 +565,7 @@ class DataSourceV2SQLSuite
     // The fact that the following line doesn't throw an exception means, the session catalog
     // can load the table.
     val t = catalog(SESSION_CATALOG_NAME).asTableCatalog
-      .loadTable(Identifier.of(Array.empty, "table_name"))
+      .loadTable(Identifier.of(Array("default"), "table_name"))
     assert(t.isInstanceOf[V1Table], "V1 table wasn't returned as an unresolved table")
   }
 
@@ -605,10 +605,10 @@ class DataSourceV2SQLSuite
   }
 
   test("DropTable: table qualified with the session catalog name") {
-    val ident = Identifier.of(Array(), "tbl")
+    val ident = Identifier.of(Array("default"), "tbl")
     sql("CREATE TABLE tbl USING json AS SELECT 1 AS i")
     assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
-    sql("DROP TABLE spark_catalog.tbl")
+    sql("DROP TABLE spark_catalog.default.tbl")
     assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === false)
   }
 
@@ -705,7 +705,6 @@ class DataSourceV2SQLSuite
 
     withTable("t") {
       sql("CREATE TABLE t USING json AS SELECT 1 AS i")
-      checkAnswer(sql("select default.t.i from spark_catalog.t"), Row(1))
       checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
       checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
 
@@ -1378,7 +1377,12 @@ class DataSourceV2SQLSuite
     val sessionCatalog = catalog(SESSION_CATALOG_NAME).asTableCatalog
 
     def checkPartitioning(cat: TableCatalog, partition: String): Unit = {
-      val table = cat.loadTable(Identifier.of(Array.empty, "tbl"))
+      val namespace = if (cat.name == SESSION_CATALOG_NAME) {
+        Array("default")
+      } else {
+        Array[String]()
+      }
+      val table = cat.loadTable(Identifier.of(namespace, "tbl"))
       val partitions = table.partitioning().map(_.references())
       assert(partitions.length === 1)
       val fieldNames = partitions.flatMap(_.map(_.fieldNames()))
@@ -1414,48 +1418,48 @@ class DataSourceV2SQLSuite
   }
 
   test("tableCreation: duplicate column names in the table definition") {
-    val errorMsg = "Found duplicate column(s) in the table definition of t"
+    val errorMsg = "Found duplicate column(s) in the table definition of"
     Seq((true, ("a", "a")), (false, ("aA", "Aa"))).foreach { case (caseSensitive, (c0, c1)) =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
         assertAnalysisError(
           s"CREATE TABLE t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE TABLE testcat.t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE testcat.t ($c0 INT, $c1 INT) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
       }
     }
   }
 
   test("tableCreation: duplicate nested column names in the table definition") {
-    val errorMsg = "Found duplicate column(s) in the table definition of t"
+    val errorMsg = "Found duplicate column(s) in the table definition of"
     Seq((true, ("a", "a")), (false, ("aA", "Aa"))).foreach { case (caseSensitive, (c0, c1)) =>
       withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
         assertAnalysisError(
           s"CREATE TABLE t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE TABLE testcat.t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg default.t"
         )
         assertAnalysisError(
           s"CREATE OR REPLACE TABLE testcat.t (d struct<$c0: INT, $c1: INT>) USING $v2Source",
-          errorMsg
+          s"$errorMsg t"
         )
       }
     }
@@ -1786,7 +1790,7 @@ class DataSourceV2SQLSuite
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
       testV1Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS")
-      testV1Command("ANALYZE TABLE", s"$t COMPUTE STATISTICS FOR ALL COLUMNS")
+      testV1CommandSupportingTempView("ANALYZE TABLE", s"$t COMPUTE STATISTICS FOR ALL COLUMNS")
     }
   }
 
@@ -1850,7 +1854,7 @@ class DataSourceV2SQLSuite
     val t = "testcat.ns1.ns2.tbl"
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
-      testV1Command("SHOW CREATE TABLE", t)
+      testV1CommandSupportingTempView("SHOW CREATE TABLE", t)
     }
   }
 
@@ -1883,8 +1887,8 @@ class DataSourceV2SQLSuite
     withTable(t) {
       spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo")
 
-      testV1Command("SHOW COLUMNS", s"FROM $t")
-      testV1Command("SHOW COLUMNS", s"IN $t")
+      testV1CommandSupportingTempView("SHOW COLUMNS", s"FROM $t")
+      testV1CommandSupportingTempView("SHOW COLUMNS", s"IN $t")
 
       val e3 = intercept[AnalysisException] {
         sql(s"SHOW COLUMNS FROM tbl IN testcat.ns1.ns2")
@@ -2121,23 +2125,39 @@ class DataSourceV2SQLSuite
     withTable("t") {
       sql("CREATE TABLE t USING json AS SELECT 1 AS i")
       checkAnswer(sql("select * from t"), Row(1))
-      checkAnswer(sql("select * from spark_catalog.t"), Row(1))
       checkAnswer(sql("select * from spark_catalog.default.t"), Row(1))
     }
   }
 
+  test("SPARK-30885: v1 table name should be fully qualified") {
+    def assertWrongTableIdent(): Unit = {
+      withTable("t") {
+        sql("CREATE TABLE t USING json AS SELECT 1 AS i")
+        val e = intercept[AnalysisException] {
+          sql("select * from spark_catalog.t")
+        }
+        assert(e.message.contains("Table or view not found: spark_catalog.t"))
+      }
+    }
+
+    assertWrongTableIdent()
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+    assertWrongTableIdent()
+  }
+
   test("SPARK-30259: session catalog can be specified in CREATE TABLE AS SELECT command") {
     withTable("tbl") {
-      val ident = Identifier.of(Array(), "tbl")
-      sql("CREATE TABLE spark_catalog.tbl USING json AS SELECT 1 AS i")
+      val ident = Identifier.of(Array("default"), "tbl")
+      sql("CREATE TABLE spark_catalog.default.tbl USING json AS SELECT 1 AS i")
       assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
     }
   }
 
   test("SPARK-30259: session catalog can be specified in CREATE TABLE command") {
     withTable("tbl") {
-      val ident = Identifier.of(Array(), "tbl")
-      sql("CREATE TABLE spark_catalog.tbl (col string) USING json")
+      val ident = Identifier.of(Array("default"), "tbl")
+      sql("CREATE TABLE spark_catalog.default.tbl (col string) USING json")
       assert(catalog("spark_catalog").asTableCatalog.tableExists(ident) === true)
     }
   }
@@ -2146,7 +2166,7 @@ class DataSourceV2SQLSuite
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
 
-    withTable("spark_catalog.t", "testcat.ns.t") {
+    withTable("spark_catalog.default.t", "testcat.ns.t") {
       sql("CREATE TABLE t USING parquet AS SELECT 1")
       sql("CREATE TABLE testcat.ns.t USING parquet AS SELECT 2")
 
@@ -2168,17 +2188,18 @@ class DataSourceV2SQLSuite
 
     withTempView("t") {
       spark.range(10).createTempView("t")
-      withView(s"$sessionCatalogName.v") {
+      withView(s"$sessionCatalogName.default.v") {
         val e = intercept[AnalysisException] {
-          sql(s"CREATE VIEW $sessionCatalogName.v AS SELECT * FROM t")
+          sql(s"CREATE VIEW $sessionCatalogName.default.v AS SELECT * FROM t")
         }
         assert(e.message.contains("referencing a temporary view"))
       }
     }
 
     withTempView("t") {
-      withView(s"$sessionCatalogName.v") {
-        sql(s"CREATE VIEW $sessionCatalogName.v AS SELECT t1.col FROM t t1 JOIN ns1.ns2.t t2")
+      withView(s"$sessionCatalogName.default.v") {
+        sql(s"CREATE VIEW $sessionCatalogName.default.v " +
+          "AS SELECT t1.col FROM t t1 JOIN ns1.ns2.t t2")
         sql(s"USE $sessionCatalogName")
         // The view should read data from table `testcat.ns1.ns2.t` not the temp view.
         spark.range(10).createTempView("t")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
index 7bff955b18360..9693a10f9afca 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
@@ -75,7 +75,12 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
     withCatalogOption.foreach(cName => dfw.option("catalog", cName))
     dfw.partitionBy(partitionBy: _*).save()
 
-    val table = catalog(withCatalogOption.getOrElse(SESSION_CATALOG_NAME)).loadTable("t1")
+    val ident = if (withCatalogOption.isEmpty) {
+      Identifier.of(Array("default"), "t1")
+    } else {
+      Identifier.of(Array(), "t1")
+    }
+    val table = catalog(withCatalogOption.getOrElse(SESSION_CATALOG_NAME)).loadTable(ident)
     val namespace = withCatalogOption.getOrElse("default")
     assert(table.name() === s"$namespace.t1", "Table identifier was wrong")
     assert(table.partitioning().length === partitionBy.length, "Partitioning did not match")
@@ -134,7 +139,7 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
     val dfw = df.write.format(format).mode(SaveMode.Ignore).option("name", "t1")
     dfw.save()
 
-    val table = catalog(SESSION_CATALOG_NAME).loadTable("t1")
+    val table = catalog(SESSION_CATALOG_NAME).loadTable(Identifier.of(Array("default"), "t1"))
     assert(table.partitioning().isEmpty, "Partitioning should be empty")
     assert(table.schema() === new StructType().add("id", LongType), "Schema did not match")
     assert(load("t1", None).count() === 0)
@@ -279,7 +284,12 @@ class CatalogSupportingInMemoryTableProvider
   override def extractIdentifier(options: CaseInsensitiveStringMap): Identifier = {
     val name = options.get("name")
     assert(name != null, "The name should be provided for this table")
-    Identifier.of(Array.empty, name)
+    val namespace = if (options.containsKey("catalog")) {
+      Array[String]()
+    } else {
+      Array("default")
+    }
+    Identifier.of(namespace, name)
   }
 
   override def extractCatalog(options: CaseInsensitiveStringMap): String = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
index 3f6ac0b7f8d3c..637cf2fd16515 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/TestV2SessionCatalogBase.scala
@@ -22,6 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
 import org.apache.spark.sql.connector.catalog.{DelegatingCatalogExtension, Identifier, Table}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.types.StructType
@@ -41,23 +42,14 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
       partitions: Array[Transform],
       properties: util.Map[String, String]): T
 
-  protected def fullIdentifier(ident: Identifier): Identifier = {
-    if (ident.namespace().isEmpty) {
-      Identifier.of(Array("default"), ident.name())
-    } else {
-      ident
-    }
-  }
-
   override def loadTable(ident: Identifier): Table = {
-    val fullIdent = fullIdentifier(ident)
-    if (tables.containsKey(fullIdent)) {
-      tables.get(fullIdent)
+    if (tables.containsKey(ident)) {
+      tables.get(ident)
     } else {
       // Table was created through the built-in catalog
-      val t = super.loadTable(fullIdent)
+      val t = super.loadTable(ident)
       val table = newTable(t.name(), t.schema(), t.partitioning(), t.properties())
-      tables.put(fullIdent, table)
+      tables.put(ident, table)
       table
     }
   }
@@ -69,13 +61,12 @@ private[connector] trait TestV2SessionCatalogBase[T <: Table] extends Delegating
       properties: util.Map[String, String]): Table = {
     val created = super.createTable(ident, schema, partitions, properties)
     val t = newTable(created.name(), schema, partitions, properties)
-    val fullIdent = fullIdentifier(ident)
-    tables.put(fullIdent, t)
+    tables.put(ident, t)
     t
   }
 
   override def dropTable(ident: Identifier): Boolean = {
-    tables.remove(fullIdentifier(ident))
+    tables.remove(ident)
     super.dropTable(ident)
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
index 9a393f19ce9bb..575efec364812 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala
@@ -79,7 +79,7 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
           var e = intercept[AnalysisException] {
             sql("CREATE VIEW jtv1 AS SELECT * FROM temp_jtv1 WHERE id < 6")
           }.getMessage
-          assert(e.contains("Not allowed to create a permanent view `jtv1` by " +
+          assert(e.contains("Not allowed to create a permanent view `default`.`jtv1` by " +
             "referencing a temporary view temp_jtv1. " +
             "Please create a temp view instead by CREATE TEMP VIEW"))
 
@@ -88,8 +88,8 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils {
           e = intercept[AnalysisException] {
             sql(s"CREATE VIEW jtv1 AS SELECT * FROM $globalTempDB.global_temp_jtv1 WHERE id < 6")
           }.getMessage
-          assert(e.contains(s"Not allowed to create a permanent view `jtv1` by referencing " +
-            s"a temporary view global_temp.global_temp_jtv1"))
+          assert(e.contains("Not allowed to create a permanent view `default`.`jtv1` by " +
+            "referencing a temporary view global_temp.global_temp_jtv1"))
         }
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index dbf4b09403423..e4bf721b1d58d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -473,17 +473,12 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       withEmptyDirInTablePath("tab1") { tableLoc =>
         val hiddenGarbageFile = new File(tableLoc.getCanonicalPath, ".garbage")
         hiddenGarbageFile.createNewFile()
-        val exMsg = "Can not create the managed table('`tab1`'). The associated location"
         val exMsgWithDefaultDB =
           "Can not create the managed table('`default`.`tab1`'). The associated location"
         var ex = intercept[AnalysisException] {
           sql(s"CREATE TABLE tab1 USING ${dataSource} AS SELECT 1, 'a'")
         }.getMessage
-        if (isUsingHiveMetastore) {
-          assert(ex.contains(exMsgWithDefaultDB))
-        } else {
-          assert(ex.contains(exMsg))
-        }
+        assert(ex.contains(exMsgWithDefaultDB))
 
         ex = intercept[AnalysisException] {
           sql(s"CREATE TABLE tab1 (col1 int, col2 string) USING ${dataSource}")
@@ -509,8 +504,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
         val ex = intercept[AnalysisException] {
           sql("ALTER TABLE tab1 RENAME TO tab2")
         }.getMessage
-        val expectedMsg = "Can not rename the managed table('`tab1`'). The associated location"
-        assert(ex.contains(expectedMsg))
+        assert(ex.contains(
+          "Can not rename the managed table('`default`.`tab1`'). The associated location"))
       }
     }
   }
@@ -640,7 +635,8 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
         val errMsg = intercept[AnalysisException] {
           sql(s"CREATE TABLE t($c0 INT, $c1 INT) USING parquet")
         }.getMessage
-        assert(errMsg.contains("Found duplicate column(s) in the table definition of `t`"))
+        assert(errMsg.contains(
+          "Found duplicate column(s) in the table definition of `default`.`t`"))
       }
     }
   }
@@ -649,7 +645,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       sql("CREATE TABLE tbl(a int, b string) USING json PARTITIONED BY (c)")
     }
-    assert(e.message == "partition column c is not defined in table tbl, " +
+    assert(e.message == "partition column c is not defined in table default.tbl, " +
       "defined table columns are: a, b")
   }
 
@@ -657,7 +653,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     val e = intercept[AnalysisException] {
       sql("CREATE TABLE tbl(a int, b string) USING json CLUSTERED BY (c) INTO 4 BUCKETS")
     }
-    assert(e.message == "bucket column c is not defined in table tbl, " +
+    assert(e.message == "bucket column c is not defined in table default.tbl, " +
       "defined table columns are: a, b")
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index 7b3fb68174234..fc793534641df 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -102,10 +102,10 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
   }
 
   test("uncache of nonexistent tables") {
-    val expectedErrorMsg = "Table or view not found: nonexistentTable"
+    val expectedErrorMsg = "Table or view not found:"
     // make sure table doesn't exist
     var e = intercept[AnalysisException](spark.table("nonexistentTable")).getMessage
-    assert(e.contains(expectedErrorMsg))
+    assert(e.contains(s"$expectedErrorMsg nonexistentTable"))
     e = intercept[AnalysisException] {
       uncacheTable("nonexistentTable")
     }.getMessage
@@ -113,7 +113,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     e = intercept[AnalysisException] {
       sql("UNCACHE TABLE nonexistentTable")
     }.getMessage
-    assert(e.contains(expectedErrorMsg))
+    assert(e.contains(s"$expectedErrorMsg default.nonexistentTable"))
     sql("UNCACHE TABLE IF EXISTS nonexistentTable")
   }
 
@@ -403,13 +403,13 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // Cache the table 'cachedTable' in default db without qualified table name , and then
           // check whether the table is cached with expected name.
           sql("CACHE TABLE cachedTable OPTIONS('storageLevel' 'DISK_ONLY')")
-          assertCached(sql("SELECT * FROM cachedTable"), "`cachedTable`", DISK_ONLY)
+          assertCached(sql("SELECT * FROM cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
           assert(spark.catalog.isCached("cachedTable"), "Table 'cachedTable' should be cached.")
 
           // Refresh the table 'cachedTable' in default db with unqualified table name, and then
           // check whether the table is still cached with the same name.
           sql("REFRESH TABLE cachedTable")
-          assertCached(sql("SELECT * FROM cachedTable"), "`cachedTable`", DISK_ONLY)
+          assertCached(sql("SELECT * FROM cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
           assert(spark.catalog.isCached("cachedTable"),
             "Table 'cachedTable' should be cached after refreshing with its unqualified name.")
 
@@ -420,7 +420,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
           // 'default.cachedTable', instead of 'cachedTable'
           activateDatabase(db) {
             sql("REFRESH TABLE default.cachedTable")
-            assertCached(sql("SELECT * FROM default.cachedTable"), "`cachedTable`", DISK_ONLY)
+            assertCached(
+              sql("SELECT * FROM default.cachedTable"), "`default`.`cachedTable`", DISK_ONLY)
             assert(spark.catalog.isCached("default.cachedTable"),
               "Table 'cachedTable' should be cached after refreshing with its qualified name.")
           }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
index e5d572c90af38..99db1e3cbd18c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
@@ -219,7 +219,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
       val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>) USING hive"
       sql(createTable)
       val shownDDL = getShowDDL("SHOW CREATE TABLE t1")
-      assert(shownDDL == createTable.dropRight(" USING hive".length))
+      assert(shownDDL == "CREATE TABLE `default`.`t1` (`a` STRUCT<`b`: STRING>)")
 
       checkCreateHiveTableOrView("t1")
     }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
index 5e6e114fc3fdc..fa43ff14fd796 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala
@@ -80,8 +80,8 @@ class HiveSQLViewSuite extends SQLViewSuite with TestHiveSingleton {
             val e = intercept[AnalysisException] {
               sql(s"CREATE VIEW view1 AS SELECT $tempFunctionName(id) from tab1")
             }.getMessage
-            assert(e.contains("Not allowed to create a permanent view `view1` by referencing " +
-              s"a temporary function `$tempFunctionName`"))
+            assert(e.contains("Not allowed to create a permanent view `default`.`view1` by " +
+              s"referencing a temporary function `$tempFunctionName`"))
           }
         }
       }

From 2bbb9958c3b017062cafca8c78fc9e6d6d33dbd7 Mon Sep 17 00:00:00 2001
From: Josh Rosen <joshrosen@databricks.com>
Date: Tue, 25 Feb 2020 17:04:13 +0900
Subject: [PATCH 0158/1280] [SPARK-30944][BUILD] Update URL for Google Cloud
 Storage mirror of Maven Central

### What changes were proposed in this pull request?

This PR is a followup to #27307: per https://travis-ci.community/t/maven-builds-that-use-the-gcs-maven-central-mirror-should-update-their-paths/5926, the Google Cloud Storage mirror of Maven Central has updated its URLs: the new paths are updated more frequently. The new paths are listed on https://storage-download.googleapis.com/maven-central/index.html

This patch updates our build files to use these new URLs.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing build + tests.

Closes #27688 from JoshRosen/update-gcs-mirror-url.

Authored-by: Josh Rosen <joshrosen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f152d2a0a80e2756dd620538a46b030dd5a6e630)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 pom.xml                                                       | 4 ++--
 project/SparkBuild.scala                                      | 2 +-
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala    | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 925fa28a291a4..b3750e4c3f442 100644
--- a/pom.xml
+++ b/pom.xml
@@ -253,7 +253,7 @@
         See https://storage-download.googleapis.com/maven-central/index.html
       -->
       <name>GCS Maven Central mirror</name>
-      <url>https://maven-central.storage-download.googleapis.com/repos/central/data/</url>
+      <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
       <releases>
         <enabled>true</enabled>
       </releases>
@@ -284,7 +284,7 @@
         See https://storage-download.googleapis.com/maven-central/index.html
       -->
       <name>GCS Maven Central mirror</name>
-      <url>https://maven-central.storage-download.googleapis.com/repos/central/data/</url>
+      <url>https://maven-central.storage-download.googleapis.com/maven2/</url>
       <releases>
         <enabled>true</enabled>
       </releases>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index a07823cff2481..fcde1e958f35a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -226,7 +226,7 @@ object SparkBuild extends PomBuild {
     resolvers := Seq(
       // Google Mirror of Maven Central, placed first so that it's used instead of flaky Maven Central.
       // See https://storage-download.googleapis.com/maven-central/index.html for more info.
-      "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/repos/central/data/",
+      "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,
       Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 529749747b205..674c6dfa2e5e7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2136,7 +2136,7 @@ object SQLConf {
         "if the default Maven Central repo is unreachable.")
       .stringConf
       .createWithDefault(
-        "https://maven-central.storage-download.googleapis.com/repos/central/data/")
+        "https://maven-central.storage-download.googleapis.com/maven2/")
 
   val LEGACY_FROM_DAYTIME_STRING =
     buildConf("spark.sql.legacy.fromDayTimeString.enabled")

From ff6662acc6ff1511cc1c6b3671c54156102b0aae Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Tue, 25 Feb 2020 17:37:34 +0900
Subject: [PATCH 0159/1280] [SPARK-28228][SQL][TESTS] Refactoring for nested
 CTE tests

### What changes were proposed in this pull request?
Split the nested CTE cases into a single file `cte-nested.sql`, which will be reused in cte-legacy.sql and cte-nonlegacy.sql.

### Why are the changes needed?
Make the cases easy to maintain.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UT.

Closes #27667 from xuanyuanking/SPARK-28228-test.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../resources/sql-tests/inputs/cte-legacy.sql | 117 +-----------
 .../resources/sql-tests/inputs/cte-nested.sql | 105 +++++++++++
 .../sql-tests/inputs/cte-nonlegacy.sql        |   2 +-
 .../test/resources/sql-tests/inputs/cte.sql   | 106 -----------
 .../sql-tests/results/cte-legacy.sql.out      |  42 +----
 .../sql-tests/results/cte-nested.sql.out      | 174 +++++++++++++++++
 .../sql-tests/results/cte-nonlegacy.sql.out   | 177 +-----------------
 .../resources/sql-tests/results/cte.sql.out   | 174 +----------------
 8 files changed, 285 insertions(+), 612 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out

diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
index d8754d3023965..29dee1a3afd38 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-legacy.sql
@@ -1,115 +1,2 @@
-create temporary view t as select * from values 0, 1, 2 as t(id);
-create temporary view t2 as select * from values 0, 1 as t(id);
-
--- CTE legacy substitution
-SET spark.sql.legacy.ctePrecedencePolicy=legacy;
-
--- CTE in CTE definition
-WITH t as (
-  WITH t2 AS (SELECT 1)
-  SELECT * FROM t2
-)
-SELECT * FROM t;
-
--- CTE in subquery
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression
-SELECT (
-  WITH t AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in CTE definition shadows outer
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 2
-WITH
-  t(c) AS (SELECT 1),
-  t2 AS (
-    SELECT (
-      SELECT max(c) FROM (
-        WITH t(c) AS (SELECT 2)
-        SELECT * FROM t
-      )
-    )
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 3
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2),
-    t2 AS (
-      WITH t AS (SELECT 3)
-      SELECT * FROM t
-    )
-    SELECT * FROM t2
-  )
-SELECT * FROM t2;
-
--- CTE in subquery shadows outer
-WITH t(c) AS (SELECT 1)
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery shadows outer 2
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery shadows outer 3
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression shadows outer 2
-WITH t AS (SELECT 1)
-SELECT (
-  SELECT (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer 3
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT (
-    WITH t AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
--- Clean up
-DROP VIEW IF EXISTS t;
-DROP VIEW IF EXISTS t2;
+--SET spark.sql.legacy.ctePrecedencePolicy = legacy
+--IMPORT cte-nested.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
new file mode 100644
index 0000000000000..5e5e3a51f1f5d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
@@ -0,0 +1,105 @@
+-- CTE in CTE definition
+WITH t as (
+  WITH t2 AS (SELECT 1)
+  SELECT * FROM t2
+)
+SELECT * FROM t;
+
+-- CTE in subquery
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 1)
+  SELECT * FROM t
+);
+
+-- CTE in subquery expression
+SELECT (
+  WITH t AS (SELECT 1)
+  SELECT * FROM t
+);
+
+-- CTE in CTE definition shadows outer
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+SELECT * FROM t2;
+
+-- CTE in CTE definition shadows outer 2
+WITH
+  t(c) AS (SELECT 1),
+  t2 AS (
+    SELECT (
+      SELECT max(c) FROM (
+        WITH t(c) AS (SELECT 2)
+        SELECT * FROM t
+      )
+    )
+  )
+SELECT * FROM t2;
+
+-- CTE in CTE definition shadows outer 3
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2),
+    t2 AS (
+      WITH t AS (SELECT 3)
+      SELECT * FROM t
+    )
+    SELECT * FROM t2
+  )
+SELECT * FROM t2;
+
+-- CTE in subquery shadows outer
+WITH t(c) AS (SELECT 1)
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+);
+
+-- CTE in subquery shadows outer 2
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 2)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery shadows outer 3
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 3)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery expression shadows outer
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT * FROM t
+);
+
+-- CTE in subquery expression shadows outer 2
+WITH t AS (SELECT 1)
+SELECT (
+  SELECT (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+);
+
+-- CTE in subquery expression shadows outer 3
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT (
+    WITH t AS (SELECT 3)
+    SELECT * FROM t
+  )
+);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
index 8b9ce8911b133..d3b4539da2f3e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nonlegacy.sql
@@ -1,2 +1,2 @@
 --SET spark.sql.legacy.ctePrecedencePolicy = corrected
---IMPORT cte.sql
+--IMPORT cte-nested.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte.sql b/sql/core/src/test/resources/sql-tests/inputs/cte.sql
index d0e145c35a9fe..ec2f4808fcfc9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte.sql
@@ -49,112 +49,6 @@ WITH
   t(x) AS (SELECT 2)
 SELECT * FROM t;
 
--- CTE in CTE definition
-WITH t as (
-  WITH t2 AS (SELECT 1)
-  SELECT * FROM t2
-)
-SELECT * FROM t;
-
--- CTE in subquery
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression
-SELECT (
-  WITH t AS (SELECT 1)
-  SELECT * FROM t
-);
-
--- CTE in CTE definition shadows outer
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 2
-WITH
-  t(c) AS (SELECT 1),
-  t2 AS (
-    SELECT (
-      SELECT max(c) FROM (
-        WITH t(c) AS (SELECT 2)
-        SELECT * FROM t
-      )
-    )
-  )
-SELECT * FROM t2;
-
--- CTE in CTE definition shadows outer 3
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2),
-    t2 AS (
-      WITH t AS (SELECT 3)
-      SELECT * FROM t
-    )
-    SELECT * FROM t2
-  )
-SELECT * FROM t2;
-
--- CTE in subquery shadows outer
-WITH t(c) AS (SELECT 1)
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery shadows outer 2
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery shadows outer 3
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT * FROM t
-);
-
--- CTE in subquery expression shadows outer 2
-WITH t AS (SELECT 1)
-SELECT (
-  SELECT (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-);
-
--- CTE in subquery expression shadows outer 3
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT (
-    WITH t AS (SELECT 3)
-    SELECT * FROM t
-  )
-);
-
 -- Clean up
 DROP VIEW IF EXISTS t;
 DROP VIEW IF EXISTS t2;
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
index a20ae55b1b876..e8020e1a454e6 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
@@ -1,29 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 17
-
-
--- !query
-create temporary view t as select * from values 0, 1, 2 as t(id)
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-create temporary view t2 as select * from values 0, 1 as t(id)
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-SET spark.sql.legacy.ctePrecedencePolicy=legacy
--- !query schema
-struct<key:string,value:string>
--- !query output
-spark.sql.legacy.ctePrecedencePolicy	legacy
+-- Number of queries: 12
 
 
 -- !query
@@ -190,19 +166,3 @@ SELECT (
 struct<scalarsubquery():int>
 -- !query output
 1
-
-
--- !query
-DROP VIEW IF EXISTS t
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-DROP VIEW IF EXISTS t2
--- !query schema
-struct<>
--- !query output
-
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
new file mode 100644
index 0000000000000..64d635adae428
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
@@ -0,0 +1,174 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 12
+
+
+-- !query
+WITH t as (
+  WITH t2 AS (SELECT 1)
+  SELECT * FROM t2
+)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+1
+
+
+-- !query
+SELECT (
+  WITH t AS (SELECT 1)
+  SELECT * FROM t
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+SELECT * FROM t2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH
+  t(c) AS (SELECT 1),
+  t2 AS (
+    SELECT (
+      SELECT max(c) FROM (
+        WITH t(c) AS (SELECT 2)
+        SELECT * FROM t
+      )
+    )
+  )
+SELECT * FROM t2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH
+  t AS (SELECT 1),
+  t2 AS (
+    WITH t AS (SELECT 2),
+    t2 AS (
+      WITH t AS (SELECT 3)
+      SELECT * FROM t
+    )
+    SELECT * FROM t2
+  )
+SELECT * FROM t2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT max(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<max(c):int>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+2
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT sum(c) FROM (
+  WITH t(c) AS (SELECT 2)
+  SELECT max(c) AS c FROM (
+    WITH t(c) AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<sum(c):bigint>
+-- !query output
+3
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  SELECT (
+    WITH t AS (SELECT 2)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t AS (SELECT 1)
+SELECT (
+  WITH t AS (SELECT 2)
+  SELECT (
+    WITH t AS (SELECT 3)
+    SELECT * FROM t
+  )
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
index 2d87781193c25..9422bb642d96f 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
@@ -1,164 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 27
-
-
--- !query
-create temporary view t as select * from values 0, 1, 2 as t(id)
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-create temporary view t2 as select * from values 0, 1 as t(id)
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-WITH s AS (SELECT 1 FROM s) SELECT * FROM s
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Table or view not found: s; line 1 pos 25
-
-
--- !query
-WITH r AS (SELECT (SELECT * FROM r))
-SELECT * FROM r
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Table or view not found: r; line 1 pos 33
-
-
--- !query
-WITH t AS (SELECT 1 FROM t) SELECT * FROM t
--- !query schema
-struct<1:int>
--- !query output
-1
-1
-1
-
-
--- !query
-WITH s1 AS (SELECT 1 FROM s2), s2 AS (SELECT 1 FROM s1) SELECT * FROM s1, s2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Table or view not found: s2; line 1 pos 26
-
-
--- !query
-WITH t1 AS (SELECT * FROM t2), t2 AS (SELECT 2 FROM t1) SELECT * FROM t1 cross join t2
--- !query schema
-struct<id:int,2:int>
--- !query output
-0	2
-0	2
-1	2
-1	2
-
-
--- !query
-WITH CTE1 AS (
-  SELECT b.id AS id
-  FROM   T2 a
-         CROSS JOIN (SELECT id AS id FROM T2) b
-)
-SELECT t1.id AS c1,
-       t2.id AS c2
-FROM   CTE1 t1
-       CROSS JOIN CTE1 t2
--- !query schema
-struct<c1:int,c2:int>
--- !query output
-0	0
-0	0
-0	0
-0	0
-0	1
-0	1
-0	1
-0	1
-1	0
-1	0
-1	0
-1	0
-1	1
-1	1
-1	1
-1	1
-
-
--- !query
-WITH t(x) AS (SELECT 1)
-SELECT * FROM t WHERE x = 1
--- !query schema
-struct<x:int>
--- !query output
-1
-
-
--- !query
-WITH t(x, y) AS (SELECT 1, 2)
-SELECT * FROM t WHERE x = 1 AND y = 2
--- !query schema
-struct<x:int,y:int>
--- !query output
-1	2
-
-
--- !query
-WITH t(x, x) AS (SELECT 1, 2)
-SELECT * FROM t
--- !query schema
-struct<x:int,x:int>
--- !query output
-1	2
-
-
--- !query
-WITH t() AS (SELECT 1)
-SELECT * FROM t
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-no viable alternative at input 'WITH t()'(line 1, pos 7)
-
-== SQL ==
-WITH t() AS (SELECT 1)
--------^^^
-SELECT * FROM t
-
-
--- !query
-WITH
-  t(x) AS (SELECT 1),
-  t(x) AS (SELECT 2)
-SELECT * FROM t
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-CTE definition can't have duplicate names: 't'.(line 1, pos 0)
-
-== SQL ==
-WITH
-^^^
-  t(x) AS (SELECT 1),
-  t(x) AS (SELECT 2)
-SELECT * FROM t
+-- Number of queries: 12
 
 
 -- !query
@@ -325,19 +166,3 @@ SELECT (
 struct<scalarsubquery():int>
 -- !query output
 3
-
-
--- !query
-DROP VIEW IF EXISTS t
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-DROP VIEW IF EXISTS t2
--- !query schema
-struct<>
--- !query output
-
diff --git a/sql/core/src/test/resources/sql-tests/results/cte.sql.out b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
index 6486fba18d20f..b8f666586ce45 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 27
+-- Number of queries: 15
 
 
 -- !query
@@ -161,178 +161,6 @@ WITH
 SELECT * FROM t
 
 
--- !query
-WITH t as (
-  WITH t2 AS (SELECT 1)
-  SELECT * FROM t2
-)
-SELECT * FROM t
--- !query schema
-struct<1:int>
--- !query output
-1
-
-
--- !query
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 1)
-  SELECT * FROM t
-)
--- !query schema
-struct<max(c):int>
--- !query output
-1
-
-
--- !query
-SELECT (
-  WITH t AS (SELECT 1)
-  SELECT * FROM t
-)
--- !query schema
-struct<scalarsubquery():int>
--- !query output
-1
-
-
--- !query
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-SELECT * FROM t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
-
-
--- !query
-WITH
-  t(c) AS (SELECT 1),
-  t2 AS (
-    SELECT (
-      SELECT max(c) FROM (
-        WITH t(c) AS (SELECT 2)
-        SELECT * FROM t
-      )
-    )
-  )
-SELECT * FROM t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
-
-
--- !query
-WITH
-  t AS (SELECT 1),
-  t2 AS (
-    WITH t AS (SELECT 2),
-    t2 AS (
-      WITH t AS (SELECT 3)
-      SELECT * FROM t
-    )
-    SELECT * FROM t2
-  )
-SELECT * FROM t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
-
-
--- !query
-WITH t(c) AS (SELECT 1)
-SELECT max(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT * FROM t
-)
--- !query schema
-struct<max(c):int>
--- !query output
-2
-
-
--- !query
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 2)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<sum(c):bigint>
--- !query output
-2
-
-
--- !query
-WITH t(c) AS (SELECT 1)
-SELECT sum(c) FROM (
-  WITH t(c) AS (SELECT 2)
-  SELECT max(c) AS c FROM (
-    WITH t(c) AS (SELECT 3)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<sum(c):bigint>
--- !query output
-3
-
-
--- !query
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT * FROM t
-)
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
-
-
--- !query
-WITH t AS (SELECT 1)
-SELECT (
-  SELECT (
-    WITH t AS (SELECT 2)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
-
-
--- !query
-WITH t AS (SELECT 1)
-SELECT (
-  WITH t AS (SELECT 2)
-  SELECT (
-    WITH t AS (SELECT 3)
-    SELECT * FROM t
-  )
-)
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
-
-
 -- !query
 DROP VIEW IF EXISTS t
 -- !query schema

From 1d746eb0afddd3c2a4e1313dddf80ac0aec00a7a Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 25 Feb 2020 22:19:24 +0800
Subject: [PATCH 0160/1280] [SPARK-30919][SQL] Make interval multiply and
 divide's overflow behavior consistent with other operations

### What changes were proposed in this pull request?

The current behavior of interval multiply and divide follows the ANSI SQL standard when overflow, it is compatible with other operations when `spark.sql.ansi.enabled` is true, but not compatible when `spark.sql.ansi.enabled` is false.

When `spark.sql.ansi.enabled` is false, as the factor is a double value, so it should use java's rounding or truncation behavior for casting double to integrals. when divided by zero, it returns `null`.  we also follow the natural rules for intervals as defined in the Gregorian calendar, so we do not add the month fraction to days but add days fraction to microseconds.

### Why are the changes needed?

Make interval multiply and divide's overflow behavior consistent with other interval operations

### Does this PR introduce any user-facing change?

no, these are new features in 3.0

### How was this patch tested?

add uts

Closes #27672 from yaooqinn/SPARK-30919.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 761209c1f2af513a9db2e08c5937531cff7aeeed)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/intervalExpressions.scala     | 34 ++++++++---
 .../sql/catalyst/util/IntervalUtils.scala     | 45 +++++++++++---
 .../IntervalExpressionsSuite.scala            | 37 +++++++----
 .../catalyst/util/IntervalUtilsSuite.scala    | 61 ++++++++++++-------
 .../resources/sql-tests/inputs/interval.sql   |  4 ++
 .../sql-tests/results/ansi/interval.sql.out   | 38 +++++++++++-
 .../sql-tests/results/interval.sql.out        | 49 ++++++++++++---
 7 files changed, 210 insertions(+), 58 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
index 831510e7f0f3e..c09350f33c7f9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
@@ -22,6 +22,7 @@ import java.util.Locale
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 
@@ -112,13 +113,14 @@ object ExtractIntervalPart {
 
 abstract class IntervalNumOperation(
     interval: Expression,
-    num: Expression,
-    operation: (CalendarInterval, Double) => CalendarInterval,
-    operationName: String)
+    num: Expression)
   extends BinaryExpression with ImplicitCastInputTypes with Serializable {
   override def left: Expression = interval
   override def right: Expression = num
 
+  protected val operation: (CalendarInterval, Double) => CalendarInterval
+  protected def operationName: String
+
   override def inputTypes: Seq[AbstractDataType] = Seq(CalendarIntervalType, DoubleType)
   override def dataType: DataType = CalendarIntervalType
 
@@ -136,11 +138,29 @@ abstract class IntervalNumOperation(
   override def prettyName: String = operationName.stripSuffix("Exact") + "_interval"
 }
 
-case class MultiplyInterval(interval: Expression, num: Expression)
-  extends IntervalNumOperation(interval, num, multiplyExact, "multiplyExact")
+case class MultiplyInterval(
+    interval: Expression,
+    num: Expression,
+    checkOverflow: Boolean = SQLConf.get.ansiEnabled)
+  extends IntervalNumOperation(interval, num) {
+
+  override protected val operation: (CalendarInterval, Double) => CalendarInterval =
+    if (checkOverflow) multiplyExact else multiply
+
+  override protected def operationName: String = if (checkOverflow) "multiplyExact" else "multiply"
+}
+
+case class DivideInterval(
+    interval: Expression,
+    num: Expression,
+    checkOverflow: Boolean = SQLConf.get.ansiEnabled)
+  extends IntervalNumOperation(interval, num) {
+
+  override protected val operation: (CalendarInterval, Double) => CalendarInterval =
+    if (checkOverflow) divideExact else divide
 
-case class DivideInterval(interval: Expression, num: Expression)
-  extends IntervalNumOperation(interval, num, divideExact, "divideExact")
+  override protected def operationName: String = if (checkOverflow) "divideExact" else "divide"
+}
 
 // scalastyle:off line.size.limit
 @ExpressionDescription(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 2790f8e998358..0a13ec87c2897 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -405,22 +405,38 @@ object IntervalUtils {
   }
 
   /**
-   * Makes an interval from months, days and micros with the fractional part by
-   * adding the month fraction to days and the days fraction to micros.
-   *
-   * @throws ArithmeticException if the result overflows any field value
+   * Makes an interval from months, days and micros with the fractional part.
+   * The overflow style here follows the way of ansi sql standard and the natural rules for
+   * intervals as defined in the Gregorian calendar. Thus, the days fraction will be added
+   * to microseconds but the months fraction will not be added to days, and it will throw
+   * exception if any part overflows.
    */
   private def fromDoubles(
       monthsWithFraction: Double,
       daysWithFraction: Double,
       microsWithFraction: Double): CalendarInterval = {
     val truncatedMonths = Math.toIntExact(monthsWithFraction.toLong)
-    val days = daysWithFraction + DAYS_PER_MONTH * (monthsWithFraction - truncatedMonths)
-    val truncatedDays = Math.toIntExact(days.toLong)
-    val micros = microsWithFraction + MICROS_PER_DAY * (days - truncatedDays)
+    val truncatedDays = Math.toIntExact(daysWithFraction.toLong)
+    val micros = microsWithFraction + MICROS_PER_DAY * (daysWithFraction - truncatedDays)
     new CalendarInterval(truncatedMonths, truncatedDays, micros.round)
   }
 
+  /**
+   * Makes an interval from months, days and micros with the fractional part.
+   * The overflow style here follows the way of casting [[java.lang.Double]] to integrals and the
+   * natural rules for intervals as defined in the Gregorian calendar. Thus, the days fraction
+   * will be added to microseconds but the months fraction will not be added to days, and there may
+   * be rounding or truncation in months(or day and microseconds) part.
+   */
+  private def safeFromDoubles(
+      monthsWithFraction: Double,
+      daysWithFraction: Double,
+      microsWithFraction: Double): CalendarInterval = {
+    val truncatedDays = daysWithFraction.toInt
+    val micros = microsWithFraction + MICROS_PER_DAY * (daysWithFraction - truncatedDays)
+    new CalendarInterval(monthsWithFraction.toInt, truncatedDays, micros.round)
+  }
+
   /**
    * Unary minus, return the negated the calendar interval value.
    *
@@ -484,6 +500,13 @@ object IntervalUtils {
     new CalendarInterval(months, days, microseconds)
   }
 
+  /**
+   * Return a new calendar interval instance of the left interval times a multiplier.
+   */
+  def multiply(interval: CalendarInterval, num: Double): CalendarInterval = {
+    safeFromDoubles(num * interval.months, num * interval.days, num * interval.microseconds)
+  }
+
   /**
    * Return a new calendar interval instance of the left interval times a multiplier.
    *
@@ -493,6 +516,14 @@ object IntervalUtils {
     fromDoubles(num * interval.months, num * interval.days, num * interval.microseconds)
   }
 
+  /**
+   * Return a new calendar interval instance of the left interval divides by a dividend.
+   */
+  def divide(interval: CalendarInterval, num: Double): CalendarInterval = {
+    if (num == 0) return null
+    safeFromDoubles(interval.months / num, interval.days / num, interval.microseconds / num)
+  }
+
   /**
    * Return a new calendar interval instance of the left interval divides by a dividend.
    *
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index bc5743811c975..e591c4984a7bf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -199,11 +199,16 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("multiply") {
-    def check(interval: String, num: Double, expected: String): Unit = {
-      val expr = MultiplyInterval(Literal(stringToInterval(interval)), Literal(num))
+    def check(
+        interval: String,
+        num: Double,
+        expected: String,
+        isAnsi: Option[Boolean] = None): Unit = {
       val expectedRes = safeStringToInterval(expected)
-      Seq("true", "false").foreach { v =>
+      val configs = if (isAnsi.isEmpty) Seq("true", "false") else isAnsi.map(_.toString).toSeq
+      configs.foreach { v =>
         withSQLConf(SQLConf.ANSI_ENABLED.key -> v) {
+          val expr = MultiplyInterval(Literal(stringToInterval(interval)), Literal(num))
           if (expectedRes == null) {
             checkExceptionInExpression[ArithmeticException](expr, expected)
           } else {
@@ -220,17 +225,23 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check("1 year 1 second", 0.5, "6 months 500 milliseconds")
     check("-100 years -1 millisecond", 0.5, "-50 years -500 microseconds")
     check("2 months 4 seconds", -0.5, "-1 months -2 seconds")
-    check("1 month 2 microseconds", 1.5, "1 months 15 days 3 microseconds")
-    check("2 months", Int.MaxValue, "integer overflow")
+    check("1 month 2 microseconds", 1.5, "1 months 3 microseconds")
+    check("2 months", Int.MaxValue, "integer overflow", Some(true))
+    check("2 months", Int.MaxValue, Int.MaxValue + " months", Some(false))
   }
 
   test("divide") {
-    def check(interval: String, num: Double, expected: String): Unit = {
-      val expr = DivideInterval(Literal(stringToInterval(interval)), Literal(num))
+    def check(
+        interval: String,
+        num: Double,
+        expected: String,
+        isAnsi: Option[Boolean] = None): Unit = {
       val expectedRes = safeStringToInterval(expected)
-      Seq("true", "false").foreach { v =>
+      val configs = if (isAnsi.isEmpty) Seq("true", "false") else isAnsi.map(_.toString).toSeq
+      configs.foreach { v =>
         withSQLConf(SQLConf.ANSI_ENABLED.key -> v) {
-          if (expectedRes == null) {
+          val expr = DivideInterval(Literal(stringToInterval(interval)), Literal(num))
+          if (expected != null && expectedRes == null) {
             checkExceptionInExpression[ArithmeticException](expr, expected)
           } else {
             checkEvaluation(expr, expectedRes)
@@ -245,9 +256,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check("6 years -7 seconds", 3, "2 years -2.333333 seconds")
     check("2 years -8 seconds", 0.5, "4 years -16 seconds")
     check("-1 month 2 microseconds", -0.25, "4 months -8 microseconds")
-    check("1 month 3 microsecond", 1.5, "20 days 2 microseconds")
-    check("1 second", 0, "divide by zero")
-    check(s"${Int.MaxValue} months", 0.9, "integer overflow")
+    check("1 month 3 microsecond", 1.5, "2 microseconds")
+    check("1 second", 0, "divide by zero", Some(true))
+    check("1 second", 0, null, Some(false))
+    check(s"${Int.MaxValue} months", 0.9, "integer overflow", Some(true))
+    check(s"${Int.MaxValue} months", 0.9, Int.MaxValue + " months", Some(false))
   }
 
   test("make interval") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index d0d79acd73883..e7c3163d0ac09 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -268,19 +268,23 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("multiply by num") {
-    var interval = new CalendarInterval(0, 0, 0)
-    assert(interval === multiplyExact(interval, 0))
-    interval = new CalendarInterval(123, 456, 789)
-    assert(new CalendarInterval(123 * 42, 456 * 42, 789 * 42) === multiplyExact(interval, 42))
-    interval = new CalendarInterval(-123, -456, -789)
-    assert(new CalendarInterval(-123 * 42, -456 * 42, -789 * 42) === multiplyExact(interval, 42))
-    assert(new CalendarInterval(1, 22, 12 * MICROS_PER_HOUR) ===
-      multiplyExact(new CalendarInterval(1, 5, 0), 1.5))
-    assert(new CalendarInterval(2, 14, 12 * MICROS_PER_HOUR) ===
-      multiplyExact(new CalendarInterval(2, 2, 2 * MICROS_PER_HOUR), 1.2))
+    Seq[(CalendarInterval, Double) => CalendarInterval](multiply, multiplyExact).foreach { func =>
+      var interval = new CalendarInterval(0, 0, 0)
+      assert(interval === func(interval, 0))
+      interval = new CalendarInterval(123, 456, 789)
+      assert(new CalendarInterval(123 * 42, 456 * 42, 789 * 42) === func(interval, 42))
+      interval = new CalendarInterval(-123, -456, -789)
+      assert(new CalendarInterval(-123 * 42, -456 * 42, -789 * 42) === func(interval, 42))
+      interval = new CalendarInterval(1, 5, 0)
+      assert(new CalendarInterval(1, 7, 12 * MICROS_PER_HOUR) === func(interval, 1.5))
+      interval = new CalendarInterval(2, 2, 2 * MICROS_PER_HOUR)
+      assert(new CalendarInterval(2, 2, 12 * MICROS_PER_HOUR) === func(interval, 1.2))
+    }
 
+    val interval = new CalendarInterval(2, 0, 0)
+    assert(multiply(interval, Integer.MAX_VALUE) === new CalendarInterval(Int.MaxValue, 0, 0))
     try {
-      multiplyExact(new CalendarInterval(2, 0, 0), Integer.MAX_VALUE)
+      multiplyExact(interval, Integer.MAX_VALUE)
       fail("Expected to throw an exception on months overflow")
     } catch {
       case e: ArithmeticException => assert(e.getMessage.contains("overflow"))
@@ -288,17 +292,32 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("divide by num") {
-    var interval = new CalendarInterval(0, 0, 0)
-    assert(interval === divideExact(interval, 10))
-    interval = new CalendarInterval(1, 3, 30 * MICROS_PER_SECOND)
-    assert(new CalendarInterval(0, 16, 12 * MICROS_PER_HOUR + 15 * MICROS_PER_SECOND) ===
-      divideExact(interval, 2))
-    assert(new CalendarInterval(2, 6, MICROS_PER_MINUTE) === divideExact(interval, 0.5))
-    interval = new CalendarInterval(-1, 0, -30 * MICROS_PER_SECOND)
-    assert(new CalendarInterval(0, -15, -15 * MICROS_PER_SECOND) === divideExact(interval, 2))
-    assert(new CalendarInterval(-2, 0, -1 * MICROS_PER_MINUTE) === divideExact(interval, 0.5))
+    Seq[(CalendarInterval, Double) => CalendarInterval](divide, divideExact).foreach { func =>
+      var interval = new CalendarInterval(0, 0, 0)
+      assert(interval === func(interval, 10))
+      interval = new CalendarInterval(1, 3, 30 * MICROS_PER_SECOND)
+      assert(new CalendarInterval(0, 1, 12 * MICROS_PER_HOUR + 15 * MICROS_PER_SECOND) ===
+        func(interval, 2))
+      assert(new CalendarInterval(2, 6, MICROS_PER_MINUTE) === func(interval, 0.5))
+      interval = new CalendarInterval(-1, 0, -30 * MICROS_PER_SECOND)
+      assert(new CalendarInterval(0, 0, -15 * MICROS_PER_SECOND) === func(interval, 2))
+      assert(new CalendarInterval(-2, 0, -MICROS_PER_MINUTE) === func(interval, 0.5))
+    }
+
+    var interval = new CalendarInterval(Int.MaxValue, Int.MaxValue, 0)
+    assert(divide(interval, 0.9) === new CalendarInterval(Int.MaxValue, Int.MaxValue,
+      ((Int.MaxValue / 9.0) * MICROS_PER_DAY).round))
+    try {
+      divideExact(interval, 0.9)
+      fail("Expected to throw an exception on integer overflow")
+    } catch {
+      case e: ArithmeticException => assert(e.getMessage.contains("integer overflow"))
+    }
+
+    interval = new CalendarInterval(123, 456, 789)
+    assert(divide(interval, 0) === null)
     try {
-      divideExact(new CalendarInterval(123, 456, 789), 0)
+      divideExact(interval, 0)
       fail("Expected to throw an exception on divide by zero")
     } catch {
       case e: ArithmeticException => assert(e.getMessage.contains("divide by zero"))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index facd6321a1bd4..4f26e75fa77f8 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -4,6 +4,10 @@
 select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15');
 select interval 4 month 2 weeks 3 microseconds * 1.5;
 select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5;
+select interval 2147483647 month * 2;
+select interval 2147483647 month / 0.5;
+select interval 2147483647 day * 2;
+select interval 2147483647 day / 0.5;
 
 -- interval operation with null and zero case
 select interval '2 seconds' / 0;
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 4b465406ee0ed..4a41dd669e1cc 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 81
+-- Number of queries: 85
 
 
 -- !query
@@ -26,6 +26,42 @@ struct<divide_interval(subtracttimestamps(TIMESTAMP '2019-10-15 00:00:00', TIMES
 16 hours
 
 
+-- !query
+select interval 2147483647 month * 2
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
+-- !query
+select interval 2147483647 month / 0.5
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
+-- !query
+select interval 2147483647 day * 2
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
+-- !query
+select interval 2147483647 day / 0.5
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+integer overflow
+
+
 -- !query
 select interval '2 seconds' / 0
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 0509594ac13d3..f1af335f2c78f 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 81
+-- Number of queries: 85
 
 
 -- !query
@@ -26,13 +26,44 @@ struct<divide_interval(subtracttimestamps(TIMESTAMP '2019-10-15 00:00:00', TIMES
 16 hours
 
 
+-- !query
+select interval 2147483647 month * 2
+-- !query schema
+struct<multiply_interval(INTERVAL '178956970 years 7 months', CAST(2 AS DOUBLE)):interval>
+-- !query output
+178956970 years 7 months
+
+
+-- !query
+select interval 2147483647 month / 0.5
+-- !query schema
+struct<divide_interval(INTERVAL '178956970 years 7 months', CAST(0.5 AS DOUBLE)):interval>
+-- !query output
+178956970 years 7 months
+
+
+-- !query
+select interval 2147483647 day * 2
+-- !query schema
+struct<multiply_interval(INTERVAL '2147483647 days', CAST(2 AS DOUBLE)):interval>
+-- !query output
+2147483647 days 2562047788 hours 54.775807 seconds
+
+
+-- !query
+select interval 2147483647 day / 0.5
+-- !query schema
+struct<divide_interval(INTERVAL '2147483647 days', CAST(0.5 AS DOUBLE)):interval>
+-- !query output
+2147483647 days 2562047788 hours 54.775807 seconds
+
+
 -- !query
 select interval '2 seconds' / 0
 -- !query schema
-struct<>
+struct<divide_interval(INTERVAL '2 seconds', CAST(0 AS DOUBLE)):interval>
 -- !query output
-java.lang.ArithmeticException
-divide by zero
+NULL
 
 
 -- !query
@@ -779,19 +810,17 @@ struct<(b + INTERVAL '1 months'):interval>
 -- !query
 select a * 1.1 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
-struct<>
+struct<multiply_interval(a, CAST(1.1 AS DOUBLE)):interval>
 -- !query output
-java.lang.ArithmeticException
-integer overflow
+-178956970 years -8 months
 
 
 -- !query
 select a / 0.5 from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
-struct<>
+struct<divide_interval(a, CAST(0.5 AS DOUBLE)):interval>
 -- !query output
-java.lang.ArithmeticException
-integer overflow
+-178956970 years -8 months
 
 
 -- !query

From b968cd37796a5730fe5c2318d23a38416f550957 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 25 Feb 2020 14:10:29 -0800
Subject: [PATCH 0161/1280] [SPARK-30918][SQL] improve the splitting of skewed
 partitions

### What changes were proposed in this pull request?

Use the average size of the non-skewed partitions as the target size when splitting skewed partitions, instead of ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD

### Why are the changes needed?

The goal of skew join optimization is to make the data distribution move even. So it makes more sense the use the average size of the non-skewed partitions as the target size.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

existing tests

Closes #27669 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
(cherry picked from commit 8f247e5d3682ad765bdbb9ea5a4315862c5a383c)
Signed-off-by: Xiao Li <gatorsmile@gmail.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 10 +--
 .../adaptive/OptimizeSkewedJoin.scala         | 62 +++++++++++++++----
 .../adaptive/AdaptiveQueryExecSuite.scala     |  4 +-
 3 files changed, 54 insertions(+), 22 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 674c6dfa2e5e7..e6f7cfdf43dcc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -432,19 +432,13 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
-  val ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD =
-    buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionSizeThreshold")
-      .doc("Configures the minimum size in bytes for a partition that is considered as a skewed " +
-        "partition in adaptive skewed join.")
-      .bytesConf(ByteUnit.BYTE)
-      .createWithDefaultString("64MB")
-
   val ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR =
     buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionFactor")
       .doc("A partition is considered as a skewed partition if its size is larger than" +
         " this factor multiple the median partition size and also larger than " +
-        s" ${ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key}")
+        s" ${SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key}")
       .intConf
+      .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(10)
 
   val NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN =
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 578d2d744f85c..d3cb8645fc7ea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -34,6 +34,30 @@ import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExcha
 import org.apache.spark.sql.execution.joins.SortMergeJoinExec
 import org.apache.spark.sql.internal.SQLConf
 
+/**
+ * A rule to optimize skewed joins to avoid straggler tasks whose share of data are significantly
+ * larger than those of the rest of the tasks.
+ *
+ * The general idea is to divide each skew partition into smaller partitions and replicate its
+ * matching partition on the other side of the join so that they can run in parallel tasks.
+ * Note that when matching partitions from the left side and the right side both have skew,
+ * it will become a cartesian product of splits from left and right joining together.
+ *
+ * For example, assume the Sort-Merge join has 4 partitions:
+ * left:  [L1, L2, L3, L4]
+ * right: [R1, R2, R3, R4]
+ *
+ * Let's say L2, L4 and R3, R4 are skewed, and each of them get split into 2 sub-partitions. This
+ * is scheduled to run 4 tasks at the beginning: (L1, R1), (L2, R2), (L2, R2), (L2, R2).
+ * This rule expands it to 9 tasks to increase parallelism:
+ * (L1, R1),
+ * (L2-1, R2), (L2-2, R2),
+ * (L3, R3-1), (L3, R3-2),
+ * (L4-1, R4-1), (L4-2, R4-1), (L4-1, R4-2), (L4-2, R4-2)
+ *
+ * Note that, when this rule is enabled, it also coalesces non-skewed partitions like
+ * `ReduceNumShufflePartitions` does.
+ */
 case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
   private val ensureRequirements = EnsureRequirements(conf)
@@ -43,12 +67,12 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
   /**
    * A partition is considered as a skewed partition if its size is larger than the median
-   * partition size * spark.sql.adaptive.skewedPartitionFactor and also larger than
-   * spark.sql.adaptive.skewedPartitionSizeThreshold.
+   * partition size * ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR and also larger than
+   * SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.
    */
   private def isSkewed(size: Long, medianSize: Long): Boolean = {
     size > medianSize * conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR) &&
-      size > conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD)
+      size > conf.getConf(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
   }
 
   private def medianSize(stats: MapOutputStatistics): Long = {
@@ -61,6 +85,19 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     }
   }
 
+  /**
+   * The goal of skew join optimization is to make the data distribution more even. The target size
+   * to split skewed partitions is the average size of non-skewed partition, or the
+   * target post-shuffle partition size if avg size is smaller than it.
+   */
+  private def targetSize(stats: MapOutputStatistics, medianSize: Long): Long = {
+    val targetPostShuffleSize = conf.getConf(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
+    val nonSkewSizes = stats.bytesByPartitionId.filterNot(isSkewed(_, medianSize))
+    // It's impossible that all the partitions are skewed, as we use median size to define skew.
+    assert(nonSkewSizes.nonEmpty)
+    math.max(targetPostShuffleSize, nonSkewSizes.sum / nonSkewSizes.length)
+  }
+
   /**
    * Get the map size of the specific reduce shuffle Id.
    */
@@ -72,19 +109,19 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   /**
    * Split the skewed partition based on the map size and the max split number.
    */
-  private def getMapStartIndices(stage: ShuffleQueryStageExec, partitionId: Int): Array[Int] = {
+  private def getMapStartIndices(
+      stage: ShuffleQueryStageExec,
+      partitionId: Int,
+      targetSize: Long): Array[Int] = {
     val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
     val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
-    val avgPartitionSize = mapPartitionSizes.sum / mapPartitionSizes.length
-    val advisoryPartitionSize = math.max(avgPartitionSize,
-      conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD))
     val partitionStartIndices = ArrayBuffer[Int]()
     partitionStartIndices += 0
     var i = 0
     var postMapPartitionSize = 0L
     while (i < mapPartitionSizes.length) {
       val nextMapPartitionSize = mapPartitionSizes(i)
-      if (i > 0 && postMapPartitionSize + nextMapPartitionSize > advisoryPartitionSize) {
+      if (i > 0 && postMapPartitionSize + nextMapPartitionSize > targetSize) {
         partitionStartIndices += i
         postMapPartitionSize = nextMapPartitionSize
       } else {
@@ -152,6 +189,8 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
         """.stripMargin)
       val canSplitLeft = canSplitLeftSide(joinType)
       val canSplitRight = canSplitRightSide(joinType)
+      val leftTargetSize = targetSize(leftStats, leftMedSize)
+      val rightTargetSize = targetSize(rightStats, rightMedSize)
 
       val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
       val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
@@ -179,7 +218,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
             leftSkewDesc.addPartitionSize(leftSize)
             createSkewPartitions(
               partitionIndex,
-              getMapStartIndices(left, partitionIndex),
+              getMapStartIndices(left, partitionIndex, leftTargetSize),
               getNumMappers(left))
           } else {
             Seq(SinglePartitionSpec(partitionIndex))
@@ -189,7 +228,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
             rightSkewDesc.addPartitionSize(rightSize)
             createSkewPartitions(
               partitionIndex,
-              getMapStartIndices(right, partitionIndex),
+              getMapStartIndices(right, partitionIndex, rightTargetSize),
               getNumMappers(right))
           } else {
             Seq(SinglePartitionSpec(partitionIndex))
@@ -236,7 +275,8 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       rightStats: MapOutputStatistics,
       nonSkewPartitionIndices: Seq[Int]): Seq[ShufflePartitionSpec] = {
     assert(nonSkewPartitionIndices.nonEmpty)
-    if (nonSkewPartitionIndices.length == 1) {
+    val shouldCoalesce = conf.getConf(SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED)
+    if (!shouldCoalesce || nonSkewPartitionIndices.length == 1) {
       Seq(SinglePartitionSpec(nonSkewPartitionIndices.head))
     } else {
       val startIndices = ShufflePartitionsCoalescer.coalescePartitions(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 379e17f01d6be..64566af332afb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -583,7 +583,6 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "100",
       SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
       withTempView("skewData1", "skewData2") {
         spark
@@ -609,8 +608,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_SIZE_THRESHOLD.key -> "2000",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
+      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "2000") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)

From 16c76688640b662038737d9de66e541e8051b345 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 25 Feb 2020 15:17:16 -0800
Subject: [PATCH 0162/1280] [SPARK-30481][DOCS][FOLLOWUP] Document event log
 compaction into new section of monitoring.md

### What changes were proposed in this pull request?

This is a FOLLOW-UP PR for review comment on #27208 : https://github.com/apache/spark/pull/27208#pullrequestreview-347451714

This PR documents a new feature `Eventlog Compaction` into the new section of `monitoring.md`, as it only has one configuration on the SHS side and it's hard to explain everything on the description on the single configuration.

### Why are the changes needed?

Event log compaction lacks the documentation for what it is and how it helps. This PR will explain it.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Built docs via jekyll.

> change on the new section

<img width="951" alt="Screen Shot 2020-02-16 at 2 23 18 PM" src="https://user-images.githubusercontent.com/1317309/74599587-eb9efa80-50c7-11ea-942c-f7744268e40b.png">

> change on the table

<img width="1126" alt="Screen Shot 2020-01-30 at 5 08 12 PM" src="https://user-images.githubusercontent.com/1317309/73431190-2e9c6680-4383-11ea-8ce0-815f10917ddd.png">

Closes #27398 from HeartSaVioR/SPARK-30481-FOLLOWUP-document-new-feature.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 02f8165343fb5c6fc4e5a1874252abdfe886b5b2)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/monitoring.md | 57 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 13 deletions(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index c30aa99679399..4cba15b35e11d 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -95,6 +95,48 @@ The history server can be configured as follows:
   </tr>
 </table>
 
+### Applying compaction on rolling event log files
+
+A long-running application (e.g. streaming) can bring a huge single event log file which may cost a lot to maintain and
+also requires a bunch of resource to replay per each update in Spark History Server.
+
+Enabling <code>spark.eventLog.rolling.enabled</code> and <code>spark.eventLog.rolling.maxFileSize</code> would
+let you have rolling event log files instead of single huge event log file which may help some scenarios on its own,
+but it still doesn't help you reducing the overall size of logs.
+
+Spark History Server can apply compaction on the rolling event log files to reduce the overall size of
+logs, via setting the configuration <code>spark.history.fs.eventLog.rolling.maxFilesToRetain</code> on the
+Spark History Server.
+
+Details will be described below, but please note in prior that compaction is LOSSY operation.
+Compaction will discard some events which will be no longer seen on UI - you may want to check which events will be discarded
+before enabling the option.
+
+When the compaction happens, the History Server lists all the available event log files for the application, and considers
+the event log files having less index than the file with smallest index which will be retained as target of compaction.
+For example, if the application A has 5 event log files and <code>spark.history.fs.eventLog.rolling.maxFilesToRetain</code> is set to 2, then first 3 log files will be selected to be compacted.
+
+Once it selects the target, it analyzes them to figure out which events can be excluded, and rewrites them
+into one compact file with discarding events which are decided to exclude.
+
+The compaction tries to exclude the events which point to the outdated data. As of now, below describes the candidates of events to be excluded:
+
+* Events for the job which is finished, and related stage/tasks events
+* Events for the executor which is terminated
+* Events for the SQL execution which is finished, and related job/stage/tasks events
+
+Once rewriting is done, original log files will be deleted, via best-effort manner. The History Server may not be able to delete
+the original log files, but it will not affect the operation of the History Server.
+
+Please note that Spark History Server may not compact the old event log files if figures out not a lot of space
+would be reduced during compaction. For streaming query we normally expect compaction
+will run as each micro-batch will trigger one or more jobs which will be finished shortly, but compaction won't run
+in many cases for batch query.
+
+Please also note that this is a new feature introduced in Spark 3.0, and may not be completely stable. Under some circumstances,
+the compaction may exclude more events than you expect, leading some UI issues on History Server for the application.
+Use it with caution.
+
 ### Spark History Server Configuration Options
 
 Security options for the Spark History Server are covered more detail in the
@@ -303,19 +345,8 @@ Security options for the Spark History Server are covered more detail in the
     <td>Int.MaxValue</td>
     <td>
       The maximum number of event log files which will be retained as non-compacted. By default,
-      all event log files will be retained.<br/>
-      Please note that compaction will happen in Spark History Server, which means this configuration
-      should be set to the configuration of Spark History server, and the same value will be applied
-      across applications which are being loaded in Spark History Server. This also means compaction
-      and cleanup would require running Spark History Server.<br/>
-      Please set the configuration in Spark History Server, and <code>spark.eventLog.rolling.maxFileSize</code>
-      in each application accordingly if you want to control the overall size of event log files.
-      The event log files older than these retained files will be compacted into single file and
-      deleted afterwards.<br/>
-      NOTE: Spark History Server may not compact the old event log files if it figures
-      out not a lot of space would be reduced during compaction. For streaming query
-      (including Structured Streaming) we normally expect compaction will run, but for
-      batch query compaction won't run in many cases.
+      all event log files will be retained. The lowest value is 1 for technical reason.<br/>
+      Please read the section of "Applying compaction of old event log files" for more details.
     </td>
   </tr>
 </table>

From 53430594587ad0134eff5cd2b5e06a7a3eec1b99 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 25 Feb 2020 15:29:36 -0800
Subject: [PATCH 0163/1280] [SPARK-30943][SS] Show "batch ID" in tool tip
 string for Structured Streaming UI graphs

### What changes were proposed in this pull request?

This patch changes the tool tip string in Structured Streaming UI graphs to show batch ID (and timestamp as well) instead of only showing timestamp, which was a key for DStream but no longer a key for Structured Streaming.

This patch does some refactoring as there're some spots on confusion between js file for streaming and structured streaming.

Note that this patch doesn't actually change the x axis, as once we change it we should decouple the logic for graphs between streaming and structured streaming. It won't change UX meaningfully as in x axis we only show min and max which we still would like to know about "time" as well as batch ID.

### Why are the changes needed?

In Structured Streaming, everything is aligned for "batch ID" where the UI is only showing timestamp - end users have to manually find and correlate batch ID and the timestamp which is clearly a huge pain.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Manually tested. Screenshots:

![Screen Shot 2020-02-25 at 7 22 38 AM](https://user-images.githubusercontent.com/1317309/75197701-40b2ce80-57a2-11ea-9578-c2eb2d1091de.png)
![Screen Shot 2020-02-25 at 7 22 44 AM](https://user-images.githubusercontent.com/1317309/75197704-427c9200-57a2-11ea-9439-e0a8303d0860.png)
![Screen Shot 2020-02-25 at 7 22 58 AM](https://user-images.githubusercontent.com/1317309/75197706-43152880-57a2-11ea-9617-1276c3ba181e.png)
![Screen Shot 2020-02-25 at 7 23 04 AM](https://user-images.githubusercontent.com/1317309/75197708-43152880-57a2-11ea-9de2-7d37eaf88102.png)
![Screen Shot 2020-02-25 at 7 23 31 AM](https://user-images.githubusercontent.com/1317309/75197710-43adbf00-57a2-11ea-9ae4-4e292de39c36.png)

Closes #27687 from HeartSaVioR/SPARK-30943.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
(cherry picked from commit 9ea6c0a8975a1277abba799b51aca4e2818c23e7)
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
---
 .../apache/spark/ui/static/streaming-page.js  |  2 +-
 .../ui/static/structured-streaming-page.js    |  4 +--
 .../ui/StreamingQueryStatisticsPage.scala     | 36 +++++++++++++++----
 .../spark/streaming/ui/StreamingPage.scala    | 13 ++++++-
 4 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
index 5b75bc3011b6d..ed3e65c386dce 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
@@ -171,7 +171,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
             .attr("cy", function(d) { return y(d.y); })
             .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
             .on('mouseover', function(d) {
-                var tip = formatYValue(d.y) + " " + unitY + " at " + timeFormat[d.x];
+                var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
                 showBootstrapTooltip(d3.select(this).node(), tip);
                 // show the point
                 d3.select(this)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
index 70250fdbd2d0c..c92226b408b6c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js
@@ -106,12 +106,12 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
         .on('mouseover', function(d) {
             var tip = '';
             var idx = 0;
-            var _values = timeToValues[d._x]
+            var _values = formattedTimeToValues[d._x];
             _values.forEach(function (k) {
                 tip += labels[idx] + ': ' + k + '   ';
                 idx += 1;
             });
-            tip += " at " + d._x
+            tip += " at " + formattedTimeTipStrings[d._x];
             showBootstrapTooltip(d3.select(this).node(), tip);
         })
         .on('mouseout',  function() {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
index 56672ce328bff..fa9896e03f3fc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
@@ -70,11 +70,30 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
     <script>{Unparsed(js)}</script>
   }
 
-  def generateVar(values: Array[(Long, ju.Map[String, JLong])]): Seq[Node] = {
+  def generateTimeTipStrings(values: Array[(Long, Long)]): Seq[Node] = {
+    val js = "var timeTipStrings = {};\n" + values.map { case (batchId, time) =>
+      val formattedTime = SparkUIUtils.formatBatchTime(time, 1, showYYYYMMSS = false)
+      s"timeTipStrings[$time] = 'batch $batchId ($formattedTime)';"
+    }.mkString("\n")
+
+    <script>{Unparsed(js)}</script>
+  }
+
+  def generateFormattedTimeTipStrings(values: Array[(Long, Long)]): Seq[Node] = {
+    val js = "var formattedTimeTipStrings = {};\n" + values.map { case (batchId, time) =>
+      val formattedTime = SparkUIUtils.formatBatchTime(time, 1, showYYYYMMSS = false)
+      s"""formattedTimeTipStrings["$formattedTime"] = 'batch $batchId ($formattedTime)';"""
+    }.mkString("\n")
+
+    <script>{Unparsed(js)}</script>
+  }
+
+  def generateTimeToValues(values: Array[(Long, ju.Map[String, JLong])]): Seq[Node] = {
     val durationDataPadding = SparkUIUtils.durationDataPadding(values)
-    val js = "var timeToValues = {};\n" + durationDataPadding.map { case (x, y) =>
+    val js = "var formattedTimeToValues = {};\n" + durationDataPadding.map { case (x, y) =>
       val s = y.toSeq.sortBy(_._1).map(e => s""""${e._2}"""").mkString("[", ",", "]")
-      s"""timeToValues["${SparkUIUtils.formatBatchTime(x, 1, showYYYYMMSS = false)}"] = $s;"""
+      val formattedTime = SparkUIUtils.formatBatchTime(x, 1, showYYYYMMSS = false)
+      s"""formattedTimeToValues["$formattedTime"] = $s;"""
     }.mkString("\n")
 
     <script>{Unparsed(js)}</script>
@@ -112,8 +131,10 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
   }
 
   def generateStatTable(query: StreamingQueryUIData): Seq[Node] = {
-    val batchTimes = withNoProgress(query,
-      query.recentProgress.map(p => df.parse(p.timestamp).getTime), Array.empty[Long])
+    val batchToTimestamps = withNoProgress(query,
+      query.recentProgress.map(p => (p.batchId, df.parse(p.timestamp).getTime)),
+      Array.empty[(Long, Long)])
+    val batchTimes = batchToTimestamps.map(_._2)
     val minBatchTime =
       withNoProgress(query, df.parse(query.recentProgress.head.timestamp).getTime, 0L)
     val maxBatchTime =
@@ -266,6 +287,9 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
       </table>
     // scalastyle:on
 
-    generateVar(operationDurationData) ++ generateTimeMap(batchTimes) ++ table ++ jsCollector.toHtml
+    generateTimeToValues(operationDurationData) ++
+      generateFormattedTimeTipStrings(batchToTimestamps) ++
+      generateTimeMap(batchTimes) ++ generateTimeTipStrings(batchToTimestamps) ++
+      table ++ jsCollector.toHtml
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
index d47287b6077f8..8596aa035391d 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
@@ -140,6 +140,16 @@ private[ui] class StreamingPage(parent: StreamingTab)
     <script>{Unparsed(js)}</script>
   }
 
+  private def generateTimeTipStrings(times: Seq[Long]): Seq[Node] = {
+    // We leverage timeFormat as the value would be same as timeFormat. This means it is
+    // sensitive to the order - generateTimeMap should be called earlier than this.
+    val js = "var timeTipStrings = {};\n" + times.map { time =>
+      s"timeTipStrings[$time] = timeFormat[$time];"
+    }.mkString("\n")
+
+    <script>{Unparsed(js)}</script>
+  }
+
   private def generateStatTable(): Seq[Node] = {
     val batches = listener.retainedBatches
 
@@ -313,7 +323,8 @@ private[ui] class StreamingPage(parent: StreamingTab)
     </table>
     // scalastyle:on
 
-    generateTimeMap(batchTimes) ++ table ++ jsCollector.toHtml
+    generateTimeMap(batchTimes) ++ generateTimeTipStrings(batchTimes) ++ table ++
+      jsCollector.toHtml
   }
 
   private def generateInputDStreamsTable(

From 742e35f1d48c2523dda2ce21d73b7ab5ade20582 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 26 Feb 2020 11:55:05 +0900
Subject: [PATCH 0164/1280] [SPARK-30689][CORE][FOLLOW-UP] Rename config name
 of discovery plugin

### What changes were proposed in this pull request?

Rename config `spark.resources.discovery.plugin` to `spark.resources.discoveryPlugin`.

Also, as a side minor change: labeled `ResourceDiscoveryScriptPlugin` as `DeveloperApi` since it's not for end user.

### Why are the changes needed?

Discovery plugin doesn't need to reserve the "discovery" namespace here and it's more consistent with the interface name `ResourceDiscoveryPlugin` if we use `discoveryPlugin` instead.

### Does this PR introduce any user-facing change?

No, it's newly added in Spark3.0.

### How was this patch tested?

Pass Jenkins.

Closes #27689 from Ngone51/spark_30689_followup.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e9fd52282e4ed4831c5922348b0e1ee71e045b4b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/internal/config/package.scala   | 2 +-
 .../apache/spark/resource/ResourceDiscoveryScriptPlugin.scala   | 2 ++
 docs/configuration.md                                           | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 3f36e61fe1127..37ce178407381 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -55,7 +55,7 @@ package object config {
       .createOptional
 
   private[spark] val RESOURCES_DISCOVERY_PLUGIN =
-    ConfigBuilder("spark.resources.discovery.plugin")
+    ConfigBuilder("spark.resources.discoveryPlugin")
       .doc("Comma-separated list of class names implementing" +
         "org.apache.spark.api.resource.ResourceDiscoveryPlugin to load into the application." +
         "This is for advanced users to replace the resource discovery class with a " +
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 2ac6d3c500f9d..7027d1e3511b5 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -21,6 +21,7 @@ import java.io.File
 import java.util.Optional
 
 import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.api.resource.ResourceDiscoveryPlugin
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils.executeAndGetOutput
@@ -32,6 +33,7 @@ import org.apache.spark.util.Utils.executeAndGetOutput
  * If the user specifies custom plugins, this is the last one to be executed and
  * throws if the resource isn't discovered.
  */
+@DeveloperApi
 class ResourceDiscoveryScriptPlugin extends ResourceDiscoveryPlugin with Logging {
   override def discoverResource(
       request: ResourceRequest,
diff --git a/docs/configuration.md b/docs/configuration.md
index 2421e00f39068..469feed1745c4 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -244,7 +244,7 @@ of the most common options to set are:
   </td>
 </tr>
 <tr>
- <td><code>spark.resources.discovery.plugin</code></td>
+ <td><code>spark.resources.discoveryPlugin</code></td>
   <td>org.apache.spark.resource.ResourceDiscoveryScriptPlugin</td>
   <td>
     Comma-separated list of class names implementing

From 84345c7e67c9dfd47ec76d5a3d2ad76b62f959b6 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 9 Feb 2020 13:14:30 +0800
Subject: [PATCH 0165/1280] [SPARK-30662][ML][PYSPARK] Put back the API changes
 for HasBlockSize in ALS/MLP

### What changes were proposed in this pull request?
Add ```HasBlockSize``` in shared Params in both Scala and Python.
Make ALS/MLP extend ```HasBlockSize```

### Why are the changes needed?
Add ```HasBlockSize ``` in ALS, so user can specify the blockSize.
Make ```HasBlockSize``` a shared param so both ALS and MLP can use it.

### Does this PR introduce any user-facing change?
Yes
```ALS.setBlockSize/getBlockSize```
```ALSModel.setBlockSize/getBlockSize```

### How was this patch tested?
Manually tested. Also added doctest.

Closes #27501 from huaxingao/spark_30662.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: zhengruifeng <ruifengz@foxmail.com>
---
 .../MultilayerPerceptronClassifier.scala      | 22 +--------
 .../ml/param/shared/SharedParamsCodeGen.scala |  6 ++-
 .../spark/ml/param/shared/sharedParams.scala  | 17 +++++++
 .../apache/spark/ml/recommendation/ALS.scala  | 46 ++++++++++++++-----
 python/pyspark/ml/classification.py           | 22 ++++-----
 .../ml/param/_shared_params_code_gen.py       |  5 +-
 python/pyspark/ml/param/shared.py             | 17 +++++++
 python/pyspark/ml/recommendation.py           | 29 +++++++++---
 8 files changed, 109 insertions(+), 55 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
index c7a8237849b5b..6e8f92b9b1e64 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala
@@ -34,7 +34,7 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
 
 /** Params for Multilayer Perceptron. */
 private[classification] trait MultilayerPerceptronParams extends ProbabilisticClassifierParams
-  with HasSeed with HasMaxIter with HasTol with HasStepSize with HasSolver {
+  with HasSeed with HasMaxIter with HasTol with HasStepSize with HasSolver with HasBlockSize {
 
   import MultilayerPerceptronClassifier._
 
@@ -54,26 +54,6 @@ private[classification] trait MultilayerPerceptronParams extends ProbabilisticCl
   @Since("1.5.0")
   final def getLayers: Array[Int] = $(layers)
 
-  /**
-   * Block size for stacking input data in matrices to speed up the computation.
-   * Data is stacked within partitions. If block size is more than remaining data in
-   * a partition then it is adjusted to the size of this data.
-   * Recommended size is between 10 and 1000.
-   * Default: 128
-   *
-   * @group expertParam
-   */
-  @Since("1.5.0")
-  final val blockSize: IntParam = new IntParam(this, "blockSize",
-    "Block size for stacking input data in matrices. Data is stacked within partitions." +
-      " If block size is more than remaining data in a partition then " +
-      "it is adjusted to the size of this data. Recommended size is between 10 and 1000",
-    ParamValidators.gt(0))
-
-  /** @group expertGetParam */
-  @Since("1.5.0")
-  final def getBlockSize: Int = $(blockSize)
-
   /**
    * The solver algorithm for optimization.
    * Supported options: "gd" (minibatch gradient descent) or "l-bfgs".
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
index 7ac680ec1183a..6194dfacf01fc 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
@@ -104,7 +104,11 @@ private[shared] object SharedParamsCodeGen {
         isValid = "ParamValidators.inArray(Array(\"euclidean\", \"cosine\"))"),
       ParamDesc[String]("validationIndicatorCol", "name of the column that indicates whether " +
         "each row is for training or for validation. False indicates training; true indicates " +
-        "validation.")
+        "validation."),
+      ParamDesc[Int]("blockSize", "block size for stacking input data in matrices. Data is " +
+        "stacked within partitions. If block size is more than remaining data in a partition " +
+        "then it is adjusted to the size of this data.",
+        isValid = "ParamValidators.gt(0)", isExpertParam = true)
     )
 
     val code = genSharedParams(params)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
index 44c993eeafddc..0c0d2b5981dbe 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
@@ -578,4 +578,21 @@ trait HasValidationIndicatorCol extends Params {
   /** @group getParam */
   final def getValidationIndicatorCol: String = $(validationIndicatorCol)
 }
+
+/**
+ * Trait for shared param blockSize. This trait may be changed or
+ * removed between minor versions.
+ */
+@DeveloperApi
+trait HasBlockSize extends Params {
+
+  /**
+   * Param for block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data..
+   * @group expertParam
+   */
+  final val blockSize: IntParam = new IntParam(this, "blockSize", "block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.", ParamValidators.gt(0))
+
+  /** @group expertGetParam */
+  final def getBlockSize: Int = $(blockSize)
+}
 // scalastyle:on
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 2fb9a276be887..002146f89e79a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -54,7 +54,8 @@ import org.apache.spark.util.random.XORShiftRandom
 /**
  * Common params for ALS and ALSModel.
  */
-private[recommendation] trait ALSModelParams extends Params with HasPredictionCol {
+private[recommendation] trait ALSModelParams extends Params with HasPredictionCol
+  with HasBlockSize {
   /**
    * Param for the column name for user ids. Ids must be integers. Other
    * numeric types are supported for this column, but will be cast to integers as long as they
@@ -125,6 +126,8 @@ private[recommendation] trait ALSModelParams extends Params with HasPredictionCo
 
   /** @group expertGetParam */
   def getColdStartStrategy: String = $(coldStartStrategy).toLowerCase(Locale.ROOT)
+
+  setDefault(blockSize -> 4096)
 }
 
 /**
@@ -288,6 +291,15 @@ class ALSModel private[ml] (
   @Since("2.2.0")
   def setColdStartStrategy(value: String): this.type = set(coldStartStrategy, value)
 
+  /**
+   * Set block size for stacking input data in matrices.
+   * Default is 4096.
+   *
+   * @group expertSetParam
+   */
+  @Since("3.0.0")
+  def setBlockSize(value: Int): this.type = set(blockSize, value)
+
   private val predict = udf { (featuresA: Seq[Float], featuresB: Seq[Float]) =>
     if (featuresA != null && featuresB != null) {
       var dotProduct = 0.0f
@@ -351,7 +363,7 @@ class ALSModel private[ml] (
    */
   @Since("2.2.0")
   def recommendForAllUsers(numItems: Int): DataFrame = {
-    recommendForAll(userFactors, itemFactors, $(userCol), $(itemCol), numItems)
+    recommendForAll(userFactors, itemFactors, $(userCol), $(itemCol), numItems, $(blockSize))
   }
 
   /**
@@ -366,7 +378,7 @@ class ALSModel private[ml] (
   @Since("2.3.0")
   def recommendForUserSubset(dataset: Dataset[_], numItems: Int): DataFrame = {
     val srcFactorSubset = getSourceFactorSubset(dataset, userFactors, $(userCol))
-    recommendForAll(srcFactorSubset, itemFactors, $(userCol), $(itemCol), numItems)
+    recommendForAll(srcFactorSubset, itemFactors, $(userCol), $(itemCol), numItems, $(blockSize))
   }
 
   /**
@@ -377,7 +389,7 @@ class ALSModel private[ml] (
    */
   @Since("2.2.0")
   def recommendForAllItems(numUsers: Int): DataFrame = {
-    recommendForAll(itemFactors, userFactors, $(itemCol), $(userCol), numUsers)
+    recommendForAll(itemFactors, userFactors, $(itemCol), $(userCol), numUsers, $(blockSize))
   }
 
   /**
@@ -392,7 +404,7 @@ class ALSModel private[ml] (
   @Since("2.3.0")
   def recommendForItemSubset(dataset: Dataset[_], numUsers: Int): DataFrame = {
     val srcFactorSubset = getSourceFactorSubset(dataset, itemFactors, $(itemCol))
-    recommendForAll(srcFactorSubset, userFactors, $(itemCol), $(userCol), numUsers)
+    recommendForAll(srcFactorSubset, userFactors, $(itemCol), $(userCol), numUsers, $(blockSize))
   }
 
   /**
@@ -441,11 +453,12 @@ class ALSModel private[ml] (
       dstFactors: DataFrame,
       srcOutputColumn: String,
       dstOutputColumn: String,
-      num: Int): DataFrame = {
+      num: Int,
+      blockSize: Int): DataFrame = {
     import srcFactors.sparkSession.implicits._
 
-    val srcFactorsBlocked = blockify(srcFactors.as[(Int, Array[Float])])
-    val dstFactorsBlocked = blockify(dstFactors.as[(Int, Array[Float])])
+    val srcFactorsBlocked = blockify(srcFactors.as[(Int, Array[Float])], blockSize)
+    val dstFactorsBlocked = blockify(dstFactors.as[(Int, Array[Float])], blockSize)
     val ratings = srcFactorsBlocked.crossJoin(dstFactorsBlocked)
       .as[(Seq[(Int, Array[Float])], Seq[(Int, Array[Float])])]
       .flatMap { case (srcIter, dstIter) =>
@@ -483,11 +496,10 @@ class ALSModel private[ml] (
 
   /**
    * Blockifies factors to improve the efficiency of cross join
-   * TODO: SPARK-20443 - expose blockSize as a param?
    */
   private def blockify(
       factors: Dataset[(Int, Array[Float])],
-      blockSize: Int = 4096): Dataset[Seq[(Int, Array[Float])]] = {
+      blockSize: Int): Dataset[Seq[(Int, Array[Float])]] = {
     import factors.sparkSession.implicits._
     factors.mapPartitions(_.grouped(blockSize))
   }
@@ -654,6 +666,15 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
   @Since("2.2.0")
   def setColdStartStrategy(value: String): this.type = set(coldStartStrategy, value)
 
+  /**
+   * Set block size for stacking input data in matrices.
+   * Default is 4096.
+   *
+   * @group expertSetParam
+   */
+  @Since("3.0.0")
+  def setBlockSize(value: Int): this.type = set(blockSize, value)
+
   /**
    * Sets both numUserBlocks and numItemBlocks to the specific value.
    *
@@ -683,7 +704,7 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
     instr.logDataset(dataset)
     instr.logParams(this, rank, numUserBlocks, numItemBlocks, implicitPrefs, alpha, userCol,
       itemCol, ratingCol, predictionCol, maxIter, regParam, nonnegative, checkpointInterval,
-      seed, intermediateStorageLevel, finalStorageLevel)
+      seed, intermediateStorageLevel, finalStorageLevel, blockSize)
 
     val (userFactors, itemFactors) = ALS.train(ratings, rank = $(rank),
       numUserBlocks = $(numUserBlocks), numItemBlocks = $(numItemBlocks),
@@ -694,7 +715,8 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
       checkpointInterval = $(checkpointInterval), seed = $(seed))
     val userDF = userFactors.toDF("id", "features")
     val itemDF = itemFactors.toDF("id", "features")
-    val model = new ALSModel(uid, $(rank), userDF, itemDF).setParent(this)
+    val model = new ALSModel(uid, $(rank), userDF, itemDF).setBlockSize($(blockSize))
+      .setParent(this)
     copyValues(model)
   }
 
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 5ab8e606bda03..1436b78b11202 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -2153,7 +2153,7 @@ def sigma(self):
 
 
 class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, HasMaxIter,
-                                  HasTol, HasStepSize, HasSolver):
+                                  HasTol, HasStepSize, HasSolver, HasBlockSize):
     """
     Params for :py:class:`MultilayerPerceptronClassifier`.
 
@@ -2164,11 +2164,6 @@ class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, H
                    "E.g., Array(780, 100, 10) means 780 inputs, one hidden layer with 100 " +
                    "neurons and output layer of 10 neurons.",
                    typeConverter=TypeConverters.toListInt)
-    blockSize = Param(Params._dummy(), "blockSize", "Block size for stacking input data in " +
-                      "matrices. Data is stacked within partitions. If block size is more than " +
-                      "remaining data in a partition then it is adjusted to the size of this " +
-                      "data. Recommended size is between 10 and 1000, default is 128.",
-                      typeConverter=TypeConverters.toInt)
     solver = Param(Params._dummy(), "solver", "The solver algorithm for optimization. Supported " +
                    "options: l-bfgs, gd.", typeConverter=TypeConverters.toString)
     initialWeights = Param(Params._dummy(), "initialWeights", "The initial weights of the model.",
@@ -2181,13 +2176,6 @@ def getLayers(self):
         """
         return self.getOrDefault(self.layers)
 
-    @since("1.6.0")
-    def getBlockSize(self):
-        """
-        Gets the value of blockSize or its default value.
-        """
-        return self.getOrDefault(self.blockSize)
-
     @since("2.0.0")
     def getInitialWeights(self):
         """
@@ -2211,11 +2199,17 @@ class MultilayerPerceptronClassifier(JavaProbabilisticClassifier, _MultilayerPer
     ...     (1.0, Vectors.dense([0.0, 1.0])),
     ...     (1.0, Vectors.dense([1.0, 0.0])),
     ...     (0.0, Vectors.dense([1.0, 1.0]))], ["label", "features"])
-    >>> mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], blockSize=1, seed=123)
+    >>> mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
     >>> mlp.setMaxIter(100)
     MultilayerPerceptronClassifier...
     >>> mlp.getMaxIter()
     100
+    >>> mlp.getBlockSize()
+    128
+    >>> mlp.setBlockSize(1)
+    MultilayerPerceptronClassifier...
+    >>> mlp.getBlockSize()
+    1
     >>> model = mlp.fit(df)
     >>> model.setFeaturesCol("features")
     MultilayerPerceptronClassificationModel...
diff --git a/python/pyspark/ml/param/_shared_params_code_gen.py b/python/pyspark/ml/param/_shared_params_code_gen.py
index ded3ca84b30f2..2086e831f4282 100644
--- a/python/pyspark/ml/param/_shared_params_code_gen.py
+++ b/python/pyspark/ml/param/_shared_params_code_gen.py
@@ -164,7 +164,10 @@ def get$Name(self):
          "'euclidean'", "TypeConverters.toString"),
         ("validationIndicatorCol", "name of the column that indicates whether each row is for " +
          "training or for validation. False indicates training; true indicates validation.",
-         None, "TypeConverters.toString")]
+         None, "TypeConverters.toString"),
+        ("blockSize", "block size for stacking input data in matrices. Data is stacked within "
+         "partitions. If block size is more than remaining data in a partition then it is "
+         "adjusted to the size of this data.", None, "TypeConverters.toInt")]
 
     code = []
     for name, doc, defaultValueStr, typeConverter in shared:
diff --git a/python/pyspark/ml/param/shared.py b/python/pyspark/ml/param/shared.py
index 8fc115691f1ab..24fb0d3e2554d 100644
--- a/python/pyspark/ml/param/shared.py
+++ b/python/pyspark/ml/param/shared.py
@@ -580,3 +580,20 @@ def getValidationIndicatorCol(self):
         Gets the value of validationIndicatorCol or its default value.
         """
         return self.getOrDefault(self.validationIndicatorCol)
+
+
+class HasBlockSize(Params):
+    """
+    Mixin for param blockSize: block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.
+    """
+
+    blockSize = Param(Params._dummy(), "blockSize", "block size for stacking input data in matrices. Data is stacked within partitions. If block size is more than remaining data in a partition then it is adjusted to the size of this data.", typeConverter=TypeConverters.toInt)
+
+    def __init__(self):
+        super(HasBlockSize, self).__init__()
+
+    def getBlockSize(self):
+        """
+        Gets the value of blockSize or its default value.
+        """
+        return self.getOrDefault(self.blockSize)
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index ee276962c898b..99d80aa867bda 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -28,7 +28,7 @@
 
 
 @inherit_doc
-class _ALSModelParams(HasPredictionCol):
+class _ALSModelParams(HasPredictionCol, HasBlockSize):
     """
     Params for :py:class:`ALS` and :py:class:`ALSModel`.
 
@@ -223,6 +223,8 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     0.1
     >>> als.clear(als.regParam)
     >>> model = als.fit(df)
+    >>> model.getBlockSize()
+    4096
     >>> model.getUserCol()
     'user'
     >>> model.setUserCol("user")
@@ -282,13 +284,13 @@ def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemB
                  implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None,
                  ratingCol="rating", nonnegative=False, checkpointInterval=10,
                  intermediateStorageLevel="MEMORY_AND_DISK",
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan"):
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096):
         """
         __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10, \
                  implicitPrefs=false, alpha=1.0, userCol="user", itemCol="item", seed=None, \
                  ratingCol="rating", nonnegative=false, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096)
         """
         super(ALS, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid)
@@ -296,7 +298,8 @@ def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemB
                          implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
                          ratingCol="rating", nonnegative=False, checkpointInterval=10,
                          intermediateStorageLevel="MEMORY_AND_DISK",
-                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
+                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan",
+                         blockSize=4096)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -306,13 +309,13 @@ def setParams(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItem
                   implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None,
                   ratingCol="rating", nonnegative=False, checkpointInterval=10,
                   intermediateStorageLevel="MEMORY_AND_DISK",
-                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan"):
+                  finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096):
         """
         setParams(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10, \
                  implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item", seed=None, \
                  ratingCol="rating", nonnegative=False, checkpointInterval=10, \
                  intermediateStorageLevel="MEMORY_AND_DISK", \
-                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
+                 finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan", blockSize=4096)
         Sets params for ALS.
         """
         kwargs = self._input_kwargs
@@ -443,6 +446,13 @@ def setSeed(self, value):
         """
         return self._set(seed=value)
 
+    @since("3.0.0")
+    def setBlockSize(self, value):
+        """
+        Sets the value of :py:attr:`blockSize`.
+        """
+        return self._set(blockSize=value)
+
 
 class ALSModel(JavaModel, _ALSModelParams, JavaMLWritable, JavaMLReadable):
     """
@@ -479,6 +489,13 @@ def setPredictionCol(self, value):
         """
         return self._set(predictionCol=value)
 
+    @since("3.0.0")
+    def setBlockSize(self, value):
+        """
+        Sets the value of :py:attr:`blockSize`.
+        """
+        return self._set(blockSize=value)
+
     @property
     @since("1.4.0")
     def rank(self):

From 3e1795b4aeba5a627dc3e45888cefbcccd1c8941 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 26 Feb 2020 20:48:43 +0900
Subject: [PATCH 0166/1280] [SPARK-30798][SQL][TESTS][FOLLOW-UP] Set the
 configuration against the current session explicitly in
 HiveShowCreateTableSuite

### What changes were proposed in this pull request?

After https://github.com/apache/spark/pull/27387 (see https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-2.7-hive-2.3/202/), the tests below fail consistently, specifically in one job https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-2.7-hive-2.3/ in Jenkins

```
org.apache.spark.sql.hive.HiveShowCreateTableSuite.simple hive table
org.apache.spark.sql.hive.HiveShowCreateTableSuite.simple external hive table
org.apache.spark.sql.hive.HiveShowCreateTableSuite.hive bucketing is supported
```

The profile is same as PR builder but seems it fails specifically in this machine. Seems the legacy configuration `spark.sql.legacy.createHiveTableByDefault.enabled` is not being set due to the inconsistency between `SQLConf.get` and the active Spark session as described in the https://github.com/apache/spark/pull/27387.

This PR proposes to explicitly set the configuration against the session used instead of `SQLConf.get`.

### Why are the changes needed?

To make `spark-master-test-sbt-hadoop-2.7-hive-2.3` job pass.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Cannot reproduce in my local. Presumably it cannot be reproduced in the PR builder. We should see if the tests pass at `spark-master-test-sbt-hadoop-2.7-hive-2.3` job after this PR is merged

Closes #27703 from HyukjinKwon/SPARK-30798-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 020b2622e597458b925d7227ed5f9fa269f2d391)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/hive/HiveShowCreateTableSuite.scala | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
index 99db1e3cbd18c..50c9018e4e2a9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive
 
 import org.apache.spark.sql.{AnalysisException, ShowCreateTableSuite}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 
@@ -30,12 +30,13 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
   protected override def beforeAll(): Unit = {
     super.beforeAll()
     origCreateHiveTableConfig =
-      SQLConf.get.getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
-    SQLConf.get.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED, true)
+      spark.conf.get(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
+    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key, true)
   }
 
   protected override def afterAll(): Unit = {
-    SQLConf.get.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED,
+    spark.conf.set(
+      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key,
       origCreateHiveTableConfig)
     super.afterAll()
   }

From 0759e5e7e58a689810d063188c598a5747096895 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Thu, 27 Feb 2020 00:21:38 +0800
Subject: [PATCH 0167/1280] [SPARK-30782][SQL] Column resolution doesn't
 respect current catalog/namespace for v2 tables

### What changes were proposed in this pull request?

This PR proposes to fix an issue where qualified columns are not matched for v2 tables if current catalog/namespace are used.

For v1 tables, you can currently perform the following:
```SQL
SELECT default.t.id FROM t;
```

For v2 tables, the following fails:
```SQL
USE testcat.ns1.ns2;
SELECT testcat.ns1.ns2.t.id FROM t;

org.apache.spark.sql.AnalysisException: cannot resolve '`testcat.ns1.ns2.t.id`' given input columns: [t.id, t.point]; line 1 pos 7;
```

### Why are the changes needed?

It is a bug since qualified column names cannot match if current catalog/namespace are used.

### Does this PR introduce any user-facing change?

Yes, now the following works:
```SQL
USE testcat.ns1.ns2;
SELECT testcat.ns1.ns2.t.id FROM t;
```

### How was this patch tested?

Added new tests

Closes #27532 from imback82/qualifed_col_respect_current.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 73305475c10f1218bd2060e8575ab4072d0cc50b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  8 +--
 .../sql/connector/DataSourceV2SQLSuite.scala  | 54 +++++++++++++------
 2 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index aec71747fde68..3d797994e77e9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -807,8 +807,10 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = ResolveTempViews(plan).resolveOperatorsUp {
       case u: UnresolvedRelation =>
         lookupV2Relation(u.multipartIdentifier)
-          .map(SubqueryAlias(u.multipartIdentifier, _))
-          .getOrElse(u)
+          .map { rel =>
+            val ident = rel.identifier.get
+            SubqueryAlias(rel.catalog.get.name +: ident.namespace :+ ident.name, rel)
+          }.getOrElse(u)
 
       case u @ UnresolvedTable(NonSessionCatalogAndIdentifier(catalog, ident)) =>
         CatalogV2Util.loadTable(catalog, ident)
@@ -933,7 +935,7 @@ class Analyzer(
               v1SessionCatalog.getRelation(v1Table.v1Table)
             case table =>
               SubqueryAlias(
-                identifier,
+                ident.asMultipartIdentifier,
                 DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
           }
           val key = catalog.name +: ident.namespace :+ ident.name
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index f642114983293..4ff20939b3dfd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -685,12 +685,21 @@ class DataSourceV2SQLSuite
       sql(s"CREATE TABLE $t (id bigint, point struct<x: bigint, y: bigint>) USING foo")
       sql(s"INSERT INTO $t VALUES (1, (10, 20))")
 
-      checkAnswer(
-        sql(s"SELECT testcat.ns1.ns2.tbl.id, testcat.ns1.ns2.tbl.point.x FROM $t"),
-        Row(1, 10))
-      checkAnswer(sql(s"SELECT ns1.ns2.tbl.id, ns1.ns2.tbl.point.x FROM $t"), Row(1, 10))
-      checkAnswer(sql(s"SELECT ns2.tbl.id, ns2.tbl.point.x FROM $t"), Row(1, 10))
-      checkAnswer(sql(s"SELECT tbl.id, tbl.point.x FROM $t"), Row(1, 10))
+      def check(tbl: String): Unit = {
+        checkAnswer(
+          sql(s"SELECT testcat.ns1.ns2.tbl.id, testcat.ns1.ns2.tbl.point.x FROM $tbl"),
+          Row(1, 10))
+        checkAnswer(sql(s"SELECT ns1.ns2.tbl.id, ns1.ns2.tbl.point.x FROM $tbl"), Row(1, 10))
+        checkAnswer(sql(s"SELECT ns2.tbl.id, ns2.tbl.point.x FROM $tbl"), Row(1, 10))
+        checkAnswer(sql(s"SELECT tbl.id, tbl.point.x FROM $tbl"), Row(1, 10))
+      }
+
+      // Test with qualified table name "testcat.ns1.ns2.tbl".
+      check(t)
+
+      // Test if current catalog and namespace is respected in column resolution.
+      sql("USE testcat.ns1.ns2")
+      check("tbl")
 
       val ex = intercept[AnalysisException] {
         sql(s"SELECT ns1.ns2.ns3.tbl.id from $t")
@@ -700,19 +709,30 @@ class DataSourceV2SQLSuite
   }
 
   test("qualified column names for v1 tables") {
-    // unset this config to use the default v2 session catalog.
-    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
-
-    withTable("t") {
-      sql("CREATE TABLE t USING json AS SELECT 1 AS i")
-      checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
-      checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
+    Seq(true, false).foreach { useV1Table =>
+      val format = if (useV1Table) "json" else v2Format
+      if (useV1Table) {
+        // unset this config to use the default v2 session catalog.
+        spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+      } else {
+        spark.conf.set(
+          V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[InMemoryTableSessionCatalog].getName)
+      }
 
-      // catalog name cannot be used for v1 tables.
-      val ex = intercept[AnalysisException] {
-        sql(s"select spark_catalog.default.t.i from spark_catalog.default.t")
+      withTable("t") {
+        sql(s"CREATE TABLE t USING $format AS SELECT 1 AS i")
+        checkAnswer(sql("select i from t"), Row(1))
+        checkAnswer(sql("select t.i from t"), Row(1))
+        checkAnswer(sql("select default.t.i from t"), Row(1))
+        checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
+        checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
+
+        // catalog name cannot be used for tables in the session catalog.
+        val ex = intercept[AnalysisException] {
+          sql(s"select spark_catalog.default.t.i from spark_catalog.default.t")
+        }
+        assert(ex.getMessage.contains("cannot resolve '`spark_catalog.default.t.i`"))
       }
-      assert(ex.getMessage.contains("cannot resolve '`spark_catalog.default.t.i`"))
     }
   }
 

From d88557fb3e27c721d2b4838d8a0eff280c243939 Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Thu, 27 Feb 2020 01:48:12 +0800
Subject: [PATCH 0168/1280] [SPARK-27619][SQL] MapType should be prohibited in
 hash expressions

### What changes were proposed in this pull request?
`hash()` and `xxhash64()` cannot be used on elements of `Maptype`. A new configuration `spark.sql.legacy.useHashOnMapType` is introduced to allow users to restore the previous behaviour.

When `spark.sql.legacy.useHashOnMapType` is set to false:

```
scala> spark.sql("select hash(map())");
org.apache.spark.sql.AnalysisException: cannot resolve 'hash(map())' due to data type mismatch: input to function hash cannot contain elements of MapType; line 1 pos 7;
'Project [unresolvedalias(hash(map(), 42), None)]
+- OneRowRelation
```

when `spark.sql.legacy.useHashOnMapType` is set to true :

```
scala> spark.sql("set spark.sql.legacy.useHashOnMapType=true");
res3: org.apache.spark.sql.DataFrame = [key: string, value: string]

scala> spark.sql("select hash(map())").first()
res4: org.apache.spark.sql.Row = [42]

```

### Why are the changes needed?

As discussed in Jira, SparkSql's map hashcodes depends on their order of insertion which is not consistent with the normal scala behaviour which might confuse users.
Code snippet from JIRA :
```
val a = spark.createDataset(Map(1->1, 2->2) :: Nil)
val b = spark.createDataset(Map(2->2, 1->1) :: Nil)

// Demonstration of how Scala Map equality is unaffected by insertion order:
assert(Map(1->1, 2->2).hashCode() == Map(2->2, 1->1).hashCode())
assert(Map(1->1, 2->2) == Map(2->2, 1->1))
assert(a.first() == b.first())

// In contrast, this will print two different hashcodes:
println(Seq(a, b).map(_.selectExpr("hash(*)").first()))
```

Also `MapType` is prohibited for aggregation / joins / equality comparisons #7819 and set operations #17236.

### Does this PR introduce any user-facing change?
Yes. Now users cannot use hash functions on elements of `mapType`. To restore the previous behaviour set `spark.sql.legacy.useHashOnMapType` to true.

### How was this patch tested?
UT added.

Closes #27580 from iRakson/SPARK-27619.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c913b9d8b5c2e125c441f7fe923e333cd8ac2c2c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |  2 ++
 .../spark/sql/catalyst/expressions/hash.scala | 15 ++++++++++++---
 .../apache/spark/sql/internal/SQLConf.scala   |  6 ++++++
 .../expressions/HashExpressionsSuite.scala    | 14 --------------
 .../org/apache/spark/sql/SQLQuerySuite.scala  | 19 +++++++++++++++++++
 5 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 7e52e6924d9fc..7e0a536d0f75f 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -224,6 +224,8 @@ license: |
 
   - Since Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, e.g. `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, while casting string to integrals and booleans, it will not trim the whitespaces from both ends, the foregoing results will be `null`, while to datetimes, only the trailing spaces (= ASCII 32) will be removed.
 
+  - Since Spark 3.0, An analysis exception will be thrown when hash expressions are applied on elements of MapType. To restore the behavior before Spark 3.0, set `spark.sql.legacy.useHashOnMapType` to true.
+    
   - Since Spark 3.0, numbers written in scientific notation(e.g. `1E2`) would be parsed as Double. In Spark version 2.4 and earlier, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
 
   - Since Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index d8eed27df79df..5be7f54107759 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{ArrayData, MapData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.Platform
 import org.apache.spark.unsafe.hash.Murmur3_x86_32
@@ -232,9 +233,6 @@ case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInp
  *  - array:              The `result` starts with seed, then use `result` as seed, recursively
  *                        calculate hash value for each element, and assign the element hash value
  *                        to `result`.
- *  - map:                The `result` starts with seed, then use `result` as seed, recursively
- *                        calculate hash value for each key-value, and assign the key-value hash
- *                        value to `result`.
  *  - struct:             The `result` starts with seed, then use `result` as seed, recursively
  *                        calculate hash value for each field, and assign the field hash value to
  *                        `result`.
@@ -249,10 +247,21 @@ abstract class HashExpression[E] extends Expression {
 
   override def nullable: Boolean = false
 
+  private def hasMapType(dt: DataType): Boolean = {
+    dt.existsRecursively(_.isInstanceOf[MapType])
+  }
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.length < 1) {
       TypeCheckResult.TypeCheckFailure(
         s"input to function $prettyName requires at least one argument")
+    } else if (children.exists(child => hasMapType(child.dataType)) &&
+      !SQLConf.get.getConf(SQLConf.LEGACY_USE_HASH_ON_MAPTYPE)) {
+      TypeCheckResult.TypeCheckFailure(
+        s"input to function $prettyName cannot contain elements of MapType. In Spark, same maps " +
+          "may have different hashcode, thus hash expressions are prohibited on MapType " +
+          s"elements. To restore previous behavior set ${SQLConf.LEGACY_USE_HASH_ON_MAPTYPE.key} " +
+          "to true.")
     } else {
       TypeCheckResult.TypeCheckSuccess
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e6f7cfdf43dcc..fdd3e9ced8957 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2200,6 +2200,12 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val LEGACY_USE_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.useHashOnMapType")
+    .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
+      "an analysis exception will be thrown.")
+    .booleanConf
+    .createWithDefault(false)
+
   /**
    * Holds information about keys that have been deprecated.
    *
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
index e6cf979649c83..68da1faaa8f45 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
@@ -554,28 +554,14 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       .add("arrayOfString", arrayOfString)
       .add("arrayOfArrayOfString", ArrayType(arrayOfString))
       .add("arrayOfArrayOfInt", ArrayType(ArrayType(IntegerType)))
-      .add("arrayOfMap", ArrayType(mapOfString))
       .add("arrayOfStruct", ArrayType(structOfString))
       .add("arrayOfUDT", arrayOfUDT))
 
-  testHash(
-    new StructType()
-      .add("mapOfIntAndString", MapType(IntegerType, StringType))
-      .add("mapOfStringAndArray", MapType(StringType, arrayOfString))
-      .add("mapOfArrayAndInt", MapType(arrayOfString, IntegerType))
-      .add("mapOfArray", MapType(arrayOfString, arrayOfString))
-      .add("mapOfStringAndStruct", MapType(StringType, structOfString))
-      .add("mapOfStructAndString", MapType(structOfString, StringType))
-      .add("mapOfStruct", MapType(structOfString, structOfString)))
-
   testHash(
     new StructType()
       .add("structOfString", structOfString)
       .add("structOfStructOfString", new StructType().add("struct", structOfString))
       .add("structOfArray", new StructType().add("array", arrayOfString))
-      .add("structOfMap", new StructType().add("map", mapOfString))
-      .add("structOfArrayAndMap",
-        new StructType().add("array", arrayOfString).add("map", mapOfString))
       .add("structOfUDT", structOfUDT))
 
   test("hive-hash for decimal") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index c87e6a4e39b10..78ceeaa09c125 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -2120,6 +2120,25 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
+  test("SPARK-27619: Throw analysis exception when hash and xxhash64 is used on MapType") {
+    Seq("hash", "xxhash64").foreach {
+      case hashExpression =>
+        intercept[AnalysisException] {
+          spark.createDataset(Map(1 -> 10, 2 -> 20) :: Nil).selectExpr(s"$hashExpression(*)")
+        }
+    }
+  }
+
+  test("SPARK-27619: when spark.sql.legacy.useHashOnMapType is true, hash can be used on Maptype") {
+    Seq("hash", "xxhash64").foreach {
+      case hashExpression =>
+        withSQLConf(SQLConf.LEGACY_USE_HASH_ON_MAPTYPE.key -> "true") {
+          val df = spark.createDataset(Map() :: Nil)
+          checkAnswer(df.selectExpr(s"$hashExpression(*)"), sql(s"SELECT $hashExpression(map())"))
+        }
+    }
+  }
+
   test("xxhash64 function") {
     val df = Seq(1 -> "a", 2 -> "b").toDF("i", "j")
     withTempView("tbl") {

From 99b0cad5247eec0c5c22e23b7b9bb39602a171cf Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 26 Feb 2020 13:59:43 -0800
Subject: [PATCH 0169/1280] [SPARK-30918][SQL][FOLLOWUP] Fix typo in
 OptimizeSkewedJoin

### What changes were proposed in this pull request?

This is a follow up of #27669 in order to fix a typo.

### Why are the changes needed?

N/A

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #27714 from cloud-fan/typo.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit eced93215f42d9d579b0480b25e143ae9bf5581a)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/execution/adaptive/OptimizeSkewedJoin.scala       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index d3cb8645fc7ea..dd238fe7785bb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -48,7 +48,7 @@ import org.apache.spark.sql.internal.SQLConf
  * right: [R1, R2, R3, R4]
  *
  * Let's say L2, L4 and R3, R4 are skewed, and each of them get split into 2 sub-partitions. This
- * is scheduled to run 4 tasks at the beginning: (L1, R1), (L2, R2), (L2, R2), (L2, R2).
+ * is scheduled to run 4 tasks at the beginning: (L1, R1), (L2, R2), (L3, R3), (L4, R4).
  * This rule expands it to 9 tasks to increase parallelism:
  * (L1, R1),
  * (L2-1, R2), (L2-2, R2),

From cfc48a8a3068972791410e8e36ff9cf1ba5af445 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Wed, 26 Feb 2020 14:25:58 -0800
Subject: [PATCH 0170/1280] [SPARK-30759][SQL][3.0] Fix cache initialization in
 StringRegexExpression

### What changes were proposed in this pull request?
In the PR, I propose to fix `cache` initialization in `StringRegexExpression` by changing of expected value type in `case Literal(value: String, StringType)` from `String` to `UTF8String`.

This is a backport of #27502 and #27547

### Why are the changes needed?
Actually, the case doesn't work at all because `Literal`'s value has type `UTF8String`, see
<img width="649" alt="Screen Shot 2020-02-08 at 22 45 50" src="https://user-images.githubusercontent.com/1580697/74091681-0d4a2180-4acb-11ea-8a0d-7e8c65f4214e.png">

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added new test by `RegexpExpressionsSuite`.

Closes #27713 from MaxGekk/str-regexp-foldable-pattern-backport.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/expressions/regexpExpressions.scala      | 2 +-
 .../sql/catalyst/expressions/RegexpExpressionsSuite.scala | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index 7891c2868e366..fc71943495bc7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -40,7 +40,7 @@ abstract class StringRegexExpression extends BinaryExpression
 
   // try cache the pattern for Literal
   private lazy val cache: Pattern = right match {
-    case Literal(value: String, StringType) => compile(value)
+    case Literal(value: UTF8String, StringType) => compile(value.toString)
     case _ => null
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 86da62bc74940..712d2bc4c4736 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -329,4 +329,12 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringSplit(s1, s2, -1), null, row3)
   }
 
+  test("SPARK-30759: cache initialization for literal patterns") {
+    val expr = "A" like Literal.create("a", StringType)
+    expr.eval()
+    val cache = expr.getClass.getSuperclass
+      .getDeclaredFields.filter(_.getName.endsWith("cache")).head
+    cache.setAccessible(true)
+    assert(cache.get(expr).asInstanceOf[java.util.regex.Pattern].pattern().contains("a"))
+  }
 }

From 20e0e8d596ddd9d612cd4e4d12ac1bef99851230 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 26 Feb 2020 14:40:32 -0800
Subject: [PATCH 0171/1280] [MINOR][BUILD] Fix make-distribution.sh to show
 usage without 'echo' cmd

### What changes were proposed in this pull request?

turn off `x` mode and do not print the command while printing usage of `make-distribution.sh`

### Why are the changes needed?

improve dev tools

### Does this PR introduce any user-facing change?

for only developers, clearer hints

#### after
```
./dev/make-distribution.sh --hel
+++ dirname ./dev/make-distribution.sh
++ cd ./dev/..
++ pwd
+ SPARK_HOME=/Users/kentyao/spark
+ DISTDIR=/Users/kentyao/spark/dist
+ MAKE_TGZ=false
+ MAKE_PIP=false
+ MAKE_R=false
+ NAME=none
+ MVN=/Users/kentyao/spark/build/mvn
+ ((  1  ))
+ case $1 in
+ echo 'Error: --hel is not supported'
Error: --hel is not supported
+ exit_with_usage
+ set +x
make-distribution.sh - tool for making binary distributions of Spark

usage:
make-distribution.sh [--name] [--tgz] [--pip] [--r] [--mvn <mvn-command>] <maven build options>
See Spark's "Building Spark" doc for correct Maven options.
```

#### before
```
+++ dirname ./dev/make-distribution.sh
++ cd ./dev/..
++ pwd
+ SPARK_HOME=/Users/kentyao/spark
+ DISTDIR=/Users/kentyao/spark/dist
+ MAKE_TGZ=false
+ MAKE_PIP=false
+ MAKE_R=false
+ NAME=none
+ MVN=/Users/kentyao/spark/build/mvn
+ ((  1  ))
+ case $1 in
+ echo 'Error: --hel is not supported'
Error: --hel is not supported
+ exit_with_usage
+ echo 'make-distribution.sh - tool for making binary distributions of Spark'
make-distribution.sh - tool for making binary distributions of Spark
+ echo ''

+ echo usage:
usage:
+ cl_options='[--name] [--tgz] [--pip] [--r] [--mvn <mvn-command>]'
+ echo 'make-distribution.sh [--name] [--tgz] [--pip] [--r] [--mvn <mvn-command>] <maven build options>'
make-distribution.sh [--name] [--tgz] [--pip] [--r] [--mvn <mvn-command>] <maven build options>
+ echo 'See Spark'\''s "Building Spark" doc for correct Maven options.'
See Spark's "Building Spark" doc for correct Maven options.
+ echo ''

+ exit 1
```

### How was this patch tested?

manually

Closes #27706 from yaooqinn/build.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit a6026c830a582af75a0d95d18f7759922a086334)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 dev/make-distribution.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/make-distribution.sh b/dev/make-distribution.sh
index 0b30eec76bb53..9e767ce5a3daa 100755
--- a/dev/make-distribution.sh
+++ b/dev/make-distribution.sh
@@ -39,6 +39,7 @@ NAME=none
 MVN="$SPARK_HOME/build/mvn"
 
 function exit_with_usage {
+  set +x
   echo "make-distribution.sh - tool for making binary distributions of Spark"
   echo ""
   echo "usage:"

From 51fd2f9e68db59cf73e70a4dc8fa927bb0151772 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Wed, 26 Feb 2020 19:45:15 -0600
Subject: [PATCH 0172/1280] [SPARK-30928][ML] Remove unnecessary MiMa excludes

### What changes were proposed in this pull request?
https://issues.apache.org/jira/browse/SPARK-30928
remove unnecessary MiMa excludes

### Why are the changes needed?

When auditing binary incompatible changes for 3.0, I found several MiMa excludes are not necessary, so remove these.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
run dev/mima to check

Closes #27696 from huaxingao/spark-mima.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 2c9db5608d2d65181a690c32bd0c3e83107b2b9c)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 project/MimaExcludes.scala | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 289ccb2cf3664..d69caa9e44649 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -47,7 +47,7 @@ object MimaExcludes {
     // [SPARK-29399][core] Remove old ExecutorPlugin interface.
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ExecutorPlugin"),
 
-    // [SPARK-][SQL][CORE][MLLIB] Remove more old deprecated items in Spark 3
+    // [SPARK-28980][SQL][CORE][MLLIB] Remove more old deprecated items in Spark 3
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.createExternalTable"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.createExternalTable"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.train"),
@@ -79,6 +79,7 @@ object MimaExcludes {
 
     // [SPARK-27090][CORE] Removing old LEGACY_DRIVER_IDENTIFIER ("<driver>")
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.SparkContext.LEGACY_DRIVER_IDENTIFIER"),
+
     // [SPARK-25838] Remove formatVersion from Saveable
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.DistributedLDAModel.formatVersion"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.LocalLDAModel.formatVersion"),
@@ -231,13 +232,6 @@ object MimaExcludes {
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.QuantileDiscretizer.relativeError"),
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.QuantileDiscretizer.getRelativeError"),
 
-    // [SPARK-25959] GBTClassifier picks wrong impurity stats on loading
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.ml.tree.HasVarianceImpurity.org$apache$spark$ml$tree$HasVarianceImpurity$_setter_$impurity_="),
-
     // [SPARK-28968][ML] Add HasNumFeatures in the scala side
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.FeatureHasher.getNumFeatures"),
     ProblemFilters.exclude[FinalMethodProblem]("org.apache.spark.ml.feature.FeatureHasher.numFeatures"),
@@ -316,12 +310,6 @@ object MimaExcludes {
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ml.feature.OneHotEncoderEstimator"),
     ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.feature.OneHotEncoder"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.transform"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.getInputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.getOutputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.inputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.setInputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.setOutputCol"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.feature.OneHotEncoder.outputCol"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ml.feature.OneHotEncoderEstimator$"),
 
     // [SPARK-30329][ML] add iterator/foreach methods for Vectors
@@ -453,7 +441,6 @@ object MimaExcludes {
 
     // [SPARK-25382][SQL][PYSPARK] Remove ImageSchema.readImages in 3.0
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.image.ImageSchema.readImages"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.image.ImageSchema.readImages"),
 
     // [SPARK-25341][CORE] Support rolling back a shuffle map stage and re-generate the shuffle files
     ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.shuffle.sort.UnsafeShuffleWriter.this"),

From 95df63c45fd9c8159ddee00014023eda10df93dc Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 26 Feb 2020 18:01:26 -0800
Subject: [PATCH 0173/1280] [SPARK-30906][SQL][TESTS][FOLLOW-UP] Set the
 configuration against TestHive explicitly in HiveSerDeSuite

### What changes were proposed in this pull request?

After https://github.com/apache/spark/pull/27659 (see https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-2.7-hive-2.3/253/), the tests below fail consistently, specifically in one job https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-sbt-hadoop-2.7-hive-2.3/ in Jenkins

```
org.apache.spark.sql.hive.execution.HiveSerDeSuite.Test the default fileformat for Hive-serde tables
```

The profile is same as PR builder but seems it fails specifically in this machine.

Several configurations used in `HiveSerDeSuite` are not being set presumably due to the inconsistency between `SQLConf.get` and the active Spark session described in the https://github.com/apache/spark/pull/27387, and as a side effect of the cloned session at https://github.com/apache/spark/pull/27659.

This PR proposes to explicitly set the configuration against `TestHive` by using `withExistingConf` at `withSQLConf`

### Why are the changes needed?

To make `spark-master-test-sbt-hadoop-2.7-hive-2.3` job pass.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Cannot reproduce in my local. Presumably it cannot be reproduced in the PR builder. We should see if the tests pass at `spark-master-test-sbt-hadoop-2.7-hive-2.3` job after this PR is merged.

Closes #27705 from HyukjinKwon/SPARK-30906.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit cd3ef2249fc6bd60e4ba62ee046dcf4115864206)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
index 9a1190af02fac..d2d350221aca0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
@@ -82,6 +82,10 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
     }.head
   }
 
+  // Make sure we set the config values to TestHive.conf.
+  override def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit =
+    SQLConf.withExistingConf(TestHive.conf)(super.withSQLConf(pairs: _*)(f))
+
   test("Test the default fileformat for Hive-serde tables") {
     withSQLConf("hive.default.fileformat" -> "orc",
       SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {

From c9463c6f8e7e4ee0a0f6bf09618569ea5a84aa03 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 26 Feb 2020 19:24:41 -0800
Subject: [PATCH 0174/1280] [SPARK-30963][INFRA] Add GitHub Action job for
 document generation

### What changes were proposed in this pull request?

This PR aims to add a new `GitHub Action` job for document generation.

### Why are the changes needed?

We had better test the document generation in PR Builder.
- https://lists.apache.org/thread.html/rd06a2154e853812652b8f7fa3c003746ed531b213c531517f055e1dc%40%3Cdev.spark.apache.org%3E

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the GitHub Action in this PR.

Closes #27715 from dongjoon-hyun/SPARK-30963.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 2b744fe88547acc82a6178ff806b78056db7a025)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/master.yml | 37 ++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 0f80c88eba2f2..33ea335e95111 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -117,3 +117,40 @@ jobs:
       run: ./R/install-dev.sh
     - name: lint-r
       run: ./dev/lint-r
+
+  docs:
+    runs-on: ubuntu-latest
+    name: Generate documents
+    steps:
+    - uses: actions/checkout@master
+    - uses: actions/cache@v1
+      with:
+        path: ~/.m2/repository
+        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          docs-maven-repo-
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '1.8'
+    - uses: actions/setup-python@v1
+      with:
+        python-version: '3.x'
+        architecture: 'x64'
+    - uses: actions/setup-ruby@v1
+      with:
+        ruby-version: '2.7'
+    - name: Install R
+      run: |
+        echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' | sudo tee -a /etc/apt/sources.list
+        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
+        sudo apt-get update
+        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
+    - name: Install packages
+      run: |
+        pip install sphinx mkdocs numpy
+        gem install jekyll jekyll-redirect-from rouge
+        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
+    - name: Run jekyll build
+      run: |
+        cd docs
+        jekyll build

From dcd2094371db90c2bfe5dcfdd7a01033db88727d Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 27 Feb 2020 13:42:02 +0800
Subject: [PATCH 0175/1280] [SPARK-27924][SQL][FOLLOW-UP] Improve ANSI SQL
 Boolean-Predicate

### What changes were proposed in this pull request?
This PR follows https://github.com/apache/spark/pull/25074 and improves the implement.

### Why are the changes needed?
Improve code.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27699 from beliefer/improve-boolean-test.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1515d45b8db69de67cf61100f7cf4fa5c7cadbd4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/expressions/predicates.scala | 60 -------------------
 .../sql/catalyst/parser/AstBuilder.scala      |  8 +--
 .../catalyst/expressions/PredicateSuite.scala | 32 +---------
 3 files changed, 6 insertions(+), 94 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index bcd442ad3cc35..bd190c3e5abc7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -927,66 +927,6 @@ case class GreaterThanOrEqual(left: Expression, right: Expression)
   protected override def nullSafeEval(input1: Any, input2: Any): Any = ordering.gteq(input1, input2)
 }
 
-trait BooleanTest extends UnaryExpression with Predicate with ExpectsInputTypes {
-
-  def boolValueForComparison: Boolean
-  def boolValueWhenNull: Boolean
-
-  override def nullable: Boolean = false
-  override def inputTypes: Seq[DataType] = Seq(BooleanType)
-
-  override def eval(input: InternalRow): Any = {
-    val value = child.eval(input)
-    Option(value) match {
-      case None => boolValueWhenNull
-      case other => if (boolValueWhenNull) {
-        value == !boolValueForComparison
-      } else {
-        value == boolValueForComparison
-      }
-    }
-  }
-
-  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val eval = child.genCode(ctx)
-    ev.copy(code = code"""
-      ${eval.code}
-      ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
-      if (${eval.isNull}) {
-        ${ev.value} = $boolValueWhenNull;
-      } else if ($boolValueWhenNull) {
-        ${ev.value} = ${eval.value} == !$boolValueForComparison;
-      } else {
-        ${ev.value} = ${eval.value} == $boolValueForComparison;
-      }
-      """, isNull = FalseLiteral)
-  }
-}
-
-case class IsTrue(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = true
-  override def boolValueWhenNull: Boolean = false
-  override def sql: String = s"(${child.sql} IS TRUE)"
-}
-
-case class IsNotTrue(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = true
-  override def boolValueWhenNull: Boolean = true
-  override def sql: String = s"(${child.sql} IS NOT TRUE)"
-}
-
-case class IsFalse(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = false
-  override def boolValueWhenNull: Boolean = false
-  override def sql: String = s"(${child.sql} IS FALSE)"
-}
-
-case class IsNotFalse(child: Expression) extends BooleanTest {
-  override def boolValueForComparison: Boolean = false
-  override def boolValueWhenNull: Boolean = true
-  override def sql: String = s"(${child.sql} IS NOT FALSE)"
-}
-
 /**
  * IS UNKNOWN and IS NOT UNKNOWN are the same as IS NULL and IS NOT NULL, respectively,
  * except that the input expression must be of a boolean type.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 2806fd49fbc5e..fe724460b8073 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1414,12 +1414,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case SqlBaseParser.NULL =>
         IsNull(e)
       case SqlBaseParser.TRUE => ctx.NOT match {
-        case null => IsTrue(e)
-        case _ => IsNotTrue(e)
+        case null => EqualNullSafe(e, Literal(true))
+        case _ => Not(EqualNullSafe(e, Literal(true)))
       }
       case SqlBaseParser.FALSE => ctx.NOT match {
-        case null => IsFalse(e)
-        case _ => IsNotFalse(e)
+        case null => EqualNullSafe(e, Literal(false))
+        case _ => Not(EqualNullSafe(e, Literal(false)))
       }
       case SqlBaseParser.UNKNOWN => ctx.NOT match {
         case null => IsUnknown(e)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index 67a41e7cc2767..1ad0a8ed758f4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -522,37 +522,9 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
     assert(expression == expected)
   }
 
-  val row0 = create_row(null)
-  val row1 = create_row(false)
-  val row2 = create_row(true)
-
-  test("istrue and isnottrue") {
-    checkEvaluation(IsTrue(Literal.create(null, BooleanType)), false, row0)
-    checkEvaluation(IsNotTrue(Literal.create(null, BooleanType)), true, row0)
-    checkEvaluation(IsTrue(Literal.create(false, BooleanType)), false, row1)
-    checkEvaluation(IsNotTrue(Literal.create(false, BooleanType)), true, row1)
-    checkEvaluation(IsTrue(Literal.create(true, BooleanType)), true, row2)
-    checkEvaluation(IsNotTrue(Literal.create(true, BooleanType)), false, row2)
-    IsTrue(Literal.create(null, IntegerType)).checkInputDataTypes() match {
-      case TypeCheckResult.TypeCheckFailure(msg) =>
-        assert(msg.contains("argument 1 requires boolean type"))
-    }
-  }
-
-  test("isfalse and isnotfalse") {
-    checkEvaluation(IsFalse(Literal.create(null, BooleanType)), false, row0)
-    checkEvaluation(IsNotFalse(Literal.create(null, BooleanType)), true, row0)
-    checkEvaluation(IsFalse(Literal.create(false, BooleanType)), true, row1)
-    checkEvaluation(IsNotFalse(Literal.create(false, BooleanType)), false, row1)
-    checkEvaluation(IsFalse(Literal.create(true, BooleanType)), false, row2)
-    checkEvaluation(IsNotFalse(Literal.create(true, BooleanType)), true, row2)
-    IsFalse(Literal.create(null, IntegerType)).checkInputDataTypes() match {
-      case TypeCheckResult.TypeCheckFailure(msg) =>
-        assert(msg.contains("argument 1 requires boolean type"))
-    }
-  }
-
   test("isunknown and isnotunknown") {
+    val row0 = create_row(null)
+
     checkEvaluation(IsUnknown(Literal.create(null, BooleanType)), true, row0)
     checkEvaluation(IsNotUnknown(Literal.create(null, BooleanType)), false, row0)
     IsUnknown(Literal.create(null, IntegerType)).checkInputDataTypes() match {

From c3f494627b65e47c1cf7b7873fa28322a4cd0afa Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Thu, 27 Feb 2020 14:09:07 +0800
Subject: [PATCH 0176/1280] [SPARK-30590][SQL] Untyped select API cannot take
 typed column expression that needs input type

### What changes were proposed in this pull request?

This patch proposes to throw clear analysis exception if untyped `Dataset.select` takes typed column expression that needs input type.

### Why are the changes needed?

`Dataset` provides few typed `select` helper functions to select typed column expressions. The maximum number of typed columns supported is 5. If wanting to select more than 5 typed columns, it silently calls untyped `Dataset.select` and can causes weird unresolved error, like:

```
org.apache.spark.sql.AnalysisException: unresolved operator 'Aggregate [fooagg(FooAgg(1), None, None, None, input[0, int, false] AS value#114, assertnotnull(cast(value#114 as int)), input[0, int, false] AS value#113, IntegerType, IntegerType, false) AS foo_agg_1#116, fooagg(FooAgg(2), None, None, None, input[0, int, false] AS value#119, assertnotnull(cast(value#119 as int)), input[0, int, false] AS value#118, IntegerType, IntegerType, false) AS foo_agg_2#121, fooagg(FooAgg(3), None, None, None, input[0, int, false] AS value#124, assertnotnull(cast(value#124 as int)), input[0, int, false] AS value#123, IntegerType, IntegerType, false) AS foo_agg_3#126, fooagg(FooAgg(4), None, None, None, input[0, int, false] AS value#129, assertnotnull(cast(value#129 as int)), input[0, int, false] AS value#128, IntegerType, IntegerType, false) AS foo_agg_4#131, fooagg(FooAgg(5), None, None, None, input[0, int, false] AS value#134, assertnotnull(cast(value#134 as int)), input[0, int, false] AS value#133, IntegerType, IntegerType, false) AS foo_agg_5#136, fooagg(FooAgg(6), None, None, None, input[0, int, false] AS value#139, assertnotnull(cast(value#139 as int)), input[0, int, false] AS value#138, IntegerType, IntegerType, false) AS foo_agg_6#141];;
'Aggregate [fooagg(FooAgg(1), None, None, None, input[0, int, false] AS value#114, assertnotnull(cast(value#114 as int)), input[0, int, false] AS value#113, IntegerType, IntegerType, false) AS foo_agg_1#116, fooagg(FooAgg(2), None, None, None, input[0, int, false] AS value#119, assertnotnull(cast(value#119 as int)), input[0, int, false] AS value#118, IntegerType, IntegerType, false) AS foo_agg_2#121, fooagg(FooAgg(3), None, None, None, input[0, int, false] AS value#124, assertnotnull(cast(value#124 as int)), input[0, int, false] AS value#123, IntegerType, IntegerType, false) AS foo_agg_3#126, fooagg(FooAgg(4), None, None, None, input[0, int, false] AS value#129, assertnotnull(cast(value#129 as int)), input[0, int, false] AS value#128, IntegerType, IntegerType, false) AS foo_agg_4#131, fooagg(FooAgg(5), None, None, None, input[0, int, false] AS value#134, assertnotnull(cast(value#134 as int)), input[0, int, false] AS value#133, IntegerType, IntegerType, false) AS foo_agg_5#136, fooagg(FooAgg(6), None, None, None, input[0, int, false] AS value#139, assertnotnull(cast(value#139 as int)), input[0, int, false] AS value#138, IntegerType, IntegerType, false) AS foo_agg_6#141]
+- Project [_1#6 AS a#13, _2#7 AS b#14, _3#8 AS c#15, _4#9 AS d#16, _5#10 AS e#17, _6#11 AS F#18]
 +- LocalRelation [_1#6, _2#7, _3#8, _4#9, _5#10, _6#11]

at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:43)
 at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:95)
 at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$3.apply(CheckAnalysis.scala:431)
 at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$3.apply(CheckAnalysis.scala:430)
 at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:127)
 at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:430)
```

However, to fully disallow typed columns as input to untyped `select` API will break current usage like `count` that is a `TypedColumn` in `functions`. In order to keep compatibility, we should allow current usage of certain `TypedColumn`s as input to untyped `select` API. For the `TypedColumn`s that will cause unresolved exception, we should explicitly let users know that they are incorrectly calling untyped `select` with typed columns which need input type.

### Does this PR introduce any user-facing change?

Yes, but this PR only refines the error message.

When users call `Dataset.select` API with typed column that needs input type, an analysis exception will be thrown. Previously an unresolved error will be thrown.

### How was this patch tested?

Unit tests.

Closes #27499 from viirya/SPARK-30590.

Lead-authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Co-authored-by: Liang-Chi Hsieh <liangchi@uber.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 160c144baa82b7f76301b718694187cf939ca465)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/Dataset.scala  | 21 +++++++++++++++-
 .../spark/sql/DatasetAggregatorSuite.scala    | 24 +++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 42f35354e864f..d85e23b82f9db 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -48,6 +48,7 @@ import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, PartitioningC
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.catalyst.util.IntervalUtils
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
 import org.apache.spark.sql.execution.arrow.{ArrowBatchStreamWriter, ArrowConverters}
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -1432,7 +1433,25 @@ class Dataset[T] private[sql](
    */
   @scala.annotation.varargs
   def select(cols: Column*): DataFrame = withPlan {
-    Project(cols.map(_.named), logicalPlan)
+    val untypedCols = cols.map {
+      case typedCol: TypedColumn[_, _] =>
+        // Checks if a `TypedColumn` has been inserted with
+        // specific input type and schema by `withInputType`.
+        val needInputType = typedCol.expr.find {
+          case ta: TypedAggregateExpression if ta.inputDeserializer.isEmpty => true
+          case _ => false
+        }.isDefined
+
+        if (!needInputType) {
+          typedCol
+        } else {
+          throw new AnalysisException(s"Typed column $typedCol that needs input type and schema " +
+            "cannot be passed in untyped `select` API. Use the typed `Dataset.select` API instead.")
+        }
+
+      case other => other
+    }
+    Project(untypedCols.map(_.named), logicalPlan)
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
index 6ffe133ee652b..a22abd505ca00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetAggregatorSuite.scala
@@ -219,6 +219,15 @@ case class OptionBooleanIntAggregator(colName: String)
   def OptionalBoolIntEncoder: Encoder[Option[(Boolean, Int)]] = ExpressionEncoder()
 }
 
+case class FooAgg(s: Int) extends Aggregator[Row, Int, Int] {
+  def zero: Int = s
+  def reduce(b: Int, r: Row): Int = b + r.getAs[Int](0)
+  def merge(b1: Int, b2: Int): Int = b1 + b2
+  def finish(b: Int): Int = b
+  def bufferEncoder: Encoder[Int] = Encoders.scalaInt
+  def outputEncoder: Encoder[Int] = Encoders.scalaInt
+}
+
 class DatasetAggregatorSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
 
@@ -394,4 +403,19 @@ class DatasetAggregatorSuite extends QueryTest with SharedSparkSession {
     checkAnswer(group, Row("bob", Row(true, 3)) :: Nil)
     checkDataset(group.as[OptionBooleanIntData], OptionBooleanIntData("bob", Some((true, 3))))
   }
+
+  test("SPARK-30590: untyped select should not accept typed column that needs input type") {
+    val df = Seq((1, 2, 3, 4, 5, 6)).toDF("a", "b", "c", "d", "e", "f")
+    val fooAgg = (i: Int) => FooAgg(i).toColumn.name(s"foo_agg_$i")
+
+    val agg1 = df.select(fooAgg(1), fooAgg(2), fooAgg(3), fooAgg(4), fooAgg(5))
+    checkDataset(agg1, (3, 5, 7, 9, 11))
+
+    // Passes typed columns to untyped `Dataset.select` API.
+    val err = intercept[AnalysisException] {
+      df.select(fooAgg(1), fooAgg(2), fooAgg(3), fooAgg(4), fooAgg(5), fooAgg(6))
+    }.getMessage
+    assert(err.contains("cannot be passed in untyped `select` API. " +
+      "Use the typed `Dataset.select` API instead."))
+  }
 }

From 5f46049dbc5bec85af3fc1ca2237cd05d11e0af9 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 27 Feb 2020 00:18:46 -0800
Subject: [PATCH 0177/1280] [SPARK-23435][INFRA][FOLLOW-UP] Remove unnecessary
 dependency in AppVeyor

### What changes were proposed in this pull request?
`testthat` version was pinned to `1.0.2` at https://github.com/apache/spark/commit/f15102b1702b64a54233ae31357e32335722f4e5 due to compatibility issue in SparkR.
The compatibility issue is finally fixed as of https://github.com/apache/spark/commit/298d0a5102e54ddc24f114e83d2b936762722eec and we now use testthat latest version.

Now we don't need to install `crayon', 'praise' and 'R6' as they are dependences in testthat (https://github.com/r-lib/testthat/blob/master/DESCRIPTION).

### Why are the changes needed?
To minimise build specification and prevent dependency confusion.

### Does this PR introduce any user-facing change?
No. Dev only change.

### How was this patch tested?
AppVeyor build will test it out.

Closes #27717 from HyukjinKwon/SPARK-23435-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit b2a74107d50a0f69d727049581e01d9e2f6b4778)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 appveyor.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/appveyor.yml b/appveyor.yml
index 5d98260265b1a..fc0b7d53ddabc 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -42,8 +42,7 @@ install:
   # Install maven and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
   # Required package for R unit tests
-  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')"
-  - cmd: R -e "install.packages(c('crayon', 'praise', 'R6', 'testthat'), repos='https://cloud.r-project.org/')"
+  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')"
   - cmd: R -e "packageVersion('knitr'); packageVersion('rmarkdown'); packageVersion('testthat'); packageVersion('e1071'); packageVersion('survival'); packageVersion('arrow')"
 
 build_script:

From b00895ceded4da49793314833e5442249d05f461 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 27 Feb 2020 21:29:42 +0800
Subject: [PATCH 0178/1280] [SPARK-30937][DOC] Group Hive upgrade guides
 together

### What changes were proposed in this pull request?

This PR groups all hive upgrade related migration guides inside Spark 3.0 together.

Also add another behavior change of `ScriptTransform` in the new Hive section.

### Why are the changes needed?

Make the doc more clearly to user.

### Does this PR introduce any user-facing change?

No, new doc for Spark 3.0.

### How was this patch tested?

N/A.

Closes #27670 from Ngone51/hive_migration.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 22dfd15a4574a5cccdc54c96f11de28d58363016)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                            | 10 +++++++---
 .../sql/hive/execution/ScriptTransformationSuite.scala |  5 ++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 7e0a536d0f75f..d241a664357f6 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -254,7 +254,7 @@ license: |
         </tr>
     </table>
     
-  - Since Spark 3.0, CREATE TABLE without a specific provider will use the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and earlier, it was hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
+  - Since Spark 3.0, `CREATE TABLE` without a specific provider will use the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and earlier, it was hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
 
   - Since Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation will be coerced to double value, e.g. `+'1'` results `1.0`. In Spark version 2.4 and earlier, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, e.g. `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, e.g. in Spark 2.4, the result of `+'1'` is string `1`.
 
@@ -332,10 +332,14 @@ license: |
 
   - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For generating Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
 
-  - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3. This may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore.
+  - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3 and it brings following impacts:
+  
+    - You may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore you want to connect to.
   For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
   
-  - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3. You need to migrate your custom SerDes to Hive 2.3 or build your own Spark with `hive-1.2` profile. See HIVE-15167 for more details.
+    - You need to migrate your custom SerDes to Hive 2.3 or build your own Spark with `hive-1.2` profile. See HIVE-15167 for more details.
+
+    - The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
 
 ## Upgrading from Spark SQL 2.4.4 to 2.4.5
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
index 7d01fc53a4099..7153d3f03cd57 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
@@ -212,9 +212,8 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
           |FROM v
         """.stripMargin)
 
-      // In Hive1.2, it does not do well on Decimal conversion. For example, in this case,
-      // it converts a decimal value's type from Decimal(38, 18) to Decimal(1, 0). So we need
-      // do extra cast here for Hive1.2. But in Hive2.3, it still keeps the original Decimal type.
+      // In Hive 1.2, the string representation of a decimal omits trailing zeroes.
+      // But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
       val decimalToString: Column => Column = if (HiveUtils.isHive23) {
         c => c.cast("string")
       } else {

From 933e576aab0a40e53f275ae960fc45b7ed2d6f06 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 27 Feb 2020 23:12:35 +0900
Subject: [PATCH 0179/1280] [SPARK-30956][SQL][TESTS] Use intercept instead of
 try-catch to assert failures in IntervalUtilsSuite

### What changes were proposed in this pull request?

In this PR, I addressed the comment from https://github.com/apache/spark/pull/27672#discussion_r383719562 to use `intercept` instead of `try-catch` block to assert  failures in the IntervalUtilsSuite

### Why are the changes needed?

improve tests
### Does this PR introduce any user-facing change?

no

### How was this patch tested?

Nah

Closes #27700 from yaooqinn/intervaltest.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 2d2706cb86ddccd2fc60378b0f47a437ec354017)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../catalyst/util/IntervalUtilsSuite.scala    | 119 ++++--------------
 1 file changed, 26 insertions(+), 93 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index e7c3163d0ac09..1628a61418262 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -35,27 +35,17 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     assert(safeStringToInterval(UTF8String.fromString(input)) === expected)
   }
 
-  private def checkFromStringWithFunc(
-      input: String,
-      months: Int,
-      days: Int,
-      us: Long,
-      func: CalendarInterval => CalendarInterval): Unit = {
-    val expected = new CalendarInterval(months, days, us)
-    assert(func(stringToInterval(UTF8String.fromString(input))) === expected)
-    assert(func(safeStringToInterval(UTF8String.fromString(input))) === expected)
+  private def checkFromInvalidString(input: String, errorMsg: String): Unit = {
+    failFuncWithInvalidInput(input, errorMsg, s => stringToInterval(UTF8String.fromString(s)))
+    assert(safeStringToInterval(UTF8String.fromString(input)) === null)
   }
 
-  private def checkFromInvalidString(input: String, errorMsg: String): Unit = {
-    try {
-      stringToInterval(UTF8String.fromString(input))
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: IllegalArgumentException =>
-        val msg = e.getMessage
-        assert(msg.contains(errorMsg))
+  private def failFuncWithInvalidInput(
+      input: String, errorMsg: String, converter: String => CalendarInterval): Unit = {
+    withClue("Expected to throw an exception for the invalid input") {
+      val e = intercept[IllegalArgumentException](converter(input))
+      assert(e.getMessage.contains(errorMsg))
     }
-    assert(safeStringToInterval(UTF8String.fromString(input)) === null)
   }
 
   private def testSingleUnit(
@@ -87,7 +77,6 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
-
   test("string to interval: multiple units") {
     Seq(
       "-1 MONTH 1 day -1 microseconds" -> new CalendarInterval(-1, 1, -1),
@@ -145,22 +134,9 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     assert(fromYearMonthString("99-10") === new CalendarInterval(99 * 12 + 10, 0, 0L))
     assert(fromYearMonthString("+99-10") === new CalendarInterval(99 * 12 + 10, 0, 0L))
     assert(fromYearMonthString("-8-10") === new CalendarInterval(-8 * 12 - 10, 0, 0L))
-
-    try {
-      fromYearMonthString("99-15")
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: IllegalArgumentException =>
-        assert(e.getMessage.contains("month 15 outside range"))
-    }
-
-    try {
-      fromYearMonthString("9a9-15")
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: IllegalArgumentException =>
-        assert(e.getMessage.contains("Interval string does not match year-month format"))
-    }
+    failFuncWithInvalidInput("99-15", "month 15 outside range", fromYearMonthString)
+    failFuncWithInvalidInput("9a9-15", "Interval string does not match year-month format",
+      fromYearMonthString)
   }
 
   test("from day-time string - legacy") {
@@ -179,29 +155,10 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
           12 * MICROS_PER_MINUTE + fromMillis(888)))
       assert(fromDayTimeString("-3 0:0:0") === new CalendarInterval(0, -3, 0L))
 
-      try {
-        fromDayTimeString("5 30:12:20")
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains("hour 30 outside range"))
-      }
-
-      try {
-        fromDayTimeString("5 30-12")
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains("must match day-time format"))
-      }
-
-      try {
-        fromDayTimeString("5 1:12:20", HOUR, MICROSECOND)
-        fail("Expected to throw an exception for the invalid convention type")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains("Cannot support (interval"))
-      }
+      failFuncWithInvalidInput("5 30:12:20", "hour 30 outside range", fromDayTimeString)
+      failFuncWithInvalidInput("5 30-12", "must match day-time format", fromDayTimeString)
+      failFuncWithInvalidInput("5 1:12:20", "Cannot support (interval",
+        s => fromDayTimeString(s, HOUR, MICROSECOND))
     }
   }
 
@@ -215,13 +172,10 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     assert(duration("1 microsecond", TimeUnit.MICROSECONDS, 30) === 1)
     assert(duration("1 month -30 days", TimeUnit.DAYS, 31) === 1)
 
-    try {
+    val e = intercept[ArithmeticException] {
       duration(Integer.MAX_VALUE + " month", TimeUnit.SECONDS, 31)
-      fail("Expected to throw an exception for the invalid input")
-    } catch {
-      case e: ArithmeticException =>
-        assert(e.getMessage.contains("overflow"))
     }
+    assert(e.getMessage.contains("overflow"))
   }
 
   test("negative interval") {
@@ -283,12 +237,9 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
 
     val interval = new CalendarInterval(2, 0, 0)
     assert(multiply(interval, Integer.MAX_VALUE) === new CalendarInterval(Int.MaxValue, 0, 0))
-    try {
-      multiplyExact(interval, Integer.MAX_VALUE)
-      fail("Expected to throw an exception on months overflow")
-    } catch {
-      case e: ArithmeticException => assert(e.getMessage.contains("overflow"))
-    }
+
+    val e = intercept[ArithmeticException](multiplyExact(interval, Integer.MAX_VALUE))
+    assert(e.getMessage.contains("overflow"))
   }
 
   test("divide by num") {
@@ -307,21 +258,13 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     var interval = new CalendarInterval(Int.MaxValue, Int.MaxValue, 0)
     assert(divide(interval, 0.9) === new CalendarInterval(Int.MaxValue, Int.MaxValue,
       ((Int.MaxValue / 9.0) * MICROS_PER_DAY).round))
-    try {
-      divideExact(interval, 0.9)
-      fail("Expected to throw an exception on integer overflow")
-    } catch {
-      case e: ArithmeticException => assert(e.getMessage.contains("integer overflow"))
-    }
+    val e1 = intercept[ArithmeticException](divideExact(interval, 0.9))
+    assert(e1.getMessage.contains("integer overflow"))
 
     interval = new CalendarInterval(123, 456, 789)
     assert(divide(interval, 0) === null)
-    try {
-      divideExact(interval, 0)
-      fail("Expected to throw an exception on divide by zero")
-    } catch {
-      case e: ArithmeticException => assert(e.getMessage.contains("divide by zero"))
-    }
+    val e2 = intercept[ArithmeticException](divideExact(interval, 0))
+    assert(e2.getMessage.contains("divide by zero"))
   }
 
   test("from day-time string") {
@@ -331,18 +274,8 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
         assert(fromDayTimeString(input, from, to) === safeStringToInterval(expectedUtf8))
       }
     }
-    def checkFail(
-        input: String,
-        from: IntervalUnit,
-        to: IntervalUnit,
-        errMsg: String): Unit = {
-      try {
-        fromDayTimeString(input, from, to)
-        fail("Expected to throw an exception for the invalid input")
-      } catch {
-        case e: IllegalArgumentException =>
-          assert(e.getMessage.contains(errMsg))
-      }
+    def checkFail(input: String, from: IntervalUnit, to: IntervalUnit, errMsg: String): Unit = {
+      failFuncWithInvalidInput(input, errMsg, s => fromDayTimeString(s, from, to))
     }
 
     check("12:40", HOUR, MINUTE, "12 hours 40 minutes")

From 3b73729dcc141c2fac735d271ba56e50a3f3536e Mon Sep 17 00:00:00 2001
From: Tigran Saluev <tigran@saluev.com>
Date: Thu, 27 Feb 2020 11:05:53 -0600
Subject: [PATCH 0180/1280] [MINOR][ML] Fix confusing error message in
 VectorAssembler

### What changes were proposed in this pull request?

When VectorAssembler encounters a NULL with handleInvalid="error", it throws an exception. This exception, though, has a typo making it confusing. Yet apparently, this same exception for NaN values is fine. Fixed it to look like the right one.

### Why are the changes needed?

Encountering this error with such message was very confusing! I hope to save time of fellow engineers by improving it.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

It's just an error message...

Closes #27709 from Saluev/patch-1.

Authored-by: Tigran Saluev <tigran@saluev.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 6f4a2e4c99dadf111e43c6e5b4d7ee5e4d4bd8f6)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../scala/org/apache/spark/ml/feature/VectorAssembler.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
index 51acbf3806b9d..30700122665de 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
@@ -288,7 +288,7 @@ object VectorAssembler extends DefaultParamsReadable[VectorAssembler] {
           featureIndex += length
         } else {
           throw new SparkException(
-            s"""Encountered null while assembling a row with handleInvalid = "keep". Consider
+            s"""Encountered null while assembling a row with handleInvalid = "error". Consider
                |removing nulls from dataset or using handleInvalid = "keep" or "skip"."""
               .stripMargin)
         }

From c6f718b5f41fe76c9c6eedcf2e9684d4d291cb4d Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 27 Feb 2020 17:05:56 -0800
Subject: [PATCH 0181/1280] [SPARK-30968][BUILD] Upgrade aws-java-sdk-sts to
 1.11.655

### What changes were proposed in this pull request?

This PR aims to upgrade `aws-java-sdk-sts` to `1.11.655`.

### Why are the changes needed?

[SPARK-29677](https://github.com/apache/spark/pull/26333) upgrades AWS Kinesis Client to 1.12.0 for Apache Spark 2.4.5 and 3.0.0.

Since AWS Kinesis Client 1.12.0 is using AWS SDK 1.11.665, `aws-java-sdk-sts` should be consistent with Kinesis client dependency.
- https://github.com/awslabs/amazon-kinesis-client/releases/tag/v1.12.0

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins.

Closes #27720 from dongjoon-hyun/SPARK-30968.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3995728c3ce9d85b0436c0220f957b9d9133d64a)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index b3750e4c3f442..9d36faf8ee319 100644
--- a/pom.xml
+++ b/pom.xml
@@ -149,7 +149,7 @@
     <avro.mapred.classifier>hadoop2</avro.mapred.classifier>
     <aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
     <!-- Should be consistent with Kinesis client dependency -->
-    <aws.java.sdk.version>1.11.271</aws.java.sdk.version>
+    <aws.java.sdk.version>1.11.655</aws.java.sdk.version>
     <!-- the producer is used in tests -->
     <aws.kinesis.producer.version>0.12.8</aws.kinesis.producer.version>
     <!--  org.apache.httpcomponents/httpclient-->

From fc13b27374c2ebe2c13a1def2368933578d13801 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Fri, 28 Feb 2020 12:29:46 +0900
Subject: [PATCH 0182/1280] [SPARK-30955][SQL] Exclude Generate output when
 aliasing in nested column pruning

### What changes were proposed in this pull request?

When aliasing in nested column pruning in Project on top of Generate, we should exclude Generate outputs.

### Why are the changes needed?

Right now we would prune nested columns in Project on top of Generate. It is possible that referred nested columns are from Generate's outputs, not from its child. To address that case, we should exclude Generate outputs when aliasing in nested column pruning.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Unit test.

Closes #27702 from viirya/fix-nested-pruning.

Lead-authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Co-authored-by: Liang-Chi Hsieh <liangchi@uber.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit ba032acf9534ea7da75429ebcd4c4912d90cd26c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../optimizer/NestedColumnAliasing.scala      |  8 +++-
 .../sql/catalyst/optimizer/Optimizer.scala    |  3 +-
 .../org/apache/spark/sql/SQLQuerySuite.scala  | 43 ++++++++++++++++---
 3 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index 46e07f3843925..9814bc247fc87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -104,10 +104,13 @@ object NestedColumnAliasing {
   /**
    * Return two maps in order to replace nested fields to aliases.
    *
+   * If `exclusiveAttrs` is given, any nested field accessors of these attributes
+   * won't be considered in nested fields aliasing.
+   *
    * 1. ExtractValue -> Alias: A new alias is created for each nested field.
    * 2. ExprId -> Seq[Alias]: A reference attribute has multiple aliases pointing it.
    */
-  def getAliasSubMap(exprList: Seq[Expression])
+  def getAliasSubMap(exprList: Seq[Expression], exclusiveAttrs: Seq[Attribute] = Seq.empty)
     : Option[(Map[ExtractValue, Alias], Map[ExprId, Seq[Alias]])] = {
     val (nestedFieldReferences, otherRootReferences) =
       exprList.flatMap(collectRootReferenceAndExtractValue).partition {
@@ -115,8 +118,9 @@ object NestedColumnAliasing {
         case _ => false
       }
 
+    val exclusiveAttrSet = AttributeSet(exclusiveAttrs ++ otherRootReferences)
     val aliasSub = nestedFieldReferences.asInstanceOf[Seq[ExtractValue]]
-      .filter(!_.references.subsetOf(AttributeSet(otherRootReferences)))
+      .filter(!_.references.subsetOf(exclusiveAttrSet))
       .groupBy(_.references.head)
       .flatMap { case (attr, nestedFields: Seq[ExtractValue]) =>
         // Each expression can contain multiple nested fields.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 08acac18f48bb..30ad6bfe3b715 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -612,7 +612,8 @@ object ColumnPruning extends Rule[LogicalPlan] {
     // prune unrequired nested fields
     case p @ Project(projectList, g: Generate) if SQLConf.get.nestedPruningOnExpressions &&
         NestedColumnAliasing.canPruneGenerator(g.generator) =>
-      NestedColumnAliasing.getAliasSubMap(projectList ++ g.generator.children).map {
+      val exprsToPrune = projectList ++ g.generator.children
+      NestedColumnAliasing.getAliasSubMap(exprsToPrune, g.qualifiedGeneratorOutput).map {
         case (nestedFieldToAlias, attrToAliases) =>
           val newGenerator = g.generator.transform {
             case f: ExtractValue if nestedFieldToAlias.contains(f) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 78ceeaa09c125..383861a448683 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -28,7 +28,8 @@ import org.apache.spark.{AccumulatorSuite, SparkException}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
 import org.apache.spark.sql.catalyst.expressions.GenericRow
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
-import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
+import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
+import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.execution.HiveResult.hiveResultString
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
@@ -3413,15 +3414,43 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("SPARK-30870: Column pruning shouldn't alias a nested column if it means the whole " +
-    "structure") {
-    val df = sql(
+  test("SPARK-30870: Column pruning shouldn't alias a nested column for the whole structure") {
+    withTable("t") {
+      val df = sql(
+        """
+          |SELECT value
+          |FROM VALUES array(named_struct('field', named_struct('a', 1, 'b', 2))) AS (value)
+        """.stripMargin)
+      df.write.format("parquet").saveAsTable("t")
+
+      val df2 = spark.table("t")
+        .limit(100)
+        .select(size(col("value.field")))
+      val projects = df2.queryExecution.optimizedPlan.collect {
+        case p: Project => p
+      }
+      assert(projects.length == 1)
+      val aliases = NestedColumnAliasingSuite.collectGeneratedAliases(projects(0))
+      assert(aliases.length == 0)
+    }
+  }
+
+  test("SPARK-30955: Exclude Generate output when aliasing in nested column pruning") {
+    val df1 = sql(
+      """
+        |SELECT explodedvalue.*
+        |FROM VALUES array(named_struct('nested', named_struct('a', 1, 'b', 2))) AS (value)
+        |LATERAL VIEW explode(value) AS explodedvalue
+      """.stripMargin)
+    checkAnswer(df1, Row(Row(1, 2)) :: Nil)
+
+    val df2 = sql(
       """
-        |SELECT explodedvalue.field
-        |FROM VALUES array(named_struct('field', named_struct('a', 1, 'b', 2))) AS (value)
+        |SELECT explodedvalue.nested.a
+        |FROM VALUES array(named_struct('nested', named_struct('a', 1, 'b', 2))) AS (value)
         |LATERAL VIEW explode(value) AS explodedvalue
       """.stripMargin)
-    checkAnswer(df, Row(Row(1, 2)) :: Nil)
+    checkAnswer(df2, Row(1) :: Nil)
   }
 }
 

From fa9ca56ef03fee37f90edd4aae18728a4b58aea7 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 28 Feb 2020 11:50:20 +0800
Subject: [PATCH 0183/1280] [SPARK-30972][SQL] PruneHiveTablePartitions should
 be executed as earlyScanPushDownRules

### What changes were proposed in this pull request?

Make rule `PruneHiveTablePartitions` to execute as `earlyScanPushDownRules`.

### Why are the changes needed?

Similar to rule `PruneFileSourcePartitions`, `PruneHiveTablePartitions` should also be executed as earlyScanPushDownRules to eliminate the impact on statistic computation later.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #27723 from Ngone51/early_hive_prune.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a1d2ce90b0954a6ff2da630893d48ba0e7594b08)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/OptimizeMetadataOnlyQuery.scala  | 12 ++++++++----
 .../sql/internal/BaseSessionStateBuilder.scala     | 11 +++++++++++
 .../spark/sql/hive/HiveSessionStateBuilder.scala   | 14 ++------------
 3 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 45e5f415e8da1..47284f3448b7b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -133,10 +133,14 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
               CaseInsensitiveMap(relation.tableMeta.storage.properties)
             val timeZoneId = caseInsensitiveProperties.get(DateTimeUtils.TIMEZONE_OPTION)
               .getOrElse(SQLConf.get.sessionLocalTimeZone)
-            val partitions = if (partFilters.nonEmpty) {
-              catalog.listPartitionsByFilter(relation.tableMeta.identifier, normalizedFilters)
-            } else {
-              catalog.listPartitions(relation.tableMeta.identifier)
+            val partitions = relation.prunedPartitions match {
+              // for the case where partitions have already been pruned by PruneHiveTablePartitions
+              case Some(parts) => parts
+              case None => if (partFilters.nonEmpty) {
+                catalog.listPartitionsByFilter(relation.tableMeta.identifier, normalizedFilters)
+              } else {
+                catalog.listPartitions(relation.tableMeta.identifier)
+              }
             }
 
             val partitionData = partitions.map { p =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 20e1b5694e952..9556d4d1ecabf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -232,6 +232,9 @@ abstract class BaseSessionStateBuilder(
    */
   protected def optimizer: Optimizer = {
     new SparkOptimizer(catalogManager, catalog, experimentalMethods) {
+      override def earlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
+        super.earlyScanPushDownRules ++ customEarlyScanPushDownRules
+
       override def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] =
         super.extendedOperatorOptimizationRules ++ customOperatorOptimizationRules
     }
@@ -247,6 +250,14 @@ abstract class BaseSessionStateBuilder(
     extensions.buildOptimizerRules(session)
   }
 
+  /**
+   * Custom early scan push down rules to add to the Optimizer. Prefer overriding this instead
+   * of creating your own Optimizer.
+   *
+   * Note that this may NOT depend on the `optimizer` function.
+   */
+  protected def customEarlyScanPushDownRules: Seq[Rule[LogicalPlan]] = Nil
+
   /**
    * Planner that converts optimized logical plans to physical plans.
    *
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index 7ec52ba31c029..64726755237a6 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -98,18 +98,8 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
         customCheckRules
   }
 
-  /**
-   * Logical query plan optimizer that takes into account Hive.
-   */
-  override protected def optimizer: Optimizer = {
-    new SparkOptimizer(catalogManager, catalog, experimentalMethods) {
-      override def postHocOptimizationBatches: Seq[Batch] = super.postHocOptimizationBatches ++
-        Seq(Batch("Prune Hive Table Partitions", Once, new PruneHiveTablePartitions(session)))
-
-      override def extendedOperatorOptimizationRules: Seq[Rule[LogicalPlan]] =
-        super.extendedOperatorOptimizationRules ++ customOperatorOptimizationRules
-    }
-  }
+  override def customEarlyScanPushDownRules: Seq[Rule[LogicalPlan]] =
+    Seq(new PruneHiveTablePartitions(session))
 
   /**
    * Planner that takes into account Hive-specific strategies.

From dd6a91b5a7c1c260f821372995789f7b25bbb30e Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 28 Feb 2020 15:14:23 +0900
Subject: [PATCH 0184/1280] [SPARK-30902][SQL] Default table provider should be
 decided by catalog implementations

### What changes were proposed in this pull request?

When `CREATE TABLE` SQL statement does not specify the provider, leave it to the catalog implementations to decide.

### Why are the changes needed?

It's super weird if we set the default provider to parquet when creating a table in a JDBC catalog.

### Does this PR introduce any user-facing change?

Yes, v2 catalog will not see a "provider" property in table properties if it's not specified in `CREATE TABLE` SQL statement. V2 catalog is new in 3.0.

### How was this patch tested?

new tests

Closes #27650 from cloud-fan/create_table.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f21894e5faacb13d8019df1e8fcca4253f68d0ab)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../catalyst/analysis/ResolveCatalogs.scala   |  4 +-
 .../sql/catalyst/parser/AstBuilder.scala      |  4 +-
 .../catalyst/plans/logical/statements.scala   |  4 +-
 .../sql/connector/catalog/CatalogV2Util.scala |  7 +-
 .../sql/catalyst/parser/DDLParserSuite.scala  | 64 +++++++++----------
 .../analysis/ResolveSessionCatalog.scala      | 18 +++---
 .../sql/connector/DataSourceV2SQLSuite.scala  | 34 ++++++++++
 .../execution/command/DDLParserSuite.scala    |  4 +-
 8 files changed, 84 insertions(+), 55 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 88a3c0a73a10b..78a31718ccf36 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -153,7 +153,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         c.tableSchema,
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
-        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+        convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
         orCreate = c.orCreate)
 
     case c @ ReplaceTableAsSelectStatement(
@@ -164,7 +164,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
-        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+        convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
         writeOptions = c.options,
         orCreate = c.orCreate)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index fe724460b8073..ce3383d8e17cb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2763,9 +2763,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       operationNotAllowed("CREATE EXTERNAL TABLE ...", ctx)
     }
     val schema = Option(ctx.colTypeList()).map(createSchema)
-    val defaultProvider = conf.defaultDataSourceName
-    val provider =
-      Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse(defaultProvider)
+    val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
     val (partitioning, bucketSpec, properties, options, location, comment) =
       visitCreateTableClauses(ctx.createTableClauses())
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index 1e6b67bf78b70..89e1539e07260 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -64,7 +64,7 @@ case class CreateTableStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
@@ -79,7 +79,7 @@ case class CreateTableAsSelectStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index 0fabe4df6c9a4..ff63201b02b64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -298,10 +298,9 @@ private[sql] object CatalogV2Util {
       options: Map[String, String],
       location: Option[String],
       comment: Option[String],
-      provider: String): Map[String, String] = {
-    properties ++
-      options ++
-      Map(TableCatalog.PROP_PROVIDER -> provider) ++
+      provider: Option[String]): Map[String, String] = {
+    properties ++ options ++
+      provider.map(TableCatalog.PROP_PROVIDER -> _) ++
       comment.map(TableCatalog.PROP_COMMENT -> _) ++
       location.map(TableCatalog.PROP_LOCATION -> _)
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 4a636bd57f42f..967d0bdee4406 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -49,26 +49,6 @@ class DDLParserSuite extends AnalysisTest {
     comparePlans(parsePlan(sql), expected, checkAnalysis = false)
   }
 
-  test("SPARK-30098: create table without provider should " +
-    "use default data source under non-legacy mode") {
-    val createSql = "CREATE TABLE my_tab(a INT COMMENT 'test', b STRING)"
-    val defaultProvider = conf.defaultDataSourceName
-    val expectedPlan = CreateTableStatement(
-      Seq("my_tab"),
-      new StructType()
-        .add("a", IntegerType, nullable = true, "test")
-        .add("b", StringType),
-      Seq.empty[Transform],
-      None,
-      Map.empty[String, String],
-      defaultProvider,
-      Map.empty[String, String],
-      None,
-      None,
-      false)
-    parseCompare(createSql, expectedPlan)
-  }
-
   test("create/replace table using - schema") {
     val createSql = "CREATE TABLE my_tab(a INT COMMENT 'test', b STRING NOT NULL) USING parquet"
     val replaceSql = "REPLACE TABLE my_tab(a INT COMMENT 'test', b STRING NOT NULL) USING parquet"
@@ -80,7 +60,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -103,7 +83,7 @@ class DDLParserSuite extends AnalysisTest {
         Seq.empty[Transform],
         None,
         Map.empty[String, String],
-        "parquet",
+        Some("parquet"),
         Map.empty[String, String],
         None,
         None),
@@ -123,7 +103,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq(IdentityTransform(FieldReference("a"))),
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -177,7 +157,7 @@ class DDLParserSuite extends AnalysisTest {
           LiteralValue(34, IntegerType)))),
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -199,7 +179,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       Some(BucketSpec(5, Seq("a"), Seq("b"))),
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -217,7 +197,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       Some("abc"))
@@ -237,7 +217,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map("test" -> "test"),
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -255,7 +235,7 @@ class DDLParserSuite extends AnalysisTest {
         Seq.empty[Transform],
         None,
         Map.empty[String, String],
-        "parquet",
+        Some("parquet"),
         Map.empty[String, String],
         Some("/tmp/file"),
         None)
@@ -273,7 +253,7 @@ class DDLParserSuite extends AnalysisTest {
       Seq.empty[Transform],
       None,
       Map.empty[String, String],
-      "parquet",
+      Some("parquet"),
       Map.empty[String, String],
       None,
       None)
@@ -334,7 +314,7 @@ class DDLParserSuite extends AnalysisTest {
           Seq.empty[Transform],
           Option.empty[BucketSpec],
           Map.empty[String, String],
-          "json",
+          Some("json"),
           Map("a" -> "1", "b" -> "0.1", "c" -> "true"),
           None,
           None),
@@ -389,7 +369,7 @@ class DDLParserSuite extends AnalysisTest {
         Seq.empty[Transform],
         None,
         Map("p1" -> "v1", "p2" -> "v2"),
-        "parquet",
+        Some("parquet"),
         Map.empty[String, String],
         Some("/user/external/page_view"),
         Some("This is the staging page view table"))
@@ -2059,7 +2039,7 @@ class DDLParserSuite extends AnalysisTest {
       partitioning: Seq[Transform],
       bucketSpec: Option[BucketSpec],
       properties: Map[String, String],
-      provider: String,
+      provider: Option[String],
       options: Map[String, String],
       location: Option[String],
       comment: Option[String])
@@ -2085,7 +2065,7 @@ class DDLParserSuite extends AnalysisTest {
             replace.partitioning,
             replace.bucketSpec,
             replace.properties,
-            replace.provider,
+            Some(replace.provider),
             replace.options,
             replace.location,
             replace.comment)
@@ -2107,7 +2087,7 @@ class DDLParserSuite extends AnalysisTest {
             rtas.partitioning,
             rtas.bucketSpec,
             rtas.properties,
-            rtas.provider,
+            Some(rtas.provider),
             rtas.options,
             rtas.location,
             rtas.comment)
@@ -2135,4 +2115,20 @@ class DDLParserSuite extends AnalysisTest {
       parsePlan("COMMENT ON TABLE a.b.c IS 'xYz'"),
       CommentOnTable(UnresolvedTable(Seq("a", "b", "c")), "xYz"))
   }
+
+  test("create table - without using") {
+    val sql = "CREATE TABLE 1m.2g(a INT)"
+    val expectedTableSpec = TableSpec(
+      Seq("1m", "2g"),
+      Some(new StructType().add("a", IntegerType)),
+      Seq.empty[Transform],
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None)
+
+    testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 9941f5666b8e2..baaa24f52ced9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -235,9 +235,10 @@ class ResolveSessionCatalog(
     // session catalog and the table provider is not v2.
     case c @ CreateTableStatement(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, c.tableSchema,
-          c.partitioning, c.bucketSpec, c.properties, c.provider, c.options, c.location,
+          c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
           c.comment, c.ifNotExists)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, None)
@@ -248,15 +249,16 @@ class ResolveSessionCatalog(
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
           ignoreIfExists = c.ifNotExists)
       }
 
     case c @ CreateTableAsSelectStatement(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, new StructType,
-          c.partitioning, c.bucketSpec, c.properties, c.provider, c.options, c.location,
+          c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
           c.comment, c.ifNotExists)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, Some(c.asSelect))
@@ -267,7 +269,7 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
           writeOptions = c.options,
           ignoreIfExists = c.ifNotExists)
       }
@@ -289,7 +291,7 @@ class ResolveSessionCatalog(
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
           orCreate = c.orCreate)
       }
 
@@ -304,7 +306,7 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
           writeOptions = c.options,
           orCreate = c.orCreate)
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 4ff20939b3dfd..c074b3352e015 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -256,6 +256,23 @@ class DataSourceV2SQLSuite
     checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), Seq.empty)
   }
 
+  test("CreateTable: without USING clause") {
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+    val testCatalog = catalog("testcat").asTableCatalog
+
+    sql("CREATE TABLE testcat.t1 (id int)")
+    val t1 = testCatalog.loadTable(Identifier.of(Array(), "t1"))
+    // Spark shouldn't set the default provider for catalog plugins.
+    assert(!t1.properties.containsKey(TableCatalog.PROP_PROVIDER))
+
+    sql("CREATE TABLE t2 (id int)")
+    val t2 = spark.sessionState.catalogManager.v2SessionCatalog.asTableCatalog
+      .loadTable(Identifier.of(Array("default"), "t2")).asInstanceOf[V1Table]
+    // Spark should set the default provider as DEFAULT_DATA_SOURCE_NAME for the session catalog.
+    assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
+  }
+
   test("CreateTable/RepalceTable: invalid schema if has interval type") {
     Seq("CREATE", "REPLACE").foreach { action =>
       val e1 = intercept[AnalysisException](
@@ -595,6 +612,23 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("CreateTableAsSelect: without USING clause") {
+    // unset this config to use the default v2 session catalog.
+    spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+    val testCatalog = catalog("testcat").asTableCatalog
+
+    sql("CREATE TABLE testcat.t1 AS SELECT 1 i")
+    val t1 = testCatalog.loadTable(Identifier.of(Array(), "t1"))
+    // Spark shouldn't set the default provider for catalog plugins.
+    assert(!t1.properties.containsKey(TableCatalog.PROP_PROVIDER))
+
+    sql("CREATE TABLE t2 AS SELECT 1 i")
+    val t2 = spark.sessionState.catalogManager.v2SessionCatalog.asTableCatalog
+      .loadTable(Identifier.of(Array("default"), "t2")).asInstanceOf[V1Table]
+    // Spark should set the default provider as DEFAULT_DATA_SOURCE_NAME for the session catalog.
+    assert(t2.v1Table.provider == Some(conf.defaultDataSourceName))
+  }
+
   test("DropTable: basic") {
     val tableName = "testcat.ns1.ns2.tbl"
     val ident = Identifier.of(Array("ns1", "ns2"), "tbl")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
index 81965e4c6c353..bacd64efedc83 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
@@ -492,7 +492,7 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
     assert(statement.partitioning.isEmpty)
     assert(statement.bucketSpec.isEmpty)
     assert(statement.properties.isEmpty)
-    assert(statement.provider == conf.defaultDataSourceName)
+    assert(statement.provider.isEmpty)
     assert(statement.options.isEmpty)
     assert(statement.location.isEmpty)
     assert(statement.comment.isEmpty)
@@ -662,7 +662,7 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
       assert(state.partitioning.isEmpty)
       assert(state.bucketSpec.isEmpty)
       assert(state.properties.isEmpty)
-      assert(state.provider == conf.defaultDataSourceName)
+      assert(state.provider.isEmpty)
       assert(state.options.isEmpty)
       assert(state.location.isEmpty)
       assert(state.comment.isEmpty)

From 790c14a0a96aae3be3485ae3e25b4087535a7f25 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 28 Feb 2020 15:18:25 +0900
Subject: [PATCH 0185/1280] [MINOR][SQL] Remove unnecessary MiMa excludes

### What changes were proposed in this pull request?
I found a few unnecessary MiMa excludes when auditing SQL binary incompatible changes.

### Why are the changes needed?
These MiMa excludes are not required any more, so remove.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually tested

Closes #27729 from huaxingao/mima.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c0d4cc3657b9dcca1d57d8776ec36b9efd84a06f)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 project/MimaExcludes.scala | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index d69caa9e44649..23f33a6a7824d 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -212,14 +212,6 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setMinInstancesPerNode"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setNumTrees"),
 
-    // [SPARK-26124] Update plugins, including MiMa
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsPushDownRequiredColumns.build"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportStatistics.fullSchema"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportStatistics.planInputPartitions"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportPartitioning.fullSchema"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsReportPartitioning.planInputPartitions"),
-    ProblemFilters.exclude[InheritedNewAbstractMethodProblem]("org.apache.spark.sql.sources.v2.reader.SupportsPushDownFilters.build"),
-
     // [SPARK-26090] Resolve most miscellaneous deprecation and build warnings for Spark 3
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.stat.test.BinarySampleBeanInfo"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.regression.LabeledPointBeanInfo"),

From b8e9cdcd14dcda68dde0c646f58d10880332691e Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 28 Feb 2020 00:01:20 -0800
Subject: [PATCH 0186/1280] [SPARK-30970][K8S][CORE] Fix NPE while resolving
 k8s master url

### What changes were proposed in this pull request?

```
bin/spark-sql --master  k8s:///https://kubernetes.docker.internal:6443 --conf spark.kubernetes.container.image=yaooqinn/spark:v2.4.4
Exception in thread "main" java.lang.NullPointerException
	at org.apache.spark.util.Utils$.checkAndGetK8sMasterUrl(Utils.scala:2739)
	at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:261)
	at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:774)
	at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
	at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
	at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
	at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
```
Althrough `k8s:///https://kubernetes.docker.internal:6443` is a wrong master url but should not throw npe
The `case null` will never be touched.
https://github.com/apache/spark/blob/3f4060c340d6bac412e8819c4388ccba226efcf3/core/src/main/scala/org/apache/spark/util/Utils.scala#L2772-L2776

### Why are the changes needed?

bug fix

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

add ut case

Closes #27721 from yaooqinn/SPARK-30970.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 1383bd459a834fb075c5b570338fab0886110df9)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../main/scala/org/apache/spark/util/Utils.scala  | 15 ++++++---------
 .../scala/org/apache/spark/util/UtilsSuite.scala  |  4 ++++
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 297cc5e4cb100..dde43232f0d65 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2772,19 +2772,16 @@ private[spark] object Utils extends Logging {
     }
 
     val masterScheme = new URI(masterWithoutK8sPrefix).getScheme
-    val resolvedURL = masterScheme.toLowerCase(Locale.ROOT) match {
-      case "https" =>
+
+    val resolvedURL = Option(masterScheme).map(_.toLowerCase(Locale.ROOT)) match {
+      case Some("https") =>
         masterWithoutK8sPrefix
-      case "http" =>
+      case Some("http") =>
         logWarning("Kubernetes master URL uses HTTP instead of HTTPS.")
         masterWithoutK8sPrefix
-      case null =>
-        val resolvedURL = s"https://$masterWithoutK8sPrefix"
-        logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " +
-          s"URL is $resolvedURL.")
-        resolvedURL
       case _ =>
-        throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme)
+        throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme
+          + " found in URL: " + masterWithoutK8sPrefix)
     }
 
     s"k8s://$resolvedURL"
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 8f8902e497d49..f5e438b0f1a52 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -1243,6 +1243,10 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
     intercept[IllegalArgumentException] {
       Utils.checkAndGetK8sMasterUrl("k8s://foo://host:port")
     }
+
+    intercept[IllegalArgumentException] {
+      Utils.checkAndGetK8sMasterUrl("k8s:///https://host:port")
+    }
   }
 
   test("stringHalfWidth") {

From 7ee2dd693b3a7fe3f854eff090ad181360785d39 Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Fri, 28 Feb 2020 22:57:50 +0800
Subject: [PATCH 0187/1280] [SPARK-27619][SQL][FOLLOWUP] Rename
 'spark.sql.legacy.useHashOnMapType' to 'spark.sql.legacy.allowHashOnMapType'

### What changes were proposed in this pull request?
Renamed configuration from `spark.sql.legacy.useHashOnMapType` to `spark.sql.legacy.allowHashOnMapType`.

### Why are the changes needed?
Better readability of configuration.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing UTs.

Closes #27719 from iRakson/SPARK-27619_FOLLOWUP.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a40a2f83383337ee1e4628b2a0b64d7c67ef0f89)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                                 | 2 +-
 .../org/apache/spark/sql/catalyst/expressions/hash.scala    | 6 +++---
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 2 +-
 .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 5 +++--
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index d241a664357f6..fb8f866c1b5b8 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -224,7 +224,7 @@ license: |
 
   - Since Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, e.g. `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, while casting string to integrals and booleans, it will not trim the whitespaces from both ends, the foregoing results will be `null`, while to datetimes, only the trailing spaces (= ASCII 32) will be removed.
 
-  - Since Spark 3.0, An analysis exception will be thrown when hash expressions are applied on elements of MapType. To restore the behavior before Spark 3.0, set `spark.sql.legacy.useHashOnMapType` to true.
+  - Since Spark 3.0, an analysis exception will be thrown when hash expressions are applied on elements of MapType. To restore the behavior before Spark 3.0, set `spark.sql.legacy.allowHashOnMapType` to `true`.
     
   - Since Spark 3.0, numbers written in scientific notation(e.g. `1E2`) would be parsed as Double. In Spark version 2.4 and earlier, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
index 5be7f54107759..4c8c58ae232f4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala
@@ -256,11 +256,11 @@ abstract class HashExpression[E] extends Expression {
       TypeCheckResult.TypeCheckFailure(
         s"input to function $prettyName requires at least one argument")
     } else if (children.exists(child => hasMapType(child.dataType)) &&
-      !SQLConf.get.getConf(SQLConf.LEGACY_USE_HASH_ON_MAPTYPE)) {
+        !SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE)) {
       TypeCheckResult.TypeCheckFailure(
         s"input to function $prettyName cannot contain elements of MapType. In Spark, same maps " +
-          "may have different hashcode, thus hash expressions are prohibited on MapType " +
-          s"elements. To restore previous behavior set ${SQLConf.LEGACY_USE_HASH_ON_MAPTYPE.key} " +
+          "may have different hashcode, thus hash expressions are prohibited on MapType elements." +
+          s" To restore previous behavior set ${SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE.key} " +
           "to true.")
     } else {
       TypeCheckResult.TypeCheckSuccess
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index fdd3e9ced8957..30f5fbaac2986 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2200,7 +2200,7 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_USE_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.useHashOnMapType")
+  val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")
     .booleanConf
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 383861a448683..87de8f50568f6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -2130,10 +2130,11 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("SPARK-27619: when spark.sql.legacy.useHashOnMapType is true, hash can be used on Maptype") {
+  test(s"SPARK-27619: When ${SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE.key} is true, hash can be " +
+    "used on Maptype") {
     Seq("hash", "xxhash64").foreach {
       case hashExpression =>
-        withSQLConf(SQLConf.LEGACY_USE_HASH_ON_MAPTYPE.key -> "true") {
+        withSQLConf(SQLConf.LEGACY_ALLOW_HASH_ON_MAPTYPE.key -> "true") {
           val df = spark.createDataset(Map() :: Nil)
           checkAnswer(df.selectExpr(s"$hashExpression(*)"), sql(s"SELECT $hashExpression(map())"))
         }

From 2342e280a57a108ff2327aae7157d85065016244 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 28 Feb 2020 11:22:08 -0800
Subject: [PATCH 0188/1280] [SPARK-28998][SQL][FOLLOW-UP] Remove unnecessary
 MiMa excludes

### What changes were proposed in this pull request?
Remove the cases for ```MissingTypesProblem```, ```InheritedNewAbstractMethodProblem```, ```DirectMissingMethodProblem``` and ```ReversedMissingMethodProblem```.

### Why are the changes needed?
After the changes, we don't have ```org.apache.spark.sql.sources.v2```  any more, so the only problem we can get is ```MissingClassProblem```

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually tested

Closes #27731 from huaxingao/spark-28998-followup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 961c539a676d2646a9315b427ad81852aa81b658)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 project/MimaExcludes.scala | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 23f33a6a7824d..7f6657760d259 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -335,14 +335,6 @@ object MimaExcludes {
     (problem: Problem) => problem match {
       case MissingClassProblem(cls) =>
         !cls.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case MissingTypesProblem(newCls, _) =>
-        !newCls.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case InheritedNewAbstractMethodProblem(cls, _) =>
-        !cls.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case DirectMissingMethodProblem(meth) =>
-        !meth.owner.fullName.startsWith("org.apache.spark.sql.sources.v2")
-      case ReversedMissingMethodProblem(meth) =>
-        !meth.owner.fullName.startsWith("org.apache.spark.sql.sources.v2")
       case _ => true
     },
 

From 00e2bf8a9a96cf421fa5257f72d46334306b92fa Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Fri, 28 Feb 2020 11:43:05 -0800
Subject: [PATCH 0189/1280] [SPARK-30987][CORE] Increase the timeout on
 local-cluster waitUntilExecutorsUp calls

### What changes were proposed in this pull request?

The ResourceDiscoveryPlugin tests intermittently timeout. They are timing out on just bringing up the local-cluster. I am not able to reproduce locally.  I suspect the jenkins boxes are overloaded and taking longer then 10 seconds. There was another jira SPARK-29139 that increased timeout for some other of these as well. So try increasing the timeout to 60 seconds.

Examples of timeouts:
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/119030/testReport/
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/119005/testReport/
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/119029/testReport/

### Why are the changes needed?

tests should no longer intermittently fail.

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

unit tests ran.

Closes #27738 from tgravescs/SPARK-30987.

Authored-by: Thomas Graves <tgraves@nvidia.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit 6c0c41fa0d1e119b16980405b5dc69b953380d7d)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../test/scala/org/apache/spark/DistributedSuite.scala    | 2 +-
 .../spark/internal/plugin/PluginContainerSuite.scala      | 4 ++--
 .../spark/resource/ResourceDiscoveryPluginSuite.scala     | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/DistributedSuite.scala b/core/src/test/scala/org/apache/spark/DistributedSuite.scala
index 3f309819065be..4d157b9607000 100644
--- a/core/src/test/scala/org/apache/spark/DistributedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/DistributedSuite.scala
@@ -174,7 +174,7 @@ class DistributedSuite extends SparkFunSuite with Matchers with LocalSparkContex
 
   private def testCaching(conf: SparkConf, storageLevel: StorageLevel): Unit = {
     sc = new SparkContext(conf.setMaster(clusterUrl).setAppName("test"))
-    TestUtils.waitUntilExecutorsUp(sc, 2, 30000)
+    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
     val data = sc.parallelize(1 to 1000, 10)
     val cachedData = data.persist(storageLevel)
     assert(cachedData.count === 1000)
diff --git a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
index cf2d9293ef822..7888796dd55e6 100644
--- a/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/internal/plugin/PluginContainerSuite.scala
@@ -139,7 +139,7 @@ class PluginContainerSuite extends SparkFunSuite with BeforeAndAfterEach with Lo
       .set(NonLocalModeSparkPlugin.TEST_PATH_CONF, path.getAbsolutePath())
 
     sc = new SparkContext(conf)
-    TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+    TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
     eventually(timeout(10.seconds), interval(100.millis)) {
       val children = path.listFiles()
@@ -169,7 +169,7 @@ class PluginContainerSuite extends SparkFunSuite with BeforeAndAfterEach with Lo
       sc = new SparkContext(conf)
 
       // Ensure all executors has started
-      TestUtils.waitUntilExecutorsUp(sc, 1, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
 
       var children = Array.empty[File]
       eventually(timeout(10.seconds), interval(100.millis)) {
diff --git a/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala b/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
index 7a05daa2ad715..437c903e77d4a 100644
--- a/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
+++ b/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
@@ -56,7 +56,7 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(EXECUTOR_FPGA_ID.amountConf, "1")
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       eventually(timeout(10.seconds), interval(100.millis)) {
         val children = dir.listFiles()
@@ -84,7 +84,7 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       eventually(timeout(10.seconds), interval(100.millis)) {
         val children = dir.listFiles()
@@ -111,7 +111,7 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       eventually(timeout(10.seconds), interval(100.millis)) {
         val children = dir.listFiles()
@@ -137,7 +137,7 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
-      TestUtils.waitUntilExecutorsUp(sc, 2, 10000)
+      TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
 
       assert(sc.resources.size === 1)
       assert(sc.resources.get(GPU).get.addresses === Array("5", "6"))

From 4cac4a50e0bd81edc7a6a18674a64045c7c247bb Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Fri, 28 Feb 2020 18:12:20 -0800
Subject: [PATCH 0190/1280] [SPARK-30977][CORE][3.0] Make ResourceProfile and
 ResourceProfileBuilder private

### What changes were proposed in this pull request?

Make the ResourceProfile and ResourceProfileBuilder apis private since the entire feature didn't make 3.0.

### Why are the changes needed?

to not expose to user to early.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

unit tests

Closes #27737 from tgravescs/SPARK-30977.

Authored-by: Thomas Graves <tgraves@nvidia.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/resource/ResourceProfile.scala    | 5 ++++-
 .../org/apache/spark/resource/ResourceProfileBuilder.scala   | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
index 14019d27fc2e6..f3c39d9107e98 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -34,9 +34,12 @@ import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
  * specify executor and task requirements for an RDD that will get applied during a
  * stage. This allows the user to change the resource requirements between stages.
  * This is meant to be immutable so user can't change it after building.
+ *
+ * This api is currently private until the rest of the pieces are in place and then it
+ * will become public.
  */
 @Evolving
-class ResourceProfile(
+private[spark] class ResourceProfile(
     val executorResources: Map[String, ExecutorResourceRequest],
     val taskResources: Map[String, TaskResourceRequest]) extends Serializable with Logging {
 
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
index 0d55c176eeb65..db1c77d7c4d40 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceProfileBuilder.scala
@@ -29,9 +29,12 @@ import org.apache.spark.annotation.Evolving
  * A ResourceProfile allows the user to specify executor and task requirements for an RDD
  * that will get applied during a stage. This allows the user to change the resource
  * requirements between stages.
+ *
+ * This api is currently private until the rest of the pieces are in place and then it
+ * will become public.
  */
 @Evolving
-class ResourceProfileBuilder() {
+private[spark] class ResourceProfileBuilder() {
 
   private val _taskResources = new ConcurrentHashMap[String, TaskResourceRequest]()
   private val _executorResources = new ConcurrentHashMap[String, ExecutorResourceRequest]()

From 8a8d1fbb10af6da481f26831cd519ef46ccbce6c Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Sun, 1 Mar 2020 10:55:41 +0900
Subject: [PATCH 0191/1280] [SPARK-30234][SQL][FOLLOWUP] Rename
 `spark.sql.legacy.addDirectory.recursive.enabled` to
 `spark.sql.legacy.addSingleFileInAddFile`

### What changes were proposed in this pull request?
Rename `spark.sql.legacy.addDirectory.recursive.enabled` to `spark.sql.legacy.addSingleFileInAddFile`

### Why are the changes needed?
To follow the naming convention

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing UTs.

Closes #27725 from iRakson/SPARK-30234_CONFIG.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 92a5ae2ae488b5b0f106411d72e1c812b918ab93)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md                          |  2 +-
 .../org/apache/spark/sql/internal/SQLConf.scala      | 12 ++++++------
 .../spark/sql/execution/command/resources.scala      |  2 +-
 .../spark/sql/execution/command/DDLSuite.scala       |  6 +++---
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index fb8f866c1b5b8..a9a39ceca8777 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -326,7 +326,7 @@ license: |
         </tr>
     </table>
 
-  - Since Spark 3.0, `ADD FILE` can be used to add file directories as well. Earlier only single files can be added using this command. To restore the behaviour of earlier versions, set `spark.sql.legacy.addDirectory.recursive.enabled` to false.
+  - Since Spark 3.0, `ADD FILE` can be used to add file directories as well. Earlier only single files can be added using this command. To restore the behaviour of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
 
   - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 30f5fbaac2986..7718dd2822974 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2152,13 +2152,13 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_ADD_DIRECTORY_USING_RECURSIVE =
-    buildConf("spark.sql.legacy.addDirectory.recursive.enabled")
+  val LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE =
+    buildConf("spark.sql.legacy.addSingleFileInAddFile")
       .internal()
-      .doc("When true, users can add directory by passing path of a directory to ADD FILE " +
-        "command of SQL. If false, then only a single file can be added.")
+      .doc("When true, only a single file can be added using ADD FILE. If false, then users " +
+        "can add directory by passing directory path to ADD FILE.")
       .booleanConf
-      .createWithDefault(true)
+      .createWithDefault(false)
 
   val LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED =
     buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
@@ -2491,7 +2491,7 @@ class SQLConf extends Serializable with Logging {
 
   def datetimeJava8ApiEnabled: Boolean = getConf(DATETIME_JAVA8API_ENABLED)
 
-  def addDirectoryRecursiveEnabled: Boolean = getConf(LEGACY_ADD_DIRECTORY_USING_RECURSIVE)
+  def addSingleFileInAddFile: Boolean = getConf(LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE)
 
   def legacyMsSqlServerNumericMappingEnabled: Boolean =
     getConf(LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
index 1119e5cb1d288..549477dbae6ba 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/resources.scala
@@ -47,7 +47,7 @@ case class AddJarCommand(path: String) extends RunnableCommand {
  */
 case class AddFileCommand(path: String) extends RunnableCommand {
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    val recursive = sparkSession.sessionState.conf.addDirectoryRecursiveEnabled
+    val recursive = !sparkSession.sessionState.conf.addSingleFileInAddFile
     sparkSession.sparkContext.addFile(path, recursive)
     Seq.empty[Row]
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index e4bf721b1d58d..6c824c27e4551 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2982,16 +2982,16 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("Add a directory when spark.sql.legacy.addDirectory.recursive.enabled set to true") {
+  test(s"Add a directory when ${SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key} set to false") {
     val directoryToAdd = Utils.createTempDir("/tmp/spark/addDirectory/")
     val testFile = File.createTempFile("testFile", "1", directoryToAdd)
     spark.sql(s"ADD FILE $directoryToAdd")
     assert(new File(SparkFiles.get(s"${directoryToAdd.getName}/${testFile.getName}")).exists())
   }
 
-  test("Add a directory when spark.sql.legacy.addDirectory.recursive.enabled not set to true") {
+  test(s"Add a directory when ${SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key} set to true") {
     withTempDir { testDir =>
-      withSQLConf(SQLConf.LEGACY_ADD_DIRECTORY_USING_RECURSIVE.key -> "false") {
+      withSQLConf(SQLConf.LEGACY_ADD_SINGLE_FILE_IN_ADD_FILE.key -> "true") {
         val msg = intercept[SparkException] {
           spark.sql(s"ADD FILE $testDir")
         }.getMessage

From aea8749a038441a8d2092d448065b0fabd9a0674 Mon Sep 17 00:00:00 2001
From: Josh Rosen <rosenville@gmail.com>
Date: Mon, 2 Mar 2020 10:19:12 +0900
Subject: [PATCH 0192/1280] [SPARK-29419][SQL] Fix Encoder thread-safety bug in
 createDataset(Seq)

### What changes were proposed in this pull request?

This PR fixes a thread-safety bug in `SparkSession.createDataset(Seq)`: if the caller-supplied `Encoder` is used in multiple threads then createDataset's usage of the encoder may lead to incorrect / corrupt results because the Encoder's internal mutable state will be updated from multiple threads.

Here is an example demonstrating the problem:

```scala
import org.apache.spark.sql._

val enc = implicitly[Encoder[(Int, Int)]]

val datasets = (1 to 100).par.map { _ =>
  val pairs = (1 to 100).map(x => (x, x))
  spark.createDataset(pairs)(enc)
}

datasets.reduce(_ union _).collect().foreach {
  pair => require(pair._1 == pair._2, s"Pair elements are mismatched: $pair")
}
```

Before this PR's change, the above example fails because Spark produces corrupted records where different input records' fields have been co-mingled.

This bug is similar to SPARK-22355 / #19577, a similar problem in `Dataset.collect()`.

The fix implemented here is based on #24735's updated version of the `Datataset.collect()` bugfix: use `.copy()`. For consistency, I used same [code comment](https://github.com/apache/spark/blob/d841b33ba3a9b0504597dbccd4b0d11fa810abf3/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala#L3414) / explanation as that PR.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Tested manually using the example listed above.

Thanks to smcnamara-stripe for identifying this bug.

Closes #26076 from JoshRosen/SPARK-29419.

Authored-by: Josh Rosen <rosenville@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f4499f678dc2e9f72c3ee5d2af083aa6b98f3fc2)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/SparkSession.scala     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 1fb97fb4b4cf1..bca841c48cacd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -459,7 +459,8 @@ class SparkSession private(
    * @since 2.0.0
    */
   def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
-    val enc = encoderFor[T]
+    // `ExpressionEncoder` is not thread-safe, here we create a new encoder.
+    val enc = encoderFor[T].copy()
     val attributes = enc.schema.toAttributes
     val encoded = data.map(d => enc.toRow(d).copy())
     val plan = new LocalRelation(attributes, encoded)

From 88d4be2c0b92bf719f77ea7063e969fe58a6d604 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 2 Mar 2020 10:33:26 +0900
Subject: [PATCH 0193/1280] [SPARK-30995][ML][DOCS] Latex doesn't work
 correctly in FMClassifier/FMRegressor Scala doc

### What changes were proposed in this pull request?
Latex doesn't work correctly

### Why are the changes needed?
Fix the doc to make Latex work

### Does this PR introduce any user-facing change?
Before fix:

![image](https://user-images.githubusercontent.com/13592258/75611743-0fa00a00-5ad2-11ea-9cc0-4b246b25d63d.png)

![image](https://user-images.githubusercontent.com/13592258/75611755-25adca80-5ad2-11ea-884d-b2792b714bd5.png)

After fix:

![image](https://user-images.githubusercontent.com/13592258/75611776-46762000-5ad2-11ea-838d-f7f6f93c8aec.png)

![image](https://user-images.githubusercontent.com/13592258/75611778-51c94b80-5ad2-11ea-85e4-8c7424268f52.png)

### How was this patch tested?
Manually build doc and test

Closes #27748 from huaxingao/fm_doc.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0a5e9a12a7d87ee1938368b971e4e18a3f8e84ef)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/ml/classification/FMClassifier.scala  | 10 +++++++---
 .../org/apache/spark/ml/regression/FMRegressor.scala   | 10 +++++++---
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index d511c1b5dda98..3057d510782c7 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -52,12 +52,16 @@ private[classification] trait FMClassifierParams extends ProbabilisticClassifier
  * FM is able to estimate interactions even in problems with huge sparsity
  * (like advertising and recommendation system).
  * FM formula is:
- * {{{
+ * <blockquote>
+ *   $$
+ *   \begin{align}
  *   y = \sigma\left( w_0 + \sum\limits^n_{i-1} w_i x_i +
  *     \sum\limits^n_{i=1} \sum\limits^n_{j=i+1} \langle v_i, v_j \rangle x_i x_j \right)
- * }}}
+ *   \end{align}
+ *   $$
+ * </blockquote>
  * First two terms denote global bias and linear term (as same as linear regression),
- * and last term denotes pairwise interactions term. {{{v_i}}} describes the i-th variable
+ * and last term denotes pairwise interactions term. v_i describes the i-th variable
  * with k factors.
  *
  * FM classification model uses logistic loss which can be solved by gradient descent method, and
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
index f7810eb17cf59..a61244867f210 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
@@ -277,12 +277,16 @@ private[regression] trait FMRegressorParams extends FactorizationMachinesParams
  * FM is able to estimate interactions even in problems with huge sparsity
  * (like advertising and recommendation system).
  * FM formula is:
- * {{{
+ * <blockquote>
+ *   $$
+ *   \begin{align}
  *   y = w_0 + \sum\limits^n_{i-1} w_i x_i +
  *     \sum\limits^n_{i=1} \sum\limits^n_{j=i+1} \langle v_i, v_j \rangle x_i x_j
- * }}}
+ *   \end{align}
+ *   $$
+ * </blockquote>
  * First two terms denote global bias and linear term (as same as linear regression),
- * and last term denotes pairwise interactions term. {{{v_i}}} describes the i-th variable
+ * and last term denotes pairwise interactions term. v_i describes the i-th variable
  * with k factors.
  *
  * FM regression model uses MSE loss which can be solved by gradient descent method, and

From 3e268763c345076801f1ff6f75bab67ecab87e8f Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Mon, 2 Mar 2020 15:48:48 +0800
Subject: [PATCH 0194/1280] [SPARK-30964][CORE][WEBUI] Accelerate InMemoryStore
 with a new index

### What changes were proposed in this pull request?

Spark uses the class `InMemoryStore` as the KV storage for live UI and history server(by default if no LevelDB file path is provided).
In `InMemoryStore`, all the task data in one application is stored in a hashmap, which key is the task ID and the value is the task data. This fine for getting or deleting with a provided task ID.
However, Spark stage UI always shows all the task data in one stage and the current implementation is to look up all the values in the hashmap. The time complexity is O(numOfTasks).
Also, when there are too many stages (>spark.ui.retainedStages), Spark will linearly try to look up all the task data of the stages to be deleted as well.

This can be very bad for a large application with many stages and tasks. We can improve it by allowing the natural key of an entity to have a real parent index. So that on each lookup with parent node provided, Spark can look up all the natural keys(in our case, the task IDs) first, and then find the data with the natural keys in the hashmap.

### Why are the changes needed?

The in-memory KV store becomes really slow for large applications. We can improve it with a new index. The performance can be 10 times, 100 times, even 1000 times faster.
This is also possible to make the Spark driver more stable for large applications.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Existing unit tests.
Also, I run a benchmark with the following code
```
  val store = new InMemoryStore()
  val numberOfTasksPerStage = 10000
   (0 until 1000).map { sId =>
     (0 until numberOfTasksPerStage).map { taskId =>
       val task = newTaskData(sId * numberOfTasksPerStage + taskId, "SUCCESS", sId)
       store.write(task)
     }
   }
  val appStatusStore = new AppStatusStore(store)
  var start = System.nanoTime()
  appStatusStore.taskSummary(2, attemptId, Array(0, 0.25, 0.5, 0.75, 1))
  println("task summary run time: " + ((System.nanoTime() - start) / 1000000))
  val stageIds = Seq(1, 11, 66, 88)
  val stageKeys = stageIds.map(Array(_, attemptId))
  start = System.nanoTime()
  store.removeAllByIndexValues(classOf[TaskDataWrapper], TaskIndexNames.STAGE,
    stageKeys.asJavaCollection)
   println("clean up tasks run time: " + ((System.nanoTime() - start) / 1000000))
```

Task summary before the changes: 98642ms
Task summary after the changes: 120ms

Task clean up before the changes:  4900ms
Task clean up before the changes: 4ms

It's 800x faster after the changes in the micro-benchmark.

Closes #27716 from gengliangwang/liveUIStore.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6b641430c37e0115bee781fed7360187b988313d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/util/kvstore/InMemoryStore.java     | 116 +++++++++++++++---
 .../apache/spark/util/kvstore/KVTypeInfo.java |   7 +-
 .../spark/util/kvstore/LevelDBTypeInfo.java   |   5 +-
 .../org/apache/spark/status/storeTypes.scala  |   2 +-
 4 files changed, 105 insertions(+), 25 deletions(-)

diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index b33c53871c32f..db08740975176 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -163,6 +163,12 @@ public void clear() {
     }
   }
 
+  /**
+   * An alias class for the type "ConcurrentHashMap<Comparable<Object>, Boolean>", which is used
+   * as a concurrent hashset for storing natural keys and the boolean value doesn't matter.
+   */
+  private static class NaturalKeys extends ConcurrentHashMap<Comparable<Object>, Boolean> {}
+
   private static class InstanceList<T> {
 
     /**
@@ -205,11 +211,19 @@ public void accept(Comparable<Object> key, T value) {
     private final KVTypeInfo ti;
     private final KVTypeInfo.Accessor naturalKey;
     private final ConcurrentMap<Comparable<Object>, T> data;
+    private final String naturalParentIndexName;
+    private final Boolean hasNaturalParentIndex;
+    // A mapping from parent to the natural keys of its children.
+    // For example, a mapping from a stage ID to all the task IDs in the stage.
+    private final ConcurrentMap<Comparable<Object>, NaturalKeys> parentToChildrenMap;
 
     private InstanceList(Class<?> klass) {
       this.ti = new KVTypeInfo(klass);
       this.naturalKey = ti.getAccessor(KVIndex.NATURAL_INDEX_NAME);
       this.data = new ConcurrentHashMap<>();
+      this.naturalParentIndexName = ti.getParentIndexName(KVIndex.NATURAL_INDEX_NAME);
+      this.parentToChildrenMap = new ConcurrentHashMap<>();
+      this.hasNaturalParentIndex = !naturalParentIndexName.isEmpty();
     }
 
     KVTypeInfo.Accessor getIndexAccessor(String indexName) {
@@ -217,11 +231,30 @@ KVTypeInfo.Accessor getIndexAccessor(String indexName) {
     }
 
     int countingRemoveAllByIndexValues(String index, Collection<?> indexValues) {
-      Predicate<? super T> filter = getPredicate(ti.getAccessor(index), indexValues);
-      CountingRemoveIfForEach<T> callback = new CountingRemoveIfForEach<>(data, filter);
+      if (hasNaturalParentIndex && naturalParentIndexName.equals(index)) {
+        // If there is a parent index for the natural index and `index` happens to be it,
+        // Spark can use the `parentToChildrenMap` to get the related natural keys, and then
+        // delete them from `data`.
+        int count = 0;
+        for (Object indexValue : indexValues) {
+          Comparable<Object> parentKey = asKey(indexValue);
+          NaturalKeys children = parentToChildrenMap.getOrDefault(parentKey, new NaturalKeys());
+          for (Comparable<Object> naturalKey : children.keySet()) {
+            data.remove(naturalKey);
+            count ++;
+          }
+          parentToChildrenMap.remove(parentKey);
+        }
+        return count;
+      } else {
+        Predicate<? super T> filter = getPredicate(ti.getAccessor(index), indexValues);
+        CountingRemoveIfForEach<T> callback = new CountingRemoveIfForEach<>(data, filter);
 
-      data.forEach(callback);
-      return callback.count();
+        // Go through all the values in `data` and delete objects that meets the predicate `filter`.
+        // This can be slow when there is a large number of entries in `data`.
+        data.forEach(callback);
+        return callback.count();
+      }
     }
 
     public T get(Object key) {
@@ -230,10 +263,27 @@ public T get(Object key) {
 
     public void put(T value) throws Exception {
       data.put(asKey(naturalKey.get(value)), value);
+      if (hasNaturalParentIndex) {
+        Comparable<Object> parentKey = asKey(getIndexAccessor(naturalParentIndexName).get(value));
+        NaturalKeys children =
+          parentToChildrenMap.computeIfAbsent(parentKey, k -> new NaturalKeys());
+        children.put(asKey(naturalKey.get(value)), true);
+      }
     }
 
     public void delete(Object key) {
       data.remove(asKey(key));
+      if (hasNaturalParentIndex) {
+        for (NaturalKeys v : parentToChildrenMap.values()) {
+          if (v.remove(asKey(key))) {
+            // `v` can be empty after removing the natural key and we can remove it from
+            // `parentToChildrenMap`. However, `parentToChildrenMap` is a ConcurrentMap and such
+            // checking and deleting can be slow.
+            // This method is to delete one object with certain key, let's make it simple here.
+            break;
+          }
+        }
+      }
     }
 
     public int size() {
@@ -241,7 +291,7 @@ public int size() {
     }
 
     public InMemoryView<T> view() {
-      return new InMemoryView<>(data.values(), ti);
+      return new InMemoryView<>(data, ti, naturalParentIndexName, parentToChildrenMap);
     }
 
     private static <T> Predicate<? super T> getPredicate(
@@ -271,22 +321,32 @@ private static Object indexValueForEntity(KVTypeInfo.Accessor getter, Object ent
 
   private static class InMemoryView<T> extends KVStoreView<T> {
     private static final InMemoryView<?> EMPTY_VIEW =
-      new InMemoryView<>(Collections.emptyList(), null);
+      new InMemoryView<>(new ConcurrentHashMap<>(), null, "", new ConcurrentHashMap<>());
 
-    private final Collection<T> elements;
+    private final ConcurrentMap<Comparable<Object>, T> data;
     private final KVTypeInfo ti;
     private final KVTypeInfo.Accessor natural;
-
-    InMemoryView(Collection<T> elements, KVTypeInfo ti) {
-      this.elements = elements;
+    private final ConcurrentMap<Comparable<Object>, NaturalKeys> parentToChildrenMap;
+    private final String naturalParentIndexName;
+    private final Boolean hasNaturalParentIndex;
+
+    InMemoryView(
+        ConcurrentMap<Comparable<Object>, T> data,
+        KVTypeInfo ti,
+        String naturalParentIndexName,
+        ConcurrentMap<Comparable<Object>, NaturalKeys> parentToChildrenMap) {
+      this.data = data;
       this.ti = ti;
       this.natural = ti != null ? ti.getAccessor(KVIndex.NATURAL_INDEX_NAME) : null;
+      this.naturalParentIndexName = naturalParentIndexName;
+      this.parentToChildrenMap = parentToChildrenMap;
+      this.hasNaturalParentIndex = !naturalParentIndexName.isEmpty();
     }
 
     @Override
     public Iterator<T> iterator() {
-      if (elements.isEmpty()) {
-        return new InMemoryIterator<>(elements.iterator());
+      if (data.isEmpty()) {
+        return new InMemoryIterator<>(Collections.emptyIterator());
       }
 
       KVTypeInfo.Accessor getter = index != null ? ti.getAccessor(index) : null;
@@ -322,15 +382,31 @@ public Iterator<T> iterator() {
      */
     private List<T> copyElements() {
       if (parent != null) {
-        KVTypeInfo.Accessor parentGetter = ti.getParentAccessor(index);
-        Preconditions.checkArgument(parentGetter != null, "Parent filter for non-child index.");
-        Comparable<?> parentKey = asKey(parent);
-
-        return elements.stream()
-          .filter(e -> compare(e, parentGetter, parentKey) == 0)
-          .collect(Collectors.toList());
+        Comparable<Object> parentKey = asKey(parent);
+        if (hasNaturalParentIndex && naturalParentIndexName.equals(ti.getParentIndexName(index))) {
+          // If there is a parent index for the natural index and the parent of `index` happens to
+          // be it, Spark can use the `parentToChildrenMap` to get the related natural keys, and
+          // then copy them from `data`.
+          NaturalKeys children = parentToChildrenMap.getOrDefault(parentKey, new NaturalKeys());
+          ArrayList<T> elements = new ArrayList<>();
+          for (Comparable<Object> naturalKey : children.keySet()) {
+            data.computeIfPresent(naturalKey, (k, v) -> {
+              elements.add(v);
+              return v;
+            });
+          }
+          return elements;
+        } else {
+          // Go through all the values in `data` and collect all the objects has certain parent
+          // value. This can be slow when there is a large number of entries in `data`.
+          KVTypeInfo.Accessor parentGetter = ti.getParentAccessor(index);
+          Preconditions.checkArgument(parentGetter != null, "Parent filter for non-child index.");
+          return data.values().stream()
+            .filter(e -> compare(e, parentGetter, parentKey) == 0)
+            .collect(Collectors.toList());
+        }
       } else {
-        return new ArrayList<>(elements);
+        return new ArrayList<>(data.values());
       }
     }
 
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
index d2a26982d8703..5404d33dba5fb 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/KVTypeInfo.java
@@ -68,8 +68,6 @@ public KVTypeInfo(Class<?> type) {
 
     Preconditions.checkArgument(indices.containsKey(KVIndex.NATURAL_INDEX_NAME),
         "No natural index defined for type %s.", type.getName());
-    Preconditions.checkArgument(indices.get(KVIndex.NATURAL_INDEX_NAME).parent().isEmpty(),
-        "Natural index of %s cannot have a parent.", type.getName());
 
     for (KVIndex idx : indices.values()) {
       if (!idx.parent().isEmpty()) {
@@ -117,6 +115,11 @@ Accessor getParentAccessor(String indexName) {
     return index.parent().isEmpty() ? null : getAccessor(index.parent());
   }
 
+  String getParentIndexName(String indexName) {
+    KVIndex index = indices.get(indexName);
+    return index.parent();
+  }
+
   /**
    * Abstracts the difference between invoking a Field and a Method.
    */
diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
index f4d359234cb9e..d7423537ddfcf 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDBTypeInfo.java
@@ -133,12 +133,13 @@ class LevelDBTypeInfo {
 
     // First create the parent indices, then the child indices.
     ti.indices().forEach(idx -> {
-      if (idx.parent().isEmpty()) {
+      // In LevelDB, there is no parent index for the NUTURAL INDEX.
+      if (idx.parent().isEmpty() || idx.value().equals(KVIndex.NATURAL_INDEX_NAME)) {
         indices.put(idx.value(), new Index(idx, ti.getAccessor(idx.value()), null));
       }
     });
     ti.indices().forEach(idx -> {
-      if (!idx.parent().isEmpty()) {
+      if (!idx.parent().isEmpty() && !idx.value().equals(KVIndex.NATURAL_INDEX_NAME)) {
         indices.put(idx.value(), new Index(idx, ti.getAccessor(idx.value()),
           indices.get(idx.parent())));
       }
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index f0a94d84d8a04..c957ff75a501f 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -154,7 +154,7 @@ private[spark] object TaskIndexNames {
 private[spark] class TaskDataWrapper(
     // Storing this as an object actually saves memory; it's also used as the key in the in-memory
     // store, so in that case you'd save the extra copy of the value here.
-    @KVIndexParam
+    @KVIndexParam(parent = TaskIndexNames.STAGE)
     val taskId: JLong,
     @KVIndexParam(value = TaskIndexNames.TASK_INDEX, parent = TaskIndexNames.STAGE)
     val index: Int,

From 6c4977d38f13628abfa24129ae6844146672d96d Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Mon, 2 Mar 2020 22:33:11 +0800
Subject: [PATCH 0195/1280] [SPARK-30993][SQL] Use its sql type for UDT when
 checking the type of length (fixed/var) or mutable

### What changes were proposed in this pull request?

This patch fixes the bug of UnsafeRow which misses to handle the UDT specifically, in `isFixedLength` and `isMutable`. These methods don't check its SQL type for UDT, always treating UDT as variable-length, and non-mutable.

It doesn't bring any issue if UDT is used to represent complicated type, but when UDT is used to represent some type which is matched with fixed length of SQL type, it exposes the chance of correctness issues, as these informations sometimes decide how the value should be handled.

We got report from user mailing list which suspected as mapGroupsWithState looks like handling UDT incorrectly, but after some investigation it was from GenerateUnsafeRowJoiner in shuffle phase.

https://github.com/apache/spark/blob/0e2ca11d80c3921387d7b077cb64c3a0c06b08d7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoiner.scala#L32-L43

Here updating position should not happen on fixed-length column, but due to this bug, the value of UDT having fixed-length as sql type would be modified, which actually corrupts the value.

### Why are the changes needed?

Misclassifying of the type of length for UDT can corrupt the value when the row is presented to the input of GenerateUnsafeRowJoiner, which brings correctness issue.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

New UT added.

Closes #27747 from HeartSaVioR/SPARK-30993.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f24a46011c8cba086193f697d653b6eccd029e8f)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/expressions/UnsafeRow.java   |  8 ++++
 .../GenerateUnsafeRowJoinerSuite.scala        | 41 ++++++++++++++++++-
 .../spark/sql/UserDefinedTypeSuite.scala      | 37 +++++++++++++++++
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 23e7d1f07e4a3..034894bd86085 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -95,6 +95,10 @@ public static int calculateBitSetWidthInBytes(int numFields) {
   }
 
   public static boolean isFixedLength(DataType dt) {
+    if (dt instanceof UserDefinedType) {
+      return isFixedLength(((UserDefinedType) dt).sqlType());
+    }
+
     if (dt instanceof DecimalType) {
       return ((DecimalType) dt).precision() <= Decimal.MAX_LONG_DIGITS();
     } else {
@@ -103,6 +107,10 @@ public static boolean isFixedLength(DataType dt) {
   }
 
   public static boolean isMutable(DataType dt) {
+    if (dt instanceof UserDefinedType) {
+      return isMutable(((UserDefinedType) dt).sqlType());
+    }
+
     return mutableFieldTypes.contains(dt) || dt instanceof DecimalType ||
       dt instanceof CalendarIntervalType;
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
index 81e29931e41dd..fb1ea7b867a6d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
@@ -17,13 +17,15 @@
 
 package org.apache.spark.sql.catalyst.expressions.codegen
 
+import java.time.{LocalDateTime, ZoneOffset}
+
 import scala.util.Random
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.RandomDataGenerator
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.{JoinedRow, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, JoinedRow, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -99,6 +101,23 @@ class GenerateUnsafeRowJoinerSuite extends SparkFunSuite {
     testConcatOnce(N, N, variable)
   }
 
+  test("SPARK-30993: UserDefinedType matched to fixed length SQL type shouldn't be corrupted") {
+    val schema1 = new StructType(Array(
+      StructField("date", new WrappedDateTimeUDT),
+      StructField("s", StringType),
+      StructField("i", IntegerType)))
+    val proj1 = UnsafeProjection.create(schema1.fields.map(_.dataType))
+    val intRow1 = new GenericInternalRow(Array[Any](
+      LocalDateTime.now().toEpochSecond(ZoneOffset.UTC),
+      UTF8String.fromString("hello"), 1))
+
+    val schema2 = new StructType(Array(StructField("i", IntegerType)))
+    val proj2 = UnsafeProjection.create(schema2.fields.map(_.dataType))
+    val intRow2 = new GenericInternalRow(Array[Any](2))
+
+    testConcat(schema1, proj1.apply(intRow1), schema2, proj2.apply(intRow2))
+  }
+
   private def testConcat(numFields1: Int, numFields2: Int, candidateTypes: Seq[DataType]): Unit = {
     for (i <- 0 until 10) {
       testConcatOnce(numFields1, numFields2, candidateTypes)
@@ -204,3 +223,23 @@ class GenerateUnsafeRowJoinerSuite extends SparkFunSuite {
   }
 
 }
+
+private[sql] case class WrappedDateTime(dt: LocalDateTime)
+
+private[sql] class WrappedDateTimeUDT extends UserDefinedType[WrappedDateTime] {
+  override def sqlType: DataType = LongType
+
+  override def serialize(obj: WrappedDateTime): Long = {
+    obj.dt.toEpochSecond(ZoneOffset.UTC)
+  }
+
+  def deserialize(datum: Any): WrappedDateTime = datum match {
+    case value: Long =>
+      val v = LocalDateTime.ofEpochSecond(value, 0, ZoneOffset.UTC)
+      WrappedDateTime(v)
+  }
+
+  override def userClass: Class[WrappedDateTime] = classOf[WrappedDateTime]
+
+  private[spark] override def asNullable: WrappedDateTimeUDT = this
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index ffc2018d2132d..157610f5d8a9e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import java.time.{LocalDateTime, ZoneOffset}
 import java.util.Arrays
 
 import org.apache.spark.rdd.RDD
@@ -103,6 +104,24 @@ private[spark] class ExampleSubTypeUDT extends UserDefinedType[IExampleSubType]
   override def userClass: Class[IExampleSubType] = classOf[IExampleSubType]
 }
 
+private[sql] case class FooWithDate(date: LocalDateTime, s: String, i: Int)
+
+private[sql] class LocalDateTimeUDT extends UserDefinedType[LocalDateTime] {
+  override def sqlType: DataType = LongType
+
+  override def serialize(obj: LocalDateTime): Long = {
+    obj.toEpochSecond(ZoneOffset.UTC)
+  }
+
+  def deserialize(datum: Any): LocalDateTime = datum match {
+    case value: Long => LocalDateTime.ofEpochSecond(value, 0, ZoneOffset.UTC)
+  }
+
+  override def userClass: Class[LocalDateTime] = classOf[LocalDateTime]
+
+  private[spark] override def asNullable: LocalDateTimeUDT = this
+}
+
 class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with ParquetTest
     with ExpressionEvalHelper {
   import testImplicits._
@@ -287,4 +306,22 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
     checkAnswer(spark.createDataFrame(data, schema).selectExpr("typeof(a)"),
       Seq(Row("array<double>")))
   }
+
+  test("SPARK-30993: UserDefinedType matched to fixed length SQL type shouldn't be corrupted") {
+    def concatFoo(a: FooWithDate, b: FooWithDate): FooWithDate = {
+      FooWithDate(b.date, a.s + b.s, a.i)
+    }
+
+    UDTRegistration.register(classOf[LocalDateTime].getName, classOf[LocalDateTimeUDT].getName)
+
+    // remove sub-millisecond part as we only use millis based timestamp while serde
+    val date = LocalDateTime.ofEpochSecond(LocalDateTime.now().toEpochSecond(ZoneOffset.UTC),
+      0, ZoneOffset.UTC)
+    val inputDS = List(FooWithDate(date, "Foo", 1), FooWithDate(date, "Foo", 3),
+      FooWithDate(date, "Foo", 3)).toDS()
+    val agg = inputDS.groupByKey(x => x.i).mapGroups((_, iter) => iter.reduce(concatFoo))
+    val result = agg.collect()
+
+    assert(result.toSet === Set(FooWithDate(date, "FooFoo", 3), FooWithDate(date, "Foo", 1)))
+  }
 }

From d64d6ee8ac2e9a105da63cd5e310f486de42b76c Mon Sep 17 00:00:00 2001
From: "Wu, Xiaochang" <xiaochang.wu@intel.com>
Date: Mon, 2 Mar 2020 08:56:17 -0600
Subject: [PATCH 0196/1280] [SPARK-30813][ML] Fix Matrices.sprand comments

### What changes were proposed in this pull request?
Fix mistakes in comments

### Why are the changes needed?
There are mistakes in comments

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
N/A

Closes #27564 from xwu99/fix-mllib-sprand-comment.

Authored-by: Wu, Xiaochang <xiaochang.wu@intel.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit ac122762f5091a7abf62d17595e0f5a99374ac5c)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../src/main/scala/org/apache/spark/ml/linalg/Matrices.scala    | 2 +-
 .../src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
index 61d35c8f7e303..34e4366701bbe 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Matrices.scala
@@ -1103,7 +1103,7 @@ object Matrices {
     DenseMatrix.rand(numRows, numCols, rng)
 
   /**
-   * Generate a `SparseMatrix` consisting of `i.i.d.` gaussian random numbers.
+   * Generate a `SparseMatrix` consisting of `i.i.d.` uniform random numbers.
    * @param numRows number of rows of the matrix
    * @param numCols number of columns of the matrix
    * @param density the desired density for the matrix
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 83187d65530ad..57edc965112ef 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -1055,7 +1055,7 @@ object Matrices {
     DenseMatrix.rand(numRows, numCols, rng)
 
   /**
-   * Generate a `SparseMatrix` consisting of `i.i.d.` gaussian random numbers.
+   * Generate a `SparseMatrix` consisting of `i.i.d.` uniform random numbers.
    * @param numRows number of rows of the matrix
    * @param numCols number of columns of the matrix
    * @param density the desired density for the matrix

From 148262f3fbdf7c5da7cd147cf43bf5ebab5f5244 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 2 Mar 2020 11:23:07 -0800
Subject: [PATCH 0197/1280] [SPARK-30969][CORE] Remove resource coordination
 support from Standalone

### What changes were proposed in this pull request?

Remove automatically resource coordination support from Standalone.

### Why are the changes needed?

Resource coordination is mainly designed for the scenario where multiple workers launched on the same host. However, it's, actually, a non-existed  scenario for today's Spark. Because, Spark now can start multiple executors in a single Worker, while it only allow one executor per Worker at very beginning. So, now, it really help nothing for user to launch multiple workers on the same host. Thus, it's not worth for us to bring over complicated implementation and potential high maintain cost for such an impossible scenario.

### Does this PR introduce any user-facing change?

No, it's Spark 3.0 feature.

### How was this patch tested?

Pass Jenkins.

Closes #27722 from Ngone51/abandon_coordination.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit b517f991fe0c95a186872d38be6a2091d9326195)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .gitignore                                    |   1 -
 .../scala/org/apache/spark/SparkContext.scala |  25 +-
 .../deploy/StandaloneResourceUtils.scala      | 263 +-----------------
 .../apache/spark/deploy/worker/Worker.scala   |  27 +-
 .../spark/internal/config/package.scala       |  17 --
 .../scala/org/apache/spark/util/Utils.scala   |  22 --
 .../org/apache/spark/SparkContextSuite.scala  |   6 +-
 .../spark/deploy/worker/WorkerSuite.scala     |  74 +----
 .../ResourceDiscoveryPluginSuite.scala        |   4 -
 docs/configuration.md                         |  27 +-
 docs/spark-standalone.md                      |  10 +-
 11 files changed, 17 insertions(+), 459 deletions(-)

diff --git a/.gitignore b/.gitignore
index 798e8acc4d43b..198fdee39be95 100644
--- a/.gitignore
+++ b/.gitignore
@@ -72,7 +72,6 @@ scalastyle-on-compile.generated.xml
 scalastyle-output.xml
 scalastyle.txt
 spark-*-bin-*.tgz
-spark-resources/
 spark-tests.log
 src_managed/
 streaming-tests.log
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 91188d58f4201..bcbb7e42f8d66 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -41,7 +41,6 @@ import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat => NewFileInputFor
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.deploy.{LocalSparkCluster, SparkHadoopUtil}
-import org.apache.spark.deploy.StandaloneResourceUtils._
 import org.apache.spark.executor.{ExecutorMetrics, ExecutorMetricsSource}
 import org.apache.spark.input.{FixedLengthBinaryInputFormat, PortableDataStream, StreamInputFormat, WholeTextFileInputFormat}
 import org.apache.spark.internal.Logging
@@ -250,15 +249,6 @@ class SparkContext(config: SparkConf) extends Logging {
 
   def isLocal: Boolean = Utils.isLocalMaster(_conf)
 
-  private def isClientStandalone: Boolean = {
-    val isSparkCluster = master match {
-      case SparkMasterRegex.SPARK_REGEX(_) => true
-      case SparkMasterRegex.LOCAL_CLUSTER_REGEX(_, _, _) => true
-      case _ => false
-    }
-    deployMode == "client" && isSparkCluster
-  }
-
   /**
    * @return true if context is stopped or in the midst of stopping.
    */
@@ -396,17 +386,7 @@ class SparkContext(config: SparkConf) extends Logging {
     _driverLogger = DriverLogger(_conf)
 
     val resourcesFileOpt = conf.get(DRIVER_RESOURCES_FILE)
-    val allResources = getOrDiscoverAllResources(_conf, SPARK_DRIVER_PREFIX, resourcesFileOpt)
-    _resources = {
-      // driver submitted in client mode under Standalone may have conflicting resources with
-      // other drivers/workers on this host. We should sync driver's resources info into
-      // SPARK_RESOURCES/SPARK_RESOURCES_COORDINATE_DIR/ to avoid collision.
-      if (isClientStandalone) {
-        acquireResources(_conf, SPARK_DRIVER_PREFIX, allResources, Utils.getProcessId)
-      } else {
-        allResources
-      }
-    }
+    _resources = getOrDiscoverAllResources(_conf, SPARK_DRIVER_PREFIX, resourcesFileOpt)
     logResourceInfo(SPARK_DRIVER_PREFIX, _resources)
 
     // log out spark.app.name in the Spark driver logs
@@ -2019,9 +1999,6 @@ class SparkContext(config: SparkConf) extends Logging {
     Utils.tryLogNonFatalError {
       _progressBar.foreach(_.stop())
     }
-    if (isClientStandalone) {
-      releaseResources(_conf, SPARK_DRIVER_PREFIX, _resources, Utils.getProcessId)
-    }
     _taskScheduler = null
     // TODO: Cache.stop()?
     if (_env != null) {
diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index 65bf4351ebfd9..e08709ec36ab0 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -17,30 +17,21 @@
 
 package org.apache.spark.deploy
 
-import java.io.{File, RandomAccessFile}
-import java.nio.channels.{FileLock, OverlappingFileLockException}
+import java.io.File
 import java.nio.file.Files
 
 import scala.collection.mutable
-import scala.util.Random
 import scala.util.control.NonFatal
 
 import org.json4s.{DefaultFormats, Extraction}
-import org.json4s.jackson.JsonMethods.{compact, parse, render}
+import org.json4s.jackson.JsonMethods.{compact, render}
 
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.SparkException
 import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.{SPARK_RESOURCES_COORDINATE, SPARK_RESOURCES_DIR}
 import org.apache.spark.resource.{ResourceAllocation, ResourceID, ResourceInformation, ResourceRequirement}
-import org.apache.spark.resource.ResourceUtils.{parseResourceRequirements, withResourcesJson}
 import org.apache.spark.util.Utils
 
 private[spark] object StandaloneResourceUtils extends Logging {
-  // These directory/files are used to coordinate the resources between
-  // the drivers/workers on the host in Spark Standalone.
-  val SPARK_RESOURCES_COORDINATE_DIR = "spark-resources"
-  val ALLOCATED_RESOURCES_FILE = "__allocated_resources__.json"
-  val RESOURCES_LOCK_FILE = "__allocated_resources__.lock"
 
   /**
    * A mutable resource information which provides more efficient modification on addresses.
@@ -86,249 +77,6 @@ private[spark] object StandaloneResourceUtils extends Logging {
     }
   }
 
-  /**
-   * Assigns (if coordinate needed) resources to workers/drivers from the same host to avoid
-   * address conflict.
-   *
-   * This function works in three steps. First, acquiring the lock on RESOURCES_LOCK_FILE
-   * to achieve synchronization among workers and drivers. Second, getting all allocated
-   * resources from ALLOCATED_RESOURCES_FILE and assigning isolated resources to the worker
-   * or driver after differentiating available resources in discovered resources from
-   * allocated resources. If available resources don't meet worker's or driver's requirement,
-   * try to update allocated resources by excluding the resource allocation if its related
-   * process has already terminated and do the assignment again. If still don't meet requirement,
-   * exception should be thrown. Third, updating ALLOCATED_RESOURCES_FILE with new allocated
-   * resources along with pid for the worker or driver. Then, return allocated resources
-   * information after releasing the lock.
-   *
-   * @param conf SparkConf
-   * @param componentName spark.driver / spark.worker
-   * @param resources the resources found by worker/driver on the host
-   * @param pid the process id of worker/driver to acquire resources.
-   * @return allocated resources for the worker/driver or throws exception if can't
-   *         meet worker/driver's requirement
-   */
-  def acquireResources(
-      conf: SparkConf,
-      componentName: String,
-      resources: Map[String, ResourceInformation],
-      pid: Int)
-    : Map[String, ResourceInformation] = {
-    if (!needCoordinate(conf)) {
-      return resources
-    }
-    val resourceRequirements = parseResourceRequirements(conf, componentName)
-    if (resourceRequirements.isEmpty) {
-      return Map.empty
-    }
-    val lock = acquireLock(conf)
-    try {
-      val resourcesFile = new File(getOrCreateResourcesDir(conf), ALLOCATED_RESOURCES_FILE)
-      // all allocated resources in ALLOCATED_RESOURCES_FILE, can be updated if any allocations'
-      // related processes detected to be terminated while checking pids below.
-      var origAllocation = Seq.empty[StandaloneResourceAllocation]
-      // Map[pid -> Map[resourceName -> Addresses[]]]
-      var allocated = {
-        if (resourcesFile.exists()) {
-          origAllocation = allocatedStandaloneResources(resourcesFile.getPath)
-          val allocations = origAllocation.map { resource =>
-            val resourceMap = {
-              resource.allocations.map { allocation =>
-                allocation.id.resourceName -> allocation.addresses.toArray
-              }.toMap
-            }
-            resource.pid -> resourceMap
-          }.toMap
-          allocations
-        } else {
-          Map.empty[Int, Map[String, Array[String]]]
-        }
-      }
-
-      // new allocated resources for worker or driver,
-      // map from resource name to its allocated addresses.
-      var newAssignments: Map[String, Array[String]] = null
-      // Whether we've checked process status and we'll only do the check at most once.
-      // Do the check iff the available resources can't meet the requirements at the first time.
-      var checked = false
-      // Whether we need to keep allocating for the worker/driver and we'll only go through
-      // the loop at most twice.
-      var keepAllocating = true
-      while (keepAllocating) {
-        keepAllocating = false
-        // store the pid whose related allocated resources conflict with
-        // discovered resources passed in.
-        val pidsToCheck = mutable.Set[Int]()
-        newAssignments = resourceRequirements.map { req =>
-          val rName = req.resourceName
-          val amount = req.amount
-          // initially, we must have available.length >= amount as we've done pre-check previously
-          var available = resources(rName).addresses
-          // gets available resource addresses by excluding all
-          // allocated resource addresses from discovered resources
-          allocated.foreach { a =>
-            val thePid = a._1
-            val resourceMap = a._2
-            val assigned = resourceMap.getOrElse(rName, Array.empty)
-            val retained = available.diff(assigned)
-            // if len(retained) < len(available) after differ to assigned, then, there must be
-            // some conflicting resources addresses between available and assigned. So, we should
-            // store its pid here to check whether it's alive in case we don't find enough
-            // resources after traversal all allocated resources.
-            if (retained.length < available.length && !checked) {
-              pidsToCheck += thePid
-            }
-            if (retained.length >= amount) {
-              available = retained
-            } else if (checked) {
-              keepAllocating = false
-              throw new SparkException(s"No more resources available since they've already" +
-                s" assigned to other workers/drivers.")
-            } else {
-              keepAllocating = true
-            }
-          }
-          val assigned = {
-            if (keepAllocating) { // can't meet the requirement
-              // excludes the allocation whose related process has already been terminated.
-              val (invalid, valid) = allocated.partition { a =>
-                pidsToCheck(a._1) && !(Utils.isTesting || Utils.isProcessRunning(a._1))}
-              allocated = valid
-              origAllocation = origAllocation.filter(
-                allocation => !invalid.contains(allocation.pid))
-              checked = true
-              // note this is a meaningless return value, just to avoid creating any new object
-              available
-            } else {
-              available.take(amount)
-            }
-          }
-          rName -> assigned
-        }.toMap
-      }
-      val newAllocation = {
-        val allocations = newAssignments.map { case (rName, addresses) =>
-          ResourceAllocation(new ResourceID(componentName, rName), addresses)
-        }.toSeq
-        StandaloneResourceAllocation(pid, allocations)
-      }
-      writeResourceAllocationJson(
-        componentName, origAllocation ++ Seq(newAllocation), resourcesFile)
-      newAllocation.toResourceInformationMap
-    } finally {
-      releaseLock(lock)
-    }
-  }
-
-  /**
-   * Frees (if coordinate needed) all the resources a worker/driver (pid) has in one shot
-   * to make those resources be available for other workers/drivers on the same host.
-   * @param conf SparkConf
-   * @param componentName spark.driver / spark.worker
-   * @param toRelease the resources expected to release
-   * @param pid the process id of worker/driver to release resources.
-   */
-  def releaseResources(
-      conf: SparkConf,
-      componentName: String,
-      toRelease: Map[String, ResourceInformation],
-      pid: Int)
-    : Unit = {
-    if (!needCoordinate(conf)) {
-      return
-    }
-    if (toRelease != null && toRelease.nonEmpty) {
-      val lock = acquireLock(conf)
-      try {
-        val resourcesFile = new File(getOrCreateResourcesDir(conf), ALLOCATED_RESOURCES_FILE)
-        if (resourcesFile.exists()) {
-          val (target, others) =
-            allocatedStandaloneResources(resourcesFile.getPath).partition(_.pid == pid)
-          if (target.nonEmpty) {
-            if (others.isEmpty) {
-              if (!resourcesFile.delete()) {
-                logError(s"Failed to delete $ALLOCATED_RESOURCES_FILE.")
-              }
-            } else {
-              writeResourceAllocationJson(componentName, others, resourcesFile)
-            }
-            logDebug(s"$componentName(pid=$pid) released resources: ${toRelease.mkString("\n")}")
-          } else {
-            logWarning(s"$componentName(pid=$pid) has already released its resources.")
-          }
-        }
-      } finally {
-        releaseLock(lock)
-      }
-    }
-  }
-
-  private def acquireLock(conf: SparkConf): FileLock = {
-    val resourcesDir = getOrCreateResourcesDir(conf)
-    val lockFile = new File(resourcesDir, RESOURCES_LOCK_FILE)
-    val lockFileChannel = new RandomAccessFile(lockFile, "rw").getChannel
-    var keepTry = true
-    var lock: FileLock = null
-    while (keepTry) {
-      try {
-        lock = lockFileChannel.lock()
-        logInfo(s"Acquired lock on $RESOURCES_LOCK_FILE.")
-        keepTry = false
-      } catch {
-        case e: OverlappingFileLockException =>
-          // This exception throws when we're in LocalSparkCluster mode. FileLock is designed
-          // to be used across JVMs, but our LocalSparkCluster is designed to launch multiple
-          // workers in the same JVM. As a result, when an worker in LocalSparkCluster try to
-          // acquire the lock on `resources.lock` which already locked by other worker, we'll
-          // hit this exception. So, we should manually control it.
-          keepTry = true
-          // there may be multiple workers race for the lock,
-          // so, sleep for a random time to avoid possible conflict
-          val duration = Random.nextInt(1000) + 1000
-          Thread.sleep(duration)
-      }
-    }
-    assert(lock != null, s"Acquired null lock on $RESOURCES_LOCK_FILE.")
-    lock
-  }
-
-  private def releaseLock(lock: FileLock): Unit = {
-    try {
-      lock.release()
-      lock.channel().close()
-      logInfo(s"Released lock on $RESOURCES_LOCK_FILE.")
-    } catch {
-      case e: Exception =>
-        logError(s"Error while releasing lock on $RESOURCES_LOCK_FILE.", e)
-    }
-  }
-
-  private def getOrCreateResourcesDir(conf: SparkConf): File = {
-    val coordinateDir = new File(conf.get(SPARK_RESOURCES_DIR).getOrElse {
-      val sparkHome = if (Utils.isTesting) {
-        assert(sys.props.contains("spark.test.home") ||
-          sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-        sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-      } else {
-        sys.env.getOrElse("SPARK_HOME", ".")
-      }
-      sparkHome
-    })
-    val resourceDir = new File(coordinateDir, SPARK_RESOURCES_COORDINATE_DIR)
-    if (!resourceDir.exists()) {
-      Utils.createDirectory(resourceDir)
-    }
-    resourceDir
-  }
-
-  private def allocatedStandaloneResources(resourcesFile: String)
-  : Seq[StandaloneResourceAllocation] = {
-    withResourcesJson[StandaloneResourceAllocation](resourcesFile) { json =>
-      implicit val formats = DefaultFormats
-      parse(json).extract[Seq[StandaloneResourceAllocation]]
-    }
-  }
-
   /**
    * Save the allocated resources of driver(cluster only) or executor into a JSON formatted
    * resources file. Used in Standalone only.
@@ -372,11 +120,6 @@ private[spark] object StandaloneResourceUtils extends Logging {
     Files.write(jsonFile.toPath, compact(render(allocationJson)).getBytes())
   }
 
-  /** Whether needs to coordinate resources among workers and drivers for user */
-  def needCoordinate(conf: SparkConf): Boolean = {
-    conf.get(SPARK_RESOURCES_COORDINATE)
-  }
-
   def toMutable(immutableResources: Map[String, ResourceInformation])
     : Map[String, MutableResourceInfo] = {
     immutableResources.map { case (rName, rInfo) =>
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index 4be495ac4f13f..73cac880d464a 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -34,7 +34,7 @@ import org.apache.spark.deploy.{Command, ExecutorDescription, ExecutorState}
 import org.apache.spark.deploy.DeployMessages._
 import org.apache.spark.deploy.ExternalShuffleService
 import org.apache.spark.deploy.StandaloneResourceUtils._
-import org.apache.spark.deploy.master.{DriverState, Master, WorkerResourceInfo}
+import org.apache.spark.deploy.master.{DriverState, Master}
 import org.apache.spark.deploy.worker.ui.WorkerWebUI
 import org.apache.spark.internal.{config, Logging}
 import org.apache.spark.internal.config.Tests.IS_TESTING
@@ -57,8 +57,7 @@ private[deploy] class Worker(
     val conf: SparkConf,
     val securityMgr: SecurityManager,
     resourceFileOpt: Option[String] = None,
-    externalShuffleServiceSupplier: Supplier[ExternalShuffleService] = null,
-    pid: Int = Utils.getProcessId)
+    externalShuffleServiceSupplier: Supplier[ExternalShuffleService] = null)
   extends ThreadSafeRpcEndpoint with Logging {
 
   private val host = rpcEnv.address.host
@@ -205,7 +204,6 @@ private[deploy] class Worker(
     logInfo("Spark home: " + sparkHome)
     createWorkDir()
     startExternalShuffleService()
-    releaseResourcesOnInterrupt()
     setupWorkerResources()
     webUi = new WorkerWebUI(this, workDir, webUiPort)
     webUi.bind()
@@ -219,26 +217,13 @@ private[deploy] class Worker(
     metricsSystem.getServletHandlers.foreach(webUi.attachHandler)
   }
 
-  /**
-   * Used to catch the TERM signal from sbin/stop-slave.sh and
-   * release resources before Worker exits
-   */
-  private def releaseResourcesOnInterrupt(): Unit = {
-    SignalUtils.register("TERM") {
-      releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
-      false
-    }
-  }
-
   private def setupWorkerResources(): Unit = {
     try {
-      val allResources = getOrDiscoverAllResources(conf, SPARK_WORKER_PREFIX, resourceFileOpt)
-      resources = acquireResources(conf, SPARK_WORKER_PREFIX, allResources, pid)
+      resources = getOrDiscoverAllResources(conf, SPARK_WORKER_PREFIX, resourceFileOpt)
       logResourceInfo(SPARK_WORKER_PREFIX, resources)
     } catch {
       case e: Exception =>
         logError("Failed to setup worker resources: ", e)
-        releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
         if (!Utils.isTesting) {
           System.exit(1)
         }
@@ -373,7 +358,6 @@ private[deploy] class Worker(
               TimeUnit.SECONDS))
         }
       } else {
-        releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
         logError("All masters are unresponsive! Giving up.")
         System.exit(1)
       }
@@ -472,7 +456,6 @@ private[deploy] class Worker(
       case RegisterWorkerFailed(message) =>
         if (!registered) {
           logError("Worker registration failed: " + message)
-          releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
           System.exit(1)
         }
 
@@ -738,7 +721,6 @@ private[deploy] class Worker(
   }
 
   override def onStop(): Unit = {
-    releaseResources(conf, SPARK_WORKER_PREFIX, resources, pid)
     cleanupThreadExecutor.shutdownNow()
     metricsSystem.report()
     cancelLastRegistrationRetry()
@@ -875,9 +857,8 @@ private[deploy] object Worker extends Logging {
     val securityMgr = new SecurityManager(conf)
     val rpcEnv = RpcEnv.create(systemName, host, port, conf, securityMgr)
     val masterAddresses = masterUrls.map(RpcAddress.fromSparkURL)
-    val pid = if (Utils.isTesting) workerNumber.get else Utils.getProcessId
     rpcEnv.setupEndpoint(ENDPOINT_NAME, new Worker(rpcEnv, webUiPort, cores, memory,
-      masterAddresses, ENDPOINT_NAME, workDir, conf, securityMgr, resourceFileOpt, pid = pid))
+      masterAddresses, ENDPOINT_NAME, workDir, conf, securityMgr, resourceFileOpt))
     rpcEnv
   }
 
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 37ce178407381..23c31a5be831d 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -37,23 +37,6 @@ package object config {
   private[spark] val SPARK_TASK_PREFIX = "spark.task"
   private[spark] val LISTENER_BUS_EVENT_QUEUE_PREFIX = "spark.scheduler.listenerbus.eventqueue"
 
-  private[spark] val SPARK_RESOURCES_COORDINATE =
-    ConfigBuilder("spark.resources.coordinateResourcesInStandalone")
-      .doc("Whether to coordinate resources automatically among workers/drivers(client only) " +
-        "in Standalone. If false, the user is responsible for configuring different resources " +
-        "for workers/drivers that run on the same host.")
-      .booleanConf
-      .createWithDefault(true)
-
-  private[spark] val SPARK_RESOURCES_DIR =
-    ConfigBuilder("spark.resources.dir")
-      .doc("Directory used to coordinate resources among workers/drivers(client only) in " +
-        "Standalone. Default is SPARK_HOME. Make sure to use the same directory for worker " +
-        "and drivers in client mode that run on the same host. Don't clean up this directory " +
-        "while workers/drivers are still alive to avoid the most likely resources conflict. ")
-      .stringConf
-      .createOptional
-
   private[spark] val RESOURCES_DISCOVERY_PLUGIN =
     ConfigBuilder("spark.resources.discoveryPlugin")
       .doc("Comma-separated list of class names implementing" +
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index dde43232f0d65..9f332ba608082 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2546,28 +2546,6 @@ private[spark] object Utils extends Logging {
     new File(path.getAbsolutePath + "." + UUID.randomUUID())
   }
 
-  /**
-   * Given a process id, return true if the process is still running.
-   */
-  def isProcessRunning(pid: Int): Boolean = {
-    val process = executeCommand(Seq("kill", "-0", pid.toString))
-    process.waitFor(10, TimeUnit.SECONDS)
-    process.exitValue() == 0
-  }
-
-  /**
-   * Returns the pid of this JVM process.
-   */
-  def getProcessId: Int = {
-    val PROCESS = "(\\d+)@(.*)".r
-    val name = getProcessName()
-    name match {
-      case PROCESS(pid, _) => pid.toInt
-      case _ =>
-        throw new SparkException(s"Unexpected process name: $name, expected to be PID@hostname.")
-    }
-  }
-
   /**
    * Returns the name of this JVM process. This is OS dependent but typically (OSX, Linux, Windows),
    * this is formatted as PID@hostname.
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index df9c7c5eaa368..9f8fa89985ae5 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -841,7 +841,6 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
         .setAppName("test-cluster")
         .set(DRIVER_GPU_ID.amountConf, "3")
         .set(DRIVER_GPU_ID.discoveryScriptConf, scriptPath)
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
 
@@ -924,7 +923,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     assume(!(Utils.isWindows))
     withTempDir { dir =>
       val discoveryScript = createTempScriptWithExpectedOutput(dir, "resourceDiscoveryScript",
-        """{"name": "gpu","addresses":["0", "1", "2", "3", "4", "5", "6", "7", "8"]}""")
+        """{"name": "gpu","addresses":["0", "1", "2"]}""")
 
       val conf = new SparkConf()
         .setMaster("local-cluster[3, 1, 1024]")
@@ -933,7 +932,6 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
         .set(WORKER_GPU_ID.discoveryScriptConf, discoveryScript)
         .set(TASK_GPU_ID.amountConf, "3")
         .set(EXECUTOR_GPU_ID.amountConf, "3")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
 
@@ -945,7 +943,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
         context.resources().get(GPU).get.addresses.iterator
       }
       val gpus = rdd.collect()
-      assert(gpus.sorted === Seq("0", "1", "2", "3", "4", "5", "6", "7", "8"))
+      assert(gpus.sorted === Seq("0", "0", "0", "1", "1", "1", "2", "2", "2"))
 
       eventually(timeout(10.seconds)) {
         assert(sc.statusTracker.getExecutorInfos.map(_.numRunningTasks()).sum == 0)
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
index bb541b4cad8bd..2d3d0afe3f80c 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
@@ -36,7 +36,6 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
 import org.apache.spark.TestUtils.{createTempJsonFile, createTempScriptWithExpectedOutput}
 import org.apache.spark.deploy.{Command, ExecutorState, ExternalShuffleService}
 import org.apache.spark.deploy.DeployMessages.{DriverStateChanged, ExecutorStateChanged, WorkDirCleanup}
-import org.apache.spark.deploy.StandaloneResourceUtils.{ALLOCATED_RESOURCES_FILE, SPARK_RESOURCES_COORDINATE_DIR}
 import org.apache.spark.deploy.master.DriverState
 import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Worker._
@@ -64,7 +63,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
   private def makeWorker(
       conf: SparkConf = new SparkConf(),
       shuffleServiceSupplier: Supplier[ExternalShuffleService] = null,
-      pid: Int = Utils.getProcessId,
       local: Boolean = false): Worker = {
     assert(_worker === null, "Some Worker's RpcEnv is leaked in tests")
     val securityMgr = new SecurityManager(conf)
@@ -72,7 +70,7 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
     val resourcesFile = conf.get(SPARK_WORKER_RESOURCE_FILE)
     val localWorker = new Worker(rpcEnv, 50000, 20, 1234 * 5,
       Array.fill(1)(RpcAddress("1.2.3.4", 1234)), "Worker", "/tmp",
-      conf, securityMgr, resourcesFile, shuffleServiceSupplier, pid)
+      conf, securityMgr, resourcesFile, shuffleServiceSupplier)
     if (local) {
       localWorker
     } else {
@@ -81,14 +79,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
     }
   }
 
-  private def assertResourcesFileDeleted(): Unit = {
-    assert(sys.props.contains("spark.test.home"))
-    val sparkHome = sys.props.get("spark.test.home")
-    val resourceFile = new File(sparkHome + "/" + SPARK_RESOURCES_COORDINATE_DIR,
-      ALLOCATED_RESOURCES_FILE)
-    assert(!resourceFile.exists())
-  }
-
   before {
     MockitoAnnotations.initMocks(this)
   }
@@ -251,7 +241,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
       worker.rpcEnv.shutdown()
       worker.rpcEnv.awaitTermination()
     }
-    assertResourcesFileDeleted()
   }
 
   test("worker could load resources from resources file while launching") {
@@ -273,7 +262,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
         worker.rpcEnv.shutdown()
         worker.rpcEnv.awaitTermination()
       }
-      assertResourcesFileDeleted()
     }
   }
 
@@ -292,7 +280,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
         worker.rpcEnv.shutdown()
         worker.rpcEnv.awaitTermination()
       }
-      assertResourcesFileDeleted()
     }
   }
 
@@ -316,65 +303,6 @@ class WorkerSuite extends SparkFunSuite with Matchers with BeforeAndAfter {
         worker.rpcEnv.shutdown()
         worker.rpcEnv.awaitTermination()
       }
-      assertResourcesFileDeleted()
-    }
-  }
-
-  test("Workers run on the same host should avoid resources conflict when coordinate is on") {
-    val conf = new SparkConf()
-    withTempDir { dir =>
-      val scriptPath = createTempScriptWithExpectedOutput(dir, "fpgaDiscoverScript",
-        """{"name": "fpga","addresses":["f1", "f2", "f3", "f4", "f5"]}""")
-      conf.set(WORKER_FPGA_ID.discoveryScriptConf, scriptPath)
-      conf.set(WORKER_FPGA_ID.amountConf, "2")
-      val workers = (0 until 3).map(id => makeWorker(conf, pid = id, local = true))
-      workers.zipWithIndex.foreach{case (w, i) => w.rpcEnv.setupEndpoint(s"worker$i", w)}
-      eventually(timeout(20.seconds)) {
-        val (empty, nonEmpty) = workers.partition(_.resources.isEmpty)
-        assert(empty.length === 1)
-        assert(nonEmpty.length === 2)
-        val totalResources = nonEmpty.flatMap(_.resources(FPGA).addresses).toSet.toSeq.sorted
-        assert(totalResources === Seq("f1", "f2", "f3", "f4"))
-        workers.foreach(_.rpcEnv.shutdown())
-        workers.foreach(_.rpcEnv.awaitTermination())
-      }
-      assertResourcesFileDeleted()
-    }
-  }
-
-  test("Workers run on the same host should load resources naively when coordinate is off") {
-    val conf = new SparkConf()
-    // disable coordination
-    conf.set(config.SPARK_RESOURCES_COORDINATE, false)
-    withTempDir { dir =>
-      val gpuArgs = ResourceAllocation(WORKER_GPU_ID, Seq("g0", "g1"))
-      val ja = Extraction.decompose(Seq(gpuArgs))
-      val resourcesPath = createTempJsonFile(dir, "resources", ja)
-      val scriptPath = createTempScriptWithExpectedOutput(dir, "fpgaDiscoverScript",
-        """{"name": "fpga","addresses":["f1", "f2", "f3", "f4", "f5"]}""")
-      conf.set(SPARK_WORKER_RESOURCE_FILE.key, resourcesPath)
-      conf.set(WORKER_GPU_ID.amountConf, "2")
-      conf.set(WORKER_FPGA_ID.discoveryScriptConf, scriptPath)
-      conf.set(WORKER_FPGA_ID.amountConf, "2")
-      val workers = (0 until 3).map(id => makeWorker(conf, pid = id, local = true))
-      workers.zipWithIndex.foreach{case (w, i) => w.rpcEnv.setupEndpoint(s"worker$i", w)}
-      eventually(timeout(20.seconds)) {
-        val (empty, nonEmpty) = workers.partition(_.resources.isEmpty)
-        assert(empty.length === 0)
-        assert(nonEmpty.length === 3)
-        // Each Worker should get the same resources from resources file and discovery script
-        // without coordination. Note that, normally, we must config different resources
-        // for workers run on the same host when coordinate config is off. Test here is used
-        // to validate the different behaviour comparing to the above test when coordinate config
-        // is on, so we admit the resources collision here.
-        nonEmpty.foreach { worker =>
-          assert(worker.resources === Map(GPU -> gpuArgs.toResourceInformation,
-            FPGA -> new ResourceInformation(FPGA, Array("f1", "f2", "f3", "f4", "f5"))))
-        }
-        workers.foreach(_.rpcEnv.shutdown())
-        workers.foreach(_.rpcEnv.awaitTermination())
-      }
-      assertResourcesFileDeleted()
     }
   }
 
diff --git a/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala b/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
index 437c903e77d4a..ff7d680352177 100644
--- a/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
+++ b/core/src/test/scala/org/apache/spark/resource/ResourceDiscoveryPluginSuite.scala
@@ -50,7 +50,6 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(WORKER_GPU_ID.amountConf, "2")
         .set(TASK_GPU_ID.amountConf, "1")
         .set(EXECUTOR_GPU_ID.amountConf, "1")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
         .set(WORKER_FPGA_ID.amountConf, "2")
         .set(TASK_FPGA_ID.amountConf, "1")
         .set(EXECUTOR_FPGA_ID.amountConf, "1")
@@ -81,7 +80,6 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(WORKER_GPU_ID.amountConf, "2")
         .set(TASK_GPU_ID.amountConf, "1")
         .set(EXECUTOR_GPU_ID.amountConf, "1")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
       TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
@@ -108,7 +106,6 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(WORKER_GPU_ID.amountConf, "2")
         .set(TASK_GPU_ID.amountConf, "1")
         .set(EXECUTOR_GPU_ID.amountConf, "1")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
       TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
@@ -134,7 +131,6 @@ class ResourceDiscoveryPluginSuite extends SparkFunSuite with LocalSparkContext
         .set(RESOURCES_DISCOVERY_PLUGIN, Seq(classOf[TestResourceDiscoveryPluginEmpty].getName()))
         .set(DRIVER_GPU_ID.discoveryScriptConf, scriptPath)
         .set(DRIVER_GPU_ID.amountConf, "2")
-        .set(SPARK_RESOURCES_DIR, dir.getName())
 
       sc = new SparkContext(conf)
       TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
diff --git a/docs/configuration.md b/docs/configuration.md
index 469feed1745c4..5e6fe9349e59d 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -193,25 +193,6 @@ of the most common options to set are:
     and <code>spark.driver.memory</code>.
   </td>
 </tr>
-<tr>
- <td><code>spark.resources.coordinateResourcesInStandalone</code></td>
-  <td>true</td>
-  <td>
-    Whether to coordinate resources automatically among workers/drivers(client only) 
-    in Standalone. If false, the user is responsible for configuring different resources
-    for workers/drivers that run on the same host.
-  </td>
-</tr>
-<tr>
- <td><code>spark.resources.dir</code></td>
-  <td>SPARK_HOME</td>
-  <td>
-    Directory used to coordinate resources among workers/drivers(client only) in Standalone.
-    Default is SPARK_HOME. Make sure to use the same directory for worker and drivers in
-    client mode that run on the same host. Don't clean up this directory while workers/drivers
-    are still alive to avoid the most likely resources conflict. 
-  </td>
-</tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.amount</code></td>
   <td>0</td>
@@ -228,9 +209,8 @@ of the most common options to set are:
   <td>
     A script for the driver to run to discover a particular resource type. This should
     write to STDOUT a JSON string in the format of the ResourceInformation class. This has a
-    name and an array of addresses. For a client-submitted driver in Standalone, discovery
-    script must assign different resource addresses to this driver comparing to workers' and
-    other drivers' when <code>spark.resources.coordinateResourcesInStandalone</code> is off.
+    name and an array of addresses. For a client-submitted driver, discovery script must assign
+    different resource addresses to this driver comparing to other drivers on the same host.
   </td>
 </tr>
 <tr>
@@ -2755,5 +2735,4 @@ There are configurations available to request resources for the driver: <code>sp
 
 Spark will use the configurations specified to first request containers with the corresponding resources from the cluster manager. Once it gets the container, Spark launches an Executor in that container which will discover what resources the container has and the addresses associated with each resource. The Executor will register with the Driver and report back the resources available to that Executor. The Spark scheduler can then schedule tasks to each Executor and assign specific resource addresses based on the resource requirements the user specified. The user can see the resources assigned to a task using the <code>TaskContext.get().resources</code> api. On the driver, the user can see the resources assigned with the SparkContext <code>resources</code> call. It's then up to the user to use the assignedaddresses to do the processing they want or pass those into the ML/AI framework they are using.
 
-See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. If using local-cluster mode see the Spark Standalone documentation but be aware only a single worker resources file or discovery script can be specified the is shared by all the Workers so you should enable resource coordination (see <code>spark.resources.coordinateResourcesInStandalone</code>).
-
+See your cluster manager specific page for requirements and details on each of - [YARN](running-on-yarn.html#resource-allocation-and-configuration-overview), [Kubernetes](running-on-kubernetes.html#resource-allocation-and-configuration-overview) and [Standalone Mode](spark-standalone.html#resource-allocation-and-configuration-overview). It is currently not available with Mesos or local mode. And please also note that local-cluster mode with multiple workers is not supported(see Standalone documentation).
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index ff310edb5a633..17b6772390d82 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -256,8 +256,6 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Path to resource discovery script, which is used to find a particular resource while worker starting up.
     And the output of the script should be formatted like the <code>ResourceInformation</code> class.
-    When <code>spark.resources.coordinateResourcesInStandalone</code> is off, the discovery script must assign different
-    resources for workers and drivers in client mode that run on the same host to avoid resource conflict.
   </td>
 </tr>
 <tr>
@@ -266,9 +264,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Path to resources file which is used to find various resources while worker starting up.
     The content of resources file should be formatted like <code>
-    [[{"id":{"componentName": "spark.worker","resourceName":"gpu"},"addresses":["0","1","2"]}]]</code>.
-    When <code>spark.resources.coordinateResourcesInStandalone</code> is off, resources file must assign different
-    resources for workers and drivers in client mode that run on the same host to avoid resource conflict.
+    [{"id":{"componentName": "spark.worker","resourceName":"gpu"},"addresses":["0","1","2"]}]</code>.
     If a particular resource is not found in the resources file, the discovery script would be used to
     find that resource. If the discovery script also does not find the resources, the worker will fail
     to start up.
@@ -346,9 +342,9 @@ Please make sure to have read the Custom Resource Scheduling and Configuration O
 
 Spark Standalone has 2 parts, the first is configuring the resources for the Worker, the second is the resource allocation for a specific application.
 
-The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup. Please take note of <code>spark.resources.coordinateResourcesInStandalone</code> as it indicates whether Spark should handle coordinating resources or if the user has made sure each Worker has separate resources. Also note that if using the resources coordination <code>spark.resources.dir</code> can be used to specify the directory used to do that coordination.
+The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup.
 
-The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resources.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers or Workers there are 2 ways to make sure the they don't use the same resources. The user can either configure <code>spark.resources.coordinateResourcesInStandalone</code> on and give all the Driver/Workers the same set or resources and Spark will handle make sure each Driver/Worker has separate resources, or the user can make sure the resources file or discovery script only returns resources the do not conflict with other Drivers or Workers running on the same node.
+The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resource.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers, please make sure the resources file or discovery script only returns resources that do not conflict with other Drivers running on the same node.
 
 Note, the user does not need to specify a discovery script when submitting an application as the Worker will start each Executor with the resources it allocates to it.
 

From 8cb23f0cb5b20b7e49fdd16c52d6451e901d9a7a Mon Sep 17 00:00:00 2001
From: Josh Rosen <rosenville@gmail.com>
Date: Mon, 2 Mar 2020 15:20:45 -0800
Subject: [PATCH 0198/1280] [SPARK-31003][TESTS] Fix incorrect uses of assume()
 in tests

### What changes were proposed in this pull request?

This patch fixes several incorrect uses of `assume()` in our tests.

If a call to `assume(condition)` fails then it will cause the test to be marked as skipped instead of failed: this feature allows test cases to be skipped if certain prerequisites are missing. For example, we use this to skip certain tests when running on Windows (or when Python dependencies are unavailable).

In contrast, `assert(condition)` will fail the test if the condition doesn't hold.

If `assume()` is accidentally substituted for `assert()`then the resulting test will be marked as skipped in cases where it should have failed, undermining the purpose of the test.

This patch fixes several such cases, replacing certain `assume()` calls with `assert()`.

Credit to ahirreddy for spotting this problem.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #27754 from JoshRosen/fix-assume-vs-assert.

Lead-authored-by: Josh Rosen <rosenville@gmail.com>
Co-authored-by: Josh Rosen <joshrosen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit f0010c81e2ef9b8859b39917bb62b48d739a4a22)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/expressions/OrderingSuite.scala      | 2 +-
 .../test/scala/org/apache/spark/sql/CachedTableSuite.scala  | 2 +-
 .../org/apache/spark/sql/execution/command/DDLSuite.scala   | 4 ++--
 .../scala/org/apache/spark/sql/internal/CatalogSuite.scala  | 6 +++---
 .../org/apache/spark/sql/sources/BucketedReadSuite.scala    | 2 +-
 .../org/apache/spark/sql/sources/BucketedWriteSuite.scala   | 2 +-
 .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala  | 6 +++---
 .../scala/org/apache/spark/sql/hive/test/TestHive.scala     | 2 +-
 .../sql/sources/BucketedReadWithHiveSupportSuite.scala      | 2 +-
 .../sql/sources/BucketedWriteWithHiveSupportSuite.scala     | 2 +-
 10 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
index 94e251d90bcfa..448890223c879 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/OrderingSuite.scala
@@ -106,7 +106,7 @@ class OrderingSuite extends SparkFunSuite with ExpressionEvalHelper {
           StructField("a", dataType, nullable = true) ::
             StructField("b", dataType, nullable = true) :: Nil)
         val maybeDataGenerator = RandomDataGenerator.forType(rowType, nullable = false)
-        assume(maybeDataGenerator.isDefined)
+        assert(maybeDataGenerator.isDefined)
         val randGenerator = maybeDataGenerator.get
         val toCatalyst = CatalystTypeConverters.createToCatalystConverter(rowType)
         for (_ <- 1 to 50) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index cd2c681dd7e0e..81893534fc2a3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -195,7 +195,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("SPARK-1669: cacheTable should be idempotent") {
-    assume(!spark.table("testData").logicalPlan.isInstanceOf[InMemoryRelation])
+    assert(!spark.table("testData").logicalPlan.isInstanceOf[InMemoryRelation])
 
     spark.catalog.cacheTable("testData")
     assertCached(spark.table("testData"))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 6c824c27e4551..5a67dce0ffe6a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -1033,7 +1033,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     df.write.insertInto("students")
     spark.catalog.cacheTable("students")
     checkAnswer(spark.table("students"), df)
-    assume(spark.catalog.isCached("students"), "bad test: table was not cached in the first place")
+    assert(spark.catalog.isCached("students"), "bad test: table was not cached in the first place")
     sql("ALTER TABLE students RENAME TO teachers")
     sql("CREATE TABLE students (age INT, name STRING) USING parquet")
     // Now we have both students and teachers.
@@ -1959,7 +1959,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     Seq("json", "parquet").foreach { format =>
       withTable("rectangles") {
         data.write.format(format).saveAsTable("rectangles")
-        assume(spark.table("rectangles").collect().nonEmpty,
+        assert(spark.table("rectangles").collect().nonEmpty,
           "bad test; table was empty to begin with")
 
         sql("TRUNCATE TABLE rectangles")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
index d6a1fde2147b1..4630a429830fb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
@@ -89,9 +89,9 @@ class CatalogSuite extends SharedSparkSession {
     val columns = dbName
       .map { db => spark.catalog.listColumns(db, tableName) }
       .getOrElse { spark.catalog.listColumns(tableName) }
-    assume(tableMetadata.schema.nonEmpty, "bad test")
-    assume(tableMetadata.partitionColumnNames.nonEmpty, "bad test")
-    assume(tableMetadata.bucketSpec.isDefined, "bad test")
+    assert(tableMetadata.schema.nonEmpty, "bad test")
+    assert(tableMetadata.partitionColumnNames.nonEmpty, "bad test")
+    assert(tableMetadata.bucketSpec.isDefined, "bad test")
     assert(columns.collect().map(_.name).toSet == tableMetadata.schema.map(_.name).toSet)
     val bucketColumnNames = tableMetadata.bucketSpec.map(_.bucketColumnNames).getOrElse(Nil).toSet
     columns.collect().foreach { col =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index c7266c886128c..b54f4f2295e57 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -42,7 +42,7 @@ import org.apache.spark.util.collection.BitSet
 class BucketedReadWithoutHiveSupportSuite extends BucketedReadSuite with SharedSparkSession {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
index 9713de988e379..a410f32d4af7e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedWriteSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
 class BucketedWriteWithoutHiveSupportSuite extends BucketedWriteSuite with SharedSparkSession {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "in-memory")
   }
 
   override protected def fileFormatsToTest: Seq[String] = Seq("parquet", "json")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 7adc68d548af2..08d2506376cd2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -981,8 +981,8 @@ class HiveDDLSuite
     val expectedSerdePropsString =
       expectedSerdeProps.map { case (k, v) => s"'$k'='$v'" }.mkString(", ")
     val oldPart = catalog.getPartition(TableIdentifier("boxes"), Map("width" -> "4"))
-    assume(oldPart.storage.serde != Some(expectedSerde), "bad test: serde was already set")
-    assume(oldPart.storage.properties.filterKeys(expectedSerdeProps.contains) !=
+    assert(oldPart.storage.serde != Some(expectedSerde), "bad test: serde was already set")
+    assert(oldPart.storage.properties.filterKeys(expectedSerdeProps.contains) !=
       expectedSerdeProps, "bad test: serde properties were already set")
     sql(s"""ALTER TABLE boxes PARTITION (width=4)
       |    SET SERDE '$expectedSerde'
@@ -1735,7 +1735,7 @@ class HiveDDLSuite
     Seq("json", "parquet").foreach { format =>
       withTable("rectangles") {
         data.write.format(format).saveAsTable("rectangles")
-        assume(spark.table("rectangles").collect().nonEmpty,
+        assert(spark.table("rectangles").collect().nonEmpty,
           "bad test; table was empty to begin with")
 
         sql("TRUNCATE TABLE rectangles")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 222244a04f5f5..868bc2712ee71 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -212,7 +212,7 @@ private[hive] class TestHiveSparkSession(
     }
   }
 
-  assume(sc.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+  assert(sc.conf.get(CATALOG_IMPLEMENTATION) == "hive")
 
   @transient
   override lazy val sharedState: TestHiveSharedState = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
index f277f99805a4a..35dab79ff6dff 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
@@ -23,6 +23,6 @@ import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 class BucketedReadWithHiveSupportSuite extends BucketedReadSuite with TestHiveSingleton {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala
index 454e2f65d5d88..bdbdcc2951072 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedWriteWithHiveSupportSuite.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 class BucketedWriteWithHiveSupportSuite extends BucketedWriteSuite with TestHiveSingleton {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
-    assume(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
+    assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")
   }
 
   override protected def fileFormatsToTest: Seq[String] = Seq("parquet", "orc")

From 597bbbbb5507448980e4fadbad85ffb104808081 Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Mon, 2 Mar 2020 16:04:00 -0800
Subject: [PATCH 0199/1280] [SPARK-30991] Refactor AQE readers and RDDs

### What changes were proposed in this pull request?
This PR combines `CustomShuffledRowRDD` and `LocalShuffledRowRDD` into `ShuffledRowRDD`, and creates `CustomShuffleReaderExec` to unify and replace all existing AQE readers: `CoalescedShuffleReaderExec`, `LocalShuffleReaderExec` and `SkewJoinShuffleReaderExec`.

### Why are the changes needed?
To reduce code redundancy.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Passed existing UTs.

Closes #27742 from maryannxue/aqe-readers.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 473a28c1d032993c7fa515b39f2cb1e3105d65d3)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../spark/sql/execution/ShuffledRowRDD.scala  | 142 +++++++++++-------
 .../spark/sql/execution/SparkPlanInfo.scala   |   2 +-
 .../adaptive/CustomShuffleReaderExec.scala    |  81 ++++++++++
 .../adaptive/CustomShuffledRowRDD.scala       | 113 --------------
 .../adaptive/LocalShuffledRowRDD.scala        | 112 --------------
 .../adaptive/OptimizeLocalShuffleReader.scala |  88 +++--------
 .../adaptive/OptimizeSkewedJoin.scala         |  72 +--------
 .../adaptive/ReduceNumShufflePartitions.scala |  49 +-----
 .../adaptive/ShufflePartitionsCoalescer.scala |  23 +--
 .../exchange/ShuffleExchangeExec.scala        |  12 +-
 .../ReduceNumShufflePartitionsSuite.scala     |  28 ++--
 .../ShufflePartitionsCoalescerSuite.scala     | 101 +++++++++----
 .../adaptive/AdaptiveQueryExecSuite.scala     |  23 +--
 13 files changed, 317 insertions(+), 529 deletions(-)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
index 4c19f95796d04..eb022596b1f74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
@@ -26,17 +26,28 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
 import org.apache.spark.sql.internal.SQLConf
 
+sealed trait ShufflePartitionSpec
+
+// A partition that reads data of one or more reducers, from `startReducerIndex` (inclusive) to
+// `endReducerIndex` (exclusive).
+case class CoalescedPartitionSpec(
+  startReducerIndex: Int, endReducerIndex: Int) extends ShufflePartitionSpec
+
+// A partition that reads partial data of one reducer, from `startMapIndex` (inclusive) to
+// `endMapIndex` (exclusive).
+case class PartialReducerPartitionSpec(
+  reducerIndex: Int, startMapIndex: Int, endMapIndex: Int) extends ShufflePartitionSpec
+
+// A partition that reads partial data of one mapper, from `startReducerIndex` (inclusive) to
+// `endReducerIndex` (exclusive).
+case class PartialMapperPartitionSpec(
+  mapIndex: Int, startReducerIndex: Int, endReducerIndex: Int) extends ShufflePartitionSpec
+
 /**
- * The [[Partition]] used by [[ShuffledRowRDD]]. A post-shuffle partition
- * (identified by `postShufflePartitionIndex`) contains a range of pre-shuffle partitions
- * (`startPreShufflePartitionIndex` to `endPreShufflePartitionIndex - 1`, inclusive).
+ * The [[Partition]] used by [[ShuffledRowRDD]].
  */
-private final class ShuffledRowRDDPartition(
-    val postShufflePartitionIndex: Int,
-    val startPreShufflePartitionIndex: Int,
-    val endPreShufflePartitionIndex: Int) extends Partition {
-  override val index: Int = postShufflePartitionIndex
-}
+private final case class ShuffledRowRDDPartition(
+  index: Int, spec: ShufflePartitionSpec) extends Partition
 
 /**
  * A dummy partitioner for use with records whose partition ids have been pre-computed (i.e. for
@@ -94,8 +105,7 @@ class CoalescedPartitioner(val parent: Partitioner, val partitionStartIndices: A
  * interfaces / internals.
  *
  * This RDD takes a [[ShuffleDependency]] (`dependency`),
- * and an optional array of partition start indices as input arguments
- * (`specifiedPartitionStartIndices`).
+ * and an array of [[ShufflePartitionSpec]] as input arguments.
  *
  * The `dependency` has the parent RDD of this RDD, which represents the dataset before shuffle
  * (i.e. map output). Elements of this RDD are (partitionId, Row) pairs.
@@ -103,79 +113,97 @@ class CoalescedPartitioner(val parent: Partitioner, val partitionStartIndices: A
  * `dependency.partitioner` is the original partitioner used to partition
  * map output, and `dependency.partitioner.numPartitions` is the number of pre-shuffle partitions
  * (i.e. the number of partitions of the map output).
- *
- * When `specifiedPartitionStartIndices` is defined, `specifiedPartitionStartIndices.length`
- * will be the number of post-shuffle partitions. For this case, the `i`th post-shuffle
- * partition includes `specifiedPartitionStartIndices[i]` to
- * `specifiedPartitionStartIndices[i+1] - 1` (inclusive).
- *
- * When `specifiedPartitionStartIndices` is not defined, there will be
- * `dependency.partitioner.numPartitions` post-shuffle partitions. For this case,
- * a post-shuffle partition is created for every pre-shuffle partition.
  */
 class ShuffledRowRDD(
     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
     metrics: Map[String, SQLMetric],
-    specifiedPartitionStartIndices: Option[Array[Int]] = None)
+    partitionSpecs: Array[ShufflePartitionSpec])
   extends RDD[InternalRow](dependency.rdd.context, Nil) {
 
+  def this(
+      dependency: ShuffleDependency[Int, InternalRow, InternalRow],
+      metrics: Map[String, SQLMetric]) = {
+    this(dependency, metrics,
+      Array.tabulate(dependency.partitioner.numPartitions)(i => CoalescedPartitionSpec(i, i + 1)))
+  }
+
   if (SQLConf.get.fetchShuffleBlocksInBatchEnabled) {
     dependency.rdd.context.setLocalProperty(
       SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY, "true")
   }
 
-  private[this] val numPreShufflePartitions = dependency.partitioner.numPartitions
-
-  private[this] val partitionStartIndices: Array[Int] = specifiedPartitionStartIndices match {
-    case Some(indices) => indices
-    case None =>
-      // When specifiedPartitionStartIndices is not defined, every post-shuffle partition
-      // corresponds to a pre-shuffle partition.
-      (0 until numPreShufflePartitions).toArray
-  }
-
-  private[this] val part: Partitioner =
-    new CoalescedPartitioner(dependency.partitioner, partitionStartIndices)
-
   override def getDependencies: Seq[Dependency[_]] = List(dependency)
 
-  override val partitioner: Option[Partitioner] = Some(part)
+  override val partitioner: Option[Partitioner] =
+    if (partitionSpecs.forall(_.isInstanceOf[CoalescedPartitionSpec])) {
+      val indices = partitionSpecs.map(_.asInstanceOf[CoalescedPartitionSpec].startReducerIndex)
+      // TODO this check is based on assumptions of callers' behavior but is sufficient for now.
+      if (indices.toSet.size == partitionSpecs.length) {
+        Some(new CoalescedPartitioner(dependency.partitioner, indices))
+      } else {
+        None
+      }
+    } else {
+      None
+    }
 
   override def getPartitions: Array[Partition] = {
-    assert(partitionStartIndices.length == part.numPartitions)
-    Array.tabulate[Partition](partitionStartIndices.length) { i =>
-      val startIndex = partitionStartIndices(i)
-      val endIndex =
-        if (i < partitionStartIndices.length - 1) {
-          partitionStartIndices(i + 1)
-        } else {
-          numPreShufflePartitions
-        }
-      new ShuffledRowRDDPartition(i, startIndex, endIndex)
+    Array.tabulate[Partition](partitionSpecs.length) { i =>
+      ShuffledRowRDDPartition(i, partitionSpecs(i))
     }
   }
 
   override def getPreferredLocations(partition: Partition): Seq[String] = {
     val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    val dep = dependencies.head.asInstanceOf[ShuffleDependency[_, _, _]]
-    tracker.getPreferredLocationsForShuffle(dep, partition.index)
+    partition.asInstanceOf[ShuffledRowRDDPartition].spec match {
+      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+        // TODO order by partition size.
+        startReducerIndex.until(endReducerIndex).flatMap { reducerIndex =>
+          tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
+        }
+
+      case PartialReducerPartitionSpec(_, startMapIndex, endMapIndex) =>
+        tracker.getMapLocation(dependency, startMapIndex, endMapIndex)
+
+      case PartialMapperPartitionSpec(mapIndex, _, _) =>
+        tracker.getMapLocation(dependency, mapIndex, mapIndex + 1)
+    }
   }
 
   override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val shuffledRowPartition = split.asInstanceOf[ShuffledRowRDDPartition]
     val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
     // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
     // as well as the `tempMetrics` for basic shuffle metrics.
     val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-    // The range of pre-shuffle partitions that we are fetching at here is
-    // [startPreShufflePartitionIndex, endPreShufflePartitionIndex - 1].
-    val reader =
-      SparkEnv.get.shuffleManager.getReader(
-        dependency.shuffleHandle,
-        shuffledRowPartition.startPreShufflePartitionIndex,
-        shuffledRowPartition.endPreShufflePartitionIndex,
-        context,
-        sqlMetricsReporter)
+    val reader = split.asInstanceOf[ShuffledRowRDDPartition].spec match {
+      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
+        SparkEnv.get.shuffleManager.getReader(
+          dependency.shuffleHandle,
+          startReducerIndex,
+          endReducerIndex,
+          context,
+          sqlMetricsReporter)
+
+      case PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex) =>
+        SparkEnv.get.shuffleManager.getReaderForRange(
+          dependency.shuffleHandle,
+          startMapIndex,
+          endMapIndex,
+          reducerIndex,
+          reducerIndex + 1,
+          context,
+          sqlMetricsReporter)
+
+      case PartialMapperPartitionSpec(mapIndex, startReducerIndex, endReducerIndex) =>
+        SparkEnv.get.shuffleManager.getReaderForRange(
+          dependency.shuffleHandle,
+          mapIndex,
+          mapIndex + 1,
+          startReducerIndex,
+          endReducerIndex,
+          context,
+          sqlMetricsReporter)
+    }
     reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
index 5b72ec058e127..357820a9d63d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, LocalShuffleReaderExec, QueryStageExec}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.execution.metric.SQLMetricInfo
 import org.apache.spark.sql.internal.SQLConf
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
new file mode 100644
index 0000000000000..be372bbfcf700
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec}
+
+
+/**
+ * A wrapper of shuffle query stage, which follows the given partition arrangement.
+ *
+ * @param child           It is usually `ShuffleQueryStageExec`, but can be the shuffle exchange
+ *                        node during canonicalization.
+ * @param partitionSpecs  The partition specs that defines the arrangement.
+ * @param description     The string description of this shuffle reader.
+ */
+case class CustomShuffleReaderExec private(
+    child: SparkPlan,
+    partitionSpecs: Array[ShufflePartitionSpec],
+    description: String) extends UnaryExecNode {
+
+  override def output: Seq[Attribute] = child.output
+  override lazy val outputPartitioning: Partitioning = {
+    // If it is a local shuffle reader with one mapper per task, then the output partitioning is
+    // the same as the plan before shuffle.
+    // TODO this check is based on assumptions of callers' behavior but is sufficient for now.
+    if (partitionSpecs.forall(_.isInstanceOf[PartialMapperPartitionSpec]) &&
+        partitionSpecs.map(_.asInstanceOf[PartialMapperPartitionSpec].mapIndex).toSet.size ==
+          partitionSpecs.length) {
+      child match {
+        case ShuffleQueryStageExec(_, s: ShuffleExchangeExec) =>
+          s.child.outputPartitioning
+        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeExec)) =>
+          s.child.outputPartitioning match {
+            case e: Expression => r.updateAttr(e).asInstanceOf[Partitioning]
+            case other => other
+          }
+        case _ =>
+          throw new IllegalStateException("operating on canonicalization plan")
+      }
+    } else {
+      UnknownPartitioning(partitionSpecs.length)
+    }
+  }
+
+  override def stringArgs: Iterator[Any] = Iterator(description)
+
+  private var cachedShuffleRDD: RDD[InternalRow] = null
+
+  override protected def doExecute(): RDD[InternalRow] = {
+    if (cachedShuffleRDD == null) {
+      cachedShuffleRDD = child match {
+        case stage: ShuffleQueryStageExec =>
+          new ShuffledRowRDD(
+            stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs)
+        case _ =>
+          throw new IllegalStateException("operating on canonicalization plan")
+      }
+    }
+    cachedShuffleRDD
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala
deleted file mode 100644
index 5aba57443d632..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffledRowRDD.scala
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.adaptive
-
-import org.apache.spark.{Dependency, MapOutputTrackerMaster, Partition, ShuffleDependency, SparkEnv, TaskContext}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
-
-sealed trait ShufflePartitionSpec
-
-// A partition that reads data of one reducer.
-case class SinglePartitionSpec(reducerIndex: Int) extends ShufflePartitionSpec
-
-// A partition that reads data of multiple reducers, from `startReducerIndex` (inclusive) to
-// `endReducerIndex` (exclusive).
-case class CoalescedPartitionSpec(
-    startReducerIndex: Int, endReducerIndex: Int) extends ShufflePartitionSpec
-
-// A partition that reads partial data of one reducer, from `startMapIndex` (inclusive) to
-// `endMapIndex` (exclusive).
-case class PartialPartitionSpec(
-    reducerIndex: Int, startMapIndex: Int, endMapIndex: Int) extends ShufflePartitionSpec
-
-private final case class CustomShufflePartition(
-    index: Int, spec: ShufflePartitionSpec) extends Partition
-
-// TODO: merge this with `ShuffledRowRDD`, and replace `LocalShuffledRowRDD` with this RDD.
-class CustomShuffledRowRDD(
-    var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
-    metrics: Map[String, SQLMetric],
-    partitionSpecs: Array[ShufflePartitionSpec])
-  extends RDD[InternalRow](dependency.rdd.context, Nil) {
-
-  override def getDependencies: Seq[Dependency[_]] = List(dependency)
-
-  override def clearDependencies() {
-    super.clearDependencies()
-    dependency = null
-  }
-
-  override def getPartitions: Array[Partition] = {
-    Array.tabulate[Partition](partitionSpecs.length) { i =>
-      CustomShufflePartition(i, partitionSpecs(i))
-    }
-  }
-
-  override def getPreferredLocations(partition: Partition): Seq[String] = {
-    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    partition.asInstanceOf[CustomShufflePartition].spec match {
-      case SinglePartitionSpec(reducerIndex) =>
-        tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
-
-      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
-        startReducerIndex.until(endReducerIndex).flatMap { reducerIndex =>
-          tracker.getPreferredLocationsForShuffle(dependency, reducerIndex)
-        }
-
-      case PartialPartitionSpec(_, startMapIndex, endMapIndex) =>
-        tracker.getMapLocation(dependency, startMapIndex, endMapIndex)
-    }
-  }
-
-  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
-    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
-    // as well as the `tempMetrics` for basic shuffle metrics.
-    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-    val reader = split.asInstanceOf[CustomShufflePartition].spec match {
-      case SinglePartitionSpec(reducerIndex) =>
-        SparkEnv.get.shuffleManager.getReader(
-          dependency.shuffleHandle,
-          reducerIndex,
-          reducerIndex + 1,
-          context,
-          sqlMetricsReporter)
-
-      case CoalescedPartitionSpec(startReducerIndex, endReducerIndex) =>
-        SparkEnv.get.shuffleManager.getReader(
-          dependency.shuffleHandle,
-          startReducerIndex,
-          endReducerIndex,
-          context,
-          sqlMetricsReporter)
-
-      case PartialPartitionSpec(reducerIndex, startMapIndex, endMapIndex) =>
-        SparkEnv.get.shuffleManager.getReaderForRange(
-          dependency.shuffleHandle,
-          startMapIndex,
-          endMapIndex,
-          reducerIndex,
-          reducerIndex + 1,
-          context,
-          sqlMetricsReporter)
-    }
-    reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala
deleted file mode 100644
index 19b78f5e36c9b..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/LocalShuffledRowRDD.scala
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution.adaptive
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark._
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.metric.{SQLMetric, SQLShuffleReadMetricsReporter}
-
-/**
- * The [[Partition]] used by [[LocalShuffledRowRDD]].
- * @param mapIndex the index of mapper.
- * @param startPartition the start partition ID in mapIndex mapper.
- * @param endPartition the end partition ID in mapIndex mapper.
- */
-private final class LocalShuffledRowRDDPartition(
-    override val index: Int,
-    val mapIndex: Int,
-    val startPartition: Int,
-    val endPartition: Int) extends Partition {
-}
-
-/**
- * This is a specialized version of [[org.apache.spark.sql.execution.ShuffledRowRDD]]. This is used
- * in Spark SQL adaptive execution when a shuffle join is converted to broadcast join at runtime
- * because the map output of one input table is small enough for broadcast. This RDD represents the
- * data of another input table of the join that reads from shuffle. Each partition of the RDD reads
- * the whole data from just one mapper output locally. So actually there is no data transferred
- * from the network.
- *
- * This RDD takes a [[ShuffleDependency]] (`dependency`).
- *
- * The `dependency` has the parent RDD of this RDD, which represents the dataset before shuffle
- * (i.e. map output). Elements of this RDD are (partitionId, Row) pairs.
- * Partition ids should be in the range [0, numPartitions - 1].
- * `dependency.partitioner.numPartitions` is the number of pre-shuffle partitions. (i.e. the number
- * of partitions of the map output). The post-shuffle partition number is the same to the parent
- * RDD's partition number.
- *
- * `partitionStartIndicesPerMapper` specifies how to split the shuffle blocks of each mapper into
- * one or more partitions. For a mapper `i`, the `j`th partition includes shuffle blocks from
- * `partitionStartIndicesPerMapper[i][j]` to `partitionStartIndicesPerMapper[i][j+1]` (exclusive).
- */
-class LocalShuffledRowRDD(
-     var dependency: ShuffleDependency[Int, InternalRow, InternalRow],
-     metrics: Map[String, SQLMetric],
-     partitionStartIndicesPerMapper: Array[Array[Int]])
-  extends RDD[InternalRow](dependency.rdd.context, Nil) {
-
-  private[this] val numReducers = dependency.partitioner.numPartitions
-  private[this] val numMappers = dependency.rdd.partitions.length
-
-  override def getDependencies: Seq[Dependency[_]] = List(dependency)
-
-  override def getPartitions: Array[Partition] = {
-    val partitions = ArrayBuffer[LocalShuffledRowRDDPartition]()
-    for (mapIndex <- 0 until numMappers) {
-      (partitionStartIndicesPerMapper(mapIndex) :+ numReducers).sliding(2, 1).foreach {
-        case Array(start, end) =>
-          partitions += new LocalShuffledRowRDDPartition(partitions.length, mapIndex, start, end)
-      }
-    }
-    partitions.toArray
-  }
-
-  override def getPreferredLocations(partition: Partition): Seq[String] = {
-    val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster]
-    tracker.getMapLocation(dependency, partition.index, partition.index + 1)
-  }
-
-  override def compute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
-    val localRowPartition = split.asInstanceOf[LocalShuffledRowRDDPartition]
-    val mapIndex = localRowPartition.mapIndex
-    val tempMetrics = context.taskMetrics().createTempShuffleReadMetrics()
-    // `SQLShuffleReadMetricsReporter` will update its own metrics for SQL exchange operator,
-    // as well as the `tempMetrics` for basic shuffle metrics.
-    val sqlMetricsReporter = new SQLShuffleReadMetricsReporter(tempMetrics, metrics)
-
-    val reader = SparkEnv.get.shuffleManager.getReaderForRange(
-      dependency.shuffleHandle,
-      mapIndex,
-      mapIndex + 1,
-      localRowPartition.startPartition,
-      localRowPartition.endPartition,
-      context,
-      sqlMetricsReporter)
-    reader.read().asInstanceOf[Iterator[Product2[Int, InternalRow]]].map(_._2)
-  }
-
-  override def clearDependencies() {
-    super.clearDependencies()
-    dependency = null
-  }
-}
-
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index e95441e28aafe..e4417632b2ee4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -17,13 +17,9 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{SparkPlan, UnaryExecNode}
-import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReusedExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildLeft, BuildRight, BuildSide}
 import org.apache.spark.sql.internal.SQLConf
 
@@ -67,28 +63,35 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
     }
   }
 
-  private def createLocalReader(plan: SparkPlan): LocalShuffleReaderExec = {
+  private def createLocalReader(plan: SparkPlan): CustomShuffleReaderExec = {
     plan match {
-      case c @ CoalescedShuffleReaderExec(s: ShuffleQueryStageExec, _) =>
-        LocalShuffleReaderExec(
-          s, getPartitionStartIndices(s, Some(c.partitionStartIndices.length)))
+      case c @ CustomShuffleReaderExec(s: ShuffleQueryStageExec, _, _) =>
+        CustomShuffleReaderExec(
+          s, getPartitionSpecs(s, Some(c.partitionSpecs.length)), LOCAL_SHUFFLE_READER_DESCRIPTION)
       case s: ShuffleQueryStageExec =>
-        LocalShuffleReaderExec(s, getPartitionStartIndices(s, None))
+        CustomShuffleReaderExec(s, getPartitionSpecs(s, None), LOCAL_SHUFFLE_READER_DESCRIPTION)
     }
   }
 
   // TODO: this method assumes all shuffle blocks are the same data size. We should calculate the
   //       partition start indices based on block size to avoid data skew.
-  private def getPartitionStartIndices(
+  private def getPartitionSpecs(
       shuffleStage: ShuffleQueryStageExec,
-      advisoryParallelism: Option[Int]): Array[Array[Int]] = {
+      advisoryParallelism: Option[Int]): Array[ShufflePartitionSpec] = {
     val shuffleDep = shuffleStage.shuffle.shuffleDependency
     val numReducers = shuffleDep.partitioner.numPartitions
     val expectedParallelism = advisoryParallelism.getOrElse(numReducers)
     val numMappers = shuffleDep.rdd.getNumPartitions
-    Array.fill(numMappers) {
+    val splitPoints = if (numMappers == 0) {
+      Array.empty
+    } else {
       equallyDivide(numReducers, math.max(1, expectedParallelism / numMappers)).toArray
     }
+    (0 until numMappers).flatMap { mapIndex =>
+      (splitPoints :+ numReducers).sliding(2).map {
+        case Array(start, end) => PartialMapperPartitionSpec(mapIndex, start, end)
+      }
+    }.toArray
   }
 
   /**
@@ -120,6 +123,8 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
 
 object OptimizeLocalShuffleReader {
 
+  val LOCAL_SHUFFLE_READER_DESCRIPTION: String = "local"
+
   object BroadcastJoinWithShuffleLeft {
     def unapply(plan: SparkPlan): Option[(SparkPlan, BuildSide)] = plan match {
       case join: BroadcastHashJoinExec if canUseLocalShuffleReader(join.left) =>
@@ -138,60 +143,7 @@ object OptimizeLocalShuffleReader {
 
   def canUseLocalShuffleReader(plan: SparkPlan): Boolean = plan match {
     case s: ShuffleQueryStageExec => s.shuffle.canChangeNumPartitions
-    case CoalescedShuffleReaderExec(s: ShuffleQueryStageExec, _) => s.shuffle.canChangeNumPartitions
+    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, _, _) => s.shuffle.canChangeNumPartitions
     case _ => false
   }
 }
-
-/**
- * A wrapper of shuffle query stage, which submits one or more reduce tasks per mapper to read the
- * shuffle files written by one mapper. By doing this, it's very likely to read the shuffle files
- * locally, as the shuffle files that a reduce task needs to read are in one node.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
- *              canonicalization.
- * @param partitionStartIndicesPerMapper A mapper usually writes many shuffle blocks, and it's
- *                                       better to launch multiple tasks to read shuffle blocks of
- *                                       one mapper. This array contains the partition start
- *                                       indices for each mapper.
- */
-case class LocalShuffleReaderExec(
-    child: SparkPlan,
-    partitionStartIndicesPerMapper: Array[Array[Int]]) extends UnaryExecNode {
-
-  override def output: Seq[Attribute] = child.output
-
-  override lazy val outputPartitioning: Partitioning = {
-    // when we read one mapper per task, then the output partitioning is the same as the plan
-    // before shuffle.
-    if (partitionStartIndicesPerMapper.forall(_.length == 1)) {
-      child match {
-        case ShuffleQueryStageExec(_, s: ShuffleExchangeExec) =>
-          s.child.outputPartitioning
-        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeExec)) =>
-          s.child.outputPartitioning match {
-            case e: Expression => r.updateAttr(e).asInstanceOf[Partitioning]
-            case other => other
-          }
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    } else {
-      UnknownPartitioning(partitionStartIndicesPerMapper.map(_.length).sum)
-    }
-  }
-
-  private var cachedShuffleRDD: RDD[InternalRow] = null
-
-  override protected def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createLocalShuffleRDD(partitionStartIndicesPerMapper)
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    }
-    cachedShuffleRDD
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index dd238fe7785bb..979fee1490598 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -23,11 +23,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.commons.io.FileUtils
 
 import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans._
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ShuffleExchangeExec}
@@ -221,7 +217,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
               getMapStartIndices(left, partitionIndex, leftTargetSize),
               getNumMappers(left))
           } else {
-            Seq(SinglePartitionSpec(partitionIndex))
+            Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
           }
 
           val rightParts = if (isRightSkew) {
@@ -231,7 +227,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
               getMapStartIndices(right, partitionIndex, rightTargetSize),
               getNumMappers(right))
           } else {
-            Seq(SinglePartitionSpec(partitionIndex))
+            Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
           }
 
           for {
@@ -259,9 +255,9 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       logDebug("number of skewed partitions: " +
         s"left ${leftSkewDesc.numPartitions}, right ${rightSkewDesc.numPartitions}")
       if (leftSkewDesc.numPartitions > 0 || rightSkewDesc.numPartitions > 0) {
-        val newLeft = SkewJoinShuffleReaderExec(
+        val newLeft = CustomShuffleReaderExec(
           left, leftSidePartitions.toArray, leftSkewDesc.toString)
-        val newRight = SkewJoinShuffleReaderExec(
+        val newRight = CustomShuffleReaderExec(
           right, rightSidePartitions.toArray, rightSkewDesc.toString)
         smj.copy(
           left = s1.copy(child = newLeft), right = s2.copy(child = newRight), isSkewJoin = true)
@@ -277,36 +273,21 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     assert(nonSkewPartitionIndices.nonEmpty)
     val shouldCoalesce = conf.getConf(SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED)
     if (!shouldCoalesce || nonSkewPartitionIndices.length == 1) {
-      Seq(SinglePartitionSpec(nonSkewPartitionIndices.head))
+      nonSkewPartitionIndices.map(i => CoalescedPartitionSpec(i, i + 1))
     } else {
-      val startIndices = ShufflePartitionsCoalescer.coalescePartitions(
+      ShufflePartitionsCoalescer.coalescePartitions(
         Array(leftStats, rightStats),
         firstPartitionIndex = nonSkewPartitionIndices.head,
         // `lastPartitionIndex` is exclusive.
         lastPartitionIndex = nonSkewPartitionIndices.last + 1,
         advisoryTargetSize = conf.targetPostShuffleInputSize)
-      startIndices.indices.map { i =>
-        val startIndex = startIndices(i)
-        val endIndex = if (i == startIndices.length - 1) {
-          // `endIndex` is exclusive.
-          nonSkewPartitionIndices.last + 1
-        } else {
-          startIndices(i + 1)
-        }
-        // Do not create `CoalescedPartitionSpec` if only need to read a singe partition.
-        if (startIndex + 1 == endIndex) {
-          SinglePartitionSpec(startIndex)
-        } else {
-          CoalescedPartitionSpec(startIndex, endIndex)
-        }
-      }
     }
   }
 
   private def createSkewPartitions(
       reducerIndex: Int,
       mapStartIndices: Array[Int],
-      numMappers: Int): Seq[PartialPartitionSpec] = {
+      numMappers: Int): Seq[PartialReducerPartitionSpec] = {
     mapStartIndices.indices.map { i =>
       val startMapIndex = mapStartIndices(i)
       val endMapIndex = if (i == mapStartIndices.length - 1) {
@@ -314,7 +295,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       } else {
         mapStartIndices(i + 1)
       }
-      PartialPartitionSpec(reducerIndex, startMapIndex, endMapIndex)
+      PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex)
     }
   }
 
@@ -387,40 +368,3 @@ private class SkewDesc {
     }
   }
 }
-
-/**
- * A wrapper of shuffle query stage, which follows the given partition arrangement.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
- *              canonicalization.
- * @param partitionSpecs The partition specs that defines the arrangement.
- * @param skewDesc The description of the skewed partitions.
- */
-case class SkewJoinShuffleReaderExec(
-    child: SparkPlan,
-    partitionSpecs: Array[ShufflePartitionSpec],
-    skewDesc: String) extends UnaryExecNode {
-
-  override def output: Seq[Attribute] = child.output
-
-  override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(partitionSpecs.length)
-  }
-
-  override def stringArgs: Iterator[Any] = Iterator(skewDesc)
-
-  private var cachedShuffleRDD: RDD[InternalRow] = null
-
-  override protected def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          new CustomShuffledRowRDD(
-            stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs)
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    }
-    cachedShuffleRDD
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
index 5bbcb14e008d8..767a4b2cfee6e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
@@ -18,12 +18,8 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.MapOutputStatistics
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.{ShuffledRowRDD, SparkPlan, UnaryExecNode}
+import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -31,12 +27,14 @@ import org.apache.spark.sql.internal.SQLConf
  * avoid many small reduce tasks that hurt performance.
  */
 case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
+  import ReduceNumShufflePartitions._
 
   override def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.reducePostShufflePartitionsEnabled) {
       return plan
     }
-    if (!plan.collectLeaves().forall(_.isInstanceOf[QueryStageExec])) {
+    if (!plan.collectLeaves().forall(_.isInstanceOf[QueryStageExec])
+        || plan.find(_.isInstanceOf[CustomShuffleReaderExec]).isDefined) {
       // If not all leaf nodes are query stages, it's not safe to reduce the number of
       // shuffle partitions, because we may break the assumption that all children of a spark plan
       // have same number of output partitions.
@@ -44,8 +42,6 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
     }
 
     def collectShuffleStages(plan: SparkPlan): Seq[ShuffleQueryStageExec] = plan match {
-      case _: LocalShuffleReaderExec => Nil
-      case _: SkewJoinShuffleReaderExec => Nil
       case stage: ShuffleQueryStageExec => Seq(stage)
       case _ => plan.children.flatMap(collectShuffleStages)
     }
@@ -70,7 +66,7 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
       val distinctNumPreShufflePartitions =
         validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
       if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1) {
-        val partitionStartIndices = ShufflePartitionsCoalescer.coalescePartitions(
+        val partitionSpecs = ShufflePartitionsCoalescer.coalescePartitions(
           validMetrics.toArray,
           firstPartitionIndex = 0,
           lastPartitionIndex = distinctNumPreShufflePartitions.head,
@@ -83,7 +79,7 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
           // `partitionStartIndices`, so that all the leaf shuffles in a stage have the same
           // number of output partitions.
           case stage: ShuffleQueryStageExec if stageIds.contains(stage.id) =>
-            CoalescedShuffleReaderExec(stage, partitionStartIndices)
+            CustomShuffleReaderExec(stage, partitionSpecs, COALESCED_SHUFFLE_READER_DESCRIPTION)
         }
       } else {
         plan
@@ -92,35 +88,6 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
   }
 }
 
-/**
- * A wrapper of shuffle query stage, which submits fewer reduce task as one reduce task may read
- * multiple shuffle partitions. This can avoid many small reduce tasks that hurt performance.
- *
- * @param child It's usually `ShuffleQueryStageExec`, but can be the shuffle exchange node during
- *              canonicalization.
- * @param partitionStartIndices The start partition indices for the coalesced partitions.
- */
-case class CoalescedShuffleReaderExec(
-    child: SparkPlan,
-    partitionStartIndices: Array[Int]) extends UnaryExecNode {
-
-  override def output: Seq[Attribute] = child.output
-
-  override def outputPartitioning: Partitioning = {
-    UnknownPartitioning(partitionStartIndices.length)
-  }
-
-  private var cachedShuffleRDD: ShuffledRowRDD = null
-
-  override protected def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          stage.shuffle.createShuffledRDD(Some(partitionStartIndices))
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
-    }
-    cachedShuffleRDD
-  }
+object ReduceNumShufflePartitions {
+  val COALESCED_SHUFFLE_READER_DESCRIPTION = "coalesced"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
index 18f0585524aa2..c3b8bf6c757f8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
@@ -21,11 +21,12 @@ import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark.MapOutputStatistics
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.execution.{CoalescedPartitionSpec, ShufflePartitionSpec}
 
 object ShufflePartitionsCoalescer extends Logging {
 
   /**
-   * Coalesce the same range of partitions (`firstPartitionIndex`` to `lastPartitionIndex`, the
+   * Coalesce the same range of partitions (`firstPartitionIndex` to `lastPartitionIndex`, the
    * start is inclusive and the end is exclusive) from multiple shuffles. This method assumes that
    * all the shuffles have the same number of partitions, and the partitions of same index will be
    * read together by one task.
@@ -46,15 +47,17 @@ object ShufflePartitionsCoalescer extends Logging {
    *  - coalesced partition 2: shuffle partition 2 (size 170 MiB)
    *  - coalesced partition 3: shuffle partition 3 and 4 (size 50 MiB)
    *
-   *  @return An array of partition indices which represents the coalesced partitions. For example,
-   *          [0, 2, 3] means 3 coalesced partitions: [0, 2), [2, 3), [3, lastPartitionIndex]
+   *  @return An array of [[CoalescedPartitionSpec]]s. For example, if partitions [0, 1, 2, 3, 4]
+   *          split at indices [0, 2, 3], the returned partition specs will be:
+   *          CoalescedPartitionSpec(0, 2), CoalescedPartitionSpec(2, 3) and
+   *          CoalescedPartitionSpec(3, 5).
    */
   def coalescePartitions(
       mapOutputStatistics: Array[MapOutputStatistics],
       firstPartitionIndex: Int,
       lastPartitionIndex: Int,
       advisoryTargetSize: Long,
-      minNumPartitions: Int = 1): Array[Int] = {
+      minNumPartitions: Int = 1): Array[ShufflePartitionSpec] = {
     // If `minNumPartitions` is very large, it is possible that we need to use a value less than
     // `advisoryTargetSize` as the target size of a coalesced task.
     val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum
@@ -82,8 +85,8 @@ object ShufflePartitionsCoalescer extends Logging {
       "There should be only one distinct value of the number of shuffle partitions " +
         "among registered Exchange operators.")
 
-    val splitPoints = ArrayBuffer[Int]()
-    splitPoints += firstPartitionIndex
+    val partitionSpecs = ArrayBuffer[CoalescedPartitionSpec]()
+    var latestSplitPoint = firstPartitionIndex
     var coalescedSize = 0L
     var i = firstPartitionIndex
     while (i < lastPartitionIndex) {
@@ -97,8 +100,9 @@ object ShufflePartitionsCoalescer extends Logging {
 
       // If including the `totalSizeOfCurrentPartition` would exceed the target size, then start a
       // new coalesced partition.
-      if (i > firstPartitionIndex && coalescedSize + totalSizeOfCurrentPartition > targetSize) {
-        splitPoints += i
+      if (i > latestSplitPoint && coalescedSize + totalSizeOfCurrentPartition > targetSize) {
+        partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, i)
+        latestSplitPoint = i
         // reset postShuffleInputSize.
         coalescedSize = totalSizeOfCurrentPartition
       } else {
@@ -106,7 +110,8 @@ object ShufflePartitionsCoalescer extends Logging {
       }
       i += 1
     }
+    partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, lastPartitionIndex)
 
-    splitPoints.toArray
+    partitionSpecs.toArray
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index 4b08da043b83e..b06742e8470c7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -34,7 +34,6 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Div
 import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.LocalShuffledRowRDD
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
@@ -88,15 +87,6 @@ case class ShuffleExchangeExec(
       writeMetrics)
   }
 
-  def createShuffledRDD(partitionStartIndices: Option[Array[Int]]): ShuffledRowRDD = {
-    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionStartIndices)
-  }
-
-  def createLocalShuffleRDD(
-      partitionStartIndicesPerMapper: Array[Array[Int]]): LocalShuffledRowRDD = {
-    new LocalShuffledRowRDD(shuffleDependency, readMetrics, partitionStartIndicesPerMapper)
-  }
-
   /**
    * Caches the created ShuffleRowRDD so we can reuse that.
    */
@@ -105,7 +95,7 @@ case class ShuffleExchangeExec(
   protected override def doExecute(): RDD[InternalRow] = attachTree(this, "execute") {
     // Returns the same ShuffleRowRDD if this plan is used by multiple plans.
     if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = createShuffledRDD(None)
+      cachedShuffleRDD = new ShuffledRowRDD(shuffleDependency, readMetrics)
     }
     cachedShuffleRDD
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
index 5565a0dd01840..a32b684bc85c8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
@@ -23,7 +23,8 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql._
 import org.apache.spark.sql.execution.adaptive._
-import org.apache.spark.sql.execution.adaptive.CoalescedShuffleReaderExec
+import org.apache.spark.sql.execution.adaptive.CustomShuffleReaderExec
+import org.apache.spark.sql.execution.adaptive.ReduceNumShufflePartitions.COALESCED_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -107,7 +108,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = agg.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 1)
         minNumPostShufflePartitions match {
@@ -154,7 +155,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = join.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 2)
         minNumPostShufflePartitions match {
@@ -206,7 +207,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = join.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 2)
         minNumPostShufflePartitions match {
@@ -258,7 +259,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         val finalPlan = join.queryExecution.executedPlan
           .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
         val shuffleReaders = finalPlan.collect {
-          case reader: CoalescedShuffleReaderExec => reader
+          case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
         }
         assert(shuffleReaders.length === 2)
         minNumPostShufflePartitions match {
@@ -301,7 +302,7 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
           val finalPlan = join.queryExecution.executedPlan
             .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
           val shuffleReaders = finalPlan.collect {
-            case reader: CoalescedShuffleReaderExec => reader
+            case r @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => r
           }
           assert(shuffleReaders.length === 0)
         } finally {
@@ -329,7 +330,10 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
       assert(finalPlan.collect {
         case ShuffleQueryStageExec(_, r: ReusedExchangeExec) => r
       }.length == 2)
-      assert(finalPlan.collect { case p: CoalescedShuffleReaderExec => p }.length == 3)
+      assert(
+        finalPlan.collect {
+          case p @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => p
+        }.length == 3)
 
 
       // test case 2: a query stage has 2 parent stages.
@@ -377,7 +381,10 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         Seq(0, 1, 2).map(i => Row(i)))
       val finalPlan = resultDf.queryExecution.executedPlan
         .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
-      assert(finalPlan.collect { case p: CoalescedShuffleReaderExec => p }.length == 0)
+      assert(
+        finalPlan.collect {
+          case p @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => p
+        }.isEmpty)
     }
     withSparkSession(test, 200, None)
   }
@@ -395,7 +402,10 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         .asInstanceOf[AdaptiveSparkPlanExec].executedPlan
       // As the pre-shuffle partition number are different, we will skip reducing
       // the shuffle partition numbers.
-      assert(finalPlan.collect { case p: CoalescedShuffleReaderExec => p }.length == 0)
+      assert(
+        finalPlan.collect {
+          case p @ CustomShuffleReaderExec(_, _, COALESCED_SHUFFLE_READER_DESCRIPTION) => p
+        }.isEmpty)
     }
     withSparkSession(test, 100, None)
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
index fcfde83b2ffd5..0befa0640efcc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
@@ -24,7 +24,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
 
   private def checkEstimation(
       bytesByPartitionIdArray: Array[Array[Long]],
-      expectedPartitionStartIndices: Array[Int],
+      expectedPartitionStartIndices: Array[CoalescedPartitionSpec],
       targetSize: Long,
       minNumPartitions: Int = 1): Unit = {
     val mapOutputStatistics = bytesByPartitionIdArray.zipWithIndex.map {
@@ -46,44 +46,54 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
     {
       // All bytes per partition are 0.
       val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // Some bytes per partition are 0 and total size is less than the target size.
       // 1 coalesced partition is expected.
       val bytesByPartitionId = Array[Long](10, 0, 20, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
-      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // 2 coalesced partitions are expected.
       val bytesByPartitionId = Array[Long](10, 0, 90, 20, 0)
-      val expectedPartitionStartIndices = Array[Int](0, 3)
-      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // There are a few large shuffle partitions.
       val bytesByPartitionId = Array[Long](110, 10, 100, 110, 0)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // All shuffle partitions are larger than the targeted size.
       val bytesByPartitionId = Array[Long](100, 110, 100, 110, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
-      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // The last shuffle partition is in a single coalesced partition.
       val bytesByPartitionId = Array[Long](30, 30, 0, 40, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 4)
-      checkEstimation(Array(bytesByPartitionId), expectedPartitionStartIndices, targetSize)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 4), CoalescedPartitionSpec(4, 5))
+      checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
   }
 
@@ -104,10 +114,10 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // All bytes per partition are 0.
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
 
@@ -116,10 +126,10 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 1 coalesced partition is expected.
       val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 20, 0, 20)
-      val expectedPartitionStartIndices = Array[Int](0)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
 
@@ -127,10 +137,13 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 2 coalesced partition are expected.
       val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 2, 4)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 2),
+        CoalescedPartitionSpec(2, 4),
+        CoalescedPartitionSpec(4, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
 
@@ -138,10 +151,14 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 4 coalesced partition are expected.
       val bytesByPartitionId1 = Array[Long](0, 99, 0, 20, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 4),
+        CoalescedPartitionSpec(4, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
 
@@ -149,10 +166,14 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 2 coalesced partition are needed.
       val bytesByPartitionId1 = Array[Long](0, 100, 0, 30, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 4)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 4),
+        CoalescedPartitionSpec(4, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
 
@@ -160,10 +181,15 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // There are a few large shuffle partitions.
       val bytesByPartitionId1 = Array[Long](0, 100, 40, 30, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 60, 0, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
 
@@ -171,10 +197,15 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // All pairs of shuffle partitions are larger than the targeted size.
       val bytesByPartitionId1 = Array[Long](100, 100, 40, 30, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 60, 70, 110)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 2, 3, 4)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 2),
+        CoalescedPartitionSpec(2, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize)
     }
   }
@@ -188,10 +219,10 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // the size of data is 0.
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionStartIndices = Array[Int](0)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize, minNumPartitions)
     }
 
@@ -199,10 +230,10 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // The minimal number of coalesced partitions is enforced.
       val bytesByPartitionId1 = Array[Long](10, 5, 5, 0, 20)
       val bytesByPartitionId2 = Array[Long](5, 10, 0, 10, 5)
-      val expectedPartitionStartIndices = Array[Int](0, 3)
+      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize, minNumPartitions)
     }
 
@@ -210,10 +241,14 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // The number of coalesced partitions is determined by the algorithm.
       val bytesByPartitionId1 = Array[Long](10, 50, 20, 80, 20)
       val bytesByPartitionId2 = Array[Long](40, 10, 0, 10, 30)
-      val expectedPartitionStartIndices = Array[Int](0, 1, 3, 4)
+      val expectedPartitionSpecs = Array(
+        CoalescedPartitionSpec(0, 1),
+        CoalescedPartitionSpec(1, 3),
+        CoalescedPartitionSpec(3, 4),
+        CoalescedPartitionSpec(4, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
-        expectedPartitionStartIndices,
+        expectedPartitionSpecs,
         targetSize, minNumPartitions)
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 64566af332afb..17f6b29a2d076 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -22,8 +22,9 @@ import java.net.URI
 
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
 import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan}
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
+import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight, SortMergeJoinExec}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
 import org.apache.spark.sql.internal.SQLConf
@@ -110,7 +111,7 @@ class AdaptiveQueryExecSuite
     }.length
 
     val numLocalReaders = collect(plan) {
-      case reader: LocalShuffleReaderExec => reader
+      case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
     }.length
 
     assert(numShuffles === (numLocalReaders + numShufflesWithoutLocalReader))
@@ -142,11 +143,11 @@ class AdaptiveQueryExecSuite
       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
       assert(bhj.size == 1)
       val localReaders = collect(adaptivePlan) {
-        case reader: LocalShuffleReaderExec => reader
+        case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
       }
       assert(localReaders.length == 2)
-      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[LocalShuffledRowRDD]
-      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[LocalShuffledRowRDD]
+      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[ShuffledRowRDD]
+      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[ShuffledRowRDD]
       // The pre-shuffle partition size is [0, 0, 0, 72, 0]
       // And the partitionStartIndices is [0, 3, 4], so advisoryParallelism = 3.
       // the final parallelism is
@@ -174,11 +175,11 @@ class AdaptiveQueryExecSuite
       val bhj = findTopLevelBroadcastHashJoin(adaptivePlan)
       assert(bhj.size == 1)
       val localReaders = collect(adaptivePlan) {
-        case reader: LocalShuffleReaderExec => reader
+        case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
       }
       assert(localReaders.length == 2)
-      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[LocalShuffledRowRDD]
-      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[LocalShuffledRowRDD]
+      val localShuffleRDD0 = localReaders(0).execute().asInstanceOf[ShuffledRowRDD]
+      val localShuffleRDD1 = localReaders(1).execute().asInstanceOf[ShuffledRowRDD]
       // the final parallelism is math.max(1, numReduces / numMappers): math.max(1, 5/2) = 2
       // and the partitions length is 2 * numMappers = 4
       assert(localShuffleRDD0.getPartitions.length == 4)
@@ -622,10 +623,10 @@ class AdaptiveQueryExecSuite
         def checkSkewJoin(joins: Seq[SortMergeJoinExec], expectedNumPartitions: Int): Unit = {
           assert(joins.size == 1 && joins.head.isSkewJoin)
           assert(joins.head.left.collect {
-            case r: SkewJoinShuffleReaderExec => r
+            case r: CustomShuffleReaderExec => r
           }.head.partitionSpecs.length == expectedNumPartitions)
           assert(joins.head.right.collect {
-            case r: SkewJoinShuffleReaderExec => r
+            case r: CustomShuffleReaderExec => r
           }.head.partitionSpecs.length == expectedNumPartitions)
         }
 

From fafa8d887ded4ca779bcd9f1cef88cc40aded0ae Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 3 Mar 2020 10:24:50 +0900
Subject: [PATCH 0200/1280] [SPARK-25202][SQL][FOLLOW-UP] Keep the old
 parameter name 'pattern' at split in Scala API

### What changes were proposed in this pull request?

To address the concern pointed out in https://github.com/apache/spark/pull/22227. This will make `split` source-compatible by removing minimal cosmetic changes.

### Why are the changes needed?

For source compatibility.

### Does this PR introduce any user-facing change?

No (it will prevent potential user-facing change from the original PR)

### How was this patch tested?

Unittest was changed (in order for us to detect that source compatibility easily).

Closes #27756 from HyukjinKwon/SPARK-25202.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3956e95f059ba9599c3cfde29225177d29b2494a)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/functions.scala      | 20 +++++++++----------
 .../org/apache/spark/sql/DataFrameSuite.scala |  2 +-
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index c60df14f04817..c6e8cf76fea1f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2460,25 +2460,25 @@ object functions {
   def soundex(e: Column): Column = withExpr { SoundEx(e.expr) }
 
   /**
-   * Splits str around matches of the given regex.
+   * Splits str around matches of the given pattern.
    *
    * @param str a string expression to split
-   * @param regex a string representing a regular expression. The regex string should be
-   *              a Java regular expression.
+   * @param pattern a string representing a regular expression. The regex string should be
+   *                a Java regular expression.
    *
    * @group string_funcs
    * @since 1.5.0
    */
-  def split(str: Column, regex: String): Column = withExpr {
-    StringSplit(str.expr, Literal(regex), Literal(-1))
+  def split(str: Column, pattern: String): Column = withExpr {
+    StringSplit(str.expr, Literal(pattern), Literal(-1))
   }
 
   /**
-   * Splits str around matches of the given regex.
+   * Splits str around matches of the given pattern.
    *
    * @param str a string expression to split
-   * @param regex a string representing a regular expression. The regex string should be
-   *              a Java regular expression.
+   * @param pattern a string representing a regular expression. The regex string should be
+   *                a Java regular expression.
    * @param limit an integer expression which controls the number of times the regex is applied.
    *        <ul>
    *          <li>limit greater than 0: The resulting array's length will not be more than limit,
@@ -2491,8 +2491,8 @@ object functions {
    * @group string_funcs
    * @since 3.0.0
    */
-  def split(str: Column, regex: String, limit: Int): Column = withExpr {
-    StringSplit(str.expr, Literal(regex), Literal(limit))
+  def split(str: Column, pattern: String, limit: Int): Column = withExpr {
+    StringSplit(str.expr, Literal(pattern), Literal(limit))
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 42a90735368f2..e74d5532849dd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -196,7 +196,7 @@ class DataFrameSuite extends QueryTest
   test("explode on output of array-valued function") {
     val df = Seq(("1,2"), ("4"), ("7,8,9")).toDF("csv")
     checkAnswer(
-      df.select(explode(split($"csv", ","))),
+      df.select(explode(split($"csv", pattern = ","))),
       Row("1") :: Row("2") :: Row("4") :: Row("7") :: Row("8") :: Row("9") :: Nil)
   }
 

From 4fa447cabc1c09170edd9da0e586660a7ae0db74 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 3 Mar 2020 14:21:20 +0800
Subject: [PATCH 0201/1280] Revert "[SPARK-30808][SQL] Enable Java 8 time API
 in Thrift server"

This reverts commit afaeb29599593f021c9ea47e52f8c70013a4afef.

### What changes were proposed in this pull request?

Based on the result and comment from https://github.com/apache/spark/pull/27552#discussion_r385531744

In the hive module, server-side provides datetime values simply use `value.toSting`, and the client-side regenerates the results back in `HiveBaseResultSet` with `java.sql.Date(Timestamp).valueOf`.
there will be inconsistency between client and server if we use java8 APIs

### Why are the changes needed?

the change is still unclear enough

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

Nah

Closes #27733 from yaooqinn/SPARK-30808.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1fac06c4307c8c7a5a48a50952d48ee5b9ebccb2)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/HiveResult.scala      | 61 +++++++------------
 .../org/apache/spark/sql/SQLQuerySuite.scala  |  3 +-
 .../apache/spark/sql/SQLQueryTestSuite.scala  |  2 +-
 .../spark/sql/execution/HiveResultSuite.scala | 25 ++++----
 .../SparkExecuteStatementOperation.scala      | 10 +--
 .../hive/thriftserver/SparkSQLDriver.scala    |  5 +-
 .../hive/execution/HiveComparisonTest.scala   |  4 +-
 7 files changed, 43 insertions(+), 67 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index b19184055268a..5a2f16d8e1526 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -21,10 +21,9 @@ import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 import java.time.{Instant, LocalDate}
 
-import org.apache.spark.sql.{Dataset, Row}
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
 import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand}
-import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -37,43 +36,27 @@ object HiveResult {
    * Returns the result as a hive compatible sequence of strings. This is used in tests and
    * `SparkSQLDriver` for CLI applications.
    */
-  def hiveResultString(ds: Dataset[_]): Seq[String] = {
-    val executedPlan = ds.queryExecution.executedPlan
-    executedPlan match {
-      case ExecutedCommandExec(_: DescribeCommandBase) =>
-        // If it is a describe command for a Hive table, we want to have the output format
-        // be similar with Hive.
-        executedPlan.executeCollectPublic().map {
-          case Row(name: String, dataType: String, comment) =>
-            Seq(name, dataType,
-              Option(comment.asInstanceOf[String]).getOrElse(""))
-              .map(s => String.format(s"%-20s", s))
-              .mkString("\t")
-        }
-      // SHOW TABLES in Hive only output table names,
-      // while ours output database, table name, isTemp.
-      case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
-        command.executeCollect().map(_.getString(1))
-      case _ =>
-        val sessionWithJava8DatetimeEnabled = {
-          val cloned = ds.sparkSession.cloneSession()
-          cloned.conf.set(SQLConf.DATETIME_JAVA8API_ENABLED.key, true)
-          cloned
-        }
-        sessionWithJava8DatetimeEnabled.withActive {
-          // We cannot collect the original dataset because its encoders could be created
-          // with disabled Java 8 date-time API.
-          val result: Seq[Seq[Any]] = Dataset.ofRows(ds.sparkSession, ds.logicalPlan)
-            .queryExecution
-            .executedPlan
-            .executeCollectPublic().map(_.toSeq).toSeq
-          // We need the types so we can output struct field names
-          val types = executedPlan.output.map(_.dataType)
-          // Reformat to match hive tab delimited output.
-          result.map(_.zip(types).map(e => toHiveString(e)))
-            .map(_.mkString("\t"))
-        }
-    }
+  def hiveResultString(executedPlan: SparkPlan): Seq[String] = executedPlan match {
+    case ExecutedCommandExec(_: DescribeCommandBase) =>
+      // If it is a describe command for a Hive table, we want to have the output format
+      // be similar with Hive.
+      executedPlan.executeCollectPublic().map {
+        case Row(name: String, dataType: String, comment) =>
+          Seq(name, dataType,
+            Option(comment.asInstanceOf[String]).getOrElse(""))
+            .map(s => String.format(s"%-20s", s))
+            .mkString("\t")
+      }
+    // SHOW TABLES in Hive only output table names, while ours output database, table name, isTemp.
+    case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
+      command.executeCollect().map(_.getString(1))
+    case other =>
+      val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
+      // We need the types so we can output struct field names
+      val types = executedPlan.output.map(_.dataType)
+      // Reformat to match hive tab delimited output.
+      result.map(_.zip(types).map(e => toHiveString(e)))
+        .map(_.mkString("\t"))
   }
 
   private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 87de8f50568f6..563b4d1427395 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -190,7 +190,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
           example.split("  > ").toList.foreach(_ match {
             case exampleRe(sql, output) =>
               val df = clonedSpark.sql(sql)
-              val actual = unindentAndTrim(hiveResultString(df).mkString("\n"))
+              val actual = unindentAndTrim(
+                hiveResultString(df.queryExecution.executedPlan).mkString("\n"))
               val expected = unindentAndTrim(output)
               assert(actual === expected)
             case _ =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 34829f18bfed2..6c661667825b2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -511,7 +511,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     val schema = df.schema.catalogString
     // Get answer, but also get rid of the #1234 expression ids that show up in explain plans
     val answer = SQLExecution.withNewExecutionId(df.queryExecution, Some(sql)) {
-      hiveResultString(df).map(replaceNotIncludedMsg)
+      hiveResultString(df.queryExecution.executedPlan).map(replaceNotIncludedMsg)
     }
 
     // If the output is not pre-sorted, sort it.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index bddd15c6e25d6..bb59b12e6f350 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -25,10 +25,11 @@ class HiveResultSuite extends SharedSparkSession {
   test("date formatting in hive result") {
     val dates = Seq("2018-12-28", "1582-10-13", "1582-10-14", "1582-10-15")
     val df = dates.toDF("a").selectExpr("cast(a as date) as b")
-    val result = HiveResult.hiveResultString(df)
+    val executedPlan1 = df.queryExecution.executedPlan
+    val result = HiveResult.hiveResultString(executedPlan1)
     assert(result == dates)
-    val df2 = df.selectExpr("array(b)")
-    val result2 = HiveResult.hiveResultString(df2)
+    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
+    val result2 = HiveResult.hiveResultString(executedPlan2)
     assert(result2 == dates.map(x => s"[$x]"))
   }
 
@@ -39,10 +40,11 @@ class HiveResultSuite extends SharedSparkSession {
       "1582-10-14 01:02:03",
       "1582-10-15 01:02:03")
     val df = timestamps.toDF("a").selectExpr("cast(a as timestamp) as b")
-    val result = HiveResult.hiveResultString(df)
+    val executedPlan1 = df.queryExecution.executedPlan
+    val result = HiveResult.hiveResultString(executedPlan1)
     assert(result == timestamps)
-    val df2 = df.selectExpr("array(b)")
-    val result2 = HiveResult.hiveResultString(df2)
+    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
+    val result2 = HiveResult.hiveResultString(executedPlan2)
     assert(result2 == timestamps.map(x => s"[$x]"))
   }
 
@@ -55,14 +57,15 @@ class HiveResultSuite extends SharedSparkSession {
   test("decimal formatting in hive result") {
     val df = Seq(new java.math.BigDecimal("1")).toDS()
     Seq(2, 6, 18).foreach { scala =>
-      val decimalDf = df.selectExpr(s"CAST(value AS decimal(38, $scala))")
-      val result = HiveResult.hiveResultString(decimalDf)
+      val executedPlan =
+        df.selectExpr(s"CAST(value AS decimal(38, $scala))").queryExecution.executedPlan
+      val result = HiveResult.hiveResultString(executedPlan)
       assert(result.head.split("\\.").last.length === scala)
     }
 
-    val df2 = Seq(java.math.BigDecimal.ZERO).toDS()
-      .selectExpr(s"CAST(value AS decimal(38, 8))")
-    val result = HiveResult.hiveResultString(df2)
+    val executedPlan = Seq(java.math.BigDecimal.ZERO).toDS()
+      .selectExpr(s"CAST(value AS decimal(38, 8))").queryExecution.executedPlan
+    val result = HiveResult.hiveResultString(executedPlan)
     assert(result.head === "0.00000000")
   }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index 7bcd8032bd6af..cf0e5ebf3a2b1 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.security.PrivilegedExceptionAction
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate}
 import java.util.{Arrays, Map => JMap, UUID}
 import java.util.concurrent.RejectedExecutionException
 
@@ -179,14 +178,7 @@ private[hive] class SparkExecuteStatementOperation(
           }
           curCol += 1
         }
-        // Convert date-time instances to types that are acceptable by Hive libs
-        // used in conversions to strings.
-        val resultRow = row.map {
-          case i: Instant => Timestamp.from(i)
-          case ld: LocalDate => Date.valueOf(ld)
-          case other => other
-        }.toArray.asInstanceOf[Array[Object]]
-        resultRowSet.addRow(resultRow)
+        resultRowSet.addRow(row.toArray.asInstanceOf[Array[Object]])
         curRow += 1
         resultOffset += 1
       }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
index 64e91f405d613..12fba0eae6dce 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala
@@ -60,10 +60,9 @@ private[hive] class SparkSQLDriver(val context: SQLContext = SparkSQLEnv.sqlCont
     // TODO unify the error code
     try {
       context.sparkContext.setJobDescription(command)
-      val df = context.sql(command)
-      val execution = df.queryExecution
+      val execution = context.sessionState.executePlan(context.sql(command).logicalPlan)
       hiveResponse = SQLExecution.withNewExecutionId(execution) {
-        hiveResultString(df)
+        hiveResultString(execution.executedPlan)
       }
       tableSchema = getResultSetSchema(execution)
       new CommandProcessorResponse(0)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index 82fe274515505..8b1f4c92755b9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -346,9 +346,7 @@ abstract class HiveComparisonTest
         val catalystResults = queryList.zip(hiveResults).map { case (queryString, hive) =>
           val query = new TestHiveQueryExecution(queryString.replace("../../data", testDataPath))
           def getResult(): Seq[String] = {
-            SQLExecution.withNewExecutionId(query) {
-              hiveResultString(Dataset.ofRows(query.sparkSession, query.logical))
-            }
+            SQLExecution.withNewExecutionId(query)(hiveResultString(query.executedPlan))
           }
           try { (query, prepareAnswer(query, getResult())) } catch {
             case e: Throwable =>

From ded0a72d81c1d34753be8a156126312506fb50b1 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 3 Mar 2020 19:00:33 +0900
Subject: [PATCH 0202/1280] [SPARK-30998][SQL] ClassCastException when a
 generator having nested inner generators

### What changes were proposed in this pull request?

A query below failed in the master;

```
scala> sql("select array(array(1, 2), array(3)) ar").select(explode(explode($"ar"))).show()
20/03/01 13:51:56 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0)/ 1]
java.lang.ClassCastException: scala.collection.mutable.ArrayOps$ofRef cannot be cast to org.apache.spark.sql.catalyst.util.ArrayData
	at org.apache.spark.sql.catalyst.expressions.ExplodeBase.eval(generators.scala:313)
	at org.apache.spark.sql.execution.GenerateExec.$anonfun$doExecute$8(GenerateExec.scala:108)
	at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490)
	at scala.collection.Iterator$ConcatIterator.hasNext(Iterator.scala:222)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
    ...
```

This pr modified the `hasNestedGenerator` code in `ExtractGenerator` for correctly catching nested inner generators.

### Why are the changes needed?

A bug fix.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #27750 from maropu/HandleNestedGenerators.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 313e62c376acab30e546df253b28452a664d3e73)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 16 +++++++++++++---
 .../analysis/AnalysisErrorSuite.scala         | 19 +++++++++++++++++++
 .../spark/sql/GeneratorFunctionSuite.scala    |  8 ++++++++
 3 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 3d797994e77e9..486b9525a0e63 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2164,10 +2164,20 @@ class Analyzer(
     }
 
     private def hasNestedGenerator(expr: NamedExpression): Boolean = {
+      def hasInnerGenerator(g: Generator): Boolean = g match {
+        // Since `GeneratorOuter` is just a wrapper of generators, we skip it here
+        case go: GeneratorOuter =>
+          hasInnerGenerator(go.child)
+        case _ =>
+          g.children.exists { _.find {
+            case _: Generator => true
+            case _ => false
+          }.isDefined }
+      }
       CleanupAliases.trimNonTopLevelAliases(expr) match {
-        case UnresolvedAlias(_: Generator, _) => false
-        case Alias(_: Generator, _) => false
-        case MultiAlias(_: Generator, _) => false
+        case UnresolvedAlias(g: Generator, _) => hasInnerGenerator(g)
+        case Alias(g: Generator, _) => hasInnerGenerator(g)
+        case MultiAlias(g: Generator, _) => hasInnerGenerator(g)
         case other => hasGenerator(other)
       }
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 8f62b0b9b77a2..3db1053ed0478 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -433,6 +433,25 @@ class AnalysisErrorSuite extends AnalysisTest {
       :: Nil
   )
 
+  errorTest(
+    "SPARK-30998: unsupported nested inner generators",
+    {
+      val nestedListRelation = LocalRelation(
+        AttributeReference("nestedList", ArrayType(ArrayType(IntegerType)))())
+      nestedListRelation.select(Explode(Explode($"nestedList")))
+    },
+    "Generators are not supported when it's nested in expressions, but got: " +
+      "explode(explode(nestedList))" :: Nil
+  )
+
+  errorTest(
+    "SPARK-30998: unsupported nested inner generators for aggregates",
+    testRelation.select(Explode(Explode(
+      CreateArray(CreateArray(min($"a") :: max($"a") :: Nil) :: Nil)))),
+    "Generators are not supported when it's nested in expressions, but got: " +
+      "explode(explode(array(array(min(a), max(a)))))" :: Nil
+  )
+
   errorTest(
     "generator appears in operator which is not Project",
     listRelation.sortBy(Explode($"list").asc),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
index 96a0eb3e32e9b..6785b3102ab0d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
@@ -343,6 +343,14 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession {
       assert(msg2.contains("Only one generator allowed per aggregate clause"))
     }
   }
+
+  test("SPARK-30998: Unsupported nested inner generators") {
+    val errMsg = intercept[AnalysisException] {
+      sql("SELECT array(array(1, 2), array(3)) v").select(explode(explode($"v"))).collect
+    }.getMessage
+    assert(errMsg.contains("Generators are not supported when it's nested in expressions, " +
+      "but got: explode(explode(v))"))
+  }
 }
 
 case class EmptyGenerator() extends Generator {

From e332198f5aab2eec30dedfab66296382d31d3245 Mon Sep 17 00:00:00 2001
From: roland-ondeviceresearch <roland@ondeviceresearch.com>
Date: Tue, 3 Mar 2020 09:24:33 -0600
Subject: [PATCH 0203/1280] [MINOR][DOCS] ForeachBatch java example fix

### What changes were proposed in this pull request?
ForEachBatch Java example was incorrect

### Why are the changes needed?
Example did not compile

### Does this PR introduce any user-facing change?
Yes, to docs.

### How was this patch tested?
In IDE.

Closes #27740 from roland1982/foreachwriter_java_example_fix.

Authored-by: roland-ondeviceresearch <roland@ondeviceresearch.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit a4aaee01fa8e71d51f49b24889d862422e0727c7)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/structured-streaming-programming-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 47d899431888b..1776d23607f7f 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -2105,7 +2105,7 @@ streamingDF.writeStream.foreachBatch { (batchDF: DataFrame, batchId: Long) =>
 
 {% highlight java %}
 streamingDatasetOfString.writeStream().foreachBatch(
-  new VoidFunction2<Dataset<String>, Long> {
+  new VoidFunction2<Dataset<String>, Long>() {
     public void call(Dataset<String> dataset, Long batchId) {
       // Transform and write batchDF
     }    

From 591bfd9466887fd41112945343722b573dc90a5b Mon Sep 17 00:00:00 2001
From: Sean Owen <srowen@gmail.com>
Date: Tue, 3 Mar 2020 09:27:18 -0600
Subject: [PATCH 0204/1280] [SPARK-30994][CORE] Update xerces to 2.12.0

### What changes were proposed in this pull request?

Manage up the version of Xerces that Hadoop uses (and potentially user apps) to 2.12.0 to match https://issues.apache.org/jira/browse/HADOOP-16530

### Why are the changes needed?

Picks up bug and security fixes: https://www.xml.com/news/2018-05-apache-xerces-j-2120/

### Does this PR introduce any user-facing change?

Should be no behavior changes.

### How was this patch tested?

Existing tests.

Closes #27746 from srowen/SPARK-30994.

Authored-by: Sean Owen <srowen@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 97d9a22b04cc831b7de19db735154c69f696773c)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +-
 pom.xml                                 | 6 ++++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 113b7d7ede7bd..364b825041c1a 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -201,7 +201,7 @@ super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xercesImpl/2.9.1//xercesImpl-2.9.1.jar
+xercesImpl/2.12.0//xercesImpl-2.12.0.jar
 xmlenc/0.52//xmlenc-0.52.jar
 xz/1.5//xz-1.5.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 7b7423aa79217..62d14366a0e09 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -215,7 +215,7 @@ transaction-api/1.1//transaction-api-1.1.jar
 univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
 velocity/1.5//velocity-1.5.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
-xercesImpl/2.9.1//xercesImpl-2.9.1.jar
+xercesImpl/2.12.0//xercesImpl-2.12.0.jar
 xmlenc/0.52//xmlenc-0.52.jar
 xz/1.5//xz-1.5.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/pom.xml b/pom.xml
index 9d36faf8ee319..f98d132396190 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1083,6 +1083,12 @@
           </exclusion>
         </exclusions>
       </dependency>
+      <!-- Managed up to match Hadoop in HADOOP-16530 -->
+      <dependency>
+        <groupId>xerces</groupId>
+        <artifactId>xercesImpl</artifactId>
+        <version>2.12.0</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.avro</groupId>
         <artifactId>avro</artifactId>

From 0aace99d1162348269848665725c7db2541807cc Mon Sep 17 00:00:00 2001
From: xuesenliang <xuesenliang@tencent.com>
Date: Tue, 3 Mar 2020 09:29:43 -0600
Subject: [PATCH 0205/1280] [SPARK-30388][CORE] Mark running map stages of
 finished job as finished, and cancel running tasks

### What changes were proposed in this pull request?

When a job finished, its running (re-submitted) map stages should be marked as finished if not used by other jobs. The running tasks of these stages are cancelled.

And the ListenerBus should be notified too, otherwise, these map stage items will stay on the "Active Stages" page of web UI and never gone.

For example:

Suppose job 0 has two stages: map stage 0 and result stage 1. Map stage 0 has two partitions, and its result stage 1 has two partitions too.

**Steps to reproduce the bug:**
1. map stage 0:    start task 0(```TID 0```) and task 1 (```TID 1```), then both finished successfully.
2. result stage 1:  start task 0(```TID 2```) and task 1 (```TID 3```)
3. result stage 1:  task 0(```TID 2```) finished successfully
4. result stage 1:  speculative task 1.1(```TID 4```) launched, but then failed due to FetchFailedException.
5. driver re-submits map stage 0 and result stage 1.
6. map stage 0 (retry 1): task0(```TID 5```) launched
7. result stage 1: task 1(```TID 3```) finished successfully, so job 0 finished.
8. map stage 0 is removed from ```runningStages``` and ```stageIdToStage```, because it doesn't belong to any job.
```
  private def DAGScheduler#cleanupStateForJobAndIndependentStages(job: ActiveJob): HashSet[Stage] = {
   ...
      stageIdToStage.filterKeys(stageId => registeredStages.get.contains(stageId)).foreach {
        case (stageId, stage) =>
            ...
            def removeStage(stageId: Int): Unit = {
              for (stage <- stageIdToStage.get(stageId)) {
                if (runningStages.contains(stage)) {
                  logDebug("Removing running stage %d".format(stageId))
                  runningStages -= stage
                }
                ...
              }
              stageIdToStage -= stageId
            }

            jobSet -= job.jobId
            if (jobSet.isEmpty) { // no other job needs this stage
              removeStage(stageId)
            }
          }
  ...
  }

```
9. map stage 0 (retry 1): task0(TID 5) finished successfully, but its stage 0 is not in ```stageIdToStage```, so the stage not ```markStageAsFinished```
```
  private[scheduler] def DAGScheduler#handleTaskCompletion(event: CompletionEvent): Unit = {
    val task = event.task
    val stageId = task.stageId
    ...
    if (!stageIdToStage.contains(task.stageId)) {
      postTaskEnd(event)
      // Skip all the actions if the stage has been cancelled.
      return
    }
    ...
```

#### Relevant spark driver logs as follows:

```
20/01/02 11:21:45 INFO DAGScheduler: Got job 0 (main at NativeMethodAccessorImpl.java:0) with 2 output partitions
20/01/02 11:21:45 INFO DAGScheduler: Final stage: ResultStage 1 (main at NativeMethodAccessorImpl.java:0)
20/01/02 11:21:45 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)
20/01/02 11:21:45 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 0)

20/01/02 11:21:45 INFO DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[2] at main at NativeMethodAccessorImpl.java:0), which has no missing parents
20/01/02 11:21:45 INFO DAGScheduler: Submitting 2 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[2] at main at NativeMethodAccessorImpl.java:0) (first 15 tasks are for partitions Vector(0, 1))
20/01/02 11:21:45 INFO YarnClusterScheduler: Adding task set 0.0 with 2 tasks
20/01/02 11:21:45 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 9.179.143.4, executor 1, partition 0, PROCESS_LOCAL, 7704 bytes)
20/01/02 11:21:45 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 9.76.13.26, executor 2, partition 1, PROCESS_LOCAL, 7705 bytes)
20/01/02 11:22:18 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 32491 ms on 9.179.143.4 (executor 1) (1/2)
20/01/02 11:22:26 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 40544 ms on 9.76.13.26 (executor 2) (2/2)
20/01/02 11:22:26 INFO DAGScheduler: ShuffleMapStage 0 (main at NativeMethodAccessorImpl.java:0) finished in 40.854 s
20/01/02 11:22:26 INFO YarnClusterScheduler: Removed TaskSet 0.0, whose tasks have all completed, from pool

20/01/02 11:22:26 INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[6] at main at NativeMethodAccessorImpl.java:0), which has no missing parents
20/01/02 11:22:26 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 1 (MapPartitionsRDD[6] at main at NativeMethodAccessorImpl.java:0) (first 15 tasks are for partitions Vector(0, 1))
20/01/02 11:22:26 INFO YarnClusterScheduler: Adding task set 1.0 with 2 tasks
20/01/02 11:22:26 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 2, 9.179.143.4, executor 1, partition 0, NODE_LOCAL, 7929 bytes)
20/01/02 11:22:26 INFO TaskSetManager: Starting task 1.0 in stage 1.0 (TID 3, 9.76.13.26, executor 2, partition 1, NODE_LOCAL, 7929 bytes)
20/01/02 11:22:26 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 2) in 79 ms on 9.179.143.4 (executor 1) (1/2)

20/01/02 11:22:26 INFO TaskSetManager: Marking task 1 in stage 1.0 (on 9.76.13.26) as speculatable because it ran more than 158 ms
20/01/02 11:22:26 INFO TaskSetManager: Starting task 1.1 in stage 1.0 (TID 4, 9.179.143.52, executor 3, partition 1, ANY, 7929 bytes)
20/01/02 11:22:26 WARN TaskSetManager: Lost task 1.1 in stage 1.0 (TID 4, 9.179.143.52, executor 3): FetchFailed(BlockManagerId(1, 9.179.143.4, 7337, None), shuffleId=0, mapId=0, reduceId=1, message=org.apache.spark.shuffle.FetchFailedException: Connection reset by peer)
20/01/02 11:22:26 INFO TaskSetManager: Task 1.1 in stage 1.0 (TID 4) failed, but the task will not be re-executed (either because the task failed with a shuffle data fetch failure, so the previous stage needs to be re-run, or because a different copy of the task has already succeeded).
20/01/02 11:22:26 INFO DAGScheduler: Marking ResultStage 1 (main at NativeMethodAccessorImpl.java:0) as failed due to a fetch failure from ShuffleMapStage 0 (main at NativeMethodAccessorImpl.java:0)
20/01/02 11:22:26 INFO DAGScheduler: ResultStage 1 (main at NativeMethodAccessorImpl.java:0) failed in 0.261 s due to org.apache.spark.shuffle.FetchFailedException: Connection reset by peer
20/01/02 11:22:26 INFO DAGScheduler: Resubmitting ShuffleMapStage 0 (main at NativeMethodAccessorImpl.java:0) and ResultStage 1 (main at NativeMethodAccessorImpl.java:0) due to fetch failure
20/01/02 11:22:26 INFO DAGScheduler: Resubmitting failed stages

20/01/02 11:22:26 INFO DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[2] at main at NativeMethodAccessorImpl.java:0), which has no missing parents
20/01/02 11:22:26 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[2] at main at NativeMethodAccessorImpl.java:0) (first 15 tasks are for partitions Vector(0))
20/01/02 11:22:26 INFO YarnClusterScheduler: Adding task set 0.1 with 1 tasks
20/01/02 11:22:26 INFO TaskSetManager: Starting task 0.0 in stage 0.1 (TID 5, 9.179.143.4, executor 1, partition 0, PROCESS_LOCAL, 7704 bytes)

// NOTE: Here should be "INFO TaskSetManager: Finished task 1.0 in stage 1.0 (TID 3) in 10000 ms on 9.76.13.26 (executor 2) (2/2)"
// and this bug is being fixed in https://issues.apache.org/jira/browse/SPARK-30404

20/01/02 11:22:36 INFO TaskSetManager: Ignoring task-finished event for 1.0 in stage 1.0 because task 1 has already completed successfully

20/01/02 11:22:36 INFO YarnClusterScheduler: Removed TaskSet 1.0, whose tasks have all completed, from pool
20/01/02 11:22:36 INFO DAGScheduler: ResultStage 1 (main at NativeMethodAccessorImpl.java:0) finished in 10.131 s
20/01/02 11:22:36 INFO DAGScheduler: Job 0 finished: main at NativeMethodAccessorImpl.java:0, took 51.031212 s

20/01/02 11:22:58 INFO TaskSetManager: Finished task 0.0 in stage 0.1 (TID 5) in 32029 ms on 9.179.143.4 (executor 1) (1/1)
20/01/02 11:22:58 INFO YarnClusterScheduler: Removed TaskSet 0.1, whose tasks have all completed, from pool
```

### Why are the changes needed?

web UI is incorrect: ```stage 0 (retry 1)``` is finished, but it stays in ```Active Stages``` Page.

![active_stage](https://user-images.githubusercontent.com/4401756/71656718-71185680-2d77-11ea-8dbc-fd8085ab3dfb.png)

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
A new test case is added.

And test manually on cluster. The result is as follows:
![cancel_stage](https://user-images.githubusercontent.com/4401756/71658434-04a15580-2d7f-11ea-952b-dd8dd685f37d.png)

Closes #27050 from liangxs/master.

Authored-by: xuesenliang <xuesenliang@tencent.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit 7a4cf339d7082b576624940253e8283de9e83e19)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../apache/spark/scheduler/DAGScheduler.scala | 27 +++++++-----
 .../spark/scheduler/DAGSchedulerSuite.scala   | 44 +++++++++++++++++++
 2 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 7bf363dd71c1b..ec6eff3eed462 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -1431,6 +1431,7 @@ private[spark] class DAGScheduler(
                   // If the whole job has finished, remove it
                   if (job.numFinished == job.numPartitions) {
                     markStageAsFinished(resultStage)
+                    cancelRunningIndependentStages(job, s"Job ${job.jobId} is finished.")
                     cleanupStateForJobAndIndependentStages(job)
                     try {
                       // killAllTaskAttempts will fail if a SchedulerBackend does not implement
@@ -1975,18 +1976,12 @@ private[spark] class DAGScheduler(
     }
   }
 
-  /** Fails a job and all stages that are only used by that job, and cleans up relevant state. */
-  private def failJobAndIndependentStages(
-      job: ActiveJob,
-      failureReason: String,
-      exception: Option[Throwable] = None): Unit = {
-    val error = new SparkException(failureReason, exception.orNull)
+  /** Cancel all independent, running stages that are only used by this job. */
+  private def cancelRunningIndependentStages(job: ActiveJob, reason: String): Boolean = {
     var ableToCancelStages = true
-
-    // Cancel all independent, running stages.
     val stages = jobIdToStageIds(job.jobId)
     if (stages.isEmpty) {
-      logError("No stages registered for job " + job.jobId)
+      logError(s"No stages registered for job ${job.jobId}")
     }
     stages.foreach { stageId =>
       val jobsForStage: Option[HashSet[Int]] = stageIdToStage.get(stageId).map(_.jobIds)
@@ -1998,12 +1993,12 @@ private[spark] class DAGScheduler(
         if (!stageIdToStage.contains(stageId)) {
           logError(s"Missing Stage for stage with id $stageId")
         } else {
-          // This is the only job that uses this stage, so fail the stage if it is running.
+          // This stage is only used by the job, so finish the stage if it is running.
           val stage = stageIdToStage(stageId)
           if (runningStages.contains(stage)) {
             try { // cancelTasks will fail if a SchedulerBackend does not implement killTask
               taskScheduler.cancelTasks(stageId, shouldInterruptTaskThread(job))
-              markStageAsFinished(stage, Some(failureReason))
+              markStageAsFinished(stage, Some(reason))
             } catch {
               case e: UnsupportedOperationException =>
                 logWarning(s"Could not cancel tasks for stage $stageId", e)
@@ -2013,11 +2008,19 @@ private[spark] class DAGScheduler(
         }
       }
     }
+    ableToCancelStages
+  }
 
-    if (ableToCancelStages) {
+  /** Fails a job and all stages that are only used by that job, and cleans up relevant state. */
+  private def failJobAndIndependentStages(
+      job: ActiveJob,
+      failureReason: String,
+      exception: Option[Throwable] = None): Unit = {
+    if (cancelRunningIndependentStages(job, failureReason)) {
       // SPARK-15783 important to cleanup state first, just for tests where we have some asserts
       // against the state.  Otherwise we have a *little* bit of flakiness in the tests.
       cleanupStateForJobAndIndependentStages(job)
+      val error = new SparkException(failureReason, exception.orNull)
       job.listener.jobFailed(error)
       listenerBus.post(SparkListenerJobEnd(job.jobId, clock.getTimeMillis(), JobFailed(error)))
     }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 101e60c73e9f8..72a2e4ca2d5a1 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -1931,6 +1931,50 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assertDataStructuresEmpty()
   }
 
+  test("shuffle fetch failed on speculative task, but original task succeed (SPARK-30388)") {
+    var completedStage: List[Int] = Nil
+    val listener = new SparkListener() {
+      override def onStageCompleted(event: SparkListenerStageCompleted): Unit = {
+        completedStage = completedStage :+ event.stageInfo.stageId
+      }
+    }
+    sc.addSparkListener(listener)
+
+    val shuffleMapRdd = new MyRDD(sc, 2, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2))
+    val reduceRdd = new MyRDD(sc, 2, List(shuffleDep))
+    submit(reduceRdd, Array(0, 1))
+    completeShuffleMapStageSuccessfully(0, 0, 2)
+    assert(completedStage === List(0))
+
+    // result task 0.0 succeed
+    runEvent(makeCompletionEvent(taskSets(1).tasks(0), Success, 42))
+    // speculative result task 1.1 fetch failed
+    val info = new TaskInfo(4, index = 1, attemptNumber = 1, 0L, "", "", TaskLocality.ANY, true)
+    runEvent(makeCompletionEvent(
+        taskSets(1).tasks(1),
+        FetchFailed(makeBlockManagerId("hostA"), shuffleDep.shuffleId, 0L, 0, 1, "ignored"),
+        null,
+        Seq.empty,
+        Array.empty,
+        info
+      )
+    )
+    assert(completedStage === List(0, 1))
+
+    Thread.sleep(DAGScheduler.RESUBMIT_TIMEOUT * 2)
+    // map stage resubmitted
+    assert(scheduler.runningStages.size === 1)
+    val mapStage = scheduler.runningStages.head
+    assert(mapStage.id === 0)
+    assert(mapStage.latestInfo.failureReason.isEmpty)
+
+    // original result task 1.0 succeed
+    runEvent(makeCompletionEvent(taskSets(1).tasks(1), Success, 42))
+    assert(completedStage === List(0, 1, 1, 0))
+    assert(scheduler.activeJobs.isEmpty)
+  }
+
   test("misbehaved accumulator should not crash DAGScheduler and SparkContext") {
     val acc = new LongAccumulator {
       override def add(v: java.lang.Long): Unit = throw new DAGSchedulerSuiteDummyException

From 4be2a79c7a9a1b1e3b0c3704e94da19c2b87ba47 Mon Sep 17 00:00:00 2001
From: Javier <jfuentes@hortonworks.com>
Date: Tue, 3 Mar 2020 09:55:15 -0600
Subject: [PATCH 0206/1280] [SPARK-30049][SQL] SQL fails to parse when comment
 contains an unmatched quote character

### What changes were proposed in this pull request?

A SQL statement that contains a comment with an unmatched quote character can lead to a parse error:
- Added a insideComment flag in the splitter method to avoid checking single and double quotes within a comment:
```
spark-sql> SELECT 1 -- someone's comment here
         > ;
Error in query:
extraneous input ';' expecting <EOF>(line 2, pos 0)

== SQL ==
SELECT 1 -- someone's comment here
;
^^^
```

### Why are the changes needed?

This misbehaviour was not present on previous spark versions.

### Does this PR introduce any user-facing change?

- No

### How was this patch tested?

- New tests were added.

Closes #27321 from javierivanov/SPARK-30049B.

Lead-authored-by: Javier <jfuentes@hortonworks.com>
Co-authored-by: Javier Fuentes <j.fuentes.m@icloud.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit 3ff213568694e265466d8480b61fd38f4fd8fdff)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../hive/thriftserver/SparkSQLCLIDriver.scala | 24 +++++++++++++++----
 .../sql/hive/thriftserver/CliSuite.scala      | 22 +++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index b665d4a31b9b1..19f7ea85c6cf0 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -509,24 +509,40 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
   private def splitSemiColon(line: String): JList[String] = {
     var insideSingleQuote = false
     var insideDoubleQuote = false
+    var insideComment = false
     var escape = false
     var beginIndex = 0
+    var endIndex = line.length
     val ret = new JArrayList[String]
+
     for (index <- 0 until line.length) {
-      if (line.charAt(index) == '\'') {
+      if (line.charAt(index) == '\'' && !insideComment) {
         // take a look to see if it is escaped
         if (!escape) {
           // flip the boolean variable
           insideSingleQuote = !insideSingleQuote
         }
-      } else if (line.charAt(index) == '\"') {
+      } else if (line.charAt(index) == '\"' && !insideComment) {
         // take a look to see if it is escaped
         if (!escape) {
           // flip the boolean variable
           insideDoubleQuote = !insideDoubleQuote
         }
+      } else if (line.charAt(index) == '-') {
+        val hasNext = index + 1 < line.length
+        if (insideDoubleQuote || insideSingleQuote || insideComment) {
+          // Ignores '-' in any case of quotes or comment.
+          // Avoids to start a comment(--) within a quoted segment or already in a comment.
+          // Sample query: select "quoted value --"
+          //                                    ^^ avoids starting a comment if it's inside quotes.
+        } else if (hasNext && line.charAt(index + 1) == '-') {
+          // ignore quotes and ;
+          insideComment = true
+          // ignore eol
+          endIndex = index
+        }
       } else if (line.charAt(index) == ';') {
-        if (insideSingleQuote || insideDoubleQuote) {
+        if (insideSingleQuote || insideDoubleQuote || insideComment) {
           // do not split
         } else {
           // split, do not include ; itself
@@ -543,7 +559,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
         escape = true
       }
     }
-    ret.add(line.substring(beginIndex))
+    ret.add(line.substring(beginIndex, endIndex))
     ret
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 6609701be0ede..43aafc3c8590c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -400,4 +400,26 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
         -> "1.000000000000000000"
     )
   }
+
+  test("SPARK-30049 Should not complain for quotes in commented lines") {
+    runCliWithin(1.minute)(
+      """SELECT concat('test', 'comment') -- someone's comment here
+        |;""".stripMargin -> "testcomment"
+    )
+  }
+
+  test("SPARK-30049 Should not complain for quotes in commented with multi-lines") {
+    runCliWithin(1.minute)(
+      """SELECT concat('test', 'comment') -- someone's comment here \\
+        | comment continues here with single ' quote \\
+        | extra ' \\
+        |;""".stripMargin -> "testcomment"
+    )
+    runCliWithin(1.minute)(
+      """SELECT concat('test', 'comment') -- someone's comment here \\
+        |   comment continues here with single ' quote \\
+        |   extra ' \\
+        |   ;""".stripMargin -> "testcomment"
+    )
+  }
 }

From 1c23be3b8addaa5e15d29de788b906bcbdae953b Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Wed, 4 Mar 2020 00:55:26 +0800
Subject: [PATCH 0207/1280] [SPARK-31015][SQL] Star(*) expression fails when
 used with qualified column names for v2 tables

### What changes were proposed in this pull request?

For a v2 table created with `CREATE TABLE testcat.ns1.ns2.tbl (id bigint, name string) USING foo`, the following works as expected
```
SELECT testcat.ns1.ns2.tbl.id FROM testcat.ns1.ns2.tbl
```
, but a query with qualified column name with star(*)
```
SELECT testcat.ns1.ns2.tbl.* FROM testcat.ns1.ns2.tbl
[info]   org.apache.spark.sql.AnalysisException: cannot resolve 'testcat.ns1.ns2.tbl.*' given input columns 'id, name';
```
fails to resolve. And this PR proposes to fix this issue.

### Why are the changes needed?

To fix a bug as describe above.

### Does this PR introduce any user-facing change?

Yes, now `SELECT testcat.ns1.ns2.tbl.* FROM testcat.ns1.ns2.tbl` works as expected.

### How was this patch tested?

Added new test.

Closes #27766 from imback82/fix_star_expression.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c263c154080e54dd07aaa584913773314c3528e5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/unresolved.scala    | 35 +++++++------------
 .../sql/connector/DataSourceV2SQLSuite.scala  | 27 ++++++++++++++
 2 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 608f39c2d86fd..6048d98033961 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -298,35 +298,26 @@ abstract class Star extends LeafExpression with NamedExpression {
 case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevaluable {
 
   /**
-   * Returns true if the nameParts match the qualifier of the attribute
+   * Returns true if the nameParts is a subset of the last elements of qualifier of the attribute.
    *
-   * There are two checks: i) Check if the nameParts match the qualifier fully.
-   * E.g. SELECT db.t1.* FROM db1.t1   In this case, the nameParts is Seq("db1", "t1") and
-   * qualifier of the attribute is Seq("db1","t1")
-   * ii) If (i) is not true, then check if nameParts is only a single element and it
-   * matches the table portion of the qualifier
-   *
-   * E.g. SELECT t1.* FROM db1.t1  In this case nameParts is Seq("t1") and
-   * qualifier is Seq("db1","t1")
-   * SELECT a.* FROM db1.t1 AS a
-   * In this case nameParts is Seq("a") and qualifier for
-   * attribute is Seq("a")
+   * For example, the following should all return true:
+   *   - `SELECT ns1.ns2.t.* FROM ns1.n2.t` where nameParts is Seq("ns1", "ns2", "t") and
+   *     qualifier is Seq("ns1", "ns2", "t").
+   *   - `SELECT ns2.t.* FROM ns1.n2.t` where nameParts is Seq("ns2", "t") and
+   *     qualifier is Seq("ns1", "ns2", "t").
+   *   - `SELECT t.* FROM ns1.n2.t` where nameParts is Seq("t") and
+   *     qualifier is Seq("ns1", "ns2", "t").
    */
   private def matchedQualifier(
       attribute: Attribute,
       nameParts: Seq[String],
       resolver: Resolver): Boolean = {
-    val qualifierList = attribute.qualifier
-
-    val matched = nameParts.corresponds(qualifierList)(resolver) || {
-      // check if it matches the table portion of the qualifier
-      if (nameParts.length == 1 && qualifierList.nonEmpty) {
-        resolver(nameParts.head, qualifierList.last)
-      } else {
-        false
-      }
+    val qualifierList = if (nameParts.length == attribute.qualifier.length) {
+      attribute.qualifier
+    } else {
+      attribute.qualifier.takeRight(nameParts.length)
     }
-    matched
+    nameParts.corresponds(qualifierList)(resolver)
   }
 
   override def expand(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index c074b3352e015..bccdce73f34d0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -2342,6 +2342,33 @@ class DataSourceV2SQLSuite
     assert(e2.message.contains("It is not allowed to add database prefix"))
   }
 
+  test("SPARK-31015: star expression should work for qualified column names for v2 tables") {
+    val t = "testcat.ns1.ns2.tbl"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (id bigint, name string) USING foo")
+      sql(s"INSERT INTO $t VALUES (1, 'hello')")
+
+      def check(tbl: String): Unit = {
+        checkAnswer(sql(s"SELECT testcat.ns1.ns2.tbl.* FROM $tbl"), Row(1, "hello"))
+        checkAnswer(sql(s"SELECT ns1.ns2.tbl.* FROM $tbl"), Row(1, "hello"))
+        checkAnswer(sql(s"SELECT ns2.tbl.* FROM $tbl"), Row(1, "hello"))
+        checkAnswer(sql(s"SELECT tbl.* FROM $tbl"), Row(1, "hello"))
+      }
+
+      // Test with qualified table name "testcat.ns1.ns2.tbl".
+      check(t)
+
+      // Test if current catalog and namespace is respected in column resolution.
+      sql("USE testcat.ns1.ns2")
+      check("tbl")
+
+      val ex = intercept[AnalysisException] {
+        sql(s"SELECT ns1.ns2.ns3.tbl.* from $t")
+      }
+      assert(ex.getMessage.contains("cannot resolve 'ns1.ns2.ns3.tbl.*"))
+    }
+  }
+
   private def testV1Command(sqlCommand: String, sqlParams: String): Unit = {
     val e = intercept[AnalysisException] {
       sql(s"$sqlCommand $sqlParams")

From 7d853ab6eba479a7cc5d8839b4fc497bc6b6d4c8 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 3 Mar 2020 12:25:12 -0800
Subject: [PATCH 0208/1280] [SPARK-30997][SQL] Fix an analysis failure in
 generators with aggregate functions

### What changes were proposed in this pull request?

We have supported generators in SQL aggregate expressions by SPARK-28782.
But, the generator(explode) query with aggregate functions in DataFrame failed as follows;

```
// SPARK-28782: Generator support in aggregate expressions
scala> spark.range(3).toDF("id").createOrReplaceTempView("t")
scala> sql("select explode(array(min(id), max(id))) from t").show()
+---+
|col|
+---+
|  0|
|  2|
+---+

// A failure case handled in this pr
scala> spark.range(3).select(explode(array(min($"id"), max($"id")))).show()
org.apache.spark.sql.AnalysisException:
The query operator `Generate` contains one or more unsupported
expression types Aggregate, Window or Generate.
Invalid expressions: [min(`id`), max(`id`)];;
Project [col#46L]
+- Generate explode(array(min(id#42L), max(id#42L))), false, [col#46L]
   +- Range (0, 3, step=1, splits=Some(4))

  at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis(CheckAnalysis.scala:49)
  at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis$(CheckAnalysis.scala:48)
  at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:129)
```

The root cause is that `ExtractGenerator` wrongly replaces a project w/ aggregate functions
before `GlobalAggregates` replaces it with an aggregate as follows;

```
scala> sql("SET spark.sql.optimizer.planChangeLog.level=warn")
scala> spark.range(3).select(explode(array(min($"id"), max($"id")))).show()

20/03/01 12:51:58 WARN HiveSessionStateBuilder$$anon$1:
=== Applying Rule org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences ===
!'Project [explode(array(min('id), max('id))) AS List()]   'Project [explode(array(min(id#72L), max(id#72L))) AS List()]
 +- Range (0, 3, step=1, splits=Some(4))                   +- Range (0, 3, step=1, splits=Some(4))

20/03/01 12:51:58 WARN HiveSessionStateBuilder$$anon$1:
=== Applying Rule org.apache.spark.sql.catalyst.analysis.Analyzer$ExtractGenerator ===
!'Project [explode(array(min(id#72L), max(id#72L))) AS List()]   Project [col#76L]
!+- Range (0, 3, step=1, splits=Some(4))                         +- Generate explode(array(min(id#72L), max(id#72L))), false, [col#76L]
!                                                                   +- Range (0, 3, step=1, splits=Some(4))

20/03/01 12:51:58 WARN HiveSessionStateBuilder$$anon$1:
=== Result of Batch Resolution ===
!'Project [explode(array(min('id), max('id))) AS List()]   Project [col#76L]
!+- Range (0, 3, step=1, splits=Some(4))                   +- Generate explode(array(min(id#72L), max(id#72L))), false, [col#76L]
!                                                             +- Range (0, 3, step=1, splits=Some(4))

// the analysis failed here...
```

To avoid the case in `ExtractGenerator`, this pr addes a condition to ignore generators having aggregate functions.
A correct sequence of rules is as follows;

```
20/03/01 13:19:06 WARN HiveSessionStateBuilder$$anon$1:
=== Applying Rule org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences ===
!'Project [explode(array(min('id), max('id))) AS List()]   'Project [explode(array(min(id#27L), max(id#27L))) AS List()]
 +- Range (0, 3, step=1, splits=Some(4))                   +- Range (0, 3, step=1, splits=Some(4))

20/03/01 13:19:06 WARN HiveSessionStateBuilder$$anon$1:
=== Applying Rule org.apache.spark.sql.catalyst.analysis.Analyzer$GlobalAggregates ===
!'Project [explode(array(min(id#27L), max(id#27L))) AS List()]   'Aggregate [explode(array(min(id#27L), max(id#27L))) AS List()]
 +- Range (0, 3, step=1, splits=Some(4))                         +- Range (0, 3, step=1, splits=Some(4))

20/03/01 13:19:06 WARN HiveSessionStateBuilder$$anon$1:
=== Applying Rule org.apache.spark.sql.catalyst.analysis.Analyzer$ExtractGenerator ===
!'Aggregate [explode(array(min(id#27L), max(id#27L))) AS List()]   'Project [explode(_gen_input_0#31) AS List()]
!+- Range (0, 3, step=1, splits=Some(4))                           +- Aggregate [array(min(id#27L), max(id#27L)) AS _gen_input_0#31]
!                                                                     +- Range (0, 3, step=1, splits=Some(4))

```

### Why are the changes needed?

A bug fix.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #27749 from maropu/ExplodeInAggregate.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 4a1d273a4aac66385b948c6130de0a26ef84bbb4)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/analysis/Analyzer.scala    | 14 ++++++++++++++
 .../catalyst/analysis/AnalysisErrorSuite.scala    | 15 +++++++++++++++
 .../apache/spark/sql/GeneratorFunctionSuite.scala |  5 +++++
 3 files changed, 34 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 486b9525a0e63..254dd44c428be 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2182,6 +2182,15 @@ class Analyzer(
       }
     }
 
+    private def hasAggFunctionInGenerator(ne: Seq[NamedExpression]): Boolean = {
+      ne.exists(_.find {
+        case g: Generator =>
+          g.children.exists(_.find(_.isInstanceOf[AggregateFunction]).isDefined)
+        case _ =>
+          false
+      }.nonEmpty)
+    }
+
     private def trimAlias(expr: NamedExpression): Expression = expr match {
       case UnresolvedAlias(child, _) => child
       case Alias(child, _) => child
@@ -2268,6 +2277,11 @@ class Analyzer(
         val newAgg = Aggregate(groupList, newAggList, child)
         Project(projectExprs.toList, newAgg)
 
+      case p @ Project(projectList, _) if hasAggFunctionInGenerator(projectList) =>
+        // If a generator has any aggregate function, we need to apply the `GlobalAggregates` rule
+        // first for replacing `Project` with `Aggregate`.
+        p
+
       case p @ Project(projectList, child) =>
         // Holds the resolved generator, if one exists in the project list.
         var resolvedGenerator: Generate = null
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
index 3db1053ed0478..09e0d9c4882fb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
@@ -452,6 +452,13 @@ class AnalysisErrorSuite extends AnalysisTest {
       "explode(explode(array(array(min(a), max(a)))))" :: Nil
   )
 
+  errorTest(
+    "generator nested in expressions for aggregates",
+    testRelation.select(Explode(CreateArray(min($"a") :: max($"a") :: Nil)) + 1),
+    "Generators are not supported when it's nested in expressions, but got: " +
+      "(explode(array(min(a), max(a))) + 1)" :: Nil
+  )
+
   errorTest(
     "generator appears in operator which is not Project",
     listRelation.sortBy(Explode($"list").asc),
@@ -476,6 +483,14 @@ class AnalysisErrorSuite extends AnalysisTest {
     "Only one generator allowed per select clause but found 2: explode(list), explode(list)" :: Nil
   )
 
+  errorTest(
+    "more than one generators for aggregates in SELECT",
+    testRelation.select(Explode(CreateArray(min($"a") :: Nil)),
+      Explode(CreateArray(max($"a") :: Nil))),
+    "Only one generator allowed per select clause but found 2: " +
+      "explode(array(min(a))), explode(array(max(a)))" :: Nil
+  )
+
   test("SPARK-6452 regression test") {
     // CheckAnalysis should throw AnalysisException when Aggregate contains missing attribute(s)
     // Since we manually construct the logical plan at here and Sum only accept
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
index 6785b3102ab0d..8f449037a5e01 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
@@ -351,6 +351,11 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession {
     assert(errMsg.contains("Generators are not supported when it's nested in expressions, " +
       "but got: explode(explode(v))"))
   }
+
+  test("SPARK-30997: generators in aggregate expressions for dataframe") {
+    val df = Seq(1, 2, 3).toDF("v")
+    checkAnswer(df.select(explode(array(min($"v"), max($"v")))), Row(1) :: Row(3) :: Nil)
+  }
 }
 
 case class EmptyGenerator() extends Generator {

From 27329806c36d0b403153fe1ad0077acb72d92606 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 3 Mar 2020 13:40:51 -0800
Subject: [PATCH 0209/1280] [SPARK-30999][SQL] Don't cancel a QueryStageExec
 which failed before call doMaterialize

### What changes were proposed in this pull request?

This PR proposes to not cancel a `QueryStageExec` which failed before calling `doMaterialize`.

Besides, this PR also includes 2 minor improvements:

* fail fast when stage failed before calling `doMaterialize`

* format Exception with Cause

### Why are the changes needed?

For a stage which failed before materializing the lazy value (e.g. `inputRDD`), calling `cancel` on it could re-trigger the same failure again, e.g. executing child node again(see `AdaptiveQueryExecSuite`.`SPARK-30291: AQE should catch the exceptions when doing materialize` for example). And finally, the same failure will be counted 2 times, one is for materialize error and another is for cancel error.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Updated test.

Closes #27752 from Ngone51/avoid_cancel_finished_stage.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 380e8876316d6ef5a74358be2a04ab20e8b6e7ca)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      | 23 +++++++++++--------
 .../adaptive/AdaptiveQueryExecSuite.scala     |  3 ++-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 4036424bbacaa..c018ca4a1eaec 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -165,7 +165,7 @@ case class AdaptiveSparkPlanExec(
           stagesToReplace = result.newStages ++ stagesToReplace
           executionId.foreach(onUpdatePlan)
 
-          // Start materialization of all new stages.
+          // Start materialization of all new stages and fail fast if any stages failed eagerly
           result.newStages.foreach { stage =>
             try {
               stage.materialize().onComplete { res =>
@@ -176,7 +176,10 @@ case class AdaptiveSparkPlanExec(
                 }
               }(AdaptiveSparkPlanExec.executionContext)
             } catch {
-              case e: Throwable => events.offer(StageFailure(stage, e))
+              case e: Throwable =>
+                val ex = new SparkException(
+                  s"Early failed query stage found: ${stage.treeString}", e)
+                cleanUpAndThrowException(Seq(ex), Some(stage.id))
             }
           }
         }
@@ -192,13 +195,12 @@ case class AdaptiveSparkPlanExec(
             stage.resultOption = Some(res)
           case StageFailure(stage, ex) =>
             errors.append(
-              new SparkException(s"Failed to materialize query stage: ${stage.treeString}." +
-                s" and the cause is ${ex.getMessage}", ex))
+              new SparkException(s"Failed to materialize query stage: ${stage.treeString}.", ex))
         }
 
         // In case of errors, we cancel all running stages and throw exception.
         if (errors.nonEmpty) {
-          cleanUpAndThrowException(errors)
+          cleanUpAndThrowException(errors, None)
         }
 
         // Try re-optimizing and re-planning. Adopt the new plan if its cost is equal to or less
@@ -522,9 +524,13 @@ case class AdaptiveSparkPlanExec(
    * Cancel all running stages with best effort and throw an Exception containing all stage
    * materialization errors and stage cancellation errors.
    */
-  private def cleanUpAndThrowException(errors: Seq[SparkException]): Unit = {
+  private def cleanUpAndThrowException(
+       errors: Seq[SparkException],
+       earlyFailedStage: Option[Int]): Unit = {
     val runningStages = currentPhysicalPlan.collect {
-      case s: QueryStageExec => s
+      // earlyFailedStage is the stage which failed before calling doMaterialize,
+      // so we should avoid calling cancel on it to re-trigger the failure again.
+      case s: QueryStageExec if !earlyFailedStage.contains(s.id) => s
     }
     val cancelErrors = new mutable.ArrayBuffer[SparkException]()
     try {
@@ -539,8 +545,7 @@ case class AdaptiveSparkPlanExec(
       }
     } finally {
       val ex = new SparkException(
-        "Adaptive execution failed due to stage materialization failures." +
-          s" and the cause is ${errors.head.getMessage}", errors.head)
+        "Adaptive execution failed due to stage materialization failures.", errors.head)
       errors.tail.foreach(ex.addSuppressed)
       cancelErrors.foreach(ex.addSuppressed)
       throw ex
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 17f6b29a2d076..500b6cc83953a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -692,7 +692,8 @@ class AdaptiveQueryExecSuite
         val error = intercept[Exception] {
           agged.count()
         }
-        assert(error.getCause().toString contains "Failed to materialize query stage")
+        assert(error.getCause().toString contains "Early failed query stage found")
+        assert(error.getSuppressed.size === 0)
       }
     }
   }

From 4d489c85aa56816992d12908298e13bfd6fb4a0d Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Wed, 4 Mar 2020 13:56:02 +0900
Subject: [PATCH 0210/1280] [SPARK-30289][FOLLOWUP][DOC] Update the migration
 guide for `spark.sql.legacy.ctePrecedencePolicy`

### What changes were proposed in this pull request?
Fix the migration guide document for `spark.sql.legacy.ctePrecedence.enabled`, which is introduced in #27579.

### Why are the changes needed?
The config value changed.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Document only.

Closes #27782 from xuanyuanking/SPARK-30829-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f7f1948a8c7f7638f34d6a485abc9b866b87700f)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index a9a39ceca8777..00500615895fa 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -101,7 +101,7 @@ license: |
 
   - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
 
-  - Since Spark 3.0, Spark throws an AnalysisException if name conflict is detected in the nested WITH clause by default. It forces the users to choose the specific substitution order they wanted, which is controlled by `spark.sql.legacy.ctePrecedence.enabled`. If set to false (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `true`, the result is `1` which is the behavior in version 2.4 and earlier.
+  - Since Spark 3.0, `spark.sql.legacy.ctePrecedencePolicy` is introduced to control the behavior for name conflicting in the nested WITH clause. By default value `EXCEPTION`, Spark throws an AnalysisException, it forces users to choose the specific substitution order they wanted. If set to `CORRECTED` (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `LEGACY`, the result is `1` which is the behavior in version 2.4 and earlier.
 
   - Since Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and earlier, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
 

From 9c583b8aff4f3d5799524619f4997281ae428da5 Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Wed, 4 Mar 2020 13:55:34 +0800
Subject: [PATCH 0211/1280] [SPARK-30984][SS] Add UI test for Structured
 Streaming UI

### What changes were proposed in this pull request?

- Add a UI test for Structured Streaming UI
- Fix the unsafe usages of `SimpleDateFormat` by using a ThreadLocal shared object.
- Use `start` to replace `submission` to be consistent with the API `StreamingQuery.start()`.

### Why are the changes needed?

Structured Streaming UI is missing UI tests.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

The new test.

Closes #27732 from zsxwing/ss-ui-test.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ebfff7af6a9b2d672871317d30161cdafaa32ca4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 sql/core/pom.xml                              |  10 ++
 .../streaming/ProgressReporter.scala          |   2 +-
 .../execution/streaming/StreamExecution.scala |   4 +-
 .../streaming/StreamingQueryListener.scala    |   4 +-
 .../sql/streaming/ui/StreamingQueryPage.scala |  22 ++-
 .../ui/StreamingQueryStatisticsPage.scala     |  38 ++---
 .../ui/StreamingQueryStatusListener.scala     |  13 +-
 .../spark/sql/streaming/ui/UIUtils.scala      |  17 ++
 .../StreamingQueryListenerSuite.scala         |   6 +-
 .../ui/StreamingQueryPageSuite.scala          |   2 +-
 .../StreamingQueryStatusListenerSuite.scala   |   9 +-
 .../sql/streaming/ui/UISeleniumSuite.scala    | 145 ++++++++++++++++++
 12 files changed, 221 insertions(+), 51 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala

diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 0e664eca6a820..37da614034a09 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -150,6 +150,16 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.seleniumhq.selenium</groupId>
+      <artifactId>selenium-java</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.seleniumhq.selenium</groupId>
+      <artifactId>selenium-htmlunit-driver</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index f20291e11fd70..feb151a5468b8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -349,7 +349,7 @@ trait ProgressReporter extends Logging {
     result
   }
 
-  private def formatTimestamp(millis: Long): String = {
+  protected def formatTimestamp(millis: Long): String = {
     timestampFormat.format(new Date(millis))
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index 8b3534bc0837a..8006437b31297 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -307,8 +307,8 @@ abstract class StreamExecution(
       }
 
       // `postEvent` does not throw non fatal exception.
-      val submissionTime = triggerClock.getTimeMillis()
-      postEvent(new QueryStartedEvent(id, runId, name, submissionTime))
+      val startTimestamp = triggerClock.getTimeMillis()
+      postEvent(new QueryStartedEvent(id, runId, name, formatTimestamp(startTimestamp)))
 
       // Unblock starting thread
       startLatch.countDown()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
index dd842cd1a3e99..7ae38c71a005f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/StreamingQueryListener.scala
@@ -82,7 +82,7 @@ object StreamingQueryListener {
    * @param id A unique query id that persists across restarts. See `StreamingQuery.id()`.
    * @param runId A query id that is unique for every start/restart. See `StreamingQuery.runId()`.
    * @param name User-specified name of the query, null if not specified.
-   * @param submissionTime The timestamp to start a query.
+   * @param timestamp The timestamp to start a query.
    * @since 2.1.0
    */
   @Evolving
@@ -90,7 +90,7 @@ object StreamingQueryListener {
       val id: UUID,
       val runId: UUID,
       val name: String,
-      val submissionTime: Long) extends Event
+      val timestamp: String) extends Event
 
   /**
    * Event representing any progress updates in a query.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
index 650f64fe1688c..733676546eab3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
@@ -17,22 +17,18 @@
 
 package org.apache.spark.sql.streaming.ui
 
-import java.text.SimpleDateFormat
 import javax.servlet.http.HttpServletRequest
 
 import scala.xml.Node
 
-import org.apache.commons.lang3.StringEscapeUtils
+import org.apache.commons.text.StringEscapeUtils
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
 import org.apache.spark.sql.streaming.ui.UIUtils._
 import org.apache.spark.ui.{UIUtils => SparkUIUtils, WebUIPage}
 
 private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     extends WebUIPage("") with Logging {
-  val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
-  df.setTimeZone(getTimeZone("UTC"))
 
   override def render(request: HttpServletRequest): Seq[Node] = {
     val content = generateStreamingQueryTable(request)
@@ -61,11 +57,11 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     val name = UIUtils.getQueryName(query)
     val status = UIUtils.getQueryStatus(query)
     val duration = if (queryActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.submissionTime)
+      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
     } else {
       withNoProgress(query, {
         val endTimeMs = query.lastProgress.timestamp
-        SparkUIUtils.formatDurationVerbose(df.parse(endTimeMs).getTime - query.submissionTime)
+        SparkUIUtils.formatDurationVerbose(parseProgressTimestamp(endTimeMs) - query.startTimestamp)
       }, "-")
     }
 
@@ -74,7 +70,7 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
       <td> {status} </td>
       <td> {query.id} </td>
       <td> <a href={statisticsLink}> {query.runId} </a> </td>
-      <td> {SparkUIUtils.formatDate(query.submissionTime)} </td>
+      <td> {SparkUIUtils.formatDate(query.startTimestamp)} </td>
       <td> {duration} </td>
       <td> {withNoProgress(query, {
         (query.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum /
@@ -94,29 +90,29 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
       .partition(_.isActive)
     val activeQueryTables = if (activeQueries.nonEmpty) {
       val headerRow = Seq(
-        "Name", "Status", "Id", "Run ID", "Submitted Time", "Duration", "Avg Input /sec",
+        "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
         "Avg Process /sec", "Lastest Batch")
 
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = true),
-        activeQueries, true, None, Seq(null), false))
+        activeQueries, true, Some("activeQueries-table"), Seq(null), false))
     } else {
       None
     }
 
     val inactiveQueryTables = if (inactiveQueries.nonEmpty) {
       val headerRow = Seq(
-        "Name", "Status", "Id", "Run ID", "Submitted Time", "Duration", "Avg Input /sec",
+        "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
         "Avg Process /sec", "Lastest Batch", "Error")
 
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = false),
-        inactiveQueries, true, None, Seq(null), false))
+        inactiveQueries, true, Some("completedQueries-table"), Seq(null), false))
     } else {
       None
     }
 
     // scalastyle:off
     val content =
-      <span id="completed" class="collapse-aggregated-activeQueries collapse-table"
+      <span id="active" class="collapse-aggregated-activeQueries collapse-table"
             onClick="collapseTable('collapse-aggregated-activeQueries','aggregated-activeQueries')">
         <h5 id="activequeries">
           <span class="collapse-table-arrow arrow-open"></span>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
index fa9896e03f3fc..65052d96a131a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
@@ -19,21 +19,17 @@ package org.apache.spark.sql.streaming.ui
 
 import java.{util => ju}
 import java.lang.{Long => JLong}
-import java.text.SimpleDateFormat
 import java.util.UUID
 import javax.servlet.http.HttpServletRequest
 
 import scala.xml.{Node, Unparsed}
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
 import org.apache.spark.sql.streaming.ui.UIUtils._
 import org.apache.spark.ui.{GraphUIData, JsCollector, UIUtils => SparkUIUtils, WebUIPage}
 
 private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
   extends WebUIPage("statistics") with Logging {
-  val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
-  df.setTimeZone(getTimeZone("UTC"))
 
   def generateLoadResources(request: HttpServletRequest): Seq[Node] = {
     // scalastyle:off
@@ -101,13 +97,13 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
 
   def generateBasicInfo(query: StreamingQueryUIData): Seq[Node] = {
     val duration = if (query.isActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.submissionTime)
+      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
     } else {
       withNoProgress(query, {
         val end = query.lastProgress.timestamp
         val start = query.recentProgress.head.timestamp
         SparkUIUtils.formatDurationVerbose(
-          df.parse(end).getTime - df.parse(start).getTime)
+          parseProgressTimestamp(end) - parseProgressTimestamp(start))
       }, "-")
     }
 
@@ -119,7 +115,7 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
       </strong>
       since
       <strong>
-        {SparkUIUtils.formatDate(query.submissionTime)}
+        {SparkUIUtils.formatDate(query.startTimestamp)}
       </strong>
       (<strong>{numBatches}</strong> completed batches)
     </div>
@@ -132,13 +128,13 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
 
   def generateStatTable(query: StreamingQueryUIData): Seq[Node] = {
     val batchToTimestamps = withNoProgress(query,
-      query.recentProgress.map(p => (p.batchId, df.parse(p.timestamp).getTime)),
+      query.recentProgress.map(p => (p.batchId, parseProgressTimestamp(p.timestamp))),
       Array.empty[(Long, Long)])
     val batchTimes = batchToTimestamps.map(_._2)
     val minBatchTime =
-      withNoProgress(query, df.parse(query.recentProgress.head.timestamp).getTime, 0L)
+      withNoProgress(query, parseProgressTimestamp(query.recentProgress.head.timestamp), 0L)
     val maxBatchTime =
-      withNoProgress(query, df.parse(query.lastProgress.timestamp).getTime, 0L)
+      withNoProgress(query, parseProgressTimestamp(query.lastProgress.timestamp), 0L)
     val maxRecordRate =
       withNoProgress(query, query.recentProgress.map(_.inputRowsPerSecond).max, 0L)
     val minRecordRate = 0L
@@ -152,22 +148,26 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
     val minBatchDuration = 0L
 
     val inputRateData = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.inputRowsPerSecond })), Array.empty[(Long, Double)])
     val processRateData = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.processedRowsPerSecond })), Array.empty[(Long, Double)])
     val inputRowsData = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.numInputRows })), Array.empty[(Long, Double)])
     val batchDurations = withNoProgress(query,
-      query.recentProgress.map(p => (df.parse(p.timestamp).getTime,
+      query.recentProgress.map(p => (parseProgressTimestamp(p.timestamp),
         withNumberInvalid { p.batchDuration })), Array.empty[(Long, Double)])
-    val operationDurationData = withNoProgress(query, query.recentProgress.map { p =>
-      val durationMs = p.durationMs
-      // remove "triggerExecution" as it count the other operation duration.
-      durationMs.remove("triggerExecution")
-      (df.parse(p.timestamp).getTime, durationMs)}, Array.empty[(Long, ju.Map[String, JLong])])
+    val operationDurationData = withNoProgress(
+      query,
+      query.recentProgress.map { p =>
+        val durationMs = p.durationMs
+        // remove "triggerExecution" as it count the other operation duration.
+        durationMs.remove("triggerExecution")
+        (parseProgressTimestamp(p.timestamp), durationMs)
+      },
+      Array.empty[(Long, ju.Map[String, JLong])])
 
     val jsCollector = new JsCollector
     val graphUIDataForInputRate =
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
index 91815110e0d39..e331083b30024 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListener.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.streaming.ui
 
-import java.text.SimpleDateFormat
 import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 
@@ -25,9 +24,9 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.spark.SparkConf
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryProgress}
+import org.apache.spark.sql.streaming.ui.UIUtils.parseProgressTimestamp
 
 /**
  * A customized StreamingQueryListener used in structured streaming UI, which contains all
@@ -36,9 +35,6 @@ import org.apache.spark.sql.streaming.{StreamingQueryListener, StreamingQueryPro
  */
 private[sql] class StreamingQueryStatusListener(conf: SparkConf) extends StreamingQueryListener {
 
-  private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
-  timestampFormat.setTimeZone(DateTimeUtils.getTimeZone("UTC"))
-
   /**
    * We use runId as the key here instead of id in active query status map,
    * because the runId is unique for every started query, even it its a restart.
@@ -51,12 +47,13 @@ private[sql] class StreamingQueryStatusListener(conf: SparkConf) extends Streami
   private val inactiveQueryStatusRetention = conf.get(StaticSQLConf.STREAMING_UI_RETAINED_QUERIES)
 
   override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = {
+    val startTimestamp = parseProgressTimestamp(event.timestamp)
     activeQueryStatus.putIfAbsent(event.runId,
-      new StreamingQueryUIData(event.name, event.id, event.runId, event.submissionTime))
+      new StreamingQueryUIData(event.name, event.id, event.runId, startTimestamp))
   }
 
   override def onQueryProgress(event: StreamingQueryListener.QueryProgressEvent): Unit = {
-    val batchTimestamp = timestampFormat.parse(event.progress.timestamp).getTime
+    val batchTimestamp = parseProgressTimestamp(event.progress.timestamp)
     val queryStatus = activeQueryStatus.getOrDefault(
       event.progress.runId,
       new StreamingQueryUIData(event.progress.name, event.progress.id, event.progress.runId,
@@ -89,7 +86,7 @@ private[ui] class StreamingQueryUIData(
     val name: String,
     val id: UUID,
     val runId: UUID,
-    val submissionTime: Long) {
+    val startTimestamp: Long) {
 
   /** Holds the most recent query progress updates. */
   private val progressBuffer = new mutable.Queue[StreamingQueryProgress]()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
index 57b9dec81f28a..cdad5ed9942b5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/UIUtils.scala
@@ -17,6 +17,11 @@
 
 package org.apache.spark.sql.streaming.ui
 
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.getTimeZone
+
 private[ui] object UIUtils {
 
   /**
@@ -57,4 +62,16 @@ private[ui] object UIUtils {
       query.exception.map(_ => "FAILED").getOrElse("FINISHED")
     }
   }
+
+  private val progressTimestampFormat = new ThreadLocal[SimpleDateFormat]() {
+    override def initialValue(): SimpleDateFormat = {
+      val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
+      format.setTimeZone(getTimeZone("UTC"))
+      format
+    }
+  }
+
+  def parseProgressTimestamp(timestamp: String): Long = {
+    progressTimestampFormat.get.parse(timestamp).getTime
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index 6bb1646becf22..e585b8a885c9c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -254,8 +254,10 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
       assert(newEvent.name === event.name)
     }
 
-    testSerialization(new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, "name", 1L))
-    testSerialization(new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, null, 1L))
+    testSerialization(
+      new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, "name", "2016-12-05T20:54:20.827Z"))
+    testSerialization(
+      new QueryStartedEvent(UUID.randomUUID, UUID.randomUUID, null, "2016-12-05T20:54:20.827Z"))
   }
 
   test("QueryProgressEvent serialization") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
index de43e470e8e13..2a1e18ab66bb7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPageSuite.scala
@@ -97,7 +97,7 @@ class StreamingQueryPageSuite extends SharedSparkSession with BeforeAndAfter {
     when(streamQuery.name).thenReturn("query")
     when(streamQuery.id).thenReturn(id)
     when(streamQuery.runId).thenReturn(id)
-    when(streamQuery.submissionTime).thenReturn(1L)
+    when(streamQuery.startTimestamp).thenReturn(1L)
     when(streamQuery.lastProgress).thenReturn(progress)
     when(streamQuery.recentProgress).thenReturn(Array(progress))
     when(streamQuery.exception).thenReturn(None)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
index adbb501f9842e..6aa440e5609c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
@@ -32,7 +32,8 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
     // hanlde query started event
     val id = UUID.randomUUID()
     val runId = UUID.randomUUID()
-    val startEvent = new StreamingQueryListener.QueryStartedEvent(id, runId, "test", 1L)
+    val startEvent = new StreamingQueryListener.QueryStartedEvent(
+      id, runId, "test", "2016-12-05T20:54:20.827Z")
     listener.onQueryStarted(startEvent)
 
     // result checking
@@ -78,7 +79,8 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
     // handle first time start
     val id = UUID.randomUUID()
     val runId0 = UUID.randomUUID()
-    val startEvent0 = new StreamingQueryListener.QueryStartedEvent(id, runId0, "test", 1L)
+    val startEvent0 = new StreamingQueryListener.QueryStartedEvent(
+      id, runId0, "test", "2016-12-05T20:54:20.827Z")
     listener.onQueryStarted(startEvent0)
 
     // handle terminate event
@@ -87,7 +89,8 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
 
     // handle second time start
     val runId1 = UUID.randomUUID()
-    val startEvent1 = new StreamingQueryListener.QueryStartedEvent(id, runId1, "test", 1L)
+    val startEvent1 = new StreamingQueryListener.QueryStartedEvent(
+      id, runId1, "test", "2016-12-05T20:54:20.827Z")
     listener.onQueryStarted(startEvent1)
 
     // result checking
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
new file mode 100644
index 0000000000000..fdf4c6634d79f
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/UISeleniumSuite.scala
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.streaming.ui
+
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.htmlunit.HtmlUnitDriver
+import org.scalatest._
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.time.SpanSugar._
+import org.scalatestplus.selenium.WebBrowser
+
+import org.apache.spark._
+import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_PORT}
+import org.apache.spark.sql.LocalSparkSession.withSparkSession
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.util.quietly
+import org.apache.spark.sql.streaming.StreamingQueryException
+import org.apache.spark.ui.SparkUICssErrorHandler
+
+class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
+
+  implicit var webDriver: WebDriver = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    webDriver = new HtmlUnitDriver {
+      getWebClient.setCssErrorHandler(new SparkUICssErrorHandler)
+    }
+  }
+
+  private def newSparkSession(
+      master: String = "local",
+      additionalConfs: Map[String, String] = Map.empty): SparkSession = {
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName("ui-test")
+      .set(UI_ENABLED, true)
+      .set(UI_PORT, 0)
+    additionalConfs.foreach { case (k, v) => conf.set(k, v) }
+    val spark = SparkSession.builder().master(master).config(conf).getOrCreate()
+    assert(spark.sparkContext.ui.isDefined)
+    spark
+  }
+
+  def goToUi(spark: SparkSession, path: String): Unit = {
+    go to (spark.sparkContext.ui.get.webUrl.stripSuffix("/") + path)
+  }
+
+  test("SPARK-30984: Structured Streaming UI should be activated when running a streaming query") {
+    quietly {
+      withSparkSession(newSparkSession()) { spark =>
+        import spark.implicits._
+        try {
+          spark.range(1, 10).count()
+
+          goToUi(spark, "/StreamingQuery")
+
+          val h3Text = findAll(cssSelector("h3")).map(_.text).toSeq
+          h3Text should not contain ("Streaming Query")
+
+          val activeQuery =
+            spark.readStream.format("rate").load().writeStream.format("noop").start()
+          val completedQuery =
+            spark.readStream.format("rate").load().writeStream.format("noop").start()
+          completedQuery.stop()
+          val failedQuery = spark.readStream.format("rate").load().select("value").as[Long]
+            .map(_ / 0).writeStream.format("noop").start()
+          try {
+            failedQuery.awaitTermination()
+          } catch {
+            case _: StreamingQueryException =>
+          }
+
+          eventually(timeout(30.seconds), interval(100.milliseconds)) {
+            // Check the query list page
+            goToUi(spark, "/StreamingQuery")
+
+            findAll(cssSelector("h3")).map(_.text).toSeq should contain("Streaming Query")
+            findAll(cssSelector("""#activeQueries-table th""")).map(_.text).toSeq should be {
+              List("Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
+                "Avg Process /sec", "Lastest Batch")
+            }
+            val activeQueries =
+              findAll(cssSelector("""#activeQueries-table td""")).map(_.text).toSeq
+            activeQueries should contain(activeQuery.id.toString)
+            activeQueries should contain(activeQuery.runId.toString)
+            findAll(cssSelector("""#completedQueries-table th"""))
+              .map(_.text).toSeq should be {
+                List("Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
+                  "Avg Process /sec", "Lastest Batch", "Error")
+              }
+            val completedQueries =
+              findAll(cssSelector("""#completedQueries-table td""")).map(_.text).toSeq
+            completedQueries should contain(completedQuery.id.toString)
+            completedQueries should contain(completedQuery.runId.toString)
+            completedQueries should contain(failedQuery.id.toString)
+            completedQueries should contain(failedQuery.runId.toString)
+
+            // Check the query statistics page
+            val activeQueryLink =
+              findAll(cssSelector("""#activeQueries-table a""")).flatMap(_.attribute("href")).next
+            go to activeQueryLink
+
+            findAll(cssSelector("h3"))
+              .map(_.text).toSeq should contain("Streaming Query Statistics")
+            val summaryText = findAll(cssSelector("div strong")).map(_.text).toSeq
+            summaryText should contain ("Name:")
+            summaryText should contain ("Id:")
+            summaryText should contain ("RunId:")
+            findAll(cssSelector("""#stat-table th""")).map(_.text).toSeq should be {
+              List("", "Timelines", "Histograms")
+            }
+          }
+        } finally {
+          spark.streams.active.foreach(_.stop())
+        }
+      }
+    }
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      if (webDriver != null) {
+        webDriver.quit()
+      }
+    } finally {
+      super.afterAll()
+    }
+  }
+}

From 6e9781ae508ef6b74c7c62954d21eaa7ba0a5730 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 4 Mar 2020 18:27:44 +0900
Subject: [PATCH 0212/1280] [SPARK-30960][SQL] add back the legacy
 date/timestamp format support in CSV/JSON parser

### What changes were proposed in this pull request?

Before Spark 3.0, the JSON/CSV parser has a special behavior that, when the parser fails to parse a timestamp/date, fallback to another way to parse it, to support some legacy format. The fallback was removed by https://issues.apache.org/jira/browse/SPARK-26178 and https://issues.apache.org/jira/browse/SPARK-26243.

This PR adds back this legacy fallback. Since we switch the API to do datetime operations, we can't be exactly the same as before. Here we add back the support of the legacy formats that are common (examples of Spark 2.4):
1. the fields can have one or two letters
```
scala> sql("""select from_json('{"time":"1123-2-22 2:22:22"}', 'time Timestamp')""").show(false)
+-------------------------------------------+
|jsontostructs({"time":"1123-2-22 2:22:22"})|
+-------------------------------------------+
|[1123-02-22 02:22:22]                      |
+-------------------------------------------+
```
2. the separator between data and time can be "T" as well
```
scala> sql("""select from_json('{"time":"2000-12-12T12:12:12"}', 'time Timestamp')""").show(false)
+---------------------------------------------+
|jsontostructs({"time":"2000-12-12T12:12:12"})|
+---------------------------------------------+
|[2000-12-12 12:12:12]                        |
+---------------------------------------------+
```
3. the second fraction can be arbitrary length
```
scala> sql("""select from_json('{"time":"1123-02-22T02:22:22.123456789123"}', 'time Timestamp')""").show(false)
+----------------------------------------------------------+
|jsontostructs({"time":"1123-02-22T02:22:22.123456789123"})|
+----------------------------------------------------------+
|[1123-02-15 02:22:22.123]                                 |
+----------------------------------------------------------+
```
4. date string can end up with any chars after "T" or space
```
scala> sql("""select from_json('{"time":"1123-02-22Tabc"}', 'time date')""").show(false)
+----------------------------------------+
|jsontostructs({"time":"1123-02-22Tabc"})|
+----------------------------------------+
|[1123-02-22]                            |
+----------------------------------------+
```
5. remove "GMT" from the string before parsing
```
scala> sql("""select from_json('{"time":"GMT1123-2-22 2:22:22.123"}', 'time Timestamp')""").show(false)
+--------------------------------------------------+
|jsontostructs({"time":"GMT1123-2-22 2:22:22.123"})|
+--------------------------------------------------+
|[1123-02-22 02:22:22.123]                         |
+--------------------------------------------------+
```
### Why are the changes needed?

It doesn't hurt to keep this legacy support. It just makes the parsing more relaxed.

### Does this PR introduce any user-facing change?

yes, to make 3.0 support parsing most of the csv/json values that were supported before.

### How was this patch tested?

new tests

Closes #27710 from cloud-fan/bug2.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e4c61e35da47b0662c19d4d9b7db26022da4853d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/csv/UnivocityParser.scala    |  24 +++-
 .../sql/catalyst/json/JacksonParser.scala     |  29 ++++-
 .../sql/catalyst/util/DateTimeUtils.scala     |  15 +++
 .../catalyst/csv/UnivocityParserSuite.scala   |  41 +++++++
 .../execution/datasources/csv/CSVSuite.scala  |   6 +
 .../datasources/json/JsonSuite.scala          | 112 ++++++++++++++++++
 6 files changed, 222 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index cd69c21a01976..f829e6b503ab4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -175,10 +175,30 @@ class UnivocityParser(
       }
 
     case _: TimestampType => (d: String) =>
-      nullSafeDatum(d, name, nullable, options)(timestampFormatter.parse)
+      nullSafeDatum(d, name, nullable, options) { datum =>
+        try {
+          timestampFormatter.parse(datum)
+        } catch {
+          case NonFatal(e) =>
+            // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+            // compatibility.
+            val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(datum))
+            DateTimeUtils.stringToTimestamp(str, options.zoneId).getOrElse(throw e)
+        }
+      }
 
     case _: DateType => (d: String) =>
-      nullSafeDatum(d, name, nullable, options)(dateFormatter.parse)
+      nullSafeDatum(d, name, nullable, options) { datum =>
+        try {
+          dateFormatter.parse(datum)
+        } catch {
+          case NonFatal(e) =>
+            // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+            // compatibility.
+            val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(datum))
+            DateTimeUtils.stringToDate(str, options.zoneId).getOrElse(throw e)
+        }
+      }
 
     case _: StringType => (d: String) =>
       nullSafeDatum(d, name, nullable, options)(UTF8String.fromString)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index 1e408cdb126b8..da3b5013fe12d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -21,7 +21,6 @@ import java.io.{ByteArrayOutputStream, CharConversionException}
 import java.nio.charset.MalformedInputException
 
 import scala.collection.mutable.ArrayBuffer
-import scala.util.Try
 import scala.util.control.NonFatal
 
 import com.fasterxml.jackson.core._
@@ -230,7 +229,15 @@ class JacksonParser(
     case TimestampType =>
       (parser: JsonParser) => parseJsonToken[java.lang.Long](parser, dataType) {
         case VALUE_STRING if parser.getTextLength >= 1 =>
-          timestampFormatter.parse(parser.getText)
+          try {
+            timestampFormatter.parse(parser.getText)
+          } catch {
+            case NonFatal(e) =>
+              // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+              // compatibility.
+              val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(parser.getText))
+              DateTimeUtils.stringToTimestamp(str, options.zoneId).getOrElse(throw e)
+          }
 
         case VALUE_NUMBER_INT =>
           parser.getLongValue * 1000000L
@@ -239,7 +246,23 @@ class JacksonParser(
     case DateType =>
       (parser: JsonParser) => parseJsonToken[java.lang.Integer](parser, dataType) {
         case VALUE_STRING if parser.getTextLength >= 1 =>
-          dateFormatter.parse(parser.getText)
+          try {
+            dateFormatter.parse(parser.getText)
+          } catch {
+            case NonFatal(e) =>
+              // If fails to parse, then tries the way used in 2.0 and 1.x for backwards
+              // compatibility.
+              val str = UTF8String.fromString(DateTimeUtils.cleanLegacyTimestampStr(parser.getText))
+              DateTimeUtils.stringToDate(str, options.zoneId).getOrElse {
+                // In Spark 1.5.0, we store the data as number of days since epoch in string.
+                // So, we just convert it to Int.
+                try {
+                  parser.getText.toInt
+                } catch {
+                  case _: NumberFormatException => throw e
+                }
+              }.asInstanceOf[Integer]
+          }
       }
 
     case BinaryType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index f5f0309dd6b7d..448e4b3e3cc13 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -163,6 +163,21 @@ object DateTimeUtils {
     instantToMicros(localDateTime.atZone(zoneId).toInstant)
   }
 
+  // A method called by JSON/CSV parser to clean up the legacy timestamp string by removing the
+  // "GMT" string.
+  def cleanLegacyTimestampStr(s: String): String = {
+    val indexOfGMT = s.indexOf("GMT")
+    if (indexOfGMT != -1) {
+      // ISO8601 with a weird time zone specifier (2000-01-01T00:00GMT+01:00)
+      val s0 = s.substring(0, indexOfGMT)
+      val s1 = s.substring(indexOfGMT + 3)
+      // Mapped to 2000-01-01T00:00+01:00
+      s0 + s1
+    } else {
+      s
+    }
+  }
+
   /**
    * Trim and parse a given UTF8 date string to the corresponding a corresponding [[Long]] value.
    * The return type is [[Option]] in order to distinguish between 0L and null. The following
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index 536c76f042d23..c8c71bcfeb7ac 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.sources.{EqualTo, Filter, StringStartsWith}
 import org.apache.spark.sql.types._
@@ -318,4 +319,44 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     }.getMessage
     assert(errMsg2.contains("i does not exist"))
   }
+
+  test("SPARK-30960: parse date/timestamp string with legacy format") {
+    def check(parser: UnivocityParser): Unit = {
+      // The legacy format allows 1 or 2 chars for some fields.
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45") ==
+        date(2020, 1, 12, 12, 3, 45, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12") ==
+        days(2020, 1, 12, 0, 0, 0))
+      // The legacy format allows arbitrary length of second fraction.
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45.1") ==
+        date(2020, 1, 12, 12, 3, 45, 100000))
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45.1234") ==
+        date(2020, 1, 12, 12, 3, 45, 123400))
+      // The legacy format allow date string to end with T or space, with arbitrary string
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12T") ==
+        days(2020, 1, 12, 0, 0, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12Txyz") ==
+        days(2020, 1, 12, 0, 0, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12 ") ==
+        days(2020, 1, 12, 0, 0, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12 xyz") ==
+        days(2020, 1, 12, 0, 0, 0))
+      // The legacy format ignores the "GMT" from the string
+      assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45GMT") ==
+        date(2020, 1, 12, 12, 3, 45, 0))
+      assert(parser.makeConverter("t", TimestampType).apply("GMT2020-1-12 12:3:45") ==
+        date(2020, 1, 12, 12, 3, 45, 0))
+      assert(parser.makeConverter("t", DateType).apply("2020-1-12GMT") ==
+        days(2020, 1, 12, 0, 0, 0))
+      assert(parser.makeConverter("t", DateType).apply("GMT2020-1-12") ==
+        days(2020, 1, 12, 0, 0, 0))
+    }
+
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
+    check(new UnivocityParser(StructType(Seq.empty), options))
+
+    val optionsWithPattern =
+      new CSVOptions(Map("timestampFormat" -> "invalid", "dateFormat" -> "invalid"), false, "UTC")
+    check(new UnivocityParser(StructType(Seq.empty), optionsWithPattern))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 0be0e1e3da3dc..43553df296545 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2301,6 +2301,12 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     assert(csv.schema.fieldNames === Seq("a", "b", "0"))
     checkAnswer(csv, Row("a", "b", 1))
   }
+
+  test("SPARK-30960: parse date/timestamp string with legacy format") {
+    val ds = Seq("2020-1-12 3:23:34.12, 2020-1-12 T").toDS()
+    val csv = spark.read.option("header", false).schema("t timestamp, d date").csv(ds)
+    checkAnswer(csv, Row(Timestamp.valueOf("2020-1-12 3:23:34.12"), Date.valueOf("2020-1-12")))
+  }
 }
 
 class CSVv1Suite extends CSVSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 7abe818a29d9f..d2ce555c04542 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -21,6 +21,7 @@ import java.io._
 import java.nio.charset.{Charset, StandardCharsets, UnsupportedCharsetException}
 import java.nio.file.Files
 import java.sql.{Date, Timestamp}
+import java.time.{LocalDate, LocalDateTime, ZoneId}
 import java.util.Locale
 
 import com.fasterxml.jackson.core.JsonFactory
@@ -39,6 +40,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.StructType.fromDDL
+import org.apache.spark.sql.types.TestUDT.{MyDenseVector, MyDenseVectorUDT}
 import org.apache.spark.util.Utils
 
 class TestFileFilter extends PathFilter {
@@ -1447,6 +1449,107 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     })
   }
 
+  test("backward compatibility") {
+    // This test we make sure our JSON support can read JSON data generated by previous version
+    // of Spark generated through toJSON method and JSON data source.
+    // The data is generated by the following program.
+    // Here are a few notes:
+    //  - Spark 1.5.0 cannot save timestamp data. So, we manually added timestamp field (col13)
+    //      in the JSON object.
+    //  - For Spark before 1.5.1, we do not generate UDTs. So, we manually added the UDT value to
+    //      JSON objects generated by those Spark versions (col17).
+    //  - If the type is NullType, we do not write data out.
+
+    // Create the schema.
+    val struct =
+      StructType(
+        StructField("f1", FloatType, true) ::
+          StructField("f2", ArrayType(BooleanType), true) :: Nil)
+
+    val dataTypes =
+      Seq(
+        StringType, BinaryType, NullType, BooleanType,
+        ByteType, ShortType, IntegerType, LongType,
+        FloatType, DoubleType, DecimalType(25, 5), DecimalType(6, 5),
+        DateType, TimestampType,
+        ArrayType(IntegerType), MapType(StringType, LongType), struct,
+        new MyDenseVectorUDT())
+    val fields = dataTypes.zipWithIndex.map { case (dataType, index) =>
+      StructField(s"col$index", dataType, nullable = true)
+    }
+    val schema = StructType(fields)
+
+    val constantValues =
+      Seq(
+        "a string in binary".getBytes(StandardCharsets.UTF_8),
+        null,
+        true,
+        1.toByte,
+        2.toShort,
+        3,
+        Long.MaxValue,
+        0.25.toFloat,
+        0.75,
+        new java.math.BigDecimal(s"1234.23456"),
+        new java.math.BigDecimal(s"1.23456"),
+        java.sql.Date.valueOf("2015-01-01"),
+        java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123"),
+        Seq(2, 3, 4),
+        Map("a string" -> 2000L),
+        Row(4.75.toFloat, Seq(false, true)),
+        new MyDenseVector(Array(0.25, 2.25, 4.25)))
+    val data =
+      Row.fromSeq(Seq("Spark " + spark.sparkContext.version) ++ constantValues) :: Nil
+
+    // Data generated by previous versions.
+    // scalastyle:off
+    val existingJSONData =
+      """{"col0":"Spark 1.2.2","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.3.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.3.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.4.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.4.1","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.5.0","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"2015-01-01","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" ::
+      """{"col0":"Spark 1.5.0","col1":"YSBzdHJpbmcgaW4gYmluYXJ5","col3":true,"col4":1,"col5":2,"col6":3,"col7":9223372036854775807,"col8":0.25,"col9":0.75,"col10":1234.23456,"col11":1.23456,"col12":"16436","col13":"2015-01-01 23:50:59.123","col14":[2,3,4],"col15":{"a string":2000},"col16":{"f1":4.75,"f2":[false,true]},"col17":[0.25,2.25,4.25]}""" :: Nil
+    // scalastyle:on
+
+    // Generate data for the current version.
+    val df = spark.createDataFrame(spark.sparkContext.parallelize(data, 1), schema)
+    withTempPath { path =>
+      df.write.format("json").mode("overwrite").save(path.getCanonicalPath)
+
+      // df.toJSON will convert internal rows to external rows first and then generate
+      // JSON objects. While, df.write.format("json") will write internal rows directly.
+      val allJSON =
+        existingJSONData ++
+          df.toJSON.collect() ++
+          sparkContext.textFile(path.getCanonicalPath).collect()
+
+      Utils.deleteRecursively(path)
+      sparkContext.parallelize(allJSON, 1).saveAsTextFile(path.getCanonicalPath)
+
+      // Read data back with the schema specified.
+      val col0Values =
+        Seq(
+          "Spark 1.2.2",
+          "Spark 1.3.1",
+          "Spark 1.3.1",
+          "Spark 1.4.1",
+          "Spark 1.4.1",
+          "Spark 1.5.0",
+          "Spark 1.5.0",
+          "Spark " + spark.sparkContext.version,
+          "Spark " + spark.sparkContext.version)
+      val expectedResult = col0Values.map { v =>
+        Row.fromSeq(Seq(v) ++ constantValues)
+      }
+      checkAnswer(
+        spark.read.format("json").schema(schema).load(path.getCanonicalPath),
+        expectedResult
+      )
+    }
+  }
+
   test("SPARK-11544 test pathfilter") {
     withTempPath { dir =>
       val path = dir.getCanonicalPath
@@ -2557,6 +2660,15 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       checkAnswer(readBack, timestampsWithFormat)
     }
   }
+
+  test("SPARK-30960: parse date/timestamp string with legacy format") {
+    val ds = Seq("{'t': '2020-1-12 3:23:34.12', 'd': '2020-1-12 T', 'd2': '12345'}").toDS()
+    val json = spark.read.schema("t timestamp, d date, d2 date").json(ds)
+    checkAnswer(json, Row(
+      Timestamp.valueOf("2020-1-12 3:23:34.12"),
+      Date.valueOf("2020-1-12"),
+      Date.valueOf(LocalDate.ofEpochDay(12345))))
+  }
 }
 
 class JsonV1Suite extends JsonSuite {

From 8e97c5c9fab37703aa98a4564884280884e2228b Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Wed, 4 Mar 2020 18:09:48 +0800
Subject: [PATCH 0213/1280] [SPARK-30885][SQL][FOLLOW-UP] Fix issues where some
 V1 commands allow tables that are not fully qualified

### What changes were proposed in this pull request?

There are few V1 commands such as `REFRESH TABLE` that still allow `spark_catalog.t` because they run the commands with parsed table names without trying to load them in the catalog. This PR addresses this issue.

The PR also addresses the issue brought up in https://github.com/apache/spark/pull/27642#discussion_r382402104.

### Why are the changes needed?

To fix a bug where for some V1 commands, `spark_catalog.t` is allowed.

### Does this PR introduce any user-facing change?

Yes, a bug is fixed and `REFRESH TABLE spark_catalog.t` is not allowed.

### How was this patch tested?

Added new test.

Closes #27718 from imback82/fix_TempViewOrV1Table.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b30278107fa34723f5a93b412b4ff21d6e3bbef0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/connector/catalog/LookupCatalog.scala |  7 +++
 .../analysis/ResolveSessionCatalog.scala      | 14 +-----
 .../sql/connector/DataSourceV2SQLSuite.scala  | 44 +++++++++++++++----
 .../command/PlanResolutionSuite.scala         | 42 +++++++++---------
 4 files changed, 65 insertions(+), 42 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
index b0b9d7b15cef3..10c15747ec4ce 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/LookupCatalog.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.connector.catalog
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 
@@ -56,8 +57,14 @@ private[sql] trait LookupCatalog extends Logging {
    * Extract session catalog and identifier from a multi-part identifier.
    */
   object SessionCatalogAndIdentifier {
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+
     def unapply(parts: Seq[String]): Option[(CatalogPlugin, Identifier)] = parts match {
       case CatalogAndIdentifier(catalog, ident) if CatalogV2Util.isSessionCatalog(catalog) =>
+        if (ident.namespace.length != 1) {
+          throw new AnalysisException(
+            s"The namespace in session catalog must have exactly one name part: ${parts.quoted}")
+        }
         Some(catalog, ident)
       case _ => None
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index baaa24f52ced9..7950b61318e2c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -572,11 +572,6 @@ class ResolveSessionCatalog(
           FunctionIdentifier(nameParts.head, None)
         } else {
           ident.namespace match {
-            // For name parts like `spark_catalog.t`, we need to fill in the default database so
-            // that the caller side won't treat it as a temp function.
-            case Array() if nameParts.head == CatalogManager.SESSION_CATALOG_NAME =>
-              FunctionIdentifier(
-                ident.name, Some(catalogManager.v1SessionCatalog.getCurrentDatabase))
             case Array(db) => FunctionIdentifier(ident.name, Some(db))
             case _ =>
               throw new AnalysisException(s"Unsupported function name '$ident'")
@@ -642,14 +637,7 @@ class ResolveSessionCatalog(
   object TempViewOrV1Table {
     def unapply(nameParts: Seq[String]): Option[Seq[String]] = nameParts match {
       case _ if isTempView(nameParts) => Some(nameParts)
-      case SessionCatalogAndTable(_, tbl) =>
-        if (nameParts.head == CatalogManager.SESSION_CATALOG_NAME && tbl.length == 1) {
-          // For name parts like `spark_catalog.t`, we need to fill in the default database so
-          // that the caller side won't treat it as a temp view.
-          Some(Seq(catalogManager.v1SessionCatalog.getCurrentDatabase, tbl.head))
-        } else {
-          Some(tbl)
-        }
+      case SessionCatalogAndIdentifier(_, tbl) => Some(tbl.asMultipartIdentifier)
       case _ => None
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index bccdce73f34d0..1fc0bb10a535c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -2090,7 +2090,8 @@ class DataSourceV2SQLSuite
     val e1 = intercept[AnalysisException] {
       sql("DESCRIBE FUNCTION default.ns1.ns2.fun")
     }
-    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+    assert(e1.message.contains(
+      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
   }
 
   test("SHOW FUNCTIONS not valid v1 namespace") {
@@ -2109,9 +2110,10 @@ class DataSourceV2SQLSuite
     assert(e.message.contains("DROP FUNCTION is only supported in v1 catalog"))
 
     val e1 = intercept[AnalysisException] {
-      sql("DESCRIBE FUNCTION default.ns1.ns2.fun")
+      sql("DROP FUNCTION default.ns1.ns2.fun")
     }
-    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+    assert(e1.message.contains(
+      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
   }
 
   test("CREATE FUNCTION: only support session catalog") {
@@ -2123,7 +2125,8 @@ class DataSourceV2SQLSuite
     val e1 = intercept[AnalysisException] {
       sql("CREATE FUNCTION default.ns1.ns2.fun as 'f'")
     }
-    assert(e1.message.contains("Unsupported function name 'default.ns1.ns2.fun'"))
+    assert(e1.message.contains(
+      "The namespace in session catalog must have exactly one name part: default.ns1.ns2.fun"))
   }
 
   test("global temp view should not be masked by v2 catalog") {
@@ -2160,7 +2163,8 @@ class DataSourceV2SQLSuite
       // the session catalog, not the `gloabl_temp` v2 catalog.
       sql(s"CREATE TABLE $globalTempDB.ns1.ns2.tbl (id bigint, data string) USING json")
     }
-    assert(e.message.contains("global_temp.ns1.ns2.tbl is not a valid TableIdentifier"))
+    assert(e.message.contains(
+      "The namespace in session catalog must have exactly one name part: global_temp.ns1.ns2.tbl"))
   }
 
   test("table name same as catalog can be used") {
@@ -2187,10 +2191,31 @@ class DataSourceV2SQLSuite
     def assertWrongTableIdent(): Unit = {
       withTable("t") {
         sql("CREATE TABLE t USING json AS SELECT 1 AS i")
-        val e = intercept[AnalysisException] {
-          sql("select * from spark_catalog.t")
+
+        val t = "spark_catalog.t"
+        def verify(sql: String): Unit = {
+          val e = intercept[AnalysisException](spark.sql(sql))
+          assert(e.message.contains(
+            s"The namespace in session catalog must have exactly one name part: $t"))
         }
-        assert(e.message.contains("Table or view not found: spark_catalog.t"))
+
+        verify(s"select * from $t")
+        // Verify V1 commands that bypass table lookups.
+        verify(s"REFRESH TABLE $t")
+        verify(s"DESCRIBE $t i")
+        verify(s"DROP TABLE $t")
+        verify(s"DROP VIEW $t")
+        verify(s"ANALYZE TABLE $t COMPUTE STATISTICS")
+        verify(s"ANALYZE TABLE $t COMPUTE STATISTICS FOR ALL COLUMNS")
+        verify(s"MSCK REPAIR TABLE $t")
+        verify(s"LOAD DATA INPATH 'filepath' INTO TABLE $t")
+        verify(s"SHOW CREATE TABLE $t")
+        verify(s"SHOW CREATE TABLE $t AS SERDE")
+        verify(s"CACHE TABLE $t")
+        verify(s"UNCACHE TABLE $t")
+        verify(s"TRUNCATE TABLE $t")
+        verify(s"SHOW PARTITIONS $t")
+        verify(s"SHOW COLUMNS FROM $t")
       }
     }
 
@@ -2334,7 +2359,8 @@ class DataSourceV2SQLSuite
       val e1 = intercept[AnalysisException](
         sql(s"CACHE TABLE $sessionCatalogName.v")
       )
-      assert(e1.message.contains("Table or view not found: default.v"))
+      assert(e1.message.contains(
+        "The namespace in session catalog must have exactly one name part: spark_catalog.v"))
     }
     val e2 = intercept[AnalysisException] {
       sql(s"CREATE TEMP VIEW $sessionCatalogName.v AS SELECT 1")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 01d16ce0bf285..9df3c336b2eee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -128,7 +128,7 @@ class PlanResolutionSuite extends AnalysisTest {
       }
     })
     when(manager.currentCatalog).thenReturn(v2SessionCatalog)
-    when(manager.currentNamespace).thenReturn(Array.empty[String])
+    when(manager.currentNamespace).thenReturn(Array("default"))
     when(manager.v1SessionCatalog).thenReturn(v1SessionCatalog)
     manager
   }
@@ -171,7 +171,7 @@ class PlanResolutionSuite extends AnalysisTest {
         "USING parquet PARTITIONED BY (a)"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType()
@@ -215,7 +215,7 @@ class PlanResolutionSuite extends AnalysisTest {
         "CLUSTERED BY (a) SORTED BY (b) INTO 5 BUCKETS"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -236,7 +236,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val sql = "CREATE TABLE my_tab(a INT, b STRING) USING parquet COMMENT 'abc'"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -256,7 +256,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val sql = "CREATE TABLE my_tab(a INT, b STRING) USING parquet TBLPROPERTIES('test' = 'test')"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty,
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -276,7 +276,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val v1 = "CREATE TABLE my_tab(a INT, b STRING) USING parquet LOCATION '/tmp/file'"
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("my_tab"),
+      identifier = TableIdentifier("my_tab", Some("default")),
       tableType = CatalogTableType.EXTERNAL,
       storage = CatalogStorageFormat.empty.copy(locationUri = Some(new URI("/tmp/file"))),
       schema = new StructType().add("a", IntegerType).add("b", StringType),
@@ -330,7 +330,7 @@ class PlanResolutionSuite extends AnalysisTest {
       """.stripMargin
 
     val expectedTableDesc = CatalogTable(
-      identifier = TableIdentifier("table_name"),
+      identifier = TableIdentifier("table_name", Some("default")),
       tableType = CatalogTableType.MANAGED,
       storage = CatalogStorageFormat.empty.copy(
         properties = Map("a" -> "1", "b" -> "0.1", "c" -> "true")
@@ -621,7 +621,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val tableName1 = "db.tab"
     val tableIdent1 = TableIdentifier("tab", Option("db"))
     val tableName2 = "tab"
-    val tableIdent2 = TableIdentifier("tab", None)
+    val tableIdent2 = TableIdentifier("tab", Some("default"))
 
     parseResolveCompare(s"DROP TABLE $tableName1",
       DropTableCommand(tableIdent1, ifExists = false, isView = false, purge = false))
@@ -657,7 +657,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val viewName1 = "db.view"
     val viewIdent1 = TableIdentifier("view", Option("db"))
     val viewName2 = "view"
-    val viewIdent2 = TableIdentifier("view")
+    val viewIdent2 = TableIdentifier("view", Option("default"))
 
     parseResolveCompare(s"DROP VIEW $viewName1",
       DropTableCommand(viewIdent1, ifExists = false, isView = true, purge = false))
@@ -688,7 +688,7 @@ class PlanResolutionSuite extends AnalysisTest {
     val parsed2_view = parseAndResolve(sql2_view)
     val parsed3_view = parseAndResolve(sql3_view)
 
-    val tableIdent = TableIdentifier("table_name", None)
+    val tableIdent = TableIdentifier("table_name", Some("default"))
     val expected1_view = AlterTableSetPropertiesCommand(
       tableIdent, Map("test" -> "test", "comment" -> "new_comment"), isView = true)
     val expected2_view = AlterTableUnsetPropertiesCommand(
@@ -715,8 +715,8 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed2 = parseAndResolve(sql2)
         val parsed3 = parseAndResolve(sql3)
 
-        val tableIdent = TableIdentifier(tblName, None)
         if (useV1Command) {
+          val tableIdent = TableIdentifier(tblName, Some("default"))
           val expected1 = AlterTableSetPropertiesCommand(
             tableIdent, Map("test" -> "test", "comment" -> "new_comment"), isView = false)
           val expected2 = AlterTableUnsetPropertiesCommand(
@@ -781,7 +781,7 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed = parseAndResolve(sql)
         if (useV1Command) {
           val expected = AlterTableSetPropertiesCommand(
-            TableIdentifier(tblName),
+            TableIdentifier(tblName, Some("default")),
             Map("a" -> "1", "b" -> "0.1", "c" -> "true"),
             isView = false)
 
@@ -806,7 +806,7 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed = parseAndResolve(sql)
         if (useV1Command) {
           val expected = AlterTableSetLocationCommand(
-            TableIdentifier(tblName, None),
+            TableIdentifier(tblName, Some("default")),
             None,
             "new location")
           comparePlans(parsed, expected)
@@ -828,8 +828,10 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed1 = parseAndResolve(sql1)
         val parsed2 = parseAndResolve(sql2)
         if (useV1Command) {
-          val expected1 = DescribeTableCommand(TableIdentifier(tblName, None), Map.empty, false)
-          val expected2 = DescribeTableCommand(TableIdentifier(tblName, None), Map.empty, true)
+          val expected1 = DescribeTableCommand(
+            TableIdentifier(tblName, Some("default")), Map.empty, false)
+          val expected2 = DescribeTableCommand(
+            TableIdentifier(tblName, Some("default")), Map.empty, true)
 
           comparePlans(parsed1, expected1)
           comparePlans(parsed2, expected2)
@@ -851,7 +853,7 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed3 = parseAndResolve(sql3)
         if (useV1Command) {
           val expected3 = DescribeTableCommand(
-            TableIdentifier(tblName, None), Map("a" -> "1"), false)
+            TableIdentifier(tblName, Some("default")), Map("a" -> "1"), false)
           comparePlans(parsed3, expected3)
         } else {
           parsed3 match {
@@ -1019,8 +1021,8 @@ class PlanResolutionSuite extends AnalysisTest {
         val parsed1 = parseAndResolve(sql1)
         val parsed2 = parseAndResolve(sql2)
 
-        val tableIdent = TableIdentifier(tblName, None)
         if (useV1Command) {
+          val tableIdent = TableIdentifier(tblName, Some("default"))
           val oldColumn = StructField("i", IntegerType)
           val newColumn = StructField("i", LongType)
           val expected1 = AlterTableChangeColumnCommand(
@@ -1093,7 +1095,7 @@ class PlanResolutionSuite extends AnalysisTest {
         } else {
           val actual = parseAndResolve(sql)
           val expected = AlterTableChangeColumnCommand(
-            TableIdentifier(tblName, None),
+            TableIdentifier(tblName, Some("default")),
             "I",
             StructField("I", IntegerType).withComment("new comment"))
           comparePlans(actual, expected)
@@ -1125,7 +1127,7 @@ class PlanResolutionSuite extends AnalysisTest {
   }
 
   val DSV2ResolutionTests = {
-    val v2SessionCatalogTable = s"${CatalogManager.SESSION_CATALOG_NAME}.v2Table"
+    val v2SessionCatalogTable = s"${CatalogManager.SESSION_CATALOG_NAME}.default.v2Table"
     Seq(
       ("ALTER TABLE testcat.tab ALTER COLUMN i TYPE bigint", false),
       ("ALTER TABLE tab ALTER COLUMN i TYPE bigint", false),
@@ -1141,7 +1143,7 @@ class PlanResolutionSuite extends AnalysisTest {
       (s"SHOW TBLPROPERTIES $v2SessionCatalogTable", true),
       ("SELECT * from tab", false),
       ("SELECT * from testcat.tab", false),
-      (s"SELECT * from ${CatalogManager.SESSION_CATALOG_NAME}.v2Table", true)
+      (s"SELECT * from $v2SessionCatalogTable", true)
     )
   }
 

From 1a50ee6f6467a9fd50160d09d937b7316595af88 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 4 Mar 2020 20:21:48 +0800
Subject: [PATCH 0214/1280] [SPARK-31017][TEST][CORE] Test for shuffle requests
 packaging with different size and numBlocks limit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

Added 2 tests for `ShuffleBlockFetcherIteratorSuite`.

### Why are the changes needed?

When packaging shuffle fetch requests in `ShuffleBlockFetcherIterator`, there are two limitations: `maxBytesInFlight` and `maxBlocksInFlightPerAddress`. However, we don’t have test cases to test them both, e.g. the size limitation is hit before the numBlocks limitation.

We should add test cases in `ShuffleBlockFetcherIteratorSuite` to test:

1. the size limitation is hit before the numBlocks limitation
2. the numBlocks limitation is hit before the size limitation

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added new tests.

Closes #27767 from Ngone51/add_test.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 87b93d32a6bfb0f2127019b97b3fc1d13e16a10b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../ShuffleBlockFetcherIteratorSuite.scala    | 92 +++++++++++++++++++
 1 file changed, 92 insertions(+)

diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
index 45f47c7c49bca..2090a5155a6f1 100644
--- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
@@ -254,6 +254,98 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     intercept[FetchFailedException] { iterator.next() }
   }
 
+  test("Hit maxBytesInFlight limitation before maxBlocksInFlightPerAddress") {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+
+    val remoteBmId1 = BlockManagerId("test-remote-client-1", "test-remote-host1", 1)
+    val remoteBmId2 = BlockManagerId("test-remote-client-2", "test-remote-host2", 2)
+    val blockId1 = ShuffleBlockId(0, 1, 0)
+    val blockId2 = ShuffleBlockId(1, 1, 0)
+    val blocksByAddress = Seq(
+      (remoteBmId1, Seq((blockId1, 1000L, 0))),
+      (remoteBmId2, Seq((blockId2, 1000L, 0)))).toIterator
+    val transfer = createMockTransfer(Map(
+      blockId1 -> createMockManagedBuffer(1000),
+      blockId2 -> createMockManagedBuffer(1000)))
+    val taskContext = TaskContext.empty()
+    val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+    val iterator = new ShuffleBlockFetcherIterator(
+      taskContext,
+      transfer,
+      blockManager,
+      blocksByAddress,
+      (_, in) => in,
+      1000L, // allow 1 FetchRequests at most at the same time
+      Int.MaxValue,
+      Int.MaxValue, // set maxBlocksInFlightPerAddress to Int.MaxValue
+      Int.MaxValue,
+      true,
+      false,
+      metrics,
+      false)
+    // After initialize() we'll have 2 FetchRequests and each is 1000 bytes. So only the
+    // first FetchRequests can be sent, and the second one will hit maxBytesInFlight so
+    // it won't be sent.
+    verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(iterator.hasNext)
+    // next() will trigger off sending deferred request
+    iterator.next()
+    // the second FetchRequest should be sent at this time
+    verify(transfer, times(2)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(iterator.hasNext)
+    iterator.next()
+    assert(!iterator.hasNext)
+  }
+
+  test("Hit maxBlocksInFlightPerAddress limitation before maxBytesInFlight") {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+
+    val remoteBmId = BlockManagerId("test-remote-client-1", "test-remote-host", 2)
+    val blockId1 = ShuffleBlockId(0, 1, 0)
+    val blockId2 = ShuffleBlockId(0, 2, 0)
+    val blockId3 = ShuffleBlockId(0, 3, 0)
+    val blocksByAddress = Seq((remoteBmId,
+      Seq((blockId1, 1000L, 0), (blockId2, 1000L, 0), (blockId3, 1000L, 0)))).toIterator
+    val transfer = createMockTransfer(Map(
+      blockId1 -> createMockManagedBuffer(),
+      blockId2 -> createMockManagedBuffer(),
+      blockId3 -> createMockManagedBuffer()))
+    val taskContext = TaskContext.empty()
+    val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+    val iterator = new ShuffleBlockFetcherIterator(
+      taskContext,
+      transfer,
+      blockManager,
+      blocksByAddress,
+      (_, in) => in,
+      Int.MaxValue, // set maxBytesInFlight to Int.MaxValue
+      Int.MaxValue,
+      2, // set maxBlocksInFlightPerAddress to 2
+      Int.MaxValue,
+      true,
+      false,
+      metrics,
+      false)
+    // After initialize(), we'll have 2 FetchRequests that one has 2 blocks inside and another one
+    // has only one block. So only the first FetchRequest can be sent. The second FetchRequest will
+    // hit maxBlocksInFlightPerAddress so it won't be sent.
+    verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    // the first request packaged 2 blocks, so we also need to
+    // call next() for 2 times to exhaust the iterator.
+    assert(iterator.hasNext)
+    iterator.next()
+    assert(iterator.hasNext)
+    iterator.next()
+    verify(transfer, times(2)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(iterator.hasNext)
+    iterator.next()
+    assert(!iterator.hasNext)
+  }
+
   test("fetch continuous blocks in batch successful 3 local + 4 host local + 2 remote reads") {
     val blockManager = mock(classOf[BlockManager])
     val localBmId = BlockManagerId("test-client", "test-local-host", 1)

From 70c86e6b166434ce6d3420f122a33d933728fa91 Mon Sep 17 00:00:00 2001
From: DB Tsai <d_tsai@apple.com>
Date: Wed, 4 Mar 2020 23:41:49 +0900
Subject: [PATCH 0215/1280] [SPARK-31027][SQL] Refactor DataSourceStrategy to
 be more extendable

### What changes were proposed in this pull request?
Refactor `DataSourceStrategy.scala` and `DataSourceStrategySuite.scala` so it's more extendable to implement nested predicate pushdown.

### Why are the changes needed?
To support nested predicate pushdown.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing tests and new tests.

Closes #27778 from dbtsai/SPARK-31027.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../datasources/DataSourceStrategy.scala      | 105 ++++++++-------
 .../datasources/DataSourceStrategySuite.scala | 121 +++++++++++-------
 2 files changed, 133 insertions(+), 93 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index 2d902b5c43c44..1641b660a271d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -449,60 +449,60 @@ object DataSourceStrategy {
   }
 
   private def translateLeafNodeFilter(predicate: Expression): Option[Filter] = predicate match {
-    case expressions.EqualTo(a: Attribute, Literal(v, t)) =>
-      Some(sources.EqualTo(a.name, convertToScala(v, t)))
-    case expressions.EqualTo(Literal(v, t), a: Attribute) =>
-      Some(sources.EqualTo(a.name, convertToScala(v, t)))
-
-    case expressions.EqualNullSafe(a: Attribute, Literal(v, t)) =>
-      Some(sources.EqualNullSafe(a.name, convertToScala(v, t)))
-    case expressions.EqualNullSafe(Literal(v, t), a: Attribute) =>
-      Some(sources.EqualNullSafe(a.name, convertToScala(v, t)))
-
-    case expressions.GreaterThan(a: Attribute, Literal(v, t)) =>
-      Some(sources.GreaterThan(a.name, convertToScala(v, t)))
-    case expressions.GreaterThan(Literal(v, t), a: Attribute) =>
-      Some(sources.LessThan(a.name, convertToScala(v, t)))
-
-    case expressions.LessThan(a: Attribute, Literal(v, t)) =>
-      Some(sources.LessThan(a.name, convertToScala(v, t)))
-    case expressions.LessThan(Literal(v, t), a: Attribute) =>
-      Some(sources.GreaterThan(a.name, convertToScala(v, t)))
-
-    case expressions.GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
-      Some(sources.GreaterThanOrEqual(a.name, convertToScala(v, t)))
-    case expressions.GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
-      Some(sources.LessThanOrEqual(a.name, convertToScala(v, t)))
-
-    case expressions.LessThanOrEqual(a: Attribute, Literal(v, t)) =>
-      Some(sources.LessThanOrEqual(a.name, convertToScala(v, t)))
-    case expressions.LessThanOrEqual(Literal(v, t), a: Attribute) =>
-      Some(sources.GreaterThanOrEqual(a.name, convertToScala(v, t)))
-
-    case expressions.InSet(a: Attribute, set) =>
-      val toScala = CatalystTypeConverters.createToScalaConverter(a.dataType)
-      Some(sources.In(a.name, set.toArray.map(toScala)))
+    case expressions.EqualTo(PushableColumn(name), Literal(v, t)) =>
+      Some(sources.EqualTo(name, convertToScala(v, t)))
+    case expressions.EqualTo(Literal(v, t), PushableColumn(name)) =>
+      Some(sources.EqualTo(name, convertToScala(v, t)))
+
+    case expressions.EqualNullSafe(PushableColumn(name), Literal(v, t)) =>
+      Some(sources.EqualNullSafe(name, convertToScala(v, t)))
+    case expressions.EqualNullSafe(Literal(v, t), PushableColumn(name)) =>
+      Some(sources.EqualNullSafe(name, convertToScala(v, t)))
+
+    case expressions.GreaterThan(PushableColumn(name), Literal(v, t)) =>
+      Some(sources.GreaterThan(name, convertToScala(v, t)))
+    case expressions.GreaterThan(Literal(v, t), PushableColumn(name)) =>
+      Some(sources.LessThan(name, convertToScala(v, t)))
+
+    case expressions.LessThan(PushableColumn(name), Literal(v, t)) =>
+      Some(sources.LessThan(name, convertToScala(v, t)))
+    case expressions.LessThan(Literal(v, t), PushableColumn(name)) =>
+      Some(sources.GreaterThan(name, convertToScala(v, t)))
+
+    case expressions.GreaterThanOrEqual(PushableColumn(name), Literal(v, t)) =>
+      Some(sources.GreaterThanOrEqual(name, convertToScala(v, t)))
+    case expressions.GreaterThanOrEqual(Literal(v, t), PushableColumn(name)) =>
+      Some(sources.LessThanOrEqual(name, convertToScala(v, t)))
+
+    case expressions.LessThanOrEqual(PushableColumn(name), Literal(v, t)) =>
+      Some(sources.LessThanOrEqual(name, convertToScala(v, t)))
+    case expressions.LessThanOrEqual(Literal(v, t), PushableColumn(name)) =>
+      Some(sources.GreaterThanOrEqual(name, convertToScala(v, t)))
+
+    case expressions.InSet(e @ PushableColumn(name), set) =>
+      val toScala = CatalystTypeConverters.createToScalaConverter(e.dataType)
+      Some(sources.In(name, set.toArray.map(toScala)))
 
     // Because we only convert In to InSet in Optimizer when there are more than certain
     // items. So it is possible we still get an In expression here that needs to be pushed
     // down.
-    case expressions.In(a: Attribute, list) if list.forall(_.isInstanceOf[Literal]) =>
+    case expressions.In(e @ PushableColumn(name), list) if list.forall(_.isInstanceOf[Literal]) =>
       val hSet = list.map(_.eval(EmptyRow))
-      val toScala = CatalystTypeConverters.createToScalaConverter(a.dataType)
-      Some(sources.In(a.name, hSet.toArray.map(toScala)))
+      val toScala = CatalystTypeConverters.createToScalaConverter(e.dataType)
+      Some(sources.In(name, hSet.toArray.map(toScala)))
 
-    case expressions.IsNull(a: Attribute) =>
-      Some(sources.IsNull(a.name))
-    case expressions.IsNotNull(a: Attribute) =>
-      Some(sources.IsNotNull(a.name))
-    case expressions.StartsWith(a: Attribute, Literal(v: UTF8String, StringType)) =>
-      Some(sources.StringStartsWith(a.name, v.toString))
+    case expressions.IsNull(PushableColumn(name)) =>
+      Some(sources.IsNull(name))
+    case expressions.IsNotNull(PushableColumn(name)) =>
+      Some(sources.IsNotNull(name))
+    case expressions.StartsWith(PushableColumn(name), Literal(v: UTF8String, StringType)) =>
+      Some(sources.StringStartsWith(name, v.toString))
 
-    case expressions.EndsWith(a: Attribute, Literal(v: UTF8String, StringType)) =>
-      Some(sources.StringEndsWith(a.name, v.toString))
+    case expressions.EndsWith(PushableColumn(name), Literal(v: UTF8String, StringType)) =>
+      Some(sources.StringEndsWith(name, v.toString))
 
-    case expressions.Contains(a: Attribute, Literal(v: UTF8String, StringType)) =>
-      Some(sources.StringContains(a.name, v.toString))
+    case expressions.Contains(PushableColumn(name), Literal(v: UTF8String, StringType)) =>
+      Some(sources.StringContains(name, v.toString))
 
     case expressions.Literal(true, BooleanType) =>
       Some(sources.AlwaysTrue)
@@ -646,3 +646,16 @@ object DataSourceStrategy {
     }
   }
 }
+
+/**
+ * Find the column name of an expression that can be pushed down.
+ */
+object PushableColumn {
+  def unapply(e: Expression): Option[String] = {
+    def helper(e: Expression) = e match {
+      case a: Attribute => Some(a.name)
+      case _ => None
+    }
+    helper(e)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
index b76db70494cf8..7bd3213b378ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
@@ -22,68 +22,82 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.sources
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
 
 class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
+  val attrInts = Seq(
+    'cint.int
+  ).zip(Seq(
+    "cint"
+  ))
 
-  test("translate simple expression") {
-    val attrInt = 'cint.int
-    val attrStr = 'cstr.string
+  val attrStrs = Seq(
+    'cstr.string
+  ).zip(Seq(
+    "cstr"
+  ))
+
+  test("translate simple expression") { attrInts.zip(attrStrs)
+    .foreach { case ((attrInt, intColName), (attrStr, strColName)) =>
 
-    testTranslateFilter(EqualTo(attrInt, 1), Some(sources.EqualTo("cint", 1)))
-    testTranslateFilter(EqualTo(1, attrInt), Some(sources.EqualTo("cint", 1)))
+    testTranslateFilter(EqualTo(attrInt, 1), Some(sources.EqualTo(intColName, 1)))
+    testTranslateFilter(EqualTo(1, attrInt), Some(sources.EqualTo(intColName, 1)))
 
     testTranslateFilter(EqualNullSafe(attrStr, Literal(null)),
-      Some(sources.EqualNullSafe("cstr", null)))
+      Some(sources.EqualNullSafe(strColName, null)))
     testTranslateFilter(EqualNullSafe(Literal(null), attrStr),
-      Some(sources.EqualNullSafe("cstr", null)))
+      Some(sources.EqualNullSafe(strColName, null)))
 
-    testTranslateFilter(GreaterThan(attrInt, 1), Some(sources.GreaterThan("cint", 1)))
-    testTranslateFilter(GreaterThan(1, attrInt), Some(sources.LessThan("cint", 1)))
+    testTranslateFilter(GreaterThan(attrInt, 1), Some(sources.GreaterThan(intColName, 1)))
+    testTranslateFilter(GreaterThan(1, attrInt), Some(sources.LessThan(intColName, 1)))
 
-    testTranslateFilter(LessThan(attrInt, 1), Some(sources.LessThan("cint", 1)))
-    testTranslateFilter(LessThan(1, attrInt), Some(sources.GreaterThan("cint", 1)))
+    testTranslateFilter(LessThan(attrInt, 1), Some(sources.LessThan(intColName, 1)))
+    testTranslateFilter(LessThan(1, attrInt), Some(sources.GreaterThan(intColName, 1)))
 
-    testTranslateFilter(GreaterThanOrEqual(attrInt, 1), Some(sources.GreaterThanOrEqual("cint", 1)))
-    testTranslateFilter(GreaterThanOrEqual(1, attrInt), Some(sources.LessThanOrEqual("cint", 1)))
+    testTranslateFilter(GreaterThanOrEqual(attrInt, 1),
+      Some(sources.GreaterThanOrEqual(intColName, 1)))
+    testTranslateFilter(GreaterThanOrEqual(1, attrInt),
+      Some(sources.LessThanOrEqual(intColName, 1)))
 
-    testTranslateFilter(LessThanOrEqual(attrInt, 1), Some(sources.LessThanOrEqual("cint", 1)))
-    testTranslateFilter(LessThanOrEqual(1, attrInt), Some(sources.GreaterThanOrEqual("cint", 1)))
+    testTranslateFilter(LessThanOrEqual(attrInt, 1),
+      Some(sources.LessThanOrEqual(intColName, 1)))
+    testTranslateFilter(LessThanOrEqual(1, attrInt),
+      Some(sources.GreaterThanOrEqual(intColName, 1)))
 
-    testTranslateFilter(InSet(attrInt, Set(1, 2, 3)), Some(sources.In("cint", Array(1, 2, 3))))
+    testTranslateFilter(InSet(attrInt, Set(1, 2, 3)), Some(sources.In(intColName, Array(1, 2, 3))))
 
-    testTranslateFilter(In(attrInt, Seq(1, 2, 3)), Some(sources.In("cint", Array(1, 2, 3))))
+    testTranslateFilter(In(attrInt, Seq(1, 2, 3)), Some(sources.In(intColName, Array(1, 2, 3))))
 
-    testTranslateFilter(IsNull(attrInt), Some(sources.IsNull("cint")))
-    testTranslateFilter(IsNotNull(attrInt), Some(sources.IsNotNull("cint")))
+    testTranslateFilter(IsNull(attrInt), Some(sources.IsNull(intColName)))
+    testTranslateFilter(IsNotNull(attrInt), Some(sources.IsNotNull(intColName)))
 
     // cint > 1 AND cint < 10
     testTranslateFilter(And(
       GreaterThan(attrInt, 1),
       LessThan(attrInt, 10)),
       Some(sources.And(
-        sources.GreaterThan("cint", 1),
-        sources.LessThan("cint", 10))))
+        sources.GreaterThan(intColName, 1),
+        sources.LessThan(intColName, 10))))
 
     // cint >= 8 OR cint <= 2
     testTranslateFilter(Or(
       GreaterThanOrEqual(attrInt, 8),
       LessThanOrEqual(attrInt, 2)),
       Some(sources.Or(
-        sources.GreaterThanOrEqual("cint", 8),
-        sources.LessThanOrEqual("cint", 2))))
+        sources.GreaterThanOrEqual(intColName, 8),
+        sources.LessThanOrEqual(intColName, 2))))
 
     testTranslateFilter(Not(GreaterThanOrEqual(attrInt, 8)),
-      Some(sources.Not(sources.GreaterThanOrEqual("cint", 8))))
+      Some(sources.Not(sources.GreaterThanOrEqual(intColName, 8))))
 
-    testTranslateFilter(StartsWith(attrStr, "a"), Some(sources.StringStartsWith("cstr", "a")))
+    testTranslateFilter(StartsWith(attrStr, "a"), Some(sources.StringStartsWith(strColName, "a")))
 
-    testTranslateFilter(EndsWith(attrStr, "a"), Some(sources.StringEndsWith("cstr", "a")))
+    testTranslateFilter(EndsWith(attrStr, "a"), Some(sources.StringEndsWith(strColName, "a")))
 
-    testTranslateFilter(Contains(attrStr, "a"), Some(sources.StringContains("cstr", "a")))
-  }
+    testTranslateFilter(Contains(attrStr, "a"), Some(sources.StringContains(strColName, "a")))
+  }}
 
-  test("translate complex expression") {
-    val attrInt = 'cint.int
+  test("translate complex expression") { attrInts.foreach { case (attrInt, intColName) =>
 
     // ABS(cint) - 2 <= 1
     testTranslateFilter(LessThanOrEqual(
@@ -102,11 +116,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         LessThan(attrInt, 100))),
       Some(sources.Or(
         sources.And(
-          sources.GreaterThan("cint", 1),
-          sources.LessThan("cint", 10)),
+          sources.GreaterThan(intColName, 1),
+          sources.LessThan(intColName, 10)),
         sources.And(
-          sources.GreaterThan("cint", 50),
-          sources.LessThan("cint", 100)))))
+          sources.GreaterThan(intColName, 50),
+          sources.LessThan(intColName, 100)))))
 
     // SPARK-22548 Incorrect nested AND expression pushed down to JDBC data source
     // (cint > 1 AND ABS(cint) < 10) OR (cint < 50 AND cint > 100)
@@ -142,11 +156,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         LessThan(attrInt, -10))),
       Some(sources.Or(
         sources.Or(
-          sources.EqualTo("cint", 1),
-          sources.EqualTo("cint", 10)),
+          sources.EqualTo(intColName, 1),
+          sources.EqualTo(intColName, 10)),
         sources.Or(
-          sources.GreaterThan("cint", 0),
-          sources.LessThan("cint", -10)))))
+          sources.GreaterThan(intColName, 0),
+          sources.LessThan(intColName, -10)))))
 
     // (cint = 1 OR ABS(cint) = 10) OR (cint > 0 OR cint < -10)
     testTranslateFilter(Or(
@@ -173,11 +187,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         IsNotNull(attrInt))),
       Some(sources.And(
         sources.And(
-          sources.GreaterThan("cint", 1),
-          sources.LessThan("cint", 10)),
+          sources.GreaterThan(intColName, 1),
+          sources.LessThan(intColName, 10)),
         sources.And(
-          sources.EqualTo("cint", 6),
-          sources.IsNotNull("cint")))))
+          sources.EqualTo(intColName, 6),
+          sources.IsNotNull(intColName)))))
 
     // (cint > 1 AND cint < 10) AND (ABS(cint) = 6 AND cint IS NOT NULL)
     testTranslateFilter(And(
@@ -201,11 +215,11 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         IsNotNull(attrInt))),
       Some(sources.And(
         sources.Or(
-          sources.GreaterThan("cint", 1),
-          sources.LessThan("cint", 10)),
+          sources.GreaterThan(intColName, 1),
+          sources.LessThan(intColName, 10)),
         sources.Or(
-          sources.EqualTo("cint", 6),
-          sources.IsNotNull("cint")))))
+          sources.EqualTo(intColName, 6),
+          sources.IsNotNull(intColName)))))
 
     // (cint > 1 OR cint < 10) AND (cint = 6 OR cint IS NOT NULL)
     testTranslateFilter(And(
@@ -217,7 +231,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
         // Functions such as 'Abs' are not supported
         EqualTo(Abs(attrInt), 6),
         IsNotNull(attrInt))), None)
-  }
+  }}
 
   test("SPARK-26865 DataSourceV2Strategy should push normalized filters") {
     val attrInt = 'cint.int
@@ -226,6 +240,19 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-31027 test `PushableColumn.unapply` that finds the column name of " +
+    "an expression that can be pushed down") {
+    attrInts.foreach { case (attrInt, colName) =>
+      assert(PushableColumn.unapply(attrInt) === Some(colName))
+    }
+    attrStrs.foreach { case (attrStr, colName) =>
+      assert(PushableColumn.unapply(attrStr) === Some(colName))
+    }
+
+    // `Abs(col)` can not be pushed down, so it returns `None`
+    assert(PushableColumn.unapply(Abs('col.int)) === None)
+  }
+
   /**
    * Translate the given Catalyst [[Expression]] into data source [[sources.Filter]]
    * then verify against the given [[sources.Filter]].

From 104a768e242bf5399bce642b9c6295476d9cdad8 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 4 Mar 2020 20:37:51 -0800
Subject: [PATCH 0216/1280] [MINOR][CORE] Expose the alias -c flag of --conf
 for spark-submit

### What changes were proposed in this pull request?

-c is short for --conf, it was introduced since v1.1.0 but hidden from users until now

### Why are the changes needed?

### Does this PR introduce any user-facing change?

no

expose hidden feature

### How was this patch tested?

Nah

Closes #27802 from yaooqinn/conf.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3edab6cc1d70c102093e973a2cf97208db19be8c)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/deploy/SparkSubmitArguments.scala    | 2 +-
 docs/configuration.md                                           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 3f7cfea778ac6..3090a3b10a97c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -513,7 +513,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
         |                              directory of each executor. File paths of these files
         |                              in executors can be accessed via SparkFiles.get(fileName).
         |
-        |  --conf PROP=VALUE           Arbitrary Spark configuration property.
+        |  --conf, -c PROP=VALUE       Arbitrary Spark configuration property.
         |  --properties-file FILE      Path to a file from which to load extra properties. If not
         |                              specified, this will look for conf/spark-defaults.conf.
         |
diff --git a/docs/configuration.md b/docs/configuration.md
index 5e6fe9349e59d..f7b7e1671a8ab 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -95,7 +95,7 @@ Then, you can supply configuration values at runtime:
 
 The Spark shell and [`spark-submit`](submitting-applications.html)
 tool support two ways to load configurations dynamically. The first is command line options,
-such as `--master`, as shown above. `spark-submit` can accept any Spark property using the `--conf`
+such as `--master`, as shown above. `spark-submit` can accept any Spark property using the `--conf/-c`
 flag, but uses special flags for properties that play a part in launching the Spark application.
 Running `./bin/spark-submit --help` will show the entire list of these options.
 

From 253fbd54e7fe3dda0672ebc7143b943af8387a2c Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Thu, 5 Mar 2020 15:29:39 +0800
Subject: [PATCH 0217/1280] [SPARK-30668][SQL][FOLLOWUP] Raise exception
 instead of silent change for new DateFormatter

This is a follow-up work for #27441. For the cases of new TimestampFormatter return null while legacy formatter can return a value, we need to throw an exception instead of silent change. The legacy config will be referenced in the error message.

Avoid silent result change for new behavior in 3.0.

Yes, an exception is thrown when we detect legacy formatter can parse the string and the new formatter return null.

Extend existing UT.

Closes #27537 from xuanyuanking/SPARK-30668-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 7db0af578585ecaeee9fd23f8189292289b52a97)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/SparkException.scala     |  7 ++
 .../sql/catalyst/csv/UnivocityParser.scala    |  2 +
 .../expressions/datetimeExpressions.scala     |  3 +
 .../sql/catalyst/json/JacksonParser.scala     |  2 +
 .../sql/catalyst/util/DateFormatter.scala     | 60 +++++++++------
 .../util/DateTimeFormatterHelper.scala        | 26 ++++++-
 .../catalyst/util/TimestampFormatter.scala    | 73 ++++++++++++-------
 .../apache/spark/sql/internal/SQLConf.scala   | 16 +++-
 .../expressions/DateExpressionsSuite.scala    | 39 +++++++---
 .../catalyst/json/JsonInferSchemaSuite.scala  | 16 ++--
 .../apache/spark/sql/CsvFunctionsSuite.scala  | 20 ++++-
 .../apache/spark/sql/DateFunctionsSuite.scala | 58 +++++++++------
 .../adaptive/AdaptiveQueryExecSuite.scala     |  2 +-
 .../execution/datasources/csv/CSVSuite.scala  | 22 +++++-
 .../datasources/json/JsonSuite.scala          | 22 +++++-
 15 files changed, 271 insertions(+), 97 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala b/core/src/main/scala/org/apache/spark/SparkException.scala
index 4ad9a0cc4b103..81c087e314be1 100644
--- a/core/src/main/scala/org/apache/spark/SparkException.scala
+++ b/core/src/main/scala/org/apache/spark/SparkException.scala
@@ -43,3 +43,10 @@ private[spark] case class SparkUserAppException(exitCode: Int)
  */
 private[spark] case class ExecutorDeadException(message: String)
   extends SparkException(message)
+
+/**
+ * Exception thrown when Spark returns different result after upgrading to a new version.
+ */
+private[spark] class SparkUpgradeException(version: String, message: String, cause: Throwable)
+  extends SparkException("You may get a different result due to the upgrading of Spark" +
+    s" $version: $message", cause)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index f829e6b503ab4..dd8537b02935e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -23,6 +23,7 @@ import scala.util.control.NonFatal
 
 import com.univocity.parsers.csv.CsvParser
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{ExprUtils, GenericInternalRow}
@@ -285,6 +286,7 @@ class UnivocityParser(
           }
         }
       } catch {
+        case e: SparkUpgradeException => throw e
         case NonFatal(e) =>
           badRecordException = badRecordException.orElse(Some(e))
           row.setNullAt(i)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 05074d97efa44..d92d7000eace3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -26,6 +26,7 @@ import scala.util.control.NonFatal
 
 import org.apache.commons.text.StringEscapeUtils
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen._
@@ -787,6 +788,7 @@ abstract class ToTimestamp
               formatter.parse(
                 t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
+              case e: SparkUpgradeException => throw e
               case NonFatal(_) => null
             }
           }
@@ -800,6 +802,7 @@ abstract class ToTimestamp
               TimestampFormatter(formatString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
                 .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
+              case e: SparkUpgradeException => throw e
               case NonFatal(_) => null
             }
           }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index da3b5013fe12d..d0db06cae8161 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -25,6 +25,7 @@ import scala.util.control.NonFatal
 
 import com.fasterxml.jackson.core._
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
@@ -382,6 +383,7 @@ class JacksonParser(
           try {
             row.update(index, fieldConverters(index).apply(parser))
           } catch {
+            case e: SparkUpgradeException => throw e
             case NonFatal(e) =>
               badRecordException = badRecordException.orElse(Some(e))
               parser.skipChildren()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 2cf82d1cfa177..caf58c1011f2e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -23,8 +23,9 @@ import java.util.{Date, Locale}
 
 import org.apache.commons.lang3.time.FastDateFormat
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{convertSpecialDate, localDateToDays}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 sealed trait DateFormatter extends Serializable {
   def parse(s: String): Int // returns days since epoch
@@ -34,16 +35,24 @@ sealed trait DateFormatter extends Serializable {
 class Iso8601DateFormatter(
     pattern: String,
     zoneId: ZoneId,
-    locale: Locale) extends DateFormatter with DateTimeFormatterHelper {
+    locale: Locale,
+    legacyFormat: LegacyDateFormats.LegacyDateFormat)
+  extends DateFormatter with DateTimeFormatterHelper {
 
   @transient
   private lazy val formatter = getOrCreateFormatter(pattern, locale)
 
+  @transient
+  private lazy val legacyFormatter = DateFormatter.getLegacyFormatter(
+    pattern, zoneId, locale, legacyFormat)
+
   override def parse(s: String): Int = {
     val specialDate = convertSpecialDate(s.trim, zoneId)
     specialDate.getOrElse {
-      val localDate = LocalDate.parse(s, formatter)
-      localDateToDays(localDate)
+      try {
+        val localDate = LocalDate.parse(s, formatter)
+        localDateToDays(localDate)
+      } catch checkDiffResult(s, legacyFormatter.parse)
     }
   }
 
@@ -87,33 +96,40 @@ object DateFormatter {
   val defaultLocale: Locale = Locale.US
 
   def defaultPattern(): String = {
-    if (SQLConf.get.legacyTimeParserEnabled) "yyyy-MM-dd" else "uuuu-MM-dd"
+    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) "yyyy-MM-dd" else "uuuu-MM-dd"
   }
 
   private def getFormatter(
-    format: Option[String],
-    zoneId: ZoneId,
-    locale: Locale = defaultLocale,
-    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): DateFormatter = {
-
+      format: Option[String],
+      zoneId: ZoneId,
+      locale: Locale = defaultLocale,
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): DateFormatter = {
     val pattern = format.getOrElse(defaultPattern)
-    if (SQLConf.get.legacyTimeParserEnabled) {
-      legacyFormat match {
-        case FAST_DATE_FORMAT =>
-          new LegacyFastDateFormatter(pattern, locale)
-        case SIMPLE_DATE_FORMAT | LENIENT_SIMPLE_DATE_FORMAT =>
-          new LegacySimpleDateFormatter(pattern, locale)
-      }
+    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
+      getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      new Iso8601DateFormatter(pattern, zoneId, locale)
+      new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat)
+    }
+  }
+
+  def getLegacyFormatter(
+      pattern: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat): DateFormatter = {
+    legacyFormat match {
+      case FAST_DATE_FORMAT =>
+        new LegacyFastDateFormatter(pattern, locale)
+      case SIMPLE_DATE_FORMAT | LENIENT_SIMPLE_DATE_FORMAT =>
+        new LegacySimpleDateFormatter(pattern, locale)
     }
   }
 
   def apply(
-    format: String,
-    zoneId: ZoneId,
-    locale: Locale,
-    legacyFormat: LegacyDateFormat): DateFormatter = {
+      format: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat): DateFormatter = {
     getFormatter(Some(format), zoneId, locale, legacyFormat)
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index a7b6309baf614..33aa733f26459 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -19,13 +19,16 @@ package org.apache.spark.sql.catalyst.util
 
 import java.time._
 import java.time.chrono.IsoChronology
-import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle}
+import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, DateTimeParseException, ResolverStyle}
 import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
 import java.util.Locale
 
 import com.google.common.cache.CacheBuilder
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 trait DateTimeFormatterHelper {
   // Converts the parsed temporal object to ZonedDateTime. It sets time components to zeros
@@ -57,6 +60,27 @@ trait DateTimeFormatterHelper {
     }
     formatter
   }
+
+  // When legacy time parser policy set to EXCEPTION, check whether we will get different results
+  // between legacy parser and new parser. If new parser fails but legacy parser works, throw a
+  // SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
+  // DateTimeParseException will address by the caller side.
+  protected def checkDiffResult[T](
+      s: String, legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
+    case e: DateTimeParseException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION =>
+      val res = try {
+        Some(legacyParseFunc(s))
+      } catch {
+        case _: Throwable => None
+      }
+      if (res.nonEmpty) {
+        throw new SparkUpgradeException("3.0", s"Fail to parse '$s' in the new parser. You can " +
+          s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
+          s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
+      } else {
+        throw e
+      }
+  }
 }
 
 private object DateTimeFormatterHelper {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index fd77326d6b016..866f81bfff4fb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -29,7 +29,9 @@ import org.apache.commons.lang3.time.FastDateFormat
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.LegacyDateFormats.{LegacyDateFormat, LENIENT_SIMPLE_DATE_FORMAT}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.types.Decimal
 
 sealed trait TimestampFormatter extends Serializable {
@@ -52,21 +54,29 @@ sealed trait TimestampFormatter extends Serializable {
 class Iso8601TimestampFormatter(
     pattern: String,
     zoneId: ZoneId,
-    locale: Locale) extends TimestampFormatter with DateTimeFormatterHelper {
+    locale: Locale,
+    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT)
+  extends TimestampFormatter with DateTimeFormatterHelper {
   @transient
   protected lazy val formatter = getOrCreateFormatter(pattern, locale)
 
+  @transient
+  protected lazy val legacyFormatter = TimestampFormatter.getLegacyFormatter(
+    pattern, zoneId, locale, legacyFormat)
+
   override def parse(s: String): Long = {
     val specialDate = convertSpecialTimestamp(s.trim, zoneId)
     specialDate.getOrElse {
-      val parsed = formatter.parse(s)
-      val parsedZoneId = parsed.query(TemporalQueries.zone())
-      val timeZoneId = if (parsedZoneId == null) zoneId else parsedZoneId
-      val zonedDateTime = toZonedDateTime(parsed, timeZoneId)
-      val epochSeconds = zonedDateTime.toEpochSecond
-      val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
-
-      Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond)
+      try {
+        val parsed = formatter.parse(s)
+        val parsedZoneId = parsed.query(TemporalQueries.zone())
+        val timeZoneId = if (parsedZoneId == null) zoneId else parsedZoneId
+        val zonedDateTime = toZonedDateTime(parsed, timeZoneId)
+        val epochSeconds = zonedDateTime.toEpochSecond
+        val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
+
+        Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond)
+      } catch checkDiffResult(s, legacyFormatter.parse)
     }
   }
 
@@ -186,31 +196,38 @@ object TimestampFormatter {
   def defaultPattern(): String = s"${DateFormatter.defaultPattern()} HH:mm:ss"
 
   private def getFormatter(
-    format: Option[String],
-    zoneId: ZoneId,
-    locale: Locale = defaultLocale,
-    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): TimestampFormatter = {
-
+      format: Option[String],
+      zoneId: ZoneId,
+      locale: Locale = defaultLocale,
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): TimestampFormatter = {
     val pattern = format.getOrElse(defaultPattern)
-    if (SQLConf.get.legacyTimeParserEnabled) {
-      legacyFormat match {
-        case FAST_DATE_FORMAT =>
-          new LegacyFastTimestampFormatter(pattern, zoneId, locale)
-        case SIMPLE_DATE_FORMAT =>
-          new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = false)
-        case LENIENT_SIMPLE_DATE_FORMAT =>
-          new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = true)
-      }
+    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
+      getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      new Iso8601TimestampFormatter(pattern, zoneId, locale)
+      new Iso8601TimestampFormatter(pattern, zoneId, locale, legacyFormat)
+    }
+  }
+
+  def getLegacyFormatter(
+      pattern: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat): TimestampFormatter = {
+    legacyFormat match {
+      case FAST_DATE_FORMAT =>
+        new LegacyFastTimestampFormatter(pattern, zoneId, locale)
+      case SIMPLE_DATE_FORMAT =>
+        new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = false)
+      case LENIENT_SIMPLE_DATE_FORMAT =>
+        new LegacySimpleTimestampFormatter(pattern, zoneId, locale, lenient = true)
     }
   }
 
   def apply(
-    format: String,
-    zoneId: ZoneId,
-    locale: Locale,
-    legacyFormat: LegacyDateFormat): TimestampFormatter = {
+      format: String,
+      zoneId: ZoneId,
+      locale: Locale,
+      legacyFormat: LegacyDateFormat): TimestampFormatter = {
     getFormatter(Some(format), zoneId, locale, legacyFormat)
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 7718dd2822974..e39d0661e914f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2116,6 +2116,18 @@ object SQLConf {
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
     .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
+  val LEGACY_TIME_PARSER_POLICY = buildConf("spark.sql.legacy.timeParserPolicy")
+    .internal()
+    .doc("When LEGACY, java.text.SimpleDateFormat is used for formatting and parsing " +
+      "dates/timestamps in a locale-sensitive manner, which is the approach before Spark 3.0. " +
+      "When set to CORRECTED, classes from java.time.* packages are used for the same purpose. " +
+      "The default value is EXCEPTION, RuntimeException is thrown when we will get different " +
+      "results.")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+    .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
   val LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC =
     buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
       .internal()
@@ -2496,7 +2508,9 @@ class SQLConf extends Serializable with Logging {
   def legacyMsSqlServerNumericMappingEnabled: Boolean =
     getConf(LEGACY_MSSQLSERVER_NUMERIC_MAPPING_ENABLED)
 
-  def legacyTimeParserEnabled: Boolean = getConf(SQLConf.LEGACY_TIME_PARSER_ENABLED)
+  def legacyTimeParserPolicy: LegacyBehaviorPolicy.Value = {
+    LegacyBehaviorPolicy.withName(getConf(SQLConf.LEGACY_TIME_PARSER_POLICY))
+  }
 
   /**
    * Returns the [[Resolver]] for the current configuration, which can be used to determine if two
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index f1ad75321776d..4a04fbba678ed 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -23,7 +23,7 @@ import java.time.{Instant, LocalDate, LocalDateTime, ZoneId, ZoneOffset}
 import java.util.{Calendar, Locale, TimeZone}
 import java.util.concurrent.TimeUnit._
 
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, TimestampFormatter}
@@ -241,8 +241,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("DateFormat") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         checkEvaluation(
           DateFormatClass(Literal.create(null, TimestampType), Literal("y"), gmtId),
           null)
@@ -710,8 +710,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("from_unixtime") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val fmt1 = "yyyy-MM-dd HH:mm:ss"
         val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
         val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
@@ -758,8 +758,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("unix_timestamp") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
         val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
@@ -824,8 +824,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("to_unix_timestamp") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val fmt1 = "yyyy-MM-dd HH:mm:ss"
         val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
         val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
@@ -1164,4 +1164,25 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         Literal(LocalDate.of(1, 1, 1))),
       IntervalUtils.stringToInterval(UTF8String.fromString("interval 9999 years")))
   }
+
+  test("to_timestamp exception mode") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkEvaluation(
+        GetTimestamp(
+          Literal("2020-01-27T20:06:11.847-0800"),
+          Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), 1580184371847000L)
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkEvaluation(
+        GetTimestamp(
+          Literal("2020-01-27T20:06:11.847-0800"),
+          Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), null)
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      checkExceptionInExpression[SparkUpgradeException](
+        GetTimestamp(
+          Literal("2020-01-27T20:06:11.847-0800"),
+          Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), "Fail to parse")
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
index c2e03bd2c3609..bce917c80f93c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
@@ -40,8 +40,8 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("inferring timestamp type") {
-    Seq(true, false).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         checkTimestampType("yyyy", """{"a": "2018"}""")
         checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
         checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
@@ -56,8 +56,8 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("prefer decimals over timestamps") {
-    Seq(true, false).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParser =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParser) {
         checkType(
           options = Map(
             "prefersDecimal" -> "true",
@@ -71,8 +71,8 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("skip decimal type inferring") {
-    Seq(true, false).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         checkType(
           options = Map(
             "prefersDecimal" -> "false",
@@ -86,8 +86,8 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("fallback to string type") {
-    Seq(true, false).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         checkType(
           options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
           json = """{"a": "20181202.210400123"}""",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
index 61f0e138cc358..89fb4d5151b37 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
@@ -59,10 +59,22 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
     val df2 = df
       .select(from_csv($"value", schemaWithCorrField1, Map(
         "mode" -> "Permissive", "columnNameOfCorruptRecord" -> columnNameOfCorruptRecord)))
-
-    checkAnswer(df2, Seq(
-      Row(Row(0, null, "0,2013-111-11 12:13:14")),
-      Row(Row(1, java.sql.Date.valueOf("1983-08-04"), null))))
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkAnswer(df2, Seq(
+        Row(Row(0, null, "0,2013-111-11 12:13:14")),
+        Row(Row(1, java.sql.Date.valueOf("1983-08-04"), null))))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkAnswer(df2, Seq(
+        Row(Row(0, java.sql.Date.valueOf("2022-03-11"), null)),
+        Row(Row(1, java.sql.Date.valueOf("1983-08-04"), null))))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      val msg = intercept[SparkException] {
+        df2.collect()
+      }.getCause.getMessage
+      assert(msg.contains("Fail to parse"))
+    }
   }
 
   test("schema_of_csv - infers schemas") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 527dfe56c2289..da7cc1c930461 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -23,7 +23,8 @@ import java.time.{Instant, LocalDateTime}
 import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils}
+import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -96,8 +97,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("date format") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val df = Seq((d, sdf.format(d), ts)).toDF("a", "b", "c")
 
         checkAnswer(
@@ -377,6 +378,13 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       Seq(Row(Date.valueOf("2015-07-30")), Row(Date.valueOf("2015-07-30"))))
   }
 
+  def checkExceptionMessage(df: DataFrame): Unit = {
+    val message = intercept[SparkException] {
+      df.collect()
+    }.getCause.getMessage
+    assert(message.contains("Fail to parse"))
+  }
+
   test("function to_date") {
     val d1 = Date.valueOf("2015-07-22")
     val d2 = Date.valueOf("2015-07-01")
@@ -422,9 +430,15 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       df.select(to_date(col("d"), "yyyy-MM-dd")),
       Seq(Row(Date.valueOf("2015-07-22")), Row(Date.valueOf("2015-07-01")),
         Row(Date.valueOf("2014-12-31"))))
-    checkAnswer(
-      df.select(to_date(col("s"), "yyyy-MM-dd")),
-      Seq(Row(null), Row(Date.valueOf("2014-12-31")), Row(null)))
+    val confKey = SQLConf.LEGACY_TIME_PARSER_POLICY.key
+    withSQLConf(confKey -> "corrected") {
+      checkAnswer(
+        df.select(to_date(col("s"), "yyyy-MM-dd")),
+        Seq(Row(null), Row(Date.valueOf("2014-12-31")), Row(null)))
+    }
+    withSQLConf(confKey -> "exception") {
+      checkExceptionMessage(df.select(to_date(col("s"), "yyyy-MM-dd")))
+    }
 
     // now switch format
     checkAnswer(
@@ -529,8 +543,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("from_unixtime") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("corrected", "legacy").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val sdf1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
         val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
@@ -562,8 +576,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   private def secs(millis: Long): Long = TimeUnit.MILLISECONDS.toSeconds(millis)
 
   test("unix_timestamp") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("corrected", "legacy").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val date1 = Date.valueOf("2015-07-24")
         val date2 = Date.valueOf("2015-07-25")
         val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
@@ -629,8 +643,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("to_unix_timestamp") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("corrected", "legacy").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val date1 = Date.valueOf("2015-07-24")
         val date2 = Date.valueOf("2015-07-25")
         val ts1 = Timestamp.valueOf("2015-07-24 10:00:00.3")
@@ -680,8 +694,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
 
   test("to_timestamp") {
-    Seq(false, true).foreach { legacyParser =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
+    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         val date1 = Date.valueOf("2015-07-24")
         val date2 = Date.valueOf("2015-07-25")
         val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00")
@@ -701,7 +715,7 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
           df.select(unix_timestamp(col("ss")).cast("timestamp")))
         checkAnswer(df.select(to_timestamp(col("ss"))), Seq(
           Row(ts1), Row(ts2)))
-        if (legacyParser) {
+        if (legacyParserPolicy == "legacy") {
           // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off
           // the fractional part of seconds. The behavior was changed by SPARK-27438.
           val legacyFmt = "yyyy/MM/dd HH:mm:ss"
@@ -819,16 +833,18 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-30668: use legacy timestamp parser in to_timestamp") {
-    def checkTimeZoneParsing(expected: Any): Unit = {
-      val df = Seq("2020-01-27T20:06:11.847-0800").toDF("ts")
+    val confKey = SQLConf.LEGACY_TIME_PARSER_POLICY.key
+    val df = Seq("2020-01-27T20:06:11.847-0800").toDF("ts")
+    withSQLConf(confKey -> "legacy") {
+      val expected = Timestamp.valueOf("2020-01-27 20:06:11.847")
       checkAnswer(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")),
         Row(expected))
     }
-    withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> "true") {
-      checkTimeZoneParsing(Timestamp.valueOf("2020-01-27 20:06:11.847"))
+    withSQLConf(confKey -> "corrected") {
+      checkAnswer(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")), Row(null))
     }
-    withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> "false") {
-      checkTimeZoneParsing(null)
+    withSQLConf(confKey -> "exception") {
+      checkExceptionMessage(df.select(to_timestamp(col("ts"), "yyyy-MM-dd'T'HH:mm:ss.SSSz")))
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 500b6cc83953a..e5f1fa6f940e9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -609,7 +609,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "2000") {
+      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZEDateFormatter.key -> "2000") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 43553df296545..30ae9dc67a1e1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2307,6 +2307,26 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     val csv = spark.read.option("header", false).schema("t timestamp, d date").csv(ds)
     checkAnswer(csv, Row(Timestamp.valueOf("2020-1-12 3:23:34.12"), Date.valueOf("2020-1-12")))
   }
+
+  test("exception mode for parsing date/timestamp string") {
+    val ds = Seq("2020-01-27T20:06:11.847-0800").toDS()
+    val csv = spark.read
+      .option("header", false)
+      .option("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      .schema("t timestamp").csv(ds)
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      val msg = intercept[SparkException] {
+        csv.collect()
+      }.getCause.getMessage
+      assert(msg.contains("Fail to parse"))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkAnswer(csv, Row(Timestamp.valueOf("2020-01-27 20:06:11.847")))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkAnswer(csv, Row(null))
+    }
+  }
 }
 
 class CSVv1Suite extends CSVSuite {
@@ -2327,5 +2347,5 @@ class CSVLegacyTimeParserSuite extends CSVSuite {
   override protected def sparkConf: SparkConf =
     super
       .sparkConf
-      .set(SQLConf.LEGACY_TIME_PARSER_ENABLED, true)
+      .set(SQLConf.LEGACY_TIME_PARSER_POLICY, "legacy")
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index d2ce555c04542..df0bca7f0258d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2669,6 +2669,26 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       Date.valueOf("2020-1-12"),
       Date.valueOf(LocalDate.ofEpochDay(12345))))
   }
+
+  test("exception mode for parsing date/timestamp string") {
+    val ds = Seq("{'t': '2020-01-27T20:06:11.847-0800'}").toDS()
+    val json = spark.read
+      .schema("t timestamp")
+      .option("timestampFormat", "yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      .json(ds)
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "exception") {
+      val msg = intercept[SparkException] {
+        json.collect()
+      }.getCause.getMessage
+      assert(msg.contains("Fail to parse"))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "legacy") {
+      checkAnswer(json, Row(Timestamp.valueOf("2020-01-27 20:06:11.847")))
+    }
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> "corrected") {
+      checkAnswer(json, Row(null))
+    }
+  }
 }
 
 class JsonV1Suite extends JsonSuite {
@@ -2689,5 +2709,5 @@ class JsonLegacyTimeParserSuite extends JsonSuite {
   override protected def sparkConf: SparkConf =
     super
       .sparkConf
-      .set(SQLConf.LEGACY_TIME_PARSER_ENABLED, true)
+      .set(SQLConf.LEGACY_TIME_PARSER_POLICY, "legacy")
 }

From 9cea92b6f2c2fc8e0effcec710e6ff6e8a7c965f Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 5 Mar 2020 00:21:32 -0800
Subject: [PATCH 0218/1280] [SPARK-31050][TEST] Disable flaky `Roundtrip` test
 in KafkaDelegationTokenSuite

### What changes were proposed in this pull request?

Disable test `KafkaDelegationTokenSuite`.

### Why are the changes needed?

`KafkaDelegationTokenSuite` is too flaky.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #27789 from Ngone51/retry_kafka.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 0a22f1966466629cb745d000a0608d521fece093)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
index 306483825ae3b..79239e57409e9 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
@@ -62,7 +62,7 @@ class KafkaDelegationTokenSuite extends StreamTest with SharedSparkSession with
     }
   }
 
-  test("Roundtrip") {
+  ignore("Roundtrip") {
     val hadoopConf = new Configuration()
     val manager = new HadoopDelegationTokenManager(spark.sparkContext.conf, hadoopConf, null)
     val credentials = new Credentials()

From 4fcb5ae22623de96247fcfe7341be0af7ed2471f Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Thu, 5 Mar 2020 18:33:59 +0800
Subject: [PATCH 0219/1280] [SPARK-31024][SQL] Allow specifying session catalog
 name `spark_catalog` in qualified column names for v1 tables

### What changes were proposed in this pull request?

Currently, the user cannot specify the session catalog name (`spark_catalog`) in qualified column names for v1 tables:
```
SELECT spark_catalog.default.t.i FROM spark_catalog.default.t
```
fails with `cannot resolve 'spark_catalog.default.t.i`.

This is inconsistent with v2 table behavior where catalog name can be used:
```
SELECT testcat.ns1.tbl.id FROM testcat.ns1.tbl.id
```

This PR proposes to fix the inconsistency and allow the user to specify session catalog name in column names for v1 tables.

### Why are the changes needed?

Fixing an inconsistent behavior.

### Does this PR introduce any user-facing change?

Yes, now the following query works:
```
SELECT spark_catalog.default.t.i FROM spark_catalog.default.t
```

### How was this patch tested?

Added new tests.

Closes #27776 from imback82/spark_catalog_col_name_resolution.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 66b4fd040e97cb6de6536a5545017278879c98fb)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |   2 +-
 .../sql/catalyst/catalog/SessionCatalog.scala |   6 +-
 .../sql/catalyst/expressions/package.scala    | 102 +++++++++++++-----
 .../catalog/SessionCatalogSuite.scala         |   6 +-
 .../results/columnresolution-negative.sql.out |  26 ++---
 .../results/postgreSQL/create_view.sql.out    |   2 +-
 .../sql-tests/results/postgreSQL/join.sql.out |   2 +-
 .../results/postgreSQL/select_having.sql.out  |   2 +-
 .../results/postgreSQL/window_part3.sql.out   |   4 +-
 .../results/udf/postgreSQL/udf-join.sql.out   |   2 +-
 .../udf/postgreSQL/udf-select_having.sql.out  |   2 +-
 .../org/apache/spark/sql/SQLQuerySuite.scala  |   4 +-
 .../sql/connector/DataSourceV2SQLSuite.scala  |   8 +-
 .../sql/hive/HiveMetastoreCatalogSuite.scala  |   2 +-
 14 files changed, 111 insertions(+), 59 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 254dd44c428be..3cb754d665b28 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -935,7 +935,7 @@ class Analyzer(
               v1SessionCatalog.getRelation(v1Table.v1Table)
             case table =>
               SubqueryAlias(
-                ident.asMultipartIdentifier,
+                catalog.name +: ident.asMultipartIdentifier,
                 DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
           }
           val key = catalog.name +: ident.namespace :+ ident.name
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index c80d9d2d7c183..3a63aff493df5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionInfo, Im
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, View}
 import org.apache.spark.sql.catalyst.util.StringUtils
+import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 import org.apache.spark.sql.types.StructType
@@ -758,6 +759,7 @@ class SessionCatalog(
     val name = metadata.identifier
     val db = formatDatabaseName(name.database.getOrElse(currentDb))
     val table = formatTableName(name.table)
+    val multiParts = Seq(CatalogManager.SESSION_CATALOG_NAME, db, table)
 
     if (metadata.tableType == CatalogTableType.VIEW) {
       val viewText = metadata.viewText.getOrElse(sys.error("Invalid view without text."))
@@ -769,9 +771,9 @@ class SessionCatalog(
         desc = metadata,
         output = metadata.schema.toAttributes,
         child = parser.parsePlan(viewText))
-      SubqueryAlias(table, db, child)
+      SubqueryAlias(multiParts, child)
     } else {
-      SubqueryAlias(table, db, UnresolvedCatalogRelation(metadata))
+      SubqueryAlias(multiParts, UnresolvedCatalogRelation(metadata))
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
index 9f42e643e4cb2..1b59056c4da66 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
@@ -142,22 +142,44 @@ package object expressions  {
     }
 
     /** Map to use for qualified case insensitive attribute lookups with 3 part key */
-    @transient private val qualified3Part: Map[(String, String, String), Seq[Attribute]] = {
+    @transient private lazy val qualified3Part: Map[(String, String, String), Seq[Attribute]] = {
       // key is 3 part: database name, table name and name
-      val grouped = attrs.filter(_.qualifier.length == 2).groupBy { a =>
-        (a.qualifier.head.toLowerCase(Locale.ROOT),
-          a.qualifier.last.toLowerCase(Locale.ROOT),
-          a.name.toLowerCase(Locale.ROOT))
+      val grouped = attrs.filter(a => a.qualifier.length >= 2 && a.qualifier.length <= 3)
+        .groupBy { a =>
+          val qualifier = if (a.qualifier.length == 2) {
+            a.qualifier
+          } else {
+            a.qualifier.takeRight(2)
+          }
+          (qualifier.head.toLowerCase(Locale.ROOT),
+            qualifier.last.toLowerCase(Locale.ROOT),
+            a.name.toLowerCase(Locale.ROOT))
+        }
+      unique(grouped)
+    }
+
+    /** Map to use for qualified case insensitive attribute lookups with 4 part key */
+    @transient
+    private lazy val qualified4Part: Map[(String, String, String, String), Seq[Attribute]] = {
+      // key is 4 part: catalog name, database name, table name and name
+      val grouped = attrs.filter(_.qualifier.length == 3).groupBy { a =>
+        a.qualifier match {
+          case Seq(catalog, db, tbl) =>
+            (catalog.toLowerCase(Locale.ROOT),
+              db.toLowerCase(Locale.ROOT),
+              tbl.toLowerCase(Locale.ROOT),
+              a.name.toLowerCase(Locale.ROOT))
+        }
       }
       unique(grouped)
     }
 
-    /** Returns true if all qualifiers in `attrs` have 2 or less parts. */
-    @transient private val hasTwoOrLessQualifierParts: Boolean =
-      attrs.forall(_.qualifier.length <= 2)
+    /** Returns true if all qualifiers in `attrs` have 3 or less parts. */
+    @transient private val hasThreeOrLessQualifierParts: Boolean =
+      attrs.forall(_.qualifier.length <= 3)
 
-    /** Match attributes for the case where all qualifiers in `attrs` have 2 or less parts. */
-    private def matchWithTwoOrLessQualifierParts(
+    /** Match attributes for the case where all qualifiers in `attrs` have 3 or less parts. */
+    private def matchWithThreeOrLessQualifierParts(
         nameParts: Seq[String],
         resolver: Resolver): (Seq[Attribute], Seq[String]) = {
       // Collect matching attributes given a name and a lookup.
@@ -167,25 +189,55 @@ package object expressions  {
         }
       }
 
-      // Find matches for the given name assuming that the 1st two parts are qualifier
-      // (i.e. database name and table name) and the 3rd part is the actual column name.
+      // Find matches for the given name assuming that the 1st three parts are qualifier
+      // (i.e. catalog name, database name and table name) and the 4th part is the actual
+      // column name.
       //
-      // For example, consider an example where "db1" is the database name, "a" is the table name
-      // and "b" is the column name and "c" is the struct field name.
-      // If the name parts is db1.a.b.c, then Attribute will match
-      // Attribute(b, qualifier("db1,"a")) and List("c") will be the second element
+      // For example, consider an example where "cat" is the catalog name, "db1" is the database
+      // name, "a" is the table name and "b" is the column name and "c" is the struct field name.
+      // If the name parts is cat.db1.a.b.c, then Attribute will match
+      // Attribute(b, qualifier("cat", "db1, "a")) and List("c") will be the second element
       var matches: (Seq[Attribute], Seq[String]) = nameParts match {
-        case dbPart +: tblPart +: name +: nestedFields =>
-          val key = (dbPart.toLowerCase(Locale.ROOT),
+        case catalogPart +: dbPart +: tblPart +: name +: nestedFields =>
+          val key = (catalogPart.toLowerCase(Locale.ROOT), dbPart.toLowerCase(Locale.ROOT),
             tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT))
-          val attributes = collectMatches(name, qualified3Part.get(key)).filter {
-            a => (resolver(dbPart, a.qualifier.head) && resolver(tblPart, a.qualifier.last))
+          val attributes = collectMatches(name, qualified4Part.get(key)).filter { a =>
+            assert(a.qualifier.length == 3)
+            resolver(catalogPart, a.qualifier(0)) && resolver(dbPart, a.qualifier(1)) &&
+              resolver(tblPart, a.qualifier(2))
           }
           (attributes, nestedFields)
         case _ =>
           (Seq.empty, Seq.empty)
       }
 
+      // Find matches for the given name assuming that the 1st two parts are qualifier
+      // (i.e. database name and table name) and the 3rd part is the actual column name.
+      //
+      // For example, consider an example where "db1" is the database name, "a" is the table name
+      // and "b" is the column name and "c" is the struct field name.
+      // If the name parts is db1.a.b.c, then it can match both
+      // Attribute(b, qualifier("cat", "db1, "a")) and Attribute(b, qualifier("db1, "a")),
+      // and List("c") will be the second element
+      if (matches._1.isEmpty) {
+        matches = nameParts match {
+          case dbPart +: tblPart +: name +: nestedFields =>
+            val key = (dbPart.toLowerCase(Locale.ROOT),
+              tblPart.toLowerCase(Locale.ROOT), name.toLowerCase(Locale.ROOT))
+            val attributes = collectMatches(name, qualified3Part.get(key)).filter { a =>
+              val qualifier = if (a.qualifier.length == 2) {
+                a.qualifier
+              } else {
+                a.qualifier.takeRight(2)
+              }
+              resolver(dbPart, qualifier.head) && resolver(tblPart, qualifier.last)
+            }
+            (attributes, nestedFields)
+          case _ =>
+            (Seq.empty, Seq.empty)
+        }
+      }
+
       // If there are no matches, then find matches for the given name assuming that
       // the 1st part is a qualifier (i.e. table name, alias, or subquery alias) and the
       // 2nd part is the actual name. This returns a tuple of
@@ -219,9 +271,9 @@ package object expressions  {
     }
 
     /**
-     * Match attributes for the case where at least one qualifier in `attrs` has more than 2 parts.
+     * Match attributes for the case where at least one qualifier in `attrs` has more than 3 parts.
      */
-    private def matchWithThreeOrMoreQualifierParts(
+    private def matchWithFourOrMoreQualifierParts(
         nameParts: Seq[String],
         resolver: Resolver): (Seq[Attribute], Seq[String]) = {
       // Returns true if the `short` qualifier is a subset of the last elements of
@@ -277,10 +329,10 @@ package object expressions  {
 
     /** Perform attribute resolution given a name and a resolver. */
     def resolve(nameParts: Seq[String], resolver: Resolver): Option[NamedExpression] = {
-      val (candidates, nestedFields) = if (hasTwoOrLessQualifierParts) {
-        matchWithTwoOrLessQualifierParts(nameParts, resolver)
+      val (candidates, nestedFields) = if (hasThreeOrLessQualifierParts) {
+        matchWithThreeOrLessQualifierParts(nameParts, resolver)
       } else {
-        matchWithThreeOrMoreQualifierParts(nameParts, resolver)
+        matchWithFourOrMoreQualifierParts(nameParts, resolver)
       }
 
       def name = UnresolvedAttribute(nameParts).name
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
index 0d9e2f61e812a..4d88a8d7ee546 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
@@ -635,11 +635,11 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       val view = View(desc = metadata, output = metadata.schema.toAttributes,
         child = CatalystSqlParser.parsePlan(metadata.viewText.get))
       comparePlans(catalog.lookupRelation(TableIdentifier("view1", Some("db3"))),
-        SubqueryAlias("view1", "db3", view))
+        SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
       // Look up a view using current database of the session catalog.
       catalog.setCurrentDatabase("db3")
       comparePlans(catalog.lookupRelation(TableIdentifier("view1")),
-        SubqueryAlias("view1", "db3", view))
+        SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view1"), view))
     }
   }
 
@@ -655,7 +655,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
       val view = View(desc = metadata, output = metadata.schema.toAttributes,
         child = CatalystSqlParser.parsePlan(metadata.viewText.get))
       comparePlans(catalog.lookupRelation(TableIdentifier("view2", Some("db3"))),
-        SubqueryAlias("view2", "db3", view))
+        SubqueryAlias(Seq(CatalogManager.SESSION_CATALOG_NAME, "db3", "view2"), view))
     }
   }
 
diff --git a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
index f34b75a379aae..04ddfe0ac128c 100644
--- a/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/columnresolution-negative.sql.out
@@ -72,7 +72,7 @@ SELECT i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -81,7 +81,7 @@ SELECT t1.i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -90,7 +90,7 @@ SELECT mydb1.t1.i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'mydb1.t1.i1' is ambiguous, could be: mydb1.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'mydb1.t1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -99,7 +99,7 @@ SELECT i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -108,7 +108,7 @@ SELECT t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb1.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb1.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -125,7 +125,7 @@ SELECT i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -134,7 +134,7 @@ SELECT t1.i1 FROM t1, mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb1.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb1.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -143,7 +143,7 @@ SELECT i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 'i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -152,7 +152,7 @@ SELECT t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 't1.i1' is ambiguous, could be: mydb2.t1.i1, mydb2.t1.i1.; line 1 pos 7
+Reference 't1.i1' is ambiguous, could be: spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1.; line 1 pos 7
 
 
 -- !query
@@ -161,7 +161,7 @@ SELECT db1.t1.i1 FROM t1, mydb2.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`db1.t1.i1`' given input columns: [mydb2.t1.i1, mydb2.t1.i1]; line 1 pos 7
+cannot resolve '`db1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1, spark_catalog.mydb2.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -186,7 +186,7 @@ SELECT mydb1.t1 FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7
+cannot resolve '`mydb1.t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -204,7 +204,7 @@ SELECT t1 FROM mydb1.t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`t1`' given input columns: [mydb1.t1.i1]; line 1 pos 7
+cannot resolve '`t1`' given input columns: [spark_catalog.mydb1.t1.i1]; line 1 pos 7
 
 
 -- !query
@@ -221,7 +221,7 @@ SELECT mydb1.t1.i1 FROM t1
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '`mydb1.t1.i1`' given input columns: [mydb2.t1.i1]; line 1 pos 7
+cannot resolve '`mydb1.t1.i1`' given input columns: [spark_catalog.mydb2.t1.i1]; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
index 85ce978657844..1f2bd5795cf1e 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
@@ -56,7 +56,7 @@ CREATE VIEW key_dependent_view AS
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-expression 'default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
+expression 'spark_catalog.default.view_base_table.`data`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
index 5332dfff9f101..20f4f6b1f213f 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/join.sql.out
@@ -536,7 +536,7 @@ SELECT '' AS `xxx`, i, k, t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 20
+Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 20
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
index cbf4cfa58cdb9..d8d33d92a7cc4 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/select_having.sql.out
@@ -143,7 +143,7 @@ SELECT a FROM test_having HAVING min(a) < max(a)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
index 5a52358fe1c53..acce688092570 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
@@ -244,7 +244,7 @@ from t1 where f1 = f2
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve '(PARTITION BY default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
+cannot resolve '(PARTITION BY spark_catalog.default.t1.`f1` RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING)' due to data type mismatch: A range window frame cannot be used in an unordered window specification.; line 1 pos 24
 
 
 -- !query
@@ -306,7 +306,7 @@ org.apache.spark.sql.AnalysisException
 
 The query operator `Join` contains one or more unsupported
 expression types Aggregate, Window or Generate.
-Invalid expressions: [row_number() OVER (ORDER BY default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
+Invalid expressions: [row_number() OVER (ORDER BY spark_catalog.default.empsalary.`salary` ASC NULLS FIRST ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)];
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
index 3cc14ff04b1ba..188b57ffd58d5 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-join.sql.out
@@ -536,7 +536,7 @@ SELECT udf('') AS `xxx`, udf(i) AS i, udf(k), udf(t) AS t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Reference 'i' is ambiguous, could be: default.j1_tbl.i, default.j2_tbl.i.; line 1 pos 29
+Reference 'i' is ambiguous, could be: spark_catalog.default.j1_tbl.i, spark_catalog.default.j2_tbl.i.; line 1 pos 29
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
index bb108a29f75ce..50b6e60086747 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-select_having.sql.out
@@ -143,7 +143,7 @@ SELECT udf(a) FROM test_having HAVING udf(min(a)) < udf(max(a))
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-grouping expressions sequence is empty, and 'default.test_having.`a`' is not an aggregate function. Wrap '(min(default.test_having.`a`) AS `min(a#x)`, max(default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
+grouping expressions sequence is empty, and 'spark_catalog.default.test_having.`a`' is not an aggregate function. Wrap '(min(spark_catalog.default.test_having.`a`) AS `min(a#x)`, max(spark_catalog.default.test_having.`a`) AS `max(a#x)`)' in windowing function(s) or wrap 'spark_catalog.default.test_having.`a`' in first() (or first_value) if you don't care which value you get.;
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 563b4d1427395..81dfa798b4da3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -2797,7 +2797,9 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
           sql("SELECT * FROM t, S WHERE c = C")
         }.message
         assert(
-          m.contains("cannot resolve '(default.t.`c` = default.S.`C`)' due to data type mismatch"))
+          m.contains(
+            "cannot resolve '(spark_catalog.default.t.`c` = spark_catalog.default.S.`C`)' " +
+            "due to data type mismatch"))
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 1fc0bb10a535c..ba4200d84d46b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -758,14 +758,10 @@ class DataSourceV2SQLSuite
         checkAnswer(sql("select i from t"), Row(1))
         checkAnswer(sql("select t.i from t"), Row(1))
         checkAnswer(sql("select default.t.i from t"), Row(1))
+        checkAnswer(sql("select spark_catalog.default.t.i from t"), Row(1))
         checkAnswer(sql("select t.i from spark_catalog.default.t"), Row(1))
         checkAnswer(sql("select default.t.i from spark_catalog.default.t"), Row(1))
-
-        // catalog name cannot be used for tables in the session catalog.
-        val ex = intercept[AnalysisException] {
-          sql(s"select spark_catalog.default.t.i from spark_catalog.default.t")
-        }
-        assert(ex.getMessage.contains("cannot resolve '`spark_catalog.default.t.i`"))
+        checkAnswer(sql("select spark_catalog.default.t.i from spark_catalog.default.t"), Row(1))
       }
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index b8ef44b096eed..9cd56f1745afc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -62,7 +62,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
       spark.sql("create view vw1 as select 1 as id")
       val plan = spark.sql("select id from vw1").queryExecution.analyzed
       val aliases = plan.collect {
-        case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("default")), _) => x
+        case x @ SubqueryAlias(AliasIdentifier("vw1", Seq("spark_catalog", "default")), _) => x
       }
       assert(aliases.size == 1)
     }

From 2247239dd5fef9506f025c993c43fb8365e57ffd Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Thu, 5 Mar 2020 18:53:01 +0800
Subject: [PATCH 0220/1280] [SPARK-31046][SQL] Make more efficient and clean up
 AQE update UI code

### What changes were proposed in this pull request?
This PR avoids sending redundant metrics (those that have been included in previous update) as well as useless metrics (those in future stages) to Spark UI in AQE UI metrics update.

### Why are the changes needed?
This change will make UI metrics update more efficient.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Manual test in Spark UI.

Closes #27799 from maryannxue/aqe-ui-cleanup.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9b602e26d2a5252623a2ed66a1a2b382665fdab4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      | 36 +++++--------------
 1 file changed, 8 insertions(+), 28 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index c018ca4a1eaec..f2ebe1a9bf2fc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -37,7 +37,6 @@ import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec._
 import org.apache.spark.sql.execution.exchange._
-import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.execution.ui.{SparkListenerSQLAdaptiveExecutionUpdate, SparkListenerSQLAdaptiveSQLMetricUpdates, SQLPlanMetric}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.ThreadUtils
@@ -133,21 +132,6 @@ case class AdaptiveSparkPlanExec(
     executedPlan.resetMetrics()
   }
 
-  private def collectSQLMetrics(plan: SparkPlan): Seq[SQLMetric] = {
-    val metrics = new mutable.ArrayBuffer[SQLMetric]()
-    plan.foreach {
-      case p: ShuffleQueryStageExec if (p.resultOption.isEmpty) =>
-        collectSQLMetrics(p.plan).foreach(metrics += _)
-      case p: BroadcastQueryStageExec if (p.resultOption.isEmpty) =>
-        collectSQLMetrics(p.plan).foreach(metrics += _)
-      case p: SparkPlan =>
-        p.metrics.foreach { case metric =>
-          metrics += metric._2
-        }
-    }
-    metrics
-  }
-
   private def getFinalPhysicalPlan(): SparkPlan = lock.synchronized {
     if (!isFinalPlan) {
       // Subqueries do not have their own execution IDs and therefore rely on the main query to
@@ -163,7 +147,7 @@ case class AdaptiveSparkPlanExec(
         currentPhysicalPlan = result.newPlan
         if (result.newStages.nonEmpty) {
           stagesToReplace = result.newStages ++ stagesToReplace
-          executionId.foreach(onUpdatePlan)
+          executionId.foreach(onUpdatePlan(_, result.newStages.map(_.plan)))
 
           // Start materialization of all new stages and fail fast if any stages failed eagerly
           result.newStages.foreach { stage =>
@@ -232,7 +216,7 @@ case class AdaptiveSparkPlanExec(
       // Run the final plan when there's no more unfinished stages.
       currentPhysicalPlan = applyPhysicalRules(result.newPlan, queryStageOptimizerRules)
       isFinalPlan = true
-      executionId.foreach(onUpdatePlan)
+      executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
       logDebug(s"Final plan: $currentPhysicalPlan")
     }
     currentPhysicalPlan
@@ -496,14 +480,18 @@ case class AdaptiveSparkPlanExec(
   /**
    * Notify the listeners of the physical plan change.
    */
-  private def onUpdatePlan(executionId: Long): Unit = {
+  private def onUpdatePlan(executionId: Long, newSubPlans: Seq[SparkPlan]): Unit = {
     if (isSubquery) {
       // When executing subqueries, we can't update the query plan in the UI as the
       // UI doesn't support partial update yet. However, the subquery may have been
       // optimized into a different plan and we must let the UI know the SQL metrics
       // of the new plan nodes, so that it can track the valid accumulator updates later
       // and display SQL metrics correctly.
-      onUpdateSQLMetrics(collectSQLMetrics(currentPhysicalPlan), executionId)
+      val newMetrics = newSubPlans.flatMap { p =>
+        p.flatMap(_.metrics.values.map(m => SQLPlanMetric(m.name.get, m.id, m.metricType)))
+      }
+      context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates(
+        executionId.toLong, newMetrics))
     } else {
       context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate(
         executionId,
@@ -512,14 +500,6 @@ case class AdaptiveSparkPlanExec(
     }
   }
 
-  private def onUpdateSQLMetrics(sqlMetrics: Seq[SQLMetric], executionId: Long): Unit = {
-    val sqlPlanMetrics = sqlMetrics.map { case sqlMetric =>
-      SQLPlanMetric(sqlMetric.name.get, sqlMetric.id, sqlMetric.metricType)
-    }
-    context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates(
-      executionId.toLong, sqlPlanMetrics))
-  }
-
   /**
    * Cancel all running stages with best effort and throw an Exception containing all stage
    * materialization errors and stage cancellation errors.

From daa140df726f8521ae7bbaf4586277ae4f4aea7c Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 5 Mar 2020 20:43:52 +0900
Subject: [PATCH 0221/1280] [SPARK-31019][SQL] make it clear that people can
 deduplicate map keys

rename the config and make it non-internal.

Now we fail the query if duplicated map keys are detected, and provide a legacy config to deduplicate it. However, we must provide a way to get users out of this situation, instead of just rejecting to run the query. This exit strategy should always be there, while legacy config indicates that it may be removed someday.

no, just rename a config which was added in 3.0

add more tests for the fail behavior.

Closes #27772 from cloud-fan/map.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md                   |  2 +-
 .../catalyst/util/ArrayBasedMapBuilder.scala  | 20 +++---
 .../apache/spark/sql/internal/SQLConf.scala   | 25 ++++---
 .../CollectionExpressionsSuite.scala          | 14 +++-
 .../expressions/ComplexTypeSuite.scala        | 15 +++-
 .../HigherOrderFunctionsSuite.scala           |  5 +-
 .../util/ArrayBasedMapBuilderSuite.scala      | 71 ++++++++++++-------
 .../spark/sql/DataFrameFunctionsSuite.scala   | 13 +++-
 8 files changed, 112 insertions(+), 53 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 00500615895fa..6c73038bbcfa0 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -49,7 +49,7 @@ license: |
 
   - In Spark version 2.4 and earlier, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys and join keys. Since Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and earlier.
 
-  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, new config `spark.sql.legacy.allowDuplicatedMapKeys` was added, with the default value `false`, Spark will throw RuntimeException while duplicated keys are found. If set to `true`, these built-in functions will remove duplicated map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
+  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, Spark will throw RuntimeException while duplicated keys are found. Users can set `spark.sql.mapKeyDedupPolicy` to LAST_WIN to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
 
   - In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
index 40e75b5b6cd25..0185b5743e446 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilder.scala
@@ -21,7 +21,6 @@ import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.array.ByteArrayMethods
 
@@ -49,8 +48,7 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
   private lazy val keyGetter = InternalRow.getAccessor(keyType)
   private lazy val valueGetter = InternalRow.getAccessor(valueType)
 
-  private val allowDuplicatedMapKey =
-    SQLConf.get.getConf(LEGACY_ALLOW_DUPLICATED_MAP_KEY)
+  private val mapKeyDedupPolicy = SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)
 
   def put(key: Any, value: Any): Unit = {
     if (key == null) {
@@ -67,13 +65,17 @@ class ArrayBasedMapBuilder(keyType: DataType, valueType: DataType) extends Seria
       keys.append(key)
       values.append(value)
     } else {
-      if (!allowDuplicatedMapKey) {
-        throw new RuntimeException(s"Duplicate map key $key was founded, please check the input " +
-          "data. If you want to remove the duplicated keys with last-win policy, you can set " +
-          s"${LEGACY_ALLOW_DUPLICATED_MAP_KEY.key} to true.")
+      if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.EXCEPTION.toString) {
+        throw new RuntimeException(s"Duplicate map key $key was found, please check the input " +
+          "data. If you want to remove the duplicated keys, you can set " +
+          s"${SQLConf.MAP_KEY_DEDUP_POLICY.key} to ${SQLConf.MapKeyDedupPolicy.LAST_WIN} so that " +
+          "the key inserted at last takes precedence.")
+      } else if (mapKeyDedupPolicy == SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
+        // Overwrite the previous value, as the policy is last wins.
+        values(index) = value
+      } else {
+        throw new IllegalStateException("Unknown map key dedup policy: " + mapKeyDedupPolicy)
       }
-      // Overwrite the previous value, as the policy is last wins.
-      values(index) = value
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e39d0661e914f..68a89b2da5110 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2095,6 +2095,21 @@ object SQLConf {
       .stringConf
       .createOptional
 
+  object MapKeyDedupPolicy extends Enumeration {
+    val EXCEPTION, LAST_WIN = Value
+  }
+
+  val MAP_KEY_DEDUP_POLICY = buildConf("spark.sql.mapKeyDedupPolicy")
+    .doc("The policy to deduplicate map keys in builtin function: CreateMap, MapFromArrays, " +
+      "MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query " +
+      "fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted " +
+      "at last takes precedence.")
+    .version("3.0.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(MapKeyDedupPolicy.values.map(_.toString))
+    .createWithDefault(MapKeyDedupPolicy.EXCEPTION.toString)
+
   val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
     .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
@@ -2202,16 +2217,6 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
-  val LEGACY_ALLOW_DUPLICATED_MAP_KEY =
-    buildConf("spark.sql.legacy.allowDuplicatedMapKeys")
-      .doc("When true, use last wins policy to remove duplicated map keys in built-in functions, " +
-        "this config takes effect in below build-in functions: CreateMap, MapFromArrays, " +
-        "MapFromEntries, StringToMap, MapConcat and TransformKeys. Otherwise, if this is false, " +
-        "which is the default, Spark will throw an exception when duplicated map keys are " +
-        "detected.")
-      .booleanConf
-      .createWithDefault(false)
-
   val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 01df6675016d2..3cfc66f5cdb03 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -139,7 +139,9 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
       MapType(IntegerType, IntegerType, valueContainsNull = true))
     val mNull = Literal.create(null, MapType(StringType, StringType))
 
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+    checkExceptionInExpression[RuntimeException](
+      MapConcat(Seq(m0, m1)), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // overlapping maps should remove duplicated map keys w.r.t. last win policy.
       checkEvaluation(MapConcat(Seq(m0, m1)), create_map("a" -> "4", "b" -> "2", "c" -> "3"))
     }
@@ -274,7 +276,10 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(MapFromEntries(ai1), create_map(1 -> null, 2 -> 20, 3 -> null))
     checkEvaluation(MapFromEntries(ai2), Map.empty)
     checkEvaluation(MapFromEntries(ai3), null)
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+
+    checkExceptionInExpression[RuntimeException](
+      MapFromEntries(ai4), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // Duplicated map keys will be removed w.r.t. the last wins policy.
       checkEvaluation(MapFromEntries(ai4), create_map(1 -> 20))
     }
@@ -298,7 +303,10 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(MapFromEntries(as1), create_map("a" -> null, "b" -> "bb", "c" -> null))
     checkEvaluation(MapFromEntries(as2), Map.empty)
     checkEvaluation(MapFromEntries(as3), null)
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+
+    checkExceptionInExpression[RuntimeException](
+      MapFromEntries(as4), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // Duplicated map keys will be removed w.r.t. the last wins policy.
       checkEvaluation(MapFromEntries(as4), create_map("a" -> "bb"))
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 2c1e0c8460468..3df7d02fb6604 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -217,7 +217,9 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
       CreateMap(interlace(strWithNull, intSeq.map(Literal(_)))),
       "Cannot use null as map key")
 
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+    checkExceptionInExpression[RuntimeException](
+      CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // Duplicated map keys will be removed w.r.t. the last wins policy.
       checkEvaluation(
         CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
@@ -284,7 +286,12 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
       MapFromArrays(intWithNullArray, strArray),
       "Cannot use null as map key")
 
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+    checkExceptionInExpression[RuntimeException](
+      MapFromArrays(
+        Literal.create(Seq(1, 1), ArrayType(IntegerType)),
+        Literal.create(Seq(2, 3), ArrayType(IntegerType))),
+      "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // Duplicated map keys will be removed w.r.t. the last wins policy.
       checkEvaluation(
         MapFromArrays(
@@ -404,7 +411,9 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     val m5 = Map("a" -> null)
     checkEvaluation(new StringToMap(s5), m5)
 
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+    checkExceptionInExpression[RuntimeException](
+      new StringToMap(Literal("a:1,b:2,a:3")), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // Duplicated map keys will be removed w.r.t. the last wins policy.
       checkEvaluation(
         new StringToMap(Literal("a:1,b:2,a:3")),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
index b3438538a3367..c07f06eac08ea 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala
@@ -465,7 +465,10 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(
       transformKeys(transformKeys(ai0, plusOne), plusValue),
       create_map(3 -> 1, 5 -> 2, 7 -> 3, 9 -> 4))
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+
+    checkExceptionInExpression[RuntimeException](
+      transformKeys(ai0, modKey), "Duplicate map key")
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       // Duplicated map keys will be removed w.r.t. the last wins policy.
       checkEvaluation(transformKeys(ai0, modKey), create_map(1 -> 4, 2 -> 2, 0 -> 3))
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
index 87bbdb7300e4c..6e07cd5d6415d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala
@@ -48,11 +48,11 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
     val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
     builder.put(1, 1)
     val e = intercept[RuntimeException](builder.put(1, 2))
-    assert(e.getMessage.contains("Duplicate map key 1 was founded"))
+    assert(e.getMessage.contains("Duplicate map key 1 was found"))
   }
 
   test("remove duplicated keys with last wins policy") {
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
       builder.put(1, 1)
       builder.put(2, 2)
@@ -63,8 +63,15 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
-  test("binary type key") {
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+  test("binary type key with duplication") {
+    val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
+    builder.put(Array(1.toByte), 1)
+    builder.put(Array(2.toByte), 2)
+    val e = intercept[RuntimeException](builder.put(Array(1.toByte), 3))
+    // By default duplicated map key fails the query.
+    assert(e.getMessage.contains("Duplicate map key"))
+
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
       builder.put(Array(1.toByte), 1)
       builder.put(Array(2.toByte), 2)
@@ -79,18 +86,26 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
-  test("struct type key") {
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+  test("struct type key with duplication") {
+    val unsafeRow = {
+      val row = new UnsafeRow(1)
+      val bytes = new Array[Byte](16)
+      row.pointTo(bytes, 16)
+      row.setInt(0, 1)
+      row
+    }
+
+    val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
+    builder.put(InternalRow(1), 1)
+    builder.put(InternalRow(2), 2)
+    val e = intercept[RuntimeException](builder.put(unsafeRow, 3))
+    // By default duplicated map key fails the query.
+    assert(e.getMessage.contains("Duplicate map key"))
+
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
       builder.put(InternalRow(1), 1)
       builder.put(InternalRow(2), 2)
-      val unsafeRow = {
-        val row = new UnsafeRow(1)
-        val bytes = new Array[Byte](16)
-        row.pointTo(bytes, 16)
-        row.setInt(0, 1)
-        row
-      }
       builder.put(unsafeRow, 3)
       val map = builder.build()
       assert(map.numElements() == 2)
@@ -98,20 +113,28 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
-  test("array type key") {
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+  test("array type key with duplication") {
+    val unsafeArray = {
+      val array = new UnsafeArrayData()
+      val bytes = new Array[Byte](24)
+      Platform.putLong(bytes, Platform.BYTE_ARRAY_OFFSET, 2)
+      array.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET, 24)
+      array.setInt(0, 1)
+      array.setInt(1, 1)
+      array
+    }
+
+    val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
+    builder.put(new GenericArrayData(Seq(1, 1)), 1)
+    builder.put(new GenericArrayData(Seq(2, 2)), 2)
+    val e = intercept[RuntimeException](builder.put(unsafeArray, 3))
+    // By default duplicated map key fails the query.
+    assert(e.getMessage.contains("Duplicate map key"))
+
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
       builder.put(new GenericArrayData(Seq(1, 1)), 1)
       builder.put(new GenericArrayData(Seq(2, 2)), 2)
-      val unsafeArray = {
-        val array = new UnsafeArrayData()
-        val bytes = new Array[Byte](24)
-        Platform.putLong(bytes, Platform.BYTE_ARRAY_OFFSET, 2)
-        array.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET, 24)
-        array.setInt(0, 1)
-        array.setInt(1, 1)
-        array
-      }
       builder.put(unsafeArray, 3)
       val map = builder.build()
       assert(map.numElements() == 2)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index b4b9a488b11c4..a613c33b6c876 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -23,6 +23,7 @@ import java.util.TimeZone
 
 import scala.util.Random
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -651,7 +652,9 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       Row(null)
     )
 
-    withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+    intercept[SparkException](df1.selectExpr("map_concat(map1, map2)").collect())
+    intercept[SparkException](df1.select(map_concat($"map1", $"map2")).collect())
+    withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
       checkAnswer(df1.selectExpr("map_concat(map1, map2)"), expected1a)
       checkAnswer(df1.select(map_concat($"map1", $"map2")), expected1a)
     }
@@ -3070,7 +3073,13 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       checkAnswer(dfExample2.select(transform_keys(col("j"), (k, v) => k + v)),
         Seq(Row(Map(2.0 -> 1.0, 3.4 -> 1.4, 4.7 -> 1.7))))
 
-      withSQLConf(SQLConf.LEGACY_ALLOW_DUPLICATED_MAP_KEY.key -> "true") {
+      intercept[SparkException] {
+        dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)").collect()
+      }
+      intercept[SparkException] {
+        dfExample3.select(transform_keys(col("x"), (k, v) => k % 2 === 0 || v)).collect()
+      }
+      withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
         checkAnswer(dfExample3.selectExpr("transform_keys(x, (k, v) ->  k % 2 = 0 OR v)"),
           Seq(Row(Map(true -> true, true -> false))))
 

From 1c165eecd3948dc12db19827dfd326d32eb4e475 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 5 Mar 2020 19:38:20 +0800
Subject: [PATCH 0222/1280] [SPARK-31038][SQL] Add checkValue for
 spark.sql.session.timeZone

### What changes were proposed in this pull request?

The `spark.sql.session.timeZone` config can accept any string value including invalid time zone ids, then it will fail other queries that rely on the time zone. We should do the value checking in the set phase and fail fast if the zone value is invalid.

### Why are the changes needed?

improve configuration
### Does this PR introduce any user-facing change?

yes, will fail fast if the value is a wrong timezone id
### How was this patch tested?

add ut

Closes #27792 from yaooqinn/SPARK-31038.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  8 ++++++++
 .../spark/sql/internal/SQLConfSuite.scala     | 19 +++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 68a89b2da5110..cc9c2ae340d3c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -24,6 +24,7 @@ import java.util.zip.Deflater
 
 import scala.collection.JavaConverters._
 import scala.collection.immutable
+import scala.util.Try
 import scala.util.matching.Regex
 
 import org.apache.hadoop.fs.Path
@@ -38,6 +39,7 @@ import org.apache.spark.sql.catalyst.analysis.{HintErrorLogger, Resolver}
 import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
 import org.apache.spark.sql.catalyst.plans.logical.HintErrorHandler
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.unsafe.array.ByteArrayMethods
 import org.apache.spark.util.Utils
@@ -1483,10 +1485,16 @@ object SQLConf {
     .doubleConf
     .createWithDefault(0.9)
 
+  private def isValidTimezone(zone: String): Boolean = {
+    Try { DateTimeUtils.getZoneId(zone) }.isSuccess
+  }
+
   val SESSION_LOCAL_TIMEZONE =
     buildConf("spark.sql.session.timeZone")
       .doc("""The ID of session local timezone, e.g. "GMT", "America/Los_Angeles", etc.""")
       .stringConf
+      .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
+        " ZoneId.of(_, ZoneId.SHORT_IDS)")
       .createWithDefaultFunction(() => TimeZone.getDefault.getID)
 
   val WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 61be3672f3ebe..48be2115507d4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -348,4 +348,23 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     }
     check(config2)
   }
+
+  test("spark.sql.session.timeZone should only accept valid zone id") {
+    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "MIT")
+    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) === "MIT")
+    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "America/Chicago")
+    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) ===
+      "America/Chicago")
+
+    intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "pst")
+    }
+    intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "GMT+8:00")
+    }
+    val e = intercept[IllegalArgumentException] {
+      spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "Asia/shanghai")
+    }
+    assert(e.getMessage === "Cannot resolve the given timezone with ZoneId.of(_, ZoneId.SHORT_IDS)")
+  }
 }

From 49c07b7335a89533bc25d6ef45f7877b43b6a98d Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 5 Mar 2020 20:26:14 +0800
Subject: [PATCH 0223/1280] fix merge mistakes

---
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 1 -
 .../spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala   | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index cc9c2ae340d3c..3dbfc65f6f98b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2112,7 +2112,6 @@ object SQLConf {
       "MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query " +
       "fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted " +
       "at last takes precedence.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(MapKeyDedupPolicy.values.map(_.toString))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index e5f1fa6f940e9..500b6cc83953a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -609,7 +609,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZEDateFormatter.key -> "2000") {
+      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "2000") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)

From 2ff711ca4f17004abf44186d24fbb0f35fa6336d Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 5 Mar 2020 20:49:43 +0800
Subject: [PATCH 0224/1280] [SPARK-31005][SQL] Support time zone ids in casting
 strings to timestamps

### What changes were proposed in this pull request?
In the PR, I propose to change `DateTimeUtils.stringToTimestamp` to support any valid time zone id at the end of input string. After the changes, the function accepts zone ids in the formats:
- no zone id. In that case, the function uses the local session time zone from the SQL config `spark.sql.session.timeZone`
- -[h]h:[m]m
- +[h]h:[m]m
- Z
- Short zone id, see https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#SHORT_IDS
- Zone ID starts with 'UTC+', 'UTC-', 'GMT+', 'GMT-', 'UT+' or 'UT-'. The ID is split in two, with a two or three letter prefix and a suffix starting with the sign. The suffix must be in the formats:
  - +|-h[h]
  - +|-hh[:]mm
  - +|-hh:mm:ss
  - +|-hhmmss
- Region-based zone IDs in the form `{area}/{city}`, such as `Europe/Paris` or `America/New_York`. The default set of region ids is supplied by the IANA Time Zone Database (TZDB).

### Why are the changes needed?
- To use `stringToTimestamp` as a substitution of removed `stringToTime`, see https://github.com/apache/spark/pull/27710#discussion_r385020173
- Improve UX of Spark SQL by allowing flexible formats of zone ids. Currently, Spark accepts only `Z` and zone offsets that can be inconvenient when a time zone offset is shifted due to daylight saving rules. For instance:
```sql
spark-sql> select cast('2015-03-18T12:03:17.123456 Europe/Moscow' as timestamp);
NULL
```

### Does this PR introduce any user-facing change?
Yes. After the changes, casting strings to timestamps allows time zone id at the end of the strings:
```sql
spark-sql> select cast('2015-03-18T12:03:17.123456 Europe/Moscow' as timestamp);
2015-03-18 12:03:17.123456
```

### How was this patch tested?
- Added new test cases to the `string to timestamp` test in `DateTimeUtilsSuite`.
- Run `CastSuite` and `AnsiCastSuite`.

Closes #27753 from MaxGekk/stringToTimestamp-uni-zoneId.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1fd9a91c662a368e348ef96604a79929f814041c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 57 +++++++++----------
 .../catalyst/util/DateTimeUtilsSuite.scala    | 23 +++++++-
 2 files changed, 48 insertions(+), 32 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 448e4b3e3cc13..6811bfc5e0def 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -187,28 +187,28 @@ object DateTimeUtils {
    * `yyyy-[m]m`
    * `yyyy-[m]m-[d]d`
    * `yyyy-[m]m-[d]d `
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]Z`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]-[h]h:[m]m`
-   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us]+[h]h:[m]m`
+   * `yyyy-[m]m-[d]d [h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   * `yyyy-[m]m-[d]dT[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   * `[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   * `T[h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]`
+   *
+   * where `zone_id` should have one of the forms:
+   *   - Z - Zulu time zone UTC+0
+   *   - +|-[h]h:[m]m
+   *   - A short id, see https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#SHORT_IDS
+   *   - An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-,
+   *     and a suffix in the formats:
+   *     - +|-h[h]
+   *     - +|-hh[:]mm
+   *     - +|-hh:mm:ss
+   *     - +|-hhmmss
+   *  - Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
    */
   def stringToTimestamp(s: UTF8String, timeZoneId: ZoneId): Option[SQLTimestamp] = {
     if (s == null) {
       return None
     }
-    var tz: Option[Byte] = None
+    var tz: Option[String] = None
     val segments: Array[Int] = Array[Int](1, 1, 1, 0, 0, 0, 0, 0, 0)
     var i = 0
     var currentSegmentValue = 0
@@ -259,22 +259,21 @@ object DateTimeUtils {
             return None
           }
         } else if (i == 5 || i == 6) {
-          if (b == 'Z') {
+          if (b == '-' || b == '+') {
             segments(i) = currentSegmentValue
             currentSegmentValue = 0
             i += 1
-            tz = Some(43)
-          } else if (b == '-' || b == '+') {
+            tz = Some(new String(bytes, j, 1))
+          } else if (b == '.' && i == 5) {
             segments(i) = currentSegmentValue
             currentSegmentValue = 0
             i += 1
-            tz = Some(b)
-          } else if (b == '.' && i == 5) {
+          } else {
             segments(i) = currentSegmentValue
             currentSegmentValue = 0
             i += 1
-          } else {
-            return None
+            tz = Some(new String(bytes, j, bytes.length - j))
+            j = bytes.length - 1
           }
           if (i == 6  && b != '.') {
             i += 1
@@ -314,11 +313,11 @@ object DateTimeUtils {
       digitsMilli -= 1
     }
     try {
-      val zoneId = if (tz.isEmpty) {
-        timeZoneId
-      } else {
-        val sign = if (tz.get.toChar == '-') -1 else 1
-        ZoneOffset.ofHoursMinutes(sign * segments(7), sign * segments(8))
+      val zoneId = tz match {
+        case None => timeZoneId
+        case Some("+") => ZoneOffset.ofHoursMinutes(segments(7), segments(8))
+        case Some("-") => ZoneOffset.ofHoursMinutes(-segments(7), -segments(8))
+        case Some(zoneName: String) => getZoneId(zoneName.trim)
       }
       val nanoseconds = MICROSECONDS.toNanos(segments(6))
       val localTime = LocalTime.of(segments(3), segments(4), segments(5), nanoseconds.toInt)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 2e689be01f338..7685b62b618ab 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -182,24 +182,29 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       var zoneId = getZoneId("GMT-13:53")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-13:53", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17GMT-13:53", expected)
 
       zoneId = getZoneId("UTC")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17Z", expected)
+      checkStringToTimestamp("2015-03-18 12:03:17UTC", expected)
 
       zoneId = getZoneId("GMT-01:00")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17-01:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17GMT-01:00", expected)
 
       zoneId = getZoneId("GMT+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17 GMT+07:30", expected)
 
       zoneId = getZoneId("GMT+07:03")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:03", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17GMT+07:03", expected)
 
       // tests for the string including milliseconds.
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zid))
@@ -212,27 +217,32 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.456Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17.456Z", expected)
+      checkStringToTimestamp("2015-03-18 12:03:17.456 UTC", expected)
 
       zoneId = getZoneId("GMT-01:00")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123-01:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123 GMT-01:00", expected)
 
       zoneId = getZoneId("GMT+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123 GMT+07:30", expected)
 
       zoneId = getZoneId("GMT+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123GMT+07:30", expected)
 
-      zoneId = getZoneId("GMT+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123121, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123121+7:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123121 GMT+0730", expected)
 
       zoneId = getZoneId("GMT+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.12312+7:30", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.12312 UT+07:30", expected)
 
       expected = Option(time(18, 12, 15, zid = zid))
       checkStringToTimestamp("18:12:15", expected)
@@ -240,10 +250,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       zoneId = getZoneId("GMT+07:30")
       expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("T18:12:15.12312+7:30", expected)
+      checkStringToTimestamp("T18:12:15.12312 UTC+07:30", expected)
 
       zoneId = getZoneId("GMT+07:30")
       expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("18:12:15.12312+7:30", expected)
+      checkStringToTimestamp("18:12:15.12312 GMT+07:30", expected)
 
       expected = Option(date(2011, 5, 6, 7, 8, 9, 100000, zid = zid))
       checkStringToTimestamp("2011-05-06 07:08:09.1000", expected)
@@ -269,8 +281,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       // Truncating the fractional seconds
       zoneId = getZoneId("GMT+00:00")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = zoneId))
-      checkStringToTimestamp(
-        "2015-03-18T12:03:17.123456789+0:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123456789+0:00", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123456789 UTC+0", expected)
+      checkStringToTimestamp("2015-03-18T12:03:17.123456789GMT+00:00", expected)
+
+      zoneId = getZoneId("Europe/Moscow")
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = zoneId))
+      checkStringToTimestamp("2015-03-18T12:03:17.123456 Europe/Moscow", expected)
     }
   }
 

From f34898c5e19c9a35c091eded9652cd5e3d661d19 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 5 Mar 2020 21:31:26 +0800
Subject: [PATCH 0225/1280] [SPARK-31034][CORE] ShuffleBlockFetcherIterator
 should always create request for last block group

### What changes were proposed in this pull request?

This is a bug fix of #27280. This PR fix the bug where `ShuffleBlockFetcherIterator` may forget to create request for the last block group.

### Why are the changes needed?

When (all blocks).sum < `targetRemoteRequestSize` and (all blocks).length > `maxBlocksInFlightPerAddress` and (last block group).size < `maxBlocksInFlightPerAddress`,
`ShuffleBlockFetcherIterator` will not create a request for the last group. Thus, it will lost data for the reduce task.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Updated test.

Closes #27786 from Ngone51/fix_no_request_bug.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2257ce24437f05c417821c02e3e44c77c93f7211)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../storage/ShuffleBlockFetcherIterator.scala |  8 +-
 .../ShuffleBlockFetcherIteratorSuite.scala    | 91 +++++++++++++++----
 2 files changed, 78 insertions(+), 21 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index cd4c86006af5a..2a0447de04c94 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -339,14 +339,14 @@ final class ShuffleBlockFetcherIterator(
         + s"with ${blocks.size} blocks")
     }
 
-    def createFetchRequests(): Unit = {
+    def createFetchRequests(isLast: Boolean): Unit = {
       val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks)
       curBlocks = new ArrayBuffer[FetchBlockInfo]
       if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
         createFetchRequest(mergedBlocks)
       } else {
         mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks =>
-          if (blocks.length == maxBlocksInFlightPerAddress) {
+          if (blocks.length == maxBlocksInFlightPerAddress || isLast) {
             createFetchRequest(blocks)
           } else {
             // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back
@@ -367,12 +367,12 @@ final class ShuffleBlockFetcherIterator(
       // For batch fetch, the actual block in flight should count for merged block.
       val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress
       if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) {
-        createFetchRequests()
+        createFetchRequests(isLast = false)
       }
     }
     // Add in the final request
     if (curBlocks.nonEmpty) {
-      createFetchRequests()
+      createFetchRequests(isLast = true)
     }
   }
 
diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
index 2090a5155a6f1..773629c2c9a49 100644
--- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
@@ -433,32 +433,86 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     assert(blockManager.hostLocalDirManager.get.getCachedHostLocalDirs().size === 1)
   }
 
-  test("fetch continuous blocks in batch respects maxSize and maxBlocks") {
+  test("fetch continuous blocks in batch should respect maxBytesInFlight") {
     val blockManager = mock(classOf[BlockManager])
     val localBmId = BlockManagerId("test-client", "test-local-host", 1)
     doReturn(localBmId).when(blockManager).blockManagerId
 
     // Make sure remote blocks would return the merged block
-    val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 2)
-    val remoteBlocks = Seq[BlockId](
+    val remoteBmId1 = BlockManagerId("test-client-1", "test-client-1", 1)
+    val remoteBmId2 = BlockManagerId("test-client-2", "test-client-2", 2)
+    val remoteBlocks1 = (0 until 15).map(ShuffleBlockId(0, 3, _))
+    val remoteBlocks2 = Seq[BlockId](ShuffleBlockId(0, 4, 0), ShuffleBlockId(0, 4, 1))
+    val mergedRemoteBlocks = Map[BlockId, ManagedBuffer](
+      ShuffleBlockBatchId(0, 3, 0, 3) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 3, 6) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 6, 9) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 9, 12) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 12, 15) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 4, 0, 2) -> createMockManagedBuffer())
+    val transfer = createMockTransfer(mergedRemoteBlocks)
+
+    val blocksByAddress = Seq[(BlockManagerId, Seq[(BlockId, Long, Int)])](
+      (remoteBmId1, remoteBlocks1.map(blockId => (blockId, 100L, 1))),
+      (remoteBmId2, remoteBlocks2.map(blockId => (blockId, 100L, 1)))).toIterator
+
+    val taskContext = TaskContext.empty()
+    val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
+    val iterator = new ShuffleBlockFetcherIterator(
+      taskContext,
+      transfer,
+      blockManager,
+      blocksByAddress,
+      (_, in) => in,
+      1500,
+      Int.MaxValue,
+      Int.MaxValue,
+      Int.MaxValue,
+      true,
+      false,
+      metrics,
+      true)
+
+    var numResults = 0
+    // After initialize(), there will be 6 FetchRequests. And each of the first 5 requests
+    // includes 1 merged block which is merged from 3 shuffle blocks. The last request has 1 merged
+    // block which merged from 2 shuffle blocks. So, only the first 5 requests(5 * 3 * 100 >= 1500)
+    // can be sent. The 6th FetchRequest will hit maxBlocksInFlightPerAddress so it won't
+    // be sent.
+    verify(transfer, times(5)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    while (iterator.hasNext) {
+      val (blockId, inputStream) = iterator.next()
+      // Make sure we release buffers when a wrapped input stream is closed.
+      val mockBuf = mergedRemoteBlocks(blockId)
+      verifyBufferRelease(mockBuf, inputStream)
+      numResults += 1
+    }
+    // The 6th request will be sent after next() is called.
+    verify(transfer, times(6)).fetchBlocks(any(), any(), any(), any(), any(), any())
+    assert(numResults == 6)
+  }
+
+  test("fetch continuous blocks in batch should respect maxBlocksInFlightPerAddress") {
+    val blockManager = mock(classOf[BlockManager])
+    val localBmId = BlockManagerId("test-client", "test-local-host", 1)
+    doReturn(localBmId).when(blockManager).blockManagerId
+
+    // Make sure remote blocks would return the merged block
+    val remoteBmId = BlockManagerId("test-client-1", "test-client-1", 1)
+    val remoteBlocks = Seq(
       ShuffleBlockId(0, 3, 0),
       ShuffleBlockId(0, 3, 1),
-      ShuffleBlockId(0, 3, 2),
       ShuffleBlockId(0, 4, 0),
       ShuffleBlockId(0, 4, 1),
-      ShuffleBlockId(0, 5, 0),
-      ShuffleBlockId(0, 5, 1),
-      ShuffleBlockId(0, 5, 2))
+      ShuffleBlockId(0, 5, 0))
     val mergedRemoteBlocks = Map[BlockId, ManagedBuffer](
-      ShuffleBlockBatchId(0, 3, 0, 3) -> createMockManagedBuffer(),
+      ShuffleBlockBatchId(0, 3, 0, 2) -> createMockManagedBuffer(),
       ShuffleBlockBatchId(0, 4, 0, 2) -> createMockManagedBuffer(),
-      ShuffleBlockBatchId(0, 5, 0, 3) -> createMockManagedBuffer())
-    val transfer = createMockTransfer(mergedRemoteBlocks)
+      ShuffleBlockBatchId(0, 5, 0, 1) -> createMockManagedBuffer())
 
+    val transfer = createMockTransfer(mergedRemoteBlocks)
     val blocksByAddress = Seq[(BlockManagerId, Seq[(BlockId, Long, Int)])](
-      (remoteBmId, remoteBlocks.map(blockId => (blockId, 1L, 1)))
-    ).toIterator
-
+      (remoteBmId, remoteBlocks.map(blockId => (blockId, 100L, 1)))).toIterator
     val taskContext = TaskContext.empty()
     val metrics = taskContext.taskMetrics.createTempShuffleReadMetrics()
     val iterator = new ShuffleBlockFetcherIterator(
@@ -467,7 +521,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       blockManager,
       blocksByAddress,
       (_, in) => in,
-      35,
+      Int.MaxValue,
       Int.MaxValue,
       2,
       Int.MaxValue,
@@ -475,8 +529,12 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       false,
       metrics,
       true)
-
     var numResults = 0
+    // After initialize(), there will be 2 FetchRequests. First one has 2 merged blocks and each
+    // of them is merged from 2 shuffle blocks, second one has 1 merged block which is merged from
+    // 1 shuffle block. So only the first FetchRequest can be sent. The second FetchRequest will
+    // hit maxBlocksInFlightPerAddress so it won't be sent.
+    verify(transfer, times(1)).fetchBlocks(any(), any(), any(), any(), any(), any())
     while (iterator.hasNext) {
       val (blockId, inputStream) = iterator.next()
       // Make sure we release buffers when a wrapped input stream is closed.
@@ -484,8 +542,7 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
       verifyBufferRelease(mockBuf, inputStream)
       numResults += 1
     }
-    // The first 2 batch block ids are in the same fetch request as they don't exceed the max size
-    // and max blocks, so 2 requests in total.
+    // The second request will be sent after next() is called.
     verify(transfer, times(2)).fetchBlocks(any(), any(), any(), any(), any(), any())
     assert(numResults == 3)
   }

From 46b7f1796bd0b96977ce9b473601033f397a3b18 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 6 Mar 2020 00:46:34 +0800
Subject: [PATCH 0226/1280] [SPARK-31037][SQL] refine AQE config names

When introducing AQE to others, I feel the config names are a bit incoherent and hard to use.
This PR refines the config names:
1. remove the "shuffle" prefix. AQE is all about shuffle and we don't need to add the "shuffle" prefix everywhere.
2. `targetPostShuffleInputSize` is obscure, rename to `advisoryShufflePartitionSizeInBytes`.
3. `reducePostShufflePartitions` doesn't match the actual optimization, rename to `coalesceShufflePartitions`
4. `minNumPostShufflePartitions` is obscure, rename it `minPartitionNum` under the `coalesceShufflePartitions` namespace
5. `maxNumPostShufflePartitions` is confusing with the word "max", rename it `initialPartitionNum`
6. `skewedJoinOptimization` is too verbose. skew join is a well-known terminology in database area, we can just say `skewJoin`

Make the config names easy to understand.

deprecate the config `spark.sql.adaptive.shuffle.targetPostShuffleInputSize`

N/A

Closes #27793 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 142 +++++++++---------
 .../spark/sql/execution/ShuffledRowRDD.scala  |   2 +-
 .../adaptive/AdaptiveSparkPlanExec.scala      |   2 +-
 ....scala => CoalesceShufflePartitions.scala} |  14 +-
 .../adaptive/OptimizeSkewedJoin.scala         |  14 +-
 .../exchange/EnsureRequirements.scala         |   4 +-
 ...a => CoalesceShufflePartitionsSuite.scala} |  12 +-
 .../adaptive/AdaptiveQueryExecSuite.scala     |   8 +-
 .../spark/sql/internal/SQLConfSuite.scala     |  26 ++--
 .../spark/sql/sources/BucketedReadSuite.scala |   2 +-
 10 files changed, 117 insertions(+), 109 deletions(-)
 rename sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/{ReduceNumShufflePartitions.scala => CoalesceShufflePartitions.scala} (90%)
 rename sql/core/src/test/scala/org/apache/spark/sql/execution/{ReduceNumShufflePartitionsSuite.scala => CoalesceShufflePartitionsSuite.scala} (96%)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 3dbfc65f6f98b..b2b3d125971a7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -356,8 +356,16 @@ object SQLConf {
     .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be positive")
     .createWithDefault(200)
 
+  val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
+    buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
+      .internal()
+      .doc("(Deprecated since Spark 3.0)")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefaultString("64MB")
+
   val ADAPTIVE_EXECUTION_ENABLED = buildConf("spark.sql.adaptive.enabled")
-    .doc("When true, enable adaptive query execution.")
+    .doc("When true, enable adaptive query execution, which re-optimizes the query plan in the " +
+      "middle of query execution, based on accurate runtime statistics.")
     .booleanConf
     .createWithDefault(false)
 
@@ -365,90 +373,90 @@ object SQLConf {
     .internal()
     .doc("Adaptive query execution is skipped when the query does not have exchanges or " +
       "sub-queries. By setting this config to true (together with " +
-      s"'${ADAPTIVE_EXECUTION_ENABLED.key}' enabled), Spark will force apply adaptive query " +
+      s"'${ADAPTIVE_EXECUTION_ENABLED.key}' set to true), Spark will force apply adaptive query " +
       "execution for all supported queries.")
     .booleanConf
     .createWithDefault(false)
 
-  val REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.reducePostShufflePartitions")
-      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables reducing " +
-        "the number of post-shuffle partitions based on map output statistics.")
-      .booleanConf
-      .createWithDefault(true)
+  val ADVISORY_PARTITION_SIZE_IN_BYTES =
+    buildConf("spark.sql.adaptive.advisoryPartitionSizeInBytes")
+      .doc("The advisory size in bytes of the shuffle partition during adaptive optimization " +
+        s"(when ${ADAPTIVE_EXECUTION_ENABLED.key} is true). It takes effect when Spark " +
+        "coalesces small shuffle partitions or splits skewed shuffle partition.")
+      .fallbackConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
 
-  val FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch")
-      .doc("Whether to fetch the continuous shuffle blocks in batch. Instead of fetching blocks " +
-        "one by one, fetching continuous shuffle blocks for the same map task in batch can " +
-        "reduce IO and improve performance. Note, multiple continuous blocks exist in single " +
-        s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled, this feature also depends " +
-        "on a relocatable serializer, the concatenation support codec in use and the new version " +
-        "shuffle fetch protocol.")
+  val COALESCE_PARTITIONS_ENABLED =
+    buildConf("spark.sql.adaptive.coalescePartitions.enabled")
+      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark will coalesce " +
+        "contiguous shuffle partitions according to the target size (specified by " +
+        s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'), to avoid too many small tasks.")
       .booleanConf
       .createWithDefault(true)
 
-  val SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS =
-    buildConf("spark.sql.adaptive.shuffle.minNumPostShufflePartitions")
-      .doc("The advisory minimum number of post-shuffle partitions used when " +
-        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
+  val COALESCE_PARTITIONS_MIN_PARTITION_NUM =
+    buildConf("spark.sql.adaptive.coalescePartitions.minPartitionNum")
+      .doc("The minimum number of shuffle partitions after coalescing. This configuration only " +
+        s"has an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
+        s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true.")
       .intConf
-      .checkValue(_ > 0, "The minimum shuffle partition number " +
-        "must be a positive integer.")
+      .checkValue(_ > 0, "The minimum number of partitions must be positive.")
       .createWithDefault(1)
 
-  val SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE =
-    buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
-      .doc("The target post-shuffle input size in bytes of a task. This configuration only has " +
-        s"an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
-      .bytesConf(ByteUnit.BYTE)
-      .createWithDefaultString("64MB")
-
-  val SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS =
-    buildConf("spark.sql.adaptive.shuffle.maxNumPostShufflePartitions")
-      .doc("The advisory maximum number of post-shuffle partitions used in adaptive execution. " +
-        "This is used as the initial number of pre-shuffle partitions. By default it equals to " +
-        "spark.sql.shuffle.partitions. This configuration only has an effect when " +
-        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
-        s"'${REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key}' is enabled.")
+  val COALESCE_PARTITIONS_INITIAL_PARTITION_NUM =
+    buildConf("spark.sql.adaptive.coalescePartitions.initialPartitionNum")
+      .doc("The initial number of shuffle partitions before coalescing. By default it equals to " +
+        s"${SHUFFLE_PARTITIONS.key}. This configuration only has an effect when " +
+        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and '${COALESCE_PARTITIONS_ENABLED.key}' " +
+        "are both true.")
       .intConf
-      .checkValue(_ > 0, "The maximum shuffle partition number " +
-        "must be a positive integer.")
+      .checkValue(_ > 0, "The initial number of partitions must be positive.")
       .createOptional
 
+  val FETCH_SHUFFLE_BLOCKS_IN_BATCH =
+    buildConf("spark.sql.adaptive.fetchShuffleBlocksInBatch")
+      .internal()
+      .doc("Whether to fetch the contiguous shuffle blocks in batch. Instead of fetching blocks " +
+        "one by one, fetching contiguous shuffle blocks for the same map task in batch can " +
+        "reduce IO and improve performance. Note, multiple contiguous blocks exist in single " +
+        s"fetch request only happen when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
+        s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true. This feature also depends " +
+        "on a relocatable serializer, the concatenation support codec in use and the new version " +
+        "shuffle fetch protocol.")
+      .booleanConf
+      .createWithDefault(true)
+
   val LOCAL_SHUFFLE_READER_ENABLED =
-    buildConf("spark.sql.adaptive.shuffle.localShuffleReader.enabled")
-    .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled, this enables the " +
-      "optimization of converting the shuffle reader to local shuffle reader for the shuffle " +
-      "exchange of the broadcast hash join in probe side.")
-    .booleanConf
-    .createWithDefault(true)
+    buildConf("spark.sql.adaptive.localShuffleReader.enabled")
+      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark tries to use local " +
+        "shuffle reader to read the shuffle data when the shuffle partitioning is not needed, " +
+        "for example, after converting sort-merge join to broadcast-hash join.")
+      .booleanConf
+      .createWithDefault(true)
 
-  val ADAPTIVE_EXECUTION_SKEWED_JOIN_ENABLED =
-    buildConf("spark.sql.adaptive.skewedJoinOptimization.enabled")
-    .doc("When true and adaptive execution is enabled, a skewed join is automatically handled at " +
-      "runtime.")
-    .booleanConf
-    .createWithDefault(true)
+  val SKEW_JOIN_ENABLED =
+    buildConf("spark.sql.adaptive.skewJoin.enabled")
+      .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark dynamically " +
+        "handles skew in sort-merge join by splitting (and replicating if needed) skewed " +
+        "partitions.")
+      .booleanConf
+      .createWithDefault(true)
 
-  val ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR =
-    buildConf("spark.sql.adaptive.skewedJoinOptimization.skewedPartitionFactor")
-      .doc("A partition is considered as a skewed partition if its size is larger than" +
-        " this factor multiple the median partition size and also larger than " +
-        s" ${SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key}")
+  val SKEW_JOIN_SKEWED_PARTITION_FACTOR =
+    buildConf("spark.sql.adaptive.skewJoin.skewedPartitionFactor")
+      .doc("A partition is considered as skewed if its size is larger than this factor " +
+        "multiplying the median partition size and also larger than " +
+        s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'")
       .intConf
       .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(10)
 
   val NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN =
     buildConf("spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin")
+      .internal()
       .doc("The relation with a non-empty partition ratio lower than this config will not be " +
         "considered as the build side of a broadcast-hash join in adaptive execution regardless " +
         "of its size.This configuration only has an effect when " +
-        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is enabled.")
+        s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is true.")
       .doubleConf
       .checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
       .createWithDefault(0.2)
@@ -2257,7 +2265,9 @@ object SQLConf {
       DeprecatedConfig(ARROW_EXECUTION_ENABLED.key, "3.0",
         s"Use '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' instead of it."),
       DeprecatedConfig(ARROW_FALLBACK_ENABLED.key, "3.0",
-        s"Use '${ARROW_PYSPARK_FALLBACK_ENABLED.key}' instead of it.")
+        s"Use '${ARROW_PYSPARK_FALLBACK_ENABLED.key}' instead of it."),
+      DeprecatedConfig(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "3.0",
+        s"Use '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}' instead of it.")
     )
 
     Map(configs.map { cfg => cfg.key -> cfg } : _*)
@@ -2418,19 +2428,17 @@ class SQLConf extends Serializable with Logging {
 
   def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
 
-  def targetPostShuffleInputSize: Long = getConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
-
-  def fetchShuffleBlocksInBatchEnabled: Boolean = getConf(FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED)
+  def fetchShuffleBlocksInBatch: Boolean = getConf(FETCH_SHUFFLE_BLOCKS_IN_BATCH)
 
   def nonEmptyPartitionRatioForBroadcastJoin: Double =
     getConf(NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN)
 
-  def reducePostShufflePartitionsEnabled: Boolean = getConf(REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED)
+  def coalesceShufflePartitionsEnabled: Boolean = getConf(COALESCE_PARTITIONS_ENABLED)
 
-  def minNumPostShufflePartitions: Int = getConf(SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS)
+  def minShufflePartitionNum: Int = getConf(COALESCE_PARTITIONS_MIN_PARTITION_NUM)
 
-  def maxNumPostShufflePartitions: Int =
-    getConf(SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS).getOrElse(numShufflePartitions)
+  def initialShufflePartitionNum: Int =
+    getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(numShufflePartitions)
 
   def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
index eb022596b1f74..53ab0493f47eb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ShuffledRowRDD.scala
@@ -127,7 +127,7 @@ class ShuffledRowRDD(
       Array.tabulate(dependency.partitioner.numPartitions)(i => CoalescedPartitionSpec(i, i + 1)))
   }
 
-  if (SQLConf.get.fetchShuffleBlocksInBatchEnabled) {
+  if (SQLConf.get.fetchShuffleBlocksInBatch) {
     dependency.rdd.context.setLocalProperty(
       SortShuffleManager.FETCH_SHUFFLE_BLOCKS_IN_BATCH_ENABLED_KEY, "true")
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index f2ebe1a9bf2fc..b74401e8e67d2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -91,7 +91,7 @@ case class AdaptiveSparkPlanExec(
     // before 'ReduceNumShufflePartitions', as the skewed partition handled
     // in 'OptimizeSkewedJoin' rule, should be omitted in 'ReduceNumShufflePartitions'.
     OptimizeSkewedJoin(conf),
-    ReduceNumShufflePartitions(conf),
+    CoalesceShufflePartitions(conf),
     // The rule of 'OptimizeLocalShuffleReader' need to make use of the 'partitionStartIndices'
     // in 'ReduceNumShufflePartitions' rule. So it must be after 'ReduceNumShufflePartitions' rule.
     OptimizeLocalShuffleReader(conf),
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
similarity index 90%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
index 767a4b2cfee6e..a8e2d8ee3c55b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ReduceNumShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -23,14 +23,14 @@ import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.internal.SQLConf
 
 /**
- * A rule to reduce the post shuffle partitions based on the map output statistics, which can
+ * A rule to coalesce the shuffle partitions based on the map output statistics, which can
  * avoid many small reduce tasks that hurt performance.
  */
-case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
-  import ReduceNumShufflePartitions._
+case class CoalesceShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
+  import CoalesceShufflePartitions._
 
   override def apply(plan: SparkPlan): SparkPlan = {
-    if (!conf.reducePostShufflePartitionsEnabled) {
+    if (!conf.coalesceShufflePartitionsEnabled) {
       return plan
     }
     if (!plan.collectLeaves().forall(_.isInstanceOf[QueryStageExec])
@@ -70,8 +70,8 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
           validMetrics.toArray,
           firstPartitionIndex = 0,
           lastPartitionIndex = distinctNumPreShufflePartitions.head,
-          advisoryTargetSize = conf.targetPostShuffleInputSize,
-          minNumPartitions = conf.minNumPostShufflePartitions)
+          advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
+          minNumPartitions = conf.minShufflePartitionNum)
         // This transformation adds new nodes, so we must use `transformUp` here.
         val stageIds = shuffleStages.map(_.id).toSet
         plan.transformUp {
@@ -88,6 +88,6 @@ case class ReduceNumShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
   }
 }
 
-object ReduceNumShufflePartitions {
+object CoalesceShufflePartitions {
   val COALESCED_SHUFFLE_READER_DESCRIPTION = "coalesced"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 979fee1490598..2e8adcf3cd083 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -64,11 +64,11 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   /**
    * A partition is considered as a skewed partition if its size is larger than the median
    * partition size * ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR and also larger than
-   * SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.
+   * ADVISORY_PARTITION_SIZE_IN_BYTES.
    */
   private def isSkewed(size: Long, medianSize: Long): Boolean = {
-    size > medianSize * conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_PARTITION_FACTOR) &&
-      size > conf.getConf(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
+    size > medianSize * conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR) &&
+      size > conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES)
   }
 
   private def medianSize(stats: MapOutputStatistics): Long = {
@@ -87,7 +87,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    * target post-shuffle partition size if avg size is smaller than it.
    */
   private def targetSize(stats: MapOutputStatistics, medianSize: Long): Long = {
-    val targetPostShuffleSize = conf.getConf(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
+    val targetPostShuffleSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES)
     val nonSkewSizes = stats.bytesByPartitionId.filterNot(isSkewed(_, medianSize))
     // It's impossible that all the partitions are skewed, as we use median size to define skew.
     assert(nonSkewSizes.nonEmpty)
@@ -271,7 +271,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       rightStats: MapOutputStatistics,
       nonSkewPartitionIndices: Seq[Int]): Seq[ShufflePartitionSpec] = {
     assert(nonSkewPartitionIndices.nonEmpty)
-    val shouldCoalesce = conf.getConf(SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED)
+    val shouldCoalesce = conf.getConf(SQLConf.COALESCE_PARTITIONS_ENABLED)
     if (!shouldCoalesce || nonSkewPartitionIndices.length == 1) {
       nonSkewPartitionIndices.map(i => CoalescedPartitionSpec(i, i + 1))
     } else {
@@ -280,7 +280,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
         firstPartitionIndex = nonSkewPartitionIndices.head,
         // `lastPartitionIndex` is exclusive.
         lastPartitionIndex = nonSkewPartitionIndices.last + 1,
-        advisoryTargetSize = conf.targetPostShuffleInputSize)
+        advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES))
     }
   }
 
@@ -300,7 +300,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   }
 
   override def apply(plan: SparkPlan): SparkPlan = {
-    if (!conf.getConf(SQLConf.ADAPTIVE_EXECUTION_SKEWED_JOIN_ENABLED)) {
+    if (!conf.getConf(SQLConf.SKEW_JOIN_ENABLED)) {
       return plan
     }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
index ab4176cada527..28ef793ed62db 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
@@ -36,8 +36,8 @@ import org.apache.spark.sql.internal.SQLConf
  */
 case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
   private def defaultNumPreShufflePartitions: Int =
-    if (conf.adaptiveExecutionEnabled && conf.reducePostShufflePartitionsEnabled) {
-      conf.maxNumPostShufflePartitions
+    if (conf.adaptiveExecutionEnabled && conf.coalesceShufflePartitionsEnabled) {
+      conf.initialShufflePartitionNum
     } else {
       conf.numShufflePartitions
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
similarity index 96%
rename from sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
rename to sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
index a32b684bc85c8..9e77f618eded6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ReduceNumShufflePartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/CoalesceShufflePartitionsSuite.scala
@@ -23,13 +23,13 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql._
 import org.apache.spark.sql.execution.adaptive._
+import org.apache.spark.sql.execution.adaptive.CoalesceShufflePartitions.COALESCED_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.adaptive.CustomShuffleReaderExec
-import org.apache.spark.sql.execution.adaptive.ReduceNumShufflePartitions.COALESCED_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 
-class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAll {
+class CoalesceShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterAll {
 
   private var originalActiveSparkSession: Option[SparkSession] = _
   private var originalInstantiatedSparkSession: Option[SparkSession] = _
@@ -65,17 +65,17 @@ class ReduceNumShufflePartitionsSuite extends SparkFunSuite with BeforeAndAfterA
         .setAppName("test")
         .set(UI_ENABLED, false)
         .set(SQLConf.SHUFFLE_PARTITIONS.key, "5")
-        .set(SQLConf.SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS.key, "5")
+        .set(SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key, "5")
         .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")
         .set(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, "-1")
         .set(
-          SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key,
+          SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key,
           targetPostShuffleInputSize.toString)
     minNumPostShufflePartitions match {
       case Some(numPartitions) =>
-        sparkConf.set(SQLConf.SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS.key, numPartitions.toString)
+        sparkConf.set(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key, numPartitions.toString)
       case None =>
-        sparkConf.set(SQLConf.SHUFFLE_MIN_NUM_POSTSHUFFLE_PARTITIONS.key, "1")
+        sparkConf.set(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key, "1")
     }
 
     val spark = SparkSession.builder()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 500b6cc83953a..a7fa63d66a4d2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -135,7 +135,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "10") {
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "10") {
       val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
         "SELECT * FROM testData join testData2 ON key = a where value = '1'")
       val smj = findTopLevelSortMergeJoin(plan)
@@ -167,7 +167,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80",
-      SQLConf.REDUCE_POST_SHUFFLE_PARTITIONS_ENABLED.key -> "false") {
+      SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false") {
       val (plan, adaptivePlan) = runAdaptiveAndVerifyResult(
         "SELECT * FROM testData join testData2 ON key = a where value = '1'")
       val smj = findTopLevelSortMergeJoin(plan)
@@ -584,7 +584,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "700") {
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "700") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
@@ -609,7 +609,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key -> "2000") {
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "2000") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 48be2115507d4..10ea948656b17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -170,33 +170,33 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     assert(e.getMessage === s"${SQLConf.CASE_SENSITIVE.key} should be boolean, but was 10")
   }
 
-  test("Test SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE's method") {
+  test("Test ADVISORY_PARTITION_SIZE_IN_BYTES's method") {
     spark.sessionState.conf.clear()
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "100")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 100)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "100")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 100)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "1k")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 1024)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "1k")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 1024)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "1M")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 1048576)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "1M")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 1048576)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "1g")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === 1073741824)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "1g")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === 1073741824)
 
-    spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "-1")
-    assert(spark.conf.get(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE) === -1)
+    spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "-1")
+    assert(spark.conf.get(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES) === -1)
 
     // Test overflow exception
     intercept[IllegalArgumentException] {
       // This value exceeds Long.MaxValue
-      spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "90000000000g")
+      spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "90000000000g")
     }
 
     intercept[IllegalArgumentException] {
       // This value less than Long.MinValue
-      spark.conf.set(SQLConf.SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "-90000000000g")
+      spark.conf.set(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key, "-90000000000g")
     }
 
     spark.sessionState.conf.clear()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index b54f4f2295e57..57bbf200ed881 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -821,7 +821,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
   test("SPARK-29655 Read bucketed tables obeys spark.sql.shuffle.partitions") {
     withSQLConf(
       SQLConf.SHUFFLE_PARTITIONS.key -> "5",
-      SQLConf.SHUFFLE_MAX_NUM_POSTSHUFFLE_PARTITIONS.key -> "7")  {
+      SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "7")  {
       val bucketSpec = Some(BucketSpec(6, Seq("i", "j"), Nil))
       Seq(false, true).foreach { enableAdaptive =>
         withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> s"$enableAdaptive") {

From 80a894785121777aedfe340d0acacbbd17630cb3 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 5 Mar 2020 10:56:49 -0800
Subject: [PATCH 0227/1280] [SPARK-31052][TEST][CORE] Fix flaky test
 "DAGSchedulerSuite.shuffle fetch failed on speculative task, but original
 task succeed"

### What changes were proposed in this pull request?

This PR fix the flaky test in #27050.

### Why are the changes needed?

`SparkListenerStageCompleted` is posted by `listenerBus` asynchronously. So, we should make sure listener has consumed the event before asserting completed stages.

See [error message](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/119308/testReport/org.apache.spark.scheduler/DAGSchedulerSuite/shuffle_fetch_failed_on_speculative_task__but_original_task_succeed__SPARK_30388_/):

```
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: List(0, 1, 1) did not equal List(0, 1, 1, 0)
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
	at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
	at org.apache.spark.scheduler.DAGSchedulerSuite.$anonfun$new$88(DAGSchedulerSuite.scala:1976)
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Update test and test locally by no failure after running hundreds of times. Note, the failure is easy to reproduce when loop running the test for hundreds of times(e.g 200)

Closes #27809 from Ngone51/fix_flaky_spark_30388.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit 8d5ef2f766166cce3cc7a15a98ec016050ede4d8)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 72a2e4ca2d5a1..02bc216d2192d 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -1931,7 +1931,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assertDataStructuresEmpty()
   }
 
-  test("shuffle fetch failed on speculative task, but original task succeed (SPARK-30388)") {
+  test("SPARK-30388: shuffle fetch failed on speculative task, but original task succeed") {
     var completedStage: List[Int] = Nil
     val listener = new SparkListener() {
       override def onStageCompleted(event: SparkListenerStageCompleted): Unit = {
@@ -1945,6 +1945,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     val reduceRdd = new MyRDD(sc, 2, List(shuffleDep))
     submit(reduceRdd, Array(0, 1))
     completeShuffleMapStageSuccessfully(0, 0, 2)
+    sc.listenerBus.waitUntilEmpty()
     assert(completedStage === List(0))
 
     // result task 0.0 succeed
@@ -1960,6 +1961,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
         info
       )
     )
+    sc.listenerBus.waitUntilEmpty()
     assert(completedStage === List(0, 1))
 
     Thread.sleep(DAGScheduler.RESUBMIT_TIMEOUT * 2)
@@ -1971,6 +1973,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
     // original result task 1.0 succeed
     runEvent(makeCompletionEvent(taskSets(1).tasks(1), Success, 42))
+    sc.listenerBus.waitUntilEmpty()
     assert(completedStage === List(0, 1, 1, 0))
     assert(scheduler.activeJobs.isEmpty)
   }

From 515eb9de2b043ccef158e22a935c7c50b5b084f2 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Thu, 5 Mar 2020 14:50:18 -0800
Subject: [PATCH 0228/1280] [SPARK-31013][CORE][WEBUI] InMemoryStore: improve
 removeAllByIndexValues over natural key index

### What changes were proposed in this pull request?

The method `removeAllByIndexValues` in KVStore is to delete all the objects which have certain values in the given index.
However, in the current implementation of `InMemoryStore`, when the given index is the natural key index, there is no special handling for it and a linear search over all the task data is performed.

We can improve it by deleting the natural keys directly from the internal hashmap.

### Why are the changes needed?

Better performance if the given index for `removeAllByIndexValues` is the natural key index in
`InMemoryStore`
### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Enhance the existing test.

Closes #27796 from gengliangwang/cherry-pick-27763.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../spark/util/kvstore/InMemoryStore.java      | 16 +++++++++++-----
 .../spark/util/kvstore/InMemoryStoreSuite.java | 18 ++++++++++++------
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index db08740975176..f1bebb4768ead 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -231,11 +231,16 @@ KVTypeInfo.Accessor getIndexAccessor(String indexName) {
     }
 
     int countingRemoveAllByIndexValues(String index, Collection<?> indexValues) {
-      if (hasNaturalParentIndex && naturalParentIndexName.equals(index)) {
+      int count = 0;
+      if (KVIndex.NATURAL_INDEX_NAME.equals(index)) {
+        for (Object naturalKey : indexValues) {
+          count += delete(asKey(naturalKey)) ? 1 : 0;
+        }
+        return count;
+      } else if (hasNaturalParentIndex && naturalParentIndexName.equals(index)) {
         // If there is a parent index for the natural index and `index` happens to be it,
         // Spark can use the `parentToChildrenMap` to get the related natural keys, and then
         // delete them from `data`.
-        int count = 0;
         for (Object indexValue : indexValues) {
           Comparable<Object> parentKey = asKey(indexValue);
           NaturalKeys children = parentToChildrenMap.getOrDefault(parentKey, new NaturalKeys());
@@ -271,9 +276,9 @@ public void put(T value) throws Exception {
       }
     }
 
-    public void delete(Object key) {
-      data.remove(asKey(key));
-      if (hasNaturalParentIndex) {
+    public boolean delete(Object key) {
+      boolean entryExists = data.remove(asKey(key)) != null;
+      if (entryExists && hasNaturalParentIndex) {
         for (NaturalKeys v : parentToChildrenMap.values()) {
           if (v.remove(asKey(key))) {
             // `v` can be empty after removing the natural key and we can remove it from
@@ -284,6 +289,7 @@ public void delete(Object key) {
           }
         }
       }
+      return entryExists;
     }
 
     public int size() {
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
index 9e34225e14e18..da5267669d5f1 100644
--- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
@@ -158,23 +158,29 @@ public void testRemoveAll() throws Exception {
 
     assertEquals(9, store.count(ArrayKeyIndexType.class));
 
+    // Try removing non-existing keys
+    assert(!store.removeAllByIndexValues(
+      ArrayKeyIndexType.class,
+      KVIndex.NATURAL_INDEX_NAME,
+      ImmutableSet.of(new int[] {10, 10, 10}, new int[] { 3, 3, 3 })));
+    assertEquals(9, store.count(ArrayKeyIndexType.class));
 
-    store.removeAllByIndexValues(
+    assert(store.removeAllByIndexValues(
       ArrayKeyIndexType.class,
       KVIndex.NATURAL_INDEX_NAME,
-      ImmutableSet.of(new int[] {0, 0, 0}, new int[] { 2, 2, 2 }));
+      ImmutableSet.of(new int[] {0, 0, 0}, new int[] { 2, 2, 2 })));
     assertEquals(7, store.count(ArrayKeyIndexType.class));
 
-    store.removeAllByIndexValues(
+    assert(store.removeAllByIndexValues(
       ArrayKeyIndexType.class,
       "id",
-      ImmutableSet.of(new String [] { "things" }));
+      ImmutableSet.of(new String [] { "things" })));
     assertEquals(4, store.count(ArrayKeyIndexType.class));
 
-    store.removeAllByIndexValues(
+    assert(store.removeAllByIndexValues(
       ArrayKeyIndexType.class,
       "id",
-      ImmutableSet.of(new String [] { "more things" }));
+      ImmutableSet.of(new String [] { "more things" })));
     assertEquals(0, store.count(ArrayKeyIndexType.class));
   }
 

From 853f69a4cf4bf138afb9075325ad175bc5f2d334 Mon Sep 17 00:00:00 2001
From: DB Tsai <d_tsai@apple.com>
Date: Fri, 6 Mar 2020 00:13:57 +0000
Subject: [PATCH 0229/1280] [SPARK-31058][SQL][TEST-HIVE1.2] Consolidate the
 implementation of `quoteIfNeeded`

### What changes were proposed in this pull request?
There are two implementation of quoteIfNeeded.  One is in `org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quote` and the other is in `OrcFiltersBase.quoteAttributeNameIfNeeded`. This PR will consolidate them into one.

### Why are the changes needed?
Simplify the codebase.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing UTs.

Closes #27814 from dbtsai/SPARK-31058.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit fe126a6a05b0de8db68c8f890e876ce96bf194ca)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 .../sql/connector/catalog/IdentifierImpl.java |  2 +-
 .../catalog/CatalogV2Implicits.scala          | 10 ++++-----
 .../spark/sql/connector/catalog/V1Table.scala | 13 +++---------
 .../analysis/ResolveSessionCatalog.scala      |  2 +-
 .../datasources/orc/OrcFiltersBase.scala      | 10 ---------
 .../datasources/orc/OrcFilters.scala          | 21 +++++++++++--------
 .../datasources/orc/OrcFilters.scala          | 21 +++++++++++--------
 7 files changed, 34 insertions(+), 45 deletions(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
index a56007b2a5ab8..30596d92faaa2 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
@@ -55,7 +55,7 @@ public String name() {
   @Override
   public String toString() {
     return Stream.concat(Stream.of(namespace), Stream.of(name))
-      .map(CatalogV2Implicits::quote)
+      .map(CatalogV2Implicits::quoteIfNeeded)
       .collect(Collectors.joining("."));
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
index 3478af8783af6..71bab624f06a8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
@@ -84,15 +84,15 @@ private[sql] object CatalogV2Implicits {
   }
 
   implicit class NamespaceHelper(namespace: Array[String]) {
-    def quoted: String = namespace.map(quote).mkString(".")
+    def quoted: String = namespace.map(quoteIfNeeded).mkString(".")
   }
 
   implicit class IdentifierHelper(ident: Identifier) {
     def quoted: String = {
       if (ident.namespace.nonEmpty) {
-        ident.namespace.map(quote).mkString(".") + "." + quote(ident.name)
+        ident.namespace.map(quoteIfNeeded).mkString(".") + "." + quoteIfNeeded(ident.name)
       } else {
-        quote(ident.name)
+        quoteIfNeeded(ident.name)
       }
     }
 
@@ -122,10 +122,10 @@ private[sql] object CatalogV2Implicits {
           s"$quoted is not a valid TableIdentifier as it has more than 2 name parts.")
     }
 
-    def quoted: String = parts.map(quote).mkString(".")
+    def quoted: String = parts.map(quoteIfNeeded).mkString(".")
   }
 
-  def quote(part: String): String = {
+  def quoteIfNeeded(part: String): String = {
     if (part.contains(".") || part.contains("`")) {
       s"`${part.replace("`", "``")}`"
     } else {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
index 91e0c58a1c6d0..70fc9689e6087 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/V1Table.scala
@@ -24,6 +24,7 @@ import scala.collection.mutable
 
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.connector.expressions.{LogicalExpressions, Transform}
 import org.apache.spark.sql.types.StructType
 
@@ -35,20 +36,12 @@ private[sql] case class V1Table(v1Table: CatalogTable) extends Table {
     def quoted: String = {
       identifier.database match {
         case Some(db) =>
-          Seq(db, identifier.table).map(quote).mkString(".")
+          Seq(db, identifier.table).map(quoteIfNeeded).mkString(".")
         case _ =>
-          quote(identifier.table)
+          quoteIfNeeded(identifier.table)
 
       }
     }
-
-    private def quote(part: String): String = {
-      if (part.contains(".") || part.contains("`")) {
-        s"`${part.replace("`", "``")}`"
-      } else {
-        part
-      }
-    }
   }
 
   def catalogTable: CatalogTable = v1Table
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 7950b61318e2c..90a3fb8056f97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -95,7 +95,7 @@ class ResolveSessionCatalog(
               .map(_._2.dataType)
               .getOrElse {
                 throw new AnalysisException(
-                  s"ALTER COLUMN cannot find column ${quote(colName)} in v1 table. " +
+                  s"ALTER COLUMN cannot find column ${quoteIfNeeded(colName)} in v1 table. " +
                     s"Available: ${v1Table.schema.fieldNames.mkString(", ")}")
               }
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
index 0b5658715377a..e673309188756 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
@@ -36,16 +36,6 @@ trait OrcFiltersBase {
     }
   }
 
-  // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
-  // in order to distinguish predicate pushdown for nested columns.
-  protected[sql] def quoteAttributeNameIfNeeded(name: String) : String = {
-    if (!name.contains("`") && name.contains(".")) {
-      s"`$name`"
-    } else {
-      name
-    }
-  }
-
   /**
    * Return true if this is a searchable type in ORC.
    * Both CharType and VarcharType are cleaned at AstBuilder.
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 995c5ed317de1..b9cbc484e1fc1 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -24,6 +24,7 @@ import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.orc.storage.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 
@@ -218,47 +219,49 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
     // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
     // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
+    // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
+    // in order to distinguish predicate pushdown for nested columns.
     expression match {
       case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
 
       case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
 
       case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
 
       case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
 
       case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
 
       case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
 
       case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
 
       case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
 
       case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
         Some(builder.startAnd().in(quotedName, getType(attribute),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 948ab44a8c19c..6e9e592be13be 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -24,6 +24,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 
@@ -218,47 +219,49 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     // NOTE: For all case branches dealing with leaf predicates below, the additional `startAnd()`
     // call is mandatory. ORC `SearchArgument` builder requires that all leaf predicates must be
     // wrapped by a "parent" predicate (`And`, `Or`, or `Not`).
+    // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
+    // in order to distinguish predicate pushdown for nested columns.
     expression match {
       case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
 
       case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
 
       case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
 
       case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
 
       case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
 
       case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
 
       case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
 
       case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
 
       case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteAttributeNameIfNeeded(attribute)
+        val quotedName = quoteIfNeeded(attribute)
         val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
         Some(builder.startAnd().in(quotedName, getType(attribute),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())

From 32dc6acdf5615cdd668fa65249d6d170692acb19 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 6 Mar 2020 09:39:02 +0900
Subject: [PATCH 0230/1280] [SPARK-30994][BUILD][FOLLOW-UP] Change scope of
 xml-apis to include it and add xerces in SBT as dependency override

### What changes were proposed in this pull request?

This PR propose

1. Explicitly include xml-apis. xml-apis is already the part of xerces 2.12.0 (https://repo1.maven.org/maven2/xerces/xercesImpl/2.12.0/xercesImpl-2.12.0.pom). However, we're excluding it by setting `scope` to `test`. This seems causing `spark-shell`, built from Maven, to fail.

    Seems like previously xml-apis wasn't reached for some reasons but after we upgrade, it seems requiring. Therefore, this PR proposes to include it.

2. Pins `xerces` version in SBT as well. Seems this dependency is resolved differently from Maven.

Note that Hadoop 3 does not looks requiring this as they replaced xerces as of [HDFS-12221](https://issues.apache.org/jira/browse/HDFS-12221).

### Why are the changes needed?

To make `spark-shell` working from Maven build, and uses the same xerces version.

### Does this PR introduce any user-facing change?

No, it's master only.

### How was this patch tested?

**1.**

```bash
./build/mvn -DskipTests -Psparkr -Phive clean package
./bin/spark-shell
```

Before:

```
Exception in thread "main" java.lang.NoClassDefFoundError: org/w3c/dom/ElementTraversal
	at java.lang.ClassLoader.defineClass1(Native Method)
	at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
	at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at org.apache.xerces.parsers.AbstractDOMParser.startDocument(Unknown Source)
	at org.apache.xerces.xinclude.XIncludeHandler.startDocument(Unknown Source)
	at org.apache.xerces.impl.dtd.XMLDTDValidator.startDocument(Unknown Source)
	at org.apache.xerces.impl.XMLDocumentScannerImpl.startEntity(Unknown Source)
	at org.apache.xerces.impl.XMLVersionDetector.startDocumentParsing(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
	at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
	at org.apache.xerces.parsers.DOMParser.parse(Unknown Source)
	at org.apache.xerces.jaxp.DocumentBuilderImpl.parse(Unknown Source)
	at javax.xml.parsers.DocumentBuilder.parse(DocumentBuilder.java:150)
	at org.apache.hadoop.conf.Configuration.parse(Configuration.java:2482)
	at org.apache.hadoop.conf.Configuration.parse(Configuration.java:2470)
	at org.apache.hadoop.conf.Configuration.loadResource(Configuration.java:2541)
	at org.apache.hadoop.conf.Configuration.loadResources(Configuration.java:2494)
	at org.apache.hadoop.conf.Configuration.getProps(Configuration.java:2407)
	at org.apache.hadoop.conf.Configuration.set(Configuration.java:1143)
	at org.apache.hadoop.conf.Configuration.set(Configuration.java:1115)
	at org.apache.spark.deploy.SparkHadoopUtil$.org$apache$spark$deploy$SparkHadoopUtil$$appendS3AndSparkHadoopHiveConfigurations(SparkHadoopUtil.scala:456)
	at org.apache.spark.deploy.SparkHadoopUtil$.newConfiguration(SparkHadoopUtil.scala:427)
	at org.apache.spark.deploy.SparkSubmit.$anonfun$prepareSubmitEnvironment$2(SparkSubmit.scala:342)
	at scala.Option.getOrElse(Option.scala:189)
	at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:342)
	at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:871)
	at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
	at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
	at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
	at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007)
	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016)
	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
	at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	... 42 more
```

After:

```
...
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 3.1.0-SNAPSHOT
      /_/

Using Scala version 2.12.10 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_202)
Type in expressions to have them evaluated.
Type :help for more information.

scala>
```

**2.**

```
./build/sbt dependencyTree -Phadoop-2.7 -Phive-2.3 -Phive-thriftserver -Phive
./build/sbt dependencyTree -Phadoop-3.2 -Phive-2.3 -Phive-thriftserver -Phive
```

Closes #27808 from HyukjinKwon/SPARK-30994.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 1 +
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 1 +
 pom.xml                                 | 1 -
 project/SparkBuild.scala                | 1 +
 4 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 364b825041c1a..828b1a66d949f 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -202,6 +202,7 @@ threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xercesImpl/2.12.0//xercesImpl-2.12.0.jar
+xml-apis/1.4.01//xml-apis-1.4.01.jar
 xmlenc/0.52//xmlenc-0.52.jar
 xz/1.5//xz-1.5.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 62d14366a0e09..8a65540ceee1b 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -216,6 +216,7 @@ univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
 velocity/1.5//velocity-1.5.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xercesImpl/2.12.0//xercesImpl-2.12.0.jar
+xml-apis/1.4.01//xml-apis-1.4.01.jar
 xmlenc/0.52//xmlenc-0.52.jar
 xz/1.5//xz-1.5.jar
 zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
diff --git a/pom.xml b/pom.xml
index f98d132396190..8a4619743e593 100644
--- a/pom.xml
+++ b/pom.xml
@@ -612,7 +612,6 @@
         <groupId>xml-apis</groupId>
         <artifactId>xml-apis</artifactId>
         <version>1.4.01</version>
-        <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.slf4j</groupId>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index fcde1e958f35a..b606bdd649d66 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -621,6 +621,7 @@ object KubernetesIntegrationTests {
 object DependencyOverrides {
   lazy val settings = Seq(
     dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
+    dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
     dependencyOverrides += "jline" % "jline" % "2.14.6")
 }
 

From ed7924a86e2c4dd6b38075de68038e1997b2b30e Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 5 Mar 2020 16:54:59 -0800
Subject: [PATCH 0231/1280] [SQL][DOCS][MINOR] Fix typos and wrong phrases in
 docs

### What changes were proposed in this pull request?

This PR intends to fix typos and phrases in the `/docs` directory. To find them, I run the Intellij typo checker.

### Why are the changes needed?

For better documents.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A

Closes #27819 from maropu/TypoFix-20200306.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit ffec7a1964984a7f911ac0da125134c098f273ba)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 docs/sql-pyspark-pandas-with-arrow.md        | 2 +-
 docs/sql-ref-ansi-compliance.md              | 2 +-
 docs/sql-ref-null-semantics.md               | 2 +-
 docs/sql-ref-syntax-aux-show-functions.md    | 2 +-
 docs/sql-ref-syntax-ddl-alter-table.md       | 2 +-
 docs/sql-ref-syntax-ddl-create-table-like.md | 2 +-
 docs/sql-ref-syntax-qry-select-limit.md      | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/sql-pyspark-pandas-with-arrow.md b/docs/sql-pyspark-pandas-with-arrow.md
index 63ba0ba5e1b9f..e8abb9fed1b2b 100644
--- a/docs/sql-pyspark-pandas-with-arrow.md
+++ b/docs/sql-pyspark-pandas-with-arrow.md
@@ -91,7 +91,7 @@ specify the type hints of `pandas.Series` and `pandas.DataFrame` as below:
 </div>
 </p>
 
-In the following sections, it describes the cominations of the supported type hints. For simplicity,
+In the following sections, it describes the combinations of the supported type hints. For simplicity,
 `pandas.DataFrame` variant is omitted.
 
 ### Series to Series
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 267184a1cb5fa..27e60b4bb8e3b 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -60,7 +60,7 @@ The following subsections present behaviour changes in arithmetic operations, ty
 ### Arithmetic Operations
 
 In Spark SQL, arithmetic operations performed on numeric types (with the exception of decimal) are not checked for overflows by default.
-This means that in case an operation causes overflows, the result is the same that the same operation returns in a Java/Scala program (e.g., if the sum of 2 integers is higher than the maximum value representable, the result is a negative number).
+This means that in case an operation causes overflows, the result is the same with the corresponding operation in a Java/Scala program (e.g., if the sum of 2 integers is higher than the maximum value representable, the result is a negative number).
 On the other hand, Spark SQL returns null for decimal overflows.
 When `spark.sql.ansi.enabled` is set to `true` and an overflow occurs in numeric and interval arithmetic operations, it throws an arithmetic exception at runtime.
 
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index 3cbc15c600cee..37b4081d6b27b 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -605,7 +605,7 @@ SELECT name, age FROM unknown_age;
 In Spark, EXISTS and NOT EXISTS expressions are allowed inside a WHERE clause. 
 These are boolean expressions which return either `TRUE` or
 `FALSE`. In other words, EXISTS is a membership condition and returns `TRUE`
-when the subquery it refers to returns one or more rows. Similary, NOT EXISTS
+when the subquery it refers to returns one or more rows. Similarly, NOT EXISTS
 is a non-membership condition and returns TRUE when no rows or zero rows are
 returned from the subquery.
 
diff --git a/docs/sql-ref-syntax-aux-show-functions.md b/docs/sql-ref-syntax-aux-show-functions.md
index 701d427039aa5..d6f9df9896afe 100644
--- a/docs/sql-ref-syntax-aux-show-functions.md
+++ b/docs/sql-ref-syntax-aux-show-functions.md
@@ -22,7 +22,7 @@ license: |
 ### Description
 Returns the list of functions after applying an optional regex pattern.
 Given number of functions supported by Spark is quite large, this statement
-in conjuction with [describe function](sql-ref-syntax-aux-describe-function.html)
+in conjunction with [describe function](sql-ref-syntax-aux-describe-function.html)
 may be used to quickly find the function and understand its usage. The `LIKE` 
 clause is optional and supported only for compatibility with other systems.
 
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index a921478daa470..373fa8d8940b0 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -260,7 +260,7 @@ ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways'
 -- SET SERDE/ SERDE Properties
 ALTER TABLE test_tab SET SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe';
 
-ALTER TABLE dbx.tab1 SET SERDE 'org.apache.madoop' WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')
+ALTER TABLE dbx.tab1 SET SERDE 'org.apache.hadoop' WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')
 
 --SET TABLE PROPERTIES
 ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('winner' = 'loser')
diff --git a/docs/sql-ref-syntax-ddl-create-table-like.md b/docs/sql-ref-syntax-ddl-create-table-like.md
index b7d7bdd1d9805..f49fd7fb24c91 100644
--- a/docs/sql-ref-syntax-ddl-create-table-like.md
+++ b/docs/sql-ref-syntax-ddl-create-table-like.md
@@ -74,7 +74,7 @@ USING data_source
 ### Examples
 {% highlight sql %}
 
---Create table using an exsisting table
+--Create table using an existing table
 CREATE TABLE Student_Dupli like Student;
 
 --Create table like using a data source
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 2b9999cc40783..06925e6ee353c 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -20,7 +20,7 @@ license: |
 ---
 The <code>LIMIT</code> clause is used to constrain the number of rows returned by
 the [SELECT](sql-ref-syntax-qry-select.html) statement. In general, this clause
-is used in conjuction with [ORDER BY](sql-ref-syntax-qry-select-orderby.html) to
+is used in conjunction with [ORDER BY](sql-ref-syntax-qry-select-orderby.html) to
 ensure that the results are deterministic.
 
 ### Syntax

From 6a7aa0d1666aa57213ef16cc858dbf105f0810f4 Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Fri, 6 Mar 2020 10:30:59 +0900
Subject: [PATCH 0232/1280] [SPARK-30563][SQL] Disable using commit coordinator
 with NoopDataSource

### What changes were proposed in this pull request?
This PR disables using commit coordinator with `NoopDataSource`.

### Why are the changes needed?
No need for a coordinator in benchmarks.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UTs.

Closes #27791 from peter-toth/SPARK-30563-disalbe-commit-coordinator.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 72b52a3cdf2b8661c0f82db645e805ff6b085dfc)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/datasources/noop/NoopDataSource.scala    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
index 4fad0a2484cde..851cc51466a91 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
@@ -61,6 +61,7 @@ private[noop] object NoopWriteBuilder extends WriteBuilder with SupportsTruncate
 private[noop] object NoopBatchWrite extends BatchWrite {
   override def createBatchWriterFactory(info: PhysicalWriteInfo): DataWriterFactory =
     NoopWriterFactory
+  override def useCommitCoordinator(): Boolean = false
   override def commit(messages: Array[WriterCommitMessage]): Unit = {}
   override def abort(messages: Array[WriterCommitMessage]): Unit = {}
 }

From 5220a1c8756a8ddcf015001c7ab5d8fa02ab8692 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 6 Mar 2020 10:33:20 +0900
Subject: [PATCH 0233/1280] [SPARK-31036][SQL] Use stringArgs in
 Expression.toString to respect hidden parameters

### What changes were proposed in this pull request?

This PR proposes to respect hidden parameters by using `stringArgs`  in `Expression.toString `. By this, we can show the strings properly in some cases such as `NonSQLExpression`.

### Why are the changes needed?

To respect "hidden" arguments in the string representation.

### Does this PR introduce any user-facing change?

Yes, for example, on the top of https://github.com/apache/spark/pull/27657,

```scala
val identify = udf((input: Seq[Int]) => input)
spark.range(10).select(identify(array("id"))).show()
```

shows hidden parameter `useStringTypeWhenEmpty`.

```
+---------------------+
|UDF(array(id, false))|
+---------------------+
|                  [0]|
|                  [1]|
...
```

whereas:

```scala
spark.range(10).select(array("id")).show()
```

```
+---------+
|array(id)|
+---------+
|      [0]|
|      [1]|
...
```

### How was this patch tested?

Manually tested as below:

```scala
val identify = udf((input: Boolean) => input)
spark.range(10).select(identify(exists(array(col("id")), _ % 2 === 0))).show()
```

Before:

```
+-------------------------------------------------------------------------------------+
|UDF(exists(array(id), lambdafunction(((lambda 'x % 2) = 0), lambda 'x, false), true))|
+-------------------------------------------------------------------------------------+
|                                                                                 true|
|                                                                                false|
|                                                                                 true|
...
```

After:

```
+-------------------------------------------------------------------------------+
|UDF(exists(array(id), lambdafunction(((lambda 'x % 2) = 0), lambda 'x, false)))|
+-------------------------------------------------------------------------------+
|                                                                           true|
|                                                                          false|
|                                                                           true|
...
```

Closes #27788 from HyukjinKwon/arguments-str-repr.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit fc12165f48b2e1dfe04116ddaa6ff6e8650a18fb)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/catalyst/expressions/Expression.scala  | 2 +-
 .../spark/sql/catalyst/expressions/higherOrderFunctions.scala   | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 4632957e7afdd..1599321982ce2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -260,7 +260,7 @@ abstract class Expression extends TreeNode[Expression] {
    */
   def prettyName: String = nodeName.toLowerCase(Locale.ROOT)
 
-  protected def flatArguments: Iterator[Any] = productIterator.flatMap {
+  protected def flatArguments: Iterator[Any] = stringArgs.flatMap {
     case t: Iterable[_] => t
     case single => single :: Nil
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 9dd4263fdeae3..e91bd0c7015ee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -533,6 +533,8 @@ case class ArrayExists(
       SQLConf.get.getConf(SQLConf.LEGACY_ARRAY_EXISTS_FOLLOWS_THREE_VALUED_LOGIC))
   }
 
+  override def stringArgs: Iterator[Any] = super.stringArgs.take(2)
+
   override def nullable: Boolean =
     if (followThreeValuedLogic) {
       super.nullable || function.nullable

From 6d8ee156adcac8fd2a442ab70da750f950a83661 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 6 Mar 2020 10:35:44 +0900
Subject: [PATCH 0234/1280] [MINOR][SQL] Remove an ignored test from JsonSuite

### What changes were proposed in this pull request?
Remove ignored and outdated test `Type conflict in primitive field values (Ignored)` from JsonSuite.

### Why are the changes needed?
The test is not maintained for long time. It can be removed to reduce size of JsonSuite, and improve maintainability.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running the command `./build/sbt "test:testOnly *JsonV2Suite"`

Closes #27795 from MaxGekk/remove-ignored-test-in-JsonSuite.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit cf7c397ede05fd106697bd5cc8062f394623bf22)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../datasources/json/JsonSuite.scala          | 53 -------------------
 1 file changed, 53 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index df0bca7f0258d..fb3328c51546e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -481,59 +481,6 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     )
   }
 
-  ignore("Type conflict in primitive field values (Ignored)") {
-    val jsonDF = spark.read.json(primitiveFieldValueTypeConflict)
-    jsonDF.createOrReplaceTempView("jsonTable")
-
-    // Right now, the analyzer does not promote strings in a boolean expression.
-    // Number and Boolean conflict: resolve the type as boolean in this query.
-    checkAnswer(
-      sql("select num_bool from jsonTable where NOT num_bool"),
-      Row(false)
-    )
-
-    checkAnswer(
-      sql("select str_bool from jsonTable where NOT str_bool"),
-      Row(false)
-    )
-
-    // Right now, the analyzer does not know that num_bool should be treated as a boolean.
-    // Number and Boolean conflict: resolve the type as boolean in this query.
-    checkAnswer(
-      sql("select num_bool from jsonTable where num_bool"),
-      Row(true)
-    )
-
-    checkAnswer(
-      sql("select str_bool from jsonTable where str_bool"),
-      Row(false)
-    )
-
-    // The plan of the following DSL is
-    // Project [(CAST(num_str#65:4, DoubleType) + 1.2) AS num#78]
-    //  Filter (CAST(CAST(num_str#65:4, DoubleType), DecimalType) > 92233720368547758060)
-    //    ExistingRdd [num_bool#61,num_num_1#62L,num_num_2#63,num_num_3#64,num_str#65,str_bool#66]
-    // We should directly cast num_str to DecimalType and also need to do the right type promotion
-    // in the Project.
-    checkAnswer(
-      jsonDF.
-        where('num_str >= BigDecimal("92233720368547758060")).
-        select(('num_str + 1.2).as("num")),
-      Row(new java.math.BigDecimal("92233720368547758071.2").doubleValue())
-    )
-
-    // The following test will fail. The type of num_str is StringType.
-    // So, to evaluate num_str + 1.2, we first need to use Cast to convert the type.
-    // In our test data, one value of num_str is 13.1.
-    // The result of (CAST(num_str#65:4, DoubleType) + 1.2) for this value is 14.299999999999999,
-    // which is not 14.3.
-    // Number and String conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_str + 1.2 from jsonTable where num_str > 13"),
-      Row(BigDecimal("14.3")) :: Row(BigDecimal("92233720368547758071.2")) :: Nil
-    )
-  }
-
   test("Type conflict in complex field values") {
     val jsonDF = spark.read.json(complexFieldValueTypeConflict)
 

From db1c3feacdca04c1b72191269f4f3910ad05bcb4 Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Fri, 6 Mar 2020 11:41:45 +0800
Subject: [PATCH 0235/1280] [SPARK-31045][SQL] Add config for AQE logging level

### What changes were proposed in this pull request?
This PR adds an internal config for changing the logging level of adaptive execution query plan evolvement.

### Why are the changes needed?
To make AQE debugging easier.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Added UT.

Closes #27798 from maryannxue/aqe-log-level.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d705d36c0c94d3a4684de6ca0f444557c3cec25e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 12 +++++
 .../adaptive/AdaptiveSparkPlanExec.scala      | 12 ++++-
 .../adaptive/AdaptiveQueryExecSuite.scala     | 47 +++++++++++++++++++
 3 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b2b3d125971a7..cd465bc32e35d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -378,6 +378,16 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
+  val ADAPTIVE_EXECUTION_LOG_LEVEL = buildConf("spark.sql.adaptive.logLevel")
+    .internal()
+    .doc("Configures the log level for adaptive execution logging of plan changes. The value " +
+      "can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.")
+    .version("3.0.0")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR"))
+    .createWithDefault("debug")
+
   val ADVISORY_PARTITION_SIZE_IN_BYTES =
     buildConf("spark.sql.adaptive.advisoryPartitionSizeInBytes")
       .doc("The advisory size in bytes of the shuffle partition during adaptive optimization " +
@@ -2428,6 +2438,8 @@ class SQLConf extends Serializable with Logging {
 
   def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
 
+  def adaptiveExecutionLogLevel: String = getConf(ADAPTIVE_EXECUTION_LOG_LEVEL)
+
   def fetchShuffleBlocksInBatch: Boolean = getConf(FETCH_SHUFFLE_BLOCKS_IN_BATCH)
 
   def nonEmptyPartitionRatioForBroadcastJoin: Double =
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index b74401e8e67d2..fc88a7f7798e3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -66,6 +66,15 @@ case class AdaptiveSparkPlanExec(
 
   @transient private val lock = new Object()
 
+  @transient private val logOnLevel: ( => String) => Unit = conf.adaptiveExecutionLogLevel match {
+    case "TRACE" => logTrace(_)
+    case "DEBUG" => logDebug(_)
+    case "INFO" => logInfo(_)
+    case "WARN" => logWarning(_)
+    case "ERROR" => logError(_)
+    case _ => logDebug(_)
+  }
+
   // The logical plan optimizer for re-optimizing the current logical plan.
   @transient private val optimizer = new RuleExecutor[LogicalPlan] {
     // TODO add more optimization rules
@@ -204,6 +213,7 @@ case class AdaptiveSparkPlanExec(
         val newCost = costEvaluator.evaluateCost(newPhysicalPlan)
         if (newCost < origCost ||
             (newCost == origCost && currentPhysicalPlan != newPhysicalPlan)) {
+          logOnLevel(s"Plan changed from $currentPhysicalPlan to $newPhysicalPlan")
           cleanUpTempTags(newPhysicalPlan)
           currentPhysicalPlan = newPhysicalPlan
           currentLogicalPlan = newLogicalPlan
@@ -217,7 +227,7 @@ case class AdaptiveSparkPlanExec(
       currentPhysicalPlan = applyPhysicalRules(result.newPlan, queryStageOptimizerRules)
       isFinalPlan = true
       executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
-      logDebug(s"Final plan: $currentPhysicalPlan")
+      logOnLevel(s"Final plan: $currentPhysicalPlan")
     }
     currentPhysicalPlan
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index a7fa63d66a4d2..25b1f8938f18f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.execution.adaptive
 import java.io.File
 import java.net.URI
 
+import org.apache.log4j.Level
+
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
@@ -729,5 +731,50 @@ class AdaptiveQueryExecSuite
         s"${SQLConf.ADAPTIVE_EXECUTION_ENABLED.key} is" +
         s" enabled but is not supported for")))
   }
+
+  test("test log level") {
+    def verifyLog(expectedLevel: Level): Unit = {
+      val logAppender = new LogAppender("adaptive execution")
+      withLogAppender(
+        logAppender,
+        loggerName = Some(AdaptiveSparkPlanExec.getClass.getName.dropRight(1)),
+        level = Some(Level.TRACE)) {
+        withSQLConf(
+          SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+          SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") {
+          sql("SELECT * FROM testData join testData2 ON key = a where value = '1'").collect()
+        }
+      }
+      Seq("Plan changed", "Final plan").foreach { msg =>
+        assert(
+          logAppender.loggingEvents.exists { event =>
+            event.getRenderedMessage.contains(msg) && event.getLevel == expectedLevel
+          })
+      }
+    }
+
+    // Verify default log level
+    verifyLog(Level.DEBUG)
+
+    // Verify custom log level
+    val levels = Seq(
+      "TRACE" -> Level.TRACE,
+      "trace" -> Level.TRACE,
+      "DEBUG" -> Level.DEBUG,
+      "debug" -> Level.DEBUG,
+      "INFO" -> Level.INFO,
+      "info" -> Level.INFO,
+      "WARN" -> Level.WARN,
+      "warn" -> Level.WARN,
+      "ERROR" -> Level.ERROR,
+      "error" -> Level.ERROR,
+      "deBUG" -> Level.DEBUG)
+
+    levels.foreach { level =>
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_LOG_LEVEL.key -> level._1) {
+        verifyLog(level._2)
+      }
+    }
+  }
 }
 

From 1535b2bb51782a89b271b1ebe53273ab610b390b Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 5 Mar 2020 20:09:39 -0800
Subject: [PATCH 0236/1280] [SPARK-30886][SQL] Deprecate two-parameter
 TRIM/LTRIM/RTRIM functions

### What changes were proposed in this pull request?

This PR aims to show a deprecation warning on two-parameter TRIM/LTRIM/RTRIM function usages based on the community decision.
- https://lists.apache.org/thread.html/r48b6c2596ab06206b7b7fd4bbafd4099dccd4e2cf9801aaa9034c418%40%3Cdev.spark.apache.org%3E

### Why are the changes needed?

For backward compatibility, SPARK-28093 is reverted. However, from Apache Spark 3.0.0, we should give a safe guideline to use SQL syntax instead of the esoteric function signatures.

### Does this PR introduce any user-facing change?

Yes. This shows a directional warning.

### How was this patch tested?

Pass the Jenkins with a newly added test case.

Closes #27643 from dongjoon-hyun/SPARK-30886.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit afb84e9d378003c57cd01d21cdb1a977ba25454b)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 20 ++++---
 .../sql/catalyst/analysis/AnalysisSuite.scala | 52 +++++++++++++++++++
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 3cb754d665b28..eadcd0f8363c7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.analysis
 
 import java.util
 import java.util.Locale
+import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
@@ -1795,6 +1796,7 @@ class Analyzer(
    * Replaces [[UnresolvedFunction]]s with concrete [[Expression]]s.
    */
   object ResolveFunctions extends Rule[LogicalPlan] {
+    val trimWarningEnabled = new AtomicBoolean(true)
     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       case q: LogicalPlan =>
         q transformExpressions {
@@ -1839,13 +1841,19 @@ class Analyzer(
                   }
                   AggregateExpression(agg, Complete, isDistinct, filter)
                 // This function is not an aggregate function, just return the resolved one.
-                case other =>
-                  if (isDistinct || filter.isDefined) {
-                    failAnalysis("DISTINCT or FILTER specified, " +
-                      s"but ${other.prettyName} is not an aggregate function")
-                  } else {
-                    other
+                case other if (isDistinct || filter.isDefined) =>
+                  failAnalysis("DISTINCT or FILTER specified, " +
+                    s"but ${other.prettyName} is not an aggregate function")
+                case e: String2TrimExpression if arguments.size == 2 =>
+                  if (trimWarningEnabled.get) {
+                    log.warn("Two-parameter TRIM/LTRIM/RTRIM function signatures are deprecated." +
+                      " Use SQL syntax `TRIM((BOTH | LEADING | TRAILING)? trimStr FROM str)`" +
+                      " instead.")
+                    trimWarningEnabled.set(false)
                   }
+                  e
+                case other =>
+                  other
               }
             }
         }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index d38513319388b..8451b9b50eff3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -21,6 +21,7 @@ import java.util.{Locale, TimeZone}
 
 import scala.reflect.ClassTag
 
+import org.apache.log4j.Level
 import org.scalatest.Matchers
 
 import org.apache.spark.api.python.PythonEvalType
@@ -768,4 +769,55 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     assert(message.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
       s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
   }
+
+  test("SPARK-30886 Deprecate two-parameter TRIM/LTRIM/RTRIM") {
+    Seq("trim", "ltrim", "rtrim").foreach { f =>
+      val logAppender = new LogAppender("deprecated two-parameter TRIM/LTRIM/RTRIM functions")
+      def check(count: Int): Unit = {
+        val message = "Two-parameter TRIM/LTRIM/RTRIM function signatures are deprecated."
+        assert(logAppender.loggingEvents.size == count)
+        assert(logAppender.loggingEvents.exists(
+          e => e.getLevel == Level.WARN &&
+            e.getRenderedMessage.contains(message)))
+      }
+
+      withLogAppender(logAppender) {
+        val testAnalyzer1 = new Analyzer(
+          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+
+        val plan1 = testRelation2.select(
+          UnresolvedFunction(f, $"a" :: Nil, isDistinct = false))
+        testAnalyzer1.execute(plan1)
+        // One-parameter is not deprecated.
+        assert(logAppender.loggingEvents.isEmpty)
+
+        val plan2 = testRelation2.select(
+          UnresolvedFunction(f, $"a" :: $"b" :: Nil, isDistinct = false))
+        testAnalyzer1.execute(plan2)
+        // Deprecation warning is printed out once.
+        check(1)
+
+        val plan3 = testRelation2.select(
+          UnresolvedFunction(f, $"b" :: $"a" :: Nil, isDistinct = false))
+        testAnalyzer1.execute(plan3)
+        // There is no change in the log.
+        check(1)
+
+        // New analyzer from new SessionState
+        val testAnalyzer2 = new Analyzer(
+          new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+        val plan4 = testRelation2.select(
+          UnresolvedFunction(f, $"c" :: $"d" :: Nil, isDistinct = false))
+        testAnalyzer2.execute(plan4)
+        // Additional deprecation warning from new analyzer
+        check(2)
+
+        val plan5 = testRelation2.select(
+          UnresolvedFunction(f, $"c" :: $"d" :: Nil, isDistinct = false))
+        testAnalyzer2.execute(plan5)
+        // There is no change in the log.
+        check(2)
+      }
+    }
+  }
 }

From 5375b40f37cefe6f18f9f943412b60c65fe2f844 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 6 Mar 2020 13:00:04 +0800
Subject: [PATCH 0237/1280] [SPARK-31010][SQL][FOLLOW-UP] Deprecate untyped
 scala UDF

### What changes were proposed in this pull request?

Use scala annotation deprecate to deprecate untyped scala UDF.

### Why are the changes needed?

After #27488, it's weird to see the untyped scala UDF will fail by default without deprecation.

### Does this PR introduce any user-facing change?

Yes, user will see the warning:
```
<console>:26: warning: method udf in object functions is deprecated (since 3.0.0): Untyped Scala UDF API is deprecated, please use typed Scala UDF API such as 'def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction' instead.
       val myudf = udf(() => Math.random(), DoubleType)
                   ^
```

### How was this patch tested?

Tested manually.

Closes #27794 from Ngone51/deprecate_untyped_scala_udf.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 587266f887f85bfa2a73a77485ab4db522c6aca1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index c6e8cf76fea1f..7a589573dd7f1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -4732,6 +4732,8 @@ object functions {
    * @group udf_funcs
    * @since 2.0.0
    */
+  @deprecated("Untyped Scala UDF API is deprecated, please use typed Scala UDF API such as " +
+    "'def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction' instead.", "3.0.0")
   def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = {
     if (!SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF)) {
       val errorMsg = "You're using untyped Scala UDF, which does not have the input type " +

From 7c09c9f4d514f47f256b0c1ec6a4b148d4707fa4 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Thu, 5 Mar 2020 21:22:32 -0800
Subject: [PATCH 0238/1280] [SPARK-23817][FOLLOWUP][TEST] Add OrcV2QuerySuite

### What changes were proposed in this pull request?

Add `OrcV2QuerySuite` which explicitly sets the configuration `USE_V1_SOURCE_LIST` as `""` to use ORC V2 implementation.

### Why are the changes needed?

As now file source V2 is disabled by default, the test suite `OrcQuerySuite` is testing V1 implementation as well as the `OrcV1QuerySuite`. We should fix it.
### Does this PR introduce any user-facing change?

No
### How was this patch tested?

Unit test.

Closes #27816 from gengliangwang/orcQuerySuite.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 1426ad89684e7b6c4c255e8714810c1dab3dfae1)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/execution/datasources/orc/OrcQuerySuite.scala    | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
index b8bf4b16fe53c..5f495263b30e3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
@@ -631,7 +631,7 @@ abstract class OrcQueryTest extends OrcTest {
   }
 }
 
-class OrcQuerySuite extends OrcQueryTest with SharedSparkSession {
+abstract class OrcQuerySuite extends OrcQueryTest with SharedSparkSession {
   import testImplicits._
 
   test("LZO compression options for writing to an ORC file") {
@@ -722,3 +722,10 @@ class OrcV1QuerySuite extends OrcQuerySuite {
       .sparkConf
       .set(SQLConf.USE_V1_SOURCE_LIST, "orc")
 }
+
+class OrcV2QuerySuite extends OrcQuerySuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}

From 9b48f3358d3efb523715a5f258e5ed83e28692f6 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Thu, 5 Mar 2020 21:31:08 -0800
Subject: [PATCH 0239/1280] [SPARK-31045][SQL][FOLLOWUP][3.0] Fix build due to
 divergence between master and 3.0

### What changes were proposed in this pull request?

This patch fixes the build failure in `branch-3.0` due to cherry-picking SPARK-31045 to branch-3.0, as `.version()` is not available in `branch-3.0` yet.

### Why are the changes needed?

The build is failing in `branch-3.0`.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Jenkins build will verify.

Closes #27826 from HeartSaVioR/SPARK-31045-branch-3.0-FOLLOWUP.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index cd465bc32e35d..fdaf0ececa5e9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -382,7 +382,6 @@ object SQLConf {
     .internal()
     .doc("Configures the log level for adaptive execution logging of plan changes. The value " +
       "can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR"))

From 35a8e1ed7bb2f9493742d7401e32eab4621fd2d0 Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Fri, 6 Mar 2020 15:40:44 +0800
Subject: [PATCH 0240/1280] [SPARK-31061][SQL] Provide ability to alter the
 provider of a table

This PR adds functionality to HiveExternalCatalog to be able to change the provider of a table.

This is useful for catalogs in Spark 3.0 to be able to use alterTable to change the provider of a table as part of an atomic REPLACE TABLE function.

No

Unit tests

Closes #27822 from brkyvz/externalCat.

Authored-by: Burak Yavuz <brkyvz@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2e3adadc6a53fe044b286a6a59529a94e7eeda6c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/hive/HiveExternalCatalog.scala  | 10 ++++-
 .../sql/hive/HiveExternalCatalogSuite.scala   | 40 +++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index ca292f65efeee..be6d824ece684 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -634,7 +634,15 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
         k.startsWith(DATASOURCE_PREFIX) || k.startsWith(STATISTICS_PREFIX) ||
           k.startsWith(CREATED_SPARK_VERSION)
       }
-      val newTableProps = propsFromOldTable ++ tableDefinition.properties + partitionProviderProp
+      val newFormatIfExists = tableDefinition.provider.flatMap { p =>
+        if (DDLUtils.isDatasourceTable(tableDefinition)) {
+          Some(DATASOURCE_PROVIDER -> p)
+        } else {
+          None
+        }
+      }
+      val newTableProps =
+        propsFromOldTable ++ tableDefinition.properties + partitionProviderProp ++ newFormatIfExists
 
       // // Add old table's owner if we need to restore
       val owner = Option(tableDefinition.owner).filter(_.nonEmpty).getOrElse(oldTableDef.owner)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
index 0a8889885df7b..473a93bf129df 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.hive
 
+import java.net.URI
+
 import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.SparkConf
@@ -178,4 +180,42 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite {
     assertThrows[QueryExecutionException](client.runSqlHive(
       "INSERT overwrite directory \"fs://localhost/tmp\" select 1 as a"))
   }
+
+  test("SPARK-31061: alterTable should be able to change table provider") {
+    val catalog = newBasicCatalog()
+    val parquetTable = CatalogTable(
+      identifier = TableIdentifier("parq_tbl", Some("db1")),
+      tableType = CatalogTableType.MANAGED,
+      storage = storageFormat.copy(locationUri = Some(new URI("file:/some/path"))),
+      schema = new StructType().add("col1", "int").add("col2", "string"),
+      provider = Some("parquet"))
+    catalog.createTable(parquetTable, ignoreIfExists = false)
+
+    val rawTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(rawTable.provider === Some("parquet"))
+
+    val fooTable = parquetTable.copy(provider = Some("foo"))
+    catalog.alterTable(fooTable)
+    val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(alteredTable.provider === Some("foo"))
+  }
+
+  test("SPARK-31061: alterTable should be able to change table provider from hive") {
+    val catalog = newBasicCatalog()
+    val hiveTable = CatalogTable(
+      identifier = TableIdentifier("parq_tbl", Some("db1")),
+      tableType = CatalogTableType.MANAGED,
+      storage = storageFormat,
+      schema = new StructType().add("col1", "int").add("col2", "string"),
+      provider = Some("hive"))
+    catalog.createTable(hiveTable, ignoreIfExists = false)
+
+    val rawTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(rawTable.provider === Some("hive"))
+
+    val fooTable = rawTable.copy(provider = Some("foo"))
+    catalog.alterTable(fooTable)
+    val alteredTable = externalCatalog.getTable("db1", "parq_tbl")
+    assert(alteredTable.provider === Some("foo"))
+  }
 }

From cf5956f058607dac1866fddbee495f0d46c19c05 Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Fri, 6 Mar 2020 16:45:06 +0800
Subject: [PATCH 0241/1280] [SPARK-30899][SQL] CreateArray/CreateMap's data
 type should not depend on SQLConf.get

### What changes were proposed in this pull request?
Introduced a new parameter `emptyCollection` for `CreateMap` and `CreateArray` functiion to remove dependency on SQLConf.get.

### Why are the changes needed?
This allows to avoid the issue when the configuration change between different phases of planning, and this can silently break a query plan which can lead to crashes or data corruption.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing UTs.

Closes #27657 from iRakson/SPARK-30899.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit cba17e07e9f15673f274de1728f6137d600026e1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/TypeCoercion.scala  |  8 ++---
 .../expressions/complexTypeCreator.scala      | 35 ++++++++++++++++---
 .../expressions/complexTypeExtractors.scala   |  4 +--
 .../sql/catalyst/optimizer/ComplexTypes.scala |  8 ++---
 .../optimizer/NormalizeFloatingNumbers.scala  |  8 ++---
 5 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index f416e8efe17ab..0a0bef6c9953a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -553,10 +553,10 @@ object TypeCoercion {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
-      case a @ CreateArray(children) if !haveSameType(children.map(_.dataType)) =>
+      case a @ CreateArray(children, _) if !haveSameType(children.map(_.dataType)) =>
         val types = children.map(_.dataType)
         findWiderCommonType(types) match {
-          case Some(finalDataType) => CreateArray(children.map(castIfNotSameType(_, finalDataType)))
+          case Some(finalDataType) => a.copy(children.map(castIfNotSameType(_, finalDataType)))
           case None => a
         }
 
@@ -592,7 +592,7 @@ object TypeCoercion {
           case None => m
         }
 
-      case m @ CreateMap(children) if m.keys.length == m.values.length &&
+      case m @ CreateMap(children, _) if m.keys.length == m.values.length &&
           (!haveSameType(m.keys.map(_.dataType)) || !haveSameType(m.values.map(_.dataType))) =>
         val keyTypes = m.keys.map(_.dataType)
         val newKeys = findWiderCommonType(keyTypes) match {
@@ -606,7 +606,7 @@ object TypeCoercion {
           case None => m.values
         }
 
-        CreateMap(newKeys.zip(newValues).flatMap { case (k, v) => Seq(k, v) })
+        m.copy(newKeys.zip(newValues).flatMap { case (k, v) => Seq(k, v) })
 
       // Promote SUM, SUM DISTINCT and AVERAGE to largest types to prevent overflows.
       case s @ Sum(e @ DecimalType()) => s // Decimal is already the biggest.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 4bd85d304ded2..6c31511571387 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -37,16 +37,23 @@ import org.apache.spark.unsafe.types.UTF8String
       > SELECT _FUNC_(1, 2, 3);
        [1,2,3]
   """)
-case class CreateArray(children: Seq[Expression]) extends Expression {
+case class CreateArray(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
+  extends Expression {
+
+  def this(children: Seq[Expression]) = {
+    this(children, SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE))
+  }
 
   override def foldable: Boolean = children.forall(_.foldable)
 
+  override def stringArgs: Iterator[Any] = super.stringArgs.take(1)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), s"function $prettyName")
   }
 
   private val defaultElementType: DataType = {
-    if (SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE)) {
+    if (useStringTypeWhenEmpty) {
       StringType
     } else {
       NullType
@@ -79,6 +86,12 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
   override def prettyName: String = "array"
 }
 
+object CreateArray {
+  def apply(children: Seq[Expression]): CreateArray = {
+    new CreateArray(children)
+  }
+}
+
 private [sql] object GenArrayData {
   /**
    * Return Java code pieces based on DataType and array size to allocate ArrayData class
@@ -141,12 +154,18 @@ private [sql] object GenArrayData {
       > SELECT _FUNC_(1.0, '2', 3.0, '4');
        {1.0:"2",3.0:"4"}
   """)
-case class CreateMap(children: Seq[Expression]) extends Expression {
+case class CreateMap(children: Seq[Expression], useStringTypeWhenEmpty: Boolean)
+  extends Expression {
+
+  def this(children: Seq[Expression]) = {
+    this(children, SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE))
+  }
+
   lazy val keys = children.indices.filter(_ % 2 == 0).map(children)
   lazy val values = children.indices.filter(_ % 2 != 0).map(children)
 
   private val defaultElementType: DataType = {
-    if (SQLConf.get.getConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE)) {
+    if (useStringTypeWhenEmpty) {
       StringType
     } else {
       NullType
@@ -155,6 +174,8 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
 
   override def foldable: Boolean = children.forall(_.foldable)
 
+  override def stringArgs: Iterator[Any] = super.stringArgs.take(1)
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.size % 2 != 0) {
       TypeCheckResult.TypeCheckFailure(
@@ -215,6 +236,12 @@ case class CreateMap(children: Seq[Expression]) extends Expression {
   override def prettyName: String = "map"
 }
 
+object CreateMap {
+  def apply(children: Seq[Expression]): CreateMap = {
+    new CreateMap(children)
+  }
+}
+
 /**
  * Returns a catalyst Map containing the two arrays in children expressions as keys and values.
  */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index e9d60ed3a481f..9c600c9d39cf7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -275,9 +275,9 @@ trait GetArrayItemUtil {
     if (ordinal.foldable && !ordinal.nullable) {
       val intOrdinal = ordinal.eval().asInstanceOf[Number].intValue()
       child match {
-        case CreateArray(ar) if intOrdinal < ar.length =>
+        case CreateArray(ar, _) if intOrdinal < ar.length =>
           ar(intOrdinal).nullable
-        case GetArrayStructFields(CreateArray(elements), field, _, _, _)
+        case GetArrayStructFields(CreateArray(elements, _), field, _, _, _)
           if intOrdinal < elements.length =>
           elements(intOrdinal).nullable || field.nullable
         case _ =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
index 28dc8e9d0d5f3..f79dabf758c14 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
@@ -41,14 +41,14 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
         createNamedStruct.valExprs(ordinal)
 
       // Remove redundant array indexing.
-      case GetArrayStructFields(CreateArray(elems), field, ordinal, _, _) =>
+      case GetArrayStructFields(CreateArray(elems, useStringTypeWhenEmpty), field, ordinal, _, _) =>
         // Instead of selecting the field on the entire array, select it from each member
         // of the array. Pushing down the operation this way may open other optimizations
         // opportunities (i.e. struct(...,x,...).x)
-        CreateArray(elems.map(GetStructField(_, ordinal, Some(field.name))))
+        CreateArray(elems.map(GetStructField(_, ordinal, Some(field.name))), useStringTypeWhenEmpty)
 
       // Remove redundant map lookup.
-      case ga @ GetArrayItem(CreateArray(elems), IntegerLiteral(idx)) =>
+      case ga @ GetArrayItem(CreateArray(elems, _), IntegerLiteral(idx)) =>
         // Instead of creating the array and then selecting one row, remove array creation
         // altogether.
         if (idx >= 0 && idx < elems.size) {
@@ -58,7 +58,7 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
           // out of bounds, mimic the runtime behavior and return null
           Literal(null, ga.dataType)
         }
-      case GetMapValue(CreateMap(elems), key) => CaseKeyWhen(key, elems)
+      case GetMapValue(CreateMap(elems, _), key) => CaseKeyWhen(key, elems)
     }
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index ea01d9e63eef7..5f94af5ffe636 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -114,11 +114,11 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
     case CreateNamedStruct(children) =>
       CreateNamedStruct(children.map(normalize))
 
-    case CreateArray(children) =>
-      CreateArray(children.map(normalize))
+    case CreateArray(children, useStringTypeWhenEmpty) =>
+      CreateArray(children.map(normalize), useStringTypeWhenEmpty)
 
-    case CreateMap(children) =>
-      CreateMap(children.map(normalize))
+    case CreateMap(children, useStringTypeWhenEmpty) =>
+      CreateMap(children.map(normalize), useStringTypeWhenEmpty)
 
     case _ if expr.dataType == FloatType || expr.dataType == DoubleType =>
       KnownFloatingPointNormalized(NormalizeNaNAndZero(expr))

From fc45b274df2096e0d6340974feafb17f1b2f2cc4 Mon Sep 17 00:00:00 2001
From: DB Tsai <d_tsai@apple.com>
Date: Fri, 6 Mar 2020 21:09:24 +0000
Subject: [PATCH 0242/1280] [SPARK-31064][SQL] New Parquet Predicate Filter
 APIs with multi-part Identifier Support

### What changes were proposed in this pull request?
Parquet's org.apache.parquet.filter2.predicate.FilterApi uses `dots` as separators to split the column name into multi-parts of nested fields. The drawback is this causes issues when the field name contains `dots`.

The new APIs that will be added will take array of string directly for multi-parts of nested fields, so no confusion as using `dots` as separators.

### Why are the changes needed?
To support nested predicate pushdown and predicate pushdown for columns containing `dots`.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UTs.

Closes #27824 from dbtsai/SPARK-31064.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit 7911f952029682e12fd1f90665f2869e1461c365)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 .../filter2/predicate/SparkFilterApi.java     |  57 ++++++
 .../datasources/parquet/ParquetFilters.scala  | 192 +++++++++---------
 2 files changed, 156 insertions(+), 93 deletions(-)
 create mode 100644 sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java

diff --git a/sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java b/sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java
new file mode 100644
index 0000000000000..884042c824046
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/parquet/filter2/predicate/SparkFilterApi.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet.filter2.predicate;
+
+import org.apache.parquet.hadoop.metadata.ColumnPath;
+import org.apache.parquet.filter2.predicate.Operators.BinaryColumn;
+import org.apache.parquet.filter2.predicate.Operators.BooleanColumn;
+import org.apache.parquet.filter2.predicate.Operators.DoubleColumn;
+import org.apache.parquet.filter2.predicate.Operators.FloatColumn;
+import org.apache.parquet.filter2.predicate.Operators.IntColumn;
+import org.apache.parquet.filter2.predicate.Operators.LongColumn;
+
+/**
+ * TODO (PARQUET-1809): This is a temporary workaround; it is intended to be moved to Parquet.
+ */
+public final class SparkFilterApi {
+  public static IntColumn intColumn(String[] path) {
+    return new IntColumn(ColumnPath.get(path));
+  }
+
+  public static LongColumn longColumn(String[] path) {
+    return new LongColumn(ColumnPath.get(path));
+  }
+
+  public static FloatColumn floatColumn(String[] path) {
+    return new FloatColumn(ColumnPath.get(path));
+  }
+
+  public static DoubleColumn doubleColumn(String[] path) {
+    return new DoubleColumn(ColumnPath.get(path));
+  }
+
+  public static BooleanColumn booleanColumn(String[] path) {
+    return new BooleanColumn(ColumnPath.get(path));
+  }
+
+  public static BinaryColumn binaryColumn(String[] path) {
+    return new BinaryColumn(ColumnPath.get(path));
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index 948a120e0d6e0..07065018343cf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -25,7 +25,7 @@ import java.util.Locale
 import scala.collection.JavaConverters.asScalaBufferConverter
 
 import org.apache.parquet.filter2.predicate._
-import org.apache.parquet.filter2.predicate.FilterApi._
+import org.apache.parquet.filter2.predicate.SparkFilterApi._
 import org.apache.parquet.io.api.Binary
 import org.apache.parquet.schema.{DecimalMetadata, MessageType, OriginalType, PrimitiveComparator}
 import org.apache.parquet.schema.OriginalType._
@@ -126,264 +126,270 @@ class ParquetFilters(
     Binary.fromConstantByteArray(fixedLengthBytes, 0, numBytes)
   }
 
-  private val makeEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
-      (n: String, v: Any) => FilterApi.eq(booleanColumn(n), v.asInstanceOf[JBoolean])
+      (n: Array[String], v: Any) => FilterApi.eq(booleanColumn(n), v.asInstanceOf[JBoolean])
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
         Option(v).map(_.asInstanceOf[Number].intValue.asInstanceOf[Integer]).orNull)
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.eq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.eq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.eq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.eq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.eq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.eq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     // Binary.fromString and Binary.fromByteArray don't accept null values
     case ParquetStringType =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         binaryColumn(n),
         Option(v).map(s => Binary.fromString(s.asInstanceOf[String])).orNull)
     case ParquetBinaryType =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         binaryColumn(n),
         Option(v).map(b => Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]])).orNull)
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
         Option(v).map(date => dateToDays(date.asInstanceOf[Date]).asInstanceOf[Integer]).orNull)
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
         Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
           .asInstanceOf[JLong]).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
         Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
         Option(v).map(d => decimalToInt32(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
         Option(v).map(d => decimalToInt64(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.eq(
+      (n: Array[String], v: Any) => FilterApi.eq(
         binaryColumn(n),
         Option(v).map(d => decimalToByteArray(d.asInstanceOf[JBigDecimal], length)).orNull)
   }
 
-  private val makeNotEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeNotEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
-      (n: String, v: Any) => FilterApi.notEq(booleanColumn(n), v.asInstanceOf[JBoolean])
+      (n: Array[String], v: Any) => FilterApi.notEq(booleanColumn(n), v.asInstanceOf[JBoolean])
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
         Option(v).map(_.asInstanceOf[Number].intValue.asInstanceOf[Integer]).orNull)
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.notEq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.notEq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.notEq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.notEq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.notEq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.notEq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         binaryColumn(n),
         Option(v).map(s => Binary.fromString(s.asInstanceOf[String])).orNull)
     case ParquetBinaryType =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         binaryColumn(n),
         Option(v).map(b => Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]])).orNull)
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
         Option(v).map(date => dateToDays(date.asInstanceOf[Date]).asInstanceOf[Integer]).orNull)
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
         Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
           .asInstanceOf[JLong]).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
         Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
         Option(v).map(d => decimalToInt32(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
         Option(v).map(d => decimalToInt64(d.asInstanceOf[JBigDecimal])).orNull)
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) => FilterApi.notEq(
+      (n: Array[String], v: Any) => FilterApi.notEq(
         binaryColumn(n),
         Option(v).map(d => decimalToByteArray(d.asInstanceOf[JBigDecimal], length)).orNull)
   }
 
-  private val makeLt: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeLt:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.lt(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.lt(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.lt(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.lt(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.lt(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.lt(
+      (n: Array[String], v: Any) => FilterApi.lt(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.lt(
+      (n: Array[String], v: Any) => FilterApi.lt(
         longColumn(n),
         v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.lt(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
-  private val makeLtEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeLtEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.ltEq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.ltEq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.ltEq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.ltEq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.ltEq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.ltEq(
+      (n: Array[String], v: Any) => FilterApi.ltEq(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.ltEq(
+      (n: Array[String], v: Any) => FilterApi.ltEq(
         longColumn(n),
         v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.ltEq(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
-  private val makeGt: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeGt:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.gt(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.gt(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.gt(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.gt(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.gt(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gt(
+      (n: Array[String], v: Any) => FilterApi.gt(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gt(
+      (n: Array[String], v: Any) => FilterApi.gt(
         longColumn(n),
         v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gt(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
-  private val makeGtEq: PartialFunction[ParquetSchemaType, (String, Any) => FilterPredicate] = {
+  private val makeGtEq:
+    PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetByteType | ParquetShortType | ParquetIntegerType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), v.asInstanceOf[Number].intValue.asInstanceOf[Integer])
     case ParquetLongType =>
-      (n: String, v: Any) => FilterApi.gtEq(longColumn(n), v.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), v.asInstanceOf[JLong])
     case ParquetFloatType =>
-      (n: String, v: Any) => FilterApi.gtEq(floatColumn(n), v.asInstanceOf[JFloat])
+      (n: Array[String], v: Any) => FilterApi.gtEq(floatColumn(n), v.asInstanceOf[JFloat])
     case ParquetDoubleType =>
-      (n: String, v: Any) => FilterApi.gtEq(doubleColumn(n), v.asInstanceOf[JDouble])
+      (n: Array[String], v: Any) => FilterApi.gtEq(doubleColumn(n), v.asInstanceOf[JDouble])
 
     case ParquetStringType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(binaryColumn(n), Binary.fromString(v.asInstanceOf[String]))
     case ParquetBinaryType =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gtEq(
+      (n: Array[String], v: Any) => FilterApi.gtEq(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: String, v: Any) => FilterApi.gtEq(
+      (n: Array[String], v: Any) => FilterApi.gtEq(
         longColumn(n),
         v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), decimalToInt32(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, INT64, _, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(longColumn(n), decimalToInt64(v.asInstanceOf[JBigDecimal]))
     case ParquetSchemaType(DECIMAL, FIXED_LEN_BYTE_ARRAY, length, _) if pushDownDecimal =>
-      (n: String, v: Any) =>
+      (n: Array[String], v: Any) =>
         FilterApi.gtEq(binaryColumn(n), decimalToByteArray(v.asInstanceOf[JBigDecimal], length))
   }
 
@@ -503,38 +509,38 @@ class ParquetFilters(
     predicate match {
       case sources.IsNull(name) if canMakeFilterOn(name, null) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, null))
+          .map(_(Array(nameToParquetField(name).fieldName), null))
       case sources.IsNotNull(name) if canMakeFilterOn(name, null) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, null))
+          .map(_(Array(nameToParquetField(name).fieldName), null))
 
       case sources.EqualTo(name, value) if canMakeFilterOn(name, value) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
       case sources.Not(sources.EqualTo(name, value)) if canMakeFilterOn(name, value) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
 
       case sources.EqualNullSafe(name, value) if canMakeFilterOn(name, value) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
       case sources.Not(sources.EqualNullSafe(name, value)) if canMakeFilterOn(name, value) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
 
       case sources.LessThan(name, value) if canMakeFilterOn(name, value) =>
         makeLt.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
       case sources.LessThanOrEqual(name, value) if canMakeFilterOn(name, value) =>
         makeLtEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
 
       case sources.GreaterThan(name, value) if canMakeFilterOn(name, value) =>
         makeGt.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
       case sources.GreaterThanOrEqual(name, value) if canMakeFilterOn(name, value) =>
         makeGtEq.lift(nameToParquetField(name).fieldType)
-          .map(_(nameToParquetField(name).fieldName, value))
+          .map(_(Array(nameToParquetField(name).fieldName), value))
 
       case sources.And(lhs, rhs) =>
         // At here, it is not safe to just convert one side and remove the other side
@@ -585,13 +591,13 @@ class ParquetFilters(
         && values.distinct.length <= pushDownInFilterThreshold =>
         values.distinct.flatMap { v =>
           makeEq.lift(nameToParquetField(name).fieldType)
-            .map(_(nameToParquetField(name).fieldName, v))
+            .map(_(Array(nameToParquetField(name).fieldName), v))
         }.reduceLeftOption(FilterApi.or)
 
       case sources.StringStartsWith(name, prefix)
           if pushDownStartWith && canMakeFilterOn(name, prefix) =>
         Option(prefix).map { v =>
-          FilterApi.userDefined(binaryColumn(nameToParquetField(name).fieldName),
+          FilterApi.userDefined(binaryColumn(Array(nameToParquetField(name).fieldName)),
             new UserDefinedPredicate[Binary] with Serializable {
               private val strToBinary = Binary.fromReusedByteArray(v.getBytes)
               private val size = strToBinary.length

From d73ea9791bff2c383b10d731cf651b2ebe159dde Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 7 Mar 2020 11:42:05 -0600
Subject: [PATCH 0243/1280] [SPARK-31012][ML][PYSPARK][DOCS] Updating ML API
 docs for 3.0 changes

### What changes were proposed in this pull request?
Updating ML docs for 3.0 changes

### Why are the changes needed?
I am auditing 3.0 ML changes, found some docs are missing or not updated. Need to update these.

### Does this PR introduce any user-facing change?
Yes, doc changes

### How was this patch tested?
Manually build and check

Closes #27762 from huaxingao/spark-doc.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 4a64901ab7176ad3d41c4598dea1e73cd92e969e)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/ml-features.md                           |  2 +-
 .../BinaryClassificationEvaluator.scala       |  3 ++-
 .../MulticlassClassificationEvaluator.scala   | 21 +++++++++++++++++++
 .../ml/evaluation/RankingEvaluator.scala      |  5 +++++
 .../ml/evaluation/RegressionEvaluator.scala   |  3 ++-
 python/pyspark/ml/evaluation.py               |  9 ++++----
 python/pyspark/ml/feature.py                  |  2 +-
 7 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/docs/ml-features.md b/docs/ml-features.md
index 9c05fd5fa1ce2..05ef848aefef8 100644
--- a/docs/ml-features.md
+++ b/docs/ml-features.md
@@ -1499,7 +1499,7 @@ for more details on the API.
 
 The `Imputer` estimator completes missing values in a dataset, either using the mean or the 
 median of the columns in which the missing values are located. The input columns should be of
-`DoubleType` or `FloatType`. Currently `Imputer` does not support categorical features and possibly
+numeric type. Currently `Imputer` does not support categorical features and possibly
 creates incorrect values for columns containing categorical features. Imputer can impute custom values 
 other than 'NaN' by `.setMissingValue(custom_value)`. For example, `.setMissingValue(0)` will impute 
 all occurrences of (0).
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
index 55b910e98d405..7733225c0c4d5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
@@ -28,7 +28,8 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.DoubleType
 
 /**
- * Evaluator for binary classification, which expects two input columns: rawPrediction and label.
+ * Evaluator for binary classification, which expects input columns rawPrediction, label and
+ *  an optional weight column.
  * The rawPrediction column can be of type double (binary 0/1 prediction, or probability of label 1)
  * or of type vector (length-2 vector of raw predictions, scores, or label probabilities).
  */
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
index 435708186242f..1d6540e970383 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
@@ -81,6 +81,14 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setProbabilityCol(value: String): this.type = set(probabilityCol, value)
 
+  /**
+   * The class whose metric will be computed in `"truePositiveRateByLabel"`,
+   * `"falsePositiveRateByLabel"`, `"precisionByLabel"`, `"recallByLabel"`,
+   * `"fMeasureByLabel"`.
+   * Must be greater than or equal to 0. The default value is 0.
+   *
+   * @group param
+   */
   @Since("3.0.0")
   final val metricLabel: DoubleParam = new DoubleParam(this, "metricLabel",
     "The class whose metric will be computed in " +
@@ -98,6 +106,13 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
 
   setDefault(metricLabel -> 0.0)
 
+  /**
+   * The beta value, which controls precision vs recall weighting,
+   * used in `"weightedFMeasure"`, `"fMeasureByLabel"`.
+   * Must be greater than 0. The default value is 1.
+   *
+   * @group param
+   */
   @Since("3.0.0")
   final val beta: DoubleParam = new DoubleParam(this, "beta",
     "The beta value, which controls precision vs recall weighting, " +
@@ -114,6 +129,12 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
 
   setDefault(beta -> 1.0)
 
+  /**
+   * param for eps. log-loss is undefined for p=0 or p=1, so probabilities are clipped to
+   * max(eps, min(1 - eps, p)). Must be in range (0, 0.5). The default value is 1e-15.
+   *
+   * @group param
+   */
   @Since("3.0.0")
   final val eps: DoubleParam = new DoubleParam(this, "eps",
     "log-loss is undefined for p=0 or p=1, so probabilities are clipped to " +
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
index ca3a8ebc1659d..8d017eb181f15 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
@@ -59,6 +59,11 @@ class RankingEvaluator (override val uid: String)
 
   setDefault(metricName -> "meanAveragePrecision")
 
+  /**
+   * param for ranking position value used in `"meanAveragePrecisionAtK"`, `"precisionAtK"`,
+   * `"ndcgAtK"`, `"recallAtK"`. Must be &gt; 0. The default value is 10.
+   * @group param
+   */
   final val k = new IntParam(this, "k",
     "The ranking position value used in " +
       s"${supportedMetricNames.filter(_.endsWith("AtK")).mkString("(", "|", ")")}  " +
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
index 9f32d40d166bd..18a8dda0c76ef 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
@@ -27,7 +27,8 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, FloatType}
 
 /**
- * Evaluator for regression, which expects two input columns: prediction and label.
+ * Evaluator for regression, which expects input columns prediction, label and
+ * an optional weight column.
  */
 @Since("1.4.0")
 final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val uid: String)
diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py
index 556a2f85c708d..265f02c1a03ac 100644
--- a/python/pyspark/ml/evaluation.py
+++ b/python/pyspark/ml/evaluation.py
@@ -110,7 +110,8 @@ def isLargerBetter(self):
 class BinaryClassificationEvaluator(JavaEvaluator, HasLabelCol, HasRawPredictionCol, HasWeightCol,
                                     JavaMLReadable, JavaMLWritable):
     """
-    Evaluator for binary classification, which expects two input columns: rawPrediction and label.
+    Evaluator for binary classification, which expects input columns rawPrediction, label
+    and an optional weight column.
     The rawPrediction column can be of type double (binary 0/1 prediction, or probability of label
     1) or of type vector (length-2 vector of raw predictions, scores, or label probabilities).
 
@@ -409,9 +410,9 @@ class MulticlassClassificationEvaluator(JavaEvaluator, HasLabelCol, HasPredictio
     """
     metricName = Param(Params._dummy(), "metricName",
                        "metric name in evaluation "
-                       "(f1|accuracy|weightedPrecision|weightedRecall|weightedTruePositiveRate|"
-                       "weightedFalsePositiveRate|weightedFMeasure|truePositiveRateByLabel|"
-                       "falsePositiveRateByLabel|precisionByLabel|recallByLabel|fMeasureByLabel|"
+                       "(f1|accuracy|weightedPrecision|weightedRecall|weightedTruePositiveRate| "
+                       "weightedFalsePositiveRate|weightedFMeasure|truePositiveRateByLabel| "
+                       "falsePositiveRateByLabel|precisionByLabel|recallByLabel|fMeasureByLabel| "
                        "logLoss|hammingLoss)",
                        typeConverter=TypeConverters.toString)
     metricLabel = Param(Params._dummy(), "metricLabel",
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 4c25bb495fdfb..6df2f74bcfc9d 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -1467,7 +1467,7 @@ class Imputer(JavaEstimator, _ImputerParams, JavaMLReadable, JavaMLWritable):
     """
     Imputation estimator for completing missing values, either using the mean or the median
     of the columns in which the missing values are located. The input columns should be of
-    DoubleType or FloatType. Currently Imputer does not support categorical features and
+    numeric type. Currently Imputer does not support categorical features and
     possibly creates incorrect values for a categorical feature.
 
     Note that the mean/median value is computed after filtering out missing values.

From 895ddde9c4ee9fcd8200dbbfe9b442f17c4c3b48 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Sat, 7 Mar 2020 12:02:43 -0800
Subject: [PATCH 0244/1280] [SPARK-31014][CORE][3.0] InMemoryStore: remove key
 from parentToChildrenMap when removing key from CountingRemoveIfForEach

### What changes were proposed in this pull request?

This patch addresses missed spot on SPARK-30964 (#27716) - SPARK-30964 added secondary index which defines the relationship between parent - children and able to operate all children for given parent faster.

While SPARK-30964 handled the addition and deletion of secondary index in InstanceList properly, it missed to add code to handle deletion of secondary index in CountingRemoveIfForEach, resulting to the leak of indices. This patch adds the deletion of secondary index in CountingRemoveIfForEach.

### Why are the changes needed?

Described above.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A, as relevant field and class are marked as private, and it cannot be checked in higher level. I'm not sure we want to adjust scope to add a test.

Closes #27825 from HeartSaVioR/SPARK-31014-branch-3.0.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/util/kvstore/InMemoryStore.java     | 30 +++++++++++++------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index f1bebb4768ead..e929c6c45ebd3 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -177,7 +177,7 @@ private static class InstanceList<T> {
      * iterators.  https://bugs.openjdk.java.net/browse/JDK-8078645
      */
     private static class CountingRemoveIfForEach<T> implements BiConsumer<Comparable<Object>, T> {
-      private final ConcurrentMap<Comparable<Object>, T> data;
+      private final InstanceList<T> instanceList;
       private final Predicate<? super T> filter;
 
       /**
@@ -189,17 +189,15 @@ private static class CountingRemoveIfForEach<T> implements BiConsumer<Comparable
        */
       private int count = 0;
 
-      CountingRemoveIfForEach(
-          ConcurrentMap<Comparable<Object>, T> data,
-          Predicate<? super T> filter) {
-        this.data = data;
+      CountingRemoveIfForEach(InstanceList<T> instanceList, Predicate<? super T> filter) {
+        this.instanceList = instanceList;
         this.filter = filter;
       }
 
       @Override
       public void accept(Comparable<Object> key, T value) {
         if (filter.test(value)) {
-          if (data.remove(key, value)) {
+          if (instanceList.delete(key, value)) {
             count++;
           }
         }
@@ -253,7 +251,7 @@ int countingRemoveAllByIndexValues(String index, Collection<?> indexValues) {
         return count;
       } else {
         Predicate<? super T> filter = getPredicate(ti.getAccessor(index), indexValues);
-        CountingRemoveIfForEach<T> callback = new CountingRemoveIfForEach<>(data, filter);
+        CountingRemoveIfForEach<T> callback = new CountingRemoveIfForEach<>(this, filter);
 
         // Go through all the values in `data` and delete objects that meets the predicate `filter`.
         // This can be slow when there is a large number of entries in `data`.
@@ -278,7 +276,22 @@ public void put(T value) throws Exception {
 
     public boolean delete(Object key) {
       boolean entryExists = data.remove(asKey(key)) != null;
-      if (entryExists && hasNaturalParentIndex) {
+      if (entryExists) {
+        deleteParentIndex(key);
+      }
+      return entryExists;
+    }
+
+    public boolean delete(Object key, T value) {
+      boolean entryExists = data.remove(asKey(key), value);
+      if (entryExists) {
+        deleteParentIndex(key);
+      }
+      return entryExists;
+    }
+
+    private void deleteParentIndex(Object key) {
+      if (hasNaturalParentIndex) {
         for (NaturalKeys v : parentToChildrenMap.values()) {
           if (v.remove(asKey(key))) {
             // `v` can be empty after removing the natural key and we can remove it from
@@ -289,7 +302,6 @@ public boolean delete(Object key) {
           }
         }
       }
-      return entryExists;
     }
 
     public int size() {

From 60accbf1b7b6c0a189620aa174bb35908b94700f Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 7 Mar 2020 18:09:00 -0600
Subject: [PATCH 0245/1280] [SPARK-30934][ML][DOCS] Update ml-guide and
 ml-migration-guide for 3.0 release

### What changes were proposed in this pull request?
Update ml-guide and ml-migration-guide for 3.0.

### Why are the changes needed?
This is required for each release.

### Does this PR introduce any user-facing change?
Yes.
![image](https://user-images.githubusercontent.com/13592258/75957386-c8699e80-5e6e-11ea-9dec-7295f8f0bf33.png)

![image](https://user-images.githubusercontent.com/13592258/75957406-cef81600-5e6e-11ea-921f-20509771b49b.png)

![image](https://user-images.githubusercontent.com/13592258/75957423-d4edf700-5e6e-11ea-8e75-d41c532c8ba9.png)

![image](https://user-images.githubusercontent.com/13592258/75957434-da4b4180-5e6e-11ea-899b-f4e080b318ff.png)

### How was this patch tested?
Manually build and check.

Closes #27785 from huaxingao/spark-30934.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 513f76ac38f174ca1adf6faff087b95f4d8f9750)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/ml-guide.md           | 46 ++++++++++++++++------------------
 docs/ml-migration-guide.md | 51 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 72 insertions(+), 25 deletions(-)

diff --git a/docs/ml-guide.md b/docs/ml-guide.md
index 7b4fa4f651e64..2037285f3f12c 100644
--- a/docs/ml-guide.md
+++ b/docs/ml-guide.md
@@ -41,8 +41,6 @@ The primary Machine Learning API for Spark is now the [DataFrame](sql-programmin
 * MLlib will still support the RDD-based API in `spark.mllib` with bug fixes.
 * MLlib will not add new features to the RDD-based API.
 * In the Spark 2.x releases, MLlib will add features to the DataFrames-based API to reach feature parity with the RDD-based API.
-* After reaching feature parity (roughly estimated for Spark 2.3), the RDD-based API will be deprecated.
-* The RDD-based API is expected to be removed in Spark 3.0.
 
 *Why is MLlib switching to the DataFrame-based API?*
 
@@ -87,31 +85,31 @@ To use MLlib in Python, you will need [NumPy](http://www.numpy.org) version 1.4
 [^1]: To learn more about the benefits and background of system optimised natives, you may wish to
     watch Sam Halliday's ScalaX talk on [High Performance Linear Algebra in Scala](http://fommil.github.io/scalax14/#/).
 
-# Highlights in 2.3
+# Highlights in 3.0
 
-The list below highlights some of the new features and enhancements added to MLlib in the `2.3`
+The list below highlights some of the new features and enhancements added to MLlib in the `3.0`
 release of Spark:
 
-* Built-in support for reading images into a `DataFrame` was added
-([SPARK-21866](https://issues.apache.org/jira/browse/SPARK-21866)).
-* [`OneHotEncoderEstimator`](ml-features.html#onehotencoderestimator) was added, and should be
-used instead of the existing `OneHotEncoder` transformer. The new estimator supports
-transforming multiple columns.
-* Multiple column support was also added to `QuantileDiscretizer` and `Bucketizer`
-([SPARK-22397](https://issues.apache.org/jira/browse/SPARK-22397) and
-[SPARK-20542](https://issues.apache.org/jira/browse/SPARK-20542))
-* A new [`FeatureHasher`](ml-features.html#featurehasher) transformer was added
- ([SPARK-13969](https://issues.apache.org/jira/browse/SPARK-13969)).
-* Added support for evaluating multiple models in parallel when performing cross-validation using
-[`TrainValidationSplit` or `CrossValidator`](ml-tuning.html)
-([SPARK-19357](https://issues.apache.org/jira/browse/SPARK-19357)).
-* Improved support for custom pipeline components in Python (see
-[SPARK-21633](https://issues.apache.org/jira/browse/SPARK-21633) and 
-[SPARK-21542](https://issues.apache.org/jira/browse/SPARK-21542)).
-* `DataFrame` functions for descriptive summary statistics over vector columns
-([SPARK-19634](https://issues.apache.org/jira/browse/SPARK-19634)).
-* Robust linear regression with Huber loss
-([SPARK-3181](https://issues.apache.org/jira/browse/SPARK-3181)).
+* Multiple columns support was added to `Binarizer` ([SPARK-23578](https://issues.apache.org/jira/browse/SPARK-23578)), `StringIndexer` ([SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215)), `StopWordsRemover` ([SPARK-29808](https://issues.apache.org/jira/browse/SPARK-29808)) and PySpark `QuantileDiscretizer` ([SPARK-22796](https://issues.apache.org/jira/browse/SPARK-22796)).
+* Support Tree-Based Feature Transformation was added
+([SPARK-13677](https://issues.apache.org/jira/browse/SPARK-13677)).
+* Two new evaluators `MultilabelClassificationEvaluator` ([SPARK-16692](https://issues.apache.org/jira/browse/SPARK-16692)) and `RankingEvaluator` ([SPARK-28045](https://issues.apache.org/jira/browse/SPARK-28045)) were added.
+* Sample weights support was added in `DecisionTreeClassifier/Regressor` ([SPARK-19591](https://issues.apache.org/jira/browse/SPARK-19591)), `RandomForestClassifier/Regressor` ([SPARK-9478](https://issues.apache.org/jira/browse/SPARK-9478)), `GBTClassifier/Regressor` ([SPARK-9612](https://issues.apache.org/jira/browse/SPARK-9612)),  `RegressionEvaluator` ([SPARK-24102](https://issues.apache.org/jira/browse/SPARK-24102)), `BinaryClassificationEvaluator` ([SPARK-24103](https://issues.apache.org/jira/browse/SPARK-24103)), `BisectingKMeans` ([SPARK-30351](https://issues.apache.org/jira/browse/SPARK-30351)), `KMeans` ([SPARK-29967](https://issues.apache.org/jira/browse/SPARK-29967)) and `GaussianMixture` ([SPARK-30102](https://issues.apache.org/jira/browse/SPARK-30102)).
+* R API for `PowerIterationClustering` was added
+([SPARK-19827](https://issues.apache.org/jira/browse/SPARK-19827)).
+* Added Spark ML listener for tracking ML pipeline status
+([SPARK-23674](https://issues.apache.org/jira/browse/SPARK-23674)).
+* Fit with validation set was added to Gradient Boosted Trees in Python
+([SPARK-24333](https://issues.apache.org/jira/browse/SPARK-24333)).
+* [`RobustScaler`](ml-features.html#robustscaler) transformer was added
+([SPARK-28399](https://issues.apache.org/jira/browse/SPARK-28399)).
+* [`Factorization Machines`](ml-classification-regression.html#factorization-machines) classifier and regressor were added
+([SPARK-29224](https://issues.apache.org/jira/browse/SPARK-29224)).
+* Gaussian Naive Bayes ([SPARK-16872](https://issues.apache.org/jira/browse/SPARK-16872)) and Complement Naive Bayes ([SPARK-29942](https://issues.apache.org/jira/browse/SPARK-29942)) were added.
+* ML function parity between Scala and Python
+([SPARK-28958](https://issues.apache.org/jira/browse/SPARK-28958)).
+* `predictRaw` is made public in all the Classification models. `predictProbability` is made public in all the Classification models except `LinearSVCModel`.
+([SPARK-30358](https://issues.apache.org/jira/browse/SPARK-30358)).
 
 # Migration Guide
 
diff --git a/docs/ml-migration-guide.md b/docs/ml-migration-guide.md
index f3cd762b6f2ac..4e6d68f5a8cf4 100644
--- a/docs/ml-migration-guide.md
+++ b/docs/ml-migration-guide.md
@@ -33,16 +33,65 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
 * `OneHotEncoder` which is deprecated in 2.3, is removed in 3.0 and `OneHotEncoderEstimator` is now renamed to `OneHotEncoder`.
 * `org.apache.spark.ml.image.ImageSchema.readImages` which is deprecated in 2.3, is removed in 3.0, use `spark.read.format('image')` instead.
+* `org.apache.spark.mllib.clustering.KMeans.train` with param Int `runs` which is deprecated in 2.1, is removed in 3.0. Use `train` method without `runs` instead.
+* `org.apache.spark.mllib.classification.LogisticRegressionWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.classification.LogisticRegression` or `spark.mllib.classification.LogisticRegressionWithLBFGS` instead.
+* `org.apache.spark.mllib.feature.ChiSqSelectorModel.isSorted ` which is deprecated in 2.1, is removed in 3.0, is not intended for subclasses to use.
+* `org.apache.spark.mllib.regression.RidgeRegressionWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.regression.LinearRegression` with `elasticNetParam` = 0.0. Note the default `regParam` is 0.01 for `RidgeRegressionWithSGD`, but is 0.0 for `LinearRegression`.
+* `org.apache.spark.mllib.regression.LassoWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.regression.LinearRegression` with `elasticNetParam` = 1.0. Note the default `regParam` is 0.01 for `LassoWithSGD`, but is 0.0 for `LinearRegression`.
+* `org.apache.spark.mllib.regression.LinearRegressionWithSGD` which is deprecated in 2.0, is removed in 3.0, use `org.apache.spark.ml.regression.LinearRegression` or `LBFGS` instead.
+* `org.apache.spark.mllib.clustering.KMeans.getRuns` and `setRuns` which are deprecated in 2.1, are removed in 3.0, have no effect since Spark 2.0.0.
+* `org.apache.spark.ml.LinearSVCModel.setWeightCol` which is deprecated in 2.4, is removed in 3.0, is not intended for users.
+* From 3.0, `org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel` extends `MultilayerPerceptronParams` to expose the training params. As a result, `layers` in `MultilayerPerceptronClassificationModel` has been changed from `Array[Int]` to `IntArrayParam`. Users should use `MultilayerPerceptronClassificationModel.getLayers` instead of `MultilayerPerceptronClassificationModel.layers` to retrieve the size of layers.
+* `org.apache.spark.ml.classification.GBTClassifier.numTrees`  which is deprecated in 2.4.5, is removed in 3.0, use `getNumTrees` instead.
+* `org.apache.spark.ml.clustering.KMeansModel.computeCost` which is deprecated in 2.4, is removed in 3.0, use `ClusteringEvaluator` instead.
+* The member variable `precision` in `org.apache.spark.mllib.evaluation.MulticlassMetrics` which is deprecated in 2.0, is removed in 3.0. Use `accuracy` instead.
+* The member variable `recall` in `org.apache.spark.mllib.evaluation.MulticlassMetrics` which is deprecated in 2.0, is removed in 3.0. Use `accuracy` instead.
+* The member variable `fMeasure` in `org.apache.spark.mllib.evaluation.MulticlassMetrics` which is deprecated in 2.0, is removed in 3.0. Use `accuracy` instead.
+* `org.apache.spark.ml.util.GeneralMLWriter.context` which is deprecated in 2.0, is removed in 3.0, use `session` instead.
+* `org.apache.spark.ml.util.MLWriter.context` which is deprecated in 2.0, is removed in 3.0, use `session` instead.
+* `org.apache.spark.ml.util.MLReader.context` which is deprecated in 2.0, is removed in 3.0, use `session` instead.
+* `abstract class UnaryTransformer[IN, OUT, T <: UnaryTransformer[IN, OUT, T]]` is changed to `abstract class UnaryTransformer[IN: TypeTag, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]` in 3.0.
 
-### Changes of behavior
+### Deprecations and changes of behavior
 {:.no_toc}
 
+**Deprecations**
+
+* [SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215):
+`labels` in `StringIndexerModel` is deprecated and will be removed in 3.1.0. Use `labelsArray` instead.
+* [SPARK-25758](https://issues.apache.org/jira/browse/SPARK-25758):
+`computeCost` in `BisectingKMeansModel` is deprecated and will be removed in future versions. Use `ClusteringEvaluator` instead.
+
+**Changes of behavior**
+
 * [SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215):
  In Spark 2.4 and previous versions, when specifying `frequencyDesc` or `frequencyAsc` as
  `stringOrderType` param in `StringIndexer`, in case of equal frequency, the order of
  strings is undefined. Since Spark 3.0, the strings with equal frequency are further
  sorted by alphabet. And since Spark 3.0, `StringIndexer` supports encoding multiple
  columns.
+ * [SPARK-20604](https://issues.apache.org/jira/browse/SPARK-20604):
+ In prior to 3.0 releases, `Imputer` requires input column to be Double or Float. In 3.0, this
+ restriction is lifted so `Imputer` can handle all numeric types.
+* [SPARK-23469](https://issues.apache.org/jira/browse/SPARK-23469):
+In Spark 3.0, the `HashingTF` Transformer uses a corrected implementation of the murmur3 hash
+function to hash elements to vectors. `HashingTF` in Spark 3.0 will map elements to
+different positions in vectors than in Spark 2. However, `HashingTF` created with Spark 2.x
+and loaded with Spark 3.0 will still use the previous hash function and will not change behavior.
+* [SPARK-28969](https://issues.apache.org/jira/browse/SPARK-28969):
+The `setClassifier` method in PySpark's `OneVsRestModel` has been removed in 3.0 for parity with
+the Scala implementation. Callers should not need to set the classifier in the model after
+creation.
+* [SPARK-25790](https://issues.apache.org/jira/browse/SPARK-25790):
+ PCA adds the support for more than 65535 column matrix in Spark 3.0.
+* [SPARK-28927](https://issues.apache.org/jira/browse/SPARK-28927):
+ When fitting ALS model on nondeterministic input data, previously if rerun happens, users
+ would see ArrayIndexOutOfBoundsException caused by mismatch between In/Out user/item blocks.
+ From 3.0, a SparkException with more clear message will be thrown, and original
+ ArrayIndexOutOfBoundsException is wrapped.
+* [SPARK-29232](https://issues.apache.org/jira/browse/SPARK-29232):
+ In prior to 3.0 releases, `RandomForestRegressionModel` doesn't update the parameter maps
+ of the DecisionTreeRegressionModels underneath. This is fixed in 3.0.
 
 ## Upgrading from MLlib 2.2 to 2.3
 

From 4287b03a9c564eb2bdb4dfd93ea78728e3a9e440 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Sun, 8 Mar 2020 11:41:09 -0700
Subject: [PATCH 0246/1280] [SPARK-31053][SQL] mark connector APIs as Evolving

### What changes were proposed in this pull request?

The newly added catalog APIs are marked as Experimental but other DS v2 APIs are marked as Evolving.

This PR makes it consistent and mark all Connector APIs as Evolving.

### Why are the changes needed?

For consistency.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #27811 from cloud-fan/tag.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 1aa184763aa49d70907669b2d8af5a713ee0d7fa)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/sql/connector/catalog/CatalogExtension.java  | 4 ++--
 .../org/apache/spark/sql/connector/catalog/CatalogPlugin.java | 4 ++--
 .../sql/connector/catalog/DelegatingCatalogExtension.java     | 4 ++--
 .../org/apache/spark/sql/connector/catalog/Identifier.java    | 4 ++--
 .../apache/spark/sql/connector/catalog/IdentifierImpl.java    | 4 ++--
 .../apache/spark/sql/connector/catalog/NamespaceChange.java   | 4 ++--
 .../org/apache/spark/sql/connector/catalog/StagedTable.java   | 4 ++--
 .../spark/sql/connector/catalog/StagingTableCatalog.java      | 4 ++--
 .../apache/spark/sql/connector/catalog/SupportsDelete.java    | 4 ++--
 .../spark/sql/connector/catalog/SupportsNamespaces.java       | 4 ++--
 .../org/apache/spark/sql/connector/catalog/SupportsRead.java  | 4 ++--
 .../org/apache/spark/sql/connector/catalog/SupportsWrite.java | 4 ++--
 .../apache/spark/sql/connector/catalog/TableCapability.java   | 4 ++--
 .../org/apache/spark/sql/connector/catalog/TableCatalog.java  | 4 ++--
 .../org/apache/spark/sql/connector/catalog/TableChange.java   | 4 ++--
 .../apache/spark/sql/connector/expressions/Expression.java    | 4 ++--
 .../apache/spark/sql/connector/expressions/Expressions.java   | 4 ++--
 .../org/apache/spark/sql/connector/expressions/Literal.java   | 4 ++--
 .../spark/sql/connector/expressions/NamedReference.java       | 4 ++--
 .../org/apache/spark/sql/connector/expressions/Transform.java | 4 ++--
 .../spark/sql/connector/write/SupportsDynamicOverwrite.java   | 3 +++
 .../apache/spark/sql/connector/write/SupportsOverwrite.java   | 2 ++
 .../apache/spark/sql/connector/write/SupportsTruncate.java    | 3 +++
 23 files changed, 48 insertions(+), 40 deletions(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
index 61cb83c4f15ad..155dca5809c37 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogExtension.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
 /**
@@ -29,7 +29,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface CatalogExtension extends TableCatalog, SupportsNamespaces {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
index 2958538c21926..8ca4f568b9f18 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/CatalogPlugin.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.internal.SQLConf;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
 
@@ -41,7 +41,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface CatalogPlugin {
   /**
    * Called to initialize configuration.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
index 5a5195934413e..d07d299d65a58 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/DelegatingCatalogExtension.java
@@ -19,7 +19,7 @@
 
 import java.util.Map;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
@@ -36,7 +36,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public abstract class DelegatingCatalogExtension implements CatalogExtension {
 
   private CatalogPlugin delegate;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
index 7b3a081aecdc0..88c51d6c43ddf 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/Identifier.java
@@ -17,14 +17,14 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Identifies an object in a catalog.
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Identifier {
 
   static Identifier of(String[] namespace, String name) {
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
index 30596d92faaa2..b6c84cdeb0d3e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/IdentifierImpl.java
@@ -24,12 +24,12 @@
 
 import com.google.common.base.Preconditions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  *  An {@link Identifier} implementation.
  */
-@Experimental
+@Evolving
 class IdentifierImpl implements Identifier {
 
   private String[] namespace;
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
index f4eee84fee968..dae29c16199df 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/NamespaceChange.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * NamespaceChange subclasses represent requested changes to a namespace. These are passed to
@@ -33,7 +33,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface NamespaceChange {
   /**
    * Create a NamespaceChange for setting a namespace property.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
index c328f83f9a0c2..60b250adb41ef 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagedTable.java
@@ -19,7 +19,7 @@
 
 import java.util.Map;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.connector.write.LogicalWriteInfo;
 import org.apache.spark.sql.types.StructType;
@@ -39,7 +39,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface StagedTable extends Table {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
index 9d4f6eafd820e..35455a0ed9975 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/StagingTableCatalog.java
@@ -19,7 +19,7 @@
 
 import java.util.Map;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.connector.write.LogicalWriteInfo;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
@@ -51,7 +51,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface StagingTableCatalog extends TableCatalog {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
index 92461882fc75f..106f3283a62c8 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsDelete.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.sources.Filter;
 
 /**
@@ -26,7 +26,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsDelete {
   /**
    * Delete data from a data source table that matches filter expressions.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
index bd7e49ef6f08a..f70746b612e92 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.catalyst.analysis.NamespaceAlreadyExistsException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
 
@@ -38,7 +38,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsNamespaces extends CatalogPlugin {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
index a4cb66b2d11e2..e0ed9772dcedf 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsRead.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.read.Scan;
 import org.apache.spark.sql.connector.read.ScanBuilder;
 import org.apache.spark.sql.util.CaseInsensitiveStringMap;
@@ -29,7 +29,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsRead extends Table {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
index 19499d1061a9e..00aa180cf030b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsWrite.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.write.BatchWrite;
 import org.apache.spark.sql.connector.write.LogicalWriteInfo;
 import org.apache.spark.sql.connector.write.WriteBuilder;
@@ -29,7 +29,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface SupportsWrite extends Table {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
index 6a64bf8441933..68161d7225fcf 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCapability.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Capabilities that can be provided by a {@link Table} implementation.
@@ -28,7 +28,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public enum TableCapability {
   /**
    * Signals that the table supports reads in batch execution mode.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
index 357cd4b84f47a..1809b9cdb52e5 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.catalog;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.connector.expressions.Transform;
 import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
@@ -37,7 +37,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface TableCatalog extends CatalogPlugin {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
index 81e988a62a058..3ed185a82452b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableChange.java
@@ -21,7 +21,7 @@
 import java.util.Objects;
 import javax.annotation.Nullable;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.types.DataType;
 
 /**
@@ -39,7 +39,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface TableChange {
 
   /**
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
index a5b8e2224e711..6540c91597582 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java
@@ -17,14 +17,14 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Base class of the public logical expression API.
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Expression {
   /**
    * Format the expression as a human readable SQL-like string.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
index 28fb71e59c320..791dc969ab008 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expressions.java
@@ -19,9 +19,9 @@
 
 import java.util.Arrays;
 
+import org.apache.spark.annotation.Evolving;
 import scala.collection.JavaConverters;
 
-import org.apache.spark.annotation.Experimental;
 import org.apache.spark.sql.types.DataType;
 
 /**
@@ -29,7 +29,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public class Expressions {
   private Expressions() {
   }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
index 6474eb4ea4d01..df9e58fa319fd 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Literal.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.types.DataType;
 
 /**
@@ -29,7 +29,7 @@
  * @param <T> the JVM type of a value held by the literal
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Literal<T> extends Expression {
   /**
    * Returns the literal value.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
index 6dd46cc063145..167432fa0e86a 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/NamedReference.java
@@ -17,14 +17,14 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Represents a field or column reference in the public logical expression API.
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface NamedReference extends Expression {
   /**
    * Returns the referenced field name as an array of String parts.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
index ad78f005d9218..297205825c6a4 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Transform.java
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector.expressions;
 
-import org.apache.spark.annotation.Experimental;
+import org.apache.spark.annotation.Evolving;
 
 /**
  * Represents a transform function in the public logical expression API.
@@ -27,7 +27,7 @@
  *
  * @since 3.0.0
  */
-@Experimental
+@Evolving
 public interface Transform extends Expression {
   /**
    * Returns the transform function name.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
index bda0eed93c98a..422cd71d3453e 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsDynamicOverwrite.java
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.connector.write;
 
+import org.apache.spark.annotation.Evolving;
+
 /**
  * Write builder trait for tables that support dynamic partition overwrite.
  * <p>
@@ -29,6 +31,7 @@
  *
  * @since 3.0.0
  */
+@Evolving
 public interface SupportsDynamicOverwrite extends WriteBuilder {
   /**
    * Configures a write to dynamically replace partitions with data committed in the write.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
index ffa106c0b72f0..b4e60257942da 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsOverwrite.java
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.connector.write;
 
+import org.apache.spark.annotation.Evolving;
 import org.apache.spark.sql.sources.AlwaysTrue$;
 import org.apache.spark.sql.sources.Filter;
 
@@ -28,6 +29,7 @@
  *
  * @since 3.0.0
  */
+@Evolving
 public interface SupportsOverwrite extends WriteBuilder, SupportsTruncate {
   /**
    * Configures a write to replace data matching the filters with data committed in the write.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
index b89d409abea6f..0763f657d3b91 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/write/SupportsTruncate.java
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.connector.write;
 
+import org.apache.spark.annotation.Evolving;
+
 /**
  * Write builder trait for tables that support truncation.
  * <p>
@@ -24,6 +26,7 @@
  *
  * @since 3.0.0
  */
+@Evolving
 public interface SupportsTruncate extends WriteBuilder {
   /**
    * Configures a write to replace all existing data with data committed in the write.

From f6ee3d061409128c301e5f0d9ae9733132173cc8 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 9 Mar 2020 15:53:34 +0800
Subject: [PATCH 0247/1280] [SPARK-31082][CORE]
 MapOutputTrackerMaster.getMapLocation should handle last mapIndex correctly

### What changes were proposed in this pull request?

In `getMapLocation`, change the condition from `...endMapIndex < statuses.length` to `...endMapIndex <= statuses.length`.

### Why are the changes needed?

`endMapIndex` is exclusive, we should include it when comparing to `statuses.length`. Otherwise, we can't get the location for last mapIndex.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Updated existed test.

Closes #27850 from Ngone51/fix_getmaploction.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ef51ff9dc8c220fcbed76cdd1783f58f400df48c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../main/scala/org/apache/spark/MapOutputTracker.scala |  5 +++--
 .../execution/adaptive/AdaptiveQueryExecSuite.scala    | 10 +++++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
index f229061a6d0f6..ec8621bc55cf3 100644
--- a/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
+++ b/core/src/main/scala/org/apache/spark/MapOutputTracker.scala
@@ -696,7 +696,7 @@ private[spark] class MapOutputTrackerMaster(
    *
    * @param dep shuffle dependency object
    * @param startMapIndex the start map index
-   * @param endMapIndex the end map index
+   * @param endMapIndex the end map index (exclusive)
    * @return a sequence of locations where task runs.
    */
   def getMapLocation(
@@ -707,7 +707,8 @@ private[spark] class MapOutputTrackerMaster(
     val shuffleStatus = shuffleStatuses.get(dep.shuffleId).orNull
     if (shuffleStatus != null) {
       shuffleStatus.withMapStatuses { statuses =>
-        if (startMapIndex < endMapIndex && (startMapIndex >= 0 && endMapIndex < statuses.length)) {
+        if (startMapIndex < endMapIndex &&
+          (startMapIndex >= 0 && endMapIndex <= statuses.length)) {
           val statusesPicked = statuses.slice(startMapIndex, endMapIndex).filter(_ != null)
           statusesPicked.map(_.location.host).toSeq
         } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 25b1f8938f18f..94947a85f8fb4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -114,9 +114,13 @@ class AdaptiveQueryExecSuite
 
     val numLocalReaders = collect(plan) {
       case reader @ CustomShuffleReaderExec(_, _, LOCAL_SHUFFLE_READER_DESCRIPTION) => reader
-    }.length
-
-    assert(numShuffles === (numLocalReaders + numShufflesWithoutLocalReader))
+    }
+    numLocalReaders.foreach { r =>
+      val rdd = r.execute()
+      val parts = rdd.partitions
+      assert(parts.forall(rdd.preferredLocations(_).nonEmpty))
+    }
+    assert(numShuffles === (numLocalReaders.length + numShufflesWithoutLocalReader))
   }
 
   test("Change merge join to broadcast join") {

From 9d770d38fdf4a733e7bd12baf04530adafa2be4a Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 9 Mar 2020 09:11:21 -0500
Subject: [PATCH 0248/1280] [SPARK-30929][ML] ML, GraphX 3.0 QA: API: New Scala
 APIs, docs

### What changes were proposed in this pull request?
Auditing new ML Scala APIs introduced in 3.0. Fix found issues.

### Why are the changes needed?

### Does this PR introduce any user-facing change?
Yes. Some doc changes

### How was this patch tested?
Existing tests

Closes #27818 from huaxingao/spark-30929.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit b6b0343e3e90f0421cac277ed5ae8fb15b278d4e)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../ml/classification/FMClassifier.scala      |  2 +-
 .../MultilabelClassificationEvaluator.scala   | 20 +++++++++++++++++--
 .../ml/evaluation/RankingEvaluator.scala      | 16 +++++++++++++--
 .../spark/ml/feature/RobustScaler.scala       | 10 +++++-----
 .../ml/regression/AFTSurvivalRegression.scala |  4 ++--
 .../spark/ml/regression/FMRegressor.scala     |  2 +-
 .../org/apache/spark/ml/tree/treeParams.scala |  3 +++
 7 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index 3057d510782c7..a4d242753a789 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -187,7 +187,7 @@ class FMClassifier @Since("3.0.0") (
   @Since("3.0.0")
   def setSeed(value: Long): this.type = set(seed, value)
 
-  override protected[spark] def train(
+  override protected def train(
       dataset: Dataset[_]
     ): FMClassificationModel = instrumented { instr =>
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
index 5216c40819b06..a8db5452bd56c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
@@ -34,12 +34,13 @@ import org.apache.spark.sql.types._
  */
 @Since("3.0.0")
 @Experimental
-class MultilabelClassificationEvaluator (override val uid: String)
+class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String)
   extends Evaluator with HasPredictionCol with HasLabelCol
     with DefaultParamsWritable {
 
   import MultilabelClassificationEvaluator.supportedMetricNames
 
+  @Since("3.0.0")
   def this() = this(Identifiable.randomUID("mlcEval"))
 
   /**
@@ -49,6 +50,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
    * `"microF1Measure"`)
    * @group param
    */
+  @Since("3.0.0")
   final val metricName: Param[String] = {
     val allowedParams = ParamValidators.inArray(supportedMetricNames)
     new Param(this, "metricName", "metric name in evaluation " +
@@ -56,13 +58,21 @@ class MultilabelClassificationEvaluator (override val uid: String)
   }
 
   /** @group getParam */
+  @Since("3.0.0")
   def getMetricName: String = $(metricName)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
   setDefault(metricName -> "f1Measure")
 
+  /**
+   * param for the class whose metric will be computed in `"precisionByLabel"`, `"recallByLabel"`,
+   * `"f1MeasureByLabel"`.
+   * @group param
+   */
+  @Since("3.0.0")
   final val metricLabel: DoubleParam = new DoubleParam(this, "metricLabel",
     "The class whose metric will be computed in " +
       s"${supportedMetricNames.filter(_.endsWith("ByLabel")).mkString("(", "|", ")")}. " +
@@ -70,6 +80,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
     ParamValidators.gtEq(0.0))
 
   /** @group getParam */
+  @Since("3.0.0")
   def getMetricLabel: Double = $(metricLabel)
 
   /** @group setParam */
@@ -78,12 +89,14 @@ class MultilabelClassificationEvaluator (override val uid: String)
   setDefault(metricLabel -> 0.0)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
-
+  @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(predictionCol),
@@ -113,6 +126,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
     }
   }
 
+  @Since("3.0.0")
   override def isLargerBetter: Boolean = {
     $(metricName) match {
       case "hammingLoss" => false
@@ -120,6 +134,7 @@ class MultilabelClassificationEvaluator (override val uid: String)
     }
   }
 
+  @Since("3.0.0")
   override def copy(extra: ParamMap): MultilabelClassificationEvaluator = defaultCopy(extra)
 
   @Since("3.0.0")
@@ -139,5 +154,6 @@ object MultilabelClassificationEvaluator
     "precisionByLabel", "recallByLabel", "f1MeasureByLabel",
     "microPrecision", "microRecall", "microF1Measure")
 
+  @Since("3.0.0")
   override def load(path: String): MultilabelClassificationEvaluator = super.load(path)
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
index 8d017eb181f15..c5dea6c177e21 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
@@ -33,11 +33,12 @@ import org.apache.spark.sql.types._
  */
 @Experimental
 @Since("3.0.0")
-class RankingEvaluator (override val uid: String)
+class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String)
   extends Evaluator with HasPredictionCol with HasLabelCol with DefaultParamsWritable {
 
   import RankingEvaluator.supportedMetricNames
 
+  @Since("3.0.0")
   def this() = this(Identifiable.randomUID("rankEval"))
 
   /**
@@ -45,6 +46,7 @@ class RankingEvaluator (override val uid: String)
    * `"meanAveragePrecisionAtK"`, `"precisionAtK"`, `"ndcgAtK"`, `"recallAtK"`)
    * @group param
    */
+  @Since("3.0.0")
   final val metricName: Param[String] = {
     val allowedParams = ParamValidators.inArray(supportedMetricNames)
     new Param(this, "metricName", "metric name in evaluation " +
@@ -52,9 +54,11 @@ class RankingEvaluator (override val uid: String)
   }
 
   /** @group getParam */
+  @Since("3.0.0")
   def getMetricName: String = $(metricName)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
   setDefault(metricName -> "meanAveragePrecision")
@@ -64,6 +68,7 @@ class RankingEvaluator (override val uid: String)
    * `"ndcgAtK"`, `"recallAtK"`. Must be &gt; 0. The default value is 10.
    * @group param
    */
+  @Since("3.0.0")
   final val k = new IntParam(this, "k",
     "The ranking position value used in " +
       s"${supportedMetricNames.filter(_.endsWith("AtK")).mkString("(", "|", ")")}  " +
@@ -71,20 +76,24 @@ class RankingEvaluator (override val uid: String)
     ParamValidators.gt(0))
 
   /** @group getParam */
+  @Since("3.0.0")
   def getK: Int = $(k)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setK(value: Int): this.type = set(k, value)
 
   setDefault(k -> 10)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
 
   /** @group setParam */
+  @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
-
+  @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
     val schema = dataset.schema
     SchemaUtils.checkColumnTypes(schema, $(predictionCol),
@@ -107,8 +116,10 @@ class RankingEvaluator (override val uid: String)
     }
   }
 
+  @Since("3.0.0")
   override def isLargerBetter: Boolean = true
 
+  @Since("3.0.0")
   override def copy(extra: ParamMap): RankingEvaluator = defaultCopy(extra)
 
   @Since("3.0.0")
@@ -124,5 +135,6 @@ object RankingEvaluator extends DefaultParamsReadable[RankingEvaluator] {
   private val supportedMetricNames = Array("meanAveragePrecision",
     "meanAveragePrecisionAtK", "precisionAtK", "ndcgAtK", "recallAtK")
 
+  @Since("3.0.0")
   override def load(path: String): RankingEvaluator = super.load(path)
 }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
index 2a1204a40bde0..bd9be779fedbd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
@@ -120,7 +120,7 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
  * Note that NaN values are ignored in the computation of medians and ranges.
  */
 @Since("3.0.0")
-class RobustScaler (override val uid: String)
+class RobustScaler @Since("3.0.0") (@Since("3.0.0") override val uid: String)
   extends Estimator[RobustScalerModel] with RobustScalerParams with DefaultParamsWritable {
 
   import RobustScaler._
@@ -186,7 +186,7 @@ class RobustScaler (override val uid: String)
 object RobustScaler extends DefaultParamsReadable[RobustScaler] {
 
   // compute QuantileSummaries for each feature
-  private[spark] def computeSummaries(
+  private[ml] def computeSummaries(
       vectors: RDD[Vector],
       numFeatures: Int,
       relativeError: Double): RDD[(Int, QuantileSummaries)] = {
@@ -229,9 +229,9 @@ object RobustScaler extends DefaultParamsReadable[RobustScaler] {
  */
 @Since("3.0.0")
 class RobustScalerModel private[ml] (
-    override val uid: String,
-    val range: Vector,
-    val median: Vector)
+    @Since("3.0.0") override val uid: String,
+    @Since("3.0.0") val range: Vector,
+    @Since("3.0.0") val median: Vector)
   extends Model[RobustScalerModel] with RobustScalerParams with MLWritable {
 
   import RobustScalerModel._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 8c95d25f3b5b8..2da65a0bcf790 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -194,8 +194,8 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
       }
   }
 
-  @Since("3.0.0")
-  override def train(dataset: Dataset[_]): AFTSurvivalRegressionModel = instrumented { instr =>
+  override protected def train(
+      dataset: Dataset[_]): AFTSurvivalRegressionModel = instrumented { instr =>
     val instances = extractAFTPoints(dataset)
     val handlePersistence = dataset.storageLevel == StorageLevel.NONE
     if (handlePersistence) instances.persist(StorageLevel.MEMORY_AND_DISK)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
index a61244867f210..b017a1a039ce0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
@@ -410,7 +410,7 @@ class FMRegressor @Since("3.0.0") (
   @Since("3.0.0")
   def setSeed(value: Long): this.type = set(seed, value)
 
-  override protected[spark] def train(
+  override protected def train(
       dataset: Dataset[_]
     ): FMRegressionModel = instrumented { instr =>
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
index 26a639eea2992..a273cd71a0beb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
@@ -47,6 +47,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams
    * (default = "")
    * @group param
    */
+  @Since("3.0.0")
   final val leafCol: Param[String] =
     new Param[String](this, "leafCol", "Leaf indices column name. " +
       "Predicted leaf index of each instance in each tree by preorder")
@@ -139,9 +140,11 @@ private[ml] trait DecisionTreeParams extends PredictorParams
     cacheNodeIds -> false, checkpointInterval -> 10)
 
   /** @group setParam */
+  @Since("3.0.0")
   final def setLeafCol(value: String): this.type = set(leafCol, value)
 
   /** @group getParam */
+  @Since("3.0.0")
   final def getLeafCol: String = $(leafCol)
 
   /** @group getParam */

From 2e0d2b96195b0a3772225501a703fb02304aa346 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Mon, 9 Mar 2020 11:06:45 -0700
Subject: [PATCH 0249/1280] [SPARK-30941][PYSPARK] Add a note to asDict to
 document its behavior when there are duplicate fields

### What changes were proposed in this pull request?

Adding a note to document `Row.asDict` behavior when there are duplicate fields.

### Why are the changes needed?

When a row contains duplicate fields, `asDict` and `_get_item_` behaves differently. We should document it to let users know the difference explicitly.

### Does this PR introduce any user-facing change?

No. Only document change.

### How was this patch tested?

Existing test.

Closes #27853 from viirya/SPARK-30941.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit d21aab403a0a32e8b705b38874c0b335e703bd5d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 python/pyspark/sql/types.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index a5302e7bfd5ab..320a68dffe7a3 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1528,6 +1528,12 @@ def asDict(self, recursive=False):
 
         :param recursive: turns the nested Rows to dict (default: False).
 
+        .. note:: If a row contains duplicate field names, e.g., the rows of a join
+            between two :class:`DataFrame` that both have the fields of same names,
+            one of the duplicate fields will be selected by ``asDict``. ``__getitem__``
+            will also return one of the duplicate fields, however returned value might
+            be different to ``asDict``.
+
         >>> Row(name="Alice", age=11).asDict() == {'name': 'Alice', 'age': 11}
         True
         >>> row = Row(key=1, value=Row(name='a', age=2))

From 9caf009ecd9041e71efe2ef56ca0e75cc94cb56e Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Mon, 9 Mar 2020 12:17:59 -0700
Subject: [PATCH 0250/1280] [SPARK-31055][DOCS] Update config docs for shuffle
 local host reads to have dep on external shuffle service

### What changes were proposed in this pull request?

with SPARK-27651 we now support host local reads for shuffle, but only when external shuffle service is enabled. Update the config docs to state that.

### Why are the changes needed?

clarify dependency

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

n/a

Closes #27812 from tgravescs/SPARK-27651-follow.

Authored-by: Thomas Graves <tgraves@nvidia.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e807118eef9e0214170ff62c828524d237bd58e3)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../main/scala/org/apache/spark/internal/config/package.scala  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 23c31a5be831d..1308a460a2cc8 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1135,7 +1135,8 @@ package object config {
 
   private[spark] val SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED =
     ConfigBuilder("spark.shuffle.readHostLocalDisk")
-      .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled), shuffle " +
+      .doc(s"If enabled (and `${SHUFFLE_USE_OLD_FETCH_PROTOCOL.key}` is disabled and external " +
+        s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " +
         "blocks requested from those block managers which are running on the same host are read " +
         "from the disk directly instead of being fetched as remote blocks over the network.")
       .booleanConf

From b2f52ed8cbdfc4b7a1f161121cc9bf35973d140f Mon Sep 17 00:00:00 2001
From: Yuchen Huo <yuchen.huo@databricks.com>
Date: Tue, 10 Mar 2020 11:37:31 +0800
Subject: [PATCH 0251/1280] [SPARK-30902][SQL][FOLLOW-UP] Allow
 ReplaceTableAsStatement to have none provider

### What changes were proposed in this pull request?

This is a follow up for https://github.com/apache/spark/pull/27650 where allow None provider for create table. Here we are doing the same thing for ReplaceTable.

### Why are the changes needed?

Although currently the ASTBuilder doesn't seem to allow `replace` without `USING` clause. This would allow `DataFrameWriterV2` to use the statements instead of commands directly.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing tests

Closes #27838 from yuchenhuo/SPARK-30902.

Authored-by: Yuchen Huo <yuchen.huo@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a22994333ae129a3edaac75101fb4217fe8881c7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/analysis/ResolveCatalogs.scala  |  4 ++--
 .../apache/spark/sql/catalyst/parser/AstBuilder.scala  |  2 +-
 .../spark/sql/catalyst/plans/logical/statements.scala  |  4 ++--
 .../spark/sql/catalyst/parser/DDLParserSuite.scala     |  4 ++--
 .../sql/catalyst/analysis/ResolveSessionCatalog.scala  | 10 ++++++----
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 78a31718ccf36..88a3c0a73a10b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -153,7 +153,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         c.tableSchema,
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
-        convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
         orCreate = c.orCreate)
 
     case c @ ReplaceTableAsSelectStatement(
@@ -164,7 +164,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         // convert the bucket spec and add it as a transform
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
-        convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+        convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
         writeOptions = c.options,
         orCreate = c.orCreate)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index ce3383d8e17cb..3c23ecb0c4ff4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2823,7 +2823,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val (partitioning, bucketSpec, properties, options, location, comment) =
       visitCreateTableClauses(ctx.createTableClauses())
     val schema = Option(ctx.colTypeList()).map(createSchema)
-    val provider = ctx.tableProvider.multipartIdentifier.getText
+    val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText)
     val orCreate = ctx.replaceTableHeader().CREATE() != null
 
     Option(ctx.query).map(plan) match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index 89e1539e07260..50c3cd0d10278 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -114,7 +114,7 @@ case class ReplaceTableStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
@@ -129,7 +129,7 @@ case class ReplaceTableAsSelectStatement(
     partitioning: Seq[Transform],
     bucketSpec: Option[BucketSpec],
     properties: Map[String, String],
-    provider: String,
+    provider: Option[String],
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 967d0bdee4406..e3570899db9fb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -2065,7 +2065,7 @@ class DDLParserSuite extends AnalysisTest {
             replace.partitioning,
             replace.bucketSpec,
             replace.properties,
-            Some(replace.provider),
+            replace.provider,
             replace.options,
             replace.location,
             replace.comment)
@@ -2087,7 +2087,7 @@ class DDLParserSuite extends AnalysisTest {
             rtas.partitioning,
             rtas.bucketSpec,
             rtas.properties,
-            Some(rtas.provider),
+            rtas.provider,
             rtas.options,
             rtas.location,
             rtas.comment)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 90a3fb8056f97..5330612dd8162 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -282,7 +282,8 @@ class ResolveSessionCatalog(
     // session catalog and the table provider is not v2.
     case c @ ReplaceTableStatement(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         throw new AnalysisException("REPLACE TABLE is only supported with v2 tables.")
       } else {
         ReplaceTable(
@@ -291,13 +292,14 @@ class ResolveSessionCatalog(
           c.tableSchema,
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
-          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
           orCreate = c.orCreate)
       }
 
     case c @ ReplaceTableAsSelectStatement(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
-      if (!isV2Provider(c.provider)) {
+      val provider = c.provider.getOrElse(conf.defaultDataSourceName)
+      if (!isV2Provider(provider)) {
         throw new AnalysisException("REPLACE TABLE AS SELECT is only supported with v2 tables.")
       } else {
         ReplaceTableAsSelect(
@@ -306,7 +308,7 @@ class ResolveSessionCatalog(
           // convert the bucket spec and add it as a transform
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
-          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(c.provider)),
+          convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
           writeOptions = c.options,
           orCreate = c.orCreate)
       }

From 3fb450c0b7c4e415a29d51cd10e6be6ad8dff114 Mon Sep 17 00:00:00 2001
From: maryannxue <maryannxue@apache.org>
Date: Tue, 10 Mar 2020 14:15:44 +0800
Subject: [PATCH 0252/1280] [SPARK-31096][SQL] Replace `Array` with `Seq` in
 AQE `CustomShuffleReaderExec`

### What changes were proposed in this pull request?
This PR changes the type of `CustomShuffleReaderExec`'s `partitionSpecs` from `Array` to `Seq`, since `Array` compares references not values for equality, which could lead to potential plan reuse problem.

### Why are the changes needed?
Unlike `Seq`, `Array` compares references not values for equality, which could lead to potential plan reuse problem.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Passes existing UTs.

Closes #27857 from maryannxue/aqe-customreader-fix.

Authored-by: maryannxue <maryannxue@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/CustomShuffleReaderExec.scala    |  4 +--
 .../adaptive/OptimizeLocalShuffleReader.scala | 10 +++---
 .../adaptive/OptimizeSkewedJoin.scala         | 12 +++----
 .../adaptive/ShufflePartitionsCoalescer.scala |  6 ++--
 .../ShufflePartitionsCoalescerSuite.scala     | 36 +++++++++----------
 5 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
index be372bbfcf700..ba3f72592908e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExcha
  */
 case class CustomShuffleReaderExec private(
     child: SparkPlan,
-    partitionSpecs: Array[ShufflePartitionSpec],
+    partitionSpecs: Seq[ShufflePartitionSpec],
     description: String) extends UnaryExecNode {
 
   override def output: Seq[Attribute] = child.output
@@ -71,7 +71,7 @@ case class CustomShuffleReaderExec private(
       cachedShuffleRDD = child match {
         case stage: ShuffleQueryStageExec =>
           new ShuffledRowRDD(
-            stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs)
+            stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs.toArray)
         case _ =>
           throw new IllegalStateException("operating on canonicalization plan")
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index e4417632b2ee4..fb6b40c710929 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -77,21 +77,21 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
   //       partition start indices based on block size to avoid data skew.
   private def getPartitionSpecs(
       shuffleStage: ShuffleQueryStageExec,
-      advisoryParallelism: Option[Int]): Array[ShufflePartitionSpec] = {
+      advisoryParallelism: Option[Int]): Seq[ShufflePartitionSpec] = {
     val shuffleDep = shuffleStage.shuffle.shuffleDependency
     val numReducers = shuffleDep.partitioner.numPartitions
     val expectedParallelism = advisoryParallelism.getOrElse(numReducers)
     val numMappers = shuffleDep.rdd.getNumPartitions
     val splitPoints = if (numMappers == 0) {
-      Array.empty
+      Seq.empty
     } else {
-      equallyDivide(numReducers, math.max(1, expectedParallelism / numMappers)).toArray
+      equallyDivide(numReducers, math.max(1, expectedParallelism / numMappers))
     }
     (0 until numMappers).flatMap { mapIndex =>
       (splitPoints :+ numReducers).sliding(2).map {
-        case Array(start, end) => PartialMapperPartitionSpec(mapIndex, start, end)
+        case Seq(start, end) => PartialMapperPartitionSpec(mapIndex, start, end)
       }
-    }.toArray
+    }
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 2e8adcf3cd083..c3bcce41489fb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -108,7 +108,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   private def getMapStartIndices(
       stage: ShuffleQueryStageExec,
       partitionId: Int,
-      targetSize: Long): Array[Int] = {
+      targetSize: Long): Seq[Int] = {
     val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
     val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
     val partitionStartIndices = ArrayBuffer[Int]()
@@ -126,7 +126,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       i += 1
     }
 
-    partitionStartIndices.toArray
+    partitionStartIndices
   }
 
   private def getStatistics(stage: ShuffleQueryStageExec): MapOutputStatistics = {
@@ -255,10 +255,8 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       logDebug("number of skewed partitions: " +
         s"left ${leftSkewDesc.numPartitions}, right ${rightSkewDesc.numPartitions}")
       if (leftSkewDesc.numPartitions > 0 || rightSkewDesc.numPartitions > 0) {
-        val newLeft = CustomShuffleReaderExec(
-          left, leftSidePartitions.toArray, leftSkewDesc.toString)
-        val newRight = CustomShuffleReaderExec(
-          right, rightSidePartitions.toArray, rightSkewDesc.toString)
+        val newLeft = CustomShuffleReaderExec(left, leftSidePartitions, leftSkewDesc.toString)
+        val newRight = CustomShuffleReaderExec(right, rightSidePartitions, rightSkewDesc.toString)
         smj.copy(
           left = s1.copy(child = newLeft), right = s2.copy(child = newRight), isSkewJoin = true)
       } else {
@@ -286,7 +284,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
   private def createSkewPartitions(
       reducerIndex: Int,
-      mapStartIndices: Array[Int],
+      mapStartIndices: Seq[Int],
       numMappers: Int): Seq[PartialReducerPartitionSpec] = {
     mapStartIndices.indices.map { i =>
       val startMapIndex = mapStartIndices(i)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
index c3b8bf6c757f8..8c5824178f665 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
@@ -47,7 +47,7 @@ object ShufflePartitionsCoalescer extends Logging {
    *  - coalesced partition 2: shuffle partition 2 (size 170 MiB)
    *  - coalesced partition 3: shuffle partition 3 and 4 (size 50 MiB)
    *
-   *  @return An array of [[CoalescedPartitionSpec]]s. For example, if partitions [0, 1, 2, 3, 4]
+   *  @return A sequence of [[CoalescedPartitionSpec]]s. For example, if partitions [0, 1, 2, 3, 4]
    *          split at indices [0, 2, 3], the returned partition specs will be:
    *          CoalescedPartitionSpec(0, 2), CoalescedPartitionSpec(2, 3) and
    *          CoalescedPartitionSpec(3, 5).
@@ -57,7 +57,7 @@ object ShufflePartitionsCoalescer extends Logging {
       firstPartitionIndex: Int,
       lastPartitionIndex: Int,
       advisoryTargetSize: Long,
-      minNumPartitions: Int = 1): Array[ShufflePartitionSpec] = {
+      minNumPartitions: Int = 1): Seq[ShufflePartitionSpec] = {
     // If `minNumPartitions` is very large, it is possible that we need to use a value less than
     // `advisoryTargetSize` as the target size of a coalesced task.
     val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum
@@ -112,6 +112,6 @@ object ShufflePartitionsCoalescer extends Logging {
     }
     partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, lastPartitionIndex)
 
-    partitionSpecs.toArray
+    partitionSpecs
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
index 0befa0640efcc..8aab2995467c2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
@@ -24,7 +24,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
 
   private def checkEstimation(
       bytesByPartitionIdArray: Array[Array[Long]],
-      expectedPartitionStartIndices: Array[CoalescedPartitionSpec],
+      expectedPartitionStartIndices: Seq[CoalescedPartitionSpec],
       targetSize: Long,
       minNumPartitions: Int = 1): Unit = {
     val mapOutputStatistics = bytesByPartitionIdArray.zipWithIndex.map {
@@ -46,7 +46,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
     {
       // All bytes per partition are 0.
       val bytesByPartitionId = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
       checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
@@ -54,21 +54,21 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // Some bytes per partition are 0 and total size is less than the target size.
       // 1 coalesced partition is expected.
       val bytesByPartitionId = Array[Long](10, 0, 20, 0, 0)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
       checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // 2 coalesced partitions are expected.
       val bytesByPartitionId = Array[Long](10, 0, 90, 20, 0)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
       checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
 
     {
       // There are a few large shuffle partitions.
       val bytesByPartitionId = Array[Long](110, 10, 100, 110, 0)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 2),
         CoalescedPartitionSpec(2, 3),
@@ -80,7 +80,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
     {
       // All shuffle partitions are larger than the targeted size.
       val bytesByPartitionId = Array[Long](100, 110, 100, 110, 110)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 2),
         CoalescedPartitionSpec(2, 3),
@@ -92,7 +92,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
     {
       // The last shuffle partition is in a single coalesced partition.
       val bytesByPartitionId = Array[Long](30, 30, 0, 40, 110)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 4), CoalescedPartitionSpec(4, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 4), CoalescedPartitionSpec(4, 5))
       checkEstimation(Array(bytesByPartitionId), expectedPartitionSpecs, targetSize)
     }
   }
@@ -106,7 +106,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0, 0)
       intercept[AssertionError] {
-        checkEstimation(Array(bytesByPartitionId1, bytesByPartitionId2), Array.empty, targetSize)
+        checkEstimation(Array(bytesByPartitionId1, bytesByPartitionId2), Seq.empty, targetSize)
       }
     }
 
@@ -114,7 +114,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // All bytes per partition are 0.
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
         expectedPartitionSpecs,
@@ -126,7 +126,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 1 coalesced partition is expected.
       val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 20, 0, 20)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
         expectedPartitionSpecs,
@@ -137,7 +137,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 2 coalesced partition are expected.
       val bytesByPartitionId1 = Array[Long](0, 10, 0, 20, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 2),
         CoalescedPartitionSpec(2, 4),
         CoalescedPartitionSpec(4, 5))
@@ -151,7 +151,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 4 coalesced partition are expected.
       val bytesByPartitionId1 = Array[Long](0, 99, 0, 20, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 2),
         CoalescedPartitionSpec(2, 4),
@@ -166,7 +166,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // 2 coalesced partition are needed.
       val bytesByPartitionId1 = Array[Long](0, 100, 0, 30, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 70, 0, 30)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 2),
         CoalescedPartitionSpec(2, 4),
@@ -181,7 +181,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // There are a few large shuffle partitions.
       val bytesByPartitionId1 = Array[Long](0, 100, 40, 30, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 60, 0, 110)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 2),
         CoalescedPartitionSpec(2, 3),
@@ -197,7 +197,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // All pairs of shuffle partitions are larger than the targeted size.
       val bytesByPartitionId1 = Array[Long](100, 100, 40, 30, 0)
       val bytesByPartitionId2 = Array[Long](30, 0, 60, 70, 110)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 2),
         CoalescedPartitionSpec(2, 3),
@@ -219,7 +219,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // the size of data is 0.
       val bytesByPartitionId1 = Array[Long](0, 0, 0, 0, 0)
       val bytesByPartitionId2 = Array[Long](0, 0, 0, 0, 0)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
         expectedPartitionSpecs,
@@ -230,7 +230,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // The minimal number of coalesced partitions is enforced.
       val bytesByPartitionId1 = Array[Long](10, 5, 5, 0, 20)
       val bytesByPartitionId2 = Array[Long](5, 10, 0, 10, 5)
-      val expectedPartitionSpecs = Array(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
+      val expectedPartitionSpecs = Seq(CoalescedPartitionSpec(0, 3), CoalescedPartitionSpec(3, 5))
       checkEstimation(
         Array(bytesByPartitionId1, bytesByPartitionId2),
         expectedPartitionSpecs,
@@ -241,7 +241,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       // The number of coalesced partitions is determined by the algorithm.
       val bytesByPartitionId1 = Array[Long](10, 50, 20, 80, 20)
       val bytesByPartitionId2 = Array[Long](40, 10, 0, 10, 30)
-      val expectedPartitionSpecs = Array(
+      val expectedPartitionSpecs = Seq(
         CoalescedPartitionSpec(0, 1),
         CoalescedPartitionSpec(1, 3),
         CoalescedPartitionSpec(3, 4),

From 0985f13bc66a99319820d0d9ba5b3f2a254f61a4 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 10 Mar 2020 00:33:32 -0700
Subject: [PATCH 0253/1280] [SPARK-31065][SQL] Match schema_of_json to the
 schema inference of JSON data source

This PR proposes two things:

1. Convert `null` to `string` type during schema inference of `schema_of_json` as JSON datasource does. This is a bug fix as well because `null` string is not the proper DDL formatted string and it is unable for SQL parser to recognise it as a type string. We should match it to JSON datasource and return a string type so `schema_of_json` returns a proper DDL formatted string.

2. Let `schema_of_json` respect `dropFieldIfAllNull` option during schema inference.

To let `schema_of_json` return a proper DDL formatted string, and respect `dropFieldIfAllNull` option.

Yes, it does.

```scala
import collection.JavaConverters._
import org.apache.spark.sql.functions._

spark.range(1).select(schema_of_json(lit("""{"id": ""}"""))).show()
spark.range(1).select(schema_of_json(lit("""{"id": "a", "drop": {"drop": null}}"""), Map("dropFieldIfAllNull" -> "true").asJava)).show(false)
```

**Before:**

```
struct<id:null>
struct<drop:struct<drop:null>,id:string>
```

**After:**

```
struct<id:string>
struct<id:string>
```

Manually tested, and unittests were added.

Closes #27854 from HyukjinKwon/SPARK-31065.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 815c7929c290d6eed86dc5c924f9f7d48cff179d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/jsonExpressions.scala         | 13 ++++++-
 .../sql/catalyst/json/JsonInferSchema.scala   | 13 ++++---
 .../apache/spark/sql/JsonFunctionsSuite.scala | 35 +++++++++++++++++++
 3 files changed, 53 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index 61afdb6c9492f..a63e541a8cad4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -773,7 +773,18 @@ case class SchemaOfJson(
   override def eval(v: InternalRow): Any = {
     val dt = Utils.tryWithResource(CreateJacksonParser.utf8String(jsonFactory, json)) { parser =>
       parser.nextToken()
-      jsonInferSchema.inferField(parser)
+      // To match with schema inference from JSON datasource.
+      jsonInferSchema.inferField(parser) match {
+        case st: StructType =>
+          jsonInferSchema.canonicalizeType(st, jsonOptions).getOrElse(StructType(Nil))
+        case at: ArrayType if at.elementType.isInstanceOf[StructType] =>
+          jsonInferSchema
+            .canonicalizeType(at.elementType, jsonOptions)
+            .map(ArrayType(_, containsNull = at.containsNull))
+            .getOrElse(ArrayType(StructType(Nil), containsNull = at.containsNull))
+        case other: DataType =>
+          jsonInferSchema.canonicalizeType(other, jsonOptions).getOrElse(StringType)
+      }
     }
 
     UTF8String.fromString(dt.catalogString)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 82dd6d0da2632..3dd8694eae418 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -92,12 +92,10 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
     }
     json.sparkContext.runJob(mergedTypesFromPartitions, foldPartition, mergeResult)
 
-    canonicalizeType(rootType, options) match {
-      case Some(st: StructType) => st
-      case _ =>
-        // canonicalizeType erases all empty structs, including the only one we want to keep
-        StructType(Nil)
-    }
+    canonicalizeType(rootType, options)
+      .find(_.isInstanceOf[StructType])
+      // canonicalizeType erases all empty structs, including the only one we want to keep
+      .getOrElse(StructType(Nil)).asInstanceOf[StructType]
   }
 
   /**
@@ -198,7 +196,8 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
    * Recursively canonicalizes inferred types, e.g., removes StructTypes with no fields,
    * drops NullTypes or converts them to StringType based on provided options.
    */
-  private def canonicalizeType(tpe: DataType, options: JSONOptions): Option[DataType] = tpe match {
+  private[catalyst] def canonicalizeType(
+      tpe: DataType, options: JSONOptions): Option[DataType] = tpe match {
     case at: ArrayType =>
       canonicalizeType(at.elementType, options)
         .map(t => at.copy(elementType = t))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index fd1e9e309558e..8cc5c227eb1a2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -653,4 +653,39 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
       assert(json_tuple_result === len)
     }
   }
+
+  test("SPARK-31065: schema_of_json - null and empty strings as strings") {
+    Seq("""{"id": null}""", """{"id": ""}""").foreach { input =>
+      checkAnswer(
+        spark.range(1).select(schema_of_json(input)),
+        Seq(Row("struct<id:string>")))
+    }
+  }
+
+  test("SPARK-31065: schema_of_json - 'dropFieldIfAllNull' option") {
+    val options = Map("dropFieldIfAllNull" -> "true")
+    // Structs
+    checkAnswer(
+      spark.range(1).select(
+        schema_of_json(
+          lit("""{"id": "a", "drop": {"drop": null}}"""),
+          options.asJava)),
+      Seq(Row("struct<id:string>")))
+
+    // Array of structs
+    checkAnswer(
+      spark.range(1).select(
+        schema_of_json(
+          lit("""[{"id": "a", "drop": {"drop": null}}]"""),
+          options.asJava)),
+      Seq(Row("array<struct<id:string>>")))
+
+    // Other types are not affected.
+    checkAnswer(
+      spark.range(1).select(
+        schema_of_json(
+          lit("""null"""),
+          options.asJava)),
+      Seq(Row("string")))
+  }
 }

From 69dcea284961668b28d702e90e9068a3b80cbc8a Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 10 Mar 2020 18:04:09 +0900
Subject: [PATCH 0254/1280] [SPARK-30992][DSTREAMS] Arrange scattered config of
 streaming module

### What changes were proposed in this pull request?
I found a lot scattered config in `Streaming`.I think should arrange these config in unified position.

### Why are the changes needed?
Arrange scattered config

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27744 from beliefer/arrange-scattered-streaming-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8ee41f3576689f3d164131d1e6041bd347394364)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/streaming/StreamingConf.scala       | 161 ++++++++++++++++++
 .../spark/streaming/StreamingContext.scala    |   3 +-
 .../spark/streaming/dstream/DStream.scala     |   3 +-
 .../streaming/receiver/BlockGenerator.scala   |   5 +-
 .../streaming/receiver/RateLimiter.scala      |   5 +-
 .../streaming/scheduler/JobGenerator.scala    |   6 +-
 .../streaming/scheduler/JobScheduler.scala    |   2 +-
 .../streaming/scheduler/RateController.scala  |   3 +-
 .../scheduler/rate/RateEstimator.scala        |  11 +-
 .../ui/StreamingJobProgressListener.scala     |   4 +-
 .../spark/streaming/util/StateMap.scala       |   4 +-
 .../streaming/util/WriteAheadLogUtils.scala   |  42 ++---
 .../streaming/ReceiverInputDStreamSuite.scala |   3 +-
 13 files changed, 201 insertions(+), 51 deletions(-)
 create mode 100644 streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala

diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
new file mode 100644
index 0000000000000..71aefd647f905
--- /dev/null
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.streaming
+
+import java.util.concurrent.TimeUnit
+
+import org.apache.spark.internal.config.ConfigBuilder
+import org.apache.spark.streaming.util.OpenHashMapBasedStateMap.DELTA_CHAIN_LENGTH_THRESHOLD
+
+object StreamingConf {
+
+  private[streaming] val BACKPRESSURE_ENABLED =
+    ConfigBuilder("spark.streaming.backpressure.enabled")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val RECEIVER_MAX_RATE =
+    ConfigBuilder("spark.streaming.receiver.maxRate")
+      .longConf
+      .createWithDefault(Long.MaxValue)
+
+  private[streaming] val BACKPRESSURE_INITIAL_RATE =
+    ConfigBuilder("spark.streaming.backpressure.initialRate")
+      .fallbackConf(RECEIVER_MAX_RATE)
+
+  private[streaming] val BLOCK_INTERVAL =
+    ConfigBuilder("spark.streaming.blockInterval")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createWithDefaultString("200ms")
+
+  private[streaming] val RECEIVER_WAL_ENABLE_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.enable")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val RECEIVER_WAL_CLASS_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.class")
+      .stringConf
+      .createOptional
+
+  private[streaming] val RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.rollingIntervalSecs")
+      .intConf
+      .createWithDefault(60)
+
+  private[streaming] val RECEIVER_WAL_MAX_FAILURES_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.maxFailures")
+      .intConf
+      .createWithDefault(3)
+
+  private[streaming] val RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
+    ConfigBuilder("spark.streaming.receiver.writeAheadLog.closeFileAfterWrite")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val DRIVER_WAL_CLASS_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.class")
+      .stringConf
+      .createOptional
+
+  private[streaming] val DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.rollingIntervalSecs")
+      .intConf
+      .createWithDefault(60)
+
+  private[streaming] val DRIVER_WAL_MAX_FAILURES_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.maxFailures")
+      .intConf
+      .createWithDefault(3)
+
+  private[streaming] val DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.closeFileAfterWrite")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val DRIVER_WAL_BATCHING_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.allowBatching")
+      .booleanConf
+      .createWithDefault(true)
+
+  private[streaming] val DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY =
+    ConfigBuilder("spark.streaming.driver.writeAheadLog.batchingTimeout")
+      .longConf
+      .createWithDefault(5000)
+
+  private[streaming] val STREAMING_UNPERSIST =
+    ConfigBuilder("spark.streaming.unpersist")
+      .booleanConf
+      .createWithDefault(true)
+
+  private[streaming] val STOP_GRACEFULLY_ON_SHUTDOWN =
+    ConfigBuilder("spark.streaming.stopGracefullyOnShutdown")
+      .booleanConf
+      .createWithDefault(false)
+
+  private[streaming] val UI_RETAINED_BATCHES =
+    ConfigBuilder("spark.streaming.ui.retainedBatches")
+      .intConf
+      .createWithDefault(1000)
+
+  private[streaming] val SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD =
+    ConfigBuilder("spark.streaming.sessionByKey.deltaChainThreshold")
+      .intConf
+      .createWithDefault(DELTA_CHAIN_LENGTH_THRESHOLD)
+
+  private[streaming] val BACKPRESSURE_RATE_ESTIMATOR =
+    ConfigBuilder("spark.streaming.backpressure.rateEstimator")
+      .stringConf
+      .createWithDefault("pid")
+
+  private[streaming] val BACKPRESSURE_PID_PROPORTIONAL =
+    ConfigBuilder("spark.streaming.backpressure.pid.proportional")
+      .doubleConf
+      .createWithDefault(1.0)
+
+  private[streaming] val BACKPRESSURE_PID_INTEGRAL =
+    ConfigBuilder("spark.streaming.backpressure.pid.integral")
+      .doubleConf
+      .createWithDefault(0.2)
+
+  private[streaming] val BACKPRESSURE_PID_DERIVED =
+    ConfigBuilder("spark.streaming.backpressure.pid.derived")
+      .doubleConf
+      .createWithDefault(0.0)
+
+  private[streaming] val BACKPRESSURE_PID_MIN_RATE =
+    ConfigBuilder("spark.streaming.backpressure.pid.minRate")
+      .doubleConf
+      .createWithDefault(100)
+
+  private[streaming] val CONCURRENT_JOBS =
+    ConfigBuilder("spark.streaming.concurrentJobs")
+      .intConf
+      .createWithDefault(1)
+
+  private[streaming] val GRACEFUL_STOP_TIMEOUT =
+    ConfigBuilder("spark.streaming.gracefulStopTimeout")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .createOptional
+
+  private[streaming] val MANUAL_CLOCK_JUMP =
+    ConfigBuilder("spark.streaming.manualClock.jump")
+      .longConf
+      .createWithDefault(0)
+
+}
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
index 440b653e45de1..e3459c96335b3 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala
@@ -41,6 +41,7 @@ import org.apache.spark.rdd.{RDD, RDDOperationScope}
 import org.apache.spark.scheduler.LiveListenerBus
 import org.apache.spark.serializer.SerializationDebugger
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.streaming.StreamingConf.STOP_GRACEFULLY_ON_SHUTDOWN
 import org.apache.spark.streaming.StreamingContextState._
 import org.apache.spark.streaming.dstream._
 import org.apache.spark.streaming.receiver.Receiver
@@ -717,7 +718,7 @@ class StreamingContext private[streaming] (
   }
 
   private def stopOnShutdown(): Unit = {
-    val stopGracefully = conf.getBoolean("spark.streaming.stopGracefullyOnShutdown", false)
+    val stopGracefully = conf.get(STOP_GRACEFULLY_ON_SHUTDOWN)
     logInfo(s"Invoking stop(stopGracefully=$stopGracefully) from shutdown hook")
     // Do not stop SparkContext, let its own shutdown hook stop it
     stop(stopSparkContext = false, stopGracefully = stopGracefully)
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
index 6c981b293ac76..e037f26088347 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala
@@ -31,6 +31,7 @@ import org.apache.spark.internal.io.SparkHadoopWriterUtils
 import org.apache.spark.rdd.{BlockRDD, RDD, RDDOperationScope}
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.streaming._
+import org.apache.spark.streaming.StreamingConf.STREAMING_UNPERSIST
 import org.apache.spark.streaming.StreamingContext.rddToFileName
 import org.apache.spark.streaming.scheduler.Job
 import org.apache.spark.ui.{UIUtils => SparkUIUtils}
@@ -447,7 +448,7 @@ abstract class DStream[T: ClassTag] (
    * this to clear their own metadata along with the generated RDDs.
    */
   private[streaming] def clearMetadata(time: Time): Unit = {
-    val unpersistData = ssc.conf.getBoolean("spark.streaming.unpersist", true)
+    val unpersistData = ssc.conf.get(STREAMING_UNPERSIST)
     val oldRDDs = generatedRDDs.filter(_._1 <= (time - rememberDuration))
     logDebug("Clearing references to old RDDs: [" +
       oldRDDs.map(x => s"${x._1} -> ${x._2.id}").mkString(", ") + "]")
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala b/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala
index 2533c53883cac..d641f55fa7f6f 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/receiver/BlockGenerator.scala
@@ -24,6 +24,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.internal.Logging
 import org.apache.spark.storage.StreamBlockId
+import org.apache.spark.streaming.StreamingConf.BLOCK_INTERVAL
 import org.apache.spark.streaming.util.RecurringTimer
 import org.apache.spark.util.{Clock, SystemClock}
 
@@ -100,8 +101,8 @@ private[streaming] class BlockGenerator(
   }
   import GeneratorState._
 
-  private val blockIntervalMs = conf.getTimeAsMs("spark.streaming.blockInterval", "200ms")
-  require(blockIntervalMs > 0, s"'spark.streaming.blockInterval' should be a positive value")
+  private val blockIntervalMs = conf.get(BLOCK_INTERVAL)
+  require(blockIntervalMs > 0, s"'${BLOCK_INTERVAL.key}' should be a positive value")
 
   private val blockIntervalTimer =
     new RecurringTimer(clock, blockIntervalMs, updateCurrentBuffer, "BlockGenerator")
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala b/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala
index c620074b4e44d..f77ca3e8fdb45 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/receiver/RateLimiter.scala
@@ -21,6 +21,7 @@ import com.google.common.util.concurrent.{RateLimiter => GuavaRateLimiter}
 
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.Logging
+import org.apache.spark.streaming.StreamingConf.{BACKPRESSURE_INITIAL_RATE, RECEIVER_MAX_RATE}
 
 /**
  * Provides waitToPush() method to limit the rate at which receivers consume data.
@@ -37,7 +38,7 @@ import org.apache.spark.internal.Logging
 private[receiver] abstract class RateLimiter(conf: SparkConf) extends Logging {
 
   // treated as an upper limit
-  private val maxRateLimit = conf.getLong("spark.streaming.receiver.maxRate", Long.MaxValue)
+  private val maxRateLimit = conf.get(RECEIVER_MAX_RATE)
   private lazy val rateLimiter = GuavaRateLimiter.create(getInitialRateLimit().toDouble)
 
   def waitToPush(): Unit = {
@@ -68,6 +69,6 @@ private[receiver] abstract class RateLimiter(conf: SparkConf) extends Logging {
    * Get the initial rateLimit to initial rateLimiter
    */
   private def getInitialRateLimit(): Long = {
-    math.min(conf.getLong("spark.streaming.backpressure.initialRate", maxRateLimit), maxRateLimit)
+    math.min(conf.get(BACKPRESSURE_INITIAL_RATE), maxRateLimit)
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
index 7e8449ee5aa7e..8008a5c495e9d 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobGenerator.scala
@@ -23,7 +23,7 @@ import scala.util.{Failure, Success, Try}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
-import org.apache.spark.streaming.{Checkpoint, CheckpointWriter, Time}
+import org.apache.spark.streaming.{Checkpoint, CheckpointWriter, StreamingConf, Time}
 import org.apache.spark.streaming.api.python.PythonDStream
 import org.apache.spark.streaming.util.RecurringTimer
 import org.apache.spark.util.{Clock, EventLoop, ManualClock, Utils}
@@ -115,7 +115,7 @@ class JobGenerator(jobScheduler: JobScheduler) extends Logging {
       logInfo("Stopping JobGenerator gracefully")
       val timeWhenStopStarted = System.nanoTime()
       val stopTimeoutMs = conf.getTimeAsMs(
-        "spark.streaming.gracefulStopTimeout", s"${10 * ssc.graph.batchDuration.milliseconds}ms")
+        StreamingConf.GRACEFUL_STOP_TIMEOUT.key, s"${10 * ssc.graph.batchDuration.milliseconds}ms")
       val pollTime = 100
 
       // To prevent graceful stop to get stuck permanently
@@ -206,7 +206,7 @@ class JobGenerator(jobScheduler: JobScheduler) extends Logging {
     // or if the property is defined set it to that time
     if (clock.isInstanceOf[ManualClock]) {
       val lastTime = ssc.initialCheckpoint.checkpointTime.milliseconds
-      val jumpTime = ssc.sc.conf.getLong("spark.streaming.manualClock.jump", 0)
+      val jumpTime = ssc.sc.conf.get(StreamingConf.MANUAL_CLOCK_JUMP)
       clock.asInstanceOf[ManualClock].setTime(lastTime + jumpTime)
     }
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
index 7eea57cc083ed..a6d8dccd7e722 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/JobScheduler.scala
@@ -47,7 +47,7 @@ class JobScheduler(val ssc: StreamingContext) extends Logging {
   // Use of ConcurrentHashMap.keySet later causes an odd runtime problem due to Java 7/8 diff
   // https://gist.github.com/AlainODea/1375759b8720a3f9f094
   private val jobSets: java.util.Map[Time, JobSet] = new ConcurrentHashMap[Time, JobSet]
-  private val numConcurrentJobs = ssc.conf.getInt("spark.streaming.concurrentJobs", 1)
+  private val numConcurrentJobs = ssc.conf.get(StreamingConf.CONCURRENT_JOBS)
   private val jobExecutor =
     ThreadUtils.newDaemonFixedThreadPool(numConcurrentJobs, "streaming-job-executor")
   private[streaming] val jobGenerator = new JobGenerator(this)
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala
index 7774e85f778a6..88f191fa022c6 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/RateController.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicLong
 import scala.concurrent.{ExecutionContext, Future}
 
 import org.apache.spark.SparkConf
+import org.apache.spark.streaming.StreamingConf.BACKPRESSURE_ENABLED
 import org.apache.spark.streaming.scheduler.rate.RateEstimator
 import org.apache.spark.util.{ThreadUtils, Utils}
 
@@ -86,5 +87,5 @@ private[streaming] abstract class RateController(val streamUID: Int, rateEstimat
 
 object RateController {
   def isBackPressureEnabled(conf: SparkConf): Boolean =
-    conf.getBoolean("spark.streaming.backpressure.enabled", false)
+    conf.get(BACKPRESSURE_ENABLED)
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala
index e4b9dffee04f4..7f4d0f298e8f2 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/scheduler/rate/RateEstimator.scala
@@ -19,6 +19,7 @@ package org.apache.spark.streaming.scheduler.rate
 
 import org.apache.spark.SparkConf
 import org.apache.spark.streaming.Duration
+import org.apache.spark.streaming.StreamingConf._
 
 /**
  * A component that estimates the rate at which an `InputDStream` should ingest
@@ -57,12 +58,12 @@ object RateEstimator {
    * @throws IllegalArgumentException if the configured RateEstimator is not `pid`.
    */
   def create(conf: SparkConf, batchInterval: Duration): RateEstimator =
-    conf.get("spark.streaming.backpressure.rateEstimator", "pid") match {
+    conf.get(BACKPRESSURE_RATE_ESTIMATOR) match {
       case "pid" =>
-        val proportional = conf.getDouble("spark.streaming.backpressure.pid.proportional", 1.0)
-        val integral = conf.getDouble("spark.streaming.backpressure.pid.integral", 0.2)
-        val derived = conf.getDouble("spark.streaming.backpressure.pid.derived", 0.0)
-        val minRate = conf.getDouble("spark.streaming.backpressure.pid.minRate", 100)
+        val proportional = conf.get(BACKPRESSURE_PID_PROPORTIONAL)
+        val integral = conf.get(BACKPRESSURE_PID_INTEGRAL)
+        val derived = conf.get(BACKPRESSURE_PID_DERIVED)
+        val minRate = conf.get(BACKPRESSURE_PID_MIN_RATE)
         new PIDRateEstimator(batchInterval.milliseconds, proportional, integral, derived, minRate)
 
       case estimator =>
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala
index de73762beb860..da351ecf1889c 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingJobProgressListener.scala
@@ -24,7 +24,7 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, Queue}
 
 import org.apache.spark.scheduler._
-import org.apache.spark.streaming.{StreamingContext, Time}
+import org.apache.spark.streaming.{StreamingConf, StreamingContext, Time}
 import org.apache.spark.streaming.scheduler._
 
 private[spark] class StreamingJobProgressListener(ssc: StreamingContext)
@@ -33,7 +33,7 @@ private[spark] class StreamingJobProgressListener(ssc: StreamingContext)
   private val waitingBatchUIData = new HashMap[Time, BatchUIData]
   private val runningBatchUIData = new HashMap[Time, BatchUIData]
   private val completedBatchUIData = new Queue[BatchUIData]
-  private val batchUIDataLimit = ssc.conf.getInt("spark.streaming.ui.retainedBatches", 1000)
+  private val batchUIDataLimit = ssc.conf.get(StreamingConf.UI_RETAINED_BATCHES)
   private var totalCompletedBatches = 0L
   private var totalReceivedRecords = 0L
   private var totalProcessedRecords = 0L
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala
index 618c036377aee..4224cef1cbae1 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/StateMap.scala
@@ -26,6 +26,7 @@ import com.esotericsoftware.kryo.io.{Input, Output}
 
 import org.apache.spark.SparkConf
 import org.apache.spark.serializer.{KryoInputObjectInputBridge, KryoOutputObjectOutputBridge}
+import org.apache.spark.streaming.StreamingConf.SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD
 import org.apache.spark.streaming.util.OpenHashMapBasedStateMap._
 import org.apache.spark.util.collection.OpenHashMap
 
@@ -61,8 +62,7 @@ private[streaming] object StateMap {
   def empty[K, S]: StateMap[K, S] = new EmptyStateMap[K, S]
 
   def create[K: ClassTag, S: ClassTag](conf: SparkConf): StateMap[K, S] = {
-    val deltaChainThreshold = conf.getInt("spark.streaming.sessionByKey.deltaChainThreshold",
-      DELTA_CHAIN_LENGTH_THRESHOLD)
+    val deltaChainThreshold = conf.get(SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD)
     new OpenHashMapBasedStateMap[K, S](deltaChainThreshold)
   }
 }
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala
index b0a4c98fc9e57..224e782066f60 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/WriteAheadLogUtils.scala
@@ -23,52 +23,34 @@ import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.internal.Logging
+import org.apache.spark.streaming.StreamingConf._
 import org.apache.spark.util.Utils
 
 /** A helper class with utility functions related to the WriteAheadLog interface */
 private[streaming] object WriteAheadLogUtils extends Logging {
-  val RECEIVER_WAL_ENABLE_CONF_KEY = "spark.streaming.receiver.writeAheadLog.enable"
-  val RECEIVER_WAL_CLASS_CONF_KEY = "spark.streaming.receiver.writeAheadLog.class"
-  val RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
-    "spark.streaming.receiver.writeAheadLog.rollingIntervalSecs"
-  val RECEIVER_WAL_MAX_FAILURES_CONF_KEY = "spark.streaming.receiver.writeAheadLog.maxFailures"
-  val RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
-    "spark.streaming.receiver.writeAheadLog.closeFileAfterWrite"
-
-  val DRIVER_WAL_CLASS_CONF_KEY = "spark.streaming.driver.writeAheadLog.class"
-  val DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
-    "spark.streaming.driver.writeAheadLog.rollingIntervalSecs"
-  val DRIVER_WAL_MAX_FAILURES_CONF_KEY = "spark.streaming.driver.writeAheadLog.maxFailures"
-  val DRIVER_WAL_BATCHING_CONF_KEY = "spark.streaming.driver.writeAheadLog.allowBatching"
-  val DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY = "spark.streaming.driver.writeAheadLog.batchingTimeout"
-  val DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
-    "spark.streaming.driver.writeAheadLog.closeFileAfterWrite"
-
-  val DEFAULT_ROLLING_INTERVAL_SECS = 60
-  val DEFAULT_MAX_FAILURES = 3
 
   def enableReceiverLog(conf: SparkConf): Boolean = {
-    conf.getBoolean(RECEIVER_WAL_ENABLE_CONF_KEY, false)
+    conf.get(RECEIVER_WAL_ENABLE_CONF_KEY)
   }
 
   def getRollingIntervalSecs(conf: SparkConf, isDriver: Boolean): Int = {
     if (isDriver) {
-      conf.getInt(DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY, DEFAULT_ROLLING_INTERVAL_SECS)
+      conf.get(DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY)
     } else {
-      conf.getInt(RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY, DEFAULT_ROLLING_INTERVAL_SECS)
+      conf.get(RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY)
     }
   }
 
   def getMaxFailures(conf: SparkConf, isDriver: Boolean): Int = {
     if (isDriver) {
-      conf.getInt(DRIVER_WAL_MAX_FAILURES_CONF_KEY, DEFAULT_MAX_FAILURES)
+      conf.get(DRIVER_WAL_MAX_FAILURES_CONF_KEY)
     } else {
-      conf.getInt(RECEIVER_WAL_MAX_FAILURES_CONF_KEY, DEFAULT_MAX_FAILURES)
+      conf.get(RECEIVER_WAL_MAX_FAILURES_CONF_KEY)
     }
   }
 
   def isBatchingEnabled(conf: SparkConf, isDriver: Boolean): Boolean = {
-    isDriver && conf.getBoolean(DRIVER_WAL_BATCHING_CONF_KEY, defaultValue = true)
+    isDriver && conf.get(DRIVER_WAL_BATCHING_CONF_KEY)
   }
 
   /**
@@ -76,14 +58,14 @@ private[streaming] object WriteAheadLogUtils extends Logging {
    * before we fail the write attempt to unblock receivers.
    */
   def getBatchingTimeout(conf: SparkConf): Long = {
-    conf.getLong(DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY, defaultValue = 5000)
+    conf.get(DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY)
   }
 
   def shouldCloseFileAfterWrite(conf: SparkConf, isDriver: Boolean): Boolean = {
     if (isDriver) {
-      conf.getBoolean(DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY, defaultValue = false)
+      conf.get(DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY)
     } else {
-      conf.getBoolean(RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY, defaultValue = false)
+      conf.get(RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY)
     }
   }
 
@@ -126,9 +108,9 @@ private[streaming] object WriteAheadLogUtils extends Logging {
     ): WriteAheadLog = {
 
     val classNameOption = if (isDriver) {
-      sparkConf.getOption(DRIVER_WAL_CLASS_CONF_KEY)
+      sparkConf.get(DRIVER_WAL_CLASS_CONF_KEY)
     } else {
-      sparkConf.getOption(RECEIVER_WAL_CLASS_CONF_KEY)
+      sparkConf.get(RECEIVER_WAL_CLASS_CONF_KEY)
     }
     val wal = classNameOption.map { className =>
       try {
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
index 5e2ce25c7c441..6b332206e8f6d 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/ReceiverInputDStreamSuite.scala
@@ -22,6 +22,7 @@ import scala.util.Random
 import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.rdd.BlockRDD
 import org.apache.spark.storage.{StorageLevel, StreamBlockId}
+import org.apache.spark.streaming.StreamingConf.RECEIVER_WAL_ENABLE_CONF_KEY
 import org.apache.spark.streaming.dstream.ReceiverInputDStream
 import org.apache.spark.streaming.rdd.WriteAheadLogBackedBlockRDD
 import org.apache.spark.streaming.receiver.{BlockManagerBasedStoreResult, Receiver, WriteAheadLogBasedStoreResult}
@@ -117,7 +118,7 @@ class ReceiverInputDStreamSuite
   private def runTest(enableWAL: Boolean, body: ReceiverInputDStream[_] => Unit): Unit = {
     val conf = new SparkConf()
     conf.setMaster("local[4]").setAppName("ReceiverInputDStreamSuite")
-    conf.set(WriteAheadLogUtils.RECEIVER_WAL_ENABLE_CONF_KEY, enableWAL.toString)
+    conf.set(StreamingConf.RECEIVER_WAL_ENABLE_CONF_KEY.key, enableWAL.toString)
     require(WriteAheadLogUtils.enableReceiverLog(conf) === enableWAL)
     ssc = new StreamingContext(conf, Seconds(1))
     val receiverStream = new ReceiverInputDStream[Int](ssc) {

From c2384558086b0386a20aa8098cdf7a4c823a5f04 Mon Sep 17 00:00:00 2001
From: Eric Wu <492960551@qq.com>
Date: Tue, 10 Mar 2020 19:08:59 +0800
Subject: [PATCH 0255/1280] [SPARK-31079][SQL] Logging QueryExecutionMetering
 in RuleExecutor logger

### What changes were proposed in this pull request?
RuleExecutor already support metering for analyzer/optimizer rules. By providing such information in `PlanChangeLogger`, user can get more information when debugging rule changes .

This PR enhanced `PlanChangeLogger` to display RuleExecutor metrics. This can be easily done by calling the existing API `resetMetrics` and `dumpTimeSpent`, but there might be conflicts if user is also collecting total metrics of a sql job. Thus I introduced `QueryExecutionMetrics`, as the snapshot of `QueryExecutionMetering`, to better support this feature.

Information added to `PlanChangeLogger`
```
=== Metrics of Executed Rules ===
Total number of runs: 554
Total time: 0.107756568 seconds
Total number of effective runs: 11
Total time of effective runs: 0.047615486 seconds
```

### Why are the changes needed?
Provide better plan change debugging user experience

### Does this PR introduce any user-facing change?
Only add more debugging info of `planChangeLog`, default log level is TRACE.

### How was this patch tested?
Update existing tests to verify the new logs

Closes #27846 from Eric5553/ExplainRuleExecMetrics.

Authored-by: Eric Wu <492960551@qq.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 15df2a3f40c74cd3950cc48c95c330217e3ef401)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../rules/QueryExecutionMetering.scala        | 27 +++++++++++++++++++
 .../sql/catalyst/rules/RuleExecutor.scala     | 24 ++++++++++++++++-
 .../optimizer/OptimizerLoggingSuite.scala     |  9 ++++++-
 3 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
index 875c46d2a417a..8efc3593d72f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/QueryExecutionMetering.scala
@@ -37,6 +37,10 @@ case class QueryExecutionMetering() {
     timeEffectiveRunsMap.clear()
   }
 
+  def getMetrics(): QueryExecutionMetrics = {
+    QueryExecutionMetrics(totalTime, totalNumRuns, totalNumEffectiveRuns, totalEffectiveTime)
+  }
+
   def totalTime: Long = {
     timeMap.sum()
   }
@@ -45,6 +49,14 @@ case class QueryExecutionMetering() {
     numRunsMap.sum()
   }
 
+  def totalNumEffectiveRuns: Long = {
+    numEffectiveRunsMap.sum()
+  }
+
+  def totalEffectiveTime: Long = {
+    timeEffectiveRunsMap.sum()
+  }
+
   def incExecutionTimeBy(ruleName: String, delta: Long): Unit = {
     timeMap.addAndGet(ruleName, delta)
   }
@@ -95,3 +107,18 @@ case class QueryExecutionMetering() {
      """.stripMargin
   }
 }
+
+case class QueryExecutionMetrics(
+    time: Long,
+    numRuns: Long,
+    numEffectiveRuns: Long,
+    timeEffective: Long) {
+
+  def -(metrics: QueryExecutionMetrics): QueryExecutionMetrics = {
+    QueryExecutionMetrics(
+      this.time - metrics.time,
+      this.numRuns - metrics.numRuns,
+      this.numEffectiveRuns - metrics.numEffectiveRuns,
+      this.timeEffective - metrics.timeEffective)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
index da5242bee28e1..bff04d317d4d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleExecutor.scala
@@ -21,6 +21,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.QueryPlanningTracker
 import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.trees.TreeNode
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.sideBySide
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.Utils
@@ -37,6 +38,10 @@ object RuleExecutor {
   def resetMetrics(): Unit = {
     queryExecutionMeter.resetMetrics()
   }
+
+  def getCurrentMetrics(): QueryExecutionMetrics = {
+    queryExecutionMeter.getMetrics()
+  }
 }
 
 abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
@@ -121,6 +126,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
     val queryExecutionMetrics = RuleExecutor.queryExecutionMeter
     val planChangeLogger = new PlanChangeLogger()
     val tracker: Option[QueryPlanningTracker] = QueryPlanningTracker.get
+    val beforeMetrics = RuleExecutor.getCurrentMetrics()
 
     // Run the structural integrity checker against the initial input
     if (!isPlanIntegral(plan)) {
@@ -199,6 +205,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
 
       planChangeLogger.logBatch(batch.name, batchStartPlan, curPlan)
     }
+    planChangeLogger.logMetrics(RuleExecutor.getCurrentMetrics() - beforeMetrics)
 
     curPlan
   }
@@ -231,7 +238,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
             s"""
                |=== Result of Batch ${batchName} ===
                |${sideBySide(oldPlan.treeString, newPlan.treeString).mkString("\n")}
-          """.stripMargin
+            """.stripMargin
           } else {
             s"Batch ${batchName} has no effect."
           }
@@ -241,6 +248,21 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
       }
     }
 
+    def logMetrics(metrics: QueryExecutionMetrics): Unit = {
+      val totalTime = metrics.time / NANOS_PER_SECOND.toDouble
+      val totalTimeEffective = metrics.timeEffective / NANOS_PER_SECOND.toDouble
+      val message =
+        s"""
+           |=== Metrics of Executed Rules ===
+           |Total number of runs: ${metrics.numRuns}
+           |Total time: ${totalTime} seconds
+           |Total number of effective runs: ${metrics.numEffectiveRuns}
+           |Total time of effective runs: ${totalTimeEffective} seconds
+        """.stripMargin
+
+      logBasedOnLevel(message)
+    }
+
     private def logBasedOnLevel(f: => String): Unit = {
       logLevel match {
         case "TRACE" => logTrace(f)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
index d3b0a0e7d9347..db22121e08601 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerLoggingSuite.scala
@@ -49,12 +49,19 @@ class OptimizerLoggingSuite extends PlanTest {
       case event => Seq(
         "Applying Rule",
         "Result of Batch",
-        "has no effect").exists(event.getRenderedMessage().contains)
+        "has no effect",
+        "Metrics of Executed Rules").exists(event.getRenderedMessage().contains)
     }
     val logMessages = events.map(_.getRenderedMessage)
     assert(expectedRulesOrBatches.forall
     (ruleOrBatch => logMessages.exists(_.contains(ruleOrBatch))))
     assert(events.forall(_.getLevel == expectedLevel))
+    val expectedMetrics = Seq(
+      "Total number of runs: 7",
+      "Total time:",
+      "Total number of effective runs: 3",
+      "Total time of effective runs:")
+    assert(expectedMetrics.forall(metrics => logMessages.exists(_.contains(metrics))))
   }
 
   test("test log level") {

From 558a82cbd4ee6d463ed5c4460b327340511aa6d4 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 10 Mar 2020 22:08:58 +0800
Subject: [PATCH 0256/1280] [SPARK-30189][SQL] Interval from
 year-month/date-time string should handle whitespaces

### What changes were proposed in this pull request?

Currently, we parse interval from multi units strings or from date-time/year-month pattern strings, the former handles all whitespace, the latter not or even spaces.

### Why are the changes needed?

behavior consistency

### Does this PR introduce any user-facing change?
yes, interval in date-time/year-month like
```
select interval '\n-\t10\t 12:34:46.789\t' day to second
-- !query 126 schema
struct<INTERVAL '-10 days -12 hours -34 minutes -46.789 seconds':interval>
-- !query 126 output
-10 days -12 hours -34 minutes -46.789 seconds
```
is valid now.

### How was this patch tested?

add ut.

Closes #26815 from yaooqinn/SPARK-30189.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3bd6ebff81a46c3bf3664c4be1714c3002d92e85)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/IntervalUtils.scala     |  5 +-
 .../parser/ExpressionParserSuite.scala        |  5 +-
 .../catalyst/util/IntervalUtilsSuite.scala    | 14 ++++++
 .../resources/sql-tests/inputs/interval.sql   |  4 ++
 .../sql-tests/results/ansi/interval.sql.out   | 47 ++++++++++++++++++-
 .../sql-tests/results/interval.sql.out        | 47 ++++++++++++++++++-
 6 files changed, 115 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 0a13ec87c2897..ccf8c5ed3603e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -136,8 +136,7 @@ object IntervalUtils {
             s"Error parsing interval year-month string: ${e.getMessage}", e)
       }
     }
-    assert(input.length == input.trim.length)
-    input match {
+    input.trim match {
       case yearMonthPattern("-", yearStr, monthStr) =>
         negateExact(toInterval(yearStr, monthStr))
       case yearMonthPattern(_, yearStr, monthStr) =>
@@ -300,7 +299,7 @@ object IntervalUtils {
     val regexp = dayTimePattern.get(from -> to)
     require(regexp.isDefined, s"Cannot support (interval '$input' $from to $to) expression")
     val pattern = regexp.get.pattern
-    val m = pattern.matcher(input)
+    val m = pattern.matcher(input.trim)
     require(m.matches, s"Interval string must match day-time format of '$pattern': $input, " +
       s"$fallbackNotice")
     var micros: Long = 0L
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index e8beb612b5134..74fd48d922a3a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -694,7 +694,7 @@ class ExpressionParserSuite extends AnalysisTest {
     intercept("interval 10 nanoseconds", "invalid unit 'nanoseconds'")
 
     // Year-Month intervals.
-    val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0")
+    val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0", "\t -1-2\t")
     yearMonthValues.foreach { value =>
       val result = Literal(IntervalUtils.fromYearMonthString(value))
       checkIntervals(s"'$value' year to month", result)
@@ -707,7 +707,8 @@ class ExpressionParserSuite extends AnalysisTest {
       "10 9:8:7.123456789",
       "1 0:0:0",
       "-1 0:0:0",
-      "1 0:0:1")
+      "1 0:0:1",
+      "\t 1 0:0:1 ")
     datTimeValues.foreach { value =>
       val result = Literal(IntervalUtils.fromDayTimeString(value))
       checkIntervals(s"'$value' day to second", result)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 1628a61418262..3d9372c9b43b7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -137,6 +137,15 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     failFuncWithInvalidInput("99-15", "month 15 outside range", fromYearMonthString)
     failFuncWithInvalidInput("9a9-15", "Interval string does not match year-month format",
       fromYearMonthString)
+
+    // whitespaces
+    assert(fromYearMonthString("99-10 ") === new CalendarInterval(99 * 12 + 10, 0, 0L))
+    assert(fromYearMonthString("+99-10\t") === new CalendarInterval(99 * 12 + 10, 0, 0L))
+    assert(fromYearMonthString("\t\t-8-10\t") === new CalendarInterval(-8 * 12 - 10, 0, 0L))
+    failFuncWithInvalidInput("99\t-15", "Interval string does not match year-month format",
+      fromYearMonthString)
+    failFuncWithInvalidInput("-\t99-15", "Interval string does not match year-month format",
+      fromYearMonthString)
   }
 
   test("from day-time string - legacy") {
@@ -312,6 +321,11 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     checkFail("5 30:12:20", DAY, SECOND, "hour 30 outside range")
     checkFail("5 30-12", DAY, SECOND, "must match day-time format")
     checkFail("5 1:12:20", HOUR, MICROSECOND, "Cannot support (interval")
+
+    // whitespaces
+    check("\t +5 12:40\t ", DAY, MINUTE, "5 days 12 hours 40 minutes")
+    checkFail("+5\t 12:40", DAY, MINUTE, "must match day-time format")
+
   }
 
   test("interval overflow check") {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index 4f26e75fa77f8..fec11b4a9038e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -153,6 +153,10 @@ select interval 'interval \t 1\tday';
 select interval 'interval\t1\tday';
 select interval '1\t' day;
 select interval '1 ' day;
+select interval '2-2\t' year to month;
+select interval '-\t2-2\t' year to month;
+select interval '\n0 12:34:46.789\t' day to second;
+select interval '\n-\t10\t 12:34:46.789\t' day to second;
 
 -- interval overflow if (ansi) exception else NULL
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 4a41dd669e1cc..d4238c73e39e1 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 85
+-- Number of queries: 89
 
 
 -- !query
@@ -804,6 +804,51 @@ struct<INTERVAL '1 days':interval>
 1 days
 
 
+-- !query
+select interval '2-2\t' year to month
+-- !query schema
+struct<INTERVAL '2 years 2 months':interval>
+-- !query output
+2 years 2 months
+
+
+-- !query
+select interval '-\t2-2\t' year to month
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Interval string does not match year-month format of 'y-m': -	2-2	(line 1, pos 16)
+
+== SQL ==
+select interval '-\t2-2\t' year to month
+----------------^^^
+
+
+-- !query
+select interval '\n0 12:34:46.789\t' day to second
+-- !query schema
+struct<INTERVAL '12 hours 34 minutes 46.789 seconds':interval>
+-- !query output
+12 hours 34 minutes 46.789 seconds
+
+
+-- !query
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 
+-	10	 12:34:46.789	, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
+
+== SQL ==
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+----------------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index f1af335f2c78f..7a3dd74b33bb5 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 85
+-- Number of queries: 89
 
 
 -- !query
@@ -783,6 +783,51 @@ struct<INTERVAL '1 days':interval>
 1 days
 
 
+-- !query
+select interval '2-2\t' year to month
+-- !query schema
+struct<INTERVAL '2 years 2 months':interval>
+-- !query output
+2 years 2 months
+
+
+-- !query
+select interval '-\t2-2\t' year to month
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Interval string does not match year-month format of 'y-m': -	2-2	(line 1, pos 16)
+
+== SQL ==
+select interval '-\t2-2\t' year to month
+----------------^^^
+
+
+-- !query
+select interval '\n0 12:34:46.789\t' day to second
+-- !query schema
+struct<INTERVAL '12 hours 34 minutes 46.789 seconds':interval>
+-- !query output
+12 hours 34 minutes 46.789 seconds
+
+
+-- !query
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+requirement failed: Interval string must match day-time format of '^(?<sign>[+|-])?(?<day>\d+) (?<hour>\d{1,2}):(?<minute>\d{1,2}):(?<second>(\d{1,2})(\.(\d{1,9}))?)$': 
+-	10	 12:34:46.789	, set spark.sql.legacy.fromDayTimeString.enabled to true to restore the behavior before Spark 3.0.(line 1, pos 16)
+
+== SQL ==
+select interval '\n-\t10\t 12:34:46.789\t' day to second
+----------------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema

From 57bf23c01b2cffe5011a9d15eb68eff5c28519f4 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 10 Mar 2020 11:09:36 -0700
Subject: [PATCH 0257/1280] [SPARK-31037][SQL][FOLLOW-UP] Replace legacy
 ReduceNumShufflePartitions with CoalesceShufflePartitions in comment

### What changes were proposed in this pull request?

Replace legacy `ReduceNumShufflePartitions` with `CoalesceShufflePartitions` in comment.

### Why are the changes needed?

Rule `ReduceNumShufflePartitions` has renamed to `CoalesceShufflePartitions`, we should update related comment as well.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A.

Closes #27865 from Ngone51/spark_31037_followup.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 34be83e08b6f5313bdd9d165d3e203d06eff677b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/execution/adaptive/AdaptiveSparkPlanExec.scala      | 6 +++---
 .../spark/sql/execution/adaptive/OptimizeSkewedJoin.scala   | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index fc88a7f7798e3..c1486aa057912 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -97,12 +97,12 @@ case class AdaptiveSparkPlanExec(
   @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq(
     ReuseAdaptiveSubquery(conf, context.subqueryCache),
     // Here the 'OptimizeSkewedJoin' rule should be executed
-    // before 'ReduceNumShufflePartitions', as the skewed partition handled
-    // in 'OptimizeSkewedJoin' rule, should be omitted in 'ReduceNumShufflePartitions'.
+    // before 'CoalesceShufflePartitions', as the skewed partition handled
+    // in 'OptimizeSkewedJoin' rule, should be omitted in 'CoalesceShufflePartitions'.
     OptimizeSkewedJoin(conf),
     CoalesceShufflePartitions(conf),
     // The rule of 'OptimizeLocalShuffleReader' need to make use of the 'partitionStartIndices'
-    // in 'ReduceNumShufflePartitions' rule. So it must be after 'ReduceNumShufflePartitions' rule.
+    // in 'CoalesceShufflePartitions' rule. So it must be after 'CoalesceShufflePartitions' rule.
     OptimizeLocalShuffleReader(conf),
     ApplyColumnarRulesAndInsertTransitions(conf, context.session.sessionState.columnarRules),
     CollapseCodegenStages(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index c3bcce41489fb..4387409e75ab3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -52,7 +52,7 @@ import org.apache.spark.sql.internal.SQLConf
  * (L4-1, R4-1), (L4-2, R4-1), (L4-1, R4-2), (L4-2, R4-2)
  *
  * Note that, when this rule is enabled, it also coalesces non-skewed partitions like
- * `ReduceNumShufflePartitions` does.
+ * `CoalesceShufflePartitions` does.
  */
 case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
@@ -191,7 +191,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
       val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
       // This is used to delay the creation of non-skew partitions so that we can potentially
-      // coalesce them like `ReduceNumShufflePartitions` does.
+      // coalesce them like `CoalesceShufflePartitions` does.
       val nonSkewPartitionIndices = mutable.ArrayBuffer.empty[Int]
       val leftSkewDesc = new SkewDesc
       val rightSkewDesc = new SkewDesc

From b8e2cb32cbc75601d6d7a841362676cf2f273bda Mon Sep 17 00:00:00 2001
From: Qianyang Yu <qyu@us.ibm.com>
Date: Wed, 11 Mar 2020 08:47:30 +0900
Subject: [PATCH 0258/1280] [SPARK-30962][SQL][DOC] Documentation for Alter
 table command phase 2

### What changes were proposed in this pull request?

### Why are the changes needed?

Based on [JIRA 30962](https://issues.apache.org/jira/browse/SPARK-30962), we want to add all the support `Alter Table` syntax for V1 table.

### Does this PR introduce any user-facing change?

Yes

### How was this patch tested?

Before:
The documentation looks like
 [Alter Table](https://github.com/apache/spark/pull/25590)

After:
<img width="850" alt="Screen Shot 2020-03-03 at 2 02 23 PM" src="https://user-images.githubusercontent.com/7550280/75824837-168c7e00-5d59-11ea-9751-d1dab0f5a892.png">
<img width="977" alt="Screen Shot 2020-03-03 at 2 02 41 PM" src="https://user-images.githubusercontent.com/7550280/75824859-21dfa980-5d59-11ea-8b49-3adf6eb55fc6.png">
<img width="1028" alt="Screen Shot 2020-03-03 at 2 02 59 PM" src="https://user-images.githubusercontent.com/7550280/75824884-2e640200-5d59-11ea-81ef-d77d0a8efee2.png">
<img width="864" alt="Screen Shot 2020-03-03 at 2 03 14 PM" src="https://user-images.githubusercontent.com/7550280/75824910-39b72d80-5d59-11ea-84d0-bffa2499f086.png">
<img width="823" alt="Screen Shot 2020-03-03 at 2 03 28 PM" src="https://user-images.githubusercontent.com/7550280/75824937-45a2ef80-5d59-11ea-932c-314924856834.png">
<img width="811" alt="Screen Shot 2020-03-03 at 2 03 42 PM" src="https://user-images.githubusercontent.com/7550280/75824965-4cc9fd80-5d59-11ea-815b-8c1ebad310b1.png">
<img width="827" alt="Screen Shot 2020-03-03 at 2 03 53 PM" src="https://user-images.githubusercontent.com/7550280/75824978-518eb180-5d59-11ea-8a55-2fa26376b9c1.png">

<img width="783" alt="Screen Shot 2020-03-03 at 2 04 03 PM" src="https://user-images.githubusercontent.com/7550280/75825001-5bb0b000-5d59-11ea-8dd9-dcfbfa1b4330.png">

Notes:
Those syntaxes are not supported by v1 Table.

- `ALTER TABLE .. RENAME COLUMN`
- `ALTER TABLE ... DROP (COLUMN | COLUMNS)`
- `ALTER TABLE ... (ALTER | CHANGE) COLUMN? alterColumnAction` only support change comments, not other actions: `datatype, position, (SET | DROP) NOT NULL`
- `ALTER TABLE .. CHANGE COLUMN?`
- `ALTER TABLE .... REPLACE COLUMNS`
- `ALTER TABLE ... RECOVER PARTITIONS`
-

Closes #27779 from kevinyu98/spark-30962-alterT.

Authored-by: Qianyang Yu <qyu@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 0f54dc7c03ed975ecb7f776a0151b9325d21e85c)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-syntax-ddl-alter-table.md | 213 ++++++++++++++++++++++++-
 1 file changed, 210 insertions(+), 3 deletions(-)

diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index 373fa8d8940b0..2dd808b131ef2 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -23,14 +23,13 @@ license: |
 `ALTER TABLE` statement changes the schema or properties of a table.
 
 ### RENAME 
-`ALTER TABLE RENAME` statement changes the table name of an existing table in the database.
+`ALTER TABLE RENAME TO` statement changes the table name of an existing table in the database.
 
 #### Syntax
 {% highlight sql %}
 ALTER TABLE table_identifier RENAME TO table_identifier
 
 ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
-
 {% endhighlight %}
 
 #### Parameters
@@ -83,6 +82,109 @@ ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , col_spec ... ] )
 </dl>
 
 
+### ALTER OR CHANGE COLUMN
+`ALTER TABLE ALTER COLUMN` or `ALTER TABLE CHANGE COLUMN` statement changes column's comment.
+
+#### Syntax
+{% highlight sql %}
+ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnAction
+{% endhighlight %}
+
+#### Parameters
+<dl>
+  <dt><code><em>table_identifier</em></code></dt>
+  <dd>
+    Specifies a table name, which may be optionally qualified with a database name.<br><br>
+    <b>Syntax:</b>
+      <code>
+        [ database_name. ] table_name
+      </code>
+  </dd>
+</dl>
+
+<dl>
+  <dt><code><em>COLUMN col_spec</em></code></dt>
+  <dd>Specifies the column to be altered or be changed.</dd>
+</dl>
+
+<dl>
+  <dt><code><em>alterColumnAction</em></code></dt>
+   <dd>
+     Change the comment string.<br><br>
+     <b>Syntax:</b>
+        <code>
+            COMMENT STRING
+        </code>     
+   </dd>     
+</dl>
+
+
+### ADD AND DROP PARTITION
+
+#### ADD PARTITION
+`ALTER TABLE ADD` statement adds partition to the partitioned table.
+
+##### Syntax
+{% highlight sql %}
+ALTER TABLE table_identifier ADD [IF NOT EXISTS] 
+    ( partition_spec [ partition_spec ... ] )
+{% endhighlight %}
+     
+##### Parameters
+<dl>
+  <dt><code><em>table_identifier</em></code></dt>
+  <dd>
+    Specifies a table name, which may be optionally qualified with a database name.<br><br>
+    <b>Syntax:</b>
+      <code>
+        [ database_name. ] table_name
+      </code>
+  </dd>
+</dl>
+
+<dl>
+  <dt><code><em>partition_spec</em></code></dt>
+  <dd>
+    Partition to be added. <br><br>
+    <b>Syntax:</b>
+      <code>
+        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
+      </code>
+  </dd>
+</dl>     
+
+#### DROP PARTITION
+`ALTER TABLE DROP` statement drops the partition of the table.
+
+##### Syntax
+{% highlight sql %}
+ALTER TABLE table_identifier DROP [ IF EXISTS ] partition_spec [PURGE]
+{% endhighlight %}
+     
+##### Parameters
+<dl>
+  <dt><code><em>table_identifier</em></code></dt>
+  <dd>
+    Specifies a table name, which may be optionally qualified with a database name.<br><br>
+    <b>Syntax:</b>
+      <code>
+        [ database_name. ] table_name
+      </code>
+  </dd>
+</dl>
+
+<dl>
+  <dt><code><em>partition_spec</em></code></dt>
+  <dd>
+    Partition to be dropped. <br><br>
+    <b>Syntax:</b>
+      <code>
+        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
+      </code>
+  </dd>
+</dl>     
+
+     
 ### SET AND UNSET
 
 #### SET TABLE PROPERTIES
@@ -217,7 +319,7 @@ SHOW PARTITIONS StudentInfo;
 | age=15     |
 +------------+--+
 
--- Add new column to a table
+-- Add new columns to a table
 
 DESC StudentInfo;
 +--------------------------+------------+----------+--+
@@ -248,6 +350,111 @@ DESC StudentInfo;
 | age                      | int        | NULL     |
 +--------------------------+------------+----------+--+
 
+-- Add a new partition to a table 
+
+SHOW PARTITIONS StudentInfo;
++------------+--+
+| partition  |
++------------+--+
+| age=11     |
+| age=12     |
+| age=15     |
++------------+--+
+
+ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);
+
+-- After adding a new partition to the table
+SHOW PARTITIONS StudentInfo;
++------------+--+
+| partition  |
++------------+--+
+| age=11     |
+| age=12     |
+| age=15     |
+| age=18     |
++------------+--+
+
+-- Drop a partition from the table 
+
+SHOW PARTITIONS StudentInfo;
++------------+--+
+| partition  |
++------------+--+
+| age=11     |
+| age=12     |
+| age=15     |
+| age=18     |
++------------+--+
+
+ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);
+
+-- After dropping the partition of the table
+SHOW PARTITIONS StudentInfo;
++------------+--+
+| partition  |
++------------+--+
+| age=11     |
+| age=12     |
+| age=15     |
++------------+--+
+
+-- Adding multiple partitions to the table
+
+SHOW PARTITIONS StudentInfo;
++------------+--+
+| partition  |
++------------+--+
+| age=11     |
+| age=12     |
+| age=15     |
++------------+--+
+
+ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18) PARTITION (age=20);
+
+-- After adding multiple partitions to the table
+SHOW PARTITIONS StudentInfo;
++------------+--+
+| partition  |
++------------+--+
+| age=11     |
+| age=12     |
+| age=15     |
+| age=18     |
+| age=20     |
++------------+--+
+
+-- ALTER OR CHANGE COLUMNS
+
+DESC StudentInfo;
++--------------------------+------------+----------+--+
+|         col_name         | data_type  | comment  |
++--------------------------+------------+----------+--+
+| name                     | string     | NULL     |
+| rollno                   | int        | NULL     |
+| LastName                 | string     | NULL     |
+| DOB                      | timestamp  | NULL     |
+| age                      | int        | NULL     |
+| # Partition Information  |            |          |
+| # col_name               | data_type  | comment  |
+| age                      | int        | NULL     |
++--------------------------+------------+----------+--+
+
+ALTER TABLE StudentInfo ALTER COLUMN name COMMENT "new comment";
+
+--After ALTER or CHANGE COLUMNS
+DESC StudentInfo;
++--------------------------+------------+------------+--+
+|         col_name         | data_type  | comment    |
++--------------------------+------------+------------+--+
+| name                     | string     | new comment|
+| rollno                   | int        | NULL       |
+| LastName                 | string     | NULL       |
+| DOB                      | timestamp  | NULL       |
+| age                      | int        | NULL       |
+| # Partition Information  |            |            |
+| # col_name               | data_type  | comment    |
+| age                      | int        | NULL       |
++--------------------------+------------+------------+--+
 
 --Change the fileformat
 ALTER TABLE loc_orc SET fileformat orc;

From d1f5df40cb7687c5fd3145d3d629fb1069227638 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Tue, 10 Mar 2020 17:50:34 -0700
Subject: [PATCH 0259/1280] [SPARK-31095][BUILD] Upgrade netty-all to
 4.1.47.Final

### What changes were proposed in this pull request?

This PR aims to bring the bug fixes from the latest netty-all.

### Why are the changes needed?

- 4.1.47.Final: https://github.com/netty/netty/milestone/222?closed=1 (15 patches or issues)
- 4.1.46.Final: https://github.com/netty/netty/milestone/221?closed=1 (80 patches or issues)
- 4.1.45.Final: https://github.com/netty/netty/milestone/220?closed=1 (23 patches or issues)
- 4.1.44.Final: https://github.com/netty/netty/milestone/218?closed=1 (113 patches or issues)
- 4.1.43.Final: https://github.com/netty/netty/milestone/217?closed=1 (63 patches or issues)

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins with the existing tests.

Closes #27869 from dongjoon-hyun/SPARK-31095.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 93def95b0801842e0288a77b3a97f84d31b57366)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +-
 pom.xml                                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 828b1a66d949f..39f7262323f55 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -155,7 +155,7 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.42.Final//netty-all-4.1.42.Final.jar
+netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 8a65540ceee1b..26ac30d6d6be8 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -170,7 +170,7 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-netty-all/4.1.42.Final//netty-all-4.1.42.Final.jar
+netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 4dddbba04405e..e908ec8a355d8 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -183,7 +183,7 @@ metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
 mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
-netty-all/4.1.42.Final//netty-all-4.1.42.Final.jar
+netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 nimbus-jose-jwt/4.41.1//nimbus-jose-jwt-4.41.1.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/2.7.5//okhttp-2.7.5.jar
diff --git a/pom.xml b/pom.xml
index 8a4619743e593..262f3ac0d98df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -698,7 +698,7 @@
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
-        <version>4.1.42.Final</version>
+        <version>4.1.47.Final</version>
       </dependency>
       <dependency>
         <groupId>org.apache.derby</groupId>

From 3f23529cac3a306afe0ed175b8034d4f24b08acb Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 10 Mar 2020 21:50:44 -0700
Subject: [PATCH 0260/1280] [SPARK-31070][SQL] make skew join split skewed
 partitions more evenly

<!--
Thanks for sending a pull request!  Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://spark.apache.org/contributing.html
  2. Ensure you have added or run the appropriate tests for your PR: https://spark.apache.org/developer-tools.html
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][SPARK-XXXX] Your PR title ...'.
  4. Be sure to keep the PR description updated to reflect all changes.
  5. Please write your PR title to summarize what this PR proposes.
  6. If possible, provide a concise example to reproduce the issue for a faster review.
  7. If you want to add a new configuration, please read the guideline first for naming configurations in
     'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
-->

### What changes were proposed in this pull request?
<!--
Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
  1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
  2. If you fix some SQL features, you can provide some references of other DBMSes.
  3. If there is design documentation, please add the link.
  4. If there is a discussion in the mailing list, please add the link.
-->
There are two problems when splitting skewed partitions:
1. It's impossible that we can't split the skewed partition, then we shouldn't create a skew join.
2. When splitting, it's possible that we create a partition for very small amount of data..

This PR fixes them
1. don't create `PartialReducerPartitionSpec` if we can't split.
2. merge small partitions to the previous partition.
### Why are the changes needed?
<!--
Please clarify why the changes are needed. For instance,
  1. If you propose a new API, clarify the use case for a new API.
  2. If you fix a bug, you can clarify why it is a bug.
-->
make skew join split skewed partitions more evenly

### Does this PR introduce any user-facing change?
<!--
If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
If no, write 'No'.
-->
no

### How was this patch tested?
<!--
If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
If tests were not added, please describe why they were not added and/or why it was difficult to add.
-->
updated test

Closes #27833 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit d5f5720efa7232f1339976462d462a7360978ab5)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../adaptive/CoalesceShufflePartitions.scala  |  2 +-
 .../adaptive/OptimizeSkewedJoin.scala         | 44 ++++++----------
 ...scer.scala => ShufflePartitionsUtil.scala} | 50 ++++++++++++++++++-
 ...scala => ShufflePartitionsUtilSuite.scala} | 32 ++++++++++--
 .../adaptive/AdaptiveQueryExecSuite.scala     | 14 +++---
 5 files changed, 102 insertions(+), 40 deletions(-)
 rename sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/{ShufflePartitionsCoalescer.scala => ShufflePartitionsUtil.scala} (73%)
 rename sql/core/src/test/scala/org/apache/spark/sql/execution/{ShufflePartitionsCoalescerSuite.scala => ShufflePartitionsUtilSuite.scala} (88%)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
index a8e2d8ee3c55b..d779a20057ac2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -66,7 +66,7 @@ case class CoalesceShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
       val distinctNumPreShufflePartitions =
         validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
       if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1) {
-        val partitionSpecs = ShufflePartitionsCoalescer.coalescePartitions(
+        val partitionSpecs = ShufflePartitionsUtil.coalescePartitions(
           validMetrics.toArray,
           firstPartitionIndex = 0,
           lastPartitionIndex = distinctNumPreShufflePartitions.head,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 4387409e75ab3..7f523937b632c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.execution.adaptive
 
 import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
 
 import org.apache.commons.io.FileUtils
 
@@ -111,22 +110,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       targetSize: Long): Seq[Int] = {
     val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
     val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
-    val partitionStartIndices = ArrayBuffer[Int]()
-    partitionStartIndices += 0
-    var i = 0
-    var postMapPartitionSize = 0L
-    while (i < mapPartitionSizes.length) {
-      val nextMapPartitionSize = mapPartitionSizes(i)
-      if (i > 0 && postMapPartitionSize + nextMapPartitionSize > targetSize) {
-        partitionStartIndices += i
-        postMapPartitionSize = nextMapPartitionSize
-      } else {
-        postMapPartitionSize += nextMapPartitionSize
-      }
-      i += 1
-    }
-
-    partitionStartIndices
+    ShufflePartitionsUtil.splitSizeListByTargetSize(mapPartitionSizes, targetSize)
   }
 
   private def getStatistics(stage: ShuffleQueryStageExec): MapOutputStatistics = {
@@ -211,21 +195,25 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
           }
 
           val leftParts = if (isLeftSkew) {
-            leftSkewDesc.addPartitionSize(leftSize)
-            createSkewPartitions(
-              partitionIndex,
-              getMapStartIndices(left, partitionIndex, leftTargetSize),
-              getNumMappers(left))
+            val mapStartIndices = getMapStartIndices(left, partitionIndex, leftTargetSize)
+            if (mapStartIndices.length > 1) {
+              leftSkewDesc.addPartitionSize(leftSize)
+              createSkewPartitions(partitionIndex, mapStartIndices, getNumMappers(left))
+            } else {
+              Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
+            }
           } else {
             Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
           }
 
           val rightParts = if (isRightSkew) {
-            rightSkewDesc.addPartitionSize(rightSize)
-            createSkewPartitions(
-              partitionIndex,
-              getMapStartIndices(right, partitionIndex, rightTargetSize),
-              getNumMappers(right))
+            val mapStartIndices = getMapStartIndices(right, partitionIndex, rightTargetSize)
+            if (mapStartIndices.length > 1) {
+              rightSkewDesc.addPartitionSize(rightSize)
+              createSkewPartitions(partitionIndex, mapStartIndices, getNumMappers(right))
+            } else {
+              Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
+            }
           } else {
             Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
           }
@@ -273,7 +261,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     if (!shouldCoalesce || nonSkewPartitionIndices.length == 1) {
       nonSkewPartitionIndices.map(i => CoalescedPartitionSpec(i, i + 1))
     } else {
-      ShufflePartitionsCoalescer.coalescePartitions(
+      ShufflePartitionsUtil.coalescePartitions(
         Array(leftStats, rightStats),
         firstPartitionIndex = nonSkewPartitionIndices.head,
         // `lastPartitionIndex` is exclusive.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
similarity index 73%
rename from sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
index 8c5824178f665..32f5fd41819b2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsCoalescer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
@@ -23,7 +23,9 @@ import org.apache.spark.MapOutputStatistics
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.execution.{CoalescedPartitionSpec, ShufflePartitionSpec}
 
-object ShufflePartitionsCoalescer extends Logging {
+object ShufflePartitionsUtil extends Logging {
+  final val SMALL_PARTITION_FACTOR = 0.2
+  final val MERGED_PARTITION_FACTOR = 1.2
 
   /**
    * Coalesce the same range of partitions (`firstPartitionIndex` to `lastPartitionIndex`, the
@@ -114,4 +116,50 @@ object ShufflePartitionsCoalescer extends Logging {
 
     partitionSpecs
   }
+
+  /**
+   * Given a list of size, return an array of indices to split the list into multiple partitions,
+   * so that the size sum of each partition is close to the target size. Each index indicates the
+   * start of a partition.
+   */
+  def splitSizeListByTargetSize(sizes: Seq[Long], targetSize: Long): Array[Int] = {
+    val partitionStartIndices = ArrayBuffer[Int]()
+    partitionStartIndices += 0
+    var i = 0
+    var currentPartitionSize = 0L
+    var lastPartitionSize = -1L
+
+    def tryMergePartitions() = {
+      // When we are going to start a new partition, it's possible that the current partition or
+      // the previous partition is very small and it's better to merge the current partition into
+      // the previous partition.
+      val shouldMergePartitions = lastPartitionSize > -1 &&
+        ((currentPartitionSize + lastPartitionSize) < targetSize * MERGED_PARTITION_FACTOR ||
+        (currentPartitionSize < targetSize * SMALL_PARTITION_FACTOR ||
+          lastPartitionSize < targetSize * SMALL_PARTITION_FACTOR))
+      if (shouldMergePartitions) {
+        // We decide to merge the current partition into the previous one, so the start index of
+        // the current partition should be removed.
+        partitionStartIndices.remove(partitionStartIndices.length - 1)
+        lastPartitionSize += currentPartitionSize
+      } else {
+        lastPartitionSize = currentPartitionSize
+      }
+    }
+
+    while (i < sizes.length) {
+      // If including the next size in the current partition exceeds the target size, package the
+      // current partition and start a new partition.
+      if (i > 0 && currentPartitionSize + sizes(i) > targetSize) {
+        tryMergePartitions()
+        partitionStartIndices += i
+        currentPartitionSize = sizes(i)
+      } else {
+        currentPartitionSize += sizes(i)
+      }
+      i += 1
+    }
+    tryMergePartitions()
+    partitionStartIndices.toArray
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
similarity index 88%
rename from sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
rename to sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
index 8aab2995467c2..2cd4c98716c28 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsCoalescerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
@@ -18,9 +18,9 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.{MapOutputStatistics, SparkFunSuite}
-import org.apache.spark.sql.execution.adaptive.ShufflePartitionsCoalescer
+import org.apache.spark.sql.execution.adaptive.ShufflePartitionsUtil
 
-class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
+class ShufflePartitionsUtilSuite extends SparkFunSuite {
 
   private def checkEstimation(
       bytesByPartitionIdArray: Array[Array[Long]],
@@ -31,7 +31,7 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
       case (bytesByPartitionId, index) =>
         new MapOutputStatistics(index, bytesByPartitionId)
     }
-    val estimatedPartitionStartIndices = ShufflePartitionsCoalescer.coalescePartitions(
+    val estimatedPartitionStartIndices = ShufflePartitionsUtil.coalescePartitions(
       mapOutputStatistics,
       0,
       bytesByPartitionIdArray.head.length,
@@ -252,4 +252,30 @@ class ShufflePartitionsCoalescerSuite extends SparkFunSuite {
         targetSize, minNumPartitions)
     }
   }
+
+  test("splitSizeListByTargetSize") {
+    val targetSize = 100
+
+    // merge the small partitions at the beginning/end
+    val sizeList1 = Seq[Long](15, 90, 15, 15, 15, 90, 15)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList1, targetSize).toSeq ==
+      Seq(0, 2, 5))
+
+    // merge the small partitions in the middle
+    val sizeList2 = Seq[Long](30, 15, 90, 10, 90, 15, 30)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList2, targetSize).toSeq ==
+      Seq(0, 2, 4, 5))
+
+    // merge small partitions if the partition itself is smaller than
+    // targetSize * SMALL_PARTITION_FACTOR
+    val sizeList3 = Seq[Long](15, 1000, 15, 1000)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList3, targetSize).toSeq ==
+      Seq(0, 3))
+
+    // merge small partitions if the combined size is smaller than
+    // targetSize * MERGED_PARTITION_FACTOR
+    val sizeList4 = Seq[Long](35, 75, 90, 20, 35, 25, 35)
+    assert(ShufflePartitionsUtil.splitSizeListByTargetSize(sizeList4, targetSize).toSeq ==
+      Seq(0, 2, 3))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 94947a85f8fb4..fcca23d7751f0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -645,11 +645,11 @@ class AdaptiveQueryExecSuite
         //              into 2 splits and right side is divided into 4 splits, so
         //              2 x 4 sub-partitions.
         // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, and divide into 3 splits, so
-        //              3 sub-partitions.
+        // Partition 4: only left side is skewed, and divide into 2 splits, so
+        //              2 sub-partitions.
         // So total (8 + 1 + 3) partitions.
         val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        checkSkewJoin(innerSmj, 8 + 1 + 3)
+        checkSkewJoin(innerSmj, 8 + 1 + 2)
 
         // skewed left outer join optimization
         val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
@@ -659,11 +659,11 @@ class AdaptiveQueryExecSuite
         // Partition 0: both left and right sides are skewed, but left join can't split right side,
         //              so only left side is divided into 2 splits, and thus 2 sub-partitions.
         // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, and divide into 3 splits, so
-        //              3 sub-partitions.
-        // So total (2 + 1 + 3) partitions.
+        // Partition 4: only left side is skewed, and divide into 2 splits, so
+        //              2 sub-partitions.
+        // So total (2 + 1 + 2) partitions.
         val leftSmj = findTopLevelSortMergeJoin(leftAdaptivePlan)
-        checkSkewJoin(leftSmj, 2 + 1 + 3)
+        checkSkewJoin(leftSmj, 2 + 1 + 2)
 
         // skewed right outer join optimization
         val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(

From 202fe11fa1d182fb86c495cada9c9568a5be6019 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Wed, 11 Mar 2020 14:11:13 +0800
Subject: [PATCH 0261/1280] [SPARK-31030][SQL] Backward Compatibility for
 Parsing and formatting Datetime

In Spark version 2.4 and earlier, datetime parsing, formatting and conversion are performed by using the hybrid calendar (Julian + Gregorian).
Since the Proleptic Gregorian calendar is de-facto calendar worldwide, as well as the chosen one in ANSI SQL standard, Spark 3.0 switches to it by using Java 8 API classes (the java.time packages that are based on ISO chronology ). The switching job is completed in SPARK-26651.
But after the switching, there are some patterns not compatible between Java 8 and Java 7, Spark needs its own definition on the patterns rather than depends on Java API.
In this PR, we achieve this by writing the document and shadow the incompatible letters. See more details in [SPARK-31030](https://issues.apache.org/jira/browse/SPARK-31030)

For backward compatibility.

No.
After we define our own datetime parsing and formatting patterns, it's same to old Spark version.

Existing and new added UT.
Locally document test:
![image](https://user-images.githubusercontent.com/4833765/76064100-f6acc280-5fc3-11ea-9ef7-82e7dc074205.png)

Closes #27830 from xuanyuanking/SPARK-31030.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3493162c78822de0563f7d736040aebdb81e936b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 R/pkg/R/functions.R                           |   4 +-
 docs/_data/menu-sql.yaml                      |   2 +
 docs/sql-ref-datetime-pattern.md              | 220 ++++++++++++++++++
 python/pyspark/sql/functions.py               |   6 +-
 python/pyspark/sql/readwriter.py              |  16 +-
 python/pyspark/sql/streaming.py               |   8 +-
 .../sql/catalyst/catalog/interface.scala      |   2 +-
 .../expressions/datetimeExpressions.scala     |   6 +-
 .../sql/catalyst/util/DateFormatter.scala     |   4 +-
 .../util/DateTimeFormatterHelper.scala        |   5 +-
 .../sql/catalyst/util/DateTimeUtils.scala     |  62 ++++-
 .../catalyst/util/TimestampFormatter.scala    |   4 +-
 .../catalyst/util/DateTimeUtilsSuite.scala    |  18 +-
 .../apache/spark/sql/DataFrameReader.scala    |   8 +-
 .../apache/spark/sql/DataFrameWriter.scala    |   8 +-
 .../datasources/PartitioningUtils.scala       |   2 +-
 .../org/apache/spark/sql/functions.scala      |  18 +-
 .../sql/streaming/DataStreamReader.scala      |   8 +-
 .../native/stringCastAndExpressions.sql.out   |   6 +-
 19 files changed, 341 insertions(+), 66 deletions(-)
 create mode 100644 docs/sql-ref-datetime-pattern.md

diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index 48f69d5769620..89c7cbedc641a 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -2776,7 +2776,7 @@ setMethod("format_string", signature(format = "character", x = "Column"),
 #' head(tmp)}
 #' @note from_unixtime since 1.5.0
 setMethod("from_unixtime", signature(x = "Column"),
-          function(x, format = "uuuu-MM-dd HH:mm:ss") {
+          function(x, format = "yyyy-MM-dd HH:mm:ss") {
             jc <- callJStatic("org.apache.spark.sql.functions",
                               "from_unixtime",
                               x@jc, format)
@@ -3062,7 +3062,7 @@ setMethod("unix_timestamp", signature(x = "Column", format = "missing"),
 #' @aliases unix_timestamp,Column,character-method
 #' @note unix_timestamp(Column, character) since 1.5.0
 setMethod("unix_timestamp", signature(x = "Column", format = "character"),
-          function(x, format = "uuuu-MM-dd HH:mm:ss") {
+          function(x, format = "yyyy-MM-dd HH:mm:ss") {
             jc <- callJStatic("org.apache.spark.sql.functions", "unix_timestamp", x@jc, format)
             column(jc)
           })
diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 38a5cf61245a6..c17bfd371f717 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -223,3 +223,5 @@
                   url: sql-ref-syntax-aux-resource-mgmt-list-file.html
                 - text: LIST JAR
                   url: sql-ref-syntax-aux-resource-mgmt-list-jar.html
+    - text: Datetime Pattern
+      url: sql-ref-datetime-pattern.html
diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
new file mode 100644
index 0000000000000..429d781248ef2
--- /dev/null
+++ b/docs/sql-ref-datetime-pattern.md
@@ -0,0 +1,220 @@
+---
+layout: global
+title: Datetime patterns
+displayTitle: Datetime Patterns for Formatting and Parsing
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+There are several common scenarios for datetime usage in Spark:
+
+- CSV/JSON datasources use the pattern string for parsing and formatting time content.
+
+- Datetime functions related to convert string to/from `DateType` or `TimestampType`. For example, unix_timestamp, date_format, to_unix_timestamp, from_unixtime, to_date, to_timestamp, from_utc_timestamp, to_utc_timestamp, etc.
+
+Spark uses the below letters in date and timestamp parsing and formatting:
+<table class="table">
+<tr>
+  <th> <b>Symbol</b> </th>
+  <th> <b>Meaning</b> </th>
+  <th> <b>Presentation</b> </th>
+  <th> <b>Examples</b> </th>
+</tr>
+<tr>
+  <td> <b>G</b> </td>
+  <td> era </td>
+  <td> text </td>
+  <td> AD; Anno Domini; A </td>
+</tr>
+<tr>
+  <td> <b>y</b> </td>
+  <td> year </td>
+  <td> year </td>
+  <td> 2020; 20 </td>
+</tr>
+<tr>
+  <td> <b>D</b> </td>
+  <td> day-of-year </td>
+  <td> number </td>
+  <td> 189 </td>
+</tr>
+<tr>
+  <td> <b>M</b> </td>
+  <td> month-of-year </td>
+  <td> number/text </td>
+  <td> 7; 07; Jul; July; J </td>
+</tr>
+<tr>
+  <td> <b>d</b> </td>
+  <td> day-of-month </td>
+  <td> number </td>
+  <td> 28 </td>
+</tr>
+<tr>
+  <td> <b>Y</b> </td>
+  <td> week-based-year </td>
+  <td> year </td>
+  <td> 1996; 96 </td>
+</tr>
+<tr>
+  <td> <b>w</b> </td>
+  <td> week-of-week-based-year </td>
+  <td> number </td>
+  <td> 27 </td>
+</tr>
+<tr>
+  <td> <b>W</b> </td>
+  <td> week-of-month </td>
+  <td> number </td>
+  <td> 4 </td>
+</tr>
+<tr>
+  <td> <b>E</b> </td>
+  <td> day-of-week </td>
+  <td> text </td>
+  <td> Tue; Tuesday; T </td>
+</tr>
+<tr>
+  <td> <b>e</b> </td>
+  <td> localized day-of-week </td>
+  <td> number/text </td>
+  <td> 2; 02; Tue; Tuesday; T </td>
+</tr>
+<tr>
+  <td> <b>F</b> </td>
+  <td> week-of-month </td>
+  <td> number </td>
+  <td> 3 </td>
+</tr>
+<tr>
+  <td> <b>a</b> </td>
+  <td> am-pm-of-day </td>
+  <td> text </td>
+  <td> PM </td>
+</tr>
+<tr>
+  <td> <b>h</b> </td>
+  <td> clock-hour-of-am-pm (1-12) </td>
+  <td> number </td>
+  <td> 12 </td>
+</tr>
+<tr>
+  <td> <b>K</b> </td>
+  <td> hour-of-am-pm (0-11) </td>
+  <td> number </td>
+  <td> 0 </td>
+</tr>
+<tr>
+  <td> <b>k</b> </td>
+  <td> clock-hour-of-day (1-24) </td>
+  <td> number </td>
+  <td> 0 </td>
+</tr>
+<tr>
+  <td> <b>H</b> </td>
+  <td> hour-of-day (0-23)  </td>
+  <td> number </td>
+  <td> 0 </td>
+</tr>
+<tr>
+  <td> <b>m</b> </td>
+  <td> minute-of-hour </td>
+  <td> number </td>
+  <td> 30 </td>
+</tr>
+<tr>
+  <td> <b>s</b> </td>
+  <td> second-of-minute </td>
+  <td> number </td>
+  <td> 55 </td>
+</tr>
+<tr>
+  <td> <b>S</b> </td>
+  <td> fraction-of-second </td>
+  <td> fraction </td>
+  <td> 978 </td>
+</tr>
+<tr>
+  <td> <b>z</b> </td>
+  <td> time-zone name </td>
+  <td> zone-name </td>
+  <td> Pacific Standard Time; PST </td>
+</tr>
+<tr>
+  <td> <b>O</b> </td>
+  <td> localized zone-offset </td>
+  <td> offset-O </td>
+  <td> GMT+8; GMT+08:00; UTC-08:00; </td>
+</tr>
+<tr>
+  <td> <b>X</b> </td>
+  <td> zone-offset 'Z' for zero </td>
+  <td> offset-X </td>
+  <td> Z; -08; -0830; -08:30; -083015; -08:30:15; </td>
+</tr>
+<tr>
+  <td> <b>x</b> </td>
+  <td> zone-offset </td>
+  <td> offset-x </td>
+  <td> +0000; -08; -0830; -08:30; -083015; -08:30:15; </td>
+</tr>
+<tr>
+  <td> <b>Z</b> </td>
+  <td> zone-offset </td>
+  <td> offset-Z </td>
+  <td> +0000; -0800; -08:00; </td>
+</tr>
+<tr>
+  <td> <b>'</b> </td>
+  <td> escape for text </td>
+  <td> delimiter </td>
+  <td></td>
+</tr>
+<tr>
+  <td> <b>''</b> </td>
+  <td> single quote </td>
+  <td> literal </td>
+  <td> ' </td>
+</tr>
+</table>
+
+The count of pattern letters determines the format.
+
+- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short form. Exactly 4 pattern letters will use the full form. Exactly 5 pattern letters will use the narrow form. Six or more letters will fail.
+
+- Number: If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary. The following pattern letters have constraints on the count of letters. Only one letter 'F' can be specified. Up to two letters of 'd', 'H', 'h', 'K', 'k', 'm', and 's' can be specified. Up to three letters of 'D' can be specified.
+
+- Number/Text: If the count of pattern letters is 3 or greater, use the Text rules above. Otherwise use the Number rules above.
+
+- Fraction: Outputs the micro-of-second field as a fraction-of-second. The micro-of-second value has six digits, thus the count of pattern letters is from 1 to 6. If it is less than 6, then the micro-of-second value is truncated, with only the most significant digits being output.
+
+- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
+
+- Zone names: This outputs the display name of the time-zone ID. If the count of letters is one, two or three, then the short name is output. If the count of letters is four, then the full name is output. Five or more letters will fail.
+
+- Offset X and x: This formats the offset based on the number of pattern letters. One letter outputs just the hour, such as '+01', unless the minute is non-zero in which case the minute is also output, such as '+0130'. Two letters outputs the hour and minute, without a colon, such as '+0130'. Three letters outputs the hour and minute, with a colon, such as '+01:30'. Four letters outputs the hour and minute and optional second, without a colon, such as '+013015'. Five letters outputs the hour and minute and optional second, with a colon, such as '+01:30:15'. Six or more letters will fail. Pattern letter 'X' (upper case) will output 'Z' when the offset to be output would be zero, whereas pattern letter 'x' (lower case) will output '+00', '+0000', or '+00:00'.
+
+- Offset O: This formats the localized offset based on the number of pattern letters. One letter outputs the short form of the localized offset, which is localized offset text, such as 'GMT', with hour without leading zero, optional 2-digit minute and second if non-zero, and colon, for example 'GMT+8'. Four letters outputs the full form, which is localized offset text, such as 'GMT, with 2-digit hour and minute field, optional second field if non-zero, and colon, for example 'GMT+08:00'. Any other count of letters will fail.
+
+- Offset Z: This formats the offset based on the number of pattern letters. One, two or three letters outputs the hour and minute, without a colon, such as '+0130'. The output will be '+0000' when the offset is zero. Four letters outputs the full form of localized offset, equivalent to four letters of Offset-O. The output will be the corresponding localized offset text if the offset is zero. Five letters outputs the hour, minute, with optional second if non-zero, with colon. It outputs 'Z' if the offset is zero. Six or more letters will fail.
+
+More details for the text style:
+
+- Short Form: Short text, typically an abbreviation. For example, day-of-week Monday might output "Mon".
+
+- Full Form: Full text, typically the full description. For example, day-of-week Monday might output "Monday".
+
+- Narrow Form: Narrow text, typically a single letter. For example, day-of-week Monday might output "M".
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 5bf8165af6a44..c23cc2bb2d4e4 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -1249,7 +1249,7 @@ def last_day(date):
 
 @ignore_unicode_prefix
 @since(1.5)
-def from_unixtime(timestamp, format="uuuu-MM-dd HH:mm:ss"):
+def from_unixtime(timestamp, format="yyyy-MM-dd HH:mm:ss"):
     """
     Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
     representing the timestamp of that moment in the current system time zone in the given
@@ -1266,9 +1266,9 @@ def from_unixtime(timestamp, format="uuuu-MM-dd HH:mm:ss"):
 
 
 @since(1.5)
-def unix_timestamp(timestamp=None, format='uuuu-MM-dd HH:mm:ss'):
+def unix_timestamp(timestamp=None, format='yyyy-MM-dd HH:mm:ss'):
     """
-    Convert time string with given pattern ('uuuu-MM-dd HH:mm:ss', by default)
+    Convert time string with given pattern ('yyyy-MM-dd HH:mm:ss', by default)
     to Unix time stamp (in seconds), using the default timezone and the default
     locale, return null if fail.
 
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 2db25875621ff..5904688e087a7 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -223,12 +223,12 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param dateFormat: sets the string that indicates a date format. Custom date formats
                            follow the formats at ``java.time.format.DateTimeFormatter``. This
                            applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at
                                 ``java.time.format.DateTimeFormatter``.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
                           set, it uses the default value, ``false``.
         :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
@@ -432,12 +432,12 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param dateFormat: sets the string that indicates a date format. Custom date formats
                            follow the formats at ``java.time.format.DateTimeFormatter``. This
                            applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at
                                 ``java.time.format.DateTimeFormatter``.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
                            set, it uses the default value, ``20480``.
         :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
@@ -852,12 +852,12 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
         :param dateFormat: sets the string that indicates a date format. Custom date formats
                            follow the formats at ``java.time.format.DateTimeFormatter``. This
                            applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at
                                 ``java.time.format.DateTimeFormatter``.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param encoding: specifies encoding (charset) of saved json files. If None is set,
                         the default UTF-8 charset will be used.
         :param lineSep: defines the line separator that should be used for writing. If None is
@@ -957,12 +957,12 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
         :param dateFormat: sets the string that indicates a date format. Custom date formats
                            follow the formats at ``java.time.format.DateTimeFormatter``. This
                            applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at
                                 ``java.time.format.DateTimeFormatter``.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
                                         values being written should be skipped. If None is set, it
                                         uses the default value, ``true``.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index f989cb3e133f1..6a7624f2fbf41 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -461,12 +461,12 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param dateFormat: sets the string that indicates a date format. Custom date formats
                            follow the formats at ``java.time.format.DateTimeFormatter``. This
                            applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at
                                 ``java.time.format.DateTimeFormatter``.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
                           set, it uses the default value, ``false``.
         :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
@@ -673,12 +673,12 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param dateFormat: sets the string that indicates a date format. Custom date formats
                            follow the formats at ``java.time.format.DateTimeFormatter``. This
                            applies to date type. If None is set, it uses the
-                           default value, ``uuuu-MM-dd``.
+                           default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at
                                 ``java.time.format.DateTimeFormatter``.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``uuuu-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
                            set, it uses the default value, ``20480``.
         :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 81561c53f83c1..6e965ef807d1c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -522,7 +522,7 @@ object CatalogColumnStat extends Logging {
   val VERSION = 2
 
   private def getTimestampFormatter(): TimestampFormatter = {
-    TimestampFormatter(format = "uuuu-MM-dd HH:mm:ss.SSSSSS", zoneId = ZoneOffset.UTC)
+    TimestampFormatter(format = "yyyy-MM-dd HH:mm:ss.SSSSSS", zoneId = ZoneOffset.UTC)
   }
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index d92d7000eace3..1dad440b59381 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -671,7 +671,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
     Arguments:
       * timeExp - A date/timestamp or string which is returned as a UNIX timestamp.
       * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "uuuu-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
+                 Default value is "yyyy-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
                  for valid date and time format patterns.
   """,
   examples = """
@@ -705,7 +705,7 @@ case class ToUnixTimestamp(
  * Converts time string with given pattern to Unix time stamp (in seconds), returns null if fail.
  * See [https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html].
  * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null.
- * If the second parameter is missing, use "uuuu-MM-dd HH:mm:ss".
+ * If the second parameter is missing, use "yyyy-MM-dd HH:mm:ss".
  * If no parameters provided, the first parameter will be current_timestamp.
  * If the first parameter is a Date or Timestamp instead of String, we will ignore the
  * second parameter.
@@ -716,7 +716,7 @@ case class ToUnixTimestamp(
     Arguments:
       * timeExp - A date/timestamp or string. If not provided, this defaults to current time.
       * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "uuuu-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
+                 Default value is "yyyy-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
                  for valid date and time format patterns.
   """,
   examples = """
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index caf58c1011f2e..d2e4e8b367a0a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -95,9 +95,7 @@ object DateFormatter {
 
   val defaultLocale: Locale = Locale.US
 
-  def defaultPattern(): String = {
-    if (SQLConf.get.legacyTimeParserPolicy == LEGACY) "yyyy-MM-dd" else "uuuu-MM-dd"
-  }
+  val defaultPattern: String = "yyyy-MM-dd"
 
   private def getFormatter(
       format: Option[String],
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 33aa733f26459..fab504174783e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -52,10 +52,11 @@ trait DateTimeFormatterHelper {
   // less synchronised.
   // The Cache.get method is not used here to avoid creation of additional instances of Callable.
   protected def getOrCreateFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
-    val key = (pattern, locale)
+    val newPattern = DateTimeUtils.convertIncompatiblePattern(pattern)
+    val key = (newPattern, locale)
     var formatter = cache.getIfPresent(key)
     if (formatter == null) {
-      formatter = buildFormatter(pattern, locale)
+      formatter = buildFormatter(newPattern, locale)
       cache.put(key, formatter)
     }
     formatter
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 6811bfc5e0def..2427a71676314 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -819,9 +819,9 @@ object DateTimeUtils {
 
   /**
    * Extracts special values from an input string ignoring case.
-   * @param input - a trimmed string
-   * @param zoneId - zone identifier used to get the current date.
-   * @return some special value in lower case or None.
+   * @param input A trimmed string
+   * @param zoneId Zone identifier used to get the current date.
+   * @return Some special value in lower case or None.
    */
   private def extractSpecialValue(input: String, zoneId: ZoneId): Option[String] = {
     def isValid(value: String, timeZoneId: String): Boolean = {
@@ -848,9 +848,9 @@ object DateTimeUtils {
 
   /**
    * Converts notational shorthands that are converted to ordinary timestamps.
-   * @param input - a trimmed string
-   * @param zoneId - zone identifier used to get the current date.
-   * @return some of microseconds since the epoch if the conversion completed
+   * @param input A trimmed string
+   * @param zoneId Zone identifier used to get the current date.
+   * @return Some of microseconds since the epoch if the conversion completed
    *         successfully otherwise None.
    */
   def convertSpecialTimestamp(input: String, zoneId: ZoneId): Option[SQLTimestamp] = {
@@ -874,9 +874,9 @@ object DateTimeUtils {
 
   /**
    * Converts notational shorthands that are converted to ordinary dates.
-   * @param input - a trimmed string
-   * @param zoneId - zone identifier used to get the current date.
-   * @return some of days since the epoch if the conversion completed successfully otherwise None.
+   * @param input A trimmed string
+   * @param zoneId Zone identifier used to get the current date.
+   * @return Some of days since the epoch if the conversion completed successfully otherwise None.
    */
   def convertSpecialDate(input: String, zoneId: ZoneId): Option[SQLDate] = {
     extractSpecialValue(input, zoneId).flatMap {
@@ -898,9 +898,9 @@ object DateTimeUtils {
 
   /**
    * Subtracts two dates.
-   * @param endDate - the end date, exclusive
-   * @param startDate - the start date, inclusive
-   * @return an interval between two dates. The interval can be negative
+   * @param endDate The end date, exclusive
+   * @param startDate The start date, inclusive
+   * @return An interval between two dates. The interval can be negative
    *         if the end date is before the start date.
    */
   def subtractDates(endDate: SQLDate, startDate: SQLDate): CalendarInterval = {
@@ -911,4 +911,42 @@ object DateTimeUtils {
     val days = period.getDays
     new CalendarInterval(months, days, 0)
   }
+
+  /**
+   * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
+   * parsing/formatting datetime values. The pattern string is incompatible with the one defined
+   * by SimpleDateFormat in Spark 2.4 and earlier. This function converts all incompatible pattern
+   * for the new parser in Spark 3.0. See more details in SPARK-31030.
+   * @param pattern The input pattern.
+   * @return The pattern for new parser
+   */
+  def convertIncompatiblePattern(pattern: String): String = {
+    val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
+      case (patternPart, index) =>
+        // Text can be quoted using single quotes, we only check the non-quote parts.
+        index % 2 == 0 && patternPart.contains("G")
+    }
+    pattern.split("'").zipWithIndex.map {
+      case (patternPart, index) =>
+        if (index % 2 == 0) {
+          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
+          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
+          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
+          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
+          // exception and ask users to change the pattern strings or turn on the legacy mode;
+          // otherwise, return NULL as what Spark 2.4 does.
+          val res = patternPart.replace("u", "e")
+          // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
+          // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
+          // We only do this substitution when there is no era designator found in the pattern.
+          if (!eraDesignatorContained) {
+            res.replace("y", "u")
+          } else {
+            res
+          }
+        } else {
+          patternPart
+        }
+    }.mkString("'")
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 866f81bfff4fb..99139ca8cd7f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -87,7 +87,7 @@ class Iso8601TimestampFormatter(
 }
 
 /**
- * The formatter parses/formats timestamps according to the pattern `uuuu-MM-dd HH:mm:ss.[..fff..]`
+ * The formatter parses/formats timestamps according to the pattern `yyyy-MM-dd HH:mm:ss.[..fff..]`
  * where `[..fff..]` is a fraction of second up to microsecond resolution. The formatter does not
  * output trailing zeros in the fraction. For example, the timestamp `2019-03-05 15:00:01.123400` is
  * formatted as the string `2019-03-05 15:00:01.1234`.
@@ -193,7 +193,7 @@ object TimestampFormatter {
 
   val defaultLocale: Locale = Locale.US
 
-  def defaultPattern(): String = s"${DateFormatter.defaultPattern()} HH:mm:ss"
+  def defaultPattern(): String = s"${DateFormatter.defaultPattern} HH:mm:ss"
 
   private def getFormatter(
       format: Option[String],
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 7685b62b618ab..d98cd0a1e785c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -89,7 +89,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
   test("SPARK-6785: java date conversion before and after epoch") {
     def format(d: Date): String = {
-      TimestampFormatter("uuuu-MM-dd", defaultTimeZone().toZoneId).format(fromMillis(d.getTime))
+      TimestampFormatter("yyyy-MM-dd", defaultTimeZone().toZoneId).format(fromMillis(d.getTime))
     }
     def checkFromToJavaDate(d1: Date): Unit = {
       val d2 = toJavaDate(fromJavaDate(d1))
@@ -667,4 +667,20 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       assert(toDate("tomorrow CET ", zoneId).get === today + 1)
     }
   }
+
+  test("check incompatible pattern") {
+    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
+    assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
+      === "uuuu-MM'y contains in quoted text'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
+      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
+    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
+      === "uuuu-MM'u contains in quoted text'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
+      === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+      === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 6cce7203127f5..9416126780506 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -390,10 +390,10 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`columnNameOfCorruptRecord` (default is the value specified in
    * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
    * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
    * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
    * <li>`multiLine` (default `false`): parse one record, which may span multiple lines,
@@ -615,10 +615,10 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * value.</li>
    * <li>`negativeInf` (default `-Inf`): sets the string representation of a negative infinity
    * value.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
    * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
    * <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index fff1f4b636dea..22b26ca8164cf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -748,10 +748,10 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`compression` (default `null`): compression codec to use when saving to file. This can be
    * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
    * `snappy` and `deflate`). </li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
    * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
    * <li>`encoding` (by default it is not set): specifies encoding (charset) of saved json
@@ -869,10 +869,10 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`compression` (default `null`): compression codec to use when saving to file. This can be
    * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
    * `snappy` and `deflate`). </li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
    * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
    * <li>`ignoreLeadingWhiteSpace` (default `true`): a flag indicating whether or not leading
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index fdad43b23c5aa..b0ec24e9b1a3d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -60,7 +60,7 @@ object PartitionSpec {
 
 object PartitioningUtils {
 
-  val timestampPartitionPattern = "uuuu-MM-dd HH:mm:ss[.S]"
+  val timestampPartitionPattern = "yyyy-MM-dd HH:mm:ss[.S]"
 
   private[datasources] case class PartitionValues(columnNames: Seq[String], literals: Seq[Literal])
   {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 7a589573dd7f1..f280ec3c22e29 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2635,8 +2635,8 @@ object functions {
    * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
    *
    * @param dateExpr A date, timestamp or string. If a string, the data must be in a format that
-   *                 can be cast to a timestamp, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
-   * @param format A pattern `dd.MM.uuuu` would return a string like `18.03.1993`
+   *                 can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
+   * @param format A pattern `dd.MM.yyyy` would return a string like `18.03.1993`
    * @return A string, or null if `dateExpr` was a string that could not be cast to a timestamp
    * @note Use specialized functions like [[year]] whenever possible as they benefit from a
    * specialized implementation.
@@ -2873,7 +2873,7 @@ object functions {
   /**
    * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
    * representing the timestamp of that moment in the current system time zone in the
-   * uuuu-MM-dd HH:mm:ss format.
+   * yyyy-MM-dd HH:mm:ss format.
    *
    * @param ut A number of a type that is castable to a long, such as string or integer. Can be
    *           negative for timestamps before the unix epoch
@@ -2918,11 +2918,11 @@ object functions {
   }
 
   /**
-   * Converts time string in format uuuu-MM-dd HH:mm:ss to Unix timestamp (in seconds),
+   * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds),
    * using the default timezone and the default locale.
    *
    * @param s A date, timestamp or string. If a string, the data must be in the
-   *          `uuuu-MM-dd HH:mm:ss` format
+   *          `yyyy-MM-dd HH:mm:ss` format
    * @return A long, or null if the input was a string not of the correct format
    * @group datetime_funcs
    * @since 1.5.0
@@ -2937,7 +2937,7 @@ object functions {
    * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
    *
    * @param s A date, timestamp or string. If a string, the data must be in a format that can be
-   *          cast to a date, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *          cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param p A date time pattern detailing the format of `s` when `s` is a string
    * @return A long, or null if `s` was a string that could not be cast to a date or `p` was
    *         an invalid format
@@ -2950,7 +2950,7 @@ object functions {
    * Converts to a timestamp by casting rules to `TimestampType`.
    *
    * @param s A date, timestamp or string. If a string, the data must be in a format that can be
-   *          cast to a timestamp, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *          cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @return A timestamp, or null if the input was a string that could not be cast to a timestamp
    * @group datetime_funcs
    * @since 2.2.0
@@ -2965,7 +2965,7 @@ object functions {
    * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
    *
    * @param s   A date, timestamp or string. If a string, the data must be in a format that can be
-   *            cast to a timestamp, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *            cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param fmt A date time pattern detailing the format of `s` when `s` is a string
    * @return A timestamp, or null if `s` was a string that could not be cast to a timestamp or
    *         `fmt` was an invalid format
@@ -2990,7 +2990,7 @@ object functions {
    * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
    *
    * @param e   A date, timestamp or string. If a string, the data must be in a format that can be
-   *            cast to a date, such as `uuuu-MM-dd` or `uuuu-MM-dd HH:mm:ss.SSSS`
+   *            cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param fmt A date time pattern detailing the format of `e` when `e`is a string
    * @return A date, or null if `e` was a string that could not be cast to a date or `fmt` was an
    *         invalid format
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 0eb4776988d9f..6848be15d42a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -252,10 +252,10 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * <li>`columnNameOfCorruptRecord` (default is the value specified in
    * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
    * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
    * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
    * <li>`multiLine` (default `false`): parse one record, which may span multiple lines,
@@ -318,10 +318,10 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * value.</li>
    * <li>`negativeInf` (default `-Inf`): sets the string representation of a negative infinity
    * value.</li>
-   * <li>`dateFormat` (default `uuuu-MM-dd`): sets the string that indicates a date format.
+   * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
    * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `uuuu-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
    * <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
index 7b419c6702586..5c56eff85b264 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
@@ -144,7 +144,7 @@ NULL
 -- !query
 select to_unix_timestamp(a) from t
 -- !query schema
-struct<to_unix_timestamp(a, uuuu-MM-dd HH:mm:ss):bigint>
+struct<to_unix_timestamp(a, yyyy-MM-dd HH:mm:ss):bigint>
 -- !query output
 NULL
 
@@ -160,7 +160,7 @@ NULL
 -- !query
 select unix_timestamp(a) from t
 -- !query schema
-struct<unix_timestamp(a, uuuu-MM-dd HH:mm:ss):bigint>
+struct<unix_timestamp(a, yyyy-MM-dd HH:mm:ss):bigint>
 -- !query output
 NULL
 
@@ -176,7 +176,7 @@ NULL
 -- !query
 select from_unixtime(a) from t
 -- !query schema
-struct<from_unixtime(CAST(a AS BIGINT), uuuu-MM-dd HH:mm:ss):string>
+struct<from_unixtime(CAST(a AS BIGINT), yyyy-MM-dd HH:mm:ss):string>
 -- !query output
 NULL
 

From 76874cfa4a9f6ada746f9c80a3a5a4d569f1a5a8 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 11 Mar 2020 20:13:59 +0800
Subject: [PATCH 0262/1280] [SPARK-31111][SQL][TESTS] Fix interval output issue
 in ExtractBenchmark

### What changes were proposed in this pull request?

fix the error caused by interval output in ExtractBenchmark
### Why are the changes needed?

fix a bug in the test

```scala
[info]   Running case: cast to interval
[error] Exception in thread "main" org.apache.spark.sql.AnalysisException: Cannot use interval type in the table schema.;;
[error] OverwriteByExpression RelationV2[] noop-table, true, true
[error] +- Project [(subtractdates(cast(cast(id#0L as timestamp) as date), -719162) + subtracttimestamps(cast(id#0L as timestamp), -30610249419876544)) AS ((CAST(CAST(id AS TIMESTAMP) AS DATE) - DATE '0001-01-01') + (CAST(id AS TIMESTAMP) - TIMESTAMP '1000-01-01 01:02:03.123456'))#2]
[error]    +- Range (1262304000, 1272304000, step=1, splits=Some(1))
[error]
[error] 	at org.apache.spark.sql.catalyst.util.TypeUtils$.failWithIntervalType(TypeUtils.scala:106)
[error] 	at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$25(CheckAnalysis.scala:389)
[error] 	at org.a
```
### Does this PR introduce any user-facing change?

no

### How was this patch tested?

re-run benchmark

Closes #27867 from yaooqinn/SPARK-31111.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2b46662bd0335320a668d2247a9c62552da07336)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../ExtractBenchmark-jdk11-results.txt        | 208 +++++++++---------
 .../benchmarks/ExtractBenchmark-results.txt   | 208 +++++++++---------
 .../benchmark/ExtractBenchmark.scala          |   4 +-
 3 files changed, 211 insertions(+), 209 deletions(-)

diff --git a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
index 0572cc25bf6c2..0873215982168 100644
--- a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
@@ -1,119 +1,119 @@
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   380            456          67         26.3          38.0       1.0X
-MILLENNIUM of timestamp                            1274           1361         138          7.9         127.4       0.3X
-CENTURY of timestamp                               1119           1132          19          8.9         111.9       0.3X
-DECADE of timestamp                                1076           1083           6          9.3         107.6       0.4X
-YEAR of timestamp                                  1066           1098          43          9.4         106.6       0.4X
-ISOYEAR of timestamp                               1190           1194           4          8.4         119.0       0.3X
-QUARTER of timestamp                               1269           1273           4          7.9         126.9       0.3X
-MONTH of timestamp                                 1060           1075          22          9.4         106.0       0.4X
-WEEK of timestamp                                  1560           1565           8          6.4         156.0       0.2X
-DAY of timestamp                                   1039           1046           8          9.6         103.9       0.4X
-DAYOFWEEK of timestamp                             1248           1274          24          8.0         124.8       0.3X
-DOW of timestamp                                   1252           1273          25          8.0         125.2       0.3X
-ISODOW of timestamp                                1195           1204           9          8.4         119.5       0.3X
-DOY of timestamp                                   1081           1086           6          9.3         108.1       0.4X
-HOUR of timestamp                                   778            781           5         12.9          77.8       0.5X
-MINUTE of timestamp                                 779            780           1         12.8          77.9       0.5X
-SECOND of timestamp                                 597            611          20         16.7          59.7       0.6X
-MILLISECONDS of timestamp                           636            642           6         15.7          63.6       0.6X
-MICROSECONDS of timestamp                           498            504           5         20.1          49.8       0.8X
-EPOCH of timestamp                                  946            956           9         10.6          94.6       0.4X
+cast to timestamp                                   343            348           5         29.1          34.3       1.0X
+MILLENNIUM of timestamp                             888            899          14         11.3          88.8       0.4X
+CENTURY of timestamp                                835            846          15         12.0          83.5       0.4X
+DECADE of timestamp                                 824            834           9         12.1          82.4       0.4X
+YEAR of timestamp                                   818            824          10         12.2          81.8       0.4X
+ISOYEAR of timestamp                                879            890          17         11.4          87.9       0.4X
+QUARTER of timestamp                                965           1073          96         10.4          96.5       0.4X
+MONTH of timestamp                                  787            793           8         12.7          78.7       0.4X
+WEEK of timestamp                                  1277           1341          57          7.8         127.7       0.3X
+DAY of timestamp                                    780            785           8         12.8          78.0       0.4X
+DAYOFWEEK of timestamp                              956            962           7         10.5          95.6       0.4X
+DOW of timestamp                                    995           1034          36         10.1          99.5       0.3X
+ISODOW of timestamp                                 885            898          11         11.3          88.5       0.4X
+DOY of timestamp                                    808            810           3         12.4          80.8       0.4X
+HOUR of timestamp                                   622            627           7         16.1          62.2       0.6X
+MINUTE of timestamp                                 629            636           8         15.9          62.9       0.5X
+SECOND of timestamp                                 800            819          30         12.5          80.0       0.4X
+MILLISECONDS of timestamp                           743            755          21         13.5          74.3       0.5X
+MICROSECONDS of timestamp                           692            730          53         14.5          69.2       0.5X
+EPOCH of timestamp                                  773            798          40         12.9          77.3       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   356            362           9         28.1          35.6       1.0X
-MILLENNIUM of timestamp                            1260           1297          41          7.9         126.0       0.3X
-CENTURY of timestamp                               1082           1085           3          9.2         108.2       0.3X
-DECADE of timestamp                                1056           1068          19          9.5         105.6       0.3X
-YEAR of timestamp                                  1045           1053          13          9.6         104.5       0.3X
-ISOYEAR of timestamp                               1300           1316          25          7.7         130.0       0.3X
-QUARTER of timestamp                               1279           1280           2          7.8         127.9       0.3X
-MONTH of timestamp                                 1037           1046          11          9.6         103.7       0.3X
-WEEK of timestamp                                  1539           1557          28          6.5         153.9       0.2X
-DAY of timestamp                                   1032           1038           6          9.7         103.2       0.3X
-DAYOFWEEK of timestamp                             1241           1244           4          8.1         124.1       0.3X
-DOW of timestamp                                   1237           1241           7          8.1         123.7       0.3X
-ISODOW of timestamp                                1155           1158           3          8.7         115.5       0.3X
-DOY of timestamp                                   1075           1080           4          9.3         107.5       0.3X
-HOUR of timestamp                                   766            770           5         13.1          76.6       0.5X
-MINUTE of timestamp                                 764            769           4         13.1          76.4       0.5X
-SECOND of timestamp                                 590            592           2         16.9          59.0       0.6X
-MILLISECONDS of timestamp                           627            636          10         16.0          62.7       0.6X
-MICROSECONDS of timestamp                           493            505          15         20.3          49.3       0.7X
-EPOCH of timestamp                                  962            966           4         10.4          96.2       0.4X
+cast to timestamp                                   303            323          24         33.0          30.3       1.0X
+MILLENNIUM of timestamp                             841            852           9         11.9          84.1       0.4X
+CENTURY of timestamp                                818            821           4         12.2          81.8       0.4X
+DECADE of timestamp                                 821            836          15         12.2          82.1       0.4X
+YEAR of timestamp                                   858            928          61         11.7          85.8       0.4X
+ISOYEAR of timestamp                                933            939           6         10.7          93.3       0.3X
+QUARTER of timestamp                                979           1004          26         10.2          97.9       0.3X
+MONTH of timestamp                                  755            784          25         13.2          75.5       0.4X
+WEEK of timestamp                                  1189           1228          64          8.4         118.9       0.3X
+DAY of timestamp                                    768            770           4         13.0          76.8       0.4X
+DAYOFWEEK of timestamp                              918            948          27         10.9          91.8       0.3X
+DOW of timestamp                                    933            983          53         10.7          93.3       0.3X
+ISODOW of timestamp                                 884            928          61         11.3          88.4       0.3X
+DOY of timestamp                                    787            797          12         12.7          78.7       0.4X
+HOUR of timestamp                                   611            664          58         16.4          61.1       0.5X
+MINUTE of timestamp                                 616            622           5         16.2          61.6       0.5X
+SECOND of timestamp                                 732            748          14         13.7          73.2       0.4X
+MILLISECONDS of timestamp                           705            716          17         14.2          70.5       0.4X
+MICROSECONDS of timestamp                           642            664          33         15.6          64.2       0.5X
+EPOCH of timestamp                                  760            781          18         13.2          76.0       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        886            905          17         11.3          88.6       1.0X
-MILLENNIUM of date                                 1253           1261           7          8.0         125.3       0.7X
-CENTURY of date                                    1068           1079          10          9.4         106.8       0.8X
-DECADE of date                                     1040           1068          25          9.6         104.0       0.9X
-YEAR of date                                       1032           1043          11          9.7         103.2       0.9X
-ISOYEAR of date                                    1304           1313          12          7.7         130.4       0.7X
-QUARTER of date                                    1284           1301          16          7.8         128.4       0.7X
-MONTH of date                                      1033           1036           4          9.7         103.3       0.9X
-WEEK of date                                       1535           1545           8          6.5         153.5       0.6X
-DAY of date                                        1023           1033          11          9.8         102.3       0.9X
-DAYOFWEEK of date                                  1230           1236           6          8.1         123.0       0.7X
-DOW of date                                        1238           1247           9          8.1         123.8       0.7X
-ISODOW of date                                     1159           1169          17          8.6         115.9       0.8X
-DOY of date                                        1082           1084           3          9.2         108.2       0.8X
-HOUR of date                                       1879           1891          11          5.3         187.9       0.5X
-MINUTE of date                                     1881           1905          21          5.3         188.1       0.5X
-SECOND of date                                     1718           1724           5          5.8         171.8       0.5X
-MILLISECONDS of date                               1733           1737           6          5.8         173.3       0.5X
-MICROSECONDS of date                               1629           1644          23          6.1         162.9       0.5X
-EPOCH of date                                      2085           2090           5          4.8         208.5       0.4X
+cast to date                                        712            786         120         14.0          71.2       1.0X
+MILLENNIUM of date                                  846            888          73         11.8          84.6       0.8X
+CENTURY of date                                     781            792          11         12.8          78.1       0.9X
+DECADE of date                                      770            824          48         13.0          77.0       0.9X
+YEAR of date                                        804            832          27         12.4          80.4       0.9X
+ISOYEAR of date                                     909            931          24         11.0          90.9       0.8X
+QUARTER of date                                     957            975          22         10.4          95.7       0.7X
+MONTH of date                                       789            794           7         12.7          78.9       0.9X
+WEEK of date                                       1141           1165          29          8.8         114.1       0.6X
+DAY of date                                         784            800          22         12.8          78.4       0.9X
+DAYOFWEEK of date                                   907            916          14         11.0          90.7       0.8X
+DOW of date                                         931            958          25         10.7          93.1       0.8X
+ISODOW of date                                      852            857           7         11.7          85.2       0.8X
+DOY of date                                         831            870          45         12.0          83.1       0.9X
+HOUR of date                                       1574           1598          26          6.4         157.4       0.5X
+MINUTE of date                                     1525           1590          58          6.6         152.5       0.5X
+SECOND of date                                     1728           1739          10          5.8         172.8       0.4X
+MILLISECONDS of date                               1715           1727          10          5.8         171.5       0.4X
+MICROSECONDS of date                               1559           1669          95          6.4         155.9       0.5X
+EPOCH of date                                      1766           1774           9          5.7         176.6       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        888            891           6         11.3          88.8       1.0X
-MILLENNIUM of date                                 1250           1260          17          8.0         125.0       0.7X
-CENTURY of date                                    1070           1076           6          9.3         107.0       0.8X
-DECADE of date                                     1036           1041           5          9.6         103.6       0.9X
-YEAR of date                                       1037           1038           1          9.6         103.7       0.9X
-ISOYEAR of date                                    1300           1307           9          7.7         130.0       0.7X
-QUARTER of date                                    1267           1277           9          7.9         126.7       0.7X
-MONTH of date                                      1034           1037           4          9.7         103.4       0.9X
-WEEK of date                                       1543           1554          10          6.5         154.3       0.6X
-DAY of date                                        1022           1030          12          9.8         102.2       0.9X
-DAYOFWEEK of date                                  1230           1232           4          8.1         123.0       0.7X
-DOW of date                                        1227           1242          15          8.1         122.7       0.7X
-ISODOW of date                                     1157           1173          20          8.6         115.7       0.8X
-DOY of date                                        1073           1083          18          9.3         107.3       0.8X
-HOUR of date                                       1873           1878           7          5.3         187.3       0.5X
-MINUTE of date                                     1861           1876          14          5.4         186.1       0.5X
-SECOND of date                                     1717           1724           6          5.8         171.7       0.5X
-MILLISECONDS of date                               1729           1736           7          5.8         172.9       0.5X
-MICROSECONDS of date                               1622           1627           5          6.2         162.2       0.5X
-EPOCH of date                                      2066           2079          19          4.8         206.6       0.4X
+cast to date                                        803            825          23         12.5          80.3       1.0X
+MILLENNIUM of date                                  918            924           8         10.9          91.8       0.9X
+CENTURY of date                                     855            879          25         11.7          85.5       0.9X
+DECADE of date                                      777            795          19         12.9          77.7       1.0X
+YEAR of date                                        772            779           9         13.0          77.2       1.0X
+ISOYEAR of date                                     903            927          21         11.1          90.3       0.9X
+QUARTER of date                                     983           1339         381         10.2          98.3       0.8X
+MONTH of date                                       764            780          20         13.1          76.4       1.1X
+WEEK of date                                       1239           1316         112          8.1         123.9       0.6X
+DAY of date                                         769            857          91         13.0          76.9       1.0X
+DAYOFWEEK of date                                   900            965          61         11.1          90.0       0.9X
+DOW of date                                         990           1041          62         10.1          99.0       0.8X
+ISODOW of date                                      974            977           5         10.3          97.4       0.8X
+DOY of date                                         862            918          50         11.6          86.2       0.9X
+HOUR of date                                       1491           1517          25          6.7         149.1       0.5X
+MINUTE of date                                     1514           1520           9          6.6         151.4       0.5X
+SECOND of date                                     1657           1677          27          6.0         165.7       0.5X
+MILLISECONDS of date                               1646           1669          26          6.1         164.6       0.5X
+MICROSECONDS of date                               1538           1556          23          6.5         153.8       0.5X
+EPOCH of date                                      1699           1718          28          5.9         169.9       0.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                   1273           1279           6          7.9         127.3       1.0X
-MILLENNIUM of interval                             1323           1352          36          7.6         132.3       1.0X
-CENTURY of interval                                1353           1356           4          7.4         135.3       0.9X
-DECADE of interval                                 1326           1339          11          7.5         132.6       1.0X
-YEAR of interval                                   1341           1345           3          7.5         134.1       0.9X
-QUARTER of interval                                1368           1372           4          7.3         136.8       0.9X
-MONTH of interval                                  1320           1326           6          7.6         132.0       1.0X
-DAY of interval                                    1306           1310           4          7.7         130.6       1.0X
-HOUR of interval                                   1341           1347           8          7.5         134.1       0.9X
-MINUTE of interval                                 1337           1349          11          7.5         133.7       1.0X
-SECOND of interval                                 1450           1451           1          6.9         145.0       0.9X
-MILLISECONDS of interval                           1476           1490          23          6.8         147.6       0.9X
-MICROSECONDS of interval                           1316           1331          25          7.6         131.6       1.0X
-EPOCH of interval                                  1461           1462           1          6.8         146.1       0.9X
+cast to interval                                    971            976           5         10.3          97.1       1.0X
+MILLENNIUM of interval                              983           1009          23         10.2          98.3       1.0X
+CENTURY of interval                                 972            984          12         10.3          97.2       1.0X
+DECADE of interval                                  954            962          10         10.5          95.4       1.0X
+YEAR of interval                                    954            973          17         10.5          95.4       1.0X
+QUARTER of interval                                1009           1020          11          9.9         100.9       1.0X
+MONTH of interval                                   946            963          16         10.6          94.6       1.0X
+DAY of interval                                     952            963          12         10.5          95.2       1.0X
+HOUR of interval                                    948            960          11         10.5          94.8       1.0X
+MINUTE of interval                                 1035           1040           4          9.7         103.5       0.9X
+SECOND of interval                                 1085           1105          31          9.2         108.5       0.9X
+MILLISECONDS of interval                           1065           1088          24          9.4         106.5       0.9X
+MICROSECONDS of interval                            992           1007          17         10.1          99.2       1.0X
+EPOCH of interval                                  1087           1103          25          9.2         108.7       0.9X
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-results.txt b/sql/core/benchmarks/ExtractBenchmark-results.txt
index 317f107bd746c..36bdbd026c046 100644
--- a/sql/core/benchmarks/ExtractBenchmark-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-results.txt
@@ -1,119 +1,119 @@
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   409            457          45         24.5          40.9       1.0X
-MILLENNIUM of timestamp                            1240           1295          57          8.1         124.0       0.3X
-CENTURY of timestamp                               1186           1240          49          8.4         118.6       0.3X
-DECADE of timestamp                                1083           1104          20          9.2         108.3       0.4X
-YEAR of timestamp                                  1061           1073          15          9.4         106.1       0.4X
-ISOYEAR of timestamp                               1198           1213          25          8.3         119.8       0.3X
-QUARTER of timestamp                               1304           1322          23          7.7         130.4       0.3X
-MONTH of timestamp                                 1052           1067          19          9.5         105.2       0.4X
-WEEK of timestamp                                  1534           1558          25          6.5         153.4       0.3X
-DAY of timestamp                                   1038           1057          26          9.6         103.8       0.4X
-DAYOFWEEK of timestamp                             1226           1239          22          8.2         122.6       0.3X
-DOW of timestamp                                   1212           1224          13          8.3         121.2       0.3X
-ISODOW of timestamp                                1148           1165          24          8.7         114.8       0.4X
-DOY of timestamp                                   1066           1075          14          9.4         106.6       0.4X
-HOUR of timestamp                                   358            362           6         27.9          35.8       1.1X
-MINUTE of timestamp                                 364            369           4         27.4          36.4       1.1X
-SECOND of timestamp                                 453            471          26         22.1          45.3       0.9X
-MILLISECONDS of timestamp                           497            500           2         20.1          49.7       0.8X
-MICROSECONDS of timestamp                           360            363           4         27.8          36.0       1.1X
-EPOCH of timestamp                                 1100           1104           5          9.1         110.0       0.4X
+cast to timestamp                                   287            308          19         34.8          28.7       1.0X
+MILLENNIUM of timestamp                             896            918          20         11.2          89.6       0.3X
+CENTURY of timestamp                                849            856           7         11.8          84.9       0.3X
+DECADE of timestamp                                 765            777          17         13.1          76.5       0.4X
+YEAR of timestamp                                   754            756           2         13.3          75.4       0.4X
+ISOYEAR of timestamp                                843            849           5         11.9          84.3       0.3X
+QUARTER of timestamp                                867            873           9         11.5          86.7       0.3X
+MONTH of timestamp                                  758            762           4         13.2          75.8       0.4X
+WEEK of timestamp                                  1049           1054           6          9.5         104.9       0.3X
+DAY of timestamp                                    750            763          11         13.3          75.0       0.4X
+DAYOFWEEK of timestamp                              890            918          25         11.2          89.0       0.3X
+DOW of timestamp                                    879            887           8         11.4          87.9       0.3X
+ISODOW of timestamp                                 862            869          11         11.6          86.2       0.3X
+DOY of timestamp                                    811            868          55         12.3          81.1       0.4X
+HOUR of timestamp                                   627            638          11         16.0          62.7       0.5X
+MINUTE of timestamp                                 600            606           6         16.7          60.0       0.5X
+SECOND of timestamp                                 743            799          51         13.5          74.3       0.4X
+MILLISECONDS of timestamp                           723            737          22         13.8          72.3       0.4X
+MICROSECONDS of timestamp                           648            653           5         15.4          64.8       0.4X
+EPOCH of timestamp                                  780            800          17         12.8          78.0       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   315            317           2         31.8          31.5       1.0X
-MILLENNIUM of timestamp                            1155           1162           8          8.7         115.5       0.3X
-CENTURY of timestamp                               1146           1152           5          8.7         114.6       0.3X
-DECADE of timestamp                                1031           1043          11          9.7         103.1       0.3X
-YEAR of timestamp                                  1033           1041          10          9.7         103.3       0.3X
-ISOYEAR of timestamp                               1274           1278           5          7.8         127.4       0.2X
-QUARTER of timestamp                               1326           1346          30          7.5         132.6       0.2X
-MONTH of timestamp                                 1027           1031           7          9.7         102.7       0.3X
-WEEK of timestamp                                  1529           1535           6          6.5         152.9       0.2X
-DAY of timestamp                                   1024           1031           9          9.8         102.4       0.3X
-DAYOFWEEK of timestamp                             1197           1201           5          8.4         119.7       0.3X
-DOW of timestamp                                   1201           1218          15          8.3         120.1       0.3X
-ISODOW of timestamp                                1143           1149           6          8.8         114.3       0.3X
-DOY of timestamp                                   1074           1085           9          9.3         107.4       0.3X
-HOUR of timestamp                                   354            354           0         28.3          35.4       0.9X
-MINUTE of timestamp                                 358            361           5         27.9          35.8       0.9X
-SECOND of timestamp                                 445            456          17         22.5          44.5       0.7X
-MILLISECONDS of timestamp                           504            514          12         19.8          50.4       0.6X
-MICROSECONDS of timestamp                           361            370           8         27.7          36.1       0.9X
-EPOCH of timestamp                                 1111           1117           9          9.0         111.1       0.3X
+cast to timestamp                                   238            248          12         42.0          23.8       1.0X
+MILLENNIUM of timestamp                             862            875          12         11.6          86.2       0.3X
+CENTURY of timestamp                                833            847          22         12.0          83.3       0.3X
+DECADE of timestamp                                 759            765           7         13.2          75.9       0.3X
+YEAR of timestamp                                   744            755          15         13.4          74.4       0.3X
+ISOYEAR of timestamp                                937           1019          73         10.7          93.7       0.3X
+QUARTER of timestamp                               1011           1091          69          9.9         101.1       0.2X
+MONTH of timestamp                                  846            888          40         11.8          84.6       0.3X
+WEEK of timestamp                                  1210           1239          41          8.3         121.0       0.2X
+DAY of timestamp                                    932            979          41         10.7          93.2       0.3X
+DAYOFWEEK of timestamp                             1174           1200          42          8.5         117.4       0.2X
+DOW of timestamp                                   1131           1172          37          8.8         113.1       0.2X
+ISODOW of timestamp                                 896            903           7         11.2          89.6       0.3X
+DOY of timestamp                                    805            818          12         12.4          80.5       0.3X
+HOUR of timestamp                                   596            597           2         16.8          59.6       0.4X
+MINUTE of timestamp                                 582            597          17         17.2          58.2       0.4X
+SECOND of timestamp                                 697            709          16         14.4          69.7       0.3X
+MILLISECONDS of timestamp                           700            710          10         14.3          70.0       0.3X
+MICROSECONDS of timestamp                           612            631          21         16.3          61.2       0.4X
+EPOCH of timestamp                                  755            760           7         13.2          75.5       0.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        849            851           3         11.8          84.9       1.0X
-MILLENNIUM of date                                 1129           1139          11          8.9         112.9       0.8X
-CENTURY of date                                    1136           1143           7          8.8         113.6       0.7X
-DECADE of date                                     1039           1043           5          9.6         103.9       0.8X
-YEAR of date                                       1030           1037          10          9.7         103.0       0.8X
-ISOYEAR of date                                    1269           1278           9          7.9         126.9       0.7X
-QUARTER of date                                    1323           1330           6          7.6         132.3       0.6X
-MONTH of date                                      1021           1023           2          9.8         102.1       0.8X
-WEEK of date                                       1541           1549           8          6.5         154.1       0.6X
-DAY of date                                        1021           1033          13          9.8         102.1       0.8X
-DAYOFWEEK of date                                  1196           1209          11          8.4         119.6       0.7X
-DOW of date                                        1214           1229          13          8.2         121.4       0.7X
-ISODOW of date                                     1148           1153           7          8.7         114.8       0.7X
-DOY of date                                        1073           1079           5          9.3         107.3       0.8X
-HOUR of date                                       1311           1314           4          7.6         131.1       0.6X
-MINUTE of date                                     1311           1311           1          7.6         131.1       0.6X
-SECOND of date                                     1420           1434          13          7.0         142.0       0.6X
-MILLISECONDS of date                               1426           1442          14          7.0         142.6       0.6X
-MICROSECONDS of date                               1312           1318           6          7.6         131.2       0.6X
-EPOCH of date                                      2034           2050          16          4.9         203.4       0.4X
+cast to date                                        633            641           7         15.8          63.3       1.0X
+MILLENNIUM of date                                  824            845          26         12.1          82.4       0.8X
+CENTURY of date                                     864            878          13         11.6          86.4       0.7X
+DECADE of date                                      746            763          17         13.4          74.6       0.8X
+YEAR of date                                        752            785          39         13.3          75.2       0.8X
+ISOYEAR of date                                     900            905           5         11.1          90.0       0.7X
+QUARTER of date                                     906            930          23         11.0          90.6       0.7X
+MONTH of date                                       747            752           6         13.4          74.7       0.8X
+WEEK of date                                       1089           1100          17          9.2         108.9       0.6X
+DAY of date                                         795            803          13         12.6          79.5       0.8X
+DAYOFWEEK of date                                   937            944          10         10.7          93.7       0.7X
+DOW of date                                         927           1003          66         10.8          92.7       0.7X
+ISODOW of date                                      977            980           7         10.2          97.7       0.6X
+DOY of date                                         827            877          45         12.1          82.7       0.8X
+HOUR of date                                       1525           1547          34          6.6         152.5       0.4X
+MINUTE of date                                     1473           1499          23          6.8         147.3       0.4X
+SECOND of date                                     1600           1615          15          6.2         160.0       0.4X
+MILLISECONDS of date                               1666           1765         156          6.0         166.6       0.4X
+MICROSECONDS of date                               1554           1627         127          6.4         155.4       0.4X
+EPOCH of date                                      1615           1646          27          6.2         161.5       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        852            879          42         11.7          85.2       1.0X
-MILLENNIUM of date                                 1131           1136           7          8.8         113.1       0.8X
-CENTURY of date                                    1138           1145           6          8.8         113.8       0.7X
-DECADE of date                                     1030           1043          13          9.7         103.0       0.8X
-YEAR of date                                       1022           1028           8          9.8         102.2       0.8X
-ISOYEAR of date                                    1260           1265           6          7.9         126.0       0.7X
-QUARTER of date                                    1326           1330           7          7.5         132.6       0.6X
-MONTH of date                                      1014           1034          26          9.9         101.4       0.8X
-WEEK of date                                       1523           1526           5          6.6         152.3       0.6X
-DAY of date                                        1022           1023           2          9.8         102.2       0.8X
-DAYOFWEEK of date                                  1197           1203           9          8.4         119.7       0.7X
-DOW of date                                        1188           1198          16          8.4         118.8       0.7X
-ISODOW of date                                     1143           1153           9          8.8         114.3       0.7X
-DOY of date                                        1052           1058           7          9.5         105.2       0.8X
-HOUR of date                                       1309           1311           4          7.6         130.9       0.7X
-MINUTE of date                                     1302           1305           6          7.7         130.2       0.7X
-SECOND of date                                     1414           1432          16          7.1         141.4       0.6X
-MILLISECONDS of date                               1441           1450          11          6.9         144.1       0.6X
-MICROSECONDS of date                               1292           1301           8          7.7         129.2       0.7X
-EPOCH of date                                      2030           2036           8          4.9         203.0       0.4X
+cast to date                                        676            690          20         14.8          67.6       1.0X
+MILLENNIUM of date                                  868            882          16         11.5          86.8       0.8X
+CENTURY of date                                     875            880           8         11.4          87.5       0.8X
+DECADE of date                                      762            776          16         13.1          76.2       0.9X
+YEAR of date                                        788            800          11         12.7          78.8       0.9X
+ISOYEAR of date                                     903            917          12         11.1          90.3       0.7X
+QUARTER of date                                     983           1018          40         10.2          98.3       0.7X
+MONTH of date                                       836            857          19         12.0          83.6       0.8X
+WEEK of date                                       1137           1168          28          8.8         113.7       0.6X
+DAY of date                                         768            817          82         13.0          76.8       0.9X
+DAYOFWEEK of date                                   890            926          36         11.2          89.0       0.8X
+DOW of date                                        1007           1033          39          9.9         100.7       0.7X
+ISODOW of date                                      962            969           7         10.4          96.2       0.7X
+DOY of date                                         797            882          80         12.6          79.7       0.8X
+HOUR of date                                       1449           1482          29          6.9         144.9       0.5X
+MINUTE of date                                     1536           1610          69          6.5         153.6       0.4X
+SECOND of date                                     1675           1823         128          6.0         167.5       0.4X
+MILLISECONDS of date                               1605           1622          18          6.2         160.5       0.4X
+MICROSECONDS of date                               1481           1504          32          6.8         148.1       0.5X
+EPOCH of date                                      1656           1843         296          6.0         165.6       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                   1249           1254           6          8.0         124.9       1.0X
-MILLENNIUM of interval                             1310           1316           9          7.6         131.0       1.0X
-CENTURY of interval                                1304           1315          10          7.7         130.4       1.0X
-DECADE of interval                                 1306           1313           7          7.7         130.6       1.0X
-YEAR of interval                                   1304           1313          11          7.7         130.4       1.0X
-QUARTER of interval                                1310           1317           7          7.6         131.0       1.0X
-MONTH of interval                                  1311           1319          12          7.6         131.1       1.0X
-DAY of interval                                    1295           1304          13          7.7         129.5       1.0X
-HOUR of interval                                   1301           1306           8          7.7         130.1       1.0X
-MINUTE of interval                                 1316           1319           3          7.6         131.6       0.9X
-SECOND of interval                                 1437           1440           3          7.0         143.7       0.9X
-MILLISECONDS of interval                           1435           1449          16          7.0         143.5       0.9X
-MICROSECONDS of interval                           1304           1314           9          7.7         130.4       1.0X
-EPOCH of interval                                  1440           1453          19          6.9         144.0       0.9X
+cast to interval                                    919            959          45         10.9          91.9       1.0X
+MILLENNIUM of interval                              959           1001          36         10.4          95.9       1.0X
+CENTURY of interval                                 974            998          21         10.3          97.4       0.9X
+DECADE of interval                                  975            989          13         10.3          97.5       0.9X
+YEAR of interval                                    987           1012          24         10.1          98.7       0.9X
+QUARTER of interval                                1014           1031          25          9.9         101.4       0.9X
+MONTH of interval                                   978           1018          45         10.2          97.8       0.9X
+DAY of interval                                    1012           1095          92          9.9         101.2       0.9X
+HOUR of interval                                   1129           1143          14          8.9         112.9       0.8X
+MINUTE of interval                                 1038           1060          25          9.6         103.8       0.9X
+SECOND of interval                                 1062           1289         356          9.4         106.2       0.9X
+MILLISECONDS of interval                           1142           1226          83          8.8         114.2       0.8X
+MICROSECONDS of interval                           1038           1129          79          9.6         103.8       0.9X
+EPOCH of interval                                  1090           1127          34          9.2         109.0       0.8X
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
index de23132284dc8..f1315d48e4d99 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
@@ -42,7 +42,9 @@ object ExtractBenchmark extends SqlBasedBenchmark {
       spark
         .range(sinceSecond, sinceSecond + cardinality, 1, 1)
         .selectExpr(exprs: _*)
-        .noop()
+        .queryExecution
+        .toRdd
+        .foreach(_ => ())
     }
   }
 

From 4e9504f8601bf4d267270fddcf075572d53f5070 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Wed, 11 Mar 2020 20:53:56 +0800
Subject: [PATCH 0263/1280] [SPARK-31076][SQL] Convert Catalyst's
 DATE/TIMESTAMP to Java Date/Timestamp via local date-time

In the PR, I propose to change conversion of java.sql.Timestamp/Date values to/from internal values of Catalyst's TimestampType/DateType before cutover day `1582-10-15` of Gregorian calendar. I propose to construct local date-time from microseconds/days since the epoch. Take each date-time component `year`, `month`, `day`, `hour`, `minute`, `second` and `second fraction`, and construct java.sql.Timestamp/Date using the extracted components.

This will rebase underlying time/date offset in the way that collected java.sql.Timestamp/Date values will have the same local time-date component as the original values in Gregorian calendar.

Here is the example which demonstrates the issue:
```sql
scala> sql("select date '1100-10-10'").collect()
res1: Array[org.apache.spark.sql.Row] = Array([1100-10-03])
```

Yes, after the changes:
```sql
scala> sql("select date '1100-10-10'").collect()
res0: Array[org.apache.spark.sql.Row] = Array([1100-10-10])
```

By running `DateTimeUtilsSuite`, `DateFunctionsSuite` and `DateExpressionsSuite`.

Closes #27807 from MaxGekk/rebase-timestamp-before-1582.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3d3e366aa836cb7d2295f54e78e544c7b15c9c08)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 39 +++++++++--
 .../spark/sql/execution/HiveResultSuite.scala |  6 +-
 .../datasources/orc/OrcColumnVector.java      |  3 +-
 .../datasources/orc/OrcColumnVector.java      |  3 +-
 .../spark/sql/hive/HiveInspectors.scala       | 66 +++++++++++++++++--
 5 files changed, 102 insertions(+), 15 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 2427a71676314..e1cc89d03c069 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -47,6 +47,15 @@ object DateTimeUtils {
   // it's 2440587.5, rounding up to compatible with Hive
   final val JULIAN_DAY_OF_EPOCH = 2440588
 
+  final val GREGORIAN_CUTOVER_DAY = LocalDate.of(1582, 10, 15).toEpochDay
+  final val GREGORIAN_CUTOVER_MICROS = instantToMicros(
+    LocalDateTime.of(1582, 10, 15, 0, 0, 0)
+      .atOffset(ZoneOffset.UTC)
+      .toInstant)
+  final val GREGORIAN_CUTOVER_MILLIS = GREGORIAN_CUTOVER_MICROS / MICROS_PER_MILLIS
+
+  final val julianCommonEraStart = Timestamp.valueOf("0001-01-01 00:00:00")
+
   final val TimeZoneGMT = TimeZone.getTimeZone("GMT")
   final val TimeZoneUTC = TimeZone.getTimeZone("UTC")
 
@@ -88,28 +97,50 @@ object DateTimeUtils {
    * Returns the number of days since epoch from java.sql.Date.
    */
   def fromJavaDate(date: Date): SQLDate = {
-    millisToDays(date.getTime)
+    if (date.getTime < GREGORIAN_CUTOVER_MILLIS) {
+      val era = if (date.before(julianCommonEraStart)) 0 else 1
+      val localDate = date.toLocalDate.`with`(ChronoField.ERA, era)
+      localDateToDays(localDate)
+    } else {
+      millisToDays(date.getTime)
+    }
   }
 
   /**
    * Returns a java.sql.Date from number of days since epoch.
    */
   def toJavaDate(daysSinceEpoch: SQLDate): Date = {
-    new Date(daysToMillis(daysSinceEpoch))
+    if (daysSinceEpoch < GREGORIAN_CUTOVER_DAY) {
+      Date.valueOf(LocalDate.ofEpochDay(daysSinceEpoch))
+    } else {
+      new Date(daysToMillis(daysSinceEpoch))
+    }
   }
 
   /**
    * Returns a java.sql.Timestamp from number of micros since epoch.
    */
   def toJavaTimestamp(us: SQLTimestamp): Timestamp = {
-    Timestamp.from(microsToInstant(us))
+    if (us < GREGORIAN_CUTOVER_MICROS) {
+      val ldt = microsToInstant(us).atZone(ZoneId.systemDefault()).toLocalDateTime
+      Timestamp.valueOf(ldt)
+    } else {
+      Timestamp.from(microsToInstant(us))
+    }
   }
 
   /**
    * Returns the number of micros since epoch from java.sql.Timestamp.
    */
   def fromJavaTimestamp(t: Timestamp): SQLTimestamp = {
-    instantToMicros(t.toInstant)
+    if (t.getTime < GREGORIAN_CUTOVER_MILLIS) {
+      val era = if (t.before(julianCommonEraStart)) 0 else 1
+      val localDateTime = t.toLocalDateTime.`with`(ChronoField.ERA, era)
+      val instant = ZonedDateTime.of(localDateTime, ZoneId.systemDefault()).toInstant
+      instantToMicros(instant)
+    } else {
+      instantToMicros(t.toInstant)
+    }
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index bb59b12e6f350..bf7cbaad7384b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -23,7 +23,7 @@ class HiveResultSuite extends SharedSparkSession {
   import testImplicits._
 
   test("date formatting in hive result") {
-    val dates = Seq("2018-12-28", "1582-10-13", "1582-10-14", "1582-10-15")
+    val dates = Seq("2018-12-28", "1582-10-03", "1582-10-04", "1582-10-15")
     val df = dates.toDF("a").selectExpr("cast(a as date) as b")
     val executedPlan1 = df.queryExecution.executedPlan
     val result = HiveResult.hiveResultString(executedPlan1)
@@ -36,8 +36,8 @@ class HiveResultSuite extends SharedSparkSession {
   test("timestamp formatting in hive result") {
     val timestamps = Seq(
       "2018-12-28 01:02:03",
-      "1582-10-13 01:02:03",
-      "1582-10-14 01:02:03",
+      "1582-10-03 01:02:03",
+      "1582-10-04 01:02:03",
       "1582-10-15 01:02:03")
     val df = timestamps.toDF("a").selectExpr("cast(a as timestamp) as b")
     val executedPlan1 = df.queryExecution.executedPlan
diff --git a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 9bfad1e83ee7b..0dfed768f87de 100644
--- a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -21,6 +21,7 @@
 
 import org.apache.orc.storage.ql.exec.vector.*;
 
+import org.apache.spark.sql.catalyst.util.DateTimeUtils;
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.TimestampType;
@@ -136,7 +137,7 @@ public int getInt(int rowId) {
   public long getLong(int rowId) {
     int index = getRowIndex(rowId);
     if (isTimestamp) {
-      return timestampData.time[index] * 1000 + timestampData.nanos[index] / 1000 % 1000;
+      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
     } else {
       return longData.vector[index];
     }
diff --git a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 2f1925e69a337..35447fe7e19c2 100644
--- a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -21,6 +21,7 @@
 
 import org.apache.hadoop.hive.ql.exec.vector.*;
 
+import org.apache.spark.sql.catalyst.util.DateTimeUtils;
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.TimestampType;
@@ -136,7 +137,7 @@ public int getInt(int rowId) {
   public long getLong(int rowId) {
     int index = getRowIndex(rowId);
     if (isTimestamp) {
-      return timestampData.time[index] * 1000 + timestampData.nanos[index] / 1000 % 1000;
+      return DateTimeUtils.fromJavaTimestamp(timestampData.asScratchTimestamp(index));
     } else {
       return longData.vector[index];
     }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index 0cd9b3641bd4a..e217c5279046e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -18,7 +18,8 @@
 package org.apache.spark.sql.hive
 
 import java.lang.reflect.{ParameterizedType, Type, WildcardType}
-import java.util.concurrent.TimeUnit._
+import java.time.LocalDate
+import java.util.Calendar
 
 import scala.collection.JavaConverters._
 
@@ -181,6 +182,33 @@ import org.apache.spark.unsafe.types.UTF8String
  */
 private[hive] trait HiveInspectors {
 
+  private final val JULIAN_CUTOVER_DAY =
+    rebaseGregorianToJulianDays(DateTimeUtils.GREGORIAN_CUTOVER_DAY.toInt)
+
+  private def rebaseJulianToGregorianDays(daysSinceEpoch: Int): Int = {
+    val localDate = LocalDate.ofEpochDay(daysSinceEpoch)
+    val utcCal = new Calendar.Builder()
+      .setCalendarType("gregory")
+      .setTimeZone(DateTimeUtils.TimeZoneUTC)
+      .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+      .build()
+    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, DateTimeConstants.MILLIS_PER_DAY))
+  }
+
+  private def rebaseGregorianToJulianDays(daysSinceEpoch: Int): Int = {
+    val millis = Math.multiplyExact(daysSinceEpoch, DateTimeConstants.MILLIS_PER_DAY)
+    val utcCal = new Calendar.Builder()
+      .setCalendarType("gregory")
+      .setTimeZone(DateTimeUtils.TimeZoneUTC)
+      .setInstant(millis)
+      .build()
+    val localDate = LocalDate.of(
+      utcCal.get(Calendar.YEAR),
+      utcCal.get(Calendar.MONTH) + 1,
+      utcCal.get(Calendar.DAY_OF_MONTH))
+    Math.toIntExact(localDate.toEpochDay)
+  }
+
   def javaTypeToDataType(clz: Type): DataType = clz match {
     // writable
     case c: Class[_] if c == classOf[hadoopIo.DoubleWritable] => DoubleType
@@ -466,7 +494,7 @@ private[hive] trait HiveInspectors {
         _ => constant
       case poi: WritableConstantTimestampObjectInspector =>
         val t = poi.getWritableConstantValue
-        val constant = SECONDS.toMicros(t.getSeconds) + NANOSECONDS.toMicros(t.getNanos)
+        val constant = DateTimeUtils.fromJavaTimestamp(t.getTimestamp)
         _ => constant
       case poi: WritableConstantIntObjectInspector =>
         val constant = poi.getWritableConstantValue.get()
@@ -618,7 +646,14 @@ private[hive] trait HiveInspectors {
         case x: DateObjectInspector if x.preferWritable() =>
           data: Any => {
             if (data != null) {
-              DateTimeUtils.fromJavaDate(x.getPrimitiveWritableObject(data).get())
+              // Rebasing written days via conversion to local dates.
+              // See the comment for `getDateWritable()`.
+              val daysSinceEpoch = x.getPrimitiveWritableObject(data).getDays
+              if (daysSinceEpoch < JULIAN_CUTOVER_DAY) {
+                rebaseJulianToGregorianDays(daysSinceEpoch)
+              } else {
+                daysSinceEpoch
+              }
             } else {
               null
             }
@@ -634,8 +669,7 @@ private[hive] trait HiveInspectors {
         case x: TimestampObjectInspector if x.preferWritable() =>
           data: Any => {
             if (data != null) {
-              val t = x.getPrimitiveWritableObject(data)
-              SECONDS.toMicros(t.getSeconds) + NANOSECONDS.toMicros(t.getNanos)
+              DateTimeUtils.fromJavaTimestamp(x.getPrimitiveWritableObject(data).getTimestamp)
             } else {
               null
             }
@@ -1012,7 +1046,27 @@ private[hive] trait HiveInspectors {
     }
 
   private def getDateWritable(value: Any): hiveIo.DateWritable =
-    if (value == null) null else new hiveIo.DateWritable(value.asInstanceOf[Int])
+    if (value == null) {
+      null
+    } else {
+      // Rebasing days since the epoch to store the same number of days
+      // as by Spark 2.4 and earlier versions. Spark 3.0 switched to
+      // Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
+      // this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
+      // Julian calendar for dates before 1582-10-15. So, the same local date may
+      // be mapped to different number of days since the epoch in different calendars.
+      // For example:
+      // Proleptic Gregorian calendar: 1582-01-01 -> -141714
+      // Julian calendar: 1582-01-01 -> -141704
+      // The code below converts -141714 to -141704.
+      val daysSinceEpoch = value.asInstanceOf[Int]
+      val rebasedDays = if (daysSinceEpoch < DateTimeUtils.GREGORIAN_CUTOVER_DAY) {
+        rebaseGregorianToJulianDays(daysSinceEpoch)
+      } else {
+        daysSinceEpoch
+      }
+      new hiveIo.DateWritable(rebasedDays)
+    }
 
   private def getTimestampWritable(value: Any): hiveIo.TimestampWritable =
     if (value == null) {

From 53059e39180604c6676bb644c831caa9265437aa Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 11 Mar 2020 23:47:13 +0800
Subject: [PATCH 0264/1280] [SPARK-31117][SQL][TEST] reduce the test time of
 DateTimeUtilsSuite

`DateTimeUtilsSuite.daysToMicros and microsToDays` takes 30 seconds, which is too long for a UT.

This PR changes the test to check random data, to reduce testing time. Now this test takes 1 second.

make test faster

no

N/A

Closes #27873 from cloud-fan/test.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5be0d04f16dce1f2a3ad94669b6cff7460287397)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index d98cd0a1e785c..a88a8dda4456c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -622,10 +622,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       "MIT" -> Set(15338))
     for (tz <- ALL_TIMEZONES) {
       val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
-      (-20000 to 20000).foreach { d =>
+      val testingData = Seq(-20000, 20000) ++
+        (1 to 1000).map(_ => (math.random() * 40000 - 20000).toInt)
+      testingData.foreach { d =>
         if (!skipped.contains(d)) {
           assert(millisToDays(daysToMillis(d, tz.toZoneId), tz.toZoneId) === d,
-            s"Round trip of ${d} did not work in tz ${tz}")
+            s"Round trip of $d did not work in tz $tz")
         }
       }
     }

From c1e6e1439d1a79560f197e2627a334fcd0bb8a28 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 11 Mar 2020 09:55:24 -0700
Subject: [PATCH 0265/1280] [SPARK-31091] Revert SPARK-24640 Return `NULL` from
 `size(NULL)` by default

### What changes were proposed in this pull request?

This PR reverts https://github.com/apache/spark/pull/26051 and https://github.com/apache/spark/pull/26066

### Why are the changes needed?

There is no standard requiring that `size(null)` must return null, and returning -1 looks reasonable as well. This is kind of a cosmetic change and we should avoid it if it breaks existing queries. This is similar to reverting TRIM function parameter order change.

### Does this PR introduce any user-facing change?

Yes, change the behavior of `size(null)` back to be the same as 2.4.

### How was this patch tested?

N/A

Closes #27834 from cloud-fan/revert.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 8efb71013d0c9e8d81430aa48f88b91929425bff)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-migration-guide.md                                   | 2 --
 .../spark/sql/catalyst/expressions/collectionOperations.scala | 4 ++--
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala    | 2 +-
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 6c73038bbcfa0..e7ac9f0fb5bff 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -214,8 +214,6 @@ license: |
     - `now` - current query start time
   For example `SELECT timestamp 'tomorrow';`.
 
-  - Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
-  
   - Since Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and earlier, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
 
   - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index cfa877b9c0892..6d95909e17f41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -79,7 +79,7 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
     _FUNC_(expr) - Returns the size of an array or a map.
     The function returns -1 if its input is null and spark.sql.legacy.sizeOfNull is set to true.
     If spark.sql.legacy.sizeOfNull is set to false, the function returns null for null input.
-    By default, the spark.sql.legacy.sizeOfNull parameter is set to false.
+    By default, the spark.sql.legacy.sizeOfNull parameter is set to true.
   """,
   examples = """
     Examples:
@@ -88,7 +88,7 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
       > SELECT _FUNC_(map('a', 1, 'b', 2));
        2
       > SELECT _FUNC_(NULL);
-       NULL
+       -1
   """)
 case class Size(child: Expression, legacySizeOfNull: Boolean)
   extends UnaryExpression with ExpectsInputTypes {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index fdaf0ececa5e9..644fe899b8591 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1942,7 +1942,7 @@ object SQLConf {
     .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " +
       "The size function returns null for null input if the flag is disabled.")
     .booleanConf
-    .createWithDefault(false)
+    .createWithDefault(true)
 
   val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED =
     buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled")

From ffcc4a27041abe97991f4bd14d0b5abf3c50a542 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 11 Mar 2020 16:52:40 -0700
Subject: [PATCH 0266/1280] [SPARK-31110][DOCS][SQL] refine sql doc for SELECT

### What changes were proposed in this pull request?

A few improvements to the sql ref SELECT doc:
1. correct the syntax of SELECT query
2. correct the default of null sort order
3. correct the GROUP BY syntax
4. several minor fixes

### Why are the changes needed?

refine document

### Does this PR introduce any user-facing change?

N/A

### How was this patch tested?

N/A

Closes #27866 from cloud-fan/doc.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 0f0ccdadb123d5839c34244e25a4ee17dde0fcdc)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-ref-syntax-qry-select-clusterby.md   | 18 +++---
 ...sql-ref-syntax-qry-select-distribute-by.md | 18 +++---
 docs/sql-ref-syntax-qry-select-groupby.md     | 48 ++++++++--------
 docs/sql-ref-syntax-qry-select-having.md      | 12 ++--
 docs/sql-ref-syntax-qry-select-limit.md       | 23 +++++---
 docs/sql-ref-syntax-qry-select-orderby.md     | 24 ++++----
 docs/sql-ref-syntax-qry-select-sortby.md      | 28 +++++-----
 docs/sql-ref-syntax-qry-select-where.md       | 10 ++--
 docs/sql-ref-syntax-qry-select.md             | 55 ++++++++++---------
 docs/sql-ref-syntax-qry.md                    |  8 +--
 10 files changed, 126 insertions(+), 118 deletions(-)

diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index bb60e8bfe25fc..8f1dc59806f80 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -41,20 +41,20 @@ CLUSTER BY { expression [ , ... ] }
 ### Examples
 {% highlight sql %}
 CREATE TABLE person (name STRING, age INT);
-INSERT INTO person VALUES 
-    ('Zen Hui', 25), 
-    ('Anil B', 18), 
-    ('Shone S', 16), 
+INSERT INTO person VALUES
+    ('Zen Hui', 25),
+    ('Anil B', 18),
+    ('Shone S', 16),
     ('Mike A', 25),
-    ('John A', 18), 
+    ('John A', 18),
     ('Jack N', 16);
 
 -- Reduce the number of shuffle partitions to 2 to illustrate the behavior of `CLUSTER BY`.
 -- It's easier to see the clustering and sorting behavior with less number of partitions.
 SET spark.sql.shuffle.partitions = 2;
-                        
+
 -- Select the rows with no ordering. Please note that without any sort directive, the results
--- of the query is not deterministic. It's included here to show the difference in behavior 
+-- of the query is not deterministic. It's included here to show the difference in behavior
 -- of a query when `CLUSTER BY` is not used vs when it's used. The query below produces rows
 -- where age column is not sorted.
 SELECT age, name FROM person;
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index 5ade9c16e4a0f..957df9c48eee7 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,7 @@ license: |
 ---
 The <code>DISTRIBUTE BY</code> clause is used to repartition the data based
 on the input expressions. Unlike the [CLUSTER BY](sql-ref-syntax-qry-select-clusterby.html)
-clause, this does not sort the data within each partition. 
+clause, this does not sort the data within each partition.
 
 ### Syntax
 {% highlight sql %}
@@ -39,19 +39,19 @@ DISTRIBUTE BY { expression [ , ... ] }
 {% highlight sql %}
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
-    ('Zen Hui', 25), 
-    ('Anil B', 18), 
-    ('Shone S', 16), 
+    ('Zen Hui', 25),
+    ('Anil B', 18),
+    ('Shone S', 16),
     ('Mike A', 25),
-    ('John A', 18), 
+    ('John A', 18),
     ('Jack N', 16);
 
 -- Reduce the number of shuffle partitions to 2 to illustrate the behavior of `DISTRIBUTE BY`.
 -- It's easier to see the clustering and sorting behavior with less number of partitions.
 SET spark.sql.shuffle.partitions = 2;
-                        
+
 -- Select the rows with no ordering. Please note that without any sort directive, the result
--- of the query is not deterministic. It's included here to just contrast it with the 
+-- of the query is not deterministic. It's included here to just contrast it with the
 -- behavior of `DISTRIBUTE BY`. The query below produces rows where age columns are not
 -- clustered together.
 SELECT age, name FROM person;
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index ab1c5d6eb5ee5..49a11cafbbd5a 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -9,42 +9,43 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-The <code>GROUP BY</code> clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on 
-the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple 
+The <code>GROUP BY</code> clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
+the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple
 aggregations for the same input record set via `GROUPING SETS`, `CUBE`, `ROLLUP` clauses.
 
 ### Syntax
 {% highlight sql %}
-GROUP BY [ GROUPING SETS grouping_sets ] group_expression [ , group_expression [ , ... ] ]
-    [ ( WITH ROLLUP | WITH CUBE | GROUPING SETS grouping_sets ) ) ]
+GROUP BY group_expression [ , group_expression [ , ... ] ]
+  [ { WITH ROLLUP | WITH CUBE | GROUPING SETS (grouping_set [ , ...]) } ]
+
+GROUP BY GROUPING SETS (grouping_set [ , ...])
 {% endhighlight %}
 
 ### Parameters
 <dl>
   <dt><code><em>GROUPING SETS</em></code></dt>
   <dd>
-    Groups the rows for each subset of the expressions specified in the grouping sets. For example, 
+    Groups the rows for each subset of the expressions specified in the grouping sets. For example,
     <code>GROUP BY GROUPING SETS (warehouse, product)</code> is semantically equivalent
     to union of results of <code>GROUP BY warehouse</code> and <code>GROUP BY product</code>. This clause
-    is shorthand for a <code>UNION ALL</code> where each leg of the <code>UNION ALL</code> 
+    is a shorthand for a <code>UNION ALL</code> where each leg of the <code>UNION ALL</code>
     operator performs aggregation of subset of the columns specified in the <code>GROUPING SETS</code> clause.
   </dd>
-  <dt><code><em>grouping_sets</em></code></dt>
+  <dt><code><em>grouping_set</em></code></dt>
   <dd>
-    Specifies one of more groupings based on which the <code>GROUP BY</code> clause performs aggregations. A grouping
-    set is specified by a list of comma-separated expressions in parentheses.<br><br>
+    A grouping set is specified by zero or more comma-separated expressions in parentheses.<br><br>
     <b>Syntax:</b>
       <code>
-        ( () | ( expression [ , ... ] ) )
+        ([expression [, ...]])
       </code>
   </dd>
   <dt><code><em>grouping_expression</em></code></dt>
@@ -55,17 +56,18 @@ GROUP BY [ GROUPING SETS grouping_sets ] group_expression [ , group_expression [
   </dd>
   <dt><code><em>ROLLUP</em></code></dt>
   <dd>
-    Specifies multiple levels of aggregations in a single statement. This clause is used to compute aggregations 
-    based on multiple grouping sets. <code>ROLLUP</code> is shorthand for <code>GROUPING SETS</code>. For example,
-    GROUP BY warehouse, product  WITH ROLLUP is equivalent to GROUP BY <code>warehouse, product</code> GROUPING SETS
-    <code> ((warehouse, product), (warehouse), ())</code>.
+    Specifies multiple levels of aggregations in a single statement. This clause is used to compute aggregations
+    based on multiple grouping sets. <code>ROLLUP</code> is a shorthand for <code>GROUPING SETS</code>. For example,
+    <code>GROUP BY warehouse, product WITH ROLLUP</code> is equivalent to <code>GROUP BY GROUPING SETS
+    ((warehouse, product), (warehouse), ())</code>.
     The N elements of a <code>ROLLUP</code> specification results in N+1 <code>GROUPING SETS</code>.
   </dd>
   <dt><code><em>CUBE</em></code></dt>
   <dd>
-    <code>CUBE</code> clause is used to perform aggregations based on combination of grouping columns specified in the 
-    <code>GROUP BY</code> clause. For example, <code>GROUP BY warehouse, product  WITH CUBE</code> is equivalent 
-    to GROUP BY <code>warehouse, product</code> GROUPING SETS <code>((warehouse, product), (warehouse), (product), ())</code>.
+    <code>CUBE</code> clause is used to perform aggregations based on combination of grouping columns specified in the
+    <code>GROUP BY</code> clause. <code>CUBE</code> is a shorthand for <code>GROUPING SETS</code>. For example,
+    <code>GROUP BY warehouse, product WITH CUBE</code> is equivalent to <code>GROUP BY GROUPING SETS
+    ((warehouse, product), (warehouse), (product), ())</code>.
     The N elements of a <code>CUBE</code> specification results in 2^N <code>GROUPING SETS</code>.
   </dd>
 </dl>
@@ -73,7 +75,7 @@ GROUP BY [ GROUPING SETS grouping_sets ] group_expression [ , group_expression [
 ### Examples
 {% highlight sql %}
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
-INSERT INTO dealer VALUES 
+INSERT INTO dealer VALUES
     (100, 'Fremont', 'Honda Civic', 10),
     (100, 'Fremont', 'Honda Accord', 15),
     (100, 'Fremont', 'Honda CRV', 7),
@@ -107,7 +109,7 @@ SELECT id, sum(quantity) FROM dealer GROUP BY 1 ORDER BY 1;
 
 -- Multiple aggregations.
 -- 1. Sum of quantity per dealership.
--- 2. Max quantity per dealership. 
+-- 2. Max quantity per dealership.
 SELECT id, sum(quantity) AS sum, max(quantity) AS max FROM dealer GROUP BY id ORDER BY id;
 
   +---+---+---+
@@ -148,7 +150,7 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
   |San Jose|Honda Civic |5  |
   +--------+------------+---+
 
--- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS` 
+-- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS`
 -- specifications are present.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
    GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
@@ -200,7 +202,7 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
 -- Group by processing with `CUBE` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model WITH CUBE 
+   GROUP BY city, car_model WITH CUBE
    ORDER BY city, car_model;
 
   +--------+------------+---+
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index 94d9be649f106..dee1e3c0e39b9 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -33,7 +33,7 @@ HAVING boolean_expression
   <dt><code><em>boolean_expression</em></code></dt>
   <dd>
     Specifies any expression that evaluates to a result type <code>boolean</code>. Two or
-    more expressions may be combined together using the logical 
+    more expressions may be combined together using the logical
     operators ( <code>AND</code>, <code>OR</code> ).<br><br>
 
     <b>Note</b><br>
@@ -70,7 +70,7 @@ SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING city = 'Fremo
 
 -- `HAVING` clause referring to aggregate function.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum(quantity) > 15;
- 
+
   +-------+---+
   |   city|sum|
   +-------+---+
@@ -100,7 +100,7 @@ SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING max(quantity)
 
 -- `HAVING` clause referring to constant expression.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING 1 > 0 ORDER BY city;
-  
+
   +--------+---+
   |    city|sum|
   +--------+---+
@@ -116,7 +116,7 @@ SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
   +---+
   | 78|
   +---+
- 
+
 {% endhighlight %}
 
 ### Related Clauses
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 06925e6ee353c..356930c879d28 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,7 +37,7 @@ LIMIT { ALL | integer_expression }
   </dd>
   <dt><code><em>integer_expression</em></code></dt>
   <dd>
-    Specifies an expression that returns an integer. 
+    Specifies a foldable expression that returns an integer.
   </dd>
 </dl>
 
@@ -45,13 +45,13 @@ LIMIT { ALL | integer_expression }
 {% highlight sql %}
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
-    ('Zen Hui', 25), 
-    ('Anil B', 18), 
-    ('Shone S', 16), 
+    ('Zen Hui', 25),
+    ('Anil B', 18),
+    ('Shone S', 16),
     ('Mike A', 25),
-    ('John A', 18), 
+    ('John A', 18),
     ('Jack N', 16);
-                        
+
 -- Select the first two rows.
 SELECT name, age FROM person ORDER BY name LIMIT 2;
 
@@ -76,7 +76,7 @@ SELECT name, age FROM person ORDER BY name LIMIT ALL;
   |Zen Hui|25 |
   +-------+---+
 
--- A function expression as an input to limit.
+-- A function expression as an input to LIMIT.
 SELECT name, age FROM person ORDER BY name LIMIT length('SPARK')
 
   +-------+---+
@@ -88,6 +88,11 @@ SELECT name, age FROM person ORDER BY name LIMIT length('SPARK')
   | Mike A| 25|
   |Shone S| 16|
   +-------+---+
+
+-- A non-foldable expression as an input to LIMIT is not allowed.
+SELECT name, age FROM person ORDER BY name LIMIT length(name)
+
+org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a constant value ...
 {% endhighlight %}
 
 ### Related Clauses
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index c4b4ced0b7e73..eb99dbb06096d 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,7 @@ license: |
 ---
 The <code>ORDER BY</code> clause is used to return the result rows in a sorted manner
 in the user specified order. Unlike the [SORT BY](sql-ref-syntax-qry-select-sortby.html)
-clause, this clause guarantees a total order in the output. 
+clause, this clause guarantees a total order in the output.
 
 ### Syntax
 {% highlight sql %}
@@ -47,20 +47,18 @@ ORDER BY { expression [ sort_direction | nulls_sort_oder ] [ , ... ] }
   </dd>
   <dt><code><em>nulls_sort_order</em></code></dt>
   <dd>
-    Optionally specifies whether NULL values are returned before/after non-NULL values, based on the 
-    sort direction. In Spark, NULL values are considered to be lower than any non-NULL values by default.
-    Therefore the ordering of NULL values depend on the sort direction. If <code>null_sort_order</code> is
-    not specified, then NULLs sort first if sort order is <code>ASC</code> and NULLS sort last if 
-    sort order is <code>DESC</code>.<br><br>
+    Optionally specifies whether NULL values are returned before/after non-NULL values. If
+    <code>null_sort_order</code> is not specified, then NULLs sort first if sort order is
+    <code>ASC</code> and NULLS sort last if sort order is <code>DESC</code>.<br><br>
     <ol>
-      <li> If <code>NULLS FIRST</code> (the default) is specified, then NULL values are returned first 
+      <li> If <code>NULLS FIRST</code> is specified, then NULL values are returned first
            regardless of the sort order.</li>
       <li>If <code>NULLS LAST</code> is specified, then NULL values are returned last regardless of
            the sort order. </li>
     </ol><br>
     <b>Syntax:</b>
     <code>
-       [ NULLS { FIRST | LAST } ] 
+       [ NULLS { FIRST | LAST } ]
     </code>
   </dd>
 </dl>
@@ -75,7 +73,7 @@ INSERT INTO person VALUES
     (400, 'Jerry', NULL),
     (500, 'Dan',  50);
 
--- Sort rows by age. By default rows are sorted in ascending manner.
+-- Sort rows by age. By default rows are sorted in ascending manner with NULL FIRST.
 SELECT name, age FROM person ORDER BY age;
 
   +-----+----+
@@ -101,9 +99,9 @@ SELECT name, age FROM person ORDER BY age NULLS LAST;
   |Jerry|null|
   +-----+----+
 
--- Sort rows by age in descending manner.
+-- Sort rows by age in descending manner, which defaults to NULL LAST.
 SELECT name, age FROM person ORDER BY age DESC;
- 
+
   +-----+----+
   |name |age |
   +-----+----+
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index 1818a69f4a34b..9b52738ee7926 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -49,20 +49,18 @@ SORT BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
   </dd>
   <dt><code><em>nulls_sort_order</em></code></dt>
   <dd>
-    Optionally specifies whether NULL values are returned before/after non-NULL values, based on the 
-    sort direction. In Spark, NULL values are considered to be lower than any non-NULL values by default.
-    Therefore the ordering of NULL values depend on the sort direction. If <code>null_sort_order</code> is
-    not specified, then NULLs sort first if sort order is <code>ASC</code> and NULLS sort last if 
-    sort order is <code>DESC</code>.<br><br>
+    Optionally specifies whether NULL values are returned before/after non-NULL values. If
+    <code>null_sort_order</code> is not specified, then NULLs sort first if sort order is
+    <code>ASC</code> and NULLS sort last if sort order is <code>DESC</code>.<br><br>
     <ol>
-      <li> If <code>NULLS FIRST</code> (the default) is specified, then NULL values are returned first 
+      <li> If <code>NULLS FIRST</code> is specified, then NULL values are returned first
            regardless of the sort order.</li>
       <li>If <code>NULLS LAST</code> is specified, then NULL values are returned last regardless of
            the sort order. </li>
     </ol><br>
     <b>Syntax:</b>
     <code>
-       [ NULLS { FIRST | LAST } ] 
+       [ NULLS { FIRST | LAST } ]
     </code>
   </dd>
 </dl>
@@ -71,15 +69,15 @@ SORT BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
 {% highlight sql %}
 CREATE TABLE person (zip_code INT, name STRING, age INT);
 INSERT INTO person VALUES
-    (94588, 'Zen Hui', 50), 
-    (94588, 'Dan Li', 18), 
+    (94588, 'Zen Hui', 50),
+    (94588, 'Dan Li', 18),
     (94588, 'Anil K', 27),
     (94588, 'John V', NULL),
     (94511, 'David K', 42),
     (94511, 'Aryan B.', 18),
     (94511, 'Lalit B.', NULL);
 
--- Use `REPARTITION` hint to partition the data by `zip_code` to 
+-- Use `REPARTITION` hint to partition the data by `zip_code` to
 -- examine the `SORT BY` behavior. This is used in rest of the
 -- examples.
 
@@ -128,9 +126,9 @@ SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age
   |null|Lalit B.|94511   |
   +----+--------+--------+
 
--- Sort rows by age within each partition in descending manner.
+-- Sort rows by age within each partition in descending manner, which defaults to NULL LAST.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC;
- 
+
   +----+--------+--------+
   |age |name    |zip_code|
   +----+--------+--------+
@@ -143,7 +141,7 @@ SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age
   |null|Lalit B.|94511   |
   +----+--------+--------+
 
--- Sort rows by age within each partition in ascending manner keeping null values to be first.
+-- Sort rows by age within each partition in descending manner keeping null values to be first.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC NULLS FIRST;
 
   +----+--------+--------+
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index a493623df480b..106053d16f8bd 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,7 +31,7 @@ WHERE boolean_expression
   <dt><code><em>boolean_expression</em></code></dt>
   <dd>
     Specifies any expression that evaluates to a result type <code>boolean</code>. Two or
-    more expressions may be combined together using the logical 
+    more expressions may be combined together using the logical
     operators ( <code>AND</code>, <code>OR</code> ).
   </dd>
 </dl>
@@ -99,8 +99,8 @@ SELECT * FROM person WHERE age > (SELECT avg(age) FROM person);
   |300|Mike|80 |
   +---+----+---+
 
--- Correlated column reference in `WHERE` clause of subquery.
-SELECT * FROM person AS parent 
+-- Correlated Subquery in `WHERE` clause.
+SELECT * FROM person AS parent
 WHERE EXISTS (
               SELECT 1 FROM person AS child
               WHERE parent.id = child.id AND child.age IS NULL
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 80b930f335d1e..e87c4a587e34c 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -1,7 +1,7 @@
 ---
 layout: global
 title: SELECT
-displayTitle: SELECT 
+displayTitle: SELECT
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,39 +19,43 @@ license: |
   limitations under the License.
 ---
 Spark supports a `SELECT` statement and conforms to the ANSI SQL standard. Queries are
-used to retrieve result sets from one or more tables. The following section 
+used to retrieve result sets from one or more tables. The following section
 describes the overall query syntax and the sub-sections cover different constructs
-of a query along with examples. 
+of a query along with examples.
 
 ### Syntax
 {% highlight sql %}
 [ WITH with_query [ , ... ] ]
+select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_statement, ... ]
+[ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
+[ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
+[ CLUSTER BY { expression [ , ...] } ]
+[ DISTRIBUTE BY { expression [, ...] } ]
+[ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
+[ LIMIT { ALL | expression } ]
+{% endhighlight %}
+
+While `select_statement` is defined as
+{% highlight sql %}
 SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
   FROM { from_item [ , ...] }
   [ WHERE boolean_expression ]
   [ GROUP BY expression [ , ...] ]
   [ HAVING boolean_expression ]
-  [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-  [ SORT  BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-  [ CLUSTER BY { expression [ , ...] } ]
-  [ DISTRIBUTE BY { expression [, ...] } ]
-  { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select ]
-  [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
-  [ LIMIT { ALL | expression } ]
 {% endhighlight %}
 
 ### Parameters
 <dl>
   <dt><code><em>with_query</em></code></dt>
   <dd>
-    Specifies the common table expressions (CTEs) before the main <code>SELECT</code> query block.
-    These table expressions are allowed to be referenced later in the main query. This is useful to abstract
-    out repeated subquery blocks in the main query and improves readability of the query.
+    Specifies the common table expressions (CTEs) before the main query block.
+    These table expressions are allowed to be referenced later in the FROM clause. This is useful to abstract
+    out repeated subquery blocks in the FROM clause and improves readability of the query.
   </dd>
   <dt><code><em>hints</em></code></dt>
   <dd>
     Hints can be specified to help spark optimizer make better planning decisions. Currently spark supports hints
-    that influence selection of join strategies and repartitioning of the data. 
+    that influence selection of join strategies and repartitioning of the data.
   </dd>
   <dt><code><em>ALL</em></code></dt>
   <dd>
@@ -77,7 +81,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
       <li>Join relation</li>
       <li>Table valued function</li>
       <li>Inlined table</li>
-      <li>Subquery</li>    
+      <li>Subquery</li>
     </ol>
   </dd>
   <dt><code><em>WHERE</em></code></dt>
@@ -87,12 +91,13 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
   <dt><code><em>GROUP BY</em></code></dt>
   <dd>
     Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
-    (MIN, MAX, COUNT, SUM, AVG) to group rows based on the grouping expressions.
+    (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
   </dd>
   <dt><code><em>HAVING</em></code></dt>
   <dd>
     Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
-    filter rows after the grouping is performed.
+    filter rows after the grouping is performed. If HAVING is specified without GROUP BY, it indicates a GROUP BY
+    without grouping expressions (global aggregate).
   </dd>
   <dt><code><em>ORDER BY</em></code></dt>
   <dd>
@@ -108,17 +113,17 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
   <dt><code><em>CLUSTER BY</em></code></dt>
   <dd>
     Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
-    the same effect of using <code>DISTRIBUTE BY</code> and <code>SORT BY</code> together. 
+    the same effect of using <code>DISTRIBUTE BY</code> and <code>SORT BY</code> together.
   </dd>
   <dt><code><em>DISTRIBUTE BY</em></code></dt>
   <dd>
-    Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually 
-    exclusive with <code>ORDER BY</code> and <code>CLUSTER BY</code> and can not be specified together. 
+    Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually
+    exclusive with <code>ORDER BY</code> and <code>CLUSTER BY</code> and can not be specified together.
   </dd>
   <dt><code><em>LIMIT</em></code></dt>
   <dd>
-    Specifies the maximum number of rows that can be returned by a statement or subquery. This clause 
-    is mostly used in the conjunction with <code>ORDER BY</code> to produce a deterministic result. 
+    Specifies the maximum number of rows that can be returned by a statement or subquery. This clause
+    is mostly used in the conjunction with <code>ORDER BY</code> to produce a deterministic result.
   </dd>
   <dt><code><em>boolean_expression</em></code></dt>
   <dd>
@@ -130,7 +135,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
   </dd>
   <dt><code><em>named_window</em></code></dt>
   <dd>
-    Specifies aliases for one or more source window specifications. The source window specifications can 
+    Specifies aliases for one or more source window specifications. The source window specifications can
     be referenced in the widow definitions in the query.
   </dd>
 </dl>
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index cd7c0ffccdc0f..37414acd57a38 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,14 +19,14 @@ license: |
   limitations under the License.
 ---
 
-Spark supports <code>SELECT</code> statement that is  used to retrieve rows
+Spark supports <code>SELECT</code> statement that is used to retrieve rows
 from one or more tables according to the specified clauses. The full syntax
 and brief description of supported clauses are explained in
 [SELECT](sql-ref-syntax-qry-select.html) section. Spark also provides the
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
- 
+
 - [WHERE Clause](sql-ref-syntax-qry-select-where.html)
 - [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
 - [HAVING Clause](sql-ref-syntax-qry-select-having.html)

From b86dc6ab76d945e5c15c12390436ad95c119a493 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 11 Mar 2020 19:26:15 -0700
Subject: [PATCH 0267/1280] [SPARK-31126][SS] Upgrade Kafka to 2.4.1

### What changes were proposed in this pull request?

This PR (SPARK-31126) aims to upgrade Kafka library to bring a client-side bug fix like KAFKA-8933

### Why are the changes needed?

The following is the full release note.
- https://downloads.apache.org/kafka/2.4.1/RELEASE_NOTES.html

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Pass the Jenkins with the existing test.

Closes #27881 from dongjoon-hyun/SPARK-KAFKA-2.4.1.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 614323d326db192540c955b4fa9b3b7af7527001)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 262f3ac0d98df..5aa100e57cb63 100644
--- a/pom.xml
+++ b/pom.xml
@@ -132,7 +132,7 @@
     <!-- Version used for internal directory structure -->
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
-    <kafka.version>2.4.0</kafka.version>
+    <kafka.version>2.4.1</kafka.version>
     <derby.version>10.12.1.1</derby.version>
     <parquet.version>1.10.1</parquet.version>
     <orc.version>1.5.9</orc.version>

From 5d616262954f0b163e74c7a5527e5b3dfa913f52 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 12 Mar 2020 18:13:52 +0800
Subject: [PATCH 0268/1280] [SPARK-31066][SQL][TEST-HIVE1.2] Disable useless
 and uncleaned hive SessionState initialization parts

### What changes were proposed in this pull request?

As a common usage and according to the spark doc, users may often just copy their `hive-site.xml` to Spark directly from hive projects. Sometimes, the config file is not that clean for spark and may cause some side effects.

for example, `hive.session.history.enabled` will create a log for the hive jobs but useless for spark and also it will not be deleted on JVM exit.

this pr
 1) disable `hive.session.history.enabled` explicitly to disable creating `hive_job_log` file, e.g.
```
Hive history file=/var/folders/01/h81cs4sn3dq2dd_k4j6fhrmc0000gn/T//kentyao/hive_job_log_79c63b29-95a4-4935-a9eb-2d89844dfe4f_493861201.txt
```
2) set `hive.execution.engine` to `spark` explicitly in case the config is `tez` and casue uneccesary problem like this:

```
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/tez/dag/api/SessionNotRunning
	at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:529)
```

### Why are the changes needed?

reduce overhead of internal complexity and users' hive cognitive load for  running spark

### Does this PR introduce any user-facing change?

yes, `hive_job_log` file will not be created even enabled, and will not try to initialize tez kinds of stuff
### How was this patch tested?

add ut and verify manually

Closes #27827 from yaooqinn/SPARK-31066.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 18f27308749e695f9942768d7ba85cef9fceb174)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../hive/thriftserver/SparkSQLCLIDriver.scala |  8 +-
 .../sql/hive/client/HiveClientImpl.scala      | 82 +++++++++++--------
 .../spark/sql/hive/client/VersionsSuite.scala | 12 +++
 3 files changed, 64 insertions(+), 38 deletions(-)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 19f7ea85c6cf0..6b76927bd4150 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -44,6 +44,7 @@ import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.hive.HiveUtils
+import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider
 import org.apache.spark.util.ShutdownHookManager
 
@@ -88,12 +89,7 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     val hadoopConf = SparkHadoopUtil.get.newConfiguration(sparkConf)
     val extraConfigs = HiveUtils.formatTimeVarsForHiveClient(hadoopConf)
 
-    val cliConf = new HiveConf(classOf[SessionState])
-    (hadoopConf.iterator().asScala.map(kv => kv.getKey -> kv.getValue)
-      ++ sparkConf.getAll.toMap ++ extraConfigs).foreach {
-      case (k, v) =>
-        cliConf.set(k, v)
-    }
+    val cliConf = HiveClientImpl.newHiveConf(sparkConf, hadoopConf, extraConfigs)
 
     val sessionState = new CliSessionState(cliConf)
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index f2c516ec0a8dc..4a3e813740b48 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -58,7 +58,6 @@ import org.apache.spark.sql.execution.QueryExecutionException
 import org.apache.spark.sql.hive.HiveExternalCatalog
 import org.apache.spark.sql.hive.HiveExternalCatalog.{DATASOURCE_SCHEMA, DATASOURCE_SCHEMA_NUMPARTS, DATASOURCE_SCHEMA_PART_PREFIX}
 import org.apache.spark.sql.hive.HiveUtils
-import org.apache.spark.sql.hive.client.HiveClientImpl._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.{CircularBuffer, Utils}
@@ -99,6 +98,8 @@ private[hive] class HiveClientImpl(
   extends HiveClient
   with Logging {
 
+  import HiveClientImpl._
+
   // Circular buffer to hold what hive prints to STDOUT and ERR.  Only printed when failures occur.
   private val outputBuffer = new CircularBuffer()
 
@@ -159,36 +160,7 @@ private[hive] class HiveClientImpl(
       s"(version ${version.fullVersion}) is ${conf.getVar(ConfVars.METASTOREWAREHOUSE)}")
 
   private def newState(): SessionState = {
-    val hiveConf = new HiveConf(classOf[SessionState])
-    // HiveConf is a Hadoop Configuration, which has a field of classLoader and
-    // the initial value will be the current thread's context class loader
-    // (i.e. initClassLoader at here).
-    // We call hiveConf.setClassLoader(initClassLoader) at here to make
-    // this action explicit.
-    hiveConf.setClassLoader(initClassLoader)
-
-    // 1: Take all from the hadoopConf to this hiveConf.
-    // This hadoopConf contains user settings in Hadoop's core-site.xml file
-    // and Hive's hive-site.xml file. Note, we load hive-site.xml file manually in
-    // SharedState and put settings in this hadoopConf instead of relying on HiveConf
-    // to load user settings. Otherwise, HiveConf's initialize method will override
-    // settings in the hadoopConf. This issue only shows up when spark.sql.hive.metastore.jars
-    // is not set to builtin. When spark.sql.hive.metastore.jars is builtin, the classpath
-    // has hive-site.xml. So, HiveConf will use that to override its default values.
-    // 2: we set all spark confs to this hiveConf.
-    // 3: we set all entries in config to this hiveConf.
-    val confMap = (hadoopConf.iterator().asScala.map(kv => kv.getKey -> kv.getValue) ++
-      sparkConf.getAll.toMap ++ extraConfig).toMap
-    confMap.foreach { case (k, v) => hiveConf.set(k, v) }
-    SQLConf.get.redactOptions(confMap).foreach { case (k, v) =>
-      logDebug(
-        s"""
-           |Applying Hadoop/Hive/Spark and extra properties to Hive Conf:
-           |$k=$v
-         """.stripMargin)
-    }
-    // Disable CBO because we removed the Calcite dependency.
-    hiveConf.setBoolean("hive.cbo.enable", false)
+    val hiveConf = newHiveConf(sparkConf, hadoopConf, extraConfig, Some(initClassLoader))
     val state = new SessionState(hiveConf)
     if (clientLoader.cachedHive != null) {
       Hive.set(clientLoader.cachedHive.asInstanceOf[Hive])
@@ -990,7 +962,7 @@ private[hive] class HiveClientImpl(
   }
 }
 
-private[hive] object HiveClientImpl {
+private[hive] object HiveClientImpl extends Logging {
   /** Converts the native StructField to Hive's FieldSchema. */
   def toHiveColumn(c: StructField): FieldSchema = {
     val typeString = if (c.metadata.contains(HIVE_TYPE_STRING)) {
@@ -1232,4 +1204,50 @@ private[hive] object HiveClientImpl {
     StatsSetupConst.RAW_DATA_SIZE,
     StatsSetupConst.TOTAL_SIZE
   )
+
+  def newHiveConf(
+      sparkConf: SparkConf,
+      hadoopConf: JIterable[JMap.Entry[String, String]],
+      extraConfig: Map[String, String],
+      classLoader: Option[ClassLoader] = None): HiveConf = {
+    val hiveConf = new HiveConf(classOf[SessionState])
+    // HiveConf is a Hadoop Configuration, which has a field of classLoader and
+    // the initial value will be the current thread's context class loader.
+    // We call hiveConf.setClassLoader(initClassLoader) at here to ensure it use the classloader
+    // we want.
+    classLoader.foreach(hiveConf.setClassLoader)
+    // 1: Take all from the hadoopConf to this hiveConf.
+    // This hadoopConf contains user settings in Hadoop's core-site.xml file
+    // and Hive's hive-site.xml file. Note, we load hive-site.xml file manually in
+    // SharedState and put settings in this hadoopConf instead of relying on HiveConf
+    // to load user settings. Otherwise, HiveConf's initialize method will override
+    // settings in the hadoopConf. This issue only shows up when spark.sql.hive.metastore.jars
+    // is not set to builtin. When spark.sql.hive.metastore.jars is builtin, the classpath
+    // has hive-site.xml. So, HiveConf will use that to override its default values.
+    // 2: we set all spark confs to this hiveConf.
+    // 3: we set all entries in config to this hiveConf.
+    val confMap = (hadoopConf.iterator().asScala.map(kv => kv.getKey -> kv.getValue) ++
+      sparkConf.getAll.toMap ++ extraConfig).toMap
+    confMap.foreach { case (k, v) => hiveConf.set(k, v) }
+    SQLConf.get.redactOptions(confMap).foreach { case (k, v) =>
+      logDebug(s"Applying Hadoop/Hive/Spark and extra properties to Hive Conf:$k=$v")
+    }
+    // Disable CBO because we removed the Calcite dependency.
+    hiveConf.setBoolean("hive.cbo.enable", false)
+    // If this is true, SessionState.start will create a file to log hive job which will not be
+    // deleted on exit and is useless for spark
+    if (hiveConf.getBoolean("hive.session.history.enabled", false)) {
+      logWarning("Detected HiveConf hive.session.history.enabled is true and will be reset to" +
+        " false to disable useless hive logic")
+      hiveConf.setBoolean("hive.session.history.enabled", false)
+    }
+    // If this is tez engine, SessionState.start might bring extra logic to initialize tez stuff,
+    // which is useless for spark.
+    if (hiveConf.get("hive.execution.engine") == "tez") {
+      logWarning("Detected HiveConf hive.execution.engine is 'tez' and will be reset to 'mr'" +
+        " to disable useless hive logic")
+      hiveConf.set("hive.execution.engine", "mr")
+    }
+    hiveConf
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index 4760af7aa46ff..7471142d4e6fa 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -82,6 +82,18 @@ class VersionsSuite extends SparkFunSuite with Logging {
     assert("success" === client.getConf("test", null))
   }
 
+  test("override useless and side-effect hive configurations ") {
+    val hadoopConf = new Configuration()
+    // These hive flags should be reset by spark
+    hadoopConf.setBoolean("hive.cbo.enable", true)
+    hadoopConf.setBoolean("hive.session.history.enabled", true)
+    hadoopConf.set("hive.execution.engine", "tez")
+    val client = buildClient(HiveUtils.builtinHiveVersion, hadoopConf)
+    assert(!client.getConf("hive.cbo.enable", "true").toBoolean)
+    assert(!client.getConf("hive.session.history.enabled", "true").toBoolean)
+    assert(client.getConf("hive.execution.engine", "tez") === "mr")
+  }
+
   private def getNestedMessages(e: Throwable): String = {
     var causes = ""
     var lastException = e

From 8163c267b0dc4d117b22936e0371f53e1b4620b6 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 12 Mar 2020 21:16:02 +0900
Subject: [PATCH 0269/1280] [SPARK-31010][SQL][FOLLOW-UP] Give an example for
 typed Scala UDF in error message

In the error message, adding an example for typed Scala UDF.

Help user to know how to migrate to typed Scala UDF.

No, it's a new error message in Spark 3.0.

Pass Jenkins.

Closes #27884 from Ngone51/spark_31010_followup.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala   | 2 +-
 sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 644fe899b8591..37389223d57d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2032,7 +2032,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.allowUntypedScalaUDF")
       .internal()
       .doc("When set to true, user is allowed to use org.apache.spark.sql.functions." +
-        "udf(f: AnyRef, dataType: DataType). Otherwise, exception will be throw.")
+        "udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.")
       .booleanConf
       .createWithDefault(false)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index f280ec3c22e29..e2d3d55812c51 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -4740,8 +4740,9 @@ object functions {
         "information. Spark may blindly pass null to the Scala closure with primitive-type " +
         "argument, and the closure will see the default value of the Java type for the null " +
         "argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. " +
-        "You could use other typed Scala UDF APIs to avoid this problem, or set " +
-        s"${SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key} to true and use this API with caution."
+        "You could use typed Scala UDF APIs (e.g. `udf((x: Int) => x)`) to avoid this problem, " +
+        s"or set ${SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key} to true and use this API with " +
+        s"caution."
       throw new AnalysisException(errorMsg)
     }
     SparkUserDefinedFunction(f, dataType, inputSchemas = Nil)

From 7659a6a66e5170b078de9af00a27a0e65d5e12de Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 12 Mar 2020 21:28:24 +0800
Subject: [PATCH 0270/1280] [SPARK-31124][SQL] change the default value of
 minPartitionNum in AQE

### What changes were proposed in this pull request?

AQE has a perf regression when using the default settings: if we coalesce the shuffle partitions into one or few partitions, we may leave many CPU cores idle and the perf is worse than with AQE off (which leverages all CPU cores).

Technically, this is not a bad thing. If there are many queries running at the same time, it's better to coalesce shuffle partitions into fewer partitions. However, the default settings of AQE should try to avoid any perf regression as possible as we can.

This PR changes the default value of minPartitionNum when coalescing shuffle partitions, to be `SparkContext.defaultParallelism`, so that AQE can leverage all the CPU cores.

### Why are the changes needed?

avoid AQE perf regression

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

existing tests

Closes #27879 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 77c49cb702862a0c60733dba797201ada2f5b51a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/internal/SQLConf.scala  |  7 +++----
 .../sql/execution/adaptive/AdaptiveSparkPlanExec.scala |  2 +-
 .../execution/adaptive/CoalesceShufflePartitions.scala | 10 ++++++++--
 .../sql/execution/adaptive/OptimizeSkewedJoin.scala    |  9 +++++++--
 .../sql/execution/adaptive/ShufflePartitionsUtil.scala |  2 +-
 5 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 37389223d57d3..06180f620219a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -404,12 +404,13 @@ object SQLConf {
 
   val COALESCE_PARTITIONS_MIN_PARTITION_NUM =
     buildConf("spark.sql.adaptive.coalescePartitions.minPartitionNum")
-      .doc("The minimum number of shuffle partitions after coalescing. This configuration only " +
+      .doc("The minimum number of shuffle partitions after coalescing. If not set, the default " +
+        "value is the default parallelism of the Spark cluster. This configuration only " +
         s"has an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true.")
       .intConf
       .checkValue(_ > 0, "The minimum number of partitions must be positive.")
-      .createWithDefault(1)
+      .createOptional
 
   val COALESCE_PARTITIONS_INITIAL_PARTITION_NUM =
     buildConf("spark.sql.adaptive.coalescePartitions.initialPartitionNum")
@@ -2446,8 +2447,6 @@ class SQLConf extends Serializable with Logging {
 
   def coalesceShufflePartitionsEnabled: Boolean = getConf(COALESCE_PARTITIONS_ENABLED)
 
-  def minShufflePartitionNum: Int = getConf(COALESCE_PARTITIONS_MIN_PARTITION_NUM)
-
   def initialShufflePartitionNum: Int =
     getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(numShufflePartitions)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index c1486aa057912..68da06de2cb7d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -100,7 +100,7 @@ case class AdaptiveSparkPlanExec(
     // before 'CoalesceShufflePartitions', as the skewed partition handled
     // in 'OptimizeSkewedJoin' rule, should be omitted in 'CoalesceShufflePartitions'.
     OptimizeSkewedJoin(conf),
-    CoalesceShufflePartitions(conf),
+    CoalesceShufflePartitions(context.session),
     // The rule of 'OptimizeLocalShuffleReader' need to make use of the 'partitionStartIndices'
     // in 'CoalesceShufflePartitions' rule. So it must be after 'CoalesceShufflePartitions' rule.
     OptimizeLocalShuffleReader(conf),
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
index d779a20057ac2..d2a7f6ab5426c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.MapOutputStatistics
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.internal.SQLConf
@@ -26,8 +27,9 @@ import org.apache.spark.sql.internal.SQLConf
  * A rule to coalesce the shuffle partitions based on the map output statistics, which can
  * avoid many small reduce tasks that hurt performance.
  */
-case class CoalesceShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
+case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPlan] {
   import CoalesceShufflePartitions._
+  private def conf = session.sessionState.conf
 
   override def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.coalesceShufflePartitionsEnabled) {
@@ -66,12 +68,16 @@ case class CoalesceShufflePartitions(conf: SQLConf) extends Rule[SparkPlan] {
       val distinctNumPreShufflePartitions =
         validMetrics.map(stats => stats.bytesByPartitionId.length).distinct
       if (validMetrics.nonEmpty && distinctNumPreShufflePartitions.length == 1) {
+        // We fall back to Spark default parallelism if the minimum number of coalesced partitions
+        // is not set, so to avoid perf regressions compared to no coalescing.
+        val minPartitionNum = conf.getConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM)
+          .getOrElse(session.sparkContext.defaultParallelism)
         val partitionSpecs = ShufflePartitionsUtil.coalescePartitions(
           validMetrics.toArray,
           firstPartitionIndex = 0,
           lastPartitionIndex = distinctNumPreShufflePartitions.head,
           advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
-          minNumPartitions = conf.minShufflePartitionNum)
+          minNumPartitions = minPartitionNum)
         // This transformation adds new nodes, so we must use `transformUp` here.
         val stageIds = shuffleStages.map(_.id).toSet
         plan.transformUp {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 7f523937b632c..a75a3f37f6672 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
+import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkContext, SparkEnv}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
@@ -261,12 +261,17 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     if (!shouldCoalesce || nonSkewPartitionIndices.length == 1) {
       nonSkewPartitionIndices.map(i => CoalescedPartitionSpec(i, i + 1))
     } else {
+      // We fall back to Spark default parallelism if the minimum number of coalesced partitions
+      // is not set, so to avoid perf regressions compared to no coalescing.
+      val minPartitionNum = conf.getConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM)
+        .getOrElse(SparkContext.getActive.get.defaultParallelism)
       ShufflePartitionsUtil.coalescePartitions(
         Array(leftStats, rightStats),
         firstPartitionIndex = nonSkewPartitionIndices.head,
         // `lastPartitionIndex` is exclusive.
         lastPartitionIndex = nonSkewPartitionIndices.last + 1,
-        advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES))
+        advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
+        minNumPartitions = minPartitionNum)
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
index 32f5fd41819b2..292df11c23ca1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
@@ -59,7 +59,7 @@ object ShufflePartitionsUtil extends Logging {
       firstPartitionIndex: Int,
       lastPartitionIndex: Int,
       advisoryTargetSize: Long,
-      minNumPartitions: Int = 1): Seq[ShufflePartitionSpec] = {
+      minNumPartitions: Int): Seq[ShufflePartitionSpec] = {
     // If `minNumPartitions` is very large, it is possible that we need to use a value less than
     // `advisoryTargetSize` as the target size of a coalesced task.
     val totalPostShuffleInputSize = mapOutputStatistics.map(_.bytesByPartitionId.sum).sum

From 74cb5094ec00c359bb70a456d6490f45bdd5ccd7 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 12 Mar 2020 09:06:29 -0700
Subject: [PATCH 0271/1280] [SPARK-31130][BUILD] Use the same version of
 `commons-io` in SBT

### What changes were proposed in this pull request?

This PR (SPARK-31130) aims to pin `Commons IO` version to `2.4` in SBT build like Maven build.

### Why are the changes needed?

[HADOOP-15261](https://issues.apache.org/jira/browse/HADOOP-15261) upgraded `commons-io` from 2.4 to 2.5 at Apache Hadoop 3.1.

In `Maven`, Apache Spark always uses `Commons IO 2.4` based on `pom.xml`.
```
$ git grep commons-io.version
pom.xml:    <commons-io.version>2.4</commons-io.version>
pom.xml:        <version>${commons-io.version}</version>
```

However, `SBT` choose `2.5`.

**branch-3.0**
```
$ build/sbt -Phadoop-3.2 "core/dependencyTree" | grep commons-io:commons-io | head -n1
[info]   | | +-commons-io:commons-io:2.5
```

**branch-2.4**
```
$ build/sbt -Phadoop-3.1 "core/dependencyTree" | grep commons-io:commons-io | head -n1
[info]   | | +-commons-io:commons-io:2.5
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins with `[test-hadoop3.2]` (the default PR Builder is `SBT`) and manually do the following locally.
```
build/sbt -Phadoop-3.2 "core/dependencyTree" | grep commons-io:commons-io | head -n1
```

Closes #27886 from dongjoon-hyun/SPARK-31130.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 972e23d18186c73026ebed95b37a886ca6eecf3e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 project/SparkBuild.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index b606bdd649d66..1a2a7c3066d26 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -621,6 +621,7 @@ object KubernetesIntegrationTests {
 object DependencyOverrides {
   lazy val settings = Seq(
     dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
+    dependencyOverrides += "commons-io" % "commons-io" % "2.4",
     dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
     dependencyOverrides += "jline" % "jline" % "2.14.6")
 }

From 4bcba6fa61e4edac3a616403af35d7e2b093fed3 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 12 Mar 2020 09:24:49 -0700
Subject: [PATCH 0272/1280] [SPARK-31131][SQL] Remove the unnecessary config
 spark.sql.legacy.timeParser.enabled

spark.sql.legacy.timeParser.enabled should be removed from SQLConf and the migration guide
spark.sql.legacy.timeParsePolicy is the right one

fix doc

no

Pass the jenkins

Closes #27889 from yaooqinn/SPARK-31131.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7b4b29e8d955b43daa9ad28667e4fadbb9fce49a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-migration-guide.md                               | 2 +-
 .../scala/org/apache/spark/sql/internal/SQLConf.scala     | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index e7ac9f0fb5bff..108107967b82f 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -67,7 +67,7 @@ license: |
 
   - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
-    - Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. Since Spark 3.0, the conversions are based on `java.time.format.DateTimeFormatter`, see https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and earlier, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html. The old behavior can be restored by setting `spark.sql.legacy.timeParser.enabled` to `true`.
+    - Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. Since Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and earlier, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html. The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
 
     - The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 06180f620219a..ba25a6816f344 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2234,14 +2234,6 @@ object SQLConf {
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
 
-  val LEGACY_TIME_PARSER_ENABLED = buildConf("spark.sql.legacy.timeParser.enabled")
-    .internal()
-    .doc("When set to true, java.text.SimpleDateFormat is used for formatting and parsing " +
-      "dates/timestamps in a locale-sensitive manner. When set to false, classes from " +
-      "java.time.* packages are used for the same purpose.")
-    .booleanConf
-    .createWithDefault(false)
-
   val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")

From fd5692477ca9ba3407a350caba01a6f192d521b2 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 12 Mar 2020 12:59:29 -0700
Subject: [PATCH 0273/1280] [SPARK-31129][SQL][TESTS] Fix IntervalBenchmark and
 DateTimeBenchmark

### What changes were proposed in this pull request?

This PR aims to recover `IntervalBenchmark` and `DataTimeBenchmark` due to banning intervals as output.

### Why are the changes needed?

This PR recovers the benchmark suite.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Manually, re-run the benchmark.

Closes #27885 from yaooqinn/SPARK-31111-2.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit fbc9dc7e9dcde8a77673b1782f4f1141e183ff00)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../DateTimeBenchmark-jdk11-results.txt       | 434 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 434 +++++++++---------
 .../IntervalBenchmark-jdk11-results.txt       |  52 +--
 .../benchmarks/IntervalBenchmark-results.txt  |  52 +--
 .../benchmark/DateTimeBenchmark.scala         |   4 +-
 .../benchmark/IntervalBenchmark.scala         |   4 +-
 6 files changed, 491 insertions(+), 489 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 7d9b147d8d053..883f9de74c22f 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -2,428 +2,428 @@
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    408            445          53         24.5          40.8       1.0X
-cast to timestamp wholestage on                     401            453          63         24.9          40.1       1.0X
+cast to timestamp wholestage off                    221            232          16         45.3          22.1       1.0X
+cast to timestamp wholestage on                     213            256          71         46.9          21.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1197           1246          69          8.4         119.7       1.0X
-year of timestamp wholestage on                    1111           1123          10          9.0         111.1       1.1X
+year of timestamp wholestage off                    863            961         139         11.6          86.3       1.0X
+year of timestamp wholestage on                     783            821          26         12.8          78.3       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1451           1462          16          6.9         145.1       1.0X
-quarter of timestamp wholestage on                 1409           1424          13          7.1         140.9       1.0X
+quarter of timestamp wholestage off                1008           1013           7          9.9         100.8       1.0X
+quarter of timestamp wholestage on                  926            963          36         10.8          92.6       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1106           1109           3          9.0         110.6       1.0X
-month of timestamp wholestage on                   1092           1095           4          9.2         109.2       1.0X
+month of timestamp wholestage off                   794            813          27         12.6          79.4       1.0X
+month of timestamp wholestage on                    737            758          23         13.6          73.7       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1675           1677           3          6.0         167.5       1.0X
-weekofyear of timestamp wholestage on              1616           1625           7          6.2         161.6       1.0X
+weekofyear of timestamp wholestage off             1063           1076          18          9.4         106.3       1.0X
+weekofyear of timestamp wholestage on              1070           1366         305          9.3         107.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1111           1115           6          9.0         111.1       1.0X
-day of timestamp wholestage on                     1090           1098          10          9.2         109.0       1.0X
+day of timestamp wholestage off                     847            854           9         11.8          84.7       1.0X
+day of timestamp wholestage on                      799            836          37         12.5          79.9       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1162           1167           7          8.6         116.2       1.0X
-dayofyear of timestamp wholestage on               1136           1142           6          8.8         113.6       1.0X
+dayofyear of timestamp wholestage off               816            830          20         12.2          81.6       1.0X
+dayofyear of timestamp wholestage on                856            925          62         11.7          85.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1102           1107           7          9.1         110.2       1.0X
-dayofmonth of timestamp wholestage on              1092           1111          16          9.2         109.2       1.0X
+dayofmonth of timestamp wholestage off             1150           1159          12          8.7         115.0       1.0X
+dayofmonth of timestamp wholestage on               945           1165         308         10.6          94.5       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1254           1260           9          8.0         125.4       1.0X
-dayofweek of timestamp wholestage on               1264           1269           7          7.9         126.4       1.0X
+dayofweek of timestamp wholestage off              1062           1074          16          9.4         106.2       1.0X
+dayofweek of timestamp wholestage on                942           1029          74         10.6          94.2       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1211           1215           5          8.3         121.1       1.0X
-weekday of timestamp wholestage on                 1197           1206          12          8.4         119.7       1.0X
+weekday of timestamp wholestage off                 870            872           3         11.5          87.0       1.0X
+weekday of timestamp wholestage on                  821            855          21         12.2          82.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    899            904           6         11.1          89.9       1.0X
-hour of timestamp wholestage on                     823            827           3         12.2          82.3       1.1X
+hour of timestamp wholestage off                    578            582           5         17.3          57.8       1.0X
+hour of timestamp wholestage on                     533            544          14         18.8          53.3       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  906            923          24         11.0          90.6       1.0X
-minute of timestamp wholestage on                   821            830          13         12.2          82.1       1.1X
+minute of timestamp wholestage off                  611            613           3         16.4          61.1       1.0X
+minute of timestamp wholestage on                   551            557           8         18.2          55.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  902            908           7         11.1          90.2       1.0X
-second of timestamp wholestage on                   823            833           9         12.2          82.3       1.1X
+second of timestamp wholestage off                  705            716          16         14.2          70.5       1.0X
+second of timestamp wholestage on                   569            669          99         17.6          56.9       1.2X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         300            301           2         33.3          30.0       1.0X
-current_date wholestage on                          309            312           3         32.4          30.9       1.0X
+current_date wholestage off                         177            177           0         56.6          17.7       1.0X
+current_date wholestage on                          164            180          13         60.9          16.4       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    303            337          47         33.0          30.3       1.0X
-current_timestamp wholestage on                     320            448         160         31.3          32.0       0.9X
+current_timestamp wholestage off                    182            182           1         55.1          18.2       1.0X
+current_timestamp wholestage on                     168            194          53         59.4          16.8       1.1X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         910            912           2         11.0          91.0       1.0X
-cast to date wholestage on                          930            937           6         10.8          93.0       1.0X
+cast to date wholestage off                         615            632          24         16.3          61.5       1.0X
+cast to date wholestage on                          596            660          40         16.8          59.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1122           1123           1          8.9         112.2       1.0X
-last_day wholestage on                             1100           1109           6          9.1         110.0       1.0X
+last_day wholestage off                             717            728          16         13.9          71.7       1.0X
+last_day wholestage on                              764            834          42         13.1          76.4       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             991            992           2         10.1          99.1       1.0X
-next_day wholestage on                              981            984           4         10.2          98.1       1.0X
+next_day wholestage off                             677            677           0         14.8          67.7       1.0X
+next_day wholestage on                              645            796         140         15.5          64.5       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             927            927           1         10.8          92.7       1.0X
-date_add wholestage on                              908            915          12         11.0          90.8       1.0X
+date_add wholestage off                             637            663          37         15.7          63.7       1.0X
+date_add wholestage on                              659            675          16         15.2          65.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             921            921           0         10.9          92.1       1.0X
-date_sub wholestage on                              907            910           4         11.0          90.7       1.0X
+date_sub wholestage off                             630            682          73         15.9          63.0       1.0X
+date_sub wholestage on                              720            762          37         13.9          72.0       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1255           1257           3          8.0         125.5       1.0X
-add_months wholestage on                           1242           1259          13          8.1         124.2       1.0X
+add_months wholestage off                          1064           1088          35          9.4         106.4       1.0X
+add_months wholestage on                            883            909          25         11.3          88.3       1.2X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         4912           4968          79          2.0         491.2       1.0X
-format date wholestage on                          4734           4757          16          2.1         473.4       1.0X
+format date wholestage off                         3667           3854         265          2.7         366.7       1.0X
+format date wholestage on                          3355           3548         195          3.0         335.5       1.1X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8692           8701          13          1.2         869.2       1.0X
-from_unixtime wholestage on                        8717           8730          14          1.1         871.7       1.0X
+from_unixtime wholestage off                       4664           4704          57          2.1         466.4       1.0X
+from_unixtime wholestage on                        4596           4772         193          2.2         459.6       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1339           1348          12          7.5         133.9       1.0X
-from_utc_timestamp wholestage on                   1398           1404           6          7.2         139.8       1.0X
+from_utc_timestamp wholestage off                   810            819          13         12.3          81.0       1.0X
+from_utc_timestamp wholestage on                    795            814          16         12.6          79.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1936           1951          21          5.2         193.6       1.0X
-to_utc_timestamp wholestage on                     1783           1792           8          5.6         178.3       1.1X
+to_utc_timestamp wholestage off                    1130           1146          22          8.8         113.0       1.0X
+to_utc_timestamp wholestage on                     1034           1063          40          9.7         103.4       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        350            353           4         28.6          35.0       1.0X
-cast interval wholestage on                         348            355           4         28.7          34.8       1.0X
+cast interval wholestage off                        192            194           2         52.1          19.2       1.0X
+cast interval wholestage on                         172            180           5         58.2          17.2       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1679           1680           2          6.0         167.9       1.0X
-datediff wholestage on                             1674           1687          11          6.0         167.4       1.0X
+datediff wholestage off                            1070           1082          17          9.3         107.0       1.0X
+datediff wholestage on                             1039           1059          13          9.6         103.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      4116           4120           5          2.4         411.6       1.0X
-months_between wholestage on                       4092           4140          90          2.4         409.2       1.0X
+months_between wholestage off                      3118           3137          27          3.2         311.8       1.0X
+months_between wholestage on                       3079           3122          48          3.2         307.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2236           2396         227          0.4        2235.9       1.0X
-window wholestage on                              47208          47298         171          0.0       47208.3       0.0X
+window wholestage off                              1147           1157          14          0.9        1146.6       1.0X
+window wholestage on                              16698          17304         715          0.1       16698.2       0.1X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1939           1947          11          5.2         193.9       1.0X
-date_trunc YEAR wholestage on                      1890           1899           6          5.3         189.0       1.0X
+date_trunc YEAR wholestage off                     1478           1485           9          6.8         147.8       1.0X
+date_trunc YEAR wholestage on                      1458           1587         213          6.9         145.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1947           1950           5          5.1         194.7       1.0X
-date_trunc YYYY wholestage on                      1903           1909           7          5.3         190.3       1.0X
+date_trunc YYYY wholestage off                     1467           1481          20          6.8         146.7       1.0X
+date_trunc YYYY wholestage on                      1457           1513          52          6.9         145.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1941           1952          15          5.2         194.1       1.0X
-date_trunc YY wholestage on                        1895           1907          10          5.3         189.5       1.0X
+date_trunc YY wholestage off                       1551           1556           7          6.4         155.1       1.0X
+date_trunc YY wholestage on                        1468           1538          78          6.8         146.8       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1912           1934          30          5.2         191.2       1.0X
-date_trunc MON wholestage on                       1899           1923          20          5.3         189.9       1.0X
+date_trunc MON wholestage off                      1481           1493          16          6.8         148.1       1.0X
+date_trunc MON wholestage on                       1476           1501          28          6.8         147.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1923           1931          11          5.2         192.3       1.0X
-date_trunc MONTH wholestage on                     1897           1908          10          5.3         189.7       1.0X
+date_trunc MONTH wholestage off                    1470           1477          10          6.8         147.0       1.0X
+date_trunc MONTH wholestage on                     1449           1610         260          6.9         144.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1923           1924           2          5.2         192.3       1.0X
-date_trunc MM wholestage on                        1902           1911          10          5.3         190.2       1.0X
+date_trunc MM wholestage off                       1487           1495          12          6.7         148.7       1.0X
+date_trunc MM wholestage on                        1467           1489          17          6.8         146.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1713           1717           6          5.8         171.3       1.0X
-date_trunc DAY wholestage on                       1693           1699           4          5.9         169.3       1.0X
+date_trunc DAY wholestage off                      1321           1328          10          7.6         132.1       1.0X
+date_trunc DAY wholestage on                       1279           1306          28          7.8         127.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1711           1712           1          5.8         171.1       1.0X
-date_trunc DD wholestage on                        1691           1697          12          5.9         169.1       1.0X
+date_trunc DD wholestage off                       1366           1368           3          7.3         136.6       1.0X
+date_trunc DD wholestage on                        1297           1314          18          7.7         129.7       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1715           1715           0          5.8         171.5       1.0X
-date_trunc HOUR wholestage on                      1699           1705           4          5.9         169.9       1.0X
+date_trunc HOUR wholestage off                     1338           1354          23          7.5         133.8       1.0X
+date_trunc HOUR wholestage on                      1403           1482          71          7.1         140.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    520            524           7         19.2          52.0       1.0X
-date_trunc MINUTE wholestage on                     500            505           6         20.0          50.0       1.0X
+date_trunc MINUTE wholestage off                    241            242           2         41.5          24.1       1.0X
+date_trunc MINUTE wholestage on                     196            204           6         51.0          19.6       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    501            502           1         20.0          50.1       1.0X
-date_trunc SECOND wholestage on                     497            501           3         20.1          49.7       1.0X
+date_trunc SECOND wholestage off                    211            222          15         47.3          21.1       1.0X
+date_trunc SECOND wholestage on                     205            243          64         48.8          20.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1839           1840           1          5.4         183.9       1.0X
-date_trunc WEEK wholestage on                      1820           1827           8          5.5         182.0       1.0X
+date_trunc WEEK wholestage off                     1492           1515          32          6.7         149.2       1.0X
+date_trunc WEEK wholestage on                      1465           1490          39          6.8         146.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  2624           2624           0          3.8         262.4       1.0X
-date_trunc QUARTER wholestage on                   2573           2583           7          3.9         257.3       1.0X
+date_trunc QUARTER wholestage off                  1908           1982         104          5.2         190.8       1.0X
+date_trunc QUARTER wholestage on                   1879           2020          99          5.3         187.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           315            317           3         31.8          31.5       1.0X
-trunc year wholestage on                            314            328          16         31.9          31.4       1.0X
+trunc year wholestage off                           326            419         132         30.7          32.6       1.0X
+trunc year wholestage on                            187            199           8         53.4          18.7       1.7X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           314            317           4         31.8          31.4       1.0X
-trunc yyyy wholestage on                            312            315           3         32.1          31.2       1.0X
+trunc yyyy wholestage off                           194            194           0         51.6          19.4       1.0X
+trunc yyyy wholestage on                            192            207          10         52.1          19.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             314            315           0         31.8          31.4       1.0X
-trunc yy wholestage on                              312            317           6         32.0          31.2       1.0X
+trunc yy wholestage off                             177            180           4         56.5          17.7       1.0X
+trunc yy wholestage on                              185            193           9         53.9          18.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            314            315           1         31.8          31.4       1.0X
-trunc mon wholestage on                             310            314           4         32.2          31.0       1.0X
+trunc mon wholestage off                            208            210           3         48.0          20.8       1.0X
+trunc mon wholestage on                             195            204           8         51.3          19.5       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          313            314           2         32.0          31.3       1.0X
-trunc month wholestage on                           312            316           6         32.1          31.2       1.0X
+trunc month wholestage off                          182            185           4         55.1          18.2       1.0X
+trunc month wholestage on                           185            222          70         54.0          18.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             312            313           1         32.0          31.2       1.0X
-trunc mm wholestage on                              309            312           3         32.3          30.9       1.0X
+trunc mm wholestage off                             208            218          14         48.0          20.8       1.0X
+trunc mm wholestage on                              197            206          10         50.8          19.7       1.1X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     170            173           5          5.9         170.0       1.0X
-to timestamp str wholestage on                      156            161           5          6.4         155.9       1.1X
+to timestamp str wholestage off                     121            130          13          8.3         120.7       1.0X
+to timestamp str wholestage on                      128            134           6          7.8         128.4       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1741           1746           8          0.6        1740.7       1.0X
-to_timestamp wholestage on                         1887           1896          11          0.5        1886.6       0.9X
+to_timestamp wholestage off                        1201           1272         100          0.8        1200.8       1.0X
+to_timestamp wholestage on                          973           1019          53          1.0         972.7       1.2X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1928           1934           9          0.5        1927.7       1.0X
-to_unix_timestamp wholestage on                    1857           1862           3          0.5        1857.5       1.0X
+to_unix_timestamp wholestage off                   1002           1077         106          1.0        1001.8       1.0X
+to_unix_timestamp wholestage on                     988           1079         100          1.0         987.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          161            162           0          6.2         161.5       1.0X
-to date str wholestage on                           151            156           3          6.6         151.4       1.1X
+to date str wholestage off                          157            165          12          6.4         157.2       1.0X
+to date str wholestage on                           137            140           3          7.3         137.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             2513           2519           8          0.4        2513.1       1.0X
-to_date wholestage on                              2484           2500          15          0.4        2484.2       1.0X
+to_date wholestage off                             1597           1614          25          0.6        1597.0       1.0X
+to_date wholestage on                              1503           1573          58          0.7        1502.8       1.1X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             340            347           8         14.7          68.0       1.0X
-Collect longs                                      1170           1217          42          4.3         234.1       0.3X
-Collect timestamps                                 1771           2267         836          2.8         354.1       0.2X
+From java.sql.Timestamp                             197            202           6         25.4          39.4       1.0X
+Collect longs                                       794            911         110          6.3         158.7       0.2X
+Collect timestamps                                 1030           1079          51          4.9         205.9       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 3567a60d1b0b8..2ea26c327afae 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -2,428 +2,428 @@
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    447            462          21         22.4          44.7       1.0X
-cast to timestamp wholestage on                     390            426          54         25.7          39.0       1.1X
+cast to timestamp wholestage off                    230            240          13         43.5          23.0       1.0X
+cast to timestamp wholestage on                     194            208          20         51.4          19.4       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1189           1285         135          8.4         118.9       1.0X
-year of timestamp wholestage on                    1146           1156           9          8.7         114.6       1.0X
+year of timestamp wholestage off                    806            822          22         12.4          80.6       1.0X
+year of timestamp wholestage on                     748            761          13         13.4          74.8       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1290           1293           4          7.8         129.0       1.0X
-quarter of timestamp wholestage on                 1237           1251          13          8.1         123.7       1.0X
+quarter of timestamp wholestage off                 828            832           5         12.1          82.8       1.0X
+quarter of timestamp wholestage on                  821            858          45         12.2          82.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1096           1101           7          9.1         109.6       1.0X
-month of timestamp wholestage on                   1088           1095           7          9.2         108.8       1.0X
+month of timestamp wholestage off                   709            713           5         14.1          70.9       1.0X
+month of timestamp wholestage on                    714            722          10         14.0          71.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1635           1636           1          6.1         163.5       1.0X
-weekofyear of timestamp wholestage on              1711           1714           4          5.8         171.1       1.0X
+weekofyear of timestamp wholestage off             1217           1220           4          8.2         121.7       1.0X
+weekofyear of timestamp wholestage on              1019           1043          24          9.8         101.9       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1094           1108          20          9.1         109.4       1.0X
-day of timestamp wholestage on                     1083           1092           8          9.2         108.3       1.0X
+day of timestamp wholestage off                     706            712           9         14.2          70.6       1.0X
+day of timestamp wholestage on                      694            704           8         14.4          69.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1145           1145           1          8.7         114.5       1.0X
-dayofyear of timestamp wholestage on               1131           1141          12          8.8         113.1       1.0X
+dayofyear of timestamp wholestage off               717            728          16         14.0          71.7       1.0X
+dayofyear of timestamp wholestage on                724            736          10         13.8          72.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1112           1124          17          9.0         111.2       1.0X
-dayofmonth of timestamp wholestage on              1082           1096          14          9.2         108.2       1.0X
+dayofmonth of timestamp wholestage off              719            721           3         13.9          71.9       1.0X
+dayofmonth of timestamp wholestage on               706            714           8         14.2          70.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1254           1255           2          8.0         125.4       1.0X
-dayofweek of timestamp wholestage on               1250           1260          10          8.0         125.0       1.0X
+dayofweek of timestamp wholestage off               904            979         106         11.1          90.4       1.0X
+dayofweek of timestamp wholestage on                796            819          14         12.6          79.6       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1190           1196           8          8.4         119.0       1.0X
-weekday of timestamp wholestage on                 1204           1218          11          8.3         120.4       1.0X
+weekday of timestamp wholestage off                 813            819           9         12.3          81.3       1.0X
+weekday of timestamp wholestage on                  788            809          13         12.7          78.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    859            862           4         11.6          85.9       1.0X
-hour of timestamp wholestage on                     825            827           2         12.1          82.5       1.0X
+hour of timestamp wholestage off                    567            582          21         17.6          56.7       1.0X
+hour of timestamp wholestage on                     539            551          11         18.6          53.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  841            851          15         11.9          84.1       1.0X
-minute of timestamp wholestage on                   817            824           6         12.2          81.7       1.0X
+minute of timestamp wholestage off                  554            563          13         18.1          55.4       1.0X
+minute of timestamp wholestage on                   520            531           7         19.2          52.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  901            905           5         11.1          90.1       1.0X
-second of timestamp wholestage on                   830            844          12         12.1          83.0       1.1X
+second of timestamp wholestage off                  651            654           5         15.4          65.1       1.0X
+second of timestamp wholestage on                   535            546          12         18.7          53.5       1.2X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         290            292           2         34.4          29.0       1.0X
-current_date wholestage on                          301            309          10         33.3          30.1       1.0X
+current_date wholestage off                         172            175           4         58.0          17.2       1.0X
+current_date wholestage on                          174            177           3         57.5          17.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    300            301           0         33.3          30.0       1.0X
-current_timestamp wholestage on                     316            348          36         31.6          31.6       0.9X
+current_timestamp wholestage off                    184            185           0         54.3          18.4       1.0X
+current_timestamp wholestage on                     196            212          14         51.0          19.6       0.9X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         964            973          13         10.4          96.4       1.0X
-cast to date wholestage on                          900            905           4         11.1          90.0       1.1X
+cast to date wholestage off                         652            662          13         15.3          65.2       1.0X
+cast to date wholestage on                          645            690          29         15.5          64.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1125           1138          19          8.9         112.5       1.0X
-last_day wholestage on                             1111           1122          12          9.0         111.1       1.0X
+last_day wholestage off                             801            808          11         12.5          80.1       1.0X
+last_day wholestage on                              765            829          37         13.1          76.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             970            982          16         10.3          97.0       1.0X
-next_day wholestage on                              955            958           4         10.5          95.5       1.0X
+next_day wholestage off                             676            676           0         14.8          67.6       1.0X
+next_day wholestage on                              674            708          33         14.8          67.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             894            914          29         11.2          89.4       1.0X
-date_add wholestage on                              880            884           4         11.4          88.0       1.0X
+date_add wholestage off                             644            646           2         15.5          64.4       1.0X
+date_add wholestage on                              640            661          32         15.6          64.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             901            901           1         11.1          90.1       1.0X
-date_sub wholestage on                              883            892          16         11.3          88.3       1.0X
+date_sub wholestage off                             704            718          20         14.2          70.4       1.0X
+date_sub wholestage on                              684            728          34         14.6          68.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1212           1213           0          8.2         121.2       1.0X
-add_months wholestage on                           1212           1219          11          8.3         121.2       1.0X
+add_months wholestage off                          1011           1017           8          9.9         101.1       1.0X
+add_months wholestage on                            837            860          25         11.9          83.7       1.2X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         4973           5069         136          2.0         497.3       1.0X
-format date wholestage on                          5061           5075          18          2.0         506.1       1.0X
+format date wholestage off                         3467           3591         176          2.9         346.7       1.0X
+format date wholestage on                          3417           3482          66          2.9         341.7       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8827           8835          11          1.1         882.7       1.0X
-from_unixtime wholestage on                        8840           8845           5          1.1         884.0       1.0X
+from_unixtime wholestage off                       4823           4850          38          2.1         482.3       1.0X
+from_unixtime wholestage on                        4774           4811          29          2.1         477.4       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1143           1145           2          8.7         114.3       1.0X
-from_utc_timestamp wholestage on                   1174           1187          12          8.5         117.4       1.0X
+from_utc_timestamp wholestage off                   703            717          20         14.2          70.3       1.0X
+from_utc_timestamp wholestage on                    665            671           7         15.0          66.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1721           1725           7          5.8         172.1       1.0X
-to_utc_timestamp wholestage on                     1605           1613           5          6.2         160.5       1.1X
+to_utc_timestamp wholestage off                    1000           1016          22         10.0         100.0       1.0X
+to_utc_timestamp wholestage on                      917            933          11         10.9          91.7       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        342            345           5         29.3          34.2       1.0X
-cast interval wholestage on                         329            336          13         30.4          32.9       1.0X
+cast interval wholestage off                        195            198           3         51.2          19.5       1.0X
+cast interval wholestage on                         176            181           5         56.9          17.6       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1643           1645           3          6.1         164.3       1.0X
-datediff wholestage on                             1645           1657           8          6.1         164.5       1.0X
+datediff wholestage off                            1022           1025           5          9.8         102.2       1.0X
+datediff wholestage on                             1003           1015          10         10.0         100.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      3369           3375           8          3.0         336.9       1.0X
-months_between wholestage on                       3356           3362           5          3.0         335.6       1.0X
+months_between wholestage off                      2562           2579          24          3.9         256.2       1.0X
+months_between wholestage on                       2503           2524          17          4.0         250.3       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2017           2029          18          0.5        2016.8       1.0X
-window wholestage on                              44825          44909          56          0.0       44824.9       0.0X
+window wholestage off                              1067           1077          15          0.9        1066.6       1.0X
+window wholestage on                              14503          19089         NaN          0.1       14502.6       0.1X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1628           1629           0          6.1         162.8       1.0X
-date_trunc YEAR wholestage on                      1589           1599           9          6.3         158.9       1.0X
+date_trunc YEAR wholestage off                     1497           1543          64          6.7         149.7       1.0X
+date_trunc YEAR wholestage on                      1412           1533         138          7.1         141.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1634           1646          17          6.1         163.4       1.0X
-date_trunc YYYY wholestage on                      1596           1606          13          6.3         159.6       1.0X
+date_trunc YYYY wholestage off                     1725           1729           5          5.8         172.5       1.0X
+date_trunc YYYY wholestage on                      1413           1481          56          7.1         141.3       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1627           1627           0          6.1         162.7       1.0X
-date_trunc YY wholestage on                        1590           1598          10          6.3         159.0       1.0X
+date_trunc YY wholestage off                       1456           1459           5          6.9         145.6       1.0X
+date_trunc YY wholestage on                        1369           1380           7          7.3         136.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1633           1634           1          6.1         163.3       1.0X
-date_trunc MON wholestage on                       1602           1607           5          6.2         160.2       1.0X
+date_trunc MON wholestage off                      1459           1471          17          6.9         145.9       1.0X
+date_trunc MON wholestage on                       1359           1377          21          7.4         135.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1637           1644          11          6.1         163.7       1.0X
-date_trunc MONTH wholestage on                     1601           1609           5          6.2         160.1       1.0X
+date_trunc MONTH wholestage off                    1455           1480          35          6.9         145.5       1.0X
+date_trunc MONTH wholestage on                     1344           1468         170          7.4         134.4       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1639           1653          20          6.1         163.9       1.0X
-date_trunc MM wholestage on                        1602           1606           4          6.2         160.2       1.0X
+date_trunc MM wholestage off                       1437           1463          37          7.0         143.7       1.0X
+date_trunc MM wholestage on                        1273           1285           9          7.9         127.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1635           1643          11          6.1         163.5       1.0X
-date_trunc DAY wholestage on                       1511           1517           6          6.6         151.1       1.1X
+date_trunc DAY wholestage off                      1198           1209          16          8.3         119.8       1.0X
+date_trunc DAY wholestage on                       1137           1192          70          8.8         113.7       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1640           1646           8          6.1         164.0       1.0X
-date_trunc DD wholestage on                        1516           1519           4          6.6         151.6       1.1X
+date_trunc DD wholestage off                       1201           1215          20          8.3         120.1       1.0X
+date_trunc DD wholestage on                        1132           1144          14          8.8         113.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1533           1535           4          6.5         153.3       1.0X
-date_trunc HOUR wholestage on                      1532           1538           7          6.5         153.2       1.0X
+date_trunc HOUR wholestage off                     1236           1238           3          8.1         123.6       1.0X
+date_trunc HOUR wholestage on                      1134           1150          15          8.8         113.4       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    381            382           1         26.2          38.1       1.0X
-date_trunc MINUTE wholestage on                     337            343           9         29.7          33.7       1.1X
+date_trunc MINUTE wholestage off                    206            207           2         48.6          20.6       1.0X
+date_trunc MINUTE wholestage on                     185            186           2         54.0          18.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    376            376           1         26.6          37.6       1.0X
-date_trunc SECOND wholestage on                     341            344           6         29.4          34.1       1.1X
+date_trunc SECOND wholestage off                    209            210           2         47.9          20.9       1.0X
+date_trunc SECOND wholestage on                     174            185           7         57.4          17.4       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1556           1556           0          6.4         155.6       1.0X
-date_trunc WEEK wholestage on                      1498           1507           9          6.7         149.8       1.0X
+date_trunc WEEK wholestage off                     1231           1293          86          8.1         123.1       1.0X
+date_trunc WEEK wholestage on                      1186           1219          35          8.4         118.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  2340           2346           9          4.3         234.0       1.0X
-date_trunc QUARTER wholestage on                   2309           2315           4          4.3         230.9       1.0X
+date_trunc QUARTER wholestage off                  1785           1792          10          5.6         178.5       1.0X
+date_trunc QUARTER wholestage on                   1671           1708          28          6.0         167.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           323            323           0         31.0          32.3       1.0X
-trunc year wholestage on                            304            307           4         32.9          30.4       1.1X
+trunc year wholestage off                           186            199          18         53.8          18.6       1.0X
+trunc year wholestage on                            161            165           5         62.3          16.1       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           323            324           2         31.0          32.3       1.0X
-trunc yyyy wholestage on                            303            313          12         33.0          30.3       1.1X
+trunc yyyy wholestage off                           184            184           0         54.4          18.4       1.0X
+trunc yyyy wholestage on                            156            162           6         64.3          15.6       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             325            327           2         30.8          32.5       1.0X
-trunc yy wholestage on                              304            308           3         32.9          30.4       1.1X
+trunc yy wholestage off                             171            174           5         58.6          17.1       1.0X
+trunc yy wholestage on                              161            172          10         62.1          16.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            327            351          34         30.5          32.7       1.0X
-trunc mon wholestage on                             301            302           2         33.3          30.1       1.1X
+trunc mon wholestage off                            175            176           1         57.1          17.5       1.0X
+trunc mon wholestage on                             166            167           1         60.3          16.6       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          319            319           1         31.4          31.9       1.0X
-trunc month wholestage on                           301            305           4         33.2          30.1       1.1X
+trunc month wholestage off                          177            186          12         56.4          17.7       1.0X
+trunc month wholestage on                           160            168           8         62.4          16.0       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             319            320           1         31.3          31.9       1.0X
-trunc mm wholestage on                              300            307           7         33.3          30.0       1.1X
+trunc mm wholestage off                             173            175           2         57.8          17.3       1.0X
+trunc mm wholestage on                              167            178          10         59.9          16.7       1.0X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     218            219           2          4.6         217.7       1.0X
-to timestamp str wholestage on                      212            217           5          4.7         212.4       1.0X
+to timestamp str wholestage off                     166            168           3          6.0         166.3       1.0X
+to timestamp str wholestage on                      170            192          37          5.9         169.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        2063           2066           5          0.5        2062.5       1.0X
-to_timestamp wholestage on                         2000           2014           8          0.5        2000.3       1.0X
+to_timestamp wholestage off                         975           1042          95          1.0         975.1       1.0X
+to_timestamp wholestage on                          976            982           5          1.0         976.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   2038           2040           4          0.5        2037.7       1.0X
-to_unix_timestamp wholestage on                    2021           2029          11          0.5        2020.5       1.0X
+to_unix_timestamp wholestage off                    978            984           8          1.0         978.4       1.0X
+to_unix_timestamp wholestage on                     956            979          20          1.0         955.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          209            215           8          4.8         209.0       1.0X
-to date str wholestage on                           203            205           2          4.9         203.1       1.0X
+to date str wholestage off                          169            173           5          5.9         168.9       1.0X
+to date str wholestage on                           155            159           4          6.4         155.4       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             2191           2201          14          0.5        2191.3       1.0X
-to_date wholestage on                              2124           2137          16          0.5        2124.4       1.0X
+to_date wholestage off                             1583           1594          15          0.6        1583.2       1.0X
+to_date wholestage on                              1586           1635          52          0.6        1586.1       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             284            289           5         17.6          56.9       1.0X
-Collect longs                                      1413           2057         930          3.5         282.6       0.2X
-Collect timestamps                                 1605           1712          93          3.1         321.0       0.2X
+From java.sql.Timestamp                             174            176           3         28.8          34.7       1.0X
+Collect longs                                       945           1275         538          5.3         189.0       0.2X
+Collect timestamps                                 1019           1422         633          4.9         203.8       0.2X
 
 
diff --git a/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt b/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
index 7d23e5467baed..8958d7c53413f 100644
--- a/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/IntervalBenchmark-jdk11-results.txt
@@ -1,29 +1,29 @@
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast strings to intervals:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare string w/ interval                          756            831          78          1.3         756.4       1.0X
-prepare string w/o interval                         622            637          15          1.6         622.4       1.2X
-1 units w/ interval                                 506            532          22          2.0         506.2       1.5X
-1 units w/o interval                                472            477           9          2.1         471.6       1.6X
-2 units w/ interval                                 699            710          11          1.4         698.5       1.1X
-2 units w/o interval                                670            674           6          1.5         669.9       1.1X
-3 units w/ interval                                1431           1437           7          0.7        1431.3       0.5X
-3 units w/o interval                               1418           1429          11          0.7        1418.2       0.5X
-4 units w/ interval                                1687           1692           8          0.6        1686.8       0.4X
-4 units w/o interval                               1679           1688           7          0.6        1679.4       0.5X
-5 units w/ interval                                1862           1864           3          0.5        1861.9       0.4X
-5 units w/o interval                               1847           1864          15          0.5        1846.9       0.4X
-6 units w/ interval                                2067           2081          12          0.5        2066.9       0.4X
-6 units w/o interval                               2070           2071           2          0.5        2069.6       0.4X
-7 units w/ interval                                2458           2468          13          0.4        2457.7       0.3X
-7 units w/o interval                               2450           2453           3          0.4        2450.1       0.3X
-8 units w/ interval                                2833           2838           8          0.4        2832.6       0.3X
-8 units w/o interval                               2830           2839           8          0.4        2829.8       0.3X
-9 units w/ interval                                2873           2880           6          0.3        2873.4       0.3X
-9 units w/o interval                               2860           2863           3          0.3        2860.1       0.3X
-10 units w/ interval                               3252           3257           5          0.3        3252.2       0.2X
-10 units w/o interval                              3212           3220           8          0.3        3211.6       0.2X
-11 units w/ interval                               3369           3376           6          0.3        3368.5       0.2X
-11 units w/o interval                              3384           3395          15          0.3        3384.2       0.2X
+prepare string w/ interval                          448            469          20          2.2         447.6       1.0X
+prepare string w/o interval                         405            409           4          2.5         404.6       1.1X
+1 units w/ interval                                 321            328           6          3.1         321.4       1.4X
+1 units w/o interval                                303            307           4          3.3         303.1       1.5X
+2 units w/ interval                                 445            458          12          2.2         444.6       1.0X
+2 units w/o interval                                416            424          10          2.4         416.2       1.1X
+3 units w/ interval                                1006           1012           8          1.0        1006.4       0.4X
+3 units w/o interval                               1240           1249           8          0.8        1239.6       0.4X
+4 units w/ interval                                1295           1418         106          0.8        1295.4       0.3X
+4 units w/o interval                               1172           1188          15          0.9        1171.6       0.4X
+5 units w/ interval                                1326           1335          11          0.8        1325.6       0.3X
+5 units w/o interval                               1309           1336          44          0.8        1308.7       0.3X
+6 units w/ interval                                1441           1464          29          0.7        1441.0       0.3X
+6 units w/o interval                               1350           1369          17          0.7        1350.1       0.3X
+7 units w/ interval                                1606           1669          99          0.6        1605.6       0.3X
+7 units w/o interval                               1546           1557          12          0.6        1546.3       0.3X
+8 units w/ interval                                1771           1875         120          0.6        1770.6       0.3X
+8 units w/o interval                               1775           1789          13          0.6        1775.2       0.3X
+9 units w/ interval                                2126           2757         849          0.5        2126.4       0.2X
+9 units w/o interval                               2053           2070          21          0.5        2053.3       0.2X
+10 units w/ interval                               2209           2243          30          0.5        2209.1       0.2X
+10 units w/o interval                              2400           2702         365          0.4        2400.2       0.2X
+11 units w/ interval                               2616           2699          72          0.4        2616.5       0.2X
+11 units w/o interval                              3218           3380         195          0.3        3218.4       0.1X
 
diff --git a/sql/core/benchmarks/IntervalBenchmark-results.txt b/sql/core/benchmarks/IntervalBenchmark-results.txt
index da2f0e66dacac..48af333b78ba4 100644
--- a/sql/core/benchmarks/IntervalBenchmark-results.txt
+++ b/sql/core/benchmarks/IntervalBenchmark-results.txt
@@ -1,29 +1,29 @@
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast strings to intervals:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-prepare string w/ interval                          648            721          94          1.5         648.3       1.0X
-prepare string w/o interval                         562            596          49          1.8         562.3       1.2X
-1 units w/ interval                                 568            590          21          1.8         568.5       1.1X
-1 units w/o interval                                522            538          20          1.9         521.7       1.2X
-2 units w/ interval                                 751            754           3          1.3         751.5       0.9X
-2 units w/o interval                                716            723           6          1.4         716.1       0.9X
-3 units w/ interval                                1402           1411          11          0.7        1401.6       0.5X
-3 units w/o interval                               1381           1387           5          0.7        1381.2       0.5X
-4 units w/ interval                                1591           1595           6          0.6        1591.2       0.4X
-4 units w/o interval                               1582           1585           3          0.6        1582.3       0.4X
-5 units w/ interval                                1747           1749           2          0.6        1747.3       0.4X
-5 units w/o interval                               1738           1746          10          0.6        1737.7       0.4X
-6 units w/ interval                                1929           1931           3          0.5        1929.1       0.3X
-6 units w/o interval                               1919           1922           2          0.5        1919.0       0.3X
-7 units w/ interval                                2345           2354           8          0.4        2345.0       0.3X
-7 units w/o interval                               2334           2336           2          0.4        2334.1       0.3X
-8 units w/ interval                                2533           2546          16          0.4        2533.0       0.3X
-8 units w/o interval                               2519           2521           1          0.4        2519.4       0.3X
-9 units w/ interval                                2885           2889           5          0.3        2884.5       0.2X
-9 units w/o interval                               2804           2813          12          0.4        2803.9       0.2X
-10 units w/ interval                               3041           3060          16          0.3        3041.3       0.2X
-10 units w/o interval                              3031           3043          15          0.3        3031.2       0.2X
-11 units w/ interval                               3270           3280           9          0.3        3269.9       0.2X
-11 units w/o interval                              3273           3280           7          0.3        3272.6       0.2X
+prepare string w/ interval                          389            410          21          2.6         388.7       1.0X
+prepare string w/o interval                         340            360          18          2.9         340.5       1.1X
+1 units w/ interval                                 378            389          16          2.6         377.8       1.0X
+1 units w/o interval                                346            350           5          2.9         346.2       1.1X
+2 units w/ interval                                 444            457          11          2.3         444.2       0.9X
+2 units w/o interval                                455            464          12          2.2         455.1       0.9X
+3 units w/ interval                                 942            964          20          1.1         941.5       0.4X
+3 units w/o interval                                927           1020          93          1.1         927.3       0.4X
+4 units w/ interval                                1114           1127          17          0.9        1113.9       0.3X
+4 units w/o interval                               1100           1105           4          0.9        1100.3       0.4X
+5 units w/ interval                                1180           1244          57          0.8        1180.1       0.3X
+5 units w/o interval                               1135           1141           6          0.9        1135.2       0.3X
+6 units w/ interval                                1284           1316          48          0.8        1284.0       0.3X
+6 units w/o interval                               1276           1357         122          0.8        1276.1       0.3X
+7 units w/ interval                                1609           1636          32          0.6        1609.1       0.2X
+7 units w/o interval                               1551           1578          36          0.6        1550.9       0.3X
+8 units w/ interval                                1787           1874         129          0.6        1787.1       0.2X
+8 units w/o interval                               1751           1767          15          0.6        1750.6       0.2X
+9 units w/ interval                                1960           2065         141          0.5        1959.7       0.2X
+9 units w/o interval                               1885           1908          39          0.5        1885.1       0.2X
+10 units w/ interval                               2178           2185          11          0.5        2177.9       0.2X
+10 units w/o interval                              2150           2255         164          0.5        2150.1       0.2X
+11 units w/ interval                               2457           2542         139          0.4        2456.7       0.2X
+11 units w/o interval                              2557           2770         188          0.4        2556.7       0.2X
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 086583fdafe6d..8de9b8d076e0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -114,12 +114,12 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
     }
     runBenchmark("Parsing") {
       val n = 1000000
-      val timestampStrExpr = "concat('2019-01-27 11:02:01.', cast(mod(id, 1000) as string))"
+      val timestampStrExpr = "concat('2019-01-27 11:02:01.', rpad(mod(id, 1000), 3, '0'))"
       val pattern = "'yyyy-MM-dd HH:mm:ss.SSS'"
       run(n, "to timestamp str", timestampStrExpr)
       run(n, "to_timestamp", s"to_timestamp($timestampStrExpr, $pattern)")
       run(n, "to_unix_timestamp", s"to_unix_timestamp($timestampStrExpr, $pattern)")
-      val dateStrExpr = "concat('2019-01-', cast(mod(id, 25) as string))"
+      val dateStrExpr = "concat('2019-01-', lpad(mod(id, 25), 2, '0'))"
       run(n, "to date str", dateStrExpr)
       run(n, "to_date", s"to_date($dateStrExpr, 'yyyy-MM-dd')")
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
index 94e763459a111..907e3f40c1911 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/IntervalBenchmark.scala
@@ -44,7 +44,9 @@ object IntervalBenchmark extends SqlBasedBenchmark {
       spark
         .range(0, cardinality, 1, 1)
         .select(exprs: _*)
-        .noop()
+        .queryExecution
+        .toRdd
+        .foreach(_ => ())
     }
   }
 

From 6770a4140312e6a37166766072c538f36cc4a374 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 13 Mar 2020 10:47:36 +0900
Subject: [PATCH 0274/1280] [SPARK-31090][SPARK-25457] Revert "IntegralDivide
 returns data type of the operands"

This reverts commit 47d6e80a2e64823fabb596503fb6a6cc6f51f713.

There is no standard requiring that `div` must return the type of the operand, and always returning long type looks fine. This is kind of a cosmetic change and we should avoid it if it breaks existing queries. This is similar to reverting TRIM function parameter order change.

Yes, change the behavior of `div` back to be the same as 2.4.

N/A

Closes #27835 from cloud-fan/revert2.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b27b3c91f113ec49ee87c877dac0a602849d76b1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../catalyst/analysis/DecimalPrecision.scala  |   5 +-
 .../sql/catalyst/expressions/arithmetic.scala |  42 ++---
 .../apache/spark/sql/internal/SQLConf.scala   |  10 --
 .../ArithmeticExpressionSuite.scala           |  64 +++-----
 .../sql-tests/inputs/operator-div.sql         |  21 ---
 .../resources/sql-tests/inputs/operators.sql  |  10 ++
 .../sql-tests/results/operator-div.sql.out    | 146 ------------------
 .../sql-tests/results/operators.sql.out       |  66 +++++++-
 .../results/postgreSQL/numeric.sql.out        |   8 +-
 9 files changed, 110 insertions(+), 262 deletions(-)
 delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/operator-div.sql
 delete mode 100644 sql/core/src/test/resources/sql-tests/results/operator-div.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
index 91489b3e46ff3..f2d607e5b737c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecision.scala
@@ -175,12 +175,11 @@ object DecimalPrecision extends TypeCoercionRule {
         resultType, nullOnOverflow)
 
     case expr @ IntegralDivide(
-        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2), returnLong) =>
+        e1 @ DecimalType.Expression(p1, s1), e2 @ DecimalType.Expression(p2, s2)) =>
       val widerType = widerDecimalType(p1, s1, p2, s2)
       val promotedExpr = IntegralDivide(
         promotePrecision(e1, widerType),
-        promotePrecision(e2, widerType),
-        returnLong)
+        promotePrecision(e2, widerType))
       if (expr.dataType.isInstanceOf[DecimalType]) {
         // This follows division rule
         val intDig = p1 - s1 + s2
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 624891c619d41..215e88a2cc8a4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -395,7 +395,7 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
 
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long if spark.sql.legacy.integralDivide.returnBigint is true, otherwise the data type of the operands is returned.",
+  usage = "expr1 _FUNC_ expr2 - Divide `expr1` by `expr2`. It returns NULL if an operand is NULL or `expr2` is 0. The result is casted to long.",
   examples = """
     Examples:
       > SELECT 3 _FUNC_ 2;
@@ -405,31 +405,15 @@ case class Divide(left: Expression, right: Expression) extends DivModLike {
 // scalastyle:on line.size.limit
 case class IntegralDivide(
     left: Expression,
-    right: Expression,
-    returnLong: Boolean) extends DivModLike {
-
-  def this(left: Expression, right: Expression) = {
-    this(left, right, SQLConf.get.integralDivideReturnLong)
-  }
+    right: Expression) extends DivModLike {
 
   override def inputType: AbstractDataType = TypeCollection(IntegralType, DecimalType)
 
-  override def dataType: DataType = if (returnLong) {
-    LongType
-  } else {
-    left.dataType
-  }
+  override def dataType: DataType = LongType
 
   override def symbol: String = "/"
   override def decimalMethod: String = "quot"
-  override def decimalToDataTypeCodeGen(decimalResult: String): String = {
-    if (returnLong) {
-      s"$decimalResult.toLong()"
-    } else {
-      decimalResult
-    }
-  }
-
+  override def decimalToDataTypeCodeGen(decimalResult: String): String = s"$decimalResult.toLong()"
   override def sqlOperator: String = "div"
 
   private lazy val div: (Any, Any) => Any = {
@@ -439,19 +423,13 @@ case class IntegralDivide(
       case d: DecimalType =>
         d.asIntegral.asInstanceOf[Integral[Any]]
     }
-    val divide = integral.quot _
-    if (returnLong) {
-      val toLong = integral.asInstanceOf[Integral[Any]].toLong _
-      (x, y) => {
-        val res = divide(x, y)
-        if (res == null) {
-          null
-        } else {
-          toLong(res)
-        }
+    (x, y) => {
+      val res = integral.quot(x, y)
+      if (res == null) {
+        null
+      } else {
+        integral.asInstanceOf[Integral[Any]].toLong(res)
       }
-    } else {
-      divide
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index ba25a6816f344..e49593ef8518f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1988,14 +1988,6 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-
-  val LEGACY_INTEGRALDIVIDE_RETURN_LONG = buildConf("spark.sql.legacy.integralDivide.returnBigint")
-    .doc("If it is set to true, the div operator returns always a bigint. This behavior was " +
-      "inherited from Hive. Otherwise, the return type is the data type of the operands.")
-    .internal()
-    .booleanConf
-    .createWithDefault(false)
-
   val LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING =
     buildConf("spark.sql.legacy.bucketedTableScan.outputOrdering")
       .internal()
@@ -2793,8 +2785,6 @@ class SQLConf extends Serializable with Logging {
   def createHiveTableByDefaultEnabled: Boolean =
     getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
 
-  def integralDivideReturnLong: Boolean = getConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG)
-
   def truncateTableIgnorePermissionAcl: Boolean =
     getConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
index b4a1ae20c3e11..675f85f9e82ea 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
@@ -174,24 +174,13 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("/ (Divide) for integral type") {
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "false") {
-      checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0.toByte)
-      checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0.toShort)
-      checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0)
-      checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0.toLong)
-      checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0.toShort)
-      checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0)
-      checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
-    }
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "true") {
-      checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0L)
-      checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0L)
-      checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0L)
-      checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0L)
-      checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0L)
-      checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0L)
-      checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
-    }
+    checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0L)
+    checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0L)
+    checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0L)
+    checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0L)
+    checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0L)
+    checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0L)
+    checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
   }
 
   test("% (Remainder)") {
@@ -407,33 +396,18 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("SPARK-28322: IntegralDivide supports decimal type") {
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "false") {
-      checkEvaluation(IntegralDivide(Literal(Decimal(1)), Literal(Decimal(2))), Decimal(0))
-      checkEvaluation(IntegralDivide(Literal(Decimal(2.4)), Literal(Decimal(1.1))), Decimal(2))
-      checkEvaluation(IntegralDivide(Literal(Decimal(1.2)), Literal(Decimal(1.1))), Decimal(1))
-      checkEvaluation(IntegralDivide(Literal(Decimal(0.2)), Literal(Decimal(0.0))), null)
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999")), Literal(Decimal(0.001)))),
-        BigDecimal("99999999999999999999999999999999999000"))
-      // overflow during promote precision
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999999")), Literal(Decimal(0.00001)))),
-        null)
-    }
-    withSQLConf(SQLConf.LEGACY_INTEGRALDIVIDE_RETURN_LONG.key -> "true") {
-      checkEvaluation(IntegralDivide(Literal(Decimal(1)), Literal(Decimal(2))), 0L)
-      checkEvaluation(IntegralDivide(Literal(Decimal(2.4)), Literal(Decimal(1.1))), 2L)
-      checkEvaluation(IntegralDivide(Literal(Decimal(1.2)), Literal(Decimal(1.1))), 1L)
-      checkEvaluation(IntegralDivide(Literal(Decimal(0.2)), Literal(Decimal(0.0))), null)
-      // overflows long and so returns a wrong result
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999")), Literal(Decimal(0.001)))),
-        687399551400672280L)
-      // overflow during promote precision
-      checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
-        Literal(Decimal("99999999999999999999999999999999999999")), Literal(Decimal(0.00001)))),
-        null)
-    }
+    checkEvaluation(IntegralDivide(Literal(Decimal(1)), Literal(Decimal(2))), 0L)
+    checkEvaluation(IntegralDivide(Literal(Decimal(2.4)), Literal(Decimal(1.1))), 2L)
+    checkEvaluation(IntegralDivide(Literal(Decimal(1.2)), Literal(Decimal(1.1))), 1L)
+    checkEvaluation(IntegralDivide(Literal(Decimal(0.2)), Literal(Decimal(0.0))), null)
+    // overflows long and so returns a wrong result
+    checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
+      Literal(Decimal("99999999999999999999999999999999999")), Literal(Decimal(0.001)))),
+      687399551400672280L)
+    // overflow during promote precision
+    checkEvaluation(DecimalPrecision.decimalAndDecimal.apply(IntegralDivide(
+      Literal(Decimal("99999999999999999999999999999999999999")), Literal(Decimal(0.00001)))),
+      null)
   }
 
   test("SPARK-24598: overflow on long returns wrong result") {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operator-div.sql b/sql/core/src/test/resources/sql-tests/inputs/operator-div.sql
deleted file mode 100644
index 67b2d39aacd18..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/operator-div.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-set spark.sql.legacy.integralDivide.returnBigint=true;
-
-select 5 div 2;
-select 5 div 0;
-select 5 div null;
-select null div 5;
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0));
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0));
-
-set spark.sql.legacy.integralDivide.returnBigint=false;
-
-select 5 div 2;
-select 5 div 0;
-select 5 div null;
-select null div 5;
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0));
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0));
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/operators.sql b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
index ba14789d48db6..20bf0eb15c5b2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/operators.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/operators.sql
@@ -22,6 +22,16 @@ select 5 / 0;
 select 5 / null;
 select null / 5;
 
+-- integral div
+select 5 div 2;
+select 5 div 0;
+select 5 div null;
+select null div 5;
+select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0));
+select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0));
+select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0));
+select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0));
+
 -- other arithmetics
 select 1 + 2;
 select 1 - 2;
diff --git a/sql/core/src/test/resources/sql-tests/results/operator-div.sql.out b/sql/core/src/test/resources/sql-tests/results/operator-div.sql.out
deleted file mode 100644
index 3f933f4c0e449..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/operator-div.sql.out
+++ /dev/null
@@ -1,146 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 18
-
-
--- !query
-set spark.sql.legacy.integralDivide.returnBigint=true
--- !query schema
-struct<key:string,value:string>
--- !query output
-spark.sql.legacy.integralDivide.returnBigint	true
-
-
--- !query
-select 5 div 2
--- !query schema
-struct<(5 div 2):bigint>
--- !query output
-2
-
-
--- !query
-select 5 div 0
--- !query schema
-struct<(5 div 0):bigint>
--- !query output
-NULL
-
-
--- !query
-select 5 div null
--- !query schema
-struct<(5 div CAST(NULL AS INT)):bigint>
--- !query output
-NULL
-
-
--- !query
-select null div 5
--- !query schema
-struct<(CAST(NULL AS INT) div 5):bigint>
--- !query output
-NULL
-
-
--- !query
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):bigint>
--- !query output
-25
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
--- !query output
-NULL
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
--- !query output
-NULL
-
-
--- !query
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
--- !query output
-NULL
-
-
--- !query
-set spark.sql.legacy.integralDivide.returnBigint=false
--- !query schema
-struct<key:string,value:string>
--- !query output
-spark.sql.legacy.integralDivide.returnBigint	false
-
-
--- !query
-select 5 div 2
--- !query schema
-struct<(5 div 2):int>
--- !query output
-2
-
-
--- !query
-select 5 div 0
--- !query schema
-struct<(5 div 0):int>
--- !query output
-NULL
-
-
--- !query
-select 5 div null
--- !query schema
-struct<(5 div CAST(NULL AS INT)):int>
--- !query output
-NULL
-
-
--- !query
-select null div 5
--- !query schema
-struct<(CAST(NULL AS INT) div 5):int>
--- !query output
-NULL
-
-
--- !query
-select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):decimal(10,0)>
--- !query output
-25
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):decimal(1,0)>
--- !query output
-NULL
-
-
--- !query
-select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):decimal(1,0)>
--- !query output
-NULL
-
-
--- !query
-select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
--- !query schema
-struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):decimal(1,0)>
--- !query output
-NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index 548281014afd7..083410f8807c4 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 49
+-- Number of queries: 57
 
 
 -- !query
@@ -154,6 +154,70 @@ struct<(CAST(NULL AS DOUBLE) / CAST(5 AS DOUBLE)):double>
 NULL
 
 
+-- !query
+select 5 div 2
+-- !query schema
+struct<(5 div 2):bigint>
+-- !query output
+2
+
+
+-- !query
+select 5 div 0
+-- !query schema
+struct<(5 div 0):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select 5 div null
+-- !query schema
+struct<(5 div CAST(NULL AS INT)):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select null div 5
+-- !query schema
+struct<(CAST(NULL AS INT) div 5):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast(51 as decimal(10, 0)) div cast(2 as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(51 AS DECIMAL(10,0)) AS DECIMAL(10,0)) div CAST(CAST(2 AS DECIMAL(2,0)) AS DECIMAL(10,0))):bigint>
+-- !query output
+25
+
+
+-- !query
+select cast(5 as decimal(1, 0)) div cast(0 as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(0 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast(5 as decimal(1, 0)) div cast(null as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(5 AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(NULL AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast(null as decimal(1, 0)) div cast(5 as decimal(2, 0))
+-- !query schema
+struct<(CAST(CAST(NULL AS DECIMAL(1,0)) AS DECIMAL(2,0)) div CAST(CAST(5 AS DECIMAL(2,0)) AS DECIMAL(2,0))):bigint>
+-- !query output
+NULL
+
+
 -- !query
 select 1 + 2
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
index bdb605e406b8a..65b6641983a1d 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
@@ -4494,7 +4494,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) / CA
 -- !query
 select div(cast(999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(38,0)>
+struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):bigint>
 -- !query output
 0
 
@@ -4510,7 +4510,7 @@ struct<(CAST(CAST(999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CA
 -- !query
 select div(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):decimal(38,0)>
+struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))):bigint>
 -- !query output
 -9
 
@@ -4526,7 +4526,7 @@ struct<(CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) %
 -- !query
 select div(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)*1000000000000000000000 + mod(cast(-9999999999999999999999 as decimal(38, 0)),1000000000000000000000)
 -- !query schema
-struct<(CAST((CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0)) * CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0)) + CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0))):decimal(38,0)>
+struct<(CAST((CAST(CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) div CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(20,0)) AS DECIMAL(22,0)) * CAST(1000000000000000000000 AS DECIMAL(22,0))) AS DECIMAL(38,0)) + CAST((CAST(CAST(-9999999999999999999999 AS DECIMAL(38,0)) AS DECIMAL(38,0)) % CAST(1000000000000000000000 AS DECIMAL(38,0))) AS DECIMAL(38,0))):decimal(38,0)>
 -- !query output
 -9999999999999999999999
 
@@ -4542,7 +4542,7 @@ struct<(CAST(70.0 AS DECIMAL(3,1)) % CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,
 -- !query
 select div (70.0,70)
 -- !query schema
-struct<(CAST(70.0 AS DECIMAL(3,1)) div CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):decimal(2,0)>
+struct<(CAST(70.0 AS DECIMAL(3,1)) div CAST(CAST(70 AS DECIMAL(2,0)) AS DECIMAL(3,1))):bigint>
 -- !query output
 1
 

From 1333b87c92788c64109629447ee92f6aa8362c30 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Fri, 13 Mar 2020 11:45:59 +0900
Subject: [PATCH 0275/1280] [SPARK-28093][FOLLOW-UP] Remove migration guide of
 TRIM changes

### What changes were proposed in this pull request?
Since we reverted the original change in https://github.com/apache/spark/pull/27540, this PR is to remove the corresponding migration guide made in the commit https://github.com/apache/spark/pull/24948

### Why are the changes needed?
N/A

### Does this PR introduce any user-facing change?
N/A

### How was this patch tested?
N/A

Closes #27896 from gatorsmile/SPARK-28093Followup.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1c8526dc87212b7a51fddf5f180bc8387805e4d0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 108107967b82f..19c744cd0206d 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -31,8 +31,6 @@ license: |
 
   - Since Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
 
-  - Since Spark 3.0, we reversed argument order of the trim function from `TRIM(trimStr, str)` to `TRIM(str, trimStr)` to be compatible with other databases.
-
   - In Spark version 2.4 and earlier, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. Therefore we will treat these queries as invalid since Spark 3.0.
 
   - Since Spark 3.0, the Dataset and DataFrame API `unionAll` is not deprecated any more. It is an alias for `union`.

From 61ede3ac7cb799fadc2549fa61b5375bc7872e7f Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Thu, 12 Mar 2020 20:01:17 -0700
Subject: [PATCH 0276/1280] [SPARK-31128][WEBUI] Fix Uncaught TypeError in
 streaming statistics page

### What changes were proposed in this pull request?

There is a minor issue in https://github.com/apache/spark/pull/26201
In the streaming statistics page, there is such error
```
streaming-page.js:211 Uncaught TypeError: Cannot read property 'top' of undefined
at SVGCircleElement.<anonymous> (streaming-page.js:211)
at SVGCircleElement.__onclick (d3.min.js:1)
```
in the console after clicking the timeline graph.
![image](https://user-images.githubusercontent.com/1097932/76479745-14b26280-63ca-11ea-9079-0065321795f9.png)

This PR is to fix it.
### Why are the changes needed?

Fix the error of javascript execution.

### Does this PR introduce any user-facing change?

No, the error shows up in the console.

### How was this patch tested?

Manual test.

Closes #27883 from gengliangwang/fixSelector.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 0f463258c214c67360f2b46a4e2292d992b3bf23)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../apache/spark/ui/static/streaming-page.js  | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
index ed3e65c386dce..528b9dfd9a4b8 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
@@ -190,29 +190,32 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
                     .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
             })
             .on("click", function(d) {
-                if (lastTimeout != null) {
-                    window.clearTimeout(lastTimeout);
-                }
-                if (lastClickedBatch != null) {
-                    clearBatchRow(lastClickedBatch);
-                    lastClickedBatch = null;
-                }
-                lastClickedBatch = d.x;
-                highlightBatchRow(lastClickedBatch);
-                lastTimeout = window.setTimeout(function () {
-                    lastTimeout = null;
+                var batchSelector = $("#batch-" + d.x);
+                // If there is a corresponding batch row, scroll down to it and highlight it.
+                if (batchSelector.length > 0) {
+                    if (lastTimeout != null) {
+                        window.clearTimeout(lastTimeout);
+                    }
                     if (lastClickedBatch != null) {
                         clearBatchRow(lastClickedBatch);
                         lastClickedBatch = null;
                     }
-                }, 3000); // Clean up after 3 seconds
-
-                var batchSelector = $("#batch-" + d.x);
-                var topOffset = batchSelector.offset().top - 15;
-                if (topOffset < 0) {
-                    topOffset = 0;
+                    lastClickedBatch = d.x;
+                    highlightBatchRow(lastClickedBatch);
+                    lastTimeout = window.setTimeout(function () {
+                        lastTimeout = null;
+                        if (lastClickedBatch != null) {
+                            clearBatchRow(lastClickedBatch);
+                            lastClickedBatch = null;
+                        }
+                    }, 3000); // Clean up after 3 seconds
+
+                    var topOffset = batchSelector.offset().top - 15;
+                    if (topOffset < 0) {
+                        topOffset = 0;
+                    }
+                    $('html,body').animate({scrollTop: topOffset}, 200);
                 }
-                $('html,body').animate({scrollTop: topOffset}, 200);
             });
 }
 

From b1976ac43ce8cdda58916c300fd5b01677fd5973 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 13 Mar 2020 12:58:49 -0700
Subject: [PATCH 0277/1280] [SPARK-31004][WEBUI][SS] Show message for empty
 Streaming Queries instead of empty timelines and histograms

### What changes were proposed in this pull request?

`StreamingQueryStatisticsPage` shows a message "No visualization information available because there is no batches" instead of showing empty timelines and histograms for empty streaming queries.

[Before this change applied]
![before-fix-for-empty-streaming-query](https://user-images.githubusercontent.com/4736016/75642391-b32e1d80-5c7e-11ea-9c07-e2f0f1b5b4f9.png)

[After this change applied]
![after-fix-for-empty-streaming-query2](https://user-images.githubusercontent.com/4736016/75694583-1904be80-5cec-11ea-9b13-dc7078775188.png)

### Why are the changes needed?

Empty charts are ugly and a little bit confusing.
It's better to clearly say "No visualization information available".

Also, this change fixes a JS error shown in the capture above.
This error occurs because `drawTimeline` in `streaming-page.js` is called even though `formattedDate` will be `undefined` for empty streaming queries.

### Does this PR introduce any user-facing change?

Yes. screen captures are shown above.

### How was this patch tested?

Manually tested by creating an empty streaming query like as follows.
```
val df = spark.readStream.format("socket").options(Map("host"->"<non-existing hostname>", "port"->"...")).load
df.writeStream.format("console").start
```
This streaming query will fail because of `non-existing hostname` and has no batches.

Closes #27755 from sarutak/fix-for-empty-batches.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 680981587d3219aea7caadb56df6519c8b3acb7f)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../ui/StreamingQueryStatisticsPage.scala          | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
index 65052d96a131a..227e5e5af3983 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatisticsPage.scala
@@ -229,14 +229,15 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
         0L,
         "ms")
 
-    val table =
-    // scalastyle:off
+    val table = if (query.lastProgress != null) {
+      // scalastyle:off
       <table id="stat-table" class="table table-bordered" style="width: auto">
         <thead>
           <tr>
             <th style="width: 160px;"></th>
             <th style="width: 492px;">Timelines</th>
-            <th style="width: 350px;">Histograms</th></tr>
+            <th style="width: 350px;">Histograms</th>
+          </tr>
         </thead>
         <tbody>
           <tr>
@@ -285,7 +286,12 @@ private[ui] class StreamingQueryStatisticsPage(parent: StreamingQueryTab)
           </tr>
         </tbody>
       </table>
-    // scalastyle:on
+    } else {
+      <div id="empty-streaming-query-message">
+        <b>No visualization information available.</b>
+      </div>
+      // scalastyle:on
+    }
 
     generateTimeToValues(operationDurationData) ++
       generateFormattedTimeTipStrings(batchToTimestamps) ++

From 339e4dd3a3daf6c11670e5ca7786c54f68a86bfa Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Fri, 13 Mar 2020 15:55:29 -0700
Subject: [PATCH 0278/1280] [SPARK-31144][SQL] Wrap Error with
 QueryExecutionException to notify QueryExecutionListener

### What changes were proposed in this pull request?

This PR manually reverts changes in #25292 and then wraps java.lang.Error with `QueryExecutionException` to notify `QueryExecutionListener` to send it to `QueryExecutionListener.onFailure` which only accepts `Exception`.

The bug fix PR for 2.4 is #27904. It needs a separate PR because the touched codes were changed a lot.

### Why are the changes needed?

Avoid API changes and fix a bug.

### Does this PR introduce any user-facing change?

Yes. Reverting an API change happening in 3.0. QueryExecutionListener APIs will be the same as 2.4.

### How was this patch tested?

The new added test.

Closes #27907 from zsxwing/SPARK-31144.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 1ddf44dfcaff53e870a3c9608e31a60805e50c29)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 project/MimaExcludes.scala                    |  4 --
 .../sql/util/QueryExecutionListener.scala     | 18 ++++--
 .../spark/sql/DataFrameWriterV2Suite.scala    |  2 +-
 .../apache/spark/sql/SessionStateSuite.scala  |  2 +-
 .../sql/TestQueryExecutionListener.scala      |  2 +-
 .../scala/org/apache/spark/sql/UDFSuite.scala |  2 +-
 .../DataSourceV2DataFrameSuite.scala          |  2 +-
 .../FileDataSourceV2FallBackSuite.scala       |  6 +-
 .../SupportsCatalogOptionsSuite.scala         |  2 +-
 .../sql/test/DataFrameReaderWriterSuite.scala |  2 +-
 .../sql/util/DataFrameCallbackSuite.scala     | 62 ++++++++++++++-----
 .../util/ExecutionListenerManagerSuite.scala  |  2 +-
 .../hive/thriftserver/DummyListeners.scala    |  2 +-
 13 files changed, 71 insertions(+), 37 deletions(-)

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 7f6657760d259..f8ad60bfac5db 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -419,10 +419,6 @@ object MimaExcludes {
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.ProcessingTime"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.streaming.ProcessingTime$"),
 
-    // [SPARK-28556][SQL] QueryExecutionListener should also notify Error
-    ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.util.QueryExecutionListener.onFailure"),
-    ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.util.QueryExecutionListener.onFailure"),
-
     // [SPARK-25382][SQL][PYSPARK] Remove ImageSchema.readImages in 3.0
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.image.ImageSchema.readImages"),
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
index 01f81825f6bfd..0b5951ec2ac97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/util/QueryExecutionListener.scala
@@ -23,7 +23,7 @@ import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.execution.QueryExecution
+import org.apache.spark.sql.execution.{QueryExecution, QueryExecutionException}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.util.{ListenerBus, Utils}
@@ -55,12 +55,13 @@ trait QueryExecutionListener {
    * @param funcName the name of the action that triggered this query.
    * @param qe the QueryExecution object that carries detail information like logical plan,
    *           physical plan, etc.
-   * @param error the error that failed this query.
-   *
+   * @param exception the exception that failed this query. If `java.lang.Error` is thrown during
+   *                  execution, it will be wrapped with an `Exception` and it can be accessed by
+   *                  `exception.getCause`.
    * @note This can be invoked by multiple different threads.
    */
   @DeveloperApi
-  def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit
+  def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit
 }
 
 
@@ -140,7 +141,14 @@ private[sql] class ExecutionListenerBus(session: SparkSession)
       val funcName = event.executionName.get
       event.executionFailure match {
         case Some(ex) =>
-          listener.onFailure(funcName, event.qe, ex)
+          val exception = ex match {
+            case e: Exception => e
+            case other: Throwable =>
+              val message = "Hit an error when executing a query" +
+                (if (other.getMessage == null) "" else s": ${other.getMessage}")
+              new QueryExecutionException(message, other)
+          }
+          listener.onFailure(funcName, event.qe, exception)
         case _ =>
           listener.onSuccess(funcName, event.qe, event.duration)
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
index cd157086a8b8e..ac2ebd8bd748b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
@@ -71,7 +71,7 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
         plan = qe.analyzed
 
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
     spark.listenerManager.register(listener)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
index 31957a99e15af..003f5bc835d5f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
@@ -135,7 +135,7 @@ class SessionStateSuite extends SparkFunSuite {
   test("fork new session and inherit listener manager") {
     class CommandCollector extends QueryExecutionListener {
       val commands: ArrayBuffer[String] = ArrayBuffer.empty[String]
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable) : Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, ex: Exception) : Unit = {}
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         commands += funcName
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala b/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala
index fd6bc9662bfad..d2a6358ee822b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TestQueryExecutionListener.scala
@@ -28,7 +28,7 @@ class TestQueryExecutionListener extends QueryExecutionListener {
     OnSuccessCall.isOnSuccessCalled.set(true)
   }
 
-  override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = { }
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = { }
 }
 
 /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index cbe2e91a20d61..e0857ed6bc35a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -341,7 +341,7 @@ class UDFSuite extends QueryTest with SharedSparkSession {
       withTempPath { path =>
         var numTotalCachedHit = 0
         val listener = new QueryExecutionListener {
-          override def onFailure(f: String, qe: QueryExecution, e: Throwable): Unit = {}
+          override def onFailure(f: String, qe: QueryExecution, e: Exception): Unit = {}
 
           override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
             qe.withCachedData match {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
index 0a6897b829994..7c7afa9cfbd41 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
@@ -141,7 +141,7 @@ class DataSourceV2DataFrameSuite
       override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
         plan = qe.analyzed
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
 
     try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
index b0da2eb697f36..51d734279414a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala
@@ -157,13 +157,13 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession {
     Seq("parquet", classOf[ParquetDataSourceV2].getCanonicalName).foreach { format =>
       withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> format) {
         val commands = ArrayBuffer.empty[(String, LogicalPlan)]
-        val errors = ArrayBuffer.empty[(String, Throwable)]
+        val exceptions = ArrayBuffer.empty[(String, Exception)]
         val listener = new QueryExecutionListener {
           override def onFailure(
               funcName: String,
               qe: QueryExecution,
-              error: Throwable): Unit = {
-            errors += funcName -> error
+              exception: Exception): Unit = {
+            exceptions += funcName -> exception
           }
 
           override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
index 9693a10f9afca..550bec7505422 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/SupportsCatalogOptionsSuite.scala
@@ -216,7 +216,7 @@ class SupportsCatalogOptionsSuite extends QueryTest with SharedSparkSession with
       override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {
         plan = qe.analyzed
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
 
     spark.listenerManager.register(listener)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index fb939007697c2..9747840ce4032 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -282,7 +282,7 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         plan = qe.analyzed
 
       }
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
     }
 
     spark.listenerManager.register(listener)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
index 6881812286b24..b17c93503804c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/util/DataFrameCallbackSuite.scala
@@ -20,15 +20,17 @@ package org.apache.spark.sql.util
 import scala.collection.mutable.ArrayBuffer
 
 import org.apache.spark._
-import org.apache.spark.sql.{functions, AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.{functions, AnalysisException, Dataset, QueryTest, Row, SparkSession}
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, InsertIntoStatement, LogicalPlan, Project}
-import org.apache.spark.sql.execution.{QueryExecution, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.{QueryExecution, QueryExecutionException, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.command.RunnableCommand
 import org.apache.spark.sql.execution.datasources.{CreateTable, InsertIntoHadoopFsRelationCommand}
 import org.apache.spark.sql.execution.datasources.json.JsonFileFormat
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StringType
 
 class DataFrameCallbackSuite extends QueryTest
   with SharedSparkSession
@@ -40,7 +42,7 @@ class DataFrameCallbackSuite extends QueryTest
     val metrics = ArrayBuffer.empty[(String, QueryExecution, Long)]
     val listener = new QueryExecutionListener {
       // Only test successful case here, so no need to implement `onFailure`
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         metrics += ((funcName, qe, duration))
@@ -67,10 +69,10 @@ class DataFrameCallbackSuite extends QueryTest
   }
 
   testQuietly("execute callback functions when a DataFrame action failed") {
-    val metrics = ArrayBuffer.empty[(String, QueryExecution, Throwable)]
+    val metrics = ArrayBuffer.empty[(String, QueryExecution, Exception)]
     val listener = new QueryExecutionListener {
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {
-        metrics += ((funcName, qe, error))
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+        metrics += ((funcName, qe, exception))
       }
 
       // Only test failed case here, so no need to implement `onSuccess`
@@ -96,7 +98,7 @@ class DataFrameCallbackSuite extends QueryTest
     val metrics = ArrayBuffer.empty[Long]
     val listener = new QueryExecutionListener {
       // Only test successful case here, so no need to implement `onFailure`
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         val metric = stripAQEPlan(qe.executedPlan) match {
@@ -136,7 +138,7 @@ class DataFrameCallbackSuite extends QueryTest
     val metrics = ArrayBuffer.empty[Long]
     val listener = new QueryExecutionListener {
       // Only test successful case here, so no need to implement `onFailure`
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
         metrics += qe.executedPlan.longMetric("dataSize").value
@@ -176,10 +178,10 @@ class DataFrameCallbackSuite extends QueryTest
 
   test("execute callback functions for DataFrameWriter") {
     val commands = ArrayBuffer.empty[(String, LogicalPlan)]
-    val errors = ArrayBuffer.empty[(String, Throwable)]
+    val exceptions = ArrayBuffer.empty[(String, Exception)]
     val listener = new QueryExecutionListener {
-      override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {
-        errors += funcName -> error
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+        exceptions += funcName -> exception
       }
 
       override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {
@@ -225,9 +227,9 @@ class DataFrameCallbackSuite extends QueryTest
         spark.range(10).select($"id", $"id").write.insertInto("tab")
       }
       sparkContext.listenerBus.waitUntilEmpty()
-      assert(errors.length == 1)
-      assert(errors.head._1 == "insertInto")
-      assert(errors.head._2 == e)
+      assert(exceptions.length == 1)
+      assert(exceptions.head._1 == "insertInto")
+      assert(exceptions.head._2 == e)
     }
   }
 
@@ -238,7 +240,7 @@ class DataFrameCallbackSuite extends QueryTest
         metricMaps += qe.observedMetrics
       }
 
-      override def onFailure(funcName: String, qe: QueryExecution, exception: Throwable): Unit = {
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
         // No-op
       }
     }
@@ -278,4 +280,32 @@ class DataFrameCallbackSuite extends QueryTest
       spark.listenerManager.unregister(listener)
     }
   }
+
+  testQuietly("SPARK-31144: QueryExecutionListener should receive `java.lang.Error`") {
+    var e: Exception = null
+    val listener = new QueryExecutionListener {
+      override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
+        e = exception
+      }
+      override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = {}
+    }
+    spark.listenerManager.register(listener)
+
+    intercept[Error] {
+      Dataset.ofRows(spark, ErrorTestCommand("foo")).collect()
+    }
+    sparkContext.listenerBus.waitUntilEmpty()
+    assert(e != null && e.isInstanceOf[QueryExecutionException]
+      && e.getCause.isInstanceOf[Error] && e.getCause.getMessage == "foo")
+    spark.listenerManager.unregister(listener)
+  }
+}
+
+/** A test command that throws `java.lang.Error` during execution. */
+case class ErrorTestCommand(foo: String) extends RunnableCommand {
+
+  override val output: Seq[Attribute] = Seq(AttributeReference("foo", StringType)())
+
+  override def run(sparkSession: SparkSession): Seq[Row] =
+    throw new java.lang.Error(foo)
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala
index 2fd6cb220ea3f..ab854a0281c6c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/util/ExecutionListenerManagerSuite.scala
@@ -57,7 +57,7 @@ private class CountingQueryExecutionListener extends QueryExecutionListener {
     CALLBACK_COUNT.incrementAndGet()
   }
 
-  override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {
     CALLBACK_COUNT.incrementAndGet()
   }
 
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
index d056b3b2153cf..4564c2209a931 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/DummyListeners.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.util.QueryExecutionListener
 
 class DummyQueryExecutionListener extends QueryExecutionListener {
   override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = {}
-  override def onFailure(funcName: String, qe: QueryExecution, error: Throwable): Unit = {}
+  override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {}
 }
 
 class DummyStreamingQueryListener extends StreamingQueryListener {

From 1fc98336cfc8390139f2548a1f496d40a6a7f784 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 13 Mar 2020 16:44:23 -0700
Subject: [PATCH 0279/1280] [MINOR][DOCS] Fix [[...]] to `...` and
 <code>...</code> in documentation

### What changes were proposed in this pull request?

Before:

- ![Screen Shot 2020-03-13 at 1 19 12 PM](https://user-images.githubusercontent.com/6477701/76589452-7c34f300-652d-11ea-9da7-3754f8575796.png)
- ![Screen Shot 2020-03-13 at 1 19 24 PM](https://user-images.githubusercontent.com/6477701/76589455-7d662000-652d-11ea-9dbe-f5fe10d1e7ad.png)
- ![Screen Shot 2020-03-13 at 1 19 03 PM](https://user-images.githubusercontent.com/6477701/76589449-7b03c600-652d-11ea-8e99-dbe47f561f9c.png)

After:

- ![Screen Shot 2020-03-13 at 1 17 37 PM](https://user-images.githubusercontent.com/6477701/76589437-74754e80-652d-11ea-99f5-14fb4761f915.png)
- ![Screen Shot 2020-03-13 at 1 17 46 PM](https://user-images.githubusercontent.com/6477701/76589442-76d7a880-652d-11ea-8c10-53e595421081.png)
- ![Screen Shot 2020-03-13 at 1 18 15 PM](https://user-images.githubusercontent.com/6477701/76589443-7808d580-652d-11ea-9b1b-e5d11d638335.png)

### Why are the changes needed?
To render the code block properly in the documentation

### Does this PR introduce any user-facing change?
Yes, code rendering in documentation.

### How was this patch tested?

Manually built the doc via `SKIP_API=1 jekyll build`.

Closes #27899 from HyukjinKwon/minor-docss.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 9628aca68ba0821b8f3fa934ed4872cabb2a5d7d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/monitoring.md  | 6 +++---
 docs/quick-start.md | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index 4cba15b35e11d..ba3f1dc86becc 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -595,7 +595,7 @@ A list of the available metrics, with a short description:
   </tr>
   <tr>
     <td>inputMetrics.*</td>
-    <td>Metrics related to reading data from [[org.apache.spark.rdd.HadoopRDD]] 
+    <td>Metrics related to reading data from <code>org.apache.spark.rdd.HadoopRDD</code>
     or from persisted data.</td>
   </tr>
   <tr>
@@ -779,11 +779,11 @@ A list of the available metrics, with a short description:
   </tr>
   <tr>
     <td>&nbsp;&nbsp;&nbsp;&nbsp;.DirectPoolMemory</td>
-    <td>Peak memory that the JVM is using for direct buffer pool ([[java.lang.management.BufferPoolMXBean]])</td>
+    <td>Peak memory that the JVM is using for direct buffer pool (<code>java.lang.management.BufferPoolMXBean</code>)</td>
   </tr>
   <tr>
     <td>&nbsp;&nbsp;&nbsp;&nbsp;.MappedPoolMemory</td>
-    <td>Peak memory that the JVM is using for mapped buffer pool ([[java.lang.management.BufferPoolMXBean]])</td>
+    <td>Peak memory that the JVM is using for mapped buffer pool (<code>java.lang.management.BufferPoolMXBean</code>)</td>
   </tr>
   <tr>
     <td>&nbsp;&nbsp;&nbsp;&nbsp;.ProcessTreeJVMVMemory</td>
diff --git a/docs/quick-start.md b/docs/quick-start.md
index 86ba2c44b0e02..e7a16a3461653 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -264,7 +264,7 @@ Spark README. Note that you'll need to replace YOUR_SPARK_HOME with the location
 installed. Unlike the earlier examples with the Spark shell, which initializes its own SparkSession,
 we initialize a SparkSession as part of the program.
 
-We call `SparkSession.builder` to construct a [[SparkSession]], then set the application name, and finally call `getOrCreate` to get the [[SparkSession]] instance.
+We call `SparkSession.builder` to construct a `SparkSession`, then set the application name, and finally call `getOrCreate` to get the `SparkSession` instance.
 
 Our application depends on the Spark API, so we'll also include an sbt configuration file,
 `build.sbt`, which explains that Spark is a dependency. This file also adds a repository that

From f83ef7d143aafbbdd1bb322567481f68db72195a Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Sun, 15 Mar 2020 07:35:20 +0900
Subject: [PATCH 0280/1280] [SPARK-31151][SQL][DOC] Reorganize the migration
 guide of SQL

### What changes were proposed in this pull request?
The current migration guide of SQL is too long for most readers to find the needed info. This PR is to group the items in the migration guide of Spark SQL based on the corresponding components.

Note. This PR does not change the contents of the migration guides. Attached figure is the screenshot after the change.

![screencapture-127-0-0-1-4000-sql-migration-guide-html-2020-03-14-12_00_40](https://user-images.githubusercontent.com/11567269/76688626-d3010200-65eb-11ea-9ce7-265bc90ebb2c.png)

### Why are the changes needed?
The current migration guide of SQL is too long for most readers to find the needed info.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
N/A

Closes #27909 from gatorsmile/migrationGuideReorg.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4d4c00c1b564b57d3016ce8c3bfcffaa6e58f012)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-migration-guide.md | 287 +++++++++++++++++++-----------------
 1 file changed, 150 insertions(+), 137 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 19c744cd0206d..31d5c68caa2dc 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -23,92 +23,119 @@ license: |
 {:toc}
 
 ## Upgrading from Spark SQL 2.4 to 3.0
-  - Since Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception will be thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and earlier, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to a integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
 
-  - In Spark 3.0, the deprecated methods `SQLContext.createExternalTable` and `SparkSession.createExternalTable` have been removed in favor of its replacement, `createTable`.
-
-  - In Spark 3.0, the deprecated `HiveContext` class has been removed. Use `SparkSession.builder.enableHiveSupport()` instead.
-
-  - Since Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
-
-  - In Spark version 2.4 and earlier, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. Therefore we will treat these queries as invalid since Spark 3.0.
+### Dataset/DataFrame APIs
 
   - Since Spark 3.0, the Dataset and DataFrame API `unionAll` is not deprecated any more. It is an alias for `union`.
 
-  - In Spark version 2.4 and earlier, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType`, `DoubleType`, `DateType` and `TimestampType`, it fails on empty strings and throws exceptions. Since Spark 3.0, we disallow empty strings and will throw exceptions for data types except for `StringType` and `BinaryType`. The previous behaviour of allowing empty string can be restored by setting `spark.sql.legacy.json.allowEmptyString.enabled` to `true`.
-
-  - Since Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
-
-  - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
-
-  - In Spark version 2.4 and earlier, users can create map values with map type key via built-in function such as `CreateMap`, `MapFromArrays`, etc. Since Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can use `map_entries` function to convert map to array<struct<key, value>> as a workaround. In addition, users can still read map values with map type key from data source or Java/Scala collections, though it is discouraged.
-
   - In Spark version 2.4 and earlier, `Dataset.groupByKey` results to a grouped dataset with key attribute wrongly named as "value", if the key is non-struct type, e.g. int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries weird. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behaviour is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
 
-  - In Spark version 2.4 and earlier, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys and join keys. Since Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and earlier.
-
-  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, Spark will throw RuntimeException while duplicated keys are found. Users can set `spark.sql.mapKeyDedupPolicy` to LAST_WIN to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
+### DDL Statements
 
-  - In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
+  - Since Spark 3.0, `CREATE TABLE` without a specific provider will use the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and earlier, it was hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
 
-  - In Spark version 2.4 and earlier, the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. Since 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
+  - Since Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception will be thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and earlier, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to a integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
 
-  - In Spark version 2.4 and earlier, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. Since Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
+  - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
 
-  - In Spark version 2.4 and earlier, JSON datasource and JSON functions like `from_json` convert a bad JSON record to a row with all `null`s in the PERMISSIVE mode when specified schema is `StructType`. Since Spark 3.0, the returned row can contain non-`null` fields if some of JSON column values were parsed and converted to desired types successfully.
+  - In Spark version 2.4 and earlier, the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. Since 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
 
   - Refreshing a cached table would trigger a table uncache operation and then a table cache (lazily) operation. In Spark version 2.4 and earlier, the cache name and storage level are not preserved before the uncache operation. Therefore, the cache name and storage level could be changed unexpectedly. Since Spark 3.0, cache name and storage level will be first preserved for cache recreation. It helps to maintain a consistent cache behavior upon table refreshing.
 
-  - Since Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inferring.
-
-  - Since Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. But please note that, in Spark version 2.4 and earlier, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF will return null if the input values is null. However, since Spark 3.0, the UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will return null in Spark 2.4 and earlier if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
-
-  - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
-
-    - Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. Since Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and earlier, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html. The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
-
-    - The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
-
-    - the JDBC options `lowerBound` and `upperBound` are converted to TimestampType/DateType values in the same way as casting strings to TimestampType/DateType values. The conversion is based on Proleptic Gregorian calendar, and time zone defined by the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion is based on the hybrid calendar (Julian + Gregorian) and on default system time zone.
+  - Since Spark 3.0, the properties listing below become reserved, commands will fail if we specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. We need their specific clauses to specify them, e.g. `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. We can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, e.g `SET DBPROTERTIES('location'='/tmp')` will affect nothing. In Spark version 2.4 and earlier, these properties are neither reserved nor have side effects, e.g. `SET DBPROTERTIES('location'='/tmp')` will not change the location of the database but only create a headless property just like `'a'='b'`.
+    <table class="table">
+        <tr>
+          <th>
+            <b>Property(case sensitive)</b>
+          </th>
+          <th>
+            <b>Database Reserved</b>
+          </th>
+          <th>
+            <b>Table Reserved</b>
+          </th>
+          <th>
+            <b>Remarks</b>
+          </th>
+        </tr>
+        <tr>
+          <td>
+            provider
+          </td>
+          <td>
+            no
+          </td>
+          <td>
+            yes
+          </td>
+          <td>
+            For tables, please use the USING clause to specify it. Once set, it can't be changed.
+          </td>
+        </tr>
+        <tr>
+          <td>
+            location
+          </td>
+          <td>
+            yes
+          </td>
+          <td>
+            yes
+          </td>
+          <td>
+            For databases and tables, please use the LOCATION clause to specify it.
+          </td>
+        </tr>
+        <tr>
+          <td>
+            owner
+          </td>
+          <td>
+            yes
+          </td>
+          <td>
+            yes
+          </td>
+          <td>
+            For databases and tables, it is determined by the user who runs spark and create the table.
+          </td>
+        </tr>
+    </table>
 
-    - Formatting of `TIMESTAMP` and `DATE` literals.
+  - Since Spark 3.0, `ADD FILE` can be used to add file directories as well. Earlier only single files can be added using this command. To restore the behaviour of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
 
-    - Creating of typed `TIMESTAMP` and `DATE` literals from strings. Since Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. When the input string does not contain information about time zone, the time zone from the SQL config `spark.sql.session.timeZone` is used in that case. In Spark version 2.4 and earlier, the conversion is based on JVM system time zone. The different sources of the default time zone may change the behavior of typed `TIMESTAMP` and `DATE` literals.
+  - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result.
 
-  - In Spark version 2.4 and earlier, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. Since Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
+  - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For generating Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
 
-  - In Spark version 2.4 and earlier, the `current_timestamp` function returns a timestamp with millisecond resolution only. Since Spark 3.0, the function can return the result with microsecond resolution if the underlying clock available on the system offers such resolution.
+### UDFs and Built-in Functions
 
-  - In Spark version 2.4 and earlier, when reading a Hive Serde table with Spark native data sources(parquet/orc), Spark will infer the actual file schema and update the table schema in metastore. Since Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, please set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
+  - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
 
-  - Since Spark 3.0, `TIMESTAMP` literals are converted to strings using the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion uses the default time zone of the Java virtual machine.
+  - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
 
-  - In Spark version 2.4, when a spark session is created via `cloneSession()`, the newly created spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent spark session. Since Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. The old behavior can be restored by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
+  - Since Spark 3.0, an analysis exception will be thrown when hash expressions are applied on elements of MapType. To restore the behavior before Spark 3.0, set `spark.sql.legacy.allowHashOnMapType` to `true`.
 
-  - Since Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and earlier, `TIMESTAMP` columns are saved as `INT96` in parquet files. Note that, some SQL systems such as Hive 1.x and Impala 2.x can only read `INT96` timestamps, you can set `spark.sql.parquet.outputTimestampType` as `INT96` to restore the previous behavior and keep interoperability.
+  - Since Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and earlier, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
 
-  - Since Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it will fallback to hive-site.xml present in the `Hadoop configuration` of `SparkContext`.
+  - Since Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
 
-  - Since Spark 3.0, Spark will cast `String` to `Date/TimeStamp` in binary comparisons with dates/timestamps. The previous behaviour of casting `Date/Timestamp` to `String` can be restored by setting `spark.sql.legacy.typeCoercion.datetimeToString.enabled` to `true`.
+  - In Spark version 2.4 and earlier, users can create map values with map type key via built-in function such as `CreateMap`, `MapFromArrays`, etc. Since Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can use `map_entries` function to convert map to array<struct<key, value>> as a workaround. In addition, users can still read map values with map type key from data source or Java/Scala collections, though it is discouraged.
 
-  - Since Spark 3.0, when Avro files are written with user provided schema, the fields will be matched by field names between catalyst schema and avro schema instead of positions.
+  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, Spark will throw RuntimeException while duplicated keys are found. Users can set `spark.sql.mapKeyDedupPolicy` to LAST_WIN to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
 
-  - Since Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark will throw runtime NPE if any of the records contains null.
+  - Since Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. But please note that, in Spark version 2.4 and earlier, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF will return null if the input values is null. However, since Spark 3.0, the UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will return null in Spark 2.4 and earlier if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
 
   - Since Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, i.e., if the `predicate` returns any `null`s and no `true` is obtained, then `exists` will return `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` will be `null`. The previous behaviour can be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
 
-  - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
-
-  - Since Spark 3.0, `spark.sql.legacy.ctePrecedencePolicy` is introduced to control the behavior for name conflicting in the nested WITH clause. By default value `EXCEPTION`, Spark throws an AnalysisException, it forces users to choose the specific substitution order they wanted. If set to `CORRECTED` (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `LEGACY`, the result is `1` which is the behavior in version 2.4 and earlier.
-
   - Since Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and earlier, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
 
+  - In Spark version 2.4 and earlier, the `current_timestamp` function returns a timestamp with millisecond resolution only. Since Spark 3.0, the function can return the result with microsecond resolution if the underlying clock available on the system offers such resolution.
+
   - Since Spark 3.0, 0-argument Java UDF is executed in the executor side identically with other UDFs. In Spark version 2.4 and earlier, 0-argument Java UDF alone was executed in the driver side, and the result was propagated to executors, which might be more performant in some cases but caused inconsistency with a correctness issue in some cases.
 
   - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
 
-  - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
-
   - Since Spark 3.0, `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in case insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behaviour change is illustrated in the table below:
     <table class="table">
         <tr>
@@ -198,6 +225,50 @@ license: |
         </tr>
     </table>
 
+  - Since Spark 3.0, when casting interval values to string type, there is no "interval" prefix, e.g. `1 days 2 hours`. In Spark version 2.4 and earlier, the string contains the "interval" prefix like `interval 1 days 2 hours`.
+
+  - Since Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, e.g. `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, while casting string to integrals and booleans, it will not trim the whitespaces from both ends, the foregoing results will be `null`, while to datetimes, only the trailing spaces (= ASCII 32) will be removed.
+
+### Query Engine
+
+  - In Spark version 2.4 and earlier, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. Therefore we will treat these queries as invalid since Spark 3.0.
+
+  - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
+
+  - Since Spark 3.0, numbers written in scientific notation(e.g. `1E2`) would be parsed as Double. In Spark version 2.4 and earlier, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
+
+  - Since Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
+  
+  - Since Spark 3.0, negative scale of decimal is not allowed by default, e.g. data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and earlier, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
+
+  - Since Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation will be coerced to double value, e.g. `+'1'` results `1.0`. In Spark version 2.4 and earlier, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, e.g. `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, e.g. in Spark 2.4, the result of `+'1'` is string `1`.
+
+  - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
+
+  - Since Spark 3.0, `spark.sql.legacy.ctePrecedencePolicy` is introduced to control the behavior for name conflicting in the nested WITH clause. By default value `EXCEPTION`, Spark throws an AnalysisException, it forces users to choose the specific substitution order they wanted. If set to `CORRECTED` (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `LEGACY`, the result is `1` which is the behavior in version 2.4 and earlier.
+
+  - Since Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
+
+  - In Spark version 2.4 and earlier, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys and join keys. Since Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and earlier.
+
+  - In Spark version 2.4 and earlier, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. Since Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
+
+  - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
+
+    - Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. Since Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and earlier, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html. The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
+
+    - The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
+
+    - the JDBC options `lowerBound` and `upperBound` are converted to TimestampType/DateType values in the same way as casting strings to TimestampType/DateType values. The conversion is based on Proleptic Gregorian calendar, and time zone defined by the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion is based on the hybrid calendar (Julian + Gregorian) and on default system time zone.
+
+    - Formatting of `TIMESTAMP` and `DATE` literals.
+
+    - Creating of typed `TIMESTAMP` and `DATE` literals from strings. Since Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. When the input string does not contain information about time zone, the time zone from the SQL config `spark.sql.session.timeZone` is used in that case. In Spark version 2.4 and earlier, the conversion is based on JVM system time zone. The different sources of the default time zone may change the behavior of typed `TIMESTAMP` and `DATE` literals.
+
+  - Since Spark 3.0, `TIMESTAMP` literals are converted to strings using the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion uses the default time zone of the Java virtual machine.
+
+  - Since Spark 3.0, Spark will cast `String` to `Date/TimeStamp` in binary comparisons with dates/timestamps. The previous behaviour of casting `Date/Timestamp` to `String` can be restored by setting `spark.sql.legacy.typeCoercion.datetimeToString.enabled` to `true`.
+
   - Since Spark 3.0, special values are supported in conversion from strings to dates and timestamps. Those values are simply notational shorthands that will be converted to ordinary date or timestamp values when read. The following string values are supported for dates:
     - `epoch [zoneId]` - 1970-01-01
     - `today [zoneId]` - the current date in the time zone specified by `spark.sql.session.timeZone`
@@ -212,17 +283,37 @@ license: |
     - `now` - current query start time
   For example `SELECT timestamp 'tomorrow';`.
 
-  - Since Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and earlier, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
+### Data Sources
 
-  - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
+  - In Spark version 2.4 and earlier, when reading a Hive Serde table with Spark native data sources(parquet/orc), Spark will infer the actual file schema and update the table schema in metastore. Since Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, please set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
 
-  - Since Spark 3.0, when casting interval values to string type, there is no "interval" prefix, e.g. `1 days 2 hours`. In Spark version 2.4 and earlier, the string contains the "interval" prefix like `interval 1 days 2 hours`.
+  - In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
 
-  - Since Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, e.g. `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, while casting string to integrals and booleans, it will not trim the whitespaces from both ends, the foregoing results will be `null`, while to datetimes, only the trailing spaces (= ASCII 32) will be removed.
+  - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
 
-  - Since Spark 3.0, an analysis exception will be thrown when hash expressions are applied on elements of MapType. To restore the behavior before Spark 3.0, set `spark.sql.legacy.allowHashOnMapType` to `true`.
-    
-  - Since Spark 3.0, numbers written in scientific notation(e.g. `1E2`) would be parsed as Double. In Spark version 2.4 and earlier, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
+  - In Spark version 2.4 and earlier, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType`, `DoubleType`, `DateType` and `TimestampType`, it fails on empty strings and throws exceptions. Since Spark 3.0, we disallow empty strings and will throw exceptions for data types except for `StringType` and `BinaryType`. The previous behaviour of allowing empty string can be restored by setting `spark.sql.legacy.json.allowEmptyString.enabled` to `true`.
+
+  - In Spark version 2.4 and earlier, JSON datasource and JSON functions like `from_json` convert a bad JSON record to a row with all `null`s in the PERMISSIVE mode when specified schema is `StructType`. Since Spark 3.0, the returned row can contain non-`null` fields if some of JSON column values were parsed and converted to desired types successfully.
+
+  - Since Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inferring.
+
+  - In Spark version 2.4 and earlier, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. Since Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
+
+  - Since Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and earlier, `TIMESTAMP` columns are saved as `INT96` in parquet files. Note that, some SQL systems such as Hive 1.x and Impala 2.x can only read `INT96` timestamps, you can set `spark.sql.parquet.outputTimestampType` as `INT96` to restore the previous behavior and keep interoperability.
+
+  - Since Spark 3.0, when Avro files are written with user provided schema, the fields will be matched by field names between catalyst schema and avro schema instead of positions.
+
+  - Since Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark will throw runtime NPE if any of the records contains null.
+
+### Others
+
+  - In Spark 3.0, the deprecated methods `SQLContext.createExternalTable` and `SparkSession.createExternalTable` have been removed in favor of its replacement, `createTable`.
+
+  - In Spark 3.0, the deprecated `HiveContext` class has been removed. Use `SparkSession.builder.enableHiveSupport()` instead.
+
+  - In Spark version 2.4, when a spark session is created via `cloneSession()`, the newly created spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent spark session. Since Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. The old behavior can be restored by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
+
+  - Since Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it will fallback to hive-site.xml present in the `Hadoop configuration` of `SparkContext`.
 
   - Since Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
     <table class="table">
@@ -249,84 +340,6 @@ license: |
           </td>
         </tr>
     </table>
-    
-  - Since Spark 3.0, `CREATE TABLE` without a specific provider will use the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and earlier, it was hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
-
-  - Since Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation will be coerced to double value, e.g. `+'1'` results `1.0`. In Spark version 2.4 and earlier, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, e.g. `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, e.g. in Spark 2.4, the result of `+'1'` is string `1`.
-
-  - Since Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
-  
-  - Since Spark 3.0, negative scale of decimal is not allowed by default, e.g. data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and earlier, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
-
-  - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
-
-  - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
-
-  - Since Spark 3.0, the properties listing below become reserved, commands will fail if we specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. We need their specific clauses to specify them, e.g. `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. We can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, e.g `SET DBPROTERTIES('location'='/tmp')` will affect nothing. In Spark version 2.4 and earlier, these properties are neither reserved nor have side effects, e.g. `SET DBPROTERTIES('location'='/tmp')` will not change the location of the database but only create a headless property just like `'a'='b'`.
-    <table class="table">
-        <tr>
-          <th>
-            <b>Property(case sensitive)</b>
-          </th>
-          <th>
-            <b>Database Reserved</b>
-          </th>
-          <th>
-            <b>Table Reserved</b>
-          </th>
-          <th>
-            <b>Remarks</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            provider
-          </td>
-          <td>
-            no
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For tables, please use the USING clause to specify it. Once set, it can't be changed.
-          </td>
-        </tr>
-        <tr>
-          <td>
-            location
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For databases and tables, please use the LOCATION clause to specify it.
-          </td>
-        </tr>
-        <tr>
-          <td>
-            owner
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For databases and tables, it is determined by the user who runs spark and create the table.
-          </td>
-        </tr>
-    </table>
-
-  - Since Spark 3.0, `ADD FILE` can be used to add file directories as well. Earlier only single files can be added using this command. To restore the behaviour of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
-
-  - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result.
-
-  - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For generating Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
 
   - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3 and it brings following impacts:
   

From 0b09b26dafd485de0f93396ea4f6a478719d6e20 Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Mon, 16 Mar 2020 10:04:43 +0900
Subject: [PATCH 0281/1280] [SPARK-31153][BUILD][3.0] Cleanup several failures
 in lint-python

Backport of #27910 / 0ce5519f179a8f88a688dece754596c3166528b7.

---

This PR cleans up several failures -- most of them silent -- in `dev/lint-python`. I don't understand how we haven't been bitten by these yet. Perhaps we've been lucky?

Fixes include:
* Fix how we compare versions. All the version checks currently in `master` silently fail with:

    ```
      File "<string>", line 2
        print(LooseVersion("""2.3.1""") >= LooseVersion("""2.4.0"""))
        ^
    IndentationError: unexpected indent
    ```
    Another problem is that `distutils.version` is undocumented and unsupported.
* Fix some basic bugs. e.g. We have an incorrect reference to `$PYDOCSTYLEBUILD`, which doesn't exist, which was causing the doc style test to silently fail with:

    ```
    ./dev/lint-python: line 193: --version: command not found
    ```
* Stop suppressing error output! It's hiding problems and serves no purpose here.

`lint-python` is part of our CI build and is currently doing any combination of the following: silently failing; incorrectly skipping tests; incorrectly downloading libraries when a suitable library is already available.

Closes #27917 from nchammas/SPARK-31153-lint-python-branch-3.0.

Authored-by: Nicholas Chammas <nicholas.chammas@liveramp.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/lint-python | 37 +++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/dev/lint-python b/dev/lint-python
index e9ed83dec5fce..9d7dfad7c534c 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -27,6 +27,19 @@ MINIMUM_PYCODESTYLE="2.4.0"
 
 SPHINX_BUILD="sphinx-build"
 
+PYTHON_EXECUTABLE="python3"
+
+function satisfies_min_version {
+    local provided_version="$1"
+    local expected_version="$2"
+    echo "$(
+        "$PYTHON_EXECUTABLE" << EOM
+from setuptools.extern.packaging import version
+print(version.parse('$provided_version') >= version.parse('$expected_version'))
+EOM
+    )"
+}
+
 function compile_python_test {
     local COMPILE_STATUS=
     local COMPILE_REPORT=
@@ -56,7 +69,7 @@ function pycodestyle_test {
     local PYCODESTYLE_STATUS=
     local PYCODESTYLE_REPORT=
     local RUN_LOCAL_PYCODESTYLE=
-    local VERSION=
+    local PYCODESTYLE_VERSION=
     local EXPECTED_PYCODESTYLE=
     local PYCODESTYLE_SCRIPT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-$MINIMUM_PYCODESTYLE.py"
     local PYCODESTYLE_SCRIPT_REMOTE_PATH="https://raw.githubusercontent.com/PyCQA/pycodestyle/$MINIMUM_PYCODESTYLE/pycodestyle.py"
@@ -69,11 +82,8 @@ function pycodestyle_test {
     # check for locally installed pycodestyle & version
     RUN_LOCAL_PYCODESTYLE="False"
     if hash "$PYCODESTYLE_BUILD" 2> /dev/null; then
-        VERSION=$( $PYCODESTYLE_BUILD --version 2> /dev/null)
-        EXPECTED_PYCODESTYLE=$( (python3 -c 'from distutils.version import LooseVersion;
-                                print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_PYCODESTYLE'"""))')\
-                                2> /dev/null)
-
+        PYCODESTYLE_VERSION="$($PYCODESTYLE_BUILD --version)"
+        EXPECTED_PYCODESTYLE="$(satisfies_min_version $PYCODESTYLE_VERSION $MINIMUM_PYCODESTYLE)"
         if [ "$EXPECTED_PYCODESTYLE" == "True" ]; then
             RUN_LOCAL_PYCODESTYLE="True"
         fi
@@ -117,7 +127,6 @@ function pycodestyle_test {
 
 function flake8_test {
     local FLAKE8_VERSION=
-    local VERSION=
     local EXPECTED_FLAKE8=
     local FLAKE8_REPORT=
     local FLAKE8_STATUS=
@@ -128,11 +137,9 @@ function flake8_test {
         exit 1
     fi
 
-    FLAKE8_VERSION="$($FLAKE8_BUILD --version  2> /dev/null)"
-    VERSION=($FLAKE8_VERSION)
-    EXPECTED_FLAKE8=$( (python3 -c 'from distutils.version import LooseVersion;
-                       print(LooseVersion("""'${VERSION[0]}'""") >= LooseVersion("""'$MINIMUM_FLAKE8'"""))') \
-                       2> /dev/null)
+    _FLAKE8_VERSION=($($FLAKE8_BUILD --version))
+    FLAKE8_VERSION="${_FLAKE8_VERSION[0]}"
+    EXPECTED_FLAKE8="$(satisfies_min_version $FLAKE8_VERSION $MINIMUM_FLAKE8)"
 
     if [[ "$EXPECTED_FLAKE8" == "False" ]]; then
         echo "\
@@ -174,10 +181,8 @@ function pydocstyle_test {
         return
     fi
 
-    PYDOCSTYLE_VERSION="$($PYDOCSTYLEBUILD --version 2> /dev/null)"
-    EXPECTED_PYDOCSTYLE=$(python3 -c 'from distutils.version import LooseVersion; \
-                             print(LooseVersion("""'$PYDOCSTYLE_VERSION'""") >= LooseVersion("""'$MINIMUM_PYDOCSTYLE'"""))' \
-                             2> /dev/null)
+    PYDOCSTYLE_VERSION="$($PYDOCSTYLE_BUILD --version)"
+    EXPECTED_PYDOCSTYLE="$(satisfies_min_version $PYDOCSTYLE_VERSION $MINIMUM_PYDOCSTYLE)"
 
     if [[ "$EXPECTED_PYDOCSTYLE" == "False" ]]; then
         echo "\

From aad1f5aa2d3e281dde2a019c1c4975533c908b66 Mon Sep 17 00:00:00 2001
From: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Date: Sun, 15 Mar 2020 23:55:04 -0700
Subject: [PATCH 0282/1280] [SPARK-31135][BUILD][TESTS] Upgrdade docker-client
 version to 8.14.1

### What changes were proposed in this pull request?
Upgrdade `docker-client` version.

### Why are the changes needed?
`docker-client` what Spark uses is super old. Snippet from the project page:
```
Spotify no longer uses recent versions of this project internally.
The version of docker-client we're using is whatever helios has in its pom.xml. => 8.14.1
```

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
```
build/mvn install -DskipTests
build/mvn -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.12 -Dtest=none -DwildcardSuites=org.apache.spark.sql.jdbc.DB2IntegrationSuite test`
build/mvn -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.12 -Dtest=none -DwildcardSuites=org.apache.spark.sql.jdbc.MsSqlServerIntegrationSuite test`
build/mvn -Pdocker-integration-tests -pl :spark-docker-integration-tests_2.12 -Dtest=none -DwildcardSuites=org.apache.spark.sql.jdbc.PostgresIntegrationSuite test`
```

Closes #27892 from gaborgsomogyi/docker-client.

Authored-by: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b0d2956a359f00e703d5ebe9a58fb9fec869721e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 external/docker-integration-tests/pom.xml | 1 +
 pom.xml                                   | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index aff79b8b8e642..cdf76e9a5f69d 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -50,6 +50,7 @@
       <groupId>com.spotify</groupId>
       <artifactId>docker-client</artifactId>
       <scope>test</scope>
+      <classifier>shaded</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
diff --git a/pom.xml b/pom.xml
index 5aa100e57cb63..978127e91e1a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -931,8 +931,9 @@
       <dependency>
         <groupId>com.spotify</groupId>
         <artifactId>docker-client</artifactId>
-        <version>5.0.2</version>
+        <version>8.14.1</version>
         <scope>test</scope>
+        <classifier>shaded</classifier>
         <exclusions>
           <exclusion>
             <artifactId>guava</artifactId>

From 4c52e42cc77bbb049e387f0fce3dda7d3585c586 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 16 Mar 2020 16:48:31 +0900
Subject: [PATCH 0283/1280] [SPARK-30958][SQL] do not set default era for
 DateTimeFormatter

### What changes were proposed in this pull request?

It's not needed at all as now we replace "y" with "u" if there is no "G". So the era is either explicitly specified (e.g. "yyyy G") or can be inferred from the year (e.g. "uuuu").

### Why are the changes needed?

By default we use "uuuu" as the year pattern, which indicates the era already. If we set a default era, it can get conflicted and fail the parsing.

### Does this PR introduce any user-facing change?

yea, now spark can parse date/timestamp with negative year via the "yyyy" pattern, which will be converted to "uuuu" under the hood.

### How was this patch tested?

new tests

Closes #27707 from cloud-fan/bug.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 50a29672e061ca17598e657593db45f55aebc34b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../util/DateTimeFormatterHelper.scala        |  1 -
 .../spark/sql/util/DateFormatterSuite.scala   | 26 +++++++++++++--
 .../sql/util/TimestampFormatterSuite.scala    | 32 +++++++++++++++++--
 3 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index fab504174783e..a59f49bbe6b42 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -95,7 +95,6 @@ private object DateTimeFormatterHelper {
 
   def toFormatter(builder: DateTimeFormatterBuilder, locale: Locale): DateTimeFormatter = {
     builder
-      .parseDefaulting(ChronoField.ERA, 1)
       .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
       .parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
       .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index d617b1c1d8237..a40dbcc2ec664 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -17,9 +17,9 @@
 
 package org.apache.spark.sql.util
 
-import java.time.{LocalDate, ZoneOffset}
+import java.time.{DateTimeException, LocalDate, ZoneOffset}
 
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays}
@@ -114,4 +114,26 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
       assert(formatter.parse("tomorrow UTC") === today + 1)
     }
   }
+
+  test("SPARK-30958: parse date with negative year") {
+    val formatter1 = DateFormatter("yyyy-MM-dd", ZoneOffset.UTC)
+    assert(formatter1.parse("-1234-02-22") === localDateToDays(LocalDate.of(-1234, 2, 22)))
+
+    def assertParsingError(f: => Unit): Unit = {
+      intercept[Exception](f) match {
+        case e: SparkUpgradeException =>
+          assert(e.getCause.isInstanceOf[DateTimeException])
+        case e =>
+          assert(e.isInstanceOf[DateTimeException])
+      }
+    }
+
+    // "yyyy" with "G" can't parse negative year or year 0000.
+    val formatter2 = DateFormatter("G yyyy-MM-dd", ZoneOffset.UTC)
+    assertParsingError(formatter2.parse("BC -1234-02-22"))
+    assertParsingError(formatter2.parse("AD 0000-02-22"))
+
+    assert(formatter2.parse("BC 1234-02-22") === localDateToDays(LocalDate.of(-1233, 2, 22)))
+    assert(formatter2.parse("AD 1234-02-22") === localDateToDays(LocalDate.of(1234, 2, 22)))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 082849c88669a..959ef8eeb1b25 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -17,12 +17,12 @@
 
 package org.apache.spark.sql.util
 
-import java.time.{Instant, LocalDateTime, LocalTime, ZoneOffset}
+import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime, ZoneOffset}
 import java.util.concurrent.TimeUnit
 
 import org.scalatest.Matchers
 
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
@@ -112,7 +112,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     }
   }
 
-  test(" case insensitive parsing of am and pm") {
+  test("case insensitive parsing of am and pm") {
     val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", ZoneOffset.UTC)
     val micros = formatter.parse("2009 Mar 20 11:30:01 am")
     assert(micros === TimeUnit.SECONDS.toMicros(
@@ -233,4 +233,30 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         "2019-10-14 09:39:07")
     }
   }
+
+  test("SPARK-30958: parse timestamp with negative year") {
+    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
+    assert(formatter1.parse("-1234-02-22 02:22:22") === instantToMicros(
+      LocalDateTime.of(-1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+
+    def assertParsingError(f: => Unit): Unit = {
+      intercept[Exception](f) match {
+        case e: SparkUpgradeException =>
+          assert(e.getCause.isInstanceOf[DateTimeException])
+        case e =>
+          assert(e.isInstanceOf[DateTimeException])
+      }
+    }
+
+    // "yyyy" with "G" can't parse negative year or year 0000.
+    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
+    assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
+    assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
+
+    assert(formatter2.parse("BC 1234-02-22 02:22:22") === instantToMicros(
+      LocalDateTime.of(-1233, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+    assert(formatter2.parse("AD 1234-02-22 02:22:22") === instantToMicros(
+      LocalDateTime.of(1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+
+  }
 }

From 5902a232c14516f893895be0d673354933d8a5d0 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 16 Mar 2020 17:06:15 +0800
Subject: [PATCH 0284/1280] [SPARK-31076][SQL][FOLLOWUP] Incapsulate date
 rebasing to `DaysWritable`

### What changes were proposed in this pull request?
Move the code related to days rebasing from/to Julian calendar from `HiveInspectors` to new class `DaysWritable`.

### Why are the changes needed?
To improve maintainability of the `HiveInspectors` trait which is already pretty complex.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By `HiveOrcHadoopFsRelationSuite`.

Closes #27890 from MaxGekk/replace-DateWritable-by-DaysWritable.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 57854c736c2ca495eb03962f61857e1600864e95)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/hive/DaysWritable.scala  | 117 ++++++++++++++++++
 .../spark/sql/hive/HiveInspectors.scala       |  58 +--------
 2 files changed, 120 insertions(+), 55 deletions(-)
 create mode 100644 sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
new file mode 100644
index 0000000000000..53a0deb270fd1
--- /dev/null
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import java.io.{DataInput, DataOutput, IOException}
+import java.sql.Date
+import java.time.LocalDate
+import java.util.Calendar
+
+import org.apache.hadoop.hive.serde2.io.DateWritable
+import org.apache.hadoop.io.WritableUtils
+
+import org.apache.spark.sql.catalyst.util.{DateTimeConstants, DateTimeUtils}
+
+/**
+ * The class accepts/returns days in Gregorian calendar and rebase them
+ * via conversion to local date in Julian calendar for dates before 1582-10-15
+ * in read/write for backward compatibility with Spark 2.4 and earlier versions.
+ *
+ * @param gregorianDays The number of days since the epoch 1970-01-01 in
+ *                      Gregorian calendar.
+ * @param julianDays The number of days since the epoch 1970-01-01 in
+ *                   Julian calendar.
+ */
+private[hive] class DaysWritable(
+    var gregorianDays: Int,
+    var julianDays: Int)
+  extends DateWritable {
+
+  def this(gregorianDays: Int) =
+    this(gregorianDays, DaysWritable.rebaseGregorianToJulianDays(gregorianDays))
+  def this(dateWritable: DateWritable) = {
+    this(
+      gregorianDays = dateWritable match {
+        case daysWritable: DaysWritable => daysWritable.gregorianDays
+        case dateWritable: DateWritable =>
+        DaysWritable.rebaseJulianToGregorianDays(dateWritable.getDays)
+      },
+      julianDays = dateWritable.getDays)
+  }
+
+  override def getDays: Int = julianDays
+  override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
+
+  @throws[IOException]
+  override def write(out: DataOutput): Unit = {
+    WritableUtils.writeVInt(out, julianDays)
+  }
+
+  @throws[IOException]
+  override def readFields(in: DataInput): Unit = {
+    julianDays = WritableUtils.readVInt(in)
+    gregorianDays = DaysWritable.rebaseJulianToGregorianDays(julianDays)
+  }
+}
+
+private[hive] object DaysWritable {
+  // Rebasing days since the epoch to store the same number of days
+  // as by Spark 2.4 and earlier versions. Spark 3.0 switched to
+  // Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
+  // this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
+  // Julian calendar for dates before 1582-10-15. So, the same local date may
+  // be mapped to different number of days since the epoch in different calendars.
+  // For example:
+  // Proleptic Gregorian calendar: 1582-01-01 -> -141714
+  // Julian calendar: 1582-01-01 -> -141704
+  // The code below converts -141714 to -141704.
+  def rebaseGregorianToJulianDays(daysSinceEpoch: Int): Int = {
+    if (daysSinceEpoch < DateTimeUtils.GREGORIAN_CUTOVER_DAY) {
+      val millis = Math.multiplyExact(daysSinceEpoch, DateTimeConstants.MILLIS_PER_DAY)
+      val utcCal = new Calendar.Builder()
+        .setCalendarType("gregory")
+        .setTimeZone(DateTimeUtils.TimeZoneUTC)
+        .setInstant(millis)
+        .build()
+      val localDate = LocalDate.of(
+        utcCal.get(Calendar.YEAR),
+        utcCal.get(Calendar.MONTH) + 1,
+        utcCal.get(Calendar.DAY_OF_MONTH))
+      Math.toIntExact(localDate.toEpochDay)
+    } else {
+      daysSinceEpoch
+    }
+  }
+
+  def rebaseJulianToGregorianDays(daysSinceEpoch: Int): Int = {
+    if (daysSinceEpoch < JULIAN_CUTOVER_DAY) {
+      val localDate = LocalDate.ofEpochDay(daysSinceEpoch)
+      val utcCal = new Calendar.Builder()
+        .setCalendarType("gregory")
+        .setTimeZone(DateTimeUtils.TimeZoneUTC)
+        .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+        .build()
+      Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, DateTimeConstants.MILLIS_PER_DAY))
+    } else {
+      daysSinceEpoch
+    }
+  }
+
+  final val JULIAN_CUTOVER_DAY =
+    rebaseGregorianToJulianDays(DateTimeUtils.GREGORIAN_CUTOVER_DAY.toInt)
+}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index e217c5279046e..e3e9a3144728d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -18,8 +18,6 @@
 package org.apache.spark.sql.hive
 
 import java.lang.reflect.{ParameterizedType, Type, WildcardType}
-import java.time.LocalDate
-import java.util.Calendar
 
 import scala.collection.JavaConverters._
 
@@ -182,33 +180,6 @@ import org.apache.spark.unsafe.types.UTF8String
  */
 private[hive] trait HiveInspectors {
 
-  private final val JULIAN_CUTOVER_DAY =
-    rebaseGregorianToJulianDays(DateTimeUtils.GREGORIAN_CUTOVER_DAY.toInt)
-
-  private def rebaseJulianToGregorianDays(daysSinceEpoch: Int): Int = {
-    val localDate = LocalDate.ofEpochDay(daysSinceEpoch)
-    val utcCal = new Calendar.Builder()
-      .setCalendarType("gregory")
-      .setTimeZone(DateTimeUtils.TimeZoneUTC)
-      .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
-      .build()
-    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, DateTimeConstants.MILLIS_PER_DAY))
-  }
-
-  private def rebaseGregorianToJulianDays(daysSinceEpoch: Int): Int = {
-    val millis = Math.multiplyExact(daysSinceEpoch, DateTimeConstants.MILLIS_PER_DAY)
-    val utcCal = new Calendar.Builder()
-      .setCalendarType("gregory")
-      .setTimeZone(DateTimeUtils.TimeZoneUTC)
-      .setInstant(millis)
-      .build()
-    val localDate = LocalDate.of(
-      utcCal.get(Calendar.YEAR),
-      utcCal.get(Calendar.MONTH) + 1,
-      utcCal.get(Calendar.DAY_OF_MONTH))
-    Math.toIntExact(localDate.toEpochDay)
-  }
-
   def javaTypeToDataType(clz: Type): DataType = clz match {
     // writable
     case c: Class[_] if c == classOf[hadoopIo.DoubleWritable] => DoubleType
@@ -646,14 +617,7 @@ private[hive] trait HiveInspectors {
         case x: DateObjectInspector if x.preferWritable() =>
           data: Any => {
             if (data != null) {
-              // Rebasing written days via conversion to local dates.
-              // See the comment for `getDateWritable()`.
-              val daysSinceEpoch = x.getPrimitiveWritableObject(data).getDays
-              if (daysSinceEpoch < JULIAN_CUTOVER_DAY) {
-                rebaseJulianToGregorianDays(daysSinceEpoch)
-              } else {
-                daysSinceEpoch
-              }
+              new DaysWritable(x.getPrimitiveWritableObject(data)).gregorianDays
             } else {
               null
             }
@@ -1045,27 +1009,11 @@ private[hive] trait HiveInspectors {
       new hadoopIo.BytesWritable(value.asInstanceOf[Array[Byte]])
     }
 
-  private def getDateWritable(value: Any): hiveIo.DateWritable =
+  private def getDateWritable(value: Any): DaysWritable =
     if (value == null) {
       null
     } else {
-      // Rebasing days since the epoch to store the same number of days
-      // as by Spark 2.4 and earlier versions. Spark 3.0 switched to
-      // Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
-      // this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
-      // Julian calendar for dates before 1582-10-15. So, the same local date may
-      // be mapped to different number of days since the epoch in different calendars.
-      // For example:
-      // Proleptic Gregorian calendar: 1582-01-01 -> -141714
-      // Julian calendar: 1582-01-01 -> -141704
-      // The code below converts -141714 to -141704.
-      val daysSinceEpoch = value.asInstanceOf[Int]
-      val rebasedDays = if (daysSinceEpoch < DateTimeUtils.GREGORIAN_CUTOVER_DAY) {
-        rebaseGregorianToJulianDays(daysSinceEpoch)
-      } else {
-        daysSinceEpoch
-      }
-      new hiveIo.DateWritable(rebasedDays)
+      new DaysWritable(value.asInstanceOf[Int])
     }
 
   private def getTimestampWritable(value: Any): hiveIo.TimestampWritable =

From c9f3370cb6b021ad2e7a8c6cc71109de353f6f04 Mon Sep 17 00:00:00 2001
From: jiake <ke.a.jia@intel.com>
Date: Mon, 16 Mar 2020 23:33:56 +0800
Subject: [PATCH 0285/1280] [SPARK-30864][SQL][DOC] add the user guide for
 Adaptive Query Execution

### What changes were proposed in this pull request?
This PR will add the user guide for AQE and the detailed configurations about the three mainly features in AQE.

### Why are the changes needed?
Add the detailed configurations.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
only add doc no need ut.

Closes #27616 from JkSelf/aqeuserguide.

Authored-by: jiake <ke.a.jia@intel.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 21c02ee5d0c0f5951927ff1654cf1351c8f066e6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-performance-tuning.md                | 60 +++++++++++++++++++
 .../adaptive/OptimizeSkewedJoin.scala         |  2 +-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 5a86c0cc31e12..489575d01ed64 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -186,3 +186,63 @@ The "REPARTITION_BY_RANGE" hint must have column names and a partition number is
     SELECT /*+ REPARTITION(3, c) */ * FROM t
     SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t
     SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t
+
+## Adaptive Query Execution
+Adaptive Query Execution (AQE) is an optimization technique in Spark SQL that makes use of the runtime statistics to choose the most efficient query execution plan. AQE is disabled by default. Spark SQL can use the umbrella configuration of `spark.sql.adaptive.enabled` to control whether turn it on/off. As of Spark 3.0, there are three major features in AQE, including coalescing post-shuffle partitions, converting sort-merge join to broadcast join, and skew join optimization.
+
+### Coalescing Post Shuffle Partitions
+This feature coalesces the post shuffle partitions based on the map output statistics when both `spark.sql.adaptive.enabled` and `spark.sql.adaptive.coalescePartitions.enabled` configurations are true. This feature simplifies the tuning of shuffle partition number when running queries. You do not need to set a proper shuffle partition number to fit your dataset. Spark can pick the proper shuffle partition number at runtime once you set a large enough initial number of shuffle partitions via `spark.sql.adaptive.coalescePartitions.initialPartitionNum` configuration.
+ <table class="table">
+   <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+   <tr>
+     <td><code>spark.sql.adaptive.coalescePartitions.enabled</code></td>
+     <td>true</td>
+     <td>
+       When true and <code>spark.sql.adaptive.enabled</code> is true, Spark will coalesce contiguous shuffle partitions according to the target size (specified by <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>), to avoid too many small tasks.
+     </td>
+   </tr>
+   <tr>
+     <td><code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code></td>
+     <td>Default Parallelism</td>
+     <td>
+       The minimum number of shuffle partitions after coalescing. If not set, the default value is the default parallelism of the Spark cluster. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
+     </td>
+   </tr>
+   <tr>
+     <td><code>spark.sql.adaptive.coalescePartitions.initialPartitionNum</code></td>
+     <td>200</td>
+     <td>
+       The initial number of shuffle partitions before coalescing. By default it equals to <code>spark.sql.shuffle.partitions</code>. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
+     </td>
+   </tr>
+   <tr>
+     <td><code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code></td>
+     <td>64 MB</td>
+     <td>
+       The advisory size in bytes of the shuffle partition during adaptive optimization (when <code>spark.sql.adaptive.enabled</code> is true). It takes effect when Spark coalesces small shuffle partitions or splits skewed shuffle partition.
+     </td>
+   </tr>
+ </table>
+ 
+### Converting sort-merge join to broadcast join
+AQE converts sort-merge join to broadcast hash join when the runtime statistics of any join side is smaller than the broadcast hash join threshold. This is not as efficient as planning a broadcast hash join in the first place, but it's better than keep doing the sort-merge join, as we can save the sorting of both the join sides, and read shuffle files locally to save network traffic(if `spark.sql.adaptive.localShuffleReader.enabled` is true)
+
+### Optimizing Skew Join
+Data skew can severely downgrade the performance of join queries. This feature dynamically handles skew in sort-merge join by splitting (and replicating if needed) skewed tasks into roughly evenly sized tasks. It takes effect when both `spark.sql.adaptive.enabled` and `spark.sql.adaptive.skewJoin.enabled` configurations are enabled.
+  <table class="table">
+     <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+     <tr>
+       <td><code>spark.sql.adaptive.skewJoin.enabled</code></td>
+       <td>true</td>
+       <td>
+         When true and <code>spark.sql.adaptive.enabled</code> is true, Spark dynamically handles skew in sort-merge join by splitting (and replicating if needed) skewed partitions.
+       </td>
+     </tr>
+     <tr>
+       <td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
+       <td>10</td>
+       <td>
+         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>.
+       </td>
+     </tr>
+   </table>
\ No newline at end of file
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index a75a3f37f6672..db65af614e0b9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -304,7 +304,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
     if (shuffleStages.length == 2) {
       // When multi table join, there will be too many complex combination to consider.
-      // Currently we only handle 2 table join like following two use cases.
+      // Currently we only handle 2 table join like following use case.
       // SMJ
       //   Sort
       //     Shuffle

From da1f95be6b9af59a91a14e01613bdc4e8ac35374 Mon Sep 17 00:00:00 2001
From: "Tae-kyeom, Kim" <kimtkyeom@devsisters.com>
Date: Mon, 16 Mar 2020 10:31:56 -0700
Subject: [PATCH 0286/1280] [SPARK-31116][SQL] Fix nested schema
 case-sensitivity in ParquetRowConverter

### What changes were proposed in this pull request?

This PR (SPARK-31116) add caseSensitive parameter to ParquetRowConverter so that it handle materialize parquet properly with respect to case sensitivity

### Why are the changes needed?

From spark 3.0.0, below statement throws IllegalArgumentException in caseInsensitive mode because of explicit field index searching in ParquetRowConverter. As we already constructed parquet requested schema and catalyst requested schema during schema clipping in ParquetReadSupport, just follow these behavior.

```scala
val path = "/some/temp/path"

spark
  .range(1L)
  .selectExpr("NAMED_STRUCT('lowercase', id, 'camelCase', id + 1) AS StructColumn")
  .write.parquet(path)

val caseInsensitiveSchema = new StructType()
  .add(
    "StructColumn",
    new StructType()
      .add("LowerCase", LongType)
      .add("camelcase", LongType))

spark.read.schema(caseInsensitiveSchema).parquet(path).show()
```

### Does this PR introduce any user-facing change?

No. The changes are only in unreleased branches (`master` and `branch-3.0`).

### How was this patch tested?

Passed new test cases that check parquet column selection with respect to schemas and case sensitivities

Closes #27888 from kimtkyeom/parquet_row_converter_case_sensitivity.

Authored-by: Tae-kyeom, Kim <kimtkyeom@devsisters.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e736c62764137b2c3af90d2dc8a77e391891200a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../parquet/ParquetRowConverter.scala         | 12 +++++-
 .../spark/sql/FileBasedDataSourceSuite.scala  | 40 +++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 850adae8a6b95..22422c02fd666 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -33,8 +33,9 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.{BINARY, DOUBLE
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -178,8 +179,15 @@ private[parquet] class ParquetRowConverter(
 
   // Converters for each field.
   private[this] val fieldConverters: Array[Converter with HasParentContainerUpdater] = {
+    // (SPARK-31116) Use case insensitive map if spark.sql.caseSensitive is false
+    // to prevent throwing IllegalArgumentException when searching catalyst type's field index
+    val catalystFieldNameToIndex = if (SQLConf.get.caseSensitiveAnalysis) {
+      catalystType.fieldNames.zipWithIndex.toMap
+    } else {
+      CaseInsensitiveMap(catalystType.fieldNames.zipWithIndex.toMap)
+    }
     parquetType.getFields.asScala.map { parquetField =>
-      val fieldIndex = catalystType.fieldIndex(parquetField.getName)
+      val fieldIndex = catalystFieldNameToIndex(parquetField.getName)
       val catalystField = catalystType(fieldIndex)
       // Converted field value should be set to the `fieldIndex`-th cell of `currentRow`
       newConverter(parquetField, catalystField.dataType, new RowUpdater(currentRow, fieldIndex))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index c870958128483..cb410b4f0d7dc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -842,6 +842,46 @@ class FileBasedDataSourceSuite extends QueryTest
       }
     }
   }
+
+  test("SPARK-31116: Select nested schema with case insensitive mode") {
+    // This test case failed at only Parquet. ORC is added for test coverage parity.
+    Seq("orc", "parquet").foreach { format =>
+      Seq("true", "false").foreach { nestedSchemaPruningEnabled =>
+        withSQLConf(
+          SQLConf.CASE_SENSITIVE.key -> "false",
+          SQLConf.NESTED_SCHEMA_PRUNING_ENABLED.key -> nestedSchemaPruningEnabled) {
+          withTempPath { dir =>
+            val path = dir.getCanonicalPath
+
+            // Prepare values for testing nested parquet data
+            spark
+              .range(1L)
+              .selectExpr("NAMED_STRUCT('lowercase', id, 'camelCase', id + 1) AS StructColumn")
+              .write
+              .format(format)
+              .save(path)
+
+            val exactSchema = "StructColumn struct<lowercase: LONG, camelCase: LONG>"
+
+            checkAnswer(spark.read.schema(exactSchema).format(format).load(path), Row(Row(0, 1)))
+
+            // In case insensitive manner, parquet's column cases are ignored
+            val innerColumnCaseInsensitiveSchema =
+              "StructColumn struct<Lowercase: LONG, camelcase: LONG>"
+            checkAnswer(
+              spark.read.schema(innerColumnCaseInsensitiveSchema).format(format).load(path),
+              Row(Row(0, 1)))
+
+            val rootColumnCaseInsensitiveSchema =
+              "structColumn struct<lowercase: LONG, camelCase: LONG>"
+            checkAnswer(
+              spark.read.schema(rootColumnCaseInsensitiveSchema).format(format).load(path),
+              Row(Row(0, 1)))
+          }
+        }
+      }
+    }
+  }
 }
 
 object TestingUDT {

From 9e8d9393f2c20d6cb66c1790cc42bd2b5d840258 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 16 Mar 2020 12:41:22 -0500
Subject: [PATCH 0287/1280] [SPARK-30930][ML] Remove ML/MLLIB DeveloperApi
 annotations

### What changes were proposed in this pull request?
jira link: https://issues.apache.org/jira/browse/SPARK-30930

Remove ML/MLLIB DeveloperApi annotations.

### Why are the changes needed?

The Developer APIs in ML/MLLIB have been there for a long time. They are stable now and are very unlikely to be changed or removed, so I unmark these Developer APIs in this PR.

### Does this PR introduce any user-facing change?
Yes. DeveloperApi annotations are removed from docs.

### How was this patch tested?
existing tests

Closes #27859 from huaxingao/spark-30930.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 3ce1dff7ba4799295a2bf89e6df35e900afc587b)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../scala/org/apache/spark/ml/Estimator.scala |  4 +--
 .../scala/org/apache/spark/ml/Model.scala     |  3 --
 .../scala/org/apache/spark/ml/Pipeline.scala  |  4 ---
 .../scala/org/apache/spark/ml/Predictor.scala |  6 +---
 .../org/apache/spark/ml/Transformer.scala     |  6 +---
 .../spark/ml/attribute/AttributeGroup.scala   |  5 ---
 .../spark/ml/attribute/AttributeType.scala    |  7 ----
 .../spark/ml/attribute/attributes.scala       | 21 ------------
 .../spark/ml/classification/Classifier.scala  |  8 +----
 .../ProbabilisticClassifier.scala             |  8 +----
 .../org/apache/spark/ml/clustering/LDA.scala  |  8 +----
 .../spark/ml/evaluation/Evaluator.scala       |  4 +--
 .../apache/spark/ml/linalg/SQLDataTypes.scala |  4 +--
 .../org/apache/spark/ml/param/params.scala    | 28 +---------------
 .../ml/param/shared/SharedParamsCodeGen.scala |  2 --
 .../spark/ml/param/shared/sharedParams.scala  | 33 -------------------
 .../apache/spark/ml/recommendation/ALS.scala  |  8 +----
 .../spark/ml/regression/Regressor.scala       |  5 ---
 .../org/apache/spark/ml/tree/treeParams.scala |  6 ++--
 .../apache/spark/ml/util/Identifiable.scala   | 10 +-----
 .../org/apache/spark/ml/util/ReadWrite.scala  |  8 +----
 .../apache/spark/mllib/clustering/LDA.scala   |  8 +----
 .../spark/mllib/clustering/LDAOptimizer.scala | 11 +------
 .../spark/mllib/feature/StandardScaler.scala  | 10 +-----
 .../mllib/feature/VectorTransformer.scala     |  4 +--
 .../spark/mllib/optimization/Gradient.scala   |  9 -----
 .../mllib/optimization/GradientDescent.scala  |  5 ---
 .../spark/mllib/optimization/LBFGS.scala      |  5 ---
 .../spark/mllib/optimization/Optimizer.scala  |  3 --
 .../spark/mllib/optimization/Updater.scala    |  9 -----
 .../spark/mllib/pmml/PMMLExportable.scala     |  4 +--
 .../mllib/random/RandomDataGenerator.scala    | 18 +---------
 .../spark/mllib/random/RandomRDDs.scala       | 20 ++---------
 .../spark/mllib/rdd/MLPairRDDFunctions.scala  |  7 ----
 .../apache/spark/mllib/rdd/RDDFunctions.scala |  8 +----
 .../spark/mllib/recommendation/ALS.scala      |  8 +----
 .../GeneralizedLinearAlgorithm.scala          |  6 +---
 .../regression/StreamingLinearAlgorithm.scala |  4 +--
 .../stat/MultivariateOnlineSummarizer.scala   |  4 +--
 .../distribution/MultivariateGaussian.scala   |  4 +--
 .../spark/mllib/tree/impurity/Entropy.scala   |  6 +---
 .../spark/mllib/tree/impurity/Gini.scala      |  6 +---
 .../spark/mllib/tree/impurity/Impurity.scala  |  6 +---
 .../spark/mllib/tree/impurity/Variance.scala  |  7 +---
 .../spark/mllib/tree/loss/AbsoluteError.scala |  4 +--
 .../spark/mllib/tree/loss/LogLoss.scala       |  4 +--
 .../apache/spark/mllib/tree/loss/Loss.scala   |  4 +--
 .../spark/mllib/tree/loss/SquaredError.scala  |  4 +--
 .../tree/model/InformationGainStats.scala     |  4 +--
 .../apache/spark/mllib/tree/model/Node.scala  |  4 +--
 .../spark/mllib/tree/model/Predict.scala      |  4 +--
 .../apache/spark/mllib/tree/model/Split.scala |  4 +--
 .../mllib/tree/model/treeEnsembleModels.scala |  6 +---
 .../spark/mllib/util/DataValidators.scala     |  4 +--
 .../mllib/util/KMeansDataGenerator.scala      |  4 +--
 .../mllib/util/LinearDataGenerator.scala      |  4 +--
 .../LogisticRegressionDataGenerator.scala     |  4 +--
 .../spark/mllib/util/MFDataGenerator.scala    |  4 +--
 .../spark/mllib/util/SVMDataGenerator.scala   |  4 +--
 .../spark/mllib/util/modelSaveLoad.scala      |  8 +----
 python/pyspark/ml/base.py                     |  2 --
 python/pyspark/ml/param/__init__.py           |  2 --
 python/pyspark/ml/pipeline.py                 |  2 --
 python/pyspark/ml/util.py                     |  8 -----
 python/pyspark/mllib/feature.py               |  2 --
 65 files changed, 49 insertions(+), 397 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
index 1247882d6c1bd..8815eb29bc860 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala
@@ -19,15 +19,13 @@ package org.apache.spark.ml
 
 import scala.annotation.varargs
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.param.{ParamMap, ParamPair}
 import org.apache.spark.sql.Dataset
 
 /**
- * :: DeveloperApi ::
  * Abstract class for estimators that fit models to data.
  */
-@DeveloperApi
 abstract class Estimator[M <: Model[M]] extends PipelineStage {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Model.scala b/mllib/src/main/scala/org/apache/spark/ml/Model.scala
index c581fed177273..709a1124cbb5e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Model.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Model.scala
@@ -17,16 +17,13 @@
 
 package org.apache.spark.ml
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.param.ParamMap
 
 /**
- * :: DeveloperApi ::
  * A fitted model, i.e., a [[Transformer]] produced by an [[Estimator]].
  *
  * @tparam M model type
  */
-@DeveloperApi
 abstract class Model[M <: Model[M]] extends Transformer {
   /**
    * The parent estimator that produced this model.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
index 69a4dbef138e3..59e95878a5fec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
@@ -36,14 +36,11 @@ import org.apache.spark.sql.{DataFrame, Dataset}
 import org.apache.spark.sql.types.StructType
 
 /**
- * :: DeveloperApi ::
  * A stage in a pipeline, either an [[Estimator]] or a [[Transformer]].
  */
-@DeveloperApi
 abstract class PipelineStage extends Params with Logging {
 
   /**
-   * :: DeveloperApi ::
    *
    * Check transform validity and derive the output schema from the input schema.
    *
@@ -54,7 +51,6 @@ abstract class PipelineStage extends Params with Logging {
    * Typical implementation should first conduct verification on schema change and parameter
    * validity, including complex parameter interaction checks.
    */
-  @DeveloperApi
   def transformSchema(schema: StructType): StructType
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
index 98dd692cbe55d..febeba7e13fcb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Predictor.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.ml
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
 import org.apache.spark.ml.param._
@@ -99,7 +99,6 @@ private[ml] trait PredictorParams extends Params
 }
 
 /**
- * :: DeveloperApi ::
  * Abstraction for prediction problems (regression and classification). It accepts all NumericType
  * labels and will automatically cast it to DoubleType in `fit()`. If this predictor supports
  * weights, it accepts all NumericType weights, which will be automatically casted to DoubleType
@@ -112,7 +111,6 @@ private[ml] trait PredictorParams extends Params
  * @tparam M  Specialization of [[PredictionModel]].  If you subclass this type, use this type
  *            parameter to specify the concrete type for the corresponding model.
  */
-@DeveloperApi
 abstract class Predictor[
     FeaturesType,
     Learner <: Predictor[FeaturesType, Learner, M],
@@ -190,7 +188,6 @@ abstract class Predictor[
 }
 
 /**
- * :: DeveloperApi ::
  * Abstraction for a model for prediction tasks (regression and classification).
  *
  * @tparam FeaturesType  Type of features.
@@ -198,7 +195,6 @@ abstract class Predictor[
  * @tparam M  Specialization of [[PredictionModel]].  If you subclass this type, use this type
  *            parameter to specify the concrete type for the corresponding model.
  */
-@DeveloperApi
 abstract class PredictionModel[FeaturesType, M <: PredictionModel[FeaturesType, M]]
   extends Model[M] with PredictorParams {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
index 1652131a9003a..3b60b5ae294ea 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Transformer.scala
@@ -20,7 +20,7 @@ package org.apache.spark.ml
 import scala.annotation.varargs
 import scala.reflect.runtime.universe.TypeTag
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.param._
 import org.apache.spark.ml.param.shared._
@@ -29,10 +29,8 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
 
 /**
- * :: DeveloperApi ::
  * Abstract class for transformers that transform one dataset into another.
  */
-@DeveloperApi
 abstract class Transformer extends PipelineStage {
 
   /**
@@ -75,11 +73,9 @@ abstract class Transformer extends PipelineStage {
 }
 
 /**
- * :: DeveloperApi ::
  * Abstract class for transformers that take one input column, apply transformation, and output the
  * result as a new column.
  */
-@DeveloperApi
 abstract class UnaryTransformer[IN: TypeTag, OUT: TypeTag, T <: UnaryTransformer[IN, OUT, T]]
   extends Transformer with HasInputCol with HasOutputCol with Logging {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
index 21a246e454c83..59d48915c0d58 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeGroup.scala
@@ -19,12 +19,10 @@ package org.apache.spark.ml.attribute
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.linalg.VectorUDT
 import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField}
 
 /**
- * :: DeveloperApi ::
  * Attributes that describe a vector ML column.
  *
  * @param name name of the attribute group (the ML column name)
@@ -33,7 +31,6 @@ import org.apache.spark.sql.types.{Metadata, MetadataBuilder, StructField}
  * @param attrs optional array of attributes. Attribute will be copied with their corresponding
  *              indices in the array.
  */
-@DeveloperApi
 class AttributeGroup private (
     val name: String,
     val numAttributes: Option[Int],
@@ -188,10 +185,8 @@ class AttributeGroup private (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods to create attribute groups.
  */
-@DeveloperApi
 object AttributeGroup {
 
   import AttributeKeys._
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala
index 078fecf088282..f1f3a97f8ac1c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/AttributeType.scala
@@ -17,20 +17,13 @@
 
 package org.apache.spark.ml.attribute
 
-import org.apache.spark.annotation.DeveloperApi
 
 /**
- * :: DeveloperApi ::
  * An enum-like type for attribute types: [[AttributeType$#Numeric]], [[AttributeType$#Nominal]],
  * and [[AttributeType$#Binary]].
  */
-@DeveloperApi
 sealed abstract class AttributeType(val name: String)
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
 object AttributeType {
 
   /** Numeric type. */
diff --git a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
index 756dd677cb579..576c771d83bec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/attribute/attributes.scala
@@ -19,14 +19,11 @@ package org.apache.spark.ml.attribute
 
 import scala.annotation.varargs
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.sql.types.{DoubleType, Metadata, MetadataBuilder, NumericType, StructField}
 
 /**
- * :: DeveloperApi ::
  * Abstract class for ML attributes.
  */
-@DeveloperApi
 sealed abstract class Attribute extends Serializable {
 
   name.foreach { n =>
@@ -150,10 +147,6 @@ private[attribute] trait AttributeFactory {
   def fromStructField(field: StructField): Attribute = decodeStructField(field, false)
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
 object Attribute extends AttributeFactory {
 
   private[attribute] override def fromMetadata(metadata: Metadata): Attribute = {
@@ -182,7 +175,6 @@ object Attribute extends AttributeFactory {
 
 
 /**
- * :: DeveloperApi ::
  * A numeric attribute with optional summary statistics.
  * @param name optional name
  * @param index optional index
@@ -191,7 +183,6 @@ object Attribute extends AttributeFactory {
  * @param std optional standard deviation
  * @param sparsity optional sparsity (ratio of zeros)
  */
-@DeveloperApi
 class NumericAttribute private[ml] (
     override val name: Option[String] = None,
     override val index: Option[Int] = None,
@@ -299,10 +290,8 @@ class NumericAttribute private[ml] (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for numeric attributes.
  */
-@DeveloperApi
 object NumericAttribute extends AttributeFactory {
 
   /** The default numeric attribute. */
@@ -321,7 +310,6 @@ object NumericAttribute extends AttributeFactory {
 }
 
 /**
- * :: DeveloperApi ::
  * A nominal attribute.
  * @param name optional name
  * @param index optional index
@@ -330,7 +318,6 @@ object NumericAttribute extends AttributeFactory {
  *                  defined.
  * @param values optional values. At most one of `numValues` and `values` can be defined.
  */
-@DeveloperApi
 class NominalAttribute private[ml] (
     override val name: Option[String] = None,
     override val index: Option[Int] = None,
@@ -464,10 +451,8 @@ class NominalAttribute private[ml] (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for nominal attributes.
  */
-@DeveloperApi
 object NominalAttribute extends AttributeFactory {
 
   /** The default nominal attribute. */
@@ -487,13 +472,11 @@ object NominalAttribute extends AttributeFactory {
 }
 
 /**
- * :: DeveloperApi ::
  * A binary attribute.
  * @param name optional name
  * @param index optional index
  * @param values optional values. If set, its size must be 2.
  */
-@DeveloperApi
 class BinaryAttribute private[ml] (
     override val name: Option[String] = None,
     override val index: Option[Int] = None,
@@ -566,10 +549,8 @@ class BinaryAttribute private[ml] (
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for binary attributes.
  */
-@DeveloperApi
 object BinaryAttribute extends AttributeFactory {
 
   /** The default binary attribute. */
@@ -586,10 +567,8 @@ object BinaryAttribute extends AttributeFactory {
 }
 
 /**
- * :: DeveloperApi ::
  * An unresolved attribute.
  */
-@DeveloperApi
 object UnresolvedAttribute extends Attribute {
 
   override def attrType: AttributeType = AttributeType.Unresolved
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
index ac92299a931bc..1f3f291644f93 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ml.classification
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
 import org.apache.spark.ml.feature.{Instance, LabeledPoint}
 import org.apache.spark.ml.linalg.{Vector, VectorUDT}
@@ -62,8 +62,6 @@ private[spark] trait ClassifierParams
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Single-label binary or multiclass classification.
  * Classes are indexed {0, 1, ..., numClasses - 1}.
  *
@@ -71,7 +69,6 @@ private[spark] trait ClassifierParams
  * @tparam E  Concrete Estimator type
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class Classifier[
     FeaturesType,
     E <: Classifier[FeaturesType, E, M],
@@ -166,15 +163,12 @@ abstract class Classifier[
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Model produced by a [[Classifier]].
  * Classes are indexed {0, 1, ..., numClasses - 1}.
  *
  * @tparam FeaturesType  Type of input features.  E.g., `Vector`
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[FeaturesType, M]]
   extends PredictionModel[FeaturesType, M] with ClassifierParams {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
index cc5da3446f1f8..9758e3ca72c38 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/ProbabilisticClassifier.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.ml.classification
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.linalg.{DenseVector, Vector, VectorUDT}
 import org.apache.spark.ml.param.shared._
 import org.apache.spark.ml.util.SchemaUtils
@@ -41,15 +41,12 @@ private[ml] trait ProbabilisticClassifierParams
 
 
 /**
- * :: DeveloperApi ::
- *
  * Single-label binary or multiclass classifier which can output class conditional probabilities.
  *
  * @tparam FeaturesType  Type of input features.  E.g., `Vector`
  * @tparam E  Concrete Estimator type
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class ProbabilisticClassifier[
     FeaturesType,
     E <: ProbabilisticClassifier[FeaturesType, E, M],
@@ -65,15 +62,12 @@ abstract class ProbabilisticClassifier[
 
 
 /**
- * :: DeveloperApi ::
- *
  * Model produced by a [[ProbabilisticClassifier]].
  * Classes are indexed {0, 1, ..., numClasses - 1}.
  *
  * @tparam FeaturesType  Type of input features.  E.g., `Vector`
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class ProbabilisticClassificationModel[
     FeaturesType,
     M <: ProbabilisticClassificationModel[FeaturesType, M]]
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index e30be8c20dcc3..4e86b3b247ace 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -26,7 +26,7 @@ import org.json4s.DefaultFormats
 import org.json4s.JsonAST.JObject
 import org.json4s.jackson.JsonMethods._
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.linalg._
@@ -762,8 +762,6 @@ class DistributedLDAModel private[ml] (
   private var _checkpointFiles: Array[String] = oldDistributedModel.checkpointFiles
 
   /**
-   * :: DeveloperApi ::
-   *
    * If using checkpointing and `LDA.keepLastCheckpoint` is set to true, then there may be
    * saved checkpoint files.  This method is provided so that users can manage those files.
    *
@@ -773,18 +771,14 @@ class DistributedLDAModel private[ml] (
    *
    * @return  Checkpoint files from training
    */
-  @DeveloperApi
   @Since("2.0.0")
   def getCheckpointFiles: Array[String] = _checkpointFiles
 
   /**
-   * :: DeveloperApi ::
-   *
    * Remove any remaining checkpoint files from training.
    *
    * @see [[getCheckpointFiles]]
    */
-  @DeveloperApi
   @Since("2.0.0")
   def deleteCheckpointFiles(): Unit = {
     val hadoopConf = sparkSession.sparkContext.hadoopConfiguration
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala
index e7b949ddce344..f7a2d0dbe998f 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/Evaluator.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.ml.evaluation
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.param.{ParamMap, Params}
 import org.apache.spark.sql.Dataset
 
 /**
- * :: DeveloperApi ::
  * Abstract class for evaluators that compute metrics from predictions.
  */
 @Since("1.5.0")
-@DeveloperApi
 abstract class Evaluator extends Params {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala
index a66ba27a7b9c5..1cab37a25ad57 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/SQLDataTypes.scala
@@ -17,15 +17,13 @@
 
 package org.apache.spark.ml.linalg
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.sql.types.DataType
 
 /**
- * :: DeveloperApi ::
  * SQL data types for vectors and matrices.
  */
 @Since("2.0.0")
-@DeveloperApi
 object SQLDataTypes {
 
   /** Data type for [[Vector]]. */
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
index 4c50f1e3292bc..53ca35ccd0073 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
@@ -29,12 +29,11 @@ import org.json4s._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.ml.linalg.{JsonMatrixConverter, JsonVectorConverter, Matrix, Vector}
 import org.apache.spark.ml.util.Identifiable
 
 /**
- * :: DeveloperApi ::
  * A param with self-contained documentation and optionally default value. Primitive-typed param
  * should use the specialized versions, which are more friendly to Java users.
  *
@@ -45,7 +44,6 @@ import org.apache.spark.ml.util.Identifiable
  *                See [[ParamValidators]] for factory methods for common validation functions.
  * @tparam T param value type
  */
-@DeveloperApi
 class Param[T](val parent: String, val name: String, val doc: String, val isValid: T => Boolean)
   extends Serializable {
 
@@ -159,11 +157,9 @@ private[ml] object Param {
 }
 
 /**
- * :: DeveloperApi ::
  * Factory methods for common validation functions for `Param.isValid`.
  * The numerical methods only support Int, Long, Float, and Double.
  */
-@DeveloperApi
 object ParamValidators {
 
   /** (private[param]) Default validation always return true */
@@ -323,10 +319,8 @@ object ParamValidators {
 // specialize primitive-typed params because Java doesn't recognize scala.Double, scala.Int, ...
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Double]` for Java.
  */
-@DeveloperApi
 class DoubleParam(parent: String, name: String, doc: String, isValid: Double => Boolean)
   extends Param[Double](parent, name, doc, isValid) {
 
@@ -383,10 +377,8 @@ private[param] object DoubleParam {
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Int]` for Java.
  */
-@DeveloperApi
 class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolean)
   extends Param[Int](parent, name, doc, isValid) {
 
@@ -412,10 +404,8 @@ class IntParam(parent: String, name: String, doc: String, isValid: Int => Boolea
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Float]` for Java.
  */
-@DeveloperApi
 class FloatParam(parent: String, name: String, doc: String, isValid: Float => Boolean)
   extends Param[Float](parent, name, doc, isValid) {
 
@@ -473,10 +463,8 @@ private object FloatParam {
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Long]` for Java.
  */
-@DeveloperApi
 class LongParam(parent: String, name: String, doc: String, isValid: Long => Boolean)
   extends Param[Long](parent, name, doc, isValid) {
 
@@ -502,10 +490,8 @@ class LongParam(parent: String, name: String, doc: String, isValid: Long => Bool
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Boolean]` for Java.
  */
-@DeveloperApi
 class BooleanParam(parent: String, name: String, doc: String) // No need for isValid
   extends Param[Boolean](parent, name, doc) {
 
@@ -525,10 +511,8 @@ class BooleanParam(parent: String, name: String, doc: String) // No need for isV
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[String]]` for Java.
  */
-@DeveloperApi
 class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array[String] => Boolean)
   extends Param[Array[String]](parent, name, doc, isValid) {
 
@@ -550,10 +534,8 @@ class StringArrayParam(parent: Params, name: String, doc: String, isValid: Array
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[Double]]` for Java.
  */
-@DeveloperApi
 class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array[Double] => Boolean)
   extends Param[Array[Double]](parent, name, doc, isValid) {
 
@@ -580,10 +562,8 @@ class DoubleArrayParam(parent: Params, name: String, doc: String, isValid: Array
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[Array[Double]]]` for Java.
  */
-@DeveloperApi
 class DoubleArrayArrayParam(
     parent: Params,
     name: String,
@@ -619,10 +599,8 @@ class DoubleArrayArrayParam(
 }
 
 /**
- * :: DeveloperApi ::
  * Specialized version of `Param[Array[Int]]` for Java.
  */
-@DeveloperApi
 class IntArrayParam(parent: Params, name: String, doc: String, isValid: Array[Int] => Boolean)
   extends Param[Array[Int]](parent, name, doc, isValid) {
 
@@ -657,11 +635,9 @@ case class ParamPair[T] @Since("1.2.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Trait for components that take parameters. This also provides an internal param map to store
  * parameter values attached to the instance.
  */
-@DeveloperApi
 trait Params extends Identifiable with Serializable {
 
   /**
@@ -915,13 +891,11 @@ private[ml] object Params {
 }
 
 /**
- * :: DeveloperApi ::
  * Java-friendly wrapper for [[Params]].
  * Java developers who need to extend [[Params]] should use this class instead.
  * If you need to extend an abstract class which already extends [[Params]], then that abstract
  * class should be Java-friendly as well.
  */
-@DeveloperApi
 abstract class JavaParams extends Params
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
index 6194dfacf01fc..7fd5f5938b565 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
@@ -207,7 +207,6 @@ private[shared] object SharedParamsCodeGen {
       | * Trait for shared param $name$defaultValueDoc. This trait may be changed or
       | * removed between minor versions.
       | */
-      |@DeveloperApi
       |trait Has$Name extends Params {
       |
       |  /**
@@ -244,7 +243,6 @@ private[shared] object SharedParamsCodeGen {
         |
         |package org.apache.spark.ml.param.shared
         |
-        |import org.apache.spark.annotation.DeveloperApi
         |import org.apache.spark.ml.param._
         |
         |// DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
index 0c0d2b5981dbe..60203eba61ea5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.ml.param.shared
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.param._
 
 // DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen.
@@ -28,7 +27,6 @@ import org.apache.spark.ml.param._
  * Trait for shared param regParam. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasRegParam extends Params {
 
   /**
@@ -45,7 +43,6 @@ trait HasRegParam extends Params {
  * Trait for shared param maxIter. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasMaxIter extends Params {
 
   /**
@@ -62,7 +59,6 @@ trait HasMaxIter extends Params {
  * Trait for shared param featuresCol (default: "features"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasFeaturesCol extends Params {
 
   /**
@@ -81,7 +77,6 @@ trait HasFeaturesCol extends Params {
  * Trait for shared param labelCol (default: "label"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasLabelCol extends Params {
 
   /**
@@ -100,7 +95,6 @@ trait HasLabelCol extends Params {
  * Trait for shared param predictionCol (default: "prediction"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasPredictionCol extends Params {
 
   /**
@@ -119,7 +113,6 @@ trait HasPredictionCol extends Params {
  * Trait for shared param rawPredictionCol (default: "rawPrediction"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasRawPredictionCol extends Params {
 
   /**
@@ -138,7 +131,6 @@ trait HasRawPredictionCol extends Params {
  * Trait for shared param probabilityCol (default: "probability"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasProbabilityCol extends Params {
 
   /**
@@ -157,7 +149,6 @@ trait HasProbabilityCol extends Params {
  * Trait for shared param varianceCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasVarianceCol extends Params {
 
   /**
@@ -174,7 +165,6 @@ trait HasVarianceCol extends Params {
  * Trait for shared param threshold. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasThreshold extends Params {
 
   /**
@@ -191,7 +181,6 @@ trait HasThreshold extends Params {
  * Trait for shared param thresholds. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasThresholds extends Params {
 
   /**
@@ -208,7 +197,6 @@ trait HasThresholds extends Params {
  * Trait for shared param inputCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasInputCol extends Params {
 
   /**
@@ -225,7 +213,6 @@ trait HasInputCol extends Params {
  * Trait for shared param inputCols. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasInputCols extends Params {
 
   /**
@@ -242,7 +229,6 @@ trait HasInputCols extends Params {
  * Trait for shared param outputCol (default: uid + "__output"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasOutputCol extends Params {
 
   /**
@@ -261,7 +247,6 @@ trait HasOutputCol extends Params {
  * Trait for shared param outputCols. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasOutputCols extends Params {
 
   /**
@@ -278,7 +263,6 @@ trait HasOutputCols extends Params {
  * Trait for shared param numFeatures (default: 262144). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasNumFeatures extends Params {
 
   /**
@@ -297,7 +281,6 @@ trait HasNumFeatures extends Params {
  * Trait for shared param checkpointInterval. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasCheckpointInterval extends Params {
 
   /**
@@ -314,7 +297,6 @@ trait HasCheckpointInterval extends Params {
  * Trait for shared param fitIntercept (default: true). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasFitIntercept extends Params {
 
   /**
@@ -333,7 +315,6 @@ trait HasFitIntercept extends Params {
  * Trait for shared param handleInvalid. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasHandleInvalid extends Params {
 
   /**
@@ -350,7 +331,6 @@ trait HasHandleInvalid extends Params {
  * Trait for shared param standardization (default: true). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasStandardization extends Params {
 
   /**
@@ -369,7 +349,6 @@ trait HasStandardization extends Params {
  * Trait for shared param seed (default: this.getClass.getName.hashCode.toLong). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasSeed extends Params {
 
   /**
@@ -388,7 +367,6 @@ trait HasSeed extends Params {
  * Trait for shared param elasticNetParam. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasElasticNetParam extends Params {
 
   /**
@@ -405,7 +383,6 @@ trait HasElasticNetParam extends Params {
  * Trait for shared param tol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasTol extends Params {
 
   /**
@@ -422,7 +399,6 @@ trait HasTol extends Params {
  * Trait for shared param relativeError (default: 0.001). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasRelativeError extends Params {
 
   /**
@@ -441,7 +417,6 @@ trait HasRelativeError extends Params {
  * Trait for shared param stepSize. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasStepSize extends Params {
 
   /**
@@ -458,7 +433,6 @@ trait HasStepSize extends Params {
  * Trait for shared param weightCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasWeightCol extends Params {
 
   /**
@@ -475,7 +449,6 @@ trait HasWeightCol extends Params {
  * Trait for shared param solver. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasSolver extends Params {
 
   /**
@@ -492,7 +465,6 @@ trait HasSolver extends Params {
  * Trait for shared param aggregationDepth (default: 2). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasAggregationDepth extends Params {
 
   /**
@@ -511,7 +483,6 @@ trait HasAggregationDepth extends Params {
  * Trait for shared param collectSubModels (default: false). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasCollectSubModels extends Params {
 
   /**
@@ -530,7 +501,6 @@ trait HasCollectSubModels extends Params {
  * Trait for shared param loss. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasLoss extends Params {
 
   /**
@@ -547,7 +517,6 @@ trait HasLoss extends Params {
  * Trait for shared param distanceMeasure (default: "euclidean"). This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasDistanceMeasure extends Params {
 
   /**
@@ -566,7 +535,6 @@ trait HasDistanceMeasure extends Params {
  * Trait for shared param validationIndicatorCol. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasValidationIndicatorCol extends Params {
 
   /**
@@ -583,7 +551,6 @@ trait HasValidationIndicatorCol extends Params {
  * Trait for shared param blockSize. This trait may be changed or
  * removed between minor versions.
  */
-@DeveloperApi
 trait HasBlockSize extends Params {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 002146f89e79a..0fa17c188209a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -32,7 +32,7 @@ import org.json4s.DefaultFormats
 import org.json4s.JsonDSL._
 
 import org.apache.spark.{Dependency, Partitioner, ShuffleDependency, SparkContext, SparkException}
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.linalg.BLAS
@@ -731,21 +731,17 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel]
 
 
 /**
- * :: DeveloperApi ::
  * An implementation of ALS that supports generic ID types, specialized for Int and Long. This is
  * exposed as a developer API for users who do need other ID types. But it is not recommended
  * because it increases the shuffle size and memory requirement during training. For simplicity,
  * users and items must have the same type. The number of distinct users/items should be smaller
  * than 2 billion.
  */
-@DeveloperApi
 object ALS extends DefaultParamsReadable[ALS] with Logging {
 
   /**
-   * :: DeveloperApi ::
    * Rating class for better code readability.
    */
-  @DeveloperApi
   case class Rating[@specialized(Int, Long) ID](user: ID, item: ID, rating: Float)
 
   @Since("1.6.0")
@@ -908,7 +904,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
   }
 
   /**
-   * :: DeveloperApi ::
    * Implementation of the ALS algorithm.
    *
    * This implementation of the ALS factorization algorithm partitions the two sets of factors among
@@ -933,7 +928,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging {
    * "block" as referring to a subset of an RDD containing the ratings rather than a contiguous
    * submatrix of the ratings matrix.
    */
-  @DeveloperApi
   def train[ID: ClassTag]( // scalastyle:ignore
       ratings: RDD[Rating[ID]],
       rank: Int = 10,
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala
index 314cf422be87e..c28dac6850e85 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/Regressor.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.ml.regression
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
 
 
@@ -28,7 +27,6 @@ import org.apache.spark.ml.{PredictionModel, Predictor, PredictorParams}
  * @tparam Learner  Concrete Estimator type
  * @tparam M  Concrete Model type
  */
-@DeveloperApi
 abstract class Regressor[
     FeaturesType,
     Learner <: Regressor[FeaturesType, Learner, M],
@@ -39,14 +37,11 @@ abstract class Regressor[
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Model produced by a `Regressor`.
  *
  * @tparam FeaturesType  Type of input features.  E.g., [[org.apache.spark.mllib.linalg.Vector]]
  * @tparam M  Concrete Model type.
  */
-@DeveloperApi
 abstract class RegressionModel[FeaturesType, M <: RegressionModel[FeaturesType, M]]
   extends PredictionModel[FeaturesType, M] with PredictorParams {
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
index a273cd71a0beb..7e2c287f146fb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
@@ -36,7 +36,7 @@ import org.apache.spark.sql.types.{DataType, DoubleType, StructType}
 /**
  * Parameters for Decision Tree-based algorithms.
  *
- * Note: Marked as private and DeveloperApi since this may be made public in the future.
+ * Note: Marked as private since this may be made public in the future.
  */
 private[ml] trait DecisionTreeParams extends PredictorParams
   with HasCheckpointInterval with HasSeed with HasWeightCol {
@@ -320,7 +320,7 @@ private[spark] object TreeEnsembleParams {
 /**
  * Parameters for Decision Tree-based ensemble algorithms.
  *
- * Note: Marked as private and DeveloperApi since this may be made public in the future.
+ * Note: Marked as private since this may be made public in the future.
  */
 private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
 
@@ -477,7 +477,7 @@ private[ml] trait RandomForestRegressorParams
 /**
  * Parameters for Gradient-Boosted Tree algorithms.
  *
- * Note: Marked as private and DeveloperApi since this may be made public in the future.
+ * Note: Marked as private since this may be made public in the future.
  */
 private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter with HasStepSize
   with HasValidationIndicatorCol {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala
index bd213e7362e94..653ffb765f610 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/Identifiable.scala
@@ -19,18 +19,13 @@ package org.apache.spark.ml.util
 
 import java.util.UUID
 
-import org.apache.spark.annotation.DeveloperApi
-
 
 /**
- * :: DeveloperApi ::
- *
  * Trait for an object with an immutable unique ID that identifies itself and its derivatives.
  *
  * WARNING: There have not yet been final discussions on this API, so it may be broken in future
  *          releases.
  */
-@DeveloperApi
 trait Identifiable {
 
   /**
@@ -41,10 +36,7 @@ trait Identifiable {
   override def toString: String = uid
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
+
 object Identifiable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
index dd0c40f33ff2a..fec05ccf15c7c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
@@ -31,7 +31,7 @@ import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.{SparkContext, SparkException}
-import org.apache.spark.annotation.{DeveloperApi, Since, Unstable}
+import org.apache.spark.annotation.{Since, Unstable}
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml._
 import org.apache.spark.ml.classification.{OneVsRest, OneVsRestModel}
@@ -301,8 +301,6 @@ trait GeneralMLWritable extends MLWritable {
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Helper trait for making simple `Params` types writable.  If a `Params` class stores
  * all data as [[org.apache.spark.ml.param.Param]] values, then extending this trait will provide
  * a default implementation of writing saved instances of the class.
@@ -311,7 +309,6 @@ trait GeneralMLWritable extends MLWritable {
  *
  * @see `DefaultParamsReadable`, the counterpart to this trait
  */
-@DeveloperApi
 trait DefaultParamsWritable extends MLWritable { self: Params =>
 
   override def write: MLWriter = new DefaultParamsWriter(this)
@@ -360,8 +357,6 @@ trait MLReadable[T] {
 
 
 /**
- * :: DeveloperApi ::
- *
  * Helper trait for making simple `Params` types readable.  If a `Params` class stores
  * all data as [[org.apache.spark.ml.param.Param]] values, then extending this trait will provide
  * a default implementation of reading saved instances of the class.
@@ -371,7 +366,6 @@ trait MLReadable[T] {
  * @tparam T ML instance type
  * @see `DefaultParamsWritable`, the counterpart to this trait
  */
-@DeveloperApi
 trait DefaultParamsReadable[T] extends MLReadable[T] {
 
   override def read: MLReader[T] = new DefaultParamsReader[T]
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
index e858ac9844a24..8c43c0b468526 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDA.scala
@@ -21,7 +21,7 @@ import java.util.Locale
 
 import breeze.linalg.{DenseVector => BDV}
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaPairRDD
 import org.apache.spark.graphx._
 import org.apache.spark.internal.Logging
@@ -281,21 +281,15 @@ class LDA private (
 
 
   /**
-   * :: DeveloperApi ::
-   *
    * LDAOptimizer used to perform the actual calculation
    */
   @Since("1.4.0")
-  @DeveloperApi
   def getOptimizer: LDAOptimizer = ldaOptimizer
 
   /**
-   * :: DeveloperApi ::
-   *
    * LDAOptimizer used to perform the actual calculation (default = EMLDAOptimizer)
    */
   @Since("1.4.0")
-  @DeveloperApi
   def setOptimizer(optimizer: LDAOptimizer): this.type = {
     this.ldaOptimizer = optimizer
     this
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
index dc90f6c3e3885..b2742ee6ecb5b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala
@@ -23,7 +23,7 @@ import breeze.linalg.{all, normalize, sum, DenseMatrix => BDM, DenseVector => BD
 import breeze.numerics.{abs, exp, trigamma}
 import breeze.stats.distributions.{Gamma, RandBasis}
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.graphx._
 import org.apache.spark.graphx.util.PeriodicGraphCheckpointer
 import org.apache.spark.internal.Logging
@@ -33,13 +33,10 @@ import org.apache.spark.storage.StorageLevel
 
 
 /**
- * :: DeveloperApi ::
- *
  * An LDAOptimizer specifies which optimization/learning/inference algorithm to use, and it can
  * hold optimizer-specific parameters for users to set.
  */
 @Since("1.4.0")
-@DeveloperApi
 trait LDAOptimizer {
 
   /*
@@ -62,8 +59,6 @@ trait LDAOptimizer {
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Optimizer for EM algorithm which stores data + parameter graph, plus algorithm parameters.
  *
  * Currently, the underlying implementation uses Expectation-Maximization (EM), implemented
@@ -78,7 +73,6 @@ trait LDAOptimizer {
  *    "On Smoothing and Inference for Topic Models."  UAI, 2009.
  */
 @Since("1.4.0")
-@DeveloperApi
 final class EMLDAOptimizer extends LDAOptimizer {
 
   import LDA._
@@ -253,8 +247,6 @@ final class EMLDAOptimizer extends LDAOptimizer {
 
 
 /**
- * :: DeveloperApi ::
- *
  * An online optimizer for LDA. The Optimizer implements the Online variational Bayes LDA
  * algorithm, which processes a subset of the corpus on each iteration, and updates the term-topic
  * distribution adaptively.
@@ -263,7 +255,6 @@ final class EMLDAOptimizer extends LDAOptimizer {
  *   Hoffman, Blei and Bach, "Online Learning for Latent Dirichlet Allocation." NIPS, 2010.
  */
 @Since("1.4.0")
-@DeveloperApi
 final class OnlineLDAOptimizer extends LDAOptimizer with Logging {
 
   // LDA common parameters
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
index 21e01ef36814f..78c974e22f2cf 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.feature
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.feature.{StandardScalerModel => NewStandardScalerModel}
 import org.apache.spark.mllib.linalg.{DenseVector, SparseVector, Vector, Vectors}
@@ -96,22 +96,14 @@ class StandardScalerModel @Since("1.3.0") (
   @Since("1.3.0")
   def this(std: Vector) = this(std, null)
 
-  /**
-   * :: DeveloperApi ::
-   */
   @Since("1.3.0")
-  @DeveloperApi
   def setWithMean(withMean: Boolean): this.type = {
     require(!(withMean && this.mean == null), "cannot set withMean to true while mean is null")
     this.withMean = withMean
     this
   }
 
-  /**
-   * :: DeveloperApi ::
-   */
   @Since("1.3.0")
-  @DeveloperApi
   def setWithStd(withStd: Boolean): this.type = {
     require(!(withStd && this.std == null),
       "cannot set withStd to true while std is null")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala
index 9db725097ae90..ac58ffe3e5509 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/VectorTransformer.scala
@@ -17,17 +17,15 @@
 
 package org.apache.spark.mllib.feature
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Trait for transformation of a vector
  */
 @Since("1.1.0")
-@DeveloperApi
 trait VectorTransformer extends Serializable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala
index 46a18295d839b..cdeed33c92381 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Gradient.scala
@@ -17,16 +17,13 @@
 
 package org.apache.spark.mllib.optimization
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.mllib.linalg.{DenseVector, Vector, Vectors}
 import org.apache.spark.mllib.linalg.BLAS.{axpy, dot, scal}
 import org.apache.spark.mllib.util.MLUtils
 
 /**
- * :: DeveloperApi ::
  * Class used to compute the gradient for a loss function, given a single data point.
  */
-@DeveloperApi
 abstract class Gradient extends Serializable {
   /**
    * Compute the gradient and loss given the features of a single data point.
@@ -58,7 +55,6 @@ abstract class Gradient extends Serializable {
 }
 
 /**
- * :: DeveloperApi ::
  * Compute gradient and loss for a multinomial logistic loss function, as used
  * in multi-class classification (it is also used in binary logistic regression).
  *
@@ -162,7 +158,6 @@ abstract class Gradient extends Serializable {
  *                   Multinomial Logistic Regression. By default, it is binary logistic regression
  *                   so numClasses will be set to 2.
  */
-@DeveloperApi
 class LogisticGradient(numClasses: Int) extends Gradient {
 
   def this() = this(2)
@@ -275,13 +270,11 @@ class LogisticGradient(numClasses: Int) extends Gradient {
 }
 
 /**
- * :: DeveloperApi ::
  * Compute gradient and loss for a Least-squared loss function, as used in linear regression.
  * This is correct for the averaged least squares loss function (mean squared error)
  *              L = 1/2n ||A weights-y||^2
  * See also the documentation for the precise formulation.
  */
-@DeveloperApi
 class LeastSquaresGradient extends Gradient {
   override def compute(data: Vector, label: Double, weights: Vector): (Vector, Double) = {
     val diff = dot(data, weights) - label
@@ -303,13 +296,11 @@ class LeastSquaresGradient extends Gradient {
 }
 
 /**
- * :: DeveloperApi ::
  * Compute gradient and loss for a Hinge loss function, as used in SVM binary classification.
  * See also the documentation for the precise formulation.
  *
  * @note This assumes that the labels are {0,1}
  */
-@DeveloperApi
 class HingeGradient extends Gradient {
   override def compute(data: Vector, label: Double, weights: Vector): (Vector, Double) = {
     val dotProduct = dot(data, weights)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
index ffe3964981a18..d9031661fbc85 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/GradientDescent.scala
@@ -21,7 +21,6 @@ import scala.collection.mutable.ArrayBuffer
 
 import breeze.linalg.{norm, DenseVector => BDV}
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 import org.apache.spark.rdd.RDD
@@ -124,13 +123,11 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
   }
 
   /**
-   * :: DeveloperApi ::
    * Runs gradient descent on the given training data.
    * @param data training data
    * @param initialWeights initial weights
    * @return solution vector
    */
-  @DeveloperApi
   def optimize(data: RDD[(Double, Vector)], initialWeights: Vector): Vector = {
     val (weights, _) = GradientDescent.runMiniBatchSGD(
       data,
@@ -148,10 +145,8 @@ class GradientDescent private[spark] (private var gradient: Gradient, private va
 }
 
 /**
- * :: DeveloperApi ::
  * Top-level method to run gradient descent.
  */
-@DeveloperApi
 object GradientDescent extends Logging {
   /**
    * Run stochastic gradient descent (SGD) in parallel using mini batches.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index 65c1159723c80..1ee9241104f87 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -22,21 +22,18 @@ import scala.collection.mutable
 import breeze.linalg.{DenseVector => BDV}
 import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS}
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 import org.apache.spark.mllib.linalg.BLAS.axpy
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Class used to solve an optimization problem using Limited-memory BFGS.
  * Reference: <a href="http://en.wikipedia.org/wiki/Limited-memory_BFGS">
  * Wikipedia on Limited-memory BFGS</a>
  * @param gradient Gradient function to be used.
  * @param updater Updater to be used to update weights after every iteration.
  */
-@DeveloperApi
 class LBFGS(private var gradient: Gradient, private var updater: Updater)
   extends Optimizer with Logging {
 
@@ -154,10 +151,8 @@ class LBFGS(private var gradient: Gradient, private var updater: Updater)
 }
 
 /**
- * :: DeveloperApi ::
  * Top-level method to run L-BFGS.
  */
-@DeveloperApi
 object LBFGS extends Logging {
   /**
    * Run Limited-memory BFGS (L-BFGS) in parallel.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala
index d8e56720967d8..582b520e08eee 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Optimizer.scala
@@ -17,15 +17,12 @@
 
 package org.apache.spark.mllib.optimization
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Trait for optimization problem solvers.
  */
-@DeveloperApi
 trait Optimizer extends Serializable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
index 142f0ec6b9021..1865dd96ce72d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/Updater.scala
@@ -21,11 +21,9 @@ import scala.math._
 
 import breeze.linalg.{axpy => brzAxpy, norm => brzNorm, Vector => BV}
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 
 /**
- * :: DeveloperApi ::
  * Class used to perform steps (weight update) using Gradient Descent methods.
  *
  * For general minimization problems, or for regularized problems of the form
@@ -37,7 +35,6 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors}
  * The updater is responsible to also perform the update coming from the
  * regularization term R(w) (if any regularization is used).
  */
-@DeveloperApi
 abstract class Updater extends Serializable {
   /**
    * Compute an updated value for weights given the gradient, stepSize, iteration number and
@@ -62,11 +59,9 @@ abstract class Updater extends Serializable {
 }
 
 /**
- * :: DeveloperApi ::
  * A simple updater for gradient descent *without* any regularization.
  * Uses a step-size decreasing with the square root of the number of iterations.
  */
-@DeveloperApi
 class SimpleUpdater extends Updater {
   override def compute(
       weightsOld: Vector,
@@ -83,7 +78,6 @@ class SimpleUpdater extends Updater {
 }
 
 /**
- * :: DeveloperApi ::
  * Updater for L1 regularized problems.
  *          R(w) = ||w||_1
  * Uses a step-size decreasing with the square root of the number of iterations.
@@ -101,7 +95,6 @@ class SimpleUpdater extends Updater {
  *
  * Equivalently, set weight component to signum(w) * max(0.0, abs(w) - shrinkageVal)
  */
-@DeveloperApi
 class L1Updater extends Updater {
   override def compute(
       weightsOld: Vector,
@@ -128,12 +121,10 @@ class L1Updater extends Updater {
 }
 
 /**
- * :: DeveloperApi ::
  * Updater for L2 regularized problems.
  *          R(w) = 1/2 ||w||^2
  * Uses a step-size decreasing with the square root of the number of iterations.
  */
-@DeveloperApi
 class SquaredL2Updater extends Updater {
   override def compute(
       weightsOld: Vector,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala b/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala
index 340386c7e7bec..32471b0a64d9e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/pmml/PMMLExportable.scala
@@ -23,16 +23,14 @@ import javax.xml.transform.stream.StreamResult
 import org.jpmml.model.JAXBUtil
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.pmml.export.PMMLModelExportFactory
 
 /**
- * :: DeveloperApi ::
  * Export model to the PMML format
  * Predictive Model Markup Language (PMML) is an XML-based file format
  * developed by the Data Mining Group (www.dmg.org).
  */
-@DeveloperApi
 @Since("1.4.0")
 trait PMMLExportable {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala
index d3b548832bb21..3a2ecf17d88d7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomDataGenerator.scala
@@ -19,14 +19,12 @@ package org.apache.spark.mllib.random
 
 import org.apache.commons.math3.distribution._
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.util.random.{Pseudorandom, XORShiftRandom}
 
 /**
- * :: DeveloperApi ::
  * Trait for random data generators that generate i.i.d. data.
  */
-@DeveloperApi
 @Since("1.1.0")
 trait RandomDataGenerator[T] extends Pseudorandom with Serializable {
 
@@ -45,10 +43,8 @@ trait RandomDataGenerator[T] extends Pseudorandom with Serializable {
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from U[0.0, 1.0]
  */
-@DeveloperApi
 @Since("1.1.0")
 class UniformGenerator extends RandomDataGenerator[Double] {
 
@@ -68,10 +64,8 @@ class UniformGenerator extends RandomDataGenerator[Double] {
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the standard normal distribution.
  */
-@DeveloperApi
 @Since("1.1.0")
 class StandardNormalGenerator extends RandomDataGenerator[Double] {
 
@@ -91,12 +85,10 @@ class StandardNormalGenerator extends RandomDataGenerator[Double] {
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the Poisson distribution with the given mean.
  *
  * @param mean mean for the Poisson distribution.
  */
-@DeveloperApi
 @Since("1.1.0")
 class PoissonGenerator @Since("1.1.0") (
     @Since("1.1.0") val mean: Double) extends RandomDataGenerator[Double] {
@@ -116,12 +108,10 @@ class PoissonGenerator @Since("1.1.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the exponential distribution with the given mean.
  *
  * @param mean mean for the exponential distribution.
  */
-@DeveloperApi
 @Since("1.3.0")
 class ExponentialGenerator @Since("1.3.0") (
     @Since("1.3.0") val mean: Double) extends RandomDataGenerator[Double] {
@@ -141,13 +131,11 @@ class ExponentialGenerator @Since("1.3.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the gamma distribution with the given shape and scale.
  *
  * @param shape shape for the gamma distribution.
  * @param scale scale for the gamma distribution
  */
-@DeveloperApi
 @Since("1.3.0")
 class GammaGenerator @Since("1.3.0") (
     @Since("1.3.0") val shape: Double,
@@ -168,14 +156,12 @@ class GammaGenerator @Since("1.3.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the log normal distribution with the
  * given mean and standard deviation.
  *
  * @param mean mean for the log normal distribution.
  * @param std standard deviation for the log normal distribution
  */
-@DeveloperApi
 @Since("1.3.0")
 class LogNormalGenerator @Since("1.3.0") (
     @Since("1.3.0") val mean: Double,
@@ -196,14 +182,12 @@ class LogNormalGenerator @Since("1.3.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * Generates i.i.d. samples from the Weibull distribution with the
  * given shape and scale parameter.
  *
  * @param alpha shape parameter for the Weibull distribution.
  * @param beta scale parameter for the Weibull distribution.
  */
-@DeveloperApi
 class WeibullGenerator(
     val alpha: Double,
     val beta: Double) extends RandomDataGenerator[Double] {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
index 258b1763bba86..eacccfcc4d59f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
@@ -20,7 +20,7 @@ package org.apache.spark.mllib.random
 import scala.reflect.ClassTag
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.{JavaDoubleRDD, JavaRDD, JavaSparkContext}
 import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
 import org.apache.spark.mllib.linalg.Vector
@@ -372,7 +372,6 @@ object RandomRDDs {
 
 
   /**
-   * :: DeveloperApi ::
    * Generates an RDD comprised of `i.i.d.` samples produced by the input RandomDataGenerator.
    *
    * @param sc SparkContext used to create the RDD.
@@ -382,7 +381,6 @@ object RandomRDDs {
    * @param seed Random seed (default: a random long integer).
    * @return RDD[T] comprised of `i.i.d.` samples produced by generator.
    */
-  @DeveloperApi
   @Since("1.1.0")
   def randomRDD[T: ClassTag](
       sc: SparkContext,
@@ -394,7 +392,6 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * Generates an RDD comprised of `i.i.d.` samples produced by the input RandomDataGenerator.
    *
    * @param jsc JavaSparkContext used to create the RDD.
@@ -404,7 +401,6 @@ object RandomRDDs {
    * @param seed Random seed (default: a random long integer).
    * @return RDD[T] comprised of `i.i.d.` samples produced by generator.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaRDD[T](
       jsc: JavaSparkContext,
@@ -418,10 +414,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * `RandomRDDs.randomJavaRDD` with the default seed.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaRDD[T](
     jsc: JavaSparkContext,
@@ -432,10 +426,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * `RandomRDDs.randomJavaRDD` with the default seed & numPartitions
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaRDD[T](
       jsc: JavaSparkContext,
@@ -831,7 +823,6 @@ object RandomRDDs {
 
 
   /**
-   * :: DeveloperApi ::
    * Generates an RDD[Vector] with vectors containing `i.i.d.` samples produced by the
    * input RandomDataGenerator.
    *
@@ -843,7 +834,6 @@ object RandomRDDs {
    * @param seed Random seed (default: a random long integer).
    * @return RDD[Vector] with vectors containing `i.i.d.` samples produced by generator.
    */
-  @DeveloperApi
   @Since("1.1.0")
   def randomVectorRDD(sc: SparkContext,
       generator: RandomDataGenerator[Double],
@@ -856,10 +846,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * Java-friendly version of `RandomRDDs.randomVectorRDD`.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaVectorRDD(
       jsc: JavaSparkContext,
@@ -871,11 +859,9 @@ object RandomRDDs {
     randomVectorRDD(jsc.sc, generator, numRows, numCols, numPartitions, seed).toJavaRDD()
   }
 
-  /**
-   * :: DeveloperApi ::
+  /** ::
    * `RandomRDDs.randomJavaVectorRDD` with the default seed.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaVectorRDD(
       jsc: JavaSparkContext,
@@ -887,10 +873,8 @@ object RandomRDDs {
   }
 
   /**
-   * :: DeveloperApi ::
    * `RandomRDDs.randomJavaVectorRDD` with the default number of partitions and the default seed.
    */
-  @DeveloperApi
   @Since("1.6.0")
   def randomJavaVectorRDD(
       jsc: JavaSparkContext,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala
index e28e1af5b0a26..8f78bcc15347f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/MLPairRDDFunctions.scala
@@ -20,15 +20,12 @@ package org.apache.spark.mllib.rdd
 import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.rdd.RDD
 import org.apache.spark.util.BoundedPriorityQueue
 
 /**
- * :: DeveloperApi ::
  * Machine learning specific Pair RDD functions.
  */
-@DeveloperApi
 class MLPairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) extends Serializable {
   /**
    * Returns the top k (largest) elements for each key from this RDD as defined by the specified
@@ -51,10 +48,6 @@ class MLPairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) extends Se
   }
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
 object MLPairRDDFunctions {
   /** Implicit conversion from a pair RDD to MLPairRDDFunctions. */
   implicit def fromPairRDD[K: ClassTag, V: ClassTag](rdd: RDD[(K, V)]): MLPairRDDFunctions[K, V] =
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala b/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala
index 32e6ecf6308e0..1a13cbb5191c6 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/rdd/RDDFunctions.scala
@@ -20,14 +20,11 @@ package org.apache.spark.mllib.rdd
 import scala.language.implicitConversions
 import scala.reflect.ClassTag
 
-import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Machine learning specific RDD functions.
  */
-@DeveloperApi
 class RDDFunctions[T: ClassTag](self: RDD[T]) extends Serializable {
 
   /**
@@ -54,10 +51,7 @@ class RDDFunctions[T: ClassTag](self: RDD[T]) extends Serializable {
 
 }
 
-/**
- * :: DeveloperApi ::
- */
-@DeveloperApi
+
 object RDDFunctions {
 
   /** Implicit conversion from an RDD to RDDFunctions. */
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
index f3f15ba0d0f2c..1ae0b580a3716 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.recommendation
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.recommendation.{ALS => NewALS}
@@ -195,12 +195,10 @@ class ALS private (
   }
 
   /**
-   * :: DeveloperApi ::
    * Sets storage level for intermediate RDDs (user/product in/out links). The default value is
    * `MEMORY_AND_DISK`. Users can change it to a serialized storage, e.g., `MEMORY_AND_DISK_SER` and
    * set `spark.rdd.compress` to `true` to reduce the space requirement, at the cost of speed.
    */
-  @DeveloperApi
   @Since("1.1.0")
   def setIntermediateRDDStorageLevel(storageLevel: StorageLevel): this.type = {
     require(storageLevel != StorageLevel.NONE,
@@ -210,13 +208,11 @@ class ALS private (
   }
 
   /**
-   * :: DeveloperApi ::
    * Sets storage level for final RDDs (user/product used in MatrixFactorizationModel). The default
    * value is `MEMORY_AND_DISK`. Users can change it to a serialized storage, e.g.
    * `MEMORY_AND_DISK_SER` and set `spark.rdd.compress` to `true` to reduce the space requirement,
    * at the cost of speed.
    */
-  @DeveloperApi
   @Since("1.3.0")
   def setFinalRDDStorageLevel(storageLevel: StorageLevel): this.type = {
     this.finalRDDStorageLevel = storageLevel
@@ -224,14 +220,12 @@ class ALS private (
   }
 
   /**
-   * :: DeveloperApi ::
    * Set period (in iterations) between checkpoints (default = 10). Checkpointing helps with
    * recovery (when nodes fail) and StackOverflow exceptions caused by long lineage. It also helps
    * with eliminating temporary shuffle files on disk, which can be important when there are many
    * ALS iterations. If the checkpoint directory is not set in [[org.apache.spark.SparkContext]],
    * this setting is ignored.
    */
-  @DeveloperApi
   @Since("1.4.0")
   def setCheckpointInterval(checkpointInterval: Int): this.type = {
     this.checkpointInterval = checkpointInterval
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala
index ef3ff2b98a5e6..6d0b3fa791d6e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/GeneralizedLinearAlgorithm.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.mllib.regression
 
 import org.apache.spark.SparkException
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.feature.StandardScaler
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
@@ -28,7 +28,6 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.storage.StorageLevel
 
 /**
- * :: DeveloperApi ::
  * GeneralizedLinearModel (GLM) represents a model trained using
  * GeneralizedLinearAlgorithm. GLMs consist of a weight vector and
  * an intercept.
@@ -38,7 +37,6 @@ import org.apache.spark.storage.StorageLevel
  *
  */
 @Since("0.8.0")
-@DeveloperApi
 abstract class GeneralizedLinearModel @Since("1.0.0") (
     @Since("1.0.0") val weights: Vector,
     @Since("0.8.0") val intercept: Double)
@@ -94,13 +92,11 @@ abstract class GeneralizedLinearModel @Since("1.0.0") (
 }
 
 /**
- * :: DeveloperApi ::
  * GeneralizedLinearAlgorithm implements methods to train a Generalized Linear Model (GLM).
  * This class should be extended with an Optimizer to create a new GLM.
  *
  */
 @Since("0.8.0")
-@DeveloperApi
 abstract class GeneralizedLinearAlgorithm[M <: GeneralizedLinearModel]
   extends Logging with Serializable {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
index f44c8fe351459..eb83f9a6b75af 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/StreamingLinearAlgorithm.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.regression
 
 import scala.reflect.ClassTag
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.Vector
@@ -27,7 +27,6 @@ import org.apache.spark.streaming.api.java.{JavaDStream, JavaPairDStream}
 import org.apache.spark.streaming.dstream.DStream
 
 /**
- * :: DeveloperApi ::
  * StreamingLinearAlgorithm implements methods for continuously
  * training a generalized linear model on streaming data,
  * and using it for prediction on (possibly different) streaming data.
@@ -56,7 +55,6 @@ import org.apache.spark.streaming.dstream.DStream
  *
  */
 @Since("1.1.0")
-@DeveloperApi
 abstract class StreamingLinearAlgorithm[
     M <: GeneralizedLinearModel,
     A <: GeneralizedLinearAlgorithm[M]] extends Logging {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
index 00c1da66d7df6..829cce6eac73e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/MultivariateOnlineSummarizer.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.stat
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{Vector, Vectors}
 
 /**
- * :: DeveloperApi ::
  * MultivariateOnlineSummarizer implements [[MultivariateStatisticalSummary]] to compute the mean,
  * variance, minimum, maximum, counts, and nonzero counts for instances in sparse or dense vector
  * format in an online fashion.
@@ -41,7 +40,6 @@ import org.apache.spark.mllib.linalg.{Vector, Vectors}
  * Reliability weights (Wikipedia)</a>.
  */
 @Since("1.1.0")
-@DeveloperApi
 class MultivariateOnlineSummarizer extends MultivariateStatisticalSummary with Serializable {
 
   private var n = 0
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala
index 9a746dcf35556..0b6341b227012 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/distribution/MultivariateGaussian.scala
@@ -19,12 +19,11 @@ package org.apache.spark.mllib.stat.distribution
 
 import breeze.linalg.{diag, eigSym, max, DenseMatrix => DBM, DenseVector => DBV, Vector => BV}
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{Matrices, Matrix, Vector, Vectors}
 import org.apache.spark.mllib.util.MLUtils
 
 /**
- * :: DeveloperApi ::
  * This class provides basic functionality for a Multivariate Gaussian (Normal) Distribution. In
  * the event that the covariance matrix is singular, the density will be computed in a
  * reduced dimensional subspace under which the distribution is supported.
@@ -35,7 +34,6 @@ import org.apache.spark.mllib.util.MLUtils
  * @param sigma The covariance matrix of the distribution
  */
 @Since("1.3.0")
-@DeveloperApi
 class MultivariateGaussian @Since("1.3.0") (
     @Since("1.3.0") val mu: Vector,
     @Since("1.3.0") val sigma: Matrix) extends Serializable {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
index f01a98e74886b..f62d3810ed3ed 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Entropy.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.tree.impurity
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Class for calculating entropy during multiclass classification.
@@ -28,14 +28,12 @@ object Entropy extends Impurity {
   private[tree] def log2(x: Double) = scala.math.log(x) / scala.math.log(2)
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   override def calculate(counts: Array[Double], totalCount: Double): Double = {
     if (totalCount == 0) {
       return 0
@@ -55,7 +53,6 @@ object Entropy extends Impurity {
   }
 
   /**
-   * :: DeveloperApi ::
    * variance calculation
    * @param count number of instances
    * @param sum sum of labels
@@ -63,7 +60,6 @@ object Entropy extends Impurity {
    * @return information value, or 0 if count = 0
    */
   @Since("1.0.0")
-  @DeveloperApi
   override def calculate(count: Double, sum: Double, sumSquares: Double): Double =
     throw new UnsupportedOperationException("Entropy.calculate")
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
index 913ffbbb2457a..5983118c05754 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Gini.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.tree.impurity
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Class for calculating the Gini impurity
@@ -28,14 +28,12 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
 object Gini extends Impurity {
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   override def calculate(counts: Array[Double], totalCount: Double): Double = {
     if (totalCount == 0) {
       return 0
@@ -52,7 +50,6 @@ object Gini extends Impurity {
   }
 
   /**
-   * :: DeveloperApi ::
    * variance calculation
    * @param count number of instances
    * @param sum sum of labels
@@ -60,7 +57,6 @@ object Gini extends Impurity {
    * @return information value, or 0 if count = 0
    */
   @Since("1.0.0")
-  @DeveloperApi
   override def calculate(count: Double, sum: Double, sumSquares: Double): Double =
     throw new UnsupportedOperationException("Gini.calculate")
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala
index 491473490eba7..9b1d8d717d40f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Impurity.scala
@@ -19,7 +19,7 @@ package org.apache.spark.mllib.tree.impurity
 
 import java.util.Locale
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Trait for calculating information gain.
@@ -31,18 +31,15 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
 trait Impurity extends Serializable {
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   def calculate(counts: Array[Double], totalCount: Double): Double
 
   /**
-   * :: DeveloperApi ::
    * information calculation for regression
    * @param count number of instances
    * @param sum sum of labels
@@ -50,7 +47,6 @@ trait Impurity extends Serializable {
    * @return information value, or 0 if count = 0
    */
   @Since("1.0.0")
-  @DeveloperApi
   def calculate(count: Double, sum: Double, sumSquares: Double): Double
 }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
index a07b919271f71..f5b2f8d514c7e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/impurity/Variance.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.mllib.tree.impurity
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
  * Class for calculating variance during regression
@@ -26,27 +26,22 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
 object Variance extends Impurity {
 
   /**
-   * :: DeveloperApi ::
    * information calculation for multiclass classification
    * @param counts Array[Double] with counts for each label
    * @param totalCount sum of counts for all labels
    * @return information value, or 0 if totalCount = 0
    */
   @Since("1.1.0")
-  @DeveloperApi
   override def calculate(counts: Array[Double], totalCount: Double): Double =
      throw new UnsupportedOperationException("Variance.calculate")
 
   /**
-   * :: DeveloperApi ::
    * variance calculation
    * @param count number of instances
    * @param sum sum of labels
    * @param sumSquares summation of squares of the labels
    * @return information value, or 0 if count = 0
    */
-  @Since("1.0.0")
-  @DeveloperApi
   override def calculate(count: Double, sum: Double, sumSquares: Double): Double = {
     if (count == 0) {
       return 0
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala
index 9b60d018d0eda..48e576afdf675 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/AbsoluteError.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 
 /**
- * :: DeveloperApi ::
  * Class for absolute error loss calculation (for regression).
  *
  * The absolute (L1) error is defined as:
@@ -29,7 +28,6 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
  * where y is the label and F(x) is the model prediction for features x.
  */
 @Since("1.2.0")
-@DeveloperApi
 object AbsoluteError extends Loss {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala
index 9339f0a23c1bd..49f1215c77ad2 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/LogLoss.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.util.MLUtils
 
 /**
- * :: DeveloperApi ::
  * Class for log loss calculation (for classification).
  * This uses twice the binomial negative log likelihood, called "deviance" in Friedman (1999).
  *
@@ -30,7 +29,6 @@ import org.apache.spark.mllib.util.MLUtils
  * where y is a label in {-1, 1} and F(x) is the model prediction for features x.
  */
 @Since("1.2.0")
-@DeveloperApi
 object LogLoss extends ClassificationLoss {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala
index e7ffb3f8f53c1..f589e0ce5d4a1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/Loss.scala
@@ -17,17 +17,15 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.mllib.tree.model.TreeEnsembleModel
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Trait for adding "pluggable" loss functions for the gradient boosting algorithm.
  */
 @Since("1.2.0")
-@DeveloperApi
 trait Loss extends Serializable {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala
index 4eb6810c46b20..3ff59fc64ce7f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/loss/SquaredError.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.loss
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 
 /**
- * :: DeveloperApi ::
  * Class for squared error loss calculation.
  *
  * The squared (L2) error is defined as:
@@ -29,7 +28,6 @@ import org.apache.spark.annotation.{DeveloperApi, Since}
  * where y is the label and F(x) is the model prediction for features x.
  */
 @Since("1.2.0")
-@DeveloperApi
 object SquaredError extends Loss {
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
index f3dbfd96e1815..54d95a3ffad1b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/InformationGainStats.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
 
 /**
- * :: DeveloperApi ::
  * Information gain statistics for each split
  * @param gain information gain value
  * @param impurity current node impurity
@@ -31,7 +30,6 @@ import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
  * @param rightPredict right node predict
  */
 @Since("1.0.0")
-@DeveloperApi
 class InformationGainStats(
     val gain: Double,
     val impurity: Double,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
index 5fd053647aa46..bac36ce6937b0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Node.scala
@@ -17,13 +17,12 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.Vector
 import org.apache.spark.mllib.tree.configuration.FeatureType._
 
 /**
- * :: DeveloperApi ::
  * Node in a decision tree.
  *
  * About node indexing:
@@ -40,7 +39,6 @@ import org.apache.spark.mllib.tree.configuration.FeatureType._
  * @param stats information gain stats
  */
 @Since("1.0.0")
-@DeveloperApi
 class Node @Since("1.2.0") (
     @Since("1.0.0") val id: Int,
     @Since("1.0.0") var predict: Predict,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
index 1dbdd2d860efd..44bc625d217f1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Predict.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 
 /**
- * :: DeveloperApi ::
  * Predicted value for a node
  * @param predict predicted value
  * @param prob probability of the label (classification only)
  */
 @Since("1.2.0")
-@DeveloperApi
 class Predict @Since("1.2.0") (
     @Since("1.2.0") val predict: Double,
     @Since("1.2.0") val prob: Double = 0.0) extends Serializable {
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
index bda5e662779c7..dbada8f983b99 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/Split.scala
@@ -17,11 +17,10 @@
 
 package org.apache.spark.mllib.tree.model
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType
 
 /**
- * :: DeveloperApi ::
  * Split applied to a feature
  * @param feature feature index
  * @param threshold Threshold for continuous feature.
@@ -30,7 +29,6 @@ import org.apache.spark.mllib.tree.configuration.FeatureType.FeatureType
  * @param categories Split left if categorical feature value is in this set, else right.
  */
 @Since("1.0.0")
-@DeveloperApi
 case class Split(
     @Since("1.0.0") feature: Int,
     @Since("1.0.0") threshold: Double,
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
index 810f528c71906..6b3afdd262e99 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
@@ -25,7 +25,7 @@ import org.json4s.JsonDSL._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.linalg.Vector
@@ -174,7 +174,6 @@ class GradientBoostedTreesModel @Since("1.2.0") (
 object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
 
   /**
-   * :: DeveloperApi ::
    * Compute the initial predictions and errors for a dataset for the first
    * iteration of gradient boosting.
    * @param data: training data.
@@ -185,7 +184,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
    *         corresponding to every sample.
    */
   @Since("1.4.0")
-  @DeveloperApi
   def computeInitialPredictionAndError(
       data: RDD[LabeledPoint],
       initTreeWeight: Double,
@@ -199,7 +197,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
   }
 
   /**
-   * :: DeveloperApi ::
    * Update a zipped predictionError RDD
    * (as obtained with computeInitialPredictionAndError)
    * @param data: training data.
@@ -211,7 +208,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
    *         corresponding to each sample.
    */
   @Since("1.4.0")
-  @DeveloperApi
   def updatePredictionError(
     data: RDD[LabeledPoint],
     predictionAndError: RDD[(Double, Double)],
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
index 2c712d8f821a0..1a8c8807f91d4 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/DataValidators.scala
@@ -17,16 +17,14 @@
 
 package org.apache.spark.mllib.util
 
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * A collection of methods used to validate data before applying ML algorithms.
  */
-@DeveloperApi
 @Since("0.8.0")
 object DataValidators extends Logging {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
index b6eb10e9de00a..a409abc125c0c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
@@ -20,16 +20,14 @@ package org.apache.spark.mllib.util
 import scala.util.Random
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate test data for KMeans. This class first chooses k cluster centers
  * from a d-dimensional Gaussian distribution scaled by factor r and then creates a Gaussian
  * cluster with scale 1 around each center.
  */
-@DeveloperApi
 @Since("0.8.0")
 object KMeansDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
index 30c77d2af8fbc..b044c86f1069a 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
@@ -23,18 +23,16 @@ import scala.util.Random
 import com.github.fommil.netlib.BLAS.{getInstance => blas}
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{BLAS, Vectors}
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate sample data used for Linear Data. This class generates
  * uniformly random values for every feature and adds Gaussian noise with mean `eps` to the
  * response variable `Y`.
  */
-@DeveloperApi
 @Since("0.8.0")
 object LinearDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
index 7e9d9465441c9..bca4ad4a3a60e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
@@ -20,17 +20,15 @@ package org.apache.spark.mllib.util
 import scala.util.Random
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.Vectors
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate test data for LogisticRegression. This class chooses positive labels
  * with probability `probOne` and scales features for positive examples by `eps`.
  */
-@DeveloperApi
 @Since("0.8.0")
 object LogisticRegressionDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
index 7a308a5ec25c0..de21db8d44d7c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
@@ -22,12 +22,11 @@ import java.{util => ju}
 import scala.util.Random
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.{BLAS, DenseMatrix}
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate RDD(s) containing data for Matrix Factorization.
  *
  * This method samples training entries according to the oversampling factor
@@ -50,7 +49,6 @@ import org.apache.spark.rdd.RDD
  *   test           (Boolean) Whether to create testing RDD.
  *   testSampFact   (Double) Percentage of training data to use as test data.
  */
-@DeveloperApi
 @Since("0.8.0")
 object MFDataGenerator {
   @Since("0.8.0")
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
index 9f6ba025aedde..9fffa508afbfb 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
@@ -22,17 +22,15 @@ import scala.util.Random
 import com.github.fommil.netlib.BLAS.{getInstance => blas}
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.mllib.linalg.Vectors
 import org.apache.spark.mllib.regression.LabeledPoint
 import org.apache.spark.rdd.RDD
 
 /**
- * :: DeveloperApi ::
  * Generate sample data used for SVM. This class generates uniform random values
  * for the features and adds Gaussian noise with weight 0.1 to generate labels.
  */
-@DeveloperApi
 @Since("0.8.0")
 object SVMDataGenerator {
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
index e8889bfd7c381..8f2d8b9014c68 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
@@ -24,17 +24,14 @@ import org.json4s._
 import org.json4s.jackson.JsonMethods._
 
 import org.apache.spark.SparkContext
-import org.apache.spark.annotation.{DeveloperApi, Since}
+import org.apache.spark.annotation.Since
 import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.types.{DataType, StructField, StructType}
 
 /**
- * :: DeveloperApi ::
- *
  * Trait for models and transformers which may be saved as files.
  * This should be inherited by the class which implements model instances.
  */
-@DeveloperApi
 @Since("1.3.0")
 trait Saveable {
 
@@ -57,12 +54,9 @@ trait Saveable {
 }
 
 /**
- * :: DeveloperApi ::
- *
  * Trait for classes which can load models and transformers from files.
  * This should be inherited by an object paired with the model class.
  */
-@DeveloperApi
 @Since("1.3.0")
 trait Loader[M <: Saveable] {
 
diff --git a/python/pyspark/ml/base.py b/python/pyspark/ml/base.py
index 542cb25172ead..3368a9d36c783 100644
--- a/python/pyspark/ml/base.py
+++ b/python/pyspark/ml/base.py
@@ -96,8 +96,6 @@ def fitMultiple(self, dataset, paramMaps):
         :return: A thread safe iterable which contains one model for each param map. Each
                  call to `next(modelIterator)` will return `(index, model)` where model was fit
                  using `paramMaps[index]`. `index` values may not be sequential.
-
-        .. note:: DeveloperApi
         """
         estimator = self.copy()
 
diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py
index fe61f9f0fffd6..1be8755c7b982 100644
--- a/python/pyspark/ml/param/__init__.py
+++ b/python/pyspark/ml/param/__init__.py
@@ -76,8 +76,6 @@ def __eq__(self, other):
 
 class TypeConverters(object):
     """
-    .. note:: DeveloperApi
-
     Factory methods for common type conversion functions for `Param.typeConverter`.
 
     .. versionadded:: 2.0.0
diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py
index 097530230cbca..09e0748ffbb3b 100644
--- a/python/pyspark/ml/pipeline.py
+++ b/python/pyspark/ml/pipeline.py
@@ -324,8 +324,6 @@ def _to_java(self):
 @inherit_doc
 class PipelineSharedReadWrite():
     """
-    .. note:: DeveloperApi
-
     Functions for :py:class:`MLReader` and :py:class:`MLWriter` shared between
     :py:class:`Pipeline` and :py:class:`PipelineModel`
 
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index fe0ed569f214d..35ad5518e1c1f 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -345,8 +345,6 @@ def read(cls):
 @inherit_doc
 class DefaultParamsWritable(MLWritable):
     """
-    .. note:: DeveloperApi
-
     Helper trait for making simple :py:class:`Params` types writable.  If a :py:class:`Params`
     class stores all data as :py:class:`Param` values, then extending this trait will provide
     a default implementation of writing saved instances of the class.
@@ -370,8 +368,6 @@ def write(self):
 @inherit_doc
 class DefaultParamsWriter(MLWriter):
     """
-    .. note:: DeveloperApi
-
     Specialization of :py:class:`MLWriter` for :py:class:`Params` types
 
     Class for writing Estimators and Transformers whose parameters are JSON-serializable.
@@ -445,8 +441,6 @@ def _get_metadata_to_save(instance, sc, extraMetadata=None, paramMap=None):
 @inherit_doc
 class DefaultParamsReadable(MLReadable):
     """
-    .. note:: DeveloperApi
-
     Helper trait for making simple :py:class:`Params` types readable.
     If a :py:class:`Params` class stores all data as :py:class:`Param` values,
     then extending this trait will provide a default implementation of reading saved
@@ -466,8 +460,6 @@ def read(cls):
 @inherit_doc
 class DefaultParamsReader(MLReader):
     """
-    .. note:: DeveloperApi
-
     Specialization of :py:class:`MLReader` for :py:class:`Params` types
 
     Default :py:class:`MLReader` implementation for transformers and estimators that
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index 905c4da2d880d..3efae6ff0ecc3 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -43,8 +43,6 @@
 
 class VectorTransformer(object):
     """
-    .. note:: DeveloperApi
-
     Base class for transformation of a vector or RDD of vector
     """
     def transform(self, vector):

From 9cf5d170910a65792d894d129a976ec486b4abc6 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 16 Mar 2020 11:22:34 -0700
Subject: [PATCH 0288/1280] [SPARK-31146][SQL] Leverage the helper method for
 aliasing in built-in SQL expressions

### What changes were proposed in this pull request?

This PR is kind of a followup of #26808. It leverages the helper method for aliasing in built-in SQL expressions to use the alias as its output column name where it's applicable.

- `Expression`, `UnaryMathExpression` and `BinaryMathExpression` search the alias in the tags by default.
- When the naming is different in its implementation, it has to be overwritten for the expression specifically. E.g., `CallMethodViaReflection`, `Remainder`, `CurrentTimestamp`,
`FormatString` and `XPathDouble`.

This PR fixes the aliases of the functions below:

| class                    | alias            |
|--------------------------|------------------|
|`Rand`                    |`random`          |
|`Ceil`                    |`ceiling`         |
|`Remainder`               |`mod`             |
|`Pow`                     |`pow`             |
|`Signum`                  |`sign`            |
|`Chr`                     |`char`            |
|`Length`                  |`char_length`     |
|`Length`                  |`character_length`|
|`FormatString`            |`printf`          |
|`Substring`               |`substr`          |
|`Upper`                   |`ucase`           |
|`XPathDouble`             |`xpath_number`    |
|`DayOfMonth`              |`day`             |
|`CurrentTimestamp`        |`now`             |
|`Size`                    |`cardinality`     |
|`Sha1`                    |`sha`             |
|`CallMethodViaReflection` |`java_method`     |

Note: `EqualTo`, `=` and `==` aliases were excluded because it's unable to leverage this helper method. It should fix the parser.

Note: this PR also excludes some instances such as `ToDegrees`, `ToRadians`, `UnaryMinus` and `UnaryPositive` that needs an explicit name overwritten to make the scope of this PR smaller.

### Why are the changes needed?

To respect expression name.

### Does this PR introduce any user-facing change?

Yes, it will change the output column name.

### How was this patch tested?

Manually tested, and unittests were added.

Closes #27901 from HyukjinKwon/31146.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6704103499d2003b1879ff0b4b8e29141e401b9f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/FunctionRegistry.scala  | 38 ++++++++++---------
 .../expressions/CallMethodViaReflection.scala |  4 +-
 .../sql/catalyst/expressions/Expression.scala |  5 ++-
 .../expressions/aggregate/First.scala         |  2 -
 .../catalyst/expressions/aggregate/Last.scala |  2 -
 .../sql/catalyst/expressions/arithmetic.scala | 14 ++++++-
 .../expressions/datetimeExpressions.scala     |  4 +-
 .../expressions/mathExpressions.scala         | 10 ++---
 .../expressions/stringExpressions.scala       |  7 ++--
 .../sql/catalyst/expressions/xml/xpath.scala  |  5 ++-
 .../sql-tests/results/operators.sql.out       | 14 +++----
 .../results/postgreSQL/insert.sql.out         |  2 +-
 .../results/postgreSQL/numeric.sql.out        |  2 +-
 .../results/postgreSQL/strings.sql.out        |  4 +-
 .../results/string-functions.sql.out          |  6 +--
 .../native/implicitTypeCasts.sql.out          |  2 +-
 .../org/apache/spark/sql/ExplainSuite.scala   |  4 +-
 17 files changed, 70 insertions(+), 55 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 6c4aee4f58b75..c11186ebc0745 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -218,7 +218,7 @@ object FunctionRegistry {
     expression[PosExplode]("posexplode"),
     expressionGeneratorOuter[PosExplode]("posexplode_outer"),
     expression[Rand]("rand"),
-    expression[Rand]("random"),
+    expression[Rand]("random", true),
     expression[Randn]("randn"),
     expression[Stack]("stack"),
     expression[CaseWhen]("when"),
@@ -235,7 +235,7 @@ object FunctionRegistry {
     expression[BRound]("bround"),
     expression[Cbrt]("cbrt"),
     expression[Ceil]("ceil"),
-    expression[Ceil]("ceiling"),
+    expression[Ceil]("ceiling", true),
     expression[Cos]("cos"),
     expression[Cosh]("cosh"),
     expression[Conv]("conv"),
@@ -252,12 +252,12 @@ object FunctionRegistry {
     expression[Log1p]("log1p"),
     expression[Log2]("log2"),
     expression[Log]("ln"),
-    expression[Remainder]("mod"),
+    expression[Remainder]("mod", true),
     expression[UnaryMinus]("negative"),
     expression[Pi]("pi"),
     expression[Pmod]("pmod"),
     expression[UnaryPositive]("positive"),
-    expression[Pow]("pow"),
+    expression[Pow]("pow", true),
     expression[Pow]("power"),
     expression[ToRadians]("radians"),
     expression[Rint]("rint"),
@@ -265,7 +265,7 @@ object FunctionRegistry {
     expression[ShiftLeft]("shiftleft"),
     expression[ShiftRight]("shiftright"),
     expression[ShiftRightUnsigned]("shiftrightunsigned"),
-    expression[Signum]("sign"),
+    expression[Signum]("sign", true),
     expression[Signum]("signum"),
     expression[Sin]("sin"),
     expression[Sinh]("sinh"),
@@ -323,12 +323,12 @@ object FunctionRegistry {
 
     // string functions
     expression[Ascii]("ascii"),
-    expression[Chr]("char"),
+    expression[Chr]("char", true),
     expression[Chr]("chr"),
     expression[Base64]("base64"),
     expression[BitLength]("bit_length"),
-    expression[Length]("char_length"),
-    expression[Length]("character_length"),
+    expression[Length]("char_length", true),
+    expression[Length]("character_length", true),
     expression[ConcatWs]("concat_ws"),
     expression[Decode]("decode"),
     expression[Elt]("elt"),
@@ -351,7 +351,7 @@ object FunctionRegistry {
     expression[JsonTuple]("json_tuple"),
     expression[ParseUrl]("parse_url"),
     expression[StringLocate]("position"),
-    expression[FormatString]("printf"),
+    expression[FormatString]("printf", true),
     expression[RegExpExtract]("regexp_extract"),
     expression[RegExpReplace]("regexp_replace"),
     expression[StringRepeat]("repeat"),
@@ -364,21 +364,21 @@ object FunctionRegistry {
     expression[SoundEx]("soundex"),
     expression[StringSpace]("space"),
     expression[StringSplit]("split"),
-    expression[Substring]("substr"),
+    expression[Substring]("substr", true),
     expression[Substring]("substring"),
     expression[Left]("left"),
     expression[Right]("right"),
     expression[SubstringIndex]("substring_index"),
     expression[StringTranslate]("translate"),
     expression[StringTrim]("trim"),
-    expression[Upper]("ucase"),
+    expression[Upper]("ucase", true),
     expression[UnBase64]("unbase64"),
     expression[Unhex]("unhex"),
     expression[Upper]("upper"),
     expression[XPathList]("xpath"),
     expression[XPathBoolean]("xpath_boolean"),
     expression[XPathDouble]("xpath_double"),
-    expression[XPathDouble]("xpath_number"),
+    expression[XPathDouble]("xpath_number", true),
     expression[XPathFloat]("xpath_float"),
     expression[XPathInt]("xpath_int"),
     expression[XPathLong]("xpath_long"),
@@ -393,7 +393,7 @@ object FunctionRegistry {
     expression[DateAdd]("date_add"),
     expression[DateFormatClass]("date_format"),
     expression[DateSub]("date_sub"),
-    expression[DayOfMonth]("day"),
+    expression[DayOfMonth]("day", true),
     expression[DayOfYear]("dayofyear"),
     expression[DayOfMonth]("dayofmonth"),
     expression[FromUnixTime]("from_unixtime"),
@@ -404,7 +404,7 @@ object FunctionRegistry {
     expression[Month]("month"),
     expression[MonthsBetween]("months_between"),
     expression[NextDay]("next_day"),
-    expression[CurrentTimestamp]("now"),
+    expression[CurrentTimestamp]("now", true),
     expression[Quarter]("quarter"),
     expression[Second]("second"),
     expression[ParseToTimestamp]("to_timestamp"),
@@ -445,7 +445,7 @@ object FunctionRegistry {
     expression[MapConcat]("map_concat"),
     expression[Size]("size"),
     expression[Slice]("slice"),
-    expression[Size]("cardinality"),
+    expression[Size]("cardinality", true),
     expression[ArraysZip]("arrays_zip"),
     expression[SortArray]("sort_array"),
     expression[Shuffle]("shuffle"),
@@ -478,7 +478,7 @@ object FunctionRegistry {
     expression[Uuid]("uuid"),
     expression[Murmur3Hash]("hash"),
     expression[XxHash64]("xxhash64"),
-    expression[Sha1]("sha"),
+    expression[Sha1]("sha", true),
     expression[Sha1]("sha1"),
     expression[Sha2]("sha2"),
     expression[SparkPartitionID]("spark_partition_id"),
@@ -488,7 +488,7 @@ object FunctionRegistry {
     expression[MonotonicallyIncreasingID]("monotonically_increasing_id"),
     expression[CurrentDatabase]("current_database"),
     expression[CallMethodViaReflection]("reflect"),
-    expression[CallMethodViaReflection]("java_method"),
+    expression[CallMethodViaReflection]("java_method", true),
     expression[SparkVersion]("version"),
     expression[TypeOf]("typeof"),
 
@@ -590,7 +590,9 @@ object FunctionRegistry {
       if (varargCtor.isDefined) {
         // If there is an apply method that accepts Seq[Expression], use that one.
         try {
-          varargCtor.get.newInstance(expressions).asInstanceOf[Expression]
+          val exp = varargCtor.get.newInstance(expressions).asInstanceOf[Expression]
+          if (setAlias) exp.setTagValue(FUNC_ALIAS, name)
+          exp
         } catch {
           // the exception is an invocation exception. To get a meaningful message, we need the
           // cause.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index 65bb9a8c642b6..e6a4c8f1d3749 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.lang.reflect.{Method, Modifier}
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.types._
@@ -55,7 +55,7 @@ import org.apache.spark.util.Utils
 case class CallMethodViaReflection(children: Seq[Expression])
   extends Expression with CodegenFallback {
 
-  override def prettyName: String = "reflect"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("reflect")
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.size < 2) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 1599321982ce2..f29ece2e03b08 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
@@ -258,7 +258,8 @@ abstract class Expression extends TreeNode[Expression] {
    * Returns a user-facing string representation of this expression's name.
    * This should usually match the name of the function in SQL.
    */
-  def prettyName: String = nodeName.toLowerCase(Locale.ROOT)
+  def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(nodeName.toLowerCase(Locale.ROOT))
 
   protected def flatArguments: Iterator[Any] = stringArgs.flatMap {
     case t: Iterable[_] => t
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
index 210acf33fc431..2c0060c22a865 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
@@ -117,7 +117,5 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
 
   override lazy val evaluateExpression: AttributeReference = first
 
-  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("first")
-
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
index 2c89a4b973a7e..6793ac7632ffd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
@@ -115,7 +115,5 @@ case class Last(child: Expression, ignoreNullsExpr: Expression)
 
   override lazy val evaluateExpression: AttributeReference = last
 
-  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("last")
-
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 215e88a2cc8a4..6a64819aabb48 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult, TypeCoercion}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{IntervalUtils, TypeUtils}
@@ -457,6 +457,18 @@ case class Remainder(left: Expression, right: Expression) extends DivModLike {
 
   override def symbol: String = "%"
   override def decimalMethod: String = "remainder"
+  override def toString: String = {
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(sqlOperator) match {
+      case operator if operator == sqlOperator => s"($left $sqlOperator $right)"
+      case funcName => s"$funcName($left, $right)"
+    }
+  }
+  override def sql: String = {
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(sqlOperator) match {
+      case operator if operator == sqlOperator => s"(${left.sql} $sqlOperator ${right.sql})"
+      case funcName => s"$funcName(${left.sql}, ${right.sql})"
+    }
+  }
 
   private lazy val mod: (Any, Any) => Any = dataType match {
     // special cases to make float/double primitive types faster
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 1dad440b59381..0b1b65fa19300 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -29,6 +29,7 @@ import org.apache.commons.text.StringEscapeUtils
 import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter}
@@ -99,7 +100,8 @@ case class CurrentTimestamp() extends LeafExpression with CodegenFallback {
 
   override def eval(input: InternalRow): Any = currentTimestamp()
 
-  override def prettyName: String = "current_timestamp"
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("current_timestamp")
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index d5b959b91c23d..66e6334e3a450 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -21,7 +21,7 @@ import java.{lang => jl}
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
@@ -62,8 +62,8 @@ abstract class UnaryMathExpression(val f: Double => Double, name: String)
   override def inputTypes: Seq[AbstractDataType] = Seq(DoubleType)
   override def dataType: DataType = DoubleType
   override def nullable: Boolean = true
-  override def toString: String = s"$name($child)"
-  override def prettyName: String = name
+  override def toString: String = s"$prettyName($child)"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(name)
 
   protected override def nullSafeEval(input: Any): Any = {
     f(input.asInstanceOf[Double])
@@ -115,9 +115,9 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
 
   override def inputTypes: Seq[DataType] = Seq(DoubleType, DoubleType)
 
-  override def toString: String = s"$name($left, $right)"
+  override def toString: String = s"$prettyName($left, $right)"
 
-  override def prettyName: String = name
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse(name)
 
   override def dataType: DataType = DoubleType
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index cc09f601db9c4..50a90ae40497a 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -27,7 +27,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.commons.codec.binary.{Base64 => CommonsBase64}
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, TypeUtils}
@@ -1450,7 +1450,7 @@ case class ParseUrl(children: Seq[Expression])
 // scalastyle:on line.size.limit
 case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes {
 
-  require(children.nonEmpty, "format_string() should take at least 1 argument")
+  require(children.nonEmpty, s"$prettyName() should take at least 1 argument")
 
   override def foldable: Boolean = children.forall(_.foldable)
   override def nullable: Boolean = children(0).nullable
@@ -1517,7 +1517,8 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC
       }""")
   }
 
-  override def prettyName: String = "format_string"
+  override def prettyName: String = getTagValue(
+    FunctionRegistry.FUNC_ALIAS).getOrElse("format_string")
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
index 073b45af51caf..55e06cb9e8471 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions.xml
 
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -160,7 +160,8 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract {
   """)
 // scalastyle:on line.size.limit
 case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract {
-  override def prettyName: String = "xpath_double"
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("xpath_double")
   override def dataType: DataType = DoubleType
 
   override def nullSafeEval(xml: Any, path: Any): Any = {
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index 083410f8807c4..cf857cf9f98ad 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -293,7 +293,7 @@ struct<COT(CAST(-1 AS DOUBLE)):double>
 -- !query
 select ceiling(0)
 -- !query schema
-struct<CEIL(CAST(0 AS DOUBLE)):bigint>
+struct<ceiling(CAST(0 AS DOUBLE)):bigint>
 -- !query output
 0
 
@@ -301,7 +301,7 @@ struct<CEIL(CAST(0 AS DOUBLE)):bigint>
 -- !query
 select ceiling(1)
 -- !query schema
-struct<CEIL(CAST(1 AS DOUBLE)):bigint>
+struct<ceiling(CAST(1 AS DOUBLE)):bigint>
 -- !query output
 1
 
@@ -317,7 +317,7 @@ struct<CEIL(1234567890123456):bigint>
 -- !query
 select ceiling(1234567890123456)
 -- !query schema
-struct<CEIL(1234567890123456):bigint>
+struct<ceiling(1234567890123456):bigint>
 -- !query output
 1234567890123456
 
@@ -333,7 +333,7 @@ struct<CEIL(0.01):decimal(1,0)>
 -- !query
 select ceiling(-0.10)
 -- !query schema
-struct<CEIL(-0.10):decimal(1,0)>
+struct<ceiling(-0.10):decimal(1,0)>
 -- !query output
 0
 
@@ -389,7 +389,7 @@ true
 -- !query
 select mod(7, 2), mod(7, 0), mod(0, 2), mod(7, null), mod(null, 2), mod(null, null)
 -- !query schema
-struct<(7 % 2):int,(7 % 0):int,(0 % 2):int,(7 % CAST(NULL AS INT)):int,(CAST(NULL AS INT) % 2):int,(CAST(NULL AS DOUBLE) % CAST(NULL AS DOUBLE)):double>
+struct<mod(7, 2):int,mod(7, 0):int,mod(0, 2):int,mod(7, CAST(NULL AS INT)):int,mod(CAST(NULL AS INT), 2):int,mod(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE)):double>
 -- !query output
 1	NULL	0	NULL	NULL	NULL
 
@@ -405,7 +405,7 @@ struct<bit_length(abc):int>
 -- !query
 select CHAR_LENGTH('abc')
 -- !query schema
-struct<length(abc):int>
+struct<char_length(abc):int>
 -- !query output
 3
 
@@ -413,7 +413,7 @@ struct<length(abc):int>
 -- !query
 select CHARACTER_LENGTH('abc')
 -- !query schema
-struct<length(abc):int>
+struct<character_length(abc):int>
 -- !query output
 3
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out
index 1046d0ec86bbd..63ad74aac32ec 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/insert.sql.out
@@ -64,7 +64,7 @@ struct<>
 -- !query
 select col1, col2, char_length(col3) from inserttest
 -- !query schema
-struct<col1:int,col2:int,length(col3):int>
+struct<col1:int,col2:int,char_length(col3):int>
 -- !query output
 30	50	10000
 NULL	3	7
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
index 65b6641983a1d..e59b9d5b63a40 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
@@ -4404,7 +4404,7 @@ struct<>
 -- !query
 SELECT a, ceil(a), ceiling(a), floor(a), round(a) FROM ceil_floor_round
 -- !query schema
-struct<a:decimal(38,18),CEIL(a):decimal(21,0),CEIL(a):decimal(21,0),FLOOR(a):decimal(21,0),round(a, 0):decimal(38,0)>
+struct<a:decimal(38,18),CEIL(a):decimal(21,0),ceiling(a):decimal(21,0),FLOOR(a):decimal(21,0),round(a, 0):decimal(38,0)>
 -- !query output
 -0.000001000000000000	0	0	-1	0
 -5.499999000000000000	-5	-5	-6	-5
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
index 3b26d561d20c3..5f89c799498ac 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
@@ -627,7 +627,7 @@ struct<>
 -- !query
 SELECT substr(f1, 99995) from toasttest
 -- !query schema
-struct<substring(f1, 99995, 2147483647):string>
+struct<substr(f1, 99995, 2147483647):string>
 -- !query output
 567890
 567890
@@ -638,7 +638,7 @@ struct<substring(f1, 99995, 2147483647):string>
 -- !query
 SELECT substr(f1, 99995, 10) from toasttest
 -- !query schema
-struct<substring(f1, 99995, 10):string>
+struct<substr(f1, 99995, 10):string>
 -- !query output
 567890
 567890
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 708dbb404c285..042d332bdb5c2 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -111,7 +111,7 @@ struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
 -- !query
 SELECT substr('Spark SQL', 5)
 -- !query schema
-struct<substring(Spark SQL, 5, 2147483647):string>
+struct<substr(Spark SQL, 5, 2147483647):string>
 -- !query output
 k SQL
 
@@ -119,7 +119,7 @@ k SQL
 -- !query
 SELECT substr('Spark SQL', -3)
 -- !query schema
-struct<substring(Spark SQL, -3, 2147483647):string>
+struct<substr(Spark SQL, -3, 2147483647):string>
 -- !query output
 SQL
 
@@ -127,7 +127,7 @@ SQL
 -- !query
 SELECT substr('Spark SQL', 5, 1)
 -- !query schema
-struct<substring(Spark SQL, 5, 1):string>
+struct<substr(Spark SQL, 5, 1):string>
 -- !query output
 k
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
index f841adf89612e..e47decbd33920 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/implicitTypeCasts.sql.out
@@ -285,7 +285,7 @@ struct<month(CAST(1996-01-10 AS DATE)):int>
 -- !query
 SELECT day( '1996-01-10') FROM t
 -- !query schema
-struct<dayofmonth(CAST(1996-01-10 AS DATE)):int>
+struct<day(CAST(1996-01-10 AS DATE)):int>
 -- !query output
 10
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index b591705274110..16c58028adfe2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -116,8 +116,8 @@ class ExplainSuite extends QueryTest with SharedSparkSession {
       // plan should show the rewritten aggregate expression.
       val df = sql("SELECT k, every(v), some(v), any(v) FROM test_agg GROUP BY k")
       checkKeywordsExistsInExplain(df,
-        "Aggregate [k#x], [k#x, min(v#x) AS every(v)#x, max(v#x) AS some(v)#x, " +
-          "max(v#x) AS any(v)#x]")
+        "Aggregate [k#x], [k#x, every(v#x) AS every(v)#x, some(v#x) AS some(v)#x, " +
+          "any(v#x) AS any(v)#x]")
     }
   }
 

From fed4a680e3c45462d0d69aeac53780a677f988ae Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 16 Mar 2020 11:45:25 -0700
Subject: [PATCH 0289/1280] [SPARK-31163][SQL] TruncateTableCommand with
 acl/permission should handle non-existed path

### What changes were proposed in this pull request?

This fix #26956
Wrap try-catch on `fs.getFileStatus(path)` within acl/permission in case of the path doesn't exist.

### Why are the changes needed?

`truncate table` may fail to re-create path in case of interruption or something else. As a result, next time we `truncate table` on the same table with acl/permission, it will fail due to `FileNotFoundException`. And it also brings behavior change compares to previous Spark version, which could still `truncate table` successfully even if the path doesn't exist.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added UT.

Closes #27923 from Ngone51/fix_truncate.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit cb26f636b08aea4c5c6bf5035a359cd3cbf335c0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/command/tables.scala  |  2 +-
 .../sql/execution/command/DDLSuite.scala      | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 6243261d3cd16..d4de822d8b036 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -508,8 +508,8 @@ case class TruncateTableCommand(
           var optPermission: Option[FsPermission] = None
           var optAcls: Option[java.util.List[AclEntry]] = None
           if (!ignorePermissionAcl) {
-            val fileStatus = fs.getFileStatus(path)
             try {
+              val fileStatus = fs.getFileStatus(path)
               optPermission = Some(fileStatus.getPermission())
             } catch {
               case NonFatal(_) => // do nothing
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 5a67dce0ffe6a..10ad8acc68937 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -2084,6 +2084,27 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  test("SPARK-31163: acl/permission should handle non-existed path when truncating table") {
+    withSQLConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL.key -> "false") {
+      withTable("tab1") {
+        sql("CREATE TABLE tab1 (col1 STRING, col2 INT) USING parquet PARTITIONED BY (col2)")
+        sql("INSERT INTO tab1 SELECT 'one', 1")
+        checkAnswer(spark.table("tab1"), Row("one", 1))
+        val part = spark.sessionState.catalog.listPartitions(TableIdentifier("tab1")).head
+        val path = new File(part.location.getPath)
+        sql("TRUNCATE TABLE tab1")
+        // simulate incomplete/unsuccessful truncate
+        assert(path.exists())
+        path.delete()
+        assert(!path.exists())
+        // execute without java.io.FileNotFoundException
+        sql("TRUNCATE TABLE tab1")
+        // partition path should be re-created
+        assert(path.exists())
+      }
+    }
+  }
+
   test("create temporary view with mismatched schema") {
     withTable("tab1") {
       spark.range(10).write.saveAsTable("tab1")

From 395dbbc031b94945e2a2e2ae938c4e23d57b41ce Mon Sep 17 00:00:00 2001
From: Nicholas Chammas <nicholas.chammas@liveramp.com>
Date: Tue, 17 Mar 2020 10:41:41 +0900
Subject: [PATCH 0290/1280] [SPARK-31155] Remove pydocstyle tests

### What changes were proposed in this pull request?

As discovered here https://github.com/apache/spark/pull/27910#issuecomment-599027190, pydocstyle tests were not running anywhere (not on Jenkins; not on GitHub).

~This PR enables those tests.~

It also seems like a [large hill to climb](https://github.com/apache/spark/pull/27912#issuecomment-599167117) to enable any meaningful checks, so we're going to just rip pydocstyle out for now.

### Why are the changes needed?

Presumably, we defined those doc style tests because we care about whatever it is they enforce. Since we're not actually testing anything, though, it's better to clear the cruft.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Will check the GitHub workflow logs on this PR.

Closes #27912 from nchammas/SPARK-31155-pydocstyle.

Authored-by: Nicholas Chammas <nicholas.chammas@liveramp.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b4748ca0ab5893affa9b980457026ce481cd0fb0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/lint-python | 46 ----------------------------------------------
 dev/tox.ini     |  2 --
 2 files changed, 48 deletions(-)

diff --git a/dev/lint-python b/dev/lint-python
index 9d7dfad7c534c..674896e3b5dd0 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -16,9 +16,6 @@
 # limitations under the License.
 #
 # define test binaries + versions
-PYDOCSTYLE_BUILD="pydocstyle"
-MINIMUM_PYDOCSTYLE="3.0.0"
-
 FLAKE8_BUILD="flake8"
 MINIMUM_FLAKE8="3.5.0"
 
@@ -165,48 +162,6 @@ flake8 checks failed."
     fi
 }
 
-function pydocstyle_test {
-    local PYDOCSTYLE_REPORT=
-    local PYDOCSTYLE_STATUS=
-    local PYDOCSTYLE_VERSION=
-    local EXPECTED_PYDOCSTYLE=
-
-    # Exclude auto-generated configuration file.
-    local DOC_PATHS_TO_CHECK="$( cd "${SPARK_ROOT_DIR}" && find . -name "*.py" | grep -vF 'functions.py' )"
-
-    # Check python document style, skip check if pydocstyle is not installed.
-    if ! hash "$PYDOCSTYLE_BUILD" 2> /dev/null; then
-        echo "The pydocstyle command was not found. Skipping pydocstyle checks for now."
-        echo
-        return
-    fi
-
-    PYDOCSTYLE_VERSION="$($PYDOCSTYLE_BUILD --version)"
-    EXPECTED_PYDOCSTYLE="$(satisfies_min_version $PYDOCSTYLE_VERSION $MINIMUM_PYDOCSTYLE)"
-
-    if [[ "$EXPECTED_PYDOCSTYLE" == "False" ]]; then
-        echo "\
-The minimum version of pydocstyle needs to be $MINIMUM_PYDOCSTYLE.
-Your current version is $PYDOCSTYLE_VERSION.
-Skipping pydocstyle checks for now."
-        echo
-        return
-    fi
-
-    echo "starting $PYDOCSTYLE_BUILD test..."
-    PYDOCSTYLE_REPORT=$( ($PYDOCSTYLE_BUILD --config=dev/tox.ini $DOC_PATHS_TO_CHECK) 2>&1)
-    PYDOCSTYLE_STATUS=$?
-
-    if [ "$PYDOCSTYLE_STATUS" -ne 0 ]; then
-        echo "pydocstyle checks failed:"
-        echo "$PYDOCSTYLE_REPORT"
-        exit "$PYDOCSTYLE_STATUS"
-    else
-        echo "pydocstyle checks passed."
-        echo
-    fi
-}
-
 function sphinx_test {
     local SPHINX_REPORT=
     local SPHINX_STATUS=
@@ -252,7 +207,6 @@ PYTHON_SOURCE="$(find . -name "*.py")"
 compile_python_test "$PYTHON_SOURCE"
 pycodestyle_test "$PYTHON_SOURCE"
 flake8_test
-pydocstyle_test
 sphinx_test
 
 echo
diff --git a/dev/tox.ini b/dev/tox.ini
index 54f65692c8303..3ee13c09b5de7 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -17,5 +17,3 @@
 ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
 max-line-length=100
 exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
-[pydocstyle]
-ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D203,D204,D205,D206,D207,D208,D209,D210,D211,D212,D213,D214,D215,D300,D301,D302,D400,D401,D402,D403,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414

From abba354a6558bfba47f8621fa6e60bc1f3b00280 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 17 Mar 2020 12:01:29 +0800
Subject: [PATCH 0291/1280] [SPARK-31166][SQL] UNION map<null, null> and other
 maps should not fail

After https://github.com/apache/spark/pull/27542, `map()` returns `map<null, null>` instead of `map<string, string>`. However, this breaks queries which union `map()` and other maps.

The reason is, `TypeCoercion` rules and `Cast` think it's illegal to cast null type map key to other types, as it makes the key nullable, but it's actually legal. This PR fixes it.

To avoid breaking queries.

Yes, now some queries that work in 2.x can work in 3.0 as well.

new test

Closes #27926 from cloud-fan/bug.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d7b97a1d0daf65710317321490a833f696a46f21)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/catalyst/analysis/TypeCoercion.scala | 2 +-
 .../org/apache/spark/sql/catalyst/expressions/Cast.scala  | 8 ++++++--
 .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala   | 6 ++++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index 0a0bef6c9953a..e149bf2f49764 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -158,7 +158,7 @@ object TypeCoercion {
       }
     case (MapType(kt1, vt1, valueContainsNull1), MapType(kt2, vt2, valueContainsNull2)) =>
       findTypeFunc(kt1, kt2)
-        .filter { kt => !Cast.forceNullable(kt1, kt) && !Cast.forceNullable(kt2, kt) }
+        .filter { kt => Cast.canCastMapKeyNullSafe(kt1, kt) && Cast.canCastMapKeyNullSafe(kt2, kt) }
         .flatMap { kt =>
           findTypeFunc(vt1, vt2).map { vt =>
             MapType(kt, vt, valueContainsNull1 || valueContainsNull2 ||
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 7c4316fe08433..8177136edfd62 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -77,8 +77,7 @@ object Cast {
         resolvableNullability(fn || forceNullable(fromType, toType), tn)
 
     case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
-      canCast(fromKey, toKey) &&
-        (!forceNullable(fromKey, toKey)) &&
+      canCast(fromKey, toKey) && canCastMapKeyNullSafe(fromKey, toKey) &&
         canCast(fromValue, toValue) &&
         resolvableNullability(fn || forceNullable(fromValue, toValue), tn)
 
@@ -98,6 +97,11 @@ object Cast {
     case _ => false
   }
 
+  def canCastMapKeyNullSafe(fromType: DataType, toType: DataType): Boolean = {
+    // If the original map key type is NullType, it's OK as the map must be empty.
+    fromType == NullType || !forceNullable(fromType, toType)
+  }
+
   /**
    * Return true if we need to use the `timeZone` information casting `from` type to `to` type.
    * The patterns matched reflect the current implementation in the Cast node.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 81dfa798b4da3..feb1450253870 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3456,6 +3456,12 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       """.stripMargin)
     checkAnswer(df2, Row(1) :: Nil)
   }
+
+  test("SPARK-31166: UNION map<null, null> and other maps should not fail") {
+    checkAnswer(
+      sql("(SELECT map()) UNION ALL (SELECT map(1, 2))"),
+      Seq(Row(Map[Int, Int]()), Row(Map(1 -> 2))))
+  }
 }
 
 case class Foo(bar: Option[String])

From 63baffdd255d0c1558ce91c1528b774ec1d35f41 Mon Sep 17 00:00:00 2001
From: Zhenhua Wang <wzh_zju@163.com>
Date: Tue, 17 Mar 2020 14:20:16 +0800
Subject: [PATCH 0292/1280] [SPARK-31164][SQL] Inconsistent rdd and output
 partitioning for bucket table when output doesn't contain all bucket columns

### What changes were proposed in this pull request?

For a bucketed table, when deciding output partitioning, if the output doesn't contain all bucket columns, the result is `UnknownPartitioning`. But when generating rdd, current Spark uses `createBucketedReadRDD` because it doesn't check if the output contains all bucket columns. So the rdd and its output partitioning are inconsistent.

### Why are the changes needed?

To fix a bug.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Modified existing tests.

Closes #27924 from wzhfy/inconsistent_rdd_partitioning.

Authored-by: Zhenhua Wang <wzh_zju@163.com>
Signed-off-by: Zhenhua Wang <wzh_zju@163.com>
(cherry picked from commit 1369a973cdefbab177871124d5ceb2ef55ac136d)
Signed-off-by: Zhenhua Wang <wzh_zju@163.com>
---
 .../sql/execution/DataSourceScanExec.scala    | 139 +++++++++---------
 .../spark/sql/sources/BucketedReadSuite.scala |  27 ++--
 2 files changed, 87 insertions(+), 79 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index c1f7f0a195491..8d488d4448506 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -253,74 +253,75 @@ case class FileSourceScanExec(
     partitionFilters.exists(ExecSubqueryExpression.hasSubquery)
   }
 
-  override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = {
-    val bucketSpec = if (relation.sparkSession.sessionState.conf.bucketingEnabled) {
-      relation.bucketSpec
+  private def toAttribute(colName: String): Option[Attribute] =
+    output.find(_.name == colName)
+
+  // exposed for testing
+  lazy val bucketedScan: Boolean = {
+    if (relation.sparkSession.sessionState.conf.bucketingEnabled && relation.bucketSpec.isDefined) {
+      val spec = relation.bucketSpec.get
+      val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
+      bucketColumns.size == spec.bucketColumnNames.size
     } else {
-      None
+      false
     }
-    bucketSpec match {
-      case Some(spec) =>
-        // For bucketed columns:
-        // -----------------------
-        // `HashPartitioning` would be used only when:
-        // 1. ALL the bucketing columns are being read from the table
-        //
-        // For sorted columns:
-        // ---------------------
-        // Sort ordering should be used when ALL these criteria's match:
-        // 1. `HashPartitioning` is being used
-        // 2. A prefix (or all) of the sort columns are being read from the table.
-        //
-        // Sort ordering would be over the prefix subset of `sort columns` being read
-        // from the table.
-        // eg.
-        // Assume (col0, col2, col3) are the columns read from the table
-        // If sort columns are (col0, col1), then sort ordering would be considered as (col0)
-        // If sort columns are (col1, col0), then sort ordering would be empty as per rule #2
-        // above
-
-        def toAttribute(colName: String): Option[Attribute] =
-          output.find(_.name == colName)
-
-        val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
-        if (bucketColumns.size == spec.bucketColumnNames.size) {
-          val partitioning = HashPartitioning(bucketColumns, spec.numBuckets)
-          val sortColumns =
-            spec.sortColumnNames.map(x => toAttribute(x)).takeWhile(x => x.isDefined).map(_.get)
-          val shouldCalculateSortOrder =
-            conf.getConf(SQLConf.LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING) &&
-              sortColumns.nonEmpty &&
-              !hasPartitionsAvailableAtRunTime
-
-          val sortOrder = if (shouldCalculateSortOrder) {
-            // In case of bucketing, its possible to have multiple files belonging to the
-            // same bucket in a given relation. Each of these files are locally sorted
-            // but those files combined together are not globally sorted. Given that,
-            // the RDD partition will not be sorted even if the relation has sort columns set
-            // Current solution is to check if all the buckets have a single file in it
-
-            val files = selectedPartitions.flatMap(partition => partition.files)
-            val bucketToFilesGrouping =
-              files.map(_.getPath.getName).groupBy(file => BucketingUtils.getBucketId(file))
-            val singleFilePartitions = bucketToFilesGrouping.forall(p => p._2.length <= 1)
-
-            if (singleFilePartitions) {
-              // TODO Currently Spark does not support writing columns sorting in descending order
-              // so using Ascending order. This can be fixed in future
-              sortColumns.map(attribute => SortOrder(attribute, Ascending))
-            } else {
-              Nil
-            }
-          } else {
-            Nil
-          }
-          (partitioning, sortOrder)
+  }
+
+  override lazy val (outputPartitioning, outputOrdering): (Partitioning, Seq[SortOrder]) = {
+    if (bucketedScan) {
+      // For bucketed columns:
+      // -----------------------
+      // `HashPartitioning` would be used only when:
+      // 1. ALL the bucketing columns are being read from the table
+      //
+      // For sorted columns:
+      // ---------------------
+      // Sort ordering should be used when ALL these criteria's match:
+      // 1. `HashPartitioning` is being used
+      // 2. A prefix (or all) of the sort columns are being read from the table.
+      //
+      // Sort ordering would be over the prefix subset of `sort columns` being read
+      // from the table.
+      // eg.
+      // Assume (col0, col2, col3) are the columns read from the table
+      // If sort columns are (col0, col1), then sort ordering would be considered as (col0)
+      // If sort columns are (col1, col0), then sort ordering would be empty as per rule #2
+      // above
+      val spec = relation.bucketSpec.get
+      val bucketColumns = spec.bucketColumnNames.flatMap(n => toAttribute(n))
+      val partitioning = HashPartitioning(bucketColumns, spec.numBuckets)
+      val sortColumns =
+        spec.sortColumnNames.map(x => toAttribute(x)).takeWhile(x => x.isDefined).map(_.get)
+      val shouldCalculateSortOrder =
+        conf.getConf(SQLConf.LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING) &&
+          sortColumns.nonEmpty &&
+          !hasPartitionsAvailableAtRunTime
+
+      val sortOrder = if (shouldCalculateSortOrder) {
+        // In case of bucketing, its possible to have multiple files belonging to the
+        // same bucket in a given relation. Each of these files are locally sorted
+        // but those files combined together are not globally sorted. Given that,
+        // the RDD partition will not be sorted even if the relation has sort columns set
+        // Current solution is to check if all the buckets have a single file in it
+
+        val files = selectedPartitions.flatMap(partition => partition.files)
+        val bucketToFilesGrouping =
+          files.map(_.getPath.getName).groupBy(file => BucketingUtils.getBucketId(file))
+        val singleFilePartitions = bucketToFilesGrouping.forall(p => p._2.length <= 1)
+
+        if (singleFilePartitions) {
+          // TODO Currently Spark does not support writing columns sorting in descending order
+          // so using Ascending order. This can be fixed in future
+          sortColumns.map(attribute => SortOrder(attribute, Ascending))
         } else {
-          (UnknownPartitioning(0), Nil)
+          Nil
         }
-      case _ =>
-        (UnknownPartitioning(0), Nil)
+      } else {
+        Nil
+      }
+      (partitioning, sortOrder)
+    } else {
+      (UnknownPartitioning(0), Nil)
     }
   }
 
@@ -393,11 +394,11 @@ case class FileSourceScanExec(
         options = relation.options,
         hadoopConf = relation.sparkSession.sessionState.newHadoopConfWithOptions(relation.options))
 
-    val readRDD = relation.bucketSpec match {
-      case Some(bucketing) if relation.sparkSession.sessionState.conf.bucketingEnabled =>
-        createBucketedReadRDD(bucketing, readFile, dynamicallySelectedPartitions, relation)
-      case _ =>
-        createNonBucketedReadRDD(readFile, dynamicallySelectedPartitions, relation)
+    val readRDD = if (bucketedScan) {
+      createBucketedReadRDD(relation.bucketSpec.get, readFile, dynamicallySelectedPartitions,
+        relation)
+    } else {
+      createNonBucketedReadRDD(readFile, dynamicallySelectedPartitions, relation)
     }
     sendDriverMetrics()
     readRDD
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index 57bbf200ed881..14ba008e89961 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.catalog.BucketSpec
 import org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
-import org.apache.spark.sql.execution.{DataSourceScanExec, SortExec}
+import org.apache.spark.sql.execution.{DataSourceScanExec, FileSourceScanExec, SortExec, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
 import org.apache.spark.sql.execution.datasources.BucketingUtils
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
@@ -100,6 +100,12 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     }
   }
 
+  private def getFileScan(plan: SparkPlan): FileSourceScanExec = {
+    val fileScan = plan.collect { case f: FileSourceScanExec => f }
+    assert(fileScan.nonEmpty, plan)
+    fileScan.head
+  }
+
   // To verify if the bucket pruning works, this function checks two conditions:
   //   1) Check if the pruned buckets (before filtering) are empty.
   //   2) Verify the final result is the same as the expected one
@@ -119,8 +125,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       // Filter could hide the bug in bucket pruning. Thus, skipping all the filters
       val plan = bucketedDataFrame.filter(filterCondition).queryExecution.executedPlan
-      val rdd = plan.find(_.isInstanceOf[DataSourceScanExec])
-      assert(rdd.isDefined, plan)
+      val fileScan = getFileScan(plan)
 
       // if nothing should be pruned, skip the pruning test
       if (bucketValues.nonEmpty) {
@@ -128,7 +133,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         bucketValues.foreach { value =>
           matchedBuckets.set(BucketingUtils.getBucketIdFromValue(bucketColumn, numBuckets, value))
         }
-        val invalidBuckets = rdd.get.execute().mapPartitionsWithIndex { case (index, iter) =>
+        val invalidBuckets = fileScan.execute().mapPartitionsWithIndex { case (index, iter) =>
           // return indexes of partitions that should have been pruned and are not empty
           if (!matchedBuckets.get(index % numBuckets) && iter.nonEmpty) {
             Iterator(index)
@@ -297,10 +302,9 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       val bucketedDataFrame = spark.table("bucketed_table").select("i", "j", "k")
       val plan = bucketedDataFrame.queryExecution.executedPlan
-      val rdd = plan.find(_.isInstanceOf[DataSourceScanExec])
-      assert(rdd.isDefined, plan)
+      val fileScan = getFileScan(plan)
 
-      val emptyBuckets = rdd.get.execute().mapPartitionsWithIndex { case (index, iter) =>
+      val emptyBuckets = fileScan.execute().mapPartitionsWithIndex { case (index, iter) =>
         // return indexes of empty partitions
         if (iter.isEmpty) {
           Iterator(index)
@@ -762,10 +766,13 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
     withTable("bucketed_table") {
       df1.write.format("parquet").bucketBy(8, "i").saveAsTable("bucketed_table")
 
-      checkAnswer(spark.table("bucketed_table").select("j"), df1.select("j"))
+      val scanDF = spark.table("bucketed_table").select("j")
+      assert(!getFileScan(scanDF.queryExecution.executedPlan).bucketedScan)
+      checkAnswer(scanDF, df1.select("j"))
 
-      checkAnswer(spark.table("bucketed_table").groupBy("j").agg(max("k")),
-        df1.groupBy("j").agg(max("k")))
+      val aggDF = spark.table("bucketed_table").groupBy("j").agg(max("k"))
+      assert(!getFileScan(aggDF.queryExecution.executedPlan).bucketedScan)
+      checkAnswer(aggDF, df1.groupBy("j").agg(max("k")))
     }
   }
 

From 0512b3f427274c8bda249fba02cd16f5694a4ea5 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 17 Mar 2020 00:23:16 -0700
Subject: [PATCH 0293/1280] [SPARK-31134][SQL] optimize skew join after shuffle
 partitions are coalesced

### What changes were proposed in this pull request?

Run the `OptimizeSkewedJoin` rule after the `CoalesceShufflePartitions` rule.

### Why are the changes needed?

Remove duplicated coalescing code in `OptimizeSkewedJoin`.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

existing tests

Closes #27893 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 30d95356f1881c32eb39e51525d2bcb331fcf867)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      |   9 +-
 .../adaptive/CoalesceShufflePartitions.scala  |   2 -
 .../adaptive/OptimizeSkewedJoin.scala         | 272 +++++++++---------
 .../adaptive/ShufflePartitionsUtil.scala      |  18 +-
 .../ShufflePartitionsUtilSuite.scala          |   2 -
 5 files changed, 146 insertions(+), 157 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 68da06de2cb7d..b54a32fa988d1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -96,13 +96,10 @@ case class AdaptiveSparkPlanExec(
   // optimizations should be stage-independent.
   @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq(
     ReuseAdaptiveSubquery(conf, context.subqueryCache),
-    // Here the 'OptimizeSkewedJoin' rule should be executed
-    // before 'CoalesceShufflePartitions', as the skewed partition handled
-    // in 'OptimizeSkewedJoin' rule, should be omitted in 'CoalesceShufflePartitions'.
-    OptimizeSkewedJoin(conf),
     CoalesceShufflePartitions(context.session),
-    // The rule of 'OptimizeLocalShuffleReader' need to make use of the 'partitionStartIndices'
-    // in 'CoalesceShufflePartitions' rule. So it must be after 'CoalesceShufflePartitions' rule.
+    // The following two rules need to make use of 'CustomShuffleReaderExec.partitionSpecs'
+    // added by `CoalesceShufflePartitions`. So they must be executed after it.
+    OptimizeSkewedJoin(conf),
     OptimizeLocalShuffleReader(conf),
     ApplyColumnarRulesAndInsertTransitions(conf, context.session.sessionState.columnarRules),
     CollapseCodegenStages(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
index d2a7f6ab5426c..226d692f7b404 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -74,8 +74,6 @@ case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPl
           .getOrElse(session.sparkContext.defaultParallelism)
         val partitionSpecs = ShufflePartitionsUtil.coalescePartitions(
           validMetrics.toArray,
-          firstPartitionIndex = 0,
-          lastPartitionIndex = distinctNumPreShufflePartitions.head,
           advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
           minNumPartitions = minPartitionNum)
         // This transformation adds new nodes, so we must use `transformUp` here.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index db65af614e0b9..e02b9afbf2d4f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkContext, SparkEnv}
+import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
@@ -83,14 +83,14 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   /**
    * The goal of skew join optimization is to make the data distribution more even. The target size
    * to split skewed partitions is the average size of non-skewed partition, or the
-   * target post-shuffle partition size if avg size is smaller than it.
+   * advisory partition size if avg size is smaller than it.
    */
-  private def targetSize(stats: MapOutputStatistics, medianSize: Long): Long = {
-    val targetPostShuffleSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES)
-    val nonSkewSizes = stats.bytesByPartitionId.filterNot(isSkewed(_, medianSize))
+  private def targetSize(sizes: Seq[Long], medianSize: Long): Long = {
+    val advisorySize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES)
+    val nonSkewSizes = sizes.filterNot(isSkewed(_, medianSize))
     // It's impossible that all the partitions are skewed, as we use median size to define skew.
     assert(nonSkewSizes.nonEmpty)
-    math.max(targetPostShuffleSize, nonSkewSizes.sum / nonSkewSizes.length)
+    math.max(advisorySize, nonSkewSizes.sum / nonSkewSizes.length)
   }
 
   /**
@@ -102,21 +102,29 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   }
 
   /**
-   * Split the skewed partition based on the map size and the max split number.
+   * Splits the skewed partition based on the map size and the target partition size
+   * after split, and create a list of `PartialMapperPartitionSpec`. Returns None if can't split.
    */
-  private def getMapStartIndices(
-      stage: ShuffleQueryStageExec,
-      partitionId: Int,
-      targetSize: Long): Seq[Int] = {
-    val shuffleId = stage.shuffle.shuffleDependency.shuffleHandle.shuffleId
-    val mapPartitionSizes = getMapSizesForReduceId(shuffleId, partitionId)
-    ShufflePartitionsUtil.splitSizeListByTargetSize(mapPartitionSizes, targetSize)
-  }
-
-  private def getStatistics(stage: ShuffleQueryStageExec): MapOutputStatistics = {
-    assert(stage.resultOption.isDefined, "ShuffleQueryStageExec should" +
-      " already be ready when executing OptimizeSkewedPartitions rule")
-    stage.resultOption.get.asInstanceOf[MapOutputStatistics]
+  private def createSkewPartitionSpecs(
+      shuffleId: Int,
+      reducerId: Int,
+      targetSize: Long): Option[Seq[PartialReducerPartitionSpec]] = {
+    val mapPartitionSizes = getMapSizesForReduceId(shuffleId, reducerId)
+    val mapStartIndices = ShufflePartitionsUtil.splitSizeListByTargetSize(
+      mapPartitionSizes, targetSize)
+    if (mapStartIndices.length > 1) {
+      Some(mapStartIndices.indices.map { i =>
+        val startMapIndex = mapStartIndices(i)
+        val endMapIndex = if (i == mapStartIndices.length - 1) {
+          mapPartitionSizes.length
+        } else {
+          mapStartIndices(i + 1)
+        }
+        PartialReducerPartitionSpec(reducerId, startMapIndex, endMapIndex)
+      })
+    } else {
+      None
+    }
   }
 
   private def canSplitLeftSide(joinType: JoinType) = {
@@ -128,12 +136,9 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
     joinType == Inner || joinType == Cross || joinType == RightOuter
   }
 
-  private def getNumMappers(stage: ShuffleQueryStageExec): Int = {
-    stage.shuffle.shuffleDependency.rdd.partitions.length
-  }
-
-  private def getSizeInfo(medianSize: Long, maxSize: Long): String = {
-    s"median size: $medianSize, max size: ${maxSize}"
+  private def getSizeInfo(medianSize: Long, sizes: Seq[Long]): String = {
+    s"median size: $medianSize, max size: ${sizes.max}, min size: ${sizes.min}, avg size: " +
+      sizes.sum / sizes.length
   }
 
   /*
@@ -150,101 +155,90 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    */
   def optimizeSkewJoin(plan: SparkPlan): SparkPlan = plan.transformUp {
     case smj @ SortMergeJoinExec(_, _, joinType, _,
-        s1 @ SortExec(_, _, left: ShuffleQueryStageExec, _),
-        s2 @ SortExec(_, _, right: ShuffleQueryStageExec, _), _)
+        s1 @ SortExec(_, _, ShuffleStage(left: ShuffleStageInfo), _),
+        s2 @ SortExec(_, _, ShuffleStage(right: ShuffleStageInfo), _), _)
         if supportedJoinTypes.contains(joinType) =>
-      val leftStats = getStatistics(left)
-      val rightStats = getStatistics(right)
-      val numPartitions = leftStats.bytesByPartitionId.length
-
-      val leftMedSize = medianSize(leftStats)
-      val rightMedSize = medianSize(rightStats)
+      assert(left.partitionsWithSizes.length == right.partitionsWithSizes.length)
+      val numPartitions = left.partitionsWithSizes.length
+      // We use the median size of the original shuffle partitions to detect skewed partitions.
+      val leftMedSize = medianSize(left.mapStats)
+      val rightMedSize = medianSize(right.mapStats)
       logDebug(
         s"""
-          |Try to optimize skewed join.
-          |Left side partition size:
-          |${getSizeInfo(leftMedSize, leftStats.bytesByPartitionId.max)}
-          |Right side partition size:
-          |${getSizeInfo(rightMedSize, rightStats.bytesByPartitionId.max)}
+          |Optimizing skewed join.
+          |Left side partitions size info:
+          |${getSizeInfo(leftMedSize, left.mapStats.bytesByPartitionId)}
+          |Right side partitions size info:
+          |${getSizeInfo(rightMedSize, right.mapStats.bytesByPartitionId)}
         """.stripMargin)
       val canSplitLeft = canSplitLeftSide(joinType)
       val canSplitRight = canSplitRightSide(joinType)
-      val leftTargetSize = targetSize(leftStats, leftMedSize)
-      val rightTargetSize = targetSize(rightStats, rightMedSize)
+      // We use the actual partition sizes (may be coalesced) to calculate target size, so that
+      // the final data distribution is even (coalesced partitions + split partitions).
+      val leftActualSizes = left.partitionsWithSizes.map(_._2)
+      val rightActualSizes = right.partitionsWithSizes.map(_._2)
+      val leftTargetSize = targetSize(leftActualSizes, leftMedSize)
+      val rightTargetSize = targetSize(rightActualSizes, rightMedSize)
 
       val leftSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
       val rightSidePartitions = mutable.ArrayBuffer.empty[ShufflePartitionSpec]
-      // This is used to delay the creation of non-skew partitions so that we can potentially
-      // coalesce them like `CoalesceShufflePartitions` does.
-      val nonSkewPartitionIndices = mutable.ArrayBuffer.empty[Int]
       val leftSkewDesc = new SkewDesc
       val rightSkewDesc = new SkewDesc
       for (partitionIndex <- 0 until numPartitions) {
-        val leftSize = leftStats.bytesByPartitionId(partitionIndex)
-        val isLeftSkew = isSkewed(leftSize, leftMedSize) && canSplitLeft
-        val rightSize = rightStats.bytesByPartitionId(partitionIndex)
-        val isRightSkew = isSkewed(rightSize, rightMedSize) && canSplitRight
-        if (isLeftSkew || isRightSkew) {
-          if (nonSkewPartitionIndices.nonEmpty) {
-            // As soon as we see a skew, we'll "flush" out unhandled non-skew partitions.
-            createNonSkewPartitions(leftStats, rightStats, nonSkewPartitionIndices).foreach { p =>
-              leftSidePartitions += p
-              rightSidePartitions += p
-            }
-            nonSkewPartitionIndices.clear()
-          }
-
-          val leftParts = if (isLeftSkew) {
-            val mapStartIndices = getMapStartIndices(left, partitionIndex, leftTargetSize)
-            if (mapStartIndices.length > 1) {
-              leftSkewDesc.addPartitionSize(leftSize)
-              createSkewPartitions(partitionIndex, mapStartIndices, getNumMappers(left))
-            } else {
-              Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
-            }
-          } else {
-            Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
-          }
-
-          val rightParts = if (isRightSkew) {
-            val mapStartIndices = getMapStartIndices(right, partitionIndex, rightTargetSize)
-            if (mapStartIndices.length > 1) {
-              rightSkewDesc.addPartitionSize(rightSize)
-              createSkewPartitions(partitionIndex, mapStartIndices, getNumMappers(right))
-            } else {
-              Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
-            }
-          } else {
-            Seq(CoalescedPartitionSpec(partitionIndex, partitionIndex + 1))
+        val isLeftSkew = isSkewed(leftActualSizes(partitionIndex), leftMedSize) && canSplitLeft
+        val leftPartSpec = left.partitionsWithSizes(partitionIndex)._1
+        val isLeftCoalesced = leftPartSpec.startReducerIndex + 1 < leftPartSpec.endReducerIndex
+
+        val isRightSkew = isSkewed(rightActualSizes(partitionIndex), rightMedSize) && canSplitRight
+        val rightPartSpec = right.partitionsWithSizes(partitionIndex)._1
+        val isRightCoalesced = rightPartSpec.startReducerIndex + 1 < rightPartSpec.endReducerIndex
+
+        // A skewed partition should never be coalesced, but skip it here just to be safe.
+        val leftParts = if (isLeftSkew && !isLeftCoalesced) {
+          val reducerId = leftPartSpec.startReducerIndex
+          val skewSpecs = createSkewPartitionSpecs(
+            left.shuffleStage.shuffle.shuffleDependency.shuffleId, reducerId, leftTargetSize)
+          if (skewSpecs.isDefined) {
+            logDebug(s"Left side partition $partitionIndex is skewed, split it into " +
+              s"${skewSpecs.get.length} parts.")
+            leftSkewDesc.addPartitionSize(leftActualSizes(partitionIndex))
           }
+          skewSpecs.getOrElse(Seq(leftPartSpec))
+        } else {
+          Seq(leftPartSpec)
+        }
 
-          for {
-            leftSidePartition <- leftParts
-            rightSidePartition <- rightParts
-          } {
-            leftSidePartitions += leftSidePartition
-            rightSidePartitions += rightSidePartition
+        // A skewed partition should never be coalesced, but skip it here just to be safe.
+        val rightParts = if (isRightSkew && !isRightCoalesced) {
+          val reducerId = rightPartSpec.startReducerIndex
+          val skewSpecs = createSkewPartitionSpecs(
+            right.shuffleStage.shuffle.shuffleDependency.shuffleId, reducerId, rightTargetSize)
+          if (skewSpecs.isDefined) {
+            logDebug(s"Right side partition $partitionIndex is skewed, split it into " +
+              s"${skewSpecs.get.length} parts.")
+            rightSkewDesc.addPartitionSize(rightActualSizes(partitionIndex))
           }
+          skewSpecs.getOrElse(Seq(rightPartSpec))
         } else {
-          // Add to `nonSkewPartitionIndices` first, and add real partitions later, in case we can
-          // coalesce the non-skew partitions.
-          nonSkewPartitionIndices += partitionIndex
-          // If this is the last partition, add real partition immediately.
-          if (partitionIndex == numPartitions - 1) {
-            createNonSkewPartitions(leftStats, rightStats, nonSkewPartitionIndices).foreach { p =>
-              leftSidePartitions += p
-              rightSidePartitions += p
-            }
-            nonSkewPartitionIndices.clear()
-          }
+          Seq(rightPartSpec)
+        }
+
+        for {
+          leftSidePartition <- leftParts
+          rightSidePartition <- rightParts
+        } {
+          leftSidePartitions += leftSidePartition
+          rightSidePartitions += rightSidePartition
         }
       }
 
       logDebug("number of skewed partitions: " +
         s"left ${leftSkewDesc.numPartitions}, right ${rightSkewDesc.numPartitions}")
       if (leftSkewDesc.numPartitions > 0 || rightSkewDesc.numPartitions > 0) {
-        val newLeft = CustomShuffleReaderExec(left, leftSidePartitions, leftSkewDesc.toString)
-        val newRight = CustomShuffleReaderExec(right, rightSidePartitions, rightSkewDesc.toString)
+        val newLeft = CustomShuffleReaderExec(
+          left.shuffleStage, leftSidePartitions, leftSkewDesc.toString)
+        val newRight = CustomShuffleReaderExec(
+          right.shuffleStage, rightSidePartitions, rightSkewDesc.toString)
         smj.copy(
           left = s1.copy(child = newLeft), right = s2.copy(child = newRight), isSkewJoin = true)
       } else {
@@ -252,44 +246,6 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       }
   }
 
-  private def createNonSkewPartitions(
-      leftStats: MapOutputStatistics,
-      rightStats: MapOutputStatistics,
-      nonSkewPartitionIndices: Seq[Int]): Seq[ShufflePartitionSpec] = {
-    assert(nonSkewPartitionIndices.nonEmpty)
-    val shouldCoalesce = conf.getConf(SQLConf.COALESCE_PARTITIONS_ENABLED)
-    if (!shouldCoalesce || nonSkewPartitionIndices.length == 1) {
-      nonSkewPartitionIndices.map(i => CoalescedPartitionSpec(i, i + 1))
-    } else {
-      // We fall back to Spark default parallelism if the minimum number of coalesced partitions
-      // is not set, so to avoid perf regressions compared to no coalescing.
-      val minPartitionNum = conf.getConf(SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM)
-        .getOrElse(SparkContext.getActive.get.defaultParallelism)
-      ShufflePartitionsUtil.coalescePartitions(
-        Array(leftStats, rightStats),
-        firstPartitionIndex = nonSkewPartitionIndices.head,
-        // `lastPartitionIndex` is exclusive.
-        lastPartitionIndex = nonSkewPartitionIndices.last + 1,
-        advisoryTargetSize = conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES),
-        minNumPartitions = minPartitionNum)
-    }
-  }
-
-  private def createSkewPartitions(
-      reducerIndex: Int,
-      mapStartIndices: Seq[Int],
-      numMappers: Int): Seq[PartialReducerPartitionSpec] = {
-    mapStartIndices.indices.map { i =>
-      val startMapIndex = mapStartIndices(i)
-      val endMapIndex = if (i == mapStartIndices.length - 1) {
-        numMappers
-      } else {
-        mapStartIndices(i + 1)
-      }
-      PartialReducerPartitionSpec(reducerIndex, startMapIndex, endMapIndex)
-    }
-  }
-
   override def apply(plan: SparkPlan): SparkPlan = {
     if (!conf.getConf(SQLConf.SKEW_JOIN_ENABLED)) {
       return plan
@@ -328,6 +284,48 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
   }
 }
 
+private object ShuffleStage {
+  def unapply(plan: SparkPlan): Option[ShuffleStageInfo] = plan match {
+    case s: ShuffleQueryStageExec =>
+      val mapStats = getMapStats(s)
+      val sizes = mapStats.bytesByPartitionId
+      val partitions = sizes.zipWithIndex.map {
+        case (size, i) => CoalescedPartitionSpec(i, i + 1) -> size
+      }
+      Some(ShuffleStageInfo(s, mapStats, partitions))
+
+    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs, _) =>
+      val mapStats = getMapStats(s)
+      val sizes = mapStats.bytesByPartitionId
+      val partitions = partitionSpecs.map {
+        case spec @ CoalescedPartitionSpec(start, end) =>
+          var sum = 0L
+          var i = start
+          while (i < end) {
+            sum += sizes(i)
+            i += 1
+          }
+          spec -> sum
+        case other => throw new IllegalArgumentException(
+          s"Expect CoalescedPartitionSpec but got $other")
+      }
+      Some(ShuffleStageInfo(s, mapStats, partitions))
+
+    case _ => None
+  }
+
+  private def getMapStats(stage: ShuffleQueryStageExec): MapOutputStatistics = {
+    assert(stage.resultOption.isDefined, "ShuffleQueryStageExec should" +
+      " already be ready when executing OptimizeSkewedPartitions rule")
+    stage.resultOption.get.asInstanceOf[MapOutputStatistics]
+  }
+}
+
+private case class ShuffleStageInfo(
+    shuffleStage: ShuffleQueryStageExec,
+    mapStats: MapOutputStatistics,
+    partitionsWithSizes: Seq[(CoalescedPartitionSpec, Long)])
+
 private class SkewDesc {
   private[this] var numSkewedPartitions: Int = 0
   private[this] var totalSize: Long = 0
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
index 292df11c23ca1..208cc059d238c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
@@ -28,10 +28,9 @@ object ShufflePartitionsUtil extends Logging {
   final val MERGED_PARTITION_FACTOR = 1.2
 
   /**
-   * Coalesce the same range of partitions (`firstPartitionIndex` to `lastPartitionIndex`, the
-   * start is inclusive and the end is exclusive) from multiple shuffles. This method assumes that
-   * all the shuffles have the same number of partitions, and the partitions of same index will be
-   * read together by one task.
+   * Coalesce the partitions from multiple shuffles. This method assumes that all the shuffles
+   * have the same number of partitions, and the partitions of same index will be read together
+   * by one task.
    *
    * The strategy used to determine the number of coalesced partitions is described as follows.
    * To determine the number of coalesced partitions, we have a target size for a coalesced
@@ -56,8 +55,6 @@ object ShufflePartitionsUtil extends Logging {
    */
   def coalescePartitions(
       mapOutputStatistics: Array[MapOutputStatistics],
-      firstPartitionIndex: Int,
-      lastPartitionIndex: Int,
       advisoryTargetSize: Long,
       minNumPartitions: Int): Seq[ShufflePartitionSpec] = {
     // If `minNumPartitions` is very large, it is possible that we need to use a value less than
@@ -87,11 +84,12 @@ object ShufflePartitionsUtil extends Logging {
       "There should be only one distinct value of the number of shuffle partitions " +
         "among registered Exchange operators.")
 
+    val numPartitions = distinctNumShufflePartitions.head
     val partitionSpecs = ArrayBuffer[CoalescedPartitionSpec]()
-    var latestSplitPoint = firstPartitionIndex
+    var latestSplitPoint = 0
     var coalescedSize = 0L
-    var i = firstPartitionIndex
-    while (i < lastPartitionIndex) {
+    var i = 0
+    while (i < numPartitions) {
       // We calculate the total size of i-th shuffle partitions from all shuffles.
       var totalSizeOfCurrentPartition = 0L
       var j = 0
@@ -112,7 +110,7 @@ object ShufflePartitionsUtil extends Logging {
       }
       i += 1
     }
-    partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, lastPartitionIndex)
+    partitionSpecs += CoalescedPartitionSpec(latestSplitPoint, numPartitions)
 
     partitionSpecs
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
index 2cd4c98716c28..7acc33c43b19d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ShufflePartitionsUtilSuite.scala
@@ -33,8 +33,6 @@ class ShufflePartitionsUtilSuite extends SparkFunSuite {
     }
     val estimatedPartitionStartIndices = ShufflePartitionsUtil.coalescePartitions(
       mapOutputStatistics,
-      0,
-      bytesByPartitionIdArray.head.length,
       targetSize,
       minNumPartitions)
     assert(estimatedPartitionStartIndices === expectedPartitionStartIndices)

From 26ea213f3c4f2acb07045bf0f6b476ddfb635436 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 17 Mar 2020 00:52:45 -0700
Subject: [PATCH 0294/1280] [MINOR][SQL] Update the DataFrameWriter.bucketBy
 comment

### What changes were proposed in this pull request?

This PR intends to update the `DataFrameWriter.bucketBy` comment for clearly describing that the bucketBy scheme follows a Spark "specific" one.

I saw the questions about the current bucketing compatibility with Hive in [SPARK-31162](https://issues.apache.org/jira/browse/SPARK-31162?focusedCommentId=17060408&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-17060408) and [SPARK-17495](https://issues.apache.org/jira/browse/SPARK-17495?focusedCommentId=17059847&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-17059847) from users and IMHO the comment is a bit confusing to users about the compatibility

### Why are the changes needed?

To make users understood smoothly.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A

Closes #27930 from maropu/UpdateBucketByComment.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 124b4ce2e6e8f84294f8fc13d3e731a82325dacb)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/DataFrameWriter.scala  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 22b26ca8164cf..6946c1f5bc895 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -198,7 +198,8 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
 
   /**
    * Buckets the output by the given columns. If specified, the output is laid out on the file
-   * system similar to Hive's bucketing scheme.
+   * system similar to Hive's bucketing scheme, but with a different bucket hash function
+   * and is not compatible with Hive's bucketing.
    *
    * This is applicable for all file-based data sources (e.g. Parquet, JSON) starting with Spark
    * 2.1.0.

From 41c1dbd2fa77816582351bb129a6a7d343f56cc2 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 17 Mar 2020 21:53:46 +0800
Subject: [PATCH 0295/1280] [SPARK-31150][SQL] Parsing seconds fraction with
 variable length for timestamp

### What changes were proposed in this pull request?
This PR is to support parsing timestamp values with variable length second fraction parts.

e.g. 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]' can parse timestamp with 0~6 digit-length second fraction but fail >=7
```sql
select to_timestamp(v, 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]') from values
 ('2019-10-06 10:11:12.'),
 ('2019-10-06 10:11:12.0'),
 ('2019-10-06 10:11:12.1'),
 ('2019-10-06 10:11:12.12'),
 ('2019-10-06 10:11:12.123UTC'),
 ('2019-10-06 10:11:12.1234'),
 ('2019-10-06 10:11:12.12345CST'),
 ('2019-10-06 10:11:12.123456PST') t(v)
2019-10-06 03:11:12.123
2019-10-06 08:11:12.12345
2019-10-06 10:11:12
2019-10-06 10:11:12
2019-10-06 10:11:12.1
2019-10-06 10:11:12.12
2019-10-06 10:11:12.1234
2019-10-06 10:11:12.123456

select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
NULL
```
Since 3.0, we use java 8 time API to parse and format timestamp values. when we create the `DateTimeFormatter`, we use `appendPattern` to create the build first, where the 'S..S' part will be parsed to a fixed-length(= `'S..S'.length`). This fits the formatting part but too strict for the parsing part because the trailing zeros are very likely to be truncated.

### Why are the changes needed?

improve timestamp parsing and more compatible with 2.4.x

### Does this PR introduce any user-facing change?

no, the related changes are newly added
### How was this patch tested?

add uts

Closes #27906 from yaooqinn/SPARK-31150.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 0946a9514f56565c78b0555383c1ece14aaf2b7b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/catalog/interface.scala      |  11 +-
 .../sql/catalyst/csv/CSVInferSchema.scala     |   3 +-
 .../sql/catalyst/csv/UnivocityGenerator.scala |   3 +-
 .../sql/catalyst/csv/UnivocityParser.scala    |   3 +-
 .../expressions/datetimeExpressions.scala     |  44 +++-
 .../sql/catalyst/json/JacksonGenerator.scala  |   3 +-
 .../sql/catalyst/json/JacksonParser.scala     |   3 +-
 .../sql/catalyst/json/JsonInferSchema.scala   |   3 +-
 .../util/DateTimeFormatterHelper.scala        |  50 +++-
 .../catalyst/util/TimestampFormatter.scala    |  37 ++-
 .../sql/util/TimestampFormatterSuite.scala    |  24 +-
 .../DateTimeBenchmark-jdk11-results.txt       | 218 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 218 +++++++++---------
 .../datasources/PartitioningUtils.scala       |   5 +-
 .../resources/sql-tests/inputs/datetime.sql   |  25 ++
 .../sql-tests/results/datetime.sql.out        | 170 +++++++++++++-
 .../benchmark/DateTimeBenchmark.scala         |   2 +-
 .../ParquetPartitionDiscoverySuite.scala      |   3 +-
 18 files changed, 555 insertions(+), 270 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 6e965ef807d1c..d02776b5d86f8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -521,8 +521,11 @@ object CatalogColumnStat extends Logging {
 
   val VERSION = 2
 
-  private def getTimestampFormatter(): TimestampFormatter = {
-    TimestampFormatter(format = "yyyy-MM-dd HH:mm:ss.SSSSSS", zoneId = ZoneOffset.UTC)
+  private def getTimestampFormatter(isParsing: Boolean): TimestampFormatter = {
+    TimestampFormatter(
+      format = "yyyy-MM-dd HH:mm:ss.SSSSSS",
+      zoneId = ZoneOffset.UTC,
+      needVarLengthSecondFraction = isParsing)
   }
 
   /**
@@ -535,7 +538,7 @@ object CatalogColumnStat extends Logging {
       case DateType => DateFormatter(ZoneOffset.UTC).parse(s)
       case TimestampType if version == 1 =>
         DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf(s))
-      case TimestampType => getTimestampFormatter().parse(s)
+      case TimestampType => getTimestampFormatter(isParsing = true).parse(s)
       case ByteType => s.toByte
       case ShortType => s.toShort
       case IntegerType => s.toInt
@@ -558,7 +561,7 @@ object CatalogColumnStat extends Logging {
   def toExternalString(v: Any, colName: String, dataType: DataType): String = {
     val externalValue = dataType match {
       case DateType => DateFormatter(ZoneOffset.UTC).format(v.asInstanceOf[Int])
-      case TimestampType => getTimestampFormatter().format(v.asInstanceOf[Long])
+      case TimestampType => getTimestampFormatter(isParsing = false).format(v.asInstanceOf[Long])
       case BooleanType | _: IntegralType | FloatType | DoubleType => v
       case _: DecimalType => v.asInstanceOf[Decimal].toJavaBigDecimal
       // This version of Spark does not use min/max for binary/string types so we ignore it.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index c6a03183ab45e..a7c243537acb7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -34,7 +34,8 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
     options.timestampFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    needVarLengthSecondFraction = true)
 
   private val decimalParser = if (options.locale == Locale.US) {
     // Special handling the default locale for backward compatibility
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
index 00e3d49787db1..4990da2bf3797 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
@@ -46,7 +46,8 @@ class UnivocityGenerator(
     options.timestampFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    needVarLengthSecondFraction = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index dd8537b02935e..2c5a9d752f55f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -89,7 +89,8 @@ class UnivocityParser(
     options.timestampFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    needVarLengthSecondFraction = true)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 0b1b65fa19300..d71db0d7a791b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -626,14 +626,22 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
   @transient private lazy val formatter: Option[TimestampFormatter] = {
     if (right.foldable) {
       Option(right.eval()).map { format =>
-        TimestampFormatter(format.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+        TimestampFormatter(
+          format.toString,
+          zoneId,
+          legacyFormat = SIMPLE_DATE_FORMAT,
+          needVarLengthSecondFraction = false)
       }
     } else None
   }
 
   override protected def nullSafeEval(timestamp: Any, format: Any): Any = {
     val tf = if (formatter.isEmpty) {
-      TimestampFormatter(format.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+      TimestampFormatter(
+        format.toString,
+        zoneId,
+        legacyFormat = SIMPLE_DATE_FORMAT,
+        needVarLengthSecondFraction = false)
     } else {
       formatter.get
     }
@@ -654,7 +662,8 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
         s"""|UTF8String.fromString($tf$$.MODULE$$.apply(
             |  $format.toString(),
             |  $zid,
-            |  $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT())
+            |  $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+            |  false)
             |.format($timestamp))""".stripMargin
       })
     }
@@ -767,7 +776,11 @@ abstract class ToTimestamp
   private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
   private lazy val formatter: TimestampFormatter =
     try {
-      TimestampFormatter(constFormat.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+      TimestampFormatter(
+        constFormat.toString,
+        zoneId,
+        legacyFormat = SIMPLE_DATE_FORMAT,
+        needVarLengthSecondFraction = true)
     } catch {
       case NonFatal(_) => null
     }
@@ -801,7 +814,11 @@ abstract class ToTimestamp
           } else {
             val formatString = f.asInstanceOf[UTF8String].toString
             try {
-              TimestampFormatter(formatString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+              TimestampFormatter(
+                formatString,
+                zoneId,
+                legacyFormat = SIMPLE_DATE_FORMAT,
+                needVarLengthSecondFraction = true)
                 .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
               case e: SparkUpgradeException => throw e
@@ -850,7 +867,8 @@ abstract class ToTimestamp
               ${ev.value} = $tf$$.MODULE$$.apply(
                 $format.toString(),
                 $zid,
-                $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT())
+                $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(),
+                true)
               .parse($string.toString()) / $downScaleFactor;
             } catch (java.lang.IllegalArgumentException e) {
               ${ev.isNull} = true;
@@ -936,7 +954,11 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
   private lazy val constFormat: UTF8String = right.eval().asInstanceOf[UTF8String]
   private lazy val formatter: TimestampFormatter =
     try {
-      TimestampFormatter(constFormat.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+      TimestampFormatter(
+        constFormat.toString,
+        zoneId,
+        legacyFormat = SIMPLE_DATE_FORMAT,
+        needVarLengthSecondFraction = false)
     } catch {
       case NonFatal(_) => null
     }
@@ -963,7 +985,11 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
         } else {
           try {
             UTF8String.fromString(
-              TimestampFormatter(f.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT)
+              TimestampFormatter(
+                f.toString,
+                zoneId,
+                legacyFormat = SIMPLE_DATE_FORMAT,
+                needVarLengthSecondFraction = false)
                 .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
             case NonFatal(_) => null
@@ -1001,7 +1027,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
         s"""
         try {
           ${ev.value} = UTF8String.fromString(
-            $tf$$.MODULE$$.apply($f.toString(), $zid, $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT())
+            $tf$$.MODULE$$.apply($f.toString(), $zid, $ldf$$.MODULE$$.SIMPLE_DATE_FORMAT(), false)
               .format($seconds * 1000000L));
         } catch (java.lang.IllegalArgumentException e) {
           ${ev.isNull} = true;
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index 141360ff02117..f3938feef0a35 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -82,7 +82,8 @@ private[sql] class JacksonGenerator(
     options.timestampFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    needVarLengthSecondFraction = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index d0db06cae8161..8965a81feee19 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -60,7 +60,8 @@ class JacksonParser(
     options.timestampFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    needVarLengthSecondFraction = true)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 3dd8694eae418..56b12784fd214 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -42,7 +42,8 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
     options.timestampFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    needVarLengthSecondFraction = true)
 
   /**
    * Infer the type of a collection of json records in three stages:
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index a59f49bbe6b42..668ce9239b699 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -51,12 +51,16 @@ trait DateTimeFormatterHelper {
   // In this way, synchronised is intentionally omitted in this method to make parallel calls
   // less synchronised.
   // The Cache.get method is not used here to avoid creation of additional instances of Callable.
-  protected def getOrCreateFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
+  protected def getOrCreateFormatter(
+      pattern: String,
+      locale: Locale,
+      needVarLengthSecondFraction: Boolean = false): DateTimeFormatter = {
     val newPattern = DateTimeUtils.convertIncompatiblePattern(pattern)
-    val key = (newPattern, locale)
+    val useVarLen = needVarLengthSecondFraction && newPattern.contains('S')
+    val key = (newPattern, locale, useVarLen)
     var formatter = cache.getIfPresent(key)
     if (formatter == null) {
-      formatter = buildFormatter(newPattern, locale)
+      formatter = buildFormatter(newPattern, locale, useVarLen)
       cache.put(key, formatter)
     }
     formatter
@@ -87,7 +91,9 @@ trait DateTimeFormatterHelper {
 private object DateTimeFormatterHelper {
   val cache = CacheBuilder.newBuilder()
     .maximumSize(128)
-    .build[(String, Locale), DateTimeFormatter]()
+    .build[(String, Locale, Boolean), DateTimeFormatter]()
+
+  final val extractor = "^([^S]*)(S*)(.*)$".r
 
   def createBuilder(): DateTimeFormatterBuilder = {
     new DateTimeFormatterBuilder().parseCaseInsensitive()
@@ -104,8 +110,40 @@ private object DateTimeFormatterHelper {
       .withResolverStyle(ResolverStyle.STRICT)
   }
 
-  def buildFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
-    val builder = createBuilder().appendPattern(pattern)
+  /**
+   * Building a formatter for parsing seconds fraction with variable length
+   */
+  def createBuilderWithVarLengthSecondFraction(
+      pattern: String): DateTimeFormatterBuilder = {
+    val builder = createBuilder()
+    pattern.split("'").zipWithIndex.foreach {
+      case (pattenPart, idx) if idx % 2 == 0 =>
+        var rest = pattenPart
+        while (rest.nonEmpty) {
+          rest match {
+            case extractor(prefix, secondFraction, suffix) =>
+              builder.appendPattern(prefix)
+              if (secondFraction.nonEmpty) {
+                builder.appendFraction(ChronoField.NANO_OF_SECOND, 1, secondFraction.length, false)
+              }
+              rest = suffix
+            case _ => throw new IllegalArgumentException(s"Unrecognized datetime pattern: $pattern")
+          }
+        }
+      case (patternPart, _) => builder.appendLiteral(patternPart)
+    }
+    builder
+  }
+
+  def buildFormatter(
+      pattern: String,
+      locale: Locale,
+      varLenEnabled: Boolean): DateTimeFormatter = {
+    val builder = if (varLenEnabled) {
+      createBuilderWithVarLengthSecondFraction(pattern)
+    } else {
+      createBuilder().appendPattern(pattern)
+    }
     toFormatter(builder, locale)
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 99139ca8cd7f6..0c4fa17cdcd20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util
 
 import java.text.{ParseException, ParsePosition, SimpleDateFormat}
 import java.time._
-import java.time.format.DateTimeParseException
+import java.time.format.{DateTimeFormatter, DateTimeParseException}
 import java.time.temporal.ChronoField.MICRO_OF_SECOND
 import java.time.temporal.TemporalQueries
 import java.util.{Calendar, GregorianCalendar, Locale, TimeZone}
@@ -55,10 +55,12 @@ class Iso8601TimestampFormatter(
     pattern: String,
     zoneId: ZoneId,
     locale: Locale,
-    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT)
+    legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+    needVarLengthSecondFraction: Boolean)
   extends TimestampFormatter with DateTimeFormatterHelper {
   @transient
-  protected lazy val formatter = getOrCreateFormatter(pattern, locale)
+  protected lazy val formatter: DateTimeFormatter =
+    getOrCreateFormatter(pattern, locale, needVarLengthSecondFraction)
 
   @transient
   protected lazy val legacyFormatter = TimestampFormatter.getLegacyFormatter(
@@ -95,7 +97,8 @@ class Iso8601TimestampFormatter(
  * @param zoneId the time zone identifier in which the formatter parses or format timestamps
  */
 class FractionTimestampFormatter(zoneId: ZoneId)
-  extends Iso8601TimestampFormatter("", zoneId, TimestampFormatter.defaultLocale) {
+  extends Iso8601TimestampFormatter(
+    "", zoneId, TimestampFormatter.defaultLocale, needVarLengthSecondFraction = false) {
 
   @transient
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
@@ -199,12 +202,14 @@ object TimestampFormatter {
       format: Option[String],
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
-      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): TimestampFormatter = {
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+      needVarLengthSecondFraction: Boolean = false): TimestampFormatter = {
     val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      new Iso8601TimestampFormatter(pattern, zoneId, locale, legacyFormat)
+      new Iso8601TimestampFormatter(
+        pattern, zoneId, locale, legacyFormat, needVarLengthSecondFraction)
     }
   }
 
@@ -227,16 +232,24 @@ object TimestampFormatter {
       format: String,
       zoneId: ZoneId,
       locale: Locale,
-      legacyFormat: LegacyDateFormat): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, locale, legacyFormat)
+      legacyFormat: LegacyDateFormat,
+      needVarLengthSecondFraction: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, needVarLengthSecondFraction)
   }
 
-  def apply(format: String, zoneId: ZoneId, legacyFormat: LegacyDateFormat): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat)
+  def apply(
+      format: String,
+      zoneId: ZoneId,
+      legacyFormat: LegacyDateFormat,
+      needVarLengthSecondFraction: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, needVarLengthSecondFraction)
   }
 
-  def apply(format: String, zoneId: ZoneId): TimestampFormatter = {
-    getFormatter(Some(format), zoneId)
+  def apply(
+      format: String,
+      zoneId: ZoneId,
+      needVarLengthSecondFraction: Boolean = false): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, needVarLengthSecondFraction = needVarLengthSecondFraction)
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 959ef8eeb1b25..3cf1437864558 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -44,7 +44,8 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
       val formatter = TimestampFormatter(
         "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId))
+        DateTimeUtils.getZoneId(zoneId),
+        needVarLengthSecondFraction = true)
       val microsSinceEpoch = formatter.parse(localDate)
       assert(microsSinceEpoch === expectedMicros(zoneId))
     }
@@ -83,9 +84,9 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         2177456523456789L,
         11858049903010203L).foreach { micros =>
         DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-          val formatter = TimestampFormatter(pattern, zoneId)
-          val timestamp = formatter.format(micros)
-          val parsed = formatter.parse(timestamp)
+          val timestamp = TimestampFormatter(pattern, zoneId).format(micros)
+          val parsed = TimestampFormatter(
+            pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
           assert(micros === parsed)
         }
       }
@@ -104,9 +105,10 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       "2039-01-01T01:02:03.456789",
       "2345-10-07T22:45:03.010203").foreach { timestamp =>
       DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-        val formatter = TimestampFormatter("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", zoneId)
-        val micros = formatter.parse(timestamp)
-        val formatted = formatter.format(micros)
+        val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
+        val micros = TimestampFormatter(
+          pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
+        val formatted = TimestampFormatter(pattern, zoneId).format(micros)
         assert(timestamp === formatted)
       }
     }
@@ -159,7 +161,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   test("parsing timestamp strings with various seconds fractions") {
     DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
       def check(pattern: String, input: String, reference: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId)
+        val formatter = TimestampFormatter(pattern, zoneId, needVarLengthSecondFraction = true)
         val expected = DateTimeUtils.stringToTimestamp(
           UTF8String.fromString(reference), zoneId).get
         val actual = formatter.parse(input)
@@ -184,7 +186,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
 
       try {
-        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId)
+        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId, true)
           .parse("2019/11/14 20#25#30.123456")
         fail("Expected to throw an exception for the invalid input")
       } catch {
@@ -235,7 +237,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("SPARK-30958: parse timestamp with negative year") {
-    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
+    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC, true)
     assert(formatter1.parse("-1234-02-22 02:22:22") === instantToMicros(
       LocalDateTime.of(-1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
 
@@ -249,7 +251,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     }
 
     // "yyyy" with "G" can't parse negative year or year 0000.
-    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
+    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC, true)
     assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
     assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 883f9de74c22f..3e415fa48f919 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,92 +6,92 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    221            232          16         45.3          22.1       1.0X
-cast to timestamp wholestage on                     213            256          71         46.9          21.3       1.0X
+cast to timestamp wholestage off                    217            225          10         46.0          21.7       1.0X
+cast to timestamp wholestage on                     195            224          35         51.2          19.5       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                    863            961         139         11.6          86.3       1.0X
-year of timestamp wholestage on                     783            821          26         12.8          78.3       1.1X
+year of timestamp wholestage off                    735            739           5         13.6          73.5       1.0X
+year of timestamp wholestage on                     728            742          10         13.7          72.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1008           1013           7          9.9         100.8       1.0X
-quarter of timestamp wholestage on                  926            963          36         10.8          92.6       1.1X
+quarter of timestamp wholestage off                 940            941           2         10.6          94.0       1.0X
+quarter of timestamp wholestage on                  910            930          12         11.0          91.0       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                   794            813          27         12.6          79.4       1.0X
-month of timestamp wholestage on                    737            758          23         13.6          73.7       1.1X
+month of timestamp wholestage off                   796            808          17         12.6          79.6       1.0X
+month of timestamp wholestage on                    688            706          11         14.5          68.8       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1063           1076          18          9.4         106.3       1.0X
-weekofyear of timestamp wholestage on              1070           1366         305          9.3         107.0       1.0X
+weekofyear of timestamp wholestage off             1013           1016           3          9.9         101.3       1.0X
+weekofyear of timestamp wholestage on               988           1011          18         10.1          98.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                     847            854           9         11.8          84.7       1.0X
-day of timestamp wholestage on                      799            836          37         12.5          79.9       1.1X
+day of timestamp wholestage off                     686            698          17         14.6          68.6       1.0X
+day of timestamp wholestage on                      693            708          13         14.4          69.3       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off               816            830          20         12.2          81.6       1.0X
-dayofyear of timestamp wholestage on                856            925          62         11.7          85.6       1.0X
+dayofyear of timestamp wholestage off               711            716           7         14.1          71.1       1.0X
+dayofyear of timestamp wholestage on                713            723          12         14.0          71.3       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1150           1159          12          8.7         115.0       1.0X
-dayofmonth of timestamp wholestage on               945           1165         308         10.6          94.5       1.2X
+dayofmonth of timestamp wholestage off              706            726          28         14.2          70.6       1.0X
+dayofmonth of timestamp wholestage on               698            714          11         14.3          69.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1062           1074          16          9.4         106.2       1.0X
-dayofweek of timestamp wholestage on                942           1029          74         10.6          94.2       1.1X
+dayofweek of timestamp wholestage off               797            798           2         12.6          79.7       1.0X
+dayofweek of timestamp wholestage on                794            807          18         12.6          79.4       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                 870            872           3         11.5          87.0       1.0X
-weekday of timestamp wholestage on                  821            855          21         12.2          82.1       1.1X
+weekday of timestamp wholestage off                 783            796          17         12.8          78.3       1.0X
+weekday of timestamp wholestage on                  759            778          21         13.2          75.9       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    578            582           5         17.3          57.8       1.0X
-hour of timestamp wholestage on                     533            544          14         18.8          53.3       1.1X
+hour of timestamp wholestage off                    573            575           2         17.5          57.3       1.0X
+hour of timestamp wholestage on                     512            522          16         19.5          51.2       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  611            613           3         16.4          61.1       1.0X
-minute of timestamp wholestage on                   551            557           8         18.2          55.1       1.1X
+minute of timestamp wholestage off                  550            555           7         18.2          55.0       1.0X
+minute of timestamp wholestage on                   507            527          21         19.7          50.7       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  705            716          16         14.2          70.5       1.0X
-second of timestamp wholestage on                   569            669          99         17.6          56.9       1.2X
+second of timestamp wholestage off                  562            587          35         17.8          56.2       1.0X
+second of timestamp wholestage on                   516            529          12         19.4          51.6       1.1X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         177            177           0         56.6          17.7       1.0X
-current_date wholestage on                          164            180          13         60.9          16.4       1.1X
+current_date wholestage off                         161            164           4         61.9          16.1       1.0X
+current_date wholestage on                          163            170           8         61.2          16.3       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    182            182           1         55.1          18.2       1.0X
-current_timestamp wholestage on                     168            194          53         59.4          16.8       1.1X
+current_timestamp wholestage off                    174            177           4         57.5          17.4       1.0X
+current_timestamp wholestage on                     159            173          12         62.8          15.9       1.1X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         615            632          24         16.3          61.5       1.0X
-cast to date wholestage on                          596            660          40         16.8          59.6       1.0X
+cast to date wholestage off                         672            688          23         14.9          67.2       1.0X
+cast to date wholestage on                          592            605          10         16.9          59.2       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                             717            728          16         13.9          71.7       1.0X
-last_day wholestage on                              764            834          42         13.1          76.4       0.9X
+last_day wholestage off                             806            809           4         12.4          80.6       1.0X
+last_day wholestage on                              700            713          14         14.3          70.0       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             677            677           0         14.8          67.7       1.0X
-next_day wholestage on                              645            796         140         15.5          64.5       1.1X
+next_day wholestage off                             639            683          63         15.7          63.9       1.0X
+next_day wholestage on                              628            640          15         15.9          62.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             637            663          37         15.7          63.7       1.0X
-date_add wholestage on                              659            675          16         15.2          65.9       1.0X
+date_add wholestage off                             688            695          10         14.5          68.8       1.0X
+date_add wholestage on                              588            603          13         17.0          58.8       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             630            682          73         15.9          63.0       1.0X
-date_sub wholestage on                              720            762          37         13.9          72.0       0.9X
+date_sub wholestage off                             588            600          18         17.0          58.8       1.0X
+date_sub wholestage on                              597            624          40         16.8          59.7       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1064           1088          35          9.4         106.4       1.0X
-add_months wholestage on                            883            909          25         11.3          88.3       1.2X
+add_months wholestage off                           802            816          20         12.5          80.2       1.0X
+add_months wholestage on                            777            836          61         12.9          77.7       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         3667           3854         265          2.7         366.7       1.0X
-format date wholestage on                          3355           3548         195          3.0         335.5       1.1X
+format date wholestage off                         3328           3635         435          3.0         332.8       1.0X
+format date wholestage on                          3335           3549         253          3.0         333.5       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       4664           4704          57          2.1         466.4       1.0X
-from_unixtime wholestage on                        4596           4772         193          2.2         459.6       1.0X
+from_unixtime wholestage off                       4414           4548         189          2.3         441.4       1.0X
+from_unixtime wholestage on                        4455           4499          65          2.2         445.5       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   810            819          13         12.3          81.0       1.0X
-from_utc_timestamp wholestage on                    795            814          16         12.6          79.5       1.0X
+from_utc_timestamp wholestage off                   781            790          13         12.8          78.1       1.0X
+from_utc_timestamp wholestage on                    819            823           5         12.2          81.9       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1130           1146          22          8.8         113.0       1.0X
-to_utc_timestamp wholestage on                     1034           1063          40          9.7         103.4       1.1X
+to_utc_timestamp wholestage off                    1101           1108          10          9.1         110.1       1.0X
+to_utc_timestamp wholestage on                     1044           1055          10          9.6         104.4       1.1X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        192            194           2         52.1          19.2       1.0X
-cast interval wholestage on                         172            180           5         58.2          17.2       1.1X
+cast interval wholestage off                        198            202           6         50.6          19.8       1.0X
+cast interval wholestage on                         181            193          14         55.2          18.1       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1070           1082          17          9.3         107.0       1.0X
-datediff wholestage on                             1039           1059          13          9.6         103.9       1.0X
+datediff wholestage off                            1037           1043          10          9.6         103.7       1.0X
+datediff wholestage on                             1006           1037          21          9.9         100.6       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      3118           3137          27          3.2         311.8       1.0X
-months_between wholestage on                       3079           3122          48          3.2         307.9       1.0X
+months_between wholestage off                      2957           2972          21          3.4         295.7       1.0X
+months_between wholestage on                       2931           2952          16          3.4         293.1       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1147           1157          14          0.9        1146.6       1.0X
-window wholestage on                              16698          17304         715          0.1       16698.2       0.1X
+window wholestage off                              1130           1147          24          0.9        1130.0       1.0X
+window wholestage on                              16253          17068         580          0.1       16253.2       0.1X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1478           1485           9          6.8         147.8       1.0X
-date_trunc YEAR wholestage on                      1458           1587         213          6.9         145.8       1.0X
+date_trunc YEAR wholestage off                     1361           1380          27          7.3         136.1       1.0X
+date_trunc YEAR wholestage on                      1346           1381          28          7.4         134.6       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1467           1481          20          6.8         146.7       1.0X
-date_trunc YYYY wholestage on                      1457           1513          52          6.9         145.7       1.0X
+date_trunc YYYY wholestage off                     1352           1364          16          7.4         135.2       1.0X
+date_trunc YYYY wholestage on                      1391           1424          37          7.2         139.1       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1551           1556           7          6.4         155.1       1.0X
-date_trunc YY wholestage on                        1468           1538          78          6.8         146.8       1.1X
+date_trunc YY wholestage off                       1346           1356          15          7.4         134.6       1.0X
+date_trunc YY wholestage on                        1355           1366           9          7.4         135.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1481           1493          16          6.8         148.1       1.0X
-date_trunc MON wholestage on                       1476           1501          28          6.8         147.6       1.0X
+date_trunc MON wholestage off                      1355           1365          15          7.4         135.5       1.0X
+date_trunc MON wholestage on                       1388           1457          78          7.2         138.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1470           1477          10          6.8         147.0       1.0X
-date_trunc MONTH wholestage on                     1449           1610         260          6.9         144.9       1.0X
+date_trunc MONTH wholestage off                    1336           1357          30          7.5         133.6       1.0X
+date_trunc MONTH wholestage on                     1361           1390          22          7.3         136.1       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1487           1495          12          6.7         148.7       1.0X
-date_trunc MM wholestage on                        1467           1489          17          6.8         146.7       1.0X
+date_trunc MM wholestage off                       1350           1361          16          7.4         135.0       1.0X
+date_trunc MM wholestage on                        1370           1384          13          7.3         137.0       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1321           1328          10          7.6         132.1       1.0X
-date_trunc DAY wholestage on                       1279           1306          28          7.8         127.9       1.0X
+date_trunc DAY wholestage off                      1328           1361          46          7.5         132.8       1.0X
+date_trunc DAY wholestage on                       1356           1432          69          7.4         135.6       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1366           1368           3          7.3         136.6       1.0X
-date_trunc DD wholestage on                        1297           1314          18          7.7         129.7       1.1X
+date_trunc DD wholestage off                       1403           1445          59          7.1         140.3       1.0X
+date_trunc DD wholestage on                        1359           1405          49          7.4         135.9       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1338           1354          23          7.5         133.8       1.0X
-date_trunc HOUR wholestage on                      1403           1482          71          7.1         140.3       1.0X
+date_trunc HOUR wholestage off                     1447           1494          67          6.9         144.7       1.0X
+date_trunc HOUR wholestage on                      1387           1416          31          7.2         138.7       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    241            242           2         41.5          24.1       1.0X
-date_trunc MINUTE wholestage on                     196            204           6         51.0          19.6       1.2X
+date_trunc MINUTE wholestage off                    206            215          13         48.5          20.6       1.0X
+date_trunc MINUTE wholestage on                     233            240           4         42.9          23.3       0.9X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    211            222          15         47.3          21.1       1.0X
-date_trunc SECOND wholestage on                     205            243          64         48.8          20.5       1.0X
+date_trunc SECOND wholestage off                    200            213          18         49.9          20.0       1.0X
+date_trunc SECOND wholestage on                     219            227           7         45.6          21.9       0.9X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1492           1515          32          6.7         149.2       1.0X
-date_trunc WEEK wholestage on                      1465           1490          39          6.8         146.5       1.0X
+date_trunc WEEK wholestage off                     1287           1293           9          7.8         128.7       1.0X
+date_trunc WEEK wholestage on                      1310           1341          28          7.6         131.0       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1908           1982         104          5.2         190.8       1.0X
-date_trunc QUARTER wholestage on                   1879           2020          99          5.3         187.9       1.0X
+date_trunc QUARTER wholestage off                  1852           1880          39          5.4         185.2       1.0X
+date_trunc QUARTER wholestage on                   1857           1877          14          5.4         185.7       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           326            419         132         30.7          32.6       1.0X
-trunc year wholestage on                            187            199           8         53.4          18.7       1.7X
+trunc year wholestage off                           172            178           9         58.2          17.2       1.0X
+trunc year wholestage on                            213            222           8         47.0          21.3       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           194            194           0         51.6          19.4       1.0X
-trunc yyyy wholestage on                            192            207          10         52.1          19.2       1.0X
+trunc yyyy wholestage off                           175            182           9         57.1          17.5       1.0X
+trunc yyyy wholestage on                            224            233          12         44.7          22.4       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             177            180           4         56.5          17.7       1.0X
-trunc yy wholestage on                              185            193           9         53.9          18.5       1.0X
+trunc yy wholestage off                             167            180          18         59.7          16.7       1.0X
+trunc yy wholestage on                              212            226           9         47.3          21.2       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            208            210           3         48.0          20.8       1.0X
-trunc mon wholestage on                             195            204           8         51.3          19.5       1.1X
+trunc mon wholestage off                            175            175           1         57.2          17.5       1.0X
+trunc mon wholestage on                             219            241          17         45.7          21.9       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          182            185           4         55.1          18.2       1.0X
-trunc month wholestage on                           185            222          70         54.0          18.5       1.0X
+trunc month wholestage off                          176            176           0         56.8          17.6       1.0X
+trunc month wholestage on                           220            226           6         45.4          22.0       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             208            218          14         48.0          20.8       1.0X
-trunc mm wholestage on                              197            206          10         50.8          19.7       1.1X
+trunc mm wholestage off                             173            173           1         57.9          17.3       1.0X
+trunc mm wholestage on                              219            230           8         45.7          21.9       0.8X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     121            130          13          8.3         120.7       1.0X
-to timestamp str wholestage on                      128            134           6          7.8         128.4       0.9X
+to timestamp str wholestage off                      98            102           5         10.2          97.9       1.0X
+to timestamp str wholestage on                      102            112           5          9.8         102.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1201           1272         100          0.8        1200.8       1.0X
-to_timestamp wholestage on                          973           1019          53          1.0         972.7       1.2X
+to_timestamp wholestage off                        1373           1380          11          0.7        1373.0       1.0X
+to_timestamp wholestage on                          928           1100         103          1.1         928.1       1.5X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1002           1077         106          1.0        1001.8       1.0X
-to_unix_timestamp wholestage on                     988           1079         100          1.0         987.8       1.0X
+to_unix_timestamp wholestage off                   1416           1438          32          0.7        1416.0       1.0X
+to_unix_timestamp wholestage on                    1094           1261         194          0.9        1093.6       1.3X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          157            165          12          6.4         157.2       1.0X
-to date str wholestage on                           137            140           3          7.3         137.0       1.1X
+to date str wholestage off                          120            123           5          8.3         120.0       1.0X
+to date str wholestage on                           142            148           6          7.0         142.1       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             1597           1614          25          0.6        1597.0       1.0X
-to_date wholestage on                              1503           1573          58          0.7        1502.8       1.1X
+to_date wholestage off                             1661           1673          17          0.6        1661.0       1.0X
+to_date wholestage on                              1567           1617          55          0.6        1567.3       1.1X
 
 
 ================================================================================================
@@ -422,8 +422,8 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             197            202           6         25.4          39.4       1.0X
-Collect longs                                       794            911         110          6.3         158.7       0.2X
-Collect timestamps                                 1030           1079          51          4.9         205.9       0.2X
+From java.sql.Timestamp                             216            234          16         23.1          43.2       1.0X
+Collect longs                                       913           1204         266          5.5         182.6       0.2X
+Collect timestamps                                 1392           1515         124          3.6         278.3       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 2ea26c327afae..c77d076762d80 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,92 +6,92 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    230            240          13         43.5          23.0       1.0X
-cast to timestamp wholestage on                     194            208          20         51.4          19.4       1.2X
+cast to timestamp wholestage off                    243            256          18         41.2          24.3       1.0X
+cast to timestamp wholestage on                     193            206          19         51.9          19.3       1.3X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                    806            822          22         12.4          80.6       1.0X
-year of timestamp wholestage on                     748            761          13         13.4          74.8       1.1X
+year of timestamp wholestage off                    784            786           3         12.8          78.4       1.0X
+year of timestamp wholestage on                     738            758          15         13.5          73.8       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                 828            832           5         12.1          82.8       1.0X
-quarter of timestamp wholestage on                  821            858          45         12.2          82.1       1.0X
+quarter of timestamp wholestage off                 823            825           3         12.2          82.3       1.0X
+quarter of timestamp wholestage on                  791            804          12         12.6          79.1       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                   709            713           5         14.1          70.9       1.0X
-month of timestamp wholestage on                    714            722          10         14.0          71.4       1.0X
+month of timestamp wholestage off                   681            692          16         14.7          68.1       1.0X
+month of timestamp wholestage on                    683            703          14         14.6          68.3       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1217           1220           4          8.2         121.7       1.0X
-weekofyear of timestamp wholestage on              1019           1043          24          9.8         101.9       1.2X
+weekofyear of timestamp wholestage off             1015           1032          23          9.8         101.5       1.0X
+weekofyear of timestamp wholestage on              1016           1106          72          9.8         101.6       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                     706            712           9         14.2          70.6       1.0X
-day of timestamp wholestage on                      694            704           8         14.4          69.4       1.0X
+day of timestamp wholestage off                     683            689           9         14.6          68.3       1.0X
+day of timestamp wholestage on                      685            702          11         14.6          68.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off               717            728          16         14.0          71.7       1.0X
-dayofyear of timestamp wholestage on                724            736          10         13.8          72.4       1.0X
+dayofyear of timestamp wholestage off               713            716           4         14.0          71.3       1.0X
+dayofyear of timestamp wholestage on                704            713           9         14.2          70.4       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off              719            721           3         13.9          71.9       1.0X
-dayofmonth of timestamp wholestage on               706            714           8         14.2          70.6       1.0X
+dayofmonth of timestamp wholestage off              738            746          11         13.5          73.8       1.0X
+dayofmonth of timestamp wholestage on               684            704          16         14.6          68.4       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off               904            979         106         11.1          90.4       1.0X
-dayofweek of timestamp wholestage on                796            819          14         12.6          79.6       1.1X
+dayofweek of timestamp wholestage off               810            821          15         12.3          81.0       1.0X
+dayofweek of timestamp wholestage on                784            789           5         12.8          78.4       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                 813            819           9         12.3          81.3       1.0X
-weekday of timestamp wholestage on                  788            809          13         12.7          78.8       1.0X
+weekday of timestamp wholestage off                 767            777          13         13.0          76.7       1.0X
+weekday of timestamp wholestage on                  765            777          11         13.1          76.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    567            582          21         17.6          56.7       1.0X
-hour of timestamp wholestage on                     539            551          11         18.6          53.9       1.1X
+hour of timestamp wholestage off                    534            543          13         18.7          53.4       1.0X
+hour of timestamp wholestage on                     499            529          22         20.0          49.9       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  554            563          13         18.1          55.4       1.0X
-minute of timestamp wholestage on                   520            531           7         19.2          52.0       1.1X
+minute of timestamp wholestage off                  526            544          26         19.0          52.6       1.0X
+minute of timestamp wholestage on                   495            515          20         20.2          49.5       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  651            654           5         15.4          65.1       1.0X
-second of timestamp wholestage on                   535            546          12         18.7          53.5       1.2X
+second of timestamp wholestage off                  537            538           2         18.6          53.7       1.0X
+second of timestamp wholestage on                   508            512           5         19.7          50.8       1.1X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         172            175           4         58.0          17.2       1.0X
-current_date wholestage on                          174            177           3         57.5          17.4       1.0X
+current_date wholestage off                         164            169           6         60.8          16.4       1.0X
+current_date wholestage on                          165            171           4         60.4          16.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    184            185           0         54.3          18.4       1.0X
-current_timestamp wholestage on                     196            212          14         51.0          19.6       0.9X
+current_timestamp wholestage off                    161            168          10         62.0          16.1       1.0X
+current_timestamp wholestage on                     161            165           5         62.1          16.1       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         652            662          13         15.3          65.2       1.0X
-cast to date wholestage on                          645            690          29         15.5          64.5       1.0X
+cast to date wholestage off                         691            692           2         14.5          69.1       1.0X
+cast to date wholestage on                          572            586          16         17.5          57.2       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                             801            808          11         12.5          80.1       1.0X
-last_day wholestage on                              765            829          37         13.1          76.5       1.0X
+last_day wholestage off                             713            723          14         14.0          71.3       1.0X
+last_day wholestage on                              695            747          41         14.4          69.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             676            676           0         14.8          67.6       1.0X
-next_day wholestage on                              674            708          33         14.8          67.4       1.0X
+next_day wholestage off                             627            630           4         15.9          62.7       1.0X
+next_day wholestage on                              615            660          39         16.3          61.5       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             644            646           2         15.5          64.4       1.0X
-date_add wholestage on                              640            661          32         15.6          64.0       1.0X
+date_add wholestage off                             604            649          63         16.5          60.4       1.0X
+date_add wholestage on                              590            598           8         17.0          59.0       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             704            718          20         14.2          70.4       1.0X
-date_sub wholestage on                              684            728          34         14.6          68.4       1.0X
+date_sub wholestage off                             588            592           5         17.0          58.8       1.0X
+date_sub wholestage on                              597            620          39         16.7          59.7       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1011           1017           8          9.9         101.1       1.0X
-add_months wholestage on                            837            860          25         11.9          83.7       1.2X
+add_months wholestage off                           777            782           8         12.9          77.7       1.0X
+add_months wholestage on                            769            786          22         13.0          76.9       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         3467           3591         176          2.9         346.7       1.0X
-format date wholestage on                          3417           3482          66          2.9         341.7       1.0X
+format date wholestage off                         3335           3498         231          3.0         333.5       1.0X
+format date wholestage on                          3386           3488         103          3.0         338.6       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       4823           4850          38          2.1         482.3       1.0X
-from_unixtime wholestage on                        4774           4811          29          2.1         477.4       1.0X
+from_unixtime wholestage off                       5291           5601         438          1.9         529.1       1.0X
+from_unixtime wholestage on                        5684           5952         256          1.8         568.4       0.9X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   703            717          20         14.2          70.3       1.0X
-from_utc_timestamp wholestage on                    665            671           7         15.0          66.5       1.1X
+from_utc_timestamp wholestage off                   963            981          25         10.4          96.3       1.0X
+from_utc_timestamp wholestage on                    731            794          40         13.7          73.1       1.3X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1000           1016          22         10.0         100.0       1.0X
-to_utc_timestamp wholestage on                      917            933          11         10.9          91.7       1.1X
+to_utc_timestamp wholestage off                    1129           1135           7          8.9         112.9       1.0X
+to_utc_timestamp wholestage on                      921            973          68         10.9          92.1       1.2X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        195            198           3         51.2          19.5       1.0X
-cast interval wholestage on                         176            181           5         56.9          17.6       1.1X
+cast interval wholestage off                        197            197           0         50.8          19.7       1.0X
+cast interval wholestage on                         172            179           8         58.3          17.2       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1022           1025           5          9.8         102.2       1.0X
-datediff wholestage on                             1003           1015          10         10.0         100.3       1.0X
+datediff wholestage off                            1039           1105          93          9.6         103.9       1.0X
+datediff wholestage on                             1129           1162          22          8.9         112.9       0.9X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      2562           2579          24          3.9         256.2       1.0X
-months_between wholestage on                       2503           2524          17          4.0         250.3       1.0X
+months_between wholestage off                      2624           2654          42          3.8         262.4       1.0X
+months_between wholestage on                       2612           2709          57          3.8         261.2       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1067           1077          15          0.9        1066.6       1.0X
-window wholestage on                              14503          19089         NaN          0.1       14502.6       0.1X
+window wholestage off                              1229           1247          25          0.8        1229.4       1.0X
+window wholestage on                              14409          15033         534          0.1       14409.2       0.1X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1497           1543          64          6.7         149.7       1.0X
-date_trunc YEAR wholestage on                      1412           1533         138          7.1         141.2       1.1X
+date_trunc YEAR wholestage off                     1203           1218          22          8.3         120.3       1.0X
+date_trunc YEAR wholestage on                      1162           1172          10          8.6         116.2       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1725           1729           5          5.8         172.5       1.0X
-date_trunc YYYY wholestage on                      1413           1481          56          7.1         141.3       1.2X
+date_trunc YYYY wholestage off                     1185           1199          20          8.4         118.5       1.0X
+date_trunc YYYY wholestage on                      1148           1173          22          8.7         114.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1456           1459           5          6.9         145.6       1.0X
-date_trunc YY wholestage on                        1369           1380           7          7.3         136.9       1.1X
+date_trunc YY wholestage off                       1189           1192           4          8.4         118.9       1.0X
+date_trunc YY wholestage on                        1168           1179          16          8.6         116.8       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1459           1471          17          6.9         145.9       1.0X
-date_trunc MON wholestage on                       1359           1377          21          7.4         135.9       1.1X
+date_trunc MON wholestage off                      1308           1317          14          7.6         130.8       1.0X
+date_trunc MON wholestage on                       1303           1352          37          7.7         130.3       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1455           1480          35          6.9         145.5       1.0X
-date_trunc MONTH wholestage on                     1344           1468         170          7.4         134.4       1.1X
+date_trunc MONTH wholestage off                    1301           1320          27          7.7         130.1       1.0X
+date_trunc MONTH wholestage on                     1171           1226          41          8.5         117.1       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1437           1463          37          7.0         143.7       1.0X
-date_trunc MM wholestage on                        1273           1285           9          7.9         127.3       1.1X
+date_trunc MM wholestage off                       1237           1259          31          8.1         123.7       1.0X
+date_trunc MM wholestage on                        1181           1195          11          8.5         118.1       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1198           1209          16          8.3         119.8       1.0X
-date_trunc DAY wholestage on                       1137           1192          70          8.8         113.7       1.1X
+date_trunc DAY wholestage off                      1364           1370           8          7.3         136.4       1.0X
+date_trunc DAY wholestage on                       1189           1223          31          8.4         118.9       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1201           1215          20          8.3         120.1       1.0X
-date_trunc DD wholestage on                        1132           1144          14          8.8         113.2       1.1X
+date_trunc DD wholestage off                       1297           1328          45          7.7         129.7       1.0X
+date_trunc DD wholestage on                        1171           1242          43          8.5         117.1       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1236           1238           3          8.1         123.6       1.0X
-date_trunc HOUR wholestage on                      1134           1150          15          8.8         113.4       1.1X
+date_trunc HOUR wholestage off                     1261           1317          79          7.9         126.1       1.0X
+date_trunc HOUR wholestage on                      1171           1225          48          8.5         117.1       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    206            207           2         48.6          20.6       1.0X
-date_trunc MINUTE wholestage on                     185            186           2         54.0          18.5       1.1X
+date_trunc MINUTE wholestage off                    210            211           2         47.7          21.0       1.0X
+date_trunc MINUTE wholestage on                     178            200          18         56.2          17.8       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    209            210           2         47.9          20.9       1.0X
-date_trunc SECOND wholestage on                     174            185           7         57.4          17.4       1.2X
+date_trunc SECOND wholestage off                    206            211           7         48.5          20.6       1.0X
+date_trunc SECOND wholestage on                     186            200           9         53.9          18.6       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1231           1293          86          8.1         123.1       1.0X
-date_trunc WEEK wholestage on                      1186           1219          35          8.4         118.6       1.0X
+date_trunc WEEK wholestage off                     1275           1307          46          7.8         127.5       1.0X
+date_trunc WEEK wholestage on                      1190           1244          63          8.4         119.0       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1785           1792          10          5.6         178.5       1.0X
-date_trunc QUARTER wholestage on                   1671           1708          28          6.0         167.1       1.1X
+date_trunc QUARTER wholestage off                  1688           1719          44          5.9         168.8       1.0X
+date_trunc QUARTER wholestage on                   1732           1795          45          5.8         173.2       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           186            199          18         53.8          18.6       1.0X
-trunc year wholestage on                            161            165           5         62.3          16.1       1.2X
+trunc year wholestage off                           201            214          19         49.7          20.1       1.0X
+trunc year wholestage on                            150            157          11         66.5          15.0       1.3X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           184            184           0         54.4          18.4       1.0X
-trunc yyyy wholestage on                            156            162           6         64.3          15.6       1.2X
+trunc yyyy wholestage off                           169            174           6         59.0          16.9       1.0X
+trunc yyyy wholestage on                            159            174          10         62.9          15.9       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             171            174           5         58.6          17.1       1.0X
-trunc yy wholestage on                              161            172          10         62.1          16.1       1.1X
+trunc yy wholestage off                             171            179          12         58.5          17.1       1.0X
+trunc yy wholestage on                              170            180           9         58.8          17.0       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            175            176           1         57.1          17.5       1.0X
-trunc mon wholestage on                             166            167           1         60.3          16.6       1.1X
+trunc mon wholestage off                            174            177           4         57.5          17.4       1.0X
+trunc mon wholestage on                             162            171           7         61.7          16.2       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          177            186          12         56.4          17.7       1.0X
-trunc month wholestage on                           160            168           8         62.4          16.0       1.1X
+trunc month wholestage off                          195            196           2         51.2          19.5       1.0X
+trunc month wholestage on                           166            176           8         60.2          16.6       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             173            175           2         57.8          17.3       1.0X
-trunc mm wholestage on                              167            178          10         59.9          16.7       1.0X
+trunc mm wholestage off                             156            160           6         64.2          15.6       1.0X
+trunc mm wholestage on                              147            162          13         68.1          14.7       1.1X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     166            168           3          6.0         166.3       1.0X
-to timestamp str wholestage on                      170            192          37          5.9         169.8       1.0X
+to timestamp str wholestage off                     154            154           1          6.5         153.8       1.0X
+to timestamp str wholestage on                      127            132           6          7.9         126.5       1.2X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                         975           1042          95          1.0         975.1       1.0X
-to_timestamp wholestage on                          976            982           5          1.0         976.0       1.0X
+to_timestamp wholestage off                         973           1001          40          1.0         972.6       1.0X
+to_timestamp wholestage on                          912            922          12          1.1         912.3       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                    978            984           8          1.0         978.4       1.0X
-to_unix_timestamp wholestage on                     956            979          20          1.0         955.8       1.0X
+to_unix_timestamp wholestage off                    970            986          23          1.0         969.7       1.0X
+to_unix_timestamp wholestage on                     907            910           3          1.1         906.9       1.1X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          169            173           5          5.9         168.9       1.0X
-to date str wholestage on                           155            159           4          6.4         155.4       1.1X
+to date str wholestage off                          154            155           1          6.5         154.3       1.0X
+to date str wholestage on                           156            165           6          6.4         155.9       1.0X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             1583           1594          15          0.6        1583.2       1.0X
-to_date wholestage on                              1586           1635          52          0.6        1586.1       1.0X
+to_date wholestage off                             1757           1760           4          0.6        1756.6       1.0X
+to_date wholestage on                              1563           1626          41          0.6        1562.7       1.1X
 
 
 ================================================================================================
@@ -422,8 +422,8 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             174            176           3         28.8          34.7       1.0X
-Collect longs                                       945           1275         538          5.3         189.0       0.2X
-Collect timestamps                                 1019           1422         633          4.9         203.8       0.2X
+From java.sql.Timestamp                             182            186           4         27.5          36.4       1.0X
+Collect longs                                      1041           1652         964          4.8         208.2       0.2X
+Collect timestamps                                 1003           1029          31          5.0         200.7       0.2X
 
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index b0ec24e9b1a3d..292ac6db04baf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -131,7 +131,10 @@ object PartitioningUtils {
     }
 
     val dateFormatter = DateFormatter(zoneId)
-    val timestampFormatter = TimestampFormatter(timestampPartitionPattern, zoneId)
+    val timestampFormatter = TimestampFormatter(
+      timestampPartitionPattern,
+      zoneId,
+      needVarLengthSecondFraction = true)
     // First, we need to parse every partition's path and see if we can find partition values.
     val (partitionValues, optDiscoveredBasePaths) = paths.map { path =>
       parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes,
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index b14778b91510e..9026427480d36 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -75,3 +75,28 @@ select date '2001-09-28' - null;
 -- subtract dates
 select null - date '2019-10-06';
 select date '2001-10-01' - date '2001-09-28';
+
+-- variable-length tests
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+-- exceeded max variable length
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+-- special cases
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]');
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]');
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]');
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]');
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]');
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS");
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm");
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm");
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm");
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm");
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index a7b098d79a706..7258d009acfbe 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 47
+-- Number of queries: 68
 
 
 -- !query
@@ -391,3 +391,171 @@ select date '2001-10-01' - date '2001-09-28'
 struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
 -- !query output
 3 days
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 03:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 08:11:12.12345
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:00
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06S10:11:12.12345', 'yyyy-MM-dd\'S\'HH:mm:ss.SSSSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12345
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.12342019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+0019-10-06 10:11:12.1234
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 8de9b8d076e0f..d5ab1b9972f1d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -114,7 +114,7 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
     }
     runBenchmark("Parsing") {
       val n = 1000000
-      val timestampStrExpr = "concat('2019-01-27 11:02:01.', rpad(mod(id, 1000), 3, '0'))"
+      val timestampStrExpr = "concat('2019-01-27 11:02:01.', cast(mod(id, 1000) as string))"
       val pattern = "'yyyy-MM-dd HH:mm:ss.SSS'"
       run(n, "to timestamp str", timestampStrExpr)
       run(n, "to_timestamp", s"to_timestamp($timestampStrExpr, $pattern)")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
index e63929470ce5f..f4499fe17fdda 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
@@ -59,7 +59,8 @@ abstract class ParquetPartitionDiscoverySuite
 
   val timeZoneId = ZoneId.systemDefault()
   val df = DateFormatter(timeZoneId)
-  val tf = TimestampFormatter(timestampPartitionPattern, timeZoneId)
+  val tf = TimestampFormatter(
+    timestampPartitionPattern, timeZoneId, needVarLengthSecondFraction = true)
 
   protected override def beforeAll(): Unit = {
     super.beforeAll()

From 321341a4c3104380035350631c82a4b385f117e4 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 17 Mar 2020 23:03:18 +0800
Subject: [PATCH 0296/1280] [SPARK-31170][SQL] Spark SQL Cli should respect
 hive-site.xml and spark.sql.warehouse.dir

### What changes were proposed in this pull request?

In Spark CLI, we create a hive `CliSessionState` and it does not load the `hive-site.xml`. So the configurations in `hive-site.xml` will not take effects like other spark-hive integration apps.

Also, the warehouse directory is not correctly picked. If the `default` database does not exist, the `CliSessionState` will create one during the first time it talks to the metastore. The `Location` of the default DB will be neither the value of `spark.sql.warehousr.dir` nor the user-specified value of `hive.metastore.warehourse.dir`, but the default value of `hive.metastore.warehourse.dir `which will always be `/user/hive/warehouse`.

### Why are the changes needed?

fix bug for Spark SQL cli to pick right confs

### Does this PR introduce any user-facing change?

yes, the non-exists default database will be created in the location specified by the users via `spark.sql.warehouse.dir` or `hive.metastore.warehouse.dir`, or the default value of `spark.sql.warehouse.dir` if none of them specified.

### How was this patch tested?

add cli ut

Closes #27933 from yaooqinn/SPARK-31170.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5bc0d76591b46f0c1c9ec283ee8e1c5da76e67d6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/internal/SharedState.scala      | 80 +++++++++----------
 .../hive/thriftserver/SparkSQLCLIDriver.scala |  2 +
 .../sql/hive/thriftserver/CliSuite.scala      | 12 +++
 .../spark/sql/hive/HiveSharedStateSuite.scala |  1 -
 .../spark/sql/hive/HiveSparkSubmitSuite.scala |  2 +-
 5 files changed, 55 insertions(+), 42 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 5347264d7c50a..eb74e96514b2d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -41,7 +41,6 @@ import org.apache.spark.sql.streaming.ui.{StreamingQueryStatusListener, Streamin
 import org.apache.spark.status.ElementTrackingStore
 import org.apache.spark.util.Utils
 
-
 /**
  * A class that holds all state shared across sessions in a given [[SQLContext]].
  *
@@ -55,45 +54,10 @@ private[sql] class SharedState(
 
   SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf)
 
-  // Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
-  // the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
-  val warehousePath: String = {
-    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
-    if (configFile != null) {
-      logInfo(s"loading hive config file: $configFile")
-      sparkContext.hadoopConfiguration.addResource(configFile)
-    }
-
-    // hive.metastore.warehouse.dir only stay in hadoopConf
-    sparkContext.conf.remove("hive.metastore.warehouse.dir")
-    // Set the Hive metastore warehouse path to the one we use
-    val hiveWarehouseDir = sparkContext.hadoopConfiguration.get("hive.metastore.warehouse.dir")
-    if (hiveWarehouseDir != null && !sparkContext.conf.contains(WAREHOUSE_PATH.key)) {
-      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
-      // we will respect the value of hive.metastore.warehouse.dir.
-      sparkContext.conf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
-      logInfo(s"${WAREHOUSE_PATH.key} is not set, but hive.metastore.warehouse.dir " +
-        s"is set. Setting ${WAREHOUSE_PATH.key} to the value of " +
-        s"hive.metastore.warehouse.dir ('$hiveWarehouseDir').")
-      hiveWarehouseDir
-    } else {
-      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
-      // the value of spark.sql.warehouse.dir.
-      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set,
-      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
-      val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH)
-      logInfo(s"Setting hive.metastore.warehouse.dir ('$hiveWarehouseDir') to the value of " +
-        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
-      sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir)
-      sparkWarehouseDir
-    }
-  }
-  logInfo(s"Warehouse path is '$warehousePath'.")
-
-  // These 2 variables should be initiated after `warehousePath`, because in the first place we need
-  // to load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
-  // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
   private val (conf, hadoopConf) = {
+    // Load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
+    // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
+    SharedState.loadHiveConfFile(sparkContext.conf, sparkContext.hadoopConfiguration)
     val confClone = sparkContext.conf.clone()
     val hadoopConfClone = new Configuration(sparkContext.hadoopConfiguration)
     // If `SparkSession` is instantiated using an existing `SparkContext` instance and no existing
@@ -166,7 +130,7 @@ private[sql] class SharedState(
     val defaultDbDefinition = CatalogDatabase(
       SessionCatalog.DEFAULT_DATABASE,
       "default database",
-      CatalogUtils.stringToURI(warehousePath),
+      CatalogUtils.stringToURI(conf.get(WAREHOUSE_PATH)),
       Map())
     // Create default database if it doesn't exist
     if (!externalCatalog.databaseExists(SessionCatalog.DEFAULT_DATABASE)) {
@@ -258,4 +222,40 @@ object SharedState extends Logging {
         throw new IllegalArgumentException(s"Error while instantiating '$className':", e)
     }
   }
+
+  /**
+   * Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
+   * the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
+   */
+  def loadHiveConfFile(sparkConf: SparkConf, hadoopConf: Configuration): Unit = {
+    val hiveWarehouseKey = "hive.metastore.warehouse.dir"
+    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+    if (configFile != null) {
+      logInfo(s"loading hive config file: $configFile")
+      hadoopConf.addResource(configFile)
+    }
+    // hive.metastore.warehouse.dir only stay in hadoopConf
+    sparkConf.remove(hiveWarehouseKey)
+    // Set the Hive metastore warehouse path to the one we use
+    val hiveWarehouseDir = hadoopConf.get(hiveWarehouseKey)
+    val warehousePath = if (hiveWarehouseDir != null && !sparkConf.contains(WAREHOUSE_PATH.key)) {
+      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
+      // we will respect the value of hive.metastore.warehouse.dir.
+      sparkConf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
+      logInfo(s"${WAREHOUSE_PATH.key} is not set, but $hiveWarehouseKey is set. Setting" +
+        s" ${WAREHOUSE_PATH.key} to the value of $hiveWarehouseKey ('$hiveWarehouseDir').")
+      hiveWarehouseDir
+    } else {
+      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
+      // the value of spark.sql.warehouse.dir.
+      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set
+      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
+      val sparkWarehouseDir = sparkConf.get(WAREHOUSE_PATH)
+      logInfo(s"Setting $hiveWarehouseKey ('$hiveWarehouseDir') to the value of " +
+        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
+      hadoopConf.set(hiveWarehouseKey, sparkWarehouseDir)
+      sparkWarehouseDir
+    }
+    logInfo(s"Warehouse path is '$warehousePath'.")
+  }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 6b76927bd4150..3ddf4eccfd7a5 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -46,6 +46,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider
+import org.apache.spark.sql.internal.SharedState
 import org.apache.spark.util.ShutdownHookManager
 
 /**
@@ -87,6 +88,7 @@ private[hive] object SparkSQLCLIDriver extends Logging {
 
     val sparkConf = new SparkConf(loadDefaults = true)
     val hadoopConf = SparkHadoopUtil.get.newConfiguration(sparkConf)
+    SharedState.loadHiveConfFile(sparkConf, hadoopConf)
     val extraConfigs = HiveUtils.formatTimeVarsForHiveClient(hadoopConf)
 
     val cliConf = HiveClientImpl.newHiveConf(sparkConf, hadoopConf, extraConfigs)
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 43aafc3c8590c..ed776633f903b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -32,6 +32,7 @@ import org.scalatest.BeforeAndAfterAll
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.hive.test.HiveTestJars
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
 import org.apache.spark.util.{ThreadUtils, Utils}
 
@@ -159,6 +160,17 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     }
   }
 
+  test("Pick spark.sql.warehouse.dir first for Spark Cli if set") {
+    val sparkWareHouseDir = Utils.createTempDir()
+    new File(warehousePath, "metastore_db").delete()
+    runCliWithin(
+      1.minute,
+      Seq("--conf", s"${StaticSQLConf.WAREHOUSE_PATH.key}=$sparkWareHouseDir"))(
+      "desc database default;" -> sparkWareHouseDir.getAbsolutePath
+    )
+    sparkWareHouseDir.delete()
+  }
+
   test("Simple commands") {
     val dataFilePath =
       Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
index 6e2dcfc04d498..78535b094b83d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
@@ -45,7 +45,6 @@ class HiveSharedStateSuite extends SparkFunSuite {
       GLOBAL_TEMP_DATABASE.key -> tmpDb)
 
     val state = new SharedState(sc, initialConfigs)
-    assert(state.warehousePath !== invalidPath, "warehouse path can't determine by session options")
     assert(sc.conf.get(WAREHOUSE_PATH.key) !== invalidPath,
       "warehouse conf in session options can't affect application wide spark conf")
     assert(sc.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) !== invalidPath,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 31ff62ed0a530..8b97489e2d818 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -787,7 +787,7 @@ object SPARK_18360 {
       .enableHiveSupport().getOrCreate()
 
     val defaultDbLocation = spark.catalog.getDatabase("default").locationUri
-    assert(new Path(defaultDbLocation) == new Path(spark.sharedState.warehousePath))
+    assert(new Path(defaultDbLocation) == new Path(spark.conf.get(WAREHOUSE_PATH)))
 
     val hiveClient =
       spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client

From 314627ee148264e170b0feaa623e1e1284932166 Mon Sep 17 00:00:00 2001
From: Adam Binford <adam.binford@radiantsolutions.com>
Date: Tue, 17 Mar 2020 11:20:10 -0700
Subject: [PATCH 0297/1280] [SPARK-30860][CORE] Use FileSystem.mkdirs to avoid
 umask at rolling event log folder and appStatusFile creation

### What changes were proposed in this pull request?
This pull request fixes an issue with rolling event logs. The rolling event log directory is created ignoring the dfs umask setting. This allows the history server to prune old rolling logs when run as the group owner of the event log folder.

### Why are the changes needed?
For non-rolling event logs, log files are created ignoring the umask setting by calling setPermission after creating the file. The default umask of 022 currently causes rolling log directories to be created without group write permissions, preventing the history server from pruning logs of applications not run as the same user as the history server. This adds the same behavior for rolling event logs so users don't need to worry about the umask setting causing different behavior.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually. The folder is created with the correct 770 permission. The status file is still affected by the umask setting, but that doesn't stop the folder from being deleted by the history server. I'm not sure if that causes any other issues. I'm not sure how to test something involving a Hadoop setting.

Closes #27764 from Kimahriman/bug/rolling-log-permissions.

Authored-by: Adam Binford <adam.binford@radiantsolutions.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 9f27a5495d0114723701c932f20f32d308a571cc)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/deploy/history/EventLogFileWriters.scala     | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
index 1d58d054b7825..7d44cbd9f64f9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
@@ -166,7 +166,8 @@ object EventLogFileWriter {
   val IN_PROGRESS = ".inprogress"
   val COMPACTED = ".compact"
 
-  val LOG_FILE_PERMISSIONS = new FsPermission(Integer.parseInt("770", 8).toShort)
+  val LOG_FILE_PERMISSIONS = new FsPermission(Integer.parseInt("660", 8).toShort)
+  val LOG_FOLDER_PERMISSIONS = new FsPermission(Integer.parseInt("770", 8).toShort)
 
   def apply(
       appId: String,
@@ -317,7 +318,8 @@ class RollingEventLogFilesWriter(
       throw new IOException(s"Target log directory already exists ($logDirForAppPath)")
     }
 
-    fileSystem.mkdirs(logDirForAppPath, EventLogFileWriter.LOG_FILE_PERMISSIONS)
+    // SPARK-30860: use the class method to avoid the umask causing permission issues
+    FileSystem.mkdirs(fileSystem, logDirForAppPath, EventLogFileWriter.LOG_FOLDER_PERMISSIONS)
     createAppStatusFile(inProgress = true)
     rollEventLogFile()
   }
@@ -361,7 +363,9 @@ class RollingEventLogFilesWriter(
 
   private def createAppStatusFile(inProgress: Boolean): Unit = {
     val appStatusPath = getAppStatusFilePath(logDirForAppPath, appId, appAttemptId, inProgress)
-    val outputStream = fileSystem.create(appStatusPath)
+    // SPARK-30860: use the class method to avoid the umask causing permission issues
+    val outputStream = FileSystem.create(fileSystem, appStatusPath,
+      EventLogFileWriter.LOG_FILE_PERMISSIONS)
     // we intentionally create zero-byte file to minimize the cost
     outputStream.close()
   }

From b9371e38abbc351106453b14172d6919be0eca82 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 17 Mar 2020 11:48:54 -0700
Subject: [PATCH 0298/1280] [SPARK-31171][SQL] size(null) should return null
 under ansi mode

Make `size(null)` return null under ANSI mode, regardless of the `spark.sql.legacy.sizeOfNull` config.

In https://github.com/apache/spark/pull/27834, we change the result of `size(null)` to be -1 to match the 2.4 behavior and avoid breaking changes.

However, it's true that the "return -1" behavior is error-prone when being used with aggregate functions. The current ANSI mode controls a bunch of "better behaviors" like failing on overflow. We don't enable these "better behaviors" by default because they are too breaking. The "return null" behavior of `size(null)` is a good fit of the ANSI mode.

No as ANSI mode is off by default.

new tests

Closes #27936 from cloud-fan/null.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit dc5ebc2d5b8122121d89a9175737bea95ae10126)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/sql/internal/SQLConf.scala      |  9 ++++++---
 .../expressions/CollectionExpressionsSuite.scala     |  6 ++++++
 .../apache/spark/sql/DataFrameFunctionsSuite.scala   | 12 ++++++++++++
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index e49593ef8518f..1331350f46bc8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1940,8 +1940,8 @@ object SQLConf {
 
   val LEGACY_SIZE_OF_NULL = buildConf("spark.sql.legacy.sizeOfNull")
     .internal()
-    .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " +
-      "The size function returns null for null input if the flag is disabled.")
+    .doc(s"If it is set to false, or ${ANSI_ENABLED.key} is true, then size of null returns " +
+      "null. Otherwise, it returns -1, which was inherited from Hive.")
     .booleanConf
     .createWithDefault(true)
 
@@ -2759,7 +2759,10 @@ class SQLConf extends Serializable with Logging {
 
   def csvColumnPruning: Boolean = getConf(SQLConf.CSV_PARSER_COLUMN_PRUNING)
 
-  def legacySizeOfNull: Boolean = getConf(SQLConf.LEGACY_SIZE_OF_NULL)
+  def legacySizeOfNull: Boolean = {
+    // size(null) should return null under ansi mode.
+    getConf(SQLConf.LEGACY_SIZE_OF_NULL) && !getConf(ANSI_ENABLED)
+  }
 
   def isReplEagerEvalEnabled: Boolean = getConf(SQLConf.REPL_EAGER_EVAL_ENABLED)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 3cfc66f5cdb03..173f24881531a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -74,6 +74,12 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     withSQLConf(SQLConf.LEGACY_SIZE_OF_NULL.key -> "false") {
       testSize(sizeOfNull = null)
     }
+    // size(null) should return null under ansi mode.
+    withSQLConf(
+      SQLConf.LEGACY_SIZE_OF_NULL.key -> "true",
+      SQLConf.ANSI_ENABLED.key -> "true") {
+      testSize(sizeOfNull = null)
+    }
   }
 
   test("MapKeys/MapValues") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index a613c33b6c876..c41eb98c13ea0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -490,6 +490,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     withSQLConf(SQLConf.LEGACY_SIZE_OF_NULL.key -> "false") {
       testSizeOfArray(sizeOfNull = null)
     }
+    // size(null) should return null under ansi mode.
+    withSQLConf(
+      SQLConf.LEGACY_SIZE_OF_NULL.key -> "true",
+      SQLConf.ANSI_ENABLED.key -> "true") {
+      testSizeOfArray(sizeOfNull = null)
+    }
   }
 
   test("dataframe arrays_zip function") {
@@ -569,6 +575,12 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     withSQLConf(SQLConf.LEGACY_SIZE_OF_NULL.key -> "false") {
       testSizeOfMap(sizeOfNull = null)
     }
+    // size(null) should return null under ansi mode.
+    withSQLConf(
+      SQLConf.LEGACY_SIZE_OF_NULL.key -> "true",
+      SQLConf.ANSI_ENABLED.key -> "true") {
+      testSizeOfMap(sizeOfNull = null)
+    }
   }
 
   test("map_keys/map_values function") {

From a7c25259c97085f6518f88fbcc3f0f4bde2995af Mon Sep 17 00:00:00 2001
From: Qianyang Yu <qyu@us.ibm.com>
Date: Tue, 17 Mar 2020 16:15:02 -0700
Subject: [PATCH 0299/1280] [SPARK-30954][ML][R] Make file name the same as
 class name

This pr solved the same issue as [pr27919](https://github.com/apache/spark/pull/27919), but this one changes the file names based on comment from previous pr.

### What changes were proposed in this pull request?

Make some of  file names the same as class name in R package.

### Why are the changes needed?

Make the file consistence

### Does this PR introduce any user-facing change?

No
### How was this patch tested?

run `./R/run-tests.sh`

Closes #27940 from kevinyu98/spark-30954-r-v2.

Authored-by: Qianyang Yu <qyu@us.ibm.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6f0b0f1655e5d8fbfb20ed0a58b740e6c9a6ae50)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 ...ificationWrapper.scala => DecisionTreeClassifierWrapper.scala} | 0
 ...RegressionWrapper.scala => DecisionTreeRegressorWrapper.scala} | 0
 ...{GBTClassificationWrapper.scala => GBTClassifierWrapper.scala} | 0
 .../r/{GBTRegressionWrapper.scala => GBTRegressorWrapper.scala}   | 0
 ...ificationWrapper.scala => RandomForestClassifierWrapper.scala} | 0
 ...RegressionWrapper.scala => RandomForestRegressorWrapper.scala} | 0
 6 files changed, 0 insertions(+), 0 deletions(-)
 rename mllib/src/main/scala/org/apache/spark/ml/r/{DecisionTreeClassificationWrapper.scala => DecisionTreeClassifierWrapper.scala} (100%)
 rename mllib/src/main/scala/org/apache/spark/ml/r/{DecisionTreeRegressionWrapper.scala => DecisionTreeRegressorWrapper.scala} (100%)
 rename mllib/src/main/scala/org/apache/spark/ml/r/{GBTClassificationWrapper.scala => GBTClassifierWrapper.scala} (100%)
 rename mllib/src/main/scala/org/apache/spark/ml/r/{GBTRegressionWrapper.scala => GBTRegressorWrapper.scala} (100%)
 rename mllib/src/main/scala/org/apache/spark/ml/r/{RandomForestClassificationWrapper.scala => RandomForestClassifierWrapper.scala} (100%)
 rename mllib/src/main/scala/org/apache/spark/ml/r/{RandomForestRegressionWrapper.scala => RandomForestRegressorWrapper.scala} (100%)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassifierWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassificationWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeClassifierWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressorWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressionWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/DecisionTreeRegressorWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GBTClassifierWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/GBTClassificationWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/GBTClassifierWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressorWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressionWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/GBTRegressorWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassifierWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassificationWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/RandomForestClassifierWrapper.scala
diff --git a/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala b/mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressorWrapper.scala
similarity index 100%
rename from mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressionWrapper.scala
rename to mllib/src/main/scala/org/apache/spark/ml/r/RandomForestRegressorWrapper.scala

From 06d31104120a6f0ce249090864a4f452af0b675a Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 18 Mar 2020 12:29:39 +0800
Subject: [PATCH 0300/1280] [SPARK-31119][SQL] Add interval value support for
 extract expression as extract source

### What changes were proposed in this pull request?

```
<extract expression> ::= EXTRACT <left paren> <extract field> FROM <extract source> <right paren>

<extract source> ::= <datetime value expression> | <interval value expression>
```
We now only support datetime values as extract source for `extract` expression but it's alternative function `date_part` supports both datetime and interval.

This pr adds interval value support for `extract` expression as extract source

### Why are the changes needed?

For ANSI compliance and the semantic consistency between extract and `date_part`, we support intervals for extract expressions.

### Does this PR introduce any user-facing change?

yes, in the `extract(abc from xyz)` expression, the `xyz` can be intervals

### How was this patch tested?

add unit tests

Closes #27876 from yaooqinn/SPARK-31119.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f1d27cdd91d0d5d107add662f7c35fc4a5dd278c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/parser/AstBuilder.scala      |   8 +-
 .../ExtractBenchmark-jdk11-results.txt        | 207 +++++-----
 .../benchmarks/ExtractBenchmark-results.txt   | 207 +++++-----
 .../resources/sql-tests/inputs/extract.sql    | 121 +++---
 .../sql-tests/results/extract.sql.out         | 358 +++++++++---------
 .../sql-tests/results/postgreSQL/date.sql.out |  16 +-
 .../benchmark/ExtractBenchmark.scala          |  16 +-
 7 files changed, 484 insertions(+), 449 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 3c23ecb0c4ff4..b94fd1dd1c0ef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1545,12 +1545,8 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a Extract expression.
    */
   override def visitExtract(ctx: ExtractContext): Expression = withOrigin(ctx) {
-    val fieldStr = ctx.field.getText
-    val source = expression(ctx.source)
-    val extractField = DatePart.parseExtractField(fieldStr, source, {
-      throw new ParseException(s"Literals of type '$fieldStr' are currently not supported.", ctx)
-    })
-    new DatePart(Literal(fieldStr), expression(ctx.source), extractField)
+    val arguments = Seq(Literal(ctx.field.getText), expression(ctx.source))
+    UnresolvedFunction("date_part", arguments, isDistinct = false)
   }
 
   /**
diff --git a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
index 0873215982168..8c56c0bcc9cb0 100644
--- a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
@@ -2,118 +2,137 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   343            348           5         29.1          34.3       1.0X
-MILLENNIUM of timestamp                             888            899          14         11.3          88.8       0.4X
-CENTURY of timestamp                                835            846          15         12.0          83.5       0.4X
-DECADE of timestamp                                 824            834           9         12.1          82.4       0.4X
-YEAR of timestamp                                   818            824          10         12.2          81.8       0.4X
-ISOYEAR of timestamp                                879            890          17         11.4          87.9       0.4X
-QUARTER of timestamp                                965           1073          96         10.4          96.5       0.4X
-MONTH of timestamp                                  787            793           8         12.7          78.7       0.4X
-WEEK of timestamp                                  1277           1341          57          7.8         127.7       0.3X
-DAY of timestamp                                    780            785           8         12.8          78.0       0.4X
-DAYOFWEEK of timestamp                              956            962           7         10.5          95.6       0.4X
-DOW of timestamp                                    995           1034          36         10.1          99.5       0.3X
-ISODOW of timestamp                                 885            898          11         11.3          88.5       0.4X
-DOY of timestamp                                    808            810           3         12.4          80.8       0.4X
-HOUR of timestamp                                   622            627           7         16.1          62.2       0.6X
-MINUTE of timestamp                                 629            636           8         15.9          62.9       0.5X
-SECOND of timestamp                                 800            819          30         12.5          80.0       0.4X
-MILLISECONDS of timestamp                           743            755          21         13.5          74.3       0.5X
-MICROSECONDS of timestamp                           692            730          53         14.5          69.2       0.5X
-EPOCH of timestamp                                  773            798          40         12.9          77.3       0.4X
+cast to timestamp                                   311            331          18         32.2          31.1       1.0X
+MILLENNIUM of timestamp                             873            893          17         11.4          87.3       0.4X
+CENTURY of timestamp                                869            873           5         11.5          86.9       0.4X
+DECADE of timestamp                                 851            872          23         11.7          85.1       0.4X
+YEAR of timestamp                                   841            856          14         11.9          84.1       0.4X
+ISOYEAR of timestamp                                927            938          12         10.8          92.7       0.3X
+QUARTER of timestamp                                959            963           6         10.4          95.9       0.3X
+MONTH of timestamp                                  852            864          18         11.7          85.2       0.4X
+WEEK of timestamp                                  1124           1252         112          8.9         112.4       0.3X
+DAY of timestamp                                    848            867          19         11.8          84.8       0.4X
+DAYOFWEEK of timestamp                              977            987          16         10.2          97.7       0.3X
+DOW of timestamp                                    945            964          18         10.6          94.5       0.3X
+ISODOW of timestamp                                 924            929           5         10.8          92.4       0.3X
+DOY of timestamp                                    852            906          67         11.7          85.2       0.4X
+HOUR of timestamp                                   665            671           5         15.0          66.5       0.5X
+MINUTE of timestamp                                 655            670          15         15.3          65.5       0.5X
+SECOND of timestamp                                 757            763           7         13.2          75.7       0.4X
+MILLISECONDS of timestamp                           745            761          14         13.4          74.5       0.4X
+MICROSECONDS of timestamp                           691            697           7         14.5          69.1       0.5X
+EPOCH of timestamp                                  794            806          12         12.6          79.4       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   303            323          24         33.0          30.3       1.0X
-MILLENNIUM of timestamp                             841            852           9         11.9          84.1       0.4X
-CENTURY of timestamp                                818            821           4         12.2          81.8       0.4X
-DECADE of timestamp                                 821            836          15         12.2          82.1       0.4X
-YEAR of timestamp                                   858            928          61         11.7          85.8       0.4X
-ISOYEAR of timestamp                                933            939           6         10.7          93.3       0.3X
-QUARTER of timestamp                                979           1004          26         10.2          97.9       0.3X
-MONTH of timestamp                                  755            784          25         13.2          75.5       0.4X
-WEEK of timestamp                                  1189           1228          64          8.4         118.9       0.3X
-DAY of timestamp                                    768            770           4         13.0          76.8       0.4X
-DAYOFWEEK of timestamp                              918            948          27         10.9          91.8       0.3X
-DOW of timestamp                                    933            983          53         10.7          93.3       0.3X
-ISODOW of timestamp                                 884            928          61         11.3          88.4       0.3X
-DOY of timestamp                                    787            797          12         12.7          78.7       0.4X
-HOUR of timestamp                                   611            664          58         16.4          61.1       0.5X
-MINUTE of timestamp                                 616            622           5         16.2          61.6       0.5X
-SECOND of timestamp                                 732            748          14         13.7          73.2       0.4X
-MILLISECONDS of timestamp                           705            716          17         14.2          70.5       0.4X
-MICROSECONDS of timestamp                           642            664          33         15.6          64.2       0.5X
-EPOCH of timestamp                                  760            781          18         13.2          76.0       0.4X
+cast to timestamp                                   273            274           1         36.6          27.3       1.0X
+MILLENNIUM of timestamp                             860            875          14         11.6          86.0       0.3X
+CENTURY of timestamp                                854            866          11         11.7          85.4       0.3X
+DECADE of timestamp                                 855            863          10         11.7          85.5       0.3X
+YEAR of timestamp                                   833            837           4         12.0          83.3       0.3X
+ISOYEAR of timestamp                               1022           1036          18          9.8         102.2       0.3X
+QUARTER of timestamp                               1067           1132          58          9.4         106.7       0.3X
+MONTH of timestamp                                  855            861           5         11.7          85.5       0.3X
+WEEK of timestamp                                  1312           1325          17          7.6         131.2       0.2X
+DAY of timestamp                                    837            847          10         12.0          83.7       0.3X
+DAYOFWEEK of timestamp                              965            969           4         10.4          96.5       0.3X
+DOW of timestamp                                    981           1038          56         10.2          98.1       0.3X
+ISODOW of timestamp                                 942            959          20         10.6          94.2       0.3X
+DOY of timestamp                                    994           1021          24         10.1          99.4       0.3X
+HOUR of timestamp                                   690            695           7         14.5          69.0       0.4X
+MINUTE of timestamp                                 764            788          21         13.1          76.4       0.4X
+SECOND of timestamp                                 776            795          34         12.9          77.6       0.4X
+MILLISECONDS of timestamp                           863            879          14         11.6          86.3       0.3X
+MICROSECONDS of timestamp                           689            695           7         14.5          68.9       0.4X
+EPOCH of timestamp                                  797            865          69         12.5          79.7       0.3X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        712            786         120         14.0          71.2       1.0X
-MILLENNIUM of date                                  846            888          73         11.8          84.6       0.8X
-CENTURY of date                                     781            792          11         12.8          78.1       0.9X
-DECADE of date                                      770            824          48         13.0          77.0       0.9X
-YEAR of date                                        804            832          27         12.4          80.4       0.9X
-ISOYEAR of date                                     909            931          24         11.0          90.9       0.8X
-QUARTER of date                                     957            975          22         10.4          95.7       0.7X
-MONTH of date                                       789            794           7         12.7          78.9       0.9X
-WEEK of date                                       1141           1165          29          8.8         114.1       0.6X
-DAY of date                                         784            800          22         12.8          78.4       0.9X
-DAYOFWEEK of date                                   907            916          14         11.0          90.7       0.8X
-DOW of date                                         931            958          25         10.7          93.1       0.8X
-ISODOW of date                                      852            857           7         11.7          85.2       0.8X
-DOY of date                                         831            870          45         12.0          83.1       0.9X
-HOUR of date                                       1574           1598          26          6.4         157.4       0.5X
-MINUTE of date                                     1525           1590          58          6.6         152.5       0.5X
-SECOND of date                                     1728           1739          10          5.8         172.8       0.4X
-MILLISECONDS of date                               1715           1727          10          5.8         171.5       0.4X
-MICROSECONDS of date                               1559           1669          95          6.4         155.9       0.5X
-EPOCH of date                                      1766           1774           9          5.7         176.6       0.4X
+cast to date                                        763            774          13         13.1          76.3       1.0X
+MILLENNIUM of date                                  857            911          73         11.7          85.7       0.9X
+CENTURY of date                                     869            884          15         11.5          86.9       0.9X
+DECADE of date                                      860            868           7         11.6          86.0       0.9X
+YEAR of date                                        848            852           7         11.8          84.8       0.9X
+ISOYEAR of date                                    1028           1046          22          9.7         102.8       0.7X
+QUARTER of date                                     979            986           7         10.2          97.9       0.8X
+MONTH of date                                       857            862           4         11.7          85.7       0.9X
+WEEK of date                                       1218           1247          29          8.2         121.8       0.6X
+DAY of date                                         855            868          11         11.7          85.5       0.9X
+DAYOFWEEK of date                                   987           1081          82         10.1          98.7       0.8X
+DOW of date                                         986            996           9         10.1          98.6       0.8X
+ISODOW of date                                      938            947          10         10.7          93.8       0.8X
+DOY of date                                         877            890          11         11.4          87.7       0.9X
+HOUR of date                                       1656           1668          12          6.0         165.6       0.5X
+MINUTE of date                                     1652           1666          15          6.1         165.2       0.5X
+SECOND of date                                     1752           1776          23          5.7         175.2       0.4X
+MILLISECONDS of date                               1757           1766          11          5.7         175.7       0.4X
+MICROSECONDS of date                               1682           1691          14          5.9         168.2       0.5X
+EPOCH of date                                      1706           1721          14          5.9         170.6       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        803            825          23         12.5          80.3       1.0X
-MILLENNIUM of date                                  918            924           8         10.9          91.8       0.9X
-CENTURY of date                                     855            879          25         11.7          85.5       0.9X
-DECADE of date                                      777            795          19         12.9          77.7       1.0X
-YEAR of date                                        772            779           9         13.0          77.2       1.0X
-ISOYEAR of date                                     903            927          21         11.1          90.3       0.9X
-QUARTER of date                                     983           1339         381         10.2          98.3       0.8X
-MONTH of date                                       764            780          20         13.1          76.4       1.1X
-WEEK of date                                       1239           1316         112          8.1         123.9       0.6X
-DAY of date                                         769            857          91         13.0          76.9       1.0X
-DAYOFWEEK of date                                   900            965          61         11.1          90.0       0.9X
-DOW of date                                         990           1041          62         10.1          99.0       0.8X
-ISODOW of date                                      974            977           5         10.3          97.4       0.8X
-DOY of date                                         862            918          50         11.6          86.2       0.9X
-HOUR of date                                       1491           1517          25          6.7         149.1       0.5X
-MINUTE of date                                     1514           1520           9          6.6         151.4       0.5X
-SECOND of date                                     1657           1677          27          6.0         165.7       0.5X
-MILLISECONDS of date                               1646           1669          26          6.1         164.6       0.5X
-MICROSECONDS of date                               1538           1556          23          6.5         153.8       0.5X
-EPOCH of date                                      1699           1718          28          5.9         169.9       0.5X
+cast to date                                        771            779          10         13.0          77.1       1.0X
+MILLENNIUM of date                                  839            858          22         11.9          83.9       0.9X
+CENTURY of date                                     853            872          22         11.7          85.3       0.9X
+DECADE of date                                      871            878           6         11.5          87.1       0.9X
+YEAR of date                                        853            863           9         11.7          85.3       0.9X
+ISOYEAR of date                                    1011           1021          11          9.9         101.1       0.8X
+QUARTER of date                                     981            988           7         10.2          98.1       0.8X
+MONTH of date                                       859            863           5         11.6          85.9       0.9X
+WEEK of date                                       1148           1159          12          8.7         114.8       0.7X
+DAY of date                                         852            855           2         11.7          85.2       0.9X
+DAYOFWEEK of date                                   937            952          12         10.7          93.7       0.8X
+DOW of date                                         953            956           3         10.5          95.3       0.8X
+ISODOW of date                                      899            916          16         11.1          89.9       0.9X
+DOY of date                                         857            865           7         11.7          85.7       0.9X
+HOUR of date                                       1644           1662          18          6.1         164.4       0.5X
+MINUTE of date                                     1633           1644          10          6.1         163.3       0.5X
+SECOND of date                                     1719           1727           9          5.8         171.9       0.4X
+MILLISECONDS of date                               1721           1733          11          5.8         172.1       0.4X
+MICROSECONDS of date                               1656           1680          34          6.0         165.6       0.5X
+EPOCH of date                                      1721           1736          15          5.8         172.1       0.4X
+
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+cast to interval                                    939            947           9         10.7          93.9       1.0X
+MILLENNIUM of interval                              953            960           6         10.5          95.3       1.0X
+CENTURY of interval                                 971            979          10         10.3          97.1       1.0X
+DECADE of interval                                  962            969           7         10.4          96.2       1.0X
+YEAR of interval                                    965            973          10         10.4          96.5       1.0X
+QUARTER of interval                                 967            979          15         10.3          96.7       1.0X
+MONTH of interval                                   951            979          26         10.5          95.1       1.0X
+DAY of interval                                     945            970          25         10.6          94.5       1.0X
+HOUR of interval                                    963            970           9         10.4          96.3       1.0X
+MINUTE of interval                                  972            991          24         10.3          97.2       1.0X
+SECOND of interval                                 1053           1057           8          9.5         105.3       0.9X
+MILLISECONDS of interval                           1042           1048          11          9.6         104.2       0.9X
+MICROSECONDS of interval                            963            967           4         10.4          96.3       1.0X
+EPOCH of interval                                  1051           1071          22          9.5         105.1       0.9X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    971            976           5         10.3          97.1       1.0X
-MILLENNIUM of interval                              983           1009          23         10.2          98.3       1.0X
-CENTURY of interval                                 972            984          12         10.3          97.2       1.0X
-DECADE of interval                                  954            962          10         10.5          95.4       1.0X
-YEAR of interval                                    954            973          17         10.5          95.4       1.0X
-QUARTER of interval                                1009           1020          11          9.9         100.9       1.0X
-MONTH of interval                                   946            963          16         10.6          94.6       1.0X
-DAY of interval                                     952            963          12         10.5          95.2       1.0X
-HOUR of interval                                    948            960          11         10.5          94.8       1.0X
-MINUTE of interval                                 1035           1040           4          9.7         103.5       0.9X
-SECOND of interval                                 1085           1105          31          9.2         108.5       0.9X
-MILLISECONDS of interval                           1065           1088          24          9.4         106.5       0.9X
-MICROSECONDS of interval                            992           1007          17         10.1          99.2       1.0X
-EPOCH of interval                                  1087           1103          25          9.2         108.7       0.9X
+cast to interval                                    944            952           8         10.6          94.4       1.0X
+MILLENNIUM of interval                              955            960           5         10.5          95.5       1.0X
+CENTURY of interval                                 958            972          12         10.4          95.8       1.0X
+DECADE of interval                                  962            967           6         10.4          96.2       1.0X
+YEAR of interval                                    958            978          18         10.4          95.8       1.0X
+QUARTER of interval                                 979            998          19         10.2          97.9       1.0X
+MONTH of interval                                   949            963          12         10.5          94.9       1.0X
+DAY of interval                                     958            971          11         10.4          95.8       1.0X
+HOUR of interval                                    947            962          15         10.6          94.7       1.0X
+MINUTE of interval                                  989           1006          15         10.1          98.9       1.0X
+SECOND of interval                                 1061           1064           4          9.4         106.1       0.9X
+MILLISECONDS of interval                           1042           1061          18          9.6         104.2       0.9X
+MICROSECONDS of interval                            982            992          12         10.2          98.2       1.0X
+EPOCH of interval                                  1047           1057          13          9.5         104.7       0.9X
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-results.txt b/sql/core/benchmarks/ExtractBenchmark-results.txt
index 36bdbd026c046..94cf0bc475582 100644
--- a/sql/core/benchmarks/ExtractBenchmark-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-results.txt
@@ -2,118 +2,137 @@ Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   287            308          19         34.8          28.7       1.0X
-MILLENNIUM of timestamp                             896            918          20         11.2          89.6       0.3X
-CENTURY of timestamp                                849            856           7         11.8          84.9       0.3X
-DECADE of timestamp                                 765            777          17         13.1          76.5       0.4X
-YEAR of timestamp                                   754            756           2         13.3          75.4       0.4X
-ISOYEAR of timestamp                                843            849           5         11.9          84.3       0.3X
-QUARTER of timestamp                                867            873           9         11.5          86.7       0.3X
-MONTH of timestamp                                  758            762           4         13.2          75.8       0.4X
-WEEK of timestamp                                  1049           1054           6          9.5         104.9       0.3X
-DAY of timestamp                                    750            763          11         13.3          75.0       0.4X
-DAYOFWEEK of timestamp                              890            918          25         11.2          89.0       0.3X
-DOW of timestamp                                    879            887           8         11.4          87.9       0.3X
-ISODOW of timestamp                                 862            869          11         11.6          86.2       0.3X
-DOY of timestamp                                    811            868          55         12.3          81.1       0.4X
-HOUR of timestamp                                   627            638          11         16.0          62.7       0.5X
-MINUTE of timestamp                                 600            606           6         16.7          60.0       0.5X
-SECOND of timestamp                                 743            799          51         13.5          74.3       0.4X
-MILLISECONDS of timestamp                           723            737          22         13.8          72.3       0.4X
-MICROSECONDS of timestamp                           648            653           5         15.4          64.8       0.4X
-EPOCH of timestamp                                  780            800          17         12.8          78.0       0.4X
+cast to timestamp                                   275            307          28         36.3          27.5       1.0X
+MILLENNIUM of timestamp                             911            924          20         11.0          91.1       0.3X
+CENTURY of timestamp                                846            850           5         11.8          84.6       0.3X
+DECADE of timestamp                                 744            761          21         13.4          74.4       0.4X
+YEAR of timestamp                                   750            764          15         13.3          75.0       0.4X
+ISOYEAR of timestamp                                815            828          11         12.3          81.5       0.3X
+QUARTER of timestamp                                882            895          12         11.3          88.2       0.3X
+MONTH of timestamp                                  739            760          19         13.5          73.9       0.4X
+WEEK of timestamp                                  1058           1082          23          9.4         105.8       0.3X
+DAY of timestamp                                    722            730           8         13.8          72.2       0.4X
+DAYOFWEEK of timestamp                              860            907          58         11.6          86.0       0.3X
+DOW of timestamp                                    853            860           7         11.7          85.3       0.3X
+ISODOW of timestamp                                 829            835           8         12.1          82.9       0.3X
+DOY of timestamp                                    757            772          20         13.2          75.7       0.4X
+HOUR of timestamp                                   586            594           7         17.1          58.6       0.5X
+MINUTE of timestamp                                 577            584           8         17.3          57.7       0.5X
+SECOND of timestamp                                 810            827          17         12.4          81.0       0.3X
+MILLISECONDS of timestamp                           687            704          16         14.6          68.7       0.4X
+MICROSECONDS of timestamp                           628            632           6         15.9          62.8       0.4X
+EPOCH of timestamp                                  750            761          12         13.3          75.0       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   238            248          12         42.0          23.8       1.0X
-MILLENNIUM of timestamp                             862            875          12         11.6          86.2       0.3X
-CENTURY of timestamp                                833            847          22         12.0          83.3       0.3X
-DECADE of timestamp                                 759            765           7         13.2          75.9       0.3X
-YEAR of timestamp                                   744            755          15         13.4          74.4       0.3X
-ISOYEAR of timestamp                                937           1019          73         10.7          93.7       0.3X
-QUARTER of timestamp                               1011           1091          69          9.9         101.1       0.2X
-MONTH of timestamp                                  846            888          40         11.8          84.6       0.3X
-WEEK of timestamp                                  1210           1239          41          8.3         121.0       0.2X
-DAY of timestamp                                    932            979          41         10.7          93.2       0.3X
-DAYOFWEEK of timestamp                             1174           1200          42          8.5         117.4       0.2X
-DOW of timestamp                                   1131           1172          37          8.8         113.1       0.2X
-ISODOW of timestamp                                 896            903           7         11.2          89.6       0.3X
-DOY of timestamp                                    805            818          12         12.4          80.5       0.3X
-HOUR of timestamp                                   596            597           2         16.8          59.6       0.4X
-MINUTE of timestamp                                 582            597          17         17.2          58.2       0.4X
-SECOND of timestamp                                 697            709          16         14.4          69.7       0.3X
-MILLISECONDS of timestamp                           700            710          10         14.3          70.0       0.3X
-MICROSECONDS of timestamp                           612            631          21         16.3          61.2       0.4X
-EPOCH of timestamp                                  755            760           7         13.2          75.5       0.3X
+cast to timestamp                                   238            242           4         42.0          23.8       1.0X
+MILLENNIUM of timestamp                             811            835          25         12.3          81.1       0.3X
+CENTURY of timestamp                                799            823          21         12.5          79.9       0.3X
+DECADE of timestamp                                 733            743          12         13.6          73.3       0.3X
+YEAR of timestamp                                   714            717           3         14.0          71.4       0.3X
+ISOYEAR of timestamp                                874            900          24         11.4          87.4       0.3X
+QUARTER of timestamp                                903            910          11         11.1          90.3       0.3X
+MONTH of timestamp                                  774            808          44         12.9          77.4       0.3X
+WEEK of timestamp                                  1053           1064          12          9.5         105.3       0.2X
+DAY of timestamp                                    755            770          14         13.2          75.5       0.3X
+DAYOFWEEK of timestamp                              831            841           9         12.0          83.1       0.3X
+DOW of timestamp                                    871            898          23         11.5          87.1       0.3X
+ISODOW of timestamp                                 832            887          59         12.0          83.2       0.3X
+DOY of timestamp                                    755            767          13         13.2          75.5       0.3X
+HOUR of timestamp                                   580            597          27         17.3          58.0       0.4X
+MINUTE of timestamp                                 572            589          15         17.5          57.2       0.4X
+SECOND of timestamp                                 716            730          12         14.0          71.6       0.3X
+MILLISECONDS of timestamp                           716            762          57         14.0          71.6       0.3X
+MICROSECONDS of timestamp                           600            610          12         16.7          60.0       0.4X
+EPOCH of timestamp                                  751            779          44         13.3          75.1       0.3X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        633            641           7         15.8          63.3       1.0X
-MILLENNIUM of date                                  824            845          26         12.1          82.4       0.8X
-CENTURY of date                                     864            878          13         11.6          86.4       0.7X
-DECADE of date                                      746            763          17         13.4          74.6       0.8X
-YEAR of date                                        752            785          39         13.3          75.2       0.8X
-ISOYEAR of date                                     900            905           5         11.1          90.0       0.7X
-QUARTER of date                                     906            930          23         11.0          90.6       0.7X
-MONTH of date                                       747            752           6         13.4          74.7       0.8X
-WEEK of date                                       1089           1100          17          9.2         108.9       0.6X
-DAY of date                                         795            803          13         12.6          79.5       0.8X
-DAYOFWEEK of date                                   937            944          10         10.7          93.7       0.7X
-DOW of date                                         927           1003          66         10.8          92.7       0.7X
-ISODOW of date                                      977            980           7         10.2          97.7       0.6X
-DOY of date                                         827            877          45         12.1          82.7       0.8X
-HOUR of date                                       1525           1547          34          6.6         152.5       0.4X
-MINUTE of date                                     1473           1499          23          6.8         147.3       0.4X
-SECOND of date                                     1600           1615          15          6.2         160.0       0.4X
-MILLISECONDS of date                               1666           1765         156          6.0         166.6       0.4X
-MICROSECONDS of date                               1554           1627         127          6.4         155.4       0.4X
-EPOCH of date                                      1615           1646          27          6.2         161.5       0.4X
+cast to date                                        629            634           6         15.9          62.9       1.0X
+MILLENNIUM of date                                  826            912          97         12.1          82.6       0.8X
+CENTURY of date                                     797            840          54         12.6          79.7       0.8X
+DECADE of date                                      734            737           4         13.6          73.4       0.9X
+YEAR of date                                        721            768          78         13.9          72.1       0.9X
+ISOYEAR of date                                     884            892          13         11.3          88.4       0.7X
+QUARTER of date                                     894            949          69         11.2          89.4       0.7X
+MONTH of date                                       726            729           2         13.8          72.6       0.9X
+WEEK of date                                       1013           1023           9          9.9         101.3       0.6X
+DAY of date                                         727            747          26         13.8          72.7       0.9X
+DAYOFWEEK of date                                   828            832           6         12.1          82.8       0.8X
+DOW of date                                         846            850           5         11.8          84.6       0.7X
+ISODOW of date                                      803            810           7         12.5          80.3       0.8X
+DOY of date                                         751            761          11         13.3          75.1       0.8X
+HOUR of date                                       1372           1376           5          7.3         137.2       0.5X
+MINUTE of date                                     1379           1398          20          7.3         137.9       0.5X
+SECOND of date                                     1530           1542          10          6.5         153.0       0.4X
+MILLISECONDS of date                               1532           1541           8          6.5         153.2       0.4X
+MICROSECONDS of date                               1442           1462          23          6.9         144.2       0.4X
+EPOCH of date                                      1573           1594          20          6.4         157.3       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        676            690          20         14.8          67.6       1.0X
-MILLENNIUM of date                                  868            882          16         11.5          86.8       0.8X
-CENTURY of date                                     875            880           8         11.4          87.5       0.8X
-DECADE of date                                      762            776          16         13.1          76.2       0.9X
-YEAR of date                                        788            800          11         12.7          78.8       0.9X
-ISOYEAR of date                                     903            917          12         11.1          90.3       0.7X
-QUARTER of date                                     983           1018          40         10.2          98.3       0.7X
-MONTH of date                                       836            857          19         12.0          83.6       0.8X
-WEEK of date                                       1137           1168          28          8.8         113.7       0.6X
-DAY of date                                         768            817          82         13.0          76.8       0.9X
-DAYOFWEEK of date                                   890            926          36         11.2          89.0       0.8X
-DOW of date                                        1007           1033          39          9.9         100.7       0.7X
-ISODOW of date                                      962            969           7         10.4          96.2       0.7X
-DOY of date                                         797            882          80         12.6          79.7       0.8X
-HOUR of date                                       1449           1482          29          6.9         144.9       0.5X
-MINUTE of date                                     1536           1610          69          6.5         153.6       0.4X
-SECOND of date                                     1675           1823         128          6.0         167.5       0.4X
-MILLISECONDS of date                               1605           1622          18          6.2         160.5       0.4X
-MICROSECONDS of date                               1481           1504          32          6.8         148.1       0.5X
-EPOCH of date                                      1656           1843         296          6.0         165.6       0.4X
+cast to date                                        622            633          14         16.1          62.2       1.0X
+MILLENNIUM of date                                  793            814          20         12.6          79.3       0.8X
+CENTURY of date                                     798            809          12         12.5          79.8       0.8X
+DECADE of date                                      735            740           6         13.6          73.5       0.8X
+YEAR of date                                        714            732          23         14.0          71.4       0.9X
+ISOYEAR of date                                     881            886           4         11.3          88.1       0.7X
+QUARTER of date                                     916            925          14         10.9          91.6       0.7X
+MONTH of date                                       732            737           5         13.7          73.2       0.8X
+WEEK of date                                       1022           1034          10          9.8         102.2       0.6X
+DAY of date                                         722            745          26         13.8          72.2       0.9X
+DAYOFWEEK of date                                   831            849          17         12.0          83.1       0.7X
+DOW of date                                         853            858           4         11.7          85.3       0.7X
+ISODOW of date                                      824            825           1         12.1          82.4       0.8X
+DOY of date                                         749            752           4         13.3          74.9       0.8X
+HOUR of date                                       1422           1425           3          7.0         142.2       0.4X
+MINUTE of date                                     1379           1394          13          7.3         137.9       0.5X
+SECOND of date                                     1525           1536           9          6.6         152.5       0.4X
+MILLISECONDS of date                               1542           1555          14          6.5         154.2       0.4X
+MICROSECONDS of date                               1446           1449           2          6.9         144.6       0.4X
+EPOCH of date                                      1589           1621          35          6.3         158.9       0.4X
+
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+cast to interval                                    913            921           8         11.0          91.3       1.0X
+MILLENNIUM of interval                              976            983          12         10.2          97.6       0.9X
+CENTURY of interval                                 976            979           3         10.2          97.6       0.9X
+DECADE of interval                                  975            987          10         10.3          97.5       0.9X
+YEAR of interval                                    964            968           5         10.4          96.4       0.9X
+QUARTER of interval                                 987            997          12         10.1          98.7       0.9X
+MONTH of interval                                   974            983          11         10.3          97.4       0.9X
+DAY of interval                                     959            962           3         10.4          95.9       1.0X
+HOUR of interval                                    972            986          13         10.3          97.2       0.9X
+MINUTE of interval                                  985            988           4         10.2          98.5       0.9X
+SECOND of interval                                 1128           1141          13          8.9         112.8       0.8X
+MILLISECONDS of interval                           1083           1086           3          9.2         108.3       0.8X
+MICROSECONDS of interval                            963            969           5         10.4          96.3       0.9X
+EPOCH of interval                                  1094           1116          23          9.1         109.4       0.8X
 
 Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    919            959          45         10.9          91.9       1.0X
-MILLENNIUM of interval                              959           1001          36         10.4          95.9       1.0X
-CENTURY of interval                                 974            998          21         10.3          97.4       0.9X
-DECADE of interval                                  975            989          13         10.3          97.5       0.9X
-YEAR of interval                                    987           1012          24         10.1          98.7       0.9X
-QUARTER of interval                                1014           1031          25          9.9         101.4       0.9X
-MONTH of interval                                   978           1018          45         10.2          97.8       0.9X
-DAY of interval                                    1012           1095          92          9.9         101.2       0.9X
-HOUR of interval                                   1129           1143          14          8.9         112.9       0.8X
-MINUTE of interval                                 1038           1060          25          9.6         103.8       0.9X
-SECOND of interval                                 1062           1289         356          9.4         106.2       0.9X
-MILLISECONDS of interval                           1142           1226          83          8.8         114.2       0.8X
-MICROSECONDS of interval                           1038           1129          79          9.6         103.8       0.9X
-EPOCH of interval                                  1090           1127          34          9.2         109.0       0.8X
+cast to interval                                    950            971          23         10.5          95.0       1.0X
+MILLENNIUM of interval                              989           1000          11         10.1          98.9       1.0X
+CENTURY of interval                                 984            990           6         10.2          98.4       1.0X
+DECADE of interval                                  962            978          14         10.4          96.2       1.0X
+YEAR of interval                                    945            963          16         10.6          94.5       1.0X
+QUARTER of interval                                 985            998          17         10.2          98.5       1.0X
+MONTH of interval                                   970            976           9         10.3          97.0       1.0X
+DAY of interval                                     953            962           8         10.5          95.3       1.0X
+HOUR of interval                                    942            956          17         10.6          94.2       1.0X
+MINUTE of interval                                  975            993          16         10.3          97.5       1.0X
+SECOND of interval                                 1110           1122          12          9.0         111.0       0.9X
+MILLISECONDS of interval                           1056           1074          22          9.5         105.6       0.9X
+MICROSECONDS of interval                            939            960          36         10.7          93.9       1.0X
+EPOCH of interval                                  1071           1093          21          9.3         107.1       0.9X
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
index 2d6cad8838704..4bcc15c875f57 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
@@ -1,43 +1,43 @@
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c;
+CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c, interval 10 year 20 month 30 day 40 hour 50 minute 6.7890 second as i;
 
-select extract(millennium from c) from t;
-select extract(millennia from c) from t;
-select extract(mil from c) from t;
-select extract(mils from c) from t;
+select extract(millennium from c), extract(millennium from i) from t;
+select extract(millennia from c), extract(millennia from i) from t;
+select extract(mil from c), extract(mil from i) from t;
+select extract(mils from c), extract(mils from i) from t;
 
-select extract(century from c) from t;
-select extract(centuries from c) from t;
-select extract(c from c) from t;
-select extract(cent from c) from t;
+select extract(century from c), extract(century from i) from t;
+select extract(centuries from c), extract(centuries from i) from t;
+select extract(c from c),  extract(c from i) from t;
+select extract(cent from c), extract(cent from i) from t;
 
-select extract(decade from c) from t;
-select extract(decades from c) from t;
-select extract(dec from c) from t;
-select extract(decs from c) from t;
+select extract(decade from c), extract(decade from i) from t;
+select extract(decades from c), extract(decades from i) from t;
+select extract(dec from c), extract(dec from i) from t;
+select extract(decs from c), extract(decs from i) from t;
 
-select extract(year from c) from t;
-select extract(y from c) from t;
-select extract(years from c) from t;
-select extract(yr from c) from t;
-select extract(yrs from c) from t;
+select extract(year from c), extract(year from i) from t;
+select extract(y from c), extract(y from i) from t;
+select extract(years from c), extract(years from i) from t;
+select extract(yr from c), extract(yr from i) from t;
+select extract(yrs from c), extract(yrs from i) from t;
 
 select extract(isoyear from c) from t;
 
-select extract(quarter from c) from t;
-select extract(qtr from c) from t;
+select extract(quarter from c), extract(quarter from i) from t;
+select extract(qtr from c), extract(qtr from i) from t;
 
-select extract(month from c) from t;
-select extract(mon from c) from t;
-select extract(mons from c) from t;
-select extract(months from c) from t;
+select extract(month from c), extract(month from i) from t;
+select extract(mon from c), extract(mon from i) from t;
+select extract(mons from c), extract(mons from i) from t;
+select extract(months from c), extract(months from i) from t;
 
 select extract(week from c) from t;
 select extract(w from c) from t;
 select extract(weeks from c) from t;
 
-select extract(day from c) from t;
-select extract(d from c) from t;
-select extract(days from c) from t;
+select extract(day from c), extract(day from i) from t;
+select extract(d from c), extract(d from i) from t;
+select extract(days from c), extract(days from i) from t;
 
 select extract(dayofweek from c) from t;
 
@@ -47,38 +47,39 @@ select extract(isodow from c) from t;
 
 select extract(doy from c) from t;
 
-select extract(hour from c) from t;
-select extract(h from c) from t;
-select extract(hours from c) from t;
-select extract(hr from c) from t;
-select extract(hrs from c) from t;
-
-select extract(minute from c) from t;
-select extract(m from c) from t;
-select extract(min from c) from t;
-select extract(mins from c) from t;
-select extract(minutes from c) from t;
-
-select extract(second from c) from t;
-select extract(s from c) from t;
-select extract(sec from c) from t;
-select extract(seconds from c) from t;
-select extract(secs from c) from t;
-
-select extract(milliseconds from c) from t;
-select extract(msec from c) from t;
-select extract(msecs from c) from t;
-select extract(millisecon from c) from t;
-select extract(mseconds from c) from t;
-select extract(ms from c) from t;
-
-select extract(microseconds from c) from t;
-select extract(usec from c) from t;
-select extract(usecs from c) from t;
-select extract(useconds from c) from t;
-select extract(microsecon from c) from t;
-select extract(us from c) from t;
-
-select extract(epoch from c) from t;
+select extract(hour from c), extract(hour from i) from t;
+select extract(h from c), extract(h from i) from t;
+select extract(hours from c), extract(hours from i) from t;
+select extract(hr from c), extract(hr from i) from t;
+select extract(hrs from c), extract(hrs from i) from t;
+
+select extract(minute from c), extract(minute from i) from t;
+select extract(m from c), extract(m from i) from t;
+select extract(min from c), extract(min from i) from t;
+select extract(mins from c), extract(mins from i) from t;
+select extract(minutes from c), extract(minutes from i) from t;
+
+select extract(second from c), extract(second from i) from t;
+select extract(s from c), extract(s from i) from t;
+select extract(sec from c), extract(sec from i) from t;
+select extract(seconds from c), extract(seconds from i) from t;
+select extract(secs from c), extract(secs from i) from t;
+
+select extract(milliseconds from c), extract(milliseconds from i) from t;
+select extract(msec from c), extract(msec from i) from t;
+select extract(msecs from c), extract(msecs from i) from t;
+select extract(millisecon from c), extract(millisecon from i) from t;
+select extract(mseconds from c), extract(mseconds from i) from t;
+select extract(ms from c), extract(ms from i) from t;
+
+select extract(microseconds from c), extract(microseconds from i) from t;
+select extract(usec from c), extract(usec from i) from t;
+select extract(usecs from c), extract(usecs from i) from t;
+select extract(useconds from c), extract(useconds from i) from t;
+select extract(microsecon from c), extract(microsecon from i) from t;
+select extract(us from c), extract(us from i) from t;
+
+select extract(epoch from c), extract(epoch from i) from t;
 
 select extract(not_supported from c) from t;
+select extract(not_supported from i) from t;
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 583459f9037b8..38c8490694088 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -1,9 +1,9 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 64
+-- Number of queries: 65
 
 
 -- !query
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c
+CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c, interval 10 year 20 month 30 day 40 hour 50 minute 6.7890 second as i
 -- !query schema
 struct<>
 -- !query output
@@ -11,201 +11,201 @@ struct<>
 
 
 -- !query
-select extract(millennium from c) from t
+select extract(millennium from c), extract(millennium from i) from t
 -- !query schema
-struct<date_part('millennium', `c`):int>
+struct<date_part('millennium', t.`c`):int,date_part('millennium', t.`i`):int>
 -- !query output
-3
+3	0
 
 
 -- !query
-select extract(millennia from c) from t
+select extract(millennia from c), extract(millennia from i) from t
 -- !query schema
-struct<date_part('millennia', `c`):int>
+struct<date_part('millennia', t.`c`):int,date_part('millennia', t.`i`):int>
 -- !query output
-3
+3	0
 
 
 -- !query
-select extract(mil from c) from t
+select extract(mil from c), extract(mil from i) from t
 -- !query schema
-struct<date_part('mil', `c`):int>
+struct<date_part('mil', t.`c`):int,date_part('mil', t.`i`):int>
 -- !query output
-3
+3	0
 
 
 -- !query
-select extract(mils from c) from t
+select extract(mils from c), extract(mils from i) from t
 -- !query schema
-struct<date_part('mils', `c`):int>
+struct<date_part('mils', t.`c`):int,date_part('mils', t.`i`):int>
 -- !query output
-3
+3	0
 
 
 -- !query
-select extract(century from c) from t
+select extract(century from c), extract(century from i) from t
 -- !query schema
-struct<date_part('century', `c`):int>
+struct<date_part('century', t.`c`):int,date_part('century', t.`i`):int>
 -- !query output
-21
+21	0
 
 
 -- !query
-select extract(centuries from c) from t
+select extract(centuries from c), extract(centuries from i) from t
 -- !query schema
-struct<date_part('centuries', `c`):int>
+struct<date_part('centuries', t.`c`):int,date_part('centuries', t.`i`):int>
 -- !query output
-21
+21	0
 
 
 -- !query
-select extract(c from c) from t
+select extract(c from c),  extract(c from i) from t
 -- !query schema
-struct<date_part('c', `c`):int>
+struct<date_part('c', t.`c`):int,date_part('c', t.`i`):int>
 -- !query output
-21
+21	0
 
 
 -- !query
-select extract(cent from c) from t
+select extract(cent from c), extract(cent from i) from t
 -- !query schema
-struct<date_part('cent', `c`):int>
+struct<date_part('cent', t.`c`):int,date_part('cent', t.`i`):int>
 -- !query output
-21
+21	0
 
 
 -- !query
-select extract(decade from c) from t
+select extract(decade from c), extract(decade from i) from t
 -- !query schema
-struct<date_part('decade', `c`):int>
+struct<date_part('decade', t.`c`):int,date_part('decade', t.`i`):int>
 -- !query output
-201
+201	1
 
 
 -- !query
-select extract(decades from c) from t
+select extract(decades from c), extract(decades from i) from t
 -- !query schema
-struct<date_part('decades', `c`):int>
+struct<date_part('decades', t.`c`):int,date_part('decades', t.`i`):int>
 -- !query output
-201
+201	1
 
 
 -- !query
-select extract(dec from c) from t
+select extract(dec from c), extract(dec from i) from t
 -- !query schema
-struct<date_part('dec', `c`):int>
+struct<date_part('dec', t.`c`):int,date_part('dec', t.`i`):int>
 -- !query output
-201
+201	1
 
 
 -- !query
-select extract(decs from c) from t
+select extract(decs from c), extract(decs from i) from t
 -- !query schema
-struct<date_part('decs', `c`):int>
+struct<date_part('decs', t.`c`):int,date_part('decs', t.`i`):int>
 -- !query output
-201
+201	1
 
 
 -- !query
-select extract(year from c) from t
+select extract(year from c), extract(year from i) from t
 -- !query schema
-struct<date_part('year', `c`):int>
+struct<date_part('year', t.`c`):int,date_part('year', t.`i`):int>
 -- !query output
-2011
+2011	11
 
 
 -- !query
-select extract(y from c) from t
+select extract(y from c), extract(y from i) from t
 -- !query schema
-struct<date_part('y', `c`):int>
+struct<date_part('y', t.`c`):int,date_part('y', t.`i`):int>
 -- !query output
-2011
+2011	11
 
 
 -- !query
-select extract(years from c) from t
+select extract(years from c), extract(years from i) from t
 -- !query schema
-struct<date_part('years', `c`):int>
+struct<date_part('years', t.`c`):int,date_part('years', t.`i`):int>
 -- !query output
-2011
+2011	11
 
 
 -- !query
-select extract(yr from c) from t
+select extract(yr from c), extract(yr from i) from t
 -- !query schema
-struct<date_part('yr', `c`):int>
+struct<date_part('yr', t.`c`):int,date_part('yr', t.`i`):int>
 -- !query output
-2011
+2011	11
 
 
 -- !query
-select extract(yrs from c) from t
+select extract(yrs from c), extract(yrs from i) from t
 -- !query schema
-struct<date_part('yrs', `c`):int>
+struct<date_part('yrs', t.`c`):int,date_part('yrs', t.`i`):int>
 -- !query output
-2011
+2011	11
 
 
 -- !query
 select extract(isoyear from c) from t
 -- !query schema
-struct<date_part('isoyear', `c`):int>
+struct<date_part('isoyear', t.`c`):int>
 -- !query output
 2011
 
 
 -- !query
-select extract(quarter from c) from t
+select extract(quarter from c), extract(quarter from i) from t
 -- !query schema
-struct<date_part('quarter', `c`):int>
+struct<date_part('quarter', t.`c`):int,date_part('quarter', t.`i`):tinyint>
 -- !query output
-2
+2	3
 
 
 -- !query
-select extract(qtr from c) from t
+select extract(qtr from c), extract(qtr from i) from t
 -- !query schema
-struct<date_part('qtr', `c`):int>
+struct<date_part('qtr', t.`c`):int,date_part('qtr', t.`i`):tinyint>
 -- !query output
-2
+2	3
 
 
 -- !query
-select extract(month from c) from t
+select extract(month from c), extract(month from i) from t
 -- !query schema
-struct<date_part('month', `c`):int>
+struct<date_part('month', t.`c`):int,date_part('month', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(mon from c) from t
+select extract(mon from c), extract(mon from i) from t
 -- !query schema
-struct<date_part('mon', `c`):int>
+struct<date_part('mon', t.`c`):int,date_part('mon', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(mons from c) from t
+select extract(mons from c), extract(mons from i) from t
 -- !query schema
-struct<date_part('mons', `c`):int>
+struct<date_part('mons', t.`c`):int,date_part('mons', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
-select extract(months from c) from t
+select extract(months from c), extract(months from i) from t
 -- !query schema
-struct<date_part('months', `c`):int>
+struct<date_part('months', t.`c`):int,date_part('months', t.`i`):tinyint>
 -- !query output
-5
+5	8
 
 
 -- !query
 select extract(week from c) from t
 -- !query schema
-struct<date_part('week', `c`):int>
+struct<date_part('week', t.`c`):int>
 -- !query output
 18
 
@@ -213,7 +213,7 @@ struct<date_part('week', `c`):int>
 -- !query
 select extract(w from c) from t
 -- !query schema
-struct<date_part('w', `c`):int>
+struct<date_part('w', t.`c`):int>
 -- !query output
 18
 
@@ -221,39 +221,39 @@ struct<date_part('w', `c`):int>
 -- !query
 select extract(weeks from c) from t
 -- !query schema
-struct<date_part('weeks', `c`):int>
+struct<date_part('weeks', t.`c`):int>
 -- !query output
 18
 
 
 -- !query
-select extract(day from c) from t
+select extract(day from c), extract(day from i) from t
 -- !query schema
-struct<date_part('day', `c`):int>
+struct<date_part('day', t.`c`):int,date_part('day', t.`i`):int>
 -- !query output
-6
+6	30
 
 
 -- !query
-select extract(d from c) from t
+select extract(d from c), extract(d from i) from t
 -- !query schema
-struct<date_part('d', `c`):int>
+struct<date_part('d', t.`c`):int,date_part('d', t.`i`):int>
 -- !query output
-6
+6	30
 
 
 -- !query
-select extract(days from c) from t
+select extract(days from c), extract(days from i) from t
 -- !query schema
-struct<date_part('days', `c`):int>
+struct<date_part('days', t.`c`):int,date_part('days', t.`i`):int>
 -- !query output
-6
+6	30
 
 
 -- !query
 select extract(dayofweek from c) from t
 -- !query schema
-struct<date_part('dayofweek', `c`):int>
+struct<date_part('dayofweek', t.`c`):int>
 -- !query output
 6
 
@@ -261,7 +261,7 @@ struct<date_part('dayofweek', `c`):int>
 -- !query
 select extract(dow from c) from t
 -- !query schema
-struct<date_part('dow', `c`):int>
+struct<date_part('dow', t.`c`):int>
 -- !query output
 5
 
@@ -269,7 +269,7 @@ struct<date_part('dow', `c`):int>
 -- !query
 select extract(isodow from c) from t
 -- !query schema
-struct<date_part('isodow', `c`):int>
+struct<date_part('isodow', t.`c`):int>
 -- !query output
 5
 
@@ -277,233 +277,233 @@ struct<date_part('isodow', `c`):int>
 -- !query
 select extract(doy from c) from t
 -- !query schema
-struct<date_part('doy', `c`):int>
+struct<date_part('doy', t.`c`):int>
 -- !query output
 126
 
 
 -- !query
-select extract(hour from c) from t
+select extract(hour from c), extract(hour from i) from t
 -- !query schema
-struct<date_part('hour', `c`):int>
+struct<date_part('hour', t.`c`):int,date_part('hour', t.`i`):bigint>
 -- !query output
-7
+7	40
 
 
 -- !query
-select extract(h from c) from t
+select extract(h from c), extract(h from i) from t
 -- !query schema
-struct<date_part('h', `c`):int>
+struct<date_part('h', t.`c`):int,date_part('h', t.`i`):bigint>
 -- !query output
-7
+7	40
 
 
 -- !query
-select extract(hours from c) from t
+select extract(hours from c), extract(hours from i) from t
 -- !query schema
-struct<date_part('hours', `c`):int>
+struct<date_part('hours', t.`c`):int,date_part('hours', t.`i`):bigint>
 -- !query output
-7
+7	40
 
 
 -- !query
-select extract(hr from c) from t
+select extract(hr from c), extract(hr from i) from t
 -- !query schema
-struct<date_part('hr', `c`):int>
+struct<date_part('hr', t.`c`):int,date_part('hr', t.`i`):bigint>
 -- !query output
-7
+7	40
 
 
 -- !query
-select extract(hrs from c) from t
+select extract(hrs from c), extract(hrs from i) from t
 -- !query schema
-struct<date_part('hrs', `c`):int>
+struct<date_part('hrs', t.`c`):int,date_part('hrs', t.`i`):bigint>
 -- !query output
-7
+7	40
 
 
 -- !query
-select extract(minute from c) from t
+select extract(minute from c), extract(minute from i) from t
 -- !query schema
-struct<date_part('minute', `c`):int>
+struct<date_part('minute', t.`c`):int,date_part('minute', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(m from c) from t
+select extract(m from c), extract(m from i) from t
 -- !query schema
-struct<date_part('m', `c`):int>
+struct<date_part('m', t.`c`):int,date_part('m', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(min from c) from t
+select extract(min from c), extract(min from i) from t
 -- !query schema
-struct<date_part('min', `c`):int>
+struct<date_part('min', t.`c`):int,date_part('min', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(mins from c) from t
+select extract(mins from c), extract(mins from i) from t
 -- !query schema
-struct<date_part('mins', `c`):int>
+struct<date_part('mins', t.`c`):int,date_part('mins', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(minutes from c) from t
+select extract(minutes from c), extract(minutes from i) from t
 -- !query schema
-struct<date_part('minutes', `c`):int>
+struct<date_part('minutes', t.`c`):int,date_part('minutes', t.`i`):tinyint>
 -- !query output
-8
+8	50
 
 
 -- !query
-select extract(second from c) from t
+select extract(second from c), extract(second from i) from t
 -- !query schema
-struct<date_part('second', `c`):decimal(8,6)>
+struct<date_part('second', t.`c`):decimal(8,6),date_part('second', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(s from c) from t
+select extract(s from c), extract(s from i) from t
 -- !query schema
-struct<date_part('s', `c`):decimal(8,6)>
+struct<date_part('s', t.`c`):decimal(8,6),date_part('s', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(sec from c) from t
+select extract(sec from c), extract(sec from i) from t
 -- !query schema
-struct<date_part('sec', `c`):decimal(8,6)>
+struct<date_part('sec', t.`c`):decimal(8,6),date_part('sec', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(seconds from c) from t
+select extract(seconds from c), extract(seconds from i) from t
 -- !query schema
-struct<date_part('seconds', `c`):decimal(8,6)>
+struct<date_part('seconds', t.`c`):decimal(8,6),date_part('seconds', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(secs from c) from t
+select extract(secs from c), extract(secs from i) from t
 -- !query schema
-struct<date_part('secs', `c`):decimal(8,6)>
+struct<date_part('secs', t.`c`):decimal(8,6),date_part('secs', t.`i`):decimal(8,6)>
 -- !query output
-9.123456
+9.123456	6.789000
 
 
 -- !query
-select extract(milliseconds from c) from t
+select extract(milliseconds from c), extract(milliseconds from i) from t
 -- !query schema
-struct<date_part('milliseconds', `c`):decimal(8,3)>
+struct<date_part('milliseconds', t.`c`):decimal(8,3),date_part('milliseconds', t.`i`):decimal(8,3)>
 -- !query output
-9123.456
+9123.456	6789.000
 
 
 -- !query
-select extract(msec from c) from t
+select extract(msec from c), extract(msec from i) from t
 -- !query schema
-struct<date_part('msec', `c`):decimal(8,3)>
+struct<date_part('msec', t.`c`):decimal(8,3),date_part('msec', t.`i`):decimal(8,3)>
 -- !query output
-9123.456
+9123.456	6789.000
 
 
 -- !query
-select extract(msecs from c) from t
+select extract(msecs from c), extract(msecs from i) from t
 -- !query schema
-struct<date_part('msecs', `c`):decimal(8,3)>
+struct<date_part('msecs', t.`c`):decimal(8,3),date_part('msecs', t.`i`):decimal(8,3)>
 -- !query output
-9123.456
+9123.456	6789.000
 
 
 -- !query
-select extract(millisecon from c) from t
+select extract(millisecon from c), extract(millisecon from i) from t
 -- !query schema
-struct<date_part('millisecon', `c`):decimal(8,3)>
+struct<date_part('millisecon', t.`c`):decimal(8,3),date_part('millisecon', t.`i`):decimal(8,3)>
 -- !query output
-9123.456
+9123.456	6789.000
 
 
 -- !query
-select extract(mseconds from c) from t
+select extract(mseconds from c), extract(mseconds from i) from t
 -- !query schema
-struct<date_part('mseconds', `c`):decimal(8,3)>
+struct<date_part('mseconds', t.`c`):decimal(8,3),date_part('mseconds', t.`i`):decimal(8,3)>
 -- !query output
-9123.456
+9123.456	6789.000
 
 
 -- !query
-select extract(ms from c) from t
+select extract(ms from c), extract(ms from i) from t
 -- !query schema
-struct<date_part('ms', `c`):decimal(8,3)>
+struct<date_part('ms', t.`c`):decimal(8,3),date_part('ms', t.`i`):decimal(8,3)>
 -- !query output
-9123.456
+9123.456	6789.000
 
 
 -- !query
-select extract(microseconds from c) from t
+select extract(microseconds from c), extract(microseconds from i) from t
 -- !query schema
-struct<date_part('microseconds', `c`):int>
+struct<date_part('microseconds', t.`c`):int,date_part('microseconds', t.`i`):bigint>
 -- !query output
-9123456
+9123456	6789000
 
 
 -- !query
-select extract(usec from c) from t
+select extract(usec from c), extract(usec from i) from t
 -- !query schema
-struct<date_part('usec', `c`):int>
+struct<date_part('usec', t.`c`):int,date_part('usec', t.`i`):bigint>
 -- !query output
-9123456
+9123456	6789000
 
 
 -- !query
-select extract(usecs from c) from t
+select extract(usecs from c), extract(usecs from i) from t
 -- !query schema
-struct<date_part('usecs', `c`):int>
+struct<date_part('usecs', t.`c`):int,date_part('usecs', t.`i`):bigint>
 -- !query output
-9123456
+9123456	6789000
 
 
 -- !query
-select extract(useconds from c) from t
+select extract(useconds from c), extract(useconds from i) from t
 -- !query schema
-struct<date_part('useconds', `c`):int>
+struct<date_part('useconds', t.`c`):int,date_part('useconds', t.`i`):bigint>
 -- !query output
-9123456
+9123456	6789000
 
 
 -- !query
-select extract(microsecon from c) from t
+select extract(microsecon from c), extract(microsecon from i) from t
 -- !query schema
-struct<date_part('microsecon', `c`):int>
+struct<date_part('microsecon', t.`c`):int,date_part('microsecon', t.`i`):bigint>
 -- !query output
-9123456
+9123456	6789000
 
 
 -- !query
-select extract(us from c) from t
+select extract(us from c), extract(us from i) from t
 -- !query schema
-struct<date_part('us', `c`):int>
+struct<date_part('us', t.`c`):int,date_part('us', t.`i`):bigint>
 -- !query output
-9123456
+9123456	6789000
 
 
 -- !query
-select extract(epoch from c) from t
+select extract(epoch from c), extract(epoch from i) from t
 -- !query schema
-struct<date_part('epoch', `c`):decimal(20,6)>
+struct<date_part('epoch', t.`c`):decimal(20,6),date_part('epoch', t.`i`):decimal(18,6)>
 -- !query output
-1304665689.123456
+1304665689.123456	370608606.789000
 
 
 -- !query
@@ -511,10 +511,14 @@ select extract(not_supported from c) from t
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
 
-Literals of type 'not_supported' are currently not supported.(line 1, pos 7)
 
-== SQL ==
-select extract(not_supported from c) from t
--------^^^
+-- !query
+select extract(not_supported from i) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index ed27317121623..e5a7c43c0a8ab 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -590,7 +590,7 @@ struct<date_part('EPOCH', TIMESTAMP '1970-01-01 00:00:00'):decimal(20,6)>
 -- !query
 SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('CENTURY', TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('CENTURY', to_date('0101-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -2
 
@@ -598,7 +598,7 @@ struct<date_part('CENTURY', TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('CENTURY', TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('CENTURY', to_date('0100-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -606,7 +606,7 @@ struct<date_part('CENTURY', TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('CENTURY', TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('CENTURY', to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -670,7 +670,7 @@ true
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('MILLENNIUM', TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('MILLENNIUM', to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -750,7 +750,7 @@ struct<date_part('DECADE', DATE '0009-12-31'):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('DECADE', to_date('0001-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 0
 
@@ -758,7 +758,7 @@ struct<date_part('DECADE', TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('DECADE', to_date('0002-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -766,7 +766,7 @@ struct<date_part('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('DECADE', to_date('0011-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -774,7 +774,7 @@ struct<date_part('DECADE', TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<date_part('DECADE', to_date('0012-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -2
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
index f1315d48e4d99..285211e918d68 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
@@ -84,8 +84,6 @@ object ExtractBenchmark extends SqlBasedBenchmark {
     }
   }
 
-  private case class Settings(fields: Seq[String], func: Seq[String], iterNum: Long)
-
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     val N = 10000000L
     val datetimeFields = Seq(
@@ -100,18 +98,16 @@ object ExtractBenchmark extends SqlBasedBenchmark {
       "HOUR", "MINUTE", "SECOND",
       "MILLISECONDS", "MICROSECONDS", "EPOCH")
     val settings = Map(
-      "timestamp" -> Settings(datetimeFields, Seq("extract", "date_part"), N),
-      "date" -> Settings(datetimeFields, Seq("extract", "date_part"), N),
-      "interval" -> Settings(intervalFields, Seq("date_part"), N))
+      "timestamp" -> datetimeFields,
+      "date" -> datetimeFields,
+      "interval" -> intervalFields)
 
-    for {
-      (dataType, Settings(fields, funcs, iterNum)) <- settings
-      func <- funcs} {
+    for {(dataType, fields) <- settings; func <- Seq("extract", "date_part")} {
 
       val benchmark = new Benchmark(s"Invoke $func for $dataType", N, output = output)
 
-      run(benchmark, iterNum, s"cast to $dataType", castExpr(dataType))
-      fields.foreach(run(benchmark, func, iterNum, _, dataType))
+      run(benchmark, N, s"cast to $dataType", castExpr(dataType))
+      fields.foreach(run(benchmark, func, N, _, dataType))
 
       benchmark.run()
     }

From 7c6ec51828656979cc3300a8badd0ae1204d9dd3 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 18 Mar 2020 20:19:50 +0800
Subject: [PATCH 0301/1280] [SPARK-31176][SQL] Remove support for 'e'/'c' as
 datetime pattern charactar

### What changes were proposed in this pull request?

The meaning of 'u' was day number of the week in SimpleDateFormat, it was changed to year in DateTimeFormatter. Now we keep the old meaning of 'u' by substituting 'u' to 'e' internally and use DateTimeFormatter to parse the pattern string. In DateTimeFormatter, the 'e' and 'c' also represents day-of-week. e.g.

```sql
select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
select date_format(timestamp '2019-10-06', 'yyyy-MM-dd eeee');
```
Because of the substitution, they all goes to `.... eeee` silently. The users may congitive problems of their meanings, so we should mark them as illegal pattern characters to stay the same as before.

This pr move the method `convertIncompatiblePattern` from `DatetimeUtils` to `DateTimeFormatterHelper` object, since it is quite specific for `DateTimeFormatterHelper` class.
And 'e' and 'c' char checking in this method.

Besides,`convertIncompatiblePattern` has a bug that will lose the last `'` if it ends with it, this pr fixes this too. e.g.

```sql
spark-sql> select date_format(timestamp "2019-10-06", "yyyy-MM-dd'S'");
20/03/18 11:19:45 ERROR SparkSQLDriver: Failed in [select date_format(timestamp "2019-10-06", "yyyy-MM-dd'S'")]
java.lang.IllegalArgumentException: Pattern ends with an incomplete string literal: uuuu-MM-dd'S

spark-sql> select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'");
NULL
```
### Why are the changes needed?

avoid vagueness
bug fix

### Does this PR introduce any user-facing change?

no, these are not  exposed yet

### How was this patch tested?

add ut

Closes #27939 from yaooqinn/SPARK-31176.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 57fcc49306296b474c5b8b685ad13082f9b50a49)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              |  2 +-
 .../util/DateTimeFormatterHelper.scala        | 43 ++++++++++++++++-
 .../sql/catalyst/util/DateTimeUtils.scala     | 38 ---------------
 .../util/DateTimeFormatterHelperSuite.scala   | 48 +++++++++++++++++++
 .../catalyst/util/DateTimeUtilsSuite.scala    | 16 -------
 .../resources/sql-tests/inputs/datetime.sql   |  7 +++
 .../sql-tests/results/datetime.sql.out        | 44 ++++++++++++++++-
 7 files changed, 141 insertions(+), 57 deletions(-)
 create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 429d781248ef2..f5c20eacae654 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -88,7 +88,7 @@ Spark uses the below letters in date and timestamp parsing and formatting:
   <td> Tue; Tuesday; T </td>
 </tr>
 <tr>
-  <td> <b>e</b> </td>
+  <td> <b>u</b> </td>
   <td> localized day-of-week </td>
   <td> number/text </td>
   <td> 2; 02; Tue; Tuesday; T </td>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 668ce9239b699..72bae28bc9148 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -55,7 +55,7 @@ trait DateTimeFormatterHelper {
       pattern: String,
       locale: Locale,
       needVarLengthSecondFraction: Boolean = false): DateTimeFormatter = {
-    val newPattern = DateTimeUtils.convertIncompatiblePattern(pattern)
+    val newPattern = convertIncompatiblePattern(pattern)
     val useVarLen = needVarLengthSecondFraction && newPattern.contains('S')
     val key = (newPattern, locale, useVarLen)
     var formatter = cache.getIfPresent(key)
@@ -157,4 +157,45 @@ private object DateTimeFormatterHelper {
       .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
     toFormatter(builder, TimestampFormatter.defaultLocale)
   }
+
+  /**
+   * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
+   * parsing/formatting datetime values. The pattern string is incompatible with the one defined
+   * by SimpleDateFormat in Spark 2.4 and earlier. This function converts all incompatible pattern
+   * for the new parser in Spark 3.0. See more details in SPARK-31030.
+   * @param pattern The input pattern.
+   * @return The pattern for new parser
+   */
+  def convertIncompatiblePattern(pattern: String): String = {
+    val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
+      case (patternPart, index) =>
+        // Text can be quoted using single quotes, we only check the non-quote parts.
+        index % 2 == 0 && patternPart.contains("G")
+    }
+    (pattern + " ").split("'").zipWithIndex.map {
+      case (patternPart, index) =>
+        if (index % 2 == 0) {
+          for (c <- patternPart if c == 'c' || c == 'e') {
+            throw new IllegalArgumentException(s"Illegal pattern character: $c")
+          }
+          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
+          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
+          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
+          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
+          // exception and ask users to change the pattern strings or turn on the legacy mode;
+          // otherwise, return NULL as what Spark 2.4 does.
+          val res = patternPart.replace("u", "e")
+          // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
+          // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
+          // We only do this substitution when there is no era designator found in the pattern.
+          if (!eraDesignatorContained) {
+            res.replace("y", "u")
+          } else {
+            res
+          }
+        } else {
+          patternPart
+        }
+    }.mkString("'").stripSuffix(" ")
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index e1cc89d03c069..a8c55e75e892f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -942,42 +942,4 @@ object DateTimeUtils {
     val days = period.getDays
     new CalendarInterval(months, days, 0)
   }
-
-  /**
-   * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
-   * parsing/formatting datetime values. The pattern string is incompatible with the one defined
-   * by SimpleDateFormat in Spark 2.4 and earlier. This function converts all incompatible pattern
-   * for the new parser in Spark 3.0. See more details in SPARK-31030.
-   * @param pattern The input pattern.
-   * @return The pattern for new parser
-   */
-  def convertIncompatiblePattern(pattern: String): String = {
-    val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
-      case (patternPart, index) =>
-        // Text can be quoted using single quotes, we only check the non-quote parts.
-        index % 2 == 0 && patternPart.contains("G")
-    }
-    pattern.split("'").zipWithIndex.map {
-      case (patternPart, index) =>
-        if (index % 2 == 0) {
-          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
-          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
-          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
-          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
-          // exception and ask users to change the pattern strings or turn on the legacy mode;
-          // otherwise, return NULL as what Spark 2.4 does.
-          val res = patternPart.replace("u", "e")
-          // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
-          // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
-          // We only do this substitution when there is no era designator found in the pattern.
-          if (!eraDesignatorContained) {
-            res.replace("y", "u")
-          } else {
-            res
-          }
-        } else {
-          patternPart
-        }
-    }.mkString("'")
-  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
new file mode 100644
index 0000000000000..811c4daa1b0e4
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper.convertIncompatiblePattern
+
+class DateTimeFormatterHelperSuite extends SparkFunSuite {
+
+  test("check incompatible pattern") {
+    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
+      === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
+    assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
+      === "uuuu-MM'y contains in quoted text'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
+      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
+    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
+      === "uuuu-MM'u contains in quoted text'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
+      === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+      === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+    val e1 = intercept[IllegalArgumentException](convertIncompatiblePattern("yyyy-MM-dd eeee G"))
+    assert(e1.getMessage === "Illegal pattern character: e")
+    val e2 = intercept[IllegalArgumentException](convertIncompatiblePattern("yyyy-MM-dd cccc G"))
+    assert(e2.getMessage === "Illegal pattern character: c")
+    assert(convertIncompatiblePattern("yyyy-MM-dd uuuu") === "uuuu-MM-dd eeee")
+    assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
+    assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'")
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index a88a8dda4456c..0df395b3cbb1c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -669,20 +669,4 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       assert(toDate("tomorrow CET ", zoneId).get === today + 1)
     }
   }
-
-  test("check incompatible pattern") {
-    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
-    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
-    assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
-      === "uuuu-MM'y contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
-    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
-      === "uuuu-MM'u contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
-      === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
-      === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
-  }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 9026427480d36..a06cdfda89ee1 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -100,3 +100,10 @@ select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm");
 select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm");
 select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm");
 select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm");
+
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'");
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
+
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc');
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 7258d009acfbe..714412f3f2bc9 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 68
+-- Number of queries: 73
 
 
 -- !query
@@ -559,3 +559,45 @@ select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
 struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
 -- !query output
 0019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp('2019-10-06S', 'yyyy-MM-dd\'S\''):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: e
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: c
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+-- !query schema
+struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+-- !query output
+2019-10-06 Sunday

From 4e3155546be6f0e4c244295b91164b112fc6c522 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Wed, 18 Mar 2020 13:21:24 -0500
Subject: [PATCH 0302/1280] [SPARK-30933][ML][DOCS] ML, GraphX 3.0 QA: Update
 user guide for new features & APIs

### What changes were proposed in this pull request?
Change ml-tuning.html.

### Why are the changes needed?
Add description for ```MultilabelClassificationEvaluator``` and ```RankingEvaluator```.

### Does this PR introduce any user-facing change?
Yes

before:
![image](https://user-images.githubusercontent.com/13592258/76437013-2c5ffb80-6376-11ea-8946-f5c2e7379b7c.png)

after:
![image](https://user-images.githubusercontent.com/13592258/76437054-397cea80-6376-11ea-867f-fe8d8fa4e5b3.png)

### How was this patch tested?

Closes #27880 from huaxingao/spark-30933.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit d22c9f6c0de76b7f3df7c9c98fdb0303c29e8e73)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/ml-tuning.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/ml-tuning.md b/docs/ml-tuning.md
index 49e23684e5974..274f1951011bd 100644
--- a/docs/ml-tuning.md
+++ b/docs/ml-tuning.md
@@ -65,9 +65,11 @@ At a high level, these model selection tools work as follows:
 
 The `Evaluator` can be a [`RegressionEvaluator`](api/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.html)
 for regression problems, a [`BinaryClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.html)
-for binary data, or a [`MulticlassClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.html)
-for multiclass problems. The default metric used to choose the best `ParamMap` can be overridden by the `setMetricName`
-method in each of these evaluators.
+for binary data, a [`MulticlassClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.html)
+for multiclass problems, a [`MultilabelClassificationEvaluator`](api/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.html)
+ for multi-label classifications, or a
+[`RankingEvaluator`](api/scala/org/apache/spark/ml/evaluation/RankingEvaluator.html) for ranking problems. The default metric used to
+choose the best `ParamMap` can be overridden by the `setMetricName` method in each of these evaluators.
 
 To help construct the parameter grid, users can use the [`ParamGridBuilder`](api/scala/org/apache/spark/ml/tuning/ParamGridBuilder.html) utility.
 By default, sets of parameters from the parameter grid are evaluated in serial. Parameter evaluation can be done in parallel by setting `parallelism` with a value of 2 or more (a value of 1 will be serial) before running model selection with `CrossValidator` or `TrainValidationSplit`.

From aba7a09da53425481893ce6d21281dc85874c619 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 19 Mar 2020 07:27:06 +0900
Subject: [PATCH 0303/1280] [SPARK-31150][SQL][FOLLOWUP] handle ' as escape for
 text

### What changes were proposed in this pull request?

pattern `''` means literal `'`

```sql
select date_format(to_timestamp("11111904-01-23 15:02:01", 'y-MM-dd HH:mm:ss'), "y-MM-dd HH:mm:ss''SSSSSSSSS");
5377-02-14 06:27:19'000000519
```
https://github.com/apache/spark/commit/0946a9514f56565c78b0555383c1ece14aaf2b7b missed this case and this pr add it back.

### Why are the changes needed?

bugfix

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

add ut

Closes #27949 from yaooqinn/SPARK-31150-2.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 3d695954e53038f978bebcb3e4444798fa8728d1)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../util/DateTimeFormatterHelper.scala        |  4 +++
 .../resources/sql-tests/inputs/datetime.sql   |  5 +++
 .../sql-tests/results/datetime.sql.out        | 34 ++++++++++++++++++-
 3 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 72bae28bc9148..4ed618ec454b0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -117,6 +117,10 @@ private object DateTimeFormatterHelper {
       pattern: String): DateTimeFormatterBuilder = {
     val builder = createBuilder()
     pattern.split("'").zipWithIndex.foreach {
+      // Split string starting with the regex itself which is `'` here will produce an extra empty
+      // string at res(0). So when the first element here is empty string we do not need append `'`
+      // literal to the DateTimeFormatterBuilder.
+      case ("", idx) if idx != 0 => builder.appendLiteral("'")
       case (pattenPart, idx) if idx % 2 == 0 =>
         var rest = pattenPart
         while (rest.nonEmpty) {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index a06cdfda89ee1..2c4ed64eda80a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -107,3 +107,8 @@ select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
 select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
 select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc');
 select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
+
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); -- middle
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss"); -- head
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss"); -- head but as single quote
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 714412f3f2bc9..f440b5fa1b893 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 73
+-- Number of queries: 77
 
 
 -- !query
@@ -601,3 +601,35 @@ select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
 struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
 -- !query output
 2019-10-06 Sunday
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12

From 39e9b554ea171e71ea152c1d3a59f72e8918dfd2 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 19 Mar 2020 07:29:31 +0900
Subject: [PATCH 0304/1280] [SPARK-31171][SQL][FOLLOWUP] update document

### What changes were proposed in this pull request?

A followup of https://github.com/apache/spark/pull/27936 to update document.

### Why are the changes needed?

correct document

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #27950 from cloud-fan/null.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 8643e5d9c50294f59b01988d99d447a38776178e)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-ansi-compliance.md                            | 7 ++++++-
 .../sql/catalyst/expressions/collectionOperations.scala    | 6 +++---
 .../src/main/scala/org/apache/spark/sql/functions.scala    | 4 ++++
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 27e60b4bb8e3b..bc5bde6778e12 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -21,7 +21,7 @@ license: |
 
 Since Spark 3.0, Spark SQL introduces two experimental options to comply with the SQL standard: `spark.sql.ansi.enabled` and `spark.sql.storeAssignmentPolicy` (See a table below for details).
 
-When `spark.sql.ansi.enabled` is set to `true`, Spark SQL follows the standard in basic behaviours (e.g., arithmetic operations, type conversion, and SQL parsing).
+When `spark.sql.ansi.enabled` is set to `true`, Spark SQL follows the standard in basic behaviours (e.g., arithmetic operations, type conversion, SQL functions and SQL parsing).
 Moreover, Spark SQL has an independent option to control implicit casting behaviours when inserting rows in a table.
 The casting behaviours are defined as store assignment rules in the standard.
 
@@ -140,6 +140,11 @@ SELECT * FROM t;
 
 {% endhighlight %}
 
+### SQL Functions
+
+The behavior of some SQL functions can be different under ANSI mode (`spark.sql.ansi.enabled=true`).
+  - `size`: This function returns null for null input under ANSI mode.
+
 ### SQL Keywords
 
 When `spark.sql.ansi.enabled` is true, Spark SQL will use the ANSI mode parser.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 6d95909e17f41..8b61bc4f22b94 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -77,9 +77,9 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
 @ExpressionDescription(
   usage = """
     _FUNC_(expr) - Returns the size of an array or a map.
-    The function returns -1 if its input is null and spark.sql.legacy.sizeOfNull is set to true.
-    If spark.sql.legacy.sizeOfNull is set to false, the function returns null for null input.
-    By default, the spark.sql.legacy.sizeOfNull parameter is set to true.
+    The function returns null for null input if spark.sql.legacy.sizeOfNull is set to false or
+    spark.sql.ansi.enabled is set to true. Otherwise, the function returns -1 for null input.
+    With the default settings, the function returns -1 for null input.
   """,
   examples = """
     Examples:
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index e2d3d55812c51..69383d44522e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -3957,6 +3957,10 @@ object functions {
   /**
    * Returns length of array or map.
    *
+   * The function returns null for null input if spark.sql.legacy.sizeOfNull is set to false or
+   * spark.sql.ansi.enabled is set to true. Otherwise, the function returns -1 for null input.
+   * With the default settings, the function returns -1 for null input.
+   *
    * @group collection_funcs
    * @since 1.5.0
    */

From a97117f1294d3625d71809f2770523ad0e14ade0 Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Wed, 18 Mar 2020 18:07:24 -0700
Subject: [PATCH 0305/1280] [SPARK-31178][SQL] Prevent V2 exec nodes from
 executing multiple times

### What changes were proposed in this pull request?

This PR prevents the execution of V2 DataSource exec nodes multiple times when `collect()` is called on them. For V1 DataSources, commands would be executed as a RunnableCommand, which would cache the result as part of the `ExecutedCommandExec` node. We extend `V2CommandExec` for all the data writing commands so that they only get executed once as well.

### Why are the changes needed?

Calling `collect()` on a SQL command that inserts data or creates a table gets executed multiple times otherwise.

### Does this PR introduce any user-facing change?

Fixes a bug

### How was this patch tested?

Unit tests

Closes #27941 from brkyvz/doubleInsert.

Authored-by: Burak Yavuz <brkyvz@gmail.com>
Signed-off-by: Burak Yavuz <brkyvz@gmail.com>
(cherry picked from commit 4237251861c79f3176de7cf5232f0388ec5d946e)
Signed-off-by: Burak Yavuz <brkyvz@gmail.com>
---
 .../datasources/v2/ShowNamespacesExec.scala   |  4 +-
 .../datasources/v2/ShowTablesExec.scala       |  4 +-
 .../datasources/v2/V1FallbackWriters.scala    | 10 +--
 .../datasources/v2/V2CommandExec.scala        |  6 +-
 .../v2/WriteToDataSourceV2Exec.scala          | 28 ++++----
 .../sql/connector/DataSourceV2SQLSuite.scala  | 68 +++++++++++++++++++
 .../spark/sql/connector/InsertIntoTests.scala | 17 +++++
 7 files changed, 112 insertions(+), 25 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
index fe3ab8023db6f..6f968481cb7cc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchem
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces
+import org.apache.spark.sql.execution.LeafExecNode
 
 /**
  * Physical plan node for showing namespaces.
@@ -33,8 +34,7 @@ case class ShowNamespacesExec(
     output: Seq[Attribute],
     catalog: SupportsNamespaces,
     namespace: Seq[String],
-    pattern: Option[String])
-    extends V2CommandExec {
+    pattern: Option[String]) extends V2CommandExec with LeafExecNode {
 
   override protected def run(): Seq[InternalRow] = {
     val namespaces = if (namespace.nonEmpty) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
index 995b00871fc2a..c740e0d370dfd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchem
 import org.apache.spark.sql.catalyst.util.StringUtils
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.execution.LeafExecNode
 
 /**
  * Physical plan node for showing tables.
@@ -33,8 +34,7 @@ case class ShowTablesExec(
     output: Seq[Attribute],
     catalog: TableCatalog,
     namespace: Seq[String],
-    pattern: Option[String])
-    extends V2CommandExec {
+    pattern: Option[String]) extends V2CommandExec with LeafExecNode {
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
     val encoder = RowEncoder(schema).resolveAndBind()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
index f97300025400d..7502a8750aee4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
@@ -41,7 +41,7 @@ case class AppendDataExecV1(
     writeOptions: CaseInsensitiveStringMap,
     plan: LogicalPlan) extends V1FallbackWriters {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     writeWithV1(newWriteBuilder().buildForV1Write())
   }
 }
@@ -67,7 +67,7 @@ case class OverwriteByExpressionExecV1(
     filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
   }
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     newWriteBuilder() match {
       case builder: SupportsTruncate if isTruncate(deleteWhere) =>
         writeWithV1(builder.truncate().asV1Builder.buildForV1Write())
@@ -82,7 +82,7 @@ case class OverwriteByExpressionExecV1(
 }
 
 /** Some helper interfaces that use V2 write semantics through the V1 writer interface. */
-sealed trait V1FallbackWriters extends SupportsV1Write {
+sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
   override def output: Seq[Attribute] = Nil
   override final def children: Seq[SparkPlan] = Nil
 
@@ -115,8 +115,8 @@ trait SupportsV1Write extends SparkPlan {
   // TODO: We should be able to work on SparkPlans at this point.
   def plan: LogicalPlan
 
-  protected def writeWithV1(relation: InsertableRelation): RDD[InternalRow] = {
+  protected def writeWithV1(relation: InsertableRelation): Seq[InternalRow] = {
     relation.insert(Dataset.ofRows(sqlContext.sparkSession, plan), overwrite = false)
-    sparkContext.emptyRDD
+    Nil
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
index a1f685d47a346..4be4a6b30edcd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2CommandExec.scala
@@ -19,13 +19,13 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.LeafExecNode
+import org.apache.spark.sql.execution.SparkPlan
 
 /**
  * A physical operator that executes run() and saves the result to prevent multiple executions.
  * Any V2 commands that do not require triggering a spark job should extend this class.
  */
-abstract class V2CommandExec extends LeafExecNode {
+abstract class V2CommandExec extends SparkPlan {
 
   /**
    * Abstract method that each concrete command needs to implement to compute the result.
@@ -53,4 +53,6 @@ abstract class V2CommandExec extends LeafExecNode {
   protected override def doExecute(): RDD[InternalRow] = {
     sqlContext.sparkContext.parallelize(result, 1)
   }
+
+  override def children: Seq[SparkPlan] = Nil
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index e360a9e656a16..616e18ee85a6b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -70,10 +70,10 @@ case class CreateTableAsSelectExec(
 
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
       if (ifNotExists) {
-        return sparkContext.parallelize(Seq.empty, 1)
+        return Nil
       }
 
       throw new TableAlreadyExistsException(ident)
@@ -125,10 +125,10 @@ case class AtomicCreateTableAsSelectExec(
     writeOptions: CaseInsensitiveStringMap,
     ifNotExists: Boolean) extends AtomicTableWriteExec {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     if (catalog.tableExists(ident)) {
       if (ifNotExists) {
-        return sparkContext.parallelize(Seq.empty, 1)
+        return Nil
       }
 
       throw new TableAlreadyExistsException(ident)
@@ -161,7 +161,7 @@ case class ReplaceTableAsSelectExec(
 
   import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.IdentifierHelper
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     // Note that this operation is potentially unsafe, but these are the strict semantics of
     // RTAS if the catalog does not support atomic operations.
     //
@@ -225,7 +225,7 @@ case class AtomicReplaceTableAsSelectExec(
     writeOptions: CaseInsensitiveStringMap,
     orCreate: Boolean) extends AtomicTableWriteExec {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     val schema = query.schema.asNullable
     val staged = if (orCreate) {
       catalog.stageCreateOrReplace(
@@ -255,7 +255,7 @@ case class AppendDataExec(
     writeOptions: CaseInsensitiveStringMap,
     query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     writeWithV2(newWriteBuilder().buildForBatch())
   }
 }
@@ -280,7 +280,7 @@ case class OverwriteByExpressionExec(
     filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
   }
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     newWriteBuilder() match {
       case builder: SupportsTruncate if isTruncate(deleteWhere) =>
         writeWithV2(builder.truncate().buildForBatch())
@@ -308,7 +308,7 @@ case class OverwritePartitionsDynamicExec(
     writeOptions: CaseInsensitiveStringMap,
     query: SparkPlan) extends V2TableWriteExec with BatchWriteHelper {
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     newWriteBuilder() match {
       case builder: SupportsDynamicOverwrite =>
         writeWithV2(builder.overwriteDynamicPartitions().buildForBatch())
@@ -325,7 +325,7 @@ case class WriteToDataSourceV2Exec(
 
   def writeOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty()
 
-  override protected def doExecute(): RDD[InternalRow] = {
+  override protected def run(): Seq[InternalRow] = {
     writeWithV2(batchWrite)
   }
 }
@@ -350,7 +350,7 @@ trait BatchWriteHelper {
 /**
  * The base physical plan for writing data into data source v2.
  */
-trait V2TableWriteExec extends UnaryExecNode {
+trait V2TableWriteExec extends V2CommandExec with UnaryExecNode {
   def query: SparkPlan
 
   var commitProgress: Option[StreamWriterCommitProgress] = None
@@ -358,7 +358,7 @@ trait V2TableWriteExec extends UnaryExecNode {
   override def child: SparkPlan = query
   override def output: Seq[Attribute] = Nil
 
-  protected def writeWithV2(batchWrite: BatchWrite): RDD[InternalRow] = {
+  protected def writeWithV2(batchWrite: BatchWrite): Seq[InternalRow] = {
     val rdd: RDD[InternalRow] = {
       val tempRdd = query.execute()
       // SPARK-23271 If we are attempting to write a zero partition rdd, create a dummy single
@@ -415,7 +415,7 @@ trait V2TableWriteExec extends UnaryExecNode {
         }
     }
 
-    sparkContext.emptyRDD
+    Nil
   }
 }
 
@@ -485,7 +485,7 @@ private[v2] trait AtomicTableWriteExec extends V2TableWriteExec with SupportsV1W
   protected def writeToStagedTable(
       stagedTable: StagedTable,
       writeOptions: CaseInsensitiveStringMap,
-      ident: Identifier): RDD[InternalRow] = {
+      ident: Identifier): Seq[InternalRow] = {
     Utils.tryWithSafeFinallyAndFailureCallbacks({
       stagedTable match {
         case table: SupportsWrite =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index ba4200d84d46b..07e0959516040 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -319,6 +319,37 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("CreateTableAsSelect: do not double execute on collect(), take() and other queries") {
+    val basicCatalog = catalog("testcat").asTableCatalog
+    val atomicCatalog = catalog("testcat_atomic").asTableCatalog
+    val basicIdentifier = "testcat.table_name"
+    val atomicIdentifier = "testcat_atomic.table_name"
+
+    Seq((basicCatalog, basicIdentifier), (atomicCatalog, atomicIdentifier)).foreach {
+      case (catalog, identifier) =>
+        val df = spark.sql(s"CREATE TABLE $identifier USING foo AS SELECT id, data FROM source")
+
+        df.collect()
+        df.take(5)
+        df.tail(5)
+        df.where("true").collect()
+        df.where("true").take(5)
+        df.where("true").tail(5)
+
+        val table = catalog.loadTable(Identifier.of(Array(), "table_name"))
+
+        assert(table.name == identifier)
+        assert(table.partitioning.isEmpty)
+        assert(table.properties == withDefaultOwnership(Map("provider" -> "foo")).asJava)
+        assert(table.schema == new StructType()
+          .add("id", LongType)
+          .add("data", StringType))
+
+        val rdd = spark.sparkContext.parallelize(table.asInstanceOf[InMemoryTable].rows)
+        checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), spark.table("source"))
+    }
+  }
+
   test("ReplaceTableAsSelect: basic v2 implementation.") {
     val basicCatalog = catalog("testcat").asTableCatalog
     val atomicCatalog = catalog("testcat_atomic").asTableCatalog
@@ -346,6 +377,43 @@ class DataSourceV2SQLSuite
     }
   }
 
+  Seq("REPLACE", "CREATE OR REPLACE").foreach { cmd =>
+    test(s"ReplaceTableAsSelect: do not double execute $cmd on collect()") {
+      val basicCatalog = catalog("testcat").asTableCatalog
+      val atomicCatalog = catalog("testcat_atomic").asTableCatalog
+      val basicIdentifier = "testcat.table_name"
+      val atomicIdentifier = "testcat_atomic.table_name"
+
+      Seq((basicCatalog, basicIdentifier), (atomicCatalog, atomicIdentifier)).foreach {
+        case (catalog, identifier) =>
+          spark.sql(s"CREATE TABLE $identifier USING foo AS SELECT id, data FROM source")
+          val originalTable = catalog.loadTable(Identifier.of(Array(), "table_name"))
+
+          val df = spark.sql(s"$cmd TABLE $identifier USING foo AS SELECT id FROM source")
+
+          df.collect()
+          df.take(5)
+          df.tail(5)
+          df.where("true").collect()
+          df.where("true").take(5)
+          df.where("true").tail(5)
+
+          val replacedTable = catalog.loadTable(Identifier.of(Array(), "table_name"))
+
+          assert(replacedTable != originalTable, "Table should have been replaced.")
+          assert(replacedTable.name == identifier)
+          assert(replacedTable.partitioning.isEmpty)
+          assert(replacedTable.properties == withDefaultOwnership(Map("provider" -> "foo")).asJava)
+          assert(replacedTable.schema == new StructType().add("id", LongType))
+
+          val rdd = spark.sparkContext.parallelize(replacedTable.asInstanceOf[InMemoryTable].rows)
+          checkAnswer(
+            spark.internalCreateDataFrame(rdd, replacedTable.schema),
+            spark.table("source").select("id"))
+      }
+    }
+  }
+
   test("ReplaceTableAsSelect: Non-atomic catalog drops the table if the write fails.") {
     spark.sql("CREATE TABLE testcat.table_name USING foo AS SELECT id, data FROM source")
     val testCatalog = catalog("testcat").asTableCatalog
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
index 0fd6cf1b6746c..b88ad5218fcd2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
@@ -463,5 +463,22 @@ trait InsertIntoSQLOnlyTests
         }
       }
     }
+
+    test("do not double insert on INSERT INTO collect()") {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTableAndData(t1) { view =>
+        sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format")
+        val df = sql(s"INSERT INTO TABLE $t1 SELECT * FROM $view")
+
+        df.collect()
+        df.take(5)
+        df.tail(5)
+        df.where("true").collect()
+        df.where("true").take(5)
+        df.where("true").tail(5)
+
+        verifyTable(t1, spark.table(view))
+      }
+    }
   }
 }

From f45be8b19ba045889bfa54a0e4b8dce54c4dd910 Mon Sep 17 00:00:00 2001
From: sarthfrey-db <sarth.frey@databricks.com>
Date: Thu, 19 Mar 2020 12:12:39 +0900
Subject: [PATCH 0306/1280] [SPARK-30667][CORE] Change BarrierTaskContext
 allGather method return type

This PR proposes that we change the return type of the `BarrierTaskContext.allGather` method to `Array[String]` instead of `ArrayBuffer[String]` since it is immutable. Based on discussion in #27640. cc zhengruifeng srowen

Closes #27951 from sarthfrey/all-gather-api.

Authored-by: sarthfrey-db <sarth.frey@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 6fd3138e9c3c1d703f2a66bcdf17555803547774)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/BarrierTaskContext.scala | 4 ++--
 .../main/scala/org/apache/spark/api/python/PythonRunner.scala | 2 +-
 .../org/apache/spark/scheduler/BarrierTaskContextSuite.scala  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 2263538a11676..0c2ceb1a02c7b 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -217,11 +217,11 @@ class BarrierTaskContext private[spark] (
    */
   @Experimental
   @Since("3.0.0")
-  def allGather(message: String): ArrayBuffer[String] = {
+  def allGather(message: String): Array[String] = {
     val json = runBarrier(RequestMethod.ALL_GATHER, message)
     val jsonArray = parse(json)
     implicit val formats = DefaultFormats
-    ArrayBuffer(jsonArray.extract[Array[String]]: _*)
+    jsonArray.extract[Array[String]]
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index fa8bf0fc06358..06c9446c7534e 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -420,7 +420,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
             context.asInstanceOf[BarrierTaskContext].barrier()
             result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
           case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
-            val messages: ArrayBuffer[String] = context.asInstanceOf[BarrierTaskContext].allGather(
+            val messages: Array[String] = context.asInstanceOf[BarrierTaskContext].allGather(
               message
             )
             result = compact(render(JArray(
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 33594c0a50d14..0dd8be72dc904 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -65,7 +65,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
       Thread.sleep(Random.nextInt(1000))
       // Pass partitionId message in
       val message: String = context.partitionId().toString
-      val messages: ArrayBuffer[String] = context.allGather(message)
+      val messages: Array[String] = context.allGather(message)
       messages.toList.iterator
     }
     // Take a sorted list of all the partitionId messages

From d712a7aeef4606a2ab3102b04545cde218eb712a Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 19 Mar 2020 12:49:51 +0800
Subject: [PATCH 0307/1280] [SPARK-31159][SQL] Rebase date/timestamp from/to
 Julian calendar in parquet

The PR addresses the issue of compatibility with Spark 2.4 and earlier version in reading/writing dates and timestamp via Parquet datasource. Previous releases are based on a hybrid calendar - Julian + Gregorian. Since Spark 3.0, Proleptic Gregorian calendar is used by default, see SPARK-26651. In particular, the issue pops up for dates/timestamps before 1582-10-15 when the hybrid calendar switches from/to Gregorian to/from Julian calendar. The same local date in different calendar is converted to different number of days since the epoch 1970-01-01. For example, the 1001-01-01 date is converted to:
- -719164 in Julian calendar. Spark 2.4 saves the number as a value of DATE type into parquet.
- -719162 in Proleptic Gregorian calendar. Spark 3.0 saves the number as a date value.

According to the parquet spec, parquet timestamps of the `TIMESTAMP_MILLIS`, `TIMESTAMP_MICROS` output type and parquet dates should be based on Proleptic Gregorian calendar but the `INT96` timestamps should be stored as Julian days. Since the version 3.0, Spark conforms the spec but for the backward compatibility with previous version, the PR proposes rebasing from/to Proleptic Gregorian calendar to the hybrid one under the SQL config:
```
spark.sql.legacy.parquet.rebaseDateTime.enabled
```
which is set to `false` by default which means the rebasing is not performed by default.

The details of the implementation:
1. Added 2 methods to `DateTimeUtils` for rebasing microseconds. `rebaseGregorianToJulianMicros()` builds a local timestamp in Proleptic Gregorian calendar, extracts date-time fields `year`, `month`, ..., `second fraction` from the local timestamp and uses them to build another local timestamp based on the hybrid calendar (using `java.util.Calendar` API). After that it calculates the number of microseconds since the epoch using the resulted local timestamp. The function performs the conversion via the system JVM time zone for compatibility with Spark 2.4 and earlier versions. The `rebaseJulianToGregorianMicros()` function does reverse conversion.
2. Added 2 methods to `DateTimeUtils` for rebasing days. `rebaseGregorianToJulianDays()` builds a local date from the passed number of days since the epoch in Proleptic Gregorian calendar, interprets the resulted date as a local date in the hybrid calendar and gets the number of days since the epoch from the resulted local date. The conversion is performed via the `UTC` time zone because the conversion is independent from time zones, and `UTC` is selected to void round issues of casting days to milliseconds and back. The `rebaseJulianToGregorianDays()` functions does revers conversion.
3. Use `rebaseGregorianToJulianMicros()` and `rebaseGregorianToJulianDays()` while saving timestamps/dates to parquet files if the SQL config is on.
4. Use `rebaseJulianToGregorianMicros()` and `rebaseJulianToGregorianDays()` while loading timestamps/dates from parquet files if the SQL config is on.
5. The SQL config `spark.sql.legacy.parquet.rebaseDateTime.enabled` controls conversions from/to dates, timestamps of `TIMESTAMP_MILLIS`, `TIMESTAMP_MICROS`, see the SQL config `spark.sql.parquet.outputTimestampType`.
6. The rebasing is always performed for `INT96` timestamps, independently from `spark.sql.legacy.parquet.rebaseDateTime.enabled`.
7. Supported the vectorized parquet reader, see the SQL config `spark.sql.parquet.enableVectorizedReader`.

- For the backward compatibility with Spark 2.4 and earlier versions. The changes allow users to read dates/timestamps saved by previous version, and get the same result. Also after the changes, users can enable the rebasing in write, and save dates/timestamps that can be loaded correctly by Spark 2.4 and earlier versions.
- It fixes the bug of incorrect saving/loading timestamps of the `INT96` type

Yes, the timestamp `1001-01-01 01:02:03.123456` saved by Spark 2.4.5 as `TIMESTAMP_MICROS` is interpreted by Spark 3.0.0-preview2 differently:
```scala
scala> spark.read.parquet("/Users/maxim/tmp/before_1582/2_4_5_ts_micros").show(false)
+--------------------------+
|ts                        |
+--------------------------+
|1001-01-07 11:32:20.123456|
+--------------------------+
```
After the changes:
```scala
scala> spark.conf.set("spark.sql.legacy.parquet.rebaseDateTime.enabled", true)

scala> spark.read.parquet("/Users/maxim/tmp/before_1582/2_4_5_ts_micros").show(false)
+--------------------------+
|ts                        |
+--------------------------+
|1001-01-01 01:02:03.123456|
+--------------------------+
```

1. Added tests to `ParquetIOSuite` to check rebasing in read for regular reader and vectorized parquet reader. The test reads back parquet files saved by Spark 2.4.5 via:
```shell
$ export TZ="America/Los_Angeles"
```
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> val df = Seq("1001-01-01").toDF("dateS").select($"dateS".cast("date").as("date"))
df: org.apache.spark.sql.DataFrame = [date: date]
scala> df.write.parquet("/Users/maxim/tmp/before_1582/2_4_5_date")

scala> val df = Seq("1001-01-01 01:02:03.123456").toDF("tsS").select($"tsS".cast("timestamp").as("ts"))
df: org.apache.spark.sql.DataFrame = [ts: timestamp]

scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS")
scala> df.write.parquet("/Users/maxim/tmp/before_1582/2_4_5_ts_micros")

scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MILLIS")
scala> df.write.parquet("/Users/maxim/tmp/before_1582/2_4_5_ts_millis")

scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "INT96")
scala> df.write.parquet("/Users/maxim/tmp/before_1582/2_4_5_ts_int96")
```
2. Manually check the write code path. Save date/timestamps (TIMESTAMP_MICROS, TIMESTAMP_MILLIS, INT96) by Spark 3.1.0-SNAPSHOT (after the changes):
```bash
$ export TZ="America/Los_Angeles"
```
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> spark.conf.set("spark.sql.legacy.parquet.rebaseDateTime.enabled", true)
scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS")
scala> val df = Seq(("1001-01-01", "1001-01-01 01:02:03.123456")).toDF("dateS", "tsS").select($"dateS".cast("date").as("d"), $"tsS".cast("timestamp").as("ts"))
df: org.apache.spark.sql.DataFrame = [d: date, ts: timestamp]
scala> df.write.parquet("/Users/maxim/tmp/before_1582/3_0_0_micros")
scala> spark.read.parquet("/Users/maxim/tmp/before_1582/3_0_0_micros").show(false)
+----------+--------------------------+
|d         |ts                        |
+----------+--------------------------+
|1001-01-01|1001-01-01 01:02:03.123456|
+----------+--------------------------+
```
Read the saved date/timestamp by Spark 2.4.5:
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> spark.read.parquet("/Users/maxim/tmp/before_1582/3_0_0_micros").show(false)
+----------+--------------------------+
|d         |ts                        |
+----------+--------------------------+
|1001-01-01|1001-01-01 01:02:03.123456|
+----------+--------------------------+
```

Closes #27915 from MaxGekk/rebase-parquet-datetime.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bb295d80e37440e6ee67a00bc09df2e9ff6e4e46)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 106 +++++++++++++++++-
 .../apache/spark/sql/internal/SQLConf.scala   |  15 +++
 .../catalyst/util/DateTimeUtilsSuite.scala    |  60 ++++++++++
 .../parquet/VectorizedColumnReader.java       |  44 +++++++-
 .../parquet/ParquetRowConverter.scala         |  49 ++++++--
 .../parquet/ParquetWriteSupport.scala         |  19 ++++
 .../before_1582_date_v2_4.snappy.parquet      | Bin 0 -> 398 bytes
 ...e_1582_timestamp_int96_v2_4.snappy.parquet | Bin 0 -> 494 bytes
 ..._1582_timestamp_micros_v2_4.snappy.parquet | Bin 0 -> 436 bytes
 ..._1582_timestamp_millis_v2_4.snappy.parquet | Bin 0 -> 436 bytes
 .../datasources/parquet/ParquetIOSuite.scala  |  66 +++++++++++
 11 files changed, 342 insertions(+), 17 deletions(-)
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index a8c55e75e892f..7f5babfca4e59 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -21,7 +21,7 @@ import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 import java.time._
 import java.time.temporal.{ChronoField, ChronoUnit, IsoFields}
-import java.util.{Locale, TimeZone}
+import java.util.{Calendar, Locale, TimeZone}
 import java.util.concurrent.TimeUnit._
 
 import scala.util.control.NonFatal
@@ -150,7 +150,9 @@ object DateTimeUtils {
   def fromJulianDay(day: Int, nanoseconds: Long): SQLTimestamp = {
     // use Long to avoid rounding errors
     val seconds = (day - JULIAN_DAY_OF_EPOCH).toLong * SECONDS_PER_DAY
-    SECONDS.toMicros(seconds) + NANOSECONDS.toMicros(nanoseconds)
+    val micros = SECONDS.toMicros(seconds) + NANOSECONDS.toMicros(nanoseconds)
+    val rebased = rebaseJulianToGregorianMicros(micros)
+    rebased
   }
 
   /**
@@ -159,7 +161,7 @@ object DateTimeUtils {
    * Note: support timestamp since 4717 BC (without negative nanoseconds, compatible with Hive).
    */
   def toJulianDay(us: SQLTimestamp): (Int, Long) = {
-    val julian_us = us + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
+    val julian_us = rebaseGregorianToJulianMicros(us) + JULIAN_DAY_OF_EPOCH * MICROS_PER_DAY
     val day = julian_us / MICROS_PER_DAY
     val micros = julian_us % MICROS_PER_DAY
     (day.toInt, MICROSECONDS.toNanos(micros))
@@ -942,4 +944,102 @@ object DateTimeUtils {
     val days = period.getDays
     new CalendarInterval(months, days, 0)
   }
+
+  /**
+   * Converts the given microseconds to a local date-time in UTC time zone in Proleptic Gregorian
+   * calendar, interprets the result as a local date-time in Julian calendar in UTC time zone.
+   * And takes microseconds since the epoch from the Julian timestamp.
+   *
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'.
+   * @return The rebased microseconds since the epoch in Julian calendar.
+   */
+  def rebaseGregorianToJulianMicros(micros: Long): Long = {
+    val ldt = microsToInstant(micros).atZone(ZoneId.systemDefault).toLocalDateTime
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
+      .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
+      .build()
+    fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
+  }
+
+  /**
+   * Converts the given microseconds to a local date-time in UTC time zone in Julian calendar,
+   * interprets the result as a local date-time in Proleptic Gregorian calendar in UTC time zone.
+   * And takes microseconds since the epoch from the Gregorian timestamp.
+   *
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'.
+   * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+   */
+  def rebaseJulianToGregorianMicros(micros: Long): Long = {
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setInstant(toMillis(micros))
+      .build()
+    val localDateTime = LocalDateTime.of(
+      cal.get(Calendar.YEAR),
+      cal.get(Calendar.MONTH) + 1,
+      cal.get(Calendar.DAY_OF_MONTH),
+      cal.get(Calendar.HOUR_OF_DAY),
+      cal.get(Calendar.MINUTE),
+      cal.get(Calendar.SECOND),
+      (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
+    instantToMicros(localDateTime.atZone(ZoneId.systemDefault).toInstant)
+  }
+
+  /**
+   * Converts the given number of days since the epoch day 1970-01-01 to
+   * a local date in Julian calendar, interprets the result as a local
+   * date in Proleptic Gregorian calendar, and take the number of days
+   * since the epoch from the Gregorian date.
+   *
+   * @param days The number of days since the epoch in Julian calendar.
+   * @return The rebased number of days in Gregorian calendar.
+   */
+  def rebaseJulianToGregorianDays(days: Int): Int = {
+    val utcCal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setTimeZone(TimeZoneUTC)
+      .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
+      .build()
+    val localDate = LocalDate.of(
+      utcCal.get(Calendar.YEAR),
+      utcCal.get(Calendar.MONTH) + 1,
+      utcCal.get(Calendar.DAY_OF_MONTH))
+    Math.toIntExact(localDate.toEpochDay)
+  }
+
+  /**
+   * Rebasing days since the epoch to store the same number of days
+   * as by Spark 2.4 and earlier versions. Spark 3.0 switched to
+   * Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
+   * this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15. So, the same local date may
+   * be mapped to different number of days since the epoch in different calendars.
+   *
+   * For example:
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   *   Julian calendar: 1582-01-01 -> -141704
+   * The code below converts -141714 to -141704.
+   *
+   * @param days The number of days since the epoch 1970-01-01. It can be negative.
+   * @return The rebased number of days since the epoch in Julian calendar.
+   */
+  def rebaseGregorianToJulianDays(days: Int): Int = {
+    val localDate = LocalDate.ofEpochDay(days)
+    val utcCal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setTimeZone(TimeZoneUTC)
+      .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+      .build()
+    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 1331350f46bc8..a47f7d9340ec4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2232,6 +2232,19 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
+  val LEGACY_PARQUET_REBASE_DATETIME =
+    buildConf("spark.sql.legacy.parquet.rebaseDateTime.enabled")
+      .internal()
+      .doc("When true, rebase dates/timestamps from Proleptic Gregorian calendar " +
+        "to the hybrid calendar (Julian + Gregorian) in write and " +
+        "from the hybrid calendar to Proleptic Gregorian calendar in read. " +
+        "The rebasing is performed by converting micros/millis/days to " +
+        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
+        "timestamp in the target calendar, and getting the number of micros/millis/days " +
+        "since the epoch 1970-01-01 00:00:00Z.")
+      .booleanConf
+      .createWithDefault(false)
+
   /**
    * Holds information about keys that have been deprecated.
    *
@@ -2807,6 +2820,8 @@ class SQLConf extends Serializable with Logging {
 
   def csvFilterPushDown: Boolean = getConf(CSV_FILTER_PUSHDOWN_ENABLED)
 
+  def parquetRebaseDateTimeEnabled: Boolean = getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME)
+
   /** ********************** SQLConf functionality methods ************ */
 
   /** Set Spark SQL configuration properties. */
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 0df395b3cbb1c..f2ad9e6fad44e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -669,4 +669,64 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       assert(toDate("tomorrow CET ", zoneId).get === today + 1)
     }
   }
+
+  test("rebase julian to/from gregorian micros") {
+    outstandingTimezones.foreach { timeZone =>
+      withDefaultTimeZone(timeZone) {
+        Seq(
+          "0001-01-01 01:02:03.654321",
+          "1000-01-01 03:02:01.123456",
+          "1582-10-04 00:00:00.000000",
+          "1582-10-15 00:00:00.999999", // Gregorian cutover day
+          "1883-11-10 00:00:00.000000", // America/Los_Angeles -7:52:58 zone offset
+          "1883-11-20 00:00:00.000000", // America/Los_Angeles -08:00 zone offset
+          "1969-12-31 11:22:33.000100",
+          "1970-01-01 00:00:00.000001", // The epoch day
+          "2020-03-14 09:33:01.500000").foreach { ts =>
+          withClue(s"time zone = ${timeZone.getID} ts = $ts") {
+            val julianTs = Timestamp.valueOf(ts)
+            val julianMicros = fromMillis(julianTs.getTime) +
+              ((julianTs.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
+            val gregorianMicros = instantToMicros(LocalDateTime.parse(ts.replace(' ', 'T'))
+              .atZone(timeZone.toZoneId)
+              .toInstant)
+
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
+            assert(rebaseGregorianToJulianMicros(gregorianMicros) === julianMicros)
+          }
+        }
+      }
+    }
+  }
+
+  test("rebase gregorian to/from julian days") {
+    // millisToDays() and fromJavaDate() are taken from Spark 2.4
+    def millisToDays(millisUtc: Long, timeZone: TimeZone): Int = {
+      val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
+      Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
+    }
+    def fromJavaDate(date: Date): Int = {
+      millisToDays(date.getTime, defaultTimeZone())
+    }
+    outstandingTimezones.foreach { timeZone =>
+      withDefaultTimeZone(timeZone) {
+        Seq(
+          "0001-01-01",
+          "1000-01-01",
+          "1582-10-04",
+          "1582-10-15", // Gregorian cutover day
+          "1883-11-10", // America/Los_Angeles -7:52:58 zone offset
+          "1883-11-20", // America/Los_Angeles -08:00 zone offset
+          "1969-12-31",
+          "1970-01-01", // The epoch day
+          "2020-03-14").foreach { date =>
+          val julianDays = fromJavaDate(Date.valueOf(date))
+          val gregorianDays = localDateToDays(LocalDate.parse(date))
+
+          assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
+          assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 329465544979d..f9b613999a65b 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -37,6 +37,7 @@
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
+import org.apache.spark.sql.internal.SQLConf;
 import org.apache.spark.sql.types.DataTypes;
 import org.apache.spark.sql.types.DecimalType;
 
@@ -101,6 +102,7 @@ public class VectorizedColumnReader {
   // The timezone conversion to apply to int96 timestamps. Null if no conversion.
   private final ZoneId convertTz;
   private static final ZoneId UTC = ZoneOffset.UTC;
+  private final boolean rebaseDateTime;
 
   public VectorizedColumnReader(
       ColumnDescriptor descriptor,
@@ -129,6 +131,7 @@ public VectorizedColumnReader(
     if (totalValueCount == 0) {
       throw new IOException("totalValueCount == 0");
     }
+    this.rebaseDateTime = SQLConf.get().parquetRebaseDateTimeEnabled();
   }
 
   /**
@@ -407,7 +410,7 @@ private void readBooleanBatch(int rowId, int num, WritableColumnVector column)
   private void readIntBatch(int rowId, int num, WritableColumnVector column) throws IOException {
     // This is where we implement support for the valid type conversions.
     // TODO: implement remaining type conversions
-    if (column.dataType() == DataTypes.IntegerType || column.dataType() == DataTypes.DateType ||
+    if (column.dataType() == DataTypes.IntegerType ||
         DecimalType.is32BitDecimalType(column.dataType())) {
       defColumn.readIntegers(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
@@ -417,6 +420,21 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
     } else if (column.dataType() == DataTypes.ShortType) {
       defColumn.readShorts(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+    } else if (column.dataType() == DataTypes.DateType ) {
+      if (rebaseDateTime) {
+        for (int i = 0; i < num; i++) {
+          if (defColumn.readInteger() == maxDefLevel) {
+            column.putInt(
+              rowId + i,
+              DateTimeUtils.rebaseJulianToGregorianDays(dataColumn.readInteger()));
+          } else {
+            column.putNull(rowId + i);
+          }
+        }
+      } else {
+        defColumn.readIntegers(
+           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+      }
     } else {
       throw constructConvertNotSupportedException(descriptor, column);
     }
@@ -425,14 +443,32 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
   private void readLongBatch(int rowId, int num, WritableColumnVector column) throws IOException {
     // This is where we implement support for the valid type conversions.
     if (column.dataType() == DataTypes.LongType ||
-        DecimalType.is64BitDecimalType(column.dataType()) ||
-        originalType == OriginalType.TIMESTAMP_MICROS) {
+        DecimalType.is64BitDecimalType(column.dataType())) {
       defColumn.readLongs(
         num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+    } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
+      if (rebaseDateTime) {
+        for (int i = 0; i < num; i++) {
+          if (defColumn.readInteger() == maxDefLevel) {
+            column.putLong(
+              rowId + i,
+              DateTimeUtils.rebaseJulianToGregorianMicros(dataColumn.readLong()));
+          } else {
+            column.putNull(rowId + i);
+          }
+        }
+      } else {
+        defColumn.readLongs(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+      }
     } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
       for (int i = 0; i < num; i++) {
         if (defColumn.readInteger() == maxDefLevel) {
-          column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong()));
+          long micros = DateTimeUtils.fromMillis(dataColumn.readLong());
+          if (rebaseDateTime) {
+            micros = DateTimeUtils.rebaseJulianToGregorianMicros(micros);
+          }
+          column.putLong(rowId + i, micros);
         } else {
           column.putNull(rowId + i);
         }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 22422c02fd666..85a0d43b5e438 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -130,6 +130,9 @@ private[parquet] class ParquetRowConverter(
     updater: ParentContainerUpdater)
   extends ParquetGroupConverter(updater) with Logging {
 
+  // Enable rebasing date/timestamp from Julian to Proleptic Gregorian calendar
+  private val rebaseDateTime = SQLConf.get.parquetRebaseDateTimeEnabled
+
   assert(
     parquetType.getFieldCount <= catalystType.length,
     s"""Field count of the Parquet schema is greater than the field count of the Catalyst schema:
@@ -271,16 +274,35 @@ private[parquet] class ParquetRowConverter(
         new ParquetStringConverter(updater)
 
       case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MICROS =>
-        new ParquetPrimitiveConverter(updater) {
-          override def addLong(value: Long): Unit = {
-            updater.setLong(value)
+        if (rebaseDateTime) {
+          new ParquetPrimitiveConverter(updater) {
+            override def addLong(value: Long): Unit = {
+              val rebased = DateTimeUtils.rebaseJulianToGregorianMicros(value)
+              updater.setLong(rebased)
+            }
+          }
+        } else {
+          new ParquetPrimitiveConverter(updater) {
+            override def addLong(value: Long): Unit = {
+              updater.setLong(value)
+            }
           }
         }
 
       case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MILLIS =>
-        new ParquetPrimitiveConverter(updater) {
-          override def addLong(value: Long): Unit = {
-            updater.setLong(DateTimeUtils.fromMillis(value))
+        if (rebaseDateTime) {
+          new ParquetPrimitiveConverter(updater) {
+            override def addLong(value: Long): Unit = {
+              val micros = DateTimeUtils.fromMillis(value)
+              val rebased = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+              updater.setLong(rebased)
+            }
+          }
+        } else {
+          new ParquetPrimitiveConverter(updater) {
+            override def addLong(value: Long): Unit = {
+              updater.setLong(DateTimeUtils.fromMillis(value))
+            }
           }
         }
 
@@ -305,10 +327,17 @@ private[parquet] class ParquetRowConverter(
         }
 
       case DateType =>
-        new ParquetPrimitiveConverter(updater) {
-          override def addInt(value: Int): Unit = {
-            // DateType is not specialized in `SpecificMutableRow`, have to box it here.
-            updater.set(value.asInstanceOf[DateType#InternalType])
+        if (rebaseDateTime) {
+          new ParquetPrimitiveConverter(updater) {
+            override def addInt(value: Int): Unit = {
+              updater.set(DateTimeUtils.rebaseJulianToGregorianDays(value))
+            }
+          }
+        } else {
+          new ParquetPrimitiveConverter(updater) {
+            override def addInt(value: Int): Unit = {
+              updater.set(value)
+            }
           }
         }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index f6490614ab05b..8904b355c4455 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -77,6 +77,9 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
   private val decimalBuffer =
     new Array[Byte](Decimal.minBytesForPrecision(DecimalType.MAX_PRECISION))
 
+  // Whether to rebase datetimes from Gregorian to Julian calendar in write
+  private val rebaseDateTime: Boolean = SQLConf.get.parquetRebaseDateTimeEnabled
+
   override def init(configuration: Configuration): WriteContext = {
     val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA)
     this.schema = StructType.fromString(schemaString)
@@ -147,6 +150,11 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
         (row: SpecializedGetters, ordinal: Int) =>
           recordConsumer.addInteger(row.getShort(ordinal))
 
+      case DateType if rebaseDateTime =>
+        (row: SpecializedGetters, ordinal: Int) =>
+          val rebasedDays = DateTimeUtils.rebaseGregorianToJulianDays(row.getInt(ordinal))
+          recordConsumer.addInteger(rebasedDays)
+
       case IntegerType | DateType =>
         (row: SpecializedGetters, ordinal: Int) =>
           recordConsumer.addInteger(row.getInt(ordinal))
@@ -177,10 +185,21 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
               buf.order(ByteOrder.LITTLE_ENDIAN).putLong(timeOfDayNanos).putInt(julianDay)
               recordConsumer.addBinary(Binary.fromReusedByteArray(timestampBuffer))
 
+          case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS if rebaseDateTime =>
+            (row: SpecializedGetters, ordinal: Int) =>
+              val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(row.getLong(ordinal))
+              recordConsumer.addLong(rebasedMicros)
+
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS =>
             (row: SpecializedGetters, ordinal: Int) =>
               recordConsumer.addLong(row.getLong(ordinal))
 
+          case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS if rebaseDateTime =>
+            (row: SpecializedGetters, ordinal: Int) =>
+              val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(row.getLong(ordinal))
+              val millis = DateTimeUtils.toMillis(rebasedMicros)
+              recordConsumer.addLong(millis)
+
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS =>
             (row: SpecializedGetters, ordinal: Int) =>
               val millis = DateTimeUtils.toMillis(row.getLong(ordinal))
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..7d5cc12eefe04f18f8a2ba99db22326dc0399a80
GIT binary patch
literal 398
zcmaKp!Arw16vneSMv)y&X&{FZ+}NOo*0F9aUc9NN;YkpYbWL^Gc3aX;Wc2F);J@lc
z-`oUGULNo9eeeCgklE#JKoB`0jz=&w9bz%3{r>U!wXZfB{Fx9!$EO(TQEO)*n0i-T
zvy%Gd(afJ!AqhqV_psAWWu<yuWMj%G>st)0<Bo;sULn}Vc4_Orb@303G6)3sfCpxb
zD07BO{Uqc{0!;jHg<0nU0<3FQ3B7z3g{maLrE|CC)!Rz(n43!1^NM>sTc|uWJig!Z
zQWiUD7ZZeo@i3Kjp35W$9Tt_xe^2sw+cXaia&8X5vqHNYrOjelx`7CM5x9e-UgW7O
tr%{?svnU#!C!xy1a5PH&&<`Uu_UBPDi6(xck~A1n#^8C^Ere$96W`?#bIbq$

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..13254bd93a5e6c54a13beb4314757d76b98117e5
GIT binary patch
literal 494
zcmaixPfNov7{=dbrS!0ah$#(3ECt5~wX}|P>+s@D#KZ6^BI%lHvE6Eu`iHS^;dk@m
z(GOxeCb)}U65i+c=6#+dv-9f^>)6Kh3L~robUk>Tu=hiN{f{HSfcNl$*CK3SliJ<i
zbhomg{|5yClOnZtH0WSNyo;`=Sp9e><`2AN7y&kpk&UDP3lU0a&!pHOBSSYyt72p!
z>r=D7w4QmkYrI?i6N7l_Dtc=Nn+>3yihxjiu&nQWRxuUG-(>T}Q{k(SPYsuTy?pUY
zE?LSXvu{?ee5<%Sb%m^(xpX}@%XyIscdF2Wl`WD&$=f!Wmt06zR!UklMZuCnwWzb)
zbAP6~)75%?y9WDm>v>Yv&NHt?zFat=9|nHt^phqpQio0AG@E2`JUmSzo<-4cm<CZ0
b#e5XZ<76C<gM=q(I3Sd&H?K5+nfmE(+~RaD

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..7d2b46e9bea416774f0b5ebc664d9383530ddf24
GIT binary patch
literal 436
zcmZ{hPfNov7{=emF?tYBrZkX432baoL+i9#hZk?+X?POGkaSJ8*zQl0Dl+yd1P@+4
z`5F9hn#sm4dI=BD`#$+SA*0Ki5Cf*z^D#1_iEZlI@b&R|J9+)yD+fA%xBzf;hRk_j
z?IIWv|LRaJMD=*5rVp|d7y)jykc+fx9fN}iCA4i*+#u&o8_34T$U)AV<-A*M`(h8>
ztAFr>n#y1R0Gl`hLf!7Xy60jc44ePf;A&ZL(`Q*r&8zt{FQpc#(83+Bn0{GEHe^ay
z^;9#TWw|U;#fD}y6Xj;4m9e*LIxnTtqFfkjRu_dx3bSIe(r3F{*>JU9PuAdGZ<!}$
z<vqzt<@1>r@-X0`*GcNUNIfx#(`=B%aql#VWEMreUK&I}6wC8q8Ylg@A0#qK!!DuJ
MJaKFRjLd)i023L1p8x;=

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..e9825455c20158d1d0cbb8add20ca5833d5be9cc
GIT binary patch
literal 436
zcmZ{hK}*9h7>2*bF?tXWLP`TUl)%OYHMGuk>+s@DJc-vKlCG(iwOiMu${70_9{hd&
zkD1BFE_w+s@AtiV-jK(;rw{|C*z++mqK$3p+G-B#;Cr__DF=FgxBzf*gUoqm9U~YK
z|NdN6qMpB~*_$i{Mu1x#<RYy)m*5~m3GLbxx5%(<1KIl+Imr36oG+_u-=3f+^$(s<
zQyB~ZU>`?7sM}xGuUu5Zu=#Hdu2uy%eO8Usyk36rQfiS3E!^pb>2)R9gehG&GtGRK
z<+4Z>o0!o;l>3oZ#y+g+yp&3dvNG19DGHGkX2oQs&yKgU$!5EK-h%sZU|yBA_bzLd
zFBe|O!+?igFKO~3^~5+%vvC&3gX<)cSriQhX%GcbEQi4?PDb%4NMw?ReL|^u;?x3o
HH2?JjJC=Zu

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 1550b3bbb6242..7f85fd2a1629a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
+import java.sql.{Date, Timestamp}
 import java.util.Locale
 
 import scala.collection.JavaConverters._
@@ -879,6 +880,71 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       assert(metaData.get(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
     }
   }
+
+  test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
+    Seq(false, true).foreach { vectorized =>
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "true") {
+          checkAnswer(
+            readResourceParquetFile("test-data/before_1582_date_v2_4.snappy.parquet"),
+            Row(java.sql.Date.valueOf("1001-01-01")))
+          checkAnswer(readResourceParquetFile(
+            "test-data/before_1582_timestamp_micros_v2_4.snappy.parquet"),
+            Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+          checkAnswer(readResourceParquetFile(
+            "test-data/before_1582_timestamp_millis_v2_4.snappy.parquet"),
+            Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123")))
+        }
+        checkAnswer(readResourceParquetFile(
+          "test-data/before_1582_timestamp_int96_v2_4.snappy.parquet"),
+          Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      }
+    }
+  }
+
+  test("SPARK-31159: rebasing timestamps in write") {
+    Seq(
+      ("TIMESTAMP_MILLIS", "1001-01-01 01:02:03.123", "1001-01-07 01:09:05.123"),
+      ("TIMESTAMP_MICROS", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456"),
+      ("INT96", "1001-01-01 01:02:03.123456", "1001-01-01 01:02:03.123456")
+    ).foreach { case (outType, tsStr, nonRebased) =>
+      withClue(s"output type $outType") {
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
+          withTempPath { dir =>
+            val path = dir.getAbsolutePath
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "true") {
+              Seq(tsStr).toDF("tsS")
+                .select($"tsS".cast("timestamp").as("ts"))
+                .write
+                .parquet(path)
+
+              checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
+            }
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "false") {
+              checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(nonRebased)))
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("SPARK-31159: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "true") {
+        Seq("1001-01-01").toDF("dateS")
+          .select($"dateS".cast("date").as("date"))
+          .write
+          .parquet(path)
+
+        checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
+      }
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-07")))
+      }
+    }
+  }
 }
 
 class JobCommitFailureParquetOutputCommitter(outputPath: Path, context: TaskAttemptContext)

From a8c08b1d81aefd1e3d7f4616b76e2285f9981cc7 Mon Sep 17 00:00:00 2001
From: Kris Mok <kris.mok@databricks.com>
Date: Thu, 19 Mar 2020 20:53:01 +0900
Subject: [PATCH 0308/1280] [SPARK-31187][SQL] Sort the whole-stage codegen
 debug output by codegenStageId

### What changes were proposed in this pull request?

Spark SQL's whole-stage codegen (WSCG) supports dumping the generated code to help with debugging. One way to get the generated code is through `df.queryExecution.debug.codegen`, or SQL `EXPLAIN CODEGEN` statement.

The generated code is currently printed without specific ordering, which can make debugging a bit annoying. This PR makes a minor improvement to sort the codegen dump by the `codegenStageId`, ascending.

After this change, the following query:
```scala
spark.range(10).agg(sum('id)).queryExecution.debug.codegen
```
will always dump the generated code in a natural, stable order. A version of this example with shorter output is:
```
spark.range(10).agg(sum('id)).queryExecution.debug.codegenToSeq.map(_._1).foreach(println)
*(1) HashAggregate(keys=[], functions=[partial_sum(id#8L)], output=[sum#15L])
+- *(1) Range (0, 10, step=1, splits=16)

*(2) HashAggregate(keys=[], functions=[sum(id#8L)], output=[sum(id)#12L])
+- Exchange SinglePartition, true, [id=#30]
   +- *(1) HashAggregate(keys=[], functions=[partial_sum(id#8L)], output=[sum#15L])
      +- *(1) Range (0, 10, step=1, splits=16)
```

The number of codegen stages within a single SQL query tends to be very small, most likely < 50, so the overhead of adding the sorting shouldn't be significant.

### Why are the changes needed?

Minor improvement to aid WSCG debugging.

### Does this PR introduce any user-facing change?

No user-facing change for end-users; minor change for developers who debug WSCG generated code.

### How was this patch tested?

Manually tested the output; all other tests still pass.

Closes #27955 from rednaxelafx/codegen.

Authored-by: Kris Mok <kris.mok@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit a1776288f48d450fea28f50fef78fd6aa10a8160)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../scala/org/apache/spark/sql/execution/debug/package.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 6a57ef2cafe23..6c40104e52a5f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -113,7 +113,7 @@ package object debug {
         s
       case s => s
     }
-    codegenSubtrees.toSeq.map { subtree =>
+    codegenSubtrees.toSeq.sortBy(_.codegenStageId).map { subtree =>
       val (_, source) = subtree.doCodeGen()
       val codeStats = try {
         CodeGenerator.compile(source)._2

From 1a5cd167e0901948d68d6c7880d39966e74d10b3 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 20 Mar 2020 00:52:09 +0900
Subject: [PATCH 0309/1280] [SPARK-30292][SQL][FOLLOWUP] ansi cast from strings
 to integral numbers (byte/short/int/long) should fail with fraction

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/26933

Fraction string like "1.23" is definitely not a valid integral format and we should fail to do the cast under the ANSI mode.

### Why are the changes needed?

correct the ANSI cast behavior from string to integral

### Does this PR introduce any user-facing change?

Yes under ANSI mode, but ANSI mode is off by default.

### How was this patch tested?

new test

Closes #27957 from cloud-fan/ansi.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit ac262cb27255f989f6a6dd864bd5114a928b96da)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../apache/spark/unsafe/types/UTF8String.java | 24 +++++++++++--------
 .../sql/catalyst/expressions/CastSuite.scala  |  2 ++
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index c5384669eb922..186597fa64780 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -1105,6 +1105,10 @@ public static class IntWrapper implements Serializable {
    * @return true if the parsing was successful else false
    */
   public boolean toLong(LongWrapper toLongResult) {
+    return toLong(toLongResult, true);
+  }
+
+  private boolean toLong(LongWrapper toLongResult, boolean allowDecimal) {
     int offset = 0;
     while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
     if (offset == this.numBytes) return false;
@@ -1129,7 +1133,7 @@ public boolean toLong(LongWrapper toLongResult) {
     while (offset <= end) {
       b = getByte(offset);
       offset++;
-      if (b == separator) {
+      if (b == separator && allowDecimal) {
         // We allow decimals and will return a truncated integral in that case.
         // Therefore we won't throw an exception here (checking the fractional
         // part happens below.)
@@ -1198,6 +1202,10 @@ public boolean toLong(LongWrapper toLongResult) {
    * @return true if the parsing was successful else false
    */
   public boolean toInt(IntWrapper intWrapper) {
+    return toInt(intWrapper, true);
+  }
+
+  private boolean toInt(IntWrapper intWrapper, boolean allowDecimal) {
     int offset = 0;
     while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
     if (offset == this.numBytes) return false;
@@ -1222,7 +1230,7 @@ public boolean toInt(IntWrapper intWrapper) {
     while (offset <= end) {
       b = getByte(offset);
       offset++;
-      if (b == separator) {
+      if (b == separator && allowDecimal) {
         // We allow decimals and will return a truncated integral in that case.
         // Therefore we won't throw an exception here (checking the fractional
         // part happens below.)
@@ -1276,9 +1284,7 @@ public boolean toShort(IntWrapper intWrapper) {
     if (toInt(intWrapper)) {
       int intValue = intWrapper.value;
       short result = (short) intValue;
-      if (result == intValue) {
-        return true;
-      }
+      return result == intValue;
     }
     return false;
   }
@@ -1287,9 +1293,7 @@ public boolean toByte(IntWrapper intWrapper) {
     if (toInt(intWrapper)) {
       int intValue = intWrapper.value;
       byte result = (byte) intValue;
-      if (result == intValue) {
-        return true;
-      }
+      return result == intValue;
     }
     return false;
   }
@@ -1302,7 +1306,7 @@ public boolean toByte(IntWrapper intWrapper) {
    */
   public long toLongExact() {
     LongWrapper result = new LongWrapper();
-    if (toLong(result)) {
+    if (toLong(result, false)) {
       return result.value;
     }
     throw new NumberFormatException("invalid input syntax for type numeric: " + this);
@@ -1316,7 +1320,7 @@ public long toLongExact() {
    */
   public int toIntExact() {
     IntWrapper result = new IntWrapper();
-    if (toInt(result)) {
+    if (toInt(result, false)) {
       return result.value;
     }
     throw new NumberFormatException("invalid input syntax for type numeric: " + this);
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index bde95f0e3add5..9d20eac369152 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -1287,6 +1287,8 @@ class AnsiCastSuite extends CastSuiteBase {
         cast("123-string", dataType), "invalid input")
       checkExceptionInExpression[NumberFormatException](
         cast("2020-07-19", dataType), "invalid input")
+      checkExceptionInExpression[NumberFormatException](
+        cast("1.23", dataType), "invalid input")
     }
 
     Seq(DoubleType, FloatType, DecimalType.USER_DEFAULT).foreach { dataType =>

From 476aaee7089d051dae49d2a44de69df3b79248a0 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 19 Mar 2020 16:13:50 -0700
Subject: [PATCH 0310/1280] Revert "[SPARK-31170][SQL] Spark SQL Cli should
 respect hive-site.xml and spark.sql.warehouse.dir"

This reverts commit 321341a4c3104380035350631c82a4b385f117e4.

Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/internal/SharedState.scala      | 80 +++++++++----------
 .../hive/thriftserver/SparkSQLCLIDriver.scala |  2 -
 .../sql/hive/thriftserver/CliSuite.scala      | 12 ---
 .../spark/sql/hive/HiveSharedStateSuite.scala |  1 +
 .../spark/sql/hive/HiveSparkSubmitSuite.scala |  2 +-
 5 files changed, 42 insertions(+), 55 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index eb74e96514b2d..5347264d7c50a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -41,6 +41,7 @@ import org.apache.spark.sql.streaming.ui.{StreamingQueryStatusListener, Streamin
 import org.apache.spark.status.ElementTrackingStore
 import org.apache.spark.util.Utils
 
+
 /**
  * A class that holds all state shared across sessions in a given [[SQLContext]].
  *
@@ -54,10 +55,45 @@ private[sql] class SharedState(
 
   SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf)
 
+  // Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
+  // the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
+  val warehousePath: String = {
+    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+    if (configFile != null) {
+      logInfo(s"loading hive config file: $configFile")
+      sparkContext.hadoopConfiguration.addResource(configFile)
+    }
+
+    // hive.metastore.warehouse.dir only stay in hadoopConf
+    sparkContext.conf.remove("hive.metastore.warehouse.dir")
+    // Set the Hive metastore warehouse path to the one we use
+    val hiveWarehouseDir = sparkContext.hadoopConfiguration.get("hive.metastore.warehouse.dir")
+    if (hiveWarehouseDir != null && !sparkContext.conf.contains(WAREHOUSE_PATH.key)) {
+      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
+      // we will respect the value of hive.metastore.warehouse.dir.
+      sparkContext.conf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
+      logInfo(s"${WAREHOUSE_PATH.key} is not set, but hive.metastore.warehouse.dir " +
+        s"is set. Setting ${WAREHOUSE_PATH.key} to the value of " +
+        s"hive.metastore.warehouse.dir ('$hiveWarehouseDir').")
+      hiveWarehouseDir
+    } else {
+      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
+      // the value of spark.sql.warehouse.dir.
+      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set,
+      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
+      val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH)
+      logInfo(s"Setting hive.metastore.warehouse.dir ('$hiveWarehouseDir') to the value of " +
+        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
+      sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir)
+      sparkWarehouseDir
+    }
+  }
+  logInfo(s"Warehouse path is '$warehousePath'.")
+
+  // These 2 variables should be initiated after `warehousePath`, because in the first place we need
+  // to load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
+  // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
   private val (conf, hadoopConf) = {
-    // Load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
-    // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
-    SharedState.loadHiveConfFile(sparkContext.conf, sparkContext.hadoopConfiguration)
     val confClone = sparkContext.conf.clone()
     val hadoopConfClone = new Configuration(sparkContext.hadoopConfiguration)
     // If `SparkSession` is instantiated using an existing `SparkContext` instance and no existing
@@ -130,7 +166,7 @@ private[sql] class SharedState(
     val defaultDbDefinition = CatalogDatabase(
       SessionCatalog.DEFAULT_DATABASE,
       "default database",
-      CatalogUtils.stringToURI(conf.get(WAREHOUSE_PATH)),
+      CatalogUtils.stringToURI(warehousePath),
       Map())
     // Create default database if it doesn't exist
     if (!externalCatalog.databaseExists(SessionCatalog.DEFAULT_DATABASE)) {
@@ -222,40 +258,4 @@ object SharedState extends Logging {
         throw new IllegalArgumentException(s"Error while instantiating '$className':", e)
     }
   }
-
-  /**
-   * Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
-   * the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
-   */
-  def loadHiveConfFile(sparkConf: SparkConf, hadoopConf: Configuration): Unit = {
-    val hiveWarehouseKey = "hive.metastore.warehouse.dir"
-    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
-    if (configFile != null) {
-      logInfo(s"loading hive config file: $configFile")
-      hadoopConf.addResource(configFile)
-    }
-    // hive.metastore.warehouse.dir only stay in hadoopConf
-    sparkConf.remove(hiveWarehouseKey)
-    // Set the Hive metastore warehouse path to the one we use
-    val hiveWarehouseDir = hadoopConf.get(hiveWarehouseKey)
-    val warehousePath = if (hiveWarehouseDir != null && !sparkConf.contains(WAREHOUSE_PATH.key)) {
-      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
-      // we will respect the value of hive.metastore.warehouse.dir.
-      sparkConf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
-      logInfo(s"${WAREHOUSE_PATH.key} is not set, but $hiveWarehouseKey is set. Setting" +
-        s" ${WAREHOUSE_PATH.key} to the value of $hiveWarehouseKey ('$hiveWarehouseDir').")
-      hiveWarehouseDir
-    } else {
-      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
-      // the value of spark.sql.warehouse.dir.
-      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set
-      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
-      val sparkWarehouseDir = sparkConf.get(WAREHOUSE_PATH)
-      logInfo(s"Setting $hiveWarehouseKey ('$hiveWarehouseDir') to the value of " +
-        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
-      hadoopConf.set(hiveWarehouseKey, sparkWarehouseDir)
-      sparkWarehouseDir
-    }
-    logInfo(s"Warehouse path is '$warehousePath'.")
-  }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 3ddf4eccfd7a5..6b76927bd4150 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -46,7 +46,6 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider
-import org.apache.spark.sql.internal.SharedState
 import org.apache.spark.util.ShutdownHookManager
 
 /**
@@ -88,7 +87,6 @@ private[hive] object SparkSQLCLIDriver extends Logging {
 
     val sparkConf = new SparkConf(loadDefaults = true)
     val hadoopConf = SparkHadoopUtil.get.newConfiguration(sparkConf)
-    SharedState.loadHiveConfFile(sparkConf, hadoopConf)
     val extraConfigs = HiveUtils.formatTimeVarsForHiveClient(hadoopConf)
 
     val cliConf = HiveClientImpl.newHiveConf(sparkConf, hadoopConf, extraConfigs)
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index ed776633f903b..43aafc3c8590c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -32,7 +32,6 @@ import org.scalatest.BeforeAndAfterAll
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.hive.test.HiveTestJars
-import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
 import org.apache.spark.util.{ThreadUtils, Utils}
 
@@ -160,17 +159,6 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     }
   }
 
-  test("Pick spark.sql.warehouse.dir first for Spark Cli if set") {
-    val sparkWareHouseDir = Utils.createTempDir()
-    new File(warehousePath, "metastore_db").delete()
-    runCliWithin(
-      1.minute,
-      Seq("--conf", s"${StaticSQLConf.WAREHOUSE_PATH.key}=$sparkWareHouseDir"))(
-      "desc database default;" -> sparkWareHouseDir.getAbsolutePath
-    )
-    sparkWareHouseDir.delete()
-  }
-
   test("Simple commands") {
     val dataFilePath =
       Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
index 78535b094b83d..6e2dcfc04d498 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
@@ -45,6 +45,7 @@ class HiveSharedStateSuite extends SparkFunSuite {
       GLOBAL_TEMP_DATABASE.key -> tmpDb)
 
     val state = new SharedState(sc, initialConfigs)
+    assert(state.warehousePath !== invalidPath, "warehouse path can't determine by session options")
     assert(sc.conf.get(WAREHOUSE_PATH.key) !== invalidPath,
       "warehouse conf in session options can't affect application wide spark conf")
     assert(sc.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) !== invalidPath,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 8b97489e2d818..31ff62ed0a530 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -787,7 +787,7 @@ object SPARK_18360 {
       .enableHiveSupport().getOrCreate()
 
     val defaultDbLocation = spark.catalog.getDatabase("default").locationUri
-    assert(new Path(defaultDbLocation) == new Path(spark.conf.get(WAREHOUSE_PATH)))
+    assert(new Path(defaultDbLocation) == new Path(spark.sharedState.warehousePath))
 
     val hiveClient =
       spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client

From 7d4c1b894ef32170b421f51c42cf30198c35c21b Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 19 Mar 2020 20:11:04 -0700
Subject: [PATCH 0311/1280] [SPARK-25121][SQL] Supports multi-part table names
 for broadcast hint resolution

### What changes were proposed in this pull request?

This pr fixed code to respect a database name for broadcast table hint resolution.
Currently, spark ignores a database name in multi-part names;
```
scala> sql("CREATE DATABASE testDb")
scala> spark.range(10).write.saveAsTable("testDb.t")

// without this patch
scala> spark.range(10).join(spark.table("testDb.t"), "id").hint("broadcast", "testDb.t").explain
== Physical Plan ==
*(2) Project [id#24L]
+- *(2) BroadcastHashJoin [id#24L], [id#26L], Inner, BuildLeft
   :- BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]))
   :  +- *(1) Range (0, 10, step=1, splits=4)
   +- *(2) Project [id#26L]
      +- *(2) Filter isnotnull(id#26L)
         +- *(2) FileScan parquet testdb.t[id#26L] Batched: true, Format: Parquet, Location: InMemoryFileIndex[file:/Users/maropu/Repositories/spark/spark-2.3.1-bin-hadoop2.7/spark-warehouse..., PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint>

// with this patch
scala> spark.range(10).join(spark.table("testDb.t"), "id").hint("broadcast", "testDb.t").explain
== Physical Plan ==
*(2) Project [id#3L]
+- *(2) BroadcastHashJoin [id#3L], [id#5L], Inner, BuildRight
   :- *(2) Range (0, 10, step=1, splits=4)
   +- BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]))
      +- *(1) Project [id#5L]
         +- *(1) Filter isnotnull(id#5L)
            +- *(1) FileScan parquet testdb.t[id#5L] Batched: true, Format: Parquet, Location: InMemoryFileIndex[file:/Users/maropu/Repositories/spark/spark-master/spark-warehouse/testdb.db/t], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint>
```

This PR comes from https://github.com/apache/spark/pull/22198

### Why are the changes needed?

For better usability.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added unit tests.

Closes #27935 from maropu/SPARK-25121-2.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ca499e94091ae62a6ee76ea779d7b2b4cf2dbc5c)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/HintErrorLogger.scala   |  7 +-
 .../sql/catalyst/analysis/ResolveHints.scala  | 71 ++++++++++----
 .../sql/catalyst/expressions/package.scala    |  2 +-
 .../sql/catalyst/plans/logical/hints.scala    |  3 +-
 .../sql/catalyst/analysis/AnalysisTest.scala  |  2 +
 .../catalyst/analysis/ResolveHintsSuite.scala | 48 +++++++++
 .../sql/catalyst/analysis/TestRelations.scala |  2 +
 .../apache/spark/sql/DataFrameJoinSuite.scala | 98 ++++++++++++++++++-
 .../sql/execution/GlobalTempViewSuite.scala   | 24 ++++-
 9 files changed, 230 insertions(+), 27 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala
index c6e0c74527b8f..71c6d40dfba16 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HintErrorLogger.scala
@@ -24,15 +24,16 @@ import org.apache.spark.sql.catalyst.plans.logical.{HintErrorHandler, HintInfo}
  * The hint error handler that logs warnings for each hint error.
  */
 object HintErrorLogger extends HintErrorHandler with Logging {
+  import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 
   override def hintNotRecognized(name: String, parameters: Seq[Any]): Unit = {
     logWarning(s"Unrecognized hint: ${hintToPrettyString(name, parameters)}")
   }
 
   override def hintRelationsNotFound(
-      name: String, parameters: Seq[Any], invalidRelations: Set[String]): Unit = {
-    invalidRelations.foreach { n =>
-      logWarning(s"Count not find relation '$n' specified in hint " +
+      name: String, parameters: Seq[Any], invalidRelations: Set[Seq[String]]): Unit = {
+    invalidRelations.foreach { ident =>
+      logWarning(s"Count not find relation '${ident.quoted}' specified in hint " +
         s"'${hintToPrettyString(name, parameters)}'.")
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
index 5b77d67bd1340..81de086e78f91 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala
@@ -64,31 +64,59 @@ object ResolveHints {
             _.toUpperCase(Locale.ROOT)).contains(hintName.toUpperCase(Locale.ROOT))))
     }
 
+    // This method checks if given multi-part identifiers are matched with each other.
+    // The [[ResolveJoinStrategyHints]] rule is applied before the resolution batch
+    // in the analyzer and we cannot semantically compare them at this stage.
+    // Therefore, we follow a simple rule; they match if an identifier in a hint
+    // is a tail of an identifier in a relation. This process is independent of a session
+    // catalog (`currentDb` in [[SessionCatalog]]) and it just compares them literally.
+    //
+    // For example,
+    //  * in a query `SELECT /*+ BROADCAST(t) */ * FROM db1.t JOIN t`,
+    //    the broadcast hint will match both tables, `db1.t` and `t`,
+    //    even when the current db is `db2`.
+    //  * in a query `SELECT /*+ BROADCAST(default.t) */ * FROM default.t JOIN t`,
+    //    the broadcast hint will match the left-side table only, `default.t`.
+    private def matchedIdentifier(identInHint: Seq[String], identInQuery: Seq[String]): Boolean = {
+      if (identInHint.length <= identInQuery.length) {
+        identInHint.zip(identInQuery.takeRight(identInHint.length))
+          .forall { case (i1, i2) => resolver(i1, i2) }
+      } else {
+        false
+      }
+    }
+
+    private def extractIdentifier(r: SubqueryAlias): Seq[String] = {
+      r.identifier.qualifier :+ r.identifier.name
+    }
+
     private def applyJoinStrategyHint(
         plan: LogicalPlan,
-        relations: mutable.HashSet[String],
+        relationsInHint: Set[Seq[String]],
+        relationsInHintWithMatch: mutable.HashSet[Seq[String]],
         hintName: String): LogicalPlan = {
       // Whether to continue recursing down the tree
       var recurse = true
 
+      def matchedIdentifierInHint(identInQuery: Seq[String]): Boolean = {
+        relationsInHint.find(matchedIdentifier(_, identInQuery))
+          .map(relationsInHintWithMatch.add).nonEmpty
+      }
+
       val newNode = CurrentOrigin.withOrigin(plan.origin) {
         plan match {
           case ResolvedHint(u @ UnresolvedRelation(ident), hint)
-              if relations.exists(resolver(_, ident.last)) =>
-            relations.remove(ident.last)
+              if matchedIdentifierInHint(ident) =>
             ResolvedHint(u, createHintInfo(hintName).merge(hint, hintErrorHandler))
 
           case ResolvedHint(r: SubqueryAlias, hint)
-              if relations.exists(resolver(_, r.alias)) =>
-            relations.remove(r.alias)
+              if matchedIdentifierInHint(extractIdentifier(r)) =>
             ResolvedHint(r, createHintInfo(hintName).merge(hint, hintErrorHandler))
 
-          case u @ UnresolvedRelation(ident) if relations.exists(resolver(_, ident.last)) =>
-            relations.remove(ident.last)
+          case UnresolvedRelation(ident) if matchedIdentifierInHint(ident) =>
             ResolvedHint(plan, createHintInfo(hintName))
 
-          case r: SubqueryAlias if relations.exists(resolver(_, r.alias)) =>
-            relations.remove(r.alias)
+          case r: SubqueryAlias if matchedIdentifierInHint(extractIdentifier(r)) =>
             ResolvedHint(plan, createHintInfo(hintName))
 
           case _: ResolvedHint | _: View | _: With | _: SubqueryAlias =>
@@ -107,7 +135,9 @@ object ResolveHints {
       }
 
       if ((plan fastEquals newNode) && recurse) {
-        newNode.mapChildren(child => applyJoinStrategyHint(child, relations, hintName))
+        newNode.mapChildren { child =>
+          applyJoinStrategyHint(child, relationsInHint, relationsInHintWithMatch, hintName)
+        }
       } else {
         newNode
       }
@@ -120,17 +150,19 @@ object ResolveHints {
           ResolvedHint(h.child, createHintInfo(h.name))
         } else {
           // Otherwise, find within the subtree query plans to apply the hint.
-          val relationNames = h.parameters.map {
-            case tableName: String => tableName
-            case tableId: UnresolvedAttribute => tableId.name
+          val relationNamesInHint = h.parameters.map {
+            case tableName: String => UnresolvedAttribute.parseAttributeName(tableName)
+            case tableId: UnresolvedAttribute => tableId.nameParts
             case unsupported => throw new AnalysisException("Join strategy hint parameter " +
               s"should be an identifier or string but was $unsupported (${unsupported.getClass}")
-          }
-          val relationNameSet = new mutable.HashSet[String]
-          relationNames.foreach(relationNameSet.add)
-
-          val applied = applyJoinStrategyHint(h.child, relationNameSet, h.name)
-          hintErrorHandler.hintRelationsNotFound(h.name, h.parameters, relationNameSet.toSet)
+          }.toSet
+          val relationsInHintWithMatch = new mutable.HashSet[Seq[String]]
+          val applied = applyJoinStrategyHint(
+            h.child, relationNamesInHint, relationsInHintWithMatch, h.name)
+
+          // Filters unmatched relation identifiers in the hint
+          val unmatchedIdents = relationNamesInHint -- relationsInHintWithMatch
+          hintErrorHandler.hintRelationsNotFound(h.name, h.parameters, unmatchedIdents)
           applied
         }
     }
@@ -246,5 +278,4 @@ object ResolveHints {
         h.child
     }
   }
-
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
index 1b59056c4da66..8bf1f19844556 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/package.scala
@@ -196,7 +196,7 @@ package object expressions  {
       // For example, consider an example where "cat" is the catalog name, "db1" is the database
       // name, "a" is the table name and "b" is the column name and "c" is the struct field name.
       // If the name parts is cat.db1.a.b.c, then Attribute will match
-      // Attribute(b, qualifier("cat", "db1, "a")) and List("c") will be the second element
+      // Attribute(b, qualifier("cat", "db1", "a")) and List("c") will be the second element
       var matches: (Seq[Attribute], Seq[String]) = nameParts match {
         case catalogPart +: dbPart +: tblPart +: name +: nestedFields =>
           val key = (catalogPart.toLowerCase(Locale.ROOT), dbPart.toLowerCase(Locale.ROOT),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
index f26e5662ee856..a325b61fcc5a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/hints.scala
@@ -186,7 +186,8 @@ trait HintErrorHandler {
    * @param parameters the hint parameters
    * @param invalidRelations the set of relation names that cannot be associated
    */
-  def hintRelationsNotFound(name: String, parameters: Seq[Any], invalidRelations: Set[String]): Unit
+  def hintRelationsNotFound(
+    name: String, parameters: Seq[Any], invalidRelations: Set[Seq[String]]): Unit
 
   /**
    * Callback for a join hint specified on a relation that is not part of a join.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
index 3f8d409992381..4473c20b2cca6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
@@ -45,6 +45,8 @@ trait AnalysisTest extends PlanTest {
     catalog.createTempView("TaBlE", TestRelations.testRelation, overrideIfExists = true)
     catalog.createTempView("TaBlE2", TestRelations.testRelation2, overrideIfExists = true)
     catalog.createTempView("TaBlE3", TestRelations.testRelation3, overrideIfExists = true)
+    catalog.createGlobalTempView("TaBlE4", TestRelations.testRelation4, overrideIfExists = true)
+    catalog.createGlobalTempView("TaBlE5", TestRelations.testRelation5, overrideIfExists = true)
     new Analyzer(catalog, conf) {
       override val extendedResolutionRules = EliminateSubqueryAliases +: extendedAnalysisRules
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
index 5e66c038738a4..ca7d28401cf2a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
@@ -241,4 +241,52 @@ class ResolveHintsSuite extends AnalysisTest {
       Project(testRelation.output, testRelation),
       caseSensitive = false)
   }
+
+  test("Supports multi-part table names for broadcast hint resolution") {
+    // local temp table (single-part identifier case)
+    checkAnalysis(
+      UnresolvedHint("MAPJOIN", Seq("table", "table2"),
+        table("TaBlE").join(table("TaBlE2"))),
+      Join(
+        ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
+        ResolvedHint(testRelation2, HintInfo(strategy = Some(BROADCAST))),
+        Inner,
+        None,
+        JoinHint.NONE),
+      caseSensitive = false)
+
+    checkAnalysis(
+      UnresolvedHint("MAPJOIN", Seq("TaBlE", "table2"),
+        table("TaBlE").join(table("TaBlE2"))),
+      Join(
+        ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
+        testRelation2,
+        Inner,
+        None,
+        JoinHint.NONE),
+      caseSensitive = true)
+
+    // global temp table (multi-part identifier case)
+    checkAnalysis(
+      UnresolvedHint("MAPJOIN", Seq("GlOBal_TeMP.table4", "table5"),
+        table("global_temp", "table4").join(table("global_temp", "table5"))),
+      Join(
+        ResolvedHint(testRelation4, HintInfo(strategy = Some(BROADCAST))),
+        ResolvedHint(testRelation5, HintInfo(strategy = Some(BROADCAST))),
+        Inner,
+        None,
+        JoinHint.NONE),
+      caseSensitive = false)
+
+    checkAnalysis(
+      UnresolvedHint("MAPJOIN", Seq("global_temp.TaBlE4", "table5"),
+        table("global_temp", "TaBlE4").join(table("global_temp", "TaBlE5"))),
+      Join(
+        ResolvedHint(testRelation4, HintInfo(strategy = Some(BROADCAST))),
+        testRelation5,
+        Inner,
+        None,
+        JoinHint.NONE),
+      caseSensitive = true)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
index e12e272aedffe..33b6029070938 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TestRelations.scala
@@ -44,6 +44,8 @@ object TestRelations {
     AttributeReference("g", StringType)(),
     AttributeReference("h", MapType(IntegerType, IntegerType))())
 
+  val testRelation5 = LocalRelation(AttributeReference("i", StringType)())
+
   val nestedRelation = LocalRelation(
     AttributeReference("top", StructType(
       StructField("duplicateField", StringType) ::
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index c7545bcad8962..6b772e53ac184 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -17,10 +17,14 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.{Inner, InnerLike, LeftOuter, RightOuter}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Filter, HintInfo, Join, JoinHint, LogicalPlan, Project}
+import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.datasources.LogicalRelation
+import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -322,4 +326,96 @@ class DataFrameJoinSuite extends QueryTest
       }
     }
   }
+
+  test("Supports multi-part names for broadcast hint resolution") {
+    val (table1Name, table2Name) = ("t1", "t2")
+
+    withTempDatabase { dbName =>
+      withTable(table1Name, table2Name) {
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          spark.range(50).write.saveAsTable(s"$dbName.$table1Name")
+          spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
+
+          def checkIfHintApplied(df: DataFrame): Unit = {
+            val sparkPlan = df.queryExecution.executedPlan
+            val broadcastHashJoins = sparkPlan.collect { case p: BroadcastHashJoinExec => p }
+            assert(broadcastHashJoins.size == 1)
+            val broadcastExchanges = broadcastHashJoins.head.collect {
+              case p: BroadcastExchangeExec => p
+            }
+            assert(broadcastExchanges.size == 1)
+            val tables = broadcastExchanges.head.collect {
+              case FileSourceScanExec(_, _, _, _, _, _, Some(tableIdent)) => tableIdent
+            }
+            assert(tables.size == 1)
+            assert(tables.head === TableIdentifier(table1Name, Some(dbName)))
+          }
+
+          def checkIfHintNotApplied(df: DataFrame): Unit = {
+            val sparkPlan = df.queryExecution.executedPlan
+            val broadcastHashJoins = sparkPlan.collect { case p: BroadcastHashJoinExec => p }
+            assert(broadcastHashJoins.isEmpty)
+          }
+
+          def sqlTemplate(tableName: String, hintTableName: String): DataFrame = {
+            sql(s"SELECT /*+ BROADCASTJOIN($hintTableName) */ * " +
+              s"FROM $tableName, $dbName.$table2Name " +
+              s"WHERE $tableName.id = $table2Name.id")
+          }
+
+          def dfTemplate(tableName: String, hintTableName: String): DataFrame = {
+            spark.table(tableName).join(spark.table(s"$dbName.$table2Name"), "id")
+              .hint("broadcast", hintTableName)
+          }
+
+          sql(s"USE $dbName")
+
+          checkIfHintApplied(sqlTemplate(table1Name, table1Name))
+          checkIfHintApplied(sqlTemplate(s"$dbName.$table1Name", s"$dbName.$table1Name"))
+          checkIfHintApplied(sqlTemplate(s"$dbName.$table1Name", table1Name))
+          checkIfHintNotApplied(sqlTemplate(table1Name, s"$dbName.$table1Name"))
+
+          checkIfHintApplied(dfTemplate(table1Name, table1Name))
+          checkIfHintApplied(dfTemplate(s"$dbName.$table1Name", s"$dbName.$table1Name"))
+          checkIfHintApplied(dfTemplate(s"$dbName.$table1Name", table1Name))
+          checkIfHintApplied(dfTemplate(table1Name, s"$dbName.$table1Name"))
+          checkIfHintApplied(dfTemplate(table1Name,
+            s"${CatalogManager.SESSION_CATALOG_NAME}.$dbName.$table1Name"))
+
+          withView("tv") {
+            sql(s"CREATE VIEW tv AS SELECT * FROM $dbName.$table1Name")
+            checkIfHintApplied(sqlTemplate("tv", "tv"))
+            checkIfHintNotApplied(sqlTemplate("tv", s"$dbName.tv"))
+
+            checkIfHintApplied(dfTemplate("tv", "tv"))
+            checkIfHintApplied(dfTemplate("tv", s"$dbName.tv"))
+          }
+        }
+      }
+    }
+  }
+
+  test("The same table name exists in two databases for broadcast hint resolution") {
+    val (db1Name, db2Name) = ("db1", "db2")
+
+    withDatabase(db1Name, db2Name) {
+      withTable("t") {
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          sql(s"CREATE DATABASE $db1Name")
+          sql(s"CREATE DATABASE $db2Name")
+          spark.range(1).write.saveAsTable(s"$db1Name.t")
+          spark.range(1).write.saveAsTable(s"$db2Name.t")
+
+          // Checks if a broadcast hint applied in both sides
+          val statement = s"SELECT /*+ BROADCASTJOIN(t) */ * FROM $db1Name.t, $db2Name.t " +
+            s"WHERE $db1Name.t.id = $db2Name.t.id"
+          sql(statement).queryExecution.optimizedPlan match {
+            case Join(_, _, _, _, JoinHint(Some(HintInfo(Some(BROADCAST))),
+              Some(HintInfo(Some(BROADCAST))))) =>
+            case _ => fail("broadcast hint not found in both tables")
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
index 7fbfa73623c85..28e82aa14e0d0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala
@@ -20,7 +20,8 @@ package org.apache.spark.sql.execution
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalog.Table
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, HintInfo, Join, JoinHint}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
 
@@ -170,4 +171,25 @@ class GlobalTempViewSuite extends QueryTest with SharedSparkSession {
         isTemporary = true).toString)
     }
   }
+
+  test("broadcast hint on global temp view") {
+    withGlobalTempView("v1") {
+      spark.range(10).createGlobalTempView("v1")
+      withTempView("v2") {
+        spark.range(10).createTempView("v2")
+
+        withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+          Seq(
+            "SELECT /*+ MAPJOIN(v1) */ * FROM global_temp.v1, v2 WHERE v1.id = v2.id",
+            "SELECT /*+ MAPJOIN(global_temp.v1) */ * FROM global_temp.v1, v2 WHERE v1.id = v2.id"
+          ).foreach { statement =>
+            sql(statement).queryExecution.optimizedPlan match {
+              case Join(_, _, _, _, JoinHint(Some(HintInfo(Some(BROADCAST))), None)) =>
+              case _ => fail("broadcast hint not found in a left-side table")
+            }
+          }
+        }
+      }
+    }
+  }
 }

From c42f9f61f40d4d796413ab2bfb58fbeff5ceb68b Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 20 Mar 2020 12:28:57 +0800
Subject: [PATCH 0312/1280] [SPARK-31181][SQL][TESTS] Remove the default value
 assumption on CREATE TABLE test cases

A few `CREATE TABLE` test cases have some assumption on the default value of `LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED`. This PR (SPARK-31181) makes the test cases more explicit from test-case side.

The configuration change was tested via https://github.com/apache/spark/pull/27894 during discussing SPARK-31136. This PR has only the test case part from that PR.

This makes our test case more robust in terms of the default value of `LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED`. Even in the case where we switch the conf value, that will be one-liner with no test case changes.

No.

Pass the Jenkins with the existing tests.

Closes #27946 from dongjoon-hyun/SPARK-EXPLICIT-TEST.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f1cc86792f825f76c2689660764d62d7be0d1989)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/catalyst/parser/DDLParserSuite.scala  | 27 ++++++++++---------
 .../inputs/describe-table-column.sql          |  2 +-
 .../inputs/postgreSQL/create_view.sql         |  2 +-
 .../results/describe-table-column.sql.out     |  2 +-
 .../results/postgreSQL/create_view.sql.out    |  2 +-
 .../sql/connector/DataSourceV2SQLSuite.scala  |  2 ++
 .../execution/command/DDLParserSuite.scala    |  3 ++-
 .../spark/sql/hive/StatisticsSuite.scala      |  4 ++-
 8 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index e3570899db9fb..35a54c8e95b72 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.expressions.{EqualTo, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition.{after, first}
 import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType, TimestampType}
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -2117,18 +2118,20 @@ class DDLParserSuite extends AnalysisTest {
   }
 
   test("create table - without using") {
-    val sql = "CREATE TABLE 1m.2g(a INT)"
-    val expectedTableSpec = TableSpec(
-      Seq("1m", "2g"),
-      Some(new StructType().add("a", IntegerType)),
-      Seq.empty[Transform],
-      None,
-      Map.empty[String, String],
-      None,
-      Map.empty[String, String],
-      None,
-      None)
+    withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "false") {
+      val sql = "CREATE TABLE 1m.2g(a INT)"
+      val expectedTableSpec = TableSpec(
+        Seq("1m", "2g"),
+        Some(new StructType().add("a", IntegerType)),
+        Seq.empty[Transform],
+        None,
+        Map.empty[String, String],
+        None,
+        Map.empty[String, String],
+        None,
+        None)
 
-    testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
+    }
   }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql b/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
index 821cb473751eb..d55e398329b76 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/describe-table-column.sql
@@ -52,7 +52,7 @@ DROP TABLE desc_complex_col_table;
 
 --Test case insensitive
 
-CREATE TABLE customer(CName STRING);
+CREATE TABLE customer(CName STRING) USING PARQUET;
 
 INSERT INTO customer VALUES('Maria');
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
index 39e708478e298..21ffd85f7d01f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/create_view.sql
@@ -41,7 +41,7 @@ DROP TABLE emp;
 -- These views are left around mainly to exercise special cases in pg_dump.
 
 -- [SPARK-19842] Informational Referential Integrity Constraints Support in Spark
-CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20));
+CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20)) USING PARQUET;
 --
 CREATE VIEW key_dependent_view AS
    SELECT * FROM view_base_table GROUP BY key;
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
index ae9240ec588da..c6d3d45879eb1 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-table-column.sql.out
@@ -267,7 +267,7 @@ struct<>
 
 
 -- !query
-CREATE TABLE customer(CName STRING)
+CREATE TABLE customer(CName STRING) USING PARQUET
 -- !query schema
 struct<>
 -- !query output
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
index 1f2bd5795cf1e..ae1cb2f171704 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out
@@ -42,7 +42,7 @@ struct<>
 
 
 -- !query
-CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20))
+CREATE TABLE view_base_table (key int /* PRIMARY KEY */, data varchar(20)) USING PARQUET
 -- !query schema
 struct<>
 -- !query output
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 07e0959516040..ffaff66492879 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -257,6 +257,7 @@ class DataSourceV2SQLSuite
   }
 
   test("CreateTable: without USING clause") {
+    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key, "false")
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
     val testCatalog = catalog("testcat").asTableCatalog
@@ -681,6 +682,7 @@ class DataSourceV2SQLSuite
   }
 
   test("CreateTableAsSelect: without USING clause") {
+    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key, "false")
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
     val testCatalog = catalog("testcat").asTableCatalog
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
index bacd64efedc83..28e5082886b67 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
@@ -40,7 +40,8 @@ import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 
 class DDLParserSuite extends AnalysisTest with SharedSparkSession {
-  private lazy val parser = new SparkSqlParser(new SQLConf)
+  private lazy val parser = new SparkSqlParser(new SQLConf().copy(
+    SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED -> false))
 
   private def assertUnsupported(sql: String, containsThesePhrases: Seq[String] = Seq()): Unit = {
     val e = intercept[ParseException] {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index 488175a22bad7..c1eab63ec073f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -1520,10 +1520,12 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
     val ext_tbl = "SPARK_30269_external"
     withTempDir { dir =>
       withTable(tbl, ext_tbl) {
-        sql(s"CREATE TABLE $tbl (key INT, value STRING, ds STRING) PARTITIONED BY (ds)")
+        sql(s"CREATE TABLE $tbl (key INT, value STRING, ds STRING)" +
+          "USING parquet PARTITIONED BY (ds)")
         sql(
           s"""
              | CREATE TABLE $ext_tbl (key INT, value STRING, ds STRING)
+             | USING PARQUET
              | PARTITIONED BY (ds)
              | LOCATION '${dir.toURI}'
            """.stripMargin)

From 028091fea4bfb3bc1a8ae114fd01be2c9bfcb82d Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 20 Mar 2020 13:57:49 +0800
Subject: [PATCH 0313/1280] [SPARK-31183][SQL] Rebase date/timestamp from/to
 Julian calendar in Avro

The PR addresses the issue of compatibility with Spark 2.4 and earlier version in reading/writing dates and timestamp via **Avro** datasource. Previous releases are based on a hybrid calendar - Julian + Gregorian. Since Spark 3.0, Proleptic Gregorian calendar is used by default, see SPARK-26651. In particular, the issue pops up for dates/timestamps before 1582-10-15 when the hybrid calendar switches from/to Gregorian to/from Julian calendar. The same local date in different calendar is converted to different number of days since the epoch 1970-01-01. For example, the 1001-01-01 date is converted to:
- -719164 in Julian calendar. Spark 2.4 saves the number as a value of DATE type into **Avro** files.
- -719162 in Proleptic Gregorian calendar. Spark 3.0 saves the number as a date value.

The PR proposes rebasing from/to Proleptic Gregorian calendar to the hybrid one under the SQL config:
```
spark.sql.legacy.avro.rebaseDateTime.enabled
```
which is set to `false` by default which means the rebasing is not performed by default.

The details of the implementation:
1. Re-use 2 methods of `DateTimeUtils` added by the PR https://github.com/apache/spark/pull/27915 for rebasing microseconds.
2. Re-use 2 methods of `DateTimeUtils` added by the PR https://github.com/apache/spark/pull/27915 for rebasing days.
3. Use `rebaseGregorianToJulianMicros()` and `rebaseGregorianToJulianDays()` while saving timestamps/dates to **Avro** files if the SQL config is on.
4. Use `rebaseJulianToGregorianMicros()` and `rebaseJulianToGregorianDays()` while loading timestamps/dates from **Avro** files if the SQL config is on.
5. The SQL config `spark.sql.legacy.avro.rebaseDateTime.enabled` controls conversions from/to dates, and timestamps of the `timestamp-millis`, `timestamp-micros` logical types.

For the backward compatibility with Spark 2.4 and earlier versions. The changes allow users to read dates/timestamps saved by previous version, and get the same result. Also after the changes, users can enable the rebasing in write, and save dates/timestamps that can be loaded correctly by Spark 2.4 and earlier versions.

Yes, the timestamp `1001-01-01 01:02:03.123456` saved by Spark 2.4.5 as `timestamp-micros` is interpreted by Spark 3.0.0-preview2 differently:
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> spark.read.format("avro").load("/Users/maxim/tmp/before_1582/2_4_5_date_avro").show(false)
+----------+
|date      |
+----------+
|1001-01-07|
+----------+
```
After the changes:
```scala
scala> spark.conf.set("spark.sql.legacy.avro.rebaseDateTime.enabled", true)
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")

scala> spark.read.format("avro").load("/Users/maxim/tmp/before_1582/2_4_5_date_avro").show(false)
+----------+
|date      |
+----------+
|1001-01-01|
+----------+
```

1. Added tests to `AvroLogicalTypeSuite` to check rebasing in read. The test reads back avro files saved by Spark 2.4.5 via:
```shell
$ export TZ="America/Los_Angeles"
```
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> val df = Seq("1001-01-01").toDF("dateS").select($"dateS".cast("date").as("date"))
df: org.apache.spark.sql.DataFrame = [date: date]
scala> df.write.format("avro").save("/Users/maxim/tmp/before_1582/2_4_5_date_avro")

scala> val df2 = Seq("1001-01-01 01:02:03.123456").toDF("tsS").select($"tsS".cast("timestamp").as("ts"))
df2: org.apache.spark.sql.DataFrame = [ts: timestamp]
scala> df2.write.format("avro").save("/Users/maxim/tmp/before_1582/2_4_5_ts_avro")

scala> :paste
// Entering paste mode (ctrl-D to finish)

  val timestampSchema = s"""
    |  {
    |    "namespace": "logical",
    |    "type": "record",
    |    "name": "test",
    |    "fields": [
    |      {"name": "ts", "type": ["null", {"type": "long","logicalType": "timestamp-millis"}], "default": null}
    |    ]
    |  }
    |""".stripMargin

// Exiting paste mode, now interpreting.
scala> df3.write.format("avro").option("avroSchema", timestampSchema).save("/Users/maxim/tmp/before_1582/2_4_5_ts_millis_avro")

```

2. Added the following tests to `AvroLogicalTypeSuite` to check rebasing of dates/timestamps (in microsecond and millisecond precision). The tests write rebased a date/timestamps and read them back w/ enabled/disabled rebasing, and compare results. :
  - `rebasing microseconds timestamps in write`
  - `rebasing milliseconds timestamps in write`
  - `rebasing dates in write`

Closes #27953 from MaxGekk/rebase-avro-datetime.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 4766a3664729644b5391b13805cdee44501025d8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/avro/AvroDeserializer.scala     |  34 ++++--
 .../spark/sql/avro/AvroSerializer.scala       |  26 ++++-
 .../test/resources/before_1582_date_v2_4.avro | Bin 0 -> 202 bytes
 .../resources/before_1582_ts_micros_v2_4.avro | Bin 0 -> 218 bytes
 .../resources/before_1582_ts_millis_v2_4.avro | Bin 0 -> 244 bytes
 .../spark/sql/avro/AvroLogicalTypeSuite.scala |  98 +++++++++++++++++-
 .../apache/spark/sql/internal/SQLConf.scala   |  15 +++
 7 files changed, 158 insertions(+), 15 deletions(-)
 create mode 100644 external/avro/src/test/resources/before_1582_date_v2_4.avro
 create mode 100644 external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro
 create mode 100644 external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 2c17c16f06da7..b98f3036686f3 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -32,8 +32,9 @@ import org.apache.avro.util.Utf8
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 /**
@@ -42,6 +43,9 @@ import org.apache.spark.unsafe.types.UTF8String
 class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
   private lazy val decimalConversions = new DecimalConversion()
 
+  // Enable rebasing date/timestamp from Julian to Proleptic Gregorian calendar
+  private val rebaseDateTime = SQLConf.get.avroRebaseDateTimeEnabled
+
   private val converter: Any => Any = rootCatalystType match {
     // A shortcut for empty schema.
     case st: StructType if st.isEmpty =>
@@ -88,6 +92,11 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
       case (INT, IntegerType) => (updater, ordinal, value) =>
         updater.setInt(ordinal, value.asInstanceOf[Int])
 
+      case (INT, DateType) if rebaseDateTime => (updater, ordinal, value) =>
+        val days = value.asInstanceOf[Int]
+        val rebasedDays = DateTimeUtils.rebaseJulianToGregorianDays(days)
+        updater.setInt(ordinal, rebasedDays)
+
       case (INT, DateType) => (updater, ordinal, value) =>
         updater.setInt(ordinal, value.asInstanceOf[Int])
 
@@ -95,14 +104,23 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
         updater.setLong(ordinal, value.asInstanceOf[Long])
 
       case (LONG, TimestampType) => avroType.getLogicalType match {
-        case _: TimestampMillis => (updater, ordinal, value) =>
-          updater.setLong(ordinal, value.asInstanceOf[Long] * 1000)
+        // For backward compatibility, if the Avro type is Long and it is not logical type
+        // (the `null` case), the value is processed as timestamp type with millisecond precision.
+        case null | _: TimestampMillis => (updater, ordinal, value) =>
+          val millis = value.asInstanceOf[Long]
+          val micros = DateTimeUtils.fromMillis(millis)
+          if (rebaseDateTime) {
+            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
+          } else {
+            updater.setLong(ordinal, micros)
+          }
         case _: TimestampMicros => (updater, ordinal, value) =>
-          updater.setLong(ordinal, value.asInstanceOf[Long])
-        case null => (updater, ordinal, value) =>
-          // For backward compatibility, if the Avro type is Long and it is not logical type,
-          // the value is processed as timestamp type with millisecond precision.
-          updater.setLong(ordinal, value.asInstanceOf[Long] * 1000)
+          val micros = value.asInstanceOf[Long]
+          if (rebaseDateTime) {
+            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
+          } else {
+            updater.setLong(ordinal, micros)
+          }
         case other => throw new IncompatibleSchemaException(
           s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
       }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index b7bf7e5543033..af9e3a5f81f82 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -34,6 +34,8 @@ import org.apache.avro.util.Utf8
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
@@ -42,6 +44,9 @@ import org.apache.spark.sql.types._
 class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean)
   extends Logging {
 
+  // Whether to rebase datetimes from Gregorian to Julian calendar in write
+  private val rebaseDateTime: Boolean = SQLConf.get.avroRebaseDateTimeEnabled
+
   def serialize(catalystData: Any): Any = {
     converter.apply(catalystData)
   }
@@ -135,15 +140,26 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
       case (BinaryType, BYTES) =>
         (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
 
+      case (DateType, INT) if rebaseDateTime =>
+        (getter, ordinal) => DateTimeUtils.rebaseGregorianToJulianDays(getter.getInt(ordinal))
+
       case (DateType, INT) =>
         (getter, ordinal) => getter.getInt(ordinal)
 
       case (TimestampType, LONG) => avroType.getLogicalType match {
-          case _: TimestampMillis => (getter, ordinal) => getter.getLong(ordinal) / 1000
-          case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
-          // For backward compatibility, if the Avro type is Long and it is not logical type,
-          // output the timestamp value as with millisecond precision.
-          case null => (getter, ordinal) => getter.getLong(ordinal) / 1000
+          // For backward compatibility, if the Avro type is Long and it is not logical type
+          // (the `null` case), output the timestamp value as with millisecond precision.
+          case null | _: TimestampMillis => (getter, ordinal) =>
+            val micros = getter.getLong(ordinal)
+            val rebasedMicros = if (rebaseDateTime) {
+              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
+            } else micros
+            DateTimeUtils.toMillis(rebasedMicros)
+          case _: TimestampMicros => (getter, ordinal) =>
+            val micros = getter.getLong(ordinal)
+            if (rebaseDateTime) {
+              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
+            } else micros
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
         }
diff --git a/external/avro/src/test/resources/before_1582_date_v2_4.avro b/external/avro/src/test/resources/before_1582_date_v2_4.avro
new file mode 100644
index 0000000000000000000000000000000000000000..96aa7cbf176a537ca3e845aab126eba8ed6aba7c
GIT binary patch
literal 202
zcmeZI%3@>@ODrqO*DFrWNX<?Bz*wzRQdy9yWTjM;nw(#hqNJmgmzWFUm*f}tq?V=T
z1i{49GE;L>ij}OQt6@q~5=&Bn8XyJ$g`p;8=9K_NbMn(OlM{17Af|(r)&dPL&B;-U
zt&Ob}0lOkOKP5Gpr#LUMprDdrzHQd^1CI?3%Fp`waqSAJTqY3~9)`~|wP*dj)rKwy
E0Gm2Ya{vGU

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro b/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro
new file mode 100644
index 0000000000000000000000000000000000000000..efe5e71a58813ecd1bb2c8105341103518c984d3
GIT binary patch
literal 218
zcmeZI%3@>@ODrqO*DFrWNX<=bW2#mvsVqoUvQjEaP0lY$QPNS$OUwoFOY#eRQp-|v
zf?(olnW;G`#Y$Gu)o>-nK;;naKtZTEIr(|%K;fMH^vvYMoDhiNC7HRY#U+Wk1-iMJ
z$wm3aO0__HN^^3QVrye-MZgY9&QD2A<|)ogEGVdC==4kz>^@w+=(}Ubgt!%Rzca~k
WDlq(?{OIYcl^1iwrcDY&R{#JKF;cAn

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro b/external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro
new file mode 100644
index 0000000000000000000000000000000000000000..dbaec814eb9549543358c68f94429140f60318ba
GIT binary patch
literal 244
zcmeZI%3@>@ODrqO*DFrWNX<>$!&I$QQdy9yWTjM;nw(#hqNJmgmzWFUm!uY#KzYRl
ziOC?Loc#36<is4HXj*1!PD-(oRdh96c`;BJVpOzJUTF?c72N2Y{JeCaEL3|4#J-Zu
zT%Zk!xdpnpnK?O`#Y(lYI!Y<2X^EvdB}!Hx6KZ2?MZj)N&QD2A<|)ogEGVdC*uGiJ
k{eGC8pMIp9k^GnQVoWj|GECDi{dxc4Gw;$xHH*>Z0d-baZvX%Q

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 82569653c1f23..9e89b69c0b33c 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.avro
 
 import java.io.File
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
 
 import org.apache.avro.{LogicalTypes, Schema}
 import org.apache.avro.Conversions.DecimalConversion
@@ -25,7 +25,7 @@ import org.apache.avro.file.DataFileWriter
 import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord}
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -348,6 +348,100 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
       assert(msg.contains("Unscaled value too large for precision"))
     }
   }
+
+  private def readResourceAvroFile(name: String): DataFrame = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(name)
+    spark.read.format("avro").load(url.toString)
+  }
+
+  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+      checkAnswer(
+        readResourceAvroFile("before_1582_date_v2_4.avro"),
+        Row(java.sql.Date.valueOf("1001-01-01")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
+    }
+  }
+
+  test("SPARK-31183: rebasing microseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val nonRebased = "1001-01-07 01:09:05.123456"
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq(tsStr).toDF("tsS")
+          .select($"tsS".cast("timestamp").as("ts"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing milliseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val rebased = "1001-01-01 01:02:03.123"
+    val nonRebased = "1001-01-07 01:09:05.123"
+    Seq(
+      """{"type": "long","logicalType": "timestamp-millis"}""",
+      """"long"""").foreach { tsType =>
+      val timestampSchema = s"""
+          |{
+          |  "namespace": "logical",
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {"name": "ts", "type": $tsType}
+          |  ]
+          |}""".stripMargin
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+          Seq(tsStr).toDF("tsS")
+            .select($"tsS".cast("timestamp").as("ts"))
+            .write
+            .option("avroSchema", timestampSchema)
+            .format("avro")
+            .save(path)
+
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(rebased)))
+        }
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(nonRebased)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq("1001-01-01").toDF("dateS")
+          .select($"dateS".cast("date").as("date"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
+      }
+    }
+  }
 }
 
 class AvroV1LogicalTypeSuite extends AvroLogicalTypeSuite {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index a47f7d9340ec4..9681a5d25306e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2245,6 +2245,19 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val LEGACY_AVRO_REBASE_DATETIME =
+    buildConf("spark.sql.legacy.avro.rebaseDateTime.enabled")
+      .internal()
+      .doc("When true, rebase dates/timestamps from Proleptic Gregorian calendar " +
+        "to the hybrid calendar (Julian + Gregorian) in write and " +
+        "from the hybrid calendar to Proleptic Gregorian calendar in read. " +
+        "The rebasing is performed by converting micros/millis/days to " +
+        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
+        "timestamp in the target calendar, and getting the number of micros/millis/days " +
+        "since the epoch 1970-01-01 00:00:00Z.")
+      .booleanConf
+      .createWithDefault(false)
+
   /**
    * Holds information about keys that have been deprecated.
    *
@@ -2822,6 +2835,8 @@ class SQLConf extends Serializable with Logging {
 
   def parquetRebaseDateTimeEnabled: Boolean = getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME)
 
+  def avroRebaseDateTimeEnabled: Boolean = getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME)
+
   /** ********************** SQLConf functionality methods ************ */
 
   /** Set Spark SQL configuration properties. */

From 5a57579fd6f8fb903d403144f70c3f353964c33d Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 20 Mar 2020 15:57:21 +0900
Subject: [PATCH 0314/1280] [SPARK-31195][SQL] Correct and reuse days rebase
 functions of `DateTimeUtils` in `DaysWritable`

### What changes were proposed in this pull request?
In the PR, I propose to correct and re-use functions from `DateTimeUtils` for rebasing days before the cutover day `1582-10-15` in `org.apache.spark.sql.hive.DaysWritable`.

### Why are the changes needed?
0. Existing rebasing of days in `DaysWritable` is not correct.
1. To deduplicate code in `DaysWritable`
2. To use functions that are better tested and cross checked by loading dates/timestamps from Parquet/Avro files written by Spark 2.4.5

### Does this PR introduce any user-facing change?
This PR can introduce behavior change because the replaced code is different from the re-used code from `DateTimeUtils`.

### How was this patch tested?
By existing test suite, for instance `HiveOrcHadoopFsRelationSuite`.

Closes #27962 from MaxGekk/reuse-rebase-funcs.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 6a668763b81cce4b4e50d5b34f8e79c493031f86)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/hive/DaysWritable.scala  | 24 +++----------------
 1 file changed, 3 insertions(+), 21 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
index 53a0deb270fd1..862be33d7c0da 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
@@ -19,13 +19,11 @@ package org.apache.spark.sql.hive
 
 import java.io.{DataInput, DataOutput, IOException}
 import java.sql.Date
-import java.time.LocalDate
-import java.util.Calendar
 
 import org.apache.hadoop.hive.serde2.io.DateWritable
 import org.apache.hadoop.io.WritableUtils
 
-import org.apache.spark.sql.catalyst.util.{DateTimeConstants, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 
 /**
  * The class accepts/returns days in Gregorian calendar and rebase them
@@ -82,17 +80,7 @@ private[hive] object DaysWritable {
   // The code below converts -141714 to -141704.
   def rebaseGregorianToJulianDays(daysSinceEpoch: Int): Int = {
     if (daysSinceEpoch < DateTimeUtils.GREGORIAN_CUTOVER_DAY) {
-      val millis = Math.multiplyExact(daysSinceEpoch, DateTimeConstants.MILLIS_PER_DAY)
-      val utcCal = new Calendar.Builder()
-        .setCalendarType("gregory")
-        .setTimeZone(DateTimeUtils.TimeZoneUTC)
-        .setInstant(millis)
-        .build()
-      val localDate = LocalDate.of(
-        utcCal.get(Calendar.YEAR),
-        utcCal.get(Calendar.MONTH) + 1,
-        utcCal.get(Calendar.DAY_OF_MONTH))
-      Math.toIntExact(localDate.toEpochDay)
+      DateTimeUtils.rebaseGregorianToJulianDays(daysSinceEpoch)
     } else {
       daysSinceEpoch
     }
@@ -100,13 +88,7 @@ private[hive] object DaysWritable {
 
   def rebaseJulianToGregorianDays(daysSinceEpoch: Int): Int = {
     if (daysSinceEpoch < JULIAN_CUTOVER_DAY) {
-      val localDate = LocalDate.ofEpochDay(daysSinceEpoch)
-      val utcCal = new Calendar.Builder()
-        .setCalendarType("gregory")
-        .setTimeZone(DateTimeUtils.TimeZoneUTC)
-        .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
-        .build()
-      Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, DateTimeConstants.MILLIS_PER_DAY))
+      DateTimeUtils.rebaseJulianToGregorianDays(daysSinceEpoch)
     } else {
       daysSinceEpoch
     }

From a6f3e3b096e2d7a39e0b2fdec6452e6d633baf7e Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 20 Mar 2020 19:02:54 +0900
Subject: [PATCH 0315/1280] [SPARK-31183][SQL][FOLLOWUP] Move rebase tests to
 `AvroSuite` and check the rebase flag out of function bodies

1. The tests added by #27953 are moved from `AvroLogicalTypeSuite` to `AvroSuite`.
2. Checking of the `rebaseDateTime` flag is moved out from functions bodies.

1. The tests are moved because they are not directly related to logical types.
2. Checking the flag out of functions bodies should improve performance.

No

By running Avro tests via the command `build/sbt avro/test`

Closes #27964 from MaxGekk/rebase-avro-datetime-followup.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/avro/AvroDeserializer.scala     |  21 +--
 .../spark/sql/avro/AvroSerializer.scala       |  18 ++-
 .../spark/sql/avro/AvroLogicalTypeSuite.scala |  98 +-------------
 .../org/apache/spark/sql/avro/AvroSuite.scala | 124 ++++++++++++++++--
 4 files changed, 131 insertions(+), 130 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index b98f3036686f3..3e8a7f95ccd96 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -106,21 +106,22 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
       case (LONG, TimestampType) => avroType.getLogicalType match {
         // For backward compatibility, if the Avro type is Long and it is not logical type
         // (the `null` case), the value is processed as timestamp type with millisecond precision.
+        case null | _: TimestampMillis if rebaseDateTime => (updater, ordinal, value) =>
+          val millis = value.asInstanceOf[Long]
+          val micros = DateTimeUtils.fromMillis(millis)
+          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+          updater.setLong(ordinal, rebasedMicros)
         case null | _: TimestampMillis => (updater, ordinal, value) =>
           val millis = value.asInstanceOf[Long]
           val micros = DateTimeUtils.fromMillis(millis)
-          if (rebaseDateTime) {
-            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
-          } else {
-            updater.setLong(ordinal, micros)
-          }
+          updater.setLong(ordinal, micros)
+        case _: TimestampMicros if rebaseDateTime => (updater, ordinal, value) =>
+          val micros = value.asInstanceOf[Long]
+          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+          updater.setLong(ordinal, rebasedMicros)
         case _: TimestampMicros => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
-          if (rebaseDateTime) {
-            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
-          } else {
-            updater.setLong(ordinal, micros)
-          }
+          updater.setLong(ordinal, micros)
         case other => throw new IncompatibleSchemaException(
           s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
       }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index af9e3a5f81f82..68df7c0c340bc 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -149,17 +149,15 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
       case (TimestampType, LONG) => avroType.getLogicalType match {
           // For backward compatibility, if the Avro type is Long and it is not logical type
           // (the `null` case), output the timestamp value as with millisecond precision.
-          case null | _: TimestampMillis => (getter, ordinal) =>
-            val micros = getter.getLong(ordinal)
-            val rebasedMicros = if (rebaseDateTime) {
-              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
-            } else micros
-            DateTimeUtils.toMillis(rebasedMicros)
-          case _: TimestampMicros => (getter, ordinal) =>
+          case null | _: TimestampMillis if rebaseDateTime => (getter, ordinal) =>
             val micros = getter.getLong(ordinal)
-            if (rebaseDateTime) {
-              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
-            } else micros
+            val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(micros)
+            DateTimeUtils.fromMillis(rebasedMicros)
+          case null | _: TimestampMillis => (getter, ordinal) =>
+            DateTimeUtils.fromMillis(getter.getLong(ordinal))
+          case _: TimestampMicros if rebaseDateTime => (getter, ordinal) =>
+            DateTimeUtils.rebaseGregorianToJulianMicros(getter.getLong(ordinal))
+          case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
         }
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 9e89b69c0b33c..82569653c1f23 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.avro
 
 import java.io.File
-import java.sql.{Date, Timestamp}
+import java.sql.Timestamp
 
 import org.apache.avro.{LogicalTypes, Schema}
 import org.apache.avro.Conversions.DecimalConversion
@@ -25,7 +25,7 @@ import org.apache.avro.file.DataFileWriter
 import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord}
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
+import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -348,100 +348,6 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
       assert(msg.contains("Unscaled value too large for precision"))
     }
   }
-
-  private def readResourceAvroFile(name: String): DataFrame = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.format("avro").load(url.toString)
-  }
-
-  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
-    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-      checkAnswer(
-        readResourceAvroFile("before_1582_date_v2_4.avro"),
-        Row(java.sql.Date.valueOf("1001-01-01")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
-    }
-  }
-
-  test("SPARK-31183: rebasing microseconds timestamps in write") {
-    val tsStr = "1001-01-01 01:02:03.123456"
-    val nonRebased = "1001-01-07 01:09:05.123456"
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-        Seq(tsStr).toDF("tsS")
-          .select($"tsS".cast("timestamp").as("ts"))
-          .write.format("avro")
-          .save(path)
-
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
-      }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
-      }
-    }
-  }
-
-  test("SPARK-31183: rebasing milliseconds timestamps in write") {
-    val tsStr = "1001-01-01 01:02:03.123456"
-    val rebased = "1001-01-01 01:02:03.123"
-    val nonRebased = "1001-01-07 01:09:05.123"
-    Seq(
-      """{"type": "long","logicalType": "timestamp-millis"}""",
-      """"long"""").foreach { tsType =>
-      val timestampSchema = s"""
-          |{
-          |  "namespace": "logical",
-          |  "type": "record",
-          |  "name": "test",
-          |  "fields": [
-          |    {"name": "ts", "type": $tsType}
-          |  ]
-          |}""".stripMargin
-      withTempPath { dir =>
-        val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-          Seq(tsStr).toDF("tsS")
-            .select($"tsS".cast("timestamp").as("ts"))
-            .write
-            .option("avroSchema", timestampSchema)
-            .format("avro")
-            .save(path)
-
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(rebased)))
-        }
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(nonRebased)))
-        }
-      }
-    }
-  }
-
-  test("SPARK-31183: rebasing dates in write") {
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-        Seq("1001-01-01").toDF("dateS")
-          .select($"dateS".cast("date").as("date"))
-          .write.format("avro")
-          .save(path)
-
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
-      }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
-      }
-    }
-  }
 }
 
 class AvroV1LogicalTypeSuite extends AvroLogicalTypeSuite {
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 360160c9c9398..34a0e2bf607a9 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -21,7 +21,7 @@ import java.io._
 import java.net.URL
 import java.nio.file.{Files, Paths}
 import java.sql.{Date, Timestamp}
-import java.util.{Locale, TimeZone, UUID}
+import java.util.{Locale, UUID}
 
 import scala.collection.JavaConverters._
 
@@ -35,9 +35,10 @@ import org.apache.commons.io.FileUtils
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql._
-import org.apache.spark.sql.TestingUDT.{IntervalData, NullData, NullUDT}
+import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -83,6 +84,11 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }, new GenericDatumReader[Any]()).getSchema.toString(false)
   }
 
+  private def readResourceAvroFile(name: String): DataFrame = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(name)
+    spark.read.format("avro").load(url.toString)
+  }
+
   test("resolve avro data source") {
     val databricksAvro = "com.databricks.spark.avro"
     // By default the backward compatibility for com.databricks.spark.avro is enabled.
@@ -402,18 +408,19 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         StructField("float", FloatType, true),
         StructField("date", DateType, true)
       ))
-      TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
-      val rdd = spark.sparkContext.parallelize(Seq(
-        Row(1f, null),
-        Row(2f, new Date(1451948400000L)),
-        Row(3f, new Date(1460066400500L))
-      ))
-      val df = spark.createDataFrame(rdd, schema)
-      df.write.format("avro").save(dir.toString)
-      assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
-      checkAnswer(
-        spark.read.format("avro").load(dir.toString).select("date"),
-        Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
+      DateTimeTestUtils.withDefaultTimeZone(DateTimeUtils.TimeZoneUTC) {
+        val rdd = spark.sparkContext.parallelize(Seq(
+          Row(1f, null),
+          Row(2f, new Date(1451948400000L)),
+          Row(3f, new Date(1460066400500L))
+        ))
+        val df = spark.createDataFrame(rdd, schema)
+        df.write.format("avro").save(dir.toString)
+        assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
+        checkAnswer(
+          spark.read.format("avro").load(dir.toString).select("date"),
+          Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
+      }
     }
   }
 
@@ -1521,6 +1528,95 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       assert(deprecatedEvents.size === 1)
     }
   }
+
+  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+      checkAnswer(
+        readResourceAvroFile("before_1582_date_v2_4.avro"),
+        Row(java.sql.Date.valueOf("1001-01-01")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
+    }
+  }
+
+  test("SPARK-31183: rebasing microseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val nonRebased = "1001-01-07 01:09:05.123456"
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq(tsStr).toDF("tsS")
+          .select($"tsS".cast("timestamp").as("ts"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing milliseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val rebased = "1001-01-01 01:02:03.123"
+    val nonRebased = "1001-01-07 01:09:05.123"
+    Seq(
+      """{"type": "long","logicalType": "timestamp-millis"}""",
+      """"long"""").foreach { tsType =>
+      val timestampSchema = s"""
+        |{
+        |  "namespace": "logical",
+        |  "type": "record",
+        |  "name": "test",
+        |  "fields": [
+        |    {"name": "ts", "type": $tsType}
+        |  ]
+        |}""".stripMargin
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+          Seq(tsStr).toDF("tsS")
+            .select($"tsS".cast("timestamp").as("ts"))
+            .write
+            .option("avroSchema", timestampSchema)
+            .format("avro")
+            .save(path)
+
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(rebased)))
+        }
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(nonRebased)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq("1001-01-01").toDF("dateS")
+          .select($"dateS".cast("date").as("date"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
+      }
+    }
+  }
 }
 
 class AvroV1Suite extends AvroSuite {

From 1421af82a429e324e40caf18c70daf687c5ad674 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 20 Mar 2020 21:59:26 +0800
Subject: [PATCH 0316/1280] [SPARK-31189][SQL][DOCS] Fix errors and missing
 parts for datetime pattern document

### What changes were proposed in this pull request?

Fix errors and missing parts for datetime pattern document
1. The pattern we use is similar to DateTimeFormatter and SimpleDateFormat but not identical. So we shouldn't use any of them in the API docs but use a link to the doc of our own.
2. Some pattern letters are missing
3. Some pattern letters are explicitly banned - Set('A', 'c', 'e', 'n', 'N')
4. the second fraction pattern different logic for parsing and formatting

### Why are the changes needed?

fix and improve doc
### Does this PR introduce any user-facing change?

yes, new and updated doc
### How was this patch tested?

pass Jenkins
viewed locally with `jekyll serve`
![image](https://user-images.githubusercontent.com/8326978/77044447-6bd3bb00-69fa-11ea-8d6f-7084166c5dea.png)

Closes #27956 from yaooqinn/SPARK-31189.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 88ae6c44816bf6882ec3cc69dc12713a1ed6f89b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              | 48 ++++++++++++++++---
 python/pyspark/sql/functions.py               | 13 +++--
 python/pyspark/sql/readwriter.py              | 37 +++++++-------
 python/pyspark/sql/streaming.py               | 17 +++----
 .../expressions/datetimeExpressions.scala     | 24 +++++++---
 .../util/DateTimeFormatterHelper.scala        |  4 +-
 .../util/DateTimeFormatterHelperSuite.scala   | 10 ++--
 .../apache/spark/sql/DataFrameReader.scala    | 16 +++++--
 .../apache/spark/sql/DataFrameWriter.scala    | 16 +++++--
 .../org/apache/spark/sql/functions.scala      | 20 ++++++--
 .../sql/streaming/DataStreamReader.scala      | 16 +++++--
 11 files changed, 155 insertions(+), 66 deletions(-)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index f5c20eacae654..80585a4fd484c 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -21,11 +21,13 @@ license: |
 
 There are several common scenarios for datetime usage in Spark:
 
-- CSV/JSON datasources use the pattern string for parsing and formatting time content.
+- CSV/JSON datasources use the pattern string for parsing and formatting datetime content.
 
-- Datetime functions related to convert string to/from `DateType` or `TimestampType`. For example, unix_timestamp, date_format, to_unix_timestamp, from_unixtime, to_date, to_timestamp, from_utc_timestamp, to_utc_timestamp, etc.
+- Datetime functions related to convert `StringType` to/from `DateType` or `TimestampType`.
+  For example, `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp`, `from_utc_timestamp`, `to_utc_timestamp`, etc.
+
+Spark uses pattern letters in the following table for date and timestamp parsing and formatting:
 
-Spark uses the below letters in date and timestamp parsing and formatting:
 <table class="table">
 <tr>
   <th> <b>Symbol</b> </th>
@@ -52,7 +54,7 @@ Spark uses the below letters in date and timestamp parsing and formatting:
   <td> 189 </td>
 </tr>
 <tr>
-  <td> <b>M</b> </td>
+  <td> <b>M/L</b> </td>
   <td> month-of-year </td>
   <td> number/text </td>
   <td> 7; 07; Jul; July; J </td>
@@ -63,6 +65,12 @@ Spark uses the below letters in date and timestamp parsing and formatting:
   <td> number </td>
   <td> 28 </td>
 </tr>
+<tr>
+  <td> <b>Q/q</b> </td>
+  <td> quarter-of-year </td>
+  <td> number/text </td>
+  <td> 3; 03; Q3; 3rd quarter </td>
+</tr>
 <tr>
   <td> <b>Y</b> </td>
   <td> week-based-year </td>
@@ -147,6 +155,12 @@ Spark uses the below letters in date and timestamp parsing and formatting:
   <td> fraction </td>
   <td> 978 </td>
 </tr>
+<tr>
+  <td> <b>V</b> </td>
+  <td> time-zone ID </td>
+  <td> zone-id </td>
+  <td> America/Los_Angeles; Z; -08:30 </td>
+</tr>
 <tr>
   <td> <b>z</b> </td>
   <td> time-zone name </td>
@@ -189,6 +203,18 @@ Spark uses the below letters in date and timestamp parsing and formatting:
   <td> literal </td>
   <td> ' </td>
 </tr>
+<tr>
+  <td> <b>[</b> </td>
+  <td> optional section start </td>
+  <td>  </td>
+  <td>  </td>
+</tr>
+<tr>
+  <td> <b>]</b> </td>
+  <td> optional section end </td>
+  <td>  </td>
+  <td>  </td>
+</tr>
 </table>
 
 The count of pattern letters determines the format.
@@ -199,11 +225,16 @@ The count of pattern letters determines the format.
 
 - Number/Text: If the count of pattern letters is 3 or greater, use the Text rules above. Otherwise use the Number rules above.
 
-- Fraction: Outputs the micro-of-second field as a fraction-of-second. The micro-of-second value has six digits, thus the count of pattern letters is from 1 to 6. If it is less than 6, then the micro-of-second value is truncated, with only the most significant digits being output.
+- Fraction: Use one or more (up to 9) contiguous `'S'` characters, e,g `SSSSSS`, to parse and format fraction of second.
+  For parsing, the acceptable fraction length can be [1, the number of contiguous 'S'].
+  For formatting, the fraction length would be padded to the number of contiguous 'S' with zeros.
+  Spark supports datetime of micro-of-second precision, which has up to 6 significant digits, but can parse nano-of-second with exceeded part truncated.
 
 - Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
 
-- Zone names: This outputs the display name of the time-zone ID. If the count of letters is one, two or three, then the short name is output. If the count of letters is four, then the full name is output. Five or more letters will fail.
+- Zone ID(V): This outputs the display the time-zone ID. Pattern letter count must be 2.
+
+- Zone names(z): This outputs the display textual name of the time-zone ID. If the count of letters is one, two or three, then the short name is output. If the count of letters is four, then the full name is output. Five or more letters will fail.
 
 - Offset X and x: This formats the offset based on the number of pattern letters. One letter outputs just the hour, such as '+01', unless the minute is non-zero in which case the minute is also output, such as '+0130'. Two letters outputs the hour and minute, without a colon, such as '+0130'. Three letters outputs the hour and minute, with a colon, such as '+01:30'. Four letters outputs the hour and minute and optional second, without a colon, such as '+013015'. Five letters outputs the hour and minute and optional second, with a colon, such as '+01:30:15'. Six or more letters will fail. Pattern letter 'X' (upper case) will output 'Z' when the offset to be output would be zero, whereas pattern letter 'x' (lower case) will output '+00', '+0000', or '+00:00'.
 
@@ -211,6 +242,11 @@ The count of pattern letters determines the format.
 
 - Offset Z: This formats the offset based on the number of pattern letters. One, two or three letters outputs the hour and minute, without a colon, such as '+0130'. The output will be '+0000' when the offset is zero. Four letters outputs the full form of localized offset, equivalent to four letters of Offset-O. The output will be the corresponding localized offset text if the offset is zero. Five letters outputs the hour, minute, with optional second if non-zero, with colon. It outputs 'Z' if the offset is zero. Six or more letters will fail.
 
+- Optional section start and end: Use `[]` to define an optional section and maybe nested.
+  During formatting, all valid data will be output even it is in the optional section.
+  During parsing, the whole section may be missing from the parsed string.
+  An optional section is started by `[` and ended using `]` (or at the end of the pattern).
+
 More details for the text style:
 
 - Short Form: Short text, typically an abbreviation. For example, day-of-week Monday might output "Mon".
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index c23cc2bb2d4e4..bbadb54dc0800 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -920,8 +920,9 @@ def date_format(date, format):
     format given by the second argument.
 
     A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All
-    pattern letters of the Java class `java.time.format.DateTimeFormatter` can be used.
+    pattern letters of `datetime pattern`_. can be used.
 
+    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
     .. note:: Use when ever possible specialized functions like `year`. These benefit from a
         specialized implementation.
 
@@ -1138,11 +1139,12 @@ def months_between(date1, date2, roundOff=True):
 @since(2.2)
 def to_date(col, format=None):
     """Converts a :class:`Column` into :class:`pyspark.sql.types.DateType`
-    using the optionally specified format. Specify formats according to
-    `DateTimeFormatter <https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html>`_. # noqa
+    using the optionally specified format. Specify formats according to `datetime pattern`_.
     By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format
     is omitted. Equivalent to ``col.cast("date")``.
 
+    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
     >>> df.select(to_date(df.t).alias('date')).collect()
     [Row(date=datetime.date(1997, 2, 28))]
@@ -1162,11 +1164,12 @@ def to_date(col, format=None):
 @since(2.2)
 def to_timestamp(col, format=None):
     """Converts a :class:`Column` into :class:`pyspark.sql.types.TimestampType`
-    using the optionally specified format. Specify formats according to
-    `DateTimeFormatter <https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html>`_. # noqa
+    using the optionally specified format. Specify formats according to `datetime pattern`_.
     By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format
     is omitted. Equivalent to ``col.cast("timestamp")``.
 
+    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
     >>> df.select(to_timestamp(df.t).alias('dt')).collect()
     [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 5904688e087a7..e7ecb3ba9fc76 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -221,12 +221,11 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
                                           it uses the value specified in
                                           ``spark.sql.columnNameOfCorruptRecord``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
                            default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
                                 default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
@@ -255,6 +254,7 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
                                     disables `partition discovery`_.
 
         .. _partition discovery: /sql-data-sources-parquet.html#partition-discovery
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
         >>> df1 = spark.read.json('python/test_support/sql/people.json')
         >>> df1.dtypes
@@ -430,12 +430,11 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param negativeInf: sets the string representation of a negative infinity value. If None
                             is set, it uses the default value, ``Inf``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
                            default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
                                 default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
@@ -491,6 +490,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
         >>> df = spark.read.csv('python/test_support/sql/ages.csv')
         >>> df.dtypes
         [('_c0', 'string'), ('_c1', 'string')]
@@ -850,12 +851,11 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
                             known case-insensitive shorten names (none, bzip2, gzip, lz4,
                             snappy and deflate).
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
                            default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
                                 default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param encoding: specifies encoding (charset) of saved json files. If None is set,
@@ -865,6 +865,8 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
         :param ignoreNullFields: Whether to ignore null fields when generating JSON objects.
                         If None is set, it uses the default value, ``true``.
 
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
         >>> df.write.json(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
@@ -954,13 +956,12 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
                        the default value, ``false``.
         :param nullValue: sets the string representation of a null value. If None is set, it uses
                           the default value, empty string.
-        :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
+        :param dateFormat: sets the string that indicates a date format. Custom date formats follow
+                           the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
                            default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
                                 default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
@@ -980,6 +981,8 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
         :param lineSep: defines the line separator that should be used for writing. If None is
                         set, it uses the default value, ``\\n``. Maximum length is 1 character.
 
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
         >>> df.write.csv(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 6a7624f2fbf41..a83167882a8d7 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -459,12 +459,11 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
                                           it uses the value specified in
                                           ``spark.sql.columnNameOfCorruptRecord``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
                            default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
                                 default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
@@ -491,6 +490,7 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
                                     disables `partition discovery`_.
 
         .. _partition discovery: /sql-data-sources-parquet.html#partition-discovery
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
         >>> json_sdf = spark.readStream.json(tempfile.mkdtemp(), schema = sdf_schema)
         >>> json_sdf.isStreaming
@@ -671,12 +671,11 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param negativeInf: sets the string representation of a negative infinity value. If None
                             is set, it uses the default value, ``Inf``.
         :param dateFormat: sets the string that indicates a date format. Custom date formats
-                           follow the formats at ``java.time.format.DateTimeFormatter``. This
-                           applies to date type. If None is set, it uses the
+                           follow the formats at `datetime pattern`_.
+                           This applies to date type. If None is set, it uses the
                            default value, ``yyyy-MM-dd``.
         :param timestampFormat: sets the string that indicates a timestamp format.
-                                Custom date formats follow the formats at
-                                ``java.time.format.DateTimeFormatter``.
+                                Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
                                 default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
@@ -726,6 +725,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
+        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
         >>> csv_sdf = spark.readStream.csv(tempfile.mkdtemp(), schema = sdf_schema)
         >>> csv_sdf.isStreaming
         True
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index d71db0d7a791b..7ca6ab14ae8c5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -601,7 +601,7 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
   arguments = """
     Arguments:
       * timestamp - A date/timestamp or string to be converted to the given format.
-      * fmt - Date/time format pattern to follow. See `java.time.format.DateTimeFormatter` for valid date
+      * fmt - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a> for valid date
               and time format patterns.
   """,
   examples = """
@@ -676,13 +676,14 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
  * Converts time string with given pattern.
  * Deterministic version of [[UnixTimestamp]], must have at least one parameter.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(timeExp[, format]) - Returns the UNIX timestamp of the given time.",
   arguments = """
     Arguments:
       * timeExp - A date/timestamp or string which is returned as a UNIX timestamp.
       * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "yyyy-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
+                 Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
                  for valid date and time format patterns.
   """,
   examples = """
@@ -691,6 +692,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
        1460098800
   """,
   since = "1.6.0")
+// scalastyle:on line.size.limit
 case class ToUnixTimestamp(
     timeExp: Expression,
     format: Expression,
@@ -712,9 +714,10 @@ case class ToUnixTimestamp(
   override def prettyName: String = "to_unix_timestamp"
 }
 
+// scalastyle:off line.size.limit
 /**
  * Converts time string with given pattern to Unix time stamp (in seconds), returns null if fail.
- * See [https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html].
+ * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>.
  * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null.
  * If the second parameter is missing, use "yyyy-MM-dd HH:mm:ss".
  * If no parameters provided, the first parameter will be current_timestamp.
@@ -727,7 +730,7 @@ case class ToUnixTimestamp(
     Arguments:
       * timeExp - A date/timestamp or string. If not provided, this defaults to current time.
       * format - Date/time format pattern to follow. Ignored if `timeExp` is not a string.
-                 Default value is "yyyy-MM-dd HH:mm:ss". See `java.time.format.DateTimeFormatter`
+                 Default value is "yyyy-MM-dd HH:mm:ss". See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html"> Datetime Patterns</a>
                  for valid date and time format patterns.
   """,
   examples = """
@@ -738,6 +741,7 @@ case class ToUnixTimestamp(
        1460041200
   """,
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Option[String] = None)
   extends UnixTime {
 
@@ -915,12 +919,13 @@ abstract class UnixTime extends ToTimestamp {
  * format. If the format is missing, using format like "1970-01-01 00:00:00".
  * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(unix_time, format) - Returns `unix_time` in the specified `format`.",
   arguments = """
     Arguments:
       * unix_time - UNIX Timestamp to be converted to the provided format.
-      * format - Date/time format pattern to follow. See `java.time.format.DateTimeFormatter`
+      * format - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
                  for valid date and time format patterns.
   """,
   examples = """
@@ -929,6 +934,7 @@ abstract class UnixTime extends ToTimestamp {
        1969-12-31 16:00:00
   """,
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[String] = None)
   extends BinaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
 
@@ -1449,6 +1455,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
 /**
  * Parses a column to a date based on the given format.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
     _FUNC_(date_str[, fmt]) - Parses the `date_str` expression with the `fmt` expression to
@@ -1458,7 +1465,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
   arguments = """
     Arguments:
       * date_str - A string to be parsed to date.
-      * fmt - Date format pattern to follow. See `java.time.format.DateTimeFormatter` for valid
+      * fmt - Date format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a> for valid
               date and time format patterns.
   """,
   examples = """
@@ -1469,6 +1476,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
        2016-12-31
   """,
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class ParseToDate(left: Expression, format: Option[Expression], child: Expression)
   extends RuntimeReplaceable {
 
@@ -1497,6 +1505,7 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
 /**
  * Parses a column to a timestamp based on the supplied format.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
     _FUNC_(timestamp_str[, fmt]) - Parses the `timestamp_str` expression with the `fmt` expression
@@ -1506,7 +1515,7 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
   arguments = """
     Arguments:
       * timestamp_str - A string to be parsed to timestamp.
-      * fmt - Timestamp format pattern to follow. See `java.time.format.DateTimeFormatter` for valid
+      * fmt - Timestamp format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a> for valid
               date and time format patterns.
   """,
   examples = """
@@ -1517,6 +1526,7 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
        2016-12-31 00:00:00
   """,
   since = "2.2.0")
+// scalastyle:on line.size.limit
 case class ParseToTimestamp(left: Expression, format: Option[Expression], child: Expression)
   extends RuntimeReplaceable {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 4ed618ec454b0..05ec23f7ad479 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -162,6 +162,8 @@ private object DateTimeFormatterHelper {
     toFormatter(builder, TimestampFormatter.defaultLocale)
   }
 
+  final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
+
   /**
    * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
    * parsing/formatting datetime values. The pattern string is incompatible with the one defined
@@ -179,7 +181,7 @@ private object DateTimeFormatterHelper {
     (pattern + " ").split("'").zipWithIndex.map {
       case (patternPart, index) =>
         if (index % 2 == 0) {
-          for (c <- patternPart if c == 'c' || c == 'e') {
+          for (c <- patternPart if unsupportedLetters.contains(c)) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
           }
           // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index 811c4daa1b0e4..817e503584324 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.util
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper.convertIncompatiblePattern
+import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
 
 class DateTimeFormatterHelperSuite extends SparkFunSuite {
 
@@ -36,10 +36,10 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
       === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
       === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
-    val e1 = intercept[IllegalArgumentException](convertIncompatiblePattern("yyyy-MM-dd eeee G"))
-    assert(e1.getMessage === "Illegal pattern character: e")
-    val e2 = intercept[IllegalArgumentException](convertIncompatiblePattern("yyyy-MM-dd cccc G"))
-    assert(e2.getMessage === "Illegal pattern character: c")
+    unsupportedLetters.foreach { l =>
+      val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
+      assert(e.getMessage === s"Illegal pattern character: $l")
+    }
     assert(convertIncompatiblePattern("yyyy-MM-dd uuuu") === "uuuu-MM-dd eeee")
     assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
     assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 9416126780506..a2a3518f2509c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -391,11 +391,15 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
    * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.</li>
    * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
    * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`multiLine` (default `false`): parse one record, which may span multiple lines,
    * per file</li>
    * <li>`encoding` (by default it is not set): allows to forcibly set one of standard basic
@@ -616,11 +620,15 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <li>`negativeInf` (default `-Inf`): sets the string representation of a negative infinity
    * value.</li>
    * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
    * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
    * a record can have.</li>
    * <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 6946c1f5bc895..11feae9170d88 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -750,11 +750,15 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
    * `snappy` and `deflate`). </li>
    * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
    * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`encoding` (by default it is not set): specifies encoding (charset) of saved json
    * files. If it is not set, the UTF-8 charset will be used. </li>
    * <li>`lineSep` (default `\n`): defines the line separator that should be used for writing.</li>
@@ -871,11 +875,15 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`,
    * `snappy` and `deflate`). </li>
    * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
    * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`ignoreLeadingWhiteSpace` (default `true`): a flag indicating whether or not leading
    * whitespaces from values being written should be skipped.</li>
    * <li>`ignoreTrailingWhiteSpace` (default `true`): a flag indicating defines whether or not
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 69383d44522e0..11c04e5212a3d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2632,7 +2632,9 @@ object functions {
    * Converts a date/timestamp/string to a value of string in the format specified by the date
    * format given by the second argument.
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param dateExpr A date, timestamp or string. If a string, the data must be in a format that
    *                 can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
@@ -2890,7 +2892,9 @@ object functions {
    * representing the timestamp of that moment in the current system time zone in the given
    * format.
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param ut A number of a type that is castable to a long, such as string or integer. Can be
    *           negative for timestamps before the unix epoch
@@ -2934,7 +2938,9 @@ object functions {
   /**
    * Converts time string with given pattern to Unix timestamp (in seconds).
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param s A date, timestamp or string. If a string, the data must be in a format that can be
    *          cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
@@ -2962,7 +2968,9 @@ object functions {
   /**
    * Converts time string with the given pattern to timestamp.
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param s   A date, timestamp or string. If a string, the data must be in a format that can be
    *            cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
@@ -2987,7 +2995,9 @@ object functions {
   /**
    * Converts the column into a `DateType` with a specified format
    *
-   * See [[java.time.format.DateTimeFormatter]] for valid date and time format patterns
+   * See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>
+   * for valid date and time format patterns
    *
    * @param e   A date, timestamp or string. If a string, the data must be in a format that can be
    *            cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 6848be15d42a5..be7f021cb2465 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -253,11 +253,15 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string
    * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.</li>
    * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
    * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`multiLine` (default `false`): parse one record, which may span multiple lines,
    * per file</li>
    * <li>`lineSep` (default covers all `\r`, `\r\n` and `\n`): defines the line separator
@@ -319,11 +323,15 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * <li>`negativeInf` (default `-Inf`): sets the string representation of a negative infinity
    * value.</li>
    * <li>`dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format.
-   * Custom date formats follow the formats at `java.time.format.DateTimeFormatter`.
+   * Custom date formats follow the formats at
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
    * This applies to date type.</li>
    * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
-   * `java.time.format.DateTimeFormatter`. This applies to timestamp type.</li>
+   * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
+   *   Datetime Patterns</a>.
+   * This applies to timestamp type.</li>
    * <li>`maxColumns` (default `20480`): defines a hard limit of how many columns
    * a record can have.</li>
    * <li>`maxCharsPerColumn` (default `-1`): defines the maximum number of characters allowed

From 7227a9404dce18e48368114c66c2e1bc4e4abfc1 Mon Sep 17 00:00:00 2001
From: yan ma <yan.ma@intel.com>
Date: Fri, 20 Mar 2020 10:32:58 -0500
Subject: [PATCH 0317/1280] [SPARK-30773][ML] Support NativeBlas for level-1
 routines

### What changes were proposed in this pull request?
Change BLAS for part of level-1 routines(axpy, dot, scal(double, denseVector)) from java implementation to NativeBLAS when vector size>256

### Why are the changes needed?
In current ML BLAS.scala, all level-1 routines are fixed to use java
implementation. But NativeBLAS(intel MKL, OpenBLAS) can bring up to 11X
performance improvement based on performance test which apply direct
calls against these methods. We should provide a way to allow user take
advantage of NativeBLAS for level-1 routines. Here we do it through
switching to NativeBLAS for these methods from f2jBLAS.

### Does this PR introduce any user-facing change?
 Yes, methods axpy, dot, scal in level-1 routines will switch to NativeBLAS when it has more than nativeL1Threshold(fixed value 256) elements and will fallback to f2jBLAS if native BLAS is not properly configured in system.

### How was this patch tested?
Perf test direct calls level-1 routines

Closes #27546 from yma11/SPARK-30773.

Lead-authored-by: yan ma <yan.ma@intel.com>
Co-authored-by: Ma Yan <yan.ma@intel.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit fae981e5f32e0ddb86591a616829423dfafb4ed0)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/ml-guide.md                              |  2 +-
 .../org/apache/spark/ml/linalg/BLAS.scala     | 27 ++++++++++++-------
 .../apache/spark/ml/linalg/BLASSuite.scala    |  6 +++++
 .../org/apache/spark/mllib/linalg/BLAS.scala  | 27 ++++++++++++-------
 .../apache/spark/mllib/linalg/BLASSuite.scala |  6 +++++
 5 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/docs/ml-guide.md b/docs/ml-guide.md
index 2037285f3f12c..5ce6b4f58c2f7 100644
--- a/docs/ml-guide.md
+++ b/docs/ml-guide.md
@@ -78,7 +78,7 @@ The most popular native BLAS such as [Intel MKL](https://software.intel.com/en-u
 
 Configuring these BLAS implementations to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of cores each Spark task is configured to use, which is 1 by default and typically left at 1.
 
-Please refer to resources like the following to understand how to configure the number of threads these BLAS implementations use: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded).
+Please refer to resources like the following to understand how to configure the number of threads these BLAS implementations use: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) or [Intel oneMKL](https://software.intel.com/en-us/onemkl-linux-developer-guide-improving-performance-with-threading) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded). Note that if nativeBLAS is not properly configured in system, java implementation(f2jBLAS) will be used as fallback option.
 
 To use MLlib in Python, you will need [NumPy](http://www.numpy.org) version 1.4 or newer.
 
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
index e054a15fc9b75..fcd33a63bd0a2 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/BLAS.scala
@@ -27,8 +27,9 @@ private[spark] object BLAS extends Serializable {
 
   @transient private var _f2jBLAS: NetlibBLAS = _
   @transient private var _nativeBLAS: NetlibBLAS = _
+  private val nativeL1Threshold: Int = 256
 
-  // For level-1 routines, we use Java implementation.
+  // For level-1 function dspmv, use f2jBLAS for better performance.
   private[ml] def f2jBLAS: NetlibBLAS = {
     if (_f2jBLAS == null) {
       _f2jBLAS = new F2jBLAS
@@ -36,6 +37,14 @@ private[spark] object BLAS extends Serializable {
     _f2jBLAS
   }
 
+  private[ml] def getBLAS(vectorSize: Int): NetlibBLAS = {
+    if (vectorSize < nativeL1Threshold) {
+      f2jBLAS
+    } else {
+      nativeBLAS
+    }
+  }
+
   /**
    * y += a * x
    */
@@ -63,7 +72,7 @@ private[spark] object BLAS extends Serializable {
    */
   private def axpy(a: Double, x: DenseVector, y: DenseVector): Unit = {
     val n = x.size
-    f2jBLAS.daxpy(n, a, x.values, 1, y.values, 1)
+    getBLAS(n).daxpy(n, a, x.values, 1, y.values, 1)
   }
 
   /**
@@ -94,7 +103,7 @@ private[spark] object BLAS extends Serializable {
   private[spark] def axpy(a: Double, X: DenseMatrix, Y: DenseMatrix): Unit = {
     require(X.numRows == Y.numRows && X.numCols == Y.numCols, "Dimension mismatch: " +
       s"size(X) = ${(X.numRows, X.numCols)} but size(Y) = ${(Y.numRows, Y.numCols)}.")
-    f2jBLAS.daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
+    getBLAS(X.values.length).daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
   }
 
   /**
@@ -123,7 +132,7 @@ private[spark] object BLAS extends Serializable {
    */
   private def dot(x: DenseVector, y: DenseVector): Double = {
     val n = x.size
-    f2jBLAS.ddot(n, x.values, 1, y.values, 1)
+    getBLAS(n).ddot(n, x.values, 1, y.values, 1)
   }
 
   /**
@@ -218,16 +227,16 @@ private[spark] object BLAS extends Serializable {
   def scal(a: Double, x: Vector): Unit = {
     x match {
       case sx: SparseVector =>
-        f2jBLAS.dscal(sx.values.length, a, sx.values, 1)
+        getBLAS(sx.values.length).dscal(sx.values.length, a, sx.values, 1)
       case dx: DenseVector =>
-        f2jBLAS.dscal(dx.values.length, a, dx.values, 1)
+        getBLAS(dx.size).dscal(dx.values.length, a, dx.values, 1)
       case _ =>
         throw new IllegalArgumentException(s"scal doesn't support vector type ${x.getClass}.")
     }
   }
 
   // For level-3 routines, we use the native BLAS.
-  private def nativeBLAS: NetlibBLAS = {
+  private[ml] def nativeBLAS: NetlibBLAS = {
     if (_nativeBLAS == null) {
       _nativeBLAS = NativeBLAS
     }
@@ -374,7 +383,7 @@ private[spark] object BLAS extends Serializable {
       // gemm: alpha is equal to 0 and beta is equal to 1. Returning C.
       return
     } else if (alpha == 0.0) {
-      f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+      getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
     } else {
       A match {
         case sparse: SparseMatrix => gemm(alpha, sparse, B, beta, C)
@@ -480,7 +489,7 @@ private[spark] object BLAS extends Serializable {
     } else {
       // Scale matrix first if `beta` is not equal to 1.0
       if (beta != 1.0) {
-        f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+        getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
       }
       // Perform matrix multiplication and add to C. The rows of A are multiplied by the columns of
       // B, and added to C.
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
index 877ac68983348..781f3da313d82 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/BLASSuite.scala
@@ -23,6 +23,12 @@ import org.apache.spark.ml.util.TestingUtils._
 
 class BLASSuite extends SparkMLFunSuite {
 
+  test("nativeL1Threshold") {
+    assert(getBLAS(128) == BLAS.f2jBLAS)
+    assert(getBLAS(256) == BLAS.nativeBLAS)
+    assert(getBLAS(512) == BLAS.nativeBLAS)
+  }
+
   test("copy") {
     val sx = Vectors.sparse(4, Array(0, 2), Array(1.0, -2.0))
     val dx = Vectors.dense(1.0, 0.0, -2.0, 0.0)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
index 1f5558dc2a50e..3371b0565acee 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
@@ -29,8 +29,9 @@ private[spark] object BLAS extends Serializable with Logging {
 
   @transient private var _f2jBLAS: NetlibBLAS = _
   @transient private var _nativeBLAS: NetlibBLAS = _
+  private val nativeL1Threshold: Int = 256
 
-  // For level-1 routines, we use Java implementation.
+  // For level-1 function dspmv, use f2jBLAS for better performance.
   private[mllib] def f2jBLAS: NetlibBLAS = {
     if (_f2jBLAS == null) {
       _f2jBLAS = new F2jBLAS
@@ -38,6 +39,14 @@ private[spark] object BLAS extends Serializable with Logging {
     _f2jBLAS
   }
 
+  private[mllib] def getBLAS(vectorSize: Int): NetlibBLAS = {
+    if (vectorSize < nativeL1Threshold) {
+      f2jBLAS
+    } else {
+      nativeBLAS
+    }
+  }
+
   /**
    * y += a * x
    */
@@ -65,7 +74,7 @@ private[spark] object BLAS extends Serializable with Logging {
    */
   private def axpy(a: Double, x: DenseVector, y: DenseVector): Unit = {
     val n = x.size
-    f2jBLAS.daxpy(n, a, x.values, 1, y.values, 1)
+    getBLAS(n).daxpy(n, a, x.values, 1, y.values, 1)
   }
 
   /**
@@ -96,7 +105,7 @@ private[spark] object BLAS extends Serializable with Logging {
   private[spark] def axpy(a: Double, X: DenseMatrix, Y: DenseMatrix): Unit = {
     require(X.numRows == Y.numRows && X.numCols == Y.numCols, "Dimension mismatch: " +
       s"size(X) = ${(X.numRows, X.numCols)} but size(Y) = ${(Y.numRows, Y.numCols)}.")
-    f2jBLAS.daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
+    getBLAS(X.values.length).daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
   }
 
   /**
@@ -125,7 +134,7 @@ private[spark] object BLAS extends Serializable with Logging {
    */
   private def dot(x: DenseVector, y: DenseVector): Double = {
     val n = x.size
-    f2jBLAS.ddot(n, x.values, 1, y.values, 1)
+    getBLAS(n).ddot(n, x.values, 1, y.values, 1)
   }
 
   /**
@@ -220,16 +229,16 @@ private[spark] object BLAS extends Serializable with Logging {
   def scal(a: Double, x: Vector): Unit = {
     x match {
       case sx: SparseVector =>
-        f2jBLAS.dscal(sx.values.length, a, sx.values, 1)
+        getBLAS(sx.values.length).dscal(sx.values.length, a, sx.values, 1)
       case dx: DenseVector =>
-        f2jBLAS.dscal(dx.values.length, a, dx.values, 1)
+        getBLAS(dx.size).dscal(dx.values.length, a, dx.values, 1)
       case _ =>
         throw new IllegalArgumentException(s"scal doesn't support vector type ${x.getClass}.")
     }
   }
 
   // For level-3 routines, we use the native BLAS.
-  private def nativeBLAS: NetlibBLAS = {
+  private[mllib] def nativeBLAS: NetlibBLAS = {
     if (_nativeBLAS == null) {
       _nativeBLAS = NativeBLAS
     }
@@ -356,7 +365,7 @@ private[spark] object BLAS extends Serializable with Logging {
     if (alpha == 0.0 && beta == 1.0) {
       logDebug("gemm: alpha is equal to 0 and beta is equal to 1. Returning C.")
     } else if (alpha == 0.0) {
-      f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+      getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
     } else {
       A match {
         case sparse: SparseMatrix => gemm(alpha, sparse, B, beta, C)
@@ -462,7 +471,7 @@ private[spark] object BLAS extends Serializable with Logging {
     } else {
       // Scale matrix first if `beta` is not equal to 1.0
       if (beta != 1.0) {
-        f2jBLAS.dscal(C.values.length, beta, C.values, 1)
+        getBLAS(C.values.length).dscal(C.values.length, beta, C.values, 1)
       }
       // Perform matrix multiplication and add to C. The rows of A are multiplied by the columns of
       // B, and added to C.
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
index 6e68c1c9d36c8..12ab2ac3cc698 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
@@ -23,6 +23,12 @@ import org.apache.spark.mllib.util.TestingUtils._
 
 class BLASSuite extends SparkFunSuite {
 
+  test("nativeL1Threshold") {
+    assert(getBLAS(128) == BLAS.f2jBLAS)
+    assert(getBLAS(256) == BLAS.nativeBLAS)
+    assert(getBLAS(512) == BLAS.nativeBLAS)
+  }
+
   test("copy") {
     val sx = Vectors.sparse(4, Array(0, 2), Array(1.0, -2.0))
     val dx = Vectors.dense(1.0, 0.0, -2.0, 0.0)

From 4a5f6955ba9667298bf13e5cdaa7703d5389b083 Mon Sep 17 00:00:00 2001
From: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Date: Sat, 21 Mar 2020 18:59:29 -0700
Subject: [PATCH 0318/1280] [SPARK-30541][TESTS] Implement
 KafkaDelegationTokenSuite with testRetry

### What changes were proposed in this pull request?
`KafkaDelegationTokenSuite` has been ignored because showed flaky behaviour. In this PR I've changed the approach how the test executed and turning it on again. This PR contains the following:
* The test runs in separate JVM in order to avoid modified security context
* The body of the test runs in `testRetry` which reties if failed
* Additional logs to analyse possible failures
* Enhanced clean-up code

### Why are the changes needed?
`KafkaDelegationTokenSuite ` is ignored.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Executed the test in loop 1k+ times in jenkins (locally much harder to reproduce).

Closes #27877 from gaborgsomogyi/SPARK-30541.

Authored-by: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit bf342bafa81738a47d511d3aa02812a1ccc0ecab)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 external/kafka-0-10-sql/src/test/resources/log4j.properties   | 4 +++-
 .../apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala | 2 +-
 .../scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala  | 4 ++++
 project/SparkBuild.scala                                      | 3 ++-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/external/kafka-0-10-sql/src/test/resources/log4j.properties b/external/kafka-0-10-sql/src/test/resources/log4j.properties
index 75e3b53a093f6..daf05723d4e7b 100644
--- a/external/kafka-0-10-sql/src/test/resources/log4j.properties
+++ b/external/kafka-0-10-sql/src/test/resources/log4j.properties
@@ -25,4 +25,6 @@ log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{
 
 # Ignore messages below warning level from Jetty, because it's a bit verbose
 log4j.logger.org.spark-project.jetty=WARN
-
+log4j.logger.org.apache.spark.sql.kafka010.KafkaTestUtils=DEBUG
+log4j.logger.org.apache.directory.server.kerberos.kdc.authentication=DEBUG
+log4j.logger.org.apache.directory.server.core.DefaultDirectoryService=DEBUG
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
index 79239e57409e9..702bd4f6ebdad 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaDelegationTokenSuite.scala
@@ -62,7 +62,7 @@ class KafkaDelegationTokenSuite extends StreamTest with SharedSparkSession with
     }
   }
 
-  ignore("Roundtrip") {
+  testRetry("Roundtrip", 3) {
     val hadoopConf = new Configuration()
     val manager = new HadoopDelegationTokenManager(spark.sparkContext.conf, hadoopConf, null)
     val credentials = new Credentials()
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 7b972fede96e5..c1ca557061ba1 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -170,6 +170,7 @@ class KafkaTestUtils(
 
     kdc.getKrb5conf.delete()
     Files.write(krb5confStr, kdc.getKrb5conf, StandardCharsets.UTF_8)
+    logDebug(s"krb5.conf file content: $krb5confStr")
   }
 
   private def addedKrb5Config(key: String, value: String): String = {
@@ -309,6 +310,7 @@ class KafkaTestUtils(
     }
     brokerReady = false
     zkReady = false
+    kdcReady = false
 
     if (producer != null) {
       producer.close()
@@ -317,6 +319,7 @@ class KafkaTestUtils(
 
     if (adminClient != null) {
       adminClient.close()
+      adminClient = null
     }
 
     if (server != null) {
@@ -351,6 +354,7 @@ class KafkaTestUtils(
     Configuration.getConfiguration.refresh()
     if (kdc != null) {
       kdc.stop()
+      kdc = null
     }
     UserGroupInformation.reset()
     teardownKrbDebug()
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 1a2a7c3066d26..3889013995800 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -479,7 +479,8 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
-    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite"
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
+    "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
   private val DEFAULT_TEST_GROUP = "default_test_group"

From f0bfdc513a15884de8f3ffc79cc1845991082642 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 21 Mar 2020 20:52:04 -0700
Subject: [PATCH 0319/1280] Revert "[SPARK-31183][SQL][FOLLOWUP] Move rebase
 tests to `AvroSuite` and check the rebase flag out of function bodies"

This reverts commit a6f3e3b096e2d7a39e0b2fdec6452e6d633baf7e.

Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/avro/AvroDeserializer.scala     |  21 ++-
 .../spark/sql/avro/AvroSerializer.scala       |  18 +--
 .../spark/sql/avro/AvroLogicalTypeSuite.scala |  98 +++++++++++++-
 .../org/apache/spark/sql/avro/AvroSuite.scala | 124 ++----------------
 4 files changed, 130 insertions(+), 131 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 3e8a7f95ccd96..b98f3036686f3 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -106,22 +106,21 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
       case (LONG, TimestampType) => avroType.getLogicalType match {
         // For backward compatibility, if the Avro type is Long and it is not logical type
         // (the `null` case), the value is processed as timestamp type with millisecond precision.
-        case null | _: TimestampMillis if rebaseDateTime => (updater, ordinal, value) =>
-          val millis = value.asInstanceOf[Long]
-          val micros = DateTimeUtils.fromMillis(millis)
-          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
-          updater.setLong(ordinal, rebasedMicros)
         case null | _: TimestampMillis => (updater, ordinal, value) =>
           val millis = value.asInstanceOf[Long]
           val micros = DateTimeUtils.fromMillis(millis)
-          updater.setLong(ordinal, micros)
-        case _: TimestampMicros if rebaseDateTime => (updater, ordinal, value) =>
-          val micros = value.asInstanceOf[Long]
-          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
-          updater.setLong(ordinal, rebasedMicros)
+          if (rebaseDateTime) {
+            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
+          } else {
+            updater.setLong(ordinal, micros)
+          }
         case _: TimestampMicros => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
-          updater.setLong(ordinal, micros)
+          if (rebaseDateTime) {
+            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
+          } else {
+            updater.setLong(ordinal, micros)
+          }
         case other => throw new IncompatibleSchemaException(
           s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
       }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index 68df7c0c340bc..af9e3a5f81f82 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -149,15 +149,17 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
       case (TimestampType, LONG) => avroType.getLogicalType match {
           // For backward compatibility, if the Avro type is Long and it is not logical type
           // (the `null` case), output the timestamp value as with millisecond precision.
-          case null | _: TimestampMillis if rebaseDateTime => (getter, ordinal) =>
-            val micros = getter.getLong(ordinal)
-            val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(micros)
-            DateTimeUtils.fromMillis(rebasedMicros)
           case null | _: TimestampMillis => (getter, ordinal) =>
-            DateTimeUtils.fromMillis(getter.getLong(ordinal))
-          case _: TimestampMicros if rebaseDateTime => (getter, ordinal) =>
-            DateTimeUtils.rebaseGregorianToJulianMicros(getter.getLong(ordinal))
-          case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
+            val micros = getter.getLong(ordinal)
+            val rebasedMicros = if (rebaseDateTime) {
+              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
+            } else micros
+            DateTimeUtils.toMillis(rebasedMicros)
+          case _: TimestampMicros => (getter, ordinal) =>
+            val micros = getter.getLong(ordinal)
+            if (rebaseDateTime) {
+              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
+            } else micros
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
         }
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 82569653c1f23..9e89b69c0b33c 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.avro
 
 import java.io.File
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
 
 import org.apache.avro.{LogicalTypes, Schema}
 import org.apache.avro.Conversions.DecimalConversion
@@ -25,7 +25,7 @@ import org.apache.avro.file.DataFileWriter
 import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord}
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{DataFrame, QueryTest, Row}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -348,6 +348,100 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
       assert(msg.contains("Unscaled value too large for precision"))
     }
   }
+
+  private def readResourceAvroFile(name: String): DataFrame = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(name)
+    spark.read.format("avro").load(url.toString)
+  }
+
+  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+      checkAnswer(
+        readResourceAvroFile("before_1582_date_v2_4.avro"),
+        Row(java.sql.Date.valueOf("1001-01-01")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
+    }
+  }
+
+  test("SPARK-31183: rebasing microseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val nonRebased = "1001-01-07 01:09:05.123456"
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq(tsStr).toDF("tsS")
+          .select($"tsS".cast("timestamp").as("ts"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing milliseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val rebased = "1001-01-01 01:02:03.123"
+    val nonRebased = "1001-01-07 01:09:05.123"
+    Seq(
+      """{"type": "long","logicalType": "timestamp-millis"}""",
+      """"long"""").foreach { tsType =>
+      val timestampSchema = s"""
+          |{
+          |  "namespace": "logical",
+          |  "type": "record",
+          |  "name": "test",
+          |  "fields": [
+          |    {"name": "ts", "type": $tsType}
+          |  ]
+          |}""".stripMargin
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+          Seq(tsStr).toDF("tsS")
+            .select($"tsS".cast("timestamp").as("ts"))
+            .write
+            .option("avroSchema", timestampSchema)
+            .format("avro")
+            .save(path)
+
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(rebased)))
+        }
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(nonRebased)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq("1001-01-01").toDF("dateS")
+          .select($"dateS".cast("date").as("date"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
+      }
+    }
+  }
 }
 
 class AvroV1LogicalTypeSuite extends AvroLogicalTypeSuite {
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 34a0e2bf607a9..360160c9c9398 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -21,7 +21,7 @@ import java.io._
 import java.net.URL
 import java.nio.file.{Files, Paths}
 import java.sql.{Date, Timestamp}
-import java.util.{Locale, UUID}
+import java.util.{Locale, TimeZone, UUID}
 
 import scala.collection.JavaConverters._
 
@@ -35,10 +35,9 @@ import org.apache.commons.io.FileUtils
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql._
-import org.apache.spark.sql.TestingUDT.IntervalData
+import org.apache.spark.sql.TestingUDT.{IntervalData, NullData, NullUDT}
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -84,11 +83,6 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }, new GenericDatumReader[Any]()).getSchema.toString(false)
   }
 
-  private def readResourceAvroFile(name: String): DataFrame = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.format("avro").load(url.toString)
-  }
-
   test("resolve avro data source") {
     val databricksAvro = "com.databricks.spark.avro"
     // By default the backward compatibility for com.databricks.spark.avro is enabled.
@@ -408,19 +402,18 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         StructField("float", FloatType, true),
         StructField("date", DateType, true)
       ))
-      DateTimeTestUtils.withDefaultTimeZone(DateTimeUtils.TimeZoneUTC) {
-        val rdd = spark.sparkContext.parallelize(Seq(
-          Row(1f, null),
-          Row(2f, new Date(1451948400000L)),
-          Row(3f, new Date(1460066400500L))
-        ))
-        val df = spark.createDataFrame(rdd, schema)
-        df.write.format("avro").save(dir.toString)
-        assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
-        checkAnswer(
-          spark.read.format("avro").load(dir.toString).select("date"),
-          Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
-      }
+      TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
+      val rdd = spark.sparkContext.parallelize(Seq(
+        Row(1f, null),
+        Row(2f, new Date(1451948400000L)),
+        Row(3f, new Date(1460066400500L))
+      ))
+      val df = spark.createDataFrame(rdd, schema)
+      df.write.format("avro").save(dir.toString)
+      assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
+      checkAnswer(
+        spark.read.format("avro").load(dir.toString).select("date"),
+        Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
     }
   }
 
@@ -1528,95 +1521,6 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       assert(deprecatedEvents.size === 1)
     }
   }
-
-  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
-    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-      checkAnswer(
-        readResourceAvroFile("before_1582_date_v2_4.avro"),
-        Row(java.sql.Date.valueOf("1001-01-01")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
-    }
-  }
-
-  test("SPARK-31183: rebasing microseconds timestamps in write") {
-    val tsStr = "1001-01-01 01:02:03.123456"
-    val nonRebased = "1001-01-07 01:09:05.123456"
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-        Seq(tsStr).toDF("tsS")
-          .select($"tsS".cast("timestamp").as("ts"))
-          .write.format("avro")
-          .save(path)
-
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
-      }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
-      }
-    }
-  }
-
-  test("SPARK-31183: rebasing milliseconds timestamps in write") {
-    val tsStr = "1001-01-01 01:02:03.123456"
-    val rebased = "1001-01-01 01:02:03.123"
-    val nonRebased = "1001-01-07 01:09:05.123"
-    Seq(
-      """{"type": "long","logicalType": "timestamp-millis"}""",
-      """"long"""").foreach { tsType =>
-      val timestampSchema = s"""
-        |{
-        |  "namespace": "logical",
-        |  "type": "record",
-        |  "name": "test",
-        |  "fields": [
-        |    {"name": "ts", "type": $tsType}
-        |  ]
-        |}""".stripMargin
-      withTempPath { dir =>
-        val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-          Seq(tsStr).toDF("tsS")
-            .select($"tsS".cast("timestamp").as("ts"))
-            .write
-            .option("avroSchema", timestampSchema)
-            .format("avro")
-            .save(path)
-
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(rebased)))
-        }
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(nonRebased)))
-        }
-      }
-    }
-  }
-
-  test("SPARK-31183: rebasing dates in write") {
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-        Seq("1001-01-01").toDF("dateS")
-          .select($"dateS".cast("date").as("date"))
-          .write.format("avro")
-          .save(path)
-
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
-      }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
-      }
-    }
-  }
 }
 
 class AvroV1Suite extends AvroSuite {

From e36901573c30f90d7698082259873cb3f5ee93df Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sun, 22 Mar 2020 14:22:44 +0900
Subject: [PATCH 0320/1280] [SPARK-31189][R][DOCS][FOLLOWUP] Replace Datetime
 pattern links in R doc

### What changes were proposed in this pull request?

Use our own docs for data pattern instructions to replace java doc.

### Why are the changes needed?

fix doc

### Does this PR introduce any user-facing change?

yes. doc changed
### How was this patch tested?

pass jenkins

Closes #27975 from yaooqinn/SPARK-31189-2.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/functions.R | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index 89c7cbedc641a..dd5dbbc500e57 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -1792,7 +1792,7 @@ setMethod("radians",
 #' @details
 #' \code{to_date}: Converts the column into a DateType. You may optionally specify
 #' a format according to the rules in:
-#' \url{https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html}.
+#' \href{https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html}{Datetime Pattern}
 #' If the string cannot be parsed according to the specified format (or default),
 #' the value of the column will be null.
 #' By default, it follows casting rules to a DateType if the format is omitted
@@ -1888,7 +1888,7 @@ setMethod("to_csv", signature(x = "Column"),
 #' @details
 #' \code{to_timestamp}: Converts the column into a TimestampType. You may optionally specify
 #' a format according to the rules in:
-#' \url{https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html}.
+#' \href{https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html}{Datetime Pattern}
 #' If the string cannot be parsed according to the specified format (or default),
 #' the value of the column will be null.
 #' By default, it follows casting rules to a TimestampType if the format is omitted
@@ -2760,8 +2760,8 @@ setMethod("format_string", signature(format = "character", x = "Column"),
 #' \code{from_unixtime}: Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC)
 #' to a string representing the timestamp of that moment in the current system time zone in the JVM
 #' in the given format.
-#' See \href{https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html}{
-#' Customizing Formats} for available options.
+#' See \href{https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html}{
+#' Datetime Pattern} for available options.
 #'
 #' @rdname column_datetime_functions
 #'

From 4bc4af31534deccb0d12c5b5b84900f720c31381 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 23 Mar 2020 13:09:14 +0900
Subject: [PATCH 0321/1280] [SPARK-31183][SQL][FOLLOWUP][3.0] Move rebase tests
 to `AvroSuite` and check the rebase flag out of function bodies

### What changes were proposed in this pull request?
1. The tests added by #27953 are moved from `AvroLogicalTypeSuite` to `AvroSuite`.
2. Checking of the `rebaseDateTime` flag is moved out from functions bodies.

This is a backport of https://github.com/apache/spark/pull/27964

### Why are the changes needed?
1. The tests are moved because they are not directly related to logical types.
2. Checking the flag out of functions bodies should improve performance.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running Avro tests via the command `build/sbt avro/test`

Closes #27977 from MaxGekk/rebase-avro-datetime-followup-3.0.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/avro/AvroDeserializer.scala     |  21 +--
 .../spark/sql/avro/AvroSerializer.scala       |  16 +--
 .../spark/sql/avro/AvroLogicalTypeSuite.scala |  98 +-------------
 .../org/apache/spark/sql/avro/AvroSuite.scala | 124 ++++++++++++++++--
 4 files changed, 130 insertions(+), 129 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index b98f3036686f3..3e8a7f95ccd96 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -106,21 +106,22 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
       case (LONG, TimestampType) => avroType.getLogicalType match {
         // For backward compatibility, if the Avro type is Long and it is not logical type
         // (the `null` case), the value is processed as timestamp type with millisecond precision.
+        case null | _: TimestampMillis if rebaseDateTime => (updater, ordinal, value) =>
+          val millis = value.asInstanceOf[Long]
+          val micros = DateTimeUtils.fromMillis(millis)
+          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+          updater.setLong(ordinal, rebasedMicros)
         case null | _: TimestampMillis => (updater, ordinal, value) =>
           val millis = value.asInstanceOf[Long]
           val micros = DateTimeUtils.fromMillis(millis)
-          if (rebaseDateTime) {
-            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
-          } else {
-            updater.setLong(ordinal, micros)
-          }
+          updater.setLong(ordinal, micros)
+        case _: TimestampMicros if rebaseDateTime => (updater, ordinal, value) =>
+          val micros = value.asInstanceOf[Long]
+          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+          updater.setLong(ordinal, rebasedMicros)
         case _: TimestampMicros => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
-          if (rebaseDateTime) {
-            updater.setLong(ordinal, DateTimeUtils.rebaseJulianToGregorianMicros(micros))
-          } else {
-            updater.setLong(ordinal, micros)
-          }
+          updater.setLong(ordinal, micros)
         case other => throw new IncompatibleSchemaException(
           s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
       }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index af9e3a5f81f82..b6e2a2b15aac6 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -149,17 +149,15 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
       case (TimestampType, LONG) => avroType.getLogicalType match {
           // For backward compatibility, if the Avro type is Long and it is not logical type
           // (the `null` case), output the timestamp value as with millisecond precision.
-          case null | _: TimestampMillis => (getter, ordinal) =>
+          case null | _: TimestampMillis if rebaseDateTime => (getter, ordinal) =>
             val micros = getter.getLong(ordinal)
-            val rebasedMicros = if (rebaseDateTime) {
-              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
-            } else micros
+            val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(micros)
             DateTimeUtils.toMillis(rebasedMicros)
-          case _: TimestampMicros => (getter, ordinal) =>
-            val micros = getter.getLong(ordinal)
-            if (rebaseDateTime) {
-              DateTimeUtils.rebaseGregorianToJulianMicros(micros)
-            } else micros
+          case null | _: TimestampMillis => (getter, ordinal) =>
+            DateTimeUtils.toMillis(getter.getLong(ordinal))
+          case _: TimestampMicros if rebaseDateTime => (getter, ordinal) =>
+            DateTimeUtils.rebaseGregorianToJulianMicros(getter.getLong(ordinal))
+          case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
         }
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 9e89b69c0b33c..82569653c1f23 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.spark.sql.avro
 
 import java.io.File
-import java.sql.{Date, Timestamp}
+import java.sql.Timestamp
 
 import org.apache.avro.{LogicalTypes, Schema}
 import org.apache.avro.Conversions.DecimalConversion
@@ -25,7 +25,7 @@ import org.apache.avro.file.DataFileWriter
 import org.apache.avro.generic.{GenericData, GenericDatumWriter, GenericRecord}
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.{DataFrame, QueryTest, Row}
+import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -348,100 +348,6 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
       assert(msg.contains("Unscaled value too large for precision"))
     }
   }
-
-  private def readResourceAvroFile(name: String): DataFrame = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.format("avro").load(url.toString)
-  }
-
-  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
-    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-      checkAnswer(
-        readResourceAvroFile("before_1582_date_v2_4.avro"),
-        Row(java.sql.Date.valueOf("1001-01-01")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
-    }
-  }
-
-  test("SPARK-31183: rebasing microseconds timestamps in write") {
-    val tsStr = "1001-01-01 01:02:03.123456"
-    val nonRebased = "1001-01-07 01:09:05.123456"
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-        Seq(tsStr).toDF("tsS")
-          .select($"tsS".cast("timestamp").as("ts"))
-          .write.format("avro")
-          .save(path)
-
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
-      }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
-      }
-    }
-  }
-
-  test("SPARK-31183: rebasing milliseconds timestamps in write") {
-    val tsStr = "1001-01-01 01:02:03.123456"
-    val rebased = "1001-01-01 01:02:03.123"
-    val nonRebased = "1001-01-07 01:09:05.123"
-    Seq(
-      """{"type": "long","logicalType": "timestamp-millis"}""",
-      """"long"""").foreach { tsType =>
-      val timestampSchema = s"""
-          |{
-          |  "namespace": "logical",
-          |  "type": "record",
-          |  "name": "test",
-          |  "fields": [
-          |    {"name": "ts", "type": $tsType}
-          |  ]
-          |}""".stripMargin
-      withTempPath { dir =>
-        val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-          Seq(tsStr).toDF("tsS")
-            .select($"tsS".cast("timestamp").as("ts"))
-            .write
-            .option("avroSchema", timestampSchema)
-            .format("avro")
-            .save(path)
-
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(rebased)))
-        }
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(nonRebased)))
-        }
-      }
-    }
-  }
-
-  test("SPARK-31183: rebasing dates in write") {
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
-        Seq("1001-01-01").toDF("dateS")
-          .select($"dateS".cast("date").as("date"))
-          .write.format("avro")
-          .save(path)
-
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
-      }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
-      }
-    }
-  }
 }
 
 class AvroV1LogicalTypeSuite extends AvroLogicalTypeSuite {
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 360160c9c9398..34a0e2bf607a9 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -21,7 +21,7 @@ import java.io._
 import java.net.URL
 import java.nio.file.{Files, Paths}
 import java.sql.{Date, Timestamp}
-import java.util.{Locale, TimeZone, UUID}
+import java.util.{Locale, UUID}
 
 import scala.collection.JavaConverters._
 
@@ -35,9 +35,10 @@ import org.apache.commons.io.FileUtils
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql._
-import org.apache.spark.sql.TestingUDT.{IntervalData, NullData, NullUDT}
+import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -83,6 +84,11 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }, new GenericDatumReader[Any]()).getSchema.toString(false)
   }
 
+  private def readResourceAvroFile(name: String): DataFrame = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(name)
+    spark.read.format("avro").load(url.toString)
+  }
+
   test("resolve avro data source") {
     val databricksAvro = "com.databricks.spark.avro"
     // By default the backward compatibility for com.databricks.spark.avro is enabled.
@@ -402,18 +408,19 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         StructField("float", FloatType, true),
         StructField("date", DateType, true)
       ))
-      TimeZone.setDefault(TimeZone.getTimeZone("UTC"))
-      val rdd = spark.sparkContext.parallelize(Seq(
-        Row(1f, null),
-        Row(2f, new Date(1451948400000L)),
-        Row(3f, new Date(1460066400500L))
-      ))
-      val df = spark.createDataFrame(rdd, schema)
-      df.write.format("avro").save(dir.toString)
-      assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
-      checkAnswer(
-        spark.read.format("avro").load(dir.toString).select("date"),
-        Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
+      DateTimeTestUtils.withDefaultTimeZone(DateTimeUtils.TimeZoneUTC) {
+        val rdd = spark.sparkContext.parallelize(Seq(
+          Row(1f, null),
+          Row(2f, new Date(1451948400000L)),
+          Row(3f, new Date(1460066400500L))
+        ))
+        val df = spark.createDataFrame(rdd, schema)
+        df.write.format("avro").save(dir.toString)
+        assert(spark.read.format("avro").load(dir.toString).count == rdd.count)
+        checkAnswer(
+          spark.read.format("avro").load(dir.toString).select("date"),
+          Seq(Row(null), Row(new Date(1451865600000L)), Row(new Date(1459987200000L))))
+      }
     }
   }
 
@@ -1521,6 +1528,95 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       assert(deprecatedEvents.size === 1)
     }
   }
+
+  test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+      checkAnswer(
+        readResourceAvroFile("before_1582_date_v2_4.avro"),
+        Row(java.sql.Date.valueOf("1001-01-01")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      checkAnswer(
+        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
+    }
+  }
+
+  test("SPARK-31183: rebasing microseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val nonRebased = "1001-01-07 01:09:05.123456"
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq(tsStr).toDF("tsS")
+          .select($"tsS".cast("timestamp").as("ts"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing milliseconds timestamps in write") {
+    val tsStr = "1001-01-01 01:02:03.123456"
+    val rebased = "1001-01-01 01:02:03.123"
+    val nonRebased = "1001-01-07 01:09:05.123"
+    Seq(
+      """{"type": "long","logicalType": "timestamp-millis"}""",
+      """"long"""").foreach { tsType =>
+      val timestampSchema = s"""
+        |{
+        |  "namespace": "logical",
+        |  "type": "record",
+        |  "name": "test",
+        |  "fields": [
+        |    {"name": "ts", "type": $tsType}
+        |  ]
+        |}""".stripMargin
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+          Seq(tsStr).toDF("tsS")
+            .select($"tsS".cast("timestamp").as("ts"))
+            .write
+            .option("avroSchema", timestampSchema)
+            .format("avro")
+            .save(path)
+
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(rebased)))
+        }
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+          checkAnswer(
+            spark.read.schema("ts timestamp").format("avro").load(path),
+            Row(Timestamp.valueOf(nonRebased)))
+        }
+      }
+    }
+  }
+
+  test("SPARK-31183: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        Seq("1001-01-01").toDF("dateS")
+          .select($"dateS".cast("date").as("date"))
+          .write.format("avro")
+          .save(path)
+
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
+      }
+    }
+  }
 }
 
 class AvroV1Suite extends AvroSuite {

From 8c09160cb8ec750826932988e1c87385a496ba22 Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Sun, 22 Mar 2020 22:22:13 -0700
Subject: [PATCH 0322/1280] [SPARK-30494][SQL] Fix cached data leakage during
 replacing an existing view

### What changes were proposed in this pull request?

The cached RDD for plan "select 1" stays in memory forever until the session close. This cached data cannot be used since the view temp1 has been replaced by another plan. It's a memory leak.

We can reproduce by below commands:
```
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 3.0.0-SNAPSHOT
      /_/

Using Scala version 2.12.10 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_201)
Type in expressions to have them evaluated.
Type :help for more information.

scala> spark.sql("create or replace temporary view temp1 as select 1")
scala> spark.sql("cache table temp1")
scala> spark.sql("create or replace temporary view temp1 as select 1, 2")
scala> spark.sql("cache table temp1")
scala> assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1, 2")).isDefined)
scala> assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isDefined)
```

### Why are the changes needed?
Fix the memory leak, specially for long running mode.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Add an unit test.

Closes #27185 from LantaoJin/SPARK-30494.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 929b794e25ff5454dadde7da304e6df25526d60e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/execution/command/CommandUtils.scala  |  8 ++++
 .../spark/sql/execution/command/tables.scala  | 13 +-----
 .../spark/sql/execution/command/views.scala   | 15 +++++++
 .../apache/spark/sql/CachedTableSuite.scala   | 42 +++++++++++++++++++
 .../hive/execution/InsertIntoHiveTable.scala  |  2 +-
 5 files changed, 68 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
index b229b238238fd..7e456a6af6fc9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
@@ -385,4 +385,12 @@ object CommandUtils extends Logging {
   private def isDataPath(path: Path, stagingDir: String): Boolean = {
     !path.getName.startsWith(stagingDir) && DataSourceUtils.isDataPath(path)
   }
+
+  def uncacheTableOrView(sparkSession: SparkSession, name: String): Unit = {
+    try {
+      sparkSession.catalog.uncacheTable(name)
+    } catch {
+      case NonFatal(e) => logWarning("Exception when attempting to uncache $name", e)
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index d4de822d8b036..61955ba4a4b55 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -196,11 +196,7 @@ case class AlterTableRenameCommand(
       // this can happen with Hive tables when the underlying catalog is in-memory.
       val wasCached = Try(sparkSession.catalog.isCached(oldName.unquotedString)).getOrElse(false)
       if (wasCached) {
-        try {
-          sparkSession.catalog.uncacheTable(oldName.unquotedString)
-        } catch {
-          case NonFatal(e) => log.warn(e.toString, e)
-        }
+        CommandUtils.uncacheTableOrView(sparkSession, oldName.unquotedString)
       }
       // Invalidate the table last, otherwise uncaching the table would load the logical plan
       // back into the hive metastore cache
@@ -230,12 +226,7 @@ case class AlterTableAddColumnsCommand(
     val catalog = sparkSession.sessionState.catalog
     val catalogTable = verifyAlterTableAddColumn(sparkSession.sessionState.conf, catalog, table)
 
-    try {
-      sparkSession.catalog.uncacheTable(table.quotedString)
-    } catch {
-      case NonFatal(e) =>
-        log.warn(s"Exception when attempting to uncache table ${table.quotedString}", e)
-    }
+    CommandUtils.uncacheTableOrView(sparkSession, table.quotedString)
     catalog.refreshTable(table)
 
     SchemaUtils.checkColumnNameDuplication(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 38481dda428a5..795f9002d30bc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable
 import org.apache.spark.sql.catalyst.expressions.{Alias, SubqueryExpression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.types.MetadataBuilder
 import org.apache.spark.sql.util.SchemaUtils
 
@@ -108,9 +109,19 @@ case class CreateViewCommand(
     verifyTemporaryObjectsNotExists(catalog)
 
     if (viewType == LocalTempView) {
+      if (replace && catalog.getTempView(name.table).isDefined) {
+        logDebug(s"Try to uncache ${name.quotedString} before replacing.")
+        CommandUtils.uncacheTableOrView(sparkSession, name.quotedString)
+      }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
       catalog.createTempView(name.table, aliasedPlan, overrideIfExists = replace)
     } else if (viewType == GlobalTempView) {
+      if (replace && catalog.getGlobalTempView(name.table).isDefined) {
+        val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
+        val globalTempView = TableIdentifier(name.table, Option(db))
+        logDebug(s"Try to uncache ${globalTempView.quotedString} before replacing.")
+        CommandUtils.uncacheTableOrView(sparkSession, globalTempView.quotedString)
+      }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
       catalog.createGlobalTempView(name.table, aliasedPlan, overrideIfExists = replace)
     } else if (catalog.tableExists(name)) {
@@ -125,6 +136,10 @@ case class CreateViewCommand(
         val viewIdent = tableMetadata.identifier
         checkCyclicViewReference(analyzedPlan, Seq(viewIdent), viewIdent)
 
+        // uncache the cached data before replacing an exists view
+        logDebug(s"Try to uncache ${viewIdent.quotedString} before replacing.")
+        CommandUtils.uncacheTableOrView(sparkSession, viewIdent.quotedString)
+
         // Handles `CREATE OR REPLACE VIEW v0 AS SELECT ...`
         // Nothing we need to retain from the old view, so just drop and create a new one
         catalog.dropTable(viewIdent, ignoreIfNotExists = false, purge = false)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 81893534fc2a3..9baaaa9e07c0e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1122,4 +1122,46 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       assert(!spark.catalog.isCached("t1"))
     }
   }
+
+  test("SPARK-30494 Fix the leak of cached data when replace an existing view") {
+    withTempView("tempView") {
+      spark.catalog.clearCache()
+      sql("create or replace temporary view tempView as select 1")
+      sql("cache table tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isDefined)
+      sql("create or replace temporary view tempView as select 1, 2")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isEmpty)
+      sql("cache table tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1, 2")).isDefined)
+    }
+
+    withGlobalTempView("tempGlobalTempView") {
+      spark.catalog.clearCache()
+      sql("create or replace global temporary view tempGlobalTempView as select 1")
+      sql("cache table global_temp.tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isDefined)
+      sql("create or replace global temporary view tempGlobalTempView as select 1, 2")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1")).isEmpty)
+      sql("cache table global_temp.tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(sql("select 1, 2")).isDefined)
+    }
+
+    withView("view1") {
+      spark.catalog.clearCache()
+      sql("create or replace view view1 as select 1")
+      sql("cache table view1")
+      sql("create or replace view view1 as select 1, 2")
+      sql("cache table view1")
+      // the cached plan of persisted view likes below,
+      // we cannot use the same assertion of temp view.
+      // SubqueryAlias
+      //    |
+      //    + View
+      //        |
+      //        + Project[1 AS 1]
+      spark.sharedState.cacheManager.uncacheQuery(spark.table("view1"), cascade = false)
+      // make sure there is no cached data leak
+      assert(spark.sharedState.cacheManager.isEmpty)
+    }
+  }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 801be64702519..19f439598142e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -106,7 +106,7 @@ case class InsertIntoHiveTable(
     }
 
     // un-cache this table.
-    sparkSession.catalog.uncacheTable(table.identifier.quotedString)
+    CommandUtils.uncacheTableOrView(sparkSession, table.identifier.quotedString)
     sparkSession.sessionState.catalog.refreshTable(table.identifier)
 
     CommandUtils.updateTableStats(sparkSession, table)

From 5229d1c66c8817d8fc37b9bc992bf1e411e302b4 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 23 Mar 2020 14:21:24 +0800
Subject: [PATCH 0323/1280] [SPARK-31211][SQL] Fix rebasing of 29 February of
 Julian leap years

In the PR, I propose to fix the issue of rebasing leap years in Julian calendar to Proleptic Gregorian calendar in which the years are not leap years. In the Julian calendar, every four years is a leap year, with a leap day added to the month of February. In Proleptic Gregorian calendar, every year that is exactly divisible by four is a leap year, except for years that are exactly divisible by 100, but these centurial years are leap years, if they are exactly divisible by 400. In this ways, the date **1000-02-29** exists in the Julian calendar but not in Proleptic Gregorian calendar.

I modified the `rebaseJulianToGregorianMicros()` and `rebaseJulianToGregorianDays()` in `DateTimeUtils` by passing 1 as a day number of month while forming `LocalDate` or `LocalDateTime`, and adding the number of days using the `plusDays()` method. For example, **1000-02-29** doesn't exist in Proleptic Gregorian calendar, and `LocalDate.of(1000, 2, 29)` throws an exception. To avoid the issue, I build the `LocalDate.of(1000, 2, 1)` date and add 28 days. The `plusDays(28)` method produces the next valid date after `1000-02-28` which is **1000-03-01**.

Before the changes, the `java.time.DateTimeException` exception is raised while loading the date `1000-02-29` from parquet files saved by Spark 2.4.5:
```scala
scala> spark.conf.set("spark.sql.legacy.parquet.rebaseDateTime.enabled", true)
scala> spark.read.parquet("/Users/maxim/tmp/before_1582/2_4_5_date_leap").show
20/03/21 03:03:59 ERROR Executor: Exception in task 0.0 in stage 3.0 (TID 3)
java.time.DateTimeException: Invalid date 'February 29' as '1000' is not a leap year
```
The parquet files were saved via the commands:
```shell
$ export TZ="America/Los_Angeles"
```
```scala
scala> scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> val df = Seq(java.sql.Date.valueOf("1000-02-29")).toDF("dateS").select($"dateS".as("date"))
df: org.apache.spark.sql.DataFrame = [date: date]
scala> df.write.mode("overwrite").parquet("/Users/maxim/tmp/before_1582/2_4_5_date_leap")
scala> spark.read.parquet("/Users/maxim/tmp/before_1582/2_4_5_date_leap").show
+----------+
|      date|
+----------+
|1000-02-29|
+----------+
```

Yes, after the fix:
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
scala> spark.conf.set("spark.sql.legacy.parquet.rebaseDateTime.enabled", true)
scala> spark.read.parquet("/Users/maxim/tmp/before_1582/2_4_5_date_leap").show
+----------+
|      date|
+----------+
|1000-03-01|
+----------+
```

Added tests to `DateTimeUtilsSuite`.

Closes #27974 from MaxGekk/julian-date-29-feb.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit db6247faa8780bca8f8d3ba71b568ea63b162973)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 14 +++-
 .../catalyst/util/DateTimeUtilsSuite.scala    | 82 +++++++++++++++----
 2 files changed, 77 insertions(+), 19 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 7f5babfca4e59..ba1c5093ee9be 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -983,11 +983,16 @@ object DateTimeUtils {
     val localDateTime = LocalDateTime.of(
       cal.get(Calendar.YEAR),
       cal.get(Calendar.MONTH) + 1,
-      cal.get(Calendar.DAY_OF_MONTH),
+      // The number of days will be added later to handle non-existing
+      // Julian dates in Proleptic Gregorian calendar.
+      // For example, 1000-02-29 exists in Julian calendar because 1000
+      // is a leap year but it is not a leap year in Gregorian calendar.
+      1,
       cal.get(Calendar.HOUR_OF_DAY),
       cal.get(Calendar.MINUTE),
       cal.get(Calendar.SECOND),
       (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
+      .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
     instantToMicros(localDateTime.atZone(ZoneId.systemDefault).toInstant)
   }
 
@@ -1011,7 +1016,12 @@ object DateTimeUtils {
     val localDate = LocalDate.of(
       utcCal.get(Calendar.YEAR),
       utcCal.get(Calendar.MONTH) + 1,
-      utcCal.get(Calendar.DAY_OF_MONTH))
+      // The number of days will be added later to handle non-existing
+      // Julian dates in Proleptic Gregorian calendar.
+      // For example, 1000-02-29 exists in Julian calendar because 1000
+      // is a leap year but it is not a leap year in Gregorian calendar.
+      1)
+      .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
     Math.toIntExact(localDate.toEpochDay)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index f2ad9e6fad44e..96da4bee7fa61 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -670,6 +670,17 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     }
   }
 
+  private def parseToJulianMicros(s: String): Long = {
+    val ts = Timestamp.valueOf(s)
+    val julianMicros = fromMillis(ts.getTime) +
+      ((ts.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
+    julianMicros
+  }
+
+  private def parseToGregMicros(s: String, zoneId: ZoneId): Long = {
+    instantToMicros(LocalDateTime.parse(s).atZone(zoneId).toInstant)
+  }
+
   test("rebase julian to/from gregorian micros") {
     outstandingTimezones.foreach { timeZone =>
       withDefaultTimeZone(timeZone) {
@@ -684,30 +695,27 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
           "1970-01-01 00:00:00.000001", // The epoch day
           "2020-03-14 09:33:01.500000").foreach { ts =>
           withClue(s"time zone = ${timeZone.getID} ts = $ts") {
-            val julianTs = Timestamp.valueOf(ts)
-            val julianMicros = fromMillis(julianTs.getTime) +
-              ((julianTs.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
-            val gregorianMicros = instantToMicros(LocalDateTime.parse(ts.replace(' ', 'T'))
-              .atZone(timeZone.toZoneId)
-              .toInstant)
+            val julianMicros = parseToJulianMicros(ts)
+            val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), timeZone.toZoneId)
 
-            assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
-            assert(rebaseGregorianToJulianMicros(gregorianMicros) === julianMicros)
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregMicros)
+            assert(rebaseGregorianToJulianMicros(gregMicros) === julianMicros)
           }
         }
       }
     }
   }
 
+  // millisToDays() and fromJavaDate() are taken from Spark 2.4
+  private def millisToDaysLegacy(millisUtc: Long, timeZone: TimeZone): Int = {
+    val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
+    Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
+  }
+  private def fromJavaDateLegacy(date: Date): Int = {
+    millisToDaysLegacy(date.getTime, defaultTimeZone())
+  }
+
   test("rebase gregorian to/from julian days") {
-    // millisToDays() and fromJavaDate() are taken from Spark 2.4
-    def millisToDays(millisUtc: Long, timeZone: TimeZone): Int = {
-      val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
-      Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
-    }
-    def fromJavaDate(date: Date): Int = {
-      millisToDays(date.getTime, defaultTimeZone())
-    }
     outstandingTimezones.foreach { timeZone =>
       withDefaultTimeZone(timeZone) {
         Seq(
@@ -720,7 +728,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
           "1969-12-31",
           "1970-01-01", // The epoch day
           "2020-03-14").foreach { date =>
-          val julianDays = fromJavaDate(Date.valueOf(date))
+          val julianDays = fromJavaDateLegacy(Date.valueOf(date))
           val gregorianDays = localDateToDays(LocalDate.parse(date))
 
           assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
@@ -729,4 +737,44 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
   }
+
+  test("rebase julian to gregorian date for leap years") {
+    outstandingTimezones.foreach { timeZone =>
+      withDefaultTimeZone(timeZone) {
+        Seq(
+          "1000-02-29" -> "1000-03-01",
+          "1600-02-29" -> "1600-02-29",
+          "1700-02-29" -> "1700-03-01",
+          "2000-02-29" -> "2000-02-29").foreach { case (julianDate, gregDate) =>
+          withClue(s"tz = ${timeZone.getID} julian date = $julianDate greg date = $gregDate") {
+            val date = Date.valueOf(julianDate)
+            val julianDays = fromJavaDateLegacy(date)
+            val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
+
+            assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+          }
+        }
+      }
+    }
+  }
+
+  test("rebase julian to gregorian timestamp for leap years") {
+    outstandingTimezones.foreach { timeZone =>
+      withDefaultTimeZone(timeZone) {
+        Seq(
+          "1000-02-29 01:02:03.123456" -> "1000-03-01T01:02:03.123456",
+          "1600-02-29 11:12:13.654321" -> "1600-02-29T11:12:13.654321",
+          "1700-02-29 21:22:23.000001" -> "1700-03-01T21:22:23.000001",
+          "2000-02-29 00:00:00.999999" -> "2000-02-29T00:00:00.999999"
+        ).foreach { case (julianTs, gregTs) =>
+          withClue(s"tz = ${timeZone.getID} julian ts = $julianTs greg ts = $gregTs") {
+            val julianMicros = parseToJulianMicros(julianTs)
+            val gregorianMicros = parseToGregMicros(gregTs, timeZone.toZoneId)
+
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
+          }
+        }
+      }
+    }
+  }
 }

From e7587df9478f96d5a949afda26fb4a6bf8636d6f Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 23 Mar 2020 16:28:34 +0800
Subject: [PATCH 0324/1280] [SPARK-31190][SQL] ScalaReflection should not
 erasure user defined AnyVal type

### What changes were proposed in this pull request?

Improve `ScalaReflection` to only don't erasure non user defined `AnyVal` type, but still erasure other types, e.g. `Any`. And this brings two benefits:

1. Give better encode error message for some unsupported types, e.g. `Any`

2. Won't miss the walk path for the `AnyVal` type

### Why are the changes needed?

Firstly, PR #15284 added encode(serializeFor/deserializeFor) support for value class, which extends `AnyVal`, by not erasure types. But, this also introduce a problem that when user try to encoder unsupported types, e.g. `Any`, it will fail on `java.lang.ClassNotFoundException: scala.Any` due to the reason that `scala.Any` doesn't erasure to `java.lang.Object`.

Also, in current `getClassNameFromType()`, it always erasure types which could missing walked path for user defined `AnyVal` types.

### Does this PR introduce any user-facing change?

Yes. For the test below:

```
case class Bar(i: Any)
case class Foo(i: Bar) extends AnyVal

test() {
  implicitly[ExpressionEncoder[Foo]]
}
```

Before:

```
java.lang.ClassNotFoundException: scala.Any
 at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
 at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
 at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:355)
 ...
````

After:
```
java.lang.UnsupportedOperationException: No Encoder found for Any
 - field (class: "java.lang.Object", name: "i")
 - field (class: "org.apache.spark.sql.catalyst.encoders.Bar", name: "i")
 - root class: "org.apache.spark.sql.catalyst.encoders.Foo"
 at org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$serializerFor$1(ScalaReflection.scala:561)
```

### How was this patch tested?

Added unit test and test manually.

Closes #27959 from Ngone51/impr_anyval.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5c4d44bb83aafc296265524ec1ee09bcd11365ae)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/ScalaReflection.scala  | 45 ++++++++++++-------
 .../encoders/ExpressionEncoderSuite.scala     |  9 ++++
 2 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 1f7634bafa420..f587245116581 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -611,10 +611,39 @@ object ScalaReflection extends ScalaReflection {
     }
   }
 
+  private def erasure(tpe: Type): Type = {
+    // For user-defined AnyVal classes, we should not erasure it. Otherwise, it will
+    // resolve to underlying type which wrapped by this class, e.g erasure
+    // `case class Foo(i: Int) extends AnyVal` will return type `Int` instead of `Foo`.
+    // But, for other types, we do need to erasure it. For example, we need to erasure
+    // `scala.Any` to `java.lang.Object` in order to load it from Java ClassLoader.
+    // Please see SPARK-17368 & SPARK-31190 for more details.
+    if (isSubtype(tpe, localTypeOf[AnyVal]) && !tpe.toString.startsWith("scala")) {
+      tpe
+    } else {
+      tpe.erasure
+    }
+  }
+
+  /**
+   * Returns the full class name for a type. The returned name is the canonical
+   * Scala name, where each component is separated by a period. It is NOT the
+   * Java-equivalent runtime name (no dollar signs).
+   *
+   * In simple cases, both the Scala and Java names are the same, however when Scala
+   * generates constructs that do not map to a Java equivalent, such as singleton objects
+   * or nested classes in package objects, it uses the dollar sign ($) to create
+   * synthetic classes, emulating behaviour in Java bytecode.
+   */
+  def getClassNameFromType(tpe: `Type`): String = {
+    erasure(tpe).dealias.typeSymbol.asClass.fullName
+  }
+
   /*
    * Retrieves the runtime class corresponding to the provided type.
    */
-  def getClassFromType(tpe: Type): Class[_] = mirror.runtimeClass(tpe.dealias.typeSymbol.asClass)
+  def getClassFromType(tpe: Type): Class[_] =
+    mirror.runtimeClass(erasure(tpe).dealias.typeSymbol.asClass)
 
   case class Schema(dataType: DataType, nullable: Boolean)
 
@@ -863,20 +892,6 @@ trait ScalaReflection extends Logging {
     tag.in(mirror).tpe.dealias
   }
 
-  /**
-   * Returns the full class name for a type. The returned name is the canonical
-   * Scala name, where each component is separated by a period. It is NOT the
-   * Java-equivalent runtime name (no dollar signs).
-   *
-   * In simple cases, both the Scala and Java names are the same, however when Scala
-   * generates constructs that do not map to a Java equivalent, such as singleton objects
-   * or nested classes in package objects, it uses the dollar sign ($) to create
-   * synthetic classes, emulating behaviour in Java bytecode.
-   */
-  def getClassNameFromType(tpe: `Type`): String = {
-    tpe.dealias.erasure.typeSymbol.asClass.fullName
-  }
-
   /**
    * Returns the parameter names and types for the primary constructor of this type.
    *
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index c1f1be3b30e4b..66a1bbe01f5c6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -107,6 +107,8 @@ class UDTForCaseClass extends UserDefinedType[UDTCaseClass] {
   }
 }
 
+case class Bar(i: Any)
+case class Foo(i: Bar) extends AnyVal
 case class PrimitiveValueClass(wrapped: Int) extends AnyVal
 case class ReferenceValueClass(wrapped: ReferenceValueClass.Container) extends AnyVal
 object ReferenceValueClass {
@@ -311,6 +313,13 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
 
   productTest(("UDT", new ExamplePoint(0.1, 0.2)))
 
+  test("AnyVal class with Any fields") {
+    val exception = intercept[UnsupportedOperationException](implicitly[ExpressionEncoder[Foo]])
+    val errorMsg = exception.getMessage
+    assert(errorMsg.contains("root class: \"org.apache.spark.sql.catalyst.encoders.Foo\""))
+    assert(errorMsg.contains("No Encoder found for Any"))
+  }
+
   test("nullable of encoder schema") {
     def checkNullable[T: ExpressionEncoder](nullable: Boolean*): Unit = {
       assert(implicitly[ExpressionEncoder[T]].schema.map(_.nullable) === nullable.toSeq)

From ac7976791819ed9a2726c72d80528982e142c818 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 23 Mar 2020 23:02:48 +0900
Subject: [PATCH 0325/1280] [SPARK-31159][SQL][FOLLOWUP] Move checking of the
 `rebaseDateTime` flag out of the loop in `VectorizedColumnReader`

In the PR, I propose to refactor reading of timestamps of the `TIMESTAMP_MILLIS` logical type from Parquet files in `VectorizedColumnReader`, and move checking of the `rebaseDateTime` flag out of the internal loop.

To avoid any additional overhead of the checking the SQL config `spark.sql.legacy.parquet.rebaseDateTime.enabled` introduced by the PR https://github.com/apache/spark/pull/27915.

No

By running the test suite `ParquetIOSuite`.

Closes #27973 from MaxGekk/rebase-parquet-datetime-followup.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit aa3a7429f44970eb95a58e1f6cfbf7d89d6753a0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../parquet/VectorizedColumnReader.java       | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index f9b613999a65b..fa1838f092c2e 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -462,15 +462,22 @@ private void readLongBatch(int rowId, int num, WritableColumnVector column) thro
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
       }
     } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
-      for (int i = 0; i < num; i++) {
-        if (defColumn.readInteger() == maxDefLevel) {
-          long micros = DateTimeUtils.fromMillis(dataColumn.readLong());
-          if (rebaseDateTime) {
-            micros = DateTimeUtils.rebaseJulianToGregorianMicros(micros);
+      if (rebaseDateTime) {
+        for (int i = 0; i < num; i++) {
+          if (defColumn.readInteger() == maxDefLevel) {
+            long micros = DateTimeUtils.fromMillis(dataColumn.readLong());
+            column.putLong(rowId + i, DateTimeUtils.rebaseJulianToGregorianMicros(micros));
+          } else {
+            column.putNull(rowId + i);
+          }
+        }
+      } else {
+        for (int i = 0; i < num; i++) {
+          if (defColumn.readInteger() == maxDefLevel) {
+            column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong()));
+          } else {
+            column.putNull(rowId + i);
           }
-          column.putLong(rowId + i, micros);
-        } else {
-          column.putNull(rowId + i);
         }
       }
     } else {

From 9fbd916b294ce500c8adf11c1cad0d46abaa7bf2 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 23 Mar 2020 22:00:50 +0800
Subject: [PATCH 0326/1280] [SPARK-31133][SQL][DOC] fix sql ref doc for DML

### What changes were proposed in this pull request?

`INSERT OVERWRITE DIRECTORY` can only use file format (class implements `org.apache.spark.sql.execution.datasources.FileFormat`). This PR fixes it and other minor improvement.

### Why are the changes needed?

### Does this PR introduce any user-facing change?

### How was this patch tested?

Closes #27891 from cloud-fan/doc.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-syntax-dml-insert-overwrite-directory.md | 10 +++++-----
 docs/sql-ref-syntax-dml-insert-overwrite-table.md     |  6 +++---
 docs/sql-ref-syntax-dml-load.md                       | 10 +++++-----
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
index 6f19c62bc3d56..7f3224de0ccee 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,7 @@ license: |
   limitations under the License.
 ---
 ### Description
-The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the directory with the new values using Spark native format. The inserted rows can be specified by value expressions or result from a query.
+The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the directory with the new values using a given Spark file format. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
 {% highlight sql %}
@@ -39,13 +39,13 @@ INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
 <dl>
   <dt><code><em>file_format</em></code></dt>
   <dd>
-  Specifies the file format to use for the insert. Valid options are <code>TEXT</code>, <code>CSV</code>, <code>JSON</code>, <code>JDBC</code>, <code>PARQUET</code>, <code>ORC</code>, <code>HIVE</code>, <code>DELTA</code>, <code>LIBSVM</code>, or a fully qualified class name of a custom implementation of <code>org.apache.spark.sql.sources.DataSourceRegister</code>.
+  Specifies the file format to use for the insert. Valid options are <code>TEXT</code>, <code>CSV</code>, <code>JSON</code>, <code>JDBC</code>, <code>PARQUET</code>, <code>ORC</code>, <code>HIVE</code>, <code>LIBSVM</code>, or a fully qualified class name of a custom implementation of <code>org.apache.spark.sql.execution.datasources.FileFormat</code>.
   </dd>
 </dl>
 
 <dl>
   <dt><code><em>OPTIONS ( key = val [ , ... ] )</em></code></dt>
-  <dd>Specifies one or more table property key and value pairs.</dd>
+  <dd>Specifies one or more options for the writing of the file format.</dd>
 </dl>
 
 <dl>
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index 1f32342642b2a..2318a8b622202 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -178,7 +178,7 @@ INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
      | Jason Wang     | 908 Bird St, Saratoga          | 121212         | true           |
      + -------------- + ------------------------------ + -------------- + -------------- +
 
- INSERT OVERWRITE students;
+ INSERT OVERWRITE students
       FROM applicants SELECT name, address, id applicants WHERE qualified = true;
 
  SELECT * FROM students;
diff --git a/docs/sql-ref-syntax-dml-load.md b/docs/sql-ref-syntax-dml-load.md
index aadfd708e274e..090c49dc5d082 100644
--- a/docs/sql-ref-syntax-dml-load.md
+++ b/docs/sql-ref-syntax-dml-load.md
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -20,7 +20,7 @@ license: |
 ---
 
 ### Description
-`LOAD DATA` statement loads the data into a table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
+`LOAD DATA` statement loads the data into a Hive serde table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
 
 ### Syntax
 {% highlight sql %}
@@ -78,7 +78,7 @@ LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ part
      | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
      + -------------- + ------------------------------ + -------------- +
 
- CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT);
+ CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT) USING HIVE;
 
  -- Assuming the students table is in '/user/hive/warehouse/'
  LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
@@ -92,7 +92,7 @@ LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ part
      + -------------- + ------------------------------ + -------------- +
 
  -- Example with partition specification.
- CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
+ CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) PARTITIONED BY (c2, c3);
 
  INSERT INTO test_partition PARTITION (c2 = 2, c3 = 3) VALUES (1);
 

From 5c1e78fec71fde04f2b54bae1fec08431c3a4427 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 24 Mar 2020 12:07:22 +0800
Subject: [PATCH 0327/1280] [SPARK-31205][SQL] support string literal as the
 second argument of date_add/date_sub functions

### What changes were proposed in this pull request?

https://github.com/apache/spark/pull/26412 introduced a behavior change that `date_add`/`date_sub` functions can't accept string and double values in the second parameter. This is reasonable as it's error-prone to cast string/double to int at runtime.

However, using string literals as function arguments is very common in SQL databases. To avoid breaking valid use cases that the string literal is indeed an integer, this PR proposes to add ansi_cast for string literal in date_add/date_sub functions. If the string value is not a valid integer, we fail at query compiling time because of constant folding.

### Why are the changes needed?

avoid breaking changes

### Does this PR introduce any user-facing change?

Yes, now 3.0 can run `date_add('2011-11-11', '1')` like 2.4

### How was this patch tested?

new tests.

Closes #27965 from cloud-fan/string.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1d0f54951ea66cfbfc712300ed04f6d848b2fd5a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |  2 +-
 .../sql/catalyst/analysis/Analyzer.scala      |  6 +-
 .../sql/catalyst/analysis/TypeCoercion.scala  | 30 ++++++++
 .../resources/sql-tests/inputs/datetime.sql   | 10 +++
 .../sql-tests/results/datetime.sql.out        | 73 ++++++++++++++++++-
 .../native/promoteStrings.sql.out             |  6 +-
 6 files changed, 118 insertions(+), 9 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 31d5c68caa2dc..b4e51c29ce4c8 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -110,7 +110,7 @@ license: |
 
 ### UDFs and Built-in Functions
 
-  - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
+  - Since Spark 3.0, the `date_add` and `date_sub` functions only accept int, smallint, tinyint as the 2nd argument, fractional and non-literal string are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), 12.34)` will cause `AnalysisException`. Note that, string literals are still allowed, but Spark will throw Analysis Exception if the string content is not a valid integer. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
 
   - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index eadcd0f8363c7..975f02332512f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -295,8 +295,8 @@ class Analyzer(
           case (CalendarIntervalType, CalendarIntervalType) => a
           case (_, CalendarIntervalType) => Cast(TimeAdd(l, r), l.dataType)
           case (CalendarIntervalType, _) => Cast(TimeAdd(r, l), r.dataType)
-          case (DateType, _) => DateAdd(l, r)
-          case (_, DateType) => DateAdd(r, l)
+          case (DateType, dt) if dt != StringType => DateAdd(l, r)
+          case (dt, DateType) if dt != StringType => DateAdd(r, l)
           case _ => a
         }
         case s @ Subtract(l, r) if s.childrenResolved => (l.dataType, r.dataType) match {
@@ -305,7 +305,7 @@ class Analyzer(
           case (TimestampType, _) => SubtractTimestamps(l, r)
           case (_, TimestampType) => SubtractTimestamps(l, r)
           case (_, DateType) => SubtractDates(l, r)
-          case (DateType, _) => DateSub(l, r)
+          case (DateType, dt) if dt != StringType => DateSub(l, r)
           case _ => s
         }
         case m @ Multiply(l, r) if m.childrenResolved => (l.dataType, r.dataType) match {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index e149bf2f49764..5a5d7c6620fb5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -23,6 +23,7 @@ import scala.annotation.tailrec
 import scala.collection.mutable
 
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -63,6 +64,7 @@ object TypeCoercion {
       ImplicitTypeCasts ::
       DateTimeOperations ::
       WindowFrameCoercion ::
+      StringLiteralCoercion ::
       Nil
 
   // See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types.
@@ -1043,6 +1045,34 @@ object TypeCoercion {
       }
     }
   }
+
+  /**
+   * A special rule to support string literal as the second argument of date_add/date_sub functions,
+   * to keep backward compatibility as a temporary workaround.
+   * TODO(SPARK-28589): implement ANSI type type coercion and handle string literals.
+   */
+  object StringLiteralCoercion extends TypeCoercionRule {
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+      // Skip nodes who's children have not been resolved yet.
+      case e if !e.childrenResolved => e
+      case DateAdd(l, r) if r.dataType == StringType && r.foldable =>
+        val days = try {
+          AnsiCast(r, IntegerType).eval().asInstanceOf[Int]
+        } catch {
+          case e: NumberFormatException => throw new AnalysisException(
+            "The second argument of 'date_add' function needs to be an integer.", cause = Some(e))
+        }
+        DateAdd(l, Literal(days))
+      case DateSub(l, r) if r.dataType == StringType && r.foldable =>
+        val days = try {
+          AnsiCast(r, IntegerType).eval().asInstanceOf[Int]
+        } catch {
+          case e: NumberFormatException => throw new AnalysisException(
+            "The second argument of 'date_sub' function needs to be an integer.", cause = Some(e))
+        }
+        DateSub(l, Literal(days))
+    }
+  }
 }
 
 trait TypeCoercionRule extends Rule[LogicalPlan] with Logging {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 2c4ed64eda80a..fd3325085df96 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -58,20 +58,30 @@ select date_add('2011-11-11', 1L);
 select date_add('2011-11-11', 1.0);
 select date_add('2011-11-11', 1E1);
 select date_add('2011-11-11', '1');
+select date_add('2011-11-11', '1.2');
 select date_add(date'2011-11-11', 1);
 select date_add(timestamp'2011-11-11', 1);
 select date_sub(date'2011-11-11', 1);
+select date_sub(date'2011-11-11', '1');
+select date_sub(date'2011-11-11', '1.2');
 select date_sub(timestamp'2011-11-11', 1);
 select date_sub(null, 1);
 select date_sub(date'2011-11-11', null);
 select date'2011-11-11' + 1E1;
+select date'2011-11-11' + '1';
 select null + date '2001-09-28';
 select date '2001-09-28' + 7Y;
 select 7S + date '2001-09-28';
 select date '2001-10-01' - 7;
+select date '2001-10-01' - '7';
 select date '2001-09-28' + null;
 select date '2001-09-28' - null;
 
+-- date add/sub with non-literal string column
+create temp view v as select '1' str;
+select date_add('2011-11-11', str) from v;
+select date_sub('2011-11-11', str) from v;
+
 -- subtract dates
 select null - date '2019-10-06';
 select date '2001-10-01' - date '2001-09-28';
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index f440b5fa1b893..aec04da4dd54d 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 77
+-- Number of queries: 85
 
 
 -- !query
@@ -266,10 +266,18 @@ cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mi
 -- !query
 select date_add('2011-11-11', '1')
 -- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2011-11-11' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+The second argument of 'date_add' function needs to be an integer.;
 
 
 -- !query
@@ -296,6 +304,23 @@ struct<date_sub(DATE '2011-11-11', 1):date>
 2011-11-10
 
 
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.;
+
+
 -- !query
 select date_sub(timestamp'2011-11-11', 1)
 -- !query schema
@@ -329,6 +354,15 @@ org.apache.spark.sql.AnalysisException
 cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
 
 
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
 -- !query
 select null + date '2001-09-28'
 -- !query schema
@@ -361,6 +395,15 @@ struct<date_sub(DATE '2001-10-01', 7):date>
 2001-09-24
 
 
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
 -- !query
 select date '2001-09-28' + null
 -- !query schema
@@ -377,6 +420,32 @@ struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
 NULL
 
 
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
 -- !query
 select null - date '2019-10-06'
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
index 31353bdedc69f..b8c190beeae19 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/promoteStrings.sql.out
@@ -107,7 +107,7 @@ SELECT '1' + cast('2017-12-11 09:30:00' as date)        FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
 
 
 -- !query
@@ -698,7 +698,7 @@ SELECT cast('2017-12-11 09:30:00' as date)        + '1' FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve 'date_add(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
 
 
 -- !query
@@ -790,7 +790,7 @@ SELECT cast('2017-12-11 09:30:00' as date)        - '1' FROM t
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'date_sub(CAST('2017-12-11 09:30:00' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+cannot resolve 'date_sub(CAST('2017-12-11 09:30:00' AS DATE), CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
 
 
 -- !query

From dad4d6af0146bd76e4c80d12dac0735a03afd303 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 24 Mar 2020 14:10:59 +0900
Subject: [PATCH 0328/1280] [SPARK-31229][SQL][TESTS] Add unit tests
 TypeCoercion.findTypeForComplex and Cast.canCast in null <> complex types

### What changes were proposed in this pull request?

This PR (SPARK-31229) is rather a followup of https://github.com/apache/spark/pull/27926 (SPARK-31166). It adds unittests for `TypeCoercion.findTypeForComplex` and `Cast.canCast` about struct, map and array with the respect to null types.

### Why are the changes needed?

To detect which scope was broken in the future easily.

### Does this PR introduce any user-facing change?

No, it's a test-only.

### How was this patch tested?

Unittests were added.

Closes #27990 from HyukjinKwon/SPARK-31166-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit bd324007d51b5dce4a4520fee21412a1587e08b6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../catalyst/analysis/TypeCoercionSuite.scala  | 18 ++++++++++++++++++
 .../sql/catalyst/expressions/CastSuite.scala   | 17 +++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index 0d6f9bcedb6a2..95005fd3f5a5e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -467,11 +467,21 @@ class TypeCoercionSuite extends AnalysisTest {
       ArrayType(IntegerType, containsNull = false),
       Some(ArrayType(IntegerType, containsNull = true)))
 
+    widenTest(
+      ArrayType(NullType, containsNull = true),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = true)))
+
     widenTest(
       MapType(IntegerType, StringType, valueContainsNull = true),
       MapType(IntegerType, StringType, valueContainsNull = false),
       Some(MapType(IntegerType, StringType, valueContainsNull = true)))
 
+    widenTest(
+      MapType(NullType, NullType, true),
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, true)))
+
     widenTest(
       new StructType()
         .add("arr", ArrayType(IntegerType, containsNull = true), nullable = false),
@@ -479,6 +489,14 @@ class TypeCoercionSuite extends AnalysisTest {
         .add("arr", ArrayType(IntegerType, containsNull = false), nullable = true),
       Some(new StructType()
         .add("arr", ArrayType(IntegerType, containsNull = true), nullable = true)))
+
+    widenTest(
+      new StructType()
+        .add("null", NullType, nullable = true),
+      new StructType()
+        .add("null", IntegerType, nullable = false),
+      Some(new StructType()
+        .add("null", IntegerType, nullable = true)))
   }
 
   test("wider common type for decimal and array") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 9d20eac369152..e5e28b6a74b39 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -1140,6 +1140,23 @@ class CastSuite extends CastSuiteBase {
     assert(Cast.canCast(set.dataType, ArrayType(StringType, false)))
   }
 
+  test("NullTypes should be able to cast to any complex types") {
+    assert(Cast.canCast(ArrayType(NullType, true), ArrayType(IntegerType, true)))
+    assert(Cast.canCast(ArrayType(NullType, false), ArrayType(IntegerType, true)))
+
+    assert(Cast.canCast(
+      MapType(NullType, NullType, true), MapType(IntegerType, IntegerType, true)))
+    assert(Cast.canCast(
+      MapType(NullType, NullType, false), MapType(IntegerType, IntegerType, true)))
+
+    assert(Cast.canCast(
+      StructType(StructField("a", NullType, true) :: Nil),
+      StructType(StructField("a", IntegerType, true) :: Nil)))
+    assert(Cast.canCast(
+      StructType(StructField("a", NullType, false) :: Nil),
+      StructType(StructField("a", IntegerType, true) :: Nil)))
+  }
+
   test("Cast should output null for invalid strings when ANSI is not enabled.") {
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
       checkEvaluation(cast("abdef", DecimalType.USER_DEFAULT), null)

From 1530bc0632297cd8330a5f64d615f8400b3fbe63 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 24 Mar 2020 17:59:43 +0900
Subject: [PATCH 0329/1280] [SPARK-31231][BUILD] Explicitly setuptools version
 as 46.0.0 in pip package test

### What changes were proposed in this pull request?

For a bit of background,
PIP packaging test started to fail (see [this logs](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120218/testReport/)) as of  setuptools 46.1.0 release. In https://github.com/pypa/setuptools/issues/1424, they decided to don't keep the modes in `package_data`.

In PySpark pip installation, we keep the executable scripts in `package_data` https://github.com/apache/spark/blob/fc4e56a54c15e20baf085e6061d3d83f5ce1185d/python/setup.py#L199-L200, and expose their symbolic links as executable scripts.

So, the symbolic links (or copied scripts) executes the scripts copied from `package_data`, which doesn't have the executable permission in its mode:

```
/tmp/tmp.UmkEGNFdKF/3.6/bin/spark-submit: line 27: /tmp/tmp.UmkEGNFdKF/3.6/lib/python3.6/site-packages/pyspark/bin/spark-class: Permission denied
/tmp/tmp.UmkEGNFdKF/3.6/bin/spark-submit: line 27: exec: /tmp/tmp.UmkEGNFdKF/3.6/lib/python3.6/site-packages/pyspark/bin/spark-class: cannot execute: Permission denied
```

The current issue is being tracked at https://github.com/pypa/setuptools/issues/2041

</br>

For what this PR proposes:
It sets the upper bound in PR builder for now to unblock other PRs.  _This PR does not solve the issue yet. I will make a fix after monitoring https://github.com/pypa/setuptools/issues/2041_

### Why are the changes needed?

It currently affects users who uses the latest setuptools. So, _users seem unable to use PySpark with the latest setuptools._ See also https://github.com/pypa/setuptools/issues/2041#issuecomment-602566667

### Does this PR introduce any user-facing change?

It makes CI pass for now. No user-facing change yet.

### How was this patch tested?

Jenkins will test.

Closes #27995 from HyukjinKwon/investigate-pip-packaging.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c181c45f863ba55e15ab9b41f635ffbddad9bac0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/run-pip-tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 470f21e69d46a..dc9582a11f549 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -76,7 +76,7 @@ for python in "${PYTHON_EXECS[@]}"; do
     VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
     rm -rf "$VIRTUALENV_PATH"
     if [ -n "$USE_CONDA" ]; then
-      conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
+      conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools=46.0.0
       source activate "$VIRTUALENV_PATH"
     else
       mkdir -p "$VIRTUALENV_PATH"

From aa20b646318e81f2ea9eb10620f5f70b8fd9b915 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Tue, 24 Mar 2020 21:14:25 +0800
Subject: [PATCH 0330/1280] [SPARK-31221][SQL] Rebase any date-times in
 conversions to/from Java types

In the PR, I propose to apply rebasing for all dates/timestamps in conversion functions `fromJavaDate()`, `toJavaDate()`, `toJavaTimestamp()` and `fromJavaTimestamp()`. The rebasing is performed via building a local date-time in an original calendar, extracting date-time fields from the result, and creating new local date-time in the target calendar.

The changes are need to be compatible with previous Spark version (2.4.5 and earlier versions) not only before the Gregorian cutover date `1582-10-15` but also for dates after the date. For instance, Gregorian calendar implementation in Java 7 `java.util.GregorianCalendar` is not accurate in resolving time zone offsets as Gregorian calendar introduced since Java 8.

Yes, this PR can introduce behavior changes for dates after `1582-10-15`, in particular conversions of zone ids to zone offsets will be much more accurate.

By existing test suites `DateTimeUtilsSuite`, `DateFunctionsSuite`, `DateExpressionsSuite`, `CollectionExpressionsSuite`, `HiveOrcHadoopFsRelationSuite`, `ParquetIOSuite`.

Closes #27980 from MaxGekk/reuse-rebase-funcs-in-java-funcs.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1fd4607d844905f10fe1edbfd6282c25b95b233d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 117 ++++++++++++------
 .../CollectionExpressionsSuite.scala          |  28 +++--
 .../catalyst/util/DateTimeUtilsSuite.scala    |  24 ++--
 .../apache/spark/sql/hive/DaysWritable.scala  |  39 +-----
 4 files changed, 109 insertions(+), 99 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index ba1c5093ee9be..2b646ccc5bad9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -47,13 +47,6 @@ object DateTimeUtils {
   // it's 2440587.5, rounding up to compatible with Hive
   final val JULIAN_DAY_OF_EPOCH = 2440588
 
-  final val GREGORIAN_CUTOVER_DAY = LocalDate.of(1582, 10, 15).toEpochDay
-  final val GREGORIAN_CUTOVER_MICROS = instantToMicros(
-    LocalDateTime.of(1582, 10, 15, 0, 0, 0)
-      .atOffset(ZoneOffset.UTC)
-      .toInstant)
-  final val GREGORIAN_CUTOVER_MILLIS = GREGORIAN_CUTOVER_MICROS / MICROS_PER_MILLIS
-
   final val julianCommonEraStart = Timestamp.valueOf("0001-01-01 00:00:00")
 
   final val TimeZoneGMT = TimeZone.getTimeZone("GMT")
@@ -94,53 +87,103 @@ object DateTimeUtils {
   }
 
   /**
-   * Returns the number of days since epoch from java.sql.Date.
+   * Converts an instance of `java.sql.Date` to a number of days since the epoch
+   * 1970-01-01 via extracting date fields `year`, `month`, `days` from the input,
+   * creating a local date in Proleptic Gregorian calendar from the fields, and
+   * getting the number of days from the resulted local date.
+   *
+   * This approach was taken to have the same local date as the triple of `year`,
+   * `month`, `day` in the original hybrid calendar used by `java.sql.Date` and
+   * Proleptic Gregorian calendar used by Spark since version 3.0.0, see SPARK-26651.
+   *
+   * @param date It represents a specific instant in time based on
+   *             the hybrid calendar which combines Julian and
+   *             Gregorian calendars.
+   * @return The number of days since epoch from java.sql.Date.
    */
   def fromJavaDate(date: Date): SQLDate = {
-    if (date.getTime < GREGORIAN_CUTOVER_MILLIS) {
-      val era = if (date.before(julianCommonEraStart)) 0 else 1
-      val localDate = date.toLocalDate.`with`(ChronoField.ERA, era)
-      localDateToDays(localDate)
-    } else {
-      millisToDays(date.getTime)
-    }
+    val era = if (date.before(julianCommonEraStart)) 0 else 1
+    val localDate = LocalDate
+      .of(date.getYear + 1900, date.getMonth + 1, 1)
+      .`with`(ChronoField.ERA, era)
+      // Add days separately to convert dates existed in Julian calendar but not
+      // in Proleptic Gregorian calendar. For example, 1000-02-29 is valid date
+      // in Julian calendar because 1000 is a leap year but 1000 is not a leap
+      // year in Proleptic Gregorian calendar. And 1000-02-29 doesn't exist in it.
+      .plusDays(date.getDate - 1) // Returns the next valid date after `date.getDate - 1` days
+    localDateToDays(localDate)
   }
 
   /**
-   * Returns a java.sql.Date from number of days since epoch.
+   * The opposite to `fromJavaDate` method which converts a number of days to an
+   * instance of `java.sql.Date`. It builds a local date in Proleptic Gregorian
+   * calendar, extracts date fields `year`, `month`, `day`, and creates a local
+   * date in the hybrid calendar (Julian + Gregorian calendars) from the fields.
+   *
+   * The purpose of the conversion is to have the same local date as the triple
+   * of `year`, `month`, `day` in the original Proleptic Gregorian calendar and
+   * in the target calender.
+   *
+   * @param daysSinceEpoch The number of days since 1970-01-01.
+   * @return A `java.sql.Date` from number of days since epoch.
    */
   def toJavaDate(daysSinceEpoch: SQLDate): Date = {
-    if (daysSinceEpoch < GREGORIAN_CUTOVER_DAY) {
-      Date.valueOf(LocalDate.ofEpochDay(daysSinceEpoch))
-    } else {
-      new Date(daysToMillis(daysSinceEpoch))
-    }
+    val localDate = LocalDate.ofEpochDay(daysSinceEpoch)
+    new Date(localDate.getYear - 1900, localDate.getMonthValue - 1, localDate.getDayOfMonth)
   }
 
   /**
-   * Returns a java.sql.Timestamp from number of micros since epoch.
+   * Converts microseconds since the epoch to an instance of `java.sql.Timestamp`
+   * via creating a local timestamp at the system time zone in Proleptic Gregorian
+   * calendar, extracting date and time fields like `year` and `hours`, and forming
+   * new timestamp in the hybrid calendar from the extracted fields.
+   *
+   * The conversion is based on the JVM system time zone because the `java.sql.Timestamp`
+   * uses the time zone internally.
+   *
+   * The method performs the conversion via local timestamp fields to have the same date-time
+   * representation as `year`, `month`, `day`, ..., `seconds` in the original calendar
+   * and in the target calendar.
+   *
+   * @param us The number of microseconds since 1970-01-01T00:00:00.000000Z.
+   * @return A `java.sql.Timestamp` from number of micros since epoch.
    */
   def toJavaTimestamp(us: SQLTimestamp): Timestamp = {
-    if (us < GREGORIAN_CUTOVER_MICROS) {
-      val ldt = microsToInstant(us).atZone(ZoneId.systemDefault()).toLocalDateTime
-      Timestamp.valueOf(ldt)
-    } else {
-      Timestamp.from(microsToInstant(us))
-    }
+    val ldt = microsToInstant(us).atZone(ZoneId.systemDefault()).toLocalDateTime
+    Timestamp.valueOf(ldt)
   }
 
   /**
-   * Returns the number of micros since epoch from java.sql.Timestamp.
+   * Converts an instance of `java.sql.Timestamp` to the number of microseconds since
+   * 1970-01-01T00:00:00.000000Z. It extracts date-time fields from the input, builds
+   * a local timestamp in Proleptic Gregorian calendar from the fields, and binds
+   * the timestamp to the system time zone. The resulted instant is converted to
+   * microseconds since the epoch.
+   *
+   * The conversion is performed via the system time zone because it is used internally
+   * in `java.sql.Timestamp` while extracting date-time fields.
+   *
+   * The goal of the function is to have the same local date-time in the original calendar
+   * - the hybrid calendar (Julian + Gregorian) and in the target calendar which is
+   * Proleptic Gregorian calendar, see SPARK-26651.
+   *
+   * @param t It represents a specific instant in time based on
+   *          the hybrid calendar which combines Julian and
+   *          Gregorian calendars.
+   * @return The number of micros since epoch from `java.sql.Timestamp`.
    */
   def fromJavaTimestamp(t: Timestamp): SQLTimestamp = {
-    if (t.getTime < GREGORIAN_CUTOVER_MILLIS) {
-      val era = if (t.before(julianCommonEraStart)) 0 else 1
-      val localDateTime = t.toLocalDateTime.`with`(ChronoField.ERA, era)
-      val instant = ZonedDateTime.of(localDateTime, ZoneId.systemDefault()).toInstant
-      instantToMicros(instant)
-    } else {
-      instantToMicros(t.toInstant)
-    }
+    val era = if (t.before(julianCommonEraStart)) 0 else 1
+    val localDateTime = LocalDateTime.of(
+      t.getYear + 1900, t.getMonth + 1, 1,
+      t.getHours, t.getMinutes, t.getSeconds, t.getNanos)
+      .`with`(ChronoField.ERA, era)
+      // Add days separately to convert dates existed in Julian calendar but not
+      // in Proleptic Gregorian calendar. For example, 1000-02-29 is valid date
+      // in Julian calendar because 1000 is a leap year but 1000 is not a leap
+      // year in Proleptic Gregorian calendar. And 1000-02-29 doesn't exist in it.
+      .plusDays(t.getDate - 1) // Returns the next valid date after `date.getDate - 1` days
+    instantToMicros(localDateTime.atZone(ZoneId.systemDefault).toInstant)
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 173f24881531a..bb31a066440c4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -27,8 +27,8 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -845,9 +845,11 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
 
   test("Sequence on DST boundaries") {
     val timeZone = TimeZone.getTimeZone("Europe/Prague")
-    val dstOffset = timeZone.getDSTSavings
 
-    def noDST(t: Timestamp): Timestamp = new Timestamp(t.getTime - dstOffset)
+    def ts(s: String, noDST: Boolean = false): Long = {
+      val offset = if (noDST) timeZone.getDSTSavings else 0
+      DateTimeUtils.fromMillis(Timestamp.valueOf(s).getTime - offset)
+    }
 
     DateTimeTestUtils.withDefaultTimeZone(timeZone) {
       // Spring time change
@@ -856,9 +858,9 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
         Literal(Timestamp.valueOf("2018-03-25 03:30:00")),
         Literal(stringToInterval("interval 30 minutes"))),
         Seq(
-          Timestamp.valueOf("2018-03-25 01:30:00"),
-          Timestamp.valueOf("2018-03-25 03:00:00"),
-          Timestamp.valueOf("2018-03-25 03:30:00")))
+          ts("2018-03-25 01:30:00"),
+          ts("2018-03-25 03:00:00"),
+          ts("2018-03-25 03:30:00")))
 
       // Autumn time change
       checkEvaluation(new Sequence(
@@ -866,13 +868,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
         Literal(Timestamp.valueOf("2018-10-28 03:30:00")),
         Literal(stringToInterval("interval 30 minutes"))),
         Seq(
-          Timestamp.valueOf("2018-10-28 01:30:00"),
-          noDST(Timestamp.valueOf("2018-10-28 02:00:00")),
-          noDST(Timestamp.valueOf("2018-10-28 02:30:00")),
-          Timestamp.valueOf("2018-10-28 02:00:00"),
-          Timestamp.valueOf("2018-10-28 02:30:00"),
-          Timestamp.valueOf("2018-10-28 03:00:00"),
-          Timestamp.valueOf("2018-10-28 03:30:00")))
+          ts("2018-10-28 01:30:00"),
+          ts("2018-10-28 02:00:00", noDST = true),
+          ts("2018-10-28 02:30:00", noDST = true),
+          ts("2018-10-28 02:00:00"),
+          ts("2018-10-28 02:30:00"),
+          ts("2018-10-28 03:00:00"),
+          ts("2018-10-28 03:30:00")))
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 96da4bee7fa61..10b2457947df8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -88,12 +88,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("SPARK-6785: java date conversion before and after epoch") {
-    def format(d: Date): String = {
-      TimestampFormatter("yyyy-MM-dd", defaultTimeZone().toZoneId).format(fromMillis(d.getTime))
-    }
     def checkFromToJavaDate(d1: Date): Unit = {
       val d2 = toJavaDate(fromJavaDate(d1))
-      assert(format(d2) === format(d1))
+      assert(d2.toString === d1.toString)
     }
 
     val df1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
@@ -499,17 +496,16 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
 
-    withDefaultTimeZone(TimeZone.getTimeZone("PST")) {
+    val tz = "America/Los_Angeles"
+    withDefaultTimeZone(TimeZone.getTimeZone(tz)) {
       // Daylight Saving Time
-      test("2016-03-13 01:59:59", "PST", "2016-03-13 09:59:59.0")
-      // 2016-03-13 02:00:00 PST does not exists
-      test("2016-03-13 02:00:00", "PST", "2016-03-13 10:00:00.0")
-      test("2016-03-13 03:00:00", "PST", "2016-03-13 10:00:00.0")
-      test("2016-11-06 00:59:59", "PST", "2016-11-06 07:59:59.0")
-      // 2016-11-06 01:00:00 PST could be 2016-11-06 08:00:00 UTC or 2016-11-06 09:00:00 UTC
-      test("2016-11-06 01:00:00", "PST", "2016-11-06 09:00:00.0")
-      test("2016-11-06 01:59:59", "PST", "2016-11-06 09:59:59.0")
-      test("2016-11-06 02:00:00", "PST", "2016-11-06 10:00:00.0")
+      test("2016-03-13 01:59:59", tz, "2016-03-13 09:59:59.0")
+      test("2016-03-13 02:00:00", tz, "2016-03-13 10:00:00.0")
+      test("2016-03-13 03:00:00", tz, "2016-03-13 10:00:00.0")
+      test("2016-11-06 00:59:59", tz, "2016-11-06 07:59:59.0")
+      test("2016-11-06 01:00:00", tz, "2016-11-06 08:00:00.0")
+      test("2016-11-06 01:59:59", tz, "2016-11-06 08:59:59.0")
+      test("2016-11-06 02:00:00", tz, "2016-11-06 10:00:00.0")
     }
   }
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
index 862be33d7c0da..1eec8d760118f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
@@ -23,7 +23,7 @@ import java.sql.Date
 import org.apache.hadoop.hive.serde2.io.DateWritable
 import org.apache.hadoop.io.WritableUtils
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
 
 /**
  * The class accepts/returns days in Gregorian calendar and rebase them
@@ -41,13 +41,13 @@ private[hive] class DaysWritable(
   extends DateWritable {
 
   def this(gregorianDays: Int) =
-    this(gregorianDays, DaysWritable.rebaseGregorianToJulianDays(gregorianDays))
+    this(gregorianDays, rebaseGregorianToJulianDays(gregorianDays))
   def this(dateWritable: DateWritable) = {
     this(
       gregorianDays = dateWritable match {
         case daysWritable: DaysWritable => daysWritable.gregorianDays
         case dateWritable: DateWritable =>
-        DaysWritable.rebaseJulianToGregorianDays(dateWritable.getDays)
+        rebaseJulianToGregorianDays(dateWritable.getDays)
       },
       julianDays = dateWritable.getDays)
   }
@@ -63,37 +63,6 @@ private[hive] class DaysWritable(
   @throws[IOException]
   override def readFields(in: DataInput): Unit = {
     julianDays = WritableUtils.readVInt(in)
-    gregorianDays = DaysWritable.rebaseJulianToGregorianDays(julianDays)
+    gregorianDays = rebaseJulianToGregorianDays(julianDays)
   }
 }
-
-private[hive] object DaysWritable {
-  // Rebasing days since the epoch to store the same number of days
-  // as by Spark 2.4 and earlier versions. Spark 3.0 switched to
-  // Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
-  // this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
-  // Julian calendar for dates before 1582-10-15. So, the same local date may
-  // be mapped to different number of days since the epoch in different calendars.
-  // For example:
-  // Proleptic Gregorian calendar: 1582-01-01 -> -141714
-  // Julian calendar: 1582-01-01 -> -141704
-  // The code below converts -141714 to -141704.
-  def rebaseGregorianToJulianDays(daysSinceEpoch: Int): Int = {
-    if (daysSinceEpoch < DateTimeUtils.GREGORIAN_CUTOVER_DAY) {
-      DateTimeUtils.rebaseGregorianToJulianDays(daysSinceEpoch)
-    } else {
-      daysSinceEpoch
-    }
-  }
-
-  def rebaseJulianToGregorianDays(daysSinceEpoch: Int): Int = {
-    if (daysSinceEpoch < JULIAN_CUTOVER_DAY) {
-      DateTimeUtils.rebaseJulianToGregorianDays(daysSinceEpoch)
-    } else {
-      daysSinceEpoch
-    }
-  }
-
-  final val JULIAN_CUTOVER_DAY =
-    rebaseGregorianToJulianDays(DateTimeUtils.GREGORIAN_CUTOVER_DAY.toInt)
-}

From a934142f2405e085d0e1ef76b1bc6fbc6e3059c1 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 24 Mar 2020 23:03:57 +0800
Subject: [PATCH 0331/1280] [SPARK-30127][SQL] Support case class parameter for
 typed Scala UDF

### What changes were proposed in this pull request?

To support  case class parameter for typed Scala UDF, e.g.

```
case class TestData(key: Int, value: String)
val f = (d: TestData) => d.key * d.value.toInt
val myUdf = udf(f)
val df = Seq(("data", TestData(50, "2"))).toDF("col1", "col2")
checkAnswer(df.select(myUdf(Column("col2"))), Row(100) :: Nil)
```

### Why are the changes needed?

Currently, Spark UDF can only work on data types like java.lang.String, o.a.s.sql.Row, Seq[_], etc. This is inconvenient if user want to apply an operation on one column, and the column is struct type. You must access data from a Row object, instead of domain object like Dataset operations. It will be great if UDF can work on types that are supported by Dataset, e.g. case class.

And here's benchmark result of using case class comparing to row:

```scala

// case class:  58ms 65ms 59ms 64ms 61ms
// row:         59ms 64ms 73ms 84ms 69ms
val f1 = (d: TestData) => s"${d.key}, ${d.value}"
val f2 = (r: Row) => s"${r.getInt(0)}, ${r.getString(1)}"
val udf1 = udf(f1)
// set spark.sql.legacy.allowUntypedScalaUDF=true
val udf2 = udf(f2, StringType)

val df = spark.range(100000).selectExpr("cast (id as int) as id")
    .select(struct('id, lit("str")).as("col"))
df.cache().collect()

// warmup to exclude some extra influence
df.select(udf1('col)).write.mode(SaveMode.Overwrite).format("noop").save()
df.select(udf2('col)).write.mode(SaveMode.Overwrite).format("noop").save()

start = System.currentTimeMillis()
df.select(udf1('col)).write.mode(SaveMode.Overwrite).format("noop").save()
println(System.currentTimeMillis() - start)

start = System.currentTimeMillis()
df.select(udf2('col)).write.mode(SaveMode.Overwrite).format("noop").save()
println(System.currentTimeMillis() - start)

```

### Does this PR introduce any user-facing change?

Yes. User now could be able to use typed Scala UDF with case class as input parameter.

### How was this patch tested?

Added unit tests.

Closes #27937 from Ngone51/udf_caseclass_support.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f6ff7d0cf8c0e562f3b086180d5418e6996055bb)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |   8 +-
 .../sql/catalyst/expressions/ScalaUDF.scala   | 595 ++++++++++--------
 .../sql/catalyst/analysis/AnalysisSuite.scala |  18 +-
 .../catalyst/expressions/ScalaUDFSuite.scala  |  18 +-
 .../optimizer/EliminateSortsSuite.scala       |   4 +-
 .../sql/catalyst/trees/TreeNodeSuite.scala    |   4 +-
 .../apache/spark/sql/UDFRegistration.scala    | 146 ++---
 .../datasources/FileFormatDataWriter.scala    |   3 +-
 .../sql/expressions/UserDefinedFunction.scala |  11 +-
 .../org/apache/spark/sql/functions.scala      |  76 +--
 .../spark/sql/IntegratedUDFTestUtils.scala    |   2 +-
 .../scala/org/apache/spark/sql/UDFSuite.scala |  44 +-
 12 files changed, 519 insertions(+), 410 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 975f02332512f..67f6f49385007 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2703,13 +2703,13 @@ class Analyzer(
 
       case p => p transformExpressionsUp {
 
-        case udf @ ScalaUDF(_, _, inputs, inputPrimitives, _, _, _, _)
-            if inputPrimitives.contains(true) =>
+        case udf @ ScalaUDF(_, _, inputs, _, _, _, _)
+            if udf.inputPrimitives.contains(true) =>
           // Otherwise, add special handling of null for fields that can't accept null.
           // The result of operations like this, when passed null, is generally to return null.
-          assert(inputPrimitives.length == inputs.length)
+          assert(udf.inputPrimitives.length == inputs.length)
 
-          val inputPrimitivesPair = inputPrimitives.zip(inputs)
+          val inputPrimitivesPair = udf.inputPrimitives.zip(inputs)
           val inputNullCheck = inputPrimitivesPair.collect {
             case (isPrimitive, input) if isPrimitive && input.nullable =>
               IsNull(input)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 10f8ec9617d1b..1ac7ca676a876 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -17,11 +17,14 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import scala.collection.mutable
+
 import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.types.{AbstractDataType, DataType}
+import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType}
 
 /**
  * User-defined function.
@@ -31,14 +34,9 @@ import org.apache.spark.sql.types.{AbstractDataType, DataType}
  *                  null. Use boxed type or [[Option]] if you wanna do the null-handling yourself.
  * @param dataType  Return type of function.
  * @param children  The input expressions of this UDF.
- * @param inputPrimitives The analyzer should be aware of Scala primitive types so as to make the
- *                        UDF return null if there is any null input value of these types. On the
- *                        other hand, Java UDFs can only have boxed types, thus this parameter will
- *                        always be all false.
- * @param inputTypes  The expected input types of this UDF, used to perform type coercion. If we do
- *                    not want to perform coercion, simply use "Nil". Note that it would've been
- *                    better to use Option of Seq[DataType] so we can use "None" as the case for no
- *                    type coercion. However, that would require more refactoring of the codebase.
+ * @param inputEncoders ExpressionEncoder for each input parameters. For a input parameter which
+ *                      serialized as struct will use encoder instead of CatalystTypeConverters to
+ *                      convert internal value to Scala value.
  * @param udfName  The user-specified name of this UDF.
  * @param nullable  True if the UDF can return null value.
  * @param udfDeterministic  True if the UDF is deterministic. Deterministic UDF returns same result
@@ -48,8 +46,7 @@ case class ScalaUDF(
     function: AnyRef,
     dataType: DataType,
     children: Seq[Expression],
-    inputPrimitives: Seq[Boolean],
-    inputTypes: Seq[AbstractDataType] = Nil,
+    inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
     udfName: Option[String] = None,
     nullable: Boolean = true,
     udfDeterministic: Boolean = true)
@@ -59,6 +56,68 @@ case class ScalaUDF(
 
   override def toString: String = s"${udfName.getOrElse("UDF")}(${children.mkString(", ")})"
 
+  /**
+   * The analyzer should be aware of Scala primitive types so as to make the
+   * UDF return null if there is any null input value of these types. On the
+   * other hand, Java UDFs can only have boxed types, thus this will return
+   * Nil(has same effect with all false) and analyzer will skip null-handling
+   * on them.
+   */
+  def inputPrimitives: Seq[Boolean] = {
+    inputEncoders.map { encoderOpt =>
+      // It's possible that some of the inputs don't have a specific encoder(e.g. `Any`)
+      if (encoderOpt.isDefined) {
+        val encoder = encoderOpt.get
+        if (encoder.isSerializedAsStruct) {
+          // struct type is not primitive
+          false
+        } else {
+          // `nullable` is false iff the type is primitive
+          !encoder.schema.head.nullable
+        }
+      } else {
+        // Any type is not primitive
+        false
+      }
+    }
+  }
+
+  /**
+   * The expected input types of this UDF, used to perform type coercion. If we do
+   * not want to perform coercion, simply use "Nil". Note that it would've been
+   * better to use Option of Seq[DataType] so we can use "None" as the case for no
+   * type coercion. However, that would require more refactoring of the codebase.
+   */
+  def inputTypes: Seq[AbstractDataType] = {
+    inputEncoders.map { encoderOpt =>
+      if (encoderOpt.isDefined) {
+        val encoder = encoderOpt.get
+        if (encoder.isSerializedAsStruct) {
+          encoder.schema
+        } else {
+          encoder.schema.head.dataType
+        }
+      } else {
+        AnyDataType
+      }
+    }
+  }
+
+  private def createToScalaConverter(i: Int, dataType: DataType): Any => Any = {
+    if (inputEncoders.isEmpty) {
+      // for untyped Scala UDF
+      CatalystTypeConverters.createToScalaConverter(dataType)
+    } else {
+      val encoder = inputEncoders(i)
+      if (encoder.isDefined && encoder.get.isSerializedAsStructForTopLevel) {
+        val enc = encoder.get.resolveAndBind()
+        row: Any => enc.fromRow(row.asInstanceOf[InternalRow])
+      } else {
+        CatalystTypeConverters.createToScalaConverter(dataType)
+      }
+    }
+  }
+
   // scalastyle:off line.size.limit
 
   /** This method has been generated by this script
@@ -66,7 +125,7 @@ case class ScalaUDF(
     (1 to 22).map { x =>
       val anys = (1 to x).map(x => "Any").reduce(_ + ", " + _)
       val childs = (0 to x - 1).map(x => s"val child$x = children($x)").reduce(_ + "\n  " + _)
-      val converters = (0 to x - 1).map(x => s"lazy val converter$x = CatalystTypeConverters.createToScalaConverter(child$x.dataType)").reduce(_ + "\n  " + _)
+      val converters = (0 to x - 1).map(x => s"lazy val converter$x = createToScalaConverter($x, child$x.dataType)").reduce(_ + "\n  " + _)
       val evals = (0 to x - 1).map(x => s"converter$x(child$x.eval(input))").reduce(_ + ",\n      " + _)
 
       s"""case $x =>
@@ -91,7 +150,7 @@ case class ScalaUDF(
     case 1 =>
       val func = function.asInstanceOf[(Any) => Any]
       val child0 = children(0)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)))
@@ -101,8 +160,8 @@ case class ScalaUDF(
       val func = function.asInstanceOf[(Any, Any) => Any]
       val child0 = children(0)
       val child1 = children(1)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -114,9 +173,9 @@ case class ScalaUDF(
       val child0 = children(0)
       val child1 = children(1)
       val child2 = children(2)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -130,10 +189,10 @@ case class ScalaUDF(
       val child1 = children(1)
       val child2 = children(2)
       val child3 = children(3)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -149,11 +208,11 @@ case class ScalaUDF(
       val child2 = children(2)
       val child3 = children(3)
       val child4 = children(4)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -171,12 +230,12 @@ case class ScalaUDF(
       val child3 = children(3)
       val child4 = children(4)
       val child5 = children(5)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -196,13 +255,13 @@ case class ScalaUDF(
       val child4 = children(4)
       val child5 = children(5)
       val child6 = children(6)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -224,14 +283,14 @@ case class ScalaUDF(
       val child5 = children(5)
       val child6 = children(6)
       val child7 = children(7)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -255,15 +314,15 @@ case class ScalaUDF(
       val child6 = children(6)
       val child7 = children(7)
       val child8 = children(8)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -289,16 +348,16 @@ case class ScalaUDF(
       val child7 = children(7)
       val child8 = children(8)
       val child9 = children(9)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -326,17 +385,17 @@ case class ScalaUDF(
       val child8 = children(8)
       val child9 = children(9)
       val child10 = children(10)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -366,18 +425,18 @@ case class ScalaUDF(
       val child9 = children(9)
       val child10 = children(10)
       val child11 = children(11)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -409,19 +468,19 @@ case class ScalaUDF(
       val child10 = children(10)
       val child11 = children(11)
       val child12 = children(12)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -455,20 +514,20 @@ case class ScalaUDF(
       val child11 = children(11)
       val child12 = children(12)
       val child13 = children(13)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -504,21 +563,21 @@ case class ScalaUDF(
       val child12 = children(12)
       val child13 = children(13)
       val child14 = children(14)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -556,22 +615,22 @@ case class ScalaUDF(
       val child13 = children(13)
       val child14 = children(14)
       val child15 = children(15)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -611,23 +670,23 @@ case class ScalaUDF(
       val child14 = children(14)
       val child15 = children(15)
       val child16 = children(16)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -669,24 +728,24 @@ case class ScalaUDF(
       val child15 = children(15)
       val child16 = children(16)
       val child17 = children(17)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -730,25 +789,25 @@ case class ScalaUDF(
       val child16 = children(16)
       val child17 = children(17)
       val child18 = children(18)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -794,26 +853,26 @@ case class ScalaUDF(
       val child17 = children(17)
       val child18 = children(18)
       val child19 = children(19)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
-      lazy val converter19 = CatalystTypeConverters.createToScalaConverter(child19.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
+      lazy val converter19 = createToScalaConverter(19, child19.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -861,27 +920,27 @@ case class ScalaUDF(
       val child18 = children(18)
       val child19 = children(19)
       val child20 = children(20)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
-      lazy val converter19 = CatalystTypeConverters.createToScalaConverter(child19.dataType)
-      lazy val converter20 = CatalystTypeConverters.createToScalaConverter(child20.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
+      lazy val converter19 = createToScalaConverter(19, child19.dataType)
+      lazy val converter20 = createToScalaConverter(20, child20.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -931,28 +990,28 @@ case class ScalaUDF(
       val child19 = children(19)
       val child20 = children(20)
       val child21 = children(21)
-      lazy val converter0 = CatalystTypeConverters.createToScalaConverter(child0.dataType)
-      lazy val converter1 = CatalystTypeConverters.createToScalaConverter(child1.dataType)
-      lazy val converter2 = CatalystTypeConverters.createToScalaConverter(child2.dataType)
-      lazy val converter3 = CatalystTypeConverters.createToScalaConverter(child3.dataType)
-      lazy val converter4 = CatalystTypeConverters.createToScalaConverter(child4.dataType)
-      lazy val converter5 = CatalystTypeConverters.createToScalaConverter(child5.dataType)
-      lazy val converter6 = CatalystTypeConverters.createToScalaConverter(child6.dataType)
-      lazy val converter7 = CatalystTypeConverters.createToScalaConverter(child7.dataType)
-      lazy val converter8 = CatalystTypeConverters.createToScalaConverter(child8.dataType)
-      lazy val converter9 = CatalystTypeConverters.createToScalaConverter(child9.dataType)
-      lazy val converter10 = CatalystTypeConverters.createToScalaConverter(child10.dataType)
-      lazy val converter11 = CatalystTypeConverters.createToScalaConverter(child11.dataType)
-      lazy val converter12 = CatalystTypeConverters.createToScalaConverter(child12.dataType)
-      lazy val converter13 = CatalystTypeConverters.createToScalaConverter(child13.dataType)
-      lazy val converter14 = CatalystTypeConverters.createToScalaConverter(child14.dataType)
-      lazy val converter15 = CatalystTypeConverters.createToScalaConverter(child15.dataType)
-      lazy val converter16 = CatalystTypeConverters.createToScalaConverter(child16.dataType)
-      lazy val converter17 = CatalystTypeConverters.createToScalaConverter(child17.dataType)
-      lazy val converter18 = CatalystTypeConverters.createToScalaConverter(child18.dataType)
-      lazy val converter19 = CatalystTypeConverters.createToScalaConverter(child19.dataType)
-      lazy val converter20 = CatalystTypeConverters.createToScalaConverter(child20.dataType)
-      lazy val converter21 = CatalystTypeConverters.createToScalaConverter(child21.dataType)
+      lazy val converter0 = createToScalaConverter(0, child0.dataType)
+      lazy val converter1 = createToScalaConverter(1, child1.dataType)
+      lazy val converter2 = createToScalaConverter(2, child2.dataType)
+      lazy val converter3 = createToScalaConverter(3, child3.dataType)
+      lazy val converter4 = createToScalaConverter(4, child4.dataType)
+      lazy val converter5 = createToScalaConverter(5, child5.dataType)
+      lazy val converter6 = createToScalaConverter(6, child6.dataType)
+      lazy val converter7 = createToScalaConverter(7, child7.dataType)
+      lazy val converter8 = createToScalaConverter(8, child8.dataType)
+      lazy val converter9 = createToScalaConverter(9, child9.dataType)
+      lazy val converter10 = createToScalaConverter(10, child10.dataType)
+      lazy val converter11 = createToScalaConverter(11, child11.dataType)
+      lazy val converter12 = createToScalaConverter(12, child12.dataType)
+      lazy val converter13 = createToScalaConverter(13, child13.dataType)
+      lazy val converter14 = createToScalaConverter(14, child14.dataType)
+      lazy val converter15 = createToScalaConverter(15, child15.dataType)
+      lazy val converter16 = createToScalaConverter(16, child16.dataType)
+      lazy val converter17 = createToScalaConverter(17, child17.dataType)
+      lazy val converter18 = createToScalaConverter(18, child18.dataType)
+      lazy val converter19 = createToScalaConverter(19, child19.dataType)
+      lazy val converter20 = createToScalaConverter(20, child20.dataType)
+      lazy val converter21 = createToScalaConverter(21, child21.dataType)
       (input: InternalRow) => {
         func(
           converter0(child0.eval(input)),
@@ -987,8 +1046,8 @@ case class ScalaUDF(
     val converterClassName = classOf[Any => Any].getName
 
     // The type converters for inputs and the result.
-    val converters: Array[Any => Any] = children.map { c =>
-      CatalystTypeConverters.createToScalaConverter(c.dataType)
+    val converters: Array[Any => Any] = children.zipWithIndex.map { case (c, i) =>
+      createToScalaConverter(i, c.dataType)
     }.toArray :+ CatalystTypeConverters.createToCatalystConverter(dataType)
     val convertersTerm = ctx.addReferenceObj("converters", converters, s"$converterClassName[]")
     val errorMsgTerm = ctx.addReferenceObj("errMsg", udfErrorMessage)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 8451b9b50eff3..02472e153b09e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count, Sum}
@@ -326,20 +327,21 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     }
 
     // non-primitive parameters do not need special null handling
-    val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil, false :: Nil)
+    val udf1 = ScalaUDF((s: String) => "x", StringType, string :: Nil,
+      Option(ExpressionEncoder[String]()) :: Nil)
     val expected1 = udf1
     checkUDF(udf1, expected1)
 
     // only primitive parameter needs special null handling
     val udf2 = ScalaUDF((s: String, d: Double) => "x", StringType, string :: double :: Nil,
-      false :: true :: Nil)
+      Option(ExpressionEncoder[String]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
     val expected2 =
       If(IsNull(double), nullResult, udf2.copy(children = string :: KnownNotNull(double) :: Nil))
     checkUDF(udf2, expected2)
 
     // special null handling should apply to all primitive parameters
     val udf3 = ScalaUDF((s: Short, d: Double) => "x", StringType, short :: double :: Nil,
-      true :: true :: Nil)
+      Option(ExpressionEncoder[Short]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
     val expected3 = If(
       IsNull(short) || IsNull(double),
       nullResult,
@@ -351,7 +353,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       (s: Short, d: Double) => "x",
       StringType,
       short :: nonNullableDouble :: Nil,
-      true :: true :: Nil)
+      Option(ExpressionEncoder[Short]()) :: Option(ExpressionEncoder[Double]()) :: Nil)
     val expected4 = If(
       IsNull(short),
       nullResult,
@@ -362,8 +364,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
   test("SPARK-24891 Fix HandleNullInputsForUDF rule") {
     val a = testRelation.output(0)
     val func = (x: Int, y: Int) => x + y
-    val udf1 = ScalaUDF(func, IntegerType, a :: a :: Nil, false :: false :: Nil)
-    val udf2 = ScalaUDF(func, IntegerType, a :: udf1 :: Nil, false :: false :: Nil)
+    val udf1 = ScalaUDF(func, IntegerType, a :: a :: Nil,
+      Option(ExpressionEncoder[java.lang.Integer]()) ::
+        Option(ExpressionEncoder[java.lang.Integer]()) :: Nil)
+    val udf2 = ScalaUDF(func, IntegerType, a :: udf1 :: Nil,
+      Option(ExpressionEncoder[java.lang.Integer]()) ::
+        Option(ExpressionEncoder[java.lang.Integer]()) :: Nil)
     val plan = Project(Alias(udf2, "")() :: Nil, testRelation)
     comparePlans(plan.analyze, plan.analyze.analyze)
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
index c5ffc381b58e2..836b2eaa642a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import java.util.Locale
 
 import org.apache.spark.{SparkException, SparkFunSuite}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType}
@@ -27,10 +28,12 @@ import org.apache.spark.sql.types.{DecimalType, IntegerType, StringType}
 class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("basic") {
-    val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil, true :: Nil)
+    val intUdf = ScalaUDF((i: Int) => i + 1, IntegerType, Literal(1) :: Nil,
+      Option(ExpressionEncoder[Int]()) :: Nil)
     checkEvaluation(intUdf, 2)
 
-    val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil, false :: Nil)
+    val stringUdf = ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil,
+      Option(ExpressionEncoder[String]()) :: Nil)
     checkEvaluation(stringUdf, "ax")
   }
 
@@ -39,7 +42,7 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       (s: String) => s.toLowerCase(Locale.ROOT),
       StringType,
       Literal.create(null, StringType) :: Nil,
-      false :: Nil)
+      Option(ExpressionEncoder[String]()) :: Nil)
 
     val e1 = intercept[SparkException](udf.eval())
     assert(e1.getMessage.contains("Failed to execute user defined function"))
@@ -52,7 +55,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("SPARK-22695: ScalaUDF should not use global variables") {
     val ctx = new CodegenContext
-    ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil, false :: Nil).genCode(ctx)
+    ScalaUDF((s: String) => s + "x", StringType, Literal("a") :: Nil,
+      Option(ExpressionEncoder[String]()) :: Nil).genCode(ctx)
     assert(ctx.inlinedMutableStates.isEmpty)
   }
 
@@ -61,7 +65,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       val udf = ScalaUDF(
         (a: java.math.BigDecimal) => a.multiply(new java.math.BigDecimal(100)),
         DecimalType.SYSTEM_DEFAULT,
-        Literal(BigDecimal("12345678901234567890.123")) :: Nil, false :: Nil)
+        Literal(BigDecimal("12345678901234567890.123")) :: Nil,
+        Option(ExpressionEncoder[java.math.BigDecimal]()) :: Nil)
       val e1 = intercept[ArithmeticException](udf.eval())
       assert(e1.getMessage.contains("cannot be represented as Decimal"))
       val e2 = intercept[SparkException] {
@@ -73,7 +78,8 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper {
       val udf = ScalaUDF(
         (a: java.math.BigDecimal) => a.multiply(new java.math.BigDecimal(100)),
         DecimalType.SYSTEM_DEFAULT,
-        Literal(BigDecimal("12345678901234567890.123")) :: Nil, false :: Nil)
+        Literal(BigDecimal("12345678901234567890.123")) :: Nil,
+        Option(ExpressionEncoder[java.math.BigDecimal]()) :: Nil)
       checkEvaluation(udf, null)
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
index d9a6fbf81de91..d7eb048ba8705 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.analysis.{Analyzer, EmptyFunctionRegistry}
 import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
@@ -244,7 +245,8 @@ class EliminateSortsSuite extends PlanTest {
   }
 
   test("should not remove orderBy in groupBy clause with ScalaUDF as aggs") {
-    val scalaUdf = ScalaUDF((s: Int) => s, IntegerType, 'a :: Nil, true :: Nil)
+    val scalaUdf = ScalaUDF((s: Int) => s, IntegerType, 'a :: Nil,
+      Option(ExpressionEncoder[Int]()) :: Nil)
     val projectPlan = testRelation.select('a, 'b)
     val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc)
     val groupByPlan = orderByPlan.groupBy('a)(scalaUdf)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index e72b2e9b1b214..f5259706325eb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.{AliasIdentifier, FunctionIdentifier, InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.dsl.expressions.DslString
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.plans.{LeftOuter, NaturalJoin, SQLHelper}
@@ -594,7 +595,8 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("toJSON should not throws java.lang.StackOverflowError") {
-    val udf = ScalaUDF(SelfReferenceUDF(), BooleanType, Seq("col1".attr), false :: Nil)
+    val udf = ScalaUDF(SelfReferenceUDF(), BooleanType, Seq("col1".attr),
+      Option(ExpressionEncoder[String]()) :: Nil)
     // Should not throw java.lang.StackOverflowError
     udf.toJSON
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
index 0f08e10c00d22..ced4af46c3f30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala
@@ -125,7 +125,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
     (0 to 22).foreach { x =>
       val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
       val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
-      val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"Try(ScalaReflection.schemaFor[A$i]).toOption :: $s"})
+      val inputEncoders = (1 to x).foldRight("Nil")((i, s) => {s"Try(ExpressionEncoder[A$i]()).toOption :: $s"})
       println(s"""
         |/**
         | * Registers a deterministic Scala closure of $x arguments as user-defined function (UDF).
@@ -134,8 +134,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
         | */
         |def register[$typeTags](name: String, func: Function$x[$types]): UserDefinedFunction = {
         |  val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-        |  val inputSchemas: Seq[Option[ScalaReflection.Schema]] = $inputSchemas
-        |  val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+        |  val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = $inputEncoders
+        |  val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
         |  val finalUdf = if (nullable) udf else udf.asNonNullable()
         |  def builder(e: Seq[Expression]) = if (e.length == $x) {
         |    finalUdf.createScalaUDF(e)
@@ -163,7 +163,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
         |def register(name: String, f: UDF$i[$extTypeArgs], returnType: DataType): Unit = {
         |  val func = $funcCall
         |  def builder(e: Seq[Expression]) = if (e.length == $i) {
-        |    ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+        |    ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
         |  } else {
         |    throw new AnalysisException("Invalid number of arguments for function " + name +
         |      ". Expected: $i; Found: " + e.length)
@@ -180,8 +180,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag](name: String, func: Function0[RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 0) {
       finalUdf.createScalaUDF(e)
@@ -200,8 +200,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 1) {
       finalUdf.createScalaUDF(e)
@@ -220,8 +220,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func: Function2[A1, A2, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 2) {
       finalUdf.createScalaUDF(e)
@@ -240,8 +240,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name: String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 3) {
       finalUdf.createScalaUDF(e)
@@ -260,8 +260,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 4) {
       finalUdf.createScalaUDF(e)
@@ -280,8 +280,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 5) {
       finalUdf.createScalaUDF(e)
@@ -300,8 +300,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 6) {
       finalUdf.createScalaUDF(e)
@@ -320,8 +320,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 7) {
       finalUdf.createScalaUDF(e)
@@ -340,8 +340,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String, func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 8) {
       finalUdf.createScalaUDF(e)
@@ -360,8 +360,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name: String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 9) {
       finalUdf.createScalaUDF(e)
@@ -380,8 +380,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 10) {
       finalUdf.createScalaUDF(e)
@@ -400,8 +400,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 11) {
       finalUdf.createScalaUDF(e)
@@ -420,8 +420,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 12) {
       finalUdf.createScalaUDF(e)
@@ -440,8 +440,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func: Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 13) {
       finalUdf.createScalaUDF(e)
@@ -460,8 +460,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String, func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 14) {
       finalUdf.createScalaUDF(e)
@@ -480,8 +480,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 15) {
       finalUdf.createScalaUDF(e)
@@ -500,8 +500,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 16) {
       finalUdf.createScalaUDF(e)
@@ -520,8 +520,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 17) {
       finalUdf.createScalaUDF(e)
@@ -540,8 +540,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 18) {
       finalUdf.createScalaUDF(e)
@@ -560,8 +560,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func: Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 19) {
       finalUdf.createScalaUDF(e)
@@ -580,8 +580,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name: String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 20) {
       finalUdf.createScalaUDF(e)
@@ -600,8 +600,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Try(ScalaReflection.schemaFor[A21]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Try(ExpressionEncoder[A21]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 21) {
       finalUdf.createScalaUDF(e)
@@ -620,8 +620,8 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
    */
   def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag, A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21: TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas: Seq[Option[ScalaReflection.Schema]] = Try(ScalaReflection.schemaFor[A1]).toOption :: Try(ScalaReflection.schemaFor[A2]).toOption :: Try(ScalaReflection.schemaFor[A3]).toOption :: Try(ScalaReflection.schemaFor[A4]).toOption :: Try(ScalaReflection.schemaFor[A5]).toOption :: Try(ScalaReflection.schemaFor[A6]).toOption :: Try(ScalaReflection.schemaFor[A7]).toOption :: Try(ScalaReflection.schemaFor[A8]).toOption :: Try(ScalaReflection.schemaFor[A9]).toOption :: Try(ScalaReflection.schemaFor[A10]).toOption :: Try(ScalaReflection.schemaFor[A11]).toOption :: Try(ScalaReflection.schemaFor[A12]).toOption :: Try(ScalaReflection.schemaFor[A13]).toOption :: Try(ScalaReflection.schemaFor[A14]).toOption :: Try(ScalaReflection.schemaFor[A15]).toOption :: Try(ScalaReflection.schemaFor[A16]).toOption :: Try(ScalaReflection.schemaFor[A17]).toOption :: Try(ScalaReflection.schemaFor[A18]).toOption :: Try(ScalaReflection.schemaFor[A19]).toOption :: Try(ScalaReflection.schemaFor[A20]).toOption :: Try(ScalaReflection.schemaFor[A21]).toOption :: Try(ScalaReflection.schemaFor[A22]).toOption :: Nil
-    val udf = SparkUserDefinedFunction(func, dataType, inputSchemas).withName(name)
+    val inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Try(ExpressionEncoder[A11]()).toOption :: Try(ExpressionEncoder[A12]()).toOption :: Try(ExpressionEncoder[A13]()).toOption :: Try(ExpressionEncoder[A14]()).toOption :: Try(ExpressionEncoder[A15]()).toOption :: Try(ExpressionEncoder[A16]()).toOption :: Try(ExpressionEncoder[A17]()).toOption :: Try(ExpressionEncoder[A18]()).toOption :: Try(ExpressionEncoder[A19]()).toOption :: Try(ExpressionEncoder[A20]()).toOption :: Try(ExpressionEncoder[A21]()).toOption :: Try(ExpressionEncoder[A22]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(func, dataType, inputEncoders).withName(name)
     val finalUdf = if (nullable) udf else udf.asNonNullable()
     def builder(e: Seq[Expression]) = if (e.length == 22) {
       finalUdf.createScalaUDF(e)
@@ -731,7 +731,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF0[_], returnType: DataType): Unit = {
     val func = () => f.asInstanceOf[UDF0[Any]].call()
     def builder(e: Seq[Expression]) = if (e.length == 0) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 0; Found: " + e.length)
@@ -746,7 +746,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF1[_, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
     def builder(e: Seq[Expression]) = if (e.length == 1) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 1; Found: " + e.length)
@@ -761,7 +761,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF2[_, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 2) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 2; Found: " + e.length)
@@ -776,7 +776,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF3[_, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 3) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 3; Found: " + e.length)
@@ -791,7 +791,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF4[_, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 4) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 4; Found: " + e.length)
@@ -806,7 +806,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF5[_, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 5) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 5; Found: " + e.length)
@@ -821,7 +821,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF6[_, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 6) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 6; Found: " + e.length)
@@ -836,7 +836,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 7) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 7; Found: " + e.length)
@@ -851,7 +851,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 8) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 8; Found: " + e.length)
@@ -866,7 +866,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 9) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 9; Found: " + e.length)
@@ -881,7 +881,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 10) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 10; Found: " + e.length)
@@ -896,7 +896,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF11[_, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF11[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 11) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 11; Found: " + e.length)
@@ -911,7 +911,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF12[_, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF12[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 12) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 12; Found: " + e.length)
@@ -926,7 +926,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF13[_, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF13[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 13) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 13; Found: " + e.length)
@@ -941,7 +941,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF14[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF14[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 14) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 14; Found: " + e.length)
@@ -956,7 +956,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF15[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF15[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 15) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 15; Found: " + e.length)
@@ -971,7 +971,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF16[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF16[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 16) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 16; Found: " + e.length)
@@ -986,7 +986,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF17[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF17[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 17) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 17; Found: " + e.length)
@@ -1001,7 +1001,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF18[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF18[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 18) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 18; Found: " + e.length)
@@ -1016,7 +1016,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF19[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF19[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 19) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 19; Found: " + e.length)
@@ -1031,7 +1031,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF20[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF20[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 20) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 20; Found: " + e.length)
@@ -1046,7 +1046,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF21[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF21[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 21) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 21; Found: " + e.length)
@@ -1061,7 +1061,7 @@ class UDFRegistration private[sql] (functionRegistry: FunctionRegistry) extends
   def register(name: String, f: UDF22[_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _], returnType: DataType): Unit = {
     val func = f.asInstanceOf[UDF22[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
     def builder(e: Seq[Expression]) = if (e.length == 22) {
-      ScalaUDF(func, returnType, e, e.map(_ => false), udfName = Some(name))
+      ScalaUDF(func, returnType, e, Nil, udfName = Some(name))
     } else {
       throw new AnalysisException("Invalid number of arguments for function " + name +
         ". Expected: 22; Found: " + e.length)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
index 50c4f6cd57a96..edb49d3f90ca3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatDataWriter.scala
@@ -182,8 +182,7 @@ class DynamicPartitionDataWriter(
       val partitionName = ScalaUDF(
         ExternalCatalogUtils.getPartitionPathString _,
         StringType,
-        Seq(Literal(c.name), Cast(c, StringType, Option(description.timeZoneId))),
-        Seq(false, false))
+        Seq(Literal(c.name), Cast(c, StringType, Option(description.timeZoneId))))
       if (i == 0) Seq(partitionName) else Seq(Literal(Path.SEPARATOR), partitionName)
     })
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index c50168cf7ac13..2ef6e3d291cef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -93,7 +93,7 @@ sealed abstract class UserDefinedFunction {
 private[spark] case class SparkUserDefinedFunction(
     f: AnyRef,
     dataType: DataType,
-    inputSchemas: Seq[Option[ScalaReflection.Schema]],
+    inputEncoders: Seq[Option[ExpressionEncoder[_]]] = Nil,
     name: Option[String] = None,
     nullable: Boolean = true,
     deterministic: Boolean = true) extends UserDefinedFunction {
@@ -104,18 +104,11 @@ private[spark] case class SparkUserDefinedFunction(
   }
 
   private[sql] def createScalaUDF(exprs: Seq[Expression]): ScalaUDF = {
-    // It's possible that some of the inputs don't have a specific type(e.g. `Any`),  skip type
-    // check.
-    val inputTypes = inputSchemas.map(_.map(_.dataType).getOrElse(AnyDataType))
-    // `ScalaReflection.Schema.nullable` is false iff the type is primitive. Also `Any` is not
-    // primitive.
-    val inputsPrimitive = inputSchemas.map(_.map(!_.nullable).getOrElse(false))
     ScalaUDF(
       f,
       dataType,
       exprs,
-      inputsPrimitive,
-      inputTypes,
+      inputEncoders,
       udfName = name,
       nullable = nullable,
       udfDeterministic = deterministic)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 11c04e5212a3d..cad7916bad73e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -4282,7 +4282,7 @@ object functions {
   (0 to 10).foreach { x =>
     val types = (1 to x).foldRight("RT")((i, s) => {s"A$i, $s"})
     val typeTags = (1 to x).map(i => s"A$i: TypeTag").foldLeft("RT: TypeTag")(_ + ", " + _)
-    val inputSchemas = (1 to x).foldRight("Nil")((i, s) => {s"Try(ScalaReflection.schemaFor(typeTag[A$i])).toOption :: $s"})
+    val inputEncoders = (1 to x).foldRight("Nil")((i, s) => {s"Try(ExpressionEncoder[A$i]()).toOption :: $s"})
     println(s"""
       |/**
       | * Defines a Scala closure of $x arguments as user-defined function (UDF).
@@ -4295,8 +4295,8 @@ object functions {
       | */
       |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = {
       |  val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-      |  val inputSchemas = $inputSchemas
-      |  val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+      |  val inputEncoders = $inputEncoders
+      |  val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
       |  if (nullable) udf else udf.asNonNullable()
       |}""".stripMargin)
   }
@@ -4319,7 +4319,7 @@ object functions {
       | */
       |def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = {
       |  val func = $funcCall
-      |  SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill($i)(None))
+      |  SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill($i)(None))
       |}""".stripMargin)
   }
 
@@ -4401,8 +4401,8 @@ object functions {
    */
   def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4417,8 +4417,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag](f: Function1[A1, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4433,8 +4433,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag](f: Function2[A1, A2, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4449,8 +4449,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](f: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4465,8 +4465,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag](f: Function4[A1, A2, A3, A4, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4481,8 +4481,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag](f: Function5[A1, A2, A3, A4, A5, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4497,8 +4497,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag](f: Function6[A1, A2, A3, A4, A5, A6, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4513,8 +4513,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](f: Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4529,8 +4529,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](f: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4545,8 +4545,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](f: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A9])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4561,8 +4561,8 @@ object functions {
    */
   def udf[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4: TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10: TypeTag](f: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, RT]): UserDefinedFunction = {
     val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
-    val inputSchemas = Try(ScalaReflection.schemaFor(typeTag[A1])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A2])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A3])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A4])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A5])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A6])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A7])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A8])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A9])).toOption :: Try(ScalaReflection.schemaFor(typeTag[A10])).toOption :: Nil
-    val udf = SparkUserDefinedFunction(f, dataType, inputSchemas)
+    val inputEncoders = Try(ExpressionEncoder[A1]()).toOption :: Try(ExpressionEncoder[A2]()).toOption :: Try(ExpressionEncoder[A3]()).toOption :: Try(ExpressionEncoder[A4]()).toOption :: Try(ExpressionEncoder[A5]()).toOption :: Try(ExpressionEncoder[A6]()).toOption :: Try(ExpressionEncoder[A7]()).toOption :: Try(ExpressionEncoder[A8]()).toOption :: Try(ExpressionEncoder[A9]()).toOption :: Try(ExpressionEncoder[A10]()).toOption :: Nil
+    val udf = SparkUserDefinedFunction(f, dataType, inputEncoders)
     if (nullable) udf else udf.asNonNullable()
   }
 
@@ -4581,7 +4581,7 @@ object functions {
    */
   def udf(f: UDF0[_], returnType: DataType): UserDefinedFunction = {
     val func = () => f.asInstanceOf[UDF0[Any]].call()
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(0)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(0)(None))
   }
 
   /**
@@ -4595,7 +4595,7 @@ object functions {
    */
   def udf(f: UDF1[_, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF1[Any, Any]].call(_: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(1)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(1)(None))
   }
 
   /**
@@ -4609,7 +4609,7 @@ object functions {
    */
   def udf(f: UDF2[_, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF2[Any, Any, Any]].call(_: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(2)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(2)(None))
   }
 
   /**
@@ -4623,7 +4623,7 @@ object functions {
    */
   def udf(f: UDF3[_, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF3[Any, Any, Any, Any]].call(_: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(3)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(3)(None))
   }
 
   /**
@@ -4637,7 +4637,7 @@ object functions {
    */
   def udf(f: UDF4[_, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF4[Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(4)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(4)(None))
   }
 
   /**
@@ -4651,7 +4651,7 @@ object functions {
    */
   def udf(f: UDF5[_, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF5[Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(5)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(5)(None))
   }
 
   /**
@@ -4665,7 +4665,7 @@ object functions {
    */
   def udf(f: UDF6[_, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF6[Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(6)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(6)(None))
   }
 
   /**
@@ -4679,7 +4679,7 @@ object functions {
    */
   def udf(f: UDF7[_, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF7[Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(7)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(7)(None))
   }
 
   /**
@@ -4693,7 +4693,7 @@ object functions {
    */
   def udf(f: UDF8[_, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF8[Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(8)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(8)(None))
   }
 
   /**
@@ -4707,7 +4707,7 @@ object functions {
    */
   def udf(f: UDF9[_, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF9[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(9)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(9)(None))
   }
 
   /**
@@ -4721,7 +4721,7 @@ object functions {
    */
   def udf(f: UDF10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType): UserDefinedFunction = {
     val func = f.asInstanceOf[UDF10[Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]].call(_: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any, _: Any)
-    SparkUserDefinedFunction(func, returnType, inputSchemas = Seq.fill(10)(None))
+    SparkUserDefinedFunction(func, returnType, inputEncoders = Seq.fill(10)(None))
   }
 
   // scalastyle:on parameter.number
@@ -4759,7 +4759,7 @@ object functions {
         s"caution."
       throw new AnalysisException(errorMsg)
     }
-    SparkUserDefinedFunction(f, dataType, inputSchemas = Nil)
+    SparkUserDefinedFunction(f, dataType, inputEncoders = Nil)
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
index 51150a1b38b49..4a4504a075060 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
@@ -337,7 +337,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
         input.toString
       },
       StringType,
-      inputSchemas = Seq.fill(1)(None),
+      inputEncoders = Seq.fill(1)(None),
       name = Some(name)) {
 
       override def apply(exprs: Column*): Column = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index e0857ed6bc35a..08f41f6819a0e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -33,7 +33,6 @@ import org.apache.spark.sql.test.SQLTestData._
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.util.QueryExecutionListener
 
-
 private case class FunctionResult(f1: String, f2: String)
 
 class UDFSuite extends QueryTest with SharedSparkSession {
@@ -551,4 +550,47 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     }
     assert(e.getMessage.contains("Invalid arguments for function cast"))
   }
+
+  test("only one case class parameter") {
+    val f = (d: TestData) => d.key * d.value.toInt
+    val myUdf = udf(f)
+    val df = Seq(("data", TestData(50, "2"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("one case class with primitive parameter") {
+    val f = (i: Int, p: TestData) => p.key * i
+    val myUdf = udf(f)
+    val df = Seq((2, TestData(50, "data"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(100) :: Nil)
+  }
+
+  test("multiple case class parameters") {
+    val f = (d1: TestData, d2: TestData) => d1.key * d2.key
+    val myUdf = udf(f)
+    val df = Seq((TestData(10, "d1"), TestData(50, "d2"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(500) :: Nil)
+  }
+
+  test("input case class parameter and return case class") {
+    val f = (d: TestData) => TestData(d.key * 2, "copy")
+    val myUdf = udf(f)
+    val df = Seq(("data", TestData(50, "d2"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col2"))), Row(Row(100, "copy")) :: Nil)
+  }
+
+  test("any and case class parameter") {
+    val f = (any: Any, d: TestData) => s"${any.toString}, ${d.value}"
+    val myUdf = udf(f)
+    val df = Seq(("Hello", TestData(50, "World"))).toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row("Hello, World") :: Nil)
+  }
+
+  test("nested case class parameter") {
+    val f = (y: Int, training: TrainingSales) => training.sales.year + y
+    val myUdf = udf(f)
+    val df = Seq((20, TrainingSales("training", CourseSales("course", 2000, 3.14))))
+      .toDF("col1", "col2")
+    checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(2020) :: Nil)
+  }
 }

From d2789600233fa5e6c01b0c973f8f31ce5a206438 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Tue, 24 Mar 2020 13:37:13 -0700
Subject: [PATCH 0332/1280] [SPARK-31081][UI][SQL] Make display of
 stageId/stageAttemptId/taskId of sql metrics toggleable

### What changes were proposed in this pull request?

This is another solution for `SPARK-31081` and #27849 .
I added a checkbox which can toggle display of stageId/taskid in the SQL's DAG page.
Mainly, I implemented the toggleable texts in boxes with HTML label feature provided by `dagre-d3`.
The additional metrics are enclosed by `<span>` and control the appearance of the text.
But the exception is additional metrics in clusters.
We can use HTML label for cluster but layout will be broken so I choosed normal text label for clusters.
Due to that, this solution contains a little bit tricky code in`spark-sql-viz.js` to manipulate the metric texts and generate DOMs.

### Why are the changes needed?

It makes metrics harder to read after #26843 and user may not interest in extra info(stageId/StageAttemptId/taskId ) when they do not need debug.
#27849 control the appearance by a new configuration property but providing a checkbox is more flexible.

### Does this PR introduce any user-facing change?

Yes.
[Additional metrics shown]
![with-checked](https://user-images.githubusercontent.com/4736016/77244214-0f6cd780-6c56-11ea-9275-a30758dd5339.png)

[Additional metrics hidden]
![without-chedked](https://user-images.githubusercontent.com/4736016/77244219-14ca2200-6c56-11ea-9874-33a466085fce.png)

### How was this patch tested?

Tested manually with a simple DataFrame operation.
* The appearance of additional metrics in the boxes are controlled by the newly added checkbox.
* No error found with JS-debugger.
* Checked/not-checked state is preserved after reloading.

Closes #27927 from sarutak/SPARK-31081.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 999c9ed10c2362d89afd3bbb48e35f3c7ac3cf89)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../sql/execution/ui/static/spark-sql-viz.css |  2 +-
 .../sql/execution/ui/static/spark-sql-viz.js  | 74 ++++++++++++++++++-
 .../sql/execution/ui/ExecutionPage.scala      |  4 +
 .../sql/execution/ui/SparkPlanGraph.scala     | 10 ++-
 4 files changed, 83 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index 20188387c9ba4..ffc9acdce928f 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -34,7 +34,7 @@
 }
 
 /* Highlight the SparkPlan node name */
-#plan-viz-graph svg text :first-child {
+#plan-viz-graph svg text :first-child:not(.stageId-and-taskId-metrics) {
   font-weight: bold;
 }
 
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index c8349149439c8..b23ae9ab99c72 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -47,6 +47,7 @@ function renderPlanViz() {
   }
 
   resizeSvg(svg);
+  postprocessForAdditionalMetrics();
 }
 
 /* -------------------- *
@@ -75,6 +76,10 @@ function setupTooltipForSparkPlanNode(nodeId) {
     })
 }
 
+// labelSeparator should be a non-graphical character in order not to affect the width of boxes.
+var labelSeparator = "\x01";
+var stageAndTaskMetricsPattern = "^(.*)(\\(stage.*attempt.*task[^)]*\\))(.*)$";
+
 /*
  * Helper function to pre-process the graph layout.
  * This step is necessary for certain styles that affect the positioning
@@ -84,8 +89,29 @@ function preprocessGraphLayout(g) {
   g.graph().ranksep = "70";
   var nodes = g.nodes();
   for (var i = 0; i < nodes.length; i++) {
-      var node = g.node(nodes[i]);
-      node.padding = "5";
+    var node = g.node(nodes[i]);
+    node.padding = "5";
+
+    var firstSearator;
+    var secondSeparator;
+    var splitter;
+    if (node.isCluster) {
+      firstSearator = secondSeparator = labelSeparator;
+      splitter = "\\n";
+    } else {
+      firstSearator = "<span class='stageId-and-taskId-metrics'>";
+      secondSeparator = "</span>";
+      splitter = "<br>";
+    }
+
+    node.label.split(splitter).forEach(function(text, i) {
+      var newTexts = text.match(stageAndTaskMetricsPattern);
+      if (newTexts) {
+        node.label = node.label.replace(
+            newTexts[0],
+            newTexts[1] + firstSearator + newTexts[2] + secondSeparator + newTexts[3]);
+      }
+    });
   }
   // Curve the edges
   var edges = g.edges();
@@ -163,3 +189,47 @@ function getAbsolutePosition(d3selection) {
   }
   return { x: _x, y: _y };
 }
+
+/*
+ * Helper function for postprocess for additional metrics.
+ */
+function postprocessForAdditionalMetrics() {
+  // With dagre-d3, we can choose normal text (default) or HTML as a label type.
+  // HTML label for node works well but not for cluster so we need to choose the default label type
+  // and manipulate DOM.
+  $("g.cluster text tspan")
+    .each(function() {
+      var originalText = $(this).text();
+        if (originalText.indexOf(labelSeparator) > 0) {
+          var newTexts = originalText.split(labelSeparator);
+          var thisD3Node = d3.selectAll($(this));
+          thisD3Node.text(newTexts[0]);
+          thisD3Node.append("tspan").attr("class", "stageId-and-taskId-metrics").text(newTexts[1]);
+          $(this).append(newTexts[2]);
+        } else {
+          return originalText;
+        }
+  });
+
+  var checkboxNode = $("#stageId-and-taskId-checkbox");
+  checkboxNode.click(function() {
+      onClickAdditionalMetricsCheckbox($(this));
+  });
+  var isChecked = window.localStorage.getItem("stageId-and-taskId-checked") == "true";
+  $("#stageId-and-taskId-checkbox").prop("checked", isChecked);
+  onClickAdditionalMetricsCheckbox(checkboxNode);
+}
+
+/*
+ * Helper function which defines the action on click the checkbox.
+ */
+function onClickAdditionalMetricsCheckbox(checkboxNode) {
+  var additionalMetrics = $(".stageId-and-taskId-metrics");
+  var isChecked = checkboxNode.prop("checked");
+  if (isChecked) {
+    additionalMetrics.show();
+  } else {
+    additionalMetrics.hide();
+  }
+  window.localStorage.setItem("stageId-and-taskId-checked", isChecked);
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
index 91360e0e50314..ca146e7cd4a3e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
@@ -71,6 +71,10 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
             {jobLinks(JobExecutionStatus.FAILED, "Failed Jobs:")}
           </ul>
         </div>
+        <div>
+          <input type="checkbox" id="stageId-and-taskId-checkbox"></input>
+          <span>Show the Stage (Stage Attempt): Task ID that corresponds to the max metric</span>
+        </div>
 
       val metrics = sqlStore.executionMetrics(executionId)
       val graph = sqlStore.planGraph(executionId)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index d31d77840b802..67628029d226b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -160,7 +160,7 @@ private[ui] class SparkPlanGraphNode(
     val metrics: Seq[SQLPlanMetric]) {
 
   def makeDotNode(metricsValue: Map[Long, String]): String = {
-    val builder = new mutable.StringBuilder(name)
+    val builder = new mutable.StringBuilder("<b>" + name + "</b>")
 
     val values = for {
       metric <- metrics
@@ -173,9 +173,10 @@ private[ui] class SparkPlanGraphNode(
       // If there are metrics, display each entry in a separate line.
       // Note: whitespace between two "\n"s is to create an empty line between the name of
       // SparkPlan and metrics. If removing it, it won't display the empty line in UI.
-      builder ++= "\n \n"
-      builder ++= values.mkString("\n")
-      s"""  $id [label="${StringEscapeUtils.escapeJava(builder.toString())}"];"""
+      builder ++= "<br><br>"
+      builder ++= values.mkString("<br>")
+      val labelStr = StringEscapeUtils.escapeJava(builder.toString().replaceAll("\n", "<br>"))
+      s"""  $id [labelType="html" label="${labelStr}"];"""
     } else {
       // SPARK-30684: when there is no metrics, add empty lines to increase the height of the node,
       // so that there won't be gaps between an edge and a small node.
@@ -210,6 +211,7 @@ private[ui] class SparkPlanGraphCluster(
     }
     s"""
        |  subgraph cluster${id} {
+       |    isCluster="true";
        |    label="${StringEscapeUtils.escapeJava(labelStr)}";
        |    ${nodes.map(_.makeDotNode(metricsValue)).mkString("    \n")}
        |  }

From 4b970099d976cb817968a128a8825401f8903deb Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 24 Mar 2020 21:54:04 -0700
Subject: [PATCH 0333/1280] [SPARK-31101][BUILD][3.0] Upgrade Janino to 3.0.16

### What changes were proposed in this pull request?

This PR(SPARK-31101) proposes to upgrade Janino to 3.0.16 which is released recently.

* Merged pull request janino-compiler/janino#114 "Grow the code for relocatables, and do fixup, and relocate".

Please see the commit log.
- https://github.com/janino-compiler/janino/commits/3.0.16

You can see the changelog from the link: http://janino-compiler.github.io/janino/changelog.html / though release note for Janino 3.0.16 is actually incorrect.

### Why are the changes needed?

We got some report on failure on user's query which Janino throws error on compiling generated code. The issue is here: janino-compiler/janino#113 It contains the information of generated code, symptom (error), and analysis of the bug, so please refer the link for more details.
Janino 3.0.16 contains the PR janino-compiler/janino#114 which would enable Janino to succeed to compile user's query properly.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing UTs.

Below test code fails on branch-3.0 and passes with this patch.

```
  /**
   * NOTE: The test code tries to control the size of for/switch statement in expand_doConsume,
   * as well as the overall size of expand_doConsume, so that the query triggers known Janino
   * bug - https://github.com/janino-compiler/janino/issues/113.
   *
   * The expected exception message from Janino when we use switch statement for "ExpandExec":
   * - "Operand stack inconsistent at offset xxx: Previous size 1, now 0"
   * which will not happen when we use if-else-if statement for "ExpandExec".
   *
   * "The number of fields" and "The number of distinct aggregation functions" are the major
   * factors to increase the size of generated code: while these values should be large enough
   * to trigger the Janino bug, these values should not also too big; otherwise one of below
   * exceptions might be thrown:
   * - "expand_doConsume would be beyond 64KB"
   * - "java.lang.ClassFormatError: Too many arguments in method signature in class file"
   */
  test("SPARK-31115 Lots of columns and distinct aggregations shouldn't break code generation") {
    withSQLConf(
      (SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true"),
      (SQLConf.WHOLESTAGE_MAX_NUM_FIELDS.key, "10000"),
      (SQLConf.CODEGEN_FALLBACK.key, "false"),
      (SQLConf.CODEGEN_LOGGING_MAX_LINES.key, "-1")
    ) {
      var df = Seq(("1", "2", 1), ("1", "2", 2), ("2", "3", 3), ("2", "3", 4)).toDF("a", "b", "c")

      // The value is tested under commit "e807118eef9e0214170ff62c828524d237bd58e3":
      // the query fails with switch statement, whereas it passes with if-else statement.
      // Note that the value depends on the Spark logic as well - different Spark versions may
      // require different value to ensure the test failing with switch statement.
      val numNewFields = 100

      df = df.withColumns(
        (1 to numNewFields).map { idx => s"a$idx" },
        (1 to numNewFields).map { idx =>
          when(col("c").mod(lit(2)).===(lit(0)), lit(idx)).otherwise(col("c"))
        }
      )

      val aggExprs: Array[Column] = Range(1, numNewFields).map { idx =>
        if (idx % 2 == 0) {
          coalesce(countDistinct(s"a$idx"), lit(0))
        } else {
          coalesce(count(s"a$idx"), lit(0))
        }
      }.toArray

      val aggDf = df
        .groupBy("a", "b")
        .agg(aggExprs.head, aggExprs.tail: _*)

      // We are only interested in whether the code compilation fails or not, so skipping
      // verification on outputs.
      aggDf.collect()
    }
  }
```

Closes #27996 from HeartSaVioR/SPARK-31101-branch-3.0.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 4 ++--
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 4 ++--
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 4 ++--
 pom.xml                                 | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 39f7262323f55..6e8368b36397b 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -35,7 +35,7 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.15//commons-compiler-3.0.15.jar
+commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.8.1//commons-compress-1.8.1.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
@@ -105,7 +105,7 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.15//janino-3.0.15.jar
+janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 26ac30d6d6be8..79214058b8f03 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -33,7 +33,7 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.15//commons-compiler-3.0.15.jar
+commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.8.1//commons-compress-1.8.1.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
@@ -118,7 +118,7 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.15//janino-3.0.15.jar
+janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index e908ec8a355d8..fd4bb7e32a492 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -30,7 +30,7 @@ commons-beanutils/1.9.4//commons-beanutils-1.9.4.jar
 commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
-commons-compiler/3.0.15//commons-compiler-3.0.15.jar
+commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.8.1//commons-compress-1.8.1.jar
 commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
@@ -117,7 +117,7 @@ jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
 jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
 jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
 jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
-janino/3.0.15//janino-3.0.15.jar
+janino/3.0.16//janino-3.0.16.jar
 javassist/3.25.0-GA//javassist-3.25.0-GA.jar
 javax.inject/1//javax.inject-1.jar
 javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
diff --git a/pom.xml b/pom.xml
index 978127e91e1a1..cc4d500d599d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,7 +178,7 @@
     <!-- org.apache.commons/commons-pool2/-->
     <commons-pool2.version>2.6.2</commons-pool2.version>
     <datanucleus-core.version>4.1.17</datanucleus-core.version>
-    <janino.version>3.0.15</janino.version>
+    <janino.version>3.0.16</janino.version>
     <jersey.version>2.30</jersey.version>
     <joda.version>2.10.5</joda.version>
     <jodd.version>3.5.2</jodd.version>

From 5da878b913606ed6dddee9b1b6c6c5770a8a5e45 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Wed, 25 Mar 2020 13:19:43 +0800
Subject: [PATCH 0334/1280] [SPARK-31207][CORE] Ensure the total number of
 blocks to fetch equals to the sum of local/hostLocal/remote blocks

### What changes were proposed in this pull request?

Assert the number of blocks to fetch equals the number of local blocks + the number of hostLocal blocks + the number of remote blocks in ShuffleBlockFetcherIterator. Also refactor the code a bit to make it easier to follow.

### Why are the changes needed?

When the numbers don't match it means something is going wrong, we should fail fast.

### Does this PR introduce any user-facing change?

No. This is basically code refactoring.

### How was this patch tested?

Tested with existing test suites.

Closes #27972 from jiangxb1987/BlockFetcher.

Authored-by: Xingbo Jiang <xingbo.jiang@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a03fbfbdd5758f6f2798510783c830c432ba0367)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../storage/ShuffleBlockFetcherIterator.scala | 84 +++++++++++--------
 1 file changed, 49 insertions(+), 35 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index 2a0447de04c94..f1a7d88213f7b 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -91,8 +91,7 @@ final class ShuffleBlockFetcherIterator(
   private val targetRemoteRequestSize = math.max(maxBytesInFlight / 5, 1L)
 
   /**
-   * Total number of blocks to fetch. This should be equal to the total number of blocks
-   * in [[blocksByAddress]] because we already filter out zero-sized blocks in [[blocksByAddress]].
+   * Total number of blocks to fetch.
    */
   private[this] var numBlocksToFetch = 0
 
@@ -290,7 +289,6 @@ final class ShuffleBlockFetcherIterator(
     var localBlockBytes = 0L
     var hostLocalBlockBytes = 0L
     var remoteBlockBytes = 0L
-    var numRemoteBlocks = 0
 
     val hostLocalDirReadingEnabled =
       blockManager.hostLocalDirManager != null && blockManager.hostLocalDirManager.isDefined
@@ -299,25 +297,29 @@ final class ShuffleBlockFetcherIterator(
       if (address.executorId == blockManager.blockManagerId.executorId) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
-          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)).to[ArrayBuffer])
+          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)))
         localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex))
         localBlockBytes += mergedBlockInfos.map(_.size).sum
       } else if (hostLocalDirReadingEnabled && address.host == blockManager.blockManagerId.host) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
-          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)).to[ArrayBuffer])
+          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)))
         val blocksForAddress =
           mergedBlockInfos.map(info => (info.blockId, info.size, info.mapIndex))
         hostLocalBlocksByExecutor += address -> blocksForAddress
         hostLocalBlocks ++= blocksForAddress.map(info => (info._1, info._3))
         hostLocalBlockBytes += mergedBlockInfos.map(_.size).sum
       } else {
-        numRemoteBlocks += blockInfos.size
         remoteBlockBytes += blockInfos.map(_._2).sum
         collectFetchRequests(address, blockInfos, collectedRemoteRequests)
       }
     }
+    val numRemoteBlocks = collectedRemoteRequests.map(_.blocks.size).sum
     val totalBytes = localBlockBytes + remoteBlockBytes + hostLocalBlockBytes
+    assert(numBlocksToFetch == localBlocks.size + hostLocalBlocks.size + numRemoteBlocks,
+      s"The number of non-empty blocks $numBlocksToFetch doesn't equal to the number of local " +
+        s"blocks ${localBlocks.size} + the number of host-local blocks ${hostLocalBlocks.size} " +
+        s"+ the number of remote blocks ${numRemoteBlocks}.")
     logInfo(s"Getting $numBlocksToFetch (${Utils.bytesToString(totalBytes)}) non-empty blocks " +
       s"including ${localBlocks.size} (${Utils.bytesToString(localBlockBytes)}) local and " +
       s"${hostLocalBlocks.size} (${Utils.bytesToString(hostLocalBlockBytes)}) " +
@@ -325,6 +327,40 @@ final class ShuffleBlockFetcherIterator(
     collectedRemoteRequests
   }
 
+  private def createFetchRequest(
+      blocks: Seq[FetchBlockInfo],
+      address: BlockManagerId,
+      curRequestSize: Long): FetchRequest = {
+    logDebug(s"Creating fetch request of $curRequestSize at $address "
+      + s"with ${blocks.size} blocks")
+    FetchRequest(address, blocks)
+  }
+
+  private def createFetchRequests(
+      curBlocks: Seq[FetchBlockInfo],
+      address: BlockManagerId,
+      curRequestSize: Long,
+      isLast: Boolean,
+      collectedRemoteRequests: ArrayBuffer[FetchRequest]): Seq[FetchBlockInfo] = {
+    val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks)
+    var retBlocks = Seq.empty[FetchBlockInfo]
+    if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
+      collectedRemoteRequests += createFetchRequest(mergedBlocks, address, curRequestSize)
+    } else {
+      mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks =>
+        if (blocks.length == maxBlocksInFlightPerAddress || isLast) {
+          collectedRemoteRequests += createFetchRequest(blocks, address, curRequestSize)
+        } else {
+          // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back
+          // to `curBlocks`.
+          retBlocks = blocks
+          numBlocksToFetch -= blocks.size
+        }
+      }
+    }
+    retBlocks
+  }
+
   private def collectFetchRequests(
       address: BlockManagerId,
       blockInfos: Seq[(BlockId, Long, Int)],
@@ -333,32 +369,6 @@ final class ShuffleBlockFetcherIterator(
     var curRequestSize = 0L
     var curBlocks = new ArrayBuffer[FetchBlockInfo]
 
-    def createFetchRequest(blocks: Seq[FetchBlockInfo]): Unit = {
-      collectedRemoteRequests += FetchRequest(address, blocks)
-      logDebug(s"Creating fetch request of $curRequestSize at $address "
-        + s"with ${blocks.size} blocks")
-    }
-
-    def createFetchRequests(isLast: Boolean): Unit = {
-      val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks)
-      curBlocks = new ArrayBuffer[FetchBlockInfo]
-      if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
-        createFetchRequest(mergedBlocks)
-      } else {
-        mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks =>
-          if (blocks.length == maxBlocksInFlightPerAddress || isLast) {
-            createFetchRequest(blocks)
-          } else {
-            // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back
-            // to `curBlocks`.
-            curBlocks = blocks
-            numBlocksToFetch -= blocks.size
-          }
-        }
-      }
-      curRequestSize = curBlocks.map(_.size).sum
-    }
-
     while (iterator.hasNext) {
       val (blockId, size, mapIndex) = iterator.next()
       assertPositiveBlockSize(blockId, size)
@@ -367,12 +377,16 @@ final class ShuffleBlockFetcherIterator(
       // For batch fetch, the actual block in flight should count for merged block.
       val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress
       if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) {
-        createFetchRequests(isLast = false)
+        curBlocks = createFetchRequests(curBlocks, address, curRequestSize, isLast = false,
+          collectedRemoteRequests).to[ArrayBuffer]
+        curRequestSize = curBlocks.map(_.size).sum
       }
     }
     // Add in the final request
     if (curBlocks.nonEmpty) {
-      createFetchRequests(isLast = true)
+      curBlocks = createFetchRequests(curBlocks, address, curRequestSize, isLast = true,
+        collectedRemoteRequests).to[ArrayBuffer]
+      curRequestSize = curBlocks.map(_.size).sum
     }
   }
 
@@ -389,7 +403,7 @@ final class ShuffleBlockFetcherIterator(
   }
 
   private[this] def mergeContinuousShuffleBlockIdsIfNeeded(
-      blocks: ArrayBuffer[FetchBlockInfo]): ArrayBuffer[FetchBlockInfo] = {
+      blocks: Seq[FetchBlockInfo]): Seq[FetchBlockInfo] = {
     val result = if (doBatchFetch) {
       var curBlocks = new ArrayBuffer[FetchBlockInfo]
       val mergedBlockInfo = new ArrayBuffer[FetchBlockInfo]

From 89f31df3ffb20af8bd49c4bdb1f4624aa4feadf9 Mon Sep 17 00:00:00 2001
From: samsetegne <samuelsetegne@gmail.com>
Date: Wed, 25 Mar 2020 15:00:15 +0800
Subject: [PATCH 0335/1280] [SPARK-30822][SQL] Remove semicolon at the end of a
 sql query

# What changes were proposed in this pull request?
This change proposes ignoring a terminating semicolon from queries submitted by the user (if included) instead of raising a parse exception.

# Why are the changes needed?
When a user submits a directly executable SQL statement terminated with a semicolon, they receive an `org.apache.spark.sql.catalyst.parser.ParseException` of `extraneous input ';' expecting <EOF>`. SQL-92 describes a direct SQL statement as having the format of `<directly executable statement> <semicolon>` and the majority of SQL implementations either require the semicolon as a statement terminator, or make it optional (meaning not raising an exception when it's included, seemingly in recognition that it's a common behavior).

# Does this PR introduce any user-facing change?
No

# How was this patch tested?
Unit test added to `PlanParserSuite`
```
sbt> project catalyst
sbt> testOnly *PlanParserSuite
[info] - case insensitive (565 milliseconds)
[info] - explain (9 milliseconds)
[info] - set operations (41 milliseconds)
[info] - common table expressions (31 milliseconds)
[info] - simple select query (47 milliseconds)
[info] - hive-style single-FROM statement (11 milliseconds)
[info] - multi select query (32 milliseconds)
[info] - query organization (41 milliseconds)
[info] - insert into (12 milliseconds)
[info] - aggregation (24 milliseconds)
[info] - limit (11 milliseconds)
[info] - window spec (11 milliseconds)
[info] - lateral view (17 milliseconds)
[info] - joins (62 milliseconds)
[info] - sampled relations (11 milliseconds)
[info] - sub-query (11 milliseconds)
[info] - scalar sub-query (9 milliseconds)
[info] - table reference (2 milliseconds)
[info] - table valued function (8 milliseconds)
[info] - SPARK-20311 range(N) as alias (2 milliseconds)
[info] - SPARK-20841 Support table column aliases in FROM clause (3 milliseconds)
[info] - SPARK-20962 Support subquery column aliases in FROM clause (4 milliseconds)
[info] - SPARK-20963 Support aliases for join relations in FROM clause (3 milliseconds)
[info] - inline table (23 milliseconds)
[info] - simple select query with !> and !< (5 milliseconds)
[info] - select hint syntax (34 milliseconds)
[info] - SPARK-20854: select hint syntax with expressions (12 milliseconds)
[info] - SPARK-20854: multiple hints (4 milliseconds)
[info] - TRIM function (16 milliseconds)
[info] - OVERLAY function (16 milliseconds)
[info] - precedence of set operations (18 milliseconds)
[info] - create/alter view as insert into table (4 milliseconds)
[info] - Invalid insert constructs in the query (10 milliseconds)
[info] - relation in v2 catalog (3 milliseconds)
[info] - CTE with column alias (2 milliseconds)
[info] - statement containing terminal semicolons (3 milliseconds)
[info] ScalaTest
[info] Run completed in 3 seconds, 129 milliseconds.
[info] Total number of tests run: 36
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 36, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
[info] Passed: Total 36, Failed 0, Errors 0, Passed 36
```

### Current behavior:
#### scala
```scala
scala> val df = sql("select 1")
// df: org.apache.spark.sql.DataFrame = [1: int]
scala> df.show()
// +---+
// |  1|
// +---+
// |  1|
// +---+

scala> val df = sql("select 1;")
// org.apache.spark.sql.catalyst.parser.ParseException:
// extraneous input ';' expecting <EOF>(line 1, pos 8)

// == SQL ==
// select 1;
// --------^^^

//   at org.apache.spark.sql.catalyst.parser.ParseException.withCommand(ParseDriver.scala:263)
//   at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parse(ParseDriver.scala:130)
//   at org.apache.spark.sql.execution.SparkSqlParser.parse(SparkSqlParser.scala:52)
//   at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parsePlan(ParseDriver.scala:76)
//   at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:605)
//   at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
//   at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:605)
//   ... 47 elided
```
#### pyspark
```python
df = spark.sql('select 1')
df.show()
#+---+
#|  1|
#+---+
#|  1|
#+---+

df = spark.sql('select 1;')
# Traceback (most recent call last):
#   File "<stdin>", line 1, in <module>
#   File "/Users/ssetegne/spark/python/pyspark/sql/session.py", line 646, in sql
#     return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)
#   File "/Users/ssetegne/spark/python/lib/py4j-0.10.8.1-src.zip/py4j/java_gateway.py", line 1286, in # __call__
#   File "/Users/ssetegne/spark/python/pyspark/sql/utils.py", line 102, in deco
#     raise converted
# pyspark.sql.utils.ParseException:
# extraneous input ';' expecting <EOF>(line 1, pos 8)

# == SQL ==
# select 1;
# --------^^^
```

### Behavior after proposed fix:
#### scala
```scala
scala> val df = sql("select 1")
// df: org.apache.spark.sql.DataFrame = [1: int]
scala> df.show()
// +---+
// |  1|
// +---+
// |  1|
// +---+

scala> val df = sql("select 1;")
// df: org.apache.spark.sql.DataFrame = [1: int]
scala> df.show()
// +---+
// |  1|
// +---+
// |  1|
// +---+
```
#### pyspark
```python
df = spark.sql('select 1')
df.show()
#+---+
#|    1 |
#+---+
#|    1 |
#+---+

df = spark.sql('select 1;')
df.show()
#+---+
#|    1 |
#+---+
#|    1 |
#+---+
```

Closes #27567 from samsetegne/semicolon.

Authored-by: samsetegne <samuelsetegne@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 44431d4b1a22c3db87d7e4a24df517d6d45905a8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 | 2 +-
 .../apache/spark/sql/catalyst/parser/DDLParserSuite.scala  | 2 +-
 .../apache/spark/sql/catalyst/parser/PlanParserSuite.scala | 7 +++++++
 .../resources/sql-tests/results/describe-query.sql.out     | 4 ++--
 .../test/resources/sql-tests/results/grouping_set.sql.out  | 4 ++--
 .../src/test/resources/sql-tests/results/interval.sql.out  | 2 +-
 6 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 563ef69b3b8ae..143a567e8082c 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -64,7 +64,7 @@ grammar SqlBase;
 }
 
 singleStatement
-    : statement EOF
+    : statement ';'* EOF
     ;
 
 singleExpression
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 35a54c8e95b72..543ea5ddf5d20 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -1480,7 +1480,7 @@ class DDLParserSuite extends AnalysisTest {
       AnalyzeColumnStatement(Seq("a", "b", "c"), None, allColumns = true))
 
     intercept("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL COLUMNS key, value",
-      "mismatched input 'key' expecting <EOF>")
+      "mismatched input 'key' expecting {<EOF>, ';'}")
     intercept("ANALYZE TABLE a.b.c COMPUTE STATISTICS FOR ALL",
       "missing 'COLUMNS' at '<EOF>'")
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 875096f615241..11230e4065419 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -922,4 +922,11 @@ class PlanParserSuite extends AnalysisTest {
       "WITH t(x) AS (SELECT c FROM a) SELECT * FROM t",
       cte(table("t").select(star()), "t" -> ((table("a").select('c), Seq("x")))))
   }
+
+  test("statement containing terminal semicolons") {
+    assertEqual("select 1;", OneRowRelation().select(1))
+    assertEqual("select a, b;", OneRowRelation().select('a, 'b))
+    assertEqual("select a, b from db.c;;;", table("db", "c").select('a, 'b))
+    assertEqual("select a, b from db.c; ;;  ;", table("db", "c").select('a, 'b))
+  }
 }
diff --git a/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out b/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
index 6b16aba268f50..2199fc0312d25 100644
--- a/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/describe-query.sql.out
@@ -112,7 +112,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-mismatched input 'desc_temp1' expecting {<EOF>, '.'}(line 1, pos 21)
+mismatched input 'desc_temp1' expecting {<EOF>, ';'}(line 1, pos 21)
 
 == SQL ==
 DESCRIBE INSERT INTO desc_temp1 values (1, 'val1')
@@ -126,7 +126,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-mismatched input 'desc_temp1' expecting {<EOF>, '.'}(line 1, pos 21)
+mismatched input 'desc_temp1' expecting {<EOF>, ';'}(line 1, pos 21)
 
 == SQL ==
 DESCRIBE INSERT INTO desc_temp1 SELECT * FROM desc_temp2
diff --git a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
index 8eeabb34b4fab..0a5fe7a727241 100644
--- a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
@@ -138,7 +138,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-extraneous input 'ROLLUP' expecting <EOF>(line 1, pos 53)
+extraneous input 'ROLLUP' expecting {<EOF>, ';'}(line 1, pos 53)
 
 == SQL ==
 SELECT a, b, c, count(d) FROM grouping GROUP BY WITH ROLLUP
@@ -152,7 +152,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-extraneous input 'CUBE' expecting <EOF>(line 1, pos 53)
+extraneous input 'CUBE' expecting {<EOF>, ';'}(line 1, pos 53)
 
 == SQL ==
 SELECT a, b, c, count(d) FROM grouping GROUP BY WITH CUBE
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 7a3dd74b33bb5..0d758ca1f24ca 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -639,7 +639,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-extraneous input 'day' expecting <EOF>(line 1, pos 27)
+extraneous input 'day' expecting {<EOF>, ';'}(line 1, pos 27)
 
 == SQL ==
 select interval 30 day day day

From b3fd5c90ae00807f3a49e2b9f1f0c8d026cc4219 Mon Sep 17 00:00:00 2001
From: Eric Wu <492960551@qq.com>
Date: Sat, 21 Mar 2020 17:41:23 -0700
Subject: [PATCH 0336/1280] [SPARK-31184][SQL] Support getTablesByType API of
 Hive Client

### What changes were proposed in this pull request?
Hive 2.3+ supports `getTablesByType` API, which will provide an efficient way to get HiveTable with specific type. Now, we have following mappings when using `HiveExternalCatalog`.
```
CatalogTableType.EXTERNAL  =>  HiveTableType.EXTERNAL_TABLE
CatalogTableType.MANAGED => HiveTableType.MANAGED_TABLE
CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW
```
Without this API, we need to achieve the goal by `getTables` + `getTablesByName` + `filter with type`.

This PR add `getTablesByType` in `HiveShim`. For those hive versions don't support this API, `UnsupportedOperationException` will be thrown. And the upper logic should catch the exception and fallback to the filter solution mentioned above.

Since the JDK11 related fix in `Hive` is not released yet, manual tests against hive 2.3.7-SNAPSHOT is done by following the instructions of SPARK-29245.

### Why are the changes needed?
This API will provide better usability and performance if we want to get a list of hiveTables with specific type. For example `HiveTableType.VIRTUAL_VIEW` corresponding to `CatalogTableType.VIEW`.

### Does this PR introduce any user-facing change?
No, this is a support function.

### How was this patch tested?
Add tests in VersionsSuite and manually run JDK11 test with following settings:

- Hive 2.3.6 Metastore on JDK8
- Hive 2.3.7-SNAPSHOT library build from source of Hive 2.3 branch
- Spark build with Hive 2.3.7-SNAPSHOT on jdk-11.0.6

Closes #27952 from Eric5553/GetTableByType.

Authored-by: Eric Wu <492960551@qq.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 3a48ea1fe0fb85253f12d86caea01ffcb7e678d0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/hive/client/HiveClient.scala    |  9 ++++
 .../sql/hive/client/HiveClientImpl.scala      | 42 ++++++++++++++-----
 .../spark/sql/hive/client/HiveShim.scala      | 35 +++++++++++++++-
 .../spark/sql/hive/client/VersionsSuite.scala | 22 ++++++++--
 4 files changed, 92 insertions(+), 16 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
index e31dffa4795c5..3ea80eaf6f714 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClient.scala
@@ -61,6 +61,15 @@ private[hive] trait HiveClient {
   /** Returns the names of tables in the given database that matches the given pattern. */
   def listTables(dbName: String, pattern: String): Seq[String]
 
+  /**
+   * Returns the names of tables with specific tableType in the given database that matches
+   * the given pattern.
+   */
+  def listTablesByType(
+      dbName: String,
+      pattern: String,
+      tableType: CatalogTableType): Seq[String]
+
   /** Sets the name of current database. */
   def setCurrentDatabase(databaseName: String): Unit
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 4a3e813740b48..6ad5e9d3c9080 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -755,6 +755,22 @@ private[hive] class HiveClientImpl(
     client.getTablesByPattern(dbName, pattern).asScala
   }
 
+  override def listTablesByType(
+      dbName: String,
+      pattern: String,
+      tableType: CatalogTableType): Seq[String] = withHiveState {
+    try {
+      // Try with Hive API getTablesByType first, it's supported from Hive 2.3+.
+      shim.getTablesByType(client, dbName, pattern, toHiveTableType(tableType))
+    } catch {
+      case _: UnsupportedOperationException =>
+        // Fallback to filter logic if getTablesByType not supported.
+        val tableNames = client.getTablesByPattern(dbName, pattern).asScala
+        val tables = getTablesByName(dbName, tableNames).filter(_.tableType == tableType)
+        tables.map(_.identifier.table)
+    }
+  }
+
   /**
    * Runs the specified SQL query using Hive.
    */
@@ -1011,25 +1027,29 @@ private[hive] object HiveClientImpl extends Logging {
   private def toOutputFormat(name: String) =
     Utils.classForName[org.apache.hadoop.hive.ql.io.HiveOutputFormat[_, _]](name)
 
+  def toHiveTableType(catalogTableType: CatalogTableType): HiveTableType = {
+    catalogTableType match {
+      case CatalogTableType.EXTERNAL => HiveTableType.EXTERNAL_TABLE
+      case CatalogTableType.MANAGED => HiveTableType.MANAGED_TABLE
+      case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW
+      case t =>
+        throw new IllegalArgumentException(
+          s"Unknown table type is found at toHiveTableType: $t")
+    }
+  }
+
   /**
    * Converts the native table metadata representation format CatalogTable to Hive's Table.
    */
   def toHiveTable(table: CatalogTable, userName: Option[String] = None): HiveTable = {
     val hiveTable = new HiveTable(table.database, table.identifier.table)
+    hiveTable.setTableType(toHiveTableType(table.tableType))
     // For EXTERNAL_TABLE, we also need to set EXTERNAL field in the table properties.
     // Otherwise, Hive metastore will change the table to a MANAGED_TABLE.
     // (metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java#L1095-L1105)
-    hiveTable.setTableType(table.tableType match {
-      case CatalogTableType.EXTERNAL =>
-        hiveTable.setProperty("EXTERNAL", "TRUE")
-        HiveTableType.EXTERNAL_TABLE
-      case CatalogTableType.MANAGED =>
-        HiveTableType.MANAGED_TABLE
-      case CatalogTableType.VIEW => HiveTableType.VIRTUAL_VIEW
-      case t =>
-        throw new IllegalArgumentException(
-          s"Unknown table type is found at toHiveTable: $t")
-    })
+    if (table.tableType == CatalogTableType.EXTERNAL) {
+      hiveTable.setProperty("EXTERNAL", "TRUE")
+    }
     // Note: In Hive the schema and partition columns must be disjoint sets
     val (partCols, schema) = table.schema.map(toHiveColumn).partition { c =>
       table.partitionColumnNames.contains(c.getName)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 50ce536a160c8..2b806609426a1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -29,6 +29,7 @@ import scala.util.control.NonFatal
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.metastore.IMetaStoreClient
+import org.apache.hadoop.hive.metastore.TableType
 import org.apache.hadoop.hive.metastore.api.{Database, EnvironmentContext, Function => HiveFunction, FunctionType, MetaException, PrincipalType, ResourceType, ResourceUri}
 import org.apache.hadoop.hive.ql.Driver
 import org.apache.hadoop.hive.ql.io.AcidUtils
@@ -90,6 +91,12 @@ private[client] sealed abstract class Shim {
 
   def alterPartitions(hive: Hive, tableName: String, newParts: JList[Partition]): Unit
 
+  def getTablesByType(
+      hive: Hive,
+      dbName: String,
+      pattern: String,
+      tableType: TableType): Seq[String]
+
   def createPartitions(
       hive: Hive,
       db: String,
@@ -363,6 +370,15 @@ private[client] class Shim_v0_12 extends Shim with Logging {
     conf.getIntVar(HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY) * 1000L
   }
 
+  override def getTablesByType(
+      hive: Hive,
+      dbName: String,
+      pattern: String,
+      tableType: TableType): Seq[String] = {
+    throw new UnsupportedOperationException("Hive 2.2 and lower versions don't support " +
+      "getTablesByType. Please use Hive 2.3 or higher version.")
+  }
+
   override def loadPartition(
       hive: Hive,
       loadPath: Path,
@@ -1220,7 +1236,24 @@ private[client] class Shim_v2_1 extends Shim_v2_0 {
 
 private[client] class Shim_v2_2 extends Shim_v2_1
 
-private[client] class Shim_v2_3 extends Shim_v2_1
+private[client] class Shim_v2_3 extends Shim_v2_1 {
+  private lazy val getTablesByTypeMethod =
+    findMethod(
+      classOf[Hive],
+      "getTablesByType",
+      classOf[String],
+      classOf[String],
+      classOf[TableType])
+
+  override def getTablesByType(
+      hive: Hive,
+      dbName: String,
+      pattern: String,
+      tableType: TableType): Seq[String] = {
+    getTablesByTypeMethod.invoke(hive, dbName, pattern, tableType)
+      .asInstanceOf[JList[String]].asScala
+  }
+}
 
 private[client] class Shim_v3_0 extends Shim_v2_3 {
   // Spark supports only non-ACID operations
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index 7471142d4e6fa..ba75dcf006e5f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -154,10 +154,11 @@ class VersionsSuite extends SparkFunSuite with Logging {
         .client.version.fullVersion.startsWith(version))
     }
 
-    def table(database: String, tableName: String): CatalogTable = {
+    def table(database: String, tableName: String,
+        tableType: CatalogTableType = CatalogTableType.MANAGED): CatalogTable = {
       CatalogTable(
         identifier = TableIdentifier(tableName, Some(database)),
-        tableType = CatalogTableType.MANAGED,
+        tableType = tableType,
         schema = new StructType().add("key", "int"),
         storage = CatalogStorageFormat(
           locationUri = None,
@@ -273,7 +274,9 @@ class VersionsSuite extends SparkFunSuite with Logging {
 
     test(s"$version: createTable") {
       client.createTable(table("default", tableName = "src"), ignoreIfExists = false)
-      client.createTable(table("default", "temporary"), ignoreIfExists = false)
+      client.createTable(table("default", tableName = "temporary"), ignoreIfExists = false)
+      client.createTable(table("default", tableName = "view1", tableType = CatalogTableType.VIEW),
+        ignoreIfExists = false)
     }
 
     test(s"$version: loadTable") {
@@ -389,7 +392,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
     }
 
     test(s"$version: listTables(database)") {
-      assert(client.listTables("default") === Seq("src", "temporary"))
+      assert(client.listTables("default") === Seq("src", "temporary", "view1"))
     }
 
     test(s"$version: listTables(database, pattern)") {
@@ -397,6 +400,13 @@ class VersionsSuite extends SparkFunSuite with Logging {
       assert(client.listTables("default", pattern = "nonexist").isEmpty)
     }
 
+    test(s"$version: listTablesByType(database, pattern, tableType)") {
+      assert(client.listTablesByType("default", pattern = "view1",
+        CatalogTableType.VIEW) === Seq("view1"))
+      assert(client.listTablesByType("default", pattern = "nonexist",
+        CatalogTableType.VIEW).isEmpty)
+    }
+
     test(s"$version: dropTable") {
       val versionsWithoutPurge =
         if (versions.contains("0.14")) versions.takeWhile(_ != "0.14") else Nil
@@ -405,12 +415,16 @@ class VersionsSuite extends SparkFunSuite with Logging {
       try {
         client.dropTable("default", tableName = "temporary", ignoreIfNotExists = false,
           purge = true)
+        client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
+          purge = true)
         assert(!versionsWithoutPurge.contains(version))
       } catch {
         case _: UnsupportedOperationException =>
           assert(versionsWithoutPurge.contains(version))
           client.dropTable("default", tableName = "temporary", ignoreIfNotExists = false,
             purge = false)
+          client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
+            purge = false)
       }
       assert(client.listTables("default") === Seq("src"))
     }

From bd999943866e9dcc1a67222f6d46e97e1726d96e Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Wed, 25 Mar 2020 16:32:28 +0800
Subject: [PATCH 0337/1280] [SPARK-31232][SQL][DOCS] Specify formats of
 `spark.sql.session.timeZone`

In the PR, I propose to update the doc for `spark.sql.session.timeZone`, and restrict format of config's values to 2 forms:
1. Geographical regions, such as `America/Los_Angeles`.
2. Fixed offsets - a fully resolved offset from UTC. For example, `-08:00`.

Other formats such as three-letter time zone IDs are ambitious, and depend on the locale. For example, `CST` could be U.S. `Central Standard Time` and `China Standard Time`. Such formats have been already deprecated in JDK, see [Three-letter time zone IDs](https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html).

No

By running `./dev/scalastyle`, and manual testing.

Closes #27999 from MaxGekk/doc-session-time-zone.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 27d53de10faa907a53a979b66282a457767b33ba)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/internal/SQLConf.scala | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 9681a5d25306e..1b00bed82da0a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1507,13 +1507,16 @@ object SQLConf {
     Try { DateTimeUtils.getZoneId(zone) }.isSuccess
   }
 
-  val SESSION_LOCAL_TIMEZONE =
-    buildConf("spark.sql.session.timeZone")
-      .doc("""The ID of session local timezone, e.g. "GMT", "America/Los_Angeles", etc.""")
-      .stringConf
-      .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
-        " ZoneId.of(_, ZoneId.SHORT_IDS)")
-      .createWithDefaultFunction(() => TimeZone.getDefault.getID)
+  val SESSION_LOCAL_TIMEZONE = buildConf("spark.sql.session.timeZone")
+    .doc("The ID of session local timezone in the format of either region-based zone IDs or " +
+      "zone offsets. Region IDs must have the form 'area/city', such as 'America/Los_Angeles'. " +
+      "Zone offsets must be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. " +
+      "Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not " +
+      "recommended to use because they can be ambiguous.")
+    .stringConf
+    .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
+      " ZoneId.of(_, ZoneId.SHORT_IDS)")
+    .createWithDefaultFunction(() => TimeZone.getDefault.getID)
 
   val WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD =
     buildConf("spark.sql.windowExec.buffer.in.memory.threshold")

From e6c85b37298d026ff7e38aad4b10098f6169f36a Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Wed, 25 Mar 2020 08:37:28 -0700
Subject: [PATCH 0338/1280] [SPARK-25121][SQL][FOLLOWUP] Add more unit tests
 for multi-part identifiers in join strategy hints

### What changes were proposed in this pull request?

This pr intends to add unit tests for the other join hints (`MERGEJOIN`, `SHUFFLE_HASH`, and `SHUFFLE_REPLICATE_NL`). This is a followup PR of #27935.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added unit tests.

Closes #28013 from maropu/SPARK-25121-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit da49f50621b201822a64358624c19be3fae0a855)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/ResolveHintsSuite.scala | 93 ++++++++++---------
 1 file changed, 49 insertions(+), 44 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
index ca7d28401cf2a..d3bd5d07a0932 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveHintsSuite.scala
@@ -242,51 +242,56 @@ class ResolveHintsSuite extends AnalysisTest {
       caseSensitive = false)
   }
 
-  test("Supports multi-part table names for broadcast hint resolution") {
-    // local temp table (single-part identifier case)
-    checkAnalysis(
-      UnresolvedHint("MAPJOIN", Seq("table", "table2"),
-        table("TaBlE").join(table("TaBlE2"))),
-      Join(
-        ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
-        ResolvedHint(testRelation2, HintInfo(strategy = Some(BROADCAST))),
-        Inner,
-        None,
-        JoinHint.NONE),
-      caseSensitive = false)
-
-    checkAnalysis(
-      UnresolvedHint("MAPJOIN", Seq("TaBlE", "table2"),
-        table("TaBlE").join(table("TaBlE2"))),
-      Join(
-        ResolvedHint(testRelation, HintInfo(strategy = Some(BROADCAST))),
-        testRelation2,
-        Inner,
-        None,
-        JoinHint.NONE),
-      caseSensitive = true)
+  test("Supports multi-part table names for join strategy hint resolution") {
+    Seq(("MAPJOIN", BROADCAST),
+        ("MERGEJOIN", SHUFFLE_MERGE),
+        ("SHUFFLE_HASH", SHUFFLE_HASH),
+        ("SHUFFLE_REPLICATE_NL", SHUFFLE_REPLICATE_NL)).foreach { case (hintName, st) =>
+      // local temp table (single-part identifier case)
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("table", "table2"),
+          table("TaBlE").join(table("TaBlE2"))),
+        Join(
+          ResolvedHint(testRelation, HintInfo(strategy = Some(st))),
+          ResolvedHint(testRelation2, HintInfo(strategy = Some(st))),
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = false)
 
-    // global temp table (multi-part identifier case)
-    checkAnalysis(
-      UnresolvedHint("MAPJOIN", Seq("GlOBal_TeMP.table4", "table5"),
-        table("global_temp", "table4").join(table("global_temp", "table5"))),
-      Join(
-        ResolvedHint(testRelation4, HintInfo(strategy = Some(BROADCAST))),
-        ResolvedHint(testRelation5, HintInfo(strategy = Some(BROADCAST))),
-        Inner,
-        None,
-        JoinHint.NONE),
-      caseSensitive = false)
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("TaBlE", "table2"),
+          table("TaBlE").join(table("TaBlE2"))),
+        Join(
+          ResolvedHint(testRelation, HintInfo(strategy = Some(st))),
+          testRelation2,
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = true)
+
+      // global temp table (multi-part identifier case)
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("GlOBal_TeMP.table4", "table5"),
+          table("global_temp", "table4").join(table("global_temp", "table5"))),
+        Join(
+          ResolvedHint(testRelation4, HintInfo(strategy = Some(st))),
+          ResolvedHint(testRelation5, HintInfo(strategy = Some(st))),
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = false)
 
-    checkAnalysis(
-      UnresolvedHint("MAPJOIN", Seq("global_temp.TaBlE4", "table5"),
-        table("global_temp", "TaBlE4").join(table("global_temp", "TaBlE5"))),
-      Join(
-        ResolvedHint(testRelation4, HintInfo(strategy = Some(BROADCAST))),
-        testRelation5,
-        Inner,
-        None,
-        JoinHint.NONE),
-      caseSensitive = true)
+      checkAnalysis(
+        UnresolvedHint(hintName, Seq("global_temp.TaBlE4", "table5"),
+          table("global_temp", "TaBlE4").join(table("global_temp", "TaBlE5"))),
+        Join(
+          ResolvedHint(testRelation4, HintInfo(strategy = Some(st))),
+          testRelation5,
+          Inner,
+          None,
+          JoinHint.NONE),
+        caseSensitive = true)
+    }
   }
 }

From 2e4a8cb1d53f6338789d554e04dd1a22d60d5405 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 25 Mar 2020 09:25:55 -0700
Subject: [PATCH 0339/1280] [SPARK-31147][SQL] Forbid CHAR type in
 non-Hive-Serde tables

Spark introduced CHAR type for hive compatibility but it only works for hive tables. CHAR type is never documented and is treated as STRING type for non-Hive tables.

However, this leads to confusing behaviors

**Apache Spark 3.0.0-preview2**
```
spark-sql> CREATE TABLE t(a CHAR(3));

spark-sql> INSERT INTO TABLE t SELECT 'a ';

spark-sql> SELECT a, length(a) FROM t;
a 	2
```

**Apache Spark 2.4.5**
```
spark-sql> CREATE TABLE t(a CHAR(3));

spark-sql> INSERT INTO TABLE t SELECT 'a ';

spark-sql> SELECT a, length(a) FROM t;
a  	3
```

According to the SQL standard, `CHAR(3)` should guarantee all the values are of length 3. Since `CHAR(3)` is treated as STRING so Spark doesn't guarantee it.

This PR forbids CHAR type in non-Hive tables as it's not supported correctly.

avoid confusing/wrong behavior

yes, now users can't create/alter non-Hive tables with CHAR type.

new tests

Closes #27902 from cloud-fan/char.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-migration-guide.md                   | 18 ++---
 .../catalyst/analysis/ResolveCatalogs.scala   |  4 ++
 .../sql/catalyst/parser/AstBuilder.scala      |  4 ++
 .../sql/catalyst/parser/ParseDriver.scala     |  5 ++
 .../sql/catalyst/parser/ParserInterface.scala |  6 ++
 .../sql/connector/catalog/CatalogV2Util.scala | 19 +++++-
 .../spark/sql/types/HiveStringType.scala      |  7 ++
 .../analysis/ResolveSessionCatalog.scala      | 14 ++++
 .../sql/SparkSessionExtensionSuite.scala      |  3 +
 .../command/PlanResolutionSuite.scala         | 67 ++++++++++++++++---
 .../sql/hive/HiveMetastoreCatalogSuite.scala  |  2 +-
 11 files changed, 129 insertions(+), 20 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index b4e51c29ce4c8..d2773d822e802 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -108,6 +108,8 @@ license: |
 
   - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For generating Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
 
+  - Since Spark 3.0, column of CHAR type is not allowed in non-Hive-Serde tables, and CREATE/ALTER TABLE commands will fail if CHAR type is detected. Please use STRING type instead. In Spark version 2.4 and earlier, CHAR type is treated as STRING type and the length parameter is simply ignored.
+
 ### UDFs and Built-in Functions
 
   - Since Spark 3.0, the `date_add` and `date_sub` functions only accept int, smallint, tinyint as the 2nd argument, fractional and non-literal string are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), 12.34)` will cause `AnalysisException`. Note that, string literals are still allowed, but Spark will throw Analysis Exception if the string content is not a valid integer. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
@@ -964,29 +966,29 @@ Spark SQL supports the vast majority of Hive features, such as:
 * Unions
 * Sub-queries
   * Sub-queries in the FROM Clause
-  
+
     ```SELECT col FROM (SELECT a + b AS col FROM t1) t2```
   * Sub-queries in WHERE Clause
     * Correlated or non-correlated IN and NOT IN statement in WHERE Clause
-    
+
       ```
       SELECT col FROM t1 WHERE col IN (SELECT a FROM t2 WHERE t1.a = t2.a)
       SELECT col FROM t1 WHERE col IN (SELECT a FROM t2)
       ```
     * Correlated or non-correlated EXISTS and NOT EXISTS statement in WHERE Clause
-    
+
       ```
       SELECT col FROM t1 WHERE EXISTS (SELECT t2.a FROM t2 WHERE t1.a = t2.a AND t2.a > 10)
       SELECT col FROM t1 WHERE EXISTS (SELECT t2.a FROM t2 WHERE t2.a > 10)
       ```
     * Non-correlated IN and NOT IN statement in JOIN Condition
-    
+
       ```SELECT t1.col FROM t1 JOIN t2 ON t1.a = t2.a AND t1.a IN (SELECT a FROM t3)```
-   
+
     * Non-correlated EXISTS and NOT EXISTS statement in JOIN Condition
-       
-      ```SELECT t1.col FROM t1 JOIN t2 ON t1.a = t2.a AND EXISTS (SELECT * FROM t3 WHERE t3.a > 10)``` 
-       
+
+      ```SELECT t1.col FROM t1 JOIN t2 ON t1.a = t2.a AND EXISTS (SELECT * FROM t3 WHERE t3.a > 10)```
+
 * Sampling
 * Explain
 * Partitioned tables including dynamic partition insertion
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 88a3c0a73a10b..895dfbb77d642 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -34,6 +34,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
   override def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
     case AlterTableAddColumnsStatement(
          nameParts @ NonSessionCatalogAndTable(catalog, tbl), cols) =>
+      cols.foreach(c => failCharType(c.dataType))
       val changes = cols.map { col =>
         TableChange.addColumn(
           col.name.toArray,
@@ -46,6 +47,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case a @ AlterTableAlterColumnStatement(
          nameParts @ NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _) =>
+      a.dataType.foreach(failCharType)
       val colName = a.column.toArray
       val typeChange = a.dataType.map { newDataType =>
         TableChange.updateColumnType(colName, newDataType)
@@ -121,6 +123,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case c @ CreateTableStatement(
          NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+      assertNoCharTypeInSchema(c.tableSchema)
       CreateV2Table(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -147,6 +150,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
 
     case c @ ReplaceTableStatement(
          NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+      assertNoCharTypeInSchema(c.tableSchema)
       ReplaceTable(
         catalog.asTableCatalog,
         tbl.asIdentifier,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index b94fd1dd1c0ef..09d316b683b4f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -104,6 +104,10 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     withOrigin(ctx)(StructType(visitColTypeList(ctx.colTypeList)))
   }
 
+  def parseRawDataType(ctx: SingleDataTypeContext): DataType = withOrigin(ctx) {
+    typedVisit[DataType](ctx.dataType())
+  }
+
   /* ********************************************************************************************
    * Plan parsing
    * ******************************************************************************************** */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
index c96f2da9a5289..590193bddafb5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
@@ -40,6 +40,11 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     astBuilder.visitSingleDataType(parser.singleDataType())
   }
 
+  /** Similar to `parseDataType`, but without CHAR/VARCHAR replacement. */
+  override def parseRawDataType(sqlText: String): DataType = parse(sqlText) { parser =>
+    astBuilder.parseRawDataType(parser.singleDataType())
+  }
+
   /** Creates Expression for a given SQL string. */
   override def parseExpression(sqlText: String): Expression = parse(sqlText) { parser =>
     astBuilder.visitSingleExpression(parser.singleExpression())
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
index 77e357ad073da..d724933bc1029 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserInterface.scala
@@ -70,4 +70,10 @@ trait ParserInterface {
    */
   @throws[ParseException]("Text cannot be parsed to a DataType")
   def parseDataType(sqlText: String): DataType
+
+  /**
+   * Parse a string to a raw [[DataType]] without CHAR/VARCHAR replacement.
+   */
+  @throws[ParseException]("Text cannot be parsed to a DataType")
+  def parseRawDataType(sqlText: String): DataType
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
index ff63201b02b64..e1f329352592f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala
@@ -22,11 +22,13 @@ import java.util.Collections
 
 import scala.collection.JavaConverters._
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.{NamedRelation, NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException, UnresolvedV2Relation}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.catalyst.plans.logical.AlterTable
 import org.apache.spark.sql.connector.catalog.TableChange._
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
-import org.apache.spark.sql.types.{ArrayType, MapType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, DataType, HIVE_TYPE_STRING, HiveStringType, MapType, StructField, StructType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.Utils
 
@@ -329,4 +331,19 @@ private[sql] object CatalogV2Util {
       .getOrElse(catalogManager.v2SessionCatalog)
       .asTableCatalog
   }
+
+  def failCharType(dt: DataType): Unit = {
+    if (HiveStringType.containsCharType(dt)) {
+      throw new AnalysisException(
+        "Cannot use CHAR type in non-Hive-Serde tables, please use STRING type instead.")
+    }
+  }
+
+  def assertNoCharTypeInSchema(schema: StructType): Unit = {
+    schema.foreach { f =>
+      if (f.metadata.contains(HIVE_TYPE_STRING)) {
+        failCharType(CatalystSqlParser.parseRawDataType(f.metadata.getString(HIVE_TYPE_STRING)))
+      }
+    }
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
index 4eb3226c5786e..a29f49ad14a77 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/HiveStringType.scala
@@ -53,6 +53,13 @@ object HiveStringType {
     case _: HiveStringType => StringType
     case _ => dt
   }
+
+  def containsCharType(dt: DataType): Boolean = dt match {
+    case ArrayType(et, _) => containsCharType(et)
+    case MapType(kt, vt, _) => containsCharType(kt) || containsCharType(vt)
+    case StructType(fields) => fields.exists(f => containsCharType(f.dataType))
+    case _ => dt.isInstanceOf[CharType]
+  }
 }
 
 /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 5330612dd8162..7b88fc6008246 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -50,6 +50,9 @@ class ResolveSessionCatalog(
          nameParts @ SessionCatalogAndTable(catalog, tbl), cols) =>
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
+          if (!DDLUtils.isHiveTable(v1Table.v1Table)) {
+            cols.foreach(c => failCharType(c.dataType))
+          }
           cols.foreach { c =>
             assertTopLevelColumn(c.name, "AlterTableAddColumnsCommand")
             if (!c.nullable) {
@@ -59,6 +62,7 @@ class ResolveSessionCatalog(
           }
           AlterTableAddColumnsCommand(tbl.asTableIdentifier, cols.map(convertToStructField))
       }.getOrElse {
+        cols.foreach(c => failCharType(c.dataType))
         val changes = cols.map { col =>
           TableChange.addColumn(
             col.name.toArray,
@@ -74,6 +78,10 @@ class ResolveSessionCatalog(
          nameParts @ SessionCatalogAndTable(catalog, tbl), _, _, _, _, _) =>
       loadTable(catalog, tbl.asIdentifier).collect {
         case v1Table: V1Table =>
+          if (!DDLUtils.isHiveTable(v1Table.v1Table)) {
+            a.dataType.foreach(failCharType)
+          }
+
           if (a.column.length > 1) {
             throw new AnalysisException(
               "ALTER COLUMN with qualified column is only supported with v2 tables.")
@@ -111,6 +119,7 @@ class ResolveSessionCatalog(
             builder.build())
           AlterTableChangeColumnCommand(tbl.asTableIdentifier, colName, newColumn)
       }.getOrElse {
+        a.dataType.foreach(failCharType)
         val colName = a.column.toArray
         val typeChange = a.dataType.map { newDataType =>
           TableChange.updateColumnType(colName, newDataType)
@@ -237,12 +246,16 @@ class ResolveSessionCatalog(
          SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
       val provider = c.provider.getOrElse(conf.defaultDataSourceName)
       if (!isV2Provider(provider)) {
+        if (!DDLUtils.isHiveTable(Some(provider))) {
+          assertNoCharTypeInSchema(c.tableSchema)
+        }
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, c.tableSchema,
           c.partitioning, c.bucketSpec, c.properties, provider, c.options, c.location,
           c.comment, c.ifNotExists)
         val mode = if (c.ifNotExists) SaveMode.Ignore else SaveMode.ErrorIfExists
         CreateTable(tableDesc, mode, None)
       } else {
+        assertNoCharTypeInSchema(c.tableSchema)
         CreateV2Table(
           catalog.asTableCatalog,
           tbl.asIdentifier,
@@ -286,6 +299,7 @@ class ResolveSessionCatalog(
       if (!isV2Provider(provider)) {
         throw new AnalysisException("REPLACE TABLE is only supported with v2 tables.")
       } else {
+        assertNoCharTypeInSchema(c.tableSchema)
         ReplaceTable(
           catalog.asTableCatalog,
           tbl.asIdentifier,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
index 99ea95089d71c..2d48f8df23e65 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
@@ -327,6 +327,9 @@ case class MyParser(spark: SparkSession, delegate: ParserInterface) extends Pars
 
   override def parseDataType(sqlText: String): DataType =
     delegate.parseDataType(sqlText)
+
+  override def parseRawDataType(sqlText: String): DataType =
+    delegate.parseRawDataType(sqlText)
 }
 
 object MyExtensions {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 9df3c336b2eee..2e88ea9299c00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -37,11 +37,13 @@ import org.apache.spark.sql.connector.catalog.TableChange.{UpdateColumnComment,
 import org.apache.spark.sql.execution.datasources.CreateTable
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.SimpleScanSource
 import org.apache.spark.sql.types.{CharType, DoubleType, HIVE_TYPE_STRING, IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType}
 
 class PlanResolutionSuite extends AnalysisTest {
   import CatalystSqlParser._
 
+  private val v1Format = classOf[SimpleScanSource].getName
   private val v2Format = classOf[FakeV2Provider].getName
 
   private val table: Table = {
@@ -61,6 +63,15 @@ class PlanResolutionSuite extends AnalysisTest {
     val t = mock(classOf[CatalogTable])
     when(t.schema).thenReturn(new StructType().add("i", "int").add("s", "string"))
     when(t.tableType).thenReturn(CatalogTableType.MANAGED)
+    when(t.provider).thenReturn(Some(v1Format))
+    V1Table(t)
+  }
+
+  private val v1HiveTable: V1Table = {
+    val t = mock(classOf[CatalogTable])
+    when(t.schema).thenReturn(new StructType().add("i", "int").add("s", "string"))
+    when(t.tableType).thenReturn(CatalogTableType.MANAGED)
+    when(t.provider).thenReturn(Some("hive"))
     V1Table(t)
   }
 
@@ -83,6 +94,7 @@ class PlanResolutionSuite extends AnalysisTest {
       invocation.getArgument[Identifier](0).name match {
         case "v1Table" => v1Table
         case "v1Table1" => v1Table
+        case "v1HiveTable" => v1HiveTable
         case "v2Table" => table
         case "v2Table1" => table
         case "v2TableWithAcceptAnySchemaCapability" => tableWithAcceptAnySchemaCapability
@@ -1046,15 +1058,6 @@ class PlanResolutionSuite extends AnalysisTest {
           }
           assert(e2.getMessage.contains(
             "ALTER COLUMN with qualified column is only supported with v2 tables"))
-
-          val sql5 = s"ALTER TABLE $tblName ALTER COLUMN i TYPE char(1)"
-          val builder = new MetadataBuilder
-          builder.putString(HIVE_TYPE_STRING, CharType(1).catalogString)
-          val newColumnWithCleanedType = StructField("i", StringType, true, builder.build())
-          val expected5 = AlterTableChangeColumnCommand(
-            tableIdent, "i", newColumnWithCleanedType)
-          val parsed5 = parseAndResolve(sql5)
-          comparePlans(parsed5, expected5)
         } else {
           parsed1 match {
             case AlterTable(_, _, _: DataSourceV2Relation, changes) =>
@@ -1071,6 +1074,15 @@ class PlanResolutionSuite extends AnalysisTest {
           }
         }
     }
+
+    val sql = s"ALTER TABLE v1HiveTable ALTER COLUMN i TYPE char(1)"
+    val builder = new MetadataBuilder
+    builder.putString(HIVE_TYPE_STRING, CharType(1).catalogString)
+    val newColumnWithCleanedType = StructField("i", StringType, true, builder.build())
+    val expected = AlterTableChangeColumnCommand(
+      TableIdentifier("v1HiveTable", Some("default")), "i", newColumnWithCleanedType)
+    val parsed = parseAndResolve(sql)
+    comparePlans(parsed, expected)
   }
 
   test("alter table: alter column action is not specified") {
@@ -1506,6 +1518,42 @@ class PlanResolutionSuite extends AnalysisTest {
       case l => fail("Expected unresolved MergeIntoTable, but got:\n" + l.treeString)
     }
   }
+
+  test("SPARK-31147: forbid CHAR type in non-Hive tables") {
+    def checkFailure(t: String, provider: String): Unit = {
+      val types = Seq(
+        "CHAR(2)",
+        "ARRAY<CHAR(2)>",
+        "MAP<INT, CHAR(2)>",
+        "MAP<CHAR(2), INT>",
+        "STRUCT<s: CHAR(2)>")
+      types.foreach { tpe =>
+        intercept[AnalysisException] {
+          parseAndResolve(s"CREATE TABLE $t(col $tpe) USING $provider")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"REPLACE TABLE $t(col $tpe) USING $provider")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"CREATE OR REPLACE TABLE $t(col $tpe) USING $provider")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"ALTER TABLE $t ADD COLUMN col $tpe")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"ALTER TABLE $t ADD COLUMN col $tpe")
+        }
+        intercept[AnalysisException] {
+          parseAndResolve(s"ALTER TABLE $t ALTER COLUMN col TYPE $tpe")
+        }
+      }
+    }
+
+    checkFailure("v1Table", v1Format)
+    checkFailure("v2Table", v2Format)
+    checkFailure("testcat.tab", "foo")
+  }
+
   // TODO: add tests for more commands.
 }
 
@@ -1515,4 +1563,3 @@ object AsDataSourceV2Relation {
     case _ => None
   }
 }
-
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 9cd56f1745afc..95e99c653d6f6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -97,7 +97,7 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton with SQLTestUtils {
           |c22 map<int,char(10)>,
           |c23 struct<a:int,b:int>,
           |c24 struct<c:varchar(10),d:int>
-          |)
+          |) USING hive
         """.stripMargin)
 
       val schema = hiveClient.getTable("default", "t").schema

From 53221cda408e9be5d0d2ff5946c200cb43647dd9 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 25 Mar 2020 12:38:15 -0700
Subject: [PATCH 0340/1280] [SPARK-31244][K8S][TEST] Use Minio instead of Ceph
 in K8S DepsTestsSuite

### What changes were proposed in this pull request?

This PR (SPARK-31244) replaces `Ceph` with `Minio` in K8S `DepsTestSuite`.

### Why are the changes needed?

Currently, `DepsTestsSuite` is using `ceph` for S3 storage. However, the used version and all new releases are broken on new `minikube` releases. We had better use more robust and small one.

```
$ minikube version
minikube version: v1.8.2

$ minikube -p minikube docker-env | source

$ docker run -it --rm -e NETWORK_AUTO_DETECT=4 -e RGW_FRONTEND_PORT=8000 -e SREE_PORT=5001 -e CEPH_DEMO_UID=nano -e CEPH_DAEMON=demo ceph/daemon:v4.0.3-stable-4.0-nautilus-centos-7-x86_64 /bin/sh
2020-03-25 04:26:21  /opt/ceph-container/bin/entrypoint.sh: ERROR- it looks like we have not been able to discover the network settings

$ docker run -it --rm -e NETWORK_AUTO_DETECT=4 -e RGW_FRONTEND_PORT=8000 -e SREE_PORT=5001 -e CEPH_DEMO_UID=nano -e CEPH_DAEMON=demo ceph/daemon:v4.0.11-stable-4.0-nautilus-centos-7 /bin/sh
2020-03-25 04:20:30  /opt/ceph-container/bin/entrypoint.sh: ERROR- it looks like we have not been able to discover the network settings
```

Also, the image size is unnecessarily big (almost `1GB`) and growing while `minio` is `55.8MB` with the same features.
```
$ docker images | grep ceph
ceph/daemon v4.0.3-stable-4.0-nautilus-centos-7-x86_64 a6a05ccdf924 6 months ago 852MB
ceph/daemon v4.0.11-stable-4.0-nautilus-centos-7       87f695550d8e 12 hours ago 901MB

$ docker images | grep minio
minio/minio latest                                     95c226551ea6 5 days ago   55.8MB
```

### Does this PR introduce any user-facing change?

No. (This is a test case change)

### How was this patch tested?

Pass the existing Jenkins K8s integration test job and test with the latest minikube.
```
$ minikube version
minikube version: v1.8.2

$ kubectl version --short
Client Version: v1.17.4
Server Version: v1.17.4

$ NO_MANUAL=1 ./dev/make-distribution.sh --r --pip --tgz -Pkubernetes
$ resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh --spark-tgz $PWD/spark-*.tgz
...
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- All pods have the same service account by default
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
- Start pod creation from template
- PVs with local storage *** FAILED *** // This is irrelevant to this PR.
- Launcher client dependencies          // This is the fixed test case by this PR.
- Test basic decommissioning
- Run SparkR on simple dataframe.R example
Run completed in 12 minutes, 4 seconds.
...
```

The following is the working snapshot of `DepsTestSuite` test.
```
$ kubectl get all -ncf9438dd8a65436686b1196a6b73000f
NAME                                                  READY   STATUS    RESTARTS   AGE
pod/minio-0                                           1/1     Running   0          70s
pod/spark-test-app-8494bddca3754390b9e59a2ef47584eb   1/1     Running   0          55s

NAME                                                 TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
service/minio-s3                                     NodePort    10.109.54.180   <none>        9000:30678/TCP               70s
service/spark-test-app-fd916b711061c7b8-driver-svc   ClusterIP   None            <none>        7078/TCP,7079/TCP,4040/TCP   55s

NAME                     READY   AGE
statefulset.apps/minio   1/1     70s
```

Closes #28015 from dongjoon-hyun/SPARK-31244.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit f206bbde3a8f64650236013d61680faba492d7a4)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../k8s/integrationtest/DepsTestsSuite.scala  | 90 ++++++++-----------
 1 file changed, 36 insertions(+), 54 deletions(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index 414126862ed69..c35aa5cc7a0b1 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -35,17 +35,16 @@ import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube
 private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
   import KubernetesSuite.k8sTestTag
 
-  val cName = "ceph-nano"
+  val cName = "minio"
   val svcName = s"$cName-s3"
-  val bucket = "spark"
-
-  private def getCephContainer(): Container = {
-    val envVars = Map ( "NETWORK_AUTO_DETECT" -> "4",
-      "RGW_FRONTEND_PORT" -> "8000",
-      "SREE_PORT" -> "5001",
-      "CEPH_DEMO_UID" -> "nano",
-      "CEPH_DAEMON" -> "demo",
-      "DEBUG" -> "verbose"
+  val BUCKET = "spark"
+  val ACCESS_KEY = "minio"
+  val SECRET_KEY = "miniostorage"
+
+  private def getMinioContainer(): Container = {
+    val envVars = Map (
+      "MINIO_ACCESS_KEY" -> ACCESS_KEY,
+      "MINIO_SECRET_KEY" -> SECRET_KEY
     ).map( envV =>
       new EnvVarBuilder()
         .withName(envV._1)
@@ -63,13 +62,14 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     ).asJava
 
     new ContainerBuilder()
-      .withImage("ceph/daemon:v4.0.3-stable-4.0-nautilus-centos-7-x86_64")
+      .withImage("minio/minio:latest")
       .withImagePullPolicy("Always")
       .withName(cName)
+      .withArgs("server", "/data")
       .withPorts(new ContainerPortBuilder()
           .withName(svcName)
           .withProtocol("TCP")
-          .withContainerPort(8000)
+          .withContainerPort(9000)
         .build()
       )
       .withResources(new ResourceRequirementsBuilder()
@@ -81,10 +81,9 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .build()
   }
 
-  // Based on https://github.com/ceph/cn
-  private def setupCephStorage(): Unit = {
-    val labels = Map("app" -> "ceph", "daemon" -> "nano").asJava
-    val cephService = new ServiceBuilder()
+  private def setupMinioStorage(): Unit = {
+    val labels = Map("app" -> "minio").asJava
+    val minioService = new ServiceBuilder()
       .withNewMetadata()
         .withName(svcName)
       .withLabels(labels)
@@ -92,9 +91,9 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .withNewSpec()
         .withPorts(new ServicePortBuilder()
           .withName("https")
-          .withPort(8000)
+          .withPort(9000)
           .withProtocol("TCP")
-          .withTargetPort(new IntOrString(8000))
+          .withTargetPort(new IntOrString(9000))
           .build()
         )
         .withType("NodePort")
@@ -102,7 +101,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .endSpec()
       .build()
 
-    val cephStatefulSet = new StatefulSetBuilder()
+    val minioStatefulSet = new StatefulSetBuilder()
       .withNewMetadata()
         .withName(cName)
         .withLabels(labels)
@@ -110,7 +109,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
       .withNewSpec()
         .withReplicas(1)
         .withNewSelector()
-          .withMatchLabels(Map("app" -> "ceph").asJava)
+          .withMatchLabels(Map("app" -> "minio").asJava)
         .endSelector()
         .withServiceName(cName)
         .withNewTemplate()
@@ -119,7 +118,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
             .withLabels(labels)
            .endMetadata()
           .withNewSpec()
-            .withContainers(getCephContainer())
+            .withContainers(getMinioContainer())
           .endSpec()
         .endTemplate()
       .endSpec()
@@ -128,16 +127,16 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     kubernetesTestComponents
       .kubernetesClient
       .services()
-      .create(cephService)
+      .create(minioService)
 
     kubernetesTestComponents
       .kubernetesClient
       .apps()
       .statefulSets()
-      .create(cephStatefulSet)
+      .create(minioStatefulSet)
   }
 
- private def deleteCephStorage(): Unit = {
+ private def deleteMinioStorage(): Unit = {
     kubernetesTestComponents
       .kubernetesClient
       .apps()
@@ -155,47 +154,30 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
   test("Launcher client dependencies", k8sTestTag, MinikubeTag) {
     val fileName = Utils.createTempFile(FILE_CONTENTS, HOST_PATH)
     try {
-      setupCephStorage()
-      val cephUrlStr = getServiceUrl(svcName)
-      val cephUrl = new URL(cephUrlStr)
-      val cephHost = cephUrl.getHost
-      val cephPort = cephUrl.getPort
+      setupMinioStorage()
+      val minioUrlStr = getServiceUrl(svcName)
+      val minioUrl = new URL(minioUrlStr)
+      val minioHost = minioUrl.getHost
+      val minioPort = minioUrl.getPort
       val examplesJar = Utils.getExamplesJarAbsolutePath(sparkHomeDir)
-      val (accessKey, secretKey) = getCephCredentials()
       sparkAppConf
-        .set("spark.hadoop.fs.s3a.access.key", accessKey)
-        .set("spark.hadoop.fs.s3a.secret.key", secretKey)
+        .set("spark.hadoop.fs.s3a.access.key", ACCESS_KEY)
+        .set("spark.hadoop.fs.s3a.secret.key", SECRET_KEY)
         .set("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
-        .set("spark.hadoop.fs.s3a.endpoint", s"$cephHost:$cephPort")
-        .set("spark.kubernetes.file.upload.path", s"s3a://$bucket")
+        .set("spark.hadoop.fs.s3a.endpoint", s"$minioHost:$minioPort")
+        .set("spark.kubernetes.file.upload.path", s"s3a://$BUCKET")
         .set("spark.files", s"$HOST_PATH/$fileName")
         .set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
         .set("spark.jars.packages", "com.amazonaws:aws-java-sdk:" +
           "1.7.4,org.apache.hadoop:hadoop-aws:2.7.6")
         .set("spark.driver.extraJavaOptions", "-Divy.cache.dir=/tmp -Divy.home=/tmp")
-      createS3Bucket(accessKey, secretKey, cephUrlStr)
+      createS3Bucket(ACCESS_KEY, SECRET_KEY, minioUrlStr)
       runSparkRemoteCheckAndVerifyCompletion(appResource = examplesJar,
         appArgs = Array(fileName),
         timeout = Option(DEPS_TIMEOUT))
     } finally {
       // make sure this always runs
-      deleteCephStorage()
-    }
-  }
-
-  // There isn't a cleaner way to get the credentials
-  // when ceph-nano runs on k8s
-  private def getCephCredentials(): (String, String) = {
-    Eventually.eventually(TIMEOUT, INTERVAL) {
-      val cephPod = kubernetesTestComponents
-        .kubernetesClient
-        .pods()
-        .withName(s"$cName-0")
-        .get()
-      implicit val podName: String = cephPod.getMetadata.getName
-      implicit val components: KubernetesTestComponents = kubernetesTestComponents
-      val contents = Utils.executeCommand("cat", "/nano_user_details")
-    (extractS3Key(contents, "access_key"), extractS3Key(contents, "secret_key"))
+      deleteMinioStorage()
     }
   }
 
@@ -215,10 +197,10 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
         val credentials = new BasicAWSCredentials(accessKey, secretKey)
         val s3client = new AmazonS3Client(credentials)
         s3client.setEndpoint(endPoint)
-        s3client.createBucket(bucket)
+        s3client.createBucket(BUCKET)
       } catch {
         case e: Exception =>
-          throw new SparkException(s"Failed to create bucket $bucket.", e)
+          throw new SparkException(s"Failed to create bucket $BUCKET.", e)
       }
     }
   }

From 49dd32dfe50b2de4d0dc18466a66703aa8ef3de2 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 25 Mar 2020 12:56:29 -0700
Subject: [PATCH 0341/1280] [SPARK-31161][WEBUI][3.0] Refactor the on-click
 timeline action in streagming-page.js

### What changes were proposed in this pull request?

Refactor `streaming-page.js` by making on-click timeline action customizable.
This is a PR backporting `SPARK-31161` to `branch-3.0`

### Why are the changes needed?

In the current implementation, `streaming-page.js` is used from Streaming page and Structured Streaming page but the implementation of the on-click timeline action is strongly dependent on Streamng page.
Structured Streaming page doesn't define the on-click action for now but it's better to remove the dependncy for the future.

Originally, I make this change to fix `SPARK-31128` but #27883 resolved it.
So, now this is just for refactoring.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Manual tests with following code and confirmed there are no regression and no error in the debug console in Firefox.

For Structured Streaming:
```
spark.readStream.format("socket").options(Map("host"->"localhost", "port"->"8765")).load.writeStream.format("console").start
```
And then, visited Structured Streaming page and there were no error in the debug console when I clicked a point in the timeline.

For Spark Streaming:
```
import org.apache.spark.streaming._
val ssc = new StreamingContext(sc, Seconds(1))
ssc.socketTextStream("localhost", 8765)
dstream.foreachRDD(rdd => rdd.foreach(println))
ssc.start
```
And then, visited Streaming page and confirmed scrolling down and hilighting work well and there were no error in the debug console when I clicked a point in the timeline.

Closes #28021 from sarutak/SPARK-31161-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../apache/spark/ui/static/streaming-page.js  | 70 +++++++++++--------
 .../spark/streaming/ui/StreamingPage.scala    |  9 ++-
 2 files changed, 50 insertions(+), 29 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
index 528b9dfd9a4b8..9a01c3ba83696 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/streaming-page.js
@@ -33,6 +33,8 @@ var yValueFormat = d3.format(",.2f");
 
 var unitLabelYOffset = -10;
 
+var onClickTimeline = function() {};
+
 // Show a tooltip "text" for "node"
 function showBootstrapTooltip(node, text) {
     $(node).tooltip({title: text, trigger: "manual", container: "body"});
@@ -44,6 +46,45 @@ function hideBootstrapTooltip(node) {
     $(node).tooltip("destroy");
 }
 
+// Return the function to scroll to the corresponding
+// row on clicking a point of batch in the timeline.
+function getOnClickTimelineFunction() {
+    // If the user click one point in the graphs, jump to the batch row and highlight it. And
+    // recovery the batch row after 3 seconds if necessary.
+    // We need to remember the last clicked batch so that we can recovery it.
+    var lastClickedBatch = null;
+    var lastTimeout = null;
+
+    return function(d) {
+        var batchSelector = $("#batch-" + d.x);
+        // If there is a corresponding batch row, scroll down to it and highlight it.
+        if (batchSelector.length > 0) {
+            if (lastTimeout != null) {
+                window.clearTimeout(lastTimeout);
+            }
+            if (lastClickedBatch != null) {
+                clearBatchRow(lastClickedBatch);
+                lastClickedBatch = null;
+            }
+            lastClickedBatch = d.x;
+            highlightBatchRow(lastClickedBatch);
+            lastTimeout = window.setTimeout(function () {
+                lastTimeout = null;
+                if (lastClickedBatch != null) {
+                    clearBatchRow(lastClickedBatch);
+                    lastClickedBatch = null;
+                }
+            }, 3000); // Clean up after 3 seconds
+
+            var topOffset = batchSelector.offset().top - 15;
+            if (topOffset < 0) {
+                topOffset = 0;
+            }
+            $('html,body').animate({scrollTop: topOffset}, 200);
+        }
+    }
+}
+
 // Register a timeline graph. All timeline graphs should be register before calling any
 // "drawTimeline" so that we can determine the max margin left for all timeline graphs.
 function registerTimeline(minY, maxY) {
@@ -189,34 +230,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
                     .attr("opacity", function(d) { return isFailedBatch(d.x) ? "1" : "0";})
                     .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";});
             })
-            .on("click", function(d) {
-                var batchSelector = $("#batch-" + d.x);
-                // If there is a corresponding batch row, scroll down to it and highlight it.
-                if (batchSelector.length > 0) {
-                    if (lastTimeout != null) {
-                        window.clearTimeout(lastTimeout);
-                    }
-                    if (lastClickedBatch != null) {
-                        clearBatchRow(lastClickedBatch);
-                        lastClickedBatch = null;
-                    }
-                    lastClickedBatch = d.x;
-                    highlightBatchRow(lastClickedBatch);
-                    lastTimeout = window.setTimeout(function () {
-                        lastTimeout = null;
-                        if (lastClickedBatch != null) {
-                            clearBatchRow(lastClickedBatch);
-                            lastClickedBatch = null;
-                        }
-                    }, 3000); // Clean up after 3 seconds
-
-                    var topOffset = batchSelector.offset().top - 15;
-                    if (topOffset < 0) {
-                        topOffset = 0;
-                    }
-                    $('html,body').animate({scrollTop: topOffset}, 200);
-                }
-            });
+            .on("click", onClickTimeline);
 }
 
 /**
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
index 8596aa035391d..bbfb013c8dfdd 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/ui/StreamingPage.scala
@@ -80,9 +80,10 @@ private[ui] class StreamingPage(parent: StreamingTab)
   /** Render the page */
   def render(request: HttpServletRequest): Seq[Node] = {
     val resources = generateLoadResources(request)
+    val onClickTimelineFunc = generateOnClickTimelineFunction()
     val basicInfo = generateBasicInfo()
     val content = resources ++
-      basicInfo ++
+      onClickTimelineFunc ++ basicInfo ++
       listener.synchronized {
         generateStatTable() ++
           generateBatchListTables()
@@ -101,6 +102,12 @@ private[ui] class StreamingPage(parent: StreamingTab)
     // scalastyle:on
   }
 
+  /** Generate html that will set onClickTimeline declared in streaming-page.js */
+  private def generateOnClickTimelineFunction(): Seq[Node] = {
+    val js = "onClickTimeline = getOnClickTimelineFunction();"
+    <script>{Unparsed(js)}</script>
+  }
+
   /** Generate basic information of the streaming program */
   private def generateBasicInfo(): Seq[Node] = {
     val timeSinceStart = System.currentTimeMillis() - startTime

From cc492447a6411bae51f91f408207272e8aeca39b Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 26 Mar 2020 10:48:11 +0900
Subject: [PATCH 0342/1280] [SPARK-31258][BUILD] Pin the avro version in SBT
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

add arvo dep in SparkBuild

### Why are the changes needed?
fix sbt unidoc like https://github.com/apache/spark/pull/28017#issuecomment-603828597
```scala
[warn] Multiple main classes detected.  Run 'show discoveredMainClasses' to see the list
[warn] Multiple main classes detected.  Run 'show discoveredMainClasses' to see the list
[info] Main Scala API documentation to /home/jenkins/workspace/SparkPullRequestBuilder6/target/scala-2.12/unidoc...
[info] Main Java API documentation to /home/jenkins/workspace/SparkPullRequestBuilder6/target/javaunidoc...
[error] /home/jenkins/workspace/SparkPullRequestBuilder6/core/src/main/scala/org/apache/spark/serializer/GenericAvroSerializer.scala:123: value createDatumWriter is not a member of org.apache.avro.generic.GenericData
[error]     writerCache.getOrElseUpdate(schema, GenericData.get.createDatumWriter(schema))
[error]                                                         ^
[info] No documentation generated with unsuccessful compiler run
[error] one error found
```

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

pass jenkins
and verify manually with `sbt dependencyTree`
```scala
 kentyaohulk  ~/spark   dep  build/sbt dependencyTree | grep avro | grep -v Resolving
[info]   +-org.apache.avro:avro-mapred:1.8.2
[info]   | +-org.apache.avro:avro-ipc:1.8.2
[info]   | | +-org.apache.avro:avro:1.8.2
[info]   +-org.apache.avro:avro:1.8.2
[info]   | | +-org.apache.avro:avro:1.8.2
[info] org.apache.spark:spark-avro_2.12:3.1.0-SNAPSHOT [S]
[info]   | | | +-org.apache.avro:avro-mapred:1.8.2
[info]   | | | | +-org.apache.avro:avro-ipc:1.8.2
[info]   | | | | | +-org.apache.avro:avro:1.8.2
[info]   | | | +-org.apache.avro:avro:1.8.2
[info]   | | | | | +-org.apache.avro:avro:1.8.2
```

Closes #28020 from yaooqinn/dep.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 336621e277794ab2e6e917391a928ec662498fcf)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 project/SparkBuild.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 3889013995800..7bd92b3f15b56 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -624,7 +624,8 @@ object DependencyOverrides {
     dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
     dependencyOverrides += "commons-io" % "commons-io" % "2.4",
     dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
-    dependencyOverrides += "jline" % "jline" % "2.14.6")
+    dependencyOverrides += "jline" % "jline" % "2.14.6",
+    dependencyOverrides += "org.apache.avro" % "avro" % "1.8.2")
 }
 
 /**

From 2be0ad897d3dc5895386eb7bc12c2a0e390e47b7 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 26 Mar 2020 12:33:17 +0900
Subject: [PATCH 0343/1280] [SPARK-31231][BUILD][FOLLOW-UP] Set the upper bound
 (before 46.1.0) for setuptools in pip package test

## What changes were proposed in this pull request?
This PR is a followup of apache/spark#27995. Rather then pining setuptools version, it sets upper bound so Python 3.5 with branch-2.4 tests can pass too.

## Why are the changes needed?
To make the CI build stable

## Does this PR introduce any user-facing change?
No, dev-only change.

## How was this patch tested?
Jenkins will test.

Closes #28005 from HyukjinKwon/investigate-pip-packaging-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 178d472e1d9f9f61fa54866d00d0a5b88ee87619)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/run-pip-tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index dc9582a11f549..928d40eb19e2e 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -76,7 +76,7 @@ for python in "${PYTHON_EXECS[@]}"; do
     VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
     rm -rf "$VIRTUALENV_PATH"
     if [ -n "$USE_CONDA" ]; then
-      conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools=46.0.0
+      conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip "setuptools<46.1.0"
       source activate "$VIRTUALENV_PATH"
     else
       mkdir -p "$VIRTUALENV_PATH"

From e0ab0228a716f36e4115a1361dd45bda0fe43beb Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 26 Mar 2020 13:06:21 +0900
Subject: [PATCH 0344/1280] [MINOR][DOCS] Fix some links for python api doc

### What changes were proposed in this pull request?

the link for `partition discovery` is malformed, because for releases, there will contains` /docs/<version>/` in the full URL.

### Why are the changes needed?

fix doc

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

`SKIP_SCALADOC=1 SKIP_RDOC=1 SKIP_SQLDOC=1 jekyll serve` locally verified

Closes #28017 from yaooqinn/doc.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b024a8a69e4ae45c6ded3dd3f9f27e73a0069891)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/functions.py  | 4 ----
 python/pyspark/sql/readwriter.py | 9 ++-------
 python/pyspark/sql/streaming.py  | 5 ++---
 3 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index bbadb54dc0800..61a221ca24b95 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -1143,8 +1143,6 @@ def to_date(col, format=None):
     By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format
     is omitted. Equivalent to ``col.cast("date")``.
 
-    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
     >>> df.select(to_date(df.t).alias('date')).collect()
     [Row(date=datetime.date(1997, 2, 28))]
@@ -1168,8 +1166,6 @@ def to_timestamp(col, format=None):
     By default, it follows casting rules to :class:`pyspark.sql.types.TimestampType` if the format
     is omitted. Equivalent to ``col.cast("timestamp")``.
 
-    .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
     >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['t'])
     >>> df.select(to_timestamp(df.t).alias('dt')).collect()
     [Row(dt=datetime.datetime(1997, 2, 28, 10, 30))]
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index e7ecb3ba9fc76..817978498fcf2 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -253,7 +253,8 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
-        .. _partition discovery: /sql-data-sources-parquet.html#partition-discovery
+        .. _partition discovery:
+          https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
         .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
         >>> df1 = spark.read.json('python/test_support/sql/people.json')
@@ -490,8 +491,6 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
-        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
         >>> df = spark.read.csv('python/test_support/sql/ages.csv')
         >>> df.dtypes
         [('_c0', 'string'), ('_c1', 'string')]
@@ -865,8 +864,6 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
         :param ignoreNullFields: Whether to ignore null fields when generating JSON objects.
                         If None is set, it uses the default value, ``true``.
 
-        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
         >>> df.write.json(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
@@ -981,8 +978,6 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
         :param lineSep: defines the line separator that should be used for writing. If None is
                         set, it uses the default value, ``\\n``. Maximum length is 1 character.
 
-        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
         >>> df.write.csv(os.path.join(tempfile.mkdtemp(), 'data'))
         """
         self.mode(mode)
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index a83167882a8d7..a5e86466579c4 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -489,7 +489,8 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
-        .. _partition discovery: /sql-data-sources-parquet.html#partition-discovery
+        .. _partition discovery:
+          https://spark.apache.org/docs/latest/sql-data-sources-parquet.html#partition-discovery
         .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
         >>> json_sdf = spark.readStream.json(tempfile.mkdtemp(), schema = sdf_schema)
@@ -725,8 +726,6 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param recursiveFileLookup: recursively scan a directory for files. Using this option
                                     disables `partition discovery`_.
 
-        .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
         >>> csv_sdf = spark.readStream.csv(tempfile.mkdtemp(), schema = sdf_schema)
         >>> csv_sdf.isStreaming
         True

From a9a185c918678285838599b6d05087212d727c06 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Thu, 26 Mar 2020 12:37:48 +0800
Subject: [PATCH 0345/1280] [SPARK-30623][CORE] Spark external shuffle allow
 disable of separate event loop group

### What changes were proposed in this pull request?
Fix the regression caused by #22173.
The original PR changes the logic of handling `ChunkFetchReqeust` from async to sync, that's causes the shuffle benchmark regression. This PR fixes the regression back to the async mode by reusing the config `spark.shuffle.server.chunkFetchHandlerThreadsPercent`.
When the user sets the config, ChunkFetchReqeust will be processed in a separate event loop group, otherwise, the code path is exactly the same as before.

### Why are the changes needed?
Fix the shuffle performance regression described in  https://github.com/apache/spark/pull/22173#issuecomment-572459561

### Does this PR introduce any user-facing change?
Yes, this PR disable the separate event loop for FetchRequest by default.

### How was this patch tested?
Existing UT.

Closes #27665 from xuanyuanking/SPARK-24355-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 0fe203e7032a326ff0c78f6660ab1b09409fe09d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/network/TransportContext.java       | 27 ++++++++-------
 .../server/ChunkFetchRequestHandler.java      | 33 ++++++++++++++-----
 .../server/TransportChannelHandler.java       |  5 ++-
 .../server/TransportRequestHandler.java       | 13 ++++++--
 .../spark/network/util/TransportConf.java     | 13 ++++++--
 .../ChunkFetchRequestHandlerSuite.java        |  4 +--
 .../network/TransportRequestHandlerSuite.java |  4 +--
 7 files changed, 66 insertions(+), 33 deletions(-)

diff --git a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java
index d99b9bdbce392..a0de9df1986f5 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/TransportContext.java
@@ -123,7 +123,7 @@ public TransportContext(
 
     if (conf.getModuleName() != null &&
         conf.getModuleName().equalsIgnoreCase("shuffle") &&
-        !isClientOnly) {
+        !isClientOnly && conf.separateChunkFetchRequest()) {
       chunkFetchWorkers = NettyUtils.createEventLoop(
           IOMode.valueOf(conf.ioMode()),
           conf.chunkFetchHandlerThreads(),
@@ -187,8 +187,6 @@ public TransportChannelHandler initializePipeline(
       RpcHandler channelRpcHandler) {
     try {
       TransportChannelHandler channelHandler = createChannelHandler(channel, channelRpcHandler);
-      ChunkFetchRequestHandler chunkFetchHandler =
-        createChunkFetchHandler(channelHandler, channelRpcHandler);
       ChannelPipeline pipeline = channel.pipeline()
         .addLast("encoder", ENCODER)
         .addLast(TransportFrameDecoder.HANDLER_NAME, NettyUtils.createFrameDecoder())
@@ -200,6 +198,9 @@ public TransportChannelHandler initializePipeline(
         .addLast("handler", channelHandler);
       // Use a separate EventLoopGroup to handle ChunkFetchRequest messages for shuffle rpcs.
       if (chunkFetchWorkers != null) {
+        ChunkFetchRequestHandler chunkFetchHandler = new ChunkFetchRequestHandler(
+          channelHandler.getClient(), rpcHandler.getStreamManager(),
+          conf.maxChunksBeingTransferred(), true /* syncModeEnabled */);
         pipeline.addLast(chunkFetchWorkers, "chunkFetchHandler", chunkFetchHandler);
       }
       return channelHandler;
@@ -217,19 +218,17 @@ public TransportChannelHandler initializePipeline(
   private TransportChannelHandler createChannelHandler(Channel channel, RpcHandler rpcHandler) {
     TransportResponseHandler responseHandler = new TransportResponseHandler(channel);
     TransportClient client = new TransportClient(channel, responseHandler);
+    boolean separateChunkFetchRequest = conf.separateChunkFetchRequest();
+    ChunkFetchRequestHandler chunkFetchRequestHandler = null;
+    if (!separateChunkFetchRequest) {
+      chunkFetchRequestHandler = new ChunkFetchRequestHandler(
+        client, rpcHandler.getStreamManager(),
+        conf.maxChunksBeingTransferred(), false /* syncModeEnabled */);
+    }
     TransportRequestHandler requestHandler = new TransportRequestHandler(channel, client,
-      rpcHandler, conf.maxChunksBeingTransferred());
+      rpcHandler, conf.maxChunksBeingTransferred(), chunkFetchRequestHandler);
     return new TransportChannelHandler(client, responseHandler, requestHandler,
-      conf.connectionTimeoutMs(), closeIdleConnections, this);
-  }
-
-  /**
-   * Creates the dedicated ChannelHandler for ChunkFetchRequest messages.
-   */
-  private ChunkFetchRequestHandler createChunkFetchHandler(TransportChannelHandler channelHandler,
-      RpcHandler rpcHandler) {
-    return new ChunkFetchRequestHandler(channelHandler.getClient(),
-      rpcHandler.getStreamManager(), conf.maxChunksBeingTransferred());
+      conf.connectionTimeoutMs(), separateChunkFetchRequest, closeIdleConnections, this);
   }
 
   public TransportConf getConf() { return conf; }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
index 94412c4db559f..82810dacdad84 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/ChunkFetchRequestHandler.java
@@ -55,14 +55,17 @@ public class ChunkFetchRequestHandler extends SimpleChannelInboundHandler<ChunkF
   private final StreamManager streamManager;
   /** The max number of chunks being transferred and not finished yet. */
   private final long maxChunksBeingTransferred;
+  private final boolean syncModeEnabled;
 
   public ChunkFetchRequestHandler(
       TransportClient client,
       StreamManager streamManager,
-      Long maxChunksBeingTransferred) {
+      Long maxChunksBeingTransferred,
+      boolean syncModeEnabled) {
     this.client = client;
     this.streamManager = streamManager;
     this.maxChunksBeingTransferred = maxChunksBeingTransferred;
+    this.syncModeEnabled = syncModeEnabled;
   }
 
   @Override
@@ -76,6 +79,11 @@ protected void channelRead0(
       ChannelHandlerContext ctx,
       final ChunkFetchRequest msg) throws Exception {
     Channel channel = ctx.channel();
+    processFetchRequest(channel, msg);
+  }
+
+  public void processFetchRequest(
+      final Channel channel, final ChunkFetchRequest msg) throws Exception {
     if (logger.isTraceEnabled()) {
       logger.trace("Received req from {} to fetch block {}", getRemoteAddress(channel),
         msg.streamChunkId);
@@ -112,19 +120,26 @@ protected void channelRead0(
    * channel will be handled by the EventLoop the channel is registered to. So even
    * though we are processing the ChunkFetchRequest in a separate thread pool, the actual I/O,
    * which is the potentially blocking call that could deplete server handler threads, is still
-   * being processed by TransportServer's default EventLoopGroup. In order to throttle the max
-   * number of threads that channel I/O for sending response to ChunkFetchRequest, the thread
-   * calling channel.writeAndFlush will wait for the completion of sending response back to
-   * client by invoking await(). This will throttle the rate at which threads from
-   * ChunkFetchRequest dedicated EventLoopGroup submit channel I/O requests to TransportServer's
-   * default EventLoopGroup, thus making sure that we can reserve some threads in
-   * TransportServer's default EventLoopGroup for handling other RPC messages.
+   * being processed by TransportServer's default EventLoopGroup.
+   *
+   * When syncModeEnabled is true, Spark will throttle the max number of threads that channel I/O
+   * for sending response to ChunkFetchRequest, the thread calling channel.writeAndFlush will wait
+   * for the completion of sending response back to client by invoking await(). This will throttle
+   * the rate at which threads from ChunkFetchRequest dedicated EventLoopGroup submit channel I/O
+   * requests to TransportServer's default EventLoopGroup, thus making sure that we can reserve
+   * some threads in TransportServer's default EventLoopGroup for handling other RPC messages.
    */
   private ChannelFuture respond(
       final Channel channel,
       final Encodable result) throws InterruptedException {
     final SocketAddress remoteAddress = channel.remoteAddress();
-    return channel.writeAndFlush(result).await().addListener((ChannelFutureListener) future -> {
+    ChannelFuture channelFuture;
+    if (syncModeEnabled) {
+      channelFuture = channel.writeAndFlush(result).await();
+    } else {
+      channelFuture = channel.writeAndFlush(result);
+    }
+    return channelFuture.addListener((ChannelFutureListener) future -> {
       if (future.isSuccess()) {
         logger.trace("Sent result {} to client {}", result, remoteAddress);
       } else {
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
index 31371f6970ffe..e53a0c1a0852e 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportChannelHandler.java
@@ -58,6 +58,7 @@ public class TransportChannelHandler extends SimpleChannelInboundHandler<Message
   private final TransportRequestHandler requestHandler;
   private final long requestTimeoutNs;
   private final boolean closeIdleConnections;
+  private final boolean skipChunkFetchRequest;
   private final TransportContext transportContext;
 
   public TransportChannelHandler(
@@ -65,12 +66,14 @@ public TransportChannelHandler(
       TransportResponseHandler responseHandler,
       TransportRequestHandler requestHandler,
       long requestTimeoutMs,
+      boolean skipChunkFetchRequest,
       boolean closeIdleConnections,
       TransportContext transportContext) {
     this.client = client;
     this.responseHandler = responseHandler;
     this.requestHandler = requestHandler;
     this.requestTimeoutNs = requestTimeoutMs * 1000L * 1000;
+    this.skipChunkFetchRequest = skipChunkFetchRequest;
     this.closeIdleConnections = closeIdleConnections;
     this.transportContext = transportContext;
   }
@@ -124,7 +127,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
    */
   @Override
   public boolean acceptInboundMessage(Object msg) throws Exception {
-    if (msg instanceof ChunkFetchRequest) {
+    if (skipChunkFetchRequest && msg instanceof ChunkFetchRequest) {
       return false;
     } else {
       return super.acceptInboundMessage(msg);
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
index 0792b58304247..f178928006902 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/TransportRequestHandler.java
@@ -62,16 +62,21 @@ public class TransportRequestHandler extends MessageHandler<RequestMessage> {
   /** The max number of chunks being transferred and not finished yet. */
   private final long maxChunksBeingTransferred;
 
+  /** The dedicated ChannelHandler for ChunkFetchRequest messages. */
+  private final ChunkFetchRequestHandler chunkFetchRequestHandler;
+
   public TransportRequestHandler(
       Channel channel,
       TransportClient reverseClient,
       RpcHandler rpcHandler,
-      Long maxChunksBeingTransferred) {
+      Long maxChunksBeingTransferred,
+      ChunkFetchRequestHandler chunkFetchRequestHandler) {
     this.channel = channel;
     this.reverseClient = reverseClient;
     this.rpcHandler = rpcHandler;
     this.streamManager = rpcHandler.getStreamManager();
     this.maxChunksBeingTransferred = maxChunksBeingTransferred;
+    this.chunkFetchRequestHandler = chunkFetchRequestHandler;
   }
 
   @Override
@@ -97,8 +102,10 @@ public void channelInactive() {
   }
 
   @Override
-  public void handle(RequestMessage request) {
-    if (request instanceof RpcRequest) {
+  public void handle(RequestMessage request) throws Exception {
+    if (request instanceof ChunkFetchRequest) {
+      chunkFetchRequestHandler.processFetchRequest(channel, (ChunkFetchRequest) request);
+    } else if (request instanceof RpcRequest) {
       processRpcRequest((RpcRequest) request);
     } else if (request instanceof OneWayMessage) {
       processOneWayMessage((OneWayMessage) request);
diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index cc0f2919568ac..6c37f9a382376 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -316,7 +316,8 @@ public long maxChunksBeingTransferred() {
 
   /**
    * Percentage of io.serverThreads used by netty to process ChunkFetchRequest.
-   * Shuffle server will use a separate EventLoopGroup to process ChunkFetchRequest messages.
+   * When the config `spark.shuffle.server.chunkFetchHandlerThreadsPercent` is set,
+   * shuffle server will use a separate EventLoopGroup to process ChunkFetchRequest messages.
    * Although when calling the async writeAndFlush on the underlying channel to send
    * response back to client, the I/O on the channel is still being handled by
    * {@link org.apache.spark.network.server.TransportServer}'s default EventLoopGroup
@@ -339,12 +340,20 @@ public int chunkFetchHandlerThreads() {
       return 0;
     }
     int chunkFetchHandlerThreadsPercent =
-      conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 100);
+      Integer.parseInt(conf.get("spark.shuffle.server.chunkFetchHandlerThreadsPercent"));
     int threads =
       this.serverThreads() > 0 ? this.serverThreads() : 2 * NettyRuntime.availableProcessors();
     return (int) Math.ceil(threads * (chunkFetchHandlerThreadsPercent / 100.0));
   }
 
+  /**
+   * Whether to use a separate EventLoopGroup to process ChunkFetchRequest messages, it is decided
+   * by the config `spark.shuffle.server.chunkFetchHandlerThreadsPercent` is set or not.
+   */
+  public boolean separateChunkFetchRequest() {
+    return conf.getInt("spark.shuffle.server.chunkFetchHandlerThreadsPercent", 0) > 0;
+  }
+
   /**
    * Whether to use the old protocol while doing the shuffle block fetching.
    * It is only enabled while we need the compatibility in the scenario of new spark version
diff --git a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java
index 7e30ed4048ca8..addb4ff332746 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchRequestHandlerSuite.java
@@ -22,7 +22,6 @@
 import java.util.List;
 
 import io.netty.channel.Channel;
-import org.apache.spark.network.server.ChunkFetchRequestHandler;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -33,6 +32,7 @@
 import org.apache.spark.network.buffer.ManagedBuffer;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.protocol.*;
+import org.apache.spark.network.server.ChunkFetchRequestHandler;
 import org.apache.spark.network.server.NoOpRpcHandler;
 import org.apache.spark.network.server.OneForOneStreamManager;
 import org.apache.spark.network.server.RpcHandler;
@@ -68,7 +68,7 @@ public void handleChunkFetchRequest() throws Exception {
     long streamId = streamManager.registerStream("test-app", managedBuffers.iterator(), channel);
     TransportClient reverseClient = mock(TransportClient.class);
     ChunkFetchRequestHandler requestHandler = new ChunkFetchRequestHandler(reverseClient,
-      rpcHandler.getStreamManager(), 2L);
+      rpcHandler.getStreamManager(), 2L, false);
 
     RequestMessage request0 = new ChunkFetchRequest(new StreamChunkId(streamId, 0));
     requestHandler.channelRead(context, request0);
diff --git a/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java b/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java
index a43a659048686..0a6447176237a 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/TransportRequestHandlerSuite.java
@@ -39,7 +39,7 @@
 public class TransportRequestHandlerSuite {
 
   @Test
-  public void handleStreamRequest() {
+  public void handleStreamRequest() throws Exception {
     RpcHandler rpcHandler = new NoOpRpcHandler();
     OneForOneStreamManager streamManager = (OneForOneStreamManager) (rpcHandler.getStreamManager());
     Channel channel = mock(Channel.class);
@@ -66,7 +66,7 @@ public void handleStreamRequest() {
 
     TransportClient reverseClient = mock(TransportClient.class);
     TransportRequestHandler requestHandler = new TransportRequestHandler(channel, reverseClient,
-      rpcHandler, 2L);
+      rpcHandler, 2L, null);
 
     RequestMessage request0 = new StreamRequest(String.format("%d_%d", streamId, 0));
     requestHandler.handle(request0);

From 6b568d77fc0284b0b8a26b16fd105a7c5f54f874 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 26 Mar 2020 13:36:00 +0800
Subject: [PATCH 0346/1280] [SPARK-31237][SQL][TESTS] Replace 3-letter time
 zones by zone offsets

In the PR, I propose to add a few `ZoneId` constant values to the `DateTimeTestUtils` object, and reuse the constants in tests. Proposed the following constants:
- PST = -08:00
- UTC = +00:00
- CEST = +02:00
- CET = +01:00
- JST = +09:00
- MIT = -09:30
- LA = America/Los_Angeles

All proposed constant values (except `LA`) are initialized by zone offsets according to their definitions. This will allow to avoid:
- Using of 3-letter time zones that have been already deprecated in JDK, see _Three-letter time zone IDs_ in https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html
- Incorrect mapping of 3-letter time zones to zone offsets, see SPARK-31237. For example, `PST` is mapped to `America/Los_Angeles` instead of the `-08:00` zone offset.

Also this should improve stability and maintainability of test suites.

No

By running affected test suites.

Closes #28001 from MaxGekk/replace-pst.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit cec9604eaec2e5ff17e705ed60565bd7506c6374)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalyst/csv/CSVInferSchemaSuite.scala    |  28 +--
 .../catalyst/csv/UnivocityParserSuite.scala   |  45 +++--
 .../sql/catalyst/expressions/CastSuite.scala  |  27 +--
 .../expressions/CodeGenerationSuite.scala     |   5 +-
 .../CollectionExpressionsSuite.scala          |   3 +-
 .../expressions/CsvExpressionsSuite.scala     |  47 +++--
 .../expressions/DateExpressionsSuite.scala    | 177 +++++++++--------
 .../expressions/JsonExpressionsSuite.scala    |  77 ++++----
 .../sql/catalyst/util/DateTimeTestUtils.scala |  17 +-
 .../catalyst/util/DateTimeUtilsSuite.scala    | 178 +++++++++---------
 .../spark/sql/util/ArrowUtilsSuite.scala      |   3 +-
 .../sql/util/TimestampFormatterSuite.scala    |  11 +-
 .../spark/sql/DataFrameFunctionsSuite.scala   |   3 +-
 .../spark/sql/DataFramePivotSuite.scala       |   2 +-
 .../org/apache/spark/sql/DataFrameSuite.scala |   4 +-
 .../apache/spark/sql/DateFunctionsSuite.scala |  21 ++-
 .../execution/datasources/csv/CSVSuite.scala  |   4 +-
 .../datasources/json/JsonSuite.scala          |  10 +-
 .../ParquetPartitionDiscoverySuite.scala      |  10 +-
 .../spark/sql/internal/SQLConfSuite.scala     |   5 +-
 .../sql/sources/PartitionedWriteSuite.scala   |   6 +-
 .../streaming/EventTimeWatermarkSuite.scala   |   3 +-
 22 files changed, 346 insertions(+), 340 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
index ee73da33a1a90..b014eb92fae50 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.types._
 class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
 
   test("String fields types are inferred correctly from null types") {
-    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "GMT")
+    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "") == NullType)
@@ -48,7 +48,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("String fields types are inferred correctly from other types") {
-    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "GMT")
+    val options = new CSVOptions(Map("timestampFormat" -> "yyyy-MM-dd HH:mm:ss"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(LongType, "1.0") == DoubleType)
@@ -69,18 +69,18 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Timestamp field types are inferred correctly via custom data format") {
-    var options = new CSVOptions(Map("timestampFormat" -> "yyyy-mm"), false, "GMT")
+    var options = new CSVOptions(Map("timestampFormat" -> "yyyy-mm"), false, "UTC")
     var inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(TimestampType, "2015-08") == TimestampType)
 
-    options = new CSVOptions(Map("timestampFormat" -> "yyyy"), false, "GMT")
+    options = new CSVOptions(Map("timestampFormat" -> "yyyy"), false, "UTC")
     inferSchema = new CSVInferSchema(options)
     assert(inferSchema.inferField(TimestampType, "2015") == TimestampType)
   }
 
   test("Timestamp field types are inferred correctly from other types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(IntegerType, "2015-08-20 14") == StringType)
@@ -89,7 +89,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Boolean fields types are inferred correctly from other types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(LongType, "Fale") == StringType)
@@ -97,7 +97,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Type arrays are merged to highest common type") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(
@@ -112,14 +112,14 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Null fields are handled properly when a nullValue is specified") {
-    var options = new CSVOptions(Map("nullValue" -> "null"), false, "GMT")
+    var options = new CSVOptions(Map("nullValue" -> "null"), false, "UTC")
     var inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "null") == NullType)
     assert(inferSchema.inferField(StringType, "null") == StringType)
     assert(inferSchema.inferField(LongType, "null") == LongType)
 
-    options = new CSVOptions(Map("nullValue" -> "\\N"), false, "GMT")
+    options = new CSVOptions(Map("nullValue" -> "\\N"), false, "UTC")
     inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(IntegerType, "\\N") == IntegerType)
@@ -130,7 +130,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Merging Nulltypes should yield Nulltype.") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     val mergedNullTypes = inferSchema.mergeRowTypes(Array(NullType), Array(NullType))
@@ -138,14 +138,14 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("SPARK-18433: Improve DataSource option keys to be more case-insensitive") {
-    val options = new CSVOptions(Map("TiMeStampFormat" -> "yyyy-mm"), false, "GMT")
+    val options = new CSVOptions(Map("TiMeStampFormat" -> "yyyy-mm"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(TimestampType, "2015-08") == TimestampType)
   }
 
   test("SPARK-18877: `inferField` on DecimalType should find a common type with `typeSoFar`") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     withSQLConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED.key -> "true") {
@@ -166,7 +166,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
 
   test("DoubleType should be inferred when user defined nan/inf are provided") {
     val options = new CSVOptions(Map("nanValue" -> "nan", "negativeInf" -> "-inf",
-      "positiveInf" -> "inf"), false, "GMT")
+      "positiveInf" -> "inf"), false, "UTC")
     val inferSchema = new CSVInferSchema(options)
 
     assert(inferSchema.inferField(NullType, "nan") == DoubleType)
@@ -179,7 +179,7 @@ class CSVInferSchemaSuite extends SparkFunSuite with SQLHelper {
       val options = new CSVOptions(
         parameters = Map("locale" -> langTag, "inferSchema" -> "true", "sep" -> "|"),
         columnPruning = false,
-        defaultTimeZoneId = "GMT")
+        defaultTimeZoneId = "UTC")
       val inferSchema = new CSVInferSchema(options)
 
       val df = new DecimalFormat("", new DecimalFormatSymbols(Locale.forLanguageTag(langTag)))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index c8c71bcfeb7ac..4853b4f162be0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.csv
 
 import java.math.BigDecimal
 import java.text.{DecimalFormat, DecimalFormatSymbols}
-import java.time.ZoneOffset
 import java.util.{Locale, TimeZone}
 
 import org.apache.commons.lang3.time.FastDateFormat
@@ -44,7 +43,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
 
     stringValues.zip(decimalValues).foreach { case (strVal, decimalVal) =>
       val decimalValue = new BigDecimal(decimalVal.toString)
-      val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+      val options = new CSVOptions(Map.empty[String, String], false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       assert(parser.makeConverter("_1", decimalType).apply(strVal) ===
         Decimal(decimalValue, decimalType.precision, decimalType.scale))
@@ -58,14 +57,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // Nullable field with nullValue option.
     types.foreach { t =>
       // Tests that a custom nullValue.
-      val nullValueOptions = new CSVOptions(Map("nullValue" -> "-"), false, "GMT")
+      val nullValueOptions = new CSVOptions(Map("nullValue" -> "-"), false, "UTC")
       var parser = new UnivocityParser(StructType(Seq.empty), nullValueOptions)
       val converter = parser.makeConverter("_1", t, nullable = true)
       assertNull(converter.apply("-"))
       assertNull(converter.apply(null))
 
       // Tests that the default nullValue is empty string.
-      val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+      val options = new CSVOptions(Map.empty[String, String], false, "UTC")
       parser = new UnivocityParser(StructType(Seq.empty), options)
       assertNull(parser.makeConverter("_1", t, nullable = true).apply(""))
     }
@@ -73,7 +72,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // Not nullable field with nullValue option.
     types.foreach { t =>
       // Casts a null to not nullable field should throw an exception.
-      val options = new CSVOptions(Map("nullValue" -> "-"), false, "GMT")
+      val options = new CSVOptions(Map("nullValue" -> "-"), false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       val converter = parser.makeConverter("_1", t, nullable = false)
       var message = intercept[RuntimeException] {
@@ -89,7 +88,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     // If nullValue is different with empty string, then, empty string should not be casted into
     // null.
     Seq(true, false).foreach { b =>
-      val options = new CSVOptions(Map("nullValue" -> "null"), false, "GMT")
+      val options = new CSVOptions(Map("nullValue" -> "null"), false, "UTC")
       val parser = new UnivocityParser(StructType(Seq.empty), options)
       val converter = parser.makeConverter("_1", StringType, nullable = b)
       assert(converter.apply("") == UTF8String.fromString(""))
@@ -97,7 +96,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Throws exception for empty string with non null type") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val exception = intercept[RuntimeException]{
       parser.makeConverter("_1", IntegerType, nullable = false).apply("")
@@ -106,7 +105,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Types are cast correctly") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), options)
     assert(parser.makeConverter("_1", ByteType).apply("10") == 10)
     assert(parser.makeConverter("_1", ShortType).apply("10") == 10)
@@ -117,7 +116,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     assert(parser.makeConverter("_1", BooleanType).apply("true") == true)
 
     var timestampsOptions =
-      new CSVOptions(Map("timestampFormat" -> "dd/MM/yyyy HH:mm"), false, "GMT")
+      new CSVOptions(Map("timestampFormat" -> "dd/MM/yyyy HH:mm"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), timestampsOptions)
     val customTimestamp = "31/01/2015 00:00"
     var format = FastDateFormat.getInstance(
@@ -130,7 +129,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     assert(castedTimestamp == expectedTime * 1000L)
 
     val customDate = "31/01/2015"
-    val dateOptions = new CSVOptions(Map("dateFormat" -> "dd/MM/yyyy"), false, "GMT")
+    val dateOptions = new CSVOptions(Map("dateFormat" -> "dd/MM/yyyy"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), dateOptions)
     format = FastDateFormat.getInstance(
       dateOptions.dateFormat,
@@ -139,7 +138,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val expectedDate = format.parse(customDate).getTime
     val castedDate = parser.makeConverter("_1", DateType, nullable = true)
         .apply(customDate)
-    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, ZoneOffset.UTC))
+    assert(castedDate == DateTimeUtils.millisToDays(expectedDate, UTC))
 
     val timestamp = "2015-01-01 00:00:00"
     timestampsOptions = new CSVOptions(Map(
@@ -154,7 +153,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Throws exception for casting an invalid string to Float and Double Types") {
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val types = Seq(DoubleType, FloatType)
     val input = Seq("10u000", "abc", "1 2/3")
@@ -169,7 +168,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Float NaN values are parsed correctly") {
-    val options = new CSVOptions(Map("nanValue" -> "nn"), false, "GMT")
+    val options = new CSVOptions(Map("nanValue" -> "nn"), false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val floatVal: Float = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("nn").asInstanceOf[Float]
@@ -180,7 +179,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Double NaN values are parsed correctly") {
-    val options = new CSVOptions(Map("nanValue" -> "-"), false, "GMT")
+    val options = new CSVOptions(Map("nanValue" -> "-"), false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
     val doubleVal: Double = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("-").asInstanceOf[Double]
@@ -189,14 +188,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Float infinite values can be parsed") {
-    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "GMT")
+    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), negativeInfOptions)
     val floatVal1 = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("max").asInstanceOf[Float]
 
     assert(floatVal1 == Float.NegativeInfinity)
 
-    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "GMT")
+    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), positiveInfOptions)
     val floatVal2 = parser.makeConverter(
       "_1", FloatType, nullable = true).apply("max").asInstanceOf[Float]
@@ -205,14 +204,14 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("Double infinite values can be parsed") {
-    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "GMT")
+    val negativeInfOptions = new CSVOptions(Map("negativeInf" -> "max"), false, "UTC")
     var parser = new UnivocityParser(StructType(Seq.empty), negativeInfOptions)
     val doubleVal1 = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("max").asInstanceOf[Double]
 
     assert(doubleVal1 == Double.NegativeInfinity)
 
-    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "GMT")
+    val positiveInfOptions = new CSVOptions(Map("positiveInf" -> "max"), false, "UTC")
     parser = new UnivocityParser(StructType(Seq.empty), positiveInfOptions)
     val doubleVal2 = parser.makeConverter(
       "_1", DoubleType, nullable = true).apply("max").asInstanceOf[Double]
@@ -228,7 +227,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
       val df = new DecimalFormat("", new DecimalFormatSymbols(Locale.forLanguageTag(langTag)))
       val input = df.format(expected.toBigDecimal)
 
-      val options = new CSVOptions(Map("locale" -> langTag), false, "GMT")
+      val options = new CSVOptions(Map("locale" -> langTag), false, "UTC")
       val parser = new UnivocityParser(new StructType().add("d", decimalType), options)
 
       assert(parser.makeConverter("_1", decimalType).apply(input) === expected)
@@ -263,7 +262,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val input = "name\t42"
     val expected = UTF8String.fromString(input)
 
-    val options = new CSVOptions(Map.empty[String, String], false, "GMT")
+    val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     val parser = new UnivocityParser(StructType(Seq.empty), options)
 
     val convertedValue = parser.makeConverter("_1", StringBasedUDT, nullable = false).apply(input)
@@ -280,7 +279,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
         filters: Seq[Filter],
         expected: Option[InternalRow]): Unit = {
       Seq(false, true).foreach { columnPruning =>
-        val options = new CSVOptions(Map.empty[String, String], columnPruning, "GMT")
+        val options = new CSVOptions(Map.empty[String, String], columnPruning, "UTC")
         val parser = new UnivocityParser(dataSchema, requiredSchema, options, filters)
         val actual = parser.parse(input)
         assert(actual === expected)
@@ -355,8 +354,8 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
     val options = new CSVOptions(Map.empty[String, String], false, "UTC")
     check(new UnivocityParser(StructType(Seq.empty), options))
 
-    val optionsWithPattern =
-      new CSVOptions(Map("timestampFormat" -> "invalid", "dateFormat" -> "invalid"), false, "UTC")
+    val optionsWithPattern = new CSVOptions(
+      Map("timestampFormat" -> "invalid", "dateFormat" -> "invalid"), false, "UTC")
     check(new UnivocityParser(StructType(Seq.empty), optionsWithPattern))
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index e5e28b6a74b39..dd3f362e71b7c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -49,7 +49,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   protected def checkNullCast(from: DataType, to: DataType): Unit = {
-    checkEvaluation(cast(Literal.create(null, from), to, Option("GMT")), null)
+    checkEvaluation(cast(Literal.create(null, from), to, UTC_OPT), null)
   }
 
   test("null cast") {
@@ -141,7 +141,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the timeZoneId parameter.
-      c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
+      c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
       c.set(2015, 2, 18, 12, 3, 17)
       c.set(Calendar.MILLISECOND, 0)
       checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis))
@@ -172,7 +172,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the timeZoneId parameter.
-      c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
+      c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
       c.set(2015, 2, 18, 12, 3, 17)
       c.set(Calendar.MILLISECOND, 456)
       checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis))
@@ -280,23 +280,23 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
         fromMillis(c.getTimeInMillis))
     }
 
-    val gmtId = Option("GMT")
-
     checkEvaluation(cast("abdef", StringType), "abdef")
-    checkEvaluation(cast("abdef", TimestampType, gmtId), null)
+    checkEvaluation(cast("abdef", TimestampType, UTC_OPT), null)
     checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65))
 
     checkEvaluation(cast(cast(sd, DateType), StringType), sd)
     checkEvaluation(cast(cast(d, StringType), DateType), 0)
-    checkEvaluation(cast(cast(nts, TimestampType, gmtId), StringType, gmtId), nts)
+    checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts)
     checkEvaluation(
-      cast(cast(ts, StringType, gmtId), TimestampType, gmtId),
+      cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT),
       DateTimeUtils.fromJavaTimestamp(ts))
 
     // all convert to string type to check
-    checkEvaluation(cast(cast(cast(nts, TimestampType, gmtId), DateType, gmtId), StringType), sd)
     checkEvaluation(
-      cast(cast(cast(ts, DateType, gmtId), TimestampType, gmtId), StringType, gmtId),
+      cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType),
+      sd)
+    checkEvaluation(
+      cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT),
       zts)
 
     checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef")
@@ -309,7 +309,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
         DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
       5.toShort)
     checkEvaluation(
-      cast(cast(cast(cast(cast(cast("5", TimestampType, gmtId), ByteType),
+      cast(cast(cast(cast(cast(cast("5", TimestampType, UTC_OPT), ByteType),
         DecimalType.SYSTEM_DEFAULT), LongType), StringType), ShortType),
       null)
     checkEvaluation(cast(cast(cast(cast(cast(cast("5", DecimalType.SYSTEM_DEFAULT),
@@ -360,8 +360,9 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(cast(d, DecimalType(10, 2)), null)
     checkEvaluation(cast(d, StringType), "1970-01-01")
 
-    val gmtId = Option("GMT")
-    checkEvaluation(cast(cast(d, TimestampType, gmtId), StringType, gmtId), "1970-01-01 00:00:00")
+    checkEvaluation(
+      cast(cast(d, TimestampType, UTC_OPT), StringType, UTC_OPT),
+      "1970-01-01 00:00:00")
   }
 
   test("cast from timestamp") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index b7628a373d0bd..db52229f860b4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.expressions.objects._
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.LA
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.ThreadUtils
@@ -190,7 +191,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     val expressions = Seq.fill(length) {
       ToUTCTimestamp(
         Literal.create(Timestamp.valueOf("2015-07-24 00:00:00"), TimestampType),
-        Literal.create("PST", StringType))
+        Literal.create(LA.getId, StringType))
     }
     val plan = GenerateMutableProjection.generate(expressions)
     val actual = plan(new GenericInternalRow(length)).toSeq(expressions.map(_.dataType))
@@ -207,7 +208,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     val expressions = Seq.fill(length) {
       ToUTCTimestamp(
         Literal.create(Timestamp.valueOf("2017-10-10 00:00:00"), TimestampType),
-        Literal.create("PST", StringType))
+        Literal.create(LA.getId, StringType))
     }
     val plan = GenerateMutableProjection.generate(expressions)
     val actual = plan(new GenericInternalRow(length)).toSeq(expressions.map(_.dataType))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index bb31a066440c4..00cdc1712c459 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MICROS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.catalyst.util.IntervalUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -879,7 +880,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("Sequence of dates") {
-    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone("UTC")) {
+    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
       checkEvaluation(new Sequence(
         Literal(Date.valueOf("2018-01-01")),
         Literal(Date.valueOf("2018-01-05")),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
index e623910e2efe1..24cf9339aa72e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
@@ -27,19 +27,18 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{PST, UTC_OPT}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with PlanTestBase {
   val badCsv = "\u0000\u0000\u0000A\u0001AAA"
 
-  val gmtId = Option(DateTimeUtils.TimeZoneGMT.getID)
-
   test("from_csv") {
     val csvData = "1"
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(csvData), gmtId),
+      CsvToStructs(schema, Map.empty, Literal(csvData), UTC_OPT),
       InternalRow(1)
     )
   }
@@ -48,17 +47,17 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     val csvData = "---"
     val schema = StructType(StructField("a", DoubleType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map("mode" -> PermissiveMode.name), Literal(csvData), gmtId),
+      CsvToStructs(schema, Map("mode" -> PermissiveMode.name), Literal(csvData), UTC_OPT),
       InternalRow(null))
 
     // Default mode is Permissive
-    checkEvaluation(CsvToStructs(schema, Map.empty, Literal(csvData), gmtId), InternalRow(null))
+    checkEvaluation(CsvToStructs(schema, Map.empty, Literal(csvData), UTC_OPT), InternalRow(null))
   }
 
   test("from_csv null input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal.create(null, StringType), gmtId),
+      CsvToStructs(schema, Map.empty, Literal.create(null, StringType), UTC_OPT),
       null
     )
   }
@@ -66,7 +65,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   test("from_csv bad UTF-8") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(badCsv), gmtId),
+      CsvToStructs(schema, Map.empty, Literal(badCsv), UTC_OPT),
       InternalRow(null))
   }
 
@@ -78,14 +77,14 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 123)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(csvData1), gmtId),
+      CsvToStructs(schema, Map.empty, Literal(csvData1), UTC_OPT),
       InternalRow(c.getTimeInMillis * 1000L)
     )
     // The result doesn't change because the CSV string includes timezone string ("Z" here),
     // which means the string represents the timestamp string in the timezone regardless of
     // the timeZoneId parameter.
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal(csvData1), Option("PST")),
+      CsvToStructs(schema, Map.empty, Literal(csvData1), Option(PST.getId)),
       InternalRow(c.getTimeInMillis * 1000L)
     )
 
@@ -108,7 +107,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
             DateTimeUtils.TIMEZONE_OPTION -> tz.getID),
           Literal(csvData2),
-          gmtId),
+          UTC_OPT),
         InternalRow(c.getTimeInMillis * 1000L)
       )
     }
@@ -117,7 +116,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   test("from_csv empty input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal.create(" ", StringType), gmtId),
+      CsvToStructs(schema, Map.empty, Literal.create(" ", StringType), UTC_OPT),
       InternalRow(null)
     )
   }
@@ -129,7 +128,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
       .add("b", StringType, nullable = false)
       .add("c", StringType, nullable = false)
     val output = InternalRow(1L, null, UTF8String.fromString("foo"))
-    val expr = CsvToStructs(csvSchema, Map.empty, Literal.create(input, StringType), gmtId)
+    val expr = CsvToStructs(csvSchema, Map.empty, Literal.create(input, StringType), UTC_OPT)
     checkEvaluation(expr, output)
     val schema = expr.dataType
     val schemaToCompare = csvSchema.asNullable
@@ -142,7 +141,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
       .add("a", IntegerType)
       .add("b", IntegerType)
     checkEvaluation(
-      CsvToStructs(schema, Map.empty, Literal.create("1"), gmtId),
+      CsvToStructs(schema, Map.empty, Literal.create("1"), UTC_OPT),
       InternalRow(1, null)
     )
   }
@@ -152,7 +151,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     val schema = StructType(StructField("a", DoubleType) :: Nil)
     val exception = intercept[TestFailedException] {
       checkEvaluation(
-        CsvToStructs(schema, Map("mode" -> DropMalformedMode.name), Literal(csvData), gmtId),
+        CsvToStructs(schema, Map("mode" -> DropMalformedMode.name), Literal(csvData), UTC_OPT),
         InternalRow(null))
     }.getCause
     assert(exception.getMessage.contains("from_csv() doesn't support the DROPMALFORMED mode"))
@@ -171,14 +170,14 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
   test("to_csv - struct") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(create_row(1), schema)
-    checkEvaluation(StructsToCsv(Map.empty, struct, gmtId), "1")
+    checkEvaluation(StructsToCsv(Map.empty, struct, UTC_OPT), "1")
   }
 
   test("to_csv null input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(null, schema)
     checkEvaluation(
-      StructsToCsv(Map.empty, struct, gmtId),
+      StructsToCsv(Map.empty, struct, UTC_OPT),
       null
     )
   }
@@ -190,24 +189,24 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     c.set(Calendar.MILLISECOND, 0)
     val struct = Literal.create(create_row(c.getTimeInMillis * 1000L), schema)
 
-    checkEvaluation(StructsToCsv(Map.empty, struct, gmtId), "2016-01-01T00:00:00.000Z")
+    checkEvaluation(StructsToCsv(Map.empty, struct, UTC_OPT), "2016-01-01T00:00:00.000Z")
     checkEvaluation(
-      StructsToCsv(Map.empty, struct, Option("PST")), "2015-12-31T16:00:00.000-08:00")
+      StructsToCsv(Map.empty, struct, Option(PST.getId)), "2015-12-31T16:00:00.000-08:00")
 
     checkEvaluation(
       StructsToCsv(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> gmtId.get),
+          DateTimeUtils.TIMEZONE_OPTION -> UTC_OPT.get),
         struct,
-        gmtId),
+        UTC_OPT),
       "2016-01-01T00:00:00"
     )
     checkEvaluation(
       StructsToCsv(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> "PST"),
+          DateTimeUtils.TIMEZONE_OPTION -> PST.getId),
         struct,
-        gmtId),
+        UTC_OPT),
       "2015-12-31T16:00:00"
     )
   }
@@ -223,7 +222,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
       val options = Map("dateFormat" -> dateFormat, "locale" -> langTag)
 
       checkEvaluation(
-        CsvToStructs(schema, options, Literal.create(dateStr), gmtId),
+        CsvToStructs(schema, options, Literal.create(dateStr), UTC_OPT),
         InternalRow(17836)) // number of days from 1970-01-01
     }
   }
@@ -234,7 +233,7 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
         schema = StructType.fromDDL("i int, _unparsed boolean"),
         options = Map("columnNameOfCorruptRecord" -> "_unparsed"),
         child = Literal.create("a"),
-        timeZoneId = gmtId),
+        timeZoneId = UTC_OPT),
       expectedErrMsg = "The field for corrupt records must be string type and nullable")
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 4a04fbba678ed..2cc62598dc433 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
-import java.time.{Instant, LocalDate, LocalDateTime, ZoneId, ZoneOffset}
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
 import java.util.{Calendar, Locale, TimeZone}
 import java.util.concurrent.TimeUnit._
 
@@ -29,7 +29,6 @@ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjectio
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneGMT
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -38,15 +37,11 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   import IntegralLiteralTestUtils._
 
-  val TimeZonePST = TimeZone.getTimeZone("PST")
-  val TimeZoneJST = TimeZone.getTimeZone("JST")
-
-  val gmtId = Option(TimeZoneGMT.getID)
-  val pstId = Option(TimeZonePST.getID)
-  val jstId = Option(TimeZoneJST.getID)
+  private val PST_OPT = Option(PST.getId)
+  private val JST_OPT = Option(JST.getId)
 
   def toMillis(timestamp: String): Long = {
-    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC)
+    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC)
     DateTimeUtils.toMillis(tf.parse(timestamp))
   }
   val date = "2015-04-08 13:10:15"
@@ -55,13 +50,13 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   val ts = new Timestamp(toMillis(time))
 
   test("datetime function current_date") {
-    val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), ZoneOffset.UTC)
-    val cd = CurrentDate(gmtId).eval(EmptyRow).asInstanceOf[Int]
-    val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), ZoneOffset.UTC)
+    val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), UTC)
+    val cd = CurrentDate(UTC_OPT).eval(EmptyRow).asInstanceOf[Int]
+    val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), UTC)
     assert(d0 <= cd && cd <= d1 && d1 - d0 <= 1)
 
-    val cdjst = CurrentDate(jstId).eval(EmptyRow).asInstanceOf[Int]
-    val cdpst = CurrentDate(pstId).eval(EmptyRow).asInstanceOf[Int]
+    val cdjst = CurrentDate(JST_OPT).eval(EmptyRow).asInstanceOf[Int]
+    val cdpst = CurrentDate(PST_OPT).eval(EmptyRow).asInstanceOf[Int]
     assert(cdpst <= cd && cd <= cdjst)
   }
 
@@ -93,8 +88,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Year") {
     checkEvaluation(Year(Literal.create(null, DateType)), null)
     checkEvaluation(Year(Literal(d)), 2015)
-    checkEvaluation(Year(Cast(Literal(date), DateType, gmtId)), 2015)
-    checkEvaluation(Year(Cast(Literal(ts), DateType, gmtId)), 2013)
+    checkEvaluation(Year(Cast(Literal(date), DateType, UTC_OPT)), 2015)
+    checkEvaluation(Year(Cast(Literal(ts), DateType, UTC_OPT)), 2013)
 
     val c = Calendar.getInstance()
     (2000 to 2002).foreach { y =>
@@ -115,8 +110,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Quarter") {
     checkEvaluation(Quarter(Literal.create(null, DateType)), null)
     checkEvaluation(Quarter(Literal(d)), 2)
-    checkEvaluation(Quarter(Cast(Literal(date), DateType, gmtId)), 2)
-    checkEvaluation(Quarter(Cast(Literal(ts), DateType, gmtId)), 4)
+    checkEvaluation(Quarter(Cast(Literal(date), DateType, UTC_OPT)), 2)
+    checkEvaluation(Quarter(Cast(Literal(ts), DateType, UTC_OPT)), 4)
 
     val c = Calendar.getInstance()
     (2003 to 2004).foreach { y =>
@@ -138,8 +133,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("Month") {
     checkEvaluation(Month(Literal.create(null, DateType)), null)
     checkEvaluation(Month(Literal(d)), 4)
-    checkEvaluation(Month(Cast(Literal(date), DateType, gmtId)), 4)
-    checkEvaluation(Month(Cast(Literal(ts), DateType, gmtId)), 11)
+    checkEvaluation(Month(Cast(Literal(date), DateType, UTC_OPT)), 4)
+    checkEvaluation(Month(Cast(Literal(ts), DateType, UTC_OPT)), 11)
 
     checkEvaluation(Month(Cast(Literal("1582-04-28 13:10:15"), DateType)), 4)
     checkEvaluation(Month(Cast(Literal("1582-10-04 13:10:15"), DateType)), 10)
@@ -163,8 +158,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(DayOfMonth(Cast(Literal("2000-02-29"), DateType)), 29)
     checkEvaluation(DayOfMonth(Literal.create(null, DateType)), null)
     checkEvaluation(DayOfMonth(Literal(d)), 8)
-    checkEvaluation(DayOfMonth(Cast(Literal(date), DateType, gmtId)), 8)
-    checkEvaluation(DayOfMonth(Cast(Literal(ts), DateType, gmtId)), 8)
+    checkEvaluation(DayOfMonth(Cast(Literal(date), DateType, UTC_OPT)), 8)
+    checkEvaluation(DayOfMonth(Cast(Literal(ts), DateType, UTC_OPT)), 8)
 
     checkEvaluation(DayOfMonth(Cast(Literal("1582-04-28 13:10:15"), DateType)), 28)
     checkEvaluation(DayOfMonth(Cast(Literal("1582-10-15 13:10:15"), DateType)), 15)
@@ -183,14 +178,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("Seconds") {
-    assert(Second(Literal.create(null, DateType), gmtId).resolved === false)
-    assert(Second(Cast(Literal(d), TimestampType, gmtId), gmtId).resolved )
-    checkEvaluation(Second(Cast(Literal(d), TimestampType, gmtId), gmtId), 0)
-    checkEvaluation(Second(Cast(Literal(date), TimestampType, gmtId), gmtId), 15)
-    checkEvaluation(Second(Literal(ts), gmtId), 15)
+    assert(Second(Literal.create(null, DateType), UTC_OPT).resolved === false)
+    assert(Second(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT).resolved )
+    checkEvaluation(Second(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT), 0)
+    checkEvaluation(Second(Cast(Literal(date), TimestampType, UTC_OPT), UTC_OPT), 15)
+    checkEvaluation(Second(Literal(ts), UTC_OPT), 15)
 
     val c = Calendar.getInstance()
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+    for (tz <- outstandingTimezones) {
       val timeZoneId = Option(tz.getID)
       c.setTimeZone(tz)
       (0 to 60 by 5).foreach { s =>
@@ -207,10 +202,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("DayOfWeek") {
     checkEvaluation(DayOfWeek(Literal.create(null, DateType)), null)
     checkEvaluation(DayOfWeek(Literal(d)), Calendar.WEDNESDAY)
-    checkEvaluation(DayOfWeek(Cast(Literal(date), DateType, gmtId)),
+    checkEvaluation(DayOfWeek(Cast(Literal(date), DateType, UTC_OPT)),
       Calendar.WEDNESDAY)
-    checkEvaluation(DayOfWeek(Cast(Literal(ts), DateType, gmtId)), Calendar.FRIDAY)
-    checkEvaluation(DayOfWeek(Cast(Literal("2011-05-06"), DateType, gmtId)), Calendar.FRIDAY)
+    checkEvaluation(DayOfWeek(Cast(Literal(ts), DateType, UTC_OPT)), Calendar.FRIDAY)
+    checkEvaluation(DayOfWeek(Cast(Literal("2011-05-06"), DateType, UTC_OPT)), Calendar.FRIDAY)
     checkEvaluation(DayOfWeek(Literal(new Date(toMillis("2017-05-27 13:10:15")))),
       Calendar.SATURDAY)
     checkEvaluation(DayOfWeek(Literal(new Date(toMillis("1582-10-15 13:10:15")))),
@@ -221,9 +216,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("WeekDay") {
     checkEvaluation(WeekDay(Literal.create(null, DateType)), null)
     checkEvaluation(WeekDay(Literal(d)), 2)
-    checkEvaluation(WeekDay(Cast(Literal(date), DateType, gmtId)), 2)
-    checkEvaluation(WeekDay(Cast(Literal(ts), DateType, gmtId)), 4)
-    checkEvaluation(WeekDay(Cast(Literal("2011-05-06"), DateType, gmtId)), 4)
+    checkEvaluation(WeekDay(Cast(Literal(date), DateType, UTC_OPT)), 2)
+    checkEvaluation(WeekDay(Cast(Literal(ts), DateType, UTC_OPT)), 4)
+    checkEvaluation(WeekDay(Cast(Literal("2011-05-06"), DateType, UTC_OPT)), 4)
     checkEvaluation(WeekDay(Literal(new Date(toMillis("2017-05-27 13:10:15")))), 5)
     checkEvaluation(WeekDay(Literal(new Date(toMillis("1582-10-15 13:10:15")))), 4)
     checkConsistencyBetweenInterpretedAndCodegen(WeekDay, DateType)
@@ -232,11 +227,11 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   test("WeekOfYear") {
     checkEvaluation(WeekOfYear(Literal.create(null, DateType)), null)
     checkEvaluation(WeekOfYear(Literal(d)), 15)
-    checkEvaluation(WeekOfYear(Cast(Literal(date), DateType, gmtId)), 15)
-    checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, gmtId)), 45)
-    checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, gmtId)), 18)
-    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-15 13:10:15"), DateType, gmtId)), 41)
-    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-04 13:10:15"), DateType, gmtId)), 40)
+    checkEvaluation(WeekOfYear(Cast(Literal(date), DateType, UTC_OPT)), 15)
+    checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, UTC_OPT)), 45)
+    checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, UTC_OPT)), 18)
+    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-15 13:10:15"), DateType, UTC_OPT)), 41)
+    checkEvaluation(WeekOfYear(Cast(Literal("1582-10-04 13:10:15"), DateType, UTC_OPT)), 40)
     checkConsistencyBetweenInterpretedAndCodegen(WeekOfYear, DateType)
   }
 
@@ -244,38 +239,38 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     Seq("legacy", "corrected").foreach { legacyParserPolicy =>
       withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
         checkEvaluation(
-          DateFormatClass(Literal.create(null, TimestampType), Literal("y"), gmtId),
+          DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT),
           null)
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-          Literal.create(null, StringType), gmtId), null)
-
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-          Literal("y"), gmtId), "2015")
-        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), gmtId), "2013")
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, gmtId),
-          Literal("H"), gmtId), "0")
-        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), gmtId), "13")
-
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
-          Literal("y"), pstId), "2015")
-        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), pstId), "2013")
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, pstId),
-          Literal("H"), pstId), "0")
-        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), pstId), "5")
-
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
-          Literal("y"), jstId), "2015")
-        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), jstId), "2013")
-        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, jstId),
-          Literal("H"), jstId), "0")
-        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), jstId), "22")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT),
+          Literal.create(null, StringType), UTC_OPT), null)
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT),
+          Literal("y"), UTC_OPT), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT),
+          Literal("H"), UTC_OPT), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13")
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT),
+          Literal("y"), PST_OPT), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT),
+          Literal("H"), PST_OPT), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5")
+
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT),
+          Literal("y"), JST_OPT), "2015")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013")
+        checkEvaluation(DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT),
+          Literal("H"), JST_OPT), "0")
+        checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22")
 
         // SPARK-28072 The codegen path should work
         checkEvaluation(
           expression = DateFormatClass(
             BoundReference(ordinal = 0, dataType = TimestampType, nullable = true),
             BoundReference(ordinal = 1, dataType = StringType, nullable = true),
-            jstId),
+            JST_OPT),
           expected = "22",
           inputRow = InternalRow(DateTimeUtils.fromJavaTimestamp(ts), UTF8String.fromString("H")))
       }
@@ -283,14 +278,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("Hour") {
-    assert(Hour(Literal.create(null, DateType), gmtId).resolved === false)
-    assert(Hour(Literal(ts), gmtId).resolved)
-    checkEvaluation(Hour(Cast(Literal(d), TimestampType, gmtId), gmtId), 0)
-    checkEvaluation(Hour(Cast(Literal(date), TimestampType, gmtId), gmtId), 13)
-    checkEvaluation(Hour(Literal(ts), gmtId), 13)
+    assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false)
+    assert(Hour(Literal(ts), UTC_OPT).resolved)
+    checkEvaluation(Hour(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT), 0)
+    checkEvaluation(Hour(Cast(Literal(date), TimestampType, UTC_OPT), UTC_OPT), 13)
+    checkEvaluation(Hour(Literal(ts), UTC_OPT), 13)
 
     val c = Calendar.getInstance()
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+    for (tz <- outstandingTimezones) {
       val timeZoneId = Option(tz.getID)
       c.setTimeZone(tz)
       (0 to 24 by 6).foreach { h =>
@@ -309,15 +304,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("Minute") {
-    assert(Minute(Literal.create(null, DateType), gmtId).resolved === false)
-    assert(Minute(Literal(ts), gmtId).resolved)
-    checkEvaluation(Minute(Cast(Literal(d), TimestampType, gmtId), gmtId), 0)
+    assert(Minute(Literal.create(null, DateType), UTC_OPT).resolved === false)
+    assert(Minute(Literal(ts), UTC_OPT).resolved)
+    checkEvaluation(Minute(Cast(Literal(d), TimestampType, UTC_OPT), UTC_OPT), 0)
     checkEvaluation(
-      Minute(Cast(Literal(date), TimestampType, gmtId), gmtId), 10)
-    checkEvaluation(Minute(Literal(ts), gmtId), 10)
+      Minute(Cast(Literal(date), TimestampType, UTC_OPT), UTC_OPT), 10)
+    checkEvaluation(Minute(Literal(ts), UTC_OPT), 10)
 
     val c = Calendar.getInstance()
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+    for (tz <- outstandingTimezones) {
       val timeZoneId = Option(tz.getID)
       c.setTimeZone(tz)
       (0 to 60 by 5).foreach { m =>
@@ -389,7 +384,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("time_add") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+    for (tz <- outstandingTimezones) {
       val timeZoneId = Option(tz.getID)
       sdf.setTimeZone(tz)
 
@@ -427,7 +422,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("time_sub") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+    for (tz <- outstandingTimezones) {
       val timeZoneId = Option(tz.getID)
       sdf.setTimeZone(tz)
 
@@ -513,7 +508,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("months_between") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+    for (tz <- outstandingTimezones) {
       val timeZoneId = Option(tz.getID)
       sdf.setTimeZone(tz)
 
@@ -646,7 +641,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         expected)
     }
 
-    withDefaultTimeZone(TimeZoneGMT) {
+    withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
       val inputDate = Timestamp.valueOf("2015-07-22 05:30:06")
 
       Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt =>
@@ -716,7 +711,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
         val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-        for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+        for (tz <- outstandingTimezones) {
           val timeZoneId = Option(tz.getID)
           sdf1.setTimeZone(tz)
           sdf2.setTimeZone(tz)
@@ -765,10 +760,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
         val fmt3 = "yy-MM-dd"
         val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-        sdf3.setTimeZone(TimeZoneGMT)
+        sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
 
-        withDefaultTimeZone(TimeZoneGMT) {
-          for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+        withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
+          for (tz <- outstandingTimezones) {
             val timeZoneId = Option(tz.getID)
             sdf1.setTimeZone(tz)
             sdf2.setTimeZone(tz)
@@ -832,10 +827,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
         val fmt3 = "yy-MM-dd"
         val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
-        sdf3.setTimeZone(TimeZoneGMT)
+        sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
 
-        withDefaultTimeZone(TimeZoneGMT) {
-          for (tz <- Seq(TimeZoneGMT, TimeZonePST, TimeZoneJST)) {
+        withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
+          for (tz <- outstandingTimezones) {
             val timeZoneId = Option(tz.getID)
             sdf1.setTimeZone(tz)
             sdf2.setTimeZone(tz)
@@ -924,8 +919,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           NonFoldableLiteral.create(tz, StringType)),
         if (expected != null) Timestamp.valueOf(expected) else null)
     }
-    test("2015-07-24 00:00:00", "PST", "2015-07-24 07:00:00")
-    test("2015-01-24 00:00:00", "PST", "2015-01-24 08:00:00")
+    test("2015-07-24 00:00:00", LA.getId, "2015-07-24 07:00:00")
+    test("2015-01-24 00:00:00", LA.getId, "2015-01-24 08:00:00")
     test(null, "UTC", null)
     test("2015-07-24 00:00:00", null, null)
     test(null, null, null)
@@ -955,8 +950,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           NonFoldableLiteral.create(tz, StringType)),
         if (expected != null) Timestamp.valueOf(expected) else null)
     }
-    test("2015-07-24 00:00:00", "PST", "2015-07-23 17:00:00")
-    test("2015-01-24 00:00:00", "PST", "2015-01-23 16:00:00")
+    test("2015-07-24 00:00:00", LA.getId, "2015-07-23 17:00:00")
+    test("2015-01-24 00:00:00", LA.getId, "2015-01-23 16:00:00")
     test(null, "UTC", null)
     test("2015-07-24 00:00:00", null, null)
     test(null, null, null)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index 3693531f47610..dec2e4665251f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.text.{DecimalFormat, DecimalFormatSymbols, SimpleDateFormat}
-import java.util.{Calendar, Locale}
+import java.util.{Calendar, Locale, TimeZone}
 
 import org.scalatest.exceptions.TestFailedException
 
@@ -27,6 +27,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{PST, UTC, UTC_OPT}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -395,13 +396,11 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       InternalRow(UTF8String.fromString("1"), null, UTF8String.fromString("1")))
   }
 
-  val gmtId = Option(DateTimeUtils.TimeZoneGMT.getID)
-
   test("from_json") {
     val jsonData = """{"a": 1}"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(jsonData), UTC_OPT),
       InternalRow(1)
     )
   }
@@ -410,13 +409,13 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val jsonData = """{"a" 1}"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(jsonData), UTC_OPT),
       InternalRow(null)
     )
 
     val exception = intercept[TestFailedException] {
       checkEvaluation(
-        JsonToStructs(schema, Map("mode" -> FailFastMode.name), Literal(jsonData), gmtId),
+        JsonToStructs(schema, Map("mode" -> FailFastMode.name), Literal(jsonData), UTC_OPT),
         InternalRow(null)
       )
     }.getCause
@@ -429,35 +428,35 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = """[{"a": 1}, {"a": 2}]"""
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = InternalRow(1) :: InternalRow(2) :: Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=object, schema=array, output=array of single row") {
     val input = """{"a": 1}"""
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = InternalRow(1) :: Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty array, schema=array, output=empty array") {
     val input = "[ ]"
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty object, schema=array, output=array of single row with null") {
     val input = "{ }"
     val schema = ArrayType(StructType(StructField("a", IntegerType) :: Nil))
     val output = InternalRow(null) :: Nil
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=array of single object, schema=struct, output=single row") {
     val input = """[{"a": 1}]"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val output = InternalRow(null)
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=array, schema=struct, output=single row") {
@@ -466,27 +465,27 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = new StructType().add("a", IntegerType).add(corrupted, StringType)
     val output = InternalRow(null, UTF8String.fromString(input))
     val options = Map("columnNameOfCorruptRecord" -> corrupted)
-    checkEvaluation(JsonToStructs(schema, options, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, options, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty array, schema=struct, output=single row with null") {
     val input = """[]"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val output = InternalRow(null)
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json - input=empty object, schema=struct, output=single row with null") {
     val input = """{  }"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val output = InternalRow(null)
-    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), gmtId), output)
+    checkEvaluation(JsonToStructs(schema, Map.empty, Literal(input), UTC_OPT), output)
   }
 
   test("from_json null input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal.create(null, StringType), gmtId),
+      JsonToStructs(schema, Map.empty, Literal.create(null, StringType), UTC_OPT),
       null
     )
   }
@@ -494,7 +493,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
   test("SPARK-20549: from_json bad UTF-8") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(badJson), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(badJson), UTC_OPT),
       InternalRow(null))
   }
 
@@ -502,18 +501,18 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = StructType(StructField("t", TimestampType) :: Nil)
 
     val jsonData1 = """{"t": "2016-01-01T00:00:00.123Z"}"""
-    var c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+    var c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 123)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData1), gmtId),
+      JsonToStructs(schema, Map.empty, Literal(jsonData1), UTC_OPT),
       InternalRow(c.getTimeInMillis * 1000L)
     )
     // The result doesn't change because the json string includes timezone string ("Z" here),
     // which means the string represents the timestamp string in the timezone regardless of
     // the timeZoneId parameter.
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal(jsonData1), Option("PST")),
+      JsonToStructs(schema, Map.empty, Literal(jsonData1), Option(PST.getId)),
       InternalRow(c.getTimeInMillis * 1000L)
     )
 
@@ -536,7 +535,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
             DateTimeUtils.TIMEZONE_OPTION -> tz.getID),
           Literal(jsonData2),
-          gmtId),
+          UTC_OPT),
         InternalRow(c.getTimeInMillis * 1000L)
       )
     }
@@ -545,7 +544,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
   test("SPARK-19543: from_json empty input column") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     checkEvaluation(
-      JsonToStructs(schema, Map.empty, Literal.create(" ", StringType), gmtId),
+      JsonToStructs(schema, Map.empty, Literal.create(" ", StringType), UTC_OPT),
       null
     )
   }
@@ -554,7 +553,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(create_row(1), schema)
     checkEvaluation(
-      StructsToJson(Map.empty, struct, gmtId),
+      StructsToJson(Map.empty, struct, UTC_OPT),
       """{"a":1}"""
     )
   }
@@ -564,7 +563,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(InternalRow(1) :: InternalRow(2) :: Nil)
     val output = """[{"a":1},{"a":2}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -573,7 +572,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(InternalRow(null) :: Nil)
     val output = """[{}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -582,7 +581,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(Nil)
     val output = """[]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -590,41 +589,41 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(null, schema)
     checkEvaluation(
-      StructsToJson(Map.empty, struct, gmtId),
+      StructsToJson(Map.empty, struct, UTC_OPT),
       null
     )
   }
 
   test("to_json with timestamp") {
     val schema = StructType(StructField("t", TimestampType) :: Nil)
-    val c = Calendar.getInstance(DateTimeUtils.TimeZoneGMT)
+    val c = Calendar.getInstance(TimeZone.getTimeZone(UTC))
     c.set(2016, 0, 1, 0, 0, 0)
     c.set(Calendar.MILLISECOND, 0)
     val struct = Literal.create(create_row(c.getTimeInMillis * 1000L), schema)
 
     checkEvaluation(
-      StructsToJson(Map.empty, struct, gmtId),
+      StructsToJson(Map.empty, struct, UTC_OPT),
       """{"t":"2016-01-01T00:00:00.000Z"}"""
     )
     checkEvaluation(
-      StructsToJson(Map.empty, struct, Option("PST")),
+      StructsToJson(Map.empty, struct, Option(PST.getId)),
       """{"t":"2015-12-31T16:00:00.000-08:00"}"""
     )
 
     checkEvaluation(
       StructsToJson(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> gmtId.get),
+          DateTimeUtils.TIMEZONE_OPTION -> UTC_OPT.get),
         struct,
-        gmtId),
+        UTC_OPT),
       """{"t":"2016-01-01T00:00:00"}"""
     )
     checkEvaluation(
       StructsToJson(
         Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-          DateTimeUtils.TIMEZONE_OPTION -> "PST"),
+          DateTimeUtils.TIMEZONE_OPTION -> PST.getId),
         struct,
-        gmtId),
+        UTC_OPT),
       """{"t":"2015-12-31T16:00:00"}"""
     )
   }
@@ -665,7 +664,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       ArrayBasedMapData(Map(UTF8String.fromString("b") -> 2)) :: Nil)
     val output = """[{"a":1},{"b":2}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -674,7 +673,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     val input = new GenericArrayData(ArrayBasedMapData(Map(UTF8String.fromString("a") -> 1)) :: Nil)
     val output = """[{"a":1}]"""
     checkEvaluation(
-      StructsToJson(Map.empty, Literal.create(input, inputSchema), gmtId),
+      StructsToJson(Map.empty, Literal.create(input, inputSchema), UTC_OPT),
       output)
   }
 
@@ -704,7 +703,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       .add("b", StringType, nullable = false)
       .add("c", StringType, nullable = false)
     val output = InternalRow(1L, null, UTF8String.fromString("foo"))
-    val expr = JsonToStructs(jsonSchema, Map.empty, Literal.create(input, StringType), gmtId)
+    val expr = JsonToStructs(jsonSchema, Map.empty, Literal.create(input, StringType), UTC_OPT)
     checkEvaluation(expr, output)
     val schema = expr.dataType
     val schemaToCompare = jsonSchema.asNullable
@@ -737,7 +736,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       val options = Map("dateFormat" -> dateFormat, "locale" -> langTag)
 
       checkEvaluation(
-        JsonToStructs(schema, options, Literal.create(dateStr), gmtId),
+        JsonToStructs(schema, options, Literal.create(dateStr), UTC_OPT),
         InternalRow(17836)) // number of days from 1970-01-01
     }
   }
@@ -748,7 +747,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
         schema = StructType.fromDDL("i int, _unparsed boolean"),
         options = Map("columnNameOfCorruptRecord" -> "_unparsed"),
         child = Literal.create("""{"i":"a"}"""),
-        timeZoneId = gmtId),
+        timeZoneId = UTC_OPT),
       expectedErrMsg = "The field for corrupt records must be string type and nullable")
   }
 
@@ -770,7 +769,7 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       val (expected, input) = decimalInput(langTag)
 
       checkEvaluation(
-        JsonToStructs(schema, options, Literal.create(input), gmtId),
+        JsonToStructs(schema, options, Literal.create(input), UTC_OPT),
         InternalRow(expected))
     }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index 5f1428f96e2b0..374400ecdb058 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -22,20 +22,31 @@ import java.util.TimeZone
 import java.util.concurrent.TimeUnit
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
 
 /**
  * Helper functions for testing date and time functionality.
  */
 object DateTimeTestUtils {
 
+  val CEST = getZoneId("+02:00")
+  val CET = getZoneId("+01:00")
+  val JST = getZoneId("+09:00")
+  val LA = getZoneId("America/Los_Angeles")
+  val MIT = getZoneId("-09:30")
+  val PST = getZoneId("-08:00")
+  val UTC = getZoneId("+00:00")
+
+  val UTC_OPT = Option("UTC")
+
   val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
 
   val outstandingTimezonesIds: Seq[String] = Seq(
     "UTC",
-    "PST",
-    "CET",
+    PST.getId,
+    CET.getId,
     "Africa/Dakar",
-    "America/Los_Angeles",
+    LA.getId,
     "Antarctica/Vostok",
     "Asia/Hong_Kong",
     "Europe/Amsterdam")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 10b2457947df8..5eca86409aa01 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util
 
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
-import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset}
+import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId}
 import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
@@ -34,9 +34,6 @@ import org.apache.spark.unsafe.types.UTF8String
 
 class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
-  val zonePST = getZoneId("PST")
-  val zoneGMT = getZoneId("GMT")
-
   private def defaultZoneId = ZoneId.systemDefault()
 
   test("nanoseconds truncation") {
@@ -124,7 +121,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     checkFromToJavaDate(new Date(df2.parse("1776-07-04 18:30:00 UTC").getTime))
   }
 
-  private def toDate(s: String, zoneId: ZoneId = ZoneOffset.UTC): Option[SQLDate] = {
+  private def toDate(s: String, zoneId: ZoneId = UTC): Option[SQLDate] = {
     stringToDate(UTF8String.fromString(s), zoneId)
   }
 
@@ -176,29 +173,28 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the tz parameter.
-      var zoneId = getZoneId("GMT-13:53")
+      var zoneId = getZoneId("-13:53")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-13:53", expected)
       checkStringToTimestamp("2015-03-18T12:03:17GMT-13:53", expected)
 
-      zoneId = getZoneId("UTC")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, zid = UTC))
       checkStringToTimestamp("2015-03-18T12:03:17Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17UTC", expected)
 
-      zoneId = getZoneId("GMT-01:00")
+      zoneId = getZoneId("-01:00")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17-01:00", expected)
       checkStringToTimestamp("2015-03-18T12:03:17GMT-01:00", expected)
 
-      zoneId = getZoneId("GMT+07:30")
+      zoneId = getZoneId("+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:30", expected)
       checkStringToTimestamp("2015-03-18T12:03:17 GMT+07:30", expected)
 
-      zoneId = getZoneId("GMT+07:03")
+      zoneId = getZoneId("+07:03")
       expected = Option(date(2015, 3, 18, 12, 3, 17, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17+07:03", expected)
       checkStringToTimestamp("2015-03-18T12:03:17GMT+07:03", expected)
@@ -210,24 +206,23 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
       // If the string value includes timezone string, it represents the timestamp string
       // in the timezone regardless of the tz parameter.
-      zoneId = getZoneId("UTC")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, zid = zoneId))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 456000, zid = UTC))
       checkStringToTimestamp("2015-03-18T12:03:17.456Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17.456Z", expected)
       checkStringToTimestamp("2015-03-18 12:03:17.456 UTC", expected)
 
-      zoneId = getZoneId("GMT-01:00")
+      zoneId = getZoneId("-01:00")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123-1:0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123-01:00", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123 GMT-01:00", expected)
 
-      zoneId = getZoneId("GMT+07:30")
+      zoneId = getZoneId("+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123 GMT+07:30", expected)
 
-      zoneId = getZoneId("GMT+07:30")
+      zoneId = getZoneId("+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123000, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.123+07:30", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123GMT+07:30", expected)
@@ -236,7 +231,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       checkStringToTimestamp("2015-03-18T12:03:17.123121+7:30", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123121 GMT+0730", expected)
 
-      zoneId = getZoneId("GMT+07:30")
+      zoneId = getZoneId("+07:30")
       expected = Option(date(2015, 3, 18, 12, 3, 17, 123120, zid = zoneId))
       checkStringToTimestamp("2015-03-18T12:03:17.12312+7:30", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.12312 UT+07:30", expected)
@@ -244,12 +239,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       expected = Option(time(18, 12, 15, zid = zid))
       checkStringToTimestamp("18:12:15", expected)
 
-      zoneId = getZoneId("GMT+07:30")
+      zoneId = getZoneId("+07:30")
       expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("T18:12:15.12312+7:30", expected)
       checkStringToTimestamp("T18:12:15.12312 UTC+07:30", expected)
 
-      zoneId = getZoneId("GMT+07:30")
+      zoneId = getZoneId("+07:30")
       expected = Option(time(18, 12, 15, 123120, zid = zoneId))
       checkStringToTimestamp("18:12:15.12312+7:30", expected)
       checkStringToTimestamp("18:12:15.12312 GMT+07:30", expected)
@@ -276,8 +271,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       checkStringToTimestamp("1999 08", None)
 
       // Truncating the fractional seconds
-      zoneId = getZoneId("GMT+00:00")
-      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = zoneId))
+      expected = Option(date(2015, 3, 18, 12, 3, 17, 123456, zid = UTC))
       checkStringToTimestamp("2015-03-18T12:03:17.123456789+0:00", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123456789 UTC+0", expected)
       checkStringToTimestamp("2015-03-18T12:03:17.123456789GMT+00:00", expected)
@@ -306,38 +300,38 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("hours") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, zonePST)
-    assert(getHours(input, zonePST) === 13)
-    assert(getHours(input, zoneGMT) === 20)
-    input = date(2015, 12, 8, 2, 7, 9, 0, zonePST)
-    assert(getHours(input, zonePST) === 2)
-    assert(getHours(input, zoneGMT) === 10)
-    input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
-    assert(getHours(input, zonePST) === 0)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, LA)
+    assert(getHours(input, LA) === 13)
+    assert(getHours(input, UTC) === 20)
+    input = date(2015, 12, 8, 2, 7, 9, 0, LA)
+    assert(getHours(input, LA) === 2)
+    assert(getHours(input, UTC) === 10)
+    input = date(10, 1, 1, 0, 0, 0, 0, LA)
+    assert(getHours(input, LA) === 0)
   }
 
   test("minutes") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, zonePST)
-    assert(getMinutes(input, zonePST) === 2)
-    assert(getMinutes(input, zoneGMT) === 2)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, LA)
+    assert(getMinutes(input, LA) === 2)
+    assert(getMinutes(input, UTC) === 2)
     assert(getMinutes(input, getZoneId("Australia/North")) === 32)
-    input = date(2015, 3, 8, 2, 7, 9, 0, zonePST)
-    assert(getMinutes(input, zonePST) === 7)
-    assert(getMinutes(input, zoneGMT) === 7)
+    input = date(2015, 3, 8, 2, 7, 9, 0, LA)
+    assert(getMinutes(input, LA) === 7)
+    assert(getMinutes(input, UTC) === 7)
     assert(getMinutes(input, getZoneId("Australia/North")) === 37)
-    input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
-    assert(getMinutes(input, zonePST) === 0)
+    input = date(10, 1, 1, 0, 0, 0, 0, LA)
+    assert(getMinutes(input, LA) === 0)
   }
 
   test("seconds") {
-    var input = date(2015, 3, 18, 13, 2, 11, 0, zonePST)
-    assert(getSeconds(input, zonePST) === 11)
-    assert(getSeconds(input, zoneGMT) === 11)
-    input = date(2015, 3, 8, 2, 7, 9, 0, zonePST)
-    assert(getSeconds(input, zonePST) === 9)
-    assert(getSeconds(input, zoneGMT) === 9)
-    input = date(10, 1, 1, 0, 0, 0, 0, zonePST)
-    assert(getSeconds(input, zonePST) === 0)
+    var input = date(2015, 3, 18, 13, 2, 11, 0, LA)
+    assert(getSeconds(input, LA) === 11)
+    assert(getSeconds(input, UTC) === 11)
+    input = date(2015, 3, 8, 2, 7, 9, 0, LA)
+    assert(getSeconds(input, LA) === 9)
+    assert(getSeconds(input, UTC) === 9)
+    input = date(10, 1, 1, 0, 0, 0, 0, LA)
+    assert(getSeconds(input, LA) === 0)
   }
 
   test("hours / minutes / seconds") {
@@ -403,58 +397,58 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     val ts2 = date(2000, 2, 28, 10, 30, 0, 123000)
     assert(timestampAddInterval(ts1, 36, 0, 123000, defaultZoneId) === ts2)
 
-    val ts3 = date(1997, 2, 27, 16, 0, 0, 0, zonePST)
-    val ts4 = date(2000, 2, 27, 16, 0, 0, 123000, zonePST)
-    val ts5 = date(2000, 2, 28, 0, 0, 0, 123000, zoneGMT)
-    assert(timestampAddInterval(ts3, 36, 0, 123000, zonePST) === ts4)
-    assert(timestampAddInterval(ts3, 36, 0, 123000, zoneGMT) === ts5)
+    val ts3 = date(1997, 2, 27, 16, 0, 0, 0, LA)
+    val ts4 = date(2000, 2, 27, 16, 0, 0, 123000, LA)
+    val ts5 = date(2000, 2, 28, 0, 0, 0, 123000, UTC)
+    assert(timestampAddInterval(ts3, 36, 0, 123000, LA) === ts4)
+    assert(timestampAddInterval(ts3, 36, 0, 123000, UTC) === ts5)
   }
 
   test("timestamp add days") {
     // 2019-3-9 is the end of Pacific Standard Time
-    val ts1 = date(2019, 3, 9, 12, 0, 0, 123000, zonePST)
+    val ts1 = date(2019, 3, 9, 12, 0, 0, 123000, LA)
     // 2019-3-10 is the start of Pacific Daylight Time
-    val ts2 = date(2019, 3, 10, 12, 0, 0, 123000, zonePST)
-    val ts3 = date(2019, 5, 9, 12, 0, 0, 123000, zonePST)
-    val ts4 = date(2019, 5, 10, 12, 0, 0, 123000, zonePST)
+    val ts2 = date(2019, 3, 10, 12, 0, 0, 123000, LA)
+    val ts3 = date(2019, 5, 9, 12, 0, 0, 123000, LA)
+    val ts4 = date(2019, 5, 10, 12, 0, 0, 123000, LA)
     // 2019-11-2 is the end of Pacific Daylight Time
-    val ts5 = date(2019, 11, 2, 12, 0, 0, 123000, zonePST)
+    val ts5 = date(2019, 11, 2, 12, 0, 0, 123000, LA)
     // 2019-11-3 is the start of Pacific Standard Time
-    val ts6 = date(2019, 11, 3, 12, 0, 0, 123000, zonePST)
+    val ts6 = date(2019, 11, 3, 12, 0, 0, 123000, LA)
 
     // transit from Pacific Standard Time to Pacific Daylight Time
     assert(timestampAddInterval(
-      ts1, 0, 0, 23 * MICROS_PER_HOUR, zonePST) === ts2)
-    assert(timestampAddInterval(ts1, 0, 1, 0, zonePST) === ts2)
+      ts1, 0, 0, 23 * MICROS_PER_HOUR, LA) === ts2)
+    assert(timestampAddInterval(ts1, 0, 1, 0, LA) === ts2)
     // just a normal day
     assert(timestampAddInterval(
-      ts3, 0, 0, 24 * MICROS_PER_HOUR, zonePST) === ts4)
-    assert(timestampAddInterval(ts3, 0, 1, 0, zonePST) === ts4)
+      ts3, 0, 0, 24 * MICROS_PER_HOUR, LA) === ts4)
+    assert(timestampAddInterval(ts3, 0, 1, 0, LA) === ts4)
     // transit from Pacific Daylight Time to Pacific Standard Time
     assert(timestampAddInterval(
-      ts5, 0, 0, 25 * MICROS_PER_HOUR, zonePST) === ts6)
-    assert(timestampAddInterval(ts5, 0, 1, 0, zonePST) === ts6)
+      ts5, 0, 0, 25 * MICROS_PER_HOUR, LA) === ts6)
+    assert(timestampAddInterval(ts5, 0, 1, 0, LA) === ts6)
   }
 
   test("monthsBetween") {
     val date1 = date(1997, 2, 28, 10, 30, 0)
     var date2 = date(1996, 10, 30)
-    assert(monthsBetween(date1, date2, true, ZoneOffset.UTC) === 3.94959677)
-    assert(monthsBetween(date1, date2, false, ZoneOffset.UTC) === 3.9495967741935485)
+    assert(monthsBetween(date1, date2, true, UTC) === 3.94959677)
+    assert(monthsBetween(date1, date2, false, UTC) === 3.9495967741935485)
     Seq(true, false).foreach { roundOff =>
       date2 = date(2000, 2, 28)
-      assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === -36)
+      assert(monthsBetween(date1, date2, roundOff, UTC) === -36)
       date2 = date(2000, 2, 29)
-      assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === -36)
+      assert(monthsBetween(date1, date2, roundOff, UTC) === -36)
       date2 = date(1996, 3, 31)
-      assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === 11)
+      assert(monthsBetween(date1, date2, roundOff, UTC) === 11)
     }
 
-    val date3 = date(2000, 2, 28, 16, zid = zonePST)
-    val date4 = date(1997, 2, 28, 16, zid = zonePST)
-    assert(monthsBetween(date3, date4, true, zonePST) === 36.0)
-    assert(monthsBetween(date3, date4, true, ZoneOffset.UTC) === 35.90322581)
-    assert(monthsBetween(date3, date4, false, ZoneOffset.UTC) === 35.903225806451616)
+    val date3 = date(2000, 2, 28, 16, zid = LA)
+    val date4 = date(1997, 2, 28, 16, zid = LA)
+    assert(monthsBetween(date3, date4, true, LA) === 36.0)
+    assert(monthsBetween(date3, date4, true, UTC) === 35.90322581)
+    assert(monthsBetween(date3, date4, false, UTC) === 35.903225806451616)
   }
 
   test("from UTC timestamp") {
@@ -465,19 +459,19 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     for (tz <- ALL_TIMEZONES) {
       withDefaultTimeZone(tz) {
         test("2011-12-25 09:00:00.123456", "UTC", "2011-12-25 09:00:00.123456")
-        test("2011-12-25 09:00:00.123456", "JST", "2011-12-25 18:00:00.123456")
-        test("2011-12-25 09:00:00.123456", "PST", "2011-12-25 01:00:00.123456")
+        test("2011-12-25 09:00:00.123456", JST.getId, "2011-12-25 18:00:00.123456")
+        test("2011-12-25 09:00:00.123456", LA.getId, "2011-12-25 01:00:00.123456")
         test("2011-12-25 09:00:00.123456", "Asia/Shanghai", "2011-12-25 17:00:00.123456")
       }
     }
 
-    withDefaultTimeZone(TimeZone.getTimeZone("PST")) {
+    withDefaultTimeZone(TimeZone.getTimeZone(LA.getId)) {
       // Daylight Saving Time
-      test("2016-03-13 09:59:59.0", "PST", "2016-03-13 01:59:59.0")
-      test("2016-03-13 10:00:00.0", "PST", "2016-03-13 03:00:00.0")
-      test("2016-11-06 08:59:59.0", "PST", "2016-11-06 01:59:59.0")
-      test("2016-11-06 09:00:00.0", "PST", "2016-11-06 01:00:00.0")
-      test("2016-11-06 10:00:00.0", "PST", "2016-11-06 02:00:00.0")
+      test("2016-03-13 09:59:59.0", LA.getId, "2016-03-13 01:59:59.0")
+      test("2016-03-13 10:00:00.0", LA.getId, "2016-03-13 03:00:00.0")
+      test("2016-11-06 08:59:59.0", LA.getId, "2016-11-06 01:59:59.0")
+      test("2016-11-06 09:00:00.0", LA.getId, "2016-11-06 01:00:00.0")
+      test("2016-11-06 10:00:00.0", LA.getId, "2016-11-06 02:00:00.0")
     }
   }
 
@@ -490,13 +484,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     for (tz <- ALL_TIMEZONES) {
       withDefaultTimeZone(tz) {
         test("2011-12-25 09:00:00.123456", "UTC", "2011-12-25 09:00:00.123456")
-        test("2011-12-25 18:00:00.123456", "JST", "2011-12-25 09:00:00.123456")
-        test("2011-12-25 01:00:00.123456", "PST", "2011-12-25 09:00:00.123456")
+        test("2011-12-25 18:00:00.123456", JST.getId, "2011-12-25 09:00:00.123456")
+        test("2011-12-25 01:00:00.123456", LA.getId, "2011-12-25 09:00:00.123456")
         test("2011-12-25 17:00:00.123456", "Asia/Shanghai", "2011-12-25 09:00:00.123456")
       }
     }
 
-    val tz = "America/Los_Angeles"
+    val tz = LA.getId
     withDefaultTimeZone(TimeZone.getTimeZone(tz)) {
       // Daylight Saving Time
       test("2016-03-13 01:59:59", tz, "2016-03-13 09:59:59.0")
@@ -596,16 +590,16 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("daysToMillis and millisToDays") {
-    val input = toMillis(date(2015, 12, 31, 16, zid = zonePST))
-    assert(millisToDays(input, zonePST) === 16800)
-    assert(millisToDays(input, ZoneOffset.UTC) === 16801)
-    assert(millisToDays(-1 * MILLIS_PER_DAY + 1, ZoneOffset.UTC) == -1)
+    val input = toMillis(date(2015, 12, 31, 16, zid = LA))
+    assert(millisToDays(input, LA) === 16800)
+    assert(millisToDays(input, UTC) === 16801)
+    assert(millisToDays(-1 * MILLIS_PER_DAY + 1, UTC) == -1)
 
-    var expected = toMillis(date(2015, 12, 31, zid = zonePST))
-    assert(daysToMillis(16800, zonePST) === expected)
+    var expected = toMillis(date(2015, 12, 31, zid = LA))
+    assert(daysToMillis(16800, LA) === expected)
 
-    expected = toMillis(date(2015, 12, 31, zid = zoneGMT))
-    assert(daysToMillis(16800, ZoneOffset.UTC) === expected)
+    expected = toMillis(date(2015, 12, 31, zid = UTC))
+    assert(daysToMillis(16800, UTC) === expected)
 
     // There are some days are skipped entirely in some timezone, skip them here.
     val skipped_days = Map[String, Set[Int]](
@@ -615,7 +609,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       "Pacific/Fakaofo" -> Set(15338),
       "Pacific/Kiritimati" -> Set(9130, 9131),
       "Pacific/Kwajalein" -> Set(8632, 8633, 8634),
-      "MIT" -> Set(15338))
+      MIT.getId -> Set(15338))
     for (tz <- ALL_TIMEZONES) {
       val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
       val testingData = Seq(-20000, 20000) ++
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
index 4439a7bb3ae87..0ffb492e702d1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/ArrowUtilsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.util
 import org.apache.arrow.vector.types.pojo.ArrowType
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.LA
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.types._
 
@@ -65,7 +66,7 @@ class ArrowUtilsSuite extends SparkFunSuite {
     roundtripWithTz(DateTimeUtils.defaultTimeZone().getID)
     roundtripWithTz("Asia/Tokyo")
     roundtripWithTz("UTC")
-    roundtripWithTz("America/Los_Angeles")
+    roundtripWithTz(LA.getId)
   }
 
   test("array") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 3cf1437864558..b2c39242aad53 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -25,6 +25,7 @@ import org.scalatest.Matchers
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CET, PST, UTC}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -34,8 +35,8 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     val localDate = "2018-12-02T10:11:12.001234"
     val expectedMicros = Map(
       "UTC" -> 1543745472001234L,
-      "PST" -> 1543774272001234L,
-      "CET" -> 1543741872001234L,
+      PST.getId -> 1543774272001234L,
+      CET.getId -> 1543741872001234L,
       "Africa/Dakar" -> 1543745472001234L,
       "America/Los_Angeles" -> 1543774272001234L,
       "Antarctica/Vostok" -> 1543723872001234L,
@@ -55,8 +56,8 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     val microsSinceEpoch = 1543745472001234L
     val expectedTimestamp = Map(
       "UTC" -> "2018-12-02T10:11:12.001234",
-      "PST" -> "2018-12-02T02:11:12.001234",
-      "CET" -> "2018-12-02T11:11:12.001234",
+      PST.getId -> "2018-12-02T02:11:12.001234",
+      CET.getId -> "2018-12-02T11:11:12.001234",
       "Africa/Dakar" -> "2018-12-02T10:11:12.001234",
       "America/Los_Angeles" -> "2018-12-02T02:11:12.001234",
       "Antarctica/Vostok" -> "2018-12-02T16:11:12.001234",
@@ -251,7 +252,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     }
 
     // "yyyy" with "G" can't parse negative year or year 0000.
-    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC, true)
+    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", UTC, true)
     assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
     assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index c41eb98c13ea0..875a671a01bfb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.plans.logical.OneRowRelation
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -1028,7 +1029,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
         Timestamp.valueOf("2018-01-01 12:00:00"),
         Timestamp.valueOf("2018-01-02 00:00:00")))))
 
-    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone("UTC")) {
+    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
       checkAnswer(
         spark.sql("select sequence(" +
           "   cast('2018-01-01' as date)" +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
index 51c6a835d58d6..51d861ec69b23 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
@@ -258,7 +258,7 @@ class DataFramePivotSuite extends QueryTest with SharedSparkSession {
     val ts = "2012-12-31 16:00:10.011"
     val tsWithZone = "2013-01-01 00:00:10.011"
 
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
       val df = Seq(java.sql.Timestamp.valueOf(ts)).toDF("a").groupBy("a").pivot("a").count()
       val expected = StructType(
         StructField("a", TimestampType) ::
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index e74d5532849dd..72aa7bf084c00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1167,7 +1167,7 @@ class DataFrameSuite extends QueryTest
                            |""".stripMargin
     assert(df.showString(1, truncate = 0) === expectedAnswer)
 
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
 
       val expectedAnswer = """+----------+-------------------+
                              ||d         |ts                 |
@@ -1188,7 +1188,7 @@ class DataFrameSuite extends QueryTest
                          " ts  | 2016-12-01 00:00:00 \n"
     assert(df.showString(1, truncate = 0, vertical = true) === expectedAnswer)
 
-    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+    withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
 
       val expectedAnswer = "-RECORD 0------------------\n" +
                            " d   | 2016-12-01          \n" +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index da7cc1c930461..6ed748304686b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -24,6 +24,7 @@ import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CEST, LA}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -761,12 +762,12 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
     ).toDF("a", "b")
     checkAnswer(
-      df.select(from_utc_timestamp(col("a"), "PST")),
+      df.select(from_utc_timestamp(col("a"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-23 17:00:00")),
         Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
     checkAnswer(
-      df.select(from_utc_timestamp(col("b"), "PST")),
+      df.select(from_utc_timestamp(col("b"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-23 17:00:00")),
         Row(Timestamp.valueOf("2015-07-24 17:00:00"))))
@@ -774,8 +775,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
   test("from_utc_timestamp with column zone") {
     val df = Seq(
-      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "CET"),
-      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "PST")
+      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", CEST.getId),
+      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", LA.getId)
     ).toDF("a", "b", "c")
     checkAnswer(
       df.select(from_utc_timestamp(col("a"), col("c"))),
@@ -804,12 +805,12 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
     ).toDF("a", "b")
     checkAnswer(
-      df.select(to_utc_timestamp(col("a"), "PST")),
+      df.select(to_utc_timestamp(col("a"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-24 07:00:00")),
         Row(Timestamp.valueOf("2015-07-25 07:00:00"))))
     checkAnswer(
-      df.select(to_utc_timestamp(col("b"), "PST")),
+      df.select(to_utc_timestamp(col("b"), LA.getId)),
       Seq(
         Row(Timestamp.valueOf("2015-07-24 07:00:00")),
         Row(Timestamp.valueOf("2015-07-25 07:00:00"))))
@@ -817,8 +818,8 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
 
   test("to_utc_timestamp with column zone") {
     val df = Seq(
-      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "PST"),
-      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "CET")
+      (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", LA.getId),
+      (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", CEST.getId)
     ).toDF("a", "b", "c")
     checkAnswer(
       df.select(to_utc_timestamp(col("a"), col("c"))),
@@ -849,10 +850,10 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-30752: convert time zones on a daylight saving day") {
-    val systemTz = "PST"
+    val systemTz = LA.getId
     val sessionTz = "UTC"
     val fromTz = "Asia/Hong_Kong"
-    val fromTs = "2019-11-03T12:00:00" // daylight saving date in PST
+    val fromTs = "2019-11-03T12:00:00" // daylight saving date in America/Los_Angeles
     val utsTs = "2019-11-03T04:00:00"
     val defaultTz = TimeZone.getDefault
     try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 30ae9dc67a1e1..f9a510df83f45 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -1080,7 +1080,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .format("csv")
         .option("header", "true")
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .save(timestampsWithFormatPath)
 
       // This will load back the timestamps as string.
@@ -1102,7 +1102,7 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
         .option("header", "true")
         .option("inferSchema", "true")
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .load(timestampsWithFormatPath)
 
       checkAnswer(readBack, timestampsWithFormat)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index fb3328c51546e..54668005a72dc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -21,7 +21,7 @@ import java.io._
 import java.nio.charset.{Charset, StandardCharsets, UnsupportedCharsetException}
 import java.nio.file.Files
 import java.sql.{Date, Timestamp}
-import java.time.{LocalDate, LocalDateTime, ZoneId}
+import java.time.LocalDate
 import java.util.Locale
 
 import com.fasterxml.jackson.core.JsonFactory
@@ -1329,7 +1329,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
 
   test("SPARK-6245 JsonInferSchema.infer on empty RDD") {
     // This is really a test that it doesn't throw an exception
-    val options = new JSONOptions(Map.empty[String, String], "GMT")
+    val options = new JSONOptions(Map.empty[String, String], "UTC")
     val emptySchema = new JsonInferSchema(options).infer(
       empty.rdd,
       CreateJacksonParser.string)
@@ -1356,7 +1356,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("SPARK-8093 Erase empty structs") {
-    val options = new JSONOptions(Map.empty[String, String], "GMT")
+    val options = new JSONOptions(Map.empty[String, String], "UTC")
     val emptySchema = new JsonInferSchema(options).infer(
       emptyRecords.rdd,
       CreateJacksonParser.string)
@@ -1730,7 +1730,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       timestampsWithFormat.write
         .format("json")
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .save(timestampsWithFormatPath)
 
       // This will load back the timestamps as string.
@@ -1748,7 +1748,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val readBack = spark.read
         .schema(customSchema)
         .option("timestampFormat", "yyyy/MM/dd HH:mm")
-        .option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+        .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .json(timestampsWithFormatPath)
 
       checkAnswer(readBack, timestampsWithFormat)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
index f4499fe17fdda..d3301ced2ba19 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
@@ -88,7 +88,7 @@ abstract class ParquetPartitionDiscoverySuite
     check("1990-02-24 12:00:30",
       Literal.create(Timestamp.valueOf("1990-02-24 12:00:30"), TimestampType))
 
-    val c = Calendar.getInstance(TimeZone.getTimeZone("GMT"))
+    val c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
     c.set(1990, 1, 24, 12, 0, 30)
     c.set(Calendar.MILLISECOND, 0)
     check("1990-02-24 12:00:30",
@@ -692,10 +692,10 @@ abstract class ParquetPartitionDiscoverySuite
     }
 
     withTempPath { dir =>
-      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .format("parquet").partitionBy(partitionColumns.map(_.name): _*).save(dir.toString)
       val fields = schema.map(f => Column(f.name).cast(f.dataType))
-      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .load(dir.toString).select(fields: _*), row)
     }
   }
@@ -734,10 +734,10 @@ abstract class ParquetPartitionDiscoverySuite
     }
 
     withTempPath { dir =>
-      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .format("parquet").partitionBy(partitionColumns.map(_.name): _*).save(dir.toString)
       val fields = schema.map(f => Column(f.name))
-      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      checkAnswer(spark.read.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .load(dir.toString).select(fields: _*), row)
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index 10ea948656b17..cde2aa71b4ef5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path
 import org.apache.log4j.Level
 
 import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.MIT
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.test.{SharedSparkSession, TestSQLContext}
 import org.apache.spark.util.Utils
@@ -350,8 +351,8 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
   }
 
   test("spark.sql.session.timeZone should only accept valid zone id") {
-    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "MIT")
-    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) === "MIT")
+    spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, MIT.getId)
+    assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) === MIT.getId)
     spark.conf.set(SQLConf.SESSION_LOCAL_TIMEZONE.key, "America/Chicago")
     assert(sql(s"set ${SQLConf.SESSION_LOCAL_TIMEZONE.key}").head().getString(1) ===
       "America/Chicago")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
index ab1d1f80e7397..1c4e2a967b0a2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
@@ -139,15 +139,15 @@ class PartitionedWriteSuite extends QueryTest with SharedSparkSession {
       checkPartitionValues(files.head, "2016-12-01 00:00:00")
     }
     withTempPath { f =>
-      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "GMT")
+      df.write.option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
         .partitionBy("ts").parquet(f.getAbsolutePath)
       val files = TestUtils.recursiveList(f).filter(_.getAbsolutePath.endsWith("parquet"))
       assert(files.length == 1)
-      // use timeZone option "GMT" to format partition value.
+      // use timeZone option utcTz.getId to format partition value.
       checkPartitionValues(files.head, "2016-12-01 08:00:00")
     }
     withTempPath { f =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
         df.write.partitionBy("ts").parquet(f.getAbsolutePath)
         val files = TestUtils.recursiveList(f).filter(_.getAbsolutePath.endsWith("parquet"))
         assert(files.length == 1)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
index 92ec2a0c172ef..f29a6c7f77079 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
@@ -29,6 +29,7 @@ import org.scalatest.{BeforeAndAfter, Matchers}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Dataset}
 import org.apache.spark.sql.catalyst.plans.logical.EventTimeWatermark
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC
 import org.apache.spark.sql.execution.streaming._
 import org.apache.spark.sql.execution.streaming.sources.MemorySink
 import org.apache.spark.sql.functions.{count, window}
@@ -773,7 +774,7 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
   }
 
   private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
-  timestampFormat.setTimeZone(ju.TimeZone.getTimeZone("UTC"))
+  timestampFormat.setTimeZone(ju.TimeZone.getTimeZone(UTC))
 
   private def formatTimestamp(sec: Long): String = {
     timestampFormat.format(new ju.Date(sec * 1000))

From a36e3c4c898b513b90e58859e3ca8e550d5cb0cd Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 26 Mar 2020 14:24:53 +0800
Subject: [PATCH 0347/1280] [SPARK-30934][ML][FOLLOW-UP] Update ml-guide to
 include MulticlassClassificationEvaluator weight support in highlights

### What changes were proposed in this pull request?
Update ml-guide to include ```MulticlassClassificationEvaluator``` weight support in highlights

### Why are the changes needed?
```MulticlassClassificationEvaluator``` weight support is very important, so should include it in highlights

### Does this PR introduce any user-facing change?
Yes

after:
![image](https://user-images.githubusercontent.com/13592258/77614952-6ccd8680-6eeb-11ea-9354-fa20004132df.png)

### How was this patch tested?
manually build and check

Closes #28031 from huaxingao/highlights-followup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: zhengruifeng <ruifengz@foxmail.com>
(cherry picked from commit ee6f8991a792e24a5b1c020d958877187af2f41b)
Signed-off-by: zhengruifeng <ruifengz@foxmail.com>
---
 docs/ml-guide.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/ml-guide.md b/docs/ml-guide.md
index 5ce6b4f58c2f7..ddce98b32f941 100644
--- a/docs/ml-guide.md
+++ b/docs/ml-guide.md
@@ -91,10 +91,10 @@ The list below highlights some of the new features and enhancements added to MLl
 release of Spark:
 
 * Multiple columns support was added to `Binarizer` ([SPARK-23578](https://issues.apache.org/jira/browse/SPARK-23578)), `StringIndexer` ([SPARK-11215](https://issues.apache.org/jira/browse/SPARK-11215)), `StopWordsRemover` ([SPARK-29808](https://issues.apache.org/jira/browse/SPARK-29808)) and PySpark `QuantileDiscretizer` ([SPARK-22796](https://issues.apache.org/jira/browse/SPARK-22796)).
-* Support Tree-Based Feature Transformation was added
+* Tree-Based Feature Transformation was added
 ([SPARK-13677](https://issues.apache.org/jira/browse/SPARK-13677)).
 * Two new evaluators `MultilabelClassificationEvaluator` ([SPARK-16692](https://issues.apache.org/jira/browse/SPARK-16692)) and `RankingEvaluator` ([SPARK-28045](https://issues.apache.org/jira/browse/SPARK-28045)) were added.
-* Sample weights support was added in `DecisionTreeClassifier/Regressor` ([SPARK-19591](https://issues.apache.org/jira/browse/SPARK-19591)), `RandomForestClassifier/Regressor` ([SPARK-9478](https://issues.apache.org/jira/browse/SPARK-9478)), `GBTClassifier/Regressor` ([SPARK-9612](https://issues.apache.org/jira/browse/SPARK-9612)),  `RegressionEvaluator` ([SPARK-24102](https://issues.apache.org/jira/browse/SPARK-24102)), `BinaryClassificationEvaluator` ([SPARK-24103](https://issues.apache.org/jira/browse/SPARK-24103)), `BisectingKMeans` ([SPARK-30351](https://issues.apache.org/jira/browse/SPARK-30351)), `KMeans` ([SPARK-29967](https://issues.apache.org/jira/browse/SPARK-29967)) and `GaussianMixture` ([SPARK-30102](https://issues.apache.org/jira/browse/SPARK-30102)).
+* Sample weights support was added in `DecisionTreeClassifier/Regressor` ([SPARK-19591](https://issues.apache.org/jira/browse/SPARK-19591)), `RandomForestClassifier/Regressor` ([SPARK-9478](https://issues.apache.org/jira/browse/SPARK-9478)), `GBTClassifier/Regressor` ([SPARK-9612](https://issues.apache.org/jira/browse/SPARK-9612)),  `MulticlassClassificationEvaluator` ([SPARK-24101](https://issues.apache.org/jira/browse/SPARK-24101)), `RegressionEvaluator` ([SPARK-24102](https://issues.apache.org/jira/browse/SPARK-24102)), `BinaryClassificationEvaluator` ([SPARK-24103](https://issues.apache.org/jira/browse/SPARK-24103)), `BisectingKMeans` ([SPARK-30351](https://issues.apache.org/jira/browse/SPARK-30351)), `KMeans` ([SPARK-29967](https://issues.apache.org/jira/browse/SPARK-29967)) and `GaussianMixture` ([SPARK-30102](https://issues.apache.org/jira/browse/SPARK-30102)).
 * R API for `PowerIterationClustering` was added
 ([SPARK-19827](https://issues.apache.org/jira/browse/SPARK-19827)).
 * Added Spark ML listener for tracking ML pipeline status
@@ -105,10 +105,10 @@ release of Spark:
 ([SPARK-28399](https://issues.apache.org/jira/browse/SPARK-28399)).
 * [`Factorization Machines`](ml-classification-regression.html#factorization-machines) classifier and regressor were added
 ([SPARK-29224](https://issues.apache.org/jira/browse/SPARK-29224)).
-* Gaussian Naive Bayes ([SPARK-16872](https://issues.apache.org/jira/browse/SPARK-16872)) and Complement Naive Bayes ([SPARK-29942](https://issues.apache.org/jira/browse/SPARK-29942)) were added.
+* Gaussian Naive Bayes Classifier ([SPARK-16872](https://issues.apache.org/jira/browse/SPARK-16872)) and Complement Naive Bayes Classifier ([SPARK-29942](https://issues.apache.org/jira/browse/SPARK-29942)) were added.
 * ML function parity between Scala and Python
 ([SPARK-28958](https://issues.apache.org/jira/browse/SPARK-28958)).
-* `predictRaw` is made public in all the Classification models. `predictProbability` is made public in all the Classification models except `LinearSVCModel`.
+* `predictRaw` is made public in all the Classification models. `predictProbability` is made public in all the Classification models except `LinearSVCModel`
 ([SPARK-30358](https://issues.apache.org/jira/browse/SPARK-30358)).
 
 # Migration Guide

From 557623b5a1ebd50b0f604843628b9d0ed8ff19ef Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 26 Mar 2020 15:03:16 +0800
Subject: [PATCH 0348/1280] [SPARK-31234][SQL] ResetCommand should reset config
 to sc.conf only

### What changes were proposed in this pull request?
Currently, ResetCommand clear all configurations, including sql configs, static sql configs and spark context level configs.
for example:
```sql
spark-sql> set xyz=abc;
xyz abc
spark-sql> set;
spark.app.id local-1585055396930
spark.app.name SparkSQL::10.242.189.214
spark.driver.host 10.242.189.214
spark.driver.port 65094
spark.executor.id driver
spark.jars
spark.master local[*]
spark.sql.catalogImplementation hive
spark.sql.hive.version 1.2.1
spark.submit.deployMode client
xyz abc
spark-sql> reset;
spark-sql> set;
spark-sql> set spark.sql.hive.version;
spark.sql.hive.version 1.2.1
spark-sql> set spark.app.id;
spark.app.id <undefined>
```
In this PR, we restore spark confs to  RuntimeConfig after it is cleared

### Why are the changes needed?
reset command overkills configs which are static.
### Does this PR introduce any user-facing change?

yes, the ResetCommand do not change static configs now

### How was this patch tested?

add ut

Closes #28003 from yaooqinn/SPARK-31234.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 44bd36ad7b315f4c7592cdc1edf04356fcd23645)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-syntax-aux-conf-mgmt-reset.md        |  4 ++--
 .../spark/sql/execution/command/SetCommand.scala  |  9 +++++++--
 .../apache/spark/sql/internal/SQLConfSuite.scala  | 15 +++++++++++++++
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
index 8ee61514ee4ef..5ebc7b97ef64f 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
@@ -20,7 +20,7 @@ license: |
 ---
 
 ### Description
-Reset all the properties specific to the current session to their default values. After RESET command, executing SET command will output empty.
+Reset any runtime configurations specific to the current session which were set via the [SET](sql-ref-syntax-aux-conf-mgmt-set.html) command to their default values.
 
 ### Syntax
 {% highlight sql %}
@@ -30,7 +30,7 @@ RESET
 
 ### Examples
 {% highlight sql %}
--- Reset all the properties specific to the current session to their default values.
+-- Reset any runtime configurations specific to the current session which were set via the SET command to their default values.
 RESET;
 {% endhighlight %}
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
index 39b08e2894dcd..a12b261ce50af 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
@@ -157,7 +157,8 @@ object SetCommand {
 }
 
 /**
- * This command is for resetting SQLConf to the default values. Command that runs
+ * This command is for resetting SQLConf to the default values. Any configurations that were set
+ * via [[SetCommand]] will get reset to default value. Command that runs
  * {{{
  *   reset;
  * }}}
@@ -165,7 +166,11 @@ object SetCommand {
 case object ResetCommand extends RunnableCommand with IgnoreCachedData {
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    sparkSession.sessionState.conf.clear()
+    val conf = sparkSession.sessionState.conf
+    conf.clear()
+    sparkSession.sparkContext.conf.getAll.foreach { case (k, v) =>
+      conf.setConfString(k, v)
+    }
     Seq.empty[Row]
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index cde2aa71b4ef5..c2d8493da92b6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -116,6 +116,21 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("reset will not change static sql configs and spark core configs") {
+    val conf = spark.sparkContext.getConf.getAll.toMap
+    val appName = conf.get("spark.app.name")
+    val driverHost = conf.get("spark.driver.host")
+    val master = conf.get("spark.master")
+    val warehouseDir = conf.get("spark.sql.warehouse.dir")
+    // ensure the conf here is not default value, and will not be reset to default value later
+    assert(warehouseDir.get.contains(this.getClass.getCanonicalName))
+    sql("RESET")
+    assert(conf.get("spark.app.name") === appName)
+    assert(conf.get("spark.driver.host") === driverHost)
+    assert(conf.get("spark.master") === master)
+    assert(conf.get("spark.sql.warehouse.dir") === warehouseDir)
+  }
+
   test("reset - public conf") {
     spark.sessionState.conf.clear()
     val original = spark.conf.get(SQLConf.GROUP_BY_ORDINAL)

From bd94cf7bcccd305cac1f301d2cefb043afba35f6 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 26 Mar 2020 15:42:54 +0800
Subject: [PATCH 0349/1280] [SPARK-31227][SQL] Non-nullable null type in
 complex types should not coerce to nullable type

### What changes were proposed in this pull request?

This PR targets for non-nullable null type not to coerce to nullable type in complex types.

Non-nullable fields in struct, elements in an array and entries in map can mean empty array, struct and map. They are empty so it does not need to force the nullability when we find common types.

This PR also reverts and supersedes https://github.com/apache/spark/commit/d7b97a1d0daf65710317321490a833f696a46f21

### Why are the changes needed?

To make type coercion coherent and consistent. Currently, we correctly keep the nullability even between non-nullable fields:

```scala
import org.apache.spark.sql.types._
import org.apache.spark.sql.functions._
spark.range(1).select(array(lit(1)).cast(ArrayType(IntegerType, false))).printSchema()
spark.range(1).select(array(lit(1)).cast(ArrayType(DoubleType, false))).printSchema()
```
```scala
spark.range(1).selectExpr("concat(array(1), array(1)) as arr").printSchema()
```

### Does this PR introduce any user-facing change?

Yes.

```scala
import org.apache.spark.sql.types._
import org.apache.spark.sql.functions._
spark.range(1).select(array().cast(ArrayType(IntegerType, false))).printSchema()
```
```scala
spark.range(1).selectExpr("concat(array(), array(1)) as arr").printSchema()
```

**Before:**

```
org.apache.spark.sql.AnalysisException: cannot resolve 'array()' due to data type mismatch: cannot cast array<null> to array<int>;;
'Project [cast(array() as array<int>) AS array()#68]
+- Range (0, 1, step=1, splits=Some(12))

  at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
  at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$$nestedInanonfun$checkAnalysis$1$2.applyOrElse(CheckAnalysis.scala:149)
  at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$$nestedInanonfun$checkAnalysis$1$2.applyOrElse(CheckAnalysis.scala:140)
  at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUp$2(TreeNode.scala:333)
  at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:72)
  at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:333)
  at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUp$1(TreeNode.scala:330)
  at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$mapChildren$1(TreeNode.scala:399)
  at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:237)
```

```
root
 |-- arr: array (nullable = false)
 |    |-- element: integer (containsNull = true)
```

**After:**

```
root
 |-- array(): array (nullable = false)
 |    |-- element: integer (containsNull = false)
```

```
root
 |-- arr: array (nullable = false)
 |    |-- element: integer (containsNull = false)
```

### How was this patch tested?

Unittests were added and manually tested.

Closes #27991 from HyukjinKwon/SPARK-31227.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3bd10ce007832522e38583592b6f358e185cdb7d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/TypeCoercion.scala  |  2 +-
 .../spark/sql/catalyst/expressions/Cast.scala | 17 +++---
 .../catalyst/analysis/TypeCoercionSuite.scala | 54 ++++++++++++-------
 .../sql/catalyst/expressions/CastSuite.scala  | 22 ++++++++
 .../spark/sql/DataFrameFunctionsSuite.scala   |  7 +++
 5 files changed, 73 insertions(+), 29 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index 5a5d7c6620fb5..eb9a4d4feb783 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -160,7 +160,7 @@ object TypeCoercion {
       }
     case (MapType(kt1, vt1, valueContainsNull1), MapType(kt2, vt2, valueContainsNull2)) =>
       findTypeFunc(kt1, kt2)
-        .filter { kt => Cast.canCastMapKeyNullSafe(kt1, kt) && Cast.canCastMapKeyNullSafe(kt2, kt) }
+        .filter { kt => !Cast.forceNullable(kt1, kt) && !Cast.forceNullable(kt2, kt) }
         .flatMap { kt =>
           findTypeFunc(vt1, vt2).map { vt =>
             MapType(kt, vt, valueContainsNull1 || valueContainsNull2 ||
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 8177136edfd62..8d82956cc6f74 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -77,7 +77,8 @@ object Cast {
         resolvableNullability(fn || forceNullable(fromType, toType), tn)
 
     case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
-      canCast(fromKey, toKey) && canCastMapKeyNullSafe(fromKey, toKey) &&
+      canCast(fromKey, toKey) &&
+        (!forceNullable(fromKey, toKey)) &&
         canCast(fromValue, toValue) &&
         resolvableNullability(fn || forceNullable(fromValue, toValue), tn)
 
@@ -97,11 +98,6 @@ object Cast {
     case _ => false
   }
 
-  def canCastMapKeyNullSafe(fromType: DataType, toType: DataType): Boolean = {
-    // If the original map key type is NullType, it's OK as the map must be empty.
-    fromType == NullType || !forceNullable(fromType, toType)
-  }
-
   /**
    * Return true if we need to use the `timeZone` information casting `from` type to `to` type.
    * The patterns matched reflect the current implementation in the Cast node.
@@ -210,8 +206,13 @@ object Cast {
     case _ => false  // overflow
   }
 
+  /**
+   * Returns `true` if casting non-nullable values from `from` type to `to` type
+   * may return null. Note that the caller side should take care of input nullability
+   * first and only call this method if the input is not nullable.
+   */
   def forceNullable(from: DataType, to: DataType): Boolean = (from, to) match {
-    case (NullType, _) => true
+    case (NullType, _) => false // empty array or map case
     case (_, _) if from == to => false
 
     case (StringType, BinaryType) => false
@@ -269,7 +270,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     }
   }
 
-  override def nullable: Boolean = Cast.forceNullable(child.dataType, dataType) || child.nullable
+  override def nullable: Boolean = child.nullable || Cast.forceNullable(child.dataType, dataType)
 
   protected def ansiEnabled: Boolean
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index 95005fd3f5a5e..ab21a9ea5ba18 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 
 class TypeCoercionSuite extends AnalysisTest {
+  import TypeCoercionSuite._
 
   // scalastyle:off line.size.limit
   // The following table shows all implicit data type conversions that are not visible to the user.
@@ -99,22 +100,6 @@ class TypeCoercionSuite extends AnalysisTest {
     case _ => Literal.create(null, dataType)
   }
 
-  val integralTypes: Seq[DataType] =
-    Seq(ByteType, ShortType, IntegerType, LongType)
-  val fractionalTypes: Seq[DataType] =
-    Seq(DoubleType, FloatType, DecimalType.SYSTEM_DEFAULT, DecimalType(10, 2))
-  val numericTypes: Seq[DataType] = integralTypes ++ fractionalTypes
-  val atomicTypes: Seq[DataType] =
-    numericTypes ++ Seq(BinaryType, BooleanType, StringType, DateType, TimestampType)
-  val complexTypes: Seq[DataType] =
-    Seq(ArrayType(IntegerType),
-      ArrayType(StringType),
-      MapType(StringType, StringType),
-      new StructType().add("a1", StringType),
-      new StructType().add("a1", StringType).add("a2", IntegerType))
-  val allTypes: Seq[DataType] =
-    atomicTypes ++ complexTypes ++ Seq(NullType, CalendarIntervalType)
-
   // Check whether the type `checkedType` can be cast to all the types in `castableTypes`,
   // but cannot be cast to the other types in `allTypes`.
   private def checkTypeCasting(checkedType: DataType, castableTypes: Seq[DataType]): Unit = {
@@ -497,6 +482,23 @@ class TypeCoercionSuite extends AnalysisTest {
         .add("null", IntegerType, nullable = false),
       Some(new StructType()
         .add("null", IntegerType, nullable = true)))
+
+    widenTest(
+      ArrayType(NullType, containsNull = false),
+      ArrayType(IntegerType, containsNull = false),
+      Some(ArrayType(IntegerType, containsNull = false)))
+
+    widenTest(MapType(NullType, NullType, false),
+      MapType(IntegerType, StringType, false),
+      Some(MapType(IntegerType, StringType, false)))
+
+    widenTest(
+      new StructType()
+        .add("null", NullType, nullable = false),
+      new StructType()
+        .add("null", IntegerType, nullable = false),
+      Some(new StructType()
+        .add("null", IntegerType, nullable = false)))
   }
 
   test("wider common type for decimal and array") {
@@ -728,8 +730,6 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("cast NullType for expressions that implement ExpectsInputTypes") {
-    import TypeCoercionSuite._
-
     ruleTest(TypeCoercion.ImplicitTypeCasts,
       AnyTypeUnaryExpression(Literal.create(null, NullType)),
       AnyTypeUnaryExpression(Literal.create(null, NullType)))
@@ -740,8 +740,6 @@ class TypeCoercionSuite extends AnalysisTest {
   }
 
   test("cast NullType for binary operators") {
-    import TypeCoercionSuite._
-
     ruleTest(TypeCoercion.ImplicitTypeCasts,
       AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)),
       AnyTypeBinaryOperator(Literal.create(null, NullType), Literal.create(null, NullType)))
@@ -1548,6 +1546,22 @@ class TypeCoercionSuite extends AnalysisTest {
 
 object TypeCoercionSuite {
 
+  val integralTypes: Seq[DataType] =
+    Seq(ByteType, ShortType, IntegerType, LongType)
+  val fractionalTypes: Seq[DataType] =
+    Seq(DoubleType, FloatType, DecimalType.SYSTEM_DEFAULT, DecimalType(10, 2))
+  val numericTypes: Seq[DataType] = integralTypes ++ fractionalTypes
+  val atomicTypes: Seq[DataType] =
+    numericTypes ++ Seq(BinaryType, BooleanType, StringType, DateType, TimestampType)
+  val complexTypes: Seq[DataType] =
+    Seq(ArrayType(IntegerType),
+      ArrayType(StringType),
+      MapType(StringType, StringType),
+      new StructType().add("a1", StringType),
+      new StructType().add("a1", StringType).add("a2", IntegerType))
+  val allTypes: Seq[DataType] =
+    atomicTypes ++ complexTypes ++ Seq(NullType, CalendarIntervalType)
+
   case class AnyTypeUnaryExpression(child: Expression)
     extends UnaryExpression with ExpectsInputTypes with Unevaluable {
     override def inputTypes: Seq[AbstractDataType] = Seq(AnyDataType)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index dd3f362e71b7c..e25d805d25a26 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion.numericPrecedence
+import org.apache.spark.sql.catalyst.analysis.TypeCoercionSuite
 import org.apache.spark.sql.catalyst.expressions.aggregate.{CollectList, CollectSet}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -413,6 +414,14 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
       assert(ret.resolved)
       checkEvaluation(ret, Seq(null, true, false, null))
     }
+
+    {
+      val array = Literal.create(Seq.empty, ArrayType(NullType, containsNull = false))
+      val ret = cast(array, ArrayType(IntegerType, containsNull = false))
+      assert(ret.resolved)
+      checkEvaluation(ret, Seq.empty)
+    }
+
     {
       val ret = cast(array, ArrayType(BooleanType, containsNull = false))
       assert(ret.resolved === false)
@@ -1158,6 +1167,19 @@ class CastSuite extends CastSuiteBase {
       StructType(StructField("a", IntegerType, true) :: Nil)))
   }
 
+  test("SPARK-31227: Non-nullable null type should not coerce to nullable type") {
+    TypeCoercionSuite.allTypes.foreach { t =>
+      assert(Cast.canCast(ArrayType(NullType, false), ArrayType(t, false)))
+
+      assert(Cast.canCast(
+        MapType(NullType, NullType, false), MapType(t, t, false)))
+
+      assert(Cast.canCast(
+        StructType(StructField("a", NullType, false) :: Nil),
+        StructType(StructField("a", t, false) :: Nil)))
+    }
+  }
+
   test("Cast should output null for invalid strings when ANSI is not enabled.") {
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
       checkEvaluation(cast("abdef", DecimalType.USER_DEFAULT), null)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 875a671a01bfb..9100ee3940a20 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -1533,6 +1533,13 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
     assert(e.getMessage.contains("string, binary or array"))
   }
 
+  test("SPARK-31227: Non-nullable null type should not coerce to nullable type in concat") {
+    val actual = spark.range(1).selectExpr("concat(array(), array(1)) as arr")
+    val expected = spark.range(1).selectExpr("array(1) as arr")
+    checkAnswer(actual, expected)
+    assert(actual.schema === expected.schema)
+  }
+
   test("flatten function") {
     // Test cases with a primitive type
     val intDF = Seq(

From b7810dd6c6a67dfda307763af8ca13ce4a6f1f99 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 26 Mar 2020 17:45:29 +0800
Subject: [PATCH 0350/1280] [SPARK-31254][SQL] Use the current session time
 zone in `HiveResult.toHiveString`

### What changes were proposed in this pull request?
In the PR, I propose to define `timestampFormatter`, `dateFormatter` and `zoneId` as methods of the `HiveResult` object. This should guarantee that the formatters pick the current session time zone in `toHiveString()`

### Why are the changes needed?
Currently, date/timestamp formatters in `HiveResult.toHiveString` are initialized once on instantiation of the `HiveResult` object, and pick up the session time zone. If the sessions time zone is changed, the formatters still use the previous one.

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
By existing test suites, in particular, by `HiveResultSuite`

Closes #28024 from MaxGekk/hive-result-datetime-formatters.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 600319dcb977827028327f9395644ae3d492e0fe)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/execution/HiveResult.scala   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index 5a2f16d8e1526..ff820bfb4e426 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -59,9 +59,9 @@ object HiveResult {
         .map(_.mkString("\t"))
   }
 
-  private lazy val zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
-  private lazy val dateFormatter = DateFormatter(zoneId)
-  private lazy val timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
+  private def zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
+  private def dateFormatter = DateFormatter(zoneId)
+  private def timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
 
   /** Formats a datum (based on the given data type) and returns the string representation. */
   def toHiveString(a: (Any, DataType), nested: Boolean = false): String = a match {

From 7b1fba2978e6e1fb7eacf693be2a5ad8b9a859b2 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 26 Mar 2020 18:52:56 +0800
Subject: [PATCH 0351/1280] [SPARK-31242][SQL][TEST] mergeSparkConf in
 WithTestConf should also respect
 spark.sql.legacy.sessionInitWithConfigDefaults

### What changes were proposed in this pull request?

Make `mergeSparkConf` in `WithTestConf` respects `spark.sql.legacy.sessionInitWithConfigDefaults`.

### Why are the changes needed?

Without the fix, conf specified by `withSQLConf` can be reverted to original value in a cloned SparkSession.  For example, you will fail test below without the fix:

```
withSQLConf(SQLConf.CODEGEN_FALLBACK.key -> "true") {
  val cloned = spark.cloneSession()
  SparkSession.setActiveSession(cloned)
  assert(SQLConf.get.getConf(SQLConf.CODEGEN_FALLBACK) === true)
}
```

So we should fix it just as  #24540 did before.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #28014 from Ngone51/sparksession_clone.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8b798c1bc501f0d2b1c2c80ab64ffd764bc72987)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/internal/BaseSessionStateBuilder.scala     | 14 ++++++++++----
 .../scala/org/apache/spark/sql/SQLQuerySuite.scala | 10 ++++++++++
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 9556d4d1ecabf..3bbdbb002cca8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -347,8 +347,14 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
 
   override protected lazy val conf: SQLConf = {
     val overrideConfigurations = overrideConfs
-    val conf = parentState.map(_.conf.clone()).getOrElse {
-      new SQLConf {
+    parentState.map { s =>
+      val cloned = s.conf.clone()
+      if (session.sparkContext.conf.get(StaticSQLConf.SQL_LEGACY_SESSION_INIT_WITH_DEFAULTS)) {
+        mergeSparkConf(conf, session.sparkContext.conf)
+      }
+      cloned
+    }.getOrElse {
+      val conf = new SQLConf {
         clear()
         override def clear(): Unit = {
           super.clear()
@@ -356,8 +362,8 @@ private[sql] trait WithTestConf { self: BaseSessionStateBuilder =>
           overrideConfigurations.foreach { case (key, value) => setConfString(key, value) }
         }
       }
+      mergeSparkConf(conf, session.sparkContext.conf)
+      conf
     }
-    mergeSparkConf(conf, session.sparkContext.conf)
-    conf
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index feb1450253870..42f0d96347eed 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3462,6 +3462,16 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       sql("(SELECT map()) UNION ALL (SELECT map(1, 2))"),
       Seq(Row(Map[Int, Int]()), Row(Map(1 -> 2))))
   }
+
+  test("SPARK-31242: clone SparkSession should respect sessionInitWithConfigDefaults") {
+    // Note, only the conf explicitly set in SparkConf(e.g. in SharedSparkSessionBase) would cause
+    // problem before the fix.
+    withSQLConf(SQLConf.CODEGEN_FALLBACK.key -> "true") {
+      val cloned = spark.cloneSession()
+      SparkSession.setActiveSession(cloned)
+      assert(SQLConf.get.getConf(SQLConf.CODEGEN_FALLBACK) === true)
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From 8d0800a0803d3c47938bddefa15328d654739bc5 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 26 Mar 2020 22:57:01 +0900
Subject: [PATCH 0352/1280] [SPARK-31201][SQL] Add an individual config for
 skewed partition threshold

Skew join handling comes with an overhead: we need to read some data repeatedly. We should treat a partition as skewed if it's large enough so that it's beneficial to do so.

Currently the size threshold is the advisory partition size, which is 64 MB by default. This is not large enough for the skewed partition size threshold.

This PR adds a new config for the threshold and set default value as 256 MB.

Avoid skew join handling that may introduce a  perf regression.

no

existing tests

Closes #27967 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 05498af72e19b058b210815e1053f3fa9b0157d9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-performance-tuning.md                        |  9 ++++++++-
 .../scala/org/apache/spark/sql/internal/SQLConf.scala | 11 ++++++++++-
 .../sql/execution/adaptive/OptimizeSkewedJoin.scala   |  2 +-
 .../execution/adaptive/AdaptiveQueryExecSuite.scala   |  2 +-
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 489575d01ed64..9a1cc891d9312 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -242,7 +242,14 @@ Data skew can severely downgrade the performance of join queries. This feature d
        <td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
        <td>10</td>
        <td>
-         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>.
+         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewedPartitionThresholdInBytes</code>.
+       </td>
+     </tr>
+     <tr>
+       <td><code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code></td>
+       <td>256MB</td>
+       <td>
+         A partition is considered as skewed if its size in bytes is larger than this threshold and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> multiplying the median partition size. Ideally this config should be set larger than <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>.
        </td>
      </tr>
    </table>
\ No newline at end of file
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 1b00bed82da0a..c61a57ebb422a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -455,11 +455,20 @@ object SQLConf {
     buildConf("spark.sql.adaptive.skewJoin.skewedPartitionFactor")
       .doc("A partition is considered as skewed if its size is larger than this factor " +
         "multiplying the median partition size and also larger than " +
-        s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'")
+        "'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
       .intConf
       .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(10)
 
+  val SKEW_JOIN_SKEWED_PARTITION_THRESHOLD =
+    buildConf("spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes")
+      .doc("A partition is considered as skewed if its size in bytes is larger than this " +
+        s"threshold and also larger than '${SKEW_JOIN_SKEWED_PARTITION_FACTOR.key}' " +
+        "multiplying the median partition size. Ideally this config should be set larger " +
+        s"than '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'.")
+      .bytesConf(ByteUnit.BYTE)
+      .createWithDefaultString("256MB")
+
   val NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN =
     buildConf("spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin")
       .internal()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index e02b9afbf2d4f..b09e5630f89c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -67,7 +67,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
    */
   private def isSkewed(size: Long, medianSize: Long): Boolean = {
     size > medianSize * conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_FACTOR) &&
-      size > conf.getConf(SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES)
+      size > conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD)
   }
 
   private def medianSize(stats: MapOutputStatistics): Long = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index fcca23d7751f0..d4c5b0d42fb11 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -615,6 +615,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
+      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "2000",
       SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "2000") {
       withTempView("skewData1", "skewData2") {
         spark
@@ -781,4 +782,3 @@ class AdaptiveQueryExecSuite
     }
   }
 }
-

From 8f93dc2f1dd8bd09d52fd3dc07a4c10e70bd237c Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 26 Mar 2020 09:11:13 -0700
Subject: [PATCH 0353/1280] [SPARK-31259][CORE] Fix log message about fetch
 request size in ShuffleBlockFetcherIterator

### What changes were proposed in this pull request?

Fix incorrect log of `cureRequestSize`.

### Why are the changes needed?

In batch mode, `curRequestSize` can be the total size of several block groups. And each group should have its own request size instead of using the total size.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

It's only affect log.

Closes #28028 from Ngone51/fix_curRequestSize.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 33f532a9f201fb9c7895d685b3dce82cf042dc61)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../storage/ShuffleBlockFetcherIterator.scala      | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index f1a7d88213f7b..404e055f914e8 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -329,9 +329,8 @@ final class ShuffleBlockFetcherIterator(
 
   private def createFetchRequest(
       blocks: Seq[FetchBlockInfo],
-      address: BlockManagerId,
-      curRequestSize: Long): FetchRequest = {
-    logDebug(s"Creating fetch request of $curRequestSize at $address "
+      address: BlockManagerId): FetchRequest = {
+    logDebug(s"Creating fetch request of ${blocks.map(_.size).sum} at $address "
       + s"with ${blocks.size} blocks")
     FetchRequest(address, blocks)
   }
@@ -339,17 +338,16 @@ final class ShuffleBlockFetcherIterator(
   private def createFetchRequests(
       curBlocks: Seq[FetchBlockInfo],
       address: BlockManagerId,
-      curRequestSize: Long,
       isLast: Boolean,
       collectedRemoteRequests: ArrayBuffer[FetchRequest]): Seq[FetchBlockInfo] = {
     val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks)
     var retBlocks = Seq.empty[FetchBlockInfo]
     if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
-      collectedRemoteRequests += createFetchRequest(mergedBlocks, address, curRequestSize)
+      collectedRemoteRequests += createFetchRequest(mergedBlocks, address)
     } else {
       mergedBlocks.grouped(maxBlocksInFlightPerAddress).foreach { blocks =>
         if (blocks.length == maxBlocksInFlightPerAddress || isLast) {
-          collectedRemoteRequests += createFetchRequest(blocks, address, curRequestSize)
+          collectedRemoteRequests += createFetchRequest(blocks, address)
         } else {
           // The last group does not exceed `maxBlocksInFlightPerAddress`. Put it back
           // to `curBlocks`.
@@ -377,14 +375,14 @@ final class ShuffleBlockFetcherIterator(
       // For batch fetch, the actual block in flight should count for merged block.
       val mayExceedsMaxBlocks = !doBatchFetch && curBlocks.size >= maxBlocksInFlightPerAddress
       if (curRequestSize >= targetRemoteRequestSize || mayExceedsMaxBlocks) {
-        curBlocks = createFetchRequests(curBlocks, address, curRequestSize, isLast = false,
+        curBlocks = createFetchRequests(curBlocks, address, isLast = false,
           collectedRemoteRequests).to[ArrayBuffer]
         curRequestSize = curBlocks.map(_.size).sum
       }
     }
     // Add in the final request
     if (curBlocks.nonEmpty) {
-      curBlocks = createFetchRequests(curBlocks, address, curRequestSize, isLast = true,
+      curBlocks = createFetchRequests(curBlocks, address, isLast = true,
         collectedRemoteRequests).to[ArrayBuffer]
       curRequestSize = curBlocks.map(_.size).sum
     }

From 78cc2ef5b663d6d605e3d4febc6fb99e20b7f165 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Thu, 26 Mar 2020 13:14:28 -0700
Subject: [PATCH 0354/1280] [SPARK-31238][SQL] Rebase dates to/from Julian
 calendar in write/read for ORC datasource

### What changes were proposed in this pull request?

This PR (SPARK-31238) aims the followings.
1. Modified ORC Vectorized Reader, in particular, OrcColumnVector v1.2 and v2.3. After the changes, it uses `DateTimeUtils. rebaseJulianToGregorianDays()` added by https://github.com/apache/spark/pull/27915 . The method performs rebasing days from the hybrid calendar (Julian + Gregorian) to Proleptic Gregorian calendar. It builds a local date in the original calendar, extracts date fields `year`, `month` and `day` from the local date, and builds another local date in the target calendar. After that, it calculates days from the epoch `1970-01-01` for the resulted local date.
2. Introduced rebasing dates while saving ORC files, in particular, I modified `OrcShimUtils. getDateWritable` v1.2 and v2.3, and returned `DaysWritable` instead of Hive's `DateWritable`. The `DaysWritable` class was added by the PR https://github.com/apache/spark/pull/27890 (and fixed by https://github.com/apache/spark/pull/27962). I moved `DaysWritable` from `sql/hive` to `sql/core` to re-use it in ORC datasource.

### Why are the changes needed?
For the backward compatibility with Spark 2.4 and earlier versions. The changes allow users to read dates/timestamps saved by previous version, and get the same result.

### Does this PR introduce any user-facing change?
Yes. Before the changes, loading the date `1200-01-01` saved by Spark 2.4.5 returns the following:
```scala
scala> spark.read.orc("/Users/maxim/tmp/before_1582/2_4_5_date_orc").show(false)
+----------+
|dt        |
+----------+
|1200-01-08|
+----------+
```
After the changes
```scala
scala> spark.read.orc("/Users/maxim/tmp/before_1582/2_4_5_date_orc").show(false)
+----------+
|dt        |
+----------+
|1200-01-01|
+----------+
```

### How was this patch tested?
- By running `OrcSourceSuite` and `HiveOrcSourceSuite`.
- Add new test `SPARK-31238: compatibility with Spark 2.4 in reading dates` to `OrcSuite` which reads an ORC file saved by Spark 2.4.5 via the commands:
```shell
$ export TZ="America/Los_Angeles"
```
```scala
scala> sql("select cast('1200-01-01' as date) dt").write.mode("overwrite").orc("/Users/maxim/tmp/before_1582/2_4_5_date_orc")
scala> spark.read.orc("/Users/maxim/tmp/before_1582/2_4_5_date_orc").show(false)
+----------+
|dt        |
+----------+
|1200-01-01|
+----------+
```
- Add round trip test `SPARK-31238: rebasing dates in write`. The test `SPARK-31238: compatibility with Spark 2.4 in reading dates` confirms rebasing in read. So, we can check rebasing in write.

Closes #28016 from MaxGekk/rebase-date-orc.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit d72ec8574113f9a7e87f3d7ec56c8447267b0506)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../execution/datasources}/DaysWritable.scala |  10 ++-
 .../before_1582_date_v2_4.snappy.orc          | Bin 0 -> 201 bytes
 .../datasources/orc/OrcSourceSuite.scala      |  28 ++++++-
 .../execution/datasources/orc/OrcTest.scala   |   5 ++
 .../datasources/orc/OrcColumnVector.java      |  15 +++-
 .../datasources/orc/DaysWritable.scala        |  79 ++++++++++++++++++
 .../datasources/orc/OrcShimUtils.scala        |   4 +-
 .../datasources/orc/OrcColumnVector.java      |  15 +++-
 .../datasources/orc/OrcShimUtils.scala        |   5 +-
 .../spark/sql/hive/HiveInspectors.scala       |   1 +
 10 files changed, 153 insertions(+), 9 deletions(-)
 rename sql/{hive/src/main/scala/org/apache/spark/sql/hive => core/src/main/scala/org/apache/spark/sql/execution/datasources}/DaysWritable.scala (92%)
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc
 create mode 100644 sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
similarity index 92%
rename from sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
rename to sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
index 1eec8d760118f..00b710f57d0f8 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/DaysWritable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.hive
+package org.apache.spark.sql.execution.datasources
 
 import java.io.{DataInput, DataOutput, IOException}
 import java.sql.Date
@@ -35,11 +35,12 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils.{rebaseGregorianToJulian
  * @param julianDays The number of days since the epoch 1970-01-01 in
  *                   Julian calendar.
  */
-private[hive] class DaysWritable(
+class DaysWritable(
     var gregorianDays: Int,
     var julianDays: Int)
   extends DateWritable {
 
+  def this() = this(0, 0)
   def this(gregorianDays: Int) =
     this(gregorianDays, rebaseGregorianToJulianDays(gregorianDays))
   def this(dateWritable: DateWritable) = {
@@ -55,6 +56,11 @@ private[hive] class DaysWritable(
   override def getDays: Int = julianDays
   override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
 
+  override def set(d: Int): Unit = {
+    gregorianDays = d
+    julianDays = rebaseGregorianToJulianDays(d)
+  }
+
   @throws[IOException]
   override def write(out: DataOutput): Unit = {
     WritableUtils.writeVInt(out, julianDays)
diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc b/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.orc
new file mode 100644
index 0000000000000000000000000000000000000000..ebe01743b2e2043d9374630c5f24586b6e5df3cb
GIT binary patch
literal 201
zcmeYdau#G@;9?VE;b074&<8R_xtJLk7=-vZ7_B%sX0K8b0O9}!ZU%-t29BxHjv&Px
zYyu1t++dnfLJCMT3NT7=2r+Rm05Ov(P>BQ=*gP%)R8t~2Y8e<fm<58RxPYXp1c!pM
z2BQ)u2agaVqZCt0i4qeBzX78aSg#d^UWW;O9t<3g0!$JO4FXC`3_KdL25e1Vn3?^9
GoW%hzBO5IM

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 1e27593584786..b5e002f34313d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.orc
 
 import java.io.File
 import java.nio.charset.StandardCharsets.UTF_8
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
 import java.util.Locale
 
 import org.apache.hadoop.conf.Configuration
@@ -482,6 +482,32 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
       }
     }
   }
+
+  test("SPARK-31238: compatibility with Spark 2.4 in reading dates") {
+    Seq(false, true).foreach { vectorized =>
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+        checkAnswer(
+          readResourceOrcFile("test-data/before_1582_date_v2_4.snappy.orc"),
+          Row(java.sql.Date.valueOf("1200-01-01")))
+      }
+    }
+  }
+
+  test("SPARK-31238: rebasing dates in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      Seq("1001-01-01").toDF("dateS")
+        .select($"dateS".cast("date").as("date"))
+        .write
+        .orc(path)
+
+      Seq(false, true).foreach { vectorized =>
+        withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+          checkAnswer(spark.read.orc(path), Row(Date.valueOf("1001-01-01")))
+        }
+      }
+    }
+  }
 }
 
 class OrcSourceSuite extends OrcSuite with SharedSparkSession {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
index 388744bd0fd6e..16772fec87dfb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
@@ -133,4 +133,9 @@ abstract class OrcTest extends QueryTest with FileBasedDataSourceTest with Befor
         throw new AnalysisException("Can not match OrcTable in the query.")
     }
   }
+
+  protected def readResourceOrcFile(name: String): DataFrame = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(name)
+    spark.read.orc(url.toString)
+  }
 }
diff --git a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 0dfed768f87de..5dc3f3743ce91 100644
--- a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -23,6 +23,7 @@
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
 import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.TimestampType;
 import org.apache.spark.sql.vectorized.ColumnarArray;
@@ -42,6 +43,7 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
   private DecimalColumnVector decimalData;
   private TimestampColumnVector timestampData;
   private final boolean isTimestamp;
+  private final boolean isDate;
 
   private int batchSize;
 
@@ -54,6 +56,12 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
       isTimestamp = false;
     }
 
+    if (type instanceof DateType) {
+      isDate = true;
+    } else {
+      isDate = false;
+    }
+
     baseData = vector;
     if (vector instanceof LongColumnVector) {
       longData = (LongColumnVector) vector;
@@ -130,7 +138,12 @@ public short getShort(int rowId) {
 
   @Override
   public int getInt(int rowId) {
-    return (int) longData.vector[getRowIndex(rowId)];
+    int value = (int) longData.vector[getRowIndex(rowId)];
+    if (isDate) {
+      return DateTimeUtils.rebaseJulianToGregorianDays(value);
+    } else {
+      return value;
+    }
   }
 
   @Override
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
new file mode 100644
index 0000000000000..4934a9650eee1
--- /dev/null
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.orc
+
+import java.io.{DataInput, DataOutput, IOException}
+import java.sql.Date
+
+import org.apache.hadoop.io.WritableUtils
+import org.apache.orc.storage.serde2.io.DateWritable
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
+
+/**
+ * The class accepts/returns days in Gregorian calendar and rebase them
+ * via conversion to local date in Julian calendar for dates before 1582-10-15
+ * in read/write for backward compatibility with Spark 2.4 and earlier versions.
+ *
+ * This is a clone of `org.apache.spark.sql.execution.datasources.DaysWritable`.
+ * The class is cloned because Hive ORC v1.2 uses different `DateWritable`:
+ *   - v1.2: `org.apache.orc.storage.serde2.io.DateWritable`
+ *   - v2.3 and `HiveInspectors`: `org.apache.hadoop.hive.serde2.io.DateWritable`
+ *
+ * @param gregorianDays The number of days since the epoch 1970-01-01 in
+ *                      Gregorian calendar.
+ * @param julianDays The number of days since the epoch 1970-01-01 in
+ *                   Julian calendar.
+ */
+class DaysWritable(
+    var gregorianDays: Int,
+    var julianDays: Int)
+  extends DateWritable {
+
+  def this() = this(0, 0)
+  def this(gregorianDays: Int) =
+    this(gregorianDays, rebaseGregorianToJulianDays(gregorianDays))
+  def this(dateWritable: DateWritable) = {
+    this(
+      gregorianDays = dateWritable match {
+        case daysWritable: DaysWritable => daysWritable.gregorianDays
+        case dateWritable: DateWritable =>
+        rebaseJulianToGregorianDays(dateWritable.getDays)
+      },
+      julianDays = dateWritable.getDays)
+  }
+
+  override def getDays: Int = julianDays
+  override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
+
+  override def set(d: Int): Unit = {
+    gregorianDays = d
+    julianDays = rebaseGregorianToJulianDays(d)
+  }
+
+  @throws[IOException]
+  override def write(out: DataOutput): Unit = {
+    WritableUtils.writeVInt(out, julianDays)
+  }
+
+  @throws[IOException]
+  override def readFields(in: DataInput): Unit = {
+    julianDays = WritableUtils.readVInt(in)
+    gregorianDays = rebaseJulianToGregorianDays(julianDays)
+  }
+}
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
index 68503aba22b40..ece528039bba8 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
@@ -47,13 +47,13 @@ private[sql] object OrcShimUtils {
 
   def getDateWritable(reuseObj: Boolean): (SpecializedGetters, Int) => DateWritable = {
     if (reuseObj) {
-      val result = new DateWritable()
+      val result = new DaysWritable()
       (getter, ordinal) =>
         result.set(getter.getInt(ordinal))
         result
     } else {
       (getter: SpecializedGetters, ordinal: Int) =>
-        new DateWritable(getter.getInt(ordinal))
+        new DaysWritable(getter.getInt(ordinal))
     }
   }
 
diff --git a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 35447fe7e19c2..7be4a6f6c72c8 100644
--- a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -23,6 +23,7 @@
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
 import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.TimestampType;
 import org.apache.spark.sql.vectorized.ColumnarArray;
@@ -42,6 +43,7 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
   private DecimalColumnVector decimalData;
   private TimestampColumnVector timestampData;
   private final boolean isTimestamp;
+  private final boolean isDate;
 
   private int batchSize;
 
@@ -54,6 +56,12 @@ public class OrcColumnVector extends org.apache.spark.sql.vectorized.ColumnVecto
       isTimestamp = false;
     }
 
+    if (type instanceof DateType) {
+      isDate = true;
+    } else {
+      isDate = false;
+    }
+
     baseData = vector;
     if (vector instanceof LongColumnVector) {
       longData = (LongColumnVector) vector;
@@ -130,7 +138,12 @@ public short getShort(int rowId) {
 
   @Override
   public int getInt(int rowId) {
-    return (int) longData.vector[getRowIndex(rowId)];
+    int value = (int) longData.vector[getRowIndex(rowId)];
+    if (isDate) {
+      return DateTimeUtils.rebaseJulianToGregorianDays(value);
+    } else {
+      return value;
+    }
   }
 
   @Override
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
index c32f024476e69..5666d319e1e90 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf.{Operator => OrcOperator}
 import org.apache.hadoop.hive.serde2.io.{DateWritable, HiveDecimalWritable}
 
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
+import org.apache.spark.sql.execution.datasources.DaysWritable
 import org.apache.spark.sql.types.Decimal
 
 /**
@@ -47,13 +48,13 @@ private[sql] object OrcShimUtils {
 
   def getDateWritable(reuseObj: Boolean): (SpecializedGetters, Int) => DateWritable = {
     if (reuseObj) {
-      val result = new DateWritable()
+      val result = new DaysWritable()
       (getter, ordinal) =>
         result.set(getter.getInt(ordinal))
         result
     } else {
       (getter: SpecializedGetters, ordinal: Int) =>
-        new DateWritable(getter.getInt(ordinal))
+        new DaysWritable(getter.getInt(ordinal))
     }
   }
 
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
index e3e9a3144728d..16e9014340244 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.execution.datasources.DaysWritable
 import org.apache.spark.sql.types
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String

From 6f30ff44cf2d3d347a516a0e0370d07e8de9352c Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Fri, 27 Mar 2020 08:09:17 +0900
Subject: [PATCH 0355/1280] [SPARK-31262][SQL][TESTS] Fix bug tests imported
 bracketed comments

### What changes were proposed in this pull request?
This PR related to https://github.com/apache/spark/pull/27481.
If test case A uses `--IMPORT` to import test case B contains bracketed comments, the output can't display bracketed comments in golden files well.
The content of `nested-comments.sql` show below:
```
-- This test case just used to test imported bracketed comments.

-- the first case of bracketed comment
--QUERY-DELIMITER-START
/* This is the first example of bracketed comment.
SELECT 'ommented out content' AS first;
*/
SELECT 'selected content' AS first;
--QUERY-DELIMITER-END
```
The test case `comments.sql` imports `nested-comments.sql` below:
`--IMPORT nested-comments.sql`
Before this PR, the output will be:
```
-- !query
/* This is the first example of bracketed comment.
SELECT 'ommented out content' AS first
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

mismatched input '/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP',
'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', '
ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
/* This is the first example of bracketed comment.
^^^
SELECT 'ommented out content' AS first

-- !query
*/
SELECT 'selected content' AS first
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

extraneous input '*/' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
*/
^^^
SELECT 'selected content' AS first
```
After this PR, the output will be:
```
-- !query
/* This is the first example of bracketed comment.
SELECT 'ommented out content' AS first;
*/
SELECT 'selected content' AS first
-- !query schema
struct<first:string>
-- !query output
selected content
```

### Why are the changes needed?
Golden files can't display the bracketed comments in imported test cases.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
New UT.

Closes #28018 from beliefer/fix-bug-tests-imported-bracketed-comments.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 9e0fee933e62eb309d4aa32bb1e5126125d0bf9f)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../scala/org/apache/spark/sql/SQLQueryTestSuite.scala   | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 6c661667825b2..848966a97a70e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -256,20 +256,23 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     def splitWithSemicolon(seq: Seq[String]) = {
       seq.mkString("\n").split("(?<=[^\\\\]);")
     }
-    val input = fileToString(new File(testCase.inputFile))
 
-    val (comments, code) = input.split("\n").partition { line =>
+    def splitCommentsAndCodes(input: String) = input.split("\n").partition { line =>
       val newLine = line.trim
       newLine.startsWith("--") && !newLine.startsWith("--QUERY-DELIMITER")
     }
 
+    val input = fileToString(new File(testCase.inputFile))
+
+    val (comments, code) = splitCommentsAndCodes(input)
+
     // If `--IMPORT` found, load code from another test case file, then insert them
     // into the head in this test.
     val importedTestCaseName = comments.filter(_.startsWith("--IMPORT ")).map(_.substring(9))
     val importedCode = importedTestCaseName.flatMap { testCaseName =>
       listTestCases.find(_.name == testCaseName).map { testCase =>
         val input = fileToString(new File(testCase.inputFile))
-        val (_, code) = input.split("\n").partition(_.trim.startsWith("--"))
+        val (_, code) = splitCommentsAndCodes(input)
         code
       }
     }.flatten

From 6fea291762af3e802cb4c237bdad51ebf5d7152c Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Fri, 27 Mar 2020 12:10:30 +0900
Subject: [PATCH 0356/1280] [SPARK-31186][PYSPARK][SQL] toPandas should not
 fail on duplicate column names

### What changes were proposed in this pull request?

When `toPandas` API works on duplicate column names produced from operators like join, we see the error like:

```
ValueError: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().
```

This patch fixes the error in `toPandas` API.

### Why are the changes needed?

To make `toPandas` work on dataframe with duplicate column names.

### Does this PR introduce any user-facing change?

Yes. Previously calling `toPandas` API on a dataframe with duplicate column names will fail. After this patch, it will produce correct result.

### How was this patch tested?

Unit test.

Closes #28025 from viirya/SPARK-31186.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 559d3e4051500d5c49e9a7f3ac33aac3de19c9c6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/pandas/conversion.py    | 48 +++++++++++++++++-----
 python/pyspark/sql/tests/test_dataframe.py | 18 ++++++++
 2 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/python/pyspark/sql/pandas/conversion.py b/python/pyspark/sql/pandas/conversion.py
index 8548cd222bf10..47cf8bbc5b688 100644
--- a/python/pyspark/sql/pandas/conversion.py
+++ b/python/pyspark/sql/pandas/conversion.py
@@ -21,6 +21,7 @@
     xrange = range
 else:
     from itertools import izip as zip
+from collections import Counter
 
 from pyspark import since
 from pyspark.rdd import _load_from_socket
@@ -131,9 +132,16 @@ def toPandas(self):
 
         # Below is toPandas without Arrow optimization.
         pdf = pd.DataFrame.from_records(self.collect(), columns=self.columns)
+        column_counter = Counter(self.columns)
+
+        dtype = [None] * len(self.schema)
+        for fieldIdx, field in enumerate(self.schema):
+            # For duplicate column name, we use `iloc` to access it.
+            if column_counter[field.name] > 1:
+                pandas_col = pdf.iloc[:, fieldIdx]
+            else:
+                pandas_col = pdf[field.name]
 
-        dtype = {}
-        for field in self.schema:
             pandas_type = PandasConversionMixin._to_corrected_pandas_type(field.dataType)
             # SPARK-21766: if an integer field is nullable and has null values, it can be
             # inferred by pandas as float column. Once we convert the column with NaN back
@@ -141,16 +149,36 @@ def toPandas(self):
             # float type, not the corrected type from the schema in this case.
             if pandas_type is not None and \
                 not(isinstance(field.dataType, IntegralType) and field.nullable and
-                    pdf[field.name].isnull().any()):
-                dtype[field.name] = pandas_type
+                    pandas_col.isnull().any()):
+                dtype[fieldIdx] = pandas_type
             # Ensure we fall back to nullable numpy types, even when whole column is null:
-            if isinstance(field.dataType, IntegralType) and pdf[field.name].isnull().any():
-                dtype[field.name] = np.float64
-            if isinstance(field.dataType, BooleanType) and pdf[field.name].isnull().any():
-                dtype[field.name] = np.object
+            if isinstance(field.dataType, IntegralType) and pandas_col.isnull().any():
+                dtype[fieldIdx] = np.float64
+            if isinstance(field.dataType, BooleanType) and pandas_col.isnull().any():
+                dtype[fieldIdx] = np.object
+
+        df = pd.DataFrame()
+        for index, t in enumerate(dtype):
+            column_name = self.schema[index].name
+
+            # For duplicate column name, we use `iloc` to access it.
+            if column_counter[column_name] > 1:
+                series = pdf.iloc[:, index]
+            else:
+                series = pdf[column_name]
+
+            if t is not None:
+                series = series.astype(t, copy=False)
+
+            # `insert` API makes copy of data, we only do it for Series of duplicate column names.
+            # `pdf.iloc[:, index] = pdf.iloc[:, index]...` doesn't always work because `iloc` could
+            # return a view or a copy depending by context.
+            if column_counter[column_name] > 1:
+                df.insert(index, column_name, series, allow_duplicates=True)
+            else:
+                df[column_name] = series
 
-        for f, t in dtype.items():
-            pdf[f] = pdf[f].astype(t, copy=False)
+        pdf = df
 
         if timezone is None:
             return pdf
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index d738449799bda..d9dcbc0f9595f 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -529,6 +529,24 @@ def test_to_pandas(self):
         self.assertEquals(types[4], np.object)  # datetime.date
         self.assertEquals(types[5], 'datetime64[ns]')
 
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    def test_to_pandas_on_cross_join(self):
+        import numpy as np
+
+        sql = """
+        select t1.*, t2.* from (
+          select explode(sequence(1, 3)) v
+        ) t1 left join (
+          select explode(sequence(1, 3)) v
+        ) t2
+        """
+        with self.sql_conf({"spark.sql.crossJoin.enabled": True}):
+            df = self.spark.sql(sql)
+            pdf = df.toPandas()
+            types = pdf.dtypes
+            self.assertEquals(types.iloc[0], np.int32)
+            self.assertEquals(types.iloc[1], np.int32)
+
     @unittest.skipIf(have_pandas, "Required Pandas was found.")
     def test_to_pandas_required_pandas_not_found(self):
         with QuietTest(self.sc):

From 5f5ee4d84acc933112c52b1818a865139c2af05a Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 27 Mar 2020 12:05:45 +0800
Subject: [PATCH 0357/1280] [SPARK-31170][SQL] Spark SQL Cli should respect
 hive-site.xml and spark.sql.warehouse.dir

### What changes were proposed in this pull request?

In Spark CLI, we create a hive `CliSessionState` and it does not load the `hive-site.xml`. So the configurations in `hive-site.xml` will not take effects like other spark-hive integration apps.

Also, the warehouse directory is not correctly picked. If the `default` database does not exist, the `CliSessionState` will create one during the first time it talks to the metastore. The `Location` of the default DB will be neither the value of `spark.sql.warehousr.dir` nor the user-specified value of `hive.metastore.warehourse.dir`, but the default value of `hive.metastore.warehourse.dir `which will always be `/user/hive/warehouse`.

This PR fixes CLiSuite failure with the hive-1.2 profile in https://github.com/apache/spark/pull/27933.

In https://github.com/apache/spark/pull/27933, we fix the issue in JIRA by deciding the warehouse dir using all properties from spark conf and Hadoop conf, but properties from `--hiveconf` is not included,  they will be applied to the `CliSessionState` instance after it initialized. When this command-line option key is `hive.metastore.warehouse.dir`, the actual warehouse dir is overridden. Because of the logic in Hive for creating the non-existing default database changed, that test passed with `Hive 2.3.6` but failed with `1.2`. So in this PR, Hadoop/Hive configurations are ordered by:
` spark.hive.xxx > spark.hadoop.xxx > --hiveconf xxx > hive-site.xml` througth `ShareState.loadHiveConfFile` before sessionState start

### Why are the changes needed?

Bugfix for Spark SQL CLI to pick right confs

### Does this PR introduce any user-facing change?

yes,
1. the non-exists default database will be created in the location specified by the users via `spark.sql.warehouse.dir` or `hive.metastore.warehouse.dir`, or the default value of `spark.sql.warehouse.dir` if none of them specified.

2. configurations from `hive-site.xml` will not override command-line options or the properties defined with `spark.hadoo(hive).` prefix in spark conf.

### How was this patch tested?

add cli ut

Closes #27969 from yaooqinn/SPARK-31170-2.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8be16907c261657f83f5d5934bcd978d8dacf7ff)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/internal/SharedState.scala      | 87 +++++++++--------
 .../hive/thriftserver/SparkSQLCLIDriver.scala |  9 +-
 .../src/test/noclasspath/hive-site.xml        | 30 ++++++
 .../sql/hive/thriftserver/CliSuite.scala      | 96 ++++++++++++++-----
 .../spark/sql/hive/HiveSharedStateSuite.scala |  1 -
 .../spark/sql/hive/HiveSparkSubmitSuite.scala |  2 +-
 6 files changed, 159 insertions(+), 66 deletions(-)
 create mode 100644 sql/hive-thriftserver/src/test/noclasspath/hive-site.xml

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 5347264d7c50a..14b8ea66c8071 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -22,6 +22,7 @@ import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 import javax.annotation.concurrent.GuardedBy
 
+import scala.collection.JavaConverters._
 import scala.reflect.ClassTag
 import scala.util.control.NonFatal
 
@@ -41,7 +42,6 @@ import org.apache.spark.sql.streaming.ui.{StreamingQueryStatusListener, Streamin
 import org.apache.spark.status.ElementTrackingStore
 import org.apache.spark.util.Utils
 
-
 /**
  * A class that holds all state shared across sessions in a given [[SQLContext]].
  *
@@ -55,45 +55,10 @@ private[sql] class SharedState(
 
   SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf)
 
-  // Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
-  // the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
-  val warehousePath: String = {
-    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
-    if (configFile != null) {
-      logInfo(s"loading hive config file: $configFile")
-      sparkContext.hadoopConfiguration.addResource(configFile)
-    }
-
-    // hive.metastore.warehouse.dir only stay in hadoopConf
-    sparkContext.conf.remove("hive.metastore.warehouse.dir")
-    // Set the Hive metastore warehouse path to the one we use
-    val hiveWarehouseDir = sparkContext.hadoopConfiguration.get("hive.metastore.warehouse.dir")
-    if (hiveWarehouseDir != null && !sparkContext.conf.contains(WAREHOUSE_PATH.key)) {
-      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
-      // we will respect the value of hive.metastore.warehouse.dir.
-      sparkContext.conf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
-      logInfo(s"${WAREHOUSE_PATH.key} is not set, but hive.metastore.warehouse.dir " +
-        s"is set. Setting ${WAREHOUSE_PATH.key} to the value of " +
-        s"hive.metastore.warehouse.dir ('$hiveWarehouseDir').")
-      hiveWarehouseDir
-    } else {
-      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
-      // the value of spark.sql.warehouse.dir.
-      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set,
-      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
-      val sparkWarehouseDir = sparkContext.conf.get(WAREHOUSE_PATH)
-      logInfo(s"Setting hive.metastore.warehouse.dir ('$hiveWarehouseDir') to the value of " +
-        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
-      sparkContext.hadoopConfiguration.set("hive.metastore.warehouse.dir", sparkWarehouseDir)
-      sparkWarehouseDir
-    }
-  }
-  logInfo(s"Warehouse path is '$warehousePath'.")
-
-  // These 2 variables should be initiated after `warehousePath`, because in the first place we need
-  // to load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
-  // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
   private val (conf, hadoopConf) = {
+    // Load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
+    // both spark conf and hadoop conf avoiding be affected by any SparkSession level options
+    SharedState.loadHiveConfFile(sparkContext.conf, sparkContext.hadoopConfiguration)
     val confClone = sparkContext.conf.clone()
     val hadoopConfClone = new Configuration(sparkContext.hadoopConfiguration)
     // If `SparkSession` is instantiated using an existing `SparkContext` instance and no existing
@@ -166,7 +131,7 @@ private[sql] class SharedState(
     val defaultDbDefinition = CatalogDatabase(
       SessionCatalog.DEFAULT_DATABASE,
       "default database",
-      CatalogUtils.stringToURI(warehousePath),
+      CatalogUtils.stringToURI(conf.get(WAREHOUSE_PATH)),
       Map())
     // Create default database if it doesn't exist
     if (!externalCatalog.databaseExists(SessionCatalog.DEFAULT_DATABASE)) {
@@ -258,4 +223,46 @@ object SharedState extends Logging {
         throw new IllegalArgumentException(s"Error while instantiating '$className':", e)
     }
   }
+
+  /**
+   * Load hive-site.xml into hadoopConf and determine the warehouse path we want to use, based on
+   * the config from both hive and Spark SQL. Finally set the warehouse config value to sparkConf.
+   */
+  def loadHiveConfFile(
+      sparkConf: SparkConf,
+      hadoopConf: Configuration): Unit = {
+    val hiveWarehouseKey = "hive.metastore.warehouse.dir"
+    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+    if (configFile != null) {
+      logInfo(s"loading hive config file: $configFile")
+      val hadoopConfTemp = new Configuration()
+      hadoopConfTemp.addResource(configFile)
+      hadoopConfTemp.asScala.foreach { entry =>
+        hadoopConf.setIfUnset(entry.getKey, entry.getValue)
+      }
+    }
+    // hive.metastore.warehouse.dir only stay in hadoopConf
+    sparkConf.remove(hiveWarehouseKey)
+    // Set the Hive metastore warehouse path to the one we use
+    val hiveWarehouseDir = hadoopConf.get(hiveWarehouseKey)
+    val warehousePath = if (hiveWarehouseDir != null && !sparkConf.contains(WAREHOUSE_PATH.key)) {
+      // If hive.metastore.warehouse.dir is set and spark.sql.warehouse.dir is not set,
+      // we will respect the value of hive.metastore.warehouse.dir.
+      sparkConf.set(WAREHOUSE_PATH.key, hiveWarehouseDir)
+      logInfo(s"${WAREHOUSE_PATH.key} is not set, but $hiveWarehouseKey is set. Setting" +
+        s" ${WAREHOUSE_PATH.key} to the value of $hiveWarehouseKey ('$hiveWarehouseDir').")
+      hiveWarehouseDir
+    } else {
+      // If spark.sql.warehouse.dir is set, we will override hive.metastore.warehouse.dir using
+      // the value of spark.sql.warehouse.dir.
+      // When neither spark.sql.warehouse.dir nor hive.metastore.warehouse.dir is set
+      // we will set hive.metastore.warehouse.dir to the default value of spark.sql.warehouse.dir.
+      val sparkWarehouseDir = sparkConf.get(WAREHOUSE_PATH)
+      logInfo(s"Setting $hiveWarehouseKey ('$hiveWarehouseDir') to the value of " +
+        s"${WAREHOUSE_PATH.key} ('$sparkWarehouseDir').")
+      hadoopConf.set(hiveWarehouseKey, sparkWarehouseDir)
+      sparkWarehouseDir
+    }
+    logInfo(s"Warehouse path is '$warehousePath'.")
+  }
 }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 6b76927bd4150..5ed0cb0407484 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -46,6 +46,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.hive.security.HiveDelegationTokenProvider
+import org.apache.spark.sql.internal.SharedState
 import org.apache.spark.util.ShutdownHookManager
 
 /**
@@ -130,6 +131,7 @@ private[hive] object SparkSQLCLIDriver extends Logging {
       UserGroupInformation.getCurrentUser.addCredentials(credentials)
     }
 
+    SharedState.loadHiveConfFile(sparkConf, conf)
     SessionState.start(sessionState)
 
     // Clean up after we exit
@@ -188,8 +190,11 @@ private[hive] object SparkSQLCLIDriver extends Logging {
     // Execute -i init files (always in silent mode)
     cli.processInitFiles(sessionState)
 
-    newHiveConf.foreach { kv =>
-      SparkSQLEnv.sqlContext.setConf(kv._1, kv._2)
+    // We don't propagate hive.metastore.warehouse.dir, because it might has been adjusted in
+    // [[SharedState.loadHiveConfFile]] based on the user specified or default values of
+    // spark.sql.warehouse.dir and hive.metastore.warehouse.dir.
+    for ((k, v) <- newHiveConf if k != "hive.metastore.warehouse.dir") {
+      SparkSQLEnv.sqlContext.setConf(k, v)
     }
 
     if (sessionState.execString != null) {
diff --git a/sql/hive-thriftserver/src/test/noclasspath/hive-site.xml b/sql/hive-thriftserver/src/test/noclasspath/hive-site.xml
new file mode 100644
index 0000000000000..d0bf04d36b00d
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/noclasspath/hive-site.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+  <property>
+    <name>hive.in.test</name>
+      <value>true</value>
+      <description>Internal marker for test.</description>
+  </property>
+  <property>
+    <name>hive.metastore.warehouse.dir</name>
+    <value>/tmp/hive_one</value>
+  </property>
+</configuration>
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 43aafc3c8590c..c393054051fc4 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -27,22 +27,23 @@ import scala.concurrent.Promise
 import scala.concurrent.duration._
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.hive.test.HiveTestJars
+import org.apache.spark.sql.internal.StaticSQLConf
 import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer
 import org.apache.spark.util.{ThreadUtils, Utils}
 
 /**
- * A test suite for the `spark-sql` CLI tool.  Note that all test cases share the same temporary
- * Hive metastore and warehouse.
+ * A test suite for the `spark-sql` CLI tool.
  */
-class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
+class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterEach with Logging {
   val warehousePath = Utils.createTempDir()
   val metastorePath = Utils.createTempDir()
   val scratchDirPath = Utils.createTempDir()
+  val sparkWareHouseDir = Utils.createTempDir()
 
   override def beforeAll(): Unit = {
     super.beforeAll()
@@ -53,14 +54,20 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
 
   override def afterAll(): Unit = {
     try {
-      warehousePath.delete()
-      metastorePath.delete()
-      scratchDirPath.delete()
+      Utils.deleteRecursively(warehousePath)
+      Utils.deleteRecursively(metastorePath)
+      Utils.deleteRecursively(scratchDirPath)
     } finally {
       super.afterAll()
     }
   }
 
+  override def afterEach(): Unit = {
+    // Only running `runCliWithin` in a single test case will share the same temporary
+    // Hive metastore
+    Utils.deleteRecursively(metastorePath)
+  }
+
   /**
    * Run a CLI operation and expect all the queries and expected answers to be returned.
    *
@@ -75,25 +82,35 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
   def runCliWithin(
       timeout: FiniteDuration,
       extraArgs: Seq[String] = Seq.empty,
-      errorResponses: Seq[String] = Seq("Error:"))(
+      errorResponses: Seq[String] = Seq("Error:"),
+      maybeWarehouse: Option[File] = Some(warehousePath),
+      useExternalHiveFile: Boolean = false)(
       queriesAndExpectedAnswers: (String, String)*): Unit = {
 
     val (queries, expectedAnswers) = queriesAndExpectedAnswers.unzip
     // Explicitly adds ENTER for each statement to make sure they are actually entered into the CLI.
     val queriesString = queries.map(_ + "\n").mkString
 
+    val extraHive = if (useExternalHiveFile) {
+      s"--driver-class-path ${System.getProperty("user.dir")}/src/test/noclasspath"
+    } else {
+      ""
+    }
+    val warehouseConf =
+      maybeWarehouse.map(dir => s"--hiveconf ${ConfVars.METASTOREWAREHOUSE}=$dir").getOrElse("")
     val command = {
       val cliScript = "../../bin/spark-sql".split("/").mkString(File.separator)
       val jdbcUrl = s"jdbc:derby:;databaseName=$metastorePath;create=true"
       s"""$cliScript
          |  --master local
          |  --driver-java-options -Dderby.system.durability=test
+         |  $extraHive
          |  --conf spark.ui.enabled=false
          |  --hiveconf ${ConfVars.METASTORECONNECTURLKEY}=$jdbcUrl
-         |  --hiveconf ${ConfVars.METASTOREWAREHOUSE}=$warehousePath
          |  --hiveconf ${ConfVars.SCRATCHDIR}=$scratchDirPath
          |  --hiveconf conf1=conftest
          |  --hiveconf conf2=1
+         |  $warehouseConf
        """.stripMargin.split("\\s+").toSeq ++ extraArgs
     }
 
@@ -159,6 +176,54 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     }
   }
 
+  test("load warehouse dir from hive-site.xml") {
+    runCliWithin(1.minute, maybeWarehouse = None, useExternalHiveFile = true)(
+      "desc database default;" -> "hive_one",
+      "set spark.sql.warehouse.dir;" -> "hive_one")
+  }
+
+  test("load warehouse dir from --hiveconf") {
+    // --hiveconf will overrides hive-site.xml
+    runCliWithin(2.minute, useExternalHiveFile = true)(
+      "desc database default;" -> warehousePath.getAbsolutePath,
+      "create database cliTestDb;" -> "",
+      "desc database cliTestDb;" -> warehousePath.getAbsolutePath,
+      "set spark.sql.warehouse.dir;" -> warehousePath.getAbsolutePath)
+  }
+
+  test("load warehouse dir from --conf spark(.hadoop).hive.*") {
+    // override conf from hive-site.xml
+    runCliWithin(
+      2.minute,
+      extraArgs = Seq("--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"),
+      maybeWarehouse = None,
+      useExternalHiveFile = true)(
+      "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
+      "create database cliTestDb;" -> "",
+      "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
+      "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+
+    // override conf from --hiveconf too
+    runCliWithin(
+      2.minute,
+      extraArgs = Seq("--conf", s"spark.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"))(
+      "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
+      "create database cliTestDb;" -> "",
+      "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
+      "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+  }
+
+  test("load warehouse dir from spark.sql.warehouse.dir") {
+    // spark.sql.warehouse.dir overrides all hive ones
+    runCliWithin(
+      2.minute,
+      extraArgs =
+        Seq("--conf",
+          s"${StaticSQLConf.WAREHOUSE_PATH.key}=${sparkWareHouseDir}1",
+          "--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=${sparkWareHouseDir}2"))(
+      "desc database default;" -> sparkWareHouseDir.getAbsolutePath.concat("1"))
+  }
+
   test("Simple commands") {
     val dataFilePath =
       Thread.currentThread().getContextClassLoader.getResource("data/files/small_kv.txt")
@@ -308,19 +373,6 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     )
   }
 
-  test("SPARK-21451: spark.sql.warehouse.dir should respect options in --hiveconf") {
-    runCliWithin(1.minute)("set spark.sql.warehouse.dir;" -> warehousePath.getAbsolutePath)
-  }
-
-  test("SPARK-21451: Apply spark.hadoop.* configurations") {
-    val tmpDir = Utils.createTempDir(namePrefix = "SPARK-21451")
-    runCliWithin(
-      1.minute,
-      Seq("--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=$tmpDir"))(
-      "set spark.sql.warehouse.dir;" -> tmpDir.getAbsolutePath)
-    tmpDir.delete()
-  }
-
   test("Support hive.aux.jars.path") {
     val hiveContribJar = HiveTestJars.getHiveContribJar().getCanonicalPath
     runCliWithin(
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
index 6e2dcfc04d498..78535b094b83d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSharedStateSuite.scala
@@ -45,7 +45,6 @@ class HiveSharedStateSuite extends SparkFunSuite {
       GLOBAL_TEMP_DATABASE.key -> tmpDb)
 
     val state = new SharedState(sc, initialConfigs)
-    assert(state.warehousePath !== invalidPath, "warehouse path can't determine by session options")
     assert(sc.conf.get(WAREHOUSE_PATH.key) !== invalidPath,
       "warehouse conf in session options can't affect application wide spark conf")
     assert(sc.hadoopConfiguration.get(ConfVars.METASTOREWAREHOUSE.varname) !== invalidPath,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 31ff62ed0a530..8b97489e2d818 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -787,7 +787,7 @@ object SPARK_18360 {
       .enableHiveSupport().getOrCreate()
 
     val defaultDbLocation = spark.catalog.getDatabase("default").locationUri
-    assert(new Path(defaultDbLocation) == new Path(spark.sharedState.warehousePath))
+    assert(new Path(defaultDbLocation) == new Path(spark.conf.get(WAREHOUSE_PATH)))
 
     val hiveClient =
       spark.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog].client

From dde7e457e8aed561dfdc5309952bbfc99ddfc1a6 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Fri, 27 Mar 2020 12:48:14 +0800
Subject: [PATCH 0358/1280] [SPARK-31204][SQL] HiveResult compatibility for
 DatasourceV2 command

### What changes were proposed in this pull request?

`HiveResult` performs some conversions for commands to be compatible with Hive output, e.g.:
```
// If it is a describe command for a Hive table, we want to have the output format be similar with Hive.
case ExecutedCommandExec(_: DescribeCommandBase) =>
...
// SHOW TABLES in Hive only output table names, while ours output database, table name, isTemp.
case command  ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
```
This conversion is needed for DatasourceV2 commands as well and this PR proposes to add the conversion for v2 commands `SHOW TABLES` and `DESCRIBE TABLE`.

### Why are the changes needed?

This is a bug where conversion is not applied to v2 commands.

### Does this PR introduce any user-facing change?

Yes, now the outputs for v2 commands `SHOW TABLES` and `DESCRIBE TABLE` are compatible with HIVE output.

For example, with a table created as:
```
CREATE TABLE testcat.ns.tbl (id bigint COMMENT 'col1') USING foo
```

The output of `SHOW TABLES` has changed from
```
ns    table
```
to
```
table
```

And the output of `DESCRIBE TABLE` has changed from
```
id    bigint    col1

# Partitioning
Not partitioned
```
to
```
id                      bigint                  col1

# Partitioning
Not partitioned
```

### How was this patch tested?

Added unit tests.

Closes #28004 from imback82/hive_result.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a97d3b9f4f4ddd215ecaa7f96c64aeba6e825f74)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/HiveResult.scala      | 29 +++++++++++------
 .../spark/sql/execution/HiveResultSuite.scala | 32 +++++++++++++++++++
 2 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index ff820bfb4e426..21874bdaf20f6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -24,6 +24,7 @@ import java.time.{Instant, LocalDate}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
 import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand}
+import org.apache.spark.sql.execution.datasources.v2.{DescribeTableExec, ShowTablesExec}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -38,18 +39,17 @@ object HiveResult {
    */
   def hiveResultString(executedPlan: SparkPlan): Seq[String] = executedPlan match {
     case ExecutedCommandExec(_: DescribeCommandBase) =>
-      // If it is a describe command for a Hive table, we want to have the output format
-      // be similar with Hive.
-      executedPlan.executeCollectPublic().map {
-        case Row(name: String, dataType: String, comment) =>
-          Seq(name, dataType,
-            Option(comment.asInstanceOf[String]).getOrElse(""))
-            .map(s => String.format(s"%-20s", s))
-            .mkString("\t")
-      }
-    // SHOW TABLES in Hive only output table names, while ours output database, table name, isTemp.
+      formatDescribeTableOutput(executedPlan.executeCollectPublic())
+    case _: DescribeTableExec =>
+      formatDescribeTableOutput(executedPlan.executeCollectPublic())
+    // SHOW TABLES in Hive only output table names while our v1 command outputs
+    // database, table name, isTemp.
     case command @ ExecutedCommandExec(s: ShowTablesCommand) if !s.isExtended =>
       command.executeCollect().map(_.getString(1))
+    // SHOW TABLES in Hive only output table names while our v2 command outputs
+    // namespace and table name.
+    case command : ShowTablesExec =>
+      command.executeCollect().map(_.getString(1))
     case other =>
       val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
       // We need the types so we can output struct field names
@@ -59,6 +59,15 @@ object HiveResult {
         .map(_.mkString("\t"))
   }
 
+  private def formatDescribeTableOutput(rows: Array[Row]): Seq[String] = {
+    rows.map {
+      case Row(name: String, dataType: String, comment) =>
+        Seq(name, dataType, Option(comment.asInstanceOf[String]).getOrElse(""))
+          .map(s => String.format(s"%-20s", s))
+          .mkString("\t")
+    }
+  }
+
   private def zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
   private def dateFormatter = DateFormatter(zoneId)
   private def timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index bf7cbaad7384b..5e81c74420fd0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.execution
 
+import org.apache.spark.sql.connector.InMemoryTableCatalog
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
 
 class HiveResultSuite extends SharedSparkSession {
@@ -68,4 +69,35 @@ class HiveResultSuite extends SharedSparkSession {
     val result = HiveResult.hiveResultString(executedPlan)
     assert(result.head === "0.00000000")
   }
+
+  test("SHOW TABLES in hive result") {
+    withSQLConf("spark.sql.catalog.testcat" -> classOf[InMemoryTableCatalog].getName) {
+      Seq(("testcat.ns", "tbl", "foo"), ("spark_catalog.default", "tbl", "csv")).foreach {
+        case (ns, tbl, source) =>
+          withTable(s"$ns.$tbl") {
+            spark.sql(s"CREATE TABLE $ns.$tbl (id bigint) USING $source")
+            val df = spark.sql(s"SHOW TABLES FROM $ns")
+            val executedPlan = df.queryExecution.executedPlan
+            assert(HiveResult.hiveResultString(executedPlan).head == tbl)
+          }
+      }
+    }
+  }
+
+  test("DESCRIBE TABLE in hive result") {
+    withSQLConf("spark.sql.catalog.testcat" -> classOf[InMemoryTableCatalog].getName) {
+      Seq(("testcat.ns", "tbl", "foo"), ("spark_catalog.default", "tbl", "csv")).foreach {
+        case (ns, tbl, source) =>
+          withTable(s"$ns.$tbl") {
+            spark.sql(s"CREATE TABLE $ns.$tbl (id bigint COMMENT 'col1') USING $source")
+            val df = spark.sql(s"DESCRIBE $ns.$tbl")
+            val executedPlan = df.queryExecution.executedPlan
+            val expected = "id                  " +
+              "\tbigint              " +
+              "\tcol1                "
+            assert(HiveResult.hiveResultString(executedPlan).head == expected)
+          }
+      }
+    }
+  }
 }

From a7c58b1ae5a05f509a034ca410a50c41ce94cf5f Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 27 Mar 2020 13:35:28 +0800
Subject: [PATCH 0359/1280] [SPARK-31275][WEBUI] Improve the metrics format in
 ExecutionPage for StageId

### What changes were proposed in this pull request?

In ExecutionPage, metrics format for stageId, attemptId and taskId are displayed like `(stageId (attemptId): taskId)` for now.
I changed this format like `(stageId.attemptId taskId)`.

### Why are the changes needed?

As cloud-fan suggested  [here](https://github.com/apache/spark/pull/27927#discussion_r398591519), `stageId.attemptId` is more standard in Spark.

### Does this PR introduce any user-facing change?

Yes. Before applying this change, we can see the UI like as follows.
![with-checked](https://user-images.githubusercontent.com/4736016/77682421-42a6c200-6fda-11ea-92e4-e9f4554adb71.png)

And after this change applied, we can like as follows.
![fix-merics-format-with-checked](https://user-images.githubusercontent.com/4736016/77682493-61a55400-6fda-11ea-801f-91a67da698fd.png)

### How was this patch tested?

Modified `SQLMetricsSuite` and manual test.

Closes #28039 from sarutak/improve-metrics-format.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bc37fdc77130ce4f60806db0bb2b1b8914452040)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/ui/static/spark-sql-viz.js  |  2 +-
 .../sql/execution/metric/SQLMetrics.scala     | 12 +++++------
 .../sql/execution/ui/ExecutionPage.scala      |  2 +-
 .../execution/metric/SQLMetricsSuite.scala    | 20 +++++++++----------
 .../metric/SQLMetricsTestUtils.scala          | 18 ++++++++---------
 5 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index b23ae9ab99c72..bded921a2b83e 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -78,7 +78,7 @@ function setupTooltipForSparkPlanNode(nodeId) {
 
 // labelSeparator should be a non-graphical character in order not to affect the width of boxes.
 var labelSeparator = "\x01";
-var stageAndTaskMetricsPattern = "^(.*)(\\(stage.*attempt.*task[^)]*\\))(.*)$";
+var stageAndTaskMetricsPattern = "^(.*)(\\(stage.*task[^)]*\\))(.*)$";
 
 /*
  * Helper function to pre-process the graph layout.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
index 65aabe004d75b..1394e0f723733 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
@@ -116,7 +116,7 @@ object SQLMetrics {
     // data size total (min, med, max):
     // 100GB (100MB, 1GB, 10GB)
     val acc = new SQLMetric(SIZE_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId (attemptId): taskId))"),
+    acc.register(sc, name = Some(s"$name total (min, med, max (stageId: taskId))"),
       countFailedValues = false)
     acc
   }
@@ -126,7 +126,7 @@ object SQLMetrics {
     // duration(min, med, max):
     // 5s (800ms, 1s, 2s)
     val acc = new SQLMetric(TIMING_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId (attemptId): taskId))"),
+    acc.register(sc, name = Some(s"$name total (min, med, max (stageId: taskId))"),
       countFailedValues = false)
     acc
   }
@@ -134,7 +134,7 @@ object SQLMetrics {
   def createNanoTimingMetric(sc: SparkContext, name: String): SQLMetric = {
     // Same with createTimingMetric, just normalize the unit of time to millisecond.
     val acc = new SQLMetric(NS_TIMING_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId (attemptId): taskId))"),
+    acc.register(sc, name = Some(s"$name total (min, med, max (stageId: taskId))"),
       countFailedValues = false)
     acc
   }
@@ -150,7 +150,7 @@ object SQLMetrics {
     // probe avg (min, med, max):
     // (1.2, 2.2, 6.3)
     val acc = new SQLMetric(AVERAGE_METRIC)
-    acc.register(sc, name = Some(s"$name (min, med, max (stageId (attemptId): taskId))"),
+    acc.register(sc, name = Some(s"$name (min, med, max (stageId: taskId))"),
       countFailedValues = false)
     acc
   }
@@ -169,11 +169,11 @@ object SQLMetrics {
    * and represent it in string for a SQL physical operator.
     */
   def stringValue(metricsType: String, values: Array[Long], maxMetrics: Array[Long]): String = {
-    // stringMetric = "(driver)" OR (stage $stageId (attempt $attemptId): task $taskId))
+    // stringMetric = "(driver)" OR (stage ${stageId}.${attemptId}: task $taskId)
     val stringMetric = if (maxMetrics.isEmpty) {
       "(driver)"
     } else {
-      s"(stage ${maxMetrics(1)} (attempt ${maxMetrics(2)}): task ${maxMetrics(3)})"
+      s"(stage ${maxMetrics(1)}.${maxMetrics(2)}: task ${maxMetrics(3)})"
     }
     if (metricsType == SUM_METRIC) {
       val numberFormat = NumberFormat.getIntegerInstance(Locale.US)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
index ca146e7cd4a3e..e3e22783e18ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/ExecutionPage.scala
@@ -73,7 +73,7 @@ class ExecutionPage(parent: SQLTab) extends WebUIPage("execution") with Logging
         </div>
         <div>
           <input type="checkbox" id="stageId-and-taskId-checkbox"></input>
-          <span>Show the Stage (Stage Attempt): Task ID that corresponds to the max metric</span>
+          <span>Show the Stage ID and Task ID that corresponds to the max metric</span>
         </div>
 
       val metrics = sqlStore.executionMetrics(executionId)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index 7d09577075d5d..11f93c8836500 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -98,7 +98,7 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val ds = spark.range(10).filter('id < 5)
     testSparkPlanMetricsWithPredicates(ds.toDF(), 1, Map(
       0L -> (("WholeStageCodegen (1)", Map(
-        "duration total (min, med, max (stageId (attemptId): taskId))" -> {
+        "duration total (min, med, max (stageId: taskId))" -> {
           _.toString.matches(timingMetricPattern)
         })))), true)
   }
@@ -110,10 +110,10 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df = testData2.groupBy().count() // 2 partitions
     val expected1 = Seq(
       Map("number of output rows" -> 2L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
           aggregateMetricsPattern),
       Map("number of output rows" -> 1L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
           aggregateMetricsPattern))
     val shuffleExpected1 = Map(
       "records read" -> 2L,
@@ -130,10 +130,10 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df2 = testData2.groupBy('a).count()
     val expected2 = Seq(
       Map("number of output rows" -> 4L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
           aggregateMetricsPattern),
       Map("number of output rows" -> 3L,
-        "avg hash probe bucket list iters (min, med, max (stageId (attemptId): taskId))" ->
+        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
           aggregateMetricsPattern))
 
     val shuffleExpected2 = Map(
@@ -181,8 +181,8 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
       }
       val metrics = getSparkPlanMetrics(df, 1, nodeIds, enableWholeStage).get
       nodeIds.foreach { nodeId =>
-        val probes = metrics(nodeId)._2("avg hash probe bucket list iters (min, med, max (stageId" +
-          " (attemptId): taskId))")
+        val probes =
+          metrics(nodeId)._2("avg hash probe bucket list iters (min, med, max (stageId: taskId))")
         // Extract min, med, max from the string and strip off everthing else.
         val index = probes.toString.stripPrefix("\n(").stripSuffix(")").indexOf(" (", 0)
         probes.toString.stripPrefix("\n(").stripSuffix(")").slice(0, index).split(", ").foreach {
@@ -231,13 +231,13 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df = Seq(1, 3, 2).toDF("id").sort('id)
     testSparkPlanMetricsWithPredicates(df, 2, Map(
       0L -> (("Sort", Map(
-        "sort time total (min, med, max (stageId (attemptId): taskId))" -> {
+        "sort time total (min, med, max (stageId: taskId))" -> {
           _.toString.matches(timingMetricPattern)
         },
-        "peak memory total (min, med, max (stageId (attemptId): taskId))" -> {
+        "peak memory total (min, med, max (stageId: taskId))" -> {
           _.toString.matches(sizeMetricPattern)
         },
-        "spill size total (min, med, max (stageId (attemptId): taskId))" -> {
+        "spill size total (min, med, max (stageId: taskId))" -> {
           _.toString.matches(sizeMetricPattern)
         })))
     ))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
index 0c1148f7b82e4..766e7a9748510 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
@@ -41,27 +41,27 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
 
   protected def statusStore: SQLAppStatusStore = spark.sharedState.statusStore
 
-  // Pattern of size SQLMetric value, e.g. "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB (stage 0
-  // (attempt 0): task 4))" OR "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB)"
+  // Pattern of size SQLMetric value, e.g. "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB (stage 0.0:
+  // task 4))" OR "\n96.2 MiB (32.1 MiB, 32.1 MiB, 32.1 MiB)"
   protected val sizeMetricPattern = {
     val bytes = "([0-9]+(\\.[0-9]+)?) (EiB|PiB|TiB|GiB|MiB|KiB|B)"
-    val maxMetrics = "\\(stage ([0-9])+ \\(attempt ([0-9])+\\)\\: task ([0-9])+\\)"
+    val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
     s"\\n$bytes \\($bytes, $bytes, $bytes( $maxMetrics)?\\)"
   }
 
-  // Pattern of timing SQLMetric value, e.g. "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms (stage 3 (attempt
-  // 0): task 217))" OR "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms)"
+  // Pattern of timing SQLMetric value, e.g. "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms (stage 3.0):
+  // task 217))" OR "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms)"
   protected val timingMetricPattern = {
     val duration = "([0-9]+(\\.[0-9]+)?) (ms|s|m|h)"
-    val maxMetrics = "\\(stage ([0-9])+ \\(attempt ([0-9])+\\)\\: task ([0-9])+\\)"
+    val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
     s"\\n$duration \\($duration, $duration, $duration( $maxMetrics)?\\)"
   }
 
   // Pattern of size SQLMetric value for Aggregate tests.
-  // e.g "\n(1, 1, 0.9 (stage 1 (attempt 0): task 8)) OR "\n(1, 1, 0.9 )"
+  // e.g "\n(1, 1, 0.9 (stage 1.0: task 8)) OR "\n(1, 1, 0.9 )"
   protected val aggregateMetricsPattern = {
     val iters = "([0-9]+(\\.[0-9]+)?)"
-    val maxMetrics = "\\(stage ([0-9])+ \\(attempt ([0-9])+\\)\\: task ([0-9])+\\)"
+    val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
     s"\\n\\($iters, $iters, $iters( $maxMetrics)?\\)"
   }
 
@@ -98,7 +98,7 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
     }
 
     val totalNumBytesMetric = executedNode.metrics.find(
-      _.name == "written output total (min, med, max (stageId (attemptId): taskId))").get
+      _.name == "written output total (min, med, max (stageId: taskId))").get
     val totalNumBytes = metrics(totalNumBytesMetric.accumulatorId).replaceAll(",", "")
       .split(" ").head.trim.toDouble
     assert(totalNumBytes > 0)

From b300ed8065f327bcad80dffdfe94e6699562e68a Mon Sep 17 00:00:00 2001
From: DB Tsai <d_tsai@apple.com>
Date: Fri, 27 Mar 2020 14:28:57 +0800
Subject: [PATCH 0360/1280] 
 [SPARK-25556][SPARK-17636][SPARK-31026][SPARK-31060][SQL][TEST-HIVE1.2]
 Nested Column Predicate Pushdown for Parquet

### What changes were proposed in this pull request?
1. `DataSourceStrategy.scala` is extended to create `org.apache.spark.sql.sources.Filter` from nested expressions.
2. Translation from nested `org.apache.spark.sql.sources.Filter` to `org.apache.parquet.filter2.predicate.FilterPredicate` is implemented to support nested predicate pushdown for Parquet.

### Why are the changes needed?
Better performance for handling nested predicate pushdown.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
New tests are added.

Closes #27728 from dbtsai/SPARK-17636.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit cb0db213736de5c5c02b09a2d5c3e17254708ce1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalog/CatalogV2Implicits.scala          |   8 +
 .../apache/spark/sql/internal/SQLConf.scala   |  13 +
 .../apache/spark/sql/sources/filters.scala    |  60 +-
 .../datasources/DataSourceStrategy.scala      |  15 +-
 .../datasources/parquet/ParquetFilters.scala  |  79 +-
 .../datasources/v2/orc/OrcScanBuilder.scala   |   7 +-
 .../datasources/DataSourceStrategySuite.scala |  54 +-
 .../parquet/ParquetFilterSuite.scala          | 754 +++++++++++-------
 .../datasources/parquet/ParquetIOSuite.scala  |  20 +-
 .../datasources/parquet/ParquetTest.scala     |  12 +-
 .../spark/sql/sources/FiltersSuite.scala      | 173 ++--
 .../datasources/orc/OrcFilters.scala          |  65 +-
 .../datasources/orc/OrcFilters.scala          |  65 +-
 .../spark/sql/hive/orc/OrcFilters.scala       |   7 +-
 14 files changed, 852 insertions(+), 480 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
index 71bab624f06a8..d90804f4b6ff6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Implicits.scala
@@ -20,7 +20,9 @@ package org.apache.spark.sql.connector.catalog
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.BucketSpec
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.connector.expressions.{BucketTransform, IdentityTransform, LogicalExpressions, Transform}
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Conversion helpers for working with v2 [[CatalogPlugin]].
@@ -132,4 +134,10 @@ private[sql] object CatalogV2Implicits {
       part
     }
   }
+
+  private lazy val catalystSqlParser = new CatalystSqlParser(SQLConf.get)
+
+  def parseColumnPath(name: String): Seq[String] = {
+    catalystSqlParser.parseMultipartIdentifier(name)
+  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index c61a57ebb422a..a6cb96d81c519 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1857,6 +1857,17 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
+  val NESTED_PREDICATE_PUSHDOWN_ENABLED =
+    buildConf("spark.sql.optimizer.nestedPredicatePushdown.enabled")
+      .internal()
+      .doc("When true, Spark tries to push down predicates for nested columns and or names " +
+        "containing `dots` to data sources. Currently, Parquet implements both optimizations " +
+        "while ORC only supports predicates for names containing `dots`. The other data sources" +
+        "don't support this feature yet.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(true)
+
   val SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.serializer.nestedSchemaPruning.enabled")
       .internal()
@@ -2790,6 +2801,8 @@ class SQLConf extends Serializable with Logging {
 
   def nestedSchemaPruningEnabled: Boolean = getConf(NESTED_SCHEMA_PRUNING_ENABLED)
 
+  def nestedPredicatePushdownEnabled: Boolean = getConf(NESTED_PREDICATE_PUSHDOWN_ENABLED)
+
   def serializerNestedSchemaPruningEnabled: Boolean =
     getConf(SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
index 020dd79f8f0d7..319073e4475be 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.sources
 
 import org.apache.spark.annotation.{Evolving, Stable}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 // This file defines all the filters that we can push down to the data sources.
@@ -32,6 +33,10 @@ import org.apache.spark.annotation.{Evolving, Stable}
 sealed abstract class Filter {
   /**
    * List of columns that are referenced by this filter.
+   *
+   * Note that, each element in `references` represents a column; `dots` are used as separators
+   * for nested columns. If any part of the names contains `dots`, it is quoted to avoid confusion.
+   *
    * @since 2.1.0
    */
   def references: Array[String]
@@ -40,12 +45,32 @@ sealed abstract class Filter {
     case f: Filter => f.references
     case _ => Array.empty
   }
+
+  /**
+   * List of columns that are referenced by this filter.
+   *
+   * @return each element is a column name as an array of string multi-identifier
+   * @since 3.0.0
+   */
+  def v2references: Array[Array[String]] = {
+    this.references.map(parseColumnPath(_).toArray)
+  }
+
+  /**
+   * If any of the references of this filter contains nested column
+   */
+  private[sql] def containsNestedColumn: Boolean = {
+    this.v2references.exists(_.length > 1)
+  }
 }
 
 /**
- * A filter that evaluates to `true` iff the attribute evaluates to a value
+ * A filter that evaluates to `true` iff the column evaluates to a value
  * equal to `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -58,6 +83,9 @@ case class EqualTo(attribute: String, value: Any) extends Filter {
  * in that it returns `true` (rather than NULL) if both inputs are NULL, and `false`
  * (rather than NULL) if one of the input is NULL and the other is not NULL.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.5.0
  */
 @Stable
@@ -69,6 +97,9 @@ case class EqualNullSafe(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * greater than `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -80,6 +111,9 @@ case class GreaterThan(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * greater than or equal to `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -91,6 +125,9 @@ case class GreaterThanOrEqual(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * less than `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -102,6 +139,9 @@ case class LessThan(attribute: String, value: Any) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to a value
  * less than or equal to `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -112,6 +152,9 @@ case class LessThanOrEqual(attribute: String, value: Any) extends Filter {
 /**
  * A filter that evaluates to `true` iff the attribute evaluates to one of the values in the array.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -139,6 +182,9 @@ case class In(attribute: String, values: Array[Any]) extends Filter {
 /**
  * A filter that evaluates to `true` iff the attribute evaluates to null.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -149,6 +195,9 @@ case class IsNull(attribute: String) extends Filter {
 /**
  * A filter that evaluates to `true` iff the attribute evaluates to a non-null value.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.0
  */
 @Stable
@@ -190,6 +239,9 @@ case class Not(child: Filter) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to
  * a string that starts with `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.1
  */
 @Stable
@@ -201,6 +253,9 @@ case class StringStartsWith(attribute: String, value: String) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to
  * a string that ends with `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.1
  */
 @Stable
@@ -212,6 +267,9 @@ case class StringEndsWith(attribute: String, value: String) extends Filter {
  * A filter that evaluates to `true` iff the attribute evaluates to
  * a string that contains the string `value`.
  *
+ * @param attribute of the column to be evaluated; `dots` are used as separators
+ *                  for nested columns. If any part of the names contains `dots`,
+ *                  it is quoted to avoid confusion.
  * @since 1.3.1
  */
 @Stable
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index 1641b660a271d..faf37609ad814 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -652,10 +652,19 @@ object DataSourceStrategy {
  */
 object PushableColumn {
   def unapply(e: Expression): Option[String] = {
-    def helper(e: Expression) = e match {
-      case a: Attribute => Some(a.name)
+    val nestedPredicatePushdownEnabled = SQLConf.get.nestedPredicatePushdownEnabled
+    import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+    def helper(e: Expression): Option[Seq[String]] = e match {
+      case a: Attribute =>
+        if (nestedPredicatePushdownEnabled || !a.name.contains(".")) {
+          Some(Seq(a.name))
+        } else {
+          None
+        }
+      case s: GetStructField if nestedPredicatePushdownEnabled =>
+        helper(s.child).map(_ :+ s.childSchema(s.ordinal).name)
       case _ => None
     }
-    helper(e)
+    helper(e).map(_.quoted)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index 07065018343cf..f206f59dacdc6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -27,7 +27,7 @@ import scala.collection.JavaConverters.asScalaBufferConverter
 import org.apache.parquet.filter2.predicate._
 import org.apache.parquet.filter2.predicate.SparkFilterApi._
 import org.apache.parquet.io.api.Binary
-import org.apache.parquet.schema.{DecimalMetadata, MessageType, OriginalType, PrimitiveComparator}
+import org.apache.parquet.schema.{DecimalMetadata, GroupType, MessageType, OriginalType, PrimitiveComparator, PrimitiveType, Type}
 import org.apache.parquet.schema.OriginalType._
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._
@@ -49,15 +49,35 @@ class ParquetFilters(
     pushDownInFilterThreshold: Int,
     caseSensitive: Boolean) {
   // A map which contains parquet field name and data type, if predicate push down applies.
-  private val nameToParquetField : Map[String, ParquetField] = {
-    // Here we don't flatten the fields in the nested schema but just look up through
-    // root fields. Currently, accessing to nested fields does not push down filters
-    // and it does not support to create filters for them.
-    val primitiveFields =
-    schema.getFields.asScala.filter(_.isPrimitive).map(_.asPrimitiveType()).map { f =>
-      f.getName -> ParquetField(f.getName,
-        ParquetSchemaType(f.getOriginalType,
-          f.getPrimitiveTypeName, f.getTypeLength, f.getDecimalMetadata))
+  //
+  // Each key in `nameToParquetField` represents a column; `dots` are used as separators for
+  // nested columns. If any part of the names contains `dots`, it is quoted to avoid confusion.
+  // See `org.apache.spark.sql.connector.catalog.quote` for implementation details.
+  private val nameToParquetField : Map[String, ParquetPrimitiveField] = {
+    // Recursively traverse the parquet schema to get primitive fields that can be pushed-down.
+    // `parentFieldNames` is used to keep track of the current nested level when traversing.
+    def getPrimitiveFields(
+        fields: Seq[Type],
+        parentFieldNames: Array[String] = Array.empty): Seq[ParquetPrimitiveField] = {
+      fields.flatMap {
+        case p: PrimitiveType =>
+          Some(ParquetPrimitiveField(fieldNames = parentFieldNames :+ p.getName,
+            fieldType = ParquetSchemaType(p.getOriginalType,
+              p.getPrimitiveTypeName, p.getTypeLength, p.getDecimalMetadata)))
+        // Note that when g is a `Struct`, `g.getOriginalType` is `null`.
+        // When g is a `Map`, `g.getOriginalType` is `MAP`.
+        // When g is a `List`, `g.getOriginalType` is `LIST`.
+        case g: GroupType if g.getOriginalType == null =>
+          getPrimitiveFields(g.getFields.asScala, parentFieldNames :+ g.getName)
+        // Parquet only supports push-down for primitive types; as a result, Map and List types
+        // are removed.
+        case _ => None
+      }
+    }
+
+    val primitiveFields = getPrimitiveFields(schema.getFields.asScala).map { field =>
+      import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
+      (field.fieldNames.toSeq.quoted, field)
     }
     if (caseSensitive) {
       primitiveFields.toMap
@@ -75,13 +95,13 @@ class ParquetFilters(
   }
 
   /**
-   * Holds a single field information stored in the underlying parquet file.
+   * Holds a single primitive field information stored in the underlying parquet file.
    *
-   * @param fieldName field name in parquet file
+   * @param fieldNames a field name as an array of string multi-identifier in parquet file
    * @param fieldType field type related info in parquet file
    */
-  private case class ParquetField(
-      fieldName: String,
+  private case class ParquetPrimitiveField(
+      fieldNames: Array[String],
       fieldType: ParquetSchemaType)
 
   private case class ParquetSchemaType(
@@ -472,13 +492,8 @@ class ParquetFilters(
     case _ => false
   }
 
-  // Parquet does not allow dots in the column name because dots are used as a column path
-  // delimiter. Since Parquet 1.8.2 (PARQUET-389), Parquet accepts the filter predicates
-  // with missing columns. The incorrect results could be got from Parquet when we push down
-  // filters for the column having dots in the names. Thus, we do not push down such filters.
-  // See SPARK-20364.
   private def canMakeFilterOn(name: String, value: Any): Boolean = {
-    nameToParquetField.contains(name) && !name.contains(".") && valueCanMakeFilterOn(name, value)
+    nameToParquetField.contains(name) && valueCanMakeFilterOn(name, value)
   }
 
   /**
@@ -509,38 +524,38 @@ class ParquetFilters(
     predicate match {
       case sources.IsNull(name) if canMakeFilterOn(name, null) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), null))
+          .map(_(nameToParquetField(name).fieldNames, null))
       case sources.IsNotNull(name) if canMakeFilterOn(name, null) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), null))
+          .map(_(nameToParquetField(name).fieldNames, null))
 
       case sources.EqualTo(name, value) if canMakeFilterOn(name, value) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.Not(sources.EqualTo(name, value)) if canMakeFilterOn(name, value) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.EqualNullSafe(name, value) if canMakeFilterOn(name, value) =>
         makeEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.Not(sources.EqualNullSafe(name, value)) if canMakeFilterOn(name, value) =>
         makeNotEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.LessThan(name, value) if canMakeFilterOn(name, value) =>
         makeLt.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.LessThanOrEqual(name, value) if canMakeFilterOn(name, value) =>
         makeLtEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.GreaterThan(name, value) if canMakeFilterOn(name, value) =>
         makeGt.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
       case sources.GreaterThanOrEqual(name, value) if canMakeFilterOn(name, value) =>
         makeGtEq.lift(nameToParquetField(name).fieldType)
-          .map(_(Array(nameToParquetField(name).fieldName), value))
+          .map(_(nameToParquetField(name).fieldNames, value))
 
       case sources.And(lhs, rhs) =>
         // At here, it is not safe to just convert one side and remove the other side
@@ -591,13 +606,13 @@ class ParquetFilters(
         && values.distinct.length <= pushDownInFilterThreshold =>
         values.distinct.flatMap { v =>
           makeEq.lift(nameToParquetField(name).fieldType)
-            .map(_(Array(nameToParquetField(name).fieldName), v))
+            .map(_(nameToParquetField(name).fieldNames, v))
         }.reduceLeftOption(FilterApi.or)
 
       case sources.StringStartsWith(name, prefix)
           if pushDownStartWith && canMakeFilterOn(name, prefix) =>
         Option(prefix).map { v =>
-          FilterApi.userDefined(binaryColumn(Array(nameToParquetField(name).fieldName)),
+          FilterApi.userDefined(binaryColumn(nameToParquetField(name).fieldNames),
             new UserDefinedPredicate[Binary] with Serializable {
               private val strToBinary = Binary.fromReusedByteArray(v.getBytes)
               private val size = strToBinary.length
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
index 1421ffd8b6de4..9f40f5faa2e99 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
@@ -22,6 +22,7 @@ import scala.collection.JavaConverters._
 import org.apache.orc.mapreduce.OrcInputFormat
 
 import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.orc.OrcFilters
@@ -59,8 +60,10 @@ case class OrcScanBuilder(
         // changed `hadoopConf` in executors.
         OrcInputFormat.setSearchArgument(hadoopConf, f, schema.fieldNames)
       }
-      val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
-      _pushedFilters = OrcFilters.convertibleFilters(schema, dataTypeMap, filters).toArray
+      val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
+      // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+      val newFilters = filters.filter(!_.containsNestedColumn)
+      _pushedFilters = OrcFilters.convertibleFilters(schema, dataTypeMap, newFilters).toArray
     }
     filters
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
index 7bd3213b378ce..a775a97895cfc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
@@ -26,15 +26,61 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructT
 
 class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
   val attrInts = Seq(
-    'cint.int
+    'cint.int,
+    Symbol("c.int").int,
+    GetStructField('a.struct(StructType(
+      StructField("cstr", StringType, nullable = true) ::
+        StructField("cint", IntegerType, nullable = true) :: Nil)), 1, None),
+    GetStructField('a.struct(StructType(
+      StructField("c.int", IntegerType, nullable = true) ::
+        StructField("cstr", StringType, nullable = true) :: Nil)), 0, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("cstr1", StringType, nullable = true) ::
+        StructField("cstr2", StringType, nullable = true) ::
+        StructField("cint", IntegerType, nullable = true) :: Nil)), 2, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("c.int", IntegerType, nullable = true) :: Nil)), 0, None),
+    GetStructField(GetStructField('a.struct(StructType(
+      StructField("cstr1", StringType, nullable = true) ::
+        StructField("b", StructType(StructField("cint", IntegerType, nullable = true) ::
+          StructField("cstr2", StringType, nullable = true) :: Nil)) :: Nil)), 1, None), 0, None)
   ).zip(Seq(
-    "cint"
+    "cint",
+    "`c.int`", // single level field that contains `dot` in name
+    "a.cint", // two level nested field
+    "a.`c.int`", // two level nested field, and nested level contains `dot`
+    "`a.b`.cint", // two level nested field, and top level contains `dot`
+    "`a.b`.`c.int`", // two level nested field, and both levels contain `dot`
+    "a.b.cint" // three level nested field
   ))
 
   val attrStrs = Seq(
-    'cstr.string
+    'cstr.string,
+    Symbol("c.str").string,
+    GetStructField('a.struct(StructType(
+      StructField("cint", IntegerType, nullable = true) ::
+        StructField("cstr", StringType, nullable = true) :: Nil)), 1, None),
+    GetStructField('a.struct(StructType(
+      StructField("c.str", StringType, nullable = true) ::
+        StructField("cint", IntegerType, nullable = true) :: Nil)), 0, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("cint1", IntegerType, nullable = true) ::
+        StructField("cint2", IntegerType, nullable = true) ::
+        StructField("cstr", StringType, nullable = true) :: Nil)), 2, None),
+    GetStructField(Symbol("a.b").struct(StructType(
+      StructField("c.str", StringType, nullable = true) :: Nil)), 0, None),
+    GetStructField(GetStructField('a.struct(StructType(
+      StructField("cint1", IntegerType, nullable = true) ::
+        StructField("b", StructType(StructField("cstr", StringType, nullable = true) ::
+          StructField("cint2", IntegerType, nullable = true) :: Nil)) :: Nil)), 1, None), 0, None)
   ).zip(Seq(
-    "cstr"
+    "cstr",
+    "`c.str`", // single level field that contains `dot` in name
+    "a.cstr", // two level nested field
+    "a.`c.str`", // two level nested field, and nested level contains `dot`
+    "`a.b`.cstr", // two level nested field, and top level contains `dot`
+    "`a.b`.`c.str`", // two level nested field, and both levels contain `dot`
+    "a.b.cstr" // three level nested field
   ))
 
   test("translate simple expression") { attrInts.zip(attrStrs)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 4e0c1c2dbe601..d1161e33b0941 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -103,22 +103,42 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     checkFilterPredicate(predicate, filterClass, Seq(Row(expected)))(df)
   }
 
-  private def checkBinaryFilterPredicate
-      (predicate: Predicate, filterClass: Class[_ <: FilterPredicate], expected: Seq[Row])
-      (implicit df: DataFrame): Unit = {
-    def checkBinaryAnswer(df: DataFrame, expected: Seq[Row]) = {
-      assertResult(expected.map(_.getAs[Array[Byte]](0).mkString(",")).sorted) {
-        df.rdd.map(_.getAs[Array[Byte]](0).mkString(",")).collect().toSeq.sorted
-      }
+  /**
+   * Takes single level `inputDF` dataframe to generate multi-level nested
+   * dataframes as new test data.
+   */
+  private def withNestedDataFrame(inputDF: DataFrame)
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit = {
+    assert(inputDF.schema.fields.length == 1)
+    assert(!inputDF.schema.fields.head.dataType.isInstanceOf[StructType])
+    val df = inputDF.toDF("temp")
+    Seq(
+      (
+        df.withColumnRenamed("temp", "a"),
+        "a", // zero nesting
+        (x: Any) => x),
+      (
+        df.withColumn("a", struct(df("temp") as "b")).drop("temp"),
+        "a.b", // one level nesting
+        (x: Any) => Row(x)),
+      (
+        df.withColumn("a", struct(struct(df("temp") as "c") as "b")).drop("temp"),
+        "a.b.c", // two level nesting
+        (x: Any) => Row(Row(x))
+      ),
+      (
+        df.withColumnRenamed("temp", "a.b"),
+        "`a.b`", // zero nesting with column name containing `dots`
+        (x: Any) => x
+      ),
+      (
+        df.withColumn("a.b", struct(df("temp") as "c.d") ).drop("temp"),
+        "`a.b`.`c.d`", // one level nesting with column names containing `dots`
+        (x: Any) => Row(x)
+      )
+    ).foreach { case (df, colName, resultFun) =>
+      runTest(df, colName, resultFun)
     }
-
-    checkFilterPredicate(df, predicate, filterClass, checkBinaryAnswer _, expected)
-  }
-
-  private def checkBinaryFilterPredicate
-      (predicate: Predicate, filterClass: Class[_ <: FilterPredicate], expected: Array[Byte])
-      (implicit df: DataFrame): Unit = {
-    checkBinaryFilterPredicate(predicate, filterClass, Seq(Row(expected)))(df)
   }
 
   private def testTimestampPushdown(data: Seq[Timestamp]): Unit = {
@@ -128,36 +148,38 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     val ts3 = data(2)
     val ts4 = data(3)
 
-    withParquetDataFrame(data.map(i => Tuple1(i))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], data.map(i => Row.apply(i)))
-
-      checkFilterPredicate('_1 === ts1, classOf[Eq[_]], ts1)
-      checkFilterPredicate('_1 <=> ts1, classOf[Eq[_]], ts1)
-      checkFilterPredicate('_1 =!= ts1, classOf[NotEq[_]],
-        Seq(ts2, ts3, ts4).map(i => Row.apply(i)))
-
-      checkFilterPredicate('_1 < ts2, classOf[Lt[_]], ts1)
-      checkFilterPredicate('_1 > ts1, classOf[Gt[_]], Seq(ts2, ts3, ts4).map(i => Row.apply(i)))
-      checkFilterPredicate('_1 <= ts1, classOf[LtEq[_]], ts1)
-      checkFilterPredicate('_1 >= ts4, classOf[GtEq[_]], ts4)
-
-      checkFilterPredicate(Literal(ts1) === '_1, classOf[Eq[_]], ts1)
-      checkFilterPredicate(Literal(ts1) <=> '_1, classOf[Eq[_]], ts1)
-      checkFilterPredicate(Literal(ts2) > '_1, classOf[Lt[_]], ts1)
-      checkFilterPredicate(Literal(ts3) < '_1, classOf[Gt[_]], ts4)
-      checkFilterPredicate(Literal(ts1) >= '_1, classOf[LtEq[_]], ts1)
-      checkFilterPredicate(Literal(ts4) <= '_1, classOf[GtEq[_]], ts4)
-
-      checkFilterPredicate(!('_1 < ts4), classOf[GtEq[_]], ts4)
-      checkFilterPredicate('_1 < ts2 || '_1 > ts3, classOf[Operators.Or], Seq(Row(ts1), Row(ts4)))
-    }
-  }
-
-  private def testDecimalPushDown(data: DataFrame)(f: DataFrame => Unit): Unit = {
-    withTempPath { file =>
-      data.write.parquet(file.getCanonicalPath)
-      readParquetFile(file.toString)(f)
+    import testImplicits._
+    withNestedDataFrame(data.map(i => Tuple1(i)).toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val tsAttr = df(colName).expr
+        assert(df(colName).expr.dataType === TimestampType)
+
+        checkFilterPredicate(tsAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
+          data.map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(tsAttr === ts1, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr <=> ts1, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr =!= ts1, classOf[NotEq[_]],
+          Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(tsAttr < ts2, classOf[Lt[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr > ts1, classOf[Gt[_]],
+          Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+        checkFilterPredicate(tsAttr <= ts1, classOf[LtEq[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr >= ts4, classOf[GtEq[_]], resultFun(ts4))
+
+        checkFilterPredicate(Literal(ts1) === tsAttr, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts1) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts2) > tsAttr, classOf[Lt[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts3) < tsAttr, classOf[Gt[_]], resultFun(ts4))
+        checkFilterPredicate(Literal(ts1) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts4) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
+
+        checkFilterPredicate(!(tsAttr < ts4), classOf[GtEq[_]], resultFun(ts4))
+        checkFilterPredicate(tsAttr < ts2 || tsAttr > ts3, classOf[Operators.Or],
+          Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
+      }
     }
   }
 
@@ -187,201 +209,273 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("filter pushdown - boolean") {
-    withParquetDataFrame((true :: false :: Nil).map(b => Tuple1.apply(Option(b)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], Seq(Row(true), Row(false)))
-
-      checkFilterPredicate('_1 === true, classOf[Eq[_]], true)
-      checkFilterPredicate('_1 <=> true, classOf[Eq[_]], true)
-      checkFilterPredicate('_1 =!= true, classOf[NotEq[_]], false)
+    val data = (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val booleanAttr = df(colName).expr
+        assert(df(colName).expr.dataType === BooleanType)
+
+        checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
+          Seq(Row(resultFun(true)), Row(resultFun(false))))
+
+        checkFilterPredicate(booleanAttr === true, classOf[Eq[_]], resultFun(true))
+        checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]], resultFun(true))
+        checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]], resultFun(false))
+      }
     }
   }
 
   test("filter pushdown - tinyint") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toByte)))) { implicit df =>
-      assert(df.schema.head.dataType === ByteType)
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1.toByte, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1.toByte, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1.toByte, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2.toByte, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3.toByte, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1.toByte, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4.toByte, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1.toByte) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1.toByte) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2.toByte) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3.toByte) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1.toByte) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4.toByte) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4.toByte), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2.toByte || '_1 > 3.toByte,
-        classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toByte)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val tinyIntAttr = df(colName).expr
+        assert(df(colName).expr.dataType === ByteType)
+
+        checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(Literal(1.toByte) === tinyIntAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]], resultFun(4))
+        checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
+          classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
+      }
     }
   }
 
   test("filter pushdown - smallint") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toShort)))) { implicit df =>
-      assert(df.schema.head.dataType === ShortType)
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1.toShort, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1.toShort, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1.toShort, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2.toShort, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3.toShort, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1.toShort, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4.toShort, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1.toShort) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1.toShort) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2.toShort) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3.toShort) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1.toShort) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4.toShort) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4.toShort), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2.toShort || '_1 > 3.toShort,
-        classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toShort)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val smallIntAttr = df(colName).expr
+        assert(df(colName).expr.dataType === ShortType)
+
+        checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(Literal(1.toShort) === smallIntAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(2.toShort) > smallIntAttr, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(Literal(3.toShort) < smallIntAttr, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(Literal(1.toShort) >= smallIntAttr, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(Literal(4.toShort) <= smallIntAttr, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]], resultFun(4))
+        checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr > 3.toShort,
+          classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
+      }
     }
   }
 
   test("filter pushdown - integer") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val intAttr = df(colName).expr
+        assert(df(colName).expr.dataType === IntegerType)
+
+        checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
+        checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
+          Seq(Row(resultFun(1)), Row(resultFun(4))))
+      }
     }
   }
 
   test("filter pushdown - long") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toLong)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toLong)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val longAttr = df(colName).expr
+        assert(df(colName).expr.dataType === LongType)
+
+        checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
+        checkFilterPredicate(longAttr < 2 || longAttr > 3, classOf[Operators.Or],
+          Seq(Row(resultFun(1)), Row(resultFun(4))))
+      }
     }
   }
 
   test("filter pushdown - float") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toFloat)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toFloat)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val floatAttr = df(colName).expr
+        assert(df(colName).expr.dataType === FloatType)
+
+        checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
+        checkFilterPredicate(floatAttr < 2 || floatAttr > 3, classOf[Operators.Or],
+          Seq(Row(resultFun(1)), Row(resultFun(4))))
+      }
     }
   }
 
   test("filter pushdown - double") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Option(i.toDouble)))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 === 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 <=> 1, classOf[Eq[_]], 1)
-      checkFilterPredicate('_1 =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-      checkFilterPredicate('_1 < 2, classOf[Lt[_]], 1)
-      checkFilterPredicate('_1 > 3, classOf[Gt[_]], 4)
-      checkFilterPredicate('_1 <= 1, classOf[LtEq[_]], 1)
-      checkFilterPredicate('_1 >= 4, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(Literal(1) === '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(1) <=> '_1, classOf[Eq[_]], 1)
-      checkFilterPredicate(Literal(2) > '_1, classOf[Lt[_]], 1)
-      checkFilterPredicate(Literal(3) < '_1, classOf[Gt[_]], 4)
-      checkFilterPredicate(Literal(1) >= '_1, classOf[LtEq[_]], 1)
-      checkFilterPredicate(Literal(4) <= '_1, classOf[GtEq[_]], 4)
-
-      checkFilterPredicate(!('_1 < 4), classOf[GtEq[_]], 4)
-      checkFilterPredicate('_1 < 2 || '_1 > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toDouble)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val doubleAttr = df(colName).expr
+        assert(df(colName).expr.dataType === DoubleType)
+
+        checkFilterPredicate(doubleAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(doubleAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(doubleAttr === 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(doubleAttr <=> 1, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(doubleAttr =!= 1, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(doubleAttr < 2, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(doubleAttr > 3, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(doubleAttr <= 1, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(doubleAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(Literal(1) === doubleAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(1) <=> doubleAttr, classOf[Eq[_]], resultFun(1))
+        checkFilterPredicate(Literal(2) > doubleAttr, classOf[Lt[_]], resultFun(1))
+        checkFilterPredicate(Literal(3) < doubleAttr, classOf[Gt[_]], resultFun(4))
+        checkFilterPredicate(Literal(1) >= doubleAttr, classOf[LtEq[_]], resultFun(1))
+        checkFilterPredicate(Literal(4) <= doubleAttr, classOf[GtEq[_]], resultFun(4))
+
+        checkFilterPredicate(!(doubleAttr < 4), classOf[GtEq[_]], resultFun(4))
+        checkFilterPredicate(doubleAttr < 2 || doubleAttr > 3, classOf[Operators.Or],
+          Seq(Row(resultFun(1)), Row(resultFun(4))))
+      }
     }
   }
 
   test("filter pushdown - string") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(i.toString))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate(
-        '_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(i => Row.apply(i.toString)))
-
-      checkFilterPredicate('_1 === "1", classOf[Eq[_]], "1")
-      checkFilterPredicate('_1 <=> "1", classOf[Eq[_]], "1")
-      checkFilterPredicate(
-        '_1 =!= "1", classOf[NotEq[_]], (2 to 4).map(i => Row.apply(i.toString)))
-
-      checkFilterPredicate('_1 < "2", classOf[Lt[_]], "1")
-      checkFilterPredicate('_1 > "3", classOf[Gt[_]], "4")
-      checkFilterPredicate('_1 <= "1", classOf[LtEq[_]], "1")
-      checkFilterPredicate('_1 >= "4", classOf[GtEq[_]], "4")
-
-      checkFilterPredicate(Literal("1") === '_1, classOf[Eq[_]], "1")
-      checkFilterPredicate(Literal("1") <=> '_1, classOf[Eq[_]], "1")
-      checkFilterPredicate(Literal("2") > '_1, classOf[Lt[_]], "1")
-      checkFilterPredicate(Literal("3") < '_1, classOf[Gt[_]], "4")
-      checkFilterPredicate(Literal("1") >= '_1, classOf[LtEq[_]], "1")
-      checkFilterPredicate(Literal("4") <= '_1, classOf[GtEq[_]], "4")
-
-      checkFilterPredicate(!('_1 < "4"), classOf[GtEq[_]], "4")
-      checkFilterPredicate('_1 < "2" || '_1 > "3", classOf[Operators.Or], Seq(Row("1"), Row("4")))
+    val data = (1 to 4).map(i => Tuple1(Option(i.toString)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val stringAttr = df(colName).expr
+        assert(df(colName).expr.dataType === StringType)
+
+        checkFilterPredicate(stringAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(stringAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+        checkFilterPredicate(stringAttr === "1", classOf[Eq[_]], resultFun("1"))
+        checkFilterPredicate(stringAttr <=> "1", classOf[Eq[_]], resultFun("1"))
+        checkFilterPredicate(stringAttr =!= "1", classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+        checkFilterPredicate(stringAttr < "2", classOf[Lt[_]], resultFun("1"))
+        checkFilterPredicate(stringAttr > "3", classOf[Gt[_]], resultFun("4"))
+        checkFilterPredicate(stringAttr <= "1", classOf[LtEq[_]], resultFun("1"))
+        checkFilterPredicate(stringAttr >= "4", classOf[GtEq[_]], resultFun("4"))
+
+        checkFilterPredicate(Literal("1") === stringAttr, classOf[Eq[_]], resultFun("1"))
+        checkFilterPredicate(Literal("1") <=> stringAttr, classOf[Eq[_]], resultFun("1"))
+        checkFilterPredicate(Literal("2") > stringAttr, classOf[Lt[_]], resultFun("1"))
+        checkFilterPredicate(Literal("3") < stringAttr, classOf[Gt[_]], resultFun("4"))
+        checkFilterPredicate(Literal("1") >= stringAttr, classOf[LtEq[_]], resultFun("1"))
+        checkFilterPredicate(Literal("4") <= stringAttr, classOf[GtEq[_]], resultFun("4"))
+
+        checkFilterPredicate(!(stringAttr < "4"), classOf[GtEq[_]], resultFun("4"))
+        checkFilterPredicate(stringAttr < "2" || stringAttr > "3", classOf[Operators.Or],
+          Seq(Row(resultFun("1")), Row(resultFun("4"))))
+      }
     }
   }
 
@@ -390,32 +484,39 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       def b: Array[Byte] = int.toString.getBytes(StandardCharsets.UTF_8)
     }
 
-    withParquetDataFrame((1 to 4).map(i => Tuple1(i.b))) { implicit df =>
-      checkBinaryFilterPredicate('_1 === 1.b, classOf[Eq[_]], 1.b)
-      checkBinaryFilterPredicate('_1 <=> 1.b, classOf[Eq[_]], 1.b)
-
-      checkBinaryFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkBinaryFilterPredicate(
-        '_1.isNotNull, classOf[NotEq[_]], (1 to 4).map(i => Row.apply(i.b)).toSeq)
-
-      checkBinaryFilterPredicate(
-        '_1 =!= 1.b, classOf[NotEq[_]], (2 to 4).map(i => Row.apply(i.b)).toSeq)
-
-      checkBinaryFilterPredicate('_1 < 2.b, classOf[Lt[_]], 1.b)
-      checkBinaryFilterPredicate('_1 > 3.b, classOf[Gt[_]], 4.b)
-      checkBinaryFilterPredicate('_1 <= 1.b, classOf[LtEq[_]], 1.b)
-      checkBinaryFilterPredicate('_1 >= 4.b, classOf[GtEq[_]], 4.b)
-
-      checkBinaryFilterPredicate(Literal(1.b) === '_1, classOf[Eq[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(1.b) <=> '_1, classOf[Eq[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(2.b) > '_1, classOf[Lt[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(3.b) < '_1, classOf[Gt[_]], 4.b)
-      checkBinaryFilterPredicate(Literal(1.b) >= '_1, classOf[LtEq[_]], 1.b)
-      checkBinaryFilterPredicate(Literal(4.b) <= '_1, classOf[GtEq[_]], 4.b)
-
-      checkBinaryFilterPredicate(!('_1 < 4.b), classOf[GtEq[_]], 4.b)
-      checkBinaryFilterPredicate(
-        '_1 < 2.b || '_1 > 3.b, classOf[Operators.Or], Seq(Row(1.b), Row(4.b)))
+    val data = (1 to 4).map(i => Tuple1(Option(i.b)))
+    import testImplicits._
+    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val binaryAttr: Expression = df(colName).expr
+        assert(df(colName).expr.dataType === BinaryType)
+
+        checkFilterPredicate(binaryAttr === 1.b, classOf[Eq[_]], resultFun(1.b))
+        checkFilterPredicate(binaryAttr <=> 1.b, classOf[Eq[_]], resultFun(1.b))
+
+        checkFilterPredicate(binaryAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(binaryAttr.isNotNull, classOf[NotEq[_]],
+          (1 to 4).map(i => Row.apply(resultFun(i.b))))
+
+        checkFilterPredicate(binaryAttr =!= 1.b, classOf[NotEq[_]],
+          (2 to 4).map(i => Row.apply(resultFun(i.b))))
+
+        checkFilterPredicate(binaryAttr < 2.b, classOf[Lt[_]], resultFun(1.b))
+        checkFilterPredicate(binaryAttr > 3.b, classOf[Gt[_]], resultFun(4.b))
+        checkFilterPredicate(binaryAttr <= 1.b, classOf[LtEq[_]], resultFun(1.b))
+        checkFilterPredicate(binaryAttr >= 4.b, classOf[GtEq[_]], resultFun(4.b))
+
+        checkFilterPredicate(Literal(1.b) === binaryAttr, classOf[Eq[_]], resultFun(1.b))
+        checkFilterPredicate(Literal(1.b) <=> binaryAttr, classOf[Eq[_]], resultFun(1.b))
+        checkFilterPredicate(Literal(2.b) > binaryAttr, classOf[Lt[_]], resultFun(1.b))
+        checkFilterPredicate(Literal(3.b) < binaryAttr, classOf[Gt[_]], resultFun(4.b))
+        checkFilterPredicate(Literal(1.b) >= binaryAttr, classOf[LtEq[_]], resultFun(1.b))
+        checkFilterPredicate(Literal(4.b) <= binaryAttr, classOf[GtEq[_]], resultFun(4.b))
+
+        checkFilterPredicate(!(binaryAttr < 4.b), classOf[GtEq[_]], resultFun(4.b))
+        checkFilterPredicate(binaryAttr < 2.b || binaryAttr > 3.b, classOf[Operators.Or],
+          Seq(Row(resultFun(1.b)), Row(resultFun(4.b))))
+      }
     }
   }
 
@@ -424,40 +525,53 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       def date: Date = Date.valueOf(s)
     }
 
-    val data = Seq("2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21")
-
-    withParquetDataFrame(data.map(i => Tuple1(i.date))) { implicit df =>
-      checkFilterPredicate('_1.isNull, classOf[Eq[_]], Seq.empty[Row])
-      checkFilterPredicate('_1.isNotNull, classOf[NotEq[_]], data.map(i => Row.apply(i.date)))
-
-      checkFilterPredicate('_1 === "2018-03-18".date, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 <=> "2018-03-18".date, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 =!= "2018-03-18".date, classOf[NotEq[_]],
-        Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(i.date)))
-
-      checkFilterPredicate('_1 < "2018-03-19".date, classOf[Lt[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 > "2018-03-20".date, classOf[Gt[_]], "2018-03-21".date)
-      checkFilterPredicate('_1 <= "2018-03-18".date, classOf[LtEq[_]], "2018-03-18".date)
-      checkFilterPredicate('_1 >= "2018-03-21".date, classOf[GtEq[_]], "2018-03-21".date)
-
-      checkFilterPredicate(
-        Literal("2018-03-18".date) === '_1, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-18".date) <=> '_1, classOf[Eq[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-19".date) > '_1, classOf[Lt[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-20".date) < '_1, classOf[Gt[_]], "2018-03-21".date)
-      checkFilterPredicate(
-        Literal("2018-03-18".date) >= '_1, classOf[LtEq[_]], "2018-03-18".date)
-      checkFilterPredicate(
-        Literal("2018-03-21".date) <= '_1, classOf[GtEq[_]], "2018-03-21".date)
-
-      checkFilterPredicate(!('_1 < "2018-03-21".date), classOf[GtEq[_]], "2018-03-21".date)
-      checkFilterPredicate(
-        '_1 < "2018-03-19".date || '_1 > "2018-03-20".date,
-        classOf[Operators.Or],
-        Seq(Row("2018-03-18".date), Row("2018-03-21".date)))
+    val data = Seq("2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21").map(_.date)
+    import testImplicits._
+    withNestedDataFrame(data.map(i => Tuple1(i)).toDF()) { case (inputDF, colName, resultFun) =>
+      withParquetDataFrame(inputDF) { implicit df =>
+        val dateAttr: Expression = df(colName).expr
+        assert(df(colName).expr.dataType === DateType)
+
+        checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+        checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
+          data.map(i => Row.apply(resultFun(i))))
+
+        checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
+          Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i.date))))
+
+        checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
+          resultFun("2018-03-21".date))
+        checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
+          resultFun("2018-03-21".date))
+
+        checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
+          resultFun("2018-03-21".date))
+        checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
+          resultFun("2018-03-18".date))
+        checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
+          resultFun("2018-03-21".date))
+
+        checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
+          resultFun("2018-03-21".date))
+        checkFilterPredicate(
+          dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
+          classOf[Operators.Or],
+          Seq(Row(resultFun("2018-03-18".date)), Row(resultFun("2018-03-21".date))))
+      }
     }
   }
 
@@ -485,7 +599,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.INT96.toString) {
-      withParquetDataFrame(millisData.map(i => Tuple1(i))) { implicit df =>
+      import testImplicits._
+      withParquetDataFrame(millisData.map(i => Tuple1(i)).toDF()) { implicit df =>
         val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
         assertResult(None) {
           createParquetFilters(schema).createFilter(sources.IsNull("_1"))
@@ -502,33 +617,39 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       (false, DecimalType.MAX_PRECISION) // binaryWriterUsingUnscaledBytes
     ).foreach { case (legacyFormat, precision) =>
       withSQLConf(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key -> legacyFormat.toString) {
-        val schema = StructType.fromDDL(s"a decimal($precision, 2)")
         val rdd =
           spark.sparkContext.parallelize((1 to 4).map(i => Row(new java.math.BigDecimal(i))))
-        val dataFrame = spark.createDataFrame(rdd, schema)
-        testDecimalPushDown(dataFrame) { implicit df =>
-          assert(df.schema === schema)
-          checkFilterPredicate('a.isNull, classOf[Eq[_]], Seq.empty[Row])
-          checkFilterPredicate('a.isNotNull, classOf[NotEq[_]], (1 to 4).map(Row.apply(_)))
-
-          checkFilterPredicate('a === 1, classOf[Eq[_]], 1)
-          checkFilterPredicate('a <=> 1, classOf[Eq[_]], 1)
-          checkFilterPredicate('a =!= 1, classOf[NotEq[_]], (2 to 4).map(Row.apply(_)))
-
-          checkFilterPredicate('a < 2, classOf[Lt[_]], 1)
-          checkFilterPredicate('a > 3, classOf[Gt[_]], 4)
-          checkFilterPredicate('a <= 1, classOf[LtEq[_]], 1)
-          checkFilterPredicate('a >= 4, classOf[GtEq[_]], 4)
-
-          checkFilterPredicate(Literal(1) === 'a, classOf[Eq[_]], 1)
-          checkFilterPredicate(Literal(1) <=> 'a, classOf[Eq[_]], 1)
-          checkFilterPredicate(Literal(2) > 'a, classOf[Lt[_]], 1)
-          checkFilterPredicate(Literal(3) < 'a, classOf[Gt[_]], 4)
-          checkFilterPredicate(Literal(1) >= 'a, classOf[LtEq[_]], 1)
-          checkFilterPredicate(Literal(4) <= 'a, classOf[GtEq[_]], 4)
-
-          checkFilterPredicate(!('a < 4), classOf[GtEq[_]], 4)
-          checkFilterPredicate('a < 2 || 'a > 3, classOf[Operators.Or], Seq(Row(1), Row(4)))
+        val dataFrame = spark.createDataFrame(rdd, StructType.fromDDL(s"a decimal($precision, 2)"))
+        withNestedDataFrame(dataFrame) { case (inputDF, colName, resultFun) =>
+          withParquetDataFrame(inputDF) { implicit df =>
+            val decimalAttr: Expression = df(colName).expr
+            assert(df(colName).expr.dataType === DecimalType(precision, 2))
+
+            checkFilterPredicate(decimalAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+            checkFilterPredicate(decimalAttr.isNotNull, classOf[NotEq[_]],
+              (1 to 4).map(i => Row.apply(resultFun(i))))
+
+            checkFilterPredicate(decimalAttr === 1, classOf[Eq[_]], resultFun(1))
+            checkFilterPredicate(decimalAttr <=> 1, classOf[Eq[_]], resultFun(1))
+            checkFilterPredicate(decimalAttr =!= 1, classOf[NotEq[_]],
+              (2 to 4).map(i => Row.apply(resultFun(i))))
+
+            checkFilterPredicate(decimalAttr < 2, classOf[Lt[_]], resultFun(1))
+            checkFilterPredicate(decimalAttr > 3, classOf[Gt[_]], resultFun(4))
+            checkFilterPredicate(decimalAttr <= 1, classOf[LtEq[_]], resultFun(1))
+            checkFilterPredicate(decimalAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+            checkFilterPredicate(Literal(1) === decimalAttr, classOf[Eq[_]], resultFun(1))
+            checkFilterPredicate(Literal(1) <=> decimalAttr, classOf[Eq[_]], resultFun(1))
+            checkFilterPredicate(Literal(2) > decimalAttr, classOf[Lt[_]], resultFun(1))
+            checkFilterPredicate(Literal(3) < decimalAttr, classOf[Gt[_]], resultFun(4))
+            checkFilterPredicate(Literal(1) >= decimalAttr, classOf[LtEq[_]], resultFun(1))
+            checkFilterPredicate(Literal(4) <= decimalAttr, classOf[GtEq[_]], resultFun(4))
+
+            checkFilterPredicate(!(decimalAttr < 4), classOf[GtEq[_]], resultFun(4))
+            checkFilterPredicate(decimalAttr < 2 || decimalAttr > 3, classOf[Operators.Or],
+              Seq(Row(resultFun(1)), Row(resultFun(4))))
+          }
         }
       }
     }
@@ -1042,7 +1163,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("SPARK-16371 Do not push down filters when inner name and outer name are the same") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Tuple1(i)))) { implicit df =>
+    import testImplicits._
+    withParquetDataFrame((1 to 4).map(i => Tuple1(Tuple1(i))).toDF()) { implicit df =>
       // Here the schema becomes as below:
       //
       // root
@@ -1107,7 +1229,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
   }
 
-  test("SPARK-20364: Disable Parquet predicate pushdown for fields having dots in the names") {
+  test("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") {
     import testImplicits._
 
     Seq(true, false).foreach { vectorized =>
@@ -1120,6 +1242,28 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
           assert(readBack.count() == 1)
         }
       }
+
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString,
+          // Makes sure disabling 'spark.sql.parquet.recordFilter' still enables
+          // row group level filtering.
+          SQLConf.PARQUET_RECORD_FILTER_ENABLED.key -> "false",
+          SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true") {
+
+        withTempPath { path =>
+          val data = (1 to 1024)
+          data.toDF("col.dots").coalesce(1)
+            .write.option("parquet.block.size", 512)
+            .parquet(path.getAbsolutePath)
+          val df = spark.read.parquet(path.getAbsolutePath).filter("`col.dots` == 500")
+          // Here, we strip the Spark side filter and check the actual results from Parquet.
+          val actual = stripSparkFilter(df).collect().length
+          // Since those are filtered at row group level, the result count should be less
+          // than the total length but should not be a single record.
+          // Note that, if record level filtering is enabled, it should be a single record.
+          // If no filter is pushed down to Parquet, it should be the total length of data.
+          assert(actual > 1 && actual < data.length)
+        }
+      }
     }
   }
 
@@ -1162,7 +1306,10 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("filter pushdown - StringStartsWith") {
-    withParquetDataFrame((1 to 4).map(i => Tuple1(i + "str" + i))) { implicit df =>
+    withParquetDataFrame {
+      import testImplicits._
+      (1 to 4).map(i => Tuple1(i + "str" + i)).toDF()
+    } { implicit df =>
       checkFilterPredicate(
         '_1.startsWith("").asInstanceOf[Predicate],
         classOf[UserDefinedByInstance[_, _]],
@@ -1208,7 +1355,10 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
 
     // SPARK-28371: make sure filter is null-safe.
-    withParquetDataFrame(Seq(Tuple1[String](null))) { implicit df =>
+    withParquetDataFrame {
+      import testImplicits._
+      Seq(Tuple1[String](null)).toDF()
+    } { implicit df =>
       checkFilterPredicate(
         '_1.startsWith("blah").asInstanceOf[Predicate],
         classOf[UserDefinedByInstance[_, _]],
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 7f85fd2a1629a..497b823868450 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -82,7 +82,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
    * Writes `data` to a Parquet file, reads it back and check file contents.
    */
   protected def checkParquetFile[T <: Product : ClassTag: TypeTag](data: Seq[T]): Unit = {
-    withParquetDataFrame(data)(r => checkAnswer(r, data.map(Row.fromTuple)))
+    withParquetDataFrame(data.toDF())(r => checkAnswer(r, data.map(Row.fromTuple)))
   }
 
   test("basic data types (without binary)") {
@@ -94,7 +94,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   test("raw binary") {
     val data = (1 to 4).map(i => Tuple1(Array.fill(3)(i.toByte)))
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       assertResult(data.map(_._1.mkString(",")).sorted) {
         df.collect().map(_.getAs[Array[Byte]](0).mkString(",")).sorted
       }
@@ -197,7 +197,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   testStandardAndLegacyModes("struct") {
     val data = (1 to 4).map(i => Tuple1((i, s"val_$i")))
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(struct) =>
         Row(Row(struct.productIterator.toSeq: _*))
@@ -214,7 +214,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(array) =>
         Row(array.map(struct => Row(struct.productIterator.toSeq: _*)))
@@ -233,7 +233,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(array) =>
         Row(array.map { case Tuple1(Tuple1(str)) => Row(Row(str))})
@@ -243,7 +243,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   testStandardAndLegacyModes("nested struct with array of array as field") {
     val data = (1 to 4).map(i => Tuple1((i, Seq(Seq(s"val_$i")))))
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(struct) =>
         Row(Row(struct.productIterator.toSeq: _*))
@@ -260,7 +260,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(m) =>
         Row(m.map { case (k, v) => Row(k.productIterator.toSeq: _*) -> v })
@@ -277,7 +277,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data) { df =>
+    withParquetDataFrame(data.toDF()) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(m) =>
         Row(m.mapValues(struct => Row(struct.productIterator.toSeq: _*)))
@@ -293,7 +293,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       null.asInstanceOf[java.lang.Float],
       null.asInstanceOf[java.lang.Double])
 
-    withParquetDataFrame(allNulls :: Nil) { df =>
+    withParquetDataFrame((allNulls :: Nil).toDF()) { df =>
       val rows = df.collect()
       assert(rows.length === 1)
       assert(rows.head === Row(Seq.fill(5)(null): _*))
@@ -306,7 +306,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       None.asInstanceOf[Option[Long]],
       None.asInstanceOf[Option[String]])
 
-    withParquetDataFrame(allNones :: Nil) { df =>
+    withParquetDataFrame((allNones :: Nil).toDF()) { df =>
       val rows = df.collect()
       assert(rows.length === 1)
       assert(rows.head === Row(Seq.fill(3)(null): _*))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index 828ba6aee026b..f2dbc536ac566 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -63,12 +63,16 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
       (f: String => Unit): Unit = withDataSourceFile(data)(f)
 
   /**
-   * Writes `data` to a Parquet file and reads it back as a [[DataFrame]],
+   * Writes `df` dataframe to a Parquet file and reads it back as a [[DataFrame]],
    * which is then passed to `f`. The Parquet file will be deleted after `f` returns.
    */
-  protected def withParquetDataFrame[T <: Product: ClassTag: TypeTag]
-      (data: Seq[T], testVectorized: Boolean = true)
-      (f: DataFrame => Unit): Unit = withDataSourceDataFrame(data, testVectorized)(f)
+  protected def withParquetDataFrame(df: DataFrame, testVectorized: Boolean = true)
+      (f: DataFrame => Unit): Unit = {
+    withTempPath { file =>
+      df.write.format(dataSourceName).save(file.getCanonicalPath)
+      readFile(file.getCanonicalPath, testVectorized)(f)
+    }
+  }
 
   /**
    * Writes `data` to a Parquet file, reads it back as a [[DataFrame]] and registers it as a
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala
index 1cb7a2156c3d3..33b2db57d9f0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/FiltersSuite.scala
@@ -24,66 +24,143 @@ import org.apache.spark.SparkFunSuite
  */
 class FiltersSuite extends SparkFunSuite {
 
-  test("EqualTo references") {
-    assert(EqualTo("a", "1").references.toSeq == Seq("a"))
-    assert(EqualTo("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
+  private def withFieldNames(f: (String, Array[String]) => Unit): Unit = {
+    Seq(("a", Array("a")),
+      ("a.b", Array("a", "b")),
+      ("`a.b`.c", Array("a.b", "c")),
+      ("`a.b`.`c.d`.`e.f`", Array("a.b", "c.d", "e.f"))
+    ).foreach { case (name, fieldNames) =>
+      f(name, fieldNames)
+    }
   }
 
-  test("EqualNullSafe references") {
-    assert(EqualNullSafe("a", "1").references.toSeq == Seq("a"))
-    assert(EqualNullSafe("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+  test("EqualTo references") { withFieldNames { (name, fieldNames) =>
+    assert(EqualTo(name, "1").references.toSeq == Seq(name))
+    assert(EqualTo(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("GreaterThan references") {
-    assert(GreaterThan("a", "1").references.toSeq == Seq("a"))
-    assert(GreaterThan("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualTo(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(EqualTo("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("GreaterThanOrEqual references") {
-    assert(GreaterThanOrEqual("a", "1").references.toSeq == Seq("a"))
-    assert(GreaterThanOrEqual("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualTo(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(EqualTo("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("LessThan references") {
-    assert(LessThan("a", "1").references.toSeq == Seq("a"))
-    assert(LessThan("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+  test("EqualNullSafe references") { withFieldNames { (name, fieldNames) =>
+    assert(EqualNullSafe(name, "1").references.toSeq == Seq(name))
+    assert(EqualNullSafe(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("LessThanOrEqual references") {
-    assert(LessThanOrEqual("a", "1").references.toSeq == Seq("a"))
-    assert(LessThanOrEqual("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualNullSafe(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(EqualNullSafe("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("In references") {
-    assert(In("a", Array("1")).references.toSeq == Seq("a"))
-    assert(In("a", Array("1", EqualTo("b", "2"))).references.toSeq == Seq("a", "b"))
-  }
+    assert(EqualNullSafe(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(EqualNullSafe("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("IsNull references") {
-    assert(IsNull("a").references.toSeq == Seq("a"))
-  }
+  test("GreaterThan references") { withFieldNames { (name, fieldNames) =>
+    assert(GreaterThan(name, "1").references.toSeq == Seq(name))
+    assert(GreaterThan(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("IsNotNull references") {
-    assert(IsNotNull("a").references.toSeq == Seq("a"))
-  }
+    assert(GreaterThan(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(GreaterThan("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("And references") {
-    assert(And(EqualTo("a", "1"), EqualTo("b", "1")).references.toSeq == Seq("a", "b"))
-  }
+    assert(GreaterThan(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(GreaterThan("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("Or references") {
-    assert(Or(EqualTo("a", "1"), EqualTo("b", "1")).references.toSeq == Seq("a", "b"))
-  }
+  test("GreaterThanOrEqual references") { withFieldNames { (name, fieldNames) =>
+    assert(GreaterThanOrEqual(name, "1").references.toSeq == Seq(name))
+    assert(GreaterThanOrEqual(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
 
-  test("StringStartsWith references") {
-    assert(StringStartsWith("a", "str").references.toSeq == Seq("a"))
-  }
+    assert(GreaterThanOrEqual(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(GreaterThanOrEqual("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
 
-  test("StringEndsWith references") {
-    assert(StringEndsWith("a", "str").references.toSeq == Seq("a"))
-  }
+    assert(GreaterThanOrEqual(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(GreaterThanOrEqual("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
 
-  test("StringContains references") {
-    assert(StringContains("a", "str").references.toSeq == Seq("a"))
-  }
+  test("LessThan references") { withFieldNames { (name, fieldNames) =>
+    assert(LessThan(name, "1").references.toSeq == Seq(name))
+    assert(LessThan(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+
+    assert(LessThan("a", EqualTo("b", "2")).references.toSeq == Seq("a", "b"))
+  }}
+
+  test("LessThanOrEqual references") { withFieldNames { (name, fieldNames) =>
+    assert(LessThanOrEqual(name, "1").references.toSeq == Seq(name))
+    assert(LessThanOrEqual(name, "1").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+
+    assert(LessThanOrEqual(name, EqualTo("b", "2")).references.toSeq == Seq(name, "b"))
+    assert(LessThanOrEqual("b", EqualTo(name, "2")).references.toSeq == Seq("b", name))
+
+    assert(LessThanOrEqual(name, EqualTo("b", "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(LessThanOrEqual("b", EqualTo(name, "2")).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("In references") { withFieldNames { (name, fieldNames) =>
+    assert(In(name, Array("1")).references.toSeq == Seq(name))
+    assert(In(name, Array("1")).v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+
+    assert(In(name, Array("1", EqualTo("b", "2"))).references.toSeq == Seq(name, "b"))
+    assert(In("b", Array("1", EqualTo(name, "2"))).references.toSeq == Seq("b", name))
+
+    assert(In(name, Array("1", EqualTo("b", "2"))).v2references.toSeq.map(_.toSeq)
+      == Seq(fieldNames.toSeq, Seq("b")))
+    assert(In("b", Array("1", EqualTo(name, "2"))).v2references.toSeq.map(_.toSeq)
+      == Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("IsNull references") { withFieldNames { (name, fieldNames) =>
+    assert(IsNull(name).references.toSeq == Seq(name))
+    assert(IsNull(name).v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("IsNotNull references") { withFieldNames { (name, fieldNames) =>
+    assert(IsNotNull(name).references.toSeq == Seq(name))
+    assert(IsNull(name).v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("And references") { withFieldNames { (name, fieldNames) =>
+    assert(And(EqualTo(name, "1"), EqualTo("b", "1")).references.toSeq == Seq(name, "b"))
+    assert(And(EqualTo("b", "1"), EqualTo(name, "1")).references.toSeq == Seq("b", name))
+
+    assert(And(EqualTo(name, "1"), EqualTo("b", "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(fieldNames.toSeq, Seq("b")))
+    assert(And(EqualTo("b", "1"), EqualTo(name, "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("Or references") { withFieldNames { (name, fieldNames) =>
+    assert(Or(EqualTo(name, "1"), EqualTo("b", "1")).references.toSeq == Seq(name, "b"))
+    assert(Or(EqualTo("b", "1"), EqualTo(name, "1")).references.toSeq == Seq("b", name))
+
+    assert(Or(EqualTo(name, "1"), EqualTo("b", "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(fieldNames.toSeq, Seq("b")))
+    assert(Or(EqualTo("b", "1"), EqualTo(name, "1")).v2references.toSeq.map(_.toSeq) ==
+      Seq(Seq("b"), fieldNames.toSeq))
+  }}
+
+  test("StringStartsWith references") { withFieldNames { (name, fieldNames) =>
+    assert(StringStartsWith(name, "str").references.toSeq == Seq(name))
+    assert(StringStartsWith(name, "str").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("StringEndsWith references") { withFieldNames { (name, fieldNames) =>
+    assert(StringEndsWith(name, "str").references.toSeq == Seq(name))
+    assert(StringEndsWith(name, "str").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
+
+  test("StringContains references") { withFieldNames { (name, fieldNames) =>
+    assert(StringContains(name, "str").references.toSeq == Seq(name))
+    assert(StringContains(name, "str").v2references.toSeq.map(_.toSeq) == Seq(fieldNames.toSeq))
+  }}
 }
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index b9cbc484e1fc1..f5abd30854e00 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -65,9 +65,11 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * Create ORC filter as a SearchArgument instance.
    */
   def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
+    val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
     // Combines all convertible filters using `And` to produce a single conjunction
-    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, filters))
+    // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+    val newFilters = filters.filter(!_.containsNestedColumn)
+    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
     conjunctionOptional.map { conjunction =>
       // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
       // The input predicate is fully convertible. There should not be any empty result in the
@@ -222,48 +224,39 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
     // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
+      case EqualTo(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().equals(name, getType(name), castedValue).end())
 
-      case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
+      case EqualNullSafe(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().nullSafeEquals(name, getType(name), castedValue).end())
 
-      case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
+      case LessThan(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().lessThan(name, getType(name), castedValue).end())
 
-      case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case LessThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().lessThanEquals(name, getType(name), castedValue).end())
 
-      case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case GreaterThan(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startNot().lessThanEquals(name, getType(name), castedValue).end())
 
-      case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
+      case GreaterThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startNot().lessThan(name, getType(name), castedValue).end())
 
-      case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
+      case IsNull(name) if isSearchableType(dataTypeMap(name)) =>
+        Some(builder.startAnd().isNull(name, getType(name)).end())
 
-      case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
+      case IsNotNull(name) if isSearchableType(dataTypeMap(name)) =>
+        Some(builder.startNot().isNull(name, getType(name)).end())
 
-      case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
-        Some(builder.startAnd().in(quotedName, getType(attribute),
+      case In(name, values) if isSearchableType(dataTypeMap(name)) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name)))
+        Some(builder.startAnd().in(name, getType(name),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
 
       case _ => None
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 6e9e592be13be..675e089153679 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -65,9 +65,11 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * Create ORC filter as a SearchArgument instance.
    */
   def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
+    val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
     // Combines all convertible filters using `And` to produce a single conjunction
-    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, filters))
+    // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+    val newFilters = filters.filter(!_.containsNestedColumn)
+    val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
     conjunctionOptional.map { conjunction =>
       // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
       // The input predicate is fully convertible. There should not be any empty result in the
@@ -222,48 +224,39 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
     // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
+      case EqualTo(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().equals(name, getType(name), castedValue).end())
 
-      case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
+      case EqualNullSafe(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().nullSafeEquals(name, getType(name), castedValue).end())
 
-      case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
+      case LessThan(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().lessThan(name, getType(name), castedValue).end())
 
-      case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case LessThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startAnd().lessThanEquals(name, getType(name), castedValue).end())
 
-      case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
+      case GreaterThan(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startNot().lessThanEquals(name, getType(name), castedValue).end())
 
-      case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValue = castLiteralValue(value, dataTypeMap(attribute))
-        Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
+      case GreaterThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name))
+        Some(builder.startNot().lessThan(name, getType(name), castedValue).end())
 
-      case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
+      case IsNull(name) if isSearchableType(dataTypeMap(name)) =>
+        Some(builder.startAnd().isNull(name, getType(name)).end())
 
-      case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
+      case IsNotNull(name) if isSearchableType(dataTypeMap(name)) =>
+        Some(builder.startNot().isNull(name, getType(name)).end())
 
-      case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
-        val quotedName = quoteIfNeeded(attribute)
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
-        Some(builder.startAnd().in(quotedName, getType(attribute),
+      case In(name, values) if isSearchableType(dataTypeMap(name)) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name)))
+        Some(builder.startAnd().in(name, getType(name),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
 
       case _ => None
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
index cd1bffb6b7ab7..f9c514567c639 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFilters.scala
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 
 import org.apache.spark.SparkException
 import org.apache.spark.internal.Logging
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.execution.datasources.orc.{OrcFilters => DatasourceOrcFilters}
 import org.apache.spark.sql.execution.datasources.orc.OrcFilters.buildTree
 import org.apache.spark.sql.hive.HiveUtils
@@ -73,9 +74,11 @@ private[orc] object OrcFilters extends Logging {
     if (HiveUtils.isHive23) {
       DatasourceOrcFilters.createFilter(schema, filters).asInstanceOf[Option[SearchArgument]]
     } else {
-      val dataTypeMap = schema.map(f => f.name -> f.dataType).toMap
+      val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
+      // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
+      val newFilters = filters.filter(!_.containsNestedColumn)
       // Combines all convertible filters using `And` to produce a single conjunction
-      val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, filters))
+      val conjunctionOptional = buildTree(convertibleFilters(schema, dataTypeMap, newFilters))
       conjunctionOptional.map { conjunction =>
         // Then tries to build a single ORC `SearchArgument` for the conjunction predicate.
         // The input predicate is fully convertible. There should not be any empty result in the

From ebc358c8d2b6d67c7319be006452c9c993b7a098 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Thu, 26 Mar 2020 23:49:24 -0700
Subject: [PATCH 0361/1280] [SPARK-31086][SQL] Add Back the Deprecated
 SQLContext methods

### What changes were proposed in this pull request?

Based on the discussion in the mailing list [[Proposal] Modification to Spark's Semantic Versioning Policy](http://apache-spark-developers-list.1001551.n3.nabble.com/Proposal-Modification-to-Spark-s-Semantic-Versioning-Policy-td28938.html) , this PR is to add back the following APIs whose maintenance cost are relatively small.

- SQLContext.applySchema
- SQLContext.parquetFile
- SQLContext.jsonFile
- SQLContext.jsonRDD
- SQLContext.load
- SQLContext.jdbc

### Why are the changes needed?
Avoid breaking the APIs that are commonly used.

### Does this PR introduce any user-facing change?
Adding back the APIs that were removed in 3.0 branch does not introduce the user-facing changes, because Spark 3.0 has not been released.

### How was this patch tested?
The existing tests.

Closes #27839 from gatorsmile/addAPIBackV3.

Lead-authored-by: gatorsmile <gatorsmile@gmail.com>
Co-authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit b7e4cc775b7eac68606d1f385911613f5139db1b)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../org/apache/spark/sql/SQLContext.scala     | 283 ++++++++++++++++++
 .../apache/spark/sql/DeprecatedAPISuite.scala | 106 +++++++
 .../org/apache/spark/sql/jdbc/JDBCSuite.scala |  14 +
 3 files changed, 403 insertions(+)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 2054874e5e07b..592c64c93c8a2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -611,6 +611,289 @@ class SQLContext private[sql](val sparkSession: SparkSession)
     sessionState.catalog.listTables(databaseName).map(_.table).toArray
   }
 
+  ////////////////////////////////////////////////////////////////////////////
+  ////////////////////////////////////////////////////////////////////////////
+  // Deprecated methods
+  ////////////////////////////////////////////////////////////////////////////
+  ////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rowRDD: RDD[Row], schema: StructType): DataFrame = {
+    createDataFrame(rowRDD, schema)
+  }
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rowRDD: JavaRDD[Row], schema: StructType): DataFrame = {
+    createDataFrame(rowRDD, schema)
+  }
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rdd: RDD[_], beanClass: Class[_]): DataFrame = {
+    createDataFrame(rdd, beanClass)
+  }
+
+  /**
+   * @deprecated As of 1.3.0, replaced by `createDataFrame()`.
+   */
+  @deprecated("Use createDataFrame instead.", "1.3.0")
+  def applySchema(rdd: JavaRDD[_], beanClass: Class[_]): DataFrame = {
+    createDataFrame(rdd, beanClass)
+  }
+
+  /**
+   * Loads a Parquet file, returning the result as a `DataFrame`. This function returns an empty
+   * `DataFrame` if no paths are passed in.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().parquet()`.
+   */
+  @deprecated("Use read.parquet() instead.", "1.4.0")
+  @scala.annotation.varargs
+  def parquetFile(paths: String*): DataFrame = {
+    if (paths.isEmpty) {
+      emptyDataFrame
+    } else {
+      read.parquet(paths : _*)
+    }
+  }
+
+  /**
+   * Loads a JSON file (one object per line), returning the result as a `DataFrame`.
+   * It goes through the entire dataset once to determine the schema.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonFile(path: String): DataFrame = {
+    read.json(path)
+  }
+
+  /**
+   * Loads a JSON file (one object per line) and applies the given schema,
+   * returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonFile(path: String, schema: StructType): DataFrame = {
+    read.schema(schema).json(path)
+  }
+
+  /**
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonFile(path: String, samplingRatio: Double): DataFrame = {
+    read.option("samplingRatio", samplingRatio.toString).json(path)
+  }
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record), returning the result as a
+   * `DataFrame`.
+   * It goes through the entire dataset once to determine the schema.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: RDD[String]): DataFrame = read.json(json)
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record), returning the result as a
+   * `DataFrame`.
+   * It goes through the entire dataset once to determine the schema.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: JavaRDD[String]): DataFrame = read.json(json)
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record) and applies the given schema,
+   * returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: RDD[String], schema: StructType): DataFrame = {
+    read.schema(schema).json(json)
+  }
+
+  /**
+   * Loads an JavaRDD[String] storing JSON objects (one object per record) and applies the given
+   * schema, returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: JavaRDD[String], schema: StructType): DataFrame = {
+    read.schema(schema).json(json)
+  }
+
+  /**
+   * Loads an RDD[String] storing JSON objects (one object per record) inferring the
+   * schema, returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: RDD[String], samplingRatio: Double): DataFrame = {
+    read.option("samplingRatio", samplingRatio.toString).json(json)
+  }
+
+  /**
+   * Loads a JavaRDD[String] storing JSON objects (one object per record) inferring the
+   * schema, returning the result as a `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().json()`.
+   */
+  @deprecated("Use read.json() instead.", "1.4.0")
+  def jsonRDD(json: JavaRDD[String], samplingRatio: Double): DataFrame = {
+    read.option("samplingRatio", samplingRatio.toString).json(json)
+  }
+
+  /**
+   * Returns the dataset stored at path as a DataFrame,
+   * using the default data source configured by spark.sql.sources.default.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().load(path)`.
+   */
+  @deprecated("Use read.load(path) instead.", "1.4.0")
+  def load(path: String): DataFrame = {
+    read.load(path)
+  }
+
+  /**
+   * Returns the dataset stored at path as a DataFrame, using the given data source.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().format(source).load(path)`.
+   */
+  @deprecated("Use read.format(source).load(path) instead.", "1.4.0")
+  def load(path: String, source: String): DataFrame = {
+    read.format(source).load(path)
+  }
+
+  /**
+   * (Java-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().format(source).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).options(options).load() instead.", "1.4.0")
+  def load(source: String, options: java.util.Map[String, String]): DataFrame = {
+    read.options(options).format(source).load()
+  }
+
+  /**
+   * (Scala-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by `read().format(source).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).options(options).load() instead.", "1.4.0")
+  def load(source: String, options: Map[String, String]): DataFrame = {
+    read.options(options).format(source).load()
+  }
+
+  /**
+   * (Java-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame, using the given schema as the schema of the DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by
+   *            `read().format(source).schema(schema).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).schema(schema).options(options).load() instead.", "1.4.0")
+  def load(
+      source: String,
+      schema: StructType,
+      options: java.util.Map[String, String]): DataFrame = {
+    read.format(source).schema(schema).options(options).load()
+  }
+
+  /**
+   * (Scala-specific) Returns the dataset specified by the given data source and
+   * a set of options as a DataFrame, using the given schema as the schema of the DataFrame.
+   *
+   * @group genericdata
+   * @deprecated As of 1.4.0, replaced by
+   *            `read().format(source).schema(schema).options(options).load()`.
+   */
+  @deprecated("Use read.format(source).schema(schema).options(options).load() instead.", "1.4.0")
+  def load(source: String, schema: StructType, options: Map[String, String]): DataFrame = {
+    read.format(source).schema(schema).options(options).load()
+  }
+
+  /**
+   * Construct a `DataFrame` representing the database table accessible via JDBC URL
+   * url named table.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().jdbc()`.
+   */
+  @deprecated("Use read.jdbc() instead.", "1.4.0")
+  def jdbc(url: String, table: String): DataFrame = {
+    read.jdbc(url, table, new Properties)
+  }
+
+  /**
+   * Construct a `DataFrame` representing the database table accessible via JDBC URL
+   * url named table.  Partitions of the table will be retrieved in parallel based on the parameters
+   * passed to this function.
+   *
+   * @param columnName the name of a column of integral type that will be used for partitioning.
+   * @param lowerBound the minimum value of `columnName` used to decide partition stride
+   * @param upperBound the maximum value of `columnName` used to decide partition stride
+   * @param numPartitions the number of partitions.  the range `minValue`-`maxValue` will be split
+   *                      evenly into this many partitions
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().jdbc()`.
+   */
+  @deprecated("Use read.jdbc() instead.", "1.4.0")
+  def jdbc(
+      url: String,
+      table: String,
+      columnName: String,
+      lowerBound: Long,
+      upperBound: Long,
+      numPartitions: Int): DataFrame = {
+    read.jdbc(url, table, columnName, lowerBound, upperBound, numPartitions, new Properties)
+  }
+
+  /**
+   * Construct a `DataFrame` representing the database table accessible via JDBC URL
+   * url named table. The theParts parameter gives a list expressions
+   * suitable for inclusion in WHERE clauses; each one defines one partition
+   * of the `DataFrame`.
+   *
+   * @group specificdata
+   * @deprecated As of 1.4.0, replaced by `read().jdbc()`.
+   */
+  @deprecated("Use read.jdbc() instead.", "1.4.0")
+  def jdbc(url: String, table: String, theParts: Array[String]): DataFrame = {
+    read.jdbc(url, table, theParts, new Properties)
+  }
 }
 
 /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
new file mode 100644
index 0000000000000..c31ef996f0362
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+
+class DeprecatedAPISuite extends QueryTest with SharedSparkSession {
+
+  test("SQLContext.applySchema") {
+    val rowRdd = sparkContext.parallelize(Seq(Row("Jack", 20), Row("Marry", 18)))
+    val schema = StructType(StructField("name", StringType, false) ::
+      StructField("age", IntegerType, true) :: Nil)
+    val sqlContext = spark.sqlContext
+    checkAnswer(sqlContext.applySchema(rowRdd, schema), Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+    checkAnswer(sqlContext.applySchema(rowRdd.toJavaRDD(), schema),
+      Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+  }
+
+  test("SQLContext.parquetFile") {
+    val sqlContext = spark.sqlContext
+    withTempDir { dir =>
+      val parquetFile = s"${dir.toString}/${System.currentTimeMillis()}"
+      val expectDF = spark.range(10).toDF()
+      expectDF.write.parquet(parquetFile)
+      val parquetDF = sqlContext.parquetFile(parquetFile)
+      checkAnswer(parquetDF, expectDF)
+    }
+  }
+
+  test("SQLContext.jsonFile") {
+    val sqlContext = spark.sqlContext
+    withTempDir { dir =>
+      val jsonFile = s"${dir.toString}/${System.currentTimeMillis()}"
+      val expectDF = spark.range(10).toDF()
+      expectDF.write.json(jsonFile)
+      var jsonDF = sqlContext.jsonFile(jsonFile)
+      checkAnswer(jsonDF, expectDF)
+      assert(jsonDF.schema === expectDF.schema.asNullable)
+
+      var schema = expectDF.schema
+      jsonDF = sqlContext.jsonFile(jsonFile, schema)
+      checkAnswer(jsonDF, expectDF)
+      assert(jsonDF.schema === schema.asNullable)
+
+      jsonDF = sqlContext.jsonFile(jsonFile, 0.9)
+      checkAnswer(jsonDF, expectDF)
+
+      val jsonRDD = sparkContext.parallelize(Seq("{\"name\":\"Jack\",\"age\":20}",
+        "{\"name\":\"Marry\",\"age\":18}"))
+      jsonDF = sqlContext.jsonRDD(jsonRDD)
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD.toJavaRDD())
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+
+      schema = StructType(StructField("name", StringType, false) ::
+        StructField("age", IntegerType, false) :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD, schema)
+      checkAnswer(jsonDF, Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD.toJavaRDD(), schema)
+      checkAnswer(jsonDF, Row("Jack", 20) :: Row("Marry", 18) :: Nil)
+
+
+      jsonDF = sqlContext.jsonRDD(jsonRDD, 0.9)
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+      jsonDF = sqlContext.jsonRDD(jsonRDD.toJavaRDD(), 0.9)
+      checkAnswer(jsonDF, Row(18, "Marry") :: Row(20, "Jack") :: Nil)
+    }
+  }
+
+  test("SQLContext.load") {
+    withTempDir { dir =>
+      val path = s"${dir.toString}/${System.currentTimeMillis()}"
+      val expectDF = spark.range(10).toDF()
+      expectDF.write.parquet(path)
+      val sqlContext = spark.sqlContext
+
+      var loadDF = sqlContext.load(path)
+      checkAnswer(loadDF, expectDF)
+
+      loadDF = sqlContext.load(path, "parquet")
+      checkAnswer(loadDF, expectDF)
+
+      loadDF = sqlContext.load("parquet", Map("path" -> path))
+      checkAnswer(loadDF, expectDF)
+
+      loadDF = sqlContext.load("parquet", expectDF.schema, Map("path" -> path))
+      checkAnswer(loadDF, expectDF)
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 9cba95f7d7df2..fd691f547eb05 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1699,4 +1699,18 @@ class JDBCSuite extends QueryTest
     assert(JdbcDialects.get("jdbc:teradata://localhost/db") === TeradataDialect)
     assert(JdbcDialects.get("jdbc:Teradata://localhost/db") === TeradataDialect)
   }
+
+  test("SQLContext.jdbc (deprecated)") {
+    val sqlContext = spark.sqlContext
+    var jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE")
+    checkAnswer(jdbcDF, Row("fred", 1) :: Row("mary", 2) :: Row ("joe 'foo' \"bar\"", 3) :: Nil)
+
+    jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", "THEID", 0, 4, 3)
+    checkNumPartitions(jdbcDF, 3)
+    checkAnswer(jdbcDF, Row("fred", 1) :: Row("mary", 2) :: Row ("joe 'foo' \"bar\"", 3) :: Nil)
+
+    val parts = Array[String]("THEID = 2")
+    jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts)
+    checkAnswer(jdbcDF, Row("mary", 2) :: Nil)
+  }
 }

From 2a449df305d5f8495959fd71d937e0f5f4fff87d Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Thu, 26 Mar 2020 23:51:15 -0700
Subject: [PATCH 0362/1280] [SPARK-31088][SQL] Add back HiveContext and
 createExternalTable

### What changes were proposed in this pull request?
Based on the discussion in the mailing list [[Proposal] Modification to Spark's Semantic Versioning Policy](http://apache-spark-developers-list.1001551.n3.nabble.com/Proposal-Modification-to-Spark-s-Semantic-Versioning-Policy-td28938.html) , this PR is to add back the following APIs whose maintenance cost are relatively small.

- HiveContext
- createExternalTable APIs

### Why are the changes needed?

Avoid breaking the APIs that are commonly used.

### Does this PR introduce any user-facing change?
Adding back the APIs that were removed in 3.0 branch does not introduce the user-facing changes, because Spark 3.0 has not been released.

### How was this patch tested?

add a new test suite for createExternalTable APIs.

Closes #27815 from gatorsmile/addAPIsBack.

Lead-authored-by: gatorsmile <gatorsmile@gmail.com>
Co-authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit b9eafcb52658b7f5ec60bb4ebcc9da0fde94e105)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 docs/sql-migration-guide.md                   |   4 -
 project/MimaExcludes.scala                    |   2 -
 python/pyspark/__init__.py                    |   2 +-
 python/pyspark/sql/__init__.py                |   4 +-
 python/pyspark/sql/catalog.py                 |  20 ++++
 python/pyspark/sql/context.py                 |  67 +++++++++++-
 .../org/apache/spark/sql/SQLContext.scala     |  91 ++++++++++++++++
 .../apache/spark/sql/catalog/Catalog.scala    | 102 ++++++++++++++++-
 .../DeprecatedCreateExternalTableSuite.scala  |  85 +++++++++++++++
 .../apache/spark/sql/hive/HiveContext.scala   |  63 +++++++++++
 .../hive/HiveContextCompatibilitySuite.scala  | 103 ++++++++++++++++++
 11 files changed, 532 insertions(+), 11 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala
 create mode 100644 sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
 create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index d2773d822e802..ab35e1fc0a005 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -309,10 +309,6 @@ license: |
 
 ### Others
 
-  - In Spark 3.0, the deprecated methods `SQLContext.createExternalTable` and `SparkSession.createExternalTable` have been removed in favor of its replacement, `createTable`.
-
-  - In Spark 3.0, the deprecated `HiveContext` class has been removed. Use `SparkSession.builder.enableHiveSupport()` instead.
-
   - In Spark version 2.4, when a spark session is created via `cloneSession()`, the newly created spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent spark session. Since Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. The old behavior can be restored by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
 
   - Since Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it will fallback to hive-site.xml present in the `Hadoop configuration` of `SparkContext`.
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index f8ad60bfac5db..9a5029e1c1a68 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -48,8 +48,6 @@ object MimaExcludes {
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.ExecutorPlugin"),
 
     // [SPARK-28980][SQL][CORE][MLLIB] Remove more old deprecated items in Spark 3
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.createExternalTable"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.createExternalTable"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeans.train"),
     ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.mllib.clustering.KMeans.train"),
     ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.mllib.classification.LogisticRegressionWithSGD$"),
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 76a5bd0d645ba..70c0b27a6aa33 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -113,7 +113,7 @@ def wrapper(self, *args, **kwargs):
 
 
 # for back compatibility
-from pyspark.sql import SQLContext, Row
+from pyspark.sql import SQLContext, HiveContext, Row
 
 __all__ = [
     "SparkConf", "SparkContext", "SparkFiles", "RDD", "StorageLevel", "Broadcast",
diff --git a/python/pyspark/sql/__init__.py b/python/pyspark/sql/__init__.py
index 0a8d71c12e060..c28cb8c3b9cbe 100644
--- a/python/pyspark/sql/__init__.py
+++ b/python/pyspark/sql/__init__.py
@@ -43,7 +43,7 @@
 
 
 from pyspark.sql.types import Row
-from pyspark.sql.context import SQLContext, UDFRegistration
+from pyspark.sql.context import SQLContext, HiveContext, UDFRegistration
 from pyspark.sql.session import SparkSession
 from pyspark.sql.column import Column
 from pyspark.sql.catalog import Catalog
@@ -55,7 +55,7 @@
 
 
 __all__ = [
-    'SparkSession', 'SQLContext', 'UDFRegistration',
+    'SparkSession', 'SQLContext', 'HiveContext', 'UDFRegistration',
     'DataFrame', 'GroupedData', 'Column', 'Catalog', 'Row',
     'DataFrameNaFunctions', 'DataFrameStatFunctions', 'Window', 'WindowSpec',
     'DataFrameReader', 'DataFrameWriter', 'PandasCogroupedOps'
diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py
index 08cf6ee330785..974251f63b37a 100644
--- a/python/pyspark/sql/catalog.py
+++ b/python/pyspark/sql/catalog.py
@@ -138,6 +138,26 @@ def listColumns(self, tableName, dbName=None):
                 isBucket=jcolumn.isBucket()))
         return columns
 
+    @since(2.0)
+    def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
+        """Creates a table based on the dataset in a data source.
+
+        It returns the DataFrame associated with the external table.
+
+        The data source is specified by the ``source`` and a set of ``options``.
+        If ``source`` is not specified, the default data source configured by
+        ``spark.sql.sources.default`` will be used.
+
+        Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
+        created external table.
+
+        :return: :class:`DataFrame`
+        """
+        warnings.warn(
+            "createExternalTable is deprecated since Spark 2.2, please use createTable instead.",
+            DeprecationWarning)
+        return self.createTable(tableName, path, source, schema, **options)
+
     @since(2.2)
     def createTable(self, tableName, path=None, source=None, schema=None, **options):
         """Creates a table based on the dataset in a data source.
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index f203e1cb55588..0b7a7dabf296e 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -32,7 +32,7 @@
 from pyspark.sql.udf import UDFRegistration
 from pyspark.sql.utils import install_exception_handler
 
-__all__ = ["SQLContext"]
+__all__ = ["SQLContext", "HiveContext"]
 
 
 class SQLContext(object):
@@ -338,6 +338,24 @@ def dropTempTable(self, tableName):
         """
         self.sparkSession.catalog.dropTempView(tableName)
 
+    @since(1.3)
+    def createExternalTable(self, tableName, path=None, source=None, schema=None, **options):
+        """Creates an external table based on the dataset in a data source.
+
+        It returns the DataFrame associated with the external table.
+
+        The data source is specified by the ``source`` and a set of ``options``.
+        If ``source`` is not specified, the default data source configured by
+        ``spark.sql.sources.default`` will be used.
+
+        Optionally, a schema can be provided as the schema of the returned :class:`DataFrame` and
+        created external table.
+
+        :return: :class:`DataFrame`
+        """
+        return self.sparkSession.catalog.createExternalTable(
+            tableName, path, source, schema, **options)
+
     @ignore_unicode_prefix
     @since(1.0)
     def sql(self, sqlQuery):
@@ -461,6 +479,53 @@ def streams(self):
         return StreamingQueryManager(self._ssql_ctx.streams())
 
 
+class HiveContext(SQLContext):
+    """A variant of Spark SQL that integrates with data stored in Hive.
+
+    Configuration for Hive is read from ``hive-site.xml`` on the classpath.
+    It supports running both SQL and HiveQL commands.
+
+    :param sparkContext: The SparkContext to wrap.
+    :param jhiveContext: An optional JVM Scala HiveContext. If set, we do not instantiate a new
+        :class:`HiveContext` in the JVM, instead we make all calls to this object.
+
+    .. note:: Deprecated in 2.0.0. Use SparkSession.builder.enableHiveSupport().getOrCreate().
+    """
+
+    def __init__(self, sparkContext, jhiveContext=None):
+        warnings.warn(
+            "HiveContext is deprecated in Spark 2.0.0. Please use " +
+            "SparkSession.builder.enableHiveSupport().getOrCreate() instead.",
+            DeprecationWarning)
+        if jhiveContext is None:
+            sparkContext._conf.set("spark.sql.catalogImplementation", "hive")
+            sparkSession = SparkSession.builder._sparkContext(sparkContext).getOrCreate()
+        else:
+            sparkSession = SparkSession(sparkContext, jhiveContext.sparkSession())
+        SQLContext.__init__(self, sparkContext, sparkSession, jhiveContext)
+
+    @classmethod
+    def _createForTesting(cls, sparkContext):
+        """(Internal use only) Create a new HiveContext for testing.
+
+        All test code that touches HiveContext *must* go through this method. Otherwise,
+        you may end up launching multiple derby instances and encounter with incredibly
+        confusing error messages.
+        """
+        jsc = sparkContext._jsc.sc()
+        jtestHive = sparkContext._jvm.org.apache.spark.sql.hive.test.TestHiveContext(jsc, False)
+        return cls(sparkContext, jtestHive)
+
+    def refreshTable(self, tableName):
+        """Invalidate and refresh all the cached the metadata of the given
+        table. For performance reasons, Spark SQL or the external data source
+        library it uses might cache certain metadata about a table, such as the
+        location of blocks. When those change outside of Spark SQL, users should
+        call this function to invalidate the cache.
+        """
+        self._ssql_ctx.refreshTable(tableName)
+
+
 def _test():
     import os
     import doctest
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 592c64c93c8a2..bbcc842877f74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -479,6 +479,97 @@ class SQLContext private[sql](val sparkSession: SparkSession)
   def readStream: DataStreamReader = sparkSession.readStream
 
 
+  /**
+   * Creates an external table from the given path and returns the corresponding DataFrame.
+   * It will use the default data source configured by spark.sql.sources.default.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(tableName: String, path: String): DataFrame = {
+    sparkSession.catalog.createTable(tableName, path)
+  }
+
+  /**
+   * Creates an external table from the given path based on a data source
+   * and returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      path: String,
+      source: String): DataFrame = {
+    sparkSession.catalog.createTable(tableName, path, source)
+  }
+
+  /**
+   * Creates an external table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: java.util.Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, options)
+  }
+
+  /**
+   * (Scala-specific)
+   * Creates an external table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, options)
+  }
+
+  /**
+   * Create an external table from the given path based on a data source, a schema and
+   * a set of options. Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: java.util.Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, schema, options)
+  }
+
+  /**
+   * (Scala-specific)
+   * Create an external table from the given path based on a data source, a schema and
+   * a set of options. Then, returns the corresponding DataFrame.
+   *
+   * @group ddl_ops
+   * @since 1.3.0
+   */
+  @deprecated("use sparkSession.catalog.createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: Map[String, String]): DataFrame = {
+    sparkSession.catalog.createTable(tableName, source, schema, options)
+  }
+
   /**
    * Registers the given `DataFrame` as a temporary table in the catalog. Temporary tables exist
    * only during the lifetime of this instance of SQLContext.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
index 318cc629e7a34..60738e6d4ef9e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalog/Catalog.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalog
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.annotation.Stable
+import org.apache.spark.annotation.{Evolving, Experimental, Stable}
 import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.StorageLevel
@@ -208,6 +208,20 @@ abstract class Catalog {
    */
   def functionExists(dbName: String, functionName: String): Boolean
 
+  /**
+   * Creates a table from the given path and returns the corresponding DataFrame.
+   * It will use the default data source configured by spark.sql.sources.default.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(tableName: String, path: String): DataFrame = {
+    createTable(tableName, path)
+  }
+
   /**
    * Creates a table from the given path and returns the corresponding DataFrame.
    * It will use the default data source configured by spark.sql.sources.default.
@@ -219,6 +233,20 @@ abstract class Catalog {
    */
   def createTable(tableName: String, path: String): DataFrame
 
+  /**
+   * Creates a table from the given path based on a data source and returns the corresponding
+   * DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(tableName: String, path: String, source: String): DataFrame = {
+    createTable(tableName, path, source)
+  }
+
   /**
    * Creates a table from the given path based on a data source and returns the corresponding
    * DataFrame.
@@ -230,6 +258,23 @@ abstract class Catalog {
    */
   def createTable(tableName: String, path: String, source: String): DataFrame
 
+  /**
+   * Creates a table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: java.util.Map[String, String]): DataFrame = {
+    createTable(tableName, source, options)
+  }
+
   /**
    * Creates a table based on the dataset in a data source and a set of options.
    * Then, returns the corresponding DataFrame.
@@ -246,6 +291,24 @@ abstract class Catalog {
     createTable(tableName, source, options.asScala.toMap)
   }
 
+  /**
+   * (Scala-specific)
+   * Creates a table from the given path based on a data source and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      options: Map[String, String]): DataFrame = {
+    createTable(tableName, source, options)
+  }
+
   /**
    * (Scala-specific)
    * Creates a table based on the dataset in a data source and a set of options.
@@ -261,6 +324,24 @@ abstract class Catalog {
       source: String,
       options: Map[String, String]): DataFrame
 
+  /**
+   * Create a table from the given path based on a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: java.util.Map[String, String]): DataFrame = {
+    createTable(tableName, source, schema, options)
+  }
+
   /**
    * Create a table based on the dataset in a data source, a schema and a set of options.
    * Then, returns the corresponding DataFrame.
@@ -278,6 +359,25 @@ abstract class Catalog {
     createTable(tableName, source, schema, options.asScala.toMap)
   }
 
+  /**
+   * (Scala-specific)
+   * Create a table from the given path based on a data source, a schema and a set of options.
+   * Then, returns the corresponding DataFrame.
+   *
+   * @param tableName is either a qualified or unqualified name that designates a table.
+   *                  If no database identifier is provided, it refers to a table in
+   *                  the current database.
+   * @since 2.0.0
+   */
+  @deprecated("use createTable instead.", "2.2.0")
+  def createExternalTable(
+      tableName: String,
+      source: String,
+      schema: StructType,
+      options: Map[String, String]): DataFrame = {
+    createTable(tableName, source, schema, options)
+  }
+
   /**
    * (Scala-specific)
    * Create a table based on the dataset in a data source, a schema and a set of options.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala
new file mode 100644
index 0000000000000..0b5cd3dae4761
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/DeprecatedCreateExternalTableSuite.scala
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal
+
+import java.io.File
+
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
+
+class DeprecatedCreateExternalTableSuite extends SharedSparkSession {
+  test("createExternalTable with explicit path") {
+    withTable("t") {
+      withTempDir { dir =>
+        val path = new File(dir, "test")
+        spark.range(100).write.parquet(path.getAbsolutePath)
+        spark.catalog.createExternalTable(
+          tableName = "t",
+          path = path.getAbsolutePath
+        )
+        assert(spark.sessionState.catalog.tableExists(TableIdentifier("t")))
+        val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+        assert(table.tableType === CatalogTableType.EXTERNAL)
+        assert(table.provider === Some("parquet"))
+        assert(table.schema === new StructType().add("id", "long"))
+        assert(table.storage.locationUri.get == makeQualifiedPath(path.getAbsolutePath))
+      }
+    }
+  }
+
+  test("createExternalTable with 'path' options") {
+    withTable("t") {
+      withTempDir { dir =>
+        val path = new File(dir, "test")
+        spark.range(100).write.parquet(path.getAbsolutePath)
+        spark.catalog.createExternalTable(
+          tableName = "t",
+          source = "parquet",
+          options = Map("path" -> path.getAbsolutePath))
+        assert(spark.sessionState.catalog.tableExists(TableIdentifier("t")))
+        val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+        assert(table.tableType === CatalogTableType.EXTERNAL)
+        assert(table.provider === Some("parquet"))
+        assert(table.schema === new StructType().add("id", "long"))
+        assert(table.storage.locationUri.get == makeQualifiedPath(path.getAbsolutePath))
+      }
+    }
+  }
+
+  test("createExternalTable with explicit schema") {
+    withTable("t") {
+      withTempDir { dir =>
+        val path = new File(dir, "test")
+        spark.range(100).write.parquet(path.getAbsolutePath)
+        spark.catalog.createExternalTable(
+          tableName = "t",
+          source = "parquet",
+          schema = new StructType().add("i", "int"),
+          options = Map("path" -> path.getAbsolutePath))
+        assert(spark.sessionState.catalog.tableExists(TableIdentifier("t")))
+        val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier("t"))
+        assert(table.tableType === CatalogTableType.EXTERNAL)
+        assert(table.provider === Some("parquet"))
+        assert(table.schema === new StructType().add("i", "int"))
+        assert(table.storage.locationUri.get == makeQualifiedPath(path.getAbsolutePath))
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
new file mode 100644
index 0000000000000..02a5117f005e8
--- /dev/null
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.SparkContext
+import org.apache.spark.api.java.JavaSparkContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{SparkSession, SQLContext}
+
+
+/**
+ * An instance of the Spark SQL execution engine that integrates with data stored in Hive.
+ * Configuration for Hive is read from hive-site.xml on the classpath.
+ */
+@deprecated("Use SparkSession.builder.enableHiveSupport instead", "2.0.0")
+class HiveContext private[hive](_sparkSession: SparkSession)
+  extends SQLContext(_sparkSession) with Logging {
+
+  self =>
+
+  def this(sc: SparkContext) = {
+    this(SparkSession.builder().sparkContext(HiveUtils.withHiveExternalCatalog(sc)).getOrCreate())
+  }
+
+  def this(sc: JavaSparkContext) = this(sc.sc)
+
+  /**
+   * Returns a new HiveContext as new session, which will have separated SQLConf, UDF/UDAF,
+   * temporary tables and SessionState, but sharing the same CacheManager, IsolatedClientLoader
+   * and Hive client (both of execution and metadata) with existing HiveContext.
+   */
+  override def newSession(): HiveContext = {
+    new HiveContext(sparkSession.newSession())
+  }
+
+  /**
+   * Invalidate and refresh all the cached the metadata of the given table. For performance reasons,
+   * Spark SQL or the external data source library it uses might cache certain metadata about a
+   * table, such as the location of blocks. When those change outside of Spark SQL, users should
+   * call this function to invalidate the cache.
+   *
+   * @since 1.3.0
+   */
+  def refreshTable(tableName: String): Unit = {
+    sparkSession.catalog.refreshTable(tableName)
+  }
+
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala
new file mode 100644
index 0000000000000..a80db765846e9
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveContextCompatibilitySuite.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.scalatest.BeforeAndAfterEach
+
+import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
+
+
+class HiveContextCompatibilitySuite extends SparkFunSuite with BeforeAndAfterEach {
+
+  override protected val enableAutoThreadAudit = false
+  private var sc: SparkContext = null
+  private var hc: HiveContext = null
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    sc = SparkContext.getOrCreate(new SparkConf().setMaster("local").setAppName("test"))
+    HiveUtils.newTemporaryConfiguration(useInMemoryDerby = true).foreach { case (k, v) =>
+      sc.hadoopConfiguration.set(k, v)
+    }
+    hc = new HiveContext(sc)
+  }
+
+  override def afterEach(): Unit = {
+    try {
+      hc.sharedState.cacheManager.clearCache()
+      hc.sessionState.catalog.reset()
+    } finally {
+      super.afterEach()
+    }
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      sc = null
+      hc = null
+    } finally {
+      super.afterAll()
+    }
+  }
+
+  test("basic operations") {
+    val _hc = hc
+    import _hc.implicits._
+    val df1 = (1 to 20).map { i => (i, i) }.toDF("a", "x")
+    val df2 = (1 to 100).map { i => (i, i % 10, i % 2 == 0) }.toDF("a", "b", "c")
+      .select($"a", $"b")
+      .filter($"a" > 10 && $"b" > 6 && $"c")
+    val df3 = df1.join(df2, "a")
+    val res = df3.collect()
+    val expected = Seq((18, 18, 8)).toDF("a", "x", "b").collect()
+    assert(res.toSeq == expected.toSeq)
+    df3.createOrReplaceTempView("mai_table")
+    val df4 = hc.table("mai_table")
+    val res2 = df4.collect()
+    assert(res2.toSeq == expected.toSeq)
+  }
+
+  test("basic DDLs") {
+    val _hc = hc
+    import _hc.implicits._
+    val databases = hc.sql("SHOW DATABASES").collect().map(_.getString(0))
+    assert(databases.toSeq == Seq("default"))
+    hc.sql("CREATE DATABASE mee_db")
+    hc.sql("USE mee_db")
+    val databases2 = hc.sql("SHOW DATABASES").collect().map(_.getString(0))
+    assert(databases2.toSet == Set("default", "mee_db"))
+    val df = (1 to 10).map { i => ("bob" + i.toString, i) }.toDF("name", "age")
+    df.createOrReplaceTempView("mee_table")
+    hc.sql("CREATE TABLE moo_table (name string, age int)")
+    hc.sql("INSERT INTO moo_table SELECT * FROM mee_table")
+    assert(
+      hc.sql("SELECT * FROM moo_table order by name").collect().toSeq ==
+      df.collect().toSeq.sortBy(_.getString(0)))
+    val tables = hc.sql("SHOW TABLES IN mee_db").select("tableName").collect().map(_.getString(0))
+    assert(tables.toSet == Set("moo_table", "mee_table"))
+    hc.sql("DROP TABLE moo_table")
+    hc.sql("DROP TABLE mee_table")
+    val tables2 = hc.sql("SHOW TABLES IN mee_db").select("tableName").collect().map(_.getString(0))
+    assert(tables2.isEmpty)
+    hc.sql("USE default")
+    hc.sql("DROP DATABASE mee_db CASCADE")
+    val databases3 = hc.sql("SHOW DATABASES").collect().map(_.getString(0))
+    assert(databases3.toSeq == Seq("default"))
+  }
+
+}

From 4868b4d119e5fd7099d9f33f23a4164007360712 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 27 Mar 2020 15:21:19 +0800
Subject: [PATCH 0363/1280] [SPARK-31225][SQL] Override sql method of
 OuterReference

### What changes were proposed in this pull request?

OuterReference is one LeafExpression, so it's children is Nil, which makes its SQL representation always be outer(). This makes our explain-command and error msg unclear when OuterReference exists.
e.g.

```scala
org.apache.spark.sql.AnalysisException:
Aggregate/Window/Generate expressions are not valid in where clause of the query.
Expression in where clause: [(in.`value` = max(outer()))]
Invalid expressions: [max(outer())];;
```
This PR override its `sql` method with its `prettyName` and single argment `e`'s `sql` methond

### Why are the changes needed?

improve err message

### Does this PR introduce any user-facing change?

yes, the err msg caused by OuterReference has changed
### How was this patch tested?

modified ut results

Closes #27985 from yaooqinn/SPARK-31225.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5945d46c11a86fd85f9e65f24c2e88f368eee01f)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/expressions/namedExpressions.scala     | 1 +
 .../sql-tests/results/postgreSQL/aggregates_part1.sql.out     | 4 ++--
 .../subquery/negative-cases/invalid-correlation.sql.out       | 2 +-
 .../results/udf/postgreSQL/udf-aggregates_part1.sql.out       | 4 ++--
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
index 02e90f8458c3d..77b4cecc263c7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala
@@ -382,6 +382,7 @@ case class OuterReference(e: NamedExpression)
   override def nullable: Boolean = e.nullable
   override def prettyName: String = "outer"
 
+  override def sql: String = s"$prettyName(${e.sql})"
   override def name: String = e.name
   override def qualifier: Seq[String] = e.qualifier
   override def exprId: ExprId = e.exprId
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
index 5efb58c7fc1b0..f7bba96738eab 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part1.sql.out
@@ -381,8 +381,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) = CAST(b.`four` AS BIGINT))]
-Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))];
+Expression in where clause: [(sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT)) = CAST(b.`four` AS BIGINT))]
+Invalid expressions: [sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT))];
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
index ec7ecf28754ef..d703d4e9112e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/negative-cases/invalid-correlation.sql.out
@@ -109,7 +109,7 @@ struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
 Expressions referencing the outer query are not supported outside of WHERE/HAVING clauses:
-Aggregate [min(outer(t2a#x)) AS min(outer())#x]
+Aggregate [min(outer(t2a#x)) AS min(outer(t2.`t2a`))#x]
 +- SubqueryAlias t3
    +- Project [t3a#x, t3b#x, t3c#x]
       +- SubqueryAlias t3
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
index adf434b1bde95..76637bf578e6f 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part1.sql.out
@@ -372,8 +372,8 @@ struct<>
 org.apache.spark.sql.AnalysisException
 
 Aggregate/Window/Generate expressions are not valid in where clause of the query.
-Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) = CAST(CAST(udf(ansi_cast(four as string)) AS INT) AS BIGINT))]
-Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))];
+Expression in where clause: [(sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT)) = CAST(CAST(udf(ansi_cast(four as string)) AS INT) AS BIGINT))]
+Invalid expressions: [sum(DISTINCT CAST((outer(a.`four`) + b.`four`) AS BIGINT))];
 
 
 -- !query

From f94d13f6da9f8f9c1cb98df63dcfc07c2846b879 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 27 Mar 2020 01:00:51 -0700
Subject: [PATCH 0364/1280] 
 [SPARK-25556][SPARK-17636][SPARK-31026][SPARK-31060][FOLLOWUP][3.0] Fix build
 error due to conf version

### What changes were proposed in this pull request?

This fixes the build error.

### Why are the changes needed?

`branch-3.0` doesn't have conf version.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Passing the GitHub Action is enough.

Closes #28046 from dongjoon-hyun/hotfix.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index a6cb96d81c519..26ddd40f76e04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1864,7 +1864,6 @@ object SQLConf {
         "containing `dots` to data sources. Currently, Parquet implements both optimizations " +
         "while ORC only supports predicates for names containing `dots`. The other data sources" +
         "don't support this feature yet.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 

From c2f79a61de2af529999ff9ef8321094f2ea7ea9b Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 27 Mar 2020 21:14:25 +0800
Subject: [PATCH 0365/1280] [SPARK-31277][SQL][TESTS] Migrate
 `DateTimeTestUtils` from `TimeZone` to `ZoneId`

In the PR, I propose to change types of `DateTimeTestUtils` values and functions by replacing `java.util.TimeZone` to `java.time.ZoneId`. In particular:
1. Type of `ALL_TIMEZONES` is changed to `Seq[ZoneId]`.
2. Remove `val outstandingTimezones: Seq[TimeZone]`.
3. Change the type of the time zone parameter in `withDefaultTimeZone` to `ZoneId`.
4. Modify affected test suites.

Currently, Spark SQL's date-time expressions and functions have been already ported on Java 8 time API but tests still use old time APIs. In particular, `DateTimeTestUtils` exposes functions that accept only TimeZone instances. This is inconvenient, and CPU consuming because need to convert TimeZone instances to ZoneId instances via strings (zone ids).

No

By affected test suites executed by jenkins builds.

Closes #28033 from MaxGekk/with-default-time-zone.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9f0c010a5c9b5c10401f8e16fa7a151714b6dbb0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/avro/AvroSuite.scala |  4 +-
 .../sql/catalyst/expressions/CastSuite.scala  |  9 +--
 .../CollectionExpressionsSuite.scala          |  4 +-
 .../expressions/CsvExpressionsSuite.scala     | 10 +--
 .../expressions/DateExpressionsSuite.scala    | 63 ++++++++++---------
 .../expressions/JsonExpressionsSuite.scala    |  8 +--
 .../parser/ExpressionParserSuite.scala        |  6 +-
 .../sql/catalyst/util/DateTimeTestUtils.scala | 15 ++---
 .../catalyst/util/DateTimeUtilsSuite.scala    | 54 ++++++++--------
 .../spark/sql/DataFrameFunctionsSuite.scala   |  2 +-
 .../spark/sql/StatisticsCollectionSuite.scala |  3 +-
 11 files changed, 91 insertions(+), 87 deletions(-)

diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 34a0e2bf607a9..a04037c17d7e6 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -38,7 +38,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, UTC}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -408,7 +408,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         StructField("float", FloatType, true),
         StructField("date", DateType, true)
       ))
-      DateTimeTestUtils.withDefaultTimeZone(DateTimeUtils.TimeZoneUTC) {
+      withDefaultTimeZone(UTC) {
         val rdd = spark.sparkContext.parallelize(Seq(
           Row(1f, null),
           Row(2f, new Date(1451948400000L)),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index e25d805d25a26..5c57843980374 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -112,13 +112,14 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
   }
 
   test("cast string to timestamp") {
-    new ParVector(ALL_TIMEZONES.toVector).foreach { tz =>
+    new ParVector(ALL_TIMEZONES.toVector).foreach { zid =>
       def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = {
-        checkEvaluation(cast(Literal(str), TimestampType, Option(tz.getID)), expected)
+        checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected)
       }
 
       checkCastStringToTimestamp("123", null)
 
+      val tz = TimeZone.getTimeZone(zid)
       var c = Calendar.getInstance(tz)
       c.set(2015, 0, 1, 0, 0, 0)
       c.set(Calendar.MILLISECOND, 0)
@@ -263,10 +264,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     val zts = sd + " 00:00:00"
     val sts = sd + " 00:00:02"
     val nts = sts + ".1"
-    val ts = withDefaultTimeZone(TimeZoneGMT)(Timestamp.valueOf(nts))
+    val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts))
 
     for (tz <- ALL_TIMEZONES) {
-      val timeZoneId = Option(tz.getID)
+      val timeZoneId = Option(tz.getId)
       var c = Calendar.getInstance(TimeZoneGMT)
       c.set(2015, 2, 8, 2, 30, 0)
       checkEvaluation(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 00cdc1712c459..91773824b845d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -852,7 +852,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
       DateTimeUtils.fromMillis(Timestamp.valueOf(s).getTime - offset)
     }
 
-    DateTimeTestUtils.withDefaultTimeZone(timeZone) {
+    DateTimeTestUtils.withDefaultTimeZone(timeZone.toZoneId) {
       // Spring time change
       checkEvaluation(new Sequence(
         Literal(Timestamp.valueOf("2018-03-25 01:30:00")),
@@ -880,7 +880,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
   }
 
   test("Sequence of dates") {
-    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
+    DateTimeTestUtils.withDefaultTimeZone(UTC) {
       checkEvaluation(new Sequence(
         Literal(Date.valueOf("2018-01-01")),
         Literal(Date.valueOf("2018-01-05")),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
index 24cf9339aa72e..16d78b0526503 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CsvExpressionsSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.text.SimpleDateFormat
-import java.util.{Calendar, Locale}
+import java.util.{Calendar, Locale, TimeZone}
 
 import org.scalatest.exceptions.TestFailedException
 
@@ -89,8 +89,8 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
     )
 
     val csvData2 = "2016-01-01T00:00:00"
-    for (tz <- DateTimeTestUtils.outstandingTimezones) {
-      c = Calendar.getInstance(tz)
+    for (zid <- DateTimeTestUtils.outstandingZoneIds) {
+      c = Calendar.getInstance(TimeZone.getTimeZone(zid))
       c.set(2016, 0, 1, 0, 0, 0)
       c.set(Calendar.MILLISECOND, 0)
       checkEvaluation(
@@ -98,14 +98,14 @@ class CsvExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with P
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"),
           Literal(csvData2),
-          Option(tz.getID)),
+          Option(zid.getId)),
         InternalRow(c.getTimeInMillis * 1000L)
       )
       checkEvaluation(
         CsvToStructs(
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-            DateTimeUtils.TIMEZONE_OPTION -> tz.getID),
+            DateTimeUtils.TIMEZONE_OPTION -> zid.getId),
           Literal(csvData2),
           UTC_OPT),
         InternalRow(c.getTimeInMillis * 1000L)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 2cc62598dc433..350b3a44baa82 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -185,9 +185,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Second(Literal(ts), UTC_OPT), 15)
 
     val c = Calendar.getInstance()
-    for (tz <- outstandingTimezones) {
-      val timeZoneId = Option(tz.getID)
-      c.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      c.setTimeZone(TimeZone.getTimeZone(zid))
       (0 to 60 by 5).foreach { s =>
         c.set(2015, 18, 3, 3, 5, s)
         checkEvaluation(
@@ -285,9 +285,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Hour(Literal(ts), UTC_OPT), 13)
 
     val c = Calendar.getInstance()
-    for (tz <- outstandingTimezones) {
-      val timeZoneId = Option(tz.getID)
-      c.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      c.setTimeZone(TimeZone.getTimeZone(zid))
       (0 to 24 by 6).foreach { h =>
         (0 to 60 by 30).foreach { m =>
           (0 to 60 by 30).foreach { s =>
@@ -312,9 +312,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Minute(Literal(ts), UTC_OPT), 10)
 
     val c = Calendar.getInstance()
-    for (tz <- outstandingTimezones) {
-      val timeZoneId = Option(tz.getID)
-      c.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      c.setTimeZone(TimeZone.getTimeZone(zid))
       (0 to 60 by 5).foreach { m =>
         (0 to 60 by 15).foreach { s =>
           c.set(2015, 18, 3, 3, m, s)
@@ -384,9 +384,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("time_add") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
-    for (tz <- outstandingTimezones) {
-      val timeZoneId = Option(tz.getID)
-      sdf.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
 
       checkEvaluation(
         TimeAdd(
@@ -422,9 +422,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("time_sub") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
-    for (tz <- outstandingTimezones) {
-      val timeZoneId = Option(tz.getID)
-      sdf.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
 
       checkEvaluation(
         TimeSub(
@@ -508,9 +508,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   test("months_between") {
     val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US)
-    for (tz <- outstandingTimezones) {
-      val timeZoneId = Option(tz.getID)
-      sdf.setTimeZone(tz)
+    for (zid <- outstandingZoneIds) {
+      val timeZoneId = Option(zid.getId)
+      sdf.setTimeZone(TimeZone.getTimeZone(zid))
 
       checkEvaluation(
         MonthsBetween(
@@ -641,7 +641,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         expected)
     }
 
-    withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
+    withDefaultTimeZone(UTC) {
       val inputDate = Timestamp.valueOf("2015-07-22 05:30:06")
 
       Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt =>
@@ -711,8 +711,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf1 = new SimpleDateFormat(fmt1, Locale.US)
         val fmt2 = "yyyy-MM-dd HH:mm:ss.SSS"
         val sdf2 = new SimpleDateFormat(fmt2, Locale.US)
-        for (tz <- outstandingTimezones) {
-          val timeZoneId = Option(tz.getID)
+        for (zid <- outstandingZoneIds) {
+          val timeZoneId = Option(zid.getId)
+          val tz = TimeZone.getTimeZone(zid)
           sdf1.setTimeZone(tz)
           sdf2.setTimeZone(tz)
 
@@ -762,9 +763,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
         sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
 
-        withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
-          for (tz <- outstandingTimezones) {
-            val timeZoneId = Option(tz.getID)
+        withDefaultTimeZone(UTC) {
+          for (zid <- outstandingZoneIds) {
+            val timeZoneId = Option(zid.getId)
+            val tz = TimeZone.getTimeZone(zid)
             sdf1.setTimeZone(tz)
             sdf2.setTimeZone(tz)
 
@@ -829,9 +831,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         val sdf3 = new SimpleDateFormat(fmt3, Locale.US)
         sdf3.setTimeZone(TimeZone.getTimeZone(UTC))
 
-        withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
-          for (tz <- outstandingTimezones) {
-            val timeZoneId = Option(tz.getID)
+        withDefaultTimeZone(UTC) {
+          for (zid <- outstandingZoneIds) {
+            val timeZoneId = Option(zid.getId)
+            val tz = TimeZone.getTimeZone(zid)
             sdf1.setTimeZone(tz)
             sdf2.setTimeZone(tz)
 
@@ -847,7 +850,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(
               ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
               MILLISECONDS.toSeconds(
-                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), zid)))
             checkEvaluation(
               ToUnixTimestamp(
                 Literal(sdf2.format(new Timestamp(-1000000))),
@@ -856,7 +859,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(ToUnixTimestamp(
               Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
               MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
-                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId)))
+                DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), zid)))
             val t1 = ToUnixTimestamp(
               CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
             val t2 = ToUnixTimestamp(
@@ -871,7 +874,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
             checkEvaluation(ToUnixTimestamp(
               Literal(date1), Literal.create(null, StringType), timeZoneId),
               MILLISECONDS.toSeconds(
-                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
+                DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), zid)))
             checkEvaluation(
               ToUnixTimestamp(
                 Literal("2015-07-24"),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index dec2e4665251f..032e0ac61884b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -517,8 +517,8 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     )
 
     val jsonData2 = """{"t": "2016-01-01T00:00:00"}"""
-    for (tz <- DateTimeTestUtils.outstandingTimezones) {
-      c = Calendar.getInstance(tz)
+    for (zid <- DateTimeTestUtils.outstandingZoneIds) {
+      c = Calendar.getInstance(TimeZone.getTimeZone(zid))
       c.set(2016, 0, 1, 0, 0, 0)
       c.set(Calendar.MILLISECOND, 0)
       checkEvaluation(
@@ -526,14 +526,14 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss"),
           Literal(jsonData2),
-          Option(tz.getID)),
+          Option(zid.getId)),
         InternalRow(c.getTimeInMillis * 1000L)
       )
       checkEvaluation(
         JsonToStructs(
           schema,
           Map("timestampFormat" -> "yyyy-MM-dd'T'HH:mm:ss",
-            DateTimeUtils.TIMEZONE_OPTION -> tz.getID),
+            DateTimeUtils.TIMEZONE_OPTION -> zid.getId),
           Literal(jsonData2),
           UTC_OPT),
         InternalRow(c.getTimeInMillis * 1000L)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 74fd48d922a3a..522d49e448f1f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -772,10 +772,10 @@ class ExpressionParserSuite extends AnalysisTest {
   }
 
   test("timestamp literals") {
-    DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
-      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone.getID) {
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId) {
         def toMicros(time: LocalDateTime): Long = {
-          val seconds = time.atZone(timeZone.toZoneId).toInstant.getEpochSecond
+          val seconds = time.atZone(zid).toInstant.getEpochSecond
           TimeUnit.SECONDS.toMicros(seconds)
         }
         assertEval(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index 374400ecdb058..bf9e8f71ba1c9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -21,6 +21,8 @@ import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId, ZoneOffset}
 import java.util.TimeZone
 import java.util.concurrent.TimeUnit
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
 
@@ -39,7 +41,7 @@ object DateTimeTestUtils {
 
   val UTC_OPT = Option("UTC")
 
-  val ALL_TIMEZONES: Seq[TimeZone] = TimeZone.getAvailableIDs.toSeq.map(TimeZone.getTimeZone)
+  val ALL_TIMEZONES: Seq[ZoneId] = ZoneId.getAvailableZoneIds.asScala.map(getZoneId).toSeq
 
   val outstandingTimezonesIds: Seq[String] = Seq(
     "UTC",
@@ -50,16 +52,15 @@ object DateTimeTestUtils {
     "Antarctica/Vostok",
     "Asia/Hong_Kong",
     "Europe/Amsterdam")
-  val outstandingTimezones: Seq[TimeZone] = outstandingTimezonesIds.map(TimeZone.getTimeZone)
-  val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(DateTimeUtils.getZoneId)
+  val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId)
 
-  def withDefaultTimeZone[T](newDefaultTimeZone: TimeZone)(block: => T): T = {
-    val originalDefaultTimeZone = TimeZone.getDefault
+  def withDefaultTimeZone[T](newDefaultTimeZone: ZoneId)(block: => T): T = {
+    val originalDefaultTimeZone = ZoneId.systemDefault()
     try {
-      TimeZone.setDefault(newDefaultTimeZone)
+      TimeZone.setDefault(TimeZone.getTimeZone(newDefaultTimeZone))
       block
     } finally {
-      TimeZone.setDefault(originalDefaultTimeZone)
+      TimeZone.setDefault(TimeZone.getTimeZone(originalDefaultTimeZone))
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 5eca86409aa01..922e09dd1e29d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -154,11 +154,10 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("string to timestamp") {
-    for (tz <- ALL_TIMEZONES) {
+    for (zid <- ALL_TIMEZONES) {
       def checkStringToTimestamp(str: String, expected: Option[Long]): Unit = {
-        assert(toTimestamp(str, tz.toZoneId) === expected)
+        assert(toTimestamp(str, zid) === expected)
       }
-      val zid = tz.toZoneId
 
       checkStringToTimestamp("1969-12-31 16:00:00", Option(date(1969, 12, 31, 16, zid = zid)))
       checkStringToTimestamp("0001", Option(date(1, 1, 1, 0, zid = zid)))
@@ -465,7 +464,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
 
-    withDefaultTimeZone(TimeZone.getTimeZone(LA.getId)) {
+    withDefaultTimeZone(LA) {
       // Daylight Saving Time
       test("2016-03-13 09:59:59.0", LA.getId, "2016-03-13 01:59:59.0")
       test("2016-03-13 10:00:00.0", LA.getId, "2016-03-13 03:00:00.0")
@@ -481,8 +480,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         === expected)
     }
 
-    for (tz <- ALL_TIMEZONES) {
-      withDefaultTimeZone(tz) {
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
         test("2011-12-25 09:00:00.123456", "UTC", "2011-12-25 09:00:00.123456")
         test("2011-12-25 18:00:00.123456", JST.getId, "2011-12-25 09:00:00.123456")
         test("2011-12-25 01:00:00.123456", LA.getId, "2011-12-25 09:00:00.123456")
@@ -490,8 +489,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
 
-    val tz = LA.getId
-    withDefaultTimeZone(TimeZone.getTimeZone(tz)) {
+    withDefaultTimeZone(LA) {
+      val tz = LA.getId
       // Daylight Saving Time
       test("2016-03-13 01:59:59", tz, "2016-03-13 09:59:59.0")
       test("2016-03-13 02:00:00", tz, "2016-03-13 10:00:00.0")
@@ -552,9 +551,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "2001-01-01", defaultInputTS.get)
     testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", defaultInputTS.get)
 
-    for (tz <- ALL_TIMEZONES) {
-      withDefaultTimeZone(tz) {
-        val zid = tz.toZoneId
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
         val inputTS = DateTimeUtils.stringToTimestamp(
           UTF8String.fromString("2015-03-05T09:32:05.359"), defaultZoneId)
         val inputTS1 = DateTimeUtils.stringToTimestamp(
@@ -610,14 +608,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       "Pacific/Kiritimati" -> Set(9130, 9131),
       "Pacific/Kwajalein" -> Set(8632, 8633, 8634),
       MIT.getId -> Set(15338))
-    for (tz <- ALL_TIMEZONES) {
-      val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
+    for (zid <- ALL_TIMEZONES) {
+      val skipped = skipped_days.getOrElse(zid.getId, Set.empty)
       val testingData = Seq(-20000, 20000) ++
         (1 to 1000).map(_ => (math.random() * 40000 - 20000).toInt)
       testingData.foreach { d =>
         if (!skipped.contains(d)) {
-          assert(millisToDays(daysToMillis(d, tz.toZoneId), tz.toZoneId) === d,
-            s"Round trip of $d did not work in tz $tz")
+          assert(millisToDays(daysToMillis(d, zid), zid) === d,
+            s"Round trip of $d did not work in tz ${zid.getId}")
         }
       }
     }
@@ -672,8 +670,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("rebase julian to/from gregorian micros") {
-    outstandingTimezones.foreach { timeZone =>
-      withDefaultTimeZone(timeZone) {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
         Seq(
           "0001-01-01 01:02:03.654321",
           "1000-01-01 03:02:01.123456",
@@ -684,9 +682,9 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
           "1969-12-31 11:22:33.000100",
           "1970-01-01 00:00:00.000001", // The epoch day
           "2020-03-14 09:33:01.500000").foreach { ts =>
-          withClue(s"time zone = ${timeZone.getID} ts = $ts") {
+          withClue(s"time zone = ${zid.getId} ts = $ts") {
             val julianMicros = parseToJulianMicros(ts)
-            val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), timeZone.toZoneId)
+            val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), zid)
 
             assert(rebaseJulianToGregorianMicros(julianMicros) === gregMicros)
             assert(rebaseGregorianToJulianMicros(gregMicros) === julianMicros)
@@ -706,8 +704,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("rebase gregorian to/from julian days") {
-    outstandingTimezones.foreach { timeZone =>
-      withDefaultTimeZone(timeZone) {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
         Seq(
           "0001-01-01",
           "1000-01-01",
@@ -729,14 +727,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("rebase julian to gregorian date for leap years") {
-    outstandingTimezones.foreach { timeZone =>
-      withDefaultTimeZone(timeZone) {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
         Seq(
           "1000-02-29" -> "1000-03-01",
           "1600-02-29" -> "1600-02-29",
           "1700-02-29" -> "1700-03-01",
           "2000-02-29" -> "2000-02-29").foreach { case (julianDate, gregDate) =>
-          withClue(s"tz = ${timeZone.getID} julian date = $julianDate greg date = $gregDate") {
+          withClue(s"tz = ${zid.getId} julian date = $julianDate greg date = $gregDate") {
             val date = Date.valueOf(julianDate)
             val julianDays = fromJavaDateLegacy(date)
             val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
@@ -749,17 +747,17 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("rebase julian to gregorian timestamp for leap years") {
-    outstandingTimezones.foreach { timeZone =>
-      withDefaultTimeZone(timeZone) {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
         Seq(
           "1000-02-29 01:02:03.123456" -> "1000-03-01T01:02:03.123456",
           "1600-02-29 11:12:13.654321" -> "1600-02-29T11:12:13.654321",
           "1700-02-29 21:22:23.000001" -> "1700-03-01T21:22:23.000001",
           "2000-02-29 00:00:00.999999" -> "2000-02-29T00:00:00.999999"
         ).foreach { case (julianTs, gregTs) =>
-          withClue(s"tz = ${timeZone.getID} julian ts = $julianTs greg ts = $gregTs") {
+          withClue(s"tz = ${zid.getId} julian ts = $julianTs greg ts = $gregTs") {
             val julianMicros = parseToJulianMicros(julianTs)
-            val gregorianMicros = parseToGregMicros(gregTs, timeZone.toZoneId)
+            val gregorianMicros = parseToGregMicros(gregTs, zid)
 
             assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
           }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 9100ee3940a20..ac98d3f0c7095 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -1029,7 +1029,7 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
         Timestamp.valueOf("2018-01-01 12:00:00"),
         Timestamp.valueOf("2018-01-02 00:00:00")))))
 
-    DateTimeTestUtils.withDefaultTimeZone(TimeZone.getTimeZone(UTC)) {
+    DateTimeTestUtils.withDefaultTimeZone(UTC) {
       checkAnswer(
         spark.sql("select sequence(" +
           "   cast('2018-01-01' as date)" +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
index 30b15a8710d8c..708b98e8fe15a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala
@@ -481,7 +481,8 @@ class StatisticsCollectionSuite extends StatisticsCollectionTestBase with Shared
       }
     }
 
-    DateTimeTestUtils.outstandingTimezones.foreach { timeZone =>
+    DateTimeTestUtils.outstandingZoneIds.foreach { zid =>
+      val timeZone = TimeZone.getTimeZone(zid)
       checkTimestampStats(DateType, DateTimeUtils.TimeZoneUTC, timeZone) { stats =>
         assert(stats.min.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(start))
         assert(stats.max.get.asInstanceOf[Int] == TimeUnit.SECONDS.toDays(end - 1))

From b6e8f64d49caf1f0a1f1b910d603e8e000270d01 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 27 Mar 2020 09:06:59 -0700
Subject: [PATCH 0366/1280] [SPARK-31284][SQL][TESTS] Check rebasing of
 timestamps in ORC datasource

### What changes were proposed in this pull request?
In the PR, I propose 2 tests to check that rebasing of timestamps from/to the hybrid calendar (Julian + Gregorian) to/from Proleptic Gregorian calendar works correctly.
1. The test `compatibility with Spark 2.4 in reading timestamps` load ORC file saved by Spark 2.4.5 via:
```shell
$ export TZ="America/Los_Angeles"
```
```scala
scala> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")

scala> val df = Seq("1001-01-01 01:02:03.123456").toDF("tsS").select($"tsS".cast("timestamp").as("ts"))
df: org.apache.spark.sql.DataFrame = [ts: timestamp]

scala> df.write.orc("/Users/maxim/tmp/before_1582/2_4_5_ts_orc")

scala> spark.read.orc("/Users/maxim/tmp/before_1582/2_4_5_ts_orc").show(false)
+--------------------------+
|ts                        |
+--------------------------+
|1001-01-01 01:02:03.123456|
+--------------------------+
```
2. The test `rebasing timestamps in write` is round trip test. Since the previous test confirms correct rebasing of timestamps in read. This test should pass only if rebasing works correctly in write.

### Why are the changes needed?
To guarantee that rebasing works correctly for timestamps in ORC datasource.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running `OrcSourceSuite` for Hive 1.2 and 2.3 via the commands:
```
$ build/sbt -Phive-2.3 "test:testOnly *OrcSourceSuite"
```
and
```
$ build/sbt -Phive-1.2 "test:testOnly *OrcSourceSuite"
```

Closes #28047 from MaxGekk/rebase-ts-orc-test.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit fc2a974e030c82bf500a81c3908f853c3eeb761d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../test-data/before_1582_ts_v2_4.snappy.orc  | Bin 0 -> 251 bytes
 .../datasources/orc/OrcSourceSuite.scala      |  28 ++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc

diff --git a/sql/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc b/sql/core/src/test/resources/test-data/before_1582_ts_v2_4.snappy.orc
new file mode 100644
index 0000000000000000000000000000000000000000..af9ef040270ac3f79570e275d33940972934b35d
GIT binary patch
literal 251
zcmeYdau#G@;9?VE;b074a0N0IxY!uLKuC;((Mv#L=As?lA3yUdK<NMmeg=jL2EJ!C
zYo2p6Fyt}t3o%85OyFP>V36Pj(~J_TK$=m2QGyFdvjS-jAtnw6AZC(c4hRl`SOd0^
zO9Hzk5kkET3>?e?rBYl#(m+B;!9asiiIamzh>=l>siatmiG$OC(F$ya6;3l8Cir<U
daCizZNi;MFC^0ecXs8;nHGN@b_78Fv2LLuXGgtrs

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index b5e002f34313d..0b7500c012686 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -508,6 +508,34 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
       }
     }
   }
+
+  test("SPARK-31284: compatibility with Spark 2.4 in reading timestamps") {
+    Seq(false, true).foreach { vectorized =>
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+        checkAnswer(
+          readResourceOrcFile("test-data/before_1582_ts_v2_4.snappy.orc"),
+          Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+      }
+    }
+  }
+
+  test("SPARK-31284: rebasing timestamps in write") {
+    withTempPath { dir =>
+      val path = dir.getAbsolutePath
+      Seq("1001-01-01 01:02:03.123456").toDF("tsS")
+        .select($"tsS".cast("timestamp").as("ts"))
+        .write
+        .orc(path)
+
+      Seq(false, true).foreach { vectorized =>
+        withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+          checkAnswer(
+            spark.read.orc(path),
+            Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+        }
+      }
+    }
+  }
 }
 
 class OrcSourceSuite extends OrcSuite with SharedSparkSession {

From c1a03b2233aee77f957dbc94180d2334c61ac088 Mon Sep 17 00:00:00 2001
From: Prashant Sharma <prashsh1@in.ibm.com>
Date: Fri, 27 Mar 2020 09:13:55 -0700
Subject: [PATCH 0367/1280] [SPARK-31200][K8S] Enforce to use `https` in
 /etc/apt/sources.list
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…n progress errors.

### What changes were proposed in this pull request?
Switching to `https` instead of `http` in the debian mirror urls.

### Why are the changes needed?
My ISP was trying to intercept (or trying to serve from cache) the `http` traffic and this was causing a very confusing errors while building the spark image. I thought by posting this, I can help someone save his time and energy, if he encounters the same issue.
```
bash-3.2$ bin/docker-image-tool.sh -r scrapcodes -t v3.1.0-f1cc86 build
Sending build context to Docker daemon  203.4MB
Step 1/18 : ARG java_image_tag=8-jre-slim
Step 2/18 : FROM openjdk:${java_image_tag}
 ---> 381b20190cf7
Step 3/18 : ARG spark_uid=185
 ---> Using cache
 ---> 65c06f86753c
Step 4/18 : RUN set -ex &&     apt-get update &&     ln -s /lib /lib64 &&     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd &&     rm -rf /var/cache/apt/*
 ---> Running in 96bcbe927d35
+ apt-get update
Get:1 http://deb.debian.org/debian buster InRelease [122 kB]
Get:2 http://deb.debian.org/debian buster-updates InRelease [49.3 kB]
Get:3 http://deb.debian.org/debian buster/main amd64 Packages [7907 kB]
Err:3 http://deb.debian.org/debian buster/main amd64 Packages
  File has unexpected size (13217 != 7906744). Mirror sync in progress? [IP: 151.101.10.133 80]
  Hashes of expected file:
   - Filesize:7906744 [weak]
   - SHA256:80ed5d1cc1f31a568b77e4fadfd9e01fa4d65e951243fd2ce29eee14d4b532cc
   - MD5Sum:80b6d9c1b6630b2234161e42f4040ab3 [weak]
  Release file created at: Sat, 08 Feb 2020 10:57:10 +0000
Get:5 http://deb.debian.org/debian buster-updates/main amd64 Packages [7380 B]
Err:5 http://deb.debian.org/debian buster-updates/main amd64 Packages
  File has unexpected size (13233 != 7380). Mirror sync in progress? [IP: 151.101.10.133 80]
  Hashes of expected file:
   - Filesize:7380 [weak]
   - SHA256:6af9ea081b6a3da33cfaf76a81978517f65d38e45230089a5612e56f2b6b789d
  Release file created at: Fri, 20 Mar 2020 02:28:11 +0000
Get:4 http://security-cdn.debian.org/debian-security buster/updates InRelease [65.4 kB]
Get:6 http://security-cdn.debian.org/debian-security buster/updates/main amd64 Packages [183 kB]
Fetched 419 kB in 1s (327 kB/s)
Reading package lists...
E: Failed to fetch http://deb.debian.org/debian/dists/buster/main/binary-amd64/by-hash/SHA256/80ed5d1cc1f31a568b77e4fadfd9e01fa4d65e951243fd2ce29eee14d4b532cc  File has unexpected size (13217 != 7906744). Mirror sync in progress? [IP: 151.101.10.133 80]
   Hashes of expected file:
    - Filesize:7906744 [weak]
    - SHA256:80ed5d1cc1f31a568b77e4fadfd9e01fa4d65e951243fd2ce29eee14d4b532cc
    - MD5Sum:80b6d9c1b6630b2234161e42f4040ab3 [weak]
   Release file created at: Sat, 08 Feb 2020 10:57:10 +0000
E: Failed to fetch http://deb.debian.org/debian/dists/buster-updates/main/binary-amd64/by-hash/SHA256/6af9ea081b6a3da33cfaf76a81978517f65d38e45230089a5612e56f2b6b789d  File has unexpected size (13233 != 7380). Mirror sync in progress? [IP: 151.101.10.133 80]
   Hashes of expected file:
    - Filesize:7380 [weak]
    - SHA256:6af9ea081b6a3da33cfaf76a81978517f65d38e45230089a5612e56f2b6b789d
   Release file created at: Fri, 20 Mar 2020 02:28:11 +0000
E: Some index files failed to download. They have been ignored, or old ones used instead.
The command '/bin/sh -c set -ex &&     apt-get update &&     ln -s /lib /lib64 &&     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd &&     rm -rf /var/cache/apt/*' returned a non-zero code: 100
Failed to build Spark JVM Docker image, please refer to Docker build output for details.
```
### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually by switching to `https` mirrors on the offending ISP (I am already on).

Closes #27966 from ScrapCodes/docker-mirror.

Authored-by: Prashant Sharma <prashsh1@in.ibm.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit f87957371dc9c723013849c65b014c0fbf93ed3a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../kubernetes/docker/src/main/dockerfiles/spark/Dockerfile      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index 6ed37fc637b31..c1df215b43004 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -28,6 +28,7 @@ ARG spark_uid=185
 # docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .
 
 RUN set -ex && \
+    sed -i 's/http:/https:/g' /etc/apt/sources.list && \
     apt-get update && \
     ln -s /lib /lib64 && \
     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 && \

From 7435f4543ea6f2b927da6055c1cfb75f4a62f19d Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 27 Mar 2020 09:35:35 -0700
Subject: [PATCH 0368/1280] [MINOR][DOC] Refine comments of QueryPlan regarding
 subquery

### What changes were proposed in this pull request?

The query plan of Spark SQL is a mutually recursive structure: QueryPlan -> Expression (PlanExpression) -> QueryPlan, but the transformations do not take this into account.

This PR refines the comments of `QueryPlan` to highlight this fact.

### Why are the changes needed?

better document.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #28050 from cloud-fan/comment.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 8a5d49610d875c473114781e92300c79e24a53cc)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/plans/QueryPlan.scala  | 32 ++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 12482667efa0d..9f86fb298877a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -23,6 +23,16 @@ import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode, TreeNodeTag
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, StructType}
 
+/**
+ * An abstraction of the Spark SQL query plan tree, which can be logical or physical. This class
+ * defines some basic properties of a query plan node, as well as some new transform APIs to
+ * transform the expressions of the plan node.
+ *
+ * Note that, the query plan is a mutually recursive structure:
+ *   QueryPlan -> Expression (subquery) -> QueryPlan
+ * The tree traverse APIs like `transform`, `foreach`, `collect`, etc. that are
+ * inherited from `TreeNode`, do not traverse into query plans inside subqueries.
+ */
 abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanType] {
   self: PlanType =>
 
@@ -133,7 +143,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
 
   /**
    * Returns the result of running [[transformExpressions]] on this node
-   * and all its children.
+   * and all its children. Note that this method skips expressions inside subqueries.
    */
   def transformAllExpressions(rule: PartialFunction[Expression, Expression]): this.type = {
     transform {
@@ -204,7 +214,7 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   /**
-   * All the subqueries of current plan.
+   * All the top-level subqueries of the current plan node. Nested subqueries are not included.
    */
   def subqueries: Seq[PlanType] = {
     expressions.flatMap(_.collect {
@@ -213,21 +223,21 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   /**
-   * Returns a sequence containing the result of applying a partial function to all elements in this
-   * plan, also considering all the plans in its (nested) subqueries
-   */
-  def collectInPlanAndSubqueries[B](f: PartialFunction[PlanType, B]): Seq[B] =
-    (this +: subqueriesAll).flatMap(_.collect(f))
-
-  /**
-   * Returns a sequence containing the subqueries in this plan, also including the (nested)
-   * subquries in its children
+   * All the subqueries of the current plan node and all its children. Nested subqueries are also
+   * included.
    */
   def subqueriesAll: Seq[PlanType] = {
     val subqueries = this.flatMap(_.subqueries)
     subqueries ++ subqueries.flatMap(_.subqueriesAll)
   }
 
+  /**
+   * Returns a sequence containing the result of applying a partial function to all elements in this
+   * plan, also considering all the plans in its (nested) subqueries
+   */
+  def collectInPlanAndSubqueries[B](f: PartialFunction[PlanType, B]): Seq[B] =
+    (this +: subqueriesAll).flatMap(_.collect(f))
+
   override def innerChildren: Seq[QueryPlan[_]] = subqueries
 
   /**

From 7c90ec065f81c3933eef1f0dd172f1a518b1232b Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 27 Mar 2020 15:45:35 -0700
Subject: [PATCH 0369/1280] [SPARK-31271][UI] fix web ui for driver side SQL
 metrics

### What changes were proposed in this pull request?

In https://github.com/apache/spark/pull/23551, we changed the metrics type of driver-side SQL metrics to size/time etc. which comes with max/min/median info.

This doesn't make sense for driver side SQL metrics as they have only one value. It makes the web UI hard to read:
![image](https://user-images.githubusercontent.com/3182036/77653892-42db9900-6fab-11ea-8e7f-92f763fa32ff.png)

This PR updates the SQL metrics UI to only display max/min/median if there are more than one metrics values:
![image](https://user-images.githubusercontent.com/3182036/77653975-5f77d100-6fab-11ea-849e-64c935377c8e.png)

### Why are the changes needed?

Makes the UI easier to read

### Does this PR introduce any user-facing change?

no

### How was this patch tested?
manual test

Closes #28037 from cloud-fan/ui.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c4e98c065c99d2cf840e6006ee5414fbaaba9937)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/execution/metric/SQLMetrics.scala     | 60 ++++++++++---------
 .../sql/execution/ui/SparkPlanGraph.scala     |  7 ++-
 .../execution/metric/SQLMetricsSuite.scala    | 33 +++++-----
 .../metric/SQLMetricsTestUtils.scala          | 12 ++--
 .../ui/SQLAppStatusListenerSuite.scala        |  9 ++-
 5 files changed, 68 insertions(+), 53 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
index 1394e0f723733..92d217983f4c5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/metric/SQLMetrics.scala
@@ -116,26 +116,23 @@ object SQLMetrics {
     // data size total (min, med, max):
     // 100GB (100MB, 1GB, 10GB)
     val acc = new SQLMetric(SIZE_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId: taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
   def createTimingMetric(sc: SparkContext, name: String): SQLMetric = {
     // The final result of this metric in physical operator UI may looks like:
-    // duration(min, med, max):
+    // duration total (min, med, max):
     // 5s (800ms, 1s, 2s)
     val acc = new SQLMetric(TIMING_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId: taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
   def createNanoTimingMetric(sc: SparkContext, name: String): SQLMetric = {
     // Same with createTimingMetric, just normalize the unit of time to millisecond.
     val acc = new SQLMetric(NS_TIMING_METRIC, -1)
-    acc.register(sc, name = Some(s"$name total (min, med, max (stageId: taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
@@ -150,8 +147,7 @@ object SQLMetrics {
     // probe avg (min, med, max):
     // (1.2, 2.2, 6.3)
     val acc = new SQLMetric(AVERAGE_METRIC)
-    acc.register(sc, name = Some(s"$name (min, med, max (stageId: taskId))"),
-      countFailedValues = false)
+    acc.register(sc, name = Some(name), countFailedValues = false)
     acc
   }
 
@@ -164,13 +160,15 @@ object SQLMetrics {
     metricsType != SUM_METRIC
   }
 
+  private val METRICS_NAME_SUFFIX = "(min, med, max (stageId: taskId))"
+
   /**
    * A function that defines how we aggregate the final accumulator results among all tasks,
    * and represent it in string for a SQL physical operator.
     */
   def stringValue(metricsType: String, values: Array[Long], maxMetrics: Array[Long]): String = {
-    // stringMetric = "(driver)" OR (stage ${stageId}.${attemptId}: task $taskId)
-    val stringMetric = if (maxMetrics.isEmpty) {
+    // taskInfo = "(driver)" OR (stage ${stageId}.${attemptId}: task $taskId)
+    val taskInfo = if (maxMetrics.isEmpty) {
       "(driver)"
     } else {
       s"(stage ${maxMetrics(1)}.${maxMetrics(2)}: task ${maxMetrics(3)})"
@@ -180,18 +178,20 @@ object SQLMetrics {
       numberFormat.format(values.sum)
     } else if (metricsType == AVERAGE_METRIC) {
       val validValues = values.filter(_ > 0)
-      val Seq(min, med, max) = {
-        val metric = if (validValues.isEmpty) {
-          val zeros = Seq.fill(3)(0L)
-          zeros.map(v => toNumberFormat(v))
-        } else {
+      // When there are only 1 metrics value (or None), no need to display max/min/median. This is
+      // common for driver-side SQL metrics.
+      if (validValues.length <= 1) {
+        toNumberFormat(validValues.headOption.getOrElse(0))
+      } else {
+        val Seq(min, med, max) = {
           Arrays.sort(validValues)
-          Seq(toNumberFormat(validValues(0)), toNumberFormat(validValues(validValues.length / 2)),
-            s"${toNumberFormat(validValues(validValues.length - 1))} $stringMetric")
+          Seq(
+            toNumberFormat(validValues(0)),
+            toNumberFormat(validValues(validValues.length / 2)),
+            toNumberFormat(validValues(validValues.length - 1)))
         }
-        metric
+        s"$METRICS_NAME_SUFFIX:\n($min, $med, $max $taskInfo)"
       }
-      s"\n($min, $med, $max)"
     } else {
       val strFormat: Long => String = if (metricsType == SIZE_METRIC) {
         Utils.bytesToString
@@ -204,19 +204,21 @@ object SQLMetrics {
       }
 
       val validValues = values.filter(_ >= 0)
-      val Seq(sum, min, med, max) = {
-        val metric = if (validValues.isEmpty) {
-          val zeros = Seq.fill(4)(0L)
-          zeros.map(v => strFormat(v))
-        } else {
+      // When there are only 1 metrics value (or None), no need to display max/min/median. This is
+      // common for driver-side SQL metrics.
+      if (validValues.length <= 1) {
+        strFormat(validValues.headOption.getOrElse(0))
+      } else {
+        val Seq(sum, min, med, max) = {
           Arrays.sort(validValues)
-          Seq(strFormat(validValues.sum), strFormat(validValues(0)),
+          Seq(
+            strFormat(validValues.sum),
+            strFormat(validValues(0)),
             strFormat(validValues(validValues.length / 2)),
-            s"${strFormat(validValues(validValues.length - 1))} $stringMetric")
+            strFormat(validValues(validValues.length - 1)))
         }
-        metric
+        s"total $METRICS_NAME_SUFFIX\n$sum ($min, $med, $max $taskInfo)"
       }
-      s"\n$sum ($min, $med, $max)"
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index 67628029d226b..274a5a414ffa2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -166,7 +166,12 @@ private[ui] class SparkPlanGraphNode(
       metric <- metrics
       value <- metricsValue.get(metric.accumulatorId)
     } yield {
-      metric.name + ": " + value
+      // The value may contain ":" to extend the name, like `total (min, med, max): ...`
+      if (value.contains(":")) {
+        metric.name + " " + value
+      } else {
+        metric.name + ": " + value
+      }
     }
 
     if (values.nonEmpty) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index 11f93c8836500..a5b07d56c03e6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -98,7 +98,7 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val ds = spark.range(10).filter('id < 5)
     testSparkPlanMetricsWithPredicates(ds.toDF(), 1, Map(
       0L -> (("WholeStageCodegen (1)", Map(
-        "duration total (min, med, max (stageId: taskId))" -> {
+        "duration" -> {
           _.toString.matches(timingMetricPattern)
         })))), true)
   }
@@ -110,10 +110,10 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df = testData2.groupBy().count() // 2 partitions
     val expected1 = Seq(
       Map("number of output rows" -> 2L,
-        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern),
       Map("number of output rows" -> 1L,
-        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern))
     val shuffleExpected1 = Map(
       "records read" -> 2L,
@@ -130,10 +130,10 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df2 = testData2.groupBy('a).count()
     val expected2 = Seq(
       Map("number of output rows" -> 4L,
-        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern),
       Map("number of output rows" -> 3L,
-        "avg hash probe bucket list iters (min, med, max (stageId: taskId))" ->
+        "avg hash probe bucket list iters" ->
           aggregateMetricsPattern))
 
     val shuffleExpected2 = Map(
@@ -181,12 +181,17 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
       }
       val metrics = getSparkPlanMetrics(df, 1, nodeIds, enableWholeStage).get
       nodeIds.foreach { nodeId =>
-        val probes =
-          metrics(nodeId)._2("avg hash probe bucket list iters (min, med, max (stageId: taskId))")
-        // Extract min, med, max from the string and strip off everthing else.
-        val index = probes.toString.stripPrefix("\n(").stripSuffix(")").indexOf(" (", 0)
-        probes.toString.stripPrefix("\n(").stripSuffix(")").slice(0, index).split(", ").foreach {
-          probe => assert(probe.toDouble > 1.0)
+        val probes = metrics(nodeId)._2("avg hash probe bucket list iters").toString
+        if (!probes.contains("\n")) {
+          // It's a single metrics value
+          assert(probes.toDouble > 1.0)
+        } else {
+          val mainValue = probes.split("\n").apply(1).stripPrefix("(").stripSuffix(")")
+          // Extract min, med, max from the string and strip off everthing else.
+          val index = mainValue.indexOf(" (", 0)
+          mainValue.slice(0, index).split(", ").foreach {
+            probe => assert(probe.toDouble > 1.0)
+          }
         }
       }
     }
@@ -231,13 +236,13 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
     val df = Seq(1, 3, 2).toDF("id").sort('id)
     testSparkPlanMetricsWithPredicates(df, 2, Map(
       0L -> (("Sort", Map(
-        "sort time total (min, med, max (stageId: taskId))" -> {
+        "sort time" -> {
           _.toString.matches(timingMetricPattern)
         },
-        "peak memory total (min, med, max (stageId: taskId))" -> {
+        "peak memory" -> {
           _.toString.matches(sizeMetricPattern)
         },
-        "spill size total (min, med, max (stageId: taskId))" -> {
+        "spill size" -> {
           _.toString.matches(sizeMetricPattern)
         })))
     ))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
index 766e7a9748510..2977b5339ab51 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsTestUtils.scala
@@ -46,23 +46,23 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
   protected val sizeMetricPattern = {
     val bytes = "([0-9]+(\\.[0-9]+)?) (EiB|PiB|TiB|GiB|MiB|KiB|B)"
     val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
-    s"\\n$bytes \\($bytes, $bytes, $bytes( $maxMetrics)?\\)"
+    s"(.*\\n$bytes \\($bytes, $bytes, $bytes( $maxMetrics)?\\))|($bytes)"
   }
 
   // Pattern of timing SQLMetric value, e.g. "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms (stage 3.0):
-  // task 217))" OR "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms)"
+  // task 217))" OR "\n2.0 ms (1.0 ms, 1.0 ms, 1.0 ms)" OR "1.0 ms"
   protected val timingMetricPattern = {
     val duration = "([0-9]+(\\.[0-9]+)?) (ms|s|m|h)"
     val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
-    s"\\n$duration \\($duration, $duration, $duration( $maxMetrics)?\\)"
+    s"(.*\\n$duration \\($duration, $duration, $duration( $maxMetrics)?\\))|($duration)"
   }
 
   // Pattern of size SQLMetric value for Aggregate tests.
-  // e.g "\n(1, 1, 0.9 (stage 1.0: task 8)) OR "\n(1, 1, 0.9 )"
+  // e.g "\n(1, 1, 0.9 (stage 1.0: task 8)) OR "\n(1, 1, 0.9 )" OR "1"
   protected val aggregateMetricsPattern = {
     val iters = "([0-9]+(\\.[0-9]+)?)"
     val maxMetrics = "\\(stage ([0-9])+\\.([0-9])+\\: task ([0-9])+\\)"
-    s"\\n\\($iters, $iters, $iters( $maxMetrics)?\\)"
+    s"(.*\\n\\($iters, $iters, $iters( $maxMetrics)?\\))|($iters)"
   }
 
   /**
@@ -98,7 +98,7 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
     }
 
     val totalNumBytesMetric = executedNode.metrics.find(
-      _.name == "written output total (min, med, max (stageId: taskId))").get
+      _.name == "written output").get
     val totalNumBytes = metrics(totalNumBytesMetric.accumulatorId).replaceAll(",", "")
       .split(" ").head.trim.toDouble
     assert(totalNumBytes > 0)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index 9f4a335309b63..949924e962bb9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -152,11 +152,14 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
       expected.foreach { case (id, value) =>
         // The values in actual can be SQL metrics meaning that they contain additional formatting
         // when converted to string. Verify that they start with the expected value.
-        // TODO: this is brittle. There is no requirement that the actual string needs to start
-        // with the accumulator value.
         assert(actual.contains(id))
         val v = actual(id).trim
-        assert(v.startsWith(value.toString), s"Wrong value for accumulator $id")
+        if (v.contains("\n")) {
+          // The actual value can be "total (max, ...)\n6 ms (5 ms, ...)".
+          assert(v.split("\n")(1).startsWith(value.toString), s"Wrong value for accumulator $id")
+        } else {
+          assert(v.startsWith(value.toString), s"Wrong value for accumulator $id")
+        }
       }
     }
 

From 4e13ba90446745fc5a9f46ed1f80c6eefb738795 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 27 Mar 2020 18:44:53 -0700
Subject: [PATCH 0370/1280] [SPARK-31238][SPARK-31284][TEST][FOLLOWUP] Fix
 readResourceOrcFile to create a local file from resource

### What changes were proposed in this pull request?

This PR aims to copy a test resource file to a local file in `OrcTest` suite before reading it.

### Why are the changes needed?

SPARK-31238 and SPARK-31284 added test cases to access the resouce file in `sql/core` module from `sql/hive` module. In **Maven** test environment, this causes a failure.
```
- SPARK-31238: compatibility with Spark 2.4 in reading dates *** FAILED ***
java.lang.IllegalArgumentException: java.net.URISyntaxException: Relative path in absolute URI:
jar:file:/home/jenkins/workspace/spark-master-test-maven-hadoop-3.2-hive-2.3-jdk-11/sql/core/target/spark-sql_2.12-3.1.0-SNAPSHOT-tests.jar!/test-data/before_1582_date_v2_4.snappy.orc
```

```
- SPARK-31284: compatibility with Spark 2.4 in reading timestamps *** FAILED ***
java.lang.IllegalArgumentException: java.net.URISyntaxException: Relative path in absolute URI:
jar:file:/home/jenkins/workspace/spark-master-test-maven-hadoop-3.2-hive-2.3/sql/core/target/spark-sql_2.12-3.1.0-SNAPSHOT-tests.jar!/test-data/before_1582_ts_v2_4.snappy.orc
```

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Pass the Jenkins with Maven.

Closes #28059 from dongjoon-hyun/SPARK-31238.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit d025ddbaa7e7b9746d8e47aeed61ed39d2f09f0e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/datasources/orc/OrcTest.scala      | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
index 16772fec87dfb..e929f904c798d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcTest.scala
@@ -22,6 +22,7 @@ import java.io.File
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.TypeTag
 
+import org.apache.commons.io.FileUtils
 import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.sql._
@@ -136,6 +137,10 @@ abstract class OrcTest extends QueryTest with FileBasedDataSourceTest with Befor
 
   protected def readResourceOrcFile(name: String): DataFrame = {
     val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.orc(url.toString)
+    // Copy to avoid URISyntaxException when `sql/hive` accesses the resources in `sql/core`
+    val file = File.createTempFile("orc-test", ".orc")
+    file.deleteOnExit();
+    FileUtils.copyURLToFile(url, file)
+    spark.read.orc(file.getAbsolutePath)
   }
 }

From 71dcf6691a48dd622b83e128aa9be30f757b45ec Mon Sep 17 00:00:00 2001
From: Kengo Seki <sekikn@apache.org>
Date: Sun, 29 Mar 2020 08:48:08 +0900
Subject: [PATCH 0371/1280] [SPARK-31292][CORE][SQL] Replace toSet.toSeq with
 distinct for readability

### What changes were proposed in this pull request?

This PR replaces the method calls of `toSet.toSeq` with `distinct`.

### Why are the changes needed?

`toSet.toSeq` is intended to make its elements unique but a bit verbose. Using `distinct` instead is easier to understand and improves readability.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Tested with the existing unit tests and found no problem.

Closes #28062 from sekikn/SPARK-31292.

Authored-by: Kengo Seki <sekikn@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 0b237bd615da4b2c2b781e72af4ad3a4f2951444)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../main/scala/org/apache/spark/resource/ResourceUtils.scala    | 2 +-
 core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala | 2 +-
 .../main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala  | 2 +-
 .../scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala    | 2 +-
 .../org/apache/spark/scheduler/TaskSchedulerImplSuite.scala     | 2 +-
 sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala      | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 7dd7fc1b99353..994b36386330c 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -149,7 +149,7 @@ private[spark] object ResourceUtils extends Logging {
   def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = {
     sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) =>
       key.substring(0, key.indexOf('.'))
-    }.toSet.toSeq.map(name => new ResourceID(componentName, name))
+    }.distinct.map(name => new ResourceID(componentName, name))
   }
 
   def parseAllResourceRequests(
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
index 857c89d7a98f5..15f2161fac39d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ResultTask.scala
@@ -69,7 +69,7 @@ private[spark] class ResultTask[T, U](
   with Serializable {
 
   @transient private[this] val preferredLocs: Seq[TaskLocation] = {
-    if (locs == null) Nil else locs.toSet.toSeq
+    if (locs == null) Nil else locs.distinct
   }
 
   override def runTask(context: TaskContext): U = {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
index 4c0c30a3caf67..a0ba9208ea647 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ShuffleMapTask.scala
@@ -71,7 +71,7 @@ private[spark] class ShuffleMapTask(
   }
 
   @transient private val preferredLocs: Seq[TaskLocation] = {
-    if (locs == null) Nil else locs.toSet.toSeq
+    if (locs == null) Nil else locs.distinct
   }
 
   override def runTask(context: TaskContext): MapStatus = {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 6a1d460e6a9d9..ed304736356c1 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -408,7 +408,7 @@ private[spark] class TaskSchedulerImpl(
         newExecAvail = true
       }
     }
-    val hosts = offers.map(_.host).toSet.toSeq
+    val hosts = offers.map(_.host).distinct
     for ((host, Some(rack)) <- hosts.zip(getRacksForHosts(hosts))) {
       hostsByRack.getOrElseUpdate(rack, new HashSet[String]()) += host
     }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
index e7ecf847ff4f4..a083cdb95b20f 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSchedulerImplSuite.scala
@@ -758,7 +758,7 @@ class TaskSchedulerImplSuite extends SparkFunSuite with LocalSparkContext with B
         // that are explicitly blacklisted, plus those that have *any* executors blacklisted.
         val nodesForBlacklistedExecutors = offers.filter { offer =>
           execBlacklist.contains(offer.executorId)
-        }.map(_.host).toSet.toSeq
+        }.map(_.host).distinct
         val nodesWithAnyBlacklisting = (nodeBlacklist ++ nodesForBlacklistedExecutors).toSet
         // Similarly, figure out which executors have any blacklisting.  This means all executors
         // that are explicitly blacklisted, plus all executors on nodes that are blacklisted.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index d85e23b82f9db..b910136e0706e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -2455,7 +2455,7 @@ class Dataset[T] private[sql](
   def dropDuplicates(colNames: Seq[String]): Dataset[T] = withTypedPlan {
     val resolver = sparkSession.sessionState.analyzer.resolver
     val allColumns = queryExecution.analyzed.output
-    val groupCols = colNames.toSet.toSeq.flatMap { (colName: String) =>
+    val groupCols = colNames.distinct.flatMap { (colName: String) =>
       // It is possibly there are more than one columns with the same name,
       // so we call filter instead of find.
       val cols = allColumns.filter(col => resolver(col.name, colName))

From 1c4fe31a5a697c80ca59ed6286ad1423d8541e6a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sat, 28 Mar 2020 18:36:34 -0700
Subject: [PATCH 0372/1280] [SPARK-30722][DOCS][FOLLOW-UP] Add Pandas Function
 API into the menu

### What changes were proposed in this pull request?

This PR adds "Pandas Function API" into the menu.

### Why are the changes needed?

To be consistent and to make easier to navigate.

### Does this PR introduce any user-facing change?

No, master only.

![Screen Shot 2020-03-27 at 11 40 29 PM](https://user-images.githubusercontent.com/6477701/77767405-60306600-7084-11ea-944a-93726259cd00.png)

### How was this patch tested?

Manually verified by `SKIP_API=1 jekyll build`.

Closes #28054 from HyukjinKwon/followup-spark-30722.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 34c7476cb5da98d5f3be354669dcd762df2b75e1)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/_data/menu-sql.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index c17bfd371f717..8a1d9374fba86 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -67,6 +67,8 @@
       url: sql-pyspark-pandas-with-arrow.html#enabling-for-conversion-tofrom-pandas
     - text: "Pandas UDFs (a.k.a. Vectorized UDFs)"
       url: sql-pyspark-pandas-with-arrow.html#pandas-udfs-aka-vectorized-udfs
+    - text: "Pandas Function APIs"
+      url: sql-pyspark-pandas-with-arrow.html#pandas-function-apis
     - text: Usage Notes
       url: sql-pyspark-pandas-with-arrow.html#usage-notes
 - text: Migration Guide

From 38c262b97e4fa10b249d51d3f69a0c97760492a3 Mon Sep 17 00:00:00 2001
From: Zhenhua Wang <wzh_zju@163.com>
Date: Sun, 29 Mar 2020 13:30:14 +0900
Subject: [PATCH 0373/1280] [SPARK-31261][SQL] Avoid npe when reading bad csv
 input with `columnNameCorruptRecord` specified

### What changes were proposed in this pull request?

SPARK-25387 avoids npe for bad csv input, but when reading bad csv input with `columnNameCorruptRecord` specified, `getCurrentInput` is called and it still throws npe.

### Why are the changes needed?

Bug fix.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Add a test.

Closes #28029 from wzhfy/corrupt_column_npe.

Authored-by: Zhenhua Wang <wzh_zju@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 791d2ba346f3358fc280adbbbe27f2cd50fd3732)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/catalyst/csv/UnivocityParser.scala   |  3 ++-
 .../sql/execution/datasources/csv/CSVSuite.scala   | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index 2c5a9d752f55f..8e87a82769471 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -101,7 +101,8 @@ class UnivocityParser(
 
   // Retrieve the raw record string.
   private def getCurrentInput: UTF8String = {
-    UTF8String.fromString(tokenizer.getContext.currentParsedContent().stripLineEnd)
+    val currentContent = tokenizer.getContext.currentParsedContent()
+    if (currentContent == null) null else UTF8String.fromString(currentContent.stripLineEnd)
   }
 
   // This parser first picks some tokens from the input tokens, according to the required schema,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index f9a510df83f45..366cf11871fa0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -1897,6 +1897,20 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
     assert(spark.read.csv(input).collect().toSet == Set(Row()))
   }
 
+  test("SPARK-31261: bad csv input with `columnNameCorruptRecord` should not cause NPE") {
+    val schema = StructType(
+      StructField("a", IntegerType) :: StructField("_corrupt_record", StringType) :: Nil)
+    val input = spark.createDataset(Seq("\u0000\u0000\u0001234"))
+
+    checkAnswer(
+      spark.read
+        .option("columnNameOfCorruptRecord", "_corrupt_record")
+        .schema(schema)
+        .csv(input),
+      Row(null, null))
+    assert(spark.read.csv(input).collect().toSet == Set(Row()))
+  }
+
   test("field names of inferred schema shouldn't compare to the first row") {
     val input = Seq("1,2").toDS()
     val df = spark.read.option("enforceSchema", false).csv(input)

From 6f8b3e0bea3c6639ba7648b41580198ad925683a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sun, 29 Mar 2020 13:59:18 +0900
Subject: [PATCH 0374/1280] [SPARK-31287][PYTHON][SQL] Ignore type hints in
 groupby.(cogroup.)applyInPandas and mapInPandas

### What changes were proposed in this pull request?

This PR proposes to make pandas function APIs (`groupby.(cogroup.)applyInPandas` and `mapInPandas`) to ignore Python type hints.

### Why are the changes needed?

Python type hints are optional. It shouldn't affect where pandas UDFs are not used.
This is also a future work for them to support other type hints. We shouldn't at least throw an exception at this moment.

### Does this PR introduce any user-facing change?

No, it's master-only change.

```python
import pandas as pd

def pandas_plus_one(pdf: pd.DataFrame) -> pd.DataFrame:
    return pdf + 1

spark.range(10).groupby('id').applyInPandas(pandas_plus_one, schema="id long").show()
```
```python
import pandas as pd

def pandas_plus_one(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:
    return left + 1

spark.range(10).groupby('id').cogroup(spark.range(10).groupby("id")).applyInPandas(pandas_plus_one, schema="id long").show()
```

```python
from typing import Iterator
import pandas as pd

def pandas_plus_one(iter: Iterator[pd.DataFrame]) -> Iterator[pd.DataFrame]:
    return map(lambda v: v + 1, iter)

spark.range(10).mapInPandas(pandas_plus_one, schema="id long").show()
```

**Before:**

Exception

**After:**

```
+---+
| id|
+---+
|  1|
|  2|
|  3|
|  4|
|  5|
|  6|
|  7|
|  8|
|  9|
| 10|
+---+
```

### How was this patch tested?

Closes #28052 from HyukjinKwon/SPARK-31287.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3165a95a04448546ae8955020566d718c6960223)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/pandas/functions.py        |  8 ++++
 .../sql/tests/test_pandas_udf_typehints.py    | 42 +++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py
index 31aa321bf5826..f43ebf8c5750a 100644
--- a/python/pyspark/sql/pandas/functions.py
+++ b/python/pyspark/sql/pandas/functions.py
@@ -384,6 +384,14 @@ def _create_pandas_udf(f, returnType, evalType):
                 "In Python 3.6+ and Spark 3.0+, it is preferred to specify type hints for "
                 "pandas UDF instead of specifying pandas UDF type which will be deprecated "
                 "in the future releases. See SPARK-28264 for more details.", UserWarning)
+        elif evalType in [PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF,
+                          PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,
+                          PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF]:
+            # In case of 'SQL_GROUPED_MAP_PANDAS_UDF',  deprecation warning is being triggered
+            # at `apply` instead.
+            # In case of 'SQL_MAP_PANDAS_ITER_UDF' and 'SQL_COGROUPED_MAP_PANDAS_UDF', the
+            # evaluation type will always be set.
+            pass
         elif len(argspec.annotations) > 0:
             evalType = infer_eval_type(signature(f))
             assert evalType is not None
diff --git a/python/pyspark/sql/tests/test_pandas_udf_typehints.py b/python/pyspark/sql/tests/test_pandas_udf_typehints.py
index 7c83c78f108d9..2582080056864 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_typehints.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_typehints.py
@@ -261,6 +261,48 @@ def test_group_agg_udf_type_hint(self):
         expected = df.groupby('id').agg(mean(df.v).alias('weighted_mean(v, 1.0)')).sort('id')
         assert_frame_equal(expected.toPandas(), actual.toPandas())
 
+    def test_ignore_type_hint_in_group_apply_in_pandas(self):
+        df = self.spark.range(10)
+        exec(
+            "def pandas_plus_one(v: pd.DataFrame) -> pd.DataFrame:\n"
+            "    return v + 1",
+            self.local)
+
+        pandas_plus_one = self.local["pandas_plus_one"]
+
+        actual = df.groupby('id').applyInPandas(pandas_plus_one, schema=df.schema).sort('id')
+        expected = df.selectExpr("id + 1 as id")
+        assert_frame_equal(expected.toPandas(), actual.toPandas())
+
+    def test_ignore_type_hint_in_cogroup_apply_in_pandas(self):
+        df = self.spark.range(10)
+        exec(
+            "def pandas_plus_one(left: pd.DataFrame, right: pd.DataFrame) -> pd.DataFrame:\n"
+            "    return left + 1",
+            self.local)
+
+        pandas_plus_one = self.local["pandas_plus_one"]
+
+        actual = df.groupby('id').cogroup(
+            self.spark.range(10).groupby("id")
+        ).applyInPandas(pandas_plus_one, schema=df.schema).sort('id')
+        expected = df.selectExpr("id + 1 as id")
+        assert_frame_equal(expected.toPandas(), actual.toPandas())
+
+    def test_ignore_type_hint_in_map_in_pandas(self):
+        df = self.spark.range(10)
+        exec(
+            "from typing import Iterator\n"
+            "def pandas_plus_one(iter: Iterator[pd.DataFrame]) -> Iterator[pd.DataFrame]:\n"
+            "    return map(lambda v: v + 1, iter)",
+            self.local)
+
+        pandas_plus_one = self.local["pandas_plus_one"]
+
+        actual = df.mapInPandas(pandas_plus_one, schema=df.schema)
+        expected = df.selectExpr("id + 1 as id")
+        assert_frame_equal(expected.toPandas(), actual.toPandas())
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_udf_typehints import *

From f375930d81337f2facbe5da71bb126d4d935e49d Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Sat, 28 Mar 2020 22:05:16 -0700
Subject: [PATCH 0375/1280] [SPARK-31087] [SQL] Add Back Multiple Removed APIs

### What changes were proposed in this pull request?

Based on the discussion in the mailing list [[Proposal] Modification to Spark's Semantic Versioning Policy](http://apache-spark-developers-list.1001551.n3.nabble.com/Proposal-Modification-to-Spark-s-Semantic-Versioning-Policy-td28938.html) , this PR is to add back the following APIs whose maintenance cost are relatively small.

- functions.toDegrees/toRadians
- functions.approxCountDistinct
- functions.monotonicallyIncreasingId
- Column.!==
- Dataset.explode
- Dataset.registerTempTable
- SQLContext.getOrCreate, setActive, clearActive, constructors

Below is the other removed APIs in the original PR, but not added back in this PR [https://issues.apache.org/jira/browse/SPARK-25908]:

- Remove some AccumulableInfo .apply() methods
- Remove non-label-specific multiclass precision/recall/fScore in favor of accuracy
- Remove unused Python StorageLevel constants
- Remove unused multiclass option in libsvm parsing
- Remove references to deprecated spark configs like spark.yarn.am.port
- Remove TaskContext.isRunningLocally
- Remove ShuffleMetrics.shuffle* methods
- Remove BaseReadWrite.context in favor of session

### Why are the changes needed?
Avoid breaking the APIs that are commonly used.

### Does this PR introduce any user-facing change?
Adding back the APIs that were removed in 3.0 branch does not introduce the user-facing changes, because Spark 3.0 has not been released.

### How was this patch tested?
Added a new test suite for these APIs.

Author: gatorsmile <gatorsmile@gmail.com>
Author: yi.wu <yi.wu@databricks.com>

Closes #27821 from gatorsmile/addAPIBackV2.

(cherry picked from commit 3884455780a214c620f309e00d5a083039746755)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 project/MimaExcludes.scala                    |   8 --
 python/pyspark/sql/dataframe.py               |  19 +++
 python/pyspark/sql/functions.py               |  11 ++
 .../scala/org/apache/spark/sql/Column.scala   |  18 +++
 .../scala/org/apache/spark/sql/Dataset.scala  |  98 +++++++++++++++
 .../org/apache/spark/sql/SQLContext.scala     |  50 +++++++-
 .../org/apache/spark/sql/functions.scala      |  79 ++++++++++++
 .../org/apache/spark/sql/DataFrameSuite.scala |  46 +++++++
 .../apache/spark/sql/DeprecatedAPISuite.scala | 114 ++++++++++++++++++
 .../apache/spark/sql/SQLContextSuite.scala    |  30 ++++-
 10 files changed, 458 insertions(+), 15 deletions(-)

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 9a5029e1c1a68..d1ed48a11e67a 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -235,14 +235,6 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.ShuffleWriteMetrics.shuffleWriteTime"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.ShuffleWriteMetrics.shuffleRecordsWritten"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.AccumulableInfo.apply"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.approxCountDistinct"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.toRadians"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.toDegrees"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.functions.monotonicallyIncreasingId"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.clearActive"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.getOrCreate"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.SQLContext.setActive"),
-    ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.sql.SQLContext.this"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.MulticlassMetrics.fMeasure"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.MulticlassMetrics.recall"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.evaluation.MulticlassMetrics.precision"),
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 44cb264714482..2a366dcb9fcbb 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -122,6 +122,25 @@ def toJSON(self, use_unicode=True):
         rdd = self._jdf.toJSON()
         return RDD(rdd.toJavaRDD(), self._sc, UTF8Deserializer(use_unicode))
 
+    @since(1.3)
+    def registerTempTable(self, name):
+        """Registers this DataFrame as a temporary table using the given name.
+
+        The lifetime of this temporary table is tied to the :class:`SparkSession`
+        that was used to create this :class:`DataFrame`.
+
+        >>> df.registerTempTable("people")
+        >>> df2 = spark.sql("select * from people")
+        >>> sorted(df.collect()) == sorted(df2.collect())
+        True
+        >>> spark.catalog.dropTempView("people")
+
+        .. note:: Deprecated in 2.0, use createOrReplaceTempView instead.
+        """
+        warnings.warn(
+            "Deprecated in 2.0, use createOrReplaceTempView instead.", DeprecationWarning)
+        self._jdf.createOrReplaceTempView(name)
+
     @since(2.0)
     def createTempView(self, name):
         """Creates a local temporary view with this :class:`DataFrame`.
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 61a221ca24b95..543bb7ffeeac5 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -296,6 +296,8 @@ def _options_to_str(options):
 
 # Wraps deprecated functions (keys) with the messages (values).
 _functions_deprecated = {
+    'toDegrees': 'Deprecated in 2.1, use degrees instead.',
+    'toRadians': 'Deprecated in 2.1, use radians instead.',
 }
 
 for _name, _doc in _functions.items():
@@ -319,6 +321,15 @@ def _options_to_str(options):
 del _name, _doc
 
 
+@since(1.3)
+def approxCountDistinct(col, rsd=None):
+    """
+    .. note:: Deprecated in 2.1, use :func:`approx_count_distinct` instead.
+    """
+    warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", DeprecationWarning)
+    return approx_count_distinct(col, rsd)
+
+
 @since(2.1)
 def approx_count_distinct(col, rsd=None):
     """Aggregate function: returns a new :class:`Column` for approximate distinct count of
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 8bd5835fd931b..49c9f830fb27e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -319,6 +319,24 @@ class Column(val expr: Expression) extends Logging {
     */
   def =!= (other: Any): Column = withExpr{ Not(EqualTo(expr, lit(other).expr)) }
 
+  /**
+   * Inequality test.
+   * {{{
+   *   // Scala:
+   *   df.select( df("colA") !== df("colB") )
+   *   df.select( !(df("colA") === df("colB")) )
+   *
+   *   // Java:
+   *   import static org.apache.spark.sql.functions.*;
+   *   df.filter( col("colA").notEqual(col("colB")) );
+   * }}}
+   *
+   * @group expr_ops
+   * @since 1.3.0
+    */
+  @deprecated("!== does not have the same precedence as ===, use =!= instead", "2.0.0")
+  def !== (other: Any): Column = this =!= other
+
   /**
    * Inequality test.
    * {{{
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index b910136e0706e..5c3a82afabb73 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -22,6 +22,7 @@ import java.io.{ByteArrayOutputStream, CharArrayWriter, DataOutputStream}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.language.implicitConversions
+import scala.reflect.runtime.universe.TypeTag
 import scala.util.control.NonFatal
 
 import org.apache.commons.lang3.StringUtils
@@ -34,6 +35,7 @@ import org.apache.spark.api.python.{PythonRDD, SerDeUtil}
 import org.apache.spark.api.r.RRDD
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.QueryPlanningTracker
 import org.apache.spark.sql.catalyst.analysis._
 import org.apache.spark.sql.catalyst.catalog.HiveTableRelation
@@ -2265,6 +2267,90 @@ class Dataset[T] private[sql](
     randomSplit(weights.toArray, seed)
   }
 
+  /**
+   * (Scala-specific) Returns a new Dataset where each row has been expanded to zero or more
+   * rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. The columns of
+   * the input row are implicitly joined with each row that is output by the function.
+   *
+   * Given that this is deprecated, as an alternative, you can explode columns either using
+   * `functions.explode()` or `flatMap()`. The following example uses these alternatives to count
+   * the number of books that contain a given word:
+   *
+   * {{{
+   *   case class Book(title: String, words: String)
+   *   val ds: Dataset[Book]
+   *
+   *   val allWords = ds.select('title, explode(split('words, " ")).as("word"))
+   *
+   *   val bookCountPerWord = allWords.groupBy("word").agg(countDistinct("title"))
+   * }}}
+   *
+   * Using `flatMap()` this can similarly be exploded as:
+   *
+   * {{{
+   *   ds.flatMap(_.words.split(" "))
+   * }}}
+   *
+   * @group untypedrel
+   * @since 2.0.0
+   */
+  @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0")
+  def explode[A <: Product : TypeTag](input: Column*)(f: Row => TraversableOnce[A]): DataFrame = {
+    val elementSchema = ScalaReflection.schemaFor[A].dataType.asInstanceOf[StructType]
+
+    val convert = CatalystTypeConverters.createToCatalystConverter(elementSchema)
+
+    val rowFunction =
+      f.andThen(_.map(convert(_).asInstanceOf[InternalRow]))
+    val generator = UserDefinedGenerator(elementSchema, rowFunction, input.map(_.expr))
+
+    withPlan {
+      Generate(generator, unrequiredChildIndex = Nil, outer = false,
+        qualifier = None, generatorOutput = Nil, logicalPlan)
+    }
+  }
+
+  /**
+   * (Scala-specific) Returns a new Dataset where a single column has been expanded to zero
+   * or more rows by the provided function. This is similar to a `LATERAL VIEW` in HiveQL. All
+   * columns of the input row are implicitly joined with each value that is output by the function.
+   *
+   * Given that this is deprecated, as an alternative, you can explode columns either using
+   * `functions.explode()`:
+   *
+   * {{{
+   *   ds.select(explode(split('words, " ")).as("word"))
+   * }}}
+   *
+   * or `flatMap()`:
+   *
+   * {{{
+   *   ds.flatMap(_.words.split(" "))
+   * }}}
+   *
+   * @group untypedrel
+   * @since 2.0.0
+   */
+  @deprecated("use flatMap() or select() with functions.explode() instead", "2.0.0")
+  def explode[A, B : TypeTag](inputColumn: String, outputColumn: String)(f: A => TraversableOnce[B])
+    : DataFrame = {
+    val dataType = ScalaReflection.schemaFor[B].dataType
+    val attributes = AttributeReference(outputColumn, dataType)() :: Nil
+    // TODO handle the metadata?
+    val elementSchema = attributes.toStructType
+
+    def rowFunction(row: Row): TraversableOnce[InternalRow] = {
+      val convert = CatalystTypeConverters.createToCatalystConverter(dataType)
+      f(row(0).asInstanceOf[A]).map(o => InternalRow(convert(o)))
+    }
+    val generator = UserDefinedGenerator(elementSchema, rowFunction, apply(inputColumn).expr :: Nil)
+
+    withPlan {
+      Generate(generator, unrequiredChildIndex = Nil, outer = false,
+        qualifier = None, generatorOutput = Nil, logicalPlan)
+    }
+  }
+
   /**
    * Returns a new Dataset by adding a column or replacing the existing column that has
    * the same name.
@@ -3129,6 +3215,18 @@ class Dataset[T] private[sql](
    */
   def javaRDD: JavaRDD[T] = toJavaRDD
 
+  /**
+   * Registers this Dataset as a temporary table using the given name. The lifetime of this
+   * temporary table is tied to the [[SparkSession]] that was used to create this Dataset.
+   *
+   * @group basic
+   * @since 1.6.0
+   */
+  @deprecated("Use createOrReplaceTempView(viewName) instead.", "2.0.0")
+  def registerTempTable(tableName: String): Unit = {
+    createOrReplaceTempView(tableName)
+  }
+
   /**
    * Creates a local temporary view using the given name. The lifetime of this
    * temporary view is tied to the [[SparkSession]] that was used to create this Dataset.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index bbcc842877f74..68ce82d5badda 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -24,7 +24,7 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.{SparkConf, SparkContext}
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Stable, Unstable}
-import org.apache.spark.api.java.JavaRDD
+import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.ConfigEntry
 import org.apache.spark.rdd.RDD
@@ -64,6 +64,15 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   // Note: Since Spark 2.0 this class has become a wrapper of SparkSession, where the
   // real functionality resides. This class remains mainly for backward compatibility.
+
+  @deprecated("Use SparkSession.builder instead", "2.0.0")
+  def this(sc: SparkContext) = {
+    this(SparkSession.builder().sparkContext(sc).getOrCreate())
+  }
+
+  @deprecated("Use SparkSession.builder instead", "2.0.0")
+  def this(sparkContext: JavaSparkContext) = this(sparkContext.sc)
+
   // TODO: move this logic into SparkSession
 
   private[sql] def sessionState: SessionState = sparkSession.sessionState
@@ -997,6 +1006,45 @@ class SQLContext private[sql](val sparkSession: SparkSession)
  */
 object SQLContext {
 
+  /**
+   * Get the singleton SQLContext if it exists or create a new one using the given SparkContext.
+   *
+   * This function can be used to create a singleton SQLContext object that can be shared across
+   * the JVM.
+   *
+   * If there is an active SQLContext for current thread, it will be returned instead of the global
+   * one.
+   *
+   * @since 1.5.0
+   */
+  @deprecated("Use SparkSession.builder instead", "2.0.0")
+  def getOrCreate(sparkContext: SparkContext): SQLContext = {
+    SparkSession.builder().sparkContext(sparkContext).getOrCreate().sqlContext
+  }
+
+  /**
+   * Changes the SQLContext that will be returned in this thread and its children when
+   * SQLContext.getOrCreate() is called. This can be used to ensure that a given thread receives
+   * a SQLContext with an isolated session, instead of the global (first created) context.
+   *
+   * @since 1.6.0
+   */
+  @deprecated("Use SparkSession.setActiveSession instead", "2.0.0")
+  def setActive(sqlContext: SQLContext): Unit = {
+    SparkSession.setActiveSession(sqlContext.sparkSession)
+  }
+
+  /**
+   * Clears the active SQLContext for current thread. Subsequent calls to getOrCreate will
+   * return the first created context instead of a thread-local override.
+   *
+   * @since 1.6.0
+   */
+  @deprecated("Use SparkSession.clearActiveSession instead", "2.0.0")
+  def clearActive(): Unit = {
+    SparkSession.clearActiveSession()
+  }
+
   /**
    * Converts an iterator of Java Beans to InternalRow using the provided
    * bean info & schema. This is not related to the singleton, but is a static
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index cad7916bad73e..0ca4238e2c1f2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -211,6 +211,36 @@ object functions {
   // Aggregate functions
   //////////////////////////////////////////////////////////////////////////////////////////////
 
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(e: Column): Column = approx_count_distinct(e)
+
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName)
+
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd)
+
+  /**
+   * @group agg_funcs
+   * @since 1.3.0
+   */
+  @deprecated("Use approx_count_distinct", "2.1.0")
+  def approxCountDistinct(columnName: String, rsd: Double): Column = {
+    approx_count_distinct(Column(columnName), rsd)
+  }
+
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
@@ -1090,6 +1120,27 @@ object functions {
    */
   def isnull(e: Column): Column = withExpr { IsNull(e.expr) }
 
+  /**
+   * A column expression that generates monotonically increasing 64-bit integers.
+   *
+   * The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive.
+   * The current implementation puts the partition ID in the upper 31 bits, and the record number
+   * within each partition in the lower 33 bits. The assumption is that the data frame has
+   * less than 1 billion partitions, and each partition has less than 8 billion records.
+   *
+   * As an example, consider a `DataFrame` with two partitions, each with 3 records.
+   * This expression would return the following IDs:
+   *
+   * {{{
+   * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.
+   * }}}
+   *
+   * @group normal_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use monotonically_increasing_id()", "2.0.0")
+  def monotonicallyIncreasingId(): Column = monotonically_increasing_id()
+
   /**
    * A column expression that generates monotonically increasing 64-bit integers.
    *
@@ -2071,6 +2122,20 @@ object functions {
    */
   def tanh(columnName: String): Column = tanh(Column(columnName))
 
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use degrees", "2.1.0")
+  def toDegrees(e: Column): Column = degrees(e)
+
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use degrees", "2.1.0")
+  def toDegrees(columnName: String): Column = degrees(Column(columnName))
+
   /**
    * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
    *
@@ -2093,6 +2158,20 @@ object functions {
    */
   def degrees(columnName: String): Column = degrees(Column(columnName))
 
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use radians", "2.1.0")
+  def toRadians(e: Column): Column = radians(e)
+
+  /**
+   * @group math_funcs
+   * @since 1.4.0
+   */
+  @deprecated("Use radians", "2.1.0")
+  def toRadians(columnName: String): Column = radians(Column(columnName))
+
   /**
    * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
    *
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 72aa7bf084c00..1762bc61d1fc7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -109,6 +109,31 @@ class DataFrameSuite extends QueryTest
     dfAlias.col("t2.c")
   }
 
+  test("simple explode") {
+    val df = Seq(Tuple1("a b c"), Tuple1("d e")).toDF("words")
+
+    checkAnswer(
+      df.explode("words", "word") { word: String => word.split(" ").toSeq }.select('word),
+      Row("a") :: Row("b") :: Row("c") :: Row("d") ::Row("e") :: Nil
+    )
+  }
+
+  test("explode") {
+    val df = Seq((1, "a b c"), (2, "a b"), (3, "a")).toDF("number", "letters")
+    val df2 =
+      df.explode('letters) {
+        case Row(letters: String) => letters.split(" ").map(Tuple1(_)).toSeq
+      }
+
+    checkAnswer(
+      df2
+        .select('_1 as 'letter, 'number)
+        .groupBy('letter)
+        .agg(countDistinct('number)),
+      Row("a", 3) :: Row("b", 2) :: Row("c", 1) :: Nil
+    )
+  }
+
   test("Star Expansion - CreateStruct and CreateArray") {
     val structDf = testData2.select("a", "b").as("record")
     // CreateStruct and CreateArray in aggregateExpressions
@@ -185,6 +210,27 @@ class DataFrameSuite extends QueryTest
     }
   }
 
+  test("Star Expansion - ds.explode should fail with a meaningful message if it takes a star") {
+    val df = Seq(("1", "1,2"), ("2", "4"), ("3", "7,8,9")).toDF("prefix", "csv")
+    val e = intercept[AnalysisException] {
+      df.explode($"*") { case Row(prefix: String, csv: String) =>
+        csv.split(",").map(v => Tuple1(prefix + ":" + v)).toSeq
+      }.queryExecution.assertAnalyzed()
+    }
+    assert(e.getMessage.contains("Invalid usage of '*' in explode/json_tuple/UDTF"))
+
+    checkAnswer(
+      df.explode('prefix, 'csv) { case Row(prefix: String, csv: String) =>
+        csv.split(",").map(v => Tuple1(prefix + ":" + v)).toSeq
+      },
+      Row("1", "1,2", "1:1") ::
+        Row("1", "1,2", "1:2") ::
+        Row("2", "4", "2:4") ::
+        Row("3", "7,8,9", "3:7") ::
+        Row("3", "7,8,9", "3:8") ::
+        Row("3", "7,8,9", "3:9") :: Nil)
+  }
+
   test("Star Expansion - explode should fail with a meaningful message if it takes a star") {
     val df = Seq(("1,2"), ("4"), ("7,8,9")).toDF("csv")
     val e = intercept[AnalysisException] {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
index c31ef996f0362..25b8849d61248 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DeprecatedAPISuite.scala
@@ -17,10 +17,124 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
 
 class DeprecatedAPISuite extends QueryTest with SharedSparkSession {
+  import MathFunctionsTestData.DoubleData
+  import testImplicits._
+
+  private lazy val doubleData = (1 to 10).map(i => DoubleData(i * 0.2 - 1, i * -0.2 + 1)).toDF()
+
+  private def testOneToOneMathFunction[
+    @specialized(Int, Long, Float, Double) T,
+    @specialized(Int, Long, Float, Double) U](
+        c: Column => Column,
+        f: T => U): Unit = {
+    checkAnswer(
+      doubleData.select(c('a)),
+      (1 to 10).map(n => Row(f((n * 0.2 - 1).asInstanceOf[T])))
+    )
+
+    checkAnswer(
+      doubleData.select(c('b)),
+      (1 to 10).map(n => Row(f((-n * 0.2 + 1).asInstanceOf[T])))
+    )
+
+    checkAnswer(
+      doubleData.select(c(lit(null))),
+      (1 to 10).map(_ => Row(null))
+    )
+  }
+
+  test("functions.toDegrees") {
+    testOneToOneMathFunction(toDegrees, math.toDegrees)
+    withView("t") {
+      val df = Seq(0, 1, 1.5).toDF("a")
+      df.createOrReplaceTempView("t")
+
+      checkAnswer(
+        sql("SELECT degrees(0), degrees(1), degrees(1.5)"),
+        Seq(0).toDF().select(toDegrees(lit(0)), toDegrees(lit(1)), toDegrees(lit(1.5)))
+      )
+      checkAnswer(
+        sql("SELECT degrees(a) FROM t"),
+        df.select(toDegrees("a"))
+      )
+    }
+  }
+
+  test("functions.toRadians") {
+    testOneToOneMathFunction(toRadians, math.toRadians)
+    withView("t") {
+      val df = Seq(0, 1, 1.5).toDF("a")
+      df.createOrReplaceTempView("t")
+
+      checkAnswer(
+        sql("SELECT radians(0), radians(1), radians(1.5)"),
+        Seq(0).toDF().select(toRadians(lit(0)), toRadians(lit(1)), toRadians(lit(1.5)))
+      )
+      checkAnswer(
+        sql("SELECT radians(a) FROM t"),
+        df.select(toRadians("a"))
+      )
+    }
+  }
+
+  test("functions.approxCountDistinct") {
+    withView("t") {
+      val df = Seq(0, 1, 2).toDF("a")
+      df.createOrReplaceTempView("t")
+      checkAnswer(
+        sql("SELECT approx_count_distinct(a) FROM t"),
+        df.select(approxCountDistinct("a")))
+    }
+  }
+
+  test("functions.monotonicallyIncreasingId") {
+    // Make sure we have 2 partitions, each with 2 records.
+    val df = sparkContext.parallelize(Seq[Int](), 2).mapPartitions { _ =>
+      Iterator(Tuple1(1), Tuple1(2))
+    }.toDF("a")
+    checkAnswer(
+      df.select(monotonicallyIncreasingId(), expr("monotonically_increasing_id()")),
+      Row(0L, 0L) ::
+        Row(1L, 1L) ::
+        Row((1L << 33) + 0L, (1L << 33) + 0L) ::
+        Row((1L << 33) + 1L, (1L << 33) + 1L) :: Nil
+    )
+  }
+
+  test("Column.!==") {
+    val nullData = Seq(
+      (Some(1), Some(1)), (Some(1), Some(2)), (Some(1), None), (None, None)).toDF("a", "b")
+    checkAnswer(
+      nullData.filter($"b" !== 1),
+      Row(1, 2) :: Nil)
+
+    checkAnswer(nullData.filter($"b" !== null), Nil)
+
+    checkAnswer(
+      nullData.filter($"a" !== $"b"),
+      Row(1, 2) :: Nil)
+  }
+
+  test("Dataset.registerTempTable") {
+    withTempView("t") {
+      Seq(1).toDF().registerTempTable("t")
+      assert(spark.catalog.tableExists("t"))
+    }
+  }
+
+  test("SQLContext.setActive/clearActive") {
+    val sc = spark.sparkContext
+    val sqlContext = new SQLContext(sc)
+    SQLContext.setActive(sqlContext)
+    assert(SparkSession.getActiveSession === Some(spark))
+    SQLContext.clearActive()
+    assert(SparkSession.getActiveSession === None)
+  }
 
   test("SQLContext.applySchema") {
     val rowRdd = sparkContext.parallelize(Seq(Row("Jack", 20), Row("Marry", 18)))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
index aab2ae4afc7f5..a1799829932b8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLContextSuite.scala
@@ -24,14 +24,32 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{BooleanType, StringType, StructField, StructType}
 
+@deprecated("This suite is deprecated to silent compiler deprecation warnings", "2.0.0")
 class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
 
   object DummyRule extends Rule[LogicalPlan] {
     def apply(p: LogicalPlan): LogicalPlan = p
   }
 
+  test("getOrCreate instantiates SQLContext") {
+    val sqlContext = SQLContext.getOrCreate(sc)
+    assert(sqlContext != null, "SQLContext.getOrCreate returned null")
+    assert(SQLContext.getOrCreate(sc).eq(sqlContext),
+      "SQLContext created by SQLContext.getOrCreate not returned by SQLContext.getOrCreate")
+  }
+
+  test("getOrCreate return the original SQLContext") {
+    val sqlContext = SQLContext.getOrCreate(sc)
+    val newSession = sqlContext.newSession()
+    assert(SQLContext.getOrCreate(sc).eq(sqlContext),
+      "SQLContext.getOrCreate after explicitly created SQLContext did not return the context")
+    SparkSession.setActiveSession(newSession.sparkSession)
+    assert(SQLContext.getOrCreate(sc).eq(newSession),
+      "SQLContext.getOrCreate after explicitly setActive() did not return the active context")
+  }
+
   test("Sessions of SQLContext") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val session1 = sqlContext.newSession()
     val session2 = sqlContext.newSession()
 
@@ -59,13 +77,13 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
   }
 
   test("Catalyst optimization passes are modifiable at runtime") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     sqlContext.experimental.extraOptimizations = Seq(DummyRule)
     assert(sqlContext.sessionState.optimizer.batches.flatMap(_.rules).contains(DummyRule))
   }
 
   test("get all tables") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val df = sqlContext.range(10)
     df.createOrReplaceTempView("listtablessuitetable")
     assert(
@@ -82,7 +100,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
   }
 
   test("getting all tables with a database name has no impact on returned table names") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val df = sqlContext.range(10)
     df.createOrReplaceTempView("listtablessuitetable")
     assert(
@@ -99,7 +117,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
   }
 
   test("query the returned DataFrame of tables") {
-    val sqlContext = SparkSession.builder().sparkContext(sc).getOrCreate().sqlContext
+    val sqlContext = SQLContext.getOrCreate(sc)
     val df = sqlContext.range(10)
     df.createOrReplaceTempView("listtablessuitetable")
 
@@ -109,7 +127,7 @@ class SQLContextSuite extends SparkFunSuite with SharedSparkContext {
         StructField("isTemporary", BooleanType, false) :: Nil)
 
     Seq(sqlContext.tables(), sqlContext.sql("SHOW TABLes")).foreach {
-      tableDF =>
+      case tableDF =>
         assert(expectedSchema === tableDF.schema)
 
         tableDF.createOrReplaceTempView("tables")

From 702f75dbde6fd96f3bcebf5cfab00c35020e77ae Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 29 Mar 2020 11:19:24 -0500
Subject: [PATCH 0376/1280] [SPARK-30363][SQL][DOCS][FOLLOWUP] Fix a broken
 link in SQL Reference

### What changes were proposed in this pull request?
Fix a broken link and make the relevant docs reference to the new doc

### Why are the changes needed?

### Does this PR introduce any user-facing change?
Yes, make CACHE TABLE, UNCACHE TABLE, CLEAR CACHE, REFRESH TABLE link to the new doc

### How was this patch tested?
Manually build and check

Closes #28065 from huaxingao/spark-30363-follow-up.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit e656e990610be257bc1696e31ec969648abccc12)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                       | 2 +-
 docs/sql-ref-syntax-aux-cache-cache-table.md   | 1 +
 docs/sql-ref-syntax-aux-cache-clear-cache.md   | 1 +
 docs/sql-ref-syntax-aux-cache-uncache-table.md | 1 +
 docs/sql-ref-syntax-aux-cache.md               | 1 +
 docs/sql-ref-syntax-aux-refresh-table.md       | 1 +
 6 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 8a1d9374fba86..3bf4952bebbcc 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -176,7 +176,7 @@
                 - text: REFRESH TABLE
                   url: sql-ref-syntax-aux-refresh-table.html
                 - text: REFRESH
-                  url: sql-ref-syntax-aux-cache-refresh.md
+                  url: sql-ref-syntax-aux-cache-refresh.html
             - text: DESCRIBE
               url: sql-ref-syntax-aux-describe.html
               subitems:
diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index 20ade1961ab0b..27cc77b938fbe 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -88,4 +88,5 @@ CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testDat
   * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
   * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
   * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+  * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
 
diff --git a/docs/sql-ref-syntax-aux-cache-clear-cache.md b/docs/sql-ref-syntax-aux-cache-clear-cache.md
index 57ba77d59ea7b..15ba3c787c177 100644
--- a/docs/sql-ref-syntax-aux-cache-clear-cache.md
+++ b/docs/sql-ref-syntax-aux-cache-clear-cache.md
@@ -36,4 +36,5 @@ CLEAR CACHE;
  * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
  * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
 
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index 69e21c258a333..7e4b8fbc35aa8 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -49,3 +49,4 @@ UNCACHE TABLE t1;
  * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
  * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache.md b/docs/sql-ref-syntax-aux-cache.md
index e4a640a6edbe5..1a48fceeb1f6b 100644
--- a/docs/sql-ref-syntax-aux-cache.md
+++ b/docs/sql-ref-syntax-aux-cache.md
@@ -23,3 +23,4 @@ license: |
 * [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
 * [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
 * [REFRESH TABLE statement](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-refresh-table.md
index 96d698a1d68b0..b248ee67fa12a 100644
--- a/docs/sql-ref-syntax-aux-refresh-table.md
+++ b/docs/sql-ref-syntax-aux-refresh-table.md
@@ -56,3 +56,4 @@ REFRESH TABLE tempDB.view1;
 - [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 - [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
 - [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+- [REFRESH](sql-ref-syntax-aux-cache-refresh.html)

From 83f943c4b922413159cdcc09dfee7ef67d586215 Mon Sep 17 00:00:00 2001
From: Kengo Seki <sekikn@apache.org>
Date: Sun, 29 Mar 2020 14:27:19 -0700
Subject: [PATCH 0377/1280] [SPARK-31293][DSTREAMS][KINESIS][DOC] Fix wrong
 examples and help messages for Kinesis integration

### What changes were proposed in this pull request?

This PR (SPARK-31293) fixes wrong command examples, parameter descriptions and help message format for Amazon Kinesis integration with Spark Streaming.

### Why are the changes needed?

To improve usability of those commands.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

I ran the fixed commands manually and confirmed they worked as expected.

Closes #28063 from sekikn/SPARK-31293.

Authored-by: Kengo Seki <sekikn@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 60dd1a690fed62b1d6442cdc8cf3f89ef4304d5a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/streaming-kinesis-integration.md                  |  3 +--
 .../python/examples/streaming/kinesis_wordcount_asl.py |  7 ++++---
 .../spark/examples/streaming/KinesisWordCountASL.scala | 10 +++++-----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index e68d51321cae3..db813c46949c2 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -246,8 +246,7 @@ To run the example,
 	</div>
 	<div data-lang="python" markdown="1">
 
-        ./bin/spark-submit --jars external/kinesis-asl/target/scala-*/\
-            spark-streaming-kinesis-asl-assembly_*.jar \
+        ./bin/spark-submit --jars 'external/kinesis-asl-assembly/target/spark-streaming-kinesis-asl-assembly_*.jar' \
             external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
             [Kinesis app name] [Kinesis stream name] [endpoint URL] [region name]
 
diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 49794faab88c4..777a33270c415 100644
--- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -26,6 +26,7 @@
     <stream-name> name of the Kinesis stream (ie. mySparkStream)
     <endpoint-url> endpoint of the Kinesis service
       (e.g. https://kinesis.us-east-1.amazonaws.com)
+    <region-name> region name of the Kinesis endpoint (e.g. us-east-1)
 
 
   Example:
@@ -34,10 +35,10 @@
       $ export AWS_SECRET_KEY=<your-secret-key>
 
       # run the example
-      $ bin/spark-submit -jars external/kinesis-asl/target/scala-*/\
-        spark-streaming-kinesis-asl-assembly_*.jar \
+      $ bin/spark-submit --jars \
+        'external/kinesis-asl-assembly/target/spark-streaming-kinesis-asl-assembly_*.jar' \
         external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py \
-        myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com
+        myAppName mySparkStream https://kinesis.us-east-1.amazonaws.com us-east-1
 
   There is a companion helper class called KinesisWordProducerASL which puts dummy data
   onto the Kinesis stream.
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index a5d5ac769b28d..32f4a6759474f 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -78,7 +78,7 @@ object KinesisWordCountASL extends Logging {
     if (args.length != 3) {
       System.err.println(
         """
-          |Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url> <region-name>
+          |Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url>
           |
           |    <app-name> is the name of the consumer app, used to track the read data in DynamoDB
           |    <stream-name> is the name of the Kinesis stream
@@ -171,11 +171,11 @@ object KinesisWordCountASL extends Logging {
  *   <endpoint-url> is the endpoint of the Kinesis service
  *     (ie. https://kinesis.us-east-1.amazonaws.com)
  *   <records-per-sec> is the rate of records per second to put onto the stream
- *   <words-per-record> is the rate of records per second to put onto the stream
+ *   <words-per-record> is the number of words per record
  *
  * Example:
  *    $ SPARK_HOME/bin/run-example streaming.KinesisWordProducerASL mySparkStream \
- *         https://kinesis.us-east-1.amazonaws.com us-east-1 10 5
+ *         https://kinesis.us-east-1.amazonaws.com 10 5
  */
 object KinesisWordProducerASL {
   def main(args: Array[String]): Unit = {
@@ -183,13 +183,13 @@ object KinesisWordProducerASL {
       System.err.println(
         """
           |Usage: KinesisWordProducerASL <stream-name> <endpoint-url> <records-per-sec>
-                                         <words-per-record>
+          |                              <words-per-record>
           |
           |    <stream-name> is the name of the Kinesis stream
           |    <endpoint-url> is the endpoint of the Kinesis service
           |                   (e.g. https://kinesis.us-east-1.amazonaws.com)
           |    <records-per-sec> is the rate of records per second to put onto the stream
-          |    <words-per-record> is the rate of records per second to put onto the stream
+          |    <words-per-record> is the number of words per record
           |
         """.stripMargin)
 

From 8d61a141d9e899c521f37b79c78d3a29fd8401fe Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 30 Mar 2020 12:20:11 +0800
Subject: [PATCH 0378/1280] [SPARK-31286][SQL][DOC] Specify formats of time
 zone ID for JSON/CSV option and from/to_utc_timestamp

### What changes were proposed in this pull request?
In the PR, I propose to update the doc for the `timeZone` option in JSON/CSV datasources and for the `tz` parameter of the `from_utc_timestamp()`/`to_utc_timestamp()` functions, and to restrict format of config's values to 2 forms:
1. Geographical regions, such as `America/Los_Angeles`.
2. Fixed offsets - a fully resolved offset from UTC. For example, `-08:00`.

### Why are the changes needed?
Other formats such as three-letter time zone IDs are ambitious, and depend on the locale. For example, `CST` could be U.S. `Central Standard Time` and `China Standard Time`. Such formats have been already deprecated in JDK, see [Three-letter time zone IDs](https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html).

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running `./dev/scalastyle`, and manual testing.

Closes #28051 from MaxGekk/doc-time-zone-option.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d2ff5c5bfb29a7b22df3cb49b6584c3e2bec397c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 R/pkg/R/functions.R                           |  8 ++-
 python/pyspark/sql/functions.py               | 14 ++++-
 python/pyspark/sql/readwriter.py              | 60 +++++++++++++++----
 python/pyspark/sql/streaming.py               | 60 +++++++++++++++----
 .../apache/spark/sql/DataFrameReader.scala    | 45 ++++++++++++--
 .../apache/spark/sql/DataFrameWriter.scala    | 45 ++++++++++++--
 .../org/apache/spark/sql/functions.scala      | 16 +++--
 .../sql/streaming/DataStreamReader.scala      | 45 ++++++++++++--
 .../sql/streaming/DataStreamWriter.scala      | 45 ++++++++++++--
 9 files changed, 283 insertions(+), 55 deletions(-)

diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index dd5dbbc500e57..d8b045041e115 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -77,7 +77,13 @@ NULL
 #'          days to be added to or subtracted from \code{y}. For class \code{character}, it is
 #'          \itemize{
 #'          \item \code{date_format}: date format specification.
-#'          \item \code{from_utc_timestamp}, \code{to_utc_timestamp}: time zone to use.
+#'          \item \code{from_utc_timestamp}, \code{to_utc_timestamp}: A string detailing
+#'            the time zone ID that the input should be adjusted to. It should be in the format
+#'            of either region-based zone IDs or zone offsets. Region IDs must have the form
+#'            'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format
+#'            (+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported
+#'            as aliases of '+00:00'. Other short names are not recommended to use
+#'            because they can be ambiguous.
 #'          \item \code{next_day}: day of the week string.
 #'          }
 #' @param ... additional argument(s).
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 543bb7ffeeac5..1ade21ccb42a0 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -1313,7 +1313,12 @@ def from_utc_timestamp(timestamp, tz):
     timestamp to string according to the session local timezone.
 
     :param timestamp: the column that contains timestamps
-    :param tz: a string that has the ID of timezone, e.g. "GMT", "America/Los_Angeles", etc
+    :param tz: A string detailing the time zone ID that the input should be adjusted to. It should
+               be in the format of either region-based zone IDs or zone offsets. Region IDs must
+               have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+               the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+               supported as aliases of '+00:00'. Other short names are not recommended to use
+               because they can be ambiguous.
 
     .. versionchanged:: 2.4
        `tz` can take a :class:`Column` containing timezone ID strings.
@@ -1347,7 +1352,12 @@ def to_utc_timestamp(timestamp, tz):
     timestamp to string according to the session local timezone.
 
     :param timestamp: the column that contains timestamps
-    :param tz: a string that has the ID of timezone, e.g. "GMT", "America/Los_Angeles", etc
+    :param tz: A string detailing the time zone ID that the input should be adjusted to. It should
+               be in the format of either region-based zone IDs or zone offsets. Region IDs must
+               have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+               the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+               supported as aliases of '+00:00'. Other short names are not recommended to use
+               because they can be ambiguous.
 
     .. versionchanged:: 2.4
        `tz` can take a :class:`Column` containing timezone ID strings.
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 817978498fcf2..92d36e7c3b1ae 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -105,9 +105,18 @@ def option(self, key, value):
         """Adds an input option for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
             * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching
                 the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter.
                 It does not change the behavior of partition discovery.
@@ -120,9 +129,18 @@ def options(self, **options):
         """Adds input options for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
             * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching
                 the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter.
                 It does not change the behavior of partition discovery.
@@ -665,9 +683,18 @@ def option(self, key, value):
         """Adds an output option for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
         """
         self._jwrite = self._jwrite.option(key, to_str(value))
         return self
@@ -677,9 +704,18 @@ def options(self, **options):
         """Adds output options for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
         """
         for k in options:
             self._jwrite = self._jwrite.option(k, to_str(options[k]))
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index a5e86466579c4..4d36a04c038c7 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -339,9 +339,18 @@ def option(self, key, value):
         """Adds an input option for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
         .. note:: Evolving.
 
@@ -355,9 +364,18 @@ def options(self, **options):
         """Adds input options for the underlying data source.
 
         You can set the following option(s) for reading files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to parse timestamps
-                in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to parse
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
         .. note:: Evolving.
 
@@ -812,9 +830,18 @@ def option(self, key, value):
         """Adds an output option for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
         .. note:: Evolving.
         """
@@ -826,9 +853,18 @@ def options(self, **options):
         """Adds output options for the underlying data source.
 
         You can set the following option(s) for writing files:
-            * ``timeZone``: sets the string that indicates a timezone to be used to format
-                timestamps in the JSON/CSV datasources or partition values.
-                If it isn't set, it uses the default value, session local timezone.
+            * ``timeZone``: sets the string that indicates a time zone ID to be used to format
+                timestamps in the JSON/CSV datasources or partition values. The following
+                formats of `timeZone` are supported:
+
+                * Region-based zone ID: It should have the form 'area/city', such as \
+                  'America/Los_Angeles'.
+                * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \
+                 '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
+
+                Other short names like 'CST' are not recommended to use because they can be
+                ambiguous. If it isn't set, the current value of the SQL config
+                ``spark.sql.session.timeZone`` is used by default.
 
        .. note:: Evolving.
         """
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index a2a3518f2509c..83e567834313a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -96,8 +96,19 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -133,8 +144,19 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -149,8 +171,19 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 11feae9170d88..7e669e0faccd3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -107,8 +107,19 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -144,8 +155,19 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
@@ -160,8 +182,19 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 1.4.0
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 0ca4238e2c1f2..8a89a3b93561a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -3136,8 +3136,12 @@ object functions {
    *
    * @param ts A date, timestamp or string. If a string, the data must be in a format that can be
    *           cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
-   * @param tz A string detailing the time zone that the input should be adjusted to, such as
-   *           `Europe/London`, `PST` or `GMT+5`
+   * @param tz A string detailing the time zone ID that the input should be adjusted to. It should
+   *           be in the format of either region-based zone IDs or zone offsets. Region IDs must
+   *           have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+   *           the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+   *           supported as aliases of '+00:00'. Other short names are not recommended to use
+   *           because they can be ambiguous.
    * @return A timestamp, or null if `ts` was a string that could not be cast to a timestamp or
    *         `tz` was an invalid value
    * @group datetime_funcs
@@ -3165,8 +3169,12 @@ object functions {
    *
    * @param ts A date, timestamp or string. If a string, the data must be in a format that can be
    *           cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
-   * @param tz A string detailing the time zone that the input belongs to, such as `Europe/London`,
-   *           `PST` or `GMT+5`
+   * @param tz A string detailing the time zone ID that the input should be adjusted to. It should
+   *           be in the format of either region-based zone IDs or zone offsets. Region IDs must
+   *           have the form 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in
+   *           the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are
+   *           supported as aliases of '+00:00'. Other short names are not recommended to use
+   *           because they can be ambiguous.
    * @return A timestamp, or null if `ts` was a string that could not be cast to a timestamp or
    *         `tz` was an invalid value
    * @group datetime_funcs
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index be7f021cb2465..a2eaed8693a4b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -81,8 +81,19 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -118,8 +129,19 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV data sources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -134,8 +156,19 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to parse timestamps in the JSON/CSV data sources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
index 1c21a30dd5bd6..1d0ca4d9453a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
@@ -161,8 +161,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -198,8 +209,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0
@@ -214,8 +236,19 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
    *
    * You can set the following option(s):
    * <ul>
-   * <li>`timeZone` (default session local timezone): sets the string that indicates a timezone
-   * to be used to format timestamps in the JSON/CSV datasources or partition values.</li>
+   * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
+   * to be used to format timestamps in the JSON/CSV datasources or partition values. The following
+   * formats of `timeZone` are supported:
+   *   <ul>
+   *     <li> Region-based zone ID: It should have the form 'area/city', such as
+   *         'America/Los_Angeles'.</li>
+   *     <li> Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00'
+   *          or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.</li>
+   *   </ul>
+   * Other short names like 'CST' are not recommended to use because they can be ambiguous.
+   * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is
+   * used by default.
+   * </li>
    * </ul>
    *
    * @since 2.0.0

From 5687b31be3fd1ff991a2e3d918a2dcd79ce1449c Mon Sep 17 00:00:00 2001
From: Oleksii Kachaiev <kachayev@gmail.com>
Date: Mon, 30 Mar 2020 13:20:57 +0800
Subject: [PATCH 0379/1280] [SPARK-30532] DataFrameStatFunctions to work with
 TABLE.COLUMN syntax

### What changes were proposed in this pull request?
`DataFrameStatFunctions` now works correctly with fully qualified column name (Table.Column syntax) by properly resolving the name instead of relying on field names from schema, notably:
* `approxQuantile`
* `freqItems`
* `cov`
* `corr`

(other functions from `DataFrameStatFunctions` already work correctly).

See code examples below.

### Why are the changes needed?
With current implementation some stat functions are impossible to use when joining datasets with similar column names.

### Does this PR introduce any user-facing change?
Yes. Before the change, the following code would fail with `AnalysisException`.

```scala
scala> val df1 = sc.parallelize(0 to 10).toDF("num").as("table1")
df1: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [num: int]

scala> val df2 = sc.parallelize(0 to 10).toDF("num").as("table2")
df2: org.apache.spark.sql.Dataset[org.apache.spark.sql.Row] = [num: int]

scala> val dfx = df2.crossJoin(df1)
dfx: org.apache.spark.sql.DataFrame = [num: int, num: int]

scala> dfx.stat.approxQuantile("table1.num", Array(0.1), 0.0)
res0: Array[Double] = Array(1.0)

scala> dfx.stat.corr("table1.num", "table2.num")
res1: Double = 1.0

scala> dfx.stat.cov("table1.num", "table2.num")
res2: Double = 11.0

scala> dfx.stat.freqItems(Array("table1.num", "table2.num"))
res3: org.apache.spark.sql.DataFrame = [table1.num_freqItems: array<int>, table2.num_freqItems: array<int>]
```

### How was this patch tested?
Corresponding unit tests are added to `DataFrameStatSuite.scala` (marked as "SPARK-30532").

Closes #27916 from kachayev/fix-spark-30532.

Authored-by: Oleksii Kachaiev <kachayev@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 22bb6b0fddb3ecd3ac0ad2b41a5024c86b8a6fc7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/stat/FrequentItems.scala    |  4 +--
 .../sql/execution/stat/StatFunctions.scala    |  9 +++----
 .../apache/spark/sql/DataFrameStatSuite.scala | 26 +++++++++++++++++++
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
index 6f1b67801bb80..bcd226f95f822 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/FrequentItems.scala
@@ -113,10 +113,8 @@ object FrequentItems extends Logging {
     val justItems = freqItems.map(m => m.baseMap.keys.toArray)
     val resultRow = Row(justItems : _*)
 
-    val originalSchema = df.schema
     val outputCols = cols.map { name =>
-      val index = originalSchema.fieldIndex(name)
-      val originalField = originalSchema.fields(index)
+      val originalField = df.resolve(name)
 
       // append frequent Items to the column name for easy debugging
       StructField(name + "_freqItems", ArrayType(originalField.dataType, originalField.nullable))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
index fffd8805a6525..5094e5eab5955 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
@@ -70,7 +70,7 @@ object StatFunctions extends Logging {
     require(relativeError >= 0,
       s"Relative Error must be non-negative but got $relativeError")
     val columns: Seq[Column] = cols.map { colName =>
-      val field = df.schema(colName)
+      val field = df.resolve(colName)
       require(field.dataType.isInstanceOf[NumericType],
         s"Quantile calculation for column $colName with data type ${field.dataType}" +
         " is not supported.")
@@ -154,10 +154,9 @@ object StatFunctions extends Logging {
               functionName: String): CovarianceCounter = {
     require(cols.length == 2, s"Currently $functionName calculation is supported " +
       "between two columns.")
-    cols.map(name => (name, df.schema.fields.find(_.name == name))).foreach { case (name, data) =>
-      require(data.nonEmpty, s"Couldn't find column with name $name")
-      require(data.get.dataType.isInstanceOf[NumericType], s"Currently $functionName calculation " +
-        s"for columns with dataType ${data.get.dataType.catalogString} not supported.")
+    cols.map(name => (name, df.resolve(name))).foreach { case (name, data) =>
+      require(data.dataType.isInstanceOf[NumericType], s"Currently $functionName calculation " +
+        s"for columns with dataType ${data.dataType.catalogString} not supported.")
     }
     val columns = cols.map(n => Column(Cast(Column(n).expr, DoubleType)))
     df.select(columns: _*).queryExecution.toRdd.treeAggregate(new CovarianceCounter)(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
index 394bad751b5ce..19601726fa393 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameStatSuite.scala
@@ -126,6 +126,32 @@ class DataFrameStatSuite extends QueryTest with SharedSparkSession {
     assert(math.abs(corr3 - 0.95723391394758572) < 1e-12)
   }
 
+  test("SPARK-30532 stat functions to understand fully-qualified column name") {
+    val df1 = spark.sparkContext.parallelize(0 to 10).toDF("num").as("table1")
+    val df2 = spark.sparkContext.parallelize(0 to 10).toDF("num").as("table2")
+    val dfx = df2.crossJoin(df1)
+
+    assert(dfx.stat.corr("table1.num", "table2.num") != 0.0)
+    assert(dfx.stat.cov("table1.num", "table2.num") != 0.0)
+    assert(dfx.stat.approxQuantile("table1.num", Array(0.1), 0.0).length == 1)
+    assert(dfx.stat.approxQuantile("table2.num", Array(0.1), 0.0).length == 1)
+    assert(dfx.stat.freqItems(Array("table1.num", "table2.num")).collect()(0).length == 2)
+
+    // this should throw "Reference 'num' is ambiguous"
+    intercept[AnalysisException] {
+      dfx.stat.freqItems(Array("num"))
+    }
+    intercept[AnalysisException] {
+      dfx.stat.approxQuantile("num", Array(0.1), 0.0)
+    }
+    intercept[AnalysisException] {
+      dfx.stat.cov("num", "num")
+    }
+    intercept[AnalysisException] {
+      dfx.stat.corr("num", "num")
+    }
+  }
+
   test("covariance") {
     val df = Seq.tabulate(10)(i => (i, 2.0 * i, toLetter(i))).toDF("singles", "doubles", "letters")
 

From 6550d0d5283efdbbd838f3aeaf0476c7f52a0fb1 Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@apache.org>
Date: Mon, 30 Mar 2020 08:42:10 +0000
Subject: [PATCH 0380/1280] Preparing Spark release v3.0.0-rc1

---
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 2 +-
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 38 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/assembly/pom.xml b/assembly/pom.xml
index 193ad3d671bcf..0a52a00c4a694 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index a1c8a8e6582eb..fa4fcb1fbfa80 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 163c250054e4d..14a1b7db9e33d 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index a6d99813a8501..e75a843f5e6bb 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 76a402bb2bd31..004af0a1ab203 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 3c3c0d2d96a1c..a35156a4bbfff 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 883b73a69c9de..dedc7dfc82056 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 93a4f67fd23f2..ebb05257d85f9 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 11faf6b1b0e38..ad743a7712812 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index a888620139207..a5c3106c4e261 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,7 +14,7 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.0-SNAPSHOT
+SPARK_VERSION: 3.0.0
 SPARK_VERSION_SHORT: 3.0.0
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
diff --git a/examples/pom.xml b/examples/pom.xml
index a099f1e042e99..ec3490545132d 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index ba6f20bfdbf58..70cada7ce95e4 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index cdf76e9a5f69d..32679c759404e 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index f2dcf5d217a89..ae22afc3faeae 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 693820da6af6b..7b10f0e78bc28 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index 01ca96b4f8f40..b2ecc0f81d7ac 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index ad7a8b7e23f1d..19e7352e3945c 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 0ce922349ea66..bb37b213c697b 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 7d69764b77de7..cfe7b7373b967 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index db64b201abc2c..d2f828421e64f 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 444568a03d6c7..ebfaccffcdded 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 02734e82ed26e..8a62a48236aab 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index e75e8345cd51d..c2f25df2f147e 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index 2eab868ac0dc8..bb4b640c68fe9 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 11769ef548d7c..c41bbe6ea85f8 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index cc4d500d599d9..d61c68c239eb1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.0-SNAPSHOT</version>
+  <version>3.0.0</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 1abc41279ebe8..8011a5f7a3423 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.0.dev0"
+__version__ = "3.0.0"
diff --git a/repl/pom.xml b/repl/pom.xml
index 3aee53cc9599a..83c8d34a3a132 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 8c46738b259bc..d49f0c80be1c1 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 369dfd491826c..261bacce5391d 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index 107ba365aa775..c61f8dff8cf55 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 0e5df14e060da..f18f2ce58588a 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 304e3a7457fcc..14a0f665ea84e 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 37da614034a09..3388aca287aa3 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 75c7f77942396..6dec472392574 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index c37582386347b..267bd6039e3fa 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 87af6388e1118..d6f4f303ad423 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index e380e869f55c7..4a58881077524 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From fc5079841907443369af98b17c20f1ac24b3727d Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@apache.org>
Date: Mon, 30 Mar 2020 08:42:27 +0000
Subject: [PATCH 0381/1280] Preparing development version 3.0.1-SNAPSHOT

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index c8cb1c3a992ad..3eff30b576748 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.0
+Version: 3.0.1
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0a52a00c4a694..8bef9d8bc56cb 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index fa4fcb1fbfa80..fc1441d80b7bf 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 14a1b7db9e33d..de2a6fb18d69b 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index e75a843f5e6bb..6c0c016fdf94f 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 004af0a1ab203..b8df19152a606 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index a35156a4bbfff..8119709a6aeff 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index dedc7dfc82056..96ac525880213 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index ebb05257d85f9..f385f07f951ce 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index ad743a7712812..0c9befc921af6 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index a5c3106c4e261..db2f6721e0cc4 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,8 +14,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.0
-SPARK_VERSION_SHORT: 3.0.0
+SPARK_VERSION: 3.0.1-SNAPSHOT
+SPARK_VERSION_SHORT: 3.0.1
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index ec3490545132d..1b59c629a8c73 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 70cada7ce95e4..87183e72f5de3 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 32679c759404e..b57e3c0e25ae8 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index ae22afc3faeae..9426db5e11354 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 7b10f0e78bc28..d77364d0210fe 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index b2ecc0f81d7ac..70a3406701315 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 19e7352e3945c..a338b6511298f 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index bb37b213c697b..599ae172671e5 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index cfe7b7373b967..9fa18d9f82833 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index d2f828421e64f..acfc2c5a36b1a 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index ebfaccffcdded..f0666536bb706 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 8a62a48236aab..48f559e4822f3 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index c2f25df2f147e..0759ce5c114b3 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index bb4b640c68fe9..ed34a370a8274 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index c41bbe6ea85f8..b8ae344e63e65 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index d61c68c239eb1..e5c8724cf6ae4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.0</version>
+  <version>3.0.1-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 8011a5f7a3423..906298ff9964a 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.0"
+__version__ = "3.0.1.dev0"
diff --git a/repl/pom.xml b/repl/pom.xml
index 83c8d34a3a132..b826b87f53efc 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index d49f0c80be1c1..9a48bf37b2cd5 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 261bacce5391d..52412570a2788 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index c61f8dff8cf55..e0c997f869aba 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index f18f2ce58588a..95299d2066ea3 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 14a0f665ea84e..6667c7bd089dc 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 3388aca287aa3..fbfcdf88fa26e 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 6dec472392574..f60820dd45f34 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 267bd6039e3fa..e8c1598b3669c 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d6f4f303ad423..1e99b6c21522b 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 4a58881077524..5430a3e600365 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 3ab2f8877e4618031b3258cb90eb249145077e08 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 30 Mar 2020 16:46:31 +0800
Subject: [PATCH 0382/1280] [SPARK-31296][SQL][TESTS] Benchmark date-time
 rebasing in Parquet datasource

### What changes were proposed in this pull request?
In the PR, I propose to add new benchmark `DateTimeRebaseBenchmark` which should measure the performance of rebasing of dates/timestamps from/to to the hybrid calendar (Julian+Gregorian) to/from Proleptic Gregorian calendar:
1. In write, it saves separately dates and timestamps before and after 1582 year w/ and w/o rebasing.
2. In read, it loads previously saved parquet files by vectorized reader and by regular reader.

Here is the summary of benchmarking:
- Saving timestamps is **~6 times slower**
- Loading timestamps w/ vectorized **off** is **~4 times slower**
- Loading timestamps w/ vectorized **on** is **~10 times slower**

### Why are the changes needed?
To know the impact of date-time rebasing introduced by #27915, #27953, #27807.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Run the `DateTimeRebaseBenchmark` benchmark using Amazon EC2:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK8/11 |

Closes #28057 from MaxGekk/rebase-bechmark.

Lead-authored-by: Maxim Gekk <max.gekk@gmail.com>
Co-authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a1dbcd13a3eeaee50cc1a46e909f9478d6d55177)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeRebaseBenchmark-jdk11-results.txt |  53 ++++++
 .../DateTimeRebaseBenchmark-results.txt       |  53 ++++++
 .../benchmark/DateTimeRebaseBenchmark.scala   | 161 ++++++++++++++++++
 3 files changed, 267 insertions(+)
 create mode 100644 sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
 create mode 100644 sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala

diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..52522f8f88c7a
--- /dev/null
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -0,0 +1,53 @@
+================================================================================================
+Rebasing dates/timestamps in Parquet datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   9272           9272           0         10.8          92.7       1.0X
+before 1582, noop                                  9142           9142           0         10.9          91.4       1.0X
+after 1582, rebase off                            21841          21841           0          4.6         218.4       0.4X
+after 1582, rebase on                             58245          58245           0          1.7         582.4       0.2X
+before 1582, rebase off                           19813          19813           0          5.0         198.1       0.5X
+before 1582, rebase on                            63737          63737           0          1.6         637.4       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off, rebase off                   13004          13063          67          7.7         130.0       1.0X
+after 1582, vec off, rebase on                    36224          36253          26          2.8         362.2       0.4X
+after 1582, vec on, rebase off                     3596           3654          54         27.8          36.0       3.6X
+after 1582, vec on, rebase on                     26144          26253         112          3.8         261.4       0.5X
+before 1582, vec off, rebase off                  12872          12914          51          7.8         128.7       1.0X
+before 1582, vec off, rebase on                   37762          37904         153          2.6         377.6       0.3X
+before 1582, vec on, rebase off                    3522           3592          94         28.4          35.2       3.7X
+before 1582, vec on, rebase on                    27580          27615          59          3.6         275.8       0.5X
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   3113           3113           0         32.1          31.1       1.0X
+before 1582, noop                                  3078           3078           0         32.5          30.8       1.0X
+after 1582, rebase off                            15749          15749           0          6.3         157.5       0.2X
+after 1582, rebase on                             69106          69106           0          1.4         691.1       0.0X
+before 1582, rebase off                           15967          15967           0          6.3         159.7       0.2X
+before 1582, rebase on                            76843          76843           0          1.3         768.4       0.0X
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off, rebase off                   15070          15172          94          6.6         150.7       1.0X
+after 1582, vec off, rebase on                    43748          43867         157          2.3         437.5       0.3X
+after 1582, vec on, rebase off                     4805           4859          60         20.8          48.1       3.1X
+after 1582, vec on, rebase on                     33960          34027          61          2.9         339.6       0.4X
+before 1582, vec off, rebase off                  15037          15071          52          6.7         150.4       1.0X
+before 1582, vec off, rebase on                   44590          44749         156          2.2         445.9       0.3X
+before 1582, vec on, rebase off                    4831           4852          30         20.7          48.3       3.1X
+before 1582, vec on, rebase on                    35460          35481          18          2.8         354.6       0.4X
+
+
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
new file mode 100644
index 0000000000000..c9320cfe660f9
--- /dev/null
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -0,0 +1,53 @@
+================================================================================================
+Rebasing dates/timestamps in Parquet datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   9472           9472           0         10.6          94.7       1.0X
+before 1582, noop                                  9226           9226           0         10.8          92.3       1.0X
+after 1582, rebase off                            21201          21201           0          4.7         212.0       0.4X
+after 1582, rebase on                             56471          56471           0          1.8         564.7       0.2X
+before 1582, rebase off                           20179          20179           0          5.0         201.8       0.5X
+before 1582, rebase on                            65717          65717           0          1.5         657.2       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off, rebase off                   12294          12434         205          8.1         122.9       1.0X
+after 1582, vec off, rebase on                    36959          36967          12          2.7         369.6       0.3X
+after 1582, vec on, rebase off                     3644           3691          49         27.4          36.4       3.4X
+after 1582, vec on, rebase on                     26764          26852          92          3.7         267.6       0.5X
+before 1582, vec off, rebase off                  12830          12917          85          7.8         128.3       1.0X
+before 1582, vec off, rebase on                   38897          39053         229          2.6         389.0       0.3X
+before 1582, vec on, rebase off                    3638           3693          85         27.5          36.4       3.4X
+before 1582, vec on, rebase on                    28956          29007          44          3.5         289.6       0.4X
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   2952           2952           0         33.9          29.5       1.0X
+before 1582, noop                                  2880           2880           0         34.7          28.8       1.0X
+after 1582, rebase off                            15928          15928           0          6.3         159.3       0.2X
+after 1582, rebase on                             82816          82816           0          1.2         828.2       0.0X
+before 1582, rebase off                           15988          15988           0          6.3         159.9       0.2X
+before 1582, rebase on                            92636          92636           0          1.1         926.4       0.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off, rebase off                   14863          14917          78          6.7         148.6       1.0X
+after 1582, vec off, rebase on                    54819          54939         140          1.8         548.2       0.3X
+after 1582, vec on, rebase off                     4905           4941          32         20.4          49.0       3.0X
+after 1582, vec on, rebase on                     44914          45008         124          2.2         449.1       0.3X
+before 1582, vec off, rebase off                  14928          14970          48          6.7         149.3       1.0X
+before 1582, vec off, rebase on                   59752          59996         245          1.7         597.5       0.2X
+before 1582, vec on, rebase off                    4892           4916          33         20.4          48.9       3.0X
+before 1582, vec on, rebase on                    46854          46977         198          2.1         468.5       0.3X
+
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
new file mode 100644
index 0000000000000..983d9b4f709f4
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneOffset}
+
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Synthetic benchmark for rebasing of date and timestamp in read/write.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/DateTimeRebaseBenchmark-results.txt".
+ * }}}
+ */
+object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
+  import spark.implicits._
+
+  private def genTs(cardinality: Int, start: LocalDateTime, end: LocalDateTime): DataFrame = {
+    val startSec = start.toEpochSecond(ZoneOffset.UTC)
+    val endSec = end.toEpochSecond(ZoneOffset.UTC)
+    spark.range(0, cardinality, 1, 1)
+      .select((($"id" % (endSec - startSec)) + startSec).as("seconds"))
+      .select($"seconds".cast("timestamp").as("ts"))
+  }
+
+  private def genTsAfter1582(cardinality: Int): DataFrame = {
+    val start = LocalDateTime.of(1582, 10, 15, 0, 0, 0)
+    val end = LocalDateTime.of(3000, 1, 1, 0, 0, 0)
+    genTs(cardinality, start, end)
+  }
+
+  private def genTsBefore1582(cardinality: Int): DataFrame = {
+    val start = LocalDateTime.of(10, 1, 1, 0, 0, 0)
+    val end = LocalDateTime.of(1580, 1, 1, 0, 0, 0)
+    genTs(cardinality, start, end)
+  }
+
+  private def genDate(cardinality: Int, start: LocalDate, end: LocalDate): DataFrame = {
+    val startSec = LocalDateTime.of(start, LocalTime.MIDNIGHT).toEpochSecond(ZoneOffset.UTC)
+    val endSec = LocalDateTime.of(end, LocalTime.MIDNIGHT).toEpochSecond(ZoneOffset.UTC)
+    spark.range(0, cardinality * SECONDS_PER_DAY, SECONDS_PER_DAY, 1)
+      .select((($"id" % (endSec - startSec)) + startSec).as("seconds"))
+      .select($"seconds".cast("timestamp").as("ts"))
+      .select($"ts".cast("date").as("date"))
+  }
+
+  private def genDateAfter1582(cardinality: Int): DataFrame = {
+    val start = LocalDate.of(1582, 10, 15)
+    val end = LocalDate.of(3000, 1, 1)
+    genDate(cardinality, start, end)
+  }
+
+  private def genDateBefore1582(cardinality: Int): DataFrame = {
+    val start = LocalDate.of(10, 1, 1)
+    val end = LocalDate.of(1580, 1, 1)
+    genDate(cardinality, start, end)
+  }
+
+  private def genDF(cardinality: Int, dateTime: String, after1582: Boolean): DataFrame = {
+    (dateTime, after1582) match {
+      case ("date", true) => genDateAfter1582(cardinality)
+      case ("date", false) => genDateBefore1582(cardinality)
+      case ("timestamp", true) => genTsAfter1582(cardinality)
+      case ("timestamp", false) => genTsBefore1582(cardinality)
+      case _ => throw new IllegalArgumentException(
+        s"cardinality = $cardinality dateTime = $dateTime after1582 = $after1582")
+    }
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    withTempPath { path =>
+      runBenchmark("Rebasing dates/timestamps in Parquet datasource") {
+        val rowsNum = 100000000
+        Seq("date", "timestamp").foreach { dateTime =>
+          val benchmark = new Benchmark(s"Save ${dateTime}s to parquet", rowsNum, output = output)
+          benchmark.addCase("after 1582, noop", 1) { _ =>
+            genDF(rowsNum, dateTime, after1582 = true).noop()
+          }
+          benchmark.addCase("before 1582, noop", 1) { _ =>
+            genDF(rowsNum, dateTime, after1582 = false).noop()
+          }
+
+          def save(after1582: Boolean, rebase: Boolean): Unit = {
+            val period = if (after1582) "after" else "before"
+            val rebaseFlag = if (rebase) "on" else "off"
+            val caseName = s"$period 1582, rebase $rebaseFlag"
+            benchmark.addCase(caseName, 1) { _ =>
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> rebase.toString) {
+                val df = genDF(rowsNum, dateTime, after1582)
+                val pathToWrite = path.getAbsolutePath + s"/${dateTime}_${period}_1582_$rebaseFlag"
+                df.write
+                  .mode("overwrite")
+                  .format("parquet")
+                  .save(pathToWrite)
+              }
+            }
+          }
+
+          Seq(true, false).foreach { after1582 =>
+            Seq(false, true).foreach { rebase =>
+              save(after1582, rebase)
+            }
+          }
+          benchmark.run()
+
+          val benchmark2 = new Benchmark(
+            s"Load ${dateTime}s from parquet", rowsNum, output = output)
+
+          def load(after1582: Boolean, vec: Boolean, rebase: Boolean): Unit = {
+            val period = if (after1582) "after" else "before"
+            val rebaseFlag = if (rebase) "on" else "off"
+            val vecFlag = if (vec) "on" else "off"
+            val caseName = s"$period 1582, vec $vecFlag, rebase $rebaseFlag"
+            benchmark2.addCase(caseName, 3) { _ =>
+              withSQLConf(
+                SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
+                SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> rebase.toString) {
+                val pathToRead = path.getAbsolutePath + s"/${dateTime}_${period}_1582_$rebaseFlag"
+                spark.read.format("parquet").load(pathToRead).noop()
+              }
+            }
+          }
+
+          Seq(true, false).foreach { after1582 =>
+            Seq(false, true).foreach { vec =>
+              Seq(false, true).foreach { rebase =>
+                load(after1582, vec, rebase)
+              }
+            }
+          }
+
+          benchmark2.run()
+        }
+      }
+    }
+  }
+}

From 7329c256c6d02cbc700d367320ef20d215bca8aa Mon Sep 17 00:00:00 2001
From: manuzhang <owenzhang1990@gmail.com>
Date: Mon, 30 Mar 2020 12:44:46 -0500
Subject: [PATCH 0383/1280] [SPARK-31219][YARN] Enable closeIdleConnections in
 YarnShuffleService

### What changes were proposed in this pull request?
Close idle connections at shuffle server side when an `IdleStateEvent` is triggered after `spark.shuffle.io.connectionTimeout` or `spark.network.timeout` time. It's based on following investigations.

1. We found connections on our clusters building up continuously (> 10k for some nodes). Is that normal ? We don't think so.
2. We looked into the connections on one node and found there were a lot of half-open connections. (connections only existed on one node)
3. We also checked those connections were very old (> 21 hours). (FYI, https://superuser.com/questions/565991/how-to-determine-the-socket-connection-up-time-on-linux)
4. Looking at the code, TransportContext registers an IdleStateHandler which should fire an IdleStateEvent when timeout. We did a heap dump of the YarnShuffleService and checked the attributes of IdleStateHandler. It turned out firstAllIdleEvent of many IdleStateHandlers were already false so IdleStateEvent were already fired.
5. Finally, we realized the IdleStateEvent would not be handled since closeIdleConnections are hardcoded to false for YarnShuffleService.

### Why are the changes needed?
Idle connections to YarnShuffleService could never be closed, and will be accumulating and taking up memory and file descriptors.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing tests.

Closes #27998 from manuzhang/spark-31219.

Authored-by: manuzhang <owenzhang1990@gmail.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit 0d997e5156a751c99cd6f8be1528ed088a585d1f)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../java/org/apache/spark/network/yarn/YarnShuffleService.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
index 815a56d765b6a..c41efbad8ffec 100644
--- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
+++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/YarnShuffleService.java
@@ -188,7 +188,7 @@ protected void serviceInit(Configuration conf) throws Exception {
 
       int port = conf.getInt(
         SPARK_SHUFFLE_SERVICE_PORT_KEY, DEFAULT_SPARK_SHUFFLE_SERVICE_PORT);
-      transportContext = new TransportContext(transportConf, blockHandler);
+      transportContext = new TransportContext(transportConf, blockHandler, true);
       shuffleServer = transportContext.createServer(port, bootstraps);
       // the port should normally be fixed, but for tests its useful to find an open port
       port = shuffleServer.getPort();

From 1caca7d97a03ab9ac99597e1ef9fa3890da90743 Mon Sep 17 00:00:00 2001
From: Ben Ryves <benjamin.ryves@getyourguide.com>
Date: Tue, 31 Mar 2020 15:16:17 +0900
Subject: [PATCH 0384/1280] [SPARK-31306][DOCS] update rand() function
 documentation to indicate exclusive upper bound

### What changes were proposed in this pull request?
A small documentation change to clarify that the `rand()` function produces values in `[0.0, 1.0)`.

### Why are the changes needed?
`rand()` uses `Rand()` - which generates values in [0, 1) ([documented here](https://github.com/apache/spark/blob/a1dbcd13a3eeaee50cc1a46e909f9478d6d55177/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala#L71)). The existing documentation suggests that 1.0 is a possible value returned by rand (i.e for a distribution written as `X ~ U(a, b)`, x can be a or b, so `U[0.0, 1.0]` suggests the value returned could include 1.0).

### Does this PR introduce any user-facing change?
Only documentation changes.

### How was this patch tested?
Documentation changes only.

Closes #28071 from Smeb/master.

Authored-by: Ben Ryves <benjamin.ryves@getyourguide.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit fa378567105ec9d9bbe30edf4b74b09c3df27658)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/functions.R                                          | 2 +-
 python/pyspark/sql/functions.py                              | 2 +-
 sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index d8b045041e115..173dbc4b39bad 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -2888,7 +2888,7 @@ setMethod("lpad", signature(x = "Column", len = "numeric", pad = "character"),
 
 #' @details
 #' \code{rand}: Generates a random column with independent and identically distributed (i.i.d.)
-#' samples from U[0.0, 1.0].
+#' samples uniformly distributed in [0.0, 1.0).
 #' Note: the function is non-deterministic in general case.
 #'
 #' @rdname column_nonaggregate_functions
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 1ade21ccb42a0..476aab433e1ec 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -599,7 +599,7 @@ def nanvl(col1, col2):
 @since(1.4)
 def rand(seed=None):
     """Generates a random column with independent and identically distributed (i.i.d.) samples
-    from U[0.0, 1.0].
+    uniformly distributed in [0.0, 1.0).
 
     .. note:: The function is non-deterministic in general case.
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 8a89a3b93561a..fd4e77f693709 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -1204,7 +1204,7 @@ object functions {
 
   /**
    * Generate a random column with independent and identically distributed (i.i.d.) samples
-   * from U[0.0, 1.0].
+   * uniformly distributed in [0.0, 1.0).
    *
    * @note The function is non-deterministic in general case.
    *
@@ -1215,7 +1215,7 @@ object functions {
 
   /**
    * Generate a random column with independent and identically distributed (i.i.d.) samples
-   * from U[0.0, 1.0].
+   * uniformly distributed in [0.0, 1.0).
    *
    * @note The function is non-deterministic in general case.
    *

From e7885b8a6686bc9179f741f1394dbbf7a9e211ef Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Tue, 31 Mar 2020 17:38:47 +0800
Subject: [PATCH 0385/1280] [SPARK-31297][SQL] Speed up dates rebasing

### What changes were proposed in this pull request?
In the PR, I propose to replace current implementation of the `rebaseGregorianToJulianDays()` and `rebaseJulianToGregorianDays()` functions in `DateTimeUtils` by new one which is based on the fact that difference between Proleptic Gregorian and the hybrid (Julian+Gregorian) calendars was changed only 14 times for entire supported range of valid dates `[0001-01-01, 9999-12-31]`:

| date | Proleptic Greg. days | Hybrid (Julian+Greg) days | diff|
| ---- | ----|----|----|
|0001-01-01|-719162|-719164|-2|
|0100-03-01|-682944|-682945|-1|
|0200-03-01|-646420|-646420|0|
|0300-03-01|-609896|-609895|1|
|0500-03-01|-536847|-536845|2|
|0600-03-01|-500323|-500320|3|
|0700-03-01|-463799|-463795|4|
|0900-03-01|-390750|-390745|5|
|1000-03-01|-354226|-354220|6|
|1100-03-01|-317702|-317695|7|
|1300-03-01|-244653|-244645|8|
|1400-03-01|-208129|-208120|9|
|1500-03-01|-171605|-171595|10|
|1582-10-15|-141427|-141427|0|

For the given days since the epoch, the proposed implementation finds the range of days which the input days belongs to, and adds the diff in days between calendars to the input. The result is rebased days since the epoch in the target calendar.

For example, if need to rebase -650000 days from Proleptic Gregorian calendar to the hybrid calendar. In that case, the input falls to the bucket [-682944, -646420), the diff associated with the range is -1. To get the rebased days in Julian calendar, we should add -1 to -650000, and the result is -650001.

### Why are the changes needed?
To make dates rebasing faster.

### Does this PR introduce any user-facing change?
No, the results should be the same for valid range of the `DATE` type `[0001-01-01, 9999-12-31]`.

### How was this patch tested?
- Added 2 tests to `DateTimeUtilsSuite` for the `rebaseGregorianToJulianDays()` and `rebaseJulianToGregorianDays()` functions. The tests check that results of old and new implementation (optimized version) are the same for all supported dates.
- Re-run `DateTimeRebaseBenchmark` on:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK8/11 |

Closes #28067 from MaxGekk/optimize-rebasing.

Lead-authored-by: Maxim Gekk <max.gekk@gmail.com>
Co-authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bb0b416f0b3a2747a420b17d1bf659891bae3274)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 79 +++++++++++++------
 .../catalyst/util/DateTimeUtilsSuite.scala    | 58 +++++++++++++-
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 64 +++++++--------
 .../DateTimeRebaseBenchmark-results.txt       | 64 +++++++--------
 4 files changed, 174 insertions(+), 91 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 2b646ccc5bad9..44cabe215b6e1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -1039,6 +1039,44 @@ object DateTimeUtils {
     instantToMicros(localDateTime.atZone(ZoneId.systemDefault).toInstant)
   }
 
+  /**
+   * Rebases days since the epoch from an original to an target calendar, from instance
+   * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+   *
+   * It finds the latest switch day which is less than `days`, and adds the difference
+   * in days associated with the switch day to the given `days`. The function is based
+   * on linear search which starts from the most recent switch days. This allows to perform
+   * less comparisons for modern dates.
+   *
+   * @param switchDays The days when difference in days between original and target
+   *                   calendar was changed.
+   * @param diffs The differences in days between calendars.
+   * @param days The number of days since the epoch 1970-01-01 to be rebased to the
+   *             target calendar.
+   * @return The rebased day
+   */
+  private def rebaseDays(switchDays: Array[Int], diffs: Array[Int], days: Int): Int = {
+    var i = switchDays.length - 1
+    while (i >= 0 && days < switchDays(i)) {
+      i -= 1
+    }
+    val rebased = days + diffs(if (i < 0) 0 else i)
+    rebased
+  }
+
+  // The differences in days between Julian and Proleptic Gregorian dates.
+  // The diff at the index `i` is applicable for all days in the date interval:
+  // [julianGregDiffSwitchDay(i), julianGregDiffSwitchDay(i+1))
+  private val julianGregDiffs = Array(2, 1, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, 0)
+  // The sorted days in Julian calendar when difference in days between Julian and
+  // Proleptic Gregorian calendars was changed.
+  // The starting point is the `0001-01-01` (-719164 days since the epoch in
+  // Julian calendar). All dates before the staring point have the same difference
+  // of 2 days in Julian and Proleptic Gregorian calendars.
+  private val julianGregDiffSwitchDay = Array(
+    -719164, -682945, -646420, -609895, -536845, -500320, -463795,
+    -390745, -354220, -317695, -244645, -208120, -171595, -141427)
+
   /**
    * Converts the given number of days since the epoch day 1970-01-01 to
    * a local date in Julian calendar, interprets the result as a local
@@ -1049,25 +1087,22 @@ object DateTimeUtils {
    * @return The rebased number of days in Gregorian calendar.
    */
   def rebaseJulianToGregorianDays(days: Int): Int = {
-    val utcCal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
-      .setCalendarType("gregory")
-      .setTimeZone(TimeZoneUTC)
-      .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
-      .build()
-    val localDate = LocalDate.of(
-      utcCal.get(Calendar.YEAR),
-      utcCal.get(Calendar.MONTH) + 1,
-      // The number of days will be added later to handle non-existing
-      // Julian dates in Proleptic Gregorian calendar.
-      // For example, 1000-02-29 exists in Julian calendar because 1000
-      // is a leap year but it is not a leap year in Gregorian calendar.
-      1)
-      .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
-    Math.toIntExact(localDate.toEpochDay)
+    rebaseDays(julianGregDiffSwitchDay, julianGregDiffs, days)
   }
 
+  // The differences in days between Proleptic Gregorian and Julian dates.
+  // The diff at the index `i` is applicable for all days in the date interval:
+  // [gregJulianDiffSwitchDay(i), gregJulianDiffSwitchDay(i+1))
+  private val grepJulianDiffs = Array(-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0)
+  // The sorted days in Proleptic Gregorian calendar when difference in days between
+  // Proleptic Gregorian and Julian was changed.
+  // The starting point is the `0001-01-01` (-719162 days since the epoch in
+  // Proleptic Gregorian calendar). All dates before the staring point have the same
+  // difference of -2 days in Proleptic Gregorian and Julian calendars.
+  private val gregJulianDiffSwitchDay = Array(
+    -719162, -682944, -646420, -609896, -536847, -500323, -463799,
+    -390750, -354226, -317702, -244653, -208129, -171605, -141427)
+
   /**
    * Rebasing days since the epoch to store the same number of days
    * as by Spark 2.4 and earlier versions. Spark 3.0 switched to
@@ -1085,14 +1120,6 @@ object DateTimeUtils {
    * @return The rebased number of days since the epoch in Julian calendar.
    */
   def rebaseGregorianToJulianDays(days: Int): Int = {
-    val localDate = LocalDate.ofEpochDay(days)
-    val utcCal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
-      .setCalendarType("gregory")
-      .setTimeZone(TimeZoneUTC)
-      .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
-      .build()
-    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+    rebaseDays(gregJulianDiffSwitchDay, grepJulianDiffs, days)
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 922e09dd1e29d..652abe7659400 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.util
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
 import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId}
-import java.util.{Locale, TimeZone}
+import java.util.{Calendar, Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
 import org.scalatest.Matchers
@@ -765,4 +765,60 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
   }
+
+  test("optimization of days rebasing - Gregorian to Julian") {
+    def refRebaseGregorianToJulianDays(days: Int): Int = {
+      val localDate = LocalDate.ofEpochDay(days)
+      val utcCal = new Calendar.Builder()
+        // `gregory` is a hybrid calendar that supports both
+        // the Julian and Gregorian calendar systems
+        .setCalendarType("gregory")
+        .setTimeZone(TimeZoneUTC)
+        .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+        .build()
+      Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+    }
+
+    val start = localDateToDays(LocalDate.of(1, 1, 1))
+    val end = localDateToDays(LocalDate.of(2030, 1, 1))
+
+    var days = start
+    while (days < end) {
+      assert(rebaseGregorianToJulianDays(days) === refRebaseGregorianToJulianDays(days))
+      days += 1
+    }
+  }
+
+  test("optimization of days rebasing - Julian to Gregorian") {
+    def refRebaseJulianToGregorianDays(days: Int): Int = {
+      val utcCal = new Calendar.Builder()
+        // `gregory` is a hybrid calendar that supports both
+        // the Julian and Gregorian calendar systems
+        .setCalendarType("gregory")
+        .setTimeZone(TimeZoneUTC)
+        .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
+        .build()
+      val localDate = LocalDate.of(
+        utcCal.get(Calendar.YEAR),
+        utcCal.get(Calendar.MONTH) + 1,
+        // The number of days will be added later to handle non-existing
+        // Julian dates in Proleptic Gregorian calendar.
+        // For example, 1000-02-29 exists in Julian calendar because 1000
+        // is a leap year but it is not a leap year in Gregorian calendar.
+        1)
+        .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
+      Math.toIntExact(localDate.toEpochDay)
+    }
+
+    val start = rebaseGregorianToJulianDays(
+      localDateToDays(LocalDate.of(1, 1, 1)))
+    val end = rebaseGregorianToJulianDays(
+      localDateToDays(LocalDate.of(2030, 1, 1)))
+
+    var days = start
+    while (days < end) {
+      assert(rebaseJulianToGregorianDays(days) === refRebaseJulianToGregorianDays(days))
+      days += 1
+    }
+  }
 }
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 52522f8f88c7a..4fed51113912f 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -2,52 +2,52 @@
 Rebasing dates/timestamps in Parquet datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9272           9272           0         10.8          92.7       1.0X
-before 1582, noop                                  9142           9142           0         10.9          91.4       1.0X
-after 1582, rebase off                            21841          21841           0          4.6         218.4       0.4X
-after 1582, rebase on                             58245          58245           0          1.7         582.4       0.2X
-before 1582, rebase off                           19813          19813           0          5.0         198.1       0.5X
-before 1582, rebase on                            63737          63737           0          1.6         637.4       0.1X
+after 1582, noop                                   9304           9304           0         10.7          93.0       1.0X
+before 1582, noop                                  9187           9187           0         10.9          91.9       1.0X
+after 1582, rebase off                            22054          22054           0          4.5         220.5       0.4X
+after 1582, rebase on                             20361          20361           0          4.9         203.6       0.5X
+before 1582, rebase off                           20286          20286           0          4.9         202.9       0.5X
+before 1582, rebase on                            22230          22230           0          4.5         222.3       0.4X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13004          13063          67          7.7         130.0       1.0X
-after 1582, vec off, rebase on                    36224          36253          26          2.8         362.2       0.4X
-after 1582, vec on, rebase off                     3596           3654          54         27.8          36.0       3.6X
-after 1582, vec on, rebase on                     26144          26253         112          3.8         261.4       0.5X
-before 1582, vec off, rebase off                  12872          12914          51          7.8         128.7       1.0X
-before 1582, vec off, rebase on                   37762          37904         153          2.6         377.6       0.3X
-before 1582, vec on, rebase off                    3522           3592          94         28.4          35.2       3.7X
-before 1582, vec on, rebase on                    27580          27615          59          3.6         275.8       0.5X
+after 1582, vec off, rebase off                   12773          12866         129          7.8         127.7       1.0X
+after 1582, vec off, rebase on                    13063          13086          39          7.7         130.6       1.0X
+after 1582, vec on, rebase off                     3678           3719          61         27.2          36.8       3.5X
+after 1582, vec on, rebase on                      5078           5121          52         19.7          50.8       2.5X
+before 1582, vec off, rebase off                  12942          12972          42          7.7         129.4       1.0X
+before 1582, vec off, rebase on                   13866          13904          58          7.2         138.7       0.9X
+before 1582, vec on, rebase off                    3678           3711          43         27.2          36.8       3.5X
+before 1582, vec on, rebase on                     5621           5657          44         17.8          56.2       2.3X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   3113           3113           0         32.1          31.1       1.0X
-before 1582, noop                                  3078           3078           0         32.5          30.8       1.0X
-after 1582, rebase off                            15749          15749           0          6.3         157.5       0.2X
-after 1582, rebase on                             69106          69106           0          1.4         691.1       0.0X
-before 1582, rebase off                           15967          15967           0          6.3         159.7       0.2X
-before 1582, rebase on                            76843          76843           0          1.3         768.4       0.0X
+after 1582, noop                                   2983           2983           0         33.5          29.8       1.0X
+before 1582, noop                                  2979           2979           0         33.6          29.8       1.0X
+after 1582, rebase off                            17452          17452           0          5.7         174.5       0.2X
+after 1582, rebase on                             70193          70193           0          1.4         701.9       0.0X
+before 1582, rebase off                           17784          17784           0          5.6         177.8       0.2X
+before 1582, rebase on                            83498          83498           0          1.2         835.0       0.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15070          15172          94          6.6         150.7       1.0X
-after 1582, vec off, rebase on                    43748          43867         157          2.3         437.5       0.3X
-after 1582, vec on, rebase off                     4805           4859          60         20.8          48.1       3.1X
-after 1582, vec on, rebase on                     33960          34027          61          2.9         339.6       0.4X
-before 1582, vec off, rebase off                  15037          15071          52          6.7         150.4       1.0X
-before 1582, vec off, rebase on                   44590          44749         156          2.2         445.9       0.3X
-before 1582, vec on, rebase off                    4831           4852          30         20.7          48.3       3.1X
-before 1582, vec on, rebase on                    35460          35481          18          2.8         354.6       0.4X
+after 1582, vec off, rebase off                   15114          15151          32          6.6         151.1       1.0X
+after 1582, vec off, rebase on                    45804          45912         126          2.2         458.0       0.3X
+after 1582, vec on, rebase off                     4900           4947          56         20.4          49.0       3.1X
+after 1582, vec on, rebase on                     34599          34650          45          2.9         346.0       0.4X
+before 1582, vec off, rebase off                  15093          15174          70          6.6         150.9       1.0X
+before 1582, vec off, rebase on                   47367          47472         121          2.1         473.7       0.3X
+before 1582, vec on, rebase off                    4884           4952          80         20.5          48.8       3.1X
+before 1582, vec on, rebase on                    35831          35883          59          2.8         358.3       0.4X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index c9320cfe660f9..ee486276653fe 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -2,52 +2,52 @@
 Rebasing dates/timestamps in Parquet datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9472           9472           0         10.6          94.7       1.0X
-before 1582, noop                                  9226           9226           0         10.8          92.3       1.0X
-after 1582, rebase off                            21201          21201           0          4.7         212.0       0.4X
-after 1582, rebase on                             56471          56471           0          1.8         564.7       0.2X
-before 1582, rebase off                           20179          20179           0          5.0         201.8       0.5X
-before 1582, rebase on                            65717          65717           0          1.5         657.2       0.1X
+after 1582, noop                                   9582           9582           0         10.4          95.8       1.0X
+before 1582, noop                                  9473           9473           0         10.6          94.7       1.0X
+after 1582, rebase off                            21431          21431           0          4.7         214.3       0.4X
+after 1582, rebase on                             22156          22156           0          4.5         221.6       0.4X
+before 1582, rebase off                           21399          21399           0          4.7         214.0       0.4X
+before 1582, rebase on                            22927          22927           0          4.4         229.3       0.4X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12294          12434         205          8.1         122.9       1.0X
-after 1582, vec off, rebase on                    36959          36967          12          2.7         369.6       0.3X
-after 1582, vec on, rebase off                     3644           3691          49         27.4          36.4       3.4X
-after 1582, vec on, rebase on                     26764          26852          92          3.7         267.6       0.5X
-before 1582, vec off, rebase off                  12830          12917          85          7.8         128.3       1.0X
-before 1582, vec off, rebase on                   38897          39053         229          2.6         389.0       0.3X
-before 1582, vec on, rebase off                    3638           3693          85         27.5          36.4       3.4X
-before 1582, vec on, rebase on                    28956          29007          44          3.5         289.6       0.4X
+after 1582, vec off, rebase off                   12637          12736         111          7.9         126.4       1.0X
+after 1582, vec off, rebase on                    13463          13531          61          7.4         134.6       0.9X
+after 1582, vec on, rebase off                     3693           3703           8         27.1          36.9       3.4X
+after 1582, vec on, rebase on                      5242           5252           9         19.1          52.4       2.4X
+before 1582, vec off, rebase off                  13055          13169         126          7.7         130.5       1.0X
+before 1582, vec off, rebase on                   14067          14270         185          7.1         140.7       0.9X
+before 1582, vec on, rebase off                    3697           3702           7         27.1          37.0       3.4X
+before 1582, vec on, rebase on                     6058           6097          34         16.5          60.6       2.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2952           2952           0         33.9          29.5       1.0X
-before 1582, noop                                  2880           2880           0         34.7          28.8       1.0X
-after 1582, rebase off                            15928          15928           0          6.3         159.3       0.2X
-after 1582, rebase on                             82816          82816           0          1.2         828.2       0.0X
-before 1582, rebase off                           15988          15988           0          6.3         159.9       0.2X
-before 1582, rebase on                            92636          92636           0          1.1         926.4       0.0X
+after 1582, noop                                   2713           2713           0         36.9          27.1       1.0X
+before 1582, noop                                  2715           2715           0         36.8          27.2       1.0X
+after 1582, rebase off                            16768          16768           0          6.0         167.7       0.2X
+after 1582, rebase on                             82811          82811           0          1.2         828.1       0.0X
+before 1582, rebase off                           17052          17052           0          5.9         170.5       0.2X
+before 1582, rebase on                            95134          95134           0          1.1         951.3       0.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1058-aws
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   14863          14917          78          6.7         148.6       1.0X
-after 1582, vec off, rebase on                    54819          54939         140          1.8         548.2       0.3X
-after 1582, vec on, rebase off                     4905           4941          32         20.4          49.0       3.0X
-after 1582, vec on, rebase on                     44914          45008         124          2.2         449.1       0.3X
-before 1582, vec off, rebase off                  14928          14970          48          6.7         149.3       1.0X
-before 1582, vec off, rebase on                   59752          59996         245          1.7         597.5       0.2X
-before 1582, vec on, rebase off                    4892           4916          33         20.4          48.9       3.0X
-before 1582, vec on, rebase on                    46854          46977         198          2.1         468.5       0.3X
+after 1582, vec off, rebase off                   15200          15321         194          6.6         152.0       1.0X
+after 1582, vec off, rebase on                    63160          63337         177          1.6         631.6       0.2X
+after 1582, vec on, rebase off                     4891           4928          43         20.4          48.9       3.1X
+after 1582, vec on, rebase on                     45474          45484          10          2.2         454.7       0.3X
+before 1582, vec off, rebase off                  15203          15330         110          6.6         152.0       1.0X
+before 1582, vec off, rebase on                   65588          65664          73          1.5         655.9       0.2X
+before 1582, vec on, rebase off                    4844           4916         105         20.6          48.4       3.1X
+before 1582, vec on, rebase on                    47815          47943         162          2.1         478.2       0.3X
 
 

From 08bb5f0ffeb4f5e37417f15931717784db544730 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Tue, 31 Mar 2020 19:01:08 +0800
Subject: [PATCH 0386/1280] [SPARK-31314][CORE] Revert SPARK-29285 to fix
 shuffle regression caused by creating temporary file eagerly

### What changes were proposed in this pull request?
This reverts commit 8cf76f8d61b393bb3abd9780421b978e98db8cae. #25962

### Why are the changes needed?
In SPARK-29285, we change to create shuffle temporary eagerly. This is helpful for not to fail the entire task in the scenario of occasional disk failure. But for the applications that many tasks don't actually create shuffle files, it caused overhead. See the below benchmark:
Env: Spark local-cluster[2, 4, 19968], each queries run 5 round, each round 5 times.
Data: TPC-DS scale=99 generate by spark-tpcds-datagen
Results:
|     | Base                                                                                        | Revert                                                                                      |
|-----|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------|
| Q20 | Vector(4.096865667, 2.76231748, 2.722007606, 2.514433591, 2.400373579)  Median 2.722007606  | Vector(3.763185446, 2.586498463, 2.593472842, 2.320522846, 2.224627274)  Median 2.586498463 |
| Q33 | Vector(5.872176321, 4.854397586, 4.568787136, 4.393378146, 4.423996818)  Median 4.568787136 | Vector(5.38746785, 4.361236877, 4.082311276, 3.867206824, 3.783188024)  Median 4.082311276  |
| Q52 | Vector(3.978870321, 3.225437871, 3.282411608, 2.869674887, 2.644490664)  Median 3.225437871 | Vector(4.000381522, 3.196025108, 3.248787619, 2.767444508, 2.606163423)  Median 3.196025108 |
| Q56 | Vector(6.238045133, 4.820535173, 4.609965579, 4.313509894, 4.221256227)  Median 4.609965579 | Vector(6.241611339, 4.225592467, 4.195202502, 3.757085755, 3.657525982)  Median 4.195202502 |

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing tests.

Closes #28072 from xuanyuanking/SPARK-29285-revert.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 07c50784d34e10bbfafac7498c0b70c4ec08048a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/storage/DiskBlockManager.scala      | 36 ++++------------
 .../scala/org/apache/spark/util/Utils.scala   |  2 +-
 .../spark/storage/DiskBlockManagerSuite.scala | 43 +------------------
 3 files changed, 10 insertions(+), 71 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
index ee43b76e17010..f2113947f6bf5 100644
--- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala
@@ -20,8 +20,6 @@ package org.apache.spark.storage
 import java.io.{File, IOException}
 import java.util.UUID
 
-import scala.util.control.NonFatal
-
 import org.apache.spark.SparkConf
 import org.apache.spark.executor.ExecutorExitCode
 import org.apache.spark.internal.{config, Logging}
@@ -119,38 +117,20 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea
 
   /** Produces a unique block id and File suitable for storing local intermediate results. */
   def createTempLocalBlock(): (TempLocalBlockId, File) = {
-    var blockId = TempLocalBlockId(UUID.randomUUID())
-    var tempLocalFile = getFile(blockId)
-    var count = 0
-    while (!canCreateFile(tempLocalFile) && count < Utils.MAX_DIR_CREATION_ATTEMPTS) {
-      blockId = TempLocalBlockId(UUID.randomUUID())
-      tempLocalFile = getFile(blockId)
-      count += 1
+    var blockId = new TempLocalBlockId(UUID.randomUUID())
+    while (getFile(blockId).exists()) {
+      blockId = new TempLocalBlockId(UUID.randomUUID())
     }
-    (blockId, tempLocalFile)
+    (blockId, getFile(blockId))
   }
 
   /** Produces a unique block id and File suitable for storing shuffled intermediate results. */
   def createTempShuffleBlock(): (TempShuffleBlockId, File) = {
-    var blockId = TempShuffleBlockId(UUID.randomUUID())
-    var tempShuffleFile = getFile(blockId)
-    var count = 0
-    while (!canCreateFile(tempShuffleFile) && count < Utils.MAX_DIR_CREATION_ATTEMPTS) {
-      blockId = TempShuffleBlockId(UUID.randomUUID())
-      tempShuffleFile = getFile(blockId)
-      count += 1
-    }
-    (blockId, tempShuffleFile)
-  }
-
-  private def canCreateFile(file: File): Boolean = {
-    try {
-      file.createNewFile()
-    } catch {
-      case NonFatal(_) =>
-        logError("Failed to create temporary block file: " + file.getAbsoluteFile)
-        false
+    var blockId = new TempShuffleBlockId(UUID.randomUUID())
+    while (getFile(blockId).exists()) {
+      blockId = new TempShuffleBlockId(UUID.randomUUID())
     }
+    (blockId, getFile(blockId))
   }
 
   /**
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 9f332ba608082..c7db2127a6f04 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -95,7 +95,7 @@ private[spark] object Utils extends Logging {
    */
   val DEFAULT_DRIVER_MEM_MB = JavaUtils.DEFAULT_DRIVER_MEM_MB.toInt
 
-  val MAX_DIR_CREATION_ATTEMPTS: Int = 10
+  private val MAX_DIR_CREATION_ATTEMPTS: Int = 10
   @volatile private var localRootDirs: Array[String] = null
 
   /** Scheme used for files that are locally available on worker nodes in the cluster. */
diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
index ccc525e854838..c757dee43808d 100644
--- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
@@ -51,7 +51,7 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B
   override def beforeEach(): Unit = {
     super.beforeEach()
     val conf = testConf.clone
-    conf.set("spark.local.dir", rootDirs).set("spark.diskStore.subDirectories", "1")
+    conf.set("spark.local.dir", rootDirs)
     diskBlockManager = new DiskBlockManager(conf, deleteFilesOnStop = true)
   }
 
@@ -90,45 +90,4 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B
     for (i <- 0 until numBytes) writer.write(i)
     writer.close()
   }
-
-  test("temporary shuffle/local file should be able to handle disk failures") {
-    try {
-      // the following two lines pre-create subdirectories under each root dir of block manager
-      diskBlockManager.getFile("1")
-      diskBlockManager.getFile("2")
-
-      val tempShuffleFile1 = diskBlockManager.createTempShuffleBlock()._2
-      val tempLocalFile1 = diskBlockManager.createTempLocalBlock()._2
-      assert(tempShuffleFile1.exists(), "There are no bad disks, so temp shuffle file exists")
-      assert(tempLocalFile1.exists(), "There are no bad disks, so temp local file exists")
-
-      // partial disks damaged
-      rootDir0.setExecutable(false)
-      val tempShuffleFile2 = diskBlockManager.createTempShuffleBlock()._2
-      val tempLocalFile2 = diskBlockManager.createTempLocalBlock()._2
-      // It's possible that after 10 retries we still not able to find the healthy disk. we need to
-      // remove the flakiness of these two asserts
-      if (tempShuffleFile2.getParentFile.getParentFile.getParent === rootDir1.getAbsolutePath) {
-        assert(tempShuffleFile2.exists(),
-          "There is only one bad disk, so temp shuffle file should be created")
-      }
-      if (tempLocalFile2.getParentFile.getParentFile.getParent === rootDir1.getAbsolutePath) {
-        assert(tempLocalFile2.exists(),
-          "There is only one bad disk, so temp local file should be created")
-      }
-
-      // all disks damaged
-      rootDir1.setExecutable(false)
-      val tempShuffleFile3 = diskBlockManager.createTempShuffleBlock()._2
-      val tempLocalFile3 = diskBlockManager.createTempLocalBlock()._2
-      assert(!tempShuffleFile3.exists(),
-        "All disks are broken, so there should be no temp shuffle file created")
-      assert(!tempLocalFile3.exists(),
-        "All disks are broken, so there should be no temp local file created")
-    } finally {
-      rootDir0.setExecutable(true)
-      rootDir1.setExecutable(true)
-    }
-
-  }
 }

From 20bb33453f85aeb5d2448252a9dd23d3ab85d251 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 31 Mar 2020 23:19:46 +0800
Subject: [PATCH 0387/1280] [SPARK-31230][SQL] Use statement plans in
 DataFrameWriter(V2)

### What changes were proposed in this pull request?

Create statement plans in `DataFrameWriter(V2)`, like the SQL API.

### Why are the changes needed?

It's better to leave all the resolution work to the analyzer.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

existing tests

Closes #27992 from cloud-fan/statement.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8b01473e8bffe349b1ed993b61420d7d68896cd8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalyst/analysis/ResolveCatalogs.scala   |  8 +--
 .../sql/catalyst/parser/AstBuilder.scala      |  4 +-
 .../catalyst/plans/logical/statements.scala   |  2 +
 .../spark/sql/connector/InMemoryTable.scala   |  1 +
 .../apache/spark/sql/DataFrameWriter.scala    | 55 ++++++++++---------
 .../apache/spark/sql/DataFrameWriterV2.scala  | 43 ++++++++-------
 .../analysis/ResolveSessionCatalog.scala      |  8 +--
 .../command/PlanResolutionSuite.scala         |  4 +-
 8 files changed, 66 insertions(+), 59 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
index 895dfbb77d642..403e4e8758e72 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveCatalogs.scala
@@ -134,7 +134,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         ignoreIfExists = c.ifNotExists)
 
     case c @ CreateTableAsSelectStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
       CreateTableAsSelect(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -142,7 +142,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
         convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
-        writeOptions = c.options,
+        writeOptions = c.writeOptions,
         ignoreIfExists = c.ifNotExists)
 
     case RefreshTableStatement(NonSessionCatalogAndTable(catalog, tbl)) =>
@@ -161,7 +161,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         orCreate = c.orCreate)
 
     case c @ ReplaceTableAsSelectStatement(
-         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         NonSessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
       ReplaceTableAsSelect(
         catalog.asTableCatalog,
         tbl.asIdentifier,
@@ -169,7 +169,7 @@ class ResolveCatalogs(val catalogManager: CatalogManager)
         c.partitioning ++ c.bucketSpec.map(_.asTransform),
         c.asSelect,
         convertTableProperties(c.properties, c.options, c.location, c.comment, c.provider),
-        writeOptions = c.options,
+        writeOptions = c.writeOptions,
         orCreate = c.orCreate)
 
     case DropTableStatement(NonSessionCatalogAndTable(catalog, tbl), ifExists, _) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 09d316b683b4f..cd4c895752a79 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2779,7 +2779,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case Some(query) =>
         CreateTableAsSelectStatement(
           table, query, partitioning, bucketSpec, properties, provider, options, location, comment,
-          ifNotExists = ifNotExists)
+          writeOptions = Map.empty, ifNotExists = ifNotExists)
 
       case None if temp =>
         // CREATE TEMPORARY TABLE ... USING ... is not supported by the catalyst parser.
@@ -2834,7 +2834,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
 
       case Some(query) =>
         ReplaceTableAsSelectStatement(table, query, partitioning, bucketSpec, properties,
-          provider, options, location, comment, orCreate = orCreate)
+          provider, options, location, comment, writeOptions = Map.empty, orCreate = orCreate)
 
       case _ =>
         ReplaceTableStatement(table, schema.getOrElse(new StructType), partitioning,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
index 50c3cd0d10278..7d7aae20c65c3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala
@@ -83,6 +83,7 @@ case class CreateTableAsSelectStatement(
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
+    writeOptions: Map[String, String],
     ifNotExists: Boolean) extends ParsedStatement {
 
   override def children: Seq[LogicalPlan] = Seq(asSelect)
@@ -133,6 +134,7 @@ case class ReplaceTableAsSelectStatement(
     options: Map[String, String],
     location: Option[String],
     comment: Option[String],
+    writeOptions: Map[String, String],
     orCreate: Boolean) extends ParsedStatement {
 
   override def children: Seq[LogicalPlan] = Seq(asSelect)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
index 0187ae31e2d1c..3d7026e180cd1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
@@ -116,6 +116,7 @@ class InMemoryTable(
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
+    InMemoryTable.maybeSimulateFailedTableWrite(new CaseInsensitiveStringMap(properties))
     InMemoryTable.maybeSimulateFailedTableWrite(info.options)
 
     new WriteBuilder with SupportsTruncate with SupportsOverwrite with SupportsDynamicOverwrite {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 7e669e0faccd3..d11e4db982035 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -26,18 +26,17 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, NoSuchTableException, UnresolvedRelation}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.Literal
-import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, InsertIntoStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelect}
-import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
-import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Implicits, CatalogV2Util, Identifier, SupportsCatalogOptions, SupportsWrite, Table, TableCatalog, TableProvider, V1Table}
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, CreateTableAsSelectStatement, InsertIntoStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
+import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Implicits, CatalogV2Util, Identifier, SupportsCatalogOptions, Table, TableCatalog, TableProvider, V1Table}
 import org.apache.spark.sql.connector.catalog.TableCapability._
-import org.apache.spark.sql.connector.expressions.{BucketTransform, FieldReference, IdentityTransform, LiteralValue, Transform}
+import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
 import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, DataSourceUtils, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf.PartitionOverwriteMode
 import org.apache.spark.sql.sources.BaseRelation
-import org.apache.spark.sql.types.{IntegerType, StructType}
+import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
@@ -574,12 +573,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
     val canUseV2 = lookupV2Provider().isDefined
 
     session.sessionState.sqlParser.parseMultipartIdentifier(tableName) match {
-      case NonSessionCatalogAndIdentifier(catalog, ident) =>
-        saveAsTable(catalog.asTableCatalog, ident)
+      case nameParts @ NonSessionCatalogAndIdentifier(catalog, ident) =>
+        saveAsTable(catalog.asTableCatalog, ident, nameParts)
 
-      case SessionCatalogAndIdentifier(catalog, ident)
+      case nameParts @ SessionCatalogAndIdentifier(catalog, ident)
           if canUseV2 && ident.namespace().length <= 1 =>
-        saveAsTable(catalog.asTableCatalog, ident)
+        saveAsTable(catalog.asTableCatalog, ident, nameParts)
 
       case AsTableIdentifier(tableIdentifier) =>
         saveAsTable(tableIdentifier)
@@ -591,16 +590,12 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   }
 
 
-  private def saveAsTable(catalog: TableCatalog, ident: Identifier): Unit = {
+  private def saveAsTable(
+      catalog: TableCatalog, ident: Identifier, nameParts: Seq[String]): Unit = {
     val tableOpt = try Option(catalog.loadTable(ident)) catch {
       case _: NoSuchTableException => None
     }
 
-    def getLocationIfExists: Option[(String, String)] = {
-      val opts = CaseInsensitiveMap(extraOptions.toMap)
-      opts.get("path").map(TableCatalog.PROP_LOCATION -> _)
-    }
-
     val command = (mode, tableOpt) match {
       case (_, Some(_: V1Table)) =>
         return saveAsTable(TableIdentifier(ident.name(), ident.namespace().headOption))
@@ -611,12 +606,16 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
         AppendData.byName(v2Relation, df.logicalPlan, extraOptions.toMap)
 
       case (SaveMode.Overwrite, _) =>
-        ReplaceTableAsSelect(
-          catalog,
-          ident,
-          partitioningAsV2,
+        ReplaceTableAsSelectStatement(
+          nameParts,
           df.queryExecution.analyzed,
-          Map(TableCatalog.PROP_PROVIDER -> source) ++ getLocationIfExists,
+          partitioningAsV2,
+          None,
+          Map.empty,
+          Some(source),
+          Map.empty,
+          extraOptions.get("path"),
+          extraOptions.get(TableCatalog.PROP_COMMENT),
           extraOptions.toMap,
           orCreate = true)      // Create the table if it doesn't exist
 
@@ -624,14 +623,18 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
         // We have a potential race condition here in AppendMode, if the table suddenly gets
         // created between our existence check and physical execution, but this can't be helped
         // in any case.
-        CreateTableAsSelect(
-          catalog,
-          ident,
-          partitioningAsV2,
+        CreateTableAsSelectStatement(
+          nameParts,
           df.queryExecution.analyzed,
-          Map(TableCatalog.PROP_PROVIDER -> source) ++ getLocationIfExists,
+          partitioningAsV2,
+          None,
+          Map.empty,
+          Some(source),
+          Map.empty,
+          extraOptions.get("path"),
+          extraOptions.get(TableCatalog.PROP_COMMENT),
           extraOptions.toMap,
-          ignoreIfExists = other == SaveMode.Ignore)
+          ifNotExists = other == SaveMode.Ignore)
     }
 
     runCommand(df.sparkSession, "saveAsTable") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
index 45a9b28154028..15ff7fa0530cc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
@@ -23,8 +23,7 @@ import scala.collection.mutable
 import org.apache.spark.annotation.Experimental
 import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Bucket, Days, Hours, Literal, Months, Years}
-import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelect}
-import org.apache.spark.sql.connector.catalog.TableCatalog
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelectStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
 import org.apache.spark.sql.connector.expressions.{LogicalExpressions, NamedReference, Transform}
 import org.apache.spark.sql.execution.SQLExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
@@ -47,8 +46,6 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
 
   private val sparkSession = ds.sparkSession
 
-  private val catalogManager = sparkSession.sessionState.analyzer.catalogManager
-
   private val tableName = sparkSession.sessionState.sqlParser.parseMultipartIdentifier(table)
 
   private val (catalog, identifier) = {
@@ -120,19 +117,19 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
   }
 
   override def create(): Unit = {
-    // create and replace could alternatively create ParsedPlan statements, like
-    // `CreateTableFromDataFrameStatement(UnresolvedRelation(tableName), ...)`, to keep the catalog
-    // resolution logic in the analyzer.
     runCommand("create") {
-      CreateTableAsSelect(
-        catalog,
-        identifier,
-        partitioning.getOrElse(Seq.empty),
+      CreateTableAsSelectStatement(
+        tableName,
         logicalPlan,
-        properties = provider.map(p => properties + (TableCatalog.PROP_PROVIDER -> p))
-          .getOrElse(properties).toMap,
-        writeOptions = options.toMap,
-        ignoreIfExists = false)
+        partitioning.getOrElse(Seq.empty),
+        None,
+        properties.toMap,
+        provider,
+        Map.empty,
+        None,
+        None,
+        options.toMap,
+        ifNotExists = false)
     }
   }
 
@@ -231,13 +228,17 @@ final class DataFrameWriterV2[T] private[sql](table: String, ds: Dataset[T])
 
   private def internalReplace(orCreate: Boolean): Unit = {
     runCommand("replace") {
-      ReplaceTableAsSelect(
-        catalog,
-        identifier,
-        partitioning.getOrElse(Seq.empty),
+      ReplaceTableAsSelectStatement(
+        tableName,
         logicalPlan,
-        properties = provider.map(p => properties + ("provider" -> p)).getOrElse(properties).toMap,
-        writeOptions = options.toMap,
+        partitioning.getOrElse(Seq.empty),
+        None,
+        properties.toMap,
+        provider,
+        Map.empty,
+        None,
+        None,
+        options.toMap,
         orCreate = orCreate)
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 7b88fc6008246..77c5701d04708 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -267,7 +267,7 @@ class ResolveSessionCatalog(
       }
 
     case c @ CreateTableAsSelectStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
       val provider = c.provider.getOrElse(conf.defaultDataSourceName)
       if (!isV2Provider(provider)) {
         val tableDesc = buildCatalogTable(tbl.asTableIdentifier, new StructType,
@@ -283,7 +283,7 @@ class ResolveSessionCatalog(
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
           convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
-          writeOptions = c.options,
+          writeOptions = c.writeOptions,
           ignoreIfExists = c.ifNotExists)
       }
 
@@ -311,7 +311,7 @@ class ResolveSessionCatalog(
       }
 
     case c @ ReplaceTableAsSelectStatement(
-         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _) =>
+         SessionCatalogAndTable(catalog, tbl), _, _, _, _, _, _, _, _, _, _) =>
       val provider = c.provider.getOrElse(conf.defaultDataSourceName)
       if (!isV2Provider(provider)) {
         throw new AnalysisException("REPLACE TABLE AS SELECT is only supported with v2 tables.")
@@ -323,7 +323,7 @@ class ResolveSessionCatalog(
           c.partitioning ++ c.bucketSpec.map(_.asTransform),
           c.asSelect,
           convertTableProperties(c.properties, c.options, c.location, c.comment, Some(provider)),
-          writeOptions = c.options,
+          writeOptions = c.writeOptions,
           orCreate = c.orCreate)
       }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
index 2e88ea9299c00..8a3e1bb968efc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala
@@ -552,7 +552,7 @@ class PlanResolutionSuite extends AnalysisTest {
         assert(ctas.catalog.name == "testcat")
         assert(ctas.tableName == Identifier.of(Array("mydb"), "table_name"))
         assert(ctas.properties == expectedProperties)
-        assert(ctas.writeOptions == Map("other" -> "20"))
+        assert(ctas.writeOptions.isEmpty)
         assert(ctas.partitioning.isEmpty)
         assert(ctas.ignoreIfExists)
 
@@ -586,7 +586,7 @@ class PlanResolutionSuite extends AnalysisTest {
         assert(ctas.catalog.name == "testcat")
         assert(ctas.tableName == Identifier.of(Array("mydb"), "table_name"))
         assert(ctas.properties == expectedProperties)
-        assert(ctas.writeOptions == Map("other" -> "20"))
+        assert(ctas.writeOptions.isEmpty)
         assert(ctas.partitioning.isEmpty)
         assert(ctas.ignoreIfExists)
 

From bd2b6aa42c8a5472c464bb1ee1a8f59a97f699f9 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 31 Mar 2020 16:17:26 +0000
Subject: [PATCH 0388/1280] [SPARK-31312][SQL] Cache Class instance for the UDF
 instance in HiveFunctionWrapper

### What changes were proposed in this pull request?

This patch proposes to cache Class instance for the UDF instance in HiveFunctionWrapper to fix the case where Hive simple UDF is somehow transformed (expression is copied) and evaluated later with another classloader (for the case current thread context classloader is somehow changed). In this case, Spark throws CNFE as of now.

It's only occurred for Hive simple UDF, as HiveFunctionWrapper caches the UDF instance whereas it doesn't do for `UDF` type. The comment says Spark has to create instance every time for UDF, so we cannot simply do the same. This patch caches Class instance instead, and switch current thread context classloader to which loads the Class instance.

This patch extends the test boundary as well. We only tested with GenericUDTF for SPARK-26560, and this patch actually requires only UDF. But to avoid regression for other types as well, this patch adds all available types (UDF, GenericUDF, AbstractGenericUDAFResolver, UDAF, GenericUDTF) into the boundary of tests.

Credit to cloud-fan as he discovered the problem and proposed the solution.

### Why are the changes needed?

Above section describes why it's a bug and how it's fixed.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

New UTs added.

Closes #28079 from HeartSaVioR/SPARK-31312.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2a6aa8e87bec39f6bfec67e151ef8566b75caecd)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/hive/HiveShim.scala  |  18 +-
 .../test/noclasspath/TestUDTF-spark-26560.jar | Bin 7462 -> 0 bytes
 .../src/test/noclasspath/hive-test-udfs.jar   | Bin 0 -> 35660 bytes
 .../sql/hive/HiveUDFDynamicLoadSuite.scala    | 190 ++++++++++++++++++
 .../sql/hive/execution/SQLQuerySuite.scala    |  47 -----
 5 files changed, 203 insertions(+), 52 deletions(-)
 delete mode 100644 sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar
 create mode 100644 sql/hive/src/test/noclasspath/hive-test-udfs.jar
 create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
index 3beef6b1df457..04a6a8f8aa9a5 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala
@@ -118,9 +118,12 @@ private[hive] object HiveShim {
    *
    * @param functionClassName UDF class name
    * @param instance optional UDF instance which contains additional information (for macro)
+   * @param clazz optional class instance to create UDF instance
    */
-  private[hive] case class HiveFunctionWrapper(var functionClassName: String,
-    private var instance: AnyRef = null) extends java.io.Externalizable {
+  private[hive] case class HiveFunctionWrapper(
+      var functionClassName: String,
+      private var instance: AnyRef = null,
+      private var clazz: Class[_ <: AnyRef] = null) extends java.io.Externalizable {
 
     // for Serialization
     def this() = this(null)
@@ -232,8 +235,10 @@ private[hive] object HiveShim {
         in.readFully(functionInBytes)
 
         // deserialize the function object via Hive Utilities
+        clazz = Utils.getContextOrSparkClassLoader.loadClass(functionClassName)
+          .asInstanceOf[Class[_ <: AnyRef]]
         instance = deserializePlan[AnyRef](new java.io.ByteArrayInputStream(functionInBytes),
-          Utils.getContextOrSparkClassLoader.loadClass(functionClassName))
+          clazz)
       }
     }
 
@@ -241,8 +246,11 @@ private[hive] object HiveShim {
       if (instance != null) {
         instance.asInstanceOf[UDFType]
       } else {
-        val func = Utils.getContextOrSparkClassLoader
-          .loadClass(functionClassName).getConstructor().newInstance().asInstanceOf[UDFType]
+        if (clazz == null) {
+          clazz = Utils.getContextOrSparkClassLoader.loadClass(functionClassName)
+            .asInstanceOf[Class[_ <: AnyRef]]
+        }
+        val func = clazz.getConstructor().newInstance().asInstanceOf[UDFType]
         if (!func.isInstanceOf[UDF]) {
           // We cache the function if it's no the Simple UDF,
           // as we always have to create new instance for Simple UDF
diff --git a/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar b/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar
deleted file mode 100644
index b73b17d5c7880a56981288d0ae2041b0970b14f3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 7462
zcmbW6bzD^Y*2ahK?(UKfX=#S;?oNRjIz&1Iqy%XYDQN~lkPf9pngNmSPD!a3&+*)=
z_gv38_ucz5Gy9L(-)F7$*?aB%TaUUD96T!E+Xi}-srK8!_b+$=IRNBl$El-@0)V4E
zLNHA_K`{07K?A_RpTGbB@c4KCQ~%}JXy4Da09#nu+x$NvxIaVeEv!Kx@c)xV^j}%_
z4qpE>IHBU7>qGTxpcTl)!_C3+_ed-Mu5{r4C_Me&gFUT1Y=5WI{~cZCKf-4Fhjbes
z3umy?_u{nwTIJhifDHJS0}gG(I0ifbV2lC)NdJ#2ayBkDZVpyDGTK0Ckf)0WFNc+r
zg}eK2*WNEuEYy9dMW#a>po-fFsa$2WZ>e;K(!=%_epX^mmXg-eYKPsl?<t8KpvfAv
z71+xu?25l$0v(tYTRFUny8#3yFF)dk8RgsH^6&K>dNg<G`Q=QU^A?sXG&(B4nCB3o
zb?2RdF+W)LT^S?ABs#t_@!Rnj1B|LO3(PN!F6NSGGkg$x^Qf0S7?x(-=&4kPCVM_J
zRYDAAQ5nc+=xbisVrv$jp4&LpqV~%#i5Bl!3h}Z&E?FZUoiYxmsUg-zkPbBXYROOu
ziUNm%e2;mwF^%_he3<Xg`F$*=AmRR)8VXrEK=k$wt9e%@ISlq%sRYlo>%E+DTPpqR
z^&Za@<kM+D*os`()KIHoJU%#t??G&eohbEX0#A=G8U3oaoxSso`?E(!a<n(ekHJ|l
z<->Hici|PZN8?H1Lm&h;n*h!gRpz$J&QChohRhD4<rOK+M`W)~<R|4(s~S7ELylwl
zl<-+Mh0rW|&pz;s>TIc`;U?u*5n5Kcsy{Urn3}4~(Bd1dB3P3*D-_w^v`l9uX67M|
z<pRqVm<)J1G!*nR2g(Rch6;MfYB-+BnQh0Qu=j)aC~01E+~awMzY(&J6wSom$>o*R
zFYYrA>hw&JS&87;WJVQP*2?Is-0LaAaBMNz>pB*^g}iNeea^LwXgWw)FOVATg<S?L
zr#0F|o>q(Kz^vQl@*$r}#J(ZMk)m<hHRiKeWPeJmUk4^*K!YQ;Cf;~D*NGj$KGUJc
z7jL2{!SU>x1<hvWKE%?3;2~mb9T;nykRQ?qzSgek=JNFKUfQbDdZ<{d#G_2fsYMj6
z*RV9`>?fFI8j*g>g1+iaQ{LsdSjJz{F<D1^qNSM&b)Ho#G@g85S9?m1p<ef>r_lFA
zaH}5uOl{&kR(-;O9p~NWjtVQnXyX@;R$kZ1#d!^rOsEPv$?@-dK;;eeg`ww^jZTwc
zhI%tqno><15)stXb>hzmpiwXU$8I7eTL6Vs_XZ>jv2;+V#spr!Xw730h2{h6U-{J{
z+qNImj|wS%;&J5)p}SZ&zc@xK$f3y|x{CV9dwi9_?2|cIBR`hTuqevrALor+lMx~q
z>5anEEO(}zcPz3rQHyplKax~L*OD_i`8aDcOpXyaBK-;uFKJ~HpC!B7_bU<DdP}N*
z{IY^sEO$Qgyx8X0E`%iH#TCqQ=K4m<bO@`J^y5pe{rKAm0X}lFK$vYJOXd3XmZzr1
zFJ(*$>mL@pPDmRNAgKGem8lX{Xq~kAWkofb9tci!amaHpDIwe)lm2WcBLWswY9n`W
z7B;3;j9$ncl=Y(m*|vg;Zkd^EJ0TLKE{vG5{0Nv7qy?JjNu&8s3%F9`<E=s#04+jc
zD!}C7O=R9%*ndBW*#4v6q1<&M6<KXbb_G@7?=3U=pENVPkb$GoyVh25_o&1F5uqZf
zssNPL(&kVB{#ADhPv~%j<;ITK4{m<fED^F@GN2N!i6$j^XgF$J7iXKqmHFoD>7Juf
zjOC-ObkNO3T6nCsmC-n9Lm@tfwjX_0*lfgUulVj{P-JEJelR{S&}g`9bEKW#s9sY4
z(*qz&5k<J$9tiG!>gO_|HnL8t<BbZx@&bKZ5ACFy!XZ7@doUQHXH#5C)m$xRbZzK8
zjib(W&1CaZBTW2|Ezj2SloE~?DL6TrMAsxqmUAmX7~Mxb7p!Z8*JO^^>h2)w!=7->
z^f5w{k$2mBPFGcIT`1p5{Cl(i_NGu<yA<J!1pt_`0RY%P)<;=ET2@s{_OIR=W(|$p
zr@ILP&Z<7S6=^vj7G?7}kgZ2muS8TxJIFM+FeXJZK&a~xA-x>b(bp#rQd@!xti8YL
zsubN<*qN$XKg!<E>OwngXp`ma&&J*NED?TXD*9zbJlXTJ$I|x2w$zo1SAH$h;mP*o
zyl8{H0Qz;NUe)R4wecvwyH$M^>D-4FzhJces{H`Nn653w<`x4J_We|WvNt~7?o49k
zRcNl>j~{uvpDeNmm<(&_^`BhN9z|lEW1yTna#U(ph&%{#bGhk!$oqvU1EoGmh0c&c
zgu7h<$;ODuS}$tUgP;ocO|_**DYov(gtk?b8IdWzn302kehbOrBbw}Du#!&HfMmH&
zpHwUC#l+L;>~qyLQZaj<!DOql#zlQjD>Zypl|YDJ6Is}l_XkvU6vC}V>J`DAr_kG?
z>*g$X)`)C!c&ElUxP)Fte!bH)S`yASbd?InQMgu-Nc6kU6Braw41-7(3^=>n;Y=kf
z$Uzv06y*-6G!>{+q?(QGV+-|GdL6GvYb!RJ>2GsCZCmuX`MhiJv18jDiLVpD=I$#U
zS?rMG<+K6Hxh_eW9ck9+)d9{74sh?i6uI%HL8MbeQSDi8#=nVTGnZu*Y0%jaotcbX
z9c0yga;-XY*l}4~ML*SGb4r?zQebR995%D;)hMU%rm`cW<uyZe>-p))KBt2S+?vHZ
zB+2CkM2PTLz>M{CG+fCQpr?75W+*GuDDQ)${y_d^W#e|ktJG|hHPgu3VMeS*QjHV>
zUw<E%Eu2~?CIL)~(JncHXcq|+Ov5Vu_yoiVg#~#NN-1Xu(y1s9Up;=PLU<cyR9iF(
z=2I{rATMuvP11*-5SGUqYVT3z>V1&Udjjb>aI2*eEQH;ToS~C6MHg2OyQ(_zsy!BY
zTc|<eKjU<R@Dg0wxPy5`@z%<TS={lV;d%8;A7(8k3arM6kiTX`P#%c%wwh2A;wx{=
z{UA5h)k9f6)6`0eM01eJ9W4=H7gWWBG&q8$`n7#yR~>WRrR*Mi7kJ}~RUqH#7z*>w
zqS9y@7@3Z}Fleun+Y%PCc~xy_0FxjpKijK<cO?F<5OxF~4w5XEZ2dkw$i*DdN3fI3
z2$gceT2}~N->6qg(ui{j7BiR9X8PSA1Ur|aGmB^`DW+8q@`a9G%jz_>4^{8O77}DB
z6xSYGKqL+@v)qhvwb?>nC#mjFM<HC5uU!ttdK&PAvisgl^h|gIWPZ~vH5f+lVGt=G
z-2ocPsAq{E>foP)j6lfG?+=SJH{i?fYoWiI94b$<t@P<5Uot*K8RUu7l=bxrKORmM
znsm(hzWSp1ge#+EU%t7DRQHW$&y{vdEk27qkub2V+z~SYX_kp+1~Cs|xge>c5nXi>
zAXAsZKoCV+mN-((r1;V^xg=?Fo;LZCG;m_MM+jSsf0~nYn8~=fPy-8}{}81MyqYeb
zVEu`|7BwJH@q$(<8b)KjeL^M3S-=G#BI*M_CpjXUk>n>vU{BY1h-Zk?th~B6<(P;^
zAY=2Ha`|z7d)oc9C5NLU8>s2WrF2_U!pTeIeu(`2grErkG0{eNtP^wGDdD82j9B-e
zk(oK+3Q(_4-TqZap*78A)6$eCel_bn+>Mej4DA*ok$?n?q@nI>S^_sDf#Qi(Y|Q)K
z+WJz8rHMBvUgEl9wycQ{SAm>7#OX4Uw$8O=;2Ad$?Nx`Kl!MW+-uN(|626ii^RP_=
zY0*v%<Es9Vr<l*`pP%AMy-?X<R_9nfuA|$DRF|Pjv={=?hCrPVPE%-ypcgxQ;qm95
zqF%l2Lz+1=K~Lg$M~W<wJ~-jO$5RK&%3~_7Ul!1NY}Ml|z3OmQf;(MviwzS{5=YFd
zT3@P?F9T24AD#yrM4j|@L3_5QY#Ehm$e$BvD<qQ0Q-=i&zF&Gm1P}_-f_=9Q1*D)X
zA`MS!KZ-TZ=bcqLY7WI)?n!O;+aV-la!(il`xqD@yx_xbLeip5c%kh+f4dm*ayo<{
z!RkTxA<8I&;C>HAIRdLx?rCh8MErcvps^#x<K-yXx%yQ;aZB#R)g+3cWcN-&uVE@)
zq*+Dy*;}}M<8~e>R+{;+8+tHpa7Y|kp3=w{1KxcY6C?|fi`*xx2;s%?{GPF*SkXa_
zfPjlY7>%bCpSW?5L^V#!6U-U2&{f*SP@6uqf*`Ye6oT31Pd}~du#%N@%w0Ko2|{R5
z`YOm=YeIKCRSi|942&dR4398VIGW|06Yb%CHXma#!-2U8N|e!6>yIM}gL94CE@Lkq
zq6d{`&0EG-BENCrNV)%2{8S1v0o6)a*k;&BSJH*yMwxn@>?}qbLR(+g-H>X{B?@!^
zuBcs+h;qu(ow<y|QNGbI;(#@;&h=k{mn%LeZ`V*U@5%<fNy_38TE~tfxEjILLf}^J
zDoP|~*0KjaE(!zEt)_AM2J5(bw&Tn0hwCSDVAr9xqIVEgpp8P*%85~1(_=)Qt3|BU
zda`1nOvPy+gVm8-csug#^eG$Q9q63+VS6>mY4U?_1*&P)rfoSE)u;^;4QY7fQ7d#=
zPTibIH=^FQBJKF$ZS90NuO!7`I~a`<L6-yIW=a>IAR7vu#Lm*(WRq)aJ{H`gZ+*o>
zdOBnM=xBja!(k^M61e^{t_wTlRMMN6Y34zr)1$Dr#G}l3MrZ6K($!g;gEn<FI&k#*
zI6m$Zom$=iQVJ@LX+f7X*t~XhC1~}*hzT{DE~>9qVj>Fa2u)$8I9WXP;D$ST$$4rT
zS-xF@<OiJyWOi*5QO1lU-a*U~@S%dL(#g0zlp#d-xD9kQ46T6<Tp#V8pg-GprG8}f
z`Qkx>o89Hg)N{%9bY-DxwhLA~0z)+vv1EdiG?tBVgBn_9g{&{OmZHTtJO`2+$X%(Q
z?&G=7b=vJJb|CVQGEmfMT_n$ODomliN9KQwH=ZonW4&$tQbD`H&huOh>5D?bs_z2|
z2^15itd3_5Dg+1Gt<_y6&utd`rjq;55besfsccYjkT(ER2Gqm4oQDaK8=08v?!#Pk
z)#|tz!<Q7^7EjDx*$!T*tOOfv3a*4w(%!G?RWI5)pP_v(tZe(G+Nrm+Gtg+q`{>EX
z0(fXkcq=^~YF#Uw+=^XZ0BR4JPGO^?6P$Clj52_5?>0mvo;wzuJc8IfqNYz~?A}It
z8q%jj!JAK9YO_XJ_cgZeJ*gXo<e;>VEU>yGpdPOG>U1Y&0Vjkl8tjPuNq4gwcr3Z6
zOcG0m?6nGNOG*qxMMt7n#v0NT>9Ohui+mM=lcq}*2;$1g?D}L1x#JkE(m$1ud&1&H
z`YR2@wV^DS-mjJrAfgg3&Wp<`qO{1@w7OKntwDPxI4?+E)9aM>JhnQA+=QVhUW5|M
zEp5yBX!A?pScY$gtfMW=JKD5H-Nn<y$*PY*F-yA`(A3z)BJ?pYrCo{?9L<4Pb<C_%
zd^30pQfrPwGVZ1Y7#4l*IvJylP38kEG3jDDt@zT9)Hy){r0+{BYW=G`W*(h41&PDp
zBf>6W6Y^4;kVp*%vv_;ag+-hZIVziV-j8r{bs;3dP;E(m+@9Wgi!6LWycQ`vj2J6j
zP{0o@od@oM;vu;r2Pi2XZY=l#V@oG|j&Qn&a~hp6Ev%Mf!AYN!E}VLc76i;N29p7G
z<1f~i6h$1f4@T<<se%JTOfb1o@=7z|1R&$SRx!`bgPP?BlrlmH`YgoGOf*kmJ-c3Z
zxW;6N$1e9rxpTT!A|!|#m3O&4z!PVi==bE>>sh^y7oghf>gk%Dwe)OjHjT+_x-z_3
z%hoRsAPh9CtPI+-Z-ohXzgeurG!n+D!_TNfBIB+}0^S0saHL0*(mq11Hvf3JdVRSK
zTi)u|VBFGVRC(ifJ$ACyiY;z3-*SypYkURqZ)?6ym)e*;3h-=Mn?xcs?cQC&BtKoe
zUQ(S2c-z!sVCMd2b-EJ~L@04f$q)6|mJr~bAy9pXAZn)qx<BeJ48|A<k5|89ct5aK
zDC+EiDTqTU(Ei3NY-cc8xULU<>n7)UQT2uFwgYc}^;xQrKPF?B?EagXjR=mRRDEBB
z;TJB{5A+fJp~Db7k1$mcvNvIwY{<DZR3sjb)Ar_Og^g@W&feT4tKEG=cFWfxQLux@
zWGysOavz*sDRK7MJ5%6VgBLcuuKWYI{8T?6VcLZBld6ziMgbah>)b@L7nipV6eawG
z+D}mTCu*kc<e91K{GSKGN7yy`(5#0ehIOZtZNx4+_It1CcUe_*8!aybQ_F>bPf0Gw
zW_|5!-)5+xbL6z(VHqw~lf8p!x?l8poACujKuDg`SJS0rY@v=P(UY12`KylfnSMLC
z4KHbZ)>N>Y5!x39c00&ek%B{ApTJ-HV6WS}w`eS<<TH6TadU#v3jl>3xP=l8ksQn#
zvniIN*oOx-zx_f>u*JqAwwKVTHg<CU`e16iu3f4>n_Cp2P+@)Lb+&C~`?RC153ITz
z>4!>f|0g*;xs=zK_}-<v2heAH*j6?ClHL~lu`i@>K)&IS(2uEAh6|!;oULu6Fp^fG
zhxt=hOz$hK`G}`@@v;WxK(!t!USzmJ^=*VNX)13?gZVm`H<gr`X7lJ})ROYp(65w<
zA2FKnidl8Kw<bAn*=H(3hbs?I&AXxLBMr6eG#b>CBhf6`au3Ix(_fIoF-q&j4cx=h
zh#6}*TwCY^3J=4uCpeMzU5UkXKjjCS2nJQ@>FMhYwIIs1ak_vO6IaFObI}|O0Z;r6
zsyJ7w_7*_}AH?PuI_9s5IGC><;*U8<<w6l(cT16pg_Qskg)a=tiripP;ps&}ZxBH&
z1Tt$osb+RK=k`2{WwSK!crQ!$zrI%W<KU=NK8rkWl}O2#Jm2R)@Z26R63!*UMv*wo
zr)(AJ{ov&#5Yw=P8MGdMNSmDDrwahX67pkm7N0waGhr~R4!O-pB-2=}*2UP{QboJI
zHJmLz{(w50fT%*wkrnL5_ZEnrV^NVd=l?a;OkD+8lwyCmWwo>AA^A`NO=RsVCa7G9
zc+_Tz6Y2a?=Ss>iG)Jz!ylr$@Oun?-E+axMu?_nk$3gJ<;OgobGwm@QGQ4r?)w$@f
z&+G+%axOPeyFifE^V(K=j7NxMDqJE(8u@If#qBHQ+e^pC*SOk63}&6fFP128vU%~7
zV#ihI!JSH>SHZ7{&u``1;0_sUDxPN)V~1qfW4uQvub7U*21}sivU&7?PA?o3mbUkf
zjlU4x!u-5!pJO3F^tl^r6x@BVeq6Szs%hUX-2ZCT{`g}xY9|+V(BgH@63_&9w<r_;
zeaquuKB3IeXJ~ze9rb$fdanX6&KEmG7MT^$IJp!92u$;rz`|zWxUl-2E2bpTH%(GL
zsVN~^Dezo!m1!|yYXm|U6$qvJfqcWNnbxQq_&03_*g9XJpXZuf&1?(GPwfE<80tze
zu#|AW@+gS!68W}a19m09{r>J)ApGuF{K>8OH3<#yBT4K#>0j=}{|?0c2^jh_@Lx{G
z|IQ`)i97UPx!+xlzlZ<X)A+Fp)OWM5Z}t3Mh3`TC2LHFiaVPT4;rNSE|JmR8QN;J}
zMZUWocS_%^fWJ`sv)A#X(&Rr>`nU6Or})kJ_^*n;@;&}gyt~JOAHe=?0e{Qw55-@)
zAb+3wpU0km1oG|9=s^VhI0U`JeH()QO<q5bK!3;GmGCYvRHPqpKMh3xD$rjI4}T=z
zL;ivMuTkP1^_v0t7x8`@N&ej1exxoE{XqSr)BOsHa91k;0R3)vxiho!e&hTfU8ex`

diff --git a/sql/hive/src/test/noclasspath/hive-test-udfs.jar b/sql/hive/src/test/noclasspath/hive-test-udfs.jar
new file mode 100644
index 0000000000000000000000000000000000000000..a5bfa456f6686da0ca010061cc3d690bfcd2ac96
GIT binary patch
literal 35660
zcmb@t1yrP2vNcLkI7#8|?oQ$E?(XjHP8II%E`@91?(XjH?hXZyo}Ty5?Y{rK?s@mJ
zSh+GGbD#4;?AQ_e#3w8983YRGw=bK*G?{-q`R4-&5EhV)y$P+7A|%jff@@Ii`D0&g
zCs$~oPat=nfPg@dKHew$Z$AzF&rj>w>KT|C{Y?-9A0OHOx&?$kf|%+V+Su6sj>Z15
znSZmP-!=vTWc-1Zdnp)I{M%m9fPgUnuUIsK3OWjoHugsUxJsT6P=o(|m1oIq;yt`r
zo$pD)ZK}}_Uy!lBU}>_~nmfVNIUyh^oXXab5H0eGipSOqkR#!Cf=|vxJ;>sAquQ|?
z{R${L+nklZ03W!BeHqlESrq~L7SEv!&kG6!wHga-kb~xppD!89KB~RQthEtd$<x0;
zoJ=44GcE`8i({8`RMwJn1|0>`dPsZ>DoRpOb-GeDrP?09%@Y?FL$0(cvu{Cj4o9LD
z2H)DxT;;L*l8Uvuww=wv;M43F!-}VNOp{H!9!`f-Z3Qik%eFyVP7buWffLqBj?BG>
zVef&OZea>NBp&))Wvs%q)Gpx&m7-}yhhN^sila4W8Et%-cMlKHZuOW0;86J|8bO6g
z8EGpmFF^x^2T3aHCj3=5r99gG8qY?KcbyJi7Z}304ZL`8qlm2KQ>Tt;_;3)$(T3i-
zrZ8N)B*`>6dy44DW0M%}gHm)^{)tma<+<qU0<cYX(*&-Mc#np@q)53RKY39|0`>I~
z2$YbTe?Q*6>EOjnHVD?7qBKf=%_3B@P{XQdhw6NY09ucux2Z-tkbn3GNT2Y>`%tp}
z6^<|7e~cv5|LcF>)Xe#xq3L$?Z^P+#b{zZ(5cl5$8rWDn+MDVB9;b4D1#J5#+7bOV
zu#=&q@$cwFe??2dqCb+F>TjYOx$0TjTK@Bo)A{dL`JHROT?0Hp;4BOX5RfJ$5Rkxs
zxQd98wUND<fs&x2uz-z|wId^qfu){<!#_Cr+a<=`f&Mx3<SSb!U@D<`OCvN_RqT<N
zwpCco5dn|nFH4Zc^9v{_bOIll4ipEC;ir!qvz%w<_eK5Mwz<?PGB7KQd;{`{-|=An
zG|hBK=Q-#$;j#YWbo0PN`wmR!7aHQN$#4bQe%hm^$!sgsQ%Zup06>yNYn%;JgQ<Mb
z`*K5Kt;+|!%2Z{l8<H~sqpw2`NW#6+I(J>IWFyuINriw0>^sA8@9Q}^oxuO(G~LNT
z-J;NEW2E#d-bb5W(hR__K~n@3@KJl#C$6vwvGucYyJb-PqIs_5O7?Z#eY6N0oqjaQ
zuWJ7i%*DmOrl*u|($;ym!gjS&C1=%syJXq97g5{BwOb9P9U{M_sa8&Nc6<WK5ltH|
zhW}<BQH3cU@H*H@-CgGp>v1hY@D}f7o1P=;uS9<aBBnSUg#qGQg=};LM7t|Z)=|;D
zsFa~jX2w}oo=A3$S$HS9APQC4+<oGE8^I)jM0vyptvC2E%rLEVER~E9msg3RUuh>B
zuF0~rRG+HAG+l|jFRGKzadD)dLWxGkfEGb#EA&Ha%-O8z#|W8^Aj^UutD}&d`Gbhg
zNf;#ci0wHJel-mRLoCvv?*(`$DRnoUbNUF6>#R+;Q=;H*Aj1(E6vvK2>QKe`Kmm+-
zo7Q>XE$e$#V^j4b-3gfXI8FmgQm8Xrsc;#A#u>zlOkmfS`ZGFLtfd&ZH#B&De9JRU
zCcQ1{a5U9=TTEi;&uE5dhvDnpaDmjTT`EjbTH?GkvA@WnjaI)_>FXg|fVJ1#!k(Zo
zR}I<zQmpBxbMoxpKCV}=5U-VBki?-?Kn+!C+8(!ZXHC}*OnD~<?785V^*L>oG8cC(
z)T7-g$Y;-4t;ytTE^wLD-ebYY)_V-(yWO!KHvk@F<{u+u=gp|$dzQP(4NyWgvpjaI
z>qR1+C(-AnSuI7FFC6DY)l}K%9&ws27yQ*#Rx9QCTWEL#@mA`2Vv*-UvOH(rg815i
z@+&FE`0`<uAaQ3{s<Ya2$vNlpgc}OoYaxs~Z;7Va#1|Rt=zIyE4s@Sxp?_rJXH2|?
zj56N7rjog)jn|0Iq!4d$Qh7$YK-8rAvIe<8GPH_3DCXXBY|qz1KW$9K6%e*%E-ZMZ
zANh-r2u}$VenyPjJwhVS=y!WYwKY8EADMkAC*#iE2zo3sx;61d_sx3!w3Bvl*tX(J
zX&~VBLU$4M9>~Ikh3WI@1XW+MA*C%`TQf&cE5E_wM|E`a7z=X!=y94<NWNj*(ao-O
zD3P#jjI~*=nN~5%*^I!li6Dn9t3(HunH9Sxfq3X<_PCHcu8ncKO~E@Esd0A|Z;3U@
zmk4IiFW4AA<N<NhdEr0k5=Em7d^dsG*!-o0<0p?G7~g^amG|om|Csj)|4t_QXEFtA
z`3nsI|Mo|4%D)Hxhg|CYS9MFLfsgD~j&(CJFc6UNM?U5G4;K*>7WzGR^BEd4{Fgk+
z+T^SK+v0z!;4&06Y!LX6-oO&&k8*28>(tNc@2YES4Q_?}=*>cj1mzLyTDPk>0_dy4
zJn(<1J?BQi{+2$kM)_sG_xDQOYqql;q_G{0zCJ#kaeO0njTb<1aai=t?{ogbu{gJ_
zjq*_A+F5n0=|<t+6<t3<4YsrBLflhVaJ?(Wc{sS4ad-=<X7L<(*dUl)%kwFssE|L<
zd<M2z6A&;H*Gj6nfqR{aA$I<#ja2j{YK@U*qEB}*>9tnbhmAQ(^T94bPc2fA2L=90
z>1-dI9-)=CmBz+3=5#o_8Hj?m6D?Vnj7Iy2j`LSs%l?2OuM-aWgh2B>sP1_=E_h@a
z<y!Feh7$I%YhVP+Pt@Y?mc_Di(=i86F~RW{auXCucc%tf>crUEaXE-ZIbRCb05Sm6
zEMTW<t|LQ@NofH38khR^+4Iqm-Adv?PqrntYTS;hBLn4e$RLy<61cW#WNo*ru0Fa-
zwX12j@KP_%eN9m^YiEu#YfsHP4n@y6PiUTN;9Qg3l##6VEC#9|YIRq%)UbzFf0Z>0
z%0wF-2X-Jb3%xhv9GbYZRhQ$>2x51_X9>Tf%;?dW0Wrqul(tIUtRYdofL(cD>uw^m
z$j{LOO4GOIvdKgpxra<guWzV7C~iACc%NqgL(2U65uZtkmeu>zt20Gn$`tuMy~Hgg
zLtqG))1eKGLtJQyX{16V;$ynqW8FTph@I5fM7vP-T3H4*-a!Axuiurv#mh%FioW^S
z;v+h}KBANNKa0-)BlN<G_GVW9H9QFpe6@rBui+_!_*;02YfywLR%nCEEr&MC8Ss<E
zlGPRz@B-)4ax=}WnQ=6?D3~<BT&Wzz^@IC-;~91>C>KW}u*2COO?hf}$g<h!@C3aC
zd169g5ZyM$Xkge>mELlOq67>{A(yy@>2c~5F%s3+k)jurQp9_-Ws^%uF?lr)Dc4cH
zHq#9s=Zy%i%E9*-E2Y&FHg}(i77}unw@8cJ<?kHRF_#?Owk8rr2~FPR-^d3}lRcOg
zYBEx|Y8ob=iqA}wEM72_?U&jJ>aey$tc>Hw^f>>#d_WFZL2ZvXjxQX|IY(DXOn|Yw
zhxLJ_{;qeFjoMEU?SxD&zUPK^tlBFc6hHo}X4>sQ0cG0B*>D$QuFlpz2||cB(s)@w
z1V+3bUPA?dh-4VGP7!fEweC!a+gH$ud-)mtusz&$HD)!bJFlt<f(r*T$Fo#$SHr%6
zFx=L_j9{ICeY1BTVCIH3G8u={Q#xH$fQdfd@Tga;zFct?Q;K!aDx{pM-m%B2vbpE?
zOA7}*Nrkc{0mDa;NdBCZVBqppN9`dW(_&{9oWCte`>4^v5NRn_4!wSSD5*m&FL9|I
zsW7@58?0ZUkt0A!RuJ=@M+tCZB4Xf{XPmLw1{YJS8Q3&q9R1cm3>jDP1vsq>a#XI&
zGc^;^JT6yEo2PHM$ry7aIXlaI0SzsISn7qyOa`1ix;e=xi|*bIXmBzUAF@OZflV`e
zW)@EPnqUZoM$g?aS5!1is@!ORhwy4EDP;FsHFNd^EIwf~msUTZEefBbQ>Tk+$cTMR
z59<aVr3y_CQiV{n#|fCG9n8UQ2mD_ma{i}U|8JG#djC`r-M=dM$^N7wf2$<_K>ve`
zwEmTh{6<t)&nM*3526->0s$%f9isl%RRsC|Um=mwb7dg<co4R<(Q_1X*0Xfd`{<be
zi?sa4-M0VE-SRlC-?$sNVb-A4tTkMiy-7+?3tbK6Ezd+RiU<RWh?M6t!%!dAves+~
zx&Fu_IV&h7NJww~15L8^wj0-ov&IUF4@Pi|#KCwx({0>if7J8+=M!=#8VOS`eG#7k
zgRp@3mq0LN0gOO9`MzMyWQNW7P&s4+ig7wJ>BLI9DuBRjiv94Lfck2FpTAP|>j+@W
z&^)u7WDT@ylEEsWSF&#i@`7YyY1b^v`DQiqAd`5(sc;*$oH&q~-^4QMMPX#M!gk{h
z?baec_L9V^y_Sf~(0Ta|iiGyYQ!ZJkeY4r9#*`YG@gmW4VmKuM)O!Mna+Q3gyVdDV
z+_;39^w#v9_LXTbu^C-)1Qq8&Y*hD}g^g+f;JkiMrDHB#SY*7<aC<?VoZ~ir@bsdl
zU7@lnzJygO2x((G(a<<X7cF@fAQYh&%-T%}M>5|@XX;?xHxd?3{FK2Vj;ppiqA7y8
z>~uSrgAR#JZ8~R%a-f*LVD||X{n=Rsk9>#S-A)!%De+e&Oio;kxKvsa)V`&)+-@;T
zkB{h8CrQ=dLXn2QE#G4o*AI@=#ayE%9ij%pN*kh-p}`Gt20|N)XI%46`oSbY7G(VA
zL(L<%fZeIsh}f+0^!_jE`9loV^ah_tiwJJ<A<ll#f2WI($=)PSy#&0N<}dh81$;J4
zIn}^+3~?Uvu_J7ctjZ}9SPF5EgwT$zPFj(@0t-!55#h=`gj?Fmu@Obj%H|XfqE{3W
z^z8xB7HZ*ptGe|Pj$a+d=nL}G<Wj`H3^lAo`1KWj1MY<y&<cI+Jhyn)YVQmYM1BLG
zN(pgCV(J|)T~%NdSR`-aA{8{O9P<c#7b)5Cz2G6Db_(xwr+&Kmgq%oKh|F6{2=^E3
zdXWR?;TNlfzP@>kLL~vEhT;h3S>Y4&qlEYJ{;@TtKwE-IhHt+Nze3I0rt%x`eEOpT
z&YUAMUVivW3m*!|_dg@nU(+aGQOjnA4v9;9B_Of8nV!4fq#vpl0s<rI^N(V!{E!M*
z$`gydf_44)#V)&&H|or2*G=%tJUA9=el^f-{1C?|FHg@2C&vt2o(?bYORzHxaa-(D
zcGpwgD7_Q^iJ*B=Cp)Gcz2S0Zz@0(9|B*kZnBaDHQ@oQCJ7gbjiXSN;t@nX{%qr4*
zcoMI<(t!<|(_j(rUUe_O+$c~lJ7VmqvLiu!`2mox?9nTGepsA+=2hH_7eUhPxvL1N
zG`GMFE3!Ih`Ibht!b<|UcwW|Y<=pqiYyBou7um);?tfq*bdceC6rbgxKc2^mw66A0
z<x-2K!1V$SsLPR#pmk>$X=<x}J4Vd13(S~NR6`GmktA}%rO2mlkf*%%xuSiaz;A-u
zI0@}#VHn>vR8i(6$TL@-V<mtN6u;*S_TW0tfMMTBOdcpWYj>0hkctr3#DqwSXAmcJ
z!FRfAN((t&>_zF}`wrV#^^kR40|W2_#u@9V)1LCDlp3~`r`y^)Hzu1_9-Fi1y3EmS
ze)Xd8UK~M%Z&A(L5W1&xFD@oq&rWGb{+w{2)73ty3$uc>fchgW4e3PUll2y}9;6FO
z3HB<n<>WF~>YB_Y5G*2RZbm0}AHz>%#Vt5({4#`jGHv2AL~X)W=|;Xq-aFJkhMEwp
zcX7{04o~~Y;d1}8Q2Q;1i&_6?8C*jKQ4T3!!7S(tbv_;*nJXWN6}TZ%>qiEM5K91t
zL<Ke2Hsh%Gn~}yY0MU8HeFN5ZofPae-Ohu3BimuOQh=CZl4BfdI$b)@u*e#T%F=27
zl-%nGV%3Pj2XaN1=5HPj;TSGV7ihuWKi8`Wx!@#um^^Ezz)M-M7x_&T(m%AOulhYP
z4o14*EcT2C4x%5?LNd+HA(3~WRjV3>qAz@}d@#I6bfGmZVLbKL@@pmd*A879`m{sE
zp{%t;X`$x|QaGsu@kExL!E)*_$RoFfk+f>68qg7{!Ss{FG2b?;6t#w359j`}-CfF=
z<(R%|gOt)Jobj>weh?WzRyJvIDJS${y^Y&+VNA@U4U*-;ZgXMg8D+x|NGWnfr|ga5
z_j@B9W1T{yCT%m)giz-YeTnHCt0M@&0OFV!=a=t^xqEw%kSut>F-8sBM&)c8Qy$Xs
zobBWdMo5K|$U+_F%kW{jpEKlEyREQYavQ@ufc?g9KQt`#Q~FF-N#^T-!Dc2|QURfa
zn*)7~nUNcTfe)PP4xELN#V8>^oz2r++JFW+M{a(;j&5TSjim%<f_z7vtN7Z)E|20j
z30a$<2E<R^s|wsj-<Fl{@%x)LEL+$ls!>mS`tkkkzI6_mo5QZ4`C6i8>%o_+cAJgo
z)*0SzQ+ec!mwjoTP&(F0gI^VyoV?G;5TwonrF5)!2h7d$UL776xr~0P!o~a*>3;>e
z?BR%+2W~2-J6)&ch#n5EbjrvNu<#g#Rjzwh!@mr#Ul7~CyJB0S)!j5wYG~-MI4PNO
z)6BjWiu?)!rjG?{4{o82BC5pwV$SXOGi*`i@I?F#t{fMmI>%i7T-D6W2@-T3-qfSa
zAee`S<v6`0^Gj06cDF~wT1gJnz-l0dwSZHDXM)ekl-WN0x0sS7O4c6k?p_FMy05Q)
zYF)g6y<x^a*i`<(CdL0bn-o5LUTc&8q$E=KAk;@YQ=rv+3#=#z2Y>0iY#oyjmZbW1
z3MvpJ8iZ)7;pn3z@+e-R{@$_4LwA*dio*mi%5*ga?h~3?6td8ZHc^|#FKZrNFviBn
zap1Y(GM?ggx3dHdl-CF+fJJH$yrmc~DV|yjMIA_D_yawDzZQTdZztSW6$%g5m$^>~
z;8LhinRE-m6v{z;$~{SRoS8f53(YbLUfgjW(lOF?F?ASnlzfL0U`^1*`<C89hPGTF
zDKe^f+IqR~ww_CSrclvU4tkzqkL#)UnyXg8NRV;EHA5aw>rmS5wO}AB$sT!Wk3yRS
z+Bf<pEI*}*(!9kUpSnO9_F6%s+^ALU?|t$-7jm_nS|tMJO`Oe+RGiND)oaTxYxg!E
zV>Y8eKV@W+;lN>MorxCoD-rn$Gd6JYUZ#zGSPn~qHISf!u4#3EO7^KKR~kO^fikCn
zWHLg%feNI6gl7tA73>)#R%`)iIb<#V&}18FQaG-ZF2Dqll~jd`wQQTSXY}2Q#EEZs
zhWZ@oHtmU6PgV!~VYAT&zM9UwY6T)9O-6;%ywN$tLq9Vx`wZSQLj9<GzvR0;=h7|F
z+GsT%+ma6%x(NOTqL#S-p-6e52ukc4DQ?aM6H4`qSrYNYphz7xofTpZ>D=U(>T-xU
zFBnjp$?yhN-WEY>w2VWNiCxpr*(3496WbUoys8S!*{6Z-A~KI$<S$j9DYS!f)xc$8
zgBb+9scsN@mKDfgr}DW(X3j=Y2oN|WH!Ti-<k=b{9upFjQ>P_34MVH2y=2G!5>?F#
z-eYXO1pW001+zitd!YoA@imjzPxCHT8H1)=OrV3R0#^iBi^W%)*dkT29!v>=r@~=)
zefZtkYY0e#pai1NfdVFIOSNq=Yz|(oMEE)H<Sub8J06>X&?zN`m!#*a17+cr)D9n@
zD3spGcTEo!&R2LY<ns^L?O(2dg9<NT;e%S6|2xzYv~kk6H2P1eB?tE=-Eua4CJzUP
z)Ec%-GdSFbDTFA1KLr$<jpoH+6?L|4-C*7D2dN56A`3}Me%$nfeS_agw^o1%M4M$w
zVPiCT_SzUtUweFdCH|%tZWg=~4+b-YmJ-kO(Q<fUJlm=FeFI3U*$VeXhw32{0zw(b
zlF(EG@sA~0jov?jx{#sOh$s4_;HOKDDJ(Unp#rECl*L9BGs-TlFUzD->d9FK&fLz`
z7%E7SJ4}@hBeNN;6XX%YRx9j(>o`h3I*xuT?P7?Mlnnbe-=Pss6(r-B1FZ&2lyV`x
z{ngkDHMN@50~e{Ey&2WiqiHFWafn?hRYC4Pq@su|isRYp?U#)*_QJ6oF^c|4r~(Ch
zsh>uvC$5d4kB}n1H39ru-be**(kC1q3f!8wBb2ANREc95Mz}V%nR{#*>MV4Z@dSF|
zwZGFDz(|5wjhDN#(VK}?M_LhLsBVU74&c-t9VQn1x$hX&8jgw1V|TaZ1-;|E@o5kh
zh->~w-*I>9J2=Jv;_^W?#OguNsx*i#-%E>$q)cahM61*nA#slDIg!|u@K)tm^ryj@
z_eqdQe(!I$?eMv{QYr1j(O67RWP4_x4Ohs;<4kr9`q`6z22RGQn}IKZbQ|jMby+ZN
z!Q9LL0Pj2O^}d$0{`ti*q-0NPRkV!2lB`s=UZ70?<U(wzAEn6kb7;Lvkh`c8{`D^}
zn;3>nem8&TJR2BzNGy^~qgwQM*pXF6LGto4S+B<aIVo&lT+w?G{&&IPEXWsVfM@xY
zGuWb&wi|n}K(A-=I21n@Bs*>nB$&D}8L5*qD7#{PcrSEO1Mx5RR2&x5yt--|4fT@}
z1yNn(oX4-o1BctLc@tC1&WDxNO^f0N>YrvGsLk_W>ZY321J^s|9(QCg1X$%}>=9lD
zEfkCNqQwn5{&B+oFOsb?NPM06AldT&PDig`YGePO)r0@)=$C0WAfUX%rmb_rsB>lM
z@Ks|mph%R$*QHi|YOS8lpCxW4@N`r9d}7fwB`{^Wnuh+(G_EHH5dfH{j*hZ%-K0OH
zv%TybB-sElsD=^*7L(-$i<pWH3Pte?F@&0p7Kw`WM1QKGn5JN$*hfQZ>2yI?jls7(
zI1PMA4L>lXN^zQLnScu?ttYdbD~1SsbE=_XR(BR_jB*<yn3!x<*q=vnkuq~h4^?4I
z1NiyV%wsK#oY|K4aSa$vUf(s_s!*?6LP0HQx~!ITVYX@_ejI*d73$57V(Z0cG#{iA
z=P7N`0)CF11rmpo!Ya?(pE2gl#C2TN#A_TI%Sy>Kt`gFxJeskkc9#d&>ahpPxE-fv
z&Spu;{u<Rw<~k7yT+baCp@nXpI_d=~t{P*9bnDfc*hLXw&t#=c!imB+gl-TY6h$3^
zqh_aWpqiIVvDg*#kYDi(vOw#N4zz2!&6N@00b^S2Y7K3{IM@4*@5KBywy{VL7*$EO
zFW~+8+bn(Ya+PbFxn*L#P|CXfb39Td6>yk+n!6x`a=^g^^_b%Ze0gUZMw1oWd5$Z0
z*5FK4Z<u{^&AaU93O;W)rX`5UD0j&$1K#?Sd%CCM_u4OU3HcdQI8!{|kAfgPF9k$Q
z&jh09$;pEAz#-EKB`m-VkWfucu3<Q`jhczy!;ZQ>gE;L04U5e(@h9UO6&n<(q6$SV
z@U`U}0-Z4qT}<9Z$*$|E%HR+XbqT8650d!EK64YkCcX}HYe75@^%o0nG9O7JatjQy
z`A!gqfbv|KVy=r($t2mP$m`_P_;w2NWK_ySujn%eQv^iT1r%Ardg<hF$35}n9C)!<
zEy8MWE3c@M27r`8^x~8>O8cxHqBI}!z-lGT3Ej7xndd1-7Q?vA1#+09le{Y%oXcQ*
zZalJR3b?5iA*U1ng(a~2r%p@6Ogrwk+A#NnTk`+&+ECKwKdTJmBz|kP2HqPooDUtz
zARwUHql3=Ah5O2cl7Qg<BscZdcXZLHINRhrrYnC8u1s#U-Re%r`UV3n1rkm|qxF~*
zeo!xkFZlVR-D>+_)(ywI!}xDKzvpMVKu8$7^vP^|^gLo>Q~|$Ge3%~AVtql30sw2i
zzG7c;DC38Eji~&tUL5(G_~<T!$yJ(^jiYeV1t?^e!x~5cYPAt$2P0C_S4>WmxI*fa
zy{5KKFNtNRLIk7_A3&MkgsXv0Ys)zWJaZyj$4P5TX35h{O}1Ba{sx+a_Ua#Y*`XV8
zjj57<%EM}BgT1j^%*NL~C>Qa1Iveej^puK>5YFWe#H@l*`LBNFpxp<b#Ly+*NCNg!
zu6s1vkwv?XkdQ|?1KYWJ8JH%(IelyrlaA-=<!J`&$=96D!BHB}!D5TKvQ0?<@FI_(
zxxE-jU{>ptFK+Y(Vx9w$)gp<m0Vj(=%-C!8X2Sd;(CSX<eQd}FPMVz`Ow|Gv9c+xm
z!II~5`n8}YtMNm6X5YH2HBke>4P)dImtnObg!209Aqu`G9N!)A^q!>Ul(C!ahg)`G
z^_yF^@ZpwKnmyx+^V06p3c4WSyBC$->I4`Xm_@~Au}fQUsF^LJpV1p2jxO>Z8howw
zn+NHxP0Y#mCE@J!uD(^LW4DWTCMi?~*UB}0qOtCshi8s=BIA&Yg6lGnzc<9BSm&KO
zJuWIlLe2GZJ>-z@CndX>vIbb;5p?MRSLTl4`KW#{OWfKhV!LNfbM3<|3peB;eTF7z
zC%UIZbJZGKQ8QS$8J^L@${7_oD#M$N%f6vM)|DE;dipV5k@_W}oZt%i*eb;vQ3@Bs
zEN`xgWkvj?q%itJwOob!?A)goiX2+mv7Z%DP&F(ZO9S0oq^xRlnAKHACf^${HqHv@
z{lAwa@?m9b<?+H4k~7`^qv`s6Z065%I2iw)Saz1b&$RsID9)cTasEFsoeckJ%>N~>
z^`EBCNd6Mn<oCIi-}f!^cP#mS;FF8P59hGu!#ULY+xY+I*o;qD!O4n&=#Q^|FO`Wt
ze)Ib?J`<Baxr(;`-JGve7_pw=MdCs&w2MPi7k7)lJ|v7s`IH#C2*ghTBUB6wjS;%M
z)|<lWXDL%P5c4c!sfzz|^YbHFZ=FF7oE#CbD#k>H!v&W^q{-{u$^k79j&5lvg7spf
znL%GahMsWXX9xal1K)%n_#!a;Pa&c4Zo`xN^Y}Tnr7f{^3-U3Z34z$?4ML>4hB^6M
z@K~X$Mkl~jDedBxs{~Z-(J|o(4+TGKT%rSEGWJ2G8TDOR;KT?JwckX~9=W%Q@(nhS
zKX2XTub^s`gInvs<iBv+&R9>}iKL%TtghVYoGIZJdP^oJgvjO4F}TbVDZ)Bpc-YGr
zHIwXGZ}^Y~`c}QNM>I+>=jDhi_9F`|L*zO3GP69k(793l!ZOPOBp2Tu;AFxIMMLH7
zJcD44RD>Je%%fxxuoY#CH@>8Cg+r}+DR00Dx9;{pL)g5}nAF*&O}=4vWc2oSGa9sK
z<+o*qLrN6xTUsLAzh~r|4N7w{bmtunLBJMP2Oz7yI(#;99nB6u{YkHZ=V`=SWVd@Q
zsWz4&lO5%@yye37d8l74GDGgu_!htOR#kAA&`n$JQmPBLWbJ&vdU@_#cYf<JL&c@$
z&MHHc#Xw`^4WR7jCfh9i#5GkUit0InvYk_2cZyP362-=!jO$2GJ$nd<kJBKOkJ-)t
z)$scp5vTg3g1LbFmPv%h1i+)y7KIr60vQXJ`E957=Nej7XFt3y>Iz7UHM^)En#lBE
z9=yM!*bKE<n&ZZHt&|Ikmel|RM{VQ^?z6yqZsts8?ZX(r4-kwd%O=8Tv)|@&;nLC4
z%JF*B^Kb#o`mG}*nD>S-eB|{y_ggIlF4G%3As!|Fw{a7D;EKzZEfJi8Z@$6o?Ns4v
zJQ4ekd&F<7w3|je>{n)=vfzKlRX$O1lWrq>X7aSF`_TqJ614Z?wrhq}Zq{M6j<a|x
z^)6gGRb0Zr+iYXLUQ2!2K)AB<6cGe(r`+_=9De|P<?Vz;Pn9WBo0~*9(ElZ{6Cf!`
zE19nZX2vQ}8C96zsXaE6$1#^BP7k5ZtU@;sh%VjlL`Y1@MogbK%qT9u;BXzzTsrJF
zCtA{0&F!Qnm8{O3m6v8cSO`O%mr|=cnu?~K$=WSzRrJKaY<Tztk&;TYqDlf7Q#>4J
znIIQdmSU)a25_*mTK3G|T5=ccJt3qPbC(!>y-@?RA@j`aa?zBD&4IC_E{#NCP$;oe
z{wZk_dICx8rA^QDcG`NI!iaGNM+<%?u1-T#xEC)~;K@_mhJ6wvS>%Nk^9^^(w3q+p
zVR9cCFivsg%a7%3i!MTAJxy4O4ZptcZO<2&<Fn#S7tGct%TQ8`(d)V4e5PN7$&%0;
zMkqdhRm}P*&p)OnN3cm9xJEvQrF1~n4f8es^5Jv9k=Y!aZ9<OSm_^`BPPHLdaEb*H
zLjMpHn5u}`bnGdGI9<Z^$!NW=CAE2`ly<;Ez?ArmTtbS<sjq-scJgkmVL+EoxQ?<h
zGF!Q!g7l#3J)L}AgW+%moIrDooFt1S#Ac+^y~33-1l(qLBJpS;a*3dLs$R)-Yo*c{
zx<UlfcNs<1rDrC{CZnCy`OiBpV%a^BdZfgQ{)tk@4(iN%6b6me@bg$9@ecd7mKRvt
z#P|i$NtFt)y10CZqVj>ebXbeN_S<YWu}1(I&cX9fZP8K`{jF_trN&3BC2Xmp9N@96
z6{z9<>_4tbYL9u(*CyN&Y~f_Y4*Nbq%4`GmWFptUBuzB)^-RJ*ir9tot?$AQBezm-
zLEh2<l=elFs&K2n;YvJ0dJofG`A}FQY@@|oekt5Hd2I3*lZBpqgD<AsLVYAocug?c
zDd3Z1lbxIv)4uj0-dcE!_-29N9p*y(`lX|=1P<vE9MY#Zi|7^o0;-W_3(T5<6Jb_f
zEdv^2KO~}7A4KWFc8PWiArtj7(_ce1TDEru7=?JX9~<&w=Sv+j@@|1hXRLq6X60(W
z3;Gvpv3h@4+o&zYy>Ikb>+anQSS!Xnd@1VMz+chD<>`^1h}vuo3@N>rvi;@bP(!bB
z)o2ihcU7#}I);sxqq{3_kOCc~*;nE#)rC`8mITU+G8aYZXoJj2Ob~!=+IiP-!=}i%
zoqh3DOXzb<7~)~=)n)R@ly)1(S7JMl{rt=cv61E*z_F`xrVYm3#XK;S+j93K>%%Ql
zG}W>5F{F%c+r6CaZ!SQHlWK--Tujeym$9YUvLjyR8C2-%YHQB1U?2q**4?KI4kUcZ
zG)s<7!*eFoHmFXzOX;0FMknj(?U1Bi<AT1-O+Jg<G>MC(802LIYmXmFoE%ong6ky@
zM9)>;TRwZ%6Pm7a=XkK3PLHHd&DHuu4-CDHS=G7Rn@FB8#&Jh)tn%Y3zs1>ca7|p&
z!G>vxeXT?{pFYC~Ca#HY*wpUS_J{wp=@*o>&UAg53G^D33X4}2^niOy<a?FrUm-Iy
zgRegDxH3qf+S>v&0b_B2Z;>BV@$MT{w~dY=b}x?A58uqVUO5Rxvj?NtCGp_Lveg#a
zf2g&=QKaQ-V-K%{E7+VHR+Je`#g|;BBRB^98dKEpfYS>)yYBCp!R$^Q&>{-B#817_
z;goD<NAH8XQ62WH2Nq$&z027Ec7*%PD$T2(h&3P17Y1$~bm#(Hh!s9Dm9a8Y<<H)&
zgd63EJT>XG_O&6?Lif{bmGJCO5{VNUkFdNtFyNSHtPpA?KCSS)ZD6rYvd>x%vMdO9
z)Oc{^I=nkj`BJsemMkTUIKgG0FsC^PC%_d>pThOS4OMbb3aJnCF(x*5>V>!a2Z;6S
z0axqm1gRBz_DuHBnWe0V_>!4vrrO=o>eZC;z2^k?NK@0da9YhaJ9^E?Q|{nVu2tvF
z^|7EV5E6{)Q<2z_;2z|nMN9mQD?!RALL44QeDi#no&dRf)<9L#{zLD?fJ<QLnFgT-
zBOm`Vh;6KXyLo;6c_aOKJ>MuW(||LRj|8UxJpP`x9?!t|{Qz4B%Xw96Y;CoDd(WIE
z>@ME5VR=sJ?2sZsRH~TpO6eNxfF11@_;O>ANDH|R@LF6$u<LfiJ+m+5zzXrzXl5>m
zJBGJ2#N8>n{%)|v&n#uz3-@veRbhBX>Mz>_+&>YP&K^Fs?4~|fBRX3-FRt-#QbSdp
zfDFmNMhKnm^(dM3M^2@3i<V{SVv9hBC4lIsK^Kg{^rDc$87LD}y=QB8l3obh0(0*v
zBF7c&D%K%9ya*p&A08sGlMu$3!bdsH*uY0sHum;2O1etR$@fkT3X}cXp^+jQ=$nB8
zRY1vH6U7rl5EQ3K5Z31ZdNyB2N)7zIzhInzUj_fMqU*p5c0EC}mf-s^CQA_%PI1(f
z;Baat-|V!OkWw9@*5>;LOkNt3W9Qoo+8FoBf^VE7ZpHmO>>tlJZGjh~f<HQLxDS&{
z<$tLn5&anJ7c??3v(mF9qW|l{WKUJY0ZHk@Ic69#OcGPY3xi~lOVZ$Hz}K7wnbKs|
z`?|!-@7oKVKALF>=4_nUw43YO;Io;w=huf=DT*JTRPCBK%%bBdFHwFi_5LkC(aX3e
zmX|rTDm!sWQbuNiZS9xS!;yn?$Lr1vEzs#e3;b*zPbh1oa1}T?@`OnCCa4(L4?{X%
zra&SW$=ID(KxG$%bP-xaHj0590jW|enh>Bc_lLD+Y?5MY*|t)kxzF$|g4M^6%SWT7
z(O5vgyfVjvpeizEHbn`T*=7TyHkxV4#CqV|qQ}CuT`e2GSxrWGMw(!i*3QbkdVWgB
zF(kRO<cn)pHSoy&5g={9S7ny=1|Rs~oTT+Kj7(DKMzh(z<6I!`(T^67LrRD?|0TIz
z6nx&_R^~DUOKcvN6>K8HZXtf+hF>T!LGQ;ZFTx9%EOLz%Am{=-cJ^n~_k5z6aoIFv
zZ}rjVo^EJ+ZUJnyv(#~tGjeOpZlx~tW9Vx54dhh@EB!_pQQC46wFW07E^#X6WYuBi
zgG~8E&MjlXk1FulS2n{@N{7U+vl<|vQ9KlApcYyF;^3A;OEU-+C2z{j<O)!l$C@xk
zpQn~~Pl`u3qRsaR9~5l@V4|Wj^%=Dnb+`lNkg4^L9606=aZalf?AG<79e}xU_lo{X
zcqVa@2sE{yhjZI-w3O)s5isl>U0#LN>{8w3n|<ZM3{k2qr=DeZ2VOv~&#Ly?QYSEo
z?Ow^)_4DELAZzMY+j}RA$$wc52ujJT@k~8ovo^WQ390G69C|x=-y({BnSv*^g_$xL
z#!>T8dACY%Jz9*ejY!{#>GA8%vS+{mz%2z!VK57rCi@R{;<|<{*rFUT?M~))b$>yQ
z47^#)=}PF0>lMk?SMLEBqpPc+z`{7l-&nY2Y;~F=;N@Kf1`@g|_s}rS@YS4xlOdy`
z1&-+0QP3~|5|*?APX*Ld9r-D&h7h<XY_o#MDXqf+cII!SQXD&@R7?Z5-c6CQg>#h$
zzr@<;Y9{tjx-zKNn%>q!>MfA9PR|9r9sBp5S_d7$<0#g1{m`h<RScU1C91{v7N;Nd
zNbI7x-Hs8J*R%3!7FY%HQB`k^M!u98RM44H*40p~8y~8Yq_5ujYmat|YNLf%lU6aV
zX2ik%u)Z>|WLo25N<Ejl&pgK<+6VCy2MfwTuheax7~134Vw%^iDvY|qiU>wlcil{5
z^Xj}c{9LI)l~ZY7QI&31Qv7&68%wQ&F7pMJe%7_i|H9s{Kr&uZr+B?3G{gFRI6_wG
zHeL^s0ZUWKLiXy5fEs0Fj*(Dbt>x9P1UE|&u5BPyDiRlt?UKXgaBAH}@l|xPoF5g%
zPX<*5Z5lhnFp8dnu~3$T>QX!3DzC_<J_p47xS_|}{UMBEAGHISir)m{oXxQ7j8Hfk
z#EcLd{k1;iTs~;U6DENr2XNjNSn`llJt7-YE*+=Y7Mcn>C8g<KVJaLN&zLkRJeoQz
zldZH#V$KmSyoW-dBIkmx6d@ioZBMC)XlABZVPNI{W;bRMoMGPd-AQGz1c6`~Ea8x)
zsH?%W)5ZVT<8~7qup?soswlWDK=_awU>wsSG)kkMgSd~eJh(C`os3Br`HQI{*7%b;
z8Z6r}o?TGzM#!MeV)phB4zkRzNCCm4j<5>R*ulzg%az+^j{Z0dj@!0qzf2B8?zi}N
zZDZC4i<sksSFt^(=wNx`f;#+F>rwZS;hB)&d-vg2uUhoJZ(vZIKYB&EO!T=Awa{$;
zaItbsQZ7wPtu*+RKwMg=o8!J7mCHR{V`0XHsr)0zw5!H?IKB0O0KMG<#LE;QQqr46
z5`Hbx)~$n8soOx{QyZ3N=6%Z@Dv4sFU!wk2Z$elp_y+PvVYu~KF3b72muMRU2uSIF
zUl{&c7V6Z@Tovcgy=9DJ(<Eu4{i~4Uv-x_W>7wDxze9aO2p$82lufK(A6=EC8ST&D
z=t4s6&9ARZ6I$`jwL(U!10{ll<C|12UzQ4U-e75YxAuCz<#Hj38)5*7Wtxa{n0ThX
zXh^>xs{HjZ$_9k|m;^0RMGBtKsKner-na%zdOvu?fpj{?hjZ@8*TvoBndIBD_kgps
zn&zkO$<O>i{%A;Je?^YU9r0px!ARqYk`>oBLHXzl4Z9fzb>Bj#C126&$9Bp^ltw$W
zm4xg`9Mp{^YdaZ0&~%{3x!sM=Nf8ufv{Ops38RzbH{fsu^2KB)nDg-mw9Hmf7y~!8
zk!OE^$ra3!r$QG!YUGYTH1t)vZ(OWulCL!=7ck@eG5#7;E9QdY9MgdBN%$A9i1V+$
z>b$(=Fm_f>ykWw3MZ9mFcN85uy!8Ar$#bfdRTVUr&s^$)H}xl(&SR)wL^#h@&q9t1
zWN07!HCzt0M<9d<U;U+I;><u`!Z0z;0+!nYjDXb&a3Vmdz7rEpGIeJneP>P!%&!A;
z6w_-k4s^`=wn#zOq)MTyF3gIgWgxhRQejBV!XN0A_btzGo#fNOag_{evZ9IqZJ=tE
z5{t&J@OR(lDivtUX-R=d#5n2FH9UfonV|;C&5);M&NtTj%<pJPsD*v$xg7bbP|`nD
zY{G8!hY}o$&nJQDBTP{wFh=UNa;(784A;>npmzl&&C&89O%dJ0AfOZ@)M!7c2r|}X
zTOW~({j@K0hmr&gP_a8om<U$F)CGWAqkaF;mjUMDj(~E?A*PO)N<G$6qaC&=-LqKO
zIM?8M7km~1B@HckbE53mo|a>H>@7838Cq9&RN%`P-he*?EJbW)Wye+olOsxxoeD6#
zaBZ=_inip_P6@Zn_}=C%$<3t*jot;b10!i%N*l9tsGZs;dmB-+xN_`lik(WD9q<aN
zQ~MR(@GGeid1jP(x3O$Z>=(gLq_7T|3qia%pkFZXhgW~lUkqukv#$rk8cFr~HXjvU
zuG<+ib4sRkNv}91h-Srt_H6ykS|SYz>O=}$H74AJzF_8mBU1RTl@!ayADbCGw;z&K
z7Q<!|v7X3HuN|1#D}kSncLbpr?#l)rtgfh~yS10DjT{-^nYx6|b37@~3R08g@j1j|
zat<ppMZ2v(`axqWn@$JMPDw66iNd-U3YqdJzpgvavXU2zWt~}|6!OU>{`bDTrZ%P9
zZMUK0ve3Q9Y~HcQS!9Yt{ghA#Q^hM^%kexpT|SKMbz3^LFH1{YN}|Y4joF(*$h2D;
z6z$aDw56_~l_if~v<><gnpgO{PAJwpT|jH!KN`m?IZw&Hhac!2v^neFGk|MEL|G`j
zzu{hYA8*WCri5oJ(GzaQq0XmKtanEtX)0bZughH-y!xk<iFU~-y#jbtuAp1<Hj^m4
z19YG@l?)#R^u)5s#63WnYD8tUBcK%pb;%whLN~TuD7>S1iXI8aDIViOd3LmreY#p&
z1^Y2egrMF^wrv{m)-Eo96Y})4w{af}d3RVPNUI!gn}l`y&@8A)Qf`q{znO85O`pn&
zHHUXLt^-pAHz?bfwhGouAl8mLBb&u}G&A0(;h9oqt|`{#<ky#2!?%=To}?X@Xq23p
zd_GwW`E?~*sdkpW&(+hbDX&49#F*#FahQ~a@_fw~L872cYQway>}Ma)X)oq=>tse_
zR)ElADL*uu%5Qm-V00jg8&d1WUi@5##Mo4P;fz+RwkfMV5=r$df!pFapwc`-odgtH
zjN47*q$^m+32O>%gEO+ZzYbWaID7{O?GB{do98B{kcNtKueB=ykP-E5G0INao4EKc
ziED^44ovC_vYT*Bm=0kQQ<dpb8`qX5vun>mEllv#J-f{C@Olz4Bz?kf0RT#8jQ+j1
zJ>pb8*tdv_VVPY)zNb}Dn)=Mxwc(c|u5zm-CDsMBrkB|!TFv#|1VyFcW2)VbYl{W{
zKnRFCZ*{v5+Nb^#S|YPw%wE8brraC(IG@aBHpx_%Xj_F;G$;D8TiLXy5_PUK(h3U3
zwx7(8`j1MQnM8I-OMQ1=I?1$Y2w%lasANF)-Pq3*vv9sjt|d`bL&hS9ExSxAkMa>G
z0@)G>BsQKJ4R)hn2i37hvuGurU^U?xLQXh;6=AW>imo}wfoRBsMjtq<u{g~@w>0?i
zgjTYou)`teSN95~w(bC49ISahw7a^FXp6<`eZ+Q<H+UA{AOXvuV>>~6#y4;5LK{y_
z8olP1PE$?k6rUwxjjA}c$sGboN_Pb*8T>$kEiAU~bXr;%GeLCN{DcIR$Xdv{qYwuw
zol~_1alO4m?yK}V8(6K7>>SR(YPp1pmvn}Mw5>eaYn2?dBKMsmWSV<+o5pfrZ85|H
zuDfm3DXV-#|F{Y}PIYZT0oOckQt=*R($t~@RZ3N+oU-@@j44h7+N9WL%R{wH*WhQU
zR6V*Y`SnB3B1+I;pm;Z1G3pHDNI3h$t97f*+fSlpL^0^aC`-;@NY}wcQ}b6*{zyx*
z_zOa!v|(Q-Pv1UgZ{i5wrvv)7_SvO?lzfm@=$dCMbzKYR=u~b=32t9ctZFvoJ{gj)
zG<bDsVq9_6rn@Qh-%Kc5AxBJXrR7gW4ctWBSSr|ksiz0g!K>D#iW#JJooPcGjUrnx
z7aVt%t&vbijRY{Y2V__(cp_o}fEF9=;lQ2ve7G7qOcl~oZE0FioP$%naon9@_~Uz`
ziU9?(KDpxuCTh2MWskr{E{O|n>ywKD6rn{3Tu021NFoGmNFu{ZNUPbFzDQt^Eh(ti
z_pKB~!(~09vY$+PmsDn^E*%*)mg-vM_haJM!{V3X`GY}fzS~_O^XJn^as*bZvd#!K
z+TU+n(LcKwr(h)vC$oZ9Q%NmyMUKqfVs8Q_9tglE2w_dkJo;g1=$N&|^`ixlfNL9i
za2|&Cr1dw2qv_*fE}z36qAT%UZ6q=A?1><|+x;cRBU*OFMmd30im9CnScTQj_*jMw
z8+hCcmzVrKE-6TEq_{vU6)ITdIi0NK>zl-6A|!B<s6~dEQ>tM=98gCEw`(BkK^#zW
z)MY37XC}h}G*-{2t*Iz>Y+#39B2Z8drRr^_Zc!oERwOA=<Emn$<+Ig-W1EpoG1f}`
zkTd~>Yo}~z-3>$v7y6j8cpcffm|?KIb~xW>uCTCvO<_DFB$vyPFw7?SLTB7x52Q2`
zUq}|@Lbc*v`-mhw0<lpEz7{5;c1uy3=t6gB)Y!lM<A$Aojm9jKN`XUs{y5eE0|Mgu
z|6MQs4ZEN&-oj!=7H#0&<RrD;d3mgOrBy1T*)7a^{Ke?Vs`SK&YONJNIsa#bv6$3K
z$OXtRAYX4FzNv;|K|w(fuRTh>SN83jTK;}TSvk?S_ZyCbp&_@-xb!F;oeoesUri8=
zs;}YJme2|k#;Y`X;62ibe2MspT2w7+_f@AxO9Q!8rzUKj3{-l8-9_K<Em+E$<%W)w
zmhOhDwdW#2$yIJ+Pn7pU>vV^|&QPZGRCBn@F&!r=l*Tq?a_*_!C}#jejeQ@D`V84-
zov3wDz`N2H&U@&urxB^dLp;gN%0vk#+DNRRt2A#v>S^FTejECnF2%K`wlI07wpNqe
z;o@xWwALtmV#$zC3pceIXp(FeHsH}OHE>I1KvG;#*Z+FiUi$1ST~mnF<7|<rFrIQ3
zsu61|+8sj{x#fz)rHa@GnI684Bn^FgLl|1lVMKgk4wDyKFWC}*&AAY!AKs&w$SUDW
zJQS-hRuf($S!Jj?UM0Exqf?KfeEJ{X-T6utE8=;MLgWp~6;va<+<0(ftg}f~cj{wh
zkiD<S^}{rD&ngZQgHY)&eR<{BDv4)}*Q6ZlcX92TzSb<PVG@1<s5S2=YXe6;p;@#A
zAT+NNE}|7G)mUrR-lMlpE~bb9M+9;tG8ls>*E_&l`Xouyq2|`1z|%{<;sRZUdL)_o
zd!TjOj)8eU)STGqbEI!@r<ll7#0~8!&Z3cMk*dkR$_R)hIEcTe#p{1|Of4&z5$bh+
zqTNK{R}FL8Z3Bs#lBV7qp$u-YAqX?|7*~0b009mlg^*;X7hT-S6GXeE7bJT^BLO_2
z;fGU)Xm_;|aSpc<wa2lY-iYr9Pn0IBJYpF{KCkwzF728e3MNl$2YnHD#&nLp=%o`L
zf}P^YB$CmC!f=C6(u+=#6^dw$)M%9Q26wTGdgaF1#D5~>>?3%BZtd~7hJEy(*gZSU
zV*pF*3(w1{jX1w^B%Sn8?grcSCV1LFO7%8y7?xU;#OVY&;%jaxu1v%F35Wd<Wan6R
zf%(UFZOrZaQ22+BcLe{xce41~2+iLy@Wxb692A6*-mTn^th$N)MW6@svf*E{+d%n#
z2-Q~dny2bZgl&GVA0lb8b+BoS;uaJXO1DeDYSD3g?NxDv^BiRzz5EU@l5v&xMjpwY
zDaD^{1qA)cEOl(j!DWK2!(lvgt&QdF=^PwrV!H<q!w%1%q}R{1Cpt9APc}Mw%oakA
zh}eTD*w0l+KCc=*ZY(DsZNNXY$9}eyBGMnOOKvHr0KRf)-wD~Kh_zq#Vh_NwobSt6
zwn|I2g;sp^eSI-GOJ-3ykuOZ6_mrjS1S(07Opik?1J#v+^cG8I#hGscU4fs+=-8V*
z&qRXYwkg+H5H<81f6oJE<no^0b5)+>XIK&wYoAn>NBs<9RD(1f*)L~xIw~2oMA7Fb
zu`R6aI1)Q$l2arW(nf<p@04OuSOy~VK)mJ*O~m9Q{lynN)gR$!04kagY`{o$w?rtq
zjbdZ_5xu{d)`g<;<ggtjF&1(hpj4ujegdA;Mpe)@3eVe~XRC=kPhZZlqTBSV7FW3v
z6M8+(FZ)G$&LeyN{%Jr=FC=%LQ=EocTe{4rBw#&G1`YJYT@$xN6tN<#X6+^PN-dIA
z1WG;*fRqEHZM`icG>->FQEC3htLi9hcl$U;qd7W!aa>QUEK_m`Ix<;_)NV^xN<5is
zpj8uvJfMr7(86xQ1!Ar)D8}`x|J_El)93P5?+rWuy*_FgR<{^t)BCXi*Jrtk3Q&vv
zZG($#P?D@CA1r1DH~t@-)9+I|+dpyC+$=Z5LXz>rI*;9m^A)7CBP=JyQ|cv_g94Ij
zW-zAVFy?jJ0$~PC2e(0b76ppbF{Ha^ql5Chg!y{NEtR&+FMlfa#9I~^G5V<?+nBLN
zG?*ErLM!y3xAs)+WxmDej+?u})0&xfF|(J>T8k)O(e#*}^;B$g1%9rpj>CTmMgbF8
z$oRS!Ffg;a^ZZF)v4lf_aIUK-FrmKCMrFYE*!rZ2HzXUiHC#It-N98G5G*-2-4a|Q
zci}Y~{dCYfS=$2*8gqWbNHI=dmL2XQm?%-(I|9+2=n3?73iNt6Mqn3uohkz?l3}*V
zJF^HMOlcgR5Is<dT0lv>!1(FOFH&@B+`vf5vvbl5HOXNB*_~M405m4_eyKfSf4uB8
zy0K|((6!pekduv$)wawfIat6DjvTA5a4q&Y*(y~?jcLBo3OaqWK!a`QDO2wlvL}I<
znKMa11yp4BjC{DIKhbqA7mEq<$p04jDL+5KGdG%)O#IhZRRloGxkW??s|<^ux-pQz
zKrXP$0`G<d7xRqL&yb2dDukF7P&tXimQR>nRUt(?j_3*Lwi+)Bx>hU9%oHkckFl(u
zm_BZ$*y*f@!O}G`BdXsN%N7+2DHG)iARCLGommWRCb$F0L3tI$BL>y}oUg8TI7^as
zK_0%Wmr%VXS9Jb_PjJ%lJ@tnOj1K-ITVM#(l#7{qb^MM_b`V7Qxds*VCPnw!F#8_+
z>?SBvnF5Ugo7A~7dj)E~5PPuPE);pqD2z|kApFxVl(HrfV?05xl<ZI~?o_9>F4qfW
z%hE*fPn|=<7fQ=}4N!ML`5`aq7y%G6X}Vt|jXnQoE3cKKd0f1tT~kj|zICCAx?I2A
zqJ!XNDMzHPjw<oK8+v4qw|;5!BOl%=gxed^>@UTK!ZK6b&3xMd`gHelQOy0Zgm#(k
z(?es>*kX7+B0m@_b@!;F@zJsGIljN<L6|z2xq}KjfcB>{glqJ(Y#6r&@(#H&*iQfx
zdJuj#G{lvg9;A6rV3XUJ_eL+JQ64Lf4ScS~Ms_1A%x_Nre_FfBs4TawOGt-wcO%{1
z-QC?G(j@}Yosv@00@B^x-3<~F5&}x+_i_@ho^w3+yD%7o_Xlguz4pW2d+legHK#(~
z2)*bl_k=}x4M7?9bxw1y2U*xj>2PNJnL5Npb?)(uAKO?#FFJ#aIbGzkKw96b2gb=)
zk8n^qsN4Ujw{epNz8(YY3fND9fVlqKL)lMU)v0MIp@^Z}CV`n^_T*vvyg^_hdtRQ-
z2+Wd{?l(|gYh2e$11*}8`Xat<>8%qjWL@Ib2I;{|DP6_5eq>A3T*0_Lp|r1{U(E_7
zH;(tpAwlZF@pG?^_ftw*94@8LKHlAW;<rk;lM9Kw^ApCnVJxr}5JduYa(zk(<;zHN
z?9V{5W5B?O)GacGQPhDLCV`Pq_7s_MSYNUIijhcgh7|K8vza<efcjlQ){%vRHewe9
zQgMG`8w)1qxRbdLAtA9h6NSXUnyihii_RBD;Y$t{86%?QRE%%mC>Qz}CvEn^7Cbq{
zq_EkX_2|)EQ?N~SM;QDRJ8obt`<=^^bVtKEiZPveZM2HxD=IW5`YkEB7TV-r4OdE+
z#He;Akz*FT+K0_wPGF?r{j@c)h&cml6dn6wyG~+>L8M_)V}I!Flgq&l)6L_s7wDB7
zf>}$w7M`Z6188=Z*hkedPNV}(&0suRyJ`FMD25XY`Sx`&tNTec$}71n&!4Z8kVtHs
z7caWb5fMhg)Mstg_2ABnzuMTuJdJo+Z|5OT7b_lpg$Ad4D7~iHP8~jhCswz`6|YWW
z5V6`CWV9(0iE_4sH!?qr)Nxj}wBTmppPIH6V5Vv9{cK^ZUp3Iz`_nxCF(Nx|dyR2G
z#y%wm)H^2ooV+(ZRv#H$+ffFsLe@{GUwR0<XU99C85#U0F~JhtW<5~O!?O~(dJez5
z5xO|Kw#b6Rl&<Af-J8&9)AyOlQ}kV`T_1{2xGiV61ygWv#`4=)!wNS+n#(SgCXV89
z=BYFlr>TuJghGOA9WVxFr7%A59%1gWed8PM8tyh>4seW!W15|S1TDoEp&#5VA}s5l
z>7g8dlm0+&AepFUi~AC)+6Tm%9tY*voFjF3Cvj^udjDuhJ79uiEP!TQP5oQavlMyN
z{WroiLO?5+Ec>qtKp-j+pYnF#8PpSuhtoF#>9xsFGp2Y5@+BCV46Q=PA7=PO9x`<J
zzn6S6R`h1M)jCg8Dz2j;!%E192f5bEP*__2g_sj4I*&IMZyOERyO1@eMML%*<xvv%
z;uV;4q-Nzq)MnK~&Sm{=Pezj^3qdJ?i>PshFY|R^h}vAjm$KSb3dOUYq$U~(&yBcC
zCKTBE-wU;~!6u}spc$D5kt!w2DFmL6SfHA+aI(dA&!7g*ba|`iymVzcw4aVUq!BW7
ziDj4!4<mz!S?}xP<xFe<i?)V!%V;fv?FgPlTjU*Ak<`Q$0S9|0BoI*?$-}93&Bfh7
z!GtkZmYi!gI99?Uxt%3)JVtbx$B$2w8EGwuUqm8YkQ?(PApHe{bB^JR>TzhC6}gdS
zn-k5nG(~|@hmhm*WVd+M=j$EFJd`1jl*=>8ID4{VCN@iOZ9K8vK@ph=5ntTBrMKG=
z7*wRF@`h|J2hv^lSSs7oKUxpKZr!P+OrW*|>p}o^5JoFV6p+2O22!q;m&!^~(3!A*
zy2lEo!Y*p^VstKGUHMrT6ub8{e{=ww1saVh4|7S#<t>ME%cU5Oy2zBt8^-8S9_2eb
z>d9uGB(A(0%ppUPO6bpDdHjf-sG@w^cu0g%o~f3q71sK0<=x5cX|>&=;+WsWDH}a2
zh1$z0F`Hm39b(heWUC5+eCzJ{67@v6h2IHu3L=4Rg`*M|ZyW)7*1QYv*~qx{$6T|q
zmqeAw???TgWSj%voI?qngAuEDv){HGw`vMSzeRd3Xeq?C|B0lJ(Q<bIZzYpNBK!UJ
zn~A&*t?ZWV_=ZZ&J<g@_ZUZ<iGEYs`vKWdo;=`tj&%2A8yLG;z<0nKObTkzkUumCC
zWFaJyIjZ$;saep0(;B2{a%lUW$Wv!aycRn=eUeeEfS1mP0!<j^uu&TVj%*>2Z2?Ai
z7trBy)V~k3=jNJ9(y`?mwcFXCeDHpm*3zk=^La@qyKc_1bCQQ}-aLM8jRm0QIp#!J
zEZ;i)lA<$|IkZA2{P<H<)UM_Ae{cj0>d&b90i@YT0BP3jzuiUtgEae)BLHB|KKS@^
z^jSq2ITQf|k1BlFhIB}ukRT%3C)}gL>muQO(3y#FWTo~i(_9N&3&y#xK7XF`!(WX&
zGG1%LUx{R#4#NYgfIxi8GWD&X9?&-9@^*cf4M?s`Q4A9X_zPIATwa0kiX4=qFvWWf
z!C5N}bSRD}1HItpJxwkI4<2QrN(sa8!}p-1%M7JOf~mddU<P|Rtn)2v(9d@R{rcng
zpS3GXVOJ%P;%>~BBQr5hlxKuJ&6&8kvngG>;)FYlqQ6tc+;BX%Yp8RGA~j=&jA;lk
z(khcyibKjB=}A@)h7@D*L<%ijKprdES>qmv=@y^v#wRJP^3-QSFThPb<9(Ge-Cc$w
z+1~DH+9*3&jJcy+(VxpOEmcxvkTbxIRlK3BOz<L&JmchSs0Fj1J!ZMS+uw50L?&mE
zpLd0kkCTic0!!fb3>eH+*!(UuR(2;9_6Ad;q(WDYLpL(I3<$10kP1!A^|h0npsVMF
zp`Tg8cmQc^RLkq&A|K0UC=8)iJ;U`leZ8$1{ddGuJ$n4_LSc|aEa5%Z_WRiK#9LVg
zOX=24qgg^ZPLXjMI#^%(HPp7XGFIrAol=@o99bmQSsNhMs6BUgLVWtH>X)98f8HX^
zg&GQm0}d=sNPk-d0^q9O^UBSB<9|QqWB&y|`zNlJrT~d7vI@%GlJjRk=YZ8QkgpV$
zoxJ8XC<qpaFOtDSEJ9@PQ50&2mPugxtY`O5Gl-t8vE2cgrr2F~vO(ZJ!DoVBM>oBu
zyJo|2;&xFIl$x-79+K3v<S?@QYWO16;Y*#%*%n{(9b&72I}$&rEW}t)8;~ZTICvMB
zB51&rA_xKL4G&JMMk8T9;v9(yQ$3;)h8hE*SgH~OrZC2qNnjFqBV{tovt>oQ=#1f^
zq%QII(8HxSv88@Sx!svfW!VXl^UKPz^ya>3WmgW3>L?-su8Yo$oOFlEs@odYvw%!C
zX+p3iX@ZBi$V^i3xU0RMD9-OJ3Yc7az)_CTeO_Cp?Lz_!{g4Bm%ZvABwxuR10-tN(
z1Y0<)HHlU?-up52d%U&^U!Y9G-in~S`X()M8BM4l0t?7<(vs$61CZ~YgvZ(idBaI&
zh8ie|)<QAtMg(OhIA989m%4B=>6_HWAw`5x_WL2xedhGpk_m<Ug48=Qjg%?I*pDWH
zMir(!J~dIElACD@JC>7Nt^0M{&#1mJR&p?^t#0srUD~>;DBYI2Othd8FUtZZ>!&SO
zG9jt=ID@)#sfDJ_)F?D<T(IGxOb*3|6kILZ5PJh;8|8?aSV=Ls1MIfQ0D7iS;|gf{
zyx2zAGfW{{eO6NFVhb%1RLTJU@j>;)6CA;d0k`Ze&US67h=Yu&g#(U=(nWy*R9RgN
zvOc%U#1ft%R851No8uN30WgHSH^_nSX+vYBE^ThqJjAQPVyd^Yg$H9zHiAKg)B;lk
zwKl!t`sYN-O~q?97Dn~?&1b!ruuchz2z>`&k$070ShNLXh4A9RvBBWf?D$0#d;>wT
z0I4i-vc_P(t5`>aYb4<{i8|cOA+K+<WQl~=3`=9k3^VgOL27)#OFa1_;12PV7@@8}
zShE5%N})|k8iIh~4L~wG0QbfIhMA%N3S0<uY7}aMaqq4F4n}ct<rkP(9Je-^-JG`!
zfm<TV*$H%OuMe!I8v-=HeUe~dUq2KS!0^!3?1BrRsf5OfUPYog8gJK_Xgjtei0_RN
z!-=M=b#VN`J|eGm{8d;4Q<*~y_I+R9&{#;9N=4(fhFG@w$8yGJyr$c_ZJ!q@p__0%
z7z<<*c~qNWF20SyMXIUA#*;dRizLki9Sdy|5)OMKk!{<VMj5Z%v!^|rAYwEGhP}-`
zG8SapXE?&*;I>HJZ-}Egm;r$X$Yw1F5#d#-KVrk-@75kaxyY^azs`(eqfoq2QAdLn
zo2gV(S=5%|&?JPxm{&usM-{qJ+bMg8r-l7_EYZKio{G>C^yW;T4&xNjDHtz|lzejj
z6i$7&tqHE*W71tD1&dihOKiP0KpfsSn;B_jjidE_w(zFX2(cg&$q5L2`t*BU63Gm@
zILEap)@jC)ebEJ-NnS5T$3t4yMW23aVsYc88hK>L54X>s@b(}m2wd7QlStJagSiEb
zTDElLVsZ25Y0c$u`(f3GU+VPpSmpLDNnQ!*Y1XD<@Ujt`3_A$%5pYu;EcV2683(Kh
zEkeXeOJ$~o8?s?{FtB=KaEWbgO3dyFV!FZ7n2SLGa#<7dMXv~1dqJKT!GX4kDW%vN
zSRVzVBF8<3T%vGJkQ91?A_xw#tMgqsa@Woe*ac~BGOILGXj?#bFj1GbkM^_Pt2eGe
ztdpFW`%-ftvE`GeE>=!PKVf91Onr&3h3^a7zg>#r&Fs#9B{eZXpVUw9JlNrC<vszj
z&N3MrMot__$576OLrQ!V^hVTqTxC;O=lKGURf7`4dr7RF&3%diVveD>4J*Ak980PA
z6XG2`2#T<;%d#o$O!t|r%lNZdu#3&SV~frObA?Y*8KRX#kza^LjE@ipIWNBSdVhHG
zpLCQ-4`arU5I{g9s6arn|D!X={r%=oB)7lSP0%7?k$@A^FB?MJOXCpwnIHqouxc2p
z+X712yAln_UqDzOtczp4WCmF;{`{tCXuLAKZw;_kno>IR&t_CPEII%RC%kW&czBv@
zOc4yDUxf8@B%J{YCu$C858&@E4-(jbxqY;FnT-ZR37C)A$yY7#7cr0rpr9o1NwPa5
zXbEY$zv*K1iqm`KA@%t5y1itj#Y0{@@$r+Ob6(B#!@n$*NV!1t-0P4#R64q>faYHB
z7&(GD`Z^$a4Lt%~u;uQGM+a#^cX^1Uix#OTbsqk-WJLwvd5s0C1ct07?l8dbBcf&G
z2g<3@Oy8b)E{O6zJRVf?UL8tZ?j(6eOctR-4(xrq<(Nuk&gZ8th_c7qqTHF;aodSl
zm`aBAX%S%->y{*0SPWHw^dUtlD>uGB`{ihOuY{14B^my94Qo9bD?Ac+KvEx>bo;Q1
ztEHQ;I6fn@5~f7n+PR-$_9TY)Faj$p6bqcvgwk6hO;Max+z~>WoVOpzYJG;@Lw(6|
zlO|O@GpDSgOU6+MS6U-j_?RUY*J;r;+SyRC&Dz6U-O!Ney@ecYWyOqClMzboUs_~A
znq63-wN1Qw#=TqRScj|4broSp8m)p+LDsguGRNYk&v7`kn{5U`0=2(e?JGoGgdwTO
zqT*}E*Qnb7sREjd<Wa9C!G(9?q#aaXK_yCZ?OjS*oaC|dM65ES8@0@ngp0EP3u&2T
zvJk+#gGB?|)g9Dsi*~}H)^KH^A*1+4vt_Kvk&vdY=2bMQ{wZ1tktMxkQa+ZP9JQg=
z_6Y?U7qzP4#>i=kQg@uVphQ=yZP^|nT2+?*KrviueYEnRdvB=^wpzv(w9H{ey97-a
z=R4(Ea7w6l@q9ZyN=<@XEIU<RO+$kZ9V9vEVIOp4Fm3RjG><_hv)2VwDoei`RXl2>
zZ@=LBoMajwUCDt}M#&|@!JZp|R-$mGkN<&;;XTPWLT-EfcFe7?zFg$fMVJ#rp5ze9
z?rfPIMJ0Dc6`T;Vln=dZ>1N@1^ZCIc$~DwQE->0AdmWybug1t6)W$?->w}!H+G%DC
zFzN}{2@*FV8gvtUHb^lM_Xr*1H{0XKLY9znjT`-Ipg`{e8Ohe|j(c6jfR1Qs2hM2&
zpu$}wlraxhJ!rp#_JB+K8XbzON_7g`-SKw^o{Ix|(7A_EG2Z&|Ilw`RpNEFY9p;CP
z*ow4oG2Vt)V9FVax2s^z9Quc0YQLeUA`axhoMGtM7wFqtrdkaMqhp{;IbmzRy;`GQ
zKd9|{p^-mHgj0JrNz7e6T<qmg%no6V8k=bNEE>5n>WnWVyDLhoO23BUwb1i{<YU2Q
zl%aH45$=o2S>Dy`d?5F^3@%4V7gTQLlp+dCOU4)Fj%s8BH`q?!3g6$c6UtN26_Cd)
z(uyFtaSwBJiij|y-)?NYqrypewX5Y_(x<;_VnV|xb5c%fJ(6h5FO`dIiLa(f8z8m4
zmyPPt{6<zwq>`o9{uQZz%%=4DisWK>ZoKU<+}GH+ydia)`E6qXAQp5<P8QQ`<Y~5^
zf*qB_dLDA-MNIYxlB2$MH6z;-80XSS#LZa>hEZo)zN(N<iXN>jp^)Wn!%fedjrPwm
zOKs+=xt!-jzGyt1a0#NAYhqPQF6Xh(#1Fm}xEl93R_x+EpPs04bPA4|#Zh8P%IN-j
z#%pA3$`oI2R_fbwnhad4KxS=}78lI5j8*3Wbgd-<(RRQawf*e;!pZYh?3qMiz$QQ8
z40V68CiBNq*?8dCfaK*B2QRC4ooseqVo10U?+~<gMLD#&zqEkQpjfuEEhQ*$JUu(a
z>9_IMY<D<>_1`3fd$lFc0AoY3XL$wPtf8#y%RS^rE45;33W_C$E2Gb))7rrt+~hch
zgjWj2-rlvxfK!b!AL)!^AQGYJo(JDUjRT2$y(^rMwZ6_>&XboWhPqpoK|c9L$Z5cF
zc@a(!geMZ(4Orzklhmm~W=*`vt25o1X+aQi@hv;iFg-*@PRj<B@=3r_#bIDmMV^LV
zWqdw=NqUN|f26JuOOvjCDxXa%e$%$m@<yI8pA@wg(Gf|SEy->2Od>IODA0lBK*lLF
z-agDwx)t;*!9t}umdPO_Wc)dtegi3ZEr(gvD<D425+kms9FregXZmg9M5Z$E!e_VH
zDLR)Sn)=|A1bLbwMcrV4-I8%n;M<RBIX^RAzT_6cS2y)nuLBZWaG}@7WKU$X9c7Ro
zoVV+@(*x6XT0#OpqhUB>4#07E%FcTX`H_XXewdhjtOtzt+b7bOI1>Iib*~)Rc{>nb
zfIwGYXA0C7l{mQucpEo9E6GwYIH<B$Jujh!8JA<H;`LA9?EpRISf@{|nHeDW8_<r|
zLa&7Uv=0Y3W9Y!nbd!egyk$GdM3#bcrXFg29h_yFbn^A9lngvDgH9S3le;Mz<3_1>
zu@UwXMNn}-BO~0!_WY)B0vzuR1U3nGm2lR<>XmFhJ*FhE3Xf4sRs#Oh(;=fRTMCrn
zjQqIQ4zA{7q}9AN>iDvb_2F2T5L?Il+|q;OC2~BTKwaZzFro-1@VvvZ7$e$+aY=Nl
zSJXh>N%J}!+}vP&9$?lp=(aRuiMUV}1L^j$eSyJjqDA}cn$&X}+Rt2$C~j-8gcC{k
z{BB%=q_%6|Z944m<Xju?Kp($G#pso6yRQ!}0$g=y{11;A_tnAo$Bm!6+)$FSnczq8
z@SUiYfTPKAj|I6QxUUW7*9Ic=j~68B@XqBquC`<CBYMR)yDQFPw{l>+>euOw-x?_x
zhAb6J5Jk~7?$&d#-}thk;r4R>8#>TgnT80|j5c+Hx?P!Q;+vyZJfv-}m28bcg&29y
z!pd?vjQQC`^4Mx=F(gyaoQ)6E)&tq3^N7UXyrD!Y>5$@HqwDvbs_d+RPmJWm2ScA#
z?{YP0;9x8fyTT2vsFI8;=lWR+Ouic22P6A@VMx&4e0-H@G}wCZA>;U?sOR+O%ciol
zyNWMO^CeD}e8%{Qxh9q>)P*=slf563sG&9@`r+uLdzU<j0j0smoD(Ue*lu}>T}4w>
ztg7lJ%r!N`S8Z#-d*M;jt6p(FEoXOD6q(ZMF_UDIv(6?~$x>YIZl5xIo~@qZTdJd8
z$T+vq6uR~Wq9Hopb;Q@#N|@hBmF!QF5mk!0c_ukEFcMPV#ygY=xY{Z1BP5Z#sQU%>
zpc8Ymt178`ceF6shV-EH_u`{ZfVC%p`C&%5r%~!T>>_Wk2c4el3=6mgPT;&+2V?hj
zxKQZ;8Ak7O2{ASszM_x5ycI7{%I2)4*K|Tma7{C?4(^Eq)$Y;&pPoIWwoEj71uyo^
z$_4R5ZKJM%De8<=+`;{$_iLFx^?%4jKmi^-84+ax8VOm^U#r_=e?|iW*=OkMI6!@+
z2=D}>JTxIAAS)p%qNGeKBl?G^!%Y2m&B*<u+AKjB5CRUo8#@~LeRlJQWc(Fj1O<*X
z$BnBaOk?Pit1eH!SW1N&zDjv(b9<5KAFgbqHHlT1g94}QM$+ZC6mZ(hw?E?*SnPk`
zjlv|VHTvQ6SSyKEjez<rk0@m>uD|^k8&H;n`48wV*zNeo+eL1L+3Kc_$}tr=BWhI7
zfZ@2$=IEu$=$rJ2TVQ+Y$6r)8Vt*7IBWEq$acVA%&!>GCkC>*2euEifxctcruKR1x
z1=Z&84YnnMyn_wFQIEgk+X*zCv5ws@R@WueT~L4ici+qFR93D9ya2ok)1?L!JUxt$
zw1kj|tdhtdQaeA0&61{;!$LQz=UK_DeXf!Pcy21Q1@=~O*{6UaAq(L;YqFR?(o%vN
z@X}sdf{>dNo`fdvY-5*8Rhe8CiM_dst*@yEDP1r}buA)v1F1*{PWhbgb-BNN;)`{f
zcU;@O*cH5%an7oQI6B##`N&;o&IWsvtX6V*b)!Ab>R?n;g0<Y=<mL@?P;%g@8QQfY
z+1R9^Lj%YeK2*87IFR!emcZD#c)W6PI9a9f)EQM$8#uXH`WpE391iN-lD1g6h>OR|
z-ukxvIny`tB&eDg8DdRRE{0YK2oo)GW3}LMN7NFesxm{z0tD5QX=S5eJ#<|ZUM&kY
z^(M@tSA?l~wo)p=Ljr{=eS*y(7t>)2spqnZSiI)0!?8vm>Q~k2jO0+LWL}rLHDLSA
zx%5NJL!s@g5`1Fc3!A#jy=hEwpbALEd16&xg@op;<<`4EsKo!mgt%DZI2g$&5Q1bs
zE{aqFO4AE-MT4%p6;xNC2*(Bv9JkN{ny?5OA4{>mbz-H)NUg1Wyt3$XBgtL*?5;tN
zy=zCEs~Pp@vB+vR1cttXvDG#)COQ*QF}pQEy|0QDYSlnz8iz>FGPrJC2*HUZp=5iu
z8c}Y8sr5yuxaw54xff@`H-@QHgKlKUj@qs&OGxJGOir<~pt7~iNBtJpo$JLUs*2l^
zn#xH-n$J&94(KenKsOCKAOzM|z)LwVffkM9V2}hpi8|@~Df&{8k2CSa47_Gtm)368
zyiQ2f+0+fZ8zp;MkEIZg`qJGMcn7g^3LX`>NoyYmjJpey9Jp?SWOBMx3yKo*^Aw(#
zrj1Hbq5sD5a~ZTdKds8#aa(2y4OE=MhH}h4lqkPUCSP;M4|XnxSxhITJ%{#{gzPyW
zyMc?u0=lq#@_yGPC(f0}T(vn0nC^>Kw_q8z1@(LI*SNJtRup`e&o$4>7W?2U;h{hj
z#yH#+1H3YAu<pvx6iZ)<YcueqC)hbkizn+E31TV^<2%4a1DSc1kV6cQ!N^{=Ztu&(
zZ&`nMM$=`x{mtk#^XUW>#onsac%m&NF-?xwmv#n2kkZfBWttkmQQYE7y)wvO`8sky
z#!x^@V|inZJN><^^}$`)+p)Ev@urPcIbhYbdIbfv=+*$&DtIOf9mAyv>A3AF=xZ^d
z&1$9Jh)I-;P80f|)%tfJO4odm%n{hQB7hdx8{w`$H<45e$h#Bca+YK%7ur4z^9;i4
zezq90=wDa*@w0t`H!%BiFD#(97E``tYKADj7Vc?~U}&tY?jVTELoZoh8mMo?`ic|o
zO3e;3$D))<+c-fqq+DL3M1smr#T?UhmR&TS&?hw=h&PsCsa7fWTq`$KqEMQnbBcZ_
zw1kg>SR!X!1kVIp&yFdohn1ZH60VLXMGYqU5IvShE}7vJn;$dtktp^G>-F?{4+nx0
z>jE9tD7kiCj>1zE)+4Ac+l{y4QO2{ZmC&BAB`=7iLVy)Mwoc1<y<oEj;^KCFvMewr
zk`&{{i)v2XeuS)v*eJd6WzI4h8CBS19&g<vt2L1&an0iESCc8-!L_%hx@a?3kOQUS
zEKy$5Kp5!T{^3>>5vOP~PQtw1!&-X!XrDyY`sB^ux8)cUUNx-EDWa57eFVLg;shqz
z0Y_)!rxegsEhj>?hhWQ_-avq7aZy$mlq`t8g>vRo<u#>>e!d|}$B6M(Sitl}CARIN
zy`%DmMNj<U_(X4{pKCsIeviK2XAL3lb_MN{fw3_7H#Kpm$b#==_9*0OH;$``_X6dG
z@uLk!M2UQ+tiVp=iAJU__L%)6&z-oPds{~o(-ysgBKOB~4I%oiP&$$2MMcEnCAY4!
zNgQ`-5ZB(fy^sPu-Lwz)W0T?o&n($mD-r)-yHIm<?yV7g(%UuFvomK(CRKqGhpH?Q
zjVVsx=QZ597K9GO;im-Bu{#A6548$0I-~q5Tsw<tN$P8(FY<a%LaW;z8aBBDfHCH(
zp#}Di8KD6}i74uwvct!_)qsozAJix#p6(;4aZ>h!9=Jj<D#7&Aa6kUYk6y#tmbf14
z!6M5w8_ax$4AC1gxFfL+?P$)U_)HK>l24ZIKo7KA8K<5m>W|vPdJ}p3L||u1jeXN#
zIsn#zFyOjK5843p&x>Sr3g><r;$;csdGQ)pAq;nx0TF^*;j}PHA8>gf)5;6o(BEuR
zy2Omjo>~|-yP-mbo!_>6ZRN~fN(9wcgl$*hyXPdUDKn>`#s6+}i~`qEFYO%t3(~y)
z1f?D#@U2(0u&UfZ1iBxnUEuBqn!FJbn}U>&hLOdPRo1leESG$zg78t$Mx2}`qgtv0
z)}*)61Y6i=p~|I1HPzj93C8r?q86f`<gPKf=|qUntS3S7suZ+nLG;Vg-PfLo<(=cS
zD#++}rP@@*q%d-9Aw-~Fk0B|6F-UjiMq^MYnTvYl`iT;6B+|X~R<UzxMG-mhSC6Je
zsD^HaZ9^}D884M9#DH#o8_E?Y7qD6BMD-MEE<yp)RvyBdsV&P)9k1?*1+f(?NUs78
zVV3tDTN#nuf+_8)9Dzo(CLyCZbdf6Msr?JA?cmyGh&?ytojw1?Pcab)7Fv_&Q|q3f
ziv_DekeVDP;Y)<KsW{4t9_(L8n%^^Goh}-``np1<V6m4~`g$uPq6@+2RKNv;e37T#
z>XlzD#yADC))@_^P+7|7VUy|#6;Kj&L|2FDb|n`eEL?or1$OI1kjz$Csj0HVF&zSG
zRp^&W(a})QV}v=$eAtl$!`lw9`R56V*jZ*#0{tohkTlAe++;~HOvCW`wnq}OQYVw<
zQ=~qs$1OJ3qf-@ALpNXdPhj61*b%%kn!n(QvNyZ>G#4k(`c|5wjQWBK8C6pbiZ>SZ
zB$0A^Qlo;1LL%jxsUdeBBIBXJHe^@AEDN&3a=Y2SWE(gm7AbDE(nahtox~h;CnT!}
z@?@+)kMXW?hJ<pRnbSEh#5akkjh8&Q{7^dNDQ$1+WKa*4o6EZL<4jiE=3)oVz|9Jk
z@lBu+A-93%Gzdmj>5ifTx0B(w97gGh%jA)gMz3&P41)CDn+{*geDc=%%>Kz2kBFtD
zS3Y;=e37V=Q`+=fnN@E=`)jQ|m#;yC*-xgL{F_OTp{tug#Xgy3dP4VLtK`&MT7kYu
z6_y4<`*P>Q70D0|ixYsMA5hUJJn?M%Z6d_%5&P{dz1)&k3X8(dv!Lqn(ln^nu@x!@
zFzBPLUX||X6wzr;gE}(|cdY%?Iz~f=Ii2YI5p$*N%KUr{Ztk&=PZM>Ck|a-ShI3y^
zc*iUhNT5a(Vw?2|=P*Q&8NHp&$8iEd4s@3qifDma(Pb&)XG6#5%*%{O$;C6sQZ%}f
zLaKmy!~T&Sr=r&?F)q9;4M&GGH<AnQseR(ki?5%*y`D&VnIvLq3fw`ISg*Qz8a-1o
z=oPxQ4>y$%zM2a=;ViX}8;__s6fO^+Qh=iO!~n~f_6VDyVFj2{ouOJ-Yx^_B;Zt6r
zJYuEDf<b~bFE*^sf}%?I634|?=M7$bz$oA#YY1pecsiJZ!`_rG&cuEJXXuvFdhIL$
zR(95Cm~gU9u^z2&o9`ewFEBO(g+{@{g|f3*rwTrb?%PC`rgI%a#XH(lqIgcMoiJO1
zs)8>ov;#L$8BTb|%*R|<_2#ay>A?-h0##4G+gg+4vP?Z3uSUc7e(j?J&j6KKkPOOJ
zI{DHlG)~{EQG7@$$p^L1fcH#C@dU)FE2GUWG>I>KeIVF@&aN0NichPs%bo|Bk9vB*
ziT+E^#!Vy}{+F(vuB9bIr<O+D(By_|&D+gX^+Hdy*LuapUSG_cfjv7v=Sh(Rdbz8x
zlF48SJ1An>?f}Wqz74@5dIepkKX|oqbF~Xn*z8uP-PE8}eCu{IakA5lz^C)E=?1Y<
z`?}P<rSa~q;P%p2Pp78M83;7p?)^1*oYU2tHQ7ba+J+_#J%_4|g?4ZoH2yn0)+xtb
zem15>RM`$NZZjDhmT?D8KpD7yr2I8$=g?*j_X|gOc0@e3)+%Sez2R8S>OR<=+qAgc
zvI~)23#Ngxvjh%zc(N{$gQ~^t0NRlR^_O6y@2m-U)WO}SMoW<${baeYtNfCwA=3%*
zF&!-z%=Pth>Z#XWxG-RDboY&zt>5?rgA5;IHxUYo^}n#gLp-2qj|Xk`UfFiOcK4)r
zlkJCqH}M(3lEJ<T2C7r7w&zM+UEevB<ags}J%K)$u2?V=ry!_ykMnvGU{>!+xaAM-
z*ZmfIJAB=8z-3dt%c!VZYkgfbp^!r~4D$ke>7|)zZIT=;ZCVrZQ_a;f><-`tmfZPW
zvUj+iKABdRx@)lr9Bn}%Gm30k8<r%=ZhJ^|8APs|G6;=ett&(OZP-*0-oAE0Pi|Zh
zwoEz=>I?Chb>2+hp1}12+4vpW`=XCv9xiE9OBO<z`+GIkej`HNp{C^h5>+oZadKXM
zIJaBfDmakJzzvoovGu7u)wH;E!BWH(L|zQ5zgXElD6J<Q?*<;lrC|SX>J2l3Q3b1j
ziver+J3&O7m;R-`g9#;?E8K~6%`M}=0!F?^S#w6@okhmX7;{X>DZ^qmm5wsb*hm~T
zEod2p#dlcV%xx5(rKHK1GD(ExVlt^=uca|wk?An;8nrt#$Gq4vPnMh-Ek1<S@1A-)
zR#!<ws6a3?7DAaS_I%>S+jlsiWI}2YL(iTngih2QZLai*a*hJiL|I|=UGs)^hp~$4
zuzQuLsi~`tG=Yn?&{^B8MsM(aOoy@11PXFHETQ{U@@3T~yPtQNwC&>!IxWTZbCd}S
z!SpHc@@_$FUcY?NXwD1G54rXr&`(IXd~d;RC{cws_Y(BX5YNpSS3fKfK0(eXIJhjA
zb)%&%mOcwSZ|0BB5;#Ag1#{Y+%;ii+M}Xo#%ED{r>g{)SW(%!bgZJ8sJR*usa#IDe
z1p&M|q02kB;3J2lkR7pK<c}pZ+NcgSH^mRJtJPe}JMM>GiUOCxp-u6&XRZ~6O*1G;
zTz0=q(36*e<i<T%Z`x>YdX6)aO&D0Y0dG^tfiZ5fMh9_zrE)Fk=9?x~Q`j=T&MRI}
zXqFTp7u|yJjP}s`e0XEyjDqNx81jjB^YuCRsO!=NYiv4$sB$(tk<*Q-&;%m~=A1u&
zyb$EsNR#~~UhS2o#|@HlE~#Go=({z%+f*i$nD9y2kGAbnzSrLGG0yMATR@M<DvIKg
z@(_Gd%;7p=af%iq5N!FO(y1MLY)&sMB-VDn9BY3=zkAR%?LKW^rbGpI1zc-n1Ad-9
zOxtDUlmUtR-(~F{gJsa|^ax(7H)(64I!ODu$tayo9^T9x$-Zx3`f}Q8)Sjrle|>Sj
z+Qzj?Apt{2FX@S@o3&=ksb?GESF?9b9>ZPLAn2M9@1qp|gkG#TG1PApmE)oatU&!W
z^QdgHG4wXdZOb8o%D1Wc<pw)F)9k`ibD$MCc`0BJJkY-q_65k{{_=$Yv@dY~%daxO
zV86=z{)a%^-&ukIdT5FFH%os={{FMU?*)G!8uWc^@DKT4fZcof-yZ_~mCEzI`0wv_
z0kh~m@<Y21q<{eq?xlc#=-|hFEzuuE;=YaSeN2Cg#BacVQ=0`Czi&nPH^#r$n*AO9
z{e<BGiroIf_@BzN04Mj#vw!2{d)?WGPF()k$!{vO08jT4t$*X`dkxx$o~C}%)1UHJ
z0FU>Iv_JIt>-2xGNBg_S`|0Nb+^zpZ0Dl?V?;ihPcm1b2?LQCwdpXMAf!xnz7oZd4
zKLWY$UJ7t@FHZTx5&!x;elJb=&=C=!kH$|p`hCE^IQ)TD<gZJ?uOa?krt)`(_aP4a
zmkxg@S_v5Sy=dhRNBt*b@39);LlC5}|7z5KO!H%HLI8?;eX}2;_%-(5%f<bU;(n5$
z0r$QC82ewu;{cxS#pC|Q)A!PG4?UUxq^C#Id@m&TLyvz(__2)K?;h``dHCOY{8>@C
zzYF@Gz#hwRJp@LM{BuG7BNUH;{Z7jHIT(*6njXS<10W0i%-sJ4jK30W`Z-vSwQe55
zS^<3cf5Q3;<(r>F_E;+9Au=wU|2ML~5Dxh{WRLX~9wJM}|9>O<vD(7V0ejq>{vohy
zqW?Fr`xnFCDXsh+t{<O)upXU(RLK4k*DsyE0rvml81#Lg@88FLzpm#0t^HrSe*d$N
z?|Xhf^Z`u%cVl?S0RLsnyzc=14{Pb;{@=g*x?f8Te%jX`8`Zz<2L5+b`sb?pxJUFu
zh%1yo8;+lW`0p0k$9*CnqOxcF-=KO>0`h0k`Ok&yaj&w6;6mB|=ivTb+#a{ZcnGaQ
z@V|xjM<v70MdxvAfQJw`MgLm}j~}qdjRJlS)#K9Chp6PG|4UT=Ze|`=H9Z6nulQdA
z_-|^QelA*%>tY^)1y=drfc;Iy;9ovlk7?NtabaowZ*V;%Y5!cT9&@-JV)8cn-(mUz
zqwDYB{kmm;&-3{m-u;$(Z36U=^#d?>&-(eB9qm5Pe$4s#)f|BS1K82{Odptg!~g<V
z{>wXqUpJ!f*)k6;>zO^U{0DdDe)xZn_xsDdhgQ*C9$5YTYVWU%f`J3Z2t)z+)&t}k
KGk`w@0{TCAtviPR

literal 0
HcmV?d00001

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala
new file mode 100644
index 0000000000000..ee8e6f4f78be5
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression}
+import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
+import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.sql.types.{IntegerType, StringType}
+import org.apache.spark.util.Utils
+
+class HiveUDFDynamicLoadSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+
+  case class UDFTestInformation(
+      identifier: String,
+      funcName: String,
+      className: String,
+      fnVerifyQuery: () => Unit,
+      fnCreateHiveUDFExpression: () => Expression)
+
+  private val udfTestInfos: Seq[UDFTestInformation] = Array(
+    // UDF
+    // UDFExampleAdd2 is slightly modified version of UDFExampleAdd in hive/contrib,
+    // which adds two integers or doubles.
+    UDFTestInformation(
+      "UDF",
+      "udf_add2",
+      "org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd2",
+      () => {
+        checkAnswer(sql("SELECT udf_add2(1, 2)"), Row(3) :: Nil)
+      },
+      () => {
+        HiveSimpleUDF(
+          "default.udf_add2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd2"),
+          Array(
+            AttributeReference("a", IntegerType, nullable = false)(),
+            AttributeReference("b", IntegerType, nullable = false)()))
+      }),
+
+    // GenericUDF
+    // GenericUDFTrim2 is cloned version of GenericUDFTrim in hive/contrib.
+    UDFTestInformation(
+      "GENERIC_UDF",
+      "generic_udf_trim2",
+      "org.apache.hadoop.hive.contrib.udf.example.GenericUDFTrim2",
+      () => {
+        checkAnswer(sql("SELECT generic_udf_trim2(' hello ')"), Row("hello") :: Nil)
+      },
+      () => {
+        HiveGenericUDF(
+          "default.generic_udf_trim2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udf.example.GenericUDFTrim2"),
+          Array(AttributeReference("a", StringType, nullable = false)())
+        )
+      }
+    ),
+
+    // AbstractGenericUDAFResolver
+    // GenericUDAFSum2 is cloned version of GenericUDAFSum in hive/exec.
+    UDFTestInformation(
+      "GENERIC_UDAF",
+      "generic_udaf_sum2",
+      "org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum2",
+      () => {
+        import spark.implicits._
+        val df = Seq((0: Integer) -> 0, (1: Integer) -> 1, (2: Integer) -> 2, (3: Integer) -> 3)
+          .toDF("key", "value").createOrReplaceTempView("t")
+        checkAnswer(sql("SELECT generic_udaf_sum2(value) FROM t GROUP BY key % 2"),
+          Row(2) :: Row(4) :: Nil)
+      },
+      () => {
+        HiveUDAFFunction(
+          "default.generic_udaf_sum2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum2"),
+          Array(AttributeReference("a", IntegerType, nullable = false)())
+        )
+      }
+    ),
+
+    // UDAF
+    // UDAFExampleMax2 is cloned version of UDAFExampleMax in hive/contrib.
+    UDFTestInformation(
+      "UDAF",
+      "udaf_max2",
+      "org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax2",
+      () => {
+        import spark.implicits._
+        val df = Seq((0: Integer) -> 0, (1: Integer) -> 1, (2: Integer) -> 2, (3: Integer) -> 3)
+          .toDF("key", "value").createOrReplaceTempView("t")
+        checkAnswer(sql("SELECT udaf_max2(value) FROM t GROUP BY key % 2"),
+          Row(2) :: Row(3) :: Nil)
+      },
+      () => {
+        HiveUDAFFunction(
+          "default.udaf_max2",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax2"),
+          Array(AttributeReference("a", IntegerType, nullable = false)()),
+          isUDAFBridgeRequired = true
+        )
+      }
+    ),
+
+    // GenericUDTF
+    // GenericUDTFCount3 is slightly modified version of GenericUDTFCount2 in hive/contrib,
+    // which emits the count for three times.
+    UDFTestInformation(
+      "GENERIC_UDTF",
+      "udtf_count3",
+      "org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3",
+      () => {
+        checkAnswer(
+          sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"),
+          Row(3) :: Row(3) :: Row(3) :: Nil)
+      },
+      () => {
+        HiveGenericUDTF(
+          "default.udtf_count3",
+          HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3"),
+          Array.empty[Expression]
+        )
+      }
+    )
+  )
+
+  udfTestInfos.foreach { udfInfo =>
+    // The test jars are built from below commit:
+    // https://github.com/HeartSaVioR/hive/commit/12f3f036b6efd0299cd1d457c0c0a65e0fd7e5f2
+    // which contain new UDF classes to be dynamically loaded and tested via Spark.
+
+    // This jar file should not be placed to the classpath.
+    val jarPath = "src/test/noclasspath/hive-test-udfs.jar"
+    val jarUrl = s"file://${System.getProperty("user.dir")}/$jarPath"
+
+    test("Spark should be able to run Hive UDF using jar regardless of " +
+      s"current thread context classloader (${udfInfo.identifier}") {
+      Utils.withContextClassLoader(Utils.getSparkClassLoader) {
+        withUserDefinedFunction(udfInfo.funcName -> false) {
+          val sparkClassLoader = Thread.currentThread().getContextClassLoader
+
+          sql(s"CREATE FUNCTION ${udfInfo.funcName} AS '${udfInfo.className}' USING JAR '$jarUrl'")
+
+          assert(Thread.currentThread().getContextClassLoader eq sparkClassLoader)
+
+          // JAR will be loaded at first usage, and it will change the current thread's
+          // context classloader to jar classloader in sharedState.
+          // See SessionState.addJar for details.
+          udfInfo.fnVerifyQuery()
+
+          assert(Thread.currentThread().getContextClassLoader ne sparkClassLoader)
+          assert(Thread.currentThread().getContextClassLoader eq
+            spark.sqlContext.sharedState.jarClassLoader)
+
+          val udfExpr = udfInfo.fnCreateHiveUDFExpression()
+          // force initializing - this is what we do in HiveSessionCatalog
+          udfExpr.dataType
+
+          // Roll back to the original classloader and run query again. Without this line, the test
+          // would pass, as thread's context classloader is changed to jar classloader. But thread
+          // context classloader can be changed from others as well which would fail the query; one
+          // example is spark-shell, which thread context classloader rolls back automatically. This
+          // mimics the behavior of spark-shell.
+          Thread.currentThread().setContextClassLoader(sparkClassLoader)
+
+          udfInfo.fnVerifyQuery()
+
+          val newExpr = udfExpr.makeCopy(udfExpr.productIterator.map(_.asInstanceOf[AnyRef])
+            .toArray)
+          newExpr.dataType
+        }
+      }
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 539b464743461..65c1db553e0d7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2492,51 +2492,4 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
       }
     }
   }
-
-  test("SPARK-26560 Spark should be able to run Hive UDF using jar regardless of " +
-    "current thread context classloader") {
-    // force to use Spark classloader as other test (even in other test suites) may change the
-    // current thread's context classloader to jar classloader
-    Utils.withContextClassLoader(Utils.getSparkClassLoader) {
-      withUserDefinedFunction("udtf_count3" -> false) {
-        val sparkClassLoader = Thread.currentThread().getContextClassLoader
-
-        // This jar file should not be placed to the classpath; GenericUDTFCount3 is slightly
-        // modified version of GenericUDTFCount2 in hive/contrib, which emits the count for
-        // three times.
-        val jarPath = "src/test/noclasspath/TestUDTF-spark-26560.jar"
-        val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath"
-
-        sql(
-          s"""
-             |CREATE FUNCTION udtf_count3
-             |AS 'org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3'
-             |USING JAR '$jarURL'
-          """.stripMargin)
-
-        assert(Thread.currentThread().getContextClassLoader eq sparkClassLoader)
-
-        // JAR will be loaded at first usage, and it will change the current thread's
-        // context classloader to jar classloader in sharedState.
-        // See SessionState.addJar for details.
-        checkAnswer(
-          sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"),
-          Row(3) :: Row(3) :: Row(3) :: Nil)
-
-        assert(Thread.currentThread().getContextClassLoader ne sparkClassLoader)
-        assert(Thread.currentThread().getContextClassLoader eq
-          spark.sqlContext.sharedState.jarClassLoader)
-
-        // Roll back to the original classloader and run query again. Without this line, the test
-        // would pass, as thread's context classloader is changed to jar classloader. But thread
-        // context classloader can be changed from others as well which would fail the query; one
-        // example is spark-shell, which thread context classloader rolls back automatically. This
-        // mimics the behavior of spark-shell.
-        Thread.currentThread().setContextClassLoader(sparkClassLoader)
-        checkAnswer(
-          sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"),
-          Row(3) :: Row(3) :: Row(3) :: Nil)
-      }
-    }
-  }
 }

From 5a96ee7619ea07edefd030c66641e6e473a890e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C4=90=E1=BA=B7ng=20Minh=20D=C5=A9ng?= <dungdm93@live.com>
Date: Mon, 30 Mar 2020 15:41:57 -0700
Subject: [PATCH 0389/1280] [SPARK-29574][K8S][FOLLOWUP] Fix bash comparison
 error in Docker entrypoint.sh
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A small change to fix an error in Docker `entrypoint.sh`

When spark running on Kubernetes, I got the following logs:
```log
+ '[' -n ']'
+ '[' -z ']'
++ /bin/hadoop classpath
/opt/entrypoint.sh: line 62: /bin/hadoop: No such file or directory
+ export SPARK_DIST_CLASSPATH=
+ SPARK_DIST_CLASSPATH=
```
This is because you are missing some quotes on bash comparisons.

No

CI

Closes #28075 from dungdm93/patch-1.

Authored-by: Đặng Minh Dũng <dungdm93@live.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 1d0fc9aa85b3ad3326b878de49b748413dee1dd9)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../docker/src/main/dockerfiles/spark/entrypoint.sh           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index 6ee3523c8edab..8218c2982c6f8 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -58,8 +58,8 @@ fi
 
 # If HADOOP_HOME is set and SPARK_DIST_CLASSPATH is not set, set it here so Hadoop jars are available to the executor.
 # It does not set SPARK_DIST_CLASSPATH if already set, to avoid overriding customizations of this value from elsewhere e.g. Docker/K8s.
-if [ -n ${HADOOP_HOME}  ] && [ -z ${SPARK_DIST_CLASSPATH}  ]; then
-  export SPARK_DIST_CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath)  
+if [ -n "${HADOOP_HOME}"  ] && [ -z "${SPARK_DIST_CLASSPATH}"  ]; then
+  export SPARK_DIST_CLASSPATH="$($HADOOP_HOME/bin/hadoop classpath)"
 fi
 
 if ! [ -z ${HADOOP_CONF_DIR+x} ]; then

From ca3887a0de31fa78097ca7ee92ead914a3ce050c Mon Sep 17 00:00:00 2001
From: Luca Canali <luca.canali@cern.ch>
Date: Mon, 30 Mar 2020 18:00:54 -0700
Subject: [PATCH 0390/1280] [SPARK-30775][DOC] Improve the description of
 executor metrics in the monitoring documentation

### What changes were proposed in this pull request?
This PR (SPARK-30775) aims to improve the description of the executor metrics in the monitoring documentation.

### Why are the changes needed?
Improve and clarify monitoring documentation by:
- adding reference to the Prometheus end point, as implemented in [SPARK-29064]
- extending the list and descripion of executor metrics, following up from [SPARK-27157]

### Does this PR introduce any user-facing change?
Documentation update.

### How was this patch tested?
n.a.

Closes #27526 from LucaCanali/docPrometheusMetricsFollowupSpark29064.

Authored-by: Luca Canali <luca.canali@cern.ch>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit aa98ac52dbbe3fc2d3b152af9324a71f48439a38)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/monitoring.md | 58 ++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 51 insertions(+), 7 deletions(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index ba3f1dc86becc..131cd2a844e44 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -689,31 +689,75 @@ A list of the available metrics, with a short description:
 ### Executor Metrics
 
 Executor-level metrics are sent from each executor to the driver as part of the Heartbeat to describe the performance metrics of Executor itself like JVM heap memory, GC information.
-Executor metric values and their measured peak values per executor are exposed via the REST API at the end point `/applications/[app-id]/executors`.
-In addition, aggregated per-stage peak values of the executor metrics are written to the event log if `spark.eventLog.logStageExecutorMetrics` is true.
-Executor metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
+Executor metric values and their measured memory peak values per executor are exposed via the REST API in JSON format and in Prometheus format.
+The JSON end point is exposed at: `/applications/[app-id]/executors`, and the Prometheus endpoint at: `/metrics/executors/prometheus`.
+The Prometheus endpoint is conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
+In addition, aggregated per-stage peak values of the executor memory metrics are written to the event log if
+`spark.eventLog.logStageExecutorMetrics` is true.  
+Executor memory metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
 A list of the available metrics, with a short description:
 
 <table class="table">
   <tr><th>Executor Level Metric name</th>
       <th>Short description</th>
   </tr>
+  <tr>
+    <td>rddBlocks</td>
+    <td>RDD blocks in the block manager of this executor.</td>
+  </tr>
+  <tr>
+    <td>memoryUsed</td>
+    <td>Storage memory used by this executor.</td>
+  </tr>
+  <tr>
+    <td>diskUsed</td>
+    <td>Disk space used for RDD storage by this executor.</td>
+  </tr>
+  <tr>
+    <td>totalCores</td>
+    <td>Number of cores available in this executor.</td>
+  </tr>
+  <tr>
+    <td>maxTasks</td>
+    <td>Maximum number of tasks that can run concurrently in this executor.</td>
+  </tr>
+  <tr>
+    <td>activeTasks</td>
+    <td>Number of tasks currently executing.</td>
+  </tr>
+  <tr>
+    <td>failedTasks</td>
+    <td>Number of tasks that have failed in this executor.</td>
+  </tr>
+  <tr>
+    <td>completedTasks</td>
+    <td>Number of tasks that have completed in this executor.</td>
+  </tr>
+  <tr>
+    <td>totalTasks</td>
+    <td>Total number of tasks (running, failed and completed) in this executor.</td>
+  </tr>
+  <tr>
+    <td>totalDuration</td>
+    <td>Elapsed time the JVM spent executing tasks in this executor.
+    The value is expressed in milliseconds.</td>
+  </tr>
   <tr>
     <td>totalGCTime</td>
-    <td>Elapsed time the JVM spent in garbage collection summed in this Executor.
+    <td>Elapsed time the JVM spent in garbage collection summed in this executor.
     The value is expressed in milliseconds.</td>
   </tr>
   <tr>
     <td>totalInputBytes</td>
-    <td>Total input bytes summed in this Executor.</td>
+    <td>Total input bytes summed in this executor.</td>
   </tr>
   <tr>
     <td>totalShuffleRead</td>
-    <td>Total shuffer read bytes summed in this Executor.</td>
+    <td>Total shuffle read bytes summed in this executor.</td>
   </tr>
   <tr>
     <td>totalShuffleWrite</td>
-    <td>Total shuffer write bytes summed in this Executor.</td>
+    <td>Total shuffle write bytes summed in this executor.</td>
   </tr>
   <tr>
     <td>maxMemory</td>

From 207344d0da86496b377c2c5f5ad613c6d02f4c33 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 31 Mar 2020 17:35:26 +0000
Subject: [PATCH 0391/1280] [SPARK-31010][SQL][FOLLOW-UP] Add Java UDF
 suggestion in error message of untyped Scala UDF

### What changes were proposed in this pull request?

Added Java UDF suggestion in the in error message of untyped Scala UDF.

### Why are the changes needed?

To help user migrate their use case from deprecate untyped Scala UDF to other supported UDF.

### Does this PR introduce any user-facing change?

No. It haven't been released.

### How was this patch tested?

Pass Jenkins.

Closes #28070 from Ngone51/spark_31010.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 590b9a0132b68d9523e663997def957b2e46dfb1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../main/scala/org/apache/spark/sql/functions.scala    | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index fd4e77f693709..782be98c3fa2c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -4841,9 +4841,13 @@ object functions {
         "information. Spark may blindly pass null to the Scala closure with primitive-type " +
         "argument, and the closure will see the default value of the Java type for the null " +
         "argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. " +
-        "You could use typed Scala UDF APIs (e.g. `udf((x: Int) => x)`) to avoid this problem, " +
-        s"or set ${SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key} to true and use this API with " +
-        s"caution."
+        "To get rid of this error, you could:\n" +
+        "1. use typed Scala UDF APIs, e.g. `udf((x: Int) => x)`\n" +
+        "2. use Java UDF APIs, e.g. `udf(new UDF1[String, Integer] { " +
+        "override def call(s: String): Integer = s.length() }, IntegerType)`, " +
+        "if input types are all non primitive\n" +
+        s"3. set ${SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF.key} to true and " +
+        s"use this API with caution"
       throw new AnalysisException(errorMsg)
     }
     SparkUserDefinedFunction(f, dataType, inputEncoders = Nil)

From 01b26c49009d8136f1f962e87ce7e35db43533ab Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Wed, 1 Apr 2020 08:42:15 +0900
Subject: [PATCH 0392/1280] [SPARK-31305][SQL][DOCS] Add a page to list all
 commands in SQL Reference

### What changes were proposed in this pull request?
Add a page to list all commands in SQL Reference...

### Why are the changes needed?
so it's easier for user to find a specific command.

### Does this PR introduce any user-facing change?
before:
![image](https://user-images.githubusercontent.com/13592258/77938658-ec03e700-726a-11ea-983c-7a559cc0aae2.png)

after:
![image](https://user-images.githubusercontent.com/13592258/77937899-d3df9800-7269-11ea-85db-749a9521576a.png)

![image](https://user-images.githubusercontent.com/13592258/77937924-db9f3c80-7269-11ea-9441-7603feee421c.png)

Also move ```use database``` from query category to ddl category.

### How was this patch tested?
Manually build and check

Closes #28074 from huaxingao/list-all.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 1a7f9649b67d2108cb14e9e466855dfe52db6d66)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml   |  4 +--
 docs/sql-ref-syntax-ddl.md |  1 +
 docs/sql-ref-syntax.md     | 62 +++++++++++++++++++++++++++++++++++++-
 3 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 3bf4952bebbcc..6534c50be2cc4 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -123,6 +123,8 @@
               url: sql-ref-syntax-ddl-truncate-table.html
             - text: REPAIR TABLE
               url: sql-ref-syntax-ddl-repair-table.html
+            - text: USE DATABASE
+              url: sql-ref-syntax-qry-select-usedb.html
         - text: Data Manipulation Statements
           url: sql-ref-syntax-dml.html
           subitems:
@@ -152,8 +154,6 @@
                   url: sql-ref-syntax-qry-select-distribute-by.html
                 - text: LIMIT Clause 
                   url: sql-ref-syntax-qry-select-limit.html
-                - text: USE database
-                  url: sql-ref-syntax-qry-select-usedb.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-ddl.md b/docs/sql-ref-syntax-ddl.md
index 954020a874455..ab4e95a1539ff 100644
--- a/docs/sql-ref-syntax-ddl.md
+++ b/docs/sql-ref-syntax-ddl.md
@@ -36,3 +36,4 @@ Data Definition Statements are used to create or modify the structure of databas
 - [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
 - [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
 - [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+- [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 2510278ab252c..3db97ac11dd84 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -19,4 +19,64 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data. The SQL Syntax section describes the SQL syntax in detail along with usage examples when applicable.
+Spark SQL is Apache Spark's module for working with structured data. The SQL Syntax section describes the SQL syntax in detail along with usage examples when applicable. This document provides a list of Data Definition and Data Manipulation Statements, as well as Data Retrieval and Auxiliary Statements.
+
+### DDL Statements
+- [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+- [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+- [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+- [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
+- [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+
+### DML Statements
+- [INSERT INTO](sql-ref-syntax-dml-insert-into.html)
+- [INSERT OVERWRITE](sql-ref-syntax-dml-insert-overwrite-table.html)
+- [INSERT OVERWRITE DIRECTORY](sql-ref-syntax-dml-insert-overwrite-directory.html)
+- [INSERT OVERWRITE DIRECTORY with Hive format](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+- [LOAD](sql-ref-syntax-dml-load.html)
+
+### Data Retrieval Statements
+- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+- [EXPLAIN](sql-ref-syntax-qry-explain.html)
+- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+
+### Auxiliary Statements
+- [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+- [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+- [ANALYZE TABLE](sql-ref-syntax-aux-analyze-table.html)
+- [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+- [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+- [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+- [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+- [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+- [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+- [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+- [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
+- [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
+- [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+- [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+- [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
+- [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+- [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+- [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
+- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+- [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)

From 233dc1260af6df7a8e9a689ba5c6fe3e81a5bc1f Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Wed, 1 Apr 2020 10:38:03 +0900
Subject: [PATCH 0393/1280] [SPARK-31290][R] Add back the deprecated R APIs

### What changes were proposed in this pull request?
Add back the deprecated R APIs removed by https://github.com/apache/spark/pull/22843/ and https://github.com/apache/spark/pull/22815.

These APIs are

- `sparkR.init`
- `sparkRSQL.init`
- `sparkRHive.init`
- `registerTempTable`
- `createExternalTable`
- `dropTempTable`

No need to port the function such as
```r
createExternalTable <- function(x, ...) {
  dispatchFunc("createExternalTable(tableName, path = NULL, source = NULL, ...)", x, ...)
}
```
because this was for the backward compatibility when SQLContext exists before assuming from https://github.com/apache/spark/pull/9192,  but seems we don't need it anymore since SparkR replaced SQLContext with Spark Session at https://github.com/apache/spark/pull/13635.

### Why are the changes needed?
Amend Spark's Semantic Versioning Policy

### Does this PR introduce any user-facing change?
Yes
The removed R APIs are put back.

### How was this patch tested?
Add back the removed tests

Closes #28058 from huaxingao/r.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit fd0b2281272daba590c6bb277688087d0b26053f)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/NAMESPACE                       |  7 ++
 R/pkg/R/DataFrame.R                   | 26 +++++++
 R/pkg/R/catalog.R                     | 54 +++++++++++++++
 R/pkg/R/generics.R                    |  3 +
 R/pkg/R/sparkR.R                      | 98 +++++++++++++++++++++++++++
 R/pkg/tests/fulltests/test_sparkSQL.R | 13 +++-
 docs/sparkr-migration-guide.md        |  3 +-
 7 files changed, 200 insertions(+), 4 deletions(-)

diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
index 7ed2e36d59531..9fd7bb4c6ff2b 100644
--- a/R/pkg/NAMESPACE
+++ b/R/pkg/NAMESPACE
@@ -28,6 +28,7 @@ importFrom("utils", "download.file", "object.size", "packageVersion", "tail", "u
 
 # S3 methods exported
 export("sparkR.session")
+export("sparkR.init")
 export("sparkR.session.stop")
 export("sparkR.stop")
 export("sparkR.conf")
@@ -41,6 +42,9 @@ export("sparkR.callJStatic")
 
 export("install.spark")
 
+export("sparkRSQL.init",
+       "sparkRHive.init")
+
 # MLlib integration
 exportMethods("glm",
               "spark.glm",
@@ -148,6 +152,7 @@ exportMethods("arrange",
               "printSchema",
               "randomSplit",
               "rbind",
+              "registerTempTable",
               "rename",
               "repartition",
               "repartitionByRange",
@@ -420,8 +425,10 @@ export("as.DataFrame",
        "cacheTable",
        "clearCache",
        "createDataFrame",
+       "createExternalTable",
        "createTable",
        "currentDatabase",
+       "dropTempTable",
        "dropTempView",
        "listColumns",
        "listDatabases",
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 593d3ca16220d..14d2076e88eff 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -521,6 +521,32 @@ setMethod("createOrReplaceTempView",
               invisible(callJMethod(x@sdf, "createOrReplaceTempView", viewName))
           })
 
+#' (Deprecated) Register Temporary Table
+#'
+#' Registers a SparkDataFrame as a Temporary Table in the SparkSession
+#' @param x A SparkDataFrame
+#' @param tableName A character vector containing the name of the table
+#'
+#' @seealso \link{createOrReplaceTempView}
+#' @rdname registerTempTable-deprecated
+#' @name registerTempTable
+#' @aliases registerTempTable,SparkDataFrame,character-method
+#' @examples
+#'\dontrun{
+#' sparkR.session()
+#' path <- "path/to/file.json"
+#' df <- read.json(path)
+#' registerTempTable(df, "json_df")
+#' new_df <- sql("SELECT * FROM json_df")
+#'}
+#' @note registerTempTable since 1.4.0
+setMethod("registerTempTable",
+          signature(x = "SparkDataFrame", tableName = "character"),
+          function(x, tableName) {
+              .Deprecated("createOrReplaceTempView")
+              invisible(callJMethod(x@sdf, "createOrReplaceTempView", tableName))
+          })
+
 #' insertInto
 #'
 #' Insert the contents of a SparkDataFrame into a table registered in the current SparkSession.
diff --git a/R/pkg/R/catalog.R b/R/pkg/R/catalog.R
index 7641f8a7a0432..275737f804bde 100644
--- a/R/pkg/R/catalog.R
+++ b/R/pkg/R/catalog.R
@@ -17,6 +17,35 @@
 
 # catalog.R: SparkSession catalog functions
 
+#' (Deprecated) Create an external table
+#'
+#' Creates an external table based on the dataset in a data source,
+#' Returns a SparkDataFrame associated with the external table.
+#'
+#' The data source is specified by the \code{source} and a set of options(...).
+#' If \code{source} is not specified, the default data source configured by
+#' "spark.sql.sources.default" will be used.
+#'
+#' @param tableName a name of the table.
+#' @param path the path of files to load.
+#' @param source the name of external data source.
+#' @param schema the schema of the data required for some data sources.
+#' @param ... additional argument(s) passed to the method.
+#' @return A SparkDataFrame.
+#' @rdname createExternalTable-deprecated
+#' @seealso \link{createTable}
+#' @examples
+#'\dontrun{
+#' sparkR.session()
+#' df <- createExternalTable("myjson", path="path/to/json", source="json", schema)
+#' }
+#' @name createExternalTable
+#' @note createExternalTable since 1.4.0
+createExternalTable <- function(tableName, path = NULL, source = NULL, schema = NULL, ...) {
+  .Deprecated("createTable", old = "createExternalTable")
+  createTable(tableName, path, source, schema, ...)
+}
+
 #' Creates a table based on the dataset in a data source
 #'
 #' Creates a table based on the dataset in a data source. Returns a SparkDataFrame associated with
@@ -130,6 +159,31 @@ clearCache <- function() {
   invisible(callJMethod(catalog, "clearCache"))
 }
 
+#' (Deprecated) Drop Temporary Table
+#'
+#' Drops the temporary table with the given table name in the catalog.
+#' If the table has been cached/persisted before, it's also unpersisted.
+#'
+#' @param tableName The name of the SparkSQL table to be dropped.
+#' @seealso \link{dropTempView}
+#' @rdname dropTempTable-deprecated
+#' @examples
+#' \dontrun{
+#' sparkR.session()
+#' df <- read.df(path, "parquet")
+#' createOrReplaceTempView(df, "table")
+#' dropTempTable("table")
+#' }
+#' @name dropTempTable
+#' @note dropTempTable since 1.4.0
+dropTempTable <- function(tableName) {
+  .Deprecated("dropTempView", old = "dropTempTable")
+  if (class(tableName) != "character") {
+    stop("tableName must be a string.")
+  }
+  dropTempView(tableName)
+}
+
 #' Drops the temporary view with the given view name in the catalog.
 #'
 #' Drops the temporary view with the given view name in the catalog.
diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R
index 4134d5cecc888..89e0982429b5f 100644
--- a/R/pkg/R/generics.R
+++ b/R/pkg/R/generics.R
@@ -528,6 +528,9 @@ setGeneric("persist", function(x, newLevel) { standardGeneric("persist") })
 #' @rdname printSchema
 setGeneric("printSchema", function(x) { standardGeneric("printSchema") })
 
+#' @rdname registerTempTable-deprecated
+setGeneric("registerTempTable", function(x, tableName) { standardGeneric("registerTempTable") })
+
 #' @rdname rename
 setGeneric("rename", function(x, ...) { standardGeneric("rename") })
 
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index cdb59093781fb..cc8c92b8ab262 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -88,6 +88,49 @@ sparkR.stop <- function() {
   sparkR.session.stop()
 }
 
+#' (Deprecated) Initialize a new Spark Context
+#'
+#' This function initializes a new SparkContext.
+#'
+#' @param master The Spark master URL
+#' @param appName Application name to register with cluster manager
+#' @param sparkHome Spark Home directory
+#' @param sparkEnvir Named list of environment variables to set on worker nodes
+#' @param sparkExecutorEnv Named list of environment variables to be used when launching executors
+#' @param sparkJars Character vector of jar files to pass to the worker nodes
+#' @param sparkPackages Character vector of package coordinates
+#' @seealso \link{sparkR.session}
+#' @rdname sparkR.init-deprecated
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init("local[2]", "SparkR", "/home/spark")
+#' sc <- sparkR.init("local[2]", "SparkR", "/home/spark",
+#'                  list(spark.executor.memory="1g"))
+#' sc <- sparkR.init("yarn-client", "SparkR", "/home/spark",
+#'                  list(spark.executor.memory="4g"),
+#'                  list(LD_LIBRARY_PATH="/directory of JVM libraries (libjvm.so) on workers/"),
+#'                  c("one.jar", "two.jar", "three.jar"),
+#'                  c("com.databricks:spark-avro_2.11:2.0.1"))
+#'}
+#' @note sparkR.init since 1.4.0
+sparkR.init <- function(
+  master = "",
+  appName = "SparkR",
+  sparkHome = Sys.getenv("SPARK_HOME"),
+  sparkEnvir = list(),
+  sparkExecutorEnv = list(),
+  sparkJars = "",
+  sparkPackages = "") {
+  .Deprecated("sparkR.session")
+  sparkR.sparkContext(master,
+     appName,
+     sparkHome,
+     convertNamedListToEnv(sparkEnvir),
+     convertNamedListToEnv(sparkExecutorEnv),
+     sparkJars,
+     sparkPackages)
+}
+
 # Internal function to handle creating the SparkContext.
 sparkR.sparkContext <- function(
   master = "",
@@ -229,6 +272,61 @@ sparkR.sparkContext <- function(
   sc
 }
 
+#' (Deprecated) Initialize a new SQLContext
+#'
+#' This function creates a SparkContext from an existing JavaSparkContext and
+#' then uses it to initialize a new SQLContext
+#'
+#' Starting SparkR 2.0, a SparkSession is initialized and returned instead.
+#' This API is deprecated and kept for backward compatibility only.
+#'
+#' @param jsc The existing JavaSparkContext created with SparkR.init()
+#' @seealso \link{sparkR.session}
+#' @rdname sparkRSQL.init-deprecated
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init()
+#' sqlContext <- sparkRSQL.init(sc)
+#'}
+#' @note sparkRSQL.init since 1.4.0
+sparkRSQL.init <- function(jsc = NULL) {
+  .Deprecated("sparkR.session")
+
+  if (exists(".sparkRsession", envir = .sparkREnv)) {
+    return(get(".sparkRsession", envir = .sparkREnv))
+  }
+
+  # Default to without Hive support for backward compatibility.
+  sparkR.session(enableHiveSupport = FALSE)
+}
+
+#' (Deprecated) Initialize a new HiveContext
+#'
+#' This function creates a HiveContext from an existing JavaSparkContext
+#'
+#' Starting SparkR 2.0, a SparkSession is initialized and returned instead.
+#' This API is deprecated and kept for backward compatibility only.
+#'
+#' @param jsc The existing JavaSparkContext created with SparkR.init()
+#' @seealso \link{sparkR.session}
+#' @rdname sparkRHive.init-deprecated
+#' @examples
+#'\dontrun{
+#' sc <- sparkR.init()
+#' sqlContext <- sparkRHive.init(sc)
+#'}
+#' @note sparkRHive.init since 1.4.0
+sparkRHive.init <- function(jsc = NULL) {
+  .Deprecated("sparkR.session")
+
+  if (exists(".sparkRsession", envir = .sparkREnv)) {
+    return(get(".sparkRsession", envir = .sparkREnv))
+  }
+
+  # Default to without Hive support for backward compatibility.
+  sparkR.session(enableHiveSupport = TRUE)
+}
+
 #' Get the existing SparkSession or initialize a new SparkSession.
 #'
 #' SparkSession is the entry point into SparkR. \code{sparkR.session} gets the existing
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index c1d277ac84be1..32536de3e9548 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -106,6 +106,15 @@ if (is_windows()) {
   Sys.setenv(TZ = "GMT")
 }
 
+test_that("calling sparkRSQL.init returns existing SQL context", {
+  sqlContext <- suppressWarnings(sparkRSQL.init(sc))
+  expect_equal(suppressWarnings(sparkRSQL.init(sc)), sqlContext)
+})
+
+test_that("calling sparkRSQL.init returns existing SparkSession", {
+  expect_equal(suppressWarnings(sparkRSQL.init(sc)), sparkSession)
+})
+
 test_that("calling sparkR.session returns existing SparkSession", {
   expect_equal(sparkR.session(), sparkSession)
 })
@@ -656,10 +665,10 @@ test_that("test tableNames and tables", {
   expect_true("tableName" %in% colnames(tables()))
   expect_true(all(c("tableName", "database", "isTemporary") %in% colnames(tables())))
 
-  createOrReplaceTempView(df, "table2")
+  suppressWarnings(registerTempTable(df, "table2"))
   tables <- listTables()
   expect_equal(count(tables), count + 2)
-  dropTempView("table1")
+  suppressWarnings(dropTempTable("table1"))
   expect_true(dropTempView("table2"))
 
   tables <- listTables()
diff --git a/docs/sparkr-migration-guide.md b/docs/sparkr-migration-guide.md
index 6fbc4c03aefc1..32836cdac53d5 100644
--- a/docs/sparkr-migration-guide.md
+++ b/docs/sparkr-migration-guide.md
@@ -28,8 +28,7 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
 ## Upgrading from SparkR 2.4 to 3.0
 
- - The deprecated methods `sparkR.init`, `sparkRSQL.init`, `sparkRHive.init` have been removed. Use `sparkR.session` instead.
- - The deprecated methods `parquetFile`, `saveAsParquetFile`, `jsonFile`, `registerTempTable`, `createExternalTable`, and `dropTempTable` have been removed. Use `read.parquet`, `write.parquet`, `read.json`, `createOrReplaceTempView`, `createTable`, `dropTempView`, `union` instead.
+ - The deprecated methods `parquetFile`, `saveAsParquetFile`, `jsonFile`, `jsonRDD` have been removed. Use `read.parquet`, `write.parquet`, `read.json` instead.
 
 ## Upgrading from SparkR 2.3 to 2.4
 

From 6f55ed4a16f9d54a1817f9ee618e4847061580b0 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Wed, 1 Apr 2020 04:56:05 +0000
Subject: [PATCH 0394/1280] [SPARK-31318][SQL] Split Parquet/Avro configs for
 rebasing dates/timestamps in read and in write

In the PR, I propose to replace the following SQL configs:
1.  `spark.sql.legacy.parquet.rebaseDateTime.enabled` by
    - `spark.sql.legacy.parquet.rebaseDateTimeInWrite.enabled` (`false` by default). The config enables rebasing dates/timestamps while saving to Parquet files. If it is set to `true`, dates/timestamps are converted to local date-time in Proleptic Gregorian calendar, date-time fields are extracted, and used in building new local date-time in the hybrid calendar (Julian + Gregorian). The resulted local date-time is converted to days or microseconds since the epoch.
    - `spark.sql.legacy.parquet.rebaseDateTimeInRead.enabled` (`false` by default). The config enables rebasing of dates/timestamps in reading from Parquet files.
2. `spark.sql.legacy.avro.rebaseDateTime.enabled` by
    - `spark.sql.legacy.avro.rebaseDateTimeInWrite.enabled` (`false` by default). It enables dates/timestamps rebasing from Proleptic Gregorian calendar to the hybrid calendar via local date/timestamps.
    - `spark.sql.legacy.avro.rebaseDateTimeInRead.enabled` (`false` by default).  It enables rebasing dates/timestamps from the hybrid calendar to Proleptic Gregorian calendar in read. The rebasing is performed by converting micros/millis/days to a local date/timestamp in the source calendar, interpreting the resulted date/timestamp in the target calendar, and getting the number of micros/millis/days since the epoch 1970-01-01 00:00:00Z.

This allows to load dates/timestamps saved by Spark 2.4, and save to Parquet/Avro files without rebasing. And the reverse use case - load data saved by Spark 3.0, and save it in the form which is compatible with Spark 2.4.

Yes, users have to use new SQL configs. Old SQL configs are removed by the PR.

By existing test suites `AvroV1Suite`, `AvroV2Suite` and `ParquetIOSuite`.

Closes #28082 from MaxGekk/split-rebase-configs.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c5323d2e8d74edddefcb0bb7e965d12de45abb18)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/avro/AvroDeserializer.scala     |  3 +-
 .../spark/sql/avro/AvroSerializer.scala       |  3 +-
 .../org/apache/spark/sql/avro/AvroSuite.scala | 23 ++++++-----
 .../apache/spark/sql/internal/SQLConf.scala   | 40 ++++++++++++++-----
 .../parquet/VectorizedColumnReader.java       |  2 +-
 .../parquet/ParquetRowConverter.scala         |  2 +-
 .../parquet/ParquetWriteSupport.scala         |  3 +-
 .../benchmark/DateTimeRebaseBenchmark.scala   |  4 +-
 .../datasources/parquet/ParquetIOSuite.scala  | 16 ++++----
 9 files changed, 63 insertions(+), 33 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 3e8a7f95ccd96..d2aa8a96aba84 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -44,7 +44,8 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
   private lazy val decimalConversions = new DecimalConversion()
 
   // Enable rebasing date/timestamp from Julian to Proleptic Gregorian calendar
-  private val rebaseDateTime = SQLConf.get.avroRebaseDateTimeEnabled
+  private val rebaseDateTime =
+    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ)
 
   private val converter: Any => Any = rootCatalystType match {
     // A shortcut for empty schema.
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index b6e2a2b15aac6..7f454e2bf9491 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -45,7 +45,8 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
   extends Logging {
 
   // Whether to rebase datetimes from Gregorian to Julian calendar in write
-  private val rebaseDateTime: Boolean = SQLConf.get.avroRebaseDateTimeEnabled
+  private val rebaseDateTime: Boolean =
+    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE)
 
   def serialize(catalystData: Any): Any = {
     converter.apply(catalystData)
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index a04037c17d7e6..9336d8e2257f9 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -1530,7 +1530,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
-    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
       checkAnswer(
         readResourceAvroFile("before_1582_date_v2_4.avro"),
         Row(java.sql.Date.valueOf("1001-01-01")))
@@ -1548,15 +1548,16 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     val nonRebased = "1001-01-07 01:09:05.123456"
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
         Seq(tsStr).toDF("tsS")
           .select($"tsS".cast("timestamp").as("ts"))
           .write.format("avro")
           .save(path)
-
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
         checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
       }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "false") {
         checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
       }
     }
@@ -1580,19 +1581,20 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         |}""".stripMargin
       withTempPath { dir =>
         val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
           Seq(tsStr).toDF("tsS")
             .select($"tsS".cast("timestamp").as("ts"))
             .write
             .option("avroSchema", timestampSchema)
             .format("avro")
             .save(path)
-
+        }
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
           checkAnswer(
             spark.read.schema("ts timestamp").format("avro").load(path),
             Row(Timestamp.valueOf(rebased)))
         }
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "false") {
           checkAnswer(
             spark.read.schema("ts timestamp").format("avro").load(path),
             Row(Timestamp.valueOf(nonRebased)))
@@ -1604,15 +1606,16 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
   test("SPARK-31183: rebasing dates in write") {
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "true") {
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
         Seq("1001-01-01").toDF("dateS")
           .select($"dateS".cast("date").as("date"))
           .write.format("avro")
           .save(path)
-
+      }
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
         checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
       }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME.key -> "false") {
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "false") {
         checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
       }
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 26ddd40f76e04..44f303766f017 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2254,11 +2254,22 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
-  val LEGACY_PARQUET_REBASE_DATETIME =
-    buildConf("spark.sql.legacy.parquet.rebaseDateTime.enabled")
+  val LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE =
+    buildConf("spark.sql.legacy.parquet.rebaseDateTimeInWrite.enabled")
       .internal()
       .doc("When true, rebase dates/timestamps from Proleptic Gregorian calendar " +
-        "to the hybrid calendar (Julian + Gregorian) in write and " +
+        "to the hybrid calendar (Julian + Gregorian) in write. " +
+        "The rebasing is performed by converting micros/millis/days to " +
+        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
+        "timestamp in the target calendar, and getting the number of micros/millis/days " +
+        "since the epoch 1970-01-01 00:00:00Z.")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_PARQUET_REBASE_DATETIME_IN_READ =
+    buildConf("spark.sql.legacy.parquet.rebaseDateTimeInRead.enabled")
+      .internal()
+      .doc("When true, rebase dates/timestamps " +
         "from the hybrid calendar to Proleptic Gregorian calendar in read. " +
         "The rebasing is performed by converting micros/millis/days to " +
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
@@ -2267,11 +2278,22 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_AVRO_REBASE_DATETIME =
-    buildConf("spark.sql.legacy.avro.rebaseDateTime.enabled")
+  val LEGACY_AVRO_REBASE_DATETIME_IN_WRITE =
+    buildConf("spark.sql.legacy.avro.rebaseDateTimeInWrite.enabled")
       .internal()
       .doc("When true, rebase dates/timestamps from Proleptic Gregorian calendar " +
-        "to the hybrid calendar (Julian + Gregorian) in write and " +
+        "to the hybrid calendar (Julian + Gregorian) in write. " +
+        "The rebasing is performed by converting micros/millis/days to " +
+        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
+        "timestamp in the target calendar, and getting the number of micros/millis/days " +
+        "since the epoch 1970-01-01 00:00:00Z.")
+      .booleanConf
+      .createWithDefault(false)
+
+  val LEGACY_AVRO_REBASE_DATETIME_IN_READ =
+    buildConf("spark.sql.legacy.avro.rebaseDateTimeInRead.enabled")
+      .internal()
+      .doc("When true, rebase dates/timestamps " +
         "from the hybrid calendar to Proleptic Gregorian calendar in read. " +
         "The rebasing is performed by converting micros/millis/days to " +
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
@@ -2857,9 +2879,9 @@ class SQLConf extends Serializable with Logging {
 
   def csvFilterPushDown: Boolean = getConf(CSV_FILTER_PUSHDOWN_ENABLED)
 
-  def parquetRebaseDateTimeEnabled: Boolean = getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME)
-
-  def avroRebaseDateTimeEnabled: Boolean = getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME)
+  def parquetRebaseDateTimeInReadEnabled: Boolean = {
+    getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ)
+  }
 
   /** ********************** SQLConf functionality methods ************ */
 
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index fa1838f092c2e..f209f7f288f58 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -131,7 +131,7 @@ public VectorizedColumnReader(
     if (totalValueCount == 0) {
       throw new IOException("totalValueCount == 0");
     }
-    this.rebaseDateTime = SQLConf.get().parquetRebaseDateTimeEnabled();
+    this.rebaseDateTime = SQLConf.get().parquetRebaseDateTimeInReadEnabled();
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 85a0d43b5e438..18011dbf6cd6b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -131,7 +131,7 @@ private[parquet] class ParquetRowConverter(
   extends ParquetGroupConverter(updater) with Logging {
 
   // Enable rebasing date/timestamp from Julian to Proleptic Gregorian calendar
-  private val rebaseDateTime = SQLConf.get.parquetRebaseDateTimeEnabled
+  private val rebaseDateTime = SQLConf.get.parquetRebaseDateTimeInReadEnabled
 
   assert(
     parquetType.getFieldCount <= catalystType.length,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index 8904b355c4455..1c4f2978a48d1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -78,7 +78,8 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
     new Array[Byte](Decimal.minBytesForPrecision(DecimalType.MAX_PRECISION))
 
   // Whether to rebase datetimes from Gregorian to Julian calendar in write
-  private val rebaseDateTime: Boolean = SQLConf.get.parquetRebaseDateTimeEnabled
+  private val rebaseDateTime: Boolean =
+    SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE)
 
   override def init(configuration: Configuration): WriteContext = {
     val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index 983d9b4f709f4..48ceccc9002c0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -109,7 +109,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
             val rebaseFlag = if (rebase) "on" else "off"
             val caseName = s"$period 1582, rebase $rebaseFlag"
             benchmark.addCase(caseName, 1) { _ =>
-              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> rebase.toString) {
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
                 val df = genDF(rowsNum, dateTime, after1582)
                 val pathToWrite = path.getAbsolutePath + s"/${dateTime}_${period}_1582_$rebaseFlag"
                 df.write
@@ -138,7 +138,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
             benchmark2.addCase(caseName, 3) { _ =>
               withSQLConf(
                 SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
-                SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> rebase.toString) {
+                SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
                 val pathToRead = path.getAbsolutePath + s"/${dateTime}_${period}_1582_$rebaseFlag"
                 spark.read.format("parquet").load(pathToRead).noop()
               }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 497b823868450..f901ce16f0965 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -884,7 +884,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
     Seq(false, true).foreach { vectorized =>
       withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "true") {
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
           checkAnswer(
             readResourceParquetFile("test-data/before_1582_date_v2_4.snappy.parquet"),
             Row(java.sql.Date.valueOf("1001-01-01")))
@@ -912,15 +912,16 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
           withTempPath { dir =>
             val path = dir.getAbsolutePath
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "true") {
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
               Seq(tsStr).toDF("tsS")
                 .select($"tsS".cast("timestamp").as("ts"))
                 .write
                 .parquet(path)
-
+            }
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
               checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
             }
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "false") {
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "false") {
               checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(nonRebased)))
             }
           }
@@ -932,15 +933,16 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   test("SPARK-31159: rebasing dates in write") {
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "true") {
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
         Seq("1001-01-01").toDF("dateS")
           .select($"dateS".cast("date").as("date"))
           .write
           .parquet(path)
-
+      }
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
         checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
       }
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME.key -> "false") {
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "false") {
         checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-07")))
       }
     }

From 3339deee0776cea64ce557d18427e8beb8fb8745 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 1 Apr 2020 07:02:26 +0000
Subject: [PATCH 0395/1280] [SPARK-31311][SQL][TESTS] Benchmark date-time
 rebasing in ORC datasource

### What changes were proposed in this pull request?
In the PR, I propose to add new benchmarks to `DateTimeRebaseBenchmark` for saving and loading dates/timestamps to/from ORC files. I extracted common code from the benchmark for Parquet datasource and place it to the methods `caseName()` and `getPath()`. Added benchmarks for ORC save/load dates before and after 1582-10-15 because an implementation may have different performance for dates before the Julian calendar cutover day, see #28067 as an example.

### Why are the changes needed?
To have the base line for future optimizations of `fromJavaDate()`/`toJavaDate()` and `toJavaTimestamp()`/`fromJavaTimestamp()` in `DateTimeUtils`. The methods are used while saving/loading dates/timestamps by ORC datasource.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running the updated benchmark `DateTimeRebaseBenchmark` via the command:
```
SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.DateTimeRebaseBenchmark"
```
in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 1.8.0_242-8u242/11.0.6+10 |

Closes #28076 from MaxGekk/rebase-benchmark-orc.

Lead-authored-by: Max Gekk <max.gekk@gmail.com>
Co-authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 91af87d34ee11490164ca5f1023d9818775dceb7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeRebaseBenchmark-jdk11-results.txt |  97 +++++++++----
 .../DateTimeRebaseBenchmark-results.txt       |  97 +++++++++----
 .../benchmark/DateTimeRebaseBenchmark.scala   | 129 ++++++++++++------
 3 files changed, 227 insertions(+), 96 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 4fed51113912f..01b063993cd9c 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,48 +6,89 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9304           9304           0         10.7          93.0       1.0X
-before 1582, noop                                  9187           9187           0         10.9          91.9       1.0X
-after 1582, rebase off                            22054          22054           0          4.5         220.5       0.4X
-after 1582, rebase on                             20361          20361           0          4.9         203.6       0.5X
-before 1582, rebase off                           20286          20286           0          4.9         202.9       0.5X
-before 1582, rebase on                            22230          22230           0          4.5         222.3       0.4X
+after 1582, noop                                   9299           9299           0         10.8          93.0       1.0X
+before 1582, noop                                  9220           9220           0         10.8          92.2       1.0X
+after 1582, rebase off                            20390          20390           0          4.9         203.9       0.5X
+after 1582, rebase on                             20378          20378           0          4.9         203.8       0.5X
+before 1582, rebase off                           20069          20069           0          5.0         200.7       0.5X
+before 1582, rebase on                            20637          20637           0          4.8         206.4       0.5X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12773          12866         129          7.8         127.7       1.0X
-after 1582, vec off, rebase on                    13063          13086          39          7.7         130.6       1.0X
-after 1582, vec on, rebase off                     3678           3719          61         27.2          36.8       3.5X
-after 1582, vec on, rebase on                      5078           5121          52         19.7          50.8       2.5X
-before 1582, vec off, rebase off                  12942          12972          42          7.7         129.4       1.0X
-before 1582, vec off, rebase on                   13866          13904          58          7.2         138.7       0.9X
-before 1582, vec on, rebase off                    3678           3711          43         27.2          36.8       3.5X
-before 1582, vec on, rebase on                     5621           5657          44         17.8          56.2       2.3X
+after 1582, vec off, rebase off                   12927          13017          78          7.7         129.3       1.0X
+after 1582, vec off, rebase on                    13127          13176          50          7.6         131.3       1.0X
+after 1582, vec on, rebase off                     3725           3779          91         26.8          37.3       3.5X
+after 1582, vec on, rebase on                      5134           5221          99         19.5          51.3       2.5X
+before 1582, vec off, rebase off                  13049          13061          16          7.7         130.5       1.0X
+before 1582, vec off, rebase on                   13877          13916          51          7.2         138.8       0.9X
+before 1582, vec on, rebase off                    3702           3736          56         27.0          37.0       3.5X
+before 1582, vec on, rebase on                     5567           5637          78         18.0          55.7       2.3X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2983           2983           0         33.5          29.8       1.0X
-before 1582, noop                                  2979           2979           0         33.6          29.8       1.0X
-after 1582, rebase off                            17452          17452           0          5.7         174.5       0.2X
-after 1582, rebase on                             70193          70193           0          1.4         701.9       0.0X
-before 1582, rebase off                           17784          17784           0          5.6         177.8       0.2X
-before 1582, rebase on                            83498          83498           0          1.2         835.0       0.0X
+after 1582, noop                                   2988           2988           0         33.5          29.9       1.0X
+before 1582, noop                                  3000           3000           0         33.3          30.0       1.0X
+after 1582, rebase off                            16163          16163           0          6.2         161.6       0.2X
+after 1582, rebase on                             68399          68399           0          1.5         684.0       0.0X
+before 1582, rebase off                           16921          16921           0          5.9         169.2       0.2X
+before 1582, rebase on                            74425          74425           0          1.3         744.3       0.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15114          15151          32          6.6         151.1       1.0X
-after 1582, vec off, rebase on                    45804          45912         126          2.2         458.0       0.3X
-after 1582, vec on, rebase off                     4900           4947          56         20.4          49.0       3.1X
-after 1582, vec on, rebase on                     34599          34650          45          2.9         346.0       0.4X
-before 1582, vec off, rebase off                  15093          15174          70          6.6         150.9       1.0X
-before 1582, vec off, rebase on                   47367          47472         121          2.1         473.7       0.3X
-before 1582, vec on, rebase off                    4884           4952          80         20.5          48.8       3.1X
-before 1582, vec on, rebase on                    35831          35883          59          2.8         358.3       0.4X
+after 1582, vec off, rebase off                   15147          15258          97          6.6         151.5       1.0X
+after 1582, vec off, rebase on                    45035          45101          60          2.2         450.3       0.3X
+after 1582, vec on, rebase off                     4934           5012         100         20.3          49.3       3.1X
+after 1582, vec on, rebase on                     34263          34360          88          2.9         342.6       0.4X
+before 1582, vec off, rebase off                  15177          15220          37          6.6         151.8       1.0X
+before 1582, vec off, rebase on                   46754          46761          12          2.1         467.5       0.3X
+before 1582, vec on, rebase off                    4892           4956          61         20.4          48.9       3.1X
+before 1582, vec on, rebase on                    35989          36014          22          2.8         359.9       0.4X
+
+
+================================================================================================
+Rebasing dates/timestamps in ORC datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   9295           9295           0         10.8          93.0       1.0X
+before 1582, noop                                  9352           9352           0         10.7          93.5       1.0X
+after 1582                                        17112          17112           0          5.8         171.1       0.5X
+before 1582                                       17979          17979           0          5.6         179.8       0.5X
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off                               20874          20905          38          4.8         208.7       1.0X
+after 1582, vec on                                 3813           3844          28         26.2          38.1       5.5X
+before 1582, vec off                              25912          25949          38          3.9         259.1       0.8X
+before 1582, vec on                                4322           4343          19         23.1          43.2       4.8X
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   3003           3003           0         33.3          30.0       1.0X
+before 1582, noop                                  3012           3012           0         33.2          30.1       1.0X
+after 1582                                        41031          41031           0          2.4         410.3       0.1X
+before 1582                                       44436          44436           0          2.3         444.4       0.1X
+
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off                               28477          28582          92          3.5         284.8       1.0X
+after 1582, vec on                                20754          20924         237          4.8         207.5       1.4X
+before 1582, vec off                              32858          32921          58          3.0         328.6       0.9X
+before 1582, vec on                               25734          25769          30          3.9         257.3       1.1X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index ee486276653fe..b35301349a1ee 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,48 +6,89 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9582           9582           0         10.4          95.8       1.0X
-before 1582, noop                                  9473           9473           0         10.6          94.7       1.0X
-after 1582, rebase off                            21431          21431           0          4.7         214.3       0.4X
-after 1582, rebase on                             22156          22156           0          4.5         221.6       0.4X
-before 1582, rebase off                           21399          21399           0          4.7         214.0       0.4X
-before 1582, rebase on                            22927          22927           0          4.4         229.3       0.4X
+after 1582, noop                                   9691           9691           0         10.3          96.9       1.0X
+before 1582, noop                                  9024           9024           0         11.1          90.2       1.1X
+after 1582, rebase off                            21195          21195           0          4.7         211.9       0.5X
+after 1582, rebase on                             20045          20045           0          5.0         200.4       0.5X
+before 1582, rebase off                           20039          20039           0          5.0         200.4       0.5X
+before 1582, rebase on                            20451          20451           0          4.9         204.5       0.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12637          12736         111          7.9         126.4       1.0X
-after 1582, vec off, rebase on                    13463          13531          61          7.4         134.6       0.9X
-after 1582, vec on, rebase off                     3693           3703           8         27.1          36.9       3.4X
-after 1582, vec on, rebase on                      5242           5252           9         19.1          52.4       2.4X
-before 1582, vec off, rebase off                  13055          13169         126          7.7         130.5       1.0X
-before 1582, vec off, rebase on                   14067          14270         185          7.1         140.7       0.9X
-before 1582, vec on, rebase off                    3697           3702           7         27.1          37.0       3.4X
-before 1582, vec on, rebase on                     6058           6097          34         16.5          60.6       2.1X
+after 1582, vec off, rebase off                   13207          13339         116          7.6         132.1       1.0X
+after 1582, vec off, rebase on                    13408          13446          57          7.5         134.1       1.0X
+after 1582, vec on, rebase off                     3680           3712          39         27.2          36.8       3.6X
+after 1582, vec on, rebase on                      5229           5261          29         19.1          52.3       2.5X
+before 1582, vec off, rebase off                  13135          13164          25          7.6         131.4       1.0X
+before 1582, vec off, rebase on                   13946          14033          94          7.2         139.5       0.9X
+before 1582, vec on, rebase off                    3689           3726          49         27.1          36.9       3.6X
+before 1582, vec on, rebase on                     5679           5687           9         17.6          56.8       2.3X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2713           2713           0         36.9          27.1       1.0X
-before 1582, noop                                  2715           2715           0         36.8          27.2       1.0X
-after 1582, rebase off                            16768          16768           0          6.0         167.7       0.2X
-after 1582, rebase on                             82811          82811           0          1.2         828.1       0.0X
-before 1582, rebase off                           17052          17052           0          5.9         170.5       0.2X
-before 1582, rebase on                            95134          95134           0          1.1         951.3       0.0X
+after 1582, noop                                   2720           2720           0         36.8          27.2       1.0X
+before 1582, noop                                  2712           2712           0         36.9          27.1       1.0X
+after 1582, rebase off                            16626          16626           0          6.0         166.3       0.2X
+after 1582, rebase on                             85136          85136           0          1.2         851.4       0.0X
+before 1582, rebase off                           16855          16855           0          5.9         168.6       0.2X
+before 1582, rebase on                           106121         106121           0          0.9        1061.2       0.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15200          15321         194          6.6         152.0       1.0X
-after 1582, vec off, rebase on                    63160          63337         177          1.6         631.6       0.2X
-after 1582, vec on, rebase off                     4891           4928          43         20.4          48.9       3.1X
-after 1582, vec on, rebase on                     45474          45484          10          2.2         454.7       0.3X
-before 1582, vec off, rebase off                  15203          15330         110          6.6         152.0       1.0X
-before 1582, vec off, rebase on                   65588          65664          73          1.5         655.9       0.2X
-before 1582, vec on, rebase off                    4844           4916         105         20.6          48.4       3.1X
-before 1582, vec on, rebase on                    47815          47943         162          2.1         478.2       0.3X
+after 1582, vec off, rebase off                   15198          15301          90          6.6         152.0       1.0X
+after 1582, vec off, rebase on                    55210          55370         140          1.8         552.1       0.3X
+after 1582, vec on, rebase off                     4859           4880          19         20.6          48.6       3.1X
+after 1582, vec on, rebase on                     44758          44824          85          2.2         447.6       0.3X
+before 1582, vec off, rebase off                  15206          15316         112          6.6         152.1       1.0X
+before 1582, vec off, rebase on                   60452          60588         222          1.7         604.5       0.3X
+before 1582, vec on, rebase off                    4892           4933          36         20.4          48.9       3.1X
+before 1582, vec on, rebase on                    46871          46950          82          2.1         468.7       0.3X
+
+
+================================================================================================
+Rebasing dates/timestamps in ORC datasource
+================================================================================================
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   9102           9102           0         11.0          91.0       1.0X
+before 1582, noop                                  9099           9099           0         11.0          91.0       1.0X
+after 1582                                        17652          17652           0          5.7         176.5       0.5X
+before 1582                                       18284          18284           0          5.5         182.8       0.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off                               25169          25215          48          4.0         251.7       1.0X
+after 1582, vec on                                 3701           3717          16         27.0          37.0       6.8X
+before 1582, vec off                              26919          27045         182          3.7         269.2       0.9X
+before 1582, vec on                                4169           4192          31         24.0          41.7       6.0X
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, noop                                   2906           2906           0         34.4          29.1       1.0X
+before 1582, noop                                  2863           2863           0         34.9          28.6       1.0X
+after 1582                                        48858          48858           0          2.0         488.6       0.1X
+before 1582                                       50945          50945           0          2.0         509.5       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1582, vec off                               40925          40955          26          2.4         409.2       1.0X
+after 1582, vec on                                31246          31404         164          3.2         312.5       1.3X
+before 1582, vec off                              44634          44680          40          2.2         446.3       0.9X
+before 1582, vec on                               35578          35834         282          2.8         355.8       1.2X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index 48ceccc9002c0..6285461d93097 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.execution.benchmark
 
+import java.io.File
 import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneOffset}
 
 import org.apache.spark.benchmark.Benchmark
@@ -91,68 +92,116 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
     }
   }
 
+  private def benchmarkInputs(benchmark: Benchmark, rowsNum: Int, dateTime: String): Unit = {
+    benchmark.addCase("after 1582, noop", 1) { _ =>
+      genDF(rowsNum, dateTime, after1582 = true).noop()
+    }
+    benchmark.addCase("before 1582, noop", 1) { _ =>
+      genDF(rowsNum, dateTime, after1582 = false).noop()
+    }
+  }
+
+  private def flagToStr(flag: Boolean): String = {
+    if (flag) "on" else "off"
+  }
+
+  private def caseName(
+      after1582: Boolean,
+      rebase: Option[Boolean] = None,
+      vec: Option[Boolean] = None): String = {
+    val period = if (after1582) "after" else "before"
+    val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("")
+    val rebaseFlag = rebase.map(flagToStr).map(flag => s", rebase $flag").getOrElse("")
+    s"$period 1582$vecFlag$rebaseFlag"
+  }
+
+  private def getPath(
+      basePath: File,
+      dateTime: String,
+      after1582: Boolean,
+      rebase: Option[Boolean] = None): String = {
+    val period = if (after1582) "after" else "before"
+    val rebaseFlag = rebase.map(flagToStr).map(flag => s"_$flag").getOrElse("")
+    basePath.getAbsolutePath + s"/${dateTime}_${period}_1582$rebaseFlag"
+  }
+
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    val rowsNum = 100000000
+
     withTempPath { path =>
       runBenchmark("Rebasing dates/timestamps in Parquet datasource") {
-        val rowsNum = 100000000
         Seq("date", "timestamp").foreach { dateTime =>
           val benchmark = new Benchmark(s"Save ${dateTime}s to parquet", rowsNum, output = output)
-          benchmark.addCase("after 1582, noop", 1) { _ =>
-            genDF(rowsNum, dateTime, after1582 = true).noop()
-          }
-          benchmark.addCase("before 1582, noop", 1) { _ =>
-            genDF(rowsNum, dateTime, after1582 = false).noop()
-          }
-
-          def save(after1582: Boolean, rebase: Boolean): Unit = {
-            val period = if (after1582) "after" else "before"
-            val rebaseFlag = if (rebase) "on" else "off"
-            val caseName = s"$period 1582, rebase $rebaseFlag"
-            benchmark.addCase(caseName, 1) { _ =>
-              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
-                val df = genDF(rowsNum, dateTime, after1582)
-                val pathToWrite = path.getAbsolutePath + s"/${dateTime}_${period}_1582_$rebaseFlag"
-                df.write
-                  .mode("overwrite")
-                  .format("parquet")
-                  .save(pathToWrite)
-              }
-            }
-          }
-
+          benchmarkInputs(benchmark, rowsNum, dateTime)
           Seq(true, false).foreach { after1582 =>
             Seq(false, true).foreach { rebase =>
-              save(after1582, rebase)
+              benchmark.addCase(caseName(after1582, Some(rebase)), 1) { _ =>
+                withSQLConf(
+                  SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
+                  genDF(rowsNum, dateTime, after1582)
+                    .write
+                    .mode("overwrite")
+                    .format("parquet")
+                    .save(getPath(path, dateTime, after1582, Some(rebase)))
+                }
+              }
             }
           }
           benchmark.run()
 
           val benchmark2 = new Benchmark(
             s"Load ${dateTime}s from parquet", rowsNum, output = output)
-
-          def load(after1582: Boolean, vec: Boolean, rebase: Boolean): Unit = {
-            val period = if (after1582) "after" else "before"
-            val rebaseFlag = if (rebase) "on" else "off"
-            val vecFlag = if (vec) "on" else "off"
-            val caseName = s"$period 1582, vec $vecFlag, rebase $rebaseFlag"
-            benchmark2.addCase(caseName, 3) { _ =>
-              withSQLConf(
-                SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
-                SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
-                val pathToRead = path.getAbsolutePath + s"/${dateTime}_${period}_1582_$rebaseFlag"
-                spark.read.format("parquet").load(pathToRead).noop()
+          Seq(true, false).foreach { after1582 =>
+            Seq(false, true).foreach { vec =>
+              Seq(false, true).foreach { rebase =>
+                benchmark2.addCase(caseName(after1582, Some(rebase), Some(vec)), 3) { _ =>
+                  withSQLConf(
+                    SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
+                    SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                    spark.read
+                      .format("parquet")
+                      .load(getPath(path, dateTime, after1582, Some(rebase)))
+                      .noop()
+                  }
+                }
               }
             }
           }
+          benchmark2.run()
+        }
+      }
+    }
+
+    withTempPath { path =>
+      runBenchmark("Rebasing dates/timestamps in ORC datasource") {
+        Seq("date", "timestamp").foreach { dateTime =>
+          val benchmark = new Benchmark(s"Save ${dateTime}s to ORC", rowsNum, output = output)
+          benchmarkInputs(benchmark, rowsNum, dateTime)
+          Seq(true, false).foreach { after1582 =>
+            benchmark.addCase(caseName(after1582), 1) { _ =>
+              genDF(rowsNum, dateTime, after1582)
+                .write
+                .mode("overwrite")
+                .format("orc")
+                .save(getPath(path, dateTime, after1582))
+            }
+          }
+          benchmark.run()
 
+          val benchmark2 = new Benchmark(s"Load ${dateTime}s from ORC", rowsNum, output = output)
           Seq(true, false).foreach { after1582 =>
             Seq(false, true).foreach { vec =>
-              Seq(false, true).foreach { rebase =>
-                load(after1582, vec, rebase)
+              benchmark2.addCase(caseName(after1582, vec = Some(vec)), 3) { _ =>
+                withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vec.toString) {
+                  spark
+                    .read
+                    .format("orc")
+                    .load(getPath(path, dateTime, after1582))
+                    .noop()
+                }
               }
             }
           }
-
           benchmark2.run()
         }
       }

From 4d5a1984bf393e15b5d6e034b38ae6167bff9921 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 1 Apr 2020 16:43:32 +0900
Subject: [PATCH 0396/1280] [SPARK-31320] Fix release script for 3.0.0

### What changes were proposed in this pull request?

The release script stops working after https://github.com/apache/spark/commit/d5865493ae71e6369e9f3350dd7e694afcf57298 , as we require `mkdocs 1.0.0`.

This PR upgrades `mkdocs` from 0.1.6.3 to 1.0.4. To do that ruby is also upgraded to 2.5.

This PR also fixes some small issues.

### Why are the changes needed?

to make RC

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

tested by 3.0.0-rc1

Closes #28088 from cloud-fan/rc.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/create-release/do-release-docker.sh |  2 +-
 dev/create-release/release-util.sh      | 10 +++++++---
 dev/create-release/spark-rm/Dockerfile  | 13 +++++--------
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index 694a87bf78084..4a003d7e07de2 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -93,7 +93,7 @@ done
 
 GPG_KEY_FILE="$WORKDIR/gpg.key"
 fcreate_secure "$GPG_KEY_FILE"
-$GPG --export-secret-key --armor "$GPG_KEY" > "$GPG_KEY_FILE"
+$GPG --export-secret-key --armor --pinentry-mode loopback --passphrase "$GPG_PASSPHRASE" "$GPG_KEY" > "$GPG_KEY_FILE"
 
 run_silent "Building spark-rm image with tag $IMGTAG..." "docker-build.log" \
   docker build -t "spark-rm:$IMGTAG" --build-arg UID=$UID "$SELF/spark-rm"
diff --git a/dev/create-release/release-util.sh b/dev/create-release/release-util.sh
index 1282e15e79913..8ee94a67f34fd 100755
--- a/dev/create-release/release-util.sh
+++ b/dev/create-release/release-util.sh
@@ -159,10 +159,14 @@ function get_release_info {
   export SPARK_PACKAGE_VERSION="$RELEASE_TAG"
 
   # Gather some user information.
-  export ASF_USERNAME=$(read_config "ASF user" "$LOGNAME")
+  if [ -z "$ASF_USERNAME" ]; then
+    export ASF_USERNAME=$(read_config "ASF user" "$LOGNAME")
+  fi
 
-  GIT_NAME=$(git config user.name || echo "")
-  export GIT_NAME=$(read_config "Full name" "$GIT_NAME")
+  if [ -z "$GIT_NAME" ]; then
+    GIT_NAME=$(git config user.name || echo "")
+    export GIT_NAME=$(read_config "Full name" "$GIT_NAME")
+  fi
 
   export GIT_EMAIL="$ASF_USERNAME@apache.org"
   export GPG_KEY=$(read_config "GPG key" "$GIT_EMAIL")
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 63451687ee8c2..540dc90f42817 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -33,8 +33,8 @@ ENV DEBCONF_NONINTERACTIVE_SEEN true
 # These arguments are just for reuse and not really meant to be customized.
 ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 
-ARG BASE_PIP_PKGS="setuptools wheel"
-ARG PIP_PKGS="pyopenssl numpy sphinx"
+ARG PIP_PKGS="sphinx==2.3.1 mkdocs==1.0.4 numpy==1.18.1"
+ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 
 # Install extra needed repos and refresh.
 # - CRAN repo
@@ -62,12 +62,11 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   curl -sL https://deb.nodesource.com/setup_11.x | bash && \
   $APT_INSTALL nodejs && \
   # Install needed python packages. Use pip for installing packages (for consistency).
-  $APT_INSTALL libpython3-dev python3-pip && \
+  $APT_INSTALL libpython3-dev python3-pip python3-setuptools && \
   # Change default python version to python3.
   update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && \
   update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 && \
   update-alternatives --set python /usr/bin/python3.6 && \
-  pip3 install $BASE_PIP_PKGS && \
   pip3 install $PIP_PKGS && \
   # Install R packages and dependencies used when building.
   # R depends on pandoc*, libssl (which are installed above).
@@ -76,10 +75,8 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" && \
   Rscript -e "devtools::install_github('jimhester/lintr')" && \
   # Install tools needed to build the documentation.
-  $APT_INSTALL ruby2.3 ruby2.3-dev mkdocs && \
-  gem install jekyll --no-rdoc --no-ri -v 3.8.6 && \
-  gem install jekyll-redirect-from -v 0.15.0 && \
-  gem install rouge
+  $APT_INSTALL ruby2.5 ruby2.5-dev && \
+  gem install --no-document $GEM_PKGS
 
 WORKDIR /opt/spark-rm/output
 

From 08a88e04f4417c5a0168592d1f01c300b4ae9de5 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 1 Apr 2020 03:42:26 +0000
Subject: [PATCH 0397/1280] [SPARK-31313][K8S][TEST] Add `m01` node name to
 support Minikube 1.8.x

### What changes were proposed in this pull request?

This PR aims to add `m01` as a node name additionally to `PVTestsSuite`.

### Why are the changes needed?

minikube 1.8.0 ~ 1.8.2 generate a cluster with a nodename `m01` while all the other versions have `minikube`. This causes `PVTestSuite` failure.
```
$ minikube --vm-driver=hyperkit start --memory 6000 --cpus 8
* minikube v1.8.2 on Darwin 10.15.3
  - MINIKUBE_ACTIVE_DOCKERD=minikube
* Using the hyperkit driver based on user configuration
* Creating hyperkit VM (CPUs=8, Memory=6000MB, Disk=20000MB) ...
* Preparing Kubernetes v1.18.0 on Docker 19.03.6 ...
* Launching Kubernetes ...
* Enabling addons: default-storageclass, storage-provisioner
* Waiting for cluster to come online ...
* Done! kubectl is now configured to use "minikube"

$ kubectl get nodes
NAME   STATUS   ROLES    AGE   VERSION
m01    Ready    master   22s   v1.17.3
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

This only adds a new node name. So, K8S Jenkins job should passed.
In addition, `K8s` integration test suite should be tested on `minikube 1.8.2` manually.

```
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- All pods have the same service account by default
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
- Start pod creation from template
- PVs with local storage
- Launcher client dependencies
- Test basic decommissioning
- Run SparkR on simple dataframe.R example
Run completed in 10 minutes, 23 seconds.
Total number of tests run: 20
Suites: completed 2, aborted 0
Tests: succeeded 20, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```

For the above test, Minikube 1.8.2 and K8s v1.18.0 is used.
```
$ minikube version
minikube version: v1.8.2
commit: eb13446e786c9ef70cb0a9f85a633194e62396a1

$ kubectl version --short
Client Version: v1.18.0
Server Version: v1.18.0
```

Closes #28080 from dongjoon-hyun/SPARK-31313.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit dba525c997b0033ac1b6fd24236cd72938f94bbf)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index f0218217e6afb..a7cb84eb4b371 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -56,7 +56,8 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
                 .withMatchExpressions(new NodeSelectorRequirementBuilder()
                   .withKey("kubernetes.io/hostname")
                   .withOperator("In")
-                  .withValues("minikube", "docker-for-desktop", "docker-desktop").build()).build())
+                  .withValues("minikube", "m01", "docker-for-desktop", "docker-desktop")
+                  .build()).build())
             .endRequired()
           .endNodeAffinity()
       .endSpec()

From 5a7395d296b57c8e06bbf100b4ebdaa1100c3a91 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 1 Apr 2020 12:03:41 -0700
Subject: [PATCH 0398/1280] [SPARK-31073][DOC][FOLLOWUP] Add description for
 Shuffle Write Time metric in StagePage to web-ui.md

### What changes were proposed in this pull request?

This PR adds description for `Shuffle Write Time` to `web-ui.md`.

### Why are the changes needed?

#27837 added `Shuffle Write Time` metric to task metrics summary but it's not documented yet.

### Does this PR introduce any user-facing change?

Yes.
We can see the description for `Shuffle Write Time` in the new `web-ui.html`.
<img width="956" alt="shuffle-write-time-description" src="https://user-images.githubusercontent.com/4736016/78175342-a9722280-7495-11ea-9cc6-62c6f3619aa3.png">

### How was this patch tested?

Built docs by `SKIP_API=1 jekyll build` in `doc` directory and then confirmed `web-ui.html`.

Closes #28093 from sarutak/SPARK-31073-doc.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b9b1b549af4dd57767fc0c23f5fe7695ca9fa9d5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/web-ui.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/web-ui.md b/docs/web-ui.md
index e28a689c8de50..c53af804d8d59 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -143,6 +143,7 @@ Summary metrics for all task are represented in a table and in a timeline.
 * **Shuffle Read Size / Records**. Total shuffle bytes read, includes both data read locally and data read from remote executors.
 * **Shuffle Read Blocked Time** is the time that tasks spent blocked waiting for shuffle data to be read from remote machines.
 * **Shuffle Remote Reads** is the total shuffle bytes read from remote executors.
+* **Shuffle Write Time** is the time that tasks spent writing shuffle data.
 * **Shuffle spill (memory)** is the size of the deserialized form of the shuffled data in memory.
 * **Shuffle spill (disk)** is the size of the serialized form of the data on disk.
 

From 0908f8e6559ca7005f854b4398adb078955c5315 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 1 Apr 2020 12:04:40 -0700
Subject: [PATCH 0399/1280] [SPARK-31322][SQL] rename
 QueryPlan.collectInPlanAndSubqueries to collectWithSubqueries

### What changes were proposed in this pull request?

rename `QueryPlan.collectInPlanAndSubqueries` to `collectWithSubqueries`

### Why are the changes needed?

The old name is too verbose. `QueryPlan` is internal but it's the core of catalyst and we'd better make the API name clearer before we release it.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

N/A

Closes #28092 from cloud-fan/rename.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 09f036a14cee4825edc73b463e1eebe85ff1c915)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala | 4 ++--
 .../org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala  | 2 +-
 .../org/apache/spark/sql/execution/CollectMetricsExec.scala   | 2 +-
 .../org/apache/spark/sql/DynamicPartitionPruningSuite.scala   | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 9f86fb298877a..13e5b129765e2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -232,10 +232,10 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   /**
-   * Returns a sequence containing the result of applying a partial function to all elements in this
+   * A variant of `collect`. This method not only apply the given function to all elements in this
    * plan, also considering all the plans in its (nested) subqueries
    */
-  def collectInPlanAndSubqueries[B](f: PartialFunction[PlanType, B]): Seq[B] =
+  def collectWithSubqueries[B](f: PartialFunction[PlanType, B]): Seq[B] =
     (this +: subqueriesAll).flatMap(_.collect(f))
 
   override def innerChildren: Seq[QueryPlan[_]] = subqueries
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
index d96f8086a3e93..91ce187f4d270 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
@@ -78,7 +78,7 @@ class QueryPlanSuite extends SparkFunSuite {
 
     val countRelationsInPlan = plan.collect({ case _: UnresolvedRelation => 1 }).sum
     val countRelationsInPlanAndSubqueries =
-      plan.collectInPlanAndSubqueries({ case _: UnresolvedRelation => 1 }).sum
+      plan.collectWithSubqueries({ case _: UnresolvedRelation => 1 }).sum
 
     assert(countRelationsInPlan == 2)
     assert(countRelationsInPlanAndSubqueries == 5)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
index e482bc9941ea9..e1b9c8f430c56 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CollectMetricsExec.scala
@@ -87,7 +87,7 @@ object CollectMetricsExec {
    * Recursively collect all collected metrics from a query tree.
    */
   def collect(plan: SparkPlan): Map[String, Row] = {
-    val metrics = plan.collectInPlanAndSubqueries {
+    val metrics = plan.collectWithSubqueries {
       case collector: CollectMetricsExec => collector.name -> collector.collectedMetrics
     }
     metrics.toMap
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index baa9f5ecafc68..cdf9ea4b31ee7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -1234,7 +1234,7 @@ abstract class DynamicPartitionPruningSuiteBase
 
       val plan = df.queryExecution.executedPlan
       val countSubqueryBroadcasts =
-        plan.collectInPlanAndSubqueries({ case _: SubqueryBroadcastExec => 1 }).sum
+        plan.collectWithSubqueries({ case _: SubqueryBroadcastExec => 1 }).sum
 
       assert(countSubqueryBroadcasts == 2)
     }

From a852dbc3adbf1017a5b673b16f8b74704858cb26 Mon Sep 17 00:00:00 2001
From: Mukul Murthy <mukul.murthy@gmail.com>
Date: Thu, 2 Apr 2020 12:37:58 +0900
Subject: [PATCH 0400/1280] [SPARK-31324][SS] Include stream ID in the
 termination timeout error message

### What changes were proposed in this pull request?

This PR (SPARK-31324) aims to include stream ID in the error thrown when a stream does not stop() in time.

### Why are the changes needed?

https://github.com/apache/spark/pull/26771/ added a conf to set a requested timeout for stopping a stream, after which the stop() method throws. From seeing this in a production use case with several streams running, it's helpful to include which stream failed to stop in the error message.

### Does this PR introduce any user-facing change?

If a stream times out when terminating, the error message now includes the stream ID.

Before:
`Stream Execution thread failed to stop within 2000 milliseconds (specified by spark.sql.streaming.stopTimeout). See the cause on what was being executed in the streaming query thread.`

After:
`Stream Execution thread for stream [id = 8513769d-b9d2-4902-9b36-3668bd022245, runId = 21ed8c35-9bfe-423f-853d-c022d91818bc] failed to stop within 2000 milliseconds (specified by spark.sql.streaming.stopTimeout). See the cause on what was being executed in the streaming query thread.`

### How was this patch tested?

Updated existing unit test

Closes #28095 from mukulmurthy/31324-id.

Authored-by: Mukul Murthy <mukul.murthy@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 34abbb677dd3867e23279ff45d411ef3010adbe7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/streaming/StreamExecution.scala     | 6 +++---
 .../scala/org/apache/spark/sql/streaming/StreamSuite.scala  | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index 8006437b31297..9b1951a834d9a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -451,9 +451,9 @@ abstract class StreamExecution(
       val stackTraceException = new SparkException("The stream thread was last executing:")
       stackTraceException.setStackTrace(queryExecutionThread.getStackTrace)
       val timeoutException = new TimeoutException(
-        s"Stream Execution thread failed to stop within $timeout milliseconds (specified by " +
-        s"${SQLConf.STREAMING_STOP_TIMEOUT.key}). See the cause on what was " +
-        "being executed in the streaming query thread.")
+        s"Stream Execution thread for stream $prettyIdString failed to stop within $timeout " +
+        s"milliseconds (specified by ${SQLConf.STREAMING_STOP_TIMEOUT.key}). See the cause on " +
+        s"what was being executed in the streaming query thread.")
       timeoutException.initCause(stackTraceException)
       throw timeoutException
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
index b6618826487c6..5bcc4e90a2043 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
@@ -1244,9 +1244,10 @@ class StreamSuite extends StreamTest {
       failAfter(60.seconds) {
         val startTime = System.nanoTime()
         withSQLConf(SQLConf.STREAMING_STOP_TIMEOUT.key -> "2000") {
-          intercept[TimeoutException] {
+          val ex = intercept[TimeoutException] {
             sq.stop()
           }
+          assert(ex.getMessage.contains(sq.id.toString))
         }
         val duration = (System.nanoTime() - startTime) / 1e6
         assert(duration >= 2000,

From 2386a15344e43166e440753fbad5490c8aee6473 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 2 Apr 2020 08:34:36 +0000
Subject: [PATCH 0401/1280] [SPARK-31321][SQL] Remove SaveMode check in v2
 FileWriteBuilder

### What changes were proposed in this pull request?

The `SaveMode` is resolved before we create `FileWriteBuilder` to build `BatchWrite`.

In https://github.com/apache/spark/pull/25876, we removed save mode for DSV2 from DataFrameWriter. So that the `mode` method is never used which makes `validateInputs` fail determinately without `mode` set.

### Why are the changes needed?
rm dead code.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

existing tests.

Closes #28090 from yaooqinn/SPARK-31321.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1ce584f6b78a109d63d1a5ea8b721422f9d8d6e0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/v2/FileWriteBuilder.scala     | 32 ++-----------------
 1 file changed, 3 insertions(+), 29 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
index d519832c57501..cd62ee7814bf2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileWriteBuilder.scala
@@ -16,7 +16,6 @@
  */
 package org.apache.spark.sql.execution.datasources.v2
 
-import java.io.IOException
 import java.util.UUID
 
 import scala.collection.JavaConverters._
@@ -27,7 +26,7 @@ import org.apache.hadoop.mapreduce.Job
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
 
 import org.apache.spark.internal.io.FileCommitProtocol
-import org.apache.spark.sql.{AnalysisException, SaveMode, SparkSession}
+import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
 import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, WriteBuilder}
@@ -46,12 +45,6 @@ abstract class FileWriteBuilder(
   private val schema = info.schema()
   private val queryId = info.queryId()
   private val options = info.options()
-  private var mode: SaveMode = _
-
-  def mode(mode: SaveMode): WriteBuilder = {
-    this.mode = mode
-    this
-  }
 
   override def buildForBatch(): BatchWrite = {
     val sparkSession = SparkSession.active
@@ -68,26 +61,8 @@ abstract class FileWriteBuilder(
     lazy val description =
       createWriteJobDescription(sparkSession, hadoopConf, job, paths.head, options.asScala.toMap)
 
-    val fs = path.getFileSystem(hadoopConf)
-    mode match {
-      case SaveMode.ErrorIfExists if fs.exists(path) =>
-        val qualifiedOutputPath = path.makeQualified(fs.getUri, fs.getWorkingDirectory)
-        throw new AnalysisException(s"path $qualifiedOutputPath already exists.")
-
-      case SaveMode.Ignore if fs.exists(path) =>
-        null
-
-      case SaveMode.Overwrite =>
-        if (fs.exists(path) && !committer.deleteWithJob(fs, path, true)) {
-          throw new IOException(s"Unable to clear directory $path prior to writing to it")
-        }
-        committer.setupJob(job)
-        new FileBatchWrite(job, description, committer)
-
-      case _ =>
-        committer.setupJob(job)
-        new FileBatchWrite(job, description, committer)
-    }
+    committer.setupJob(job)
+    new FileBatchWrite(job, description, committer)
   }
 
   /**
@@ -104,7 +79,6 @@ abstract class FileWriteBuilder(
   private def validateInputs(caseSensitiveAnalysis: Boolean): Unit = {
     assert(schema != null, "Missing input data schema")
     assert(queryId != null, "Missing query ID")
-    assert(mode != null, "Missing save mode")
 
     if (paths.length != 1) {
       throw new IllegalArgumentException("Expected exactly one path to be specified, but " +

From 22496535c96a6d210640d240f338840e9f655c49 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Fri, 3 Apr 2020 04:35:31 +0000
Subject: [PATCH 0402/1280] [SPARK-31328][SQL] Fix rebasing of overlapped local
 timestamps during daylight saving time

### What changes were proposed in this pull request?
1. Fix the `rebaseGregorianToJulianMicros()` function in `DateTimeUtils` by passing the daylight saving offset associated with the input `micros` to the constructed instance of `GregorianCalendar`. The problem is in `cal.getTimeInMillis` which returns earliest instant in the case of local date-time overlaps, see https://github.com/AdoptOpenJDK/openjdk-jdk8u/blob/master/jdk/src/share/classes/java/util/GregorianCalendar.java#L2783-L2786 . I fixed the issue by keeping the standard zone offset as is, and set the DST offset only. I don't set `ZONE_OFFSET` because time zone resolution works differently in Java 8 and Java 7 time APIs. So, if I would set the standard zone offsets too, this could change the behavior, and rebasing won't give the same result as Spark 2.4.
2. Fix `rebaseJulianToGregorianMicros()` by changing resulted zoned date-time if `DST_OFFSET` is zero which means the input date-time has passed an autumn daylight savings cutover. So, I take the latest local timestamp out of 2 overlapped timestamps. Otherwise I return a zoned date-time w/o any modification because it is equal to calling the `withEarlierOffsetAtOverlap()` method, so, we can optimize the case.

### Why are the changes needed?
This fixes the bug of loosing of DST offset info in rebasing timestamps via local date-time. For example, there are 2 different timestamps in the `America/Los_Angeles` time zone: `2019-11-03T01:00:00-07:00` and `2019-11-03T01:00:00-08:00`, though they are mapped to the same local date-time `2019-11-03T01:00`, see
<img width="456" alt="Screen Shot 2020-04-02 at 10 19 24" src="https://user-images.githubusercontent.com/1580697/78245697-95a7da00-74f0-11ea-9eba-c08138851cb3.png">
Currently, the UTC timestamp `2019-11-03T09:00:00Z` is converted to `2019-11-03T01:00:00-08:00`, and then to `2019-11-03T01:00:00` (in the original calendar, for instance Proleptic Gregorian calendar) and back to the UTC timestamp `2019-11-03T08:00:00Z` (in the hybrid calendar - Gregorian for the timestamp). That's wrong because the local timestamp must be converted to the original timestamp `2019-11-03T09:00:00Z`.

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
- Added a test to `DateTimeUtilsSuite` which checks that rebased micros are the same as the input during DST. The result must be the same if Java 8 and 7 time API functions return the same time zone offsets.
- Run the following code to check that there is no difference between rebased and original micros for modern timestamps:
```scala
    test("rebasing differences") {
      withDefaultTimeZone(getZoneId("America/Los_Angeles")) {
        val start = instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
          .atZone(getZoneId("America/Los_Angeles"))
          .toInstant)
        val end = instantToMicros(LocalDateTime.of(2030, 1, 1, 0, 0, 0)
          .atZone(getZoneId("America/Los_Angeles"))
          .toInstant)

        var micros = start
        var diff = Long.MaxValue
        var counter = 0
        while (micros < end) {
          val rebased = rebaseGregorianToJulianMicros(micros)
          val curDiff = rebased - micros
          if (curDiff != diff) {
            counter += 1
            diff = curDiff
            val ldt = microsToInstant(micros).atZone(getZoneId("America/Los_Angeles")).toLocalDateTime
            println(s"local date-time = $ldt diff = ${diff / MICROS_PER_MINUTE} minutes")
          }
          micros += 30 * MICROS_PER_MINUTE
        }
        println(s"counter = $counter")
      }
    }
```
```
local date-time = 0001-01-01T00:00 diff = -2872 minutes
local date-time = 0100-03-01T00:00 diff = -1432 minutes
local date-time = 0200-03-01T00:00 diff = 7 minutes
local date-time = 0300-03-01T00:00 diff = 1447 minutes
local date-time = 0500-03-01T00:00 diff = 2887 minutes
local date-time = 0600-03-01T00:00 diff = 4327 minutes
local date-time = 0700-03-01T00:00 diff = 5767 minutes
local date-time = 0900-03-01T00:00 diff = 7207 minutes
local date-time = 1000-03-01T00:00 diff = 8647 minutes
local date-time = 1100-03-01T00:00 diff = 10087 minutes
local date-time = 1300-03-01T00:00 diff = 11527 minutes
local date-time = 1400-03-01T00:00 diff = 12967 minutes
local date-time = 1500-03-01T00:00 diff = 14407 minutes
local date-time = 1582-10-15T00:00 diff = 7 minutes
local date-time = 1883-11-18T12:22:58 diff = 0 minutes
counter = 15
```
The code is not added to `DateTimeUtilsSuite` because it takes > 30 seconds.
- By running the updated benchmark `DateTimeRebaseBenchmark` via the command:
```
SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.DateTimeRebaseBenchmark"
```
in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 1.8.0_242-8u242/11.0.6+10 |

Closes #28101 from MaxGekk/fix-local-date-overlap.

Lead-authored-by: Maxim Gekk <max.gekk@gmail.com>
Co-authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 820bb9985a76a567b79ffb01bbd2d32788e1dba0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     | 18 +++-
 .../catalyst/util/DateTimeUtilsSuite.scala    | 16 ++++
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 88 +++++++++----------
 .../DateTimeRebaseBenchmark-results.txt       | 88 +++++++++----------
 4 files changed, 120 insertions(+), 90 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 44cabe215b6e1..3512e3b5bfa65 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -997,13 +997,18 @@ object DateTimeUtils {
    * @return The rebased microseconds since the epoch in Julian calendar.
    */
   def rebaseGregorianToJulianMicros(micros: Long): Long = {
-    val ldt = microsToInstant(micros).atZone(ZoneId.systemDefault).toLocalDateTime
+    val instant = microsToInstant(micros)
+    val zoneId = ZoneId.systemDefault
+    val ldt = instant.atZone(zoneId).toLocalDateTime
     val cal = new Calendar.Builder()
       // `gregory` is a hybrid calendar that supports both
       // the Julian and Gregorian calendar systems
       .setCalendarType("gregory")
       .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
       .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
+      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
+      // This setting selects the original local timestamp mapped to the given `micros`.
+      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
       .build()
     fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
   }
@@ -1036,7 +1041,16 @@ object DateTimeUtils {
       cal.get(Calendar.SECOND),
       (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
       .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
-    instantToMicros(localDateTime.atZone(ZoneId.systemDefault).toInstant)
+    val zonedDateTime = localDateTime.atZone(ZoneId.systemDefault)
+    // Zero DST offset means that local clocks have switched to the winter time already.
+    // So, clocks go back one hour. We should correct zoned date-time and change
+    // the zone offset to the later of the two valid offsets at a local time-line overlap.
+    val adjustedZdt = if (cal.get(Calendar.DST_OFFSET) == 0) {
+      zonedDateTime.withLaterOffsetAtOverlap()
+    } else {
+      zonedDateTime
+    }
+    instantToMicros(adjustedZdt.toInstant)
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 652abe7659400..f9c15f394f5ca 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -821,4 +821,20 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       days += 1
     }
   }
+
+  test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") {
+    Seq(
+      LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"),
+      "Europe/Amsterdam" ->
+        Seq("2019-10-27T00:00:00Z", "2019-10-27T00:30:00Z", "2019-10-27T01:00:00Z")
+    ).foreach { case (tz, ts) =>
+      withDefaultTimeZone(getZoneId(tz)) {
+        ts.foreach { str =>
+          val micros = instantToMicros(Instant.parse(str))
+          assert(rebaseGregorianToJulianMicros(micros) === micros)
+          assert(rebaseJulianToGregorianMicros(micros) === micros)
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 01b063993cd9c..570f7beda6b4e 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9299           9299           0         10.8          93.0       1.0X
-before 1582, noop                                  9220           9220           0         10.8          92.2       1.0X
-after 1582, rebase off                            20390          20390           0          4.9         203.9       0.5X
-after 1582, rebase on                             20378          20378           0          4.9         203.8       0.5X
-before 1582, rebase off                           20069          20069           0          5.0         200.7       0.5X
-before 1582, rebase on                            20637          20637           0          4.8         206.4       0.5X
+after 1582, noop                                   9392           9392           0         10.6          93.9       1.0X
+before 1582, noop                                  9324           9324           0         10.7          93.2       1.0X
+after 1582, rebase off                            20975          20975           0          4.8         209.7       0.4X
+after 1582, rebase on                             20016          20016           0          5.0         200.2       0.5X
+before 1582, rebase off                           20088          20088           0          5.0         200.9       0.5X
+before 1582, rebase on                            20310          20310           0          4.9         203.1       0.5X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12927          13017          78          7.7         129.3       1.0X
-after 1582, vec off, rebase on                    13127          13176          50          7.6         131.3       1.0X
-after 1582, vec on, rebase off                     3725           3779          91         26.8          37.3       3.5X
-after 1582, vec on, rebase on                      5134           5221          99         19.5          51.3       2.5X
-before 1582, vec off, rebase off                  13049          13061          16          7.7         130.5       1.0X
-before 1582, vec off, rebase on                   13877          13916          51          7.2         138.8       0.9X
-before 1582, vec on, rebase off                    3702           3736          56         27.0          37.0       3.5X
-before 1582, vec on, rebase on                     5567           5637          78         18.0          55.7       2.3X
+after 1582, vec off, rebase off                   13371          13463         154          7.5         133.7       1.0X
+after 1582, vec off, rebase on                    13482          13533          57          7.4         134.8       1.0X
+after 1582, vec on, rebase off                     3713           3781          96         26.9          37.1       3.6X
+after 1582, vec on, rebase on                      5153           5173          29         19.4          51.5       2.6X
+before 1582, vec off, rebase off                  12939          12998          97          7.7         129.4       1.0X
+before 1582, vec off, rebase on                   14160          14255          85          7.1         141.6       0.9X
+before 1582, vec on, rebase off                    3748           3776          28         26.7          37.5       3.6X
+before 1582, vec on, rebase on                     5532           5575          54         18.1          55.3       2.4X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2988           2988           0         33.5          29.9       1.0X
-before 1582, noop                                  3000           3000           0         33.3          30.0       1.0X
-after 1582, rebase off                            16163          16163           0          6.2         161.6       0.2X
-after 1582, rebase on                             68399          68399           0          1.5         684.0       0.0X
-before 1582, rebase off                           16921          16921           0          5.9         169.2       0.2X
-before 1582, rebase on                            74425          74425           0          1.3         744.3       0.0X
+after 1582, noop                                   2795           2795           0         35.8          27.9       1.0X
+before 1582, noop                                  2806           2806           0         35.6          28.1       1.0X
+after 1582, rebase off                            16113          16113           0          6.2         161.1       0.2X
+after 1582, rebase on                             70198          70198           0          1.4         702.0       0.0X
+before 1582, rebase off                           16690          16690           0          6.0         166.9       0.2X
+before 1582, rebase on                            75706          75706           0          1.3         757.1       0.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15147          15258          97          6.6         151.5       1.0X
-after 1582, vec off, rebase on                    45035          45101          60          2.2         450.3       0.3X
-after 1582, vec on, rebase off                     4934           5012         100         20.3          49.3       3.1X
-after 1582, vec on, rebase on                     34263          34360          88          2.9         342.6       0.4X
-before 1582, vec off, rebase off                  15177          15220          37          6.6         151.8       1.0X
-before 1582, vec off, rebase on                   46754          46761          12          2.1         467.5       0.3X
-before 1582, vec on, rebase off                    4892           4956          61         20.4          48.9       3.1X
-before 1582, vec on, rebase on                    35989          36014          22          2.8         359.9       0.4X
+after 1582, vec off, rebase off                   15631          15753         111          6.4         156.3       1.0X
+after 1582, vec off, rebase on                    45834          46027         193          2.2         458.3       0.3X
+after 1582, vec on, rebase off                     4883           4964          70         20.5          48.8       3.2X
+after 1582, vec on, rebase on                     34514          34563          63          2.9         345.1       0.5X
+before 1582, vec off, rebase off                  15253          15354         104          6.6         152.5       1.0X
+before 1582, vec off, rebase on                   47353          47412          59          2.1         473.5       0.3X
+before 1582, vec on, rebase off                    4848           4894          69         20.6          48.5       3.2X
+before 1582, vec on, rebase on                    36125          36143          22          2.8         361.3       0.4X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9295           9295           0         10.8          93.0       1.0X
-before 1582, noop                                  9352           9352           0         10.7          93.5       1.0X
-after 1582                                        17112          17112           0          5.8         171.1       0.5X
-before 1582                                       17979          17979           0          5.6         179.8       0.5X
+after 1582, noop                                   9160           9160           0         10.9          91.6       1.0X
+before 1582, noop                                  9235           9235           0         10.8          92.4       1.0X
+after 1582                                        17154          17154           0          5.8         171.5       0.5X
+before 1582                                       17545          17545           0          5.7         175.5       0.5X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               20874          20905          38          4.8         208.7       1.0X
-after 1582, vec on                                 3813           3844          28         26.2          38.1       5.5X
-before 1582, vec off                              25912          25949          38          3.9         259.1       0.8X
-before 1582, vec on                                4322           4343          19         23.1          43.2       4.8X
+after 1582, vec off                               21024          21146         205          4.8         210.2       1.0X
+after 1582, vec on                                 3814           3838          21         26.2          38.1       5.5X
+before 1582, vec off                              24293          24347          82          4.1         242.9       0.9X
+before 1582, vec on                                4143           4168          22         24.1          41.4       5.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   3003           3003           0         33.3          30.0       1.0X
-before 1582, noop                                  3012           3012           0         33.2          30.1       1.0X
-after 1582                                        41031          41031           0          2.4         410.3       0.1X
-before 1582                                       44436          44436           0          2.3         444.4       0.1X
+after 1582, noop                                   2797           2797           0         35.8          28.0       1.0X
+before 1582, noop                                  2826           2826           0         35.4          28.3       1.0X
+after 1582                                        40021          40021           0          2.5         400.2       0.1X
+before 1582                                       41500          41500           0          2.4         415.0       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               28477          28582          92          3.5         284.8       1.0X
-after 1582, vec on                                20754          20924         237          4.8         207.5       1.4X
-before 1582, vec off                              32858          32921          58          3.0         328.6       0.9X
-before 1582, vec on                               25734          25769          30          3.9         257.3       1.1X
+after 1582, vec off                               32517          32541          23          3.1         325.2       1.0X
+after 1582, vec on                                19644          19725         128          5.1         196.4       1.7X
+before 1582, vec off                              37204          37305         104          2.7         372.0       0.9X
+before 1582, vec on                               24105          24120          13          4.1         241.1       1.3X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index b35301349a1ee..2f8712c859bc9 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9691           9691           0         10.3          96.9       1.0X
-before 1582, noop                                  9024           9024           0         11.1          90.2       1.1X
-after 1582, rebase off                            21195          21195           0          4.7         211.9       0.5X
-after 1582, rebase on                             20045          20045           0          5.0         200.4       0.5X
-before 1582, rebase off                           20039          20039           0          5.0         200.4       0.5X
-before 1582, rebase on                            20451          20451           0          4.9         204.5       0.5X
+after 1582, noop                                   9488           9488           0         10.5          94.9       1.0X
+before 1582, noop                                  9301           9301           0         10.8          93.0       1.0X
+after 1582, rebase off                            20109          20109           0          5.0         201.1       0.5X
+after 1582, rebase on                             20004          20004           0          5.0         200.0       0.5X
+before 1582, rebase off                           19906          19906           0          5.0         199.1       0.5X
+before 1582, rebase on                            20466          20466           0          4.9         204.7       0.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13207          13339         116          7.6         132.1       1.0X
-after 1582, vec off, rebase on                    13408          13446          57          7.5         134.1       1.0X
-after 1582, vec on, rebase off                     3680           3712          39         27.2          36.8       3.6X
-after 1582, vec on, rebase on                      5229           5261          29         19.1          52.3       2.5X
-before 1582, vec off, rebase off                  13135          13164          25          7.6         131.4       1.0X
-before 1582, vec off, rebase on                   13946          14033          94          7.2         139.5       0.9X
-before 1582, vec on, rebase off                    3689           3726          49         27.1          36.9       3.6X
-before 1582, vec on, rebase on                     5679           5687           9         17.6          56.8       2.3X
+after 1582, vec off, rebase off                   12593          12653          52          7.9         125.9       1.0X
+after 1582, vec off, rebase on                    13350          13489         121          7.5         133.5       0.9X
+after 1582, vec on, rebase off                     3665           3681          25         27.3          36.6       3.4X
+after 1582, vec on, rebase on                      5193           5210          16         19.3          51.9       2.4X
+before 1582, vec off, rebase off                  13023          13059          32          7.7         130.2       1.0X
+before 1582, vec off, rebase on                   13855          13937         115          7.2         138.6       0.9X
+before 1582, vec on, rebase off                    3651           3665          12         27.4          36.5       3.4X
+before 1582, vec on, rebase on                     5623           5671          45         17.8          56.2       2.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2720           2720           0         36.8          27.2       1.0X
-before 1582, noop                                  2712           2712           0         36.9          27.1       1.0X
-after 1582, rebase off                            16626          16626           0          6.0         166.3       0.2X
-after 1582, rebase on                             85136          85136           0          1.2         851.4       0.0X
-before 1582, rebase off                           16855          16855           0          5.9         168.6       0.2X
-before 1582, rebase on                           106121         106121           0          0.9        1061.2       0.0X
+after 1582, noop                                   2798           2798           0         35.7          28.0       1.0X
+before 1582, noop                                  2955           2955           0         33.8          29.6       0.9X
+after 1582, rebase off                            15889          15889           0          6.3         158.9       0.2X
+after 1582, rebase on                             84247          84247           0          1.2         842.5       0.0X
+before 1582, rebase off                           16134          16134           0          6.2         161.3       0.2X
+before 1582, rebase on                           100006         100006           0          1.0        1000.1       0.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15198          15301          90          6.6         152.0       1.0X
-after 1582, vec off, rebase on                    55210          55370         140          1.8         552.1       0.3X
-after 1582, vec on, rebase off                     4859           4880          19         20.6          48.6       3.1X
-after 1582, vec on, rebase on                     44758          44824          85          2.2         447.6       0.3X
-before 1582, vec off, rebase off                  15206          15316         112          6.6         152.1       1.0X
-before 1582, vec off, rebase on                   60452          60588         222          1.7         604.5       0.3X
-before 1582, vec on, rebase off                    4892           4933          36         20.4          48.9       3.1X
-before 1582, vec on, rebase on                    46871          46950          82          2.1         468.7       0.3X
+after 1582, vec off, rebase off                   14920          15045         116          6.7         149.2       1.0X
+after 1582, vec off, rebase on                    55062          55171         140          1.8         550.6       0.3X
+after 1582, vec on, rebase off                     4871           4952          72         20.5          48.7       3.1X
+after 1582, vec on, rebase on                     44955          44981          23          2.2         449.5       0.3X
+before 1582, vec off, rebase off                  15236          15386         142          6.6         152.4       1.0X
+before 1582, vec off, rebase on                   57290          57368          79          1.7         572.9       0.3X
+before 1582, vec on, rebase off                    4919           4930          15         20.3          49.2       3.0X
+before 1582, vec on, rebase on                    47351          47713         400          2.1         473.5       0.3X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9102           9102           0         11.0          91.0       1.0X
-before 1582, noop                                  9099           9099           0         11.0          91.0       1.0X
-after 1582                                        17652          17652           0          5.7         176.5       0.5X
-before 1582                                       18284          18284           0          5.5         182.8       0.5X
+after 1582, noop                                   9451           9451           0         10.6          94.5       1.0X
+before 1582, noop                                  9765           9765           0         10.2          97.7       1.0X
+after 1582                                        18722          18722           0          5.3         187.2       0.5X
+before 1582                                       18864          18864           0          5.3         188.6       0.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               25169          25215          48          4.0         251.7       1.0X
-after 1582, vec on                                 3701           3717          16         27.0          37.0       6.8X
-before 1582, vec off                              26919          27045         182          3.7         269.2       0.9X
-before 1582, vec on                                4169           4192          31         24.0          41.7       6.0X
+after 1582, vec off                               24897          25095         247          4.0         249.0       1.0X
+after 1582, vec on                                 3719           3780          84         26.9          37.2       6.7X
+before 1582, vec off                              31290          31347          50          3.2         312.9       0.8X
+before 1582, vec on                                4166           4188          25         24.0          41.7       6.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2906           2906           0         34.4          29.1       1.0X
-before 1582, noop                                  2863           2863           0         34.9          28.6       1.0X
-after 1582                                        48858          48858           0          2.0         488.6       0.1X
-before 1582                                       50945          50945           0          2.0         509.5       0.1X
+after 1582, noop                                   2882           2882           0         34.7          28.8       1.0X
+before 1582, noop                                  2991           2991           0         33.4          29.9       1.0X
+after 1582                                        53951          53951           0          1.9         539.5       0.1X
+before 1582                                       54276          54276           0          1.8         542.8       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               40925          40955          26          2.4         409.2       1.0X
-after 1582, vec on                                31246          31404         164          3.2         312.5       1.3X
-before 1582, vec off                              44634          44680          40          2.2         446.3       0.9X
-before 1582, vec on                               35578          35834         282          2.8         355.8       1.2X
+after 1582, vec off                               41411          41514          97          2.4         414.1       1.0X
+after 1582, vec on                                32163          32201          36          3.1         321.6       1.3X
+before 1582, vec off                              43013          43111         131          2.3         430.1       1.0X
+before 1582, vec on                               34114          34152          45          2.9         341.1       1.2X
 
 

From 308a8fd3c67704b7fce0067a199707f46c6e6f1e Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 3 Apr 2020 14:36:03 +0900
Subject: [PATCH 0403/1280] [SPARK-31326][SQL][DOCS] Create Function docs
 structure for SQL Reference

### What changes were proposed in this pull request?
Create Function docs structure for SQL Reference...

### Why are the changes needed?
so the Function docs can be added later, also want to get a consensus about what to document for Functions in SQL Reference.

### Does this PR introduce any user-facing change?
Yes
<img width="1050" alt="Screen Shot 2020-04-02 at 12 09 20 AM" src="https://user-images.githubusercontent.com/13592258/78220451-68b6e100-7476-11ea-9a21-733b41652785.png">

<img width="1051" alt="Screen Shot 2020-04-02 at 12 09 44 AM" src="https://user-images.githubusercontent.com/13592258/78220460-6ce2fe80-7476-11ea-887c-defefd55c19d.png">

<img width="1051" alt="Screen Shot 2020-04-02 at 12 10 05 AM" src="https://user-images.githubusercontent.com/13592258/78220463-6f455880-7476-11ea-81fc-fd4137db7c3f.png">

### How was this patch tested?
Manually build and check

Closes #28099 from huaxingao/function.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4e45c07f5dbc4b178c41449320b7405b20aa05e9)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml                    | 21 ++++++++++++++++++++
 docs/sql-ref-functions-builtin-aggregate.md | 10 +++++-----
 docs/sql-ref-functions-builtin-array.md     | 22 +++++++++++++++++++++
 docs/sql-ref-functions-builtin-date-time.md | 22 +++++++++++++++++++++
 docs/sql-ref-functions-builtin.md           | 17 ++++++++--------
 docs/sql-ref-functions-udf-hive.md          | 22 +++++++++++++++++++++
 docs/sql-ref-functions-udf.md               | 17 ++++++++--------
 docs/sql-ref-functions.md                   | 13 +++++-------
 8 files changed, 115 insertions(+), 29 deletions(-)
 create mode 100644 docs/sql-ref-functions-builtin-array.md
 create mode 100644 docs/sql-ref-functions-builtin-date-time.md
 create mode 100644 docs/sql-ref-functions-udf-hive.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 6534c50be2cc4..500895aab4a6c 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -225,5 +225,26 @@
                   url: sql-ref-syntax-aux-resource-mgmt-list-file.html
                 - text: LIST JAR
                   url: sql-ref-syntax-aux-resource-mgmt-list-jar.html
+    - text: Functions
+      url: sql-ref-functions.html
+      subitems:
+      - text: Build-in Functions
+        url: sql-ref-functions-builtin.html
+        subitems:
+        - text: Build-in Aggregate Functions
+          url: sql-ref-functions-builtin-aggregate.html
+        - text: Build-in Array Functions
+          url: sql-ref-functions-builtin-array.html
+        - text: Build-in Date Time Functions
+          url: sql-ref-functions-builtin-date-time.html
+      - text: UDFs (User-Defined Functions)
+        url: sql-ref-functions-udf.html
+        subitems:
+        - text: Scalar UDFs (User-Defined Functions)
+          url: sql-ref-functions-udf-scalar.html
+        - text: UDAFs (User-Defined Aggregate Functions)
+          url: sql-ref-functions-udf-aggregate.html
+        - text: Integration with Hive UDFs/UDAFs/UDTFs
+          url: sql-ref-functions-udf-hive.html
     - text: Datetime Pattern
       url: sql-ref-datetime-pattern.html
diff --git a/docs/sql-ref-functions-builtin-aggregate.md b/docs/sql-ref-functions-builtin-aggregate.md
index 3fcd78245ca04..d59543647e025 100644
--- a/docs/sql-ref-functions-builtin-aggregate.md
+++ b/docs/sql-ref-functions-builtin-aggregate.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Builtin Aggregate Functions
-displayTitle: Builtin Aggregate Functions
+title: Built-in Aggregate Functions
+displayTitle: Built-in Aggregate Functions
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,4 +19,4 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+Aggregate functions
\ No newline at end of file
diff --git a/docs/sql-ref-functions-builtin-array.md b/docs/sql-ref-functions-builtin-array.md
new file mode 100644
index 0000000000000..599d10c23b616
--- /dev/null
+++ b/docs/sql-ref-functions-builtin-array.md
@@ -0,0 +1,22 @@
+---
+layout: global
+title: Built-in Array Functions
+displayTitle: Built-in Array Functions
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Array Functions
diff --git a/docs/sql-ref-functions-builtin-date-time.md b/docs/sql-ref-functions-builtin-date-time.md
new file mode 100644
index 0000000000000..10421324a3448
--- /dev/null
+++ b/docs/sql-ref-functions-builtin-date-time.md
@@ -0,0 +1,22 @@
+---
+layout: global
+title: Built-in Date and Time Functions
+displayTitle: Built-in Date and Time Functions
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Date-Time Functions
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
index 6c57b0d6fdf6f..48e5c0e6e13e2 100644
--- a/docs/sql-ref-functions-builtin.md
+++ b/docs/sql-ref-functions-builtin.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: Built-in Functions
+displayTitle: Built-in Functions
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,8 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+Spark SQL defines built-in functions to use, a complete list of which can be found [here](api/sql/). Among them, Spark SQL has several special categories of built-in functions: [Aggregate Functions](sql-ref-functions-builtin-aggregate.html) to operate on a group of rows, [Array Functions](sql-ref-functions-builtin-array.html) to operate on Array columns, and [Date and Time Functions](sql-ref-functions-builtin-date-time.html) to operate on Date and Time.
+
+* [Aggregate Functions](sql-ref-functions-builtin-aggregate.html)
+* [Array Functions](sql-ref-functions-builtin-array.html)
+* [Date and Time Functions](sql-ref-functions-builtin-date-time.html)
diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md
new file mode 100644
index 0000000000000..8698be707ea43
--- /dev/null
+++ b/docs/sql-ref-functions-udf-hive.md
@@ -0,0 +1,22 @@
+---
+layout: global
+title: Integration with Hive UDFs/UDAFs/UDTFs
+displayTitle: Integration with Hive UDFs/UDAFs/UDTFs
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+Integration with Hive UDFs/UDAFs/UDTFs
\ No newline at end of file
diff --git a/docs/sql-ref-functions-udf.md b/docs/sql-ref-functions-udf.md
index 6c57b0d6fdf6f..91c04f16049e2 100644
--- a/docs/sql-ref-functions-udf.md
+++ b/docs/sql-ref-functions-udf.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: UDFs (User-Defined Functions)
+displayTitle: UDFs (User-Defined Functions)
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,8 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.
+
+* [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+* [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index 6c57b0d6fdf6f..dabf90bc52855 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: Functions
+displayTitle: Functions
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,7 +19,4 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+Spark SQL defines commonly used functions as built-in functions. It also provides User-Defined Functions (UDFs) to allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.

From 015f066acbed0a98000f3ccf9c20ddc49fdfb2f6 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Sat, 22 Feb 2020 09:46:42 +0900
Subject: [PATCH 0404/1280] [SPARK-30840][CORE][SQL] Add version property for
 ConfigEntry and ConfigBuilder
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Spark `ConfigEntry` and `ConfigBuilder` missing Spark version information of each configuration at release. This is not good for Spark user when they visiting the page of spark configuration.
http://spark.apache.org/docs/latest/configuration.html
The new Spark SQL config docs looks like:
![sql配置截屏](https://user-images.githubusercontent.com/8486025/74604522-cb882f00-50f9-11ea-8683-57a90f9e3347.png)

```
> SET -v
spark.sql.adaptive.enabled      false   When true, enable adaptive query execution.
spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin       0.2     The relation with a non-empty partition ratio lower than this config will not be considered as the build side of a broadcast-hash join in adaptive execution regardless of its size.This configuration only has an effect when 'spark.sql.adaptive.enabled' is enabled.
spark.sql.adaptive.optimizeSkewedJoin.enabled   true    When true and adaptive execution is enabled, a skewed join is automatically handled at runtime.
spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionFactor     10      A partition is considered as a skewed partition if its size is larger than this factor multiple the median partition size and also larger than  spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionSizeThreshold
spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionMaxSplits  5       Configures the maximum number of task to handle a skewed partition in adaptive skewedjoin.
spark.sql.adaptive.optimizeSkewedJoin.skewedPartitionSizeThreshold      64MB    Configures the minimum size in bytes for a partition that is considered as a skewed partition in adaptive skewed join.
spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch.enabled    true    Whether to fetch the continuous shuffle blocks in batch. Instead of fetching blocks one by one, fetching continuous shuffle blocks for the same map task in batch can reduce IO and improve performance. Note, multiple continuous blocks exist in single fetch request only happen when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled' is enabled, this feature also depends on a relocatable serializer, the concatenation support codec in use and the new version shuffle fetch protocol.
spark.sql.adaptive.shuffle.localShuffleReader.enabled   true    When true and 'spark.sql.adaptive.enabled' is enabled, this enables the optimization of converting the shuffle reader to local shuffle reader for the shuffle exchange of the broadcast hash join in probe side.
spark.sql.adaptive.shuffle.maxNumPostShufflePartitions  <undefined>     The advisory maximum number of post-shuffle partitions used in adaptive execution. This is used as the initial number of pre-shuffle partitions. By default it equals to spark.sql.shuffle.partitions. This configuration only has an effect when 'spark.sql.adaptive.enabled' and 'spark.sql.adaptive.shuffle.reducePostShufflePartitions.enabled' is enabled.
```

**Note**: Because there are so many configuration items that are exposed and require a lot of finishing, I will add the version numbers of these configuration items in another PR.

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
Exists UT

Closes #27592 from beliefer/add-version-to-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/ConfigBuilder.scala | 16 ++++++---
 .../spark/internal/config/ConfigEntry.scala   | 35 +++++++++++++------
 .../apache/spark/sql/internal/SQLConf.scala   |  4 +--
 .../spark/sql/api/python/PythonSQLUtils.scala |  2 +-
 .../sql/execution/command/SetCommand.scala    | 11 ++++--
 .../spark/sql/internal/SQLConfSuite.scala     |  8 ++---
 sql/gen-sql-config-docs.py                    | 10 ++++--
 7 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala b/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala
index 68e1994f0f94f..8d5959a0c8b7f 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/ConfigBuilder.scala
@@ -129,7 +129,7 @@ private[spark] class TypedConfigBuilder[T](
   def createOptional: OptionalConfigEntry[T] = {
     val entry = new OptionalConfigEntry[T](parent.key, parent._prependedKey,
       parent._prependSeparator, parent._alternatives, converter, stringConverter, parent._doc,
-      parent._public)
+      parent._public, parent._version)
     parent._onCreate.foreach(_(entry))
     entry
   }
@@ -144,7 +144,7 @@ private[spark] class TypedConfigBuilder[T](
       val transformedDefault = converter(stringConverter(default))
       val entry = new ConfigEntryWithDefault[T](parent.key, parent._prependedKey,
         parent._prependSeparator, parent._alternatives, transformedDefault, converter,
-        stringConverter, parent._doc, parent._public)
+        stringConverter, parent._doc, parent._public, parent._version)
       parent._onCreate.foreach(_(entry))
       entry
     }
@@ -154,7 +154,7 @@ private[spark] class TypedConfigBuilder[T](
   def createWithDefaultFunction(defaultFunc: () => T): ConfigEntry[T] = {
     val entry = new ConfigEntryWithDefaultFunction[T](parent.key, parent._prependedKey,
       parent._prependSeparator, parent._alternatives, defaultFunc, converter, stringConverter,
-      parent._doc, parent._public)
+      parent._doc, parent._public, parent._version)
     parent._onCreate.foreach(_ (entry))
     entry
   }
@@ -166,7 +166,7 @@ private[spark] class TypedConfigBuilder[T](
   def createWithDefaultString(default: String): ConfigEntry[T] = {
     val entry = new ConfigEntryWithDefaultString[T](parent.key, parent._prependedKey,
       parent._prependSeparator, parent._alternatives, default, converter, stringConverter,
-      parent._doc, parent._public)
+      parent._doc, parent._public, parent._version)
     parent._onCreate.foreach(_(entry))
     entry
   }
@@ -186,6 +186,7 @@ private[spark] case class ConfigBuilder(key: String) {
   private[config] var _prependSeparator: String = ""
   private[config] var _public = true
   private[config] var _doc = ""
+  private[config] var _version = ""
   private[config] var _onCreate: Option[ConfigEntry[_] => Unit] = None
   private[config] var _alternatives = List.empty[String]
 
@@ -199,6 +200,11 @@ private[spark] case class ConfigBuilder(key: String) {
     this
   }
 
+  def version(v: String): ConfigBuilder = {
+    _version = v
+    this
+  }
+
   /**
    * Registers a callback for when the config entry is finally instantiated. Currently used by
    * SQLConf to keep track of SQL configuration entries.
@@ -255,7 +261,7 @@ private[spark] case class ConfigBuilder(key: String) {
 
   def fallbackConf[T](fallback: ConfigEntry[T]): ConfigEntry[T] = {
     val entry = new FallbackConfigEntry(key, _prependedKey, _prependSeparator, _alternatives, _doc,
-      _public, fallback)
+      _public, _version, fallback)
     _onCreate.foreach(_(entry))
     entry
   }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala b/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
index c5df4c8820098..8c0b11d46312e 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala
@@ -39,6 +39,7 @@ package org.apache.spark.internal.config
  * @param doc the documentation for the configuration
  * @param isPublic if this configuration is public to the user. If it's `false`, this
  *                 configuration is only used internally and we should not expose it to users.
+ * @param version the spark version when the configuration was released.
  * @tparam T the value type
  */
 private[spark] abstract class ConfigEntry[T] (
@@ -49,7 +50,8 @@ private[spark] abstract class ConfigEntry[T] (
     val valueConverter: String => T,
     val stringConverter: T => String,
     val doc: String,
-    val isPublic: Boolean) {
+    val isPublic: Boolean,
+    val version: String) {
 
   import ConfigEntry._
 
@@ -74,7 +76,8 @@ private[spark] abstract class ConfigEntry[T] (
   def defaultValue: Option[T] = None
 
   override def toString: String = {
-    s"ConfigEntry(key=$key, defaultValue=$defaultValueString, doc=$doc, public=$isPublic)"
+    s"ConfigEntry(key=$key, defaultValue=$defaultValueString, doc=$doc, " +
+      s"public=$isPublic, version=$version)"
   }
 }
 
@@ -87,7 +90,8 @@ private class ConfigEntryWithDefault[T] (
     valueConverter: String => T,
     stringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry(
     key,
     prependedKey,
@@ -96,7 +100,8 @@ private class ConfigEntryWithDefault[T] (
     valueConverter,
     stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValue: Option[T] = Some(_defaultValue)
@@ -117,7 +122,8 @@ private class ConfigEntryWithDefaultFunction[T] (
     valueConverter: String => T,
     stringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry(
     key,
     prependedKey,
@@ -126,7 +132,8 @@ private class ConfigEntryWithDefaultFunction[T] (
     valueConverter,
     stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValue: Option[T] = Some(_defaultFunction())
@@ -147,7 +154,8 @@ private class ConfigEntryWithDefaultString[T] (
     valueConverter: String => T,
     stringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry(
     key,
     prependedKey,
@@ -156,7 +164,8 @@ private class ConfigEntryWithDefaultString[T] (
     valueConverter,
     stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValue: Option[T] = Some(valueConverter(_defaultValue))
@@ -181,7 +190,8 @@ private[spark] class OptionalConfigEntry[T](
     val rawValueConverter: String => T,
     val rawStringConverter: T => String,
     doc: String,
-    isPublic: Boolean)
+    isPublic: Boolean,
+    version: String)
   extends ConfigEntry[Option[T]](
     key,
     prependedKey,
@@ -190,7 +200,8 @@ private[spark] class OptionalConfigEntry[T](
     s => Some(rawValueConverter(s)),
     v => v.map(rawStringConverter).orNull,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValueString: String = ConfigEntry.UNDEFINED
@@ -210,6 +221,7 @@ private[spark] class FallbackConfigEntry[T] (
     alternatives: List[String],
     doc: String,
     isPublic: Boolean,
+    version: String,
     val fallback: ConfigEntry[T])
   extends ConfigEntry[T](
     key,
@@ -219,7 +231,8 @@ private[spark] class FallbackConfigEntry[T] (
     fallback.valueConverter,
     fallback.stringConverter,
     doc,
-    isPublic
+    isPublic,
+    version
   ) {
 
   override def defaultValueString: String = s"<value of ${fallback.key}>"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 44f303766f017..9f9e55601d69e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2982,10 +2982,10 @@ class SQLConf extends Serializable with Logging {
    * Return all the configuration definitions that have been defined in [[SQLConf]]. Each
    * definition contains key, defaultValue and doc.
    */
-  def getAllDefinedConfs: Seq[(String, String, String)] = sqlConfEntries.synchronized {
+  def getAllDefinedConfs: Seq[(String, String, String, String)] = sqlConfEntries.synchronized {
     sqlConfEntries.values.asScala.filter(_.isPublic).map { entry =>
       val displayValue = Option(getConfString(entry.key, null)).getOrElse(entry.defaultValueString)
-      (entry.key, displayValue, entry.doc)
+      (entry.key, displayValue, entry.doc, entry.version)
     }.toSeq
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index bf3055d5e3e09..03f5a60aec438 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -40,7 +40,7 @@ private[sql] object PythonSQLUtils {
     FunctionRegistry.functionSet.flatMap(f => FunctionRegistry.builtin.lookupFunction(f)).toArray
   }
 
-  def listSQLConfigs(): Array[(String, String, String)] = {
+  def listSQLConfigs(): Array[(String, String, String, String)] = {
     val conf = new SQLConf()
     // Py4J doesn't seem to translate Seq well, so we convert to an Array.
     conf.getAllDefinedConfs.toArray
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
index a12b261ce50af..3dc1d52697714 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala
@@ -115,14 +115,19 @@ case class SetCommand(kv: Option[(String, Option[String])]) extends RunnableComm
     case Some(("-v", None)) =>
       val runFunc = (sparkSession: SparkSession) => {
         sparkSession.sessionState.conf.getAllDefinedConfs.sorted.map {
-          case (key, defaultValue, doc) =>
-            Row(key, Option(defaultValue).getOrElse("<undefined>"), doc)
+          case (key, defaultValue, doc, version) =>
+            Row(
+              key,
+              Option(defaultValue).getOrElse("<undefined>"),
+              doc,
+              Option(version).getOrElse("<unknown>"))
         }
       }
       val schema = StructType(
         StructField("key", StringType, nullable = false) ::
           StructField("value", StringType, nullable = false) ::
-          StructField("meaning", StringType, nullable = false) :: Nil)
+          StructField("meaning", StringType, nullable = false) ::
+          StructField("Since version", StringType, nullable = false) :: Nil)
       (schema.toAttributes, runFunc)
 
     // Queries the deprecated "mapred.reduce.tasks" property.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index c2d8493da92b6..f389465d4b489 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -301,8 +301,8 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     assert(spark.sessionState.conf.getConfString(fallback.key, "lzo") === "lzo")
 
     val displayValue = spark.sessionState.conf.getAllDefinedConfs
-      .find { case (key, _, _) => key == fallback.key }
-      .map { case (_, v, _) => v }
+      .find { case (key, _, _, _) => key == fallback.key }
+      .map { case (_, v, _, _) => v }
       .get
     assert(displayValue === fallback.defaultValueString)
 
@@ -313,8 +313,8 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     assert(spark.sessionState.conf.getConfString(fallback.key) === "lzo")
 
     val newDisplayValue = spark.sessionState.conf.getAllDefinedConfs
-      .find { case (key, _, _) => key == fallback.key }
-      .map { case (_, v, _) => v }
+      .find { case (key, _, _, _) => key == fallback.key }
+      .map { case (_, v, _, _) => v }
       .get
     assert(newDisplayValue === "lzo")
 
diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py
index 04f5a850c9980..98212ad373370 100644
--- a/sql/gen-sql-config-docs.py
+++ b/sql/gen-sql-config-docs.py
@@ -25,7 +25,7 @@
 from pyspark.java_gateway import launch_gateway
 
 SQLConfEntry = namedtuple(
-    "SQLConfEntry", ["name", "default", "description"])
+    "SQLConfEntry", ["name", "default", "description", "version"])
 
 
 def get_public_sql_configs(jvm):
@@ -34,6 +34,7 @@ def get_public_sql_configs(jvm):
             name=_sql_config._1(),
             default=_sql_config._2(),
             description=_sql_config._3(),
+            version=_sql_config._4()
         )
         for _sql_config in jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
     ]
@@ -49,12 +50,13 @@ def generate_sql_configs_table(sql_configs, path):
 
     ```html
     <table class="table">
-    <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+    <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 
     <tr>
         <td><code>spark.sql.adaptive.enabled</code></td>
         <td>false</td>
         <td><p>When true, enable adaptive query execution.</p></td>
+        <td>2.1.0</td>
     </tr>
 
     ...
@@ -68,7 +70,7 @@ def generate_sql_configs_table(sql_configs, path):
         f.write(dedent(
             """
             <table class="table">
-            <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+            <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
             """
         ))
         for config in sorted(sql_configs, key=lambda x: x.name):
@@ -96,12 +98,14 @@ def generate_sql_configs_table(sql_configs, path):
                     <td><code>{name}</code></td>
                     <td>{default}</td>
                     <td>{description}</td>
+                    <td>{version}</td>
                 </tr>
                 """
                 .format(
                     name=config.name,
                     default=default,
                     description=markdown.markdown(config.description),
+                    version=config.version
                 )
             ))
         f.write("</table>\n")

From 0843f2df9b36b38735d08c2bd6f0dfe4ca0c432c Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 3 Apr 2020 16:05:07 +0900
Subject: [PATCH 0405/1280] [SPARK-31249][CORE] Fix flaky
 CoarseGrainedSchedulerBackendSuite.custom log url for Spark UI is applied

### What changes were proposed in this pull request?

In `CoarseGrainedSchedulerBackendSuite.RegisterExecutor`, change it to post `SparkListenerExecutorAdded` before `context.reply(true)`.

### Why are the changes needed?

To fix flaky `CoarseGrainedSchedulerBackendSuite.custom log url for Spark UI is applied`.

In this test, though we use `askSync` to register executor but `askSync` could be finished before we posting the 3rd `SparkListenerExecutorAdded` event to the listener bus due to the reason that `context.reply(true)` comes before `listenerBus.post`.

The error can be reproduced if you:
- loop it for 500 times in one turn
- or, insert a `Thread.sleep(1000)` between `post` and `reply`.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Loop the flaky tests for 1000 times without any error.

Closes #28100 from Ngone51/fix_spark_31249.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a4fc6a6e98fcd7ea5ec7a006864ade855c4b5a59)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scheduler/cluster/CoarseGrainedSchedulerBackend.scala     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 55f4005ef1b45..9d8fb8f83856d 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -249,10 +249,10 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
               logDebug(s"Decremented number of pending executors ($numPendingExecutors left)")
             }
           }
-          // Note: some tests expect the reply to come after we put the executor in the map
-          context.reply(true)
           listenerBus.post(
             SparkListenerExecutorAdded(System.currentTimeMillis(), executorId, data))
+          // Note: some tests expect the reply to come after we put the executor in the map
+          context.reply(true)
         }
 
       case StopDriver =>

From d36e2a721d0e3827477ca5a1aaf55700587380cf Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 3 Apr 2020 12:43:33 +0000
Subject: [PATCH 0406/1280] [SPARK-31327][SQL] Write Spark version into Avro
 file metadata

### What changes were proposed in this pull request?

Write Spark version into Avro file metadata

### Why are the changes needed?

The version info is very useful for backward compatibility. This is also done in parquet/orc.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

new test

Closes #28102 from cloud-fan/avro.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6b1ca886c0066f4e10534336f3fce64cdebc79a5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/avro/SparkAvroKeyOutputFormat.java    | 94 +++++++++++++++++++
 .../spark/sql/avro/AvroOutputWriter.scala     | 12 ++-
 .../org/apache/spark/sql/avro/AvroSuite.scala | 14 ++-
 .../scala/org/apache/spark/sql/package.scala  |  1 +
 4 files changed, 116 insertions(+), 5 deletions(-)
 create mode 100644 external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java

diff --git a/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
new file mode 100644
index 0000000000000..55696a6ac2e0f
--- /dev/null
+++ b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.avro;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.CodecFactory;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.mapred.AvroKey;
+import org.apache.avro.mapreduce.AvroKeyOutputFormat;
+import org.apache.avro.mapreduce.Syncable;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+// A variant of `AvroKeyOutputFormat`, which is used to inject the custom `RecordWriterFactory` so
+// that we can set avro file metadata.
+public class SparkAvroKeyOutputFormat extends AvroKeyOutputFormat<GenericRecord> {
+  public SparkAvroKeyOutputFormat(Map<String, String> metadata) {
+    super(new SparkRecordWriterFactory(metadata));
+  }
+
+  static class SparkRecordWriterFactory extends RecordWriterFactory<GenericRecord> {
+    private final Map<String, String> metadata;
+    SparkRecordWriterFactory(Map<String, String> metadata) {
+      this.metadata = metadata;
+    }
+
+    protected RecordWriter<AvroKey<GenericRecord>, NullWritable> create(
+        Schema writerSchema,
+        GenericData dataModel,
+        CodecFactory compressionCodec,
+        OutputStream outputStream,
+        int syncInterval) throws IOException {
+      return new SparkAvroKeyRecordWriter(
+        writerSchema, dataModel, compressionCodec, outputStream, syncInterval, metadata);
+    }
+  }
+}
+
+// This a fork of org.apache.avro.mapreduce.AvroKeyRecordWriter, in order to set file metadata.
+class SparkAvroKeyRecordWriter<T> extends RecordWriter<AvroKey<T>, NullWritable>
+    implements Syncable {
+
+  private final DataFileWriter<T> mAvroFileWriter;
+
+  SparkAvroKeyRecordWriter(
+      Schema writerSchema,
+      GenericData dataModel,
+      CodecFactory compressionCodec,
+      OutputStream outputStream,
+      int syncInterval,
+      Map<String, String> metadata) throws IOException {
+    this.mAvroFileWriter = new DataFileWriter(dataModel.createDatumWriter(writerSchema));
+    for (Map.Entry<String, String> entry : metadata.entrySet()) {
+      this.mAvroFileWriter.setMeta(entry.getKey(), entry.getValue());
+    }
+    this.mAvroFileWriter.setCodec(compressionCodec);
+    this.mAvroFileWriter.setSyncInterval(syncInterval);
+    this.mAvroFileWriter.create(writerSchema, outputStream);
+  }
+
+  public void write(AvroKey<T> record, NullWritable ignore) throws IOException {
+    this.mAvroFileWriter.append(record.datum());
+  }
+
+  public void close(TaskAttemptContext context) throws IOException {
+    this.mAvroFileWriter.close();
+  }
+
+  public long sync() throws IOException {
+    return this.mAvroFileWriter.sync();
+  }
+}
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
index 06507115f5ed8..2cfa3a4826ed8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
@@ -19,14 +19,17 @@ package org.apache.spark.sql.avro
 
 import java.io.{IOException, OutputStream}
 
+import scala.collection.JavaConverters._
+
 import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
 import org.apache.avro.mapred.AvroKey
-import org.apache.avro.mapreduce.AvroKeyOutputFormat
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext}
 
+import org.apache.spark.SPARK_VERSION_SHORT
+import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriter
 import org.apache.spark.sql.types._
@@ -45,8 +48,9 @@ private[avro] class AvroOutputWriter(
    * Overrides the couple of methods responsible for generating the output streams / files so
    * that the data can be correctly partitioned
    */
-  private val recordWriter: RecordWriter[AvroKey[GenericRecord], NullWritable] =
-    new AvroKeyOutputFormat[GenericRecord]() {
+  private val recordWriter: RecordWriter[AvroKey[GenericRecord], NullWritable] = {
+    val sparkVersion = Map(SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT).asJava
+    new SparkAvroKeyOutputFormat(sparkVersion) {
 
       override def getDefaultWorkFile(context: TaskAttemptContext, extension: String): Path = {
         new Path(path)
@@ -57,8 +61,8 @@ private[avro] class AvroOutputWriter(
         val path = getDefaultWorkFile(context, ".avro")
         path.getFileSystem(context.getConfiguration).create(path)
       }
-
     }.getRecordWriter(context)
+  }
 
   override def write(row: InternalRow): Unit = {
     val key = new AvroKey(serializer.serialize(row).asInstanceOf[GenericRecord])
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 9336d8e2257f9..a5224fd104ea9 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -33,7 +33,7 @@ import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWri
 import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed}
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
@@ -1620,6 +1620,18 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-31327: Write Spark version into Avro file metadata") {
+    withTempPath { path =>
+      spark.range(1).repartition(1).write.format("avro").save(path.getCanonicalPath)
+      val avroFiles = path.listFiles()
+        .filter(f => f.isFile && !f.getName.startsWith(".") && !f.getName.startsWith("_"))
+      assert(avroFiles.length === 1)
+      val reader = DataFileReader.openReader(avroFiles(0), new GenericDatumReader[GenericRecord]())
+      val version = reader.asInstanceOf[DataFileReader[_]].getMetaString(SPARK_VERSION_METADATA_KEY)
+      assert(version === SPARK_VERSION_SHORT)
+    }
+  }
 }
 
 class AvroV1Suite extends AvroSuite {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
index 61875931d226e..58de6758f4d7b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
@@ -49,6 +49,7 @@ package object sql {
    * Metadata key which is used to write Spark version in the followings:
    * - Parquet file metadata
    * - ORC file metadata
+   * - Avro file metadata
    *
    * Note that Hive table property `spark.sql.create.version` also has Spark version.
    */

From 7c5262c6fdb76bdfce16181b74ed85b48224b959 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sat, 4 Apr 2020 08:09:15 +0900
Subject: [PATCH 0407/1280] [SPARK-31231][BUILD] Unset setuptools version in
 pip packaging test

### What changes were proposed in this pull request?

This PR unsets `setuptools` version in CI. This was fixed in the 0.46.1.2+ `setuptools` - pypa/setuptools#2046. `setuptools` 0.46.1.0 and 0.46.1.1 still have this problem.

### Why are the changes needed?

To test the latest setuptools out to see if users can actually install and use it.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Jenkins will test.

Closes #28111 from HyukjinKwon/SPARK-31231-revert.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 9f58f0385758f31179943a681e4353eb9279cb20)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/run-pip-tests | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 928d40eb19e2e..470f21e69d46a 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -76,7 +76,7 @@ for python in "${PYTHON_EXECS[@]}"; do
     VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
     rm -rf "$VIRTUALENV_PATH"
     if [ -n "$USE_CONDA" ]; then
-      conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip "setuptools<46.1.0"
+      conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
       source activate "$VIRTUALENV_PATH"
     else
       mkdir -p "$VIRTUALENV_PATH"

From af7fa9d12622c3b2b169c4a58d143bc21ed9e784 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Sun, 5 Apr 2020 13:12:56 +0900
Subject: [PATCH 0408/1280] [SPARK-30841][SQL][DOC][3.0] Add version
 information to the configuration of SQL

### What changes were proposed in this pull request?
This PR follows https://github.com/apache/spark/pull/27691 and used to spark3.0.
The configuration only have one difference.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.pandas.udf.buffer.size | 3.0.0 | SPARK-27870 | 26998b86c13e79582a3df31f6184f825cde45e73#diff-9a6b543db706f1a90f790783d6930a13 | this config changed to spark.sql.execution.pandas.udf.buffer.size in master.

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test.

Closes #28116 from beliefer/backport-sqlconf-version.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 267 +++++++++++++++++-
 1 file changed, 261 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 9f9e55601d69e..7a10aba7ef6e7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -177,6 +177,7 @@ object SQLConf {
   val ANALYZER_MAX_ITERATIONS = buildConf("spark.sql.analyzer.maxIterations")
     .internal()
     .doc("The max number of iterations the analyzer runs.")
+    .version("3.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -185,12 +186,14 @@ object SQLConf {
       "specified by their rule names and separated by comma. It is not guaranteed that all the " +
       "rules in this configuration will eventually be excluded, as some rules are necessary " +
       "for correctness. The optimizer will log the rules that have indeed been excluded.")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val OPTIMIZER_MAX_ITERATIONS = buildConf("spark.sql.optimizer.maxIterations")
     .internal()
     .doc("The max number of iterations the optimizer runs.")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -198,6 +201,7 @@ object SQLConf {
     buildConf("spark.sql.optimizer.inSetConversionThreshold")
       .internal()
       .doc("The threshold of set size for InSet conversion.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -206,6 +210,7 @@ object SQLConf {
       .internal()
       .doc("Configures the max set size in InSet for which Spark will generate code with " +
         "switch statements. This is applicable only to bytes, shorts, ints, dates.")
+      .version("3.0.0")
       .intConf
       .checkValue(threshold => threshold >= 0 && threshold <= 600, "The max set size " +
         "for using switch statements in InSet must be non-negative and less than or equal to 600")
@@ -216,6 +221,7 @@ object SQLConf {
     .doc("Configures the log level for logging the change from the original plan to the new " +
       "plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', " +
       "'warn', or 'error'. The default log level is 'trace'.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValue(logLevel => Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR").contains(logLevel),
@@ -227,6 +233,7 @@ object SQLConf {
     .internal()
     .doc("Configures a list of rules to be logged in the optimizer, in which the rules are " +
       "specified by their rule names and separated by comma.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -234,12 +241,14 @@ object SQLConf {
     .internal()
     .doc("Configures a list of batches to be logged in the optimizer, in which the batches " +
       "are specified by their batch names and separated by comma.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val DYNAMIC_PARTITION_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.dynamicPartitionPruning.enabled")
       .doc("When true, we will generate predicate for partition column when it's used as join key")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -249,6 +258,7 @@ object SQLConf {
       .doc("When true, distinct count statistics will be used for computing the data size of the " +
         "partitioned table after dynamic partition pruning, in order to evaluate if it is worth " +
         "adding an extra subquery as the pruning filter if broadcast reuse is not applicable.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -259,6 +269,7 @@ object SQLConf {
       "used as the fallback filter ratio for computing the data size of the partitioned table " +
       "after dynamic partition pruning, in order to evaluate if it is worth adding an extra " +
       "subquery as the pruning filter if broadcast reuse is not applicable.")
+    .version("3.0.0")
     .doubleConf
     .createWithDefault(0.5)
 
@@ -267,18 +278,21 @@ object SQLConf {
       .internal()
       .doc("When true, dynamic partition pruning will only apply when the broadcast exchange of " +
         "a broadcast hash join operation can be reused as the dynamic pruning filter.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val COMPRESS_CACHED = buildConf("spark.sql.inMemoryColumnarStorage.compressed")
     .doc("When set to true Spark SQL will automatically select a compression codec for each " +
       "column based on statistics of the data.")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(true)
 
   val COLUMN_BATCH_SIZE = buildConf("spark.sql.inMemoryColumnarStorage.batchSize")
     .doc("Controls the size of batches for columnar caching.  Larger batch sizes can improve " +
       "memory utilization and compression, but risk OOMs when caching data.")
+    .version("1.1.1")
     .intConf
     .createWithDefault(10000)
 
@@ -286,6 +300,7 @@ object SQLConf {
     buildConf("spark.sql.inMemoryColumnarStorage.partitionPruning")
       .internal()
       .doc("When true, enable partition pruning for in-memory columnar tables.")
+      .version("1.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -293,12 +308,14 @@ object SQLConf {
     buildConf("spark.sql.inMemoryTableScanStatistics.enable")
       .internal()
       .doc("When true, enable in-memory table scan accumulators.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val CACHE_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.inMemoryColumnarStorage.enableVectorizedReader")
       .doc("Enables vectorized reader for columnar caching.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -306,12 +323,14 @@ object SQLConf {
     buildConf("spark.sql.columnVector.offheap.enabled")
       .internal()
       .doc("When true, use OffHeapColumnVector in ColumnarBatch.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val PREFER_SORTMERGEJOIN = buildConf("spark.sql.join.preferSortMergeJoin")
     .internal()
     .doc("When true, prefer sort merge join over shuffle hash join.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -320,6 +339,7 @@ object SQLConf {
     .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " +
       "requires additional memory to be reserved up-front. The memory overhead may be " +
       "significant when sorting very small rows (up to 50% more in this case).")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -330,6 +350,7 @@ object SQLConf {
       "command `ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan` has been " +
       "run, and file-based data source tables where the statistics are computed directly on " +
       "the files of data.")
+    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("10MB")
 
@@ -338,6 +359,7 @@ object SQLConf {
     .doc("Minimal increase rate in number of partitions between attempts when executing a take " +
       "on a query. Higher values lead to more partitions read. Lower values might lead to " +
       "longer execution times as more jobs will be run")
+    .version("2.1.1")
     .intConf
     .createWithDefault(4)
 
@@ -345,6 +367,7 @@ object SQLConf {
     buildConf("spark.sql.hive.advancedPartitionPredicatePushdown.enabled")
       .internal()
       .doc("When true, advanced partition predicate pushdown into Hive metastore is enabled.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -352,6 +375,7 @@ object SQLConf {
     .doc("The default number of partitions to use when shuffling data for joins or aggregations. " +
       "Note: For structured streaming, this configuration cannot be changed between query " +
       "restarts from the same checkpoint location.")
+    .version("1.1.0")
     .intConf
     .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be positive")
     .createWithDefault(200)
@@ -360,12 +384,14 @@ object SQLConf {
     buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
       .internal()
       .doc("(Deprecated since Spark 3.0)")
+      .version("1.6.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("64MB")
 
   val ADAPTIVE_EXECUTION_ENABLED = buildConf("spark.sql.adaptive.enabled")
     .doc("When true, enable adaptive query execution, which re-optimizes the query plan in the " +
       "middle of query execution, based on accurate runtime statistics.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -375,6 +401,7 @@ object SQLConf {
       "sub-queries. By setting this config to true (together with " +
       s"'${ADAPTIVE_EXECUTION_ENABLED.key}' set to true), Spark will force apply adaptive query " +
       "execution for all supported queries.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -382,6 +409,7 @@ object SQLConf {
     .internal()
     .doc("Configures the log level for adaptive execution logging of plan changes. The value " +
       "can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR"))
@@ -392,6 +420,7 @@ object SQLConf {
       .doc("The advisory size in bytes of the shuffle partition during adaptive optimization " +
         s"(when ${ADAPTIVE_EXECUTION_ENABLED.key} is true). It takes effect when Spark " +
         "coalesces small shuffle partitions or splits skewed shuffle partition.")
+      .version("3.0.0")
       .fallbackConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
 
   val COALESCE_PARTITIONS_ENABLED =
@@ -399,6 +428,7 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark will coalesce " +
         "contiguous shuffle partitions according to the target size (specified by " +
         s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'), to avoid too many small tasks.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -408,6 +438,7 @@ object SQLConf {
         "value is the default parallelism of the Spark cluster. This configuration only " +
         s"has an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The minimum number of partitions must be positive.")
       .createOptional
@@ -418,6 +449,7 @@ object SQLConf {
         s"${SHUFFLE_PARTITIONS.key}. This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and '${COALESCE_PARTITIONS_ENABLED.key}' " +
         "are both true.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The initial number of partitions must be positive.")
       .createOptional
@@ -432,6 +464,7 @@ object SQLConf {
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true. This feature also depends " +
         "on a relocatable serializer, the concatenation support codec in use and the new version " +
         "shuffle fetch protocol.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -440,6 +473,7 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark tries to use local " +
         "shuffle reader to read the shuffle data when the shuffle partitioning is not needed, " +
         "for example, after converting sort-merge join to broadcast-hash join.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -448,6 +482,7 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark dynamically " +
         "handles skew in sort-merge join by splitting (and replicating if needed) skewed " +
         "partitions.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -456,6 +491,7 @@ object SQLConf {
       .doc("A partition is considered as skewed if its size is larger than this factor " +
         "multiplying the median partition size and also larger than " +
         "'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(10)
@@ -466,6 +502,7 @@ object SQLConf {
         s"threshold and also larger than '${SKEW_JOIN_SKEWED_PARTITION_FACTOR.key}' " +
         "multiplying the median partition size. Ideally this config should be set larger " +
         s"than '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("256MB")
 
@@ -476,6 +513,7 @@ object SQLConf {
         "considered as the build side of a broadcast-hash join in adaptive execution regardless " +
         "of its size.This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is true.")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
       .createWithDefault(0.2)
@@ -484,6 +522,7 @@ object SQLConf {
     buildConf("spark.sql.subexpressionElimination.enabled")
       .internal()
       .doc("When true, common subexpressions will be eliminated.")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -491,6 +530,7 @@ object SQLConf {
     .internal()
     .doc("Whether the query analyzer should be case sensitive or not. " +
       "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -500,6 +540,7 @@ object SQLConf {
       "plan to optimize them. Constraint propagation can sometimes be computationally expensive " +
       "for certain kinds of query plans (such as those with a large number of predicates and " +
       "aliases) which might negatively impact overall runtime.")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -508,6 +549,7 @@ object SQLConf {
     .doc("When true, string literals (including regex patterns) remain escaped in our SQL " +
       "parser. The default is false since Spark 2.0. Setting it to true can restore the behavior " +
       "prior to Spark 2.0.")
+    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -516,6 +558,7 @@ object SQLConf {
     .doc("When estimating the output data size of a table scan, multiply the file size with this " +
       "factor as the estimated data size, in case the data is compressed in the file and lead to" +
       " a heavily underestimated result.")
+    .version("2.3.1")
     .doubleConf
     .checkValue(_ > 0, "the value of fileDataSizeFactor must be greater than 0")
     .createWithDefault(1.0)
@@ -524,6 +567,7 @@ object SQLConf {
     .doc("When true, the Parquet data source merges schemas collected from all data files, " +
          "otherwise the schema is picked from the summary file or a random data file " +
          "if no summary file is available.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -532,6 +576,7 @@ object SQLConf {
          "summary files and we will ignore them when merging schema. Otherwise, if this is " +
          "false, which is the default, we will merge all part-files. This should be considered " +
          "as expert-only option, and shouldn't be enabled before knowing what it means exactly.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -540,6 +585,7 @@ object SQLConf {
       "Spark SQL, do not differentiate between binary data and strings when writing out the " +
       "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " +
       "compatibility with these systems.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -548,6 +594,7 @@ object SQLConf {
       "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " +
       "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " +
       "provide compatibility with these systems.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -555,6 +602,7 @@ object SQLConf {
     .doc("This controls whether timestamp adjustments should be applied to INT96 data when " +
       "converting to timestamps, for data written by Impala.  This is necessary because Impala " +
       "stores INT96 data with a different timezone offset than Hive & Spark.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -568,6 +616,7 @@ object SQLConf {
       "is a standard timestamp type in Parquet, which stores number of microseconds from the " +
       "Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which " +
       "means Spark has to truncate the microsecond portion of its timestamp value.")
+    .version("2.3.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(ParquetOutputTimestampType.values.map(_.toString))
@@ -579,6 +628,7 @@ object SQLConf {
       "precedence would be `compression`, `parquet.compression`, " +
       "`spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, " +
       "snappy, gzip, lzo, brotli, lz4, zstd.")
+    .version("1.1.1")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "gzip", "lzo", "lz4", "brotli", "zstd"))
@@ -586,6 +636,7 @@ object SQLConf {
 
   val PARQUET_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.parquet.filterPushdown")
     .doc("Enables Parquet filter push-down optimization when set to true.")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -593,6 +644,7 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for Date. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
+    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -602,15 +654,17 @@ object SQLConf {
       .doc("If true, enables Parquet filter push-down optimization for Timestamp. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.")
-    .internal()
-    .booleanConf
-    .createWithDefault(true)
+      .version("2.4.0")
+      .internal()
+      .booleanConf
+      .createWithDefault(true)
 
   val PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED =
     buildConf("spark.sql.parquet.filterPushdown.decimal")
       .doc("If true, enables Parquet filter push-down optimization for Decimal. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
+      .version("2.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -620,6 +674,7 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for string startsWith function. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
+    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -632,6 +687,7 @@ object SQLConf {
         "By setting this value to 0 this feature can be disabled. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
+      .version("2.4.0")
       .internal()
       .intConf
       .checkValue(threshold => threshold >= 0, "The threshold must not be negative.")
@@ -643,6 +699,7 @@ object SQLConf {
       "systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet " +
       "will be used. For example, decimals will be written in int-based format. If Parquet " +
       "output is intended for use with systems that do not support this newer format, set to true.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -652,6 +709,7 @@ object SQLConf {
       "of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata " +
       "summaries will never be created, irrespective of the value of " +
       "parquet.summary.metadata.level")
+    .version("1.5.0")
     .internal()
     .stringConf
     .createWithDefault("org.apache.parquet.hadoop.ParquetOutputCommitter")
@@ -659,6 +717,7 @@ object SQLConf {
   val PARQUET_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.parquet.enableVectorizedReader")
       .doc("Enables vectorized parquet decoding.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -668,12 +727,14 @@ object SQLConf {
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' " +
       "is enabled and the vectorized reader is not used. You can ensure the vectorized reader " +
       s"is not used by setting '${PARQUET_VECTORIZED_READER_ENABLED.key}' to false.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val PARQUET_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.parquet.columnarReaderBatchSize")
     .doc("The number of rows to include in a parquet vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
@@ -682,6 +743,7 @@ object SQLConf {
       "`orc.compress` is specified in the table-specific options/properties, the precedence " +
       "would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`." +
       "Acceptable values include: none, uncompressed, snappy, zlib, lzo.")
+    .version("2.3.0")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "zlib", "lzo"))
@@ -690,6 +752,7 @@ object SQLConf {
   val ORC_IMPLEMENTATION = buildConf("spark.sql.orc.impl")
     .doc("When native, use the native version of ORC support instead of the ORC library in Hive. " +
       "It is 'hive' by default prior to Spark 2.4.")
+    .version("2.3.0")
     .internal()
     .stringConf
     .checkValues(Set("hive", "native"))
@@ -697,23 +760,27 @@ object SQLConf {
 
   val ORC_VECTORIZED_READER_ENABLED = buildConf("spark.sql.orc.enableVectorizedReader")
     .doc("Enables vectorized orc decoding.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.orc.columnarReaderBatchSize")
     .doc("The number of rows to include in a orc vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
   val ORC_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.orc.filterPushdown")
     .doc("When true, enable filter pushdown for ORC files.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_SCHEMA_MERGING_ENABLED = buildConf("spark.sql.orc.mergeSchema")
     .doc("When true, the Orc data source merges schemas collected from all data files, " +
       "otherwise the schema is picked from a random data file.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -721,6 +788,7 @@ object SQLConf {
     .doc("When true, check all the partition paths under the table\'s root directory " +
          "when reading data stored in HDFS. This configuration will be deprecated in the future " +
          s"releases and replaced by ${SPARK_IGNORE_MISSING_FILES.key}.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -730,6 +798,7 @@ object SQLConf {
            "unmatching partitions can be eliminated earlier. This only affects Hive tables " +
            "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " +
            "HiveUtils.CONVERT_METASTORE_ORC for more information).")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -739,6 +808,7 @@ object SQLConf {
            "This includes both datasource and converted Hive tables. When partition management " +
            "is enabled, datasource tables store partition in the Hive metastore, and use the " +
            "metastore to prune partitions during query planning.")
+      .version("2.1.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -747,6 +817,7 @@ object SQLConf {
       .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " +
            "a cache that can use up to specified num bytes for file metadata. This conf only " +
            "has an effect when hive filesource partition management is enabled.")
+      .version("2.1.1")
       .longConf
       .createWithDefault(250 * 1024 * 1024)
 
@@ -762,6 +833,7 @@ object SQLConf {
       "and write it back to the table properties), INFER_ONLY (infer the schema but don't " +
       "attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback " +
       "to using the case-insensitive metastore schema instead of inferring).")
+    .version("2.1.1")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(HiveCaseSensitiveInferenceMode.values.map(_.toString))
@@ -774,23 +846,27 @@ object SQLConf {
       "scanned are partition columns and the query has an aggregate operator that satisfies " +
       "distinct semantics. By default the optimization is disabled, since it may return " +
       "incorrect results when the files are empty.")
+    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
   val COLUMN_NAME_OF_CORRUPT_RECORD = buildConf("spark.sql.columnNameOfCorruptRecord")
     .doc("The name of internal column for storing raw/un-parsed JSON and CSV records that fail " +
       "to parse.")
+    .version("1.2.0")
     .stringConf
     .createWithDefault("_corrupt_record")
 
   val BROADCAST_TIMEOUT = buildConf("spark.sql.broadcastTimeout")
     .doc("Timeout in seconds for the broadcast wait time in broadcast joins.")
+    .version("1.3.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString(s"${5 * 60}")
 
   // This is only used for the thriftserver
   val THRIFTSERVER_POOL = buildConf("spark.sql.thriftserver.scheduler.pool")
     .doc("Set a Fair Scheduler pool for a JDBC client session.")
+    .version("1.1.1")
     .stringConf
     .createOptional
 
@@ -798,23 +874,27 @@ object SQLConf {
     buildConf("spark.sql.thriftServer.incrementalCollect")
       .internal()
       .doc("When true, enable incremental collection for execution in Thrift Server.")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   val THRIFTSERVER_UI_STATEMENT_LIMIT =
     buildConf("spark.sql.thriftserver.ui.retainedStatements")
       .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.")
+      .version("1.4.0")
       .intConf
       .createWithDefault(200)
 
   val THRIFTSERVER_UI_SESSION_LIMIT = buildConf("spark.sql.thriftserver.ui.retainedSessions")
     .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.")
+    .version("1.4.0")
     .intConf
     .createWithDefault(200)
 
   // This is used to set the default data source
   val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default")
     .doc("The default data source to use in input/output.")
+    .version("1.3.0")
     .stringConf
     .createWithDefault("parquet")
 
@@ -823,6 +903,7 @@ object SQLConf {
     .doc("When true, a table created by a Hive CTAS statement (no USING clause) " +
       "without specifying any storage property will be converted to a data source table, " +
       s"using the data source set by ${DEFAULT_DATA_SOURCE_NAME.key}.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -831,22 +912,26 @@ object SQLConf {
       .doc("When true, fast stats (number of files and total size of all files) will be gathered" +
         " in parallel while repairing table partitions to avoid the sequential listing in Hive" +
         " metastore.")
+      .version("2.0.1")
       .booleanConf
       .createWithDefault(true)
 
   val PARTITION_COLUMN_TYPE_INFERENCE =
     buildConf("spark.sql.sources.partitionColumnTypeInference.enabled")
       .doc("When true, automatically infer the data types for partitioned columns.")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   val BUCKETING_ENABLED = buildConf("spark.sql.sources.bucketing.enabled")
     .doc("When false, we will treat bucketed table as normal table")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val BUCKETING_MAX_BUCKETS = buildConf("spark.sql.sources.bucketing.maxBuckets")
     .doc("The maximum number of buckets allowed.")
+    .version("2.4.0")
     .intConf
     .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
     .createWithDefault(100000)
@@ -855,36 +940,42 @@ object SQLConf {
     .internal()
     .doc("When false, we will throw an error if a query contains a cartesian product without " +
         "explicit CROSS JOIN syntax.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORDER_BY_ORDINAL = buildConf("spark.sql.orderByOrdinal")
     .doc("When true, the ordinal numbers are treated as the position in the select list. " +
          "When false, the ordinal numbers in order/sort by clause are ignored.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ORDINAL = buildConf("spark.sql.groupByOrdinal")
     .doc("When true, the ordinal numbers in group by clauses are treated as the position " +
       "in the select list. When false, the ordinal numbers are ignored.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ALIASES = buildConf("spark.sql.groupByAliases")
     .doc("When true, aliases in a select list can be used in group by clauses. When false, " +
       "an analysis exception is thrown in the case.")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
   // The output committer class used by data sources. The specified class needs to be a
   // subclass of org.apache.hadoop.mapreduce.OutputCommitter.
   val OUTPUT_COMMITTER_CLASS = buildConf("spark.sql.sources.outputCommitterClass")
+    .version("1.4.0")
     .internal()
     .stringConf
     .createOptional
 
   val FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.sources.commitProtocolClass")
+      .version("2.1.1")
       .internal()
       .stringConf
       .createWithDefault(
@@ -896,6 +987,7 @@ object SQLConf {
         "of detected paths exceeds this value during partition discovery, it tries to list the " +
         "files with another Spark distributed job. This configuration is effective only when " +
         "using file-based sources such as Parquet, JSON and ORC.")
+      .version("1.5.0")
       .intConf
       .checkValue(parallel => parallel >= 0, "The maximum number of paths allowed for listing " +
         "files at driver side must not be negative")
@@ -905,6 +997,7 @@ object SQLConf {
     buildConf("spark.sql.sources.parallelPartitionDiscovery.parallelism")
       .doc("The number of parallelism to list a collection of path recursively, Set the " +
         "number to prevent file listing from generating too many tasks.")
+      .version("2.1.1")
       .internal()
       .intConf
       .createWithDefault(10000)
@@ -916,6 +1009,7 @@ object SQLConf {
         "schedule tasks to take advantage of data locality. It can be particularly " +
         "useful if data is read from a remote cluster so the scheduler could never " +
         "take advantage of locality anyway.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(false)
@@ -924,6 +1018,7 @@ object SQLConf {
   // See SPARK-6231.
   val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY =
     buildConf("spark.sql.selfJoinAutoResolveAmbiguity")
+      .version("1.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -931,12 +1026,14 @@ object SQLConf {
   val FAIL_AMBIGUOUS_SELF_JOIN_ENABLED =
     buildConf("spark.sql.analyzer.failAmbiguousSelfJoin")
       .doc("When true, fail the Dataset query if it contains ambiguous self-join.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
 
   // Whether to retain group by columns or not in GroupedData.agg.
   val DATAFRAME_RETAIN_GROUP_COLUMNS = buildConf("spark.sql.retainGroupColumns")
+    .version("1.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -944,12 +1041,14 @@ object SQLConf {
   val DATAFRAME_PIVOT_MAX_VALUES = buildConf("spark.sql.pivotMaxValues")
     .doc("When doing a pivot without specifying values for the pivot column this is the maximum " +
       "number of (distinct) values that will be collected without error.")
+    .version("1.6.0")
     .intConf
     .createWithDefault(10000)
 
   val RUN_SQL_ON_FILES = buildConf("spark.sql.runSQLOnFiles")
     .internal()
     .doc("When true, we could use `datasource`.`path` as table in SQL query.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -957,6 +1056,7 @@ object SQLConf {
     .internal()
     .doc("When true, the whole stage (of multiple operators) will be compiled into single java" +
       " method.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -965,6 +1065,7 @@ object SQLConf {
     .internal()
     .doc("When true, embed the (whole-stage) codegen stage ID into " +
       "the class name of the generated class as a suffix")
+    .version("2.3.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -972,6 +1073,7 @@ object SQLConf {
     .internal()
     .doc("The maximum number of fields (including nested fields) that will be supported before" +
       " deactivating whole-stage codegen.")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -981,6 +1083,7 @@ object SQLConf {
       "interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. " +
       "`NO_CODEGEN` skips codegen and goes interpreted path always. Note that " +
       "this config works only for tests.")
+    .version("2.4.0")
     .internal()
     .stringConf
     .checkValues(CodegenObjectFactoryMode.values.map(_.toString))
@@ -990,12 +1093,14 @@ object SQLConf {
     .internal()
     .doc("When true, (whole stage) codegen could be temporary disabled for the part of query that" +
       " fail to compile generated code")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val CODEGEN_LOGGING_MAX_LINES = buildConf("spark.sql.codegen.logging.maxLines")
     .internal()
     .doc("The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.")
+    .version("2.3.0")
     .intConf
     .checkValue(maxLines => maxLines >= -1, "The maximum must be a positive integer, 0 to " +
       "disable logging or -1 to apply no limit.")
@@ -1009,6 +1114,7 @@ object SQLConf {
       "is the largest bytecode size possible for a valid Java method. When running on HotSpot, " +
       s"it may be preferable to set the value to ${CodeGenerator.DEFAULT_JVM_HUGE_METHOD_LIMIT} " +
       "to match HotSpot's implementation.")
+    .version("2.3.0")
     .intConf
     .createWithDefault(65535)
 
@@ -1020,6 +1126,7 @@ object SQLConf {
       "generated, so use the code length as metric. When running on HotSpot, a function's " +
       "bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not " +
       "be too small, otherwise there will be many function calls.")
+    .version("3.0.0")
     .intConf
     .checkValue(threshold => threshold > 0, "The threshold must be a positive integer.")
     .createWithDefault(1024)
@@ -1030,6 +1137,7 @@ object SQLConf {
       .doc("When true, whole stage codegen would put the logic of consuming rows of each " +
         "physical operator into individual methods, instead of a single big method. This can be " +
         "used to avoid oversized function that can miss the opportunity of JIT optimization.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1037,6 +1145,7 @@ object SQLConf {
     .doc("The maximum number of bytes to pack into a single partition when reading files. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.0.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("128MB") // parquet.block.size
 
@@ -1047,6 +1156,7 @@ object SQLConf {
       " over estimated, then the partitions with small files will be faster than partitions with" +
       " bigger files (which is scheduled first). This configuration is effective only when using" +
       " file-based sources such as Parquet, JSON and ORC.")
+    .version("2.0.0")
     .longConf
     .createWithDefault(4 * 1024 * 1024)
 
@@ -1055,6 +1165,7 @@ object SQLConf {
       "encountering corrupted files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -1063,24 +1174,28 @@ object SQLConf {
       "encountering missing files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val MAX_RECORDS_PER_FILE = buildConf("spark.sql.files.maxRecordsPerFile")
     .doc("Maximum number of records to write out to a single file. " +
       "If this value is zero or negative, there is no limit.")
+    .version("2.2.0")
     .longConf
     .createWithDefault(0)
 
   val EXCHANGE_REUSE_ENABLED = buildConf("spark.sql.exchange.reuse")
     .internal()
     .doc("When true, the planner will try to find out duplicated exchanges and re-use them.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.reuseSubquery")
     .internal()
     .doc("When true, the planner will try to find out duplicated subqueries and re-use them.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1092,6 +1207,7 @@ object SQLConf {
           "be a subclass of StateStoreProvider, and must have a zero-arg constructor. " +
           "Note: For structured streaming, this configuration cannot be changed between query " +
           "restarts from the same checkpoint location.")
+      .version("2.3.0")
       .stringConf
       .createWithDefault(
         "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider")
@@ -1101,6 +1217,7 @@ object SQLConf {
       .internal()
       .doc("Minimum number of state store delta files that needs to be generated before they " +
         "consolidated into snapshots.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1108,24 +1225,28 @@ object SQLConf {
     buildConf("spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion")
       .internal()
       .doc("State format version used by flatMapGroupsWithState operation in a streaming query")
+      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
 
   val CHECKPOINT_LOCATION = buildConf("spark.sql.streaming.checkpointLocation")
     .doc("The default location for storing checkpoint data for streaming queries.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   val FORCE_DELETE_TEMP_CHECKPOINT_LOCATION =
     buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation")
       .doc("When true, enable temporary checkpoint locations force delete.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val MIN_BATCHES_TO_RETAIN = buildConf("spark.sql.streaming.minBatchesToRetain")
     .internal()
     .doc("The minimum number of batches that must be retained and made recoverable.")
+    .version("2.1.1")
     .intConf
     .createWithDefault(100)
 
@@ -1135,6 +1256,7 @@ object SQLConf {
       "loading from files. The value adjusts a trade-off between memory usage vs cache miss: " +
       "'2' covers both success and direct failure cases, '1' covers only success case, " +
       "and '0' covers extreme case - disable cache to maximize memory size of executors.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(2)
 
@@ -1144,6 +1266,7 @@ object SQLConf {
       .doc("State format version used by streaming aggregation operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
+      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1154,6 +1277,7 @@ object SQLConf {
       "If we find a concurrent active run for a streaming query (in the same or different " +
       "SparkSessions on the same cluster) and this flag is true, we will stop the old streaming " +
       "query run to start the new one.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1163,6 +1287,7 @@ object SQLConf {
       .doc("State format version used by streaming join operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
+      .version("3.0.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1172,6 +1297,7 @@ object SQLConf {
       .internal()
       .doc("When true, the logical plan for streaming query will be checked for unsupported" +
         " operations.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1179,6 +1305,7 @@ object SQLConf {
     buildConf("spark.sql.variable.substitute")
       .doc("This enables substitution using syntax like `${var}`, `${system:var}`, " +
         "and `${env:var}`.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1189,6 +1316,7 @@ object SQLConf {
         "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " +
         "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " +
         "When disabled, records go directly to the 2nd level.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1196,6 +1324,7 @@ object SQLConf {
     buildConf("spark.sql.codegen.aggregate.map.vectorized.enable")
       .internal()
       .doc("Enable vectorized aggregate hash map. This is for testing/benchmarking only.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1205,6 +1334,7 @@ object SQLConf {
       .doc("When true, the code generator would split aggregate code into individual methods " +
         "instead of a single big method. This can be used to avoid oversized function that " +
         "can miss the opportunity of JIT optimization.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1216,6 +1346,7 @@ object SQLConf {
         "However the DAG depth may become too large and cause unexpected behavior. This " +
         "configuration puts a limit on this: when the depth of a view exceeds this value during " +
         "analysis, we terminate the resolution to avoid potential errors.")
+      .version("2.2.0")
       .intConf
       .checkValue(depth => depth > 0, "The maximum depth of a view reference in a nested view " +
         "must be positive.")
@@ -1223,6 +1354,7 @@ object SQLConf {
 
   val STREAMING_FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.streaming.commitProtocolClass")
+      .version("2.1.0")
       .internal()
       .stringConf
       .createWithDefault("org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol")
@@ -1235,6 +1367,7 @@ object SQLConf {
         "'max' which chooses the maximum across multiple operators. " +
         "Note: This configuration cannot be changed between query restarts from the same " +
         "checkpoint location.")
+      .version("2.4.0")
       .stringConf
       .transform(_.toLowerCase(Locale.ROOT))
       .checkValue(
@@ -1249,6 +1382,7 @@ object SQLConf {
       .doc("In the case of ObjectHashAggregateExec, when the size of the in-memory hash map " +
         "grows too large, we will fall back to sort-based aggregation. This option sets a row " +
         "count threshold for the size of the hash map.")
+      .version("2.2.0")
       .intConf
       // We are trying to be conservative and use a relatively small default count threshold here
       // since the state object of some TypedImperativeAggregate function can be quite large (e.g.
@@ -1258,6 +1392,7 @@ object SQLConf {
   val USE_OBJECT_HASH_AGG = buildConf("spark.sql.execution.useObjectHashAggregateExec")
     .internal()
     .doc("Decides if we use ObjectHashAggregateExec")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1266,12 +1401,14 @@ object SQLConf {
       .doc("Whether to ignore null fields when generating JSON objects in JSON data source and " +
         "JSON functions such as to_json. " +
         "If false, it generates null for null fields in JSON objects.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SINK_LOG_DELETION = buildConf("spark.sql.streaming.fileSink.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream sink.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1280,6 +1417,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1287,12 +1425,14 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSink.log.cleanupDelay")
       .internal()
       .doc("How long that a file is guaranteed to be visible for all readers.")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
   val FILE_SOURCE_LOG_DELETION = buildConf("spark.sql.streaming.fileSource.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream source.")
+    .version("2.0.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -1301,6 +1441,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
+      .version("2.0.1")
       .intConf
       .createWithDefault(10)
 
@@ -1308,6 +1449,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSource.log.cleanupDelay")
       .internal()
       .doc("How long in milliseconds a file is guaranteed to be visible for all readers.")
+      .version("2.0.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
@@ -1317,12 +1459,14 @@ object SQLConf {
       .doc("When true, force the schema of streaming file source to be nullable (including all " +
         "the fields). Otherwise, the schema might not be compatible with actual data, which " +
         "leads to corruptions.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SOURCE_CLEANER_NUM_THREADS =
     buildConf("spark.sql.streaming.fileSource.cleaner.numThreads")
       .doc("Number of threads used in the file source completed file cleaner.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(1)
 
@@ -1330,6 +1474,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.schemaInference")
       .internal()
       .doc("Whether file-based streaming sources will infer its own schema")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1337,6 +1482,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.pollingDelay")
       .internal()
       .doc("How long to delay polling new data when no data is available")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10L)
 
@@ -1344,6 +1490,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.stopTimeout")
       .doc("How long to wait in milliseconds for the streaming execution thread to stop when " +
         "calling the streaming query's stop() method. 0 or negative values wait indefinitely.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
@@ -1351,6 +1498,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.noDataProgressEventInterval")
       .internal()
       .doc("How long to wait between two progress events when there is no data")
+      .version("2.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10000L)
 
@@ -1359,18 +1507,21 @@ object SQLConf {
       .doc(
         "Whether streaming micro-batch engine will execute batches without data " +
           "for eager state management for stateful streaming queries.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
   val STREAMING_METRICS_ENABLED =
     buildConf("spark.sql.streaming.metricsEnabled")
       .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.")
+      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   val STREAMING_PROGRESS_RETENTION =
     buildConf("spark.sql.streaming.numRecentProgressUpdates")
       .doc("The number of progress updates to retain for a streaming query")
+      .version("2.1.1")
       .intConf
       .createWithDefault(100)
 
@@ -1378,6 +1529,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpointFileManagerClass")
       .doc("The class used to write checkpoint files atomically. This class must be a subclass " +
         "of the interface CheckpointFileManager.")
+      .version("2.4.0")
       .internal()
       .stringConf
 
@@ -1385,6 +1537,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpoint.escapedPathCheck.enabled")
       .doc("Whether to detect a streaming query may pick up an incorrect checkpoint path due " +
         "to SPARK-26824.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -1395,6 +1548,7 @@ object SQLConf {
       .doc("When true, SQL commands use parallel file listing, " +
         "as opposed to single thread listing. " +
         "This usually speeds up commands that need to list many directories.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1404,6 +1558,7 @@ object SQLConf {
       s"which is larger than `${AUTO_BROADCASTJOIN_THRESHOLD.key}` to be more conservative. " +
       "That is to say by default the optimizer will not choose to broadcast a table unless it " +
       "knows for sure its size is small enough.")
+    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(Long.MaxValue)
 
@@ -1414,6 +1569,7 @@ object SQLConf {
       "For non-partitioned data source tables, it will be automatically recalculated if table " +
       "statistics are not available. For partitioned data source and partitioned Hive tables, " +
       s"It is '${DEFAULT_SIZE_IN_BYTES.key}' if table statistics are not available.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1422,6 +1578,7 @@ object SQLConf {
       .internal()
       .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
         "column level statistics.")
+      .version("2.1.1")
       .doubleConf
       .createWithDefault(0.05)
 
@@ -1432,6 +1589,7 @@ object SQLConf {
         "histogram. Note that collecting histograms takes extra cost. For example, collecting " +
         "column statistics usually takes only one table scan, but generating equi-height " +
         "histogram will cause an extra table scan.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1439,6 +1597,7 @@ object SQLConf {
     buildConf("spark.sql.statistics.histogram.numBins")
       .internal()
       .doc("The number of bins when generating histograms.")
+      .version("2.3.0")
       .intConf
       .checkValue(num => num > 1, "The number of bins must be greater than 1.")
       .createWithDefault(254)
@@ -1449,6 +1608,7 @@ object SQLConf {
       .doc("Accuracy of percentile approximation when generating equi-height histograms. " +
         "Larger value means better accuracy. The relative error can be deduced by " +
         "1.0 / PERCENTILE_ACCURACY.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1457,30 +1617,35 @@ object SQLConf {
       .doc("Enables automatic update for table size once table's data is changed. Note that if " +
         "the total number of files of the table is very large, this can be expensive and slow " +
         "down data change commands.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val CBO_ENABLED =
     buildConf("spark.sql.cbo.enabled")
       .doc("Enables CBO for estimation of plan statistics when set true.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val PLAN_STATS_ENABLED =
     buildConf("spark.sql.cbo.planStats.enabled")
       .doc("When true, the logical plan will fetch row counts and column statistics from catalog.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_ENABLED =
     buildConf("spark.sql.cbo.joinReorder.enabled")
       .doc("Enables join reorder in CBO.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_DP_THRESHOLD =
     buildConf("spark.sql.cbo.joinReorder.dp.threshold")
       .doc("The maximum number of joined nodes allowed in the dynamic programming algorithm.")
+      .version("2.2.0")
       .intConf
       .checkValue(number => number > 0, "The maximum number must be a positive integer.")
       .createWithDefault(12)
@@ -1490,6 +1655,7 @@ object SQLConf {
       .internal()
       .doc("The weight of cardinality (number of rows) for plan cost comparison in join reorder: " +
         "rows * weight + size * (1 - weight).")
+      .version("2.2.0")
       .doubleConf
       .checkValue(weight => weight >= 0 && weight <= 1, "The weight value must be in [0, 1].")
       .createWithDefault(0.7)
@@ -1497,11 +1663,13 @@ object SQLConf {
   val JOIN_REORDER_DP_STAR_FILTER =
     buildConf("spark.sql.cbo.joinReorder.dp.star.filter")
       .doc("Applies star-join filter heuristics to cost based join enumeration.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val STARSCHEMA_DETECTION = buildConf("spark.sql.cbo.starSchemaDetection")
     .doc("When true, it enables join reordering based on star schema detection. ")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1509,6 +1677,7 @@ object SQLConf {
     .internal()
     .doc("Specifies the upper limit of the ratio between the largest fact tables" +
       " for a star join to be considered. ")
+    .version("2.2.0")
     .doubleConf
     .createWithDefault(0.9)
 
@@ -1522,6 +1691,7 @@ object SQLConf {
       "Zone offsets must be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. " +
       "Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not " +
       "recommended to use because they can be ambiguous.")
+    .version("2.2.0")
     .stringConf
     .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
       " ZoneId.of(_, ZoneId.SHORT_IDS)")
@@ -1531,6 +1701,7 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.in.memory.threshold")
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the window operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1538,6 +1709,7 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by window operator")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1546,6 +1718,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the sort merge " +
         "join operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1553,6 +1726,7 @@ object SQLConf {
     buildConf("spark.sql.sortMergeJoinExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by sort merge join operator")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1561,6 +1735,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the cartesian " +
         "product operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1568,12 +1743,14 @@ object SQLConf {
     buildConf("spark.sql.cartesianProductExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by cartesian product operator")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
   val SUPPORT_QUOTED_REGEX_COLUMN_NAME = buildConf("spark.sql.parser.quotedRegexColumnNames")
     .doc("When true, quoted Identifiers (using backticks) in SELECT statement are interpreted" +
       " as regular expressions.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1582,12 +1759,14 @@ object SQLConf {
       .internal()
       .doc("Number of points to sample per partition in order to determine the range boundaries" +
           " for range partitioning, typically used in global sorting (without limit).")
+      .version("2.3.0")
       .intConf
       .createWithDefault(100)
 
   val ARROW_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.enabled")
       .doc("(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1599,6 +1778,7 @@ object SQLConf {
         "2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame " +
         "The following data types are unsupported: " +
         "BinaryType, MapType, ArrayType of TimestampType, and nested StructType.")
+      .version("3.0.0")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
   val ARROW_SPARKR_EXECUTION_ENABLED =
@@ -1611,6 +1791,7 @@ object SQLConf {
         "4. gapply " +
         "The following data types are unsupported: " +
         "FloatType, BinaryType, ArrayType, StructType and MapType.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1618,6 +1799,7 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.fallback.enabled")
       .doc("(Deprecated since Spark 3.0, please set " +
         "'spark.sql.execution.arrow.pyspark.fallback.enabled'.)")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1625,12 +1807,14 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.pyspark.fallback.enabled")
       .doc(s"When true, optimizations enabled by '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' will " +
         "fallback automatically to non-optimized implementations if an error occurs.")
+      .version("3.0.0")
       .fallbackConf(ARROW_FALLBACK_ENABLED)
 
   val ARROW_EXECUTION_MAX_RECORDS_PER_BATCH =
     buildConf("spark.sql.execution.arrow.maxRecordsPerBatch")
       .doc("When using Apache Arrow, limit the maximum number of records that can be written " +
         "to a single ArrowRecordBatch in memory. If set to zero or negative there is no limit.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1641,6 +1825,7 @@ object SQLConf {
         s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
         "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
         "pipelined; however, it might degrade performance. See SPARK-27870.")
+      .version("3.0.0")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
@@ -1650,6 +1835,7 @@ object SQLConf {
         "to use position if not. When false, a grouped map Pandas UDF will assign columns from " +
         "the returned Pandas DataFrame based on position, regardless of column label type. " +
         "This configuration will be deprecated in future releases.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1660,6 +1846,7 @@ object SQLConf {
         "Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
         "when detecting unsafe type conversion like overflow. When false, disabling Arrow's type " +
         "check and do type conversions anyway. This config only works for Arrow 0.11.0+.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1673,6 +1860,7 @@ object SQLConf {
       " column names of the left node are distinct. If all the conditions are met, the" +
       " rule will replace the except operation with a Filter by flipping the filter" +
       " condition(s) of the right node.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1683,6 +1871,7 @@ object SQLConf {
         "happens according to Hive behavior and SQL ANSI 2011 specification, ie. rounding the " +
         "decimal part of the result if an exact representation is not possible. Otherwise, NULL " +
         "is returned in those cases, as previously.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1692,6 +1881,7 @@ object SQLConf {
       .doc("When integral literal is used in decimal operations, pick a minimum precision " +
         "required by the literal if this config is true, to make the resulting precision and/or " +
         "scale smaller. This can reduce the possibility of precision lose and/or overflow.")
+      .version("2.3.3")
       .booleanConf
       .createWithDefault(true)
 
@@ -1701,6 +1891,7 @@ object SQLConf {
         "information. The values of options whose names that match this regex will be redacted " +
         "in the explain output. This redaction is applied on top of the global redaction " +
         s"configuration defined by ${SECRET_REDACTION_PATTERN.key}.")
+    .version("2.2.2")
     .regexConf
     .createWithDefault("(?i)url".r)
 
@@ -1710,17 +1901,20 @@ object SQLConf {
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands. " +
         "When this conf is not set, the value from `spark.redaction.string.regex` is used.")
+      .version("2.3.0")
       .fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN)
 
   val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString")
     .doc("When this option is set to false and all inputs are binary, `functions.concat` returns " +
       "an output as binary. Otherwise, it returns as a string.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString")
     .doc("When this option is set to false and all inputs are binary, `elt` returns " +
       "an output as binary. Otherwise, it returns as a string.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1731,6 +1925,7 @@ object SQLConf {
         "user-specified schema. If the validation fails, a runtime exception is thrown. " +
         "When this option is set to false, the partition column value will be converted to null " +
         "if it can not be casted to corresponding user-specified schema.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1738,6 +1933,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.continuous.epochBacklogQueueSize")
       .doc("The max number of entries to be stored in queue to wait for late epochs. " +
         "If this parameter is exceeded by the size of the queue, stream will stop with an error.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1746,6 +1942,7 @@ object SQLConf {
     .internal()
     .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
       " buffer the results of a ContinuousDataReader.")
+    .version("2.3.0")
     .intConf
     .createWithDefault(1024)
 
@@ -1754,6 +1951,7 @@ object SQLConf {
       .internal()
       .doc("The interval at which continuous execution readers will poll to check whether" +
         " the epoch has advanced on the driver.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
@@ -1762,12 +1960,14 @@ object SQLConf {
     .doc("A comma-separated list of data source short names or fully qualified data source " +
       "implementation class names for which Data Source V2 code path is disabled. These data " +
       "sources will fallback to Data Source V1 code path.")
+    .version("3.0.0")
     .stringConf
     .createWithDefault("avro,csv,json,kafka,orc,parquet,text")
 
   val DISABLED_V2_STREAMING_WRITERS = buildConf("spark.sql.streaming.disabledV2Writers")
     .doc("A comma-separated list of fully qualified data source register class names for which" +
       " StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.")
+    .version("2.3.1")
     .stringConf
     .createWithDefault("")
 
@@ -1778,6 +1978,7 @@ object SQLConf {
         "A comma-separated list of fully qualified data source register class names for which " +
           "MicroBatchReadSupport is disabled. Reads from these sources will fall back to the " +
           "V1 Sources.")
+      .version("2.4.0")
       .stringConf
       .createWithDefault("")
 
@@ -1798,6 +1999,7 @@ object SQLConf {
         "(which takes precedence over this setting), e.g. " +
         "dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path)."
       )
+      .version("2.3.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(PartitionOverwriteMode.values.map(_.toString))
@@ -1822,6 +2024,7 @@ object SQLConf {
         "in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is " +
         "not allowed."
       )
+      .version("3.0.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(StoreAssignmentPolicy.values.map(_.toString))
@@ -1832,6 +2035,7 @@ object SQLConf {
       "throw a runtime exception if an overflow occurs in any operation on integral/decimal " +
       "field. 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in " +
       "the SQL parser.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1844,6 +2048,7 @@ object SQLConf {
         "issues. Turn on this config to insert a local sort before actually doing repartition " +
         "to generate consistent repartition results. The performance of repartition() may go " +
         "down since we insert extra local sort before it.")
+        .version("2.1.4")
         .booleanConf
         .createWithDefault(true)
 
@@ -1854,6 +2059,7 @@ object SQLConf {
         "satisfying a query. This optimization allows columnar file format readers to avoid " +
         "reading unnecessary nested column data. Currently Parquet and ORC are the " +
         "data sources that implement this optimization.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1864,6 +2070,7 @@ object SQLConf {
         "containing `dots` to data sources. Currently, Parquet implements both optimizations " +
         "while ORC only supports predicates for names containing `dots`. The other data sources" +
         "don't support this feature yet.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1873,6 +2080,7 @@ object SQLConf {
       .doc("Prune nested fields from object serialization operator which are unnecessary in " +
         "satisfying a query. This optimization allows object serializers to avoid " +
         "executing unnecessary nested expressions.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1883,6 +2091,7 @@ object SQLConf {
         "satisfying a query. Note that this optimization doesn't prune nested fields from " +
         "physical data source scanning. For pruning nested fields from scanning, please use " +
         "`spark.sql.optimizer.nestedSchemaPruning.enabled` config.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1892,6 +2101,7 @@ object SQLConf {
       .doc("In SQL queries with a SORT followed by a LIMIT like " +
           "'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort" +
           " in memory, otherwise do a global sort which spills to disk if necessary.")
+      .version("2.4.0")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1907,6 +2117,7 @@ object SQLConf {
     .internal()
     .doc("If it is set to true, column names of the requested schema are passed to CSV parser. " +
       "Other column values can be ignored during parsing even if they are malformed.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1917,6 +2128,7 @@ object SQLConf {
       "Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python " +
       "REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned " +
       "outputs are showed similar to R data.frame would.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1925,12 +2137,14 @@ object SQLConf {
       s"effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true. The valid range of this " +
       "config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and " +
       "greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).")
+    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
   val REPL_EAGER_EVAL_TRUNCATE = buildConf("spark.sql.repl.eagerEval.truncate")
     .doc("The max number of characters for each cell that is returned by eager evaluation. " +
       s"This only takes effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
@@ -1941,6 +2155,7 @@ object SQLConf {
         "by the fast hash aggregate product operator. The bit is not for actual value, " +
         "but the actual numBuckets is determined by loadFactor " +
         "(e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).")
+      .version("2.4.0")
       .intConf
       .checkValue(bit => bit >= 10 && bit <= 30, "The bit value must be in [10, 30].")
       .createWithDefault(16)
@@ -1948,6 +2163,7 @@ object SQLConf {
   val AVRO_COMPRESSION_CODEC = buildConf("spark.sql.avro.compression.codec")
     .doc("Compression codec used in writing of AVRO files. Supported codecs: " +
       "uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.")
+    .version("2.4.0")
     .stringConf
     .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz"))
     .createWithDefault("snappy")
@@ -1956,6 +2172,7 @@ object SQLConf {
     .doc("Compression level for the deflate codec used in writing of AVRO files. " +
       "Valid value must be in the range of from 1 to 9 inclusive or -1. " +
       "The default value is -1 which corresponds to 6 level in the current implementation.")
+    .version("2.4.0")
     .intConf
     .checkValues((1 to 9).toSet + Deflater.DEFAULT_COMPRESSION)
     .createWithDefault(Deflater.DEFAULT_COMPRESSION)
@@ -1964,6 +2181,7 @@ object SQLConf {
     .internal()
     .doc(s"If it is set to false, or ${ANSI_ENABLED.key} is true, then size of null returns " +
       "null. Otherwise, it returns -1, which was inherited from Hive.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1972,6 +2190,7 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, the data source provider com.databricks.spark.avro is mapped " +
         "to the built-in but external Avro data source module for backward compatibility.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1982,6 +2201,7 @@ object SQLConf {
         "set operations are performed from left to right as they appear in the query. When set " +
         "to false and order of evaluation is not specified by parentheses, INTERSECT operations " +
         "are performed before any UNION, EXCEPT and MINUS operations.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1990,6 +2210,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, a literal with an exponent (e.g. 1E-30) would be parsed " +
         "as Decimal rather than Double.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1999,6 +2220,7 @@ object SQLConf {
       .doc("When set to true, negative scale of Decimal type is allowed. For example, " +
         "the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is " +
         "Decimal(11, 0) in non legacy mode.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2007,6 +2229,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, CREATE TABLE syntax without a provider will use hive " +
         s"instead of the value of ${DEFAULT_DATA_SOURCE_NAME.key}.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2015,14 +2238,16 @@ object SQLConf {
       .internal()
       .doc("When true, the bucketed table scan will list files during planning to figure out the " +
         "output ordering, which is expensive and may make the planning quite slow.")
-    .booleanConf
-    .createWithDefault(false)
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   val LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE =
     buildConf("spark.sql.legacy.parser.havingWithoutGroupByAsWhere")
       .internal()
       .doc("If it is set to true, the parser will treat HAVING without GROUP BY as a normal " +
         "WHERE, which does not follow SQL standard.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(false)
 
@@ -2030,7 +2255,8 @@ object SQLConf {
     buildConf("spark.sql.legacy.json.allowEmptyString.enabled")
       .internal()
       .doc("When set to true, the parser of JSON data source treats empty strings as null for " +
-      "some data types such as `IntegerType`.")
+        "some data types such as `IntegerType`.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2040,6 +2266,7 @@ object SQLConf {
       .doc("When set to true, Spark returns an empty collection with `StringType` as element " +
         "type if the `array`/`map` function is called without any parameters. Otherwise, Spark " +
         "returns an empty collection with `NullType` as element type.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2048,6 +2275,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, user is allowed to use org.apache.spark.sql.functions." +
         "udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2056,6 +2284,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, TRUNCATE TABLE command will not try to set back original " +
         "permission and ACLs when re-creating the table/partition paths.")
+      .version("2.4.6")
       .booleanConf
       .createWithDefault(false)
 
@@ -2065,6 +2294,7 @@ object SQLConf {
       .doc("When set to true, the key attribute resulted from running `Dataset.groupByKey` " +
         "for non-struct key type, will be named as `value`, following the behavior of Spark " +
         "version 2.4 and earlier.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2072,6 +2302,7 @@ object SQLConf {
     .doc("Maximum number of fields of sequence-like entries can be converted to strings " +
       "in debug output. Any elements beyond the limit will be dropped and replaced by a" +
       """ "... N more fields" placeholder.""")
+    .version("3.0.0")
     .intConf
     .createWithDefault(25)
 
@@ -2080,6 +2311,7 @@ object SQLConf {
       "longer, further output will be truncated.  The default setting always generates a full " +
       "plan.  Set this to a lower value such as 8k if plan strings are taking up too much " +
       "memory or are causing OutOfMemory errors in the driver or UI processes.")
+    .version("3.0.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(i => i >= 0 && i <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, "Invalid " +
       "value for 'spark.sql.maxPlanStringLength'.  Length must be a valid string length " +
@@ -2091,6 +2323,7 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, SET command will fail when the key is registered as " +
         "a SparkConf entry.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2099,6 +2332,7 @@ object SQLConf {
       "java.time.LocalDate classes of Java 8 API are used as external types for " +
       "Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp " +
       "and java.sql.Date are used for the same purpose.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2106,6 +2340,7 @@ object SQLConf {
     .doc("The max length of a file that can be read by the binary file data source. " +
       "Spark will fail fast and not attempt to read the file if its length exceeds this value. " +
       "The theoretical max is Int.MaxValue, though VMs might implement a smaller max.")
+    .version("3.0.0")
     .internal()
     .intConf
     .createWithDefault(Int.MaxValue)
@@ -2115,12 +2350,14 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
         "with String")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog")
     .doc("Name of the default catalog. This will be the current catalog if users have not " +
       "explicitly set the current catalog yet.")
+    .version("3.0.0")
     .stringConf
     .createWithDefault(SESSION_CATALOG_NAME)
 
@@ -2132,6 +2369,7 @@ object SQLConf {
         s"be loaded by the $SESSION_CATALOG_NAME, this catalog must also return the table " +
         s"metadata. To delegate operations to the $SESSION_CATALOG_NAME, implementations can " +
         "extend 'CatalogExtension'.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -2144,6 +2382,7 @@ object SQLConf {
       "MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query " +
       "fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted " +
       "at last takes precedence.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(MapKeyDedupPolicy.values.map(_.toString))
@@ -2152,6 +2391,7 @@ object SQLConf {
   val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
     .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2165,6 +2405,7 @@ object SQLConf {
       "CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, " +
       "AnalysisException is thrown while name conflict is detected in nested CTE. This config " +
       "will be removed in future versions and CORRECTED will be the only behavior.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2177,6 +2418,7 @@ object SQLConf {
       "When set to CORRECTED, classes from java.time.* packages are used for the same purpose. " +
       "The default value is EXCEPTION, RuntimeException is thrown when we will get different " +
       "results.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2186,6 +2428,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
       .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2194,6 +2437,7 @@ object SQLConf {
       .doc("A comma-delimited string config of the optional additional remote Maven mirror " +
         "repositories. This is only used for downloading Hive jars in IsolatedClientLoader " +
         "if the default Maven Central repo is unreachable.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(
         "https://maven-central.storage-download.googleapis.com/maven2/")
@@ -2206,6 +2450,7 @@ object SQLConf {
         "all interval units out of the specified range. If it is set to `false`, " +
         "`ParseException` is thrown if the input does not match to the pattern " +
         "defined by `from` and `to`.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2215,6 +2460,7 @@ object SQLConf {
       .doc("When true, all database and table properties are not reserved and available for " +
         "create/alter syntaxes. But please be aware that the reserved properties will be " +
         "silently removed.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2223,6 +2469,7 @@ object SQLConf {
       .internal()
       .doc("When true, only a single file can be added using ADD FILE. If false, then users " +
         "can add directory by passing directory path to ADD FILE.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2230,11 +2477,13 @@ object SQLConf {
     buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
       .internal()
       .doc("When true, use legacy MySqlServer SMALLINT and REAL type mapping.")
+      .version("2.4.5")
       .booleanConf
       .createWithDefault(false)
 
   val CSV_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.csv.filterPushdown.enabled")
     .doc("When true, enable filter pushdown to CSV datasource.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2244,6 +2493,7 @@ object SQLConf {
       .doc("The number of partitions to be handled in one turn when use " +
         "`AlterTableAddPartitionCommand` to add partitions into table. The smaller " +
         "batch size is, the less memory is required for the real handler, e.g. Hive Metastore.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
@@ -2251,6 +2501,7 @@ object SQLConf {
   val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2263,6 +2514,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2275,6 +2527,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2287,6 +2540,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2299,6 +2553,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 

From a5444db82ace1bfca32ec8d5b9f9823e91ac9511 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Sun, 5 Apr 2020 13:14:30 +0900
Subject: [PATCH 0409/1280] [SPARK-31002][CORE][DOC][3.0] Add version
 information to the configuration of Core

### What changes were proposed in this pull request?
This PR follows https://github.com/apache/spark/pull/27847 and used to spark3.0.

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test.

Closes #28118 from beliefer/backport-core-conf-version.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/package.scala       | 340 ++++++++++++++++--
 1 file changed, 312 insertions(+), 28 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 1308a460a2cc8..fe0e476cdd213 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -45,6 +45,7 @@ package object config {
         "custom implementation. Spark will try each class specified until one of them " +
         "returns the resource information for that resource. It tries the discovery " +
         "script last if none of the plugins return information for that resource.")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -55,89 +56,113 @@ package object config {
       .doc("Path to a file containing the resources allocated to the driver. " +
         "The file should be formatted as a JSON array of ResourceAllocation objects. " +
         "Only used internally in standalone mode.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val DRIVER_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS)
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val DRIVER_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.driver.userClassPathFirst").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.driver.userClassPathFirst")
+      .version("1.3.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DRIVER_CORES = ConfigBuilder("spark.driver.cores")
     .doc("Number of cores to use for the driver process, only in cluster mode.")
+    .version("1.3.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val DRIVER_MEMORY = ConfigBuilder(SparkLauncher.DRIVER_MEMORY)
     .doc("Amount of memory to use for the driver process, in MiB unless otherwise specified.")
+    .version("1.1.1")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per driver in cluster mode, " +
       "in MiB unless otherwise specified.")
+    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
   private[spark] val DRIVER_LOG_DFS_DIR =
-    ConfigBuilder("spark.driver.log.dfsDir").stringConf.createOptional
+    ConfigBuilder("spark.driver.log.dfsDir").version("3.0.0").stringConf.createOptional
 
   private[spark] val DRIVER_LOG_LAYOUT =
     ConfigBuilder("spark.driver.log.layout")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LOG_PERSISTTODFS =
     ConfigBuilder("spark.driver.log.persistToDfs.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DRIVER_LOG_ALLOW_EC =
     ConfigBuilder("spark.driver.log.allowErasureCoding")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLED = ConfigBuilder("spark.eventLog.enabled")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val EVENT_LOG_DIR = ConfigBuilder("spark.eventLog.dir")
+    .version("1.0.0")
     .stringConf
     .createWithDefault(EventLoggingListener.DEFAULT_LOG_DIR)
 
   private[spark] val EVENT_LOG_COMPRESS =
     ConfigBuilder("spark.eventLog.compress")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_BLOCK_UPDATES =
     ConfigBuilder("spark.eventLog.logBlockUpdates.enabled")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ALLOW_EC =
     ConfigBuilder("spark.eventLog.erasureCoding.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_TESTING =
     ConfigBuilder("spark.eventLog.testing")
       .internal()
+      .version("1.0.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_OUTPUT_BUFFER_SIZE = ConfigBuilder("spark.eventLog.buffer.kb")
     .doc("Buffer size to use when writing to output streams, in KiB unless otherwise specified.")
+    .version("1.0.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("100k")
 
@@ -145,6 +170,7 @@ package object config {
     ConfigBuilder("spark.eventLog.logStageExecutorMetrics")
       .doc("Whether to write per-stage peaks of executor metrics (for each executor) " +
         "to the event log.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -153,6 +179,7 @@ package object config {
       .doc("Names of supported young generation garbage collector. A name usually is " +
         " the return of GarbageCollectorMXBean.getName. The built-in young generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
@@ -162,20 +189,28 @@ package object config {
       .doc("Names of supported old generation garbage collector. A name usually is " +
         "the return of GarbageCollectorMXBean.getName. The built-in old generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
 
   private[spark] val EVENT_LOG_OVERWRITE =
-    ConfigBuilder("spark.eventLog.overwrite").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.eventLog.overwrite")
+      .version("1.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EVENT_LOG_CALLSITE_LONG_FORM =
-    ConfigBuilder("spark.eventLog.longForm.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.eventLog.longForm.enabled")
+      .version("2.4.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLE_ROLLING =
     ConfigBuilder("spark.eventLog.rolling.enabled")
       .doc("Whether rolling over event log files is enabled. If set to true, it cuts down " +
         "each event log file to the configured size.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -183,35 +218,46 @@ package object config {
     ConfigBuilder("spark.eventLog.rolling.maxFileSize")
       .doc(s"When ${EVENT_LOG_ENABLE_ROLLING.key}=true, specifies the max size of event log file" +
         " to be rolled over.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
         "configured to be at least 10 MiB.")
       .createWithDefaultString("128m")
 
   private[spark] val EXECUTOR_ID =
-    ConfigBuilder("spark.executor.id").stringConf.createOptional
+    ConfigBuilder("spark.executor.id").version("1.2.0").stringConf.createOptional
 
   private[spark] val EXECUTOR_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val EXECUTOR_HEARTBEAT_DROP_ZERO_ACCUMULATOR_UPDATES =
     ConfigBuilder("spark.executor.heartbeat.dropZeroAccumulatorUpdates")
       .internal()
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val EXECUTOR_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.executor.heartbeatInterval")
+      .version("1.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val EXECUTOR_HEARTBEAT_MAX_FAILURES =
-    ConfigBuilder("spark.executor.heartbeat.maxFailures").internal().intConf.createWithDefault(60)
+    ConfigBuilder("spark.executor.heartbeat.maxFailures")
+      .internal()
+      .version("1.6.2")
+      .intConf
+      .createWithDefault(60)
 
   private[spark] val EXECUTOR_PROCESS_TREE_METRICS_ENABLED =
     ConfigBuilder("spark.executor.processTreeMetrics.enabled")
       .doc("Whether to collect process tree metrics (from the /proc filesystem) when collecting " +
         "executor metrics.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -220,33 +266,44 @@ package object config {
       .doc("How often to collect executor metrics (in milliseconds). " +
         "If 0, the polling is done on executor heartbeats. " +
         "If positive, the polling is done at this interval.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
   private[spark] val EXECUTOR_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS)
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val EXECUTOR_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val EXECUTOR_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.executor.userClassPathFirst").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.executor.userClassPathFirst")
+      .version("1.3.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EXECUTOR_CORES = ConfigBuilder(SparkLauncher.EXECUTOR_CORES)
+    .version("1.0.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val EXECUTOR_MEMORY = ConfigBuilder(SparkLauncher.EXECUTOR_MEMORY)
     .doc("Amount of memory to use per executor process, in MiB unless otherwise specified.")
+    .version("0.7.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " +
       "in MiB unless otherwise specified.")
+    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
@@ -256,12 +313,14 @@ package object config {
       "the cluster (not from each machine). If not set, the default will be " +
       "`spark.deploy.defaultCores` on Spark's standalone cluster manager, or infinite " +
       "(all available cores) on Mesos.")
+    .version("0.6.0")
     .intConf
     .createOptional
 
   private[spark] val MEMORY_OFFHEAP_ENABLED = ConfigBuilder("spark.memory.offHeap.enabled")
     .doc("If true, Spark will attempt to use off-heap memory for certain operations. " +
       "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive.")
+    .version("1.6.0")
     .withAlternative("spark.unsafe.offHeap")
     .booleanConf
     .createWithDefault(false)
@@ -272,6 +331,7 @@ package object config {
       "This setting has no impact on heap memory usage, so if your executors' total memory " +
       "consumption must fit within some hard limit then be sure to shrink your JVM heap size " +
       "accordingly. This must be set to a positive value when spark.memory.offHeap.enabled=true.")
+    .version("1.6.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(_ >= 0, "The off-heap memory size must not be negative")
     .createWithDefault(0)
@@ -281,6 +341,7 @@ package object config {
       "size of the region set aside by spark.memory.fraction. The higher this is, the " +
       "less working memory may be available to execution and tasks may spill to disk more " +
       "often. Leaving this at the default value is recommended. ")
+    .version("1.6.0")
     .doubleConf
     .checkValue(v => v >= 0.0 && v < 1.0, "Storage fraction must be in [0,1)")
     .createWithDefault(0.5)
@@ -291,16 +352,19 @@ package object config {
       "The purpose of this config is to set aside memory for internal metadata, " +
       "user data structures, and imprecise size estimation in the case of sparse, " +
       "unusually large records. Leaving this at the default value is recommended.  ")
+    .version("1.6.0")
     .doubleConf
     .createWithDefault(0.6)
 
   private[spark] val STORAGE_SAFETY_FRACTION = ConfigBuilder("spark.storage.safetyFraction")
+    .version("1.1.0")
     .doubleConf
     .createWithDefault(0.9)
 
   private[spark] val STORAGE_UNROLL_MEMORY_THRESHOLD =
     ConfigBuilder("spark.storage.unrollMemoryThreshold")
       .doc("Initial memory to request before unrolling any block")
+      .version("1.1.0")
       .longConf
       .createWithDefault(1024 * 1024)
 
@@ -310,6 +374,7 @@ package object config {
         "Cached RDD block replicas lost due to executor failures are replenished " +
         "if there are any existing available replicas. This tries to " +
         "get the replication level of the block to the initial number")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -320,41 +385,54 @@ package object config {
         "This prevents Spark from memory mapping very small blocks. " +
         "In general, memory mapping has high overhead for blocks close to or below " +
         "the page size of the operating system.")
+      .version("0.9.2")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("2m")
 
   private[spark] val STORAGE_REPLICATION_POLICY =
     ConfigBuilder("spark.storage.replication.policy")
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[RandomBlockReplicationPolicy].getName)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_MAPPER =
     ConfigBuilder("spark.storage.replication.topologyMapper")
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[DefaultTopologyMapper].getName)
 
   private[spark] val STORAGE_CACHED_PEERS_TTL = ConfigBuilder("spark.storage.cachedPeersTtl")
-    .intConf.createWithDefault(60 * 1000)
+    .version("1.1.1")
+    .intConf
+    .createWithDefault(60 * 1000)
 
   private[spark] val STORAGE_MAX_REPLICATION_FAILURE =
     ConfigBuilder("spark.storage.maxReplicationFailures")
-      .intConf.createWithDefault(1)
+      .version("1.1.1")
+      .intConf
+      .createWithDefault(1)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_FILE =
-    ConfigBuilder("spark.storage.replication.topologyFile").stringConf.createOptional
+    ConfigBuilder("spark.storage.replication.topologyFile")
+      .version("2.1.0")
+      .stringConf
+      .createOptional
 
   private[spark] val STORAGE_EXCEPTION_PIN_LEAK =
     ConfigBuilder("spark.storage.exceptionOnPinLeak")
+      .version("1.6.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STORAGE_BLOCKMANAGER_TIMEOUTINTERVAL =
     ConfigBuilder("spark.storage.blockManagerTimeoutIntervalMs")
+      .version("0.7.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("60s")
 
   private[spark] val STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT =
     ConfigBuilder("spark.storage.blockManagerSlaveTimeoutMs")
+      .version("0.7.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString(Network.NETWORK_TIMEOUT.defaultValueString)
 
@@ -362,6 +440,7 @@ package object config {
     ConfigBuilder("spark.storage.cleanupFilesAfterExecutorExit")
       .doc("Whether or not cleanup the files not served by the external shuffle service " +
         "on executor exits.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -369,6 +448,7 @@ package object config {
     ConfigBuilder("spark.diskStore.subDirectories")
       .doc("Number of subdirectories inside each path listed in spark.local.dir for " +
         "hashing Block files into.")
+      .version("0.6.0")
       .intConf
       .checkValue(_ > 0, "The number of subdirectories must be positive.")
       .createWithDefault(64)
@@ -377,71 +457,101 @@ package object config {
     ConfigBuilder("spark.block.failures.beforeLocationRefresh")
       .doc("Max number of failures before this block manager refreshes " +
         "the block locations from the driver.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(5)
 
-  private[spark] val IS_PYTHON_APP = ConfigBuilder("spark.yarn.isPython").internal()
-    .booleanConf.createWithDefault(false)
+  private[spark] val IS_PYTHON_APP =
+    ConfigBuilder("spark.yarn.isPython")
+      .internal()
+      .version("1.5.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val CPUS_PER_TASK = ConfigBuilder("spark.task.cpus").intConf.createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_ENABLED =
-    ConfigBuilder("spark.dynamicAllocation.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.enabled")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_TESTING =
-    ConfigBuilder("spark.dynamicAllocation.testing").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.testing")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_MIN_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.minExecutors").intConf.createWithDefault(0)
+    ConfigBuilder("spark.dynamicAllocation.minExecutors")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(0)
 
   private[spark] val DYN_ALLOCATION_INITIAL_EXECUTORS =
     ConfigBuilder("spark.dynamicAllocation.initialExecutors")
+      .version("1.3.0")
       .fallbackConf(DYN_ALLOCATION_MIN_EXECUTORS)
 
   private[spark] val DYN_ALLOCATION_MAX_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.maxExecutors").intConf.createWithDefault(Int.MaxValue)
+    ConfigBuilder("spark.dynamicAllocation.maxExecutors")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(Int.MaxValue)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO =
     ConfigBuilder("spark.dynamicAllocation.executorAllocationRatio")
-      .doubleConf.createWithDefault(1.0)
+      .version("2.4.0")
+      .doubleConf
+      .createWithDefault(1.0)
 
   private[spark] val DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.cachedExecutorIdleTimeout")
+      .version("1.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.executorIdleTimeout")
+      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(60)
 
   private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING =
     ConfigBuilder("spark.dynamicAllocation.shuffleTracking.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_SHUFFLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.shuffleTimeout")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Long.MaxValue)
 
   private[spark] val DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.schedulerBacklogTimeout")
+      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS).createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.sustainedSchedulerBacklogTimeout")
+      .version("1.2.0")
       .fallbackConf(DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT)
 
   private[spark] val LOCALITY_WAIT = ConfigBuilder("spark.locality.wait")
+    .version("0.5.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("3s")
 
   private[spark] val SHUFFLE_SERVICE_ENABLED =
-    ConfigBuilder("spark.shuffle.service.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.shuffle.service.enabled")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val SHUFFLE_SERVICE_FETCH_RDD_ENABLED =
     ConfigBuilder(Constants.SHUFFLE_SERVICE_FETCH_RDD_ENABLED)
@@ -449,6 +559,7 @@ package object config {
         "In case of dynamic allocation if this feature is enabled executors having only disk " +
         "persisted blocks are considered idle after " +
         "'spark.dynamicAllocation.executorIdleTimeout' and will be released accordingly.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -456,21 +567,25 @@ package object config {
     ConfigBuilder("spark.shuffle.service.db.enabled")
       .doc("Whether to use db in ExternalShuffleService. Note that this only affects " +
         "standalone mode.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_SERVICE_PORT =
-    ConfigBuilder("spark.shuffle.service.port").intConf.createWithDefault(7337)
+    ConfigBuilder("spark.shuffle.service.port").version("1.2.0").intConf.createWithDefault(7337)
 
   private[spark] val KEYTAB = ConfigBuilder("spark.kerberos.keytab")
     .doc("Location of user's keytab.")
+    .version("3.0.0")
     .stringConf.createOptional
 
   private[spark] val PRINCIPAL = ConfigBuilder("spark.kerberos.principal")
     .doc("Name of the Kerberos principal.")
+    .version("3.0.0")
     .stringConf.createOptional
 
   private[spark] val KERBEROS_RELOGIN_PERIOD = ConfigBuilder("spark.kerberos.relogin.period")
+    .version("3.0.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("1m")
 
@@ -480,6 +595,7 @@ package object config {
         "Which credentials to use when renewing delegation tokens for executors. Can be either " +
         "'keytab', the default, which requires a keytab to be provided, or 'ccache', which uses " +
         "the local credentials cache.")
+      .version("3.0.0")
       .stringConf
       .checkValues(Set("keytab", "ccache"))
       .createWithDefault("keytab")
@@ -488,104 +604,124 @@ package object config {
     ConfigBuilder("spark.kerberos.access.hadoopFileSystems")
     .doc("Extra Hadoop filesystem URLs for which to request delegation tokens. The filesystem " +
       "that hosts fs.defaultFS does not need to be listed here.")
+    .version("3.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances")
+    .version("1.0.0")
     .intConf
     .createOptional
 
   private[spark] val PY_FILES = ConfigBuilder("spark.yarn.dist.pyFiles")
     .internal()
+    .version("2.2.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val TASK_MAX_DIRECT_RESULT_SIZE =
     ConfigBuilder("spark.task.maxDirectResultSize")
+      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1L << 20)
 
   private[spark] val TASK_MAX_FAILURES =
     ConfigBuilder("spark.task.maxFailures")
+      .version("0.8.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val TASK_REAPER_ENABLED =
     ConfigBuilder("spark.task.reaper.enabled")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val TASK_REAPER_KILL_TIMEOUT =
     ConfigBuilder("spark.task.reaper.killTimeout")
+      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(-1)
 
   private[spark] val TASK_REAPER_POLLING_INTERVAL =
     ConfigBuilder("spark.task.reaper.pollingInterval")
+      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val TASK_REAPER_THREAD_DUMP =
     ConfigBuilder("spark.task.reaper.threadDump")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(true)
 
   // Blacklist confs
   private[spark] val BLACKLIST_ENABLED =
     ConfigBuilder("spark.blacklist.enabled")
+      .version("2.1.0")
       .booleanConf
       .createOptional
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerExecutor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(1)
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerNode")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC =
     ConfigBuilder("spark.blacklist.application.maxFailedTasksPerExecutor")
+      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedTasksPerExecutor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE =
     ConfigBuilder("spark.blacklist.application.maxFailedExecutorsPerNode")
+      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedExecutorsPerNode")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val BLACKLIST_TIMEOUT_CONF =
     ConfigBuilder("spark.blacklist.timeout")
+      .version("2.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_KILL_ENABLED =
     ConfigBuilder("spark.blacklist.killBlacklistedExecutors")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val BLACKLIST_LEGACY_TIMEOUT_CONF =
     ConfigBuilder("spark.scheduler.executorTaskBlacklistTime")
       .internal()
+      .version("1.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_FETCH_FAILURE_ENABLED =
     ConfigBuilder("spark.blacklist.application.fetchFailure.enabled")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
   // End blacklist confs
@@ -595,6 +731,7 @@ package object config {
       .doc("Whether to un-register all the outputs on the host in condition that we receive " +
         " a FetchFailure. This is set default to false, which means, we only un-register the " +
         " outputs related to the exact executor(instead of the host) on a FetchFailure.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -604,6 +741,7 @@ package object config {
         "an event queue using capacity specified by `spark.scheduler.listenerbus" +
         ".eventqueue.queueName.capacity` first. If it's not configured, Spark will " +
         "use the default capacity specified by this config.")
+      .version("2.3.0")
       .intConf
       .checkValue(_ > 0, "The capacity of listener bus event queue must be positive")
       .createWithDefault(10000)
@@ -611,6 +749,7 @@ package object config {
   private[spark] val LISTENER_BUS_METRICS_MAX_LISTENER_CLASSES_TIMED =
     ConfigBuilder("spark.scheduler.listenerbus.metrics.maxListenerClassesTimed")
       .internal()
+      .version("2.3.0")
       .intConf
       .createWithDefault(128)
 
@@ -620,6 +759,7 @@ package object config {
       .doc("When enabled, log the event that takes too much time to process. This helps us " +
         "discover the event types that cause performance bottlenecks. The time threshold is " +
         "controlled by spark.scheduler.listenerbus.logSlowEvent.threshold.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -628,52 +768,65 @@ package object config {
       .internal()
       .doc("The time threshold of whether a event is considered to be taking too much time to " +
         s"process. Log the event if ${LISTENER_BUS_LOG_SLOW_EVENT_ENABLED.key} is true.")
+      .version("3.0.0")
       .timeConf(TimeUnit.NANOSECONDS)
       .createWithDefaultString("1s")
 
   // This property sets the root namespace for metrics reporting
   private[spark] val METRICS_NAMESPACE = ConfigBuilder("spark.metrics.namespace")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_CONF = ConfigBuilder("spark.metrics.conf")
+    .version("0.8.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_EXECUTORMETRICS_SOURCE_ENABLED =
     ConfigBuilder("spark.metrics.executorMetricsSource.enabled")
       .doc("Whether to register the ExecutorMetrics source with the metrics system.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val METRICS_STATIC_SOURCES_ENABLED =
     ConfigBuilder("spark.metrics.staticSources.enabled")
       .doc("Whether to register static sources with the metrics system.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val PYSPARK_DRIVER_PYTHON = ConfigBuilder("spark.pyspark.driver.python")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   // To limit how many applications are shown in the History Server summary ui
   private[spark] val HISTORY_UI_MAX_APPS =
-    ConfigBuilder("spark.history.ui.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
+    ConfigBuilder("spark.history.ui.maxApplications")
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val IO_ENCRYPTION_ENABLED = ConfigBuilder("spark.io.encryption.enabled")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IO_ENCRYPTION_KEYGEN_ALGORITHM =
     ConfigBuilder("spark.io.encryption.keygen.algorithm")
+      .version("2.1.0")
       .stringConf
       .createWithDefault("HmacSHA1")
 
   private[spark] val IO_ENCRYPTION_KEY_SIZE_BITS = ConfigBuilder("spark.io.encryption.keySizeBits")
+    .version("2.1.0")
     .intConf
     .checkValues(Set(128, 192, 256))
     .createWithDefault(128)
@@ -681,57 +834,68 @@ package object config {
   private[spark] val IO_CRYPTO_CIPHER_TRANSFORMATION =
     ConfigBuilder("spark.io.crypto.cipher.transformation")
       .internal()
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString("AES/CTR/NoPadding")
 
   private[spark] val DRIVER_HOST_ADDRESS = ConfigBuilder("spark.driver.host")
     .doc("Address of driver endpoints.")
+    .version("0.7.0")
     .stringConf
     .createWithDefault(Utils.localCanonicalHostName())
 
   private[spark] val DRIVER_PORT = ConfigBuilder("spark.driver.port")
     .doc("Port of driver endpoints.")
+    .version("0.7.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_SUPERVISE = ConfigBuilder("spark.driver.supervise")
     .doc("If true, restarts the driver automatically if it fails with a non-zero exit status. " +
       "Only has effect in Spark standalone mode or Mesos cluster deploy mode.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val DRIVER_BIND_ADDRESS = ConfigBuilder("spark.driver.bindAddress")
     .doc("Address where to bind network listen sockets on the driver.")
+    .version("2.1.0")
     .fallbackConf(DRIVER_HOST_ADDRESS)
 
   private[spark] val BLOCK_MANAGER_PORT = ConfigBuilder("spark.blockManager.port")
     .doc("Port to use for the block manager when a more specific setting is not provided.")
+    .version("1.1.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_BLOCK_MANAGER_PORT = ConfigBuilder("spark.driver.blockManager.port")
     .doc("Port to use for the block manager on the driver.")
+    .version("2.1.0")
     .fallbackConf(BLOCK_MANAGER_PORT)
 
   private[spark] val IGNORE_CORRUPT_FILES = ConfigBuilder("spark.files.ignoreCorruptFiles")
     .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " +
       "encountering corrupted or non-existing files and contents that have been read will still " +
       "be returned.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IGNORE_MISSING_FILES = ConfigBuilder("spark.files.ignoreMissingFiles")
     .doc("Whether to ignore missing files. If true, the Spark jobs will continue to run when " +
       "encountering missing files and the contents that have been read will still be returned.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val APP_CALLER_CONTEXT = ConfigBuilder("spark.log.callerContext")
+    .version("2.2.0")
     .stringConf
     .createOptional
 
   private[spark] val FILES_MAX_PARTITION_BYTES = ConfigBuilder("spark.files.maxPartitionBytes")
     .doc("The maximum number of bytes to pack into a single partition when reading files.")
+    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(128 * 1024 * 1024)
 
@@ -740,6 +904,7 @@ package object config {
       " the same time. This is used when putting multiple files into a partition. It's better to" +
       " over estimate, then the partitions with small files will be faster than partitions with" +
       " bigger files.")
+    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(4 * 1024 * 1024)
 
@@ -747,6 +912,7 @@ package object config {
     ConfigBuilder("spark.hadoopRDD.ignoreEmptySplits")
       .internal()
       .doc("When true, HadoopRDD/NewHadoopRDD will not create partitions for empty input splits.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -756,6 +922,7 @@ package object config {
         "driver and executor environments contain sensitive information. When this regex matches " +
         "a property key or value, the value is redacted from the environment UI and various logs " +
         "like YARN and event logs.")
+      .version("2.1.2")
       .regexConf
       .createWithDefault("(?i)secret|password|token".r)
 
@@ -764,26 +931,31 @@ package object config {
       .doc("Regex to decide which parts of strings produced by Spark contain sensitive " +
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands.")
+      .version("2.2.0")
       .regexConf
       .createOptional
 
   private[spark] val AUTH_SECRET =
     ConfigBuilder("spark.authenticate.secret")
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val AUTH_SECRET_BIT_LENGTH =
     ConfigBuilder("spark.authenticate.secretBitLength")
+      .version("1.6.0")
       .intConf
       .createWithDefault(256)
 
   private[spark] val NETWORK_AUTH_ENABLED =
     ConfigBuilder("spark.authenticate")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SASL_ENCRYPTION_ENABLED =
     ConfigBuilder("spark.authenticate.enableSaslEncryption")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -793,6 +965,7 @@ package object config {
         "loaded from this path on both the driver and the executors if overrides are not set for " +
         "either entity (see below). File-based secret keys are only allowed when using " +
         "Kubernetes.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -805,6 +978,7 @@ package object config {
         "be specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
+      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val AUTH_SECRET_FILE_EXECUTOR =
@@ -816,12 +990,14 @@ package object config {
         "specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
+      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val BUFFER_WRITE_CHUNK_SIZE =
     ConfigBuilder("spark.buffer.write.chunkSize")
       .internal()
       .doc("The chunk size in bytes during writing out the bytes of ChunkedByteBuffer.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         "The chunk size during writing out the bytes of ChunkedByteBuffer should" +
@@ -832,6 +1008,7 @@ package object config {
     ConfigBuilder("spark.checkpoint.compress")
       .doc("Whether to compress RDD checkpoints. Generally a good idea. Compression will use " +
         "spark.io.compression.codec.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -842,6 +1019,7 @@ package object config {
         "Caching preferred locations can relieve query loading to DFS and save the query " +
         "time. The drawback is that the cached locations can be possibly outdated and " +
         "lose data locality. If this config is not specified, it will not cache.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MINUTES)
       .checkValue(_ > 0, "The expire time for caching preferred locations cannot be non-positive.")
       .createOptional
@@ -851,12 +1029,14 @@ package object config {
       .doc("Threshold in bytes above which the size of shuffle blocks in " +
         "HighlyCompressedMapStatus is accurately recorded. This helps to prevent OOM " +
         "by avoiding underestimating shuffle block size when fetch shuffle blocks.")
+      .version("2.2.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(100 * 1024 * 1024)
 
   private[spark] val SHUFFLE_REGISTRATION_TIMEOUT =
     ConfigBuilder("spark.shuffle.registration.timeout")
       .doc("Timeout in milliseconds for registration to the external shuffle service.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(5000)
 
@@ -864,6 +1044,7 @@ package object config {
     ConfigBuilder("spark.shuffle.registration.maxAttempts")
       .doc("When we fail to register to the external shuffle service, we will " +
         "retry for maxAttempts times.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(3)
 
@@ -874,6 +1055,7 @@ package object config {
         "address in a single fetch or simultaneously, this could crash the serving executor or " +
         "Node Manager. This is especially useful to reduce the load on the Node Manager when " +
         "external shuffle is enabled. You can mitigate the issue by setting it to a lower value.")
+      .version("2.2.1")
       .intConf
       .checkValue(_ > 0, "The max no. of blocks in flight cannot be non-positive.")
       .createWithDefault(Int.MaxValue)
@@ -885,6 +1067,7 @@ package object config {
         "configuration will affect both shuffle fetch and block manager remote block fetch. " +
         "For users who enabled external shuffle service, this feature can only work when " +
         "external shuffle service is at least 2.3.0.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       // fetch-to-mem is guaranteed to fail if the message is bigger than 2 GB, so we might
       // as well use fetch-to-disk in that case.  The message includes some metadata in addition
@@ -900,12 +1083,14 @@ package object config {
       .doc("Enable tracking of updatedBlockStatuses in the TaskMetrics. Off by default since " +
         "tracking the block statuses can use a lot of memory and its not used anywhere within " +
         "spark.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_IO_PLUGIN_CLASS =
     ConfigBuilder("spark.shuffle.sort.io.plugin.class")
       .doc("Name of the class to use for shuffle IO.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(classOf[LocalDiskShuffleDataIO].getName)
 
@@ -914,6 +1099,7 @@ package object config {
       .doc("Size of the in-memory buffer for each shuffle file output stream, in KiB unless " +
         "otherwise specified. These buffers reduce the number of disk seeks and system calls " +
         "made in creating intermediate shuffle files.")
+      .version("1.4.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The file buffer size must be positive and less than or equal to" +
@@ -924,6 +1110,7 @@ package object config {
     ConfigBuilder("spark.shuffle.unsafe.file.output.buffer")
       .doc("The file system for this buffer size after each partition " +
         "is written in unsafe shuffle writer. In KiB unless otherwise specified.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The buffer size must be positive and less than or equal to" +
@@ -933,6 +1120,7 @@ package object config {
   private[spark] val SHUFFLE_DISK_WRITE_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.spill.diskWriteBufferSize")
       .doc("The buffer size, in bytes, to use when writing the sorted records to an on-disk file.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 12 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         s"The buffer size must be greater than 12 and less than or equal to " +
@@ -944,6 +1132,7 @@ package object config {
       .internal()
       .doc("The memory check period is used to determine how often we should check whether "
         + "there is a need to request more memory when we try to unroll the given block in memory.")
+      .version("2.3.0")
       .longConf
       .createWithDefault(16)
 
@@ -951,6 +1140,7 @@ package object config {
     ConfigBuilder("spark.storage.unrollMemoryGrowthFactor")
       .internal()
       .doc("Memory to request as a multiple of the size that used to unroll the block.")
+      .version("2.3.0")
       .doubleConf
       .createWithDefault(1.5)
 
@@ -961,12 +1151,14 @@ package object config {
         "where the YARN service does not support schemes that are supported by Spark, like http, " +
         "https and ftp, or jars required to be in the local YARN client's classpath. Wildcard " +
         "'*' is denoted to download resources for all the schemes.")
+      .version("2.3.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val EXTRA_LISTENERS = ConfigBuilder("spark.extraListeners")
     .doc("Class names of listeners to add to SparkContext during initialization.")
+    .version("1.3.0")
     .stringConf
     .toSequence
     .createOptional
@@ -978,6 +1170,7 @@ package object config {
         "By default it's Integer.MAX_VALUE, which means we never force the sorter to spill, " +
         "until we reach some limitations, like the max page size limitation for the pointer " +
         "array in the sorter.")
+      .version("1.6.0")
       .intConf
       .createWithDefault(Integer.MAX_VALUE)
 
@@ -987,30 +1180,35 @@ package object config {
       .doc("Multi-thread is used when the number of mappers * shuffle partitions is greater than " +
         "or equal to this threshold. Note that the actual parallelism is calculated by number of " +
         "mappers * shuffle partitions / this threshold + 1, so this threshold should be positive.")
+      .version("2.3.0")
       .intConf
       .checkValue(v => v > 0, "The threshold should be positive.")
       .createWithDefault(10000000)
 
   private[spark] val MAX_RESULT_SIZE = ConfigBuilder("spark.driver.maxResultSize")
     .doc("Size limit for results.")
+    .version("1.2.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("1g")
 
   private[spark] val CREDENTIALS_RENEWAL_INTERVAL_RATIO =
     ConfigBuilder("spark.security.credentials.renewalRatio")
       .doc("Ratio of the credential's expiration time when Spark should fetch new credentials.")
+      .version("2.4.0")
       .doubleConf
       .createWithDefault(0.75d)
 
   private[spark] val CREDENTIALS_RENEWAL_RETRY_WAIT =
     ConfigBuilder("spark.security.credentials.retryWait")
       .doc("How long to wait before retrying to fetch new credentials after a failure.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("1h")
 
   private[spark] val SHUFFLE_SORT_INIT_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.sort.initialBufferSize")
       .internal()
+      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 0 && v <= Int.MaxValue,
         s"The buffer size must be greater than 0 and less than or equal to ${Int.MaxValue}.")
@@ -1020,6 +1218,7 @@ package object config {
     ConfigBuilder("spark.shuffle.compress")
       .doc("Whether to compress shuffle output. Compression will use " +
         "spark.io.compression.codec.")
+      .version("0.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1027,6 +1226,7 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.compress")
       .doc("Whether to compress data spilled during shuffles. Compression will use " +
         "spark.io.compression.codec.")
+      .version("0.9.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1036,6 +1236,7 @@ package object config {
       .doc("The codec used to compress MapStatus, which is generated by ShuffleMapTask. " +
         "By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also " +
         "use fully qualified class names to specify the codec.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault("zstd")
 
@@ -1044,6 +1245,7 @@ package object config {
       .internal()
       .doc("Initial threshold for the size of a collection before we start tracking its " +
         "memory usage.")
+      .version("1.1.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(5 * 1024 * 1024)
 
@@ -1051,6 +1253,7 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.batchSize")
       .internal()
       .doc("Size of object batches when reading/writing from serializers.")
+      .version("0.9.0")
       .longConf
       .createWithDefault(10000)
 
@@ -1058,34 +1261,40 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.bypassMergeThreshold")
       .doc("In the sort-based shuffle manager, avoid merge-sorting data if there is no " +
         "map-side aggregation and there are at most this many reduce partitions")
+      .version("1.1.1")
       .intConf
       .createWithDefault(200)
 
   private[spark] val SHUFFLE_MANAGER =
     ConfigBuilder("spark.shuffle.manager")
+      .version("1.1.0")
       .stringConf
       .createWithDefault("sort")
 
   private[spark] val SHUFFLE_REDUCE_LOCALITY_ENABLE =
     ConfigBuilder("spark.shuffle.reduceLocality.enabled")
       .doc("Whether to compute locality preferences for reduce tasks")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST =
     ConfigBuilder("spark.shuffle.mapOutput.minSizeForBroadcast")
       .doc("The size at which we use Broadcast to send the map output statuses to the executors.")
+      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("512k")
 
   private[spark] val SHUFFLE_MAPOUTPUT_DISPATCHER_NUM_THREADS =
     ConfigBuilder("spark.shuffle.mapOutput.dispatcher.numThreads")
+      .version("2.0.0")
       .intConf
       .createWithDefault(8)
 
   private[spark] val SHUFFLE_DETECT_CORRUPT =
     ConfigBuilder("spark.shuffle.detectCorrupt")
       .doc("Whether to detect any corruption in fetched blocks.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1095,18 +1304,21 @@ package object config {
         "by using extra memory to detect early corruption. Any IOException thrown will cause " +
         "the task to be retried once and if it fails again with same exception, then " +
         "FetchFailedException will be thrown to retry previous stage")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_SYNC =
     ConfigBuilder("spark.shuffle.sync")
       .doc("Whether to force outstanding writes to disk.")
+      .version("0.8.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_UNSAFE_FAST_MERGE_ENABLE =
     ConfigBuilder("spark.shuffle.unsafe.fastMergeEnabled")
       .doc("Whether to perform a fast spill merge.")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1114,6 +1326,7 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.useRadixSort")
       .doc("Whether to use radix sort for sorting in-memory partition ids. Radix sort is much " +
         "faster, but requires additional memory to be reserved memory as pointers are added.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1121,6 +1334,7 @@ package object config {
     ConfigBuilder("spark.shuffle.minNumPartitionsToHighlyCompress")
       .internal()
       .doc("Number of partitions to determine if MapStatus should use HighlyCompressedMapStatus")
+      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The value should be a positive integer.")
       .createWithDefault(2000)
@@ -1130,6 +1344,7 @@ package object config {
       .doc("Whether to use the old protocol while doing the shuffle block fetching. " +
         "It is only enabled while we need the compatibility in the scenario of new Spark " +
         "version job fetching shuffle blocks from old version external shuffle service.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1139,6 +1354,7 @@ package object config {
         s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " +
         "blocks requested from those block managers which are running on the same host are read " +
         "from the disk directly instead of being fetched as remote blocks over the network.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1149,6 +1365,7 @@ package object config {
         "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
         s"persisted RDD blocks or shuffle blocks " +
         s"(when `${SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED.key}` is set) from the same host.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(1000)
 
@@ -1156,6 +1373,7 @@ package object config {
     ConfigBuilder("spark.storage.memoryMapLimitForTests")
       .internal()
       .doc("For testing only, controls the size of chunks when memory mapping a file")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1165,6 +1383,7 @@ package object config {
         "coordinator didn't receive all the sync messages from barrier tasks within the " +
         "configured time, throw a SparkException to fail all the tasks. The default value is set " +
         "to 31536000(3600 * 24 * 365) so the barrier() call shall wait for one year.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v > 0, "The value should be a positive time value.")
       .createWithDefaultString("365d")
@@ -1173,6 +1392,7 @@ package object config {
     ConfigBuilder("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
       .doc("The timeout in seconds to wait to acquire a new executor and schedule a task " +
         "before aborting a TaskSet which is unschedulable because of being completely blacklisted.")
+      .version("2.4.1")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v >= 0, "The value should be a non negative time value.")
       .createWithDefault(120)
@@ -1187,6 +1407,7 @@ package object config {
         "configured max failure times for a job then fail current job submission. Note this " +
         "config only applies to jobs that contain one or more barrier stages, we won't perform " +
         "the check on non-barrier jobs.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("15s")
 
@@ -1200,6 +1421,7 @@ package object config {
         "max failure times for a job then fail current job submission. Note this config only " +
         "applies to jobs that contain one or more barrier stages, we won't perform the check on " +
         "non-barrier jobs.")
+      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The max failures should be a positive value.")
       .createWithDefault(40)
@@ -1207,18 +1429,21 @@ package object config {
   private[spark] val UNSAFE_EXCEPTION_ON_MEMORY_LEAK =
     ConfigBuilder("spark.unsafe.exceptionOnMemoryLeak")
       .internal()
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val UNSAFE_SORTER_SPILL_READ_AHEAD_ENABLED =
     ConfigBuilder("spark.unsafe.sorter.spill.read.ahead.enabled")
       .internal()
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val UNSAFE_SORTER_SPILL_READER_BUFFER_SIZE =
     ConfigBuilder("spark.unsafe.sorter.spill.reader.buffer.size")
       .internal()
+      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => 1024 * 1024 <= v && v <= MAX_BUFFER_SIZE_BYTES,
         s"The value must be in allowed range [1,048,576, ${MAX_BUFFER_SIZE_BYTES}].")
@@ -1231,63 +1456,83 @@ package object config {
       .withPrepended(DEFAULT_PLUGINS_LIST, separator = ",")
       .doc("Comma-separated list of class names implementing " +
         "org.apache.spark.api.plugin.SparkPlugin to load into the application.")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val CLEANER_PERIODIC_GC_INTERVAL =
     ConfigBuilder("spark.cleaner.periodicGC.interval")
+      .version("1.6.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("30min")
 
   private[spark] val CLEANER_REFERENCE_TRACKING =
     ConfigBuilder("spark.cleaner.referenceTracking")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING_SHUFFLE =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking.shuffle")
+      .version("1.1.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_CLEAN_CHECKPOINTS =
     ConfigBuilder("spark.cleaner.referenceTracking.cleanCheckpoints")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_STRATEGY =
-    ConfigBuilder("spark.executor.logs.rolling.strategy").stringConf.createWithDefault("")
+    ConfigBuilder("spark.executor.logs.rolling.strategy")
+      .version("1.1.0")
+      .stringConf
+      .createWithDefault("")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_TIME_INTERVAL =
-    ConfigBuilder("spark.executor.logs.rolling.time.interval").stringConf.createWithDefault("daily")
+    ConfigBuilder("spark.executor.logs.rolling.time.interval")
+      .version("1.1.0")
+      .stringConf
+      .createWithDefault("daily")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_SIZE =
     ConfigBuilder("spark.executor.logs.rolling.maxSize")
+      .version("1.4.0")
       .stringConf
       .createWithDefault((1024 * 1024).toString)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_RETAINED_FILES =
-    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles").intConf.createWithDefault(-1)
+    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles")
+      .version("1.1.0")
+      .intConf
+      .createWithDefault(-1)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_ENABLE_COMPRESSION =
     ConfigBuilder("spark.executor.logs.rolling.enableCompression")
+      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_ENABLED = ConfigBuilder("spark.master.rest.enabled")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_PORT = ConfigBuilder("spark.master.rest.port")
+    .version("1.3.0")
     .intConf
     .createWithDefault(6066)
 
   private[spark] val MASTER_UI_PORT = ConfigBuilder("spark.master.ui.port")
+    .version("1.1.0")
     .intConf
     .createWithDefault(8080)
 
@@ -1296,6 +1541,7 @@ package object config {
       .doc("Block size in bytes used in Snappy compression, in the case when " +
         "Snappy compression codec is used. Lowering this block size " +
         "will also lower shuffle memory usage when Snappy is used")
+      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1304,6 +1550,7 @@ package object config {
       .doc("Block size in bytes used in LZ4 compression, in the case when LZ4 compression" +
         "codec is used. Lowering this block size will also lower shuffle memory " +
         "usage when LZ4 is used.")
+      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1313,6 +1560,7 @@ package object config {
         "broadcast variables and shuffle outputs. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec")
+      .version("0.8.0")
       .stringConf
       .createWithDefaultString("lz4")
 
@@ -1322,6 +1570,7 @@ package object config {
         "compression codec is used. Lowering this size will lower the shuffle " +
         "memory usage when Zstd is used, but it might increase the compression " +
         "cost because of excessive JNI call overhead")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1329,6 +1578,7 @@ package object config {
     ConfigBuilder("spark.io.compression.zstd.level")
       .doc("Compression level for Zstd compression codec. Increasing the compression " +
         "level will result in better compression at the expense of more CPU and memory")
+      .version("2.3.0")
       .intConf
       .createWithDefault(1)
 
@@ -1337,6 +1587,7 @@ package object config {
       .internal()
       .doc("If the size in bytes of a file loaded by Spark exceeds this threshold, " +
         "a warning is logged with the possible reasons.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1024 * 1024 * 1024)
 
@@ -1345,28 +1596,34 @@ package object config {
       .doc("The codec used to compress event log. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec. If this is not given, spark.io.compression.codec will be used.")
+      .version("3.0.0")
       .fallbackConf(IO_COMPRESSION_CODEC)
 
   private[spark] val BUFFER_SIZE =
     ConfigBuilder("spark.buffer.size")
+      .version("0.5.0")
       .intConf
       .checkValue(_ >= 0, "The buffer size must not be negative")
       .createWithDefault(65536)
 
   private[spark] val LOCALITY_WAIT_PROCESS = ConfigBuilder("spark.locality.wait.process")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_NODE = ConfigBuilder("spark.locality.wait.node")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_RACK = ConfigBuilder("spark.locality.wait.rack")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
-    private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
+  private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
     .doc("Maximum size of map outputs to fetch simultaneously from each reduce task, " +
       "in MiB unless otherwise specified. Since each output requires us to create a " +
       "buffer to receive it, this represents a fixed memory overhead per reduce task, " +
       "so keep it small unless you have a large amount of memory")
+    .version("1.4.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("48m")
 
@@ -1376,12 +1633,14 @@ package object config {
       "it might lead to very large number of inbound connections to one or more nodes, " +
       "causing the workers to fail under load. By allowing it to limit the number of " +
       "fetch requests, this scenario can be mitigated")
+    .version("2.0.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
   private[spark] val BROADCAST_COMPRESS = ConfigBuilder("spark.broadcast.compress")
     .doc("Whether to compress broadcast variables before sending them. " +
       "Generally a good idea. Compression will use spark.io.compression.codec")
+    .version("0.6.0")
     .booleanConf.createWithDefault(true)
 
   private[spark] val BROADCAST_BLOCKSIZE = ConfigBuilder("spark.broadcast.blockSize")
@@ -1389,6 +1648,7 @@ package object config {
       "KiB unless otherwise specified. Too large a value decreases " +
       "parallelism during broadcast (makes it slower); however, " +
       "if it is too small, BlockManager might take a performance hit")
+    .version("0.5.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("4m")
 
@@ -1398,12 +1658,14 @@ package object config {
       "corrupted blocks, at the cost of computing and sending a little " +
       "more data. It's possible to disable it if the network has other " +
       "mechanisms to guarantee data won't be corrupted during broadcast")
+    .version("2.1.1")
     .booleanConf.createWithDefault(true)
 
   private[spark] val BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD =
     ConfigBuilder("spark.broadcast.UDFCompressionThreshold")
       .doc("The threshold at which user-defined functions (UDFs) and Python RDD commands " +
         "are compressed by broadcast in bytes unless otherwise specified")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v >= 0, "The threshold should be non-negative.")
       .createWithDefault(1L * 1024 * 1024)
@@ -1414,92 +1676,111 @@ package object config {
       "or StorageLevel.MEMORY_ONLY in Python). Can save substantial " +
       "space at the cost of some extra CPU time. " +
       "Compression will use spark.io.compression.codec")
+    .version("0.6.0")
     .booleanConf.createWithDefault(false)
 
   private[spark] val RDD_PARALLEL_LISTING_THRESHOLD =
     ConfigBuilder("spark.rdd.parallelListingThreshold")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
   private[spark] val RDD_LIMIT_SCALE_UP_FACTOR =
     ConfigBuilder("spark.rdd.limit.scaleUpFactor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val SERIALIZER = ConfigBuilder("spark.serializer")
+    .version("0.5.0")
     .stringConf
     .createWithDefault("org.apache.spark.serializer.JavaSerializer")
 
   private[spark] val SERIALIZER_OBJECT_STREAM_RESET =
     ConfigBuilder("spark.serializer.objectStreamReset")
+      .version("1.0.0")
       .intConf
       .createWithDefault(100)
 
   private[spark] val SERIALIZER_EXTRA_DEBUG_INFO = ConfigBuilder("spark.serializer.extraDebugInfo")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
   private[spark] val JARS = ConfigBuilder("spark.jars")
+    .version("0.9.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val FILES = ConfigBuilder("spark.files")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SUBMIT_DEPLOY_MODE = ConfigBuilder("spark.submit.deployMode")
+    .version("1.5.0")
     .stringConf
     .createWithDefault("client")
 
   private[spark] val SUBMIT_PYTHON_FILES = ConfigBuilder("spark.submit.pyFiles")
+    .version("1.0.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SCHEDULER_ALLOCATION_FILE =
     ConfigBuilder("spark.scheduler.allocation.file")
+      .version("0.8.1")
       .stringConf
       .createOptional
 
   private[spark] val SCHEDULER_MIN_REGISTERED_RESOURCES_RATIO =
     ConfigBuilder("spark.scheduler.minRegisteredResourcesRatio")
+      .version("1.1.1")
       .doubleConf
       .createOptional
 
   private[spark] val SCHEDULER_MAX_REGISTERED_RESOURCE_WAITING_TIME =
     ConfigBuilder("spark.scheduler.maxRegisteredResourcesWaitingTime")
+      .version("1.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("30s")
 
   private[spark] val SCHEDULER_MODE =
     ConfigBuilder("spark.scheduler.mode")
+      .version("0.8.0")
       .stringConf
       .createWithDefault(SchedulingMode.FIFO.toString)
 
   private[spark] val SCHEDULER_REVIVE_INTERVAL =
     ConfigBuilder("spark.scheduler.revive.interval")
+      .version("0.8.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val SPECULATION_ENABLED =
     ConfigBuilder("spark.speculation")
+      .version("0.6.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SPECULATION_INTERVAL =
     ConfigBuilder("spark.speculation.interval")
+      .version("0.6.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
   private[spark] val SPECULATION_MULTIPLIER =
     ConfigBuilder("spark.speculation.multiplier")
+      .version("0.6.0")
       .doubleConf
       .createWithDefault(1.5)
 
   private[spark] val SPECULATION_QUANTILE =
     ConfigBuilder("spark.speculation.quantile")
+      .version("0.6.0")
       .doubleConf
       .createWithDefault(0.75)
 
@@ -1513,16 +1794,19 @@ package object config {
         "large enough. E.g. tasks might be re-launched if there are enough successful runs " +
         "even though the threshold hasn't been reached. The number of slots is computed based " +
         "on the conf values of spark.executor.cores and spark.task.cpus minimum 1.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val STAGING_DIR = ConfigBuilder("spark.yarn.stagingDir")
     .doc("Staging directory used while submitting applications.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   private[spark] val BUFFER_PAGESIZE = ConfigBuilder("spark.buffer.pageSize")
     .doc("The amount of memory used per page in bytes")
+    .version("1.5.0")
     .bytesConf(ByteUnit.BYTE)
     .createOptional
 

From d7846cbc670bc7daa8e1520eedf80410c8943722 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 25 Feb 2020 11:39:11 +0900
Subject: [PATCH 0410/1280] [SPARK-30887][CORE][DOC] Add version information to
 the configuration of Deploy

### What changes were proposed in this pull request?
1.Add version information to the configuration of `Deploy`.
2.Update the docs of `Deploy`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.deploy.recoveryMode | 0.8.1 | None | d66c01f2b6defb3db6c1be99523b734a4d960532#diff-29dffdccd5a7f4c8b496c293e87c8668 |
spark.deploy.recoveryMode.factory | 1.2.0 | SPARK-1830 |		deefd9d7377a8091a1d184b99066febd0e9f6afd#diff-29dffdccd5a7f4c8b496c293e87c8668 | This configuration appears in branch-1.3, but the version number in the pom.xml file corresponding to the commit is 1.2.0-SNAPSHOT
spark.deploy.recoveryDirectory | 0.8.1 | None |			d66c01f2b6defb3db6c1be99523b734a4d960532#diff-29dffdccd5a7f4c8b496c293e87c8668 |
spark.deploy.zookeeper.url | 0.8.1 | None |			d66c01f2b6defb3db6c1be99523b734a4d960532#diff-4457313ca662a1cd60197122d924585c |
spark.deploy.zookeeper.dir | 0.8.1 | None | d66c01f2b6defb3db6c1be99523b734a4d960532#diff-a84228cb45c7d5bd93305a1f5bf720b6 |
spark.deploy.retainedApplications | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-29dffdccd5a7f4c8b496c293e87c8668 |
spark.deploy.retainedDrivers | 1.1.0 | None | 7446f5ff93142d2dd5c79c63fa947f47a1d4db8b#diff-29dffdccd5a7f4c8b496c293e87c8668 |
spark.dead.worker.persistence | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-29dffdccd5a7f4c8b496c293e87c8668 |
spark.deploy.maxExecutorRetries | 1.6.3 | SPARK-16956 | ace458f0330f22463ecf7cbee7c0465e10fba8a8#diff-29dffdccd5a7f4c8b496c293e87c8668 |
spark.deploy.spreadOut | 0.6.1 | None | bb2b9ff37cd2503cc6ea82c5dd395187b0910af0#diff-0e7ae91819fc8f7b47b0f97be7116325 |
spark.deploy.defaultCores | 0.9.0 | None | d8bcc8e9a095c1b20dd7a17b6535800d39bff80e#diff-29dffdccd5a7f4c8b496c293e87c8668 |

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27668 from beliefer/add-version-to-deploy-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/internal/config/Deploy.scala     | 11 +++++++++++
 docs/configuration.md                                 |  5 ++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
index ceab957b36634..d494c5ec019c7 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
@@ -19,48 +19,59 @@ package org.apache.spark.internal.config
 
 private[spark] object Deploy {
   val RECOVERY_MODE = ConfigBuilder("spark.deploy.recoveryMode")
+    .version("0.8.1")
     .stringConf
     .createWithDefault("NONE")
 
   val RECOVERY_MODE_FACTORY = ConfigBuilder("spark.deploy.recoveryMode.factory")
+    .version("1.2.0")
     .stringConf
     .createWithDefault("")
 
   val RECOVERY_DIRECTORY = ConfigBuilder("spark.deploy.recoveryDirectory")
+    .version("0.8.1")
     .stringConf
     .createWithDefault("")
 
   val ZOOKEEPER_URL = ConfigBuilder("spark.deploy.zookeeper.url")
     .doc(s"When `${RECOVERY_MODE.key}` is set to ZOOKEEPER, this " +
       "configuration is used to set the zookeeper URL to connect to.")
+    .version("0.8.1")
     .stringConf
     .createOptional
 
   val ZOOKEEPER_DIRECTORY = ConfigBuilder("spark.deploy.zookeeper.dir")
+    .version("0.8.1")
     .stringConf
     .createOptional
 
   val RETAINED_APPLICATIONS = ConfigBuilder("spark.deploy.retainedApplications")
+    .version("0.8.0")
     .intConf
     .createWithDefault(200)
 
   val RETAINED_DRIVERS = ConfigBuilder("spark.deploy.retainedDrivers")
+    .version("1.1.0")
     .intConf
     .createWithDefault(200)
 
   val REAPER_ITERATIONS = ConfigBuilder("spark.dead.worker.persistence")
+    .version("0.8.0")
     .intConf
     .createWithDefault(15)
 
   val MAX_EXECUTOR_RETRIES = ConfigBuilder("spark.deploy.maxExecutorRetries")
+    .version("1.6.3")
     .intConf
     .createWithDefault(10)
 
   val SPREAD_OUT_APPS = ConfigBuilder("spark.deploy.spreadOut")
+    .version("0.6.1")
     .booleanConf
     .createWithDefault(true)
 
   val DEFAULT_CORES = ConfigBuilder("spark.deploy.defaultCores")
+    .version("0.9.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
diff --git a/docs/configuration.md b/docs/configuration.md
index f7b7e1671a8ab..2f5b570cb3114 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2577,22 +2577,25 @@ Spark subsystems.
 ### Deploy
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.deploy.recoveryMode</code></td>
     <td>NONE</td>
     <td>The recovery mode setting to recover submitted Spark jobs with cluster mode when it failed and relaunches.
     This is only applicable for cluster mode when running with Standalone or Mesos.</td>
+    <td>0.8.1</td>
   </tr>
   <tr>
     <td><code>spark.deploy.zookeeper.url</code></td>
     <td>None</td>
     <td>When `spark.deploy.recoveryMode` is set to ZOOKEEPER, this configuration is used to set the zookeeper URL to connect to.</td>
+    <td>0.8.1</td>
   </tr>
   <tr>
     <td><code>spark.deploy.zookeeper.dir</code></td>
     <td>None</td>
     <td>When `spark.deploy.recoveryMode` is set to ZOOKEEPER, this configuration is used to set the zookeeper directory to store recovery state.</td>
+    <td>0.8.1</td>
   </tr>
 </table>
 

From 42b26cdafc9f3bafe0d225724b97c8e6bdb74e8d Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 27 Feb 2020 10:56:38 +0900
Subject: [PATCH 0411/1280] [SPARK-30910][CORE][DOC] Add version information to
 the configuration of R
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `R`.
2.Update the docs of `R`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.r.backendConnectionTimeout | 2.1.0 | SPARK-17919 | 2881a2d1d1a650a91df2c6a01275eba14a43b42a#diff-025470e1b7094d7cf4a78ea353fb3981 |  
spark.r.numRBackendThreads | 1.4.0 | SPARK-8282 | 28e8a6ea65fd08ab9cefc4d179d5c66ffefd3eb4#diff-697f7f2fc89808e0113efc71ed235db2 |  
spark.r.heartBeatInterval | 2.1.0 | SPARK-17919 | 2881a2d1d1a650a91df2c6a01275eba14a43b42a#diff-fe903bf14db371aa320b7cc516f2463c |  
spark.sparkr.r.command | 1.5.3 | SPARK-10971 | 9695f452e86a88bef3bcbd1f3c0b00ad9e9ac6e1#diff-025470e1b7094d7cf4a78ea353fb3981 |  
spark.r.command | 1.5.3 | SPARK-10971 | 9695f452e86a88bef3bcbd1f3c0b00ad9e9ac6e1#diff-025470e1b7094d7cf4a78ea353fb3981 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27708 from beliefer/add-version-to-R-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/internal/config/R.scala   | 5 +++++
 docs/configuration.md                                     | 8 +++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/R.scala b/core/src/main/scala/org/apache/spark/internal/config/R.scala
index 26e06a5231c42..46fc198cd4cf5 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/R.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/R.scala
@@ -19,22 +19,27 @@ package org.apache.spark.internal.config
 private[spark] object R {
 
   val R_BACKEND_CONNECTION_TIMEOUT = ConfigBuilder("spark.r.backendConnectionTimeout")
+    .version("2.1.0")
     .intConf
     .createWithDefault(6000)
 
   val R_NUM_BACKEND_THREADS = ConfigBuilder("spark.r.numRBackendThreads")
+    .version("1.4.0")
     .intConf
     .createWithDefault(2)
 
   val R_HEARTBEAT_INTERVAL = ConfigBuilder("spark.r.heartBeatInterval")
+    .version("2.1.0")
     .intConf
     .createWithDefault(100)
 
   val SPARKR_COMMAND = ConfigBuilder("spark.sparkr.r.command")
+    .version("1.5.3")
     .stringConf
     .createWithDefault("Rscript")
 
   val R_COMMAND = ConfigBuilder("spark.r.command")
+    .version("1.5.3")
     .stringConf
     .createOptional
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index 2f5b570cb3114..4aff1ef537459 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2513,13 +2513,14 @@ Spark subsystems.
 ### SparkR
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.r.numRBackendThreads</code></td>
   <td>2</td>
   <td>
     Number of threads used by RBackend to handle RPC calls from SparkR package.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.r.command</code></td>
@@ -2527,6 +2528,7 @@ Spark subsystems.
   <td>
     Executable for executing R scripts in cluster modes for both driver and workers.
   </td>
+  <td>1.5.3</td>
 </tr>
 <tr>
   <td><code>spark.r.driver.command</code></td>
@@ -2534,6 +2536,7 @@ Spark subsystems.
   <td>
     Executable for executing R scripts in client modes for driver. Ignored in cluster modes.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.r.shell.command</code></td>
@@ -2542,6 +2545,7 @@ Spark subsystems.
     Executable for executing sparkR shell in client modes for driver. Ignored in cluster modes. It is the same as environment variable <code>SPARKR_DRIVER_R</code>, but take precedence over it.
     <code>spark.r.shell.command</code> is used for sparkR shell while <code>spark.r.driver.command</code> is used for running R script.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.r.backendConnectionTimeout</code></td>
@@ -2549,6 +2553,7 @@ Spark subsystems.
   <td>
     Connection timeout set by R process on its connection to RBackend in seconds.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.r.heartBeatInterval</code></td>
@@ -2556,6 +2561,7 @@ Spark subsystems.
   <td>
     Interval for heartbeats sent from SparkR backend to R process to prevent connection timeout.
   </td>
+  <td>2.1.0</td>
 </tr>
 
 </table>

From 6e04912c528038834501301587865c1a621112f7 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 27 Feb 2020 10:57:34 +0900
Subject: [PATCH 0412/1280] [SPARK-30909][CORE][DOC] Add version information to
 the configuration of Python
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `Python`.
2.Update the docs of `Python`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.python.worker.reuse | 1.2.0 | SPARK-3030 | 2aea0da84c58a179917311290083456dfa043db7#diff-0a67bc4d171abe4df8eb305b0f4123a2 |  
spark.python.task.killTimeout | 2.2.2 | SPARK-22535 | be68f86e11d64209d9e325ce807025318f383bea#diff-0a67bc4d171abe4df8eb305b0f4123a2 |  
spark.python.use.daemon | 2.3.0 | SPARK-22554 | 57c5514de9dba1c14e296f85fb13fef23ce8c73f#diff-9008ad45db34a7eee2e265a50626841b |  
spark.python.daemon.module | 2.4.0 | SPARK-22959 | afae8f2bc82597593595af68d1aa2d802210ea8b#diff-9008ad45db34a7eee2e265a50626841b |  
spark.python.worker.module | 2.4.0 | SPARK-22959 | afae8f2bc82597593595af68d1aa2d802210ea8b#diff-9008ad45db34a7eee2e265a50626841b |  
spark.executor.pyspark.memory | 2.4.0 | SPARK-25004 | 7ad18ee9f26e75dbe038c6034700f9cd4c0e2baa#diff-6bdad48cfc34314e89599655442ff210 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27704 from beliefer/add-version-to-python-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/internal/config/Python.scala     | 6 ++++++
 docs/configuration.md                                       | 6 ++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Python.scala b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
index 26a0598f49411..188d884319644 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Python.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Python.scala
@@ -22,26 +22,32 @@ import org.apache.spark.network.util.ByteUnit
 
 private[spark] object Python {
   val PYTHON_WORKER_REUSE = ConfigBuilder("spark.python.worker.reuse")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
   val PYTHON_TASK_KILL_TIMEOUT = ConfigBuilder("spark.python.task.killTimeout")
+    .version("2.2.2")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("2s")
 
   val PYTHON_USE_DAEMON = ConfigBuilder("spark.python.use.daemon")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val PYTHON_DAEMON_MODULE = ConfigBuilder("spark.python.daemon.module")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val PYTHON_WORKER_MODULE = ConfigBuilder("spark.python.worker.module")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val PYSPARK_EXECUTOR_MEMORY = ConfigBuilder("spark.executor.pyspark.memory")
+    .version("2.4.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index 4aff1ef537459..88edaf6dade66 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -136,7 +136,7 @@ of the most common options to set are:
 ### Application Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.app.name</code></td>
   <td>(none)</td>
@@ -258,6 +258,7 @@ of the most common options to set are:
     limitations are inherited. For instance, Windows does not support resource limiting and actual 
     resource is not limited on MacOS.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.memoryOverhead</code></td>
@@ -420,7 +421,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Runtime Environment
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.driver.extraClassPath</code></td>
   <td>(none)</td>
@@ -659,6 +660,7 @@ Apart from these, the following properties are also available, and may be useful
     if there is a large broadcast, then the broadcast will not need to be transferred
     from JVM to Python worker for every task.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.files</code></td>

From 2c5b13f33add9943248c66873ada2c2e1e5d703a Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 27 Feb 2020 11:05:11 +0900
Subject: [PATCH 0413/1280] [SPARK-30888][CORE][DOC] Add version information to
 the configuration of Network
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `Network`.
2.Update the docs of `Network`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.network.crypto.saslFallback | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.network.crypto.enabled | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-6bdad48cfc34314e89599655442ff210 |  
spark.network.remoteReadNioBufferConversion | 2.4.0 | SPARK-24307 | 2c82745686f4456c4d5c84040a431dcb5b6cb60b#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.network.timeout | 1.3.0 | SPARK-4688 | d3f07fd23cc26a70f44c52e24445974d4885d58a#diff-1df6b5af3d8f9f16255ff8c7a06f402f |  
spark.network.timeoutInterval | 1.3.2 | SPARK-5529 | ec196ab1c7569d7ab0a50c9d7338c2835f2c84d5#diff-47779b72f095f7e7f926898fa1a425ee |  
spark.rpc.askTimeout | 1.4.0 | SPARK-6490 | 8136810dfad12008ac300116df7bc8448740f1ae#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.rpc.connect.threads | 1.6.0 | SPARK-6028 | 084e4e126211d74a79e8dbd2d0e604dd3c650822#diff-0c89b4a60c30a7cd2224bb64d93da942 |  
spark.rpc.io.numConnectionsPerPeer | 1.6.0 | SPARK-10745 | 34a77679877bc40b58a10ec539a8da00fed7db39#diff-0c89b4a60c30a7cd2224bb64d93da942 |  
spark.rpc.io.threads | 1.6.0 | SPARK-6028 | 084e4e126211d74a79e8dbd2d0e604dd3c650822#diff-0c89b4a60c30a7cd2224bb64d93da942 |  
spark.rpc.lookupTimeout | 1.4.0 | SPARK-6490 | 8136810dfad12008ac300116df7bc8448740f1ae#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.rpc.message.maxSize | 2.0.0 | SPARK-7997 | bc1babd63da4ee56e6d371eb24805a5d714e8295#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.rpc.netty.dispatcher.numThreads | 1.6.0 | SPARK-11079 | 1797055dbf1d2fd7714d7c65c8d2efde2f15efc1#diff-05133dfc4bfdb6a27aa092d86ce24866 |  
spark.rpc.numRetries | 1.4.0 | SPARK-6490 | 8136810dfad12008ac300116df7bc8448740f1ae#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.rpc.retry.wait | 1.4.0 | SPARK-6490 | 8136810dfad12008ac300116df7bc8448740f1ae#diff-529fc5c06b9731c1fbda6f3db60b16aa |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27674 from beliefer/add-version-to-network-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/Network.scala       | 14 +++++++++++++
 docs/configuration.md                         | 20 +++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Network.scala b/core/src/main/scala/org/apache/spark/internal/config/Network.scala
index 129e31a82979f..0961d062cc04f 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Network.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Network.scala
@@ -23,71 +23,85 @@ private[spark] object Network {
 
   private[spark] val NETWORK_CRYPTO_SASL_FALLBACK =
     ConfigBuilder("spark.network.crypto.saslFallback")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val NETWORK_CRYPTO_ENABLED =
     ConfigBuilder("spark.network.crypto.enabled")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val NETWORK_REMOTE_READ_NIO_BUFFER_CONVERSION =
     ConfigBuilder("spark.network.remoteReadNioBufferConversion")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val NETWORK_TIMEOUT =
     ConfigBuilder("spark.network.timeout")
+      .version("1.3.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("120s")
 
   private[spark] val NETWORK_TIMEOUT_INTERVAL =
     ConfigBuilder("spark.network.timeoutInterval")
+      .version("1.3.2")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString(STORAGE_BLOCKMANAGER_TIMEOUTINTERVAL.defaultValueString)
 
   private[spark] val RPC_ASK_TIMEOUT =
     ConfigBuilder("spark.rpc.askTimeout")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
   private[spark] val RPC_CONNECT_THREADS =
     ConfigBuilder("spark.rpc.connect.threads")
+      .version("1.6.0")
       .intConf
       .createWithDefault(64)
 
   private[spark] val RPC_IO_NUM_CONNECTIONS_PER_PEER =
     ConfigBuilder("spark.rpc.io.numConnectionsPerPeer")
+      .version("1.6.0")
       .intConf
       .createWithDefault(1)
 
   private[spark] val RPC_IO_THREADS =
     ConfigBuilder("spark.rpc.io.threads")
+      .version("1.6.0")
       .intConf
       .createOptional
 
   private[spark] val RPC_LOOKUP_TIMEOUT =
     ConfigBuilder("spark.rpc.lookupTimeout")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
   private[spark] val RPC_MESSAGE_MAX_SIZE =
     ConfigBuilder("spark.rpc.message.maxSize")
+      .version("2.0.0")
       .intConf
       .createWithDefault(128)
 
   private[spark] val RPC_NETTY_DISPATCHER_NUM_THREADS =
     ConfigBuilder("spark.rpc.netty.dispatcher.numThreads")
+      .version("1.6.0")
       .intConf
       .createOptional
 
   private[spark] val RPC_NUM_RETRIES =
     ConfigBuilder("spark.rpc.numRetries")
+      .version("1.4.0")
       .intConf
       .createWithDefault(3)
 
   private[spark] val RPC_RETRY_WAIT =
     ConfigBuilder("spark.rpc.retry.wait")
+      .version("1.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("3s")
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index 88edaf6dade66..295cb3d2f22a2 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1658,7 +1658,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Networking
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.rpc.message.maxSize</code></td>
   <td>128</td>
@@ -1667,6 +1667,7 @@ Apart from these, the following properties are also available, and may be useful
     output size information sent between executors and the driver. Increase this if you are running
     jobs with many thousands of map and reduce tasks and see messages about the RPC message size.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.blockManager.port</code></td>
@@ -1674,6 +1675,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port for all block managers to listen on. These exist on both the driver and the executors.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.driver.blockManager.port</code></td>
@@ -1682,6 +1684,7 @@ Apart from these, the following properties are also available, and may be useful
     Driver-specific port for the block manager to listen on, for cases where it cannot use the same
     configuration as executors.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.driver.bindAddress</code></td>
@@ -1695,6 +1698,7 @@ Apart from these, the following properties are also available, and may be useful
     the different ports used by the driver (RPC, block manager and UI) need to be forwarded from the
     container's host.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.driver.host</code></td>
@@ -1703,6 +1707,7 @@ Apart from these, the following properties are also available, and may be useful
     Hostname or IP address for the driver.
     This is used for communicating with the executors and the standalone Master.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.driver.port</code></td>
@@ -1711,6 +1716,7 @@ Apart from these, the following properties are also available, and may be useful
     Port for the driver to listen on.
     This is used for communicating with the executors and the standalone Master.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.rpc.io.backLog</code></td>
@@ -1720,6 +1726,7 @@ Apart from these, the following properties are also available, and may be useful
     need to be increased, so that incoming connections are not dropped when a large number of
     connections arrives in a short period of time.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.network.timeout</code></td>
@@ -1731,6 +1738,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.shuffle.io.connectionTimeout</code>, <code>spark.rpc.askTimeout</code> or
     <code>spark.rpc.lookupTimeout</code> if they are not configured.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.network.io.preferDirectBufs</code></td>
@@ -1740,7 +1748,8 @@ Apart from these, the following properties are also available, and may be useful
     Off-heap buffers are used to reduce garbage collection during shuffle and cache
     block transfer. For environments where off-heap memory is tightly limited, users may wish to
     turn this off to force all allocations to be on-heap.
-    </td>
+  </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.port.maxRetries</code></td>
@@ -1752,6 +1761,7 @@ Apart from these, the following properties are also available, and may be useful
     essentially allows it to try a range of ports from the start port specified
     to port + maxRetries.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.rpc.numRetries</code></td>
@@ -1760,6 +1770,7 @@ Apart from these, the following properties are also available, and may be useful
     Number of times to retry before an RPC task gives up.
     An RPC task will run at most times of this number.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.retry.wait</code></td>
@@ -1767,6 +1778,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Duration for an RPC ask operation to wait before retrying.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.askTimeout</code></td>
@@ -1774,6 +1786,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Duration for an RPC ask operation to wait before timing out.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.lookupTimeout</code></td>
@@ -1781,6 +1794,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Duration for an RPC remote endpoint lookup operation to wait before timing out.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.core.connection.ack.wait.timeout</code></td>
@@ -1790,6 +1804,7 @@ Apart from these, the following properties are also available, and may be useful
     out and giving up. To avoid unwilling timeout caused by long pause like GC,
     you can set larger value.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.network.maxRemoteBlockSizeFetchToMem</code></td>
@@ -1801,6 +1816,7 @@ Apart from these, the following properties are also available, and may be useful
     For users who enabled external shuffle service, this feature can only work when
     external shuffle service is at least 2.3.0.
   </td>
+  <td></td>
 </tr>
 </table>
 

From dd31736c0ca55dfe3842a6ad920d85f4301c563f Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Mon, 2 Mar 2020 15:14:47 +0900
Subject: [PATCH 0414/1280] [SPARK-30908][CORE][DOC] Add version information to
 the configuration of Kryo
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `Kryo`.
2.Update the docs of `Kryo`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.kryo.registrationRequired | 1.1.0 | SPARK-2102 | efdaeb111917dd0314f1d00ee8524bed1e2e21ca#diff-1f81c62dad0e2dfc387a974bb08c497c |  
spark.kryo.registrator | 0.5.0 | None | 91c07a33d90ab0357e8713507134ecef5c14e28a#diff-792ed56b3398163fa14e8578549d0d98 | This is not a release version, do we need to record it?
spark.kryo.classesToRegister | 1.2.0 | SPARK-1813 | 6bb56faea8d238ea22c2de33db93b1b39f492b3a#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.kryo.unsafe | 2.1.0 | SPARK-928 | bc167a2a53f5a795d089e8a884569b1b3e2cd439#diff-1f81c62dad0e2dfc387a974bb08c497c |  
spark.kryo.pool | 3.0.0 | SPARK-26466 | 38f030725c561979ca98b2a6cc7ca6c02a1f80ed#diff-a3c6b992784f9abeb9f3047d3dcf3ed9 |  
spark.kryo.referenceTracking | 0.8.0 | None | 0a8cc309211c62f8824d76618705c817edcf2424#diff-1f81c62dad0e2dfc387a974bb08c497c |  
spark.kryoserializer.buffer | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-1f81c62dad0e2dfc387a974bb08c497c |  
spark.kryoserializer.buffer.max | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-1f81c62dad0e2dfc387a974bb08c497c |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27734 from beliefer/add-version-to-kryo-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/internal/config/Kryo.scala    | 8 ++++++++
 docs/configuration.md                                    | 9 ++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
index 717a09914a2f5..646d8556538c2 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
@@ -22,35 +22,43 @@ import org.apache.spark.network.util.ByteUnit
 private[spark] object Kryo {
 
   val KRYO_REGISTRATION_REQUIRED = ConfigBuilder("spark.kryo.registrationRequired")
+    .version("1.1.0")
     .booleanConf
     .createWithDefault(false)
 
   val KRYO_USER_REGISTRATORS = ConfigBuilder("spark.kryo.registrator")
+    .version("0.5.0")
     .stringConf
     .createOptional
 
   val KRYO_CLASSES_TO_REGISTER = ConfigBuilder("spark.kryo.classesToRegister")
+    .version("1.2.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val KRYO_USE_UNSAFE = ConfigBuilder("spark.kryo.unsafe")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   val KRYO_USE_POOL = ConfigBuilder("spark.kryo.pool")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val KRYO_REFERENCE_TRACKING = ConfigBuilder("spark.kryo.referenceTracking")
+    .version("0.8.0")
     .booleanConf
     .createWithDefault(true)
 
   val KRYO_SERIALIZER_BUFFER_SIZE = ConfigBuilder("spark.kryoserializer.buffer")
+    .version("1.4.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("64k")
 
   val KRYO_SERIALIZER_MAX_BUFFER_SIZE = ConfigBuilder("spark.kryoserializer.buffer.max")
+    .version("1.4.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("64m")
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 295cb3d2f22a2..c586ecf15a8f4 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1204,7 +1204,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Compression and Serialization
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.broadcast.compress</code></td>
   <td>true</td>
@@ -1278,6 +1278,7 @@ Apart from these, the following properties are also available, and may be useful
     with Kryo.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
+  <th>1.2.0</th>
 </tr>
 <tr>
   <td><code>spark.kryo.referenceTracking</code></td>
@@ -1288,6 +1289,7 @@ Apart from these, the following properties are also available, and may be useful
     copies of the same object. Can be disabled to improve performance if you know this is not the
     case.
   </td>
+  <th>0.8.0</th>
 </tr>
 <tr>
   <td><code>spark.kryo.registrationRequired</code></td>
@@ -1299,6 +1301,7 @@ Apart from these, the following properties are also available, and may be useful
     significant performance overhead, so enabling this option can enforce strictly that a
     user has not omitted classes from registration.
   </td>
+  <th>1.1.0</th>
 </tr>
 <tr>
   <td><code>spark.kryo.registrator</code></td>
@@ -1312,6 +1315,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>KryoRegistrator</code></a>.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
+  <th>0.5.0</th>
 </tr>
 <tr>
   <td><code>spark.kryo.unsafe</code></td>
@@ -1320,6 +1324,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to use unsafe based Kryo serializer. Can be
     substantially faster by using Unsafe Based IO.
   </td>
+  <th>2.1.0</th>
 </tr>
 <tr>
   <td><code>spark.kryoserializer.buffer.max</code></td>
@@ -1329,6 +1334,7 @@ Apart from these, the following properties are also available, and may be useful
     This must be larger than any object you attempt to serialize and must be less than 2048m.
     Increase this if you get a "buffer limit exceeded" exception inside Kryo.
   </td>
+  <th>1.4.0</th>
 </tr>
 <tr>
   <td><code>spark.kryoserializer.buffer</code></td>
@@ -1338,6 +1344,7 @@ Apart from these, the following properties are also available, and may be useful
     Note that there will be one buffer <i>per core</i> on each worker. This buffer will grow up to
     <code>spark.kryoserializer.buffer.max</code> if needed.
   </td>
+  <th>1.4.0</th>
 </tr>
 <tr>
   <td><code>spark.rdd.compress</code></td>

From 231c0bbe61c07eb237fa4e13457acd88e0038794 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Mon, 2 Mar 2020 15:15:49 +0900
Subject: [PATCH 0415/1280] [SPARK-30891][CORE][DOC] Add version information to
 the configuration of History
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `History`.
2.Update the docs of `History`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.history.fs.logDirectory | 1.1.0 | SPARK-1768 | 21ddd7d1e9f8e2a726427f32422c31706a20ba3f#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.fs.safemodeCheck.interval | 1.6.0 | SPARK-11020 | cf04fdfe71abc395163a625cc1f99ec5e54cc07e#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.fs.update.interval | 1.4.0 | SPARK-6046 | 4527761bcd6501c362baf2780905a0018b9a74ba#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.fs.cleaner.enabled | 1.3.0 | SPARK-3562 | 8942b522d8a3269a2a357e3a274ed4b3e66ebdde#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e | Branch branch-1.3 does not exist, exists in branch-1.4, but it is 1.3.0-SNAPSHOT in pom.xml
spark.history.fs.cleaner.interval | 1.4.0 | SPARK-5933 | 1991337336596f94698e79c2366f065c374128ab#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |
spark.history.fs.cleaner.maxAge | 1.4.0 | SPARK-5933 | 1991337336596f94698e79c2366f065c374128ab#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |
spark.history.fs.cleaner.maxNum | 3.0.0 | SPARK-28294 | bbc2be4f425c4c26450e1bf21db407e81046ce21#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.store.path | 2.3.0 | SPARK-20642 | 74daf622de4e534d5a5929b424a6e836850eefad#diff-19f35f981fdc5b0a46f070b879a9a9fc |  
spark.history.store.maxDiskUsage | 2.3.0 | SPARK-20654 | 8b497046c647a21bbed1bdfbdcb176745a1d5cd5#diff-19f35f981fdc5b0a46f070b879a9a9fc |  
spark.history.ui.port | 1.0.0 | SPARK-1276 | 9ae80bf9bd3e4da7443af97b41fe26aa5d35d70b#diff-b49b5b9c31ddb36a9061004b5b723058 |  
spark.history.fs.inProgressOptimization.enabled | 2.4.0 | SPARK-6951 | 653fe02415a537299e15f92b56045569864b6183#diff-19f35f981fdc5b0a46f070b879a9a9fc |  
spark.history.fs.endEventReparseChunkSize | 2.4.0 | SPARK-6951 | 653fe02415a537299e15f92b56045569864b6183#diff-19f35f981fdc5b0a46f070b879a9a9fc |  
spark.history.fs.eventLog.rolling.maxFilesToRetain | 3.0.0 | SPARK-30481 | a2fe73b83c0e7c61d1c83b236565a71e3d005a71#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.fs.eventLog.rolling.compaction.score.threshold | 3.0.0 | SPARK-30481 | a2fe73b83c0e7c61d1c83b236565a71e3d005a71#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.fs.driverlog.cleaner.enabled | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.fs.driverlog.cleaner.interval | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.fs.driverlog.cleaner.maxAge | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.ui.acls.enable | 1.0.1 | Spark 1489 | c8dd13221215275948b1a6913192d40e0c8cbadd#diff-b49b5b9c31ddb36a9061004b5b723058 |  
spark.history.ui.admin.acls | 2.1.1 | SPARK-19033 | 4ca1788805e4a0131ba8f0ccb7499ee0e0242837#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.ui.admin.acls.groups | 2.1.1 | SPARK-19033 | 4ca1788805e4a0131ba8f0ccb7499ee0e0242837#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.fs.numReplayThreads | 2.0.0 | SPARK-13988 | 6fdd0e32a6c3fdce1f3f7e1f8d252af05c419f7b#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.retainedApplications | 1.0.0 | SPARK-1276 | 9ae80bf9bd3e4da7443af97b41fe26aa5d35d70b#diff-b49b5b9c31ddb36a9061004b5b723058 |
spark.history.provider | 1.1.0 | SPARK-1768 | 21ddd7d1e9f8e2a726427f32422c31706a20ba3f#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.kerberos.enabled | 1.0.1 | Spark-1490 | 866b03ef4d27b2160563b58d577de29ba6eb4442#diff-b49b5b9c31ddb36a9061004b5b723058 |  
spark.history.kerberos.principal | 1.0.1 | Spark-1490 | 866b03ef4d27b2160563b58d577de29ba6eb4442#diff-b49b5b9c31ddb36a9061004b5b723058 |  
spark.history.kerberos.keytab | 1.0.1 | Spark-1490 | 866b03ef4d27b2160563b58d577de29ba6eb4442#diff-b49b5b9c31ddb36a9061004b5b723058 |  
spark.history.custom.executor.log.url | 3.0.0 | SPARK-26311 | ae5b2a6a92be4986ef5b8062d7fb59318cff6430#diff-6bddeb5e25239974fc13db66266b167b |  
spark.history.custom.executor.log.url.applyIncompleteApplication | 3.0.0 | SPARK-26311 | ae5b2a6a92be4986ef5b8062d7fb59318cff6430#diff-6bddeb5e25239974fc13db66266b167b |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27751 from beliefer/add-version-to-history-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/History.scala       | 28 +++++++++++++++++++
 docs/monitoring.md                            | 26 ++++++++++++++++-
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 8f99908507ceb..581777de366ef 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -26,46 +26,56 @@ private[spark] object History {
   val DEFAULT_LOG_DIR = "file:/tmp/spark-events"
 
   val HISTORY_LOG_DIR = ConfigBuilder("spark.history.fs.logDirectory")
+    .version("1.1.0")
     .stringConf
     .createWithDefault(DEFAULT_LOG_DIR)
 
   val SAFEMODE_CHECK_INTERVAL_S = ConfigBuilder("spark.history.fs.safemodeCheck.interval")
+    .version("1.6.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("5s")
 
   val UPDATE_INTERVAL_S = ConfigBuilder("spark.history.fs.update.interval")
+    .version("1.4.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("10s")
 
   val CLEANER_ENABLED = ConfigBuilder("spark.history.fs.cleaner.enabled")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
   val CLEANER_INTERVAL_S = ConfigBuilder("spark.history.fs.cleaner.interval")
+    .version("1.4.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("1d")
 
   val MAX_LOG_AGE_S = ConfigBuilder("spark.history.fs.cleaner.maxAge")
+    .version("1.4.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("7d")
 
   val MAX_LOG_NUM = ConfigBuilder("spark.history.fs.cleaner.maxNum")
     .doc("The maximum number of log files in the event log directory.")
+    .version("3.0.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
   val LOCAL_STORE_DIR = ConfigBuilder("spark.history.store.path")
     .doc("Local directory where to cache application history information. By default this is " +
       "not set, meaning all history information will be kept in memory.")
+    .version("2.3.0")
     .stringConf
     .createOptional
 
   val MAX_LOCAL_DISK_USAGE = ConfigBuilder("spark.history.store.maxDiskUsage")
+    .version("2.3.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("10g")
 
   val HISTORY_SERVER_UI_PORT = ConfigBuilder("spark.history.ui.port")
     .doc("Web UI port to bind Spark History Server")
+    .version("1.0.0")
     .intConf
     .createWithDefault(18080)
 
@@ -73,6 +83,7 @@ private[spark] object History {
     ConfigBuilder("spark.history.fs.inProgressOptimization.enabled")
       .doc("Enable optimized handling of in-progress logs. This option may leave finished " +
         "applications that fail to rename their event logs listed as in-progress.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -81,6 +92,7 @@ private[spark] object History {
       .doc("How many bytes to parse at the end of log files looking for the end event. " +
         "This is used to speed up generation of application listings by skipping unnecessary " +
         "parts of event log files. It can be disabled by setting this config to 0.")
+      .version("2.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("1m")
 
@@ -90,6 +102,7 @@ private[spark] object History {
         "By default, all event log files will be retained. Please set the configuration " +
         s"and ${EVENT_LOG_ROLLING_MAX_FILE_SIZE.key} accordingly if you want to control " +
         "the overall size of event log files.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "Max event log files to retain should be higher than 0.")
       .createWithDefault(Integer.MAX_VALUE)
@@ -99,54 +112,67 @@ private[spark] object History {
       .doc("The threshold score to determine whether it's good to do the compaction or not. " +
         "The compaction score is calculated in analyzing, and being compared to this value. " +
         "Compaction will proceed only when the score is higher than the threshold value.")
+      .version("3.0.0")
       .internal()
       .doubleConf
       .createWithDefault(0.7d)
 
   val DRIVER_LOG_CLEANER_ENABLED = ConfigBuilder("spark.history.fs.driverlog.cleaner.enabled")
+    .version("3.0.0")
     .fallbackConf(CLEANER_ENABLED)
 
   val DRIVER_LOG_CLEANER_INTERVAL = ConfigBuilder("spark.history.fs.driverlog.cleaner.interval")
+    .version("3.0.0")
     .fallbackConf(CLEANER_INTERVAL_S)
 
   val MAX_DRIVER_LOG_AGE_S = ConfigBuilder("spark.history.fs.driverlog.cleaner.maxAge")
+    .version("3.0.0")
     .fallbackConf(MAX_LOG_AGE_S)
 
   val HISTORY_SERVER_UI_ACLS_ENABLE = ConfigBuilder("spark.history.ui.acls.enable")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(false)
 
   val HISTORY_SERVER_UI_ADMIN_ACLS = ConfigBuilder("spark.history.ui.admin.acls")
+    .version("2.1.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val HISTORY_SERVER_UI_ADMIN_ACLS_GROUPS = ConfigBuilder("spark.history.ui.admin.acls.groups")
+    .version("2.1.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val NUM_REPLAY_THREADS = ConfigBuilder("spark.history.fs.numReplayThreads")
+    .version("2.0.0")
     .intConf
     .createWithDefaultFunction(() => Math.ceil(Runtime.getRuntime.availableProcessors() / 4f).toInt)
 
   val RETAINED_APPLICATIONS = ConfigBuilder("spark.history.retainedApplications")
+    .version("1.0.0")
     .intConf
     .createWithDefault(50)
 
   val PROVIDER = ConfigBuilder("spark.history.provider")
+    .version("1.1.0")
     .stringConf
     .createOptional
 
   val KERBEROS_ENABLED = ConfigBuilder("spark.history.kerberos.enabled")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(false)
 
   val KERBEROS_PRINCIPAL = ConfigBuilder("spark.history.kerberos.principal")
+    .version("1.0.1")
     .stringConf
     .createOptional
 
   val KERBEROS_KEYTAB = ConfigBuilder("spark.history.kerberos.keytab")
+    .version("1.0.1")
     .stringConf
     .createOptional
 
@@ -156,6 +182,7 @@ private[spark] object History {
       "some path variables via patterns which can vary on cluster manager. Please check the " +
       "documentation for your cluster manager to see which patterns are supported, if any. " +
       "This configuration has no effect on a live application, it only affects the history server.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -165,6 +192,7 @@ private[spark] object History {
         s"${CUSTOM_EXECUTOR_LOG_URL.key}, to incomplete application as well. " +
         "Even if this is true, this still only affects the behavior of the history server, " +
         "not running spark applications.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 }
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 131cd2a844e44..b72fd027da597 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -143,13 +143,14 @@ Security options for the Spark History Server are covered more detail in the
 [Security](security.html#web-ui) page.
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td>spark.history.provider</td>
     <td><code>org.apache.spark.deploy.history.FsHistoryProvider</code></td>
     <td>Name of the class implementing the application history backend. Currently there is only
     one implementation, provided by Spark, which looks for application logs stored in the
     file system.</td>
+    <td>1.1.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.logDirectory</td>
@@ -160,6 +161,7 @@ Security options for the Spark History Server are covered more detail in the
     an HDFS path <code>hdfs://namenode/shared/spark-logs</code>
     or that of an alternative filesystem supported by the Hadoop APIs.
     </td>
+    <td>1.1.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.update.interval</td>
@@ -171,6 +173,7 @@ Security options for the Spark History Server are covered more detail in the
       As soon as an update has completed, listings of the completed and incomplete applications
       will reflect the changes.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.retainedApplications</td>
@@ -180,6 +183,7 @@ Security options for the Spark History Server are covered more detail in the
       the oldest applications will be removed from the cache. If an application is not in the cache,
       it will have to be loaded from disk if it is accessed from the UI.
     </td>
+    <td>1.0.0</td>
   </tr>
   <tr>
     <td>spark.history.ui.maxApplications</td>
@@ -188,6 +192,7 @@ Security options for the Spark History Server are covered more detail in the
       The number of applications to display on the history summary page. Application UIs are still
       available by accessing their URLs directly even if they are not displayed on the history summary page.
     </td>
+    <td>2.0.1</td>
   </tr>
   <tr>
     <td>spark.history.ui.port</td>
@@ -195,6 +200,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       The port to which the web interface of the history server binds.
     </td>
+    <td>1.0.0</td>
   </tr>
   <tr>
     <td>spark.history.kerberos.enabled</td>
@@ -203,6 +209,7 @@ Security options for the Spark History Server are covered more detail in the
       Indicates whether the history server should use kerberos to login. This is required
       if the history server is accessing HDFS files on a secure Hadoop cluster.
     </td>
+    <td>1.0.1</td>
   </tr>
   <tr>
     <td>spark.history.kerberos.principal</td>
@@ -210,6 +217,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       When <code>spark.history.kerberos.enabled=true</code>, specifies kerberos principal name for the History Server.
     </td>
+    <td>1.0.1</td>
   </tr>
   <tr>
     <td>spark.history.kerberos.keytab</td>
@@ -217,6 +225,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       When <code>spark.history.kerberos.enabled=true</code>, specifies location of the kerberos keytab file for the History Server.
     </td>
+    <td>1.0.1</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.enabled</td>
@@ -224,6 +233,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       Specifies whether the History Server should periodically clean up event logs from storage.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.interval</td>
@@ -236,6 +246,7 @@ Security options for the Spark History Server are covered more detail in the
       <code>spark.history.fs.cleaner.maxNum</code>, Spark tries to clean up the completed attempts
       from the applications based on the order of their oldest attempt time.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.maxAge</td>
@@ -243,6 +254,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       When <code>spark.history.fs.cleaner.enabled=true</code>, job history files older than this will be deleted when the filesystem history cleaner runs.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.cleaner.maxNum</td>
@@ -253,6 +265,7 @@ Security options for the Spark History Server are covered more detail in the
       This should be smaller than the underlying file system limit like
       `dfs.namenode.fs-limits.max-directory-items` in HDFS.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.endEventReparseChunkSize</td>
@@ -262,6 +275,7 @@ Security options for the Spark History Server are covered more detail in the
       This is used to speed up generation of application listings by skipping unnecessary
       parts of event log files. It can be disabled by setting this config to 0.
     </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.inProgressOptimization.enabled</td>
@@ -270,6 +284,7 @@ Security options for the Spark History Server are covered more detail in the
       Enable optimized handling of in-progress logs. This option may leave finished
       applications that fail to rename their event logs listed as in-progress.
     </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.driverlog.cleaner.enabled</td>
@@ -277,6 +292,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       Specifies whether the History Server should periodically clean up driver logs from storage.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.driverlog.cleaner.interval</td>
@@ -285,6 +301,7 @@ Security options for the Spark History Server are covered more detail in the
       When <code>spark.history.fs.driverlog.cleaner.enabled=true</code>, specifies how often the filesystem driver log cleaner checks for files to delete.
       Files are only deleted if they are older than <code>spark.history.fs.driverlog.cleaner.maxAge</code>
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.driverlog.cleaner.maxAge</td>
@@ -292,6 +309,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       When <code>spark.history.fs.driverlog.cleaner.enabled=true</code>, driver log files older than this will be deleted when the driver log cleaner runs.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.numReplayThreads</td>
@@ -299,6 +317,7 @@ Security options for the Spark History Server are covered more detail in the
     <td>
       Number of threads that will be used by history server to process event logs.
     </td>
+    <td>2.0.0</td>
   </tr>
   <tr>
     <td>spark.history.store.maxDiskUsage</td>
@@ -307,6 +326,7 @@ Security options for the Spark History Server are covered more detail in the
       Maximum disk usage for the local directory where the cache application history information
       are stored.
     </td>
+    <td>2.3.0</td>
   </tr>
   <tr>
     <td>spark.history.store.path</td>
@@ -316,6 +336,7 @@ Security options for the Spark History Server are covered more detail in the
         server will store application data on disk instead of keeping it in memory. The data
         written to disk will be re-used in the event of a history server restart.
     </td>
+    <td>2.3.0</td>
   </tr>
   <tr>
     <td>spark.history.custom.executor.log.url</td>
@@ -329,6 +350,7 @@ Security options for the Spark History Server are covered more detail in the
         <p/>
         For now, only YARN mode supports this configuration
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.custom.executor.log.url.applyIncompleteApplication</td>
@@ -339,6 +361,7 @@ Security options for the Spark History Server are covered more detail in the
         Please note that incomplete applications may include applications which didn't shutdown gracefully.
         Even this is set to `true`, this configuration has no effect on a live application, it only affects the history server.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td>spark.history.fs.eventLog.rolling.maxFilesToRetain</td>
@@ -348,6 +371,7 @@ Security options for the Spark History Server are covered more detail in the
       all event log files will be retained. The lowest value is 1 for technical reason.<br/>
       Please read the section of "Applying compaction of old event log files" for more details.
     </td>
+    <td>3.0.0</td>
   </tr>
 </table>
 

From 056e60fa7c7e45a59be769c189456a83760c4aef Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Mon, 2 Mar 2020 15:16:40 +0900
Subject: [PATCH 0416/1280] [SPARK-30912][CORE][DOC] Add version information to
 the configuration of Streaming.scala

### What changes were proposed in this pull request?
1.Add version information to the configuration of `Streaming`.
2.Update the docs of `Streaming`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.streaming.dynamicAllocation.enabled | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |
spark.streaming.dynamicAllocation.testing | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |
spark.streaming.dynamicAllocation.minExecutors | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |
spark.streaming.dynamicAllocation.maxExecutors | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |
spark.streaming.dynamicAllocation.scalingInterval | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |
spark.streaming.dynamicAllocation.scalingUpRatio | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |
spark.streaming.dynamicAllocation.scalingDownRatio | 3.0.0 | SPARK-26941 | cad475dcc9376557f882859856286e858002389a#diff-335c3bbf4ca27cf65a6f850b1a7c89dd |

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27745 from beliefer/add-version-to-streaming-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/internal/config/Streaming.scala | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala b/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala
index 6e58c090e8126..44a467b278614 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Streaming.scala
@@ -23,16 +23,19 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_ENABLED =
     ConfigBuilder("spark.streaming.dynamicAllocation.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STREAMING_DYN_ALLOCATION_TESTING =
     ConfigBuilder("spark.streaming.dynamicAllocation.testing")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STREAMING_DYN_ALLOCATION_MIN_EXECUTORS =
     ConfigBuilder("spark.streaming.dynamicAllocation.minExecutors")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The min executor number of streaming dynamic " +
         "allocation must be positive.")
@@ -40,6 +43,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_MAX_EXECUTORS =
     ConfigBuilder("spark.streaming.dynamicAllocation.maxExecutors")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The max executor number of streaming dynamic " +
         "allocation must be positive.")
@@ -47,6 +51,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_SCALING_INTERVAL =
     ConfigBuilder("spark.streaming.dynamicAllocation.scalingInterval")
+      .version("3.0.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ > 0, "The scaling interval of streaming dynamic " +
         "allocation must be positive.")
@@ -54,6 +59,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_SCALING_UP_RATIO =
     ConfigBuilder("spark.streaming.dynamicAllocation.scalingUpRatio")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ > 0, "The scaling up ratio of streaming dynamic " +
         "allocation must be positive.")
@@ -61,6 +67,7 @@ private[spark] object Streaming {
 
   private[spark] val STREAMING_DYN_ALLOCATION_SCALING_DOWN_RATIO =
     ConfigBuilder("spark.streaming.dynamicAllocation.scalingDownRatio")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ > 0, "The scaling down ratio of streaming dynamic " +
         "allocation must be positive.")

From dd97bcbda5be70ff6d3e7192bb7782092e80483c Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Fri, 6 Mar 2020 11:08:57 +0900
Subject: [PATCH 0417/1280] [SPARK-30914][CORE][DOC] Add version information to
 the configuration of UI
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `UI`.
2.Update the docs of `UI`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.ui.showConsoleProgress | 1.2.1 | SPARK-4017 | 04b1bdbae31c3039125100e703121daf7d9dabf5#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.ui.consoleProgress.update.interval | 2.1.0 | SPARK-16919 | e076fb05ac83a3ed6995e29bb03ea07ea05e39db#diff-fbf4e388a66b6a37e984b91cd71a3e2c |  
spark.ui.enabled | 1.1.1 | SPARK-3490 | 937de93e80e6d299c4d08be426da2d5bc2d66f98#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.ui.port | 0.7.0 | None | f03d9760fd8ac67fd0865cb355ba75d2eff507fe#diff-ed8dbcebe16fda5ecd6df1a981dc6fee |  
spark.ui.filters | 1.0.0 | SPARK-1189 | 7edbea41b43e0dc11a2de156be220db8b7952d01#diff-f79a5ead735b3d0b34b6b94486918e1c |  
spark.ui.allowFramingFrom | 1.6.0 | SPARK-10589 | 5dbaf3d3911bbfa003bc75459aaad66b4f6e0c67#diff-f79a5ead735b3d0b34b6b94486918e1c |  
spark.ui.reverseProxy | 2.1.0 | SPARK-15487 | 92ce8d4849a0341c4636e70821b7be57ad3055b1#diff-364713d7776956cb8b0a771e9b62f82d |
spark.ui.reverseProxyUrl | 2.1.0 | SPARK-15487 | 92ce8d4849a0341c4636e70821b7be57ad3055b1#diff-364713d7776956cb8b0a771e9b62f82d |
spark.ui.killEnabled | 1.0.0 | SPARK-1202 | 211f97447b5f078afcb1619a08d2e2349325f61a#diff-a40023c80383451b6e29ee7a6e0593e9 |
spark.ui.threadDumpsEnabled | 1.2.0 | SPARK-611 | 866c7bbe56f9c7fd96d3f4afe8a76405dc877a6e#diff-5d18fb70c572369a0fff0b97de94f265 |  
spark.ui.prometheus.enabled | 3.0.0 | SPARK-29064 | bbfaadb280a80b511a98d18881641c6d9851dd51#diff-f70174ad0759db1fb4cb36a7ff9324a7 |  
spark.ui.xXssProtection | 2.3.0 | SPARK-22188 | 5a07aca4d464e96d75ea17bf6768e24b829872ec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.xContentTypeOptions.enabled | 2.3.0 | SPARK-22188 | 5a07aca4d464e96d75ea17bf6768e24b829872ec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.strictTransportSecurity | 2.3.0 | SPARK-22188 | 5a07aca4d464e96d75ea17bf6768e24b829872ec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.requestHeaderSize | 2.2.3 | SPARK-26118 | 9ceee6f188e6c3794d31ce15cc61d29f907bebf7#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.timeline.tasks.maximum | 1.4.0 | SPARK-7296 | a5f7b3b9c7f05598a1cc8e582e5facee1029cd5e#diff-fa4cfb2cce1b925f55f41f2dfa8c8501 |  
spark.acls.enable | 1.1.0 | SPARK-1890 and SPARK-1891 | e3fe6571decfdc406ec6d505fd92f9f2b85a618c#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.ui.view.acls | 1.0.0 | SPARK-1189 | 7edbea41b43e0dc11a2de156be220db8b7952d01#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.ui.view.acls.groups | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.admin.acls | 1.1.0 | SPARK-1890 and SPARK-1891 | e3fe6571decfdc406ec6d505fd92f9f2b85a618c#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.admin.acls.groups | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.modify.acls | 1.1.0 | SPARK-1890 and SPARK-1891 | e3fe6571decfdc406ec6d505fd92f9f2b85a618c#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.modify.acls.groups | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.user.groups.mapping | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.ui.proxyRedirectUri | 3.0.0 | SPARK-30240 | a9fbd310300e57ed58818d7347f3c3172701c491#diff-f70174ad0759db1fb4cb36a7ff9324a7 |  
spark.ui.custom.executor.log.url | 3.0.0 | SPARK-26792 | d5bda2c9e8dde6afc075cc7f65b15fa9aa82231c#diff-f70174ad0759db1fb4cb36a7ff9324a7 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27806 from beliefer/add-version-to-UI-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/internal/config/UI.scala | 26 +++++++++++++++++++
 docs/configuration.md                         | 10 +++++++
 2 files changed, 36 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/UI.scala b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
index 60d985713d30e..231eecf086bbe 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/UI.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
@@ -25,31 +25,37 @@ private[spark] object UI {
 
   val UI_SHOW_CONSOLE_PROGRESS = ConfigBuilder("spark.ui.showConsoleProgress")
     .doc("When true, show the progress bar in the console.")
+    .version("1.2.1")
     .booleanConf
     .createWithDefault(false)
 
   val UI_CONSOLE_PROGRESS_UPDATE_INTERVAL =
     ConfigBuilder("spark.ui.consoleProgress.update.interval")
+      .version("2.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(200)
 
   val UI_ENABLED = ConfigBuilder("spark.ui.enabled")
     .doc("Whether to run the web UI for the Spark application.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(true)
 
   val UI_PORT = ConfigBuilder("spark.ui.port")
     .doc("Port for your application's dashboard, which shows memory and workload data.")
+    .version("0.7.0")
     .intConf
     .createWithDefault(4040)
 
   val UI_FILTERS = ConfigBuilder("spark.ui.filters")
     .doc("Comma separated list of filter class names to apply to the Spark Web UI.")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val UI_ALLOW_FRAMING_FROM = ConfigBuilder("spark.ui.allowFramingFrom")
+    .version("1.6.0")
     .stringConf
     .createOptional
 
@@ -61,6 +67,7 @@ private[spark] object UI {
       "through spark master/proxy public URL. This setting affects all the workers and " +
       "application UIs running in the cluster and must be set on all the workers, drivers " +
       " and masters.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -69,15 +76,18 @@ private[spark] object UI {
       "in front of Spark Master. This is useful when running proxy for authentication e.g. " +
       "OAuth proxy. Make sure this is a complete URL including scheme (http/https) and port to " +
       "reach your proxy.")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   val UI_KILL_ENABLED = ConfigBuilder("spark.ui.killEnabled")
     .doc("Allows jobs and stages to be killed from the web UI.")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val UI_THREAD_DUMPS_ENABLED = ConfigBuilder("spark.ui.threadDumpsEnabled")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -85,73 +95,88 @@ private[spark] object UI {
     .internal()
     .doc("Expose executor metrics at /metrics/executors/prometheus. " +
       "For master/worker/driver metrics, you need to configure `conf/metrics.properties`.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val UI_X_XSS_PROTECTION = ConfigBuilder("spark.ui.xXssProtection")
     .doc("Value for HTTP X-XSS-Protection response header")
+    .version("2.3.0")
     .stringConf
     .createWithDefaultString("1; mode=block")
 
   val UI_X_CONTENT_TYPE_OPTIONS = ConfigBuilder("spark.ui.xContentTypeOptions.enabled")
     .doc("Set to 'true' for setting X-Content-Type-Options HTTP response header to 'nosniff'")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val UI_STRICT_TRANSPORT_SECURITY = ConfigBuilder("spark.ui.strictTransportSecurity")
     .doc("Value for HTTP Strict Transport Security Response Header")
+    .version("2.3.0")
     .stringConf
     .createOptional
 
   val UI_REQUEST_HEADER_SIZE = ConfigBuilder("spark.ui.requestHeaderSize")
     .doc("Value for HTTP request header size in bytes.")
+    .version("2.2.3")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("8k")
 
   val UI_TIMELINE_TASKS_MAXIMUM = ConfigBuilder("spark.ui.timeline.tasks.maximum")
+    .version("1.4.0")
     .intConf
     .createWithDefault(1000)
 
   val ACLS_ENABLE = ConfigBuilder("spark.acls.enable")
+    .version("1.1.0")
     .booleanConf
     .createWithDefault(false)
 
   val UI_VIEW_ACLS = ConfigBuilder("spark.ui.view.acls")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val UI_VIEW_ACLS_GROUPS = ConfigBuilder("spark.ui.view.acls.groups")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val ADMIN_ACLS = ConfigBuilder("spark.admin.acls")
+    .version("1.1.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val ADMIN_ACLS_GROUPS = ConfigBuilder("spark.admin.acls.groups")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val MODIFY_ACLS = ConfigBuilder("spark.modify.acls")
+    .version("1.1.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val MODIFY_ACLS_GROUPS = ConfigBuilder("spark.modify.acls.groups")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val USER_GROUPS_MAPPING = ConfigBuilder("spark.user.groups.mapping")
+    .version("2.0.0")
     .stringConf
     .createWithDefault("org.apache.spark.security.ShellBasedGroupsMappingProvider")
 
   val PROXY_REDIRECT_URI = ConfigBuilder("spark.ui.proxyRedirectUri")
     .doc("Proxy address to use when responding with HTTP redirects.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -163,6 +188,7 @@ private[spark] object UI {
       "This configuration replaces original log urls in event log, which will be also effective " +
       "when accessing the application on history server. The new log urls must be permanent, " +
       "otherwise you might have dead link for executor log urls.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index c586ecf15a8f4..9cbe341db9117 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1031,6 +1031,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to run the web UI for the Spark application.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.ui.killEnabled</code></td>
@@ -1038,6 +1039,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Allows jobs and stages to be killed from the web UI.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.liveUpdate.period</code></td>
@@ -1062,6 +1064,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port for your application's dashboard, which shows memory and workload data.
   </td>
+  <td>0.7.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedJobs</code></td>
@@ -1093,6 +1096,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Enable running Spark Master as reverse proxy for worker and application UIs. In this mode, Spark master will reverse proxy the worker and application UIs to enable access without requiring direct access to their hosts. Use it with caution, as worker and application UI will not be accessible directly, you will only be able to access them through spark master/proxy public URL. This setting affects all the workers and application UIs running in the cluster and must be set on all the workers, drivers and masters.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.reverseProxyUrl</code></td>
@@ -1100,6 +1104,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     This is the URL where your proxy is running. This URL is for proxy which is running in front of Spark Master. This is useful when running proxy for authentication e.g. OAuth proxy. Make sure this is a complete URL including scheme (http/https) and port to reach your proxy.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.proxyRedirectUri</code></td>
@@ -1112,6 +1117,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>X-Forwarded-Context</code> request header), or by setting the proxy base in the Spark
     app's configuration.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.showConsoleProgress</code></td>
@@ -1123,6 +1129,7 @@ Apart from these, the following properties are also available, and may be useful
     <br/>
     <em>Note:</em> In shell environment, the default value of spark.ui.showConsoleProgress is true.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.ui.custom.executor.log.url</code></td>
@@ -1138,6 +1145,7 @@ Apart from these, the following properties are also available, and may be useful
     <p/>
     For now, only YARN mode supports this configuration
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.retainedExecutors</code></td>
@@ -1190,6 +1198,7 @@ Apart from these, the following properties are also available, and may be useful
     <br /><code>spark.com.test.filter1.param.name1=foo</code>
     <br /><code>spark.com.test.filter1.param.name2=bar</code>
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.requestHeaderSize</code></td>
@@ -1198,6 +1207,7 @@ Apart from these, the following properties are also available, and may be useful
     The maximum allowed size for a HTTP request header, in bytes unless otherwise specified.
     This setting applies for the Spark History Server too.
   </td>
+  <td>2.2.3</td>
 </tr>
 </table>
 

From 2b93493c202388d010bdd19aef0d9319be56e40e Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 5 Mar 2020 11:58:21 +0900
Subject: [PATCH 0418/1280] [SPARK-30889][SPARK-30913][CORE][DOC] Add version
 information to the configuration of Tests.scala and Worker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1.Add version information to the configuration of `Tests` and `Worker`.
2.Update the docs of `Worker`.

I sorted out some information of `Tests` show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.testing.memory | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-395d07dcd46359cca610ce74357f0bb4 |  
spark.testing.dynamicAllocation.scheduleInterval | 2.3.0 | SPARK-22864 | 4e9e6aee44bb2ddb41b567d659358b22fd824222#diff-b096353602813e47074ace09a3890d56 |  
spark.testing | 1.0.1 | SPARK-1606 | ce57624b8232159fe3ec6db228afc622133df591#diff-d239aee594001f8391676e1047a0381e |  
spark.test.noStageRetry | 1.2.0 | SPARK-3796 | f55218aeb1e9d638df6229b36a59a15ce5363482#diff-6a9ff7fb74fd490a50462d45db2d5e11 |  
spark.testing.reservedMemory | 1.6.0 | SPARK-12081 | 84c44b500b5c90dffbe1a6b0aa86f01699b09b96#diff-395d07dcd46359cca610ce74357f0bb4 |
spark.testing.nHosts | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.testing.nExecutorsPerHost | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.testing.nCoresPerExecutor | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.resources.warnings.testing | 3.1.0 | SPARK-29148 | 496f6ac86001d284cbfb7488a63dd3a168919c0f#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.testing.resourceProfileManager | 3.1.0 | SPARK-29148 | 496f6ac86001d284cbfb7488a63dd3a168919c0f#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  

I sorted out some information of `Worker` show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.worker.resourcesFile | 3.0.0 | SPARK-27369 | 7cbe01e8efc3f6cd3a0cac4bcfadea8fcc74a955#diff-b2fc8d6ab7ac5735085e2d6cfacb95da |  
spark.worker.timeout | 0.6.2 | None | e395aa295aeec6767df798bf1002b1f30983c1cd#diff-776a630ac2b2ec5fe85c07ca20a58fc0 |  
spark.worker.driverTerminateTimeout | 2.1.2 | SPARK-20843 | ebd72f453aa0b4f68760d28b3e93e6dd33856659#diff-829a8674171f92acd61007bedb1bfa4f |  
spark.worker.cleanup.enabled | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.cleanup.interval | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.cleanup.appDataTtl | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.preferConfiguredMasterAddress | 2.2.1 | SPARK-20529 | 75e5ea294c15ecfb7366ae15dce196aa92c87ca4#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.ui.port | 1.1.0 | SPARK-2857 | 12f99cf5f88faf94d9dbfe85cb72d0010a3a25ac#diff-48ca297b6536cb92362bec1487581f05 |  
spark.worker.ui.retainedExecutors | 1.5.0 | SPARK-9202 | c0686668ae6a92b6bb4801a55c3b78aedbee816a#diff-916ca56b663f178f302c265b7ef38499 |
spark.worker.ui.retainedDrivers | 1.5.0 | SPARK-9202 | c0686668ae6a92b6bb4801a55c3b78aedbee816a#diff-916ca56b663f178f302c265b7ef38499 |
spark.worker.ui.compressedLogFileLengthCacheSize | 2.0.2 | SPARK-17711 | 26e978a93f029e1a1b5c7524d0b52c8141b70997#diff-d239aee594001f8391676e1047a0381e |  
spark.worker.decommission.enabled | 3.1.0 | SPARK-20628 | d273a2bb0fac452a97f5670edd69d3e452e3e57e#diff-b2fc8d6ab7ac5735085e2d6cfacb95da |  

Supplemental configuration version information.

No

Exists UT

Closes #27783 from beliefer/add-version-to-tests-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/internal/config/Tests.scala  |  8 +++++++
 .../apache/spark/internal/config/Worker.scala | 22 +++++++++++++++++++
 docs/configuration.md                         |  4 +++-
 docs/spark-standalone.md                      | 10 +++++++--
 4 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
index 21660ab3a9512..232264d6c8e49 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
@@ -22,35 +22,43 @@ private[spark] object Tests {
   val TEST_USE_COMPRESSED_OOPS_KEY = "spark.test.useCompressedOops"
 
   val TEST_MEMORY = ConfigBuilder("spark.testing.memory")
+    .version("1.6.0")
     .longConf
     .createWithDefault(Runtime.getRuntime.maxMemory)
 
   val TEST_SCHEDULE_INTERVAL =
     ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval")
+      .version("2.3.0")
       .longConf
       .createWithDefault(100)
 
   val IS_TESTING = ConfigBuilder("spark.testing")
+    .version("1.0.1")
     .booleanConf
     .createOptional
 
   val TEST_NO_STAGE_RETRY = ConfigBuilder("spark.test.noStageRetry")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(false)
 
   val TEST_RESERVED_MEMORY = ConfigBuilder("spark.testing.reservedMemory")
+    .version("1.6.0")
     .longConf
     .createOptional
 
   val TEST_N_HOSTS = ConfigBuilder("spark.testing.nHosts")
+    .version("3.0.0")
     .intConf
     .createWithDefault(5)
 
   val TEST_N_EXECUTORS_HOST = ConfigBuilder("spark.testing.nExecutorsPerHost")
+    .version("3.0.0")
     .intConf
     .createWithDefault(4)
 
   val TEST_N_CORES_EXECUTOR = ConfigBuilder("spark.testing.nCoresPerExecutor")
+    .version("3.0.0")
     .intConf
     .createWithDefault(2)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
index f1eaae29f18df..619b636237e7c 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
@@ -28,47 +28,69 @@ private[spark] object Worker {
     .doc("Path to a file containing the resources allocated to the worker. " +
       "The file should be formatted as a JSON array of ResourceAllocation objects. " +
       "Only used internally in standalone mode.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val WORKER_TIMEOUT = ConfigBuilder("spark.worker.timeout")
+    .version("0.6.2")
     .longConf
     .createWithDefault(60)
 
   val WORKER_DRIVER_TERMINATE_TIMEOUT = ConfigBuilder("spark.worker.driverTerminateTimeout")
+    .version("2.1.2")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("10s")
 
   val WORKER_CLEANUP_ENABLED = ConfigBuilder("spark.worker.cleanup.enabled")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_CLEANUP_INTERVAL = ConfigBuilder("spark.worker.cleanup.interval")
+    .version("1.0.0")
     .longConf
     .createWithDefault(60 * 30)
 
   val APP_DATA_RETENTION = ConfigBuilder("spark.worker.cleanup.appDataTtl")
+    .version("1.0.0")
     .longConf
     .createWithDefault(7 * 24 * 3600)
 
   val PREFER_CONFIGURED_MASTER_ADDRESS = ConfigBuilder("spark.worker.preferConfiguredMasterAddress")
+    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_UI_PORT = ConfigBuilder("spark.worker.ui.port")
+    .version("1.1.0")
     .intConf
     .createOptional
 
   val WORKER_UI_RETAINED_EXECUTORS = ConfigBuilder("spark.worker.ui.retainedExecutors")
+    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val WORKER_UI_RETAINED_DRIVERS = ConfigBuilder("spark.worker.ui.retainedDrivers")
+    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val UNCOMPRESSED_LOG_FILE_LENGTH_CACHE_SIZE_CONF =
     ConfigBuilder("spark.worker.ui.compressedLogFileLengthCacheSize")
+<<<<<<< HEAD
     .intConf
     .createWithDefault(100)
+=======
+      .version("2.0.2")
+      .intConf
+      .createWithDefault(100)
+
+  private[spark] val WORKER_DECOMMISSION_ENABLED =
+    ConfigBuilder("spark.worker.decommission.enabled")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+>>>>>>> ebcff675e0c... [SPARK-30889][SPARK-30913][CORE][DOC] Add version information to the configuration of Tests.scala and Worker
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index 9cbe341db9117..b336289741a86 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -929,7 +929,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Spark UI
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.eventLog.logBlockUpdates.enabled</code></td>
   <td>false</td>
@@ -1153,6 +1153,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished executors the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.retainedDrivers</code></td>
@@ -1160,6 +1161,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished drivers the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.ui.retainedExecutions</code></td>
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 17b6772390d82..4d4b85e31c8ff 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -185,7 +185,7 @@ You can optionally configure the cluster further by setting environment variable
 SPARK_MASTER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.deploy.retainedApplications</code></td>
   <td>200</td>
@@ -242,6 +242,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Number of seconds after which the standalone deploy master considers a worker lost if it
     receives no heartbeats.
   </td>
+  <td>0.6.2</td>
 </tr>
 <tr>
   <td><code>spark.worker.resource.{resourceName}.amount</code></td>
@@ -269,13 +270,14 @@ SPARK_MASTER_OPTS supports the following system properties:
     find that resource. If the discovery script also does not find the resources, the worker will fail
     to start up.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
 SPARK_WORKER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.worker.cleanup.enabled</code></td>
   <td>false</td>
@@ -284,6 +286,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     mode, as YARN works differently. Only the directories of stopped applications are cleaned up.
     This should be enabled if spark.shuffle.service.db.enabled is "true"
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.interval</code></td>
@@ -292,6 +295,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     Controls the interval, in seconds, at which the worker cleans up old application work dirs
     on the local machine.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.appDataTtl</code></td>
@@ -302,6 +306,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     downloaded to each application work dir.  Over time, the work dirs can quickly fill up disk space,
     especially if you run jobs very frequently.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.db.enabled</code></td>
@@ -333,6 +338,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     Spark caches the uncompressed file size of compressed log files. This property controls the cache
     size.
   </td>
+  <td>2.0.2</td>
 </tr>
 </table>
 

From 528a07b1ee4dd1a53d7a8821e8bd41e328ee302c Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sun, 5 Apr 2020 13:27:21 +0900
Subject: [PATCH 0419/1280] Revert "[SPARK-30889][SPARK-30913][CORE][DOC] Add
 version information to the configuration of Tests.scala and Worker"

This reverts commit 2b93493c202388d010bdd19aef0d9319be56e40e.
---
 .../apache/spark/internal/config/Tests.scala  |  8 -------
 .../apache/spark/internal/config/Worker.scala | 22 -------------------
 docs/configuration.md                         |  4 +---
 docs/spark-standalone.md                      | 10 ++-------
 4 files changed, 3 insertions(+), 41 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
index 232264d6c8e49..21660ab3a9512 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
@@ -22,43 +22,35 @@ private[spark] object Tests {
   val TEST_USE_COMPRESSED_OOPS_KEY = "spark.test.useCompressedOops"
 
   val TEST_MEMORY = ConfigBuilder("spark.testing.memory")
-    .version("1.6.0")
     .longConf
     .createWithDefault(Runtime.getRuntime.maxMemory)
 
   val TEST_SCHEDULE_INTERVAL =
     ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval")
-      .version("2.3.0")
       .longConf
       .createWithDefault(100)
 
   val IS_TESTING = ConfigBuilder("spark.testing")
-    .version("1.0.1")
     .booleanConf
     .createOptional
 
   val TEST_NO_STAGE_RETRY = ConfigBuilder("spark.test.noStageRetry")
-    .version("1.2.0")
     .booleanConf
     .createWithDefault(false)
 
   val TEST_RESERVED_MEMORY = ConfigBuilder("spark.testing.reservedMemory")
-    .version("1.6.0")
     .longConf
     .createOptional
 
   val TEST_N_HOSTS = ConfigBuilder("spark.testing.nHosts")
-    .version("3.0.0")
     .intConf
     .createWithDefault(5)
 
   val TEST_N_EXECUTORS_HOST = ConfigBuilder("spark.testing.nExecutorsPerHost")
-    .version("3.0.0")
     .intConf
     .createWithDefault(4)
 
   val TEST_N_CORES_EXECUTOR = ConfigBuilder("spark.testing.nCoresPerExecutor")
-    .version("3.0.0")
     .intConf
     .createWithDefault(2)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
index 619b636237e7c..f1eaae29f18df 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
@@ -28,69 +28,47 @@ private[spark] object Worker {
     .doc("Path to a file containing the resources allocated to the worker. " +
       "The file should be formatted as a JSON array of ResourceAllocation objects. " +
       "Only used internally in standalone mode.")
-    .version("3.0.0")
     .stringConf
     .createOptional
 
   val WORKER_TIMEOUT = ConfigBuilder("spark.worker.timeout")
-    .version("0.6.2")
     .longConf
     .createWithDefault(60)
 
   val WORKER_DRIVER_TERMINATE_TIMEOUT = ConfigBuilder("spark.worker.driverTerminateTimeout")
-    .version("2.1.2")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("10s")
 
   val WORKER_CLEANUP_ENABLED = ConfigBuilder("spark.worker.cleanup.enabled")
-    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_CLEANUP_INTERVAL = ConfigBuilder("spark.worker.cleanup.interval")
-    .version("1.0.0")
     .longConf
     .createWithDefault(60 * 30)
 
   val APP_DATA_RETENTION = ConfigBuilder("spark.worker.cleanup.appDataTtl")
-    .version("1.0.0")
     .longConf
     .createWithDefault(7 * 24 * 3600)
 
   val PREFER_CONFIGURED_MASTER_ADDRESS = ConfigBuilder("spark.worker.preferConfiguredMasterAddress")
-    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_UI_PORT = ConfigBuilder("spark.worker.ui.port")
-    .version("1.1.0")
     .intConf
     .createOptional
 
   val WORKER_UI_RETAINED_EXECUTORS = ConfigBuilder("spark.worker.ui.retainedExecutors")
-    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val WORKER_UI_RETAINED_DRIVERS = ConfigBuilder("spark.worker.ui.retainedDrivers")
-    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val UNCOMPRESSED_LOG_FILE_LENGTH_CACHE_SIZE_CONF =
     ConfigBuilder("spark.worker.ui.compressedLogFileLengthCacheSize")
-<<<<<<< HEAD
     .intConf
     .createWithDefault(100)
-=======
-      .version("2.0.2")
-      .intConf
-      .createWithDefault(100)
-
-  private[spark] val WORKER_DECOMMISSION_ENABLED =
-    ConfigBuilder("spark.worker.decommission.enabled")
-      .version("3.1.0")
-      .booleanConf
-      .createWithDefault(false)
->>>>>>> ebcff675e0c... [SPARK-30889][SPARK-30913][CORE][DOC] Add version information to the configuration of Tests.scala and Worker
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index b336289741a86..9cbe341db9117 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -929,7 +929,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Spark UI
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
 <tr>
   <td><code>spark.eventLog.logBlockUpdates.enabled</code></td>
   <td>false</td>
@@ -1153,7 +1153,6 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished executors the Spark UI and status APIs remember before garbage collecting.
   </td>
-  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.retainedDrivers</code></td>
@@ -1161,7 +1160,6 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished drivers the Spark UI and status APIs remember before garbage collecting.
   </td>
-  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.ui.retainedExecutions</code></td>
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 4d4b85e31c8ff..17b6772390d82 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -185,7 +185,7 @@ You can optionally configure the cluster further by setting environment variable
 SPARK_MASTER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
 <tr>
   <td><code>spark.deploy.retainedApplications</code></td>
   <td>200</td>
@@ -242,7 +242,6 @@ SPARK_MASTER_OPTS supports the following system properties:
     Number of seconds after which the standalone deploy master considers a worker lost if it
     receives no heartbeats.
   </td>
-  <td>0.6.2</td>
 </tr>
 <tr>
   <td><code>spark.worker.resource.{resourceName}.amount</code></td>
@@ -270,14 +269,13 @@ SPARK_MASTER_OPTS supports the following system properties:
     find that resource. If the discovery script also does not find the resources, the worker will fail
     to start up.
   </td>
-  <td>3.0.0</td>
 </tr>
 </table>
 
 SPARK_WORKER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
 <tr>
   <td><code>spark.worker.cleanup.enabled</code></td>
   <td>false</td>
@@ -286,7 +284,6 @@ SPARK_WORKER_OPTS supports the following system properties:
     mode, as YARN works differently. Only the directories of stopped applications are cleaned up.
     This should be enabled if spark.shuffle.service.db.enabled is "true"
   </td>
-  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.interval</code></td>
@@ -295,7 +292,6 @@ SPARK_WORKER_OPTS supports the following system properties:
     Controls the interval, in seconds, at which the worker cleans up old application work dirs
     on the local machine.
   </td>
-  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.appDataTtl</code></td>
@@ -306,7 +302,6 @@ SPARK_WORKER_OPTS supports the following system properties:
     downloaded to each application work dir.  Over time, the work dirs can quickly fill up disk space,
     especially if you run jobs very frequently.
   </td>
-  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.db.enabled</code></td>
@@ -338,7 +333,6 @@ SPARK_WORKER_OPTS supports the following system properties:
     Spark caches the uncompressed file size of compressed log files. This property controls the cache
     size.
   </td>
-  <td>2.0.2</td>
 </tr>
 </table>
 

From 5017bd6e42ef8b0d4dcf7eabf4293da1eecef16e Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 5 Mar 2020 11:58:21 +0900
Subject: [PATCH 0420/1280] [SPARK-30889][SPARK-30913][CORE][DOC] Add version
 information to the configuration of Tests.scala and Worker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

1.Add version information to the configuration of `Tests` and `Worker`.
2.Update the docs of `Worker`.

I sorted out some information of `Tests` show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.testing.memory | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-395d07dcd46359cca610ce74357f0bb4 |  
spark.testing.dynamicAllocation.scheduleInterval | 2.3.0 | SPARK-22864 | 4e9e6aee44bb2ddb41b567d659358b22fd824222#diff-b096353602813e47074ace09a3890d56 |  
spark.testing | 1.0.1 | SPARK-1606 | ce57624b8232159fe3ec6db228afc622133df591#diff-d239aee594001f8391676e1047a0381e |  
spark.test.noStageRetry | 1.2.0 | SPARK-3796 | f55218aeb1e9d638df6229b36a59a15ce5363482#diff-6a9ff7fb74fd490a50462d45db2d5e11 |  
spark.testing.reservedMemory | 1.6.0 | SPARK-12081 | 84c44b500b5c90dffbe1a6b0aa86f01699b09b96#diff-395d07dcd46359cca610ce74357f0bb4 |
spark.testing.nHosts | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.testing.nExecutorsPerHost | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.testing.nCoresPerExecutor | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.resources.warnings.testing | 3.1.0 | SPARK-29148 | 496f6ac86001d284cbfb7488a63dd3a168919c0f#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  
spark.testing.resourceProfileManager | 3.1.0 | SPARK-29148 | 496f6ac86001d284cbfb7488a63dd3a168919c0f#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 |  

I sorted out some information of `Worker` show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.worker.resourcesFile | 3.0.0 | SPARK-27369 | 7cbe01e8efc3f6cd3a0cac4bcfadea8fcc74a955#diff-b2fc8d6ab7ac5735085e2d6cfacb95da |  
spark.worker.timeout | 0.6.2 | None | e395aa295aeec6767df798bf1002b1f30983c1cd#diff-776a630ac2b2ec5fe85c07ca20a58fc0 |  
spark.worker.driverTerminateTimeout | 2.1.2 | SPARK-20843 | ebd72f453aa0b4f68760d28b3e93e6dd33856659#diff-829a8674171f92acd61007bedb1bfa4f |  
spark.worker.cleanup.enabled | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.cleanup.interval | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.cleanup.appDataTtl | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.preferConfiguredMasterAddress | 2.2.1 | SPARK-20529 | 75e5ea294c15ecfb7366ae15dce196aa92c87ca4#diff-916ca56b663f178f302c265b7ef38499 |  
spark.worker.ui.port | 1.1.0 | SPARK-2857 | 12f99cf5f88faf94d9dbfe85cb72d0010a3a25ac#diff-48ca297b6536cb92362bec1487581f05 |  
spark.worker.ui.retainedExecutors | 1.5.0 | SPARK-9202 | c0686668ae6a92b6bb4801a55c3b78aedbee816a#diff-916ca56b663f178f302c265b7ef38499 |
spark.worker.ui.retainedDrivers | 1.5.0 | SPARK-9202 | c0686668ae6a92b6bb4801a55c3b78aedbee816a#diff-916ca56b663f178f302c265b7ef38499 |
spark.worker.ui.compressedLogFileLengthCacheSize | 2.0.2 | SPARK-17711 | 26e978a93f029e1a1b5c7524d0b52c8141b70997#diff-d239aee594001f8391676e1047a0381e |  
spark.worker.decommission.enabled | 3.1.0 | SPARK-20628 | d273a2bb0fac452a97f5670edd69d3e452e3e57e#diff-b2fc8d6ab7ac5735085e2d6cfacb95da |  

Supplemental configuration version information.

No

Exists UT

Closes #27783 from beliefer/add-version-to-tests-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/internal/config/Tests.scala  |  8 ++++++++
 .../org/apache/spark/internal/config/Worker.scala | 15 +++++++++++++--
 docs/configuration.md                             |  4 +++-
 docs/spark-standalone.md                          | 10 ++++++++--
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
index 21660ab3a9512..232264d6c8e49 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
@@ -22,35 +22,43 @@ private[spark] object Tests {
   val TEST_USE_COMPRESSED_OOPS_KEY = "spark.test.useCompressedOops"
 
   val TEST_MEMORY = ConfigBuilder("spark.testing.memory")
+    .version("1.6.0")
     .longConf
     .createWithDefault(Runtime.getRuntime.maxMemory)
 
   val TEST_SCHEDULE_INTERVAL =
     ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval")
+      .version("2.3.0")
       .longConf
       .createWithDefault(100)
 
   val IS_TESTING = ConfigBuilder("spark.testing")
+    .version("1.0.1")
     .booleanConf
     .createOptional
 
   val TEST_NO_STAGE_RETRY = ConfigBuilder("spark.test.noStageRetry")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(false)
 
   val TEST_RESERVED_MEMORY = ConfigBuilder("spark.testing.reservedMemory")
+    .version("1.6.0")
     .longConf
     .createOptional
 
   val TEST_N_HOSTS = ConfigBuilder("spark.testing.nHosts")
+    .version("3.0.0")
     .intConf
     .createWithDefault(5)
 
   val TEST_N_EXECUTORS_HOST = ConfigBuilder("spark.testing.nExecutorsPerHost")
+    .version("3.0.0")
     .intConf
     .createWithDefault(4)
 
   val TEST_N_CORES_EXECUTOR = ConfigBuilder("spark.testing.nCoresPerExecutor")
+    .version("3.0.0")
     .intConf
     .createWithDefault(2)
 }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
index f1eaae29f18df..a8072712c46ce 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala
@@ -28,47 +28,58 @@ private[spark] object Worker {
     .doc("Path to a file containing the resources allocated to the worker. " +
       "The file should be formatted as a JSON array of ResourceAllocation objects. " +
       "Only used internally in standalone mode.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val WORKER_TIMEOUT = ConfigBuilder("spark.worker.timeout")
+    .version("0.6.2")
     .longConf
     .createWithDefault(60)
 
   val WORKER_DRIVER_TERMINATE_TIMEOUT = ConfigBuilder("spark.worker.driverTerminateTimeout")
+    .version("2.1.2")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("10s")
 
   val WORKER_CLEANUP_ENABLED = ConfigBuilder("spark.worker.cleanup.enabled")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_CLEANUP_INTERVAL = ConfigBuilder("spark.worker.cleanup.interval")
+    .version("1.0.0")
     .longConf
     .createWithDefault(60 * 30)
 
   val APP_DATA_RETENTION = ConfigBuilder("spark.worker.cleanup.appDataTtl")
+    .version("1.0.0")
     .longConf
     .createWithDefault(7 * 24 * 3600)
 
   val PREFER_CONFIGURED_MASTER_ADDRESS = ConfigBuilder("spark.worker.preferConfiguredMasterAddress")
+    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
   val WORKER_UI_PORT = ConfigBuilder("spark.worker.ui.port")
+    .version("1.1.0")
     .intConf
     .createOptional
 
   val WORKER_UI_RETAINED_EXECUTORS = ConfigBuilder("spark.worker.ui.retainedExecutors")
+    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val WORKER_UI_RETAINED_DRIVERS = ConfigBuilder("spark.worker.ui.retainedDrivers")
+    .version("1.5.0")
     .intConf
     .createWithDefault(1000)
 
   val UNCOMPRESSED_LOG_FILE_LENGTH_CACHE_SIZE_CONF =
     ConfigBuilder("spark.worker.ui.compressedLogFileLengthCacheSize")
-    .intConf
-    .createWithDefault(100)
+      .version("2.0.2")
+      .intConf
+      .createWithDefault(100)
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index 9cbe341db9117..b336289741a86 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -929,7 +929,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Spark UI
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.eventLog.logBlockUpdates.enabled</code></td>
   <td>false</td>
@@ -1153,6 +1153,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished executors the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.retainedDrivers</code></td>
@@ -1160,6 +1161,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished drivers the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.ui.retainedExecutions</code></td>
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 17b6772390d82..4d4b85e31c8ff 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -185,7 +185,7 @@ You can optionally configure the cluster further by setting environment variable
 SPARK_MASTER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.deploy.retainedApplications</code></td>
   <td>200</td>
@@ -242,6 +242,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Number of seconds after which the standalone deploy master considers a worker lost if it
     receives no heartbeats.
   </td>
+  <td>0.6.2</td>
 </tr>
 <tr>
   <td><code>spark.worker.resource.{resourceName}.amount</code></td>
@@ -269,13 +270,14 @@ SPARK_MASTER_OPTS supports the following system properties:
     find that resource. If the discovery script also does not find the resources, the worker will fail
     to start up.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
 SPARK_WORKER_OPTS supports the following system properties:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.worker.cleanup.enabled</code></td>
   <td>false</td>
@@ -284,6 +286,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     mode, as YARN works differently. Only the directories of stopped applications are cleaned up.
     This should be enabled if spark.shuffle.service.db.enabled is "true"
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.interval</code></td>
@@ -292,6 +295,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     Controls the interval, in seconds, at which the worker cleans up old application work dirs
     on the local machine.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.cleanup.appDataTtl</code></td>
@@ -302,6 +306,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     downloaded to each application work dir.  Over time, the work dirs can quickly fill up disk space,
     especially if you run jobs very frequently.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.db.enabled</code></td>
@@ -333,6 +338,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     Spark caches the uncompressed file size of compressed log files. This property controls the cache
     size.
   </td>
+  <td>2.0.2</td>
 </tr>
 </table>
 

From 3263f01af74f711b8f6656994cf2e2a293ec1c05 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Sun, 5 Apr 2020 16:37:03 -0700
Subject: [PATCH 0421/1280] [SPARK-29554][SQL][FOLLOWUP] Update Auto-generated
 Alias Name for Version

### What changes were proposed in this pull request?
The auto-generated alias name of built-in function `version()` is `sparkversion()`. After this PR, it is updated to `version()`.

### Why are the changes needed?
Based on our auto-generated alias name convention for the built-in functions, the alias names should be consistent with the function names.

This built-in function `version` is added in the upcoming Spark 3.0. Thus, we should fix it before the release.

### Does this PR introduce any user-facing change?
Yes. Update the column name in schema if users do not specify the alias.

### How was this patch tested?
Added a test case.

Closes #28131 from gatorsmile/spark-29554followup.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 39a6f518cbc1e203a447c109d62a762cce031ac9)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/sql/catalyst/expressions/misc.scala      | 1 +
 .../test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala  | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index f576873829f27..8ce3ddd30a69e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -173,6 +173,7 @@ case class SparkVersion() extends LeafExpression with CodegenFallback {
   override def nullable: Boolean = false
   override def foldable: Boolean = true
   override def dataType: DataType = StringType
+  override def prettyName: String = "version"
   override def eval(input: InternalRow): Any = {
     UTF8String.fromString(SPARK_VERSION_SHORT + " " + SPARK_REVISION)
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
index 5ab06b1ebebf6..16edf35bb99f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MiscFunctionsSuite.scala
@@ -34,9 +34,11 @@ class MiscFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("version") {
+    val df = sql("SELECT version()")
     checkAnswer(
-      Seq("").toDF("a").selectExpr("version()"),
+      df,
       Row(SPARK_VERSION_SHORT + " " + SPARK_REVISION))
+    assert(df.schema.fieldNames === Seq("version()"))
   }
 }
 

From 2a5f0aba73fa7a933b605a4108001dca51a91eb5 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Mon, 6 Apr 2020 09:36:20 +0900
Subject: [PATCH 0422/1280] [SPARK-30921][PYSPARK] Predicates on python udf
 should not be pushdown through Aggregate

### What changes were proposed in this pull request?

This patch proposed to skip predicates on PythonUDFs to be pushdown through Aggregate.

### Why are the changes needed?

The predicates on PythonUDFs cannot be pushdown through Aggregate. Pushed down predicates cannot be evaluate because PythonUDFs cannot be evaluated on Filter and cause error like:

```
Caused by: java.lang.UnsupportedOperationException: Cannot generate code for expression: mean(input[1, struct<bar:bigint>, true].bar)
        at org.apache.spark.sql.catalyst.expressions.Unevaluable.doGenCode(Expression.scala:304)
        at org.apache.spark.sql.catalyst.expressions.Unevaluable.doGenCode$(Expression.scala:303)
        at org.apache.spark.sql.catalyst.expressions.PythonUDF.doGenCode(PythonUDF.scala:52)
        at org.apache.spark.sql.catalyst.expressions.Expression.$anonfun$genCode$3(Expression.scala:146)
        at scala.Option.getOrElse(Option.scala:189)
        at org.apache.spark.sql.catalyst.expressions.Expression.genCode(Expression.scala:141)
        at org.apache.spark.sql.catalyst.expressions.CastBase.doGenCode(Cast.scala:821)
        at org.apache.spark.sql.catalyst.expressions.Expression.$anonfun$genCode$3(Expression.scala:146)
        at scala.Option.getOrElse(Option.scala:189)
```

### Does this PR introduce any user-facing change?

Yes. Previously the predicates on PythonUDFs will be pushdown through Aggregate can cause error. After this change, the query can work.

### How was this patch tested?

Unit test.

Closes #28089 from viirya/SPARK-30921.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1f0287148977adb416001cb0988e919a2698c8e0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/tests/test_pandas_udf_grouped_agg.py    | 17 +++++++++++++++++
 .../sql/catalyst/optimizer/Optimizer.scala      |  5 +++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
index 21679785a769e..224c8ce97f559 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
@@ -491,6 +491,23 @@ def max_udf(v):
             agg2 = self.spark.sql("select max_udf(id) from table")
             assert_frame_equal(agg1.toPandas(), agg2.toPandas())
 
+    def test_no_predicate_pushdown_through(self):
+        # SPARK-30921: We should not pushdown predicates of PythonUDFs through Aggregate.
+        import numpy as np
+
+        @pandas_udf('float', PandasUDFType.GROUPED_AGG)
+        def mean(x):
+            return np.mean(x)
+
+        df = self.spark.createDataFrame([
+            Row(id=1, foo=42), Row(id=2, foo=1), Row(id=2, foo=2)
+        ])
+
+        agg = df.groupBy('id').agg(mean('foo').alias("mean"))
+        filtered = agg.filter(agg['mean'] > 40.0)
+
+        assert(filtered.collect()[0]["mean"] == 42.0)
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_udf_grouped_agg import *
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 30ad6bfe3b715..d93c4a5bc459a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -1204,9 +1204,10 @@ object PushPredicateThroughNonJoin extends Rule[LogicalPlan] with PredicateHelpe
 
   def getAliasMap(plan: Aggregate): AttributeMap[Expression] = {
     // Find all the aliased expressions in the aggregate list that don't include any actual
-    // AggregateExpression, and create a map from the alias to the expression
+    // AggregateExpression or PythonUDF, and create a map from the alias to the expression
     val aliasMap = plan.aggregateExpressions.collect {
-      case a: Alias if a.child.find(_.isInstanceOf[AggregateExpression]).isEmpty =>
+      case a: Alias if a.child.find(e => e.isInstanceOf[AggregateExpression] ||
+          PythonUDF.isGroupedAggPandasUDF(e)).isEmpty =>
         (a.toAttribute, a.child)
     }
     AttributeMap(aliasMap)

From e87572b60e6a4ee7274f414c75ab9c5b11c26a29 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Sun, 8 Mar 2020 12:31:57 +0900
Subject: [PATCH 0423/1280] [SPARK-31002][CORE][DOC] Add version information to
 the configuration of Core
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the configuration of `Core`.
Note: Because `Core` has a lot of configuration items, I split the items into four PR. Other PR will follows this PR.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.resources.discoveryPlugin | 3.0.0 | SPARK-30689 | 742e35f1d48c2523dda2ce21d73b7ab5ade20582#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.resourcesFile | 3.0.0 | SPARK-27835 | 6748b486a9afe8370786efb64a8c9f3470c62dcf#diff-6bdad48cfc34314e89599655442ff210 |  
SparkLauncher.DRIVER_EXTRA_CLASSPATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.extraClassPath
SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.extraJavaOptions
SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.extraLibraryPath
spark.driver.userClassPathFirst | 1.3.0 | SPARK-2996 | 6a1e0f967286945db13d94aeb6ed19f0a347c236#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
spark.driver.cores | 1.3.0 | SPARK-1507 | 2be82b1e66cd188456bbf1e5abb13af04d1629d5#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
SparkLauncher.DRIVER_MEMORY | 1.1.1 | SPARK-3243 | c1ffa3e4cdfbd1f84b5c8d8de5d0fb958a19e211#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.memory
spark.driver.memoryOverhead | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.dfsDir | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.layout | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.persistToDfs.enabled | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.allowErasureCoding | 3.0.0 | SPARK-29105 | 276aaaae8d404975f8701089e9f4dfecd16e0d9f#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.enabled | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.dir | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.compress | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.logBlockUpdates.enabled | 2.3.0 | SPARK-22050 | 1437e344ec0c29a44a19f4513986f5f184c44695#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.erasureCoding.enabled | 3.0.0 | SPARK-25855 | 35506dced739ef16136e9f3d5d48c638899d3cec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.testing | 1.0.1 | None | d4c8af87994acf3707027e6fab25363f51fd4615#diff-e4a5a68c15eed95d038acfed84b0b66a |  
spark.eventLog.buffer.kb | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.logStageExecutorMetrics | 3.0.0 | SPARK-30812 | 68d7edf9497bea2f73707d32ab55dd8e53088e7c#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.gcMetrics.youngGenerationGarbageCollectors | 3.0.0 | SPARK-25865 | e5c502c596563dce8eb58f86e42c1aea2c51ed17#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.gcMetrics.oldGenerationGarbageCollectors | 3.0.0 | SPARK-25865 | e5c502c596563dce8eb58f86e42c1aea2c51ed17#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.overwrite | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.longForm.enabled | 2.4.0 | SPARK-23820 | 71f70130f1b2b4ec70595627f0a02a88e2c0e27d#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.rolling.enabled | 3.0.0 | SPARK-28869 | 100fc58da54e026cda87832a10e2d06eaeccdf87#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.rolling.maxFileSize | 3.0.0 | SPARK-28869 | 100fc58da54e026cda87832a10e2d06eaeccdf87#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.id | 1.2.0 | SPARK-3377 | 79e45c9323455a51f25ed9acd0edd8682b4bbb88#diff-364713d7776956cb8b0a771e9b62f82d |  
SparkLauncher.EXECUTOR_EXTRA_CLASSPATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.extraClassPath
spark.executor.heartbeat.dropZeroAccumulatorUpdates | 3.0.0 | SPARK-25449 | 9362c5cc273fdd09f9b3b512e2f6b64bcefc25ab#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.heartbeatInterval | 1.1.0 | SPARK-2099 | 8d338f64c4eda45d22ae33f61ef7928011cc2846#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.executor.heartbeat.maxFailures | 1.6.2 | SPARK-13522 | 86bf93e65481b8fe5d7532ca6d4cd29cafc9e9dd#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.executor.processTreeMetrics.enabled | 3.0.0 | SPARK-27324 | 387ce89a0631f1a4c6668b90ff2a7bbcf11919cd#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.metrics.pollingInterval | 3.0.0 | SPARK-26329 | 80ab19b9fd268adfc419457f12b99a5da7b6d1c7#diff-6bdad48cfc34314e89599655442ff210 |  
SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.extraJavaOptions
SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.extraLibraryPath
spark.executor.userClassPathFirst | 1.3.0 | SPARK-2996 | 6a1e0f967286945db13d94aeb6ed19f0a347c236#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
SparkLauncher.EXECUTOR_CORES | 1.0.0 | SPARK-1126 | 1617816090e7b20124a512a43860a21232ebf511#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.cores
SparkLauncher.EXECUTOR_MEMORY | 0.7.0 | None | 696eec32c982ca516c506de33f383a173bcbd131#diff-4f50ad37deb6742ad45472636c9a870b | spark.executor.memory
spark.executor.memoryOverhead | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6bdad48cfc34314e89599655442ff210 |  
spark.cores.max | 0.6.0 | None | 0a472840030e4e7e84fe748f7bfa49f1ece599c5#diff-b6cc54c092b861f645c3cd69ea0f91e2 |  
spark.memory.offHeap.enabled | 1.6.0 | SPARK-12251 | 9870e5c7af87190167ca3845ede918671b9420ca#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.memory.offHeap.size | 1.6.0 | SPARK-12251 | 9870e5c7af87190167ca3845ede918671b9420ca#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.memory.storageFraction | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.memory.fraction | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.storage.safetyFraction | 1.1.0 | [SPARK-1777 | ecf30ee7e78ea59c462c54db0fde5328f997466c#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.unrollMemoryThreshold | 1.1.0 | SPARK-1777 | ecf30ee7e78ea59c462c54db0fde5328f997466c#diff-692a329b5a7fb4134c55d559457b94e4 |
spark.storage.replication.proactive | 2.2.0 | SPARK-15355 | fa7c582e9442b985a0493fb1dd15b3fb9b6031b4#diff-186864190089a718680accb51de5f0d4 |  
spark.storage.memoryMapThreshold | 0.9.2 | SPARK-1145 | 76339495153dd895667ad609815c887b2c8960ea#diff-abd96f2ae793cd6ea6aab5b96a3c1d7a |
spark.storage.replication.policy | 2.1.0 | SPARK-15353 | a26afd52198523dbd51dc94053424494638c7de5#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.replication.topologyMapper | 2.1.0 | SPARK-15353 | a26afd52198523dbd51dc94053424494638c7de5#diff-186864190089a718680accb51de5f0d4 |
spark.storage.cachedPeersTtl | 1.1.1 | SPARK-3495 and SPARK-3496 | be0cc9952d6c8b4cfe9ff10a761e0677cba64489#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.maxReplicationFailures | 1.1.1 | SPARK-3495 and SPARK-3496 | be0cc9952d6c8b4cfe9ff10a761e0677cba64489#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.replication.topologyFile | 2.1.0 | SPARK-15353 | a26afd52198523dbd51dc94053424494638c7de5#diff-e550ce522c12a31d805a7d0f41e802af |  
spark.storage.exceptionOnPinLeak | 1.6.2 | SPARK-13566 | ab006523b840b1d2dbf3f5ff0a238558e7665a1e#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.storage.blockManagerTimeoutIntervalMs | 0.7.3 | None | 9085ebf3750c7d9bb7c6b5f6b4bdc5b807af93c2#diff-76170a9c8f67b542bc58240a0a12fe08 |  
spark.storage.blockManagerSlaveTimeoutMs | 0.7.0 | None | 97434f49b8c029e9b78c91ec5f58557cd1b5c943#diff-2ce6374aac24d70c69182b067216e684 |
spark.storage.cleanupFilesAfterExecutorExit | 2.4.0 | SPARK-24340 | 8ef167a5f9ba8a79bb7ca98a9844fe9cfcfea060#diff-916ca56b663f178f302c265b7ef38499 |  
spark.diskStore.subDirectories | 0.6.0 | None | 815d6bd69a0c1ba0e94fc0785f5c3619b37f19c5#diff-e8b73c5b81c403a5e5d581f97624c510 |  
spark.block.failures.beforeLocationRefresh | 2.0.0 | SPARK-13328 | ff776b2fc1cd4c571fd542dbf807e6fa3373cb34#diff-2b643ea78c1add0381754b1f47eec132 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27847 from beliefer/add-version-to-core-config-part-one.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md | 42 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index b336289741a86..7afb715bc765b 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -150,6 +150,7 @@ of the most common options to set are:
   <td>
     Number of cores to use for the driver process, only in cluster mode.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.maxResultSize</code></td>
@@ -176,6 +177,7 @@ of the most common options to set are:
     Instead, please set this through the <code>--driver-memory</code> command line option
     or in your default properties file.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.driver.memoryOverhead</code></td>
@@ -192,6 +194,7 @@ of the most common options to set are:
     driver is determined by the sum of <code>spark.driver.memoryOverhead</code> 
     and <code>spark.driver.memory</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.amount</code></td>
@@ -234,6 +237,7 @@ of the most common options to set are:
     returns the resource information for that resource. It tries the discovery
     script last if none of the plugins return information for that resource.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.memory</code></td>
@@ -242,6 +246,7 @@ of the most common options to set are:
     Amount of memory to use per executor process, in the same format as JVM memory strings with
     a size unit suffix ("k", "m", "g" or "t") (e.g. <code>512m</code>, <code>2g</code>).
   </td>
+  <td>0.7.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.pyspark.memory</code></td>
@@ -276,6 +281,7 @@ of the most common options to set are:
     <code>spark.executor.memory</code>, <code>spark.memory.offHeap.size</code> and 
     <code>spark.executor.pyspark.memory</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.resource.{resourceName}.amount</code></td>
@@ -384,6 +390,7 @@ of the most common options to set are:
     <code>spark.history.fs.driverlog.cleaner.enabled</code> is true and, if they are older than max age configured
     by setting <code>spark.history.fs.driverlog.cleaner.maxAge</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.persistToDfs.enabled</code></td>
@@ -394,6 +401,7 @@ of the most common options to set are:
     will not be persisted. Additionally, enable the cleaner by setting <code>spark.history.fs.driverlog.cleaner.enabled</code>
     to true in <a href="monitoring.html#spark-history-server-configuration-options">Spark History Server</a>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.layout</code></td>
@@ -403,6 +411,7 @@ of the most common options to set are:
     it uses the layout for the first appender defined in log4j.properties. If that is also not configured, driver logs
     use the default layout.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.allowErasureCoding</code></td>
@@ -413,6 +422,7 @@ of the most common options to set are:
     written by the application. Note that even if this is true, Spark will still not force the
     file to use erasure coding, it will simply use file system defaults.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -433,6 +443,7 @@ Apart from these, the following properties are also available, and may be useful
     Instead, please set this through the <code>--driver-class-path</code> command line option or in
     your default properties file.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.defaultJavaOptions</code></td>
@@ -470,6 +481,7 @@ Apart from these, the following properties are also available, and may be useful
 
     <code>spark.driver.defaultJavaOptions</code> will be prepended to this configuration.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.extraLibraryPath</code></td>
@@ -482,6 +494,7 @@ Apart from these, the following properties are also available, and may be useful
     Instead, please set this through the <code>--driver-library-path</code> command line option or in
     your default properties file.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.userClassPathFirst</code></td>
@@ -493,6 +506,7 @@ Apart from these, the following properties are also available, and may be useful
 
     This is used in cluster mode only.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.extraClassPath</code></td>
@@ -502,6 +516,7 @@ Apart from these, the following properties are also available, and may be useful
     backwards-compatibility with older versions of Spark. Users typically should not need to set
     this option.
   </td>
+  <td>1.0.0</td> 
 </tr>
 <tr>
   <td><code>spark.executor.defaultJavaOptions</code></td>
@@ -539,6 +554,7 @@ Apart from these, the following properties are also available, and may be useful
 
     <code>spark.executor.defaultJavaOptions</code> will be prepended to this configuration.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.extraLibraryPath</code></td>
@@ -546,6 +562,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Set a special library path to use when launching executor JVM's.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.maxRetainedFiles</code></td>
@@ -600,6 +617,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) Same functionality as <code>spark.driver.userClassPathFirst</code>, but
     applied to executor instances.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.executorEnv.[EnvironmentVariableName]</code></td>
@@ -937,6 +955,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to log events for every block update, if <code>spark.eventLog.enabled</code> is true.
     *Warning*: This will increase the size of the event log considerably.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.longForm.enabled</code></td>
@@ -944,6 +963,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     If true, use the long form of call sites in the event log. Otherwise use the short form.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.compress</code></td>
@@ -951,6 +971,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to compress logged events, if <code>spark.eventLog.enabled</code> is true.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.compression.codec</code></td>
@@ -970,6 +991,7 @@ Apart from these, the following properties are also available, and may be useful
     Note that even if this is true, Spark will still not force the file to use erasure coding, it
     will simply use filesystem defaults.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.dir</code></td>
@@ -980,6 +1002,7 @@ Apart from these, the following properties are also available, and may be useful
     events specific to the application in this directory. Users may want to set this to
     a unified location like an HDFS directory so history files can be read by the history server.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.enabled</code></td>
@@ -988,6 +1011,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to log Spark events, useful for reconstructing the Web UI after the application has
     finished.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.overwrite</code></td>
@@ -995,6 +1019,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to overwrite any existing files.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.buffer.kb</code></td>
@@ -1002,6 +1027,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Buffer size to use when writing to output streams, in KiB unless otherwise specified.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.rolling.enabled</code></td>
@@ -1010,6 +1036,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether rolling over event log files is enabled. If set to true, it cuts down each event
     log file to the configured size.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.rolling.maxFileSize</code></td>
@@ -1017,6 +1044,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     When <code>spark.eventLog.rolling.enabled=true</code>, specifies the max size of event log file before it's rolled over.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.dagGraph.retainedRootRDDs</code></td>
@@ -1413,6 +1441,7 @@ Apart from these, the following properties are also available, and may be useful
     garbage collection when increasing this value, see
     <a href="tuning.html#memory-management-overview">this description</a>.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.memory.storageFraction</code></td>
@@ -1424,6 +1453,7 @@ Apart from these, the following properties are also available, and may be useful
     Leaving this at the default value is recommended. For more detail, see
     <a href="tuning.html#memory-management-overview">this description</a>.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.memory.offHeap.enabled</code></td>
@@ -1432,6 +1462,7 @@ Apart from these, the following properties are also available, and may be useful
     If true, Spark will attempt to use off-heap memory for certain operations. If off-heap memory 
     use is enabled, then <code>spark.memory.offHeap.size</code> must be positive.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.memory.offHeap.size</code></td>
@@ -1442,6 +1473,7 @@ Apart from these, the following properties are also available, and may be useful
     must fit within some hard limit then be sure to shrink your JVM heap size accordingly.
     This must be set to a positive value when <code>spark.memory.offHeap.enabled=true</code>.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.storage.replication.proactive</code></td>
@@ -1451,6 +1483,7 @@ Apart from these, the following properties are also available, and may be useful
     executor failures are replenished if there are any existing available replicas. This tries
     to get the replication level of the block to the initial number.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.periodicGC.interval</code></td>
@@ -1528,6 +1561,7 @@ Apart from these, the following properties are also available, and may be useful
     In standalone and Mesos coarse-grained modes, for more detail, see
     <a href="spark-standalone.html#Executors Scheduling">this description</a>.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.default.parallelism</code></td>
@@ -1553,6 +1587,7 @@ Apart from these, the following properties are also available, and may be useful
     the driver know that the executor is still alive and update it with metrics for in-progress
     tasks. spark.executor.heartbeatInterval should be significantly less than
     spark.network.timeout</td>
+    <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.files.fetchTimeout</code></td>
@@ -1625,6 +1660,7 @@ Apart from these, the following properties are also available, and may be useful
     unless specified otherwise. This prevents Spark from memory mapping very small blocks. In general, 
     memory mapping has high overhead for blocks close to or below the page size of the operating system.
   </td>
+  <td>0.9.2</td>
 </tr>
 <tr>
   <td><code>spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version</code></td>
@@ -1640,7 +1676,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Executor Metrics
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.eventLog.logStageExecutorMetrics</code></td>
   <td>false</td>
@@ -1651,6 +1687,7 @@ Apart from these, the following properties are also available, and may be useful
     and this is always done; this configuration is only to determine if aggregated metric peaks
     are written to the event log.
   </td>
+  <td>3.0.0</td>
 </tr>
   <td><code>spark.executor.processTreeMetrics.enabled</code></td>
   <td>false</td>
@@ -1661,6 +1698,7 @@ Apart from these, the following properties are also available, and may be useful
     <em>Note:</em> The process tree metrics are collected only if the /proc filesystem
     exists.
   </td>
+  <td>3.0.0</td>
 <tr>
   <td><code>spark.executor.metrics.pollingInterval</code></td>
   <td>0</td>
@@ -1671,6 +1709,7 @@ Apart from these, the following properties are also available, and may be useful
     specified by <code>spark.executor.heartbeatInterval</code>).
     If positive, the polling is done at this interval.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -1854,6 +1893,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.deploy.defaultCores</code> on Spark's standalone cluster manager, or
     infinite (all available cores) on Mesos.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait</code></td>

From e323696accdaa852c1d953b097fdd5ab0599a646 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 12 Mar 2020 11:03:47 +0900
Subject: [PATCH 0424/1280] [SPARK-30911][CORE][DOC] Add version information to
 the configuration of Status
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1.Add version information to the configuration of `Status`.
2.Update the docs of `Status`.
3.By the way supplementary documentation about https://github.com/apache/spark/pull/27847

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.appStateStore.asyncTracking.enable | 2.3.0 | SPARK-20653 | 772e4648d95bda3353723337723543c741ea8476#diff-9ab674b7af7b2097f7d28cb6f5fd1e8c |  
spark.ui.liveUpdate.period | 2.3.0 | SPARK-20644 | c7f38e5adb88d43ef60662c5d6ff4e7a95bff580#diff-9ab674b7af7b2097f7d28cb6f5fd1e8c |  
spark.ui.liveUpdate.minFlushPeriod | 2.4.2 | SPARK-27394 | a8a2ba11ac10051423e58920062b50f328b06421#diff-9ab674b7af7b2097f7d28cb6f5fd1e8c |  
spark.ui.retainedJobs | 1.2.0 | SPARK-2321 | 9530316887612dca060a128fca34dd5a6ab2a9a9#diff-1f32bcb61f51133bd0959a4177a066a5 |  
spark.ui.retainedStages | 0.9.0 | None | 112c0a1776bbc866a1026a9579c6f72f293414c4#diff-1f32bcb61f51133bd0959a4177a066a5 | 0.9.0-incubating-SNAPSHOT
spark.ui.retainedTasks | 2.0.1 | SPARK-15083 | 55db26245d69bb02b7d7d5f25029b1a1cd571644#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.retainedDeadExecutors | 2.0.0 | SPARK-7729 | 9f4263392e492b5bc0acecec2712438ff9a257b7#diff-a0ba36f9b1f9829bf3c4689b05ab6cf2 |  
spark.ui.dagGraph.retainedRootRDDs | 2.1.0 | SPARK-17171 | cc87280fcd065b01667ca7a59a1a32c7ab757355#diff-3f492c527ea26679d4307041b28455b8 |  
spark.metrics.appStatusSource.enabled | 3.0.0 | SPARK-30060 | 60f20e5ea2000ab8f4a593b5e4217fd5637c5e22#diff-9f796ae06b0272c1f0a012652a5b68d0 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27848 from beliefer/add-version-to-status-config.

Lead-authored-by: beliefer <beliefer@163.com>
Co-authored-by: Jiaan Geng <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/internal/config/Status.scala |  9 ++++++++
 docs/configuration.md                         | 21 +++++++++++++------
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Status.scala b/core/src/main/scala/org/apache/spark/internal/config/Status.scala
index 3cc00a6f094cf..669fa07053cad 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Status.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Status.scala
@@ -22,36 +22,44 @@ import java.util.concurrent.TimeUnit
 private[spark] object Status {
 
   val ASYNC_TRACKING_ENABLED = ConfigBuilder("spark.appStateStore.asyncTracking.enable")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val LIVE_ENTITY_UPDATE_PERIOD = ConfigBuilder("spark.ui.liveUpdate.period")
+    .version("2.3.0")
     .timeConf(TimeUnit.NANOSECONDS)
     .createWithDefaultString("100ms")
 
   val LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD = ConfigBuilder("spark.ui.liveUpdate.minFlushPeriod")
     .doc("Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when " +
       "incoming task events are not fired frequently.")
+    .version("2.4.2")
     .timeConf(TimeUnit.NANOSECONDS)
     .createWithDefaultString("1s")
 
   val MAX_RETAINED_JOBS = ConfigBuilder("spark.ui.retainedJobs")
+    .version("1.2.0")
     .intConf
     .createWithDefault(1000)
 
   val MAX_RETAINED_STAGES = ConfigBuilder("spark.ui.retainedStages")
+    .version("0.9.0")
     .intConf
     .createWithDefault(1000)
 
   val MAX_RETAINED_TASKS_PER_STAGE = ConfigBuilder("spark.ui.retainedTasks")
+    .version("2.0.1")
     .intConf
     .createWithDefault(100000)
 
   val MAX_RETAINED_DEAD_EXECUTORS = ConfigBuilder("spark.ui.retainedDeadExecutors")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
   val MAX_RETAINED_ROOT_NODES = ConfigBuilder("spark.ui.dagGraph.retainedRootRDDs")
+    .version("2.1.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
@@ -59,6 +67,7 @@ private[spark] object Status {
     ConfigBuilder("spark.metrics.appStatusSource.enabled")
       .doc("Whether Dropwizard/Codahale metrics " +
         "will be reported for the status of the running spark app.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 }
diff --git a/docs/configuration.md b/docs/configuration.md
index 7afb715bc765b..d22038b5cb530 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -516,7 +516,7 @@ Apart from these, the following properties are also available, and may be useful
     backwards-compatibility with older versions of Spark. Users typically should not need to set
     this option.
   </td>
-  <td>1.0.0</td> 
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.defaultJavaOptions</code></td>
@@ -1052,6 +1052,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many DAG graph nodes the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.enabled</code></td>
@@ -1077,6 +1078,7 @@ Apart from these, the following properties are also available, and may be useful
     meaning only the last write will happen. For live applications, this avoids a few
     operations that we can live without when rapidly processing incoming task events.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.liveUpdate.minFlushPeriod</code></td>
@@ -1085,6 +1087,7 @@ Apart from these, the following properties are also available, and may be useful
     Minimum time elapsed before stale UI data is flushed. This avoids UI staleness when incoming
     task events are not fired frequently.
   </td>
+  <td>2.4.2</td>
 </tr>
 <tr>
   <td><code>spark.ui.port</code></td>
@@ -1101,6 +1104,7 @@ Apart from these, the following properties are also available, and may be useful
     How many jobs the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedStages</code></td>
@@ -1109,6 +1113,7 @@ Apart from these, the following properties are also available, and may be useful
     How many stages the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedTasks</code></td>
@@ -1117,6 +1122,7 @@ Apart from these, the following properties are also available, and may be useful
     How many tasks in one stage the Spark UI and status APIs remember before garbage collecting.
     This is a target maximum, and fewer elements may be retained in some circumstances.
   </td>
+  <td>2.0.1</td>
 </tr>
 <tr>
   <td><code>spark.ui.reverseProxy</code></td>
@@ -1211,6 +1217,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many dead executors the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.filters</code></td>
@@ -1581,13 +1588,15 @@ Apart from these, the following properties are also available, and may be useful
   </td>
 </tr>
 <tr>
-    <td><code>spark.executor.heartbeatInterval</code></td>
-    <td>10s</td>
-    <td>Interval between each executor's heartbeats to the driver.  Heartbeats let
+  <td><code>spark.executor.heartbeatInterval</code></td>
+  <td>10s</td>
+  <td>
+    Interval between each executor's heartbeats to the driver.  Heartbeats let
     the driver know that the executor is still alive and update it with metrics for in-progress
     tasks. spark.executor.heartbeatInterval should be significantly less than
-    spark.network.timeout</td>
-    <td>1.1.0</td>
+    spark.network.timeout
+  </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.files.fetchTimeout</code></td>

From 7d6aff0f1a0ece47a49a49ce62745da6ccbd9c8e Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 6 Apr 2020 11:04:21 +0900
Subject: [PATCH 0425/1280] [SPARK-27963][DOCS][FOLLOWUP] Update requirements
 for spark.dynamicAllocation.enabled

### What changes were proposed in this pull request?

This PR fixes the outdated requirement for `spark.dynamicAllocation.enabled=true`.

### Why are the changes needed?

This is found during 3.0.0 RC1 document review and testing. As described at `spark.dynamicAllocation.shuffleTracking.enabled` in the same table, we can enabled Dynamic Allocation without external shuffle service.

### Does this PR introduce any user-facing change?

Yes. (Doc.)

### How was this patch tested?

Manually generate the doc by `SKIP_API=1 jekyll build`

**BEFORE**
![Screen Shot 2020-04-05 at 2 31 23 PM](https://user-images.githubusercontent.com/9700541/78510472-29c0ae00-774a-11ea-9916-ba80015fae82.png)

**AFTER**
![Screen Shot 2020-04-05 at 2 29 25 PM](https://user-images.githubusercontent.com/9700541/78510434-ea925d00-7749-11ea-8db8-018955507fd5.png)

Closes #28132 from dongjoon-hyun/SPARK-DA-DOC.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 388644233285bc71e43f7d063da57d22ed35ae23)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index d22038b5cb530..8d1cc849bffe8 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2315,7 +2315,8 @@ Apart from these, the following properties are also available, and may be useful
     For more detail, see the description
     <a href="job-scheduling.html#dynamic-resource-allocation">here</a>.
     <br><br>
-    This requires <code>spark.shuffle.service.enabled</code> to be set.
+    This requires <code>spark.shuffle.service.enabled</code> or
+    <code>spark.dynamicAllocation.shuffleTracking.enabled</code> to be set.
     The following configurations are also relevant:
     <code>spark.dynamicAllocation.minExecutors</code>,
     <code>spark.dynamicAllocation.maxExecutors</code>, and

From 6b2f347ed2b4113eb9a94b3f9a54c13f5a61c132 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 6 Apr 2020 05:21:04 +0000
Subject: [PATCH 0426/1280] [SPARK-31353][SQL] Set a time zone in
 DateTimeBenchmark and DateTimeRebaseBenchmark

### What changes were proposed in this pull request?
In the PR, I propose to set the `America/Los_Angeles` time zone in the date-time benchmarks `DateTimeBenchmark` and `DateTimeRebaseBenchmark` via `withDefaultTimeZone(LA)` and `withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId)`.

The results of affected benchmarks was given on an Amazon EC2 instance w/ the configuration:
| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK8/11 |

### Why are the changes needed?
Performance of date-time functions can depend on the system JVM time zone or SQL config `spark.sql.session.timeZone`. The changes allow to avoid any fluctuations of benchmarks results related to time zones, and set a reliable baseline for future optimization.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By regenerating results of DateTimeBenchmark and DateTimeRebaseBenchmark.

Closes #28127 from MaxGekk/set-timezone-in-benchmarks.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 35e6a9deee91bf84001d7330bfe0677e4ecfe74a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeBenchmark-jdk11-results.txt       | 434 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 434 +++++++++---------
 .../DateTimeRebaseBenchmark-jdk11-results.txt |  88 ++--
 .../DateTimeRebaseBenchmark-results.txt       |  88 ++--
 .../benchmark/DateTimeBenchmark.scala         | 184 ++++----
 .../benchmark/DateTimeRebaseBenchmark.scala   | 133 +++---
 6 files changed, 689 insertions(+), 672 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 3e415fa48f919..df9735cb7b0d1 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -2,428 +2,428 @@
 Extract components
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    217            225          10         46.0          21.7       1.0X
-cast to timestamp wholestage on                     195            224          35         51.2          19.5       1.1X
+cast to timestamp wholestage off                    411            449          54         24.3          41.1       1.0X
+cast to timestamp wholestage on                     392            408          11         25.5          39.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                    735            739           5         13.6          73.5       1.0X
-year of timestamp wholestage on                     728            742          10         13.7          72.8       1.0X
+year of timestamp wholestage off                   1368           1376          11          7.3         136.8       1.0X
+year of timestamp wholestage on                    1272           1293          21          7.9         127.2       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                 940            941           2         10.6          94.0       1.0X
-quarter of timestamp wholestage on                  910            930          12         11.0          91.0       1.0X
+quarter of timestamp wholestage off                1607           1632          34          6.2         160.7       1.0X
+quarter of timestamp wholestage on                 1534           1553          22          6.5         153.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                   796            808          17         12.6          79.6       1.0X
-month of timestamp wholestage on                    688            706          11         14.5          68.8       1.2X
+month of timestamp wholestage off                  1252           1259          10          8.0         125.2       1.0X
+month of timestamp wholestage on                   1270           1282          13          7.9         127.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1013           1016           3          9.9         101.3       1.0X
-weekofyear of timestamp wholestage on               988           1011          18         10.1          98.8       1.0X
+weekofyear of timestamp wholestage off             2234           2242          12          4.5         223.4       1.0X
+weekofyear of timestamp wholestage on              1830           1844          10          5.5         183.0       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                     686            698          17         14.6          68.6       1.0X
-day of timestamp wholestage on                      693            708          13         14.4          69.3       1.0X
+day of timestamp wholestage off                    1248           1256          11          8.0         124.8       1.0X
+day of timestamp wholestage on                     1248           1255          10          8.0         124.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off               711            716           7         14.1          71.1       1.0X
-dayofyear of timestamp wholestage on                713            723          12         14.0          71.3       1.0X
+dayofyear of timestamp wholestage off              1290           1303          18          7.8         129.0       1.0X
+dayofyear of timestamp wholestage on               1285           1306          20          7.8         128.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off              706            726          28         14.2          70.6       1.0X
-dayofmonth of timestamp wholestage on               698            714          11         14.3          69.8       1.0X
+dayofmonth of timestamp wholestage off             1261           1263           2          7.9         126.1       1.0X
+dayofmonth of timestamp wholestage on              1253           1259           5          8.0         125.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off               797            798           2         12.6          79.7       1.0X
-dayofweek of timestamp wholestage on                794            807          18         12.6          79.4       1.0X
+dayofweek of timestamp wholestage off              1419           1421           2          7.0         141.9       1.0X
+dayofweek of timestamp wholestage on               1406           1417          11          7.1         140.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                 783            796          17         12.8          78.3       1.0X
-weekday of timestamp wholestage on                  759            778          21         13.2          75.9       1.0X
+weekday of timestamp wholestage off                1346           1350           6          7.4         134.6       1.0X
+weekday of timestamp wholestage on                 1348           1360          10          7.4         134.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    573            575           2         17.5          57.3       1.0X
-hour of timestamp wholestage on                     512            522          16         19.5          51.2       1.1X
+hour of timestamp wholestage off                   1051           1057           8          9.5         105.1       1.0X
+hour of timestamp wholestage on                     991           1002           9         10.1          99.1       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  550            555           7         18.2          55.0       1.0X
-minute of timestamp wholestage on                   507            527          21         19.7          50.7       1.1X
+minute of timestamp wholestage off                 1041           1042           1          9.6         104.1       1.0X
+minute of timestamp wholestage on                   971            978           9         10.3          97.1       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  562            587          35         17.8          56.2       1.0X
-second of timestamp wholestage on                   516            529          12         19.4          51.6       1.1X
+second of timestamp wholestage off                  989            991           3         10.1          98.9       1.0X
+second of timestamp wholestage on                   986            999          12         10.1          98.6       1.0X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         161            164           4         61.9          16.1       1.0X
-current_date wholestage on                          163            170           8         61.2          16.3       1.0X
+current_date wholestage off                         305            313          12         32.8          30.5       1.0X
+current_date wholestage on                          302            316          17         33.1          30.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    174            177           4         57.5          17.4       1.0X
-current_timestamp wholestage on                     159            173          12         62.8          15.9       1.1X
+current_timestamp wholestage off                    359            388          41         27.8          35.9       1.0X
+current_timestamp wholestage on                     300            412         143         33.3          30.0       1.2X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         672            688          23         14.9          67.2       1.0X
-cast to date wholestage on                          592            605          10         16.9          59.2       1.1X
+cast to date wholestage off                        1080           1083           5          9.3         108.0       1.0X
+cast to date wholestage on                         1045           1055          10          9.6         104.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                             806            809           4         12.4          80.6       1.0X
-last_day wholestage on                              700            713          14         14.3          70.0       1.2X
+last_day wholestage off                            1253           1253           1          8.0         125.3       1.0X
+last_day wholestage on                             1257           1272          20          8.0         125.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             639            683          63         15.7          63.9       1.0X
-next_day wholestage on                              628            640          15         15.9          62.8       1.0X
+next_day wholestage off                            1129           1137          11          8.9         112.9       1.0X
+next_day wholestage on                             1083           1092          11          9.2         108.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             688            695          10         14.5          68.8       1.0X
-date_add wholestage on                              588            603          13         17.0          58.8       1.2X
+date_add wholestage off                            1067           1071           6          9.4         106.7       1.0X
+date_add wholestage on                             1061           1072           7          9.4         106.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             588            600          18         17.0          58.8       1.0X
-date_sub wholestage on                              597            624          40         16.8          59.7       1.0X
+date_sub wholestage off                            1065           1068           4          9.4         106.5       1.0X
+date_sub wholestage on                             1065           1072           6          9.4         106.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                           802            816          20         12.5          80.2       1.0X
-add_months wholestage on                            777            836          61         12.9          77.7       1.0X
+add_months wholestage off                          1404           1409           7          7.1         140.4       1.0X
+add_months wholestage on                           1429           1439          15          7.0         142.9       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         3328           3635         435          3.0         332.8       1.0X
-format date wholestage on                          3335           3549         253          3.0         333.5       1.0X
+format date wholestage off                         5642           5741         140          1.8         564.2       1.0X
+format date wholestage on                          5874           5900          18          1.7         587.4       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       4414           4548         189          2.3         441.4       1.0X
-from_unixtime wholestage on                        4455           4499          65          2.2         445.5       1.0X
+from_unixtime wholestage off                       7504           7515          15          1.3         750.4       1.0X
+from_unixtime wholestage on                        7579           7612          37          1.3         757.9       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   781            790          13         12.8          78.1       1.0X
-from_utc_timestamp wholestage on                    819            823           5         12.2          81.9       1.0X
+from_utc_timestamp wholestage off                  1250           1254           6          8.0         125.0       1.0X
+from_utc_timestamp wholestage on                   1287           1294          10          7.8         128.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1101           1108          10          9.1         110.1       1.0X
-to_utc_timestamp wholestage on                     1044           1055          10          9.6         104.4       1.1X
+to_utc_timestamp wholestage off                    1916           1918           2          5.2         191.6       1.0X
+to_utc_timestamp wholestage on                     1781           1792          12          5.6         178.1       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        198            202           6         50.6          19.8       1.0X
-cast interval wholestage on                         181            193          14         55.2          18.1       1.1X
+cast interval wholestage off                        347            354           9         28.8          34.7       1.0X
+cast interval wholestage on                         308            313           3         32.5          30.8       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1037           1043          10          9.6         103.7       1.0X
-datediff wholestage on                             1006           1037          21          9.9         100.6       1.0X
+datediff wholestage off                            1884           1895          15          5.3         188.4       1.0X
+datediff wholestage on                             1824           1833           9          5.5         182.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      2957           2972          21          3.4         295.7       1.0X
-months_between wholestage on                       2931           2952          16          3.4         293.1       1.0X
+months_between wholestage off                      5674           5685          16          1.8         567.4       1.0X
+months_between wholestage on                       5600           5616          14          1.8         560.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1130           1147          24          0.9        1130.0       1.0X
-window wholestage on                              16253          17068         580          0.1       16253.2       0.1X
+window wholestage off                              1971           2180         295          0.5        1971.1       1.0X
+window wholestage on                              47323          47354          35          0.0       47322.7       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1361           1380          27          7.3         136.1       1.0X
-date_trunc YEAR wholestage on                      1346           1381          28          7.4         134.6       1.0X
+date_trunc YEAR wholestage off                     2571           2588          23          3.9         257.1       1.0X
+date_trunc YEAR wholestage on                      2581           2591           6          3.9         258.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1352           1364          16          7.4         135.2       1.0X
-date_trunc YYYY wholestage on                      1391           1424          37          7.2         139.1       1.0X
+date_trunc YYYY wholestage off                     2564           2568           5          3.9         256.4       1.0X
+date_trunc YYYY wholestage on                      2583           2592           6          3.9         258.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1346           1356          15          7.4         134.6       1.0X
-date_trunc YY wholestage on                        1355           1366           9          7.4         135.5       1.0X
+date_trunc YY wholestage off                       2574           2577           5          3.9         257.4       1.0X
+date_trunc YY wholestage on                        2579           2588           8          3.9         257.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1355           1365          15          7.4         135.5       1.0X
-date_trunc MON wholestage on                       1388           1457          78          7.2         138.8       1.0X
+date_trunc MON wholestage off                      2598           2599           1          3.8         259.8       1.0X
+date_trunc MON wholestage on                       2593           2604          11          3.9         259.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1336           1357          30          7.5         133.6       1.0X
-date_trunc MONTH wholestage on                     1361           1390          22          7.3         136.1       1.0X
+date_trunc MONTH wholestage off                    2599           2613          19          3.8         259.9       1.0X
+date_trunc MONTH wholestage on                     2604           2610           5          3.8         260.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1350           1361          16          7.4         135.0       1.0X
-date_trunc MM wholestage on                        1370           1384          13          7.3         137.0       1.0X
+date_trunc MM wholestage off                       2605           2613          10          3.8         260.5       1.0X
+date_trunc MM wholestage on                        2595           2601           4          3.9         259.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1328           1361          46          7.5         132.8       1.0X
-date_trunc DAY wholestage on                       1356           1432          69          7.4         135.6       1.0X
+date_trunc DAY wholestage off                      2361           2367           9          4.2         236.1       1.0X
+date_trunc DAY wholestage on                       2370           2383          11          4.2         237.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1403           1445          59          7.1         140.3       1.0X
-date_trunc DD wholestage on                        1359           1405          49          7.4         135.9       1.0X
+date_trunc DD wholestage off                       2376           2385          13          4.2         237.6       1.0X
+date_trunc DD wholestage on                        2368           2372           3          4.2         236.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1447           1494          67          6.9         144.7       1.0X
-date_trunc HOUR wholestage on                      1387           1416          31          7.2         138.7       1.0X
+date_trunc HOUR wholestage off                     2399           2404           7          4.2         239.9       1.0X
+date_trunc HOUR wholestage on                      2387           2399          14          4.2         238.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    206            215          13         48.5          20.6       1.0X
-date_trunc MINUTE wholestage on                     233            240           4         42.9          23.3       0.9X
+date_trunc MINUTE wholestage off                    357            370          18         28.0          35.7       1.0X
+date_trunc MINUTE wholestage on                     373            378           5         26.8          37.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    200            213          18         49.9          20.0       1.0X
-date_trunc SECOND wholestage on                     219            227           7         45.6          21.9       0.9X
+date_trunc SECOND wholestage off                    382            382           1         26.2          38.2       1.0X
+date_trunc SECOND wholestage on                     372            380           8         26.9          37.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1287           1293           9          7.8         128.7       1.0X
-date_trunc WEEK wholestage on                      1310           1341          28          7.6         131.0       1.0X
+date_trunc WEEK wholestage off                     2501           2512          14          4.0         250.1       1.0X
+date_trunc WEEK wholestage on                      2484           2498          12          4.0         248.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1852           1880          39          5.4         185.2       1.0X
-date_trunc QUARTER wholestage on                   1857           1877          14          5.4         185.7       1.0X
+date_trunc QUARTER wholestage off                  3407           3410           4          2.9         340.7       1.0X
+date_trunc QUARTER wholestage on                   3371           3378           5          3.0         337.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           172            178           9         58.2          17.2       1.0X
-trunc year wholestage on                            213            222           8         47.0          21.3       0.8X
+trunc year wholestage off                           318            319           2         31.4          31.8       1.0X
+trunc year wholestage on                            343            347           8         29.2          34.3       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           175            182           9         57.1          17.5       1.0X
-trunc yyyy wholestage on                            224            233          12         44.7          22.4       0.8X
+trunc yyyy wholestage off                           315            316           2         31.7          31.5       1.0X
+trunc yyyy wholestage on                            342            355          18         29.2          34.2       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             167            180          18         59.7          16.7       1.0X
-trunc yy wholestage on                              212            226           9         47.3          21.2       0.8X
+trunc yy wholestage off                             317            318           1         31.5          31.7       1.0X
+trunc yy wholestage on                              339            348          12         29.5          33.9       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            175            175           1         57.2          17.5       1.0X
-trunc mon wholestage on                             219            241          17         45.7          21.9       0.8X
+trunc mon wholestage off                            317            319           3         31.5          31.7       1.0X
+trunc mon wholestage on                             337            349          10         29.7          33.7       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          176            176           0         56.8          17.6       1.0X
-trunc month wholestage on                           220            226           6         45.4          22.0       0.8X
+trunc month wholestage off                          315            316           1         31.7          31.5       1.0X
+trunc month wholestage on                           341            352          14         29.3          34.1       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             173            173           1         57.9          17.3       1.0X
-trunc mm wholestage on                              219            230           8         45.7          21.9       0.8X
+trunc mm wholestage off                             314            315           1         31.8          31.4       1.0X
+trunc mm wholestage on                              343            348           8         29.2          34.3       0.9X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                      98            102           5         10.2          97.9       1.0X
-to timestamp str wholestage on                      102            112           5          9.8         102.5       1.0X
+to timestamp str wholestage off                     163            165           2          6.1         163.2       1.0X
+to timestamp str wholestage on                      158            159           1          6.3         157.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1373           1380          11          0.7        1373.0       1.0X
-to_timestamp wholestage on                          928           1100         103          1.1         928.1       1.5X
+to_timestamp wholestage off                        1490           1491           2          0.7        1489.5       1.0X
+to_timestamp wholestage on                         1522           1530          10          0.7        1522.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1416           1438          32          0.7        1416.0       1.0X
-to_unix_timestamp wholestage on                    1094           1261         194          0.9        1093.6       1.3X
+to_unix_timestamp wholestage off                   1561           1574          18          0.6        1561.4       1.0X
+to_unix_timestamp wholestage on                    1531           1540           9          0.7        1531.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          120            123           5          8.3         120.0       1.0X
-to date str wholestage on                           142            148           6          7.0         142.1       0.8X
+to date str wholestage off                          206            219          18          4.9         206.1       1.0X
+to date str wholestage on                           211            214           4          4.7         211.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             1661           1673          17          0.6        1661.0       1.0X
-to_date wholestage on                              1567           1617          55          0.6        1567.3       1.1X
+to_date wholestage off                             3195           3208          18          0.3        3195.3       1.0X
+to_date wholestage on                              3179           3189          12          0.3        3179.2       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             216            234          16         23.1          43.2       1.0X
-Collect longs                                       913           1204         266          5.5         182.6       0.2X
-Collect timestamps                                 1392           1515         124          3.6         278.3       0.2X
+From java.sql.Timestamp                            1390           1400           9          3.6         277.9       1.0X
+Collect longs                                      1781           2342         496          2.8         356.2       0.8X
+Collect timestamps                                 5837           6037         180          0.9        1167.3       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index c77d076762d80..346fbd3499bab 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -2,428 +2,428 @@
 Extract components
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    243            256          18         41.2          24.3       1.0X
-cast to timestamp wholestage on                     193            206          19         51.9          19.3       1.3X
+cast to timestamp wholestage off                    413            435          31         24.2          41.3       1.0X
+cast to timestamp wholestage on                     357            375          16         28.0          35.7       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                    784            786           3         12.8          78.4       1.0X
-year of timestamp wholestage on                     738            758          15         13.5          73.8       1.1X
+year of timestamp wholestage off                   1311           1335          34          7.6         131.1       1.0X
+year of timestamp wholestage on                    1541           1561          21          6.5         154.1       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                 823            825           3         12.2          82.3       1.0X
-quarter of timestamp wholestage on                  791            804          12         12.6          79.1       1.0X
+quarter of timestamp wholestage off                1449           1467          26          6.9         144.9       1.0X
+quarter of timestamp wholestage on                 1403           1418          12          7.1         140.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                   681            692          16         14.7          68.1       1.0X
-month of timestamp wholestage on                    683            703          14         14.6          68.3       1.0X
+month of timestamp wholestage off                  1221           1231          14          8.2         122.1       1.0X
+month of timestamp wholestage on                   1252           1266           8          8.0         125.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1015           1032          23          9.8         101.5       1.0X
-weekofyear of timestamp wholestage on              1016           1106          72          9.8         101.6       1.0X
+weekofyear of timestamp wholestage off             1939           1947          11          5.2         193.9       1.0X
+weekofyear of timestamp wholestage on              1932           1942           9          5.2         193.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                     683            689           9         14.6          68.3       1.0X
-day of timestamp wholestage on                      685            702          11         14.6          68.5       1.0X
+day of timestamp wholestage off                    1216           1222           9          8.2         121.6       1.0X
+day of timestamp wholestage on                     1246           1256          14          8.0         124.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off               713            716           4         14.0          71.3       1.0X
-dayofyear of timestamp wholestage on                704            713           9         14.2          70.4       1.0X
+dayofyear of timestamp wholestage off              1278           1286          12          7.8         127.8       1.0X
+dayofyear of timestamp wholestage on               1283           1291           8          7.8         128.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off              738            746          11         13.5          73.8       1.0X
-dayofmonth of timestamp wholestage on               684            704          16         14.6          68.4       1.1X
+dayofmonth of timestamp wholestage off             1220           1226           9          8.2         122.0       1.0X
+dayofmonth of timestamp wholestage on              1232           1249          15          8.1         123.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off               810            821          15         12.3          81.0       1.0X
-dayofweek of timestamp wholestage on                784            789           5         12.8          78.4       1.0X
+dayofweek of timestamp wholestage off              1409           1423          21          7.1         140.9       1.0X
+dayofweek of timestamp wholestage on               1396           1403           7          7.2         139.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                 767            777          13         13.0          76.7       1.0X
-weekday of timestamp wholestage on                  765            777          11         13.1          76.5       1.0X
+weekday of timestamp wholestage off                1346           1362          22          7.4         134.6       1.0X
+weekday of timestamp wholestage on                 1330           1341          11          7.5         133.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    534            543          13         18.7          53.4       1.0X
-hour of timestamp wholestage on                     499            529          22         20.0          49.9       1.1X
+hour of timestamp wholestage off                    962            963           2         10.4          96.2       1.0X
+hour of timestamp wholestage on                     981            999          19         10.2          98.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  526            544          26         19.0          52.6       1.0X
-minute of timestamp wholestage on                   495            515          20         20.2          49.5       1.1X
+minute of timestamp wholestage off                  996            999           5         10.0          99.6       1.0X
+minute of timestamp wholestage on                   975            984          11         10.3          97.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  537            538           2         18.6          53.7       1.0X
-second of timestamp wholestage on                   508            512           5         19.7          50.8       1.1X
+second of timestamp wholestage off                 1004           1005           2         10.0         100.4       1.0X
+second of timestamp wholestage on                   979            988           6         10.2          97.9       1.0X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         164            169           6         60.8          16.4       1.0X
-current_date wholestage on                          165            171           4         60.4          16.5       1.0X
+current_date wholestage off                         285            289           6         35.1          28.5       1.0X
+current_date wholestage on                          302            308           8         33.1          30.2       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    161            168          10         62.0          16.1       1.0X
-current_timestamp wholestage on                     161            165           5         62.1          16.1       1.0X
+current_timestamp wholestage off                    296            298           2         33.8          29.6       1.0X
+current_timestamp wholestage on                     303            311           7         33.0          30.3       1.0X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         691            692           2         14.5          69.1       1.0X
-cast to date wholestage on                          572            586          16         17.5          57.2       1.2X
+cast to date wholestage off                        1041           1042           1          9.6         104.1       1.0X
+cast to date wholestage on                         1027           1039           9          9.7         102.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                             713            723          14         14.0          71.3       1.0X
-last_day wholestage on                              695            747          41         14.4          69.5       1.0X
+last_day wholestage off                            1250           1254           6          8.0         125.0       1.0X
+last_day wholestage on                             1267           1273           8          7.9         126.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             627            630           4         15.9          62.7       1.0X
-next_day wholestage on                              615            660          39         16.3          61.5       1.0X
+next_day wholestage off                            1118           1122           6          8.9         111.8       1.0X
+next_day wholestage on                             1081           1096          10          9.2         108.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             604            649          63         16.5          60.4       1.0X
-date_add wholestage on                              590            598           8         17.0          59.0       1.0X
+date_add wholestage off                            1053           1055           4          9.5         105.3       1.0X
+date_add wholestage on                             1059           1061           2          9.4         105.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             588            592           5         17.0          58.8       1.0X
-date_sub wholestage on                              597            620          39         16.7          59.7       1.0X
+date_sub wholestage off                            1055           1064          13          9.5         105.5       1.0X
+date_sub wholestage on                             1055           1060           6          9.5         105.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                           777            782           8         12.9          77.7       1.0X
-add_months wholestage on                            769            786          22         13.0          76.9       1.0X
+add_months wholestage off                          1373           1375           2          7.3         137.3       1.0X
+add_months wholestage on                           1383           1393          13          7.2         138.3       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         3335           3498         231          3.0         333.5       1.0X
-format date wholestage on                          3386           3488         103          3.0         338.6       1.0X
+format date wholestage off                         5590           5603          19          1.8         559.0       1.0X
+format date wholestage on                          5974           5985           8          1.7         597.4       0.9X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       5291           5601         438          1.9         529.1       1.0X
-from_unixtime wholestage on                        5684           5952         256          1.8         568.4       0.9X
+from_unixtime wholestage off                       8650           8662          18          1.2         865.0       1.0X
+from_unixtime wholestage on                        8671           8685          15          1.2         867.1       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   963            981          25         10.4          96.3       1.0X
-from_utc_timestamp wholestage on                    731            794          40         13.7          73.1       1.3X
+from_utc_timestamp wholestage off                  1152           1157           7          8.7         115.2       1.0X
+from_utc_timestamp wholestage on                   1193           1200           7          8.4         119.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1129           1135           7          8.9         112.9       1.0X
-to_utc_timestamp wholestage on                      921            973          68         10.9          92.1       1.2X
+to_utc_timestamp wholestage off                    1414           1417           3          7.1         141.4       1.0X
+to_utc_timestamp wholestage on                     1390           1397           7          7.2         139.0       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        197            197           0         50.8          19.7       1.0X
-cast interval wholestage on                         172            179           8         58.3          17.2       1.1X
+cast interval wholestage off                        332            350          25         30.1          33.2       1.0X
+cast interval wholestage on                         325            333           7         30.8          32.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1039           1105          93          9.6         103.9       1.0X
-datediff wholestage on                             1129           1162          22          8.9         112.9       0.9X
+datediff wholestage off                            1814           1819           7          5.5         181.4       1.0X
+datediff wholestage on                             1811           1822           9          5.5         181.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      2624           2654          42          3.8         262.4       1.0X
-months_between wholestage on                       2612           2709          57          3.8         261.2       1.0X
+months_between wholestage off                      4660           4661           1          2.1         466.0       1.0X
+months_between wholestage on                       4577           4586           5          2.2         457.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1229           1247          25          0.8        1229.4       1.0X
-window wholestage on                              14409          15033         534          0.1       14409.2       0.1X
+window wholestage off                              2006           2119         161          0.5        2005.6       1.0X
+window wholestage on                              43855          43884          41          0.0       43855.4       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1203           1218          22          8.3         120.3       1.0X
-date_trunc YEAR wholestage on                      1162           1172          10          8.6         116.2       1.0X
+date_trunc YEAR wholestage off                     2336           2339           5          4.3         233.6       1.0X
+date_trunc YEAR wholestage on                      2204           2211           6          4.5         220.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1185           1199          20          8.4         118.5       1.0X
-date_trunc YYYY wholestage on                      1148           1173          22          8.7         114.8       1.0X
+date_trunc YYYY wholestage off                     2338           2339           1          4.3         233.8       1.0X
+date_trunc YYYY wholestage on                      2200           2208           5          4.5         220.0       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1189           1192           4          8.4         118.9       1.0X
-date_trunc YY wholestage on                        1168           1179          16          8.6         116.8       1.0X
+date_trunc YY wholestage off                       2337           2340           5          4.3         233.7       1.0X
+date_trunc YY wholestage on                        2202           2209           6          4.5         220.2       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1308           1317          14          7.6         130.8       1.0X
-date_trunc MON wholestage on                       1303           1352          37          7.7         130.3       1.0X
+date_trunc MON wholestage off                      2238           2239           1          4.5         223.8       1.0X
+date_trunc MON wholestage on                       2222           2233          11          4.5         222.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1301           1320          27          7.7         130.1       1.0X
-date_trunc MONTH wholestage on                     1171           1226          41          8.5         117.1       1.1X
+date_trunc MONTH wholestage off                    2243           2243           1          4.5         224.3       1.0X
+date_trunc MONTH wholestage on                     2216           2224           9          4.5         221.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1237           1259          31          8.1         123.7       1.0X
-date_trunc MM wholestage on                        1181           1195          11          8.5         118.1       1.0X
+date_trunc MM wholestage off                       2237           2239           3          4.5         223.7       1.0X
+date_trunc MM wholestage on                        2217           2221           3          4.5         221.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1364           1370           8          7.3         136.4       1.0X
-date_trunc DAY wholestage on                       1189           1223          31          8.4         118.9       1.1X
+date_trunc DAY wholestage off                      1894           1896           3          5.3         189.4       1.0X
+date_trunc DAY wholestage on                       1851           1856           5          5.4         185.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1297           1328          45          7.7         129.7       1.0X
-date_trunc DD wholestage on                        1171           1242          43          8.5         117.1       1.1X
+date_trunc DD wholestage off                       1887           1899          17          5.3         188.7       1.0X
+date_trunc DD wholestage on                        1846           1855           6          5.4         184.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1261           1317          79          7.9         126.1       1.0X
-date_trunc HOUR wholestage on                      1171           1225          48          8.5         117.1       1.1X
+date_trunc HOUR wholestage off                     1947           1951           5          5.1         194.7       1.0X
+date_trunc HOUR wholestage on                      1909           1918           7          5.2         190.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    210            211           2         47.7          21.0       1.0X
-date_trunc MINUTE wholestage on                     178            200          18         56.2          17.8       1.2X
+date_trunc MINUTE wholestage off                    381            386           7         26.3          38.1       1.0X
+date_trunc MINUTE wholestage on                     321            323           1         31.2          32.1       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    206            211           7         48.5          20.6       1.0X
-date_trunc SECOND wholestage on                     186            200           9         53.9          18.6       1.1X
+date_trunc SECOND wholestage off                    368            368           0         27.2          36.8       1.0X
+date_trunc SECOND wholestage on                     324            327           3         30.9          32.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1275           1307          46          7.8         127.5       1.0X
-date_trunc WEEK wholestage on                      1190           1244          63          8.4         119.0       1.1X
+date_trunc WEEK wholestage off                     2213           2213           0          4.5         221.3       1.0X
+date_trunc WEEK wholestage on                      2113           2123          11          4.7         211.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  1688           1719          44          5.9         168.8       1.0X
-date_trunc QUARTER wholestage on                   1732           1795          45          5.8         173.2       1.0X
+date_trunc QUARTER wholestage off                  3071           3081          14          3.3         307.1       1.0X
+date_trunc QUARTER wholestage on                   3031           3035           3          3.3         303.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           201            214          19         49.7          20.1       1.0X
-trunc year wholestage on                            150            157          11         66.5          15.0       1.3X
+trunc year wholestage off                           326            328           3         30.7          32.6       1.0X
+trunc year wholestage on                            307            318          12         32.6          30.7       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           169            174           6         59.0          16.9       1.0X
-trunc yyyy wholestage on                            159            174          10         62.9          15.9       1.1X
+trunc yyyy wholestage off                           327            333           7         30.5          32.7       1.0X
+trunc yyyy wholestage on                            305            320          17         32.8          30.5       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             171            179          12         58.5          17.1       1.0X
-trunc yy wholestage on                              170            180           9         58.8          17.0       1.0X
+trunc yy wholestage off                             322            323           1         31.1          32.2       1.0X
+trunc yy wholestage on                              304            343          67         32.9          30.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            174            177           4         57.5          17.4       1.0X
-trunc mon wholestage on                             162            171           7         61.7          16.2       1.1X
+trunc mon wholestage off                            321            321           1         31.2          32.1       1.0X
+trunc mon wholestage on                             303            313           9         33.0          30.3       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          195            196           2         51.2          19.5       1.0X
-trunc month wholestage on                           166            176           8         60.2          16.6       1.2X
+trunc month wholestage off                          319            323           4         31.3          31.9       1.0X
+trunc month wholestage on                           305            359          75         32.8          30.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             156            160           6         64.2          15.6       1.0X
-trunc mm wholestage on                              147            162          13         68.1          14.7       1.1X
+trunc mm wholestage off                             322            323           1         31.0          32.2       1.0X
+trunc mm wholestage on                              317            351          53         31.5          31.7       1.0X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     154            154           1          6.5         153.8       1.0X
-to timestamp str wholestage on                      127            132           6          7.9         126.5       1.2X
+to timestamp str wholestage off                     219            219           0          4.6         219.4       1.0X
+to timestamp str wholestage on                      214            216           2          4.7         214.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                         973           1001          40          1.0         972.6       1.0X
-to_timestamp wholestage on                          912            922          12          1.1         912.3       1.1X
+to_timestamp wholestage off                        1947           1951           6          0.5        1946.8       1.0X
+to_timestamp wholestage on                         1950           1958          12          0.5        1949.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                    970            986          23          1.0         969.7       1.0X
-to_unix_timestamp wholestage on                     907            910           3          1.1         906.9       1.1X
+to_unix_timestamp wholestage off                   1975           1979           5          0.5        1974.8       1.0X
+to_unix_timestamp wholestage on                    1895           1908          12          0.5        1895.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          154            155           1          6.5         154.3       1.0X
-to date str wholestage on                           156            165           6          6.4         155.9       1.0X
+to date str wholestage off                          284            286           2          3.5         284.3       1.0X
+to date str wholestage on                           263            266           4          3.8         263.3       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             1757           1760           4          0.6        1756.6       1.0X
-to_date wholestage on                              1563           1626          41          0.6        1562.7       1.1X
+to_date wholestage off                             3460           3485          36          0.3        3459.8       1.0X
+to_date wholestage on                              3413           3431          25          0.3        3413.0       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             182            186           4         27.5          36.4       1.0X
-Collect longs                                      1041           1652         964          4.8         208.2       0.2X
-Collect timestamps                                 1003           1029          31          5.0         200.7       0.2X
+From java.sql.Timestamp                            2032           2038           6          2.5         406.5       1.0X
+Collect longs                                      1276           1597         419          3.9         255.1       1.6X
+Collect timestamps                                 6254           7606        1172          0.8        1250.7       0.3X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 570f7beda6b4e..5eb0e6b3134d0 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9392           9392           0         10.6          93.9       1.0X
-before 1582, noop                                  9324           9324           0         10.7          93.2       1.0X
-after 1582, rebase off                            20975          20975           0          4.8         209.7       0.4X
-after 1582, rebase on                             20016          20016           0          5.0         200.2       0.5X
-before 1582, rebase off                           20088          20088           0          5.0         200.9       0.5X
-before 1582, rebase on                            20310          20310           0          4.9         203.1       0.5X
+after 1582, noop                                  18597          18597           0          5.4         186.0       1.0X
+before 1582, noop                                 10565          10565           0          9.5         105.7       1.8X
+after 1582, rebase off                            29811          29811           0          3.4         298.1       0.6X
+after 1582, rebase on                             31110          31110           0          3.2         311.1       0.6X
+before 1582, rebase off                           23144          23144           0          4.3         231.4       0.8X
+before 1582, rebase on                            23689          23689           0          4.2         236.9       0.8X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13371          13463         154          7.5         133.7       1.0X
-after 1582, vec off, rebase on                    13482          13533          57          7.4         134.8       1.0X
-after 1582, vec on, rebase off                     3713           3781          96         26.9          37.1       3.6X
-after 1582, vec on, rebase on                      5153           5173          29         19.4          51.5       2.6X
-before 1582, vec off, rebase off                  12939          12998          97          7.7         129.4       1.0X
-before 1582, vec off, rebase on                   14160          14255          85          7.1         141.6       0.9X
-before 1582, vec on, rebase off                    3748           3776          28         26.7          37.5       3.6X
-before 1582, vec on, rebase on                     5532           5575          54         18.1          55.3       2.4X
+after 1582, vec off, rebase off                   12944          13064         104          7.7         129.4       1.0X
+after 1582, vec off, rebase on                    13223          13255          34          7.6         132.2       1.0X
+after 1582, vec on, rebase off                     3656           3793         219         27.4          36.6       3.5X
+after 1582, vec on, rebase on                      5176           5205          38         19.3          51.8       2.5X
+before 1582, vec off, rebase off                  12926          12970          45          7.7         129.3       1.0X
+before 1582, vec off, rebase on                   13836          13872          40          7.2         138.4       0.9X
+before 1582, vec on, rebase off                    3664           3672           7         27.3          36.6       3.5X
+before 1582, vec on, rebase on                     6049           6078          30         16.5          60.5       2.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2795           2795           0         35.8          27.9       1.0X
-before 1582, noop                                  2806           2806           0         35.6          28.1       1.0X
-after 1582, rebase off                            16113          16113           0          6.2         161.1       0.2X
-after 1582, rebase on                             70198          70198           0          1.4         702.0       0.0X
-before 1582, rebase off                           16690          16690           0          6.0         166.9       0.2X
-before 1582, rebase on                            75706          75706           0          1.3         757.1       0.0X
+after 1582, noop                                   2831           2831           0         35.3          28.3       1.0X
+before 1582, noop                                  2769           2769           0         36.1          27.7       1.0X
+after 1582, rebase off                            17296          17296           0          5.8         173.0       0.2X
+after 1582, rebase on                             81434          81434           0          1.2         814.3       0.0X
+before 1582, rebase off                           17563          17563           0          5.7         175.6       0.2X
+before 1582, rebase on                            94977          94977           0          1.1         949.8       0.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15631          15753         111          6.4         156.3       1.0X
-after 1582, vec off, rebase on                    45834          46027         193          2.2         458.3       0.3X
-after 1582, vec on, rebase off                     4883           4964          70         20.5          48.8       3.2X
-after 1582, vec on, rebase on                     34514          34563          63          2.9         345.1       0.5X
-before 1582, vec off, rebase off                  15253          15354         104          6.6         152.5       1.0X
-before 1582, vec off, rebase on                   47353          47412          59          2.1         473.5       0.3X
-before 1582, vec on, rebase off                    4848           4894          69         20.6          48.5       3.2X
-before 1582, vec on, rebase on                    36125          36143          22          2.8         361.3       0.4X
+after 1582, vec off, rebase off                   14824          14920          87          6.7         148.2       1.0X
+after 1582, vec off, rebase on                    54660          54859         210          1.8         546.6       0.3X
+after 1582, vec on, rebase off                     4876           4954          69         20.5          48.8       3.0X
+after 1582, vec on, rebase on                     44509          44573          65          2.2         445.1       0.3X
+before 1582, vec off, rebase off                  14909          14939          30          6.7         149.1       1.0X
+before 1582, vec off, rebase on                   56092          56346         326          1.8         560.9       0.3X
+before 1582, vec on, rebase off                    4846           4858          13         20.6          48.5       3.1X
+before 1582, vec on, rebase on                    46267          46341          64          2.2         462.7       0.3X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9160           9160           0         10.9          91.6       1.0X
-before 1582, noop                                  9235           9235           0         10.8          92.4       1.0X
-after 1582                                        17154          17154           0          5.8         171.5       0.5X
-before 1582                                       17545          17545           0          5.7         175.5       0.5X
+after 1582, noop                                  18035          18035           0          5.5         180.3       1.0X
+before 1582, noop                                 10571          10571           0          9.5         105.7       1.7X
+after 1582                                        26341          26341           0          3.8         263.4       0.7X
+before 1582                                       19589          19589           0          5.1         195.9       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               21024          21146         205          4.8         210.2       1.0X
-after 1582, vec on                                 3814           3838          21         26.2          38.1       5.5X
-before 1582, vec off                              24293          24347          82          4.1         242.9       0.9X
-before 1582, vec on                                4143           4168          22         24.1          41.4       5.1X
+after 1582, vec off                               34955          35025          98          2.9         349.6       1.0X
+after 1582, vec on                                 3892           3942          43         25.7          38.9       9.0X
+before 1582, vec off                              33360          33457          87          3.0         333.6       1.0X
+before 1582, vec on                                4514           4526          11         22.2          45.1       7.7X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2797           2797           0         35.8          28.0       1.0X
-before 1582, noop                                  2826           2826           0         35.4          28.3       1.0X
-after 1582                                        40021          40021           0          2.5         400.2       0.1X
-before 1582                                       41500          41500           0          2.4         415.0       0.1X
+after 1582, noop                                   2867           2867           0         34.9          28.7       1.0X
+before 1582, noop                                  2812           2812           0         35.6          28.1       1.0X
+after 1582                                        57133          57133           0          1.8         571.3       0.1X
+before 1582                                       53435          53435           0          1.9         534.4       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               32517          32541          23          3.1         325.2       1.0X
-after 1582, vec on                                19644          19725         128          5.1         196.4       1.7X
-before 1582, vec off                              37204          37305         104          2.7         372.0       0.9X
-before 1582, vec on                               24105          24120          13          4.1         241.1       1.3X
+after 1582, vec off                               38475          38551         127          2.6         384.7       1.0X
+after 1582, vec on                                31382          31537         136          3.2         313.8       1.2X
+before 1582, vec off                              42159          42240         128          2.4         421.6       0.9X
+before 1582, vec on                               34735          35129         393          2.9         347.3       1.1X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index 2f8712c859bc9..cc28244e4c973 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9488           9488           0         10.5          94.9       1.0X
-before 1582, noop                                  9301           9301           0         10.8          93.0       1.0X
-after 1582, rebase off                            20109          20109           0          5.0         201.1       0.5X
-after 1582, rebase on                             20004          20004           0          5.0         200.0       0.5X
-before 1582, rebase off                           19906          19906           0          5.0         199.1       0.5X
-before 1582, rebase on                            20466          20466           0          4.9         204.7       0.5X
+after 1582, noop                                  23840          23840           0          4.2         238.4       1.0X
+before 1582, noop                                 10842          10842           0          9.2         108.4       2.2X
+after 1582, rebase off                            35883          35883           0          2.8         358.8       0.7X
+after 1582, rebase on                             36090          36090           0          2.8         360.9       0.7X
+before 1582, rebase off                           23293          23293           0          4.3         232.9       1.0X
+before 1582, rebase on                            23803          23803           0          4.2         238.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12593          12653          52          7.9         125.9       1.0X
-after 1582, vec off, rebase on                    13350          13489         121          7.5         133.5       0.9X
-after 1582, vec on, rebase off                     3665           3681          25         27.3          36.6       3.4X
-after 1582, vec on, rebase on                      5193           5210          16         19.3          51.9       2.4X
-before 1582, vec off, rebase off                  13023          13059          32          7.7         130.2       1.0X
-before 1582, vec off, rebase on                   13855          13937         115          7.2         138.6       0.9X
-before 1582, vec on, rebase off                    3651           3665          12         27.4          36.5       3.4X
-before 1582, vec on, rebase on                     5623           5671          45         17.8          56.2       2.2X
+after 1582, vec off, rebase off                   12608          12658          43          7.9         126.1       1.0X
+after 1582, vec off, rebase on                    13385          13585         239          7.5         133.9       0.9X
+after 1582, vec on, rebase off                     3749           3792          58         26.7          37.5       3.4X
+after 1582, vec on, rebase on                      5256           5296          49         19.0          52.6       2.4X
+before 1582, vec off, rebase off                  13015          13140         109          7.7         130.1       1.0X
+before 1582, vec off, rebase on                   14273          14372         109          7.0         142.7       0.9X
+before 1582, vec on, rebase off                    3742           3766          22         26.7          37.4       3.4X
+before 1582, vec on, rebase on                     6068           6076           7         16.5          60.7       2.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2798           2798           0         35.7          28.0       1.0X
-before 1582, noop                                  2955           2955           0         33.8          29.6       0.9X
-after 1582, rebase off                            15889          15889           0          6.3         158.9       0.2X
-after 1582, rebase on                             84247          84247           0          1.2         842.5       0.0X
-before 1582, rebase off                           16134          16134           0          6.2         161.3       0.2X
-before 1582, rebase on                           100006         100006           0          1.0        1000.1       0.0X
+after 1582, noop                                   2783           2783           0         35.9          27.8       1.0X
+before 1582, noop                                  2783           2783           0         35.9          27.8       1.0X
+after 1582, rebase off                            17016          17016           0          5.9         170.2       0.2X
+after 1582, rebase on                            107890         107890           0          0.9        1078.9       0.0X
+before 1582, rebase off                           17352          17352           0          5.8         173.5       0.2X
+before 1582, rebase on                           122807         122807           0          0.8        1228.1       0.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   14920          15045         116          6.7         149.2       1.0X
-after 1582, vec off, rebase on                    55062          55171         140          1.8         550.6       0.3X
-after 1582, vec on, rebase off                     4871           4952          72         20.5          48.7       3.1X
-after 1582, vec on, rebase on                     44955          44981          23          2.2         449.5       0.3X
-before 1582, vec off, rebase off                  15236          15386         142          6.6         152.4       1.0X
-before 1582, vec off, rebase on                   57290          57368          79          1.7         572.9       0.3X
-before 1582, vec on, rebase off                    4919           4930          15         20.3          49.2       3.0X
-before 1582, vec on, rebase on                    47351          47713         400          2.1         473.5       0.3X
+after 1582, vec off, rebase off                   15066          15104          38          6.6         150.7       1.0X
+after 1582, vec off, rebase on                    68057          68312         222          1.5         680.6       0.2X
+after 1582, vec on, rebase off                     4878           4953          77         20.5          48.8       3.1X
+after 1582, vec on, rebase on                     57278          57320          41          1.7         572.8       0.3X
+before 1582, vec off, rebase off                  15293          15389          84          6.5         152.9       1.0X
+before 1582, vec off, rebase on                   72951          73139         163          1.4         729.5       0.2X
+before 1582, vec on, rebase off                    4883           4951          78         20.5          48.8       3.1X
+before 1582, vec on, rebase on                    59369          59543         226          1.7         593.7       0.3X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   9451           9451           0         10.6          94.5       1.0X
-before 1582, noop                                  9765           9765           0         10.2          97.7       1.0X
-after 1582                                        18722          18722           0          5.3         187.2       0.5X
-before 1582                                       18864          18864           0          5.3         188.6       0.5X
+after 1582, noop                                  23498          23498           0          4.3         235.0       1.0X
+before 1582, noop                                 10828          10828           0          9.2         108.3       2.2X
+after 1582                                        32267          32267           0          3.1         322.7       0.7X
+before 1582                                       20339          20339           0          4.9         203.4       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               24897          25095         247          4.0         249.0       1.0X
-after 1582, vec on                                 3719           3780          84         26.9          37.2       6.7X
-before 1582, vec off                              31290          31347          50          3.2         312.9       0.8X
-before 1582, vec on                                4166           4188          25         24.0          41.7       6.0X
+after 1582, vec off                               46140          46161          21          2.2         461.4       1.0X
+after 1582, vec on                                 3711           3748          33         26.9          37.1      12.4X
+before 1582, vec off                              42527          42745         189          2.4         425.3       1.1X
+before 1582, vec on                                4159           4169          15         24.0          41.6      11.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2882           2882           0         34.7          28.8       1.0X
-before 1582, noop                                  2991           2991           0         33.4          29.9       1.0X
-after 1582                                        53951          53951           0          1.9         539.5       0.1X
-before 1582                                       54276          54276           0          1.8         542.8       0.1X
+after 1582, noop                                   2876           2876           0         34.8          28.8       1.0X
+before 1582, noop                                  2957           2957           0         33.8          29.6       1.0X
+after 1582                                        68393          68393           0          1.5         683.9       0.0X
+before 1582                                       68847          68847           0          1.5         688.5       0.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               41411          41514          97          2.4         414.1       1.0X
-after 1582, vec on                                32163          32201          36          3.1         321.6       1.3X
-before 1582, vec off                              43013          43111         131          2.3         430.1       1.0X
-before 1582, vec on                               34114          34152          45          2.9         341.1       1.2X
+after 1582, vec off                               55967          56011          43          1.8         559.7       1.0X
+after 1582, vec on                                48930          49156         342          2.0         489.3       1.1X
+before 1582, vec off                              60544          60863         311          1.7         605.4       0.9X
+before 1582, vec on                               52539          52665         109          1.9         525.4       1.1X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index d5ab1b9972f1d..5a857622da6f6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -20,6 +20,8 @@ package org.apache.spark.sql.execution.benchmark
 import java.sql.Timestamp
 
 import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * Synthetic benchmark for date and timestamp functions.
@@ -53,96 +55,100 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
   }
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
-    val N = 10000000
-    runBenchmark("Extract components") {
-      run(N, "cast to timestamp", "cast(id as timestamp)")
-      run(N, "year")
-      run(N, "quarter")
-      run(N, "month")
-      run(N, "weekofyear")
-      run(N, "day")
-      run(N, "dayofyear")
-      run(N, "dayofmonth")
-      run(N, "dayofweek")
-      run(N, "weekday")
-      run(N, "hour")
-      run(N, "minute")
-      run(N, "second")
-    }
-    runBenchmark("Current date and time") {
-      run(N, "current_date", "current_date")
-      run(N, "current_timestamp", "current_timestamp")
-    }
-    runBenchmark("Date arithmetic") {
-      val dateExpr = "cast(cast(id as timestamp) as date)"
-      run(N, "cast to date", dateExpr)
-      run(N, "last_day", s"last_day($dateExpr)")
-      run(N, "next_day", s"next_day($dateExpr, 'TU')")
-      run(N, "date_add", s"date_add($dateExpr, 10)")
-      run(N, "date_sub", s"date_sub($dateExpr, 10)")
-      run(N, "add_months", s"add_months($dateExpr, 10)")
-    }
-    runBenchmark("Formatting dates") {
-      val dateExpr = "cast(cast(id as timestamp) as date)"
-      run(N, "format date", s"date_format($dateExpr, 'MMM yyyy')")
-    }
-    runBenchmark("Formatting timestamps") {
-      run(N, "from_unixtime", "from_unixtime(id, 'yyyy-MM-dd HH:mm:ss.SSSSSS')")
-    }
-    runBenchmark("Convert timestamps") {
-      val timestampExpr = "cast(id as timestamp)"
-      run(N, "from_utc_timestamp", s"from_utc_timestamp($timestampExpr, 'CET')")
-      run(N, "to_utc_timestamp", s"to_utc_timestamp($timestampExpr, 'CET')")
-    }
-    runBenchmark("Intervals") {
-      val (start, end) = ("cast(id as timestamp)", "cast((id+8640000) as timestamp)")
-      run(N, "cast interval", start, end)
-      run(N, "datediff", s"datediff($start, $end)")
-      run(N, "months_between", s"months_between($start, $end)")
-      run(1000000, "window", s"window($start, 100, 10, 1)")
-    }
-    runBenchmark("Truncation") {
-      val timestampExpr = "cast(id as timestamp)"
-      Seq("YEAR", "YYYY", "YY", "MON", "MONTH", "MM", "DAY", "DD", "HOUR", "MINUTE",
-          "SECOND", "WEEK", "QUARTER").foreach { level =>
-        run(N, s"date_trunc $level", s"date_trunc('$level', $timestampExpr)")
-      }
-      val dateExpr = "cast(cast(id as timestamp) as date)"
-      Seq("year", "yyyy", "yy", "mon", "month", "mm").foreach { level =>
-        run(N, s"trunc $level", s"trunc('$level', $dateExpr)")
-      }
-    }
-    runBenchmark("Parsing") {
-      val n = 1000000
-      val timestampStrExpr = "concat('2019-01-27 11:02:01.', cast(mod(id, 1000) as string))"
-      val pattern = "'yyyy-MM-dd HH:mm:ss.SSS'"
-      run(n, "to timestamp str", timestampStrExpr)
-      run(n, "to_timestamp", s"to_timestamp($timestampStrExpr, $pattern)")
-      run(n, "to_unix_timestamp", s"to_unix_timestamp($timestampStrExpr, $pattern)")
-      val dateStrExpr = "concat('2019-01-', lpad(mod(id, 25), 2, '0'))"
-      run(n, "to date str", dateStrExpr)
-      run(n, "to_date", s"to_date($dateStrExpr, 'yyyy-MM-dd')")
-    }
-    runBenchmark("Conversion from/to external types") {
-      import spark.implicits._
-      val rowsNum = 5000000
-      val numIters = 3
-      val benchmark = new Benchmark("To/from java.sql.Timestamp", rowsNum, output = output)
-      benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
-        spark.range(rowsNum)
-          .map(millis => new Timestamp(millis))
-          .noop()
-      }
-      benchmark.addCase("Collect longs", numIters) { _ =>
-        spark.range(0, rowsNum, 1, 1)
-          .collect()
-      }
-      benchmark.addCase("Collect timestamps", numIters) { _ =>
-        spark.range(0, rowsNum, 1, 1)
-          .map(millis => new Timestamp(millis))
-          .collect()
+    withDefaultTimeZone(LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        val N = 10000000
+        runBenchmark("Extract components") {
+          run(N, "cast to timestamp", "cast(id as timestamp)")
+          run(N, "year")
+          run(N, "quarter")
+          run(N, "month")
+          run(N, "weekofyear")
+          run(N, "day")
+          run(N, "dayofyear")
+          run(N, "dayofmonth")
+          run(N, "dayofweek")
+          run(N, "weekday")
+          run(N, "hour")
+          run(N, "minute")
+          run(N, "second")
+        }
+        runBenchmark("Current date and time") {
+          run(N, "current_date", "current_date")
+          run(N, "current_timestamp", "current_timestamp")
+        }
+        runBenchmark("Date arithmetic") {
+          val dateExpr = "cast(cast(id as timestamp) as date)"
+          run(N, "cast to date", dateExpr)
+          run(N, "last_day", s"last_day($dateExpr)")
+          run(N, "next_day", s"next_day($dateExpr, 'TU')")
+          run(N, "date_add", s"date_add($dateExpr, 10)")
+          run(N, "date_sub", s"date_sub($dateExpr, 10)")
+          run(N, "add_months", s"add_months($dateExpr, 10)")
+        }
+        runBenchmark("Formatting dates") {
+          val dateExpr = "cast(cast(id as timestamp) as date)"
+          run(N, "format date", s"date_format($dateExpr, 'MMM yyyy')")
+        }
+        runBenchmark("Formatting timestamps") {
+          run(N, "from_unixtime", "from_unixtime(id, 'yyyy-MM-dd HH:mm:ss.SSSSSS')")
+        }
+        runBenchmark("Convert timestamps") {
+          val timestampExpr = "cast(id as timestamp)"
+          run(N, "from_utc_timestamp", s"from_utc_timestamp($timestampExpr, 'CET')")
+          run(N, "to_utc_timestamp", s"to_utc_timestamp($timestampExpr, 'CET')")
+        }
+        runBenchmark("Intervals") {
+          val (start, end) = ("cast(id as timestamp)", "cast((id+8640000) as timestamp)")
+          run(N, "cast interval", start, end)
+          run(N, "datediff", s"datediff($start, $end)")
+          run(N, "months_between", s"months_between($start, $end)")
+          run(1000000, "window", s"window($start, 100, 10, 1)")
+        }
+        runBenchmark("Truncation") {
+          val timestampExpr = "cast(id as timestamp)"
+          Seq("YEAR", "YYYY", "YY", "MON", "MONTH", "MM", "DAY", "DD", "HOUR", "MINUTE",
+            "SECOND", "WEEK", "QUARTER").foreach { level =>
+            run(N, s"date_trunc $level", s"date_trunc('$level', $timestampExpr)")
+          }
+          val dateExpr = "cast(cast(id as timestamp) as date)"
+          Seq("year", "yyyy", "yy", "mon", "month", "mm").foreach { level =>
+            run(N, s"trunc $level", s"trunc('$level', $dateExpr)")
+          }
+        }
+        runBenchmark("Parsing") {
+          val n = 1000000
+          val timestampStrExpr = "concat('2019-01-27 11:02:01.', cast(mod(id, 1000) as string))"
+          val pattern = "'yyyy-MM-dd HH:mm:ss.SSS'"
+          run(n, "to timestamp str", timestampStrExpr)
+          run(n, "to_timestamp", s"to_timestamp($timestampStrExpr, $pattern)")
+          run(n, "to_unix_timestamp", s"to_unix_timestamp($timestampStrExpr, $pattern)")
+          val dateStrExpr = "concat('2019-01-', lpad(mod(id, 25), 2, '0'))"
+          run(n, "to date str", dateStrExpr)
+          run(n, "to_date", s"to_date($dateStrExpr, 'yyyy-MM-dd')")
+        }
+        runBenchmark("Conversion from/to external types") {
+          import spark.implicits._
+          val rowsNum = 5000000
+          val numIters = 3
+          val benchmark = new Benchmark("To/from java.sql.Timestamp", rowsNum, output = output)
+          benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
+            spark.range(rowsNum)
+              .map(millis => new Timestamp(millis))
+              .noop()
+          }
+          benchmark.addCase("Collect longs", numIters) { _ =>
+            spark.range(0, rowsNum, 1, 1)
+              .collect()
+          }
+          benchmark.addCase("Collect timestamps", numIters) { _ =>
+            spark.range(0, rowsNum, 1, 1)
+              .map(millis => new Timestamp(millis))
+              .collect()
+          }
+          benchmark.run()
+        }
       }
-      benchmark.run()
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index 6285461d93097..077ac28c149ee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -23,6 +23,7 @@ import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneOffset}
 import org.apache.spark.benchmark.Benchmark
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -128,81 +129,91 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     val rowsNum = 100000000
 
-    withTempPath { path =>
-      runBenchmark("Rebasing dates/timestamps in Parquet datasource") {
-        Seq("date", "timestamp").foreach { dateTime =>
-          val benchmark = new Benchmark(s"Save ${dateTime}s to parquet", rowsNum, output = output)
-          benchmarkInputs(benchmark, rowsNum, dateTime)
-          Seq(true, false).foreach { after1582 =>
-            Seq(false, true).foreach { rebase =>
-              benchmark.addCase(caseName(after1582, Some(rebase)), 1) { _ =>
-                withSQLConf(
-                  SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
-                  genDF(rowsNum, dateTime, after1582)
-                    .write
-                    .mode("overwrite")
-                    .format("parquet")
-                    .save(getPath(path, dateTime, after1582, Some(rebase)))
+    withDefaultTimeZone(LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        withTempPath { path =>
+          runBenchmark("Rebasing dates/timestamps in Parquet datasource") {
+            Seq("date", "timestamp").foreach { dateTime =>
+              val benchmark = new Benchmark(
+                s"Save ${dateTime}s to parquet",
+                rowsNum,
+                output = output)
+              benchmarkInputs(benchmark, rowsNum, dateTime)
+              Seq(true, false).foreach { after1582 =>
+                Seq(false, true).foreach { rebase =>
+                  benchmark.addCase(caseName(after1582, Some(rebase)), 1) { _ =>
+                    withSQLConf(
+                      SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
+                      genDF(rowsNum, dateTime, after1582)
+                        .write
+                        .mode("overwrite")
+                        .format("parquet")
+                        .save(getPath(path, dateTime, after1582, Some(rebase)))
+                    }
+                  }
                 }
               }
-            }
-          }
-          benchmark.run()
-
-          val benchmark2 = new Benchmark(
-            s"Load ${dateTime}s from parquet", rowsNum, output = output)
-          Seq(true, false).foreach { after1582 =>
-            Seq(false, true).foreach { vec =>
-              Seq(false, true).foreach { rebase =>
-                benchmark2.addCase(caseName(after1582, Some(rebase), Some(vec)), 3) { _ =>
-                  withSQLConf(
-                    SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
-                    SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
-                    spark.read
-                      .format("parquet")
-                      .load(getPath(path, dateTime, after1582, Some(rebase)))
-                      .noop()
+              benchmark.run()
+
+              val benchmark2 = new Benchmark(
+                s"Load ${dateTime}s from parquet", rowsNum, output = output)
+              Seq(true, false).foreach { after1582 =>
+                Seq(false, true).foreach { vec =>
+                  Seq(false, true).foreach { rebase =>
+                    benchmark2.addCase(caseName(after1582, Some(rebase), Some(vec)), 3) { _ =>
+                      withSQLConf(
+                        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
+                        SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                        spark.read
+                          .format("parquet")
+                          .load(getPath(path, dateTime, after1582, Some(rebase)))
+                          .noop()
+                      }
+                    }
                   }
                 }
               }
+              benchmark2.run()
             }
           }
-          benchmark2.run()
         }
-      }
-    }
 
-    withTempPath { path =>
-      runBenchmark("Rebasing dates/timestamps in ORC datasource") {
-        Seq("date", "timestamp").foreach { dateTime =>
-          val benchmark = new Benchmark(s"Save ${dateTime}s to ORC", rowsNum, output = output)
-          benchmarkInputs(benchmark, rowsNum, dateTime)
-          Seq(true, false).foreach { after1582 =>
-            benchmark.addCase(caseName(after1582), 1) { _ =>
-              genDF(rowsNum, dateTime, after1582)
-                .write
-                .mode("overwrite")
-                .format("orc")
-                .save(getPath(path, dateTime, after1582))
-            }
-          }
-          benchmark.run()
-
-          val benchmark2 = new Benchmark(s"Load ${dateTime}s from ORC", rowsNum, output = output)
-          Seq(true, false).foreach { after1582 =>
-            Seq(false, true).foreach { vec =>
-              benchmark2.addCase(caseName(after1582, vec = Some(vec)), 3) { _ =>
-                withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vec.toString) {
-                  spark
-                    .read
+        withTempPath { path =>
+          runBenchmark("Rebasing dates/timestamps in ORC datasource") {
+            Seq("date", "timestamp").foreach { dateTime =>
+              val benchmark = new Benchmark(s"Save ${dateTime}s to ORC", rowsNum, output = output)
+              benchmarkInputs(benchmark, rowsNum, dateTime)
+              Seq(true, false).foreach { after1582 =>
+                benchmark.addCase(caseName(after1582), 1) { _ =>
+                  genDF(rowsNum, dateTime, after1582)
+                    .write
+                    .mode("overwrite")
                     .format("orc")
-                    .load(getPath(path, dateTime, after1582))
-                    .noop()
+                    .save(getPath(path, dateTime, after1582))
+                }
+              }
+              benchmark.run()
+
+              val benchmark2 = new Benchmark(
+                s"Load ${dateTime}s from ORC",
+                rowsNum,
+                output = output)
+              Seq(true, false).foreach { after1582 =>
+                Seq(false, true).foreach { vec =>
+                  benchmark2.addCase(caseName(after1582, vec = Some(vec)), 3) { _ =>
+                    withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vec.toString) {
+                      spark
+                        .read
+                        .format("orc")
+                        .load(getPath(path, dateTime, after1582))
+                        .noop()
+                    }
+                  }
                 }
               }
+              benchmark2.run()
             }
           }
-          benchmark2.run()
         }
       }
     }

From dc552a9907f716efff17d7f1f3d3668d91bc50db Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Mon, 6 Apr 2020 05:34:59 +0000
Subject: [PATCH 0427/1280] [SPARK-31224][SQL] Add view support to SHOW CREATE
 TABLE

### What changes were proposed in this pull request?

For now `SHOW CREATE TABLE` command doesn't support views, but `SHOW CREATE TABLE AS SERDE` supports it. Since the views syntax are the same between Hive DDL and Spark DDL, we should be able to support views in both two commands.

This is Hive syntax for creating views:

```
CREATE VIEW [IF NOT EXISTS] [db_name.]view_name [(column_name [COMMENT column_comment], ...) ]
  [COMMENT view_comment]
  [TBLPROPERTIES (property_name = property_value, ...)]
  AS SELECT ...;
```

https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-CreateView

This is Spark syntax for creating views:

```
CREATE [OR REPLACE] [[GLOBAL] TEMPORARY] VIEW [IF NOT EXISTS [db_name.]view_name
    create_view_clauses
    AS query;
```
https://spark.apache.org/docs/3.0.0-preview/sql-ref-syntax-ddl-create-view.html

Looks like it is the same. We could support views in both commands.

This patch proposes to add views support to `SHOW CREATE TABLE`.

### Why are the changes needed?

To extend the view support of `SHOW CREATE TABLE`, so users can use `SHOW CREATE TABLE` to show Spark DDL for views.

### Does this PR introduce any user-facing change?

Yes. `SHOW CREATE TABLE` can be used to show Spark DDL for views.

### How was this patch tested?

Unit tests.

Closes #27984 from viirya/spark-view.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d782a1c4565c401a02531db3b8aa3cb6fc698fb1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/command/tables.scala  | 101 ++++++++++--------
 .../sql-tests/inputs/show-create-table.sql    |  12 ++-
 .../results/show-create-table.sql.out         |  51 +++++----
 .../spark/sql/ShowCreateTableSuite.scala      |  20 ++--
 .../sql/hive/HiveShowCreateTableSuite.scala   | 100 ++++++++---------
 5 files changed, 151 insertions(+), 133 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 61955ba4a4b55..2c9ca36c7fcdc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -1057,6 +1057,42 @@ trait ShowCreateTableCommandBase {
   protected def concatByMultiLines(iter: Iterable[String]): String = {
     iter.mkString("(\n  ", ",\n  ", ")\n")
   }
+
+  protected def showCreateView(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    showViewDataColumns(metadata, builder)
+    showTableComment(metadata, builder)
+    showViewProperties(metadata, builder)
+    showViewText(metadata, builder)
+  }
+
+  private def showViewDataColumns(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    if (metadata.schema.nonEmpty) {
+      val viewColumns = metadata.schema.map { f =>
+        val comment = f.getComment()
+          .map(escapeSingleQuotedString)
+          .map(" COMMENT '" + _ + "'")
+
+        // view columns shouldn't have data type info
+        s"${quoteIdentifier(f.name)}${comment.getOrElse("")}"
+      }
+      builder ++= concatByMultiLines(viewColumns)
+    }
+  }
+
+  private def showViewProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    val viewProps = metadata.properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX))
+    if (viewProps.nonEmpty) {
+      val props = viewProps.map { case (key, value) =>
+        s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
+      }
+
+      builder ++= s"TBLPROPERTIES ${concatByMultiLines(props)}"
+    }
+  }
+
+  private def showViewText(metadata: CatalogTable, builder: StringBuilder): Unit = {
+    builder ++= metadata.viewText.mkString("AS ", "", "\n")
+  }
 }
 
 /**
@@ -1100,10 +1136,6 @@ case class ShowCreateTableCommand(table: TableIdentifier)
           )
         }
 
-        if (tableMetadata.tableType == VIEW) {
-          throw new AnalysisException("Hive view isn't supported by SHOW CREATE TABLE")
-        }
-
         if ("true".equalsIgnoreCase(tableMetadata.properties.getOrElse("transactional", "false"))) {
           throw new AnalysisException(
             "SHOW CREATE TABLE doesn't support transactional Hive table. " +
@@ -1111,10 +1143,26 @@ case class ShowCreateTableCommand(table: TableIdentifier)
               "to show Hive DDL instead.")
         }
 
-        convertTableMetadata(tableMetadata)
+        if (tableMetadata.tableType == VIEW) {
+          tableMetadata
+        } else {
+          convertTableMetadata(tableMetadata)
+        }
       }
 
-      val stmt = showCreateDataSourceTable(metadata)
+      val builder = StringBuilder.newBuilder
+
+      val stmt = if (tableMetadata.tableType == VIEW) {
+        builder ++= s"CREATE VIEW ${table.quotedString} "
+        showCreateView(metadata, builder)
+
+        builder.toString()
+      } else {
+        builder ++= s"CREATE TABLE ${table.quotedString} "
+
+        showCreateDataSourceTable(metadata, builder)
+        builder.toString()
+      }
 
       Seq(Row(stmt))
     }
@@ -1194,18 +1242,13 @@ case class ShowCreateTableCommand(table: TableIdentifier)
     }
   }
 
-  private def showCreateDataSourceTable(metadata: CatalogTable): String = {
-    val builder = StringBuilder.newBuilder
-
-    builder ++= s"CREATE TABLE ${table.quotedString} "
+  private def showCreateDataSourceTable(metadata: CatalogTable, builder: StringBuilder): Unit = {
     showDataSourceTableDataColumns(metadata, builder)
     showDataSourceTableOptions(metadata, builder)
     showDataSourceTableNonDataColumns(metadata, builder)
     showTableComment(metadata, builder)
     showTableLocation(metadata, builder)
     showTableProperties(metadata, builder)
-
-    builder.toString()
   }
 }
 
@@ -1264,10 +1307,7 @@ case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
     builder ++= s"CREATE$tableTypeString ${table.quotedString}"
 
     if (metadata.tableType == VIEW) {
-      showViewDataColumns(metadata, builder)
-      showTableComment(metadata, builder)
-      showViewProperties(metadata, builder)
-      showViewText(metadata, builder)
+      showCreateView(metadata, builder)
     } else {
       showHiveTableHeader(metadata, builder)
       showTableComment(metadata, builder)
@@ -1280,35 +1320,6 @@ case class ShowCreateTableAsSerdeCommand(table: TableIdentifier)
     builder.toString()
   }
 
-  private def showViewDataColumns(metadata: CatalogTable, builder: StringBuilder): Unit = {
-    if (metadata.schema.nonEmpty) {
-      val viewColumns = metadata.schema.map { f =>
-        val comment = f.getComment()
-          .map(escapeSingleQuotedString)
-          .map(" COMMENT '" + _ + "'")
-
-        // view columns shouldn't have data type info
-        s"${quoteIdentifier(f.name)}${comment.getOrElse("")}"
-      }
-      builder ++= concatByMultiLines(viewColumns)
-    }
-  }
-
-  private def showViewProperties(metadata: CatalogTable, builder: StringBuilder): Unit = {
-    val viewProps = metadata.properties.filterKeys(!_.startsWith(CatalogTable.VIEW_PREFIX))
-    if (viewProps.nonEmpty) {
-      val props = viewProps.map { case (key, value) =>
-        s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'"
-      }
-
-      builder ++= s"TBLPROPERTIES ${concatByMultiLines(props)}"
-    }
-  }
-
-  private def showViewText(metadata: CatalogTable, builder: StringBuilder): Unit = {
-    builder ++= metadata.viewText.mkString("AS ", "", "\n")
-  }
-
   private def showHiveTableHeader(metadata: CatalogTable, builder: StringBuilder): Unit = {
     val columns = metadata.schema.filterNot { column =>
       metadata.partitionColumnNames.contains(column.name)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql
index dc77f87d9743a..00b46d1951fcf 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-create-table.sql
@@ -74,6 +74,9 @@ CREATE VIEW view_SPARK_30302 (aaa, bbb)
 AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302 AS SERDE;
+
+SHOW CREATE TABLE view_SPARK_30302;
+
 DROP VIEW view_SPARK_30302;
 
 
@@ -83,6 +86,9 @@ COMMENT 'This is a comment with \'quoted text\' for view'
 AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302 AS SERDE;
+
+SHOW CREATE TABLE view_SPARK_30302;
+
 DROP VIEW view_SPARK_30302;
 
 
@@ -92,13 +98,9 @@ TBLPROPERTIES ('a' = '1', 'b' = '2')
 AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302 AS SERDE;
-DROP VIEW view_SPARK_30302;
-
--- SHOW CREATE TABLE does not support view
-CREATE VIEW view_SPARK_30302 (aaa, bbb)
-AS SELECT a, b FROM tbl;
 
 SHOW CREATE TABLE view_SPARK_30302;
+
 DROP VIEW view_SPARK_30302;
 
 DROP TABLE tbl;
diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
index f5ca1eff9f0c3..88fef8908638e 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out
@@ -301,6 +301,17 @@ CREATE VIEW `default`.`view_SPARK_30302`(
 AS SELECT a, b FROM tbl
 
 
+-- !query
+SHOW CREATE TABLE view_SPARK_30302
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE VIEW `default`.`view_SPARK_30302` (
+  `aaa`,
+  `bbb`)
+AS SELECT a, b FROM tbl
+
+
 -- !query
 DROP VIEW view_SPARK_30302
 -- !query schema
@@ -331,6 +342,18 @@ COMMENT 'This is a comment with \'quoted text\' for view'
 AS SELECT a, b FROM tbl
 
 
+-- !query
+SHOW CREATE TABLE view_SPARK_30302
+-- !query schema
+struct<createtab_stmt:string>
+-- !query output
+CREATE VIEW `default`.`view_SPARK_30302` (
+  `aaa` COMMENT 'comment with \'quoted text\' for aaa',
+  `bbb`)
+COMMENT 'This is a comment with \'quoted text\' for view'
+AS SELECT a, b FROM tbl
+
+
 -- !query
 DROP VIEW view_SPARK_30302
 -- !query schema
@@ -363,30 +386,18 @@ TBLPROPERTIES (
 AS SELECT a, b FROM tbl
 
 
--- !query
-DROP VIEW view_SPARK_30302
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-CREATE VIEW view_SPARK_30302 (aaa, bbb)
-AS SELECT a, b FROM tbl
--- !query schema
-struct<>
--- !query output
-
-
-
 -- !query
 SHOW CREATE TABLE view_SPARK_30302
 -- !query schema
-struct<>
+struct<createtab_stmt:string>
 -- !query output
-org.apache.spark.sql.AnalysisException
-Hive view isn't supported by SHOW CREATE TABLE;
+CREATE VIEW `default`.`view_SPARK_30302` (
+  `aaa`,
+  `bbb`)
+TBLPROPERTIES (
+  'a' = '1',
+  'b' = '2')
+AS SELECT a, b FROM tbl
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
index 04257642fac81..4e85f739b95a2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
@@ -188,18 +188,26 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
     if (result.length > 1) result(0) + result(1) else result.head
   }
 
-  protected def checkCreateTable(table: String): Unit = {
-    checkCreateTableOrView(TableIdentifier(table, Some("default")), "TABLE")
+  protected def checkCreateTable(table: String, serde: Boolean = false): Unit = {
+    checkCreateTableOrView(TableIdentifier(table, Some("default")), "TABLE", serde)
   }
 
-  protected def checkCreateView(table: String): Unit = {
-    checkCreateTableOrView(TableIdentifier(table, Some("default")), "VIEW")
+  protected def checkCreateView(table: String, serde: Boolean = false): Unit = {
+    checkCreateTableOrView(TableIdentifier(table, Some("default")), "VIEW", serde)
   }
 
-  private def checkCreateTableOrView(table: TableIdentifier, checkType: String): Unit = {
+  protected def checkCreateTableOrView(
+      table: TableIdentifier,
+      checkType: String,
+      serde: Boolean): Unit = {
     val db = table.database.getOrElse("default")
     val expected = spark.sharedState.externalCatalog.getTable(db, table.table)
-    val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0)
+    val shownDDL = if (serde) {
+      sql(s"SHOW CREATE TABLE ${table.quotedString} AS SERDE").head().getString(0)
+    } else {
+      sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0)
+    }
+
     sql(s"DROP $checkType ${table.quotedString}")
 
     try {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
index 50c9018e4e2a9..1e31e8b1bf234 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
@@ -42,16 +42,43 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
   }
 
   test("view") {
-    withView("v1") {
-      sql("CREATE VIEW v1 AS SELECT 1 AS a")
-      checkCreateHiveTableOrView("v1", "VIEW")
+    Seq(true, false).foreach { serde =>
+      withView("v1") {
+        sql("CREATE VIEW v1 AS SELECT 1 AS a")
+        checkCreateView("v1", serde)
+      }
     }
   }
 
-  test("view  with output columns") {
-    withView("v1") {
-      sql("CREATE VIEW v1 (b) AS SELECT 1 AS a")
-      checkCreateHiveTableOrView("v1", "VIEW")
+  test("view with output columns") {
+    Seq(true, false).foreach { serde =>
+      withView("v1") {
+        sql("CREATE VIEW v1 (a, b COMMENT 'b column') AS SELECT 1 AS a, 2 AS b")
+        checkCreateView("v1", serde)
+      }
+    }
+  }
+
+  test("view with table comment and properties") {
+    Seq(true, false).foreach { serde =>
+      withView("v1") {
+        sql(
+          s"""
+             |CREATE VIEW v1 (
+             |  c1 COMMENT 'bla',
+             |  c2
+             |)
+             |COMMENT 'table comment'
+             |TBLPROPERTIES (
+             |  'prop1' = 'value1',
+             |  'prop2' = 'value2'
+             |)
+             |AS SELECT 1 AS c1, '2' AS c2
+         """.stripMargin
+        )
+
+        checkCreateView("v1", serde)
+      }
     }
   }
 
@@ -69,7 +96,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -89,7 +116,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
            """.stripMargin
         )
 
-        checkCreateHiveTableOrView("t1")
+        checkCreateTable("t1", serde = true)
       }
     }
   }
@@ -109,7 +136,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -127,7 +154,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -142,7 +169,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -164,7 +191,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
          """.stripMargin
       )
 
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -177,7 +204,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
            |INTO 2 BUCKETS
          """.stripMargin
       )
-      checkCreateHiveTableOrView("t1")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -222,27 +249,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
       val shownDDL = getShowDDL("SHOW CREATE TABLE t1")
       assert(shownDDL == "CREATE TABLE `default`.`t1` (`a` STRUCT<`b`: STRING>)")
 
-      checkCreateHiveTableOrView("t1")
-    }
-  }
-
-  /**
-   * This method compares the given table with the table created by the DDL generated by
-   * `SHOW CREATE TABLE AS SERDE`.
-   */
-  private def checkCreateHiveTableOrView(tableName: String, checkType: String = "TABLE"): Unit = {
-    val table = TableIdentifier(tableName, Some("default"))
-    val db = table.database.getOrElse("default")
-    val expected = spark.sharedState.externalCatalog.getTable(db, table.table)
-    val shownDDL = sql(s"SHOW CREATE TABLE ${table.quotedString} AS SERDE").head().getString(0)
-    sql(s"DROP $checkType ${table.quotedString}")
-
-    try {
-      sql(shownDDL)
-      val actual = spark.sharedState.externalCatalog.getTable(db, table.table)
-      checkCatalogTables(expected, actual)
-    } finally {
-      sql(s"DROP $checkType IF EXISTS ${table.table}")
+      checkCreateTable("t1", serde = true)
     }
   }
 
@@ -344,7 +351,7 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
       )
 
       val cause = intercept[AnalysisException] {
-        checkCreateHiveTableOrView("t1")
+        checkCreateTable("t1", serde = true)
       }
 
       assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` without `AS SERDE` instead"))
@@ -446,27 +453,6 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet
     }
   }
 
-  test("hive view is not supported by show create table without as serde") {
-    withTable("t1") {
-      withView("v1") {
-        sql("CREATE TABLE t1 (c1 STRING, c2 STRING)")
-
-        createRawHiveTable(
-          s"""
-             |CREATE VIEW v1
-             |AS SELECT * from t1
-           """.stripMargin
-        )
-
-        val cause = intercept[AnalysisException] {
-          sql("SHOW CREATE TABLE v1")
-        }
-
-        assert(cause.getMessage.contains("view isn't supported"))
-      }
-    }
-  }
-
   test("partitioned, bucketed hive table in Spark DDL") {
     withTable("t1") {
       sql(

From 22e1c4f7ed0c5e8d63994ceda256a03f2aca09a2 Mon Sep 17 00:00:00 2001
From: Maxim Gekk <max.gekk@gmail.com>
Date: Mon, 6 Apr 2020 05:43:29 +0000
Subject: [PATCH 0428/1280] [SPARK-31343][SQL][TESTS] Check codegen does not
 fail on expressions with escape chars in string parameters

### What changes were proposed in this pull request?
In the PR, I propose to add tests to check that code generation doesn't fail if expressions string argument contains escape chars. The PR adds similar tests added by https://github.com/apache/spark/pull/20182 for `from_utc_timestamp` / `to_utc_timestamp`.

### Why are the changes needed?
To prevent regressions in the future.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running the affected tests

Closes #28115 from MaxGekk/tests-arg-escape.

Authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 09198528b98d43169b4d0c78e36de7e543d45fc5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../CallMethodViaReflectionSuite.scala        |  6 +++
 .../expressions/DateExpressionsSuite.scala    | 17 +++++++
 .../expressions/JsonExpressionsSuite.scala    | 23 ++++++++++
 .../expressions/RegexpExpressionsSuite.scala  | 13 ++++++
 .../expressions/StringExpressionsSuite.scala  | 44 +++++++++++++++++++
 5 files changed, 103 insertions(+)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
index 88d4d460751b6..d8f3ad24246a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflectionSuite.scala
@@ -21,6 +21,7 @@ import java.sql.Timestamp
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.types.{IntegerType, StringType}
 
 /** A static class for testing purpose. */
@@ -101,6 +102,11 @@ class CallMethodViaReflectionSuite extends SparkFunSuite with ExpressionEvalHelp
     checkEvaluation(createExpr(staticClassName, "method4", 4, "four"), "m4four")
   }
 
+  test("escaping of class and method names") {
+    GenerateUnsafeProjection.generate(
+      CallMethodViaReflection(Seq(Literal("\"quote"), Literal("\"quote"), Literal(null))) :: Nil)
+  }
+
   private def createExpr(className: String, methodName: String, args: Any*) = {
     CallMethodViaReflection(
       Literal.create(className, StringType) +:
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 350b3a44baa82..7f235d6bb4f30 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -265,6 +265,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           Literal("H"), JST_OPT), "0")
         checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22")
 
+        // Test escaping of format
+        GenerateUnsafeProjection.generate(
+          DateFormatClass(Literal(ts), Literal("\"quote"), JST_OPT) :: Nil)
+
         // SPARK-28072 The codegen path should work
         checkEvaluation(
           expression = DateFormatClass(
@@ -602,6 +606,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(NextDay(Literal.create(null, DateType), Literal("xx")), null)
     checkEvaluation(
       NextDay(Literal(Date.valueOf("2015-07-23")), Literal.create(null, StringType)), null)
+    // Test escaping of dayOfWeek
+    GenerateUnsafeProjection.generate(
+      NextDay(Literal(Date.valueOf("2015-07-23")), Literal("\"quote")) :: Nil)
   }
 
   test("TruncDate") {
@@ -625,6 +632,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     testTrunc(date, null, null)
     testTrunc(null, "MON", null)
     testTrunc(null, null, null)
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(TruncDate(Literal(0, DateType), Literal("\"quote")) :: Nil)
 
     testTrunc(Date.valueOf("2000-03-08"), "decade", Date.valueOf("2000-01-01"))
     testTrunc(Date.valueOf("2000-03-08"), "century", Date.valueOf("1901-01-01"))
@@ -751,6 +760,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         }
       }
     }
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\"quote")) :: Nil)
   }
 
   test("unix_timestamp") {
@@ -818,6 +829,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         }
       }
     }
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(
+      UnixTimestamp(Literal("2015-07-24"), Literal("\"quote")) :: Nil)
   }
 
   test("to_unix_timestamp") {
@@ -893,6 +907,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         }
       }
     }
+    // Test escaping of format
+    GenerateUnsafeProjection.generate(
+      ToUnixTimestamp(Literal("2015-07-24"), Literal("\"quote")) :: Nil)
   }
 
   test("datediff") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index 032e0ac61884b..90c4d8f789660 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -25,6 +25,7 @@ import org.scalatest.exceptions.TestFailedException
 import org.apache.spark.{SparkException, SparkFunSuite}
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{PST, UTC, UTC_OPT}
@@ -49,6 +50,10 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
    in Jackson's JsonFactory.createParser(byte[]) due to RFC-4627 encoding detection */
   val badJson = "\u0000\u0000\u0000A\u0001AAA"
 
+  test("get_json_object escaping") {
+    GenerateUnsafeProjection.generate(GetJsonObject(Literal("\"quote"), Literal("\"quote")) :: Nil)
+  }
+
   test("$.store.bicycle") {
     checkEvaluation(
       GetJsonObject(Literal(json), Literal("$.store.bicycle")),
@@ -265,6 +270,11 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     assert(jt.eval(null).toSeq.head === expected)
   }
 
+  test("json_tuple escaping") {
+    GenerateUnsafeProjection.generate(
+      JsonTuple(Literal("\"quote") ::  Literal("\"quote") :: Nil) :: Nil)
+  }
+
   test("json_tuple - hive key 1") {
     checkJsonTuple(
       JsonTuple(
@@ -396,6 +406,12 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
       InternalRow(UTF8String.fromString("1"), null, UTF8String.fromString("1")))
   }
 
+  test("from_json escaping") {
+    val schema = StructType(StructField("\"quote", IntegerType) :: Nil)
+    GenerateUnsafeProjection.generate(
+      JsonToStructs(schema, Map.empty, Literal("\"quote"), UTC_OPT) :: Nil)
+  }
+
   test("from_json") {
     val jsonData = """{"a": 1}"""
     val schema = StructType(StructField("a", IntegerType) :: Nil)
@@ -549,6 +565,13 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with
     )
   }
 
+  test("to_json escaping") {
+    val schema = StructType(StructField("\"quote", IntegerType) :: Nil)
+    val struct = Literal.create(create_row(1), schema)
+    GenerateUnsafeProjection.generate(
+      StructsToJson(Map.empty, struct, UTC_OPT) :: Nil)
+  }
+
   test("to_json - struct") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
     val struct = Literal.create(create_row(1), schema)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index 712d2bc4c4736..ad9492a8d3ab1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.types.StringType
 
 /**
@@ -255,6 +256,10 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
     val nonNullExpr = RegExpReplace(Literal("100-200"), Literal("(\\d+)"), Literal("num"))
     checkEvaluation(nonNullExpr, "num-num", row1)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      RegExpReplace(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("SPARK-22570: RegExpReplace should not create a lot of global variables") {
@@ -305,6 +310,10 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       expr, row9, "Regex group count is 1, but the specified group index is 2")
     checkExceptionInExpression[IllegalArgumentException](
       expr, row10, "Regex group count is 0, but the specified group index is 1")
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      RegExpExtract(Literal("\"quote"), Literal("\"quote"), Literal(1)) :: Nil)
   }
 
   test("SPLIT") {
@@ -327,6 +336,10 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       StringSplit(s1, s2, -1), Seq("aa", "bb", "cc"), row1)
     checkEvaluation(StringSplit(s1, s2, -1), null, row2)
     checkEvaluation(StringSplit(s1, s2, -1), null, row3)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      StringSplit(Literal("\"quote"), Literal("\"quote"), Literal(-1)) :: Nil)
   }
 
   test("SPARK-30759: cache initialization for literal patterns") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 4308f98d6969a..f18364d844ce1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.types._
 
 class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -155,6 +156,11 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(c1 endsWith "b", false, row)
     checkEvaluation(c2 endsWith "b", null, row)
     checkEvaluation(c1 endsWith Literal.create(null, StringType), null, row)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(Contains(Literal("\"quote"), Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(EndsWith(Literal("\"quote"), Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(StartsWith(Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("Substring") {
@@ -352,6 +358,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Decode(b, Literal("utf-8")), null, create_row(null))
     checkEvaluation(Decode(Literal.create(null, BinaryType), Literal("utf-8")), null)
     checkEvaluation(Decode(b, Literal.create(null, StringType)), null, create_row(null))
+
+    // Test escaping of charset
+    GenerateUnsafeProjection.generate(Encode(a, Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(Decode(b, Literal("\"quote")) :: Nil)
   }
 
   test("initcap unit test") {
@@ -379,6 +389,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(Levenshtein(Literal("千世"), Literal("fog")), 3)
     checkEvaluation(Levenshtein(Literal("世界千世"), Literal("大a界b")), 4)
     // scalastyle:on
+
+    // Test escaping of arguments:
+    GenerateUnsafeProjection.generate(Levenshtein(Literal("\"quotea"), Literal("\"quoteb")) :: Nil)
   }
 
   test("soundex unit test") {
@@ -560,6 +573,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringTrim(Literal("a"), Literal.create(null, StringType)), null)
     checkEvaluation(StringTrim(Literal.create(null, StringType), Literal("a")), null)
 
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringTrim(Literal("\"quote"), Literal("\"quote")) :: Nil)
+
     checkEvaluation(StringTrim(Literal("yxTomxx"), Literal("xyz")), "Tom")
     checkEvaluation(StringTrim(Literal("xxxbarxxx"), Literal("x")), "bar")
   }
@@ -587,6 +603,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringTrimLeft(Literal.create(null, StringType), Literal("a")), null)
     checkEvaluation(StringTrimLeft(Literal("a"), Literal.create(null, StringType)), null)
 
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      StringTrimLeft(Literal("\"quote"), Literal("\"quote")) :: Nil)
+
     checkEvaluation(StringTrimLeft(Literal("zzzytest"), Literal("xyz")), "test")
     checkEvaluation(StringTrimLeft(Literal("zzzytestxyz"), Literal("xyz")), "testxyz")
     checkEvaluation(StringTrimLeft(Literal("xyxXxyLAST WORD"), Literal("xy")), "XxyLAST WORD")
@@ -616,6 +636,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringTrimRight(Literal("a"), Literal.create(null, StringType)), null)
     checkEvaluation(StringTrimRight(Literal.create(null, StringType), Literal("a")), null)
 
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      StringTrimRight(Literal("\"quote"), Literal("\"quote")) :: Nil)
+
     checkEvaluation(StringTrimRight(Literal("testxxzx"), Literal("xyz")), "test")
     checkEvaluation(StringTrimRight(Literal("xyztestxxzx"), Literal("xyz")), "xyztest")
     checkEvaluation(StringTrimRight(Literal("TURNERyxXxy"), Literal("xy")), "TURNERyxX")
@@ -632,6 +656,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       FormatString(Literal("aa%d%s"), Literal.create(null, IntegerType), "cc"), "aanullcc")
     checkEvaluation(
       FormatString(Literal("aa%d%s"), 12, Literal.create(null, StringType)), "aa12null")
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(FormatString(Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("SPARK-22603: FormatString should not generate codes beyond 64KB") {
@@ -662,6 +689,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringInstr(s1, s2), 1, create_row("花花世界", "花"))
     checkEvaluation(StringInstr(s1, s2), 0, create_row("花花世界", "小"))
     // scalastyle:on
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringInstr(Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 
   test("LOCATE") {
@@ -718,6 +748,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringRPad(s1, s2, s3), null, row3)
     checkEvaluation(StringRPad(s1, s2, s3), null, row4)
     checkEvaluation(StringRPad(s1, s2, s3), null, row5)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringLPad(Literal("\"quote"), s2, Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(StringRPad(Literal("\"quote"), s2, Literal("\"quote")) :: Nil)
     checkEvaluation(StringRPad(Literal("hi"), Literal(5)), "hi   ")
     checkEvaluation(StringRPad(Literal("hi"), Literal(1)), "h")
   }
@@ -732,6 +766,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringRepeat(Literal("hi"), Literal(-1)), "", row1)
     checkEvaluation(StringRepeat(s1, s2), "hihi", row1)
     checkEvaluation(StringRepeat(s1, s2), null, row2)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(StringRepeat(Literal("\"quote"), Literal(2)) :: Nil)
   }
 
   test("REVERSE") {
@@ -897,6 +934,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     assert(ParseUrl(Seq(Literal("1"), Literal(2))).checkInputDataTypes().isFailure)
     assert(ParseUrl(Seq(Literal(1), Literal("2"))).checkInputDataTypes().isFailure)
     assert(ParseUrl(Seq(Literal("1"), Literal("2"), Literal(3))).checkInputDataTypes().isFailure)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(ParseUrl(Seq(Literal("\"quote"), Literal("\"quote"))) :: Nil)
   }
 
   test("Sentences") {
@@ -919,5 +959,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       answer)
     checkEvaluation(Sentences("Hi there! The price was $1,234.56.... But, not now.", "XXX", "YYY"),
       answer)
+
+    // Test escaping of arguments
+    GenerateUnsafeProjection.generate(
+      Sentences(Literal("\"quote"), Literal("\"quote"), Literal("\"quote")) :: Nil)
   }
 }

From a21a0769b20c5fc613eb1b75362b349ba39290a4 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 31 Mar 2020 12:35:01 +0900
Subject: [PATCH 0429/1280] [SPARK-31279][SQL][DOC] Add version information to
 the configuration of Hive
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the configuration of `Hive`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.hive.metastore.version | 1.4.0 | SPARK-6908 | 05454fd8aef75b129cbbd0288f5089c5259f4a15#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.version | 1.1.1 | SPARK-3971 | 64945f868443fbc59cb34b34c16d782dda0fb63d#diff-12fa2178364a810b3262b30d8d48aa2d |  
spark.sql.hive.metastore.jars | 1.4.0 | SPARK-6908 | 05454fd8aef75b129cbbd0288f5089c5259f4a15#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.convertMetastoreParquet | 1.1.1 | SPARK-2406 | cc4015d2fa3785b92e6ab079b3abcf17627f7c56#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.convertMetastoreParquet.mergeSchema | 1.3.1 | SPARK-6575 | 778c87686af0c04df9dfe144b8f744f271a988ad#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.convertMetastoreOrc | 2.0.0 | SPARK-14070 | 1e886159849e3918445d3fdc3c4cef86c6c1a236#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.convertInsertingPartitionedTable | 3.0.0 | SPARK-28573 | d5688dc732890923c326f272b0c18c329a69459a#diff-842e3447fc453de26c706db1cac8f2c4 |  
spark.sql.hive.convertMetastoreCtas | 3.0.0 | SPARK-25271 | 5ad03607d1487e7ab3e3b6d00eef9c4028ed4975#diff-842e3447fc453de26c706db1cac8f2c4 |  
spark.sql.hive.metastore.sharedPrefixes | 1.4.0 | SPARK-7491 | a8556086d33cb993fab0ae2751e31455e6c664ab#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.metastore.barrierPrefixes | 1.4.0 | SPARK-7491 | a8556086d33cb993fab0ae2751e31455e6c664ab#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.hive.thriftServer.async | 1.5.0 | SPARK-6964 | eb19d3f75cbd002f7e72ce02017a8de67f562792#diff-ff50aea397a607b79df9bec6f2a841db |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #28042 from beliefer/add-version-to-hive-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-data-sources-hive-tables.md                  |  6 +++++-
 .../scala/org/apache/spark/sql/hive/HiveUtils.scala   | 11 +++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md
index 0054d466db987..22514cd082ecf 100644
--- a/docs/sql-data-sources-hive-tables.md
+++ b/docs/sql-data-sources-hive-tables.md
@@ -124,7 +124,7 @@ will compile against built-in Hive and use those classes for internal execution
 The following options can be used to configure the version of Hive that is used to retrieve metadata:
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.hive.metastore.version</code></td>
     <td><code>2.3.6</code></td>
@@ -132,6 +132,7 @@ The following options can be used to configure the version of Hive that is used
       Version of the Hive metastore. Available
       options are <code>0.12.0</code> through <code>2.3.6</code> and <code>3.0.0</code> through <code>3.1.2</code>.
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.jars</code></td>
@@ -153,6 +154,7 @@ The following options can be used to configure the version of Hive that is used
         they are packaged with your application.</li>
       </ol>
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.sharedPrefixes</code></td>
@@ -166,6 +168,7 @@ The following options can be used to configure the version of Hive that is used
         custom appenders that are used by log4j.
       </p>
     </td>
+    <td>1.4.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.hive.metastore.barrierPrefixes</code></td>
@@ -177,5 +180,6 @@ The following options can be used to configure the version of Hive that is used
         prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).
       </p>
     </td>
+    <td>1.4.0</td>
   </tr>
 </table>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 9c4b8a5819a33..3c20e68278e3e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -65,6 +65,7 @@ private[spark] object HiveUtils extends Logging {
     .doc("Version of the Hive metastore. Available options are " +
         "<code>0.12.0</code> through <code>2.3.6</code> and " +
         "<code>3.0.0</code> through <code>3.1.2</code>.")
+    .version("1.4.0")
     .stringConf
     .createWithDefault(builtinHiveVersion)
 
@@ -73,6 +74,7 @@ private[spark] object HiveUtils extends Logging {
   // already rely on this config.
   val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version")
     .doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.")
+    .version("1.1.1")
     .stringConf
     .createWithDefault(builtinHiveVersion)
 
@@ -89,12 +91,14 @@ private[spark] object HiveUtils extends Logging {
       |   Use Hive jars of specified version downloaded from Maven repositories.
       | 3. A classpath in the standard format for both Hive and Hadoop.
       """.stripMargin)
+    .version("1.4.0")
     .stringConf
     .createWithDefault("builtin")
 
   val CONVERT_METASTORE_PARQUET = buildConf("spark.sql.hive.convertMetastoreParquet")
     .doc("When set to true, the built-in Parquet reader and writer are used to process " +
       "parquet tables created by using the HiveQL syntax, instead of Hive serde.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -103,12 +107,14 @@ private[spark] object HiveUtils extends Logging {
       .doc("When true, also tries to merge possibly different but compatible Parquet schemas in " +
         "different Parquet data files. This configuration is only effective " +
         "when \"spark.sql.hive.convertMetastoreParquet\" is true.")
+      .version("1.3.1")
       .booleanConf
       .createWithDefault(false)
 
   val CONVERT_METASTORE_ORC = buildConf("spark.sql.hive.convertMetastoreOrc")
     .doc("When set to true, the built-in ORC reader and writer are used to process " +
       "ORC tables created by using the HiveQL syntax, instead of Hive serde.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -118,6 +124,7 @@ private[spark] object HiveUtils extends Logging {
         "`spark.sql.hive.convertMetastoreOrc` is true, the built-in ORC/Parquet writer is used" +
         "to process inserting into partitioned ORC/Parquet tables created by using the HiveSQL " +
         "syntax.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -126,6 +133,7 @@ private[spark] object HiveUtils extends Logging {
       "instead of Hive serde in CTAS. This flag is effective only if " +
       "`spark.sql.hive.convertMetastoreParquet` or `spark.sql.hive.convertMetastoreOrc` is " +
       "enabled respectively for Parquet and ORC formats")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -135,6 +143,7 @@ private[spark] object HiveUtils extends Logging {
       "that should be shared is JDBC drivers that are needed to talk to the metastore. Other " +
       "classes that need to be shared are those that interact with classes that are already " +
       "shared. For example, custom appenders that are used by log4j.")
+    .version("1.4.0")
     .stringConf
     .toSequence
     .createWithDefault(jdbcPrefixes)
@@ -146,12 +155,14 @@ private[spark] object HiveUtils extends Logging {
     .doc("A comma separated list of class prefixes that should explicitly be reloaded for each " +
       "version of Hive that Spark SQL is communicating with. For example, Hive UDFs that are " +
       "declared in a prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).")
+    .version("1.4.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   val HIVE_THRIFT_SERVER_ASYNC = buildConf("spark.sql.hive.thriftServer.async")
     .doc("When set to true, Hive Thrift server executes SQL queries in an asynchronous way.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(true)
 

From 88252acd72a3bd7780e2680264fea076d725b9ef Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 31 Mar 2020 12:31:25 +0900
Subject: [PATCH 0430/1280] [SPARK-31215][SQL][DOC] Add version information to
 the static configuration of SQL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the static configuration of `SQL`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.warehouse.dir | 2.0.0 | SPARK-14994 | 054f991c4350af1350af7a4109ee77f4a34822f0#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.catalogImplementation | 2.0.0 | SPARK-14720 and SPARK-13643 | 8fc267ab3322e46db81e725a5cb1adb5a71b2b4d#diff-6bdad48cfc34314e89599655442ff210 |  
spark.sql.globalTempDatabase | 2.1.0 | SPARK-17338 | 23ddff4b2b2744c3dc84d928e144c541ad5df376#diff-6bdad48cfc34314e89599655442ff210 |  
spark.sql.sources.schemaStringLengthThreshold | 1.3.1 | SPARK-6024 | 6200f0709c5c8440decae8bf700d7859f32ac9d5#diff-41ef65b9ef5b518f77e2a03559893f4d | 1.3
spark.sql.filesourceTableRelationCacheSize | 2.2.0 | SPARK-19265 | 9d9d67c7957f7cbbdbe889bdbc073568b2bfbb16#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.codegen.cache.maxEntries | 2.4.0 | SPARK-24727 | b2deef64f604ddd9502a31105ed47cb63470ec85#diff-5081b9388de3add800b6e4a6ddf55c01 |
spark.sql.codegen.comments | 2.0.0 | SPARK-15680 | f0e8738c1ec0e4c5526aeada6f50cf76428f9afd#diff-8bcc5aea39c73d4bf38aef6f6951d42c |  
spark.sql.debug | 2.1.0 | SPARK-17899 | db8784feaa605adcbd37af4bc8b7146479b631f8#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.hive.thriftServer.singleSession | 1.6.0 | SPARK-11089 | 167ea61a6a604fd9c0b00122a94d1bc4b1de24ff#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.extensions | 2.2.0 | SPARK-18127 | f0de600797ff4883927d0c70732675fd8629e239#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.queryExecutionListeners | 2.3.0 | SPARK-19558 | bd4eb9ce57da7bacff69d9ed958c94f349b7e6fb#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.streaming.streamingQueryListeners | 2.4.0 | SPARK-24479 | 7703b46d2843db99e28110c4c7ccf60934412504#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.ui.retainedExecutions | 1.5.0 | SPARK-8861 and SPARK-8862 | ebc3aad272b91cf58e2e1b4aa92b49b8a947a045#diff-81764e4d52817f83bdd5336ef1226bd9 |  
spark.sql.broadcastExchange.maxThreadThreshold | 3.0.0 | SPARK-26601 | 126310ca68f2f248ea8b312c4637eccaba2fdc2b#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.subquery.maxThreadThreshold | 2.4.6 | SPARK-30556 | 2fc562cafd71ec8f438f37a28b65118906ab2ad2#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.event.truncate.length | 3.0.0 | SPARK-27045 | e60d8fce0b0cf2a6d766ea2fc5f994546550570a#diff-5081b9388de3add800b6e4a6ddf55c01 |
spark.sql.legacy.sessionInitWithConfigDefaults | 3.0.0 | SPARK-27253 | 83f628b57da39ad9732d1393aebac373634a2eb9#diff-5081b9388de3add800b6e4a6ddf55c01 |
spark.sql.defaultUrlStreamHandlerFactory.enabled | 3.0.0 | SPARK-25694 | 8469614c0513fbed87977d4e741649db3fdd8add#diff-5081b9388de3add800b6e4a6ddf55c01 |
spark.sql.streaming.ui.enabled | 3.0.0 | SPARK-29543 | f9b86370cb04b72a4f00cbd4d60873960aa2792c#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.streaming.ui.retainedProgressUpdates | 3.0.0 | SPARK-29543 | f9b86370cb04b72a4f00cbd4d60873960aa2792c#diff-5081b9388de3add800b6e4a6ddf55c01 |  
spark.sql.streaming.ui.retainedQueries | 3.0.0 | SPARK-29543 | f9b86370cb04b72a4f00cbd4d60873960aa2792c#diff-5081b9388de3add800b6e4a6ddf55c01 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #27981 from beliefer/add-version-to-sql-static-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md                         |  1 +
 .../spark/sql/internal/StaticSQLConf.scala    | 25 +++++++++++++++++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 8d1cc849bffe8..8bfd4d522efb6 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1203,6 +1203,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished executions the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.ui.retainedBatches</code></td>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index 563e51ed597b2..d20252825c55b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -32,17 +32,20 @@ object StaticSQLConf {
 
   val WAREHOUSE_PATH = buildStaticConf("spark.sql.warehouse.dir")
     .doc("The default location for managed databases and tables.")
+    .version("2.0.0")
     .stringConf
     .createWithDefault(Utils.resolveURI("spark-warehouse").toString)
 
   val CATALOG_IMPLEMENTATION = buildStaticConf("spark.sql.catalogImplementation")
     .internal()
+    .version("2.0.0")
     .stringConf
     .checkValues(Set("hive", "in-memory"))
     .createWithDefault("in-memory")
 
   val GLOBAL_TEMP_DATABASE = buildStaticConf("spark.sql.globalTempDatabase")
     .internal()
+    .version("2.1.0")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .createWithDefault("global_temp")
@@ -55,9 +58,10 @@ object StaticSQLConf {
   // that's why this conf has to be a static SQL conf.
   val SCHEMA_STRING_LENGTH_THRESHOLD =
     buildStaticConf("spark.sql.sources.schemaStringLengthThreshold")
+      .internal()
       .doc("The maximum length allowed in a single cell when " +
         "storing additional schema information in Hive's metastore.")
-      .internal()
+      .version("1.3.1")
       .intConf
       .createWithDefault(4000)
 
@@ -65,6 +69,7 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.filesourceTableRelationCacheSize")
       .internal()
       .doc("The maximum size of the cache that maps qualified table names to table relation plans.")
+      .version("2.2.0")
       .intConf
       .checkValue(cacheSize => cacheSize >= 0, "The maximum size of the cache must not be negative")
       .createWithDefault(1000)
@@ -73,6 +78,7 @@ object StaticSQLConf {
       .internal()
       .doc("When nonzero, enable caching of generated classes for operators and expressions. " +
         "All jobs share the cache that can use up to the specified number for generated classes.")
+      .version("2.4.0")
       .intConf
       .checkValue(maxEntries => maxEntries >= 0, "The maximum must not be negative")
       .createWithDefault(100)
@@ -82,6 +88,7 @@ object StaticSQLConf {
     .doc("When true, put comment in the generated code. Since computing huge comments " +
       "can be extremely expensive in certain cases, such as deeply-nested expressions which " +
       "operate over inputs with wide schemas, default is false.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -90,6 +97,7 @@ object StaticSQLConf {
   val DEBUG_MODE = buildStaticConf("spark.sql.debug")
     .internal()
     .doc("Only used for internal debugging. Not all functions are supported when it is enabled.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -98,6 +106,7 @@ object StaticSQLConf {
       .doc("When set to true, Hive Thrift server is running in a single session mode. " +
         "All the JDBC/ODBC connections share the temporary views, function registries, " +
         "SQL configuration and the current database.")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -109,6 +118,7 @@ object StaticSQLConf {
       "applied in the specified order. For the case of parsers, the last parser is used and each " +
       "parser can delegate to its predecessor. For the case of function name conflicts, the last " +
       "registered function name is used.")
+    .version("2.2.0")
     .stringConf
     .toSequence
     .createOptional
@@ -117,6 +127,7 @@ object StaticSQLConf {
     .doc("List of class names implementing QueryExecutionListener that will be automatically " +
       "added to newly created sessions. The classes should have either a no-arg constructor, " +
       "or a constructor that expects a SparkConf argument.")
+    .version("2.3.0")
     .stringConf
     .toSequence
     .createOptional
@@ -125,6 +136,7 @@ object StaticSQLConf {
     .doc("List of class names implementing StreamingQueryListener that will be automatically " +
       "added to newly created sessions. The classes should have either a no-arg constructor, " +
       "or a constructor that expects a SparkConf argument.")
+    .version("2.4.0")
     .stringConf
     .toSequence
     .createOptional
@@ -132,6 +144,7 @@ object StaticSQLConf {
   val UI_RETAINED_EXECUTIONS =
     buildStaticConf("spark.sql.ui.retainedExecutions")
       .doc("Number of executions to retain in the Spark UI.")
+      .version("1.5.0")
       .intConf
       .createWithDefault(1000)
 
@@ -144,6 +157,7 @@ object StaticSQLConf {
         "Notice the number should be carefully chosen since decreasing parallelism might " +
         "cause longer waiting for other broadcasting. Also, increasing parallelism may " +
         "cause memory problem.")
+      .version("3.0.0")
       .intConf
       .checkValue(thres => thres > 0 && thres <= 128, "The threshold must be in (0,128].")
       .createWithDefault(128)
@@ -152,6 +166,7 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.subquery.maxThreadThreshold")
       .internal()
       .doc("The maximum degree of parallelism to execute the subquery.")
+      .version("2.4.6")
       .intConf
       .checkValue(thres => thres > 0 && thres <= 128, "The threshold must be in (0,128].")
       .createWithDefault(16)
@@ -159,6 +174,7 @@ object StaticSQLConf {
   val SQL_EVENT_TRUNCATE_LENGTH = buildStaticConf("spark.sql.event.truncate.length")
     .doc("Threshold of SQL length beyond which it will be truncated before adding to " +
       "event. Defaults to no truncation. If set to 0, callsite will be logged instead.")
+    .version("3.0.0")
     .intConf
     .checkValue(_ >= 0, "Must be set greater or equal to zero")
     .createWithDefault(Int.MaxValue)
@@ -167,11 +183,13 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.legacy.sessionInitWithConfigDefaults")
       .doc("Flag to revert to legacy behavior where a cloned SparkSession receives SparkConf " +
         "defaults, dropping any overrides in its parent SparkSession.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val DEFAULT_URL_STREAM_HANDLER_FACTORY_ENABLED =
     buildStaticConf("spark.sql.defaultUrlStreamHandlerFactory.enabled")
+      .internal()
       .doc(
         "When true, register Hadoop's FsUrlStreamHandlerFactory to support " +
         "ADD JAR against HDFS locations. " +
@@ -179,7 +197,7 @@ object StaticSQLConf {
         "to support a particular protocol type, or if Hadoop's FsUrlStreamHandlerFactory " +
         "conflicts with other protocol types such as `http` or `https`. See also SPARK-25694 " +
         "and HADOOP-14598.")
-      .internal()
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -187,6 +205,7 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.streaming.ui.enabled")
       .doc("Whether to run the Structured Streaming Web UI for the Spark application when the " +
         "Spark Web UI is enabled.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -194,12 +213,14 @@ object StaticSQLConf {
     buildStaticConf("spark.sql.streaming.ui.retainedProgressUpdates")
       .doc("The number of progress updates to retain for a streaming query for Structured " +
         "Streaming UI.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(100)
 
   val STREAMING_UI_RETAINED_QUERIES =
     buildStaticConf("spark.sql.streaming.ui.retainedQueries")
       .doc("The number of inactive queries to retain for Structured Streaming UI.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(100)
 }

From 51c80a48024242a51940c9c0aafdfd7e3a0c481f Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Mon, 6 Apr 2020 21:36:51 +0900
Subject: [PATCH 0431/1280] [SPARK-31358][SQL][DOC] Document FILTER clauses of
 aggregate functions in SQL references

### What changes were proposed in this pull request?

This PR intends to improve the SQL document of `GROUP BY`; it added the description about FILTER clauses of aggregate functions.

### Why are the changes needed?

To improve the SQL documents

### Does this PR introduce any user-facing change?

Yes.

<img src="https://user-images.githubusercontent.com/692303/78558612-e2234a80-784d-11ea-9353-b3feac4d57a7.png" width="500">

### How was this patch tested?

Manually checked.

Closes #28134 from maropu/SPARK-31358.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit e24f0dcd2754a1db27e9b0f3cf27ee6d7229f717)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-syntax-qry-select-groupby.md | 44 +++++++++++++++++++++++
 docs/sql-ref-syntax-qry-select.md         |  1 +
 2 files changed, 45 insertions(+)

diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index 49a11cafbbd5a..c461a18e004a4 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -21,6 +21,7 @@ license: |
 The <code>GROUP BY</code> clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
 the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple
 aggregations for the same input record set via `GROUPING SETS`, `CUBE`, `ROLLUP` clauses.
+When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
 
 ### Syntax
 {% highlight sql %}
@@ -30,6 +31,11 @@ GROUP BY group_expression [ , group_expression [ , ... ] ]
 GROUP BY GROUPING SETS (grouping_set [ , ...])
 {% endhighlight %}
 
+While aggregate functions are defined as
+{% highlight sql %}
+aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_expression ) ]
+{% endhighlight %}
+
 ### Parameters
 <dl>
   <dt><code><em>GROUPING SETS</em></code></dt>
@@ -70,6 +76,19 @@ GROUP BY GROUPING SETS (grouping_set [ , ...])
     ((warehouse, product), (warehouse), (product), ())</code>.
     The N elements of a <code>CUBE</code> specification results in 2^N <code>GROUPING SETS</code>.
   </dd>
+  <dt><code><em>aggregate_name</em></code></dt>
+  <dd>
+    Specifies an aggregate function name (MIN, MAX, COUNT, SUM, AVG, etc.).
+  </dd>
+  <dt><code><em>DISTINCT</em></code></dt>
+  <dd>
+    Removes duplicates in input rows before they are passed to aggregate functions.
+  </dd>
+  <dt><code><em>FILTER</em></code></dt>
+  <dd>
+    Filters the input rows for which the <code>boolean_expression</code> in the <code>WHERE</code> clause evaluates
+    to true are passed to the aggregate function; other rows are discarded.
+  </dd>
 </dl>
 
 ### Examples
@@ -120,6 +139,31 @@ SELECT id, sum(quantity) AS sum, max(quantity) AS max FROM dealer GROUP BY id OR
   |300|13 |8  |
   +---+---+---+
 
+-- Count the number of distinct dealer cities per car_model.
+SELECT car_model, count(DISTINCT city) AS count FROM dealer GROUP BY car_model;
+
+  +------------+-----+
+  |   car_model|count|
+  +------------+-----+
+  | Honda Civic|    3|
+  |   Honda CRV|    2|
+  |Honda Accord|    3|
+  +------------+-----+
+
+-- Sum of only 'Honda Civic' and 'Honda CRV' quantities per dealership.
+SELECT id, sum(quantity) FILTER (
+            WHERE car_model IN ('Honda Civic', 'Honda CRV')
+        ) AS `sum(quantity)` FROM dealer
+    GROUP BY id ORDER BY id;
+
+   +---+-------------+
+   | id|sum(quantity)|
+   +---+-------------+
+   |100|           17|
+   |200|           23|
+   |300|            5|
+   +---+-------------+
+
 -- Aggregations using multiple sets of grouping columns in a single statement.
 -- Following performs aggregations based on four sets of grouping columns.
 -- 1. city, car_model
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index e87c4a587e34c..7ad1dd15c4812 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -92,6 +92,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
   <dd>
     Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
     (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
+    When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
   </dd>
   <dt><code><em>HAVING</em></code></dt>
   <dd>

From 95b3e01bc4a650019c66bc0c286ac984b5c30524 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 6 Apr 2020 09:02:22 -0500
Subject: [PATCH 0432/1280] [SPARK-31333][SQL][DOCS] Document Join Hints

### What changes were proposed in this pull request?
Document Join Hints

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes

<img width="1049" alt="Screen Shot 2020-04-03 at 9 20 15 AM" src="https://user-images.githubusercontent.com/13592258/78382976-7c546b80-758c-11ea-9a8e-e46cfb7106f5.png">

<img width="1051" alt="Screen Shot 2020-04-03 at 10 39 55 AM" src="https://user-images.githubusercontent.com/13592258/78389778-356c7300-7598-11ea-8e6c-3742dadda11c.png">

### How was this patch tested?
Manually build and check

Closes #28113 from huaxingao/join-hints.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 44d37efba2606217d131038ad81b61fedb16cb07)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                |  2 +
 docs/sql-ref-syntax-qry-select-hints.md | 71 +++++++++++++++++++++++--
 2 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 500895aab4a6c..787ee3ecebd39 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -154,6 +154,8 @@
                   url: sql-ref-syntax-qry-select-distribute-by.html
                 - text: LIMIT Clause 
                   url: sql-ref-syntax-qry-select-limit.html
+                - text: Join Hints
+                  url: sql-ref-syntax-qry-select-hints.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index eb9655fc0d731..4713ea5439b81 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Query Hints
-displayTitle: Query Hints
+title: Join Hints
+displayTitle: Join Hints
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,14 +9,75 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+### Description
+
+Join Hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
+
+### Join Hints Types
+
+<dl>
+  <dt><code><em>BROADCAST</em></code></dt>
+  <dd>
+    Suggests that Spark use broadcast join. The join side with the hint will be broadcast regardless of <code>autoBroadcastJoinThreshold</code>. If both sides of the join have the broadcast hints, the one with the smaller size (based on stats) will be broadcast. The aliases for <code>BROADCAST</code> are <code>BROADCASTJOIN</code> and <code>MAPJOIN</code>.
+  </dd>
+</dl>
+
+<dl>
+  <dt><code><em>MERGE</em></code></dt>
+  <dd>
+     Suggests that Spark use shuffle sort merge join. The aliases for <code>MERGE</code> are <code>SHUFFLE_MERGE</code> and <code>MERGEJOIN</code>.
+  </dd>
+</dl>
+
+<dl>
+  <dt><code><em>SHUFFLE_HASH</em></code></dt>
+  <dd>
+     Suggests that Spark use shuffle hash join. If both sides have the shuffle hash hints, Spark chooses the smaller side (based on stats) as the build side.
+  </dd>
+</dl>
+
+<dl>
+  <dt><code><em>SHUFFLE_REPLICATE_NL</em></code></dt>
+  <dd>
+    Suggests that Spark use shuffle-and-replicate nested loop join.
+  </dd>
+</dl>
+
+### Examples
+
+{% highlight sql %}
+
+-- Join Hints for broadcast join
+SELECT /*+ BROADCAST(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+SELECT /*+ BROADCASTJOIN (t1) */ * FROM t1 left JOIN t2 ON t1.key = t2.key;
+SELECT /*+ MAPJOIN(t2) */ * FROM t1 right JOIN t2 ON t1.key = t2.key;
+
+-- Join Hints for shuffle sort merge join
+SELECT /*+ SHUFFLE_MERGE(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+SELECT /*+ MERGEJOIN(t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+SELECT /*+ MERGE(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+
+-- Join Hints for shuffle hash join
+SELECT /*+ SHUFFLE_HASH(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+
+-- Join Hints for shuffle-and-replicate nested loop join
+SELECT /*+ SHUFFLE_REPLICATE_NL(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+
+-- When different join strategy hints are specified on both sides of a join, Spark
+-- prioritizes the BROADCAST hint over the MERGE hint over the SHUFFLE_HASH hint
+-- over the SHUFFLE_REPLICATE_NL hint.
+-- Spark will issue Warning in the following example
+-- org.apache.spark.sql.catalyst.analysis.HintErrorLogger: Hint (strategy=merge)
+-- is overridden by another hint and will not take effect.
+SELECT /*+ BROADCAST(t1) */ /*+ MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
 
-**This page is under construction**
+{% endhighlight %}

From ad8c5377999b38aadab427b5fb15fee1f73bbb5c Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 6 Apr 2020 12:42:38 -0700
Subject: [PATCH 0433/1280] Revert "[SPARK-31073][DOC][FOLLOWUP] Add
 description for Shuffle Write Time metric in StagePage to web-ui.md"

This reverts commit 5a7395d296b57c8e06bbf100b4ebdaa1100c3a91.
---
 docs/web-ui.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/web-ui.md b/docs/web-ui.md
index c53af804d8d59..e28a689c8de50 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -143,7 +143,6 @@ Summary metrics for all task are represented in a table and in a timeline.
 * **Shuffle Read Size / Records**. Total shuffle bytes read, includes both data read locally and data read from remote executors.
 * **Shuffle Read Blocked Time** is the time that tasks spent blocked waiting for shuffle data to be read from remote machines.
 * **Shuffle Remote Reads** is the total shuffle bytes read from remote executors.
-* **Shuffle Write Time** is the time that tasks spent writing shuffle data.
 * **Shuffle spill (memory)** is the size of the deserialized form of the shuffled data in memory.
 * **Shuffle spill (disk)** is the size of the serialized form of the data on disk.
 

From 87d41c11cb1aaaeb903ac80c3fd0baf1159c3676 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 12 Mar 2020 09:52:57 +0900
Subject: [PATCH 0434/1280] [SPARK-31092][YARN][DOC] Add version information to
 the configuration of Yarn
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the configuration of `Yarn`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.yarn.tags | 1.5.0 | SPARK-9782 | 9b731fad2b43ca18f3c5274062d4c7bc2622ab72#diff-b050df3f55b82065803d6e83453b9706 |  
spark.yarn.priority | 3.0.0 | SPARK-29603 | 4615769736f4c052ae1a2de26e715e229154cd2f#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.am.attemptFailuresValidityInterval | 1.6.0 | SPARK-10739 | f97e9323b526b3d0b0fee0ca03f4276f37bb5750#diff-b050df3f55b82065803d6e83453b9706 |
spark.yarn.executor.failuresValidityInterval | 2.0.0 | SPARK-6735 | 8b44bd52fa40c0fc7d34798c3654e31533fd3008#diff-14b8ed2ef4e3da985300b8d796a38fa9 |
spark.yarn.maxAppAttempts | 1.3.0 | SPARK-2165 | 8fdd48959c93b9cf809f03549e2ae6c4687d1fcd#diff-b050df3f55b82065803d6e83453b9706 |
spark.yarn.user.classpath.first | 1.3.0 | SPARK-5087 | 8d45834debc6986e61831d0d6e982d5528dccc51#diff-b050df3f55b82065803d6e83453b9706 |  
spark.yarn.config.gatewayPath | 1.5.0 | SPARK-8302 | 37bf76a2de2143ec6348a3d43b782227849520cc#diff-b050df3f55b82065803d6e83453b9706 |  
spark.yarn.config.replacementPath | 1.5.0 | SPARK-8302 | 37bf76a2de2143ec6348a3d43b782227849520cc#diff-b050df3f55b82065803d6e83453b9706 |  
spark.yarn.queue | 1.0.0 | SPARK-1126 | 1617816090e7b20124a512a43860a21232ebf511#diff-ae6a41a938a767e5bb97b5d738371a5b |  
spark.yarn.historyServer.address | 1.0.0 | SPARK-1408 | 0058b5d2c74147d24b127a5432f89ebc7050dc18#diff-923ae58523a12397f74dd590744b8b41 |  
spark.yarn.historyServer.allowTracking | 2.2.0 | SPARK-19554 | 4661d30b988bf773ab45a15b143efb2908d33743#diff-4804e0f83ca7f891183eb0db229b4b9a |
spark.yarn.archive | 2.0.0 | SPARK-13577 | 07f1c5447753a3d593cd6ececfcb03c11b1cf8ff#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.jars | 2.0.0 | SPARK-13577 | 07f1c5447753a3d593cd6ececfcb03c11b1cf8ff#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.dist.archives | 1.0.0 | SPARK-1126 | 1617816090e7b20124a512a43860a21232ebf511#diff-ae6a41a938a767e5bb97b5d738371a5b |  
spark.yarn.dist.files | 1.0.0 | SPARK-1126 | 1617816090e7b20124a512a43860a21232ebf511#diff-ae6a41a938a767e5bb97b5d738371a5b |  
spark.yarn.dist.jars | 2.0.0 | SPARK-12343 | 8ba2b7f28fee39c4839e5ea125bd25f5091a3a1e#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.preserve.staging.files | 1.1.0 | SPARK-2933 | b92d823ad13f6fcc325eeb99563bea543871c6aa#diff-85a1f4b2810b3e11b8434dcefac5bb85 |  
spark.yarn.submit.file.replication | 0.8.1 | None | 4668fcb9ff8f9c176c4866480d52dde5d67c8522#diff-b050df3f55b82065803d6e83453b9706 |
spark.yarn.submit.waitAppCompletion | 1.4.0 | SPARK-3591 | b65bad65c3500475b974ca0219f218eef296db2c#diff-b050df3f55b82065803d6e83453b9706 |
spark.yarn.report.interval | 0.9.0 | None | ebdfa6bb9766209bc5a3c4241fa47141c5e9c5cb#diff-e0a7ae95b6d8e04a67ebca0945d27b65 |  
spark.yarn.clientLaunchMonitorInterval | 2.3.0 | SPARK-16019 | 1cad31f00644d899d8e74d58c6eb4e9f72065473#diff-4804e0f83ca7f891183eb0db229b4b9a |
spark.yarn.am.waitTime | 1.3.0 | SPARK-3779 | 253b72b56fe908bbab5d621eae8a5f359c639dfd#diff-87125050a2e2eaf87ea83aac9c19b200 |  
spark.yarn.metrics.namespace | 2.4.0 | SPARK-24594 | d2436a85294a178398525c37833dae79d45c1452#diff-4804e0f83ca7f891183eb0db229b4b9a |
spark.yarn.am.nodeLabelExpression | 1.6.0 | SPARK-7173 | 7db3610327d0725ec2ad378bc873b127a59bb87a#diff-b050df3f55b82065803d6e83453b9706 |
spark.yarn.containerLauncherMaxThreads | 1.2.0 | SPARK-1713 | 1f4a648d4e30e837d6cf3ea8de1808e2254ad70b#diff-801a04f9e67321f3203399f7f59234c1 |  
spark.yarn.max.executor.failures | 1.0.0 | SPARK-1183 | 698373211ef3cdf841c82d48168cd5dbe00a57b4#diff-0c239e58b37779967e0841fb42f3415a |  
spark.yarn.scheduler.reporterThread.maxFailures | 1.2.0 | SPARK-3304 | 11c10df825419372df61a8d23c51e8c3cc78047f#diff-85a1f4b2810b3e11b8434dcefac5bb85 |  
spark.yarn.scheduler.heartbeat.interval-ms | 0.8.1 | None | ee22be0e6c302fb2cdb24f83365c2b8a43a1baab#diff-87125050a2e2eaf87ea83aac9c19b200 |  
spark.yarn.scheduler.initial-allocation.interval | 1.4.0 | SPARK-7533 | 3ddf051ee7256f642f8a17768d161c7b5f55c7e1#diff-87125050a2e2eaf87ea83aac9c19b200 |  
spark.yarn.am.finalMessageLimit | 2.4.0 | SPARK-25174 | f8346d2fc01f1e881e4e3f9c4499bf5f9e3ceb3f#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.am.cores | 1.3.0 | SPARK-1507 | 2be82b1e66cd188456bbf1e5abb13af04d1629d5#diff-746d34aa06bfa57adb9289011e725472 |  
spark.yarn.am.extraJavaOptions | 1.3.0 | SPARK-5087 | 8d45834debc6986e61831d0d6e982d5528dccc51#diff-b050df3f55b82065803d6e83453b9706 |  
spark.yarn.am.extraLibraryPath | 1.4.0 | SPARK-7281 | 7b5dd3e3c0030087eea5a8224789352c03717c1d#diff-b050df3f55b82065803d6e83453b9706 |  
spark.yarn.am.memoryOverhead | 1.3.0 | SPARK-1953 | e96645206006a009e5c1a23bbd177dcaf3ef9b83#diff-746d34aa06bfa57adb9289011e725472 |  
spark.yarn.am.memory | 1.3.0 | SPARK-1953 | e96645206006a009e5c1a23bbd177dcaf3ef9b83#diff-746d34aa06bfa57adb9289011e725472 |  
spark.driver.appUIAddress | 1.1.0 | SPARK-1291 | 72ea56da8e383c61c6f18eeefef03b9af00f5158#diff-2b4617e158e9c5999733759550440b96 |  
spark.yarn.executor.nodeLabelExpression | 1.4.0 | SPARK-6470 | 82fee9d9aad2c9ba2fb4bd658579fe99218cafac#diff-d4620cf162e045960d84c88b2e0aa428 |  
spark.yarn.unmanagedAM.enabled | 3.0.0 | SPARK-22404 | f06bc0cd1dee2a58e04ebf24bf719a2f7ef2dc4e#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.rolledLog.includePattern | 2.0.0 | SPARK-15990 | 272a2f78f3ff801b94a81fa8fcc6633190eaa2f4#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.rolledLog.excludePattern | 2.0.0 | SPARK-15990 | 272a2f78f3ff801b94a81fa8fcc6633190eaa2f4#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.user.jar | 1.1.0 | SPARK-1395 | e380767de344fd6898429de43da592658fd86a39#diff-50e237ea17ce94c3ccfc44143518a5f7 |  
spark.yarn.secondary.jars | 0.9.2 | SPARK-1870 | 1d3aab96120c6770399e78a72b5692cf8f61a144#diff-50b743cff4885220c828b16c44eeecfd |  
spark.yarn.cache.filenames | 2.0.0 | SPARK-14602 | f47dbf27fa034629fab12d0f3c89ab75edb03f86#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.cache.sizes | 2.0.0 | SPARK-14602 | f47dbf27fa034629fab12d0f3c89ab75edb03f86#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.cache.timestamps | 2.0.0 | SPARK-14602 | f47dbf27fa034629fab12d0f3c89ab75edb03f86#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.cache.visibilities | 2.0.0 | SPARK-14602 | f47dbf27fa034629fab12d0f3c89ab75edb03f86#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.cache.types | 2.0.0 | SPARK-14602 | f47dbf27fa034629fab12d0f3c89ab75edb03f86#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.cache.confArchive | 2.0.0 | SPARK-14602 | f47dbf27fa034629fab12d0f3c89ab75edb03f86#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.yarn.blacklist.executor.launch.blacklisting.enabled | 2.4.0 | SPARK-16630 | b56e9c613fb345472da3db1a567ee129621f6bf3#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.exclude.nodes | 3.0.0 | SPARK-26688 | caceaec93203edaea1d521b88e82ef67094cdea9#diff-4804e0f83ca7f891183eb0db229b4b9a |  
The following appears in the document |   |   |   |  
spark.yarn.am.resource.{resource-type}.amount | 3.0.0 | SPARK-20327 | 3946de773498621f88009c309254b019848ed490#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.driver.resource.{resource-type}.amount | 3.0.0 | SPARK-20327 | 3946de773498621f88009c309254b019848ed490#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.executor.resource.{resource-type}.amount | 3.0.0 | SPARK-20327 | 3946de773498621f88009c309254b019848ed490#diff-4804e0f83ca7f891183eb0db229b4b9a |  
spark.yarn.appMasterEnv.[EnvironmentVariableName] | 1.1.0 | SPARK-1680 | 7b798e10e214cd407d3399e2cab9e3789f9a929e#diff-50e237ea17ce94c3ccfc44143518a5f7 |  
spark.yarn.kerberos.relogin.period | 2.3.0 | SPARK-22290 | dc2714da50ecba1bf1fdf555a82a4314f763a76e#diff-4804e0f83ca7f891183eb0db229b4b9a |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #27856 from beliefer/add-version-to-yarn-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/running-on-yarn.md                       | 64 ++++++++++++++++---
 .../org/apache/spark/deploy/yarn/config.scala | 56 +++++++++++++++-
 2 files changed, 107 insertions(+), 13 deletions(-)

diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index ab4e96329a69b..eec73e8ff67e7 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -130,7 +130,7 @@ To use a custom metrics.properties for the application master and executors, upd
 #### Spark Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.yarn.am.memory</code></td>
   <td><code>512m</code></td>
@@ -140,6 +140,7 @@ To use a custom metrics.properties for the application master and executors, upd
     <p/>
     Use lower-case suffixes, e.g. <code>k</code>, <code>m</code>, <code>g</code>, <code>t</code>, and <code>p</code>, for kibi-, mebi-, gibi-, tebi-, and pebibytes, respectively.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.resource.{resource-type}.amount</code></td>
@@ -153,6 +154,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Example: 
     To request GPU resources from YARN, use: <code>spark.yarn.am.resource.yarn.io/gpu.amount</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.driver.resource.{resource-type}.amount</code></td>
@@ -165,18 +167,20 @@ To use a custom metrics.properties for the application master and executors, upd
     Example: 
     To request GPU resources from YARN, use: <code>spark.yarn.driver.resource.yarn.io/gpu.amount</code>
   </td>
+  <td>3.0.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.executor.resource.{resource-type}.amount</code></td>
   <td><code>(none)</code></td>
- <td>
-     Amount of resource to use per executor process.
-     Please note that this feature can be used only with YARN 3.0+
-     For reference, see YARN Resource Model documentation: https://hadoop.apache.org/docs/r3.0.1/hadoop-yarn/hadoop-yarn-site/ResourceModel.html
-     <p/>
-     Example: 
-     To request GPU resources from YARN, use: <code>spark.yarn.executor.resource.yarn.io/gpu.amount</code>
- </td>
+  <td>
+    Amount of resource to use per executor process.
+    Please note that this feature can be used only with YARN 3.0+
+    For reference, see YARN Resource Model documentation: https://hadoop.apache.org/docs/r3.0.1/hadoop-yarn/hadoop-yarn-site/ResourceModel.html
+    <p/>
+    Example: 
+    To request GPU resources from YARN, use: <code>spark.yarn.executor.resource.yarn.io/gpu.amount</code>
+  </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.cores</code></td>
@@ -185,6 +189,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Number of cores to use for the YARN Application Master in client mode.
     In cluster mode, use <code>spark.driver.cores</code> instead.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.waitTime</code></td>
@@ -193,6 +198,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Only used in <code>cluster</code> mode. Time for the YARN Application Master to wait for the
     SparkContext to be initialized.
   </td>
+ <td>1.3.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.submit.file.replication</code></td>
@@ -200,6 +206,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     HDFS replication level for the files uploaded into HDFS for the application. These include things like the Spark jar, the app jar, and any distributed cache files/archives.
   </td>
+  <td>0.8.1</td>
 </tr>
 <tr>
   <td><code>spark.yarn.stagingDir</code></td>
@@ -207,6 +214,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Staging directory used while submitting applications.
   </td>
+ <td>2.0.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.preserve.staging.files</code></td>
@@ -214,6 +222,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Set to <code>true</code> to preserve the staged files (Spark jar, app jar, distributed cache files) at the end of the job rather than delete them.
   </td>
+  <td>1.1.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.scheduler.heartbeat.interval-ms</code></td>
@@ -223,6 +232,7 @@ To use a custom metrics.properties for the application master and executors, upd
     The value is capped at half the value of YARN's configuration for the expiry interval, i.e.
     <code>yarn.am.liveness-monitor.expiry-interval-ms</code>.
   </td>
+  <td>0.8.1</td>
 </tr>
 <tr>
   <td><code>spark.yarn.scheduler.initial-allocation.interval</code></td>
@@ -234,6 +244,7 @@ To use a custom metrics.properties for the application master and executors, upd
     successive eager heartbeats if pending containers still exist, until
     <code>spark.yarn.scheduler.heartbeat.interval-ms</code> is reached.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.max.executor.failures</code></td>
@@ -241,6 +252,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The maximum number of executor failures before failing the application.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.historyServer.address</code></td>
@@ -249,6 +261,7 @@ To use a custom metrics.properties for the application master and executors, upd
     The address of the Spark history server, e.g. <code>host.com:18080</code>. The address should not contain a scheme (<code>http://</code>). Defaults to not being set since the history server is an optional service. This address is given to the YARN ResourceManager when the Spark application finishes to link the application from the ResourceManager UI to the Spark history server UI.
     For this property, YARN properties can be used as variables, and these are substituted by Spark at runtime. For example, if the Spark history server runs on the same node as the YARN ResourceManager, it can be set to <code>${hadoopconf-yarn.resourcemanager.hostname}:18080</code>.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.archives</code></td>
@@ -256,6 +269,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Comma separated list of archives to be extracted into the working directory of each executor.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.files</code></td>
@@ -263,6 +277,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Comma-separated list of files to be placed in the working directory of each executor.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.jars</code></td>
@@ -270,6 +285,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Comma-separated list of jars to be placed in the working directory of each executor.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.dist.forceDownloadSchemes</code></td>
@@ -280,6 +296,7 @@ To use a custom metrics.properties for the application master and executors, upd
     support schemes that are supported by Spark, like http, https and ftp, or jars required to be in the
     local YARN client's classpath. Wildcard '*' is denoted to download resources for all the schemes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.instances</code></td>
@@ -287,6 +304,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The number of executors for static allocation. With <code>spark.dynamicAllocation.enabled</code>, the initial set of executors will be at least this large.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.memoryOverhead</code></td>
@@ -294,6 +312,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Same as <code>spark.driver.memoryOverhead</code>, but for the YARN Application Master in client mode.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.queue</code></td>
@@ -301,6 +320,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The name of the YARN queue to which the application is submitted.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.jars</code></td>
@@ -312,6 +332,7 @@ To use a custom metrics.properties for the application master and executors, upd
     need to be distributed each time an application runs. To point to jars on HDFS, for example,
     set this configuration to <code>hdfs:///some/path</code>. Globs are allowed.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.archive</code></td>
@@ -323,6 +344,7 @@ To use a custom metrics.properties for the application master and executors, upd
     Like with the previous option, the archive can also be hosted on HDFS to speed up file
     distribution.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.appMasterEnv.[EnvironmentVariableName]</code></td>
@@ -334,6 +356,7 @@ To use a custom metrics.properties for the application master and executors, upd
      the environment of the Spark driver and in <code>client</code> mode it only controls
      the environment of the executor launcher.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.containerLauncherMaxThreads</code></td>
@@ -341,6 +364,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     The maximum number of threads to use in the YARN Application Master for launching executor containers.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.extraJavaOptions</code></td>
@@ -351,6 +375,7 @@ To use a custom metrics.properties for the application master and executors, upd
   to set maximum heap size (-Xmx) settings with this option. Maximum heap size settings can be set
   with <code>spark.yarn.am.memory</code>
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.extraLibraryPath</code></td>
@@ -358,6 +383,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
     Set a special library path to use when launching the YARN Application Master in client mode.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.maxAppAttempts</code></td>
@@ -366,6 +392,7 @@ To use a custom metrics.properties for the application master and executors, upd
   The maximum number of attempts that will be made to submit the application.
   It should be no larger than the global number of max attempts in the YARN configuration.
   </td>
+  <td>1.3.0</td> 
 </tr>
 <tr>
   <td><code>spark.yarn.am.attemptFailuresValidityInterval</code></td>
@@ -375,6 +402,7 @@ To use a custom metrics.properties for the application master and executors, upd
   If the AM has been running for at least the defined interval, the AM failure count will be reset.
   This feature is not enabled if not configured.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.executor.failuresValidityInterval</code></td>
@@ -383,6 +411,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Defines the validity interval for executor failure tracking.
   Executor failures which are older than the validity interval will be ignored.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.submit.waitAppCompletion</code></td>
@@ -392,6 +421,7 @@ To use a custom metrics.properties for the application master and executors, upd
   If set to <code>true</code>, the client process will stay alive reporting the application's status.
   Otherwise, the client process will exit after submission.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.am.nodeLabelExpression</code></td>
@@ -401,6 +431,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Only versions of YARN greater than or equal to 2.6 support node label expressions, so when
   running against earlier versions, this property will be ignored.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.executor.nodeLabelExpression</code></td>
@@ -410,6 +441,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Only versions of YARN greater than or equal to 2.6 support node label expressions, so when
   running against earlier versions, this property will be ignored.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.tags</code></td>
@@ -418,6 +450,7 @@ To use a custom metrics.properties for the application master and executors, upd
   Comma-separated list of strings to pass through as YARN application tags appearing
   in YARN ApplicationReports, which can be used for filtering when querying YARN apps.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.priority</code></td>
@@ -427,6 +460,7 @@ To use a custom metrics.properties for the application master and executors, upd
   integer value have a better opportunity to be activated. Currently, YARN only supports application
   priority when using FIFO ordering policy.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.config.gatewayPath</code></td>
@@ -446,6 +480,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <code>$HADOOP_HOME</code> will make sure that paths used to launch remote processes properly
   reference the local YARN configuration.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.config.replacementPath</code></td>
@@ -453,6 +488,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
   See <code>spark.yarn.config.gatewayPath</code>.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.rolledLog.includePattern</code></td>
@@ -467,6 +503,7 @@ To use a custom metrics.properties for the application master and executors, upd
   on the file name configured in the log4j configuration (like spark.log), the user should set the
   regex (spark*) to include all the log files that need to be aggregated.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.rolledLog.excludePattern</code></td>
@@ -476,6 +513,7 @@ To use a custom metrics.properties for the application master and executors, upd
   and those log files will not be aggregated in a rolling fashion. If the log file
   name matches both the include and the exclude pattern, this file will be excluded eventually.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.blacklist.executor.launch.blacklisting.enabled</code></td>
@@ -485,6 +523,7 @@ To use a custom metrics.properties for the application master and executors, upd
   The error limit for blacklisting can be configured by
   <code>spark.blacklist.application.maxFailedExecutorsPerNode</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.exclude.nodes</code></td>
@@ -492,6 +531,7 @@ To use a custom metrics.properties for the application master and executors, upd
   <td>
   Comma-separated list of YARN node names which are excluded from resource allocation.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.metrics.namespace</code></td>
@@ -500,6 +540,7 @@ To use a custom metrics.properties for the application master and executors, upd
   The root namespace for AM metrics reporting. 
   If it is not set then the YARN application ID is used.
   </td>
+  <td>2.4.0</td>
 </tr>
 </table>
 
@@ -583,7 +624,7 @@ staging directory of the Spark application.
 ## YARN-specific Kerberos Configuration
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.kerberos.keytab</code></td>
   <td>(none)</td>
@@ -595,6 +636,7 @@ staging directory of the Spark application.
 
   <br /> (Works also with the "local" master.)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kerberos.principal</code></td>
@@ -605,6 +647,7 @@ staging directory of the Spark application.
 
   <br /> (Works also with the "local" master.)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.yarn.kerberos.relogin.period</code></td>
@@ -614,6 +657,7 @@ staging directory of the Spark application.
   that is shorter than the TGT renewal period (or the TGT lifetime if TGT renewal is not enabled).
   The default value should be enough for most deployments.
   </td>
+  <td>2.3.0</td>
 </tr>
 </table>
 
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
index 8dfbef27be2fb..b3a3570f7107a 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -29,6 +29,7 @@ package object config {
   private[spark] val APPLICATION_TAGS = ConfigBuilder("spark.yarn.tags")
     .doc("Comma-separated list of strings to pass through as YARN application tags appearing " +
       "in YARN Application Reports, which can be used for filtering when querying YARN.")
+    .version("1.5.0")
     .stringConf
     .toSequence
     .createOptional
@@ -37,6 +38,7 @@ package object config {
     .doc("Application priority for YARN to define pending applications ordering policy, those" +
       " with higher value have a better opportunity to be activated. Currently, YARN only" +
       " supports application priority when using FIFO ordering policy.")
+    .version("3.0.0")
     .intConf
     .createOptional
 
@@ -44,6 +46,7 @@ package object config {
     ConfigBuilder("spark.yarn.am.attemptFailuresValidityInterval")
       .doc("Interval after which AM failures will be considered independent and " +
         "not accumulate towards the attempt count.")
+      .version("1.6.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
@@ -51,36 +54,43 @@ package object config {
     ConfigBuilder("spark.yarn.executor.failuresValidityInterval")
       .doc("Interval after which Executor failures will be considered independent and not " +
         "accumulate towards the attempt count.")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val MAX_APP_ATTEMPTS = ConfigBuilder("spark.yarn.maxAppAttempts")
     .doc("Maximum number of AM attempts before failing the app.")
+    .version("1.3.0")
     .intConf
     .createOptional
 
   private[spark] val USER_CLASS_PATH_FIRST = ConfigBuilder("spark.yarn.user.classpath.first")
     .doc("Whether to place user jars in front of Spark's classpath.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val GATEWAY_ROOT_PATH = ConfigBuilder("spark.yarn.config.gatewayPath")
     .doc("Root of configuration paths that is present on gateway nodes, and will be replaced " +
       "with the corresponding path in cluster machines.")
+    .version("1.5.0")
     .stringConf
     .createWithDefault(null)
 
   private[spark] val REPLACEMENT_ROOT_PATH = ConfigBuilder("spark.yarn.config.replacementPath")
     .doc(s"Path to use as a replacement for ${GATEWAY_ROOT_PATH.key} when launching processes " +
       "in the YARN cluster.")
+    .version("1.5.0")
     .stringConf
     .createWithDefault(null)
 
   private[spark] val QUEUE_NAME = ConfigBuilder("spark.yarn.queue")
+    .version("1.0.0")
     .stringConf
     .createWithDefault("default")
 
   private[spark] val HISTORY_SERVER_ADDRESS = ConfigBuilder("spark.yarn.historyServer.address")
+    .version("1.0.0")
     .stringConf
     .createOptional
 
@@ -88,6 +98,7 @@ package object config {
     ConfigBuilder("spark.yarn.historyServer.allowTracking")
       .doc("Allow using the History Server URL for the application as the tracking URL for the " +
         "application when the Web UI is not enabled.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -95,37 +106,44 @@ package object config {
 
   private[spark] val SPARK_ARCHIVE = ConfigBuilder("spark.yarn.archive")
     .doc("Location of archive containing jars files with Spark classes.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   private[spark] val SPARK_JARS = ConfigBuilder("spark.yarn.jars")
     .doc("Location of jars containing Spark classes.")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createOptional
 
   private[spark] val ARCHIVES_TO_DISTRIBUTE = ConfigBuilder("spark.yarn.dist.archives")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val FILES_TO_DISTRIBUTE = ConfigBuilder("spark.yarn.dist.files")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val JARS_TO_DISTRIBUTE = ConfigBuilder("spark.yarn.dist.jars")
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val PRESERVE_STAGING_FILES = ConfigBuilder("spark.yarn.preserve.staging.files")
     .doc("Whether to preserve temporary files created by the job in HDFS.")
+    .version("1.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val STAGING_FILE_REPLICATION = ConfigBuilder("spark.yarn.submit.file.replication")
     .doc("Replication factor for files uploaded by Spark to HDFS.")
+    .version("0.8.1")
     .intConf
     .createOptional
 
@@ -134,93 +152,111 @@ package object config {
   private[spark] val WAIT_FOR_APP_COMPLETION = ConfigBuilder("spark.yarn.submit.waitAppCompletion")
     .doc("In cluster mode, whether to wait for the application to finish before exiting the " +
       "launcher process.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(true)
 
   private[spark] val REPORT_INTERVAL = ConfigBuilder("spark.yarn.report.interval")
     .doc("Interval between reports of the current app status.")
+    .version("0.9.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("1s")
 
   private[spark] val CLIENT_LAUNCH_MONITOR_INTERVAL =
     ConfigBuilder("spark.yarn.clientLaunchMonitorInterval")
       .doc("Interval between requests for status the client mode AM when starting the app.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1s")
 
   /* Shared Client-mode AM / Driver configuration. */
 
   private[spark] val AM_MAX_WAIT_TIME = ConfigBuilder("spark.yarn.am.waitTime")
+    .version("1.3.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("100s")
 
   private[spark] val YARN_METRICS_NAMESPACE = ConfigBuilder("spark.yarn.metrics.namespace")
     .doc("The root namespace for AM metrics reporting.")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   private[spark] val AM_NODE_LABEL_EXPRESSION = ConfigBuilder("spark.yarn.am.nodeLabelExpression")
     .doc("Node label expression for the AM.")
+    .version("1.6.0")
     .stringConf
     .createOptional
 
   private[spark] val CONTAINER_LAUNCH_MAX_THREADS =
     ConfigBuilder("spark.yarn.containerLauncherMaxThreads")
+      .version("1.2.0")
       .intConf
       .createWithDefault(25)
 
   private[spark] val MAX_EXECUTOR_FAILURES = ConfigBuilder("spark.yarn.max.executor.failures")
+    .version("1.0.0")
     .intConf
     .createOptional
 
   private[spark] val MAX_REPORTER_THREAD_FAILURES =
     ConfigBuilder("spark.yarn.scheduler.reporterThread.maxFailures")
+      .version("1.2.0")
       .intConf
       .createWithDefault(5)
 
   private[spark] val RM_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.yarn.scheduler.heartbeat.interval-ms")
+      .version("0.8.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("3s")
 
   private[spark] val INITIAL_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.yarn.scheduler.initial-allocation.interval")
+      .version("1.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("200ms")
 
   private[spark] val AM_FINAL_MSG_LIMIT = ConfigBuilder("spark.yarn.am.finalMessageLimit")
     .doc("The limit size of final diagnostic message for our ApplicationMaster to unregister from" +
       " the ResourceManager.")
+    .version("2.4.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("1m")
 
   /* Client-mode AM configuration. */
 
   private[spark] val AM_CORES = ConfigBuilder("spark.yarn.am.cores")
+    .version("1.3.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val AM_JAVA_OPTIONS = ConfigBuilder("spark.yarn.am.extraJavaOptions")
     .doc("Extra Java options for the client-mode AM.")
+    .version("1.3.0")
     .stringConf
     .createOptional
 
   private[spark] val AM_LIBRARY_PATH = ConfigBuilder("spark.yarn.am.extraLibraryPath")
     .doc("Extra native library path for the client-mode AM.")
+    .version("1.4.0")
     .stringConf
     .createOptional
 
   private[spark] val AM_MEMORY_OVERHEAD = ConfigBuilder("spark.yarn.am.memoryOverhead")
+    .version("1.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
   private[spark] val AM_MEMORY = ConfigBuilder("spark.yarn.am.memory")
+    .version("1.3.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("512m")
 
   /* Driver configuration. */
 
   private[spark] val DRIVER_APP_UI_ADDRESS = ConfigBuilder("spark.driver.appUIAddress")
+    .version("1.1.0")
     .stringConf
     .createOptional
 
@@ -229,6 +265,7 @@ package object config {
   private[spark] val EXECUTOR_NODE_LABEL_EXPRESSION =
     ConfigBuilder("spark.yarn.executor.nodeLabelExpression")
       .doc("Node label expression for executors.")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
@@ -237,6 +274,7 @@ package object config {
   private[spark] val YARN_UNMANAGED_AM = ConfigBuilder("spark.yarn.unmanagedAM.enabled")
     .doc("In client mode, whether to launch the Application Master service as part of the client " +
       "using unmanaged am.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -246,6 +284,7 @@ package object config {
     ConfigBuilder("spark.yarn.rolledLog.includePattern")
       .doc("Java Regex to filter the log files which match the defined include pattern and those " +
         "log files will be aggregated in a rolling fashion.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -253,6 +292,7 @@ package object config {
     ConfigBuilder("spark.yarn.rolledLog.excludePattern")
       .doc("Java Regex to filter the log files which match the defined exclude pattern and those " +
         "log files will not be aggregated in a rolling fashion.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -261,6 +301,7 @@ package object config {
   // Internal config to propagate the location of the user's jar to the driver/executors
   private[spark] val APP_JAR = ConfigBuilder("spark.yarn.user.jar")
     .internal()
+    .version("1.1.0")
     .stringConf
     .createOptional
 
@@ -268,6 +309,7 @@ package object config {
   // of the executors
   private[spark] val SECONDARY_JARS = ConfigBuilder("spark.yarn.secondary.jars")
     .internal()
+    .version("0.9.2")
     .stringConf
     .toSequence
     .createOptional
@@ -276,24 +318,28 @@ package object config {
 
   private[spark] val CACHED_FILES = ConfigBuilder("spark.yarn.cache.filenames")
     .internal()
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val CACHED_FILES_SIZES = ConfigBuilder("spark.yarn.cache.sizes")
     .internal()
+    .version("2.0.0")
     .longConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val CACHED_FILES_TIMESTAMPS = ConfigBuilder("spark.yarn.cache.timestamps")
     .internal()
+    .version("2.0.0")
     .longConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val CACHED_FILES_VISIBILITIES = ConfigBuilder("spark.yarn.cache.visibilities")
     .internal()
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
@@ -301,6 +347,7 @@ package object config {
   // Either "file" or "archive", for each file.
   private[spark] val CACHED_FILES_TYPES = ConfigBuilder("spark.yarn.cache.types")
     .internal()
+    .version("2.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
@@ -308,20 +355,23 @@ package object config {
   // The location of the conf archive in HDFS.
   private[spark] val CACHED_CONF_ARCHIVE = ConfigBuilder("spark.yarn.cache.confArchive")
     .internal()
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   /* YARN allocator-level blacklisting related config entries. */
   private[spark] val YARN_EXECUTOR_LAUNCH_BLACKLIST_ENABLED =
     ConfigBuilder("spark.yarn.blacklist.executor.launch.blacklisting.enabled")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
   /* Initially blacklisted YARN nodes. */
   private[spark] val YARN_EXCLUDE_NODES = ConfigBuilder("spark.yarn.exclude.nodes")
-      .stringConf
-      .toSequence
-      .createWithDefault(Nil)
+    .version("3.0.0")
+    .stringConf
+    .toSequence
+    .createWithDefault(Nil)
 
   private[yarn] val YARN_EXECUTOR_RESOURCE_TYPES_PREFIX = "spark.yarn.executor.resource."
   private[yarn] val YARN_DRIVER_RESOURCE_TYPES_PREFIX = "spark.yarn.driver.resource."

From d8b86fd5fb491c3e188fb9b30ca1f4b6a8384188 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 12 Mar 2020 11:02:29 +0900
Subject: [PATCH 0435/1280] [SPARK-31109][MESOS][DOC] Add version information
 to the configuration of Mesos
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the configuration of `Mesos`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.mesos.$taskType.secret.names | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.$taskType.secret.values | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.$taskType.secret.envkeys | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.$taskType.secret.filenames | 2.3.0 | SPARK-22131 | 5415963d2caaf95604211419ffc4e29fff38e1d7#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.principal | 1.5.0 | SPARK-6284 | d86bbb4e286f16f77ba125452b07827684eafeed#diff-02a6d899f7a529eb7cfbb12182a110b0 |  
spark.mesos.principal.file | 2.4.0 | SPARK-16501 | 7f10cf83f311526737fc96d5bb8281d12e41932f#diff-daf48dabbe58afaeed8787751750b01d |  
spark.mesos.secret | 1.5.0 | SPARK-6284 | d86bbb4e286f16f77ba125452b07827684eafeed#diff-02a6d899f7a529eb7cfbb12182a110b0 |  
spark.mesos.secret.file | 2.4.0 | SPARK-16501 | 7f10cf83f311526737fc96d5bb8281d12e41932f#diff-daf48dabbe58afaeed8787751750b01d |  
spark.shuffle.cleaner.interval | 2.0.0 | SPARK-12583 | 310981d49a332bd329303f610b150bbe02cf5f87#diff-2fafefee94f2a2023ea9765536870258 |  
spark.mesos.dispatcher.webui.url | 2.0.0 | SPARK-13492 | a4a0addccffb7cd0ece7947d55ce2538afa54c97#diff-f541460c7a74cee87cbb460b3b01665e |  
spark.mesos.dispatcher.historyServer.url | 2.1.0 | SPARK-16809 | 62e62124419f3fa07b324f5e42feb2c5b4fde715#diff-3779e2035d9a09fa5f6af903925b9512 |  
spark.mesos.driver.labels | 2.3.0 | SPARK-21000 | 8da3f7041aafa71d7596b531625edb899970fec2#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.driver.webui.url | 2.0.0 | SPARK-13492 | a4a0addccffb7cd0ece7947d55ce2538afa54c97#diff-e3a5e67b8de2069ce99801372e214b8e |  
spark.mesos.driver.failoverTimeout | 2.3.0 | SPARK-21456 | c42ef953343073a50ef04c5ce848b574ff7f2238#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.network.name | 2.1.0 | SPARK-18232 | d89bfc92302424406847ac7a9cfca714e6b742fc#diff-ab5bf34f1951a8f7ea83c9456a6c3ab7 |  
spark.mesos.network.labels | 2.3.0 | SPARK-21694 | ce0d3bb377766bdf4df7852272557ae846408877#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.driver.constraints | 2.2.1 | SPARK-19606 | f6ee3d90d5c299e67ae6e2d553c16c0d9759d4b5#diff-91e6e5f871160782dc50d4060d6faea3 |  
spark.mesos.driver.frameworkId | 2.1.0 | SPARK-16809 | 62e62124419f3fa07b324f5e42feb2c5b4fde715#diff-02a6d899f7a529eb7cfbb12182a110b0 |  
spark.executor.uri | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-a885e7df97790e9b59c21c63353e7476 |  
spark.mesos.proxy.baseURL | 2.3.0 | SPARK-13041 | 663f30d14a0c9219e07697af1ab56e11a714d9a6#diff-0b9b4e122eb666155aa189a4321a6ca8 |  
spark.mesos.coarse | 0.6.0 | None | 63051dd2bcc4bf09d413ff7cf89a37967edc33ba#diff-eaf125f56ce786d64dcef99cf446a751 |  
spark.mesos.coarse.shutdownTimeout | 2.0.0 | SPARK-12330 | c756bda477f458ba4aad7fdb2026263507e0ad9b#diff-d425d35aa23c47a62fbb538554f2f2cf |  
spark.mesos.maxDrivers | 1.4.0 | SPARK-5338 | 53befacced828bbac53c6e3a4976ec3f036bae9e#diff-b964c449b99c51f0a5fd77270b2951a4 |  
spark.mesos.retainedDrivers | 1.4.0 | SPARK-5338 | 53befacced828bbac53c6e3a4976ec3f036bae9e#diff-b964c449b99c51f0a5fd77270b2951a4 |  
spark.mesos.cluster.retry.wait.max | 1.4.0 | SPARK-5338 | 53befacced828bbac53c6e3a4976ec3f036bae9e#diff-b964c449b99c51f0a5fd77270b2951a4 |  
spark.mesos.fetcherCache.enable | 2.1.0 | SPARK-15994 | e34b4e12673fb76c92f661d7c03527410857a0f8#diff-772ea7311566edb25f11a4c4f882179a |  
spark.mesos.appJar.local.resolution.mode | 2.4.0 | SPARK-24326 | 22df953f6bb191858053eafbabaa5b3ebca29f56#diff-6e4d0a0445975f03f975fdc1e3d80e49 |  
spark.mesos.rejectOfferDuration | 2.2.0 | SPARK-19702 | 2e30c0b9bcaa6f7757bd85d1f1ec392d5f916f83#diff-daf48dabbe58afaeed8787751750b01d |  
spark.mesos.rejectOfferDurationForUnmetConstraints | 1.6.0 | SPARK-10471 | 74f50275e429e649212928a9f36552941b862edc#diff-02a6d899f7a529eb7cfbb12182a110b0 |  
spark.mesos.rejectOfferDurationForReachedMaxCores | 2.0.0 | SPARK-13001 | 1e7d9bfb5a41f5c2479ab3b4d4081f00bf00bd31#diff-02a6d899f7a529eb7cfbb12182a110b0 |  
spark.mesos.uris | 1.5.0 | SPARK-8798 | a2f805729b401c68b60bd690ad02533b8db57b58#diff-e3a5e67b8de2069ce99801372e214b8e |  
spark.mesos.executor.home | 1.1.1 | SPARK-3264 | 069ecfef02c4af69fc0d3755bd78be321b68b01d#diff-e3a5e67b8de2069ce99801372e214b8e |  
spark.mesos.mesosExecutor.cores | 1.4.0 | SPARK-6350 | 6fbeb82e13db7117d8f216e6148632490a4bc5be#diff-e3a5e67b8de2069ce99801372e214b8e |  
spark.mesos.extra.cores | 0.6.0 | None | 2d761e3353651049f6707c74bb5ffdd6e86f6f35#diff-37af8c6e3634f97410ade813a5172621 |  
spark.mesos.executor.memoryOverhead | 1.1.1 | SPARK-3535 | 6f150978477830bbc14ba983786dd2bce12d1fe2#diff-6b498f5407d10e848acac4a1b182457c |  
spark.mesos.executor.docker.image | 1.4.0 | SPARK-2691 | 8f50a07d2188ccc5315d979755188b1e5d5b5471#diff-e3a5e67b8de2069ce99801372e214b8e |  
spark.mesos.executor.docker.forcePullImage | 2.1.0 | SPARK-15271 | 978cd5f125eb5a410bad2e60bf8385b11cf1b978#diff-0dd025320c7ecda2ea310ed7172d7f5a |  
spark.mesos.executor.docker.portmaps | 1.4.0 | SPARK-7373 | 226033cfffa2f37ebaf8bc2c653f094e91ef0c9b#diff-b964c449b99c51f0a5fd77270b2951a4 |  
spark.mesos.executor.docker.parameters | 2.2.0 | SPARK-19740 | a888fed3099e84c2cf45e9419f684a3658ada19d#diff-4139e6605a8c7f242f65cde538770c99 |  
spark.mesos.executor.docker.volumes | 1.4.0 | SPARK-7373 | 226033cfffa2f37ebaf8bc2c653f094e91ef0c9b#diff-b964c449b99c51f0a5fd77270b2951a4 |  
spark.mesos.gpus.max | 2.1.0 | SPARK-14082 | 29f186bfdf929b1e8ffd8e33ee37b76d5dc5af53#diff-d427ee890b913c5a7056be21eb4f39d7 |  
spark.mesos.task.labels | 2.2.0 | SPARK-20085 | c8fc1f3badf61bcfc4bd8eeeb61f73078ca068d1#diff-387c5d0c916278495fc28420571adf9e |  
spark.mesos.constraints | 1.5.0 | SPARK-6707 | 1165b17d24cdf1dbebb2faca14308dfe5c2a652c#diff-e3a5e67b8de2069ce99801372e214b8e |  
spark.mesos.containerizer | 2.1.0 | SPARK-16637 | 266b92faffb66af24d8ed2725beb80770a2d91f8#diff-0dd025320c7ecda2ea310ed7172d7f5a |  
spark.mesos.role | 1.5.0 | SPARK-6284 | d86bbb4e286f16f77ba125452b07827684eafeed#diff-02a6d899f7a529eb7cfbb12182a110b0 |  
The following appears in the document |   |   |   |  
spark.mesos.driverEnv.[EnvironmentVariableName] | 2.1.0 | SPARK-16194 | 235cb256d06653bcde4c3ed6b081503a94996321#diff-b964c449b99c51f0a5fd77270b2951a4 |  
spark.mesos.dispatcher.driverDefault.[PropertyName] | 2.1.0 | SPARK-16927 and SPARK-16923 | eca58755fbbc11937b335ad953a3caff89b818e6#diff-b964c449b99c51f0a5fd77270b2951a4 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #27863 from beliefer/add-version-to-mesos-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/running-on-mesos.md                      | 47 ++++++++++++++---
 .../apache/spark/deploy/mesos/config.scala    | 52 +++++++++++++++++--
 2 files changed, 87 insertions(+), 12 deletions(-)

diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md
index cf51620a700bc..6f6ae1c0ff264 100644
--- a/docs/running-on-mesos.md
+++ b/docs/running-on-mesos.md
@@ -371,7 +371,7 @@ See the [configuration page](configuration.html) for information on Spark config
 #### Spark Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.mesos.coarse</code></td>
   <td>true</td>
@@ -380,6 +380,7 @@ See the [configuration page](configuration.html) for information on Spark config
     If set to <code>false</code>, runs over Mesos cluster in "fine-grained" sharing mode, where one Mesos task is created per Spark task.
     Detailed information in <a href="running-on-mesos.html#mesos-run-modes">'Mesos Run Modes'</a>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.extra.cores</code></td>
@@ -391,6 +392,7 @@ See the [configuration page](configuration.html) for information on Spark config
     send it more tasks.  Use this to increase parallelism.  This
     setting is only used for Mesos coarse-grained mode.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.mesosExecutor.cores</code></td>
@@ -401,6 +403,7 @@ See the [configuration page](configuration.html) for information on Spark config
     is being run, each Mesos executor will occupy the number of cores configured here.
     The value can be a floating point number.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.image</code></td>
@@ -411,6 +414,7 @@ See the [configuration page](configuration.html) for information on Spark config
     The installed path of Spark in the image can be specified with <code>spark.mesos.executor.home</code>;
     the installed path of the Mesos library can be specified with <code>spark.executorEnv.MESOS_NATIVE_JAVA_LIBRARY</code>.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.forcePullImage</code></td>
@@ -419,6 +423,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Force Mesos agents to pull the image specified in <code>spark.mesos.executor.docker.image</code>.
     By default Mesos agents will not pull images they already have cached.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.parameters</code></td>
@@ -429,6 +434,7 @@ See the [configuration page](configuration.html) for information on Spark config
 
     <pre>key1=val1,key2=val2,key3=val3</pre>
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.docker.volumes</code></td>
@@ -440,6 +446,7 @@ See the [configuration page](configuration.html) for information on Spark config
 
     <pre>[host_path:]container_path[:ro|:rw]</pre>
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.task.labels</code></td>
@@ -450,6 +457,7 @@ See the [configuration page](configuration.html) for information on Spark config
     list more than one.  If your label includes a colon or comma, you
     can escape it with a backslash.  Ex. key:value,key2:a\:b.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.home</code></td>
@@ -460,6 +468,7 @@ See the [configuration page](configuration.html) for information on Spark config
     them. Note that this is only relevant if a Spark binary package is not specified through
     <code>spark.executor.uri</code>.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.mesos.executor.memoryOverhead</code></td>
@@ -469,6 +478,7 @@ See the [configuration page](configuration.html) for information on Spark config
     the overhead will be larger of either 384 or 10% of <code>spark.executor.memory</code>. If set,
     the final overhead will be this value.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.mesos.uris</code></td>
@@ -478,6 +488,7 @@ See the [configuration page](configuration.html) for information on Spark config
     when driver or executor is launched by Mesos.  This applies to
     both coarse-grained and fine-grained mode.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.principal</code></td>
@@ -485,6 +496,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Set the principal with which Spark framework will use to authenticate with Mesos.  You can also specify this via the environment variable `SPARK_MESOS_PRINCIPAL`.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.principal.file</code></td>
@@ -492,6 +504,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Set the file containing the principal with which Spark framework will use to authenticate with Mesos.  Allows specifying the principal indirectly in more security conscious deployments.  The file must be readable by the user launching the job and be UTF-8 encoded plaintext.  You can also specify this via the environment variable `SPARK_MESOS_PRINCIPAL_FILE`.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.secret</code></td>
@@ -500,6 +513,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
     authenticating with the registry.  You can also specify this via the environment variable `SPARK_MESOS_SECRET`.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.secret.file</code></td>
@@ -508,6 +522,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the file containing the secret with which Spark framework will use to authenticate with Mesos. Used, for example, when
     authenticating with the registry.  Allows for specifying the secret indirectly in more security conscious deployments.  The file must be readable by the user launching the job and be UTF-8 encoded plaintext.  You can also specify this via the environment variable `SPARK_MESOS_SECRET_FILE`.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.role</code></td>
@@ -516,6 +531,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the role of this Spark framework for Mesos. Roles are used in Mesos for reservations
     and resource weight sharing.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.constraints</code></td>
@@ -532,6 +548,7 @@ See the [configuration page](configuration.html) for information on Spark config
       <li>In case there is no value present as a part of the constraint any offer with the corresponding attribute will be accepted (without value check).</li>
     </ul>
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.constraints</code></td>
@@ -540,6 +557,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Same as <code>spark.mesos.constraints</code> except applied to drivers when launched through the dispatcher. By default,
     all offers with sufficient resources will be accepted.
   </td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td><code>spark.mesos.containerizer</code></td>
@@ -550,6 +568,7 @@ See the [configuration page](configuration.html) for information on Spark config
     containerizers for docker: the "docker" containerizer, and the preferred
     "mesos" containerizer.  Read more here: http://mesos.apache.org/documentation/latest/container-image/
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.webui.url</code></td>
@@ -558,6 +577,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the Spark Mesos driver webui_url for interacting with the framework.
     If unset it will point to Spark's internal web UI.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.labels</code></td>
@@ -566,8 +586,8 @@ See the [configuration page](configuration.html) for information on Spark config
     Mesos labels to add to the driver.  See <code>spark.mesos.task.labels</code>
     for formatting information.
   </td>
+  <td>2.3.0</td>
 </tr>
-
 <tr>
   <td>
     <code>spark.mesos.driver.secret.values</code>,
@@ -616,8 +636,8 @@ See the [configuration page](configuration.html) for information on Spark config
       <pre>spark.mesos.driver.secret.names=password1,password2</pre>
     </p>
   </td>
+  <td>2.3.0</td>
 </tr>
-
 <tr>
   <td>
     <code>spark.mesos.driver.secret.envkeys</code>,
@@ -670,8 +690,8 @@ See the [configuration page](configuration.html) for information on Spark config
       <pre>spark.mesos.driver.secret.filenames=pwdfile1,pwdfile2</pre>
     </p>
   </td>
+  <td>2.3.0</td>
 </tr>
-
 <tr>
   <td><code>spark.mesos.driverEnv.[EnvironmentVariableName]</code></td>
   <td><code>(none)</code></td>
@@ -681,6 +701,7 @@ See the [configuration page](configuration.html) for information on Spark config
     driver process. The user can specify multiple of these to set
     multiple environment variables.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.dispatcher.webui.url</code></td>
@@ -689,7 +710,8 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the Spark Mesos dispatcher webui_url for interacting with the framework.
     If unset it will point to Spark's internal web UI.
   </td>
-  </tr>
+  <td>2.0.0</td>
+</tr>
 <tr>
   <td><code>spark.mesos.dispatcher.driverDefault.[PropertyName]</code></td>
   <td><code>(none)</code></td>
@@ -699,7 +721,8 @@ See the [configuration page](configuration.html) for information on Spark config
     spark.mesos.dispatcher.driverProperty.spark.executor.memory=32g
     results in the executors for all drivers submitted in cluster mode
     to run in 32g containers.
-</td>
+  </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.dispatcher.historyServer.url</code></td>
@@ -709,6 +732,7 @@ See the [configuration page](configuration.html) for information on Spark config
     server</a>.  The dispatcher will then link each driver to its entry
     in the history server.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.gpus.max</code></td>
@@ -717,7 +741,8 @@ See the [configuration page](configuration.html) for information on Spark config
     Set the maximum number GPU resources to acquire for this job. Note that executors will still launch when no GPU resources are found
     since this configuration is just an upper limit and not a guaranteed amount.
   </td>
-  </tr>
+  <td>2.1.0</td>
+</tr>
 <tr>
   <td><code>spark.mesos.network.name</code></td>
   <td><code>(none)</code></td>
@@ -728,6 +753,7 @@ See the [configuration page](configuration.html) for information on Spark config
     <a href="http://mesos.apache.org/documentation/latest/cni/">the Mesos CNI docs</a>
     for more details.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.network.labels</code></td>
@@ -742,6 +768,7 @@ See the [configuration page](configuration.html) for information on Spark config
     <a href="http://mesos.apache.org/documentation/latest/cni/#mesos-meta-data-to-cni-plugins">the Mesos CNI docs</a>
     for more details.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.fetcherCache.enable</code></td>
@@ -752,6 +779,7 @@ See the [configuration page](configuration.html) for information on Spark config
     href="http://mesos.apache.org/documentation/latest/fetcher/">Mesos
     Fetcher Cache</a>
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.driver.failoverTimeout</code></td>
@@ -763,6 +791,7 @@ See the [configuration page](configuration.html) for information on Spark config
     executors. The default value is zero, meaning no timeout: if the 
     driver disconnects, the master immediately tears down the framework.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.rejectOfferDuration</code></td>
@@ -772,6 +801,7 @@ See the [configuration page](configuration.html) for information on Spark config
     `spark.mesos.rejectOfferDurationForUnmetConstraints`,
     `spark.mesos.rejectOfferDurationForReachedMaxCores`
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.rejectOfferDurationForUnmetConstraints</code></td>
@@ -779,6 +809,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Time to consider unused resources refused with unmet constraints
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.rejectOfferDurationForReachedMaxCores</code></td>
@@ -787,6 +818,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Time to consider unused resources refused when maximum number of cores
     <code>spark.cores.max</code> is reached
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.mesos.appJar.local.resolution.mode</code></td>
@@ -799,6 +831,7 @@ See the [configuration page](configuration.html) for information on Spark config
     If the value is `container` then spark submit in the container will use the jar in the container's path:
     `/path/to/jar`.
   </td>
+  <td>2.4.0</td>
 </tr>
 </table>
 
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
index 79a113792de91..e1c0d18b73a2b 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/deploy/mesos/config.scala
@@ -28,6 +28,7 @@ package object config {
       ConfigBuilder(s"spark.mesos.$taskType.secret.names")
         .doc("A comma-separated list of secret reference names. Consult the Mesos Secret " +
           "protobuf for more information.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -35,6 +36,7 @@ package object config {
     private[spark] val SECRET_VALUES =
       ConfigBuilder(s"spark.mesos.$taskType.secret.values")
         .doc("A comma-separated list of secret values.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -43,6 +45,7 @@ package object config {
       ConfigBuilder(s"spark.mesos.$taskType.secret.envkeys")
         .doc("A comma-separated list of the environment variables to contain the secrets." +
           "The environment variable will be set on the driver.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -51,6 +54,7 @@ package object config {
       ConfigBuilder(s"spark.mesos.$taskType.secret.filenames")
         .doc("A comma-separated list of file paths secret will be written to.  Consult the Mesos " +
           "Secret protobuf for more information.")
+        .version("2.3.0")
         .stringConf
         .toSequence
         .createOptional
@@ -59,6 +63,7 @@ package object config {
   private[spark] val CREDENTIAL_PRINCIPAL =
     ConfigBuilder("spark.mesos.principal")
       .doc("Name of the Kerberos principal to authenticate Spark to Mesos.")
+      .version("1.5.0")
       .stringConf
       .createOptional
 
@@ -66,18 +71,21 @@ package object config {
     ConfigBuilder("spark.mesos.principal.file")
       .doc("The path of file which contains the name of the Kerberos principal " +
         "to authenticate Spark to Mesos.")
+      .version("2.4.0")
       .stringConf
       .createOptional
 
   private[spark] val CREDENTIAL_SECRET =
     ConfigBuilder("spark.mesos.secret")
       .doc("The secret value to authenticate Spark to Mesos.")
+      .version("1.5.0")
       .stringConf
       .createOptional
 
   private[spark] val CREDENTIAL_SECRET_FILE =
     ConfigBuilder("spark.mesos.secret.file")
       .doc("The path of file which contains the secret value to authenticate Spark to Mesos.")
+      .version("2.4.0")
       .stringConf
       .createOptional
 
@@ -85,6 +93,7 @@ package object config {
 
   private[spark] val SHUFFLE_CLEANER_INTERVAL_S =
     ConfigBuilder("spark.shuffle.cleaner.interval")
+      .version("2.0.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("30s")
 
@@ -92,6 +101,7 @@ package object config {
     ConfigBuilder("spark.mesos.dispatcher.webui.url")
       .doc("Set the Spark Mesos dispatcher webui_url for interacting with the " +
         "framework. If unset it will point to Spark's internal web UI.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -99,6 +109,7 @@ package object config {
     ConfigBuilder("spark.mesos.dispatcher.historyServer.url")
       .doc("Set the URL of the history server. The dispatcher will then " +
         "link each driver to its entry in the history server.")
+      .version("2.1.0")
       .stringConf
       .createOptional
 
@@ -107,6 +118,7 @@ package object config {
       .doc("Mesos labels to add to the driver.  Labels are free-form key-value pairs. Key-value " +
         "pairs should be separated by a colon, and commas used to list more than one." +
         "Ex. key:value,key2:value2")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
@@ -114,6 +126,7 @@ package object config {
     ConfigBuilder("spark.mesos.driver.webui.url")
       .doc("Set the Spark Mesos driver webui_url for interacting with the framework. " +
         "If unset it will point to Spark's internal web UI.")
+      .version("2.0.0")
       .stringConf
       .createOptional
 
@@ -125,6 +138,7 @@ package object config {
     ConfigBuilder("spark.mesos.driver.failoverTimeout")
       .doc("Amount of time in seconds that the master will wait to hear from the driver, " +
           "during a temporary disconnection, before tearing down all the executors.")
+      .version("2.3.0")
       .doubleConf
       .createWithDefault(0.0)
 
@@ -132,6 +146,7 @@ package object config {
     ConfigBuilder("spark.mesos.network.name")
       .doc("Attach containers to the given named network. If this job is launched " +
         "in cluster mode, also launch the driver in the given named network.")
+      .version("2.1.0")
       .stringConf
       .createOptional
 
@@ -140,6 +155,7 @@ package object config {
       .doc("Network labels to pass to CNI plugins.  This is a comma-separated list " +
         "of key-value pairs, where each key-value pair has the format key:value. " +
         "Example: key1:val1,key2:val2")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
@@ -147,19 +163,21 @@ package object config {
     ConfigBuilder("spark.mesos.driver.constraints")
       .doc("Attribute based constraints on mesos resource offers. Applied by the dispatcher " +
         "when launching drivers. Default is to accept all offers with sufficient resources.")
+      .version("2.2.1")
       .stringConf
       .createWithDefault("")
 
   private[spark] val DRIVER_FRAMEWORK_ID =
     ConfigBuilder("spark.mesos.driver.frameworkId")
+      .version("2.1.0")
       .stringConf
       .createOptional
 
   private[spark] val EXECUTOR_URI =
-    ConfigBuilder("spark.executor.uri").stringConf.createOptional
+    ConfigBuilder("spark.executor.uri").version("0.8.0").stringConf.createOptional
 
   private[spark] val PROXY_BASE_URL =
-    ConfigBuilder("spark.mesos.proxy.baseURL").stringConf.createOptional
+    ConfigBuilder("spark.mesos.proxy.baseURL").version("2.3.0").stringConf.createOptional
 
   private[spark] val COARSE_MODE =
     ConfigBuilder("spark.mesos.coarse")
@@ -167,22 +185,26 @@ package object config {
         "Spark acquires one long-lived Mesos task on each machine. If set to false, runs over " +
         "Mesos cluster in \"fine-grained\" sharing mode, where one Mesos task is created per " +
         "Spark task.")
-      .booleanConf.createWithDefault(true)
+      .version("0.6.0")
+      .booleanConf
+      .createWithDefault(true)
 
   private[spark] val COARSE_SHUTDOWN_TIMEOUT =
     ConfigBuilder("spark.mesos.coarse.shutdownTimeout")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0, s"spark.mesos.coarse.shutdownTimeout must be >= 0")
       .createWithDefaultString("10s")
 
   private[spark] val MAX_DRIVERS =
-    ConfigBuilder("spark.mesos.maxDrivers").intConf.createWithDefault(200)
+    ConfigBuilder("spark.mesos.maxDrivers").version("1.4.0").intConf.createWithDefault(200)
 
   private[spark] val RETAINED_DRIVERS =
-    ConfigBuilder("spark.mesos.retainedDrivers").intConf.createWithDefault(200)
+    ConfigBuilder("spark.mesos.retainedDrivers").version("1.4.0").intConf.createWithDefault(200)
 
   private[spark] val CLUSTER_RETRY_WAIT_MAX_SECONDS =
     ConfigBuilder("spark.mesos.cluster.retry.wait.max")
+      .version("1.4.0")
       .intConf
       .createWithDefault(60) // 1 minute
 
@@ -190,6 +212,7 @@ package object config {
     ConfigBuilder("spark.mesos.fetcherCache.enable")
       .doc("If set to true, all URIs (example: `spark.executor.uri`, `spark.mesos.uris`) will be " +
         "cached by the Mesos Fetcher Cache.")
+      .version("2.1.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -201,6 +224,7 @@ package object config {
         "resource from the host's file system. If the value is unknown it prints a warning msg " +
         "in the dispatcher logs and defaults to `host`. If the value is `container` then spark " +
         "submit in the container will use the jar in the container's path: `/path/to/jar`.")
+      .version("2.4.0")
       .stringConf
       .checkValues(Set("host", "container"))
       .createWithDefault("host")
@@ -210,12 +234,14 @@ package object config {
       .doc("Time to consider unused resources refused, serves as a fallback of " +
         "`spark.mesos.rejectOfferDurationForUnmetConstraints`, " +
         "`spark.mesos.rejectOfferDurationForReachedMaxCores`.")
+      .version("2.2.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("120s")
 
   private[spark] val REJECT_OFFER_DURATION_FOR_UNMET_CONSTRAINTS =
     ConfigBuilder("spark.mesos.rejectOfferDurationForUnmetConstraints")
       .doc("Time to consider unused resources refused with unmet constraints.")
+      .version("1.6.0")
       .timeConf(TimeUnit.SECONDS)
       .createOptional
 
@@ -223,6 +249,7 @@ package object config {
     ConfigBuilder("spark.mesos.rejectOfferDurationForReachedMaxCores")
       .doc("Time to consider unused resources refused when maximum number of cores " +
         "`spark.cores.max` is reached.")
+      .version("2.0.0")
       .timeConf(TimeUnit.SECONDS)
       .createOptional
 
@@ -231,6 +258,7 @@ package object config {
       .doc("A comma-separated list of URIs to be downloaded to the sandbox when driver or " +
         "executor is launched by Mesos. This applies to both coarse-grained and fine-grained " +
         "mode.")
+      .version("1.5.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -241,6 +269,7 @@ package object config {
         "By default, the executors will simply use the driver's Spark home directory, which may " +
         "not be visible to them. Note that this is only relevant if a Spark binary package is " +
         "not specified through `spark.executor.uri`.")
+      .version("1.1.1")
       .stringConf
       .createOptional
 
@@ -250,6 +279,7 @@ package object config {
         "include the cores used to run the Spark tasks. In other words, even if no Spark task " +
         "is being run, each Mesos executor will occupy the number of cores configured here. " +
         "The value can be a floating point number.")
+      .version("1.4.0")
       .doubleConf
       .createWithDefault(1.0)
 
@@ -259,6 +289,7 @@ package object config {
         "more cores allocated. It instead means that an executor will \"pretend\" it has more " +
         "cores, so that the driver will send it more tasks. Use this to increase parallelism. " +
         "This setting is only used for Mesos coarse-grained mode.")
+      .version("0.6.0")
       .intConf
       .createWithDefault(0)
 
@@ -267,6 +298,7 @@ package object config {
       .doc("The amount of additional memory, specified in MiB, to be allocated per executor. " +
         "By default, the overhead will be larger of either 384 or 10% of " +
         "`spark.executor.memory`. If set, the final overhead will be this value.")
+      .version("1.1.1")
       .intConf
       .createOptional
 
@@ -277,6 +309,7 @@ package object config {
         "The installed path of Spark in the image can be specified with " +
         "`spark.mesos.executor.home`; the installed path of the Mesos library can be specified " +
         "with `spark.executorEnv.MESOS_NATIVE_JAVA_LIBRARY`.")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
@@ -285,11 +318,13 @@ package object config {
       .doc("Force Mesos agents to pull the image specified in " +
         "`spark.mesos.executor.docker.image`. By default Mesos agents will not pull images they " +
         "already have cached.")
+      .version("2.1.0")
       .booleanConf
       .createOptional
 
   private[spark] val EXECUTOR_DOCKER_PORT_MAPS =
     ConfigBuilder("spark.mesos.executor.docker.portmaps")
+      .version("1.4.0")
       .stringConf
       .toSequence
       .createOptional
@@ -299,6 +334,7 @@ package object config {
       .doc("Set the list of custom parameters which will be passed into the `docker run` " +
         "command when launching the Spark executor on Mesos using the docker containerizer. " +
         "The format of this property is a list of key/value pairs which pair looks key1=value1.")
+      .version("2.2.0")
       .stringConf
       .toSequence
       .createOptional
@@ -309,6 +345,7 @@ package object config {
         "using `spark.mesos.executor.docker.image`. The format of this property is a list of " +
         "mappings following the form passed to `docker run -v`. That is they take the form:  " +
         "`[host_path:]container_path[:ro|:rw]`")
+      .version("1.4.0")
       .stringConf
       .toSequence
       .createOptional
@@ -318,6 +355,7 @@ package object config {
       .doc("Set the maximum number GPU resources to acquire for this job. Note that executors " +
         "will still launch when no GPU resources are found since this configuration is just an " +
         "upper limit and not a guaranteed amount.")
+      .version("2.1.0")
       .intConf
       .createWithDefault(0)
 
@@ -327,6 +365,7 @@ package object config {
         "Key-value pairs should be separated by a colon, and commas used to list more than one. " +
         "If your label includes a colon or comma, you can escape it with a backslash. " +
         "Ex. key:value,key2:a\\:b.")
+      .version("2.2.0")
       .stringConf
       .createWithDefault("")
 
@@ -335,6 +374,7 @@ package object config {
       .doc("Attribute-based constraints on mesos resource offers. By default, all resource " +
         "offers will be accepted. This setting applies only to executors. Refer to Mesos " +
         "Attributes & Resources doc for more information on attributes.")
+      .version("1.5.0")
       .stringConf
       .createWithDefault("")
 
@@ -344,6 +384,7 @@ package object config {
         "Mesos supports two types of containerizers for docker: the \"docker\" containerizer, " +
         "and the preferred \"mesos\" containerizer. " +
         "Read more here: http://mesos.apache.org/documentation/latest/container-image/")
+      .version("2.1.0")
       .stringConf
       .checkValues(Set("docker", "mesos"))
       .createWithDefault("docker")
@@ -352,6 +393,7 @@ package object config {
     ConfigBuilder("spark.mesos.role")
       .doc("Set the role of this Spark framework for Mesos. Roles are used in Mesos for " +
         "reservations and resource weight sharing.")
+      .version("1.5.0")
       .stringConf
       .createOptional
 

From aedaa6214144d953f1d8c2ce4fc85ac0f6a6f58a Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 12 Mar 2020 09:54:08 +0900
Subject: [PATCH 0436/1280] [SPARK-31118][K8S][DOC] Add version information to
 the configuration of K8S
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add version information to the configuration of `K8S`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.kubernetes.context | 3.0.0 | SPARK-25887 | c542c247bbfe1214c0bf81076451718a9e8931dc#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.master | 3.0.0 | SPARK-30371 | f14061c6a4729ad419902193aa23575d8f17f597#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.namespace | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.container.image | 2.3.0 | SPARK-22994 | b94debd2b01b87ef1d2a34d48877e38ade0969e6#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.container.image | 2.3.0 | SPARK-22807 | fb3636b482be3d0940345b1528c1d5090bbc25e6#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.container.image | 2.3.0 | SPARK-22807 | fb3636b482be3d0940345b1528c1d5090bbc25e6#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.container.image.pullPolicy | 2.3.0 | SPARK-22807 | fb3636b482be3d0940345b1528c1d5090bbc25e6#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.container.image.pullSecrets | 2.4.0 | SPARK-23668 | cccaaa14ad775fb981e501452ba2cc06ff5c0f0a#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.submission.requestTimeout | 3.0.0 | SPARK-27023 | e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.submission.connectionTimeout | 3.0.0 | SPARK-27023 | e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.requestTimeout | 3.0.0 | SPARK-27023 | e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.connectionTimeout | 3.0.0 | SPARK-27023 | e9e8bb33ef9ad785473ded168bc85867dad4ee70#diff-6e882d5561424e7e6651eb46f10104b8 |  
KUBERNETES_AUTH_DRIVER_CONF_PREFIX.serviceAccountName | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 | spark.kubernetes.authenticate.driver
KUBERNETES_AUTH_EXECUTOR_CONF_PREFIX.serviceAccountName | 3.1.0 | SPARK-30122 | f9f06eee9853ad4b6458ac9d31233e729a1ca226#diff-6e882d5561424e7e6651eb46f10104b8 | spark.kubernetes.authenticate.executor
spark.kubernetes.driver.limit.cores | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.request.cores | 3.0.0 | SPARK-27754 | 1a8c09334db87b0e938c38cd6b59d326bdcab3c3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.submitInDriver | 2.4.0 | SPARK-22839 | f15906da153f139b698e192ec6f82f078f896f1e#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.limit.cores | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.scheduler.name | 3.0.0 | SPARK-29436 | f800fa383131559c4e841bf062c9775d09190935#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.request.cores | 2.4.0 | SPARK-23285 | fe2b7a4568d65a62da6e6eb00fff05f248b4332c#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.pod.name | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.resourceNamePrefix | 3.0.0 | SPARK-25876 | 6be272b75b4ae3149869e19df193675cc4117763#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.podNamePrefix | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.allocation.batch.size | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.allocation.batch.delay | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.lostCheck.maxAttempts | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.submission.waitAppCompletion | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.report.interval | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.apiPollingInterval | 2.4.0 | SPARK-24248 | 270a9a3cac25f3e799460320d0fc94ccd7ecfaea#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.eventProcessingInterval | 2.4.0 | SPARK-24248 | 270a9a3cac25f3e799460320d0fc94ccd7ecfaea#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.memoryOverheadFactor | 2.4.0 | SPARK-23984 | 1a644afbac35c204f9ad55f86999319a9ab458c6#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.pyspark.pythonVersion | 2.4.0 | SPARK-23984 | a791c29bd824adadfb2d85594bc8dad4424df936#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.kerberos.krb5.path | 3.0.0 | SPARK-23257 | 6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.kerberos.krb5.configMapName | 3.0.0 | SPARK-23257 | 6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.hadoop.configMapName | 3.0.0 | SPARK-23257 | 6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.kerberos.tokenSecret.name | 3.0.0 | SPARK-23257 | 6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.kerberos.tokenSecret.itemKey | 3.0.0 | SPARK-23257 | 6c9c84ffb9c8d98ee2ece7ba4b010856591d383d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.resource.type | 2.4.1 | SPARK-25021 | 9031c784847353051bc0978f63ef4146ae9095ff#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.local.dirs.tmpfs | 3.0.0 | SPARK-25262 | da6fa3828bb824b65f50122a8a0a0d4741551257#diff-6e882d5561424e7e6651eb46f10104b8 | It exists in branch-3.0, but in pom.xml it is 2.4.0-snapshot
spark.kubernetes.driver.podTemplateFile | 3.0.0 | SPARK-24434 | f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.podTemplateFile | 3.0.0 | SPARK-24434 | f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.podTemplateContainerName | 3.0.0 | SPARK-24434 | f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.podTemplateContainerName | 3.0.0 | SPARK-24434 | f6cc354d83c2c9a757f9b507aadd4dbdc5825cca#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.deleteOnTermination | 3.0.0 | SPARK-25515 | 0c2935b01def8a5f631851999d9c2d57b63763e6#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.dynamicAllocation.deleteGracePeriod | 3.0.0 | SPARK-28487 | 0343854f54b48b206ca434accec99355011560c2#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.appKillPodDeletionGracePeriod | 3.0.0 | SPARK-24793 | 05168e725d2a17c4164ee5f9aa068801ec2454f4#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.file.upload.path | 3.0.0 | SPARK-23153 | 5e74570c8f5e7dfc1ca1c53c177827c5cea57bf1#diff-6e882d5561424e7e6651eb46f10104b8 |  
The following appears in the document |   |   |   |  
spark.kubernetes.authenticate.submission.caCertFile | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.submission.clientKeyFile | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.submission.clientCertFile | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.submission.oauthToken | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.submission.oauthTokenFile | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.caCertFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.clientKeyFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.clientCertFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.oauthToken | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.oauthTokenFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.mounted.caCertFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.mounted.clientKeyFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.mounted.clientCertFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.driver.mounted.oauthTokenFile | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.caCertFile | 2.4.0 | SPARK-23146 | 571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.clientKeyFile | 2.4.0 | SPARK-23146 | 571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.clientCertFile | 2.4.0 | SPARK-23146 | 571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.oauthToken | 2.4.0 | SPARK-23146 | 571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.authenticate.oauthTokenFile | 2.4.0 | SPARK-23146 | 571a6f0574e50e53cea403624ec3795cd03aa204#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.label.[LabelName] | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.annotation.[AnnotationName] | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.label.[LabelName] | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.annotation.[AnnotationName] | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.node.selector.[labelKey] | 2.3.0 | SPARK-18278 | e9b2070ab2d04993b1c0c1d6c6aba249e6664c8d#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driverEnv.[EnvironmentVariableName] | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.secrets.[SecretName] | 2.3.0 | SPARK-22757 | 171f6ddadc6185ffcc6ad82e5f48952fb49095b2#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.secrets.[SecretName] | 2.3.0 | SPARK-22757 | 171f6ddadc6185ffcc6ad82e5f48952fb49095b2#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.secretKeyRef.[EnvName] | 2.4.0 | SPARK-24232 | 21e1fc7d4aed688d7b685be6ce93f76752159c98#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.secretKeyRef.[EnvName] | 2.4.0 | SPARK-24232 | 21e1fc7d4aed688d7b685be6ce93f76752159c98#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.path | 2.4.0 | SPARK-23529 | 5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.subPath | 3.0.0 | SPARK-25960 | 3df307aa515b3564686e75d1b71754bbcaaf2dec#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.readOnly | 2.4.0 | SPARK-23529 | 5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].options.[OptionName] | 2.4.0 | SPARK-23529 | 5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-b5527f236b253e0d9f5db5164bdb43e9 |  
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.path | 2.4.0 | SPARK-23529 | 5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.subPath | 3.0.0 | SPARK-25960 | 3df307aa515b3564686e75d1b71754bbcaaf2dec#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.readOnly | 2.4.0 | SPARK-23529 | 5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-6e882d5561424e7e6651eb46f10104b8 |  
spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].options.[OptionName] | 2.4.0 | SPARK-23529 | 5ff1b9ba1983d5601add62aef64a3e87d07050eb#diff-b5527f236b253e0d9f5db5164bdb43e9 |  

Supplemental configuration version information.

'No'

Exists UT

Closes #27875 from beliefer/add-version-to-k8s-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/running-on-kubernetes.md                 | 77 ++++++++++++++++++-
 .../org/apache/spark/deploy/k8s/Config.scala  | 48 +++++++++++-
 2 files changed, 123 insertions(+), 2 deletions(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index a941b8c04cb54..4f228a5a101f5 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -494,7 +494,7 @@ See the [configuration page](configuration.html) for information on Spark config
 #### Spark Properties
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.kubernetes.context</code></td>
   <td><code>(none)</code></td>
@@ -505,6 +505,7 @@ See the [configuration page](configuration.html) for information on Spark config
     auto-configured settings can be overridden by the use of other Spark
     configuration properties e.g. <code>spark.kubernetes.namespace</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.master</code></td>
@@ -512,6 +513,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     The internal Kubernetes master (API server) address to be used for driver to request executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.namespace</code></td>
@@ -519,6 +521,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     The namespace that will be used for running the driver and executor pods.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.container.image</code></td>
@@ -529,6 +532,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This configuration is required and must be provided by the user, unless explicit
     images are provided for each different container type.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.container.image</code></td>
@@ -536,6 +540,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Custom container image to use for the driver.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.container.image</code></td>
@@ -543,6 +548,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Custom container image to use for executors.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.container.image.pullPolicy</code></td>
@@ -550,6 +556,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Container image pull policy used when pulling images within Kubernetes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.container.image.pullSecrets</code></td>
@@ -557,6 +564,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Comma separated list of Kubernetes secrets used to pull images from private image registries.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.allocation.batch.size</code></td>
@@ -564,6 +572,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Number of pods to launch at once in each round of executor pod allocation.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.allocation.batch.delay</code></td>
@@ -572,6 +581,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Time to wait between each round of executor pod allocation. Specifying values less than 1 second may lead to
     excessive CPU usage on the spark driver.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.caCertFile</code></td>
@@ -581,6 +591,7 @@ See the [configuration page](configuration.html) for information on Spark config
     must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not provide
     a scheme). In client mode, use <code>spark.kubernetes.authenticate.caCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.clientKeyFile</code></td>
@@ -590,6 +601,7 @@ See the [configuration page](configuration.html) for information on Spark config
     must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not provide
     a scheme). In client mode, use <code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.clientCertFile</code></td>
@@ -599,6 +611,7 @@ See the [configuration page](configuration.html) for information on Spark config
     file must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not
     provide a scheme). In client mode, use <code>spark.kubernetes.authenticate.clientCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.oauthToken</code></td>
@@ -608,6 +621,7 @@ See the [configuration page](configuration.html) for information on Spark config
     that unlike the other authentication options, this is expected to be the exact string value of the token to use for
     the authentication. In client mode, use <code>spark.kubernetes.authenticate.oauthToken</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.submission.oauthTokenFile</code></td>
@@ -617,6 +631,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This file must be located on the submitting machine's disk. Specify this as a path as opposed to a URI (i.e. do not
     provide a scheme). In client mode, use <code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.caCertFile</code></td>
@@ -627,6 +642,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.caCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.clientKeyFile</code></td>
@@ -637,6 +653,7 @@ See the [configuration page](configuration.html) for information on Spark config
     a Kubernetes secret. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
     In client mode, use <code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.clientCertFile</code></td>
@@ -647,6 +664,7 @@ See the [configuration page](configuration.html) for information on Spark config
     driver pod as a Kubernetes secret. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
     In client mode, use <code>spark.kubernetes.authenticate.clientCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.oauthToken</code></td>
@@ -657,6 +675,7 @@ See the [configuration page](configuration.html) for information on Spark config
     the token to use for the authentication. This token value is uploaded to the driver pod as a Kubernetes secret.
     In client mode, use <code>spark.kubernetes.authenticate.oauthToken</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.oauthTokenFile</code></td>
@@ -667,6 +686,7 @@ See the [configuration page](configuration.html) for information on Spark config
     the token to use for the authentication. This token value is uploaded to the driver pod as a secret. In client mode, use
     <code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.caCertFile</code></td>
@@ -677,6 +697,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.caCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.clientKeyFile</code></td>
@@ -687,6 +708,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.clientKeyFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.clientCertFile</code></td>
@@ -697,6 +719,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify this as a path as opposed to a URI (i.e. do not provide a scheme). In client mode, use
     <code>spark.kubernetes.authenticate.clientCertFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.mounted.oauthTokenFile</code></td>
@@ -707,6 +730,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Note that unlike the other authentication options, this file must contain the exact string value of the token to use
     for the authentication. In client mode, use <code>spark.kubernetes.authenticate.oauthTokenFile</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.driver.serviceAccountName</code></td>
@@ -716,6 +740,7 @@ See the [configuration page](configuration.html) for information on Spark config
     executor pods from the API server. Note that this cannot be specified alongside a CA cert file, client key file,
     client cert file, and/or OAuth token. In client mode, use <code>spark.kubernetes.authenticate.serviceAccountName</code> instead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.caCertFile</code></td>
@@ -724,6 +749,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the CA cert file for connecting to the Kubernetes API server over TLS when
     requesting executors. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.clientKeyFile</code></td>
@@ -732,6 +758,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the client key file for authenticating against the Kubernetes API server
     when requesting executors. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.clientCertFile</code></td>
@@ -740,6 +767,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the client cert file for authenticating against the Kubernetes API server
     when requesting executors. Specify this as a path as opposed to a URI (i.e. do not provide a scheme).
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.oauthToken</code></td>
@@ -749,6 +777,7 @@ See the [configuration page](configuration.html) for information on Spark config
     requesting executors. Note that unlike the other authentication options, this must be the exact string value of
     the token to use for the authentication.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.authenticate.oauthTokenFile</code></td>
@@ -757,6 +786,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In client mode, path to the file containing the OAuth token to use when authenticating against the Kubernetes API
     server when requesting executors.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.label.[LabelName]</code></td>
@@ -767,6 +797,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Note that Spark also adds its own labels to the driver pod
     for bookkeeping purposes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.annotation.[AnnotationName]</code></td>
@@ -775,6 +806,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Add the annotation specified by <code>AnnotationName</code> to the driver pod.
     For example, <code>spark.kubernetes.driver.annotation.something=true</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.label.[LabelName]</code></td>
@@ -785,6 +817,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Note that Spark also adds its own labels to the executor pod
     for bookkeeping purposes.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.annotation.[AnnotationName]</code></td>
@@ -793,6 +826,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Add the annotation specified by <code>AnnotationName</code> to the executor pods.
     For example, <code>spark.kubernetes.executor.annotation.something=true</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.pod.name</code></td>
@@ -804,6 +838,7 @@ See the [configuration page](configuration.html) for information on Spark config
     value in client mode allows the driver to become the owner of its executor pods, which in turn allows the executor
     pods to be garbage collected by the cluster.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.lostCheck.maxAttempts</code></td>
@@ -813,6 +848,7 @@ See the [configuration page](configuration.html) for information on Spark config
     The loss reason is used to ascertain whether the executor failure is due to a framework or an application error
     which in turn decides whether the executor is removed and replaced, or placed into a failed state for debugging.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.submission.waitAppCompletion</code></td>
@@ -821,6 +857,7 @@ See the [configuration page](configuration.html) for information on Spark config
     In cluster mode, whether to wait for the application to finish before exiting the launcher process.  When changed to
     false, the launcher has a "fire-and-forget" behavior when launching the Spark job.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.report.interval</code></td>
@@ -828,6 +865,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Interval between reports of the current Spark job status in cluster mode.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.request.cores</code></td>
@@ -837,6 +875,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Example values include 0.1, 500m, 1.5, 5, etc., with the definition of cpu units documented in <a href="https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units">CPU units</a>.
     This takes precedence over <code>spark.driver.cores</code> for specifying the driver pod cpu request if set.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.limit.cores</code></td>
@@ -844,6 +883,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Specify a hard cpu <a href="https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container">limit</a> for the driver pod.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.request.cores</code></td>
@@ -854,6 +894,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This is distinct from <code>spark.executor.cores</code>: it is only used and takes precedence over <code>spark.executor.cores</code> for specifying the executor pod cpu request if set. Task
     parallelism, e.g., number of tasks an executor can run concurrently is not affected by this.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.limit.cores</code></td>
@@ -861,6 +902,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Specify a hard cpu <a href="https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container">limit</a> for each executor pod launched for the Spark Application.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.node.selector.[labelKey]</code></td>
@@ -871,6 +913,7 @@ See the [configuration page](configuration.html) for information on Spark config
     will result in the driver pod and executors having a node selector with key <code>identifier</code> and value
      <code>myIdentifier</code>. Multiple node selector keys can be added by setting multiple configurations with this prefix.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driverEnv.[EnvironmentVariableName]</code></td>
@@ -879,6 +922,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Add the environment variable specified by <code>EnvironmentVariableName</code> to
     the Driver process. The user can specify multiple of these to set multiple environment variables.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.secrets.[SecretName]</code></td>
@@ -887,6 +931,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/configuration/secret/">Kubernetes Secret</a> named <code>SecretName</code> to the driver pod on the path specified in the value. For example,
    <code>spark.kubernetes.driver.secrets.spark-secret=/etc/secrets</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.secrets.[SecretName]</code></td>
@@ -895,6 +940,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/configuration/secret/">Kubernetes Secret</a> named <code>SecretName</code> to the executor pod on the path specified in the value. For example,
    <code>spark.kubernetes.executor.secrets.spark-secret=/etc/secrets</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.secretKeyRef.[EnvName]</code></td>
@@ -903,6 +949,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add as an environment variable to the driver container with name EnvName (case sensitive), the value referenced by key <code> key </code> in the data of the referenced <a href="https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables">Kubernetes Secret</a>. For example,
    <code>spark.kubernetes.driver.secretKeyRef.ENV_VAR=spark-secret:key</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.secretKeyRef.[EnvName]</code></td>
@@ -911,6 +958,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add as an environment variable to the executor container with name EnvName (case sensitive), the value referenced by key <code> key </code> in the data of the referenced <a href="https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables">Kubernetes Secret</a>. For example,
    <code>spark.kubernetes.executor.secrets.ENV_VAR=spark-secret:key</code>.
   </td>
+  <td>2.4.0</td>
 </tr>   
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.path</code></td>
@@ -919,6 +967,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> named <code>VolumeName</code> of the <code>VolumeType</code> type to the driver pod on the path specified in the value. For example,
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/checkpoint</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.subPath</code></td>
@@ -927,6 +976,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specifies a <a href="https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath">subpath</a> to be mounted from the volume into the driver pod.
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.subPath=checkpoint</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].mount.readOnly</code></td>
@@ -935,6 +985,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify if the mounted volume is read only or not. For example,
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.readOnly=false</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.volumes.[VolumeType].[VolumeName].options.[OptionName]</code></td>
@@ -943,6 +994,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Configure <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> options passed to the Kubernetes with <code>OptionName</code> as key having specified value, must conform with Kubernetes option format. For example,
    <code>spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=spark-pvc-claim</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.path</code></td>
@@ -951,6 +1003,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Add the <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> named <code>VolumeName</code> of the <code>VolumeType</code> type to the executor pod on the path specified in the value. For example,
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/checkpoint</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.subPath</code></td>
@@ -959,6 +1012,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specifies a <a href="https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath">subpath</a> to be mounted from the volume into the executor pod.
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.subPath=checkpoint</code>.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].mount.readOnly</code></td>
@@ -967,6 +1021,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify if the mounted volume is read only or not. For example,
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.mount.readOnly=false</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.volumes.[VolumeType].[VolumeName].options.[OptionName]</code></td>
@@ -975,6 +1030,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Configure <a href="https://kubernetes.io/docs/concepts/storage/volumes/">Kubernetes Volume</a> options passed to the Kubernetes with <code>OptionName</code> as key having specified value. For example,
    <code>spark.kubernetes.executor.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=spark-pvc-claim</code>.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.local.dirs.tmpfs</code></td>
@@ -983,6 +1039,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Configure the <code>emptyDir</code> volumes used to back <code>SPARK_LOCAL_DIRS</code> within the Spark driver and executor pods to use <code>tmpfs</code> backing i.e. RAM.  See <a href="#local-storage">Local Storage</a> earlier on this page
    for more discussion of this.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.memoryOverheadFactor</code></td>
@@ -991,6 +1048,7 @@ See the [configuration page](configuration.html) for information on Spark config
     This sets the Memory Overhead Factor that will allocate memory to non-JVM memory, which includes off-heap memory allocations, non-JVM tasks, and various systems processes. For JVM-based jobs this value will default to 0.10 and 0.40 for non-JVM jobs.
     This is done as non-JVM tasks need more non-JVM heap space and such tasks commonly fail with "Memory Overhead Exceeded" errors. This prempts this error with a higher default.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.pyspark.pythonVersion</code></td>
@@ -998,6 +1056,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
    This sets the major Python version of the docker image used to run the driver and executor containers. Can either be 2 or 3.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.krb5.path</code></td>
@@ -1006,6 +1065,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the local location of the krb5.conf file to be mounted on the driver and executors for Kerberos interaction.
    It is important to note that the KDC defined needs to be visible from inside the containers.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.krb5.configMapName</code></td>
@@ -1015,6 +1075,7 @@ See the [configuration page](configuration.html) for information on Spark config
    for Kerberos interaction. The KDC defined needs to be visible from inside the containers. The ConfigMap must also
    be in the same namespace of the driver and executor pods.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.hadoop.configMapName</code></td>
@@ -1023,6 +1084,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify the name of the ConfigMap, containing the HADOOP_CONF_DIR files, to be mounted on the driver
     and executors for custom Hadoop configuration.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.tokenSecret.name</code></td>
@@ -1031,6 +1093,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify the name of the secret where your existing delegation tokens are stored. This removes the need for the job user
     to provide any kerberos credentials for launching a job.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.kerberos.tokenSecret.itemKey</code></td>
@@ -1039,6 +1102,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Specify the item key of the data where your existing delegation tokens are stored. This removes the need for the job user
     to provide any kerberos credentials for launching a job.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.podTemplateFile</code></td>
@@ -1047,6 +1111,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the local file that contains the driver <a href="#pod-template">pod template</a>. For example
    <code>spark.kubernetes.driver.podTemplateFile=/path/to/driver-pod-template.yaml</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.podTemplateContainerName</code></td>
@@ -1055,6 +1120,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the container name to be used as a basis for the driver in the given <a href="#pod-template">pod template</a>.
    For example <code>spark.kubernetes.driver.podTemplateContainerName=spark-driver</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.podTemplateFile</code></td>
@@ -1063,6 +1129,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the local file that contains the executor <a href="#pod-template">pod template</a>. For example
    <code>spark.kubernetes.executor.podTemplateFile=/path/to/executor-pod-template.yaml</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.podTemplateContainerName</code></td>
@@ -1071,6 +1138,7 @@ See the [configuration page](configuration.html) for information on Spark config
    Specify the container name to be used as a basis for the executor in the given <a href="#pod-template">pod template</a>.
    For example <code>spark.kubernetes.executor.podTemplateContainerName=spark-executor</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.deleteOnTermination</code></td>
@@ -1078,6 +1146,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
   Specify whether executor pods should be deleted in case of failure or normal termination.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.submission.connectionTimeout</code></td>
@@ -1085,6 +1154,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Connection timeout in milliseconds for the kubernetes client to use for starting the driver.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.submission.requestTimeout</code></td>
@@ -1092,6 +1162,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Request timeout in milliseconds for the kubernetes client to use for starting the driver.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.connectionTimeout</code></td>
@@ -1099,6 +1170,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Connection timeout in milliseconds for the kubernetes client in driver to use when requesting executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.driver.requestTimeout</code></td>
@@ -1106,6 +1178,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
     Request timeout in milliseconds for the kubernetes client in driver to use when requesting executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>  
   <td><code>spark.kubernetes.appKillPodDeletionGracePeriod</code></td>
@@ -1113,6 +1186,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td>
   Specify the grace period in seconds when deleting a Spark application using spark-submit.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.file.upload.path</code></td>
@@ -1122,6 +1196,7 @@ See the [configuration page](configuration.html) for information on Spark config
     <code>spark.kubernetes.file.upload.path=s3a://&lt;s3-bucket&gt;/path</code>
     File should specified as <code>file://path/to/file </code> or absolute path.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index b326650476add..8684a602f2760 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -33,6 +33,7 @@ private[spark] object Config extends Logging {
         "to .kube/config under your home directory.  If not specified then your current " +
         "context is used.  You can always override specific aspects of the config file " +
         "provided configuration using other Spark on K8S configuration options.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -40,12 +41,14 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.driver.master")
       .doc("The internal Kubernetes master (API server) address " +
         "to be used for driver to request executors.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(KUBERNETES_MASTER_INTERNAL_URL)
 
   val KUBERNETES_NAMESPACE =
     ConfigBuilder("spark.kubernetes.namespace")
       .doc("The namespace that will be used for running the driver and executor pods.")
+      .version("2.3.0")
       .stringConf
       .createWithDefault("default")
 
@@ -54,22 +57,26 @@ private[spark] object Config extends Logging {
       .doc("Container image to use for Spark containers. Individual container types " +
         "(e.g. driver or executor) can also be configured to use different images if desired, " +
         "by setting the container type-specific image name.")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val DRIVER_CONTAINER_IMAGE =
     ConfigBuilder("spark.kubernetes.driver.container.image")
       .doc("Container image to use for the driver.")
+      .version("2.3.0")
       .fallbackConf(CONTAINER_IMAGE)
 
   val EXECUTOR_CONTAINER_IMAGE =
     ConfigBuilder("spark.kubernetes.executor.container.image")
       .doc("Container image to use for the executors.")
+      .version("2.3.0")
       .fallbackConf(CONTAINER_IMAGE)
 
   val CONTAINER_IMAGE_PULL_POLICY =
     ConfigBuilder("spark.kubernetes.container.image.pullPolicy")
       .doc("Kubernetes image pull policy. Valid values are Always, Never, and IfNotPresent.")
+      .version("2.3.0")
       .stringConf
       .checkValues(Set("Always", "Never", "IfNotPresent"))
       .createWithDefault("IfNotPresent")
@@ -78,6 +85,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.container.image.pullSecrets")
       .doc("Comma separated list of the Kubernetes secrets used " +
         "to access private image registries.")
+      .version("2.4.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -95,24 +103,28 @@ private[spark] object Config extends Logging {
   val SUBMISSION_CLIENT_REQUEST_TIMEOUT =
     ConfigBuilder("spark.kubernetes.submission.requestTimeout")
       .doc("request timeout to be used in milliseconds for starting the driver")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val SUBMISSION_CLIENT_CONNECTION_TIMEOUT =
     ConfigBuilder("spark.kubernetes.submission.connectionTimeout")
       .doc("connection timeout to be used in milliseconds for starting the driver")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val DRIVER_CLIENT_REQUEST_TIMEOUT =
     ConfigBuilder("spark.kubernetes.driver.requestTimeout")
       .doc("request timeout to be used in milliseconds for driver to request executors")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val DRIVER_CLIENT_CONNECTION_TIMEOUT =
     ConfigBuilder("spark.kubernetes.driver.connectionTimeout")
       .doc("connection timeout to be used in milliseconds for driver to request executors")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
@@ -122,48 +134,56 @@ private[spark] object Config extends Logging {
         "this service account when requesting executor pods from the API server. If specific " +
         "credentials are given for the driver pod to use, the driver will favor " +
         "using those credentials instead.")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_LIMIT_CORES =
     ConfigBuilder("spark.kubernetes.driver.limit.cores")
       .doc("Specify the hard cpu limit for the driver pod")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_REQUEST_CORES =
     ConfigBuilder("spark.kubernetes.driver.request.cores")
       .doc("Specify the cpu request for the driver pod")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_SUBMIT_CHECK =
     ConfigBuilder("spark.kubernetes.submitInDriver")
     .internal()
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
   val KUBERNETES_EXECUTOR_LIMIT_CORES =
     ConfigBuilder("spark.kubernetes.executor.limit.cores")
       .doc("Specify the hard cpu limit for each executor pod")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_SCHEDULER_NAME =
     ConfigBuilder("spark.kubernetes.executor.scheduler.name")
       .doc("Specify the scheduler name for each executor pod")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_REQUEST_CORES =
     ConfigBuilder("spark.kubernetes.executor.request.cores")
       .doc("Specify the cpu request for each executor pod")
+      .version("2.4.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_POD_NAME =
     ConfigBuilder("spark.kubernetes.driver.pod.name")
       .doc("Name of the driver pod.")
+      .version("2.3.0")
       .stringConf
       .createOptional
 
@@ -171,12 +191,14 @@ private[spark] object Config extends Logging {
   val KUBERNETES_DRIVER_POD_NAME_PREFIX =
     ConfigBuilder("spark.kubernetes.driver.resourceNamePrefix")
       .internal()
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_POD_NAME_PREFIX =
     ConfigBuilder("spark.kubernetes.executor.podNamePrefix")
       .doc("Prefix to use in front of the executor pod names.")
+      .version("2.3.0")
       .internal()
       .stringConf
       .createOptional
@@ -184,6 +206,7 @@ private[spark] object Config extends Logging {
   val KUBERNETES_ALLOCATION_BATCH_SIZE =
     ConfigBuilder("spark.kubernetes.allocation.batch.size")
       .doc("Number of pods to launch at once in each round of executor allocation.")
+      .version("2.3.0")
       .intConf
       .checkValue(value => value > 0, "Allocation batch size should be a positive integer")
       .createWithDefault(5)
@@ -191,6 +214,7 @@ private[spark] object Config extends Logging {
   val KUBERNETES_ALLOCATION_BATCH_DELAY =
     ConfigBuilder("spark.kubernetes.allocation.batch.delay")
       .doc("Time to wait between each round of executor allocation.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(value => value > 0, "Allocation batch delay must be a positive time value.")
       .createWithDefaultString("1s")
@@ -199,6 +223,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.lostCheck.maxAttempts")
       .doc("Maximum number of attempts allowed for checking the reason of an executor loss " +
         "before it is assumed that the executor failed.")
+      .version("2.3.0")
       .intConf
       .checkValue(value => value > 0, "Maximum attempts of checks of executor lost reason " +
         "must be a positive integer")
@@ -208,12 +233,14 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.submission.waitAppCompletion")
       .doc("In cluster mode, whether to wait for the application to finish before exiting the " +
         "launcher process.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
   val REPORT_INTERVAL =
     ConfigBuilder("spark.kubernetes.report.interval")
       .doc("Interval between reports of the current app status in cluster mode.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(interval => interval > 0, s"Logging interval must be a positive time value.")
       .createWithDefaultString("1s")
@@ -222,6 +249,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.apiPollingInterval")
       .doc("Interval between polls against the Kubernetes API server to inspect the " +
         "state of executors.")
+      .version("2.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(interval => interval > 0, s"API server polling interval must be a" +
         " positive time value.")
@@ -231,6 +259,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.eventProcessingInterval")
       .doc("Interval between successive inspection of executor events sent from the" +
         " Kubernetes API.")
+      .version("2.4.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(interval => interval > 0, s"Event processing interval must be a positive" +
         " time value.")
@@ -240,6 +269,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.memoryOverheadFactor")
       .doc("This sets the Memory Overhead Factor that will allocate memory to non-JVM jobs " +
         "which in the case of JVM tasks will default to 0.10 and 0.40 for non-JVM jobs")
+      .version("2.4.0")
       .doubleConf
       .checkValue(mem_overhead => mem_overhead >= 0 && mem_overhead < 1,
         "Ensure that memory overhead is a double between 0 --> 1.0")
@@ -248,6 +278,7 @@ private[spark] object Config extends Logging {
   val PYSPARK_MAJOR_PYTHON_VERSION =
     ConfigBuilder("spark.kubernetes.pyspark.pythonVersion")
       .doc("This sets the major Python version. Either 2 or 3. (Python2 or Python3)")
+      .version("2.4.0")
       .stringConf
       .checkValue(pv => List("2", "3").contains(pv),
         "Ensure that major Python version is either Python2 or Python3")
@@ -258,6 +289,7 @@ private[spark] object Config extends Logging {
       .doc("Specify the local location of the krb5.conf file to be mounted on the driver " +
         "and executors for Kerberos. Note: The KDC defined needs to be " +
         "visible from inside the containers ")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -266,6 +298,7 @@ private[spark] object Config extends Logging {
       .doc("Specify the name of the ConfigMap, containing the krb5.conf file, to be mounted " +
         "on the driver and executors for Kerberos. Note: The KDC defined" +
         "needs to be visible from inside the containers ")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -273,6 +306,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.hadoop.configMapName")
       .doc("Specify the name of the ConfigMap, containing the HADOOP_CONF_DIR files, " +
         "to be mounted on the driver and executors for custom Hadoop configuration.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -280,6 +314,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.kerberos.tokenSecret.name")
       .doc("Specify the name of the secret where your existing delegation tokens are stored. " +
         "This removes the need for the job user to provide any keytab for launching a job")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -287,13 +322,15 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.kerberos.tokenSecret.itemKey")
       .doc("Specify the item key of the data where your existing delegation tokens are stored. " +
         "This removes the need for the job user to provide any keytab for launching a job")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val APP_RESOURCE_TYPE =
     ConfigBuilder("spark.kubernetes.resource.type")
-      .doc("This sets the resource type internally")
       .internal()
+      .doc("This sets the resource type internally")
+      .version("2.4.1")
       .stringConf
       .checkValues(Set(APP_RESOURCE_TYPE_JAVA, APP_RESOURCE_TYPE_PYTHON, APP_RESOURCE_TYPE_R))
       .createOptional
@@ -304,30 +341,35 @@ private[spark] object Config extends Logging {
         "their medium set to Memory so that they will be created as tmpfs (i.e. RAM) backed " +
         "volumes. This may improve performance but scratch space usage will count towards " +
         "your pods memory limit so you may wish to request more memory.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val KUBERNETES_DRIVER_PODTEMPLATE_FILE =
     ConfigBuilder("spark.kubernetes.driver.podTemplateFile")
       .doc("File containing a template pod spec for the driver")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_PODTEMPLATE_FILE =
     ConfigBuilder("spark.kubernetes.executor.podTemplateFile")
       .doc("File containing a template pod spec for executors")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_DRIVER_PODTEMPLATE_CONTAINER_NAME =
     ConfigBuilder("spark.kubernetes.driver.podTemplateContainerName")
       .doc("container name to be used as a basis for the driver in the given pod template")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   val KUBERNETES_EXECUTOR_PODTEMPLATE_CONTAINER_NAME =
     ConfigBuilder("spark.kubernetes.executor.podTemplateContainerName")
       .doc("container name to be used as a basis for executors in the given pod template")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -340,12 +382,14 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.executor.deleteOnTermination")
       .doc("If set to false then executor pods will not be deleted in case " +
         "of failure or normal termination.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val KUBERNETES_DYN_ALLOC_KILL_GRACE_PERIOD =
     ConfigBuilder("spark.kubernetes.dynamicAllocation.deleteGracePeriod")
       .doc("How long to wait for executors to shut down gracefully before a forceful kill.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5s")
 
@@ -353,6 +397,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.appKillPodDeletionGracePeriod")
       .doc("Time to wait for graceful deletion of Spark pods when spark-submit" +
         " is used for killing an application.")
+      .version("3.0.0")
       .timeConf(TimeUnit.SECONDS)
       .createOptional
 
@@ -360,6 +405,7 @@ private[spark] object Config extends Logging {
     ConfigBuilder("spark.kubernetes.file.upload.path")
       .doc("Hadoop compatible file system path where files from the local file system " +
         "will be uploded to in cluster mode.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 

From 796f9cf3fd24141cfc43328ab30b20f7bd659263 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 7 Apr 2020 07:09:45 +0000
Subject: [PATCH 0437/1280] [SPARK-29311][SQL][FOLLOWUP] Add migration guide
 for extracting second from datetimes

### What changes were proposed in this pull request?

Add migration guide for extracting second from datetimes

### Why are the changes needed?

doc the behavior change for extract expression

### Does this PR introduce any user-facing change?

No
### How was this patch tested?

N/A, just passing jenkins

Closes #28140 from yaooqinn/SPARK-29311.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 3c94a7c8f548d7023c363720fbdd122980905ba8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index ab35e1fc0a005..5a2b848fb7a61 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -285,6 +285,8 @@ license: |
     - `now` - current query start time
   For example `SELECT timestamp 'tomorrow';`.
 
+  - Since Spark 3.0, when using `EXTRACT` expression to extract the second field from date/timestamp values, the result will be a `DecimalType(8, 6)` value with 2 digits for second part, and 6 digits for the fractional part with microsecond precision. e.g. `extract(second from to_timestamp('2019-09-20 10:10:10.1'))` results `10.100000`.  In Spark version 2.4 and earlier, it returns an `IntegerType` value and the result for the former example is `10`.
+
 ### Data Sources
 
   - In Spark version 2.4 and earlier, when reading a Hive Serde table with Spark native data sources(parquet/orc), Spark will infer the actual file schema and update the table schema in metastore. Since Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, please set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.

From 2a10e9fcb705a238573cc78bc5be8cd801989dbe Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 7 Apr 2020 20:29:12 +0900
Subject: [PATCH 0438/1280] Revert "[SPARK-31002][CORE][DOC][3.0] Add version
 information to the configuration of Core"

This reverts commit a5444db82ace1bfca32ec8d5b9f9823e91ac9511.
---
 .../spark/internal/config/package.scala       | 340 ++----------------
 1 file changed, 28 insertions(+), 312 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index fe0e476cdd213..1308a460a2cc8 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -45,7 +45,6 @@ package object config {
         "custom implementation. Spark will try each class specified until one of them " +
         "returns the resource information for that resource. It tries the discovery " +
         "script last if none of the plugins return information for that resource.")
-      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -56,113 +55,89 @@ package object config {
       .doc("Path to a file containing the resources allocated to the driver. " +
         "The file should be formatted as a JSON array of ResourceAllocation objects. " +
         "Only used internally in standalone mode.")
-      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH)
-      .version("1.0.0")
-      .stringConf
-      .createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH).stringConf.createOptional
 
   private[spark] val DRIVER_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS)
-      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)
-      .version("1.0.0")
-      .stringConf
-      .createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH).stringConf.createOptional
 
   private[spark] val DRIVER_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.driver.userClassPathFirst")
-      .version("1.3.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.driver.userClassPathFirst").booleanConf.createWithDefault(false)
 
   private[spark] val DRIVER_CORES = ConfigBuilder("spark.driver.cores")
     .doc("Number of cores to use for the driver process, only in cluster mode.")
-    .version("1.3.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val DRIVER_MEMORY = ConfigBuilder(SparkLauncher.DRIVER_MEMORY)
     .doc("Amount of memory to use for the driver process, in MiB unless otherwise specified.")
-    .version("1.1.1")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per driver in cluster mode, " +
       "in MiB unless otherwise specified.")
-    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
   private[spark] val DRIVER_LOG_DFS_DIR =
-    ConfigBuilder("spark.driver.log.dfsDir").version("3.0.0").stringConf.createOptional
+    ConfigBuilder("spark.driver.log.dfsDir").stringConf.createOptional
 
   private[spark] val DRIVER_LOG_LAYOUT =
     ConfigBuilder("spark.driver.log.layout")
-      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LOG_PERSISTTODFS =
     ConfigBuilder("spark.driver.log.persistToDfs.enabled")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DRIVER_LOG_ALLOW_EC =
     ConfigBuilder("spark.driver.log.allowErasureCoding")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLED = ConfigBuilder("spark.eventLog.enabled")
-    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val EVENT_LOG_DIR = ConfigBuilder("spark.eventLog.dir")
-    .version("1.0.0")
     .stringConf
     .createWithDefault(EventLoggingListener.DEFAULT_LOG_DIR)
 
   private[spark] val EVENT_LOG_COMPRESS =
     ConfigBuilder("spark.eventLog.compress")
-      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_BLOCK_UPDATES =
     ConfigBuilder("spark.eventLog.logBlockUpdates.enabled")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ALLOW_EC =
     ConfigBuilder("spark.eventLog.erasureCoding.enabled")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_TESTING =
     ConfigBuilder("spark.eventLog.testing")
       .internal()
-      .version("1.0.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_OUTPUT_BUFFER_SIZE = ConfigBuilder("spark.eventLog.buffer.kb")
     .doc("Buffer size to use when writing to output streams, in KiB unless otherwise specified.")
-    .version("1.0.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("100k")
 
@@ -170,7 +145,6 @@ package object config {
     ConfigBuilder("spark.eventLog.logStageExecutorMetrics")
       .doc("Whether to write per-stage peaks of executor metrics (for each executor) " +
         "to the event log.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -179,7 +153,6 @@ package object config {
       .doc("Names of supported young generation garbage collector. A name usually is " +
         " the return of GarbageCollectorMXBean.getName. The built-in young generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
-      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
@@ -189,28 +162,20 @@ package object config {
       .doc("Names of supported old generation garbage collector. A name usually is " +
         "the return of GarbageCollectorMXBean.getName. The built-in old generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
-      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
 
   private[spark] val EVENT_LOG_OVERWRITE =
-    ConfigBuilder("spark.eventLog.overwrite")
-      .version("1.0.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.eventLog.overwrite").booleanConf.createWithDefault(false)
 
   private[spark] val EVENT_LOG_CALLSITE_LONG_FORM =
-    ConfigBuilder("spark.eventLog.longForm.enabled")
-      .version("2.4.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.eventLog.longForm.enabled").booleanConf.createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLE_ROLLING =
     ConfigBuilder("spark.eventLog.rolling.enabled")
       .doc("Whether rolling over event log files is enabled. If set to true, it cuts down " +
         "each event log file to the configured size.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -218,46 +183,35 @@ package object config {
     ConfigBuilder("spark.eventLog.rolling.maxFileSize")
       .doc(s"When ${EVENT_LOG_ENABLE_ROLLING.key}=true, specifies the max size of event log file" +
         " to be rolled over.")
-      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
         "configured to be at least 10 MiB.")
       .createWithDefaultString("128m")
 
   private[spark] val EXECUTOR_ID =
-    ConfigBuilder("spark.executor.id").version("1.2.0").stringConf.createOptional
+    ConfigBuilder("spark.executor.id").stringConf.createOptional
 
   private[spark] val EXECUTOR_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH)
-      .version("1.0.0")
-      .stringConf
-      .createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH).stringConf.createOptional
 
   private[spark] val EXECUTOR_HEARTBEAT_DROP_ZERO_ACCUMULATOR_UPDATES =
     ConfigBuilder("spark.executor.heartbeat.dropZeroAccumulatorUpdates")
       .internal()
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val EXECUTOR_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.executor.heartbeatInterval")
-      .version("1.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val EXECUTOR_HEARTBEAT_MAX_FAILURES =
-    ConfigBuilder("spark.executor.heartbeat.maxFailures")
-      .internal()
-      .version("1.6.2")
-      .intConf
-      .createWithDefault(60)
+    ConfigBuilder("spark.executor.heartbeat.maxFailures").internal().intConf.createWithDefault(60)
 
   private[spark] val EXECUTOR_PROCESS_TREE_METRICS_ENABLED =
     ConfigBuilder("spark.executor.processTreeMetrics.enabled")
       .doc("Whether to collect process tree metrics (from the /proc filesystem) when collecting " +
         "executor metrics.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -266,44 +220,33 @@ package object config {
       .doc("How often to collect executor metrics (in milliseconds). " +
         "If 0, the polling is done on executor heartbeats. " +
         "If positive, the polling is done at this interval.")
-      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
   private[spark] val EXECUTOR_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS)
-      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val EXECUTOR_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH)
-      .version("1.0.0")
-      .stringConf
-      .createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH).stringConf.createOptional
 
   private[spark] val EXECUTOR_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.executor.userClassPathFirst")
-      .version("1.3.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.executor.userClassPathFirst").booleanConf.createWithDefault(false)
 
   private[spark] val EXECUTOR_CORES = ConfigBuilder(SparkLauncher.EXECUTOR_CORES)
-    .version("1.0.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val EXECUTOR_MEMORY = ConfigBuilder(SparkLauncher.EXECUTOR_MEMORY)
     .doc("Amount of memory to use per executor process, in MiB unless otherwise specified.")
-    .version("0.7.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " +
       "in MiB unless otherwise specified.")
-    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
@@ -313,14 +256,12 @@ package object config {
       "the cluster (not from each machine). If not set, the default will be " +
       "`spark.deploy.defaultCores` on Spark's standalone cluster manager, or infinite " +
       "(all available cores) on Mesos.")
-    .version("0.6.0")
     .intConf
     .createOptional
 
   private[spark] val MEMORY_OFFHEAP_ENABLED = ConfigBuilder("spark.memory.offHeap.enabled")
     .doc("If true, Spark will attempt to use off-heap memory for certain operations. " +
       "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive.")
-    .version("1.6.0")
     .withAlternative("spark.unsafe.offHeap")
     .booleanConf
     .createWithDefault(false)
@@ -331,7 +272,6 @@ package object config {
       "This setting has no impact on heap memory usage, so if your executors' total memory " +
       "consumption must fit within some hard limit then be sure to shrink your JVM heap size " +
       "accordingly. This must be set to a positive value when spark.memory.offHeap.enabled=true.")
-    .version("1.6.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(_ >= 0, "The off-heap memory size must not be negative")
     .createWithDefault(0)
@@ -341,7 +281,6 @@ package object config {
       "size of the region set aside by spark.memory.fraction. The higher this is, the " +
       "less working memory may be available to execution and tasks may spill to disk more " +
       "often. Leaving this at the default value is recommended. ")
-    .version("1.6.0")
     .doubleConf
     .checkValue(v => v >= 0.0 && v < 1.0, "Storage fraction must be in [0,1)")
     .createWithDefault(0.5)
@@ -352,19 +291,16 @@ package object config {
       "The purpose of this config is to set aside memory for internal metadata, " +
       "user data structures, and imprecise size estimation in the case of sparse, " +
       "unusually large records. Leaving this at the default value is recommended.  ")
-    .version("1.6.0")
     .doubleConf
     .createWithDefault(0.6)
 
   private[spark] val STORAGE_SAFETY_FRACTION = ConfigBuilder("spark.storage.safetyFraction")
-    .version("1.1.0")
     .doubleConf
     .createWithDefault(0.9)
 
   private[spark] val STORAGE_UNROLL_MEMORY_THRESHOLD =
     ConfigBuilder("spark.storage.unrollMemoryThreshold")
       .doc("Initial memory to request before unrolling any block")
-      .version("1.1.0")
       .longConf
       .createWithDefault(1024 * 1024)
 
@@ -374,7 +310,6 @@ package object config {
         "Cached RDD block replicas lost due to executor failures are replenished " +
         "if there are any existing available replicas. This tries to " +
         "get the replication level of the block to the initial number")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -385,54 +320,41 @@ package object config {
         "This prevents Spark from memory mapping very small blocks. " +
         "In general, memory mapping has high overhead for blocks close to or below " +
         "the page size of the operating system.")
-      .version("0.9.2")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("2m")
 
   private[spark] val STORAGE_REPLICATION_POLICY =
     ConfigBuilder("spark.storage.replication.policy")
-      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[RandomBlockReplicationPolicy].getName)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_MAPPER =
     ConfigBuilder("spark.storage.replication.topologyMapper")
-      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[DefaultTopologyMapper].getName)
 
   private[spark] val STORAGE_CACHED_PEERS_TTL = ConfigBuilder("spark.storage.cachedPeersTtl")
-    .version("1.1.1")
-    .intConf
-    .createWithDefault(60 * 1000)
+    .intConf.createWithDefault(60 * 1000)
 
   private[spark] val STORAGE_MAX_REPLICATION_FAILURE =
     ConfigBuilder("spark.storage.maxReplicationFailures")
-      .version("1.1.1")
-      .intConf
-      .createWithDefault(1)
+      .intConf.createWithDefault(1)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_FILE =
-    ConfigBuilder("spark.storage.replication.topologyFile")
-      .version("2.1.0")
-      .stringConf
-      .createOptional
+    ConfigBuilder("spark.storage.replication.topologyFile").stringConf.createOptional
 
   private[spark] val STORAGE_EXCEPTION_PIN_LEAK =
     ConfigBuilder("spark.storage.exceptionOnPinLeak")
-      .version("1.6.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STORAGE_BLOCKMANAGER_TIMEOUTINTERVAL =
     ConfigBuilder("spark.storage.blockManagerTimeoutIntervalMs")
-      .version("0.7.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("60s")
 
   private[spark] val STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT =
     ConfigBuilder("spark.storage.blockManagerSlaveTimeoutMs")
-      .version("0.7.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString(Network.NETWORK_TIMEOUT.defaultValueString)
 
@@ -440,7 +362,6 @@ package object config {
     ConfigBuilder("spark.storage.cleanupFilesAfterExecutorExit")
       .doc("Whether or not cleanup the files not served by the external shuffle service " +
         "on executor exits.")
-      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -448,7 +369,6 @@ package object config {
     ConfigBuilder("spark.diskStore.subDirectories")
       .doc("Number of subdirectories inside each path listed in spark.local.dir for " +
         "hashing Block files into.")
-      .version("0.6.0")
       .intConf
       .checkValue(_ > 0, "The number of subdirectories must be positive.")
       .createWithDefault(64)
@@ -457,101 +377,71 @@ package object config {
     ConfigBuilder("spark.block.failures.beforeLocationRefresh")
       .doc("Max number of failures before this block manager refreshes " +
         "the block locations from the driver.")
-      .version("2.0.0")
       .intConf
       .createWithDefault(5)
 
-  private[spark] val IS_PYTHON_APP =
-    ConfigBuilder("spark.yarn.isPython")
-      .internal()
-      .version("1.5.0")
-      .booleanConf
-      .createWithDefault(false)
+  private[spark] val IS_PYTHON_APP = ConfigBuilder("spark.yarn.isPython").internal()
+    .booleanConf.createWithDefault(false)
 
   private[spark] val CPUS_PER_TASK = ConfigBuilder("spark.task.cpus").intConf.createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_ENABLED =
-    ConfigBuilder("spark.dynamicAllocation.enabled")
-      .version("1.2.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.enabled").booleanConf.createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_TESTING =
-    ConfigBuilder("spark.dynamicAllocation.testing")
-      .version("1.2.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.testing").booleanConf.createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_MIN_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.minExecutors")
-      .version("1.2.0")
-      .intConf
-      .createWithDefault(0)
+    ConfigBuilder("spark.dynamicAllocation.minExecutors").intConf.createWithDefault(0)
 
   private[spark] val DYN_ALLOCATION_INITIAL_EXECUTORS =
     ConfigBuilder("spark.dynamicAllocation.initialExecutors")
-      .version("1.3.0")
       .fallbackConf(DYN_ALLOCATION_MIN_EXECUTORS)
 
   private[spark] val DYN_ALLOCATION_MAX_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.maxExecutors")
-      .version("1.2.0")
-      .intConf
-      .createWithDefault(Int.MaxValue)
+    ConfigBuilder("spark.dynamicAllocation.maxExecutors").intConf.createWithDefault(Int.MaxValue)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO =
     ConfigBuilder("spark.dynamicAllocation.executorAllocationRatio")
-      .version("2.4.0")
-      .doubleConf
-      .createWithDefault(1.0)
+      .doubleConf.createWithDefault(1.0)
 
   private[spark] val DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.cachedExecutorIdleTimeout")
-      .version("1.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.executorIdleTimeout")
-      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(60)
 
   private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING =
     ConfigBuilder("spark.dynamicAllocation.shuffleTracking.enabled")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_SHUFFLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.shuffleTimeout")
-      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Long.MaxValue)
 
   private[spark] val DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.schedulerBacklogTimeout")
-      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS).createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.sustainedSchedulerBacklogTimeout")
-      .version("1.2.0")
       .fallbackConf(DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT)
 
   private[spark] val LOCALITY_WAIT = ConfigBuilder("spark.locality.wait")
-    .version("0.5.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("3s")
 
   private[spark] val SHUFFLE_SERVICE_ENABLED =
-    ConfigBuilder("spark.shuffle.service.enabled")
-      .version("1.2.0")
-      .booleanConf
-      .createWithDefault(false)
+    ConfigBuilder("spark.shuffle.service.enabled").booleanConf.createWithDefault(false)
 
   private[spark] val SHUFFLE_SERVICE_FETCH_RDD_ENABLED =
     ConfigBuilder(Constants.SHUFFLE_SERVICE_FETCH_RDD_ENABLED)
@@ -559,7 +449,6 @@ package object config {
         "In case of dynamic allocation if this feature is enabled executors having only disk " +
         "persisted blocks are considered idle after " +
         "'spark.dynamicAllocation.executorIdleTimeout' and will be released accordingly.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -567,25 +456,21 @@ package object config {
     ConfigBuilder("spark.shuffle.service.db.enabled")
       .doc("Whether to use db in ExternalShuffleService. Note that this only affects " +
         "standalone mode.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_SERVICE_PORT =
-    ConfigBuilder("spark.shuffle.service.port").version("1.2.0").intConf.createWithDefault(7337)
+    ConfigBuilder("spark.shuffle.service.port").intConf.createWithDefault(7337)
 
   private[spark] val KEYTAB = ConfigBuilder("spark.kerberos.keytab")
     .doc("Location of user's keytab.")
-    .version("3.0.0")
     .stringConf.createOptional
 
   private[spark] val PRINCIPAL = ConfigBuilder("spark.kerberos.principal")
     .doc("Name of the Kerberos principal.")
-    .version("3.0.0")
     .stringConf.createOptional
 
   private[spark] val KERBEROS_RELOGIN_PERIOD = ConfigBuilder("spark.kerberos.relogin.period")
-    .version("3.0.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("1m")
 
@@ -595,7 +480,6 @@ package object config {
         "Which credentials to use when renewing delegation tokens for executors. Can be either " +
         "'keytab', the default, which requires a keytab to be provided, or 'ccache', which uses " +
         "the local credentials cache.")
-      .version("3.0.0")
       .stringConf
       .checkValues(Set("keytab", "ccache"))
       .createWithDefault("keytab")
@@ -604,124 +488,104 @@ package object config {
     ConfigBuilder("spark.kerberos.access.hadoopFileSystems")
     .doc("Extra Hadoop filesystem URLs for which to request delegation tokens. The filesystem " +
       "that hosts fs.defaultFS does not need to be listed here.")
-    .version("3.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances")
-    .version("1.0.0")
     .intConf
     .createOptional
 
   private[spark] val PY_FILES = ConfigBuilder("spark.yarn.dist.pyFiles")
     .internal()
-    .version("2.2.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val TASK_MAX_DIRECT_RESULT_SIZE =
     ConfigBuilder("spark.task.maxDirectResultSize")
-      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1L << 20)
 
   private[spark] val TASK_MAX_FAILURES =
     ConfigBuilder("spark.task.maxFailures")
-      .version("0.8.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val TASK_REAPER_ENABLED =
     ConfigBuilder("spark.task.reaper.enabled")
-      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val TASK_REAPER_KILL_TIMEOUT =
     ConfigBuilder("spark.task.reaper.killTimeout")
-      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(-1)
 
   private[spark] val TASK_REAPER_POLLING_INTERVAL =
     ConfigBuilder("spark.task.reaper.pollingInterval")
-      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val TASK_REAPER_THREAD_DUMP =
     ConfigBuilder("spark.task.reaper.threadDump")
-      .version("2.0.3")
       .booleanConf
       .createWithDefault(true)
 
   // Blacklist confs
   private[spark] val BLACKLIST_ENABLED =
     ConfigBuilder("spark.blacklist.enabled")
-      .version("2.1.0")
       .booleanConf
       .createOptional
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerExecutor")
-      .version("2.1.0")
       .intConf
       .createWithDefault(1)
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerNode")
-      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC =
     ConfigBuilder("spark.blacklist.application.maxFailedTasksPerExecutor")
-      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedTasksPerExecutor")
-      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE =
     ConfigBuilder("spark.blacklist.application.maxFailedExecutorsPerNode")
-      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedExecutorsPerNode")
-      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val BLACKLIST_TIMEOUT_CONF =
     ConfigBuilder("spark.blacklist.timeout")
-      .version("2.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_KILL_ENABLED =
     ConfigBuilder("spark.blacklist.killBlacklistedExecutors")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val BLACKLIST_LEGACY_TIMEOUT_CONF =
     ConfigBuilder("spark.scheduler.executorTaskBlacklistTime")
       .internal()
-      .version("1.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_FETCH_FAILURE_ENABLED =
     ConfigBuilder("spark.blacklist.application.fetchFailure.enabled")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
   // End blacklist confs
@@ -731,7 +595,6 @@ package object config {
       .doc("Whether to un-register all the outputs on the host in condition that we receive " +
         " a FetchFailure. This is set default to false, which means, we only un-register the " +
         " outputs related to the exact executor(instead of the host) on a FetchFailure.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -741,7 +604,6 @@ package object config {
         "an event queue using capacity specified by `spark.scheduler.listenerbus" +
         ".eventqueue.queueName.capacity` first. If it's not configured, Spark will " +
         "use the default capacity specified by this config.")
-      .version("2.3.0")
       .intConf
       .checkValue(_ > 0, "The capacity of listener bus event queue must be positive")
       .createWithDefault(10000)
@@ -749,7 +611,6 @@ package object config {
   private[spark] val LISTENER_BUS_METRICS_MAX_LISTENER_CLASSES_TIMED =
     ConfigBuilder("spark.scheduler.listenerbus.metrics.maxListenerClassesTimed")
       .internal()
-      .version("2.3.0")
       .intConf
       .createWithDefault(128)
 
@@ -759,7 +620,6 @@ package object config {
       .doc("When enabled, log the event that takes too much time to process. This helps us " +
         "discover the event types that cause performance bottlenecks. The time threshold is " +
         "controlled by spark.scheduler.listenerbus.logSlowEvent.threshold.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -768,65 +628,52 @@ package object config {
       .internal()
       .doc("The time threshold of whether a event is considered to be taking too much time to " +
         s"process. Log the event if ${LISTENER_BUS_LOG_SLOW_EVENT_ENABLED.key} is true.")
-      .version("3.0.0")
       .timeConf(TimeUnit.NANOSECONDS)
       .createWithDefaultString("1s")
 
   // This property sets the root namespace for metrics reporting
   private[spark] val METRICS_NAMESPACE = ConfigBuilder("spark.metrics.namespace")
-    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_CONF = ConfigBuilder("spark.metrics.conf")
-    .version("0.8.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_EXECUTORMETRICS_SOURCE_ENABLED =
     ConfigBuilder("spark.metrics.executorMetricsSource.enabled")
       .doc("Whether to register the ExecutorMetrics source with the metrics system.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val METRICS_STATIC_SOURCES_ENABLED =
     ConfigBuilder("spark.metrics.staticSources.enabled")
       .doc("Whether to register static sources with the metrics system.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val PYSPARK_DRIVER_PYTHON = ConfigBuilder("spark.pyspark.driver.python")
-    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python")
-    .version("2.1.0")
     .stringConf
     .createOptional
 
   // To limit how many applications are shown in the History Server summary ui
   private[spark] val HISTORY_UI_MAX_APPS =
-    ConfigBuilder("spark.history.ui.maxApplications")
-      .version("2.0.1")
-      .intConf
-      .createWithDefault(Integer.MAX_VALUE)
+    ConfigBuilder("spark.history.ui.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val IO_ENCRYPTION_ENABLED = ConfigBuilder("spark.io.encryption.enabled")
-    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IO_ENCRYPTION_KEYGEN_ALGORITHM =
     ConfigBuilder("spark.io.encryption.keygen.algorithm")
-      .version("2.1.0")
       .stringConf
       .createWithDefault("HmacSHA1")
 
   private[spark] val IO_ENCRYPTION_KEY_SIZE_BITS = ConfigBuilder("spark.io.encryption.keySizeBits")
-    .version("2.1.0")
     .intConf
     .checkValues(Set(128, 192, 256))
     .createWithDefault(128)
@@ -834,68 +681,57 @@ package object config {
   private[spark] val IO_CRYPTO_CIPHER_TRANSFORMATION =
     ConfigBuilder("spark.io.crypto.cipher.transformation")
       .internal()
-      .version("2.1.0")
       .stringConf
       .createWithDefaultString("AES/CTR/NoPadding")
 
   private[spark] val DRIVER_HOST_ADDRESS = ConfigBuilder("spark.driver.host")
     .doc("Address of driver endpoints.")
-    .version("0.7.0")
     .stringConf
     .createWithDefault(Utils.localCanonicalHostName())
 
   private[spark] val DRIVER_PORT = ConfigBuilder("spark.driver.port")
     .doc("Port of driver endpoints.")
-    .version("0.7.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_SUPERVISE = ConfigBuilder("spark.driver.supervise")
     .doc("If true, restarts the driver automatically if it fails with a non-zero exit status. " +
       "Only has effect in Spark standalone mode or Mesos cluster deploy mode.")
-    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val DRIVER_BIND_ADDRESS = ConfigBuilder("spark.driver.bindAddress")
     .doc("Address where to bind network listen sockets on the driver.")
-    .version("2.1.0")
     .fallbackConf(DRIVER_HOST_ADDRESS)
 
   private[spark] val BLOCK_MANAGER_PORT = ConfigBuilder("spark.blockManager.port")
     .doc("Port to use for the block manager when a more specific setting is not provided.")
-    .version("1.1.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_BLOCK_MANAGER_PORT = ConfigBuilder("spark.driver.blockManager.port")
     .doc("Port to use for the block manager on the driver.")
-    .version("2.1.0")
     .fallbackConf(BLOCK_MANAGER_PORT)
 
   private[spark] val IGNORE_CORRUPT_FILES = ConfigBuilder("spark.files.ignoreCorruptFiles")
     .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " +
       "encountering corrupted or non-existing files and contents that have been read will still " +
       "be returned.")
-    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IGNORE_MISSING_FILES = ConfigBuilder("spark.files.ignoreMissingFiles")
     .doc("Whether to ignore missing files. If true, the Spark jobs will continue to run when " +
       "encountering missing files and the contents that have been read will still be returned.")
-    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val APP_CALLER_CONTEXT = ConfigBuilder("spark.log.callerContext")
-    .version("2.2.0")
     .stringConf
     .createOptional
 
   private[spark] val FILES_MAX_PARTITION_BYTES = ConfigBuilder("spark.files.maxPartitionBytes")
     .doc("The maximum number of bytes to pack into a single partition when reading files.")
-    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(128 * 1024 * 1024)
 
@@ -904,7 +740,6 @@ package object config {
       " the same time. This is used when putting multiple files into a partition. It's better to" +
       " over estimate, then the partitions with small files will be faster than partitions with" +
       " bigger files.")
-    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(4 * 1024 * 1024)
 
@@ -912,7 +747,6 @@ package object config {
     ConfigBuilder("spark.hadoopRDD.ignoreEmptySplits")
       .internal()
       .doc("When true, HadoopRDD/NewHadoopRDD will not create partitions for empty input splits.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -922,7 +756,6 @@ package object config {
         "driver and executor environments contain sensitive information. When this regex matches " +
         "a property key or value, the value is redacted from the environment UI and various logs " +
         "like YARN and event logs.")
-      .version("2.1.2")
       .regexConf
       .createWithDefault("(?i)secret|password|token".r)
 
@@ -931,31 +764,26 @@ package object config {
       .doc("Regex to decide which parts of strings produced by Spark contain sensitive " +
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands.")
-      .version("2.2.0")
       .regexConf
       .createOptional
 
   private[spark] val AUTH_SECRET =
     ConfigBuilder("spark.authenticate.secret")
-      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val AUTH_SECRET_BIT_LENGTH =
     ConfigBuilder("spark.authenticate.secretBitLength")
-      .version("1.6.0")
       .intConf
       .createWithDefault(256)
 
   private[spark] val NETWORK_AUTH_ENABLED =
     ConfigBuilder("spark.authenticate")
-      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SASL_ENCRYPTION_ENABLED =
     ConfigBuilder("spark.authenticate.enableSaslEncryption")
-      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -965,7 +793,6 @@ package object config {
         "loaded from this path on both the driver and the executors if overrides are not set for " +
         "either entity (see below). File-based secret keys are only allowed when using " +
         "Kubernetes.")
-      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -978,7 +805,6 @@ package object config {
         "be specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
-      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val AUTH_SECRET_FILE_EXECUTOR =
@@ -990,14 +816,12 @@ package object config {
         "specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
-      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val BUFFER_WRITE_CHUNK_SIZE =
     ConfigBuilder("spark.buffer.write.chunkSize")
       .internal()
       .doc("The chunk size in bytes during writing out the bytes of ChunkedByteBuffer.")
-      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         "The chunk size during writing out the bytes of ChunkedByteBuffer should" +
@@ -1008,7 +832,6 @@ package object config {
     ConfigBuilder("spark.checkpoint.compress")
       .doc("Whether to compress RDD checkpoints. Generally a good idea. Compression will use " +
         "spark.io.compression.codec.")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1019,7 +842,6 @@ package object config {
         "Caching preferred locations can relieve query loading to DFS and save the query " +
         "time. The drawback is that the cached locations can be possibly outdated and " +
         "lose data locality. If this config is not specified, it will not cache.")
-      .version("3.0.0")
       .timeConf(TimeUnit.MINUTES)
       .checkValue(_ > 0, "The expire time for caching preferred locations cannot be non-positive.")
       .createOptional
@@ -1029,14 +851,12 @@ package object config {
       .doc("Threshold in bytes above which the size of shuffle blocks in " +
         "HighlyCompressedMapStatus is accurately recorded. This helps to prevent OOM " +
         "by avoiding underestimating shuffle block size when fetch shuffle blocks.")
-      .version("2.2.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(100 * 1024 * 1024)
 
   private[spark] val SHUFFLE_REGISTRATION_TIMEOUT =
     ConfigBuilder("spark.shuffle.registration.timeout")
       .doc("Timeout in milliseconds for registration to the external shuffle service.")
-      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(5000)
 
@@ -1044,7 +864,6 @@ package object config {
     ConfigBuilder("spark.shuffle.registration.maxAttempts")
       .doc("When we fail to register to the external shuffle service, we will " +
         "retry for maxAttempts times.")
-      .version("2.3.0")
       .intConf
       .createWithDefault(3)
 
@@ -1055,7 +874,6 @@ package object config {
         "address in a single fetch or simultaneously, this could crash the serving executor or " +
         "Node Manager. This is especially useful to reduce the load on the Node Manager when " +
         "external shuffle is enabled. You can mitigate the issue by setting it to a lower value.")
-      .version("2.2.1")
       .intConf
       .checkValue(_ > 0, "The max no. of blocks in flight cannot be non-positive.")
       .createWithDefault(Int.MaxValue)
@@ -1067,7 +885,6 @@ package object config {
         "configuration will affect both shuffle fetch and block manager remote block fetch. " +
         "For users who enabled external shuffle service, this feature can only work when " +
         "external shuffle service is at least 2.3.0.")
-      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       // fetch-to-mem is guaranteed to fail if the message is bigger than 2 GB, so we might
       // as well use fetch-to-disk in that case.  The message includes some metadata in addition
@@ -1083,14 +900,12 @@ package object config {
       .doc("Enable tracking of updatedBlockStatuses in the TaskMetrics. Off by default since " +
         "tracking the block statuses can use a lot of memory and its not used anywhere within " +
         "spark.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_IO_PLUGIN_CLASS =
     ConfigBuilder("spark.shuffle.sort.io.plugin.class")
       .doc("Name of the class to use for shuffle IO.")
-      .version("3.0.0")
       .stringConf
       .createWithDefault(classOf[LocalDiskShuffleDataIO].getName)
 
@@ -1099,7 +914,6 @@ package object config {
       .doc("Size of the in-memory buffer for each shuffle file output stream, in KiB unless " +
         "otherwise specified. These buffers reduce the number of disk seeks and system calls " +
         "made in creating intermediate shuffle files.")
-      .version("1.4.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The file buffer size must be positive and less than or equal to" +
@@ -1110,7 +924,6 @@ package object config {
     ConfigBuilder("spark.shuffle.unsafe.file.output.buffer")
       .doc("The file system for this buffer size after each partition " +
         "is written in unsafe shuffle writer. In KiB unless otherwise specified.")
-      .version("2.3.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The buffer size must be positive and less than or equal to" +
@@ -1120,7 +933,6 @@ package object config {
   private[spark] val SHUFFLE_DISK_WRITE_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.spill.diskWriteBufferSize")
       .doc("The buffer size, in bytes, to use when writing the sorted records to an on-disk file.")
-      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 12 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         s"The buffer size must be greater than 12 and less than or equal to " +
@@ -1132,7 +944,6 @@ package object config {
       .internal()
       .doc("The memory check period is used to determine how often we should check whether "
         + "there is a need to request more memory when we try to unroll the given block in memory.")
-      .version("2.3.0")
       .longConf
       .createWithDefault(16)
 
@@ -1140,7 +951,6 @@ package object config {
     ConfigBuilder("spark.storage.unrollMemoryGrowthFactor")
       .internal()
       .doc("Memory to request as a multiple of the size that used to unroll the block.")
-      .version("2.3.0")
       .doubleConf
       .createWithDefault(1.5)
 
@@ -1151,14 +961,12 @@ package object config {
         "where the YARN service does not support schemes that are supported by Spark, like http, " +
         "https and ftp, or jars required to be in the local YARN client's classpath. Wildcard " +
         "'*' is denoted to download resources for all the schemes.")
-      .version("2.3.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val EXTRA_LISTENERS = ConfigBuilder("spark.extraListeners")
     .doc("Class names of listeners to add to SparkContext during initialization.")
-    .version("1.3.0")
     .stringConf
     .toSequence
     .createOptional
@@ -1170,7 +978,6 @@ package object config {
         "By default it's Integer.MAX_VALUE, which means we never force the sorter to spill, " +
         "until we reach some limitations, like the max page size limitation for the pointer " +
         "array in the sorter.")
-      .version("1.6.0")
       .intConf
       .createWithDefault(Integer.MAX_VALUE)
 
@@ -1180,35 +987,30 @@ package object config {
       .doc("Multi-thread is used when the number of mappers * shuffle partitions is greater than " +
         "or equal to this threshold. Note that the actual parallelism is calculated by number of " +
         "mappers * shuffle partitions / this threshold + 1, so this threshold should be positive.")
-      .version("2.3.0")
       .intConf
       .checkValue(v => v > 0, "The threshold should be positive.")
       .createWithDefault(10000000)
 
   private[spark] val MAX_RESULT_SIZE = ConfigBuilder("spark.driver.maxResultSize")
     .doc("Size limit for results.")
-    .version("1.2.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("1g")
 
   private[spark] val CREDENTIALS_RENEWAL_INTERVAL_RATIO =
     ConfigBuilder("spark.security.credentials.renewalRatio")
       .doc("Ratio of the credential's expiration time when Spark should fetch new credentials.")
-      .version("2.4.0")
       .doubleConf
       .createWithDefault(0.75d)
 
   private[spark] val CREDENTIALS_RENEWAL_RETRY_WAIT =
     ConfigBuilder("spark.security.credentials.retryWait")
       .doc("How long to wait before retrying to fetch new credentials after a failure.")
-      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("1h")
 
   private[spark] val SHUFFLE_SORT_INIT_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.sort.initialBufferSize")
       .internal()
-      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 0 && v <= Int.MaxValue,
         s"The buffer size must be greater than 0 and less than or equal to ${Int.MaxValue}.")
@@ -1218,7 +1020,6 @@ package object config {
     ConfigBuilder("spark.shuffle.compress")
       .doc("Whether to compress shuffle output. Compression will use " +
         "spark.io.compression.codec.")
-      .version("0.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1226,7 +1027,6 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.compress")
       .doc("Whether to compress data spilled during shuffles. Compression will use " +
         "spark.io.compression.codec.")
-      .version("0.9.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1236,7 +1036,6 @@ package object config {
       .doc("The codec used to compress MapStatus, which is generated by ShuffleMapTask. " +
         "By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also " +
         "use fully qualified class names to specify the codec.")
-      .version("3.0.0")
       .stringConf
       .createWithDefault("zstd")
 
@@ -1245,7 +1044,6 @@ package object config {
       .internal()
       .doc("Initial threshold for the size of a collection before we start tracking its " +
         "memory usage.")
-      .version("1.1.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(5 * 1024 * 1024)
 
@@ -1253,7 +1051,6 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.batchSize")
       .internal()
       .doc("Size of object batches when reading/writing from serializers.")
-      .version("0.9.0")
       .longConf
       .createWithDefault(10000)
 
@@ -1261,40 +1058,34 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.bypassMergeThreshold")
       .doc("In the sort-based shuffle manager, avoid merge-sorting data if there is no " +
         "map-side aggregation and there are at most this many reduce partitions")
-      .version("1.1.1")
       .intConf
       .createWithDefault(200)
 
   private[spark] val SHUFFLE_MANAGER =
     ConfigBuilder("spark.shuffle.manager")
-      .version("1.1.0")
       .stringConf
       .createWithDefault("sort")
 
   private[spark] val SHUFFLE_REDUCE_LOCALITY_ENABLE =
     ConfigBuilder("spark.shuffle.reduceLocality.enabled")
       .doc("Whether to compute locality preferences for reduce tasks")
-      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST =
     ConfigBuilder("spark.shuffle.mapOutput.minSizeForBroadcast")
       .doc("The size at which we use Broadcast to send the map output statuses to the executors.")
-      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("512k")
 
   private[spark] val SHUFFLE_MAPOUTPUT_DISPATCHER_NUM_THREADS =
     ConfigBuilder("spark.shuffle.mapOutput.dispatcher.numThreads")
-      .version("2.0.0")
       .intConf
       .createWithDefault(8)
 
   private[spark] val SHUFFLE_DETECT_CORRUPT =
     ConfigBuilder("spark.shuffle.detectCorrupt")
       .doc("Whether to detect any corruption in fetched blocks.")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1304,21 +1095,18 @@ package object config {
         "by using extra memory to detect early corruption. Any IOException thrown will cause " +
         "the task to be retried once and if it fails again with same exception, then " +
         "FetchFailedException will be thrown to retry previous stage")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_SYNC =
     ConfigBuilder("spark.shuffle.sync")
       .doc("Whether to force outstanding writes to disk.")
-      .version("0.8.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_UNSAFE_FAST_MERGE_ENABLE =
     ConfigBuilder("spark.shuffle.unsafe.fastMergeEnabled")
       .doc("Whether to perform a fast spill merge.")
-      .version("1.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1326,7 +1114,6 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.useRadixSort")
       .doc("Whether to use radix sort for sorting in-memory partition ids. Radix sort is much " +
         "faster, but requires additional memory to be reserved memory as pointers are added.")
-      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1334,7 +1121,6 @@ package object config {
     ConfigBuilder("spark.shuffle.minNumPartitionsToHighlyCompress")
       .internal()
       .doc("Number of partitions to determine if MapStatus should use HighlyCompressedMapStatus")
-      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The value should be a positive integer.")
       .createWithDefault(2000)
@@ -1344,7 +1130,6 @@ package object config {
       .doc("Whether to use the old protocol while doing the shuffle block fetching. " +
         "It is only enabled while we need the compatibility in the scenario of new Spark " +
         "version job fetching shuffle blocks from old version external shuffle service.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1354,7 +1139,6 @@ package object config {
         s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " +
         "blocks requested from those block managers which are running on the same host are read " +
         "from the disk directly instead of being fetched as remote blocks over the network.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1365,7 +1149,6 @@ package object config {
         "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
         s"persisted RDD blocks or shuffle blocks " +
         s"(when `${SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED.key}` is set) from the same host.")
-      .version("3.0.0")
       .intConf
       .createWithDefault(1000)
 
@@ -1373,7 +1156,6 @@ package object config {
     ConfigBuilder("spark.storage.memoryMapLimitForTests")
       .internal()
       .doc("For testing only, controls the size of chunks when memory mapping a file")
-      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1383,7 +1165,6 @@ package object config {
         "coordinator didn't receive all the sync messages from barrier tasks within the " +
         "configured time, throw a SparkException to fail all the tasks. The default value is set " +
         "to 31536000(3600 * 24 * 365) so the barrier() call shall wait for one year.")
-      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v > 0, "The value should be a positive time value.")
       .createWithDefaultString("365d")
@@ -1392,7 +1173,6 @@ package object config {
     ConfigBuilder("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
       .doc("The timeout in seconds to wait to acquire a new executor and schedule a task " +
         "before aborting a TaskSet which is unschedulable because of being completely blacklisted.")
-      .version("2.4.1")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v >= 0, "The value should be a non negative time value.")
       .createWithDefault(120)
@@ -1407,7 +1187,6 @@ package object config {
         "configured max failure times for a job then fail current job submission. Note this " +
         "config only applies to jobs that contain one or more barrier stages, we won't perform " +
         "the check on non-barrier jobs.")
-      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("15s")
 
@@ -1421,7 +1200,6 @@ package object config {
         "max failure times for a job then fail current job submission. Note this config only " +
         "applies to jobs that contain one or more barrier stages, we won't perform the check on " +
         "non-barrier jobs.")
-      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The max failures should be a positive value.")
       .createWithDefault(40)
@@ -1429,21 +1207,18 @@ package object config {
   private[spark] val UNSAFE_EXCEPTION_ON_MEMORY_LEAK =
     ConfigBuilder("spark.unsafe.exceptionOnMemoryLeak")
       .internal()
-      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val UNSAFE_SORTER_SPILL_READ_AHEAD_ENABLED =
     ConfigBuilder("spark.unsafe.sorter.spill.read.ahead.enabled")
       .internal()
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val UNSAFE_SORTER_SPILL_READER_BUFFER_SIZE =
     ConfigBuilder("spark.unsafe.sorter.spill.reader.buffer.size")
       .internal()
-      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => 1024 * 1024 <= v && v <= MAX_BUFFER_SIZE_BYTES,
         s"The value must be in allowed range [1,048,576, ${MAX_BUFFER_SIZE_BYTES}].")
@@ -1456,83 +1231,63 @@ package object config {
       .withPrepended(DEFAULT_PLUGINS_LIST, separator = ",")
       .doc("Comma-separated list of class names implementing " +
         "org.apache.spark.api.plugin.SparkPlugin to load into the application.")
-      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val CLEANER_PERIODIC_GC_INTERVAL =
     ConfigBuilder("spark.cleaner.periodicGC.interval")
-      .version("1.6.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("30min")
 
   private[spark] val CLEANER_REFERENCE_TRACKING =
     ConfigBuilder("spark.cleaner.referenceTracking")
-      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking")
-      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING_SHUFFLE =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking.shuffle")
-      .version("1.1.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_CLEAN_CHECKPOINTS =
     ConfigBuilder("spark.cleaner.referenceTracking.cleanCheckpoints")
-      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_STRATEGY =
-    ConfigBuilder("spark.executor.logs.rolling.strategy")
-      .version("1.1.0")
-      .stringConf
-      .createWithDefault("")
+    ConfigBuilder("spark.executor.logs.rolling.strategy").stringConf.createWithDefault("")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_TIME_INTERVAL =
-    ConfigBuilder("spark.executor.logs.rolling.time.interval")
-      .version("1.1.0")
-      .stringConf
-      .createWithDefault("daily")
+    ConfigBuilder("spark.executor.logs.rolling.time.interval").stringConf.createWithDefault("daily")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_SIZE =
     ConfigBuilder("spark.executor.logs.rolling.maxSize")
-      .version("1.4.0")
       .stringConf
       .createWithDefault((1024 * 1024).toString)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_RETAINED_FILES =
-    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles")
-      .version("1.1.0")
-      .intConf
-      .createWithDefault(-1)
+    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles").intConf.createWithDefault(-1)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_ENABLE_COMPRESSION =
     ConfigBuilder("spark.executor.logs.rolling.enableCompression")
-      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_ENABLED = ConfigBuilder("spark.master.rest.enabled")
-    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_PORT = ConfigBuilder("spark.master.rest.port")
-    .version("1.3.0")
     .intConf
     .createWithDefault(6066)
 
   private[spark] val MASTER_UI_PORT = ConfigBuilder("spark.master.ui.port")
-    .version("1.1.0")
     .intConf
     .createWithDefault(8080)
 
@@ -1541,7 +1296,6 @@ package object config {
       .doc("Block size in bytes used in Snappy compression, in the case when " +
         "Snappy compression codec is used. Lowering this block size " +
         "will also lower shuffle memory usage when Snappy is used")
-      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1550,7 +1304,6 @@ package object config {
       .doc("Block size in bytes used in LZ4 compression, in the case when LZ4 compression" +
         "codec is used. Lowering this block size will also lower shuffle memory " +
         "usage when LZ4 is used.")
-      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1560,7 +1313,6 @@ package object config {
         "broadcast variables and shuffle outputs. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec")
-      .version("0.8.0")
       .stringConf
       .createWithDefaultString("lz4")
 
@@ -1570,7 +1322,6 @@ package object config {
         "compression codec is used. Lowering this size will lower the shuffle " +
         "memory usage when Zstd is used, but it might increase the compression " +
         "cost because of excessive JNI call overhead")
-      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1578,7 +1329,6 @@ package object config {
     ConfigBuilder("spark.io.compression.zstd.level")
       .doc("Compression level for Zstd compression codec. Increasing the compression " +
         "level will result in better compression at the expense of more CPU and memory")
-      .version("2.3.0")
       .intConf
       .createWithDefault(1)
 
@@ -1587,7 +1337,6 @@ package object config {
       .internal()
       .doc("If the size in bytes of a file loaded by Spark exceeds this threshold, " +
         "a warning is logged with the possible reasons.")
-      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1024 * 1024 * 1024)
 
@@ -1596,34 +1345,28 @@ package object config {
       .doc("The codec used to compress event log. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec. If this is not given, spark.io.compression.codec will be used.")
-      .version("3.0.0")
       .fallbackConf(IO_COMPRESSION_CODEC)
 
   private[spark] val BUFFER_SIZE =
     ConfigBuilder("spark.buffer.size")
-      .version("0.5.0")
       .intConf
       .checkValue(_ >= 0, "The buffer size must not be negative")
       .createWithDefault(65536)
 
   private[spark] val LOCALITY_WAIT_PROCESS = ConfigBuilder("spark.locality.wait.process")
-    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_NODE = ConfigBuilder("spark.locality.wait.node")
-    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_RACK = ConfigBuilder("spark.locality.wait.rack")
-    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
-  private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
+    private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
     .doc("Maximum size of map outputs to fetch simultaneously from each reduce task, " +
       "in MiB unless otherwise specified. Since each output requires us to create a " +
       "buffer to receive it, this represents a fixed memory overhead per reduce task, " +
       "so keep it small unless you have a large amount of memory")
-    .version("1.4.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("48m")
 
@@ -1633,14 +1376,12 @@ package object config {
       "it might lead to very large number of inbound connections to one or more nodes, " +
       "causing the workers to fail under load. By allowing it to limit the number of " +
       "fetch requests, this scenario can be mitigated")
-    .version("2.0.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
   private[spark] val BROADCAST_COMPRESS = ConfigBuilder("spark.broadcast.compress")
     .doc("Whether to compress broadcast variables before sending them. " +
       "Generally a good idea. Compression will use spark.io.compression.codec")
-    .version("0.6.0")
     .booleanConf.createWithDefault(true)
 
   private[spark] val BROADCAST_BLOCKSIZE = ConfigBuilder("spark.broadcast.blockSize")
@@ -1648,7 +1389,6 @@ package object config {
       "KiB unless otherwise specified. Too large a value decreases " +
       "parallelism during broadcast (makes it slower); however, " +
       "if it is too small, BlockManager might take a performance hit")
-    .version("0.5.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("4m")
 
@@ -1658,14 +1398,12 @@ package object config {
       "corrupted blocks, at the cost of computing and sending a little " +
       "more data. It's possible to disable it if the network has other " +
       "mechanisms to guarantee data won't be corrupted during broadcast")
-    .version("2.1.1")
     .booleanConf.createWithDefault(true)
 
   private[spark] val BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD =
     ConfigBuilder("spark.broadcast.UDFCompressionThreshold")
       .doc("The threshold at which user-defined functions (UDFs) and Python RDD commands " +
         "are compressed by broadcast in bytes unless otherwise specified")
-      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v >= 0, "The threshold should be non-negative.")
       .createWithDefault(1L * 1024 * 1024)
@@ -1676,111 +1414,92 @@ package object config {
       "or StorageLevel.MEMORY_ONLY in Python). Can save substantial " +
       "space at the cost of some extra CPU time. " +
       "Compression will use spark.io.compression.codec")
-    .version("0.6.0")
     .booleanConf.createWithDefault(false)
 
   private[spark] val RDD_PARALLEL_LISTING_THRESHOLD =
     ConfigBuilder("spark.rdd.parallelListingThreshold")
-      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
   private[spark] val RDD_LIMIT_SCALE_UP_FACTOR =
     ConfigBuilder("spark.rdd.limit.scaleUpFactor")
-      .version("2.1.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val SERIALIZER = ConfigBuilder("spark.serializer")
-    .version("0.5.0")
     .stringConf
     .createWithDefault("org.apache.spark.serializer.JavaSerializer")
 
   private[spark] val SERIALIZER_OBJECT_STREAM_RESET =
     ConfigBuilder("spark.serializer.objectStreamReset")
-      .version("1.0.0")
       .intConf
       .createWithDefault(100)
 
   private[spark] val SERIALIZER_EXTRA_DEBUG_INFO = ConfigBuilder("spark.serializer.extraDebugInfo")
-    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
   private[spark] val JARS = ConfigBuilder("spark.jars")
-    .version("0.9.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val FILES = ConfigBuilder("spark.files")
-    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SUBMIT_DEPLOY_MODE = ConfigBuilder("spark.submit.deployMode")
-    .version("1.5.0")
     .stringConf
     .createWithDefault("client")
 
   private[spark] val SUBMIT_PYTHON_FILES = ConfigBuilder("spark.submit.pyFiles")
-    .version("1.0.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SCHEDULER_ALLOCATION_FILE =
     ConfigBuilder("spark.scheduler.allocation.file")
-      .version("0.8.1")
       .stringConf
       .createOptional
 
   private[spark] val SCHEDULER_MIN_REGISTERED_RESOURCES_RATIO =
     ConfigBuilder("spark.scheduler.minRegisteredResourcesRatio")
-      .version("1.1.1")
       .doubleConf
       .createOptional
 
   private[spark] val SCHEDULER_MAX_REGISTERED_RESOURCE_WAITING_TIME =
     ConfigBuilder("spark.scheduler.maxRegisteredResourcesWaitingTime")
-      .version("1.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("30s")
 
   private[spark] val SCHEDULER_MODE =
     ConfigBuilder("spark.scheduler.mode")
-      .version("0.8.0")
       .stringConf
       .createWithDefault(SchedulingMode.FIFO.toString)
 
   private[spark] val SCHEDULER_REVIVE_INTERVAL =
     ConfigBuilder("spark.scheduler.revive.interval")
-      .version("0.8.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val SPECULATION_ENABLED =
     ConfigBuilder("spark.speculation")
-      .version("0.6.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SPECULATION_INTERVAL =
     ConfigBuilder("spark.speculation.interval")
-      .version("0.6.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
   private[spark] val SPECULATION_MULTIPLIER =
     ConfigBuilder("spark.speculation.multiplier")
-      .version("0.6.0")
       .doubleConf
       .createWithDefault(1.5)
 
   private[spark] val SPECULATION_QUANTILE =
     ConfigBuilder("spark.speculation.quantile")
-      .version("0.6.0")
       .doubleConf
       .createWithDefault(0.75)
 
@@ -1794,19 +1513,16 @@ package object config {
         "large enough. E.g. tasks might be re-launched if there are enough successful runs " +
         "even though the threshold hasn't been reached. The number of slots is computed based " +
         "on the conf values of spark.executor.cores and spark.task.cpus minimum 1.")
-      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val STAGING_DIR = ConfigBuilder("spark.yarn.stagingDir")
     .doc("Staging directory used while submitting applications.")
-    .version("2.0.0")
     .stringConf
     .createOptional
 
   private[spark] val BUFFER_PAGESIZE = ConfigBuilder("spark.buffer.pageSize")
     .doc("The amount of memory used per page in bytes")
-    .version("1.5.0")
     .bytesConf(ByteUnit.BYTE)
     .createOptional
 

From 26d84653ed886e66067b25bb784f7dd5c289c53b Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 7 Apr 2020 20:29:22 +0900
Subject: [PATCH 0439/1280] Revert "[SPARK-30841][SQL][DOC][3.0] Add version
 information to the configuration of SQL"

This reverts commit af7fa9d12622c3b2b169c4a58d143bc21ed9e784.
---
 .../apache/spark/sql/internal/SQLConf.scala   | 267 +-----------------
 1 file changed, 6 insertions(+), 261 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 7a10aba7ef6e7..9f9e55601d69e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -177,7 +177,6 @@ object SQLConf {
   val ANALYZER_MAX_ITERATIONS = buildConf("spark.sql.analyzer.maxIterations")
     .internal()
     .doc("The max number of iterations the analyzer runs.")
-    .version("3.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -186,14 +185,12 @@ object SQLConf {
       "specified by their rule names and separated by comma. It is not guaranteed that all the " +
       "rules in this configuration will eventually be excluded, as some rules are necessary " +
       "for correctness. The optimizer will log the rules that have indeed been excluded.")
-    .version("2.4.0")
     .stringConf
     .createOptional
 
   val OPTIMIZER_MAX_ITERATIONS = buildConf("spark.sql.optimizer.maxIterations")
     .internal()
     .doc("The max number of iterations the optimizer runs.")
-    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -201,7 +198,6 @@ object SQLConf {
     buildConf("spark.sql.optimizer.inSetConversionThreshold")
       .internal()
       .doc("The threshold of set size for InSet conversion.")
-      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -210,7 +206,6 @@ object SQLConf {
       .internal()
       .doc("Configures the max set size in InSet for which Spark will generate code with " +
         "switch statements. This is applicable only to bytes, shorts, ints, dates.")
-      .version("3.0.0")
       .intConf
       .checkValue(threshold => threshold >= 0 && threshold <= 600, "The max set size " +
         "for using switch statements in InSet must be non-negative and less than or equal to 600")
@@ -221,7 +216,6 @@ object SQLConf {
     .doc("Configures the log level for logging the change from the original plan to the new " +
       "plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', " +
       "'warn', or 'error'. The default log level is 'trace'.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValue(logLevel => Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR").contains(logLevel),
@@ -233,7 +227,6 @@ object SQLConf {
     .internal()
     .doc("Configures a list of rules to be logged in the optimizer, in which the rules are " +
       "specified by their rule names and separated by comma.")
-    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -241,14 +234,12 @@ object SQLConf {
     .internal()
     .doc("Configures a list of batches to be logged in the optimizer, in which the batches " +
       "are specified by their batch names and separated by comma.")
-    .version("3.0.0")
     .stringConf
     .createOptional
 
   val DYNAMIC_PARTITION_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.dynamicPartitionPruning.enabled")
       .doc("When true, we will generate predicate for partition column when it's used as join key")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -258,7 +249,6 @@ object SQLConf {
       .doc("When true, distinct count statistics will be used for computing the data size of the " +
         "partitioned table after dynamic partition pruning, in order to evaluate if it is worth " +
         "adding an extra subquery as the pruning filter if broadcast reuse is not applicable.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -269,7 +259,6 @@ object SQLConf {
       "used as the fallback filter ratio for computing the data size of the partitioned table " +
       "after dynamic partition pruning, in order to evaluate if it is worth adding an extra " +
       "subquery as the pruning filter if broadcast reuse is not applicable.")
-    .version("3.0.0")
     .doubleConf
     .createWithDefault(0.5)
 
@@ -278,21 +267,18 @@ object SQLConf {
       .internal()
       .doc("When true, dynamic partition pruning will only apply when the broadcast exchange of " +
         "a broadcast hash join operation can be reused as the dynamic pruning filter.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val COMPRESS_CACHED = buildConf("spark.sql.inMemoryColumnarStorage.compressed")
     .doc("When set to true Spark SQL will automatically select a compression codec for each " +
       "column based on statistics of the data.")
-    .version("1.0.1")
     .booleanConf
     .createWithDefault(true)
 
   val COLUMN_BATCH_SIZE = buildConf("spark.sql.inMemoryColumnarStorage.batchSize")
     .doc("Controls the size of batches for columnar caching.  Larger batch sizes can improve " +
       "memory utilization and compression, but risk OOMs when caching data.")
-    .version("1.1.1")
     .intConf
     .createWithDefault(10000)
 
@@ -300,7 +286,6 @@ object SQLConf {
     buildConf("spark.sql.inMemoryColumnarStorage.partitionPruning")
       .internal()
       .doc("When true, enable partition pruning for in-memory columnar tables.")
-      .version("1.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -308,14 +293,12 @@ object SQLConf {
     buildConf("spark.sql.inMemoryTableScanStatistics.enable")
       .internal()
       .doc("When true, enable in-memory table scan accumulators.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val CACHE_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.inMemoryColumnarStorage.enableVectorizedReader")
       .doc("Enables vectorized reader for columnar caching.")
-      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -323,14 +306,12 @@ object SQLConf {
     buildConf("spark.sql.columnVector.offheap.enabled")
       .internal()
       .doc("When true, use OffHeapColumnVector in ColumnarBatch.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val PREFER_SORTMERGEJOIN = buildConf("spark.sql.join.preferSortMergeJoin")
     .internal()
     .doc("When true, prefer sort merge join over shuffle hash join.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -339,7 +320,6 @@ object SQLConf {
     .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " +
       "requires additional memory to be reserved up-front. The memory overhead may be " +
       "significant when sorting very small rows (up to 50% more in this case).")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -350,7 +330,6 @@ object SQLConf {
       "command `ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan` has been " +
       "run, and file-based data source tables where the statistics are computed directly on " +
       "the files of data.")
-    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("10MB")
 
@@ -359,7 +338,6 @@ object SQLConf {
     .doc("Minimal increase rate in number of partitions between attempts when executing a take " +
       "on a query. Higher values lead to more partitions read. Lower values might lead to " +
       "longer execution times as more jobs will be run")
-    .version("2.1.1")
     .intConf
     .createWithDefault(4)
 
@@ -367,7 +345,6 @@ object SQLConf {
     buildConf("spark.sql.hive.advancedPartitionPredicatePushdown.enabled")
       .internal()
       .doc("When true, advanced partition predicate pushdown into Hive metastore is enabled.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -375,7 +352,6 @@ object SQLConf {
     .doc("The default number of partitions to use when shuffling data for joins or aggregations. " +
       "Note: For structured streaming, this configuration cannot be changed between query " +
       "restarts from the same checkpoint location.")
-    .version("1.1.0")
     .intConf
     .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be positive")
     .createWithDefault(200)
@@ -384,14 +360,12 @@ object SQLConf {
     buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
       .internal()
       .doc("(Deprecated since Spark 3.0)")
-      .version("1.6.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("64MB")
 
   val ADAPTIVE_EXECUTION_ENABLED = buildConf("spark.sql.adaptive.enabled")
     .doc("When true, enable adaptive query execution, which re-optimizes the query plan in the " +
       "middle of query execution, based on accurate runtime statistics.")
-    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -401,7 +375,6 @@ object SQLConf {
       "sub-queries. By setting this config to true (together with " +
       s"'${ADAPTIVE_EXECUTION_ENABLED.key}' set to true), Spark will force apply adaptive query " +
       "execution for all supported queries.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -409,7 +382,6 @@ object SQLConf {
     .internal()
     .doc("Configures the log level for adaptive execution logging of plan changes. The value " +
       "can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR"))
@@ -420,7 +392,6 @@ object SQLConf {
       .doc("The advisory size in bytes of the shuffle partition during adaptive optimization " +
         s"(when ${ADAPTIVE_EXECUTION_ENABLED.key} is true). It takes effect when Spark " +
         "coalesces small shuffle partitions or splits skewed shuffle partition.")
-      .version("3.0.0")
       .fallbackConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
 
   val COALESCE_PARTITIONS_ENABLED =
@@ -428,7 +399,6 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark will coalesce " +
         "contiguous shuffle partitions according to the target size (specified by " +
         s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'), to avoid too many small tasks.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -438,7 +408,6 @@ object SQLConf {
         "value is the default parallelism of the Spark cluster. This configuration only " +
         s"has an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true.")
-      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The minimum number of partitions must be positive.")
       .createOptional
@@ -449,7 +418,6 @@ object SQLConf {
         s"${SHUFFLE_PARTITIONS.key}. This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and '${COALESCE_PARTITIONS_ENABLED.key}' " +
         "are both true.")
-      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The initial number of partitions must be positive.")
       .createOptional
@@ -464,7 +432,6 @@ object SQLConf {
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true. This feature also depends " +
         "on a relocatable serializer, the concatenation support codec in use and the new version " +
         "shuffle fetch protocol.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -473,7 +440,6 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark tries to use local " +
         "shuffle reader to read the shuffle data when the shuffle partitioning is not needed, " +
         "for example, after converting sort-merge join to broadcast-hash join.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -482,7 +448,6 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark dynamically " +
         "handles skew in sort-merge join by splitting (and replicating if needed) skewed " +
         "partitions.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -491,7 +456,6 @@ object SQLConf {
       .doc("A partition is considered as skewed if its size is larger than this factor " +
         "multiplying the median partition size and also larger than " +
         "'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
-      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(10)
@@ -502,7 +466,6 @@ object SQLConf {
         s"threshold and also larger than '${SKEW_JOIN_SKEWED_PARTITION_FACTOR.key}' " +
         "multiplying the median partition size. Ideally this config should be set larger " +
         s"than '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'.")
-      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("256MB")
 
@@ -513,7 +476,6 @@ object SQLConf {
         "considered as the build side of a broadcast-hash join in adaptive execution regardless " +
         "of its size.This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is true.")
-      .version("3.0.0")
       .doubleConf
       .checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
       .createWithDefault(0.2)
@@ -522,7 +484,6 @@ object SQLConf {
     buildConf("spark.sql.subexpressionElimination.enabled")
       .internal()
       .doc("When true, common subexpressions will be eliminated.")
-      .version("1.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -530,7 +491,6 @@ object SQLConf {
     .internal()
     .doc("Whether the query analyzer should be case sensitive or not. " +
       "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.")
-    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -540,7 +500,6 @@ object SQLConf {
       "plan to optimize them. Constraint propagation can sometimes be computationally expensive " +
       "for certain kinds of query plans (such as those with a large number of predicates and " +
       "aliases) which might negatively impact overall runtime.")
-    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -549,7 +508,6 @@ object SQLConf {
     .doc("When true, string literals (including regex patterns) remain escaped in our SQL " +
       "parser. The default is false since Spark 2.0. Setting it to true can restore the behavior " +
       "prior to Spark 2.0.")
-    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -558,7 +516,6 @@ object SQLConf {
     .doc("When estimating the output data size of a table scan, multiply the file size with this " +
       "factor as the estimated data size, in case the data is compressed in the file and lead to" +
       " a heavily underestimated result.")
-    .version("2.3.1")
     .doubleConf
     .checkValue(_ > 0, "the value of fileDataSizeFactor must be greater than 0")
     .createWithDefault(1.0)
@@ -567,7 +524,6 @@ object SQLConf {
     .doc("When true, the Parquet data source merges schemas collected from all data files, " +
          "otherwise the schema is picked from the summary file or a random data file " +
          "if no summary file is available.")
-    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -576,7 +532,6 @@ object SQLConf {
          "summary files and we will ignore them when merging schema. Otherwise, if this is " +
          "false, which is the default, we will merge all part-files. This should be considered " +
          "as expert-only option, and shouldn't be enabled before knowing what it means exactly.")
-    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -585,7 +540,6 @@ object SQLConf {
       "Spark SQL, do not differentiate between binary data and strings when writing out the " +
       "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " +
       "compatibility with these systems.")
-    .version("1.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -594,7 +548,6 @@ object SQLConf {
       "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " +
       "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " +
       "provide compatibility with these systems.")
-    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -602,7 +555,6 @@ object SQLConf {
     .doc("This controls whether timestamp adjustments should be applied to INT96 data when " +
       "converting to timestamps, for data written by Impala.  This is necessary because Impala " +
       "stores INT96 data with a different timezone offset than Hive & Spark.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -616,7 +568,6 @@ object SQLConf {
       "is a standard timestamp type in Parquet, which stores number of microseconds from the " +
       "Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which " +
       "means Spark has to truncate the microsecond portion of its timestamp value.")
-    .version("2.3.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(ParquetOutputTimestampType.values.map(_.toString))
@@ -628,7 +579,6 @@ object SQLConf {
       "precedence would be `compression`, `parquet.compression`, " +
       "`spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, " +
       "snappy, gzip, lzo, brotli, lz4, zstd.")
-    .version("1.1.1")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "gzip", "lzo", "lz4", "brotli", "zstd"))
@@ -636,7 +586,6 @@ object SQLConf {
 
   val PARQUET_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.parquet.filterPushdown")
     .doc("Enables Parquet filter push-down optimization when set to true.")
-    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -644,7 +593,6 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for Date. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
-    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -654,17 +602,15 @@ object SQLConf {
       .doc("If true, enables Parquet filter push-down optimization for Timestamp. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.")
-      .version("2.4.0")
-      .internal()
-      .booleanConf
-      .createWithDefault(true)
+    .internal()
+    .booleanConf
+    .createWithDefault(true)
 
   val PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED =
     buildConf("spark.sql.parquet.filterPushdown.decimal")
       .doc("If true, enables Parquet filter push-down optimization for Decimal. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
-      .version("2.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -674,7 +620,6 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for string startsWith function. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
-    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -687,7 +632,6 @@ object SQLConf {
         "By setting this value to 0 this feature can be disabled. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
-      .version("2.4.0")
       .internal()
       .intConf
       .checkValue(threshold => threshold >= 0, "The threshold must not be negative.")
@@ -699,7 +643,6 @@ object SQLConf {
       "systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet " +
       "will be used. For example, decimals will be written in int-based format. If Parquet " +
       "output is intended for use with systems that do not support this newer format, set to true.")
-    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -709,7 +652,6 @@ object SQLConf {
       "of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata " +
       "summaries will never be created, irrespective of the value of " +
       "parquet.summary.metadata.level")
-    .version("1.5.0")
     .internal()
     .stringConf
     .createWithDefault("org.apache.parquet.hadoop.ParquetOutputCommitter")
@@ -717,7 +659,6 @@ object SQLConf {
   val PARQUET_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.parquet.enableVectorizedReader")
       .doc("Enables vectorized parquet decoding.")
-      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -727,14 +668,12 @@ object SQLConf {
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' " +
       "is enabled and the vectorized reader is not used. You can ensure the vectorized reader " +
       s"is not used by setting '${PARQUET_VECTORIZED_READER_ENABLED.key}' to false.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val PARQUET_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.parquet.columnarReaderBatchSize")
     .doc("The number of rows to include in a parquet vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
-    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
@@ -743,7 +682,6 @@ object SQLConf {
       "`orc.compress` is specified in the table-specific options/properties, the precedence " +
       "would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`." +
       "Acceptable values include: none, uncompressed, snappy, zlib, lzo.")
-    .version("2.3.0")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "zlib", "lzo"))
@@ -752,7 +690,6 @@ object SQLConf {
   val ORC_IMPLEMENTATION = buildConf("spark.sql.orc.impl")
     .doc("When native, use the native version of ORC support instead of the ORC library in Hive. " +
       "It is 'hive' by default prior to Spark 2.4.")
-    .version("2.3.0")
     .internal()
     .stringConf
     .checkValues(Set("hive", "native"))
@@ -760,27 +697,23 @@ object SQLConf {
 
   val ORC_VECTORIZED_READER_ENABLED = buildConf("spark.sql.orc.enableVectorizedReader")
     .doc("Enables vectorized orc decoding.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.orc.columnarReaderBatchSize")
     .doc("The number of rows to include in a orc vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
-    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
   val ORC_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.orc.filterPushdown")
     .doc("When true, enable filter pushdown for ORC files.")
-    .version("1.4.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_SCHEMA_MERGING_ENABLED = buildConf("spark.sql.orc.mergeSchema")
     .doc("When true, the Orc data source merges schemas collected from all data files, " +
       "otherwise the schema is picked from a random data file.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -788,7 +721,6 @@ object SQLConf {
     .doc("When true, check all the partition paths under the table\'s root directory " +
          "when reading data stored in HDFS. This configuration will be deprecated in the future " +
          s"releases and replaced by ${SPARK_IGNORE_MISSING_FILES.key}.")
-    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -798,7 +730,6 @@ object SQLConf {
            "unmatching partitions can be eliminated earlier. This only affects Hive tables " +
            "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " +
            "HiveUtils.CONVERT_METASTORE_ORC for more information).")
-      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -808,7 +739,6 @@ object SQLConf {
            "This includes both datasource and converted Hive tables. When partition management " +
            "is enabled, datasource tables store partition in the Hive metastore, and use the " +
            "metastore to prune partitions during query planning.")
-      .version("2.1.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -817,7 +747,6 @@ object SQLConf {
       .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " +
            "a cache that can use up to specified num bytes for file metadata. This conf only " +
            "has an effect when hive filesource partition management is enabled.")
-      .version("2.1.1")
       .longConf
       .createWithDefault(250 * 1024 * 1024)
 
@@ -833,7 +762,6 @@ object SQLConf {
       "and write it back to the table properties), INFER_ONLY (infer the schema but don't " +
       "attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback " +
       "to using the case-insensitive metastore schema instead of inferring).")
-    .version("2.1.1")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(HiveCaseSensitiveInferenceMode.values.map(_.toString))
@@ -846,27 +774,23 @@ object SQLConf {
       "scanned are partition columns and the query has an aggregate operator that satisfies " +
       "distinct semantics. By default the optimization is disabled, since it may return " +
       "incorrect results when the files are empty.")
-    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
   val COLUMN_NAME_OF_CORRUPT_RECORD = buildConf("spark.sql.columnNameOfCorruptRecord")
     .doc("The name of internal column for storing raw/un-parsed JSON and CSV records that fail " +
       "to parse.")
-    .version("1.2.0")
     .stringConf
     .createWithDefault("_corrupt_record")
 
   val BROADCAST_TIMEOUT = buildConf("spark.sql.broadcastTimeout")
     .doc("Timeout in seconds for the broadcast wait time in broadcast joins.")
-    .version("1.3.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString(s"${5 * 60}")
 
   // This is only used for the thriftserver
   val THRIFTSERVER_POOL = buildConf("spark.sql.thriftserver.scheduler.pool")
     .doc("Set a Fair Scheduler pool for a JDBC client session.")
-    .version("1.1.1")
     .stringConf
     .createOptional
 
@@ -874,27 +798,23 @@ object SQLConf {
     buildConf("spark.sql.thriftServer.incrementalCollect")
       .internal()
       .doc("When true, enable incremental collection for execution in Thrift Server.")
-      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   val THRIFTSERVER_UI_STATEMENT_LIMIT =
     buildConf("spark.sql.thriftserver.ui.retainedStatements")
       .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.")
-      .version("1.4.0")
       .intConf
       .createWithDefault(200)
 
   val THRIFTSERVER_UI_SESSION_LIMIT = buildConf("spark.sql.thriftserver.ui.retainedSessions")
     .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.")
-    .version("1.4.0")
     .intConf
     .createWithDefault(200)
 
   // This is used to set the default data source
   val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default")
     .doc("The default data source to use in input/output.")
-    .version("1.3.0")
     .stringConf
     .createWithDefault("parquet")
 
@@ -903,7 +823,6 @@ object SQLConf {
     .doc("When true, a table created by a Hive CTAS statement (no USING clause) " +
       "without specifying any storage property will be converted to a data source table, " +
       s"using the data source set by ${DEFAULT_DATA_SOURCE_NAME.key}.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -912,26 +831,22 @@ object SQLConf {
       .doc("When true, fast stats (number of files and total size of all files) will be gathered" +
         " in parallel while repairing table partitions to avoid the sequential listing in Hive" +
         " metastore.")
-      .version("2.0.1")
       .booleanConf
       .createWithDefault(true)
 
   val PARTITION_COLUMN_TYPE_INFERENCE =
     buildConf("spark.sql.sources.partitionColumnTypeInference.enabled")
       .doc("When true, automatically infer the data types for partitioned columns.")
-      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   val BUCKETING_ENABLED = buildConf("spark.sql.sources.bucketing.enabled")
     .doc("When false, we will treat bucketed table as normal table")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val BUCKETING_MAX_BUCKETS = buildConf("spark.sql.sources.bucketing.maxBuckets")
     .doc("The maximum number of buckets allowed.")
-    .version("2.4.0")
     .intConf
     .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
     .createWithDefault(100000)
@@ -940,42 +855,36 @@ object SQLConf {
     .internal()
     .doc("When false, we will throw an error if a query contains a cartesian product without " +
         "explicit CROSS JOIN syntax.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORDER_BY_ORDINAL = buildConf("spark.sql.orderByOrdinal")
     .doc("When true, the ordinal numbers are treated as the position in the select list. " +
          "When false, the ordinal numbers in order/sort by clause are ignored.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ORDINAL = buildConf("spark.sql.groupByOrdinal")
     .doc("When true, the ordinal numbers in group by clauses are treated as the position " +
       "in the select list. When false, the ordinal numbers are ignored.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ALIASES = buildConf("spark.sql.groupByAliases")
     .doc("When true, aliases in a select list can be used in group by clauses. When false, " +
       "an analysis exception is thrown in the case.")
-    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
   // The output committer class used by data sources. The specified class needs to be a
   // subclass of org.apache.hadoop.mapreduce.OutputCommitter.
   val OUTPUT_COMMITTER_CLASS = buildConf("spark.sql.sources.outputCommitterClass")
-    .version("1.4.0")
     .internal()
     .stringConf
     .createOptional
 
   val FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.sources.commitProtocolClass")
-      .version("2.1.1")
       .internal()
       .stringConf
       .createWithDefault(
@@ -987,7 +896,6 @@ object SQLConf {
         "of detected paths exceeds this value during partition discovery, it tries to list the " +
         "files with another Spark distributed job. This configuration is effective only when " +
         "using file-based sources such as Parquet, JSON and ORC.")
-      .version("1.5.0")
       .intConf
       .checkValue(parallel => parallel >= 0, "The maximum number of paths allowed for listing " +
         "files at driver side must not be negative")
@@ -997,7 +905,6 @@ object SQLConf {
     buildConf("spark.sql.sources.parallelPartitionDiscovery.parallelism")
       .doc("The number of parallelism to list a collection of path recursively, Set the " +
         "number to prevent file listing from generating too many tasks.")
-      .version("2.1.1")
       .internal()
       .intConf
       .createWithDefault(10000)
@@ -1009,7 +916,6 @@ object SQLConf {
         "schedule tasks to take advantage of data locality. It can be particularly " +
         "useful if data is read from a remote cluster so the scheduler could never " +
         "take advantage of locality anyway.")
-      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(false)
@@ -1018,7 +924,6 @@ object SQLConf {
   // See SPARK-6231.
   val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY =
     buildConf("spark.sql.selfJoinAutoResolveAmbiguity")
-      .version("1.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -1026,14 +931,12 @@ object SQLConf {
   val FAIL_AMBIGUOUS_SELF_JOIN_ENABLED =
     buildConf("spark.sql.analyzer.failAmbiguousSelfJoin")
       .doc("When true, fail the Dataset query if it contains ambiguous self-join.")
-      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
 
   // Whether to retain group by columns or not in GroupedData.agg.
   val DATAFRAME_RETAIN_GROUP_COLUMNS = buildConf("spark.sql.retainGroupColumns")
-    .version("1.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -1041,14 +944,12 @@ object SQLConf {
   val DATAFRAME_PIVOT_MAX_VALUES = buildConf("spark.sql.pivotMaxValues")
     .doc("When doing a pivot without specifying values for the pivot column this is the maximum " +
       "number of (distinct) values that will be collected without error.")
-    .version("1.6.0")
     .intConf
     .createWithDefault(10000)
 
   val RUN_SQL_ON_FILES = buildConf("spark.sql.runSQLOnFiles")
     .internal()
     .doc("When true, we could use `datasource`.`path` as table in SQL query.")
-    .version("1.6.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1056,7 +957,6 @@ object SQLConf {
     .internal()
     .doc("When true, the whole stage (of multiple operators) will be compiled into single java" +
       " method.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1065,7 +965,6 @@ object SQLConf {
     .internal()
     .doc("When true, embed the (whole-stage) codegen stage ID into " +
       "the class name of the generated class as a suffix")
-    .version("2.3.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -1073,7 +972,6 @@ object SQLConf {
     .internal()
     .doc("The maximum number of fields (including nested fields) that will be supported before" +
       " deactivating whole-stage codegen.")
-    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -1083,7 +981,6 @@ object SQLConf {
       "interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. " +
       "`NO_CODEGEN` skips codegen and goes interpreted path always. Note that " +
       "this config works only for tests.")
-    .version("2.4.0")
     .internal()
     .stringConf
     .checkValues(CodegenObjectFactoryMode.values.map(_.toString))
@@ -1093,14 +990,12 @@ object SQLConf {
     .internal()
     .doc("When true, (whole stage) codegen could be temporary disabled for the part of query that" +
       " fail to compile generated code")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val CODEGEN_LOGGING_MAX_LINES = buildConf("spark.sql.codegen.logging.maxLines")
     .internal()
     .doc("The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.")
-    .version("2.3.0")
     .intConf
     .checkValue(maxLines => maxLines >= -1, "The maximum must be a positive integer, 0 to " +
       "disable logging or -1 to apply no limit.")
@@ -1114,7 +1009,6 @@ object SQLConf {
       "is the largest bytecode size possible for a valid Java method. When running on HotSpot, " +
       s"it may be preferable to set the value to ${CodeGenerator.DEFAULT_JVM_HUGE_METHOD_LIMIT} " +
       "to match HotSpot's implementation.")
-    .version("2.3.0")
     .intConf
     .createWithDefault(65535)
 
@@ -1126,7 +1020,6 @@ object SQLConf {
       "generated, so use the code length as metric. When running on HotSpot, a function's " +
       "bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not " +
       "be too small, otherwise there will be many function calls.")
-    .version("3.0.0")
     .intConf
     .checkValue(threshold => threshold > 0, "The threshold must be a positive integer.")
     .createWithDefault(1024)
@@ -1137,7 +1030,6 @@ object SQLConf {
       .doc("When true, whole stage codegen would put the logic of consuming rows of each " +
         "physical operator into individual methods, instead of a single big method. This can be " +
         "used to avoid oversized function that can miss the opportunity of JIT optimization.")
-      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1145,7 +1037,6 @@ object SQLConf {
     .doc("The maximum number of bytes to pack into a single partition when reading files. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
-    .version("2.0.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("128MB") // parquet.block.size
 
@@ -1156,7 +1047,6 @@ object SQLConf {
       " over estimated, then the partitions with small files will be faster than partitions with" +
       " bigger files (which is scheduled first). This configuration is effective only when using" +
       " file-based sources such as Parquet, JSON and ORC.")
-    .version("2.0.0")
     .longConf
     .createWithDefault(4 * 1024 * 1024)
 
@@ -1165,7 +1055,6 @@ object SQLConf {
       "encountering corrupted files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
-    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -1174,28 +1063,24 @@ object SQLConf {
       "encountering missing files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val MAX_RECORDS_PER_FILE = buildConf("spark.sql.files.maxRecordsPerFile")
     .doc("Maximum number of records to write out to a single file. " +
       "If this value is zero or negative, there is no limit.")
-    .version("2.2.0")
     .longConf
     .createWithDefault(0)
 
   val EXCHANGE_REUSE_ENABLED = buildConf("spark.sql.exchange.reuse")
     .internal()
     .doc("When true, the planner will try to find out duplicated exchanges and re-use them.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.reuseSubquery")
     .internal()
     .doc("When true, the planner will try to find out duplicated subqueries and re-use them.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1207,7 +1092,6 @@ object SQLConf {
           "be a subclass of StateStoreProvider, and must have a zero-arg constructor. " +
           "Note: For structured streaming, this configuration cannot be changed between query " +
           "restarts from the same checkpoint location.")
-      .version("2.3.0")
       .stringConf
       .createWithDefault(
         "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider")
@@ -1217,7 +1101,6 @@ object SQLConf {
       .internal()
       .doc("Minimum number of state store delta files that needs to be generated before they " +
         "consolidated into snapshots.")
-      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1225,28 +1108,24 @@ object SQLConf {
     buildConf("spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion")
       .internal()
       .doc("State format version used by flatMapGroupsWithState operation in a streaming query")
-      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
 
   val CHECKPOINT_LOCATION = buildConf("spark.sql.streaming.checkpointLocation")
     .doc("The default location for storing checkpoint data for streaming queries.")
-    .version("2.0.0")
     .stringConf
     .createOptional
 
   val FORCE_DELETE_TEMP_CHECKPOINT_LOCATION =
     buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation")
       .doc("When true, enable temporary checkpoint locations force delete.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val MIN_BATCHES_TO_RETAIN = buildConf("spark.sql.streaming.minBatchesToRetain")
     .internal()
     .doc("The minimum number of batches that must be retained and made recoverable.")
-    .version("2.1.1")
     .intConf
     .createWithDefault(100)
 
@@ -1256,7 +1135,6 @@ object SQLConf {
       "loading from files. The value adjusts a trade-off between memory usage vs cache miss: " +
       "'2' covers both success and direct failure cases, '1' covers only success case, " +
       "and '0' covers extreme case - disable cache to maximize memory size of executors.")
-    .version("2.4.0")
     .intConf
     .createWithDefault(2)
 
@@ -1266,7 +1144,6 @@ object SQLConf {
       .doc("State format version used by streaming aggregation operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
-      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1277,7 +1154,6 @@ object SQLConf {
       "If we find a concurrent active run for a streaming query (in the same or different " +
       "SparkSessions on the same cluster) and this flag is true, we will stop the old streaming " +
       "query run to start the new one.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1287,7 +1163,6 @@ object SQLConf {
       .doc("State format version used by streaming join operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
-      .version("3.0.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1297,7 +1172,6 @@ object SQLConf {
       .internal()
       .doc("When true, the logical plan for streaming query will be checked for unsupported" +
         " operations.")
-      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1305,7 +1179,6 @@ object SQLConf {
     buildConf("spark.sql.variable.substitute")
       .doc("This enables substitution using syntax like `${var}`, `${system:var}`, " +
         "and `${env:var}`.")
-      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1316,7 +1189,6 @@ object SQLConf {
         "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " +
         "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " +
         "When disabled, records go directly to the 2nd level.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1324,7 +1196,6 @@ object SQLConf {
     buildConf("spark.sql.codegen.aggregate.map.vectorized.enable")
       .internal()
       .doc("Enable vectorized aggregate hash map. This is for testing/benchmarking only.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1334,7 +1205,6 @@ object SQLConf {
       .doc("When true, the code generator would split aggregate code into individual methods " +
         "instead of a single big method. This can be used to avoid oversized function that " +
         "can miss the opportunity of JIT optimization.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1346,7 +1216,6 @@ object SQLConf {
         "However the DAG depth may become too large and cause unexpected behavior. This " +
         "configuration puts a limit on this: when the depth of a view exceeds this value during " +
         "analysis, we terminate the resolution to avoid potential errors.")
-      .version("2.2.0")
       .intConf
       .checkValue(depth => depth > 0, "The maximum depth of a view reference in a nested view " +
         "must be positive.")
@@ -1354,7 +1223,6 @@ object SQLConf {
 
   val STREAMING_FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.streaming.commitProtocolClass")
-      .version("2.1.0")
       .internal()
       .stringConf
       .createWithDefault("org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol")
@@ -1367,7 +1235,6 @@ object SQLConf {
         "'max' which chooses the maximum across multiple operators. " +
         "Note: This configuration cannot be changed between query restarts from the same " +
         "checkpoint location.")
-      .version("2.4.0")
       .stringConf
       .transform(_.toLowerCase(Locale.ROOT))
       .checkValue(
@@ -1382,7 +1249,6 @@ object SQLConf {
       .doc("In the case of ObjectHashAggregateExec, when the size of the in-memory hash map " +
         "grows too large, we will fall back to sort-based aggregation. This option sets a row " +
         "count threshold for the size of the hash map.")
-      .version("2.2.0")
       .intConf
       // We are trying to be conservative and use a relatively small default count threshold here
       // since the state object of some TypedImperativeAggregate function can be quite large (e.g.
@@ -1392,7 +1258,6 @@ object SQLConf {
   val USE_OBJECT_HASH_AGG = buildConf("spark.sql.execution.useObjectHashAggregateExec")
     .internal()
     .doc("Decides if we use ObjectHashAggregateExec")
-    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1401,14 +1266,12 @@ object SQLConf {
       .doc("Whether to ignore null fields when generating JSON objects in JSON data source and " +
         "JSON functions such as to_json. " +
         "If false, it generates null for null fields in JSON objects.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SINK_LOG_DELETION = buildConf("spark.sql.streaming.fileSink.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream sink.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1417,7 +1280,6 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
-      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1425,14 +1287,12 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSink.log.cleanupDelay")
       .internal()
       .doc("How long that a file is guaranteed to be visible for all readers.")
-      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
   val FILE_SOURCE_LOG_DELETION = buildConf("spark.sql.streaming.fileSource.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream source.")
-    .version("2.0.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -1441,7 +1301,6 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
-      .version("2.0.1")
       .intConf
       .createWithDefault(10)
 
@@ -1449,7 +1308,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSource.log.cleanupDelay")
       .internal()
       .doc("How long in milliseconds a file is guaranteed to be visible for all readers.")
-      .version("2.0.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
@@ -1459,14 +1317,12 @@ object SQLConf {
       .doc("When true, force the schema of streaming file source to be nullable (including all " +
         "the fields). Otherwise, the schema might not be compatible with actual data, which " +
         "leads to corruptions.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SOURCE_CLEANER_NUM_THREADS =
     buildConf("spark.sql.streaming.fileSource.cleaner.numThreads")
       .doc("Number of threads used in the file source completed file cleaner.")
-      .version("3.0.0")
       .intConf
       .createWithDefault(1)
 
@@ -1474,7 +1330,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.schemaInference")
       .internal()
       .doc("Whether file-based streaming sources will infer its own schema")
-      .version("2.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1482,7 +1337,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.pollingDelay")
       .internal()
       .doc("How long to delay polling new data when no data is available")
-      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10L)
 
@@ -1490,7 +1344,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.stopTimeout")
       .doc("How long to wait in milliseconds for the streaming execution thread to stop when " +
         "calling the streaming query's stop() method. 0 or negative values wait indefinitely.")
-      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
@@ -1498,7 +1351,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.noDataProgressEventInterval")
       .internal()
       .doc("How long to wait between two progress events when there is no data")
-      .version("2.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10000L)
 
@@ -1507,21 +1359,18 @@ object SQLConf {
       .doc(
         "Whether streaming micro-batch engine will execute batches without data " +
           "for eager state management for stateful streaming queries.")
-      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
   val STREAMING_METRICS_ENABLED =
     buildConf("spark.sql.streaming.metricsEnabled")
       .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.")
-      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   val STREAMING_PROGRESS_RETENTION =
     buildConf("spark.sql.streaming.numRecentProgressUpdates")
       .doc("The number of progress updates to retain for a streaming query")
-      .version("2.1.1")
       .intConf
       .createWithDefault(100)
 
@@ -1529,7 +1378,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpointFileManagerClass")
       .doc("The class used to write checkpoint files atomically. This class must be a subclass " +
         "of the interface CheckpointFileManager.")
-      .version("2.4.0")
       .internal()
       .stringConf
 
@@ -1537,7 +1385,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpoint.escapedPathCheck.enabled")
       .doc("Whether to detect a streaming query may pick up an incorrect checkpoint path due " +
         "to SPARK-26824.")
-      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -1548,7 +1395,6 @@ object SQLConf {
       .doc("When true, SQL commands use parallel file listing, " +
         "as opposed to single thread listing. " +
         "This usually speeds up commands that need to list many directories.")
-      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1558,7 +1404,6 @@ object SQLConf {
       s"which is larger than `${AUTO_BROADCASTJOIN_THRESHOLD.key}` to be more conservative. " +
       "That is to say by default the optimizer will not choose to broadcast a table unless it " +
       "knows for sure its size is small enough.")
-    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(Long.MaxValue)
 
@@ -1569,7 +1414,6 @@ object SQLConf {
       "For non-partitioned data source tables, it will be automatically recalculated if table " +
       "statistics are not available. For partitioned data source and partitioned Hive tables, " +
       s"It is '${DEFAULT_SIZE_IN_BYTES.key}' if table statistics are not available.")
-    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1578,7 +1422,6 @@ object SQLConf {
       .internal()
       .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
         "column level statistics.")
-      .version("2.1.1")
       .doubleConf
       .createWithDefault(0.05)
 
@@ -1589,7 +1432,6 @@ object SQLConf {
         "histogram. Note that collecting histograms takes extra cost. For example, collecting " +
         "column statistics usually takes only one table scan, but generating equi-height " +
         "histogram will cause an extra table scan.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1597,7 +1439,6 @@ object SQLConf {
     buildConf("spark.sql.statistics.histogram.numBins")
       .internal()
       .doc("The number of bins when generating histograms.")
-      .version("2.3.0")
       .intConf
       .checkValue(num => num > 1, "The number of bins must be greater than 1.")
       .createWithDefault(254)
@@ -1608,7 +1449,6 @@ object SQLConf {
       .doc("Accuracy of percentile approximation when generating equi-height histograms. " +
         "Larger value means better accuracy. The relative error can be deduced by " +
         "1.0 / PERCENTILE_ACCURACY.")
-      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1617,35 +1457,30 @@ object SQLConf {
       .doc("Enables automatic update for table size once table's data is changed. Note that if " +
         "the total number of files of the table is very large, this can be expensive and slow " +
         "down data change commands.")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val CBO_ENABLED =
     buildConf("spark.sql.cbo.enabled")
       .doc("Enables CBO for estimation of plan statistics when set true.")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val PLAN_STATS_ENABLED =
     buildConf("spark.sql.cbo.planStats.enabled")
       .doc("When true, the logical plan will fetch row counts and column statistics from catalog.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_ENABLED =
     buildConf("spark.sql.cbo.joinReorder.enabled")
       .doc("Enables join reorder in CBO.")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_DP_THRESHOLD =
     buildConf("spark.sql.cbo.joinReorder.dp.threshold")
       .doc("The maximum number of joined nodes allowed in the dynamic programming algorithm.")
-      .version("2.2.0")
       .intConf
       .checkValue(number => number > 0, "The maximum number must be a positive integer.")
       .createWithDefault(12)
@@ -1655,7 +1490,6 @@ object SQLConf {
       .internal()
       .doc("The weight of cardinality (number of rows) for plan cost comparison in join reorder: " +
         "rows * weight + size * (1 - weight).")
-      .version("2.2.0")
       .doubleConf
       .checkValue(weight => weight >= 0 && weight <= 1, "The weight value must be in [0, 1].")
       .createWithDefault(0.7)
@@ -1663,13 +1497,11 @@ object SQLConf {
   val JOIN_REORDER_DP_STAR_FILTER =
     buildConf("spark.sql.cbo.joinReorder.dp.star.filter")
       .doc("Applies star-join filter heuristics to cost based join enumeration.")
-      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val STARSCHEMA_DETECTION = buildConf("spark.sql.cbo.starSchemaDetection")
     .doc("When true, it enables join reordering based on star schema detection. ")
-    .version("2.2.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1677,7 +1509,6 @@ object SQLConf {
     .internal()
     .doc("Specifies the upper limit of the ratio between the largest fact tables" +
       " for a star join to be considered. ")
-    .version("2.2.0")
     .doubleConf
     .createWithDefault(0.9)
 
@@ -1691,7 +1522,6 @@ object SQLConf {
       "Zone offsets must be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. " +
       "Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not " +
       "recommended to use because they can be ambiguous.")
-    .version("2.2.0")
     .stringConf
     .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
       " ZoneId.of(_, ZoneId.SHORT_IDS)")
@@ -1701,7 +1531,6 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.in.memory.threshold")
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the window operator")
-      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1709,7 +1538,6 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by window operator")
-      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1718,7 +1546,6 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the sort merge " +
         "join operator")
-      .version("2.2.1")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1726,7 +1553,6 @@ object SQLConf {
     buildConf("spark.sql.sortMergeJoinExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by sort merge join operator")
-      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1735,7 +1561,6 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the cartesian " +
         "product operator")
-      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1743,14 +1568,12 @@ object SQLConf {
     buildConf("spark.sql.cartesianProductExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by cartesian product operator")
-      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
   val SUPPORT_QUOTED_REGEX_COLUMN_NAME = buildConf("spark.sql.parser.quotedRegexColumnNames")
     .doc("When true, quoted Identifiers (using backticks) in SELECT statement are interpreted" +
       " as regular expressions.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1759,14 +1582,12 @@ object SQLConf {
       .internal()
       .doc("Number of points to sample per partition in order to determine the range boundaries" +
           " for range partitioning, typically used in global sorting (without limit).")
-      .version("2.3.0")
       .intConf
       .createWithDefault(100)
 
   val ARROW_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.enabled")
       .doc("(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)")
-      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1778,7 +1599,6 @@ object SQLConf {
         "2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame " +
         "The following data types are unsupported: " +
         "BinaryType, MapType, ArrayType of TimestampType, and nested StructType.")
-      .version("3.0.0")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
   val ARROW_SPARKR_EXECUTION_ENABLED =
@@ -1791,7 +1611,6 @@ object SQLConf {
         "4. gapply " +
         "The following data types are unsupported: " +
         "FloatType, BinaryType, ArrayType, StructType and MapType.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1799,7 +1618,6 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.fallback.enabled")
       .doc("(Deprecated since Spark 3.0, please set " +
         "'spark.sql.execution.arrow.pyspark.fallback.enabled'.)")
-      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1807,14 +1625,12 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.pyspark.fallback.enabled")
       .doc(s"When true, optimizations enabled by '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' will " +
         "fallback automatically to non-optimized implementations if an error occurs.")
-      .version("3.0.0")
       .fallbackConf(ARROW_FALLBACK_ENABLED)
 
   val ARROW_EXECUTION_MAX_RECORDS_PER_BATCH =
     buildConf("spark.sql.execution.arrow.maxRecordsPerBatch")
       .doc("When using Apache Arrow, limit the maximum number of records that can be written " +
         "to a single ArrowRecordBatch in memory. If set to zero or negative there is no limit.")
-      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1825,7 +1641,6 @@ object SQLConf {
         s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
         "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
         "pipelined; however, it might degrade performance. See SPARK-27870.")
-      .version("3.0.0")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
@@ -1835,7 +1650,6 @@ object SQLConf {
         "to use position if not. When false, a grouped map Pandas UDF will assign columns from " +
         "the returned Pandas DataFrame based on position, regardless of column label type. " +
         "This configuration will be deprecated in future releases.")
-      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1846,7 +1660,6 @@ object SQLConf {
         "Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
         "when detecting unsafe type conversion like overflow. When false, disabling Arrow's type " +
         "check and do type conversions anyway. This config only works for Arrow 0.11.0+.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1860,7 +1673,6 @@ object SQLConf {
       " column names of the left node are distinct. If all the conditions are met, the" +
       " rule will replace the except operation with a Filter by flipping the filter" +
       " condition(s) of the right node.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1871,7 +1683,6 @@ object SQLConf {
         "happens according to Hive behavior and SQL ANSI 2011 specification, ie. rounding the " +
         "decimal part of the result if an exact representation is not possible. Otherwise, NULL " +
         "is returned in those cases, as previously.")
-      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1881,7 +1692,6 @@ object SQLConf {
       .doc("When integral literal is used in decimal operations, pick a minimum precision " +
         "required by the literal if this config is true, to make the resulting precision and/or " +
         "scale smaller. This can reduce the possibility of precision lose and/or overflow.")
-      .version("2.3.3")
       .booleanConf
       .createWithDefault(true)
 
@@ -1891,7 +1701,6 @@ object SQLConf {
         "information. The values of options whose names that match this regex will be redacted " +
         "in the explain output. This redaction is applied on top of the global redaction " +
         s"configuration defined by ${SECRET_REDACTION_PATTERN.key}.")
-    .version("2.2.2")
     .regexConf
     .createWithDefault("(?i)url".r)
 
@@ -1901,20 +1710,17 @@ object SQLConf {
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands. " +
         "When this conf is not set, the value from `spark.redaction.string.regex` is used.")
-      .version("2.3.0")
       .fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN)
 
   val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString")
     .doc("When this option is set to false and all inputs are binary, `functions.concat` returns " +
       "an output as binary. Otherwise, it returns as a string.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString")
     .doc("When this option is set to false and all inputs are binary, `elt` returns " +
       "an output as binary. Otherwise, it returns as a string.")
-    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1925,7 +1731,6 @@ object SQLConf {
         "user-specified schema. If the validation fails, a runtime exception is thrown. " +
         "When this option is set to false, the partition column value will be converted to null " +
         "if it can not be casted to corresponding user-specified schema.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1933,7 +1738,6 @@ object SQLConf {
     buildConf("spark.sql.streaming.continuous.epochBacklogQueueSize")
       .doc("The max number of entries to be stored in queue to wait for late epochs. " +
         "If this parameter is exceeded by the size of the queue, stream will stop with an error.")
-      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1942,7 +1746,6 @@ object SQLConf {
     .internal()
     .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
       " buffer the results of a ContinuousDataReader.")
-    .version("2.3.0")
     .intConf
     .createWithDefault(1024)
 
@@ -1951,7 +1754,6 @@ object SQLConf {
       .internal()
       .doc("The interval at which continuous execution readers will poll to check whether" +
         " the epoch has advanced on the driver.")
-      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
@@ -1960,14 +1762,12 @@ object SQLConf {
     .doc("A comma-separated list of data source short names or fully qualified data source " +
       "implementation class names for which Data Source V2 code path is disabled. These data " +
       "sources will fallback to Data Source V1 code path.")
-    .version("3.0.0")
     .stringConf
     .createWithDefault("avro,csv,json,kafka,orc,parquet,text")
 
   val DISABLED_V2_STREAMING_WRITERS = buildConf("spark.sql.streaming.disabledV2Writers")
     .doc("A comma-separated list of fully qualified data source register class names for which" +
       " StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.")
-    .version("2.3.1")
     .stringConf
     .createWithDefault("")
 
@@ -1978,7 +1778,6 @@ object SQLConf {
         "A comma-separated list of fully qualified data source register class names for which " +
           "MicroBatchReadSupport is disabled. Reads from these sources will fall back to the " +
           "V1 Sources.")
-      .version("2.4.0")
       .stringConf
       .createWithDefault("")
 
@@ -1999,7 +1798,6 @@ object SQLConf {
         "(which takes precedence over this setting), e.g. " +
         "dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path)."
       )
-      .version("2.3.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(PartitionOverwriteMode.values.map(_.toString))
@@ -2024,7 +1822,6 @@ object SQLConf {
         "in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is " +
         "not allowed."
       )
-      .version("3.0.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(StoreAssignmentPolicy.values.map(_.toString))
@@ -2035,7 +1832,6 @@ object SQLConf {
       "throw a runtime exception if an overflow occurs in any operation on integral/decimal " +
       "field. 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in " +
       "the SQL parser.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2048,7 +1844,6 @@ object SQLConf {
         "issues. Turn on this config to insert a local sort before actually doing repartition " +
         "to generate consistent repartition results. The performance of repartition() may go " +
         "down since we insert extra local sort before it.")
-        .version("2.1.4")
         .booleanConf
         .createWithDefault(true)
 
@@ -2059,7 +1854,6 @@ object SQLConf {
         "satisfying a query. This optimization allows columnar file format readers to avoid " +
         "reading unnecessary nested column data. Currently Parquet and ORC are the " +
         "data sources that implement this optimization.")
-      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -2070,7 +1864,6 @@ object SQLConf {
         "containing `dots` to data sources. Currently, Parquet implements both optimizations " +
         "while ORC only supports predicates for names containing `dots`. The other data sources" +
         "don't support this feature yet.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2080,7 +1873,6 @@ object SQLConf {
       .doc("Prune nested fields from object serialization operator which are unnecessary in " +
         "satisfying a query. This optimization allows object serializers to avoid " +
         "executing unnecessary nested expressions.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2091,7 +1883,6 @@ object SQLConf {
         "satisfying a query. Note that this optimization doesn't prune nested fields from " +
         "physical data source scanning. For pruning nested fields from scanning, please use " +
         "`spark.sql.optimizer.nestedSchemaPruning.enabled` config.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2101,7 +1892,6 @@ object SQLConf {
       .doc("In SQL queries with a SORT followed by a LIMIT like " +
           "'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort" +
           " in memory, otherwise do a global sort which spills to disk if necessary.")
-      .version("2.4.0")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -2117,7 +1907,6 @@ object SQLConf {
     .internal()
     .doc("If it is set to true, column names of the requested schema are passed to CSV parser. " +
       "Other column values can be ignored during parsing even if they are malformed.")
-    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2128,7 +1917,6 @@ object SQLConf {
       "Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python " +
       "REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned " +
       "outputs are showed similar to R data.frame would.")
-    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2137,14 +1925,12 @@ object SQLConf {
       s"effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true. The valid range of this " +
       "config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and " +
       "greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).")
-    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
   val REPL_EAGER_EVAL_TRUNCATE = buildConf("spark.sql.repl.eagerEval.truncate")
     .doc("The max number of characters for each cell that is returned by eager evaluation. " +
       s"This only takes effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true.")
-    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
@@ -2155,7 +1941,6 @@ object SQLConf {
         "by the fast hash aggregate product operator. The bit is not for actual value, " +
         "but the actual numBuckets is determined by loadFactor " +
         "(e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).")
-      .version("2.4.0")
       .intConf
       .checkValue(bit => bit >= 10 && bit <= 30, "The bit value must be in [10, 30].")
       .createWithDefault(16)
@@ -2163,7 +1948,6 @@ object SQLConf {
   val AVRO_COMPRESSION_CODEC = buildConf("spark.sql.avro.compression.codec")
     .doc("Compression codec used in writing of AVRO files. Supported codecs: " +
       "uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.")
-    .version("2.4.0")
     .stringConf
     .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz"))
     .createWithDefault("snappy")
@@ -2172,7 +1956,6 @@ object SQLConf {
     .doc("Compression level for the deflate codec used in writing of AVRO files. " +
       "Valid value must be in the range of from 1 to 9 inclusive or -1. " +
       "The default value is -1 which corresponds to 6 level in the current implementation.")
-    .version("2.4.0")
     .intConf
     .checkValues((1 to 9).toSet + Deflater.DEFAULT_COMPRESSION)
     .createWithDefault(Deflater.DEFAULT_COMPRESSION)
@@ -2181,7 +1964,6 @@ object SQLConf {
     .internal()
     .doc(s"If it is set to false, or ${ANSI_ENABLED.key} is true, then size of null returns " +
       "null. Otherwise, it returns -1, which was inherited from Hive.")
-    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2190,7 +1972,6 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, the data source provider com.databricks.spark.avro is mapped " +
         "to the built-in but external Avro data source module for backward compatibility.")
-      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2201,7 +1982,6 @@ object SQLConf {
         "set operations are performed from left to right as they appear in the query. When set " +
         "to false and order of evaluation is not specified by parentheses, INTERSECT operations " +
         "are performed before any UNION, EXCEPT and MINUS operations.")
-      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2210,7 +1990,6 @@ object SQLConf {
       .internal()
       .doc("When set to true, a literal with an exponent (e.g. 1E-30) would be parsed " +
         "as Decimal rather than Double.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2220,7 +1999,6 @@ object SQLConf {
       .doc("When set to true, negative scale of Decimal type is allowed. For example, " +
         "the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is " +
         "Decimal(11, 0) in non legacy mode.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2229,7 +2007,6 @@ object SQLConf {
       .internal()
       .doc("When set to true, CREATE TABLE syntax without a provider will use hive " +
         s"instead of the value of ${DEFAULT_DATA_SOURCE_NAME.key}.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2238,16 +2015,14 @@ object SQLConf {
       .internal()
       .doc("When true, the bucketed table scan will list files during planning to figure out the " +
         "output ordering, which is expensive and may make the planning quite slow.")
-      .version("3.0.0")
-      .booleanConf
-      .createWithDefault(false)
+    .booleanConf
+    .createWithDefault(false)
 
   val LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE =
     buildConf("spark.sql.legacy.parser.havingWithoutGroupByAsWhere")
       .internal()
       .doc("If it is set to true, the parser will treat HAVING without GROUP BY as a normal " +
         "WHERE, which does not follow SQL standard.")
-      .version("2.4.1")
       .booleanConf
       .createWithDefault(false)
 
@@ -2255,8 +2030,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.json.allowEmptyString.enabled")
       .internal()
       .doc("When set to true, the parser of JSON data source treats empty strings as null for " +
-        "some data types such as `IntegerType`.")
-      .version("3.0.0")
+      "some data types such as `IntegerType`.")
       .booleanConf
       .createWithDefault(false)
 
@@ -2266,7 +2040,6 @@ object SQLConf {
       .doc("When set to true, Spark returns an empty collection with `StringType` as element " +
         "type if the `array`/`map` function is called without any parameters. Otherwise, Spark " +
         "returns an empty collection with `NullType` as element type.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2275,7 +2048,6 @@ object SQLConf {
       .internal()
       .doc("When set to true, user is allowed to use org.apache.spark.sql.functions." +
         "udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2284,7 +2056,6 @@ object SQLConf {
       .internal()
       .doc("When set to true, TRUNCATE TABLE command will not try to set back original " +
         "permission and ACLs when re-creating the table/partition paths.")
-      .version("2.4.6")
       .booleanConf
       .createWithDefault(false)
 
@@ -2294,7 +2065,6 @@ object SQLConf {
       .doc("When set to true, the key attribute resulted from running `Dataset.groupByKey` " +
         "for non-struct key type, will be named as `value`, following the behavior of Spark " +
         "version 2.4 and earlier.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2302,7 +2072,6 @@ object SQLConf {
     .doc("Maximum number of fields of sequence-like entries can be converted to strings " +
       "in debug output. Any elements beyond the limit will be dropped and replaced by a" +
       """ "... N more fields" placeholder.""")
-    .version("3.0.0")
     .intConf
     .createWithDefault(25)
 
@@ -2311,7 +2080,6 @@ object SQLConf {
       "longer, further output will be truncated.  The default setting always generates a full " +
       "plan.  Set this to a lower value such as 8k if plan strings are taking up too much " +
       "memory or are causing OutOfMemory errors in the driver or UI processes.")
-    .version("3.0.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(i => i >= 0 && i <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, "Invalid " +
       "value for 'spark.sql.maxPlanStringLength'.  Length must be a valid string length " +
@@ -2323,7 +2091,6 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, SET command will fail when the key is registered as " +
         "a SparkConf entry.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2332,7 +2099,6 @@ object SQLConf {
       "java.time.LocalDate classes of Java 8 API are used as external types for " +
       "Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp " +
       "and java.sql.Date are used for the same purpose.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2340,7 +2106,6 @@ object SQLConf {
     .doc("The max length of a file that can be read by the binary file data source. " +
       "Spark will fail fast and not attempt to read the file if its length exceeds this value. " +
       "The theoretical max is Int.MaxValue, though VMs might implement a smaller max.")
-    .version("3.0.0")
     .internal()
     .intConf
     .createWithDefault(Int.MaxValue)
@@ -2350,14 +2115,12 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
         "with String")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog")
     .doc("Name of the default catalog. This will be the current catalog if users have not " +
       "explicitly set the current catalog yet.")
-    .version("3.0.0")
     .stringConf
     .createWithDefault(SESSION_CATALOG_NAME)
 
@@ -2369,7 +2132,6 @@ object SQLConf {
         s"be loaded by the $SESSION_CATALOG_NAME, this catalog must also return the table " +
         s"metadata. To delegate operations to the $SESSION_CATALOG_NAME, implementations can " +
         "extend 'CatalogExtension'.")
-      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -2382,7 +2144,6 @@ object SQLConf {
       "MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query " +
       "fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted " +
       "at last takes precedence.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(MapKeyDedupPolicy.values.map(_.toString))
@@ -2391,7 +2152,6 @@ object SQLConf {
   val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
     .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2405,7 +2165,6 @@ object SQLConf {
       "CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, " +
       "AnalysisException is thrown while name conflict is detected in nested CTE. This config " +
       "will be removed in future versions and CORRECTED will be the only behavior.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2418,7 +2177,6 @@ object SQLConf {
       "When set to CORRECTED, classes from java.time.* packages are used for the same purpose. " +
       "The default value is EXCEPTION, RuntimeException is thrown when we will get different " +
       "results.")
-    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2428,7 +2186,6 @@ object SQLConf {
     buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
       .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2437,7 +2194,6 @@ object SQLConf {
       .doc("A comma-delimited string config of the optional additional remote Maven mirror " +
         "repositories. This is only used for downloading Hive jars in IsolatedClientLoader " +
         "if the default Maven Central repo is unreachable.")
-      .version("3.0.0")
       .stringConf
       .createWithDefault(
         "https://maven-central.storage-download.googleapis.com/maven2/")
@@ -2450,7 +2206,6 @@ object SQLConf {
         "all interval units out of the specified range. If it is set to `false`, " +
         "`ParseException` is thrown if the input does not match to the pattern " +
         "defined by `from` and `to`.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2460,7 +2215,6 @@ object SQLConf {
       .doc("When true, all database and table properties are not reserved and available for " +
         "create/alter syntaxes. But please be aware that the reserved properties will be " +
         "silently removed.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2469,7 +2223,6 @@ object SQLConf {
       .internal()
       .doc("When true, only a single file can be added using ADD FILE. If false, then users " +
         "can add directory by passing directory path to ADD FILE.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2477,13 +2230,11 @@ object SQLConf {
     buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
       .internal()
       .doc("When true, use legacy MySqlServer SMALLINT and REAL type mapping.")
-      .version("2.4.5")
       .booleanConf
       .createWithDefault(false)
 
   val CSV_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.csv.filterPushdown.enabled")
     .doc("When true, enable filter pushdown to CSV datasource.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2493,7 +2244,6 @@ object SQLConf {
       .doc("The number of partitions to be handled in one turn when use " +
         "`AlterTableAddPartitionCommand` to add partitions into table. The smaller " +
         "batch size is, the less memory is required for the real handler, e.g. Hive Metastore.")
-      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
@@ -2501,7 +2251,6 @@ object SQLConf {
   val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")
-    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2514,7 +2263,6 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2527,7 +2275,6 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2540,7 +2287,6 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2553,7 +2299,6 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
-      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 

From f85b1725531f7773c541cf349c9d778e299bf090 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Mon, 23 Mar 2020 13:01:44 +0900
Subject: [PATCH 0440/1280] [SPARK-31141][DSTREAMS][DOC] Add version
 information to the configuration of Dstreams
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the configuration of `Dstreams`.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.streaming.backpressure.enabled | 1.5.0 | SPARK-9967 and SPARK-10099 | 392bd19d678567751cd3844d9d166a7491c5887e#diff-1b584c4ed88a9022abb11d594f760997 |  
spark.streaming.backpressure.initialRate | 2.0.0 | SPARK-11627 | 7218c0eba957e0a079a407b79c3a050cce9647b2#diff-c64d571ef32d2dbf76e965ecd04a9f52 |  
spark.streaming.blockInterval | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-54d85b29e4349628a0de525c119399b5 |  
spark.streaming.receiver.maxRate | 1.0.2 | SPARK-1341 | ca19cfbcd5cfac9ad731350dfeea14355aec87d6#diff-c64d571ef32d2dbf76e965ecd04a9f52 |  
spark.streaming.receiver.writeAheadLog.enable | 1.2.1 | SPARK-4482 | ce5ea0fd611ce560f6e1fac83562469bdb97091e#diff-0607b70e4e79cbbc1a128c45784cb813 |  
spark.streaming.unpersist | 0.9.0 | None | 08b9fec93d00ff0ebb49af4d9ac72d2806eded02#diff-bcf5f84f78d23ebde7d532bea756bc57 |  
spark.streaming.stopGracefullyOnShutdown | 1.4.0 | SPARK-7776 | a17a5cb302c5fa6a4d3e9e3e0fa2100c0b5436d6#diff-8a7f0e3f26c15ba484e6312c3caf033d |  
spark.streaming.kafka.maxRetries | 1.3.0 | SPARK-4964 | a119cae48030520da9f26ee9a1270bed7f33031e#diff-26cb4369f86050dc2e75cd16291b2844 |  
spark.streaming.ui.retainedBatches | 1.0.0 | SPARK-1386 | f36dc3fed0a0671b0712d664db859da28c0a98e2#diff-56b8d67d07284cfab165d5363bd3500e |
spark.streaming.driver.writeAheadLog.closeFileAfterWrite | 1.6.0 | SPARK-11324 | 4f030b9e82172659d250281782ac573cbd1438fc#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.receiver.writeAheadLog.closeFileAfterWrite | 1.6.0 | SPARK-11324 | 4f030b9e82172659d250281782ac573cbd1438fc#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.receiver.writeAheadLog.class | 1.4.0 | SPARK-7056 | 1868bd40dcce23990b98748b0239bd00452b1ca5#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.receiver.writeAheadLog.rollingIntervalSecs | 1.4.0 | SPARK-7056 | 1868bd40dcce23990b98748b0239bd00452b1ca5#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.receiver.writeAheadLog.maxFailures | 1.2.0 | SPARK-4028 | 234de9232bcfa212317a8073c4a82c3863b36b14#diff-8cec1a581eebcad673dc8930b1a2801c |  
spark.streaming.driver.writeAheadLog.class | 1.4.0 | SPARK-7056 | 1868bd40dcce23990b98748b0239bd00452b1ca5#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.driver.writeAheadLog.rollingIntervalSecs | 1.4.0 | SPARK-7056 | 1868bd40dcce23990b98748b0239bd00452b1ca5#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.driver.writeAheadLog.maxFailures | 1.4.0 | SPARK-7056 | 1868bd40dcce23990b98748b0239bd00452b1ca5#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.driver.writeAheadLog.allowBatching | 1.6.0 | SPARK-11141 | dccc4645df629f35c4788d50b2c0a6ab381db4b7#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.driver.writeAheadLog.batchingTimeout | 1.6.0 | SPARK-11141 | dccc4645df629f35c4788d50b2c0a6ab381db4b7#diff-a1b3ec72e8d7cc91433a1cc64fe6e91d |  
spark.streaming.sessionByKey.deltaChainThreshold | 1.6.0 | SPARK-11290 | daa74be6f863061221bb0c2f94e70672e6fcbeaa#diff-e0a40541298f885606a2361ff9c5af6c |  
spark.streaming.backpressure.rateEstimator | 1.5.0 | SPARK-8977 | 819be46e5a73f2d19230354ebba30c58538590f5#diff-5dcaea3a4eca07f898fa88fe6d69e5c3 |  
spark.streaming.backpressure.pid.proportional | 1.5.0 | SPARK-8979 | 0a1d2ca42c8b31d6b0e70163795f0185d4622f87#diff-5dcaea3a4eca07f898fa88fe6d69e5c3 |  
spark.streaming.backpressure.pid.integral | 1.5.0 | SPARK-8979 | 0a1d2ca42c8b31d6b0e70163795f0185d4622f87#diff-5dcaea3a4eca07f898fa88fe6d69e5c3 |  
spark.streaming.backpressure.pid.derived | 1.5.0 | SPARK-8979 | 0a1d2ca42c8b31d6b0e70163795f0185d4622f87#diff-5dcaea3a4eca07f898fa88fe6d69e5c3 |  
spark.streaming.backpressure.pid.minRate | 1.5.0 | SPARK-9966 | 612b4609bdd38763725ae07d77c2176aa6756e64#diff-5dcaea3a4eca07f898fa88fe6d69e5c3 |  
spark.streaming.concurrentJobs | 0.7.0 | None | c97ebf64377e853ab7c616a103869a4417f25954#diff-839f06302b2d648a85436486fc13c85d |  
spark.streaming.internal.batchTime | 1.4.0 | SPARK-6862 | 1b7106b867bc0aa4d64b669d79b646f862acaf47#diff-25124e4f06a1da237bf486eceb1f7967 | It's not a configuration, it's a property
spark.streaming.internal.outputOpId | 1.4.0 | SPARK-6862 | 1b7106b867bc0aa4d64b669d79b646f862acaf47#diff-25124e4f06a1da237bf486eceb1f7967 | It's not a configuration, it's a property
spark.streaming.clock | 0.7.0 | None | cae894ee7aefa4cf9b1952038a48be81e1d2a856#diff-839f06302b2d648a85436486fc13c85d |  
spark.streaming.gracefulStopTimeout | 1.0.0 | SPARK-1332 | 94cbe2329021296b660d88f3e8ef3734374020d2#diff-2f8c5c038fda47b9875e10785fdd2498 |  
spark.streaming.manualClock.jump | 0.7.0 | None | fc3d0b602a08fdd182c2138506d1cd9952631f95#diff-839f06302b2d648a85436486fc13c85d |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'

### How was this patch tested?
Exists UT

Closes #27898 from beliefer/add-version-to-dstream-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md                         | 24 ++++++++++++-----
 .../spark/streaming/StreamingConf.scala       | 27 +++++++++++++++++++
 2 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 8bfd4d522efb6..ce4afc56e330a 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2481,7 +2481,7 @@ Spark subsystems.
 ### Spark Streaming
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.streaming.backpressure.enabled</code></td>
   <td>false</td>
@@ -2494,6 +2494,7 @@ Spark subsystems.
     <code>spark.streaming.receiver.maxRate</code> and <code>spark.streaming.kafka.maxRatePerPartition</code>
     if they are set (see below).
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.backpressure.initialRate</code></td>
@@ -2502,6 +2503,7 @@ Spark subsystems.
     This is the initial maximum receiving rate at which each receiver will receive data for the
     first batch when the backpressure mechanism is enabled.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.blockInterval</code></td>
@@ -2512,6 +2514,7 @@ Spark subsystems.
     <a href="streaming-programming-guide.html#level-of-parallelism-in-data-receiving">performance
      tuning</a> section in the Spark Streaming programming guide for more details.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.receiver.maxRate</code></td>
@@ -2523,6 +2526,7 @@ Spark subsystems.
     See the <a href="streaming-programming-guide.html#deploying-applications">deployment guide</a>
     in the Spark Streaming programming guide for mode details.
   </td>
+  <td>1.0.2</td>
 </tr>
 <tr>
   <td><code>spark.streaming.receiver.writeAheadLog.enable</code></td>
@@ -2533,6 +2537,7 @@ Spark subsystems.
     See the <a href="streaming-programming-guide.html#deploying-applications">deployment guide</a>
     in the Spark Streaming programming guide for more details.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.streaming.unpersist</code></td>
@@ -2544,6 +2549,7 @@ Spark subsystems.
     streaming application as they will not be cleared automatically. But it comes at the cost of
     higher memory usage in Spark.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.stopGracefullyOnShutdown</code></td>
@@ -2552,6 +2558,7 @@ Spark subsystems.
     If <code>true</code>, Spark shuts down the <code>StreamingContext</code> gracefully on JVM
     shutdown rather than immediately.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.kafka.maxRatePerPartition</code></td>
@@ -2564,12 +2571,12 @@ Spark subsystems.
   </td>
 </tr>
 <tr>
-    <td><code>spark.streaming.kafka.minRatePerPartition</code></td>
-    <td>1</td>
-    <td>
-      Minimum rate (number of records per second) at which data will be read from each Kafka
-      partition when using the new Kafka direct stream API.
-    </td>
+  <td><code>spark.streaming.kafka.minRatePerPartition</code></td>
+  <td>1</td>
+  <td>
+    Minimum rate (number of records per second) at which data will be read from each Kafka
+    partition when using the new Kafka direct stream API.
+  </td>
 </tr>
 <tr>
   <td><code>spark.streaming.ui.retainedBatches</code></td>
@@ -2577,6 +2584,7 @@ Spark subsystems.
   <td>
     How many batches the Spark Streaming UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.driver.writeAheadLog.closeFileAfterWrite</code></td>
@@ -2586,6 +2594,7 @@ Spark subsystems.
     when you want to use S3 (or any file system that does not support flushing) for the metadata WAL
     on the driver.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.receiver.writeAheadLog.closeFileAfterWrite</code></td>
@@ -2595,6 +2604,7 @@ Spark subsystems.
     when you want to use S3 (or any file system that does not support flushing) for the data WAL
     on the receivers.
   </td>
+  <td>1.6.0</td>
 </tr>
 </table>
 
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala b/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
index 71aefd647f905..bb80bd7072e8e 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/StreamingConf.scala
@@ -26,135 +26,162 @@ object StreamingConf {
 
   private[streaming] val BACKPRESSURE_ENABLED =
     ConfigBuilder("spark.streaming.backpressure.enabled")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(false)
 
   private[streaming] val RECEIVER_MAX_RATE =
     ConfigBuilder("spark.streaming.receiver.maxRate")
+      .version("1.0.2")
       .longConf
       .createWithDefault(Long.MaxValue)
 
   private[streaming] val BACKPRESSURE_INITIAL_RATE =
     ConfigBuilder("spark.streaming.backpressure.initialRate")
+      .version("2.0.0")
       .fallbackConf(RECEIVER_MAX_RATE)
 
   private[streaming] val BLOCK_INTERVAL =
     ConfigBuilder("spark.streaming.blockInterval")
+      .version("0.8.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("200ms")
 
   private[streaming] val RECEIVER_WAL_ENABLE_CONF_KEY =
     ConfigBuilder("spark.streaming.receiver.writeAheadLog.enable")
+      .version("1.2.1")
       .booleanConf
       .createWithDefault(false)
 
   private[streaming] val RECEIVER_WAL_CLASS_CONF_KEY =
     ConfigBuilder("spark.streaming.receiver.writeAheadLog.class")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
   private[streaming] val RECEIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
     ConfigBuilder("spark.streaming.receiver.writeAheadLog.rollingIntervalSecs")
+      .version("1.4.0")
       .intConf
       .createWithDefault(60)
 
   private[streaming] val RECEIVER_WAL_MAX_FAILURES_CONF_KEY =
     ConfigBuilder("spark.streaming.receiver.writeAheadLog.maxFailures")
+      .version("1.2.0")
       .intConf
       .createWithDefault(3)
 
   private[streaming] val RECEIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
     ConfigBuilder("spark.streaming.receiver.writeAheadLog.closeFileAfterWrite")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(false)
 
   private[streaming] val DRIVER_WAL_CLASS_CONF_KEY =
     ConfigBuilder("spark.streaming.driver.writeAheadLog.class")
+      .version("1.4.0")
       .stringConf
       .createOptional
 
   private[streaming] val DRIVER_WAL_ROLLING_INTERVAL_CONF_KEY =
     ConfigBuilder("spark.streaming.driver.writeAheadLog.rollingIntervalSecs")
+      .version("1.4.0")
       .intConf
       .createWithDefault(60)
 
   private[streaming] val DRIVER_WAL_MAX_FAILURES_CONF_KEY =
     ConfigBuilder("spark.streaming.driver.writeAheadLog.maxFailures")
+      .version("1.4.0")
       .intConf
       .createWithDefault(3)
 
   private[streaming] val DRIVER_WAL_CLOSE_AFTER_WRITE_CONF_KEY =
     ConfigBuilder("spark.streaming.driver.writeAheadLog.closeFileAfterWrite")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(false)
 
   private[streaming] val DRIVER_WAL_BATCHING_CONF_KEY =
     ConfigBuilder("spark.streaming.driver.writeAheadLog.allowBatching")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(true)
 
   private[streaming] val DRIVER_WAL_BATCHING_TIMEOUT_CONF_KEY =
     ConfigBuilder("spark.streaming.driver.writeAheadLog.batchingTimeout")
+      .version("1.6.0")
       .longConf
       .createWithDefault(5000)
 
   private[streaming] val STREAMING_UNPERSIST =
     ConfigBuilder("spark.streaming.unpersist")
+      .version("0.9.0")
       .booleanConf
       .createWithDefault(true)
 
   private[streaming] val STOP_GRACEFULLY_ON_SHUTDOWN =
     ConfigBuilder("spark.streaming.stopGracefullyOnShutdown")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[streaming] val UI_RETAINED_BATCHES =
     ConfigBuilder("spark.streaming.ui.retainedBatches")
+      .version("1.0.0")
       .intConf
       .createWithDefault(1000)
 
   private[streaming] val SESSION_BY_KEY_DELTA_CHAIN_THRESHOLD =
     ConfigBuilder("spark.streaming.sessionByKey.deltaChainThreshold")
+      .version("1.6.0")
       .intConf
       .createWithDefault(DELTA_CHAIN_LENGTH_THRESHOLD)
 
   private[streaming] val BACKPRESSURE_RATE_ESTIMATOR =
     ConfigBuilder("spark.streaming.backpressure.rateEstimator")
+      .version("1.5.0")
       .stringConf
       .createWithDefault("pid")
 
   private[streaming] val BACKPRESSURE_PID_PROPORTIONAL =
     ConfigBuilder("spark.streaming.backpressure.pid.proportional")
+      .version("1.5.0")
       .doubleConf
       .createWithDefault(1.0)
 
   private[streaming] val BACKPRESSURE_PID_INTEGRAL =
     ConfigBuilder("spark.streaming.backpressure.pid.integral")
+      .version("1.5.0")
       .doubleConf
       .createWithDefault(0.2)
 
   private[streaming] val BACKPRESSURE_PID_DERIVED =
     ConfigBuilder("spark.streaming.backpressure.pid.derived")
+      .version("1.5.0")
       .doubleConf
       .createWithDefault(0.0)
 
   private[streaming] val BACKPRESSURE_PID_MIN_RATE =
     ConfigBuilder("spark.streaming.backpressure.pid.minRate")
+      .version("1.5.0")
       .doubleConf
       .createWithDefault(100)
 
   private[streaming] val CONCURRENT_JOBS =
     ConfigBuilder("spark.streaming.concurrentJobs")
+      .version("0.7.0")
       .intConf
       .createWithDefault(1)
 
   private[streaming] val GRACEFUL_STOP_TIMEOUT =
     ConfigBuilder("spark.streaming.gracefulStopTimeout")
+      .version("1.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[streaming] val MANUAL_CLOCK_JUMP =
     ConfigBuilder("spark.streaming.manualClock.jump")
+      .version("0.7.0")
       .longConf
       .createWithDefault(0)
 

From 05615812d21521ccb248439e9c3122ca7592380a Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 26 Mar 2020 20:11:15 +0900
Subject: [PATCH 0441/1280] [SPARK-31228][DSTREAMS] Add version information to
 the configuration of Kafka
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Add version information to the configuration of Kafka.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.streaming.kafka.consumer.cache.enabled | 2.2.1 | SPARK-19185 | 02cf178bb2a7dc8b4c06eb040c44b6453e41ed15#diff-c465bbcc83b2ecc7530d1c0128e4432b |  
spark.streaming.kafka.consumer.poll.ms | 2.0.1 | SPARK-12177 | 3134f116a3565c3a299fa2e7094acd7304d64280#diff-4597d93a0e951f7199697dba7dd0dc32 |  
spark.streaming.kafka.consumer.cache.initialCapacity | 2.0.1 | SPARK-12177 | 3134f116a3565c3a299fa2e7094acd7304d64280#diff-4597d93a0e951f7199697dba7dd0dc32 |  
spark.streaming.kafka.consumer.cache.maxCapacity | 2.0.1 | SPARK-12177 | 3134f116a3565c3a299fa2e7094acd7304d64280#diff-4597d93a0e951f7199697dba7dd0dc32 |  
spark.streaming.kafka.consumer.cache.loadFactor | 2.0.1 | SPARK-12177 | 3134f116a3565c3a299fa2e7094acd7304d64280#diff-4597d93a0e951f7199697dba7dd0dc32 |  
spark.streaming.kafka.maxRatePerPartition | 1.3.0 | SPARK-4964 | a119cae48030520da9f26ee9a1270bed7f33031e#diff-26cb4369f86050dc2e75cd16291b2844 |  
spark.streaming.kafka.minRatePerPartition | 2.4.0 | SPARK-25233 | 135ff16a3510a4dfb3470904004dae9848005019#diff-815f6ec5caf9e4beb355f5f981171f1f |  
spark.streaming.kafka.allowNonConsecutiveOffsets | 2.3.1 | SPARK-24067 | 1d598b771de3b588a2f377ae7ccf8193156641f2#diff-4597d93a0e951f7199697dba7dd0dc32 |  
spark.kafka.producer.cache.timeout | 2.2.1 | SPARK-19968 | f6730a70cb47ebb3df7f42209df7b076aece1093#diff-ac8844e8d791a75aaee3d0d10bfc1f2a |  
spark.kafka.producer.cache.evictorThreadRunInterval | 3.0.0 | SPARK-21869 | 7bff2db9ed803e05a43c2d875c1dea819d81248a#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.consumer.cache.capacity | 3.0.0 | SPARK-27687 | efa303581ac61d6f517aacd08883da2d01530bd2#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.consumer.cache.jmx.enable | 3.0.0 | SPARK-25151 | 594c9c5a3ece0e913949c7160bb4925e5d289e44#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.consumer.cache.timeout | 3.0.0 | SPARK-25151 | 594c9c5a3ece0e913949c7160bb4925e5d289e44#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.consumer.cache.evictorThreadRunInterval | 3.0.0 | SPARK-25151 | 594c9c5a3ece0e913949c7160bb4925e5d289e44#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.consumer.fetchedData.cache.timeout | 3.0.0 | SPARK-25151 | 594c9c5a3ece0e913949c7160bb4925e5d289e44#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval | 3.0.0 | SPARK-25151 | 594c9c5a3ece0e913949c7160bb4925e5d289e44#diff-ea8349d528fe8d1b0a8ffa2840ff4bcd |  
spark.kafka.clusters.${cluster}.auth.bootstrap.servers | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.target.bootstrap.servers.regex | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.security.protocol | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.sasl.kerberos.service.name | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.ssl.truststore.location | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.ssl.truststore.password | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.ssl.keystore.location | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.ssl.keystore.password | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.ssl.key.password | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  
spark.kafka.clusters.${cluster}.sasl.token.mechanism | 3.0.0 | SPARK-27294 | 2f558094257c38d26650049f2ac93be6d65d6d85#diff-7df71bd47f5a3428ebdb05ced3c31f49 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #27989 from beliefer/add-version-to-kafka-config.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md                         |  2 ++
 .../structured-streaming-kafka-integration.md | 26 +++++++++++---
 .../apache/spark/sql/kafka010/package.scala   |  8 +++++
 .../spark/streaming/kafka010/package.scala    | 36 +++++++++++--------
 4 files changed, 54 insertions(+), 18 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index ce4afc56e330a..4f3f542048d9c 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2569,6 +2569,7 @@ Spark subsystems.
     <a href="streaming-kafka-0-10-integration.html">Kafka Integration guide</a>
     for more details.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.kafka.minRatePerPartition</code></td>
@@ -2577,6 +2578,7 @@ Spark subsystems.
     Minimum rate (number of records per second) at which data will be read from each Kafka
     partition when using the new Kafka direct stream API.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.streaming.ui.retainedBatches</code></td>
diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md
index a1eeee54987db..016faa735acd6 100644
--- a/docs/structured-streaming-kafka-integration.md
+++ b/docs/structured-streaming-kafka-integration.md
@@ -525,21 +525,24 @@ The caching key is built up from the following information:
 The following properties are available to configure the consumer pool:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.cache.capacity</td>
   <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
   <td>64</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.timeout</td>
   <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.evictorThreadRunInterval</td>
   <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.jmx.enable</td>
@@ -547,6 +550,7 @@ The following properties are available to configure the consumer pool:
   The prefix of JMX name is set to "kafka010-cached-simple-kafka-consumer-pool".
   </td>
   <td>false</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -571,16 +575,18 @@ Note that it doesn't leverage Apache Commons Pool due to the difference of chara
 The following properties are available to configure the fetched data pool:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.timeout</td>
   <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval</td>
   <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -816,16 +822,18 @@ It will use different Kafka producer when delegation token is renewed; Kafka pro
 The following properties are available to configure the producer pool:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.producer.cache.timeout</td>
   <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>10m (10 minutes)</td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td>spark.kafka.producer.cache.evictorThreadRunInterval</td>
   <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -935,7 +943,7 @@ When none of the above applies then unsecure connection assumed.
 Delegation tokens can be obtained from multiple clusters and <code>${cluster}</code> is an arbitrary unique identifier which helps to group different configurations.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.auth.bootstrap.servers</code></td>
     <td>None</td>
@@ -943,6 +951,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       A list of coma separated host/port pairs to use for establishing the initial connection
       to the Kafka cluster. For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.target.bootstrap.servers.regex</code></td>
@@ -953,6 +962,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       If multiple clusters match the address, an exception will be thrown and the query won't be started.
       Kafka's secure and unsecure listeners are bound to different ports. When both used the secure listener port has to be part of the regular expression.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.security.protocol</code></td>
@@ -962,6 +972,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       <code>bootstrap.servers</code> config matches (for further details please see <code>spark.kafka.clusters.${cluster}.target.bootstrap.servers.regex</code>),
       and can be overridden by setting <code>kafka.security.protocol</code> on the source or sink.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.sasl.kerberos.service.name</code></td>
@@ -970,6 +981,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The Kerberos principal name that Kafka runs as. This can be defined either in Kafka's JAAS config or in Kafka's config.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.truststore.location</code></td>
@@ -977,6 +989,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
     <td>
       The location of the trust store file. For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.truststore.password</code></td>
@@ -985,6 +998,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The store password for the trust store file. This is optional and only needed if <code>spark.kafka.clusters.${cluster}.ssl.truststore.location</code> is configured.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.keystore.location</code></td>
@@ -993,6 +1007,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The location of the key store file. This is optional for client and can be used for two-way authentication for client.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.keystore.password</code></td>
@@ -1001,6 +1016,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The store password for the key store file. This is optional and only needed if <code>spark.kafka.clusters.${cluster}.ssl.keystore.location</code> is configured.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.ssl.key.password</code></td>
@@ -1009,6 +1025,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       The password of the private key in the key store file. This is optional for client.
       For further details please see Kafka documentation. Only used to obtain delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
   <tr>
     <td><code>spark.kafka.clusters.${cluster}.sasl.token.mechanism</code></td>
@@ -1017,6 +1034,7 @@ Delegation tokens can be obtained from multiple clusters and <code>${cluster}</c
       SASL mechanism used for client connections with delegation token. Because SCRAM login module used for authentication a compatible mechanism has to be set here.
       For further details please see Kafka documentation (<code>sasl.mechanism</code>). Only used to authenticate against Kafka broker with delegation token.
     </td>
+    <td>3.0.0</td>
   </tr>
 </table>
 
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala
index 460bb8bd34ec6..b7e42c00b8cff 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/package.scala
@@ -29,6 +29,7 @@ package object kafka010 {   // scalastyle:ignore
   private[kafka010] val PRODUCER_CACHE_TIMEOUT =
     ConfigBuilder("spark.kafka.producer.cache.timeout")
       .doc("The expire time to remove the unused producers.")
+      .version("2.2.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10m")
 
@@ -36,6 +37,7 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.producer.cache.evictorThreadRunInterval")
       .doc("The interval of time between runs of the idle evictor thread for producer pool. " +
         "When non-positive, no idle evictor thread will be run.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1m")
 
@@ -43,12 +45,14 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.consumer.cache.capacity")
       .doc("The maximum number of consumers cached. Please note it's a soft limit" +
         " (check Structured Streaming Kafka integration guide for further details).")
+      .version("3.0.0")
       .intConf
       .createWithDefault(64)
 
   private[kafka010] val CONSUMER_CACHE_JMX_ENABLED =
     ConfigBuilder("spark.kafka.consumer.cache.jmx.enable")
       .doc("Enable or disable JMX for pools created with this configuration instance.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -57,6 +61,7 @@ package object kafka010 {   // scalastyle:ignore
       .doc("The minimum amount of time a consumer may sit idle in the pool before " +
         "it is eligible for eviction by the evictor. " +
         "When non-positive, no consumers will be evicted from the pool due to idle time alone.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5m")
 
@@ -64,6 +69,7 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.consumer.cache.evictorThreadRunInterval")
       .doc("The interval of time between runs of the idle evictor thread for consumer pool. " +
         "When non-positive, no idle evictor thread will be run.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1m")
 
@@ -72,6 +78,7 @@ package object kafka010 {   // scalastyle:ignore
       .doc("The minimum amount of time a fetched data may sit idle in the pool before " +
         "it is eligible for eviction by the evictor. " +
         "When non-positive, no fetched data will be evicted from the pool due to idle time alone.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("5m")
 
@@ -79,6 +86,7 @@ package object kafka010 {   // scalastyle:ignore
     ConfigBuilder("spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval")
       .doc("The interval of time between runs of the idle evictor thread for fetched data pool. " +
         "When non-positive, no idle evictor thread will be run.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1m")
 }
diff --git a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala
index 3d2921fa900a3..0679a496fc3c7 100644
--- a/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala
+++ b/external/kafka-0-10/src/main/scala/org/apache/spark/streaming/kafka010/package.scala
@@ -26,43 +26,51 @@ package object kafka010 { //scalastyle:ignore
 
   private[spark] val CONSUMER_CACHE_ENABLED =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.enabled")
+      .version("2.2.1")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CONSUMER_POLL_MS =
     ConfigBuilder("spark.streaming.kafka.consumer.poll.ms")
-    .longConf
-    .createOptional
+      .version("2.0.1")
+      .longConf
+      .createOptional
 
   private[spark] val CONSUMER_CACHE_INITIAL_CAPACITY =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.initialCapacity")
-    .intConf
-    .createWithDefault(16)
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(16)
 
   private[spark] val CONSUMER_CACHE_MAX_CAPACITY =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.maxCapacity")
-    .intConf
-    .createWithDefault(64)
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(64)
 
   private[spark] val CONSUMER_CACHE_LOAD_FACTOR =
     ConfigBuilder("spark.streaming.kafka.consumer.cache.loadFactor")
-    .doubleConf
-    .createWithDefault(0.75)
+      .version("2.0.1")
+      .doubleConf
+      .createWithDefault(0.75)
 
   private[spark] val MAX_RATE_PER_PARTITION =
     ConfigBuilder("spark.streaming.kafka.maxRatePerPartition")
-    .longConf
-    .createWithDefault(0)
+      .version("1.3.0")
+      .longConf
+      .createWithDefault(0)
 
   private[spark] val MIN_RATE_PER_PARTITION =
     ConfigBuilder("spark.streaming.kafka.minRatePerPartition")
-    .longConf
-    .createWithDefault(1)
+      .version("2.4.0")
+      .longConf
+      .createWithDefault(1)
 
   private[spark] val ALLOW_NON_CONSECUTIVE_OFFSETS =
     ConfigBuilder("spark.streaming.kafka.allowNonConsecutiveOffsets")
-    .booleanConf
-    .createWithDefault(false)
+      .version("2.3.1")
+      .booleanConf
+      .createWithDefault(false)
 
 }
 

From 2d520368bd50584696a04f99d0b1cb110ab70076 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 27 Feb 2020 10:58:44 +0900
Subject: [PATCH 0442/1280] [SPARK-30841][SQL][DOC] Add version information to
 the configuration of SQL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add version information to the configuration of Spark SQL.
Note: Because SQLConf has a lot of configuration items, I split the items into two PR. Another PR will follows this PR.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.analyzer.maxIterations | 3.0.0 | SPARK-30138 | c2f29d5ea58eb4565cc5602937d6d0bb75558513#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.excludedRules | 2.4.0 | SPARK-24802 | 434319e73f8cb6e080671bdde42a72228bd814ef#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.maxIterations | 2.0.0 | SPARK-14677 | f4be0946af219379fb2476e6f80b2e50463adeb2#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.optimizer.inSetConversionThreshold | 2.0.0 | SPARK-14796 | 3647120a5a879edf3a96a5fd68fb7aa849ad57ef#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.optimizer.inSetSwitchThreshold | 3.0.0 | SPARK-26205 | 0c23a39384b7ae5fb4aeb4f7f6fe72007b84bbd2#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.planChangeLog.level | 3.0.0 | SPARK-25415 | 8b702e1e0aba1d3e4b0aa582f20cf99f80a44a09#diff-9a6b543db706f1a90f790783d6930a13 | This configuration does not exist in branch-2.4 branch, but from the branch-3.0 git log, it is found that the version number of the pom.xml file is 2.4.0-SNAPSHOT
spark.sql.optimizer.planChangeLog.rules | 3.0.0 | SPARK-25415 | 8b702e1e0aba1d3e4b0aa582f20cf99f80a44a09#diff-9a6b543db706f1a90f790783d6930a13 | This configuration does not exist in branch-2.4 branch, but from the branch-3.0 git log, it is found that the version number of the pom.xml file is 2.4.0-SNAPSHOT
spark.sql.optimizer.planChangeLog.batches | 3.0.0 | SPARK-27088 | 074533334d01afdd7862a1ac6c5a7a672bcce3f8#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.dynamicPartitionPruning.enabled | 3.0.0 | SPARK-11150 | a7a3935c97d1fe6060cae42bbc9229c087b648ab#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.dynamicPartitionPruning.useStats | 3.0.0 | SPARK-11150 | a7a3935c97d1fe6060cae42bbc9229c087b648ab#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio | 3.0.0 | SPARK-11150 | a7a3935c97d1fe6060cae42bbc9229c087b648ab#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly | 3.0.0 | SPARK-30528 | 59a13c9b7bc3b3aa5b5bc30a60344f849c0f8012#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.inMemoryColumnarStorage.compressed | 1.0.1 | SPARK-2631 | 86534d0f5255362618c05a07b0171ec35c915822#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.inMemoryColumnarStorage.batchSize | 1.1.1 | SPARK-2650 | 779d1eb26d0f031791e93c908d51a59c3b422a55#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.inMemoryColumnarStorage.partitionPruning | 1.2.0 | SPARK-2961 | 248067adbe90f93c7d5e23aa61b3072dfdf48a8a#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.inMemoryTableScanStatistics.enable | 3.0.0 | SPARK-28257 | 42b80ae128ab1aa8a87c1376fe88e2cde52e6e4f#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.inMemoryColumnarStorage.enableVectorizedReader | 2.3.1 | SPARK-23312 | e5e9f9a430c827669ecfe9d5c13cc555fc89c980#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.columnVector.offheap.enabled | 2.3.0 | SPARK-20101 | 572af5027e45ca96e0d283a8bf7c84dcf476f9bc#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.join.preferSortMergeJoin | 2.0.0 | SPARK-13977 | 9c23c818ca0175c8f2a4a66eac261ec251d27c97#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.sort.enableRadixSort | 2.0.0 | SPARK-14724 | e2b5647ab92eb478b3f7b36a0ce6faf83e24c0e5#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.autoBroadcastJoinThreshold | 1.1.0 | SPARK-2393 | c7db274be79f448fda566208946cb50958ea9b1a#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.limit.scaleUpFactor | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.hive.advancedPartitionPredicatePushdown.enabled | 2.3.0 | SPARK-20331 | d8cada8d1d3fce979a4bc1f9879593206722a3b9#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.shuffle.partitions | 1.1.0 | SPARK-1508 | 08ed9ad81397b71206c4dc903bfb94b6105691ed#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.adaptive.enabled | 1.6.0 | SPARK-9858 and SPARK-9859 and SPARK-9861 | d728d5c98658c44ed2949b55d36edeaa46f8c980#diff-41ef65b9ef5b518f77e2a03559893f4d |
spark.sql.adaptive.forceApply | 3.0.0 | SPARK-30719 | b29cb1a82b1a1facf1dd040025db93d998dad4cd#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.shuffle.reducePostShufflePartitions | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.shuffle.fetchShuffleBlocksInBatch | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.shuffle.minNumPostShufflePartitions | 3.0.0 | SPARK-9853 | 8616109061efc5b23b24bb9ec4a3c0f2745903c1#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.shuffle.targetPostShuffleInputSize | 1.6.0 | SPARK-9858 and SPARK-9859 and SPARK-9861 | d728d5c98658c44ed2949b55d36edeaa46f8c980#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.adaptive.shuffle.maxNumPostShufflePartitions | 3.0.0 | SPARK-9853 | 8616109061efc5b23b24bb9ec4a3c0f2745903c1#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.shuffle.localShuffleReader.enabled | 3.0.0 | SPARK-29893 | 6e581cf164c3a2930966b270ac1406dc1195c942#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.skewedJoinOptimization.enabled | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.skewedJoinOptimization.skewedPartitionFactor | 3.0.0 | SPARK-30812 | 5b36cdbbfef147e93b35eaa4f8e0bea9690b6d06#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin | 3.0.0 | SPARK-9853 and SPARK-29002 | 8616109061efc5b23b24bb9ec4a3c0f2745903c1#diff-9a6b543db706f1a90f790783d6930a13 and b2f06608b785f577999318c00f2c315f39d90889#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.subexpressionElimination.enabled | 1.6.0 | SPARK-10371 | f38509a763816f43a224653fe65e4645894c9fc4#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.caseSensitive | 1.4.0 | SPARK-4699 | 21bd7222e55b9cf684c072141998a0623a69f514#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.constraintPropagation.enabled | 2.2.0 | SPARK-19846 | e011004bedca47be998a0c14fe22a6f9bb5090cd#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parser.escapedStringLiterals | 2.2.1 | SPARK-20399 | 3d1908fd58fd9b1970cbffebdb731bfe4c776ad9#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sources.fileCompressionFactor | 2.3.1 | SPARK-22790 | 0fc5533e53ad03eb67590ddd231f40c2713150c3#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parquet.mergeSchema | 1.5.0 | SPARK-8690 | 246265f2bb056d5e9011d3331b809471a24ff8d7#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.respectSummaryFiles | 1.5.0 | SPARK-8838 | 6175d6cfe795fbd88e3ee713fac375038a3993a8#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.binaryAsString | 1.1.1 | SPARK-2927 | de501e169f24e4573747aec85b7651c98633c028#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.int96AsTimestamp | 1.3.0 | SPARK-4987 | 67d52207b5cf2df37ca70daff2a160117510f55e#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.int96TimestampConversion | 2.3.0 | SPARK-12297 | acf7ef3154e094875fa89f30a78ab111b267db91#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parquet.outputTimestampType | 2.3.0 | SPARK-10365 | 21a7bfd5c324e6c82152229f1394f26afeae771c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parquet.compression.codec | 1.1.1 | SPARK-3131 | 3a9d874d7a46ab8b015631d91ba479d9a0ba827f#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.filterPushdown | 1.2.0 | SPARK-4391 | 576688aa2a19bd4ba239a2b93af7947f983e5124#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.filterPushdown.date | 2.4.0 | SPARK-23727 | b02e76cbffe9e589b7a4e60f91250ca12a4420b2#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parquet.filterPushdown.timestamp | 2.4.0 | SPARK-24718 | 43e4e851b642bbee535d22e1b9e72ec6b99f6ed4#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.parquet.filterPushdown.decimal | 2.4.0 | SPARK-24549 | 9549a2814951f9ba969955d78ac4bd2240f85989#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.parquet.filterPushdown.string.startsWith | 2.4.0 | SPARK-24638 | 03545ce6de08bd0ad685c5f59b73bc22dfc40887#diff-9a6b543db706f1a90f790783d6930a13 | 
spark.sql.parquet.pushdown.inFilterThreshold | 2.4.0 | SPARK-17091 | e1de34113e057707dfc5ff54a8109b3ec7c16dfb#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parquet.writeLegacyFormat | 1.6.0 | SPARK-10400 | 01cd688f5245cbb752863100b399b525b31c3510#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.output.committer.class | 1.5.0 | SPARK-8139 | 111d6b9b8a584b962b6ae80c7aa8c45845ce0099#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.enableVectorizedReader | 2.0.0 | SPARK-13486 | 2b2c8c33236677c916541f956f7b94bba014a9ce#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.parquet.recordLevelFilter.enabled | 2.3.0 | SPARK-17310 | 673c67046598d33b9ecf864024ca7a937c1998d6#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.parquet.columnarReaderBatchSize | 2.4.0 | SPARK-23188 | cc41245fa3f954f961541bf4b4275c28473042b8#diff-9a6b543db706f1a90f790783d6930a13 |  

Supplemental configuration version information.

No

Exists UT

Closes #27691 from beliefer/add-version-to-sql-config-part-one.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 66 ++++++++++++++++++-
 1 file changed, 63 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 9f9e55601d69e..265eb629a28d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -177,6 +177,7 @@ object SQLConf {
   val ANALYZER_MAX_ITERATIONS = buildConf("spark.sql.analyzer.maxIterations")
     .internal()
     .doc("The max number of iterations the analyzer runs.")
+    .version("3.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -185,12 +186,14 @@ object SQLConf {
       "specified by their rule names and separated by comma. It is not guaranteed that all the " +
       "rules in this configuration will eventually be excluded, as some rules are necessary " +
       "for correctness. The optimizer will log the rules that have indeed been excluded.")
+    .version("2.4.0")
     .stringConf
     .createOptional
 
   val OPTIMIZER_MAX_ITERATIONS = buildConf("spark.sql.optimizer.maxIterations")
     .internal()
     .doc("The max number of iterations the optimizer runs.")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -198,6 +201,7 @@ object SQLConf {
     buildConf("spark.sql.optimizer.inSetConversionThreshold")
       .internal()
       .doc("The threshold of set size for InSet conversion.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -206,6 +210,7 @@ object SQLConf {
       .internal()
       .doc("Configures the max set size in InSet for which Spark will generate code with " +
         "switch statements. This is applicable only to bytes, shorts, ints, dates.")
+      .version("3.0.0")
       .intConf
       .checkValue(threshold => threshold >= 0 && threshold <= 600, "The max set size " +
         "for using switch statements in InSet must be non-negative and less than or equal to 600")
@@ -216,6 +221,7 @@ object SQLConf {
     .doc("Configures the log level for logging the change from the original plan to the new " +
       "plan after a rule or batch is applied. The value can be 'trace', 'debug', 'info', " +
       "'warn', or 'error'. The default log level is 'trace'.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValue(logLevel => Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR").contains(logLevel),
@@ -227,6 +233,7 @@ object SQLConf {
     .internal()
     .doc("Configures a list of rules to be logged in the optimizer, in which the rules are " +
       "specified by their rule names and separated by comma.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
@@ -234,12 +241,14 @@ object SQLConf {
     .internal()
     .doc("Configures a list of batches to be logged in the optimizer, in which the batches " +
       "are specified by their batch names and separated by comma.")
+    .version("3.0.0")
     .stringConf
     .createOptional
 
   val DYNAMIC_PARTITION_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.dynamicPartitionPruning.enabled")
       .doc("When true, we will generate predicate for partition column when it's used as join key")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -249,6 +258,7 @@ object SQLConf {
       .doc("When true, distinct count statistics will be used for computing the data size of the " +
         "partitioned table after dynamic partition pruning, in order to evaluate if it is worth " +
         "adding an extra subquery as the pruning filter if broadcast reuse is not applicable.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -259,6 +269,7 @@ object SQLConf {
       "used as the fallback filter ratio for computing the data size of the partitioned table " +
       "after dynamic partition pruning, in order to evaluate if it is worth adding an extra " +
       "subquery as the pruning filter if broadcast reuse is not applicable.")
+    .version("3.0.0")
     .doubleConf
     .createWithDefault(0.5)
 
@@ -267,18 +278,21 @@ object SQLConf {
       .internal()
       .doc("When true, dynamic partition pruning will only apply when the broadcast exchange of " +
         "a broadcast hash join operation can be reused as the dynamic pruning filter.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val COMPRESS_CACHED = buildConf("spark.sql.inMemoryColumnarStorage.compressed")
     .doc("When set to true Spark SQL will automatically select a compression codec for each " +
       "column based on statistics of the data.")
+    .version("1.0.1")
     .booleanConf
     .createWithDefault(true)
 
   val COLUMN_BATCH_SIZE = buildConf("spark.sql.inMemoryColumnarStorage.batchSize")
     .doc("Controls the size of batches for columnar caching.  Larger batch sizes can improve " +
       "memory utilization and compression, but risk OOMs when caching data.")
+    .version("1.1.1")
     .intConf
     .createWithDefault(10000)
 
@@ -286,6 +300,7 @@ object SQLConf {
     buildConf("spark.sql.inMemoryColumnarStorage.partitionPruning")
       .internal()
       .doc("When true, enable partition pruning for in-memory columnar tables.")
+      .version("1.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -293,12 +308,14 @@ object SQLConf {
     buildConf("spark.sql.inMemoryTableScanStatistics.enable")
       .internal()
       .doc("When true, enable in-memory table scan accumulators.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val CACHE_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.inMemoryColumnarStorage.enableVectorizedReader")
       .doc("Enables vectorized reader for columnar caching.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -306,12 +323,14 @@ object SQLConf {
     buildConf("spark.sql.columnVector.offheap.enabled")
       .internal()
       .doc("When true, use OffHeapColumnVector in ColumnarBatch.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val PREFER_SORTMERGEJOIN = buildConf("spark.sql.join.preferSortMergeJoin")
     .internal()
     .doc("When true, prefer sort merge join over shuffle hash join.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -320,6 +339,7 @@ object SQLConf {
     .doc("When true, enable use of radix sort when possible. Radix sort is much faster but " +
       "requires additional memory to be reserved up-front. The memory overhead may be " +
       "significant when sorting very small rows (up to 50% more in this case).")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -330,6 +350,7 @@ object SQLConf {
       "command `ANALYZE TABLE <tableName> COMPUTE STATISTICS noscan` has been " +
       "run, and file-based data source tables where the statistics are computed directly on " +
       "the files of data.")
+    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("10MB")
 
@@ -338,6 +359,7 @@ object SQLConf {
     .doc("Minimal increase rate in number of partitions between attempts when executing a take " +
       "on a query. Higher values lead to more partitions read. Lower values might lead to " +
       "longer execution times as more jobs will be run")
+    .version("2.1.1")
     .intConf
     .createWithDefault(4)
 
@@ -345,6 +367,7 @@ object SQLConf {
     buildConf("spark.sql.hive.advancedPartitionPredicatePushdown.enabled")
       .internal()
       .doc("When true, advanced partition predicate pushdown into Hive metastore is enabled.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -352,6 +375,7 @@ object SQLConf {
     .doc("The default number of partitions to use when shuffling data for joins or aggregations. " +
       "Note: For structured streaming, this configuration cannot be changed between query " +
       "restarts from the same checkpoint location.")
+    .version("1.1.0")
     .intConf
     .checkValue(_ > 0, "The value of spark.sql.shuffle.partitions must be positive")
     .createWithDefault(200)
@@ -360,12 +384,14 @@ object SQLConf {
     buildConf("spark.sql.adaptive.shuffle.targetPostShuffleInputSize")
       .internal()
       .doc("(Deprecated since Spark 3.0)")
+      .version("1.6.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("64MB")
 
   val ADAPTIVE_EXECUTION_ENABLED = buildConf("spark.sql.adaptive.enabled")
     .doc("When true, enable adaptive query execution, which re-optimizes the query plan in the " +
       "middle of query execution, based on accurate runtime statistics.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -375,6 +401,7 @@ object SQLConf {
       "sub-queries. By setting this config to true (together with " +
       s"'${ADAPTIVE_EXECUTION_ENABLED.key}' set to true), Spark will force apply adaptive query " +
       "execution for all supported queries.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -382,6 +409,7 @@ object SQLConf {
     .internal()
     .doc("Configures the log level for adaptive execution logging of plan changes. The value " +
       "can be 'trace', 'debug', 'info', 'warn', or 'error'. The default log level is 'debug'.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(Set("TRACE", "DEBUG", "INFO", "WARN", "ERROR"))
@@ -392,6 +420,7 @@ object SQLConf {
       .doc("The advisory size in bytes of the shuffle partition during adaptive optimization " +
         s"(when ${ADAPTIVE_EXECUTION_ENABLED.key} is true). It takes effect when Spark " +
         "coalesces small shuffle partitions or splits skewed shuffle partition.")
+      .version("3.0.0")
       .fallbackConf(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE)
 
   val COALESCE_PARTITIONS_ENABLED =
@@ -399,6 +428,7 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark will coalesce " +
         "contiguous shuffle partitions according to the target size (specified by " +
         s"'${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'), to avoid too many small tasks.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -408,6 +438,7 @@ object SQLConf {
         "value is the default parallelism of the Spark cluster. This configuration only " +
         s"has an effect when '${ADAPTIVE_EXECUTION_ENABLED.key}' and " +
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The minimum number of partitions must be positive.")
       .createOptional
@@ -432,6 +463,7 @@ object SQLConf {
         s"'${COALESCE_PARTITIONS_ENABLED.key}' are both true. This feature also depends " +
         "on a relocatable serializer, the concatenation support codec in use and the new version " +
         "shuffle fetch protocol.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -440,6 +472,7 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark tries to use local " +
         "shuffle reader to read the shuffle data when the shuffle partitioning is not needed, " +
         "for example, after converting sort-merge join to broadcast-hash join.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -448,6 +481,7 @@ object SQLConf {
       .doc(s"When true and '${ADAPTIVE_EXECUTION_ENABLED.key}' is true, Spark dynamically " +
         "handles skew in sort-merge join by splitting (and replicating if needed) skewed " +
         "partitions.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -456,6 +490,7 @@ object SQLConf {
       .doc("A partition is considered as skewed if its size is larger than this factor " +
         "multiplying the median partition size and also larger than " +
         "'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes'")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The skew factor must be positive.")
       .createWithDefault(10)
@@ -466,6 +501,7 @@ object SQLConf {
         s"threshold and also larger than '${SKEW_JOIN_SKEWED_PARTITION_FACTOR.key}' " +
         "multiplying the median partition size. Ideally this config should be set larger " +
         s"than '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}'.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("256MB")
 
@@ -476,6 +512,7 @@ object SQLConf {
         "considered as the build side of a broadcast-hash join in adaptive execution regardless " +
         "of its size.This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' is true.")
+      .version("3.0.0")
       .doubleConf
       .checkValue(_ >= 0, "The non-empty partition ratio must be positive number.")
       .createWithDefault(0.2)
@@ -484,6 +521,7 @@ object SQLConf {
     buildConf("spark.sql.subexpressionElimination.enabled")
       .internal()
       .doc("When true, common subexpressions will be eliminated.")
+      .version("1.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -491,6 +529,7 @@ object SQLConf {
     .internal()
     .doc("Whether the query analyzer should be case sensitive or not. " +
       "Default to case insensitive. It is highly discouraged to turn on case sensitive mode.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -500,6 +539,7 @@ object SQLConf {
       "plan to optimize them. Constraint propagation can sometimes be computationally expensive " +
       "for certain kinds of query plans (such as those with a large number of predicates and " +
       "aliases) which might negatively impact overall runtime.")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -508,6 +548,7 @@ object SQLConf {
     .doc("When true, string literals (including regex patterns) remain escaped in our SQL " +
       "parser. The default is false since Spark 2.0. Setting it to true can restore the behavior " +
       "prior to Spark 2.0.")
+    .version("2.2.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -516,6 +557,7 @@ object SQLConf {
     .doc("When estimating the output data size of a table scan, multiply the file size with this " +
       "factor as the estimated data size, in case the data is compressed in the file and lead to" +
       " a heavily underestimated result.")
+    .version("2.3.1")
     .doubleConf
     .checkValue(_ > 0, "the value of fileDataSizeFactor must be greater than 0")
     .createWithDefault(1.0)
@@ -524,6 +566,7 @@ object SQLConf {
     .doc("When true, the Parquet data source merges schemas collected from all data files, " +
          "otherwise the schema is picked from the summary file or a random data file " +
          "if no summary file is available.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -532,6 +575,7 @@ object SQLConf {
          "summary files and we will ignore them when merging schema. Otherwise, if this is " +
          "false, which is the default, we will merge all part-files. This should be considered " +
          "as expert-only option, and shouldn't be enabled before knowing what it means exactly.")
+    .version("1.5.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -540,6 +584,7 @@ object SQLConf {
       "Spark SQL, do not differentiate between binary data and strings when writing out the " +
       "Parquet schema. This flag tells Spark SQL to interpret binary data as a string to provide " +
       "compatibility with these systems.")
+    .version("1.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -548,6 +593,7 @@ object SQLConf {
       "Spark would also store Timestamp as INT96 because we need to avoid precision lost of the " +
       "nanoseconds field. This flag tells Spark SQL to interpret INT96 data as a timestamp to " +
       "provide compatibility with these systems.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -555,6 +601,7 @@ object SQLConf {
     .doc("This controls whether timestamp adjustments should be applied to INT96 data when " +
       "converting to timestamps, for data written by Impala.  This is necessary because Impala " +
       "stores INT96 data with a different timezone offset than Hive & Spark.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -568,6 +615,7 @@ object SQLConf {
       "is a standard timestamp type in Parquet, which stores number of microseconds from the " +
       "Unix epoch. TIMESTAMP_MILLIS is also standard, but with millisecond precision, which " +
       "means Spark has to truncate the microsecond portion of its timestamp value.")
+    .version("2.3.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(ParquetOutputTimestampType.values.map(_.toString))
@@ -579,6 +627,7 @@ object SQLConf {
       "precedence would be `compression`, `parquet.compression`, " +
       "`spark.sql.parquet.compression.codec`. Acceptable values include: none, uncompressed, " +
       "snappy, gzip, lzo, brotli, lz4, zstd.")
+    .version("1.1.1")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "gzip", "lzo", "lz4", "brotli", "zstd"))
@@ -586,6 +635,7 @@ object SQLConf {
 
   val PARQUET_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.parquet.filterPushdown")
     .doc("Enables Parquet filter push-down optimization when set to true.")
+    .version("1.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -593,6 +643,7 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for Date. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
+    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -602,15 +653,17 @@ object SQLConf {
       .doc("If true, enables Parquet filter push-down optimization for Timestamp. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled and Timestamp stored as TIMESTAMP_MICROS or TIMESTAMP_MILLIS type.")
-    .internal()
-    .booleanConf
-    .createWithDefault(true)
+      .version("2.4.0")
+      .internal()
+      .booleanConf
+      .createWithDefault(true)
 
   val PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED =
     buildConf("spark.sql.parquet.filterPushdown.decimal")
       .doc("If true, enables Parquet filter push-down optimization for Decimal. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
+      .version("2.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -620,6 +673,7 @@ object SQLConf {
     .doc("If true, enables Parquet filter push-down optimization for string startsWith function. " +
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
       "enabled.")
+    .version("2.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -632,6 +686,7 @@ object SQLConf {
         "By setting this value to 0 this feature can be disabled. " +
         s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' is " +
         "enabled.")
+      .version("2.4.0")
       .internal()
       .intConf
       .checkValue(threshold => threshold >= 0, "The threshold must not be negative.")
@@ -643,6 +698,7 @@ object SQLConf {
       "systems such as Apache Hive and Apache Impala use. If false, the newer format in Parquet " +
       "will be used. For example, decimals will be written in int-based format. If Parquet " +
       "output is intended for use with systems that do not support this newer format, set to true.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -652,6 +708,7 @@ object SQLConf {
       "of org.apache.parquet.hadoop.ParquetOutputCommitter. If it is not, then metadata " +
       "summaries will never be created, irrespective of the value of " +
       "parquet.summary.metadata.level")
+    .version("1.5.0")
     .internal()
     .stringConf
     .createWithDefault("org.apache.parquet.hadoop.ParquetOutputCommitter")
@@ -659,6 +716,7 @@ object SQLConf {
   val PARQUET_VECTORIZED_READER_ENABLED =
     buildConf("spark.sql.parquet.enableVectorizedReader")
       .doc("Enables vectorized parquet decoding.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -668,12 +726,14 @@ object SQLConf {
       s"This configuration only has an effect when '${PARQUET_FILTER_PUSHDOWN_ENABLED.key}' " +
       "is enabled and the vectorized reader is not used. You can ensure the vectorized reader " +
       s"is not used by setting '${PARQUET_VECTORIZED_READER_ENABLED.key}' to false.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val PARQUET_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.parquet.columnarReaderBatchSize")
     .doc("The number of rows to include in a parquet vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 

From 0e9dca81d6187f9e51ed3eb03e2d954568779428 Mon Sep 17 00:00:00 2001
From: Jiaan Geng <beliefer@163.com>
Date: Mon, 2 Mar 2020 15:13:58 +0900
Subject: [PATCH 0443/1280] [SPARK-30841][SQL][DOC][FOLLOW-UP] Add version
 information to the configuration of SQL

This PR follows https://github.com/apache/spark/pull/27691
I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.orc.compression.codec | 2.3.0 | SPARK-21839 | d8f45408635d4fccac557cb1e877dfe9267fb326#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.orc.impl | 2.3.0 | SPARK-20728 | 326f1d6728a7734c228d8bfaa69442a1c7b92e9b#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.orc.enableVectorizedReader | 2.3.0 | SPARK-16060 | 60f6b994505e3f82091a04eed2dc0a9e8bd523ce#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.orc.columnarReaderBatchSize | 2.4.0 | SPARK-23188 | cc41245fa3f954f961541bf4b4275c28473042b8#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.orc.filterPushdown | 1.4.0 | SPARK-2883 | 65d71bd9fbfe6fe1b741c80fed72d6ae3d22b028#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.orc.mergeSchema | 3.0.0 | SPARK-11412 | 73183b3c8c2022846587f08e8dea5c387ed3b8d5#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.hive.verifyPartitionPath | 1.4.0 | Spark-5068 | 1f39a61118184e136f38381a9f3ba0b2d5d589d9#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.hive.metastorePartitionPruning | 1.5.0 | SPARK-9386 | ce89ff477aea6def68265ed218f6105680755c9a#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.hive.manageFilesourcePartitions | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.hive.filesourcePartitionFileCacheSize | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.hive.caseSensitiveInferenceMode | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.optimizer.metadataOnly | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.columnNameOfCorruptRecord | 1.2.0 | SPARK-3339 | 1c7f0ab302de9f82b1bd6da852d133823bc67c66#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.broadcastTimeout | 1.3.0 | SPARK-4269 | fa66ef6c97e87c9255b67b03836a4ba50598ebae#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.thriftserver.scheduler.pool | 1.1.1 | SPARK-3025 | 496f62d9a98067256d8a51fd1e7a485ff6492fa8#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.thriftServer.incrementalCollect | 2.0.3 | SPARK-18857 | c94288b57b5ce2232e58e35cada558d8d5b8ec6e#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.thriftserver.ui.retainedStatements | 1.4.0 | SPARK-5100 | 343d3bfafd449a0371feb6a88f78e07302fa7143#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.thriftserver.ui.retainedSessions | 1.4.0 | SPARK-5100 | 343d3bfafd449a0371feb6a88f78e07302fa7143#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.sources.default | 1.3.0 | SPARK-5658 | a21090ebe1ef7a709709300712de7d928a923244#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.hive.convertCTAS | 2.0.0 | SPARK-15646 | 5a835b99f9852b0c2a35f9c75a51d493474994ea#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.hive.gatherFastStats | 2.0.1 | SPARK-17063 | 3d283f6c9d9daef53fa4e90b0ead2a94710a37a7#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.sources.partitionColumnTypeInference.enabled | 1.5.0 | SPARK-7939 | 03ef6be9ce61a13dcd9d8c71298fb4be39119411#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.sources.bucketing.enabled | 2.0.0 | SPARK-13486 | 2b2c8c33236677c916541f956f7b94bba014a9ce#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.sources.bucketing.maxBuckets | 2.4.0 | SPARK-23997 | de46df549acee7fda56bb0871f444d2f3b49e582#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.crossJoin.enabled | 2.0.0 | SPARK-15425 | 4462da7071462084c5b55cc414c7faa0e1396a18#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.orderByOrdinal | 2.0.0 | SPARK-12789 | 2c5b18fb0fdeabd378dd97e91f72d1eac4e21cc7#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.groupByOrdinal | 2.0.0 | SPARK-13957 | 05f652d6c2bbd764a1dd5a45301811e14519486f#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.groupByAliases | 2.2.0 | SPARK-14471 | af3a1411a28796d4d9a100eefb093b1d91532754#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.sources.outputCommitterClass | 1.4.0 | SPARK-7567 | a385f4b8dd22e0e056569cffc4fa63047cb7c8f2#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.sources.commitProtocolClass | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.sources.parallelPartitionDiscovery.threshold | 1.5.0 | SPARK-8125 | a1064df0ee3daf496800be84293345a10e1497d9#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.sources.parallelPartitionDiscovery.parallelism | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.sources.ignoreDataLocality | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.selfJoinAutoResolveAmbiguity | 1.4.0 | SPARK-6231 | e61083ccab7764d1929248490a3d2e83987241e0#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.analyzer.failAmbiguousSelfJoin | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.retainGroupColumns | 1.4.0 | SPARK-7462 | 9c35f02b35fda80d6558573466735e79b3dd9124#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.pivotMaxValues | 1.6.0 | SPARK-8992 | 5940fc71d2a245cc6e50edb455c3dd3dbb8de43a#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.runSQLOnFiles | 1.6.0 | SPARK-11197 | f8c6bec65784de89b47e96a367d3f9790c1b3115#diff-41ef65b9ef5b518f77e2a03559893f4d
spark.sql.codegen.wholeStage | 2.0.0 | SPARK-13486 | 2b2c8c33236677c916541f956f7b94bba014a9ce#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.codegen.useIdInClassName | 2.3.1 | SPARK-23032 | 26a8b4e398ee6d1de06a5f3ac1d6d342c9b67d78#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.maxFields | 2.0.0 | SPARK-14224 and SPARK-14223 and SPARK-14310 | 5a4b11a901703464b9261dea0642d80cf8d4856c#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.codegen.factoryMode | 2.4.0 | SPARK-23711 | a40ffc656d62372da85e0fa932b67207839e7fde#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.fallback | 2.0.0 | SPARK-15759 | f0fa0a8946fb4bdf0f4697a8e389f49e98422871#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.codegen.logging.maxLines | 2.3.0 | SPARK-20871 | 2a53fbfce72b3faef020e39a1e8628d68bc95beb#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.hugeMethodLimit | 2.3.0 | SPARK-21871 | 4a779bdac3e75c17b7d36c5a009ba6c948fa9fb6#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.methodSplitThreshold | 3.0.0 | SPARK-25850 | e017cb39642a5039abd8ce8127ad41712901bdbc#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.splitConsumeFuncByOperator | 2.3.1 | SPARK-21717 | c79e771f8952e6773c3a84cc617145216feddbcf#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.files.maxPartitionBytes | 2.0.0 | SPARK-13664 | 17eec0a71ba8713c559d641e3f43a1be726b037c#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.files.openCostInBytes | 2.0.0 | SPARK-14259 | 400b2f863ffaa01a34a8dae1541c61526fef908b#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.files.ignoreCorruptFiles | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.files.ignoreMissingFiles | 2.3.0 | SPARK-22366 | 8e9863531bebbd4d83eafcbc2b359b8bd0ac5734#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.files.maxRecordsPerFile | 2.2.0 | SPARK-19944 | 0ee38a39e43dd7ad9d50457e446ae36f64621a1b#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.exchange.reuse | 2.0.0 | SPARK-13523 | 3dc9ae2e158e5b51df6f799767946fe1d190156b#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.execution.reuseSubquery | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.stateStore.providerClass | 2.3.0 | SPARK-20883 and SPARK-20376 | fa757ee1d41396ad8734a3f2dd045bb09bc82a2e#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.stateStore.minDeltasForSnapshot | 2.0.0 | SPARK-13809 | 8c826880f5eaa3221c4e9e7d3fece54e821a0b98#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion | 2.4.0 | SPARK-22187 | b3d88ac02940eff4c867d3acb79fe5ff9d724e83#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.checkpointLocation | 2.0.0 | SPARK-13985 | caea15214571d9b12dcf1553e5c1cc8b83a8ba5b#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.streaming.forceDeleteTempCheckpointLocation | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.minBatchesToRetain | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.maxBatchesToRetainInMemory | 2.4.0 | SPARK-24717 | 8b7d4f842fdc90b8d1c37080bdd9b5e1d070f5c0#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.aggregation.stateFormatVersion | 2.4.0 | SPARK-24763 | 6c5cb85856235efd464b109558896f81ae2c4c75#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.stopActiveRunOnRestart | 3.0.0 | SPARK-29568 | 363af16c72abe19fc5cc5b5bdf9d8dc34975f2ba#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.join.stateFormatVersion | 3.0.0 | SPARK-26154 | c941362cb94b24bdf48d4928a1a4dff1b13a1484#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.unsupportedOperationCheck | 2.0.0 | SPARK-14473 | 775cf17eaaae1a38efe47b282b1d6bbdb99bd759#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.variable.substitute | 2.0.0 | SPARK-14769 | 334c293ec0bcc2195d502c574ca40dbc4769d666#diff-32bb9518401c0948c5ea19377b5069ab
spark.sql.codegen.aggregate.map.twolevel.enabled | 2.3.0 | SPARK-22159 | d29d1e87995e02cb57ba3026c945c3cd66bb06e2#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.aggregate.map.vectorized.enable | 3.0.0 | SPARK-28257 | 42b80ae128ab1aa8a87c1376fe88e2cde52e6e4f#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.codegen.aggregate.splitAggregateFunc.enabled | 3.0.0 | SPARK-21870 | cb0cddffe9452937033e0e6b1fc0e600d2c787ad#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.view.maxNestedViewDepth | 2.2.0 | SPARK-19877 | ee36bc1c9043ead3c3ba4fba7e68c6c47ad7ae7a#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.commitProtocolClass | 2.1.0 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13
spark.sql.streaming.multipleWatermarkPolicy | 2.4.0 | SPARK-24730 | 6078b891da8fe7fc36579699473168ae7443284c#diff-9a6b543db706f1a90f790783d6930a13

Supplemental configuration version information.

No

Exists UT

Closes #27730 from beliefer/add-version-to-sql-config-part-two.

Lead-authored-by: Jiaan Geng <beliefer@163.com>
Co-authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 210 +++++++++++++++++-
 1 file changed, 200 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 265eb629a28d3..9358dc7ef6ee3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -742,6 +742,7 @@ object SQLConf {
       "`orc.compress` is specified in the table-specific options/properties, the precedence " +
       "would be `compression`, `orc.compress`, `spark.sql.orc.compression.codec`." +
       "Acceptable values include: none, uncompressed, snappy, zlib, lzo.")
+    .version("2.3.0")
     .stringConf
     .transform(_.toLowerCase(Locale.ROOT))
     .checkValues(Set("none", "uncompressed", "snappy", "zlib", "lzo"))
@@ -750,6 +751,7 @@ object SQLConf {
   val ORC_IMPLEMENTATION = buildConf("spark.sql.orc.impl")
     .doc("When native, use the native version of ORC support instead of the ORC library in Hive. " +
       "It is 'hive' by default prior to Spark 2.4.")
+    .version("2.3.0")
     .internal()
     .stringConf
     .checkValues(Set("hive", "native"))
@@ -757,23 +759,27 @@ object SQLConf {
 
   val ORC_VECTORIZED_READER_ENABLED = buildConf("spark.sql.orc.enableVectorizedReader")
     .doc("Enables vectorized orc decoding.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_VECTORIZED_READER_BATCH_SIZE = buildConf("spark.sql.orc.columnarReaderBatchSize")
     .doc("The number of rows to include in a orc vectorized reader batch. The number should " +
       "be carefully chosen to minimize overhead and avoid OOMs in reading data.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(4096)
 
   val ORC_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.orc.filterPushdown")
     .doc("When true, enable filter pushdown for ORC files.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORC_SCHEMA_MERGING_ENABLED = buildConf("spark.sql.orc.mergeSchema")
     .doc("When true, the Orc data source merges schemas collected from all data files, " +
       "otherwise the schema is picked from a random data file.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -781,6 +787,7 @@ object SQLConf {
     .doc("When true, check all the partition paths under the table\'s root directory " +
          "when reading data stored in HDFS. This configuration will be deprecated in the future " +
          s"releases and replaced by ${SPARK_IGNORE_MISSING_FILES.key}.")
+    .version("1.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -790,6 +797,7 @@ object SQLConf {
            "unmatching partitions can be eliminated earlier. This only affects Hive tables " +
            "not converted to filesource relations (see HiveUtils.CONVERT_METASTORE_PARQUET and " +
            "HiveUtils.CONVERT_METASTORE_ORC for more information).")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -799,6 +807,7 @@ object SQLConf {
            "This includes both datasource and converted Hive tables. When partition management " +
            "is enabled, datasource tables store partition in the Hive metastore, and use the " +
            "metastore to prune partitions during query planning.")
+      .version("2.1.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -807,6 +816,7 @@ object SQLConf {
       .doc("When nonzero, enable caching of partition file metadata in memory. All tables share " +
            "a cache that can use up to specified num bytes for file metadata. This conf only " +
            "has an effect when hive filesource partition management is enabled.")
+      .version("2.1.1")
       .longConf
       .createWithDefault(250 * 1024 * 1024)
 
@@ -822,6 +832,7 @@ object SQLConf {
       "and write it back to the table properties), INFER_ONLY (infer the schema but don't " +
       "attempt to write it to the table properties) and NEVER_INFER (the default mode-- fallback " +
       "to using the case-insensitive metastore schema instead of inferring).")
+    .version("2.1.1")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(HiveCaseSensitiveInferenceMode.values.map(_.toString))
@@ -834,23 +845,27 @@ object SQLConf {
       "scanned are partition columns and the query has an aggregate operator that satisfies " +
       "distinct semantics. By default the optimization is disabled, since it may return " +
       "incorrect results when the files are empty.")
+    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
   val COLUMN_NAME_OF_CORRUPT_RECORD = buildConf("spark.sql.columnNameOfCorruptRecord")
     .doc("The name of internal column for storing raw/un-parsed JSON and CSV records that fail " +
       "to parse.")
+    .version("1.2.0")
     .stringConf
     .createWithDefault("_corrupt_record")
 
   val BROADCAST_TIMEOUT = buildConf("spark.sql.broadcastTimeout")
     .doc("Timeout in seconds for the broadcast wait time in broadcast joins.")
+    .version("1.3.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString(s"${5 * 60}")
 
   // This is only used for the thriftserver
   val THRIFTSERVER_POOL = buildConf("spark.sql.thriftserver.scheduler.pool")
     .doc("Set a Fair Scheduler pool for a JDBC client session.")
+    .version("1.1.1")
     .stringConf
     .createOptional
 
@@ -858,23 +873,27 @@ object SQLConf {
     buildConf("spark.sql.thriftServer.incrementalCollect")
       .internal()
       .doc("When true, enable incremental collection for execution in Thrift Server.")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   val THRIFTSERVER_UI_STATEMENT_LIMIT =
     buildConf("spark.sql.thriftserver.ui.retainedStatements")
       .doc("The number of SQL statements kept in the JDBC/ODBC web UI history.")
+      .version("1.4.0")
       .intConf
       .createWithDefault(200)
 
   val THRIFTSERVER_UI_SESSION_LIMIT = buildConf("spark.sql.thriftserver.ui.retainedSessions")
     .doc("The number of SQL client sessions kept in the JDBC/ODBC web UI history.")
+    .version("1.4.0")
     .intConf
     .createWithDefault(200)
 
   // This is used to set the default data source
   val DEFAULT_DATA_SOURCE_NAME = buildConf("spark.sql.sources.default")
     .doc("The default data source to use in input/output.")
+    .version("1.3.0")
     .stringConf
     .createWithDefault("parquet")
 
@@ -883,6 +902,7 @@ object SQLConf {
     .doc("When true, a table created by a Hive CTAS statement (no USING clause) " +
       "without specifying any storage property will be converted to a data source table, " +
       s"using the data source set by ${DEFAULT_DATA_SOURCE_NAME.key}.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -891,22 +911,26 @@ object SQLConf {
       .doc("When true, fast stats (number of files and total size of all files) will be gathered" +
         " in parallel while repairing table partitions to avoid the sequential listing in Hive" +
         " metastore.")
+      .version("2.0.1")
       .booleanConf
       .createWithDefault(true)
 
   val PARTITION_COLUMN_TYPE_INFERENCE =
     buildConf("spark.sql.sources.partitionColumnTypeInference.enabled")
       .doc("When true, automatically infer the data types for partitioned columns.")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   val BUCKETING_ENABLED = buildConf("spark.sql.sources.bucketing.enabled")
     .doc("When false, we will treat bucketed table as normal table")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val BUCKETING_MAX_BUCKETS = buildConf("spark.sql.sources.bucketing.maxBuckets")
     .doc("The maximum number of buckets allowed.")
+    .version("2.4.0")
     .intConf
     .checkValue(_ > 0, "the value of spark.sql.sources.bucketing.maxBuckets must be greater than 0")
     .createWithDefault(100000)
@@ -915,36 +939,42 @@ object SQLConf {
     .internal()
     .doc("When false, we will throw an error if a query contains a cartesian product without " +
         "explicit CROSS JOIN syntax.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val ORDER_BY_ORDINAL = buildConf("spark.sql.orderByOrdinal")
     .doc("When true, the ordinal numbers are treated as the position in the select list. " +
          "When false, the ordinal numbers in order/sort by clause are ignored.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ORDINAL = buildConf("spark.sql.groupByOrdinal")
     .doc("When true, the ordinal numbers in group by clauses are treated as the position " +
       "in the select list. When false, the ordinal numbers are ignored.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val GROUP_BY_ALIASES = buildConf("spark.sql.groupByAliases")
     .doc("When true, aliases in a select list can be used in group by clauses. When false, " +
       "an analysis exception is thrown in the case.")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
   // The output committer class used by data sources. The specified class needs to be a
   // subclass of org.apache.hadoop.mapreduce.OutputCommitter.
   val OUTPUT_COMMITTER_CLASS = buildConf("spark.sql.sources.outputCommitterClass")
+    .version("1.4.0")
     .internal()
     .stringConf
     .createOptional
 
   val FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.sources.commitProtocolClass")
+      .version("2.1.1")
       .internal()
       .stringConf
       .createWithDefault(
@@ -956,6 +986,7 @@ object SQLConf {
         "of detected paths exceeds this value during partition discovery, it tries to list the " +
         "files with another Spark distributed job. This configuration is effective only when " +
         "using file-based sources such as Parquet, JSON and ORC.")
+      .version("1.5.0")
       .intConf
       .checkValue(parallel => parallel >= 0, "The maximum number of paths allowed for listing " +
         "files at driver side must not be negative")
@@ -965,6 +996,7 @@ object SQLConf {
     buildConf("spark.sql.sources.parallelPartitionDiscovery.parallelism")
       .doc("The number of parallelism to list a collection of path recursively, Set the " +
         "number to prevent file listing from generating too many tasks.")
+      .version("2.1.1")
       .internal()
       .intConf
       .createWithDefault(10000)
@@ -976,6 +1008,7 @@ object SQLConf {
         "schedule tasks to take advantage of data locality. It can be particularly " +
         "useful if data is read from a remote cluster so the scheduler could never " +
         "take advantage of locality anyway.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(false)
@@ -984,6 +1017,7 @@ object SQLConf {
   // See SPARK-6231.
   val DATAFRAME_SELF_JOIN_AUTO_RESOLVE_AMBIGUITY =
     buildConf("spark.sql.selfJoinAutoResolveAmbiguity")
+      .version("1.4.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -991,12 +1025,14 @@ object SQLConf {
   val FAIL_AMBIGUOUS_SELF_JOIN_ENABLED =
     buildConf("spark.sql.analyzer.failAmbiguousSelfJoin")
       .doc("When true, fail the Dataset query if it contains ambiguous self-join.")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
 
   // Whether to retain group by columns or not in GroupedData.agg.
   val DATAFRAME_RETAIN_GROUP_COLUMNS = buildConf("spark.sql.retainGroupColumns")
+    .version("1.4.0")
     .internal()
     .booleanConf
     .createWithDefault(true)
@@ -1004,12 +1040,14 @@ object SQLConf {
   val DATAFRAME_PIVOT_MAX_VALUES = buildConf("spark.sql.pivotMaxValues")
     .doc("When doing a pivot without specifying values for the pivot column this is the maximum " +
       "number of (distinct) values that will be collected without error.")
+    .version("1.6.0")
     .intConf
     .createWithDefault(10000)
 
   val RUN_SQL_ON_FILES = buildConf("spark.sql.runSQLOnFiles")
     .internal()
     .doc("When true, we could use `datasource`.`path` as table in SQL query.")
+    .version("1.6.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1017,6 +1055,7 @@ object SQLConf {
     .internal()
     .doc("When true, the whole stage (of multiple operators) will be compiled into single java" +
       " method.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1025,6 +1064,7 @@ object SQLConf {
     .internal()
     .doc("When true, embed the (whole-stage) codegen stage ID into " +
       "the class name of the generated class as a suffix")
+    .version("2.3.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -1032,6 +1072,7 @@ object SQLConf {
     .internal()
     .doc("The maximum number of fields (including nested fields) that will be supported before" +
       " deactivating whole-stage codegen.")
+    .version("2.0.0")
     .intConf
     .createWithDefault(100)
 
@@ -1041,6 +1082,7 @@ object SQLConf {
       "interpreted if any compile error happens. Disabling fallback if `CODEGEN_ONLY`. " +
       "`NO_CODEGEN` skips codegen and goes interpreted path always. Note that " +
       "this config works only for tests.")
+    .version("2.4.0")
     .internal()
     .stringConf
     .checkValues(CodegenObjectFactoryMode.values.map(_.toString))
@@ -1050,12 +1092,14 @@ object SQLConf {
     .internal()
     .doc("When true, (whole stage) codegen could be temporary disabled for the part of query that" +
       " fail to compile generated code")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val CODEGEN_LOGGING_MAX_LINES = buildConf("spark.sql.codegen.logging.maxLines")
     .internal()
     .doc("The maximum number of codegen lines to log when errors occur. Use -1 for unlimited.")
+    .version("2.3.0")
     .intConf
     .checkValue(maxLines => maxLines >= -1, "The maximum must be a positive integer, 0 to " +
       "disable logging or -1 to apply no limit.")
@@ -1069,6 +1113,7 @@ object SQLConf {
       "is the largest bytecode size possible for a valid Java method. When running on HotSpot, " +
       s"it may be preferable to set the value to ${CodeGenerator.DEFAULT_JVM_HUGE_METHOD_LIMIT} " +
       "to match HotSpot's implementation.")
+    .version("2.3.0")
     .intConf
     .createWithDefault(65535)
 
@@ -1080,6 +1125,7 @@ object SQLConf {
       "generated, so use the code length as metric. When running on HotSpot, a function's " +
       "bytecode should not go beyond 8KB, otherwise it will not be JITted; it also should not " +
       "be too small, otherwise there will be many function calls.")
+    .version("3.0.0")
     .intConf
     .checkValue(threshold => threshold > 0, "The threshold must be a positive integer.")
     .createWithDefault(1024)
@@ -1090,6 +1136,7 @@ object SQLConf {
       .doc("When true, whole stage codegen would put the logic of consuming rows of each " +
         "physical operator into individual methods, instead of a single big method. This can be " +
         "used to avoid oversized function that can miss the opportunity of JIT optimization.")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1097,6 +1144,7 @@ object SQLConf {
     .doc("The maximum number of bytes to pack into a single partition when reading files. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.0.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("128MB") // parquet.block.size
 
@@ -1107,6 +1155,7 @@ object SQLConf {
       " over estimated, then the partitions with small files will be faster than partitions with" +
       " bigger files (which is scheduled first). This configuration is effective only when using" +
       " file-based sources such as Parquet, JSON and ORC.")
+    .version("2.0.0")
     .longConf
     .createWithDefault(4 * 1024 * 1024)
 
@@ -1115,6 +1164,7 @@ object SQLConf {
       "encountering corrupted files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
 
@@ -1123,24 +1173,28 @@ object SQLConf {
       "encountering missing files and the contents that have been read will still be returned. " +
       "This configuration is effective only when using file-based sources such as Parquet, JSON " +
       "and ORC.")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val MAX_RECORDS_PER_FILE = buildConf("spark.sql.files.maxRecordsPerFile")
     .doc("Maximum number of records to write out to a single file. " +
       "If this value is zero or negative, there is no limit.")
+    .version("2.2.0")
     .longConf
     .createWithDefault(0)
 
   val EXCHANGE_REUSE_ENABLED = buildConf("spark.sql.exchange.reuse")
     .internal()
     .doc("When true, the planner will try to find out duplicated exchanges and re-use them.")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
   val SUBQUERY_REUSE_ENABLED = buildConf("spark.sql.execution.reuseSubquery")
     .internal()
     .doc("When true, the planner will try to find out duplicated subqueries and re-use them.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1152,6 +1206,7 @@ object SQLConf {
           "be a subclass of StateStoreProvider, and must have a zero-arg constructor. " +
           "Note: For structured streaming, this configuration cannot be changed between query " +
           "restarts from the same checkpoint location.")
+      .version("2.3.0")
       .stringConf
       .createWithDefault(
         "org.apache.spark.sql.execution.streaming.state.HDFSBackedStateStoreProvider")
@@ -1161,6 +1216,7 @@ object SQLConf {
       .internal()
       .doc("Minimum number of state store delta files that needs to be generated before they " +
         "consolidated into snapshots.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1168,24 +1224,28 @@ object SQLConf {
     buildConf("spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion")
       .internal()
       .doc("State format version used by flatMapGroupsWithState operation in a streaming query")
+      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
 
   val CHECKPOINT_LOCATION = buildConf("spark.sql.streaming.checkpointLocation")
     .doc("The default location for storing checkpoint data for streaming queries.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   val FORCE_DELETE_TEMP_CHECKPOINT_LOCATION =
     buildConf("spark.sql.streaming.forceDeleteTempCheckpointLocation")
       .doc("When true, enable temporary checkpoint locations force delete.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val MIN_BATCHES_TO_RETAIN = buildConf("spark.sql.streaming.minBatchesToRetain")
     .internal()
     .doc("The minimum number of batches that must be retained and made recoverable.")
+    .version("2.1.1")
     .intConf
     .createWithDefault(100)
 
@@ -1195,6 +1255,7 @@ object SQLConf {
       "loading from files. The value adjusts a trade-off between memory usage vs cache miss: " +
       "'2' covers both success and direct failure cases, '1' covers only success case, " +
       "and '0' covers extreme case - disable cache to maximize memory size of executors.")
+    .version("2.4.0")
     .intConf
     .createWithDefault(2)
 
@@ -1204,6 +1265,7 @@ object SQLConf {
       .doc("State format version used by streaming aggregation operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
+      .version("2.4.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1214,6 +1276,7 @@ object SQLConf {
       "If we find a concurrent active run for a streaming query (in the same or different " +
       "SparkSessions on the same cluster) and this flag is true, we will stop the old streaming " +
       "query run to start the new one.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1223,6 +1286,7 @@ object SQLConf {
       .doc("State format version used by streaming join operations in a streaming query. " +
         "State between versions are tend to be incompatible, so state format version shouldn't " +
         "be modified after running.")
+      .version("3.0.0")
       .intConf
       .checkValue(v => Set(1, 2).contains(v), "Valid versions are 1 and 2")
       .createWithDefault(2)
@@ -1232,6 +1296,7 @@ object SQLConf {
       .internal()
       .doc("When true, the logical plan for streaming query will be checked for unsupported" +
         " operations.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1239,6 +1304,7 @@ object SQLConf {
     buildConf("spark.sql.variable.substitute")
       .doc("This enables substitution using syntax like `${var}`, `${system:var}`, " +
         "and `${env:var}`.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1249,6 +1315,7 @@ object SQLConf {
         "inserted/looked-up at a 1st-level, small, fast map, and then fallback to a " +
         "2nd-level, larger, slower map when 1st level is full or keys cannot be found. " +
         "When disabled, records go directly to the 2nd level.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1256,6 +1323,7 @@ object SQLConf {
     buildConf("spark.sql.codegen.aggregate.map.vectorized.enable")
       .internal()
       .doc("Enable vectorized aggregate hash map. This is for testing/benchmarking only.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1265,6 +1333,7 @@ object SQLConf {
       .doc("When true, the code generator would split aggregate code into individual methods " +
         "instead of a single big method. This can be used to avoid oversized function that " +
         "can miss the opportunity of JIT optimization.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1276,6 +1345,7 @@ object SQLConf {
         "However the DAG depth may become too large and cause unexpected behavior. This " +
         "configuration puts a limit on this: when the depth of a view exceeds this value during " +
         "analysis, we terminate the resolution to avoid potential errors.")
+      .version("2.2.0")
       .intConf
       .checkValue(depth => depth > 0, "The maximum depth of a view reference in a nested view " +
         "must be positive.")
@@ -1283,6 +1353,7 @@ object SQLConf {
 
   val STREAMING_FILE_COMMIT_PROTOCOL_CLASS =
     buildConf("spark.sql.streaming.commitProtocolClass")
+      .version("2.1.0")
       .internal()
       .stringConf
       .createWithDefault("org.apache.spark.sql.execution.streaming.ManifestFileCommitProtocol")
@@ -1295,6 +1366,7 @@ object SQLConf {
         "'max' which chooses the maximum across multiple operators. " +
         "Note: This configuration cannot be changed between query restarts from the same " +
         "checkpoint location.")
+      .version("2.4.0")
       .stringConf
       .transform(_.toLowerCase(Locale.ROOT))
       .checkValue(
@@ -1309,6 +1381,7 @@ object SQLConf {
       .doc("In the case of ObjectHashAggregateExec, when the size of the in-memory hash map " +
         "grows too large, we will fall back to sort-based aggregation. This option sets a row " +
         "count threshold for the size of the hash map.")
+      .version("2.2.0")
       .intConf
       // We are trying to be conservative and use a relatively small default count threshold here
       // since the state object of some TypedImperativeAggregate function can be quite large (e.g.
@@ -1318,6 +1391,7 @@ object SQLConf {
   val USE_OBJECT_HASH_AGG = buildConf("spark.sql.execution.useObjectHashAggregateExec")
     .internal()
     .doc("Decides if we use ObjectHashAggregateExec")
+    .version("")
     .booleanConf
     .createWithDefault(true)
 
@@ -1326,12 +1400,14 @@ object SQLConf {
       .doc("Whether to ignore null fields when generating JSON objects in JSON data source and " +
         "JSON functions such as to_json. " +
         "If false, it generates null for null fields in JSON objects.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SINK_LOG_DELETION = buildConf("spark.sql.streaming.fileSink.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream sink.")
+    .version("")
     .booleanConf
     .createWithDefault(true)
 
@@ -1340,6 +1416,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
+      .version("")
       .intConf
       .createWithDefault(10)
 
@@ -1347,12 +1424,14 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSink.log.cleanupDelay")
       .internal()
       .doc("How long that a file is guaranteed to be visible for all readers.")
+      .version("")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
   val FILE_SOURCE_LOG_DELETION = buildConf("spark.sql.streaming.fileSource.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream source.")
+    .version("")
     .booleanConf
     .createWithDefault(true)
 
@@ -1361,6 +1440,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
+      .version("")
       .intConf
       .createWithDefault(10)
 
@@ -1368,6 +1448,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSource.log.cleanupDelay")
       .internal()
       .doc("How long in milliseconds a file is guaranteed to be visible for all readers.")
+      .version("")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
@@ -1377,12 +1458,14 @@ object SQLConf {
       .doc("When true, force the schema of streaming file source to be nullable (including all " +
         "the fields). Otherwise, the schema might not be compatible with actual data, which " +
         "leads to corruptions.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SOURCE_CLEANER_NUM_THREADS =
     buildConf("spark.sql.streaming.fileSource.cleaner.numThreads")
       .doc("Number of threads used in the file source completed file cleaner.")
+      .version("")
       .intConf
       .createWithDefault(1)
 
@@ -1390,6 +1473,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.schemaInference")
       .internal()
       .doc("Whether file-based streaming sources will infer its own schema")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -1397,6 +1481,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.pollingDelay")
       .internal()
       .doc("How long to delay polling new data when no data is available")
+      .version("")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10L)
 
@@ -1404,6 +1489,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.stopTimeout")
       .doc("How long to wait in milliseconds for the streaming execution thread to stop when " +
         "calling the streaming query's stop() method. 0 or negative values wait indefinitely.")
+      .version("")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
@@ -1411,6 +1497,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.noDataProgressEventInterval")
       .internal()
       .doc("How long to wait between two progress events when there is no data")
+      .version("")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10000L)
 
@@ -1419,18 +1506,21 @@ object SQLConf {
       .doc(
         "Whether streaming micro-batch engine will execute batches without data " +
           "for eager state management for stateful streaming queries.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
   val STREAMING_METRICS_ENABLED =
     buildConf("spark.sql.streaming.metricsEnabled")
       .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val STREAMING_PROGRESS_RETENTION =
     buildConf("spark.sql.streaming.numRecentProgressUpdates")
       .doc("The number of progress updates to retain for a streaming query")
+      .version("")
       .intConf
       .createWithDefault(100)
 
@@ -1438,6 +1528,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpointFileManagerClass")
       .doc("The class used to write checkpoint files atomically. This class must be a subclass " +
         "of the interface CheckpointFileManager.")
+      .version("")
       .internal()
       .stringConf
 
@@ -1445,6 +1536,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpoint.escapedPathCheck.enabled")
       .doc("Whether to detect a streaming query may pick up an incorrect checkpoint path due " +
         "to SPARK-26824.")
+      .version("")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -1455,6 +1547,7 @@ object SQLConf {
       .doc("When true, SQL commands use parallel file listing, " +
         "as opposed to single thread listing. " +
         "This usually speeds up commands that need to list many directories.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1464,6 +1557,7 @@ object SQLConf {
       s"which is larger than `${AUTO_BROADCASTJOIN_THRESHOLD.key}` to be more conservative. " +
       "That is to say by default the optimizer will not choose to broadcast a table unless it " +
       "knows for sure its size is small enough.")
+    .version("")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(Long.MaxValue)
 
@@ -1474,6 +1568,7 @@ object SQLConf {
       "For non-partitioned data source tables, it will be automatically recalculated if table " +
       "statistics are not available. For partitioned data source and partitioned Hive tables, " +
       s"It is '${DEFAULT_SIZE_IN_BYTES.key}' if table statistics are not available.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -1482,6 +1577,7 @@ object SQLConf {
       .internal()
       .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
         "column level statistics.")
+      .version("")
       .doubleConf
       .createWithDefault(0.05)
 
@@ -1492,6 +1588,7 @@ object SQLConf {
         "histogram. Note that collecting histograms takes extra cost. For example, collecting " +
         "column statistics usually takes only one table scan, but generating equi-height " +
         "histogram will cause an extra table scan.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -1499,6 +1596,7 @@ object SQLConf {
     buildConf("spark.sql.statistics.histogram.numBins")
       .internal()
       .doc("The number of bins when generating histograms.")
+      .version("")
       .intConf
       .checkValue(num => num > 1, "The number of bins must be greater than 1.")
       .createWithDefault(254)
@@ -1509,6 +1607,7 @@ object SQLConf {
       .doc("Accuracy of percentile approximation when generating equi-height histograms. " +
         "Larger value means better accuracy. The relative error can be deduced by " +
         "1.0 / PERCENTILE_ACCURACY.")
+      .version("")
       .intConf
       .createWithDefault(10000)
 
@@ -1517,30 +1616,35 @@ object SQLConf {
       .doc("Enables automatic update for table size once table's data is changed. Note that if " +
         "the total number of files of the table is very large, this can be expensive and slow " +
         "down data change commands.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val CBO_ENABLED =
     buildConf("spark.sql.cbo.enabled")
       .doc("Enables CBO for estimation of plan statistics when set true.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val PLAN_STATS_ENABLED =
     buildConf("spark.sql.cbo.planStats.enabled")
       .doc("When true, the logical plan will fetch row counts and column statistics from catalog.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_ENABLED =
     buildConf("spark.sql.cbo.joinReorder.enabled")
       .doc("Enables join reorder in CBO.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_DP_THRESHOLD =
     buildConf("spark.sql.cbo.joinReorder.dp.threshold")
       .doc("The maximum number of joined nodes allowed in the dynamic programming algorithm.")
+      .version("")
       .intConf
       .checkValue(number => number > 0, "The maximum number must be a positive integer.")
       .createWithDefault(12)
@@ -1550,6 +1654,7 @@ object SQLConf {
       .internal()
       .doc("The weight of cardinality (number of rows) for plan cost comparison in join reorder: " +
         "rows * weight + size * (1 - weight).")
+      .version("")
       .doubleConf
       .checkValue(weight => weight >= 0 && weight <= 1, "The weight value must be in [0, 1].")
       .createWithDefault(0.7)
@@ -1557,11 +1662,13 @@ object SQLConf {
   val JOIN_REORDER_DP_STAR_FILTER =
     buildConf("spark.sql.cbo.joinReorder.dp.star.filter")
       .doc("Applies star-join filter heuristics to cost based join enumeration.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val STARSCHEMA_DETECTION = buildConf("spark.sql.cbo.starSchemaDetection")
     .doc("When true, it enables join reordering based on star schema detection. ")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -1569,6 +1676,7 @@ object SQLConf {
     .internal()
     .doc("Specifies the upper limit of the ratio between the largest fact tables" +
       " for a star join to be considered. ")
+    .version("2.2.0")
     .doubleConf
     .createWithDefault(0.9)
 
@@ -1582,6 +1690,7 @@ object SQLConf {
       "Zone offsets must be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. " +
       "Also 'UTC' and 'Z' are supported as aliases of '+00:00'. Other short names are not " +
       "recommended to use because they can be ambiguous.")
+    .version("2.2.0")
     .stringConf
     .checkValue(isValidTimezone, s"Cannot resolve the given timezone with" +
       " ZoneId.of(_, ZoneId.SHORT_IDS)")
@@ -1591,6 +1700,7 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.in.memory.threshold")
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the window operator")
+      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1598,6 +1708,7 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by window operator")
+      .version("")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1606,6 +1717,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the sort merge " +
         "join operator")
+      .version("")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1613,6 +1725,7 @@ object SQLConf {
     buildConf("spark.sql.sortMergeJoinExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by sort merge join operator")
+      .version("")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1621,6 +1734,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the cartesian " +
         "product operator")
+      .version("")
       .intConf
       .createWithDefault(4096)
 
@@ -1628,12 +1742,14 @@ object SQLConf {
     buildConf("spark.sql.cartesianProductExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by cartesian product operator")
+      .version("")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
   val SUPPORT_QUOTED_REGEX_COLUMN_NAME = buildConf("spark.sql.parser.quotedRegexColumnNames")
     .doc("When true, quoted Identifiers (using backticks) in SELECT statement are interpreted" +
       " as regular expressions.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -1642,12 +1758,14 @@ object SQLConf {
       .internal()
       .doc("Number of points to sample per partition in order to determine the range boundaries" +
           " for range partitioning, typically used in global sorting (without limit).")
+      .version("")
       .intConf
       .createWithDefault(100)
 
   val ARROW_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.enabled")
       .doc("(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -1659,6 +1777,7 @@ object SQLConf {
         "2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame " +
         "The following data types are unsupported: " +
         "BinaryType, MapType, ArrayType of TimestampType, and nested StructType.")
+      .version("")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
   val ARROW_SPARKR_EXECUTION_ENABLED =
@@ -1671,6 +1790,7 @@ object SQLConf {
         "4. gapply " +
         "The following data types are unsupported: " +
         "FloatType, BinaryType, ArrayType, StructType and MapType.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -1678,6 +1798,7 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.fallback.enabled")
       .doc("(Deprecated since Spark 3.0, please set " +
         "'spark.sql.execution.arrow.pyspark.fallback.enabled'.)")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1685,12 +1806,14 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.pyspark.fallback.enabled")
       .doc(s"When true, optimizations enabled by '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' will " +
         "fallback automatically to non-optimized implementations if an error occurs.")
+      .version("")
       .fallbackConf(ARROW_FALLBACK_ENABLED)
 
   val ARROW_EXECUTION_MAX_RECORDS_PER_BATCH =
     buildConf("spark.sql.execution.arrow.maxRecordsPerBatch")
       .doc("When using Apache Arrow, limit the maximum number of records that can be written " +
         "to a single ArrowRecordBatch in memory. If set to zero or negative there is no limit.")
+      .version("")
       .intConf
       .createWithDefault(10000)
 
@@ -1701,6 +1824,7 @@ object SQLConf {
         s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
         "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
         "pipelined; however, it might degrade performance. See SPARK-27870.")
+      .version("")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
@@ -1710,6 +1834,7 @@ object SQLConf {
         "to use position if not. When false, a grouped map Pandas UDF will assign columns from " +
         "the returned Pandas DataFrame based on position, regardless of column label type. " +
         "This configuration will be deprecated in future releases.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1720,6 +1845,7 @@ object SQLConf {
         "Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
         "when detecting unsafe type conversion like overflow. When false, disabling Arrow's type " +
         "check and do type conversions anyway. This config only works for Arrow 0.11.0+.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -1733,6 +1859,7 @@ object SQLConf {
       " column names of the left node are distinct. If all the conditions are met, the" +
       " rule will replace the except operation with a Filter by flipping the filter" +
       " condition(s) of the right node.")
+    .version("")
     .booleanConf
     .createWithDefault(true)
 
@@ -1743,6 +1870,7 @@ object SQLConf {
         "happens according to Hive behavior and SQL ANSI 2011 specification, ie. rounding the " +
         "decimal part of the result if an exact representation is not possible. Otherwise, NULL " +
         "is returned in those cases, as previously.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1752,6 +1880,7 @@ object SQLConf {
       .doc("When integral literal is used in decimal operations, pick a minimum precision " +
         "required by the literal if this config is true, to make the resulting precision and/or " +
         "scale smaller. This can reduce the possibility of precision lose and/or overflow.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1770,17 +1899,20 @@ object SQLConf {
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands. " +
         "When this conf is not set, the value from `spark.redaction.string.regex` is used.")
+      .version("")
       .fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN)
 
   val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString")
     .doc("When this option is set to false and all inputs are binary, `functions.concat` returns " +
       "an output as binary. Otherwise, it returns as a string.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
   val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString")
     .doc("When this option is set to false and all inputs are binary, `elt` returns " +
       "an output as binary. Otherwise, it returns as a string.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -1791,6 +1923,7 @@ object SQLConf {
         "user-specified schema. If the validation fails, a runtime exception is thrown. " +
         "When this option is set to false, the partition column value will be converted to null " +
         "if it can not be casted to corresponding user-specified schema.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1798,22 +1931,25 @@ object SQLConf {
     buildConf("spark.sql.streaming.continuous.epochBacklogQueueSize")
       .doc("The max number of entries to be stored in queue to wait for late epochs. " +
         "If this parameter is exceeded by the size of the queue, stream will stop with an error.")
+      .version("")
       .intConf
       .createWithDefault(10000)
 
   val CONTINUOUS_STREAMING_EXECUTOR_QUEUE_SIZE =
     buildConf("spark.sql.streaming.continuous.executorQueueSize")
-    .internal()
-    .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
-      " buffer the results of a ContinuousDataReader.")
-    .intConf
-    .createWithDefault(1024)
+      .internal()
+      .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
+        " buffer the results of a ContinuousDataReader.")
+      .version("")
+      .intConf
+      .createWithDefault(1024)
 
   val CONTINUOUS_STREAMING_EXECUTOR_POLL_INTERVAL_MS =
     buildConf("spark.sql.streaming.continuous.executorPollIntervalMs")
       .internal()
       .doc("The interval at which continuous execution readers will poll to check whether" +
         " the epoch has advanced on the driver.")
+      .version("")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
@@ -1822,12 +1958,14 @@ object SQLConf {
     .doc("A comma-separated list of data source short names or fully qualified data source " +
       "implementation class names for which Data Source V2 code path is disabled. These data " +
       "sources will fallback to Data Source V1 code path.")
+    .version("")
     .stringConf
     .createWithDefault("avro,csv,json,kafka,orc,parquet,text")
 
   val DISABLED_V2_STREAMING_WRITERS = buildConf("spark.sql.streaming.disabledV2Writers")
     .doc("A comma-separated list of fully qualified data source register class names for which" +
       " StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.")
+    .version("")
     .stringConf
     .createWithDefault("")
 
@@ -1838,6 +1976,7 @@ object SQLConf {
         "A comma-separated list of fully qualified data source register class names for which " +
           "MicroBatchReadSupport is disabled. Reads from these sources will fall back to the " +
           "V1 Sources.")
+      .version("")
       .stringConf
       .createWithDefault("")
 
@@ -1858,6 +1997,7 @@ object SQLConf {
         "(which takes precedence over this setting), e.g. " +
         "dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path)."
       )
+      .version("")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(PartitionOverwriteMode.values.map(_.toString))
@@ -1882,6 +2022,7 @@ object SQLConf {
         "in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is " +
         "not allowed."
       )
+      .version("")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(StoreAssignmentPolicy.values.map(_.toString))
@@ -1892,6 +2033,7 @@ object SQLConf {
       "throw a runtime exception if an overflow occurs in any operation on integral/decimal " +
       "field. 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in " +
       "the SQL parser.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -1904,8 +2046,9 @@ object SQLConf {
         "issues. Turn on this config to insert a local sort before actually doing repartition " +
         "to generate consistent repartition results. The performance of repartition() may go " +
         "down since we insert extra local sort before it.")
-        .booleanConf
-        .createWithDefault(true)
+      .version("")
+      .booleanConf
+      .createWithDefault(true)
 
   val NESTED_SCHEMA_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.nestedSchemaPruning.enabled")
@@ -1914,6 +2057,7 @@ object SQLConf {
         "satisfying a query. This optimization allows columnar file format readers to avoid " +
         "reading unnecessary nested column data. Currently Parquet and ORC are the " +
         "data sources that implement this optimization.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1933,6 +2077,7 @@ object SQLConf {
       .doc("Prune nested fields from object serialization operator which are unnecessary in " +
         "satisfying a query. This optimization allows object serializers to avoid " +
         "executing unnecessary nested expressions.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1943,6 +2088,7 @@ object SQLConf {
         "satisfying a query. Note that this optimization doesn't prune nested fields from " +
         "physical data source scanning. For pruning nested fields from scanning, please use " +
         "`spark.sql.optimizer.nestedSchemaPruning.enabled` config.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -1952,6 +2098,7 @@ object SQLConf {
       .doc("In SQL queries with a SORT followed by a LIMIT like " +
           "'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort" +
           " in memory, otherwise do a global sort which spills to disk if necessary.")
+      .version("")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1967,6 +2114,7 @@ object SQLConf {
     .internal()
     .doc("If it is set to true, column names of the requested schema are passed to CSV parser. " +
       "Other column values can be ignored during parsing even if they are malformed.")
+    .version("")
     .booleanConf
     .createWithDefault(true)
 
@@ -1977,6 +2125,7 @@ object SQLConf {
       "Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python " +
       "REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned " +
       "outputs are showed similar to R data.frame would.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -1985,12 +2134,14 @@ object SQLConf {
       s"effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true. The valid range of this " +
       "config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and " +
       "greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).")
+    .version("")
     .intConf
     .createWithDefault(20)
 
   val REPL_EAGER_EVAL_TRUNCATE = buildConf("spark.sql.repl.eagerEval.truncate")
     .doc("The max number of characters for each cell that is returned by eager evaluation. " +
       s"This only takes effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true.")
+    .version("")
     .intConf
     .createWithDefault(20)
 
@@ -2001,6 +2152,7 @@ object SQLConf {
         "by the fast hash aggregate product operator. The bit is not for actual value, " +
         "but the actual numBuckets is determined by loadFactor " +
         "(e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).")
+      .version("")
       .intConf
       .checkValue(bit => bit >= 10 && bit <= 30, "The bit value must be in [10, 30].")
       .createWithDefault(16)
@@ -2008,6 +2160,7 @@ object SQLConf {
   val AVRO_COMPRESSION_CODEC = buildConf("spark.sql.avro.compression.codec")
     .doc("Compression codec used in writing of AVRO files. Supported codecs: " +
       "uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.")
+    .version("")
     .stringConf
     .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz"))
     .createWithDefault("snappy")
@@ -2016,6 +2169,7 @@ object SQLConf {
     .doc("Compression level for the deflate codec used in writing of AVRO files. " +
       "Valid value must be in the range of from 1 to 9 inclusive or -1. " +
       "The default value is -1 which corresponds to 6 level in the current implementation.")
+    .version("2.4.0")
     .intConf
     .checkValues((1 to 9).toSet + Deflater.DEFAULT_COMPRESSION)
     .createWithDefault(Deflater.DEFAULT_COMPRESSION)
@@ -2024,6 +2178,7 @@ object SQLConf {
     .internal()
     .doc(s"If it is set to false, or ${ANSI_ENABLED.key} is true, then size of null returns " +
       "null. Otherwise, it returns -1, which was inherited from Hive.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2032,6 +2187,7 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, the data source provider com.databricks.spark.avro is mapped " +
         "to the built-in but external Avro data source module for backward compatibility.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2042,6 +2198,7 @@ object SQLConf {
         "set operations are performed from left to right as they appear in the query. When set " +
         "to false and order of evaluation is not specified by parentheses, INTERSECT operations " +
         "are performed before any UNION, EXCEPT and MINUS operations.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2050,6 +2207,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, a literal with an exponent (e.g. 1E-30) would be parsed " +
         "as Decimal rather than Double.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2059,6 +2217,7 @@ object SQLConf {
       .doc("When set to true, negative scale of Decimal type is allowed. For example, " +
         "the type of number 1E10BD under legacy mode is DecimalType(2, -9), but is " +
         "Decimal(11, 0) in non legacy mode.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2067,6 +2226,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, CREATE TABLE syntax without a provider will use hive " +
         s"instead of the value of ${DEFAULT_DATA_SOURCE_NAME.key}.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2075,14 +2235,16 @@ object SQLConf {
       .internal()
       .doc("When true, the bucketed table scan will list files during planning to figure out the " +
         "output ordering, which is expensive and may make the planning quite slow.")
-    .booleanConf
-    .createWithDefault(false)
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   val LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE =
     buildConf("spark.sql.legacy.parser.havingWithoutGroupByAsWhere")
       .internal()
       .doc("If it is set to true, the parser will treat HAVING without GROUP BY as a normal " +
         "WHERE, which does not follow SQL standard.")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(false)
 
@@ -2090,7 +2252,8 @@ object SQLConf {
     buildConf("spark.sql.legacy.json.allowEmptyString.enabled")
       .internal()
       .doc("When set to true, the parser of JSON data source treats empty strings as null for " +
-      "some data types such as `IntegerType`.")
+        "some data types such as `IntegerType`.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2100,6 +2263,7 @@ object SQLConf {
       .doc("When set to true, Spark returns an empty collection with `StringType` as element " +
         "type if the `array`/`map` function is called without any parameters. Otherwise, Spark " +
         "returns an empty collection with `NullType` as element type.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2108,6 +2272,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, user is allowed to use org.apache.spark.sql.functions." +
         "udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2116,6 +2281,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, TRUNCATE TABLE command will not try to set back original " +
         "permission and ACLs when re-creating the table/partition paths.")
+      .version("2.4.6")
       .booleanConf
       .createWithDefault(false)
 
@@ -2125,6 +2291,7 @@ object SQLConf {
       .doc("When set to true, the key attribute resulted from running `Dataset.groupByKey` " +
         "for non-struct key type, will be named as `value`, following the behavior of Spark " +
         "version 2.4 and earlier.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2132,6 +2299,7 @@ object SQLConf {
     .doc("Maximum number of fields of sequence-like entries can be converted to strings " +
       "in debug output. Any elements beyond the limit will be dropped and replaced by a" +
       """ "... N more fields" placeholder.""")
+    .version("")
     .intConf
     .createWithDefault(25)
 
@@ -2140,6 +2308,7 @@ object SQLConf {
       "longer, further output will be truncated.  The default setting always generates a full " +
       "plan.  Set this to a lower value such as 8k if plan strings are taking up too much " +
       "memory or are causing OutOfMemory errors in the driver or UI processes.")
+    .version("")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(i => i >= 0 && i <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, "Invalid " +
       "value for 'spark.sql.maxPlanStringLength'.  Length must be a valid string length " +
@@ -2151,6 +2320,7 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, SET command will fail when the key is registered as " +
         "a SparkConf entry.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -2159,6 +2329,7 @@ object SQLConf {
       "java.time.LocalDate classes of Java 8 API are used as external types for " +
       "Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp " +
       "and java.sql.Date are used for the same purpose.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -2166,6 +2337,7 @@ object SQLConf {
     .doc("The max length of a file that can be read by the binary file data source. " +
       "Spark will fail fast and not attempt to read the file if its length exceeds this value. " +
       "The theoretical max is Int.MaxValue, though VMs might implement a smaller max.")
+    .version("")
     .internal()
     .intConf
     .createWithDefault(Int.MaxValue)
@@ -2175,12 +2347,14 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
         "with String")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog")
     .doc("Name of the default catalog. This will be the current catalog if users have not " +
       "explicitly set the current catalog yet.")
+    .version("")
     .stringConf
     .createWithDefault(SESSION_CATALOG_NAME)
 
@@ -2192,6 +2366,7 @@ object SQLConf {
         s"be loaded by the $SESSION_CATALOG_NAME, this catalog must also return the table " +
         s"metadata. To delegate operations to the $SESSION_CATALOG_NAME, implementations can " +
         "extend 'CatalogExtension'.")
+      .version("")
       .stringConf
       .createOptional
 
@@ -2212,6 +2387,7 @@ object SQLConf {
   val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
     .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
+    .version("")
     .booleanConf
     .createWithDefault(false)
 
@@ -2225,6 +2401,7 @@ object SQLConf {
       "CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, " +
       "AnalysisException is thrown while name conflict is detected in nested CTE. This config " +
       "will be removed in future versions and CORRECTED will be the only behavior.")
+    .version("")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2246,6 +2423,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
       .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
+      .version("")
       .booleanConf
       .createWithDefault(true)
 
@@ -2254,6 +2432,7 @@ object SQLConf {
       .doc("A comma-delimited string config of the optional additional remote Maven mirror " +
         "repositories. This is only used for downloading Hive jars in IsolatedClientLoader " +
         "if the default Maven Central repo is unreachable.")
+      .version("")
       .stringConf
       .createWithDefault(
         "https://maven-central.storage-download.googleapis.com/maven2/")
@@ -2266,6 +2445,7 @@ object SQLConf {
         "all interval units out of the specified range. If it is set to `false`, " +
         "`ParseException` is thrown if the input does not match to the pattern " +
         "defined by `from` and `to`.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2275,6 +2455,7 @@ object SQLConf {
       .doc("When true, all database and table properties are not reserved and available for " +
         "create/alter syntaxes. But please be aware that the reserved properties will be " +
         "silently removed.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2283,6 +2464,7 @@ object SQLConf {
       .internal()
       .doc("When true, only a single file can be added using ADD FILE. If false, then users " +
         "can add directory by passing directory path to ADD FILE.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
@@ -2290,11 +2472,13 @@ object SQLConf {
     buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
       .internal()
       .doc("When true, use legacy MySqlServer SMALLINT and REAL type mapping.")
+      .version("")
       .booleanConf
       .createWithDefault(false)
 
   val CSV_FILTER_PUSHDOWN_ENABLED = buildConf("spark.sql.csv.filterPushdown.enabled")
     .doc("When true, enable filter pushdown to CSV datasource.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2304,6 +2488,7 @@ object SQLConf {
       .doc("The number of partitions to be handled in one turn when use " +
         "`AlterTableAddPartitionCommand` to add partitions into table. The smaller " +
         "batch size is, the less memory is required for the real handler, e.g. Hive Metastore.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The value of spark.sql.addPartitionInBatch.size must be positive")
       .createWithDefault(100)
@@ -2311,6 +2496,7 @@ object SQLConf {
   val LEGACY_ALLOW_HASH_ON_MAPTYPE = buildConf("spark.sql.legacy.allowHashOnMapType")
     .doc("When set to true, hash expressions can be applied on elements of MapType. Otherwise, " +
       "an analysis exception will be thrown.")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2323,6 +2509,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2335,6 +2522,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2347,6 +2535,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2359,6 +2548,7 @@ object SQLConf {
         "a local date/timestamp in the source calendar, interpreting the resulted date/" +
         "timestamp in the target calendar, and getting the number of micros/millis/days " +
         "since the epoch 1970-01-01 00:00:00Z.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 

From 71620870d59807b5efb63fa76f5b3a83ad8ae2d2 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Fri, 6 Mar 2020 11:19:08 +0900
Subject: [PATCH 0444/1280] [SPARK-30841][SQL][DOC][FOLLOW-UP] Add version
 information to the configuration of SQL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR follows https://github.com/apache/spark/pull/27691 and https://github.com/apache/spark/pull/27730
I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.execution.useObjectHashAggregateExec | 2.2.0 | SPARK-19944 | 0ee38a39e43dd7ad9d50457e446ae36f64621a1b#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.jsonGenerator.ignoreNullFields | 3.0.0 | SPARK-29444 | 78b0cbe265c4e8cc3d4d8bf5d734f2998c04d376#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.streaming.fileSink.log.deletion | 2.0.0 | SPARK-14678 | 7bc948557bb6169cbeec335f8400af09375a62d3#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.fileSink.log.compactInterval | 2.0.0 | SPARK-14678 | 7bc948557bb6169cbeec335f8400af09375a62d3#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.fileSink.log.cleanupDelay | 2.0.0 | SPARK-14678 | 7bc948557bb6169cbeec335f8400af09375a62d3#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.fileSource.log.deletion | 2.0.1 | SPARK-15698 | 8d8e2332ca12067817de45a8d3812928150975d0#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.fileSource.log.compactInterval | 2.0.1 | SPARK-15698 | 8d8e2332ca12067817de45a8d3812928150975d0#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.fileSource.log.cleanupDelay | 2.0.1 | SPARK-15698 | 8d8e2332ca12067817de45a8d3812928150975d0#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.fileSource.schema.forceNullable | 3.0.0 | SPARK-28651 | 5bb69945e4aaf519cd10a5c5083332f618039af0#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.fileSource.cleaner.numThreads | 3.0.0 | SPARK-29876 | abf759a91e01497586b8bb6b7a314dd28fd6cff1#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.schemaInference | 2.0.0 | SPARK-15458 | 1fb7b3a0a2e3a5c5f784aab662df93fcc1449c36#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.streaming.pollingDelay | 2.0.0 | SPARK-16002 | afa14b71b28d788c53816bd2616ccff0c3967f40#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.streaming.stopTimeout | 3.0.0 | SPARK-30143 | 4c37a8a3f4a489b52f1919d2db84f6e32c6a05cd#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.noDataProgressEventInterval | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.streaming.noDataMicroBatches.enabled | 2.4.1 | SPARK-24157 | 535bf1cc9e6b54df7059ac3109b8cba30057d040#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.metricsEnabled | 2.0.2 | SPARK-17731 | 881e0eb05782ea74cf92a62954466b14ea9e05b6#diff-32bb9518401c0948c5ea19377b5069ab |
spark.sql.streaming.numRecentProgressUpdates | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.streaming.checkpointFileManagerClass | 2.4.0 | SPARK-23966 | cbb41a0c5b01579c85f06ef42cc0585fbef216c5#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.checkpoint.escapedPathCheck.enabled | 3.0.0 | SPARK-26824 | 77b99af57330cf2e5016a6acc69642d54041b041#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.statistics.parallelFileListingInStatsComputation.enabled | 2.4.1 | SPARK-24626 | f11f44548903bbab7ab764574d6bed326cf4cd8d#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.defaultSizeInBytes | 1.1.0 | SPARK-2393 | c7db274be79f448fda566208946cb50958ea9b1a#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.statistics.fallBackToHdfs | 2.0.0 | SPARK-15960 | 5c53442cc098dd618ba1430962727c74b2de2e68#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.statistics.ndv.maxError | 2.1.1 | SPARK-19944 | 80ebca62cbdb7d5c8606e95a944164ab1a943694#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.statistics.histogram.enabled | 2.3.0 | SPARK-17074 | 11b60af737a04d931356aa74ebf3c6cf4a6b08d6#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.statistics.histogram.numBins | 2.3.0 | SPARK-17074 | 11b60af737a04d931356aa74ebf3c6cf4a6b08d6#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.statistics.percentile.accuracy | 2.3.0 | SPARK-17074 | 11b60af737a04d931356aa74ebf3c6cf4a6b08d6#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.statistics.size.autoUpdate.enabled | 2.3.0 | SPARK-21127 | d5202259d9aa9ad95d572af253bf4a722b7b437a#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.cbo.enabled | 2.2.0 | SPARK-19944 | 0ee38a39e43dd7ad9d50457e446ae36f64621a1b#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.cbo.planStats.enabled | 3.0.0 | SPARK-24690 | 3f3a18fff116a02ff7996d45a1061f48a2de3102#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.cbo.joinReorder.enabled | 2.2.0 | SPARK-19944 | 0ee38a39e43dd7ad9d50457e446ae36f64621a1b#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.cbo.joinReorder.dp.threshold | 2.2.0 | SPARK-19944 | 0ee38a39e43dd7ad9d50457e446ae36f64621a1b#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.cbo.joinReorder.card.weight | 2.2.0 | SPARK-19915 | c083b6b7dec337d680b54dabeaa40e7a0f69ae69#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.cbo.joinReorder.dp.star.filter | 2.2.0 | SPARK-20233 | fbe4216e1e83d243a7f0521b76bfb20c25278281#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.cbo.starSchemaDetection | 2.2.0 | SPARK-17791 | 81639115947a13017d1637549a8f66ba599b27b8#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.cbo.starJoinFTRatio | 2.2.0 | SPARK-17791 | 81639115947a13017d1637549a8f66ba599b27b8#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.session.timeZone | 2.2.0 | SPARK-19944 | 0ee38a39e43dd7ad9d50457e446ae36f64621a1b#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.windowExec.buffer.in.memory.threshold | 2.2.1 | SPARK-21595 | 406eb1c2ee670c2f14f2737c32c9aa0b8d35bf7c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.windowExec.buffer.spill.threshold | 2.2.0 | SPARK-13450 | 02c274eaba0a8e7611226e0d4e93d3c36253f4ce#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sortMergeJoinExec.buffer.in.memory.threshold | 2.2.1 | SPARK-21595 | 406eb1c2ee670c2f14f2737c32c9aa0b8d35bf7c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sortMergeJoinExec.buffer.spill.threshold | 2.2.0 | SPARK-13450 | 02c274eaba0a8e7611226e0d4e93d3c36253f4ce#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.cartesianProductExec.buffer.in.memory.threshold | 2.2.1 | SPARK-21595 | 406eb1c2ee670c2f14f2737c32c9aa0b8d35bf7c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.cartesianProductExec.buffer.spill.threshold | 2.2.0 | SPARK-13450 | 02c274eaba0a8e7611226e0d4e93d3c36253f4ce#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.parser.quotedRegexColumnNames | 2.3.0 | SPARK-12139 | 2cbfc975ba937a4eb761de7a6473b7747941f386#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.rangeExchange.sampleSizePerPartition | 2.3.0 | SPARK-22160 | 323806e68f91f3c7521327186a37ddd1436267d0#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.arrow.enabled | 2.3.0 | SPARK-22159 | d29d1e87995e02cb57ba3026c945c3cd66bb06e2#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.execution.arrow.pyspark.enabled | 3.0.0 | SPARK-27834 | db48da87f02e2e89710ba65fab8b07e9c85b9e74#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.arrow.sparkr.enabled | 3.0.0 | SPARK-27834 | db48da87f02e2e89710ba65fab8b07e9c85b9e74#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.arrow.fallback.enabled | 2.4.0 | SPARK-23380 | d6632d185e147fcbe6724545488ad80dce20277e#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.execution.arrow.pyspark.fallback.enabled | 3.0.0 | SPARK-27834 | db48da87f02e2e89710ba65fab8b07e9c85b9e74#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.arrow.maxRecordsPerBatch | 2.3.0 | SPARK-13534 | d03aebbe6508ba441dc87f9546f27aeb27553d77#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.pandas.udf.buffer.size | 3.1.0 | SPARK-27870 | 692e3ddb4e517638156f7427ade8b62fb37634a7#diff-9a6b543db706f1a90f790783d6930a13 | Exists in master, not branch-3.0
spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName | 2.4.1 | SPARK-24324 | 3f203050ac764516e68fb43628bba0df5963e44d#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.execution.pandas.convertToArrowArraySafely | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.replaceExceptWithFilter | 2.3.0 | SPARK-22181 | 01f6ba0e7a12ef818d56e7d5b1bd889b79f2b57c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.decimalOperations.allowPrecisionLoss | 2.3.1 | SPARK-22036 | 8a98274823a4671cee85081dd19f40146e736325#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.literal.pickMinimumPrecision | 2.3.3 | SPARK-25454 | 26d893a4f64de18222942568f7735114447a6ab7#diff-9a6b543db706f1a90f790783d6930a13 |  

Supplemental configuration version information.

No

Exists UT

Closes #27770 from beliefer/add-version-to-sql-config-part-three.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 106 +++++++++---------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 9358dc7ef6ee3..378e900e6676f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1391,7 +1391,7 @@ object SQLConf {
   val USE_OBJECT_HASH_AGG = buildConf("spark.sql.execution.useObjectHashAggregateExec")
     .internal()
     .doc("Decides if we use ObjectHashAggregateExec")
-    .version("")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1400,14 +1400,14 @@ object SQLConf {
       .doc("Whether to ignore null fields when generating JSON objects in JSON data source and " +
         "JSON functions such as to_json. " +
         "If false, it generates null for null fields in JSON objects.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SINK_LOG_DELETION = buildConf("spark.sql.streaming.fileSink.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream sink.")
-    .version("")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1416,7 +1416,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
-      .version("")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
@@ -1424,14 +1424,14 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSink.log.cleanupDelay")
       .internal()
       .doc("How long that a file is guaranteed to be visible for all readers.")
-      .version("")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
   val FILE_SOURCE_LOG_DELETION = buildConf("spark.sql.streaming.fileSource.log.deletion")
     .internal()
     .doc("Whether to delete the expired log files in file stream source.")
-    .version("")
+    .version("2.0.1")
     .booleanConf
     .createWithDefault(true)
 
@@ -1440,7 +1440,7 @@ object SQLConf {
       .internal()
       .doc("Number of log files after which all the previous files " +
         "are compacted into the next log file.")
-      .version("")
+      .version("2.0.1")
       .intConf
       .createWithDefault(10)
 
@@ -1448,7 +1448,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.fileSource.log.cleanupDelay")
       .internal()
       .doc("How long in milliseconds a file is guaranteed to be visible for all readers.")
-      .version("")
+      .version("2.0.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(TimeUnit.MINUTES.toMillis(10)) // 10 minutes
 
@@ -1458,14 +1458,14 @@ object SQLConf {
       .doc("When true, force the schema of streaming file source to be nullable (including all " +
         "the fields). Otherwise, the schema might not be compatible with actual data, which " +
         "leads to corruptions.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   val FILE_SOURCE_CLEANER_NUM_THREADS =
     buildConf("spark.sql.streaming.fileSource.cleaner.numThreads")
       .doc("Number of threads used in the file source completed file cleaner.")
-      .version("")
+      .version("3.0.0")
       .intConf
       .createWithDefault(1)
 
@@ -1473,7 +1473,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.schemaInference")
       .internal()
       .doc("Whether file-based streaming sources will infer its own schema")
-      .version("")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1481,7 +1481,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.pollingDelay")
       .internal()
       .doc("How long to delay polling new data when no data is available")
-      .version("")
+      .version("2.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10L)
 
@@ -1489,7 +1489,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.stopTimeout")
       .doc("How long to wait in milliseconds for the streaming execution thread to stop when " +
         "calling the streaming query's stop() method. 0 or negative values wait indefinitely.")
-      .version("")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
@@ -1497,7 +1497,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.noDataProgressEventInterval")
       .internal()
       .doc("How long to wait between two progress events when there is no data")
-      .version("")
+      .version("2.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(10000L)
 
@@ -1506,21 +1506,21 @@ object SQLConf {
       .doc(
         "Whether streaming micro-batch engine will execute batches without data " +
           "for eager state management for stateful streaming queries.")
-      .version("")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
   val STREAMING_METRICS_ENABLED =
     buildConf("spark.sql.streaming.metricsEnabled")
       .doc("Whether Dropwizard/Codahale metrics will be reported for active streaming queries.")
-      .version("")
+      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   val STREAMING_PROGRESS_RETENTION =
     buildConf("spark.sql.streaming.numRecentProgressUpdates")
       .doc("The number of progress updates to retain for a streaming query")
-      .version("")
+      .version("2.1.1")
       .intConf
       .createWithDefault(100)
 
@@ -1528,7 +1528,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpointFileManagerClass")
       .doc("The class used to write checkpoint files atomically. This class must be a subclass " +
         "of the interface CheckpointFileManager.")
-      .version("")
+      .version("2.4.0")
       .internal()
       .stringConf
 
@@ -1536,7 +1536,7 @@ object SQLConf {
     buildConf("spark.sql.streaming.checkpoint.escapedPathCheck.enabled")
       .doc("Whether to detect a streaming query may pick up an incorrect checkpoint path due " +
         "to SPARK-26824.")
-      .version("")
+      .version("3.0.0")
       .internal()
       .booleanConf
       .createWithDefault(true)
@@ -1547,7 +1547,7 @@ object SQLConf {
       .doc("When true, SQL commands use parallel file listing, " +
         "as opposed to single thread listing. " +
         "This usually speeds up commands that need to list many directories.")
-      .version("")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1557,7 +1557,7 @@ object SQLConf {
       s"which is larger than `${AUTO_BROADCASTJOIN_THRESHOLD.key}` to be more conservative. " +
       "That is to say by default the optimizer will not choose to broadcast a table unless it " +
       "knows for sure its size is small enough.")
-    .version("")
+    .version("1.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(Long.MaxValue)
 
@@ -1568,7 +1568,7 @@ object SQLConf {
       "For non-partitioned data source tables, it will be automatically recalculated if table " +
       "statistics are not available. For partitioned data source and partitioned Hive tables, " +
       s"It is '${DEFAULT_SIZE_IN_BYTES.key}' if table statistics are not available.")
-    .version("")
+    .version("2.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1577,7 +1577,7 @@ object SQLConf {
       .internal()
       .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
         "column level statistics.")
-      .version("")
+      .version("2.1.1")
       .doubleConf
       .createWithDefault(0.05)
 
@@ -1588,7 +1588,7 @@ object SQLConf {
         "histogram. Note that collecting histograms takes extra cost. For example, collecting " +
         "column statistics usually takes only one table scan, but generating equi-height " +
         "histogram will cause an extra table scan.")
-      .version("")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1596,7 +1596,7 @@ object SQLConf {
     buildConf("spark.sql.statistics.histogram.numBins")
       .internal()
       .doc("The number of bins when generating histograms.")
-      .version("")
+      .version("2.3.0")
       .intConf
       .checkValue(num => num > 1, "The number of bins must be greater than 1.")
       .createWithDefault(254)
@@ -1607,7 +1607,7 @@ object SQLConf {
       .doc("Accuracy of percentile approximation when generating equi-height histograms. " +
         "Larger value means better accuracy. The relative error can be deduced by " +
         "1.0 / PERCENTILE_ACCURACY.")
-      .version("")
+      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1616,35 +1616,35 @@ object SQLConf {
       .doc("Enables automatic update for table size once table's data is changed. Note that if " +
         "the total number of files of the table is very large, this can be expensive and slow " +
         "down data change commands.")
-      .version("")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   val CBO_ENABLED =
     buildConf("spark.sql.cbo.enabled")
       .doc("Enables CBO for estimation of plan statistics when set true.")
-      .version("")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val PLAN_STATS_ENABLED =
     buildConf("spark.sql.cbo.planStats.enabled")
       .doc("When true, the logical plan will fetch row counts and column statistics from catalog.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_ENABLED =
     buildConf("spark.sql.cbo.joinReorder.enabled")
       .doc("Enables join reorder in CBO.")
-      .version("")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val JOIN_REORDER_DP_THRESHOLD =
     buildConf("spark.sql.cbo.joinReorder.dp.threshold")
       .doc("The maximum number of joined nodes allowed in the dynamic programming algorithm.")
-      .version("")
+      .version("2.2.0")
       .intConf
       .checkValue(number => number > 0, "The maximum number must be a positive integer.")
       .createWithDefault(12)
@@ -1654,7 +1654,7 @@ object SQLConf {
       .internal()
       .doc("The weight of cardinality (number of rows) for plan cost comparison in join reorder: " +
         "rows * weight + size * (1 - weight).")
-      .version("")
+      .version("2.2.0")
       .doubleConf
       .checkValue(weight => weight >= 0 && weight <= 1, "The weight value must be in [0, 1].")
       .createWithDefault(0.7)
@@ -1662,13 +1662,13 @@ object SQLConf {
   val JOIN_REORDER_DP_STAR_FILTER =
     buildConf("spark.sql.cbo.joinReorder.dp.star.filter")
       .doc("Applies star-join filter heuristics to cost based join enumeration.")
-      .version("")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   val STARSCHEMA_DETECTION = buildConf("spark.sql.cbo.starSchemaDetection")
     .doc("When true, it enables join reordering based on star schema detection. ")
-    .version("")
+    .version("2.2.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1708,7 +1708,7 @@ object SQLConf {
     buildConf("spark.sql.windowExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by window operator")
-      .version("")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1717,7 +1717,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the sort merge " +
         "join operator")
-      .version("")
+      .version("2.2.1")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1725,7 +1725,7 @@ object SQLConf {
     buildConf("spark.sql.sortMergeJoinExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by sort merge join operator")
-      .version("")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
@@ -1734,7 +1734,7 @@ object SQLConf {
       .internal()
       .doc("Threshold for number of rows guaranteed to be held in memory by the cartesian " +
         "product operator")
-      .version("")
+      .version("2.2.1")
       .intConf
       .createWithDefault(4096)
 
@@ -1742,14 +1742,14 @@ object SQLConf {
     buildConf("spark.sql.cartesianProductExec.buffer.spill.threshold")
       .internal()
       .doc("Threshold for number of rows to be spilled by cartesian product operator")
-      .version("")
+      .version("2.2.0")
       .intConf
       .createWithDefault(SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD.defaultValue.get)
 
   val SUPPORT_QUOTED_REGEX_COLUMN_NAME = buildConf("spark.sql.parser.quotedRegexColumnNames")
     .doc("When true, quoted Identifiers (using backticks) in SELECT statement are interpreted" +
       " as regular expressions.")
-    .version("")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1758,14 +1758,14 @@ object SQLConf {
       .internal()
       .doc("Number of points to sample per partition in order to determine the range boundaries" +
           " for range partitioning, typically used in global sorting (without limit).")
-      .version("")
+      .version("2.3.0")
       .intConf
       .createWithDefault(100)
 
   val ARROW_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.enabled")
       .doc("(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)")
-      .version("")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1777,7 +1777,7 @@ object SQLConf {
         "2. pyspark.sql.SparkSession.createDataFrame when its input is a Pandas DataFrame " +
         "The following data types are unsupported: " +
         "BinaryType, MapType, ArrayType of TimestampType, and nested StructType.")
-      .version("")
+      .version("3.0.0")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
   val ARROW_SPARKR_EXECUTION_ENABLED =
@@ -1790,7 +1790,7 @@ object SQLConf {
         "4. gapply " +
         "The following data types are unsupported: " +
         "FloatType, BinaryType, ArrayType, StructType and MapType.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1798,7 +1798,7 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.fallback.enabled")
       .doc("(Deprecated since Spark 3.0, please set " +
         "'spark.sql.execution.arrow.pyspark.fallback.enabled'.)")
-      .version("")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1806,14 +1806,14 @@ object SQLConf {
     buildConf("spark.sql.execution.arrow.pyspark.fallback.enabled")
       .doc(s"When true, optimizations enabled by '${ARROW_PYSPARK_EXECUTION_ENABLED.key}' will " +
         "fallback automatically to non-optimized implementations if an error occurs.")
-      .version("")
+      .version("3.0.0")
       .fallbackConf(ARROW_FALLBACK_ENABLED)
 
   val ARROW_EXECUTION_MAX_RECORDS_PER_BATCH =
     buildConf("spark.sql.execution.arrow.maxRecordsPerBatch")
       .doc("When using Apache Arrow, limit the maximum number of records that can be written " +
         "to a single ArrowRecordBatch in memory. If set to zero or negative there is no limit.")
-      .version("")
+      .version("2.3.0")
       .intConf
       .createWithDefault(10000)
 
@@ -1824,7 +1824,7 @@ object SQLConf {
         s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
         "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
         "pipelined; however, it might degrade performance. See SPARK-27870.")
-      .version("")
+      .version("3.1.0")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =
@@ -1834,7 +1834,7 @@ object SQLConf {
         "to use position if not. When false, a grouped map Pandas UDF will assign columns from " +
         "the returned Pandas DataFrame based on position, regardless of column label type. " +
         "This configuration will be deprecated in future releases.")
-      .version("")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1845,7 +1845,7 @@ object SQLConf {
         "Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
         "when detecting unsafe type conversion like overflow. When false, disabling Arrow's type " +
         "check and do type conversions anyway. This config only works for Arrow 0.11.0+.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1859,7 +1859,7 @@ object SQLConf {
       " column names of the left node are distinct. If all the conditions are met, the" +
       " rule will replace the except operation with a Filter by flipping the filter" +
       " condition(s) of the right node.")
-    .version("")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -1870,7 +1870,7 @@ object SQLConf {
         "happens according to Hive behavior and SQL ANSI 2011 specification, ie. rounding the " +
         "decimal part of the result if an exact representation is not possible. Otherwise, NULL " +
         "is returned in those cases, as previously.")
-      .version("")
+      .version("2.3.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -1880,7 +1880,7 @@ object SQLConf {
       .doc("When integral literal is used in decimal operations, pick a minimum precision " +
         "required by the literal if this config is true, to make the resulting precision and/or " +
         "scale smaller. This can reduce the possibility of precision lose and/or overflow.")
-      .version("")
+      .version("2.3.3")
       .booleanConf
       .createWithDefault(true)
 

From cae889c50fec42fc23c7618b7db39602776945f5 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Sun, 8 Mar 2020 12:33:02 +0900
Subject: [PATCH 0445/1280] [SPARK-30841][SQL][DOC][FOLLOW-UP] Add version
 information to the configuration of SQL
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR follows https://github.com/apache/spark/pull/27691, https://github.com/apache/spark/pull/27730 and https://github.com/apache/spark/pull/27770
I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.redaction.options.regex | 2.2.2 | SPARK-23850 | 6a55d8b03053e616dcacb79cd2c29a06d219dc32#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.redaction.string.regex | 2.3.0 | SPARK-22791 | 28315714ddef3ddcc192375e98dd5207cf4ecc98#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.function.concatBinaryAsString | 2.3.0 | SPARK-22771 | f2b3525c17d660cf6f082bbafea8632615b4f58e#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.function.eltOutputAsString | 2.3.0 | SPARK-22937 | bf853018cabcd3b3abf84bfe534d2981020b4a71#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sources.validatePartitionColumns | 3.0.0 | SPARK-26263 | 5a140b7844936cf2b65f08853b8cfd8c499d4f13#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.continuous.epochBacklogQueueSize | 3.0.0 | SPARK-24063 | c4bbfd177b4e7cb46f47b39df9fd71d2d9a12c6d#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.continuous.executorQueueSize | 2.3.0 | SPARK-22789 | 8941a4abcada873c26af924e129173dc33d66d71#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.continuous.executorPollIntervalMs | 2.3.0 | SPARK-22789 | 8941a4abcada873c26af924e129173dc33d66d71#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sources.useV1SourceList | 3.0.0 | SPARK-28747 | cb06209fc908bac6ce6a8f20653865489773cbc3#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.streaming.disabledV2Writers | 2.3.1 | SPARK-23196 | 588b9694c1967ff45774431441e84081ee6eb515#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.streaming.disabledV2MicroBatchReaders | 2.4.0 | SPARK-23362 | 0a73aa31f41c83503d5d99eff3c9d7b406014ab3#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sources.partitionOverwriteMode | 2.3.0 | SPARK-20236 | b96248862589bae1ddcdb14ce4c802789a001306#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.storeAssignmentPolicy | 3.0.0 | SPARK-28730 | 895c90b582cc2b2667241f66d5b733852aeef9eb#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.ansi.enabled | 3.0.0 | SPARK-30125 | d9b30694122f8716d3acb448638ef1e2b96ebc7a#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.sortBeforeRepartition | 2.1.4 | SPARK-23207 and SPARK-22905 and SPARK-24564 and SPARK-25114 | 4d2d3d47e00e78893b1ecd5a9a9070adc5243ac9#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.nestedSchemaPruning.enabled | 2.4.1 | SPARK-4502 | dfcff38394929970fee454c69864d0e10d59f8d4#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.serializer.nestedSchemaPruning.enabled | 3.0.0 | SPARK-26837 | 0f2c0b53e8fb18c86c67b5dd679c006db93f94a5#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.optimizer.expression.nestedPruning.enabled | 3.0.0 | SPARK-27707 | 127bc899ae78d73332a87f0972b5db3c9936c1f1#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.execution.topKSortFallbackThreshold | 2.4.0 | SPARK-24193 | 8a837bf4f3f2758f7825d2362cf9de209026651a#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.csv.parser.columnPruning.enabled | 2.4.0 | SPARK-24244 and SPARK-24368 | 64fad0b519cf35b8c0a0dec18dd3df9488a5ed25#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.repl.eagerEval.enabled | 2.4.0 | SPARK-24215 | 6a0b77a55d53e74ac0a0892556c3a7a933474948#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.repl.eagerEval.maxNumRows | 2.4.0 | SPARK-24215 | 6a0b77a55d53e74ac0a0892556c3a7a933474948#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.repl.eagerEval.truncate | 2.4.0 | SPARK-24215 | 6a0b77a55d53e74ac0a0892556c3a7a933474948#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.codegen.aggregate.fastHashMap.capacityBit | 2.4.0 | SPARK-24978 | 6193a202aab0271b4532ee4b740318290f2c44a1#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.avro.compression.codec | 2.4.0 | SPARK-24881 | 0a0f68bae6c0a1bf30184b1e9ac6bf3805bd7511#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.avro.deflate.level | 2.4.0 | SPARK-24881 | 0a0f68bae6c0a1bf30184b1e9ac6bf3805bd7511#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.sizeOfNull | 2.4.0 | SPARK-24605 | d08f53dc61f662f5291f71bcbe1a7b9f531a34d2#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.replaceDatabricksSparkAvro.enabled | 2.4.0 | SPARK-25129 | ac0174e55af2e935d41545721e9f430c942b3a0c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.setopsPrecedence.enabled | 2.4.0 | SPARK-24966 | 73dd6cf9b558f9d752e1f3c13584344257ad7863#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.exponentLiteralAsDecimal.enabled | 3.0.0 | SPARK-29956 | 87ebfaf003fcd05a7f6d23b3ecd4661409ce5f2f#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.allowNegativeScaleOfDecimal | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.createHiveTableByDefault.enabled | 3.0.0 | SPARK-30098 | 58be82ad4b98fc17e821e916e69e77a6aa36209d#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.integralDivide.returnBigint | 3.0.0 | SPARK-25457 | 47d6e80a2e64823fabb596503fb6a6cc6f51f713#diff-9a6b543db706f1a90f790783d6930a13 | Exists in branch-3.0 branch, but the pom.xml file corresponding to the commit log is 2.5.0-SNAPSHOT
spark.sql.legacy.bucketedTableScan.outputOrdering | 3.0.0 | SPARK-28595 | 469423f33887a966aaa33eb75f5e7974a0a97beb#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.parser.havingWithoutGroupByAsWhere | 2.4.1 | SPARK-25708 | 3dba5d41f1a66ae5eb08404d103284110c45a351#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.json.allowEmptyString.enabled | 3.0.0 | SPARK-25040 | d3de7568f32e298442f07b0a28b2c906de72c797#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.createEmptyCollectionUsingStringType | 3.0.0 | SPARK-30790 | 8ab6ae3ede96adb093347470a5cbbf17fe8c04e9#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.allowUntypedScalaUDF | 3.0.0 | SPARK-26580 | bc30a07ce262840c99a752db4fbd3a423f652017#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.truncateTable.ignorePermissionAcl.enabled | 2.4.6 | SPARK-30312 | 830a4ec59b86253f18eb7dfd6ed0bbe0d7920e5b#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue | 3.0.0 | SPARK-26085 | ab2eafb3cdc7631452650c6cac03a92629255347#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.debug.maxToStringFields | 3.0.0 | SPARK-26066 | 81550b38e43fb20f89f529d2127575c71a54a538#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.maxPlanStringLength | 3.0.0 | SPARK-26103 | 812ad5546148d2194ab0e4230ee85b8f6a5be2fb#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.setCommandRejectsSparkCoreConfs | 3.0.0 | SPARK-26060 | 1ab3d3e474ce2e36d58aea8ad09fb61f0c73e5c5#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.datetime.java8API.enabled | 3.0.0 | SPARK-27008 | 52671d631d2a64ed1cfa0c6e01168908faf92df8#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.sources.binaryFile.maxLength | 3.0.0 | SPARK-27588 | 618d6bff71073c8c93501ab7392c3cc579730f0b#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.typeCoercion.datetimeToString.enabled | 3.0.0 | SPARK-27638 | 83d289eef492de8c7f3e5145f9bd75431608b500#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.defaultCatalog | 3.0.0 | SPARK-29753 | 942753a44beeae5f0142ceefa307e90cbc1234c5#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.catalog.$SESSION_CATALOG_NAME | 3.0.0 | SPARK-29412 | 9407fba0375675d6ee6461253f3b8230e8d67509#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.legacy.doLooseUpcast | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.ctePrecedencePolicy | 3.0.0 | SPARK-30829 | 00943be81afbca6be13e1e72b24536cd98a788d6#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.timeParserPolicy | 3.1.0 | SPARK-30668 | 7db0af578585ecaeee9fd23f8189292289b52a97#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.followThreeValuedLogicInArrayExists | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.maven.additionalRemoteRepositories | 3.0.0 | SPARK-29175 | 3d7359ad4202067b26a199657b6a3e1f38be0e4d#diff-9a6b543db706f1a90f790783d6930a13 |
spark.sql.legacy.fromDayTimeString.enabled | 3.0.0 | SPARK-29864 and SPARK-29920 | e933539cdd557297daf97ff5e532a3f098896979#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.notReserveProperties | 3.0.0 | SPARK-30812 | b76bc0b1b8b2abd00a84f805af90ca4c5925faaa#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.addSingleFileInAddFile | 3.0.0 | SPARK-30234 | 8a8d1fbb10af6da481f26831cd519ef46ccbce6c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.mssqlserver.numericMapping.enabled | 2.4.5 | SPARK-28152 | 69de7f31c37a7e0298e66cc814afc1b0aa948bbb#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.csv.filterPushdown.enabled | 3.0.0 | SPARK-30323 | 4e50f0291f032b4a5c0b46ed01fdef14e4cbb050#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.addPartitionInBatch.size | 3.0.0 | SPARK-29938 | 5ccbb38a71890b114c707279e7395d1f6284ebfd#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.timeParser.enabled | 3.0.0 | SPARK-30668 | 92f57237871400ab9d499e1174af22a867c01988#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.legacy.allowDuplicatedMapKeys | 3.0.0 | SPARK-25829 | 33329caa81827a245b84158b13234b88a4746e56#diff-9a6b543db706f1a90f790783d6930a13 |  

Supplemental configuration version information.

No

Exists UT

Closes #27829 from beliefer/add-version-to-sql-config-part-four.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 103 +++++++++---------
 1 file changed, 54 insertions(+), 49 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 378e900e6676f..420102b0f50a7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -449,6 +449,7 @@ object SQLConf {
         s"${SHUFFLE_PARTITIONS.key}. This configuration only has an effect when " +
         s"'${ADAPTIVE_EXECUTION_ENABLED.key}' and '${COALESCE_PARTITIONS_ENABLED.key}' " +
         "are both true.")
+      .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The initial number of partitions must be positive.")
       .createOptional
@@ -1890,6 +1891,7 @@ object SQLConf {
         "information. The values of options whose names that match this regex will be redacted " +
         "in the explain output. This redaction is applied on top of the global redaction " +
         s"configuration defined by ${SECRET_REDACTION_PATTERN.key}.")
+    .version("2.2.2")
     .regexConf
     .createWithDefault("(?i)url".r)
 
@@ -1899,20 +1901,20 @@ object SQLConf {
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands. " +
         "When this conf is not set, the value from `spark.redaction.string.regex` is used.")
-      .version("")
+      .version("2.3.0")
       .fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN)
 
   val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString")
     .doc("When this option is set to false and all inputs are binary, `functions.concat` returns " +
       "an output as binary. Otherwise, it returns as a string.")
-    .version("")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
   val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString")
     .doc("When this option is set to false and all inputs are binary, `elt` returns " +
       "an output as binary. Otherwise, it returns as a string.")
-    .version("")
+    .version("2.3.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -1923,7 +1925,7 @@ object SQLConf {
         "user-specified schema. If the validation fails, a runtime exception is thrown. " +
         "When this option is set to false, the partition column value will be converted to null " +
         "if it can not be casted to corresponding user-specified schema.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1931,25 +1933,25 @@ object SQLConf {
     buildConf("spark.sql.streaming.continuous.epochBacklogQueueSize")
       .doc("The max number of entries to be stored in queue to wait for late epochs. " +
         "If this parameter is exceeded by the size of the queue, stream will stop with an error.")
-      .version("")
+      .version("3.0.0")
       .intConf
       .createWithDefault(10000)
 
   val CONTINUOUS_STREAMING_EXECUTOR_QUEUE_SIZE =
     buildConf("spark.sql.streaming.continuous.executorQueueSize")
-      .internal()
-      .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
-        " buffer the results of a ContinuousDataReader.")
-      .version("")
-      .intConf
-      .createWithDefault(1024)
+    .internal()
+    .doc("The size (measured in number of rows) of the queue used in continuous execution to" +
+      " buffer the results of a ContinuousDataReader.")
+    .version("2.3.0")
+    .intConf
+    .createWithDefault(1024)
 
   val CONTINUOUS_STREAMING_EXECUTOR_POLL_INTERVAL_MS =
     buildConf("spark.sql.streaming.continuous.executorPollIntervalMs")
       .internal()
       .doc("The interval at which continuous execution readers will poll to check whether" +
         " the epoch has advanced on the driver.")
-      .version("")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
@@ -1958,14 +1960,14 @@ object SQLConf {
     .doc("A comma-separated list of data source short names or fully qualified data source " +
       "implementation class names for which Data Source V2 code path is disabled. These data " +
       "sources will fallback to Data Source V1 code path.")
-    .version("")
+    .version("3.0.0")
     .stringConf
     .createWithDefault("avro,csv,json,kafka,orc,parquet,text")
 
   val DISABLED_V2_STREAMING_WRITERS = buildConf("spark.sql.streaming.disabledV2Writers")
     .doc("A comma-separated list of fully qualified data source register class names for which" +
       " StreamWriteSupport is disabled. Writes to these sources will fall back to the V1 Sinks.")
-    .version("")
+    .version("2.3.1")
     .stringConf
     .createWithDefault("")
 
@@ -1976,7 +1978,7 @@ object SQLConf {
         "A comma-separated list of fully qualified data source register class names for which " +
           "MicroBatchReadSupport is disabled. Reads from these sources will fall back to the " +
           "V1 Sources.")
-      .version("")
+      .version("2.4.0")
       .stringConf
       .createWithDefault("")
 
@@ -1997,7 +1999,7 @@ object SQLConf {
         "(which takes precedence over this setting), e.g. " +
         "dataframe.write.option(\"partitionOverwriteMode\", \"dynamic\").save(path)."
       )
-      .version("")
+      .version("2.3.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(PartitionOverwriteMode.values.map(_.toString))
@@ -2022,7 +2024,7 @@ object SQLConf {
         "in type coercion, e.g. converting `double` to `int` or `decimal` to `double` is " +
         "not allowed."
       )
-      .version("")
+      .version("3.0.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
       .checkValues(StoreAssignmentPolicy.values.map(_.toString))
@@ -2033,7 +2035,7 @@ object SQLConf {
       "throw a runtime exception if an overflow occurs in any operation on integral/decimal " +
       "field. 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in " +
       "the SQL parser.")
-    .version("")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2046,7 +2048,7 @@ object SQLConf {
         "issues. Turn on this config to insert a local sort before actually doing repartition " +
         "to generate consistent repartition results. The performance of repartition() may go " +
         "down since we insert extra local sort before it.")
-      .version("")
+      .version("2.1.4")
       .booleanConf
       .createWithDefault(true)
 
@@ -2057,7 +2059,7 @@ object SQLConf {
         "satisfying a query. This optimization allows columnar file format readers to avoid " +
         "reading unnecessary nested column data. Currently Parquet and ORC are the " +
         "data sources that implement this optimization.")
-      .version("")
+      .version("2.4.1")
       .booleanConf
       .createWithDefault(true)
 
@@ -2068,6 +2070,7 @@ object SQLConf {
         "containing `dots` to data sources. Currently, Parquet implements both optimizations " +
         "while ORC only supports predicates for names containing `dots`. The other data sources" +
         "don't support this feature yet.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2077,7 +2080,7 @@ object SQLConf {
       .doc("Prune nested fields from object serialization operator which are unnecessary in " +
         "satisfying a query. This optimization allows object serializers to avoid " +
         "executing unnecessary nested expressions.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2088,7 +2091,7 @@ object SQLConf {
         "satisfying a query. Note that this optimization doesn't prune nested fields from " +
         "physical data source scanning. For pruning nested fields from scanning, please use " +
         "`spark.sql.optimizer.nestedSchemaPruning.enabled` config.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2098,7 +2101,7 @@ object SQLConf {
       .doc("In SQL queries with a SORT followed by a LIMIT like " +
           "'SELECT x FROM t ORDER BY y LIMIT m', if m is under this threshold, do a top-K sort" +
           " in memory, otherwise do a global sort which spills to disk if necessary.")
-      .version("")
+      .version("2.4.0")
       .intConf
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -2114,7 +2117,7 @@ object SQLConf {
     .internal()
     .doc("If it is set to true, column names of the requested schema are passed to CSV parser. " +
       "Other column values can be ignored during parsing even if they are malformed.")
-    .version("")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(true)
 
@@ -2125,7 +2128,7 @@ object SQLConf {
       "Jupyter, the HTML table (generated by _repr_html_) will be returned. For plain Python " +
       "REPL, the returned outputs are formatted like dataframe.show(). In SparkR, the returned " +
       "outputs are showed similar to R data.frame would.")
-    .version("")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2134,14 +2137,14 @@ object SQLConf {
       s"effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true. The valid range of this " +
       "config is from 0 to (Int.MaxValue - 1), so the invalid config like negative and " +
       "greater than (Int.MaxValue - 1) will be normalized to 0 and (Int.MaxValue - 1).")
-    .version("")
+    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
   val REPL_EAGER_EVAL_TRUNCATE = buildConf("spark.sql.repl.eagerEval.truncate")
     .doc("The max number of characters for each cell that is returned by eager evaluation. " +
       s"This only takes effect when ${REPL_EAGER_EVAL_ENABLED.key} is set to true.")
-    .version("")
+    .version("2.4.0")
     .intConf
     .createWithDefault(20)
 
@@ -2152,7 +2155,7 @@ object SQLConf {
         "by the fast hash aggregate product operator. The bit is not for actual value, " +
         "but the actual numBuckets is determined by loadFactor " +
         "(e.g: default bit value 16 , the actual numBuckets is ((1 << 16) / 0.5).")
-      .version("")
+      .version("2.4.0")
       .intConf
       .checkValue(bit => bit >= 10 && bit <= 30, "The bit value must be in [10, 30].")
       .createWithDefault(16)
@@ -2160,7 +2163,7 @@ object SQLConf {
   val AVRO_COMPRESSION_CODEC = buildConf("spark.sql.avro.compression.codec")
     .doc("Compression codec used in writing of AVRO files. Supported codecs: " +
       "uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.")
-    .version("")
+    .version("2.4.0")
     .stringConf
     .checkValues(Set("uncompressed", "deflate", "snappy", "bzip2", "xz"))
     .createWithDefault("snappy")
@@ -2198,7 +2201,7 @@ object SQLConf {
         "set operations are performed from left to right as they appear in the query. When set " +
         "to false and order of evaluation is not specified by parentheses, INTERSECT operations " +
         "are performed before any UNION, EXCEPT and MINUS operations.")
-      .version("")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2207,7 +2210,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, a literal with an exponent (e.g. 1E-30) would be parsed " +
         "as Decimal rather than Double.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2253,7 +2256,7 @@ object SQLConf {
       .internal()
       .doc("When set to true, the parser of JSON data source treats empty strings as null for " +
         "some data types such as `IntegerType`.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2291,7 +2294,7 @@ object SQLConf {
       .doc("When set to true, the key attribute resulted from running `Dataset.groupByKey` " +
         "for non-struct key type, will be named as `value`, following the behavior of Spark " +
         "version 2.4 and earlier.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2299,7 +2302,7 @@ object SQLConf {
     .doc("Maximum number of fields of sequence-like entries can be converted to strings " +
       "in debug output. Any elements beyond the limit will be dropped and replaced by a" +
       """ "... N more fields" placeholder.""")
-    .version("")
+    .version("3.0.0")
     .intConf
     .createWithDefault(25)
 
@@ -2308,7 +2311,7 @@ object SQLConf {
       "longer, further output will be truncated.  The default setting always generates a full " +
       "plan.  Set this to a lower value such as 8k if plan strings are taking up too much " +
       "memory or are causing OutOfMemory errors in the driver or UI processes.")
-    .version("")
+    .version("3.0.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(i => i >= 0 && i <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH, "Invalid " +
       "value for 'spark.sql.maxPlanStringLength'.  Length must be a valid string length " +
@@ -2320,7 +2323,7 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, SET command will fail when the key is registered as " +
         "a SparkConf entry.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2329,7 +2332,7 @@ object SQLConf {
       "java.time.LocalDate classes of Java 8 API are used as external types for " +
       "Catalyst's TimestampType and DateType. If it is set to false, java.sql.Timestamp " +
       "and java.sql.Date are used for the same purpose.")
-    .version("")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2337,7 +2340,7 @@ object SQLConf {
     .doc("The max length of a file that can be read by the binary file data source. " +
       "Spark will fail fast and not attempt to read the file if its length exceeds this value. " +
       "The theoretical max is Int.MaxValue, though VMs might implement a smaller max.")
-    .version("")
+    .version("3.0.0")
     .internal()
     .intConf
     .createWithDefault(Int.MaxValue)
@@ -2347,14 +2350,14 @@ object SQLConf {
       .internal()
       .doc("If it is set to true, date/timestamp will cast to string in binary comparisons " +
         "with String")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   val DEFAULT_CATALOG = buildConf("spark.sql.defaultCatalog")
     .doc("Name of the default catalog. This will be the current catalog if users have not " +
       "explicitly set the current catalog yet.")
-    .version("")
+    .version("3.0.0")
     .stringConf
     .createWithDefault(SESSION_CATALOG_NAME)
 
@@ -2366,7 +2369,7 @@ object SQLConf {
         s"be loaded by the $SESSION_CATALOG_NAME, this catalog must also return the table " +
         s"metadata. To delegate operations to the $SESSION_CATALOG_NAME, implementations can " +
         "extend 'CatalogExtension'.")
-      .version("")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -2379,6 +2382,7 @@ object SQLConf {
       "MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query " +
       "fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted " +
       "at last takes precedence.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(MapKeyDedupPolicy.values.map(_.toString))
@@ -2387,7 +2391,7 @@ object SQLConf {
   val LEGACY_LOOSE_UPCAST = buildConf("spark.sql.legacy.doLooseUpcast")
     .internal()
     .doc("When true, the upcast will be loose and allows string to atomic types.")
-    .version("")
+    .version("3.0.0")
     .booleanConf
     .createWithDefault(false)
 
@@ -2401,7 +2405,7 @@ object SQLConf {
       "CORRECTED, inner CTE definitions take precedence. The default value is EXCEPTION, " +
       "AnalysisException is thrown while name conflict is detected in nested CTE. This config " +
       "will be removed in future versions and CORRECTED will be the only behavior.")
-    .version("")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2414,6 +2418,7 @@ object SQLConf {
       "When set to CORRECTED, classes from java.time.* packages are used for the same purpose. " +
       "The default value is EXCEPTION, RuntimeException is thrown when we will get different " +
       "results.")
+    .version("3.0.0")
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
@@ -2423,7 +2428,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.followThreeValuedLogicInArrayExists")
       .internal()
       .doc("When true, the ArrayExists will follow the three-valued boolean logic.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -2432,7 +2437,7 @@ object SQLConf {
       .doc("A comma-delimited string config of the optional additional remote Maven mirror " +
         "repositories. This is only used for downloading Hive jars in IsolatedClientLoader " +
         "if the default Maven Central repo is unreachable.")
-      .version("")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(
         "https://maven-central.storage-download.googleapis.com/maven2/")
@@ -2445,7 +2450,7 @@ object SQLConf {
         "all interval units out of the specified range. If it is set to `false`, " +
         "`ParseException` is thrown if the input does not match to the pattern " +
         "defined by `from` and `to`.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2455,7 +2460,7 @@ object SQLConf {
       .doc("When true, all database and table properties are not reserved and available for " +
         "create/alter syntaxes. But please be aware that the reserved properties will be " +
         "silently removed.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2464,7 +2469,7 @@ object SQLConf {
       .internal()
       .doc("When true, only a single file can be added using ADD FILE. If false, then users " +
         "can add directory by passing directory path to ADD FILE.")
-      .version("")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -2472,7 +2477,7 @@ object SQLConf {
     buildConf("spark.sql.legacy.mssqlserver.numericMapping.enabled")
       .internal()
       .doc("When true, use legacy MySqlServer SMALLINT and REAL type mapping.")
-      .version("")
+      .version("2.4.5")
       .booleanConf
       .createWithDefault(false)
 

From 5cef257ce04e844cbc48aea9e60b7286cd2e83ee Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Sun, 8 Mar 2020 12:31:57 +0900
Subject: [PATCH 0446/1280] [SPARK-31002][CORE][DOC] Add version information to
 the configuration of Core
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add version information to the configuration of `Core`.
Note: Because `Core` has a lot of configuration items, I split the items into four PR. Other PR will follows this PR.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.resources.discoveryPlugin | 3.0.0 | SPARK-30689 | 742e35f1d48c2523dda2ce21d73b7ab5ade20582#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.resourcesFile | 3.0.0 | SPARK-27835 | 6748b486a9afe8370786efb64a8c9f3470c62dcf#diff-6bdad48cfc34314e89599655442ff210 |  
SparkLauncher.DRIVER_EXTRA_CLASSPATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.extraClassPath
SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.extraJavaOptions
SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.extraLibraryPath
spark.driver.userClassPathFirst | 1.3.0 | SPARK-2996 | 6a1e0f967286945db13d94aeb6ed19f0a347c236#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
spark.driver.cores | 1.3.0 | SPARK-1507 | 2be82b1e66cd188456bbf1e5abb13af04d1629d5#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
SparkLauncher.DRIVER_MEMORY | 1.1.1 | SPARK-3243 | c1ffa3e4cdfbd1f84b5c8d8de5d0fb958a19e211#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.driver.memory
spark.driver.memoryOverhead | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.dfsDir | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.layout | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.persistToDfs.enabled | 3.0.0 | SPARK-25118 | 5f11e8c4cb9a5db037ac239b8fcc97f3a746e772#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.log.allowErasureCoding | 3.0.0 | SPARK-29105 | 276aaaae8d404975f8701089e9f4dfecd16e0d9f#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.enabled | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.dir | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.compress | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.logBlockUpdates.enabled | 2.3.0 | SPARK-22050 | 1437e344ec0c29a44a19f4513986f5f184c44695#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.erasureCoding.enabled | 3.0.0 | SPARK-25855 | 35506dced739ef16136e9f3d5d48c638899d3cec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.testing | 1.0.1 | None | d4c8af87994acf3707027e6fab25363f51fd4615#diff-e4a5a68c15eed95d038acfed84b0b66a |  
spark.eventLog.buffer.kb | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.logStageExecutorMetrics | 3.0.0 | SPARK-30812 | 68d7edf9497bea2f73707d32ab55dd8e53088e7c#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.gcMetrics.youngGenerationGarbageCollectors | 3.0.0 | SPARK-25865 | e5c502c596563dce8eb58f86e42c1aea2c51ed17#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.gcMetrics.oldGenerationGarbageCollectors | 3.0.0 | SPARK-25865 | e5c502c596563dce8eb58f86e42c1aea2c51ed17#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.overwrite | 1.0.0 | SPARK-1132 | 79d07d66040f206708e14de393ab0b80020ed96a#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.eventLog.longForm.enabled | 2.4.0 | SPARK-23820 | 71f70130f1b2b4ec70595627f0a02a88e2c0e27d#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.rolling.enabled | 3.0.0 | SPARK-28869 | 100fc58da54e026cda87832a10e2d06eaeccdf87#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.rolling.maxFileSize | 3.0.0 | SPARK-28869 | 100fc58da54e026cda87832a10e2d06eaeccdf87#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.id | 1.2.0 | SPARK-3377 | 79e45c9323455a51f25ed9acd0edd8682b4bbb88#diff-364713d7776956cb8b0a771e9b62f82d |  
SparkLauncher.EXECUTOR_EXTRA_CLASSPATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.extraClassPath
spark.executor.heartbeat.dropZeroAccumulatorUpdates | 3.0.0 | SPARK-25449 | 9362c5cc273fdd09f9b3b512e2f6b64bcefc25ab#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.heartbeatInterval | 1.1.0 | SPARK-2099 | 8d338f64c4eda45d22ae33f61ef7928011cc2846#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.executor.heartbeat.maxFailures | 1.6.2 | SPARK-13522 | 86bf93e65481b8fe5d7532ca6d4cd29cafc9e9dd#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.executor.processTreeMetrics.enabled | 3.0.0 | SPARK-27324 | 387ce89a0631f1a4c6668b90ff2a7bbcf11919cd#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.metrics.pollingInterval | 3.0.0 | SPARK-26329 | 80ab19b9fd268adfc419457f12b99a5da7b6d1c7#diff-6bdad48cfc34314e89599655442ff210 |  
SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.extraJavaOptions
SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.extraLibraryPath
spark.executor.userClassPathFirst | 1.3.0 | SPARK-2996 | 6a1e0f967286945db13d94aeb6ed19f0a347c236#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
SparkLauncher.EXECUTOR_CORES | 1.0.0 | SPARK-1126 | 1617816090e7b20124a512a43860a21232ebf511#diff-4d2ab44195558d5a9d5f15b8803ef39d | spark.executor.cores
SparkLauncher.EXECUTOR_MEMORY | 0.7.0 | None | 696eec32c982ca516c506de33f383a173bcbd131#diff-4f50ad37deb6742ad45472636c9a870b | spark.executor.memory
spark.executor.memoryOverhead | 2.3.0 | SPARK-22646 | 3f4060c340d6bac412e8819c4388ccba226efcf3#diff-6bdad48cfc34314e89599655442ff210 |  
spark.cores.max | 0.6.0 | None | 0a472840030e4e7e84fe748f7bfa49f1ece599c5#diff-b6cc54c092b861f645c3cd69ea0f91e2 |  
spark.memory.offHeap.enabled | 1.6.0 | SPARK-12251 | 9870e5c7af87190167ca3845ede918671b9420ca#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.memory.offHeap.size | 1.6.0 | SPARK-12251 | 9870e5c7af87190167ca3845ede918671b9420ca#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.memory.storageFraction | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.memory.fraction | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.storage.safetyFraction | 1.1.0 | [SPARK-1777 | ecf30ee7e78ea59c462c54db0fde5328f997466c#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.unrollMemoryThreshold | 1.1.0 | SPARK-1777 | ecf30ee7e78ea59c462c54db0fde5328f997466c#diff-692a329b5a7fb4134c55d559457b94e4 |
spark.storage.replication.proactive | 2.2.0 | SPARK-15355 | fa7c582e9442b985a0493fb1dd15b3fb9b6031b4#diff-186864190089a718680accb51de5f0d4 |  
spark.storage.memoryMapThreshold | 0.9.2 | SPARK-1145 | 76339495153dd895667ad609815c887b2c8960ea#diff-abd96f2ae793cd6ea6aab5b96a3c1d7a |
spark.storage.replication.policy | 2.1.0 | SPARK-15353 | a26afd52198523dbd51dc94053424494638c7de5#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.replication.topologyMapper | 2.1.0 | SPARK-15353 | a26afd52198523dbd51dc94053424494638c7de5#diff-186864190089a718680accb51de5f0d4 |
spark.storage.cachedPeersTtl | 1.1.1 | SPARK-3495 and SPARK-3496 | be0cc9952d6c8b4cfe9ff10a761e0677cba64489#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.maxReplicationFailures | 1.1.1 | SPARK-3495 and SPARK-3496 | be0cc9952d6c8b4cfe9ff10a761e0677cba64489#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.storage.replication.topologyFile | 2.1.0 | SPARK-15353 | a26afd52198523dbd51dc94053424494638c7de5#diff-e550ce522c12a31d805a7d0f41e802af |  
spark.storage.exceptionOnPinLeak | 1.6.2 | SPARK-13566 | ab006523b840b1d2dbf3f5ff0a238558e7665a1e#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.storage.blockManagerTimeoutIntervalMs | 0.7.3 | None | 9085ebf3750c7d9bb7c6b5f6b4bdc5b807af93c2#diff-76170a9c8f67b542bc58240a0a12fe08 |  
spark.storage.blockManagerSlaveTimeoutMs | 0.7.0 | None | 97434f49b8c029e9b78c91ec5f58557cd1b5c943#diff-2ce6374aac24d70c69182b067216e684 |
spark.storage.cleanupFilesAfterExecutorExit | 2.4.0 | SPARK-24340 | 8ef167a5f9ba8a79bb7ca98a9844fe9cfcfea060#diff-916ca56b663f178f302c265b7ef38499 |  
spark.diskStore.subDirectories | 0.6.0 | None | 815d6bd69a0c1ba0e94fc0785f5c3619b37f19c5#diff-e8b73c5b81c403a5e5d581f97624c510 |  
spark.block.failures.beforeLocationRefresh | 2.0.0 | SPARK-13328 | ff776b2fc1cd4c571fd542dbf807e6fa3373cb34#diff-2b643ea78c1add0381754b1f47eec132 |  

Supplemental configuration version information.

No

Exists UT

Closes #27847 from beliefer/add-version-to-core-config-part-one.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/package.scala       | 109 +++++++++++++++---
 1 file changed, 95 insertions(+), 14 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 1308a460a2cc8..f3195d978ec6d 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -45,6 +45,7 @@ package object config {
         "custom implementation. Spark will try each class specified until one of them " +
         "returns the resource information for that resource. It tries the discovery " +
         "script last if none of the plugins return information for that resource.")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
@@ -55,89 +56,113 @@ package object config {
       .doc("Path to a file containing the resources allocated to the driver. " +
         "The file should be formatted as a JSON array of ResourceAllocation objects. " +
         "Only used internally in standalone mode.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_CLASSPATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val DRIVER_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.DRIVER_DEFAULT_JAVA_OPTIONS)
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val DRIVER_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.driver.userClassPathFirst").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.driver.userClassPathFirst")
+      .version("1.3.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DRIVER_CORES = ConfigBuilder("spark.driver.cores")
     .doc("Number of cores to use for the driver process, only in cluster mode.")
+    .version("1.3.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val DRIVER_MEMORY = ConfigBuilder(SparkLauncher.DRIVER_MEMORY)
     .doc("Amount of memory to use for the driver process, in MiB unless otherwise specified.")
+    .version("1.1.1")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val DRIVER_MEMORY_OVERHEAD = ConfigBuilder("spark.driver.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per driver in cluster mode, " +
       "in MiB unless otherwise specified.")
+    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
   private[spark] val DRIVER_LOG_DFS_DIR =
-    ConfigBuilder("spark.driver.log.dfsDir").stringConf.createOptional
+    ConfigBuilder("spark.driver.log.dfsDir").version("3.0.0").stringConf.createOptional
 
   private[spark] val DRIVER_LOG_LAYOUT =
     ConfigBuilder("spark.driver.log.layout")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
   private[spark] val DRIVER_LOG_PERSISTTODFS =
     ConfigBuilder("spark.driver.log.persistToDfs.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DRIVER_LOG_ALLOW_EC =
     ConfigBuilder("spark.driver.log.allowErasureCoding")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLED = ConfigBuilder("spark.eventLog.enabled")
+    .version("1.0.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val EVENT_LOG_DIR = ConfigBuilder("spark.eventLog.dir")
+    .version("1.0.0")
     .stringConf
     .createWithDefault(EventLoggingListener.DEFAULT_LOG_DIR)
 
   private[spark] val EVENT_LOG_COMPRESS =
     ConfigBuilder("spark.eventLog.compress")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_BLOCK_UPDATES =
     ConfigBuilder("spark.eventLog.logBlockUpdates.enabled")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ALLOW_EC =
     ConfigBuilder("spark.eventLog.erasureCoding.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_TESTING =
     ConfigBuilder("spark.eventLog.testing")
       .internal()
+      .version("1.0.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EVENT_LOG_OUTPUT_BUFFER_SIZE = ConfigBuilder("spark.eventLog.buffer.kb")
     .doc("Buffer size to use when writing to output streams, in KiB unless otherwise specified.")
+    .version("1.0.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("100k")
 
@@ -145,6 +170,7 @@ package object config {
     ConfigBuilder("spark.eventLog.logStageExecutorMetrics")
       .doc("Whether to write per-stage peaks of executor metrics (for each executor) " +
         "to the event log.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -153,6 +179,7 @@ package object config {
       .doc("Names of supported young generation garbage collector. A name usually is " +
         " the return of GarbageCollectorMXBean.getName. The built-in young generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
@@ -162,20 +189,28 @@ package object config {
       .doc("Names of supported old generation garbage collector. A name usually is " +
         "the return of GarbageCollectorMXBean.getName. The built-in old generation garbage " +
         s"collectors are ${GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS}")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(GarbageCollectionMetrics.OLD_GENERATION_BUILTIN_GARBAGE_COLLECTORS)
 
   private[spark] val EVENT_LOG_OVERWRITE =
-    ConfigBuilder("spark.eventLog.overwrite").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.eventLog.overwrite")
+      .version("1.0.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EVENT_LOG_CALLSITE_LONG_FORM =
-    ConfigBuilder("spark.eventLog.longForm.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.eventLog.longForm.enabled")
+      .version("2.4.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EVENT_LOG_ENABLE_ROLLING =
     ConfigBuilder("spark.eventLog.rolling.enabled")
       .doc("Whether rolling over event log files is enabled. If set to true, it cuts down " +
         "each event log file to the configured size.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -183,35 +218,46 @@ package object config {
     ConfigBuilder("spark.eventLog.rolling.maxFileSize")
       .doc(s"When ${EVENT_LOG_ENABLE_ROLLING.key}=true, specifies the max size of event log file" +
         " to be rolled over.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ >= ByteUnit.MiB.toBytes(10), "Max file size of event log should be " +
         "configured to be at least 10 MiB.")
       .createWithDefaultString("128m")
 
   private[spark] val EXECUTOR_ID =
-    ConfigBuilder("spark.executor.id").stringConf.createOptional
+    ConfigBuilder("spark.executor.id").version("1.2.0").stringConf.createOptional
 
   private[spark] val EXECUTOR_CLASS_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_CLASSPATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val EXECUTOR_HEARTBEAT_DROP_ZERO_ACCUMULATOR_UPDATES =
     ConfigBuilder("spark.executor.heartbeat.dropZeroAccumulatorUpdates")
       .internal()
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val EXECUTOR_HEARTBEAT_INTERVAL =
     ConfigBuilder("spark.executor.heartbeatInterval")
+      .version("1.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val EXECUTOR_HEARTBEAT_MAX_FAILURES =
-    ConfigBuilder("spark.executor.heartbeat.maxFailures").internal().intConf.createWithDefault(60)
+    ConfigBuilder("spark.executor.heartbeat.maxFailures")
+      .internal()
+      .version("1.6.2")
+      .intConf
+      .createWithDefault(60)
 
   private[spark] val EXECUTOR_PROCESS_TREE_METRICS_ENABLED =
     ConfigBuilder("spark.executor.processTreeMetrics.enabled")
       .doc("Whether to collect process tree metrics (from the /proc filesystem) when collecting " +
         "executor metrics.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -220,33 +266,44 @@ package object config {
       .doc("How often to collect executor metrics (in milliseconds). " +
         "If 0, the polling is done on executor heartbeats. " +
         "If positive, the polling is done at this interval.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("0")
 
   private[spark] val EXECUTOR_JAVA_OPTIONS =
     ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_JAVA_OPTIONS)
       .withPrepended(SparkLauncher.EXECUTOR_DEFAULT_JAVA_OPTIONS)
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val EXECUTOR_LIBRARY_PATH =
-    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH).stringConf.createOptional
+    ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH)
+      .version("1.0.0")
+      .stringConf
+      .createOptional
 
   private[spark] val EXECUTOR_USER_CLASS_PATH_FIRST =
-    ConfigBuilder("spark.executor.userClassPathFirst").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.executor.userClassPathFirst")
+      .version("1.3.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val EXECUTOR_CORES = ConfigBuilder(SparkLauncher.EXECUTOR_CORES)
+    .version("1.0.0")
     .intConf
     .createWithDefault(1)
 
   private[spark] val EXECUTOR_MEMORY = ConfigBuilder(SparkLauncher.EXECUTOR_MEMORY)
     .doc("Amount of memory to use per executor process, in MiB unless otherwise specified.")
+    .version("0.7.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("1g")
 
   private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead")
     .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " +
       "in MiB unless otherwise specified.")
+    .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
 
@@ -256,12 +313,14 @@ package object config {
       "the cluster (not from each machine). If not set, the default will be " +
       "`spark.deploy.defaultCores` on Spark's standalone cluster manager, or infinite " +
       "(all available cores) on Mesos.")
+    .version("0.6.0")
     .intConf
     .createOptional
 
   private[spark] val MEMORY_OFFHEAP_ENABLED = ConfigBuilder("spark.memory.offHeap.enabled")
     .doc("If true, Spark will attempt to use off-heap memory for certain operations. " +
       "If off-heap memory use is enabled, then spark.memory.offHeap.size must be positive.")
+    .version("1.6.0")
     .withAlternative("spark.unsafe.offHeap")
     .booleanConf
     .createWithDefault(false)
@@ -272,6 +331,7 @@ package object config {
       "This setting has no impact on heap memory usage, so if your executors' total memory " +
       "consumption must fit within some hard limit then be sure to shrink your JVM heap size " +
       "accordingly. This must be set to a positive value when spark.memory.offHeap.enabled=true.")
+    .version("1.6.0")
     .bytesConf(ByteUnit.BYTE)
     .checkValue(_ >= 0, "The off-heap memory size must not be negative")
     .createWithDefault(0)
@@ -281,6 +341,7 @@ package object config {
       "size of the region set aside by spark.memory.fraction. The higher this is, the " +
       "less working memory may be available to execution and tasks may spill to disk more " +
       "often. Leaving this at the default value is recommended. ")
+    .version("1.6.0")
     .doubleConf
     .checkValue(v => v >= 0.0 && v < 1.0, "Storage fraction must be in [0,1)")
     .createWithDefault(0.5)
@@ -291,16 +352,19 @@ package object config {
       "The purpose of this config is to set aside memory for internal metadata, " +
       "user data structures, and imprecise size estimation in the case of sparse, " +
       "unusually large records. Leaving this at the default value is recommended.  ")
+    .version("1.6.0")
     .doubleConf
     .createWithDefault(0.6)
 
   private[spark] val STORAGE_SAFETY_FRACTION = ConfigBuilder("spark.storage.safetyFraction")
+    .version("1.1.0")
     .doubleConf
     .createWithDefault(0.9)
 
   private[spark] val STORAGE_UNROLL_MEMORY_THRESHOLD =
     ConfigBuilder("spark.storage.unrollMemoryThreshold")
       .doc("Initial memory to request before unrolling any block")
+      .version("1.1.0")
       .longConf
       .createWithDefault(1024 * 1024)
 
@@ -310,6 +374,7 @@ package object config {
         "Cached RDD block replicas lost due to executor failures are replenished " +
         "if there are any existing available replicas. This tries to " +
         "get the replication level of the block to the initial number")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -320,41 +385,54 @@ package object config {
         "This prevents Spark from memory mapping very small blocks. " +
         "In general, memory mapping has high overhead for blocks close to or below " +
         "the page size of the operating system.")
+      .version("0.9.2")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("2m")
 
   private[spark] val STORAGE_REPLICATION_POLICY =
     ConfigBuilder("spark.storage.replication.policy")
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[RandomBlockReplicationPolicy].getName)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_MAPPER =
     ConfigBuilder("spark.storage.replication.topologyMapper")
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString(classOf[DefaultTopologyMapper].getName)
 
   private[spark] val STORAGE_CACHED_PEERS_TTL = ConfigBuilder("spark.storage.cachedPeersTtl")
-    .intConf.createWithDefault(60 * 1000)
+    .version("1.1.1")
+    .intConf
+    .createWithDefault(60 * 1000)
 
   private[spark] val STORAGE_MAX_REPLICATION_FAILURE =
     ConfigBuilder("spark.storage.maxReplicationFailures")
-      .intConf.createWithDefault(1)
+      .version("1.1.1")
+      .intConf
+      .createWithDefault(1)
 
   private[spark] val STORAGE_REPLICATION_TOPOLOGY_FILE =
-    ConfigBuilder("spark.storage.replication.topologyFile").stringConf.createOptional
+    ConfigBuilder("spark.storage.replication.topologyFile")
+      .version("2.1.0")
+      .stringConf
+      .createOptional
 
   private[spark] val STORAGE_EXCEPTION_PIN_LEAK =
     ConfigBuilder("spark.storage.exceptionOnPinLeak")
+      .version("1.6.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val STORAGE_BLOCKMANAGER_TIMEOUTINTERVAL =
     ConfigBuilder("spark.storage.blockManagerTimeoutIntervalMs")
+      .version("0.7.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("60s")
 
   private[spark] val STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT =
     ConfigBuilder("spark.storage.blockManagerSlaveTimeoutMs")
+      .version("0.7.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString(Network.NETWORK_TIMEOUT.defaultValueString)
 
@@ -362,6 +440,7 @@ package object config {
     ConfigBuilder("spark.storage.cleanupFilesAfterExecutorExit")
       .doc("Whether or not cleanup the files not served by the external shuffle service " +
         "on executor exits.")
+      .version("2.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -369,6 +448,7 @@ package object config {
     ConfigBuilder("spark.diskStore.subDirectories")
       .doc("Number of subdirectories inside each path listed in spark.local.dir for " +
         "hashing Block files into.")
+      .version("0.6.0")
       .intConf
       .checkValue(_ > 0, "The number of subdirectories must be positive.")
       .createWithDefault(64)
@@ -377,6 +457,7 @@ package object config {
     ConfigBuilder("spark.block.failures.beforeLocationRefresh")
       .doc("Max number of failures before this block manager refreshes " +
         "the block locations from the driver.")
+      .version("2.0.0")
       .intConf
       .createWithDefault(5)
 

From b889e66a6cbc8642485fa4bda8a1e6e34ba1c621 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 12 Mar 2020 09:52:20 +0900
Subject: [PATCH 0447/1280] [SPARK-31002][CORE][DOC][FOLLOWUP] Add version
 information to the configuration of Core
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR follows up https://github.com/apache/spark/pull/27847.
I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.yarn.isPython | 1.5.0 | SPARK-5479 | 38112905bc3b33f2ae75274afba1c30e116f6e46#diff-4d2ab44195558d5a9d5f15b8803ef39d |
spark.task.cpus | 0.5.0 | None | e5c4cd8a5e188592f8786a265c0cd073c69ac886#diff-391214d132a0fb4478f4f9c2313d8966 |  
spark.dynamicAllocation.enabled | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.dynamicAllocation.testing | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.dynamicAllocation.minExecutors | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.dynamicAllocation.initialExecutors | 1.3.0 | SPARK-4585 | b2047b55c5fc85de6b63276d8ab9610d2496e08b#diff-b096353602813e47074ace09a3890d56 |  
spark.dynamicAllocation.maxExecutors | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.dynamicAllocation.executorAllocationRatio | 2.4.0 | SPARK-22683 | 55c4ca88a3b093ee197a8689631be8d1fac1f10f#diff-6bdad48cfc34314e89599655442ff210 |  
spark.dynamicAllocation.cachedExecutorIdleTimeout | 1.4.0 | SPARK-7955 | 6faaf15ba311bc3a79aae40a6c9c4befabb6889f#diff-b096353602813e47074ace09a3890d56 |  
spark.dynamicAllocation.executorIdleTimeout | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.dynamicAllocation.shuffleTracking.enabled | 3.0.0 | SPARK-27963 | 2ddeff97d7329942a98ef363991eeabc3fa71a76#diff-6bdad48cfc34314e89599655442ff210 |  
spark.dynamicAllocation.shuffleTimeout | 3.0.0 | SPARK-27963 | 2ddeff97d7329942a98ef363991eeabc3fa71a76#diff-6bdad48cfc34314e89599655442ff210 |  
spark.dynamicAllocation.schedulerBacklogTimeout | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.dynamicAllocation.sustainedSchedulerBacklogTimeout | 1.2.0 | SPARK-3795 | 8d59b37b02eb36f37bcefafb952519d7dca744ad#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.locality.wait | 0.5.0 | None | e5c4cd8a5e188592f8786a265c0cd073c69ac886#diff-391214d132a0fb4478f4f9c2313d8966 |  
spark.shuffle.service.enabled | 1.2.0 | SPARK-3796 | f55218aeb1e9d638df6229b36a59a15ce5363482#diff-2b643ea78c1add0381754b1f47eec132 |  
Constants.SHUFFLE_SERVICE_FETCH_RDD_ENABLED | 3.0.0 | SPARK-27677 | e9f3f62b2c0f521f3cc23fef381fc6754853ad4f#diff-6bdad48cfc34314e89599655442ff210 | spark.shuffle.service.fetch.rdd.enabled
spark.shuffle.service.db.enabled | 3.0.0 | SPARK-26288 | 8b0aa59218c209d39cbba5959302d8668b885cf6#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.service.port | 1.2.0 | SPARK-3796 | f55218aeb1e9d638df6229b36a59a15ce5363482#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.kerberos.keytab | 3.0.0 | SPARK-25372 | 51540c2fa677658be954c820bc18ba748e4c8583#diff-6bdad48cfc34314e89599655442ff210 |
spark.kerberos.principal | 3.0.0 | SPARK-25372 | 51540c2fa677658be954c820bc18ba748e4c8583#diff-6bdad48cfc34314e89599655442ff210 |
spark.kerberos.relogin.period | 3.0.0 | SPARK-23781 | 68dde3481ea458b0b8deeec2f99233c2d4c1e056#diff-6bdad48cfc34314e89599655442ff210 |
spark.kerberos.renewal.credentials | 3.0.0 | SPARK-26595 | 2a67dbfbd341af166b1c85904875f26a6dea5ba8#diff-6bdad48cfc34314e89599655442ff210 |  
spark.kerberos.access.hadoopFileSystems | 3.0.0 | SPARK-26766 | d0443a74d185ec72b747fa39994fa9a40ce974cf#diff-6bdad48cfc34314e89599655442ff210 |  
spark.executor.instances | 1.0.0 | SPARK-1126 | 1617816090e7b20124a512a43860a21232ebf511#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
spark.yarn.dist.pyFiles | 2.2.1 | SPARK-21714 | d10c9dc3f631a26dbbbd8f5c601ca2001a5d7c80#diff-6bdad48cfc34314e89599655442ff210 |  
spark.task.maxDirectResultSize | 2.0.0 | SPARK-13830 | 2ef4c5963bff3574fe17e669d703b25ddd064e5d#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.task.maxFailures | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-264da78fe625d594eae59d1adabc8ae9 |  
spark.task.reaper.enabled | 2.0.3 | SPARK-18761 | 678d91c1d2283d9965a39656af9d383bad093ba8#diff-5a0de266c82b95adb47d9bca714e1f1b |
spark.task.reaper.killTimeout | 2.0.3 | SPARK-18761 | 678d91c1d2283d9965a39656af9d383bad093ba8#diff-5a0de266c82b95adb47d9bca714e1f1b |
spark.task.reaper.pollingInterval | 2.0.3 | SPARK-18761 | 678d91c1d2283d9965a39656af9d383bad093ba8#diff-5a0de266c82b95adb47d9bca714e1f1b |
spark.task.reaper.threadDump | 2.0.3 | SPARK-18761 | 678d91c1d2283d9965a39656af9d383bad093ba8#diff-5a0de266c82b95adb47d9bca714e1f1b |
spark.blacklist.enabled | 2.1.0 | SPARK-17675 | 9ce7d3e542e786c62f047c13f3001e178f76e06a#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.task.maxTaskAttemptsPerExecutor | 2.1.0 | SPARK-17675 | 9ce7d3e542e786c62f047c13f3001e178f76e06a#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.task.maxTaskAttemptsPerNode | 2.1.0 | SPARK-17675 | 9ce7d3e542e786c62f047c13f3001e178f76e06a#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.application.maxFailedTasksPerExecutor | 2.2.0 | SPARK-8425 | 93cdb8a7d0f124b4db069fd8242207c82e263c52#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.stage.maxFailedTasksPerExecutor | 2.1.0 | SPARK-17675 | 9ce7d3e542e786c62f047c13f3001e178f76e06a#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.application.maxFailedExecutorsPerNode | 2.2.0 | SPARK-8425 | 93cdb8a7d0f124b4db069fd8242207c82e263c52#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.stage.maxFailedExecutorsPerNode | 2.1.0 | SPARK-17675 | 9ce7d3e542e786c62f047c13f3001e178f76e06a#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.timeout | 2.1.0 | SPARK-17675 | 9ce7d3e542e786c62f047c13f3001e178f76e06a#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blacklist.killBlacklistedExecutors | 2.2.0 | SPARK-16554 | 6287c94f08200d548df5cc0a401b73b84f9968c4#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.executorTaskBlacklistTime | 1.0.0 | None | ab747d39ddc7c8a314ed2fb26548fc5652af0d74#diff-bad3987c83bd22d46416d3dd9d208e76 |
spark.blacklist.application.fetchFailure.enabled | 2.3.0 | SPARK-13669 and SPARK-20898 | 9e50a1d37a4cf0c34e20a7c1a910ceaff41535a2#diff-6bdad48cfc34314e89599655442ff210 |  
spark.files.fetchFailure.unRegisterOutputOnHost | 2.3.0 | SPARK-19753 | dccc0aa3cf957c8eceac598ac81ac82f03b52105#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.listenerbus.eventqueue.capacity | 2.3.0 | SPARK-20887 | 629f38e171409da614fd635bd8dd951b7fde17a4#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.listenerbus.metrics.maxListenerClassesTimed | 2.3.0 | SPARK-20863 | 2a23cdd078a7409d0bb92cf27718995766c41b1d#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.listenerbus.logSlowEvent | 3.0.0 | SPARK-30812 | 68d7edf9497bea2f73707d32ab55dd8e53088e7c#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.listenerbus.logSlowEvent.threshold | 3.0.0 | SPARK-29001 | 0346afa8fc348aa1b3f5110df747a64e3b2da388#diff-6bdad48cfc34314e89599655442ff210 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Exists UT

Closes #27852 from beliefer/add-version-to-core-config-part-two.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/package.scala       | 84 ++++++++++++++++---
 docs/configuration.md                         | 31 +++++++
 2 files changed, 104 insertions(+), 11 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index f3195d978ec6d..12ae56c3be76e 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -461,68 +461,98 @@ package object config {
       .intConf
       .createWithDefault(5)
 
-  private[spark] val IS_PYTHON_APP = ConfigBuilder("spark.yarn.isPython").internal()
-    .booleanConf.createWithDefault(false)
+  private[spark] val IS_PYTHON_APP =
+    ConfigBuilder("spark.yarn.isPython")
+      .internal()
+      .version("1.5.0")
+      .booleanConf
+      .createWithDefault(false)
 
-  private[spark] val CPUS_PER_TASK = ConfigBuilder("spark.task.cpus").intConf.createWithDefault(1)
+  private[spark] val CPUS_PER_TASK =
+    ConfigBuilder("spark.task.cpus").version("0.5.0").intConf.createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_ENABLED =
-    ConfigBuilder("spark.dynamicAllocation.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.enabled")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_TESTING =
-    ConfigBuilder("spark.dynamicAllocation.testing").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.dynamicAllocation.testing")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_MIN_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.minExecutors").intConf.createWithDefault(0)
+    ConfigBuilder("spark.dynamicAllocation.minExecutors")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(0)
 
   private[spark] val DYN_ALLOCATION_INITIAL_EXECUTORS =
     ConfigBuilder("spark.dynamicAllocation.initialExecutors")
+      .version("1.3.0")
       .fallbackConf(DYN_ALLOCATION_MIN_EXECUTORS)
 
   private[spark] val DYN_ALLOCATION_MAX_EXECUTORS =
-    ConfigBuilder("spark.dynamicAllocation.maxExecutors").intConf.createWithDefault(Int.MaxValue)
+    ConfigBuilder("spark.dynamicAllocation.maxExecutors")
+      .version("1.2.0")
+      .intConf
+      .createWithDefault(Int.MaxValue)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_ALLOCATION_RATIO =
     ConfigBuilder("spark.dynamicAllocation.executorAllocationRatio")
-      .doubleConf.createWithDefault(1.0)
+      .version("2.4.0")
+      .doubleConf
+      .createWithDefault(1.0)
 
   private[spark] val DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.cachedExecutorIdleTimeout")
+      .version("1.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.executorIdleTimeout")
+      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(60)
 
   private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING =
     ConfigBuilder("spark.dynamicAllocation.shuffleTracking.enabled")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val DYN_ALLOCATION_SHUFFLE_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.shuffleTimeout")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(Long.MaxValue)
 
   private[spark] val DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.schedulerBacklogTimeout")
+      .version("1.2.0")
       .timeConf(TimeUnit.SECONDS).createWithDefault(1)
 
   private[spark] val DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT =
     ConfigBuilder("spark.dynamicAllocation.sustainedSchedulerBacklogTimeout")
+      .version("1.2.0")
       .fallbackConf(DYN_ALLOCATION_SCHEDULER_BACKLOG_TIMEOUT)
 
   private[spark] val LOCALITY_WAIT = ConfigBuilder("spark.locality.wait")
+    .version("0.5.0")
     .timeConf(TimeUnit.MILLISECONDS)
     .createWithDefaultString("3s")
 
   private[spark] val SHUFFLE_SERVICE_ENABLED =
-    ConfigBuilder("spark.shuffle.service.enabled").booleanConf.createWithDefault(false)
+    ConfigBuilder("spark.shuffle.service.enabled")
+      .version("1.2.0")
+      .booleanConf
+      .createWithDefault(false)
 
   private[spark] val SHUFFLE_SERVICE_FETCH_RDD_ENABLED =
     ConfigBuilder(Constants.SHUFFLE_SERVICE_FETCH_RDD_ENABLED)
@@ -530,6 +560,7 @@ package object config {
         "In case of dynamic allocation if this feature is enabled executors having only disk " +
         "persisted blocks are considered idle after " +
         "'spark.dynamicAllocation.executorIdleTimeout' and will be released accordingly.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -537,21 +568,26 @@ package object config {
     ConfigBuilder("spark.shuffle.service.db.enabled")
       .doc("Whether to use db in ExternalShuffleService. Note that this only affects " +
         "standalone mode.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_SERVICE_PORT =
-    ConfigBuilder("spark.shuffle.service.port").intConf.createWithDefault(7337)
+    ConfigBuilder("spark.shuffle.service.port").version("1.2.0").intConf.createWithDefault(7337)
 
   private[spark] val KEYTAB = ConfigBuilder("spark.kerberos.keytab")
     .doc("Location of user's keytab.")
+    .version("3.0.0")
     .stringConf.createOptional
 
   private[spark] val PRINCIPAL = ConfigBuilder("spark.kerberos.principal")
     .doc("Name of the Kerberos principal.")
-    .stringConf.createOptional
+    .version("3.0.0")
+    .stringConf
+    .createOptional
 
   private[spark] val KERBEROS_RELOGIN_PERIOD = ConfigBuilder("spark.kerberos.relogin.period")
+    .version("3.0.0")
     .timeConf(TimeUnit.SECONDS)
     .createWithDefaultString("1m")
 
@@ -561,6 +597,7 @@ package object config {
         "Which credentials to use when renewing delegation tokens for executors. Can be either " +
         "'keytab', the default, which requires a keytab to be provided, or 'ccache', which uses " +
         "the local credentials cache.")
+      .version("3.0.0")
       .stringConf
       .checkValues(Set("keytab", "ccache"))
       .createWithDefault("keytab")
@@ -569,104 +606,124 @@ package object config {
     ConfigBuilder("spark.kerberos.access.hadoopFileSystems")
     .doc("Extra Hadoop filesystem URLs for which to request delegation tokens. The filesystem " +
       "that hosts fs.defaultFS does not need to be listed here.")
+    .version("3.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances")
+    .version("1.0.0")
     .intConf
     .createOptional
 
   private[spark] val PY_FILES = ConfigBuilder("spark.yarn.dist.pyFiles")
     .internal()
+    .version("2.2.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val TASK_MAX_DIRECT_RESULT_SIZE =
     ConfigBuilder("spark.task.maxDirectResultSize")
+      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1L << 20)
 
   private[spark] val TASK_MAX_FAILURES =
     ConfigBuilder("spark.task.maxFailures")
+      .version("0.8.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val TASK_REAPER_ENABLED =
     ConfigBuilder("spark.task.reaper.enabled")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val TASK_REAPER_KILL_TIMEOUT =
     ConfigBuilder("spark.task.reaper.killTimeout")
+      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(-1)
 
   private[spark] val TASK_REAPER_POLLING_INTERVAL =
     ConfigBuilder("spark.task.reaper.pollingInterval")
+      .version("2.0.3")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10s")
 
   private[spark] val TASK_REAPER_THREAD_DUMP =
     ConfigBuilder("spark.task.reaper.threadDump")
+      .version("2.0.3")
       .booleanConf
       .createWithDefault(true)
 
   // Blacklist confs
   private[spark] val BLACKLIST_ENABLED =
     ConfigBuilder("spark.blacklist.enabled")
+      .version("2.1.0")
       .booleanConf
       .createOptional
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_EXECUTOR =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerExecutor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(1)
 
   private[spark] val MAX_TASK_ATTEMPTS_PER_NODE =
     ConfigBuilder("spark.blacklist.task.maxTaskAttemptsPerNode")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC =
     ConfigBuilder("spark.blacklist.application.maxFailedTasksPerExecutor")
+      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILURES_PER_EXEC_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedTasksPerExecutor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE =
     ConfigBuilder("spark.blacklist.application.maxFailedExecutorsPerNode")
+      .version("2.2.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val MAX_FAILED_EXEC_PER_NODE_STAGE =
     ConfigBuilder("spark.blacklist.stage.maxFailedExecutorsPerNode")
+      .version("2.1.0")
       .intConf
       .createWithDefault(2)
 
   private[spark] val BLACKLIST_TIMEOUT_CONF =
     ConfigBuilder("spark.blacklist.timeout")
+      .version("2.1.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_KILL_ENABLED =
     ConfigBuilder("spark.blacklist.killBlacklistedExecutors")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val BLACKLIST_LEGACY_TIMEOUT_CONF =
     ConfigBuilder("spark.scheduler.executorTaskBlacklistTime")
       .internal()
+      .version("1.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val BLACKLIST_FETCH_FAILURE_ENABLED =
     ConfigBuilder("spark.blacklist.application.fetchFailure.enabled")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
   // End blacklist confs
@@ -676,6 +733,7 @@ package object config {
       .doc("Whether to un-register all the outputs on the host in condition that we receive " +
         " a FetchFailure. This is set default to false, which means, we only un-register the " +
         " outputs related to the exact executor(instead of the host) on a FetchFailure.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -685,6 +743,7 @@ package object config {
         "an event queue using capacity specified by `spark.scheduler.listenerbus" +
         ".eventqueue.queueName.capacity` first. If it's not configured, Spark will " +
         "use the default capacity specified by this config.")
+      .version("2.3.0")
       .intConf
       .checkValue(_ > 0, "The capacity of listener bus event queue must be positive")
       .createWithDefault(10000)
@@ -692,6 +751,7 @@ package object config {
   private[spark] val LISTENER_BUS_METRICS_MAX_LISTENER_CLASSES_TIMED =
     ConfigBuilder("spark.scheduler.listenerbus.metrics.maxListenerClassesTimed")
       .internal()
+      .version("2.3.0")
       .intConf
       .createWithDefault(128)
 
@@ -701,6 +761,7 @@ package object config {
       .doc("When enabled, log the event that takes too much time to process. This helps us " +
         "discover the event types that cause performance bottlenecks. The time threshold is " +
         "controlled by spark.scheduler.listenerbus.logSlowEvent.threshold.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -709,6 +770,7 @@ package object config {
       .internal()
       .doc("The time threshold of whether a event is considered to be taking too much time to " +
         s"process. Log the event if ${LISTENER_BUS_LOG_SLOW_EVENT_ENABLED.key} is true.")
+      .version("3.0.0")
       .timeConf(TimeUnit.NANOSECONDS)
       .createWithDefaultString("1s")
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 4f3f542048d9c..4f601c82a461b 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -877,6 +877,7 @@ Apart from these, the following properties are also available, and may be useful
     <a href="job-scheduling.html#configuration-and-setup">dynamic allocation
     configuration and setup documentation</a> for more information.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.port</code></td>
@@ -884,6 +885,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port on which the external shuffle service will run.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.index.cache.size</code></td>
@@ -1916,6 +1918,7 @@ Apart from these, the following properties are also available, and may be useful
     You should increase this setting if your tasks are long and see poor locality, but the
     default usually works well.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait.node</code></td>
@@ -1987,6 +1990,7 @@ Apart from these, the following properties are also available, and may be useful
     config. Note that capacity must be greater than 0. Consider increasing value (e.g. 20000) 
     if listener events are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.shared.capacity</code></td>
@@ -2051,6 +2055,7 @@ Apart from these, the following properties are also available, and may be useful
     due to too many task failures. The blacklisting algorithm can be further controlled by the
     other "spark.blacklist" configuration options.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.timeout</code></td>
@@ -2059,6 +2064,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) How long a node or executor is blacklisted for the entire application, before it
     is unconditionally removed from the blacklist to attempt running new tasks.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.task.maxTaskAttemptsPerExecutor</code></td>
@@ -2067,6 +2073,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) For a given task, how many times it can be retried on one executor before the
     executor is blacklisted for that task.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.task.maxTaskAttemptsPerNode</code></td>
@@ -2075,6 +2082,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) For a given task, how many times it can be retried on one node, before the entire
     node is blacklisted for that task.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.stage.maxFailedTasksPerExecutor</code></td>
@@ -2083,6 +2091,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) How many different tasks must fail on one executor, within one stage, before the
     executor is blacklisted for that stage.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.stage.maxFailedExecutorsPerNode</code></td>
@@ -2091,6 +2100,7 @@ Apart from these, the following properties are also available, and may be useful
     (Experimental) How many different executors are marked as blacklisted for a given stage, before
     the entire node is marked as failed for the stage.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.application.maxFailedTasksPerExecutor</code></td>
@@ -2102,6 +2112,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.blacklist.timeout</code>.  Note that with dynamic allocation, though, the executors
     may get marked as idle and be reclaimed by the cluster manager.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.application.maxFailedExecutorsPerNode</code></td>
@@ -2113,6 +2124,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.blacklist.timeout</code>.  Note that with dynamic allocation, though, the executors
     on the node may get marked as idle and be reclaimed by the cluster manager.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.killBlacklistedExecutors</code></td>
@@ -2123,6 +2135,7 @@ Apart from these, the following properties are also available, and may be useful
     as controlled by spark.blacklist.application.*. Note that, when an entire node is added 
     to the blacklist, all of the executors on that node will be killed.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.application.fetchFailure.enabled</code></td>
@@ -2132,6 +2145,7 @@ Apart from these, the following properties are also available, and may be useful
     failure happens. If external shuffle service is enabled, then the whole node will be
     blacklisted.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation</code></td>
@@ -2182,6 +2196,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Number of cores to allocate for each task.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.task.resource.{resourceName}.amount</code></td>
@@ -2207,6 +2222,7 @@ Apart from these, the following properties are also available, and may be useful
     to fail; a particular task has to fail this number of attempts.
     Should be greater than or equal to 1. Number of allowed retries = this value - 1.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.enabled</code></td>
@@ -2218,6 +2234,7 @@ Apart from these, the following properties are also available, and may be useful
     of this monitoring. When set to false (the default), task killing will use an older code
     path which lacks such monitoring.
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.pollingInterval</code></td>
@@ -2229,6 +2246,7 @@ Apart from these, the following properties are also available, and may be useful
     (this thread dump can be disabled via the <code>spark.task.reaper.threadDump</code> setting,
     which is documented below).
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.threadDump</code></td>
@@ -2238,6 +2256,7 @@ Apart from these, the following properties are also available, and may be useful
     dumps are logged during periodic polling of killed tasks. Set this to false to disable
     collection of thread dumps.
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.task.reaper.killTimeout</code></td>
@@ -2249,6 +2268,7 @@ Apart from these, the following properties are also available, and may be useful
     of this setting is to act as a safety-net to prevent runaway noncancellable tasks from rendering
     an executor unusable.
   </td>
+  <td>2.0.3</td>
 </tr>
 <tr>
   <td><code>spark.stage.maxConsecutiveAttempts</code></td>
@@ -2324,6 +2344,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.dynamicAllocation.initialExecutors</code>
     <code>spark.dynamicAllocation.executorAllocationRatio</code>
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.executorIdleTimeout</code></td>
@@ -2333,6 +2354,7 @@ Apart from these, the following properties are also available, and may be useful
     the executor will be removed. For more detail, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.cachedExecutorIdleTimeout</code></td>
@@ -2342,6 +2364,7 @@ Apart from these, the following properties are also available, and may be useful
     the executor will be removed. For more details, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.initialExecutors</code></td>
@@ -2352,6 +2375,7 @@ Apart from these, the following properties are also available, and may be useful
     If `--num-executors` (or `spark.executor.instances`) is set and larger than this value, it will
     be used as the initial number of executors.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.maxExecutors</code></td>
@@ -2359,6 +2383,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Upper bound for the number of executors if dynamic allocation is enabled.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.minExecutors</code></td>
@@ -2366,6 +2391,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Lower bound for the number of executors if dynamic allocation is enabled.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.executorAllocationRatio</code></td>
@@ -2383,6 +2409,7 @@ Apart from these, the following properties are also available, and may be useful
     by the <code>spark.dynamicAllocation.minExecutors</code> and
     <code>spark.dynamicAllocation.maxExecutors</code> settings
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.schedulerBacklogTimeout</code></td>
@@ -2392,6 +2419,7 @@ Apart from these, the following properties are also available, and may be useful
     this duration, new executors will be requested. For more detail, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code></td>
@@ -2401,6 +2429,7 @@ Apart from these, the following properties are also available, and may be useful
     subsequent executor requests. For more detail, see this
     <a href="job-scheduling.html#resource-allocation-policy">description</a>.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.shuffleTracking.enabled</code></td>
@@ -2410,6 +2439,7 @@ Apart from these, the following properties are also available, and may be useful
     without the need for an external shuffle service. This option will try to keep alive executors
     that are storing shuffle data for active jobs.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.dynamicAllocation.shuffleTimeout</code></td>
@@ -2421,6 +2451,7 @@ Apart from these, the following properties are also available, and may be useful
     quickly enough, this option can be used to control when to time out executors even when they are
     storing shuffle data.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 

From 5e7fa2e713d510aa502b2c83542cf4e57f454fb3 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Mon, 16 Mar 2020 10:08:07 +0900
Subject: [PATCH 0448/1280] [SPARK-31002][CORE][DOC][FOLLOWUP] Add version
 information to the configuration of Core
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR follows up #27847 and https://github.com/apache/spark/pull/27852.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.metrics.namespace | 2.1.0 | SPARK-5847 | 70f846a313061e4db6174e0dc6c12c8c806ccf78#diff-6bdad48cfc34314e89599655442ff210 |
spark.metrics.conf | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-7ea2624e832b166ca27cd4baca8691d9 |  
spark.metrics.executorMetricsSource.enabled | 3.0.0 | SPARK-27189 | 729f43f499f3dd2718c0b28d73f2ca29cc811eac#diff-6bdad48cfc34314e89599655442ff210 |  
spark.metrics.staticSources.enabled | 3.0.0 | SPARK-30060 | 60f20e5ea2000ab8f4a593b5e4217fd5637c5e22#diff-6bdad48cfc34314e89599655442ff210 |  
spark.pyspark.driver.python | 2.1.0 | SPARK-13081 | 7a9e25c38380e6c62080d62ad38a4830e44fe753#diff-6bdad48cfc34314e89599655442ff210 |  
spark.pyspark.python | 2.1.0 | SPARK-13081 | 7a9e25c38380e6c62080d62ad38a4830e44fe753#diff-6bdad48cfc34314e89599655442ff210 |  
spark.history.ui.maxApplications | 2.0.1 | SPARK-17243 | 021aa28f439443cda1bc7c5e3eee7c85b40c1a2d#diff-6bdad48cfc34314e89599655442ff210 |  
spark.io.encryption.enabled | 2.1.0 | SPARK-5682 | 4b4e329e49f8af28fa6301bd06c48d7097eaf9e6#diff-6bdad48cfc34314e89599655442ff210 |  
spark.io.encryption.keygen.algorithm | 2.1.0 | SPARK-5682 | 4b4e329e49f8af28fa6301bd06c48d7097eaf9e6#diff-6bdad48cfc34314e89599655442ff210 |  
spark.io.encryption.keySizeBits | 2.1.0 | SPARK-5682 | 4b4e329e49f8af28fa6301bd06c48d7097eaf9e6#diff-6bdad48cfc34314e89599655442ff210 |  
spark.io.encryption.commons.config.* | 2.1.0 | SPARK-5682 | 4b4e329e49f8af28fa6301bd06c48d7097eaf9e6 |  
spark.io.crypto.cipher.transformation | 2.1.0 | SPARK-5682 | 4b4e329e49f8af28fa6301bd06c48d7097eaf9e6#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.host | 0.7.0 | None | 02a6761589c35f15f1a6e3b63a7964ba057d3ba6#diff-eaf125f56ce786d64dcef99cf446a751 |  
spark.driver.port | 0.7.0 | None | 02a6761589c35f15f1a6e3b63a7964ba057d3ba6#diff-eaf125f56ce786d64dcef99cf446a751 |  
spark.driver.supervise | 1.3.0 | SPARK-5388 | 6ec0cdc14390d4dc45acf31040f21e1efc476fc0#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
spark.driver.bindAddress | 2.1.0 | SPARK-4563 | 2cd1bfa4f0c6625b0ab1dbeba2b9586b9a6a9f42#diff-6bdad48cfc34314e89599655442ff210 |  
spark.blockManager.port | 1.1.0 | SPARK-2157 | 31090e43ca91f687b0bc6e25c824dc25bd7027cd#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.driver.blockManager.port | 2.1.0 | SPARK-4563 | 2cd1bfa4f0c6625b0ab1dbeba2b9586b9a6a9f42#diff-6bdad48cfc34314e89599655442ff210 |  
spark.files.ignoreCorruptFiles | 2.1.0 | SPARK-17850 | 47776e7c0c68590fe446cef910900b1aaead06f9#diff-6bdad48cfc34314e89599655442ff210 |  
spark.files.ignoreMissingFiles | 2.4.0 | SPARK-22676 | ed4101d29f50d54fd7846421e4c00e9ecd3599d0#diff-6bdad48cfc34314e89599655442ff210 |  
spark.log.callerContext | 2.2.0 | SPARK-16759 | 3af894511be6fcc17731e28b284dba432fe911f5#diff-6bdad48cfc34314e89599655442ff210 | In branch-2.2 but pom.xml is 2.1.0-SNAPSHOT
spark.files.maxPartitionBytes | 2.1.0 | SPARK-16575 | c8879bf1ee2af9ccd5d5656571d931d2fc1da024#diff-6bdad48cfc34314e89599655442ff210 |  
spark.files.openCostInBytes | 2.1.0 | SPARK-16575 | c8879bf1ee2af9ccd5d5656571d931d2fc1da024#diff-6bdad48cfc34314e89599655442ff210 |  
spark.hadoopRDD.ignoreEmptySplits | 2.3.0 | SPARK-22233 | 0fa10666cf75e3c4929940af49c8a6f6ea874759#diff-6bdad48cfc34314e89599655442ff210 |  
spark.redaction.regex | 2.1.2 | SPARK-18535 and SPARK-19720 | 444cca14d7ac8c5ab5d7e9d080b11f4d6babe3bf#diff-6bdad48cfc34314e89599655442ff210 |  
spark.redaction.string.regex | 2.2.0 | SPARK-20070 | 91fa80fe8a2480d64c430bd10f97b3d44c007bcc#diff-6bdad48cfc34314e89599655442ff210 |  
spark.authenticate.secret | 1.0.0 | SPARK-1189 | 7edbea41b43e0dc11a2de156be220db8b7952d01#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.authenticate.secretBitLength | 1.6.0 | SPARK-11073 | f8d93edec82eedab59d50aec06ca2de7e4cf14f6#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.authenticate | 1.0.0 | SPARK-1189 | 7edbea41b43e0dc11a2de156be220db8b7952d01#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.authenticate.enableSaslEncryption | 1.4.0 | SPARK-6229 | 38d4e9e446b425ca6a8fe8d8080f387b08683842#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |
spark.authenticate.secret.file | 3.0.0 | SPARK-26239 | 57d6fbfa8c803ce1791e7be36aba0219a1fcaa63#diff-6bdad48cfc34314e89599655442ff210 |  
spark.authenticate.secret.driver.file | 3.0.0 | SPARK-26239 | 57d6fbfa8c803ce1791e7be36aba0219a1fcaa63#diff-6bdad48cfc34314e89599655442ff210 |  
spark.authenticate.secret.executor.file | 3.0.0 | SPARK-26239 | 57d6fbfa8c803ce1791e7be36aba0219a1fcaa63#diff-6bdad48cfc34314e89599655442ff210 |  
spark.buffer.write.chunkSize | 2.3.0 | SPARK-21527 | 574ef6c987c636210828e96d2f797d8f10aff05e#diff-6bdad48cfc34314e89599655442ff210 |  
spark.checkpoint.compress | 2.2.0 | SPARK-19525 | 1405862382185e04b09f84af18f82f2f0295a755#diff-6bdad48cfc34314e89599655442ff210 |  
spark.rdd.checkpoint.cachePreferredLocsExpireTime | 3.0.0 | SPARK-29182 | 4ecbdbb6a7bd3908da32c82832e886b4f9f9e596#diff-6bdad48cfc34314e89599655442ff210 |
spark.shuffle.accurateBlockThreshold | 2.2.1 | SPARK-20801 | 81f63c8923416014d5c6bc227dd3c4e2a62bac8e#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.registration.timeout | 2.3.0 | SPARK-20640 | d107b3b910d8f434fb15b663a9db4c2dfe0a9f43#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.registration.maxAttempts | 2.3.0 | SPARK-20640 | d107b3b910d8f434fb15b663a9db4c2dfe0a9f43#diff-6bdad48cfc34314e89599655442ff210 |  
spark.reducer.maxBlocksInFlightPerAddress | 2.2.1 | SPARK-21243 | 88dccda393bc79dc6032f71b6acf8eb2b4b152be#diff-6bdad48cfc34314e89599655442ff210 |  
spark.network.maxRemoteBlockSizeFetchToMem | 3.0.0 | SPARK-26700 | d8613571bc1847775dd5c1945757279234cb388c#diff-6bdad48cfc34314e89599655442ff210 |
spark.taskMetrics.trackUpdatedBlockStatuses | 2.3.0 | SPARK-20923 | 5b5a69bea9de806e2c39b04b248ee82a7b664d7b#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.sort.io.plugin.class | 3.0.0 | SPARK-28209 | abef84a868e9e15f346eea315bbab0ec8ac8e389#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.file.buffer | 1.4.0 | SPARK-7081 | c53ebea9db418099df50f9adc1a18cee7849cd97#diff-ecdafc46b901740134261d2cab24ccd9 |  
spark.shuffle.unsafe.file.output.buffer | 2.3.0 | SPARK-20950 | 565e7a8d4ae7879ee704fb94ae9b3da31e202d7e#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.spill.diskWriteBufferSize | 2.3.0 | SPARK-20950 | 565e7a8d4ae7879ee704fb94ae9b3da31e202d7e#diff-6bdad48cfc34314e89599655442ff210 |  
spark.storage.unrollMemoryCheckPeriod | 2.3.0 | SPARK-21923 | a11db942aaf4c470a85f8a1b180f034f7a584254#diff-6bdad48cfc34314e89599655442ff210 |  
spark.storage.unrollMemoryGrowthFactor | 2.3.0 | SPARK-21923 | a11db942aaf4c470a85f8a1b180f034f7a584254#diff-6bdad48cfc34314e89599655442ff210 |  
spark.yarn.dist.forceDownloadSchemes | 2.3.0 | SPARK-21917 | 8319432af60b8e1dc00f08d794f7d80591e24d0c#diff-6bdad48cfc34314e89599655442ff210 |  
spark.extraListeners | 1.3.0 | SPARK-5411 | 47e4d579eb4a9aab8e0dd9c1400394d80c8d0388#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.shuffle.spill.numElementsForceSpillThreshold | 1.6.0 | SPARK-10708 | f6d06adf05afa9c5386dc2396c94e7a98730289f#diff-3eedc75de4787b842477138d8cc7f150 |  
spark.shuffle.mapOutput.parallelAggregationThreshold | 2.3.0 | SPARK-22537 | efd0036ec88bdc385f5a9ea568d2e2bbfcda2912#diff-6bdad48cfc34314e89599655442ff210 |  
spark.driver.maxResultSize | 1.2.0 | SPARK-3466 | 6181577e9935f46b646ba3925b873d031aa3d6ba#diff-d239aee594001f8391676e1047a0381e |
spark.security.credentials.renewalRatio | 2.4.0 | SPARK-23361 | 5fa438471110afbf4e2174df449ac79e292501f8#diff-6bdad48cfc34314e89599655442ff210 |  
spark.security.credentials.retryWait | 2.4.0 | SPARK-23361 | 5fa438471110afbf4e2174df449ac79e292501f8#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.sort.initialBufferSize | 2.1.0 | SPARK-15958 | bf665a958631125a1670504ef5966ef1a0e14798#diff-a1d00506391c1c4b2209f9bbff590c5b | On branch-2.1, but in pom.xml it is 2.0.0-SNAPSHOT
spark.shuffle.compress | 0.6.0 | None | efc5423210d1aadeaea78273a4a8f10425753079#diff-76170a9c8f67b542bc58240a0a12fe08 |  
spark.shuffle.spill.compress | 0.9.0 | None | c3816de5040e3c48e58ed4762d2f4eb606812938#diff-2b643ea78c1add0381754b1f47eec132 |  
spark.shuffle.mapStatus.compression.codec | 3.0.0 | SPARK-29939 | 456cfe6e4693efd26d64f089d53c4e01bf8150a2#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.spill.initialMemoryThreshold | 1.1.1 | SPARK-4480 | 16bf5f3d17624db2a96c921fe8a1e153cdafb06c#diff-31417c461d8901d8e08167b0cbc344c1 |  
spark.shuffle.spill.batchSize | 0.9.0 | None | c3816de5040e3c48e58ed4762d2f4eb606812938#diff-a470b9812a5ac8c37d732da7d9fbe39a |
spark.shuffle.sort.bypassMergeThreshold | 1.1.1 | SPARK-2787 | 0f2274f8ed6131ad17326e3fff7f7e093863b72d#diff-31417c461d8901d8e08167b0cbc344c1 |  
spark.shuffle.manager | 1.1.0 | SPARK-2044 | 508fd371d6dbb826fd8a00787d347235b549e189#diff-60df49b5d3c59f2c4540fa16a90033a1 |  
spark.shuffle.reduceLocality.enabled | 1.5.0 | SPARK-2774 | 96a7c888d806adfdb2c722025a1079ed7eaa2052#diff-6a9ff7fb74fd490a50462d45db2d5e11 |  
spark.shuffle.mapOutput.minSizeForBroadcast | 2.0.0 | SPARK-1239 | d98dd72e7baeb59eacec4fefd66397513a607b2f#diff-609c3f8c26150ca96a94cd27146a809b |  
spark.shuffle.mapOutput.dispatcher.numThreads | 2.0.0 | SPARK-1239 | d98dd72e7baeb59eacec4fefd66397513a607b2f#diff-609c3f8c26150ca96a94cd27146a809b |  
spark.shuffle.detectCorrupt | 2.2.0 | SPARK-4105 | cf33a86285629abe72c1acf235b8bfa6057220a8#diff-eb30a71e0d04150b8e0b64929852e38b |
spark.shuffle.detectCorrupt.useExtraMemory | 3.0.0 | SPARK-26089 | 688b0c01fac0db80f6473181673a89f1ce1be65b#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.sync | 0.8.0 | None | 31da065b1d08c1fad5283e4bcf8e0ed01818c03e#diff-ad46ed23fcc3fa87f30d05204917b917 |  
spark.shuffle.unsafe.fastMergeEnabled | 1.4.0 | SPARK-7081 | c53ebea9db418099df50f9adc1a18cee7849cd97#diff-642ce9f439435408382c3ac3b5c5e0a0 |  
spark.shuffle.sort.useRadixSort | 2.0.0 | SPARK-14724 | e2b5647ab92eb478b3f7b36a0ce6faf83e24c0e5#diff-3eedc75de4787b842477138d8cc7f150 |  
spark.shuffle.minNumPartitionsToHighlyCompress | 2.4.0 | SPARK-24519 | 39dfaf2fd167cafc84ec9cc637c114ed54a331e3#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.useOldFetchProtocol | 3.0.0 | SPARK-25341 | f725d472f51fb80c6ce1882ec283ff69bafb0de4#diff-6bdad48cfc34314e89599655442ff210 |  
spark.shuffle.readHostLocalDisk | 3.0.0 | SPARK-30812 | 68d7edf9497bea2f73707d32ab55dd8e53088e7c#diff-6bdad48cfc34314e89599655442ff210 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #27913 from beliefer/add-version-to-core-config-part-three.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/package.scala       | 77 ++++++++++++++++++-
 docs/configuration.md                         | 31 ++++++--
 docs/security.md                              | 22 +++++-
 3 files changed, 119 insertions(+), 11 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 12ae56c3be76e..74a2e0af2952a 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -776,47 +776,59 @@ package object config {
 
   // This property sets the root namespace for metrics reporting
   private[spark] val METRICS_NAMESPACE = ConfigBuilder("spark.metrics.namespace")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_CONF = ConfigBuilder("spark.metrics.conf")
+    .version("0.8.0")
     .stringConf
     .createOptional
 
   private[spark] val METRICS_EXECUTORMETRICS_SOURCE_ENABLED =
     ConfigBuilder("spark.metrics.executorMetricsSource.enabled")
       .doc("Whether to register the ExecutorMetrics source with the metrics system.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val METRICS_STATIC_SOURCES_ENABLED =
     ConfigBuilder("spark.metrics.staticSources.enabled")
       .doc("Whether to register static sources with the metrics system.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val PYSPARK_DRIVER_PYTHON = ConfigBuilder("spark.pyspark.driver.python")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   private[spark] val PYSPARK_PYTHON = ConfigBuilder("spark.pyspark.python")
+    .version("2.1.0")
     .stringConf
     .createOptional
 
   // To limit how many applications are shown in the History Server summary ui
   private[spark] val HISTORY_UI_MAX_APPS =
-    ConfigBuilder("spark.history.ui.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
+    ConfigBuilder("spark.history.ui.maxApplications")
+      .version("2.0.1")
+      .intConf
+      .createWithDefault(Integer.MAX_VALUE)
 
   private[spark] val IO_ENCRYPTION_ENABLED = ConfigBuilder("spark.io.encryption.enabled")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IO_ENCRYPTION_KEYGEN_ALGORITHM =
     ConfigBuilder("spark.io.encryption.keygen.algorithm")
+      .version("2.1.0")
       .stringConf
       .createWithDefault("HmacSHA1")
 
   private[spark] val IO_ENCRYPTION_KEY_SIZE_BITS = ConfigBuilder("spark.io.encryption.keySizeBits")
+    .version("2.1.0")
     .intConf
     .checkValues(Set(128, 192, 256))
     .createWithDefault(128)
@@ -824,57 +836,68 @@ package object config {
   private[spark] val IO_CRYPTO_CIPHER_TRANSFORMATION =
     ConfigBuilder("spark.io.crypto.cipher.transformation")
       .internal()
+      .version("2.1.0")
       .stringConf
       .createWithDefaultString("AES/CTR/NoPadding")
 
   private[spark] val DRIVER_HOST_ADDRESS = ConfigBuilder("spark.driver.host")
     .doc("Address of driver endpoints.")
+    .version("0.7.0")
     .stringConf
     .createWithDefault(Utils.localCanonicalHostName())
 
   private[spark] val DRIVER_PORT = ConfigBuilder("spark.driver.port")
     .doc("Port of driver endpoints.")
+    .version("0.7.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_SUPERVISE = ConfigBuilder("spark.driver.supervise")
     .doc("If true, restarts the driver automatically if it fails with a non-zero exit status. " +
       "Only has effect in Spark standalone mode or Mesos cluster deploy mode.")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val DRIVER_BIND_ADDRESS = ConfigBuilder("spark.driver.bindAddress")
     .doc("Address where to bind network listen sockets on the driver.")
+    .version("2.1.0")
     .fallbackConf(DRIVER_HOST_ADDRESS)
 
   private[spark] val BLOCK_MANAGER_PORT = ConfigBuilder("spark.blockManager.port")
     .doc("Port to use for the block manager when a more specific setting is not provided.")
+    .version("1.1.0")
     .intConf
     .createWithDefault(0)
 
   private[spark] val DRIVER_BLOCK_MANAGER_PORT = ConfigBuilder("spark.driver.blockManager.port")
     .doc("Port to use for the block manager on the driver.")
+    .version("2.1.0")
     .fallbackConf(BLOCK_MANAGER_PORT)
 
   private[spark] val IGNORE_CORRUPT_FILES = ConfigBuilder("spark.files.ignoreCorruptFiles")
     .doc("Whether to ignore corrupt files. If true, the Spark jobs will continue to run when " +
       "encountering corrupted or non-existing files and contents that have been read will still " +
       "be returned.")
+    .version("2.1.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val IGNORE_MISSING_FILES = ConfigBuilder("spark.files.ignoreMissingFiles")
     .doc("Whether to ignore missing files. If true, the Spark jobs will continue to run when " +
       "encountering missing files and the contents that have been read will still be returned.")
+    .version("2.4.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val APP_CALLER_CONTEXT = ConfigBuilder("spark.log.callerContext")
+    .version("2.2.0")
     .stringConf
     .createOptional
 
   private[spark] val FILES_MAX_PARTITION_BYTES = ConfigBuilder("spark.files.maxPartitionBytes")
     .doc("The maximum number of bytes to pack into a single partition when reading files.")
+    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(128 * 1024 * 1024)
 
@@ -883,6 +906,7 @@ package object config {
       " the same time. This is used when putting multiple files into a partition. It's better to" +
       " over estimate, then the partitions with small files will be faster than partitions with" +
       " bigger files.")
+    .version("2.1.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(4 * 1024 * 1024)
 
@@ -890,6 +914,7 @@ package object config {
     ConfigBuilder("spark.hadoopRDD.ignoreEmptySplits")
       .internal()
       .doc("When true, HadoopRDD/NewHadoopRDD will not create partitions for empty input splits.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -899,6 +924,7 @@ package object config {
         "driver and executor environments contain sensitive information. When this regex matches " +
         "a property key or value, the value is redacted from the environment UI and various logs " +
         "like YARN and event logs.")
+      .version("2.1.2")
       .regexConf
       .createWithDefault("(?i)secret|password|token".r)
 
@@ -907,26 +933,31 @@ package object config {
       .doc("Regex to decide which parts of strings produced by Spark contain sensitive " +
         "information. When this regex matches a string part, that string part is replaced by a " +
         "dummy value. This is currently used to redact the output of SQL explain commands.")
+      .version("2.2.0")
       .regexConf
       .createOptional
 
   private[spark] val AUTH_SECRET =
     ConfigBuilder("spark.authenticate.secret")
+      .version("1.0.0")
       .stringConf
       .createOptional
 
   private[spark] val AUTH_SECRET_BIT_LENGTH =
     ConfigBuilder("spark.authenticate.secretBitLength")
+      .version("1.6.0")
       .intConf
       .createWithDefault(256)
 
   private[spark] val NETWORK_AUTH_ENABLED =
     ConfigBuilder("spark.authenticate")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SASL_ENCRYPTION_ENABLED =
     ConfigBuilder("spark.authenticate.enableSaslEncryption")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -936,6 +967,7 @@ package object config {
         "loaded from this path on both the driver and the executors if overrides are not set for " +
         "either entity (see below). File-based secret keys are only allowed when using " +
         "Kubernetes.")
+      .version("3.0.0")
       .stringConf
       .createOptional
 
@@ -948,6 +980,7 @@ package object config {
         "be specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
+      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val AUTH_SECRET_FILE_EXECUTOR =
@@ -959,12 +992,14 @@ package object config {
         "specified for the executors. The fallback configuration allows the same path to be " +
         "used for both the driver and the executors when running in cluster mode. File-based " +
         "secret keys are only allowed when using Kubernetes.")
+      .version("3.0.0")
       .fallbackConf(AUTH_SECRET_FILE)
 
   private[spark] val BUFFER_WRITE_CHUNK_SIZE =
     ConfigBuilder("spark.buffer.write.chunkSize")
       .internal()
       .doc("The chunk size in bytes during writing out the bytes of ChunkedByteBuffer.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(_ <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         "The chunk size during writing out the bytes of ChunkedByteBuffer should" +
@@ -975,6 +1010,7 @@ package object config {
     ConfigBuilder("spark.checkpoint.compress")
       .doc("Whether to compress RDD checkpoints. Generally a good idea. Compression will use " +
         "spark.io.compression.codec.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -985,6 +1021,7 @@ package object config {
         "Caching preferred locations can relieve query loading to DFS and save the query " +
         "time. The drawback is that the cached locations can be possibly outdated and " +
         "lose data locality. If this config is not specified, it will not cache.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MINUTES)
       .checkValue(_ > 0, "The expire time for caching preferred locations cannot be non-positive.")
       .createOptional
@@ -994,12 +1031,14 @@ package object config {
       .doc("Threshold in bytes above which the size of shuffle blocks in " +
         "HighlyCompressedMapStatus is accurately recorded. This helps to prevent OOM " +
         "by avoiding underestimating shuffle block size when fetch shuffle blocks.")
+      .version("2.2.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(100 * 1024 * 1024)
 
   private[spark] val SHUFFLE_REGISTRATION_TIMEOUT =
     ConfigBuilder("spark.shuffle.registration.timeout")
       .doc("Timeout in milliseconds for registration to the external shuffle service.")
+      .version("2.3.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(5000)
 
@@ -1007,6 +1046,7 @@ package object config {
     ConfigBuilder("spark.shuffle.registration.maxAttempts")
       .doc("When we fail to register to the external shuffle service, we will " +
         "retry for maxAttempts times.")
+      .version("2.3.0")
       .intConf
       .createWithDefault(3)
 
@@ -1017,6 +1057,7 @@ package object config {
         "address in a single fetch or simultaneously, this could crash the serving executor or " +
         "Node Manager. This is especially useful to reduce the load on the Node Manager when " +
         "external shuffle is enabled. You can mitigate the issue by setting it to a lower value.")
+      .version("2.2.1")
       .intConf
       .checkValue(_ > 0, "The max no. of blocks in flight cannot be non-positive.")
       .createWithDefault(Int.MaxValue)
@@ -1028,6 +1069,7 @@ package object config {
         "configuration will affect both shuffle fetch and block manager remote block fetch. " +
         "For users who enabled external shuffle service, this feature can only work when " +
         "external shuffle service is at least 2.3.0.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       // fetch-to-mem is guaranteed to fail if the message is bigger than 2 GB, so we might
       // as well use fetch-to-disk in that case.  The message includes some metadata in addition
@@ -1043,12 +1085,14 @@ package object config {
       .doc("Enable tracking of updatedBlockStatuses in the TaskMetrics. Off by default since " +
         "tracking the block statuses can use a lot of memory and its not used anywhere within " +
         "spark.")
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_IO_PLUGIN_CLASS =
     ConfigBuilder("spark.shuffle.sort.io.plugin.class")
       .doc("Name of the class to use for shuffle IO.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault(classOf[LocalDiskShuffleDataIO].getName)
 
@@ -1057,6 +1101,7 @@ package object config {
       .doc("Size of the in-memory buffer for each shuffle file output stream, in KiB unless " +
         "otherwise specified. These buffers reduce the number of disk seeks and system calls " +
         "made in creating intermediate shuffle files.")
+      .version("1.4.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The file buffer size must be positive and less than or equal to" +
@@ -1067,6 +1112,7 @@ package object config {
     ConfigBuilder("spark.shuffle.unsafe.file.output.buffer")
       .doc("The file system for this buffer size after each partition " +
         "is written in unsafe shuffle writer. In KiB unless otherwise specified.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.KiB)
       .checkValue(v => v > 0 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH / 1024,
         s"The buffer size must be positive and less than or equal to" +
@@ -1076,6 +1122,7 @@ package object config {
   private[spark] val SHUFFLE_DISK_WRITE_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.spill.diskWriteBufferSize")
       .doc("The buffer size, in bytes, to use when writing the sorted records to an on-disk file.")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 12 && v <= ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH,
         s"The buffer size must be greater than 12 and less than or equal to " +
@@ -1087,6 +1134,7 @@ package object config {
       .internal()
       .doc("The memory check period is used to determine how often we should check whether "
         + "there is a need to request more memory when we try to unroll the given block in memory.")
+      .version("2.3.0")
       .longConf
       .createWithDefault(16)
 
@@ -1094,6 +1142,7 @@ package object config {
     ConfigBuilder("spark.storage.unrollMemoryGrowthFactor")
       .internal()
       .doc("Memory to request as a multiple of the size that used to unroll the block.")
+      .version("2.3.0")
       .doubleConf
       .createWithDefault(1.5)
 
@@ -1104,12 +1153,14 @@ package object config {
         "where the YARN service does not support schemes that are supported by Spark, like http, " +
         "https and ftp, or jars required to be in the local YARN client's classpath. Wildcard " +
         "'*' is denoted to download resources for all the schemes.")
+      .version("2.3.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val EXTRA_LISTENERS = ConfigBuilder("spark.extraListeners")
     .doc("Class names of listeners to add to SparkContext during initialization.")
+    .version("1.3.0")
     .stringConf
     .toSequence
     .createOptional
@@ -1121,6 +1172,7 @@ package object config {
         "By default it's Integer.MAX_VALUE, which means we never force the sorter to spill, " +
         "until we reach some limitations, like the max page size limitation for the pointer " +
         "array in the sorter.")
+      .version("1.6.0")
       .intConf
       .createWithDefault(Integer.MAX_VALUE)
 
@@ -1130,30 +1182,35 @@ package object config {
       .doc("Multi-thread is used when the number of mappers * shuffle partitions is greater than " +
         "or equal to this threshold. Note that the actual parallelism is calculated by number of " +
         "mappers * shuffle partitions / this threshold + 1, so this threshold should be positive.")
+      .version("2.3.0")
       .intConf
       .checkValue(v => v > 0, "The threshold should be positive.")
       .createWithDefault(10000000)
 
   private[spark] val MAX_RESULT_SIZE = ConfigBuilder("spark.driver.maxResultSize")
     .doc("Size limit for results.")
+    .version("1.2.0")
     .bytesConf(ByteUnit.BYTE)
     .createWithDefaultString("1g")
 
   private[spark] val CREDENTIALS_RENEWAL_INTERVAL_RATIO =
     ConfigBuilder("spark.security.credentials.renewalRatio")
       .doc("Ratio of the credential's expiration time when Spark should fetch new credentials.")
+      .version("2.4.0")
       .doubleConf
       .createWithDefault(0.75d)
 
   private[spark] val CREDENTIALS_RENEWAL_RETRY_WAIT =
     ConfigBuilder("spark.security.credentials.retryWait")
       .doc("How long to wait before retrying to fetch new credentials after a failure.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("1h")
 
   private[spark] val SHUFFLE_SORT_INIT_BUFFER_SIZE =
     ConfigBuilder("spark.shuffle.sort.initialBufferSize")
       .internal()
+      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v > 0 && v <= Int.MaxValue,
         s"The buffer size must be greater than 0 and less than or equal to ${Int.MaxValue}.")
@@ -1163,6 +1220,7 @@ package object config {
     ConfigBuilder("spark.shuffle.compress")
       .doc("Whether to compress shuffle output. Compression will use " +
         "spark.io.compression.codec.")
+      .version("0.6.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1170,6 +1228,7 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.compress")
       .doc("Whether to compress data spilled during shuffles. Compression will use " +
         "spark.io.compression.codec.")
+      .version("0.9.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1179,6 +1238,7 @@ package object config {
       .doc("The codec used to compress MapStatus, which is generated by ShuffleMapTask. " +
         "By default, Spark provides four codecs: lz4, lzf, snappy, and zstd. You can also " +
         "use fully qualified class names to specify the codec.")
+      .version("3.0.0")
       .stringConf
       .createWithDefault("zstd")
 
@@ -1187,6 +1247,7 @@ package object config {
       .internal()
       .doc("Initial threshold for the size of a collection before we start tracking its " +
         "memory usage.")
+      .version("1.1.1")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(5 * 1024 * 1024)
 
@@ -1194,6 +1255,7 @@ package object config {
     ConfigBuilder("spark.shuffle.spill.batchSize")
       .internal()
       .doc("Size of object batches when reading/writing from serializers.")
+      .version("0.9.0")
       .longConf
       .createWithDefault(10000)
 
@@ -1201,34 +1263,40 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.bypassMergeThreshold")
       .doc("In the sort-based shuffle manager, avoid merge-sorting data if there is no " +
         "map-side aggregation and there are at most this many reduce partitions")
+      .version("1.1.1")
       .intConf
       .createWithDefault(200)
 
   private[spark] val SHUFFLE_MANAGER =
     ConfigBuilder("spark.shuffle.manager")
+      .version("1.1.0")
       .stringConf
       .createWithDefault("sort")
 
   private[spark] val SHUFFLE_REDUCE_LOCALITY_ENABLE =
     ConfigBuilder("spark.shuffle.reduceLocality.enabled")
       .doc("Whether to compute locality preferences for reduce tasks")
+      .version("1.5.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val SHUFFLE_MAPOUTPUT_MIN_SIZE_FOR_BROADCAST =
     ConfigBuilder("spark.shuffle.mapOutput.minSizeForBroadcast")
       .doc("The size at which we use Broadcast to send the map output statuses to the executors.")
+      .version("2.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("512k")
 
   private[spark] val SHUFFLE_MAPOUTPUT_DISPATCHER_NUM_THREADS =
     ConfigBuilder("spark.shuffle.mapOutput.dispatcher.numThreads")
+      .version("2.0.0")
       .intConf
       .createWithDefault(8)
 
   private[spark] val SHUFFLE_DETECT_CORRUPT =
     ConfigBuilder("spark.shuffle.detectCorrupt")
       .doc("Whether to detect any corruption in fetched blocks.")
+      .version("2.2.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1238,18 +1306,21 @@ package object config {
         "by using extra memory to detect early corruption. Any IOException thrown will cause " +
         "the task to be retried once and if it fails again with same exception, then " +
         "FetchFailedException will be thrown to retry previous stage")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_SYNC =
     ConfigBuilder("spark.shuffle.sync")
       .doc("Whether to force outstanding writes to disk.")
+      .version("0.8.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SHUFFLE_UNSAFE_FAST_MERGE_ENABLE =
     ConfigBuilder("spark.shuffle.unsafe.fastMergeEnabled")
       .doc("Whether to perform a fast spill merge.")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1257,6 +1328,7 @@ package object config {
     ConfigBuilder("spark.shuffle.sort.useRadixSort")
       .doc("Whether to use radix sort for sorting in-memory partition ids. Radix sort is much " +
         "faster, but requires additional memory to be reserved memory as pointers are added.")
+      .version("2.0.0")
       .booleanConf
       .createWithDefault(true)
 
@@ -1264,6 +1336,7 @@ package object config {
     ConfigBuilder("spark.shuffle.minNumPartitionsToHighlyCompress")
       .internal()
       .doc("Number of partitions to determine if MapStatus should use HighlyCompressedMapStatus")
+      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The value should be a positive integer.")
       .createWithDefault(2000)
@@ -1273,6 +1346,7 @@ package object config {
       .doc("Whether to use the old protocol while doing the shuffle block fetching. " +
         "It is only enabled while we need the compatibility in the scenario of new Spark " +
         "version job fetching shuffle blocks from old version external shuffle service.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
@@ -1282,6 +1356,7 @@ package object config {
         s"shuffle `${SHUFFLE_SERVICE_ENABLED.key}` is enabled), shuffle " +
         "blocks requested from those block managers which are running on the same host are read " +
         "from the disk directly instead of being fetched as remote blocks over the network.")
+      .version("3.0.0")
       .booleanConf
       .createWithDefault(true)
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 4f601c82a461b..e4ea25c05c23f 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -163,6 +163,7 @@ of the most common options to set are:
     and memory overhead of objects in JVM). Setting a proper limit can protect the driver from
     out-of-memory errors.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.memory</code></td>
@@ -322,6 +323,7 @@ of the most common options to set are:
     will be called; otherwise, a zero-argument constructor will be called. If no valid constructor
     can be found, the SparkContext creation will fail with an exception.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.local.dir</code></td>
@@ -368,6 +370,7 @@ of the most common options to set are:
     Its length depends on the Hadoop configuration <code>hadoop.caller.context.max.size</code>. It should be concise,
     and typically can have up to 50 characters.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.supervise</code></td>
@@ -376,6 +379,7 @@ of the most common options to set are:
     If true, restarts the driver automatically if it fails with a non-zero exit status.
     Only has effect in Spark standalone mode or Mesos cluster deploy mode.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.log.dfsDir</code></td>
@@ -635,6 +639,7 @@ Apart from these, the following properties are also available, and may be useful
     executor environments contain sensitive information. When this regex matches a property key or
     value, the value is redacted from the environment UI and various logs like YARN and event logs.
   </td>
+  <td>2.1.2</td>
 </tr>
 <tr>
   <td><code>spark.python.profile</code></td>
@@ -648,6 +653,7 @@ Apart from these, the following properties are also available, and may be useful
     By default the <code>pyspark.profiler.BasicProfiler</code> will be used, but this can be overridden by
     passing a profiler class in as a parameter to the <code>SparkContext</code> constructor.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.python.profile.dump</code></td>
@@ -757,6 +763,7 @@ Apart from these, the following properties are also available, and may be useful
     Python binary executable to use for PySpark in driver.
     (default is <code>spark.pyspark.python</code>)
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.pyspark.python</code></td>
@@ -764,6 +771,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Python binary executable to use for PySpark in both driver and executors.
   </td>
+  <td>2.1.0</td>
 </tr>
 </table>
 
@@ -801,6 +809,7 @@ Apart from these, the following properties are also available, and may be useful
     is especially useful to reduce the load on the Node Manager when external shuffle is enabled.
     You can mitigate this issue by setting it to a lower value.
   </td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.compress</code></td>
@@ -809,6 +818,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to compress map output files. Generally a good idea. Compression will use
     <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.file.buffer</code></td>
@@ -818,6 +828,7 @@ Apart from these, the following properties are also available, and may be useful
     specified. These buffers reduce the number of disk seeks and system calls made in creating 
     intermediate shuffle files.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.maxRetries</code></td>
@@ -912,6 +923,7 @@ Apart from these, the following properties are also available, and may be useful
     (Advanced) In the sort-based shuffle manager, avoid merge-sorting data if there is no
     map-side aggregation and there are at most this many reduce partitions.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.spill.compress</code></td>
@@ -920,6 +932,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to compress data spilled during shuffles. Compression will use
     <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.accurateBlockThreshold</code></td>
@@ -929,6 +942,7 @@ Apart from these, the following properties are also available, and may be useful
     accurately recorded. This helps to prevent OOM by avoiding underestimating shuffle 
     block size when fetch shuffle blocks.
   </td>
+  <td>2.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.registration.timeout</code></td>
@@ -936,6 +950,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Timeout in milliseconds for registration to the external shuffle service.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.registration.maxAttempts</code></td>
@@ -943,6 +958,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     When we fail to register to the external shuffle service, we will retry for maxAttempts times.
   </td>
+  <td>2.3.0</td>
 </tr>
 </table>
 
@@ -1269,7 +1285,8 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Whether to compress RDD checkpoints. Generally a good idea.
     Compression will use <code>spark.io.compression.codec</code>.
-   </td>
+  </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.codec</code></td>
@@ -1635,6 +1652,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     The maximum number of bytes to pack into a single partition when reading files.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.files.openCostInBytes</code></td>
@@ -1644,6 +1662,7 @@ Apart from these, the following properties are also available, and may be useful
     time. This is used when putting multiple files into a partition. It is better to overestimate,
     then the partitions with small files will be faster than partitions with bigger files.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
     <td><code>spark.hadoop.cloneConf</code></td>
@@ -1745,7 +1764,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Port for all block managers to listen on. These exist on both the driver and the executors.
   </td>
-  <td></td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.blockManager.port</code></td>
@@ -1754,7 +1773,7 @@ Apart from these, the following properties are also available, and may be useful
     Driver-specific port for the block manager to listen on, for cases where it cannot use the same
     configuration as executors.
   </td>
-  <td></td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.bindAddress</code></td>
@@ -1768,7 +1787,7 @@ Apart from these, the following properties are also available, and may be useful
     the different ports used by the driver (RPC, block manager and UI) need to be forwarded from the
     container's host.
   </td>
-  <td></td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.host</code></td>
@@ -1777,7 +1796,7 @@ Apart from these, the following properties are also available, and may be useful
     Hostname or IP address for the driver.
     This is used for communicating with the executors and the standalone Master.
   </td>
-  <td></td>
+  <td>0.7.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.port</code></td>
@@ -1786,7 +1805,7 @@ Apart from these, the following properties are also available, and may be useful
     Port for the driver to listen on.
     This is used for communicating with the executors and the standalone Master.
   </td>
-  <td></td>
+  <td>0.7.0</td>
 </tr>
 <tr>
   <td><code>spark.rpc.io.backLog</code></td>
diff --git a/docs/security.md b/docs/security.md
index a4556e242cffc..5496879d8c840 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -66,11 +66,12 @@ also see their authentication secret. Access control rules should be properly se
 Kubernetes admin to ensure that Spark authentication is secure.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.authenticate</code></td>
   <td>false</td>
   <td>Whether Spark authenticates its internal connections.</td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.secret</code></td>
@@ -78,6 +79,7 @@ Kubernetes admin to ensure that Spark authentication is secure.
   <td>
     The secret key used authentication. See above for when this configuration should be set.
   </td>
+  <td>1.0.0</td>
 </tr>
 </table>
 
@@ -85,7 +87,7 @@ Alternatively, one can mount authentication secrets using files and Kubernetes s
 the user mounts into their pods.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.authenticate.secret.file</code></td>
   <td>None</td>
@@ -94,6 +96,7 @@ the user mounts into their pods.
     contents of the file have been securely generated. This file is loaded on both the driver
     and the executors unless other settings override this (see below).
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.secret.driver.file</code></td>
@@ -106,6 +109,7 @@ the user mounts into their pods.
     and the executors can both use files to load the secret key. Ensure that the contents of the file
     on the driver is identical to the contents of the file on the executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.secret.executor.file</code></td>
@@ -118,6 +122,7 @@ the user mounts into their pods.
     and the executors can both use files to load the secret key. Ensure that the contents of the file
     on the driver is identical to the contents of the file on the executors.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -138,13 +143,14 @@ is still required when talking to shuffle services from Spark versions older tha
 The following table describes the different options available for configuring this feature.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.network.crypto.enabled</code></td>
   <td>false</td>
   <td>
     Enable AES-based RPC encryption, including the new authentication protocol added in 2.2.0.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.keyLength</code></td>
@@ -152,6 +158,7 @@ The following table describes the different options available for configuring th
   <td>
     The length in bits of the encryption key to generate. Valid values are 128, 192 and 256.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.keyFactoryAlgorithm</code></td>
@@ -160,6 +167,7 @@ The following table describes the different options available for configuring th
     The key factory algorithm to use when generating encryption keys. Should be one of the
     algorithms supported by the javax.crypto.SecretKeyFactory class in the JRE being used.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.config.*</code></td>
@@ -169,6 +177,7 @@ The following table describes the different options available for configuring th
     use. The config name should be the name of commons-crypto configuration without the
     <code>commons.crypto</code> prefix.
   </td>
+  <td></td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.saslFallback</code></td>
@@ -179,6 +188,7 @@ The following table describes the different options available for configuring th
     do not support the internal Spark authentication protocol. On the shuffle service side,
     disabling this feature will block older clients from authenticating.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.authenticate.enableSaslEncryption</code></td>
@@ -208,7 +218,7 @@ encrypting output data generated by applications with APIs such as `saveAsHadoop
 The following settings cover enabling encryption for data written to disk:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.io.encryption.enabled</code></td>
   <td>false</td>
@@ -216,6 +226,7 @@ The following settings cover enabling encryption for data written to disk:
     Enable local disk I/O encryption. Currently supported by all modes except Mesos. It's strongly
     recommended that RPC encryption be enabled when using this feature.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.io.encryption.keySizeBits</code></td>
@@ -223,6 +234,7 @@ The following settings cover enabling encryption for data written to disk:
   <td>
     IO encryption key size in bits. Supported values are 128, 192 and 256.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.io.encryption.keygen.algorithm</code></td>
@@ -232,6 +244,7 @@ The following settings cover enabling encryption for data written to disk:
     described in the KeyGenerator section of the Java Cryptography Architecture Standard Algorithm
     Name Documentation.
   </td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.io.encryption.commons.config.*</code></td>
@@ -241,6 +254,7 @@ The following settings cover enabling encryption for data written to disk:
     use. The config name should be the name of commons-crypto configuration without the
     <code>commons.crypto</code> prefix.
   </td>
+  <td>2.1.0</td>
 </tr>
 </table>
 

From afab532189a4e224f1334172b07e2f927f530a9f Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Mon, 23 Mar 2020 11:07:43 +0900
Subject: [PATCH 0449/1280] [SPARK-31002][CORE][DOC][FOLLOWUP] Add version
 information to the configuration of Core
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR follows up #27847, #27852 and https://github.com/apache/spark/pull/27913.

I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.storage.localDiskByExecutors.cacheSize | 3.0.0 | SPARK-27651 | fd2bf55abaab08798a428d4e47d4050ba2b82a95#diff-6bdad48cfc34314e89599655442ff210 |
spark.storage.memoryMapLimitForTests | 2.3.0 | SPARK-3151 | b8ffb51055108fd606b86f034747006962cd2df3#diff-abd96f2ae793cd6ea6aab5b96a3c1d7a |  
spark.barrier.sync.timeout | 2.4.0 | SPARK-24817 | 388f5a0635a2812cd71b08352e3ddc20293ec189#diff-6bdad48cfc34314e89599655442ff210 |
spark.scheduler.blacklist.unschedulableTaskSetTimeout | 2.4.1 | SPARK-22148 | 52e9711d01694158ecb3691f2ec25c0ebe4b0207#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.barrier.maxConcurrentTasksCheck.interval | 2.4.0 | SPARK-24819 | bfb74394a5513134ea1da9fcf4a1783b77dd64e4#diff-6bdad48cfc34314e89599655442ff210 |  
spark.scheduler.barrier.maxConcurrentTasksCheck.maxFailures | 2.4.0 | SPARK-24819 | bfb74394a5513134ea1da9fcf4a1783b77dd64e4#diff-6bdad48cfc34314e89599655442ff210 |  
spark.unsafe.exceptionOnMemoryLeak | 1.4.0 | SPARK-7076 and SPARK-7077 and SPARK-7080 | f49284b5bf3a69ed91a5e3e6e0ed3be93a6ab9e4#diff-5a0de266c82b95adb47d9bca714e1f1b |  
spark.unsafe.sorter.spill.read.ahead.enabled | 2.3.0 | SPARK-21113 | 1e978b17d63d7ba20368057aa4e65f5ef6e87369#diff-93a086317cea72a113cf81056882c206 |  
spark.unsafe.sorter.spill.reader.buffer.size | 2.1.0 | SPARK-16862 | c1937dd19a23bd096a4707656c7ba19fb5c16966#diff-93a086317cea72a113cf81056882c206 |  
spark.plugins | 3.0.0 | SPARK-29397 | d51d228048d519a9a666f48dc532625de13e7587#diff-6bdad48cfc34314e89599655442ff210 |  
spark.cleaner.periodicGC.interval | 1.6.0 | SPARK-8414 | 72da2a21f0940b97757ace5975535e559d627688#diff-75141521b1d55bc32d72b70032ad96c0 |
spark.cleaner.referenceTracking | 1.0.0 | SPARK-1103 | 11eabbe125b2ee572fad359c33c93f5e6fdf0b2d#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.cleaner.referenceTracking.blocking | 1.0.0 | SPARK-1103 | 11eabbe125b2ee572fad359c33c93f5e6fdf0b2d#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.cleaner.referenceTracking.blocking.shuffle | 1.1.1 | SPARK-3139 | 5cf1e440137006eedd6846ac8fa57ccf9fd1958d#diff-75141521b1d55bc32d72b70032ad96c0 |  
spark.cleaner.referenceTracking.cleanCheckpoints | 1.4.0 | SPARK-2033 | 25998e4d73bcc95ac85d9af71adfdc726ec89568#diff-440e866c5df0b8386aff57f9f8bd8db1 |  
spark.executor.logs.rolling.strategy | 1.1.0 | SPARK-1940 | 4823bf470ec1b47a6f404834d4453e61d3dcbec9#diff-2b4575e096e4db7165e087f9429f2a02 |
spark.executor.logs.rolling.time.interval | 1.1.0 | SPARK-1940 | 4823bf470ec1b47a6f404834d4453e61d3dcbec9#diff-2b4575e096e4db7165e087f9429f2a02 |
spark.executor.logs.rolling.maxSize | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.executor.logs.rolling.maxRetainedFiles | 1.1.0 | SPARK-1940 | 4823bf470ec1b47a6f404834d4453e61d3dcbec9#diff-2b4575e096e4db7165e087f9429f2a02 |
spark.executor.logs.rolling.enableCompression | 2.0.2 | SPARK-17711 | 26e978a93f029e1a1b5c7524d0b52c8141b70997#diff-2b4575e096e4db7165e087f9429f2a02 |  
spark.master.rest.enabled | 1.3.0 | SPARK-5388 | 6ec0cdc14390d4dc45acf31040f21e1efc476fc0#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.master.rest.port | 1.3.0 | SPARK-5388 | 6ec0cdc14390d4dc45acf31040f21e1efc476fc0#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.master.ui.port | 1.1.0 | SPARK-2857 | 12f99cf5f88faf94d9dbfe85cb72d0010a3a25ac#diff-366c88f47e9b5cfa4d4305febeb8b026 |  
spark.io.compression.snappy.blockSize | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.io.compression.lz4.blockSize | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.io.compression.codec | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-df9e6118c481ceb27faa399114fac0a1 |  
spark.io.compression.zstd.bufferSize | 2.3.0 | SPARK-19112 | 444bce1c98c45147fe63e2132e9743a0c5e49598#diff-df9e6118c481ceb27faa399114fac0a1 |  
spark.io.compression.zstd.level | 2.3.0 | SPARK-19112 | 444bce1c98c45147fe63e2132e9743a0c5e49598#diff-df9e6118c481ceb27faa399114fac0a1 |  
spark.io.warning.largeFileThreshold | 3.0.0 | SPARK-28366 | 26d03b62e20d053943d03b5c5573dd349e49654c#diff-6bdad48cfc34314e89599655442ff210 |  
spark.eventLog.compression.codec | 3.0.0 | SPARK-28118 | 47f54b1ec717d0d744bf3ad46bb1ed3542b667c8#diff-6bdad48cfc34314e89599655442ff210 |  
spark.buffer.size | 0.5.0 | None | 4b1646a25f7581cecae108553da13833e842e68a#diff-eaf125f56ce786d64dcef99cf446a751 |  
spark.locality.wait.process | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-264da78fe625d594eae59d1adabc8ae9 |  
spark.locality.wait.node | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-264da78fe625d594eae59d1adabc8ae9 |  
spark.locality.wait.rack | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-264da78fe625d594eae59d1adabc8ae9 |  
spark.reducer.maxSizeInFlight | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.reducer.maxReqsInFlight | 2.0.0 | SPARK-6166 | 894921d813a259f2f266fde7d86d2ecb5a0af24b#diff-eb30a71e0d04150b8e0b64929852e38b |  
spark.broadcast.compress | 0.6.0 | None | efc5423210d1aadeaea78273a4a8f10425753079#diff-76170a9c8f67b542bc58240a0a12fe08 |  
spark.broadcast.blockSize | 0.5.0 | None | b8ab7862b8bd168bca60bd930cd97c1099fbc8a8#diff-271d7958e14cdaa46cf3737cfcf51341 |  
spark.broadcast.checksum | 2.1.1 | SPARK-18188 | 06a56df226aa0c03c21f23258630d8a96385c696#diff-4f43d14923008c6650a8eb7b40c07f74 |
spark.broadcast.UDFCompressionThreshold | 3.0.0 | SPARK-28355 | 79e204770300dab4a669b9f8e2421ef905236e7b#diff-6bdad48cfc34314e89599655442ff210 |
spark.rdd.compress | 0.6.0 | None | efc5423210d1aadeaea78273a4a8f10425753079#diff-76170a9c8f67b542bc58240a0a12fe08 |  
spark.rdd.parallelListingThreshold | 2.0.0 | SPARK-9926 | 80a4bfa4d1c86398b90b26c34d8dcbc2355f5a6a#diff-eaababfc87ea4949f97860e8b89b7586 |
spark.rdd.limit.scaleUpFactor | 2.1.0 | SPARK-16984 | 806d8a8e980d8ba2f4261bceb393c40bafaa2f73#diff-1d55e54678eff2076263f2fe36150c17 |  
spark.serializer | 0.5.0 | None | fd1d255821bde844af28e897fabd59a715659038#diff-b920b65c23bf3a1b3326325b0d6a81b2 |  
spark.serializer.objectStreamReset | 1.0.0 | SPARK-942 | 40566e10aae4b21ffc71ea72702b8df118ac5c8e#diff-6a59dfc43d1b31dc1c3072ceafa829f5 |  
spark.serializer.extraDebugInfo | 1.3.0 | SPARK-5307 | 636408311deeebd77fb83d2249e0afad1a1ba149#diff-6a59dfc43d1b31dc1c3072ceafa829f5 |  
spark.jars | 0.9.0 | None | f1d206c6b4c0a5b2517b05af05fdda6049e2f7c2#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.files | 1.0.0 | None | 29ee101c73bf066bf7f4f8141c475b8d1bd3cf1c#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.submit.deployMode | 1.5.0 | SPARK-6797 | 7f487c8bde14dbdd244a3493ad11a129ef2bb327#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
spark.submit.pyFiles | 1.0.1 | SPARK-1549 | d7ddb26e1fa02e773999cc4a97c48d2cd1723956#diff-4d2ab44195558d5a9d5f15b8803ef39d |
spark.scheduler.allocation.file | 0.8.1 | None | 976fe60f7609d7b905a34f18743efabd966407f0#diff-9bc0105ee454005379abed710cd20ced |  
spark.scheduler.minRegisteredResourcesRatio | 1.1.1 | SPARK-2635 | 3311da2f9efc5ff2c7d01273ac08f719b067d11d#diff-7d99a7c7a051e5e851aaaefb275a44a1 |  
spark.scheduler.maxRegisteredResourcesWaitingTime | 1.1.1 | SPARK-2635 | 3311da2f9efc5ff2c7d01273ac08f719b067d11d#diff-7d99a7c7a051e5e851aaaefb275a44a1 |  
spark.scheduler.mode | 0.8.0 | None | 98fb69822cf780160bca51abeaab7c82e49fab54#diff-cb7a25b3c9a7341c6d99bcb8e9780c92 |  
spark.scheduler.revive.interval | 0.8.1 | None | d0c9d41a061969d409715b86a91937d8de4c29f7#diff-7d99a7c7a051e5e851aaaefb275a44a1 |  
spark.speculation | 0.6.0 | None | e72afdb817bcc8388aeb8b8d31628fd5fd67acf1#diff-4e188f32951dc989d97fa7577858bc7c |  
spark.speculation.interval | 0.6.0 | None | e72afdb817bcc8388aeb8b8d31628fd5fd67acf1#diff-4e188f32951dc989d97fa7577858bc7c |  
spark.speculation.multiplier | 0.6.0 | None | e72afdb817bcc8388aeb8b8d31628fd5fd67acf1#diff-fff59f72dfe6ca4ccb607ad12535da07 |  
spark.speculation.quantile | 0.6.0 | None | e72afdb817bcc8388aeb8b8d31628fd5fd67acf1#diff-fff59f72dfe6ca4ccb607ad12535da07 |  
spark.speculation.task.duration.threshold | 3.0.0 | SPARK-29976 | ad238a2238a9d0da89be4424574436cbfaee579d#diff-6bdad48cfc34314e89599655442ff210 |
spark.yarn.stagingDir | 2.0.0 | SPARK-13063 | bc36df127d3b9f56b4edaeb5eca7697d4aef761a#diff-14b8ed2ef4e3da985300b8d796a38fa9 |  
spark.buffer.pageSize | 1.5.0 | SPARK-9411 | 1b0099fc62d02ff6216a76fbfe17a4ec5b2f3536#diff-1b22e54318c04824a6d53ed3f4d1bb35 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Exists UT

Closes #27931 from beliefer/add-version-to-core-config-part-four.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/internal/config/package.scala       | 82 +++++++++++++++++--
 docs/configuration.md                         | 72 ++++++++++++----
 2 files changed, 134 insertions(+), 20 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 74a2e0af2952a..f70ee2e5c71c7 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1367,6 +1367,7 @@ package object config {
         "unbounded store. This cache will be used to avoid the network in case of fetching disk " +
         s"persisted RDD blocks or shuffle blocks " +
         s"(when `${SHUFFLE_HOST_LOCAL_DISK_READING_ENABLED.key}` is set) from the same host.")
+      .version("3.0.0")
       .intConf
       .createWithDefault(1000)
 
@@ -1374,6 +1375,7 @@ package object config {
     ConfigBuilder("spark.storage.memoryMapLimitForTests")
       .internal()
       .doc("For testing only, controls the size of chunks when memory mapping a file")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH)
 
@@ -1383,6 +1385,7 @@ package object config {
         "coordinator didn't receive all the sync messages from barrier tasks within the " +
         "configured time, throw a SparkException to fail all the tasks. The default value is set " +
         "to 31536000(3600 * 24 * 365) so the barrier() call shall wait for one year.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v > 0, "The value should be a positive time value.")
       .createWithDefaultString("365d")
@@ -1391,6 +1394,7 @@ package object config {
     ConfigBuilder("spark.scheduler.blacklist.unschedulableTaskSetTimeout")
       .doc("The timeout in seconds to wait to acquire a new executor and schedule a task " +
         "before aborting a TaskSet which is unschedulable because of being completely blacklisted.")
+      .version("2.4.1")
       .timeConf(TimeUnit.SECONDS)
       .checkValue(v => v >= 0, "The value should be a non negative time value.")
       .createWithDefault(120)
@@ -1405,6 +1409,7 @@ package object config {
         "configured max failure times for a job then fail current job submission. Note this " +
         "config only applies to jobs that contain one or more barrier stages, we won't perform " +
         "the check on non-barrier jobs.")
+      .version("2.4.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("15s")
 
@@ -1418,6 +1423,7 @@ package object config {
         "max failure times for a job then fail current job submission. Note this config only " +
         "applies to jobs that contain one or more barrier stages, we won't perform the check on " +
         "non-barrier jobs.")
+      .version("2.4.0")
       .intConf
       .checkValue(v => v > 0, "The max failures should be a positive value.")
       .createWithDefault(40)
@@ -1425,18 +1431,21 @@ package object config {
   private[spark] val UNSAFE_EXCEPTION_ON_MEMORY_LEAK =
     ConfigBuilder("spark.unsafe.exceptionOnMemoryLeak")
       .internal()
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val UNSAFE_SORTER_SPILL_READ_AHEAD_ENABLED =
     ConfigBuilder("spark.unsafe.sorter.spill.read.ahead.enabled")
       .internal()
+      .version("2.3.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val UNSAFE_SORTER_SPILL_READER_BUFFER_SIZE =
     ConfigBuilder("spark.unsafe.sorter.spill.reader.buffer.size")
       .internal()
+      .version("2.1.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => 1024 * 1024 <= v && v <= MAX_BUFFER_SIZE_BYTES,
         s"The value must be in allowed range [1,048,576, ${MAX_BUFFER_SIZE_BYTES}].")
@@ -1449,63 +1458,83 @@ package object config {
       .withPrepended(DEFAULT_PLUGINS_LIST, separator = ",")
       .doc("Comma-separated list of class names implementing " +
         "org.apache.spark.api.plugin.SparkPlugin to load into the application.")
+      .version("3.0.0")
       .stringConf
       .toSequence
       .createWithDefault(Nil)
 
   private[spark] val CLEANER_PERIODIC_GC_INTERVAL =
     ConfigBuilder("spark.cleaner.periodicGC.interval")
+      .version("1.6.0")
       .timeConf(TimeUnit.SECONDS)
       .createWithDefaultString("30min")
 
   private[spark] val CLEANER_REFERENCE_TRACKING =
     ConfigBuilder("spark.cleaner.referenceTracking")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking")
+      .version("1.0.0")
       .booleanConf
       .createWithDefault(true)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_BLOCKING_SHUFFLE =
     ConfigBuilder("spark.cleaner.referenceTracking.blocking.shuffle")
+      .version("1.1.1")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val CLEANER_REFERENCE_TRACKING_CLEAN_CHECKPOINTS =
     ConfigBuilder("spark.cleaner.referenceTracking.cleanCheckpoints")
+      .version("1.4.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_STRATEGY =
-    ConfigBuilder("spark.executor.logs.rolling.strategy").stringConf.createWithDefault("")
+    ConfigBuilder("spark.executor.logs.rolling.strategy")
+      .version("1.1.0")
+      .stringConf
+      .createWithDefault("")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_TIME_INTERVAL =
-    ConfigBuilder("spark.executor.logs.rolling.time.interval").stringConf.createWithDefault("daily")
+    ConfigBuilder("spark.executor.logs.rolling.time.interval")
+      .version("1.1.0")
+      .stringConf
+      .createWithDefault("daily")
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_SIZE =
     ConfigBuilder("spark.executor.logs.rolling.maxSize")
+      .version("1.4.0")
       .stringConf
       .createWithDefault((1024 * 1024).toString)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_MAX_RETAINED_FILES =
-    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles").intConf.createWithDefault(-1)
+    ConfigBuilder("spark.executor.logs.rolling.maxRetainedFiles")
+      .version("1.1.0")
+      .intConf
+      .createWithDefault(-1)
 
   private[spark] val EXECUTOR_LOGS_ROLLING_ENABLE_COMPRESSION =
     ConfigBuilder("spark.executor.logs.rolling.enableCompression")
+      .version("2.0.2")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_ENABLED = ConfigBuilder("spark.master.rest.enabled")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(false)
 
   private[spark] val MASTER_REST_SERVER_PORT = ConfigBuilder("spark.master.rest.port")
+    .version("1.3.0")
     .intConf
     .createWithDefault(6066)
 
   private[spark] val MASTER_UI_PORT = ConfigBuilder("spark.master.ui.port")
+    .version("1.1.0")
     .intConf
     .createWithDefault(8080)
 
@@ -1514,6 +1543,7 @@ package object config {
       .doc("Block size in bytes used in Snappy compression, in the case when " +
         "Snappy compression codec is used. Lowering this block size " +
         "will also lower shuffle memory usage when Snappy is used")
+      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1522,6 +1552,7 @@ package object config {
       .doc("Block size in bytes used in LZ4 compression, in the case when LZ4 compression" +
         "codec is used. Lowering this block size will also lower shuffle memory " +
         "usage when LZ4 is used.")
+      .version("1.4.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1531,6 +1562,7 @@ package object config {
         "broadcast variables and shuffle outputs. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec")
+      .version("0.8.0")
       .stringConf
       .createWithDefaultString("lz4")
 
@@ -1540,6 +1572,7 @@ package object config {
         "compression codec is used. Lowering this size will lower the shuffle " +
         "memory usage when Zstd is used, but it might increase the compression " +
         "cost because of excessive JNI call overhead")
+      .version("2.3.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefaultString("32k")
 
@@ -1547,6 +1580,7 @@ package object config {
     ConfigBuilder("spark.io.compression.zstd.level")
       .doc("Compression level for Zstd compression codec. Increasing the compression " +
         "level will result in better compression at the expense of more CPU and memory")
+      .version("2.3.0")
       .intConf
       .createWithDefault(1)
 
@@ -1555,6 +1589,7 @@ package object config {
       .internal()
       .doc("If the size in bytes of a file loaded by Spark exceeds this threshold, " +
         "a warning is logged with the possible reasons.")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .createWithDefault(1024 * 1024 * 1024)
 
@@ -1563,28 +1598,34 @@ package object config {
       .doc("The codec used to compress event log. By default, Spark provides four codecs: " +
         "lz4, lzf, snappy, and zstd. You can also use fully qualified class names to specify " +
         "the codec. If this is not given, spark.io.compression.codec will be used.")
+      .version("3.0.0")
       .fallbackConf(IO_COMPRESSION_CODEC)
 
   private[spark] val BUFFER_SIZE =
     ConfigBuilder("spark.buffer.size")
+      .version("0.5.0")
       .intConf
       .checkValue(_ >= 0, "The buffer size must not be negative")
       .createWithDefault(65536)
 
   private[spark] val LOCALITY_WAIT_PROCESS = ConfigBuilder("spark.locality.wait.process")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_NODE = ConfigBuilder("spark.locality.wait.node")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
   private[spark] val LOCALITY_WAIT_RACK = ConfigBuilder("spark.locality.wait.rack")
+    .version("0.8.0")
     .fallbackConf(LOCALITY_WAIT)
 
-    private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
+  private[spark] val REDUCER_MAX_SIZE_IN_FLIGHT = ConfigBuilder("spark.reducer.maxSizeInFlight")
     .doc("Maximum size of map outputs to fetch simultaneously from each reduce task, " +
       "in MiB unless otherwise specified. Since each output requires us to create a " +
       "buffer to receive it, this represents a fixed memory overhead per reduce task, " +
       "so keep it small unless you have a large amount of memory")
+    .version("1.4.0")
     .bytesConf(ByteUnit.MiB)
     .createWithDefaultString("48m")
 
@@ -1594,12 +1635,14 @@ package object config {
       "it might lead to very large number of inbound connections to one or more nodes, " +
       "causing the workers to fail under load. By allowing it to limit the number of " +
       "fetch requests, this scenario can be mitigated")
+    .version("2.0.0")
     .intConf
     .createWithDefault(Int.MaxValue)
 
   private[spark] val BROADCAST_COMPRESS = ConfigBuilder("spark.broadcast.compress")
     .doc("Whether to compress broadcast variables before sending them. " +
       "Generally a good idea. Compression will use spark.io.compression.codec")
+    .version("0.6.0")
     .booleanConf.createWithDefault(true)
 
   private[spark] val BROADCAST_BLOCKSIZE = ConfigBuilder("spark.broadcast.blockSize")
@@ -1607,6 +1650,7 @@ package object config {
       "KiB unless otherwise specified. Too large a value decreases " +
       "parallelism during broadcast (makes it slower); however, " +
       "if it is too small, BlockManager might take a performance hit")
+    .version("0.5.0")
     .bytesConf(ByteUnit.KiB)
     .createWithDefaultString("4m")
 
@@ -1616,12 +1660,15 @@ package object config {
       "corrupted blocks, at the cost of computing and sending a little " +
       "more data. It's possible to disable it if the network has other " +
       "mechanisms to guarantee data won't be corrupted during broadcast")
-    .booleanConf.createWithDefault(true)
+    .version("2.1.1")
+    .booleanConf
+    .createWithDefault(true)
 
   private[spark] val BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD =
     ConfigBuilder("spark.broadcast.UDFCompressionThreshold")
       .doc("The threshold at which user-defined functions (UDFs) and Python RDD commands " +
         "are compressed by broadcast in bytes unless otherwise specified")
+      .version("3.0.0")
       .bytesConf(ByteUnit.BYTE)
       .checkValue(v => v >= 0, "The threshold should be non-negative.")
       .createWithDefault(1L * 1024 * 1024)
@@ -1632,92 +1679,112 @@ package object config {
       "or StorageLevel.MEMORY_ONLY in Python). Can save substantial " +
       "space at the cost of some extra CPU time. " +
       "Compression will use spark.io.compression.codec")
-    .booleanConf.createWithDefault(false)
+    .version("0.6.0")
+    .booleanConf
+    .createWithDefault(false)
 
   private[spark] val RDD_PARALLEL_LISTING_THRESHOLD =
     ConfigBuilder("spark.rdd.parallelListingThreshold")
+      .version("2.0.0")
       .intConf
       .createWithDefault(10)
 
   private[spark] val RDD_LIMIT_SCALE_UP_FACTOR =
     ConfigBuilder("spark.rdd.limit.scaleUpFactor")
+      .version("2.1.0")
       .intConf
       .createWithDefault(4)
 
   private[spark] val SERIALIZER = ConfigBuilder("spark.serializer")
+    .version("0.5.0")
     .stringConf
     .createWithDefault("org.apache.spark.serializer.JavaSerializer")
 
   private[spark] val SERIALIZER_OBJECT_STREAM_RESET =
     ConfigBuilder("spark.serializer.objectStreamReset")
+      .version("1.0.0")
       .intConf
       .createWithDefault(100)
 
   private[spark] val SERIALIZER_EXTRA_DEBUG_INFO = ConfigBuilder("spark.serializer.extraDebugInfo")
+    .version("1.3.0")
     .booleanConf
     .createWithDefault(true)
 
   private[spark] val JARS = ConfigBuilder("spark.jars")
+    .version("0.9.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val FILES = ConfigBuilder("spark.files")
+    .version("1.0.0")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SUBMIT_DEPLOY_MODE = ConfigBuilder("spark.submit.deployMode")
+    .version("1.5.0")
     .stringConf
     .createWithDefault("client")
 
   private[spark] val SUBMIT_PYTHON_FILES = ConfigBuilder("spark.submit.pyFiles")
+    .version("1.0.1")
     .stringConf
     .toSequence
     .createWithDefault(Nil)
 
   private[spark] val SCHEDULER_ALLOCATION_FILE =
     ConfigBuilder("spark.scheduler.allocation.file")
+      .version("0.8.1")
       .stringConf
       .createOptional
 
   private[spark] val SCHEDULER_MIN_REGISTERED_RESOURCES_RATIO =
     ConfigBuilder("spark.scheduler.minRegisteredResourcesRatio")
+      .version("1.1.1")
       .doubleConf
       .createOptional
 
   private[spark] val SCHEDULER_MAX_REGISTERED_RESOURCE_WAITING_TIME =
     ConfigBuilder("spark.scheduler.maxRegisteredResourcesWaitingTime")
+      .version("1.1.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("30s")
 
   private[spark] val SCHEDULER_MODE =
     ConfigBuilder("spark.scheduler.mode")
+      .version("0.8.0")
       .stringConf
       .createWithDefault(SchedulingMode.FIFO.toString)
 
   private[spark] val SCHEDULER_REVIVE_INTERVAL =
     ConfigBuilder("spark.scheduler.revive.interval")
+      .version("0.8.1")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val SPECULATION_ENABLED =
     ConfigBuilder("spark.speculation")
+      .version("0.6.0")
       .booleanConf
       .createWithDefault(false)
 
   private[spark] val SPECULATION_INTERVAL =
     ConfigBuilder("spark.speculation.interval")
+      .version("0.6.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefault(100)
 
   private[spark] val SPECULATION_MULTIPLIER =
     ConfigBuilder("spark.speculation.multiplier")
+      .version("0.6.0")
       .doubleConf
       .createWithDefault(1.5)
 
   private[spark] val SPECULATION_QUANTILE =
     ConfigBuilder("spark.speculation.quantile")
+      .version("0.6.0")
       .doubleConf
       .createWithDefault(0.75)
 
@@ -1731,16 +1798,19 @@ package object config {
         "large enough. E.g. tasks might be re-launched if there are enough successful runs " +
         "even though the threshold hasn't been reached. The number of slots is computed based " +
         "on the conf values of spark.executor.cores and spark.task.cpus minimum 1.")
+      .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .createOptional
 
   private[spark] val STAGING_DIR = ConfigBuilder("spark.yarn.stagingDir")
     .doc("Staging directory used while submitting applications.")
+    .version("2.0.0")
     .stringConf
     .createOptional
 
   private[spark] val BUFFER_PAGESIZE = ConfigBuilder("spark.buffer.pageSize")
     .doc("The amount of memory used per page in bytes")
+    .version("1.5.0")
     .bytesConf(ByteUnit.BYTE)
     .createOptional
 
diff --git a/docs/configuration.md b/docs/configuration.md
index e4ea25c05c23f..6d018979ababa 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -361,6 +361,7 @@ of the most common options to set are:
     Which means to launch driver program locally ("client")
     or remotely ("cluster") on one of the nodes inside the cluster.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.log.callerContext</code></td>
@@ -575,6 +576,7 @@ Apart from these, the following properties are also available, and may be useful
     Sets the number of latest rolling log files that are going to be retained by the system.
     Older log files will be deleted. Disabled by default.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.enableCompression</code></td>
@@ -583,6 +585,7 @@ Apart from these, the following properties are also available, and may be useful
     Enable executor log compression. If it is enabled, the rolled executor logs will be compressed.
     Disabled by default.
   </td>
+  <td>2.0.2</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.maxSize</code></td>
@@ -592,6 +595,7 @@ Apart from these, the following properties are also available, and may be useful
     Rolling is disabled by default. See <code>spark.executor.logs.rolling.maxRetainedFiles</code>
     for automatic cleaning of old logs.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.strategy</code></td>
@@ -603,6 +607,7 @@ Apart from these, the following properties are also available, and may be useful
     For "size", use <code>spark.executor.logs.rolling.maxSize</code> to set
     the maximum file size for rolling.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.logs.rolling.time.interval</code></td>
@@ -613,6 +618,7 @@ Apart from these, the following properties are also available, and may be useful
     any interval in seconds. See <code>spark.executor.logs.rolling.maxRetainedFiles</code>
     for automatic cleaning of old logs.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.userClassPathFirst</code></td>
@@ -692,6 +698,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Comma-separated list of files to be placed in the working directory of each executor. Globs are allowed.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.submit.pyFiles</code></td>
@@ -699,6 +706,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Comma-separated list of .zip, .egg, or .py files to place on the PYTHONPATH for Python apps. Globs are allowed.
   </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.jars</code></td>
@@ -706,6 +714,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Comma-separated list of jars to include on the driver and executor classpaths. Globs are allowed.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.packages</code></td>
@@ -778,7 +787,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Shuffle Behavior
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.reducer.maxSizeInFlight</code></td>
   <td>48m</td>
@@ -788,6 +797,7 @@ Apart from these, the following properties are also available, and may be useful
     represents a fixed memory overhead per reduce task, so keep it small unless you have a
     large amount of memory.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.reducer.maxReqsInFlight</code></td>
@@ -798,6 +808,7 @@ Apart from these, the following properties are also available, and may be useful
     of inbound connections to one or more nodes, causing the workers to fail under load.
     By allowing it to limit the number of fetch requests, this scenario can be mitigated.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.reducer.maxBlocksInFlightPerAddress</code></td>
@@ -998,6 +1009,7 @@ Apart from these, the following properties are also available, and may be useful
     The codec to compress logged events. If this is not given,
     <code>spark.io.compression.codec</code> will be used.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.eventLog.erasureCoding.enabled</code></td>
@@ -1278,6 +1290,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to compress broadcast variables before sending them. Generally a good idea.
     Compression will use <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.checkpoint.compress</code></td>
@@ -1301,6 +1314,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>org.apache.spark.io.SnappyCompressionCodec</code>,
     and <code>org.apache.spark.io.ZStdCompressionCodec</code>.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.lz4.blockSize</code></td>
@@ -1310,6 +1324,7 @@ Apart from these, the following properties are also available, and may be useful
     is used. Lowering this block size will also lower shuffle memory usage when LZ4 is used.
     Default unit is bytes, unless otherwise specified.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.snappy.blockSize</code></td>
@@ -1319,6 +1334,7 @@ Apart from these, the following properties are also available, and may be useful
     Lowering this block size will also lower shuffle memory usage when Snappy is used.
     Default unit is bytes, unless otherwise specified.
   </td>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.zstd.level</code></td>
@@ -1327,6 +1343,7 @@ Apart from these, the following properties are also available, and may be useful
     Compression level for Zstd compression codec. Increasing the compression level will result in better
     compression at the expense of more CPU and memory.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.io.compression.zstd.bufferSize</code></td>
@@ -1336,6 +1353,7 @@ Apart from these, the following properties are also available, and may be useful
     is used. Lowering this size will lower the shuffle memory usage when Zstd is used, but it
     might increase the compression cost because of excessive JNI call overhead.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.classesToRegister</code></td>
@@ -1345,7 +1363,7 @@ Apart from these, the following properties are also available, and may be useful
     with Kryo.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
-  <th>1.2.0</th>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.referenceTracking</code></td>
@@ -1356,7 +1374,7 @@ Apart from these, the following properties are also available, and may be useful
     copies of the same object. Can be disabled to improve performance if you know this is not the
     case.
   </td>
-  <th>0.8.0</th>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.registrationRequired</code></td>
@@ -1368,7 +1386,7 @@ Apart from these, the following properties are also available, and may be useful
     significant performance overhead, so enabling this option can enforce strictly that a
     user has not omitted classes from registration.
   </td>
-  <th>1.1.0</th>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.registrator</code></td>
@@ -1382,7 +1400,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>KryoRegistrator</code></a>.
     See the <a href="tuning.html#data-serialization">tuning guide</a> for more details.
   </td>
-  <th>0.5.0</th>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.kryo.unsafe</code></td>
@@ -1391,7 +1409,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to use unsafe based Kryo serializer. Can be
     substantially faster by using Unsafe Based IO.
   </td>
-  <th>2.1.0</th>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.kryoserializer.buffer.max</code></td>
@@ -1401,7 +1419,7 @@ Apart from these, the following properties are also available, and may be useful
     This must be larger than any object you attempt to serialize and must be less than 2048m.
     Increase this if you get a "buffer limit exceeded" exception inside Kryo.
   </td>
-  <th>1.4.0</th>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.kryoserializer.buffer</code></td>
@@ -1411,7 +1429,7 @@ Apart from these, the following properties are also available, and may be useful
     Note that there will be one buffer <i>per core</i> on each worker. This buffer will grow up to
     <code>spark.kryoserializer.buffer.max</code> if needed.
   </td>
-  <th>1.4.0</th>
+  <td>1.4.0</td>
 </tr>
 <tr>
   <td><code>spark.rdd.compress</code></td>
@@ -1423,6 +1441,7 @@ Apart from these, the following properties are also available, and may be useful
     Can save substantial space at the cost of some extra CPU time.
     Compression will use <code>spark.io.compression.codec</code>.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.serializer</code></td>
@@ -1438,6 +1457,7 @@ Apart from these, the following properties are also available, and may be useful
     <a href="api/scala/org/apache/spark/serializer/Serializer.html">
     <code>org.apache.spark.Serializer</code></a>.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.serializer.objectStreamReset</code></td>
@@ -1449,13 +1469,14 @@ Apart from these, the following properties are also available, and may be useful
     objects to be collected. To turn off this periodic reset set it to -1.
     By default it will reset the serializer every 100 objects.
   </td>
+  <td>1.0.0</td>
 </tr>
 </table>
 
 ### Memory Management
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.memory.fraction</code></td>
   <td>0.6</td>
@@ -1522,6 +1543,7 @@ Apart from these, the following properties are also available, and may be useful
     on the driver, this may happen very occasionally or not at all. Not cleaning at all may
     lead to executors running out of disk space after a while.
   </td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking</code></td>
@@ -1529,6 +1551,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Enables or disables context cleaning.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking.blocking</code></td>
@@ -1537,6 +1560,7 @@ Apart from these, the following properties are also available, and may be useful
     Controls whether the cleaning thread should block on cleanup tasks (other than shuffle, which is controlled by
     <code>spark.cleaner.referenceTracking.blocking.shuffle</code> Spark property).
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking.blocking.shuffle</code></td>
@@ -1544,6 +1568,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Controls whether the cleaning thread should block on shuffle cleanup tasks.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.cleaner.referenceTracking.cleanCheckpoints</code></td>
@@ -1551,13 +1576,14 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Controls whether to clean checkpoint files if the reference is out of scope.
   </td>
+  <td>1.4.0</td>
 </tr>
 </table>
 
 ### Execution Behavior
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.broadcast.blockSize</code></td>
   <td>4m</td>
@@ -1566,6 +1592,7 @@ Apart from these, the following properties are also available, and may be useful
     specified. Too large a value decreases parallelism during broadcast (makes it slower); however, 
     if it is too small, <code>BlockManager</code> might take a performance hit.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.broadcast.checksum</code></td>
@@ -1575,6 +1602,7 @@ Apart from these, the following properties are also available, and may be useful
     help detect corrupted blocks, at the cost of computing and sending a little more data. It's possible
     to disable it if the network has other mechanisms to guarantee data won't be corrupted during broadcast.
   </td>
+  <td>2.1.1</td>
 </tr>
 <tr>
   <td><code>spark.executor.cores</code></td>
@@ -1905,14 +1933,14 @@ Apart from these, the following properties are also available, and may be useful
     For users who enabled external shuffle service, this feature can only work when
     external shuffle service is at least 2.3.0.
   </td>
-  <td></td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
 ### Scheduling
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.cores.max</code></td>
   <td>(not set)</td>
@@ -1946,6 +1974,7 @@ Apart from these, the following properties are also available, and may be useful
     Customize the locality wait for node locality. For example, you can set this to 0 to skip
     node locality and search immediately for rack locality (if your cluster has rack information).
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait.process</code></td>
@@ -1954,6 +1983,7 @@ Apart from these, the following properties are also available, and may be useful
     Customize the locality wait for process locality. This affects tasks that attempt to access
     cached data in a particular executor process.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.locality.wait.rack</code></td>
@@ -1961,6 +1991,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Customize the locality wait for rack locality.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.maxRegisteredResourcesWaitingTime</code></td>
@@ -1968,6 +1999,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Maximum amount of time to wait for resources to register before scheduling begins.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.minRegisteredResourcesRatio</code></td>
@@ -1981,6 +2013,7 @@ Apart from these, the following properties are also available, and may be useful
     the maximum amount of time it will wait before scheduling begins is controlled by config
     <code>spark.scheduler.maxRegisteredResourcesWaitingTime</code>.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.mode</code></td>
@@ -1991,6 +2024,7 @@ Apart from these, the following properties are also available, and may be useful
     to use fair sharing instead of queueing jobs one after another. Useful for
     multi-user services.
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.revive.interval</code></td>
@@ -1998,6 +2032,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     The interval length for the scheduler to revive the worker resource offers to run tasks.
   </td>
+  <td>0.8.1</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.capacity</code></td>
@@ -2063,6 +2098,7 @@ Apart from these, the following properties are also available, and may be useful
     The timeout in seconds to wait to acquire a new executor and schedule a task before aborting a
     TaskSet which is unschedulable because of being completely blacklisted.
   </td>
+  <td>2.4.1</td>
 </tr>
 <tr>
   <td><code>spark.blacklist.enabled</code></td>
@@ -2173,6 +2209,7 @@ Apart from these, the following properties are also available, and may be useful
     If set to "true", performs speculative execution of tasks. This means if one or more tasks are
     running slowly in a stage, they will be re-launched.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.interval</code></td>
@@ -2180,6 +2217,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How often Spark will check for tasks to speculate.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.multiplier</code></td>
@@ -2187,6 +2225,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many times slower a task is than the median to be considered for speculation.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.quantile</code></td>
@@ -2194,6 +2233,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Fraction of tasks which must be complete before speculation is enabled for a particular stage.
   </td>
+  <td>0.6.0</td>
 </tr>
 <tr>
   <td><code>spark.speculation.task.duration.threshold</code></td>
@@ -2208,6 +2248,7 @@ Apart from these, the following properties are also available, and may be useful
     the conf values of spark.executor.cores and spark.task.cpus minimum 1.
     Default unit is bytes, unless otherwise specified.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.task.cpus</code></td>
@@ -2301,7 +2342,7 @@ Apart from these, the following properties are also available, and may be useful
 ### Barrier Execution Mode
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.barrier.sync.timeout</code></td>
   <td>365d</td>
@@ -2311,6 +2352,7 @@ Apart from these, the following properties are also available, and may be useful
     configured time, throw a SparkException to fail all the tasks. The default value is set
     to 31536000(3600 * 24 * 365) so the <code>barrier()</code> call shall wait for one year.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.barrier.maxConcurrentTasksCheck.interval</code></td>
@@ -2325,6 +2367,7 @@ Apart from these, the following properties are also available, and may be useful
     config only applies to jobs that contain one or more barrier stages, we won't perform
     the check on non-barrier jobs.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.barrier.maxConcurrentTasksCheck.maxFailures</code></td>
@@ -2339,13 +2382,14 @@ Apart from these, the following properties are also available, and may be useful
     applies to jobs that contain one or more barrier stages, we won't perform the check on
     non-barrier jobs.
   </td>
+  <td>2.4.0</td>
 </tr>
 </table>
 
 ### Dynamic Allocation
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.dynamicAllocation.enabled</code></td>
   <td>false</td>

From dec53bedbd4acbde0b7a6064476dbd148e4c6d08 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 31 Mar 2020 12:33:46 +0900
Subject: [PATCH 0450/1280] [SPARK-31295][DOC] Supplement version for
 configuration appear in doc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR supplements version for configuration appear in docs.
I sorted out some information show below.

**docs/spark-standalone.md**
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.deploy.retainedApplications | 0.8.0 | None | 46eecd110a4017ea0c86cbb1010d0ccd6a5eb2ef#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.deploy.retainedDrivers | 1.1.0 | None | 7446f5ff93142d2dd5c79c63fa947f47a1d4db8b#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.deploy.spreadOut | 0.6.1 | None | bb2b9ff37cd2503cc6ea82c5dd395187b0910af0#diff-0e7ae91819fc8f7b47b0f97be7116325 |  
spark.deploy.defaultCores | 0.9.0 | None | d8bcc8e9a095c1b20dd7a17b6535800d39bff80e#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.deploy.maxExecutorRetries | 1.6.3 | SPARK-16956 | ace458f0330f22463ecf7cbee7c0465e10fba8a8#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.worker.resource.{resourceName}.amount | 3.0.0 | SPARK-27371 | cbad616d4cb0c58993a88df14b5e30778c7f7e85#diff-d25032e4a3ae1b85a59e4ca9ccf189a8 |  
spark.worker.resource.{resourceName}.discoveryScript | 3.0.0 | SPARK-27371 | cbad616d4cb0c58993a88df14b5e30778c7f7e85#diff-d25032e4a3ae1b85a59e4ca9ccf189a8 |  
spark.worker.resourcesFile | 3.0.0 | SPARK-27369 | 7cbe01e8efc3f6cd3a0cac4bcfadea8fcc74a955#diff-b2fc8d6ab7ac5735085e2d6cfacb95da |  
spark.shuffle.service.db.enabled | 3.0.0 | SPARK-26288 | 8b0aa59218c209d39cbba5959302d8668b885cf6#diff-6bdad48cfc34314e89599655442ff210 |  
spark.storage.cleanupFilesAfterExecutorExit | 2.4.0 | SPARK-24340 | 8ef167a5f9ba8a79bb7ca98a9844fe9cfcfea060#diff-916ca56b663f178f302c265b7ef38499 |  
spark.deploy.recoveryMode | 0.8.1 | None | d66c01f2b6defb3db6c1be99523b734a4d960532#diff-29dffdccd5a7f4c8b496c293e87c8668 |  
spark.deploy.recoveryDirectory | 0.8.1 | None | d66c01f2b6defb3db6c1be99523b734a4d960532#diff-29dffdccd5a7f4c8b496c293e87c8668 |  

**docs/sql-data-sources-avro.md**
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.legacy.replaceDatabricksSparkAvro.enabled | 2.4.0 | SPARK-25129 | ac0174e55af2e935d41545721e9f430c942b3a0c#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.avro.compression.codec | 2.4.0 | SPARK-24881 | 0a0f68bae6c0a1bf30184b1e9ac6bf3805bd7511#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.avro.deflate.level | 2.4.0 | SPARK-24881 | 0a0f68bae6c0a1bf30184b1e9ac6bf3805bd7511#diff-9a6b543db706f1a90f790783d6930a13 |  

**docs/sql-data-sources-orc.md**
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.orc.impl | 2.3.0 | SPARK-20728 | 326f1d6728a7734c228d8bfaa69442a1c7b92e9b#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.orc.enableVectorizedReader | 2.3.0 | SPARK-16060 | 60f6b994505e3f82091a04eed2dc0a9e8bd523ce#diff-9a6b543db706f1a90f790783d6930a13 |  

**docs/sql-data-sources-parquet.md**
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.parquet.binaryAsString | 1.1.1 | SPARK-2927 | de501e169f24e4573747aec85b7651c98633c028#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.int96AsTimestamp | 1.3.0 | SPARK-4987 | 67d52207b5cf2df37ca70daff2a160117510f55e#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.compression.codec | 1.1.1 | SPARK-3131 | 3a9d874d7a46ab8b015631d91ba479d9a0ba827f#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.filterPushdown | 1.2.0 | SPARK-4391 | 576688aa2a19bd4ba239a2b93af7947f983e5124#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.hive.convertMetastoreParquet | 1.1.1 | SPARK-2406 | cc4015d2fa3785b92e6ab079b3abcf17627f7c56#diff-ff50aea397a607b79df9bec6f2a841db |  
spark.sql.parquet.mergeSchema | 1.5.0 | SPARK-8690 | 246265f2bb056d5e9011d3331b809471a24ff8d7#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.parquet.writeLegacyFormat | 1.6.0 | SPARK-10400 | 01cd688f5245cbb752863100b399b525b31c3510#diff-41ef65b9ef5b518f77e2a03559893f4d |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test

Closes #28064 from beliefer/supplement-doc-for-data-sources.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/spark-standalone.md         | 13 ++++++++++++-
 docs/sql-data-sources-avro.md    | 21 +++++++++++++++++----
 docs/sql-data-sources-orc.md     | 16 +++++++++++++---
 docs/sql-data-sources-parquet.md |  9 ++++++++-
 4 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 4d4b85e31c8ff..2c2ed53b478c3 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -192,6 +192,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     The maximum number of completed applications to display. Older applications will be dropped from the UI to maintain this limit.<br/>
   </td>
+  <td>0.8.0</td>
 </tr>
 <tr>
   <td><code>spark.deploy.retainedDrivers</code></td>
@@ -199,6 +200,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
    The maximum number of completed drivers to display. Older drivers will be dropped from the UI to maintain this limit.<br/>
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.deploy.spreadOut</code></td>
@@ -208,6 +210,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     to consolidate them onto as few nodes as possible. Spreading out is usually better for
     data locality in HDFS, but consolidating is more efficient for compute-intensive workloads. <br/>
   </td>
+  <td>0.6.1</td>
 </tr>
 <tr>
   <td><code>spark.deploy.defaultCores</code></td>
@@ -219,6 +222,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Set this lower on a shared cluster to prevent users from grabbing
     the whole cluster by default. <br/>
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.deploy.maxExecutorRetries</code></td>
@@ -234,6 +238,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     <code>-1</code>.
     <br/>
   </td>
+  <td>1.6.3</td>
 </tr>
 <tr>
   <td><code>spark.worker.timeout</code></td>
@@ -250,6 +255,7 @@ SPARK_MASTER_OPTS supports the following system properties:
   <td>
     Amount of a particular resource to use on the worker.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.resource.{resourceName}.discoveryScript</code></td>
@@ -258,6 +264,7 @@ SPARK_MASTER_OPTS supports the following system properties:
     Path to resource discovery script, which is used to find a particular resource while worker starting up.
     And the output of the script should be formatted like the <code>ResourceInformation</code> class.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.resourcesFile</code></td>
@@ -317,6 +324,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     enabled).  You should also enable <code>spark.worker.cleanup.enabled</code>, to ensure that the state
     eventually gets cleaned up.  This config may be removed in the future.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.storage.cleanupFilesAfterExecutorExit</code></td>
@@ -329,6 +337,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     all files/subdirectories of a stopped and timeout application.
     This only affects Standalone mode, support of other cluster manangers can be added in the future.
   </td>
+  <td>2.4.0</td>
 </tr>
 <tr>
   <td><code>spark.worker.ui.compressedLogFileLengthCacheSize</code></td>
@@ -490,14 +499,16 @@ ZooKeeper is the best way to go for production-level high availability, but if y
 In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env using this configuration:
 
 <table class="table">
-  <tr><th style="width:21%">System property</th><th>Meaning</th></tr>
+  <tr><th style="width:21%">System property</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.deploy.recoveryMode</code></td>
     <td>Set to FILESYSTEM to enable single-node recovery mode (default: NONE).</td>
+    <td>0.8.1</td>
   </tr>
   <tr>
     <td><code>spark.deploy.recoveryDirectory</code></td>
     <td>The directory in which Spark will store recovery state, accessible from the Master's perspective.</td>
+    <td>0.8.1</td>
   </tr>
 </table>
 
diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index 8e6a4079cd5de..d926ae7703268 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -258,21 +258,34 @@ Data source options of Avro can be set via:
 ## Configuration
 Configuration of Avro can be done using the `setConf` method on SparkSession or by running `SET key=value` commands using SQL.
 <table class="table">
-  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th></tr>
+  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Since Version</b></th></tr>
   <tr>
     <td>spark.sql.legacy.replaceDatabricksSparkAvro.enabled</td>
     <td>true</td>
-    <td>If it is set to true, the data source provider <code>com.databricks.spark.avro</code> is mapped to the built-in but external Avro data source module for backward compatibility.</td>
+    <td>
+      If it is set to true, the data source provider <code>com.databricks.spark.avro</code> is mapped
+      to the built-in but external Avro data source module for backward compatibility.
+    </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.sql.avro.compression.codec</td>
     <td>snappy</td>
-    <td>Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate, snappy, bzip2 and xz. Default codec is snappy.</td>
+    <td>
+      Compression codec used in writing of AVRO files. Supported codecs: uncompressed, deflate,
+      snappy, bzip2 and xz. Default codec is snappy.
+    </td>
+    <td>2.4.0</td>
   </tr>
   <tr>
     <td>spark.sql.avro.deflate.level</td>
     <td>-1</td>
-    <td>Compression level for the deflate codec used in writing of AVRO files. Valid value must be in the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level in the current implementation.</td>
+    <td>
+      Compression level for the deflate codec used in writing of AVRO files. Valid value must be in
+      the range of from 1 to 9 inclusive or -1. The default value is -1 which corresponds to 6 level
+      in the current implementation.
+    </td>
+    <td>2.4.0</td>
   </tr>
 </table>
 
diff --git a/docs/sql-data-sources-orc.md b/docs/sql-data-sources-orc.md
index bddffe02602e8..4c4b3b1eee8c2 100644
--- a/docs/sql-data-sources-orc.md
+++ b/docs/sql-data-sources-orc.md
@@ -27,15 +27,25 @@ serde tables (e.g., the ones created using the clause `USING HIVE OPTIONS (fileF
 the vectorized reader is used when `spark.sql.hive.convertMetastoreOrc` is also set to `true`.
 
 <table class="table">
-  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th></tr>
+  <tr><th><b>Property Name</b></th><th><b>Default</b></th><th><b>Meaning</b></th><th><b>Since Version</b></th></tr>
   <tr>
     <td><code>spark.sql.orc.impl</code></td>
     <td><code>native</code></td>
-    <td>The name of ORC implementation. It can be one of <code>native</code> and <code>hive</code>. <code>native</code> means the native ORC support. <code>hive</code> means the ORC library in Hive.</td>
+    <td>
+      The name of ORC implementation. It can be one of <code>native</code> and <code>hive</code>.
+      <code>native</code> means the native ORC support. <code>hive</code> means the ORC library
+      in Hive.
+    </td>
+    <td>2.3.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.orc.enableVectorizedReader</code></td>
     <td><code>true</code></td>
-    <td>Enables vectorized orc decoding in <code>native</code> implementation. If <code>false</code>, a new non-vectorized ORC reader is used in <code>native</code> implementation. For <code>hive</code> implementation, this is ignored.</td>
+    <td>
+      Enables vectorized orc decoding in <code>native</code> implementation. If <code>false</code>,
+      a new non-vectorized ORC reader is used in <code>native</code> implementation.
+      For <code>hive</code> implementation, this is ignored.
+    </td>
+    <td>2.3.0</td>
   </tr>
 </table>
diff --git a/docs/sql-data-sources-parquet.md b/docs/sql-data-sources-parquet.md
index 53a1111cd8286..6e52446c9e39e 100644
--- a/docs/sql-data-sources-parquet.md
+++ b/docs/sql-data-sources-parquet.md
@@ -258,7 +258,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
 `SET key=value` commands using SQL.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.sql.parquet.binaryAsString</code></td>
   <td>false</td>
@@ -267,6 +267,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     not differentiate between binary data and strings when writing out the Parquet schema. This
     flag tells Spark SQL to interpret binary data as a string to provide compatibility with these systems.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.int96AsTimestamp</code></td>
@@ -275,6 +276,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     Some Parquet-producing systems, in particular Impala and Hive, store Timestamp into INT96. This
     flag tells Spark SQL to interpret INT96 data as a timestamp to provide compatibility with these systems.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.compression.codec</code></td>
@@ -287,11 +289,13 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     Note that <code>zstd</code> requires <code>ZStandardCodec</code> to be installed before Hadoop 2.9.0, <code>brotli</code> requires
     <code>BrotliCodec</code> to be installed.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.filterPushdown</code></td>
   <td>true</td>
   <td>Enables Parquet filter push-down optimization when set to true.</td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.hive.convertMetastoreParquet</code></td>
@@ -300,6 +304,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     When set to false, Spark SQL will use the Hive SerDe for parquet tables instead of the built in
     support.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.mergeSchema</code></td>
@@ -310,6 +315,7 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
       schema is picked from the summary file or a random data file if no summary file is available.
     </p>
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.parquet.writeLegacyFormat</code></td>
@@ -321,5 +327,6 @@ Configuration of Parquet can be done using the `setConf` method on `SparkSession
     example, decimals will be written in int-based format. If Parquet output is intended for use
     with systems that do not support this newer format, set to true.
   </td>
+  <td>1.6.0</td>
 </tr>
 </table>

From 8af58ebd958813e9ff29d2f0d1b070d529ba1275 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 2 Apr 2020 16:01:54 +0900
Subject: [PATCH 0451/1280] [SPARK-31295][DOC][FOLLOWUP] Supplement version for
 configuration appear in doc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR supplements version for configuration appear in docs.
I sorted out some information show below.

**docs/sql-performance-tuning.md**
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.inMemoryColumnarStorage.compressed | 1.0.1 | SPARK-2631 | 86534d0f5255362618c05a07b0171ec35c915822#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.inMemoryColumnarStorage.batchSize | 1.1.1 | SPARK-2650 | 779d1eb26d0f031791e93c908d51a59c3b422a55#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.files.maxPartitionBytes | 2.0.0 | SPARK-13664 | 17eec0a71ba8713c559d641e3f43a1be726b037c#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.files.openCostInBytes | 2.0.0 | SPARK-14259 | 400b2f863ffaa01a34a8dae1541c61526fef908b#diff-32bb9518401c0948c5ea19377b5069ab |  
spark.sql.broadcastTimeout | 1.3.0 | SPARK-4269 | fa66ef6c97e87c9255b67b03836a4ba50598ebae#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.autoBroadcastJoinThreshold | 1.1.0 | SPARK-2393 | c7db274be79f448fda566208946cb50958ea9b1a#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.shuffle.partitions | 1.1.0 | SPARK-1508 | 08ed9ad81397b71206c4dc903bfb94b6105691ed#diff-41ef65b9ef5b518f77e2a03559893f4d |  
spark.sql.adaptive.coalescePartitions.enabled | 3.0.0 | SPARK-31037 | 46b7f1796bd0b96977ce9b473601033f397a3b18#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.coalescePartitions.minPartitionNum | 3.0.0 | SPARK-31037 | 46b7f1796bd0b96977ce9b473601033f397a3b18#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.coalescePartitions.initialPartitionNum | 3.0.0 | SPARK-31037 | 46b7f1796bd0b96977ce9b473601033f397a3b18#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.advisoryPartitionSizeInBytes | 3.0.0 | SPARK-31037 | 46b7f1796bd0b96977ce9b473601033f397a3b18#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.skewJoin.enabled | 3.0.0 | SPARK-31037 | 46b7f1796bd0b96977ce9b473601033f397a3b18#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.skewJoin.skewedPartitionFactor | 3.0.0 | SPARK-31037 | 46b7f1796bd0b96977ce9b473601033f397a3b18#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes | 3.0.0 | SPARK-31201 | 8d0800a0803d3c47938bddefa15328d654739bc5#diff-9a6b543db706f1a90f790783d6930a13 |  

**docs/sql-ref-ansi-compliance.md**
Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.sql.ansi.enabled | 3.0.0 | SPARK-30125 | d9b30694122f8716d3acb448638ef1e2b96ebc7a#diff-9a6b543db706f1a90f790783d6930a13 |  
spark.sql.storeAssignmentPolicy | 3.0.0 | SPARK-28730 | 895c90b582cc2b2667241f66d5b733852aeef9eb#diff-9a6b543db706f1a90f790783d6930a13 |

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test

Closes #28096 from beliefer/supplement-version-of-performance.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-performance-tuning.md  | 30 ++++++++++++++++++++++--------
 docs/sql-ref-ansi-compliance.md |  4 +++-
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 9a1cc891d9312..279aad6ae8496 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -35,7 +35,7 @@ Configuration of in-memory caching can be done using the `setConf` method on `Sp
 `SET key=value` commands using SQL.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.sql.inMemoryColumnarStorage.compressed</code></td>
   <td>true</td>
@@ -43,6 +43,7 @@ Configuration of in-memory caching can be done using the `setConf` method on `Sp
     When set to true Spark SQL will automatically select a compression codec for each column based
     on statistics of the data.
   </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.sql.inMemoryColumnarStorage.batchSize</code></td>
@@ -51,6 +52,7 @@ Configuration of in-memory caching can be done using the `setConf` method on `Sp
     Controls the size of batches for columnar caching. Larger batch sizes can improve memory utilization
     and compression, but risk OOMs when caching data.
   </td>
+  <td>1.1.1</td>
 </tr>
 
 </table>
@@ -61,7 +63,7 @@ The following options can also be used to tune the performance of query executio
 that these options will be deprecated in future release as more optimizations are performed automatically.
 
 <table class="table">
-  <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+  <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.files.maxPartitionBytes</code></td>
     <td>134217728 (128 MB)</td>
@@ -69,6 +71,7 @@ that these options will be deprecated in future release as more optimizations ar
       The maximum number of bytes to pack into a single partition when reading files.
       This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.
     </td>
+    <td>2.0.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.files.openCostInBytes</code></td>
@@ -80,15 +83,17 @@ that these options will be deprecated in future release as more optimizations ar
       scheduled first). This configuration is effective only when using file-based sources such as Parquet,
       JSON and ORC.
     </td>
+    <td>2.0.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.broadcastTimeout</code></td>
     <td>300</td>
     <td>
-    <p>
-      Timeout in seconds for the broadcast wait time in broadcast joins
-    </p>
+      <p>
+        Timeout in seconds for the broadcast wait time in broadcast joins
+      </p>
     </td>
+    <td>1.3.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.autoBroadcastJoinThreshold</code></td>
@@ -99,6 +104,7 @@ that these options will be deprecated in future release as more optimizations ar
       statistics are only supported for Hive Metastore tables where the command
       <code>ANALYZE TABLE &lt;tableName&gt; COMPUTE STATISTICS noscan</code> has been run.
     </td>
+    <td>1.1.0</td>
   </tr>
   <tr>
     <td><code>spark.sql.shuffle.partitions</code></td>
@@ -106,6 +112,7 @@ that these options will be deprecated in future release as more optimizations ar
     <td>
       Configures the number of partitions to use when shuffling data for joins or aggregations.
     </td>
+    <td>1.1.0</td>
   </tr>
 </table>
 
@@ -193,13 +200,14 @@ Adaptive Query Execution (AQE) is an optimization technique in Spark SQL that ma
 ### Coalescing Post Shuffle Partitions
 This feature coalesces the post shuffle partitions based on the map output statistics when both `spark.sql.adaptive.enabled` and `spark.sql.adaptive.coalescePartitions.enabled` configurations are true. This feature simplifies the tuning of shuffle partition number when running queries. You do not need to set a proper shuffle partition number to fit your dataset. Spark can pick the proper shuffle partition number at runtime once you set a large enough initial number of shuffle partitions via `spark.sql.adaptive.coalescePartitions.initialPartitionNum` configuration.
  <table class="table">
-   <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+   <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
    <tr>
      <td><code>spark.sql.adaptive.coalescePartitions.enabled</code></td>
      <td>true</td>
      <td>
        When true and <code>spark.sql.adaptive.enabled</code> is true, Spark will coalesce contiguous shuffle partitions according to the target size (specified by <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>), to avoid too many small tasks.
      </td>
+     <td>3.0.0</td>
    </tr>
    <tr>
      <td><code>spark.sql.adaptive.coalescePartitions.minPartitionNum</code></td>
@@ -207,6 +215,7 @@ This feature coalesces the post shuffle partitions based on the map output stati
      <td>
        The minimum number of shuffle partitions after coalescing. If not set, the default value is the default parallelism of the Spark cluster. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
      </td>
+     <td>3.0.0</td>
    </tr>
    <tr>
      <td><code>spark.sql.adaptive.coalescePartitions.initialPartitionNum</code></td>
@@ -214,6 +223,7 @@ This feature coalesces the post shuffle partitions based on the map output stati
      <td>
        The initial number of shuffle partitions before coalescing. By default it equals to <code>spark.sql.shuffle.partitions</code>. This configuration only has an effect when <code>spark.sql.adaptive.enabled</code> and <code>spark.sql.adaptive.coalescePartitions.enabled</code> are both enabled.
      </td>
+     <td>3.0.0</td>
    </tr>
    <tr>
      <td><code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code></td>
@@ -221,6 +231,7 @@ This feature coalesces the post shuffle partitions based on the map output stati
      <td>
        The advisory size in bytes of the shuffle partition during adaptive optimization (when <code>spark.sql.adaptive.enabled</code> is true). It takes effect when Spark coalesces small shuffle partitions or splits skewed shuffle partition.
      </td>
+     <td>3.0.0</td>
    </tr>
  </table>
  
@@ -230,13 +241,14 @@ AQE converts sort-merge join to broadcast hash join when the runtime statistics
 ### Optimizing Skew Join
 Data skew can severely downgrade the performance of join queries. This feature dynamically handles skew in sort-merge join by splitting (and replicating if needed) skewed tasks into roughly evenly sized tasks. It takes effect when both `spark.sql.adaptive.enabled` and `spark.sql.adaptive.skewJoin.enabled` configurations are enabled.
   <table class="table">
-     <tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+     <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
      <tr>
        <td><code>spark.sql.adaptive.skewJoin.enabled</code></td>
        <td>true</td>
        <td>
          When true and <code>spark.sql.adaptive.enabled</code> is true, Spark dynamically handles skew in sort-merge join by splitting (and replicating if needed) skewed partitions.
        </td>
+       <td>3.0.0</td>
      </tr>
      <tr>
        <td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
@@ -244,6 +256,7 @@ Data skew can severely downgrade the performance of join queries. This feature d
        <td>
          A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewedPartitionThresholdInBytes</code>.
        </td>
+       <td>3.0.0</td>
      </tr>
      <tr>
        <td><code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code></td>
@@ -251,5 +264,6 @@ Data skew can severely downgrade the performance of join queries. This feature d
        <td>
          A partition is considered as skewed if its size in bytes is larger than this threshold and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> multiplying the median partition size. Ideally this config should be set larger than <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>.
        </td>
+       <td>3.0.0</td>
      </tr>
-   </table>
\ No newline at end of file
+   </table>
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index bc5bde6778e12..83affb9adfe35 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -28,7 +28,7 @@ The casting behaviours are defined as store assignment rules in the standard.
 When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with the ANSI store assignment rules. This is a separate configuration because its default value is `ANSI`, while the configuration `spark.sql.ansi.enabled` is disabled by default.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.sql.ansi.enabled</code></td>
   <td>false</td>
@@ -37,6 +37,7 @@ When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with
     1. Spark will throw a runtime exception if an overflow occurs in any operation on integral/decimal field.
     2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in the SQL parser.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.sql.storeAssignmentPolicy</code></td>
@@ -52,6 +53,7 @@ When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with
     With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion,
     e.g. converting double to int or decimal to double is not allowed.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 

From 70bf2ff09d3d71d8fb9dcee7f2298288caa641e7 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 31 Mar 2020 12:33:01 +0900
Subject: [PATCH 0452/1280] [SPARK-31282][DOC] Supplement version for
 configuration appear in security doc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR supplements version for configuration appear in security doc.
I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.network.crypto.keyLength | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.network.crypto.keyFactoryAlgorithm | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.network.crypto.config.* | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.network.crypto.saslFallback | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.authenticate.enableSaslEncryption | 2.2.0 | SPARK-19139 | 8f3f73abc1fe62496722476460c174af0250e3fe#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.network.sasl.serverAlwaysEncrypt | 1.4.0 | SPARK-6229 | 38d4e9e446b425ca6a8fe8d8080f387b08683842#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.ui.filters | 1.0.0 | SPARK-1189 | 7edbea41b43e0dc11a2de156be220db8b7952d01#diff-f79a5ead735b3d0b34b6b94486918e1c |  
spark.acls.enable | 1.1.0 | SPARK-1890 and SPARK-1891 | e3fe6571decfdc406ec6d505fd92f9f2b85a618c#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.ui.view.acls | 1.0.0 | SPARK-1189 | 7edbea41b43e0dc11a2de156be220db8b7952d01#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.ui.view.acls.groups | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.admin.acls | 1.1.0 | SPARK-1890 and SPARK-1891 | e3fe6571decfdc406ec6d505fd92f9f2b85a618c#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.admin.acls.groups | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.modify.acls | 1.1.0 | SPARK-1890 and SPARK-1891 | e3fe6571decfdc406ec6d505fd92f9f2b85a618c#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.modify.acls.groups | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.user.groups.mapping | 2.0.0 | SPARK-4224 | ae79032dcf160796851ca29116cca146c4d86ada#diff-afd88f677ec5ff8b5e96a5cbbe00cd98 |  
spark.history.ui.acls.enable | 1.0.1 | Spark 1489 | c8dd13221215275948b1a6913192d40e0c8cbadd#diff-b49b5b9c31ddb36a9061004b5b723058 |  
spark.history.ui.admin.acls | 2.1.1 | SPARK-19033 | 4ca1788805e4a0131ba8f0ccb7499ee0e0242837#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.history.ui.admin.acls.groups | 2.1.1 | SPARK-19033 | 4ca1788805e4a0131ba8f0ccb7499ee0e0242837#diff-a7befb99e7bd7e3ab5c46c2568aa5b3e |  
spark.ui.xXssProtection | 2.3.0 | SPARK-22188 | 5a07aca4d464e96d75ea17bf6768e24b829872ec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.xContentTypeOptions.enabled | 2.3.0 | SPARK-22188 | 5a07aca4d464e96d75ea17bf6768e24b829872ec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.ui.strictTransportSecurity | 2.3.0 | SPARK-22188 | 5a07aca4d464e96d75ea17bf6768e24b829872ec#diff-6bdad48cfc34314e89599655442ff210 |  
spark.security.credentials.${service}.enabled | 2.3.0 | SPARK-20434 | a18d637112b97d2caaca0a8324bdd99086664b24#diff-da6c1fd6d8b0c7538a3e77a09e06a083 |  
spark.kerberos.access.hadoopFileSystems | 3.0.0 | SPARK-26766 | d0443a74d185ec72b747fa39994fa9a40ce974cf#diff-6bdad48cfc34314e89599655442ff210 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test

Closes #28044 from beliefer/supplement-version-to-security-doc.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/security.md | 43 +++++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/docs/security.md b/docs/security.md
index 5496879d8c840..aef6e69382a80 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -158,7 +158,7 @@ The following table describes the different options available for configuring th
   <td>
     The length in bits of the encryption key to generate. Valid values are 128, 192 and 256.
   </td>
-  <td></td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.keyFactoryAlgorithm</code></td>
@@ -167,7 +167,7 @@ The following table describes the different options available for configuring th
     The key factory algorithm to use when generating encryption keys. Should be one of the
     algorithms supported by the javax.crypto.SecretKeyFactory class in the JRE being used.
   </td>
-  <td></td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.config.*</code></td>
@@ -177,7 +177,7 @@ The following table describes the different options available for configuring th
     use. The config name should be the name of commons-crypto configuration without the
     <code>commons.crypto</code> prefix.
   </td>
-  <td></td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.crypto.saslFallback</code></td>
@@ -196,6 +196,7 @@ The following table describes the different options available for configuring th
   <td>
     Enable SASL-based encrypted communication.
   </td>
+  <td>2.2.0</td>
 </tr>
 <tr>
   <td><code>spark.network.sasl.serverAlwaysEncrypt</code></td>
@@ -204,6 +205,7 @@ The following table describes the different options available for configuring th
     Disable unencrypted connections for ports using SASL authentication. This will deny connections
     from clients that have authentication enabled, but do not request SASL-based encryption.
   </td>
+  <td>1.4.0</td>
 </tr>
 </table>
 
@@ -286,7 +288,7 @@ below.
 The following options control the authentication of Web UIs:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.ui.filters</code></td>
   <td>None</td>
@@ -294,6 +296,7 @@ The following options control the authentication of Web UIs:
     See the <a href="configuration.html#spark-ui">Spark UI</a> configuration for how to configure
     filters.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.acls.enable</code></td>
@@ -303,6 +306,7 @@ The following options control the authentication of Web UIs:
     permissions to view or modify the application. Note this requires the user to be authenticated,
     so if no authentication filter is installed, this option does not do anything.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.admin.acls</code></td>
@@ -310,6 +314,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of users that have view and modify access to the Spark application.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.admin.acls.groups</code></td>
@@ -317,6 +322,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of groups that have view and modify access to the Spark application.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.modify.acls</code></td>
@@ -324,6 +330,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of users that have modify access to the Spark application.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.modify.acls.groups</code></td>
@@ -331,6 +338,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of groups that have modify access to the Spark application.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.view.acls</code></td>
@@ -338,6 +346,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of users that have view access to the Spark application.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.view.acls.groups</code></td>
@@ -345,6 +354,7 @@ The following options control the authentication of Web UIs:
   <td>
     Comma-separated list of groups that have view access to the Spark application.
   </td>
+  <td>2.0.0</td>
 </tr>
 <tr>
   <td><code>spark.user.groups.mapping</code></td>
@@ -361,6 +371,7 @@ The following options control the authentication of Web UIs:
     Windows environment is currently <b>not</b> supported. However, a new platform/protocol can
     be supported by implementing the trait mentioned above.
   </td>
+  <td>2.0.0</td>
 </tr>
 </table>
 
@@ -375,7 +386,7 @@ servlet filters.
 To enable authorization in the SHS, a few extra options are used:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.history.ui.acls.enable</code></td>
   <td>false</td>
@@ -389,6 +400,7 @@ To enable authorization in the SHS, a few extra options are used:
     If disabled, no access control checks are made for any application UIs available through
     the history server.
   </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.history.ui.admin.acls</code></td>
@@ -397,6 +409,7 @@ To enable authorization in the SHS, a few extra options are used:
     Comma separated list of users that have view access to all the Spark applications in history
     server.
   </td>
+  <td>2.1.1</td>
 </tr>
 <tr>
   <td><code>spark.history.ui.admin.acls.groups</code></td>
@@ -405,6 +418,7 @@ To enable authorization in the SHS, a few extra options are used:
     Comma separated list of groups that have view access to all the Spark applications in history
     server.
   </td>
+  <td>2.1.1</td>
 </tr>
 </table>
 
@@ -620,7 +634,7 @@ Apache Spark can be configured to include HTTP headers to aid in preventing Cros
 Security.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.ui.xXssProtection</code></td>
   <td><code>1; mode=block</code></td>
@@ -635,6 +649,7 @@ Security.
         of the page if an attack is detected.)</li>
     </ul>
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.xContentTypeOptions.enabled</code></td>
@@ -642,7 +657,8 @@ Security.
   <td>
     When enabled, X-Content-Type-Options HTTP response header will be set to "nosniff".
   </td>
-  </tr>
+  <td>2.3.0</td>
+</tr>
 <tr>
   <td><code>spark.ui.strictTransportSecurity</code></td>
   <td>None</td>
@@ -656,6 +672,7 @@ Security.
       <li><code>max-age=&lt;expire-time&gt;; preload</code></li>
     </ul>
   </td>
+  <td>2.3.0</td>
 </tr>
 </table>
 
@@ -796,16 +813,17 @@ deployment-specific page for more information.
 The following options provides finer-grained control for this feature:
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.security.credentials.${service}.enabled</code></td>
   <td><code>true</code></td>
   <td>
-  Controls whether to obtain credentials for services when security is enabled.
-  By default, credentials for all supported services are retrieved when those services are
-  configured, but it's possible to disable that behavior if it somehow conflicts with the
-  application being run.
+    Controls whether to obtain credentials for services when security is enabled.
+    By default, credentials for all supported services are retrieved when those services are
+    configured, but it's possible to disable that behavior if it somehow conflicts with the
+    application being run.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.kerberos.access.hadoopFileSystems</code></td>
@@ -818,6 +836,7 @@ The following options provides finer-grained control for this feature:
     or in a trusted realm). Spark acquires security tokens for each of the filesystems so that
     the Spark application can access those remote Hadoop filesystems.
   </td>
+  <td>3.0.0</td>
 </tr>
 </table>
 

From 304f7f293068bdb3b55208e473ee01c256c8bd2a Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 31 Mar 2020 12:32:04 +0900
Subject: [PATCH 0453/1280] [SPARK-31269][DOC] Supplement version for
 configuration only appear in configuration doc
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
The `configuration.md` exists some config not organized by `ConfigEntry`.
This PR supplements version for configuration only appear in configuration doc.
I sorted out some information show below.

Item name | Since version | JIRA ID | Commit ID | Note
-- | -- | -- | -- | --
spark.app.name | 0.9.0 | None | 994f080f8ae3372366e6004600ba791c8a372ff0#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.driver.resource.{resourceName}.amount | 3.0.0 | SPARK-27760 | d30284b5a51dd784f663eb4eea37087b35a54d00#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.driver.resource.{resourceName}.discoveryScript | 3.0.0 | SPARK-27488 | 74e5e41eebf9ed596b48e6db52a2a9c642e5cbc3#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.driver.resource.{resourceName}.vendor | 3.0.0 | SPARK-27362 | 1277f8fa92da85d9e39d9146e3099fcb75c71a8f#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.executor.resource.{resourceName}.amount | 3.0.0 | SPARK-27760 | d30284b5a51dd784f663eb4eea37087b35a54d00#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.executor.resource.{resourceType}.discoveryScript | 3.0.0 | SPARK-27024 | db2e3c43412e4a7fb4a46c58d73d9ab304a1e949#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.executor.resource.{resourceName}.vendor | 3.0.0 | SPARK-27362 | 1277f8fa92da85d9e39d9146e3099fcb75c71a8f#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.local.dir | 0.5.0 | None | 0e93891d3d7df849cff6442038c111ffd42a5243#diff-17fd275d280b667722664ed833c6402a |  
spark.logConf | 0.9.0 | None | d8bcc8e9a095c1b20dd7a17b6535800d39bff80e#diff-364713d7776956cb8b0a771e9b62f82d |  
spark.master | 0.9.0 | SPARK-544 | 2573add94cf920a88f74d80d8ea94218d812704d#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.driver.defaultJavaOptions | 3.0.0 | SPARK-23472 | f83000597f250868de9722d8285fed013abc5ecf#diff-a78ecfc6a89edfaf0b60a5eaa0381970 |  
spark.executor.defaultJavaOptions | 3.0.0 | SPARK-23472 | f83000597f250868de9722d8285fed013abc5ecf#diff-a78ecfc6a89edfaf0b60a5eaa0381970 |  
spark.executorEnv.[EnvironmentVariableName] | 0.9.0 | None | 642029e7f43322f84abe4f7f36bb0b1b95d8101d#diff-529fc5c06b9731c1fbda6f3db60b16aa |  
spark.python.profile | 1.2.0 | SPARK-3478 | 1aa549ba9839565274a12c52fa1075b424f138a6#diff-d6fe2792e44f6babc94aabfefc8b9bce |  
spark.python.profile.dump | 1.2.0 | SPARK-3478 | 1aa549ba9839565274a12c52fa1075b424f138a6#diff-d6fe2792e44f6babc94aabfefc8b9bce |  
spark.python.worker.memory | 1.1.0 | SPARK-2538 | 14174abd421318e71c16edd24224fd5094bdfed4#diff-d6fe2792e44f6babc94aabfefc8b9bce |  
spark.jars.packages | 1.5.0 | SPARK-9263 | 34335719a372c1951fdb4dd25b75b086faf1076f#diff-63a5d817d2d45ae24de577f6a1bd80f9 |  
spark.jars.excludes | 1.5.0 | SPARK-9263 | 34335719a372c1951fdb4dd25b75b086faf1076f#diff-63a5d817d2d45ae24de577f6a1bd80f9 |  
spark.jars.ivy | 1.3.0 | SPARK-5341 | 3b7acd22ab4a134c74746e3b9a803dbd34d43855#diff-63a5d817d2d45ae24de577f6a1bd80f9 |  
spark.jars.ivySettings | 2.2.0 | SPARK-17568 | 3bc2eff8880a3ba8d4318118715ea1a47048e3de#diff-4d2ab44195558d5a9d5f15b8803ef39d |  
spark.jars.repositories | 2.3.0 | SPARK-21403 | d8257b99ddae23f702f312640a5335ddb4554403#diff-4d2ab44195558d5a9d5f15b8803ef39d |
spark.shuffle.io.maxRetries | 1.2.0 | SPARK-4188 | c1ea5c542f3267c0b23a7775887e3a6ece793fe3#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.shuffle.io.numConnectionsPerPeer | 1.2.1 | SPARK-4740 | 441ec3451730c7ae3dbef8952e313071d6147ab6#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.shuffle.io.preferDirectBufs | 1.2.0 | SPARK-4188 | c1ea5c542f3267c0b23a7775887e3a6ece793fe3#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.shuffle.io.retryWait | 1.2.1 | None | 5e5d8f469a1bea9bbe606f772ccdcab7c184c651#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.shuffle.io.backLog | 1.1.1 | SPARK-2468 | 66b4c81db7e826c00f7fb449b8a8af810cf7dd9a#diff-bdee8e601924d41e93baa7287189e878 |  
spark.shuffle.service.index.cache.size | 2.3.0 | SPARK-21501 | 1662e93119d68498942386906de309d35f4a135f#diff-97d5edc927a83a678e013ae00343df94 |
spark.shuffle.maxChunksBeingTransferred | 2.3.0 | SPARK-21175 | 799e13161e89f1ea96cb1bc7b507a05af2e89cd0#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.sql.ui.retainedExecutions | 1.5.0 | SPARK-8861 and SPARK-8862 | ebc3aad272b91cf58e2e1b4aa92b49b8a947a045#diff-81764e4d52817f83bdd5336ef1226bd9 |  
spark.streaming.ui.retainedBatches | 1.0.0 | SPARK-1386 | f36dc3fed0a0671b0712d664db859da28c0a98e2#diff-56b8d67d07284cfab165d5363bd3500e |
spark.default.parallelism | 0.5.0 | None | e5c4cd8a5e188592f8786a265c0cd073c69ac886#diff-0544ebf7533fa70ff5103e0fe1f0b036 |  
spark.files.fetchTimeout | 1.0.0 | None | f6f9d02e85d17da2f742ed0062f1648a9293e73c#diff-d239aee594001f8391676e1047a0381e |  
spark.files.useFetchCache | 1.2.2 | SPARK-6313 | a2a94a154bdd00753b8d5e344d712664c7151050#diff-d239aee594001f8391676e1047a0381e |
spark.files.overwrite | 1.0.0 | None | 84670f2715392859624df290c1b52eb4ed4a9cb1#diff-d239aee594001f8391676e1047a0381e | Exists in branch-1.0, but the version of pom is 0.9.0-incubating-SNAPSHOT
spark.hadoop.cloneConf | 1.0.3 | SPARK-2546 | 6d8f1dd15afdc7432b5721c89f9b2b402460322b#diff-83eb37f7b0ebed3c14ccb7bff0d577c2 |  
spark.hadoop.validateOutputSpecs | 1.0.1 | SPARK-1677 | 8100cbdb7546e8438019443cfc00683017c81278#diff-f70e97c099b5eac05c75288cb215e080 |
spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version | 2.2.0 | SPARK-20107 | edc87d76efea7b4d19d9d0c4ddba274a3ccb8752#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.rpc.io.backLog | 3.0.0 | SPARK-27868 | 09ed64d795d3199a94e175273fff6fcea6b52131#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.network.io.preferDirectBufs | 3.0.0 | SPARK-24920 | e103c4a5e72bab8862ff49d6d4c1e62e642fc412#diff-0ac65da2bc6b083fb861fe410c7688c2 |  
spark.port.maxRetries | 1.1.1 | SPARK-3565 | 32f2222e915f31422089139944a077e2cbd442f9#diff-d239aee594001f8391676e1047a0381e |  
spark.core.connection.ack.wait.timeout | 1.1.1 | SPARK-2677 | bd3ce2ffb8964abb4d59918ebb2c230fe4614aa2#diff-f748e95f2aa97ed715afa53ddeeac9de |  
spark.scheduler.listenerbus.eventqueue.shared.capacity | 3.0.0 | SPARK-28574 | c212c9d9ed7375cd1ea16c118733edd84037ec0d#diff-eb519ad78cc3cf0b95839cc37413b509 |  
spark.scheduler.listenerbus.eventqueue.appStatus.capacity | 3.0.0 | SPARK-28574 | c212c9d9ed7375cd1ea16c118733edd84037ec0d#diff-eb519ad78cc3cf0b95839cc37413b509 |  
spark.scheduler.listenerbus.eventqueue.executorManagement.capacity | 3.0.0 | SPARK-28574 | c212c9d9ed7375cd1ea16c118733edd84037ec0d#diff-eb519ad78cc3cf0b95839cc37413b509 |  
spark.scheduler.listenerbus.eventqueue.eventLog.capacity | 3.0.0 | SPARK-28574 | c212c9d9ed7375cd1ea16c118733edd84037ec0d#diff-eb519ad78cc3cf0b95839cc37413b509 |  
spark.scheduler.listenerbus.eventqueue.streams.capacity | 3.0.0 | SPARK-28574 | c212c9d9ed7375cd1ea16c118733edd84037ec0d#diff-eb519ad78cc3cf0b95839cc37413b509 |  
spark.task.resource.{resourceName}.amount | 3.0.0 | SPARK-27760 | d30284b5a51dd784f663eb4eea37087b35a54d00#diff-76e731333fb756df3bff5ddb3b731c46 |  
spark.stage.maxConsecutiveAttempts | 2.2.0 | SPARK-13369 | 7b5d873aef672aa0aee41e338bab7428101e1ad3#diff-6a9ff7fb74fd490a50462d45db2d5e11 |  
spark.{driver\|executor}.rpc.io.serverThreads | 1.6.0 | SPARK-10745 | 7c5b641808740ba5eed05ba8204cdbaf3fc579f5#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.{driver\|executor}.rpc.io.clientThreads | 1.6.0 | SPARK-10745 | 7c5b641808740ba5eed05ba8204cdbaf3fc579f5#diff-d2ce9b38bdc38ca9d7119f9c2cf79907 |  
spark.{driver\|executor}.rpc.netty.dispatcher.numThreads | 3.0.0 | SPARK-29398 | 2f0a38cb50e3e8b4b72219c7b2b8b15d51f6b931#diff-a68a21481fea5053848ca666dd3201d8 |  
spark.r.driver.command | 1.5.3 | SPARK-10971 | 9695f452e86a88bef3bcbd1f3c0b00ad9e9ac6e1#diff-025470e1b7094d7cf4a78ea353fb3981 |  
spark.r.shell.command | 2.1.0 | SPARK-17178 | fa6347938fc1c72ddc03a5f3cd2e929b5694f0a6#diff-a78ecfc6a89edfaf0b60a5eaa0381970 |  
spark.graphx.pregel.checkpointInterval | 2.2.0 | SPARK-5484 | f971ce5dd0788fe7f5d2ca820b9ea3db72033ddc#diff-e399679417ffa6eeedf26a7630baca16 |  

### Why are the changes needed?
Supplemental configuration version information.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test

Closes #28035 from beliefer/supplement-configuration-version.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md | 88 +++++++++++++++++++++++++++++++++----------
 1 file changed, 69 insertions(+), 19 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 6d018979ababa..fae3bb4d8309f 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -143,6 +143,7 @@ of the most common options to set are:
   <td>
     The name of your application. This will appear in the UI and in log data.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.cores</code></td>
@@ -206,6 +207,7 @@ of the most common options to set are:
     <code>spark.driver.resource.{resourceName}.discoveryScript</code>
     for the driver to find the resource on startup.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.discoveryScript</code></td>
@@ -216,6 +218,7 @@ of the most common options to set are:
     name and an array of addresses. For a client-submitted driver, discovery script must assign
     different resource addresses to this driver comparing to other drivers on the same host.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.driver.resource.{resourceName}.vendor</code></td>
@@ -226,6 +229,7 @@ of the most common options to set are:
     the Kubernetes device plugin naming convention. (e.g. For GPUs on Kubernetes
     this config would be set to nvidia.com or amd.com)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.resources.discoveryPlugin</code></td>
@@ -293,6 +297,7 @@ of the most common options to set are:
     <code>spark.executor.resource.{resourceName}.discoveryScript</code>
     for the executor to find the resource on startup.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.resource.{resourceName}.discoveryScript</code></td>
@@ -302,6 +307,7 @@ of the most common options to set are:
     write to STDOUT a JSON string in the format of the ResourceInformation class. This has a
     name and an array of addresses.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
  <td><code>spark.executor.resource.{resourceName}.vendor</code></td>
@@ -312,6 +318,7 @@ of the most common options to set are:
     the Kubernetes device plugin naming convention. (e.g. For GPUs on Kubernetes
     this config would be set to nvidia.com or amd.com)
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.extraListeners</code></td>
@@ -337,6 +344,7 @@ of the most common options to set are:
     <em>Note:</em> This will be overridden by SPARK_LOCAL_DIRS (Standalone), MESOS_SANDBOX (Mesos) or
     LOCAL_DIRS (YARN) environment variables set by the cluster manager.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.logConf</code></td>
@@ -344,6 +352,7 @@ of the most common options to set are:
   <td>
     Logs the effective SparkConf as INFO when a SparkContext is started.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.master</code></td>
@@ -352,6 +361,7 @@ of the most common options to set are:
     The cluster manager to connect to. See the list of
     <a href="submitting-applications.html#master-urls"> allowed master URL's</a>.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.submit.deployMode</code></td>
@@ -467,6 +477,7 @@ Apart from these, the following properties are also available, and may be useful
     Instead, please set this through the <code>--driver-java-options</code> command line option or in
     your default properties file.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.driver.extraJavaOptions</code></td>
@@ -540,6 +551,7 @@ Apart from these, the following properties are also available, and may be useful
     verbose gc logging to a file named for the executor ID of the app in /tmp, pass a 'value' of:
     <code>-verbose:gc -Xloggc:/tmp/{{APP_ID}}-{{EXECUTOR_ID}}.gc</code>
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.extraJavaOptions</code></td>
@@ -636,6 +648,7 @@ Apart from these, the following properties are also available, and may be useful
     Add the environment variable specified by <code>EnvironmentVariableName</code> to the Executor
     process. The user can specify multiple of these to set multiple environment variables.
   </td>
+  <td>0.9.0</td>
 </tr>
 <tr>
   <td><code>spark.redaction.regex</code></td>
@@ -659,7 +672,7 @@ Apart from these, the following properties are also available, and may be useful
     By default the <code>pyspark.profiler.BasicProfiler</code> will be used, but this can be overridden by
     passing a profiler class in as a parameter to the <code>SparkContext</code> constructor.
   </td>
-  <td></td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.python.profile.dump</code></td>
@@ -670,6 +683,7 @@ Apart from these, the following properties are also available, and may be useful
     by <code>pstats.Stats()</code>. If this is specified, the profile result will not be displayed
     automatically.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.python.worker.memory</code></td>
@@ -680,6 +694,7 @@ Apart from these, the following properties are also available, and may be useful
     (e.g. <code>512m</code>, <code>2g</code>).
     If the memory used during aggregation goes above this amount, it will spill the data into disks.
   </td>
+  <td>1.1.0</td>
 </tr>
 <tr>
   <td><code>spark.python.worker.reuse</code></td>
@@ -727,6 +742,7 @@ Apart from these, the following properties are also available, and may be useful
     repositories given by the command-line option <code>--repositories</code>. For more details, see
     <a href="submitting-applications.html#advanced-dependency-management">Advanced Dependency Management</a>.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.excludes</code></td>
@@ -735,6 +751,7 @@ Apart from these, the following properties are also available, and may be useful
     Comma-separated list of groupId:artifactId, to exclude while resolving the dependencies
     provided in <code>spark.jars.packages</code> to avoid dependency conflicts.
   </td>
+  <td>1.5.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.ivy</code></td>
@@ -744,6 +761,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.jars.packages</code>. This will override the Ivy property <code>ivy.default.ivy.user.dir</code>
     which defaults to ~/.ivy2.
   </td>
+  <td>1.3.0</td>
 </tr>
 <tr>
   <td><code>spark.jars.ivySettings</code></td>
@@ -756,6 +774,7 @@ Apart from these, the following properties are also available, and may be useful
     artifact server like Artifactory. Details on the settings file format can be
     found at <a href="http://ant.apache.org/ivy/history/latest-milestone/settings.html">Settings Files</a>
   </td>
+  <td>2.2.0</td>
 </tr>
  <tr>
   <td><code>spark.jars.repositories</code></td>
@@ -764,6 +783,7 @@ Apart from these, the following properties are also available, and may be useful
     Comma-separated list of additional remote repositories to search for the maven coordinates
     given with <code>--packages</code> or <code>spark.jars.packages</code>.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.pyspark.driver.python</code></td>
@@ -849,6 +869,7 @@ Apart from these, the following properties are also available, and may be useful
     set to a non-zero value. This retry logic helps stabilize large shuffles in the face of long GC
     pauses or transient network connectivity issues.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.numConnectionsPerPeer</code></td>
@@ -858,6 +879,7 @@ Apart from these, the following properties are also available, and may be useful
     large clusters. For clusters with many hard disks and few hosts, this may result in insufficient
     concurrency to saturate all disks, and so users may consider increasing this value.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.preferDirectBufs</code></td>
@@ -867,6 +889,7 @@ Apart from these, the following properties are also available, and may be useful
     block transfer. For environments where off-heap memory is tightly limited, users may wish to
     turn this off to force all allocations from Netty to be on-heap.
   </td>
+  <td>1.2.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.retryWait</code></td>
@@ -875,6 +898,7 @@ Apart from these, the following properties are also available, and may be useful
     (Netty only) How long to wait between retries of fetches. The maximum delay caused by retrying
     is 15 seconds by default, calculated as <code>maxRetries * retryWait</code>.
   </td>
+  <td>1.2.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.io.backLog</code></td>
@@ -887,6 +911,7 @@ Apart from these, the following properties are also available, and may be useful
     application (see <code>spark.shuffle.service.enabled</code> option below). If set below 1,
     will fallback to OS default defined by Netty's <code>io.netty.util.NetUtil#SOMAXCONN</code>.
   </td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.service.enabled</code></td>
@@ -915,6 +940,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Cache entries limited to the specified memory footprint, in bytes unless otherwise specified.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.maxChunksBeingTransferred</code></td>
@@ -926,6 +952,7 @@ Apart from these, the following properties are also available, and may be useful
     <code>spark.shuffle.io.retryWait</code>), if those limits are reached the task will fail with
     fetch failure.
   </td>
+  <td>2.3.0</td>
 </tr>
 <tr>
   <td><code>spark.shuffle.sort.bypassMergeThreshold</code></td>
@@ -1241,6 +1268,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     How many finished batches the Spark UI and status APIs remember before garbage collecting.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.ui.retainedDeadExecutors</code></td>
@@ -1634,6 +1662,7 @@ Apart from these, the following properties are also available, and may be useful
     Default number of partitions in RDDs returned by transformations like <code>join</code>,
     <code>reduceByKey</code>, and <code>parallelize</code> when not set by user.
   </td>
+  <td>0.5.0</td>
 </tr>
 <tr>
   <td><code>spark.executor.heartbeatInterval</code></td>
@@ -1653,6 +1682,7 @@ Apart from these, the following properties are also available, and may be useful
     Communication timeout to use when fetching files added through SparkContext.addFile() from
     the driver.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.files.useFetchCache</code></td>
@@ -1665,6 +1695,7 @@ Apart from these, the following properties are also available, and may be useful
     disabled in order to use Spark local directories that reside on NFS filesystems (see
     <a href="https://issues.apache.org/jira/browse/SPARK-6313">SPARK-6313</a> for more details).
   </td>
+  <td>1.2.2</td>
 </tr>
 <tr>
   <td><code>spark.files.overwrite</code></td>
@@ -1673,6 +1704,7 @@ Apart from these, the following properties are also available, and may be useful
     Whether to overwrite files added through SparkContext.addFile() when the target file exists and
     its contents do not match those of the source.
   </td>
+  <td>1.0.0</td>
 </tr>
 <tr>
   <td><code>spark.files.maxPartitionBytes</code></td>
@@ -1693,23 +1725,29 @@ Apart from these, the following properties are also available, and may be useful
   <td>2.1.0</td>
 </tr>
 <tr>
-    <td><code>spark.hadoop.cloneConf</code></td>
-    <td>false</td>
-    <td>If set to true, clones a new Hadoop <code>Configuration</code> object for each task.  This
+  <td><code>spark.hadoop.cloneConf</code></td>
+  <td>false</td>
+  <td>
+    If set to true, clones a new Hadoop <code>Configuration</code> object for each task.  This
     option should be enabled to work around <code>Configuration</code> thread-safety issues (see
     <a href="https://issues.apache.org/jira/browse/SPARK-2546">SPARK-2546</a> for more details).
     This is disabled by default in order to avoid unexpected performance regressions for jobs that
-    are not affected by these issues.</td>
+    are not affected by these issues.
+  </td>
+  <td>1.0.3</td>
 </tr>
 <tr>
-    <td><code>spark.hadoop.validateOutputSpecs</code></td>
-    <td>true</td>
-    <td>If set to true, validates the output specification (e.g. checking if the output directory already exists)
+  <td><code>spark.hadoop.validateOutputSpecs</code></td>
+  <td>true</td>
+  <td>
+    If set to true, validates the output specification (e.g. checking if the output directory already exists)
     used in saveAsHadoopFile and other variants. This can be disabled to silence exceptions due to pre-existing
-    output directories. We recommend that users do not disable this except if trying to achieve compatibility with
-    previous versions of Spark. Simply use Hadoop's FileSystem API to delete output directories by hand.
-    This setting is ignored for jobs generated through Spark Streaming's StreamingContext, since
-    data may need to be rewritten to pre-existing output directories during checkpoint recovery.</td>
+    output directories. We recommend that users do not disable this except if trying to achieve compatibility
+    with previous versions of Spark. Simply use Hadoop's FileSystem API to delete output directories by hand.
+    This setting is ignored for jobs generated through Spark Streaming's StreamingContext, since data may
+    need to be rewritten to pre-existing output directories during checkpoint recovery.
+  </td>
+  <td>1.0.1</td>
 </tr>
 <tr>
   <td><code>spark.storage.memoryMapThreshold</code></td>
@@ -1729,6 +1767,7 @@ Apart from these, the following properties are also available, and may be useful
     Version 2 may have better performance, but version 1 may handle failures better in certain situations,
     as per <a href="https://issues.apache.org/jira/browse/MAPREDUCE-4815">MAPREDUCE-4815</a>.
   </td>
+  <td>2.2.0</td>
 </tr>
 </table>
 
@@ -1843,7 +1882,7 @@ Apart from these, the following properties are also available, and may be useful
     need to be increased, so that incoming connections are not dropped when a large number of
     connections arrives in a short period of time.
   </td>
-  <td></td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.network.timeout</code></td>
@@ -1866,7 +1905,7 @@ Apart from these, the following properties are also available, and may be useful
     block transfer. For environments where off-heap memory is tightly limited, users may wish to
     turn this off to force all allocations to be on-heap.
   </td>
-  <td></td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.port.maxRetries</code></td>
@@ -1878,7 +1917,7 @@ Apart from these, the following properties are also available, and may be useful
     essentially allows it to try a range of ports from the start port specified
     to port + maxRetries.
   </td>
-  <td></td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.rpc.numRetries</code></td>
@@ -1921,7 +1960,7 @@ Apart from these, the following properties are also available, and may be useful
     out and giving up. To avoid unwilling timeout caused by long pause like GC,
     you can set larger value.
   </td>
-  <td></td>
+  <td>1.1.1</td>
 </tr>
 <tr>
   <td><code>spark.network.maxRemoteBlockSizeFetchToMem</code></td>
@@ -2054,6 +2093,7 @@ Apart from these, the following properties are also available, and may be useful
     that register to the listener bus. Consider increasing value, if the listener events corresponding
     to shared queue are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.appStatus.capacity</code></td>
@@ -2063,6 +2103,7 @@ Apart from these, the following properties are also available, and may be useful
     Consider increasing value, if the listener events corresponding to appStatus queue are dropped.
     Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.executorManagement.capacity</code></td>
@@ -2072,6 +2113,7 @@ Apart from these, the following properties are also available, and may be useful
     executor management listeners. Consider increasing value if the listener events corresponding to
     executorManagement queue are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.eventLog.capacity</code></td>
@@ -2081,6 +2123,7 @@ Apart from these, the following properties are also available, and may be useful
     that write events to eventLogs. Consider increasing value if the listener events corresponding to eventLog queue
     are dropped. Increasing this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.listenerbus.eventqueue.streams.capacity</code></td>
@@ -2090,6 +2133,7 @@ Apart from these, the following properties are also available, and may be useful
     Consider increasing value if the listener events corresponding to streams queue are dropped. Increasing
     this value may result in the driver using more memory.
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.scheduler.blacklist.unschedulableTaskSetTimeout</code></td>
@@ -2272,6 +2316,7 @@ Apart from these, the following properties are also available, and may be useful
     in order to assign resource slots (e.g. a 0.2222 configuration, or 1/0.2222 slots will become 
     4 tasks/resource, not 5).
   </td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.task.maxFailures</code></td>
@@ -2336,6 +2381,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>
     Number of consecutive stage attempts allowed before a stage is aborted.
   </td>
+  <td>2.2.0</td>
 </tr>
 </table>
 
@@ -2528,13 +2574,14 @@ like shuffle, just replace "rpc" with "shuffle" in the property names except
 <code>spark.{driver|executor}.rpc.netty.dispatcher.numThreads</code>, which is only for RPC module.
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.{driver|executor}.rpc.io.serverThreads</code></td>
   <td>
     Fall back on <code>spark.rpc.io.serverThreads</code>
   </td>
   <td>Number of threads used in the server thread pool</td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.{driver|executor}.rpc.io.clientThreads</code></td>
@@ -2542,6 +2589,7 @@ like shuffle, just replace "rpc" with "shuffle" in the property names except
     Fall back on <code>spark.rpc.io.clientThreads</code>
   </td>
   <td>Number of threads used in the client thread pool</td>
+  <td>1.6.0</td>
 </tr>
 <tr>
   <td><code>spark.{driver|executor}.rpc.netty.dispatcher.numThreads</code></td>
@@ -2549,6 +2597,7 @@ like shuffle, just replace "rpc" with "shuffle" in the property names except
     Fall back on <code>spark.rpc.netty.dispatcher.numThreads</code>
   </td>
   <td>Number of threads used in RPC message dispatcher thread pool</td>
+  <td>3.0.0</td>
 </tr>
 </table>
 
@@ -2730,7 +2779,7 @@ Spark subsystems.
   <td>
     Executable for executing R scripts in client modes for driver. Ignored in cluster modes.
   </td>
-  <td></td>
+  <td>1.5.3</td>
 </tr>
 <tr>
   <td><code>spark.r.shell.command</code></td>
@@ -2739,7 +2788,7 @@ Spark subsystems.
     Executable for executing sparkR shell in client modes for driver. Ignored in cluster modes. It is the same as environment variable <code>SPARKR_DRIVER_R</code>, but take precedence over it.
     <code>spark.r.shell.command</code> is used for sparkR shell while <code>spark.r.driver.command</code> is used for running R script.
   </td>
-  <td></td>
+  <td>2.1.0</td>
 </tr>
 <tr>
   <td><code>spark.r.backendConnectionTimeout</code></td>
@@ -2771,6 +2820,7 @@ Spark subsystems.
     Checkpoint interval for graph and message in Pregel. It used to avoid stackOverflowError due to long lineage chains
   after lots of iterations. The checkpoint is disabled by default.
   </td>
+  <td>2.2.0</td>
 </tr>
 </table>
 

From 9c591742d3c862158cf599faa24db0e201f4f037 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 7 Apr 2020 23:06:53 +0900
Subject: [PATCH 0454/1280] [SPARK-30841][SQL][FOLLOW-UP] Change 'version' of
 spark.sql.execution.pandas.udf.buffer.size to 3.0.0

### What changes were proposed in this pull request?

This PR fixes the added version of `spark.sql.execution.pandas.udf.buffer.size` to 3.0.0 (see also SPARK-27870)

### Why are the changes needed?

To show the correct version added.

### Does this PR introduce any user-facing change?

Yes but only in the unreleased branches. It will change the version shown in SQL documentation.

### How was this patch tested?

Not tested. Jenkins will test it out.

Closes #28144 from HyukjinKwon/SPARK-30841-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 6cf7336a0795ca35bd44f90207f744cfe8e12464)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 420102b0f50a7..2b5c68edbfa56 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1825,7 +1825,7 @@ object SQLConf {
         s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +
         "than 4 bytes. Lowering this value could make small Pandas UDF batch iterated and " +
         "pipelined; however, it might degrade performance. See SPARK-27870.")
-      .version("3.1.0")
+      .version("3.0.0")
       .fallbackConf(BUFFER_SIZE)
 
   val PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME =

From 2dd6a38725b86b0cf248625ec213a0ea8e4e5815 Mon Sep 17 00:00:00 2001
From: Eric Wu <492960551@qq.com>
Date: Tue, 7 Apr 2020 09:25:01 -0700
Subject: [PATCH 0455/1280] [SPARK-31113][SQL] Add SHOW VIEWS command

### What changes were proposed in this pull request?
Previously, user can issue `SHOW TABLES` to get info of both tables and views.
This PR (SPARK-31113) implements `SHOW VIEWS` SQL command similar to HIVE to get views only.(https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-ShowViews)

**Hive** -- Only show view names
```
hive> SHOW VIEWS;
OK
view_1
view_2
...
```

**Spark(Hive-Compatible)** -- Only show view names, used in tests and `SparkSQLDriver` for CLI applications
```
SHOW VIEWS IN showdb;
view_1
view_2
...
```

**Spark** -- Show more information database/viewName/isTemporary
```
spark-sql> SHOW VIEWS;
userdb	view_1	false
userdb	view_2	false
...
```

### Why are the changes needed?
`SHOW VIEWS` command provides better granularity to only get information of views.

### Does this PR introduce any user-facing change?
Add new `SHOW VIEWS` SQL command

### How was this patch tested?
Add new test `show-views.sql` and pass existing tests

Closes #27897 from Eric5553/ShowViews.

Authored-by: Eric Wu <492960551@qq.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a28ed86a387b286745b30cd4d90b3d558205a5a7)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/_data/menu-sql.yaml                      |   2 +
 docs/sql-ref-ansi-compliance.md               |   1 +
 docs/sql-ref-syntax-aux-show-tables.md        |   2 +-
 docs/sql-ref-syntax-aux-show-views.md         | 121 +++++++++++
 docs/sql-ref-syntax-aux-show.md               |   1 +
 docs/sql-ref-syntax-ddl-alter-view.md         |   1 +
 docs/sql-ref-syntax-ddl-create-view.md        |   1 +
 docs/sql-ref-syntax-ddl-drop-view.md          |   1 +
 docs/sql-ref-syntax.md                        |   1 +
 .../spark/sql/catalyst/parser/SqlBase.g4      |   7 +-
 .../sql/catalyst/analysis/Analyzer.scala      |   2 +
 .../catalyst/catalog/ExternalCatalog.scala    |   2 +
 .../catalog/ExternalCatalogWithListener.scala |   4 +
 .../catalyst/catalog/InMemoryCatalog.scala    |   6 +
 .../sql/catalyst/catalog/SessionCatalog.scala |  19 ++
 .../sql/catalyst/parser/AstBuilder.scala      |  10 +
 .../catalyst/plans/logical/v2Commands.scala   |  16 ++
 .../sql/catalyst/parser/DDLParserSuite.scala  |  37 +++-
 .../parser/TableIdentifierParserSuite.scala   |   2 +
 .../analysis/ResolveSessionCatalog.scala      |  14 ++
 .../spark/sql/execution/HiveResult.scala      |   6 +-
 .../spark/sql/execution/command/views.scala   |  39 +++-
 .../sql-tests/inputs/show-tables.sql          |   1 +
 .../resources/sql-tests/inputs/show-views.sql |  28 +++
 .../sql-tests/results/show-tables.sql.out     |  12 +-
 .../sql-tests/results/show-views.sql.out      | 201 ++++++++++++++++++
 .../sql/connector/DataSourceV2SQLSuite.scala  |  17 ++
 .../spark/sql/hive/HiveExternalCatalog.scala  |   5 +
 .../spark/sql/hive/client/VersionsSuite.scala |  10 +-
 .../sql/hive/execution/HiveCommandSuite.scala |  46 ++++
 30 files changed, 605 insertions(+), 10 deletions(-)
 create mode 100644 docs/sql-ref-syntax-aux-show-views.md
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/show-views.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/show-views.sql.out

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 787ee3ecebd39..ffc3b3c7b70f9 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -209,6 +209,8 @@
                   url: sql-ref-syntax-aux-show-partitions.html
                 - text: SHOW CREATE TABLE
                   url: sql-ref-syntax-aux-show-create-table.html
+                - text: SHOW VIEWS
+                  url: sql-ref-syntax-aux-show-views.html
             - text: CONFIGURATION MANAGEMENT
               url: sql-ref-syntax-aux-conf-mgmt.html
               subitems:
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 83affb9adfe35..7543180f5db35 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -411,6 +411,7 @@ Below is a list of all the keywords in Spark SQL.
   <tr><td>USING</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
   <tr><td>VALUES</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>VIEW</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
+  <tr><td>VIEWS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
   <tr><td>WHEN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>WHERE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
   <tr><td>WINDOW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index 2a078abf911e7..311401cf580be 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -89,7 +89,7 @@ SHOW TABLES FROM default LIKE 'sam*';
   +-----------+------------+--------------+--+
   
 -- List all tables matching the pattern `sam*|suj`
-SHOW TABLES  LIKE 'sam*|suj';
+SHOW TABLES LIKE 'sam*|suj';
   +-----------+------------+--------------+--+
   | database  | tableName  | isTemporary  |
   +-----------+------------+--------------+--+
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
new file mode 100644
index 0000000000000..a5e840d97bf2d
--- /dev/null
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -0,0 +1,121 @@
+---
+layout: global
+title: SHOW VIEWS
+displayTitle: SHOW VIEWS
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+### Description
+
+The `SHOW VIEWS` statement returns all the views for an optionally specified database.
+Additionally, the output of this statement may be filtered by an optional matching
+pattern. If no database is specified then the views are returned from the 
+current database. If the specified database is global temporary view database, we will
+list global temporary views. Note that the command also lists local temporary views 
+regardless of a given database.
+
+### Syntax
+{% highlight sql %}
+SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
+{% endhighlight %}
+
+### Parameters
+<dl>
+  <dt><code><em>{ FROM | IN } database_name</em></code></dt>
+  <dd>
+     Specifies the database name from which views are listed.
+  </dd>
+  <dt><code><em>LIKE regex_pattern</em></code></dt>
+  <dd>
+     Specifies the regular expression pattern that is used to filter out unwanted views. 
+     <ul> 
+          <li> Except for `*` and `|` character, the pattern works like a regex.</li>
+          <li> `*` alone matches 0 or more characters and `|` is used to separate multiple different regexes,
+           any of which can match. </li>
+          <li> The leading and trailing blanks are trimmed in the input pattern before processing.</li>
+     </ul>
+  </dd>
+</dl>
+
+### Example
+{% highlight sql %}
+-- Create views in different databases, also create global/local temp views.
+CREATE VIEW sam AS SELECT id, salary FROM employee WHERE name = 'sam';
+CREATE VIEW sam1 AS SELECT id, salary FROM employee WHERE name = 'sam1';
+CREATE VIEW suj AS SELECT id, salary FROM employee WHERE name = 'suj';
+USE userdb;
+CREATE VIEW user1 AS SELECT id, salary FROM default.employee WHERE name = 'user1';
+CREATE VIEW user2 AS SELECT id, salary FROM default.employee WHERE name = 'user2';
+USE default;
+CREATE GLOBAL TEMP VIEW temp1 AS SELECT 1 as col1;
+CREATE TEMP VIEW temp2 AS SELECT 1 as col1;
+
+-- List all views in default database
+SHOW VIEWS;
+  +-------------+------------+--------------+--+
+  | namespace   | viewName   | isTemporary  |
+  +-------------+------------+--------------+--+
+  | default     | sam        | false        |
+  | default     | sam1       | false        |
+  | default     | suj        | false        |
+  |             | temp2      | true         |
+  +-------------+------------+--------------+--+
+
+-- List all views from userdb database 
+SHOW VIEWS FROM userdb;
+  +-------------+------------+--------------+--+
+  | namespace   | viewName   | isTemporary  |
+  +-------------+------------+--------------+--+
+  | userdb      | user1      | false        |
+  | userdb      | user2      | false        |
+  |             | temp2      | true         |
+  +-------------+------------+--------------+--+
+  
+-- List all views in global temp view database 
+SHOW VIEWS IN global_temp;
+  +-------------+------------+--------------+--+
+  | namespace   | viewName   | isTemporary  |
+  +-------------+------------+--------------+--+
+  | global_temp | temp1      | true         |
+  |             | temp2      | true         |
+  +-------------+------------+--------------+--+
+
+-- List all views from default database matching the pattern `sam*`
+SHOW VIEWS FROM default LIKE 'sam*';
+  +-----------+------------+--------------+--+
+  | namespace | viewName   | isTemporary  |
+  +-----------+------------+--------------+--+
+  | default   | sam        | false        |
+  | default   | sam1       | false        |
+  +-----------+------------+--------------+--+
+
+-- List all views from the current database matching the pattern `sam|suj｜temp*`
+SHOW VIEWS LIKE 'sam|suj|temp*';
+  +-------------+------------+--------------+--+
+  | namespace   | viewName   | isTemporary  |
+  +-------------+------------+--------------+--+
+  | default     | sam        | false        |
+  | default     | suj        | false        |
+  |             | temp2      | true         |
+  +-------------+------------+--------------+--+
+
+{% endhighlight %}
+
+### Related statements
+- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show.md b/docs/sql-ref-syntax-aux-show.md
index f6d700acf9eef..dd56d467815c7 100644
--- a/docs/sql-ref-syntax-aux-show.md
+++ b/docs/sql-ref-syntax-aux-show.md
@@ -26,3 +26,4 @@ license: |
 * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
 * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
 * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
+* [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index d42d040522cae..a29f2b4f632a1 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -228,6 +228,7 @@ DESC TABLE EXTENDED tempdb1.v2;
 - [describe-table](sql-ref-syntax-aux-describe-table.html)
 - [create-view](sql-ref-syntax-ddl-create-view.html)
 - [drop-view](sql-ref-syntax-ddl-drop-view.html)
+- [show-views](sql-ref-syntax-aux-show-views.html)
 
 #### Note:
 
diff --git a/docs/sql-ref-syntax-ddl-create-view.md b/docs/sql-ref-syntax-ddl-create-view.md
index 67060d70f0790..53b402771e421 100644
--- a/docs/sql-ref-syntax-ddl-create-view.md
+++ b/docs/sql-ref-syntax-ddl-create-view.md
@@ -89,3 +89,4 @@ CREATE GLOBAL TEMPORARY VIEW IF NOT EXISTS subscribed_movies
 ### Related Statements
 - [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
 - [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+- [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-view.md b/docs/sql-ref-syntax-ddl-drop-view.md
index bf9e497b3a418..f313995022089 100644
--- a/docs/sql-ref-syntax-ddl-drop-view.md
+++ b/docs/sql-ref-syntax-ddl-drop-view.md
@@ -78,5 +78,6 @@ DROP VIEW IF EXISTS employeeView;
 ### Related Statements
 - [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
 - [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+- [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
 - [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
 - [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 3db97ac11dd84..dd611d75d3429 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -78,5 +78,6 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
 - [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
 - [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
 - [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
+- [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
 - [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
 - [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 143a567e8082c..39f2942df5921 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -158,7 +158,7 @@ statement
         SET TBLPROPERTIES tablePropertyList                            #setTableProperties
     | ALTER (TABLE | VIEW) multipartIdentifier
         UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList             #unsetTableProperties
-    |ALTER TABLE table=multipartIdentifier
+    | ALTER TABLE table=multipartIdentifier
         (ALTER | CHANGE) COLUMN? column=multipartIdentifier
         alterColumnAction?                                             #alterTableAlterColumn
     | ALTER TABLE table=multipartIdentifier partitionSpec?
@@ -204,6 +204,8 @@ statement
         ('(' key=tablePropertyKey ')')?                                #showTblProperties
     | SHOW COLUMNS (FROM | IN) table=multipartIdentifier
         ((FROM | IN) ns=multipartIdentifier)?                          #showColumns
+    | SHOW VIEWS ((FROM | IN) multipartIdentifier)?
+        (LIKE? pattern=STRING)?                                        #showViews
     | SHOW PARTITIONS multipartIdentifier partitionSpec?               #showPartitions
     | SHOW identifier? FUNCTIONS
         (LIKE? (multipartIdentifier | pattern=STRING))?                #showFunctions
@@ -1171,6 +1173,7 @@ ansiNonReserved
     | USE
     | VALUES
     | VIEW
+    | VIEWS
     | WINDOW
     ;
 
@@ -1434,6 +1437,7 @@ nonReserved
     | USER
     | VALUES
     | VIEW
+    | VIEWS
     | WHEN
     | WHERE
     | WINDOW
@@ -1694,6 +1698,7 @@ USER: 'USER';
 USING: 'USING';
 VALUES: 'VALUES';
 VIEW: 'VIEW';
+VIEWS: 'VIEWS';
 WHEN: 'WHEN';
 WHERE: 'WHERE';
 WINDOW: 'WINDOW';
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 67f6f49385007..e4be634d0d2f2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -743,6 +743,8 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
       case s @ ShowTables(UnresolvedNamespace(Seq()), _) =>
         s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
+      case s @ ShowViews(UnresolvedNamespace(Seq()), _) =>
+        s.copy(namespace = ResolvedNamespace(currentCatalog, catalogManager.currentNamespace))
       case UnresolvedNamespace(Seq()) =>
         ResolvedNamespace(currentCatalog, Seq.empty[String])
       case UnresolvedNamespace(CatalogAndNamespace(catalog, ns)) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
index dcc143982a4aa..db930cf7890e6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalog.scala
@@ -136,6 +136,8 @@ trait ExternalCatalog {
 
   def listTables(db: String, pattern: String): Seq[String]
 
+  def listViews(db: String, pattern: String): Seq[String]
+
   /**
    * Loads data into a table.
    *
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala
index 86113d3ec3ead..c2613ff74da4a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogWithListener.scala
@@ -154,6 +154,10 @@ class ExternalCatalogWithListener(delegate: ExternalCatalog)
     delegate.listTables(db, pattern)
   }
 
+  override def listViews(db: String, pattern: String): Seq[String] = {
+    delegate.listViews(db, pattern)
+  }
+
   override def loadTable(
       db: String,
       table: String,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
index abf69939dea13..31644a5ae4e35 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
@@ -346,6 +346,12 @@ class InMemoryCatalog(
     StringUtils.filterPattern(listTables(db), pattern)
   }
 
+  override def listViews(db: String, pattern: String): Seq[String] = synchronized {
+    requireDbExists(db)
+    val views = catalog(db).tables.filter(_._2.table.tableType == CatalogTableType.VIEW).keySet
+    StringUtils.filterPattern(views.toSeq.sorted, pattern)
+  }
+
   override def loadTable(
       db: String,
       table: String,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 3a63aff493df5..b79857cdccd22 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -880,6 +880,25 @@ class SessionCatalog(
     }
   }
 
+  /**
+   * List all matching views in the specified database, including local temporary views.
+   */
+  def listViews(db: String, pattern: String): Seq[TableIdentifier] = {
+    val dbName = formatDatabaseName(db)
+    val dbViews = if (dbName == globalTempViewManager.database) {
+      globalTempViewManager.listViewNames(pattern).map { name =>
+        TableIdentifier(name, Some(globalTempViewManager.database))
+      }
+    } else {
+      requireDbExists(dbName)
+      externalCatalog.listViews(dbName, pattern).map { name =>
+        TableIdentifier(name, Some(dbName))
+      }
+    }
+
+    dbViews ++ listLocalTempViews(pattern)
+  }
+
   /**
    * List all matching local temporary views.
    */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index cd4c895752a79..b2ef96db0fb26 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2897,6 +2897,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec))
   }
 
+  /**
+   * Create a [[ShowViews]] command.
+   */
+  override def visitShowViews(ctx: ShowViewsContext): LogicalPlan = withOrigin(ctx) {
+    val multiPart = Option(ctx.multipartIdentifier).map(visitMultipartIdentifier)
+    ShowViews(
+      UnresolvedNamespace(multiPart.getOrElse(Seq.empty[String])),
+      Option(ctx.pattern).map(string))
+  }
+
   override def visitColPosition(ctx: ColPositionContext): ColumnPosition = {
     ctx.position.getType match {
       case SqlBaseParser.FIRST => ColumnPosition.first()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index c04e56355a68f..579157a6f2f2e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -436,6 +436,22 @@ case class ShowTables(
     AttributeReference("tableName", StringType, nullable = false)())
 }
 
+/**
+ * The logical plan of the SHOW VIEWS command that works for v1 and v2 catalogs.
+ *
+ * Notes: v2 catalogs do not support views API yet, the command will fallback to
+ * v1 ShowViewsCommand during ResolveSessionCatalog.
+ */
+case class ShowViews(
+    namespace: LogicalPlan,
+    pattern: Option[String]) extends Command {
+  override def children: Seq[LogicalPlan] = Seq(namespace)
+
+  override val output: Seq[Attribute] = Seq(
+    AttributeReference("namespace", StringType, nullable = false)(),
+    AttributeReference("viewName", StringType, nullable = false)())
+}
+
 /**
  * The logical plan of the USE/USE NAMESPACE command that works for v2 catalogs.
  */
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 543ea5ddf5d20..1bef63898085c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -1179,6 +1179,12 @@ class DDLParserSuite extends AnalysisTest {
     comparePlans(
       parsePlan("SHOW TABLES"),
       ShowTables(UnresolvedNamespace(Seq.empty[String]), None))
+    comparePlans(
+      parsePlan("SHOW TABLES '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW TABLES LIKE '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
     comparePlans(
       parsePlan("SHOW TABLES FROM testcat.ns1.ns2.tbl"),
       ShowTables(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
@@ -1186,8 +1192,11 @@ class DDLParserSuite extends AnalysisTest {
       parsePlan("SHOW TABLES IN testcat.ns1.ns2.tbl"),
       ShowTables(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
     comparePlans(
-      parsePlan("SHOW TABLES IN tbl LIKE '*dog*'"),
-      ShowTables(UnresolvedNamespace(Seq("tbl")), Some("*dog*")))
+      parsePlan("SHOW TABLES IN ns1 '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW TABLES IN ns1 LIKE '*test*'"),
+      ShowTables(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
   }
 
   test("show table extended") {
@@ -1215,6 +1224,30 @@ class DDLParserSuite extends AnalysisTest {
         Some(Map("ds" -> "2008-04-09"))))
   }
 
+  test("show views") {
+    comparePlans(
+      parsePlan("SHOW VIEWS"),
+      ShowViews(UnresolvedNamespace(Seq.empty[String]), None))
+    comparePlans(
+      parsePlan("SHOW VIEWS '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW VIEWS LIKE '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq.empty[String]), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW VIEWS FROM testcat.ns1.ns2.tbl"),
+      ShowViews(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
+    comparePlans(
+      parsePlan("SHOW VIEWS IN testcat.ns1.ns2.tbl"),
+      ShowViews(UnresolvedNamespace(Seq("testcat", "ns1", "ns2", "tbl")), None))
+    comparePlans(
+      parsePlan("SHOW VIEWS IN ns1 '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+    comparePlans(
+      parsePlan("SHOW VIEWS IN ns1 LIKE '*test*'"),
+      ShowViews(UnresolvedNamespace(Seq("ns1")), Some("*test*")))
+  }
+
   test("create namespace -- backward compatibility with DATABASE/DBPROPERTIES") {
     val expected = CreateNamespaceStatement(
       Seq("a", "b", "c"),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 053d57846ce8d..d5b0885555462 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -254,6 +254,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "utctimestamp",
     "values",
     "view",
+    "views",
     "while",
     "with",
     "work",
@@ -525,6 +526,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "using",
     "values",
     "view",
+    "views",
     "when",
     "where",
     "window",
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 77c5701d04708..58d74e041ecc6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -521,6 +521,20 @@ class ResolveSessionCatalog(
         replace,
         viewType)
 
+    case ShowViews(resolved: ResolvedNamespace, pattern) =>
+      resolved match {
+        case SessionCatalogAndNamespace(_, ns) =>
+          // Fallback to v1 ShowViewsCommand since there is no view API in v2 catalog
+          assert(ns.nonEmpty)
+          if (ns.length != 1) {
+            throw new AnalysisException(s"The database name is not valid: ${ns.quoted}")
+          }
+          ShowViewsCommand(ns.head, pattern)
+        case _ =>
+          throw new AnalysisException(s"Catalog ${resolved.catalog.name} doesn't support " +
+            "SHOW VIEWS, only SessionCatalog supports this command.")
+      }
+
     case ShowTableProperties(r: ResolvedTable, propertyKey) if isSessionCatalog(r.catalog) =>
       ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index 21874bdaf20f6..1a84db1970449 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -23,7 +23,7 @@ import java.time.{Instant, LocalDate}
 
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
-import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand}
+import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand, ShowViewsCommand}
 import org.apache.spark.sql.execution.datasources.v2.{DescribeTableExec, ShowTablesExec}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -50,6 +50,10 @@ object HiveResult {
     // namespace and table name.
     case command : ShowTablesExec =>
       command.executeCollect().map(_.getString(1))
+    // SHOW VIEWS in Hive only outputs view names while our v1 command outputs
+    // namespace, viewName, and isTemporary.
+    case command @ ExecutedCommandExec(_: ShowViewsCommand) =>
+      command.executeCollect().map(_.getString(1))
     case other =>
       val result: Seq[Seq[Any]] = other.executeCollectPublic().map(_.toSeq).toSeq
       // We need the types so we can output struct field names
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 795f9002d30bc..23f1d6c983413 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -23,11 +23,12 @@ import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{GlobalTempView, LocalTempView, PersistedView, UnresolvedFunction, UnresolvedRelation, ViewType}
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Alias, SubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, SubqueryExpression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, View}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper
 import org.apache.spark.sql.internal.StaticSQLConf
-import org.apache.spark.sql.types.MetadataBuilder
+import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType}
 import org.apache.spark.sql.util.SchemaUtils
 
 /**
@@ -295,6 +296,40 @@ case class AlterViewAsCommand(
   }
 }
 
+/**
+ * A command for users to get views in the given database.
+ * If a databaseName is not given, the current database will be used.
+ * The syntax of using this command in SQL is:
+ * {{{
+ *   SHOW VIEWS [(IN|FROM) database_name] [[LIKE] 'identifier_with_wildcards'];
+ * }}}
+ */
+case class ShowViewsCommand(
+    databaseName: String,
+    tableIdentifierPattern: Option[String]) extends RunnableCommand {
+
+  // The result of SHOW VIEWS has three basic columns: namespace, viewName and isTemporary.
+  override val output: Seq[Attribute] = Seq(
+    AttributeReference("namespace", StringType, nullable = false)(),
+    AttributeReference("viewName", StringType, nullable = false)(),
+    AttributeReference("isTemporary", BooleanType, nullable = false)())
+
+  override def run(sparkSession: SparkSession): Seq[Row] = {
+    val catalog = sparkSession.sessionState.catalog
+
+    // Show the information of views.
+    val views = tableIdentifierPattern.map(catalog.listViews(databaseName, _))
+      .getOrElse(catalog.listViews(databaseName, "*"))
+    views.map { tableIdent =>
+      val namespace = tableIdent.database.toArray.quoted
+      val tableName = tableIdent.table
+      val isTemp = catalog.isTemporaryTable(tableIdent)
+
+      Row(namespace, tableName, isTemp)
+    }
+  }
+}
+
 object ViewHelper {
 
   import CatalogTable._
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql b/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql
index 3c77c9977d80f..8f46c93ee3233 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-tables.sql
@@ -15,6 +15,7 @@ SHOW TABLES IN showdb;
 SHOW TABLES 'show_t*';
 SHOW TABLES LIKE 'show_t1*|show_t2*';
 SHOW TABLES IN showdb 'show_t*';
+SHOW TABLES IN showdb LIKE 'show_t*';
 
 -- SHOW TABLE EXTENDED
 SHOW TABLE EXTENDED LIKE 'show_t*';
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-views.sql b/sql/core/src/test/resources/sql-tests/inputs/show-views.sql
new file mode 100644
index 0000000000000..bdf9ef4aa6780
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-views.sql
@@ -0,0 +1,28 @@
+-- Test data.
+CREATE DATABASE showdb;
+USE showdb;
+CREATE TABLE tbl(a STRING, b INT, c STRING, d STRING) USING parquet;
+CREATE VIEW view_1 AS SELECT * FROM tbl;
+CREATE VIEW view_2 AS SELECT * FROM tbl WHERE c='a';
+CREATE GLOBAL TEMP VIEW view_3 AS SELECT 1 as col1;
+CREATE TEMPORARY VIEW view_4(e INT) USING parquet;
+
+-- SHOW VIEWS
+SHOW VIEWS;
+SHOW VIEWS FROM showdb;
+SHOW VIEWS IN showdb;
+SHOW VIEWS IN global_temp;
+
+-- SHOW VIEWS WITH wildcard match
+SHOW VIEWS 'view_*';
+SHOW VIEWS LIKE 'view_1*|view_2*';
+SHOW VIEWS IN showdb 'view_*';
+SHOW VIEWS IN showdb LIKE 'view_*';
+-- Error when database not exists
+SHOW VIEWS IN wrongdb LIKE 'view_*';
+
+-- Clean Up
+DROP VIEW global_temp.view_3;
+DROP VIEW view_4;
+USE default;
+DROP DATABASE showdb CASCADE;
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
index 501e185b07f7a..dd2e7b168698e 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 26
+-- Number of queries: 27
 
 
 -- !query
@@ -119,6 +119,16 @@ show_t2
 show_t3
 
 
+-- !query
+SHOW TABLES IN showdb LIKE 'show_t*'
+-- !query schema
+struct<database:string,tableName:string,isTemporary:boolean>
+-- !query output
+show_t1
+show_t2
+show_t3
+
+
 -- !query
 SHOW TABLE EXTENDED LIKE 'show_t*'
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
new file mode 100644
index 0000000000000..ee8a7474e6d5d
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
@@ -0,0 +1,201 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 20
+
+
+-- !query
+CREATE DATABASE showdb
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+USE showdb
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TABLE tbl(a STRING, b INT, c STRING, d STRING) USING parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE VIEW view_1 AS SELECT * FROM tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE VIEW view_2 AS SELECT * FROM tbl WHERE c='a'
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE GLOBAL TEMP VIEW view_3 AS SELECT 1 as col1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TEMPORARY VIEW view_4(e INT) USING parquet
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW VIEWS
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+aggtest
+arraydata
+mapdata
+onek
+tenk1
+testdata
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS FROM showdb
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+aggtest
+arraydata
+mapdata
+onek
+tenk1
+testdata
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN showdb
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+aggtest
+arraydata
+mapdata
+onek
+tenk1
+testdata
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN global_temp
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+aggtest
+arraydata
+mapdata
+onek
+tenk1
+testdata
+view_3
+view_4
+
+
+-- !query
+SHOW VIEWS 'view_*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS LIKE 'view_1*|view_2*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+
+
+-- !query
+SHOW VIEWS IN showdb 'view_*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN showdb LIKE 'view_*'
+-- !query schema
+struct<namespace:string,viewName:string,isTemporary:boolean>
+-- !query output
+view_1
+view_2
+view_4
+
+
+-- !query
+SHOW VIEWS IN wrongdb LIKE 'view_*'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException
+Database 'wrongdb' not found;
+
+
+-- !query
+DROP VIEW global_temp.view_3
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW view_4
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+USE default
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP DATABASE showdb CASCADE
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index ffaff66492879..3244684c33965 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -909,6 +909,23 @@ class DataSourceV2SQLSuite
     assert(exception.getMessage.contains("The database name is not valid: a.b"))
   }
 
+  test("ShowViews: using v1 catalog, db name with multipartIdentifier ('a.b') is not allowed.") {
+    val exception = intercept[AnalysisException] {
+      sql("SHOW TABLES FROM a.b")
+    }
+
+    assert(exception.getMessage.contains("The database name is not valid: a.b"))
+  }
+
+  test("ShowViews: using v2 catalog, command not supported.") {
+    val exception = intercept[AnalysisException] {
+      sql("SHOW VIEWS FROM testcat")
+    }
+
+    assert(exception.getMessage.contains("Catalog testcat doesn't support SHOW VIEWS," +
+      " only SessionCatalog supports this command."))
+  }
+
   test("ShowTables: using v2 catalog with empty namespace") {
     spark.sql("CREATE TABLE testcat.table (id bigint, data string) USING foo")
     runShowTablesSql("SHOW TABLES FROM testcat", Seq(Row("", "table")))
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index be6d824ece684..2faf42028f3a2 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -861,6 +861,11 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
     client.listTables(db, pattern)
   }
 
+  override def listViews(db: String, pattern: String): Seq[String] = withClient {
+    requireDbExists(db)
+    client.listTablesByType(db, pattern, CatalogTableType.VIEW)
+  }
+
   override def loadTable(
       db: String,
       table: String,
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index ba75dcf006e5f..d1dd13623650d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -415,14 +415,20 @@ class VersionsSuite extends SparkFunSuite with Logging {
       try {
         client.dropTable("default", tableName = "temporary", ignoreIfNotExists = false,
           purge = true)
-        client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
-          purge = true)
         assert(!versionsWithoutPurge.contains(version))
       } catch {
         case _: UnsupportedOperationException =>
           assert(versionsWithoutPurge.contains(version))
           client.dropTable("default", tableName = "temporary", ignoreIfNotExists = false,
             purge = false)
+      }
+      // Drop table with type CatalogTableType.VIEW.
+      try {
+        client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
+          purge = true)
+        assert(!versionsWithoutPurge.contains(version))
+      } catch {
+        case _: UnsupportedOperationException =>
           client.dropTable("default", tableName = "view1", ignoreIfNotExists = false,
             purge = false)
       }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index dbbf2b29fe8b7..bd85ad4e4944c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -117,6 +117,52 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     }
   }
 
+  test("show views") {
+    withView("show1a", "show2b", "global_temp.temp1", "temp2") {
+      sql("CREATE VIEW show1a AS SELECT 1 AS id")
+      sql("CREATE VIEW show2b AS SELECT 1 AS id")
+      sql("CREATE GLOBAL TEMP VIEW temp1 AS SELECT 1 AS id")
+      sql("CREATE TEMP VIEW temp2 AS SELECT 1 AS id")
+      checkAnswer(
+        sql("SHOW VIEWS"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) ::
+          Row("default", "parquet_view1", false) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS IN default"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) ::
+          Row("default", "parquet_view1", false) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS FROM default"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) ::
+          Row("default", "parquet_view1", false) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS FROM global_temp"),
+        Row("global_temp", "temp1", true) ::
+          Row("", "temp2", true) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS 'show1*|show2*'"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS LIKE 'show1*|show2*'"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS IN default 'show1*'"),
+        Row("default", "show1a", false) :: Nil)
+      checkAnswer(
+        sql("SHOW VIEWS IN default LIKE 'show1*|show2*'"),
+        Row("default", "show1a", false) ::
+          Row("default", "show2b", false) :: Nil)
+    }
+  }
+
   test("show tblproperties of data source tables - basic") {
     checkAnswer(
       sql("SHOW TBLPROPERTIES parquet_tab1").filter(s"key = 'my_key1'"),

From a856eea42949810f54c5f2f41b9c9abdd2da37c6 Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Tue, 7 Apr 2020 17:17:47 -0700
Subject: [PATCH 0456/1280] [SPARK-31278][SS] Fix StreamingQuery output rows
 metric

### What changes were proposed in this pull request?

In Structured Streaming, we provide progress updates every 10 seconds when a stream doesn't have any new data upstream. When providing this progress though, we zero out the input information but not the output information. This PR fixes that bug.

### Why are the changes needed?

Fixes a bug around incorrect metrics

### Does this PR introduce any user-facing change?

Fixes a bug in the metrics

### How was this patch tested?

New regression test

Closes #28040 from brkyvz/sinkMetrics.

Lead-authored-by: Burak Yavuz <brkyvz@gmail.com>
Co-authored-by: Burak Yavuz <burak@databricks.com>
Signed-off-by: Burak Yavuz <brkyvz@gmail.com>
(cherry picked from commit 8ab2a0c5f23a59c00a9b4191afd976af50d913ba)
Signed-off-by: Burak Yavuz <brkyvz@gmail.com>
---
 .../spark/sql/kafka010/KafkaSinkSuite.scala   |  2 +-
 .../streaming/MicroBatchExecution.scala       |  3 +-
 .../streaming/ProgressReporter.scala          | 32 ++++++---
 .../streaming/StreamingAggregationSuite.scala | 71 ++++++++++++-------
 .../StreamingDeduplicationSuite.scala         |  3 +-
 ...StreamingQueryStatusAndProgressSuite.scala |  1 +
 6 files changed, 73 insertions(+), 39 deletions(-)

diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
index 5c8c5b1f3b307..4e808a5277a98 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaSinkSuite.scala
@@ -314,7 +314,7 @@ class KafkaSinkMicroBatchStreamingSuite extends KafkaSinkStreamingSuiteBase {
     try {
       input.addData("1", "2", "3")
       verifyResult(writer) {
-        assert(writer.lastProgress.sink.numOutputRows == 3L)
+        assert(writer.recentProgress.exists(_.sink.numOutputRows == 3L))
       }
     } finally {
       writer.stop()
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index 45a2ce16183a5..e022bfb6835d2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -226,7 +226,8 @@ class MicroBatchExecution(
           }
         }
 
-        finishTrigger(currentBatchHasNewData)  // Must be outside reportTimeTaken so it is recorded
+        // Must be outside reportTimeTaken so it is recorded
+        finishTrigger(currentBatchHasNewData, isCurrentBatchConstructed)
 
         // Signal waiting threads. Note this must be after finishTrigger() to ensure all
         // activities (progress generation, etc.) have completed before signaling.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index feb151a5468b8..d1086cd42d133 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -85,8 +85,8 @@ trait ProgressReporter extends Logging {
   private val noDataProgressEventInterval =
     sparkSession.sessionState.conf.streamingNoDataProgressEventInterval
 
-  // The timestamp we report an event that has no input data
-  private var lastNoDataProgressEventTime = Long.MinValue
+  // The timestamp we report an event that has not executed anything
+  private var lastNoExecutionProgressEventTime = Long.MinValue
 
   private val timestampFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'") // ISO8601
   timestampFormat.setTimeZone(DateTimeUtils.getTimeZone("UTC"))
@@ -142,8 +142,15 @@ trait ProgressReporter extends Logging {
     logInfo(s"Streaming query made progress: $newProgress")
   }
 
-  /** Finalizes the query progress and adds it to list of recent status updates. */
-  protected def finishTrigger(hasNewData: Boolean): Unit = {
+  /**
+   * Finalizes the query progress and adds it to list of recent status updates.
+   *
+   * @param hasNewData Whether the sources of this stream had new data for this trigger.
+   * @param hasExecuted Whether any batch was executed during this trigger. Streaming queries that
+   *                    perform stateful aggregations with timeouts can still run batches even
+   *                    though the sources don't have any new data.
+   */
+  protected def finishTrigger(hasNewData: Boolean, hasExecuted: Boolean): Unit = {
     assert(currentTriggerStartOffsets != null && currentTriggerEndOffsets != null)
     currentTriggerEndTimestamp = triggerClock.getTimeMillis()
 
@@ -170,9 +177,12 @@ trait ProgressReporter extends Logging {
       )
     }
 
-    val sinkProgress = SinkProgress(
-      sink.toString,
-      sinkCommitProgress.map(_.numOutputRows))
+    val sinkOutput = if (hasExecuted) {
+      sinkCommitProgress.map(_.numOutputRows)
+    } else {
+      sinkCommitProgress.map(_ => 0L)
+    }
+    val sinkProgress = SinkProgress(sink.toString, sinkOutput)
     val observedMetrics = extractObservedMetrics(hasNewData, lastExecution)
 
     val newProgress = new StreamingQueryProgress(
@@ -189,14 +199,14 @@ trait ProgressReporter extends Logging {
       sink = sinkProgress,
       observedMetrics = new java.util.HashMap(observedMetrics.asJava))
 
-    if (hasNewData) {
+    if (hasExecuted) {
       // Reset noDataEventTimestamp if we processed any data
-      lastNoDataProgressEventTime = Long.MinValue
+      lastNoExecutionProgressEventTime = Long.MinValue
       updateProgress(newProgress)
     } else {
       val now = triggerClock.getTimeMillis()
-      if (now - noDataProgressEventInterval >= lastNoDataProgressEventTime) {
-        lastNoDataProgressEventTime = now
+      if (now - noDataProgressEventInterval >= lastNoExecutionProgressEventTime) {
+        lastNoExecutionProgressEventTime = now
         updateProgress(newProgress)
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
index 741355381222d..85e1b85b84d26 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingAggregationSuite.scala
@@ -202,47 +202,68 @@ class StreamingAggregationSuite extends StateStoreMetricsTest with Assertions {
         }
       }
 
-      def stateOperatorProgresses: Seq[StateOperatorProgress] = {
-        val operatorProgress = mutable.ArrayBuffer[StateOperatorProgress]()
-        var progress = query.recentProgress.last
-
-        operatorProgress ++= progress.stateOperators.map { op => op.copy(op.numRowsUpdated) }
-        if (progress.numInputRows == 0) {
-          // empty batch, merge metrics from previous batch as well
-          progress = query.recentProgress.takeRight(2).head
-          operatorProgress.zipWithIndex.foreach { case (sop, index) =>
-            // "numRowsUpdated" should be merged, as it could be updated in both batches.
-            // (for now it is only updated from previous batch, but things can be changed.)
-            // other metrics represent current status of state so picking up the latest values.
-            val newOperatorProgress = sop.copy(
-              sop.numRowsUpdated + progress.stateOperators(index).numRowsUpdated)
-            operatorProgress(index) = newOperatorProgress
-          }
-        }
+      // Pick the latest progress that actually ran a batch
+      def lastExecutedBatch: StreamingQueryProgress = {
+        query.recentProgress.filter(_.durationMs.containsKey("addBatch")).last
+      }
 
-        operatorProgress
+      def stateOperatorProgresses: Seq[StateOperatorProgress] = {
+        lastExecutedBatch.stateOperators
       }
     }
 
+    val clock = new StreamManualClock()
+
     testStream(aggWithWatermark)(
+      // batchId 0
       AddData(inputData, 15),
-      CheckAnswer(), // watermark = 5
+      StartStream(Trigger.ProcessingTime("interval 1 second"), clock),
+      CheckAnswer(), // watermark = 0
       AssertOnQuery { _.stateNodes.size === 1 },
       AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 1 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 1 without data
+      AdvanceManualClock(1000L), // watermark = 5
+      Execute { q =>             // wait for the no data batch to complete
+        eventually(timeout(streamingTimeout)) { assert(q.lastProgress.batchId === 1) }
+      },
+      CheckAnswer(),
+      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 0 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 1 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 2 with data
       AddData(inputData, 10, 12, 14),
-      CheckAnswer(), // watermark = 5
-      AssertOnQuery { _.stateNodes.size === 1 },
+      AdvanceManualClock(1000L), // watermark = 5
+      CheckAnswer(),
       AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 3 with data
       AddData(inputData, 25),
-      CheckAnswer((10, 3)), // watermark = 15
-      AssertOnQuery { _.stateNodes.size === 1 },
-      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 1 },
+      AdvanceManualClock(1000L), // watermark = 5
+      CheckAnswer(),
+      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 0 },
       AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 1 },
-      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 }
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 3 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 0 },
+
+      // batchId 4 without data
+      AdvanceManualClock(1000L), // watermark = 15
+      Execute { q =>             // wait for the no data batch to complete
+        eventually(timeout(streamingTimeout)) { assert(q.lastProgress.batchId === 4) }
+      },
+      CheckAnswer((10, 3)),
+      AssertOnQuery { _.stateNodes.head.metrics("numOutputRows").value === 1 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsUpdated === 0 },
+      AssertOnQuery { _.stateOperatorProgresses.head.numRowsTotal === 2 },
+      AssertOnQuery { _.lastExecutedBatch.sink.numOutputRows == 1 }
     )
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
index cfd7204ea2931..f63778aef5a7f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
@@ -280,7 +280,8 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
         { // State should have been cleaned if flag is set, otherwise should not have been cleaned
           if (flag) assertNumStateRows(total = 1, updated = 1)
           else assertNumStateRows(total = 7, updated = 1)
-        }
+        },
+        AssertOnQuery(q => q.lastProgress.sink.numOutputRows == 0L)
       )
     }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
index 6f00b528cb8bd..08b3644745f9a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryStatusAndProgressSuite.scala
@@ -241,6 +241,7 @@ class StreamingQueryStatusAndProgressSuite extends StreamTest with Eventually {
           assert(nextProgress.numInputRows === 0)
           assert(nextProgress.stateOperators.head.numRowsTotal === 2)
           assert(nextProgress.stateOperators.head.numRowsUpdated === 0)
+          assert(nextProgress.sink.numOutputRows === 0)
         }
       } finally {
         query.stop()

From d09d49df2ec457bbc4bcb6357920d16943bc4016 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 5 Feb 2020 11:38:33 +0900
Subject: [PATCH 0457/1280] [SPARK-27870][PYTHON][FOLLOW-UP] Rename
 spark.sql.pandas.udf.buffer.size to
 spark.sql.execution.pandas.udf.buffer.size

### What changes were proposed in this pull request?

This PR renames `spark.sql.pandas.udf.buffer.size` to `spark.sql.execution.pandas.udf.buffer.size` to be more consistent with other pandas configuration prefixes, given:
-  `spark.sql.execution.pandas.arrowSafeTypeConversion`
- `spark.sql.execution.pandas.respectSessionTimeZone`
- `spark.sql.legacy.execution.pandas.groupedMap.assignColumnsByName`
- other configurations like `spark.sql.execution.arrow.*`.

### Why are the changes needed?

To make configuration names consistent.

### Does this PR introduce any user-facing change?

No because this configuration was not released yet.

### How was this patch tested?

Existing tests should cover.

Closes #27450 from HyukjinKwon/SPARK-27870-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/tests/test_pandas_udf_scalar.py              | 2 +-
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index 8e7e85ffe6b73..b07de3c0b3cc9 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -868,7 +868,7 @@ def test_close(batch_iter):
 
             with QuietTest(self.sc):
                 with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": 1,
-                                    "spark.sql.pandas.udf.buffer.size": 4}):
+                                    "spark.sql.execution.pandas.udf.buffer.size": 4}):
                     self.spark.range(10).repartition(1) \
                         .select(test_close(col("id"))).limit(2).collect()
                     # wait here because python udf worker will take some time to detect
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 2b5c68edbfa56..f6b0bbdc627ae 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1819,7 +1819,7 @@ object SQLConf {
       .createWithDefault(10000)
 
   val PANDAS_UDF_BUFFER_SIZE =
-    buildConf("spark.sql.pandas.udf.buffer.size")
+    buildConf("spark.sql.execution.pandas.udf.buffer.size")
       .doc(
         s"Same as `${BUFFER_SIZE.key}` but only applies to Pandas UDF executions. If it is not " +
         s"set, the fallback is `${BUFFER_SIZE.key}`. Note that Pandas execution requires more " +

From c75d4aa9c7b7c3a6a090b61ae82b2152d780eabb Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Wed, 8 Apr 2020 12:25:06 +0900
Subject: [PATCH 0458/1280] [SPARK-31269][DOC][FOLLOWUP][MINOR] Add version
 head of GraphX table

### What changes were proposed in this pull request?
HyukjinKwon have ported back all the PR about version to branch-3.0.
I make a double check and found GraphX table lost version head.
This PR will fix the issue.
HyukjinKwon, please help me merge this PR to master and branch-3.0

### Why are the changes needed?
Add version head of GraphX table

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test.

Closes #28149 from beliefer/fix-head-of-graphx-table.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0fc859b4d56fc13bffa1937720d0a988bc7486ac)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index fae3bb4d8309f..8e78ef2d0dcda 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2812,7 +2812,7 @@ Spark subsystems.
 ### GraphX
 
 <table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td><code>spark.graphx.pregel.checkpointInterval</code></td>
   <td>-1</td>

From dea237c08d5dad3ac186ec15d7c59e3be86c021e Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Wed, 8 Apr 2020 12:27:40 +0900
Subject: [PATCH 0459/1280] [SPARK-31351][DOC] Migration Guide Auditing for
 Spark 3.0 Release

This PR is to audit the migration guides in Spark 3.0 release:

- correct the grammar errors
- clean up some items
- replace HTML table by markdown table

N/A

No

Screenshot:

![screencapture-127-0-0-1-4000-sql-migration-guide-html-2020-04-04-21_36_29](https://user-images.githubusercontent.com/11567269/78467043-9477d800-76bd-11ea-8ab0-3d51ea5e9fa5.png)
![Screen Shot 2020-04-04 at 9 28 13 PM](https://user-images.githubusercontent.com/11567269/78467045-98a3f580-76bd-11ea-9e4b-927bf12e683a.png)
![Screen Shot 2020-04-04 at 9 28 02 PM](https://user-images.githubusercontent.com/11567269/78467046-98a3f580-76bd-11ea-8ea3-9f13cb8d200b.png)
![Screen Shot 2020-04-04 at 9 21 40 PM](https://user-images.githubusercontent.com/11567269/78467047-993c8c00-76bd-11ea-8c29-91afc68eb590.png)

Closes #28125 from gatorsmile/updateMigrationGuide3.0.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a3d83948b81cb5c47f6e084801373443f54747d8)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/core-migration-guide.md    |   6 +-
 docs/css/main.css               |  31 +++
 docs/pyspark-migration-guide.md |  78 ++-----
 docs/sql-migration-guide.md     | 358 ++++++++++----------------------
 docs/ss-migration-guide.md      |   2 +-
 5 files changed, 162 insertions(+), 313 deletions(-)

diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index fdb0afad6af9b..66a489bcc8be3 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -26,7 +26,7 @@ license: |
 
 - The `org.apache.spark.ExecutorPlugin` interface and related configuration has been replaced with
   `org.apache.spark.plugin.SparkPlugin`, which adds new functionality. Plugins using the old
-  interface need to be modified to extend the new interfaces. Check the
+  interface must be modified to extend the new interfaces. Check the
   [Monitoring](monitoring.html) guide for more details.
 
 - Deprecated method `TaskContext.isRunningLocally` has been removed. Local execution was removed and it always has returned `false`.
@@ -35,6 +35,6 @@ license: |
 
 - Deprecated method `AccumulableInfo.apply` have been removed because creating `AccumulableInfo` is disallowed.
 
-- Event log file will be written as UTF-8 encoding, and Spark History Server will replay event log files as UTF-8 encoding. Previously Spark writes event log file as default charset of driver JVM process, so Spark History Server of Spark 2.x is needed to read the old event log files in case of incompatible encoding.
+- Event log file will be written as UTF-8 encoding, and Spark History Server will replay event log files as UTF-8 encoding. Previously Spark wrote the event log file as default charset of driver JVM process, so Spark History Server of Spark 2.x is needed to read the old event log files in case of incompatible encoding.
 
-- A new protocol for fetching shuffle blocks is used. It's recommended that external shuffle services be upgraded when running Spark 3.0 apps. Old external shuffle services can still be used by setting the configuration `spark.shuffle.useOldFetchProtocol` to `true`. Otherwise, Spark may run into errors with messages like `IllegalArgumentException: Unexpected message type: <number>`.
+- A new protocol for fetching shuffle blocks is used. It's recommended that external shuffle services be upgraded when running Spark 3.0 apps. You can still use old external shuffle services by setting the configuration `spark.shuffle.useOldFetchProtocol` to `true`. Otherwise, Spark may run into errors with messages like `IllegalArgumentException: Unexpected message type: <number>`.
diff --git a/docs/css/main.css b/docs/css/main.css
index dc05d287996be..bb34d6e9bec0d 100755
--- a/docs/css/main.css
+++ b/docs/css/main.css
@@ -2,6 +2,37 @@
    Author's custom styles
    ========================================================================== */
 
+table {
+  margin: 15px 0;
+  padding: 0;
+}
+
+table tr {
+  border-top: 1px solid #cccccc;
+  background-color: white;
+  margin: 0;
+  padding: 0;
+}
+
+table tr:nth-child(2n) {
+  background-color: #F1F4F5;
+}
+
+table tr th {
+  font-weight: bold;
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px;
+}
+
+table tr td {
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px;
+}
+
 .navbar .brand {
   height: 50px;
   width: 110px;
diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index 03e062f002680..92388fffcea52 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -27,67 +27,25 @@ Many items of SQL migration can be applied when migrating PySpark to higher vers
 Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.html).
 
 ## Upgrading from PySpark 2.4 to 3.0
+- In Spark 3.0, PySpark requires a pandas version of 0.23.2 or higher to use pandas related functionality, such as `toPandas`, `createDataFrame` from pandas DataFrame, and so on.
 
-  - Since Spark 3.0, PySpark requires a Pandas version of 0.23.2 or higher to use Pandas related functionality, such as `toPandas`, `createDataFrame` from Pandas DataFrame, etc.
-
-  - Since Spark 3.0, PySpark requires a PyArrow version of 0.12.1 or higher to use PyArrow related functionality, such as `pandas_udf`, `toPandas` and `createDataFrame` with "spark.sql.execution.arrow.enabled=true", etc.
-
-  - In PySpark, when creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. Since 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a `SparkSession`.
-
-  - In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting Pandas.Series to Arrow array during serialization. Arrow will raise errors when detecting unsafe type conversion like overflow. Setting `spark.sql.execution.pandas.convertToArrowArraySafely` to true can enable it. The default setting is false. PySpark's behavior for Arrow versions is illustrated in the table below:
-    <table class="table">
-        <tr>
-          <th>
-            <b>PyArrow version</b>
-          </th>
-          <th>
-            <b>Integer Overflow</b>
-          </th>
-          <th>
-            <b>Floating Point Truncation</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            version < 0.11.0
-          </td>
-          <td>
-            Raise error
-          </td>
-          <td>
-            Silently allows
-          </td>
-        </tr>
-        <tr>
-          <td>
-            version > 0.11.0, arrowSafeTypeConversion=false
-          </td>
-          <td>
-            Silent overflow
-          </td>
-          <td>
-            Silently allows
-          </td>
-        </tr>
-        <tr>
-          <td>
-            version > 0.11.0, arrowSafeTypeConversion=true
-          </td>
-          <td>
-            Raise error
-          </td>
-          <td>
-            Raise error
-          </td>
-        </tr>
-    </table>
-
-  - Since Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation.
-
-  - Since Spark 3.0, `Column.getItem` is fixed such that it does not call `Column.apply`. Consequently, if `Column` is used as an argument to `getItem`, the indexing operator should be used.
-    For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
-
-  - As of Spark 3.0 `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to "true" for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
+- In Spark 3.0, PySpark requires a PyArrow version of 0.12.1 or higher to use PyArrow related functionality, such as `pandas_udf`, `toPandas` and `createDataFrame` with "spark.sql.execution.arrow.enabled=true", etc.
+
+- In PySpark, when creating a `SparkSession` with `SparkSession.builder.getOrCreate()`, if there is an existing `SparkContext`, the builder was trying to update the `SparkConf` of the existing `SparkContext` with configurations specified to the builder, but the `SparkContext` is shared by all `SparkSession`s, so we should not update them. In 3.0, the builder comes to not update the configurations. This is the same behavior as Java/Scala API in 2.3 and above. If you want to update them, you need to update them prior to creating a `SparkSession`.
+
+- In PySpark, when Arrow optimization is enabled, if Arrow version is higher than 0.11.0, Arrow can perform safe type conversion when converting `pandas.Series` to an Arrow array during serialization. Arrow raises errors when detecting unsafe type conversions like overflow. You enable it by setting `spark.sql.execution.pandas.convertToArrowArraySafely` to `true`. The default setting is `false`. PySpark behavior for Arrow versions is illustrated in the following table:
+
+  | PyArrow version  | Integer overflow | Floating point truncation |
+  | ---------------- | ---------------- | ------------------------- |
+  | 0.11.0 and below | Raise error      | Silently allows           |
+  | \> 0.11.0, arrowSafeTypeConversion=false | Silent overflow  | Silently allows |
+  | \> 0.11.0, arrowSafeTypeConversion=true  | Raise error      | Raise error |
+   
+- In Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation.
+
+- In Spark 3.0, `Column.getItem` is fixed such that it does not call `Column.apply`. Consequently, if `Column` is used as an argument to `getItem`, the indexing operator should be used. For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
+
+- As of Spark 3.0, `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to `true` for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
 
 ## Upgrading from PySpark 2.3 to 2.4
 
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 5a2b848fb7a61..f5c81e9e575f6 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -26,329 +26,189 @@ license: |
 
 ### Dataset/DataFrame APIs
 
-  - Since Spark 3.0, the Dataset and DataFrame API `unionAll` is not deprecated any more. It is an alias for `union`.
+  - In Spark 3.0, the Dataset and DataFrame API `unionAll` is no longer deprecated. It is an alias for `union`.
 
-  - In Spark version 2.4 and earlier, `Dataset.groupByKey` results to a grouped dataset with key attribute wrongly named as "value", if the key is non-struct type, e.g. int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries weird. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behaviour is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
+  - In Spark 2.4 and below, `Dataset.groupByKey` results to a grouped dataset with key attribute is wrongly named as "value", if the key is non-struct type, for example, int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries unexpected. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behavior is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
 
 ### DDL Statements
 
-  - Since Spark 3.0, `CREATE TABLE` without a specific provider will use the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and earlier, it was hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
+  - In Spark 3.0, `CREATE TABLE` without a specific provider uses the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and below, it was Hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
 
-  - Since Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception will be thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and earlier, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to a integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
+  - In Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception is thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and below, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to a integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
 
   - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
 
-  - In Spark version 2.4 and earlier, the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. Since 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
+  - Spark 2.4 and below: the `SET` command works without any warnings even if the specified key is for `SparkConf` entries and it has no effect because the command does not update `SparkConf`, but the behavior might confuse users. In 3.0, the command fails if a `SparkConf` key is used. You can disable such a check by setting `spark.sql.legacy.setCommandRejectsSparkCoreConfs` to `false`.
 
-  - Refreshing a cached table would trigger a table uncache operation and then a table cache (lazily) operation. In Spark version 2.4 and earlier, the cache name and storage level are not preserved before the uncache operation. Therefore, the cache name and storage level could be changed unexpectedly. Since Spark 3.0, cache name and storage level will be first preserved for cache recreation. It helps to maintain a consistent cache behavior upon table refreshing.
+  - Refreshing a cached table would trigger a table uncache operation and then a table cache (lazily) operation. In Spark version 2.4 and below, the cache name and storage level are not preserved before the uncache operation. Therefore, the cache name and storage level could be changed unexpectedly. In Spark 3.0, cache name and storage level are first preserved for cache recreation. It helps to maintain a consistent cache behavior upon table refreshing.
 
-  - Since Spark 3.0, the properties listing below become reserved, commands will fail if we specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. We need their specific clauses to specify them, e.g. `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. We can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, e.g `SET DBPROTERTIES('location'='/tmp')` will affect nothing. In Spark version 2.4 and earlier, these properties are neither reserved nor have side effects, e.g. `SET DBPROTERTIES('location'='/tmp')` will not change the location of the database but only create a headless property just like `'a'='b'`.
-    <table class="table">
-        <tr>
-          <th>
-            <b>Property(case sensitive)</b>
-          </th>
-          <th>
-            <b>Database Reserved</b>
-          </th>
-          <th>
-            <b>Table Reserved</b>
-          </th>
-          <th>
-            <b>Remarks</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            provider
-          </td>
-          <td>
-            no
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For tables, please use the USING clause to specify it. Once set, it can't be changed.
-          </td>
-        </tr>
-        <tr>
-          <td>
-            location
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For databases and tables, please use the LOCATION clause to specify it.
-          </td>
-        </tr>
-        <tr>
-          <td>
-            owner
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            yes
-          </td>
-          <td>
-            For databases and tables, it is determined by the user who runs spark and create the table.
-          </td>
-        </tr>
-    </table>
+  - In Spark 3.0, the properties listing below become reserved; commands fail if you specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. You need their specific clauses to specify them, for example, `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. You can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, for example: `SET DBPROTERTIES('location'='/tmp')` will have no effect. In Spark version 2.4 and below, these properties are neither reserved nor have side effects, for example, `SET DBPROTERTIES('location'='/tmp')` do not change the location of the database but only create a headless property just like `'a'='b'`.
+
+    | Property (case sensitive) | Database Reserved | Table Reserved | Remarks |
+    | ------------------------- | ----------------- | -------------- | ------- |
+    | provider                 | no                | yes            | For tables, use the `USING` clause to specify it. Once set, it can't be changed. |
+    | location                 | yes               | yes            | For databases and tables, use the `LOCATION` clause to specify it. |
+    | owner                    | yes               | yes            | For databases and tables, it is determined by the user who runs spark and create the table. |
 
-  - Since Spark 3.0, `ADD FILE` can be used to add file directories as well. Earlier only single files can be added using this command. To restore the behaviour of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
+ 
+  - In Spark 3.0, you can use `ADD FILE` to add file directories as well. Earlier you could add only single files using this command. To restore the behavior of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
 
-  - Since Spark 3.0, `SHOW TBLPROPERTIES` will cause `AnalysisException` if the table does not exist. In Spark version 2.4 and earlier, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view will cause `AnalysisException`. In Spark version 2.4 and earlier, it returned an empty result.
+  - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view causes `AnalysisException`. In Spark version 2.4 and below, it returned an empty result.
 
-  - Since Spark 3.0, `SHOW CREATE TABLE` will always return Spark DDL, even when the given table is a Hive serde table. For generating Hive DDL, please use `SHOW CREATE TABLE AS SERDE` command instead.
+  - In Spark 3.0, `SHOW CREATE TABLE` always returns Spark DDL, even when the given table is a Hive SerDe table. For generating Hive DDL, use `SHOW CREATE TABLE AS SERDE` command instead.
 
-  - Since Spark 3.0, column of CHAR type is not allowed in non-Hive-Serde tables, and CREATE/ALTER TABLE commands will fail if CHAR type is detected. Please use STRING type instead. In Spark version 2.4 and earlier, CHAR type is treated as STRING type and the length parameter is simply ignored.
+  - In Spark 3.0, column of CHAR type is not allowed in non-Hive-Serde tables, and CREATE/ALTER TABLE commands will fail if CHAR type is detected. Please use STRING type instead. In Spark version 2.4 and below, CHAR type is treated as STRING type and the length parameter is simply ignored.
 
 ### UDFs and Built-in Functions
 
-  - Since Spark 3.0, the `date_add` and `date_sub` functions only accept int, smallint, tinyint as the 2nd argument, fractional and non-literal string are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), 12.34)` will cause `AnalysisException`. Note that, string literals are still allowed, but Spark will throw Analysis Exception if the string content is not a valid integer. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
+  - In Spark 3.0, the `date_add` and `date_sub` functions accepts only int, smallint, tinyint as the 2nd argument; fractional and non-literal strings are not valid anymore, for example: `date_add(cast('1964-05-23' as date), '12.34')` causes `AnalysisException`. Note that, string literals are still allowed, but Spark will throw `AnalysisException` if the string content is not a valid integer. In Spark version 2.4 and below, if the 2nd argument is fractional or string value, it is coerced to int value, and the result is a date value of `1964-06-04`.
 
-  - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
+  - In Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, for example, `percentile_approx(10.0, 0.2, 1.8D)` causes `AnalysisException`. In Spark version 2.4 and below, if `accuracy` is fractional or string value, it is coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
 
-  - Since Spark 3.0, an analysis exception will be thrown when hash expressions are applied on elements of MapType. To restore the behavior before Spark 3.0, set `spark.sql.legacy.allowHashOnMapType` to `true`.
+  - In Spark 3.0, an analysis exception is thrown when hash expressions are applied on elements of `MapType`. To restore the behavior before Spark 3.0, set `spark.sql.legacy.allowHashOnMapType` to `true`.
 
-  - Since Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and earlier, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
+  - In Spark 3.0, when the `array`/`map` function is called without any parameters, it returns an empty collection with `NullType` as element type. In Spark version 2.4 and below, it returns an empty collection with `StringType` as element type. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createEmptyCollectionUsingStringType` to `true`.
 
-  - Since Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
+  - In Spark 3.0, the `from_json` functions supports two modes - `PERMISSIVE` and `FAILFAST`. The modes can be set via the `mode` option. The default mode became `PERMISSIVE`. In previous versions, behavior of `from_json` did not conform to either `PERMISSIVE` nor `FAILFAST`, especially in processing of malformed JSON records. For example, the JSON string `{"a" 1}` with the schema `a INT` is converted to `null` by previous versions but Spark 3.0 converts it to `Row(null)`.
 
-  - In Spark version 2.4 and earlier, users can create map values with map type key via built-in function such as `CreateMap`, `MapFromArrays`, etc. Since Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can use `map_entries` function to convert map to array<struct<key, value>> as a workaround. In addition, users can still read map values with map type key from data source or Java/Scala collections, though it is discouraged.
+  - In Spark version 2.4 and below, you can create map values with map type key via built-in function such as `CreateMap`, `MapFromArrays`, etc. In Spark 3.0, it's not allowed to create map values with map type key with these built-in functions. Users can use `map_entries` function to convert map to array<struct<key, value>> as a workaround. In addition, users can still read map values with map type key from data source or Java/Scala collections, though it is discouraged.
 
-  - In Spark version 2.4 and earlier, users can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, e.g. map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. Since Spark 3.0, Spark will throw RuntimeException while duplicated keys are found. Users can set `spark.sql.mapKeyDedupPolicy` to LAST_WIN to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (e.g. Parquet), the behavior will be undefined.
+  - In Spark version 2.4 and below, you can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, for example, map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. In Spark 3.0, Spark throws `RuntimeException` when duplicated keys are found. You can set `spark.sql.mapKeyDedupPolicy` to `LAST_WIN` to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (for example, Parquet), the behavior is undefined.
 
-  - Since Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. But please note that, in Spark version 2.4 and earlier, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF will return null if the input values is null. However, since Spark 3.0, the UDF will return the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` will return null in Spark 2.4 and earlier if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
+  - In Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. In Spark version 2.4 and below, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF returns null if the input values is null. However, in Spark 3.0, the UDF returns the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` returns null in Spark 2.4 and below if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
 
-  - Since Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, i.e., if the `predicate` returns any `null`s and no `true` is obtained, then `exists` will return `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` will be `null`. The previous behaviour can be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
+  - In Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, that is, if the `predicate` returns any `null`s and no `true` is obtained, then `exists` returns `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` is `null`. The previous behaviorcan be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
 
-  - Since Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and earlier, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
+  - In Spark 3.0, the `add_months` function does not adjust the resulting date to a last day of month if the original date is a last day of months. For example, `select add_months(DATE'2019-02-28', 1)` results `2019-03-28`. In Spark version 2.4 and below, the resulting date is adjusted when the original date is a last day of months. For example, adding a month to `2019-02-28` results in `2019-03-31`.
 
-  - In Spark version 2.4 and earlier, the `current_timestamp` function returns a timestamp with millisecond resolution only. Since Spark 3.0, the function can return the result with microsecond resolution if the underlying clock available on the system offers such resolution.
+  - In Spark version 2.4 and below, the `current_timestamp` function returns a timestamp with millisecond resolution only. In Spark 3.0, the function can return the result with microsecond resolution if the underlying clock available on the system offers such resolution.
 
-  - Since Spark 3.0, 0-argument Java UDF is executed in the executor side identically with other UDFs. In Spark version 2.4 and earlier, 0-argument Java UDF alone was executed in the driver side, and the result was propagated to executors, which might be more performant in some cases but caused inconsistency with a correctness issue in some cases.
+  - In Spark 3.0, a 0-argument Java UDF is executed in the executor side identically with other UDFs. In Spark version 2.4 and below, the 0-argument Java UDF alone was executed in the driver side, and the result was propagated to executors, which might be more performant in some cases but caused inconsistency with a correctness issue in some cases.
 
   - The result of `java.lang.Math`'s `log`, `log1p`, `exp`, `expm1`, and `pow` may vary across platforms. In Spark 3.0, the result of the equivalent SQL functions (including related SQL functions like `LOG10`) return values consistent with `java.lang.StrictMath`. In virtually all cases this makes no difference in the return value, and the difference is very small, but may not exactly match `java.lang.Math` on x86 platforms in cases like, for example, `log(3.0)`, whose value varies between `Math.log()` and `StrictMath.log()`.
 
-  - Since Spark 3.0, `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in case insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behaviour change is illustrated in the table below:
-    <table class="table">
-        <tr>
-          <th>
-            <b>Operation</b>
-          </th>
-          <th>
-            <b>Result prior to Spark 3.0</b>
-          </th>
-          <th>
-            <b>Result starting Spark 3.0</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            CAST('infinity' AS DOUBLE)<br>
-            CAST('+infinity' AS DOUBLE)<br>
-            CAST('inf' AS DOUBLE)<br>
-            CAST('+inf' AS DOUBLE)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Double.PositiveInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('-infinity' AS DOUBLE)<br>
-            CAST('-inf' AS DOUBLE)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Double.NegativeInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('infinity' AS FLOAT)<br>
-            CAST('+infinity' AS FLOAT)<br>
-            CAST('inf' AS FLOAT)<br>
-            CAST('+inf' AS FLOAT)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Float.PositiveInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('-infinity' AS FLOAT)<br>
-            CAST('-inf' AS FLOAT)<br>
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Float.NegativeInfinity
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('nan' AS DOUBLE)
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Double.NaN
-          </td>
-        </tr>
-        <tr>
-          <td>
-            CAST('nan' AS FLOAT)
-          </td>
-          <td>
-            NULL
-          </td>
-          <td>
-            Float.NaN
-          </td>
-        </tr>
-    </table>
+  - In Spark 3.0, the `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in a case-insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behavior change is illustrated in the table below:
+
+    | Operation | Result before Spark 3.0 | Result in Spark 3.0 |
+    | --------- | ----------------------- | ------------------- |
+    | CAST('infinity' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('+infinity' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('inf' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('inf' AS DOUBLE) | NULL | Double.PositiveInfinity |
+    | CAST('-infinity' AS DOUBLE) | NULL | Double.NegativeInfinity |
+    | CAST('-inf' AS DOUBLE) | NULL | Double.NegativeInfinity |
+    | CAST('infinity' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('+infinity' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('inf' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('+inf' AS FLOAT) | NULL | Float.PositiveInfinity |
+    | CAST('-infinity' AS FLOAT) | NULL | Float.NegativeInfinity |
+    | CAST('-inf' AS FLOAT) | NULL | Float.NegativeInfinity |
+    | CAST('nan' AS DOUBLE) | NULL | Double.Nan |
+    | CAST('nan' AS FLOAT) | NULL | Float.NaN |
+
+  - In Spark 3.0, when casting interval values to string type, there is no "interval" prefix, for example, `1 days 2 hours`. In Spark version 2.4 and below, the string contains the "interval" prefix like `interval 1 days 2 hours`.
+
+  - In Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, for example, `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and below, when casting string to integrals and booleans, it does not trim the whitespaces from both ends; the foregoing results is `null`, while to datetimes, only the trailing spaces (= ASCII 32) are removed.
 
-  - Since Spark 3.0, when casting interval values to string type, there is no "interval" prefix, e.g. `1 days 2 hours`. In Spark version 2.4 and earlier, the string contains the "interval" prefix like `interval 1 days 2 hours`.
+### Query Engine
 
-  - Since Spark 3.0, when casting string value to integral types(tinyint, smallint, int and bigint), datetime types(date, timestamp and interval) and boolean type, the leading and trailing whitespaces (<= ASCII 32) will be trimmed before converted to these type values, e.g. `cast(' 1\t' as int)` results `1`, `cast(' 1\t' as boolean)` results `true`, `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, while casting string to integrals and booleans, it will not trim the whitespaces from both ends, the foregoing results will be `null`, while to datetimes, only the trailing spaces (= ASCII 32) will be removed.
+  - In Spark version 2.4 and below, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. These queries are treated as invalid in Spark 3.0.
 
-### Query Engine
+  - In Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
 
-  - In Spark version 2.4 and earlier, SQL queries such as `FROM <table>` or `FROM <table> UNION ALL FROM <table>` are supported by accident. In hive-style `FROM <table> SELECT <expr>`, the `SELECT` clause is not negligible. Neither Hive nor Presto support this syntax. Therefore we will treat these queries as invalid since Spark 3.0.
+  - In Spark 3.0, numbers written in scientific notation(for example, `1E2`) would be parsed as Double. In Spark version 2.4 and below, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
 
-  - Since Spark 3.0, the interval literal syntax does not allow multiple from-to units anymore. For example, `SELECT INTERVAL '1-1' YEAR TO MONTH '2-2' YEAR TO MONTH'` throws parser exception.
+  - In Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
 
-  - Since Spark 3.0, numbers written in scientific notation(e.g. `1E2`) would be parsed as Double. In Spark version 2.4 and earlier, they're parsed as Decimal. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.exponentLiteralAsDecimal.enabled` to `true`.
+  - In Spark 3.0, negative scale of decimal is not allowed by default, for example, data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and below, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
 
-  - Since Spark 3.0, day-time interval strings are converted to intervals with respect to the `from` and `to` bounds. If an input string does not match to the pattern defined by specified bounds, the `ParseException` exception is thrown. For example, `interval '2 10:20' hour to minute` raises the exception because the expected format is `[+|-]h[h]:[m]m`. In Spark version 2.4, the `from` bound was not taken into account, and the `to` bound was used to truncate the resulted interval. For instance, the day-time interval string from the showed example is converted to `interval 10 hours 20 minutes`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.fromDayTimeString.enabled` to `true`.
-  
-  - Since Spark 3.0, negative scale of decimal is not allowed by default, e.g. data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and earlier, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
+  - In Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation is coerced to a double value, for example, `+'1'` returns `1.0`. In Spark version 2.4 and below, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, for example, `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, for example, in Spark 2.4, the result of `+'1'` is string `1`.
 
-  - Since Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation will be coerced to double value, e.g. `+'1'` results `1.0`. In Spark version 2.4 and earlier, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, e.g. `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, e.g. in Spark 2.4, the result of `+'1'` is string `1`.
+  - In Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
 
-  - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
+  - In Spark 3.0, `spark.sql.legacy.ctePrecedencePolicy` is introduced to control the behavior for name conflicting in the nested WITH clause. By default value `EXCEPTION`, Spark throws an AnalysisException, it forces users to choose the specific substitution order they wanted. If set to `CORRECTED` (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `LEGACY`, the result is `1` which is the behavior in version 2.4 and below.
 
-  - Since Spark 3.0, `spark.sql.legacy.ctePrecedencePolicy` is introduced to control the behavior for name conflicting in the nested WITH clause. By default value `EXCEPTION`, Spark throws an AnalysisException, it forces users to choose the specific substitution order they wanted. If set to `CORRECTED` (which is recommended), inner CTE definitions take precedence over outer definitions. For example, set the config to `false`, `WITH t AS (SELECT 1), t2 AS (WITH t AS (SELECT 2) SELECT * FROM t) SELECT * FROM t2` returns `2`, while setting it to `LEGACY`, the result is `1` which is the behavior in version 2.4 and earlier.
+  - In Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
 
-  - Since Spark 3.0, configuration `spark.sql.crossJoin.enabled` become internal configuration, and is true by default, so by default spark won't raise exception on sql with implicit cross join.
+  - In Spark version 2.4 and below, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys, and join keys. In Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and below.
 
-  - In Spark version 2.4 and earlier, float/double -0.0 is semantically equal to 0.0, but -0.0 and 0.0 are considered as different values when used in aggregate grouping keys, window partition keys and join keys. Since Spark 3.0, this bug is fixed. For example, `Seq(-0.0, 0.0).toDF("d").groupBy("d").count()` returns `[(0.0, 2)]` in Spark 3.0, and `[(0.0, 1), (-0.0, 1)]` in Spark 2.4 and earlier.
+  - In Spark version 2.4 and below, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. In Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
 
-  - In Spark version 2.4 and earlier, invalid time zone ids are silently ignored and replaced by GMT time zone, for example, in the from_utc_timestamp function. Since Spark 3.0, such time zone ids are rejected, and Spark throws `java.time.DateTimeException`.
+  - In Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and so on. Spark 3.0 uses Java 8 API classes from the `java.time` packages that are based on [ISO chronology](https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and below, those operations are performed using the hybrid calendar ([Julian + Gregorian](https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
-  - Since Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and etc. Spark 3.0 uses Java 8 API classes from the java.time packages that based on ISO chronology (https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and earlier, those operations are performed by using the hybrid calendar (Julian + Gregorian, see https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
+    * Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. In Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and below, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in [simpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
 
-    - Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. Since Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and earlier, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html. The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
+    * The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
 
-    - The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
+    * The JDBC options `lowerBound` and `upperBound` are converted to TimestampType/DateType values in the same way as casting strings to TimestampType/DateType values. The conversion is based on Proleptic Gregorian calendar, and time zone defined by the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and below, the conversion is based on the hybrid calendar (Julian + Gregorian) and on default system time zone.
 
-    - the JDBC options `lowerBound` and `upperBound` are converted to TimestampType/DateType values in the same way as casting strings to TimestampType/DateType values. The conversion is based on Proleptic Gregorian calendar, and time zone defined by the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion is based on the hybrid calendar (Julian + Gregorian) and on default system time zone.
+    * Formatting `TIMESTAMP` and `DATE` literals.
 
-    - Formatting of `TIMESTAMP` and `DATE` literals.
+    * Creating typed `TIMESTAMP` and `DATE` literals from strings. In Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. When the input string does not contain information about time zone, the time zone from the SQL config `spark.sql.session.timeZone` is used in that case. In Spark version 2.4 and below, the conversion is based on JVM system time zone. The different sources of the default time zone may change the behavior of typed `TIMESTAMP` and `DATE` literals.
 
-    - Creating of typed `TIMESTAMP` and `DATE` literals from strings. Since Spark 3.0, string conversion to typed `TIMESTAMP`/`DATE` literals is performed via casting to `TIMESTAMP`/`DATE` values. For example, `TIMESTAMP '2019-12-23 12:59:30'` is semantically equal to `CAST('2019-12-23 12:59:30' AS TIMESTAMP)`. When the input string does not contain information about time zone, the time zone from the SQL config `spark.sql.session.timeZone` is used in that case. In Spark version 2.4 and earlier, the conversion is based on JVM system time zone. The different sources of the default time zone may change the behavior of typed `TIMESTAMP` and `DATE` literals.
+  - In Spark 3.0, `TIMESTAMP` literals are converted to strings using the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and below, the conversion uses the default time zone of the Java virtual machine.
 
-  - Since Spark 3.0, `TIMESTAMP` literals are converted to strings using the SQL config `spark.sql.session.timeZone`. In Spark version 2.4 and earlier, the conversion uses the default time zone of the Java virtual machine.
+  - In Spark 3.0, Spark casts `String` to `Date/Timestamp` in binary comparisons with dates/timestamps. The previous behavior of casting `Date/Timestamp` to `String` can be restored by setting `spark.sql.legacy.typeCoercion.datetimeToString.enabled` to `true`.
 
-  - Since Spark 3.0, Spark will cast `String` to `Date/TimeStamp` in binary comparisons with dates/timestamps. The previous behaviour of casting `Date/Timestamp` to `String` can be restored by setting `spark.sql.legacy.typeCoercion.datetimeToString.enabled` to `true`.
+  - In Spark 3.0, special values are supported in conversion from strings to dates and timestamps. Those values are simply notational shorthands that are converted to ordinary date or timestamp values when read. The following string values are supported for dates:
+    * `epoch [zoneId]` - 1970-01-01
+    * `today [zoneId]` - the current date in the time zone specified by `spark.sql.session.timeZone`
+    * `yesterday [zoneId]` - the current date - 1
+    * `tomorrow [zoneId]` - the current date + 1
+    * `now` - the date of running the current query. It has the same notion as today
 
-  - Since Spark 3.0, special values are supported in conversion from strings to dates and timestamps. Those values are simply notational shorthands that will be converted to ordinary date or timestamp values when read. The following string values are supported for dates:
-    - `epoch [zoneId]` - 1970-01-01
-    - `today [zoneId]` - the current date in the time zone specified by `spark.sql.session.timeZone`
-    - `yesterday [zoneId]` - the current date - 1
-    - `tomorrow [zoneId]` - the current date + 1
-    - `now` - the date of running the current query. It has the same notion as today
-  For example `SELECT date 'tomorrow' - date 'yesterday';` should output `2`. Here are special timestamp values:
-    - `epoch [zoneId]` - 1970-01-01 00:00:00+00 (Unix system time zero)
-    - `today [zoneId]` - midnight today
-    - `yesterday [zoneId]` - midnight yesterday
-    - `tomorrow [zoneId]` - midnight tomorrow
-    - `now` - current query start time
-  For example `SELECT timestamp 'tomorrow';`.
+    For example `SELECT date 'tomorrow' - date 'yesterday';` should output `2`. Here are special timestamp values:
+    * `epoch [zoneId]` - 1970-01-01 00:00:00+00 (Unix system time zero)
+    * `today [zoneId]` - midnight today
+    * `yesterday [zoneId]` - midnight yesterday
+    * `tomorrow [zoneId]` - midnight tomorrow
+    * `now` - current query start time
+
+    For example `SELECT timestamp 'tomorrow';`.
 
   - Since Spark 3.0, when using `EXTRACT` expression to extract the second field from date/timestamp values, the result will be a `DecimalType(8, 6)` value with 2 digits for second part, and 6 digits for the fractional part with microsecond precision. e.g. `extract(second from to_timestamp('2019-09-20 10:10:10.1'))` results `10.100000`.  In Spark version 2.4 and earlier, it returns an `IntegerType` value and the result for the former example is `10`.
 
 ### Data Sources
 
-  - In Spark version 2.4 and earlier, when reading a Hive Serde table with Spark native data sources(parquet/orc), Spark will infer the actual file schema and update the table schema in metastore. Since Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, please set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
+  - In Spark version 2.4 and below, when reading a Hive SerDe table with Spark native data sources(parquet/orc), Spark infers the actual file schema and update the table schema in metastore. In Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
 
-  - In Spark version 2.4 and earlier, partition column value is converted as null if it can't be casted to corresponding user provided schema. Since 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting `spark.sql.sources.validatePartitionColumns` to `false`.
+  - In Spark version 2.4 and below, partition column value is converted as null if it can't be casted to corresponding user provided schema. In 3.0, partition column value is validated with user provided schema. An exception is thrown if the validation fails. You can disable such validation by setting  `spark.sql.sources.validatePartitionColumns` to `false`.
 
-  - Since Spark 3.0, if files or subdirectories disappear during recursive directory listing (i.e. they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
+  - In Spark 3.0, if files or subdirectories disappear during recursive directory listing (that is, they appear in an intermediate listing but then cannot be read or listed during later phases of the recursive directory listing, due to either concurrent file deletions or object store consistency issues) then the listing will fail with an exception unless `spark.sql.files.ignoreMissingFiles` is `true` (default `false`). In previous versions, these missing files or subdirectories would be ignored. Note that this change of behavior only applies during initial table file listing (or during `REFRESH TABLE`), not during query execution: the net change is that `spark.sql.files.ignoreMissingFiles` is now obeyed during table file listing / query planning, not only at query execution time.
 
-  - In Spark version 2.4 and earlier, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType`, `DoubleType`, `DateType` and `TimestampType`, it fails on empty strings and throws exceptions. Since Spark 3.0, we disallow empty strings and will throw exceptions for data types except for `StringType` and `BinaryType`. The previous behaviour of allowing empty string can be restored by setting `spark.sql.legacy.json.allowEmptyString.enabled` to `true`.
+  - In Spark version 2.4 and below, the parser of JSON data source treats empty strings as null for some data types such as `IntegerType`. For `FloatType`, `DoubleType`, `DateType` and `TimestampType`, it fails on empty strings and throws exceptions. Spark 3.0 disallows empty strings and will throw an exception for data types except for `StringType` and `BinaryType`. The previous behavior of allowing an empty string can be restored by setting `spark.sql.legacy.json.allowEmptyString.enabled` to `true`.
 
-  - In Spark version 2.4 and earlier, JSON datasource and JSON functions like `from_json` convert a bad JSON record to a row with all `null`s in the PERMISSIVE mode when specified schema is `StructType`. Since Spark 3.0, the returned row can contain non-`null` fields if some of JSON column values were parsed and converted to desired types successfully.
+  - In Spark version 2.4 and below, JSON datasource and JSON functions like `from_json` convert a bad JSON record to a row with all `null`s in the PERMISSIVE mode when specified schema is `StructType`. In Spark 3.0, the returned row can contain non-`null` fields if some of JSON column values were parsed and converted to desired types successfully.
 
-  - Since Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inferring.
+  - In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Set JSON option `inferTimestamp` to `false` to disable such type inference.
 
-  - In Spark version 2.4 and earlier, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. Since Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
+  - In Spark version 2.4 and below, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. In Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
 
-  - Since Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and earlier, `TIMESTAMP` columns are saved as `INT96` in parquet files. Note that, some SQL systems such as Hive 1.x and Impala 2.x can only read `INT96` timestamps, you can set `spark.sql.parquet.outputTimestampType` as `INT96` to restore the previous behavior and keep interoperability.
+  - In Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and below, `TIMESTAMP` columns are saved as `INT96` in parquet files. Note that, some SQL systems such as Hive 1.x and Impala 2.x can only read `INT96` timestamps, you can set `spark.sql.parquet.outputTimestampType` as `INT96` to restore the previous behavior and keep interoperability.
 
-  - Since Spark 3.0, when Avro files are written with user provided schema, the fields will be matched by field names between catalyst schema and avro schema instead of positions.
+  - In Spark 3.0, when Avro files are written with user provided schema, the fields are matched by field names between catalyst schema and Avro schema instead of positions.
 
-  - Since Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark will throw runtime NPE if any of the records contains null.
+  - In Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark throws runtime NullPointerException if any of the records contains null.
 
 ### Others
 
-  - In Spark version 2.4, when a spark session is created via `cloneSession()`, the newly created spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent spark session. Since Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. The old behavior can be restored by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
+  - In Spark version 2.4, when a Spark session is created via `cloneSession()`, the newly created Spark session inherits its configuration from its parent `SparkContext` even though the same configuration may exist with a different value in its parent Spark session. In Spark 3.0, the configurations of a parent `SparkSession` have a higher precedence over the parent `SparkContext`. You can restore the old behavior by setting `spark.sql.legacy.sessionInitWithConfigDefaults` to `true`.
 
-  - Since Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it will fallback to hive-site.xml present in the `Hadoop configuration` of `SparkContext`.
+  - In Spark 3.0, if `hive.default.fileformat` is not found in `Spark SQL configuration` then it falls back to the `hive-site.xml` file present in the `Hadoop configuration` of `SparkContext`.
 
-  - Since Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
-    <table class="table">
-        <tr>
-          <th>
-            <b>Query</b>
-          </th>
-          <th>
-            <b>Spark 2.4 or Prior</b>
-          </th>
-          <th>
-            <b>Spark 3.0</b>
-          </th>
-        </tr>
-        <tr>
-          <td>
-            <code>SELECT CAST(1 AS decimal(38, 18));</code>
-          </td>
-          <td>
-            <code>1</code>
-          </td>
-          <td>
-            <code>1.000000000000000000</code>
-          </td>
-        </tr>
-    </table>
+  - In Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
+
+    | Query | Spark 2.4 | Spark 3.0 |
+    | ----- | --------- | --------- |
+    |`SELECT CAST(1 AS decimal(38, 18));` | 1 | 1.000000000000000000 |
+
+  - In Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3 and it brings following impacts:
+
+    * You may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore you want to connect to. For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
 
-  - Since Spark 3.0, we upgraded the built-in Hive from 1.2 to 2.3 and it brings following impacts:
-  
-    - You may need to set `spark.sql.hive.metastore.version` and `spark.sql.hive.metastore.jars` according to the version of the Hive metastore you want to connect to.
-  For example: set `spark.sql.hive.metastore.version` to `1.2.1` and `spark.sql.hive.metastore.jars` to `maven` if your Hive metastore version is 1.2.1.
-  
-    - You need to migrate your custom SerDes to Hive 2.3 or build your own Spark with `hive-1.2` profile. See HIVE-15167 for more details.
+    * You need to migrate your custom SerDes to Hive 2.3 or build your own Spark with `hive-1.2` profile. See [HIVE-15167](https://issues.apache.org/jira/browse/HIVE-15167) for more details.
 
-    - The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
+    * The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
 
 ## Upgrading from Spark SQL 2.4.4 to 2.4.5
 
diff --git a/docs/ss-migration-guide.md b/docs/ss-migration-guide.md
index db8fdff8b2ac4..963ef07af7ace 100644
--- a/docs/ss-migration-guide.md
+++ b/docs/ss-migration-guide.md
@@ -30,4 +30,4 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
 - In Spark 3.0, Structured Streaming forces the source schema into nullable when file-based datasources such as text, json, csv, parquet and orc are used via `spark.readStream(...)`. Previously, it respected the nullability in source schema; however, it caused issues tricky to debug with NPE. To restore the previous behavior, set `spark.sql.streaming.fileSource.schema.forceNullable` to `false`.
 
-- Spark 3.0 fixes the correctness issue on Stream-stream outer join, which changes the schema of state. (SPARK-26154 for more details) Spark 3.0 will fail the query if you start your query from checkpoint constructed from Spark 2.x which uses stream-stream outer join. Please discard the checkpoint and replay previous inputs to recalculate outputs.
\ No newline at end of file
+- Spark 3.0 fixes the correctness issue on Stream-stream outer join, which changes the schema of state. (See [SPARK-26154](https://issues.apache.org/jira/browse/SPARK-26154) for more details). If you start your query from checkpoint constructed from Spark 2.x which uses stream-stream outer join, Spark 3.0 fails the query. To recalculate outputs, discard the checkpoint and replay previous inputs.

From 2ced9825913d4cb406263329c909619c1c386f75 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 8 Apr 2020 17:54:28 +0900
Subject: [PATCH 0460/1280] [SPARK-31379][CORE][TEST] Fix flaky
 o.a.s.scheduler.CoarseGrainedSchedulerBackendSuite.extra resources from
 executor

### What changes were proposed in this pull request?

This PR (SPARK-31379) adds one line `when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]])).thenReturn(Seq.empty)` to avoid allocating resources.

### Why are the changes needed?

The test is flaky and here's part of error stack:

```
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedDueToTimeoutException:
The code passed to eventually never returned normally. Attempted 325 times over 5.01070979 seconds.
Last failure message: ArrayBuffer("1", "3") did not equal Array("0", "1", "3").
...
org.apache.spark.scheduler.CoarseGrainedSchedulerBackendSuite.eventually(CoarseGrainedSchedulerBackendSuite.scala:45)
```

You can check [here](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120786/testReport/org.apache.spark.scheduler/CoarseGrainedSchedulerBackendSuite/extra_resources_from_executor/) for details.

And it is flaky because:  after sending `StatusUpdate` to `CoarseGrainedSchedulerBackend`, `CoarseGrainedSchedulerBackend` will call `makeOffer` immediately once releasing the resources. So, it's possible that `availableAddrs` has allocated again before we assert `execResources(GPU).availableAddrs.sorted === Array("0", "1", "3")`.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

The issue can be stably reproduced by inserting `Thread.sleep(3000)` after the line of sending `StatusUpdate`. After applying this fix, the issue is gone.

Closes #28145 from Ngone51/fix_flaky.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a2789c2a5179f6ed5613c24130bc80bf1bbfd753)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
index c063301673598..fad971fd4ddec 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
@@ -251,6 +251,9 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
       assert(execResources(GPU).assignedAddrs === Array("0"))
     }
 
+    // To avoid allocating any resources immediately after releasing the resource from the task to
+    // make sure that `availableAddrs` below won't change
+    when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]])).thenReturn(Seq.empty)
     backend.driverEndpoint.send(
       StatusUpdate("1", 1, TaskState.FINISHED, buffer, taskResources))
 

From 4dbcda4672682333e1041a24b9b1ab3d1e10e99d Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Wed, 8 Apr 2020 10:51:04 -0500
Subject: [PATCH 0461/1280] [SPARK-31362][SQL][DOCS] Document Set Operators in
 SQL Reference

### What changes were proposed in this pull request?
Document Set Operators in SQL Reference

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes

<img width="1050" alt="Screen Shot 2020-04-07 at 9 20 05 AM" src="https://user-images.githubusercontent.com/13592258/78694605-c6ea2680-78b1-11ea-8590-afb43dbe5933.png">

<img width="1050" alt="Screen Shot 2020-04-07 at 9 20 41 AM" src="https://user-images.githubusercontent.com/13592258/78694613-c8b3ea00-78b1-11ea-89b9-d6cd71ee86a0.png">

<img width="1050" alt="Screen Shot 2020-04-07 at 9 21 29 AM" src="https://user-images.githubusercontent.com/13592258/78694622-ca7dad80-78b1-11ea-9acf-7611ee57d4f2.png">

<img width="1050" alt="Screen Shot 2020-04-07 at 9 21 54 AM" src="https://user-images.githubusercontent.com/13592258/78694626-cc477100-78b1-11ea-82f8-4deaf0048de7.png">

### How was this patch tested?
Manually build and check

Closes #28139 from huaxingao/set-operators.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 5dc9b9c7c18b853cb3bdcb50041ef46888922308)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                 |   2 +
 docs/sql-ref-syntax-qry-select-setops.md | 163 ++++++++++++++++++++++-
 docs/sql-ref-syntax-qry-select.md        |   1 +
 3 files changed, 163 insertions(+), 3 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index ffc3b3c7b70f9..6f300e2a607bc 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -156,6 +156,8 @@
                   url: sql-ref-syntax-qry-select-limit.html
                 - text: Join Hints
                   url: sql-ref-syntax-qry-select-hints.html
+                - text: Set Operators
+                  url: sql-ref-syntax-qry-select-setops.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-qry-select-setops.md b/docs/sql-ref-syntax-qry-select-setops.md
index 526f857520aa2..8ed6e4880f7db 100644
--- a/docs/sql-ref-syntax-qry-select-setops.md
+++ b/docs/sql-ref-syntax-qry-select-setops.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Set Operations
-displayTitle: Set Operations
+title: Set Operators
+displayTitle: Set Operators
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -19,4 +19,161 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+Set operators are used to combine two input relations into a single one. Spark SQL supports three types of set operators:
+- `EXCEPT` or `MINUS`
+- `INTERSECT`
+- `UNION`
+
+Note that input relations must have the same number of columns and compatible data types for the respective columns.
+
+### EXCEPT
+`EXCEPT` and `EXCEPT ALL` return the rows that are found in one relation but not the other. `EXCEPT` (alternatively, `EXCEPT DISTINCT`) takes only distinct rows while `EXCEPT ALL` does not remove duplicates from the result rows. Note that `MINUS` is an alias for `EXCEPT`.
+
+#### Syntax
+{% highlight sql %}
+[ ( ] relation [ ) ] EXCEPT | MINUS [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+{% endhighlight %}
+
+### INTERSECT
+`INTERSECT` and `INTERSECT ALL` return the rows that are found in both relations. `INTERSECT` (alternatively, `INTERSECT DISTINCT`) takes only distinct rows while `INTERSECT ALL` does not remove duplicates from the result rows.
+
+#### Syntax
+{% highlight sql %}
+[ ( ] relation [ ) ] INTERSECT [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+{% endhighlight %}
+
+### UNION
+`UNION` and `UNION ALL` return the rows that are found in either relation. `UNION` (alternatively, `UNION DISTINCT`) takes only distinct rows while `UNION ALL` does not remove duplicates from the result rows.
+
+#### Syntax
+{% highlight sql %}
+[ ( ] relation [ ) ] UNION [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+{% endhighlight %}
+
+### Examples
+{% highlight sql %}
+-- Use number1 and number2 tables to demonstrate set operators.
+SELECT * FROM number1;
++---+
+|  c|
++---+
+|  3|
+|  1|
+|  2|
+|  2|
+|  3|
+|  4|
++---+
+
+SELECT * FROM number2;
++---+
+|  c|
++---+
+|  5|
+|  1|
+|  2|
+|  2|
++---+
+
+SELECT c FROM number1 EXCEPT SELECT c FROM number2;
++---+
+|  c|
++---+
+|  3|
+|  4|
++---+
+
+SELECT c FROM number1 MINUS SELECT c FROM number2;
++---+
+|  c|
++---+
+|  3|
+|  4|
++---+
+
+SELECT c FROM number1 EXCEPT ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  3|
+|  3|
+|  4|
++---+
+
+SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  3|
+|  3|
+|  4|
++---+
+
+(SELECT c FROM number1) INTERSECT (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  2|
++---+
+
+(SELECT c FROM number1) INTERSECT DISTINCT (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  2|
++---+
+
+(SELECT c FROM number1) INTERSECT ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  2|
+|  2|
++---+
+
+(SELECT c FROM number1) UNION (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  3|
+|  5|
+|  4|
+|  2|
++---+
+
+(SELECT c FROM number1) UNION DISTINCT (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  1|
+|  3|
+|  5|
+|  4|
+|  2|
++---+
+
+SELECT c FROM number1 UNION ALL (SELECT c FROM number2);
++---+
+|  c|
++---+
+|  3|
+|  1|
+|  2|
+|  2|
+|  3|
+|  4|
+|  5|
+|  1|
+|  2|
+|  2|
++---+
+
+{% endhighlight %}
+
+### Related Statement
+- [SELECT Statement](sql-ref-syntax-qry-select.html)
+
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 7ad1dd15c4812..420cf1f4a13ee 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -150,3 +150,4 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 - [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 - [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 - [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+- [SET Operators](sql-ref-syntax-qry-select-setops.html)

From 2221d3e0183140a0a98f6de92f84d2d924aab703 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Wed, 8 Apr 2020 16:59:39 -0700
Subject: [PATCH 0462/1280] [SPARK-29314][SS] Don't overwrite the metric
 "updated" of state operator to 0 if empty batch is run

### What changes were proposed in this pull request?

This patch fixes the behavior of ProgressReporter which always overwrite the value of "updated" of state operator to 0 if there's no new data. The behavior is correct only when we copy the state progress from "previous" executed plan, meaning no batch has been run. (Nonzero value of "updated" would be odd if batch didn't run, so it was correct.)

It was safe to assume no data is no batch, but SPARK-24156 enables empty data can run the batch if Spark needs to deal with watermark. After the patch, it only overwrites the value if both two conditions are met: 1) no data 2) no batch.

### Why are the changes needed?

Currently Spark doesn't reflect correct metrics when empty batch is run and this patch fixes it.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Modified UT. Note that FlatMapGroupsWithState increases the value of "updated" when state rows are removed.
Also manually tested via below query (not a simple query to test with spark-shell, as you'll meet closure issue in spark-shell while playing with state func):

> query

```
case class RunningCount(count: Long)

object TestFlatMapGroupsWithState {
  def main(args: Array[String]): Unit = {
    import org.apache.spark.sql.SparkSession

    val ss = SparkSession
      .builder()
      .appName("TestFlatMapGroupsWithState")
      .getOrCreate()

    ss.conf.set("spark.sql.shuffle.partitions", "5")

    import ss.implicits._

    val stateFunc = (key: String, values: Iterator[String], state: GroupState[RunningCount]) => {
      if (state.hasTimedOut) {
        // End users are not restricted to remove the state here - they can update the
        // state as well. For example, event time session window would have list of
        // sessions here and it cannot remove entire state.
        state.update(RunningCount(-1))
        Iterator((key, "-1"))
      } else {
        val count = state.getOption.map(_.count).getOrElse(0L) + values.size
        state.update(RunningCount(count))
        state.setTimeoutDuration("1 seconds")
        Iterator((key, count.toString))
      }
    }

    implicit val sqlContext = ss.sqlContext
    val inputData = MemoryStream[String]

    val result = inputData
      .toDF()
      .as[String]
      .groupByKey { v => v }
      .flatMapGroupsWithState(OutputMode.Append(), GroupStateTimeout.ProcessingTimeTimeout())(stateFunc)

    val query = result
      .writeStream
      .format("memory")
      .option("queryName", "test")
      .outputMode("append")
      .trigger(Trigger.ProcessingTime("5 second"))
      .start()

    Thread.sleep(1000)

    var chIdx: Long = 0

    while (true) {
      (chIdx to chIdx + 4).map { idx => inputData.addData(idx.toString) }
      chIdx += 5
      // intentionally sleep much more than trigger to enable "empty" batch
      Thread.sleep(10 * 1000)
    }
  }
}
```

> before the patch (batch 3 which was an "empty" batch)

```
{
   "id":"de945a5c-882b-4dae-aa58-cb8261cbaf9e",
   "runId":"f1eb6d0d-3cd5-48b2-a03b-5e989b6c151b",
   "name":"test",
   "timestamp":"2019-11-18T07:00:25.005Z",
   "batchId":3,
   "numInputRows":0,
   "inputRowsPerSecond":0.0,
   "processedRowsPerSecond":0.0,
   "durationMs":{
      "addBatch":1664,
      "getBatch":0,
      "latestOffset":0,
      "queryPlanning":29,
      "triggerExecution":1789,
      "walCommit":51
   },
   "stateOperators":[
      {
         "numRowsTotal":10,
         "numRowsUpdated":0,
         "memoryUsedBytes":5130,
         "customMetrics":{
            "loadedMapCacheHitCount":15,
            "loadedMapCacheMissCount":0,
            "stateOnCurrentVersionSizeBytes":2722
         }
      }
   ],
   "sources":[
      {
         "description":"MemoryStream[value#1]",
         "startOffset":9,
         "endOffset":9,
         "numInputRows":0,
         "inputRowsPerSecond":0.0,
         "processedRowsPerSecond":0.0
      }
   ],
   "sink":{
      "description":"MemorySink",
      "numOutputRows":5
   }
}
```

> after the patch (batch 3 which was an "empty" batch)

```
{
   "id":"7cb41623-6b9a-408e-ae02-6796ec636fa0",
   "runId":"17847710-ddfe-45f5-a7ab-b160e149382f",
   "name":"test",
   "timestamp":"2019-11-18T07:02:25.005Z",
   "batchId":3,
   "numInputRows":0,
   "inputRowsPerSecond":0.0,
   "processedRowsPerSecond":0.0,
   "durationMs":{
      "addBatch":1196,
      "getBatch":0,
      "latestOffset":0,
      "queryPlanning":30,
      "triggerExecution":1333,
      "walCommit":46
   },
   "stateOperators":[
      {
         "numRowsTotal":10,
         "numRowsUpdated":5,
         "memoryUsedBytes":5130,
         "customMetrics":{
            "loadedMapCacheHitCount":15,
            "loadedMapCacheMissCount":0,
            "stateOnCurrentVersionSizeBytes":2722
         }
      }
   ],
   "sources":[
      {
         "description":"MemoryStream[value#1]",
         "startOffset":9,
         "endOffset":9,
         "numInputRows":0,
         "inputRowsPerSecond":0.0,
         "processedRowsPerSecond":0.0
      }
   ],
   "sink":{
      "description":"MemorySink",
      "numOutputRows":5
   }
}
```

"numRowsUpdated" is `0` in "stateOperators" before applying the patch which is "wrong", as we "update" the state when timeout occurs. After applying the patch, it correctly represents the "numRowsUpdated" as `5` in "stateOperators".

Closes #25987 from HeartSaVioR/SPARK-29314.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Burak Yavuz <brkyvz@gmail.com>
(cherry picked from commit ca2ba4fe647cd60668410b68014a3991ad7fd5c9)
Signed-off-by: Burak Yavuz <brkyvz@gmail.com>
---
 .../sql/execution/streaming/ProgressReporter.scala   | 12 ++++++------
 .../sql/streaming/FlatMapGroupsWithStateSuite.scala  |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index d1086cd42d133..0dff1c2fe5768 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -154,7 +154,7 @@ trait ProgressReporter extends Logging {
     assert(currentTriggerStartOffsets != null && currentTriggerEndOffsets != null)
     currentTriggerEndTimestamp = triggerClock.getTimeMillis()
 
-    val executionStats = extractExecutionStats(hasNewData)
+    val executionStats = extractExecutionStats(hasNewData, hasExecuted)
     val processingTimeMills = currentTriggerEndTimestamp - currentTriggerStartTimestamp
     val processingTimeSec = Math.max(1L, processingTimeMills).toDouble / MILLIS_PER_SECOND
 
@@ -215,26 +215,26 @@ trait ProgressReporter extends Logging {
   }
 
   /** Extract statistics about stateful operators from the executed query plan. */
-  private def extractStateOperatorMetrics(hasNewData: Boolean): Seq[StateOperatorProgress] = {
+  private def extractStateOperatorMetrics(hasExecuted: Boolean): Seq[StateOperatorProgress] = {
     if (lastExecution == null) return Nil
-    // lastExecution could belong to one of the previous triggers if `!hasNewData`.
+    // lastExecution could belong to one of the previous triggers if `!hasExecuted`.
     // Walking the plan again should be inexpensive.
     lastExecution.executedPlan.collect {
       case p if p.isInstanceOf[StateStoreWriter] =>
         val progress = p.asInstanceOf[StateStoreWriter].getProgress()
-        if (hasNewData) progress else progress.copy(newNumRowsUpdated = 0)
+        if (hasExecuted) progress else progress.copy(newNumRowsUpdated = 0)
     }
   }
 
   /** Extracts statistics from the most recent query execution. */
-  private def extractExecutionStats(hasNewData: Boolean): ExecutionStats = {
+  private def extractExecutionStats(hasNewData: Boolean, hasExecuted: Boolean): ExecutionStats = {
     val hasEventTime = logicalPlan.collect { case e: EventTimeWatermark => e }.nonEmpty
     val watermarkTimestamp =
       if (hasEventTime) Map("watermark" -> formatTimestamp(offsetSeqMetadata.batchWatermarkMs))
       else Map.empty[String, String]
 
     // SPARK-19378: Still report metrics even though no data was processed while reporting progress.
-    val stateOperators = extractStateOperatorMetrics(hasNewData)
+    val stateOperators = extractStateOperatorMetrics(hasExecuted)
 
     if (!hasNewData) {
       return ExecutionStats(Map.empty, stateOperators, watermarkTimestamp)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
index d36c64f61a726..b04f8b0d4d174 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateSuite.scala
@@ -798,7 +798,7 @@ class FlatMapGroupsWithStateSuite extends StateStoreMetricsTest {
         }
       },
       CheckNewAnswer(("c", "-1")),
-      assertNumStateRows(total = 0, updated = 0)
+      assertNumStateRows(total = 0, updated = 1)
     )
   }
 

From 86e8dbfd8229f4d5f4f701f0d6c24c6b3c1c813a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 9 Apr 2020 11:04:35 +0900
Subject: [PATCH 0463/1280] [SPARK-31382][BUILD] Show a better error message
 for different python and pip installation mistake

### What changes were proposed in this pull request?

This PR proposes to show a better error message when a user mistakenly installs `pyspark` from PIP but the default `python` does not point out the corresponding `pip`. See https://stackoverflow.com/questions/46286436/running-pyspark-after-pip-install-pyspark/49587560 as an example.

It can be reproduced as below:

I have two Python executables. `python` is Python 3.7, `pip` binds with Python 3.7 and `python2.7` is Python 2.7.

```bash
pip install pyspark
```

```bash
pyspark
```

```
...
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 2.4.5
      /_/

Using Python version 3.7.3 (default, Mar 27 2019 09:23:15)
SparkSession available as 'spark'.
...
```

```bash
PYSPARK_PYTHON=python2.7 pyspark
```

```
Could not find valid SPARK_HOME while searching ['/Users', '/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/bin']
/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/bin/pyspark: line 24: /bin/load-spark-env.sh: No such file or directory
/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/bin/pyspark: line 77: /bin/spark-submit: No such file or directory
/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/bin/pyspark: line 77: exec: /bin/spark-submit: cannot execute: No such file or directory
```

### Why are the changes needed?

There are multiple questions outside about this error and they have no idea what's going on. See:

- https://stackoverflow.com/questions/46286436/running-pyspark-after-pip-install-pyspark/49587560
- https://stackoverflow.com/questions/45991888/path-issue-could-not-find-valid-spark-home-while-searching
- https://stackoverflow.com/questions/49707239/pyspark-could-not-find-valid-spark-home
- https://stackoverflow.com/questions/55569985/pyspark-could-not-find-valid-spark-home
- https://stackoverflow.com/questions/48296474/error-could-not-find-valid-spark-home-while-searching-pycharm-in-windows
- https://github.com/ContinuumIO/anaconda-issues/issues/8076

The answer is usually setting `SPARK_HOME`; however this isn't completely correct.

It works if you set `SPARK_HOME` because `pyspark` executable script directly imports the library by using `SPARK_HOME` (see https://github.com/apache/spark/blob/master/bin/pyspark#L52-L53) instead of the default package location specified via `python` executable. So, this way you use a package installed in a different Python, which isn't ideal.

### Does this PR introduce any user-facing change?

Yes, it fixes the error message better.

**Before:**

```
Could not find valid SPARK_HOME while searching ['/Users', '/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/bin']
...
```

**After:**

```
Could not find valid SPARK_HOME while searching ['/Users', '/usr/local/Cellar/python/3.7.5/Frameworks/Python.framework/Versions/3.7/bin']

Did you install PySpark via a package manager such as pip or Conda? If so,
PySpark was not found in your Python environment. It is possible your
Python environment does not properly bind with your package manager.

Please check your default 'python' and if you set PYSPARK_PYTHON and/or
PYSPARK_DRIVER_PYTHON environment variables, and see if you can import
PySpark, for example, 'python -c 'import pyspark'.

If you cannot import, you can install by using the Python executable directly,
for example, 'python -m pip install pyspark [--user]'. Otherwise, you can also
explicitly set the Python executable, that has PySpark installed, to
PYSPARK_PYTHON or PYSPARK_DRIVER_PYTHON environment variables, for example,
'PYSPARK_PYTHON=python3 pyspark'.
...
```

### How was this patch tested?

Manually tested as described above.

Closes #28152 from HyukjinKwon/SPARK-31382.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0248b329726527a07d688122f56dd2ada0e51337)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/find_spark_home.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/find_spark_home.py b/python/pyspark/find_spark_home.py
index 9c4ed46598632..52f6ea9a37100 100755
--- a/python/pyspark/find_spark_home.py
+++ b/python/pyspark/find_spark_home.py
@@ -40,6 +40,7 @@ def is_spark_home(path):
     paths = ["../", os.path.dirname(os.path.realpath(__file__))]
 
     # Add the path of the PySpark module if it exists
+    import_error_raised = False
     if sys.version < "3":
         import imp
         try:
@@ -49,7 +50,7 @@ def is_spark_home(path):
             paths.append(os.path.join(module_home, "../../"))
         except ImportError:
             # Not pip installed no worries
-            pass
+            import_error_raised = True
     else:
         from importlib.util import find_spec
         try:
@@ -59,7 +60,7 @@ def is_spark_home(path):
             paths.append(os.path.join(module_home, "../../"))
         except ImportError:
             # Not pip installed no worries
-            pass
+            import_error_raised = True
 
     # Normalize the paths
     paths = [os.path.abspath(p) for p in paths]
@@ -68,6 +69,19 @@ def is_spark_home(path):
         return next(path for path in paths if is_spark_home(path))
     except StopIteration:
         print("Could not find valid SPARK_HOME while searching {0}".format(paths), file=sys.stderr)
+        if import_error_raised:
+            print(
+                "\nDid you install PySpark via a package manager such as pip or Conda? If so,\n"
+                "PySpark was not found in your Python environment. It is possible your\n"
+                "Python environment does not properly bind with your package manager.\n"
+                "\nPlease check your default 'python' and if you set PYSPARK_PYTHON and/or\n"
+                "PYSPARK_DRIVER_PYTHON environment variables, and see if you can import\n"
+                "PySpark, for example, 'python -c 'import pyspark'.\n"
+                "\nIf you cannot import, you can install by using the Python executable directly,\n"
+                "for example, 'python -m pip install pyspark [--user]'. Otherwise, you can also\n"
+                "explicitly set the Python executable, that has PySpark installed, to\n"
+                "PYSPARK_PYTHON or PYSPARK_DRIVER_PYTHON environment variables, for example,\n"
+                "'PYSPARK_PYTHON=python3 pyspark'.\n", file=sys.stderr)
         sys.exit(-1)
 
 if __name__ == "__main__":

From a9ed01f05772c31362e56df59f396b824acbe27b Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 9 Apr 2020 13:14:41 +0900
Subject: [PATCH 0464/1280] [SPARK-26412][PYTHON][FOLLOW-UP] Improve error
 messages in Scala iterator pandas UDF

### What changes were proposed in this pull request?

This PR proposes to improve the error message from Scalar iterator pandas UDF.

### Why are the changes needed?

To show the correct error messages.

### Does this PR introduce any user-facing change?

Yes, but only in unreleased branches.

```python
import pandas as pd
from pyspark.sql.functions import pandas_udf, PandasUDFType

pandas_udf('long', PandasUDFType.SCALAR_ITER)
def pandas_plus_one(iterator):
      for _ in iterator:
            yield pd.Series(1)

spark.range(10).repartition(1).select(pandas_plus_one("id")).show()
```
```python
import pandas as pd
from pyspark.sql.functions import pandas_udf, PandasUDFType

pandas_udf('long', PandasUDFType.SCALAR_ITER)
def pandas_plus_one(iterator):
      for _ in iterator:
            yield pd.Series(list(range(20)))

spark.range(10).repartition(1).select(pandas_plus_one("id")).show()
```

**Before:**

```
RuntimeError: The number of output rows of pandas iterator UDF should
be the same with input rows. The input rows number is 10 but the output
rows number is 1.
```
```
AssertionError: Pandas MAP_ITER UDF outputted more rows than input rows.
```

**After:**

```
RuntimeError: The length of output in Scalar iterator pandas UDF should be
the same with the input's; however, the length of output was 1 and the length
of input was 10.
```
```
AssertionError: Pandas SCALAR_ITER UDF outputted more rows than input rows.
```

### How was this patch tested?

Unittests were fixed accordingly.

Closes #28135 from HyukjinKwon/SPARK-26412-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 4fafdcd63b8c4961d7788c13047636ac3d26bfb9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../pyspark/sql/tests/test_pandas_udf_scalar.py  |  6 +++---
 python/pyspark/worker.py                         | 16 ++++++++--------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index b07de3c0b3cc9..7260e80e2cfca 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -445,8 +445,8 @@ def iter_udf_wong_output_size(it):
         with QuietTest(self.sc):
             with self.assertRaisesRegexp(
                     Exception,
-                    "The number of output rows of pandas iterator UDF should be "
-                    "the same with input rows"):
+                    "The length of output in Scalar iterator.*"
+                    "the length of output was 1"):
                 df.select(iter_udf_wong_output_size(col('id'))).collect()
 
         @pandas_udf(LongType(), PandasUDFType.SCALAR_ITER)
@@ -461,7 +461,7 @@ def iter_udf_not_reading_all_input(it):
             with QuietTest(self.sc):
                 with self.assertRaisesRegexp(
                         Exception,
-                        "SQL_SCALAR_PANDAS_ITER_UDF should exhaust the input iterator"):
+                        "pandas iterator UDF should exhaust"):
                     df1.select(iter_udf_not_reading_all_input(col('id'))).collect()
 
     def test_vectorized_udf_chained(self):
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 7d62bf1783931..10764728ff042 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -341,7 +341,7 @@ def read_udfs(pickleSer, infile, eval_type):
             pickleSer, infile, eval_type, runner_conf, udf_index=0)
 
         def func(_, iterator):
-            num_input_rows = [0]
+            num_input_rows = [0]  # TODO(SPARK-29909): Use nonlocal after we drop Python 2.
 
             def map_batch(batch):
                 udf_args = [batch[offset] for offset in arg_offsets]
@@ -358,7 +358,7 @@ def map_batch(batch):
             for result_batch, result_type in result_iter:
                 num_output_rows += len(result_batch)
                 assert is_map_iter or num_output_rows <= num_input_rows[0], \
-                    "Pandas MAP_ITER UDF outputted more rows than input rows."
+                    "Pandas SCALAR_ITER UDF outputted more rows than input rows."
                 yield (result_batch, result_type)
 
             if is_scalar_iter:
@@ -367,14 +367,14 @@ def map_batch(batch):
                 except StopIteration:
                     pass
                 else:
-                    raise RuntimeError("SQL_SCALAR_PANDAS_ITER_UDF should exhaust the input "
+                    raise RuntimeError("pandas iterator UDF should exhaust the input "
                                        "iterator.")
 
-            if is_scalar_iter and num_output_rows != num_input_rows[0]:
-                raise RuntimeError("The number of output rows of pandas iterator UDF should be "
-                                   "the same with input rows. The input rows number is %d but the "
-                                   "output rows number is %d." %
-                                   (num_input_rows[0], num_output_rows))
+                if num_output_rows != num_input_rows[0]:
+                    raise RuntimeError(
+                        "The length of output in Scalar iterator pandas UDF should be "
+                        "the same with the input's; however, the length of output was %d and the "
+                        "length of input was %d." % (num_output_rows, num_input_rows[0]))
 
         # profiling is not supported for UDF
         return func, None, ser, ser

From 9f0fea178ecd0a06c0a5138e9c6b083f1706cc17 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 9 Apr 2020 16:46:27 +0900
Subject: [PATCH 0465/1280] [SPARK-30722][DOCS][FOLLOW-UP] Explicitly mention
 the same entire input/output length restriction of Series Iterator UDF

### What changes were proposed in this pull request?

This PR explicitly mention that the requirement of Iterator of Series to Iterator of Series and Iterator of Multiple Series to Iterator of Series (previously Scalar Iterator pandas UDF).

The actual limitation of this UDF is the same length of the _entire input and output_, instead of each series's length. Namely you can do something as below:

```python
from typing import Iterator, Tuple
import pandas as pd
from pyspark.sql.functions import pandas_udf

pandas_udf("long")
def func(
        iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
    return iter([pd.concat(iterator)])

spark.range(100).select(func("id")).show()
```

This characteristic allows you to prefetch the data from the iterator to speed up, compared to the regular Scalar to Scalar (previously Scalar pandas UDF).

### Why are the changes needed?

To document the correct restriction and characteristics of a feature.

### Does this PR introduce any user-facing change?

Yes in the documentation but only in unreleased branches.

### How was this patch tested?

Github Actions should test the documentation build

Closes #28160 from HyukjinKwon/SPARK-30722-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c279e6b0914014a99b84076420240dceaf3b895d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-pyspark-pandas-with-arrow.md  | 35 +++++++++++++++-----------
 python/pyspark/sql/pandas/functions.py | 12 ++++-----
 python/pyspark/worker.py               |  5 ++++
 3 files changed, 31 insertions(+), 21 deletions(-)

diff --git a/docs/sql-pyspark-pandas-with-arrow.md b/docs/sql-pyspark-pandas-with-arrow.md
index e8abb9fed1b2b..a11162963b380 100644
--- a/docs/sql-pyspark-pandas-with-arrow.md
+++ b/docs/sql-pyspark-pandas-with-arrow.md
@@ -98,7 +98,7 @@ In the following sections, it describes the combinations of the supported type h
 
 The type hint can be expressed as `pandas.Series`, ... -> `pandas.Series`.
 
-By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF where the given
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
 function takes one or more `pandas.Series` and outputs one `pandas.Series`. The output of the function should
 always be of the same length as the input. Internally, PySpark will execute a Pandas UDF by splitting
 columns into batches and calling the function for each batch as a subset of the data, then concatenating
@@ -118,13 +118,15 @@ For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/p
 
 The type hint can be expressed as `Iterator[pandas.Series]` -> `Iterator[pandas.Series]`.
 
-By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF where the given
-function takes an iterator of `pandas.Series` and outputs an iterator of `pandas.Series`. The output of each
-series from the function should always be of the same length as the input. In this case, the created
-Pandas UDF requires one input column when the Pandas UDF is called. To use multiple input columns,
-a different type hint is required. See Iterator of Multiple Series to Iterator of Series.
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the given
+function takes an iterator of `pandas.Series` and outputs an iterator of `pandas.Series`. The
+length of the entire output from the function should be the same length of the entire input; therefore, it can
+prefetch the data from the input iterator as long as the lengths are the same.
+In this case, the created Pandas UDF requires one input column when the Pandas UDF is called. To use
+multiple input columns, a different type hint is required. See Iterator of Multiple Series to Iterator
+of Series.
 
-It is useful when the UDF execution requires initializing some states although internally it works
+It is also useful when the UDF execution requires initializing some states although internally it works
 identically as Series to Series case. The pseudocode below illustrates the example.
 
 {% highlight python %}
@@ -153,10 +155,11 @@ For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/p
 
 The type hint can be expressed as `Iterator[Tuple[pandas.Series, ...]]` -> `Iterator[pandas.Series]`.
 
-By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF where the
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF where the
 given function takes an iterator of a tuple of multiple `pandas.Series` and outputs an iterator of `pandas.Series`.
 In this case, the created pandas UDF requires multiple input columns as many as the series in the tuple
-when the Pandas UDF is called. It works identically as Iterator of Series to Iterator of Series case except the parameter difference.
+when the Pandas UDF is called. Otherwise, it has the same characteristics and restrictions as Iterator of Series
+to Iterator of Series case.
 
 The following example shows how to create this Pandas UDF:
 
@@ -172,7 +175,7 @@ For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/p
 
 The type hint can be expressed as `pandas.Series`, ... -> `Any`.
 
-By using `pandas_udf` with the function having such type hints, it creates a Pandas UDF similar
+By using `pandas_udf` with the function having such type hints above, it creates a Pandas UDF similar
 to PySpark's aggregate functions. The given function takes `pandas.Series` and returns a scalar value.
 The return type should be a primitive data type, and the returned scalar can be either a python
 primitive type, e.g., `int` or `float` or a numpy data type, e.g., `numpy.int64` or `numpy.float64`.
@@ -198,12 +201,14 @@ For detailed usage, please see [`pyspark.sql.functions.pandas_udf`](api/python/p
 
 ## Pandas Function APIs
 
-Pandas function APIs can directly apply a Python native function against the whole DataFrame by
-using Pandas instances. Internally it works similarly with Pandas UDFs by Spark using Arrow to transfer
-data and Pandas to work with the data, which allows vectorized operations. A Pandas function API behaves
-as a regular API under PySpark `DataFrame` in general.
+Pandas Function APIs can directly apply a Python native function against the whole `DataFrame` by
+using Pandas instances. Internally it works similarly with Pandas UDFs by using Arrow to transfer
+data and Pandas to work with the data, which allows vectorized operations. However, A Pandas Function
+API behaves as a regular API under PySpark `DataFrame` instead of `Column`, and Python type hints in Pandas
+Functions APIs are optional and do not affect how it works internally at this moment although they
+might be required in the future.
 
-From Spark 3.0, Grouped map pandas UDF is now categorized as a separate Pandas Function API,
+From Spark 3.0, grouped map pandas UDF is now categorized as a separate Pandas Function API,
 `DataFrame.groupby().applyInPandas()`. It is still possible to use it with `PandasUDFType`
 and `DataFrame.groupby().apply()` as it was; however, it is preferred to use
 `DataFrame.groupby().applyInPandas()` directly. Using `PandasUDFType` will be deprecated
diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py
index f43ebf8c5750a..094dc357b6822 100644
--- a/python/pyspark/sql/pandas/functions.py
+++ b/python/pyspark/sql/pandas/functions.py
@@ -150,10 +150,11 @@ def pandas_udf(f=None, returnType=None, functionType=None):
 
         The function takes an iterator of `pandas.Series` and outputs an iterator of
         `pandas.Series`. In this case, the created pandas UDF instance requires one input
-        column when this is called as a PySpark column. The output of each series from
-        the function should always be of the same length as the input.
+        column when this is called as a PySpark column. The length of the entire output from
+        the function should be the same length of the entire input; therefore, it can
+        prefetch the data from the input iterator as long as the lengths are the same.
 
-        It is useful when the UDF execution
+        It is also useful when the UDF execution
         requires initializing some states although internally it works identically as
         Series to Series case. The pseudocode below illustrates the example.
 
@@ -194,9 +195,8 @@ def calculate(iterator: Iterator[pd.Series]) -> Iterator[pd.Series]:
         The function takes an iterator of a tuple of multiple `pandas.Series` and outputs an
         iterator of `pandas.Series`. In this case, the created pandas UDF instance requires
         input columns as many as the series when this is called as a PySpark column.
-        It works identically as Iterator of Series to Iterator of Series case except
-        the parameter difference. The output of each series from the function should always
-        be of the same length as the input.
+        Otherwise, it has the same characteristics and restrictions as Iterator of Series
+        to Iterator of Series case.
 
         >>> from typing import Iterator, Tuple
         >>> from pyspark.sql.functions import struct, col
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 10764728ff042..a9a32ed150c0b 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -357,6 +357,11 @@ def map_batch(batch):
             num_output_rows = 0
             for result_batch, result_type in result_iter:
                 num_output_rows += len(result_batch)
+                # This assert is for Scalar Iterator UDF to fail fast.
+                # The length of the entire input can only be explicitly known
+                # by consuming the input iterator in user side. Therefore,
+                # it's very unlikely the output length is higher than
+                # input length.
                 assert is_map_iter or num_output_rows <= num_input_rows[0], \
                     "Pandas SCALAR_ITER UDF outputted more rows than input rows."
                 yield (result_batch, result_type)

From 886d71290b77c2e2ad5ac0076339f91aad162193 Mon Sep 17 00:00:00 2001
From: gengjiaan <gengjiaan@360.cn>
Date: Thu, 9 Apr 2020 12:16:43 +0000
Subject: [PATCH 0466/1280] [SPARK-31291][SQL][TEST] SQLQueryTestSuite: Sharing
 test data and test tables among multiple test cases
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
`SQLQueryTestSuite` spend 35 minutes time to test.
I've listed the 10 test cases that took the longest time in the `SQL` module below.

Class | Spend time  ↑ | Failure | Skip | Pass | Total test case
-- | -- | -- | -- | -- | --
SQLQueryTestSuite | 35 minutes | 0 | 1 | 230 | 231
TPCDSQuerySuite | 3 minutes 8 seconds | 0 | 0 | 156 | 156
SQLQuerySuite | 2 minutes 52 seconds | 0 | 0 | 185 | 185
DynamicPartitionPruningSuiteAEOff | 1 minutes 52 seconds | 0 | 0 | 22 | 22
DataFrameFunctionsSuite | 1 minutes 37 seconds | 0 | 0 | 102 | 102
DynamicPartitionPruningSuiteAEOn | 1 minutes 24 seconds | 0 | 0 | 22 | 22
DataFrameSuite | 1 minutes 14 seconds | 0 | 2 | 157 | 159
SubquerySuite | 1 minutes 12 seconds | 0 | 1 | 70 | 71
SingleLevelAggregateHashMapSuite | 1 minutes 1 seconds | 0 | 0 | 50 | 50
DataFrameAggregateSuite | 59 seconds | 0 | 0 | 50 | 50

I checked the code of `SQLQueryTestSuite` and found `SQLQueryTestSuite` load test data repeatedly.
This PR will improve the performance of `SQLQueryTestSuite`.

The total time run `SQLQueryTestSuite` before and after this PR show below.
Before
No | Time
-- | --
1 | 20 minutes, 22 seconds
2 | 23 minutes, 21 seconds
3 | 21 minutes, 19 seconds
4 | 22 minutes, 26 seconds
5 | 20 minutes, 8 seconds

After
No | Time
-- | --
1 | 20 minutes, 52 seconds
2 | 20 minutes, 47 seconds
3 | 20 minutes, 7 seconds
4 | 21 minutes, 10 seconds
5 | 20 minutes, 4 seconds

### Why are the changes needed?
Improve the performance of `SQLQueryTestSuite`.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test

Closes #28060 from beliefer/avoid-load-test-data-repeatedly.

Lead-authored-by: gengjiaan <gengjiaan@360.cn>
Co-authored-by: beliefer <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 014d33570ba1518529a080a1ab79d33c536a0ee3)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../resources/sql-tests/results/limit.sql.out |  2 +-
 .../sql-tests/results/show-tables.sql.out     | 12 ------
 .../sql-tests/results/show-views.sql.out      | 24 -----------
 .../apache/spark/sql/SQLQueryTestSuite.scala  | 40 +++++++++++++++----
 4 files changed, 33 insertions(+), 45 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/results/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
index 281326e22a97a..074e7a6d28c47 100644
--- a/sql/core/src/test/resources/sql-tests/results/limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
@@ -88,7 +88,7 @@ SELECT * FROM testdata LIMIT key > 3
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-The limit expression must evaluate to a constant value, but got (testdata.`key` > 3);
+The limit expression must evaluate to a constant value, but got (spark_catalog.default.testdata.`key` > 3);
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
index dd2e7b168698e..a95b02c7f7743 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
@@ -63,15 +63,9 @@ SHOW TABLES
 -- !query schema
 struct<database:string,tableName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
 show_t1
 show_t2
 show_t3
-tenk1
-testdata
 
 
 -- !query
@@ -79,15 +73,9 @@ SHOW TABLES IN showdb
 -- !query schema
 struct<database:string,tableName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
 show_t1
 show_t2
 show_t3
-tenk1
-testdata
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
index ee8a7474e6d5d..d88790d8b5ec8 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
@@ -63,12 +63,6 @@ SHOW VIEWS
 -- !query schema
 struct<namespace:string,viewName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
 view_1
 view_2
 view_4
@@ -79,12 +73,6 @@ SHOW VIEWS FROM showdb
 -- !query schema
 struct<namespace:string,viewName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
 view_1
 view_2
 view_4
@@ -95,12 +83,6 @@ SHOW VIEWS IN showdb
 -- !query schema
 struct<namespace:string,viewName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
 view_1
 view_2
 view_4
@@ -111,12 +93,6 @@ SHOW VIEWS IN global_temp
 -- !query schema
 struct<namespace:string,viewName:string,isTemporary:boolean>
 -- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
 view_3
 view_4
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 848966a97a70e..4a39d95773e6b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -361,7 +361,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     // Create a local SparkSession to have stronger isolation between different test cases.
     // This does not isolate catalog changes.
     val localSparkSession = spark.newSession()
-    loadTestData(localSparkSession)
 
     testCase match {
       case udfTestCase: UDFTest =>
@@ -570,14 +569,20 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
   }
 
   /** Load built-in test tables into the SparkSession. */
-  private def loadTestData(session: SparkSession): Unit = {
+  private def createTestTables(session: SparkSession): Unit = {
     import session.implicits._
 
-    (1 to 100).map(i => (i, i.toString)).toDF("key", "value").createOrReplaceTempView("testdata")
+    (1 to 100).map(i => (i, i.toString)).toDF("key", "value")
+      .repartition(1)
+      .write
+      .format("parquet")
+      .saveAsTable("testdata")
 
     ((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4))) :: Nil)
       .toDF("arraycol", "nestedarraycol")
-      .createOrReplaceTempView("arraydata")
+      .write
+      .format("parquet")
+      .saveAsTable("arraydata")
 
     (Tuple1(Map(1 -> "a1", 2 -> "b1", 3 -> "c1", 4 -> "d1", 5 -> "e1")) ::
       Tuple1(Map(1 -> "a2", 2 -> "b2", 3 -> "c2", 4 -> "d2")) ::
@@ -585,7 +590,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       Tuple1(Map(1 -> "a4", 2 -> "b4")) ::
       Tuple1(Map(1 -> "a5")) :: Nil)
       .toDF("mapcol")
-      .createOrReplaceTempView("mapdata")
+      .write
+      .format("parquet")
+      .saveAsTable("mapdata")
 
     session
       .read
@@ -593,7 +600,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
       .options(Map("delimiter" -> "\t", "header" -> "false"))
       .schema("a int, b float")
       .load(testFile("test-data/postgresql/agg.data"))
-      .createOrReplaceTempView("aggtest")
+      .write
+      .format("parquet")
+      .saveAsTable("aggtest")
 
     session
       .read
@@ -619,7 +628,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
           |string4 string
         """.stripMargin)
       .load(testFile("test-data/postgresql/onek.data"))
-      .createOrReplaceTempView("onek")
+      .write
+      .format("parquet")
+      .saveAsTable("onek")
 
     session
       .read
@@ -645,7 +656,18 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
           |string4 string
         """.stripMargin)
       .load(testFile("test-data/postgresql/tenk.data"))
-      .createOrReplaceTempView("tenk1")
+      .write
+      .format("parquet")
+      .saveAsTable("tenk1")
+  }
+
+  private def removeTestTables(session: SparkSession): Unit = {
+    session.sql("DROP TABLE IF EXISTS testdata")
+    session.sql("DROP TABLE IF EXISTS arraydata")
+    session.sql("DROP TABLE IF EXISTS mapdata")
+    session.sql("DROP TABLE IF EXISTS aggtest")
+    session.sql("DROP TABLE IF EXISTS onek")
+    session.sql("DROP TABLE IF EXISTS tenk1")
   }
 
   private val originalTimeZone = TimeZone.getDefault
@@ -653,6 +675,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
 
   override def beforeAll(): Unit = {
     super.beforeAll()
+    createTestTables(spark)
     // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
     TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
     // Add Locale setting
@@ -664,6 +687,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     try {
       TimeZone.setDefault(originalTimeZone)
       Locale.setDefault(originalLocale)
+      removeTestTables(spark)
 
       // For debugging dump some statistics about how much time was spent in various optimizer rules
       logWarning(RuleExecutor.dumpTimeSpent())

From 1b69e51e8fa7f5b76550742b9c3af9fcaf5356a0 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 9 Apr 2020 13:28:01 -0500
Subject: [PATCH 0467/1280] [SPARK-31331][SQL][DOCS] Document Spark integration
 with Hive UDFs/UDAFs/UDTFs

### What changes were proposed in this pull request?
Document Spark integration with Hive UDFs/UDAFs/UDTFs

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes
<img width="1031" alt="Screen Shot 2020-04-02 at 2 22 42 PM" src="https://user-images.githubusercontent.com/13592258/78301971-cc7cf080-74ee-11ea-93c8-7d4c75213b47.png">

### How was this patch tested?
Manually build and check

Closes #28104 from huaxingao/hive-udfs.

Lead-authored-by: Huaxin Gao <huaxing@us.ibm.com>
Co-authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 61f903fa7aa853e6462362aa8d335e3fc192d831)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/sql-ref-functions-udf-hive.md | 88 +++++++++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 1 deletion(-)

diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md
index 8698be707ea43..a87266dec2ea1 100644
--- a/docs/sql-ref-functions-udf-hive.md
+++ b/docs/sql-ref-functions-udf-hive.md
@@ -19,4 +19,90 @@ license: |
   limitations under the License.
 ---
 
-Integration with Hive UDFs/UDAFs/UDTFs
\ No newline at end of file
+### Description
+
+Spark SQL supports integration of Hive UDFs, UDAFs and UDTFs. Similar to Spark UDFs and UDAFs, Hive UDFs work on a single row as input and generate a single row as output, while Hive UDAFs operate on multiple rows and return a single aggregated row as a result. In addition, Hive also supports UDTFs (User Defined Tabular Functions) that act on one row as input and return multiple rows as output. To use Hive UDFs/UDAFs/UTFs, the user should register them in Spark, and then use them in Spark SQL queries.
+
+### Examples
+
+Hive has two UDF interfaces: [UDF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDF.java) and [GenericUDF](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java).
+An example below uses [GenericUDFAbs](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java) derived from `GenericUDF`.
+
+{% highlight sql %}
+-- Register `GenericUDFAbs` and use it in Spark SQL.
+-- Note that, if you use your own programmed one, you need to add a JAR containig it
+-- into a classpath,
+-- e.g., ADD JAR yourHiveUDF.jar;
+CREATE TEMPORARY FUNCTION testUDF AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs';
+
+SELECT * FROM t;
+  +-----+
+  |value|
+  +-----+
+  | -1.0|
+  |  2.0|
+  | -3.0|
+  +-----+
+
+SELECT testUDF(value) FROM t;
+  +--------------+
+  |testUDF(value)|
+  +--------------+
+  |           1.0|
+  |           2.0|
+  |           3.0|
+  +--------------+
+{% endhighlight %}
+
+
+An example below uses [GenericUDTFExplode](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java) derived from [GenericUDTF](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java).
+
+{% highlight sql %}
+-- Register `GenericUDTFExplode` and use it in Spark SQL
+CREATE TEMPORARY FUNCTION hiveUDTF
+    AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode';
+
+SELECT * FROM t;
+  +------+
+  | value|
+  +------+
+  |[1, 2]|
+  |[3, 4]|
+  +------+
+
+SELECT hiveUDTF(value) FROM t;
+  +---+
+  |col|
+  +---+
+  |  1|
+  |  2|
+  |  3|
+  |  4|
+  +---+
+{% endhighlight %}
+
+Hive has two UDAF interfaces: [UDAF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDAF.java) and [GenericUDAFResolver](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver.java).
+An example below uses [GenericUDAFSum](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java) derived from `GenericUDAFResolver`.
+
+{% highlight sql %}
+-- Register `GenericUDAFSum` and use it in Spark SQL
+CREATE TEMPORARY FUNCTION hiveUDAF
+    AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum';
+
+SELECT * FROM t;
+  +---+-----+
+  |key|value|
+  +---+-----+
+  |  a|    1|
+  |  a|    2|
+  |  b|    3|
+  +---+-----+
+
+SELECT key, hiveUDAF(value) FROM t GROUP BY key;
+  +---+---------------+
+  |key|hiveUDAF(value)|
+  +---+---------------+
+  |  b|              3|
+  |  a|              3|
+  +---+---------------+
+{% endhighlight %}
\ No newline at end of file

From 471f0df08b9b59fe739b3081bb8f923a3097a97d Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 9 Apr 2020 19:39:34 -0500
Subject: [PATCH 0468/1280] [SPARK-31355][SQL][DOCS] Document TABLESAMPLE in
 SQL Reference

### What changes were proposed in this pull request?
Document TABLESAMPLE in SQL Reference

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes

<img width="1049" alt="Screen Shot 2020-04-06 at 10 23 52 PM" src="https://user-images.githubusercontent.com/13592258/78633123-96749f00-7855-11ea-9509-b7ee21da7fbd.png">

<img width="1050" alt="Screen Shot 2020-04-06 at 10 24 26 PM" src="https://user-images.githubusercontent.com/13592258/78633130-98d6f900-7855-11ea-8675-fd4b6163dfb6.png">

### How was this patch tested?
Manually build and check.

Closes #28130 from huaxingao/sampling.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit f69b0ef25df744f83c57c50735e8db9f6802e98c)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml            |  2 +
 docs/sql-ref-syntax-qry-sampling.md | 74 ++++++++++++++++++++++++++++-
 docs/sql-ref-syntax-qry-select.md   |  1 +
 3 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 6f300e2a607bc..225450b01c529 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -158,6 +158,8 @@
                   url: sql-ref-syntax-qry-select-hints.html
                 - text: Set Operators
                   url: sql-ref-syntax-qry-select-setops.html
+                - text: TABLESAMPLE
+                  url: sql-ref-syntax-qry-sampling.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-sampling.md
index a5efb361ee847..061f21c3d16dd 100644
--- a/docs/sql-ref-syntax-qry-sampling.md
+++ b/docs/sql-ref-syntax-qry-sampling.md
@@ -19,4 +19,76 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+The `TABLESAMPLE` statement is used to sample the table. It supports the following sampling methods:
+  * `TABLESAMPLE`(x `ROWS`): Sample the table down to the given number of rows.
+  * `TABLESAMPLE`(x `PERCENT`): Sample the table down to the given percentage. Note that percentages are defined as a number between 0 and 100.
+  * `TABLESAMPLE`(`BUCKET` x `OUT OF` y): Sample the table down to a `x` out of `y` fraction.
+
+Note: `TABLESAMPLE` returns the approximate number of rows or fraction requested.
+
+### Syntax
+
+{% highlight sql %}
+    TABLESAMPLE ((integer_expression | decimal_expression) PERCENT)
+        | TABLESAMPLE (integer_expression ROWS)
+        | TABLESAMPLE (BUCKET integer_expression OUT OF integer_expression)
+{% endhighlight %}
+
+### Examples
+
+{% highlight sql %}
+SELECT * FROM test;
+  +--+----+
+  |id|name|
+  +--+----+
+  | 5|Alex|
+  | 8|Lucy|
+  | 2|Mary|
+  | 4|Fred|
+  | 1|Lisa|
+  | 9|Eric|
+  |10|Adam|
+  | 6|Mark|
+  | 7|Lily|
+  | 3|Evan|
+  +--+----+
+
+SELECT * FROM test TABLESAMPLE (50 PERCENT);
+  +--+----+
+  |id|name|
+  +--+----+
+  | 5|Alex|
+  | 2|Mary|
+  | 4|Fred|
+  | 9|Eric|
+  |10|Adam|
+  | 3|Evan|
+  +--+----+
+
+SELECT * FROM test TABLESAMPLE (5 ROWS);
+  +--+----+
+  |id|name|
+  +--+----+
+  | 5|Alex|
+  | 8|Lucy|
+  | 2|Mary|
+  | 4|Fred|
+  | 1|Lisa|
+  +--+----+
+
+SELECT * FROM test TABLESAMPLE (BUCKET 4 OUT OF 10);
+  +--+----+
+  |id|name|
+  +--+----+
+  | 8|Lucy|
+  | 2|Mary|
+  | 9|Eric|
+  | 6|Mark|
+  +--+----+
+{% endhighlight %}
+
+### Related Statement
+
+  * [SELECT](sql-ref-syntax-qry-select.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 420cf1f4a13ee..17c141196d830 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -150,4 +150,5 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 - [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 - [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 - [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+- [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
 - [SET Operators](sql-ref-syntax-qry-select-setops.html)

From 7758fa43f1e58b443ca8eb1b73dfbfba1385d7ba Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 10 Apr 2020 12:02:41 +0900
Subject: [PATCH 0469/1280] [SPARK-18886][CORE][TESTS][FOLLOWUP] Fix a test
 failure due to InvalidUseOfMatchersException

### What changes were proposed in this pull request?

This fixes one UT failure.
```
[info] - extra resources from executor *** FAILED *** (218 milliseconds)
[info]   org.mockito.exceptions.misusing.InvalidUseOfMatchersException: Invalid use of argument matchers!
[info] 0 matchers expected, 1 recorded:
```

### Why are the changes needed?

The original PR was merged with an outdated Jenkins result (7 days before the merging).

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins or manually do the following.
```
$ build/sbt "core/testOnly *.CoarseGrainedSchedulerBackendSuite"
```

Closes #28174 from dongjoon-hyun/SPARK-18886.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c6ea6933e2ff5bc89f609594d34390111f88af88)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
index fad971fd4ddec..0adeec15caabe 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
@@ -253,7 +253,7 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
 
     // To avoid allocating any resources immediately after releasing the resource from the task to
     // make sure that `availableAddrs` below won't change
-    when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]])).thenReturn(Seq.empty)
+    when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]], any[Boolean])).thenReturn(Seq.empty)
     backend.driverEndpoint.send(
       StatusUpdate("1", 1, TaskState.FINISHED, buffer, taskResources))
 

From 7b37ee95cfbbebf92589f633d7d48a199d09e6b7 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 10 Apr 2020 12:08:33 +0900
Subject: [PATCH 0470/1280] Revert "[SPARK-18886][CORE][TESTS][FOLLOWUP] Fix a
 test failure due to InvalidUseOfMatchersException"

This reverts commit 7758fa43f1e58b443ca8eb1b73dfbfba1385d7ba.
---
 .../spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
index 0adeec15caabe..fad971fd4ddec 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/CoarseGrainedSchedulerBackendSuite.scala
@@ -253,7 +253,7 @@ class CoarseGrainedSchedulerBackendSuite extends SparkFunSuite with LocalSparkCo
 
     // To avoid allocating any resources immediately after releasing the resource from the task to
     // make sure that `availableAddrs` below won't change
-    when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]], any[Boolean])).thenReturn(Seq.empty)
+    when(ts.resourceOffers(any[IndexedSeq[WorkerOffer]])).thenReturn(Seq.empty)
     backend.driverEndpoint.send(
       StatusUpdate("1", 1, TaskState.FINISHED, buffer, taskResources))
 

From 1da71126d9d900c0cb87ea13a64cdfc3c3f277ce Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 10 Apr 2020 03:23:32 +0000
Subject: [PATCH 0471/1280] [SPARK-31359][SQL][3.0] Speed up timestamps
 rebasing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
In the PR, I propose to optimise the `DateTimeUtils`.`rebaseJulianToGregorianMicros()` and `rebaseGregorianToJulianMicros()` functions, and make them faster by using pre-calculated rebasing tables. This approach allows to avoid expensive conversions via local timestamps. For example, the `America/Los_Angeles` time zone has just a few time points when difference between Proleptic Gregorian calendar and the hybrid calendar (Julian + Gregorian since 1582-10-15) is changed in the time interval 0001-01-01 .. 2100-01-01:

| i | local  timestamp | Proleptic Greg. seconds | Hybrid (Julian+Greg) seconds | difference in minutes|
| -- | ------- |----|----| ---- |
|0|0001-01-01 00:00|-62135568422|-62135740800|-2872|
|1|0100-03-01 00:00|-59006333222|-59006419200|-1432|
|...|...|...|...|...|
|13|1582-10-15 00:00|-12219264422|-12219264000|7|
|14|1883-11-18 12:00|-2717640000|-2717640000|0|

The difference in microseconds between Proleptic and hybrid calendars for any local timestamp in time intervals `[local timestamp(i), local timestamp(i+1))`, and for any microseconds in the time interval `[Gregorian micros(i), Gregorian micros(i+1))` is the same. In this way, we can rebase an input micros by following the steps:
1. Look at the table, and find the time interval where the micros falls to
2. Take the difference between 2 calendars for this time interval
3. Add the difference to the input micros. The result is rebased microseconds that has the same local timestamp representation.

Here are details of the implementation:
- Pre-calculated tables are stored to JSON files `gregorian-julian-rebase-micros.json` and `julian-gregorian-rebase-micros.json` in the resource folder of `sql/catalyst`. The diffs and switch time points are stored as seconds, for example:
```json
[
  {
    "tz" : "America/Los_Angeles",
    "switches" : [ -62135740800, -59006419200, ... , -2717640000 ],
    "diffs" : [ 172378, 85978, ..., 0 ]
  }
]
```
  The JSON files are generated by 2 tests in `RebaseDateTimeSuite` - `generate 'gregorian-julian-rebase-micros.json'` and `generate 'julian-gregorian-rebase-micros.json'`. Both tests are disabled by default.
  The `switches` time points are ordered from old to recent timestamps. This condition is checked by the test `validate rebase records in JSON files` in `RebaseDateTimeSuite`. Also sizes of the `switches` and `diffs` arrays are the same (this is checked by the same test).

- The **_Asia/Tehran, Iran, Africa/Casablanca and Africa/El_Aaiun_** time zones weren't added to the JSON files, see [SPARK-31385](https://issues.apache.org/jira/browse/SPARK-31385)
- The rebase info from the JSON files is placed to hash tables - `gregJulianRebaseMap` and `julianGregRebaseMap`. I use `AnyRefMap` because it is almost 2 times faster than Scala's immutable Map. Also I tried `java.util.HashMap` but it has worse lookup time than `AnyRefMap` in our case.
The hash maps store the switch time points and diffs in microseconds precision to avoid conversions from microseconds to seconds in the runtime.

- I moved the code related to days and microseconds rebasing to the separate object `RebaseDateTime` to do not pollute `DateTimeUtils`. Tests related to date-time rebasing are moved to `RebaseDateTimeSuite` for the same reason.

- I placed rebasing via local timestamp to separate methods that require zone id as the first parameter assuming that the caller has zone id already. This allows to void unnecessary retrieving the default time zone. The methods are marked as `private[sql]` because they are used in `RebaseDateTimeSuite` as reference implementation.

- Modified the `rebaseGregorianToJulianMicros()` and `rebaseJulianToGregorianMicros()` methods in `RebaseDateTime` to look up the rebase tables first of all. If hash maps don't contain rebasing info for the given time zone id, the methods falls back to the implementation via local timestamps. This allows to support time zones specified as zone offsets like '-08:00'.

### Why are the changes needed?
To make timestamps rebasing faster:
- Saving timestamps to parquet files is ~ **x3.8 faster**
- Loading timestamps from parquet files is ~**x2.8 faster**.
- Loading timestamps by Vectorized reader ~**x4.6 faster**.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
- Added the test `validate rebase records in JSON files` to `RebaseDateTimeSuite`. The test validates 2 json files from the resource folder - `gregorian-julian-rebase-micros.json` and `julian-gregorian-rebase-micros.json`, and it checks per each time zone records that
  - the number of switch points is equal to the number of diffs between calendars. If the numbers are different, this will violate the assumption made in `RebaseDateTime.rebaseMicros`.
  - swith points are ordered from old to recent timestamps. This pre-condition is required for linear search in the `rebaseMicros` function.
- Added the test `optimization of micros rebasing - Gregorian to Julian` to `RebaseDateTimeSuite` which iterates over timestamps from 0001-01-01 to 2100-01-01 with the steps 1 ± 0.5 months, and checks that optimised function `RebaseDateTime`.`rebaseGregorianToJulianMicros()` returns the same result as non-optimised one. The check is performed for the UTC, PST, CET, Africa/Dakar, America/Los_Angeles, Antarctica/Vostok, Asia/Hong_Kong, Europe/Amsterdam time zones.
- Added the test `optimization of micros rebasing - Julian to Gregorian` to `RebaseDateTimeSuite` which does similar checks as the test above but for rebasing from the hybrid calendar (Julian + Gregorian) to Proleptic Gregorian calendar.
- The tests for days rebasing are moved from `DateTimeUtilsSuite` to `RebaseDateTimeSuite` because the rebasing related code is moved from `DateTimeUtils` to the separate object `RebaseDateTime`.
- Re-run `DateTimeRebaseBenchmark` at the America/Los_Angeles time zone (it is set explicitly in the PR #28127):

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

Closes #28163 from MaxGekk/optimize-rebase-micros-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/avro/AvroDeserializer.scala     |    7 +-
 .../spark/sql/avro/AvroSerializer.scala       |    7 +-
 .../gregorian-julian-rebase-micros.json       | 2385 +++++++++++++++++
 .../julian-gregorian-rebase-micros.json       | 2385 +++++++++++++++++
 .../sql/catalyst/util/DateTimeUtils.scala     |  152 +-
 .../sql/catalyst/util/RebaseDateTime.scala    |  338 +++
 .../catalyst/util/DateTimeUtilsSuite.scala    |  182 +-
 .../catalyst/util/RebaseDateTimeSuite.scala   |  363 +++
 .../DateTimeRebaseBenchmark-jdk11-results.txt |   88 +-
 .../DateTimeRebaseBenchmark-results.txt       |   88 +-
 .../parquet/VectorizedColumnReader.java       |    7 +-
 .../execution/datasources/DaysWritable.scala  |    2 +-
 .../parquet/ParquetRowConverter.scala         |    7 +-
 .../parquet/ParquetWriteSupport.scala         |    7 +-
 .../datasources/orc/OrcColumnVector.java      |    3 +-
 .../datasources/orc/DaysWritable.scala        |    2 +-
 .../datasources/orc/OrcColumnVector.java      |    3 +-
 17 files changed, 5588 insertions(+), 438 deletions(-)
 create mode 100644 sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
 create mode 100644 sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
 create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
 create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index d2aa8a96aba84..ce8f9762751a0 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -95,7 +96,7 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
 
       case (INT, DateType) if rebaseDateTime => (updater, ordinal, value) =>
         val days = value.asInstanceOf[Int]
-        val rebasedDays = DateTimeUtils.rebaseJulianToGregorianDays(days)
+        val rebasedDays = rebaseJulianToGregorianDays(days)
         updater.setInt(ordinal, rebasedDays)
 
       case (INT, DateType) => (updater, ordinal, value) =>
@@ -110,7 +111,7 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
         case null | _: TimestampMillis if rebaseDateTime => (updater, ordinal, value) =>
           val millis = value.asInstanceOf[Long]
           val micros = DateTimeUtils.fromMillis(millis)
-          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+          val rebasedMicros = rebaseJulianToGregorianMicros(micros)
           updater.setLong(ordinal, rebasedMicros)
         case null | _: TimestampMillis => (updater, ordinal, value) =>
           val millis = value.asInstanceOf[Long]
@@ -118,7 +119,7 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
           updater.setLong(ordinal, micros)
         case _: TimestampMicros if rebaseDateTime => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
-          val rebasedMicros = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+          val rebasedMicros = rebaseJulianToGregorianMicros(micros)
           updater.setLong(ordinal, rebasedMicros)
         case _: TimestampMicros => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index 7f454e2bf9491..350d7a3005316 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -35,6 +35,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -142,7 +143,7 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
         (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
 
       case (DateType, INT) if rebaseDateTime =>
-        (getter, ordinal) => DateTimeUtils.rebaseGregorianToJulianDays(getter.getInt(ordinal))
+        (getter, ordinal) => rebaseGregorianToJulianDays(getter.getInt(ordinal))
 
       case (DateType, INT) =>
         (getter, ordinal) => getter.getInt(ordinal)
@@ -152,12 +153,12 @@ class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable:
           // (the `null` case), output the timestamp value as with millisecond precision.
           case null | _: TimestampMillis if rebaseDateTime => (getter, ordinal) =>
             val micros = getter.getLong(ordinal)
-            val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(micros)
+            val rebasedMicros = rebaseGregorianToJulianMicros(micros)
             DateTimeUtils.toMillis(rebasedMicros)
           case null | _: TimestampMillis => (getter, ordinal) =>
             DateTimeUtils.toMillis(getter.getLong(ordinal))
           case _: TimestampMicros if rebaseDateTime => (getter, ordinal) =>
-            DateTimeUtils.rebaseGregorianToJulianMicros(getter.getLong(ordinal))
+            rebaseGregorianToJulianMicros(getter.getLong(ordinal))
           case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
diff --git a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
new file mode 100644
index 0000000000000..050aa02918fc3
--- /dev/null
+++ b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
@@ -0,0 +1,2385 @@
+[ {
+  "tz" : "Africa/Abidjan",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Accra",
+  "switches" : [ -62135596748, -59006361548, -55850687948, -52695014348, -46383580748, -43227907148, -40072233548, -33760799948, -30605126348, -27449452748, -21138019148, -17982345548, -14826671948, -12219292748, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, -52, 0 ]
+}, {
+  "tz" : "Africa/Addis_Ababa",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Algiers",
+  "switches" : [ -62135597532, -59006362332, -55850688732, -52695015132, -46383581532, -43227907932, -40072234332, -33760800732, -30605127132, -27449453532, -21138019932, -17982346332, -14826672732, -12219293532, -2486679072, -2208988800 ],
+  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, -2868, -3039, 0 ]
+}, {
+  "tz" : "Africa/Asmara",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Asmera",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Bamako",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Bangui",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Banjul",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Bissau",
+  "switches" : [ -62135593060, -59006357860, -55850684260, -52695010660, -46383577060, -43227903460, -40072229860, -33760796260, -30605122660, -27449449060, -21138015460, -17982341860, -14826668260, -12219289060, -2208988800 ],
+  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, -3740, 0 ]
+}, {
+  "tz" : "Africa/Blantyre",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Brazzaville",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Bujumbura",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Cairo",
+  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12219300309, -2208988800 ],
+  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 309, 0 ]
+}, {
+  "tz" : "Africa/Ceuta",
+  "switches" : [ -62135595524, -59006360324, -55850686724, -52695013124, -46383579524, -43227905924, -40072232324, -33760798724, -30605125124, -27449451524, -21138017924, -17982344324, -14826670724, -12219291524, -2208988800 ],
+  "diffs" : [ -177676, -91276, -4876, 81524, 167924, 254324, 340724, 427124, 513524, 599924, 686324, 772724, 859124, -4876, 0 ]
+}, {
+  "tz" : "Africa/Conakry",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Dakar",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Dar_es_Salaam",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Djibouti",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Douala",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Freetown",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Gaborone",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Harare",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Johannesburg",
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, -480, -1800, 0 ]
+}, {
+  "tz" : "Africa/Juba",
+  "switches" : [ -62135604388, -59006369188, -55850695588, -52695021988, -46383588388, -43227914788, -40072241188, -33760807588, -30605133988, -27449460388, -21138026788, -17982353188, -14826679588, -12219300388, -2208988800 ],
+  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, -3212, 0 ]
+}, {
+  "tz" : "Africa/Kampala",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Khartoum",
+  "switches" : [ -62135604608, -59006369408, -55850695808, -52695022208, -46383588608, -43227915008, -40072241408, -33760807808, -30605134208, -27449460608, -21138027008, -17982353408, -14826679808, -12219300608, -2208988800 ],
+  "diffs" : [ -172192, -85792, 608, 87008, 173408, 259808, 346208, 432608, 519008, 605408, 691808, 778208, 864608, 608, 0 ]
+}, {
+  "tz" : "Africa/Kigali",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Kinshasa",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Lagos",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Libreville",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Lome",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Luanda",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Lubumbashi",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Lusaka",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Malabo",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Maputo",
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+}, {
+  "tz" : "Africa/Maseru",
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, -480, -1800, 0 ]
+}, {
+  "tz" : "Africa/Mbabane",
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, -480, -1800, 0 ]
+}, {
+  "tz" : "Africa/Mogadishu",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Monrovia",
+  "switches" : [ -62135594212, -59006359012, -55850685412, -52695011812, -46383578212, -43227904612, -40072231012, -33760797412, -30605123812, -27449450212, -21138016612, -17982343012, -14826669412, -12219290212, -2208988800 ],
+  "diffs" : [ -175388, -88988, -2588, 83812, 170212, 256612, 343012, 429412, 515812, 602212, 688612, 775012, 861412, -2588, 0 ]
+}, {
+  "tz" : "Africa/Nairobi",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Africa/Ndjamena",
+  "switches" : [ -62135600412, -59006365212, -55850691612, -52695018012, -46383584412, -43227910812, -40072237212, -33760803612, -30605130012, -27449456412, -21138022812, -17982349212, -14826675612, -12219296412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 12, 0 ]
+}, {
+  "tz" : "Africa/Niamey",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Nouakchott",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Ouagadougou",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Porto-Novo",
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Africa/Sao_Tome",
+  "switches" : [ -62135598416, -59006363216, -55850689616, -52695016016, -46383582416, -43227908816, -40072235216, -33760801616, -30605128016, -27449454416, -21138020816, -17982347216, -14826673616, -12219294416, -2713912016, -2208988800 ],
+  "diffs" : [ -171184, -84784, 1616, 88016, 174416, 260816, 347216, 433616, 520016, 606416, 692816, 779216, 865616, 1616, -2205, 0 ]
+}, {
+  "tz" : "Africa/Timbuktu",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Africa/Tripoli",
+  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12219295964, -2208988800 ],
+  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, -4036, 0 ]
+}, {
+  "tz" : "Africa/Tunis",
+  "switches" : [ -62135599244, -59006364044, -55850690444, -52695016844, -46383583244, -43227909644, -40072236044, -33760802444, -30605128844, -27449455244, -21138021644, -17982348044, -14826674444, -12219295244, -2797202444, -2208988800 ],
+  "diffs" : [ -173956, -87556, -1156, 85244, 171644, 258044, 344444, 430844, 517244, 603644, 690044, 776444, 862844, -1156, -3039, 0 ]
+}, {
+  "tz" : "Africa/Windhoek",
+  "switches" : [ -62135600904, -59006365704, -55850692104, -52695018504, -46383584904, -43227911304, -40072237704, -33760804104, -30605130504, -27449456904, -21138023304, -17982349704, -14826676104, -12219296904, -2458170504, -2208988800 ],
+  "diffs" : [ -175896, -89496, -3096, 83304, 169704, 256104, 342504, 428904, 515304, 601704, 688104, 774504, 860904, -3096, -1800, 0 ]
+}, {
+  "tz" : "America/Adak",
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 80002, -6398, 0 ]
+}, {
+  "tz" : "America/Anchorage",
+  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12219343224, -3225223727, -2208988800 ],
+  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 82824, -3576, 0 ]
+}, {
+  "tz" : "America/Anguilla",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Antigua",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Araguaina",
+  "switches" : [ -62135585232, -59006350032, -55850676432, -52695002832, -46383569232, -43227895632, -40072222032, -33760788432, -30605114832, -27449441232, -21138007632, -17982334032, -14826660432, -12219281232, -2208988800 ],
+  "diffs" : [ -173568, -87168, -768, 85632, 172032, 258432, 344832, 431232, 517632, 604032, 690432, 776832, 863232, -768, 0 ]
+}, {
+  "tz" : "America/Argentina/Buenos_Aires",
+  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12219278772, -2372097972, -2208988800 ],
+  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, -3228, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Catamarca",
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, -4988, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/ComodRivadavia",
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, -4988, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Cordoba",
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Jujuy",
+  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12219277128, -2372096328, -2208988800 ],
+  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, -4872, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/La_Rioja",
+  "switches" : [ -62135580756, -59006345556, -55850671956, -52694998356, -46383564756, -43227891156, -40072217556, -33760783956, -30605110356, -27449436756, -21138003156, -17982329556, -14826655956, -12219276756, -2372095956, -2208988800 ],
+  "diffs" : [ -178044, -91644, -5244, 81156, 167556, 253956, 340356, 426756, 513156, 599556, 685956, 772356, 858756, -5244, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Mendoza",
+  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12219276284, -2372095484, -2208988800 ],
+  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, -5716, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Rio_Gallegos",
+  "switches" : [ -62135580188, -59006344988, -55850671388, -52694997788, -46383564188, -43227890588, -40072216988, -33760783388, -30605109788, -27449436188, -21138002588, -17982328988, -14826655388, -12219276188, -2372095388, -2208988800 ],
+  "diffs" : [ -178612, -92212, -5812, 80588, 166988, 253388, 339788, 426188, 512588, 598988, 685388, 771788, 858188, -5812, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Salta",
+  "switches" : [ -62135581100, -59006345900, -55850672300, -52694998700, -46383565100, -43227891500, -40072217900, -33760784300, -30605110700, -27449437100, -21138003500, -17982329900, -14826656300, -12219277100, -2372096300, -2208988800 ],
+  "diffs" : [ -177700, -91300, -4900, 81500, 167900, 254300, 340700, 427100, 513500, 599900, 686300, 772700, 859100, -4900, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Juan",
+  "switches" : [ -62135580356, -59006345156, -55850671556, -52694997956, -46383564356, -43227890756, -40072217156, -33760783556, -30605109956, -27449436356, -21138002756, -17982329156, -14826655556, -12219276356, -2372095556, -2208988800 ],
+  "diffs" : [ -178444, -92044, -5644, 80756, 167156, 253556, 339956, 426356, 512756, 599156, 685556, 771956, 858356, -5644, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Luis",
+  "switches" : [ -62135580876, -59006345676, -55850672076, -52694998476, -46383564876, -43227891276, -40072217676, -33760784076, -30605110476, -27449436876, -21138003276, -17982329676, -14826656076, -12219276876, -2372096076, -2208988800 ],
+  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, -5124, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Tucuman",
+  "switches" : [ -62135581148, -59006345948, -55850672348, -52694998748, -46383565148, -43227891548, -40072217948, -33760784348, -30605110748, -27449437148, -21138003548, -17982329948, -14826656348, -12219277148, -2372096348, -2208988800 ],
+  "diffs" : [ -177652, -91252, -4852, 81548, 167948, 254348, 340748, 427148, 513548, 599948, 686348, 772748, 859148, -4852, -4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Ushuaia",
+  "switches" : [ -62135580408, -59006345208, -55850671608, -52694998008, -46383564408, -43227890808, -40072217208, -33760783608, -30605110008, -27449436408, -21138002808, -17982329208, -14826655608, -12219276408, -2372095608, -2208988800 ],
+  "diffs" : [ -178392, -91992, -5592, 80808, 167208, 253608, 340008, 426408, 512808, 599208, 685608, 772008, 858408, -5592, -4608, 0 ]
+}, {
+  "tz" : "America/Aruba",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+}, {
+  "tz" : "America/Asuncion",
+  "switches" : [ -62135582960, -59006347760, -55850674160, -52695000560, -46383566960, -43227893360, -40072219760, -33760786160, -30605112560, -27449438960, -21138005360, -17982331760, -14826658160, -12219278960, -2208988800 ],
+  "diffs" : [ -172240, -85840, 560, 86960, 173360, 259760, 346160, 432560, 518960, 605360, 691760, 778160, 864560, 560, 0 ]
+}, {
+  "tz" : "America/Atikokan",
+  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12219270812, -2366733212, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, -3600, 0 ]
+}, {
+  "tz" : "America/Atka",
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 80002, -6398, 0 ]
+}, {
+  "tz" : "America/Bahia",
+  "switches" : [ -62135587556, -59006352356, -55850678756, -52695005156, -46383571556, -43227897956, -40072224356, -33760790756, -30605117156, -27449443556, -21138009956, -17982336356, -14826662756, -12219283556, -2208988800 ],
+  "diffs" : [ -171244, -84844, 1556, 87956, 174356, 260756, 347156, 433556, 519956, 606356, 692756, 779156, 865556, 1556, 0 ]
+}, {
+  "tz" : "America/Bahia_Banderas",
+  "switches" : [ -62135571540, -59006336340, -55850662740, -52694989140, -46383555540, -43227881940, -40072208340, -33760774740, -30605101140, -27449427540, -21137993940, -17982320340, -14826646740, -12219267540, -2208988800 ],
+  "diffs" : [ -176460, -90060, -3660, 82740, 169140, 255540, 341940, 428340, 514740, 601140, 687540, 773940, 860340, -3660, 0 ]
+}, {
+  "tz" : "America/Barbados",
+  "switches" : [ -62135582491, -59006347291, -55850673691, -52695000091, -46383566491, -43227892891, -40072219291, -33760785691, -30605112091, -27449438491, -21138004891, -17982331291, -14826657691, -12219278491, -2208988800 ],
+  "diffs" : [ -172709, -86309, 91, 86491, 172891, 259291, 345691, 432091, 518491, 604891, 691291, 777691, 864091, 91, 0 ]
+}, {
+  "tz" : "America/Belem",
+  "switches" : [ -62135585164, -59006349964, -55850676364, -52695002764, -46383569164, -43227895564, -40072221964, -33760788364, -30605114764, -27449441164, -21138007564, -17982333964, -14826660364, -12219281164, -2208988800 ],
+  "diffs" : [ -173636, -87236, -836, 85564, 171964, 258364, 344764, 431164, 517564, 603964, 690364, 776764, 863164, -836, 0 ]
+}, {
+  "tz" : "America/Belize",
+  "switches" : [ -62135575632, -59006340432, -55850666832, -52694993232, -46383559632, -43227886032, -40072212432, -33760778832, -30605105232, -27449431632, -21137998032, -17982324432, -14826650832, -12219271632, -2208988800 ],
+  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 432, 0 ]
+}, {
+  "tz" : "America/Blanc-Sablon",
+  "switches" : [ -62135583092, -59006347892, -55850674292, -52695000692, -46383567092, -43227893492, -40072219892, -33760786292, -30605112692, -27449439092, -21138005492, -17982331892, -14826658292, -12219279092, -2713896692 ],
+  "diffs" : [ -172108, -85708, 692, 87092, 173492, 259892, 346292, 432692, 519092, 605492, 691892, 778292, 864692, 692, 0 ]
+}, {
+  "tz" : "America/Boa_Vista",
+  "switches" : [ -62135582240, -59006347040, -55850673440, -52694999840, -46383566240, -43227892640, -40072219040, -33760785440, -30605111840, -27449438240, -21138004640, -17982331040, -14826657440, -12219278240, -2208988800 ],
+  "diffs" : [ -172960, -86560, -160, 86240, 172640, 259040, 345440, 431840, 518240, 604640, 691040, 777440, 863840, -160, 0 ]
+}, {
+  "tz" : "America/Bogota",
+  "switches" : [ -62135579024, -59006343824, -55850670224, -52694996624, -46383563024, -43227889424, -40072215824, -33760782224, -30605108624, -27449435024, -21138001424, -17982327824, -14826654224, -12219275024, -2208988800 ],
+  "diffs" : [ -172576, -86176, 224, 86624, 173024, 259424, 345824, 432224, 518624, 605024, 691424, 777824, 864224, 224, 0 ]
+}, {
+  "tz" : "America/Boise",
+  "switches" : [ -62135568911, -59006333711, -55850660111, -52694986511, -46383552911, -43227879311, -40072205711, -33760772111, -30605098511, -27449424911, -21137991311, -17982317711, -14826644111, -12219264911, -2717640000, -2208988800 ],
+  "diffs" : [ -175489, -89089, -2689, 83711, 170111, 256511, 342911, 429311, 515711, 602111, 688511, 774911, 861311, -2689, -3600, 0 ]
+}, {
+  "tz" : "America/Buenos_Aires",
+  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12219278772, -2372097972, -2208988800 ],
+  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, -3228, -4608, 0 ]
+}, {
+  "tz" : "America/Cambridge_Bay",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 25200, 0 ]
+}, {
+  "tz" : "America/Campo_Grande",
+  "switches" : [ -62135583692, -59006348492, -55850674892, -52695001292, -46383567692, -43227894092, -40072220492, -33760786892, -30605113292, -27449439692, -21138006092, -17982332492, -14826658892, -12219279692, -2208988800 ],
+  "diffs" : [ -171508, -85108, 1292, 87692, 174092, 260492, 346892, 433292, 519692, 606092, 692492, 778892, 865292, 1292, 0 ]
+}, {
+  "tz" : "America/Cancun",
+  "switches" : [ -62135575976, -59006340776, -55850667176, -52694993576, -46383559976, -43227886376, -40072212776, -33760779176, -30605105576, -27449431976, -21137998376, -17982324776, -14826651176, -12219271976, -2208988800 ],
+  "diffs" : [ -175624, -89224, -2824, 83576, 169976, 256376, 342776, 429176, 515576, 601976, 688376, 774776, 861176, -2824, 0 ]
+}, {
+  "tz" : "America/Caracas",
+  "switches" : [ -62135580736, -59006345536, -55850671936, -52694998336, -46383564736, -43227891136, -40072217536, -33760783936, -30605110336, -27449436736, -21138003136, -17982329536, -14826655936, -12219276736, -2524505536, -2208988800 ],
+  "diffs" : [ -174464, -88064, -1664, 84736, 171136, 257536, 343936, 430336, 516736, 603136, 689536, 775936, 862336, -1664, -1660, 0 ]
+}, {
+  "tz" : "America/Catamarca",
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, -4988, -4608, 0 ]
+}, {
+  "tz" : "America/Cayenne",
+  "switches" : [ -62135584240, -59006349040, -55850675440, -52695001840, -46383568240, -43227894640, -40072221040, -33760787440, -30605113840, -27449440240, -21138006640, -17982333040, -14826659440, -12219280240, -2208988800 ],
+  "diffs" : [ -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, 862240, -1760, 0 ]
+}, {
+  "tz" : "America/Cayman",
+  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12219273712, -2524502512, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, -1088, -1176, 0 ]
+}, {
+  "tz" : "America/Chicago",
+  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12219271764, -2717647200 ],
+  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 564, 0 ]
+}, {
+  "tz" : "America/Chihuahua",
+  "switches" : [ -62135571340, -59006336140, -55850662540, -52694988940, -46383555340, -43227881740, -40072208140, -33760774540, -30605100940, -27449427340, -21137993740, -17982320140, -14826646540, -12219267340, -2208988800 ],
+  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, -260, 0 ]
+}, {
+  "tz" : "America/Coral_Harbour",
+  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12219270812, -2366733212, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, -3600, 0 ]
+}, {
+  "tz" : "America/Cordoba",
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, -4608, 0 ]
+}, {
+  "tz" : "America/Costa_Rica",
+  "switches" : [ -62135576627, -59006341427, -55850667827, -52694994227, -46383560627, -43227887027, -40072213427, -33760779827, -30605106227, -27449432627, -21137999027, -17982325427, -14826651827, -12219272627, -2208988800 ],
+  "diffs" : [ -171373, -84973, 1427, 87827, 174227, 260627, 347027, 433427, 519827, 606227, 692627, 779027, 865427, 1427, 0 ]
+}, {
+  "tz" : "America/Creston",
+  "switches" : [ -62135568836, -59006333636, -55850660036, -52694986436, -46383552836, -43227879236, -40072205636, -33760772036, -30605098436, -27449424836, -21137991236, -17982317636, -14826644036, -12219264836, -2713882436 ],
+  "diffs" : [ -175564, -89164, -2764, 83636, 170036, 256436, 342836, 429236, 515636, 602036, 688436, 774836, 861236, -2764, 0 ]
+}, {
+  "tz" : "America/Cuiaba",
+  "switches" : [ -62135583340, -59006348140, -55850674540, -52695000940, -46383567340, -43227893740, -40072220140, -33760786540, -30605112940, -27449439340, -21138005740, -17982332140, -14826658540, -12219279340, -2208988800 ],
+  "diffs" : [ -171860, -85460, 940, 87340, 173740, 260140, 346540, 432940, 519340, 605740, 692140, 778540, 864940, 940, 0 ]
+}, {
+  "tz" : "America/Curacao",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+}, {
+  "tz" : "America/Danmarkshavn",
+  "switches" : [ -62135592320, -59006357120, -55850683520, -52695009920, -46383576320, -43227902720, -40072229120, -33760795520, -30605121920, -27449448320, -21138014720, -17982341120, -14826667520, -12219288320, -2208988800 ],
+  "diffs" : [ -177280, -90880, -4480, 81920, 168320, 254720, 341120, 427520, 513920, 600320, 686720, 773120, 859520, -4480, 0 ]
+}, {
+  "tz" : "America/Dawson",
+  "switches" : [ -62135563340, -59006328140, -55850654540, -52694980940, -46383547340, -43227873740, -40072200140, -33760766540, -30605092940, -27449419340, -21137985740, -17982312140, -14826638540, -12219259340, -2208988800 ],
+  "diffs" : [ -177460, -91060, -4660, 81740, 168140, 254540, 340940, 427340, 513740, 600140, 686540, 772940, 859340, -4660, 0 ]
+}, {
+  "tz" : "America/Dawson_Creek",
+  "switches" : [ -62135567944, -59006332744, -55850659144, -52694985544, -46383551944, -43227878344, -40072204744, -33760771144, -30605097544, -27449423944, -21137990344, -17982316744, -14826643144, -12219263944, -2713881544, -2208988800 ],
+  "diffs" : [ -176456, -90056, -3656, 82744, 169144, 255544, 341944, 428344, 514744, 601144, 687544, 773944, 860344, -3656, -3600, 0 ]
+}, {
+  "tz" : "America/Denver",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+}, {
+  "tz" : "America/Detroit",
+  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12219272869, -2208988800 ],
+  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, -1931, 0 ]
+}, {
+  "tz" : "America/Dominica",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Edmonton",
+  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12219265568, -2208988800 ],
+  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, -2032, 0 ]
+}, {
+  "tz" : "America/Eirunepe",
+  "switches" : [ -62135580032, -59006344832, -55850671232, -52694997632, -46383564032, -43227890432, -40072216832, -33760783232, -30605109632, -27449436032, -21138002432, -17982328832, -14826655232, -12219276032, -2208988800 ],
+  "diffs" : [ -171568, -85168, 1232, 87632, 174032, 260432, 346832, 433232, 519632, 606032, 692432, 778832, 865232, 1232, 0 ]
+}, {
+  "tz" : "America/El_Salvador",
+  "switches" : [ -62135575392, -59006340192, -55850666592, -52694992992, -46383559392, -43227885792, -40072212192, -33760778592, -30605104992, -27449431392, -21137997792, -17982324192, -14826650592, -12219271392, -2208988800 ],
+  "diffs" : [ -172608, -86208, 192, 86592, 172992, 259392, 345792, 432192, 518592, 604992, 691392, 777792, 864192, 192, 0 ]
+}, {
+  "tz" : "America/Ensenada",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+}, {
+  "tz" : "America/Fort_Nelson",
+  "switches" : [ -62135567353, -59006332153, -55850658553, -52694984953, -46383551353, -43227877753, -40072204153, -33760770553, -30605096953, -27449423353, -21137989753, -17982316153, -14826642553, -12219263353, -2713880953, -2208988800 ],
+  "diffs" : [ -177047, -90647, -4247, 82153, 168553, 254953, 341353, 427753, 514153, 600553, 686953, 773353, 859753, -4247, -3600, 0 ]
+}, {
+  "tz" : "America/Fort_Wayne",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+}, {
+  "tz" : "America/Fortaleza",
+  "switches" : [ -62135587560, -59006352360, -55850678760, -52695005160, -46383571560, -43227897960, -40072224360, -33760790760, -30605117160, -27449443560, -21138009960, -17982336360, -14826662760, -12219283560, -2208988800 ],
+  "diffs" : [ -171240, -84840, 1560, 87960, 174360, 260760, 347160, 433560, 519960, 606360, 692760, 779160, 865560, 1560, 0 ]
+}, {
+  "tz" : "America/Glace_Bay",
+  "switches" : [ -62135582412, -59006347212, -55850673612, -52695000012, -46383566412, -43227892812, -40072219212, -33760785612, -30605112012, -27449438412, -21138004812, -17982331212, -14826657612, -12219278412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 12, 0 ]
+}, {
+  "tz" : "America/Godthab",
+  "switches" : [ -62135584384, -59006349184, -55850675584, -52695001984, -46383568384, -43227894784, -40072221184, -33760787584, -30605113984, -27449440384, -21138006784, -17982333184, -14826659584, -12219280384, -2208988800 ],
+  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, -1616, 0 ]
+}, {
+  "tz" : "America/Goose_Bay",
+  "switches" : [ -62135582300, -59006347100, -55850673500, -52694999900, -46383566300, -43227892700, -40072219100, -33760785500, -30605111900, -27449438300, -21138004700, -17982331100, -14826657500, -12219278300, -2713895900, -2208988800 ],
+  "diffs" : [ -172900, -86500, -100, 86300, 172700, 259100, 345500, 431900, 518300, 604700, 691100, 777500, 863900, -100, 1748, 0 ]
+}, {
+  "tz" : "America/Grand_Turk",
+  "switches" : [ -62135579728, -59006344528, -55850670928, -52694997328, -46383563728, -43227890128, -40072216528, -33760782928, -30605109328, -27449435728, -21138002128, -17982328528, -14826654928, -12219275728, -2524504528, -2208988800 ],
+  "diffs" : [ -171872, -85472, 928, 87328, 173728, 260128, 346528, 432928, 519328, 605728, 692128, 778528, 864928, 928, -430, 0 ]
+}, {
+  "tz" : "America/Grenada",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Guadeloupe",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Guatemala",
+  "switches" : [ -62135575076, -59006339876, -55850666276, -52694992676, -46383559076, -43227885476, -40072211876, -33760778276, -30605104676, -27449431076, -21137997476, -17982323876, -14826650276, -12219271076, -2208988800 ],
+  "diffs" : [ -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, 863876, -124, 0 ]
+}, {
+  "tz" : "America/Guayaquil",
+  "switches" : [ -62135577640, -59006342440, -55850668840, -52694995240, -46383561640, -43227888040, -40072214440, -33760780840, -30605107240, -27449433640, -21138000040, -17982326440, -14826652840, -12219273640, -2524502440, -2208988800 ],
+  "diffs" : [ -173960, -87560, -1160, 85240, 171640, 258040, 344440, 430840, 517240, 603640, 690040, 776440, 862840, -1160, -840, 0 ]
+}, {
+  "tz" : "America/Guyana",
+  "switches" : [ -62135582840, -59006347640, -55850674040, -52695000440, -46383566840, -43227893240, -40072219640, -33760786040, -30605112440, -27449438840, -21138005240, -17982331640, -14826658040, -12219278840, -2208988800 ],
+  "diffs" : [ -172360, -85960, 440, 86840, 173240, 259640, 346040, 432440, 518840, 605240, 691640, 778040, 864440, 440, 0 ]
+}, {
+  "tz" : "America/Halifax",
+  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12219277536, -2208988800 ],
+  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, -864, 0 ]
+}, {
+  "tz" : "America/Havana",
+  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12219273032, -2524501832, -2208988800 ],
+  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, -1768, -1776, 0 ]
+}, {
+  "tz" : "America/Hermosillo",
+  "switches" : [ -62135570168, -59006334968, -55850661368, -52694987768, -46383554168, -43227880568, -40072206968, -33760773368, -30605099768, -27449426168, -21137992568, -17982318968, -14826645368, -12219266168, -2208988800 ],
+  "diffs" : [ -174232, -87832, -1432, 84968, 171368, 257768, 344168, 430568, 516968, 603368, 689768, 776168, 862568, -1432, 0 ]
+}, {
+  "tz" : "America/Indiana/Indianapolis",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Knox",
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 810, 0 ]
+}, {
+  "tz" : "America/Indiana/Marengo",
+  "switches" : [ -62135576077, -59006340877, -55850667277, -52694993677, -46383560077, -43227886477, -40072212877, -33760779277, -30605105677, -27449432077, -21137998477, -17982324877, -14826651277, -12219272077, -2717647200, -2208988800 ],
+  "diffs" : [ -175523, -89123, -2723, 83677, 170077, 256477, 342877, 429277, 515677, 602077, 688477, 774877, 861277, -2723, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Petersburg",
+  "switches" : [ -62135575853, -59006340653, -55850667053, -52694993453, -46383559853, -43227886253, -40072212653, -33760779053, -30605105453, -27449431853, -21137998253, -17982324653, -14826651053, -12219271853, -2717647200, -2208988800 ],
+  "diffs" : [ -175747, -89347, -2947, 83453, 169853, 256253, 342653, 429053, 515453, 601853, 688253, 774653, 861053, -2947, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Tell_City",
+  "switches" : [ -62135575977, -59006340777, -55850667177, -52694993577, -46383559977, -43227886377, -40072212777, -33760779177, -30605105577, -27449431977, -21137998377, -17982324777, -14826651177, -12219271977, -2717647200 ],
+  "diffs" : [ -172023, -85623, 777, 87177, 173577, 259977, 346377, 432777, 519177, 605577, 691977, 778377, 864777, 777, 0 ]
+}, {
+  "tz" : "America/Indiana/Vevay",
+  "switches" : [ -62135576384, -59006341184, -55850667584, -52694993984, -46383560384, -43227886784, -40072213184, -33760779584, -30605105984, -27449432384, -21137998784, -17982325184, -14826651584, -12219272384, -2717647200, -2208988800 ],
+  "diffs" : [ -175216, -88816, -2416, 83984, 170384, 256784, 343184, 429584, 515984, 602384, 688784, 775184, 861584, -2416, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Vincennes",
+  "switches" : [ -62135575793, -59006340593, -55850666993, -52694993393, -46383559793, -43227886193, -40072212593, -33760778993, -30605105393, -27449431793, -21137998193, -17982324593, -14826650993, -12219271793, -2717647200, -2208988800 ],
+  "diffs" : [ -175807, -89407, -3007, 83393, 169793, 256193, 342593, 428993, 515393, 601793, 688193, 774593, 860993, -3007, -3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Winamac",
+  "switches" : [ -62135576015, -59006340815, -55850667215, -52694993615, -46383560015, -43227886415, -40072212815, -33760779215, -30605105615, -27449432015, -21137998415, -17982324815, -14826651215, -12219272015, -2717647200, -2208988800 ],
+  "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, -2785, -3600, 0 ]
+}, {
+  "tz" : "America/Indianapolis",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+}, {
+  "tz" : "America/Inuvik",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 25200, 0 ]
+}, {
+  "tz" : "America/Iqaluit",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 18000, 0 ]
+}, {
+  "tz" : "America/Jamaica",
+  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12219274370, -2208988800 ],
+  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, -430, 0 ]
+}, {
+  "tz" : "America/Jujuy",
+  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12219277128, -2372096328, -2208988800 ],
+  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, -4872, -4608, 0 ]
+}, {
+  "tz" : "America/Juneau",
+  "switches" : [ -62135650939, -59006415739, -55850742139, -52695068539, -46383634939, -43227961339, -40072287739, -33760854139, -30605180539, -27449506939, -21138073339, -17982399739, -14826726139, -12219346939, -3225223727, -2208988800 ],
+  "diffs" : [ -86261, 139, 86539, 172939, 259339, 345739, 432139, 518539, 604939, 691339, 777739, 864139, 950539, 86539, 139, 0 ]
+}, {
+  "tz" : "America/Kentucky/Louisville",
+  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12219272218, -2717647200, -2208988800 ],
+  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, -2582, -3600, 0 ]
+}, {
+  "tz" : "America/Kentucky/Monticello",
+  "switches" : [ -62135576436, -59006341236, -55850667636, -52694994036, -46383560436, -43227886836, -40072213236, -33760779636, -30605106036, -27449432436, -21137998836, -17982325236, -14826651636, -12219272436, -2717647200, -2208988800 ],
+  "diffs" : [ -175164, -88764, -2364, 84036, 170436, 256836, 343236, 429636, 516036, 602436, 688836, 775236, 861636, -2364, -3600, 0 ]
+}, {
+  "tz" : "America/Knox_IN",
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 810, 0 ]
+}, {
+  "tz" : "America/Kralendijk",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+}, {
+  "tz" : "America/La_Paz",
+  "switches" : [ -62135580444, -59006345244, -55850671644, -52694998044, -46383564444, -43227890844, -40072217244, -33760783644, -30605110044, -27449436444, -21138002844, -17982329244, -14826655644, -12219276444, -2208988800 ],
+  "diffs" : [ -174756, -88356, -1956, 84444, 170844, 257244, 343644, 430044, 516444, 602844, 689244, 775644, 862044, -1956, 0 ]
+}, {
+  "tz" : "America/Lima",
+  "switches" : [ -62135578308, -59006343108, -55850669508, -52694995908, -46383562308, -43227888708, -40072215108, -33760781508, -30605107908, -27449434308, -21138000708, -17982327108, -14826653508, -12219274308, -2524503108, -2208988800 ],
+  "diffs" : [ -173292, -86892, -492, 85908, 172308, 258708, 345108, 431508, 517908, 604308, 690708, 777108, 863508, -492, -516, 0 ]
+}, {
+  "tz" : "America/Los_Angeles",
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 422, 0 ]
+}, {
+  "tz" : "America/Louisville",
+  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12219272218, -2717647200, -2208988800 ],
+  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, -2582, -3600, 0 ]
+}, {
+  "tz" : "America/Lower_Princes",
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+}, {
+  "tz" : "America/Maceio",
+  "switches" : [ -62135588228, -59006353028, -55850679428, -52695005828, -46383572228, -43227898628, -40072225028, -33760791428, -30605117828, -27449444228, -21138010628, -17982337028, -14826663428, -12219284228, -2208988800 ],
+  "diffs" : [ -170572, -84172, 2228, 88628, 175028, 261428, 347828, 434228, 520628, 607028, 693428, 779828, 866228, 2228, 0 ]
+}, {
+  "tz" : "America/Managua",
+  "switches" : [ -62135576092, -59006340892, -55850667292, -52694993692, -46383560092, -43227886492, -40072212892, -33760779292, -30605105692, -27449432092, -21137998492, -17982324892, -14826651292, -12219272092, -2524500892, -2208988800 ],
+  "diffs" : [ -171908, -85508, 892, 87292, 173692, 260092, 346492, 432892, 519292, 605692, 692092, 778492, 864892, 892, 888, 0 ]
+}, {
+  "tz" : "America/Manaus",
+  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12219278396, -2208988800 ],
+  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, -4, 0 ]
+}, {
+  "tz" : "America/Marigot",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Martinique",
+  "switches" : [ -62135582140, -59006346940, -55850673340, -52694999740, -46383566140, -43227892540, -40072218940, -33760785340, -30605111740, -27449438140, -21138004540, -17982330940, -14826657340, -12219278140, -2208988800 ],
+  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, -260, 0 ]
+}, {
+  "tz" : "America/Matamoros",
+  "switches" : [ -62135572800, -59006337600, -55850664000, -52694990400, -46383556800, -43227883200, -40072209600, -33760776000, -30605102400, -27449428800, -21137995200, -17982321600, -14826648000, -12219268800, -2208988800 ],
+  "diffs" : [ -175200, -88800, -2400, 84000, 170400, 256800, 343200, 429600, 516000, 602400, 688800, 775200, 861600, -2400, 0 ]
+}, {
+  "tz" : "America/Mazatlan",
+  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12219267260, -2208988800 ],
+  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, -340, 0 ]
+}, {
+  "tz" : "America/Mendoza",
+  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12219276284, -2372095484, -2208988800 ],
+  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, -5716, -4608, 0 ]
+}, {
+  "tz" : "America/Menominee",
+  "switches" : [ -62135575773, -59006340573, -55850666973, -52694993373, -46383559773, -43227886173, -40072212573, -33760778973, -30605105373, -27449431773, -21137998173, -17982324573, -14826650973, -12219271773, -2659759773 ],
+  "diffs" : [ -172227, -85827, 573, 86973, 173373, 259773, 346173, 432573, 518973, 605373, 691773, 778173, 864573, 573, 0 ]
+}, {
+  "tz" : "America/Merida",
+  "switches" : [ -62135575292, -59006340092, -55850666492, -52694992892, -46383559292, -43227885692, -40072212092, -33760778492, -30605104892, -27449431292, -21137997692, -17982324092, -14826650492, -12219271292, -2208988800 ],
+  "diffs" : [ -172708, -86308, 92, 86492, 172892, 259292, 345692, 432092, 518492, 604892, 691292, 777692, 864092, 92, 0 ]
+}, {
+  "tz" : "America/Metlakatla",
+  "switches" : [ -62135651622, -59006416422, -55850742822, -52695069222, -46383635622, -43227962022, -40072288422, -33760854822, -30605181222, -27449507622, -21138074022, -17982400422, -14826726822, -12219347622, -3225223727, -2208988800 ],
+  "diffs" : [ -85578, 822, 87222, 173622, 260022, 346422, 432822, 519222, 605622, 692022, 778422, 864822, 951222, 87222, 822, 0 ]
+}, {
+  "tz" : "America/Mexico_City",
+  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12219269004, -2208988800 ],
+  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, -2196, 0 ]
+}, {
+  "tz" : "America/Miquelon",
+  "switches" : [ -62135583320, -59006348120, -55850674520, -52695000920, -46383567320, -43227893720, -40072220120, -33760786520, -30605112920, -27449439320, -21138005720, -17982332120, -14826658520, -12219279320, -2208988800 ],
+  "diffs" : [ -175480, -89080, -2680, 83720, 170120, 256520, 342920, 429320, 515720, 602120, 688520, 774920, 861320, -2680, 0 ]
+}, {
+  "tz" : "America/Moncton",
+  "switches" : [ -62135581252, -59006346052, -55850672452, -52694998852, -46383565252, -43227891652, -40072218052, -33760784452, -30605110852, -27449437252, -21138003652, -17982330052, -14826656452, -12219277252, -2715882052, -2208988800 ],
+  "diffs" : [ -173948, -87548, -1148, 85252, 171652, 258052, 344452, 430852, 517252, 603652, 690052, 776452, 862852, -1148, -3600, 0 ]
+}, {
+  "tz" : "America/Monterrey",
+  "switches" : [ -62135572724, -59006337524, -55850663924, -52694990324, -46383556724, -43227883124, -40072209524, -33760775924, -30605102324, -27449428724, -21137995124, -17982321524, -14826647924, -12219268724, -2208988800 ],
+  "diffs" : [ -175276, -88876, -2476, 83924, 170324, 256724, 343124, 429524, 515924, 602324, 688724, 775124, 861524, -2476, 0 ]
+}, {
+  "tz" : "America/Montevideo",
+  "switches" : [ -62135583309, -59006348109, -55850674509, -52695000909, -46383567309, -43227893709, -40072220109, -33760786509, -30605112909, -27449439309, -21138005709, -17982332109, -14826658509, -12219279309, -2208988800 ],
+  "diffs" : [ -175491, -89091, -2691, 83709, 170109, 256509, 342909, 429309, 515709, 602109, 688509, 774909, 861309, -2691, 0 ]
+}, {
+  "tz" : "America/Montreal",
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, -1052, 0 ]
+}, {
+  "tz" : "America/Montserrat",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Nassau",
+  "switches" : [ -62135578230, -59006343030, -55850669430, -52694995830, -46383562230, -43227888630, -40072215030, -33760781430, -30605107830, -27449434230, -21138000630, -17982327030, -14826653430, -12219274230, -2208988800 ],
+  "diffs" : [ -173370, -86970, -570, 85830, 172230, 258630, 345030, 431430, 517830, 604230, 690630, 777030, 863430, -570, 0 ]
+}, {
+  "tz" : "America/New_York",
+  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12219275038, -2717650800 ],
+  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 238, 0 ]
+}, {
+  "tz" : "America/Nipigon",
+  "switches" : [ -62135575616, -59006340416, -55850666816, -52694993216, -46383559616, -43227886016, -40072212416, -33760778816, -30605105216, -27449431616, -21137998016, -17982324416, -14826650816, -12219271616, -2366734016 ],
+  "diffs" : [ -175984, -89584, -3184, 83216, 169616, 256016, 342416, 428816, 515216, 601616, 688016, 774416, 860816, -3184, 0 ]
+}, {
+  "tz" : "America/Nome",
+  "switches" : [ -62135643502, -59006408302, -55850734702, -52695061102, -46383627502, -43227953902, -40072280302, -33760846702, -30605173102, -27449499502, -21138065902, -17982392302, -14826718702, -12219339502, -3225223727, -2208988800 ],
+  "diffs" : [ -93698, -7298, 79102, 165502, 251902, 338302, 424702, 511102, 597502, 683902, 770302, 856702, 943102, 79102, -7298, 0 ]
+}, {
+  "tz" : "America/Noronha",
+  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12219285020, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Beulah",
+  "switches" : [ -62135572373, -59006337173, -55850663573, -52694989973, -46383556373, -43227882773, -40072209173, -33760775573, -30605101973, -27449428373, -21137994773, -17982321173, -14826647573, -12219268373, -2717643600, -2208988800 ],
+  "diffs" : [ -175627, -89227, -2827, 83573, 169973, 256373, 342773, 429173, 515573, 601973, 688373, 774773, 861173, -2827, -3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Center",
+  "switches" : [ -62135572488, -59006337288, -55850663688, -52694990088, -46383556488, -43227882888, -40072209288, -33760775688, -30605102088, -27449428488, -21137994888, -17982321288, -14826647688, -12219268488, -2717643600, -2208988800 ],
+  "diffs" : [ -175512, -89112, -2712, 83688, 170088, 256488, 342888, 429288, 515688, 602088, 688488, 774888, 861288, -2712, -3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/New_Salem",
+  "switches" : [ -62135572461, -59006337261, -55850663661, -52694990061, -46383556461, -43227882861, -40072209261, -33760775661, -30605102061, -27449428461, -21137994861, -17982321261, -14826647661, -12219268461, -2717643600, -2208988800 ],
+  "diffs" : [ -175539, -89139, -2739, 83661, 170061, 256461, 342861, 429261, 515661, 602061, 688461, 774861, 861261, -2739, -3600, 0 ]
+}, {
+  "tz" : "America/Ojinaga",
+  "switches" : [ -62135571740, -59006336540, -55850662940, -52694989340, -46383555740, -43227882140, -40072208540, -33760774940, -30605101340, -27449427740, -21137994140, -17982320540, -14826646940, -12219267740, -2208988800 ],
+  "diffs" : [ -172660, -86260, 140, 86540, 172940, 259340, 345740, 432140, 518540, 604940, 691340, 777740, 864140, 140, 0 ]
+}, {
+  "tz" : "America/Panama",
+  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12219273712, -2524502512, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, -1088, -1176, 0 ]
+}, {
+  "tz" : "America/Pangnirtung",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 18000, 0 ]
+}, {
+  "tz" : "America/Paramaribo",
+  "switches" : [ -62135583560, -59006348360, -55850674760, -52695001160, -46383567560, -43227893960, -40072220360, -33760786760, -30605113160, -27449439560, -21138005960, -17982332360, -14826658760, -12219279560, -2208988800 ],
+  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, -2440, 0 ]
+}, {
+  "tz" : "America/Phoenix",
+  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12219265902, -2717643600 ],
+  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, -1698, 0 ]
+}, {
+  "tz" : "America/Port-au-Prince",
+  "switches" : [ -62135579440, -59006344240, -55850670640, -52694997040, -46383563440, -43227889840, -40072216240, -33760782640, -30605109040, -27449435440, -21138001840, -17982328240, -14826654640, -12219275440, -2524504240, -2208988800 ],
+  "diffs" : [ -172160, -85760, 640, 87040, 173440, 259840, 346240, 432640, 519040, 605440, 691840, 778240, 864640, 640, 660, 0 ]
+}, {
+  "tz" : "America/Port_of_Spain",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Porto_Acre",
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 1728, 0 ]
+}, {
+  "tz" : "America/Porto_Velho",
+  "switches" : [ -62135581464, -59006346264, -55850672664, -52694999064, -46383565464, -43227891864, -40072218264, -33760784664, -30605111064, -27449437464, -21138003864, -17982330264, -14826656664, -12219277464, -2208988800 ],
+  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, -936, 0 ]
+}, {
+  "tz" : "America/Puerto_Rico",
+  "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12219276935, -2233035335 ],
+  "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, -1465, 0 ]
+}, {
+  "tz" : "America/Punta_Arenas",
+  "switches" : [ -62135579780, -59006344580, -55850670980, -52694997380, -46383563780, -43227890180, -40072216580, -33760782980, -30605109380, -27449435780, -21138002180, -17982328580, -14826654980, -12219275780, -2524504580, -2208988800 ],
+  "diffs" : [ -179020, -92620, -6220, 80180, 166580, 252980, 339380, 425780, 512180, 598580, 684980, 771380, 857780, -6220, -6166, 0 ]
+}, {
+  "tz" : "America/Rainy_River",
+  "switches" : [ -62135574104, -59006338904, -55850665304, -52694991704, -46383558104, -43227884504, -40072210904, -33760777304, -30605103704, -27449430104, -21137996504, -17982322904, -14826649304, -12219270104, -2366732504 ],
+  "diffs" : [ -173896, -87496, -1096, 85304, 171704, 258104, 344504, 430904, 517304, 603704, 690104, 776504, 862904, -1096, 0 ]
+}, {
+  "tz" : "America/Rankin_Inlet",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 21600, 0 ]
+}, {
+  "tz" : "America/Recife",
+  "switches" : [ -62135588424, -59006353224, -55850679624, -52695006024, -46383572424, -43227898824, -40072225224, -33760791624, -30605118024, -27449444424, -21138010824, -17982337224, -14826663624, -12219284424, -2208988800 ],
+  "diffs" : [ -170376, -83976, 2424, 88824, 175224, 261624, 348024, 434424, 520824, 607224, 693624, 780024, 866424, 2424, 0 ]
+}, {
+  "tz" : "America/Regina",
+  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12219267684, -2208988800 ],
+  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, -3516, 0 ]
+}, {
+  "tz" : "America/Resolute",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 21600, 0 ]
+}, {
+  "tz" : "America/Rio_Branco",
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 1728, 0 ]
+}, {
+  "tz" : "America/Rosario",
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, -4608, 0 ]
+}, {
+  "tz" : "America/Santa_Isabel",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+}, {
+  "tz" : "America/Santarem",
+  "switches" : [ -62135583672, -59006348472, -55850674872, -52695001272, -46383567672, -43227894072, -40072220472, -33760786872, -30605113272, -27449439672, -21138006072, -17982332472, -14826658872, -12219279672, -2208988800 ],
+  "diffs" : [ -175128, -88728, -2328, 84072, 170472, 256872, 343272, 429672, 516072, 602472, 688872, 775272, 861672, -2328, 0 ]
+}, {
+  "tz" : "America/Santiago",
+  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12219275834, -2208988800 ],
+  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, -2566, 0 ]
+}, {
+  "tz" : "America/Santo_Domingo",
+  "switches" : [ -62135580024, -59006344824, -55850671224, -52694997624, -46383564024, -43227890424, -40072216824, -33760783224, -30605109624, -27449436024, -21138002424, -17982328824, -14826655224, -12219276024, -2524504824, -2208988800 ],
+  "diffs" : [ -175176, -88776, -2376, 84024, 170424, 256824, 343224, 429624, 516024, 602424, 688824, 775224, 861624, -2376, -2400, 0 ]
+}, {
+  "tz" : "America/Sao_Paulo",
+  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12219281612, -2208988800 ],
+  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, -388, 0 ]
+}, {
+  "tz" : "America/Scoresbysund",
+  "switches" : [ -62135591528, -59006356328, -55850682728, -52695009128, -46383575528, -43227901928, -40072228328, -33760794728, -30605121128, -27449447528, -21138013928, -17982340328, -14826666728, -12219287528, -2208988800 ],
+  "diffs" : [ -174472, -88072, -1672, 84728, 171128, 257528, 343928, 430328, 516728, 603128, 689528, 775928, 862328, -1672, 0 ]
+}, {
+  "tz" : "America/Shiprock",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+}, {
+  "tz" : "America/Sitka",
+  "switches" : [ -62135650727, -59006415527, -55850741927, -52695068327, -46383634727, -43227961127, -40072287527, -33760853927, -30605180327, -27449506727, -21138073127, -17982399527, -14826725927, -12219346727, -3225223727, -2208988800 ],
+  "diffs" : [ -86473, -73, 86327, 172727, 259127, 345527, 431927, 518327, 604727, 691127, 777527, 863927, 950327, 86327, -73, 0 ]
+}, {
+  "tz" : "America/St_Barthelemy",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/St_Johns",
+  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12219280148, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, -52, 0 ]
+}, {
+  "tz" : "America/St_Kitts",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/St_Lucia",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/St_Thomas",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/St_Vincent",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Swift_Current",
+  "switches" : [ -62135570920, -59006335720, -55850662120, -52694988520, -46383554920, -43227881320, -40072207720, -33760774120, -30605100520, -27449426920, -21137993320, -17982319720, -14826646120, -12219266920, -2208988800 ],
+  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, -4280, 0 ]
+}, {
+  "tz" : "America/Tegucigalpa",
+  "switches" : [ -62135575868, -59006340668, -55850667068, -52694993468, -46383559868, -43227886268, -40072212668, -33760779068, -30605105468, -27449431868, -21137998268, -17982324668, -14826651068, -12219271868, -2208988800 ],
+  "diffs" : [ -172132, -85732, 668, 87068, 173468, 259868, 346268, 432668, 519068, 605468, 691868, 778268, 864668, 668, 0 ]
+}, {
+  "tz" : "America/Thule",
+  "switches" : [ -62135580292, -59006345092, -55850671492, -52694997892, -46383564292, -43227890692, -40072217092, -33760783492, -30605109892, -27449436292, -21138002692, -17982329092, -14826655492, -12219276292, -2208988800 ],
+  "diffs" : [ -174908, -88508, -2108, 84292, 170692, 257092, 343492, 429892, 516292, 602692, 689092, 775492, 861892, -2108, 0 ]
+}, {
+  "tz" : "America/Thunder_Bay",
+  "switches" : [ -62135575380, -59006340180, -55850666580, -52694992980, -46383559380, -43227885780, -40072212180, -33760778580, -30605104980, -27449431380, -21137997780, -17982324180, -14826650580, -12219271380, -2366733780, -2208988800 ],
+  "diffs" : [ -176220, -89820, -3420, 82980, 169380, 255780, 342180, 428580, 514980, 601380, 687780, 774180, 860580, -3420, -3600, 0 ]
+}, {
+  "tz" : "America/Tijuana",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+}, {
+  "tz" : "America/Toronto",
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, -1052, 0 ]
+}, {
+  "tz" : "America/Tortola",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Vancouver",
+  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12219263252, -2713880852 ],
+  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, -748, 0 ]
+}, {
+  "tz" : "America/Virgin",
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+}, {
+  "tz" : "America/Whitehorse",
+  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12219260388, -2208988800 ],
+  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, -3612, 0 ]
+}, {
+  "tz" : "America/Winnipeg",
+  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12219269484, -2602258284 ],
+  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, -1716, 0 ]
+}, {
+  "tz" : "America/Yakutat",
+  "switches" : [ -62135649665, -59006414465, -55850740865, -52695067265, -46383633665, -43227960065, -40072286465, -33760852865, -30605179265, -27449505665, -21138072065, -17982398465, -14826724865, -12219345665, -3225223727, -2208988800 ],
+  "diffs" : [ -87535, -1135, 85265, 171665, 258065, 344465, 430865, 517265, 603665, 690065, 776465, 862865, 949265, 85265, -1135, 0 ]
+}, {
+  "tz" : "America/Yellowknife",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 25200, 0 ]
+}, {
+  "tz" : "Antarctica/Casey",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -201600, -115200, -28800, 57600, 144000, 230400, 316800, 403200, 489600, 576000, 662400, 748800, 835200, -28800, 0 ]
+}, {
+  "tz" : "Antarctica/Davis",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -198000, -111600, -25200, 61200, 147600, 234000, 320400, 406800, 493200, 579600, 666000, 752400, 838800, -25200, 0 ]
+}, {
+  "tz" : "Antarctica/DumontDUrville",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -208800, -122400, -36000, 50400, 136800, 223200, 309600, 396000, 482400, 568800, 655200, 741600, 828000, -36000, 0 ]
+}, {
+  "tz" : "Antarctica/Macquarie",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2214259200, -2208988800 ],
+  "diffs" : [ -212400, -126000, -39600, 46800, 133200, 219600, 306000, 392400, 478800, 565200, 651600, 738000, 824400, -39600, -3600, 0 ]
+}, {
+  "tz" : "Antarctica/Mawson",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, -18000, 0 ]
+}, {
+  "tz" : "Antarctica/McMurdo",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+}, {
+  "tz" : "Antarctica/Palmer",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 10800, 0 ]
+}, {
+  "tz" : "Antarctica/Rothera",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 10800, 0 ]
+}, {
+  "tz" : "Antarctica/South_Pole",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+}, {
+  "tz" : "Antarctica/Syowa",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -183600, -97200, -10800, 75600, 162000, 248400, 334800, 421200, 507600, 594000, 680400, 766800, 853200, -10800, 0 ]
+}, {
+  "tz" : "Antarctica/Troll",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Antarctica/Vostok",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -194400, -108000, -21600, 64800, 151200, 237600, 324000, 410400, 496800, 583200, 669600, 756000, 842400, -21600, 0 ]
+}, {
+  "tz" : "Arctic/Longyearbyen",
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, -1020, 0 ]
+}, {
+  "tz" : "Asia/Aden",
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 412, 0 ]
+}, {
+  "tz" : "Asia/Almaty",
+  "switches" : [ -62135615268, -59006380068, -55850706468, -52695032868, -46383599268, -43227925668, -40072252068, -33760818468, -30605144868, -27449471268, -21138037668, -17982364068, -14826690468, -12219311268, -2208988800 ],
+  "diffs" : [ -175932, -89532, -3132, 83268, 169668, 256068, 342468, 428868, 515268, 601668, 688068, 774468, 860868, -3132, 0 ]
+}, {
+  "tz" : "Asia/Amman",
+  "switches" : [ -62135605424, -59006370224, -55850696624, -52695023024, -46383589424, -43227915824, -40072242224, -33760808624, -30605135024, -27449461424, -21138027824, -17982354224, -14826680624, -12219301424, -2208988800 ],
+  "diffs" : [ -171376, -84976, 1424, 87824, 174224, 260624, 347024, 433424, 519824, 606224, 692624, 779024, 865424, 1424, 0 ]
+}, {
+  "tz" : "Asia/Anadyr",
+  "switches" : [ -62135639396, -59006404196, -55850730596, -52695056996, -46383623396, -43227949796, -40072276196, -33760842596, -30605168996, -27449495396, -21138061796, -17982388196, -14826714596, -12219335396, -2208988800 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+}, {
+  "tz" : "Asia/Aqtau",
+  "switches" : [ -62135608864, -59006373664, -55850700064, -52695026464, -46383592864, -43227919264, -40072245664, -33760812064, -30605138464, -27449464864, -21138031264, -17982357664, -14826684064, -12219304864, -2208988800 ],
+  "diffs" : [ -178736, -92336, -5936, 80464, 166864, 253264, 339664, 426064, 512464, 598864, 685264, 771664, 858064, -5936, 0 ]
+}, {
+  "tz" : "Asia/Aqtobe",
+  "switches" : [ -62135610520, -59006375320, -55850701720, -52695028120, -46383594520, -43227920920, -40072247320, -33760813720, -30605140120, -27449466520, -21138032920, -17982359320, -14826685720, -12219306520, -2208988800 ],
+  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, -4280, 0 ]
+}, {
+  "tz" : "Asia/Ashgabat",
+  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12219306812, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, 0 ]
+}, {
+  "tz" : "Asia/Ashkhabad",
+  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12219306812, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, 0 ]
+}, {
+  "tz" : "Asia/Atyrau",
+  "switches" : [ -62135609264, -59006374064, -55850700464, -52695026864, -46383593264, -43227919664, -40072246064, -33760812464, -30605138864, -27449465264, -21138031664, -17982358064, -14826684464, -12219305264, -2208988800 ],
+  "diffs" : [ -178336, -91936, -5536, 80864, 167264, 253664, 340064, 426464, 512864, 599264, 685664, 772064, 858464, -5536, 0 ]
+}, {
+  "tz" : "Asia/Baghdad",
+  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12219303460, -2524532260, -2208988800 ],
+  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, -140, -144, 0 ]
+}, {
+  "tz" : "Asia/Bahrain",
+  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12219305168, -2208988800 ],
+  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 1568, 0 ]
+}, {
+  "tz" : "Asia/Baku",
+  "switches" : [ -62135608764, -59006373564, -55850699964, -52695026364, -46383592764, -43227919164, -40072245564, -33760811964, -30605138364, -27449464764, -21138031164, -17982357564, -14826683964, -12219304764, -2208988800 ],
+  "diffs" : [ -175236, -88836, -2436, 83964, 170364, 256764, 343164, 429564, 515964, 602364, 688764, 775164, 861564, -2436, 0 ]
+}, {
+  "tz" : "Asia/Bangkok",
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, -1076, 0 ]
+}, {
+  "tz" : "Asia/Barnaul",
+  "switches" : [ -62135616900, -59006381700, -55850708100, -52695034500, -46383600900, -43227927300, -40072253700, -33760820100, -30605146500, -27449472900, -21138039300, -17982365700, -14826692100, -12219312900, -2208988800 ],
+  "diffs" : [ -177900, -91500, -5100, 81300, 167700, 254100, 340500, 426900, 513300, 599700, 686100, 772500, 858900, -5100, 0 ]
+}, {
+  "tz" : "Asia/Beirut",
+  "switches" : [ -62135605320, -59006370120, -55850696520, -52695022920, -46383589320, -43227915720, -40072242120, -33760808520, -30605134920, -27449461320, -21138027720, -17982354120, -14826680520, -12219301320, -2840149320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Asia/Bishkek",
+  "switches" : [ -62135614704, -59006379504, -55850705904, -52695032304, -46383598704, -43227925104, -40072251504, -33760817904, -30605144304, -27449470704, -21138037104, -17982363504, -14826689904, -12219310704, -2208988800 ],
+  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, -3696, 0 ]
+}, {
+  "tz" : "Asia/Brunei",
+  "switches" : [ -62135624380, -59006389180, -55850715580, -52695041980, -46383608380, -43227934780, -40072261180, -33760827580, -30605153980, -27449480380, -21138046780, -17982373180, -14826699580, -12219320380, -2208988800 ],
+  "diffs" : [ -174020, -87620, -1220, 85180, 171580, 257980, 344380, 430780, 517180, 603580, 689980, 776380, 862780, -1220, 0 ]
+}, {
+  "tz" : "Asia/Calcutta",
+  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12219314008, -3645237208, -3155694800, -2208988800 ],
+  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 1408, 1400, -530, 0 ]
+}, {
+  "tz" : "Asia/Chita",
+  "switches" : [ -62135624032, -59006388832, -55850715232, -52695041632, -46383608032, -43227934432, -40072260832, -33760827232, -30605153632, -27449480032, -21138046432, -17982372832, -14826699232, -12219320032, -2208988800 ],
+  "diffs" : [ -177968, -91568, -5168, 81232, 167632, 254032, 340432, 426832, 513232, 599632, 686032, 772432, 858832, -5168, 0 ]
+}, {
+  "tz" : "Asia/Choibalsan",
+  "switches" : [ -62135624280, -59006389080, -55850715480, -52695041880, -46383608280, -43227934680, -40072261080, -33760827480, -30605153880, -27449480280, -21138046680, -17982373080, -14826699480, -12219320280, -2208988800 ],
+  "diffs" : [ -174120, -87720, -1320, 85080, 171480, 257880, 344280, 430680, 517080, 603480, 689880, 776280, 862680, -1320, 0 ]
+}, {
+  "tz" : "Asia/Chongqing",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+}, {
+  "tz" : "Asia/Chungking",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+}, {
+  "tz" : "Asia/Colombo",
+  "switches" : [ -62135615964, -59006380764, -55850707164, -52695033564, -46383599964, -43227926364, -40072252764, -33760819164, -30605145564, -27449471964, -21138038364, -17982364764, -14826691164, -12219311964, -2840159964, -2208988800 ],
+  "diffs" : [ -173436, -87036, -636, 85764, 172164, 258564, 344964, 431364, 517764, 604164, 690564, 776964, 863364, -636, -628, 0 ]
+}, {
+  "tz" : "Asia/Dacca",
+  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12219314500, -2524543300, -2208988800 ],
+  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 100, -400, 0 ]
+}, {
+  "tz" : "Asia/Damascus",
+  "switches" : [ -62135605512, -59006370312, -55850696712, -52695023112, -46383589512, -43227915912, -40072242312, -33760808712, -30605135112, -27449461512, -21138027912, -17982354312, -14826680712, -12219301512, -2208988800 ],
+  "diffs" : [ -171288, -84888, 1512, 87912, 174312, 260712, 347112, 433512, 519912, 606312, 692712, 779112, 865512, 1512, 0 ]
+}, {
+  "tz" : "Asia/Dhaka",
+  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12219314500, -2524543300, -2208988800 ],
+  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 100, -400, 0 ]
+}, {
+  "tz" : "Asia/Dili",
+  "switches" : [ -62135626940, -59006391740, -55850718140, -52695044540, -46383610940, -43227937340, -40072263740, -33760830140, -30605156540, -27449482940, -21138049340, -17982375740, -14826702140, -12219322940, -2208988800 ],
+  "diffs" : [ -175060, -88660, -2260, 84140, 170540, 256940, 343340, 429740, 516140, 602540, 688940, 775340, 861740, -2260, 0 ]
+}, {
+  "tz" : "Asia/Dubai",
+  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12219306072, -2208988800 ],
+  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, -1128, 0 ]
+}, {
+  "tz" : "Asia/Dushanbe",
+  "switches" : [ -62135613312, -59006378112, -55850704512, -52695030912, -46383597312, -43227923712, -40072250112, -33760816512, -30605142912, -27449469312, -21138035712, -17982362112, -14826688512, -12219309312, -2208988800 ],
+  "diffs" : [ -174288, -87888, -1488, 84912, 171312, 257712, 344112, 430512, 516912, 603312, 689712, 776112, 862512, -1488, 0 ]
+}, {
+  "tz" : "Asia/Famagusta",
+  "switches" : [ -62135604948, -59006369748, -55850696148, -52695022548, -46383588948, -43227915348, -40072241748, -33760808148, -30605134548, -27449460948, -21138027348, -17982353748, -14826680148, -12219300948, -2208988800 ],
+  "diffs" : [ -171852, -85452, 948, 87348, 173748, 260148, 346548, 432948, 519348, 605748, 692148, 778548, 864948, 948, 0 ]
+}, {
+  "tz" : "Asia/Gaza",
+  "switches" : [ -62135605072, -59006369872, -55850696272, -52695022672, -46383589072, -43227915472, -40072241872, -33760808272, -30605134672, -27449461072, -21138027472, -17982353872, -14826680272, -12219301072, -2208988800 ],
+  "diffs" : [ -171728, -85328, 1072, 87472, 173872, 260272, 346672, 433072, 519472, 605872, 692272, 778672, 865072, 1072, 0 ]
+}, {
+  "tz" : "Asia/Harbin",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+}, {
+  "tz" : "Asia/Hebron",
+  "switches" : [ -62135605223, -59006370023, -55850696423, -52695022823, -46383589223, -43227915623, -40072242023, -33760808423, -30605134823, -27449461223, -21138027623, -17982354023, -14826680423, -12219301223, -2208988800 ],
+  "diffs" : [ -171577, -85177, 1223, 87623, 174023, 260423, 346823, 433223, 519623, 606023, 692423, 778823, 865223, 1223, 0 ]
+}, {
+  "tz" : "Asia/Ho_Chi_Minh",
+  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12219318400, -2208988800 ],
+  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 400, 0 ]
+}, {
+  "tz" : "Asia/Hong_Kong",
+  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12219320202, -2208988800 ],
+  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, -1398, 0 ]
+}, {
+  "tz" : "Asia/Hovd",
+  "switches" : [ -62135618796, -59006383596, -55850709996, -52695036396, -46383602796, -43227929196, -40072255596, -33760821996, -30605148396, -27449474796, -21138041196, -17982367596, -14826693996, -12219314796, -2208988800 ],
+  "diffs" : [ -176004, -89604, -3204, 83196, 169596, 255996, 342396, 428796, 515196, 601596, 687996, 774396, 860796, -3204, 0 ]
+}, {
+  "tz" : "Asia/Irkutsk",
+  "switches" : [ -62135621825, -59006386625, -55850713025, -52695039425, -46383605825, -43227932225, -40072258625, -33760825025, -30605151425, -27449477825, -21138044225, -17982370625, -14826697025, -12219317825, -2208988800 ],
+  "diffs" : [ -176575, -90175, -3775, 82625, 169025, 255425, 341825, 428225, 514625, 601025, 687425, 773825, 860225, -3775, 0 ]
+}, {
+  "tz" : "Asia/Istanbul",
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, -3848, -3784, 0 ]
+}, {
+  "tz" : "Asia/Jakarta",
+  "switches" : [ -62135622432, -59006387232, -55850713632, -52695040032, -46383606432, -43227932832, -40072259232, -33760825632, -30605152032, -27449478432, -21138044832, -17982371232, -14826697632, -12219318432, -2208988800 ],
+  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 432, 0 ]
+}, {
+  "tz" : "Asia/Jayapura",
+  "switches" : [ -62135630568, -59006395368, -55850721768, -52695048168, -46383614568, -43227940968, -40072267368, -33760833768, -30605160168, -27449486568, -21138052968, -17982379368, -14826705768, -12219326568, -2208988800 ],
+  "diffs" : [ -171432, -85032, 1368, 87768, 174168, 260568, 346968, 433368, 519768, 606168, 692568, 778968, 865368, 1368, 0 ]
+}, {
+  "tz" : "Asia/Jerusalem",
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 1254, 1240, 0 ]
+}, {
+  "tz" : "Asia/Kabul",
+  "switches" : [ -62135613408, -59006378208, -55850704608, -52695031008, -46383597408, -43227923808, -40072250208, -33760816608, -30605143008, -27449469408, -21138035808, -17982362208, -14826688608, -12219309408, -2524538208, -2208988800 ],
+  "diffs" : [ -172392, -85992, 408, 86808, 173208, 259608, 346008, 432408, 518808, 605208, 691608, 778008, 864408, 408, -1800, 0 ]
+}, {
+  "tz" : "Asia/Kamchatka",
+  "switches" : [ -62135634876, -59006399676, -55850726076, -52695052476, -46383618876, -43227945276, -40072271676, -33760838076, -30605164476, -27449490876, -21138057276, -17982383676, -14826710076, -12219330876, -2208988800 ],
+  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, -5124, 0 ]
+}, {
+  "tz" : "Asia/Karachi",
+  "switches" : [ -62135612892, -59006377692, -55850704092, -52695030492, -46383596892, -43227923292, -40072249692, -33760816092, -30605142492, -27449468892, -21138035292, -17982361692, -14826688092, -12219308892, -2208988800 ],
+  "diffs" : [ -174708, -88308, -1908, 84492, 170892, 257292, 343692, 430092, 516492, 602892, 689292, 775692, 862092, -1908, 0 ]
+}, {
+  "tz" : "Asia/Kashgar",
+  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12219313820, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+}, {
+  "tz" : "Asia/Kathmandu",
+  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12219313276, -2208988800 ],
+  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, -224, 0 ]
+}, {
+  "tz" : "Asia/Katmandu",
+  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12219313276, -2208988800 ],
+  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, -224, 0 ]
+}, {
+  "tz" : "Asia/Khandyga",
+  "switches" : [ -62135629333, -59006394133, -55850720533, -52695046933, -46383613333, -43227939733, -40072266133, -33760832533, -30605158933, -27449485333, -21138051733, -17982378133, -14826704533, -12219325333, -2208988800 ],
+  "diffs" : [ -172667, -86267, 133, 86533, 172933, 259333, 345733, 432133, 518533, 604933, 691333, 777733, 864133, 133, 0 ]
+}, {
+  "tz" : "Asia/Kolkata",
+  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12219314008, -3645237208, -3155694800, -2208988800 ],
+  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 1408, 1400, -530, 0 ]
+}, {
+  "tz" : "Asia/Krasnoyarsk",
+  "switches" : [ -62135619086, -59006383886, -55850710286, -52695036686, -46383603086, -43227929486, -40072255886, -33760822286, -30605148686, -27449475086, -21138041486, -17982367886, -14826694286, -12219315086, -2208988800 ],
+  "diffs" : [ -175714, -89314, -2914, 83486, 169886, 256286, 342686, 429086, 515486, 601886, 688286, 774686, 861086, -2914, 0 ]
+}, {
+  "tz" : "Asia/Kuala_Lumpur",
+  "switches" : [ -62135621206, -59006386006, -55850712406, -52695038806, -46383605206, -43227931606, -40072258006, -33760824406, -30605150806, -27449477206, -21138043606, -17982370006, -14826696406, -12219317206, -2208988800 ],
+  "diffs" : [ -177194, -90794, -4394, 82006, 168406, 254806, 341206, 427606, 514006, 600406, 686806, 773206, 859606, -4394, 0 ]
+}, {
+  "tz" : "Asia/Kuching",
+  "switches" : [ -62135623280, -59006388080, -55850714480, -52695040880, -46383607280, -43227933680, -40072260080, -33760826480, -30605152880, -27449479280, -21138045680, -17982372080, -14826698480, -12219319280, -2208988800 ],
+  "diffs" : [ -175120, -88720, -2320, 84080, 170480, 256880, 343280, 429680, 516080, 602480, 688880, 775280, 861680, -2320, 0 ]
+}, {
+  "tz" : "Asia/Kuwait",
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 412, 0 ]
+}, {
+  "tz" : "Asia/Macao",
+  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12219320050, -2208988800 ],
+  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, -1550, 0 ]
+}, {
+  "tz" : "Asia/Macau",
+  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12219320050, -2208988800 ],
+  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, -1550, 0 ]
+}, {
+  "tz" : "Asia/Magadan",
+  "switches" : [ -62135632992, -59006397792, -55850724192, -52695050592, -46383616992, -43227943392, -40072269792, -33760836192, -30605162592, -27449488992, -21138055392, -17982381792, -14826708192, -12219328992, -2208988800 ],
+  "diffs" : [ -176208, -89808, -3408, 82992, 169392, 255792, 342192, 428592, 514992, 601392, 687792, 774192, 860592, -3408, 0 ]
+}, {
+  "tz" : "Asia/Makassar",
+  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12219321456, -2208988800 ],
+  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, -144, 0 ]
+}, {
+  "tz" : "Asia/Manila",
+  "switches" : [ -62135539440, -59006304240, -55850630640, -52694957040, -46383523440, -43227849840, -40072176240, -33760742640, -30605069040, -27449395440, -21137961840, -17982288240, -14826614640, -12219235440, -3944621040, -2229321840 ],
+  "diffs" : [ -258960, -172560, -86160, 240, 86640, 173040, 259440, 345840, 432240, 518640, 605040, 691440, 777840, -86160, 240, 0 ]
+}, {
+  "tz" : "Asia/Muscat",
+  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12219306072, -2208988800 ],
+  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, -1128, 0 ]
+}, {
+  "tz" : "Asia/Nicosia",
+  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12219300808, -2208988800 ],
+  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 808, 0 ]
+}, {
+  "tz" : "Asia/Novokuznetsk",
+  "switches" : [ -62135617728, -59006382528, -55850708928, -52695035328, -46383601728, -43227928128, -40072254528, -33760820928, -30605147328, -27449473728, -21138040128, -17982366528, -14826692928, -12219313728, -2208988800 ],
+  "diffs" : [ -177072, -90672, -4272, 82128, 168528, 254928, 341328, 427728, 514128, 600528, 686928, 773328, 859728, -4272, 0 ]
+}, {
+  "tz" : "Asia/Novosibirsk",
+  "switches" : [ -62135616700, -59006381500, -55850707900, -52695034300, -46383600700, -43227927100, -40072253500, -33760819900, -30605146300, -27449472700, -21138039100, -17982365500, -14826691900, -12219312700, -2208988800 ],
+  "diffs" : [ -178100, -91700, -5300, 81100, 167500, 253900, 340300, 426700, 513100, 599500, 685900, 772300, 858700, -5300, 0 ]
+}, {
+  "tz" : "Asia/Omsk",
+  "switches" : [ -62135614410, -59006379210, -55850705610, -52695032010, -46383598410, -43227924810, -40072251210, -33760817610, -30605144010, -27449470410, -21138036810, -17982363210, -14826689610, -12219310410, -2208988800 ],
+  "diffs" : [ -176790, -90390, -3990, 82410, 168810, 255210, 341610, 428010, 514410, 600810, 687210, 773610, 860010, -3990, 0 ]
+}, {
+  "tz" : "Asia/Oral",
+  "switches" : [ -62135609124, -59006373924, -55850700324, -52695026724, -46383593124, -43227919524, -40072245924, -33760812324, -30605138724, -27449465124, -21138031524, -17982357924, -14826684324, -12219305124, -2208988800 ],
+  "diffs" : [ -178476, -92076, -5676, 80724, 167124, 253524, 339924, 426324, 512724, 599124, 685524, 771924, 858324, -5676, 0 ]
+}, {
+  "tz" : "Asia/Phnom_Penh",
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, -1076, 0 ]
+}, {
+  "tz" : "Asia/Pontianak",
+  "switches" : [ -62135623040, -59006387840, -55850714240, -52695040640, -46383607040, -43227933440, -40072259840, -33760826240, -30605152640, -27449479040, -21138045440, -17982371840, -14826698240, -12219319040, -2208988800 ],
+  "diffs" : [ -171760, -85360, 1040, 87440, 173840, 260240, 346640, 433040, 519440, 605840, 692240, 778640, 865040, 1040, 0 ]
+}, {
+  "tz" : "Asia/Pyongyang",
+  "switches" : [ -62135626980, -59006391780, -55850718180, -52695044580, -46383610980, -43227937380, -40072263780, -33760830180, -30605156580, -27449482980, -21138049380, -17982375780, -14826702180, -12219322980, -2208988800 ],
+  "diffs" : [ -175020, -88620, -2220, 84180, 170580, 256980, 343380, 429780, 516180, 602580, 688980, 775380, 861780, -2220, 0 ]
+}, {
+  "tz" : "Asia/Qatar",
+  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12219305168, -2208988800 ],
+  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 1568, 0 ]
+}, {
+  "tz" : "Asia/Qostanay",
+  "switches" : [ -62135612068, -59006376868, -55850703268, -52695029668, -46383596068, -43227922468, -40072248868, -33760815268, -30605141668, -27449468068, -21138034468, -17982360868, -14826687268, -12219308068, -2208988800 ],
+  "diffs" : [ -179132, -92732, -6332, 80068, 166468, 252868, 339268, 425668, 512068, 598468, 684868, 771268, 857668, -6332, 0 ]
+}, {
+  "tz" : "Asia/Qyzylorda",
+  "switches" : [ -62135612512, -59006377312, -55850703712, -52695030112, -46383596512, -43227922912, -40072249312, -33760815712, -30605142112, -27449468512, -21138034912, -17982361312, -14826687712, -12219308512, -2208988800 ],
+  "diffs" : [ -175088, -88688, -2288, 84112, 170512, 256912, 343312, 429712, 516112, 602512, 688912, 775312, 861712, -2288, 0 ]
+}, {
+  "tz" : "Asia/Rangoon",
+  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12219315887, -2208988800 ],
+  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, -313, 0 ]
+}, {
+  "tz" : "Asia/Riyadh",
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 412, 0 ]
+}, {
+  "tz" : "Asia/Saigon",
+  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12219318400, -2208988800 ],
+  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 400, 0 ]
+}, {
+  "tz" : "Asia/Sakhalin",
+  "switches" : [ -62135631048, -59006395848, -55850722248, -52695048648, -46383615048, -43227941448, -40072267848, -33760834248, -30605160648, -27449487048, -21138053448, -17982379848, -14826706248, -12219327048, -2208988800 ],
+  "diffs" : [ -178152, -91752, -5352, 81048, 167448, 253848, 340248, 426648, 513048, 599448, 685848, 772248, 858648, -5352, 0 ]
+}, {
+  "tz" : "Asia/Samarkand",
+  "switches" : [ -62135612873, -59006377673, -55850704073, -52695030473, -46383596873, -43227923273, -40072249673, -33760816073, -30605142473, -27449468873, -21138035273, -17982361673, -14826688073, -12219308873, -2208988800 ],
+  "diffs" : [ -174727, -88327, -1927, 84473, 170873, 257273, 343673, 430073, 516473, 602873, 689273, 775673, 862073, -1927, 0 ]
+}, {
+  "tz" : "Asia/Seoul",
+  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12219323272, -2208988800 ],
+  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, -1928, 0 ]
+}, {
+  "tz" : "Asia/Shanghai",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+}, {
+  "tz" : "Asia/Singapore",
+  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12219317725, -2208988800 ],
+  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, -3875, 0 ]
+}, {
+  "tz" : "Asia/Srednekolymsk",
+  "switches" : [ -62135633692, -59006398492, -55850724892, -52695051292, -46383617692, -43227944092, -40072270492, -33760836892, -30605163292, -27449489692, -21138056092, -17982382492, -14826708892, -12219329692, -2208988800 ],
+  "diffs" : [ -175508, -89108, -2708, 83692, 170092, 256492, 342892, 429292, 515692, 602092, 688492, 774892, 861292, -2708, 0 ]
+}, {
+  "tz" : "Asia/Taipei",
+  "switches" : [ -62135625960, -59006390760, -55850717160, -52695043560, -46383609960, -43227936360, -40072262760, -33760829160, -30605155560, -27449481960, -21138048360, -17982374760, -14826701160, -12219321960, -2335248360 ],
+  "diffs" : [ -172440, -86040, 360, 86760, 173160, 259560, 345960, 432360, 518760, 605160, 691560, 777960, 864360, 360, 0 ]
+}, {
+  "tz" : "Asia/Tashkent",
+  "switches" : [ -62135613431, -59006378231, -55850704631, -52695031031, -46383597431, -43227923831, -40072250231, -33760816631, -30605143031, -27449469431, -21138035831, -17982362231, -14826688631, -12219309431, -2208988800 ],
+  "diffs" : [ -174169, -87769, -1369, 85031, 171431, 257831, 344231, 430631, 517031, 603431, 689831, 776231, 862631, -1369, 0 ]
+}, {
+  "tz" : "Asia/Tbilisi",
+  "switches" : [ -62135607551, -59006372351, -55850698751, -52695025151, -46383591551, -43227917951, -40072244351, -33760810751, -30605137151, -27449463551, -21138029951, -17982356351, -14826682751, -12219303551, -2208988800 ],
+  "diffs" : [ -176449, -90049, -3649, 82751, 169151, 255551, 341951, 428351, 514751, 601151, 687551, 773951, 860351, -3649, 0 ]
+}, {
+  "tz" : "Asia/Tel_Aviv",
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 1254, 1240, 0 ]
+}, {
+  "tz" : "Asia/Thimbu",
+  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12219314316, -2208988800 ],
+  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, -84, 0 ]
+}, {
+  "tz" : "Asia/Thimphu",
+  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12219314316, -2208988800 ],
+  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, -84, 0 ]
+}, {
+  "tz" : "Asia/Tokyo",
+  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12219326339, -2587712400 ],
+  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 1139, 0 ]
+}, {
+  "tz" : "Asia/Tomsk",
+  "switches" : [ -62135617191, -59006381991, -55850708391, -52695034791, -46383601191, -43227927591, -40072253991, -33760820391, -30605146791, -27449473191, -21138039591, -17982365991, -14826692391, -12219313191, -2208988800 ],
+  "diffs" : [ -177609, -91209, -4809, 81591, 167991, 254391, 340791, 427191, 513591, 599991, 686391, 772791, 859191, -4809, 0 ]
+}, {
+  "tz" : "Asia/Ujung_Pandang",
+  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12219321456, -2208988800 ],
+  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, -144, 0 ]
+}, {
+  "tz" : "Asia/Ulaanbaatar",
+  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12219318452, -2208988800 ],
+  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, -3148, 0 ]
+}, {
+  "tz" : "Asia/Ulan_Bator",
+  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12219318452, -2208988800 ],
+  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, -3148, 0 ]
+}, {
+  "tz" : "Asia/Urumqi",
+  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12219313820, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+}, {
+  "tz" : "Asia/Ust-Nera",
+  "switches" : [ -62135631174, -59006395974, -55850722374, -52695048774, -46383615174, -43227941574, -40072267974, -33760834374, -30605160774, -27449487174, -21138053574, -17982379974, -14826706374, -12219327174, -2208988800 ],
+  "diffs" : [ -174426, -88026, -1626, 84774, 171174, 257574, 343974, 430374, 516774, 603174, 689574, 775974, 862374, -1626, 0 ]
+}, {
+  "tz" : "Asia/Vientiane",
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, -1076, 0 ]
+}, {
+  "tz" : "Asia/Vladivostok",
+  "switches" : [ -62135628451, -59006393251, -55850719651, -52695046051, -46383612451, -43227938851, -40072265251, -33760831651, -30605158051, -27449484451, -21138050851, -17982377251, -14826703651, -12219324451, -2208988800 ],
+  "diffs" : [ -177149, -90749, -4349, 82051, 168451, 254851, 341251, 427651, 514051, 600451, 686851, 773251, 859651, -4349, 0 ]
+}, {
+  "tz" : "Asia/Yakutsk",
+  "switches" : [ -62135627938, -59006392738, -55850719138, -52695045538, -46383611938, -43227938338, -40072264738, -33760831138, -30605157538, -27449483938, -21138050338, -17982376738, -14826703138, -12219323938, -2208988800 ],
+  "diffs" : [ -174062, -87662, -1262, 85138, 171538, 257938, 344338, 430738, 517138, 603538, 689938, 776338, 862738, -1262, 0 ]
+}, {
+  "tz" : "Asia/Yangon",
+  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12219315887, -2208988800 ],
+  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, -313, 0 ]
+}, {
+  "tz" : "Asia/Yekaterinburg",
+  "switches" : [ -62135611353, -59006376153, -55850702553, -52695028953, -46383595353, -43227921753, -40072248153, -33760814553, -30605140953, -27449467353, -21138033753, -17982360153, -14826686553, -12219307353, -2208988800 ],
+  "diffs" : [ -176247, -89847, -3447, 82953, 169353, 255753, 342153, 428553, 514953, 601353, 687753, 774153, 860553, -3447, 0 ]
+}, {
+  "tz" : "Asia/Yerevan",
+  "switches" : [ -62135607480, -59006372280, -55850698680, -52695025080, -46383591480, -43227917880, -40072244280, -33760810680, -30605137080, -27449463480, -21138029880, -17982356280, -14826682680, -12219303480, -2208988800 ],
+  "diffs" : [ -176520, -90120, -3720, 82680, 169080, 255480, 341880, 428280, 514680, 601080, 687480, 773880, 860280, -3720, 0 ]
+}, {
+  "tz" : "Atlantic/Azores",
+  "switches" : [ -62135590640, -59006355440, -55850681840, -52695008240, -46383574640, -43227901040, -40072227440, -33760793840, -30605120240, -27449446640, -21138013040, -17982339440, -14826665840, -12219286640, -2713904240, -2208988800 ],
+  "diffs" : [ -175360, -88960, -2560, 83840, 170240, 256640, 343040, 429440, 515840, 602240, 688640, 775040, 861440, -2560, -3272, 0 ]
+}, {
+  "tz" : "Atlantic/Bermuda",
+  "switches" : [ -62135581242, -59006346042, -55850672442, -52694998842, -46383565242, -43227891642, -40072218042, -33760784442, -30605110842, -27449437242, -21138003642, -17982330042, -14826656442, -12219277242, -2208988800 ],
+  "diffs" : [ -173958, -87558, -1158, 85242, 171642, 258042, 344442, 430842, 517242, 603642, 690042, 776442, 862842, -1158, 0 ]
+}, {
+  "tz" : "Atlantic/Canary",
+  "switches" : [ -62135593104, -59006357904, -55850684304, -52695010704, -46383577104, -43227903504, -40072229904, -33760796304, -30605122704, -27449449104, -21138015504, -17982341904, -14826668304, -12219289104, -2208988800 ],
+  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, -3696, 0 ]
+}, {
+  "tz" : "Atlantic/Cape_Verde",
+  "switches" : [ -62135591156, -59006355956, -55850682356, -52695008756, -46383575156, -43227901556, -40072227956, -33760794356, -30605120756, -27449447156, -21138013556, -17982339956, -14826666356, -12219287156, -2208988800 ],
+  "diffs" : [ -174844, -88444, -2044, 84356, 170756, 257156, 343556, 429956, 516356, 602756, 689156, 775556, 861956, -2044, 0 ]
+}, {
+  "tz" : "Atlantic/Faeroe",
+  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12219291176, -2208988800 ],
+  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, -1624, 0 ]
+}, {
+  "tz" : "Atlantic/Faroe",
+  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12219291176, -2208988800 ],
+  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, -1624, 0 ]
+}, {
+  "tz" : "Atlantic/Jan_Mayen",
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, -1020, 0 ]
+}, {
+  "tz" : "Atlantic/Madeira",
+  "switches" : [ -62135592744, -59006357544, -55850683944, -52695010344, -46383576744, -43227903144, -40072229544, -33760795944, -30605122344, -27449448744, -21138015144, -17982341544, -14826667944, -12219288744, -2208988800 ],
+  "diffs" : [ -176856, -90456, -4056, 82344, 168744, 255144, 341544, 427944, 514344, 600744, 687144, 773544, 859944, -4056, 0 ]
+}, {
+  "tz" : "Atlantic/Reykjavik",
+  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12219287520, -2208988800 ],
+  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, -5280, 0 ]
+}, {
+  "tz" : "Atlantic/South_Georgia",
+  "switches" : [ -62135588032, -59006352832, -55850679232, -52695005632, -46383572032, -43227898432, -40072224832, -33760791232, -30605117632, -27449444032, -21138010432, -17982336832, -14826663232, -12219284032, -2524512832 ],
+  "diffs" : [ -174368, -87968, -1568, 84832, 171232, 257632, 344032, 430432, 516832, 603232, 689632, 776032, 862432, -1568, 0 ]
+}, {
+  "tz" : "Atlantic/St_Helena",
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+}, {
+  "tz" : "Atlantic/Stanley",
+  "switches" : [ -62135582916, -59006347716, -55850674116, -52695000516, -46383566916, -43227893316, -40072219716, -33760786116, -30605112516, -27449438916, -21138005316, -17982331716, -14826658116, -12219278916, -2208988800 ],
+  "diffs" : [ -175884, -89484, -3084, 83316, 169716, 256116, 342516, 428916, 515316, 601716, 688116, 774516, 860916, -3084, 0 ]
+}, {
+  "tz" : "Australia/ACT",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+}, {
+  "tz" : "Australia/Adelaide",
+  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12219326060, -2364110060, -2230189200 ],
+  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, -940, -1800, 0 ]
+}, {
+  "tz" : "Australia/Brisbane",
+  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12219329528, -2366791928 ],
+  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 728, 0 ]
+}, {
+  "tz" : "Australia/Broken_Hill",
+  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12219326748, -2364110748, -2314951200, -2230189200 ],
+  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, -252, 1800, -1800, 0 ]
+}, {
+  "tz" : "Australia/Canberra",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+}, {
+  "tz" : "Australia/Currie",
+  "switches" : [ -62135631328, -59006396128, -55850722528, -52695048928, -46383615328, -43227941728, -40072268128, -33760834528, -30605160928, -27449487328, -21138053728, -17982380128, -14826706528, -12219327328, -2345794528 ],
+  "diffs" : [ -174272, -87872, -1472, 84928, 171328, 257728, 344128, 430528, 516928, 603328, 689728, 776128, 862528, -1472, 0 ]
+}, {
+  "tz" : "Australia/Darwin",
+  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12219324200, -2364108200, -2230189200 ],
+  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, -2800, -1800, 0 ]
+}, {
+  "tz" : "Australia/Eucla",
+  "switches" : [ -62135627728, -59006392528, -55850718928, -52695045328, -46383611728, -43227938128, -40072264528, -33760830928, -30605157328, -27449483728, -21138050128, -17982376528, -14826702928, -12219323728, -2337928528 ],
+  "diffs" : [ -173372, -86972, -572, 85828, 172228, 258628, 345028, 431428, 517828, 604228, 690628, 777028, 863428, -572, 0 ]
+}, {
+  "tz" : "Australia/Hobart",
+  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12219328156, -2345795356 ],
+  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, -644, 0 ]
+}, {
+  "tz" : "Australia/LHI",
+  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12219330980, -2364114980, -2208988800 ],
+  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 380, -1800, 0 ]
+}, {
+  "tz" : "Australia/Lindeman",
+  "switches" : [ -62135632556, -59006397356, -55850723756, -52695050156, -46383616556, -43227942956, -40072269356, -33760835756, -30605162156, -27449488556, -21138054956, -17982381356, -14826707756, -12219328556, -2366790956 ],
+  "diffs" : [ -173044, -86644, -244, 86156, 172556, 258956, 345356, 431756, 518156, 604556, 690956, 777356, 863756, -244, 0 ]
+}, {
+  "tz" : "Australia/Lord_Howe",
+  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12219330980, -2364114980, -2208988800 ],
+  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 380, -1800, 0 ]
+}, {
+  "tz" : "Australia/Melbourne",
+  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12219327592, -2364111592 ],
+  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, -1208, 0 ]
+}, {
+  "tz" : "Australia/NSW",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+}, {
+  "tz" : "Australia/North",
+  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12219324200, -2364108200, -2230189200 ],
+  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, -2800, -1800, 0 ]
+}, {
+  "tz" : "Australia/Perth",
+  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12219320604, -2337925404 ],
+  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, -996, 0 ]
+}, {
+  "tz" : "Australia/Queensland",
+  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12219329528, -2366791928 ],
+  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 728, 0 ]
+}, {
+  "tz" : "Australia/South",
+  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12219326060, -2364110060, -2230189200 ],
+  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, -940, -1800, 0 ]
+}, {
+  "tz" : "Australia/Sydney",
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+}, {
+  "tz" : "Australia/Tasmania",
+  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12219328156, -2345795356 ],
+  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, -644, 0 ]
+}, {
+  "tz" : "Australia/Victoria",
+  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12219327592, -2364111592 ],
+  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, -1208, 0 ]
+}, {
+  "tz" : "Australia/West",
+  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12219320604, -2337925404 ],
+  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, -996, 0 ]
+}, {
+  "tz" : "Australia/Yancowinna",
+  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12219326748, -2364110748, -2314951200, -2230189200 ],
+  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, -252, 1800, -1800, 0 ]
+}, {
+  "tz" : "Brazil/Acre",
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 1728, 0 ]
+}, {
+  "tz" : "Brazil/DeNoronha",
+  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12219285020, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+}, {
+  "tz" : "Brazil/East",
+  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12219281612, -2208988800 ],
+  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, -388, 0 ]
+}, {
+  "tz" : "Brazil/West",
+  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12219278396, -2208988800 ],
+  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, -4, 0 ]
+}, {
+  "tz" : "CET",
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "CST6CDT",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Canada/Atlantic",
+  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12219277536, -2208988800 ],
+  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, -864, 0 ]
+}, {
+  "tz" : "Canada/Central",
+  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12219269484, -2602258284 ],
+  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, -1716, 0 ]
+}, {
+  "tz" : "Canada/Eastern",
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, -1052, 0 ]
+}, {
+  "tz" : "Canada/Mountain",
+  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12219265568, -2208988800 ],
+  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, -2032, 0 ]
+}, {
+  "tz" : "Canada/Newfoundland",
+  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12219280148, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, -52, 0 ]
+}, {
+  "tz" : "Canada/Pacific",
+  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12219263252, -2713880852 ],
+  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, -748, 0 ]
+}, {
+  "tz" : "Canada/Saskatchewan",
+  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12219267684, -2208988800 ],
+  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, -3516, 0 ]
+}, {
+  "tz" : "Canada/Yukon",
+  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12219260388, -2208988800 ],
+  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, -3612, 0 ]
+}, {
+  "tz" : "Chile/Continental",
+  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12219275834, -2208988800 ],
+  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, -2566, 0 ]
+}, {
+  "tz" : "Chile/EasterIsland",
+  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12219266552, -2208988800 ],
+  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, -4648, 0 ]
+}, {
+  "tz" : "Cuba",
+  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12219273032, -2524501832, -2208988800 ],
+  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, -1768, -1776, 0 ]
+}, {
+  "tz" : "EET",
+  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12219300000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "EST5EDT",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Egypt",
+  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12219300309, -2208988800 ],
+  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 309, 0 ]
+}, {
+  "tz" : "Eire",
+  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12219291300, -2821649700, -2208988800 ],
+  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, -1500, -1521, 0 ]
+}, {
+  "tz" : "Etc/GMT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+1",
+  "switches" : [ -62135593200, -59006358000, -55850684400, -52695010800, -46383577200, -43227903600, -40072230000, -33760796400, -30605122800, -27449449200, -21138015600, -17982342000, -14826668400, -12219289200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+10",
+  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12219256800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+11",
+  "switches" : [ -62135557200, -59006322000, -55850648400, -52694974800, -46383541200, -43227867600, -40072194000, -33760760400, -30605086800, -27449413200, -21137979600, -17982306000, -14826632400, -12219253200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+12",
+  "switches" : [ -62135553600, -59006318400, -55850644800, -52694971200, -46383537600, -43227864000, -40072190400, -33760756800, -30605083200, -27449409600, -21137976000, -17982302400, -14826628800, -12219249600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+2",
+  "switches" : [ -62135589600, -59006354400, -55850680800, -52695007200, -46383573600, -43227900000, -40072226400, -33760792800, -30605119200, -27449445600, -21138012000, -17982338400, -14826664800, -12219285600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+3",
+  "switches" : [ -62135586000, -59006350800, -55850677200, -52695003600, -46383570000, -43227896400, -40072222800, -33760789200, -30605115600, -27449442000, -21138008400, -17982334800, -14826661200, -12219282000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+4",
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+5",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+6",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+7",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+8",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+9",
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-1",
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-10",
+  "switches" : [ -62135632800, -59006397600, -55850724000, -52695050400, -46383616800, -43227943200, -40072269600, -33760836000, -30605162400, -27449488800, -21138055200, -17982381600, -14826708000, -12219328800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-11",
+  "switches" : [ -62135636400, -59006401200, -55850727600, -52695054000, -46383620400, -43227946800, -40072273200, -33760839600, -30605166000, -27449492400, -21138058800, -17982385200, -14826711600, -12219332400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-12",
+  "switches" : [ -62135640000, -59006404800, -55850731200, -52695057600, -46383624000, -43227950400, -40072276800, -33760843200, -30605169600, -27449496000, -21138062400, -17982388800, -14826715200, -12219336000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-13",
+  "switches" : [ -62135643600, -59006408400, -55850734800, -52695061200, -46383627600, -43227954000, -40072280400, -33760846800, -30605173200, -27449499600, -21138066000, -17982392400, -14826718800, -12219339600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-14",
+  "switches" : [ -62135647200, -59006412000, -55850738400, -52695064800, -46383631200, -43227957600, -40072284000, -33760850400, -30605176800, -27449503200, -21138069600, -17982396000, -14826722400, -12219343200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-2",
+  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12219300000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-3",
+  "switches" : [ -62135607600, -59006372400, -55850698800, -52695025200, -46383591600, -43227918000, -40072244400, -33760810800, -30605137200, -27449463600, -21138030000, -17982356400, -14826682800, -12219303600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-4",
+  "switches" : [ -62135611200, -59006376000, -55850702400, -52695028800, -46383595200, -43227921600, -40072248000, -33760814400, -30605140800, -27449467200, -21138033600, -17982360000, -14826686400, -12219307200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-5",
+  "switches" : [ -62135614800, -59006379600, -55850706000, -52695032400, -46383598800, -43227925200, -40072251600, -33760818000, -30605144400, -27449470800, -21138037200, -17982363600, -14826690000, -12219310800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-6",
+  "switches" : [ -62135618400, -59006383200, -55850709600, -52695036000, -46383602400, -43227928800, -40072255200, -33760821600, -30605148000, -27449474400, -21138040800, -17982367200, -14826693600, -12219314400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-7",
+  "switches" : [ -62135622000, -59006386800, -55850713200, -52695039600, -46383606000, -43227932400, -40072258800, -33760825200, -30605151600, -27449478000, -21138044400, -17982370800, -14826697200, -12219318000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-8",
+  "switches" : [ -62135625600, -59006390400, -55850716800, -52695043200, -46383609600, -43227936000, -40072262400, -33760828800, -30605155200, -27449481600, -21138048000, -17982374400, -14826700800, -12219321600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-9",
+  "switches" : [ -62135629200, -59006394000, -55850720400, -52695046800, -46383613200, -43227939600, -40072266000, -33760832400, -30605158800, -27449485200, -21138051600, -17982378000, -14826704400, -12219325200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/GMT0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/Greenwich",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/UCT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/UTC",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/Universal",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Etc/Zulu",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Europe/Amsterdam",
+  "switches" : [ -62135597972, -59006362772, -55850689172, -52695015572, -46383581972, -43227908372, -40072234772, -33760801172, -30605127572, -27449453972, -21138020372, -17982346772, -14826673172, -12219293972, -2208988800 ],
+  "diffs" : [ -175228, -88828, -2428, 83972, 170372, 256772, 343172, 429572, 515972, 602372, 688772, 775172, 861572, -2428, 0 ]
+}, {
+  "tz" : "Europe/Andorra",
+  "switches" : [ -62135597164, -59006361964, -55850688364, -52695014764, -46383581164, -43227907564, -40072233964, -33760800364, -30605126764, -27449453164, -21138019564, -17982345964, -14826672364, -12219293164, -2208988800 ],
+  "diffs" : [ -176036, -89636, -3236, 83164, 169564, 255964, 342364, 428764, 515164, 601564, 687964, 774364, 860764, -3236, 0 ]
+}, {
+  "tz" : "Europe/Astrakhan",
+  "switches" : [ -62135608332, -59006373132, -55850699532, -52695025932, -46383592332, -43227918732, -40072245132, -33760811532, -30605137932, -27449464332, -21138030732, -17982357132, -14826683532, -12219304332, -2208988800 ],
+  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, -2868, 0 ]
+}, {
+  "tz" : "Europe/Athens",
+  "switches" : [ -62135602492, -59006367292, -55850693692, -52695020092, -46383586492, -43227912892, -40072239292, -33760805692, -30605132092, -27449458492, -21138024892, -17982351292, -14826677692, -12219298492, -2208988800 ],
+  "diffs" : [ -174308, -87908, -1508, 84892, 171292, 257692, 344092, 430492, 516892, 603292, 689692, 776092, 862492, -1508, 0 ]
+}, {
+  "tz" : "Europe/Belfast",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "Europe/Belgrade",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Europe/Berlin",
+  "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12219296008, -2422054408 ],
+  "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, -392, 0 ]
+}, {
+  "tz" : "Europe/Bratislava",
+  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12219296264, -2469401864 ],
+  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, -136, 0 ]
+}, {
+  "tz" : "Europe/Brussels",
+  "switches" : [ -62135597850, -59006362650, -55850689050, -52695015450, -46383581850, -43227908250, -40072234650, -33760801050, -30605127450, -27449453850, -21138020250, -17982346650, -14826673050, -12219293850, -2450995200, -2208988800 ],
+  "diffs" : [ -175350, -88950, -2550, 83850, 170250, 256650, 343050, 429450, 515850, 602250, 688650, 775050, 861450, -2550, -3600, 0 ]
+}, {
+  "tz" : "Europe/Bucharest",
+  "switches" : [ -62135603064, -59006367864, -55850694264, -52695020664, -46383587064, -43227913464, -40072239864, -33760806264, -30605132664, -27449459064, -21138025464, -17982351864, -14826678264, -12219299064, -2208988800 ],
+  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, -936, 0 ]
+}, {
+  "tz" : "Europe/Budapest",
+  "switches" : [ -62135601380, -59006366180, -55850692580, -52695018980, -46383585380, -43227911780, -40072238180, -33760804580, -30605130980, -27449457380, -21138023780, -17982350180, -14826676580, -12219297380, -2500938980 ],
+  "diffs" : [ -171820, -85420, 980, 87380, 173780, 260180, 346580, 432980, 519380, 605780, 692180, 778580, 864980, 980, 0 ]
+}, {
+  "tz" : "Europe/Busingen",
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, -1552, -1814, 0 ]
+}, {
+  "tz" : "Europe/Chisinau",
+  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12219299720, -2840147720, -2208988800 ],
+  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, -280, -300, 0 ]
+}, {
+  "tz" : "Europe/Copenhagen",
+  "switches" : [ -62135599820, -59006364620, -55850691020, -52695017420, -46383583820, -43227910220, -40072236620, -33760803020, -30605129420, -27449455820, -21138022220, -17982348620, -14826675020, -12219295820, -2398294220 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+}, {
+  "tz" : "Europe/Dublin",
+  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12219291300, -2821649700, -2208988800 ],
+  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, -1500, -1521, 0 ]
+}, {
+  "tz" : "Europe/Gibraltar",
+  "switches" : [ -62135595516, -59006360316, -55850686716, -52695013116, -46383579516, -43227905916, -40072232316, -33760798716, -30605125116, -27449451516, -21138017916, -17982344316, -14826670716, -12219291516, -2821649916, -2208988800 ],
+  "diffs" : [ -177684, -91284, -4884, 81516, 167916, 254316, 340716, 427116, 513516, 599916, 686316, 772716, 859116, -4884, -3600, 0 ]
+}, {
+  "tz" : "Europe/Guernsey",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "Europe/Helsinki",
+  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12219298789, -2208988800 ],
+  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, -1211, 0 ]
+}, {
+  "tz" : "Europe/Isle_of_Man",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "Europe/Istanbul",
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, -3848, -3784, 0 ]
+}, {
+  "tz" : "Europe/Jersey",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "Europe/Kaliningrad",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2422056120, -2208988800 ],
+  "diffs" : [ -175080, -88680, -2280, 84120, 170520, 256920, 343320, 429720, 516120, 602520, 688920, 775320, 861720, -2280, -3600, 0 ]
+}, {
+  "tz" : "Europe/Kiev",
+  "switches" : [ -62135604124, -59006368924, -55850695324, -52695021724, -46383588124, -43227914524, -40072240924, -33760807324, -30605133724, -27449460124, -21138026524, -17982352924, -14826679324, -12219300124, -2208988800 ],
+  "diffs" : [ -172676, -86276, 124, 86524, 172924, 259324, 345724, 432124, 518524, 604924, 691324, 777724, 864124, 124, 0 ]
+}, {
+  "tz" : "Europe/Kirov",
+  "switches" : [ -62135608728, -59006373528, -55850699928, -52695026328, -46383592728, -43227919128, -40072245528, -33760811928, -30605138328, -27449464728, -21138031128, -17982357528, -14826683928, -12219304728, -2208988800 ],
+  "diffs" : [ -171672, -85272, 1128, 87528, 173928, 260328, 346728, 433128, 519528, 605928, 692328, 778728, 865128, 1128, 0 ]
+}, {
+  "tz" : "Europe/Lisbon",
+  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12219290595, -2208988800 ],
+  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, -2205, 0 ]
+}, {
+  "tz" : "Europe/Ljubljana",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Europe/London",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "Europe/Luxembourg",
+  "switches" : [ -62135598276, -59006363076, -55850689476, -52695015876, -46383582276, -43227908676, -40072235076, -33760801476, -30605127876, -27449454276, -21138020676, -17982347076, -14826673476, -12219294276, -2208988800 ],
+  "diffs" : [ -174924, -88524, -2124, 84276, 170676, 257076, 343476, 429876, 516276, 602676, 689076, 775476, 861876, -2124, 0 ]
+}, {
+  "tz" : "Europe/Madrid",
+  "switches" : [ -62135595916, -59006360716, -55850687116, -52695013516, -46383579916, -43227906316, -40072232716, -33760799116, -30605125516, -27449451916, -21138018316, -17982344716, -14826671116, -12219291916, -2208988800 ],
+  "diffs" : [ -177284, -90884, -4484, 81916, 168316, 254716, 341116, 427516, 513916, 600316, 686716, 773116, 859516, -4484, 0 ]
+}, {
+  "tz" : "Europe/Malta",
+  "switches" : [ -62135600284, -59006365084, -55850691484, -52695017884, -46383584284, -43227910684, -40072237084, -33760803484, -30605129884, -27449456284, -21138022684, -17982349084, -14826675484, -12219296284, -2403478684 ],
+  "diffs" : [ -172916, -86516, -116, 86284, 172684, 259084, 345484, 431884, 518284, 604684, 691084, 777484, 863884, -116, 0 ]
+}, {
+  "tz" : "Europe/Mariehamn",
+  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12219298789, -2208988800 ],
+  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, -1211, 0 ]
+}, {
+  "tz" : "Europe/Minsk",
+  "switches" : [ -62135603416, -59006368216, -55850694616, -52695021016, -46383587416, -43227913816, -40072240216, -33760806616, -30605133016, -27449459416, -21138025816, -17982352216, -14826678616, -12219299416, -2840147416, -2208988800 ],
+  "diffs" : [ -176984, -90584, -4184, 82216, 168616, 255016, 341416, 427816, 514216, 600616, 687016, 773416, 859816, -4184, -4200, 0 ]
+}, {
+  "tz" : "Europe/Monaco",
+  "switches" : [ -62135598572, -59006363372, -55850689772, -52695016172, -46383582572, -43227908972, -40072235372, -33760801772, -30605128172, -27449454572, -21138020972, -17982347372, -14826673772, -12219294572, -2486680172, -2208988800 ],
+  "diffs" : [ -174628, -88228, -1828, 84572, 170972, 257372, 343772, 430172, 516572, 602972, 689372, 775772, 862172, -1828, -3039, 0 ]
+}, {
+  "tz" : "Europe/Moscow",
+  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12219301817, -2208988800 ],
+  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, -1783, 0 ]
+}, {
+  "tz" : "Europe/Nicosia",
+  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12219300808, -2208988800 ],
+  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 808, 0 ]
+}, {
+  "tz" : "Europe/Oslo",
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, -1020, 0 ]
+}, {
+  "tz" : "Europe/Paris",
+  "switches" : [ -62135597361, -59006362161, -55850688561, -52695014961, -46383581361, -43227907761, -40072234161, -33760800561, -30605126961, -27449453361, -21138019761, -17982346161, -14826672561, -12219293361, -2208988800 ],
+  "diffs" : [ -175839, -89439, -3039, 83361, 169761, 256161, 342561, 428961, 515361, 601761, 688161, 774561, 860961, -3039, 0 ]
+}, {
+  "tz" : "Europe/Podgorica",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Europe/Prague",
+  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12219296264, -2469401864 ],
+  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, -136, 0 ]
+}, {
+  "tz" : "Europe/Riga",
+  "switches" : [ -62135602594, -59006367394, -55850693794, -52695020194, -46383586594, -43227912994, -40072239394, -33760805794, -30605132194, -27449458594, -21138024994, -17982351394, -14826677794, -12219298594, -2208988800 ],
+  "diffs" : [ -174206, -87806, -1406, 84994, 171394, 257794, 344194, 430594, 516994, 603394, 689794, 776194, 862594, -1406, 0 ]
+}, {
+  "tz" : "Europe/Rome",
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+}, {
+  "tz" : "Europe/Samara",
+  "switches" : [ -62135608820, -59006373620, -55850700020, -52695026420, -46383592820, -43227919220, -40072245620, -33760812020, -30605138420, -27449464820, -21138031220, -17982357620, -14826684020, -12219304820, -2208988800 ],
+  "diffs" : [ -175180, -88780, -2380, 84020, 170420, 256820, 343220, 429620, 516020, 602420, 688820, 775220, 861620, -2380, 0 ]
+}, {
+  "tz" : "Europe/San_Marino",
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+}, {
+  "tz" : "Europe/Sarajevo",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Europe/Saratov",
+  "switches" : [ -62135607858, -59006372658, -55850699058, -52695025458, -46383591858, -43227918258, -40072244658, -33760811058, -30605137458, -27449463858, -21138030258, -17982356658, -14826683058, -12219303858, -2208988800 ],
+  "diffs" : [ -176142, -89742, -3342, 83058, 169458, 255858, 342258, 428658, 515058, 601458, 687858, 774258, 860658, -3342, 0 ]
+}, {
+  "tz" : "Europe/Simferopol",
+  "switches" : [ -62135604984, -59006369784, -55850696184, -52695022584, -46383588984, -43227915384, -40072241784, -33760808184, -30605134584, -27449460984, -21138027384, -17982353784, -14826680184, -12219300984, -2840148984, -2208988800 ],
+  "diffs" : [ -175416, -89016, -2616, 83784, 170184, 256584, 342984, 429384, 515784, 602184, 688584, 774984, 861384, -2616, -2640, 0 ]
+}, {
+  "tz" : "Europe/Skopje",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Europe/Sofia",
+  "switches" : [ -62135602396, -59006367196, -55850693596, -52695019996, -46383586396, -43227912796, -40072239196, -33760805596, -30605131996, -27449458396, -21138024796, -17982351196, -14826677596, -12219298396, -2840146396, -2369527016 ],
+  "diffs" : [ -174404, -88004, -1604, 84796, 171196, 257596, 343996, 430396, 516796, 603196, 689596, 775996, 862396, -1604, -184, 0 ]
+}, {
+  "tz" : "Europe/Stockholm",
+  "switches" : [ -62135601132, -59006365932, -55850692332, -52695018732, -46383585132, -43227911532, -40072237932, -33760804332, -30605130732, -27449457132, -21138023532, -17982349932, -14826676332, -12219297132, -2871681132, -2208992414 ],
+  "diffs" : [ -172068, -85668, 732, 87132, 173532, 259932, 346332, 432732, 519132, 605532, 691932, 778332, 864732, 732, 14, 0 ]
+}, {
+  "tz" : "Europe/Tallinn",
+  "switches" : [ -62135602740, -59006367540, -55850693940, -52695020340, -46383586740, -43227913140, -40072239540, -33760805940, -30605132340, -27449458740, -21138025140, -17982351540, -14826677940, -12219298740, -2208988800 ],
+  "diffs" : [ -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 862740, -1260, 0 ]
+}, {
+  "tz" : "Europe/Tirane",
+  "switches" : [ -62135601560, -59006366360, -55850692760, -52695019160, -46383585560, -43227911960, -40072238360, -33760804760, -30605131160, -27449457560, -21138023960, -17982350360, -14826676760, -12219297560, -2208988800 ],
+  "diffs" : [ -171640, -85240, 1160, 87560, 173960, 260360, 346760, 433160, 519560, 605960, 692360, 778760, 865160, 1160, 0 ]
+}, {
+  "tz" : "Europe/Tiraspol",
+  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12219299720, -2840147720, -2208988800 ],
+  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, -280, -300, 0 ]
+}, {
+  "tz" : "Europe/Ulyanovsk",
+  "switches" : [ -62135608416, -59006373216, -55850699616, -52695026016, -46383592416, -43227918816, -40072245216, -33760811616, -30605138016, -27449464416, -21138030816, -17982357216, -14826683616, -12219304416, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+}, {
+  "tz" : "Europe/Uzhgorod",
+  "switches" : [ -62135602152, -59006366952, -55850693352, -52695019752, -46383586152, -43227912552, -40072238952, -33760805352, -30605131752, -27449458152, -21138024552, -17982350952, -14826677352, -12219298152, -2500939752, -2208988800 ],
+  "diffs" : [ -174648, -88248, -1848, 84552, 170952, 257352, 343752, 430152, 516552, 602952, 689352, 775752, 862152, -1848, -3600, 0 ]
+}, {
+  "tz" : "Europe/Vaduz",
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, -1552, -1814, 0 ]
+}, {
+  "tz" : "Europe/Vatican",
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+}, {
+  "tz" : "Europe/Vienna",
+  "switches" : [ -62135600721, -59006365521, -55850691921, -52695018321, -46383584721, -43227911121, -40072237521, -33760803921, -30605130321, -27449456721, -21138023121, -17982349521, -14826675921, -12219296721, -2422055121 ],
+  "diffs" : [ -172479, -86079, 321, 86721, 173121, 259521, 345921, 432321, 518721, 605121, 691521, 777921, 864321, 321, 0 ]
+}, {
+  "tz" : "Europe/Vilnius",
+  "switches" : [ -62135602876, -59006367676, -55850694076, -52695020476, -46383586876, -43227913276, -40072239676, -33760806076, -30605132476, -27449458876, -21138025276, -17982351676, -14826678076, -12219298876, -2840146876, -2208988800 ],
+  "diffs" : [ -173924, -87524, -1124, 85276, 171676, 258076, 344476, 430876, 517276, 603676, 690076, 776476, 862876, -1124, -2160, 0 ]
+}, {
+  "tz" : "Europe/Volgograd",
+  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12219303460, -2208988800 ],
+  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, -3740, 0 ]
+}, {
+  "tz" : "Europe/Warsaw",
+  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12219297840, -2208988800 ],
+  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 1440, 0 ]
+}, {
+  "tz" : "Europe/Zagreb",
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+}, {
+  "tz" : "Europe/Zaporozhye",
+  "switches" : [ -62135605240, -59006370040, -55850696440, -52695022840, -46383589240, -43227915640, -40072242040, -33760808440, -30605134840, -27449461240, -21138027640, -17982354040, -14826680440, -12219301240, -2840149240, -2208988800 ],
+  "diffs" : [ -171560, -85160, 1240, 87640, 174040, 260440, 346840, 433240, 519640, 606040, 692440, 778840, 865240, 1240, 1200, 0 ]
+}, {
+  "tz" : "Europe/Zurich",
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, -1552, -1814, 0 ]
+}, {
+  "tz" : "GB",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "GB-Eire",
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+}, {
+  "tz" : "GMT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "GMT0",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Greenwich",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Hongkong",
+  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12219320202, -2208988800 ],
+  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, -1398, 0 ]
+}, {
+  "tz" : "Iceland",
+  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12219287520, -2208988800 ],
+  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, -5280, 0 ]
+}, {
+  "tz" : "Indian/Antananarivo",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Indian/Chagos",
+  "switches" : [ -62135614180, -59006378980, -55850705380, -52695031780, -46383598180, -43227924580, -40072250980, -33760817380, -30605143780, -27449470180, -21138036580, -17982362980, -14826689380, -12219310180, -2208988800 ],
+  "diffs" : [ -177020, -90620, -4220, 82180, 168580, 254980, 341380, 427780, 514180, 600580, 686980, 773380, 859780, -4220, 0 ]
+}, {
+  "tz" : "Indian/Christmas",
+  "switches" : [ -62135622172, -59006386972, -55850713372, -52695039772, -46383606172, -43227932572, -40072258972, -33760825372, -30605151772, -27449478172, -21138044572, -17982370972, -14826697372, -12219318172, -2364102172 ],
+  "diffs" : [ -172628, -86228, 172, 86572, 172972, 259372, 345772, 432172, 518572, 604972, 691372, 777772, 864172, 172, 0 ]
+}, {
+  "tz" : "Indian/Cocos",
+  "switches" : [ -62135620060, -59006384860, -55850711260, -52695037660, -46383604060, -43227930460, -40072256860, -33760823260, -30605149660, -27449476060, -21138042460, -17982368860, -14826695260, -12219316060, -2209012060 ],
+  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, -140, 0 ]
+}, {
+  "tz" : "Indian/Comoro",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Indian/Kerguelen",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
+  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, -18000, 0 ]
+}, {
+  "tz" : "Indian/Mahe",
+  "switches" : [ -62135610108, -59006374908, -55850701308, -52695027708, -46383594108, -43227920508, -40072246908, -33760813308, -30605139708, -27449466108, -21138032508, -17982358908, -14826685308, -12219306108, -2208988800 ],
+  "diffs" : [ -173892, -87492, -1092, 85308, 171708, 258108, 344508, 430908, 517308, 603708, 690108, 776508, 862908, -1092, 0 ]
+}, {
+  "tz" : "Indian/Maldives",
+  "switches" : [ -62135614440, -59006379240, -55850705640, -52695032040, -46383598440, -43227924840, -40072251240, -33760817640, -30605144040, -27449470440, -21138036840, -17982363240, -14826689640, -12219310440, -2208988800 ],
+  "diffs" : [ -173160, -86760, -360, 86040, 172440, 258840, 345240, 431640, 518040, 604440, 690840, 777240, 863640, -360, 0 ]
+}, {
+  "tz" : "Indian/Mauritius",
+  "switches" : [ -62135610600, -59006375400, -55850701800, -52695028200, -46383594600, -43227921000, -40072247400, -33760813800, -30605140200, -27449466600, -21138033000, -17982359400, -14826685800, -12219306600, -2208988800 ],
+  "diffs" : [ -173400, -87000, -600, 85800, 172200, 258600, 345000, 431400, 517800, 604200, 690600, 777000, 863400, -600, 0 ]
+}, {
+  "tz" : "Indian/Mayotte",
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+}, {
+  "tz" : "Indian/Reunion",
+  "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12219306112, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, -1088, 0 ]
+}, {
+  "tz" : "Israel",
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 1254, 1240, 0 ]
+}, {
+  "tz" : "Jamaica",
+  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12219274370, -2208988800 ],
+  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, -430, 0 ]
+}, {
+  "tz" : "Japan",
+  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12219326339, -2587712400 ],
+  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 1139, 0 ]
+}, {
+  "tz" : "Kwajalein",
+  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12219332960, -2208988800 ],
+  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, -3040, 0 ]
+}, {
+  "tz" : "Libya",
+  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12219295964, -2208988800 ],
+  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, -4036, 0 ]
+}, {
+  "tz" : "MET",
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "MST7MDT",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Mexico/BajaNorte",
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+}, {
+  "tz" : "Mexico/BajaSur",
+  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12219267260, -2208988800 ],
+  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, -340, 0 ]
+}, {
+  "tz" : "Mexico/General",
+  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12219269004, -2208988800 ],
+  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, -2196, 0 ]
+}, {
+  "tz" : "NZ",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+}, {
+  "tz" : "NZ-CHAT",
+  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12219336828, -3192437628, -2208988800 ],
+  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, -1872, -1800, 0 ]
+}, {
+  "tz" : "Navajo",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+}, {
+  "tz" : "PRC",
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+}, {
+  "tz" : "PST8PDT",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Pacific/Apia",
+  "switches" : [ -62135641984, -59006406784, -55850733184, -52695059584, -46383625984, -43227952384, -40072278784, -33760845184, -30605171584, -27449497984, -21138064384, -17982390784, -14826717184, -12219337984, -2445424384, -2208988800 ],
+  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, -1616, -88016, 0 ]
+}, {
+  "tz" : "Pacific/Auckland",
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Bougainville",
+  "switches" : [ -62135634136, -59006398936, -55850725336, -52695051736, -46383618136, -43227944536, -40072270936, -33760837336, -30605163736, -27449490136, -21138056536, -17982382936, -14826709336, -12219330136, -2840178136, -2366790512, -2208988800 ],
+  "diffs" : [ -175064, -88664, -2264, 84136, 170536, 256936, 343336, 429736, 516136, 602536, 688936, 775336, 861736, -2264, -4288, -3600, 0 ]
+}, {
+  "tz" : "Pacific/Chatham",
+  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12219336828, -3192437628, -2208988800 ],
+  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, -1872, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Chuuk",
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, -85972, 428, 0 ]
+}, {
+  "tz" : "Pacific/Easter",
+  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12219266552, -2208988800 ],
+  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, -4648, 0 ]
+}, {
+  "tz" : "Pacific/Efate",
+  "switches" : [ -62135637196, -59006401996, -55850728396, -52695054796, -46383621196, -43227947596, -40072273996, -33760840396, -30605166796, -27449493196, -21138059596, -17982385996, -14826712396, -12219333196, -2208988800 ],
+  "diffs" : [ -172004, -85604, 796, 87196, 173596, 259996, 346396, 432796, 519196, 605596, 691996, 778396, 864796, 796, 0 ]
+}, {
+  "tz" : "Pacific/Enderbury",
+  "switches" : [ -62135555740, -59006320540, -55850646940, -52694973340, -46383539740, -43227866140, -40072192540, -33760758940, -30605085340, -27449411740, -21137978140, -17982304540, -14826630940, -12219251740, -2208988800 ],
+  "diffs" : [ -260660, -174260, -87860, -1460, 84940, 171340, 257740, 344140, 430540, 516940, 603340, 689740, 776140, -87860, 0 ]
+}, {
+  "tz" : "Pacific/Fakaofo",
+  "switches" : [ -62135555704, -59006320504, -55850646904, -52694973304, -46383539704, -43227866104, -40072192504, -33760758904, -30605085304, -27449411704, -21137978104, -17982304504, -14826630904, -12219251704, -2208988800 ],
+  "diffs" : [ -260696, -174296, -87896, -1496, 84904, 171304, 257704, 344104, 430504, 516904, 603304, 689704, 776104, -87896, 0 ]
+}, {
+  "tz" : "Pacific/Fiji",
+  "switches" : [ -62135639744, -59006404544, -55850730944, -52695057344, -46383623744, -43227950144, -40072276544, -33760842944, -30605169344, -27449495744, -21138062144, -17982388544, -14826714944, -12219335744, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, -256, 0 ]
+}, {
+  "tz" : "Pacific/Funafuti",
+  "switches" : [ -62135639812, -59006404612, -55850731012, -52695057412, -46383623812, -43227950212, -40072276612, -33760843012, -30605169412, -27449495812, -21138062212, -17982388612, -14826715012, -12219335812, -2208988800 ],
+  "diffs" : [ -172988, -86588, -188, 86212, 172612, 259012, 345412, 431812, 518212, 604612, 691012, 777412, 863812, -188, 0 ]
+}, {
+  "tz" : "Pacific/Galapagos",
+  "switches" : [ -62135575296, -59006340096, -55850666496, -52694992896, -46383559296, -43227885696, -40072212096, -33760778496, -30605104896, -27449431296, -21137997696, -17982324096, -14826650496, -12219271296, -2208988800 ],
+  "diffs" : [ -172704, -86304, 96, 86496, 172896, 259296, 345696, 432096, 518496, 604896, 691296, 777696, 864096, 96, 0 ]
+}, {
+  "tz" : "Pacific/Gambier",
+  "switches" : [ -62135564412, -59006329212, -55850655612, -52694982012, -46383548412, -43227874812, -40072201212, -33760767612, -30605094012, -27449420412, -21137986812, -17982313212, -14826639612, -12219260412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 12, 0 ]
+}, {
+  "tz" : "Pacific/Guadalcanal",
+  "switches" : [ -62135635188, -59006399988, -55850726388, -52695052788, -46383619188, -43227945588, -40072271988, -33760838388, -30605164788, -27449491188, -21138057588, -17982383988, -14826710388, -12219331188, -2208988800 ],
+  "diffs" : [ -174012, -87612, -1212, 85188, 171588, 257988, 344388, 430788, 517188, 603588, 689988, 776388, 862788, -1212, 0 ]
+}, {
+  "tz" : "Pacific/Guam",
+  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12219241140, -3944626740, -2208988800 ],
+  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, -87660, -1260, 0 ]
+}, {
+  "tz" : "Pacific/Honolulu",
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, -1886, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Johnston",
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, -1886, -1800, 0 ]
+}, {
+  "tz" : "Pacific/Kiritimati",
+  "switches" : [ -62135559040, -59006323840, -55850650240, -52694976640, -46383543040, -43227869440, -40072195840, -33760762240, -30605088640, -27449415040, -21137981440, -17982307840, -14826634240, -12219255040, -2208988800 ],
+  "diffs" : [ -260960, -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, -88160, 0 ]
+}, {
+  "tz" : "Pacific/Kosrae",
+  "switches" : [ -62135549516, -59006314316, -55850640716, -52694967116, -46383533516, -43227859916, -40072186316, -33760752716, -30605079116, -27449405516, -21137971916, -17982298316, -14826624716, -12219245516, -3944631116, -2208988800 ],
+  "diffs" : [ -259684, -173284, -86884, -484, 85916, 172316, 258716, 345116, 431516, 517916, 604316, 690716, 777116, -86884, -484, 0 ]
+}, {
+  "tz" : "Pacific/Kwajalein",
+  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12219332960, -2208988800 ],
+  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, -3040, 0 ]
+}, {
+  "tz" : "Pacific/Majuro",
+  "switches" : [ -62135637888, -59006402688, -55850729088, -52695055488, -46383621888, -43227948288, -40072274688, -33760841088, -30605167488, -27449493888, -21138060288, -17982386688, -14826713088, -12219333888, -2208988800 ],
+  "diffs" : [ -174912, -88512, -2112, 84288, 170688, 257088, 343488, 429888, 516288, 602688, 689088, 775488, 861888, -2112, 0 ]
+}, {
+  "tz" : "Pacific/Marquesas",
+  "switches" : [ -62135563320, -59006328120, -55850654520, -52694980920, -46383547320, -43227873720, -40072200120, -33760766520, -30605092920, -27449419320, -21137985720, -17982312120, -14826638520, -12219259320, -2208988800 ],
+  "diffs" : [ -172080, -85680, 720, 87120, 173520, 259920, 346320, 432720, 519120, 605520, 691920, 778320, 864720, 720, 0 ]
+}, {
+  "tz" : "Pacific/Midway",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+}, {
+  "tz" : "Pacific/Nauru",
+  "switches" : [ -62135636860, -59006401660, -55850728060, -52695054460, -46383620860, -43227947260, -40072273660, -33760840060, -30605166460, -27449492860, -21138059260, -17982385660, -14826712060, -12219332860, -2208988800 ],
+  "diffs" : [ -175940, -89540, -3140, 83260, 169660, 256060, 342460, 428860, 515260, 601660, 688060, 774460, 860860, -3140, 0 ]
+}, {
+  "tz" : "Pacific/Niue",
+  "switches" : [ -62135556020, -59006320820, -55850647220, -52694973620, -46383540020, -43227866420, -40072192820, -33760759220, -30605085620, -27449412020, -21137978420, -17982304820, -14826631220, -12219252020, -2208988800 ],
+  "diffs" : [ -173980, -87580, -1180, 85220, 171620, 258020, 344420, 430820, 517220, 603620, 690020, 776420, 862820, -1180, 0 ]
+}, {
+  "tz" : "Pacific/Norfolk",
+  "switches" : [ -62135637112, -59006401912, -55850728312, -52695054712, -46383621112, -43227947512, -40072273912, -33760840312, -30605166712, -27449493112, -21138059512, -17982385912, -14826712312, -12219333112, -2208988800 ],
+  "diffs" : [ -172088, -85688, 712, 87112, 173512, 259912, 346312, 432712, 519112, 605512, 691912, 778312, 864712, 712, 0 ]
+}, {
+  "tz" : "Pacific/Noumea",
+  "switches" : [ -62135636748, -59006401548, -55850727948, -52695054348, -46383620748, -43227947148, -40072273548, -33760839948, -30605166348, -27449492748, -21138059148, -17982385548, -14826711948, -12219332748, -2208988800 ],
+  "diffs" : [ -172452, -86052, 348, 86748, 173148, 259548, 345948, 432348, 518748, 605148, 691548, 777948, 864348, 348, 0 ]
+}, {
+  "tz" : "Pacific/Pago_Pago",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+}, {
+  "tz" : "Pacific/Palau",
+  "switches" : [ -62135542676, -59006307476, -55850633876, -52694960276, -46383526676, -43227853076, -40072179476, -33760745876, -30605072276, -27449398676, -21137965076, -17982291476, -14826617876, -12219238676, -3944624276, -2208988800 ],
+  "diffs" : [ -259324, -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, -86524, -124, 0 ]
+}, {
+  "tz" : "Pacific/Pitcairn",
+  "switches" : [ -62135565580, -59006330380, -55850656780, -52694983180, -46383549580, -43227875980, -40072202380, -33760768780, -30605095180, -27449421580, -21137987980, -17982314380, -14826640780, -12219261580, -2208988800 ],
+  "diffs" : [ -175220, -88820, -2420, 83980, 170380, 256780, 343180, 429580, 515980, 602380, 688780, 775180, 861580, -2420, 0 ]
+}, {
+  "tz" : "Pacific/Pohnpei",
+  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12219244372, -3944629972, -2208988800 ],
+  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, -88028, -1628, 0 ]
+}, {
+  "tz" : "Pacific/Ponape",
+  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12219244372, -3944629972, -2208988800 ],
+  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, -88028, -1628, 0 ]
+}, {
+  "tz" : "Pacific/Port_Moresby",
+  "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12219328120, -2840176120, -2366790512 ],
+  "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, -680, -688, 0 ]
+}, {
+  "tz" : "Pacific/Rarotonga",
+  "switches" : [ -62135558456, -59006323256, -55850649656, -52694976056, -46383542456, -43227868856, -40072195256, -33760761656, -30605088056, -27449414456, -21137980856, -17982307256, -14826633656, -12219254456, -2208988800 ],
+  "diffs" : [ -175144, -88744, -2344, 84056, 170456, 256856, 343256, 429656, 516056, 602456, 688856, 775256, 861656, -2344, 0 ]
+}, {
+  "tz" : "Pacific/Saipan",
+  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12219241140, -3944626740, -2208988800 ],
+  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, -87660, -1260, 0 ]
+}, {
+  "tz" : "Pacific/Samoa",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+}, {
+  "tz" : "Pacific/Tahiti",
+  "switches" : [ -62135560904, -59006325704, -55850652104, -52694978504, -46383544904, -43227871304, -40072197704, -33760764104, -30605090504, -27449416904, -21137983304, -17982309704, -14826636104, -12219256904, -2208988800 ],
+  "diffs" : [ -172696, -86296, 104, 86504, 172904, 259304, 345704, 432104, 518504, 604904, 691304, 777704, 864104, 104, 0 ]
+}, {
+  "tz" : "Pacific/Tarawa",
+  "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12219334324, -2208988800 ],
+  "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, -1676, 0 ]
+}, {
+  "tz" : "Pacific/Tongatapu",
+  "switches" : [ -62135641160, -59006405960, -55850732360, -52695058760, -46383625160, -43227951560, -40072277960, -33760844360, -30605170760, -27449497160, -21138063560, -17982389960, -14826716360, -12219337160, -2208988800 ],
+  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, -2440, 0 ]
+}, {
+  "tz" : "Pacific/Truk",
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, -85972, 428, 0 ]
+}, {
+  "tz" : "Pacific/Wake",
+  "switches" : [ -62135636788, -59006401588, -55850727988, -52695054388, -46383620788, -43227947188, -40072273588, -33760839988, -30605166388, -27449492788, -21138059188, -17982385588, -14826711988, -12219332788, -2208988800 ],
+  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, -3212, 0 ]
+}, {
+  "tz" : "Pacific/Wallis",
+  "switches" : [ -62135640920, -59006405720, -55850732120, -52695058520, -46383624920, -43227951320, -40072277720, -33760844120, -30605170520, -27449496920, -21138063320, -17982389720, -14826716120, -12219336920, -2208988800 ],
+  "diffs" : [ -171880, -85480, 920, 87320, 173720, 260120, 346520, 432920, 519320, 605720, 692120, 778520, 864920, 920, 0 ]
+}, {
+  "tz" : "Pacific/Yap",
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, -85972, 428, 0 ]
+}, {
+  "tz" : "Poland",
+  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12219297840, -2208988800 ],
+  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 1440, 0 ]
+}, {
+  "tz" : "Portugal",
+  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12219290595, -2208988800 ],
+  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, -2205, 0 ]
+}, {
+  "tz" : "ROK",
+  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12219323272, -2208988800 ],
+  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, -1928, 0 ]
+}, {
+  "tz" : "Singapore",
+  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12219317725, -2208988800 ],
+  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, -3875, 0 ]
+}, {
+  "tz" : "SystemV/AST4",
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/AST4ADT",
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/CST6",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/CST6CDT",
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/EST5",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/EST5EDT",
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/HST10",
+  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12219256800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/MST7",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/MST7MDT",
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/PST8",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/PST8PDT",
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/YST9",
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "SystemV/YST9YDT",
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Turkey",
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, -3848, -3784, 0 ]
+}, {
+  "tz" : "UCT",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "US/Alaska",
+  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12219343224, -3225223727, -2208988800 ],
+  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 82824, -3576, 0 ]
+}, {
+  "tz" : "US/Aleutian",
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 80002, -6398, 0 ]
+}, {
+  "tz" : "US/Arizona",
+  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12219265902, -2717643600 ],
+  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, -1698, 0 ]
+}, {
+  "tz" : "US/Central",
+  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12219271764, -2717647200 ],
+  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 564, 0 ]
+}, {
+  "tz" : "US/East-Indiana",
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+}, {
+  "tz" : "US/Eastern",
+  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12219275038, -2717650800 ],
+  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 238, 0 ]
+}, {
+  "tz" : "US/Hawaii",
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, -1886, -1800, 0 ]
+}, {
+  "tz" : "US/Indiana-Starke",
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 810, 0 ]
+}, {
+  "tz" : "US/Michigan",
+  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12219272869, -2208988800 ],
+  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, -1931, 0 ]
+}, {
+  "tz" : "US/Mountain",
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+}, {
+  "tz" : "US/Pacific",
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 422, 0 ]
+}, {
+  "tz" : "US/Pacific-New",
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 422, 0 ]
+}, {
+  "tz" : "US/Samoa",
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+}, {
+  "tz" : "UTC",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Universal",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "W-SU",
+  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12219301817, -2208988800 ],
+  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, -1783, 0 ]
+}, {
+  "tz" : "WET",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+}, {
+  "tz" : "Zulu",
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+} ]
\ No newline at end of file
diff --git a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
new file mode 100644
index 0000000000000..b733ac43cbf1f
--- /dev/null
+++ b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
@@ -0,0 +1,2385 @@
+[ {
+  "tz" : "Africa/Abidjan",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Accra",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ]
+}, {
+  "tz" : "Africa/Addis_Ababa",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Algiers",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486682111, -2208988800 ],
+  "diffs" : [ 175668, 89268, 2868, -83532, -169932, -256332, -342732, -429132, -515532, -601932, -688332, -774732, -861132, 2868, 3039, 0 ]
+}, {
+  "tz" : "Africa/Asmara",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Asmera",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Bamako",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Bangui",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Banjul",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Bissau",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 176540, 90140, 3740, -82660, -169060, -255460, -341860, -428260, -514660, -601060, -687460, -773860, -860260, 3740, 0 ]
+}, {
+  "tz" : "Africa/Blantyre",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Brazzaville",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Bujumbura",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Cairo",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ]
+}, {
+  "tz" : "Africa/Ceuta",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 177676, 91276, 4876, -81524, -167924, -254324, -340724, -427124, -513524, -599924, -686324, -772724, -859124, 4876, 0 ]
+}, {
+  "tz" : "Africa/Conakry",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Dakar",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Dar_es_Salaam",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Djibouti",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Douala",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Freetown",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Gaborone",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Harare",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Johannesburg",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458174920, -2208988800 ],
+  "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ]
+}, {
+  "tz" : "Africa/Juba",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 176012, 89612, 3212, -83188, -169588, -255988, -342388, -428788, -515188, -601588, -687988, -774388, -860788, 3212, 0 ]
+}, {
+  "tz" : "Africa/Kampala",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Khartoum",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172192, 85792, -608, -87008, -173408, -259808, -346208, -432608, -519008, -605408, -691808, -778208, -864608, -608, 0 ]
+}, {
+  "tz" : "Africa/Kigali",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Kinshasa",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Lagos",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Libreville",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Lome",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Luanda",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Lubumbashi",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Lusaka",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Malabo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Maputo",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172180, 85780, -620, -87020, -173420, -259820, -346220, -432620, -519020, -605420, -691820, -778220, -864620, -620, 0 ]
+}, {
+  "tz" : "Africa/Maseru",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458174920, -2208988800 ],
+  "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ]
+}, {
+  "tz" : "Africa/Mbabane",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458174920, -2208988800 ],
+  "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ]
+}, {
+  "tz" : "Africa/Mogadishu",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Monrovia",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175388, 88988, 2588, -83812, -170212, -256612, -343012, -429412, -515812, -602212, -688612, -775012, -861412, 2588, 0 ]
+}, {
+  "tz" : "Africa/Nairobi",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Africa/Ndjamena",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 172788, 86388, -12, -86412, -172812, -259212, -345612, -432012, -518412, -604812, -691212, -777612, -864012, -12, 0 ]
+}, {
+  "tz" : "Africa/Niamey",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Nouakchott",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Ouagadougou",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Porto-Novo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/Sao_Tome",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2713914221, -2208988800 ],
+  "diffs" : [ 171184, 84784, -1616, -88016, -174416, -260816, -347216, -433616, -520016, -606416, -692816, -779216, -865616, -1616, 2205, 0 ]
+}, {
+  "tz" : "Africa/Timbuktu",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Africa/Tripoli",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 176836, 90436, 4036, -82364, -168764, -255164, -341564, -427964, -514364, -600764, -687164, -773564, -859964, 4036, 0 ]
+}, {
+  "tz" : "Africa/Tunis",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2797205483, -2208988800 ],
+  "diffs" : [ 173956, 87556, 1156, -85244, -171644, -258044, -344444, -430844, -517244, -603644, -690044, -776444, -862844, 1156, 3039, 0 ]
+}, {
+  "tz" : "Africa/Windhoek",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2458172304, -2208988800 ],
+  "diffs" : [ 175896, 89496, 3096, -83304, -169704, -256104, -342504, -428904, -515304, -601704, -688104, -774504, -860904, 3096, 1800, 0 ]
+}, {
+  "tz" : "America/Adak",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ],
+  "diffs" : [ 92798, 6398, -80002, -166402, -252802, -339202, -425602, -512002, -598402, -684802, -771202, -857602, -944002, -80002, 6398, 0 ]
+}, {
+  "tz" : "America/Anchorage",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225227303, -2208988800 ],
+  "diffs" : [ 89976, 3576, -82824, -169224, -255624, -342024, -428424, -514824, -601224, -687624, -774024, -860424, -946824, -82824, 3576, 0 ]
+}, {
+  "tz" : "America/Anguilla",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Antigua",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Araguaina",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173568, 87168, 768, -85632, -172032, -258432, -344832, -431232, -517632, -604032, -690432, -776832, -863232, 768, 0 ]
+}, {
+  "tz" : "America/Argentina/Buenos_Aires",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372102580, -2208988800 ],
+  "diffs" : [ 176028, 89628, 3228, -83172, -169572, -255972, -342372, -428772, -515172, -601572, -687972, -774372, -860772, 3228, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Catamarca",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100820, -2208988800 ],
+  "diffs" : [ 177788, 91388, 4988, -81412, -167812, -254212, -340612, -427012, -513412, -599812, -686212, -772612, -859012, 4988, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/ComodRivadavia",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100820, -2208988800 ],
+  "diffs" : [ 177788, 91388, 4988, -81412, -167812, -254212, -340612, -427012, -513412, -599812, -686212, -772612, -859012, 4988, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Cordoba",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 177408, 91008, 4608, -81792, -168192, -254592, -340992, -427392, -513792, -600192, -686592, -772992, -859392, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Jujuy",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100936, -2208988800 ],
+  "diffs" : [ 177672, 91272, 4872, -81528, -167928, -254328, -340728, -427128, -513528, -599928, -686328, -772728, -859128, 4872, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/La_Rioja",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100564, -2208988800 ],
+  "diffs" : [ 178044, 91644, 5244, -81156, -167556, -253956, -340356, -426756, -513156, -599556, -685956, -772356, -858756, 5244, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Mendoza",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100092, -2208988800 ],
+  "diffs" : [ 178516, 92116, 5716, -80684, -167084, -253484, -339884, -426284, -512684, -599084, -685484, -771884, -858284, 5716, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Rio_Gallegos",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372099996, -2208988800 ],
+  "diffs" : [ 178612, 92212, 5812, -80588, -166988, -253388, -339788, -426188, -512588, -598988, -685388, -771788, -858188, 5812, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Salta",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100908, -2208988800 ],
+  "diffs" : [ 177700, 91300, 4900, -81500, -167900, -254300, -340700, -427100, -513500, -599900, -686300, -772700, -859100, 4900, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Juan",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100164, -2208988800 ],
+  "diffs" : [ 178444, 92044, 5644, -80756, -167156, -253556, -339956, -426356, -512756, -599156, -685556, -771956, -858356, 5644, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/San_Luis",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100684, -2208988800 ],
+  "diffs" : [ 177924, 91524, 5124, -81276, -167676, -254076, -340476, -426876, -513276, -599676, -686076, -772476, -858876, 5124, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Tucuman",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100956, -2208988800 ],
+  "diffs" : [ 177652, 91252, 4852, -81548, -167948, -254348, -340748, -427148, -513548, -599948, -686348, -772748, -859148, 4852, 4608, 0 ]
+}, {
+  "tz" : "America/Argentina/Ushuaia",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100216, -2208988800 ],
+  "diffs" : [ 178392, 91992, 5592, -80808, -167208, -253608, -340008, -426408, -512808, -599208, -685608, -772008, -858408, 5592, 4608, 0 ]
+}, {
+  "tz" : "America/Aruba",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/Asuncion",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172240, 85840, -560, -86960, -173360, -259760, -346160, -432560, -518960, -605360, -691760, -778160, -864560, -560, 0 ]
+}, {
+  "tz" : "America/Atikokan",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736812, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 3600, 0 ]
+}, {
+  "tz" : "America/Atka",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ],
+  "diffs" : [ 92798, 6398, -80002, -166402, -252802, -339202, -425602, -512002, -598402, -684802, -771202, -857602, -944002, -80002, 6398, 0 ]
+}, {
+  "tz" : "America/Bahia",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 171244, 84844, -1556, -87956, -174356, -260756, -347156, -433556, -519956, -606356, -692756, -779156, -865556, -1556, 0 ]
+}, {
+  "tz" : "America/Bahia_Banderas",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 176460, 90060, 3660, -82740, -169140, -255540, -341940, -428340, -514740, -601140, -687540, -773940, -860340, 3660, 0 ]
+}, {
+  "tz" : "America/Barbados",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172709, 86309, -91, -86491, -172891, -259291, -345691, -432091, -518491, -604891, -691291, -777691, -864091, -91, 0 ]
+}, {
+  "tz" : "America/Belem",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173636, 87236, 836, -85564, -171964, -258364, -344764, -431164, -517564, -603964, -690364, -776764, -863164, 836, 0 ]
+}, {
+  "tz" : "America/Belize",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172368, 85968, -432, -86832, -173232, -259632, -346032, -432432, -518832, -605232, -691632, -778032, -864432, -432, 0 ]
+}, {
+  "tz" : "America/Blanc-Sablon",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2713896692 ],
+  "diffs" : [ 172108, 85708, -692, -87092, -173492, -259892, -346292, -432692, -519092, -605492, -691892, -778292, -864692, -692, 0 ]
+}, {
+  "tz" : "America/Boa_Vista",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172960, 86560, 160, -86240, -172640, -259040, -345440, -431840, -518240, -604640, -691040, -777440, -863840, 160, 0 ]
+}, {
+  "tz" : "America/Bogota",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 172576, 86176, -224, -86624, -173024, -259424, -345824, -432224, -518624, -605024, -691424, -777824, -864224, -224, 0 ]
+}, {
+  "tz" : "America/Boise",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600, -2208988800 ],
+  "diffs" : [ 175489, 89089, 2689, -83711, -170111, -256511, -342911, -429311, -515711, -602111, -688511, -774911, -861311, 2689, 3600, 0 ]
+}, {
+  "tz" : "America/Buenos_Aires",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372102580, -2208988800 ],
+  "diffs" : [ 176028, 89628, 3228, -83172, -169572, -255972, -342372, -428772, -515172, -601572, -687972, -774372, -860772, 3228, 4608, 0 ]
+}, {
+  "tz" : "America/Cambridge_Bay",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ]
+}, {
+  "tz" : "America/Campo_Grande",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 171508, 85108, -1292, -87692, -174092, -260492, -346892, -433292, -519692, -606092, -692492, -778892, -865292, -1292, 0 ]
+}, {
+  "tz" : "America/Cancun",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 175624, 89224, 2824, -83576, -169976, -256376, -342776, -429176, -515576, -601976, -688376, -774776, -861176, 2824, 0 ]
+}, {
+  "tz" : "America/Caracas",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2524507196, -2208988800 ],
+  "diffs" : [ 174464, 88064, 1664, -84736, -171136, -257536, -343936, -430336, -516736, -603136, -689536, -775936, -862336, 1664, 1660, 0 ]
+}, {
+  "tz" : "America/Catamarca",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100820, -2208988800 ],
+  "diffs" : [ 177788, 91388, 4988, -81412, -167812, -254212, -340612, -427012, -513412, -599812, -686212, -772612, -859012, 4988, 4608, 0 ]
+}, {
+  "tz" : "America/Cayenne",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 174560, 88160, 1760, -84640, -171040, -257440, -343840, -430240, -516640, -603040, -689440, -775840, -862240, 1760, 0 ]
+}, {
+  "tz" : "America/Cayman",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503688, -2208988800 ],
+  "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 1176, 0 ]
+}, {
+  "tz" : "America/Chicago",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 172236, 85836, -564, -86964, -173364, -259764, -346164, -432564, -518964, -605364, -691764, -778164, -864564, -564, 0 ]
+}, {
+  "tz" : "America/Chihuahua",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 173060, 86660, 260, -86140, -172540, -258940, -345340, -431740, -518140, -604540, -690940, -777340, -863740, 260, 0 ]
+}, {
+  "tz" : "America/Coral_Harbour",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736812, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 3600, 0 ]
+}, {
+  "tz" : "America/Cordoba",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 177408, 91008, 4608, -81792, -168192, -254592, -340992, -427392, -513792, -600192, -686592, -772992, -859392, 4608, 0 ]
+}, {
+  "tz" : "America/Costa_Rica",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 171373, 84973, -1427, -87827, -174227, -260627, -347027, -433427, -519827, -606227, -692627, -779027, -865427, -1427, 0 ]
+}, {
+  "tz" : "America/Creston",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713882436 ],
+  "diffs" : [ 175564, 89164, 2764, -83636, -170036, -256436, -342836, -429236, -515636, -602036, -688436, -774836, -861236, 2764, 0 ]
+}, {
+  "tz" : "America/Cuiaba",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 171860, 85460, -940, -87340, -173740, -260140, -346540, -432940, -519340, -605740, -692140, -778540, -864940, -940, 0 ]
+}, {
+  "tz" : "America/Curacao",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/Danmarkshavn",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 177280, 90880, 4480, -81920, -168320, -254720, -341120, -427520, -513920, -600320, -686720, -773120, -859520, 4480, 0 ]
+}, {
+  "tz" : "America/Dawson",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 177460, 91060, 4660, -81740, -168140, -254540, -340940, -427340, -513740, -600140, -686540, -772940, -859340, 4660, 0 ]
+}, {
+  "tz" : "America/Dawson_Creek",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713885144, -2208988800 ],
+  "diffs" : [ 176456, 90056, 3656, -82744, -169144, -255544, -341944, -428344, -514744, -601144, -687544, -773944, -860344, 3656, 3600, 0 ]
+}, {
+  "tz" : "America/Denver",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "America/Detroit",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 174731, 88331, 1931, -84469, -170869, -257269, -343669, -430069, -516469, -602869, -689269, -775669, -862069, 1931, 0 ]
+}, {
+  "tz" : "America/Dominica",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Edmonton",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 174832, 88432, 2032, -84368, -170768, -257168, -343568, -429968, -516368, -602768, -689168, -775568, -861968, 2032, 0 ]
+}, {
+  "tz" : "America/Eirunepe",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171568, 85168, -1232, -87632, -174032, -260432, -346832, -433232, -519632, -606032, -692432, -778832, -865232, -1232, 0 ]
+}, {
+  "tz" : "America/El_Salvador",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172608, 86208, -192, -86592, -172992, -259392, -345792, -432192, -518592, -604992, -691392, -777792, -864192, -192, 0 ]
+}, {
+  "tz" : "America/Ensenada",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "America/Fort_Nelson",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713884553, -2208988800 ],
+  "diffs" : [ 177047, 90647, 4247, -82153, -168553, -254953, -341353, -427753, -514153, -600553, -686953, -773353, -859753, 4247, 3600, 0 ]
+}, {
+  "tz" : "America/Fort_Wayne",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "America/Fortaleza",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 171240, 84840, -1560, -87960, -174360, -260760, -347160, -433560, -519960, -606360, -692760, -779160, -865560, -1560, 0 ]
+}, {
+  "tz" : "America/Glace_Bay",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172788, 86388, -12, -86412, -172812, -259212, -345612, -432012, -518412, -604812, -691212, -777612, -864012, -12, 0 ]
+}, {
+  "tz" : "America/Godthab",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 174416, 88016, 1616, -84784, -171184, -257584, -343984, -430384, -516784, -603184, -689584, -775984, -862384, 1616, 0 ]
+}, {
+  "tz" : "America/Goose_Bay",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2713894152, -2208988800 ],
+  "diffs" : [ 172900, 86500, 100, -86300, -172700, -259100, -345500, -431900, -518300, -604700, -691100, -777500, -863900, 100, -1748, 0 ]
+}, {
+  "tz" : "America/Grand_Turk",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524504958, -2208988800 ],
+  "diffs" : [ 171872, 85472, -928, -87328, -173728, -260128, -346528, -432928, -519328, -605728, -692128, -778528, -864928, -928, 430, 0 ]
+}, {
+  "tz" : "America/Grenada",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Guadeloupe",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Guatemala",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172924, 86524, 124, -86276, -172676, -259076, -345476, -431876, -518276, -604676, -691076, -777476, -863876, 124, 0 ]
+}, {
+  "tz" : "America/Guayaquil",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503280, -2208988800 ],
+  "diffs" : [ 173960, 87560, 1160, -85240, -171640, -258040, -344440, -430840, -517240, -603640, -690040, -776440, -862840, 1160, 840, 0 ]
+}, {
+  "tz" : "America/Guyana",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172360, 85960, -440, -86840, -173240, -259640, -346040, -432440, -518840, -605240, -691640, -778040, -864440, -440, 0 ]
+}, {
+  "tz" : "America/Halifax",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173664, 87264, 864, -85536, -171936, -258336, -344736, -431136, -517536, -603936, -690336, -776736, -863136, 864, 0 ]
+}, {
+  "tz" : "America/Havana",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503608, -2208988800 ],
+  "diffs" : [ 174568, 88168, 1768, -84632, -171032, -257432, -343832, -430232, -516632, -603032, -689432, -775832, -862232, 1768, 1776, 0 ]
+}, {
+  "tz" : "America/Hermosillo",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 174232, 87832, 1432, -84968, -171368, -257768, -344168, -430568, -516968, -603368, -689768, -776168, -862568, 1432, 0 ]
+}, {
+  "tz" : "America/Indiana/Indianapolis",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Knox",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ]
+}, {
+  "tz" : "America/Indiana/Marengo",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175523, 89123, 2723, -83677, -170077, -256477, -342877, -429277, -515677, -602077, -688477, -774877, -861277, 2723, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Petersburg",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175747, 89347, 2947, -83453, -169853, -256253, -342653, -429053, -515453, -601853, -688253, -774653, -861053, 2947, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Tell_City",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 172023, 85623, -777, -87177, -173577, -259977, -346377, -432777, -519177, -605577, -691977, -778377, -864777, -777, 0 ]
+}, {
+  "tz" : "America/Indiana/Vevay",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175216, 88816, 2416, -83984, -170384, -256784, -343184, -429584, -515984, -602384, -688784, -775184, -861584, 2416, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Vincennes",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175807, 89407, 3007, -83393, -169793, -256193, -342593, -428993, -515393, -601793, -688193, -774593, -860993, 3007, 3600, 0 ]
+}, {
+  "tz" : "America/Indiana/Winamac",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 3600, 0 ]
+}, {
+  "tz" : "America/Indianapolis",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "America/Inuvik",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ]
+}, {
+  "tz" : "America/Iqaluit",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 154800, 68400, -18000, -104400, -190800, -277200, -363600, -450000, -536400, -622800, -709200, -795600, -882000, -18000, 0 ]
+}, {
+  "tz" : "America/Jamaica",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 173230, 86830, 430, -85970, -172370, -258770, -345170, -431570, -517970, -604370, -690770, -777170, -863570, 430, 0 ]
+}, {
+  "tz" : "America/Jujuy",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100936, -2208988800 ],
+  "diffs" : [ 177672, 91272, 4872, -81528, -167928, -254328, -340728, -427128, -513528, -599928, -686328, -772728, -859128, 4872, 4608, 0 ]
+}, {
+  "tz" : "America/Juneau",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225223588, -2208988800 ],
+  "diffs" : [ 86261, -139, -86539, -172939, -259339, -345739, -432139, -518539, -604939, -691339, -777739, -864139, -950539, -86539, -139, 0 ]
+}, {
+  "tz" : "America/Kentucky/Louisville",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175382, 88982, 2582, -83818, -170218, -256618, -343018, -429418, -515818, -602218, -688618, -775018, -861418, 2582, 3600, 0 ]
+}, {
+  "tz" : "America/Kentucky/Monticello",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175164, 88764, 2364, -84036, -170436, -256836, -343236, -429636, -516036, -602436, -688836, -775236, -861636, 2364, 3600, 0 ]
+}, {
+  "tz" : "America/Knox_IN",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ]
+}, {
+  "tz" : "America/Kralendijk",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/La_Paz",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174756, 88356, 1956, -84444, -170844, -257244, -343644, -430044, -516444, -602844, -689244, -775644, -862044, 1956, 0 ]
+}, {
+  "tz" : "America/Lima",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503624, -2208988800 ],
+  "diffs" : [ 173292, 86892, 492, -85908, -172308, -258708, -345108, -431508, -517908, -604308, -690708, -777108, -863508, 492, 516, 0 ]
+}, {
+  "tz" : "America/Los_Angeles",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ],
+  "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ]
+}, {
+  "tz" : "America/Louisville",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175382, 88982, 2582, -83818, -170218, -256618, -343018, -429418, -515818, -602218, -688618, -775018, -861418, 2582, 3600, 0 ]
+}, {
+  "tz" : "America/Lower_Princes",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ]
+}, {
+  "tz" : "America/Maceio",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 170572, 84172, -2228, -88628, -175028, -261428, -347828, -434228, -520628, -607028, -693428, -779828, -866228, -2228, 0 ]
+}, {
+  "tz" : "America/Managua",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2524500004, -2208988800 ],
+  "diffs" : [ 171908, 85508, -892, -87292, -173692, -260092, -346492, -432892, -519292, -605692, -692092, -778492, -864892, -892, -888, 0 ]
+}, {
+  "tz" : "America/Manaus",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172804, 86404, 4, -86396, -172796, -259196, -345596, -431996, -518396, -604796, -691196, -777596, -863996, 4, 0 ]
+}, {
+  "tz" : "America/Marigot",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Martinique",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173060, 86660, 260, -86140, -172540, -258940, -345340, -431740, -518140, -604540, -690940, -777340, -863740, 260, 0 ]
+}, {
+  "tz" : "America/Matamoros",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 175200, 88800, 2400, -84000, -170400, -256800, -343200, -429600, -516000, -602400, -688800, -775200, -861600, 2400, 0 ]
+}, {
+  "tz" : "America/Mazatlan",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 173140, 86740, 340, -86060, -172460, -258860, -345260, -431660, -518060, -604460, -690860, -777260, -863660, 340, 0 ]
+}, {
+  "tz" : "America/Mendoza",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2372100092, -2208988800 ],
+  "diffs" : [ 178516, 92116, 5716, -80684, -167084, -253484, -339884, -426284, -512684, -599084, -685484, -771884, -858284, 5716, 4608, 0 ]
+}, {
+  "tz" : "America/Menominee",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2659759773 ],
+  "diffs" : [ 172227, 85827, -573, -86973, -173373, -259773, -346173, -432573, -518973, -605373, -691773, -778173, -864573, -573, 0 ]
+}, {
+  "tz" : "America/Merida",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172708, 86308, -92, -86492, -172892, -259292, -345692, -432092, -518492, -604892, -691292, -777692, -864092, -92, 0 ]
+}, {
+  "tz" : "America/Metlakatla",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225222905, -2208988800 ],
+  "diffs" : [ 85578, -822, -87222, -173622, -260022, -346422, -432822, -519222, -605622, -692022, -778422, -864822, -951222, -87222, -822, 0 ]
+}, {
+  "tz" : "America/Mexico_City",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 174996, 88596, 2196, -84204, -170604, -257004, -343404, -429804, -516204, -602604, -689004, -775404, -861804, 2196, 0 ]
+}, {
+  "tz" : "America/Miquelon",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175480, 89080, 2680, -83720, -170120, -256520, -342920, -429320, -515720, -602120, -688520, -774920, -861320, 2680, 0 ]
+}, {
+  "tz" : "America/Moncton",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2715885652, -2208988800 ],
+  "diffs" : [ 173948, 87548, 1148, -85252, -171652, -258052, -344452, -430852, -517252, -603652, -690052, -776452, -862852, 1148, 3600, 0 ]
+}, {
+  "tz" : "America/Monterrey",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 175276, 88876, 2476, -83924, -170324, -256724, -343124, -429524, -515924, -602324, -688724, -775124, -861524, 2476, 0 ]
+}, {
+  "tz" : "America/Montevideo",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175491, 89091, 2691, -83709, -170109, -256509, -342909, -429309, -515709, -602109, -688509, -774909, -861309, 2691, 0 ]
+}, {
+  "tz" : "America/Montreal",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ],
+  "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ]
+}, {
+  "tz" : "America/Montserrat",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Nassau",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 173370, 86970, 570, -85830, -172230, -258630, -345030, -431430, -517830, -604230, -690630, -777030, -863430, 570, 0 ]
+}, {
+  "tz" : "America/New_York",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800 ],
+  "diffs" : [ 172562, 86162, -238, -86638, -173038, -259438, -345838, -432238, -518638, -605038, -691438, -777838, -864238, -238, 0 ]
+}, {
+  "tz" : "America/Nipigon",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366734016 ],
+  "diffs" : [ 175984, 89584, 3184, -83216, -169616, -256016, -342416, -428816, -515216, -601616, -688016, -774416, -860816, 3184, 0 ]
+}, {
+  "tz" : "America/Nome",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225231025, -2208988800 ],
+  "diffs" : [ 93698, 7298, -79102, -165502, -251902, -338302, -424702, -511102, -597502, -683902, -770302, -856702, -943102, -79102, 7298, 0 ]
+}, {
+  "tz" : "America/Noronha",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Beulah",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ],
+  "diffs" : [ 175627, 89227, 2827, -83573, -169973, -256373, -342773, -429173, -515573, -601973, -688373, -774773, -861173, 2827, 3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/Center",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ],
+  "diffs" : [ 175512, 89112, 2712, -83688, -170088, -256488, -342888, -429288, -515688, -602088, -688488, -774888, -861288, 2712, 3600, 0 ]
+}, {
+  "tz" : "America/North_Dakota/New_Salem",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ],
+  "diffs" : [ 175539, 89139, 2739, -83661, -170061, -256461, -342861, -429261, -515661, -602061, -688461, -774861, -861261, 2739, 3600, 0 ]
+}, {
+  "tz" : "America/Ojinaga",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 172660, 86260, -140, -86540, -172940, -259340, -345740, -432140, -518540, -604940, -691340, -777740, -864140, -140, 0 ]
+}, {
+  "tz" : "America/Panama",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503688, -2208988800 ],
+  "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 1176, 0 ]
+}, {
+  "tz" : "America/Pangnirtung",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 154800, 68400, -18000, -104400, -190800, -277200, -363600, -450000, -536400, -622800, -709200, -795600, -882000, -18000, 0 ]
+}, {
+  "tz" : "America/Paramaribo",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175240, 88840, 2440, -83960, -170360, -256760, -343160, -429560, -515960, -602360, -688760, -775160, -861560, 2440, 0 ]
+}, {
+  "tz" : "America/Phoenix",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 174498, 88098, 1698, -84702, -171102, -257502, -343902, -430302, -516702, -603102, -689502, -775902, -862302, 1698, 0 ]
+}, {
+  "tz" : "America/Port-au-Prince",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503580, -2208988800 ],
+  "diffs" : [ 172160, 85760, -640, -87040, -173440, -259840, -346240, -432640, -519040, -605440, -691840, -778240, -864640, -640, -660, 0 ]
+}, {
+  "tz" : "America/Port_of_Spain",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Porto_Acre",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171072, 84672, -1728, -88128, -174528, -260928, -347328, -433728, -520128, -606528, -692928, -779328, -865728, -1728, 0 ]
+}, {
+  "tz" : "America/Porto_Velho",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173736, 87336, 936, -85464, -171864, -258264, -344664, -431064, -517464, -603864, -690264, -776664, -863064, 936, 0 ]
+}, {
+  "tz" : "America/Puerto_Rico",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ],
+  "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ]
+}, {
+  "tz" : "America/Punta_Arenas",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2524510746, -2208988800 ],
+  "diffs" : [ 179020, 92620, 6220, -80180, -166580, -252980, -339380, -425780, -512180, -598580, -684980, -771380, -857780, 6220, 6166, 0 ]
+}, {
+  "tz" : "America/Rainy_River",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2366732504 ],
+  "diffs" : [ 173896, 87496, 1096, -85304, -171704, -258104, -344504, -430904, -517304, -603704, -690104, -776504, -862904, 1096, 0 ]
+}, {
+  "tz" : "America/Rankin_Inlet",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 151200, 64800, -21600, -108000, -194400, -280800, -367200, -453600, -540000, -626400, -712800, -799200, -885600, -21600, 0 ]
+}, {
+  "tz" : "America/Recife",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 170376, 83976, -2424, -88824, -175224, -261624, -348024, -434424, -520824, -607224, -693624, -780024, -866424, -2424, 0 ]
+}, {
+  "tz" : "America/Regina",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 176316, 89916, 3516, -82884, -169284, -255684, -342084, -428484, -514884, -601284, -687684, -774084, -860484, 3516, 0 ]
+}, {
+  "tz" : "America/Resolute",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 151200, 64800, -21600, -108000, -194400, -280800, -367200, -453600, -540000, -626400, -712800, -799200, -885600, -21600, 0 ]
+}, {
+  "tz" : "America/Rio_Branco",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171072, 84672, -1728, -88128, -174528, -260928, -347328, -433728, -520128, -606528, -692928, -779328, -865728, -1728, 0 ]
+}, {
+  "tz" : "America/Rosario",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 177408, 91008, 4608, -81792, -168192, -254592, -340992, -427392, -513792, -600192, -686592, -772992, -859392, 4608, 0 ]
+}, {
+  "tz" : "America/Santa_Isabel",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "America/Santarem",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175128, 88728, 2328, -84072, -170472, -256872, -343272, -429672, -516072, -602472, -688872, -775272, -861672, 2328, 0 ]
+}, {
+  "tz" : "America/Santiago",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 175366, 88966, 2566, -83834, -170234, -256634, -343034, -429434, -515834, -602234, -688634, -775034, -861434, 2566, 0 ]
+}, {
+  "tz" : "America/Santo_Domingo",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2524507224, -2208988800 ],
+  "diffs" : [ 175176, 88776, 2376, -84024, -170424, -256824, -343224, -429624, -516024, -602424, -688824, -775224, -861624, 2376, 2400, 0 ]
+}, {
+  "tz" : "America/Sao_Paulo",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173188, 86788, 388, -86012, -172412, -258812, -345212, -431612, -518012, -604412, -690812, -777212, -863612, 388, 0 ]
+}, {
+  "tz" : "America/Scoresbysund",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200, -2208988800 ],
+  "diffs" : [ 174472, 88072, 1672, -84728, -171128, -257528, -343928, -430328, -516728, -603128, -689528, -775928, -862328, 1672, 0 ]
+}, {
+  "tz" : "America/Shiprock",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "America/Sitka",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225223800, -2208988800 ],
+  "diffs" : [ 86473, 73, -86327, -172727, -259127, -345527, -431927, -518327, -604727, -691127, -777527, -863927, -950327, -86327, 73, 0 ]
+}, {
+  "tz" : "America/St_Barthelemy",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Johns",
+  "switches" : [ -62135757000, -59006435400, -55850675400, -52694915400, -46383395400, -43227635400, -40071875400, -33760355400, -30604595400, -27448835400, -21137315400, -17981555400, -14825795400, -12219280200, -2208988800 ],
+  "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ]
+}, {
+  "tz" : "America/St_Kitts",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Lucia",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Thomas",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/St_Vincent",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Swift_Current",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 177080, 90680, 4280, -82120, -168520, -254920, -341320, -427720, -514120, -600520, -686920, -773320, -859720, 4280, 0 ]
+}, {
+  "tz" : "America/Tegucigalpa",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172132, 85732, -668, -87068, -173468, -259868, -346268, -432668, -519068, -605468, -691868, -778268, -864668, -668, 0 ]
+}, {
+  "tz" : "America/Thule",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 174908, 88508, 2108, -84292, -170692, -257092, -343492, -429892, -516292, -602692, -689092, -775492, -861892, 2108, 0 ]
+}, {
+  "tz" : "America/Thunder_Bay",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366737380, -2208988800 ],
+  "diffs" : [ 176220, 89820, 3420, -82980, -169380, -255780, -342180, -428580, -514980, -601380, -687780, -774180, -860580, 3420, 3600, 0 ]
+}, {
+  "tz" : "America/Tijuana",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "America/Toronto",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ],
+  "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ]
+}, {
+  "tz" : "America/Tortola",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Vancouver",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2713880852 ],
+  "diffs" : [ 173548, 87148, 748, -85652, -172052, -258452, -344852, -431252, -517652, -604052, -690452, -776852, -863252, 748, 0 ]
+}, {
+  "tz" : "America/Virgin",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ]
+}, {
+  "tz" : "America/Whitehorse",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 176412, 90012, 3612, -82788, -169188, -255588, -341988, -428388, -514788, -601188, -687588, -773988, -860388, 3612, 0 ]
+}, {
+  "tz" : "America/Winnipeg",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2602258284 ],
+  "diffs" : [ 174516, 88116, 1716, -84684, -171084, -257484, -343884, -430284, -516684, -603084, -689484, -775884, -862284, 1716, 0 ]
+}, {
+  "tz" : "America/Yakutat",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225224862, -2208988800 ],
+  "diffs" : [ 87535, 1135, -85265, -171665, -258065, -344465, -430865, -517265, -603665, -690065, -776465, -862865, -949265, -85265, 1135, 0 ]
+}, {
+  "tz" : "America/Yellowknife",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ]
+}, {
+  "tz" : "Antarctica/Casey",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 201600, 115200, 28800, -57600, -144000, -230400, -316800, -403200, -489600, -576000, -662400, -748800, -835200, 28800, 0 ]
+}, {
+  "tz" : "Antarctica/Davis",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 198000, 111600, 25200, -61200, -147600, -234000, -320400, -406800, -493200, -579600, -666000, -752400, -838800, 25200, 0 ]
+}, {
+  "tz" : "Antarctica/DumontDUrville",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ],
+  "diffs" : [ 208800, 122400, 36000, -50400, -136800, -223200, -309600, -396000, -482400, -568800, -655200, -741600, -828000, 36000, 0 ]
+}, {
+  "tz" : "Antarctica/Macquarie",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2214262800, -2208988800 ],
+  "diffs" : [ 212400, 126000, 39600, -46800, -133200, -219600, -306000, -392400, -478800, -565200, -651600, -738000, -824400, 39600, 3600, 0 ]
+}, {
+  "tz" : "Antarctica/Mawson",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 190800, 104400, 18000, -68400, -154800, -241200, -327600, -414000, -500400, -586800, -673200, -759600, -846000, 18000, 0 ]
+}, {
+  "tz" : "Antarctica/McMurdo",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "Antarctica/Palmer",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 162000, 75600, -10800, -97200, -183600, -270000, -356400, -442800, -529200, -615600, -702000, -788400, -874800, -10800, 0 ]
+}, {
+  "tz" : "Antarctica/Rothera",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 162000, 75600, -10800, -97200, -183600, -270000, -356400, -442800, -529200, -615600, -702000, -788400, -874800, -10800, 0 ]
+}, {
+  "tz" : "Antarctica/South_Pole",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "Antarctica/Syowa",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 183600, 97200, 10800, -75600, -162000, -248400, -334800, -421200, -507600, -594000, -680400, -766800, -853200, 10800, 0 ]
+}, {
+  "tz" : "Antarctica/Troll",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Antarctica/Vostok",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 194400, 108000, 21600, -64800, -151200, -237600, -324000, -410400, -496800, -583200, -669600, -756000, -842400, 21600, 0 ]
+}, {
+  "tz" : "Arctic/Longyearbyen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ],
+  "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ]
+}, {
+  "tz" : "Asia/Aden",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ]
+}, {
+  "tz" : "Asia/Almaty",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 175932, 89532, 3132, -83268, -169668, -256068, -342468, -428868, -515268, -601668, -688068, -774468, -860868, 3132, 0 ]
+}, {
+  "tz" : "Asia/Amman",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171376, 84976, -1424, -87824, -174224, -260624, -347024, -433424, -519824, -606224, -692624, -779024, -865424, -1424, 0 ]
+}, {
+  "tz" : "Asia/Anadyr",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Asia/Aqtau",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 178736, 92336, 5936, -80464, -166864, -253264, -339664, -426064, -512464, -598864, -685264, -771664, -858064, 5936, 0 ]
+}, {
+  "tz" : "Asia/Aqtobe",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 177080, 90680, 4280, -82120, -168520, -254920, -341320, -427720, -514120, -600520, -686920, -773320, -859720, 4280, 0 ]
+}, {
+  "tz" : "Asia/Ashgabat",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 0 ]
+}, {
+  "tz" : "Asia/Ashkhabad",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 0 ]
+}, {
+  "tz" : "Asia/Atyrau",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 178336, 91936, 5536, -80864, -167264, -253664, -340064, -426464, -512864, -599264, -685664, -772064, -858464, 5536, 0 ]
+}, {
+  "tz" : "Asia/Baghdad",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2524532404, -2208988800 ],
+  "diffs" : [ 172940, 86540, 140, -86260, -172660, -259060, -345460, -431860, -518260, -604660, -691060, -777460, -863860, 140, 144, 0 ]
+}, {
+  "tz" : "Asia/Bahrain",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 171232, 84832, -1568, -87968, -174368, -260768, -347168, -433568, -519968, -606368, -692768, -779168, -865568, -1568, 0 ]
+}, {
+  "tz" : "Asia/Baku",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175236, 88836, 2436, -83964, -170364, -256764, -343164, -429564, -515964, -602364, -688764, -775164, -861564, 2436, 0 ]
+}, {
+  "tz" : "Asia/Bangkok",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ]
+}, {
+  "tz" : "Asia/Barnaul",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 177900, 91500, 5100, -81300, -167700, -254100, -340500, -426900, -513300, -599700, -686100, -772500, -858900, 5100, 0 ]
+}, {
+  "tz" : "Asia/Beirut",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840149320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Asia/Bishkek",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 176496, 90096, 3696, -82704, -169104, -255504, -341904, -428304, -514704, -601104, -687504, -773904, -860304, 3696, 0 ]
+}, {
+  "tz" : "Asia/Brunei",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174020, 87620, 1220, -85180, -171580, -257980, -344380, -430780, -517180, -603580, -689980, -776380, -862780, 1220, 0 ]
+}, {
+  "tz" : "Asia/Calcutta",
+  "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -3645235808, -3155695330, -2208988800 ],
+  "diffs" : [ 171392, 84992, -1408, -87808, -174208, -260608, -347008, -433408, -519808, -606208, -692608, -779008, -865408, -1408, -1400, 530, 0 ]
+}, {
+  "tz" : "Asia/Chita",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 177968, 91568, 5168, -81232, -167632, -254032, -340432, -426832, -513232, -599632, -686032, -772432, -858832, 5168, 0 ]
+}, {
+  "tz" : "Asia/Choibalsan",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174120, 87720, 1320, -85080, -171480, -257880, -344280, -430680, -517080, -603480, -689880, -776280, -862680, 1320, 0 ]
+}, {
+  "tz" : "Asia/Chongqing",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Chungking",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Colombo",
+  "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -2840160592, -2208988800 ],
+  "diffs" : [ 173436, 87036, 636, -85764, -172164, -258564, -344964, -431364, -517764, -604164, -690564, -776964, -863364, 636, 628, 0 ]
+}, {
+  "tz" : "Asia/Dacca",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2524543700, -2208988800 ],
+  "diffs" : [ 172700, 86300, -100, -86500, -172900, -259300, -345700, -432100, -518500, -604900, -691300, -777700, -864100, -100, 400, 0 ]
+}, {
+  "tz" : "Asia/Damascus",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171288, 84888, -1512, -87912, -174312, -260712, -347112, -433512, -519912, -606312, -692712, -779112, -865512, -1512, 0 ]
+}, {
+  "tz" : "Asia/Dhaka",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2524543700, -2208988800 ],
+  "diffs" : [ 172700, 86300, -100, -86500, -172900, -259300, -345700, -432100, -518500, -604900, -691300, -777700, -864100, -100, 400, 0 ]
+}, {
+  "tz" : "Asia/Dili",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 175060, 88660, 2260, -84140, -170540, -256940, -343340, -429740, -516140, -602540, -688940, -775340, -861740, 2260, 0 ]
+}, {
+  "tz" : "Asia/Dubai",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173928, 87528, 1128, -85272, -171672, -258072, -344472, -430872, -517272, -603672, -690072, -776472, -862872, 1128, 0 ]
+}, {
+  "tz" : "Asia/Dushanbe",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174288, 87888, 1488, -84912, -171312, -257712, -344112, -430512, -516912, -603312, -689712, -776112, -862512, 1488, 0 ]
+}, {
+  "tz" : "Asia/Famagusta",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171852, 85452, -948, -87348, -173748, -260148, -346548, -432948, -519348, -605748, -692148, -778548, -864948, -948, 0 ]
+}, {
+  "tz" : "Asia/Gaza",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171728, 85328, -1072, -87472, -173872, -260272, -346672, -433072, -519472, -605872, -692272, -778672, -865072, -1072, 0 ]
+}, {
+  "tz" : "Asia/Harbin",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Hebron",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171577, 85177, -1223, -87623, -174023, -260423, -346823, -433223, -519623, -606023, -692423, -778823, -865223, -1223, 0 ]
+}, {
+  "tz" : "Asia/Ho_Chi_Minh",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 172400, 86000, -400, -86800, -173200, -259600, -346000, -432400, -518800, -605200, -691600, -778000, -864400, -400, 0 ]
+}, {
+  "tz" : "Asia/Hong_Kong",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174198, 87798, 1398, -85002, -171402, -257802, -344202, -430602, -517002, -603402, -689802, -776202, -862602, 1398, 0 ]
+}, {
+  "tz" : "Asia/Hovd",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 176004, 89604, 3204, -83196, -169596, -255996, -342396, -428796, -515196, -601596, -687996, -774396, -860796, 3204, 0 ]
+}, {
+  "tz" : "Asia/Irkutsk",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 176575, 90175, 3775, -82625, -169025, -255425, -341825, -428225, -514625, -601025, -687425, -773825, -860225, 3775, 0 ]
+}, {
+  "tz" : "Asia/Istanbul",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151536, -2208988800 ],
+  "diffs" : [ 176648, 90248, 3848, -82552, -168952, -255352, -341752, -428152, -514552, -600952, -687352, -773752, -860152, 3848, 3784, 0 ]
+}, {
+  "tz" : "Asia/Jakarta",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 172368, 85968, -432, -86832, -173232, -259632, -346032, -432432, -518832, -605232, -691632, -778032, -864432, -432, 0 ]
+}, {
+  "tz" : "Asia/Jayapura",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 171432, 85032, -1368, -87768, -174168, -260568, -346968, -433368, -519768, -606168, -692568, -778968, -865368, -1368, 0 ]
+}, {
+  "tz" : "Asia/Jerusalem",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
+  "diffs" : [ 171546, 85146, -1254, -87654, -174054, -260454, -346854, -433254, -519654, -606054, -692454, -778854, -865254, -1254, -1240, 0 ]
+}, {
+  "tz" : "Asia/Kabul",
+  "switches" : [ -62135785800, -59006464200, -55850704200, -52694944200, -46383424200, -43227664200, -40071904200, -33760384200, -30604624200, -27448864200, -21137344200, -17981584200, -14825824200, -12219309000, -2524540008, -2208988800 ],
+  "diffs" : [ 172392, 85992, -408, -86808, -173208, -259608, -346008, -432408, -518808, -605208, -691608, -778008, -864408, -408, 1800, 0 ]
+}, {
+  "tz" : "Asia/Kamchatka",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 177924, 91524, 5124, -81276, -167676, -254076, -340476, -426876, -513276, -599676, -686076, -772476, -858876, 5124, 0 ]
+}, {
+  "tz" : "Asia/Karachi",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174708, 88308, 1908, -84492, -170892, -257292, -343692, -430092, -516492, -602892, -689292, -775692, -862092, 1908, 0 ]
+}, {
+  "tz" : "Asia/Kashgar",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Asia/Kathmandu",
+  "switches" : [ -62135790300, -59006468700, -55850708700, -52694948700, -46383428700, -43227668700, -40071908700, -33760388700, -30604628700, -27448868700, -21137348700, -17981588700, -14825828700, -12219313500, -2208988800 ],
+  "diffs" : [ 173024, 86624, 224, -86176, -172576, -258976, -345376, -431776, -518176, -604576, -690976, -777376, -863776, 224, 0 ]
+}, {
+  "tz" : "Asia/Katmandu",
+  "switches" : [ -62135790300, -59006468700, -55850708700, -52694948700, -46383428700, -43227668700, -40071908700, -33760388700, -30604628700, -27448868700, -21137348700, -17981588700, -14825828700, -12219313500, -2208988800 ],
+  "diffs" : [ 173024, 86624, 224, -86176, -172576, -258976, -345376, -431776, -518176, -604576, -690976, -777376, -863776, 224, 0 ]
+}, {
+  "tz" : "Asia/Khandyga",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 172667, 86267, -133, -86533, -172933, -259333, -345733, -432133, -518533, -604933, -691333, -777733, -864133, -133, 0 ]
+}, {
+  "tz" : "Asia/Kolkata",
+  "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -3645235808, -3155695330, -2208988800 ],
+  "diffs" : [ 171392, 84992, -1408, -87808, -174208, -260608, -347008, -433408, -519808, -606208, -692608, -779008, -865408, -1408, -1400, 530, 0 ]
+}, {
+  "tz" : "Asia/Krasnoyarsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 175714, 89314, 2914, -83486, -169886, -256286, -342686, -429086, -515486, -601886, -688286, -774686, -861086, 2914, 0 ]
+}, {
+  "tz" : "Asia/Kuala_Lumpur",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 177194, 90794, 4394, -82006, -168406, -254806, -341206, -427606, -514006, -600406, -686806, -773206, -859606, 4394, 0 ]
+}, {
+  "tz" : "Asia/Kuching",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 175120, 88720, 2320, -84080, -170480, -256880, -343280, -429680, -516080, -602480, -688880, -775280, -861680, 2320, 0 ]
+}, {
+  "tz" : "Asia/Kuwait",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ]
+}, {
+  "tz" : "Asia/Macao",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174350, 87950, 1550, -84850, -171250, -257650, -344050, -430450, -516850, -603250, -689650, -776050, -862450, 1550, 0 ]
+}, {
+  "tz" : "Asia/Macau",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174350, 87950, 1550, -84850, -171250, -257650, -344050, -430450, -516850, -603250, -689650, -776050, -862450, 1550, 0 ]
+}, {
+  "tz" : "Asia/Magadan",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 176208, 89808, 3408, -82992, -169392, -255792, -342192, -428592, -514992, -601392, -687792, -774192, -860592, 3408, 0 ]
+}, {
+  "tz" : "Asia/Makassar",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172944, 86544, 144, -86256, -172656, -259056, -345456, -431856, -518256, -604656, -691056, -777456, -863856, 144, 0 ]
+}, {
+  "tz" : "Asia/Manila",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -3944620800, -2229321840 ],
+  "diffs" : [ 258960, 172560, 86160, -240, -86640, -173040, -259440, -345840, -432240, -518640, -605040, -691440, -777840, 86160, -240, 0 ]
+}, {
+  "tz" : "Asia/Muscat",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173928, 87528, 1128, -85272, -171672, -258072, -344472, -430872, -517272, -603672, -690072, -776472, -862872, 1128, 0 ]
+}, {
+  "tz" : "Asia/Nicosia",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171992, 85592, -808, -87208, -173608, -260008, -346408, -432808, -519208, -605608, -692008, -778408, -864808, -808, 0 ]
+}, {
+  "tz" : "Asia/Novokuznetsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 177072, 90672, 4272, -82128, -168528, -254928, -341328, -427728, -514128, -600528, -686928, -773328, -859728, 4272, 0 ]
+}, {
+  "tz" : "Asia/Novosibirsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 178100, 91700, 5300, -81100, -167500, -253900, -340300, -426700, -513100, -599500, -685900, -772300, -858700, 5300, 0 ]
+}, {
+  "tz" : "Asia/Omsk",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 176790, 90390, 3990, -82410, -168810, -255210, -341610, -428010, -514410, -600810, -687210, -773610, -860010, 3990, 0 ]
+}, {
+  "tz" : "Asia/Oral",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 178476, 92076, 5676, -80724, -167124, -253524, -339924, -426324, -512724, -599124, -685524, -771924, -858324, 5676, 0 ]
+}, {
+  "tz" : "Asia/Phnom_Penh",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ]
+}, {
+  "tz" : "Asia/Pontianak",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 171760, 85360, -1040, -87440, -173840, -260240, -346640, -433040, -519440, -605840, -692240, -778640, -865040, -1040, 0 ]
+}, {
+  "tz" : "Asia/Pyongyang",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 175020, 88620, 2220, -84180, -170580, -256980, -343380, -429780, -516180, -602580, -688980, -775380, -861780, 2220, 0 ]
+}, {
+  "tz" : "Asia/Qatar",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 171232, 84832, -1568, -87968, -174368, -260768, -347168, -433568, -519968, -606368, -692768, -779168, -865568, -1568, 0 ]
+}, {
+  "tz" : "Asia/Qostanay",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 179132, 92732, 6332, -80068, -166468, -252868, -339268, -425668, -512068, -598468, -684868, -771268, -857668, 6332, 0 ]
+}, {
+  "tz" : "Asia/Qyzylorda",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 175088, 88688, 2288, -84112, -170512, -256912, -343312, -429712, -516112, -602512, -688912, -775312, -861712, 2288, 0 ]
+}, {
+  "tz" : "Asia/Rangoon",
+  "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2208988800 ],
+  "diffs" : [ 173113, 86713, 313, -86087, -172487, -258887, -345287, -431687, -518087, -604487, -690887, -777287, -863687, 313, 0 ]
+}, {
+  "tz" : "Asia/Riyadh",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ]
+}, {
+  "tz" : "Asia/Saigon",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 172400, 86000, -400, -86800, -173200, -259600, -346000, -432400, -518800, -605200, -691600, -778000, -864400, -400, 0 ]
+}, {
+  "tz" : "Asia/Sakhalin",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 178152, 91752, 5352, -81048, -167448, -253848, -340248, -426648, -513048, -599448, -685848, -772248, -858648, 5352, 0 ]
+}, {
+  "tz" : "Asia/Samarkand",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174727, 88327, 1927, -84473, -170873, -257273, -343673, -430073, -516473, -602873, -689273, -775673, -862073, 1927, 0 ]
+}, {
+  "tz" : "Asia/Seoul",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 174728, 88328, 1928, -84472, -170872, -257272, -343672, -430072, -516472, -602872, -689272, -775672, -862072, 1928, 0 ]
+}, {
+  "tz" : "Asia/Shanghai",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "Asia/Singapore",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 176675, 90275, 3875, -82525, -168925, -255325, -341725, -428125, -514525, -600925, -687325, -773725, -860125, 3875, 0 ]
+}, {
+  "tz" : "Asia/Srednekolymsk",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 175508, 89108, 2708, -83692, -170092, -256492, -342892, -429292, -515692, -602092, -688492, -774892, -861292, 2708, 0 ]
+}, {
+  "tz" : "Asia/Taipei",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2335248360 ],
+  "diffs" : [ 172440, 86040, -360, -86760, -173160, -259560, -345960, -432360, -518760, -605160, -691560, -777960, -864360, -360, 0 ]
+}, {
+  "tz" : "Asia/Tashkent",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 174169, 87769, 1369, -85031, -171431, -257831, -344231, -430631, -517031, -603431, -689831, -776231, -862631, 1369, 0 ]
+}, {
+  "tz" : "Asia/Tbilisi",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176449, 90049, 3649, -82751, -169151, -255551, -341951, -428351, -514751, -601151, -687551, -773951, -860351, 3649, 0 ]
+}, {
+  "tz" : "Asia/Tel_Aviv",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
+  "diffs" : [ 171546, 85146, -1254, -87654, -174054, -260454, -346854, -433254, -519654, -606054, -692454, -778854, -865254, -1254, -1240, 0 ]
+}, {
+  "tz" : "Asia/Thimbu",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 172884, 86484, 84, -86316, -172716, -259116, -345516, -431916, -518316, -604716, -691116, -777516, -863916, 84, 0 ]
+}, {
+  "tz" : "Asia/Thimphu",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 172884, 86484, 84, -86316, -172716, -259116, -345516, -431916, -518316, -604716, -691116, -777516, -863916, 84, 0 ]
+}, {
+  "tz" : "Asia/Tokyo",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2587712400 ],
+  "diffs" : [ 171661, 85261, -1139, -87539, -173939, -260339, -346739, -433139, -519539, -605939, -692339, -778739, -865139, -1139, 0 ]
+}, {
+  "tz" : "Asia/Tomsk",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 177609, 91209, 4809, -81591, -167991, -254391, -340791, -427191, -513591, -599991, -686391, -772791, -859191, 4809, 0 ]
+}, {
+  "tz" : "Asia/Ujung_Pandang",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172944, 86544, 144, -86256, -172656, -259056, -345456, -431856, -518256, -604656, -691056, -777456, -863856, 144, 0 ]
+}, {
+  "tz" : "Asia/Ulaanbaatar",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 175948, 89548, 3148, -83252, -169652, -256052, -342452, -428852, -515252, -601652, -688052, -774452, -860852, 3148, 0 ]
+}, {
+  "tz" : "Asia/Ulan_Bator",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 175948, 89548, 3148, -83252, -169652, -256052, -342452, -428852, -515252, -601652, -688052, -774452, -860852, 3148, 0 ]
+}, {
+  "tz" : "Asia/Urumqi",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Asia/Ust-Nera",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ],
+  "diffs" : [ 174426, 88026, 1626, -84774, -171174, -257574, -343974, -430374, -516774, -603174, -689574, -775974, -862374, 1626, 0 ]
+}, {
+  "tz" : "Asia/Vientiane",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ],
+  "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ]
+}, {
+  "tz" : "Asia/Vladivostok",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ],
+  "diffs" : [ 177149, 90749, 4349, -82051, -168451, -254851, -341251, -427651, -514051, -600451, -686851, -773251, -859651, 4349, 0 ]
+}, {
+  "tz" : "Asia/Yakutsk",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 174062, 87662, 1262, -85138, -171538, -257938, -344338, -430738, -517138, -603538, -689938, -776338, -862738, 1262, 0 ]
+}, {
+  "tz" : "Asia/Yangon",
+  "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2208988800 ],
+  "diffs" : [ 173113, 86713, 313, -86087, -172487, -258887, -345287, -431687, -518087, -604487, -690887, -777287, -863687, 313, 0 ]
+}, {
+  "tz" : "Asia/Yekaterinburg",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 176247, 89847, 3447, -82953, -169353, -255753, -342153, -428553, -514953, -601353, -687753, -774153, -860553, 3447, 0 ]
+}, {
+  "tz" : "Asia/Yerevan",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176520, 90120, 3720, -82680, -169080, -255480, -341880, -428280, -514680, -601080, -687480, -773880, -860280, 3720, 0 ]
+}, {
+  "tz" : "Atlantic/Azores",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200, -2713907512, -2208988800 ],
+  "diffs" : [ 175360, 88960, 2560, -83840, -170240, -256640, -343040, -429440, -515840, -602240, -688640, -775040, -861440, 2560, 3272, 0 ]
+}, {
+  "tz" : "Atlantic/Bermuda",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173958, 87558, 1158, -85242, -171642, -258042, -344442, -430842, -517242, -603642, -690042, -776442, -862842, 1158, 0 ]
+}, {
+  "tz" : "Atlantic/Canary",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 176496, 90096, 3696, -82704, -169104, -255504, -341904, -428304, -514704, -601104, -687504, -773904, -860304, 3696, 0 ]
+}, {
+  "tz" : "Atlantic/Cape_Verde",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200, -2208988800 ],
+  "diffs" : [ 174844, 88444, 2044, -84356, -170756, -257156, -343556, -429956, -516356, -602756, -689156, -775556, -861956, 2044, 0 ]
+}, {
+  "tz" : "Atlantic/Faeroe",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 174424, 88024, 1624, -84776, -171176, -257576, -343976, -430376, -516776, -603176, -689576, -775976, -862376, 1624, 0 ]
+}, {
+  "tz" : "Atlantic/Faroe",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 174424, 88024, 1624, -84776, -171176, -257576, -343976, -430376, -516776, -603176, -689576, -775976, -862376, 1624, 0 ]
+}, {
+  "tz" : "Atlantic/Jan_Mayen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ],
+  "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ]
+}, {
+  "tz" : "Atlantic/Madeira",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 176856, 90456, 4056, -82344, -168744, -255144, -341544, -427944, -514344, -600744, -687144, -773544, -859944, 4056, 0 ]
+}, {
+  "tz" : "Atlantic/Reykjavik",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 178080, 91680, 5280, -81120, -167520, -253920, -340320, -426720, -513120, -599520, -685920, -772320, -858720, 5280, 0 ]
+}, {
+  "tz" : "Atlantic/South_Georgia",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2524512832 ],
+  "diffs" : [ 174368, 87968, 1568, -84832, -171232, -257632, -344032, -430432, -516832, -603232, -689632, -776032, -862432, 1568, 0 ]
+}, {
+  "tz" : "Atlantic/St_Helena",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ]
+}, {
+  "tz" : "Atlantic/Stanley",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 175884, 89484, 3084, -83316, -169716, -256116, -342516, -428916, -515316, -601716, -688116, -774516, -860916, 3084, 0 ]
+}, {
+  "tz" : "Australia/ACT",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/Adelaide",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364111860, -2230189200 ],
+  "diffs" : [ 173740, 87340, 940, -85460, -171860, -258260, -344660, -431060, -517460, -603860, -690260, -776660, -863060, 940, 1800, 0 ]
+}, {
+  "tz" : "Australia/Brisbane",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2366791928 ],
+  "diffs" : [ 172072, 85672, -728, -87128, -173528, -259928, -346328, -432728, -519128, -605528, -691928, -778328, -864728, -728, 0 ]
+}, {
+  "tz" : "Australia/Broken_Hill",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364108948, -2314953000, -2230189200 ],
+  "diffs" : [ 173052, 86652, 252, -86148, -172548, -258948, -345348, -431748, -518148, -604548, -690948, -777348, -863748, 252, -1800, 1800, 0 ]
+}, {
+  "tz" : "Australia/Canberra",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/Currie",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345794528 ],
+  "diffs" : [ 174272, 87872, 1472, -84928, -171328, -257728, -344128, -430528, -516928, -603328, -689728, -776128, -862528, 1472, 0 ]
+}, {
+  "tz" : "Australia/Darwin",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364110000, -2230189200 ],
+  "diffs" : [ 175600, 89200, 2800, -83600, -170000, -256400, -342800, -429200, -515600, -602000, -688400, -774800, -861200, 2800, 1800, 0 ]
+}, {
+  "tz" : "Australia/Eucla",
+  "switches" : [ -62135801100, -59006479500, -55850719500, -52694959500, -46383439500, -43227679500, -40071919500, -33760399500, -30604639500, -27448879500, -21137359500, -17981599500, -14825839500, -12219324300, -2337928528 ],
+  "diffs" : [ 173372, 86972, 572, -85828, -172228, -258628, -345028, -431428, -517828, -604228, -690628, -777028, -863428, 572, 0 ]
+}, {
+  "tz" : "Australia/Hobart",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345795356 ],
+  "diffs" : [ 173444, 87044, 644, -85756, -172156, -258556, -344956, -431356, -517756, -604156, -690556, -776956, -863356, 644, 0 ]
+}, {
+  "tz" : "Australia/LHI",
+  "switches" : [ -62135807400, -59006485800, -55850725800, -52694965800, -46383445800, -43227685800, -40071925800, -33760405800, -30604645800, -27448885800, -21137365800, -17981605800, -14825845800, -12219330600, -2364116780, -2208988800 ],
+  "diffs" : [ 172420, 86020, -380, -86780, -173180, -259580, -345980, -432380, -518780, -605180, -691580, -777980, -864380, -380, 1800, 0 ]
+}, {
+  "tz" : "Australia/Lindeman",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2366790956 ],
+  "diffs" : [ 173044, 86644, 244, -86156, -172556, -258956, -345356, -431756, -518156, -604556, -690956, -777356, -863756, 244, 0 ]
+}, {
+  "tz" : "Australia/Lord_Howe",
+  "switches" : [ -62135807400, -59006485800, -55850725800, -52694965800, -46383445800, -43227685800, -40071925800, -33760405800, -30604645800, -27448885800, -21137365800, -17981605800, -14825845800, -12219330600, -2364116780, -2208988800 ],
+  "diffs" : [ 172420, 86020, -380, -86780, -173180, -259580, -345980, -432380, -518780, -605180, -691580, -777980, -864380, -380, 1800, 0 ]
+}, {
+  "tz" : "Australia/Melbourne",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364111592 ],
+  "diffs" : [ 174008, 87608, 1208, -85192, -171592, -257992, -344392, -430792, -517192, -603592, -689992, -776392, -862792, 1208, 0 ]
+}, {
+  "tz" : "Australia/NSW",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/North",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364110000, -2230189200 ],
+  "diffs" : [ 175600, 89200, 2800, -83600, -170000, -256400, -342800, -429200, -515600, -602000, -688400, -774800, -861200, 2800, 1800, 0 ]
+}, {
+  "tz" : "Australia/Perth",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2337925404 ],
+  "diffs" : [ 173796, 87396, 996, -85404, -171804, -258204, -344604, -431004, -517404, -603804, -690204, -776604, -863004, 996, 0 ]
+}, {
+  "tz" : "Australia/Queensland",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2366791928 ],
+  "diffs" : [ 172072, 85672, -728, -87128, -173528, -259928, -346328, -432728, -519128, -605528, -691928, -778328, -864728, -728, 0 ]
+}, {
+  "tz" : "Australia/South",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364111860, -2230189200 ],
+  "diffs" : [ 173740, 87340, 940, -85460, -171860, -258260, -344660, -431060, -517460, -603860, -690260, -776660, -863060, 940, 1800, 0 ]
+}, {
+  "tz" : "Australia/Sydney",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364113092 ],
+  "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ]
+}, {
+  "tz" : "Australia/Tasmania",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345795356 ],
+  "diffs" : [ 173444, 87044, 644, -85756, -172156, -258556, -344956, -431356, -517756, -604156, -690556, -776956, -863356, 644, 0 ]
+}, {
+  "tz" : "Australia/Victoria",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2364111592 ],
+  "diffs" : [ 174008, 87608, 1208, -85192, -171592, -257992, -344392, -430792, -517192, -603592, -689992, -776392, -862792, 1208, 0 ]
+}, {
+  "tz" : "Australia/West",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2337925404 ],
+  "diffs" : [ 173796, 87396, 996, -85404, -171804, -258204, -344604, -431004, -517404, -603804, -690204, -776604, -863004, 996, 0 ]
+}, {
+  "tz" : "Australia/Yancowinna",
+  "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364108948, -2314953000, -2230189200 ],
+  "diffs" : [ 173052, 86652, 252, -86148, -172548, -258948, -345348, -431748, -518148, -604548, -690948, -777348, -863748, 252, -1800, 1800, 0 ]
+}, {
+  "tz" : "Brazil/Acre",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 171072, 84672, -1728, -88128, -174528, -260928, -347328, -433728, -520128, -606528, -692928, -779328, -865728, -1728, 0 ]
+}, {
+  "tz" : "Brazil/DeNoronha",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2208988800 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Brazil/East",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ],
+  "diffs" : [ 173188, 86788, 388, -86012, -172412, -258812, -345212, -431612, -518012, -604412, -690812, -777212, -863612, 388, 0 ]
+}, {
+  "tz" : "Brazil/West",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 172804, 86404, 4, -86396, -172796, -259196, -345596, -431996, -518396, -604796, -691196, -777596, -863996, 4, 0 ]
+}, {
+  "tz" : "CET",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "CST6CDT",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Canada/Atlantic",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 173664, 87264, 864, -85536, -171936, -258336, -344736, -431136, -517536, -603936, -690336, -776736, -863136, 864, 0 ]
+}, {
+  "tz" : "Canada/Central",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2602258284 ],
+  "diffs" : [ 174516, 88116, 1716, -84684, -171084, -257484, -343884, -430284, -516684, -603084, -689484, -775884, -862284, 1716, 0 ]
+}, {
+  "tz" : "Canada/Eastern",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ],
+  "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ]
+}, {
+  "tz" : "Canada/Mountain",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 174832, 88432, 2032, -84368, -170768, -257168, -343568, -429968, -516368, -602768, -689168, -775568, -861968, 2032, 0 ]
+}, {
+  "tz" : "Canada/Newfoundland",
+  "switches" : [ -62135757000, -59006435400, -55850675400, -52694915400, -46383395400, -43227635400, -40071875400, -33760355400, -30604595400, -27448835400, -21137315400, -17981555400, -14825795400, -12219280200, -2208988800 ],
+  "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ]
+}, {
+  "tz" : "Canada/Pacific",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2713880852 ],
+  "diffs" : [ 173548, 87148, 748, -85652, -172052, -258452, -344852, -431252, -517652, -604052, -690452, -776852, -863252, 748, 0 ]
+}, {
+  "tz" : "Canada/Saskatchewan",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 176316, 89916, 3516, -82884, -169284, -255684, -342084, -428484, -514884, -601284, -687684, -774084, -860484, 3516, 0 ]
+}, {
+  "tz" : "Canada/Yukon",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 176412, 90012, 3612, -82788, -169188, -255588, -341988, -428388, -514788, -601188, -687588, -773988, -860388, 3612, 0 ]
+}, {
+  "tz" : "Chile/Continental",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ],
+  "diffs" : [ 175366, 88966, 2566, -83834, -170234, -256634, -343034, -429434, -515834, -602234, -688634, -775034, -861434, 2566, 0 ]
+}, {
+  "tz" : "Chile/EasterIsland",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 177448, 91048, 4648, -81752, -168152, -254552, -340952, -427352, -513752, -600152, -686552, -772952, -859352, 4648, 0 ]
+}, {
+  "tz" : "Cuba",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503608, -2208988800 ],
+  "diffs" : [ 174568, 88168, 1768, -84632, -171032, -257432, -343832, -430232, -516632, -603032, -689432, -775832, -862232, 1768, 1776, 0 ]
+}, {
+  "tz" : "EET",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "EST5EDT",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Egypt",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ]
+}, {
+  "tz" : "Eire",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2821651221, -2208988800 ],
+  "diffs" : [ 174300, 87900, 1500, -84900, -171300, -257700, -344100, -430500, -516900, -603300, -689700, -776100, -862500, 1500, 1521, 0 ]
+}, {
+  "tz" : "Etc/GMT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+1",
+  "switches" : [ -62135766000, -59006444400, -55850684400, -52694924400, -46383404400, -43227644400, -40071884400, -33760364400, -30604604400, -27448844400, -21137324400, -17981564400, -14825804400, -12219289200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+10",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+11",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+12",
+  "switches" : [ -62135726400, -59006404800, -55850644800, -52694884800, -46383364800, -43227604800, -40071844800, -33760324800, -30604564800, -27448804800, -21137284800, -17981524800, -14825764800, -12219249600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+2",
+  "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+3",
+  "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+4",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+5",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+6",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+7",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+8",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT+9",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-1",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-10",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-11",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-12",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-13",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-14",
+  "switches" : [ -62135820000, -59006498400, -55850738400, -52694978400, -46383458400, -43227698400, -40071938400, -33760418400, -30604658400, -27448898400, -21137378400, -17981618400, -14825858400, -12219343200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-2",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-3",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-4",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-5",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-6",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-7",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-8",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT-9",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/GMT0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/Greenwich",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/UCT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/UTC",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/Universal",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Etc/Zulu",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Europe/Amsterdam",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175228, 88828, 2428, -83972, -170372, -256772, -343172, -429572, -515972, -602372, -688772, -775172, -861572, 2428, 0 ]
+}, {
+  "tz" : "Europe/Andorra",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 176036, 89636, 3236, -83164, -169564, -255964, -342364, -428764, -515164, -601564, -687964, -774364, -860764, 3236, 0 ]
+}, {
+  "tz" : "Europe/Astrakhan",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175668, 89268, 2868, -83532, -169932, -256332, -342732, -429132, -515532, -601932, -688332, -774732, -861132, 2868, 0 ]
+}, {
+  "tz" : "Europe/Athens",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174308, 87908, 1508, -84892, -171292, -257692, -344092, -430492, -516892, -603292, -689692, -776092, -862492, 1508, 0 ]
+}, {
+  "tz" : "Europe/Belfast",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Belgrade",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Berlin",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ],
+  "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ]
+}, {
+  "tz" : "Europe/Bratislava",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2469401864 ],
+  "diffs" : [ 172936, 86536, 136, -86264, -172664, -259064, -345464, -431864, -518264, -604664, -691064, -777464, -863864, 136, 0 ]
+}, {
+  "tz" : "Europe/Brussels",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2450998800, -2208988800 ],
+  "diffs" : [ 175350, 88950, 2550, -83850, -170250, -256650, -343050, -429450, -515850, -602250, -688650, -775050, -861450, 2550, 3600, 0 ]
+}, {
+  "tz" : "Europe/Bucharest",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 173736, 87336, 936, -85464, -171864, -258264, -344664, -431064, -517464, -603864, -690264, -776664, -863064, 936, 0 ]
+}, {
+  "tz" : "Europe/Budapest",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2500938980 ],
+  "diffs" : [ 171820, 85420, -980, -87380, -173780, -260180, -346580, -432980, -519380, -605780, -692180, -778580, -864980, -980, 0 ]
+}, {
+  "tz" : "Europe/Busingen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -3675200662, -2385246586 ],
+  "diffs" : [ 174352, 87952, 1552, -84848, -171248, -257648, -344048, -430448, -516848, -603248, -689648, -776048, -862448, 1552, 1814, 0 ]
+}, {
+  "tz" : "Europe/Chisinau",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148020, -2208988800 ],
+  "diffs" : [ 173080, 86680, 280, -86120, -172520, -258920, -345320, -431720, -518120, -604520, -690920, -777320, -863720, 280, 300, 0 ]
+}, {
+  "tz" : "Europe/Copenhagen",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2398294220 ],
+  "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ]
+}, {
+  "tz" : "Europe/Dublin",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2821651221, -2208988800 ],
+  "diffs" : [ 174300, 87900, 1500, -84900, -171300, -257700, -344100, -430500, -516900, -603300, -689700, -776100, -862500, 1500, 1521, 0 ]
+}, {
+  "tz" : "Europe/Gibraltar",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2821653516, -2208988800 ],
+  "diffs" : [ 177684, 91284, 4884, -81516, -167916, -254316, -340716, -427116, -513516, -599916, -686316, -772716, -859116, 4884, 3600, 0 ]
+}, {
+  "tz" : "Europe/Guernsey",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Helsinki",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174011, 87611, 1211, -85189, -171589, -257989, -344389, -430789, -517189, -603589, -689989, -776389, -862789, 1211, 0 ]
+}, {
+  "tz" : "Europe/Isle_of_Man",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Istanbul",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151536, -2208988800 ],
+  "diffs" : [ 176648, 90248, 3848, -82552, -168952, -255352, -341752, -428152, -514552, -600952, -687352, -773752, -860152, 3848, 3784, 0 ]
+}, {
+  "tz" : "Europe/Jersey",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Kaliningrad",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2422059720, -2208988800 ],
+  "diffs" : [ 175080, 88680, 2280, -84120, -170520, -256920, -343320, -429720, -516120, -602520, -688920, -775320, -861720, 2280, 3600, 0 ]
+}, {
+  "tz" : "Europe/Kiev",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 172676, 86276, -124, -86524, -172924, -259324, -345724, -432124, -518524, -604924, -691324, -777724, -864124, -124, 0 ]
+}, {
+  "tz" : "Europe/Kirov",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 171672, 85272, -1128, -87528, -173928, -260328, -346728, -433128, -519528, -605928, -692328, -778728, -865128, -1128, 0 ]
+}, {
+  "tz" : "Europe/Lisbon",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175005, 88605, 2205, -84195, -170595, -256995, -343395, -429795, -516195, -602595, -688995, -775395, -861795, 2205, 0 ]
+}, {
+  "tz" : "Europe/Ljubljana",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/London",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "Europe/Luxembourg",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 174924, 88524, 2124, -84276, -170676, -257076, -343476, -429876, -516276, -602676, -689076, -775476, -861876, 2124, 0 ]
+}, {
+  "tz" : "Europe/Madrid",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 177284, 90884, 4484, -81916, -168316, -254716, -341116, -427516, -513916, -600316, -686716, -773116, -859516, 4484, 0 ]
+}, {
+  "tz" : "Europe/Malta",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403478684 ],
+  "diffs" : [ 172916, 86516, 116, -86284, -172684, -259084, -345484, -431884, -518284, -604684, -691084, -777484, -863884, 116, 0 ]
+}, {
+  "tz" : "Europe/Mariehamn",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174011, 87611, 1211, -85189, -171589, -257989, -344389, -430789, -517189, -603589, -689989, -776389, -862789, 1211, 0 ]
+}, {
+  "tz" : "Europe/Minsk",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151616, -2208988800 ],
+  "diffs" : [ 176984, 90584, 4184, -82216, -168616, -255016, -341416, -427816, -514216, -600616, -687016, -773416, -859816, 4184, 4200, 0 ]
+}, {
+  "tz" : "Europe/Monaco",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486683211, -2208988800 ],
+  "diffs" : [ 174628, 88228, 1828, -84572, -170972, -257372, -343772, -430172, -516572, -602972, -689372, -775772, -862172, 1828, 3039, 0 ]
+}, {
+  "tz" : "Europe/Moscow",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174583, 88183, 1783, -84617, -171017, -257417, -343817, -430217, -516617, -603017, -689417, -775817, -862217, 1783, 0 ]
+}, {
+  "tz" : "Europe/Nicosia",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 171992, 85592, -808, -87208, -173608, -260008, -346408, -432808, -519208, -605608, -692008, -778408, -864808, -808, 0 ]
+}, {
+  "tz" : "Europe/Oslo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ],
+  "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ]
+}, {
+  "tz" : "Europe/Paris",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 175839, 89439, 3039, -83361, -169761, -256161, -342561, -428961, -515361, -601761, -688161, -774561, -860961, 3039, 0 ]
+}, {
+  "tz" : "Europe/Podgorica",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Prague",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2469401864 ],
+  "diffs" : [ 172936, 86536, 136, -86264, -172664, -259064, -345464, -431864, -518264, -604664, -691064, -777464, -863864, 136, 0 ]
+}, {
+  "tz" : "Europe/Riga",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174206, 87806, 1406, -84994, -171394, -257794, -344194, -430594, -516994, -603394, -689794, -776194, -862594, 1406, 0 ]
+}, {
+  "tz" : "Europe/Rome",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403565200 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Europe/Samara",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175180, 88780, 2380, -84020, -170420, -256820, -343220, -429620, -516020, -602420, -688820, -775220, -861620, 2380, 0 ]
+}, {
+  "tz" : "Europe/San_Marino",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403565200 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Europe/Sarajevo",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Saratov",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176142, 89742, 3342, -83058, -169458, -255858, -342258, -428658, -515058, -601458, -687858, -774258, -860658, 3342, 0 ]
+}, {
+  "tz" : "Europe/Simferopol",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151624, -2208988800 ],
+  "diffs" : [ 175416, 89016, 2616, -83784, -170184, -256584, -342984, -429384, -515784, -602184, -688584, -774984, -861384, 2616, 2640, 0 ]
+}, {
+  "tz" : "Europe/Skopje",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Sofia",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840146580, -2369527016 ],
+  "diffs" : [ 174404, 88004, 1604, -84796, -171196, -257596, -343996, -430396, -516796, -603196, -689596, -775996, -862396, 1604, 184, 0 ]
+}, {
+  "tz" : "Europe/Stockholm",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2871681118, -2208992414 ],
+  "diffs" : [ 172068, 85668, -732, -87132, -173532, -259932, -346332, -432732, -519132, -605532, -691932, -778332, -864732, -732, -14, 0 ]
+}, {
+  "tz" : "Europe/Tallinn",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 174060, 87660, 1260, -85140, -171540, -257940, -344340, -430740, -517140, -603540, -689940, -776340, -862740, 1260, 0 ]
+}, {
+  "tz" : "Europe/Tirane",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 171640, 85240, -1160, -87560, -173960, -260360, -346760, -433160, -519560, -605960, -692360, -778760, -865160, -1160, 0 ]
+}, {
+  "tz" : "Europe/Tiraspol",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148020, -2208988800 ],
+  "diffs" : [ 173080, 86680, 280, -86120, -172520, -258920, -345320, -431720, -518120, -604520, -690920, -777320, -863720, 280, 300, 0 ]
+}, {
+  "tz" : "Europe/Ulyanovsk",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Europe/Uzhgorod",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2500943352, -2208988800 ],
+  "diffs" : [ 174648, 88248, 1848, -84552, -170952, -257352, -343752, -430152, -516552, -602952, -689352, -775752, -862152, 1848, 3600, 0 ]
+}, {
+  "tz" : "Europe/Vaduz",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -3675200662, -2385246586 ],
+  "diffs" : [ 174352, 87952, 1552, -84848, -171248, -257648, -344048, -430448, -516848, -603248, -689648, -776048, -862448, 1552, 1814, 0 ]
+}, {
+  "tz" : "Europe/Vatican",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2403565200 ],
+  "diffs" : [ 173404, 87004, 604, -85796, -172196, -258596, -344996, -431396, -517796, -604196, -690596, -776996, -863396, 604, 0 ]
+}, {
+  "tz" : "Europe/Vienna",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422055121 ],
+  "diffs" : [ 172479, 86079, -321, -86721, -173121, -259521, -345921, -432321, -518721, -605121, -691521, -777921, -864321, -321, 0 ]
+}, {
+  "tz" : "Europe/Vilnius",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840149036, -2208988800 ],
+  "diffs" : [ 173924, 87524, 1124, -85276, -171676, -258076, -344476, -430876, -517276, -603676, -690076, -776476, -862876, 1124, 2160, 0 ]
+}, {
+  "tz" : "Europe/Volgograd",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 176540, 90140, 3740, -82660, -169060, -255460, -341860, -428260, -514660, -601060, -687460, -773860, -860260, 3740, 0 ]
+}, {
+  "tz" : "Europe/Warsaw",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 171360, 84960, -1440, -87840, -174240, -260640, -347040, -433440, -519840, -606240, -692640, -779040, -865440, -1440, 0 ]
+}, {
+  "tz" : "Europe/Zagreb",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2713915320 ],
+  "diffs" : [ 171480, 85080, -1320, -87720, -174120, -260520, -346920, -433320, -519720, -606120, -692520, -778920, -865320, -1320, 0 ]
+}, {
+  "tz" : "Europe/Zaporozhye",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148040, -2208988800 ],
+  "diffs" : [ 171560, 85160, -1240, -87640, -174040, -260440, -346840, -433240, -519640, -606040, -692440, -778840, -865240, -1240, -1200, 0 ]
+}, {
+  "tz" : "Europe/Zurich",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -3675200662, -2385246586 ],
+  "diffs" : [ 174352, 87952, 1552, -84848, -171248, -257648, -344048, -430448, -516848, -603248, -689648, -776048, -862448, 1552, 1814, 0 ]
+}, {
+  "tz" : "GB",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "GB-Eire",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -3852662325 ],
+  "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ]
+}, {
+  "tz" : "GMT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "GMT0",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Greenwich",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Hongkong",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 174198, 87798, 1398, -85002, -171402, -257802, -344202, -430602, -517002, -603402, -689802, -776202, -862602, 1398, 0 ]
+}, {
+  "tz" : "Iceland",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 178080, 91680, 5280, -81120, -167520, -253920, -340320, -426720, -513120, -599520, -685920, -772320, -858720, 5280, 0 ]
+}, {
+  "tz" : "Indian/Antananarivo",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Indian/Chagos",
+  "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ],
+  "diffs" : [ 177020, 90620, 4220, -82180, -168580, -254980, -341380, -427780, -514180, -600580, -686980, -773380, -859780, 4220, 0 ]
+}, {
+  "tz" : "Indian/Christmas",
+  "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2364102172 ],
+  "diffs" : [ 172628, 86228, -172, -86572, -172972, -259372, -345772, -432172, -518572, -604972, -691372, -777772, -864172, -172, 0 ]
+}, {
+  "tz" : "Indian/Cocos",
+  "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2209012060 ],
+  "diffs" : [ 172940, 86540, 140, -86260, -172660, -259060, -345460, -431860, -518260, -604660, -691060, -777460, -863860, 140, 0 ]
+}, {
+  "tz" : "Indian/Comoro",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Indian/Kerguelen",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 190800, 104400, 18000, -68400, -154800, -241200, -327600, -414000, -500400, -586800, -673200, -759600, -846000, 18000, 0 ]
+}, {
+  "tz" : "Indian/Mahe",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173892, 87492, 1092, -85308, -171708, -258108, -344508, -430908, -517308, -603708, -690108, -776508, -862908, 1092, 0 ]
+}, {
+  "tz" : "Indian/Maldives",
+  "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ],
+  "diffs" : [ 173160, 86760, 360, -86040, -172440, -258840, -345240, -431640, -518040, -604440, -690840, -777240, -863640, 360, 0 ]
+}, {
+  "tz" : "Indian/Mauritius",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173400, 87000, 600, -85800, -172200, -258600, -345000, -431400, -517800, -604200, -690600, -777000, -863400, 600, 0 ]
+}, {
+  "tz" : "Indian/Mayotte",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ]
+}, {
+  "tz" : "Indian/Reunion",
+  "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
+  "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 0 ]
+}, {
+  "tz" : "Israel",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
+  "diffs" : [ 171546, 85146, -1254, -87654, -174054, -260454, -346854, -433254, -519654, -606054, -692454, -778854, -865254, -1254, -1240, 0 ]
+}, {
+  "tz" : "Jamaica",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 173230, 86830, 430, -85970, -172370, -258770, -345170, -431570, -517970, -604370, -690770, -777170, -863570, 430, 0 ]
+}, {
+  "tz" : "Japan",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2587712400 ],
+  "diffs" : [ 171661, 85261, -1139, -87539, -173939, -260339, -346739, -433139, -519539, -605939, -692339, -778739, -865139, -1139, 0 ]
+}, {
+  "tz" : "Kwajalein",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 175840, 89440, 3040, -83360, -169760, -256160, -342560, -428960, -515360, -601760, -688160, -774560, -860960, 3040, 0 ]
+}, {
+  "tz" : "Libya",
+  "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
+  "diffs" : [ 176836, 90436, 4036, -82364, -168764, -255164, -341564, -427964, -514364, -600764, -687164, -773564, -859964, 4036, 0 ]
+}, {
+  "tz" : "MET",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "MST7MDT",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Mexico/BajaNorte",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 172084, 85684, -716, -87116, -173516, -259916, -346316, -432716, -519116, -605516, -691916, -778316, -864716, -716, 0 ]
+}, {
+  "tz" : "Mexico/BajaSur",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ],
+  "diffs" : [ 173140, 86740, 340, -86060, -172460, -258860, -345260, -431660, -518060, -604460, -690860, -777260, -863660, 340, 0 ]
+}, {
+  "tz" : "Mexico/General",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 174996, 88596, 2196, -84204, -170604, -257004, -343404, -429804, -516204, -602604, -689004, -775404, -861804, 2196, 0 ]
+}, {
+  "tz" : "NZ",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "NZ-CHAT",
+  "switches" : [ -62135815500, -59006493900, -55850733900, -52694973900, -46383453900, -43227693900, -40071933900, -33760413900, -30604653900, -27448893900, -21137373900, -17981613900, -14825853900, -12219338700, -3192439428, -2208988800 ],
+  "diffs" : [ 174672, 88272, 1872, -84528, -170928, -257328, -343728, -430128, -516528, -602928, -689328, -775728, -862128, 1872, 1800, 0 ]
+}, {
+  "tz" : "Navajo",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "PRC",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 172457, 86057, -343, -86743, -173143, -259543, -345943, -432343, -518743, -605143, -691543, -777943, -864343, -343, 0 ]
+}, {
+  "tz" : "PST8PDT",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Pacific/Apia",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2445512400, -2208988800 ],
+  "diffs" : [ 174416, 88016, 1616, -84784, -171184, -257584, -343984, -430384, -516784, -603184, -689584, -775984, -862384, 1616, 88016, 0 ]
+}, {
+  "tz" : "Pacific/Auckland",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -3192437344, -2208988800 ],
+  "diffs" : [ 174056, 87656, 1256, -85144, -171544, -257944, -344344, -430744, -517144, -603544, -689944, -776344, -862744, 1256, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Bougainville",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2840182424, -2366794112, -2208988800 ],
+  "diffs" : [ 175064, 88664, 2264, -84136, -170536, -256936, -343336, -429736, -516136, -602536, -688936, -775336, -861736, 2264, 4288, 3600, 0 ]
+}, {
+  "tz" : "Pacific/Chatham",
+  "switches" : [ -62135815500, -59006493900, -55850733900, -52694973900, -46383453900, -43227693900, -40071933900, -33760413900, -30604653900, -27448893900, -21137373900, -17981613900, -14825853900, -12219338700, -3192439428, -2208988800 ],
+  "diffs" : [ 174672, 88272, 1872, -84528, -170928, -257328, -343728, -430128, -516528, -602928, -689328, -775728, -862128, 1872, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Chuuk",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ]
+}, {
+  "tz" : "Pacific/Easter",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 177448, 91048, 4648, -81752, -168152, -254552, -340952, -427352, -513752, -600152, -686552, -772952, -859352, 4648, 0 ]
+}, {
+  "tz" : "Pacific/Efate",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 172004, 85604, -796, -87196, -173596, -259996, -346396, -432796, -519196, -605596, -691996, -778396, -864796, -796, 0 ]
+}, {
+  "tz" : "Pacific/Enderbury",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ],
+  "diffs" : [ 260660, 174260, 87860, 1460, -84940, -171340, -257740, -344140, -430540, -516940, -603340, -689740, -776140, 87860, 0 ]
+}, {
+  "tz" : "Pacific/Fakaofo",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ],
+  "diffs" : [ 260696, 174296, 87896, 1496, -84904, -171304, -257704, -344104, -430504, -516904, -603304, -689704, -776104, 87896, 0 ]
+}, {
+  "tz" : "Pacific/Fiji",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ]
+}, {
+  "tz" : "Pacific/Funafuti",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 172988, 86588, 188, -86212, -172612, -259012, -345412, -431812, -518212, -604612, -691012, -777412, -863812, 188, 0 ]
+}, {
+  "tz" : "Pacific/Galapagos",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ],
+  "diffs" : [ 172704, 86304, -96, -86496, -172896, -259296, -345696, -432096, -518496, -604896, -691296, -777696, -864096, -96, 0 ]
+}, {
+  "tz" : "Pacific/Gambier",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -2208988800 ],
+  "diffs" : [ 172788, 86388, -12, -86412, -172812, -259212, -345612, -432012, -518412, -604812, -691212, -777612, -864012, -12, 0 ]
+}, {
+  "tz" : "Pacific/Guadalcanal",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 174012, 87612, 1212, -85188, -171588, -257988, -344388, -430788, -517188, -603588, -689988, -776388, -862788, 1212, 0 ]
+}, {
+  "tz" : "Pacific/Guam",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 260460, 174060, 87660, 1260, -85140, -171540, -257940, -344340, -430740, -517140, -603540, -689940, -776340, 87660, 1260, 0 ]
+}, {
+  "tz" : "Pacific/Honolulu",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ],
+  "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Johnston",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ],
+  "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ]
+}, {
+  "tz" : "Pacific/Kiritimati",
+  "switches" : [ -62135820000, -59006498400, -55850738400, -52694978400, -46383458400, -43227698400, -40071938400, -33760418400, -30604658400, -27448898400, -21137378400, -17981618400, -14825858400, -12219343200, -2208988800 ],
+  "diffs" : [ 260960, 174560, 88160, 1760, -84640, -171040, -257440, -343840, -430240, -516640, -603040, -689440, -775840, 88160, 0 ]
+}, {
+  "tz" : "Pacific/Kosrae",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ],
+  "diffs" : [ 259684, 173284, 86884, 484, -85916, -172316, -258716, -345116, -431516, -517916, -604316, -690716, -777116, 86884, 484, 0 ]
+}, {
+  "tz" : "Pacific/Kwajalein",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 175840, 89440, 3040, -83360, -169760, -256160, -342560, -428960, -515360, -601760, -688160, -774560, -860960, 3040, 0 ]
+}, {
+  "tz" : "Pacific/Majuro",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 174912, 88512, 2112, -84288, -170688, -257088, -343488, -429888, -516288, -602688, -689088, -775488, -861888, 2112, 0 ]
+}, {
+  "tz" : "Pacific/Marquesas",
+  "switches" : [ -62135735400, -59006413800, -55850653800, -52694893800, -46383373800, -43227613800, -40071853800, -33760333800, -30604573800, -27448813800, -21137293800, -17981533800, -14825773800, -12219258600, -2208988800 ],
+  "diffs" : [ 172080, 85680, -720, -87120, -173520, -259920, -346320, -432720, -519120, -605520, -691920, -778320, -864720, -720, 0 ]
+}, {
+  "tz" : "Pacific/Midway",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "Pacific/Nauru",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 175940, 89540, 3140, -83260, -169660, -256060, -342460, -428860, -515260, -601660, -688060, -774460, -860860, 3140, 0 ]
+}, {
+  "tz" : "Pacific/Niue",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2208988800 ],
+  "diffs" : [ 173980, 87580, 1180, -85220, -171620, -258020, -344420, -430820, -517220, -603620, -690020, -776420, -862820, 1180, 0 ]
+}, {
+  "tz" : "Pacific/Norfolk",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 172088, 85688, -712, -87112, -173512, -259912, -346312, -432712, -519112, -605512, -691912, -778312, -864712, -712, 0 ]
+}, {
+  "tz" : "Pacific/Noumea",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ],
+  "diffs" : [ 172452, 86052, -348, -86748, -173148, -259548, -345948, -432348, -518748, -605148, -691548, -777948, -864348, -348, 0 ]
+}, {
+  "tz" : "Pacific/Pago_Pago",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "Pacific/Palau",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -3944624400, -2208988800 ],
+  "diffs" : [ 259324, 172924, 86524, 124, -86276, -172676, -259076, -345476, -431876, -518276, -604676, -691076, -777476, 86524, 124, 0 ]
+}, {
+  "tz" : "Pacific/Pitcairn",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ],
+  "diffs" : [ 175220, 88820, 2420, -83980, -170380, -256780, -343180, -429580, -515980, -602380, -688780, -775180, -861580, 2420, 0 ]
+}, {
+  "tz" : "Pacific/Pohnpei",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ],
+  "diffs" : [ 260828, 174428, 88028, 1628, -84772, -171172, -257572, -343972, -430372, -516772, -603172, -689572, -775972, 88028, 1628, 0 ]
+}, {
+  "tz" : "Pacific/Ponape",
+  "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ],
+  "diffs" : [ 260828, 174428, 88028, 1628, -84772, -171172, -257572, -343972, -430372, -516772, -603172, -689572, -775972, 88028, 1628, 0 ]
+}, {
+  "tz" : "Pacific/Port_Moresby",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ],
+  "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ]
+}, {
+  "tz" : "Pacific/Rarotonga",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2208988800 ],
+  "diffs" : [ 175144, 88744, 2344, -84056, -170456, -256856, -343256, -429656, -516056, -602456, -688856, -775256, -861656, 2344, 0 ]
+}, {
+  "tz" : "Pacific/Saipan",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 260460, 174060, 87660, 1260, -85140, -171540, -257940, -344340, -430740, -517140, -603540, -689940, -776340, 87660, 1260, 0 ]
+}, {
+  "tz" : "Pacific/Samoa",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "Pacific/Tahiti",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2208988800 ],
+  "diffs" : [ 172696, 86296, -104, -86504, -172904, -259304, -345704, -432104, -518504, -604904, -691304, -777704, -864104, -104, 0 ]
+}, {
+  "tz" : "Pacific/Tarawa",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 174476, 88076, 1676, -84724, -171124, -257524, -343924, -430324, -516724, -603124, -689524, -775924, -862324, 1676, 0 ]
+}, {
+  "tz" : "Pacific/Tongatapu",
+  "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ],
+  "diffs" : [ 175240, 88840, 2440, -83960, -170360, -256760, -343160, -429560, -515960, -602360, -688760, -775160, -861560, 2440, 0 ]
+}, {
+  "tz" : "Pacific/Truk",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ]
+}, {
+  "tz" : "Pacific/Wake",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 176012, 89612, 3212, -83188, -169588, -255988, -342388, -428788, -515188, -601588, -687988, -774388, -860788, 3212, 0 ]
+}, {
+  "tz" : "Pacific/Wallis",
+  "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ],
+  "diffs" : [ 171880, 85480, -920, -87320, -173720, -260120, -346520, -432920, -519320, -605720, -692120, -778520, -864920, -920, 0 ]
+}, {
+  "tz" : "Pacific/Yap",
+  "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ],
+  "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ]
+}, {
+  "tz" : "Poland",
+  "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
+  "diffs" : [ 171360, 84960, -1440, -87840, -174240, -260640, -347040, -433440, -519840, -606240, -692640, -779040, -865440, -1440, 0 ]
+}, {
+  "tz" : "Portugal",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175005, 88605, 2205, -84195, -170595, -256995, -343395, -429795, -516195, -602595, -688995, -775395, -861795, 2205, 0 ]
+}, {
+  "tz" : "ROK",
+  "switches" : [ -62135802000, -59006480400, -55850720400, -52694960400, -46383440400, -43227680400, -40071920400, -33760400400, -30604640400, -27448880400, -21137360400, -17981600400, -14825840400, -12219325200, -2208988800 ],
+  "diffs" : [ 174728, 88328, 1928, -84472, -170872, -257272, -343672, -430072, -516472, -602872, -689272, -775672, -862072, 1928, 0 ]
+}, {
+  "tz" : "Singapore",
+  "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ],
+  "diffs" : [ 176675, 90275, 3875, -82525, -168925, -255325, -341725, -428125, -514525, -600925, -687325, -773725, -860125, 3875, 0 ]
+}, {
+  "tz" : "SystemV/AST4",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/AST4ADT",
+  "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/CST6",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/CST6CDT",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/EST5",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/EST5EDT",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/HST10",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/MST7",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/MST7MDT",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/PST8",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/PST8PDT",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/YST9",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "SystemV/YST9YDT",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Turkey",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2840151536, -2208988800 ],
+  "diffs" : [ 176648, 90248, 3848, -82552, -168952, -255352, -341752, -428152, -514552, -600952, -687352, -773752, -860152, 3848, 3784, 0 ]
+}, {
+  "tz" : "UCT",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "US/Alaska",
+  "switches" : [ -62135737200, -59006415600, -55850655600, -52694895600, -46383375600, -43227615600, -40071855600, -33760335600, -30604575600, -27448815600, -21137295600, -17981535600, -14825775600, -12219260400, -3225227303, -2208988800 ],
+  "diffs" : [ 89976, 3576, -82824, -169224, -255624, -342024, -428424, -514824, -601224, -687624, -774024, -860424, -946824, -82824, 3576, 0 ]
+}, {
+  "tz" : "US/Aleutian",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ],
+  "diffs" : [ 92798, 6398, -80002, -166402, -252802, -339202, -425602, -512002, -598402, -684802, -771202, -857602, -944002, -80002, 6398, 0 ]
+}, {
+  "tz" : "US/Arizona",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 174498, 88098, 1698, -84702, -171102, -257502, -343902, -430302, -516702, -603102, -689502, -775902, -862302, 1698, 0 ]
+}, {
+  "tz" : "US/Central",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 172236, 85836, -564, -86964, -173364, -259764, -346164, -432564, -518964, -605364, -691764, -778164, -864564, -564, 0 ]
+}, {
+  "tz" : "US/East-Indiana",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800, -2208988800 ],
+  "diffs" : [ 175478, 89078, 2678, -83722, -170122, -256522, -342922, -429322, -515722, -602122, -688522, -774922, -861322, 2678, 3600, 0 ]
+}, {
+  "tz" : "US/Eastern",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800 ],
+  "diffs" : [ 172562, 86162, -238, -86638, -173038, -259438, -345838, -432238, -518638, -605038, -691438, -777838, -864238, -238, 0 ]
+}, {
+  "tz" : "US/Hawaii",
+  "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ],
+  "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ]
+}, {
+  "tz" : "US/Indiana-Starke",
+  "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200 ],
+  "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ]
+}, {
+  "tz" : "US/Michigan",
+  "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ],
+  "diffs" : [ 174731, 88331, 1931, -84469, -170869, -257269, -343669, -430069, -516469, -602869, -689269, -775669, -862069, 1931, 0 ]
+}, {
+  "tz" : "US/Mountain",
+  "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ],
+  "diffs" : [ 172796, 86396, -4, -86404, -172804, -259204, -345604, -432004, -518404, -604804, -691204, -777604, -864004, -4, 0 ]
+}, {
+  "tz" : "US/Pacific",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ],
+  "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ]
+}, {
+  "tz" : "US/Pacific-New",
+  "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ],
+  "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ]
+}, {
+  "tz" : "US/Samoa",
+  "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ],
+  "diffs" : [ 87768, 1368, -85032, -171432, -257832, -344232, -430632, -517032, -603432, -689832, -776232, -862632, -949032, -85032, 1368, 0 ]
+}, {
+  "tz" : "UTC",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Universal",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "W-SU",
+  "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ],
+  "diffs" : [ 174583, 88183, 1783, -84617, -171017, -257417, -343817, -430217, -516617, -603017, -689417, -775817, -862217, 1783, 0 ]
+}, {
+  "tz" : "WET",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+}, {
+  "tz" : "Zulu",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ],
+  "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ]
+} ]
\ No newline at end of file
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 3512e3b5bfa65..33e05c5c20a42 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -21,12 +21,13 @@ import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 import java.time._
 import java.time.temporal.{ChronoField, ChronoUnit, IsoFields}
-import java.util.{Calendar, Locale, TimeZone}
+import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit._
 
 import scala.util.control.NonFatal
 
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.types.Decimal
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -987,153 +988,4 @@ object DateTimeUtils {
     val days = period.getDays
     new CalendarInterval(months, days, 0)
   }
-
-  /**
-   * Converts the given microseconds to a local date-time in UTC time zone in Proleptic Gregorian
-   * calendar, interprets the result as a local date-time in Julian calendar in UTC time zone.
-   * And takes microseconds since the epoch from the Julian timestamp.
-   *
-   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'.
-   * @return The rebased microseconds since the epoch in Julian calendar.
-   */
-  def rebaseGregorianToJulianMicros(micros: Long): Long = {
-    val instant = microsToInstant(micros)
-    val zoneId = ZoneId.systemDefault
-    val ldt = instant.atZone(zoneId).toLocalDateTime
-    val cal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
-      .setCalendarType("gregory")
-      .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
-      .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
-      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
-      // This setting selects the original local timestamp mapped to the given `micros`.
-      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
-      .build()
-    fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
-  }
-
-  /**
-   * Converts the given microseconds to a local date-time in UTC time zone in Julian calendar,
-   * interprets the result as a local date-time in Proleptic Gregorian calendar in UTC time zone.
-   * And takes microseconds since the epoch from the Gregorian timestamp.
-   *
-   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'.
-   * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
-   */
-  def rebaseJulianToGregorianMicros(micros: Long): Long = {
-    val cal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
-      .setCalendarType("gregory")
-      .setInstant(toMillis(micros))
-      .build()
-    val localDateTime = LocalDateTime.of(
-      cal.get(Calendar.YEAR),
-      cal.get(Calendar.MONTH) + 1,
-      // The number of days will be added later to handle non-existing
-      // Julian dates in Proleptic Gregorian calendar.
-      // For example, 1000-02-29 exists in Julian calendar because 1000
-      // is a leap year but it is not a leap year in Gregorian calendar.
-      1,
-      cal.get(Calendar.HOUR_OF_DAY),
-      cal.get(Calendar.MINUTE),
-      cal.get(Calendar.SECOND),
-      (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
-      .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
-    val zonedDateTime = localDateTime.atZone(ZoneId.systemDefault)
-    // Zero DST offset means that local clocks have switched to the winter time already.
-    // So, clocks go back one hour. We should correct zoned date-time and change
-    // the zone offset to the later of the two valid offsets at a local time-line overlap.
-    val adjustedZdt = if (cal.get(Calendar.DST_OFFSET) == 0) {
-      zonedDateTime.withLaterOffsetAtOverlap()
-    } else {
-      zonedDateTime
-    }
-    instantToMicros(adjustedZdt.toInstant)
-  }
-
-  /**
-   * Rebases days since the epoch from an original to an target calendar, from instance
-   * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
-   *
-   * It finds the latest switch day which is less than `days`, and adds the difference
-   * in days associated with the switch day to the given `days`. The function is based
-   * on linear search which starts from the most recent switch days. This allows to perform
-   * less comparisons for modern dates.
-   *
-   * @param switchDays The days when difference in days between original and target
-   *                   calendar was changed.
-   * @param diffs The differences in days between calendars.
-   * @param days The number of days since the epoch 1970-01-01 to be rebased to the
-   *             target calendar.
-   * @return The rebased day
-   */
-  private def rebaseDays(switchDays: Array[Int], diffs: Array[Int], days: Int): Int = {
-    var i = switchDays.length - 1
-    while (i >= 0 && days < switchDays(i)) {
-      i -= 1
-    }
-    val rebased = days + diffs(if (i < 0) 0 else i)
-    rebased
-  }
-
-  // The differences in days between Julian and Proleptic Gregorian dates.
-  // The diff at the index `i` is applicable for all days in the date interval:
-  // [julianGregDiffSwitchDay(i), julianGregDiffSwitchDay(i+1))
-  private val julianGregDiffs = Array(2, 1, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, 0)
-  // The sorted days in Julian calendar when difference in days between Julian and
-  // Proleptic Gregorian calendars was changed.
-  // The starting point is the `0001-01-01` (-719164 days since the epoch in
-  // Julian calendar). All dates before the staring point have the same difference
-  // of 2 days in Julian and Proleptic Gregorian calendars.
-  private val julianGregDiffSwitchDay = Array(
-    -719164, -682945, -646420, -609895, -536845, -500320, -463795,
-    -390745, -354220, -317695, -244645, -208120, -171595, -141427)
-
-  /**
-   * Converts the given number of days since the epoch day 1970-01-01 to
-   * a local date in Julian calendar, interprets the result as a local
-   * date in Proleptic Gregorian calendar, and take the number of days
-   * since the epoch from the Gregorian date.
-   *
-   * @param days The number of days since the epoch in Julian calendar.
-   * @return The rebased number of days in Gregorian calendar.
-   */
-  def rebaseJulianToGregorianDays(days: Int): Int = {
-    rebaseDays(julianGregDiffSwitchDay, julianGregDiffs, days)
-  }
-
-  // The differences in days between Proleptic Gregorian and Julian dates.
-  // The diff at the index `i` is applicable for all days in the date interval:
-  // [gregJulianDiffSwitchDay(i), gregJulianDiffSwitchDay(i+1))
-  private val grepJulianDiffs = Array(-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0)
-  // The sorted days in Proleptic Gregorian calendar when difference in days between
-  // Proleptic Gregorian and Julian was changed.
-  // The starting point is the `0001-01-01` (-719162 days since the epoch in
-  // Proleptic Gregorian calendar). All dates before the staring point have the same
-  // difference of -2 days in Proleptic Gregorian and Julian calendars.
-  private val gregJulianDiffSwitchDay = Array(
-    -719162, -682944, -646420, -609896, -536847, -500323, -463799,
-    -390750, -354226, -317702, -244653, -208129, -171605, -141427)
-
-  /**
-   * Rebasing days since the epoch to store the same number of days
-   * as by Spark 2.4 and earlier versions. Spark 3.0 switched to
-   * Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
-   * this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
-   * Julian calendar for dates before 1582-10-15. So, the same local date may
-   * be mapped to different number of days since the epoch in different calendars.
-   *
-   * For example:
-   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
-   *   Julian calendar: 1582-01-01 -> -141704
-   * The code below converts -141714 to -141704.
-   *
-   * @param days The number of days since the epoch 1970-01-01. It can be negative.
-   * @return The rebased number of days since the epoch in Julian calendar.
-   */
-  def rebaseGregorianToJulianDays(days: Int): Int = {
-    rebaseDays(gregJulianDiffSwitchDay, grepJulianDiffs, days)
-  }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
new file mode 100644
index 0000000000000..793eb17f72ae0
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.time.{LocalDateTime, ZoneId}
+import java.time.temporal.ChronoField
+import java.util.{Calendar, TimeZone}
+
+import scala.collection.mutable.AnyRefMap
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
+
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+
+/**
+ * The collection of functions for rebasing days and microseconds from/to the hybrid calendar
+ * (Julian + Gregorian since 1582-10-15) which is used by Spark 2.4 and earlier versions
+ * to/from Proleptic Gregorian calendar used in Spark SQL since version 3.0, see SPARK-26651.
+ */
+object RebaseDateTime {
+  /**
+   * Rebases days since the epoch from an original to an target calendar, for instance,
+   * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+   *
+   * It finds the latest switch day which is less than `value`, and adds the difference
+   * in days associated with the switch days to the given `value`.
+   * The function is based on linear search which starts from the most recent switch days.
+   * This allows to perform less comparisons for modern dates.
+   *
+   * @param switches The days when difference in days between original
+   *                 and target calendar was changed.
+   * @param diffs The differences in days between calendars.
+   * @param days The number of days since the epoch 1970-01-01 to be rebased
+   *             to the target calendar.
+   * @return The rebased days.
+   */
+  private def rebaseDays(switches: Array[Int], diffs: Array[Int], days: Int): Int = {
+    var i = switches.length
+    do { i -= 1 } while (i > 0 && days < switches(i))
+    days + diffs(i)
+  }
+
+  // The differences in days between Julian and Proleptic Gregorian dates.
+  // The diff at the index `i` is applicable for all days in the date interval:
+  // [julianGregDiffSwitchDay(i), julianGregDiffSwitchDay(i+1))
+  private val julianGregDiffs = Array(2, 1, 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, 0)
+  // The sorted days in Julian calendar when difference in days between Julian and
+  // Proleptic Gregorian calendars was changed.
+  // The starting point is the `0001-01-01` (-719164 days since the epoch in
+  // Julian calendar). All dates before the staring point have the same difference
+  // of 2 days in Julian and Proleptic Gregorian calendars.
+  private val julianGregDiffSwitchDay = Array(
+    -719164, -682945, -646420, -609895, -536845, -500320, -463795,
+    -390745, -354220, -317695, -244645, -208120, -171595, -141427)
+
+  /**
+   * Converts the given number of days since the epoch day 1970-01-01 to
+   * a local date in Julian calendar, interprets the result as a local
+   * date in Proleptic Gregorian calendar, and take the number of days
+   * since the epoch from the Gregorian date.
+   *
+   * @param days The number of days since the epoch in Julian calendar.
+   * @return The rebased number of days in Gregorian calendar.
+   */
+  def rebaseJulianToGregorianDays(days: Int): Int = {
+    rebaseDays(julianGregDiffSwitchDay, julianGregDiffs, days)
+  }
+
+  // The differences in days between Proleptic Gregorian and Julian dates.
+  // The diff at the index `i` is applicable for all days in the date interval:
+  // [gregJulianDiffSwitchDay(i), gregJulianDiffSwitchDay(i+1))
+  private val gregJulianDiffs = Array(-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0)
+  // The sorted days in Proleptic Gregorian calendar when difference in days between
+  // Proleptic Gregorian and Julian was changed.
+  // The starting point is the `0001-01-01` (-719162 days since the epoch in
+  // Proleptic Gregorian calendar). All dates before the staring point have the same
+  // difference of -2 days in Proleptic Gregorian and Julian calendars.
+  private val gregJulianDiffSwitchDay = Array(
+    -719162, -682944, -646420, -609896, -536847, -500323, -463799,
+    -390750, -354226, -317702, -244653, -208129, -171605, -141427)
+
+  /**
+   * Rebasing days since the epoch to store the same number of days
+   * as by Spark 2.4 and earlier versions. Spark 3.0 switched to
+   * Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
+   * this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15. So, the same local date may
+   * be mapped to different number of days since the epoch in different calendars.
+   *
+   * For example:
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   *   Julian calendar: 1582-01-01 -> -141704
+   * The code below converts -141714 to -141704.
+   *
+   * @param days The number of days since the epoch 1970-01-01. It can be negative.
+   * @return The rebased number of days since the epoch in Julian calendar.
+   */
+  def rebaseGregorianToJulianDays(days: Int): Int = {
+    rebaseDays(gregJulianDiffSwitchDay, gregJulianDiffs, days)
+  }
+
+  /**
+   * Converts the given microseconds to a local date-time in UTC time zone in Proleptic Gregorian
+   * calendar, interprets the result as a local date-time in Julian calendar in UTC time zone.
+   * And takes microseconds since the epoch from the Julian timestamp.
+   *
+   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in Proleptic Gregorian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Julian calendar.
+   */
+  private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
+    val instant = microsToInstant(micros)
+    val ldt = instant.atZone(zoneId).toLocalDateTime
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
+      .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
+      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
+      // This setting selects the original local timestamp mapped to the given `micros`.
+      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
+      .build()
+    fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
+  }
+
+
+  /**
+   * The class describes JSON records with microseconds rebasing info.
+   * Here is an example of JSON file:
+   * {{{
+   *   [
+   *     {
+   *       "tz": "Europe/Paris",
+   *       "switches": [-123, 0],
+   *       "diffs": [422000000, 0]
+   *     }
+   *   ]
+   * }}}
+   *
+   * @param tz One of time zone ID which is expected to be acceptable by `ZoneId.of`.
+   * @param switches An ordered array of seconds since the epoch when the diff between
+   *                 two calendars are changed.
+   * @param diffs Differences in seconds associated with elements of `switches`.
+   */
+  private case class JsonRebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
+
+  /**
+   * Rebasing info used to convert microseconds from an original to a target calendar.
+   *
+   * @param switches An ordered array of microseconds since the epoch when the diff between
+   *                 two calendars are changed.
+   * @param diffs Differences in microseconds associated with elements of `switches`.
+   */
+  private[sql] case class RebaseInfo(switches: Array[Long], diffs: Array[Long])
+
+  /**
+   * Rebases micros since the epoch from an original to an target calendar, for instance,
+   * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
+   *
+   * It finds the latest switch micros which is less than `value`, and adds the difference
+   * in micros associated with the switch micros to the given `value`.
+   * The function is based on linear search which starts from the most recent switch micros.
+   * This allows to perform less comparisons for modern timestamps.
+   *
+   * @param rebaseInfo The rebasing info contains an ordered micros when difference in micros
+   *                   between original and target calendar was changed,
+   *                   and differences in micros between calendars
+   * @param micros The number of micros since the epoch 1970-01-01T00:00:00Z to be rebased
+   *               to the target calendar. It can be negative.
+   * @return The rebased micros.
+   */
+  private def rebaseMicros(rebaseInfo: RebaseInfo, micros: Long): Long = {
+    val switches = rebaseInfo.switches
+    var i = switches.length
+    do { i -= 1 } while (i > 0 && micros < switches(i))
+    micros + rebaseInfo.diffs(i)
+  }
+
+  // Loads rebasing info from an JSON file. JSON records in the files should conform to
+  // `JsonRebaseRecord`. AnyRefMap is used here instead of Scala's immutable map because
+  // it is 2 times faster in DateTimeRebaseBenchmark.
+  private[sql] def loadRebaseRecords(fileName: String): AnyRefMap[String, RebaseInfo] = {
+    val file = Thread.currentThread().getContextClassLoader.getResource(fileName)
+    val mapper = new ObjectMapper() with ScalaObjectMapper
+    mapper.registerModule(DefaultScalaModule)
+    val jsonRebaseRecords = mapper.readValue[Seq[JsonRebaseRecord]](file)
+    val anyRefMap = new AnyRefMap[String, RebaseInfo]((3 * jsonRebaseRecords.size) / 2)
+    jsonRebaseRecords.foreach { jsonRecord =>
+      val rebaseInfo = RebaseInfo(jsonRecord.switches, jsonRecord.diffs)
+      var i = 0
+      while (i < rebaseInfo.switches.length) {
+        rebaseInfo.switches(i) = rebaseInfo.switches(i) * MICROS_PER_SECOND
+        rebaseInfo.diffs(i) = rebaseInfo.diffs(i) * MICROS_PER_SECOND
+        i += 1
+      }
+      anyRefMap.update(jsonRecord.tz, rebaseInfo)
+    }
+    anyRefMap
+  }
+
+  /**
+   * A map of time zone IDs to its ordered time points (instants in microseconds since the epoch)
+   * when the difference between 2 instances associated with the same local timestamp in
+   * Proleptic Gregorian and the hybrid calendar was changed, and to the diff at the index `i` is
+   * applicable for all microseconds in the time interval:
+   *   [gregJulianDiffSwitchMicros(i), gregJulianDiffSwitchMicros(i+1))
+   */
+  private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json")
+
+  /**
+   * Rebases the given `micros` to the number of microseconds since the epoch via a local
+   * timestamp that have the same date-time fields in Proleptic Gregorian and in the hybrid
+   * calendars.
+   *
+   * The function may optimize the rebasing by using pre-calculated rebasing maps. If the maps
+   * don't contains information about the current JVM system time zone, the functions falls back
+   * to regular conversion mechanism via building local timestamps.
+   *
+   * Note: The function assumes that the input micros was derived from a local timestamp
+   *       at the default system JVM time zone in Proleptic Gregorian calendar.
+   *
+   * @param micros The microseconds since the epoch 1970-01-01T00:00:00Z
+   *               in Proleptic Gregorian calendar. It can be negative.
+   * @return The microseconds since the epoch that have the same local timestamp representation
+   *         in the hybrid calendar (Julian + Gregorian) as the original `micros` in
+   *         Proleptic Gregorian calendar.
+   */
+  def rebaseGregorianToJulianMicros(micros: Long): Long = {
+    val timeZone = TimeZone.getDefault
+    val tzId = timeZone.getID
+    val rebaseRecord = gregJulianRebaseMap.getOrNull(tzId)
+    if (rebaseRecord == null) {
+      rebaseGregorianToJulianMicros(timeZone.toZoneId, micros)
+    } else {
+      rebaseMicros(rebaseRecord, micros)
+    }
+  }
+
+  /**
+   * Converts the given microseconds to a local date-time in UTC time zone in the hybrid
+   * calendar (Julian + Gregorian since 1582-10-15), interprets the result as a local date-time
+   * in Proleptic Gregorian calendar in UTC time zone. And takes microseconds since the epoch
+   * from the Gregorian timestamp.
+   *
+   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in the hybrid calendar (Julian + Gregorian). It can be negative.
+   * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
+   */
+  private[sql] def rebaseJulianToGregorianMicros(zoneId: ZoneId, micros: Long): Long = {
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setInstant(toMillis(micros))
+      .build()
+    val localDateTime = LocalDateTime.of(
+      cal.get(Calendar.YEAR),
+      cal.get(Calendar.MONTH) + 1,
+      // The number of days will be added later to handle non-existing
+      // Julian dates in Proleptic Gregorian calendar.
+      // For example, 1000-02-29 exists in Julian calendar because 1000
+      // is a leap year but it is not a leap year in Gregorian calendar.
+      1,
+      cal.get(Calendar.HOUR_OF_DAY),
+      cal.get(Calendar.MINUTE),
+      cal.get(Calendar.SECOND),
+      (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
+      .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
+    val zonedDateTime = localDateTime.atZone(zoneId)
+    // Zero DST offset means that local clocks have switched to the winter time already.
+    // So, clocks go back one hour. We should correct zoned date-time and change
+    // the zone offset to the later of the two valid offsets at a local time-line overlap.
+    val adjustedZdt = if (cal.get(Calendar.DST_OFFSET) == 0) {
+      zonedDateTime.withLaterOffsetAtOverlap()
+    } else {
+      zonedDateTime
+    }
+    instantToMicros(adjustedZdt.toInstant)
+  }
+
+  // The rebasing maps to convert microseconds from the hybrid calendar (Julian + Gregorian)
+  // to Proleptic Gregorian calendar. It maps time zone IDs to ordered timestamps (ascending order)
+  // where at every timestamps the difference between 2 calendars was changed, and to ordered
+  // differences between 2 calendars. The diff at the index `i` is applicable for all timestamps
+  // in the interval: [julianGregDiffSwitchMicros(i), julianGregDiffSwitchMicros(i+1))
+  private val julianGregRebaseMap = loadRebaseRecords("julian-gregorian-rebase-micros.json")
+
+  /**
+   * This is an opposite to `rebaseGregorianToJulianMicros` function which rebases the given
+   * microseconds since the epoch 1970-01-01T00:00:00Z via local timestamps from the
+   * hybrid calendar (Julian + Gregorian) to Proleptic Gregorian calendar.
+   * For example, the input `micros` -12243196799876544 is mapped to 1582-01-01 00:00:00.123456 in
+   * Julian calendar. The same local timestamp in Proleptic Gregorian calendar is mapped to
+   * the different number of micros -12244061221876544 since the epoch. Semantically, the function
+   * performs such conversion either via direct conversion to local timestamps,
+   * or via pre-calculated rebasing tables.
+   *
+   * Note: The function assumes that the input micros was derived from a local timestamp
+   *       at the default system JVM time zone in the hybrid calendar (Julian and Gregorian
+   *       since 1582-10-15)
+   *
+   * @param micros The number of microseconds since the epoch 1970-01-01T00:00:00Z
+   *               in the hybrid calendar (Julian + Gregorian). It can be negative.
+   * @return The rebased number of microseconds since the epoch which is mapped to the same
+   *         local timestamp in Proleptic Gregorian calendar as `micros` in the hybrid calendar
+   *         at the system time zone.
+   */
+  def rebaseJulianToGregorianMicros(micros: Long): Long = {
+    val timeZone = TimeZone.getDefault
+    val tzId = timeZone.getID
+    val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
+    if (rebaseRecord == null) {
+      rebaseJulianToGregorianMicros(timeZone.toZoneId, micros)
+    } else {
+      rebaseMicros(rebaseRecord, micros)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index f9c15f394f5ca..7459044e12429 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.util
 import java.sql.{Date, Timestamp}
 import java.text.SimpleDateFormat
 import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId}
-import java.util.{Calendar, Locale, TimeZone}
+import java.util.Locale
 import java.util.concurrent.TimeUnit
 
 import org.scalatest.Matchers
@@ -657,184 +657,4 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       assert(toDate("tomorrow CET ", zoneId).get === today + 1)
     }
   }
-
-  private def parseToJulianMicros(s: String): Long = {
-    val ts = Timestamp.valueOf(s)
-    val julianMicros = fromMillis(ts.getTime) +
-      ((ts.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
-    julianMicros
-  }
-
-  private def parseToGregMicros(s: String, zoneId: ZoneId): Long = {
-    instantToMicros(LocalDateTime.parse(s).atZone(zoneId).toInstant)
-  }
-
-  test("rebase julian to/from gregorian micros") {
-    outstandingZoneIds.foreach { zid =>
-      withDefaultTimeZone(zid) {
-        Seq(
-          "0001-01-01 01:02:03.654321",
-          "1000-01-01 03:02:01.123456",
-          "1582-10-04 00:00:00.000000",
-          "1582-10-15 00:00:00.999999", // Gregorian cutover day
-          "1883-11-10 00:00:00.000000", // America/Los_Angeles -7:52:58 zone offset
-          "1883-11-20 00:00:00.000000", // America/Los_Angeles -08:00 zone offset
-          "1969-12-31 11:22:33.000100",
-          "1970-01-01 00:00:00.000001", // The epoch day
-          "2020-03-14 09:33:01.500000").foreach { ts =>
-          withClue(s"time zone = ${zid.getId} ts = $ts") {
-            val julianMicros = parseToJulianMicros(ts)
-            val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), zid)
-
-            assert(rebaseJulianToGregorianMicros(julianMicros) === gregMicros)
-            assert(rebaseGregorianToJulianMicros(gregMicros) === julianMicros)
-          }
-        }
-      }
-    }
-  }
-
-  // millisToDays() and fromJavaDate() are taken from Spark 2.4
-  private def millisToDaysLegacy(millisUtc: Long, timeZone: TimeZone): Int = {
-    val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
-    Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
-  }
-  private def fromJavaDateLegacy(date: Date): Int = {
-    millisToDaysLegacy(date.getTime, defaultTimeZone())
-  }
-
-  test("rebase gregorian to/from julian days") {
-    outstandingZoneIds.foreach { zid =>
-      withDefaultTimeZone(zid) {
-        Seq(
-          "0001-01-01",
-          "1000-01-01",
-          "1582-10-04",
-          "1582-10-15", // Gregorian cutover day
-          "1883-11-10", // America/Los_Angeles -7:52:58 zone offset
-          "1883-11-20", // America/Los_Angeles -08:00 zone offset
-          "1969-12-31",
-          "1970-01-01", // The epoch day
-          "2020-03-14").foreach { date =>
-          val julianDays = fromJavaDateLegacy(Date.valueOf(date))
-          val gregorianDays = localDateToDays(LocalDate.parse(date))
-
-          assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
-          assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
-        }
-      }
-    }
-  }
-
-  test("rebase julian to gregorian date for leap years") {
-    outstandingZoneIds.foreach { zid =>
-      withDefaultTimeZone(zid) {
-        Seq(
-          "1000-02-29" -> "1000-03-01",
-          "1600-02-29" -> "1600-02-29",
-          "1700-02-29" -> "1700-03-01",
-          "2000-02-29" -> "2000-02-29").foreach { case (julianDate, gregDate) =>
-          withClue(s"tz = ${zid.getId} julian date = $julianDate greg date = $gregDate") {
-            val date = Date.valueOf(julianDate)
-            val julianDays = fromJavaDateLegacy(date)
-            val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
-
-            assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
-          }
-        }
-      }
-    }
-  }
-
-  test("rebase julian to gregorian timestamp for leap years") {
-    outstandingZoneIds.foreach { zid =>
-      withDefaultTimeZone(zid) {
-        Seq(
-          "1000-02-29 01:02:03.123456" -> "1000-03-01T01:02:03.123456",
-          "1600-02-29 11:12:13.654321" -> "1600-02-29T11:12:13.654321",
-          "1700-02-29 21:22:23.000001" -> "1700-03-01T21:22:23.000001",
-          "2000-02-29 00:00:00.999999" -> "2000-02-29T00:00:00.999999"
-        ).foreach { case (julianTs, gregTs) =>
-          withClue(s"tz = ${zid.getId} julian ts = $julianTs greg ts = $gregTs") {
-            val julianMicros = parseToJulianMicros(julianTs)
-            val gregorianMicros = parseToGregMicros(gregTs, zid)
-
-            assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
-          }
-        }
-      }
-    }
-  }
-
-  test("optimization of days rebasing - Gregorian to Julian") {
-    def refRebaseGregorianToJulianDays(days: Int): Int = {
-      val localDate = LocalDate.ofEpochDay(days)
-      val utcCal = new Calendar.Builder()
-        // `gregory` is a hybrid calendar that supports both
-        // the Julian and Gregorian calendar systems
-        .setCalendarType("gregory")
-        .setTimeZone(TimeZoneUTC)
-        .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
-        .build()
-      Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
-    }
-
-    val start = localDateToDays(LocalDate.of(1, 1, 1))
-    val end = localDateToDays(LocalDate.of(2030, 1, 1))
-
-    var days = start
-    while (days < end) {
-      assert(rebaseGregorianToJulianDays(days) === refRebaseGregorianToJulianDays(days))
-      days += 1
-    }
-  }
-
-  test("optimization of days rebasing - Julian to Gregorian") {
-    def refRebaseJulianToGregorianDays(days: Int): Int = {
-      val utcCal = new Calendar.Builder()
-        // `gregory` is a hybrid calendar that supports both
-        // the Julian and Gregorian calendar systems
-        .setCalendarType("gregory")
-        .setTimeZone(TimeZoneUTC)
-        .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
-        .build()
-      val localDate = LocalDate.of(
-        utcCal.get(Calendar.YEAR),
-        utcCal.get(Calendar.MONTH) + 1,
-        // The number of days will be added later to handle non-existing
-        // Julian dates in Proleptic Gregorian calendar.
-        // For example, 1000-02-29 exists in Julian calendar because 1000
-        // is a leap year but it is not a leap year in Gregorian calendar.
-        1)
-        .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
-      Math.toIntExact(localDate.toEpochDay)
-    }
-
-    val start = rebaseGregorianToJulianDays(
-      localDateToDays(LocalDate.of(1, 1, 1)))
-    val end = rebaseGregorianToJulianDays(
-      localDateToDays(LocalDate.of(2030, 1, 1)))
-
-    var days = start
-    while (days < end) {
-      assert(rebaseJulianToGregorianDays(days) === refRebaseJulianToGregorianDays(days))
-      days += 1
-    }
-  }
-
-  test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") {
-    Seq(
-      LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"),
-      "Europe/Amsterdam" ->
-        Seq("2019-10-27T00:00:00Z", "2019-10-27T00:30:00Z", "2019-10-27T01:00:00Z")
-    ).foreach { case (tz, ts) =>
-      withDefaultTimeZone(getZoneId(tz)) {
-        ts.foreach { str =>
-          val micros = instantToMicros(Instant.parse(str))
-          assert(rebaseGregorianToJulianMicros(micros) === micros)
-          assert(rebaseJulianToGregorianMicros(micros) === micros)
-        }
-      }
-    }
-  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
new file mode 100644
index 0000000000000..dcc80b1d6f3e7
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
+import java.util.{Calendar, TimeZone}
+
+import org.scalatest.Matchers
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+
+class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
+
+  private def parseToJulianMicros(s: String): Long = {
+    val ts = Timestamp.valueOf(s)
+    val julianMicros = fromMillis(ts.getTime) +
+      ((ts.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
+    julianMicros
+  }
+
+  private def parseToGregMicros(s: String, zoneId: ZoneId): Long = {
+    instantToMicros(LocalDateTime.parse(s).atZone(zoneId).toInstant)
+  }
+
+  test("rebase julian to/from gregorian micros") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "0001-01-01 01:02:03.654321",
+          "1000-01-01 03:02:01.123456",
+          "1582-10-04 00:00:00.000000",
+          "1582-10-15 00:00:00.999999", // Gregorian cutover day
+          "1883-11-10 00:00:00.000000", // America/Los_Angeles -7:52:58 zone offset
+          "1883-11-20 00:00:00.000000", // America/Los_Angeles -08:00 zone offset
+          "1969-12-31 11:22:33.000100",
+          "1970-01-01 00:00:00.000001", // The epoch day
+          "2020-03-14 09:33:01.500000").foreach { ts =>
+          withClue(s"time zone = ${zid.getId} ts = $ts") {
+            val julianMicros = parseToJulianMicros(ts)
+            val gregMicros = parseToGregMicros(ts.replace(' ', 'T'), zid)
+
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregMicros)
+            assert(rebaseGregorianToJulianMicros(gregMicros) === julianMicros)
+          }
+        }
+      }
+    }
+  }
+
+  // millisToDays() and fromJavaDate() are taken from Spark 2.4
+  private def millisToDaysLegacy(millisUtc: Long, timeZone: TimeZone): Int = {
+    val millisLocal = millisUtc + timeZone.getOffset(millisUtc)
+    Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
+  }
+  private def fromJavaDateLegacy(date: Date): Int = {
+    millisToDaysLegacy(date.getTime, defaultTimeZone())
+  }
+
+  test("rebase gregorian to/from julian days") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "0001-01-01",
+          "1000-01-01",
+          "1582-10-04",
+          "1582-10-15", // Gregorian cutover day
+          "1883-11-10", // America/Los_Angeles -7:52:58 zone offset
+          "1883-11-20", // America/Los_Angeles -08:00 zone offset
+          "1969-12-31",
+          "1970-01-01", // The epoch day
+          "2020-03-14").foreach { date =>
+          val julianDays = fromJavaDateLegacy(Date.valueOf(date))
+          val gregorianDays = localDateToDays(LocalDate.parse(date))
+
+          assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
+          assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+        }
+      }
+    }
+  }
+
+  test("rebase julian to gregorian date for leap years") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1000-02-29" -> "1000-03-01",
+          "1600-02-29" -> "1600-02-29",
+          "1700-02-29" -> "1700-03-01",
+          "2000-02-29" -> "2000-02-29").foreach { case (julianDate, gregDate) =>
+          withClue(s"tz = ${zid.getId} julian date = $julianDate greg date = $gregDate") {
+            val date = Date.valueOf(julianDate)
+            val julianDays = fromJavaDateLegacy(date)
+            val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
+
+            assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+          }
+        }
+      }
+    }
+  }
+
+  test("rebase julian to gregorian timestamp for leap years") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1000-02-29 01:02:03.123456" -> "1000-03-01T01:02:03.123456",
+          "1600-02-29 11:12:13.654321" -> "1600-02-29T11:12:13.654321",
+          "1700-02-29 21:22:23.000001" -> "1700-03-01T21:22:23.000001",
+          "2000-02-29 00:00:00.999999" -> "2000-02-29T00:00:00.999999"
+        ).foreach { case (julianTs, gregTs) =>
+          withClue(s"tz = ${zid.getId} julian ts = $julianTs greg ts = $gregTs") {
+            val julianMicros = parseToJulianMicros(julianTs)
+            val gregorianMicros = parseToGregMicros(gregTs, zid)
+
+            assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
+          }
+        }
+      }
+    }
+  }
+
+  test("optimization of days rebasing - Gregorian to Julian") {
+    // Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
+    // in `RebaseDateTime` was generated by using this function.
+    def refRebaseGregorianToJulianDays(days: Int): Int = {
+      val localDate = LocalDate.ofEpochDay(days)
+      val utcCal = new Calendar.Builder()
+        // `gregory` is a hybrid calendar that supports both
+        // the Julian and Gregorian calendar systems
+        .setCalendarType("gregory")
+        .setTimeZone(TimeZoneUTC)
+        .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+        .build()
+      Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+    }
+
+    val start = localDateToDays(LocalDate.of(1, 1, 1))
+    val end = localDateToDays(LocalDate.of(2030, 1, 1))
+
+    var days = start
+    while (days < end) {
+      assert(rebaseGregorianToJulianDays(days) === refRebaseGregorianToJulianDays(days))
+      days += 1
+    }
+  }
+
+  test("optimization of days rebasing - Julian to Gregorian") {
+    // Rebasing switch days and diffs `julianGregDiffSwitchDay` and `julianGregDiffs`
+    // in `RebaseDateTime` was generated by using this function.
+    def refRebaseJulianToGregorianDays(days: Int): Int = {
+      val utcCal = new Calendar.Builder()
+        // `gregory` is a hybrid calendar that supports both
+        // the Julian and Gregorian calendar systems
+        .setCalendarType("gregory")
+        .setTimeZone(TimeZoneUTC)
+        .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
+        .build()
+      val localDate = LocalDate.of(
+        utcCal.get(Calendar.YEAR),
+        utcCal.get(Calendar.MONTH) + 1,
+        // The number of days will be added later to handle non-existing
+        // Julian dates in Proleptic Gregorian calendar.
+        // For example, 1000-02-29 exists in Julian calendar because 1000
+        // is a leap year but it is not a leap year in Gregorian calendar.
+        1)
+        .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
+      Math.toIntExact(localDate.toEpochDay)
+    }
+
+    val start = rebaseGregorianToJulianDays(
+      localDateToDays(LocalDate.of(1, 1, 1)))
+    val end = rebaseGregorianToJulianDays(
+      localDateToDays(LocalDate.of(2030, 1, 1)))
+
+    var days = start
+    while (days < end) {
+      assert(rebaseJulianToGregorianDays(days) === refRebaseJulianToGregorianDays(days))
+      days += 1
+    }
+  }
+
+  test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") {
+    Seq(
+      LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"),
+      "Europe/Amsterdam" ->
+        Seq("2019-10-27T00:00:00Z", "2019-10-27T00:30:00Z", "2019-10-27T01:00:00Z")
+    ).foreach { case (tz, ts) =>
+      withDefaultTimeZone(getZoneId(tz)) {
+        ts.foreach { str =>
+          val micros = instantToMicros(Instant.parse(str))
+          assert(rebaseGregorianToJulianMicros(micros) === micros)
+          assert(rebaseJulianToGregorianMicros(micros) === micros)
+        }
+      }
+    }
+  }
+
+  test("validate rebase records in JSON files") {
+    Seq(
+      "gregorian-julian-rebase-micros.json",
+      "julian-gregorian-rebase-micros.json").foreach { json =>
+      withClue(s"JSON file = $json") {
+        val rebaseRecords = loadRebaseRecords(json)
+        rebaseRecords.foreach { case (_, rebaseRecord) =>
+          assert(rebaseRecord.switches.size === rebaseRecord.diffs.size)
+          // Check ascending order of switches values
+          assert(rebaseRecord.switches.toSeq === rebaseRecord.switches.sorted.toSeq)
+        }
+      }
+    }
+  }
+
+  test("optimization of micros rebasing - Gregorian to Julian") {
+    outstandingZoneIds.foreach { zid =>
+      withClue(s"zone id = $zid") {
+        withDefaultTimeZone(zid) {
+          val start = instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
+            .atZone(zid)
+            .toInstant)
+          val end = instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0)
+            .atZone(zid)
+            .toInstant)
+          var micros = start
+          do {
+            val rebased = rebaseGregorianToJulianMicros(zid, micros)
+            val rebasedAndOptimized = rebaseGregorianToJulianMicros(micros)
+            assert(rebasedAndOptimized === rebased)
+            micros += (MICROS_PER_MONTH * (0.5 + Math.random())).toLong
+          } while (micros <= end)
+        }
+      }
+    }
+  }
+
+  test("optimization of micros rebasing - Julian to Gregorian") {
+    outstandingZoneIds.foreach { zid =>
+      withClue(s"zone id = $zid") {
+        withDefaultTimeZone(zid) {
+          val start = rebaseGregorianToJulianMicros(
+            instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0).atZone(zid).toInstant))
+          val end = rebaseGregorianToJulianMicros(
+            instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0).atZone(zid).toInstant))
+          var micros = start
+          do {
+            val rebased = rebaseJulianToGregorianMicros(zid, micros)
+            val rebasedAndOptimized = rebaseJulianToGregorianMicros(micros)
+            assert(rebasedAndOptimized === rebased)
+            micros += (MICROS_PER_MONTH * (0.5 + Math.random())).toLong
+          } while (micros <= end)
+        }
+      }
+    }
+  }
+
+  private def generateRebaseJson(
+      adjustFunc: Long => Long,
+      rebaseFunc: (ZoneId, Long) => Long,
+      dir: String,
+      fileName: String): Unit = {
+    import java.nio.file.{Files, Paths}
+    import java.nio.file.StandardOpenOption
+
+    import scala.collection.mutable.ArrayBuffer
+
+    import com.fasterxml.jackson.databind.ObjectMapper
+    import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
+
+    case class RebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
+
+    val result = new ArrayBuffer[RebaseRecord]()
+    // The time zones are excluded because:
+    // 1. Julian to Gregorian rebasing doesn't match to the opposite rebasing from
+    //    Gregorian to Julian rebasing.
+    // 2. Linear searching over switch points might be slow.
+    // 3. Results after the end time point 2100-01-01 are wrong.
+    // See SPARK-31385
+    val blacklist = Set("Asia/Tehran", "Iran", "Africa/Casablanca", "Africa/El_Aaiun")
+    ALL_TIMEZONES
+      .filterNot(zid => blacklist.contains(zid.getId))
+      .sortBy(_.getId)
+      .foreach { zid =>
+      withDefaultTimeZone(zid) {
+        val start = adjustFunc(instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
+          .atZone(zid)
+          .toInstant))
+        val end = adjustFunc(instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0)
+          .atZone(zid)
+          .toInstant))
+
+        var micros = start
+        var diff = Long.MaxValue
+        val maxStep = DAYS_PER_WEEK * MICROS_PER_DAY
+        var step: Long = MICROS_PER_SECOND
+        val switches = new ArrayBuffer[Long]()
+        val diffs = new ArrayBuffer[Long]()
+        while (micros < end) {
+          val rebased = rebaseFunc(zid, micros)
+          val curDiff = rebased - micros
+          if (curDiff != diff) {
+            if (step > MICROS_PER_SECOND) {
+              micros -= step
+              step = (Math.max(MICROS_PER_SECOND, step / 2) / MICROS_PER_SECOND) * MICROS_PER_SECOND
+            } else {
+              diff = curDiff
+              step = maxStep
+              assert(diff % MICROS_PER_SECOND == 0)
+              diffs.append(diff / MICROS_PER_SECOND)
+              assert(micros % MICROS_PER_SECOND == 0)
+              switches.append(micros / MICROS_PER_SECOND)
+            }
+          }
+          micros += step
+        }
+        result.append(RebaseRecord(zid.getId, switches.toArray, diffs.toArray))
+      }
+    }
+    val mapper = (new ObjectMapper() with ScalaObjectMapper)
+      .registerModule(DefaultScalaModule)
+      .writerWithDefaultPrettyPrinter()
+    mapper.writeValue(
+      Files.newOutputStream(
+        Paths.get(dir, fileName),
+        StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING),
+      result.toArray)
+  }
+
+  ignore("generate 'gregorian-julian-rebase-micros.json'") {
+    generateRebaseJson(
+      adjustFunc = identity[Long],
+      rebaseFunc = rebaseGregorianToJulianMicros,
+      dir = "/Users/maximgekk/tmp",
+      fileName = "gregorian-julian-rebase-micros.json")
+  }
+
+  ignore("generate 'julian-gregorian-rebase-micros.json'") {
+    generateRebaseJson(
+      adjustFunc = rebaseGregorianToJulianMicros,
+      rebaseFunc = rebaseJulianToGregorianMicros,
+      dir = "/Users/maximgekk/tmp",
+      fileName = "julian-gregorian-rebase-micros.json")
+  }
+}
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 5eb0e6b3134d0..bb7d8b0956d4b 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  18597          18597           0          5.4         186.0       1.0X
-before 1582, noop                                 10565          10565           0          9.5         105.7       1.8X
-after 1582, rebase off                            29811          29811           0          3.4         298.1       0.6X
-after 1582, rebase on                             31110          31110           0          3.2         311.1       0.6X
-before 1582, rebase off                           23144          23144           0          4.3         231.4       0.8X
-before 1582, rebase on                            23689          23689           0          4.2         236.9       0.8X
+after 1582, noop                                  19037          19037           0          5.3         190.4       1.0X
+before 1582, noop                                 11543          11543           0          8.7         115.4       1.6X
+after 1582, rebase off                            31999          31999           0          3.1         320.0       0.6X
+after 1582, rebase on                             31657          31657           0          3.2         316.6       0.6X
+before 1582, rebase off                           23777          23777           0          4.2         237.8       0.8X
+before 1582, rebase on                            24767          24767           0          4.0         247.7       0.8X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12944          13064         104          7.7         129.4       1.0X
-after 1582, vec off, rebase on                    13223          13255          34          7.6         132.2       1.0X
-after 1582, vec on, rebase off                     3656           3793         219         27.4          36.6       3.5X
-after 1582, vec on, rebase on                      5176           5205          38         19.3          51.8       2.5X
-before 1582, vec off, rebase off                  12926          12970          45          7.7         129.3       1.0X
-before 1582, vec off, rebase on                   13836          13872          40          7.2         138.4       0.9X
-before 1582, vec on, rebase off                    3664           3672           7         27.3          36.6       3.5X
-before 1582, vec on, rebase on                     6049           6078          30         16.5          60.5       2.1X
+after 1582, vec off, rebase off                   12901          13119         211          7.8         129.0       1.0X
+after 1582, vec off, rebase on                    12968          13078         153          7.7         129.7       1.0X
+after 1582, vec on, rebase off                     3633           3673          37         27.5          36.3       3.6X
+after 1582, vec on, rebase on                      4988           5035          42         20.0          49.9       2.6X
+before 1582, vec off, rebase off                  12767          12856          77          7.8         127.7       1.0X
+before 1582, vec off, rebase on                   14012          14051          41          7.1         140.1       0.9X
+before 1582, vec on, rebase off                    3626           3641          15         27.6          36.3       3.6X
+before 1582, vec on, rebase on                     5388           5458         114         18.6          53.9       2.4X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2831           2831           0         35.3          28.3       1.0X
-before 1582, noop                                  2769           2769           0         36.1          27.7       1.0X
-after 1582, rebase off                            17296          17296           0          5.8         173.0       0.2X
-after 1582, rebase on                             81434          81434           0          1.2         814.3       0.0X
-before 1582, rebase off                           17563          17563           0          5.7         175.6       0.2X
-before 1582, rebase on                            94977          94977           0          1.1         949.8       0.0X
+after 1582, noop                                   3075           3075           0         32.5          30.7       1.0X
+before 1582, noop                                  3058           3058           0         32.7          30.6       1.0X
+after 1582, rebase off                            17236          17236           0          5.8         172.4       0.2X
+after 1582, rebase on                             19237          19237           0          5.2         192.4       0.2X
+before 1582, rebase off                           17317          17317           0          5.8         173.2       0.2X
+before 1582, rebase on                            19982          19982           0          5.0         199.8       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   14824          14920          87          6.7         148.2       1.0X
-after 1582, vec off, rebase on                    54660          54859         210          1.8         546.6       0.3X
-after 1582, vec on, rebase off                     4876           4954          69         20.5          48.8       3.0X
-after 1582, vec on, rebase on                     44509          44573          65          2.2         445.1       0.3X
-before 1582, vec off, rebase off                  14909          14939          30          6.7         149.1       1.0X
-before 1582, vec off, rebase on                   56092          56346         326          1.8         560.9       0.3X
-before 1582, vec on, rebase off                    4846           4858          13         20.6          48.5       3.1X
-before 1582, vec on, rebase on                    46267          46341          64          2.2         462.7       0.3X
+after 1582, vec off, rebase off                   14865          14957         142          6.7         148.6       1.0X
+after 1582, vec off, rebase on                    17154          17190          45          5.8         171.5       0.9X
+after 1582, vec on, rebase off                     4878           4912          56         20.5          48.8       3.0X
+after 1582, vec on, rebase on                      8604           8653          42         11.6          86.0       1.7X
+before 1582, vec off, rebase off                  14891          14929          33          6.7         148.9       1.0X
+before 1582, vec off, rebase on                   17708          17758          68          5.6         177.1       0.8X
+before 1582, vec on, rebase off                    4928           4946          17         20.3          49.3       3.0X
+before 1582, vec on, rebase on                     9147           9152           7         10.9          91.5       1.6X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  18035          18035           0          5.5         180.3       1.0X
-before 1582, noop                                 10571          10571           0          9.5         105.7       1.7X
-after 1582                                        26341          26341           0          3.8         263.4       0.7X
-before 1582                                       19589          19589           0          5.1         195.9       0.9X
+after 1582, noop                                  18943          18943           0          5.3         189.4       1.0X
+before 1582, noop                                 11151          11151           0          9.0         111.5       1.7X
+after 1582                                        26738          26738           0          3.7         267.4       0.7X
+before 1582                                       19636          19636           0          5.1         196.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               34955          35025          98          2.9         349.6       1.0X
-after 1582, vec on                                 3892           3942          43         25.7          38.9       9.0X
-before 1582, vec off                              33360          33457          87          3.0         333.6       1.0X
-before 1582, vec on                                4514           4526          11         22.2          45.1       7.7X
+after 1582, vec off                               36176          36378         206          2.8         361.8       1.0X
+after 1582, vec on                                 3802           3829          29         26.3          38.0       9.5X
+before 1582, vec off                              35815          35882          74          2.8         358.1       1.0X
+before 1582, vec on                                4163           4174          10         24.0          41.6       8.7X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2867           2867           0         34.9          28.7       1.0X
-before 1582, noop                                  2812           2812           0         35.6          28.1       1.0X
-after 1582                                        57133          57133           0          1.8         571.3       0.1X
-before 1582                                       53435          53435           0          1.9         534.4       0.1X
+after 1582, noop                                   3033           3033           0         33.0          30.3       1.0X
+before 1582, noop                                  3004           3004           0         33.3          30.0       1.0X
+after 1582                                        53907          53907           0          1.9         539.1       0.1X
+before 1582                                       57241          57241           0          1.7         572.4       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               38475          38551         127          2.6         384.7       1.0X
-after 1582, vec on                                31382          31537         136          3.2         313.8       1.2X
-before 1582, vec off                              42159          42240         128          2.4         421.6       0.9X
-before 1582, vec on                               34735          35129         393          2.9         347.3       1.1X
+after 1582, vec off                               38266          38306          66          2.6         382.7       1.0X
+after 1582, vec on                                30233          30339         162          3.3         302.3       1.3X
+before 1582, vec off                              43584          43753         259          2.3         435.8       0.9X
+before 1582, vec on                               35681          35832         140          2.8         356.8       1.1X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index cc28244e4c973..a79f6fba92286 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23840          23840           0          4.2         238.4       1.0X
-before 1582, noop                                 10842          10842           0          9.2         108.4       2.2X
-after 1582, rebase off                            35883          35883           0          2.8         358.8       0.7X
-after 1582, rebase on                             36090          36090           0          2.8         360.9       0.7X
-before 1582, rebase off                           23293          23293           0          4.3         232.9       1.0X
-before 1582, rebase on                            23803          23803           0          4.2         238.0       1.0X
+after 1582, noop                                  24030          24030           0          4.2         240.3       1.0X
+before 1582, noop                                 10900          10900           0          9.2         109.0       2.2X
+after 1582, rebase off                            35666          35666           0          2.8         356.7       0.7X
+after 1582, rebase on                             35872          35872           0          2.8         358.7       0.7X
+before 1582, rebase off                           22131          22131           0          4.5         221.3       1.1X
+before 1582, rebase on                            22967          22967           0          4.4         229.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12608          12658          43          7.9         126.1       1.0X
-after 1582, vec off, rebase on                    13385          13585         239          7.5         133.9       0.9X
-after 1582, vec on, rebase off                     3749           3792          58         26.7          37.5       3.4X
-after 1582, vec on, rebase on                      5256           5296          49         19.0          52.6       2.4X
-before 1582, vec off, rebase off                  13015          13140         109          7.7         130.1       1.0X
-before 1582, vec off, rebase on                   14273          14372         109          7.0         142.7       0.9X
-before 1582, vec on, rebase off                    3742           3766          22         26.7          37.4       3.4X
-before 1582, vec on, rebase on                     6068           6076           7         16.5          60.7       2.1X
+after 1582, vec off, rebase off                   12523          12589          58          8.0         125.2       1.0X
+after 1582, vec off, rebase on                    13387          13466          86          7.5         133.9       0.9X
+after 1582, vec on, rebase off                     3646           3678          41         27.4          36.5       3.4X
+after 1582, vec on, rebase on                      5104           5198         123         19.6          51.0       2.5X
+before 1582, vec off, rebase off                  13041          13101          98          7.7         130.4       1.0X
+before 1582, vec off, rebase on                   14077          14104          23          7.1         140.8       0.9X
+before 1582, vec on, rebase off                    3667           3726          87         27.3          36.7       3.4X
+before 1582, vec on, rebase on                     5504           5521          21         18.2          55.0       2.3X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2783           2783           0         35.9          27.8       1.0X
-before 1582, noop                                  2783           2783           0         35.9          27.8       1.0X
-after 1582, rebase off                            17016          17016           0          5.9         170.2       0.2X
-after 1582, rebase on                            107890         107890           0          0.9        1078.9       0.0X
-before 1582, rebase off                           17352          17352           0          5.8         173.5       0.2X
-before 1582, rebase on                           122807         122807           0          0.8        1228.1       0.0X
+after 1582, noop                                   2782           2782           0         36.0          27.8       1.0X
+before 1582, noop                                  2778           2778           0         36.0          27.8       1.0X
+after 1582, rebase off                            16980          16980           0          5.9         169.8       0.2X
+after 1582, rebase on                             20023          20023           0          5.0         200.2       0.1X
+before 1582, rebase off                           17618          17618           0          5.7         176.2       0.2X
+before 1582, rebase on                            20416          20416           0          4.9         204.2       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15066          15104          38          6.6         150.7       1.0X
-after 1582, vec off, rebase on                    68057          68312         222          1.5         680.6       0.2X
-after 1582, vec on, rebase off                     4878           4953          77         20.5          48.8       3.1X
-after 1582, vec on, rebase on                     57278          57320          41          1.7         572.8       0.3X
-before 1582, vec off, rebase off                  15293          15389          84          6.5         152.9       1.0X
-before 1582, vec off, rebase on                   72951          73139         163          1.4         729.5       0.2X
-before 1582, vec on, rebase off                    4883           4951          78         20.5          48.8       3.1X
-before 1582, vec on, rebase on                    59369          59543         226          1.7         593.7       0.3X
+after 1582, vec off, rebase off                   15098          15146          55          6.6         151.0       1.0X
+after 1582, vec off, rebase on                    18127          18214          86          5.5         181.3       0.8X
+after 1582, vec on, rebase off                     4936           4984          42         20.3          49.4       3.1X
+after 1582, vec on, rebase on                      9685           9745          58         10.3          96.8       1.6X
+before 1582, vec off, rebase off                  15233          15259          25          6.6         152.3       1.0X
+before 1582, vec off, rebase on                   18710          18727          24          5.3         187.1       0.8X
+before 1582, vec on, rebase off                    4954           4980          24         20.2          49.5       3.0X
+before 1582, vec on, rebase on                    10224          10240          22          9.8         102.2       1.5X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23498          23498           0          4.3         235.0       1.0X
-before 1582, noop                                 10828          10828           0          9.2         108.3       2.2X
-after 1582                                        32267          32267           0          3.1         322.7       0.7X
-before 1582                                       20339          20339           0          4.9         203.4       1.2X
+after 1582, noop                                  23770          23770           0          4.2         237.7       1.0X
+before 1582, noop                                 10939          10939           0          9.1         109.4       2.2X
+after 1582                                        32879          32879           0          3.0         328.8       0.7X
+before 1582                                       20267          20267           0          4.9         202.7       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               46140          46161          21          2.2         461.4       1.0X
-after 1582, vec on                                 3711           3748          33         26.9          37.1      12.4X
-before 1582, vec off                              42527          42745         189          2.4         425.3       1.1X
-before 1582, vec on                                4159           4169          15         24.0          41.6      11.1X
+after 1582, vec off                               39651          39686          31          2.5         396.5       1.0X
+after 1582, vec on                                 3647           3660          13         27.4          36.5      10.9X
+before 1582, vec off                              38155          38219          61          2.6         381.6       1.0X
+before 1582, vec on                                4041           4046           6         24.7          40.4       9.8X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2876           2876           0         34.8          28.8       1.0X
-before 1582, noop                                  2957           2957           0         33.8          29.6       1.0X
-after 1582                                        68393          68393           0          1.5         683.9       0.0X
-before 1582                                       68847          68847           0          1.5         688.5       0.0X
+after 1582, noop                                   2802           2802           0         35.7          28.0       1.0X
+before 1582, noop                                  2797           2797           0         35.8          28.0       1.0X
+after 1582                                        59877          59877           0          1.7         598.8       0.0X
+before 1582                                       61361          61361           0          1.6         613.6       0.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               55967          56011          43          1.8         559.7       1.0X
-after 1582, vec on                                48930          49156         342          2.0         489.3       1.1X
-before 1582, vec off                              60544          60863         311          1.7         605.4       0.9X
-before 1582, vec on                               52539          52665         109          1.9         525.4       1.1X
+after 1582, vec off                               48197          48288         118          2.1         482.0       1.0X
+after 1582, vec on                                38247          38351         128          2.6         382.5       1.3X
+before 1582, vec off                              53179          53359         249          1.9         531.8       0.9X
+before 1582, vec on                               44076          44268         269          2.3         440.8       1.1X
 
 
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index f209f7f288f58..79edec315ac2f 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -35,6 +35,7 @@
 import org.apache.parquet.schema.PrimitiveType;
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 import org.apache.spark.sql.internal.SQLConf;
@@ -426,7 +427,7 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
           if (defColumn.readInteger() == maxDefLevel) {
             column.putInt(
               rowId + i,
-              DateTimeUtils.rebaseJulianToGregorianDays(dataColumn.readInteger()));
+              RebaseDateTime.rebaseJulianToGregorianDays(dataColumn.readInteger()));
           } else {
             column.putNull(rowId + i);
           }
@@ -452,7 +453,7 @@ private void readLongBatch(int rowId, int num, WritableColumnVector column) thro
           if (defColumn.readInteger() == maxDefLevel) {
             column.putLong(
               rowId + i,
-              DateTimeUtils.rebaseJulianToGregorianMicros(dataColumn.readLong()));
+              RebaseDateTime.rebaseJulianToGregorianMicros(dataColumn.readLong()));
           } else {
             column.putNull(rowId + i);
           }
@@ -466,7 +467,7 @@ private void readLongBatch(int rowId, int num, WritableColumnVector column) thro
         for (int i = 0; i < num; i++) {
           if (defColumn.readInteger() == maxDefLevel) {
             long micros = DateTimeUtils.fromMillis(dataColumn.readLong());
-            column.putLong(rowId + i, DateTimeUtils.rebaseJulianToGregorianMicros(micros));
+            column.putLong(rowId + i, RebaseDateTime.rebaseJulianToGregorianMicros(micros));
           } else {
             column.putNull(rowId + i);
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
index 00b710f57d0f8..486f678e49d7d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
@@ -23,7 +23,7 @@ import java.sql.Date
 import org.apache.hadoop.hive.serde2.io.DateWritable
 import org.apache.hadoop.io.WritableUtils
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
 
 /**
  * The class accepts/returns days in Gregorian calendar and rebase them
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 18011dbf6cd6b..a5e769c2dc1d2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -277,7 +278,7 @@ private[parquet] class ParquetRowConverter(
         if (rebaseDateTime) {
           new ParquetPrimitiveConverter(updater) {
             override def addLong(value: Long): Unit = {
-              val rebased = DateTimeUtils.rebaseJulianToGregorianMicros(value)
+              val rebased = rebaseJulianToGregorianMicros(value)
               updater.setLong(rebased)
             }
           }
@@ -294,7 +295,7 @@ private[parquet] class ParquetRowConverter(
           new ParquetPrimitiveConverter(updater) {
             override def addLong(value: Long): Unit = {
               val micros = DateTimeUtils.fromMillis(value)
-              val rebased = DateTimeUtils.rebaseJulianToGregorianMicros(micros)
+              val rebased = rebaseJulianToGregorianMicros(micros)
               updater.setLong(rebased)
             }
           }
@@ -330,7 +331,7 @@ private[parquet] class ParquetRowConverter(
         if (rebaseDateTime) {
           new ParquetPrimitiveConverter(updater) {
             override def addInt(value: Int): Unit = {
-              updater.set(DateTimeUtils.rebaseJulianToGregorianDays(value))
+              updater.set(rebaseJulianToGregorianDays(value))
             }
           }
         } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index 1c4f2978a48d1..409ea88d6e0c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
@@ -153,7 +154,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
 
       case DateType if rebaseDateTime =>
         (row: SpecializedGetters, ordinal: Int) =>
-          val rebasedDays = DateTimeUtils.rebaseGregorianToJulianDays(row.getInt(ordinal))
+          val rebasedDays = rebaseGregorianToJulianDays(row.getInt(ordinal))
           recordConsumer.addInteger(rebasedDays)
 
       case IntegerType | DateType =>
@@ -188,7 +189,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
 
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS if rebaseDateTime =>
             (row: SpecializedGetters, ordinal: Int) =>
-              val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(row.getLong(ordinal))
+              val rebasedMicros = rebaseGregorianToJulianMicros(row.getLong(ordinal))
               recordConsumer.addLong(rebasedMicros)
 
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS =>
@@ -197,7 +198,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
 
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS if rebaseDateTime =>
             (row: SpecializedGetters, ordinal: Int) =>
-              val rebasedMicros = DateTimeUtils.rebaseGregorianToJulianMicros(row.getLong(ordinal))
+              val rebasedMicros = rebaseGregorianToJulianMicros(row.getLong(ordinal))
               val millis = DateTimeUtils.toMillis(rebasedMicros)
               recordConsumer.addLong(millis)
 
diff --git a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 5dc3f3743ce91..6601bcb9018f4 100644
--- a/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v1.2/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -22,6 +22,7 @@
 import org.apache.orc.storage.ql.exec.vector.*;
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
@@ -140,7 +141,7 @@ public short getShort(int rowId) {
   public int getInt(int rowId) {
     int value = (int) longData.vector[getRowIndex(rowId)];
     if (isDate) {
-      return DateTimeUtils.rebaseJulianToGregorianDays(value);
+      return RebaseDateTime.rebaseJulianToGregorianDays(value);
     } else {
       return value;
     }
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
index 4934a9650eee1..1dccf0ca1faef 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/DaysWritable.scala
@@ -23,7 +23,7 @@ import java.sql.Date
 import org.apache.hadoop.io.WritableUtils
 import org.apache.orc.storage.serde2.io.DateWritable
 
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.{rebaseGregorianToJulianDays, rebaseJulianToGregorianDays}
 
 /**
  * The class accepts/returns days in Gregorian calendar and rebase them
diff --git a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
index 7be4a6f6c72c8..6e55fedfc4deb 100644
--- a/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
+++ b/sql/core/v2.3/src/main/java/org/apache/spark/sql/execution/datasources/orc/OrcColumnVector.java
@@ -22,6 +22,7 @@
 import org.apache.hadoop.hive.ql.exec.vector.*;
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
@@ -140,7 +141,7 @@ public short getShort(int rowId) {
   public int getInt(int rowId) {
     int value = (int) longData.vector[getRowIndex(rowId)];
     if (isDate) {
-      return DateTimeUtils.rebaseJulianToGregorianDays(value);
+      return RebaseDateTime.rebaseJulianToGregorianDays(value);
     } else {
       return value;
     }

From d2ba5b2e6962b0069e28243639215ee6f5ee50a3 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 10 Apr 2020 03:43:32 +0000
Subject: [PATCH 0472/1280] [SPARK-31359][DOC][FOLLOWUP] improve code comments
 in RebaseDateTime

improve the code comment and make them consistent between `rebaseJulianToGregorian*` and `rebaseGregorianToJulian*`

improve readability.

no

N/A

Closes #28166 from cloud-fan/comment.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 148950fa2ba7ad03d7536c8c2201ba18b9345c0e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    | 176 ++++++++++--------
 1 file changed, 98 insertions(+), 78 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 793eb17f72ae0..74407873b19f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -32,20 +32,20 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 /**
  * The collection of functions for rebasing days and microseconds from/to the hybrid calendar
  * (Julian + Gregorian since 1582-10-15) which is used by Spark 2.4 and earlier versions
- * to/from Proleptic Gregorian calendar used in Spark SQL since version 3.0, see SPARK-26651.
+ * to/from Proleptic Gregorian calendar which is used by Spark since version 3.0. See SPARK-26651.
  */
 object RebaseDateTime {
   /**
    * Rebases days since the epoch from an original to an target calendar, for instance,
    * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
    *
-   * It finds the latest switch day which is less than `value`, and adds the difference
-   * in days associated with the switch days to the given `value`.
+   * It finds the latest switch day which is less than the given `days`, and adds the difference
+   * in days associated with the switch days to the given `days`.
    * The function is based on linear search which starts from the most recent switch days.
    * This allows to perform less comparisons for modern dates.
    *
-   * @param switches The days when difference in days between original
-   *                 and target calendar was changed.
+   * @param switches The days when difference in days between original and target calendar
+   *                 was changed.
    * @param diffs The differences in days between calendars.
    * @param days The number of days since the epoch 1970-01-01 to be rebased
    *             to the target calendar.
@@ -71,12 +71,20 @@ object RebaseDateTime {
     -390745, -354220, -317695, -244645, -208120, -171595, -141427)
 
   /**
-   * Converts the given number of days since the epoch day 1970-01-01 to
-   * a local date in Julian calendar, interprets the result as a local
-   * date in Proleptic Gregorian calendar, and take the number of days
-   * since the epoch from the Gregorian date.
+   * Converts the given number of days since the epoch day 1970-01-01 to a local date in Julian
+   * calendar, interprets the result as a local date in Proleptic Gregorian calendar, and takes the
+   * number of days since the epoch from the Gregorian local date.
    *
-   * @param days The number of days since the epoch in Julian calendar.
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Julian calendar: 1582-01-01 -> -141704
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   * The code below converts -141704 to -141714.
+   *
+   * @param days The number of days since the epoch in Julian calendar. It can be negative.
    * @return The rebased number of days in Gregorian calendar.
    */
   def rebaseJulianToGregorianDays(days: Int): Int = {
@@ -97,51 +105,26 @@ object RebaseDateTime {
     -390750, -354226, -317702, -244653, -208129, -171605, -141427)
 
   /**
-   * Rebasing days since the epoch to store the same number of days
-   * as by Spark 2.4 and earlier versions. Spark 3.0 switched to
-   * Proleptic Gregorian calendar (see SPARK-26651), and as a consequence of that,
-   * this affects dates before 1582-10-15. Spark 2.4 and earlier versions use
-   * Julian calendar for dates before 1582-10-15. So, the same local date may
-   * be mapped to different number of days since the epoch in different calendars.
+   * Converts the given number of days since the epoch day 1970-01-01 to a local date in Proleptic
+   * Gregorian calendar, interprets the result as a local date in Julian calendar, and takes the
+   * number of days since the epoch from the Julian local date.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
    *
    * For example:
    *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
    *   Julian calendar: 1582-01-01 -> -141704
    * The code below converts -141714 to -141704.
    *
-   * @param days The number of days since the epoch 1970-01-01. It can be negative.
+   * @param days The number of days since the epoch in Gregorian calendar. It can be negative.
    * @return The rebased number of days since the epoch in Julian calendar.
    */
   def rebaseGregorianToJulianDays(days: Int): Int = {
     rebaseDays(gregJulianDiffSwitchDay, gregJulianDiffs, days)
   }
 
-  /**
-   * Converts the given microseconds to a local date-time in UTC time zone in Proleptic Gregorian
-   * calendar, interprets the result as a local date-time in Julian calendar in UTC time zone.
-   * And takes microseconds since the epoch from the Julian timestamp.
-   *
-   * @param zoneId The time zone ID at which the rebasing should be performed.
-   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
-   *               in Proleptic Gregorian calendar. It can be negative.
-   * @return The rebased microseconds since the epoch in Julian calendar.
-   */
-  private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
-    val instant = microsToInstant(micros)
-    val ldt = instant.atZone(zoneId).toLocalDateTime
-    val cal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
-      .setCalendarType("gregory")
-      .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
-      .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
-      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
-      // This setting selects the original local timestamp mapped to the given `micros`.
-      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
-      .build()
-    fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
-  }
-
 
   /**
    * The class describes JSON records with microseconds rebasing info.
@@ -176,8 +159,8 @@ object RebaseDateTime {
    * Rebases micros since the epoch from an original to an target calendar, for instance,
    * from a hybrid (Julian + Gregorian) to Proleptic Gregorian calendar.
    *
-   * It finds the latest switch micros which is less than `value`, and adds the difference
-   * in micros associated with the switch micros to the given `value`.
+   * It finds the latest switch micros which is less than the given `micros`, and adds the
+   * difference in micros associated with the switch micros to the given `micros`.
    * The function is based on linear search which starts from the most recent switch micros.
    * This allows to perform less comparisons for modern timestamps.
    *
@@ -227,22 +210,53 @@ object RebaseDateTime {
   private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json")
 
   /**
-   * Rebases the given `micros` to the number of microseconds since the epoch via a local
-   * timestamp that have the same date-time fields in Proleptic Gregorian and in the hybrid
-   * calendars.
+   * Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
+   * date-time in Proleptic Gregorian calendar with timezone identified by `zoneId`, interprets the
+   * result as a local date-time in Julian calendar with the same timezone, and takes microseconds
+   * since the epoch from the Julian local date-time.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
    *
-   * The function may optimize the rebasing by using pre-calculated rebasing maps. If the maps
-   * don't contains information about the current JVM system time zone, the functions falls back
-   * to regular conversion mechanism via building local timestamps.
+   * For example:
+   *   Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
+   *   Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
+   * The code below converts -12244061221876544 to -12243196799876544.
+   *
+   * @param zoneId The time zone ID at which the rebasing should be performed.
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in Proleptic Gregorian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Julian calendar.
+   */
+  private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
+    val instant = microsToInstant(micros)
+    val ldt = instant.atZone(zoneId).toLocalDateTime
+    val cal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
+      .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
+      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
+      // This setting selects the original local timestamp mapped to the given `micros`.
+      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
+      .build()
+    fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
+  }
+
+  /**
+   * An optimized version of [[rebaseGregorianToJulianMicros(ZoneId, Long)]]. This method leverages
+   * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
+   * information about the current JVM system time zone, the function falls back to the regular
+   * unoptimized version.
    *
    * Note: The function assumes that the input micros was derived from a local timestamp
    *       at the default system JVM time zone in Proleptic Gregorian calendar.
    *
-   * @param micros The microseconds since the epoch 1970-01-01T00:00:00Z
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
    *               in Proleptic Gregorian calendar. It can be negative.
-   * @return The microseconds since the epoch that have the same local timestamp representation
-   *         in the hybrid calendar (Julian + Gregorian) as the original `micros` in
-   *         Proleptic Gregorian calendar.
+   * @return The rebased microseconds since the epoch in Julian calendar.
    */
   def rebaseGregorianToJulianMicros(micros: Long): Long = {
     val timeZone = TimeZone.getDefault
@@ -256,14 +270,23 @@ object RebaseDateTime {
   }
 
   /**
-   * Converts the given microseconds to a local date-time in UTC time zone in the hybrid
-   * calendar (Julian + Gregorian since 1582-10-15), interprets the result as a local date-time
-   * in Proleptic Gregorian calendar in UTC time zone. And takes microseconds since the epoch
-   * from the Gregorian timestamp.
+   * Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
+   * date-time in Julian calendar with timezone identified by `zoneId`, interprets the result as a
+   * local date-time in Proleptic Gregorian calendar with the same timezone, and takes microseconds
+   * since the epoch from the Gregorian local date-time.
+   *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Julian calendar: 1582-01-01 00:00:00.123456 -> -12243196799876544
+   *   Proleptic Gregorian calendar: 1582-01-01 00:00:00.123456 -> -12244061221876544
+   * The code below converts -12243196799876544 to -12244061221876544.
    *
    * @param zoneId The time zone ID at which the rebasing should be performed.
    * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
-   *               in the hybrid calendar (Julian + Gregorian). It can be negative.
+   *               in the Julian calendar. It can be negative.
    * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
    */
   private[sql] def rebaseJulianToGregorianMicros(zoneId: ZoneId, micros: Long): Long = {
@@ -287,9 +310,13 @@ object RebaseDateTime {
       (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
       .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
     val zonedDateTime = localDateTime.atZone(zoneId)
-    // Zero DST offset means that local clocks have switched to the winter time already.
-    // So, clocks go back one hour. We should correct zoned date-time and change
-    // the zone offset to the later of the two valid offsets at a local time-line overlap.
+    // Assuming the daylight saving switchover time is 2:00, the local clock will go back to
+    // 2:00 after hitting 2:59. This means the local time between [2:00, 3:00) appears twice, and
+    // can map to two different physical times (seconds from the UTC epoch).
+    // Java 8 time API resolves the ambiguity by picking the earlier physical time. This is the same
+    // as Java 7 time API, except for 2:00 where Java 7 picks the later physical time.
+    // Here we detect the "2:00" case and pick the latter physical time, to be compatible with the
+    // Java 7 date-time.
     val adjustedZdt = if (cal.get(Calendar.DST_OFFSET) == 0) {
       zonedDateTime.withLaterOffsetAtOverlap()
     } else {
@@ -306,24 +333,17 @@ object RebaseDateTime {
   private val julianGregRebaseMap = loadRebaseRecords("julian-gregorian-rebase-micros.json")
 
   /**
-   * This is an opposite to `rebaseGregorianToJulianMicros` function which rebases the given
-   * microseconds since the epoch 1970-01-01T00:00:00Z via local timestamps from the
-   * hybrid calendar (Julian + Gregorian) to Proleptic Gregorian calendar.
-   * For example, the input `micros` -12243196799876544 is mapped to 1582-01-01 00:00:00.123456 in
-   * Julian calendar. The same local timestamp in Proleptic Gregorian calendar is mapped to
-   * the different number of micros -12244061221876544 since the epoch. Semantically, the function
-   * performs such conversion either via direct conversion to local timestamps,
-   * or via pre-calculated rebasing tables.
+   * An optimized version of [[rebaseJulianToGregorianMicros(ZoneId, Long)]]. This method leverages
+   * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
+   * information about the current JVM system time zone, the function falls back to the regular
+   * unoptimized version.
    *
    * Note: The function assumes that the input micros was derived from a local timestamp
-   *       at the default system JVM time zone in the hybrid calendar (Julian and Gregorian
-   *       since 1582-10-15)
+   *       at the default system JVM time zone in the Julian calendar.
    *
-   * @param micros The number of microseconds since the epoch 1970-01-01T00:00:00Z
-   *               in the hybrid calendar (Julian + Gregorian). It can be negative.
-   * @return The rebased number of microseconds since the epoch which is mapped to the same
-   *         local timestamp in Proleptic Gregorian calendar as `micros` in the hybrid calendar
-   *         at the system time zone.
+   * @param micros The number of microseconds since the epoch '1970-01-01T00:00:00Z'
+   *               in the Julian calendar. It can be negative.
+   * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
    */
   def rebaseJulianToGregorianMicros(micros: Long): Long = {
     val timeZone = TimeZone.getDefault

From 8bac0ee52a018b88324efc2a4ddbedfb3694b58f Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 9 Apr 2020 21:36:26 -0700
Subject: [PATCH 0473/1280] [SPARK-31401][K8S] Show JDK11 usage in
 `bin/docker-image-tool.sh`

### What changes were proposed in this pull request?

This PR adds an JDK11-based build example in `bin/docker-image-tool.sh`.

### Why are the changes needed?

This helps the users migrate to JDK11 more easily.

### Does this PR introduce any user-facing change?

Yes, but this is a usage help.

### How was this patch tested?

First, check the help usage manually.
```
$ bin/docker-image-tool.sh -h
...
  - Build and push JDK11-based image with tag "v3.0.0" to docker.io/myrepo
    bin/docker-image-tool.sh -r docker.io/myrepo -t v3.0.0 -b java_image_tag=11-jre-slim build
    bin/docker-image-tool.sh -r docker.io/myrepo -t v3.0.0 push {code}
```

Then, build the image and check Java version.
```
$ docker run -it --rm docker.io/myrepo/spark:v3.0.0 java --version | tail -n3
openjdk 11.0.6 2020-01-14
OpenJDK Runtime Environment 18.9 (build 11.0.6+10)
OpenJDK 64-Bit Server VM 18.9 (build 11.0.6+10, mixed mode)
```

Closes #28171 from dongjoon-hyun/SPARK-31401.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e42a3945acd614a26c7941a9eed161b500fb4520)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 bin/docker-image-tool.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 68fafbb848001..57b86254ab424 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -248,6 +248,10 @@ Examples:
   - Build and push image with tag "v2.3.0" to docker.io/myrepo
     $0 -r docker.io/myrepo -t v2.3.0 build
     $0 -r docker.io/myrepo -t v2.3.0 push
+
+  - Build and push JDK11-based image with tag "v3.0.0" to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v3.0.0 -b java_image_tag=11-jre-slim build
+    $0 -r docker.io/myrepo -t v3.0.0 push
 EOF
 }
 

From 6c9827f5c20ba2d477eab28288039abb2baf9324 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 10 Apr 2020 08:16:48 +0000
Subject: [PATCH 0474/1280] [SPARK-31384][SQL] Fix NPE in OptimizeSkewedJoin

1. Fix NPE in `OptimizeSkewedJoin`

2. prevent other potential NPE errors in AQE.

When there's a `inputRDD` of a plan has 0 partition, rule `OptimizeSkewedJoin` can hit NPE error because this kind of RDD means a null `MapOutputStatistics` due to:

https://github.com/apache/spark/blob/d98df7626b8a88cb9a1fee4f454b19333a9f3ced/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala#L68-L69

Thus, we should take care of such NPE errors in other places too.

No

Added a test.

Closes #28153 from Ngone51/npe.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6cddda7847ceabd257008986ede7ec844f1f3607)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/CoalesceShufflePartitions.scala     | 11 +++--------
 .../adaptive/DemoteBroadcastHashJoin.scala       |  9 ++++-----
 .../execution/adaptive/OptimizeSkewedJoin.scala  | 15 +++++----------
 .../sql/execution/adaptive/QueryStageExec.scala  | 10 ++++++++++
 .../adaptive/AdaptiveQueryExecSuite.scala        | 16 +++++++++++++++-
 5 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
index 226d692f7b404..096d65f16e42f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CoalesceShufflePartitions.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.MapOutputStatistics
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.SparkPlan
@@ -54,14 +53,10 @@ case class CoalesceShufflePartitions(session: SparkSession) extends Rule[SparkPl
     if (!shuffleStages.forall(_.shuffle.canChangeNumPartitions)) {
       plan
     } else {
-      val shuffleMetrics = shuffleStages.map { stage =>
-        assert(stage.resultOption.isDefined, "ShuffleQueryStageExec should already be ready")
-        stage.resultOption.get.asInstanceOf[MapOutputStatistics]
-      }
-
-      // `ShuffleQueryStageExec` gives null mapOutputStatistics when the input RDD has 0 partitions,
+      // `ShuffleQueryStageExec#mapStats` returns None when the input RDD has 0 partitions,
       // we should skip it when calculating the `partitionStartIndices`.
-      val validMetrics = shuffleMetrics.filter(_ != null)
+      val validMetrics = shuffleStages.flatMap(_.mapStats)
+
       // We may have different pre-shuffle partition numbers, don't reduce shuffle partition number
       // in that case. For example when we union fully aggregated data (data is arranged to a single
       // partition) and a result of a SortMergeJoin (multiple partitions).
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
index e5642991c59a3..0f2868e41cc39 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/DemoteBroadcastHashJoin.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import org.apache.spark.MapOutputStatistics
 import org.apache.spark.sql.catalyst.plans.logical.{HintInfo, Join, LogicalPlan, NO_BROADCAST_HASH}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
@@ -30,10 +29,10 @@ case class DemoteBroadcastHashJoin(conf: SQLConf) extends Rule[LogicalPlan] {
 
   private def shouldDemote(plan: LogicalPlan): Boolean = plan match {
     case LogicalQueryStage(_, stage: ShuffleQueryStageExec) if stage.resultOption.isDefined
-      && stage.resultOption.get != null =>
-      val mapOutputStatistics = stage.resultOption.get.asInstanceOf[MapOutputStatistics]
-      val partitionCnt = mapOutputStatistics.bytesByPartitionId.length
-      val nonZeroCnt = mapOutputStatistics.bytesByPartitionId.count(_ > 0)
+      && stage.mapStats.isDefined =>
+      val mapStats = stage.mapStats.get
+      val partitionCnt = mapStats.bytesByPartitionId.length
+      val nonZeroCnt = mapStats.bytesByPartitionId.count(_ > 0)
       partitionCnt > 0 && nonZeroCnt > 0 &&
         (nonZeroCnt * 1.0 / partitionCnt) < conf.nonEmptyPartitionRatioForBroadcastJoin
     case _ => false
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index b09e5630f89c3..d94999a7387a4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -286,16 +286,17 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 
 private object ShuffleStage {
   def unapply(plan: SparkPlan): Option[ShuffleStageInfo] = plan match {
-    case s: ShuffleQueryStageExec =>
-      val mapStats = getMapStats(s)
+    case s: ShuffleQueryStageExec if s.mapStats.isDefined =>
+      val mapStats = s.mapStats.get
       val sizes = mapStats.bytesByPartitionId
       val partitions = sizes.zipWithIndex.map {
         case (size, i) => CoalescedPartitionSpec(i, i + 1) -> size
       }
       Some(ShuffleStageInfo(s, mapStats, partitions))
 
-    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs, _) =>
-      val mapStats = getMapStats(s)
+    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs, _)
+      if s.mapStats.isDefined =>
+      val mapStats = s.mapStats.get
       val sizes = mapStats.bytesByPartitionId
       val partitions = partitionSpecs.map {
         case spec @ CoalescedPartitionSpec(start, end) =>
@@ -313,12 +314,6 @@ private object ShuffleStage {
 
     case _ => None
   }
-
-  private def getMapStats(stage: ShuffleQueryStageExec): MapOutputStatistics = {
-    assert(stage.resultOption.isDefined, "ShuffleQueryStageExec should" +
-      " already be ready when executing OptimizeSkewedPartitions rule")
-    stage.resultOption.get.asInstanceOf[MapOutputStatistics]
-  }
 }
 
 private case class ShuffleStageInfo(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
index d5dc1be63f06e..beaa9721fb5e5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
@@ -161,6 +161,16 @@ case class ShuffleQueryStageExec(
       case _ =>
     }
   }
+
+  /**
+   * Returns the Option[MapOutputStatistics]. If the shuffle map stage has no partition,
+   * this method returns None, as there is no map statistics.
+   */
+  def mapStats: Option[MapOutputStatistics] = {
+    assert(resultOption.isDefined, "ShuffleQueryStageExec should already be ready")
+    val stats = resultOption.get.asInstanceOf[MapOutputStatistics]
+    Option(stats)
+  }
 }
 
 /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index d4c5b0d42fb11..bfde042e9208f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -23,7 +23,7 @@ import java.net.URI
 import org.apache.log4j.Level
 
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
-import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
@@ -31,6 +31,7 @@ import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight,
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{IntegerType, StructType}
 import org.apache.spark.util.Utils
 
 class AdaptiveQueryExecSuite
@@ -781,4 +782,17 @@ class AdaptiveQueryExecSuite
       }
     }
   }
+
+  test("SPARK-31384: avoid NPE in OptimizeSkewedJoin when there's 0 partition plan") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+      withTempView("t2") {
+        // create DataFrame with 0 partition
+        spark.createDataFrame(sparkContext.emptyRDD[Row], new StructType().add("b", IntegerType))
+          .createOrReplaceTempView("t2")
+        // should run successfully without NPE
+        runAdaptiveAndVerifyResult("SELECT * FROM testData2 t1 left semi join t2 ON t1.a=t2.b")
+      }
+    }
+  }
 }

From 15d45c0dfe60a4353791fbfaca953bad801f267c Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Fri, 10 Apr 2020 13:08:19 +0000
Subject: [PATCH 0475/1280] [SPARK-31406][SQL][TEST]
 ThriftServerQueryTestSuite: Sharing test data and test tables among multiple
 test cases
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR is related to https://github.com/apache/spark/pull/28060.
`ThriftServerQueryTestSuite` spend 17 minutes time to test.
I checked the code and found `ThriftServerQueryTestSuite` load test data repeatedly.
I've listed all the test cases order by time with desc in the `hive-thriftserver` module below.

Class | Spend time  ↑ | Failure | Skip | Pass | Total test case
-- | -- | -- | -- | -- | --
ThriftServerQueryTestSuite | 17 minutes | 0 | 15 | 140 | 155
CliSuite | 8 minutes 24 seconds | 0 | 0 | 24 | 24
SparkThriftServerProtocolVersionsSuite | 59 seconds | 0 | 0 | 210 | 210
HiveThriftBinaryServerSuite | 36 seconds | 0 | 1 | 21 | 22
SparkMetadataOperationSuite | 19 seconds | 0 | 0 | 7 | 7
HiveCliSessionStateSuite | 16 seconds | 0 | 0 | 2 | 2
SparkSQLEnvSuite | 16 seconds | 0 | 0 | 1 | 1
HiveThriftHttpServerSuite | 15 seconds | 0 | 0 | 3 | 3
SingleSessionSuite | 14 seconds | 0 | 0 | 3 | 3
JdbcConnectionUriSuite | 2.1 seconds | 0 | 0 | 1 | 1
ThriftServerWithSparkContextSuite | 1.4 seconds | 0 | 0 | 1 | 1
SparkExecuteStatementOperationSuite | 63 millseconds | 0 | 0 | 2 | 2
UISeleniumSuite | -1 millseconds | 0 | 1 | 0 | 1

I checked the code of `ThriftServerQueryTestSuite` and found `ThriftServerQueryTestSuite` load test data repeatedly.
This PR will improve the performance of `ThriftServerQueryTestSuite`.
Because https://github.com/apache/spark/pull/28060 provides `createTestTables`(https://github.com/apache/spark/blob/e42a3945acd614a26c7941a9eed161b500fb4520/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala#L574) and `removeTestTables`(https://github.com/apache/spark/blob/e42a3945acd614a26c7941a9eed161b500fb4520/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala#L666), this PR will still uses them.
The total time run `ThriftServerQueryTestSuite` before and after this PR show below.
Before
No | Time
-- | --
1 | 18 minutes, 8 seconds
2 | 22 minutes, 44 seconds
3 | 17 minutes, 48 seconds
4 | 18 minutes, 30 seconds

After
No | Time
-- | --
1 | 16 minutes, 11 seconds
2 | 17 minutes, 19 seconds
3 | 18 minutes, 15 seconds
4 | 17 minutes, 27 seconds

### Why are the changes needed?
Improve the performance of `ThriftServerQueryTestSuite`.

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test

Closes #28180 from beliefer/avoid-load-thrift-test-data-repeatedly.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2d3692ed45affb4a0583c684b5167fd6bdff5b55)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../ThriftServerQueryTestSuite.scala          | 57 -------------------
 1 file changed, 57 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index d9ac9ab441f0c..48e3485f8e7a9 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -80,8 +80,6 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
     // We do not test with configSet.
     withJdbcStatement { statement =>
 
-      loadTestData(statement)
-
       configSet.foreach { case (k, v) =>
         statement.execute(s"SET $k = $v")
       }
@@ -262,61 +260,6 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ
     }
   }
 
-  /** Load built-in test tables. */
-  private def loadTestData(statement: Statement): Unit = {
-    // Prepare the data
-    statement.execute(
-      """
-        |CREATE OR REPLACE TEMPORARY VIEW testdata as
-        |SELECT id AS key, CAST(id AS string) AS value FROM range(1, 101)
-      """.stripMargin)
-    statement.execute(
-      """
-        |CREATE OR REPLACE TEMPORARY VIEW arraydata as
-        |SELECT * FROM VALUES
-        |(ARRAY(1, 2, 3), ARRAY(ARRAY(1, 2, 3))),
-        |(ARRAY(2, 3, 4), ARRAY(ARRAY(2, 3, 4))) AS v(arraycol, nestedarraycol)
-      """.stripMargin)
-    statement.execute(
-      """
-        |CREATE OR REPLACE TEMPORARY VIEW mapdata as
-        |SELECT * FROM VALUES
-        |MAP(1, 'a1', 2, 'b1', 3, 'c1', 4, 'd1', 5, 'e1'),
-        |MAP(1, 'a2', 2, 'b2', 3, 'c2', 4, 'd2'),
-        |MAP(1, 'a3', 2, 'b3', 3, 'c3'),
-        |MAP(1, 'a4', 2, 'b4'),
-        |MAP(1, 'a5') AS v(mapcol)
-      """.stripMargin)
-    statement.execute(
-      s"""
-         |CREATE TEMPORARY VIEW aggtest
-         |  (a int, b float)
-         |USING csv
-         |OPTIONS (path '${baseResourcePath.getParent}/test-data/postgresql/agg.data',
-         |  header 'false', delimiter '\t')
-      """.stripMargin)
-    statement.execute(
-      s"""
-         |CREATE OR REPLACE TEMPORARY VIEW onek
-         |  (unique1 int, unique2 int, two int, four int, ten int, twenty int, hundred int,
-         |    thousand int, twothousand int, fivethous int, tenthous int, odd int, even int,
-         |    stringu1 string, stringu2 string, string4 string)
-         |USING csv
-         |OPTIONS (path '${baseResourcePath.getParent}/test-data/postgresql/onek.data',
-         |  header 'false', delimiter '\t')
-      """.stripMargin)
-    statement.execute(
-      s"""
-         |CREATE OR REPLACE TEMPORARY VIEW tenk1
-         |  (unique1 int, unique2 int, two int, four int, ten int, twenty int, hundred int,
-         |    thousand int, twothousand int, fivethous int, tenthous int, odd int, even int,
-         |    stringu1 string, stringu2 string, string4 string)
-         |USING csv
-         |  OPTIONS (path '${baseResourcePath.getParent}/test-data/postgresql/tenk.data',
-         |  header 'false', delimiter '\t')
-      """.stripMargin)
-  }
-
   // Returns true if sql is retrieving data.
   private def isNeedSort(sql: String): Boolean = {
     val upperCase = sql.toUpperCase(Locale.ROOT)

From a0778501aba8177f65082b2a520875951af21068 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 11 Apr 2020 08:23:59 -0700
Subject: [PATCH 0476/1280] [SPARK-31291][SQL][TEST][FOLLOWUP] Fix resource
 loading error in ThriftServerQueryTestSuite

### What changes were proposed in this pull request?

[SPARK-31291](https://github.com/apache/spark/pull/28060) broke `ThriftServerQueryTestSuite` in Maven environment. This PR fixes it by copying the resource file from jars to local temp file.

### Why are the changes needed?

To recover the Jenkins jobs in `master` and `branch-3.0`.
- https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test%20(Dashboard)/job/spark-branch-3.0-test-maven-hadoop-2.7-hive-2.3/211/
```
org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite *** ABORTED ***
...
java.lang.IllegalArgumentException: java.net.URISyntaxException:
Relative path in absolute URI: jar:file:/home/jenkins/workspace/spark-branch-3.0-test-maven-hadoop-2.7-hive-2.3/sql/core/target/
spark-sql_2.12-3.0.1-SNAPSHOT-tests.jar!/test-data/postgresql/agg.data
```

![Screen Shot 2020-04-10 at 9 54 28 PM](https://user-images.githubusercontent.com/9700541/79035702-f03ad900-7b75-11ea-9eee-0c1581a28838.png)

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins with SBT and Maven.
- [x] Sbt (`Test build #121117` https://github.com/apache/spark/pull/28186#issuecomment-612329068)
- [x] Maven (`Test build #121118` https://github.com/apache/spark/pull/28186#issuecomment-612414382)

Closes #28186 from dongjoon-hyun/SPARK-31291.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b4c438a5e09a7b1cd98f9b53e883f339367cfda8)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../hive/thriftserver/ThriftServerQueryTestSuite.scala | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 48e3485f8e7a9..15cc3109da3f7 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -23,6 +23,7 @@ import java.util.{Locale, MissingFormatArgumentException}
 
 import scala.util.control.NonFatal
 
+import org.apache.commons.io.FileUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
 
 import org.apache.spark.SparkException
@@ -53,6 +54,15 @@ import org.apache.spark.sql.types._
  */
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
+  override protected def testFile(fileName: String): String = {
+    val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
+    // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
+    val file = File.createTempFile("thriftserver-test", ".data")
+    file.deleteOnExit()
+    FileUtils.copyURLToFile(url, file)
+    file.getAbsolutePath
+  }
+
   /** List of test cases to ignore, in lower cases. */
   override def blackList: Set[String] = super.blackList ++ Set(
     // Missing UDF

From 9236fc407fe11db13063129d6a2494e573891766 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 11 Apr 2020 08:27:30 -0700
Subject: [PATCH 0477/1280] [SPARK-31422][CORE] Fix NPE when BlockManagerSource
 is used after BlockManagerMaster stops

### What changes were proposed in this pull request?

This PR (SPARK-31422) aims to return empty result in order to avoid `NullPointerException` at `getStorageStatus` and `getMemoryStatus` which happens after `BlockManagerMaster` stops. The empty result is consistent with the current status of `SparkContext` because `BlockManager` and `BlockManagerMaster` is already stopped.

### Why are the changes needed?

In `SparkEnv.stop`, the following stop sequence is used and `metricsSystem.stop` invokes `sink.stop`.
```
blockManager.master.stop()
metricsSystem.stop() --> sinks.foreach(_.stop)
```

However, some sink can invoke `BlockManagerSource` and ends up with `NullPointerException` because `BlockManagerMaster` is already stopped and `driverEndpoint` became `null`.
```
java.lang.NullPointerException
at org.apache.spark.storage.BlockManagerMaster.getStorageStatus(BlockManagerMaster.scala:170)
at org.apache.spark.storage.BlockManagerSource$$anonfun$10.apply(BlockManagerSource.scala:63)
at org.apache.spark.storage.BlockManagerSource$$anonfun$10.apply(BlockManagerSource.scala:63)
at org.apache.spark.storage.BlockManagerSource$$anon$1.getValue(BlockManagerSource.scala:31)
at org.apache.spark.storage.BlockManagerSource$$anon$1.getValue(BlockManagerSource.scala:30)
```

Since `SparkContext` registers and forgets `BlockManagerSource` without deregistering, we had better avoid `NullPointerException` inside `BlockManagerMaster` preventively.
```scala
_env.metricsSystem.registerSource(new BlockManagerSource(_env.blockManager))
```

### Does this PR introduce any user-facing change?

Yes. This will remove NPE for the users who uses `BlockManagerSource`.

### How was this patch tested?

Pass the Jenkins with the newly added test cases.

Closes #28187 from dongjoon-hyun/SPARK-31422.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a6e6fbf2ca23e51d43f175907ce6f29c946e1acf)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/storage/BlockManagerMaster.scala    |  2 ++
 .../storage/BlockManagerMasterSuite.scala     | 35 +++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala

diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
index e335eb6ddb761..e440c1ab7bcd9 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMaster.scala
@@ -167,10 +167,12 @@ class BlockManagerMaster(
    * amount of remaining memory.
    */
   def getMemoryStatus: Map[BlockManagerId, (Long, Long)] = {
+    if (driverEndpoint == null) return Map.empty
     driverEndpoint.askSync[Map[BlockManagerId, (Long, Long)]](GetMemoryStatus)
   }
 
   def getStorageStatus: Array[StorageStatus] = {
+    if (driverEndpoint == null) return Array.empty
     driverEndpoint.askSync[Array[StorageStatus]](GetStorageStatus)
   }
 
diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala
new file mode 100644
index 0000000000000..0d54726af7ee8
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerMasterSuite.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.storage
+
+import org.junit.Assert.assertTrue
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+
+class BlockManagerMasterSuite extends SparkFunSuite {
+
+  test("SPARK-31422: getMemoryStatus should not fail after BlockManagerMaster stops") {
+    val bmm = new BlockManagerMaster(null, null, new SparkConf, true)
+    assertTrue(bmm.getMemoryStatus.isEmpty)
+  }
+
+  test("SPARK-31422: getStorageStatus should not fail after BlockManagerMaster stops") {
+    val bmm = new BlockManagerMaster(null, null, new SparkConf, true)
+    assertTrue(bmm.getStorageStatus.isEmpty)
+  }
+}

From 888b06925f60eabc844f6463cc0d333d1b82b9da Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 12 Apr 2020 13:57:54 -0500
Subject: [PATCH 0478/1280] [SPARK-31348][SQL][DOCS] Document Join in SQL
 Reference

### What changes were proposed in this pull request?
Document join in SQL Reference.

### Why are the changes needed?
To make SQL Reference complete.

### Does this PR introduce any user-facing change?
Yes
<img width="1050" alt="Screen Shot 2020-04-05 at 8 46 47 PM" src="https://user-images.githubusercontent.com/13592258/78521722-ab7efe80-777f-11ea-90f5-1fac09282721.png">

<img width="1049" alt="Screen Shot 2020-04-05 at 8 47 20 PM" src="https://user-images.githubusercontent.com/13592258/78521724-ade15880-777f-11ea-9238-183d999ed918.png">

<img width="1049" alt="Screen Shot 2020-04-05 at 8 47 41 PM" src="https://user-images.githubusercontent.com/13592258/78521726-b043b280-777f-11ea-996f-a8e86d453c01.png">

<img width="1049" alt="Screen Shot 2020-04-05 at 8 48 11 PM" src="https://user-images.githubusercontent.com/13592258/78521731-b3d73980-777f-11ea-85c8-c24798ef41ac.png">

<img width="1049" alt="Screen Shot 2020-04-05 at 8 48 33 PM" src="https://user-images.githubusercontent.com/13592258/78521734-b5a0fd00-777f-11ea-8b2c-96af30f3bf49.png">

### How was this patch tested?
Manually build and check.

Closes #28121 from huaxingao/join.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit fda910d4e2de832c36e97a404ad811022e5a5d52)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                |   2 +
 docs/sql-ref-syntax-qry-select-hints.md |   4 +
 docs/sql-ref-syntax-qry-select-join.md  | 239 +++++++++++++++++++++++-
 docs/sql-ref-syntax-qry-select.md       |   1 +
 4 files changed, 245 insertions(+), 1 deletion(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 225450b01c529..75efb30c87080 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -154,6 +154,8 @@
                   url: sql-ref-syntax-qry-select-distribute-by.html
                 - text: LIMIT Clause 
                   url: sql-ref-syntax-qry-select-limit.html
+                - text: JOIN
+                  url: sql-ref-syntax-qry-select-join.html
                 - text: Join Hints
                   url: sql-ref-syntax-qry-select-hints.html
                 - text: Set Operators
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index 4713ea5439b81..688ba10c3b1f7 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -81,3 +81,7 @@ SELECT /*+ SHUFFLE_REPLICATE_NL(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.ke
 SELECT /*+ BROADCAST(t1) */ /*+ MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
 
 {% endhighlight %}
+
+### Related Statements
+- [JOIN](sql-ref-syntax-qry-select-join.html)
+- [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index ee66dbdb8c823..4759b12f4db70 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -18,5 +18,242 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+### Description
 
-**This page is under construction**
+A SQL join is used to combine rows from two relations based on join criteria. The following section describes the overall join syntax and the sub-sections cover different types of joins along with examples.
+
+### Syntax
+
+{% highlight sql %}
+relation { [ join_type ] JOIN relation [ join_criteria ] | NATURAL join_type JOIN relation }
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>relation</em></code></dt>
+  <dd>
+    Specifies the relation to be joined.
+  </dd>
+  <dt><code><em>join_type</em></code></dt>
+  <dd>
+    Specifies the join type.<br><br>
+    <b>Syntax:</b><br>
+      <code>
+        [ INNER ]
+        | CROSS
+        | LEFT [ OUTER ]
+        | [ LEFT ] SEMI
+        | RIGHT [ OUTER ]
+        | FULL [ OUTER ]
+        | [ LEFT ] ANTI
+      </code>
+  </dd>
+  <dt><code><em>join_criteria</em></code></dt>
+  <dd>
+    Specifies how the rows from one relation will be combined with the rows of another relation.<br><br>
+    <b>Syntax:</b>
+      <code>
+        ON boolean_expression | USING ( column_name [ , column_name ... ] )
+      </code> <br><br>
+      <code>boolean_expression</code><br>
+      Specifies an expression with a return type of boolean.
+  </dd>
+</dl>
+
+### Join Types
+
+#### <b>Inner Join</b>
+
+<dd>
+The inner join is the default join in Spark SQL. It selects rows that have matching values in both relations.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation [ INNER ] JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+#### <b>Left Join </b>
+
+<dd>
+A left join returns all values from the left relation and the matched values from the right relation, or appends NULL if there is no match. It is also referred to as a left outer join.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation LEFT [ OUTER ] JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+#### <b>Right Join </b>
+<dd>
+A right join returns all values from the right relation and the matched values from the left relation, or appends NULL if there is no match. It is also referred to as a right outer join.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation RIGHT [ OUTER ] JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+#### <b>Full Join </b>
+
+<dd>
+A full join returns all values from both relations, appending NULL values on the side that does not have a match. It is also referred to as a full outer join.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation FULL [ OUTER ] JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+#### <b>Cross Join </b>
+
+<dd>
+A cross join returns the Cartesian product of two relations.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation CROSS JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+#### <b>Semi Join </b>
+
+<dd>
+A semi join returns values from the left side of the relation that has a match with the right. It is also referred to as a left semi join.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation [ LEFT ] SEMI JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+#### <b>Anti Join </b>
+
+<dd>
+An anti join returns values from the left relation that has no match with the right. It is also referred to as a left anti join.<br><br>
+  <b>Syntax:</b><br>
+    <code>
+    relation [ LEFT ] ANTI JOIN relation [ join_criteria ]
+    </code>
+</dd>
+
+### Examples
+
+{% highlight sql %}
+-- Use employee and department tables to demonstrate different type of joins.
+SELECT * FROM employee;
+
+  +---+-----+------+
+  | id| name|deptno|
+  +---+-----+------+
+  |105|Chloe|     5|
+  |103| Paul|     3|
+  |101| John|     1|
+  |102| Lisa|     2|
+  |104| Evan|     4|
+  |106|  Amy|     6|
+  +---+-----+------+
+
+SELECT * FROM department;
+  +------+-----------+
+  |deptno|   deptname|
+  +------+-----------+
+  |     3|Engineering|
+  |     2|      Sales|
+  |     1|  Marketing|
+  +------+-----------+
+
+-- Use employee and department tables to demonstrate inner join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee INNER JOIN department ON employee.deptno = department.deptno;
+  +---+-----+------+-----------|
+  | id| name|deptno|   deptname|
+  +---+-----+------+-----------|
+  |103| Paul|     3|Engineering|
+  |101| John|     1|  Marketing|
+  |102| Lisa|     2|      Sales|
+  +---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate left join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee LEFT JOIN department ON employee.deptno = department.deptno;
+  +---+-----+------+-----------|
+  | id| name|deptno|   deptname|
+  +---+-----+------+-----------|
+  |105|Chloe|     5|       NULL|
+  |103| Paul|     3|Engineering|
+  |101| John|     1|  Marketing|
+  |102| Lisa|     2|      Sales|
+  |104| Evan|     4|       NULL|
+  |106|  Amy|     6|       NULL|
+  +---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate right join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee RIGHT JOIN department ON employee.deptno = department.deptno;
+  +---+-----+------+-----------|
+  | id| name|deptno|   deptname|
+  +---+-----+------+-----------|
+  |103| Paul|     3|Engineering|
+  |101| John|     1|  Marketing|
+  |102| Lisa|     2|      Sales|
+  +---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate full join.
+SELECT id, name, employee.deptno, deptname
+    FROM employee FULL JOIN department ON employee.deptno = department.deptno;
+  +---+-----+------+-----------|
+  | id| name|deptno|   deptname|
+  +---+-----+------+-----------|
+  |101| John|     1|  Marketing|
+  |106|  Amy|     6|       NULL|
+  |103| Paul|     3|Engineering|
+  |105|Chloe|     5|       NULL|
+  |104| Evan|     4|       NULL|
+  |102| Lisa|     2|      Sales|
+  +---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate cross join.
+SELECT id, name, employee.deptno, deptname FROM employee CROSS JOIN department;
+  +---+-----+------+-----------|
+  | id| name|deptno|   deptname|
+  +---+-----+------+-----------|
+  |105|Chloe|     5|Engineering|
+  |105|Chloe|     5|  Marketing|
+  |105|Chloe|     5|      Sales|
+  |103| Paul|     3|Engineering|
+  |103| Paul|     3|  Marketing|
+  |103| Paul|     3|      Sales|
+  |101| John|     1|Engineering|
+  |101| John|     1|  Marketing|
+  |101| John|     1|      Sales|
+  |102| Lisa|     2|Engineering|
+  |102| Lisa|     2|  Marketing|
+  |102| Lisa|     2|      Sales|
+  |104| Evan|     4|Engineering|
+  |104| Evan|     4|  Marketing|
+  |104| Evan|     4|      Sales|
+  |106|  Amy|     4|Engineering|
+  |106|  Amy|     4|  Marketing|
+  |106|  Amy|     4|      Sales|
+  +---+-----+------+-----------|
+
+-- Use employee and department tables to demonstrate semi join.
+SELECT * FROM employee SEMI JOIN department ON employee.deptno = department.deptno;
+  +---+-----+------+
+  | id| name|deptno|
+  +---+-----+------+
+  |103| Paul|     3|
+  |101| John|     1|
+  |102| Lisa|     2|
+  +---+-----+------+
+
+-- Use employee and department tables to demonstrate anti join.
+SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.deptno;
+  +---+-----+------+
+  | id| name|deptno|
+  +---+-----+------+
+  |105|Chloe|     5|
+  |104| Evan|     4|
+  |106|  Amy|     6|
+  +---+-----+------+
+{% endhighlight %}
+
+### Related Statement
+  * [SELECT](sql-ref-syntax-qry-select.html)
+  * [Join Hints](sql-ref-syntax-qry-select-hints.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 17c141196d830..f1585df14e734 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -151,4 +151,5 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 - [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 - [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
 - [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+- [JOIN](sql-ref-syntax-qry-select-join.html)
 - [SET Operators](sql-ref-syntax-qry-select-setops.html)

From 36b0767a845a153c9972c6c863b197f3d2ecf33b Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Sun, 12 Apr 2020 13:10:57 -0700
Subject: [PATCH 0479/1280] [SPARK-31424][SQL] Rename
 AdaptiveSparkPlanHelper.collectInPlanAndSubqueries to collectWithSubqueries

### What changes were proposed in this pull request?
Like https://github.com/apache/spark/pull/28092, this PR is to rename `QueryPlan.collectInPlanAndSubqueries` in AdaptiveSparkPlanHelper to `collectWithSubqueries`

### Why are the changes needed?
The old name is too verbose. `QueryPlan` is internal but it's the core of catalyst and we'd better make the API name clearer before we release it.

### Does this PR introduce any user-facing change?
no

### How was this patch tested?
N/A

Closes #28193 from gatorsmile/spark-31322.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ad79ae11ba1cf30cd496f5edc4c8a9a109fd4a0e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/execution/adaptive/AdaptiveSparkPlanHelper.scala      | 2 +-
 .../spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
index 61ae6cb14ccd3..cd87230c593f9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
@@ -109,7 +109,7 @@ trait AdaptiveSparkPlanHelper {
    * Returns a sequence containing the result of applying a partial function to all elements in this
    * plan, also considering all the plans in its (nested) subqueries
    */
-  def collectInPlanAndSubqueries[B](p: SparkPlan)(f: PartialFunction[SparkPlan, B]): Seq[B] = {
+  def collectWithSubqueries[B](p: SparkPlan)(f: PartialFunction[SparkPlan, B]): Seq[B] = {
     (p +: subqueriesAll(p)).flatMap(collect(_)(f))
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index bfde042e9208f..b8ac4ddc61553 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -95,14 +95,14 @@ class AdaptiveQueryExecSuite
   }
 
   private def findReusedExchange(plan: SparkPlan): Seq[ReusedExchangeExec] = {
-    collectInPlanAndSubqueries(plan) {
+    collectWithSubqueries(plan) {
       case ShuffleQueryStageExec(_, e: ReusedExchangeExec) => e
       case BroadcastQueryStageExec(_, e: ReusedExchangeExec) => e
     }
   }
 
   private def findReusedSubquery(plan: SparkPlan): Seq[ReusedSubqueryExec] = {
-    collectInPlanAndSubqueries(plan) {
+    collectWithSubqueries(plan) {
       case e: ReusedSubqueryExec => e
     }
   }

From a83e86877de620931922b7fca89460ff60d52470 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sun, 12 Apr 2020 13:14:41 -0700
Subject: [PATCH 0480/1280] [SPARK-31416][SQL] Check more strictly that a field
 name can be used as a valid Java identifier for codegen

### What changes were proposed in this pull request?

Check more strictly that a field name can be used as a valid Java identifier in `ScalaReflection.serializerFor`
To check that, `SourceVersion` is used so that we need not add reserved keywords to be checked manually for the future Java versions (e.g, underscore, var, yield), .

### Why are the changes needed?

In the current implementation, `enum` is not checked even though it's a reserved keyword.
Also, there are lots of characters and sequences of character including numeric literals but they are not checked.
So we can't get better error message with following code.
```
case class  Data(`0`: Int)
Seq(Data(1)).toDF.show

20/04/11 03:24:24 ERROR CodeGenerator: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 43, Column 1: Expression "value_0 = value_3" is not a type
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 43, Column 1: Expression "value_0 = value_3" is not a type

...

```

### Does this PR introduce any user-facing change?

Yes. With this change and the code example above, we can get following error message.
```
java.lang.UnsupportedOperationException: `0` is not a valid identifier of Java and cannot be used as field name
- root class: "Data"

...
```

### How was this patch tested?

Add another assertion to existing test case.

Closes #28184 from sarutak/improve-identifier-check.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6cd0bef7fe872b5d9472a2140db188fb9f1eede8)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/ScalaReflection.scala  | 20 +++++++++---------
 .../expressions/objects/objects.scala         |  3 +--
 .../org/apache/spark/sql/DatasetSuite.scala   | 10 ---------
 .../sql/ScalaReflectionRelationSuite.scala    | 21 +++++++++++++++++++
 4 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index f587245116581..bb61e20cb7d0f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.catalyst
 
+import javax.lang.model.SourceVersion
+
 import org.apache.commons.lang3.reflect.ConstructorUtils
 
 import org.apache.spark.internal.Logging
@@ -539,9 +541,10 @@ object ScalaReflection extends ScalaReflection {
 
         val params = getConstructorParameters(t)
         val fields = params.map { case (fieldName, fieldType) =>
-          if (javaKeywords.contains(fieldName)) {
-            throw new UnsupportedOperationException(s"`$fieldName` is a reserved keyword and " +
-              "cannot be used as field name\n" + walkedTypePath)
+          if (SourceVersion.isKeyword(fieldName) ||
+              !SourceVersion.isIdentifier(encodeFieldNameToIdentifier(fieldName))) {
+            throw new UnsupportedOperationException(s"`$fieldName` is not a valid identifier of " +
+              "Java and cannot be used as field name\n" + walkedTypePath)
           }
 
           // SPARK-26730 inputObject won't be null with If's guard below. And KnownNotNul
@@ -782,13 +785,6 @@ object ScalaReflection extends ScalaReflection {
     }
   }
 
-  private val javaKeywords = Set("abstract", "assert", "boolean", "break", "byte", "case", "catch",
-    "char", "class", "const", "continue", "default", "do", "double", "else", "extends", "false",
-    "final", "finally", "float", "for", "goto", "if", "implements", "import", "instanceof", "int",
-    "interface", "long", "native", "new", "null", "package", "private", "protected", "public",
-    "return", "short", "static", "strictfp", "super", "switch", "synchronized", "this", "throw",
-    "throws", "transient", "true", "try", "void", "volatile", "while")
-
   val typeJavaMapping = Map[DataType, Class[_]](
     BooleanType -> classOf[Boolean],
     ByteType -> classOf[Byte],
@@ -847,6 +843,10 @@ object ScalaReflection extends ScalaReflection {
       Seq.empty
     }
   }
+
+  def encodeFieldNameToIdentifier(fieldName: String): String = {
+    TermName(fieldName).encodedName.toString
+  }
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 54abd09d89ddb..d5de95c65e49e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -28,7 +28,6 @@ import org.apache.spark.{SparkConf, SparkEnv}
 import org.apache.spark.serializer._
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, ScalaReflection}
-import org.apache.spark.sql.catalyst.ScalaReflection.universe.TermName
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen._
@@ -311,7 +310,7 @@ case class Invoke(
   override def nullable: Boolean = targetObject.nullable || needNullCheck || returnNullable
   override def children: Seq[Expression] = targetObject +: arguments
 
-  private lazy val encodedFunctionName = TermName(functionName).encodedName.toString
+  private lazy val encodedFunctionName = ScalaReflection.encodeFieldNameToIdentifier(functionName)
 
   @transient lazy val method = targetObject.dataType match {
     case ObjectType(cls) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index b0bd612e88d98..4e4558b6f3e2a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1222,14 +1222,6 @@ class DatasetSuite extends QueryTest
     assert(result == Set(ClassData("a", 1) -> null, ClassData("b", 2) -> ClassData("x", 2)))
   }
 
-  test("better error message when use java reserved keyword as field name") {
-    val e = intercept[UnsupportedOperationException] {
-      Seq(InvalidInJava(1)).toDS()
-    }
-    assert(e.getMessage.contains(
-      "`abstract` is a reserved keyword and cannot be used as field name"))
-  }
-
   test("Dataset should support flat input object to be null") {
     checkDataset(Seq("a", null).toDS(), "a", null)
   }
@@ -1949,8 +1941,6 @@ case class ClassNullableData(a: String, b: Integer)
 case class NestedStruct(f: ClassData)
 case class DeepNestedStruct(f: NestedStruct)
 
-case class InvalidInJava(`abstract`: Int)
-
 /**
  * A class used to test serialization using encoders. This class throws exceptions when using
  * Java serialization -- so the only way it can be "serialized" is through our encoders.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
index 7e305e0504729..d51a4618af519 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
@@ -74,9 +74,14 @@ case class ComplexReflectData(
     mapFieldContainsNull: Map[Int, Option[Long]],
     dataField: Data)
 
+case class InvalidInJava(`abstract`: Int)
+
 class ScalaReflectionRelationSuite extends SparkFunSuite with SharedSparkSession {
   import testImplicits._
 
+  // To avoid syntax error thrown by genjavadoc, make this case class non-top level and private.
+  private case class InvalidInJava2(`0`: Int)
+
   test("query case class RDD") {
     val data = ReflectData("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
       new java.math.BigDecimal(1), Date.valueOf("1970-01-01"), new Timestamp(12345), Seq(1, 2, 3),
@@ -142,4 +147,20 @@ class ScalaReflectionRelationSuite extends SparkFunSuite with SharedSparkSession
           Map(10 -> 100L, 20 -> 200L, 30 -> null),
           Row(null, "abc"))))
   }
+
+  test("better error message when use java reserved keyword as field name") {
+    val e = intercept[UnsupportedOperationException] {
+      Seq(InvalidInJava(1)).toDS()
+    }
+    assert(e.getMessage.contains(
+      "`abstract` is not a valid identifier of Java and cannot be used as field name"))
+  }
+
+  test("better error message when use invalid java identifier as field name") {
+    val e1 = intercept[UnsupportedOperationException] {
+      Seq(InvalidInJava2(1)).toDS()
+    }
+    assert(e1.getMessage.contains(
+      "`0` is not a valid identifier of Java and cannot be used as field name"))
+  }
 }

From 6fe52ad0a7eafe4293338a075ee25917127d4497 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Mon, 13 Apr 2020 03:11:28 +0000
Subject: [PATCH 0481/1280] [SPARK-31414][SQL] Fix performance regression with
 new TimestampFormatter for json and csv time parsing

With benchmark original, where the timestamp values are valid to the new parser

the result is
```scala
[info] Running benchmark: Read dates and timestamps
[info]   Running case: timestamp strings
[info]   Stopped after 3 iterations, 5781 ms
[info]   Running case: parse timestamps from Dataset[String]
[info]   Stopped after 3 iterations, 44764 ms
[info]   Running case: infer timestamps from Dataset[String]
[info]   Stopped after 3 iterations, 93764 ms
[info]   Running case: from_json(timestamp)
[info]   Stopped after 3 iterations, 59021 ms
```
When we modify the benchmark to

```scala
     def timestampStr: Dataset[String] = {
        spark.range(0, rowsNum, 1, 1).mapPartitions { iter =>
          iter.map(i => s"""{"timestamp":"1970-01-01T01:02:03.${i % 100}"}""")
        }.select($"value".as("timestamp")).as[String]
      }

      readBench.addCase("timestamp strings", numIters) { _ =>
        timestampStr.noop()
      }

      readBench.addCase("parse timestamps from Dataset[String]", numIters) { _ =>
        spark.read.schema(tsSchema).json(timestampStr).noop()
      }

      readBench.addCase("infer timestamps from Dataset[String]", numIters) { _ =>
        spark.read.json(timestampStr).noop()
      }
```
where the timestamp values are invalid for the new parser which causes a fallback to legacy parser(2.4).
the result is

```scala
[info] Running benchmark: Read dates and timestamps
[info]   Running case: timestamp strings
[info]   Stopped after 3 iterations, 5623 ms
[info]   Running case: parse timestamps from Dataset[String]
[info]   Stopped after 3 iterations, 506637 ms
[info]   Running case: infer timestamps from Dataset[String]
[info]   Stopped after 3 iterations, 509076 ms
```
About 10x perf-regression

BUT if we modify the timestamp pattern to `....HH:mm:ss[.SSS][XXX]` which make all timestamp values valid for the new parser to prohibit fallback, the result is

```scala
[info] Running benchmark: Read dates and timestamps
[info]   Running case: timestamp strings
[info]   Stopped after 3 iterations, 5623 ms
[info]   Running case: parse timestamps from Dataset[String]
[info]   Stopped after 3 iterations, 506637 ms
[info]   Running case: infer timestamps from Dataset[String]
[info]   Stopped after 3 iterations, 509076 ms
```

 Fix performance regression.

NO

new tests added.

Closes #28181 from yaooqinn/SPARK-31414.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d65f534c5ad4385b7c5198f15cb014e1d24e47c9)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/csv/CSVOptions.scala   |   9 +-
 .../spark/sql/catalyst/json/JSONOptions.scala |   9 +-
 .../benchmarks/CSVBenchmark-jdk11-results.txt |  88 ++++++------
 sql/core/benchmarks/CSVBenchmark-results.txt  |  88 ++++++------
 .../JsonBenchmark-jdk11-results.txt           | 130 +++++++++---------
 sql/core/benchmarks/JsonBenchmark-results.txt | 130 +++++++++---------
 .../apache/spark/sql/CsvFunctionsSuite.scala  |  12 ++
 .../apache/spark/sql/JsonFunctionsSuite.scala |  12 ++
 .../datasources/csv/CSVBenchmark.scala        |   4 +-
 .../datasources/json/JsonBenchmark.scala      |   4 +-
 10 files changed, 262 insertions(+), 224 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
index 8892037e03a7d..9d09cab4fd482 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -26,6 +26,7 @@ import com.univocity.parsers.csv.{CsvParserSettings, CsvWriterSettings, Unescape
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
 class CSVOptions(
     @transient val parameters: CaseInsensitiveMap[String],
@@ -148,8 +149,12 @@ class CSVOptions(
 
   val dateFormat: String = parameters.getOrElse("dateFormat", DateFormatter.defaultPattern)
 
-  val timestampFormat: String =
-    parameters.getOrElse("timestampFormat", s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX")
+  val timestampFormat: String = parameters.getOrElse("timestampFormat",
+    if (SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY) {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX"
+    } else {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss[.SSS][XXX]"
+    })
 
   val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
index 45c4edff47070..f9222f5af54da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
@@ -27,6 +27,7 @@ import com.fasterxml.jackson.core.json.JsonReadFeature
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
 /**
  * Options for parsing JSON data into Spark SQL rows.
@@ -90,8 +91,12 @@ private[sql] class JSONOptions(
 
   val dateFormat: String = parameters.getOrElse("dateFormat", DateFormatter.defaultPattern)
 
-  val timestampFormat: String =
-    parameters.getOrElse("timestampFormat", s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX")
+  val timestampFormat: String = parameters.getOrElse("timestampFormat",
+    if (SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY) {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss.SSSXXX"
+    } else {
+      s"${DateFormatter.defaultPattern}'T'HH:mm:ss[.SSS][XXX]"
+    })
 
   val multiLine = parameters.get("multiLine").map(_.toBoolean).getOrElse(false)
 
diff --git a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
index d8071e7bbdb35..147a77ff098d0 100644
--- a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 44297          44515         373          0.0      885948.7       1.0X
+One quoted string                                 24907          29374         NaN          0.0      498130.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                              196720         197783        1560          0.0      196719.8       1.0X
-Select 100 columns                                46691          46861         219          0.0       46691.4       4.2X
-Select one column                                 36811          36922         111          0.0       36811.3       5.3X
-count()                                            8520           8610         106          0.1        8520.5      23.1X
-Select 100 columns, one bad input field           67914          67994         136          0.0       67914.0       2.9X
-Select 100 columns, corrupt record field          77272          77445         214          0.0       77272.0       2.5X
+Select 1000 columns                               62811          63690        1416          0.0       62811.4       1.0X
+Select 100 columns                                23839          24064         230          0.0       23839.5       2.6X
+Select one column                                 19936          20641         827          0.1       19936.4       3.2X
+count()                                            4174           4380         206          0.2        4174.4      15.0X
+Select 100 columns, one bad input field           41015          42380        1688          0.0       41015.4       1.5X
+Select 100 columns, corrupt record field          46281          46338          93          0.0       46280.5       1.4X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                       25965          26054         103          0.4        2596.5       1.0X
-Select 1 column + count()                         18591          18666          91          0.5        1859.1       1.4X
-count()                                            6102           6119          18          1.6         610.2       4.3X
+Select 10 columns + count()                       10810          10997         163          0.9        1081.0       1.0X
+Select 1 column + count()                          7608           7641          47          1.3         760.8       1.4X
+count()                                            2415           2462          77          4.1         241.5       4.5X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     2142           2161          17          4.7         214.2       1.0X
-to_csv(timestamp)                                 14744          14950         182          0.7        1474.4       0.1X
-write timestamps to files                         12078          12202         175          0.8        1207.8       0.2X
-Create a dataset of dates                          2275           2291          18          4.4         227.5       0.9X
-to_csv(date)                                      11407          11464          51          0.9        1140.7       0.2X
-write dates to files                               7638           7702          90          1.3         763.8       0.3X
+Create a dataset of timestamps                      874            914          37         11.4          87.4       1.0X
+to_csv(timestamp)                                  7051           7223         250          1.4         705.1       0.1X
+write timestamps to files                          6712           6741          31          1.5         671.2       0.1X
+Create a dataset of dates                           909            945          35         11.0          90.9       1.0X
+to_csv(date)                                       4222           4231           8          2.4         422.2       0.2X
+write dates to files                               3799           3813          14          2.6         379.9       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2578           2590          10          3.9         257.8       1.0X
-read timestamps from files                        60103          60694         512          0.2        6010.3       0.0X
-infer timestamps from files                      107871         108268         351          0.1       10787.1       0.0X
-read date text from files                          2306           2310           4          4.3         230.6       1.1X
-read date from files                              47415          47657         367          0.2        4741.5       0.1X
-infer date from files                             35261          35447         164          0.3        3526.1       0.1X
-timestamp strings                                  3045           3056          11          3.3         304.5       0.8X
-parse timestamps from Dataset[String]             62221          63173         849          0.2        6222.1       0.0X
-infer timestamps from Dataset[String]            118838         119629         697          0.1       11883.8       0.0X
-date strings                                       3459           3481          19          2.9         345.9       0.7X
-parse dates from Dataset[String]                  51026          51447         503          0.2        5102.6       0.1X
-from_csv(timestamp)                               60738          61818         936          0.2        6073.8       0.0X
-from_csv(date)                                    46012          46278         370          0.2        4601.2       0.1X
+read timestamp text from files                     1342           1364          35          7.5         134.2       1.0X
+read timestamps from files                        20300          20473         247          0.5        2030.0       0.1X
+infer timestamps from files                       40705          40744          54          0.2        4070.5       0.0X
+read date text from files                          1146           1151           6          8.7         114.6       1.2X
+read date from files                              12278          12408         117          0.8        1227.8       0.1X
+infer date from files                             12734          12872         220          0.8        1273.4       0.1X
+timestamp strings                                  1467           1482          15          6.8         146.7       0.9X
+parse timestamps from Dataset[String]             21708          22234         477          0.5        2170.8       0.1X
+infer timestamps from Dataset[String]             42357          43253         922          0.2        4235.7       0.0X
+date strings                                       1512           1532          18          6.6         151.2       0.9X
+parse dates from Dataset[String]                  13436          13470          33          0.7        1343.6       0.1X
+from_csv(timestamp)                               20390          20486          95          0.5        2039.0       0.1X
+from_csv(date)                                    12592          12693         139          0.8        1259.2       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.5+10 on Mac OS X 10.15.2
-Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       11889          11945          52          0.0      118893.1       1.0X
-pushdown disabled                                 11790          11860         115          0.0      117902.3       1.0X
-w/ filters                                         1240           1278          33          0.1       12400.8       9.6X
+w/o filters                                       12535          12606          67          0.0      125348.8       1.0X
+pushdown disabled                                 12611          12672          91          0.0      126112.9       1.0X
+w/ filters                                         1093           1099          11          0.1       10928.3      11.5X
 
 
diff --git a/sql/core/benchmarks/CSVBenchmark-results.txt b/sql/core/benchmarks/CSVBenchmark-results.txt
index b3ba69c9eb6b1..498ca4caa0e45 100644
--- a/sql/core/benchmarks/CSVBenchmark-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 51602          51659          59          0.0     1032039.4       1.0X
+One quoted string                                 24073          24109          33          0.0      481463.5       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                              191926         192879        1615          0.0      191925.6       1.0X
-Select 100 columns                                46766          46846          69          0.0       46766.1       4.1X
-Select one column                                 35877          35930          83          0.0       35876.8       5.3X
-count()                                           11186          11262          65          0.1       11186.0      17.2X
-Select 100 columns, one bad input field           59943          60107         232          0.0       59943.0       3.2X
-Select 100 columns, corrupt record field          73062          73406         479          0.0       73062.2       2.6X
+Select 1000 columns                               58415          59611        2071          0.0       58414.8       1.0X
+Select 100 columns                                22568          23020         594          0.0       22568.0       2.6X
+Select one column                                 18995          19058          99          0.1       18995.0       3.1X
+count()                                            5301           5332          30          0.2        5300.9      11.0X
+Select 100 columns, one bad input field           39736          40153         361          0.0       39736.1       1.5X
+Select 100 columns, corrupt record field          47195          47826         590          0.0       47195.2       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                       22389          22447          87          0.4        2238.9       1.0X
-Select 1 column + count()                         14844          14890          43          0.7        1484.4       1.5X
-count()                                            5519           5538          18          1.8         551.9       4.1X
+Select 10 columns + count()                        9884           9904          25          1.0         988.4       1.0X
+Select 1 column + count()                          6794           6835          46          1.5         679.4       1.5X
+count()                                            2060           2065           5          4.9         206.0       4.8X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     1949           1977          25          5.1         194.9       1.0X
-to_csv(timestamp)                                 14944          15702         714          0.7        1494.4       0.1X
-write timestamps to files                         12983          12998          14          0.8        1298.3       0.2X
-Create a dataset of dates                          2156           2164           7          4.6         215.6       0.9X
-to_csv(date)                                       9675           9709          41          1.0         967.5       0.2X
-write dates to files                               7880           7897          15          1.3         788.0       0.2X
+Create a dataset of timestamps                      717            732          18         14.0          71.7       1.0X
+to_csv(timestamp)                                  6994           7100         121          1.4         699.4       0.1X
+write timestamps to files                          6417           6435          27          1.6         641.7       0.1X
+Create a dataset of dates                           827            855          24         12.1          82.7       0.9X
+to_csv(date)                                       4408           4438          32          2.3         440.8       0.2X
+write dates to files                               3738           3758          28          2.7         373.8       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2235           2245          10          4.5         223.5       1.0X
-read timestamps from files                        54490          54690         283          0.2        5449.0       0.0X
-infer timestamps from files                      104501         104737         236          0.1       10450.1       0.0X
-read date text from files                          2035           2040           6          4.9         203.5       1.1X
-read date from files                              39650          39707          52          0.3        3965.0       0.1X
-infer date from files                             29235          29363         164          0.3        2923.5       0.1X
-timestamp strings                                  3412           3426          18          2.9         341.2       0.7X
-parse timestamps from Dataset[String]             66864          67804         981          0.1        6686.4       0.0X
-infer timestamps from Dataset[String]            118780         119284         837          0.1       11878.0       0.0X
-date strings                                       3730           3734           4          2.7         373.0       0.6X
-parse dates from Dataset[String]                  48728          49071         309          0.2        4872.8       0.0X
-from_csv(timestamp)                               62294          62493         260          0.2        6229.4       0.0X
-from_csv(date)                                    44581          44665         117          0.2        4458.1       0.1X
+read timestamp text from files                     1121           1176          52          8.9         112.1       1.0X
+read timestamps from files                        21298          21366         105          0.5        2129.8       0.1X
+infer timestamps from files                       41008          41051          39          0.2        4100.8       0.0X
+read date text from files                           962            967           5         10.4          96.2       1.2X
+read date from files                              11749          11772          22          0.9        1174.9       0.1X
+infer date from files                             12426          12459          29          0.8        1242.6       0.1X
+timestamp strings                                  1508           1519           9          6.6         150.8       0.7X
+parse timestamps from Dataset[String]             21674          21997         455          0.5        2167.4       0.1X
+infer timestamps from Dataset[String]             42141          42230         105          0.2        4214.1       0.0X
+date strings                                       1694           1701           8          5.9         169.4       0.7X
+parse dates from Dataset[String]                  12929          12951          25          0.8        1292.9       0.1X
+from_csv(timestamp)                               20603          20786         166          0.5        2060.3       0.1X
+from_csv(date)                                    12325          12338          12          0.8        1232.5       0.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.2
-Intel(R) Core(TM) i7-4850HQ CPU @ 2.30GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       12557          12634          78          0.0      125572.9       1.0X
-pushdown disabled                                 12449          12509          65          0.0      124486.4       1.0X
-w/ filters                                         1372           1393          18          0.1       13724.8       9.1X
+w/o filters                                       12455          12474          22          0.0      124553.8       1.0X
+pushdown disabled                                 12462          12486          29          0.0      124624.9       1.0X
+w/ filters                                         1073           1092          18          0.1       10727.6      11.6X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
index 920e0a7723e70..03bc334471e56 100644
--- a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
@@ -3,110 +3,110 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       84774          84927         264          1.2         847.7       1.0X
-UTF-8 is set                                     119081         120155        1773          0.8        1190.8       0.7X
+No encoding                                       46010          46118         113          2.2         460.1       1.0X
+UTF-8 is set                                      54407          55427        1718          1.8         544.1       0.8X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       49293          49356          70          2.0         492.9       1.0X
-UTF-8 is set                                      80183          80211          25          1.2         801.8       0.6X
+No encoding                                       26614          28220        1461          3.8         266.1       1.0X
+UTF-8 is set                                      42765          43400         550          2.3         427.6       0.6X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       61070          61476         536          0.2        6107.0       1.0X
-UTF-8 is set                                     109765         109881         102          0.1       10976.5       0.6X
+No encoding                                       35696          35821         113          0.3        3569.6       1.0X
+UTF-8 is set                                      55441          56176        1037          0.2        5544.1       0.6X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      176999         178163        1008          0.0      353997.9       1.0X
-UTF-8 is set                                     201209         201641         614          0.0      402419.0       0.9X
+No encoding                                       61514          62968         NaN          0.0      123027.2       1.0X
+UTF-8 is set                                      72096          72933        1162          0.0      144192.7       0.9X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18768          20587         496          0.5        1876.8       1.0X
-Select 1 column                                   22642          22644           3          0.4        2264.2       0.8X
+Select 10 columns                                  9859           9913          79          1.0         985.9       1.0X
+Select 1 column                                   10981          11003          36          0.9        1098.1       0.9X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      7697           7738          55          1.3         769.7       1.0X
-Short column with UTF-8                           14051          14189         176          0.7        1405.1       0.5X
-Wide column without encoding                     108999         110075        1085          0.1       10899.9       0.1X
-Wide column with UTF-8                           157433         157779         308          0.1       15743.3       0.0X
+Short column without encoding                      3555           3579          27          2.8         355.5       1.0X
+Short column with UTF-8                            5204           5227          35          1.9         520.4       0.7X
+Wide column without encoding                      60458          60637         164          0.2        6045.8       0.1X
+Wide column with UTF-8                            77544          78111         551          0.1        7754.4       0.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           644            647           4         15.5          64.4       1.0X
-from_json                                         25859          25872          12          0.4        2585.9       0.0X
-json_tuple                                        31679          31761          71          0.3        3167.9       0.0X
-get_json_object                                   24772          25220         389          0.4        2477.2       0.0X
+Text read                                           342            346           3         29.2          34.2       1.0X
+from_json                                          7123           7318         179          1.4         712.3       0.0X
+json_tuple                                         9843           9957         132          1.0         984.3       0.0X
+get_json_object                                    7827           8046         194          1.3         782.7       0.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          3135           3165          52         15.9          62.7       1.0X
-schema inferring                                  29383          29389          10          1.7         587.7       0.1X
-parsing                                           32623          35183         NaN          1.5         652.5       0.1X
+Text read                                          1856           1884          32         26.9          37.1       1.0X
+schema inferring                                  16734          16900         153          3.0         334.7       0.1X
+parsing                                           14884          15203         470          3.4         297.7       0.1X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                         11874          11948          82          4.2         237.5       1.0X
-Schema inferring                                  42382          42398          23          1.2         847.6       0.3X
-Parsing without charset                           36410          36442          54          1.4         728.2       0.3X
-Parsing with UTF-8                                62412          62463          48          0.8        1248.2       0.2X
+Text read                                          5932           6148         228          8.4         118.6       1.0X
+Schema inferring                                  20836          21938        1086          2.4         416.7       0.3X
+Parsing without charset                           18134          18661         457          2.8         362.7       0.3X
+Parsing with UTF-8                                27734          28069         378          1.8         554.7       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     2191           2209          20          4.6         219.1       1.0X
-to_json(timestamp)                                18670          19042         565          0.5        1867.0       0.1X
-write timestamps to files                         11836          13156         NaN          0.8        1183.6       0.2X
-Create a dataset of dates                          2321           2351          33          4.3         232.1       0.9X
-to_json(date)                                     12703          12726          24          0.8        1270.3       0.2X
-write dates to files                               8230           8303          76          1.2         823.0       0.3X
+Create a dataset of timestamps                      889            914          28         11.2          88.9       1.0X
+to_json(timestamp)                                 7920           8172         353          1.3         792.0       0.1X
+write timestamps to files                          6726           6822         129          1.5         672.6       0.1X
+Create a dataset of dates                           953            963          12         10.5          95.3       0.9X
+to_json(date)                                      5370           5705         320          1.9         537.0       0.2X
+write dates to files                               4109           4166          52          2.4         410.9       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.5+10-post-Ubuntu-0ubuntu1.118.04 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2780           2795          13          3.6         278.0       1.0X
-read timestamps from files                        37158          37305         137          0.3        3715.8       0.1X
-infer timestamps from files                       73666          73838         149          0.1        7366.6       0.0X
-read date text from files                          2597           2609          10          3.9         259.7       1.1X
-read date from files                              24439          24501          56          0.4        2443.9       0.1X
-timestamp strings                                  3052           3064          12          3.3         305.2       0.9X
-parse timestamps from Dataset[String]             43611          43665          52          0.2        4361.1       0.1X
-infer timestamps from Dataset[String]             83745          84153         376          0.1        8374.5       0.0X
-date strings                                       4068           4076          10          2.5         406.8       0.7X
-parse dates from Dataset[String]                  34700          34807         118          0.3        3470.0       0.1X
-from_json(timestamp)                              64074          64124          53          0.2        6407.4       0.0X
-from_json(date)                                   52520          52617         101          0.2        5252.0       0.1X
+read timestamp text from files                     1614           1675          55          6.2         161.4       1.0X
+read timestamps from files                        16640          16858         209          0.6        1664.0       0.1X
+infer timestamps from files                       33239          33388         227          0.3        3323.9       0.0X
+read date text from files                          1310           1340          44          7.6         131.0       1.2X
+read date from files                               9470           9513          41          1.1         947.0       0.2X
+timestamp strings                                  1303           1342          47          7.7         130.3       1.2X
+parse timestamps from Dataset[String]             17650          18073         380          0.6        1765.0       0.1X
+infer timestamps from Dataset[String]             32623          34065        1330          0.3        3262.3       0.0X
+date strings                                       1864           1871           7          5.4         186.4       0.9X
+parse dates from Dataset[String]                  10914          11316         482          0.9        1091.4       0.1X
+from_json(timestamp)                              21102          21990         929          0.5        2110.2       0.1X
+from_json(date)                                   15275          15961         598          0.7        1527.5       0.1X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt
index e435f573a5d92..0f188c4cdea56 100644
--- a/sql/core/benchmarks/JsonBenchmark-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-results.txt
@@ -3,110 +3,110 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       61888          61918          27          1.6         618.9       1.0X
-UTF-8 is set                                     109057         113663         NaN          0.9        1090.6       0.6X
+No encoding                                       38998          41002         NaN          2.6         390.0       1.0X
+UTF-8 is set                                      61231          63282        1854          1.6         612.3       0.6X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       44517          44535          29          2.2         445.2       1.0X
-UTF-8 is set                                      75722          75840         111          1.3         757.2       0.6X
+No encoding                                       28272          28338          70          3.5         282.7       1.0X
+UTF-8 is set                                      58681          62243        1517          1.7         586.8       0.5X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       63677          64090         633          0.2        6367.7       1.0X
-UTF-8 is set                                      99424          99615         185          0.1        9942.4       0.6X
+No encoding                                       44026          51829        1329          0.2        4402.6       1.0X
+UTF-8 is set                                      65839          68596         500          0.2        6583.9       0.7X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      174052         174251         174          0.0      348104.1       1.0X
-UTF-8 is set                                     189000         189098         113          0.0      378000.9       0.9X
+No encoding                                       72144          74820         NaN          0.0      144287.6       1.0X
+UTF-8 is set                                      69571          77888         NaN          0.0      139142.3       1.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18387          18473         142          0.5        1838.7       1.0X
-Select 1 column                                   25560          25571          13          0.4        2556.0       0.7X
+Select 10 columns                                  9502           9604         106          1.1         950.2       1.0X
+Select 1 column                                   11861          11948         109          0.8        1186.1       0.8X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      9323           9384          58          1.1         932.3       1.0X
-Short column with UTF-8                           14016          14058          55          0.7        1401.6       0.7X
-Wide column without encoding                     133258         133532         382          0.1       13325.8       0.1X
-Wide column with UTF-8                           181212         181283          61          0.1       18121.2       0.1X
+Short column without encoding                      3830           3846          15          2.6         383.0       1.0X
+Short column with UTF-8                            5538           5543           7          1.8         553.8       0.7X
+Wide column without encoding                      66899          69158         NaN          0.1        6689.9       0.1X
+Wide column with UTF-8                            90052          93235         NaN          0.1        9005.2       0.0X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          1168           1174           5          8.6         116.8       1.0X
-from_json                                         22604          23571         883          0.4        2260.4       0.1X
-json_tuple                                        29979          30053          91          0.3        2997.9       0.0X
-get_json_object                                   21987          22263         241          0.5        2198.7       0.1X
+Text read                                           659            674          13         15.2          65.9       1.0X
+from_json                                          7676           7943         405          1.3         767.6       0.1X
+json_tuple                                         9881          10172         273          1.0         988.1       0.1X
+get_json_object                                    7949           8055         119          1.3         794.9       0.1X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5831           5842          14          8.6         116.6       1.0X
-schema inferring                                  31372          31456          73          1.6         627.4       0.2X
-parsing                                           35911          36191         254          1.4         718.2       0.2X
+Text read                                          3314           3326          17         15.1          66.3       1.0X
+schema inferring                                  16549          17037         484          3.0         331.0       0.2X
+parsing                                           15138          15283         172          3.3         302.8       0.2X
 
 Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                         10249          10314          77          4.9         205.0       1.0X
-Schema inferring                                  35403          35436          40          1.4         708.1       0.3X
-Parsing without charset                           32875          32879           4          1.5         657.5       0.3X
-Parsing with UTF-8                                53444          53519         100          0.9        1068.9       0.2X
+Text read                                          5136           5446         268          9.7         102.7       1.0X
+Schema inferring                                  19864          20568        1191          2.5         397.3       0.3X
+Parsing without charset                           17535          17888         329          2.9         350.7       0.3X
+Parsing with UTF-8                                25609          25758         218          2.0         512.2       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     1909           1924          17          5.2         190.9       1.0X
-to_json(timestamp)                                18956          19122         208          0.5        1895.6       0.1X
-write timestamps to files                         13446          13472          43          0.7        1344.6       0.1X
-Create a dataset of dates                          2180           2200          28          4.6         218.0       0.9X
-to_json(date)                                     12780          12899         109          0.8        1278.0       0.1X
-write dates to files                               7835           7865          29          1.3         783.5       0.2X
+Create a dataset of timestamps                      784            790           7         12.8          78.4       1.0X
+to_json(timestamp)                                 8005           8055          50          1.2         800.5       0.1X
+write timestamps to files                          6515           6559          45          1.5         651.5       0.1X
+Create a dataset of dates                           854            881          24         11.7          85.4       0.9X
+to_json(date)                                      5187           5194           7          1.9         518.7       0.2X
+write dates to files                               3663           3684          22          2.7         366.3       0.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_232-8u232-b09-0ubuntu1~18.04.1-b09 on Linux 4.15.0-1044-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2467           2477           9          4.1         246.7       1.0X
-read timestamps from files                        40186          40342         135          0.2        4018.6       0.1X
-infer timestamps from files                       82005          82079          71          0.1        8200.5       0.0X
-read date text from files                          2243           2264          22          4.5         224.3       1.1X
-read date from files                              24852          24863          19          0.4        2485.2       0.1X
-timestamp strings                                  3836           3854          16          2.6         383.6       0.6X
-parse timestamps from Dataset[String]             51521          51697         242          0.2        5152.1       0.0X
-infer timestamps from Dataset[String]             97300          97398         133          0.1        9730.0       0.0X
-date strings                                       4488           4491           5          2.2         448.8       0.5X
-parse dates from Dataset[String]                  37918          37976          68          0.3        3791.8       0.1X
-from_json(timestamp)                              69611          69632          36          0.1        6961.1       0.0X
-from_json(date)                                   56598          56974         347          0.2        5659.8       0.0X
+read timestamp text from files                     1297           1316          26          7.7         129.7       1.0X
+read timestamps from files                        16915          17723         963          0.6        1691.5       0.1X
+infer timestamps from files                       33967          34304         360          0.3        3396.7       0.0X
+read date text from files                          1095           1100           7          9.1         109.5       1.2X
+read date from files                               8376           8513         209          1.2         837.6       0.2X
+timestamp strings                                  1807           1816           8          5.5         180.7       0.7X
+parse timestamps from Dataset[String]             18189          18242          74          0.5        1818.9       0.1X
+infer timestamps from Dataset[String]             37906          38547         571          0.3        3790.6       0.0X
+date strings                                       2191           2194           4          4.6         219.1       0.6X
+parse dates from Dataset[String]                  11593          11625          33          0.9        1159.3       0.1X
+from_json(timestamp)                              22589          22650         101          0.4        2258.9       0.1X
+from_json(date)                                   16479          16619         159          0.6        1647.9       0.1X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
index 89fb4d5151b37..b9e0d509eeece 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
@@ -212,4 +212,16 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
       assert(readback(0).getAs[Row](0).getAs[Date](0).getTime >= 0)
     }
   }
+
+  test("optional datetime parser does not affect csv time formatting") {
+    val s = "2015-08-26 12:34:46"
+    def toDF(p: String): DataFrame = sql(
+      s"""
+         |SELECT
+         | to_csv(
+         |   named_struct('time', timestamp'$s'), map('timestampFormat', "$p")
+         | )
+         | """.stripMargin)
+    checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index 8cc5c227eb1a2..b989b5dadd224 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -688,4 +688,16 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
           options.asJava)),
       Seq(Row("string")))
   }
+
+  test("optional datetime parser does not affect json time formatting") {
+    val s = "2015-08-26 12:34:46"
+    def toDF(p: String): DataFrame = sql(
+      s"""
+         |SELECT
+         | to_json(
+         |   named_struct('time', timestamp'$s'), map('timestampFormat', "$p")
+         | )
+         | """.stripMargin)
+    checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
index e2abb39c986a7..53d287b32f8db 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVBenchmark.scala
@@ -238,7 +238,9 @@ object CSVBenchmark extends SqlBasedBenchmark {
 
       def timestampStr: Dataset[String] = {
         spark.range(0, rowsNum, 1, 1).mapPartitions { iter =>
-          iter.map(i => s"1970-01-01T01:02:03.${100 + i % 100}Z")
+          iter.map {
+            i => s"1970-01-01T01:02:03.${i % 200}Z".stripSuffix(".0Z")
+          }
         }.select($"value".as("timestamp")).as[String]
       }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
index bcecaccc8cc89..56930880ed5da 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
@@ -445,7 +445,9 @@ object JsonBenchmark extends SqlBasedBenchmark {
 
       def timestampStr: Dataset[String] = {
         spark.range(0, rowsNum, 1, 1).mapPartitions { iter =>
-          iter.map(i => s"""{"timestamp":"1970-01-01T01:02:03.${100 + i % 100}Z"}""")
+          iter.map { i =>
+            s"""{"timestamp":"1970-01-01T01:02:03.${i % 200}Z"}""".stripSuffix(".0Z")
+          }
         }.select($"value".as("timestamp")).as[String]
       }
 

From 8f7a1a9ce1edc2c5ad64fd65f8ee98480bc5829e Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 12 Apr 2020 23:38:17 -0500
Subject: [PATCH 0482/1280] [SPARK-31319][SQL][DOCS] Document UDFs/UDAFs in SQL
 Reference

### What changes were proposed in this pull request?
Document UDF in SQL Reference

### Why are the changes needed?
To make SQL Reference complete.

### Does this PR introduce any user-facing change?
Yes. Here are the new pages:
<img width="1050" alt="Screen Shot 2020-04-09 at 5 06 42 PM" src="https://user-images.githubusercontent.com/13592258/78950977-585dc200-7a85-11ea-875c-ce14c3795e0f.png">

<img width="1049" alt="Screen Shot 2020-04-09 at 5 07 06 PM" src="https://user-images.githubusercontent.com/13592258/78950979-5b58b280-7a85-11ea-81f3-bd5d91bd07e3.png">

<img width="1049" alt="Screen Shot 2020-04-09 at 5 07 26 PM" src="https://user-images.githubusercontent.com/13592258/78950985-5e53a300-7a85-11ea-86be-f63152c1501b.png">

<img width="1051" alt="Screen Shot 2020-04-09 at 5 07 54 PM" src="https://user-images.githubusercontent.com/13592258/78950991-63185700-7a85-11ea-9379-8da46cfc434c.png">

<img width="1060" alt="Screen Shot 2020-04-09 at 5 08 17 PM" src="https://user-images.githubusercontent.com/13592258/78950994-657ab100-7a85-11ea-8b34-d2c87f94b03b.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 09 27 PM" src="https://user-images.githubusercontent.com/13592258/78951001-6875a180-7a85-11ea-874e-8abd14a3d3d3.png">

<img width="1060" alt="Screen Shot 2020-04-09 at 5 10 00 PM" src="https://user-images.githubusercontent.com/13592258/78951005-6f041900-7a85-11ea-9e57-520eb8db59ec.png">

<img width="1049" alt="Screen Shot 2020-04-09 at 5 11 10 PM" src="https://user-images.githubusercontent.com/13592258/78951014-73303680-7a85-11ea-93ab-32d68d2e2d59.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 11 41 PM" src="https://user-images.githubusercontent.com/13592258/78951019-75929080-7a85-11ea-9d3b-600e8e157c05.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 16 22 PM" src="https://user-images.githubusercontent.com/13592258/78951137-dfab3580-7a85-11ea-8512-c6b660aa271e.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 22 15 PM" src="https://user-images.githubusercontent.com/13592258/78951466-22214200-7a87-11ea-93dd-6e36492421f1.png">

<img width="1049" alt="Screen Shot 2020-04-09 at 5 22 46 PM" src="https://user-images.githubusercontent.com/13592258/78951469-24839c00-7a87-11ea-93a9-fe30d689adbd.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 23 08 PM" src="https://user-images.githubusercontent.com/13592258/78951472-26e5f600-7a87-11ea-84db-087a3528aa53.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 23 34 PM" src="https://user-images.githubusercontent.com/13592258/78951474-29e0e680-7a87-11ea-8be4-2a5be1bc3788.png">

<img width="1049" alt="Screen Shot 2020-04-09 at 5 23 57 PM" src="https://user-images.githubusercontent.com/13592258/78951481-2cdbd700-7a87-11ea-8894-0a39abf54a3b.png">

<img width="1050" alt="Screen Shot 2020-04-09 at 5 24 15 PM" src="https://user-images.githubusercontent.com/13592258/78951483-2f3e3100-7a87-11ea-8845-ffebf89d7898.png">

### How was this patch tested?
Manually build and check

Closes #28087 from huaxingao/udf.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 3bbd80dbc3c30d05aa01d8369e1ff90a41850d26)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/sql-getting-started.md                   | 29 +-----
 docs/sql-ref-functions-udf-aggregate.md       | 89 ++++++++++++++++-
 docs/sql-ref-functions-udf-scalar.md          | 49 +++++++++-
 .../examples/sql/JavaUserDefinedScalar.java   | 98 +++++++++++++++++++
 .../examples/sql/UserDefinedScalar.scala      | 80 +++++++++++++++
 5 files changed, 309 insertions(+), 36 deletions(-)
 create mode 100644 examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java
 create mode 100644 examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala

diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index 9df0f768268d0..fc0a5d01ec3af 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -356,31 +356,8 @@ aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, et
 While those functions are designed for DataFrames, Spark SQL also has type-safe versions for some of them in
 [Scala](api/scala/org/apache/spark/sql/expressions/scalalang/typed$.html) and
 [Java](api/java/org/apache/spark/sql/expressions/javalang/typed.html) to work with strongly typed Datasets.
-Moreover, users are not limited to the predefined aggregate functions and can create their own.
+Moreover, users are not limited to the predefined aggregate functions and can create their own. For more details
+about user defined aggregate functions, please refer to the documentation of
+[User Defined Aggregate Functions](sql-ref-functions-udf-aggregate.html).
 
-### Type-Safe User-Defined Aggregate Functions
 
-User-defined aggregations for strongly typed Datasets revolve around the [Aggregator](api/scala/org/apache/spark/sql/expressions/Aggregator.html) abstract class.
-For example, a type-safe user-defined average can look like:
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-{% include_example typed_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala%}
-</div>
-<div data-lang="java"  markdown="1">
-{% include_example typed_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java%}
-</div>
-</div>
-
-### Untyped User-Defined Aggregate Functions
-Typed aggregations, as described above, may also be registered as untyped aggregating UDFs for use with DataFrames.
-For example, a user-defined average for untyped DataFrames can look like:
-
-<div class="codetabs">
-<div data-lang="scala"  markdown="1">
-{% include_example untyped_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala%}
-</div>
-<div data-lang="java"  markdown="1">
-{% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
-</div>
-</div>
diff --git a/docs/sql-ref-functions-udf-aggregate.md b/docs/sql-ref-functions-udf-aggregate.md
index 49c7b5824dfd6..3d8a64ea3039c 100644
--- a/docs/sql-ref-functions-udf-aggregate.md
+++ b/docs/sql-ref-functions-udf-aggregate.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: User defined Aggregate Functions (UDAF)
-displayTitle: User defined Aggregate Functions (UDAF)
+title: User Defined Aggregate Functions (UDAFs)
+displayTitle: User Defined Aggregate Functions (UDAFs)
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,4 +19,83 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+User-Defined Aggregate Functions (UDAFs) are user-programmable routines that act on multiple rows at once and return a single aggregated value as a result. This documentation lists the classes that are required for creating and registering UDAFs. It also contains examples that demonstrate how to define and register UDAFs in Scala and invoke them in Spark SQL.
+
+### Aggregator[-IN, BUF, OUT]
+
+A base class for user-defined aggregations, which can be used in Dataset operations to take all of the elements of a group and reduce them to a single value.
+
+ * IN - The input type for the aggregation.
+ * BUF - The type of the intermediate value of the reduction.
+ * OUT - The type of the final output result.
+
+<dl>
+  <dt><code><em>bufferEncoder: Encoder[BUF]</em></code></dt>
+  <dd>
+    Specifies the Encoder for the intermediate value type.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>finish(reduction: BUF): OUT</em></code></dt>
+  <dd>
+    Transform the output of the reduction.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>merge(b1: BUF, b2: BUF): BUF</em></code></dt>
+  <dd>
+    Merge two intermediate values.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>outputEncoder: Encoder[OUT]</em></code></dt>
+  <dd>
+    Specifies the Encoder for the final output value type.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>reduce(b: BUF, a: IN): BUF</em></code></dt>
+  <dd>
+     Aggregate input value <code>a</code> into current intermediate value. For performance, the function may modify <code>b</code> and return it instead of constructing new object for <code>b</code>.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>zero: BUF</em></code></dt>
+  <dd>
+    The initial value of the intermediate result for this aggregation.
+  </dd>
+</dl>
+
+### Examples
+
+#### Type-Safe User-Defined Aggregate Functions
+
+User-defined aggregations for strongly typed Datasets revolve around the [Aggregator](api/scala/org/apache/spark/sql/expressions/Aggregator.html) abstract class.
+For example, a type-safe user-defined average can look like:
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+  {% include_example typed_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala%}
+</div>
+<div data-lang="java"  markdown="1">
+  {% include_example typed_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java%}
+</div>
+</div>
+
+#### Untyped User-Defined Aggregate Functions
+
+Typed aggregations, as described above, may also be registered as untyped aggregating UDFs for use with DataFrames.
+For example, a user-defined average for untyped DataFrames can look like:
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+  {% include_example untyped_custom_aggregation scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala%}
+</div>
+<div data-lang="java"  markdown="1">
+  {% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
+</div>
+</div>
+
+### Related Statements
+ * [Scalar User Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+ * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-functions-udf-scalar.md b/docs/sql-ref-functions-udf-scalar.md
index cee135b572aea..2cb25f275cb59 100644
--- a/docs/sql-ref-functions-udf-scalar.md
+++ b/docs/sql-ref-functions-udf-scalar.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: User defined Scalar Functions (UDF)
-displayTitle: User defined Scalar Functions (UDF)
+title: Scalar User Defined Functions (UDFs)
+displayTitle: Scalar User Defined Functions (UDFs)
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -9,9 +9,9 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,4 +19,43 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+User-Defined Functions (UDFs) are user-programmable routines that act on one row. This documentation lists the classes that are required for creating and registering UDFs. It also contains examples that demonstrate how to define and register UDFs and invoke them in Spark SQL.
+
+### UserDefinedFunction
+
+To define the properties of a user-defined function, the user can use some of the methods defined in this class.
+<dl>
+  <dt><code><em>asNonNullable(): UserDefinedFunction</em></code></dt>
+  <dd>
+    Updates UserDefinedFunction to non-nullable.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>asNondeterministic(): UserDefinedFunction</em></code></dt>
+  <dd>
+    Updates UserDefinedFunction to nondeterministic.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>withName(name: String): UserDefinedFunction</em></code></dt>
+  <dd>
+    Updates UserDefinedFunction with a given name.
+  </dd>
+</dl>
+
+### Examples
+
+<div class="codetabs">
+<div data-lang="scala"  markdown="1">
+{% include_example udf_scalar scala/org/apache/spark/examples/sql/UserDefinedScalar.scala%}
+</div>
+<div data-lang="java"  markdown="1">
+  {% include_example udf_scalar java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java%}
+</div>
+</div>
+
+### Related Statements
+ * [User Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+ * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java
new file mode 100644
index 0000000000000..e5e698809b215
--- /dev/null
+++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.examples.sql;
+
+// $example on:udf_scalar$
+import org.apache.spark.sql.*;
+import org.apache.spark.sql.api.java.UDF1;
+import org.apache.spark.sql.expressions.UserDefinedFunction;
+import static org.apache.spark.sql.functions.udf;
+import org.apache.spark.sql.types.DataTypes;
+// $example off:udf_scalar$
+
+public class JavaUserDefinedScalar {
+
+  public static void main(String[] args) {
+
+    // $example on:udf_scalar$
+    SparkSession spark = SparkSession
+      .builder()
+      .appName("Java Spark SQL UDF scalar example")
+      .getOrCreate();
+
+    // Define and register a zero-argument non-deterministic UDF
+    // UDF is deterministic by default, i.e. produces the same result for the same input.
+    UserDefinedFunction random = udf(
+      () -> Math.random(), DataTypes.DoubleType
+    );
+    random.asNondeterministic();
+    spark.udf().register("random", random);
+    spark.sql("SELECT random()").show();
+    // +-------+
+    // |UDF()  |
+    // +-------+
+    // |xxxxxxx|
+    // +-------+
+
+    // Define and register a one-argument UDF
+    spark.udf().register("plusOne", new UDF1<Integer, Integer>() {
+      @Override
+      public Integer call(Integer x) {
+        return x + 1;
+      }
+    }, DataTypes.IntegerType);
+    spark.sql("SELECT plusOne(5)").show();
+    // +----------+
+    // |plusOne(5)|
+    // +----------+
+    // |         6|
+    // +----------+
+
+    // Define and register a two-argument UDF
+    UserDefinedFunction strLen = udf(
+      (String s, Integer x) -> s.length() + x, DataTypes.IntegerType
+    );
+    spark.udf().register("strLen", strLen);
+    spark.sql("SELECT strLen('test', 1)").show();
+    // +------------+
+    // |UDF(test, 1)|
+    // +------------+
+    // |           5|
+    // +------------+
+
+    // UDF in a WHERE clause
+    spark.udf().register("oneArgFilter", new UDF1<Long, Boolean>() {
+      @Override
+      public Boolean call(Long x) {
+        return  x > 5;
+      }
+    }, DataTypes.BooleanType);
+    spark.range(1, 10).createOrReplaceTempView("test");
+    spark.sql("SELECT * FROM test WHERE oneArgFilter(id)").show();
+    // +---+
+    // | id|
+    // +---+
+    // |  6|
+    // |  7|
+    // |  8|
+    // |  9|
+    // +---+
+
+    // $example off:udf_scalar$
+    spark.stop();
+  }
+}
diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala b/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala
new file mode 100644
index 0000000000000..25218a6dcc5af
--- /dev/null
+++ b/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.examples.sql
+
+// $example on:udf_scalar$
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.functions.udf
+// $example off:udf_scalar$
+
+object UserDefinedScalar {
+
+  def main(args: Array[String]): Unit = {
+    // $example on:udf_scalar$
+    val spark = SparkSession
+      .builder()
+      .appName("Spark SQL UDF scalar example")
+      .getOrCreate()
+
+    // Define and register a zero-argument non-deterministic UDF
+    // UDF is deterministic by default, i.e. produces the same result for the same input.
+    val random = udf(() => Math.random())
+    spark.udf.register("random", random.asNondeterministic())
+    spark.sql("SELECT random()").show()
+    // +-------+
+    // |UDF()  |
+    // +-------+
+    // |xxxxxxx|
+    // +-------+
+
+    // Define and register a one-argument UDF
+    val plusOne = udf((x: Int) => x + 1)
+    spark.udf.register("plusOne", plusOne)
+    spark.sql("SELECT plusOne(5)").show()
+    // +------+
+    // |UDF(5)|
+    // +------+
+    // |     6|
+    // +------+
+
+    // Define a two-argument UDF and register it with Spark in one step
+    spark.udf.register("strLenScala", (_: String).length + (_: Int))
+    spark.sql("SELECT strLenScala('test', 1)").show()
+    // +--------------------+
+    // |strLenScala(test, 1)|
+    // +--------------------+
+    // |                   5|
+    // +--------------------+
+
+    // UDF in a WHERE clause
+    spark.udf.register("oneArgFilter", (n: Int) => { n > 5 })
+    spark.range(1, 10).createOrReplaceTempView("test")
+    spark.sql("SELECT * FROM test WHERE oneArgFilter(id)").show()
+    // +---+
+    // | id|
+    // +---+
+    // |  6|
+    // |  7|
+    // |  8|
+    // |  9|
+    // +---+
+
+    // $example off:udf_scalar$
+
+    spark.stop()
+  }
+}

From 7ddbc76158c4fe1a48cb0c99e102b173355055c8 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 12 Apr 2020 23:39:27 -0500
Subject: [PATCH 0483/1280] [SPARK-31419][SQL][DOCS] Document Table-valued
 Function and Inline Table

### What changes were proposed in this pull request?
Document Table-valued Function and Inline Table

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes

<img width="1050" alt="Screen Shot 2020-04-11 at 5 34 25 PM" src="https://user-images.githubusercontent.com/13592258/79057852-cedff880-7c1a-11ea-9e1e-7882594ab573.png">

<img width="1050" alt="Screen Shot 2020-04-11 at 5 34 46 PM" src="https://user-images.githubusercontent.com/13592258/79057854-d4d5d980-7c1a-11ea-94cc-92ef1121fa43.png">

<img width="1050" alt="Screen Shot 2020-04-10 at 7 36 00 PM" src="https://user-images.githubusercontent.com/13592258/79033391-c2986480-7b62-11ea-9d0a-6c60de823256.png">

<img width="1051" alt="Screen Shot 2020-04-10 at 7 36 21 PM" src="https://user-images.githubusercontent.com/13592258/79033392-c5935500-7b62-11ea-88d4-e7d7812a7add.png">

<img width="1051" alt="Screen Shot 2020-04-11 at 5 09 48 PM" src="https://user-images.githubusercontent.com/13592258/79057555-6ba09700-7c17-11ea-9683-16bbde63a529.png">

Also, linked the newly added pages to select statement

<img width="1050" alt="Screen Shot 2020-04-10 at 3 27 59 PM" src="https://user-images.githubusercontent.com/13592258/79027245-5147ba00-7b40-11ea-9b10-527fd9639958.png">

### How was this patch tested?
Manually build and check

Closes #28185 from huaxingao/tvf.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 310bef1ac76f48bffcae8e46eb7766a84cb9ef6b)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                      |   4 +
 .../sql-ref-syntax-qry-select-inline-table.md |  84 +++++++++++
 docs/sql-ref-syntax-qry-select-tvf.md         | 131 ++++++++++++++++++
 docs/sql-ref-syntax-qry-select.md             |   6 +-
 4 files changed, 222 insertions(+), 3 deletions(-)
 create mode 100644 docs/sql-ref-syntax-qry-select-inline-table.md
 create mode 100644 docs/sql-ref-syntax-qry-select-tvf.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 75efb30c87080..b8c6e50e53fb0 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -162,6 +162,10 @@
                   url: sql-ref-syntax-qry-select-setops.html
                 - text: TABLESAMPLE
                   url: sql-ref-syntax-qry-sampling.html
+                - text: Table-valued Function
+                  url: sql-ref-syntax-qry-select-tvf.html
+                - text: Inline Table
+                  url: sql-ref-syntax-qry-select-inline-table.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-qry-select-inline-table.md b/docs/sql-ref-syntax-qry-select-inline-table.md
new file mode 100644
index 0000000000000..7a532857bc322
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-inline-table.md
@@ -0,0 +1,84 @@
+---
+layout: global
+title: Inline Table
+displayTitle: Inline Table
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+An inline table is a temporary table created using a VALUES clause.
+
+### Syntax
+
+{% highlight sql %}
+VALUES ( expression [ , ... ] ) [ table_alias ]
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>expression</em></code></dt>
+  <dd>
+    Specifies a combination of one or more values, operators and SQL functions that results in a value.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>table_alias</em></code></dt>
+  <dd>
+    Specifies a temporary name with an optional column name list. <br><br>
+    <b>Syntax:</b>
+      <code>
+        [ AS ] table_name [ ( column_name [ , ... ] ) ]
+      </code>
+  </dd>
+</dl>
+
+### Examples
+
+{% highlight sql %}
+-- single row, without a table alias
+SELECT * FROM VALUES ("one", 1);
+  +----+----+
+  |col1|col2|
+  +----+----+
+  | one|   1|
+  +----+----+
+
+-- three rows with a table alias
+SELECT * FROM VALUES ("one", 1), ("two", 2), ("three", null) AS data(a, b);
+  +-----+----+
+  |    a|   b|
+  +-----+----+
+  |  one|   1|
+  |  two|   2|
+  |three|null|
+  +-----+----+
+
+-- complex types with a table alias
+SELECT * FROM VALUES ("one", array(0, 1)), ("two", array(2, 3)) AS data(a, b);
+  +---+------+
+  |  a|     b|
+  +---+------+
+  |one|[0, 1]|
+  |two|[2, 3]|
+  +---+------+
+{% endhighlight %}
+
+### Related Statement
+
+ * [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
new file mode 100644
index 0000000000000..bbfd870d0cbe7
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -0,0 +1,131 @@
+---
+layout: global
+title: Table-valued Functions (TVF)
+displayTitle: Table-valued Functions (TVF)
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+A table-valued function (TVF) is a function that returns a relation or a set of rows.
+
+### Syntax
+
+{% highlight sql %}
+function_name ( expression [ , ... ] ) [ table_alias ]
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>expression</em></code></dt>
+  <dd>
+    Specifies a combination of one or more values, operators and SQL functions that results in a value.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>table_alias</em></code></dt>
+  <dd>
+    Specifies a temporary name with an optional column name list. <br><br>
+    <b>Syntax:</b>
+      <code>
+        [ AS ] table_name [ ( column_name [ , ... ] ) ]
+      </code>
+  </dd>
+</dl>
+
+### Supported Table-valued Functions
+
+<table class="table">
+  <thead>
+    <tr><th style="width:25%">Function</th><th>Argument Type(s)</th><th>Description</th></tr>
+  </thead>
+    <tr>
+      <td><b> range </b>( <i>end</i> )</td>
+      <td> Long </td>
+      <td>Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from 0 to <code>end</code> (exclusive) with step value 1.</td>
+    </tr>
+    <tr>
+      <td><b> range </b>( <i> start, end</i> )</td>
+      <td> Long, Long </td>
+      <td width="60%">Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from <code>start</code> to <code>end</code> (exclusive) with step value 1.</td>
+    </tr>
+    <tr>
+      <td><b> range </b>( <i> start, end, step</i> )</td>
+      <td> Long, Long, Long </td>
+      <td width="60%">Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from <code>start</code> to <code>end</code> (exclusive) with <code>step</code> value.</td>
+     </tr>
+    <tr>
+      <td><b> range </b>( <i> start, end, step, numPartitions</i> )</td>
+      <td> Long, Long, Long, Int </td>
+      <td width="60%">Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from <code>start</code> to <code>end</code> (exclusive) with <code>step</code> value, with partition number <code>numPartitions</code> specified. </td>
+    </tr>
+</table>
+
+### Examples
+
+{% highlight sql %}
+-- range call with end
+SELECT * FROM range(6 + cos(3));
+  +---+
+  | id|
+  +---+
+  |  0|
+  |  1|
+  |  2|
+  |  3|
+  |  4|
+  +---+
+
+-- range call with start and end
+SELECT * FROM range(5, 10);
+  +---+
+  | id|
+  +---+
+  |  5|
+  |  6|
+  |  7|
+  |  8|
+  |  9|
+  +---+
+
+-- range call with numPartitions
+SELECT * FROM range(0, 10, 2, 200);
+  +---+
+  | id|
+  +---+
+  |  0|
+  |  2|
+  |  4|
+  |  6|
+  |  8|
+  +---+
+
+-- range call with a table alias
+SELECT * FROM range(5, 8) AS test;
+  +---+
+  | id|
+  +---+
+  |  5|
+  |  6|
+  |  7|
+  +---+
+{% endhighlight %}
+
+### Related Statement
+
+ * [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index f1585df14e734..8ecc2c630f090 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -78,9 +78,9 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
     Specifies a source of input for the query. It can be one of the following:
     <ol>
       <li>Table relation</li>
-      <li>Join relation</li>
-      <li>Table valued function</li>
-      <li>Inlined table</li>
+      <li><a href="sql-ref-syntax-qry-select-join.html">Join relation</a></li>
+      <li><a href="sql-ref-syntax-qry-select-tvf.html">Table-value function</a></li>
+      <li><a href="sql-ref-syntax-qry-select-inline-table.html">Inline table</a></li>
       <li>Subquery</li>
     </ol>
   </dd>

From 44866125cae3f1cca7dc3a9f876fd0a3125405aa Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sun, 12 Apr 2020 23:40:36 -0500
Subject: [PATCH 0484/1280] [SPARK-31383][SQL][DOC] Clean up the SQL documents
 in docs/sql-ref*

### What changes were proposed in this pull request?

This PR intends to clean up the SQL documents in `doc/sql-ref*`.
Main changes are as follows;

 - Fixes wrong syntaxes and capitalize sub-titles
 - Adds some DDL queries in `Examples` so that users can run examples there
 - Makes query output in `Examples` follows the `Dataset.showString` (right-aligned) format
 - Adds/Removes spaces, Indents, or blank lines to follow the format below;

```
---
license...
---

### Description

Writes what's the syntax is.

### Syntax

{% highlight sql %}
SELECT...
    WHERE... // 4 indents after the second line
    ...
{% endhighlight %}

### Parameters

<dl>

  <dt><code><em>Param Name</em></code></dt>
  <dd>
    Param Description
  </dd>
  ...
</dl>

### Examples

{% highlight sql %}
-- It is better that users are able to execute example queries here.
-- So, we prepare test data in the first section if possible.
CREATE TABLE t (key STRING, value DOUBLE);
INSERT INTO t VALUES
    ('a', 1.0), ('a', 2.0), ('b', 3.0), ('c', 4.0);

-- query output has 2 indents and it follows the `Dataset.showString`
-- format (right-aligned).
SELECT * FROM t;
  +---+-----+
  |key|value|
  +---+-----+
  |  a|  1.0|
  |  a|  2.0|
  |  b|  3.0|
  |  c|  4.0|
  +---+-----+

-- Query statements after the second line have 4 indents.
SELECT key, SUM(value)
    FROM t
    GROUP BY key;
  +---+----------+
  |key|sum(value)|
  +---+----------+
  |  c|       4.0|
  |  b|       3.0|
  |  a|       3.0|
  +---+----------+
...
{% endhighlight %}

### Related Statements

 * [XXX](xxx.html)
 * ...
```

### Why are the changes needed?

The most changes of this PR are pretty minor, but I think the consistent formats/rules to write documents are important for long-term maintenance in our community

### Does this PR introduce any user-facing change?

Yes.

### How was this patch tested?

Manually checked.

Closes #28151 from maropu/MakeRightAligned.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 179289f0bfc183866d635dc72b507902a3452d3a)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/sql-ref-ansi-compliance.md               |  10 -
 docs/sql-ref-datatypes.md                     |   1 -
 docs/sql-ref-functions-builtin.md             |   6 +-
 docs/sql-ref-functions-udf.md                 |   6 +-
 docs/sql-ref-null-semantics.md                | 314 +++++++++--------
 docs/sql-ref-syntax-aux-analyze-table.md      | 103 ++++--
 docs/sql-ref-syntax-aux-analyze.md            |   2 +-
 docs/sql-ref-syntax-aux-cache-cache-table.md  |  12 +-
 docs/sql-ref-syntax-aux-cache-clear-cache.md  |   5 +-
 docs/sql-ref-syntax-aux-cache-refresh.md      |  16 +-
 .../sql-ref-syntax-aux-cache-uncache-table.md |   5 +
 docs/sql-ref-syntax-aux-cache.md              |  10 +-
 docs/sql-ref-syntax-aux-conf-mgmt-reset.md    |   7 +-
 docs/sql-ref-syntax-aux-conf-mgmt-set.md      |  21 +-
 docs/sql-ref-syntax-aux-conf-mgmt.md          |   4 +-
 docs/sql-ref-syntax-aux-describe-database.md  |  36 +-
 docs/sql-ref-syntax-aux-describe-function.md  |  11 +-
 docs/sql-ref-syntax-aux-describe-query.md     |  39 +-
 docs/sql-ref-syntax-aux-describe-table.md     | 159 +++++----
 docs/sql-ref-syntax-aux-describe.md           |   8 +-
 docs/sql-ref-syntax-aux-refresh-table.md      |  13 +-
 ...l-ref-syntax-aux-resource-mgmt-add-file.md |   5 +
 ...ql-ref-syntax-aux-resource-mgmt-add-jar.md |   5 +
 ...-ref-syntax-aux-resource-mgmt-list-file.md |   4 +
 ...l-ref-syntax-aux-resource-mgmt-list-jar.md |   4 +
 docs/sql-ref-syntax-aux-resource-mgmt.md      |   8 +-
 docs/sql-ref-syntax-aux-show-columns.md       |  35 +-
 docs/sql-ref-syntax-aux-show-create-table.md  |  34 +-
 docs/sql-ref-syntax-aux-show-databases.md     |  14 +-
 docs/sql-ref-syntax-aux-show-functions.md     |  56 +--
 docs/sql-ref-syntax-aux-show-partitions.md    |  35 +-
 docs/sql-ref-syntax-aux-show-table.md         | 211 +++++------
 docs/sql-ref-syntax-aux-show-tables.md        |  80 +++--
 docs/sql-ref-syntax-aux-show-tblproperties.md |  31 +-
 docs/sql-ref-syntax-aux-show-views.md         |   1 +
 docs/sql-ref-syntax-aux-show.md               |  19 +-
 docs/sql-ref-syntax-aux.md                    |  12 +-
 docs/sql-ref-syntax-ddl-alter-database.md     |  25 +-
 docs/sql-ref-syntax-ddl-alter-table.md        | 332 +++++++++---------
 docs/sql-ref-syntax-ddl-alter-view.md         | 132 ++++---
 docs/sql-ref-syntax-ddl-create-database.md    |  33 +-
 docs/sql-ref-syntax-ddl-create-function.md    |  32 +-
 ...-ref-syntax-ddl-create-table-datasource.md |  47 +--
 ...-ref-syntax-ddl-create-table-hiveformat.md |  57 ++-
 docs/sql-ref-syntax-ddl-create-table-like.md  |  36 +-
 docs/sql-ref-syntax-ddl-create-table.md       |  13 +-
 docs/sql-ref-syntax-ddl-create-view.md        |  11 +-
 docs/sql-ref-syntax-ddl-drop-database.md      |  20 +-
 docs/sql-ref-syntax-ddl-drop-function.md      |  56 ++-
 docs/sql-ref-syntax-ddl-drop-table.md         |  27 +-
 docs/sql-ref-syntax-ddl-drop-view.md          |  32 +-
 docs/sql-ref-syntax-ddl-repair-table.md       |  44 +--
 docs/sql-ref-syntax-ddl-truncate-table.md     |  51 ++-
 docs/sql-ref-syntax-ddl.md                    |  30 +-
 docs/sql-ref-syntax-dml-insert-into.md        | 251 ++++++-------
 ...tax-dml-insert-overwrite-directory-hive.md |  27 +-
 ...f-syntax-dml-insert-overwrite-directory.md |  12 +-
 ...l-ref-syntax-dml-insert-overwrite-table.md | 211 ++++++-----
 docs/sql-ref-syntax-dml-insert.md             |   9 +-
 docs/sql-ref-syntax-dml-load.md               | 115 +++---
 docs/sql-ref-syntax-dml.md                    |   4 +-
 docs/sql-ref-syntax-qry-explain.md            | 124 ++++---
 docs/sql-ref-syntax-qry-sampling.md           |   6 +-
 docs/sql-ref-syntax-qry-select-clusterby.md   |  55 +--
 ...sql-ref-syntax-qry-select-distribute-by.md |  55 +--
 docs/sql-ref-syntax-qry-select-groupby.md     | 213 ++++++-----
 docs/sql-ref-syntax-qry-select-having.md      |  39 +-
 docs/sql-ref-syntax-qry-select-hints.md       |   8 +-
 docs/sql-ref-syntax-qry-select-join.md        |   5 +-
 docs/sql-ref-syntax-qry-select-limit.md       |  55 +--
 docs/sql-ref-syntax-qry-select-orderby.md     |  80 +++--
 docs/sql-ref-syntax-qry-select-setops.md      | 255 +++++++-------
 docs/sql-ref-syntax-qry-select-sortby.md      | 129 +++----
 docs/sql-ref-syntax-qry-select-usedb.md       |  17 +-
 docs/sql-ref-syntax-qry-select-where.md       |  58 +--
 docs/sql-ref-syntax-qry-select.md             |  50 +--
 docs/sql-ref-syntax-qry.md                    |  19 +-
 docs/sql-ref-syntax.md                        | 110 +++---
 78 files changed, 2203 insertions(+), 2024 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 7543180f5db35..3a0c2c123d81b 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -69,18 +69,15 @@ When `spark.sql.ansi.enabled` is set to `true` and an overflow occurs in numeric
 {% highlight sql %}
 -- `spark.sql.ansi.enabled=true`
 SELECT 2147483647 + 1;
-
   java.lang.ArithmeticException: integer overflow
 
 -- `spark.sql.ansi.enabled=false`
 SELECT 2147483647 + 1;
-
   +----------------+
   |(2147483647 + 1)|
   +----------------+
   |     -2147483648|
   +----------------+
-
 {% endhighlight %}
 
 ### Type Conversion
@@ -97,16 +94,13 @@ In future releases, the behaviour of type coercion might change along with the o
 
 -- `spark.sql.ansi.enabled=true`
 SELECT CAST('a' AS INT);
-
   java.lang.NumberFormatException: invalid input syntax for type numeric: a
 
 SELECT CAST(2147483648L AS INT);
-
   java.lang.ArithmeticException: Casting 2147483648 to int causes overflow
 
 -- `spark.sql.ansi.enabled=false` (This is a default behaviour)
 SELECT CAST('a' AS INT);
-
   +--------------+
   |CAST(a AS INT)|
   +--------------+
@@ -114,7 +108,6 @@ SELECT CAST('a' AS INT);
   +--------------+
 
 SELECT CAST(2147483648L AS INT);
-
   +-----------------------+
   |CAST(2147483648 AS INT)|
   +-----------------------+
@@ -126,20 +119,17 @@ CREATE TABLE t (v INT);
 
 -- `spark.sql.storeAssignmentPolicy=ANSI`
 INSERT INTO t VALUES ('1');
-
   org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
   - Cannot safely cast 'v': StringType to IntegerType;
 
 -- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
 INSERT INTO t VALUES ('1');
 SELECT * FROM t;
-
   +---+
   |  v|
   +---+
   |  1|
   +---+
-
 {% endhighlight %}
 
 ### SQL Functions
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 9700608fe8a34..1e0d0511a114f 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -19,7 +19,6 @@ license: |
   limitations under the License.
 ---
 
-
 Spark SQL and DataFrames support the following data types:
 
 * Numeric types
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
index 48e5c0e6e13e2..917c081adb7a0 100644
--- a/docs/sql-ref-functions-builtin.md
+++ b/docs/sql-ref-functions-builtin.md
@@ -21,6 +21,6 @@ license: |
 
 Spark SQL defines built-in functions to use, a complete list of which can be found [here](api/sql/). Among them, Spark SQL has several special categories of built-in functions: [Aggregate Functions](sql-ref-functions-builtin-aggregate.html) to operate on a group of rows, [Array Functions](sql-ref-functions-builtin-array.html) to operate on Array columns, and [Date and Time Functions](sql-ref-functions-builtin-date-time.html) to operate on Date and Time.
 
-* [Aggregate Functions](sql-ref-functions-builtin-aggregate.html)
-* [Array Functions](sql-ref-functions-builtin-array.html)
-* [Date and Time Functions](sql-ref-functions-builtin-date-time.html)
+ * [Aggregate Functions](sql-ref-functions-builtin-aggregate.html)
+ * [Array Functions](sql-ref-functions-builtin-array.html)
+ * [Date and Time Functions](sql-ref-functions-builtin-date-time.html)
diff --git a/docs/sql-ref-functions-udf.md b/docs/sql-ref-functions-udf.md
index 91c04f16049e2..2c5f204d25736 100644
--- a/docs/sql-ref-functions-udf.md
+++ b/docs/sql-ref-functions-udf.md
@@ -21,6 +21,6 @@ license: |
 
 User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.
 
-* [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
-* [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
-* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
+ * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+ * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+ * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index 37b4081d6b27b..dc48a36cadb3c 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -20,6 +20,7 @@ license: |
 ---
 
 ### Description
+
 A table consists of a set of rows and each row contains a set of columns.
 A column is associated with a data type and represents
 a specific attribute of an entity (for example, `age` is a column of an
@@ -61,7 +62,7 @@ the `age` column and this table will be used in various examples in the sections
 <tr><td>700</td><td>Dan</td><td>50</td></tr>
 </table>
 
-### Comparison operators <a name="comp-operators"></a>
+### Comparison Operators <a name="comp-operators"></a>
 
 Apache spark supports the standard comparison operators such as '>', '>=', '=', '<' and '<='.
 The result of these operators is unknown or `NULL` when one of the operands or both the operands are
@@ -114,13 +115,14 @@ one or both operands are `NULL`:
 </table>
 
 ### Examples
+
 {% highlight sql %}
 -- Normal comparison operators return `NULL` when one of the operand is `NULL`.
 SELECT 5 > null AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 
 -- Normal comparison operators return `NULL` when both the operands are `NULL`.
@@ -128,7 +130,7 @@ SELECT null = null AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 
 -- Null-safe equal operator return `False` when one of the operand is `NULL`
@@ -136,7 +138,7 @@ SELECT 5 <=> null AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |false            |
+  |            false|
   +-----------------+
 
 -- Null-safe equal operator return `True` when one of the operand is `NULL`
@@ -144,11 +146,12 @@ SELECT NULL <=> NULL;
   +-----------------+
   |expression_output|
   +-----------------+
-  |true             |
+  |             true|
   +-----------------+
 {% endhighlight %}
 
-### Logical operators <a name="logical-operators"></a>
+### Logical Operators <a name="logical-operators"></a>
+
 Spark supports standard logical operators such as `AND`, `OR` and `NOT`. These operators take `Boolean` expressions
 as the arguments and return a `Boolean` value.  
 
@@ -205,13 +208,14 @@ The following tables illustrate the behavior of logical operators when one or bo
 </table>
 
 ### Examples
+
 {% highlight sql %}
 -- Normal comparison operators return `NULL` when one of the operands is `NULL`.
 SELECT (true OR null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |true             |
+  |             true|
   +-----------------+
 
 -- Normal comparison operators return `NULL` when both the operands are `NULL`.
@@ -219,7 +223,7 @@ SELECT (null OR false) AS expression_output
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 
 -- Null-safe equal operator returns `False` when one of the operands is `NULL`
@@ -227,11 +231,12 @@ SELECT NOT(null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 {% endhighlight %}
 
 ### Expressions <a name="expressions"></a>
+
 The comparison operators and logical operators are treated as expressions in
 Spark. Other than these two kinds of expressions, Spark supports other form of
 expressions such as function expressions, cast expressions, etc. The expressions
@@ -240,35 +245,37 @@ in Spark can be broadly classified as :
 - Expressions that can process `NULL` value operands
   - The result of these expressions depends on the expression itself.
 
-#### Null intolerant expressions <a name="null-intolerant"></a>
+#### Null Intolerant Expressions <a name="null-intolerant"></a>
+
 Null intolerant expressions return `NULL` when one or more arguments of 
 expression are `NULL` and most of the expressions fall in this category.
 
 ##### Examples
+
 {% highlight sql %}
-SELECT concat('John', null) as expression_output;
+SELECT concat('John', null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 
-SELECT positive(null) as expression_output;
+SELECT positive(null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 
-SELECT to_date(null) as expression_output;
+SELECT to_date(null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 {% endhighlight %}
 
-#### Expressions that can process null value operands. <a name="can-process-null"></a>
+#### Expressions That Can Process Null Value Operands <a name="can-process-null"></a>
 
 This class of expressions are designed to handle `NULL` values. The result of the 
 expressions depends on the expression itself. As an example, function expression `isnull`
@@ -287,14 +294,14 @@ returns the first non `NULL` value in its list of operands. However, `coalesce`
   - ATLEASTNNONNULLS 
   - IN
 
-
 ##### Examples
+
 {% highlight sql %}
 SELECT isnull(null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |true             |
+  |             true|
   +-----------------+
 
 -- Returns the first occurrence of non `NULL` value.
@@ -302,7 +309,7 @@ SELECT coalesce(null, null, 3, null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |3                |
+  |                3|
   +-----------------+
 
 -- Returns `NULL` as all its operands are `NULL`. 
@@ -310,18 +317,19 @@ SELECT coalesce(null, null, null, null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |null             |
+  |             null|
   +-----------------+
 
-SELECT isnan(null) as expression_output;
+SELECT isnan(null) AS expression_output;
   +-----------------+
   |expression_output|
   +-----------------+
-  |false            |
+  |            false|
   +-----------------+
 {% endhighlight %}
 
 #### Builtin Aggregate Expressions <a name="built-in-aggregate"></a>
+
 Aggregate functions compute a single result by processing a set of input rows. Below are
 the rules of how `NULL` values are handled by aggregate functions.
 - `NULL` values are ignored from processing by all the aggregate functions.
@@ -337,13 +345,14 @@ the rules of how `NULL` values are handled by aggregate functions.
   - SOME
    
 #### Examples
+
 {% highlight sql %}
 -- `count(*)` does not skip `NULL` values.
 SELECT count(*) FROM person;
   +--------+
   |count(1)|
   +--------+
-  |7       |
+  |       7|
   +--------+
 
 -- `NULL` values in column `age` are skipped from processing.
@@ -351,7 +360,7 @@ SELECT count(age) FROM person;
   +----------+
   |count(age)|
   +----------+
-  |5         |
+  |         5|
   +----------+
 
 -- `count(*)` on an empty input set returns 0. This is unlike the other
@@ -360,7 +369,7 @@ SELECT count(*) FROM person where 1 = 0;
   +--------+
   |count(1)|
   +--------+
-  |0       |
+  |       0|
   +--------+
 
 -- `NULL` values are excluded from computation of maximum value.
@@ -368,7 +377,7 @@ SELECT max(age) FROM person;
   +--------+
   |max(age)|
   +--------+
-  |50      |
+  |      50|
   +--------+
 
 -- `max` returns `NULL` on an empty input set.
@@ -376,44 +385,45 @@ SELECT max(age) FROM person where 1 = 0;
   +--------+
   |max(age)|
   +--------+
-  |null    |
+  |    null|
   +--------+
-
 {% endhighlight %}
 
-### Condition expressions in WHERE, HAVING and JOIN clauses. <a name="condition-expressions"></a>
+### Condition Expressions in WHERE, HAVING and JOIN Clauses <a name="condition-expressions"></a>
+
 `WHERE`, `HAVING` operators filter rows based on the user specified condition.
 A `JOIN` operator is used to combine rows from two tables based on a join condition.
 For all the three operators, a condition expression is a boolean expression and can return
  <code>True, False or Unknown (NULL)</code>. They are "satisfied" if the result of the condition is `True`.
 
 #### Examples
+
 {% highlight sql %}
 -- Persons whose age is unknown (`NULL`) are filtered out from the result set.
 SELECT * FROM person WHERE age > 0;
   +--------+---+
-  |name    |age|
+  |    name|age|
   +--------+---+
-  |Michelle|30 |
-  |Fred    |50 |
-  |Mike    |18 |
-  |Dan     |50 |
-  |Joe     |30 |
+  |Michelle| 30|
+  |    Fred| 50|
+  |    Mike| 18|
+  |     Dan| 50|
+  |     Joe| 30|
   +--------+---+
 
 -- `IS NULL` expression is used in disjunction to select the persons
 -- with unknown (`NULL`) records.
 SELECT * FROM person WHERE age > 0 OR age IS NULL;
   +--------+----+
-  |name    |age |
+  |    name| age|
   +--------+----+
-  |Albert  |null|
-  |Michelle|30  |
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  |Marry   |null|
-  |Joe     |30  |
+  |  Albert|null|
+  |Michelle|  30|
+  |    Fred|  50|
+  |    Mike|  18|
+  |     Dan|  50|
+  |   Marry|null|
+  |     Joe|  30|
   +--------+----+
 
 -- Person with unknown(`NULL`) ages are skipped from processing.
@@ -421,135 +431,139 @@ SELECT * FROM person GROUP BY age HAVING max(age) > 18;
   +---+--------+                                                                  
   |age|count(1)|
   +---+--------+
-  |50 |2       |
-  |30 |2       |
+  | 50|       2|
+  | 30|       2|
   +---+--------+
 
 -- A self join case with a join condition `p1.age = p2.age AND p1.name = p2.name`.
 -- The persons with unknown age (`NULL`) are filtered out by the join operator.
 SELECT * FROM person p1, person p2
-WHERE p1.age = p2.age
-  AND p1.name = p2.name;
+    WHERE p1.age = p2.age
+    AND p1.name = p2.name;
   +--------+---+--------+---+
-  |name    |age|name    |age|
+  |    name|age|    name|age|
   +--------+---+--------+---+
-  |Michelle|30 |Michelle|30 |
-  |Fred    |50 |Fred    |50 |
-  |Mike    |18 |Mike    |18 |
-  |Dan     |50 |Dan     |50 |
-  |Joe     |30 |Joe     |30 |
+  |Michelle| 30|Michelle| 30|
+  |    Fred| 50|    Fred| 50|
+  |    Mike| 18|    Mike| 18|
+  |     Dan| 50|     Dan| 50|
+  |     Joe| 30|     Joe| 30|
   +--------+---+--------+---+
 
 -- The age column from both legs of join are compared using null-safe equal which
 -- is why the persons with unknown age (`NULL`) are qualified by the join.
 SELECT * FROM person p1, person p2
-WHERE p1.age <=> p2.age
-  AND p1.name = p2.name;
-+--------+----+--------+----+
-|    name| age|    name| age|
-+--------+----+--------+----+
-|  Albert|null|  Albert|null|
-|Michelle|  30|Michelle|  30|
-|    Fred|  50|    Fred|  50|
-|    Mike|  18|    Mike|  18|
-|     Dan|  50|     Dan|  50|
-|   Marry|null|   Marry|null|
-|     Joe|  30|     Joe|  30|
-+--------+----+--------+----+
-
+    WHERE p1.age <=> p2.age
+    AND p1.name = p2.name;
+  +--------+----+--------+----+
+  |    name| age|    name| age|
+  +--------+----+--------+----+
+  |  Albert|null|  Albert|null|
+  |Michelle|  30|Michelle|  30|
+  |    Fred|  50|    Fred|  50|
+  |    Mike|  18|    Mike|  18|
+  |     Dan|  50|     Dan|  50|
+  |   Marry|null|   Marry|null|
+  |     Joe|  30|     Joe|  30|
+  +--------+----+--------+----+
 {% endhighlight %}
 
-### Aggregate operator (GROUP BY, DISTINCT) <a name="aggregate-operator"></a>
+### Aggregate Operator (GROUP BY, DISTINCT) <a name="aggregate-operator"></a>
+
 As discussed in the previous section [comparison operator](sql-ref-null-semantics.html#comparison-operators),
 two `NULL` values are not equal. However, for the purpose of grouping and distinct processing, the two or more
 values with `NULL data`are grouped together into the same bucket. This behaviour is conformant with SQL
 standard and with other enterprise database management systems.
 
 #### Examples
+
 {% highlight sql %}
 -- `NULL` values are put in one bucket in `GROUP BY` processing.
 SELECT age, count(*) FROM person GROUP BY age;
   +----+--------+                                                                 
-  |age |count(1)|
+  | age|count(1)|
   +----+--------+
-  |null|2       |
-  |50  |2       |
-  |30  |2       |
-  |18  |1       |
+  |null|       2|
+  |  50|       2|
+  |  30|       2|
+  |  18|       1|
   +----+--------+
 
 -- All `NULL` ages are considered one distinct value in `DISTINCT` processing.
 SELECT DISTINCT age FROM person;
   +----+
-  |age |
+  | age|
   +----+
   |null|
-  |50  |
-  |30  |
-  |18  |
+  |  50|
+  |  30|
+  |  18|
   +----+
-
 {% endhighlight %}
 
-### Sort operator (ORDER BY Clause) <a name="order-by"></a>
+### Sort Operator (ORDER BY Clause) <a name="order-by"></a>
+
 Spark SQL supports null ordering specification in `ORDER BY` clause. Spark processes the `ORDER BY` clause by
 placing all the `NULL` values at first or at last depending on the null ordering specification. By default, all
 the `NULL` values are placed at first.
 
 #### Examples
+
 {% highlight sql %}
 -- `NULL` values are shown at first and other values
 -- are sorted in ascending way.
 SELECT age, name FROM person ORDER BY age;
   +----+--------+
-  |age |name    |
+  | age|    name|
   +----+--------+
-  |null|Marry   |
-  |null|Albert  |
-  |18  |Mike    |
-  |30  |Michelle|
-  |30  |Joe     |
-  |50  |Fred    |
-  |50  |Dan     |
+  |null|   Marry|
+  |null|  Albert|
+  |  18|    Mike|
+  |  30|Michelle|
+  |  30|     Joe|
+  |  50|    Fred|
+  |  50|     Dan|
   +----+--------+
 
 -- Column values other than `NULL` are sorted in ascending
 -- way and `NULL` values are shown at the last.
 SELECT age, name FROM person ORDER BY age NULLS LAST;
   +----+--------+
-  |age |name    |
+  | age|    name|
   +----+--------+
-  |18  |Mike    |
-  |30  |Michelle|
-  |30  |Joe     |
-  |50  |Dan     |
-  |50  |Fred    |
-  |null|Marry   |
-  |null|Albert  |
+  |  18|    Mike|
+  |  30|Michelle|
+  |  30|     Joe|
+  |  50|     Dan|
+  |  50|    Fred|
+  |null|   Marry|
+  |null|  Albert|
   +----+--------+
 
 -- Columns other than `NULL` values are sorted in descending
 -- and `NULL` values are shown at the last.
 SELECT age, name FROM person ORDER BY age DESC NULLS LAST;
   +----+--------+
-  |age |name    |
+  | age|    name|
   +----+--------+
-  |50  |Fred    |
-  |50  |Dan     |
-  |30  |Michelle|
-  |30  |Joe     |
-  |18  |Mike    |
-  |null|Marry   |
-  |null|Albert  |
+  |  50|    Fred|
+  |  50|     Dan|
+  |  30|Michelle|
+  |  30|     Joe|
+  |  18|    Mike|
+  |null|   Marry|
+  |null|  Albert|
   +----+--------+
 {% endhighlight %}
 
-### Set operators (UNION, INTERSECT, EXCEPT) <a name="set-operators"></a>
+### Set Operators (UNION, INTERSECT, EXCEPT) <a name="set-operators"></a>
+
 `NULL` values are compared in a null-safe manner for equality in the context of
 set operations. That means when comparing rows, two `NULL` values are considered 
 equal unlike the regular `EqualTo`(`=`) operator.
 
 #### Examples
+
 {% highlight sql %}
 CREATE VIEW unknown_age SELECT * FROM person WHERE age IS NULL;
 
@@ -557,51 +571,51 @@ CREATE VIEW unknown_age SELECT * FROM person WHERE age IS NULL;
 -- result set. The comparison between columns of the row are done
 -- in a null-safe manner.
 SELECT name, age FROM person
-INTERSECT
-SELECT name, age from unknown_age;
+    INTERSECT
+    SELECT name, age from unknown_age;
   +------+----+                                                                   
-  |name  |age |
+  |  name| age|
   +------+----+
   |Albert|null|
-  |Marry |null|
+  | Marry|null|
   +------+----+
 
 -- `NULL` values from two legs of the `EXCEPT` are not in output. 
 -- This basically shows that the comparison happens in a null-safe manner.
 SELECT age, name FROM person
-EXCEPT
-SELECT age FROM unknown_age;
+    EXCEPT
+    SELECT age FROM unknown_age;
   +---+--------+                                                                  
-  |age|name    |
+  |age|    name|
   +---+--------+
-  |30 |Joe     |
-  |50 |Fred    |
-  |30 |Michelle|
-  |18 |Mike    |
-  |50 |Dan     |
+  | 30|     Joe|
+  | 50|    Fred|
+  | 30|Michelle|
+  | 18|    Mike|
+  | 50|     Dan|
   +---+--------+
 
 -- Performs `UNION` operation between two sets of data. 
 -- The comparison between columns of the row ae done in
 -- null-safe manner.
 SELECT name, age FROM person
-UNION 
-SELECT name, age FROM unknown_age;
+    UNION 
+    SELECT name, age FROM unknown_age;
   +--------+----+                                                                 
-  |name    |age |
+  |    name| age|
   +--------+----+
-  |Albert  |null|
-  |Joe     |30  |
-  |Michelle|30  |
-  |Marry   |null|
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
+  |  Albert|null|
+  |     Joe|  30|
+  |Michelle|  30|
+  |   Marry|null|
+  |    Fred|  50|
+  |    Mike|  18|
+  |     Dan|  50|
   +--------+----+
 {% endhighlight %}
 
-
 ### EXISTS/NOT EXISTS Subquery <a name="exists-not-exists"></a>
+
 In Spark, EXISTS and NOT EXISTS expressions are allowed inside a WHERE clause. 
 These are boolean expressions which return either `TRUE` or
 `FALSE`. In other words, EXISTS is a membership condition and returns `TRUE`
@@ -614,20 +628,21 @@ the subquery. They are normally faster because they can be converted to
 semijoins / anti-semijoins without special provisions for null awareness.
 
 #### Examples
+
 {% highlight sql %}
 -- Even if subquery produces rows with `NULL` values, the `EXISTS` expression
 -- evaluates to `TRUE` as the subquery produces 1 row.
 SELECT * FROM person WHERE EXISTS (SELECT null);
   +--------+----+                                                                 
-  |name    |age |
+  |    name| age|
   +--------+----+
-  |Albert  |null|
-  |Michelle|30  |
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  |Marry   |null|
-  |Joe     |30  |
+  |  Albert|null|
+  |Michelle|  30|
+  |    Fred|  50|
+  |    Mike|  18|
+  |     Dan|  50|
+  |   Marry|null|
+  |     Joe|  30|
   +--------+----+
 
 -- `NOT EXISTS` expression returns `FALSE`. It returns `TRUE` only when
@@ -641,19 +656,20 @@ SELECT * FROM person WHERE NOT EXISTS (SELECT null);
 -- `NOT EXISTS` expression returns `TRUE`.
 SELECT * FROM person WHERE NOT EXISTS (SELECT 1 WHERE 1 = 0);
   +--------+----+
-  |name    |age |
+  |    name| age|
   +--------+----+
-  |Albert  |null|
-  |Michelle|30  |
-  |Fred    |50  |
-  |Mike    |18  |
-  |Dan     |50  |
-  |Marry   |null|
-  |Joe     |30  |
+  |  Albert|null|
+  |Michelle|  30|
+  |    Fred|  50|
+  |    Mike|  18|
+  |     Dan|  50|
+  |   Marry|null|
+  |     Joe|  30|
   +--------+----+
 {% endhighlight %}
 
 ### IN/NOT IN Subquery <a name="in-not-in"></a>
+
 In Spark, `IN` and `NOT IN` expressions are allowed inside a WHERE clause of
 a query. Unlike the `EXISTS` expression, `IN` expression can return a `TRUE`,
 `FALSE` or `UNKNOWN (NULL)` value. Conceptually a `IN` expression is semantically
@@ -675,6 +691,7 @@ This is because IN returns UNKNOWN if the value is not in the list containing `N
 and because NOT UNKNOWN is again UNKNOWN.
  
 #### Examples
+
 {% highlight sql %}
 -- The subquery has only `NULL` value in its result set. Therefore,
 -- the result of `IN` predicate is UNKNOWN.
@@ -687,22 +704,21 @@ SELECT * FROM person WHERE age IN (SELECT null);
 -- The subquery has `NULL` value in the result set as well as a valid 
 -- value `50`. Rows with age = 50 are returned. 
 SELECT * FROM person
-WHERE age IN (SELECT age FROM VALUES (50), (null) sub(age));
+    WHERE age IN (SELECT age FROM VALUES (50), (null) sub(age));
   +----+---+
   |name|age|
   +----+---+
-  |Fred|50 |
-  |Dan |50 |
+  |Fred| 50|
+  | Dan| 50|
   +----+---+
 
 -- Since subquery has `NULL` value in the result set, the `NOT IN`
 -- predicate would return UNKNOWN. Hence, no rows are
 -- qualified for this query.
 SELECT * FROM person
-WHERE age NOT IN (SELECT age FROM VALUES (50), (null) sub(age));
+    WHERE age NOT IN (SELECT age FROM VALUES (50), (null) sub(age));
   +----+---+
   |name|age|
   +----+---+
   +----+---+
-
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-aux-analyze-table.md b/docs/sql-ref-syntax-aux-analyze-table.md
index 40513e836b026..739e692680233 100644
--- a/docs/sql-ref-syntax-aux-analyze-table.md
+++ b/docs/sql-ref-syntax-aux-analyze-table.md
@@ -24,13 +24,14 @@ license: |
 The `ANALYZE TABLE` statement collects statistics about the table to be used by the query optimizer to find a better query execution plan.
 
 ### Syntax
+
 {% highlight sql %}
 ANALYZE TABLE table_identifier [ partition_spec ]
     COMPUTE STATISTICS [ NOSCAN | FOR COLUMNS col [ , ... ] | FOR ALL COLUMNS ]
-
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -69,41 +70,69 @@ ANALYZE TABLE table_identifier [ partition_spec ]
 </dl>
 
 ### Examples
-{% highlight sql %}
- ANALYZE TABLE students COMPUTE STATISTICS NOSCAN;
-
- DESC EXTENDED students;
-     ......
-     Statistics	2820 bytes
-     ......
-
- ANALYZE TABLE students COMPUTE STATISTICS;
-
- DESC EXTENDED students;
-     ......
-     Statistics	2820 bytes, 3 rows
-     ......
-
- ANALYZE TABLE students PARTITION (student_id = 111111) COMPUTE STATISTICS;
-
- DESC EXTENDED students PARTITION (student_id = 111111);
-     ......
-     Partition Statistics	919 bytes, 1 rows
-     ......
-
- ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name;
-
- DESC EXTENDED students name;
-     =default tbl=students
-     col_name	name
-     data_type	string
-     comment	NULL
-     min	NULL
-     max	NULL
-     num_nulls	0
-     distinct_count	3
-     avg_col_len	11
-     max_col_len	13
-     histogram	NULL
 
+{% highlight sql %}
+CREATE TABLE students (name STRING, student_id INT) PARTITIONED BY (student_id);
+INSERT INTO students PARTITION (student_id = 111111) VALUES ('Mark');
+INSERT INTO students PARTITION (student_id = 222222) VALUES ('John');
+
+ANALYZE TABLE students COMPUTE STATISTICS NOSCAN;
+
+DESC EXTENDED students;
+  +--------------------+--------------------+-------+
+  |            col_name|           data_type|comment|
+  +--------------------+--------------------+-------+
+  |                name|              string|   null|
+  |          student_id|                 int|   null|
+  |                 ...|                 ...|    ...|
+  |          Statistics|           864 bytes|       |
+  |                 ...|                 ...|    ...|
+  |  Partition Provider|             Catalog|       |
+  +--------------------+--------------------+-------+
+
+ANALYZE TABLE students COMPUTE STATISTICS;
+
+DESC EXTENDED students;
+  +--------------------+--------------------+-------+
+  |            col_name|           data_type|comment|
+  +--------------------+--------------------+-------+
+  |                name|              string|   null|
+  |          student_id|                 int|   null|
+  |                 ...|                 ...|    ...|
+  |          Statistics|   864 bytes, 2 rows|       |
+  |                 ...|                 ...|    ...|
+  |  Partition Provider|             Catalog|       |
+  +--------------------+--------------------+-------+
+
+ANALYZE TABLE students PARTITION (student_id = 111111) COMPUTE STATISTICS;
+
+DESC EXTENDED students PARTITION (student_id = 111111);
+  +--------------------+--------------------+-------+
+  |            col_name|           data_type|comment|
+  +--------------------+--------------------+-------+
+  |                name|              string|   null|
+  |          student_id|                 int|   null|
+  |                 ...|                 ...|    ...|
+  |Partition Statistics|   432 bytes, 1 rows|       |
+  |                 ...|                 ...|    ...|
+  |        OutputFormat|org.apache.hadoop...|       |
+  +--------------------+--------------------+-------+
+
+ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name;
+
+DESC EXTENDED students name;
+  +--------------+----------+
+  |     info_name|info_value|
+  +--------------+----------+
+  |      col_name|      name|
+  |     data_type|    string|
+  |       comment|      NULL|
+  |           min|      NULL|
+  |           max|      NULL|
+  |     num_nulls|         0|
+  |distinct_count|         2|
+  |   avg_col_len|         4|
+  |   max_col_len|         4|
+  |     histogram|      NULL|
+  +--------------+----------+
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-aux-analyze.md b/docs/sql-ref-syntax-aux-analyze.md
index b1bdc73657724..4c68e6b9ec974 100644
--- a/docs/sql-ref-syntax-aux-analyze.md
+++ b/docs/sql-ref-syntax-aux-analyze.md
@@ -19,4 +19,4 @@ license: |
   limitations under the License.
 ---
 
-* [ANALYZE TABLE statement](sql-ref-syntax-aux-analyze-table.html)
+ * [ANALYZE TABLE statement](sql-ref-syntax-aux-analyze-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index 27cc77b938fbe..11f682cc10891 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -20,16 +20,19 @@ license: |
 ---
 
 ### Description
+
 `CACHE TABLE` statement caches contents of a table or output of a query with the given storage level. If a query is cached, then a temp view will be created for this query.
 This reduces scanning of the original files in future queries. 
 
 ### Syntax
+
 {% highlight sql %}
 CACHE [ LAZY ] TABLE table_identifier
     [ OPTIONS ( 'storageLevel' [ = ] value ) ] [ [ AS ] query ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>LAZY</em></code></dt>
   <dd>Only cache the table when it is first used, instead of immediately.</dd>
@@ -80,13 +83,14 @@ CACHE [ LAZY ] TABLE table_identifier
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testData;
 {% endhighlight %}
 
 ### Related Statements
-  * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-  * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
-  * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
 
+ * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+ * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-clear-cache.md b/docs/sql-ref-syntax-aux-cache-clear-cache.md
index 15ba3c787c177..47889691148b7 100644
--- a/docs/sql-ref-syntax-aux-cache-clear-cache.md
+++ b/docs/sql-ref-syntax-aux-cache-clear-cache.md
@@ -20,21 +20,24 @@ license: |
 ---
 
 ### Description
+
 `CLEAR CACHE` removes the entries and associated data from the in-memory and/or on-disk cache for all cached tables and views.
 
 ### Syntax
+
 {% highlight sql %}
 CLEAR CACHE
 {% endhighlight %}
 
 ### Examples
+
 {% highlight sql %}
 CLEAR CACHE;
 {% endhighlight %}
 
 ### Related Statements
+
  * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
  * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
  * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
-
diff --git a/docs/sql-ref-syntax-aux-cache-refresh.md b/docs/sql-ref-syntax-aux-cache-refresh.md
index 4c56893aeca98..25f7ede1d324e 100644
--- a/docs/sql-ref-syntax-aux-cache-refresh.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh.md
@@ -20,35 +20,39 @@ license: |
 ---
 
 ### Description
+
 `REFRESH` is used to invalidate and refresh all the cached data (and the associated metadata) for
 all Datasets that contains the given data source path. Path matching is by prefix, i.e. "/" would
 invalidate everything that is cached. 
 
 ### Syntax
+
 {% highlight sql %}
 REFRESH resource_path
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
  <dt><code><em>resource_path</em></code></dt>
  <dd>The path of the resource that is to be refreshed.</dd>
 </dl>
 
 ### Examples
+
 {% highlight sql %}
- -- The Path is resolved using the datasource's File Index.
+-- The Path is resolved using the datasource's File Index.
  
 CREATE TABLE test(ID INT) using parquet;
 INSERT INTO test SELECT 1000;
 CACHE TABLE test;
 INSERT INTO test SELECT 100;
 REFRESH "hdfs://path/to/table";
- 
 {% endhighlight %}
 
 ### Related Statements
-- [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
-- [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-- [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+
+ * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+ * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+ * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index 7e4b8fbc35aa8..95fd91c3c4807 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -20,15 +20,18 @@ license: |
 ---
 
 ### Description
+
 `UNCACHE TABLE` removes the entries and associated data from the in-memory and/or on-disk cache for a given table or view. The
 underlying entries should already have been brought to cache by previous `CACHE TABLE` operation. `UNCACHE TABLE` on a non-existent table throws an exception if `IF EXISTS` is not specified.
 
 ### Syntax
+
 {% highlight sql %}
 UNCACHE TABLE [ IF EXISTS ] table_identifier
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -41,11 +44,13 @@ UNCACHE TABLE [ IF EXISTS ] table_identifier
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 UNCACHE TABLE t1;
 {% endhighlight %}
 
 ### Related Statements
+
  * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
  * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache.md b/docs/sql-ref-syntax-aux-cache.md
index 1a48fceeb1f6b..418b8cc3403b5 100644
--- a/docs/sql-ref-syntax-aux-cache.md
+++ b/docs/sql-ref-syntax-aux-cache.md
@@ -19,8 +19,8 @@ license: |
   limitations under the License.
 ---
 
-* [CACHE TABLE statement](sql-ref-syntax-aux-cache-cache-table.html)
-* [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
-* [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
-* [REFRESH TABLE statement](sql-ref-syntax-aux-refresh-table.html)
-* [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
\ No newline at end of file
+ * [CACHE TABLE statement](sql-ref-syntax-aux-cache-cache-table.html)
+ * [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
+ * [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [REFRESH TABLE statement](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
index 5ebc7b97ef64f..e7e6dda4e25ee 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
@@ -20,19 +20,22 @@ license: |
 ---
 
 ### Description
+
 Reset any runtime configurations specific to the current session which were set via the [SET](sql-ref-syntax-aux-conf-mgmt-set.html) command to their default values.
 
 ### Syntax
+
 {% highlight sql %}
 RESET
 {% endhighlight %}
 
-
 ### Examples
+
 {% highlight sql %}
 -- Reset any runtime configurations specific to the current session which were set via the SET command to their default values.
 RESET;
 {% endhighlight %}
 
 ### Related Statements
-- [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+
+ * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-set.md b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
index f05dde3f567ee..2ca51307c3aae 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-set.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
@@ -20,9 +20,11 @@ license: |
 ---
 
 ### Description
+
 The SET command sets a property, returns the value of an existing property or returns all SQLConf properties with value and meaning.
 
 ### Syntax
+
 {% highlight sql %}
 SET
 SET [ -v ]
@@ -30,6 +32,7 @@ SET property_key[ = property_value ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>-v</em></code></dt>
   <dd>Outputs the key, value and meaning of existing SQLConf properties.</dd>
@@ -46,9 +49,10 @@ SET property_key[ = property_value ]
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Set a property.
-SET  spark.sql.variable.substitute=false;
+SET spark.sql.variable.substitute=false;
 
 -- List all SQLConf properties with value and meaning.
 SET -v;
@@ -57,13 +61,14 @@ SET -v;
 SET;
 
 -- List the value of specified property key.
-SET  spark.sql.variable.substitute;
-    +--------------------------------+--------+
-    |              key               | value  |
-    +--------------------------------+--------+
-    | spark.sql.variable.substitute  | false  |
-    +--------------------------------+--------+
+SET spark.sql.variable.substitute;
+  +-----------------------------+-----+
+  |                          key|value|
+  +-----------------------------+-----+
+  |spark.sql.variable.substitute|false|
+  +-----------------------------+-----+
 {% endhighlight %}
 
 ### Related Statements
-- [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt.md b/docs/sql-ref-syntax-aux-conf-mgmt.md
index 7c5d9cc895c10..f5e48ef2fee30 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt.md
@@ -19,5 +19,5 @@ license: |
   limitations under the License.
 ---
 
-* [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
-* [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+ * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+ * [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref-syntax-aux-describe-database.md b/docs/sql-ref-syntax-aux-describe-database.md
index 05a64ab2060b4..2f7b1ce984d3e 100644
--- a/docs/sql-ref-syntax-aux-describe-database.md
+++ b/docs/sql-ref-syntax-aux-describe-database.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 ​
 `DESCRIBE DATABASE` statement returns the metadata of an existing database. The metadata information includes database
@@ -26,11 +27,13 @@ returns the basic metadata information along with the database properties. The `
 interchangeable.
 
 ### Syntax
+
 {% highlight sql %}
 { DESC | DESCRIBE } DATABASE [ EXTENDED ] db_name
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>db_name</em></code></dt>
   <dd>
@@ -40,6 +43,7 @@ interchangeable.
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- Create employees DATABASE
 CREATE DATABASE employees COMMENT 'For software companies';
@@ -49,11 +53,11 @@ CREATE DATABASE employees COMMENT 'For software companies';
 -- for the employees DATABASE.
 DESCRIBE DATABASE employees;
   +-------------------------+-----------------------------+
-  |database_description_item|database_description_value   |
+  |database_description_item|   database_description_value|
   +-------------------------+-----------------------------+
-  |Database Name            |employees                    |
-  |Description              |For software companies       |
-  |Location                 |file:/Users/Temp/employees.db|
+  |            Database Name|                    employees|
+  |              Description|       For software companies|
+  |                 Location|file:/Users/Temp/employees.db|
   +-------------------------+-----------------------------+
 
 -- Create employees DATABASE
@@ -65,12 +69,12 @@ ALTER DATABASE employees SET DBPROPERTIES ('Create-by' = 'Kevin', 'Create-date'
 -- Describe employees DATABASE with EXTENDED option to return additional database properties
 DESCRIBE DATABASE EXTENDED employees;
   +-------------------------+---------------------------------------------+
-  |database_description_item|database_description_value                   |
+  |database_description_item|                   database_description_value|
   +-------------------------+---------------------------------------------+
-  |Database Name            |employees                                    |
-  |Description              |For software companies                       |
-  |Location                 |file:/Users/Temp/employees.db                |
-  |Properties               |((Create-by,kevin), (Create-date,09/01/2019))|
+  |            Database Name|                                    employees|
+  |              Description|                       For software companies|
+  |                 Location|                file:/Users/Temp/employees.db|
+  |               Properties|((Create-by,kevin), (Create-date,09/01/2019))|
   +-------------------------+---------------------------------------------+
 
 -- Create deployment SCHEMA
@@ -81,14 +85,14 @@ DESC DATABASE deployment;
   +-------------------------+------------------------------+
   |database_description_item|database_description_value    |
   +-------------------------+------------------------------+
-  |Database Name            |deployment                    |
-  |Description              |Deployment environment        |
-  |Location                 |file:/Users/Temp/deployment.db|
+  |            Database Name|                    deployment|
+  |              Description|        Deployment environment|
+  |                 Location|file:/Users/Temp/deployment.db|
   +-------------------------+------------------------------+
-
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-describe-function.md b/docs/sql-ref-syntax-aux-describe-function.md
index f3c9c625b97b8..a4ff76bddf782 100644
--- a/docs/sql-ref-syntax-aux-describe-function.md
+++ b/docs/sql-ref-syntax-aux-describe-function.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 `DESCRIBE FUNCTION` statement returns the basic metadata information of an
@@ -26,11 +27,13 @@ class and the usage details.  If the optional `EXTENDED` option is specified, th
 metadata information is returned along with the extended usage information.
 
 ### Syntax
+
 {% highlight sql %}
 { DESC | DESCRIBE } FUNCTION [ EXTENDED ] function_name
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>function_name</em></code></dt>
   <dd>
@@ -46,6 +49,7 @@ metadata information is returned along with the extended usage information.
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Describe a builtin scalar function.
 -- Returns function name, implementing class and usage
@@ -107,6 +111,7 @@ DESC FUNCTION EXTENDED explode
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-describe-query.md b/docs/sql-ref-syntax-aux-describe-query.md
index b07ebe78193d1..f64416adc556d 100644
--- a/docs/sql-ref-syntax-aux-describe-query.md
+++ b/docs/sql-ref-syntax-aux-describe-query.md
@@ -20,16 +20,19 @@ license: |
 ---
 
 ### Description
+
 The `DESCRIBE QUERY` statement is used to return the metadata of output
 of a query. A shorthand `DESC` may be used instead of `DESCRIBE` to
 describe the query output.
 
 ### Syntax
+
 {% highlight sql %}
 { DESC | DESCRIBE } [ QUERY ] input_statement
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>QUERY</em></code></dt>
   <dd>This clause is optional and may be omitted.</dd>
@@ -49,6 +52,7 @@ describe the query output.
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Create table `person`
 CREATE TABLE person (name STRING , age INT COMMENT 'Age column', address STRING);
@@ -56,19 +60,19 @@ CREATE TABLE person (name STRING , age INT COMMENT 'Age column', address STRING)
 -- Returns column metadata information for a simple select query
 DESCRIBE QUERY select age, sum(age) FROM person GROUP BY age;
   +--------+---------+----------+
-  |col_name|data_type|comment   |
+  |col_name|data_type|   comment|
   +--------+---------+----------+
-  |age     |int      |Age column|
-  |sum(age)|bigint   |null      |
+  |     age|      int|Age column|
+  |sum(age)|   bigint|      null|
   +--------+---------+----------+
 
 -- Returns column metadata information for common table expression (`CTE`).
 DESCRIBE QUERY WITH all_names_cte
-  AS (SELECT name from person) SELECT * FROM all_names_cte;
+    AS (SELECT name from person) SELECT * FROM all_names_cte;
   +--------+---------+-------+
   |col_name|data_type|comment|
   +--------+---------+-------+
-  |name    |string   |null   |
+  |    name|   string|   null|
   +--------+---------+-------+
 
 -- Returns column metadata information for a inline table.
@@ -76,32 +80,33 @@ DESC QUERY VALUES(100, 'John', 10000.20D) AS employee(id, name, salary);
   +--------+---------+-------+
   |col_name|data_type|comment|
   +--------+---------+-------+
-  |id      |int      |null   |
-  |name    |string   |null   |
-  |salary  |double   |null   |
+  |      id|      int|   null|
+  |    name|   string|   null|
+  |  salary|   double|   null|
   +--------+---------+-------+
 
 -- Returns column metadata information for `TABLE` statement.
 DESC QUERY TABLE person;
   +--------+---------+----------+
-  |col_name|data_type|comment   |
+  |col_name|data_type|   comment|
   +--------+---------+----------+
-  |name    |string   |null      |
-  |age     |int      |Age column|
-  |address |string   |null      |
+  |    name|   string|      null|
+  |     age|      int| Agecolumn|
+  | address|   string|      null|
   +--------+---------+----------+
 
 -- Returns column metadata information for a `FROM` statement.
 -- `QUERY` clause is optional and can be omitted.
 DESCRIBE FROM person SELECT age;
   +--------+---------+----------+
-  |col_name|data_type|comment   |
+  |col_name|data_type|   comment|
   +--------+---------+----------+
-  |age     |int      |Age column|
+  |     age|      int| Agecolumn|
   +--------+---------+----------+
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-describe-table.md b/docs/sql-ref-syntax-aux-describe-table.md
index 4e6aeb5b6f349..a8eee97b4dc1e 100644
--- a/docs/sql-ref-syntax-aux-describe-table.md
+++ b/docs/sql-ref-syntax-aux-describe-table.md
@@ -18,18 +18,22 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
+
 `DESCRIBE TABLE` statement returns the basic metadata information of a
 table. The metadata information includes column name, column type
 and column comment. Optionally a partition spec or column name may be specified
 to return the metadata pertaining to a partition or column respectively.
 
 ### Syntax
+
 {% highlight sql %}
 { DESC | DESCRIBE } [ TABLE ] [ format ] table_identifier [ partition_spec ] [ col_name ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>format</em></code></dt>
   <dd>
@@ -69,101 +73,105 @@ to return the metadata pertaining to a partition or column respectively.
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Creates a table `customer`. Assumes current database is `salesdb`.
 CREATE TABLE customer(
-    cust_id INT,
-    state VARCHAR(20),
-    name STRING COMMENT 'Short name'
-  )
-  USING parquet
-  PARTITION BY state;
-  ;
-
+        cust_id INT,
+        state VARCHAR(20),
+        name STRING COMMENT 'Short name'
+    )
+    USING parquet
+    PARTITIONED BY (state);
+    
+INSERT INTO customer PARTITION (state = 'AR') VALUES (100, 'Mike');
+    
 -- Returns basic metadata information for unqualified table `customer`
 DESCRIBE TABLE customer;
   +-----------------------+---------+----------+
-  |col_name               |data_type|comment   |
+  |               col_name|data_type|   comment|
   +-----------------------+---------+----------+
-  |cust_id                |int      |null      |
-  |name                   |string   |Short name|
-  |state                  |string   |null      |
+  |                cust_id|      int|      null|
+  |                   name|   string|Short name|
+  |                  state|   string|      null|
   |# Partition Information|         |          |
-  |# col_name             |data_type|comment   |
-  |state                  |string   |null      |
+  |             # col_name|data_type|   comment|
+  |                  state|   string|      null|
   +-----------------------+---------+----------+
 
 -- Returns basic metadata information for qualified table `customer`
 DESCRIBE TABLE salesdb.customer;
   +-----------------------+---------+----------+
-  |col_name               |data_type|comment   |
+  |               col_name|data_type|   comment|
   +-----------------------+---------+----------+
-  |cust_id                |int      |null      |
-  |name                   |string   |Short name|
-  |state                  |string   |null      |
+  |                cust_id|      int|      null|
+  |                   name|   string|Short name|
+  |                  state|   string|      null|
   |# Partition Information|         |          |
-  |# col_name             |data_type|comment   |
-  |state                  |string   |null      |
+  |             # col_name|data_type|   comment|
+  |                  state|   string|      null|
   +-----------------------+---------+----------+
 
 -- Returns additional metadata such as parent database, owner, access time etc.
 DESCRIBE TABLE EXTENDED customer;
   +----------------------------+------------------------------+----------+
-  |col_name                    |data_type                     |comment   |
+  |                    col_name|                     data_type|   comment|
   +----------------------------+------------------------------+----------+
-  |cust_id                     |int                           |null      |
-  |name                        |string                        |Short name|
-  |state                       |string                        |null      |
-  |# Partition Information     |                              |          |
-  |# col_name                  |data_type                     |comment   |
-  |state                       |string                        |null      |
+  |                     cust_id|                           int|      null|
+  |                        name|                        string|Short name|
+  |                       state|                        string|      null|
+  |     # Partition Information|                              |          |
+  |                  # col_name|                     data_type|   comment|
+  |                       state|                        string|      null|
   |                            |                              |          |
   |# Detailed Table Information|                              |          |
-  |Database                    |salesdb                       |          |
-  |Table                       |customer                      |          |
-  |Owner                       |<table owner>                 |          |
-  |Created Time                |Fri Aug 30 09:26:04 PDT 2019  |          |
-  |Last Access                 |Wed Dec 31 16:00:00 PST 1969  |          |
-  |Created By                  |<spark version>               |          |
-  |Type                        |MANAGED                       |          |
-  |Provider                    |parquet                       |          |
-  |Location                    |file:.../salesdb.db/customer  |          |
-  |Serde Library               |...serde.ParquetHiveSerDe     |          |
-  |InputFormat                 |...MapredParquetInputFormat   |          |
-  |OutputFormat                |...MapredParquetOutputFormat  |          |
+  |                    Database|                       default|          |
+  |                       Table|                      customer|          |
+  |                       Owner|                 <TABLE OWNER>|          |
+  |                Created Time|  Tue Apr 07 22:56:34 JST 2020|          |
+  |                 Last Access|                       UNKNOWN|          |
+  |                  Created By|               <SPARK VERSION>|          |
+  |                        Type|                       MANAGED|          |
+  |                    Provider|                       parquet|          |
+  |                    Location|file:/tmp/salesdb.db/custom...|          |
+  |               Serde Library|org.apache.hadoop.hive.ql.i...|          |
+  |                 InputFormat|org.apache.hadoop.hive.ql.i...|          |
+  |                OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+  |          Partition Provider|                       Catalog|          |
   +----------------------------+------------------------------+----------+
 
 -- Returns partition metadata such as partitioning column name, column type and comment.
-DESCRIBE TABLE customer PARTITION (state = 'AR');
-
-  +--------------------------------+-----------------------------------------+----------+
-  |col_name                        |data_type                                |comment   |
-  +--------------------------------+-----------------------------------------+----------+
-  |cust_id                         |int                                      |null      |
-  |name                            |string                                   |Short name|
-  |state                           |string                                   |null      |
-  |# Partition Information         |                                         |          |
-  |# col_name                      |data_type                                |comment   |
-  |state                           |string                                   |null      |
-  |                                |                                         |          |
-  |# Detailed Partition Information|                                         |          |
-  |Database                        |salesdb                                  |          |
-  |Table                           |customer                                 |          |
-  |Partition Values                |[state=AR]                               |          |
-  |Location                        |file:.../salesdb.db/customer/state=AR    |          |
-  |Serde Library                   |...serde.ParquetHiveSerDe                |          |
-  |InputFormat                     |...parquet.MapredParquetInputFormat      |          |
-  |OutputFormat                    |...parquet.MapredParquetOutputFormat     |          |
-  |Storage Properties              |[path=file:.../salesdb.db/customer,      |          |
-  |                                | serialization.format=1]                 |          |
-  |Partition Parameters            |{rawDataSize=-1, numFiles=1l,            |          |
-  |                                | transient_lastDdlTime=1567185245,       |          |
-  |                                | totalSize=688,                          |          |
-  |                                | COLUMN_STATS_ACCURATE=false, numRows=-1}|          |
-  |Created Time                    |Fri Aug 30 10:14:05 PDT 2019             |          |
-  |Last Access                     |Wed Dec 31 16:00:00 PST 1969             |          |
-  |Partition Statistics            |688 bytes                                |          |
-  +--------------------------------+-----------------------------------------+----------+
+DESCRIBE TABLE EXTENDED customer PARTITION (state = 'AR');
+  +------------------------------+------------------------------+----------+
+  |                      col_name|                     data_type|   comment|
+  +------------------------------+------------------------------+----------+
+  |                       cust_id|                           int|      null|
+  |                          name|                        string|Short name|
+  |                         state|                        string|      null|
+  |       # Partition Information|                              |          |
+  |                    # col_name|                     data_type|   comment|
+  |                         state|                        string|      null|
+  |                              |                              |          |
+  |# Detailed Partition Inform...|                              |          |
+  |                      Database|                       default|          |
+  |                         Table|                      customer|          |
+  |              Partition Values|                    [state=AR]|          |
+  |                      Location|file:/tmp/salesdb.db/custom...|          |
+  |                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
+  |                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
+  |                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+  |            Storage Properties|[serialization.format=1, pa...|          |
+  |          Partition Parameters|{transient_lastDdlTime=1586...|          |
+  |                  Created Time|  Tue Apr 07 23:05:43 JST 2020|          |
+  |                   Last Access|                       UNKNOWN|          |
+  |          Partition Statistics|                     659 bytes|          |
+  |                              |                              |          |
+  |         # Storage Information|                              |          |
+  |                      Location|file:/tmp/salesdb.db/custom...|          |
+  |                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
+  |                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
+  |                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+  +------------------------------+------------------------------+----------+
 
 -- Returns the metadata for `name` column.
 -- Optional `TABLE` clause is omitted and column is fully qualified.
@@ -171,13 +179,14 @@ DESCRIBE customer salesdb.customer.name;
   +---------+----------+
   |info_name|info_value|
   +---------+----------+
-  |col_name |name      |
-  |data_type|string    |
-  |comment  |Short name|
+  | col_name|      name|
+  |data_type|    string|
+  |  comment|Short name|
   +---------+----------+
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-describe.md b/docs/sql-ref-syntax-aux-describe.md
index 9f17746316480..723943f97aa07 100644
--- a/docs/sql-ref-syntax-aux-describe.md
+++ b/docs/sql-ref-syntax-aux-describe.md
@@ -19,7 +19,7 @@ license: |
   limitations under the License.
 ---
 
-* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-refresh-table.md
index b248ee67fa12a..165ca68309f4a 100644
--- a/docs/sql-ref-syntax-aux-refresh-table.md
+++ b/docs/sql-ref-syntax-aux-refresh-table.md
@@ -20,16 +20,19 @@ license: |
 ---
 
 ### Description
+
 `REFRESH TABLE` statement invalidates the cached entries, which include data
 and metadata of the given table or view. The invalidated cache is populated in
 lazy manner when the cached table or the query associated with it is executed again.
 
 ### Syntax
+
 {% highlight sql %}
 REFRESH [TABLE] table_identifier
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -42,6 +45,7 @@ REFRESH [TABLE] table_identifier
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- The cached entries of the table will be refreshed  
 -- The table is resolved from the current database as the table name is unqualified.
@@ -53,7 +57,8 @@ REFRESH TABLE tempDB.view1;
 {% endhighlight %}
 
 ### Related Statements
-- [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
-- [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-- [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+
+ * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+ * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+ * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
index 7e485cbafe709..0028884308890 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
@@ -20,20 +20,24 @@ license: |
 ---
 
 ### Description
+
 `ADD FILE` can be used to add a single file as well as a directory to the list of resources. The added resource can be listed using [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html).
 
 ### Syntax
+
 {% highlight sql %}
 ADD FILE resource_name
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
  <dt><code><em>resource_name</em></code></dt>
  <dd>The name of the file or directory to be added.</dd>
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 ADD FILE /tmp/test;
 ADD FILE "/path/to/file/abc.txt";
@@ -43,6 +47,7 @@ ADD FILE "/path/to/some/directory";
 {% endhighlight %}
 
 ### Related Statements
+
  * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
  * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
index db0a85013321d..c4020347c1be0 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
@@ -20,20 +20,24 @@ license: |
 ---
 
 ### Description
+
 `ADD JAR` adds a JAR file to the list of resources. The added JAR file can be listed using [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html).
 
 ### Syntax
+
 {% highlight sql %}
 ADD JAR file_name
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
  <dt><code><em>file_name</em></code></dt>
  <dd>The name of the JAR file to be added. It could be either on a local file system or a distributed file system.</dd>
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 ADD JAR /tmp/test.jar;
 ADD JAR "/path/to/some.jar";
@@ -42,6 +46,7 @@ ADD JAR "/path with space/abc.jar";
 {% endhighlight %}
 
 ### Related Statements
+
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
  * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
  * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
index c42bf7ae8dd41..eec98e1fbffb5 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
@@ -20,14 +20,17 @@ license: |
 ---
 
 ### Description
+
 `LIST FILE` lists the resources added by [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html).
 
 ### Syntax
+
 {% highlight sql %}
 LIST FILE
 {% endhighlight %}
 
 ### Examples
+
 {% highlight sql %}
 ADD FILE /tmp/test;
 ADD FILE /tmp/test_2;
@@ -42,6 +45,7 @@ file:/private/tmp/test
 {% endhighlight %}
 
 ### Related Statements
+
  * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
  * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
index 9d1739753099e..dca4252c90ef2 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
@@ -20,14 +20,17 @@ license: |
 ---
 
 ### Description
+
 `LIST JAR` lists the JARs added by [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html).
 
 ### Syntax
+
 {% highlight sql %}
 LIST JAR
 {% endhighlight %}
 
 ### Examples
+
 {% highlight sql %}
 ADD JAR /tmp/test.jar;
 ADD JAR /tmp/test_2.jar;
@@ -42,6 +45,7 @@ spark://192.168.1.112:62859/jars/test.jar
 {% endhighlight %}
 
 ### Related Statements
+
  * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
  * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
  * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt.md b/docs/sql-ref-syntax-aux-resource-mgmt.md
index 0885f56bdb7cf..50c12ef7c2beb 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt.md
@@ -19,7 +19,7 @@ license: |
   limitations under the License.
 ---
 
-* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
-* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
-* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
-* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+ * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+ * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+ * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+ * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
diff --git a/docs/sql-ref-syntax-aux-show-columns.md b/docs/sql-ref-syntax-aux-show-columns.md
index 0c8aba83a8403..8f73aac0e3a61 100644
--- a/docs/sql-ref-syntax-aux-show-columns.md
+++ b/docs/sql-ref-syntax-aux-show-columns.md
@@ -18,15 +18,19 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
+
 Return the list of columns in a table. If the table does not exist, an exception is thrown.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW COLUMNS table_identifier [ database ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -54,44 +58,47 @@ SHOW COLUMNS table_identifier [ database ]
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Create `customer` table in `salesdb` database;
 USE salesdb;
-CREATE TABLE customer(cust_cd INT,
-  name VARCHAR(100),
-  cust_addr STRING);
+CREATE TABLE customer(
+    cust_cd INT,
+    name VARCHAR(100),
+    cust_addr STRING);
 
 -- List the columns of `customer` table in current database.
 SHOW COLUMNS IN customer;
   +---------+
-  |col_name |
+  | col_name|
   +---------+
-  |cust_cd  |
-  |name     |
+  |  cust_cd|
+  |     name|
   |cust_addr|
   +---------+
 
 -- List the columns of `customer` table in `salesdb` database.
 SHOW COLUMNS IN salesdb.customer;
   +---------+
-  |col_name |
+  | col_name|
   +---------+
-  |cust_cd  |
-  |name     |
+  |  cust_cd|
+  |     name|
   |cust_addr|
   +---------+
 
 -- List the columns of `customer` table in `salesdb` database
 SHOW COLUMNS IN customer IN salesdb;
   +---------+
-  |col_name |
+  | col_name|
   +---------+
-  |cust_cd  |
-  |name     |
+  |  cust_cd|
+  |     name|
   |cust_addr|
   +---------+
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
+
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-create-table.md b/docs/sql-ref-syntax-aux-show-create-table.md
index 24aba602ab3cf..0a37c96bfc5ab 100644
--- a/docs/sql-ref-syntax-aux-show-create-table.md
+++ b/docs/sql-ref-syntax-aux-show-create-table.md
@@ -20,14 +20,17 @@ license: |
 ---
 
 ### Description
+
 `SHOW CREATE TABLE` returns the [CREATE TABLE statement](sql-ref-syntax-ddl-create-table.html) or [CREATE VIEW statement](sql-ref-syntax-ddl-create-view.html) that was used to create a given table or view. `SHOW CREATE TABLE` on a non-existent table or a temporary view throws an exception.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW CREATE TABLE table_identifier
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -40,31 +43,26 @@ SHOW CREATE TABLE table_identifier
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE test (c INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE
     TBLPROPERTIES ('prop1' = 'value1', 'prop2' = 'value2');
 
-show create table test;
-
--- the result of SHOW CREATE TABLE test
-CREATE TABLE `test`(`c` INT)
-ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-WITH SERDEPROPERTIES (
-  'field.delim' = ',',
-  'serialization.format' = ','
-)
-STORED AS
-  INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
-  OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
-TBLPROPERTIES (
-  'transient_lastDdlTime' = '1569350233',
-  'prop1' = 'value1',
-  'prop2' = 'value2'
-)
-
+SHOW CREATE TABLE test;
+  +----------------------------------------------------+
+  |                                      createtab_stmt|
+  +----------------------------------------------------+
+  |CREATE TABLE `default`.`test` (`c` INT)
+   USING text
+   TBLPROPERTIES (
+     'transient_lastDdlTime' = '1586269021',
+     'prop1' = 'value1',
+     'prop2' = 'value2')
+  +----------------------------------------------------+
 {% endhighlight %}
 
 ### Related Statements
+
  * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
  * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
diff --git a/docs/sql-ref-syntax-aux-show-databases.md b/docs/sql-ref-syntax-aux-show-databases.md
index 9d4be21daeabc..0ed34527bcbf3 100644
--- a/docs/sql-ref-syntax-aux-show-databases.md
+++ b/docs/sql-ref-syntax-aux-show-databases.md
@@ -20,17 +20,20 @@ license: |
 ---
 
 ### Description
+
 Lists the databases that match an optionally supplied string pattern. If no
 pattern is supplied then the command lists all the databases in the system.
 Please note that the usage of `SCHEMAS` and `DATABASES` are interchangeable
 and mean the same thing.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW { DATABASES | SCHEMAS } [ LIKE string_pattern ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>LIKE string_pattern</em></code></dt>
   <dd>
@@ -40,6 +43,7 @@ SHOW { DATABASES | SCHEMAS } [ LIKE string_pattern ]
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Create database. Assumes a database named `default` already exists in
 -- the system. 
@@ -55,6 +59,7 @@ SHOW DATABASES;
   | payments_db|
   |  payroll_db|
   +------------+
+  
 -- Lists databases with name starting with string pattern `pay`
 SHOW DATABASES LIKE 'pay*';
   +------------+
@@ -63,6 +68,7 @@ SHOW DATABASES LIKE 'pay*';
   | payments_db|
   |  payroll_db|
   +------------+
+  
 -- Lists all databases. Keywords SCHEMAS and DATABASES are interchangeable. 
 SHOW SCHEMAS;
   +------------+
@@ -73,7 +79,9 @@ SHOW SCHEMAS;
   |  payroll_db|
   +------------+
 {% endhighlight %}
+
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
diff --git a/docs/sql-ref-syntax-aux-show-functions.md b/docs/sql-ref-syntax-aux-show-functions.md
index d6f9df9896afe..da33d999f0b38 100644
--- a/docs/sql-ref-syntax-aux-show-functions.md
+++ b/docs/sql-ref-syntax-aux-show-functions.md
@@ -20,6 +20,7 @@ license: |
 ---
 
 ### Description
+
 Returns the list of functions after applying an optional regex pattern.
 Given number of functions supported by Spark is quite large, this statement
 in conjunction with [describe function](sql-ref-syntax-aux-describe-function.html)
@@ -27,11 +28,13 @@ may be used to quickly find the function and understand its usage. The `LIKE`
 clause is optional and supported only for compatibility with other systems.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW [ function_kind ] FUNCTIONS ( [ LIKE ] function_name | regex_pattern )
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>function_kind</em></code></dt>
   <dd>
@@ -66,6 +69,7 @@ SHOW [ function_kind ] FUNCTIONS ( [ LIKE ] function_name | regex_pattern )
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- List a system function `trim` by searching both user defined and system
 -- defined functions.
@@ -73,7 +77,7 @@ SHOW FUNCTIONS trim;
   +--------+
   |function|
   +--------+
-  |trim    |
+  |    trim|
   +--------+
 
 -- List a system function `concat` by searching system defined functions.
@@ -81,7 +85,7 @@ SHOW SYSTEM FUNCTIONS concat;
   +--------+
   |function|
   +--------+
-  |concat  |
+  |  concat|
   +--------+
 
 -- List a qualified function `max` from database `salesdb`. 
@@ -89,30 +93,31 @@ SHOW SYSTEM FUNCTIONS salesdb.max;
   +--------+
   |function|
   +--------+
-  |max     |
+  |     max|
   +--------+
 
 -- List all functions starting with `t`
 SHOW FUNCTIONS LIKE 't*';
   +-----------------+
-  |function         |
+  |         function|
   +-----------------+
-  |tan              |
-  |tanh             |
-  |timestamp        |
-  |tinyint          |
-  |to_csv           |
-  |to_date          |
-  |to_json          |
-  |to_timestamp     |
+  |              tan|
+  |             tanh|
+  |        timestamp|
+  |          tinyint|
+  |           to_csv|
+  |          to_date|
+  |          to_json|
+  |     to_timestamp|
   |to_unix_timestamp|
-  |to_utc_timestamp |
-  |transform        |
-  |transform_keys   |
-  |transform_values |
-  |translate        |
-  |trim             |
-  |trunc            |
+  | to_utc_timestamp|
+  |        transform|
+  |   transform_keys|
+  | transform_values|
+  |        translate|
+  |             trim|
+  |            trunc|
+  |           typeof|
   +-----------------+
 
 -- List all functions starting with `yea` or `windo`
@@ -120,8 +125,8 @@ SHOW FUNCTIONS LIKE 'yea*|windo*';
   +--------+
   |function|
   +--------+
-  |window  |
-  |year    |
+  |  window|
+  |    year|
   +--------+
 
 -- Use normal regex pattern to list function names that has 4 characters
@@ -130,10 +135,11 @@ SHOW FUNCTIONS LIKE 't[a-z][a-z][a-z]';
   +--------+
   |function|
   +--------+
-  |tanh    |
-  |trim    |
+  |    tanh|
+  |    trim|
   +--------+
 {% endhighlight %}
 
-### Related statements
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+### Related Statements
+
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-show-partitions.md b/docs/sql-ref-syntax-aux-show-partitions.md
index 6c8401e8e4718..31b881ea8f141 100644
--- a/docs/sql-ref-syntax-aux-show-partitions.md
+++ b/docs/sql-ref-syntax-aux-show-partitions.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `SHOW PARTITIONS` statement is used to list partitions of a table. An optional
@@ -25,11 +26,13 @@ partition spec may be specified to return the partitions matching the supplied
 partition spec.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW PARTITIONS table_identifier [ partition_spec ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -53,6 +56,7 @@ SHOW PARTITIONS table_identifier [ partition_spec ]
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- create a partitioned table and insert a few rows.
 USE salesdb;
@@ -64,27 +68,27 @@ INSERT INTO customer PARTITION (state = 'AZ', city = 'Peoria') VALUES (300, 'Dan
 -- Lists all partitions for table `customer`
 SHOW PARTITIONS customer;
   +----------------------+
-  |partition             |
+  |             partition|
   +----------------------+
-  |state=AZ/city=Peoria  |
-  |state=CA/city=Fremont |
+  |  state=AZ/city=Peoria|
+  | state=CA/city=Fremont|
   |state=CA/city=San Jose|
   +----------------------+
 
 -- Lists all partitions for the qualified table `customer`
 SHOW PARTITIONS salesdb.customer;
   +----------------------+
-  |partition             |
+  |             partition|
   +----------------------+
-  |state=AZ/city=Peoria  |
-  |state=CA/city=Fremont |
+  |  state=AZ/city=Peoria|
+  | state=CA/city=Fremont|
   |state=CA/city=San Jose|
   +----------------------+
 
 -- Specify a full partition spec to list specific partition
 SHOW PARTITIONS customer PARTITION (state = 'CA', city = 'Fremont');
   +---------------------+
-  |partition            |
+  |            partition|
   +---------------------+
   |state=CA/city=Fremont|
   +---------------------+
@@ -92,23 +96,24 @@ SHOW PARTITIONS customer PARTITION (state = 'CA', city = 'Fremont');
 -- Specify a partial partition spec to list the specific partitions
 SHOW PARTITIONS customer PARTITION (state = 'CA');
   +----------------------+
-  |partition             |
+  |             partition|
   +----------------------+
-  |state=CA/city=Fremont |
+  | state=CA/city=Fremont|
   |state=CA/city=San Jose|
   +----------------------+
 
 -- Specify a partial spec to list specific partition
 SHOW PARTITIONS customer PARTITION (city =  'San Jose');
   +----------------------+
-  |partition             |
+  |             partition|
   +----------------------+
   |state=CA/city=San Jose|
   +----------------------+
 {% endhighlight %}
 
-### Related statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [INSERT STATEMENT](sql-ref-syntax-dml-insert.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
+### Related Statements
+
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [INSERT STATEMENT](sql-ref-syntax-dml-insert.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index 49696585ba581..1aa44d3ab30ea 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 `SHOW TABLE EXTENDED` will show information for all tables matching the given regular expression.
@@ -30,12 +31,14 @@ information such as `Partition Parameters` and `Partition Statistics`. Note that
 cannot be used with a partition specification.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE 'identifier_with_wildcards'
     [ partition_spec ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
  <dt><code><em>IN|FROM database_name</em></code></dt>
   <dd>
@@ -61,123 +64,123 @@ SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE 'identifier_with_wildcards'
       </code>
   </dd>
 </dl>
+
 ### Examples
+
 {% highlight sql %}
 -- Assumes `employee` table created with partitioned by column `grade`
--- +-------+--------+--+
--- | name  | grade  |
--- +-------+--------+--+
--- | sam   | 1      |
--- | suj   | 2      |
--- +-------+--------+--+
+CREATE TABLE employee(name STRING, grade INT) PARTITIONED BY (grade);
+INSERT INTO employee PARTITION (grade = 1) VALUES ('sam');
+INSERT INTO employee PARTITION (grade = 2) VALUES ('suj');
 
  -- Show the details of the table
-SHOW TABLE EXTENDED  LIKE `employee`;
-+--------+---------+-----------+---------------------------------------------------------------
-|database|tableName|isTemporary|                         information
-+--------+---------+-----------+---------------------------------------------------------------
-|default |employee |false      |Database: default
-                                Table: employee
-                                Owner: root
-                                Created Time: Fri Aug 30 15:10:21 IST 2019
-                                Last Access: Thu Jan 01 05:30:00 IST 1970
-                                Created By: Spark 3.0.0-SNAPSHOT
-                                Type: MANAGED
-                                Provider: hive
-                                Table Properties: [transient_lastDdlTime=1567158021]
-                                Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                Serde Library: org.apache.hadoop.hive.serde2.lazy   
-                                .LazySimpleSerDe
-                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                OutputFormat: org.apache.hadoop.hive.ql.io
-                                .HiveIgnoreKeyTextOutputFormat
-                                Storage Properties: [serialization.format=1]
-                                Partition Provider: Catalog
-                                Partition Columns: [`grade`]
-                                Schema: root
-                                 |-- name: string (nullable = true)
-                                 |-- grade: integer (nullable = true)
-                                                                                                          
-+--------+---------+-----------+---------------------------------------------------------------
+SHOW TABLE EXTENDED LIKE 'employee';
+  +--------+---------+-----------+--------------------------------------------------------------+
+  |database|tableName|isTemporary|                         information                          |
+  +--------+---------+-----------+--------------------------------------------------------------+
+  |default |employee |false      |Database: default
+                                  Table: employee
+                                  Owner: root
+                                  Created Time: Fri Aug 30 15:10:21 IST 2019
+                                  Last Access: Thu Jan 01 05:30:00 IST 1970
+                                  Created By: Spark 3.0.0-SNAPSHOT
+                                  Type: MANAGED
+                                  Provider: hive
+                                  Table Properties: [transient_lastDdlTime=1567158021]
+                                  Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                  Serde Library: org.apache.hadoop.hive.serde2.lazy   
+                                  .LazySimpleSerDe
+                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                  OutputFormat: org.apache.hadoop.hive.ql.io
+                                  .HiveIgnoreKeyTextOutputFormat
+                                  Storage Properties: [serialization.format=1]
+                                  Partition Provider: Catalog
+                                  Partition Columns: [`grade`]
+                                  Schema: root
+                                   |-- name: string (nullable = true)
+                                   |-- grade: integer (nullable = true)
+                                                                                                            
+  +--------+---------+-----------+--------------------------------------------------------------+
 
 -- showing the multiple table details with pattern matching
 SHOW TABLE EXTENDED  LIKE `employe*`;
-+--------+---------+-----------+---------------------------------------------------------------
-|database|tableName|isTemporary|                         information
-+--------+---------+-----------+---------------------------------------------------------------
-|default |employee |false      |Database: default
-                                 Table: employee
-                                 Owner: root
-                                 Created Time: Fri Aug 30 15:10:21 IST 2019
-                                 Last Access: Thu Jan 01 05:30:00 IST 1970
-                                 Created By: Spark 3.0.0-SNAPSHOT
-                                 Type: MANAGED
-                                 Provider: hive
-                                 Table Properties: [transient_lastDdlTime=1567158021]
-                                 Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                 Serde Library: org.apache.hadoop.hive.serde2.lazy
-                                 .LazySimpleSerDe
-                                 InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                 OutputFormat: org.apache.hadoop.hive.ql.io
-                                 .HiveIgnoreKeyTextOutputFormat
-                                 Storage Properties: [serialization.format=1]
-                                 Partition Provider: Catalog
-                                 Partition Columns: [`grade`]
-                                 Schema: root
-                                  |-- name: string (nullable = true)
-                                  |-- grade: integer (nullable = true)
-
-|default |employee1|false      |Database: default
-                                Table: employee1
-                                Owner: root
-                                Created Time: Fri Aug 30 15:22:33 IST 2019
-                                Last Access: Thu Jan 01 05:30:00 IST 1970
-                                Created By: Spark 3.0.0-SNAPSHOT
-                                Type: MANAGED
-                                Provider: hive
-                                Table Properties: [transient_lastDdlTime=1567158753]
-                                Location: file:/opt/spark1/spark/spark-warehouse/employee1
-                                Serde Library: org.apache.hadoop.hive.serde2.lazy
-                                .LazySimpleSerDe
-                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                OutputFormat: org.apache.hadoop.hive.ql.io
-                                .HiveIgnoreKeyTextOutputFormat
-                                Storage Properties: [serialization.format=1]
-                                Partition Provider: Catalog
-                                Schema: root
-                                |-- name: string (nullable = true)
-                                                                                                             
-+--------+---------+----------+----------------------------------------------------------------
+  +--------+---------+-----------+--------------------------------------------------------------+
+  |database|tableName|isTemporary|                         information                          |
+  +--------+---------+-----------+--------------------------------------------------------------+
+  |default |employee |false      |Database: default
+                                  Table: employee
+                                  Owner: root
+                                  Created Time: Fri Aug 30 15:10:21 IST 2019
+                                  Last Access: Thu Jan 01 05:30:00 IST 1970
+                                  Created By: Spark 3.0.0-SNAPSHOT
+                                  Type: MANAGED
+                                  Provider: hive
+                                  Table Properties: [transient_lastDdlTime=1567158021]
+                                  Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                  Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                  .LazySimpleSerDe
+                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                  OutputFormat: org.apache.hadoop.hive.ql.io
+                                  .HiveIgnoreKeyTextOutputFormat
+                                  Storage Properties: [serialization.format=1]
+                                  Partition Provider: Catalog
+                                  Partition Columns: [`grade`]
+                                  Schema: root
+                                   |-- name: string (nullable = true)
+                                   |-- grade: integer (nullable = true)
+  
+  |default |employee1|false      |Database: default
+                                  Table: employee1
+                                  Owner: root
+                                  Created Time: Fri Aug 30 15:22:33 IST 2019
+                                  Last Access: Thu Jan 01 05:30:00 IST 1970
+                                  Created By: Spark 3.0.0-SNAPSHOT
+                                  Type: MANAGED
+                                  Provider: hive
+                                  Table Properties: [transient_lastDdlTime=1567158753]
+                                  Location: file:/opt/spark1/spark/spark-warehouse/employee1
+                                  Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                  .LazySimpleSerDe
+                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                  OutputFormat: org.apache.hadoop.hive.ql.io
+                                  .HiveIgnoreKeyTextOutputFormat
+                                  Storage Properties: [serialization.format=1]
+                                  Partition Provider: Catalog
+                                  Schema: root
+                                   |-- name: string (nullable = true)
+                                                                                                               
+  +--------+---------+----------+---------------------------------------------------------------+
   
 -- show partition file system details
 SHOW TABLE EXTENDED  IN `default` LIKE `employee` PARTITION (`grade=1`);
-+--------+---------+-----------+---------------------------------------------------------------
-|database|tableName|isTemporary|                         information                           
-+--------+---------+-----------+---------------------------------------------------------------
-|default |employee |false      | Partition Values: [grade=1]
-                               Location: file:/opt/spark1/spark/spark-warehouse/employee
-                               /grade=1
-                               Serde Library: org.apache.hadoop.hive.serde2.lazy
-                               .LazySimpleSerDe
-                               InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                               OutputFormat: org.apache.hadoop.hive.ql.io
-                               .HiveIgnoreKeyTextOutputFormat
-                               Storage Properties: [serialization.format=1]
-                               Partition Parameters: {rawDataSize=-1, numFiles=1, 
-                               transient_lastDdlTime=1567158221, totalSize=4, 
-                               COLUMN_STATS_ACCURATE=false, numRows=-1}
-                               Created Time: Fri Aug 30 15:13:41 IST 2019
-                               Last Access: Thu Jan 01 05:30:00 IST 1970
-                               Partition Statistics: 4 bytes
-                                                                                                                                                                             |
-+--------+---------+-----------+---------------------------------------------------------------
+  +--------+---------+-----------+--------------------------------------------------------------+
+  |database|tableName|isTemporary|                         information                          | 
+  +--------+---------+-----------+--------------------------------------------------------------+
+  |default |employee |false      |Partition Values: [grade=1]
+                                  Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                  /grade=1
+                                  Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                  .LazySimpleSerDe
+                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                  OutputFormat: org.apache.hadoop.hive.ql.io
+                                  .HiveIgnoreKeyTextOutputFormat
+                                  Storage Properties: [serialization.format=1]
+                                  Partition Parameters: {rawDataSize=-1, numFiles=1, 
+                                  transient_lastDdlTime=1567158221, totalSize=4, 
+                                  COLUMN_STATS_ACCURATE=false, numRows=-1}
+                                  Created Time: Fri Aug 30 15:13:41 IST 2019
+                                  Last Access: Thu Jan 01 05:30:00 IST 1970
+                                  Partition Statistics: 4 bytes
+                                                                                                                                                                            |
+  +--------+---------+-----------+--------------------------------------------------------------+
 
 -- show partition file system details with regex fails as shown below
 SHOW TABLE EXTENDED  IN `default` LIKE `empl*` PARTITION (`grade=1`);
-Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
- Table or view 'emplo*' not found in database 'default'; (state=,code=0)
-
+  Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
+   Table or view 'emplo*' not found in database 'default'; (state=,code=0)
 {% endhighlight %}
+
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index 311401cf580be..0b7062ec8eff7 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `SHOW TABLES` statement returns all the tables for an optionally specified database.
@@ -26,11 +27,13 @@ pattern. If no database is specified then the tables are returned from the
 current database.
 
 ### Syntax
+
 {% highlight sql %}
 SHOW TABLES [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>{ FROM | IN } database_name</em></code></dt>
   <dd>
@@ -50,58 +53,59 @@ SHOW TABLES [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- List all tables in default database
 SHOW TABLES;
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  | default   | suj        | false        |
-  +-----------+------------+--------------+--+
+  +--------+---------+-----------+
+  |database|tableName|isTemporary|
+  +--------+---------+-----------+
+  | default|      sam|      false|
+  | default|     sam1|      false|
+  | default|      suj|      false|
+  +--------+---------+-----------+
 
 -- List all tables from userdb database 
 SHOW TABLES FROM userdb;
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | userdb    | user1      | false        |
-  | userdb    | user2      | false        |
-  +-----------+------------+--------------+--+
+  +--------+---------+-----------+
+  |database|tableName|isTemporary|
+  +--------+---------+-----------+
+  |  userdb|    user1|      false|
+  |  userdb|    user2|      false|
+  +--------+---------+-----------+
 
 -- List all tables in userdb database
 SHOW TABLES IN userdb;
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | userdb    | user1      | false        |
-  | userdb    | user2      | false        |
-  +-----------+------------+--------------+--+
+  +--------+---------+-----------+
+  |database|tableName|isTemporary|
+  +--------+---------+-----------+
+  |  userdb|    user1|      false|
+  |  userdb|    user2|      false|
+  +--------+---------+-----------+
 
 -- List all tables from default database matching the pattern `sam*`
 SHOW TABLES FROM default LIKE 'sam*';
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  +-----------+------------+--------------+--+
+  +--------+---------+-----------+
+  |database|tableName|isTemporary|
+  +--------+---------+-----------+
+  | default|      sam|      false|
+  | default|     sam1|      false|
+  +--------+---------+-----------+
   
 -- List all tables matching the pattern `sam*|suj`
 SHOW TABLES LIKE 'sam*|suj';
-  +-----------+------------+--------------+--+
-  | database  | tableName  | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  | default   | suj        | false        |
-  +-----------+------------+--------------+--+
-
+  +--------+---------+-----------+
+  |database|tableName|isTemporary|
+  +--------+---------+-----------+
+  | default|      sam|      false|
+  | default|     sam1|      false|
+  | default|      suj|      false|
+  +--------+---------+-----------+
 {% endhighlight %}
 
-### Related statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+### Related Statements
+
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show-tblproperties.md b/docs/sql-ref-syntax-aux-show-tblproperties.md
index 451fd4abc85bb..26e57ef0aba24 100644
--- a/docs/sql-ref-syntax-aux-show-tblproperties.md
+++ b/docs/sql-ref-syntax-aux-show-tblproperties.md
@@ -20,16 +20,19 @@ license: |
 ---
 
 ### Description
+
 This statement returns the value of a table property given an optional value for
 a property key. If no key is specified then all the properties are returned. 
 
 ### Syntax
+
 {% highlight sql %}
 SHOW TBLPROPERTIES table_identifier 
    [ ( unquoted_property_key | property_key_as_string_literal ) ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -64,19 +67,20 @@ SHOW TBLPROPERTIES table_identifier
     properties are: `numFiles`, `numPartitions`, `numRows`.
 
 ### Examples
+
 {% highlight sql %}
 -- create a table `customer` in database `salesdb`
 USE salesdb;
 CREATE TABLE customer(cust_code INT, name VARCHAR(100), cust_addr STRING)
-  TBLPROPERTIES ('created.by.user' = 'John', 'created.date' = '01-01-2001');
+    TBLPROPERTIES ('created.by.user' = 'John', 'created.date' = '01-01-2001');
 
 -- show all the user specified properties for table `customer`
 SHOW TBLPROPERTIES customer;
   +---------------------+----------+
-  |key                  |value     |
+  |                  key|     value|
   +---------------------+----------+
-  |created.by.user      |John      |
-  |created.date         |01-01-2001|
+  |      created.by.user|      John|
+  |         created.date|01-01-2001|
   |transient_lastDdlTime|1567554931|
   +---------------------+----------+
 
@@ -84,10 +88,10 @@ SHOW TBLPROPERTIES customer;
 -- in database `salesdb`
 SHOW TBLPROPERTIES salesdb.customer;
   +---------------------+----------+
-  |key                  |value     |
+  |                  key|     value|
   +---------------------+----------+
-  |created.by.user      |John      |
-  |created.date         |01-01-2001|
+  |      created.by.user|      John|
+  |         created.date|01-01-2001|
   |transient_lastDdlTime|1567554931|
   +---------------------+----------+
 
@@ -96,20 +100,21 @@ SHOW TBLPROPERTIES customer (created.by.user);
   +-----+
   |value|
   +-----+
-  |John |
+  | John|
   +-----+
 
 -- show value for property `created.date`` specified as string literal
 SHOW TBLPROPERTIES customer ('created.date');
   +----------+
-  |value     |
+  |     value|
   +----------+
   |01-01-2001|
   +----------+
 {% endhighlight %}
 
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [ALTER TABLE SET TBLPROPERTIES](sql-ref-syntax-ddl-alter-table.html)
-- [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
-- [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [ALTER TABLE SET TBLPROPERTIES](sql-ref-syntax-ddl-alter-table.html)
+ * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+ * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
index a5e840d97bf2d..aec3716c2889f 100644
--- a/docs/sql-ref-syntax-aux-show-views.md
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `SHOW VIEWS` statement returns all the views for an optionally specified database.
diff --git a/docs/sql-ref-syntax-aux-show.md b/docs/sql-ref-syntax-aux-show.md
index dd56d467815c7..424fe71370897 100644
--- a/docs/sql-ref-syntax-aux-show.md
+++ b/docs/sql-ref-syntax-aux-show.md
@@ -18,12 +18,13 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
-* [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
-* [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
-* [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
-* [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
-* [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
-* [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
-* [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
-* [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
-* [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+
+ * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
+ * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+ * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+ * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+ * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+ * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
+ * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
+ * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
+ * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-aux.md b/docs/sql-ref-syntax-aux.md
index ba09d70b437a9..3cd758f076e99 100644
--- a/docs/sql-ref-syntax-aux.md
+++ b/docs/sql-ref-syntax-aux.md
@@ -21,9 +21,9 @@ license: |
 
 Besides the major SQL statements such as Data Definition Statements, Data Manipulation Statements and Data Retrieval Statements, Spark SQL also supports the following Auxiliary Statements:
 
-- [ANALYZE](sql-ref-syntax-aux-analyze.html)
-- [CACHE](sql-ref-syntax-aux-cache.html)
-- [DESCRIBE](sql-ref-syntax-aux-describe.html)
-- [SHOW](sql-ref-syntax-aux-show.html)
-- [CONFIGURATION MANAGEMENT](sql-ref-syntax-aux-conf-mgmt.html)
-- [RESOURCE MANAGEMENT](sql-ref-syntax-aux-resource-mgmt.html)
+ * [ANALYZE](sql-ref-syntax-aux-analyze.html)
+ * [CACHE](sql-ref-syntax-aux-cache.html)
+ * [DESCRIBE](sql-ref-syntax-aux-describe.html)
+ * [SHOW](sql-ref-syntax-aux-show.html)
+ * [CONFIGURATION MANAGEMENT](sql-ref-syntax-aux-conf-mgmt.html)
+ * [RESOURCE MANAGEMENT](sql-ref-syntax-aux-resource-mgmt.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-database.md b/docs/sql-ref-syntax-ddl-alter-database.md
index a32343674feb0..520aba35567e8 100644
--- a/docs/sql-ref-syntax-ddl-alter-database.md
+++ b/docs/sql-ref-syntax-ddl-alter-database.md
@@ -18,7 +18,9 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
+
 You can alter metadata associated with a database by setting `DBPROPERTIES`.  The specified property
 values override any existing value with the same property name. Please note that the usage of 
 `SCHEMA` and `DATABASE` are interchangeable and one can be used in place of the other. An error message
@@ -26,18 +28,21 @@ is issued if the database is not found in the system. This command is mostly use
 for a database and may be used for auditing purposes.
 
 ### Syntax
+
 {% highlight sql %}
 ALTER { DATABASE | SCHEMA } database_name
     SET DBPROPERTIES ( property_name = property_value, ... )
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>database_name</em></code></dt>
   <dd>Specifies the name of the database to be altered.</dd>
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Creates a database named `inventory`.
 CREATE DATABASE inventory;
@@ -47,16 +52,16 @@ ALTER DATABASE inventory SET DBPROPERTIES ('Edited-by' = 'John', 'Edit-date' = '
 
 -- Verify that properties are set.
 DESCRIBE DATABASE EXTENDED inventory;
-
-   +-------------------------+--------------------------------------------+
-   |database_description_item|database_description_value                  |
-   +-------------------------+--------------------------------------------+
-   |Database Name            |inventory                                   |
-   |Description              |                                            |
-   |Location                 |file:/temp/spark-warehouse/inventory.db     |
-   |Properties               |((Edit-date,01/01/2001), (Edited-by,John))  |
-   +-------------------------+--------------------------------------------+
+   +-------------------------+------------------------------------------+
+   |database_description_item|                database_description_value|
+   +-------------------------+------------------------------------------+
+   |            Database Name|                                 inventory|
+   |              Description|                                          |
+   |                 Location|   file:/temp/spark-warehouse/inventory.db|
+   |               Properties|((Edit-date,01/01/2001), (Edited-by,John))|
+   +-------------------------+------------------------------------------+
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index 2dd808b131ef2..edb081b7f45c0 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -20,12 +20,15 @@ license: |
 ---
 
 ### Description
+
 `ALTER TABLE` statement changes the schema or properties of a table.
 
 ### RENAME 
+
 `ALTER TABLE RENAME TO` statement changes the table name of an existing table in the database.
 
 #### Syntax
+
 {% highlight sql %}
 ALTER TABLE table_identifier RENAME TO table_identifier
 
@@ -33,6 +36,7 @@ ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
 {% endhighlight %}
 
 #### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -55,16 +59,18 @@ ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
   </dd>
 </dl>
 
-
 ### ADD COLUMNS
+
 `ALTER TABLE ADD COLUMNS` statement adds mentioned columns to an existing table.
 
 #### Syntax
+
 {% highlight sql %}
 ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , col_spec ... ] )
 {% endhighlight %}
 
 #### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -81,16 +87,18 @@ ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , col_spec ... ] )
   <dd>Specifies the columns to be added to be renamed.</dd>
 </dl>
 
-
 ### ALTER OR CHANGE COLUMN
+
 `ALTER TABLE ALTER COLUMN` or `ALTER TABLE CHANGE COLUMN` statement changes column's comment.
 
 #### Syntax
+
 {% highlight sql %}
 ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnAction
 {% endhighlight %}
 
 #### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -118,19 +126,21 @@ ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnA
    </dd>     
 </dl>
 
-
 ### ADD AND DROP PARTITION
 
 #### ADD PARTITION
+
 `ALTER TABLE ADD` statement adds partition to the partitioned table.
 
 ##### Syntax
+
 {% highlight sql %}
 ALTER TABLE table_identifier ADD [IF NOT EXISTS] 
     ( partition_spec [ partition_spec ... ] )
 {% endhighlight %}
      
 ##### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -154,14 +164,17 @@ ALTER TABLE table_identifier ADD [IF NOT EXISTS]
 </dl>     
 
 #### DROP PARTITION
+
 `ALTER TABLE DROP` statement drops the partition of the table.
 
 ##### Syntax
+
 {% highlight sql %}
 ALTER TABLE table_identifier DROP [ IF EXISTS ] partition_spec [PURGE]
 {% endhighlight %}
      
 ##### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -183,35 +196,35 @@ ALTER TABLE table_identifier DROP [ IF EXISTS ] partition_spec [PURGE]
       </code>
   </dd>
 </dl>     
-
      
 ### SET AND UNSET
 
 #### SET TABLE PROPERTIES
+
 `ALTER TABLE SET` command is used for setting the table properties. If a particular property was already set, 
 this overrides the old value with the new one.
 
 `ALTER TABLE UNSET` is used to drop the table property. 
 
 ##### Syntax
-{% highlight sql %}
 
---Set Table Properties 
+{% highlight sql %}
+-- Set Table Properties 
 ALTER TABLE table_identifier SET TBLPROPERTIES ( key1 = val1, key2 = val2, ... )
 
---Unset Table Properties
+-- Unset Table Properties
 ALTER TABLE table_identifier UNSET TBLPROPERTIES [ IF EXISTS ] ( key1, key2, ... )
-  
 {% endhighlight %}
 
 #### SET SERDE
+
 `ALTER TABLE SET` command is used for setting the SERDE or SERDE properties in Hive tables. If a particular property was already set,  
 this overrides the old value with the new one.
 
 ##### Syntax
-{% highlight sql %}
 
---Set SERDE Properties
+{% highlight sql %}
+-- Set SERDE Properties
 ALTER TABLE table_identifier [ partition_spec ]
     SET SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )
 
@@ -221,21 +234,22 @@ ALTER TABLE table_identifier [ partition_spec ] SET SERDE serde_class_name
 {% endhighlight %}
 
 #### SET LOCATION And SET FILE FORMAT
+
 `ALTER TABLE SET` command can also be used for changing the file location and file format for 
 existing tables. 
 
 ##### Syntax
-{% highlight sql %}
 
---Changing File Format
+{% highlight sql %}
+-- Changing File Format
 ALTER TABLE table_identifier [ partition_spec ] SET FILEFORMAT file_format
 
---Changing File Location
+-- Changing File Location
 ALTER TABLE table_identifier [ partition_spec ] SET LOCATION 'new_location'
-
 {% endhighlight %}
 
 #### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -263,205 +277,198 @@ ALTER TABLE table_identifier [ partition_spec ] SET LOCATION 'new_location'
   <dd>Specifies the SERDE properties to be set.</dd>
 </dl>
 
-
 ### Examples
-{% highlight sql %}
 
---RENAME table 
+{% highlight sql %}
+-- RENAME table 
 DESC student;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
+  +-----------------------+---------+-------+
+  |               col_name|data_type|comment|
+  +-----------------------+---------+-------+
+  |                   name|   string|   NULL|
+  |                 rollno|      int|   NULL|
+  |                    age|      int|   NULL|
+  |# Partition Information|         |       |
+  |             # col_name|data_type|comment|
+  |                    age|      int|   NULL|
+  +-----------------------+---------+-------+
 
 ALTER TABLE Student RENAME TO StudentInfo;
 
---After Renaming the table
-
+-- After Renaming the table
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
-
---RENAME partition
+  +-----------------------+---------+-------+
+  |               col_name|data_type|comment|
+  +-----------------------+---------+-------+
+  |                   name|   string|   NULL|
+  |                 rollno|      int|   NULL|
+  |                    age|      int|   NULL|
+  |# Partition Information|         |       |
+  |             # col_name|data_type|comment|
+  |                    age|      int|   NULL|
+  +-----------------------+---------+-------+
+
+-- RENAME partition
 
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=10     |
-| age=11     |
-| age=12     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=10|
+  |   age=11|
+  |   age=12|
+  +---------+
 
 ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');
 
---After renaming Partition
+-- After renaming Partition
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  +---------+
 
 -- Add new columns to a table
-
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+
+  +-----------------------+---------+-------+
+  |               col_name|data_type|comment|
+  +-----------------------+---------+-------+
+  |                   name|   string|   NULL|
+  |                 rollno|      int|   NULL|
+  |                    age|      int|   NULL|
+  |# Partition Information|         |       |
+  |             # col_name|data_type|comment|
+  |                    age|      int|   NULL|
+  +-----------------------+---------+-------+
 
 ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);
 
---After Adding New columns to the table
+-- After Adding New columns to the table
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| LastName                 | string     | NULL     |
-| DOB                      | timestamp  | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
+  +-----------------------+---------+-------+
+  |               col_name|data_type|comment|
+  +-----------------------+---------+-------+
+  |                   name|   string|   NULL|
+  |                 rollno|      int|   NULL|
+  |               LastName|   string|   NULL|
+  |                    DOB|timestamp|   NULL|
+  |                    age|      int|   NULL|
+  |# Partition Information|         |       |
+  |             # col_name|data_type|comment|
+  |                    age|      int|   NULL|
+  +-----------------------+---------+-------+
 
 -- Add a new partition to a table 
-
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  +---------+
 
 ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);
 
 -- After adding a new partition to the table
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-| age=18     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  |   age=18|
+  +---------+
 
 -- Drop a partition from the table 
-
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-| age=18     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  |   age=18|
+  +---------+
 
 ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);
 
 -- After dropping the partition of the table
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  +---------+
 
 -- Adding multiple partitions to the table
-
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  +---------+
 
 ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18) PARTITION (age=20);
 
 -- After adding multiple partitions to the table
 SHOW PARTITIONS StudentInfo;
-+------------+--+
-| partition  |
-+------------+--+
-| age=11     |
-| age=12     |
-| age=15     |
-| age=18     |
-| age=20     |
-+------------+--+
+  +---------+
+  |partition|
+  +---------+
+  |   age=11|
+  |   age=12|
+  |   age=15|
+  |   age=18|
+  |   age=20|
+  +---------+
 
 -- ALTER OR CHANGE COLUMNS
-
 DESC StudentInfo;
-+--------------------------+------------+----------+--+
-|         col_name         | data_type  | comment  |
-+--------------------------+------------+----------+--+
-| name                     | string     | NULL     |
-| rollno                   | int        | NULL     |
-| LastName                 | string     | NULL     |
-| DOB                      | timestamp  | NULL     |
-| age                      | int        | NULL     |
-| # Partition Information  |            |          |
-| # col_name               | data_type  | comment  |
-| age                      | int        | NULL     |
-+--------------------------+------------+----------+--+
+  +-----------------------+---------+-------+
+  |               col_name|data_type|comment|
+  +-----------------------+---------+-------+
+  |                   name|   string|   NULL|
+  |                 rollno|      int|   NULL|
+  |               LastName|   string|   NULL|
+  |                    DOB|timestamp|   NULL|
+  |                    age|      int|   NULL|
+  |# Partition Information|         |       |
+  |             # col_name|data_type|comment|
+  |                    age|      int|   NULL|
+  +-----------------------+---------+-------+
 
 ALTER TABLE StudentInfo ALTER COLUMN name COMMENT "new comment";
 
 --After ALTER or CHANGE COLUMNS
 DESC StudentInfo;
-+--------------------------+------------+------------+--+
-|         col_name         | data_type  | comment    |
-+--------------------------+------------+------------+--+
-| name                     | string     | new comment|
-| rollno                   | int        | NULL       |
-| LastName                 | string     | NULL       |
-| DOB                      | timestamp  | NULL       |
-| age                      | int        | NULL       |
-| # Partition Information  |            |            |
-| # col_name               | data_type  | comment    |
-| age                      | int        | NULL       |
-+--------------------------+------------+------------+--+
-
---Change the fileformat
+  +-----------------------+---------+-----------+
+  |               col_name|data_type|    comment|
+  +-----------------------+---------+-----------+
+  |                   name|   string|new comment|
+  |                 rollno|      int|       NULL|
+  |               LastName|   string|       NULL|
+  |                    DOB|timestamp|       NULL|
+  |                    age|      int|       NULL|
+  |# Partition Information|         |           |
+  |             # col_name|data_type|    comment|
+  |                    age|      int|       NULL|
+  +-----------------------+---------+-----------+
+
+-- Change the fileformat
 ALTER TABLE loc_orc SET fileformat orc;
 
 ALTER TABLE p1 partition (month=2, day=2) SET fileformat parquet;
 
---Change the file Location
+-- Change the file Location
 ALTER TABLE dbx.tab1 PARTITION (a='1', b='2') SET LOCATION '/path/to/part/ways'
 
 -- SET SERDE/ SERDE Properties
@@ -469,17 +476,14 @@ ALTER TABLE test_tab SET SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinar
 
 ALTER TABLE dbx.tab1 SET SERDE 'org.apache.hadoop' WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')
 
---SET TABLE PROPERTIES
+-- SET TABLE PROPERTIES
 ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('winner' = 'loser')
 
---DROP TABLE PROPERTIES
+-- DROP TABLE PROPERTIES
 ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner')
-
 {% endhighlight %}
 
-
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-
 
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index a29f2b4f632a1..8116c97cc2f41 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -113,6 +113,8 @@ and the `view_identifier` must exist.
 ALTER VIEW view_identifier AS select_statement
 {% endhighlight %}
 
+Note that `ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPERTIES` properties.
+
 #### Parameters
 <dl>
   <dt><code><em>view_identifier</em></code></dt>
@@ -139,98 +141,88 @@ ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;
 
 -- Verify that the new view is created.
 DESCRIBE TABLE EXTENDED tempdb1.v2;
-
-+----------------------------+----------+-------+
-|col_name                    |data_type |comment|
-+----------------------------+----------+-------+
-|c1                          |int       |null   |
-|c2                          |string    |null   |
-|                            |          |       |
-|# Detailed Table Information|          |       |
-|Database                    |tempdb1   |       |
-|Table                       |v2        |       |
-+----------------------------+----------+-------+
+  +----------------------------+----------+-------+
+  |                    col_name|data_type |comment|
+  +----------------------------+----------+-------+
+  |                          c1|       int|   null|
+  |                          c2|    string|   null|
+  |                            |          |       |
+  |# Detailed Table Information|          |       |
+  |                    Database|   tempdb1|       |
+  |                       Table|        v2|       |
+  +----------------------------+----------+-------+
 
 -- Before ALTER VIEW SET TBLPROPERTIES
 DESC TABLE EXTENDED tempdb1.v2;
-
-+----------------------------+----------+-------+
-|col_name                    |data_type |comment|
-+----------------------------+----------+-------+
-|c1                          |int       |null   |
-|c2                          |string    |null   |
-|                            |          |       |
-|# Detailed Table Information|          |       |
-|Database                    |tempdb1   |       |
-|Table                       |v2        |       |
-|Table Properties            |[....]    |       |
-+----------------------------+----------+-------+
+  +----------------------------+----------+-------+
+  |                    col_name| data_type|comment|
+  +----------------------------+----------+-------+
+  |                          c1|       int|   null|
+  |                          c2|    string|   null|
+  |                            |          |       |
+  |# Detailed Table Information|          |       |
+  |                    Database|   tempdb1|       |
+  |                       Table|        v2|       |
+  |            Table Properties|    [....]|       |
+  +----------------------------+----------+-------+
 
 -- Set properties in TBLPROPERTIES
 ALTER VIEW tempdb1.v2 SET TBLPROPERTIES ('created.by.user' = "John", 'created.date' = '01-01-2001' );
 
 -- Use `DESCRIBE TABLE EXTENDED tempdb1.v2` to verify
 DESC TABLE EXTENDED tempdb1.v2;
-
-+----------------------------+-----------------------------------------------------+-------+
-|col_name                    |data_type                                            |comment|
-+----------------------------+-----------------------------------------------------+-------+
-|c1                          |int                                                  |null   |
-|c2                          |string                                               |null   |
-|                            |                                                     |       |
-|# Detailed Table Information|                                                     |       |
-|Database                    |tempdb1                                              |       |
-|Table                       |v2                                                   |       |
-|Table Properties            |[created.by.user=John, created.date=01-01-2001, ....]|       |
-+----------------------------+-----------------------------------------------------+-------+
+  +----------------------------+-----------------------------------------------------+-------+
+  |                    col_name|                                            data_type|comment|
+  +----------------------------+-----------------------------------------------------+-------+
+  |                          c1|                                                  int|   null|
+  |                          c2|                                               string|   null|
+  |                            |                                                     |       |
+  |# Detailed Table Information|                                                     |       |
+  |                    Database|                                              tempdb1|       |
+  |                       Table|                                                   v2|       |
+  |            Table Properties|[created.by.user=John, created.date=01-01-2001, ....]|       |
+  +----------------------------+-----------------------------------------------------+-------+
 
 -- Remove the key `created.by.user` and `created.date` from `TBLPROPERTIES`
 ALTER VIEW tempdb1.v2 UNSET TBLPROPERTIES ('created.by.user', 'created.date');
 
 --Use `DESC TABLE EXTENDED tempdb1.v2` to verify the changes
 DESC TABLE EXTENDED tempdb1.v2;
-
-+----------------------------+----------+-------+
-|col_name                    |data_type |comment|
-+----------------------------+----------+-------+
-|c1                          |int       |null   |
-|c2                          |string    |null   |
-|                            |          |       |
-|# Detailed Table Information|          |       |
-|Database                    |tempdb1   |       |
-|Table                       |v2        |       |
-|Table Properties            |[....]    |       |
-+----------------------------+----------+-------+
+  +----------------------------+----------+-------+
+  |                    col_name| data_type|comment|
+  +----------------------------+----------+-------+
+  |                          c1|       int|   null|
+  |                          c2|    string|   null|
+  |                            |          |       |
+  |# Detailed Table Information|          |       |
+  |                    Database|   tempdb1|       |
+  |                       Table|        v2|       |
+  |            Table Properties|    [....]|       |
+  +----------------------------+----------+-------+
 
 -- Change the view definition
 ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;
 
 -- Use `DESC TABLE EXTENDED` to verify
 DESC TABLE EXTENDED tempdb1.v2;
-
-+----------------------------+---------------------------+-------+
-|col_name                    |data_type                  |comment|
-+----------------------------+---------------------------+-------+
-|c1                          |int                        |null   |
-|c2                          |string                     |null   |
-|                            |                           |       |
-|# Detailed Table Information|                           |       |
-|Database                    |tempdb1                    |       |
-|Table                       |v2                         |       |
-|Type                        |VIEW                       |       |
-|View Text                   |select * from tempdb1.v1   |       |
-|View Original Text          |select * from tempdb1.v1   |       |
-+----------------------------+---------------------------+-------+
+  +----------------------------+---------------------------+-------+
+  |                    col_name|                  data_type|comment|
+  +----------------------------+---------------------------+-------+
+  |                          c1|                        int|   null|
+  |                          c2|                     string|   null|
+  |                            |                           |       |
+  |# Detailed Table Information|                           |       |
+  |                    Database|                    tempdb1|       |
+  |                       Table|                         v2|       |
+  |                        Type|                       VIEW|       |
+  |                   View Text|   select * from tempdb1.v1|       |
+  |          View Original Text|   select * from tempdb1.v1|       |
+  +----------------------------+---------------------------+-------+
 {% endhighlight %}
 
 ### Related Statements
 
-- [describe-table](sql-ref-syntax-aux-describe-table.html)
-- [create-view](sql-ref-syntax-ddl-create-view.html)
-- [drop-view](sql-ref-syntax-ddl-drop-view.html)
-- [show-views](sql-ref-syntax-aux-show-views.html)
-
-#### Note:
-
-`ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPERTIES` properties
-
+ * [describe-table](sql-ref-syntax-aux-describe-table.html)
+ * [create-view](sql-ref-syntax-ddl-create-view.html)
+ * [drop-view](sql-ref-syntax-ddl-drop-view.html)
+ * [show-views](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-create-database.md b/docs/sql-ref-syntax-ddl-create-database.md
index 4d2211c650953..6f74acdb60bf7 100644
--- a/docs/sql-ref-syntax-ddl-create-database.md
+++ b/docs/sql-ref-syntax-ddl-create-database.md
@@ -20,17 +20,20 @@ license: |
 ---
 
 ### Description
+
 Creates a database with the specified name. If database with the same name already exists, an exception will be thrown.
 
 ### Syntax
+
 {% highlight sql %}
 CREATE { DATABASE | SCHEMA } [ IF NOT EXISTS ] database_name
-  [ COMMENT database_comment ]
-  [ LOCATION database_directory ]
-  [ WITH DBPROPERTIES ( property_name = property_value [ , ... ] ) ]
+    [ COMMENT database_comment ]
+    [ LOCATION database_directory ]
+    [ WITH DBPROPERTIES ( property_name = property_value [ , ... ] ) ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
     <dt><code><em>database_name</em></code></dt>
     <dd>Specifies the name of the database to be created.</dd>
@@ -49,6 +52,7 @@ CREATE { DATABASE | SCHEMA } [ IF NOT EXISTS ] database_name
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Create database `customer_db`. This throws exception if database with name customer_db
 -- already exists.
@@ -60,20 +64,21 @@ CREATE DATABASE IF NOT EXISTS customer_db;
 -- Create database `customer_db` only if database with same name doesn't exist with 
 -- `Comments`,`Specific Location` and `Database properties`.
 CREATE DATABASE IF NOT EXISTS customer_db COMMENT 'This is customer database' LOCATION '/user'
- WITH DBPROPERTIES (ID=001, Name='John');
+    WITH DBPROPERTIES (ID=001, Name='John');
 
 -- Verify that properties are set.
 DESCRIBE DATABASE EXTENDED customer_db;
-   +----------------------------+-----------------------------+
-   | database_description_item  | database_description_value  |
-   +----------------------------+-----------------------------+
-   | Database Name              | customer_db                 |
-   | Description                | This is customer database   |
-   | Location                   | hdfs://hacluster/user       |
-   | Properties                 | ((ID,001), (Name,John))     |
-   +----------------------------+-----------------------------+
+   +-------------------------+--------------------------+
+   |database_description_item|database_description_value|
+   +-------------------------+--------------------------+
+   |            Database Name|               customer_db|
+   |              Description| This is customer database|
+   |                 Location|     hdfs://hacluster/user|
+   |               Properties|   ((ID,001), (Name,John))|
+   +-------------------------+--------------------------+
 {% endhighlight %}
 
 ### Related Statements
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index 1f94bf6d25aa5..2bd26d18c7736 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -20,6 +20,7 @@ license: |
 ---
 
 ### Description
+
 The `CREATE FUNCTION` statement is used to create a temporary or permanent function
 in Spark. Temporary functions are scoped at a session level where as permanent
 functions are created in the persistent catalog and are made available to
@@ -31,12 +32,14 @@ aggregate functions using Scala, Python and Java APIs. Please refer to
 [aggregate functions](sql-getting-started#aggregations) for more information.
 
 ### Syntax
+
 {% highlight sql %}
 CREATE [ OR REPLACE ] [ TEMPORARY ] FUNCTION [ IF NOT EXISTS ]
     function_name AS class_name [ resource_locations ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>OR REPLACE</em></code></dt>
   <dd>
@@ -90,6 +93,7 @@ CREATE [ OR REPLACE ] [ TEMPORARY ] FUNCTION [ IF NOT EXISTS ]
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- 1. Create a simple UDF `SimpleUdf` that increments the supplied integral value by 10.
 --    import org.apache.hadoop.hive.ql.exec.UDF;
@@ -106,7 +110,7 @@ INSERT INTO test VALUES (1), (2);
 
 -- Create a permanent function called `simple_udf`. 
 CREATE FUNCTION simple_udf AS 'SimpleUdf'
-  USING JAR '/tmp/SimpleUdf.jar';
+    USING JAR '/tmp/SimpleUdf.jar';
 
 -- Verify that the function is in the registry.
 SHOW USER FUNCTIONS;
@@ -127,7 +131,7 @@ SELECT simple_udf(c1) AS function_return_value FROM t1;
 
 -- Created a temporary function.
 CREATE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf' 
-  USING JAR '/tmp/SimpleUdf.jar';
+    USING JAR '/tmp/SimpleUdf.jar';
 
 -- Verify that the newly created temporary function is in the registry.
 -- Please note that the temporary function does not have a qualified
@@ -152,20 +156,20 @@ SHOW USER FUNCTIONS;
 
 -- Replace the implementation of `simple_udf`
 CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
-  USING JAR '/tmp/SimpleUdfR.jar';
+    USING JAR '/tmp/SimpleUdfR.jar';
 
 -- Invoke the function. Every selected value should be incremented by 20.
 SELECT simple_udf(c1) AS function_return_value FROM t1;
-+---------------------+                                                         
-|function_return_value|
-+---------------------+
-|                   21|
-|                   22|
-+---------------------+
-
+  +---------------------+                                                         
+  |function_return_value|
+  +---------------------+
+  |                   21|
+  |                   22|
+  +---------------------+
 {% endhighlight %}
 
-### Related statements
-- [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+### Related Statements
+
+ * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index 532377d7fcec3..715b64c33baed 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -24,19 +24,20 @@ license: |
 The `CREATE TABLE` statement defines a new table using a Data Source. 
 
 ### Syntax
+
 {% highlight sql %}
 CREATE TABLE [ IF NOT EXISTS ] table_identifier
-  [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
-  [USING data_source]
-  [ OPTIONS ( key1=val1, key2=val2, ... ) ]
-  [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
-  [ CLUSTERED BY ( col_name3, col_name4, ... ) 
-    [ SORTED BY ( col_name [ ASC | DESC ], ... ) ] 
-    INTO num_buckets BUCKETS ]
-  [ LOCATION path ]
-  [ COMMENT table_comment ]
-  [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-  [ AS select_statement ]
+    [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
+    [USING data_source]
+    [ OPTIONS ( key1=val1, key2=val2, ... ) ]
+    [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
+    [ CLUSTERED BY ( col_name3, col_name4, ... ) 
+        [ SORTED BY ( col_name [ ASC | DESC ], ... ) ] 
+        INTO num_buckets BUCKETS ]
+    [ LOCATION path ]
+    [ COMMENT table_comment ]
+    [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
+    [ AS select_statement ]
 {% endhighlight %}
 
 Note that, the clauses between the USING clause and the AS SELECT clause can come in
@@ -95,6 +96,7 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
 </dl>
 
 ### Data Source Interaction
+
 A Data Source table acts like a pointer to the underlying data source. For example, you can create
 a table "foo" in Spark which points to a table "bar" in MySQL using JDBC Data Source. When you
 read/write table "foo", you actually read/write table "bar".
@@ -107,6 +109,7 @@ For CREATE TABLE AS SELECT, Spark will overwrite the underlying data source with
 input query, to make sure the table gets created contains exactly the same data as the input query.
 
 ### Examples
+
 {% highlight sql %}
 
 --Use data source
@@ -114,29 +117,29 @@ CREATE TABLE student (id INT, name STRING, age INT) USING CSV;
 
 --Use data from another table
 CREATE TABLE student_copy USING CSV
-  AS SELECT * FROM student;
+    AS SELECT * FROM student;
   
 --Omit the USING clause, which uses the default data source (parquet by default)
 CREATE TABLE student (id INT, name STRING, age INT);
 
 --Specify table comment and properties
 CREATE TABLE student (id INT, name STRING, age INT) USING CSV
-  COMMENT 'this is a comment'
-  TBLPROPERTIES ('foo'='bar');
+    COMMENT 'this is a comment'
+    TBLPROPERTIES ('foo'='bar');
 
 --Specify table comment and properties with different clauses order
 CREATE TABLE student (id INT, name STRING, age INT) USING CSV
-  TBLPROPERTIES ('foo'='bar')
-  COMMENT 'this is a comment';
+    TBLPROPERTIES ('foo'='bar')
+    COMMENT 'this is a comment';
 
 --Create partitioned and bucketed table
 CREATE TABLE student (id INT, name STRING, age INT)
-  USING CSV
-  PARTITIONED BY (age)
-  CLUSTERED BY (Id) INTO 4 buckets;
-
+    USING CSV
+    PARTITIONED BY (age)
+    CLUSTERED BY (Id) INTO 4 buckets;
 {% endhighlight %}
 
 ### Related Statements
-* [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
-* [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
+
+ * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
+ * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 0425bafd94398..06f353ad2f103 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -18,23 +18,24 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `CREATE TABLE` statement defines a new table using Hive format.
 
 ### Syntax
+
 {% highlight sql %}
 CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
-  [ ( col_name1[:] col_type1 [ COMMENT col_comment1 ], ... ) ]
-  [ COMMENT table_comment ]
-  [ PARTITIONED BY ( col_name2[:] col_type2 [ COMMENT col_comment2 ], ... ) 
-      | ( col_name1, col_name2, ... ) ]
-  [ ROW FORMAT row_format ]
-  [ STORED AS file_format ]
-  [ LOCATION path ]
-  [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-  [ AS select_statement ]
-
+    [ ( col_name1[:] col_type1 [ COMMENT col_comment1 ], ... ) ]
+    [ COMMENT table_comment ]
+    [ PARTITIONED BY ( col_name2[:] col_type2 [ COMMENT col_comment2 ], ... ) 
+        | ( col_name1, col_name2, ... ) ]
+    [ ROW FORMAT row_format ]
+    [ STORED AS file_format ]
+    [ LOCATION path ]
+    [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
+    [ AS select_statement ]
 {% endhighlight %}
 
 Note that, the clauses between the columns definition clause and the AS SELECT clause can come in
@@ -93,47 +94,45 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
   <dd>The table is populated using the data from the select statement.</dd>
 </dl>
 
-
 ### Examples
-{% highlight sql %}
 
+{% highlight sql %}
 --Use hive format
 CREATE TABLE student (id INT, name STRING, age INT) STORED AS ORC;
 
 --Use data from another table
 CREATE TABLE student_copy STORED AS ORC
-  AS SELECT * FROM student;
+    AS SELECT * FROM student;
 
 --Specify table comment and properties
 CREATE TABLE student (id INT, name STRING, age INT)
-  COMMENT 'this is a comment'
-  STORED AS ORC
-  TBLPROPERTIES ('foo'='bar');  
+    COMMENT 'this is a comment'
+    STORED AS ORC
+    TBLPROPERTIES ('foo'='bar');  
 
 --Specify table comment and properties with different clauses order
 CREATE TABLE student (id INT, name STRING, age INT)
-  STORED AS ORC
-  TBLPROPERTIES ('foo'='bar')
-  COMMENT 'this is a comment';
+    STORED AS ORC
+    TBLPROPERTIES ('foo'='bar')
+    COMMENT 'this is a comment';
 
 --Create partitioned table
 CREATE TABLE student (id INT, name STRING)
-  PARTITIONED BY (age INT)
-  STORED AS ORC;
+    PARTITIONED BY (age INT)
+    STORED AS ORC;
 
 --Create partitioned table with different clauses order
 CREATE TABLE student (id INT, name STRING)
-  STORED AS ORC
-  PARTITIONED BY (age INT);
+    STORED AS ORC
+    PARTITIONED BY (age INT);
 
 --Use Row Format and file format
 CREATE TABLE student (id INT,name STRING)
-  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-  STORED AS TEXTFILE;
-
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+    STORED AS TEXTFILE;
 {% endhighlight %}
 
-
 ### Related Statements
-* [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
-* [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
+
+ * [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+ * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-like.md b/docs/sql-ref-syntax-ddl-create-table-like.md
index f49fd7fb24c91..fe1dc4b1ef258 100644
--- a/docs/sql-ref-syntax-ddl-create-table-like.md
+++ b/docs/sql-ref-syntax-ddl-create-table-like.md
@@ -18,21 +18,24 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 The `CREATE TABLE` statement defines a new table using the definition/metadata of an existing table or view.
 
 ### Syntax
+
 {% highlight sql %}
 CREATE TABLE [IF NOT EXISTS] table_identifier LIKE source_table_identifier
-USING data_source
-[ ROW FORMAT row_format ]
-[ STORED AS file_format ]
-[ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-[ LOCATION path ]
+    USING data_source
+    [ ROW FORMAT row_format ]
+    [ STORED AS file_format ]
+    [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
+    [ LOCATION path ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -70,28 +73,27 @@ USING data_source
   <dd>Path to the directory where table data is stored,Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc. Location to create an external table.</dd>
 </dl>
 
-
 ### Examples
-{% highlight sql %}
 
---Create table using an existing table
+{% highlight sql %}
+-- Create table using an existing table
 CREATE TABLE Student_Dupli like Student;
 
---Create table like using a data source
+-- Create table like using a data source
 CREATE TABLE Student_Dupli like Student USING CSV;
 
---Table is created as external table at the location specified
+-- Table is created as external table at the location specified
 CREATE TABLE Student_Dupli like Student location  '/root1/home';
 
---Create table like using a rowformat
+-- Create table like using a rowformat
 CREATE TABLE Student_Dupli like Student
-  ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-  STORED AS TEXTFILE
-  TBLPROPERTIES ('owner'='xxxx');
-
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+    STORED AS TEXTFILE
+    TBLPROPERTIES ('owner'='xxxx');
 {% endhighlight %}
 
 ### Related Statements
-* [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
-* [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
+
+ * [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+ * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
 
diff --git a/docs/sql-ref-syntax-ddl-create-table.md b/docs/sql-ref-syntax-ddl-create-table.md
index 20aff6fb823cb..b0388adbc9a38 100644
--- a/docs/sql-ref-syntax-ddl-create-table.md
+++ b/docs/sql-ref-syntax-ddl-create-table.md
@@ -20,13 +20,16 @@ license: |
 ---
 
 ### Description
+
 `CREATE TABLE` statement is used to define a table in an existing database. 
 
 The CREATE statements:
-* [CREATE TABLE USING DATA_SOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
-* [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
-* [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
+
+ * [CREATE TABLE USING DATA_SOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+ * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
+ * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
 
 ### Related Statements
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+
+ * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
diff --git a/docs/sql-ref-syntax-ddl-create-view.md b/docs/sql-ref-syntax-ddl-create-view.md
index 53b402771e421..5d9db2ab36b5d 100644
--- a/docs/sql-ref-syntax-ddl-create-view.md
+++ b/docs/sql-ref-syntax-ddl-create-view.md
@@ -20,17 +20,20 @@ license: |
 ---
 
 ### Description
+
 Views are based on the result-set of an `SQL` query. `CREATE VIEW` constructs
 a virtual table that has no physical data therefore other operations like
 `ALTER VIEW` and `DROP VIEW` only change metadata. 
 
 ### Syntax
+
 {% highlight sql %}
 CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_identifier
     create_view_clauses AS query
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>OR REPLACE</em></code></dt>
   <dd>If a view of same name already exists, it will be replaced.</dd>
@@ -71,6 +74,7 @@ CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_ident
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 -- Create or replace view for `experienced_employee` with comments.
 CREATE OR REPLACE VIEW experienced_employee
@@ -87,6 +91,7 @@ CREATE GLOBAL TEMPORARY VIEW IF NOT EXISTS subscribed_movies
 {% endhighlight %}
 
 ### Related Statements
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
-- [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+
+ * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+ * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+ * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-database.md b/docs/sql-ref-syntax-ddl-drop-database.md
index 0bdb98f2b129c..431139101eba4 100644
--- a/docs/sql-ref-syntax-ddl-drop-database.md
+++ b/docs/sql-ref-syntax-ddl-drop-database.md
@@ -30,7 +30,6 @@ exception will be thrown if the database does not exist in the system.
 DROP ( DATABASE | SCHEMA ) [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
 {% endhighlight %}
 
-
 ### Parameters
 
 <dl>
@@ -54,27 +53,20 @@ DROP ( DATABASE | SCHEMA ) [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- Create `inventory_db` Database
 CREATE DATABASE inventory_db COMMENT 'This database is used to maintain Inventory';
 
 -- Drop the database and it's tables
 DROP DATABASE inventory_db CASCADE;
-+---------+
-| Result  |
-+---------+
-+---------+
 
 -- Drop the database using IF EXISTS
 DROP DATABASE IF EXISTS inventory_db CASCADE;
-+---------+
-| Result  |
-+---------+
-+---------+
-
 {% endhighlight %}
 
-### Related statements
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+### Related Statements
+
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-function.md b/docs/sql-ref-syntax-ddl-drop-function.md
index 16d08d1ae8e99..f7ad18553e304 100644
--- a/docs/sql-ref-syntax-ddl-drop-function.md
+++ b/docs/sql-ref-syntax-ddl-drop-function.md
@@ -20,15 +20,16 @@ license: |
 ---
 
 ### Description
+
 The `DROP FUNCTION` statement drops a temporary or user defined function (UDF). An exception will
- be thrown if the function does not exist. 
+be thrown if the function does not exist. 
 
 ### Syntax
+
 {% highlight sql %}
 DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] [ db_name. ] function_name
 {% endhighlight %}
 
-
 ### Parameters
 
 <dl>
@@ -47,36 +48,33 @@ DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] [ db_name. ] function_name
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- Create a permanent function `test_avg`
 CREATE FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
 
 -- List user functions
 SHOW USER FUNCTIONS;
-  +-------------------+
-  |     function      |
-  +-------------------+
-  | default.test_avg  |
-  +-------------------+
+  +----------------+
+  |        function|
+  +----------------+
+  |default.test_avg|
+  +----------------+
 
 -- Create Temporary function `test_avg`
 CREATE TEMPORARY FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
 
 -- List user functions
 SHOW USER FUNCTIONS;
-  +-------------------+
-  |     function      |
-  +-------------------+
-  | default.test_avg  |
-  | test_avg          |
-  +-------------------+
+  +----------------+
+  |        function|
+  +----------------+
+  |default.test_avg|
+  |        test_avg|
+  +----------------+
 
 -- Drop Permanent function
 DROP FUNCTION test_avg;
-  +---------+
-  | Result  |
-  +---------+
-  +---------+
 
 -- Try to drop Permanent function which is not present
 DROP FUNCTION test_avg;
@@ -86,20 +84,18 @@ DROP FUNCTION test_avg;
 
 -- List the functions after dropping, it should list only temporary function
 SHOW USER FUNCTIONS;
-  +-----------+
-  | function  |
-  +-----------+
-  | test_avg  |
-  +-----------+
+  +--------+
+  |function|
+  +--------+
+  |test_avg|
+  +--------+
   
 -- Drop Temporary function
 DROP TEMPORARY FUNCTION IF EXISTS test_avg;
-  +---------+
-  | Result  |
-  +---------+
-  +---------+
 {% endhighlight %}
-### Related statements
-- [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
+
+### Related Statements
+
+ * [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-table.md b/docs/sql-ref-syntax-ddl-drop-table.md
index d1d8534efe7a2..32a9cc7bb27db 100644
--- a/docs/sql-ref-syntax-ddl-drop-table.md
+++ b/docs/sql-ref-syntax-ddl-drop-table.md
@@ -27,11 +27,13 @@ if the table is not `EXTERNAL` table. If the table is not present it throws an e
 In case of an external table, only the associated metadata information is removed from the metastore database.
 
 ### Syntax
+
 {% highlight sql %}
 DROP TABLE [ IF EXISTS ] table_identifier
 {% endhighlight %}
 
 ### Parameter
+
 <dl>
   <dt><code><em>IF EXISTS</em></code></dt>
   <dd>
@@ -48,40 +50,27 @@ DROP TABLE [ IF EXISTS ] table_identifier
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- Assumes a table named `employeetable` exists.
 DROP TABLE employeetable;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Assumes a table named `employeetable` exists in the `userdb` database
 DROP TABLE userdb.employeetable;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Assumes a table named `employeetable` does not exists.
 -- Throws exception
 DROP TABLE employeetable;
-Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeetable;
-(state=,code=0)
+  Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeetable;
+  (state=,code=0)
 
 -- Assumes a table named `employeetable` does not exists,Try with IF EXISTS
 -- this time it will not throw exception
 DROP TABLE IF EXISTS employeetable;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
-
 {% endhighlight %}
 
 ### Related Statements
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-
 
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-view.md b/docs/sql-ref-syntax-ddl-drop-view.md
index f313995022089..ae976c125f5f8 100644
--- a/docs/sql-ref-syntax-ddl-drop-view.md
+++ b/docs/sql-ref-syntax-ddl-drop-view.md
@@ -20,14 +20,17 @@ license: |
 ---
 
 ### Description
+
 `DROP VIEW` removes the metadata associated with a specified view from the catalog.
 
 ### Syntax
+
 {% highlight sql %}
 DROP VIEW [ IF EXISTS ] view_identifier
 {% endhighlight %}
 
 ### Parameter
+
 <dl>
   <dt><code><em>IF EXISTS</em></code></dt>
   <dd>
@@ -44,40 +47,29 @@ DROP VIEW [ IF EXISTS ] view_identifier
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- Assumes a view named `employeeView` exists.
 DROP VIEW employeeView;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Assumes a view named `employeeView` exists in the `userdb` database
 DROP VIEW userdb.employeeView;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Assumes a view named `employeeView` does not exists.
 -- Throws exception
 DROP VIEW employeeView;
-Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeeView;
-(state=,code=0)
+  Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeeView;
+  (state=,code=0)
 
 -- Assumes a view named `employeeView` does not exists,Try with IF EXISTS
 -- this time it will not throw exception
 DROP VIEW IF EXISTS employeeView;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
-
 {% endhighlight %}
 
 ### Related Statements
-- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+
+ * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+ * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+ * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-repair-table.md b/docs/sql-ref-syntax-ddl-repair-table.md
index daa6a46fca58f..499b2bff54d8d 100644
--- a/docs/sql-ref-syntax-ddl-repair-table.md
+++ b/docs/sql-ref-syntax-ddl-repair-table.md
@@ -20,14 +20,17 @@ license: |
 ---
 
 ### Description
+
 `MSCK REPAIR TABLE` recovers all the partitions in the directory of a table and updates the Hive metastore. When creating a table using `PARTITIONED BY` clause, partitions are generated and registered in the Hive metastore. However, if the partitioned table is created from existing data, partitions are not registered automatically in the Hive metastore. User needs to run `MSCK REPAIR TABLE` to register the partitions. `MSCK REPAIR TABLE` on a non-existent table or a table without partitions throws an exception. Another way to recover partitions is to use `ALTER TABLE RECOVER PARTITIONS`.
 
 ### Syntax
+
 {% highlight sql %}
 MSCK REPAIR TABLE table_identifier
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -40,30 +43,31 @@ MSCK REPAIR TABLE table_identifier
 </dl>
 
 ### Examples
-{% highlight sql %}
- -- create a partitioned table from existing data /tmp/namesAndAges.parquet
- CREATE TABLE t1 (name STRING, age INT) USING parquet PARTITIONED BY (age)
-     location "/tmp/namesAndAges.parquet";
 
- -- SELECT * FROM t1 does not return results
- SELECT * FROM t1;
-
- -- run MSCK REPAIR TABLE to recovers all the partitions
- MSCK REPAIR TABLE t1;
+{% highlight sql %}
+-- create a partitioned table from existing data /tmp/namesAndAges.parquet
+CREATE TABLE t1 (name STRING, age INT) USING parquet PARTITIONED BY (age)
+    LOCATION "/tmp/namesAndAges.parquet";
 
- -- SELECT * FROM t1 returns results
- SELECT * FROM t1;
+-- SELECT * FROM t1 does not return results
+SELECT * FROM t1;
 
-     + -------------- + ------+
-     | name           | age   |
-     + -------------- + ------+
-     | Michael        | 20    |
-     + -------------- + ------+
-     | Justin         | 19    |
-     + -------------- + ----- +
-     | Andy           | 30    |
-     + -------------- + ----- +
+-- run MSCK REPAIR TABLE to recovers all the partitions
+MSCK REPAIR TABLE t1;
 
+-- SELECT * FROM t1 returns results
+SELECT * FROM t1;
+  +-------+---+
+  |   name|age|
+  +-------+---+
+  |Michael| 20|
+  +-------+---+
+  | Justin| 19|
+  +-------+---+
+  |   Andy| 30|
+  +-------+---+
 {% endhighlight %}
+
 ### Related Statements
+
  * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
diff --git a/docs/sql-ref-syntax-ddl-truncate-table.md b/docs/sql-ref-syntax-ddl-truncate-table.md
index 3a0569e809d84..6377e83570207 100644
--- a/docs/sql-ref-syntax-ddl-truncate-table.md
+++ b/docs/sql-ref-syntax-ddl-truncate-table.md
@@ -20,16 +20,19 @@ license: |
 ---
 
 ### Description
+
 The `TRUNCATE TABLE` statement removes all the rows from a table or partition(s). The table must not be a view 
 or an external/temporary table. In order to truncate multiple partitions at once, the user can specify the partitions 
 in `partition_spec`. If no `partition_spec` is specified it will remove all partitions in the table.
 
 ### Syntax
+
 {% highlight sql %}
 TRUNCATE TABLE table_identifier [ partition_spec ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -52,47 +55,43 @@ TRUNCATE TABLE table_identifier [ partition_spec ]
   </dd>
 </dl>
 
-
 ### Examples
-{% highlight sql %}
 
---Create table Student with partition
-CREATE TABLE Student ( name String, rollno INT) PARTITIONED BY (age int);
+{% highlight sql %}
+-- Create table Student with partition
+CREATE TABLE Student (name STRING, rollno INT) PARTITIONED BY (age INT);
 
 SELECT * from Student;
-+-------+---------+------+--+
-| name  | rollno  | age  |
-+-------+---------+------+--+
-| ABC   | 1       | 10   |
-| DEF   | 2       | 10   |
-| XYZ   | 3       | 12   |
-+-------+---------+------+--+
+  +----+------+---+
+  |name|rollno|age|
+  +----+------+---+
+  | ABC|     1| 10|
+  | DEF|     2| 10|
+  | XYZ|     3| 12|
+  +----+------+---+
 
 -- Removes all rows from the table in the partition specified
 TRUNCATE TABLE Student partition(age=10);
 
---After truncate execution, records belonging to partition age=10 are removed
+-- After truncate execution, records belonging to partition age=10 are removed
 SELECT * from Student;
-+-------+---------+------+--+
-| name  | rollno  | age  |
-+-------+---------+------+--+
-| XYZ   | 3       | 12   |
-+-------+---------+------+--+
+  +----+------+---+
+  |name|rollno|age|
+  +----+------+---+
+  | XYZ|     3| 12|
+  +----+------+---+
 
 -- Removes all rows from the table from all partitions
 TRUNCATE TABLE Student;
 
 SELECT * from Student;
-+-------+---------+------+--+
-| name  | rollno  | age  |
-+-------+---------+------+--+
-+-------+---------+------+--+
-No rows selected 
-
+  +----+------+---+
+  |name|rollno|age|
+  +----+------+---+
+  +----+------+---+
 {% endhighlight %}
 
-
 ### Related Statements
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
 
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+ * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
diff --git a/docs/sql-ref-syntax-ddl.md b/docs/sql-ref-syntax-ddl.md
index ab4e95a1539ff..82fbf0498a20f 100644
--- a/docs/sql-ref-syntax-ddl.md
+++ b/docs/sql-ref-syntax-ddl.md
@@ -19,21 +19,19 @@ license: |
   limitations under the License.
 ---
 
-
 Data Definition Statements are used to create or modify the structure of database objects in a database. Spark SQL supports the following Data Definition Statements:
 
-
-- [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
-- [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
-- [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
-- [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+ * [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+ * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+ * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+ * [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+ * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+ * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
+ * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+ * [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
diff --git a/docs/sql-ref-syntax-dml-insert-into.md b/docs/sql-ref-syntax-dml-insert-into.md
index 715f43c9b80ea..ba65334ef8f61 100644
--- a/docs/sql-ref-syntax-dml-insert-into.md
+++ b/docs/sql-ref-syntax-dml-insert-into.md
@@ -24,12 +24,14 @@ license: |
 The `INSERT INTO` statement inserts new rows into a table. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
+
 {% highlight sql %}
 INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
     { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -70,149 +72,148 @@ INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
 </dl>
 
 ### Examples
-#### Single Row Insert Using a VALUES Clause
-{% highlight sql %}
- CREATE TABLE students (name VARCHAR(64), address VARCHAR(64), student_id INT)
-     USING PARQUET PARTITIONED BY (student_id);
-
- INSERT INTO students
-     VALUES ('Amy Smith', '123 Park Ave, San Jose', 111111);
 
- SELECT * FROM students;
+#### Single Row Insert Using a VALUES Clause
 
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
+{% highlight sql %}
+CREATE TABLE students (name VARCHAR(64), address VARCHAR(64), student_id INT)
+    USING PARQUET PARTITIONED BY (student_id);
+
+INSERT INTO students VALUES
+    ('Amy Smith', '123 Park Ave, San Jose', 111111);
+
+SELECT * FROM students;
+  +---------+---------------------+----------+
+  |     name|              address|student_id|
+  +---------+---------------------+----------+
+  |Amy Smith|123 Park Ave,San Jose|    111111|
+  +---------+---------------------+----------+
 {% endhighlight %}
 
 #### Multi-Row Insert Using a VALUES Clause
+
 {% highlight sql %}
- INSERT INTO students
-     VALUES ('Bob Brown', '456 Taylor St, Cupertino', 222222),
-            ('Cathy Johnson', '789 Race Ave, Palo Alto', 333333);
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
+INSERT INTO students VALUES
+    ('Bob Brown', '456 Taylor St, Cupertino', 222222),
+    ('Cathy Johnson', '789 Race Ave, Palo Alto', 333333);
+
+SELECT * FROM students;
+  +-------------+------------------------+----------+
+  |         name|                 address|student_id|
+  +-------------+------------------------+----------+
+  |    Amy Smith|  123 Park Ave, San Jose|    111111|
+  +-------------+------------------------+----------+
+  |    Bob Brown|456 Taylor St, Cupertino|    222222|
+  +-------------+------------------------+----------+
+  |Cathy Johnson| 789 Race Ave, Palo Alto|    333333|
+  +--------------+-----------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a SELECT Statement
+
 {% highlight sql %}
- -- Assuming the persons table has already been created and populated.
- SELECT * FROM persons;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | ssn            |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 123456789      |
-     + -------------- + ------------------------------ + -------------- +
-     | Eddie Davis    | 245 Market St, Milpitas        | 345678901      |
-     + -------------- + ------------------------------ + ---------------+
-
- INSERT INTO students PARTITION (student_id = 444444)
-     SELECT name, address FROM persons WHERE name = "Dora Williams";
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
+-- Assuming the persons table has already been created and populated.
+SELECT * FROM persons;
+  +-------------+-------------------------+---------+
+  |         name|                  address|      ssn|
+  +-------------+-------------------------+---------+
+  |Dora Williams|134 Forest Ave, Melo Park|123456789|
+  +-------------+-------------------------+---------+
+  |  Eddie Davis|  245 Market St, Milpitas|345678901|
+  +-------------+-------------------------+---------+
+
+INSERT INTO students PARTITION (student_id = 444444)
+    SELECT name, address FROM persons WHERE name = "Dora Williams";
+
+SELECT * FROM students;
+  +-------------+-------------------------+----------+
+  |         name|                  address|student_id|
+  +-------------+-------------------------+----------+
+  |    Amy Smith|   123 Park Ave, San Jose|    111111|
+  +-------------+-------------------------+----------+
+  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
+  +-------------+-------------------------+----------+
+  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
+  +-------------+-------------------------+----------+
+  |Dora Williams|134 Forest Ave, Melo Park|    444444|
+  +-------------+-------------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a TABLE Statement
+
 {% highlight sql %}
- -- Assuming the visiting_students table has already been created and populated.
- SELECT * FROM visiting_students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-
- INSERT INTO students TABLE visiting_students;
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
+-- Assuming the visiting_students table has already been created and populated.
+SELECT * FROM visiting_students;
+  +-------------+---------------------+----------+
+  |         name|              address|student_id|
+  +-------------+---------------------+----------+
+  |Fleur Laurent|345 Copper St, London|    777777|
+  +-------------+---------------------+----------+
+  |Gordon Martin| 779 Lake Ave, Oxford|    888888|
+  +-------------+---------------------+----------+
+
+INSERT INTO students TABLE visiting_students;
+
+SELECT * FROM students;
+  +-------------+-------------------------+----------+
+  |         name|                  address|student_id|
+  +-------------+-------------------------+----------+
+  |    Amy Smith|    123 Park Ave,San Jose|    111111|
+  +-------------+-------------------------+----------+
+  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
+  +-------------+-------------------------+----------+
+  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
+  +-------------+-------------------------+----------+
+  |Dora Williams|134 Forest Ave, Melo Park|    444444|
+  +-------------+-------------------------+----------+
+  |Fleur Laurent|    345 Copper St, London|    777777|
+  +-------------+-------------------------+----------+
+  |Gordon Martin|     779 Lake Ave, Oxford|    888888|
+  +-------------+-------------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a FROM Statement
+
 {% highlight sql %}
- -- Assuming the applicants table has already been created and populated.
- SELECT * FROM applicants;
-
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | name           | address                        | student_id     | qualified      |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Ivy King       | 367 Leigh Ave, Santa Clara     | 101010         | false          |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-
- INSERT INTO students
-      FROM applicants SELECT name, address, id applicants WHERE qualified = true;
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         |
-     + -------------- + ------------------------------ + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         |
-     + -------------- + ------------------------------ + -------------- +
+-- Assuming the applicants table has already been created and populated.
+SELECT * FROM applicants;
+  +-----------+--------------------------+----------+---------+
+  |       name|                   address|student_id|qualified|
+  +-----------+--------------------------+----------+---------+
+  |Helen Davis| 469 Mission St, San Diego|    999999|     true|
+  +-----------+--------------------------+----------+---------+
+  |   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
+  +-----------+--------------------------+----------+---------+
+  | Jason Wang|     908 Bird St, Saratoga|    121212|     true|
+  +-----------+--------------------------+----------+---------+
+
+INSERT INTO students
+     FROM applicants SELECT name, address, id applicants WHERE qualified = true;
+
+SELECT * FROM students;
+  +-------------+-------------------------+----------+
+  |         name|                  address|student_id|
+  +-------------+-------------------------+----------+
+  |    Amy Smith|   123 Park Ave, San Jose|    111111|
+  +-------------+-------------------------+----------+
+  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
+  +-------------+-------------------------+----------+
+  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
+  +-------------+-------------------------+----------+
+  |Dora Williams|134 Forest Ave, Melo Park|    444444|
+  +-------------+-------------------------+----------+
+  |Fleur Laurent|    345 Copper St, London|    777777|
+  +-------------+-------------------------+----------+
+  |Gordon Martin|     779 Lake Ave, Oxford|    888888|
+  +-------------+-------------------------+----------+
+  |  Helen Davis|469 Mission St, San Diego|    999999|
+  +-------------+-------------------------+----------+
+  |   Jason Wang|    908 Bird St, Saratoga|    121212|
+  +-------------+-------------------------+----------+
 {% endhighlight %}
 
 ### Related Statements
-  * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+
+ * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+ * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+ * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
index 3b0475aef1015..3ab0994cf06e7 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
@@ -20,17 +20,20 @@ license: |
 ---
 
 ### Description
+
 The `INSERT OVERWRITE DIRECTORY` with Hive format overwrites the existing data in the directory with the new values using Hive `SerDe`.
 Hive support must be enabled to use this command. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
+
 {% highlight sql %}
 INSERT OVERWRITE [ LOCAL ] DIRECTORY directory_path
-   [ ROW FORMAT row_format ] [ STORED AS file_format ]
-   { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
+    [ ROW FORMAT row_format ] [ STORED AS file_format ]
+    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>directory_path</em></code></dt>
   <dd>
@@ -71,17 +74,19 @@ INSERT OVERWRITE [ LOCAL ] DIRECTORY directory_path
 </dl>
 
 ### Examples
+
 {% highlight sql %}
- INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
-     STORED AS orc
-     SELECT * FROM test_table;
+INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
+    STORED AS orc
+    SELECT * FROM test_table;
 
- INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
-     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-     SELECT * FROM test_table;
+INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+    SELECT * FROM test_table;
 {% endhighlight %}
 
 ### Related Statements
-  * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-  * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+
+ * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+ * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+ * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
index 7f3224de0ccee..645396620f21a 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
@@ -18,10 +18,13 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
+
 The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the directory with the new values using a given Spark file format. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
+
 {% highlight sql %}
 INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
     USING file_format [ OPTIONS ( key = val [ , ... ] ) ]
@@ -29,6 +32,7 @@ INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>directory_path</em></code></dt>
   <dd>
@@ -67,6 +71,7 @@ INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 INSERT OVERWRITE DIRECTORY '/tmp/destination'
     USING parquet
@@ -80,6 +85,7 @@ INSERT OVERWRITE DIRECTORY
 {% endhighlight %}
 
 ### Related Statements
-  * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-  * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+
+ * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+ * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+ * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index 2318a8b622202..added8e1976be 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -24,12 +24,14 @@ license: |
 The `INSERT OVERWRITE` statement overwrites the existing data in the table using the new values. The inserted rows can be specified by value expressions or result from a query.
 
 ### Syntax
+
 {% highlight sql %}
 INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
     { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>table_identifier</em></code></dt>
   <dd>
@@ -70,129 +72,120 @@ INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
 </dl>
 
 ### Examples
+
 #### Insert Using a VALUES Clause
+
 {% highlight sql %}
- -- Assuming the students table has already been created and populated.
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Bob Brown      | 456 Taylor St, Cupertino       | 222222         |
-     + -------------- + ------------------------------ + -------------- +
-     | Cathy Johnson  | 789 Race Ave, Palo Alto        | 333333         |
-     + -------------- + ------------------------------ + -------------- +
-     | Dora Williams  | 134 Forest Ave, Melo Park      | 444444         |
-     + -------------- + ------------------------------ + -------------- +
-     | Fleur Laurent  | 345 Copper St, London          | 777777         |
-     + -------------- + ------------------------------ + -------------- +
-     | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-     + -------------- + ------------------------------ + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         |
-     + -------------- + ------------------------------ + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         |
-     + -------------- + ------------------------------ + -------------- +
-
- INSERT OVERWRITE students
-     VALUES ('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
-            ('Brian Reed', '723 Kern Ave, Palo Alto', 222222);
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Ashua Hill     | 456 Erica Ct, Cupertino        | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-     | Brian Reed     | 723 Kern Ave, Palo Alto        | 222222         |
-     + -------------- + ------------------------------ + -------------- +
+-- Assuming the students table has already been created and populated.
+SELECT * FROM students;
+  +-------------+-------------------------+----------+
+  |         name|                  address|student_id|
+  +-------------+-------------------------+----------+
+  |    Amy Smith|   123 Park Ave, San Jose|    111111|
+  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
+  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
+  |Dora Williams|134 Forest Ave, Melo Park|    444444|
+  |Fleur Laurent|    345 Copper St, London|    777777|
+  |Gordon Martin|     779 Lake Ave, Oxford|    888888|
+  |  Helen Davis|469 Mission St, San Diego|    999999|
+  |   Jason Wang|    908 Bird St, Saratoga|    121212|
+  +-------------+-------------------------+----------+
+
+INSERT OVERWRITE students VALUES
+    ('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
+    ('Brian Reed', '723 Kern Ave, Palo Alto', 222222);
+
+SELECT * FROM students;
+  +----------+-----------------------+----------+
+  |      name|                address|student_id|
+  +----------+-----------------------+----------+
+  |Ashua Hill|456 Erica Ct, Cupertino|    111111|
+  |Brian Reed|723 Kern Ave, Palo Alto|    222222|
+  +----------+-----------------------+----------+
+
 {% endhighlight %}
 
 #### Insert Using a SELECT Statement
+
 {% highlight sql %}
- -- Assuming the persons table has already been created and populated.
- SELECT * FROM persons;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | ssn            |
-      + -------------- + ------------------------------ + -------------- +
-      | Dora Williams  | 134 Forest Ave, Melo Park      | 123456789      |
-      + -------------- + ------------------------------ + -------------- +
-      | Eddie Davis    | 245 Market St, Milpitas        | 345678901      |
-      + -------------- + ------------------------------ + ---------------+
-
- INSERT OVERWRITE students PARTITION (student_id = 222222)
-     SELECT name, address FROM persons WHERE name = "Dora Williams";
-
- SELECT * FROM students;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | student_id     |
-      + -------------- + ------------------------------ + -------------- +
-      | Ashua Hill     | 456 Erica Ct, Cupertino        | 111111         |
-      + -------------- + ------------------------------ + -------------- +
-      | Dora Williams  | 134 Forest Ave, Melo Park      | 222222         |
-      + -------------- + ------------------------------ + -------------- +
+-- Assuming the persons table has already been created and populated.
+SELECT * FROM persons;
+  +-------------+-------------------------+---------+
+  |         name|                  address|      ssn|
+  +-------------+-------------------------+---------+
+  |Dora Williams|134 Forest Ave, Melo Park|123456789|
+  +-------------+-------------------------+---------+
+  |  Eddie Davis|   245 Market St,Milpitas|345678901|
+  +-------------+-------------------------+---------+
+
+INSERT OVERWRITE students PARTITION (student_id = 222222)
+    SELECT name, address FROM persons WHERE name = "Dora Williams";
+
+SELECT * FROM students;
+  +-------------+-------------------------+----------+
+  |         name|                  address|student_id|
+  +-------------+-------------------------+----------+
+  |   Ashua Hill|  456 Erica Ct, Cupertino|    111111|
+  +-------------+-------------------------+----------+
+  |Dora Williams|134 Forest Ave, Melo Park|    222222|
+  +-------------+-------------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a TABLE Statement
+
 {% highlight sql %}
- -- Assuming the visiting_students table has already been created and populated.
- SELECT * FROM visiting_students;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | student_id     |
-      + -------------- + ------------------------------ + -------------- +
-      | Fleur Laurent  | 345 Copper St, London          | 777777         |
-      + -------------- + ------------------------------ + -------------- +
-      | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-      + -------------- + ------------------------------ + -------------- +
-
- INSERT OVERWRITE students TABLE visiting_students;
-
- SELECT * FROM students;
-
-      + -------------- + ------------------------------ + -------------- +
-      | name           | address                        | student_id     |
-      + -------------- + ------------------------------ + -------------- +
-      | Fleur Laurent  | 345 Copper St, London          | 777777         |
-      + -------------- + ------------------------------ + -------------- +
-      | Gordon Martin  | 779 Lake Ave, Oxford           | 888888         |
-      + -------------- + ------------------------------ + -------------- +
+-- Assuming the visiting_students table has already been created and populated.
+SELECT * FROM visiting_students;
+  +-------------+---------------------+----------+
+  |         name|              address|student_id|
+  +-------------+---------------------+----------+
+  |Fleur Laurent|345 Copper St, London|    777777|
+  +-------------+---------------------+----------+
+  |Gordon Martin| 779 Lake Ave, Oxford|    888888|
+  +-------------+---------------------+----------+
+
+INSERT OVERWRITE students TABLE visiting_students;
+
+SELECT * FROM students;
+  +-------------+---------------------+----------+
+  |         name|              address|student_id|
+  +-------------+---------------------+----------+
+  |Fleur Laurent|345 Copper St, London|    777777|
+  +-------------+---------------------+----------+
+  |Gordon Martin| 779 Lake Ave, Oxford|    888888|
+  +-------------+---------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a FROM Statement
+
 {% highlight sql %}
- -- Assuming the applicants table has already been created and populated.
- SELECT * FROM applicants;
-
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | name           | address                        | student_id     | qualified      |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Ivy King       | 367 Leigh Ave, Santa Clara     | 101010         | false          |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         | true           |
-     + -------------- + ------------------------------ + -------------- + -------------- +
-
- INSERT OVERWRITE students
-      FROM applicants SELECT name, address, id applicants WHERE qualified = true;
-
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Helen Davis    | 469 Mission St, San Diego      | 999999         |
-     + -------------- + ------------------------------ + -------------- +
-     | Jason Wang     | 908 Bird St, Saratoga          | 121212         |
-     + -------------- + ------------------------------ + -------------- +
+-- Assuming the applicants table has already been created and populated.
+SELECT * FROM applicants;
+  +-----------+--------------------------+----------+---------+
+  |       name|                   address|student_id|qualified|
+  +-----------+--------------------------+----------+---------+
+  |Helen Davis| 469 Mission St, San Diego|    999999|     true|
+  +-----------+--------------------------+----------+---------+
+  |   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
+  +-----------+--------------------------+----------+---------+
+  | Jason Wang|     908 Bird St, Saratoga|    121212|     true|
+  +-----------+--------------------------+----------+---------+
+
+INSERT OVERWRITE students
+    FROM applicants SELECT name, address, id applicants WHERE qualified = true;
+
+SELECT * FROM students;
+  +-----------+-------------------------+----------+
+  |       name|                  address|student_id|
+  +-----------+-------------------------+----------+
+  |Helen Davis|469 Mission St, San Diego|    999999|
+  +-----------+-------------------------+----------+
+  | Jason Wang|    908 Bird St, Saratoga|    121212|
+  +-----------+-------------------------+----------+
 {% endhighlight %}
 
 ### Related Statements
-  * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-  * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
-  * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+
+ * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+ * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+ * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert.md b/docs/sql-ref-syntax-dml-insert.md
index 15a2e28896943..2345add2460c8 100644
--- a/docs/sql-ref-syntax-dml-insert.md
+++ b/docs/sql-ref-syntax-dml-insert.md
@@ -20,7 +20,8 @@ license: |
 ---
 
 The INSERT statements:
-* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
-* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
-* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
-* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+
+ * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+ * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+ * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+ * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-load.md b/docs/sql-ref-syntax-dml-load.md
index 090c49dc5d082..9a9bf230e3101 100644
--- a/docs/sql-ref-syntax-dml-load.md
+++ b/docs/sql-ref-syntax-dml-load.md
@@ -20,14 +20,17 @@ license: |
 ---
 
 ### Description
+
 `LOAD DATA` statement loads the data into a Hive serde table from the user specified directory or file. If a directory is specified then all the files from the directory are loaded. If a file is specified then only the single file is loaded. Additionally the `LOAD DATA` statement takes an optional partition specification. When a partition is specified, the data files (when input source is a directory) or the single file (when input source is a file) are loaded into the partition of the target table.
 
 ### Syntax
+
 {% highlight sql %}
 LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ partition_spec ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>path</em></code></dt>
   <dd>Path of the file system. It can be either an absolute or a relative path.</dd>
@@ -67,65 +70,57 @@ LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ part
 </dl>
 
 ### Examples
-{% highlight sql %}
- -- Example without partition specification.
- -- Assuming the students table has already been created and populated.
- SELECT * FROM students;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-
- CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT) USING HIVE;
-
- -- Assuming the students table is in '/user/hive/warehouse/'
- LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
-
- SELECT * FROM test_load;
-
-     + -------------- + ------------------------------ + -------------- +
-     | name           | address                        | student_id     |
-     + -------------- + ------------------------------ + -------------- +
-     | Amy Smith      | 123 Park Ave, San Jose         | 111111         |
-     + -------------- + ------------------------------ + -------------- +
-
- -- Example with partition specification.
- CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) PARTITIONED BY (c2, c3);
-
- INSERT INTO test_partition PARTITION (c2 = 2, c3 = 3) VALUES (1);
-
- INSERT INTO test_partition PARTITION (c2 = 5, c3 = 6) VALUES (4);
-
- INSERT INTO test_partition PARTITION (c2 = 8, c3 = 9) VALUES (7);
-
- SELECT * FROM test_partition;
-
-      + ------- + ------- + ----- +
-      | c1      | c2      | c3    |
-      + ------- + --------------- +
-      | 1       | 2       | 3     |
-      + ------- + ------- + ----- +
-      | 4       | 5       | 6     |
-      + ------- + ------- + ----- +
-      | 7       | 8       | 9     |
-      + ------- + ------- + ----- +
-
- CREATE TABLE test_load_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
-
- -- Assuming the test_partition table is in '/user/hive/warehouse/'
- LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
-     OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);
-
- SELECT * FROM test_load_partition;
-
-      + ------- + ------- + ----- +
-      | c1      | c2      | c3    |
-      + ------- + --------------- +
-      | 1       | 2       | 3     |
-      + ------- + ------- + ----- +
-
 
+{% highlight sql %}
+-- Example without partition specification.
+-- Assuming the students table has already been created and populated.
+SELECT * FROM students;
+  +---------+----------------------+----------+
+  |     name|               address|student_id|
+  +---------+----------------------+----------+
+  |Amy Smith|123 Park Ave, San Jose|    111111|
+  +---------+----------------------+----------+
+
+CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT) USING HIVE;
+
+-- Assuming the students table is in '/user/hive/warehouse/'
+LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
+
+SELECT * FROM test_load;
+  +---------+----------------------+----------+
+  |     name|               address|student_id|
+  +---------+----------------------+----------+
+  |Amy Smith|123 Park Ave, San Jose|    111111|
+  +---------+----------------------+----------+
+
+-- Example with partition specification.
+CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) PARTITIONED BY (c2, c3);
+
+INSERT INTO test_partition PARTITION (c2 = 2, c3 = 3) VALUES (1);
+
+INSERT INTO test_partition PARTITION (c2 = 5, c3 = 6) VALUES (4);
+
+INSERT INTO test_partition PARTITION (c2 = 8, c3 = 9) VALUES (7);
+
+SELECT * FROM test_partition;
+  +---+---+---+
+  | c1| c2| c3|
+  +---+---+---+
+  |  1|  2|  3|
+  |  4|  5|  6|
+  |  7|  8|  9|
+  +---+---+---+
+
+CREATE TABLE test_load_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
+
+-- Assuming the test_partition table is in '/user/hive/warehouse/'
+LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
+    OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);
+
+SELECT * FROM test_load_partition;
+  +---+---+---+
+  | c1| c2| c3|
+  +---+---+---+
+  |  1|  2|  3|
+  +---+---+---+
 {% endhighlight %}
-
diff --git a/docs/sql-ref-syntax-dml.md b/docs/sql-ref-syntax-dml.md
index b5dd45f8962c9..9f75990555f64 100644
--- a/docs/sql-ref-syntax-dml.md
+++ b/docs/sql-ref-syntax-dml.md
@@ -21,5 +21,5 @@ license: |
 
 Data Manipulation Statements are used to add, change, or delete data. Spark SQL supports the following Data Manipulation Statements:
 
-- [INSERT](sql-ref-syntax-dml-insert.html)
-- [LOAD](sql-ref-syntax-dml-load.html)
+ * [INSERT](sql-ref-syntax-dml-insert.html)
+ * [LOAD](sql-ref-syntax-dml-load.html)
diff --git a/docs/sql-ref-syntax-qry-explain.md b/docs/sql-ref-syntax-qry-explain.md
index 7e18e16bc8ea6..6a7c2ace8223b 100644
--- a/docs/sql-ref-syntax-qry-explain.md
+++ b/docs/sql-ref-syntax-qry-explain.md
@@ -24,8 +24,8 @@ license: |
 The `EXPLAIN` statement is used to provide logical/physical plans for an input statement. 
 By default, this clause provides information about a physical plan only.
 
-
 ### Syntax
+
 {% highlight sql %}
 EXPLAIN [EXTENDED | CODEGEN | COST | FORMATTED] statement
 {% endhighlight %}
@@ -64,76 +64,72 @@ EXPLAIN [EXTENDED | CODEGEN | COST | FORMATTED] statement
  </dl>
 
 ### Examples
-{% highlight sql %}
-
---Default Output
 
+{% highlight sql %}
+-- Default Output
 EXPLAIN select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
-+----------------------------------------------------+
-|                        plan                        |
-+----------------------------------------------------+
-| == Physical Plan ==
-*(2) HashAggregate(keys=[k#33], functions=[sum(cast(v#34 as bigint))])
-+- Exchange hashpartitioning(k#33, 200), true, [id=#59]
-   +- *(1) HashAggregate(keys=[k#33], functions=[partial_sum(cast(v#34 as bigint))])
-      +- *(1) LocalTableScan [k#33, v#34]
-|
- +----------------------------------------------------
+  +----------------------------------------------------+
+  |                                                plan|
+  +----------------------------------------------------+
+  | == Physical Plan ==
+   *(2) HashAggregate(keys=[k#33], functions=[sum(cast(v#34 as bigint))])
+   +- Exchange hashpartitioning(k#33, 200), true, [id=#59]
+      +- *(1) HashAggregate(keys=[k#33], functions=[partial_sum(cast(v#34 as bigint))])
+         +- *(1) LocalTableScan [k#33, v#34]
+  |
+  +----------------------------------------------------
 
 -- Using Extended
-
 EXPLAIN EXTENDED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
-+----------------------------------------------------+
-|                        plan                        |
-+----------------------------------------------------+
-| == Parsed Logical Plan ==
-'Aggregate ['k], ['k, unresolvedalias('sum('v), None)]
-+- 'SubqueryAlias `t`
-   +- 'UnresolvedInlineTable [k, v], [List(1, 2), List(1, 3)]
-
-== Analyzed Logical Plan ==
-k: int, sum(v): bigint
-Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
-+- SubqueryAlias `t`
+  +----------------------------------------------------+
+  |                                                plan|
+  +----------------------------------------------------+
+  | == Parsed Logical Plan ==
+   'Aggregate ['k], ['k, unresolvedalias('sum('v), None)]
+   +- 'SubqueryAlias `t`
+      +- 'UnresolvedInlineTable [k, v], [List(1, 2), List(1, 3)]
+   
+   == Analyzed Logical Plan ==
+   k: int, sum(v): bigint
+   Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
+   +- SubqueryAlias `t`
+      +- LocalRelation [k#47, v#48]
+   
+   == Optimized Logical Plan ==
+   Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
    +- LocalRelation [k#47, v#48]
-
-== Optimized Logical Plan ==
-Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
-+- LocalRelation [k#47, v#48]
-
-== Physical Plan ==
-*(2) HashAggregate(keys=[k#47], functions=[sum(cast(v#48 as bigint))], output=[k#47, sum(v)#50L])
-+- Exchange hashpartitioning(k#47, 200), true, [id=#79]
-   +- *(1) HashAggregate(keys=[k#47], functions=[partial_sum(cast(v#48 as bigint))], output=[k#47, sum#52L])
+   
+   == Physical Plan ==
+   *(2) HashAggregate(keys=[k#47], functions=[sum(cast(v#48 as bigint))], output=[k#47, sum(v)#50L])
+  +- Exchange hashpartitioning(k#47, 200), true, [id=#79]
+     +- *(1) HashAggregate(keys=[k#47], functions=[partial_sum(cast(v#48 as bigint))], output=[k#47, sum#52L])
       +- *(1) LocalTableScan [k#47, v#48]
- |
-+----------------------------------------------------+
-
---Using Formatted
+  |
+  +----------------------------------------------------+
 
+-- Using Formatted
 EXPLAIN FORMATTED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
-+----------------------------------------------------+
-|                        plan                        |
-+----------------------------------------------------+
-| == Physical Plan ==
-* HashAggregate (4)
-+- Exchange (3)
-   +- * HashAggregate (2)
-      +- * LocalTableScan (1)
-
-
-(1) LocalTableScan [codegen id : 1]
-Output: [k#19, v#20]
-     
-(2) HashAggregate [codegen id : 1]
-Input: [k#19, v#20]
-     
-(3) Exchange 
-Input: [k#19, sum#24L]
-     
-(4) HashAggregate [codegen id : 2]
-Input: [k#19, sum#24L] 
- |
-+----------------------------------------------------+
-
+  +----------------------------------------------------+
+  |                                                plan|
+  +----------------------------------------------------+
+  | == Physical Plan ==
+   * HashAggregate (4)
+   +- Exchange (3)
+      +- * HashAggregate (2)
+         +- * LocalTableScan (1)
+   
+   
+   (1) LocalTableScan [codegen id : 1]
+   Output: [k#19, v#20]
+        
+   (2) HashAggregate [codegen id : 1]
+   Input: [k#19, v#20]
+        
+   (3) Exchange 
+   Input: [k#19, sum#24L]
+        
+   (4) HashAggregate [codegen id : 2]
+   Input: [k#19, sum#24L] 
+  |
+  +----------------------------------------------------+
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-sampling.md
index 061f21c3d16dd..3bc45cc48b78f 100644
--- a/docs/sql-ref-syntax-qry-sampling.md
+++ b/docs/sql-ref-syntax-qry-sampling.md
@@ -31,9 +31,9 @@ Note: `TABLESAMPLE` returns the approximate number of rows or fraction requested
 ### Syntax
 
 {% highlight sql %}
-    TABLESAMPLE ((integer_expression | decimal_expression) PERCENT)
-        | TABLESAMPLE (integer_expression ROWS)
-        | TABLESAMPLE (BUCKET integer_expression OUT OF integer_expression)
+TABLESAMPLE ((integer_expression | decimal_expression) PERCENT)
+    | TABLESAMPLE (integer_expression ROWS)
+    | TABLESAMPLE (BUCKET integer_expression OUT OF integer_expression)
 {% endhighlight %}
 
 ### Examples
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index 8f1dc59806f80..687b2b512cd90 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -18,6 +18,9 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>CLUSTER BY</code> clause is used to first repartition the data based
 on the input expressions and then sort the data within each partition. This is
 semantically equivalent to performing a
@@ -26,11 +29,13 @@ semantically equivalent to performing a
 resultant rows are sorted within each partition and does not guarantee a total order of output.
 
 ### Syntax
+
 {% highlight sql %}
 CLUSTER BY { expression [ , ... ] }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>expression</em></code></dt>
   <dd>
@@ -39,6 +44,7 @@ CLUSTER BY { expression [ , ... ] }
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
@@ -58,16 +64,15 @@ SET spark.sql.shuffle.partitions = 2;
 -- of a query when `CLUSTER BY` is not used vs when it's used. The query below produces rows
 -- where age column is not sorted.
 SELECT age, name FROM person;
-
   +---+-------+
-  |age|name   |
+  |age|   name|
   +---+-------+
-  |16 |Shone S|
-  |25 |Zen Hui|
-  |16 |Jack N |
-  |25 |Mike A |
-  |18 |John A |
-  |18 |Anil B |
+  | 16|Shone S|
+  | 25|Zen Hui|
+  | 16| Jack N|
+  | 25| Mike A|
+  | 18| John A|
+  | 18| Anil B|
   +---+-------+
 
 -- Produces rows clustered by age. Persons with same age are clustered together.
@@ -75,25 +80,25 @@ SELECT age, name FROM person;
 -- persons with age 16 are in the second partition. The rows are sorted based
 -- on age within each partition.
 SELECT age, name FROM person CLUSTER BY age;
-
   +---+-------+
-  |age|name   |
+  |age|   name|
   +---+-------+
-  |18 |John A |
-  |18 |Anil B |
-  |25 |Zen Hui|
-  |25 |Mike A |
-  |16 |Shone S|
-  |16 |Jack N |
+  | 18| John A|
+  | 18| Anil B|
+  | 25|Zen Hui|
+  | 25| Mike A|
+  | 16|Shone S|
+  | 16| Jack N|
   +---+-------+
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index 957df9c48eee7..18d73c7cdff19 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -18,16 +18,21 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>DISTRIBUTE BY</code> clause is used to repartition the data based
 on the input expressions. Unlike the [CLUSTER BY](sql-ref-syntax-qry-select-clusterby.html)
 clause, this does not sort the data within each partition.
 
 ### Syntax
+
 {% highlight sql %}
 DISTRIBUTE BY { expression [ , ... ] }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>expression</em></code></dt>
   <dd>
@@ -36,6 +41,7 @@ DISTRIBUTE BY { expression [ , ... ] }
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
@@ -55,40 +61,39 @@ SET spark.sql.shuffle.partitions = 2;
 -- behavior of `DISTRIBUTE BY`. The query below produces rows where age columns are not
 -- clustered together.
 SELECT age, name FROM person;
-
   +---+-------+
-  |age|name   |
+  |age|   name|
   +---+-------+
-  |16 |Shone S|
-  |25 |Zen Hui|
-  |16 |Jack N |
-  |25 |Mike A |
-  |18 |John A |
-  |18 |Anil B |
+  | 16|Shone S|
+  | 25|Zen Hui|
+  | 16| Jack N|
+  | 25| Mike A|
+  | 18| John A|
+  | 18| Anil B|
   +---+-------+
 
 -- Produces rows clustered by age. Persons with same age are clustered together.
 -- Unlike `CLUSTER BY` clause, the rows are not sorted within a partition.
 SELECT age, name FROM person DISTRIBUTE BY age;
-
   +---+-------+
-  |age|name   |
+  |age|   name|
   +---+-------+
-  |25 |Zen Hui|
-  |25 |Mike A |
-  |18 |John A |
-  |18 |Anil B |
-  |16 |Shone S|
-  |16 |Jack N |
+  | 25|Zen Hui|
+  | 25| Mike A|
+  | 18| John A|
+  | 18| Anil B|
+  | 16|Shone S|
+  | 16| Jack N|
   +---+-------+
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index c461a18e004a4..1676ca9c6d6aa 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -18,15 +18,19 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>GROUP BY</code> clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
 the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple
 aggregations for the same input record set via `GROUPING SETS`, `CUBE`, `ROLLUP` clauses.
 When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
 
 ### Syntax
+
 {% highlight sql %}
 GROUP BY group_expression [ , group_expression [ , ... ] ]
-  [ { WITH ROLLUP | WITH CUBE | GROUPING SETS (grouping_set [ , ...]) } ]
+    [ { WITH ROLLUP | WITH CUBE | GROUPING SETS (grouping_set [ , ...]) } ]
 
 GROUP BY GROUPING SETS (grouping_set [ , ...])
 {% endhighlight %}
@@ -37,6 +41,7 @@ aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_ex
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>GROUPING SETS</em></code></dt>
   <dd>
@@ -92,6 +97,7 @@ aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_ex
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
 INSERT INTO dealer VALUES
@@ -106,42 +112,38 @@ INSERT INTO dealer VALUES
 
 -- Sum of quantity per dealership. Group by `id`.
 SELECT id, sum(quantity) FROM dealer GROUP BY id ORDER BY id;
-
   +---+-------------+
-  |id |sum(quantity)|
+  | id|sum(quantity)|
   +---+-------------+
-  |100|32           |
-  |200|33           |
-  |300|13           |
+  |100|           32|
+  |200|           33|
+  |300|           13|
   +---+-------------+
 
 -- Use column position in GROUP by clause.
 SELECT id, sum(quantity) FROM dealer GROUP BY 1 ORDER BY 1;
-
   +---+-------------+
-  |id |sum(quantity)|
+  | id|sum(quantity)|
   +---+-------------+
-  |100|32           |
-  |200|33           |
-  |300|13           |
+  |100|           32|
+  |200|           33|
+  |300|           13|
   +---+-------------+
 
 -- Multiple aggregations.
 -- 1. Sum of quantity per dealership.
 -- 2. Max quantity per dealership.
 SELECT id, sum(quantity) AS sum, max(quantity) AS max FROM dealer GROUP BY id ORDER BY id;
-
   +---+---+---+
-  |id |sum|max|
+  | id|sum|max|
   +---+---+---+
-  |100|32 |15 |
-  |200|33 |20 |
-  |300|13 |8  |
+  |100| 32| 15|
+  |200| 33| 20|
+  |300| 13|  8|
   +---+---+---+
 
 -- Count the number of distinct dealer cities per car_model.
 SELECT car_model, count(DISTINCT city) AS count FROM dealer GROUP BY car_model;
-
   +------------+-----+
   |   car_model|count|
   +------------+-----+
@@ -155,14 +157,13 @@ SELECT id, sum(quantity) FILTER (
             WHERE car_model IN ('Honda Civic', 'Honda CRV')
         ) AS `sum(quantity)` FROM dealer
     GROUP BY id ORDER BY id;
-
-   +---+-------------+
-   | id|sum(quantity)|
-   +---+-------------+
-   |100|           17|
-   |200|           23|
-   |300|            5|
-   +---+-------------+
+  +---+-------------+
+  | id|sum(quantity)|
+  +---+-------------+
+  |100|           17|
+  |200|           23|
+  |300|            5|
+  +---+-------------+
 
 -- Aggregations using multiple sets of grouping columns in a single statement.
 -- Following performs aggregations based on four sets of grouping columns.
@@ -171,112 +172,108 @@ SELECT id, sum(quantity) FILTER (
 -- 3. car_model
 -- 4. Empty grouping set. Returns quantities for all city and car models.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
-   ORDER BY city;
-
+    GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
+    ORDER BY city;
   +--------+------------+---+
-  |city    |car_model   |sum|
+  |    city|   car_model|sum|
   +--------+------------+---+
-  |null    |null        |78 |
-  |null    |Honda Accord|33 |
-  |null    |Honda CRV   |10 |
-  |null    |Honda Civic |35 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
+  |    null|        null| 78|
+  |    null| HondaAccord| 33|
+  |    null|    HondaCRV| 10|
+  |    null|  HondaCivic| 35|
+  |  Dublin|        null| 33|
+  |  Dublin| HondaAccord| 10|
+  |  Dublin|    HondaCRV|  3|
+  |  Dublin|  HondaCivic| 20|
+  | Fremont|        null| 32|
+  | Fremont| HondaAccord| 15|
+  | Fremont|    HondaCRV|  7|
+  | Fremont|  HondaCivic| 10|
+  | SanJose|        null| 13|
+  | SanJose| HondaAccord|  8|
+  | SanJose|  HondaCivic|  5|
   +--------+------------+---+
 
 -- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS`
 -- specifications are present.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
-   ORDER BY city, car_model;
-
+    GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
+    ORDER BY city, car_model;
   +--------+------------+---+
-  |city    |car_model   |sum|
+  |    city|   car_model|sum|
   +--------+------------+---+
-  |null    |null        |78 |
-  |null    |Honda Accord|33 |
-  |null    |Honda CRV   |10 |
-  |null    |Honda Civic |35 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
+  |    null|        null| 78|
+  |    null| HondaAccord| 33|
+  |    null|    HondaCRV| 10|
+  |    null|  HondaCivic| 35|
+  |  Dublin|        null| 33|
+  |  Dublin| HondaAccord| 10|
+  |  Dublin|    HondaCRV|  3|
+  |  Dublin|  HondaCivic| 20|
+  | Fremont|        null| 32|
+  | Fremont| HondaAccord| 15|
+  | Fremont|    HondaCRV|  7|
+  | Fremont|  HondaCivic| 10|
+  | SanJose|        null| 13|
+  | SanJose| HondaAccord|  8|
+  | SanJose|  HondaCivic|  5|
   +--------+------------+---+
 
 -- Group by processing with `ROLLUP` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model WITH ROLLUP
-   ORDER BY city, car_model;
-
+    GROUP BY city, car_model WITH ROLLUP
+    ORDER BY city, car_model;
   +--------+------------+---+
-  |city    |car_model   |sum|
+  |    city|   car_model|sum|
   +--------+------------+---+
-  |null    |null        |78 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
+  |    null|        null| 78|
+  |  Dublin|        null| 33|
+  |  Dublin| HondaAccord| 10|
+  |  Dublin|    HondaCRV|  3|
+  |  Dublin|  HondaCivic| 20|
+  | Fremont|        null| 32|
+  | Fremont| HondaAccord| 15|
+  | Fremont|    HondaCRV|  7|
+  | Fremont|  HondaCivic| 10|
+  | SanJose|        null| 13|
+  | SanJose| HondaAccord|  8|
+  | SanJose|  HondaCivic|  5|
   +--------+------------+---+
 
 -- Group by processing with `CUBE` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
-   GROUP BY city, car_model WITH CUBE
-   ORDER BY city, car_model;
-
+    GROUP BY city, car_model WITH CUBE
+    ORDER BY city, car_model;
   +--------+------------+---+
-  |city    |car_model   |sum|
+  |    city|   car_model|sum|
   +--------+------------+---+
-  |null    |null        |78 |
-  |null    |Honda Accord|33 |
-  |null    |Honda CRV   |10 |
-  |null    |Honda Civic |35 |
-  |Dublin  |null        |33 |
-  |Dublin  |Honda Accord|10 |
-  |Dublin  |Honda CRV   |3  |
-  |Dublin  |Honda Civic |20 |
-  |Fremont |null        |32 |
-  |Fremont |Honda Accord|15 |
-  |Fremont |Honda CRV   |7  |
-  |Fremont |Honda Civic |10 |
-  |San Jose|null        |13 |
-  |San Jose|Honda Accord|8  |
-  |San Jose|Honda Civic |5  |
+  |    null|        null| 78|
+  |    null| HondaAccord| 33|
+  |    null|    HondaCRV| 10|
+  |    null|  HondaCivic| 35|
+  |  Dublin|        null| 33|
+  |  Dublin| HondaAccord| 10|
+  |  Dublin|    HondaCRV|  3|
+  |  Dublin|  HondaCivic| 20|
+  | Fremont|        null| 32|
+  | Fremont| HondaAccord| 15|
+  | Fremont|    HondaCRV|  7|
+  | Fremont|  HondaCivic| 10|
+  | SanJose|        null| 13|
+  | SanJose| HondaAccord|  8|
+  | SanJose|  HondaCivic|  5|
   +--------+------------+---+
-
 {% endhighlight %}
 
-### Related clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index dee1e3c0e39b9..b84ad17955864 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -18,17 +18,22 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>HAVING</code> clause is used to filter the results produced by
 <code>GROUP BY</code> based on the specified condition. It is often used
 in conjunction with a [GROUP BY](sql-ref-syntax-qry-select-groupby.html)
 clause.
 
 ### Syntax
+
 {% highlight sql %}
 HAVING boolean_expression
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>boolean_expression</em></code></dt>
   <dd>
@@ -47,6 +52,7 @@ HAVING boolean_expression
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
 INSERT INTO dealer VALUES
@@ -61,16 +67,14 @@ INSERT INTO dealer VALUES
 
 -- `HAVING` clause referring to column in `GROUP BY`.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING city = 'Fremont';
-
   +-------+---+
-  |city   |sum|
+  |   city|sum|
   +-------+---+
-  |Fremont|32 |
+  |Fremont| 32|
   +-------+---+
 
 -- `HAVING` clause referring to aggregate function.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum(quantity) > 15;
-
   +-------+---+
   |   city|sum|
   +-------+---+
@@ -80,7 +84,6 @@ SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum(quantity)
 
 -- `HAVING` clause referring to aggregate function by its alias.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum > 15;
-
   +-------+---+
   |   city|sum|
   +-------+---+
@@ -91,16 +94,14 @@ SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum > 15;
 -- `HAVING` clause referring to a different aggregate function than what is present in
 -- `SELECT` list.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING max(quantity) > 15;
-
   +------+---+
-  |city  |sum|
+  |  city|sum|
   +------+---+
-  |Dublin|33 |
+  |Dublin| 33|
   +------+---+
 
 -- `HAVING` clause referring to constant expression.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING 1 > 0 ORDER BY city;
-
   +--------+---+
   |    city|sum|
   +--------+---+
@@ -116,15 +117,15 @@ SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
   +---+
   | 78|
   +---+
-
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index 688ba10c3b1f7..16f4f95f90ea1 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 Join Hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
@@ -55,7 +56,6 @@ Join Hints allow users to suggest the join strategy that Spark should use. Prior
 ### Examples
 
 {% highlight sql %}
-
 -- Join Hints for broadcast join
 SELECT /*+ BROADCAST(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
 SELECT /*+ BROADCASTJOIN (t1) */ * FROM t1 left JOIN t2 ON t1.key = t2.key;
@@ -79,9 +79,9 @@ SELECT /*+ SHUFFLE_REPLICATE_NL(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.ke
 -- org.apache.spark.sql.catalyst.analysis.HintErrorLogger: Hint (strategy=merge)
 -- is overridden by another hint and will not take effect.
 SELECT /*+ BROADCAST(t1) */ /*+ MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
-
 {% endhighlight %}
 
 ### Related Statements
-- [JOIN](sql-ref-syntax-qry-select-join.html)
-- [SELECT](sql-ref-syntax-qry-select.html)
+
+ * [JOIN](sql-ref-syntax-qry-select-join.html)
+ * [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index 4759b12f4db70..41b7603a3a25e 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -18,6 +18,7 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
 ### Description
 
 A SQL join is used to combine rows from two relations based on join criteria. The following section describes the overall join syntax and the sub-sections cover different types of joins along with examples.
@@ -84,6 +85,7 @@ A left join returns all values from the left relation and the matched values fro
 </dd>
 
 #### <b>Right Join </b>
+
 <dd>
 A right join returns all values from the right relation and the matched values from the left relation, or appends NULL if there is no match. It is also referred to as a right outer join.<br><br>
   <b>Syntax:</b><br>
@@ -254,6 +256,7 @@ SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.dept
   +---+-----+------+
 {% endhighlight %}
 
-### Related Statement
+### Related Statements
+
   * [SELECT](sql-ref-syntax-qry-select.html)
   * [Join Hints](sql-ref-syntax-qry-select-hints.html)
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 356930c879d28..0ceb705889b47 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -18,17 +18,22 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>LIMIT</code> clause is used to constrain the number of rows returned by
 the [SELECT](sql-ref-syntax-qry-select.html) statement. In general, this clause
 is used in conjunction with [ORDER BY](sql-ref-syntax-qry-select-orderby.html) to
 ensure that the results are deterministic.
 
 ### Syntax
+
 {% highlight sql %}
 LIMIT { ALL | integer_expression }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>ALL</em></code></dt>
   <dd>
@@ -42,6 +47,7 @@ LIMIT { ALL | integer_expression }
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
@@ -54,31 +60,28 @@ INSERT INTO person VALUES
 
 -- Select the first two rows.
 SELECT name, age FROM person ORDER BY name LIMIT 2;
-
   +------+---+
-  |name  |age|
+  |  name|age|
   +------+---+
-  |Anil B|18 |
-  |Jack N|16 |
+  |Anil B| 18|
+  |Jack N| 16|
   +------+---+
 
 -- Specifying ALL option on LIMIT returns all the rows.
 SELECT name, age FROM person ORDER BY name LIMIT ALL;
-
   +-------+---+
-  |name   |age|
+  |   name|age|
   +-------+---+
-  |Anil B |18 |
-  |Jack N |16 |
-  |John A |18 |
-  |Mike A |25 |
-  |Shone S|16 |
-  |Zen Hui|25 |
+  | Anil B| 18|
+  | Jack N| 16|
+  | John A| 18|
+  | Mike A| 25|
+  |Shone S| 16|
+  |Zen Hui| 25|
   +-------+---+
 
 -- A function expression as an input to LIMIT.
-SELECT name, age FROM person ORDER BY name LIMIT length('SPARK')
-
+SELECT name, age FROM person ORDER BY name LIMIT length('SPARK');
   +-------+---+
   |   name|age|
   +-------+---+
@@ -90,17 +93,17 @@ SELECT name, age FROM person ORDER BY name LIMIT length('SPARK')
   +-------+---+
 
 -- A non-foldable expression as an input to LIMIT is not allowed.
-SELECT name, age FROM person ORDER BY name LIMIT length(name)
-
-org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a constant value ...
+SELECT name, age FROM person ORDER BY name LIMIT length(name);
+  org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a constant value ...
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index eb99dbb06096d..cc75367968053 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -18,16 +18,21 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>ORDER BY</code> clause is used to return the result rows in a sorted manner
 in the user specified order. Unlike the [SORT BY](sql-ref-syntax-qry-select-sortby.html)
 clause, this clause guarantees a total order in the output.
 
 ### Syntax
+
 {% highlight sql %}
 ORDER BY { expression [ sort_direction | nulls_sort_oder ] [ , ... ] }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>ORDER BY</em></code></dt>
   <dd>
@@ -64,6 +69,7 @@ ORDER BY { expression [ sort_direction | nulls_sort_oder ] [ , ... ] }
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
@@ -75,77 +81,73 @@ INSERT INTO person VALUES
 
 -- Sort rows by age. By default rows are sorted in ascending manner with NULL FIRST.
 SELECT name, age FROM person ORDER BY age;
-
   +-----+----+
-  |name |age |
+  | name| age|
   +-----+----+
   |Jerry|null|
-  |Mary |null|
-  |John |30  |
-  |Dan  |50  |
-  |Mike |80  |
+  | Mary|null|
+  | John|  30|
+  |  Dan|  50|
+  | Mike|  80|
   +-----+----+
 
 -- Sort rows in ascending manner keeping null values to be last.
 SELECT name, age FROM person ORDER BY age NULLS LAST;
-
   +-----+----+
-  |name |age |
+  | name| age|
   +-----+----+
-  |John |30  |
-  |Dan  |50  |
-  |Mike |80  |
-  |Mary |null|
+  | John|  30|
+  |  Dan|  50|
+  | Mike|  80|
+  | Mary|null|
   |Jerry|null|
   +-----+----+
 
 -- Sort rows by age in descending manner, which defaults to NULL LAST.
 SELECT name, age FROM person ORDER BY age DESC;
-
   +-----+----+
-  |name |age |
+  | name| age|
   +-----+----+
-  |Mike |80  |
-  |Dan  |50  |
-  |John |30  |
+  | Mike|  80|
+  |  Dan|  50|
+  | John|  30|
   |Jerry|null|
-  |Mary |null|
+  | Mary|null|
   +-----+----+
 
 -- Sort rows in ascending manner keeping null values to be first.
 SELECT name, age FROM person ORDER BY age DESC NULLS FIRST;
-
   +-----+----+
-  |name |age |
+  | name| age|
   +-----+----+
   |Jerry|null|
-  |Mary |null|
-  |Mike |80  |
-  |Dan  |50  |
-  |John |30  |
+  | Mary|null|
+  | Mike|  80|
+  |  Dan|  50|
+  | John|  30|
   +-----+----+
 
 -- Sort rows based on more than one column with each column having different
 -- sort direction.
 SELECT * FROM person ORDER BY name ASC, age DESC;
-
   +---+-----+----+
-  |id |name |age |
+  | id| name| age|
   +---+-----+----+
-  |500|Dan  |50  |
+  |500|  Dan|  50|
   |400|Jerry|null|
-  |100|John |30  |
-  |200|Mary |null|
-  |300|Mike |80  |
+  |100| John|  30|
+  |200| Mary|null|
+  |300| Mike|  80|
   +---+-----+----+
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-setops.md b/docs/sql-ref-syntax-qry-select-setops.md
index 8ed6e4880f7db..09a207a70c149 100644
--- a/docs/sql-ref-syntax-qry-select-setops.md
+++ b/docs/sql-ref-syntax-qry-select-setops.md
@@ -19,161 +19,178 @@ license: |
   limitations under the License.
 ---
 
+### Description
+
 Set operators are used to combine two input relations into a single one. Spark SQL supports three types of set operators:
-- `EXCEPT` or `MINUS`
-- `INTERSECT`
-- `UNION`
+
+ - `EXCEPT` or `MINUS`
+ - `INTERSECT`
+ - `UNION`
 
 Note that input relations must have the same number of columns and compatible data types for the respective columns.
 
 ### EXCEPT
+
 `EXCEPT` and `EXCEPT ALL` return the rows that are found in one relation but not the other. `EXCEPT` (alternatively, `EXCEPT DISTINCT`) takes only distinct rows while `EXCEPT ALL` does not remove duplicates from the result rows. Note that `MINUS` is an alias for `EXCEPT`.
 
 #### Syntax
+
 {% highlight sql %}
 [ ( ] relation [ ) ] EXCEPT | MINUS [ ALL | DISTINCT ] [ ( ] relation [ ) ]
 {% endhighlight %}
 
-### INTERSECT
-`INTERSECT` and `INTERSECT ALL` return the rows that are found in both relations. `INTERSECT` (alternatively, `INTERSECT DISTINCT`) takes only distinct rows while `INTERSECT ALL` does not remove duplicates from the result rows.
+#### Examples
 
-#### Syntax
 {% highlight sql %}
-[ ( ] relation [ ) ] INTERSECT [ ALL | DISTINCT ] [ ( ] relation [ ) ]
-{% endhighlight %}
-
-### UNION
-`UNION` and `UNION ALL` return the rows that are found in either relation. `UNION` (alternatively, `UNION DISTINCT`) takes only distinct rows while `UNION ALL` does not remove duplicates from the result rows.
-
-#### Syntax
-{% highlight sql %}
-[ ( ] relation [ ) ] UNION [ ALL | DISTINCT ] [ ( ] relation [ ) ]
-{% endhighlight %}
-
-### Examples
-{% highlight sql %}
--- Use number1 and number2 tables to demonstrate set operators.
+-- Use number1 and number2 tables to demonstrate set operators in this page.
 SELECT * FROM number1;
-+---+
-|  c|
-+---+
-|  3|
-|  1|
-|  2|
-|  2|
-|  3|
-|  4|
-+---+
-
+  +---+
+  |  c|
+  +---+
+  |  3|
+  |  1|
+  |  2|
+  |  2|
+  |  3|
+  |  4|
+  +---+
+  
 SELECT * FROM number2;
-+---+
-|  c|
-+---+
-|  5|
-|  1|
-|  2|
-|  2|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  5|
+  |  1|
+  |  2|
+  |  2|
+  +---+
 
 SELECT c FROM number1 EXCEPT SELECT c FROM number2;
-+---+
-|  c|
-+---+
-|  3|
-|  4|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  3|
+  |  4|
+  +---+
 
 SELECT c FROM number1 MINUS SELECT c FROM number2;
-+---+
-|  c|
-+---+
-|  3|
-|  4|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  3|
+  |  4|
+  +---+
 
 SELECT c FROM number1 EXCEPT ALL (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  3|
-|  3|
-|  4|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  3|
+  |  3|
+  |  4|
+  +---+
 
 SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  3|
-|  3|
-|  4|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  3|
+  |  3|
+  |  4|
+  +---+
+{% endhighlight %}
 
+### INTERSECT
+
+`INTERSECT` and `INTERSECT ALL` return the rows that are found in both relations. `INTERSECT` (alternatively, `INTERSECT DISTINCT`) takes only distinct rows while `INTERSECT ALL` does not remove duplicates from the result rows.
+
+#### Syntax
+
+{% highlight sql %}
+[ ( ] relation [ ) ] INTERSECT [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+{% endhighlight %}
+
+#### Examples
+
+{% highlight sql %}
 (SELECT c FROM number1) INTERSECT (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  1|
-|  2|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  1|
+  |  2|
+  +---+
 
 (SELECT c FROM number1) INTERSECT DISTINCT (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  1|
-|  2|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  1|
+  |  2|
+  +---+
 
 (SELECT c FROM number1) INTERSECT ALL (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  1|
-|  2|
-|  2|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  1|
+  |  2|
+  |  2|
+  +---+
+{% endhighlight %}
+
+### UNION
 
+`UNION` and `UNION ALL` return the rows that are found in either relation. `UNION` (alternatively, `UNION DISTINCT`) takes only distinct rows while `UNION ALL` does not remove duplicates from the result rows.
+
+#### Syntax
+
+{% highlight sql %}
+[ ( ] relation [ ) ] UNION [ ALL | DISTINCT ] [ ( ] relation [ ) ]
+{% endhighlight %}
+
+### Examples
+
+{% highlight sql %}
 (SELECT c FROM number1) UNION (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  1|
-|  3|
-|  5|
-|  4|
-|  2|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  1|
+  |  3|
+  |  5|
+  |  4|
+  |  2|
+  +---+
 
 (SELECT c FROM number1) UNION DISTINCT (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  1|
-|  3|
-|  5|
-|  4|
-|  2|
-+---+
+  +---+
+  |  c|
+  +---+
+  |  1|
+  |  3|
+  |  5|
+  |  4|
+  |  2|
+  +---+
 
 SELECT c FROM number1 UNION ALL (SELECT c FROM number2);
-+---+
-|  c|
-+---+
-|  3|
-|  1|
-|  2|
-|  2|
-|  3|
-|  4|
-|  5|
-|  1|
-|  2|
-|  2|
-+---+
-
+  +---+
+  |  c|
+  +---+
+  |  3|
+  |  1|
+  |  2|
+  |  2|
+  |  3|
+  |  4|
+  |  5|
+  |  1|
+  |  2|
+  |  2|
+  +---+
 {% endhighlight %}
 
-### Related Statement
-- [SELECT Statement](sql-ref-syntax-qry-select.html)
+### Related Statements
 
+ * [SELECT Statement](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index 9b52738ee7926..315faa5e7d501 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -18,6 +18,9 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>SORT BY</code> clause is used to return the result rows sorted
 within each partition in the user specified order. When there is more than one partition
 <code>SORT BY</code> may return result that is partially ordered. This is different
@@ -25,11 +28,13 @@ than [ORDER BY](sql-ref-syntax-qry-select-orderby.html) clause which guarantees
 total order of the output.
 
 ### Syntax
+
 {% highlight sql %}
 SORT BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>SORT BY</em></code></dt>
   <dd>
@@ -66,6 +71,7 @@ SORT BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE person (zip_code INT, name STRING, age INT);
 INSERT INTO person VALUES
@@ -83,103 +89,98 @@ INSERT INTO person VALUES
 
 -- Sort rows by `name` within each partition in ascending manner
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY name;
-
   +--------+----+--------+
-  |name    |age |zip_code|
+  |    name| age|zip_code|
   +--------+----+--------+
-  |Anil K  |27  |94588   |
-  |Dan Li  |18  |94588   |
-  |John V  |null|94588   |
-  |Zen Hui |50  |94588   |
-  |Aryan B.|18  |94511   |
-  |David K |42  |94511   |
-  |Lalit B.|null|94511   |
+  |  Anil K|  27|   94588|
+  |  Dan Li|  18|   94588|
+  |  John V|null|   94588|
+  | Zen Hui|  50|   94588|
+  |Aryan B.|  18|   94511|
+  | David K|  42|   94511|
+  |Lalit B.|null|   94511|
   +--------+----+--------+
 
 -- Sort rows within each partition using column position.
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY 1;
-
   +--------+----+--------+
-  |name    |age |zip_code|
+  |    name| age|zip_code|
   +--------+----+--------+
-  |Anil K  |27  |94588   |
-  |Dan Li  |18  |94588   |
-  |John V  |null|94588   |
-  |Zen Hui |50  |94588   |
-  |Aryan B.|18  |94511   |
-  |David K |42  |94511   |
-  |Lalit B.|null|94511   |
+  |  Anil K|  27|   94588|
+  |  Dan Li|  18|   94588|
+  |  John V|null|   94588|
+  | Zen Hui|  50|   94588|
+  |Aryan B.|  18|   94511|
+  | David K|  42|   94511|
+  |Lalit B.|null|   94511|
   +--------+----+--------+
 
 -- Sort rows within partition in ascending manner keeping null values to be last.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age NULLS LAST;
-
   +----+--------+--------+
-  |age |name    |zip_code|
+  | age|    name|zip_code|
   +----+--------+--------+
-  |18  |Dan Li  |94588   |
-  |27  |Anil K  |94588   |
-  |50  |Zen Hui |94588   |
-  |null|John V  |94588   |
-  |18  |Aryan B.|94511   |
-  |42  |David K |94511   |
-  |null|Lalit B.|94511   |
+  |  18|  Dan Li|   94588|
+  |  27|  Anil K|   94588|
+  |  50| Zen Hui|   94588|
+  |null|  John V|   94588|
+  |  18|Aryan B.|   94511|
+  |  42| David K|   94511|
+  |null|Lalit B.|   94511|
   +----+--------+--------+
 
 -- Sort rows by age within each partition in descending manner, which defaults to NULL LAST.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC;
-
   +----+--------+--------+
-  |age |name    |zip_code|
+  | age|    name|zip_code|
   +----+--------+--------+
-  |50  |Zen Hui |94588   |
-  |27  |Anil K  |94588   |
-  |18  |Dan Li  |94588   |
-  |null|John V  |94588   |
-  |42  |David K |94511   |
-  |18  |Aryan B.|94511   |
-  |null|Lalit B.|94511   |
+  |  50| Zen Hui|   94588|
+  |  27|  Anil K|   94588|
+  |  18|  Dan Li|   94588|
+  |null|  John V|   94588|
+  |  42| David K|   94511|
+  |  18|Aryan B.|   94511|
+  |null|Lalit B.|   94511|
   +----+--------+--------+
 
 -- Sort rows by age within each partition in descending manner keeping null values to be first.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC NULLS FIRST;
-
   +----+--------+--------+
-  |age |name    |zip_code|
+  | age|    name|zip_code|
   +----+--------+--------+
-  |null|John V  |94588   |
-  |50  |Zen Hui |94588   |
-  |27  |Anil K  |94588   |
-  |18  |Dan Li  |94588   |
-  |null|Lalit B.|94511   |
-  |42  |David K |94511   |
-  |18  |Aryan B.|94511   |
+  |null|  John V|   94588|
+  |  50| Zen Hui|   94588|
+  |  27|  Anil K|   94588|
+  |  18|  Dan Li|   94588|
+  |null|Lalit B.|   94511|
+  |  42| David K|   94511|
+  |  18|Aryan B.|   94511|
   +----+--------+--------+
 
 -- Sort rows within each partition based on more than one column with each column having
 -- different sort direction.
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
-   SORT BY name ASC, age DESC;
-
+    SORT BY name ASC, age DESC;
   +--------+----+--------+
-  |name    |age |zip_code|
+  |    name| age|zip_code|
   +--------+----+--------+
-  |Anil K  |27  |94588   |
-  |Dan Li  |18  |94588   |
-  |John V  |null|94588   |
-  |Zen Hui |50  |94588   |
-  |Aryan B.|18  |94511   |
-  |David K |42  |94511   |
-  |Lalit B.|null|94511   |
+  |  Anil K|  27|   94588|
+  |  Dan Li|  18|   94588|
+  |  John V|null|   94588|
+  | Zen Hui|  50|   94588|
+  |Aryan B.|  18|   94511|
+  | David K|  42|   94511|
+  |Lalit B.|null|   94511|
   +--------+----+--------+
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-usedb.md b/docs/sql-ref-syntax-qry-select-usedb.md
index 92ac91ac51769..2a05085218978 100644
--- a/docs/sql-ref-syntax-qry-select-usedb.md
+++ b/docs/sql-ref-syntax-qry-select-usedb.md
@@ -20,12 +20,14 @@ license: |
 ---
 
 ### Description
+
 `USE` statement is used to set the current database. After the current database is set,
 the unqualified database artifacts such as tables, functions and views that are 
 referenced by SQLs are resolved from the current database. 
 The default database name is 'default'.
 
 ### Syntax
+
 {% highlight sql %}
 USE database_name
 {% endhighlight %}
@@ -40,21 +42,18 @@ USE database_name
 </dl>
 
 ### Example
+
 {% highlight sql %}
 -- Use the 'userdb' which exists.
 USE userdb;
-+---------+--+
-| Result  |
-+---------+--+
-+---------+--+
 
 -- Use the 'userdb1' which doesn't exist
 USE userdb1;
-Error: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: Database 'userdb1' not found;(state=,code=0)
+  Error: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: Database 'userdb1' not found;(state=,code=0)
 {% endhighlight %}
 
-### Related statements.
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-- [CREATE TABLE ](sql-ref-syntax-ddl-create-table.html)
+### Related Statements
 
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+ * [CREATE TABLE ](sql-ref-syntax-ddl-create-table.html)
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index 106053d16f8bd..1960367cd42f0 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -18,15 +18,20 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 The <code>WHERE</code> clause is used to limit the results of the <code>FROM</code>
 clause of a query or a subquery based on the specified condition.
 
 ### Syntax
+
 {% highlight sql %}
 WHERE boolean_expression
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>boolean_expression</em></code></dt>
   <dd>
@@ -37,6 +42,7 @@ WHERE boolean_expression
 </dl>
 
 ### Examples
+
 {% highlight sql %}
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
@@ -48,38 +54,38 @@ INSERT INTO person VALUES
 -- Comparison operator in `WHERE` clause.
 SELECT * FROM person WHERE id > 200 ORDER BY id;
   +---+----+---+
-  |id |name|age|
+  | id|name|age|
   +---+----+---+
-  |300|Mike|80 |
-  |400|Dan |50 |
+  |300|Mike| 80|
+  |400| Dan| 50|
   +---+----+---+
 
 -- Comparison and logical operators in `WHERE` clause.
 SELECT * FROM person WHERE id = 200 OR id = 300 ORDER BY id;
   +---+----+----+
-  |id |name|age |
+  | id|name| age|
   +---+----+----+
   |200|Mary|null|
-  |300|Mike|80  |
+  |300|Mike|  80|
   +---+----+----+
 
 -- IS NULL expression in `WHERE` clause.
 SELECT * FROM person WHERE id > 300 OR age IS NULL ORDER BY id;
   +---+----+----+
-  |id |name|age |
+  | id|name| age|
   +---+----+----+
   |200|Mary|null|
-  |400|Dan |50  |
+  |400| Dan|  50|
   +---+----+----+
 
 -- Function expression in `WHERE` clause.
 SELECT * FROM person WHERE length(name) > 3 ORDER BY id;
   +---+----+----+
-  |id |name|age |
+  | id|name| age|
   +---+----+----+
-  |100|John|30  |
+  |100|John|  30|
   |200|Mary|null|
-  |300|Mike|80  |
+  |300|Mike|  80|
   +---+----+----+
 
 -- `BETWEEN` expression in `WHERE` clause.
@@ -94,31 +100,31 @@ SELECT * FROM person WHERE id BETWEEN 200 AND 300 ORDER BY id;
 -- Scalar Subquery in `WHERE` clause.
 SELECT * FROM person WHERE age > (SELECT avg(age) FROM person);
   +---+----+---+
-  |id |name|age|
+  | id|name|age|
   +---+----+---+
-  |300|Mike|80 |
+  |300|Mike| 80|
   +---+----+---+
 
 -- Correlated Subquery in `WHERE` clause.
 SELECT * FROM person AS parent
-WHERE EXISTS (
-              SELECT 1 FROM person AS child
-              WHERE parent.id = child.id AND child.age IS NULL
-             );
+    WHERE EXISTS (
+        SELECT 1 FROM person AS child
+        WHERE parent.id = child.id AND child.age IS NULL
+    );
   +---+----+----+
   |id |name|age |
   +---+----+----+
   |200|Mary|null|
   +---+----+----+
-
 {% endhighlight %}
 
-### Related Clauses
-- [SELECT Main](sql-ref-syntax-qry-select.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+### Related Statements
+
+ * [SELECT Main](sql-ref-syntax-qry-select.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 8ecc2c630f090..94f69d4d733c4 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -18,33 +18,38 @@ license: |
   See the License for the specific language governing permissions and
   limitations under the License.
 ---
+
+### Description
+
 Spark supports a `SELECT` statement and conforms to the ANSI SQL standard. Queries are
 used to retrieve result sets from one or more tables. The following section
 describes the overall query syntax and the sub-sections cover different constructs
 of a query along with examples.
 
 ### Syntax
+
 {% highlight sql %}
 [ WITH with_query [ , ... ] ]
 select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_statement, ... ]
-[ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-[ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-[ CLUSTER BY { expression [ , ...] } ]
-[ DISTRIBUTE BY { expression [, ...] } ]
-[ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
-[ LIMIT { ALL | expression } ]
+    [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
+    [ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
+    [ CLUSTER BY { expression [ , ...] } ]
+    [ DISTRIBUTE BY { expression [, ...] } ]
+    [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
+    [ LIMIT { ALL | expression } ]
 {% endhighlight %}
 
 While `select_statement` is defined as
 {% highlight sql %}
 SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
-  FROM { from_item [ , ...] }
-  [ WHERE boolean_expression ]
-  [ GROUP BY expression [ , ...] ]
-  [ HAVING boolean_expression ]
+    FROM { from_item [ , ...] }
+    [ WHERE boolean_expression ]
+    [ GROUP BY expression [ , ...] ]
+    [ HAVING boolean_expression ]
 {% endhighlight %}
 
 ### Parameters
+
 <dl>
   <dt><code><em>with_query</em></code></dt>
   <dd>
@@ -141,15 +146,16 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
   </dd>
 </dl>
 
-### Related Clauses
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
-- [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
-- [JOIN](sql-ref-syntax-qry-select-join.html)
-- [SET Operators](sql-ref-syntax-qry-select-setops.html)
+### Related Statements
+
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+ * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+ * [JOIN](sql-ref-syntax-qry-select-join.html)
+ * [SET Operators](sql-ref-syntax-qry-select-setops.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 37414acd57a38..477c347eed800 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -26,13 +26,12 @@ and brief description of supported clauses are explained in
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
-
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
-- [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+ * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index dd611d75d3429..94bd476ffb7b1 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -22,62 +22,66 @@ license: |
 Spark SQL is Apache Spark's module for working with structured data. The SQL Syntax section describes the SQL syntax in detail along with usage examples when applicable. This document provides a list of Data Definition and Data Manipulation Statements, as well as Data Retrieval and Auxiliary Statements.
 
 ### DDL Statements
-- [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
-- [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
-- [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
-- [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
-- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
-- [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
-- [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
-- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
-- [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
-- [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
-- [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+
+ * [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+ * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+ * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+ * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+ * [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+ * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+ * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+ * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+ * [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+ * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+ * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+ * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
+ * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
+ * [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
 
 ### DML Statements
-- [INSERT INTO](sql-ref-syntax-dml-insert-into.html)
-- [INSERT OVERWRITE](sql-ref-syntax-dml-insert-overwrite-table.html)
-- [INSERT OVERWRITE DIRECTORY](sql-ref-syntax-dml-insert-overwrite-directory.html)
-- [INSERT OVERWRITE DIRECTORY with Hive format](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
-- [LOAD](sql-ref-syntax-dml-load.html)
+
+ * [INSERT INTO](sql-ref-syntax-dml-insert-into.html)
+ * [INSERT OVERWRITE](sql-ref-syntax-dml-insert-overwrite-table.html)
+ * [INSERT OVERWRITE DIRECTORY](sql-ref-syntax-dml-insert-overwrite-directory.html)
+ * [INSERT OVERWRITE DIRECTORY with Hive format](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+ * [LOAD](sql-ref-syntax-dml-load.html)
 
 ### Data Retrieval Statements
-- [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-- [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-- [EXPLAIN](sql-ref-syntax-qry-explain.html)
-- [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-- [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-- [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
-- [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-- [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-- [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+
+ * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+ * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [EXPLAIN](sql-ref-syntax-qry-explain.html)
+ * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+ * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+ * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+ * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+ * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
 
 ### Auxiliary Statements
-- [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
-- [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
-- [ANALYZE TABLE](sql-ref-syntax-aux-analyze-table.html)
-- [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
-- [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-- [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
-- [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
-- [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
-- [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
-- [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
-- [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
-- [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
-- [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
-- [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
-- [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
-- [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
-- [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
-- [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
-- [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
-- [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
-- [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
-- [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
-- [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
-- [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-- [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+
+ * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+ * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+ * [ANALYZE TABLE](sql-ref-syntax-aux-analyze-table.html)
+ * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+ * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+ * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+ * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+ * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+ * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+ * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+ * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+ * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+ * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+ * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
+ * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
+ * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+ * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+ * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
+ * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+ * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+ * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
+ * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+ * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+ * [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)

From 4121c43c69c7dee1f9a2d0001a7d91ef590f8f91 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 13 Apr 2020 05:29:54 +0000
Subject: [PATCH 0485/1280] [SPARK-31398][SQL] Fix perf regression of loading
 dates before 1582 year by non-vectorized ORC reader

### What changes were proposed in this pull request?
In regular ORC reader when `spark.sql.orc.enableVectorizedReader` is set to `false`, I propose to use `DaysWritable` in reading DATE values from ORC files. Currently, days from ORC files are converted to java.sql.Date, and then to days in Proleptic Gregorian calendar. So, the conversion to Java type can be eliminated.

### Why are the changes needed?
- The PR fixes regressions in loading dates before the 1582 year from ORC files by when vectorised ORC reader is off.
- The changes improve performance of regular ORC reader for DATE columns.
  - x3.6 faster comparing to the current master
  - x1.9-x4.3 faster against Spark 2.4.6

Before (on JDK 8):
```
Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582, vec off                               39651          39686          31          2.5         396.5       1.0X
after 1582, vec on                                 3647           3660          13         27.4          36.5      10.9X
before 1582, vec off                              38155          38219          61          2.6         381.6       1.0X
before 1582, vec on                                4041           4046           6         24.7          40.4       9.8X
```

After (on JDK 8):
```
Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582, vec off                               10947          10971          28          9.1         109.5       1.0X
after 1582, vec on                                 3677           3702          36         27.2          36.8       3.0X
before 1582, vec off                              11456          11472          21          8.7         114.6       1.0X
before 1582, vec on                                4079           4103          21         24.5          40.8       2.7X
```

Spark 2.4.6:
```
Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582, vec off                               48169          48276          96          2.1         481.7       1.0X
after 1582, vec on                                 5375           5410          41         18.6          53.7       9.0X
before 1582, vec off                              22353          22482         198          4.5         223.5       2.2X
before 1582, vec on                                5474           5475           1         18.3          54.7       8.8X
```

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
- By existing tests suites like `DateTimeUtilsSuite`
- Checked for `hive-1.2` by:
```
./build/sbt -Phive-1.2 "test:testOnly *OrcHadoopFsRelationSuite"
```
- Re-run `DateTimeRebaseBenchmark` in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

Closes #28169 from MaxGekk/orc-optimize-dates.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit cac8d1b3520c1196eb1fe72d9bcd9965604eed4a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 88 +++++++++----------
 .../DateTimeRebaseBenchmark-results.txt       | 88 +++++++++----------
 .../datasources/orc/OrcDeserializer.scala     |  3 +-
 .../datasources/orc/OrcShimUtils.scala        |  6 +-
 .../datasources/orc/OrcShimUtils.scala        |  6 +-
 5 files changed, 96 insertions(+), 95 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index bb7d8b0956d4b..edf9eef5813d1 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  19037          19037           0          5.3         190.4       1.0X
-before 1582, noop                                 11543          11543           0          8.7         115.4       1.6X
-after 1582, rebase off                            31999          31999           0          3.1         320.0       0.6X
-after 1582, rebase on                             31657          31657           0          3.2         316.6       0.6X
-before 1582, rebase off                           23777          23777           0          4.2         237.8       0.8X
-before 1582, rebase on                            24767          24767           0          4.0         247.7       0.8X
+after 1582, noop                                  19946          19946           0          5.0         199.5       1.0X
+before 1582, noop                                 10910          10910           0          9.2         109.1       1.8X
+after 1582, rebase off                            32065          32065           0          3.1         320.7       0.6X
+after 1582, rebase on                             31424          31424           0          3.2         314.2       0.6X
+before 1582, rebase off                           21764          21764           0          4.6         217.6       0.9X
+before 1582, rebase on                            22700          22700           0          4.4         227.0       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12901          13119         211          7.8         129.0       1.0X
-after 1582, vec off, rebase on                    12968          13078         153          7.7         129.7       1.0X
-after 1582, vec on, rebase off                     3633           3673          37         27.5          36.3       3.6X
-after 1582, vec on, rebase on                      4988           5035          42         20.0          49.9       2.6X
-before 1582, vec off, rebase off                  12767          12856          77          7.8         127.7       1.0X
-before 1582, vec off, rebase on                   14012          14051          41          7.1         140.1       0.9X
-before 1582, vec on, rebase off                    3626           3641          15         27.6          36.3       3.6X
-before 1582, vec on, rebase on                     5388           5458         114         18.6          53.9       2.4X
+after 1582, vec off, rebase off                   13054          13273         193          7.7         130.5       1.0X
+after 1582, vec off, rebase on                    13337          13418         117          7.5         133.4       1.0X
+after 1582, vec on, rebase off                     3666           3740         116         27.3          36.7       3.6X
+after 1582, vec on, rebase on                      5097           5160          70         19.6          51.0       2.6X
+before 1582, vec off, rebase off                  12908          12984          76          7.7         129.1       1.0X
+before 1582, vec off, rebase on                   14514          14575          53          6.9         145.1       0.9X
+before 1582, vec on, rebase off                    3718           3790          63         26.9          37.2       3.5X
+before 1582, vec on, rebase on                     5472           5527          48         18.3          54.7       2.4X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   3075           3075           0         32.5          30.7       1.0X
-before 1582, noop                                  3058           3058           0         32.7          30.6       1.0X
-after 1582, rebase off                            17236          17236           0          5.8         172.4       0.2X
-after 1582, rebase on                             19237          19237           0          5.2         192.4       0.2X
-before 1582, rebase off                           17317          17317           0          5.8         173.2       0.2X
-before 1582, rebase on                            19982          19982           0          5.0         199.8       0.2X
+after 1582, noop                                   2718           2718           0         36.8          27.2       1.0X
+before 1582, noop                                  2774           2774           0         36.0          27.7       1.0X
+after 1582, rebase off                            16515          16515           0          6.1         165.2       0.2X
+after 1582, rebase on                             18619          18619           0          5.4         186.2       0.1X
+before 1582, rebase off                           16382          16382           0          6.1         163.8       0.2X
+before 1582, rebase on                            19580          19580           0          5.1         195.8       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   14865          14957         142          6.7         148.6       1.0X
-after 1582, vec off, rebase on                    17154          17190          45          5.8         171.5       0.9X
-after 1582, vec on, rebase off                     4878           4912          56         20.5          48.8       3.0X
-after 1582, vec on, rebase on                      8604           8653          42         11.6          86.0       1.7X
-before 1582, vec off, rebase off                  14891          14929          33          6.7         148.9       1.0X
-before 1582, vec off, rebase on                   17708          17758          68          5.6         177.1       0.8X
-before 1582, vec on, rebase off                    4928           4946          17         20.3          49.3       3.0X
-before 1582, vec on, rebase on                     9147           9152           7         10.9          91.5       1.6X
+after 1582, vec off, rebase off                   15082          15112          43          6.6         150.8       1.0X
+after 1582, vec off, rebase on                    18035          18105         107          5.5         180.3       0.8X
+after 1582, vec on, rebase off                     4948           4970          19         20.2          49.5       3.0X
+after 1582, vec on, rebase on                      8761           8780          25         11.4          87.6       1.7X
+before 1582, vec off, rebase off                  15285          15389         103          6.5         152.8       1.0X
+before 1582, vec off, rebase on                   18277          18368          80          5.5         182.8       0.8X
+before 1582, vec on, rebase off                    4938           4954          15         20.3          49.4       3.1X
+before 1582, vec on, rebase on                     9220           9248          33         10.8          92.2       1.6X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  18943          18943           0          5.3         189.4       1.0X
-before 1582, noop                                 11151          11151           0          9.0         111.5       1.7X
-after 1582                                        26738          26738           0          3.7         267.4       0.7X
-before 1582                                       19636          19636           0          5.1         196.4       1.0X
+after 1582, noop                                  19820          19820           0          5.0         198.2       1.0X
+before 1582, noop                                 10825          10825           0          9.2         108.2       1.8X
+after 1582                                        27705          27705           0          3.6         277.1       0.7X
+before 1582                                       19390          19390           0          5.2         193.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               36176          36378         206          2.8         361.8       1.0X
-after 1582, vec on                                 3802           3829          29         26.3          38.0       9.5X
-before 1582, vec off                              35815          35882          74          2.8         358.1       1.0X
-before 1582, vec on                                4163           4174          10         24.0          41.6       8.7X
+after 1582, vec off                               10401          10426          33          9.6         104.0       1.0X
+after 1582, vec on                                 3788           3813          29         26.4          37.9       2.7X
+before 1582, vec off                              10832          10843          11          9.2         108.3       1.0X
+before 1582, vec on                                4182           4219          46         23.9          41.8       2.5X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   3033           3033           0         33.0          30.3       1.0X
-before 1582, noop                                  3004           3004           0         33.3          30.0       1.0X
-after 1582                                        53907          53907           0          1.9         539.1       0.1X
-before 1582                                       57241          57241           0          1.7         572.4       0.1X
+after 1582, noop                                   2787           2787           0         35.9          27.9       1.0X
+before 1582, noop                                  2777           2777           0         36.0          27.8       1.0X
+after 1582                                        52456          52456           0          1.9         524.6       0.1X
+before 1582                                       54983          54983           0          1.8         549.8       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               38266          38306          66          2.6         382.7       1.0X
-after 1582, vec on                                30233          30339         162          3.3         302.3       1.3X
-before 1582, vec off                              43584          43753         259          2.3         435.8       0.9X
-before 1582, vec on                               35681          35832         140          2.8         356.8       1.1X
+after 1582, vec off                               38336          38425          88          2.6         383.4       1.0X
+after 1582, vec on                                30089          30198          96          3.3         300.9       1.3X
+before 1582, vec off                              42739          42801          87          2.3         427.4       0.9X
+before 1582, vec on                               31530          31591          61          3.2         315.3       1.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index a79f6fba92286..602f3e9dc5287 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  24030          24030           0          4.2         240.3       1.0X
-before 1582, noop                                 10900          10900           0          9.2         109.0       2.2X
-after 1582, rebase off                            35666          35666           0          2.8         356.7       0.7X
-after 1582, rebase on                             35872          35872           0          2.8         358.7       0.7X
-before 1582, rebase off                           22131          22131           0          4.5         221.3       1.1X
-before 1582, rebase on                            22967          22967           0          4.4         229.7       1.0X
+after 1582, noop                                  23453          23453           0          4.3         234.5       1.0X
+before 1582, noop                                 10821          10821           0          9.2         108.2       2.2X
+after 1582, rebase off                            35558          35558           0          2.8         355.6       0.7X
+after 1582, rebase on                             35892          35892           0          2.8         358.9       0.7X
+before 1582, rebase off                           22700          22700           0          4.4         227.0       1.0X
+before 1582, rebase on                            23247          23247           0          4.3         232.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12523          12589          58          8.0         125.2       1.0X
-after 1582, vec off, rebase on                    13387          13466          86          7.5         133.9       0.9X
-after 1582, vec on, rebase off                     3646           3678          41         27.4          36.5       3.4X
-after 1582, vec on, rebase on                      5104           5198         123         19.6          51.0       2.5X
-before 1582, vec off, rebase off                  13041          13101          98          7.7         130.4       1.0X
-before 1582, vec off, rebase on                   14077          14104          23          7.1         140.8       0.9X
-before 1582, vec on, rebase off                    3667           3726          87         27.3          36.7       3.4X
-before 1582, vec on, rebase on                     5504           5521          21         18.2          55.0       2.3X
+after 1582, vec off, rebase off                   12641          12690          43          7.9         126.4       1.0X
+after 1582, vec off, rebase on                    13318          13380          67          7.5         133.2       0.9X
+after 1582, vec on, rebase off                     3648           3659          11         27.4          36.5       3.5X
+after 1582, vec on, rebase on                      5160           5212          69         19.4          51.6       2.4X
+before 1582, vec off, rebase off                  13024          13065          36          7.7         130.2       1.0X
+before 1582, vec off, rebase on                   13810          13932         106          7.2         138.1       0.9X
+before 1582, vec on, rebase off                    3631           3695          57         27.5          36.3       3.5X
+before 1582, vec on, rebase on                     5791           5860          71         17.3          57.9       2.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2782           2782           0         36.0          27.8       1.0X
-before 1582, noop                                  2778           2778           0         36.0          27.8       1.0X
-after 1582, rebase off                            16980          16980           0          5.9         169.8       0.2X
-after 1582, rebase on                             20023          20023           0          5.0         200.2       0.1X
-before 1582, rebase off                           17618          17618           0          5.7         176.2       0.2X
-before 1582, rebase on                            20416          20416           0          4.9         204.2       0.1X
+after 1582, noop                                   2750           2750           0         36.4          27.5       1.0X
+before 1582, noop                                  2833           2833           0         35.3          28.3       1.0X
+after 1582, rebase off                            16832          16832           0          5.9         168.3       0.2X
+after 1582, rebase on                             19688          19688           0          5.1         196.9       0.1X
+before 1582, rebase off                           17548          17548           0          5.7         175.5       0.2X
+before 1582, rebase on                            21343          21343           0          4.7         213.4       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15098          15146          55          6.6         151.0       1.0X
-after 1582, vec off, rebase on                    18127          18214          86          5.5         181.3       0.8X
-after 1582, vec on, rebase off                     4936           4984          42         20.3          49.4       3.1X
-after 1582, vec on, rebase on                      9685           9745          58         10.3          96.8       1.6X
-before 1582, vec off, rebase off                  15233          15259          25          6.6         152.3       1.0X
-before 1582, vec off, rebase on                   18710          18727          24          5.3         187.1       0.8X
-before 1582, vec on, rebase off                    4954           4980          24         20.2          49.5       3.0X
-before 1582, vec on, rebase on                    10224          10240          22          9.8         102.2       1.5X
+after 1582, vec off, rebase off                   15243          15329          82          6.6         152.4       1.0X
+after 1582, vec off, rebase on                    18296          18330          54          5.5         183.0       0.8X
+after 1582, vec on, rebase off                     4925           4927           2         20.3          49.2       3.1X
+after 1582, vec on, rebase on                      9647           9686          35         10.4          96.5       1.6X
+before 1582, vec off, rebase off                  14880          15105         267          6.7         148.8       1.0X
+before 1582, vec off, rebase on                   18474          18514          51          5.4         184.7       0.8X
+before 1582, vec on, rebase off                    4970           4978          10         20.1          49.7       3.1X
+before 1582, vec on, rebase on                     9938          10012          64         10.1          99.4       1.5X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23770          23770           0          4.2         237.7       1.0X
-before 1582, noop                                 10939          10939           0          9.1         109.4       2.2X
-after 1582                                        32879          32879           0          3.0         328.8       0.7X
-before 1582                                       20267          20267           0          4.9         202.7       1.2X
+after 1582, noop                                  23500          23500           0          4.3         235.0       1.0X
+before 1582, noop                                 10788          10788           0          9.3         107.9       2.2X
+after 1582                                        32237          32237           0          3.1         322.4       0.7X
+before 1582                                       20187          20187           0          5.0         201.9       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               39651          39686          31          2.5         396.5       1.0X
-after 1582, vec on                                 3647           3660          13         27.4          36.5      10.9X
-before 1582, vec off                              38155          38219          61          2.6         381.6       1.0X
-before 1582, vec on                                4041           4046           6         24.7          40.4       9.8X
+after 1582, vec off                               10947          10971          28          9.1         109.5       1.0X
+after 1582, vec on                                 3677           3702          36         27.2          36.8       3.0X
+before 1582, vec off                              11456          11472          21          8.7         114.6       1.0X
+before 1582, vec on                                4079           4103          21         24.5          40.8       2.7X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2802           2802           0         35.7          28.0       1.0X
-before 1582, noop                                  2797           2797           0         35.8          28.0       1.0X
-after 1582                                        59877          59877           0          1.7         598.8       0.0X
-before 1582                                       61361          61361           0          1.6         613.6       0.0X
+after 1582, noop                                   2891           2891           0         34.6          28.9       1.0X
+before 1582, noop                                  2906           2906           0         34.4          29.1       1.0X
+after 1582                                        55812          55812           0          1.8         558.1       0.1X
+before 1582                                       57512          57512           0          1.7         575.1       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               48197          48288         118          2.1         482.0       1.0X
-after 1582, vec on                                38247          38351         128          2.6         382.5       1.3X
-before 1582, vec off                              53179          53359         249          1.9         531.8       0.9X
-before 1582, vec on                               44076          44268         269          2.3         440.8       1.1X
+after 1582, vec off                               46376          46410          33          2.2         463.8       1.0X
+after 1582, vec on                                35003          35189         163          2.9         350.0       1.3X
+before 1582, vec off                              52942          52979          34          1.9         529.4       0.9X
+before 1582, vec on                               42596          42747         158          2.3         426.0       1.1X
 
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
index 6d52d40d6dd03..4ab009c6bd014 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcDeserializer.scala
@@ -23,6 +23,7 @@ import org.apache.orc.mapred.{OrcList, OrcMap, OrcStruct, OrcTimestamp}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.RebaseDateTime.rebaseJulianToGregorianDays
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -108,7 +109,7 @@ class OrcDeserializer(
         updater.set(ordinal, bytes)
 
       case DateType => (ordinal, value) =>
-        updater.setInt(ordinal, DateTimeUtils.fromJavaDate(OrcShimUtils.getSqlDate(value)))
+        updater.setInt(ordinal, OrcShimUtils.getGregorianDays(value))
 
       case TimestampType => (ordinal, value) =>
         updater.setLong(ordinal, DateTimeUtils.fromJavaTimestamp(value.asInstanceOf[OrcTimestamp]))
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
index ece528039bba8..7fbc1cd205b13 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.sql.Date
-
 import org.apache.orc.storage.common.`type`.HiveDecimal
 import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch
 import org.apache.orc.storage.ql.io.sarg.{SearchArgument => OrcSearchArgument}
@@ -38,7 +36,9 @@ private[sql] object OrcShimUtils {
   private[sql] type Operator = OrcOperator
   private[sql] type SearchArgument = OrcSearchArgument
 
-  def getSqlDate(value: Any): Date = value.asInstanceOf[DateWritable].get
+  def getGregorianDays(value: Any): Int = {
+    new DaysWritable(value.asInstanceOf[DateWritable]).gregorianDays
+  }
 
   def getDecimal(value: Any): Decimal = {
     val decimal = value.asInstanceOf[HiveDecimalWritable].getHiveDecimal()
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
index 5666d319e1e90..60c5b7a266c51 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcShimUtils.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.sql.Date
-
 import org.apache.hadoop.hive.common.`type`.HiveDecimal
 import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch
 import org.apache.hadoop.hive.ql.io.sarg.{SearchArgument => OrcSearchArgument}
@@ -39,7 +37,9 @@ private[sql] object OrcShimUtils {
   private[sql] type Operator = OrcOperator
   private[sql] type SearchArgument = OrcSearchArgument
 
-  def getSqlDate(value: Any): Date = value.asInstanceOf[DateWritable].get
+  def getGregorianDays(value: Any): Int = {
+    new DaysWritable(value.asInstanceOf[DateWritable]).gregorianDays
+  }
 
   def getDecimal(value: Any): Decimal = {
     val decimal = value.asInstanceOf[HiveDecimalWritable].getHiveDecimal()

From d9ad11110428b2d89fcb2f0c0cbdf84a9970910a Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 13 Apr 2020 06:07:31 +0000
Subject: [PATCH 0486/1280] [SPARK-31402][SQL] Fix rebasing of BCE
 dates/timestamps

In the PR, I propose to fallback to rebasing via local dates/timestamps for days/micros of before common era (BCE).

It fixes the bug of rebasing dates/timestamps of BCE.

Yes

- By existing tests in `RebaseDateTimeSuite` and `DateTimeUtilsSuite`
- Added tests for negative years to `RebaseDateTimeSuite`

Closes #28172 from MaxGekk/fix-era-in-date-micros-rebasing.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit cf63ad61f517a44a2b60c0e4f06cb387569039b3)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    | 96 +++++++++++++++++--
 .../catalyst/util/RebaseDateTimeSuite.scala   | 88 +++++++++--------
 2 files changed, 133 insertions(+), 51 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 74407873b19f6..f4a98cf3585e2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.time.{LocalDateTime, ZoneId}
+import java.time.{LocalDate, LocalDateTime, ZoneId}
 import java.time.temporal.ChronoField
 import java.util.{Calendar, TimeZone}
 
@@ -66,15 +66,53 @@ object RebaseDateTime {
   // The starting point is the `0001-01-01` (-719164 days since the epoch in
   // Julian calendar). All dates before the staring point have the same difference
   // of 2 days in Julian and Proleptic Gregorian calendars.
+  // Rebasing switch days and diffs `julianGregDiffSwitchDay` and `julianGregDiffs`
+  // was generated by the `localRebaseJulianToGregorianDays` function.
   private val julianGregDiffSwitchDay = Array(
     -719164, -682945, -646420, -609895, -536845, -500320, -463795,
     -390745, -354220, -317695, -244645, -208120, -171595, -141427)
 
+  // The first days of Common Era (CE) which is mapped to the '0001-01-01' date in Julian calendar.
+  private final val julianCommonEraStartDay = julianGregDiffSwitchDay(0)
+
   /**
    * Converts the given number of days since the epoch day 1970-01-01 to a local date in Julian
    * calendar, interprets the result as a local date in Proleptic Gregorian calendar, and takes the
    * number of days since the epoch from the Gregorian local date.
    *
+   * @param days The number of days since the epoch in Julian calendar. It can be negative.
+   * @return The rebased number of days in Gregorian calendar.
+   */
+  private[sql] def localRebaseJulianToGregorianDays(days: Int): Int = {
+    val utcCal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setTimeZone(TimeZoneUTC)
+      .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
+      .build()
+    val localDate = LocalDate.of(
+      utcCal.get(Calendar.YEAR),
+      utcCal.get(Calendar.MONTH) + 1,
+      // The number of days will be added later to handle non-existing
+      // Julian dates in Proleptic Gregorian calendar.
+      // For example, 1000-02-29 exists in Julian calendar because 1000
+      // is a leap year but it is not a leap year in Gregorian calendar.
+      1)
+      .`with`(ChronoField.ERA, utcCal.get(Calendar.ERA))
+      .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
+    Math.toIntExact(localDate.toEpochDay)
+  }
+
+  /**
+   * For dates of Before Common Era, the function converts the given number of days since the epoch
+   * day 1970-01-01 to a local date in Julian calendar, interprets the result as a local date in
+   * Proleptic Gregorian calendar, and takes the number of days since the epoch from the Gregorian
+   * local date.
+   *
+   * For dates of Common Era, the function performs rebasing via the pre-calculated array of diffs
+   * between Julian and Proleptic Gregorian calendars.
+   *
    * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
    * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
    * calendar. See SPARK-26651.
@@ -88,7 +126,11 @@ object RebaseDateTime {
    * @return The rebased number of days in Gregorian calendar.
    */
   def rebaseJulianToGregorianDays(days: Int): Int = {
-    rebaseDays(julianGregDiffSwitchDay, julianGregDiffs, days)
+    if (days < julianCommonEraStartDay) {
+      localRebaseJulianToGregorianDays(days)
+    } else {
+      rebaseDays(julianGregDiffSwitchDay, julianGregDiffs, days)
+    }
   }
 
   // The differences in days between Proleptic Gregorian and Julian dates.
@@ -100,15 +142,46 @@ object RebaseDateTime {
   // The starting point is the `0001-01-01` (-719162 days since the epoch in
   // Proleptic Gregorian calendar). All dates before the staring point have the same
   // difference of -2 days in Proleptic Gregorian and Julian calendars.
+  // Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
+  // was generated by the `localRebaseGregorianToJulianDays` function.
   private val gregJulianDiffSwitchDay = Array(
     -719162, -682944, -646420, -609896, -536847, -500323, -463799,
     -390750, -354226, -317702, -244653, -208129, -171605, -141427)
 
+  // The first days of Common Era (CE) which is mapped to the '0001-01-01' date
+  // in Proleptic Gregorian calendar.
+  private final val gregorianCommonEraStartDay = gregJulianDiffSwitchDay(0)
+
   /**
    * Converts the given number of days since the epoch day 1970-01-01 to a local date in Proleptic
    * Gregorian calendar, interprets the result as a local date in Julian calendar, and takes the
    * number of days since the epoch from the Julian local date.
    *
+   * @param days The number of days since the epoch in Proleptic Gregorian calendar.
+   *             It can be negative.
+   * @return The rebased number of days in Julian calendar.
+   */
+  private[sql] def localRebaseGregorianToJulianDays(days: Int): Int = {
+    val localDate = LocalDate.ofEpochDay(days)
+    val utcCal = new Calendar.Builder()
+      // `gregory` is a hybrid calendar that supports both
+      // the Julian and Gregorian calendar systems
+      .setCalendarType("gregory")
+      .setTimeZone(TimeZoneUTC)
+      .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+      .build()
+    Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
+  }
+
+  /**
+   * For dates of Before Common Era, Converts the given number of days since the epoch day
+   * 1970-01-01 to a local date in Proleptic Gregorian calendar, interprets the result as a local
+   * date in Julian calendar, and takes the number of days since the epoch from the Julian
+   * local date.
+   *
+   * For dates of Common Era, the function performs rebasing via the pre-calculated array of diffs
+   * between Proleptic Gregorian and Julian calendars.
+   *
    * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
    * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
    * calendar. See SPARK-26651.
@@ -122,7 +195,11 @@ object RebaseDateTime {
    * @return The rebased number of days since the epoch in Julian calendar.
    */
   def rebaseGregorianToJulianDays(days: Int): Int = {
-    rebaseDays(gregJulianDiffSwitchDay, gregJulianDiffs, days)
+    if (days < gregorianCommonEraStartDay) {
+      localRebaseGregorianToJulianDays(days)
+    } else {
+      rebaseDays(gregJulianDiffSwitchDay, gregJulianDiffs, days)
+    }
   }
 
 
@@ -248,8 +325,8 @@ object RebaseDateTime {
   /**
    * An optimized version of [[rebaseGregorianToJulianMicros(ZoneId, Long)]]. This method leverages
    * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
-   * information about the current JVM system time zone, the function falls back to the regular
-   * unoptimized version.
+   * information about the current JVM system time zone or `micros` is related to Before Common Era,
+   * the function falls back to the regular unoptimized version.
    *
    * Note: The function assumes that the input micros was derived from a local timestamp
    *       at the default system JVM time zone in Proleptic Gregorian calendar.
@@ -262,7 +339,7 @@ object RebaseDateTime {
     val timeZone = TimeZone.getDefault
     val tzId = timeZone.getID
     val rebaseRecord = gregJulianRebaseMap.getOrNull(tzId)
-    if (rebaseRecord == null) {
+    if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
       rebaseGregorianToJulianMicros(timeZone.toZoneId, micros)
     } else {
       rebaseMicros(rebaseRecord, micros)
@@ -308,6 +385,7 @@ object RebaseDateTime {
       cal.get(Calendar.MINUTE),
       cal.get(Calendar.SECOND),
       (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
+      .`with`(ChronoField.ERA, cal.get(Calendar.ERA))
       .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
     val zonedDateTime = localDateTime.atZone(zoneId)
     // Assuming the daylight saving switchover time is 2:00, the local clock will go back to
@@ -335,8 +413,8 @@ object RebaseDateTime {
   /**
    * An optimized version of [[rebaseJulianToGregorianMicros(ZoneId, Long)]]. This method leverages
    * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
-   * information about the current JVM system time zone, the function falls back to the regular
-   * unoptimized version.
+   * information about the current JVM system time zone or `micros` is related to Before Common Era,
+   * the function falls back to the regular unoptimized version.
    *
    * Note: The function assumes that the input micros was derived from a local timestamp
    *       at the default system JVM time zone in the Julian calendar.
@@ -349,7 +427,7 @@ object RebaseDateTime {
     val timeZone = TimeZone.getDefault
     val tzId = timeZone.getID
     val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
-    if (rebaseRecord == null) {
+    if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
       rebaseJulianToGregorianMicros(timeZone.toZoneId, micros)
     } else {
       rebaseMicros(rebaseRecord, micros)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index dcc80b1d6f3e7..1e70e60484850 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util
 
 import java.sql.{Date, Timestamp}
 import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
-import java.util.{Calendar, TimeZone}
+import java.util.TimeZone
 
 import org.scalatest.Matchers
 
@@ -32,15 +32,18 @@ import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 
 class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
 
-  private def parseToJulianMicros(s: String): Long = {
-    val ts = Timestamp.valueOf(s)
+  private def toJulianMicros(ts: Timestamp): Long = {
     val julianMicros = fromMillis(ts.getTime) +
       ((ts.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS)
     julianMicros
   }
+  private def parseToJulianMicros(s: String): Long = toJulianMicros(Timestamp.valueOf(s))
 
+  private def toGregorianMicros(ldt: LocalDateTime, zoneId: ZoneId): Long = {
+    instantToMicros(ldt.atZone(zoneId).toInstant)
+  }
   private def parseToGregMicros(s: String, zoneId: ZoneId): Long = {
-    instantToMicros(LocalDateTime.parse(s).atZone(zoneId).toInstant)
+    toGregorianMicros(LocalDateTime.parse(s), zoneId)
   }
 
   test("rebase julian to/from gregorian micros") {
@@ -141,53 +144,17 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("optimization of days rebasing - Gregorian to Julian") {
-    // Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
-    // in `RebaseDateTime` was generated by using this function.
-    def refRebaseGregorianToJulianDays(days: Int): Int = {
-      val localDate = LocalDate.ofEpochDay(days)
-      val utcCal = new Calendar.Builder()
-        // `gregory` is a hybrid calendar that supports both
-        // the Julian and Gregorian calendar systems
-        .setCalendarType("gregory")
-        .setTimeZone(TimeZoneUTC)
-        .setDate(localDate.getYear, localDate.getMonthValue - 1, localDate.getDayOfMonth)
-        .build()
-      Math.toIntExact(Math.floorDiv(utcCal.getTimeInMillis, MILLIS_PER_DAY))
-    }
-
     val start = localDateToDays(LocalDate.of(1, 1, 1))
     val end = localDateToDays(LocalDate.of(2030, 1, 1))
 
     var days = start
     while (days < end) {
-      assert(rebaseGregorianToJulianDays(days) === refRebaseGregorianToJulianDays(days))
+      assert(rebaseGregorianToJulianDays(days) === localRebaseGregorianToJulianDays(days))
       days += 1
     }
   }
 
   test("optimization of days rebasing - Julian to Gregorian") {
-    // Rebasing switch days and diffs `julianGregDiffSwitchDay` and `julianGregDiffs`
-    // in `RebaseDateTime` was generated by using this function.
-    def refRebaseJulianToGregorianDays(days: Int): Int = {
-      val utcCal = new Calendar.Builder()
-        // `gregory` is a hybrid calendar that supports both
-        // the Julian and Gregorian calendar systems
-        .setCalendarType("gregory")
-        .setTimeZone(TimeZoneUTC)
-        .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
-        .build()
-      val localDate = LocalDate.of(
-        utcCal.get(Calendar.YEAR),
-        utcCal.get(Calendar.MONTH) + 1,
-        // The number of days will be added later to handle non-existing
-        // Julian dates in Proleptic Gregorian calendar.
-        // For example, 1000-02-29 exists in Julian calendar because 1000
-        // is a leap year but it is not a leap year in Gregorian calendar.
-        1)
-        .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
-      Math.toIntExact(localDate.toEpochDay)
-    }
-
     val start = rebaseGregorianToJulianDays(
       localDateToDays(LocalDate.of(1, 1, 1)))
     val end = rebaseGregorianToJulianDays(
@@ -195,7 +162,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
 
     var days = start
     while (days < end) {
-      assert(rebaseJulianToGregorianDays(days) === refRebaseJulianToGregorianDays(days))
+      assert(rebaseJulianToGregorianDays(days) === localRebaseJulianToGregorianDays(days))
       days += 1
     }
   }
@@ -360,4 +327,41 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
       dir = "/Users/maximgekk/tmp",
       fileName = "julian-gregorian-rebase-micros.json")
   }
+
+  test("rebase gregorian to/from julian days - BCE era") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          (-1100, 1, 1),
+          (-1044, 3, 5),
+          (-44, 3, 5)).foreach { case (year, month, day) =>
+          val julianDays = fromJavaDateLegacy(new Date(year - 1900, month - 1, day))
+          val gregorianDays = localDateToDays(LocalDate.of(year, month, day))
+
+          assert(rebaseGregorianToJulianDays(gregorianDays) === julianDays)
+          assert(rebaseJulianToGregorianDays(julianDays) === gregorianDays)
+        }
+      }
+    }
+  }
+
+  test("rebase gregorian to/from julian micros - BCE era") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          (-1100, 1, 1, 1, 2, 3, 0),
+          (-1044, 3, 5, 0, 0, 0, 123456000),
+          (-44, 3, 5, 23, 59, 59, 99999000)
+        ).foreach { case (year, month, day, hour, minute, second, nanos) =>
+          val julianMicros = toJulianMicros(new Timestamp(
+            year - 1900, month - 1, day, hour, minute, second, nanos))
+          val gregorianMicros = toGregorianMicros(LocalDateTime.of(
+            year, month, day, hour, minute, second, nanos), zid)
+
+          assert(rebaseGregorianToJulianMicros(gregorianMicros) === julianMicros)
+          assert(rebaseJulianToGregorianMicros(julianMicros) === gregorianMicros)
+        }
+      }
+    }
+  }
 }

From 94027d47aadb0329650f8ac12ba8d8823a8b9e8e Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 13 Apr 2020 19:04:36 +0900
Subject: [PATCH 0487/1280] [SPARK-31407][SQL][TEST] TestHiveQueryExecution
 should respect database when creating table

### What changes were proposed in this pull request?

In `TestHiveQueryExecution`, if we detect a database in the referenced table, we should create the table under that database.

### Why are the changes needed?

This fix the test `Fix hive/SQLQuerySuite.derived from Hive query file: drop_database_removes_partition_dirs.q` which currently only pass when we run it with the whole test suit but fail when run it separately.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Run the test separately and together with the whole test suite.

Closes #28177 from Ngone51/fix_derived.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 4de8ae1a0f74e24e76ea337285bf5a6aa2a6af29)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/hive/test/TestHive.scala | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index 868bc2712ee71..b9f10c348480d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
 import org.apache.spark.sql.execution.{QueryExecution, SQLExecution}
 import org.apache.spark.sql.execution.command.CacheTableCommand
 import org.apache.spark.sql.hive._
@@ -588,20 +589,29 @@ private[hive] class TestHiveQueryExecution(
 
   override lazy val analyzed: LogicalPlan = {
     val describedTables = logical match {
-      case CacheTableCommand(tbl, _, _, _) => tbl.table :: Nil
+      case CacheTableCommand(tbl, _, _, _) => tbl :: Nil
       case _ => Nil
     }
 
     // Make sure any test tables referenced are loaded.
     val referencedTables =
       describedTables ++
-        logical.collect { case UnresolvedRelation(ident) => ident.last }
+        logical.collect { case UnresolvedRelation(ident) => ident.asTableIdentifier }
     val resolver = sparkSession.sessionState.conf.resolver
-    val referencedTestTables = sparkSession.testTables.keys.filter { testTable =>
-      referencedTables.exists(resolver(_, testTable))
+    val referencedTestTables = referencedTables.flatMap { tbl =>
+      val testTableOpt = sparkSession.testTables.keys.find(resolver(_, tbl.table))
+      testTableOpt.map(testTable => tbl.copy(table = testTable))
+    }
+    logDebug(s"Query references test tables: ${referencedTestTables.map(_.table).mkString(", ")}")
+    referencedTestTables.foreach { tbl =>
+      val curDB = sparkSession.catalog.currentDatabase
+      try {
+        tbl.database.foreach(db => sparkSession.catalog.setCurrentDatabase(db))
+        sparkSession.loadTestTable(tbl.table)
+      } finally {
+        tbl.database.foreach(_ => sparkSession.catalog.setCurrentDatabase(curDB))
+      }
     }
-    logDebug(s"Query references test tables: ${referencedTestTables.mkString(", ")}")
-    referencedTestTables.foreach(sparkSession.loadTestTable)
     // Proceed with analysis.
     sparkSession.sessionState.analyzer.executeAndCheck(logical, tracker)
   }

From 61f5b9a7c09a89dbacdec067cee4aadef069c4ef Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 13 Apr 2020 14:36:25 +0000
Subject: [PATCH 0488/1280] [SPARK-31409][SQL][TEST] Fix failed tests due to
 result order changing when enable AQE

### What changes were proposed in this pull request?

This PR fix two tests by avoid result order changing when we enable AQE:

1. In `SQLQueryTestSuite`, disable BHJ optimization to avoid changing result order

2. In test `SQLQuerySuite#check outputs of expression examples`, disable  `spark.sql.adaptive.coalescePartitions.enabled`  to avoid changing result order

### Why are the changes needed?

query 147 in SQLQueryTestSuite#"udf/postgreSQL/udf-join.sql - Scala UDF" and test sql/SQLQuerySuite#"check outputs of expression examples" can fail when enable AQE due to result order changing. And this PR fix them.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Tested manually with AQE enabled.

Closes #28178 from Ngone51/fix_order.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f6512903da6caef44fd2833327546dd38cc32a59)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql     | 3 +++
 .../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala    | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
index e6fe1078b0d24..77bcfe7e10cbc 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql
@@ -8,6 +8,9 @@
 --
 -- This test file was converted from postgreSQL/join.sql.
 
+-- Disable BroadcastHashJoin optimization to avoid changing result order when we enable AQE
+--SET spark.sql.autoBroadcastJoinThreshold = -1
+
 CREATE OR REPLACE TEMPORARY VIEW INT4_TBL AS SELECT * FROM
   (VALUES (0), (123456), (-123456), (2147483647), (-2147483647))
   AS v(f1);
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 42f0d96347eed..3c514f25b835c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -181,6 +181,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     parFuncs.foreach { funcId =>
       // Examples can change settings. We clone the session to prevent tests clashing.
       val clonedSpark = spark.cloneSession()
+      // Coalescing partitions can change result order, so disable it.
+      clonedSpark.sessionState.conf.setConf(SQLConf.COALESCE_PARTITIONS_ENABLED, false)
       val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId)
       val className = info.getClassName
       if (!ignoreSet.contains(className)) {

From b11a099ca6aa259c0818170fb962fe7cc5b746e2 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 13 Apr 2020 14:40:53 +0000
Subject: [PATCH 0489/1280] [SPARK-31391][SQL][TEST] Add AdaptiveTestUtils to
 ease the test of AQE

### What changes were proposed in this pull request?

This PR adds `AdaptiveTestUtils` to make AQE test simpler, which includes:

`DisableAdaptiveExecution` - a test tag to skip a single test case if AQE is enabled.
`EnableAdaptiveExecutionSuite` - a helper trait to enable AQE for all tests except those tagged with `DisableAdaptiveExecution`.
`DisableAdaptiveExecutionSuite` - a helper trait to disable AQE for all tests.
`assertExceptionMessage` - a method to handle message of normal or AQE exception in a consistent way.
`assertExceptionCause` - a method to handle cause of normal or AQE exception in a consistent way.

### Why are the changes needed?

With this utils, we can:
- reduce much more duplicate codes;
- handle normal or AQE exception in a consistent way;
- improve the stability of AQE tests;

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Updated tests with the util.

Closes #28162 from Ngone51/add_aqe_test_utils.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bbb3cd9c5e659a89c95fe718f8b63a540187fa43)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/ConfigBehaviorSuite.scala       |  9 +-
 .../org/apache/spark/sql/ExplainSuite.scala   | 15 +--
 .../apache/spark/sql/MetadataCacheSuite.scala |  9 +-
 .../org/apache/spark/sql/SubquerySuite.scala  |  8 +-
 .../DeprecatedWholeStageCodegenSuite.scala    | 25 +++--
 .../LogicalPlanTagInSparkPlanSuite.scala      | 18 +---
 .../spark/sql/execution/PlannerSuite.scala    |  8 +-
 .../execution/WholeStageCodegenSuite.scala    | 18 +---
 .../adaptive/AdaptiveTestUtils.scala          | 93 +++++++++++++++++++
 .../datasources/json/JsonSuite.scala          |  4 +-
 .../datasources/orc/OrcQuerySuite.scala       | 25 ++---
 .../sql/execution/debug/DebuggingSuite.scala  | 19 +---
 .../execution/metric/SQLMetricsSuite.scala    | 18 +---
 .../ui/SQLAppStatusListenerSuite.scala        | 14 +--
 .../internal/ExecutorSideSQLConfSuite.scala   |  7 +-
 .../spark/sql/sources/BucketedReadSuite.scala |  3 +-
 .../apache/spark/sql/test/SQLTestUtils.scala  | 17 +++-
 .../sql/hive/HiveMetadataCacheSuite.scala     |  5 +-
 .../sql/hive/execution/HiveExplainSuite.scala | 32 +++----
 .../sql/hive/execution/SQLMetricsSuite.scala  | 19 +---
 .../sql/hive/execution/SQLQuerySuite.scala    |  7 +-
 .../apache/spark/sql/hive/test/TestHive.scala | 13 ++-
 22 files changed, 217 insertions(+), 169 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala
index c3dbbb325d842..36989efbe870d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ConfigBehaviorSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql
 
 import org.apache.commons.math3.stat.inference.ChiSquareTest
 
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -27,7 +28,8 @@ class ConfigBehaviorSuite extends QueryTest with SharedSparkSession {
 
   import testImplicits._
 
-  test("SPARK-22160 spark.sql.execution.rangeExchange.sampleSizePerPartition") {
+  test("SPARK-22160 spark.sql.execution.rangeExchange.sampleSizePerPartition",
+    DisableAdaptiveExecution("Post shuffle partition number can be different")) {
     // In this test, we run a sort and compute the histogram for partition size post shuffle.
     // With a high sample count, the partition size should be more evenly distributed, and has a
     // low chi-sq test value.
@@ -53,11 +55,8 @@ class ConfigBehaviorSuite extends QueryTest with SharedSparkSession {
         dist)
     }
 
-    // When enable AQE, the post partition number is changed.
     // And the ChiSquareTest result is also need updated. So disable AQE.
-    withSQLConf(
-        SQLConf.SHUFFLE_PARTITIONS.key -> numPartitions.toString,
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+    withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> numPartitions.toString) {
       // The default chi-sq value should be low
       assert(computeChiSquareTest() < 100)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index 16c58028adfe2..1a35e5b729c00 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -18,26 +18,15 @@
 package org.apache.spark.sql
 
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
 
-class ExplainSuite extends QueryTest with SharedSparkSession {
+class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiveExecutionSuite {
   import testImplicits._
 
-  var originalValue: String = _
-  protected override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  protected override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
-
   private def getNormalizedExplain(df: DataFrame, mode: ExplainMode): String = {
     val output = new java.io.ByteArrayOutputStream()
     Console.withOut(output) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
index 956bd7861d99d..a9f443be69cb2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
 import java.io.File
 
 import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -55,8 +56,8 @@ abstract class MetadataCacheSuite extends QueryTest with SharedSparkSession {
       val e = intercept[SparkException] {
         df.count()
       }
-      assert(e.getMessage.contains("FileNotFoundException"))
-      assert(e.getMessage.contains("recreating the Dataset/DataFrame involved"))
+      assertExceptionMessage(e, "FileNotFoundException")
+      assertExceptionMessage(e, "recreating the Dataset/DataFrame involved")
     }
   }
 }
@@ -84,8 +85,8 @@ class MetadataCacheV1Suite extends MetadataCacheSuite {
       val e = intercept[SparkException] {
         sql("select count(*) from view_refresh").first()
       }
-      assert(e.getMessage.contains("FileNotFoundException"))
-      assert(e.getMessage.contains("REFRESH"))
+      assertExceptionMessage(e, "FileNotFoundException")
+      assertExceptionMessage(e, "REFRESH")
 
       // Refresh and we should be able to read it again.
       spark.catalog.refreshTable("view_refresh")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index ff8f94c68c5ee..760b03d5e25f8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -22,7 +22,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
 import org.apache.spark.sql.catalyst.plans.logical.{Join, LogicalPlan, Sort}
 import org.apache.spark.sql.execution.{ColumnarToRowExec, ExecSubqueryExpression, FileSourceScanExec, InputAdapter, ReusedSubqueryExec, ScalarSubquery, SubqueryExec, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.datasources.FileScanRDD
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -1357,11 +1357,9 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("SPARK-27279: Reuse Subquery") {
+  test("SPARK-27279: Reuse Subquery", DisableAdaptiveExecution("reuse is dynamic in AQE")) {
     Seq(true, false).foreach { reuse =>
-      withSQLConf(SQLConf.SUBQUERY_REUSE_ENABLED.key -> reuse.toString,
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-        // when enable AQE, the reusedExchange is inserted when executed.
+      withSQLConf(SQLConf.SUBQUERY_REUSE_ENABLED.key -> reuse.toString) {
         val df = sql(
           """
             |SELECT (SELECT avg(key) FROM testData) + (SELECT avg(key) FROM testData)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala
index 1e90754ad7721..b27a940c364a4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/DeprecatedWholeStageCodegenSuite.scala
@@ -18,30 +18,27 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.expressions.scalalang.typed
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
 @deprecated("This test suite will be removed.", "3.0.0")
 class DeprecatedWholeStageCodegenSuite extends QueryTest
   with SharedSparkSession
-  with AdaptiveSparkPlanHelper {
+  with DisableAdaptiveExecutionSuite {
 
   test("simple typed UDAF should be included in WholeStageCodegen") {
-    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-      // With enable AQE, the WholeStageCodegenExec rule is applied when running QueryStageExec.
-      import testImplicits._
+    import testImplicits._
 
-      val ds = Seq(("a", 10), ("b", 1), ("b", 2), ("c", 1)).toDS()
-        .groupByKey(_._1).agg(typed.sum(_._2))
+    val ds = Seq(("a", 10), ("b", 1), ("b", 2), ("c", 1)).toDS()
+      .groupByKey(_._1).agg(typed.sum(_._2))
 
-      val plan = ds.queryExecution.executedPlan
-      assert(find(plan)(p =>
-        p.isInstanceOf[WholeStageCodegenExec] &&
-          p.asInstanceOf[WholeStageCodegenExec].child.isInstanceOf[HashAggregateExec]).isDefined)
-      assert(ds.collect() === Array(("a", 10.0), ("b", 3.0), ("c", 1.0)))
-    }
+    val plan = ds.queryExecution.executedPlan
+    assert(plan.find(p =>
+      p.isInstanceOf[WholeStageCodegenExec] &&
+        p.asInstanceOf[WholeStageCodegenExec].child.isInstanceOf[HashAggregateExec]).isDefined)
+    assert(ds.collect() === Array(("a", 10.0), ("b", 3.0), ("c", 1.0)))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
index 311f84c07a955..5bcec9b1e517c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
@@ -22,6 +22,7 @@ import scala.reflect.ClassTag
 import org.apache.spark.sql.TPCDSQuerySuite
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete, Final}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Generate, Join, LocalRelation, LogicalPlan, Range, Sample, Union, Window}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec}
 import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -29,22 +30,9 @@ import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, DataSourceV
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExchangeExec, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins._
 import org.apache.spark.sql.execution.window.WindowExec
-import org.apache.spark.sql.internal.SQLConf
 
-class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite {
-
-  var originalValue: String = _
-  // when enable AQE, the 'AdaptiveSparkPlanExec' node does not have a logical plan link
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+// Disable AQE because AdaptiveSparkPlanExec does not have a logical plan link
+class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite with DisableAdaptiveExecutionSuite {
 
   override protected def checkGeneratedCode(
       plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
index 0c5e2e3c7d1d4..9c8e44323ce98 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range, Repartition, Sort, Union}
 import org.apache.spark.sql.catalyst.plans.physical._
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecution}
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec}
 import org.apache.spark.sql.execution.exchange.{EnsureRequirements, ReusedExchangeExec, ReuseExchange, ShuffleExchangeExec}
@@ -752,7 +752,8 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
   }
 
   test("SPARK-24556: always rewrite output partitioning in ReusedExchangeExec " +
-    "and InMemoryTableScanExec") {
+    "and InMemoryTableScanExec",
+    DisableAdaptiveExecution("Reuse is dynamic in AQE")) {
     def checkOutputPartitioningRewrite(
         plans: Seq[SparkPlan],
         expectedPartitioningClass: Class[_]): Unit = {
@@ -782,8 +783,7 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
       checkOutputPartitioningRewrite(inMemoryScan, expectedPartitioningClass)
     }
     // when enable AQE, the reusedExchange is inserted when executed.
-    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+    withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
       // ReusedExchange is HashPartitioning
       val df1 = Seq(1 -> "a").toDF("i", "j").repartition($"i")
       val df2 = Seq(1 -> "a").toDF("i", "j").repartition($"i")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
index 06a016fac5300..f7396ee2a89c8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import org.apache.spark.sql.{Dataset, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeAndComment, CodeGenerator}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
@@ -28,23 +29,12 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
 
-class WholeStageCodegenSuite extends QueryTest with SharedSparkSession {
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
+class WholeStageCodegenSuite extends QueryTest with SharedSparkSession
+  with DisableAdaptiveExecutionSuite {
 
   import testImplicits._
 
-  var originalValue: String = _
-  // With on AQE, the WholeStageCodegenExec is added when running QueryStageExec.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
-
   test("range/filter should be combined") {
     val df = spark.range(10).filter("id = 1").selectExpr("id + 1")
     val plan = df.queryExecution.executedPlan
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
new file mode 100644
index 0000000000000..ddaeb57d31547
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.adaptive
+
+import java.io.{PrintWriter, StringWriter}
+
+import org.scalactic.source.Position
+import org.scalatest.Tag
+
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SQLTestUtils
+
+/**
+ * Test with this tag will be ignored if the test suite extends `EnableAdaptiveExecutionSuite`.
+ * Otherwise, it will be executed with adaptive execution disabled.
+ */
+case class DisableAdaptiveExecution(reason: String) extends Tag("DisableAdaptiveExecution")
+
+/**
+ * Helper trait that enables AQE for all tests regardless of default config values, except that
+ * tests tagged with [[DisableAdaptiveExecution]] will be skipped.
+ */
+trait EnableAdaptiveExecutionSuite extends SQLTestUtils {
+  protected val forceApply = true
+
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+      (implicit pos: Position): Unit = {
+    if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
+      // we ignore the test here but assume that another test suite which extends
+      // `DisableAdaptiveExecutionSuite` will test it anyway to ensure test coverage
+      ignore(testName + " (disabled when AQE is on)", testTags: _*)(testFun)
+    } else {
+      super.test(testName, testTags: _*) {
+        withSQLConf(
+          SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+          SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> forceApply.toString) {
+          testFun
+        }
+      }
+    }
+  }
+}
+
+/**
+ * Helper trait that disables AQE for all tests regardless of default config values.
+ */
+trait DisableAdaptiveExecutionSuite extends SQLTestUtils {
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+      (implicit pos: Position): Unit = {
+    super.test(testName, testTags: _*) {
+      withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+        testFun
+      }
+    }
+  }
+}
+
+object AdaptiveTestUtils {
+  def assertExceptionMessage(e: Exception, expected: String): Unit = {
+    val stringWriter = new StringWriter()
+    e.printStackTrace(new PrintWriter(stringWriter))
+    val errorMsg = stringWriter.toString
+    assert(errorMsg.contains(expected))
+  }
+
+  def assertExceptionCause(t: Throwable, causeClass: Class[_]): Unit = {
+    var c = t.getCause
+    var foundCause = false
+    while (c != null && !foundCause) {
+      if (causeClass.isAssignableFrom(c.getClass)) {
+        foundCause = true
+      } else {
+        c = c.getCause
+      }
+    }
+    assert(foundCause, s"Can not find cause: $causeClass")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 54668005a72dc..be95f336598f4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.{functions => F, _}
 import org.apache.spark.sql.catalyst.json._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.ExternalRDD
+import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -2192,9 +2193,8 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         .json(testFile(fileName))
         .count()
     }
-    val errMsg = exception.getMessage
 
-    assert(errMsg.contains("Malformed records are detected in record parsing"))
+    assertExceptionMessage(exception, "Malformed records are detected in record parsing")
   }
 
   def checkEncoding(expectedEncoding: String, pathToJsonFiles: String,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
index 5f495263b30e3..60f278b8e5bb0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
@@ -34,6 +34,7 @@ import org.apache.orc.mapreduce.OrcInputFormat
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, RecordReaderIterator}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -595,22 +596,22 @@ abstract class OrcQueryTest extends OrcTest {
     }
 
     withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "false") {
-      val m1 = intercept[SparkException] {
+      val e1 = intercept[SparkException] {
         testIgnoreCorruptFiles()
-      }.getMessage
-      assert(m1.contains("Malformed ORC file"))
-      val m2 = intercept[SparkException] {
+      }
+      assertExceptionMessage(e1, "Malformed ORC file")
+      val e2 = intercept[SparkException] {
         testIgnoreCorruptFilesWithoutSchemaInfer()
-      }.getMessage
-      assert(m2.contains("Malformed ORC file"))
-      val m3 = intercept[SparkException] {
+      }
+      assertExceptionMessage(e2, "Malformed ORC file")
+      val e3 = intercept[SparkException] {
         testAllCorruptFiles()
-      }.getMessage
-      assert(m3.contains("Could not read footer for file"))
-      val m4 = intercept[SparkException] {
+      }
+      assertExceptionMessage(e3, "Could not read footer for file")
+      val e4 = intercept[SparkException] {
         testAllCorruptFilesWithoutSchemaInfer()
-      }.getMessage
-      assert(m4.contains("Malformed ORC file"))
+      }
+      assertExceptionMessage(e4, "Malformed ORC file")
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
index 4cb845b2487d6..e9ef7c1a0c540 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/debug/DebuggingSuite.scala
@@ -24,27 +24,14 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
 import org.apache.spark.sql.execution.{CodegenSupport, LeafExecNode, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.functions._
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.test.SQLTestData.TestData
 import org.apache.spark.sql.types.StructType
 
-class DebuggingSuite extends SharedSparkSession {
-
-
-  var originalValue: String = _
-  // With on AQE, the WholeStageCodegenExec is added when running QueryStageExec.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+// Disable AQE because the WholeStageCodegenExec is added when running QueryStageExec
+class DebuggingSuite extends SharedSparkSession with DisableAdaptiveExecutionSuite {
 
   test("DataFrame.debug()") {
     testData.debug()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index a5b07d56c03e6..1c4a8189349ea 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial}
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.execution.{FilterExec, RangeExec, SparkPlan, WholeStageCodegenExec}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.functions._
@@ -33,22 +34,11 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.util.{AccumulatorContext, JsonProtocol}
 
-class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils {
+// Disable AQE because metric info is different with AQE on/off
+class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
+  with DisableAdaptiveExecutionSuite {
   import testImplicits._
 
-  var originalValue: String = _
-  // With AQE on/off, the metric info is different.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
-
   /**
    * Generates a `DataFrame` by filling randomly generated bytes for hash collision.
    */
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
index 949924e962bb9..0746059365004 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
@@ -37,6 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.quietly
 import org.apache.spark.sql.execution.{LeafExecNode, QueryExecution, SparkPlanInfo, SQLExecution}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.functions.count
 import org.apache.spark.sql.internal.SQLConf
@@ -623,13 +624,12 @@ class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTestUtils
     assert(statusStore.execution(2) === None)
   }
 
-  test("SPARK-29894 test Codegen Stage Id in SparkPlanInfo") {
-    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-      // with AQE on, the WholeStageCodegen rule is applied when running QueryStageExec.
-      val df = createTestDataFrame.select(count("*"))
-      val sparkPlanInfo = SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)
-      assert(sparkPlanInfo.nodeName === "WholeStageCodegen (2)")
-    }
+  test("SPARK-29894 test Codegen Stage Id in SparkPlanInfo",
+    DisableAdaptiveExecution("WSCG rule is applied later in AQE")) {
+    // with AQE on, the WholeStageCodegen rule is applied when running QueryStageExec.
+    val df = createTestDataFrame.select(count("*"))
+    val sparkPlanInfo = SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)
+    assert(sparkPlanInfo.nodeName === "WholeStageCodegen (2)")
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
index 888772c35d0ee..567524ac75c2e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/ExecutorSideSQLConfSuite.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.execution.{LeafExecNode, QueryExecution, SparkPlan}
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.debug.codegenStringSeq
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.test.SQLTestUtils
@@ -98,10 +99,10 @@ class ExecutorSideSQLConfSuite extends SparkFunSuite with SQLTestUtils {
     }
   }
 
-  test("SPARK-22219: refactor to control to generate comment") {
+  test("SPARK-22219: refactor to control to generate comment",
+    DisableAdaptiveExecution("WSCG rule is applied later in AQE")) {
     Seq(true, false).foreach { flag =>
-      withSQLConf(StaticSQLConf.CODEGEN_COMMENTS.key -> flag.toString,
-        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      withSQLConf(StaticSQLConf.CODEGEN_COMMENTS.key -> flag.toString) {
         // with AQE on, the WholeStageCodegen rule is applied when running QueryStageExec.
         val res = codegenStringSeq(spark.range(10).groupBy(col("id") * 2).count()
           .queryExecution.executedPlan)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index 14ba008e89961..707d9c2245570 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
 import org.apache.spark.sql.execution.{DataSourceScanExec, FileSourceScanExec, SortExec, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
+import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.BucketingUtils
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.joins.SortMergeJoinExec
@@ -758,7 +759,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         agged.count()
       }
 
-      assert(error.getCause().toString contains "Invalid bucket file")
+      assertExceptionMessage(error, "Invalid bucket file")
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
index 38893f846e5a4..7be15e9d87004 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
@@ -27,7 +27,8 @@ import scala.language.implicitConversions
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.Path
-import org.scalatest.{BeforeAndAfterAll, Suite}
+import org.scalactic.source.Position
+import org.scalatest.{BeforeAndAfterAll, Suite, Tag}
 import org.scalatest.concurrent.Eventually
 
 import org.apache.spark.SparkFunSuite
@@ -40,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTestBase
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.execution.FilterExec
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.util.UninterruptibleThread
@@ -114,6 +116,19 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
     }
   }
 
+  override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
+      (implicit pos: Position): Unit = {
+    if (testTags.exists(_.isInstanceOf[DisableAdaptiveExecution])) {
+      super.test(testName, testTags: _*) {
+        withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+          testFun
+        }
+      }
+    } else {
+      super.test(testName, testTags: _*)(testFun)
+    }
+  }
+
   /**
    * Run a test on a separate `UninterruptibleThread`.
    */
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
index 94a55b911f092..743cdbd6457d7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.Path
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
@@ -99,7 +100,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assert(e.getMessage.contains("FileNotFoundException"))
+            assertExceptionMessage(e, "FileNotFoundException")
 
             // Test refreshing the cache.
             spark.catalog.refreshTable("test")
@@ -114,7 +115,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e2 = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assert(e2.getMessage.contains("FileNotFoundException"))
+            assertExceptionMessage(e2, "FileNotFoundException")
             spark.catalog.refreshByPath(dir.getAbsolutePath)
             assert(sql("select * from test").count() == 3)
           }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
index f9a4e2cd210e3..7a913e99fdbe6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.parser.ParseException
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecution
 import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
@@ -133,22 +134,21 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       "src")
   }
 
-  test("explain output of physical plan should contain proper codegen stage ID") {
-    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
-      checkKeywordsExist(sql(
-        """
-          |EXPLAIN SELECT t1.id AS a, t2.id AS b FROM
-          |(SELECT * FROM range(3)) t1 JOIN
-          |(SELECT * FROM range(10)) t2 ON t1.id == t2.id % 3
-        """.stripMargin),
-        "== Physical Plan ==",
-        "*(2) Project ",
-        "+- *(2) BroadcastHashJoin ",
-        "   :- BroadcastExchange ",
-        "   :  +- *(1) Range ",
-        "   +- *(2) Range "
-      )
-    }
+  test("explain output of physical plan should contain proper codegen stage ID",
+    DisableAdaptiveExecution("Adaptive explain is different")) {
+    checkKeywordsExist(sql(
+      """
+        |EXPLAIN SELECT t1.id AS a, t2.id AS b FROM
+        |(SELECT * FROM range(3)) t1 JOIN
+        |(SELECT * FROM range(10)) t2 ON t1.id == t2.id % 3
+      """.stripMargin),
+      "== Physical Plan ==",
+      "*(2) Project ",
+      "+- *(2) BroadcastHashJoin ",
+      "   :- BroadcastExchange ",
+      "   :  +- *(1) Range ",
+      "   +- *(2) Range "
+    )
   }
 
   test("EXPLAIN CODEGEN command") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
index 16668f93bd4e7..4d6dafd598a2e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLMetricsSuite.scala
@@ -17,24 +17,13 @@
 
 package org.apache.spark.sql.hive.execution
 
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.execution.metric.SQLMetricsTestUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.SQLConf
 
-class SQLMetricsSuite extends SQLMetricsTestUtils with TestHiveSingleton {
-
-  var originalValue: String = _
-  // With AQE on/off, the metric info is different.
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    originalValue = spark.conf.get(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key)
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false")
-  }
-
-  override def afterAll(): Unit = {
-    spark.conf.set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, originalValue)
-    super.afterAll()
-  }
+// Disable AQE because metric info is different with AQE on/off
+class SQLMetricsSuite extends SQLMetricsTestUtils with TestHiveSingleton
+  with DisableAdaptiveExecutionSuite {
 
   test("writing data out metrics: hive") {
     testMetricsNonDynamicPartition("hive", "t1")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 65c1db553e0d7..138dcc586a46a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, Functio
 import org.apache.spark.sql.catalyst.catalog.{CatalogTableType, CatalogUtils, HiveTableRelation}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.command.{FunctionsCommand, LoadDataCommand}
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation}
 import org.apache.spark.sql.functions._
@@ -67,7 +68,7 @@ case class Order(
  * Hive to generate them (in contrast to HiveQuerySuite).  Often this is because the query is
  * valid, but Hive currently cannot execute it.
  */
-class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHiveSingleton {
   import hiveContext._
   import spark.implicits._
 
@@ -2493,3 +2494,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
     }
   }
 }
+
+class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite
+class SQLQuerySuiteAE extends SQLQuerySuiteBase with EnableAdaptiveExecutionSuite
+
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
index b9f10c348480d..a3e2444cae887 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -234,16 +234,16 @@ private[hive] class TestHiveSparkSession(
    * Dataset.ofRows that creates a TestHiveQueryExecution (rather than a normal QueryExecution
    * which wouldn't load all the test tables).
    */
-  override def sql(sqlText: String): DataFrame = {
+  override def sql(sqlText: String): DataFrame = withActive {
     val plan = sessionState.sqlParser.parsePlan(sqlText)
     Dataset.ofRows(self, plan)
   }
 
-  override def newSession(): TestHiveSparkSession = {
+  override def newSession(): TestHiveSparkSession = withActive {
     new TestHiveSparkSession(sc, Some(sharedState), None, loadTestTables)
   }
 
-  override def cloneSession(): SparkSession = {
+  override def cloneSession(): SparkSession = withActive {
     val result = new TestHiveSparkSession(
       sparkContext,
       Some(sharedState),
@@ -264,7 +264,10 @@ private[hive] class TestHiveSparkSession(
   System.clearProperty("spark.hostPort")
 
   // For some hive test case which contain ${system:test.tmp.dir}
-  System.setProperty("test.tmp.dir", Utils.createTempDir().toURI.getPath)
+  // Make sure it is not called again when cloning sessions.
+  if (parentSessionState.isEmpty) {
+    System.setProperty("test.tmp.dir", Utils.createTempDir().toURI.getPath)
+  }
 
   /** The location of the compiled hive distribution */
   lazy val hiveHome = envVarToFile("HIVE_HOME")
@@ -587,7 +590,7 @@ private[hive] class TestHiveQueryExecution(
     this(TestHive.sparkSession, sql)
   }
 
-  override lazy val analyzed: LogicalPlan = {
+  override lazy val analyzed: LogicalPlan = sparkSession.withActive {
     val describedTables = logical match {
       case CacheTableCommand(tbl, _, _, _) => tbl :: Nil
       case _ => Nil

From 43fedb861733bddea32936a7cbc1d4e3d7a2c9e4 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 14 Apr 2020 10:22:46 +0900
Subject: [PATCH 0490/1280] [SPARK-31434][SQL][DOCS] Drop builtin function
 pages from SQL references

### What changes were proposed in this pull request?

This PR intends to drop the built-in function pages from SQL references. We've already had a complete list of built-in functions in the API documents.

See related discussions for more details:
https://github.com/apache/spark/pull/28170#issuecomment-611917191

### Why are the changes needed?

For better SQL documents.

### Does this PR introduce any user-facing change?

![functions](https://user-images.githubusercontent.com/692303/79109009-793e5400-7db2-11ea-8cb7-4c3cf31ccb77.png)

### How was this patch tested?

Manually checked.

Closes #28203 from maropu/DropBuiltinFunctionDocs.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 853c6c990999ecb35e4a2b3222d47be3e209eb81)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/_data/menu-sql.yaml                    | 24 +++++--------------
 docs/sql-ref-functions-builtin-aggregate.md | 22 -----------------
 docs/sql-ref-functions-builtin-array.md     | 22 -----------------
 docs/sql-ref-functions-builtin-date-time.md | 22 -----------------
 docs/sql-ref-functions-builtin-scalar.md    | 22 -----------------
 docs/sql-ref-functions-builtin.md           | 26 ---------------------
 docs/sql-ref-functions-udf.md               | 26 ---------------------
 docs/sql-ref-functions.md                   | 11 ++++++++-
 8 files changed, 16 insertions(+), 159 deletions(-)
 delete mode 100644 docs/sql-ref-functions-builtin-aggregate.md
 delete mode 100644 docs/sql-ref-functions-builtin-array.md
 delete mode 100644 docs/sql-ref-functions-builtin-date-time.md
 delete mode 100644 docs/sql-ref-functions-builtin-scalar.md
 delete mode 100644 docs/sql-ref-functions-builtin.md
 delete mode 100644 docs/sql-ref-functions-udf.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index b8c6e50e53fb0..badb98d2e593b 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -242,23 +242,11 @@
     - text: Functions
       url: sql-ref-functions.html
       subitems:
-      - text: Build-in Functions
-        url: sql-ref-functions-builtin.html
-        subitems:
-        - text: Build-in Aggregate Functions
-          url: sql-ref-functions-builtin-aggregate.html
-        - text: Build-in Array Functions
-          url: sql-ref-functions-builtin-array.html
-        - text: Build-in Date Time Functions
-          url: sql-ref-functions-builtin-date-time.html
-      - text: UDFs (User-Defined Functions)
-        url: sql-ref-functions-udf.html
-        subitems:
-        - text: Scalar UDFs (User-Defined Functions)
-          url: sql-ref-functions-udf-scalar.html
-        - text: UDAFs (User-Defined Aggregate Functions)
-          url: sql-ref-functions-udf-aggregate.html
-        - text: Integration with Hive UDFs/UDAFs/UDTFs
-          url: sql-ref-functions-udf-hive.html
+      - text: Scalar UDFs (User-Defined Functions)
+        url: sql-ref-functions-udf-scalar.html
+      - text: UDAFs (User-Defined Aggregate Functions)
+        url: sql-ref-functions-udf-aggregate.html
+      - text: Integration with Hive UDFs/UDAFs/UDTFs
+        url: sql-ref-functions-udf-hive.html
     - text: Datetime Pattern
       url: sql-ref-datetime-pattern.html
diff --git a/docs/sql-ref-functions-builtin-aggregate.md b/docs/sql-ref-functions-builtin-aggregate.md
deleted file mode 100644
index d59543647e025..0000000000000
--- a/docs/sql-ref-functions-builtin-aggregate.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Built-in Aggregate Functions
-displayTitle: Built-in Aggregate Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Aggregate functions
\ No newline at end of file
diff --git a/docs/sql-ref-functions-builtin-array.md b/docs/sql-ref-functions-builtin-array.md
deleted file mode 100644
index 599d10c23b616..0000000000000
--- a/docs/sql-ref-functions-builtin-array.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Built-in Array Functions
-displayTitle: Built-in Array Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Array Functions
diff --git a/docs/sql-ref-functions-builtin-date-time.md b/docs/sql-ref-functions-builtin-date-time.md
deleted file mode 100644
index 10421324a3448..0000000000000
--- a/docs/sql-ref-functions-builtin-date-time.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Built-in Date and Time Functions
-displayTitle: Built-in Date and Time Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Date-Time Functions
diff --git a/docs/sql-ref-functions-builtin-scalar.md b/docs/sql-ref-functions-builtin-scalar.md
deleted file mode 100644
index 1d818a25c4ac1..0000000000000
--- a/docs/sql-ref-functions-builtin-scalar.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Builtin Scalar Functions
-displayTitle: Builtin Scalar Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
deleted file mode 100644
index 917c081adb7a0..0000000000000
--- a/docs/sql-ref-functions-builtin.md
+++ /dev/null
@@ -1,26 +0,0 @@
----
-layout: global
-title: Built-in Functions
-displayTitle: Built-in Functions
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-Spark SQL defines built-in functions to use, a complete list of which can be found [here](api/sql/). Among them, Spark SQL has several special categories of built-in functions: [Aggregate Functions](sql-ref-functions-builtin-aggregate.html) to operate on a group of rows, [Array Functions](sql-ref-functions-builtin-array.html) to operate on Array columns, and [Date and Time Functions](sql-ref-functions-builtin-date-time.html) to operate on Date and Time.
-
- * [Aggregate Functions](sql-ref-functions-builtin-aggregate.html)
- * [Array Functions](sql-ref-functions-builtin-array.html)
- * [Date and Time Functions](sql-ref-functions-builtin-date-time.html)
diff --git a/docs/sql-ref-functions-udf.md b/docs/sql-ref-functions-udf.md
deleted file mode 100644
index 2c5f204d25736..0000000000000
--- a/docs/sql-ref-functions-udf.md
+++ /dev/null
@@ -1,26 +0,0 @@
----
-layout: global
-title: UDFs (User-Defined Functions)
-displayTitle: UDFs (User-Defined Functions)
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.
-
- * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
- * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
- * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index dabf90bc52855..e8a0353579301 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -19,4 +19,13 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL defines commonly used functions as built-in functions. It also provides User-Defined Functions (UDFs) to allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.
+Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs).
+Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.
+
+### UDFs (User-Defined Functions)
+
+User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.
+
+ * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+ * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+ * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)

From b6b19db2098ddbeda8609e0609e839154a1bb483 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 14 Apr 2020 10:25:37 +0900
Subject: [PATCH 0491/1280] [SPARK-31414][SQL][DOCS][FOLLOWUP] Update default
 datetime pattern for json/csv APIs documentations

### What changes were proposed in this pull request?

Update default datetime pattern from `yyyy-MM-dd'T'HH:mm:ss.SSSXXX ` to `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX] ` for JSON/CSV APIs documentations

### Why are the changes needed?

doc fix

### Does this PR introduce any user-facing change?

Yes, the documentation will change

### How was this patch tested?

Passing Jenkins

Closes #28204 from yaooqinn/SPARK-31414-F.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 31b907748d21f056828a475ca6161aebf2bda536)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/readwriter.py                          | 8 ++++----
 python/pyspark/sql/streaming.py                           | 4 ++--
 .../main/scala/org/apache/spark/sql/DataFrameReader.scala | 4 ++--
 .../main/scala/org/apache/spark/sql/DataFrameWriter.scala | 4 ++--
 .../org/apache/spark/sql/streaming/DataStreamReader.scala | 4 ++--
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 92d36e7c3b1ae..6ad6377288ec5 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -245,7 +245,7 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
                           set, it uses the default value, ``false``.
         :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
@@ -455,7 +455,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
                            set, it uses the default value, ``20480``.
         :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
@@ -892,7 +892,7 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param encoding: specifies encoding (charset) of saved json files. If None is set,
                         the default UTF-8 charset will be used.
         :param lineSep: defines the line separator that should be used for writing. If None is
@@ -996,7 +996,7 @@ def csv(self, path, mode=None, compression=None, sep=None, quote=None, escape=No
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param ignoreLeadingWhiteSpace: a flag indicating whether or not leading whitespaces from
                                         values being written should be skipped. If None is set, it
                                         uses the default value, ``true``.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 4d36a04c038c7..05cf331d897a2 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -483,7 +483,7 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param multiLine: parse one record, which may span multiple lines, per file. If None is
                           set, it uses the default value, ``false``.
         :param allowUnquotedControlChars: allows JSON Strings to contain unquoted control
@@ -696,7 +696,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param timestampFormat: sets the string that indicates a timestamp format.
                                 Custom date formats follow the formats at `datetime pattern`_.
                                 This applies to timestamp type. If None is set, it uses the
-                                default value, ``yyyy-MM-dd'T'HH:mm:ss.SSSXXX``.
+                                default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``.
         :param maxColumns: defines a hard limit of how many columns a record can have. If None is
                            set, it uses the default value, ``20480``.
         :param maxCharsPerColumn: defines the maximum number of characters allowed for any given
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 83e567834313a..a7b3d08ec4df7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -428,7 +428,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
@@ -657,7 +657,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index d11e4db982035..9aef382470bcc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -790,7 +790,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
@@ -915,7 +915,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index a2eaed8693a4b..1d7e4d3059c68 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -290,7 +290,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
@@ -360,7 +360,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.
    * This applies to date type.</li>
-   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`): sets the string that
+   * <li>`timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that
    * indicates a timestamp format. Custom date formats follow the formats at
    * <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">
    *   Datetime Patterns</a>.

From 21f46e828b3c4c57548d2e9862e2f6dba4082b5b Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Tue, 14 Apr 2020 13:29:44 +0900
Subject: [PATCH 0492/1280] [SPARK-31319][SQL][FOLLOW-UP] Add a SQL example for
 UDAF

### What changes were proposed in this pull request?
Add a SQL example for UDAF

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes.
Add the following page, also change ```Sql``` to ```SQL``` in the example tab for all the sql examples.
<img width="1110" alt="Screen Shot 2020-04-13 at 6 09 24 PM" src="https://user-images.githubusercontent.com/13592258/79175240-06cd7400-7db2-11ea-8f3e-af71a591a64b.png">

### How was this patch tested?
Manually build and check

Closes #28209 from huaxingao/udf_followup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 46be1e01e977788f00f1f5aa8d64bc5f191bc578)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-data-sources-jdbc.md                |  2 +-
 docs/sql-data-sources-json.md                |  2 +-
 docs/sql-data-sources-load-save-functions.md |  8 ++---
 docs/sql-data-sources-parquet.md             |  4 +--
 docs/sql-getting-started.md                  |  2 +-
 docs/sql-performance-tuning.md               |  2 +-
 docs/sql-ref-datatypes.md                    |  2 +-
 docs/sql-ref-functions-udf-aggregate.md      | 37 ++++++++++++++++++++
 8 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/docs/sql-data-sources-jdbc.md b/docs/sql-data-sources-jdbc.md
index 3cdff42057ce1..eaee6706e5b30 100644
--- a/docs/sql-data-sources-jdbc.md
+++ b/docs/sql-data-sources-jdbc.md
@@ -217,7 +217,7 @@ the following case-insensitive options:
 {% include_example jdbc_dataset r/RSparkSQLExample.R %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
diff --git a/docs/sql-data-sources-json.md b/docs/sql-data-sources-json.md
index 588f6cdaa0ad5..0f1ca432b704a 100644
--- a/docs/sql-data-sources-json.md
+++ b/docs/sql-data-sources-json.md
@@ -77,7 +77,7 @@ For a regular multi-line JSON file, set a named parameter `multiLine` to `TRUE`.
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
diff --git a/docs/sql-data-sources-load-save-functions.md b/docs/sql-data-sources-load-save-functions.md
index a7efb9347ac64..0866f37fbdc71 100644
--- a/docs/sql-data-sources-load-save-functions.md
+++ b/docs/sql-data-sources-load-save-functions.md
@@ -127,7 +127,7 @@ visit the official Apache ORC/Parquet websites.
 {% include_example manual_save_options_orc r/RSparkSQLExample.R %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 CREATE TABLE users_with_options (
@@ -257,7 +257,7 @@ Bucketing and sorting are applicable only to persistent tables:
 {% include_example write_sorting_and_bucketing python/sql/datasource.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -291,7 +291,7 @@ while partitioning can be used with both `save` and `saveAsTable` when using the
 {% include_example write_partitioning python/sql/datasource.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -323,7 +323,7 @@ It is possible to use both partitioning and bucketing for a single table:
 {% include_example write_partition_and_bucket python/sql/datasource.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
diff --git a/docs/sql-data-sources-parquet.md b/docs/sql-data-sources-parquet.md
index 6e52446c9e39e..7875b10b0f697 100644
--- a/docs/sql-data-sources-parquet.md
+++ b/docs/sql-data-sources-parquet.md
@@ -52,7 +52,7 @@ Using the data from the above example:
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
@@ -242,7 +242,7 @@ refreshTable("my_table")
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 REFRESH TABLE my_table;
diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index fc0a5d01ec3af..dab34afaaabb2 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -205,7 +205,7 @@ refer it, e.g. `SELECT * FROM global_temp.view1`.
 {% include_example global_temp_view python/sql/basic.py %}
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 
diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 279aad6ae8496..5b784a56d476c 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -169,7 +169,7 @@ head(join(src, hint(records, "broadcast"), src$key == records$key))
 
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 {% highlight sql %}
 -- We accept BROADCAST, BROADCASTJOIN and MAPJOIN for broadcast hint
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 1e0d0511a114f..150e194ae6cde 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -631,7 +631,7 @@ from pyspark.sql.types import *
 </table>
 </div>
 
-<div data-lang="sql"  markdown="1">
+<div data-lang="SQL"  markdown="1">
 
 The following table shows the type names as well as aliases used in Spark SQL parser for each data type.
 
diff --git a/docs/sql-ref-functions-udf-aggregate.md b/docs/sql-ref-functions-udf-aggregate.md
index 3d8a64ea3039c..3fde94d6bc4bf 100644
--- a/docs/sql-ref-functions-udf-aggregate.md
+++ b/docs/sql-ref-functions-udf-aggregate.md
@@ -94,8 +94,45 @@ For example, a user-defined average for untyped DataFrames can look like:
 <div data-lang="java"  markdown="1">
   {% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
 </div>
+<div data-lang="SQL"  markdown="1">
+{% highlight sql %}
+-- Compile and place UDAF MyAverage in a JAR file called `MyAverage.jar` in /tmp.
+CREATE FUNCTION myAverage AS 'MyAverage' USING JAR '/tmp/MyAverage.jar';
+
+SHOW USER FUNCTIONS;
+-- +------------------+
+-- |          function|
+-- +------------------+
+-- | default.myAverage|
+-- +------------------+
+
+CREATE TEMPORARY VIEW employees
+USING org.apache.spark.sql.json
+OPTIONS (
+    path "examples/src/main/resources/employees.json"
+);
+
+SELECT * FROM employees;
+-- +-------+------+
+-- |   name|salary|
+-- +-------+------+
+-- |Michael|  3000|
+-- |   Andy|  4500|
+-- | Justin|  3500|
+-- |  Berta|  4000|
+-- +-------+------+
+
+SELECT myAverage(salary) as average_salary FROM employees;
+-- +--------------+
+-- |average_salary|
+-- +--------------+
+-- |        3750.0|
+-- +--------------+
+{% endhighlight %}
+</div>
 </div>
 
 ### Related Statements
+
  * [Scalar User Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
  * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)

From fef8721949286bf8443db99df51cc9c0648703bc Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 14 Apr 2020 04:50:20 +0000
Subject: [PATCH 0493/1280] [SPARK-31426][SQL] Fix perf regressions of
 toJavaTimestamp/fromJavaTimestamp

### What changes were proposed in this pull request?
Reuse the `rebaseGregorianToJulianMicros()` and `rebaseJulianToGregorianMicros()` functions introduced by the PR #28119 in `DateTimeUtils`.`toJavaTimestamp()` and `fromJavaTimestamp()`. Actually, new implementation is derived from Spark 2.4 + rebasing via pre-calculated rebasing maps.

### Why are the changes needed?
The changes speed up conversions to/from java.sql.Timestamp, and as a consequence the PR improve performance of ORC datasource in loading/saving timestamps:
- Saving ~ **x2.8 faster** in master, and -11% against Spark 2.4.6
- Loading - **x3.2-4.5 faster** in master, -5% against Spark 2.4.6

Before:
```
Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582                                        59877          59877           0          1.7         598.8       0.0X
before 1582                                       61361          61361           0          1.6         613.6       0.0X

Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582, vec off                               48197          48288         118          2.1         482.0       1.0X
after 1582, vec on                                38247          38351         128          2.6         382.5       1.3X
before 1582, vec off                              53179          53359         249          1.9         531.8       0.9X
before 1582, vec on                               44076          44268         269          2.3         440.8       1.1X
```

After:
```
Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582                                        21250          21250           0          4.7         212.5       0.1X
before 1582                                       22105          22105           0          4.5         221.0       0.1X

Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582, vec off                               14903          14933          40          6.7         149.0       1.0X
after 1582, vec on                                 8342           8426          73         12.0          83.4       1.8X
before 1582, vec off                              15528          15575          76          6.4         155.3       1.0X
before 1582, vec on                                9025           9075          61         11.1          90.2       1.7X
```

Spark 2.4.6-SNAPSHOT:
```
Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582                                        18858          18858           0          5.3         188.6       1.0X
before 1582                                       18508          18508           0          5.4         185.1       1.0X

Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
after 1582, vec off                               14063          14177         143          7.1         140.6       1.0X
after 1582, vec on                                 5955           6029         100         16.8          59.5       2.4X
before 1582, vec off                              14119          14126           7          7.1         141.2       1.0X
before 1582, vec on                                5991           6007          25         16.7          59.9       2.3X
```

### Does this PR introduce any user-facing change?
Yes, the `to_utc_timestamp` function returns the later local timestamp in the case of overlapping local timestamps at daylight saving time. it's changed back to the 2.4 behavior.

### How was this patch tested?
- By existing test suite `DateTimeUtilsSuite`, `RebaseDateTimeSuite`, `DateFunctionsSuite`, `DateExpressionsSuites`, `ParquetIOSuite`, `OrcHadoopFsRelationSuite`.
- Re-generating results of the benchmarks `DateTimeBenchmark` and `DateTimeRebaseBenchmark` in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

Closes #28189 from MaxGekk/optimize-to-from-java-timestamp.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a0f8cc08a3ca006adf2e63c7c8d5f266dba46328)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     |  21 +-
 .../catalyst/util/DateTimeUtilsSuite.scala    |   4 +-
 .../DateTimeBenchmark-jdk11-results.txt       | 218 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 218 +++++++++---------
 .../DateTimeRebaseBenchmark-jdk11-results.txt |  88 +++----
 .../DateTimeRebaseBenchmark-results.txt       |  88 +++----
 6 files changed, 316 insertions(+), 321 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 33e05c5c20a42..7ce0fa22d7c54 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -150,8 +150,12 @@ object DateTimeUtils {
    * @return A `java.sql.Timestamp` from number of micros since epoch.
    */
   def toJavaTimestamp(us: SQLTimestamp): Timestamp = {
-    val ldt = microsToInstant(us).atZone(ZoneId.systemDefault()).toLocalDateTime
-    Timestamp.valueOf(ldt)
+    val rebasedMicros = rebaseGregorianToJulianMicros(us)
+    val seconds = Math.floorDiv(rebasedMicros, MICROS_PER_SECOND)
+    val ts = new Timestamp(seconds * MILLIS_PER_SECOND)
+    val nanos = (rebasedMicros - seconds * MICROS_PER_SECOND) * NANOS_PER_MICROS
+    ts.setNanos(nanos.toInt)
+    ts
   }
 
   /**
@@ -174,17 +178,8 @@ object DateTimeUtils {
    * @return The number of micros since epoch from `java.sql.Timestamp`.
    */
   def fromJavaTimestamp(t: Timestamp): SQLTimestamp = {
-    val era = if (t.before(julianCommonEraStart)) 0 else 1
-    val localDateTime = LocalDateTime.of(
-      t.getYear + 1900, t.getMonth + 1, 1,
-      t.getHours, t.getMinutes, t.getSeconds, t.getNanos)
-      .`with`(ChronoField.ERA, era)
-      // Add days separately to convert dates existed in Julian calendar but not
-      // in Proleptic Gregorian calendar. For example, 1000-02-29 is valid date
-      // in Julian calendar because 1000 is a leap year but 1000 is not a leap
-      // year in Proleptic Gregorian calendar. And 1000-02-29 doesn't exist in it.
-      .plusDays(t.getDate - 1) // Returns the next valid date after `date.getDate - 1` days
-    instantToMicros(localDateTime.atZone(ZoneId.systemDefault).toInstant)
+    val micros = millisToMicros(t.getTime) + (t.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS
+    rebaseJulianToGregorianMicros(micros)
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 7459044e12429..6a1ee9f72e1a6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -496,8 +496,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
       test("2016-03-13 02:00:00", tz, "2016-03-13 10:00:00.0")
       test("2016-03-13 03:00:00", tz, "2016-03-13 10:00:00.0")
       test("2016-11-06 00:59:59", tz, "2016-11-06 07:59:59.0")
-      test("2016-11-06 01:00:00", tz, "2016-11-06 08:00:00.0")
-      test("2016-11-06 01:59:59", tz, "2016-11-06 08:59:59.0")
+      test("2016-11-06 01:00:00", tz, "2016-11-06 09:00:00.0")
+      test("2016-11-06 01:59:59", tz, "2016-11-06 09:59:59.0")
       test("2016-11-06 02:00:00", tz, "2016-11-06 10:00:00.0")
     }
   }
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index df9735cb7b0d1..27917c204f577 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    411            449          54         24.3          41.1       1.0X
-cast to timestamp wholestage on                     392            408          11         25.5          39.2       1.0X
+cast to timestamp wholestage off                    447            486          54         22.4          44.7       1.0X
+cast to timestamp wholestage on                     364            381          14         27.5          36.4       1.2X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1368           1376          11          7.3         136.8       1.0X
-year of timestamp wholestage on                    1272           1293          21          7.9         127.2       1.1X
+year of timestamp wholestage off                   1366           1374          11          7.3         136.6       1.0X
+year of timestamp wholestage on                    1278           1302          21          7.8         127.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1607           1632          34          6.2         160.7       1.0X
-quarter of timestamp wholestage on                 1534           1553          22          6.5         153.4       1.0X
+quarter of timestamp wholestage off                1606           1622          23          6.2         160.6       1.0X
+quarter of timestamp wholestage on                 1569           1584          16          6.4         156.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1252           1259          10          8.0         125.2       1.0X
-month of timestamp wholestage on                   1270           1282          13          7.9         127.0       1.0X
+month of timestamp wholestage off                  1243           1259          24          8.0         124.3       1.0X
+month of timestamp wholestage on                   1267           1282          13          7.9         126.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             2234           2242          12          4.5         223.4       1.0X
-weekofyear of timestamp wholestage on              1830           1844          10          5.5         183.0       1.2X
+weekofyear of timestamp wholestage off             1837           1846          12          5.4         183.7       1.0X
+weekofyear of timestamp wholestage on              1842           1848           4          5.4         184.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1248           1256          11          8.0         124.8       1.0X
-day of timestamp wholestage on                     1248           1255          10          8.0         124.8       1.0X
+day of timestamp wholestage off                    1262           1289          39          7.9         126.2       1.0X
+day of timestamp wholestage on                     1263           1268           6          7.9         126.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1290           1303          18          7.8         129.0       1.0X
-dayofyear of timestamp wholestage on               1285           1306          20          7.8         128.5       1.0X
+dayofyear of timestamp wholestage off              1302           1308           8          7.7         130.2       1.0X
+dayofyear of timestamp wholestage on               1300           1308           8          7.7         130.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1261           1263           2          7.9         126.1       1.0X
-dayofmonth of timestamp wholestage on              1253           1259           5          8.0         125.3       1.0X
+dayofmonth of timestamp wholestage off             1244           1248           6          8.0         124.4       1.0X
+dayofmonth of timestamp wholestage on              1253           1263          10          8.0         125.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1419           1421           2          7.0         141.9       1.0X
-dayofweek of timestamp wholestage on               1406           1417          11          7.1         140.6       1.0X
+dayofweek of timestamp wholestage off              1407           1408           2          7.1         140.7       1.0X
+dayofweek of timestamp wholestage on               1411           1432          14          7.1         141.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1346           1350           6          7.4         134.6       1.0X
-weekday of timestamp wholestage on                 1348           1360          10          7.4         134.8       1.0X
+weekday of timestamp wholestage off                1349           1352           5          7.4         134.9       1.0X
+weekday of timestamp wholestage on                 1347           1354          10          7.4         134.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1051           1057           8          9.5         105.1       1.0X
-hour of timestamp wholestage on                     991           1002           9         10.1          99.1       1.1X
+hour of timestamp wholestage off                   1038           1041           5          9.6         103.8       1.0X
+hour of timestamp wholestage on                     983            995           9         10.2          98.3       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1041           1042           1          9.6         104.1       1.0X
-minute of timestamp wholestage on                   971            978           9         10.3          97.1       1.1X
+minute of timestamp wholestage off                 1042           1043           1          9.6         104.2       1.0X
+minute of timestamp wholestage on                  1022           1030           9          9.8         102.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  989            991           3         10.1          98.9       1.0X
-second of timestamp wholestage on                   986            999          12         10.1          98.6       1.0X
+second of timestamp wholestage off                  978            979           2         10.2          97.8       1.0X
+second of timestamp wholestage on                  1016           1028          21          9.8         101.6       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         305            313          12         32.8          30.5       1.0X
-current_date wholestage on                          302            316          17         33.1          30.2       1.0X
+current_date wholestage off                         295            299           6         33.9          29.5       1.0X
+current_date wholestage on                          321            327           7         31.2          32.1       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    359            388          41         27.8          35.9       1.0X
-current_timestamp wholestage on                     300            412         143         33.3          30.0       1.2X
+current_timestamp wholestage off                    342            387          63         29.2          34.2       1.0X
+current_timestamp wholestage on                     324            376          97         30.9          32.4       1.1X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1080           1083           5          9.3         108.0       1.0X
-cast to date wholestage on                         1045           1055          10          9.6         104.5       1.0X
+cast to date wholestage off                        1065           1067           3          9.4         106.5       1.0X
+cast to date wholestage on                         1044           1053          10          9.6         104.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1253           1253           1          8.0         125.3       1.0X
-last_day wholestage on                             1257           1272          20          8.0         125.7       1.0X
+last_day wholestage off                            1241           1245           6          8.1         124.1       1.0X
+last_day wholestage on                             1276           1287          11          7.8         127.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1129           1137          11          8.9         112.9       1.0X
-next_day wholestage on                             1083           1092          11          9.2         108.3       1.0X
+next_day wholestage off                            1130           1150          28          8.8         113.0       1.0X
+next_day wholestage on                             1086           1096           8          9.2         108.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1067           1071           6          9.4         106.7       1.0X
-date_add wholestage on                             1061           1072           7          9.4         106.1       1.0X
+date_add wholestage off                            1048           1054           9          9.5         104.8       1.0X
+date_add wholestage on                             1077           1079           2          9.3         107.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1065           1068           4          9.4         106.5       1.0X
-date_sub wholestage on                             1065           1072           6          9.4         106.5       1.0X
+date_sub wholestage off                            1047           1052           7          9.6         104.7       1.0X
+date_sub wholestage on                             1078           1087           7          9.3         107.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1404           1409           7          7.1         140.4       1.0X
-add_months wholestage on                           1429           1439          15          7.0         142.9       1.0X
+add_months wholestage off                          1392           1421          41          7.2         139.2       1.0X
+add_months wholestage on                           1441           1447           6          6.9         144.1       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5642           5741         140          1.8         564.2       1.0X
-format date wholestage on                          5874           5900          18          1.7         587.4       1.0X
+format date wholestage off                         6034           6047          19          1.7         603.4       1.0X
+format date wholestage on                          5874           5891          16          1.7         587.4       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7504           7515          15          1.3         750.4       1.0X
-from_unixtime wholestage on                        7579           7612          37          1.3         757.9       1.0X
+from_unixtime wholestage off                       7350           7371          29          1.4         735.0       1.0X
+from_unixtime wholestage on                        7415           7428          13          1.3         741.5       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1250           1254           6          8.0         125.0       1.0X
-from_utc_timestamp wholestage on                   1287           1294          10          7.8         128.7       1.0X
+from_utc_timestamp wholestage off                  1235           1236           2          8.1         123.5       1.0X
+from_utc_timestamp wholestage on                   1300           1309          10          7.7         130.0       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1916           1918           2          5.2         191.6       1.0X
-to_utc_timestamp wholestage on                     1781           1792          12          5.6         178.1       1.1X
+to_utc_timestamp wholestage off                    1798           1805          10          5.6         179.8       1.0X
+to_utc_timestamp wholestage on                     1765           1779          11          5.7         176.5       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        347            354           9         28.8          34.7       1.0X
-cast interval wholestage on                         308            313           3         32.5          30.8       1.1X
+cast interval wholestage off                        345            350           7         29.0          34.5       1.0X
+cast interval wholestage on                         325            337          11         30.7          32.5       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1884           1895          15          5.3         188.4       1.0X
-datediff wholestage on                             1824           1833           9          5.5         182.4       1.0X
+datediff wholestage off                            1833           1834           2          5.5         183.3       1.0X
+datediff wholestage on                             1807           1815           9          5.5         180.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5674           5685          16          1.8         567.4       1.0X
-months_between wholestage on                       5600           5616          14          1.8         560.0       1.0X
+months_between wholestage off                      5938           5941           4          1.7         593.8       1.0X
+months_between wholestage on                       5891           5920          19          1.7         589.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1971           2180         295          0.5        1971.1       1.0X
-window wholestage on                              47323          47354          35          0.0       47322.7       0.0X
+window wholestage off                              2153           2313         226          0.5        2153.3       1.0X
+window wholestage on                              47097          47143          31          0.0       47097.0       0.0X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2571           2588          23          3.9         257.1       1.0X
-date_trunc YEAR wholestage on                      2581           2591           6          3.9         258.1       1.0X
+date_trunc YEAR wholestage off                     2458           2461           5          4.1         245.8       1.0X
+date_trunc YEAR wholestage on                      2433           2441           6          4.1         243.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2564           2568           5          3.9         256.4       1.0X
-date_trunc YYYY wholestage on                      2583           2592           6          3.9         258.3       1.0X
+date_trunc YYYY wholestage off                     2454           2457           4          4.1         245.4       1.0X
+date_trunc YYYY wholestage on                      2436           2442           7          4.1         243.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2574           2577           5          3.9         257.4       1.0X
-date_trunc YY wholestage on                        2579           2588           8          3.9         257.9       1.0X
+date_trunc YY wholestage off                       2467           2469           3          4.1         246.7       1.0X
+date_trunc YY wholestage on                        2435           2448          18          4.1         243.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2598           2599           1          3.8         259.8       1.0X
-date_trunc MON wholestage on                       2593           2604          11          3.9         259.3       1.0X
+date_trunc MON wholestage off                      2494           2494           1          4.0         249.4       1.0X
+date_trunc MON wholestage on                       2428           2441          14          4.1         242.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2599           2613          19          3.8         259.9       1.0X
-date_trunc MONTH wholestage on                     2604           2610           5          3.8         260.4       1.0X
+date_trunc MONTH wholestage off                    2502           2510          12          4.0         250.2       1.0X
+date_trunc MONTH wholestage on                     2438           2444           6          4.1         243.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2605           2613          10          3.8         260.5       1.0X
-date_trunc MM wholestage on                        2595           2601           4          3.9         259.5       1.0X
+date_trunc MM wholestage off                       2486           2487           1          4.0         248.6       1.0X
+date_trunc MM wholestage on                        2436           2441           4          4.1         243.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2361           2367           9          4.2         236.1       1.0X
-date_trunc DAY wholestage on                       2370           2383          11          4.2         237.0       1.0X
+date_trunc DAY wholestage off                      2406           2421          21          4.2         240.6       1.0X
+date_trunc DAY wholestage on                       2352           2370          23          4.3         235.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2376           2385          13          4.2         237.6       1.0X
-date_trunc DD wholestage on                        2368           2372           3          4.2         236.8       1.0X
+date_trunc DD wholestage off                       2414           2417           5          4.1         241.4       1.0X
+date_trunc DD wholestage on                        2355           2366           8          4.2         235.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2399           2404           7          4.2         239.9       1.0X
-date_trunc HOUR wholestage on                      2387           2399          14          4.2         238.7       1.0X
+date_trunc HOUR wholestage off                     2419           2426          10          4.1         241.9       1.0X
+date_trunc HOUR wholestage on                      2369           2374           4          4.2         236.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    357            370          18         28.0          35.7       1.0X
-date_trunc MINUTE wholestage on                     373            378           5         26.8          37.3       1.0X
+date_trunc MINUTE wholestage off                    357            378          30         28.0          35.7       1.0X
+date_trunc MINUTE wholestage on                     331            335           4         30.2          33.1       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    382            382           1         26.2          38.2       1.0X
-date_trunc SECOND wholestage on                     372            380           8         26.9          37.2       1.0X
+date_trunc SECOND wholestage off                    373            375           3         26.8          37.3       1.0X
+date_trunc SECOND wholestage on                     330            335           4         30.3          33.0       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2501           2512          14          4.0         250.1       1.0X
-date_trunc WEEK wholestage on                      2484           2498          12          4.0         248.4       1.0X
+date_trunc WEEK wholestage off                     2479           2480           2          4.0         247.9       1.0X
+date_trunc WEEK wholestage on                      2441           2450           8          4.1         244.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3407           3410           4          2.9         340.7       1.0X
-date_trunc QUARTER wholestage on                   3371           3378           5          3.0         337.1       1.0X
+date_trunc QUARTER wholestage off                  3337           3337           0          3.0         333.7       1.0X
+date_trunc QUARTER wholestage on                   3235           3244           8          3.1         323.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           318            319           2         31.4          31.8       1.0X
-trunc year wholestage on                            343            347           8         29.2          34.3       0.9X
+trunc year wholestage off                           315            316           1         31.7          31.5       1.0X
+trunc year wholestage on                            293            302          16         34.2          29.3       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           315            316           2         31.7          31.5       1.0X
-trunc yyyy wholestage on                            342            355          18         29.2          34.2       0.9X
+trunc yyyy wholestage off                           312            314           3         32.0          31.2       1.0X
+trunc yyyy wholestage on                            295            300           8         33.9          29.5       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             317            318           1         31.5          31.7       1.0X
-trunc yy wholestage on                              339            348          12         29.5          33.9       0.9X
+trunc yy wholestage off                             311            317           8         32.1          31.1       1.0X
+trunc yy wholestage on                              294            302           7         34.0          29.4       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            317            319           3         31.5          31.7       1.0X
-trunc mon wholestage on                             337            349          10         29.7          33.7       0.9X
+trunc mon wholestage off                            312            314           3         32.0          31.2       1.0X
+trunc mon wholestage on                             296            299           5         33.8          29.6       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          315            316           1         31.7          31.5       1.0X
-trunc month wholestage on                           341            352          14         29.3          34.1       0.9X
+trunc month wholestage off                          311            311           0         32.1          31.1       1.0X
+trunc month wholestage on                           293            297           4         34.2          29.3       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             314            315           1         31.8          31.4       1.0X
-trunc mm wholestage on                              343            348           8         29.2          34.3       0.9X
+trunc mm wholestage off                             315            320           7         31.7          31.5       1.0X
+trunc mm wholestage on                              291            296           8         34.4          29.1       1.1X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     163            165           2          6.1         163.2       1.0X
-to timestamp str wholestage on                      158            159           1          6.3         157.7       1.0X
+to timestamp str wholestage off                     161            171          15          6.2         160.6       1.0X
+to timestamp str wholestage on                      152            154           1          6.6         151.6       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1490           1491           2          0.7        1489.5       1.0X
-to_timestamp wholestage on                         1522           1530          10          0.7        1522.0       1.0X
+to_timestamp wholestage off                        1441           1450          13          0.7        1440.6       1.0X
+to_timestamp wholestage on                         1371           1381          20          0.7        1370.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1561           1574          18          0.6        1561.4       1.0X
-to_unix_timestamp wholestage on                    1531           1540           9          0.7        1531.4       1.0X
+to_unix_timestamp wholestage off                   1329           1335           9          0.8        1328.7       1.0X
+to_unix_timestamp wholestage on                    1354           1357           4          0.7        1353.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          206            219          18          4.9         206.1       1.0X
-to date str wholestage on                           211            214           4          4.7         211.1       1.0X
+to date str wholestage off                          212            213           2          4.7         211.7       1.0X
+to date str wholestage on                           206            207           1          4.9         205.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3195           3208          18          0.3        3195.3       1.0X
-to_date wholestage on                              3179           3189          12          0.3        3179.2       1.0X
+to_date wholestage off                             3255           3309          76          0.3        3255.0       1.0X
+to_date wholestage on                              3146           3168          23          0.3        3146.4       1.0X
 
 
 ================================================================================================
@@ -422,8 +422,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                            1390           1400           9          3.6         277.9       1.0X
-Collect longs                                      1781           2342         496          2.8         356.2       0.8X
-Collect timestamps                                 5837           6037         180          0.9        1167.3       0.2X
+From java.sql.Timestamp                             359            365           8         13.9          71.8       1.0X
+Collect longs                                      1832           1895          84          2.7         366.3       0.2X
+Collect timestamps                                 1811           2046         247          2.8         362.2       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 346fbd3499bab..ee70aa389f326 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    413            435          31         24.2          41.3       1.0X
-cast to timestamp wholestage on                     357            375          16         28.0          35.7       1.2X
+cast to timestamp wholestage off                    416            439          32         24.0          41.6       1.0X
+cast to timestamp wholestage on                     370            399          31         27.1          37.0       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1311           1335          34          7.6         131.1       1.0X
-year of timestamp wholestage on                    1541           1561          21          6.5         154.1       0.9X
+year of timestamp wholestage off                   1328           1343          21          7.5         132.8       1.0X
+year of timestamp wholestage on                    1204           1247          27          8.3         120.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1449           1467          26          6.9         144.9       1.0X
-quarter of timestamp wholestage on                 1403           1418          12          7.1         140.3       1.0X
+quarter of timestamp wholestage off                1447           1451           6          6.9         144.7       1.0X
+quarter of timestamp wholestage on                 1384           1392           7          7.2         138.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1221           1231          14          8.2         122.1       1.0X
-month of timestamp wholestage on                   1252           1266           8          8.0         125.2       1.0X
+month of timestamp wholestage off                  1233           1238           6          8.1         123.3       1.0X
+month of timestamp wholestage on                   1232           1252          15          8.1         123.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1939           1947          11          5.2         193.9       1.0X
-weekofyear of timestamp wholestage on              1932           1942           9          5.2         193.2       1.0X
+weekofyear of timestamp wholestage off             1921           1923           3          5.2         192.1       1.0X
+weekofyear of timestamp wholestage on              1893           1914          14          5.3         189.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1216           1222           9          8.2         121.6       1.0X
-day of timestamp wholestage on                     1246           1256          14          8.0         124.6       1.0X
+day of timestamp wholestage off                    1224           1227           4          8.2         122.4       1.0X
+day of timestamp wholestage on                     1219           1238          15          8.2         121.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1278           1286          12          7.8         127.8       1.0X
-dayofyear of timestamp wholestage on               1283           1291           8          7.8         128.3       1.0X
+dayofyear of timestamp wholestage off              1283           1283           1          7.8         128.3       1.0X
+dayofyear of timestamp wholestage on               1256           1261           5          8.0         125.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1220           1226           9          8.2         122.0       1.0X
-dayofmonth of timestamp wholestage on              1232           1249          15          8.1         123.2       1.0X
+dayofmonth of timestamp wholestage off             1227           1233           8          8.2         122.7       1.0X
+dayofmonth of timestamp wholestage on              1220           1228           6          8.2         122.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1409           1423          21          7.1         140.9       1.0X
-dayofweek of timestamp wholestage on               1396           1403           7          7.2         139.6       1.0X
+dayofweek of timestamp wholestage off              1404           1408           5          7.1         140.4       1.0X
+dayofweek of timestamp wholestage on               1376           1386           6          7.3         137.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1346           1362          22          7.4         134.6       1.0X
-weekday of timestamp wholestage on                 1330           1341          11          7.5         133.0       1.0X
+weekday of timestamp wholestage off                1346           1350           5          7.4         134.6       1.0X
+weekday of timestamp wholestage on                 1313           1319           6          7.6         131.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    962            963           2         10.4          96.2       1.0X
-hour of timestamp wholestage on                     981            999          19         10.2          98.1       1.0X
+hour of timestamp wholestage off                   1005           1011           8         10.0         100.5       1.0X
+hour of timestamp wholestage on                     953            961           9         10.5          95.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  996            999           5         10.0          99.6       1.0X
-minute of timestamp wholestage on                   975            984          11         10.3          97.5       1.0X
+minute of timestamp wholestage off                  995           1000           8         10.1          99.5       1.0X
+minute of timestamp wholestage on                   957            959           3         10.5          95.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1004           1005           2         10.0         100.4       1.0X
-second of timestamp wholestage on                   979            988           6         10.2          97.9       1.0X
+second of timestamp wholestage off                  960            963           4         10.4          96.0       1.0X
+second of timestamp wholestage on                   957            960           3         10.4          95.7       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         285            289           6         35.1          28.5       1.0X
-current_date wholestage on                          302            308           8         33.1          30.2       0.9X
+current_date wholestage off                         290            295           7         34.4          29.0       1.0X
+current_date wholestage on                          287            295          13         34.9          28.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    296            298           2         33.8          29.6       1.0X
-current_timestamp wholestage on                     303            311           7         33.0          30.3       1.0X
+current_timestamp wholestage off                    294            296           4         34.1          29.4       1.0X
+current_timestamp wholestage on                     280            303          19         35.7          28.0       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1041           1042           1          9.6         104.1       1.0X
-cast to date wholestage on                         1027           1039           9          9.7         102.7       1.0X
+cast to date wholestage off                        1066           1067           1          9.4         106.6       1.0X
+cast to date wholestage on                         1007           1014           6          9.9         100.7       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1250           1254           6          8.0         125.0       1.0X
-last_day wholestage on                             1267           1273           8          7.9         126.7       1.0X
+last_day wholestage off                            1273           1289          22          7.9         127.3       1.0X
+last_day wholestage on                             1238           1246           9          8.1         123.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1118           1122           6          8.9         111.8       1.0X
-next_day wholestage on                             1081           1096          10          9.2         108.1       1.0X
+next_day wholestage off                            1124           1125           2          8.9         112.4       1.0X
+next_day wholestage on                             1058           1064           8          9.5         105.8       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1053           1055           4          9.5         105.3       1.0X
-date_add wholestage on                             1059           1061           2          9.4         105.9       1.0X
+date_add wholestage off                            1049           1049           0          9.5         104.9       1.0X
+date_add wholestage on                             1036           1041           7          9.6         103.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1055           1064          13          9.5         105.5       1.0X
-date_sub wholestage on                             1055           1060           6          9.5         105.5       1.0X
+date_sub wholestage off                            1051           1052           2          9.5         105.1       1.0X
+date_sub wholestage on                             1040           1046           5          9.6         104.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1373           1375           2          7.3         137.3       1.0X
-add_months wholestage on                           1383           1393          13          7.2         138.3       1.0X
+add_months wholestage off                          1381           1388          11          7.2         138.1       1.0X
+add_months wholestage on                           1359           1367           8          7.4         135.9       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5590           5603          19          1.8         559.0       1.0X
-format date wholestage on                          5974           5985           8          1.7         597.4       0.9X
+format date wholestage off                         5766           5793          38          1.7         576.6       1.0X
+format date wholestage on                          5803           5812           7          1.7         580.3       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8650           8662          18          1.2         865.0       1.0X
-from_unixtime wholestage on                        8671           8685          15          1.2         867.1       1.0X
+from_unixtime wholestage off                       8891           8915          34          1.1         889.1       1.0X
+from_unixtime wholestage on                        8870           8885          12          1.1         887.0       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1152           1157           7          8.7         115.2       1.0X
-from_utc_timestamp wholestage on                   1193           1200           7          8.4         119.3       1.0X
+from_utc_timestamp wholestage off                  1089           1094           7          9.2         108.9       1.0X
+from_utc_timestamp wholestage on                   1105           1107           3          9.0         110.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1414           1417           3          7.1         141.4       1.0X
-to_utc_timestamp wholestage on                     1390           1397           7          7.2         139.0       1.0X
+to_utc_timestamp wholestage off                    1620           1628          11          6.2         162.0       1.0X
+to_utc_timestamp wholestage on                     1600           1608           7          6.3         160.0       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        332            350          25         30.1          33.2       1.0X
-cast interval wholestage on                         325            333           7         30.8          32.5       1.0X
+cast interval wholestage off                        334            336           3         30.0          33.4       1.0X
+cast interval wholestage on                         305            310           4         32.7          30.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1814           1819           7          5.5         181.4       1.0X
-datediff wholestage on                             1811           1822           9          5.5         181.1       1.0X
+datediff wholestage off                            1811           1816           7          5.5         181.1       1.0X
+datediff wholestage on                             1758           1761           3          5.7         175.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      4660           4661           1          2.1         466.0       1.0X
-months_between wholestage on                       4577           4586           5          2.2         457.7       1.0X
+months_between wholestage off                      4780           4789          12          2.1         478.0       1.0X
+months_between wholestage on                       4765           4772           8          2.1         476.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2006           2119         161          0.5        2005.6       1.0X
-window wholestage on                              43855          43884          41          0.0       43855.4       0.0X
+window wholestage off                              1840           1967         179          0.5        1840.4       1.0X
+window wholestage on                              46710          46792          57          0.0       46709.5       0.0X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2336           2339           5          4.3         233.6       1.0X
-date_trunc YEAR wholestage on                      2204           2211           6          4.5         220.4       1.1X
+date_trunc YEAR wholestage off                     2350           2351           1          4.3         235.0       1.0X
+date_trunc YEAR wholestage on                      2317           2324           8          4.3         231.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2338           2339           1          4.3         233.8       1.0X
-date_trunc YYYY wholestage on                      2200           2208           5          4.5         220.0       1.1X
+date_trunc YYYY wholestage off                     2341           2343           3          4.3         234.1       1.0X
+date_trunc YYYY wholestage on                      2314           2332          22          4.3         231.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2337           2340           5          4.3         233.7       1.0X
-date_trunc YY wholestage on                        2202           2209           6          4.5         220.2       1.1X
+date_trunc YY wholestage off                       2348           2354           9          4.3         234.8       1.0X
+date_trunc YY wholestage on                        2313           2317           3          4.3         231.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2238           2239           1          4.5         223.8       1.0X
-date_trunc MON wholestage on                       2222           2233          11          4.5         222.2       1.0X
+date_trunc MON wholestage off                      2383           2385           3          4.2         238.3       1.0X
+date_trunc MON wholestage on                       2293           2307          22          4.4         229.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2243           2243           1          4.5         224.3       1.0X
-date_trunc MONTH wholestage on                     2216           2224           9          4.5         221.6       1.0X
+date_trunc MONTH wholestage off                    2380           2393          18          4.2         238.0       1.0X
+date_trunc MONTH wholestage on                     2296           2304           9          4.4         229.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2237           2239           3          4.5         223.7       1.0X
-date_trunc MM wholestage on                        2217           2221           3          4.5         221.7       1.0X
+date_trunc MM wholestage off                       2375           2380           7          4.2         237.5       1.0X
+date_trunc MM wholestage on                        2295           2302           8          4.4         229.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1894           1896           3          5.3         189.4       1.0X
-date_trunc DAY wholestage on                       1851           1856           5          5.4         185.1       1.0X
+date_trunc DAY wholestage off                      2145           2145           1          4.7         214.5       1.0X
+date_trunc DAY wholestage on                       2090           2096           6          4.8         209.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1887           1899          17          5.3         188.7       1.0X
-date_trunc DD wholestage on                        1846           1855           6          5.4         184.6       1.0X
+date_trunc DD wholestage off                       2142           2149          11          4.7         214.2       1.0X
+date_trunc DD wholestage on                        2090           2096           5          4.8         209.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1947           1951           5          5.1         194.7       1.0X
-date_trunc HOUR wholestage on                      1909           1918           7          5.2         190.9       1.0X
+date_trunc HOUR wholestage off                     2204           2212          11          4.5         220.4       1.0X
+date_trunc HOUR wholestage on                      2196           2201           5          4.6         219.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    381            386           7         26.3          38.1       1.0X
-date_trunc MINUTE wholestage on                     321            323           1         31.2          32.1       1.2X
+date_trunc MINUTE wholestage off                    353            353           1         28.3          35.3       1.0X
+date_trunc MINUTE wholestage on                     331            335           4         30.2          33.1       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    368            368           0         27.2          36.8       1.0X
-date_trunc SECOND wholestage on                     324            327           3         30.9          32.4       1.1X
+date_trunc SECOND wholestage off                    343            345           3         29.2          34.3       1.0X
+date_trunc SECOND wholestage on                     335            336           2         29.9          33.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2213           2213           0          4.5         221.3       1.0X
-date_trunc WEEK wholestage on                      2113           2123          11          4.7         211.3       1.0X
+date_trunc WEEK wholestage off                     2277           2279           3          4.4         227.7       1.0X
+date_trunc WEEK wholestage on                      2262           2270          10          4.4         226.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3071           3081          14          3.3         307.1       1.0X
-date_trunc QUARTER wholestage on                   3031           3035           3          3.3         303.1       1.0X
+date_trunc QUARTER wholestage off                  3395           3395           0          2.9         339.5       1.0X
+date_trunc QUARTER wholestage on                   3185           3193           6          3.1         318.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           326            328           3         30.7          32.6       1.0X
-trunc year wholestage on                            307            318          12         32.6          30.7       1.1X
+trunc year wholestage off                           321            330          14         31.2          32.1       1.0X
+trunc year wholestage on                            293            308          14         34.1          29.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           327            333           7         30.5          32.7       1.0X
-trunc yyyy wholestage on                            305            320          17         32.8          30.5       1.1X
+trunc yyyy wholestage off                           322            324           2         31.1          32.2       1.0X
+trunc yyyy wholestage on                            293            296           4         34.1          29.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             322            323           1         31.1          32.2       1.0X
-trunc yy wholestage on                              304            343          67         32.9          30.4       1.1X
+trunc yy wholestage off                             322            322           0         31.0          32.2       1.0X
+trunc yy wholestage on                              292            296           4         34.2          29.2       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            321            321           1         31.2          32.1       1.0X
-trunc mon wholestage on                             303            313           9         33.0          30.3       1.1X
+trunc mon wholestage off                            319            320           1         31.3          31.9       1.0X
+trunc mon wholestage on                             293            295           1         34.1          29.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          319            323           4         31.3          31.9       1.0X
-trunc month wholestage on                           305            359          75         32.8          30.5       1.0X
+trunc month wholestage off                          319            320           1         31.3          31.9       1.0X
+trunc month wholestage on                           293            295           2         34.1          29.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             322            323           1         31.0          32.2       1.0X
-trunc mm wholestage on                              317            351          53         31.5          31.7       1.0X
+trunc mm wholestage off                             318            319           1         31.4          31.8       1.0X
+trunc mm wholestage on                              293            312          26         34.1          29.3       1.1X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     219            219           0          4.6         219.4       1.0X
-to timestamp str wholestage on                      214            216           2          4.7         214.0       1.0X
+to timestamp str wholestage off                     220            226           9          4.5         220.3       1.0X
+to timestamp str wholestage on                      212            214           4          4.7         212.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1947           1951           6          0.5        1946.8       1.0X
-to_timestamp wholestage on                         1950           1958          12          0.5        1949.6       1.0X
+to_timestamp wholestage off                        1875           1876           2          0.5        1874.7       1.0X
+to_timestamp wholestage on                         2075           2091          25          0.5        2075.1       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1975           1979           5          0.5        1974.8       1.0X
-to_unix_timestamp wholestage on                    1895           1908          12          0.5        1895.3       1.0X
+to_unix_timestamp wholestage off                   2141           2143           2          0.5        2141.2       1.0X
+to_unix_timestamp wholestage on                    2145           2177          21          0.5        2144.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          284            286           2          3.5         284.3       1.0X
-to date str wholestage on                           263            266           4          3.8         263.3       1.1X
+to date str wholestage off                          276            284          11          3.6         276.1       1.0X
+to date str wholestage on                           264            270           8          3.8         264.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3460           3485          36          0.3        3459.8       1.0X
-to_date wholestage on                              3413           3431          25          0.3        3413.0       1.0X
+to_date wholestage off                             3876           3903          38          0.3        3876.1       1.0X
+to_date wholestage on                              3833           3843           6          0.3        3833.4       1.0X
 
 
 ================================================================================================
@@ -422,8 +422,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                            2032           2038           6          2.5         406.5       1.0X
-Collect longs                                      1276           1597         419          3.9         255.1       1.6X
-Collect timestamps                                 6254           7606        1172          0.8        1250.7       0.3X
+From java.sql.Timestamp                             394            399           5         12.7          78.7       1.0X
+Collect longs                                      1336           2676        1201          3.7         267.2       0.3X
+Collect timestamps                                 2025           2091          65          2.5         405.0       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index edf9eef5813d1..2a9322a4b462a 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  19946          19946           0          5.0         199.5       1.0X
-before 1582, noop                                 10910          10910           0          9.2         109.1       1.8X
-after 1582, rebase off                            32065          32065           0          3.1         320.7       0.6X
-after 1582, rebase on                             31424          31424           0          3.2         314.2       0.6X
-before 1582, rebase off                           21764          21764           0          4.6         217.6       0.9X
-before 1582, rebase on                            22700          22700           0          4.4         227.0       0.9X
+after 1582, noop                                  21171          21171           0          4.7         211.7       1.0X
+before 1582, noop                                 11036          11036           0          9.1         110.4       1.9X
+after 1582, rebase off                            34321          34321           0          2.9         343.2       0.6X
+after 1582, rebase on                             33269          33269           0          3.0         332.7       0.6X
+before 1582, rebase off                           22016          22016           0          4.5         220.2       1.0X
+before 1582, rebase on                            23338          23338           0          4.3         233.4       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13054          13273         193          7.7         130.5       1.0X
-after 1582, vec off, rebase on                    13337          13418         117          7.5         133.4       1.0X
-after 1582, vec on, rebase off                     3666           3740         116         27.3          36.7       3.6X
-after 1582, vec on, rebase on                      5097           5160          70         19.6          51.0       2.6X
-before 1582, vec off, rebase off                  12908          12984          76          7.7         129.1       1.0X
-before 1582, vec off, rebase on                   14514          14575          53          6.9         145.1       0.9X
-before 1582, vec on, rebase off                    3718           3790          63         26.9          37.2       3.5X
-before 1582, vec on, rebase on                     5472           5527          48         18.3          54.7       2.4X
+after 1582, vec off, rebase off                   12791          13089         287          7.8         127.9       1.0X
+after 1582, vec off, rebase on                    13203          13271          81          7.6         132.0       1.0X
+after 1582, vec on, rebase off                     3709           3764          49         27.0          37.1       3.4X
+after 1582, vec on, rebase on                      5082           5114          29         19.7          50.8       2.5X
+before 1582, vec off, rebase off                  13059          13153          87          7.7         130.6       1.0X
+before 1582, vec off, rebase on                   14211          14236          27          7.0         142.1       0.9X
+before 1582, vec on, rebase off                    3687           3749          72         27.1          36.9       3.5X
+before 1582, vec on, rebase on                     5449           5497          56         18.4          54.5       2.3X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2718           2718           0         36.8          27.2       1.0X
-before 1582, noop                                  2774           2774           0         36.0          27.7       1.0X
-after 1582, rebase off                            16515          16515           0          6.1         165.2       0.2X
-after 1582, rebase on                             18619          18619           0          5.4         186.2       0.1X
-before 1582, rebase off                           16382          16382           0          6.1         163.8       0.2X
-before 1582, rebase on                            19580          19580           0          5.1         195.8       0.1X
+after 1582, noop                                   2831           2831           0         35.3          28.3       1.0X
+before 1582, noop                                  2816           2816           0         35.5          28.2       1.0X
+after 1582, rebase off                            15543          15543           0          6.4         155.4       0.2X
+after 1582, rebase on                             18391          18391           0          5.4         183.9       0.2X
+before 1582, rebase off                           15747          15747           0          6.4         157.5       0.2X
+before 1582, rebase on                            18846          18846           0          5.3         188.5       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15082          15112          43          6.6         150.8       1.0X
-after 1582, vec off, rebase on                    18035          18105         107          5.5         180.3       0.8X
-after 1582, vec on, rebase off                     4948           4970          19         20.2          49.5       3.0X
-after 1582, vec on, rebase on                      8761           8780          25         11.4          87.6       1.7X
-before 1582, vec off, rebase off                  15285          15389         103          6.5         152.8       1.0X
-before 1582, vec off, rebase on                   18277          18368          80          5.5         182.8       0.8X
-before 1582, vec on, rebase off                    4938           4954          15         20.3          49.4       3.1X
-before 1582, vec on, rebase on                     9220           9248          33         10.8          92.2       1.6X
+after 1582, vec off, rebase off                   16126          16216          78          6.2         161.3       1.0X
+after 1582, vec off, rebase on                    18277          18453         165          5.5         182.8       0.9X
+after 1582, vec on, rebase off                     5030           5067          42         19.9          50.3       3.2X
+after 1582, vec on, rebase on                      8553           8583          43         11.7          85.5       1.9X
+before 1582, vec off, rebase off                  15828          15872          39          6.3         158.3       1.0X
+before 1582, vec off, rebase on                   18899          18959         103          5.3         189.0       0.9X
+before 1582, vec on, rebase off                    4961           5009          43         20.2          49.6       3.3X
+before 1582, vec on, rebase on                     9099           9140          40         11.0          91.0       1.8X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  19820          19820           0          5.0         198.2       1.0X
-before 1582, noop                                 10825          10825           0          9.2         108.2       1.8X
-after 1582                                        27705          27705           0          3.6         277.1       0.7X
-before 1582                                       19390          19390           0          5.2         193.9       1.0X
+after 1582, noop                                  21026          21026           0          4.8         210.3       1.0X
+before 1582, noop                                 11040          11040           0          9.1         110.4       1.9X
+after 1582                                        28171          28171           0          3.5         281.7       0.7X
+before 1582                                       18955          18955           0          5.3         189.5       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10401          10426          33          9.6         104.0       1.0X
-after 1582, vec on                                 3788           3813          29         26.4          37.9       2.7X
-before 1582, vec off                              10832          10843          11          9.2         108.3       1.0X
-before 1582, vec on                                4182           4219          46         23.9          41.8       2.5X
+after 1582, vec off                               10876          10931          49          9.2         108.8       1.0X
+after 1582, vec on                                 3900           3913          20         25.6          39.0       2.8X
+before 1582, vec off                              11165          11174          12          9.0         111.6       1.0X
+before 1582, vec on                                4208           4214           7         23.8          42.1       2.6X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2787           2787           0         35.9          27.9       1.0X
-before 1582, noop                                  2777           2777           0         36.0          27.8       1.0X
-after 1582                                        52456          52456           0          1.9         524.6       0.1X
-before 1582                                       54983          54983           0          1.8         549.8       0.1X
+after 1582, noop                                   2924           2924           0         34.2          29.2       1.0X
+before 1582, noop                                  2820           2820           0         35.5          28.2       1.0X
+after 1582                                        22228          22228           0          4.5         222.3       0.1X
+before 1582                                       22590          22590           0          4.4         225.9       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               38336          38425          88          2.6         383.4       1.0X
-after 1582, vec on                                30089          30198          96          3.3         300.9       1.3X
-before 1582, vec off                              42739          42801          87          2.3         427.4       0.9X
-before 1582, vec on                               31530          31591          61          3.2         315.3       1.2X
+after 1582, vec off                               13591          13658          59          7.4         135.9       1.0X
+after 1582, vec on                                 7399           7488         126         13.5          74.0       1.8X
+before 1582, vec off                              14065          14096          30          7.1         140.7       1.0X
+before 1582, vec on                                7950           8127         249         12.6          79.5       1.7X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index 602f3e9dc5287..050950571511d 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,49 +6,49 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23453          23453           0          4.3         234.5       1.0X
-before 1582, noop                                 10821          10821           0          9.2         108.2       2.2X
-after 1582, rebase off                            35558          35558           0          2.8         355.6       0.7X
-after 1582, rebase on                             35892          35892           0          2.8         358.9       0.7X
-before 1582, rebase off                           22700          22700           0          4.4         227.0       1.0X
-before 1582, rebase on                            23247          23247           0          4.3         232.5       1.0X
+after 1582, noop                                  24114          24114           0          4.1         241.1       1.0X
+before 1582, noop                                 10250          10250           0          9.8         102.5       2.4X
+after 1582, rebase off                            36672          36672           0          2.7         366.7       0.7X
+after 1582, rebase on                             37123          37123           0          2.7         371.2       0.6X
+before 1582, rebase off                           21925          21925           0          4.6         219.2       1.1X
+before 1582, rebase on                            22341          22341           0          4.5         223.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12641          12690          43          7.9         126.4       1.0X
-after 1582, vec off, rebase on                    13318          13380          67          7.5         133.2       0.9X
-after 1582, vec on, rebase off                     3648           3659          11         27.4          36.5       3.5X
-after 1582, vec on, rebase on                      5160           5212          69         19.4          51.6       2.4X
-before 1582, vec off, rebase off                  13024          13065          36          7.7         130.2       1.0X
-before 1582, vec off, rebase on                   13810          13932         106          7.2         138.1       0.9X
-before 1582, vec on, rebase off                    3631           3695          57         27.5          36.3       3.5X
-before 1582, vec on, rebase on                     5791           5860          71         17.3          57.9       2.2X
+after 1582, vec off, rebase off                   12456          12601         126          8.0         124.6       1.0X
+after 1582, vec off, rebase on                    13299          13336          32          7.5         133.0       0.9X
+after 1582, vec on, rebase off                     3623           3660          40         27.6          36.2       3.4X
+after 1582, vec on, rebase on                      5160           5177          15         19.4          51.6       2.4X
+before 1582, vec off, rebase off                  13177          13264          76          7.6         131.8       0.9X
+before 1582, vec off, rebase on                   14102          14149          46          7.1         141.0       0.9X
+before 1582, vec on, rebase off                    3649           3670          34         27.4          36.5       3.4X
+before 1582, vec on, rebase on                     5652           5667          15         17.7          56.5       2.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2750           2750           0         36.4          27.5       1.0X
-before 1582, noop                                  2833           2833           0         35.3          28.3       1.0X
-after 1582, rebase off                            16832          16832           0          5.9         168.3       0.2X
-after 1582, rebase on                             19688          19688           0          5.1         196.9       0.1X
-before 1582, rebase off                           17548          17548           0          5.7         175.5       0.2X
-before 1582, rebase on                            21343          21343           0          4.7         213.4       0.1X
+after 1582, noop                                   2871           2871           0         34.8          28.7       1.0X
+before 1582, noop                                  2753           2753           0         36.3          27.5       1.0X
+after 1582, rebase off                            15927          15927           0          6.3         159.3       0.2X
+after 1582, rebase on                             19138          19138           0          5.2         191.4       0.1X
+before 1582, rebase off                           16137          16137           0          6.2         161.4       0.2X
+before 1582, rebase on                            19584          19584           0          5.1         195.8       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   15243          15329          82          6.6         152.4       1.0X
-after 1582, vec off, rebase on                    18296          18330          54          5.5         183.0       0.8X
-after 1582, vec on, rebase off                     4925           4927           2         20.3          49.2       3.1X
-after 1582, vec on, rebase on                      9647           9686          35         10.4          96.5       1.6X
-before 1582, vec off, rebase off                  14880          15105         267          6.7         148.8       1.0X
-before 1582, vec off, rebase on                   18474          18514          51          5.4         184.7       0.8X
-before 1582, vec on, rebase off                    4970           4978          10         20.1          49.7       3.1X
-before 1582, vec on, rebase on                     9938          10012          64         10.1          99.4       1.5X
+after 1582, vec off, rebase off                   14995          15047          47          6.7         150.0       1.0X
+after 1582, vec off, rebase on                    18111          18146          37          5.5         181.1       0.8X
+after 1582, vec on, rebase off                     4837           4873          44         20.7          48.4       3.1X
+after 1582, vec on, rebase on                      9542           9669         111         10.5          95.4       1.6X
+before 1582, vec off, rebase off                  14993          15090          94          6.7         149.9       1.0X
+before 1582, vec off, rebase on                   18675          18712          64          5.4         186.7       0.8X
+before 1582, vec on, rebase off                    4908           4923          15         20.4          49.1       3.1X
+before 1582, vec on, rebase on                    10128          10148          19          9.9         101.3       1.5X
 
 
 ================================================================================================
@@ -59,36 +59,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23500          23500           0          4.3         235.0       1.0X
-before 1582, noop                                 10788          10788           0          9.3         107.9       2.2X
-after 1582                                        32237          32237           0          3.1         322.4       0.7X
-before 1582                                       20187          20187           0          5.0         201.9       1.2X
+after 1582, noop                                  23977          23977           0          4.2         239.8       1.0X
+before 1582, noop                                 10094          10094           0          9.9         100.9       2.4X
+after 1582                                        33115          33115           0          3.0         331.2       0.7X
+before 1582                                       19430          19430           0          5.1         194.3       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10947          10971          28          9.1         109.5       1.0X
-after 1582, vec on                                 3677           3702          36         27.2          36.8       3.0X
-before 1582, vec off                              11456          11472          21          8.7         114.6       1.0X
-before 1582, vec on                                4079           4103          21         24.5          40.8       2.7X
+after 1582, vec off                               10217          10241          21          9.8         102.2       1.0X
+after 1582, vec on                                 3671           3691          31         27.2          36.7       2.8X
+before 1582, vec off                              10800          10874         114          9.3         108.0       0.9X
+before 1582, vec on                                4118           4165          74         24.3          41.2       2.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2891           2891           0         34.6          28.9       1.0X
-before 1582, noop                                  2906           2906           0         34.4          29.1       1.0X
-after 1582                                        55812          55812           0          1.8         558.1       0.1X
-before 1582                                       57512          57512           0          1.7         575.1       0.1X
+after 1582, noop                                   2691           2691           0         37.2          26.9       1.0X
+before 1582, noop                                  2743           2743           0         36.5          27.4       1.0X
+after 1582                                        21409          21409           0          4.7         214.1       0.1X
+before 1582                                       22554          22554           0          4.4         225.5       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               46376          46410          33          2.2         463.8       1.0X
-after 1582, vec on                                35003          35189         163          2.9         350.0       1.3X
-before 1582, vec off                              52942          52979          34          1.9         529.4       0.9X
-before 1582, vec on                               42596          42747         158          2.3         426.0       1.1X
+after 1582, vec off                               14752          14855         103          6.8         147.5       1.0X
+after 1582, vec on                                 8146           8185          34         12.3          81.5       1.8X
+before 1582, vec off                              15247          15294          46          6.6         152.5       1.0X
+before 1582, vec on                                8414           8466          52         11.9          84.1       1.8X
 
 

From 78f5cb3f07f9cb4a90ffd51fea4f491447cc6ee0 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 14 Apr 2020 05:08:48 +0000
Subject: [PATCH 0494/1280] =?UTF-8?q?[SPARK-31392][SQL][3.0]=20Support=20C?=
 =?UTF-8?q?alendarInterval=20to=20be=20reflect=20to=20Ca=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

This PR backport https://github.com/apache/spark/pull/28165 to 3.0.0, previous description shows as:

Since 3.0.0, we make CalendarInterval public for input, it's better for it to be inferred to CalendarIntervalType.
In the PR, we add a rule for CalendarInterval to be mapped to CalendarIntervalType in ScalaRelection, then records(e.g case class, tuples ...) contains interval fields are able to convert to a Dataframe.

### Why are the changes needed?

CalendarInterval is public but can not be used as input for Datafame.

```scala
scala> import org.apache.spark.unsafe.types.CalendarInterval
import org.apache.spark.unsafe.types.CalendarInterval

scala> Seq((1, new CalendarInterval(1, 2, 3))).toDF("a", "b")
java.lang.UnsupportedOperationException: Schema for type org.apache.spark.unsafe.types.CalendarInterval is not supported
  at org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$schemaFor$1(ScalaReflection.scala:735)
```

this should be supported as well as
```scala
scala> sql("select interval 2 month 1 day a")
res2: org.apache.spark.sql.DataFrame = [a: interval]
```
### Does this PR introduce any user-facing change?

Yes, records(e.g case class, tuples ...) contains interval fields are able to convert to a Dataframe
### How was this patch tested?

add uts

Closes #28198 from yaooqinn/SPARK-31392-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/ScalaReflection.scala    |  2 ++
 .../sql/catalyst/ScalaReflectionSuite.scala     | 17 +++++++++++------
 .../encoders/ExpressionEncoderSuite.scala       |  6 +++---
 .../org/apache/spark/sql/DataFrameSuite.scala   |  6 ++++++
 .../execution/arrow/ArrowConvertersSuite.scala  | 12 +++++-------
 5 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index bb61e20cb7d0f..9c8da32cbc0ae 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -703,6 +703,8 @@ object ScalaReflection extends ScalaReflection {
         Schema(TimestampType, nullable = true)
       case t if isSubtype(t, localTypeOf[java.time.LocalDate]) => Schema(DateType, nullable = true)
       case t if isSubtype(t, localTypeOf[java.sql.Date]) => Schema(DateType, nullable = true)
+      case t if isSubtype(t, localTypeOf[CalendarInterval]) =>
+        Schema(CalendarIntervalType, nullable = true)
       case t if isSubtype(t, localTypeOf[BigDecimal]) =>
         Schema(DecimalType.SYSTEM_DEFAULT, nullable = true)
       case t if isSubtype(t, localTypeOf[java.math.BigDecimal]) =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
index f9cd9c3c398f6..b981a50499bf5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedExtractValue
 import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, If, SpecificInternalRow, UpCast}
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, NewInstance}
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
 
 case class PrimitiveData(
     intField: Int,
@@ -48,7 +49,8 @@ case class NullableData(
     decimalField: java.math.BigDecimal,
     dateField: Date,
     timestampField: Timestamp,
-    binaryField: Array[Byte])
+    binaryField: Array[Byte],
+    intervalField: CalendarInterval)
 
 case class OptionalData(
     intField: Option[Int],
@@ -58,7 +60,8 @@ case class OptionalData(
     shortField: Option[Short],
     byteField: Option[Byte],
     booleanField: Option[Boolean],
-    structField: Option[PrimitiveData])
+    structField: Option[PrimitiveData],
+    intervalField: Option[CalendarInterval])
 
 case class ComplexData(
     arrayField: Seq[Int],
@@ -200,7 +203,8 @@ class ScalaReflectionSuite extends SparkFunSuite {
         StructField("decimalField", DecimalType.SYSTEM_DEFAULT, nullable = true),
         StructField("dateField", DateType, nullable = true),
         StructField("timestampField", TimestampType, nullable = true),
-        StructField("binaryField", BinaryType, nullable = true))),
+        StructField("binaryField", BinaryType, nullable = true),
+        StructField("intervalField", CalendarIntervalType, nullable = true))),
       nullable = true))
   }
 
@@ -215,7 +219,8 @@ class ScalaReflectionSuite extends SparkFunSuite {
         StructField("shortField", ShortType, nullable = true),
         StructField("byteField", ByteType, nullable = true),
         StructField("booleanField", BooleanType, nullable = true),
-        StructField("structField", schemaFor[PrimitiveData].dataType, nullable = true))),
+        StructField("structField", schemaFor[PrimitiveData].dataType, nullable = true),
+        StructField("intervalField", CalendarIntervalType, nullable = true))),
       nullable = true))
   }
 
@@ -295,10 +300,10 @@ class ScalaReflectionSuite extends SparkFunSuite {
   test("convert Option[Product] to catalyst") {
     val primitiveData = PrimitiveData(1, 1, 1, 1, 1, 1, true)
     val data = OptionalData(Some(2), Some(2), Some(2), Some(2), Some(2), Some(2), Some(true),
-      Some(primitiveData))
+      Some(primitiveData), Some(new CalendarInterval(1, 2, 3)))
     val dataType = schemaFor[OptionalData].dataType
     val convertedData = InternalRow(2, 2.toLong, 2.toDouble, 2.toFloat, 2.toShort, 2.toByte, true,
-      InternalRow(1, 1, 1, 1, 1, 1, true))
+      InternalRow(1, 1, 1, 1, 1, 1, true), new CalendarInterval(1, 2, 3))
     assert(CatalystTypeConverters.createToCatalystConverter(dataType)(data) === convertedData)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 66a1bbe01f5c6..1036dc725c205 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 import org.apache.spark.util.ClosureCleaner
 
 case class RepeatedStruct(s: Seq[PrimitiveData])
@@ -209,9 +209,9 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
 
   productTest(
     OptionalData(Some(2), Some(2), Some(2), Some(2), Some(2), Some(2), Some(true),
-      Some(PrimitiveData(1, 1, 1, 1, 1, 1, true))))
+      Some(PrimitiveData(1, 1, 1, 1, 1, 1, true)), Some(new CalendarInterval(1, 2, 3))))
 
-  productTest(OptionalData(None, None, None, None, None, None, None, None))
+  productTest(OptionalData(None, None, None, None, None, None, None, None, None))
 
   encodeDecodeTest(Seq(Some(1), None), "Option in array")
   encodeDecodeTest(Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None), "Option in map")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 1762bc61d1fc7..f797290028e6f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -43,6 +43,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
 import org.apache.spark.sql.test.SQLTestData.{DecimalData, NullStrings, TestData2}
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
 import org.apache.spark.util.Utils
 import org.apache.spark.util.random.XORShiftRandom
 
@@ -2352,6 +2353,11 @@ class DataFrameSuite extends QueryTest
     }
     assert(e.getMessage.contains("Table or view not found:"))
   }
+
+  test("CalendarInterval reflection support") {
+    val df = Seq((1, new CalendarInterval(1, 2, 3))).toDF("a", "b")
+    checkAnswer(df.selectExpr("b"), Row(new CalendarInterval(1, 2, 3)))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
index fdb23d5be78a1..1e6e59456c887 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/arrow/ArrowConvertersSuite.scala
@@ -1208,15 +1208,13 @@ class ArrowConvertersSuite extends SharedSparkSession {
     spark.conf.unset(SQLConf.ARROW_EXECUTION_MAX_RECORDS_PER_BATCH.key)
   }
 
-  testQuietly("unsupported types") {
-    def runUnsupported(block: => Unit): Unit = {
-      val msg = intercept[UnsupportedOperationException] {
-        block
-      }
-      assert(msg.getMessage.contains("is not supported"))
+  testQuietly("interval is unsupported for arrow") {
+    val e = intercept[SparkException] {
+      calenderIntervalData.toDF().toArrowBatchRdd.collect()
     }
 
-    runUnsupported { calenderIntervalData.toDF().toArrowBatchRdd.collect() }
+    assert(e.getCause.isInstanceOf[UnsupportedOperationException])
+    assert(e.getCause.getMessage.contains("Unsupported data type: interval"))
   }
 
   test("test Arrow Validator") {

From f57f7a766d30854aaba72068dcfbee7acfa50cc5 Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Tue, 14 Apr 2020 14:08:56 +0900
Subject: [PATCH 0495/1280] [SPARK-31441] Support duplicated column names for
 toPandas with arrow execution

### What changes were proposed in this pull request?

This PR is adding support duplicated column names for `toPandas` with Arrow execution.

### Why are the changes needed?

When we execute `toPandas()` with Arrow execution, it fails if the column names have duplicates.

```py
>>> spark.sql("select 1 v, 1 v").toPandas()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/path/to/lib/python3.7/site-packages/pyspark/sql/dataframe.py", line 2132, in toPandas
    pdf = table.to_pandas()
  File "pyarrow/array.pxi", line 441, in pyarrow.lib._PandasConvertible.to_pandas
  File "pyarrow/table.pxi", line 1367, in pyarrow.lib.Table._to_pandas
  File "/path/to/lib/python3.7/site-packages/pyarrow/pandas_compat.py", line 653, in table_to_blockmanager
    columns = _deserialize_column_index(table, all_columns, column_indexes)
  File "/path/to/lib/python3.7/site-packages/pyarrow/pandas_compat.py", line 704, in _deserialize_column_index
    columns = _flatten_single_level_multiindex(columns)
  File "/path/to/lib/python3.7/site-packages/pyarrow/pandas_compat.py", line 937, in _flatten_single_level_multiindex
    raise ValueError('Found non-unique column index')
ValueError: Found non-unique column index
```

### Does this PR introduce any user-facing change?

Yes, previously we will face an error above, but after this PR, we will see the result:

```py
>>> spark.sql("select 1 v, 1 v").toPandas()
   v  v
0  1  1
```

### How was this patch tested?

Added and modified related tests.

Closes #28210 from ueshin/issues/SPARK-31441/to_pandas.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 87be3641eb3517862dd5073903d5b37275852066)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/pandas/conversion.py    |  6 ++++-
 python/pyspark/sql/tests/test_dataframe.py | 27 +++++++++++++++++-----
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/python/pyspark/sql/pandas/conversion.py b/python/pyspark/sql/pandas/conversion.py
index 47cf8bbc5b688..251625ae412cf 100644
--- a/python/pyspark/sql/pandas/conversion.py
+++ b/python/pyspark/sql/pandas/conversion.py
@@ -103,13 +103,17 @@ def toPandas(self):
                 try:
                     from pyspark.sql.pandas.types import _check_series_localize_timestamps
                     import pyarrow
-                    batches = self._collect_as_arrow()
+                    # Rename columns to avoid duplicated column names.
+                    tmp_column_names = ['col_{}'.format(i) for i in range(len(self.columns))]
+                    batches = self.toDF(*tmp_column_names)._collect_as_arrow()
                     if len(batches) > 0:
                         table = pyarrow.Table.from_batches(batches)
                         # Pandas DataFrame created from PyArrow uses datetime64[ns] for date type
                         # values, but we should use datetime.date to match the behavior with when
                         # Arrow optimization is disabled.
                         pdf = table.to_pandas(date_as_object=True)
+                        # Rename back to the original column names.
+                        pdf.columns = self.columns
                         for field in self.schema:
                             if isinstance(field.dataType, TimestampType):
                                 pdf[field.name] = \
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index d9dcbc0f9595f..738c9846169d1 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -529,6 +529,19 @@ def test_to_pandas(self):
         self.assertEquals(types[4], np.object)  # datetime.date
         self.assertEquals(types[5], 'datetime64[ns]')
 
+    @unittest.skipIf(not have_pandas, pandas_requirement_message)
+    def test_to_pandas_with_duplicated_column_names(self):
+        import numpy as np
+
+        sql = "select 1 v, 1 v"
+        for arrowEnabled in [False, True]:
+            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrowEnabled}):
+                df = self.spark.sql(sql)
+                pdf = df.toPandas()
+                types = pdf.dtypes
+                self.assertEquals(types.iloc[0], np.int32)
+                self.assertEquals(types.iloc[1], np.int32)
+
     @unittest.skipIf(not have_pandas, pandas_requirement_message)
     def test_to_pandas_on_cross_join(self):
         import numpy as np
@@ -540,12 +553,14 @@ def test_to_pandas_on_cross_join(self):
           select explode(sequence(1, 3)) v
         ) t2
         """
-        with self.sql_conf({"spark.sql.crossJoin.enabled": True}):
-            df = self.spark.sql(sql)
-            pdf = df.toPandas()
-            types = pdf.dtypes
-            self.assertEquals(types.iloc[0], np.int32)
-            self.assertEquals(types.iloc[1], np.int32)
+        for arrowEnabled in [False, True]:
+            with self.sql_conf({"spark.sql.crossJoin.enabled": True,
+                                "spark.sql.execution.arrow.pyspark.enabled": arrowEnabled}):
+                df = self.spark.sql(sql)
+                pdf = df.toPandas()
+                types = pdf.dtypes
+                self.assertEquals(types.iloc[0], np.int32)
+                self.assertEquals(types.iloc[1], np.int32)
 
     @unittest.skipIf(have_pandas, "Required Pandas was found.")
     def test_to_pandas_required_pandas_not_found(self):

From ca1adf76b1d1fc15fa8d03450080072062d6f672 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 14 Apr 2020 05:18:58 +0000
Subject: [PATCH 0496/1280] [SPARK-30953][SQL] InsertAdaptiveSparkPlan should
 apply AQE on child plan of write commands

This PR changes `InsertAdaptiveSparkPlan` to apply AQE on the child plan of V1/V2 write commands rather than the command itself.

Apply AQE on write commands with child plan will expose `LogicalQueryStage` to `Analyzer` while it should hider under `AdaptiveSparkPlanExec` only to avoid unexpected broken.

No.

Pass Jenkins.

Closes #27701 from Ngone51/skip_v2_commands.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5d4f5d36a26f07327dcc0928d437dd23582ad756)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../execution/adaptive/InsertAdaptiveSparkPlan.scala   |  5 ++++-
 .../execution/adaptive/AdaptiveQueryExecSuite.scala    | 10 ++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
index 621c063e5a7d8..ea586f0bb2561 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
@@ -25,7 +25,8 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.plans.physical.UnspecifiedDistribution
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.command.ExecutedCommandExec
+import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec}
+import org.apache.spark.sql.execution.datasources.v2.V2CommandExec
 import org.apache.spark.sql.execution.exchange.Exchange
 import org.apache.spark.sql.internal.SQLConf
 
@@ -45,6 +46,8 @@ case class InsertAdaptiveSparkPlan(
   private def applyInternal(plan: SparkPlan, isSubquery: Boolean): SparkPlan = plan match {
     case _ if !conf.adaptiveExecutionEnabled => plan
     case _: ExecutedCommandExec => plan
+    case c: DataWritingCommandExec => c.copy(child = apply(c.child))
+    case c: V2CommandExec => c.withNewChildren(c.children.map(apply))
     case _ if shouldApplyAQE(plan, isSubquery) =>
       if (supportAdaptive(plan)) {
         try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index b8ac4ddc61553..64dd9aae96e2b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListe
 import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
+import org.apache.spark.sql.execution.command.DataWritingCommandExec
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight, SortMergeJoinExec}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
@@ -795,4 +796,13 @@ class AdaptiveQueryExecSuite
       }
     }
   }
+
+  test("SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of write commands") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      val plan = sql("CREATE TABLE t1 AS SELECT 1 col").queryExecution.executedPlan
+      assert(plan.isInstanceOf[DataWritingCommandExec])
+      assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec])
+    }
+  }
 }

From e0baca414f5c73729592a90218b3f5af0370a499 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 14 Apr 2020 13:40:38 +0800
Subject: [PATCH 0497/1280] fix compilation

---
 .../org/apache/spark/sql/catalyst/util/DateTimeUtils.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 7ce0fa22d7c54..88ffd48a28270 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -178,7 +178,7 @@ object DateTimeUtils {
    * @return The number of micros since epoch from `java.sql.Timestamp`.
    */
   def fromJavaTimestamp(t: Timestamp): SQLTimestamp = {
-    val micros = millisToMicros(t.getTime) + (t.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS
+    val micros = fromMillis(t.getTime) + (t.getNanos / NANOS_PER_MICROS) % MICROS_PER_MILLIS
     rebaseJulianToGregorianMicros(micros)
   }
 

From be5dfcd2f2e42976db65cf2306371f0e7a74d4cc Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 14 Apr 2020 08:06:55 +0000
Subject: [PATCH 0498/1280] [SPARK-31402][SQL][FOLLOWUP] Refine code comments
 in RebaseDateTime

### What changes were proposed in this pull request?

Refine the code comments of days rebasing, to be consistent with the micros rebasing. i.e. one method is the actual implementation and the other variant is the optimized version.

### Why are the changes needed?

improve code comments

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

N/A

Closes #28199 from cloud-fan/comment.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6b88d136deb99afd9363b208fd6fe5684fe8c3b8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    | 62 ++++++++-----------
 1 file changed, 26 insertions(+), 36 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index f4a98cf3585e2..50b552ea502d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -64,8 +64,7 @@ object RebaseDateTime {
   // The sorted days in Julian calendar when difference in days between Julian and
   // Proleptic Gregorian calendars was changed.
   // The starting point is the `0001-01-01` (-719164 days since the epoch in
-  // Julian calendar). All dates before the staring point have the same difference
-  // of 2 days in Julian and Proleptic Gregorian calendars.
+  // Julian calendar). This array is not applicable for dates before the staring point.
   // Rebasing switch days and diffs `julianGregDiffSwitchDay` and `julianGregDiffs`
   // was generated by the `localRebaseJulianToGregorianDays` function.
   private val julianGregDiffSwitchDay = Array(
@@ -80,6 +79,15 @@ object RebaseDateTime {
    * calendar, interprets the result as a local date in Proleptic Gregorian calendar, and takes the
    * number of days since the epoch from the Gregorian local date.
    *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Julian calendar: 1582-01-01 -> -141704
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   * The code below converts -141704 to -141714.
+   *
    * @param days The number of days since the epoch in Julian calendar. It can be negative.
    * @return The rebased number of days in Gregorian calendar.
    */
@@ -105,22 +113,9 @@ object RebaseDateTime {
   }
 
   /**
-   * For dates of Before Common Era, the function converts the given number of days since the epoch
-   * day 1970-01-01 to a local date in Julian calendar, interprets the result as a local date in
-   * Proleptic Gregorian calendar, and takes the number of days since the epoch from the Gregorian
-   * local date.
-   *
-   * For dates of Common Era, the function performs rebasing via the pre-calculated array of diffs
-   * between Julian and Proleptic Gregorian calendars.
-   *
-   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
-   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
-   * calendar. See SPARK-26651.
-   *
-   * For example:
-   *   Julian calendar: 1582-01-01 -> -141704
-   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
-   * The code below converts -141704 to -141714.
+   * An optimized version of [[localRebaseJulianToGregorianDays(Int)]]. This method leverages the
+   * pre-calculated rebasing array to save calculation. For dates of Before Common Era, the
+   * function falls back to the regular unoptimized version.
    *
    * @param days The number of days since the epoch in Julian calendar. It can be negative.
    * @return The rebased number of days in Gregorian calendar.
@@ -140,8 +135,7 @@ object RebaseDateTime {
   // The sorted days in Proleptic Gregorian calendar when difference in days between
   // Proleptic Gregorian and Julian was changed.
   // The starting point is the `0001-01-01` (-719162 days since the epoch in
-  // Proleptic Gregorian calendar). All dates before the staring point have the same
-  // difference of -2 days in Proleptic Gregorian and Julian calendars.
+  // Proleptic Gregorian calendar). This array is not applicable for dates before the staring point.
   // Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
   // was generated by the `localRebaseGregorianToJulianDays` function.
   private val gregJulianDiffSwitchDay = Array(
@@ -157,6 +151,15 @@ object RebaseDateTime {
    * Gregorian calendar, interprets the result as a local date in Julian calendar, and takes the
    * number of days since the epoch from the Julian local date.
    *
+   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
+   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
+   * calendar. See SPARK-26651.
+   *
+   * For example:
+   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
+   *   Julian calendar: 1582-01-01 -> -141704
+   * The code below converts -141714 to -141704.
+   *
    * @param days The number of days since the epoch in Proleptic Gregorian calendar.
    *             It can be negative.
    * @return The rebased number of days in Julian calendar.
@@ -174,22 +177,9 @@ object RebaseDateTime {
   }
 
   /**
-   * For dates of Before Common Era, Converts the given number of days since the epoch day
-   * 1970-01-01 to a local date in Proleptic Gregorian calendar, interprets the result as a local
-   * date in Julian calendar, and takes the number of days since the epoch from the Julian
-   * local date.
-   *
-   * For dates of Common Era, the function performs rebasing via the pre-calculated array of diffs
-   * between Proleptic Gregorian and Julian calendars.
-   *
-   * This is used to guarantee backward compatibility, as Spark 2.4 and earlier versions use
-   * Julian calendar for dates before 1582-10-15, while Spark 3.0 and later use Proleptic Gregorian
-   * calendar. See SPARK-26651.
-   *
-   * For example:
-   *   Proleptic Gregorian calendar: 1582-01-01 -> -141714
-   *   Julian calendar: 1582-01-01 -> -141704
-   * The code below converts -141714 to -141704.
+   * An optimized version of [[localRebaseGregorianToJulianDays(Int)]]. This method leverages the
+   * pre-calculated rebasing array to save calculation. For dates of Before Common Era, the
+   * function falls back to the regular unoptimized version.
    *
    * @param days The number of days since the epoch in Gregorian calendar. It can be negative.
    * @return The rebased number of days since the epoch in Julian calendar.

From e415a428bba8da40fe79523a8ad1be44d4e50d82 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 14 Apr 2020 14:44:00 +0000
Subject: [PATCH 0499/1280] [SPARK-31439][SQL] Fix perf regression of
 fromJavaDate

### What changes were proposed in this pull request?
In the PR, I propose to re-use optimized implementation of days rebase function `rebaseJulianToGregorianDays()` introduced by the PR #28067 in conversion of `java.sql.Date` values to Catalyst's `DATE` values. The function `fromJavaDate` in `DateTimeUtils` was re-written by taking the implementation from Spark 2.4, and by rebasing the final results via `rebaseJulianToGregorianDays()`.

Also I updated `DateTimeBenchmark`, and added a benchmark for conversion from `java.sql.Date`.

### Why are the changes needed?
The PR fixes the regression of parallelizing a collection of `java.sql.Date` values, and improves performance of converting external values to Catalyst's `DATE` values:
- x4 on the master branch
- 30% against Spark 2.4.6-SNAPSHOT

Spark 2.4.6-SNAPSHOT:
```
To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Date                                  614            655          43          8.1         122.8       1.0X
```

Before the changes:
```
To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Date                                 1154           1206          46          4.3         230.9       1.0X
```

After:
```
To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Date                                  427            434           7         11.7          85.3       1.0X
```

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
- By existing tests suites, in particular, `DateTimeUtilsSuite`, `RebaseDateTimeSuite`, `DateFunctionsSuite`, `DateExpressionsSuite`.
- Re-run `DateTimeBenchmark` in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

Closes #28205 from MaxGekk/optimize-fromJavaDate.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2c5d489679ba3814973680d65853877664bcd931)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     |  14 +-
 .../DateTimeBenchmark-jdk11-results.txt       | 221 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 221 +++++++++---------
 .../benchmark/DateTimeBenchmark.scala         |   7 +-
 4 files changed, 232 insertions(+), 231 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 88ffd48a28270..dede92f0df518 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -103,16 +103,10 @@ object DateTimeUtils {
    * @return The number of days since epoch from java.sql.Date.
    */
   def fromJavaDate(date: Date): SQLDate = {
-    val era = if (date.before(julianCommonEraStart)) 0 else 1
-    val localDate = LocalDate
-      .of(date.getYear + 1900, date.getMonth + 1, 1)
-      .`with`(ChronoField.ERA, era)
-      // Add days separately to convert dates existed in Julian calendar but not
-      // in Proleptic Gregorian calendar. For example, 1000-02-29 is valid date
-      // in Julian calendar because 1000 is a leap year but 1000 is not a leap
-      // year in Proleptic Gregorian calendar. And 1000-02-29 doesn't exist in it.
-      .plusDays(date.getDate - 1) // Returns the next valid date after `date.getDate - 1` days
-    localDateToDays(localDate)
+    val millisUtc = date.getTime
+    val millisLocal = millisUtc + TimeZone.getDefault.getOffset(millisUtc)
+    val julianDays = Math.toIntExact(Math.floorDiv(millisLocal, MILLIS_PER_DAY))
+    rebaseJulianToGregorianDays(julianDays)
   }
 
   /**
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 27917c204f577..1323a013332d2 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    447            486          54         22.4          44.7       1.0X
-cast to timestamp wholestage on                     364            381          14         27.5          36.4       1.2X
+cast to timestamp wholestage off                    440            459          28         22.7          44.0       1.0X
+cast to timestamp wholestage on                     384            407          15         26.1          38.4       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1366           1374          11          7.3         136.6       1.0X
-year of timestamp wholestage on                    1278           1302          21          7.8         127.8       1.1X
+year of timestamp wholestage off                   1334           1352          27          7.5         133.4       1.0X
+year of timestamp wholestage on                    1301           1309           9          7.7         130.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1606           1622          23          6.2         160.6       1.0X
-quarter of timestamp wholestage on                 1569           1584          16          6.4         156.9       1.0X
+quarter of timestamp wholestage off                1623           1636          18          6.2         162.3       1.0X
+quarter of timestamp wholestage on                 1592           1603           9          6.3         159.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1243           1259          24          8.0         124.3       1.0X
-month of timestamp wholestage on                   1267           1282          13          7.9         126.7       1.0X
+month of timestamp wholestage off                  1246           1250           5          8.0         124.6       1.0X
+month of timestamp wholestage on                   1277           1290          15          7.8         127.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1837           1846          12          5.4         183.7       1.0X
-weekofyear of timestamp wholestage on              1842           1848           4          5.4         184.2       1.0X
+weekofyear of timestamp wholestage off             1881           1896          20          5.3         188.1       1.0X
+weekofyear of timestamp wholestage on              1865           1879          16          5.4         186.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1262           1289          39          7.9         126.2       1.0X
-day of timestamp wholestage on                     1263           1268           6          7.9         126.3       1.0X
+day of timestamp wholestage off                    1223           1249          37          8.2         122.3       1.0X
+day of timestamp wholestage on                     1261           1276          19          7.9         126.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1302           1308           8          7.7         130.2       1.0X
-dayofyear of timestamp wholestage on               1300           1308           8          7.7         130.0       1.0X
+dayofyear of timestamp wholestage off              1291           1312          29          7.7         129.1       1.0X
+dayofyear of timestamp wholestage on               1284           1290           6          7.8         128.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1244           1248           6          8.0         124.4       1.0X
-dayofmonth of timestamp wholestage on              1253           1263          10          8.0         125.3       1.0X
+dayofmonth of timestamp wholestage off             1241           1246           7          8.1         124.1       1.0X
+dayofmonth of timestamp wholestage on              1268           1276           6          7.9         126.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1407           1408           2          7.1         140.7       1.0X
-dayofweek of timestamp wholestage on               1411           1432          14          7.1         141.1       1.0X
+dayofweek of timestamp wholestage off              1402           1408           9          7.1         140.2       1.0X
+dayofweek of timestamp wholestage on               1414           1423           8          7.1         141.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1349           1352           5          7.4         134.9       1.0X
-weekday of timestamp wholestage on                 1347           1354          10          7.4         134.7       1.0X
+weekday of timestamp wholestage off                1349           1350           2          7.4         134.9       1.0X
+weekday of timestamp wholestage on                 1344           1352           5          7.4         134.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1038           1041           5          9.6         103.8       1.0X
-hour of timestamp wholestage on                     983            995           9         10.2          98.3       1.1X
+hour of timestamp wholestage off                    974            974           0         10.3          97.4       1.0X
+hour of timestamp wholestage on                    1009           1019          15          9.9         100.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1042           1043           1          9.6         104.2       1.0X
-minute of timestamp wholestage on                  1022           1030           9          9.8         102.2       1.0X
+minute of timestamp wholestage off                 1024           1028           6          9.8         102.4       1.0X
+minute of timestamp wholestage on                   978            992          11         10.2          97.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  978            979           2         10.2          97.8       1.0X
-second of timestamp wholestage on                  1016           1028          21          9.8         101.6       1.0X
+second of timestamp wholestage off                 1023           1026           4          9.8         102.3       1.0X
+second of timestamp wholestage on                  1008           1024          14          9.9         100.8       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         295            299           6         33.9          29.5       1.0X
-current_date wholestage on                          321            327           7         31.2          32.1       0.9X
+current_date wholestage off                         296            299           5         33.8          29.6       1.0X
+current_date wholestage on                          316            318           2         31.7          31.6       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    342            387          63         29.2          34.2       1.0X
-current_timestamp wholestage on                     324            376          97         30.9          32.4       1.1X
+current_timestamp wholestage off                    310            345          49         32.2          31.0       1.0X
+current_timestamp wholestage on                     316            332          11         31.7          31.6       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1065           1067           3          9.4         106.5       1.0X
-cast to date wholestage on                         1044           1053          10          9.6         104.4       1.0X
+cast to date wholestage off                        1067           1071           7          9.4         106.7       1.0X
+cast to date wholestage on                         1050           1072          20          9.5         105.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1241           1245           6          8.1         124.1       1.0X
-last_day wholestage on                             1276           1287          11          7.8         127.6       1.0X
+last_day wholestage off                            1240           1245           8          8.1         124.0       1.0X
+last_day wholestage on                             1273           1290          14          7.9         127.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1130           1150          28          8.8         113.0       1.0X
-next_day wholestage on                             1086           1096           8          9.2         108.6       1.0X
+next_day wholestage off                            1121           1130          13          8.9         112.1       1.0X
+next_day wholestage on                             1099           1106          11          9.1         109.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1048           1054           9          9.5         104.8       1.0X
-date_add wholestage on                             1077           1079           2          9.3         107.7       1.0X
+date_add wholestage off                            1045           1051           8          9.6         104.5       1.0X
+date_add wholestage on                             1078           1086          16          9.3         107.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1047           1052           7          9.6         104.7       1.0X
-date_sub wholestage on                             1078           1087           7          9.3         107.8       1.0X
+date_sub wholestage off                            1049           1050           1          9.5         104.9       1.0X
+date_sub wholestage on                             1078           1083           4          9.3         107.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1392           1421          41          7.2         139.2       1.0X
-add_months wholestage on                           1441           1447           6          6.9         144.1       1.0X
+add_months wholestage off                          1411           1415           7          7.1         141.1       1.0X
+add_months wholestage on                           1435           1447          13          7.0         143.5       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         6034           6047          19          1.7         603.4       1.0X
-format date wholestage on                          5874           5891          16          1.7         587.4       1.0X
+format date wholestage off                         5736           5786          71          1.7         573.6       1.0X
+format date wholestage on                          5550           5575          29          1.8         555.0       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7350           7371          29          1.4         735.0       1.0X
-from_unixtime wholestage on                        7415           7428          13          1.3         741.5       1.0X
+from_unixtime wholestage off                       7378           7379           1          1.4         737.8       1.0X
+from_unixtime wholestage on                        7441           7465          24          1.3         744.1       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1235           1236           2          8.1         123.5       1.0X
-from_utc_timestamp wholestage on                   1300           1309          10          7.7         130.0       0.9X
+from_utc_timestamp wholestage off                  1238           1260          30          8.1         123.8       1.0X
+from_utc_timestamp wholestage on                   1300           1308           5          7.7         130.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1798           1805          10          5.6         179.8       1.0X
-to_utc_timestamp wholestage on                     1765           1779          11          5.7         176.5       1.0X
+to_utc_timestamp wholestage off                    1790           1793           5          5.6         179.0       1.0X
+to_utc_timestamp wholestage on                     1737           1748          13          5.8         173.7       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        345            350           7         29.0          34.5       1.0X
-cast interval wholestage on                         325            337          11         30.7          32.5       1.1X
+cast interval wholestage off                        347            350           4         28.8          34.7       1.0X
+cast interval wholestage on                         327            329           4         30.6          32.7       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1833           1834           2          5.5         183.3       1.0X
-datediff wholestage on                             1807           1815           9          5.5         180.7       1.0X
+datediff wholestage off                            1836           1860          33          5.4         183.6       1.0X
+datediff wholestage on                             1811           1826          16          5.5         181.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5938           5941           4          1.7         593.8       1.0X
-months_between wholestage on                       5891           5920          19          1.7         589.1       1.0X
+months_between wholestage off                      5613           5617           6          1.8         561.3       1.0X
+months_between wholestage on                       5590           5624          35          1.8         559.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2153           2313         226          0.5        2153.3       1.0X
-window wholestage on                              47097          47143          31          0.0       47097.0       0.0X
+window wholestage off                              2222           2308         122          0.5        2222.2       1.0X
+window wholestage on                              48721          48828         101          0.0       48720.7       0.0X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2458           2461           5          4.1         245.8       1.0X
-date_trunc YEAR wholestage on                      2433           2441           6          4.1         243.3       1.0X
+date_trunc YEAR wholestage off                     2482           2503          30          4.0         248.2       1.0X
+date_trunc YEAR wholestage on                      2520           2532          12          4.0         252.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2454           2457           4          4.1         245.4       1.0X
-date_trunc YYYY wholestage on                      2436           2442           7          4.1         243.6       1.0X
+date_trunc YYYY wholestage off                     2487           2493           9          4.0         248.7       1.0X
+date_trunc YYYY wholestage on                      2521           2527           8          4.0         252.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2467           2469           3          4.1         246.7       1.0X
-date_trunc YY wholestage on                        2435           2448          18          4.1         243.5       1.0X
+date_trunc YY wholestage off                       2479           2492          18          4.0         247.9       1.0X
+date_trunc YY wholestage on                        2519           2529           8          4.0         251.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2494           2494           1          4.0         249.4       1.0X
-date_trunc MON wholestage on                       2428           2441          14          4.1         242.8       1.0X
+date_trunc MON wholestage off                      2504           2519          21          4.0         250.4       1.0X
+date_trunc MON wholestage on                       2549           2563          15          3.9         254.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2502           2510          12          4.0         250.2       1.0X
-date_trunc MONTH wholestage on                     2438           2444           6          4.1         243.8       1.0X
+date_trunc MONTH wholestage off                    2517           2527          14          4.0         251.7       1.0X
+date_trunc MONTH wholestage on                     2538           2547          11          3.9         253.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2486           2487           1          4.0         248.6       1.0X
-date_trunc MM wholestage on                        2436           2441           4          4.1         243.6       1.0X
+date_trunc MM wholestage off                       2506           2507           2          4.0         250.6       1.0X
+date_trunc MM wholestage on                        2536           2546          10          3.9         253.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2406           2421          21          4.2         240.6       1.0X
-date_trunc DAY wholestage on                       2352           2370          23          4.3         235.2       1.0X
+date_trunc DAY wholestage off                      2443           2454          17          4.1         244.3       1.0X
+date_trunc DAY wholestage on                       2468           2479          12          4.1         246.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2414           2417           5          4.1         241.4       1.0X
-date_trunc DD wholestage on                        2355           2366           8          4.2         235.5       1.0X
+date_trunc DD wholestage off                       2440           2440           0          4.1         244.0       1.0X
+date_trunc DD wholestage on                        2459           2481          23          4.1         245.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2419           2426          10          4.1         241.9       1.0X
-date_trunc HOUR wholestage on                      2369           2374           4          4.2         236.9       1.0X
+date_trunc HOUR wholestage off                     2452           2453           2          4.1         245.2       1.0X
+date_trunc HOUR wholestage on                      2428           2435           5          4.1         242.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    357            378          30         28.0          35.7       1.0X
-date_trunc MINUTE wholestage on                     331            335           4         30.2          33.1       1.1X
+date_trunc MINUTE wholestage off                    359            359           1         27.8          35.9       1.0X
+date_trunc MINUTE wholestage on                     382            388           8         26.2          38.2       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    373            375           3         26.8          37.3       1.0X
-date_trunc SECOND wholestage on                     330            335           4         30.3          33.0       1.1X
+date_trunc SECOND wholestage off                    382            386           6         26.2          38.2       1.0X
+date_trunc SECOND wholestage on                     379            382           5         26.4          37.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2479           2480           2          4.0         247.9       1.0X
-date_trunc WEEK wholestage on                      2441           2450           8          4.1         244.1       1.0X
+date_trunc WEEK wholestage off                     2427           2429           3          4.1         242.7       1.0X
+date_trunc WEEK wholestage on                      2427           2441          12          4.1         242.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3337           3337           0          3.0         333.7       1.0X
-date_trunc QUARTER wholestage on                   3235           3244           8          3.1         323.5       1.0X
+date_trunc QUARTER wholestage off                  3551           3583          46          2.8         355.1       1.0X
+date_trunc QUARTER wholestage on                   3517           3526           6          2.8         351.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           315            316           1         31.7          31.5       1.0X
-trunc year wholestage on                            293            302          16         34.2          29.3       1.1X
+trunc year wholestage off                           312            313           1         32.0          31.2       1.0X
+trunc year wholestage on                            319            326           7         31.3          31.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           312            314           3         32.0          31.2       1.0X
-trunc yyyy wholestage on                            295            300           8         33.9          29.5       1.1X
+trunc yyyy wholestage off                           318            318           0         31.5          31.8       1.0X
+trunc yyyy wholestage on                            317            328          17         31.5          31.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             311            317           8         32.1          31.1       1.0X
-trunc yy wholestage on                              294            302           7         34.0          29.4       1.1X
+trunc yy wholestage off                             315            322          10         31.8          31.5       1.0X
+trunc yy wholestage on                              318            328           9         31.5          31.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            312            314           3         32.0          31.2       1.0X
-trunc mon wholestage on                             296            299           5         33.8          29.6       1.1X
+trunc mon wholestage off                            311            313           2         32.1          31.1       1.0X
+trunc mon wholestage on                             322            326           6         31.0          32.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          311            311           0         32.1          31.1       1.0X
-trunc month wholestage on                           293            297           4         34.2          29.3       1.1X
+trunc month wholestage off                          310            318          11         32.2          31.0       1.0X
+trunc month wholestage on                           319            326           8         31.3          31.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             315            320           7         31.7          31.5       1.0X
-trunc mm wholestage on                              291            296           8         34.4          29.1       1.1X
+trunc mm wholestage off                             311            314           4         32.2          31.1       1.0X
+trunc mm wholestage on                              318            324           7         31.4          31.8       1.0X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     161            171          15          6.2         160.6       1.0X
-to timestamp str wholestage on                      152            154           1          6.6         151.6       1.1X
+to timestamp str wholestage off                     166            167           2          6.0         165.7       1.0X
+to timestamp str wholestage on                      158            162           4          6.3         158.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1441           1450          13          0.7        1440.6       1.0X
-to_timestamp wholestage on                         1371           1381          20          0.7        1370.8       1.1X
+to_timestamp wholestage off                        1487           1488           1          0.7        1487.1       1.0X
+to_timestamp wholestage on                         1418           1422           4          0.7        1418.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1329           1335           9          0.8        1328.7       1.0X
-to_unix_timestamp wholestage on                    1354           1357           4          0.7        1353.7       1.0X
+to_unix_timestamp wholestage off                   1409           1420          15          0.7        1409.2       1.0X
+to_unix_timestamp wholestage on                    1422           1428           8          0.7        1422.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          212            213           2          4.7         211.7       1.0X
-to date str wholestage on                           206            207           1          4.9         205.9       1.0X
+to date str wholestage off                          207            207           0          4.8         206.9       1.0X
+to date str wholestage on                           209            210           1          4.8         209.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3255           3309          76          0.3        3255.0       1.0X
-to_date wholestage on                              3146           3168          23          0.3        3146.4       1.0X
+to_date wholestage off                             3378           3398          28          0.3        3377.7       1.0X
+to_date wholestage on                              3372           3379           8          0.3        3372.2       1.0X
 
 
 ================================================================================================
@@ -420,10 +420,11 @@ Conversion from/to external types
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             359            365           8         13.9          71.8       1.0X
-Collect longs                                      1832           1895          84          2.7         366.3       0.2X
-Collect timestamps                                 1811           2046         247          2.8         362.2       0.2X
+From java.sql.Date                                  427            443          23         11.7          85.4       1.0X
+From java.sql.Timestamp                             361            369          11         13.8          72.3       1.2X
+Collect longs                                      1942           2044         170          2.6         388.3       0.2X
+Collect timestamps                                 1891           2093         192          2.6         378.2       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index ee70aa389f326..77b6f833221e5 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    416            439          32         24.0          41.6       1.0X
-cast to timestamp wholestage on                     370            399          31         27.1          37.0       1.1X
+cast to timestamp wholestage off                    410            428          25         24.4          41.0       1.0X
+cast to timestamp wholestage on                     364            379          17         27.5          36.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1328           1343          21          7.5         132.8       1.0X
-year of timestamp wholestage on                    1204           1247          27          8.3         120.4       1.1X
+year of timestamp wholestage off                   1308           1318          15          7.6         130.8       1.0X
+year of timestamp wholestage on                    1204           1234          19          8.3         120.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1447           1451           6          6.9         144.7       1.0X
-quarter of timestamp wholestage on                 1384           1392           7          7.2         138.4       1.0X
+quarter of timestamp wholestage off                1439           1442           6          7.0         143.9       1.0X
+quarter of timestamp wholestage on                 1374           1385           9          7.3         137.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1233           1238           6          8.1         123.3       1.0X
-month of timestamp wholestage on                   1232           1252          15          8.1         123.2       1.0X
+month of timestamp wholestage off                  1228           1234           8          8.1         122.8       1.0X
+month of timestamp wholestage on                   1225           1241          25          8.2         122.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1921           1923           3          5.2         192.1       1.0X
-weekofyear of timestamp wholestage on              1893           1914          14          5.3         189.3       1.0X
+weekofyear of timestamp wholestage off             1878           1880           3          5.3         187.8       1.0X
+weekofyear of timestamp wholestage on              1864           1870           6          5.4         186.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1224           1227           4          8.2         122.4       1.0X
-day of timestamp wholestage on                     1219           1238          15          8.2         121.9       1.0X
+day of timestamp wholestage off                    1218           1220           3          8.2         121.8       1.0X
+day of timestamp wholestage on                     1217           1226          10          8.2         121.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1283           1283           1          7.8         128.3       1.0X
-dayofyear of timestamp wholestage on               1256           1261           5          8.0         125.6       1.0X
+dayofyear of timestamp wholestage off              1274           1282          12          7.8         127.4       1.0X
+dayofyear of timestamp wholestage on               1263           1274          14          7.9         126.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1227           1233           8          8.2         122.7       1.0X
-dayofmonth of timestamp wholestage on              1220           1228           6          8.2         122.0       1.0X
+dayofmonth of timestamp wholestage off             1221           1226           6          8.2         122.1       1.0X
+dayofmonth of timestamp wholestage on              1215           1222           9          8.2         121.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1404           1408           5          7.1         140.4       1.0X
-dayofweek of timestamp wholestage on               1376           1386           6          7.3         137.6       1.0X
+dayofweek of timestamp wholestage off              1404           1431          38          7.1         140.4       1.0X
+dayofweek of timestamp wholestage on               1374           1381           8          7.3         137.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1346           1350           5          7.4         134.6       1.0X
-weekday of timestamp wholestage on                 1313           1319           6          7.6         131.3       1.0X
+weekday of timestamp wholestage off                1342           1350          12          7.5         134.2       1.0X
+weekday of timestamp wholestage on                 1311           1317           5          7.6         131.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1005           1011           8         10.0         100.5       1.0X
-hour of timestamp wholestage on                     953            961           9         10.5          95.3       1.1X
+hour of timestamp wholestage off                    997           1000           4         10.0          99.7       1.0X
+hour of timestamp wholestage on                     959            969           8         10.4          95.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  995           1000           8         10.1          99.5       1.0X
-minute of timestamp wholestage on                   957            959           3         10.5          95.7       1.0X
+minute of timestamp wholestage off                  968            970           2         10.3          96.8       1.0X
+minute of timestamp wholestage on                   952            960           8         10.5          95.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  960            963           4         10.4          96.0       1.0X
-second of timestamp wholestage on                   957            960           3         10.4          95.7       1.0X
+second of timestamp wholestage off                  971            974           5         10.3          97.1       1.0X
+second of timestamp wholestage on                   952            953           0         10.5          95.2       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         290            295           7         34.4          29.0       1.0X
-current_date wholestage on                          287            295          13         34.9          28.7       1.0X
+current_date wholestage off                         277            282           7         36.1          27.7       1.0X
+current_date wholestage on                          283            290          11         35.3          28.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    294            296           4         34.1          29.4       1.0X
-current_timestamp wholestage on                     280            303          19         35.7          28.0       1.0X
+current_timestamp wholestage off                    302            324          31         33.1          30.2       1.0X
+current_timestamp wholestage on                     281            292          13         35.6          28.1       1.1X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1066           1067           1          9.4         106.6       1.0X
-cast to date wholestage on                         1007           1014           6          9.9         100.7       1.1X
+cast to date wholestage off                        1064           1066           2          9.4         106.4       1.0X
+cast to date wholestage on                         1002           1010           8         10.0         100.2       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1273           1289          22          7.9         127.3       1.0X
-last_day wholestage on                             1238           1246           9          8.1         123.8       1.0X
+last_day wholestage off                            1247           1250           4          8.0         124.7       1.0X
+last_day wholestage on                             1238           1245           4          8.1         123.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1124           1125           2          8.9         112.4       1.0X
-next_day wholestage on                             1058           1064           8          9.5         105.8       1.1X
+next_day wholestage off                            1112           1113           1          9.0         111.2       1.0X
+next_day wholestage on                             1055           1059           5          9.5         105.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1049           1049           0          9.5         104.9       1.0X
-date_add wholestage on                             1036           1041           7          9.6         103.6       1.0X
+date_add wholestage off                            1049           1051           3          9.5         104.9       1.0X
+date_add wholestage on                             1036           1043           9          9.6         103.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1051           1052           2          9.5         105.1       1.0X
-date_sub wholestage on                             1040           1046           5          9.6         104.0       1.0X
+date_sub wholestage off                            1043           1063          29          9.6         104.3       1.0X
+date_sub wholestage on                             1036           1041           5          9.6         103.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1381           1388          11          7.2         138.1       1.0X
-add_months wholestage on                           1359           1367           8          7.4         135.9       1.0X
+add_months wholestage off                          1371           1373           2          7.3         137.1       1.0X
+add_months wholestage on                           1402           1409          10          7.1         140.2       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5766           5793          38          1.7         576.6       1.0X
-format date wholestage on                          5803           5812           7          1.7         580.3       1.0X
+format date wholestage off                         5741           5862         171          1.7         574.1       1.0X
+format date wholestage on                          5589           5600          19          1.8         558.9       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8891           8915          34          1.1         889.1       1.0X
-from_unixtime wholestage on                        8870           8885          12          1.1         887.0       1.0X
+from_unixtime wholestage off                       8681           8684           4          1.2         868.1       1.0X
+from_unixtime wholestage on                        8669           8707          33          1.2         866.9       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1089           1094           7          9.2         108.9       1.0X
-from_utc_timestamp wholestage on                   1105           1107           3          9.0         110.5       1.0X
+from_utc_timestamp wholestage off                  1096           1101           7          9.1         109.6       1.0X
+from_utc_timestamp wholestage on                   1108           1116           6          9.0         110.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1620           1628          11          6.2         162.0       1.0X
-to_utc_timestamp wholestage on                     1600           1608           7          6.3         160.0       1.0X
+to_utc_timestamp wholestage off                    1626           1638          16          6.1         162.6       1.0X
+to_utc_timestamp wholestage on                     1594           1606          10          6.3         159.4       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        334            336           3         30.0          33.4       1.0X
-cast interval wholestage on                         305            310           4         32.7          30.5       1.1X
+cast interval wholestage off                        331            335           6         30.2          33.1       1.0X
+cast interval wholestage on                         298            305           5         33.5          29.8       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1811           1816           7          5.5         181.1       1.0X
-datediff wholestage on                             1758           1761           3          5.7         175.8       1.0X
+datediff wholestage off                            1821           1822           2          5.5         182.1       1.0X
+datediff wholestage on                             1759           1763           3          5.7         175.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      4780           4789          12          2.1         478.0       1.0X
-months_between wholestage on                       4765           4772           8          2.1         476.5       1.0X
+months_between wholestage off                      4752           4770          25          2.1         475.2       1.0X
+months_between wholestage on                       4740           4746           5          2.1         474.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1840           1967         179          0.5        1840.4       1.0X
-window wholestage on                              46710          46792          57          0.0       46709.5       0.0X
+window wholestage off                              1936           2049         160          0.5        1935.7       1.0X
+window wholestage on                              44821          44850          44          0.0       44820.6       0.0X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2350           2351           1          4.3         235.0       1.0X
-date_trunc YEAR wholestage on                      2317           2324           8          4.3         231.7       1.0X
+date_trunc YEAR wholestage off                     2382           2384           3          4.2         238.2       1.0X
+date_trunc YEAR wholestage on                      2309           2316           7          4.3         230.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2341           2343           3          4.3         234.1       1.0X
-date_trunc YYYY wholestage on                      2314           2332          22          4.3         231.4       1.0X
+date_trunc YYYY wholestage off                     2368           2375          10          4.2         236.8       1.0X
+date_trunc YYYY wholestage on                      2304           2313           7          4.3         230.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2348           2354           9          4.3         234.8       1.0X
-date_trunc YY wholestage on                        2313           2317           3          4.3         231.3       1.0X
+date_trunc YY wholestage off                       2381           2396          22          4.2         238.1       1.0X
+date_trunc YY wholestage on                        2303           2305           1          4.3         230.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2383           2385           3          4.2         238.3       1.0X
-date_trunc MON wholestage on                       2293           2307          22          4.4         229.3       1.0X
+date_trunc MON wholestage off                      2325           2333          11          4.3         232.5       1.0X
+date_trunc MON wholestage on                       2293           2301           6          4.4         229.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2380           2393          18          4.2         238.0       1.0X
-date_trunc MONTH wholestage on                     2296           2304           9          4.4         229.6       1.0X
+date_trunc MONTH wholestage off                    2335           2335           0          4.3         233.5       1.0X
+date_trunc MONTH wholestage on                     2296           2298           2          4.4         229.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2375           2380           7          4.2         237.5       1.0X
-date_trunc MM wholestage on                        2295           2302           8          4.4         229.5       1.0X
+date_trunc MM wholestage off                       2328           2329           1          4.3         232.8       1.0X
+date_trunc MM wholestage on                        2290           2295           4          4.4         229.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2145           2145           1          4.7         214.5       1.0X
-date_trunc DAY wholestage on                       2090           2096           6          4.8         209.0       1.0X
+date_trunc DAY wholestage off                      2139           2140           1          4.7         213.9       1.0X
+date_trunc DAY wholestage on                       2095           2101           7          4.8         209.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2142           2149          11          4.7         214.2       1.0X
-date_trunc DD wholestage on                        2090           2096           5          4.8         209.0       1.0X
+date_trunc DD wholestage off                       2143           2145           2          4.7         214.3       1.0X
+date_trunc DD wholestage on                        2087           2092           3          4.8         208.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2204           2212          11          4.5         220.4       1.0X
-date_trunc HOUR wholestage on                      2196           2201           5          4.6         219.6       1.0X
+date_trunc HOUR wholestage off                     2201           2207           8          4.5         220.1       1.0X
+date_trunc HOUR wholestage on                      2176           2185           6          4.6         217.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    353            353           1         28.3          35.3       1.0X
-date_trunc MINUTE wholestage on                     331            335           4         30.2          33.1       1.1X
+date_trunc MINUTE wholestage off                    356            356           0         28.1          35.6       1.0X
+date_trunc MINUTE wholestage on                     333            344           7         30.1          33.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    343            345           3         29.2          34.3       1.0X
-date_trunc SECOND wholestage on                     335            336           2         29.9          33.5       1.0X
+date_trunc SECOND wholestage off                    341            342           2         29.3          34.1       1.0X
+date_trunc SECOND wholestage on                     335            349          18         29.9          33.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2277           2279           3          4.4         227.7       1.0X
-date_trunc WEEK wholestage on                      2262           2270          10          4.4         226.2       1.0X
+date_trunc WEEK wholestage off                     2257           2257           1          4.4         225.7       1.0X
+date_trunc WEEK wholestage on                      2202           2211          11          4.5         220.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3395           3395           0          2.9         339.5       1.0X
-date_trunc QUARTER wholestage on                   3185           3193           6          3.1         318.5       1.1X
+date_trunc QUARTER wholestage off                  3180           3181           2          3.1         318.0       1.0X
+date_trunc QUARTER wholestage on                   3036           3043          12          3.3         303.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           321            330          14         31.2          32.1       1.0X
-trunc year wholestage on                            293            308          14         34.1          29.3       1.1X
+trunc year wholestage off                           327            328           1         30.6          32.7       1.0X
+trunc year wholestage on                            310            325          13         32.3          31.0       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           322            324           2         31.1          32.2       1.0X
-trunc yyyy wholestage on                            293            296           4         34.1          29.3       1.1X
+trunc yyyy wholestage off                           323            327           6         31.0          32.3       1.0X
+trunc yyyy wholestage on                            304            317          14         32.9          30.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             322            322           0         31.0          32.2       1.0X
-trunc yy wholestage on                              292            296           4         34.2          29.2       1.1X
+trunc yy wholestage off                             324            325           2         30.8          32.4       1.0X
+trunc yy wholestage on                              308            311           3         32.4          30.8       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            319            320           1         31.3          31.9       1.0X
-trunc mon wholestage on                             293            295           1         34.1          29.3       1.1X
+trunc mon wholestage off                            323            323           1         31.0          32.3       1.0X
+trunc mon wholestage on                             303            310           7         33.0          30.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          319            320           1         31.3          31.9       1.0X
-trunc month wholestage on                           293            295           2         34.1          29.3       1.1X
+trunc month wholestage off                          322            323           0         31.0          32.2       1.0X
+trunc month wholestage on                           307            337          65         32.6          30.7       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             318            319           1         31.4          31.8       1.0X
-trunc mm wholestage on                              293            312          26         34.1          29.3       1.1X
+trunc mm wholestage off                             324            325           2         30.9          32.4       1.0X
+trunc mm wholestage on                              294            306           7         34.0          29.4       1.1X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     220            226           9          4.5         220.3       1.0X
-to timestamp str wholestage on                      212            214           4          4.7         212.2       1.0X
+to timestamp str wholestage off                     216            216           1          4.6         215.7       1.0X
+to timestamp str wholestage on                      207            209           3          4.8         207.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1875           1876           2          0.5        1874.7       1.0X
-to_timestamp wholestage on                         2075           2091          25          0.5        2075.1       0.9X
+to_timestamp wholestage off                        1751           1751           1          0.6        1750.6       1.0X
+to_timestamp wholestage on                         1798           1832          31          0.6        1798.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   2141           2143           2          0.5        2141.2       1.0X
-to_unix_timestamp wholestage on                    2145           2177          21          0.5        2144.7       1.0X
+to_unix_timestamp wholestage off                   1770           1773           4          0.6        1770.2       1.0X
+to_unix_timestamp wholestage on                    1743           1754          10          0.6        1743.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          276            284          11          3.6         276.1       1.0X
-to date str wholestage on                           264            270           8          3.8         264.3       1.0X
+to date str wholestage off                          260            266           9          3.8         260.1       1.0X
+to date str wholestage on                           257            260           3          3.9         257.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3876           3903          38          0.3        3876.1       1.0X
-to_date wholestage on                              3833           3843           6          0.3        3833.4       1.0X
+to_date wholestage off                             3589           3620          44          0.3        3589.0       1.0X
+to_date wholestage on                              3565           3579          15          0.3        3565.0       1.0X
 
 
 ================================================================================================
@@ -420,10 +420,11 @@ Conversion from/to external types
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-To/from java.sql.Timestamp:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Timestamp                             394            399           5         12.7          78.7       1.0X
-Collect longs                                      1336           2676        1201          3.7         267.2       0.3X
-Collect timestamps                                 2025           2091          65          2.5         405.0       0.2X
+From java.sql.Date                                  416            420           6         12.0          83.2       1.0X
+From java.sql.Timestamp                             375            381           5         13.3          75.1       1.1X
+Collect longs                                      1225           1962        1101          4.1         245.1       0.3X
+Collect timestamps                                 2013           2829        1401          2.5         402.5       0.2X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 5a857622da6f6..5fa5b9b06a60b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -131,7 +131,12 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           import spark.implicits._
           val rowsNum = 5000000
           val numIters = 3
-          val benchmark = new Benchmark("To/from java.sql.Timestamp", rowsNum, output = output)
+          val benchmark = new Benchmark("To/from Java's date-time", rowsNum, output = output)
+          benchmark.addCase("From java.sql.Date", numIters) { _ =>
+            spark.range(rowsNum)
+              .map(millis => new java.sql.Date(millis))
+              .noop()
+          }
           benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
             spark.range(rowsNum)
               .map(millis => new Timestamp(millis))

From 2a2736688b01a6502dc969bec9a52c4bd671fd8b Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Tue, 14 Apr 2020 21:28:02 -0700
Subject: [PATCH 0500/1280] [SPARK-31420][WEBUI][3.0] Infinite timeline redraw
 in job details page

### What changes were proposed in this pull request?

This PR backports #28192 to branch-3.0.

### Why are the changes needed?

SPARK-31420 affects branch-3.0 too.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Same as #28192

Closes #28213 from sarutak/SPARK-31420-branch-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../apache/spark/ui/static/timeline-view.css  | 12 ----
 .../apache/spark/ui/static/timeline-view.js   | 29 ++++++---
 .../ui/static/vis-timeline-graph2d.min.css    |  2 +
 .../static/vis-timeline-graph2d.min.css.map   |  1 +
 .../ui/static/vis-timeline-graph2d.min.js     | 60 +++++++++++++++++++
 .../ui/static/vis-timeline-graph2d.min.js.map |  1 +
 .../org/apache/spark/ui/static/vis.min.css    |  1 -
 .../org/apache/spark/ui/static/vis.min.js     | 45 --------------
 .../scala/org/apache/spark/ui/UIUtils.scala   |  5 +-
 .../org/apache/spark/ui/UISeleniumSuite.scala |  2 +-
 dev/.rat-excludes                             |  6 +-
 licenses-binary/LICENSE-vis-timeline.txt      | 23 +++++++
 licenses-binary/LICENSE-vis.txt               | 22 -------
 licenses/LICENSE-vis-timeline.txt             | 23 +++++++
 licenses/LICENSE-vis.txt                      | 22 -------
 15 files changed, 140 insertions(+), 114 deletions(-)
 create mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
 create mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
 create mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
 create mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
 delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis.min.css
 delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis.min.js
 create mode 100644 licenses-binary/LICENSE-vis-timeline.txt
 delete mode 100644 licenses-binary/LICENSE-vis.txt
 create mode 100644 licenses/LICENSE-vis-timeline.txt
 delete mode 100644 licenses/LICENSE-vis.txt

diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
index 3f31403eaeef3..c9bf83ca98b4f 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.css
@@ -238,18 +238,6 @@ tr.corresponding-item-hover > td, tr.corresponding-item-hover > th {
   background-color: #D6FFE4 !important;
 }
 
-#application-timeline.collapsed {
-  display: none;
-}
-
-#job-timeline.collapsed {
-  display: none;
-}
-
-#task-assignment-timeline.collapsed {
-  display: none;
-}
-
 .control-panel {
   margin-bottom: 5px;
 }
diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
index b2cd616791734..a63ee8696f906 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -26,7 +26,7 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
     editable: false,
     align: 'left',
     showCurrentTime: false,
-    min: startTime,
+    start: startTime,
     zoomable: false,
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
@@ -50,7 +50,7 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
       };
 
       $(this).click(function() {
-        var jobPagePath = $(getSelectorForJobEntry(this)).find("a.name-link").attr("href")
+        var jobPagePath = $(getSelectorForJobEntry(this)).find("a.name-link").attr("href");
           window.location.href = jobPagePath
       });
 
@@ -75,6 +75,9 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
 
     $("#application-timeline").toggleClass('collapsed');
 
+    var visibilityState = status ? "" : "none";
+    $("#application-timeline").css("display", visibilityState);
+
     // Switch the class of the arrow from open to closed.
     $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-open');
     $(this).find('.expand-application-timeline-arrow').toggleClass('arrow-closed');
@@ -89,6 +92,8 @@ $(function () {
     // Set it to false so that the click function can revert it
     window.localStorage.setItem("expand-application-timeline", "false");
     $("span.expand-application-timeline").trigger('click');
+  } else {
+    $("#application-timeline").css("display", "none");
   }
 });
 
@@ -103,7 +108,7 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
     editable: false,
     align: 'left',
     showCurrentTime: false,
-    min: startTime,
+    start: startTime,
     zoomable: false,
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
@@ -152,6 +157,9 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
 
     $("#job-timeline").toggleClass('collapsed');
 
+    var visibilityState = status ? "" : "none";
+    $("#job-timeline").css("display", visibilityState);
+
     // Switch the class of the arrow from open to closed.
     $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-open');
     $(this).find('.expand-job-timeline-arrow').toggleClass('arrow-closed');
@@ -166,13 +174,15 @@ $(function () {
     // Set it to false so that the click function can revert it
     window.localStorage.setItem("expand-job-timeline", "false");
     $("span.expand-job-timeline").trigger('click');
+  } else {
+    $("#job-timeline").css("display", "none");
   }
 });
 
 function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, maxFinishTime, offset) {
   var groups = new vis.DataSet(groupArray);
   var items = new vis.DataSet(eventObjArray);
-  var container = $("#task-assignment-timeline")[0]
+  var container = $("#task-assignment-timeline")[0];
   var options = {
     groupOrder: function(a, b) {
       return a.value - b.value
@@ -181,15 +191,15 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
     align: 'left',
     selectable: false,
     showCurrentTime: false,
-    min: minLaunchTime,
-    max: maxFinishTime,
+    start: minLaunchTime,
+    end: maxFinishTime,
     zoomable: false,
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }
   };
 
-  var taskTimeline = new vis.Timeline(container)
+  var taskTimeline = new vis.Timeline(container);
   taskTimeline.setOptions(options);
   taskTimeline.setGroups(groups);
   taskTimeline.setItems(items);
@@ -220,6 +230,9 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
 
     $("#task-assignment-timeline").toggleClass("collapsed");
 
+    var visibilityState = status ? "" : "none";
+    $("#task-assignment-timeline").css("display", visibilityState);
+
      // Switch the class of the arrow from open to closed.
     $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-open");
     $(this).find(".expand-task-assignment-timeline-arrow").toggleClass("arrow-closed");
@@ -234,6 +247,8 @@ $(function () {
     // Set it to false so that the click function can revert it
     window.localStorage.setItem("expand-task-assignment-timeline", "false");
     $("span.expand-task-assignment-timeline").trigger('click');
+  } else {
+    $("#task-assignment-timeline").css("display", "none");
   }
 });
 
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
new file mode 100644
index 0000000000000..d22a9ac801891
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
@@ -0,0 +1,2 @@
+.vis [class*=span]{min-height:0;width:auto}.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper:after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:transparent;width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2,#385380);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(180deg,#3876c2 0,#385380);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#3876c2",endColorstr="#385380",GradientType=0);box-shadow:0 0 1px 0 #111927;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:none}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(180deg,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#9d9d9d",endColorstr="#c8c8c8",GradientType=0)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:transparent;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0) rgba(136,183,213,0) rgba(136,183,213,0) rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0) rgba(194,225,245,0) rgba(194,225,245,0) #f2faff;border-width:12px;margin-top:-12px}div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}.vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content,.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\26F6"}.vis-rolling-mode-btn:hover{opacity:1}.vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-custom-time>.vis-custom-time-marker{background-color:inherit;color:#fff;font-size:12px;white-space:nowrap;padding:3px 5px;top:0;cursor:auto;z-index:inherit}.vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}.vis-loading-screen{width:100%;height:100%;position:absolute;top:0;left:0}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px hsla(0,0%,60.4%,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1{background:#f5f5f5}.vis-label.vis-nested-group.vis-group-level-0{background-color:#fff}.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-left:0}.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-right:0}.vis-label.vis-nested-group.vis-group-level-1{background-color:rgba(0,0,0,.05)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-left:15px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-right:15px}.vis-label.vis-nested-group.vis-group-level-2{background-color:rgba(0,0,0,.1)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-left:30px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-right:30px}.vis-label.vis-nested-group.vis-group-level-3{background-color:rgba(0,0,0,.15)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-left:45px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-right:45px}.vis-label.vis-nested-group.vis-group-level-4{background-color:rgba(0,0,0,.2)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-left:60px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-right:60px}.vis-label.vis-nested-group.vis-group-level-5{background-color:rgba(0,0,0,.25)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-left:75px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-right:75px}.vis-label.vis-nested-group.vis-group-level-6{background-color:rgba(0,0,0,.3)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-left:90px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-right:90px}.vis-label.vis-nested-group.vis-group-level-7{background-color:rgba(0,0,0,.35)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-left:105px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-right:105px}.vis-label.vis-nested-group.vis-group-level-8{background-color:rgba(0,0,0,.4)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-left:120px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-right:120px}.vis-label.vis-nested-group.vis-group-level-9{background-color:rgba(0,0,0,.45)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-left:135px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-right:135px}.vis-label.vis-nested-group{background-color:rgba(0,0,0,.5)}.vis-ltr .vis-label.vis-nested-group .vis-inner{padding-left:150px}.vis-rtl .vis-label.vis-nested-group .vis-inner{padding-right:150px}.vis-group-level-unknown-but-gte1{border:1px solid red}.vis-label.vis-nesting-group:before{display:inline-block;width:15px}.vis-label.vis-nesting-group.expanded:before{content:"\25BC"}.vis-label.vis-nesting-group.collapsed:before{content:"\25B6"}.vis-rtl .vis-label.vis-nesting-group.collapsed:before{content:"\25C0"}.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-left:15px}.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-right:15px}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-labelset{overflow:hidden}.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-group-is-dragging{background:rgba(0,0,0,.1)}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:none}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-left,.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0}.vis-item.vis-range .vis-drag-right{right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-item.vis-cluster{vertical-align:center;text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-cluster-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item.vis-cluster-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}
+/*# sourceMappingURL=vis-timeline-graph2d.min.css.map */
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
new file mode 100644
index 0000000000000..3c165a792d0f3
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
@@ -0,0 +1 @@
+{"version":3,"sources":["bootstrap.css","activator.css","configuration.css","tooltip.css","panel.css","currenttime.css","customtime.css","timeline.css","pathStyles.css","dataaxis.css","itemset.css","labelset.css","timeaxis.css","item.css"],"names":[],"mappings":"AAEA,mBACE,YAAa,CACb,UACF,CCLA,cACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WAAY,CAGZ,UACF,CAEA,YACE,2BACF,CCbA,sBACI,iBAAiB,CACjB,aAAa,CACb,UAAU,CACV,cACJ,CAEA,8BACI,aAAa,CACb,WACJ,CAEA,oCACE,UAAW,CACX,UAAW,CACX,aACF,CAEA,kDACI,aAAa,CACb,WAAW,CACX,qBAAyB,CACzB,wBAAwB,CACxB,iBAAiB,CACjB,eAAe,CACf,SAAS,CACT,gBACJ,CAEA,wCACI,aAAa,CACb,WAAW,CACX,WAAW,CACX,qBAAsB,CACtB,gBAAgB,CAChB,wBAAyB,CACzB,wBAAwB,CACxB,iBAAiB,CACjB,eAAe,CACf,SAAS,CACT,gBAAgB,CAChB,cAAe,CACf,kBACJ,CAEA,8CACI,wBAAyB,CACzB,wBAAwB,CACxB,UACJ,CAEA,sCACI,aAAa,CACb,UAAU,CACV,WAAW,CACX,WAAW,CACX,qBAAsB,CACtB,gBACJ,CAGA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CACA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CACA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CAEA,wCACI,cAAc,CACd,eACJ,CAEA,uCACI,WAAW,CACX,WAAW,CACX,gBACJ,CAEA,qDACI,WACJ,CACA,qDACI,WACJ,CAEA,4CACI,OAAO,CACP,UAAU,CACV,WAAW,CACX,qBAAwB,CACxB,iBAAiB,CACjB,SAAW,CACX,QAAU,CACV,cACJ,CAEA,4CACI,SACJ,CAGA,8CACI,iBAAiB,CACjB,QAAQ,CACR,UAAU,CAEV,WAAW,CACX,QAAQ,CACR,mBACJ,CAEA,yCAEI,uBAAwB,CAGxB,mBAAuB,CACvB,4BAA8B,CAG9B,WAAY,CACZ,WACJ,CACA,wEACI,WAAY,CACZ,UAAW,CACX,kBAAmB,CACnB,0DAA+D,CAC/D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GAAmH,CAEnH,qBAAyB,CACzB,yBAAmC,CACnC,iBACJ,CACA,+DACI,uBAAwB,CACxB,wBAAyB,CACzB,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,kBAAmB,CACnB,2DAAgE,CAChE,uGAA6G,CAC7G,uDAAkE,CAClE,yDAA6D,CAC7D,0DAA8D,CAC9D,oDAAgE,CAChE,+GAAmH,CACnH,4BAAmC,CACnC,eACJ,CACA,+CACI,YACJ,CACA,8EACI,kBAAmB,CACnB,0DAA8D,CAC9D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GACJ,CAEA,2DACI,WAAY,CACZ,WAAY,CACZ,kBAAmB,CACnB,0DAA+D,CAC/D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GAAmH,CAEnH,qBAAyB,CACzB,yBAAmC,CACnC,iBACJ,CACA,2DACI,WAAY,CACZ,WAAY,CACZ,UAAW,CAEX,iBAAkB,CAClB,kBACJ,CAGA,wDACI,sBAAwB,CACxB,mBACJ,CAEA,oDACI,WAAY,CACZ,UAAW,CAGX,sBAAuB,CAGvB,wBAAyB,CACzB,kBAAmB,CAGnB,iBACJ,CACA,yDACI,eAAgB,CAChB,kBACJ,CACA,yDACI,eAAgB,CAChB,kBACJ,CACA,oDACI,WAAY,CACZ,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,kBACJ,CACA,+DACI,eACJ,CACA,+DACI,eACJ,CAEA,yBACI,iBAAkB,CAClB,6BAAkC,CAClC,wBAAyB,CACzB,gBAAgB,CAChB,WAAW,CACX,WAAW,CACX,iBAAiB,CACjB,UAAc,CACd,cAAc,CACd,iBAAiB,CACjB,0CAA4C,CAC5C,uCAAyC,CACzC,kCACJ,CACA,+DACI,SAAU,CACV,OAAQ,CACR,wBAAyB,CACzB,WAAY,CACZ,QAAS,CACT,OAAQ,CACR,iBAAkB,CAClB,mBACJ,CAEA,+BAEI,2FAAyC,CACzC,gBAAiB,CACjB,eACJ,CACA,gCAEI,gFAA0B,CAC1B,iBAAkB,CAClB,gBACJ,CC/RA,gBACE,iBAAkB,CAClB,iBAAkB,CAClB,WAAY,CACZ,kBAAmB,CAEnB,mBAAoB,CACpB,cAAc,CACd,UAAa,CACb,wBAAyB,CAEzB,sBAAuB,CACvB,yBAA0B,CAC1B,iBAAkB,CAClB,wBAAyB,CAEzB,sCAA2C,CAC3C,mBAAoB,CAEpB,SACF,CCpBA,WACE,iBAAkB,CAElB,SAAU,CACV,QAAS,CAET,qBACF,CAEA,wGAKE,kBACF,CAEA,+DAGE,sBAAuB,CACvB,yBAA0B,CAC1B,eACF,CAEA,iFACE,WAAY,CACZ,iBAAkB,CAClB,iBACF,CAEA,wCACE,aACF,CAMA,8FACE,aACF,CAEA,sDACE,aACF,CAEA,+DAGE,uBAAwB,CACxB,wBACF,CAEA,gBACE,eACF,CAEA,wBACE,iBACF,CAEA,uBACE,iBAAkB,CAClB,UAAW,CACX,UAAW,CACX,kCAIF,CAEA,+BACE,QAAS,CACT,MACF,CAEA,kCACE,WAAY,CACZ,MACF,CChFA,kBACE,wBAAyB,CACzB,SAAU,CACV,SAAU,CACV,mBACF,CAEA,sBACE,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,OAAQ,CACR,UAAW,CACX,iBAAkB,CAClB,cAAe,CACf,cAAe,CACf,UAAY,CACZ,UAAY,CACZ,eAAiB,CACjB,iBAAkB,CAClB,kBACF,CACA,6BACE,eACF,CAEA,4BACE,SACF,CC5BA,iBACE,wBAAyB,CACzB,SAAU,CACV,WAAY,CACZ,SACF,CAEA,yCACE,wBAAyB,CACzB,UAAY,CACZ,cAAe,CACf,kBAAmB,CACnB,eAAgB,CAChB,KAAQ,CACR,WAAe,CACf,eACF,CCfA,cACE,iBAAkB,CAClB,wBAAyB,CACzB,eAAgB,CAChB,SAAU,CACV,QAAS,CACT,qBACF,CAEA,oBACE,UAAW,CACX,WAAY,CACZ,iBAAkB,CAClB,KAAM,CACN,MACF,CChBA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,SAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,WACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,wBACI,eAAgB,CAChB,WACJ,CAGA,uBACI,eAAgB,CAChB,gBACJ,CAEA,yBACI,gBAAgB,CAChB,cACJ,CAGA,qCACI,gBAAgB,CAChB,eAAgB,CAChB,SAAa,CACb,cACJ,CAGA,2BACI,gBAAgB,CAChB,cAAc,CACd,SAAa,CACb,cACJ,CAEA,6BACI,eAAgB,CAChB,WACJ,CCxGA,kEACE,iBAAkB,CAClB,UAAW,CACX,QAAS,CACT,uBACF,CAEA,6DACE,oBACF,CAEA,6DACE,oBACF,CAGA,qCACE,UAAW,CACX,iBAAkB,CAClB,aAAc,CACd,kBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,QAAS,CACT,iBAAkB,CAClB,UACF,CAGA,qCACE,iBAAkB,CAClB,UAAW,CACX,aAAc,CACd,kBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,QAAS,CACT,iBAAkB,CAClB,UACF,CAEA,qCACE,iBAAkB,CAClB,aAAc,CACd,kBAAmB,CACnB,WAAY,CACZ,iBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,iBAAkB,CAClB,UACF,CAEA,8CACE,QAAS,CACT,iCAAkC,CAClC,8BAA+B,CAC/B,6BAA8B,CAC9B,4BAA6B,CAC7B,4BAA6B,CAC7B,gCAAiC,CACjC,6BAA8B,CAC9B,4BAA6B,CAC7B,2BAA4B,CAC5B,wBACF,CAEA,+CACE,QAAS,CACT,qCAAsC,CACtC,kCAAmC,CACnC,iCAAkC,CAClC,gCAAiC,CACjC,6BAA8B,CAC9B,+BAAgC,CAChC,4BAA6B,CAC7B,2BAA4B,CAC5B,0BAA2B,CAC3B,uBACF,CAEA,YACE,sCAA2C,CAC3C,WAAY,CACZ,wBAAyB,CACzB,4CACF,CAEA,iBAEE,kBAAmB,CACnB,oBACF,CCrGA,aACE,iBAAkB,CAClB,SAAU,CACV,QAAS,CAET,qBACF,CAEA,0DAEE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,gBACF,CAEA,UACE,iBAAkB,CAClB,UAAW,CACX,QAAS,CACT,MAAO,CACP,SACF,CAEA,2BACE,iBAAkB,CAClB,qBAAsB,CACtB,+BACF,CAEA,sCACE,kBACF,CAEA,mBACE,cACF,CAEA,6DACE,kBACF,CACA,8CACE,qBACF,CACA,kEACE,cACF,CACA,kEACE,eACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CACA,8CACE,+BACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CACA,8CACE,gCACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CAGA,4BACE,+BACF,CACA,gDACE,kBACF,CACA,gDACE,mBACF,CAEA,kCACE,oBACF,CAGA,oCAEE,oBAAqB,CACrB,UACF,CACA,6CACE,eACF,CACA,8CACE,eACF,CACA,uDACE,eACF,CAEA,oEACE,iBACF,CACA,oEACE,kBACF,CAEA,aACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WAAY,CACZ,UACF,CCjLA,cAGE,eAGF,CAEA,uCAPE,iBAAkB,CAIlB,qBAWF,CARA,yBAEE,MAAO,CACP,KAAM,CACN,UAAW,CACX,aAAc,CAMd,+BAHF,CAMA,mCACE,cACF,CAEA,uBACE,yBACF,CAEA,oCACE,kBACF,CAEA,oCACE,oBAAqB,CACrB,WACF,CAEA,+CACE,SACF,CC1CA,eACE,iBAAkB,CAClB,eACF,CAEA,8BACE,KAAM,CACN,MAAO,CACP,UACF,CAEA,8BACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WACF,CAEA,yBACE,iBAAkB,CAClB,aAAc,CACd,WAAY,CACZ,eAAgB,CAChB,qBAAsB,CAEtB,kBACF,CAEA,qCACE,iBAAkB,CAClB,cAAe,CACf,eAAgB,CAChB,aAAc,CACd,cAAe,CACf,iBACF,CAEA,sCACE,iBAAkB,CAClB,qBACF,CAEA,0CACE,iBAAkB,CAClB,sBACF,CAEA,mCACE,oBACF,CAEA,mCACE,oBACF,CCrDA,UACE,iBAAkB,CAClB,aAAc,CACd,oBAAqB,CACrB,gBAAiB,CACjB,wBAAyB,CACzB,oBAAqB,CACrB,SAEF,CAEA,uBACE,oBAAqB,CACrB,wBAAyB,CAGzB,SACF,CAEA,2BACE,WACF,CAEA,iCACE,wBACF,CAEA,kBACE,iBAAkB,CAClB,kBAAmB,CACnB,iBACF,CAEA,oBACE,eACF,CAEA,kBACE,iBAAkB,CAClB,SAAU,CACV,gBAAiB,CACjB,kBAAmB,CACnB,iBACF,CAEA,oBACE,kBAAmB,CACnB,iBAAkB,CAClB,qBACF,CAEA,yBACE,WAAY,CACZ,qCAA0C,CAC1C,qBAAsB,CACtB,SAAU,CACV,QACF,CAEA,6BACE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,SAAU,CACV,QAAS,CACT,eACF,CAEA,wBACE,kBACF,CAEA,sCACE,iBAAkB,CAClB,oBACF,CAEA,2CACE,iBAAkB,CAClB,oBACF,CAEA,mBACE,SAAU,CACV,iBAAkB,CAClB,OAAQ,CACR,qBAAsB,CACtB,uBACF,CAEA,4BACE,kBAAmB,CACnB,qBAAsB,CACtB,WACF,CAEA,oCACE,iBAAkB,CAClB,kBAAmB,CACnB,UAAY,CACZ,WAAY,CACZ,iBAAkB,CAClB,kBAAmB,CACnB,WAAY,CACZ,iBAAkB,CAClB,cAAgB,CAChB,iBAAmB,CACnB,mBAAqB,CACrB,sBACF,CAEA,gDACE,iBAAkB,CAClB,KAAQ,CACR,UAAW,CACX,WAAY,CACZ,qBAAsB,CACtB,aAAgB,CAChB,cAAe,CAEf,wCAA0C,CAC1C,qCAAuC,CACvC,oCAAsC,CACtC,mCAAqC,CACrC,gCACF,CAEA,sBACE,WACF,CAEA,0BACE,UACF,CAEA,4DACE,eAAgB,CAChB,SAAU,CACV,4BAA8B,CAC9B,cAAe,CACf,eAAiB,CAEjB,mCAAqC,CACrC,gCAAkC,CAClC,+BAAiC,CACjC,8BAAgC,CAChC,2BACF,CAEA,4DACE,cACF,CAEA,wEACE,UACF,CAEA,2BACE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,KAAM,CACN,MAAS,CACT,WACF,CAEA,mCAOE,SAAU,CAEV,eACF,CAEA,uEAXE,iBAAkB,CAClB,UAAW,CACX,aAAc,CACd,aAAc,CACd,WAAY,CACZ,KAgBF,CAVA,oCAOE,UAAW,CAEX,eACF,CAEA,iGAEE,WACF,CAEA,sBACE,qBAAsB,CACtB,iBAAkB,CAClB,kBAAmB,CACnB,iBACF,CAEA,2BACE,SAAU,CACV,iBAAkB,CAClB,OAAQ,CACR,qBAAsB,CACtB,uBACF,CAEA,0BACE,iBAAkB,CAClB,SAAU,CACV,gBAAiB,CACjB,kBAAmB,CACnB,iBACF","file":"vis-timeline-graph2d.min.css","sourcesContent":["/* override some bootstrap styles screwing up the timelines css */\n\n.vis [class*=\"span\"] {\n  min-height: 0;\n  width: auto;\n}\n",".vis .overlay {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n\n  /* Must be displayed above for example selected Timeline items */\n  z-index: 10;\n}\n\n.vis-active {\n  box-shadow: 0 0 10px #86d5f8;\n}\n","div.vis-configuration {\n    position:relative;\n    display:block;\n    float:left;\n    font-size:12px;\n}\n\ndiv.vis-configuration-wrapper {\n    display:block;\n    width:700px;\n}\n\ndiv.vis-configuration-wrapper::after {\n  clear: both;\n  content: \"\";\n  display: block;\n}\n\ndiv.vis-configuration.vis-config-option-container{\n    display:block;\n    width:495px;\n    background-color: #ffffff;\n    border:2px solid #f7f8fa;\n    border-radius:4px;\n    margin-top:20px;\n    left:10px;\n    padding-left:5px;\n}\n\ndiv.vis-configuration.vis-config-button{\n    display:block;\n    width:495px;\n    height:25px;\n    vertical-align: middle;\n    line-height:25px;\n    background-color: #f7f8fa;\n    border:2px solid #ceced0;\n    border-radius:4px;\n    margin-top:20px;\n    left:10px;\n    padding-left:5px;\n    cursor: pointer;\n    margin-bottom:30px;\n}\n\ndiv.vis-configuration.vis-config-button.hover{\n    background-color: #4588e6;\n    border:2px solid #214373;\n    color:#ffffff;\n}\n\ndiv.vis-configuration.vis-config-item{\n    display:block;\n    float:left;\n    width:495px;\n    height:25px;\n    vertical-align: middle;\n    line-height:25px;\n}\n\n\ndiv.vis-configuration.vis-config-item.vis-config-s2{\n    left:10px;\n    background-color: #f7f8fa;\n    padding-left:5px;\n    border-radius:3px;\n}\ndiv.vis-configuration.vis-config-item.vis-config-s3{\n    left:20px;\n    background-color: #e4e9f0;\n    padding-left:5px;\n    border-radius:3px;\n}\ndiv.vis-configuration.vis-config-item.vis-config-s4{\n    left:30px;\n    background-color: #cfd8e6;\n    padding-left:5px;\n    border-radius:3px;\n}\n\ndiv.vis-configuration.vis-config-header{\n    font-size:18px;\n    font-weight: bold;\n}\n\ndiv.vis-configuration.vis-config-label{\n    width:120px;\n    height:25px;\n    line-height: 25px;\n}\n\ndiv.vis-configuration.vis-config-label.vis-config-s3{\n    width:110px;\n}\ndiv.vis-configuration.vis-config-label.vis-config-s4{\n    width:100px;\n}\n\ndiv.vis-configuration.vis-config-colorBlock{\n    top:1px;\n    width:30px;\n    height:19px;\n    border:1px solid #444444;\n    border-radius:2px;\n    padding:0px;\n    margin:0px;\n    cursor:pointer;\n}\n\ninput.vis-configuration.vis-config-checkbox {\n    left:-5px;\n}\n\n\ninput.vis-configuration.vis-config-rangeinput{\n    position:relative;\n    top:-5px;\n    width:60px;\n    /*height:13px;*/\n    padding:1px;\n    margin:0;\n    pointer-events:none;\n}\n\ninput.vis-configuration.vis-config-range{\n    /*removes default webkit styles*/\n    -webkit-appearance: none;\n\n    /*fix for FF unable to apply focus style bug */\n    border: 0px solid white;\n    background-color:rgba(0,0,0,0);\n\n    /*required for proper track sizing in FF*/\n    width: 300px;\n    height:20px;\n}\ninput.vis-configuration.vis-config-range::-webkit-slider-runnable-track {\n    width: 300px;\n    height: 5px;\n    background: #dedede; /* Old browsers */\n    background: -moz-linear-gradient(top,  #dedede 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#dedede), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top, #dedede 0%, #c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #dedede 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n\n    border: 1px solid #999999;\n    box-shadow: #aaaaaa 0px 0px 3px 0px;\n    border-radius: 3px;\n}\ninput.vis-configuration.vis-config-range::-webkit-slider-thumb {\n    -webkit-appearance: none;\n    border: 1px solid #14334b;\n    height: 17px;\n    width: 17px;\n    border-radius: 50%;\n    background: #3876c2; /* Old browsers */\n    background: -moz-linear-gradient(top,  #3876c2 0%, #385380 100%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#3876c2), color-stop(100%,#385380)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #3876c2 0%,#385380 100%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top,  #3876c2 0%,#385380 100%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #3876c2 0%,#385380 100%); /* IE10+ */\n    background: linear-gradient(to bottom,  #3876c2 0%,#385380 100%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#3876c2', endColorstr='#385380',GradientType=0 ); /* IE6-9 */\n    box-shadow: #111927 0px 0px 1px 0px;\n    margin-top: -7px;\n}\ninput.vis-configuration.vis-config-range:focus {\n    outline: none;\n}\ninput.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track {\n    background: #9d9d9d; /* Old browsers */\n    background: -moz-linear-gradient(top, #9d9d9d 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#9d9d9d), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #9d9d9d 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#9d9d9d', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n}\n\ninput.vis-configuration.vis-config-range::-moz-range-track {\n    width: 300px;\n    height: 10px;\n    background: #dedede; /* Old browsers */\n    background: -moz-linear-gradient(top,  #dedede 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#dedede), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top, #dedede 0%, #c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #dedede 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n\n    border: 1px solid #999999;\n    box-shadow: #aaaaaa 0px 0px 3px 0px;\n    border-radius: 3px;\n}\ninput.vis-configuration.vis-config-range::-moz-range-thumb {\n    border: none;\n    height: 16px;\n    width: 16px;\n\n    border-radius: 50%;\n    background:  #385380;\n}\n\n/*hide the outline behind the border*/\ninput.vis-configuration.vis-config-range:-moz-focusring{\n    outline: 1px solid white;\n    outline-offset: -1px;\n}\n\ninput.vis-configuration.vis-config-range::-ms-track {\n    width: 300px;\n    height: 5px;\n\n    /*remove bg colour from the track, we'll use ms-fill-lower and ms-fill-upper instead */\n    background: transparent;\n\n    /*leave room for the larger thumb to overflow with a transparent border */\n    border-color: transparent;\n    border-width: 6px 0;\n\n    /*remove default tick marks*/\n    color: transparent;\n}\ninput.vis-configuration.vis-config-range::-ms-fill-lower {\n    background: #777;\n    border-radius: 10px;\n}\ninput.vis-configuration.vis-config-range::-ms-fill-upper {\n    background: #ddd;\n    border-radius: 10px;\n}\ninput.vis-configuration.vis-config-range::-ms-thumb {\n    border: none;\n    height: 16px;\n    width: 16px;\n    border-radius: 50%;\n    background:  #385380;\n}\ninput.vis-configuration.vis-config-range:focus::-ms-fill-lower {\n    background: #888;\n}\ninput.vis-configuration.vis-config-range:focus::-ms-fill-upper {\n    background: #ccc;\n}\n\n.vis-configuration-popup {\n    position: absolute;\n    background: rgba(57, 76, 89, 0.85);\n    border: 2px solid #f2faff;\n    line-height:30px;\n    height:30px;\n    width:150px;\n    text-align:center;\n    color: #ffffff;\n    font-size:14px;\n    border-radius:4px;\n    -webkit-transition: opacity 0.3s ease-in-out;\n    -moz-transition: opacity 0.3s ease-in-out;\n    transition: opacity 0.3s ease-in-out;\n}\n.vis-configuration-popup:after, .vis-configuration-popup:before {\n    left: 100%;\n    top: 50%;\n    border: solid transparent;\n    content: \" \";\n    height: 0;\n    width: 0;\n    position: absolute;\n    pointer-events: none;\n}\n\n.vis-configuration-popup:after {\n    border-color: rgba(136, 183, 213, 0);\n    border-left-color: rgba(57, 76, 89, 0.85);\n    border-width: 8px;\n    margin-top: -8px;\n}\n.vis-configuration-popup:before {\n    border-color: rgba(194, 225, 245, 0);\n    border-left-color: #f2faff;\n    border-width: 12px;\n    margin-top: -12px;\n}","div.vis-tooltip {\n  position: absolute;\n  visibility: hidden;\n  padding: 5px;\n  white-space: nowrap;\n\n  font-family: verdana;\n  font-size:14px;\n  color:#000000;\n  background-color: #f5f4ed;\n\n  -moz-border-radius: 3px;\n  -webkit-border-radius: 3px;\n  border-radius: 3px;\n  border: 1px solid #808074;\n\n  box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.2);\n  pointer-events: none;\n\n  z-index: 5;\n}\n",".vis-panel {\n  position: absolute;\n\n  padding: 0;\n  margin: 0;\n\n  box-sizing: border-box;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-left,\n.vis-panel.vis-right,\n.vis-panel.vis-top,\n.vis-panel.vis-bottom {\n  border: 1px #bfbfbf;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-left,\n.vis-panel.vis-right {\n  border-top-style: solid;\n  border-bottom-style: solid;\n  overflow: hidden;\n}\n\n.vis-left.vis-panel.vis-vertical-scroll, .vis-right.vis-panel.vis-vertical-scroll {\n  height: 100%;\n  overflow-x: hidden;\n  overflow-y: scroll;\n} \n\n.vis-left.vis-panel.vis-vertical-scroll {\n  direction: rtl;\n}\n\n.vis-left.vis-panel.vis-vertical-scroll .vis-content {\n  direction: ltr;\n}\n\n.vis-right.vis-panel.vis-vertical-scroll {\n  direction: ltr;\n}\n\n.vis-right.vis-panel.vis-vertical-scroll .vis-content {\n  direction: rtl;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-top,\n.vis-panel.vis-bottom {\n  border-left-style: solid;\n  border-right-style: solid;\n}\n\n.vis-background {\n  overflow: hidden;\n}\n\n.vis-panel > .vis-content {\n  position: relative;\n}\n\n.vis-panel .vis-shadow {\n  position: absolute;\n  width: 100%;\n  height: 1px;\n  box-shadow: 0 0 10px rgba(0,0,0,0.8);\n  /* TODO: find a nice way to ensure vis-shadows are drawn on top of items\n  z-index: 1;\n  */\n}\n\n.vis-panel .vis-shadow.vis-top {\n  top: -1px;\n  left: 0;\n}\n\n.vis-panel .vis-shadow.vis-bottom {\n  bottom: -1px;\n  left: 0;\n}",".vis-current-time {\n  background-color: #FF7F6E;\n  width: 2px;\n  z-index: 1;\n  pointer-events: none;\n}\n\n.vis-rolling-mode-btn {\n  height: 40px;\n  width: 40px;\n  position: absolute;\n  top: 7px;\n  right: 20px;\n  border-radius: 50%;\n  font-size: 28px;\n  cursor: pointer;\n  opacity: 0.8;\n  color: white;\n  font-weight: bold;\n  text-align: center;\n  background: #3876c2;\n}\n.vis-rolling-mode-btn:before {\n  content: \"\\26F6\";\n}\n\n.vis-rolling-mode-btn:hover {\n  opacity: 1;\n}",".vis-custom-time {\n  background-color: #6E94FF;\n  width: 2px;\n  cursor: move;\n  z-index: 1;\n}\n\n.vis-custom-time > .vis-custom-time-marker {\n  background-color: inherit;\n  color: white;\n  font-size: 12px;\n  white-space: nowrap;\n  padding: 3px 5px;\n  top: 0px;\n  cursor: initial;\n  z-index: inherit;\n}","\n.vis-timeline {\n  position: relative;\n  border: 1px solid #bfbfbf;\n  overflow: hidden;\n  padding: 0;\n  margin: 0;\n  box-sizing: border-box;\n}\n\n.vis-loading-screen {\n  width: 100%;\n  height: 100%;\n  position: absolute;\n  top: 0;\n  left: 0;\n}",".vis-graph-group0 {\n    fill:#4f81bd;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #4f81bd;\n}\n\n.vis-graph-group1 {\n    fill:#f79646;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #f79646;\n}\n\n.vis-graph-group2 {\n    fill: #8c51cf;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #8c51cf;\n}\n\n.vis-graph-group3 {\n    fill: #75c841;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #75c841;\n}\n\n.vis-graph-group4 {\n    fill: #ff0100;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #ff0100;\n}\n\n.vis-graph-group5 {\n    fill: #37d8e6;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #37d8e6;\n}\n\n.vis-graph-group6 {\n    fill: #042662;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #042662;\n}\n\n.vis-graph-group7 {\n    fill:#00ff26;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #00ff26;\n}\n\n.vis-graph-group8 {\n    fill:#ff00ff;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #ff00ff;\n}\n\n.vis-graph-group9 {\n    fill: #8f3938;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #8f3938;\n}\n\n.vis-timeline .vis-fill {\n    fill-opacity:0.1;\n    stroke: none;\n}\n\n\n.vis-timeline .vis-bar {\n    fill-opacity:0.5;\n    stroke-width:1px;\n}\n\n.vis-timeline .vis-point {\n    stroke-width:2px;\n    fill-opacity:1.0;\n}\n\n\n.vis-timeline .vis-legend-background {\n    stroke-width:1px;\n    fill-opacity:0.9;\n    fill: #ffffff;\n    stroke: #c2c2c2;\n}\n\n\n.vis-timeline .vis-outline {\n    stroke-width:1px;\n    fill-opacity:1;\n    fill: #ffffff;\n    stroke: #e5e5e5;\n}\n\n.vis-timeline .vis-icon-fill {\n    fill-opacity:0.3;\n    stroke: none;\n}\n","\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal {\n  position: absolute;\n  width: 100%;\n  height: 0;\n  border-bottom: 1px solid;\n}\n\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor {\n  border-color: #e5e5e5;\n}\n\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major {\n  border-color: #bfbfbf;\n}\n\n\n.vis-data-axis .vis-y-axis.vis-major {\n  width: 100%;\n  position: absolute;\n  color: #4d4d4d;\n  white-space: nowrap;\n}\n\n.vis-data-axis .vis-y-axis.vis-major.vis-measure {\n  padding: 0;\n  margin: 0;\n  border: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n\n.vis-data-axis .vis-y-axis.vis-minor {\n  position: absolute;\n  width: 100%;\n  color: #bebebe;\n  white-space: nowrap;\n}\n\n.vis-data-axis .vis-y-axis.vis-minor.vis-measure {\n  padding: 0;\n  margin: 0;\n  border: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n.vis-data-axis .vis-y-axis.vis-title {\n  position: absolute;\n  color: #4d4d4d;\n  white-space: nowrap;\n  bottom: 20px;\n  text-align: center;\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-measure {\n  padding: 0;\n  margin: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-left {\n  bottom: 0;\n  -webkit-transform-origin: left top;\n  -moz-transform-origin: left top;\n  -ms-transform-origin: left top;\n  -o-transform-origin: left top;\n  transform-origin: left bottom;\n  -webkit-transform: rotate(-90deg);\n  -moz-transform: rotate(-90deg);\n  -ms-transform: rotate(-90deg);\n  -o-transform: rotate(-90deg);\n  transform: rotate(-90deg);\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-right {\n  bottom: 0;\n  -webkit-transform-origin: right bottom;\n  -moz-transform-origin: right bottom;\n  -ms-transform-origin: right bottom;\n  -o-transform-origin: right bottom;\n  transform-origin: right bottom;\n  -webkit-transform: rotate(90deg);\n  -moz-transform: rotate(90deg);\n  -ms-transform: rotate(90deg);\n  -o-transform: rotate(90deg);\n  transform: rotate(90deg);\n}\n\n.vis-legend {\n  background-color: rgba(247, 252, 255, 0.65);\n  padding: 5px;\n  border: 1px solid #b3b3b3;\n  box-shadow: 2px 2px 10px rgba(154, 154, 154, 0.55);\n}\n\n.vis-legend-text {\n  /*font-size: 10px;*/\n  white-space: nowrap;\n  display: inline-block\n}","\n.vis-itemset {\n  position: relative;\n  padding: 0;\n  margin: 0;\n\n  box-sizing: border-box;\n}\n\n.vis-itemset .vis-background,\n.vis-itemset .vis-foreground {\n  position: absolute;\n  width: 100%;\n  height: 100%;\n  overflow: visible;\n}\n\n.vis-axis {\n  position: absolute;\n  width: 100%;\n  height: 0;\n  left: 0;\n  z-index: 1;\n}\n\n.vis-foreground .vis-group {\n  position: relative;\n  box-sizing: border-box;\n  border-bottom: 1px solid #bfbfbf;\n}\n\n.vis-foreground .vis-group:last-child {\n  border-bottom: none;\n}\n\n.vis-nesting-group {\n  cursor: pointer;\n}\n\n.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1 {\n  background: #f5f5f5;\n}\n.vis-label.vis-nested-group.vis-group-level-0 {\n  background-color: #ffffff;\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner {\n  padding-left: 0;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner {\n  padding-right: 0;\n}\n.vis-label.vis-nested-group.vis-group-level-1 {\n  background-color: rgba(0, 0, 0, 0.05);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner {\n  padding-left: 15px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner {\n  padding-right: 15px;\n}\n.vis-label.vis-nested-group.vis-group-level-2 {\n  background-color: rgba(0, 0, 0, 0.1);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner {\n  padding-left: 30px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner {\n  padding-right: 30px;\n}\n.vis-label.vis-nested-group.vis-group-level-3 {\n  background-color: rgba(0, 0, 0, 0.15);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner {\n  padding-left: 45px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner {\n  padding-right: 45px;\n}\n.vis-label.vis-nested-group.vis-group-level-4 {\n  background-color: rgba(0, 0, 0, 0.2);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner {\n  padding-left: 60px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner {\n  padding-right: 60px;\n}\n.vis-label.vis-nested-group.vis-group-level-5 {\n  background-color: rgba(0, 0, 0, 0.25);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner {\n  padding-left: 75px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner {\n  padding-right: 75px;\n}\n.vis-label.vis-nested-group.vis-group-level-6 {\n  background-color: rgba(0, 0, 0, 0.3);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner {\n  padding-left: 90px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner {\n  padding-right: 90px;\n}\n.vis-label.vis-nested-group.vis-group-level-7 {\n  background-color: rgba(0, 0, 0, 0.35);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner {\n  padding-left: 105px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner {\n  padding-right: 105px;\n}\n.vis-label.vis-nested-group.vis-group-level-8 {\n  background-color: rgba(0, 0, 0, 0.4);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner {\n  padding-left: 120px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner {\n  padding-right: 120px;\n}\n.vis-label.vis-nested-group.vis-group-level-9 {\n  background-color: rgba(0, 0, 0, 0.45);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner {\n  padding-left: 135px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner {\n  padding-right: 135px;\n}\n/* default takes over beginning with level-10 (thats why we add .vis-nested-group\n  to the selectors above, to have higher specifity than these rules for the defaults) */\n.vis-label.vis-nested-group {\n  background-color: rgba(0, 0, 0, 0.5);\n}\n.vis-ltr .vis-label.vis-nested-group .vis-inner {\n  padding-left: 150px;\n}\n.vis-rtl .vis-label.vis-nested-group .vis-inner {\n  padding-right: 150px;\n}\n\n.vis-group-level-unknown-but-gte1 {\n  border: 1px solid red;\n}\n\n/* expanded/collapsed indicators */\n.vis-label.vis-nesting-group:before,\n.vis-label.vis-nesting-group:before {\n  display: inline-block;\n  width: 15px;\n}\n.vis-label.vis-nesting-group.expanded:before {\n  content: \"\\25BC\";\n}\n.vis-label.vis-nesting-group.collapsed:before {\n  content: \"\\25B6\";\n}\n.vis-rtl .vis-label.vis-nesting-group.collapsed:before {\n  content: \"\\25C0\";\n}\n/* compensate missing expanded/collapsed indicator, but only at levels > 0 */\n.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0) {\n  padding-left: 15px;\n}\n.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0) {\n  padding-right: 15px;\n}\n\n.vis-overlay {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n  z-index: 10;\n}","\n.vis-labelset {\n  position: relative;\n\n  overflow: hidden;\n\n  box-sizing: border-box;\n}\n\n.vis-labelset .vis-label {\n  position: relative;\n  left: 0;\n  top: 0;\n  width: 100%;\n  color: #4d4d4d;\n\n  box-sizing: border-box;\n}\n\n.vis-labelset .vis-label {\n  border-bottom: 1px solid #bfbfbf;\n}\n\n.vis-labelset .vis-label.draggable {\n  cursor: pointer;\n}\n\n.vis-group-is-dragging {\n  background: rgba(0, 0, 0, .1);\n}\n\n.vis-labelset .vis-label:last-child {\n  border-bottom: none;\n}\n\n.vis-labelset .vis-label .vis-inner {\n  display: inline-block;\n  padding: 5px;\n}\n\n.vis-labelset .vis-label .vis-inner.vis-hidden {\n  padding: 0;\n}\n",".vis-time-axis {\n  position: relative;\n  overflow: hidden;\n}\n\n.vis-time-axis.vis-foreground {\n  top: 0;\n  left: 0;\n  width: 100%;\n}\n\n.vis-time-axis.vis-background {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n}\n\n.vis-time-axis .vis-text {\n  position: absolute;\n  color: #4d4d4d;\n  padding: 3px;\n  overflow: hidden;\n  box-sizing: border-box;\n\n  white-space: nowrap;\n}\n\n.vis-time-axis .vis-text.vis-measure {\n  position: absolute;\n  padding-left: 0;\n  padding-right: 0;\n  margin-left: 0;\n  margin-right: 0;\n  visibility: hidden;\n}\n\n.vis-time-axis .vis-grid.vis-vertical {\n  position: absolute;\n  border-left: 1px solid;\n}\n\n.vis-time-axis .vis-grid.vis-vertical-rtl {\n  position: absolute;\n  border-right: 1px solid;\n}\n\n.vis-time-axis .vis-grid.vis-minor {\n  border-color: #e5e5e5;\n}\n\n.vis-time-axis .vis-grid.vis-major {\n  border-color: #bfbfbf;\n}\n","\n.vis-item {\n  position: absolute;\n  color: #1A1A1A;\n  border-color: #97B0F8;\n  border-width: 1px;\n  background-color: #D5DDF6;\n  display: inline-block;\n  z-index: 1;\n  /*overflow: hidden;*/\n}\n\n.vis-item.vis-selected {\n  border-color: #FFC200;\n  background-color: #FFF785;\n\n  /* z-index must be higher than the z-index of custom time bar and current time bar */\n  z-index: 2;\n}\n\n.vis-editable.vis-selected {\n  cursor: move;\n}\n\n.vis-item.vis-point.vis-selected {\n  background-color: #FFF785;\n}\n\n.vis-item.vis-box {\n  text-align: center;\n  border-style: solid;\n  border-radius: 2px;\n}\n\n.vis-item.vis-point {\n  background: none;\n}\n\n.vis-item.vis-dot {\n  position: absolute;\n  padding: 0;\n  border-width: 4px;\n  border-style: solid;\n  border-radius: 4px;\n}\n\n.vis-item.vis-range {\n  border-style: solid;\n  border-radius: 2px;\n  box-sizing: border-box;\n}\n\n.vis-item.vis-background {\n  border: none;\n  background-color: rgba(213, 221, 246, 0.4);\n  box-sizing: border-box;\n  padding: 0;\n  margin: 0;\n}\n\n.vis-item .vis-item-overflow {\n  position: relative;\n  width: 100%;\n  height: 100%;\n  padding: 0;\n  margin: 0;\n  overflow: hidden;\n}\n\n.vis-item-visible-frame {\n  white-space: nowrap;\n}\n\n.vis-item.vis-range .vis-item-content {\n  position: relative;\n  display: inline-block;\n}\n\n.vis-item.vis-background .vis-item-content {\n  position: absolute;\n  display: inline-block;\n}\n\n.vis-item.vis-line {\n  padding: 0;\n  position: absolute;\n  width: 0;\n  border-left-width: 1px;\n  border-left-style: solid;\n}\n\n.vis-item .vis-item-content {\n  white-space: nowrap;\n  box-sizing: border-box;\n  padding: 5px;\n}\n\n.vis-item .vis-onUpdateTime-tooltip {\n  position: absolute;\n  background: #4f81bd;\n  color: white;\n  width: 200px;\n  text-align: center;\n  white-space: nowrap;\n  padding: 5px;\n  border-radius: 1px;\n  transition: 0.4s;\n  -o-transition: 0.4s;\n  -moz-transition: 0.4s;\n  -webkit-transition: 0.4s;\n}\n\n.vis-item .vis-delete, .vis-item .vis-delete-rtl {\n  position: absolute;\n  top: 0px;\n  width: 24px;\n  height: 24px;\n  box-sizing: border-box;\n  padding: 0px 5px;\n  cursor: pointer;\n\n  -webkit-transition: background 0.2s linear;\n  -moz-transition: background 0.2s linear;\n  -ms-transition: background 0.2s linear;\n  -o-transition: background 0.2s linear;\n  transition: background 0.2s linear;\n}\n\n.vis-item .vis-delete {\n  right: -24px;\n}\n\n.vis-item .vis-delete-rtl {\n  left: -24px;\n}\n\n.vis-item .vis-delete:after, .vis-item .vis-delete-rtl:after {\n  content: \"\\00D7\"; /* MULTIPLICATION SIGN */\n  color: red;\n  font-family: arial, sans-serif;\n  font-size: 22px;\n  font-weight: bold;\n\n  -webkit-transition: color 0.2s linear;\n  -moz-transition: color 0.2s linear;\n  -ms-transition: color 0.2s linear;\n  -o-transition: color 0.2s linear;\n  transition: color 0.2s linear;\n}\n\n.vis-item .vis-delete:hover, .vis-item .vis-delete-rtl:hover {\n  background: red;\n}\n\n.vis-item .vis-delete:hover:after, .vis-item .vis-delete-rtl:hover:after {\n  color: white;\n}\n\n.vis-item .vis-drag-center {\n  position: absolute;\n  width: 100%;\n  height: 100%;\n  top: 0;\n  left: 0px;\n  cursor: move;\n}\n\n.vis-item.vis-range .vis-drag-left {\n  position: absolute;\n  width: 24px;\n  max-width: 20%;\n  min-width: 2px;\n  height: 100%;\n  top: 0;\n  left: -4px;\n\n  cursor: w-resize;\n}\n\n.vis-item.vis-range .vis-drag-right {\n  position: absolute;\n  width: 24px;\n  max-width: 20%;\n  min-width: 2px;\n  height: 100%;\n  top: 0;\n  right: -4px;\n\n  cursor: e-resize;\n}\n\n.vis-range.vis-item.vis-readonly .vis-drag-left,\n.vis-range.vis-item.vis-readonly .vis-drag-right {\n  cursor: auto;\n}\n\n.vis-item.vis-cluster {\n  vertical-align: center;\n  text-align: center;\n  border-style: solid;\n  border-radius: 2px;\n}\n\n.vis-item.vis-cluster-line {\n  padding: 0;\n  position: absolute;\n  width: 0;\n  border-left-width: 1px;\n  border-left-style: solid;\n}\n\n.vis-item.vis-cluster-dot {\n  position: absolute;\n  padding: 0;\n  border-width: 4px;\n  border-style: solid;\n  border-radius: 4px;\n}"]}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
new file mode 100644
index 0000000000000..df730409ee491
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
@@ -0,0 +1,60 @@
+/**
+ * vis-timeline and vis-graph2d
+ * https://visjs.github.io/vis-timeline/
+ *
+ * Create a fully customizable, interactive timeline with items and ranges.
+ *
+ * @version 7.3.4
+ * @date    2020-03-18T17:03:58.105Z
+ *
+ * @copyright (c) 2011-2017 Almende B.V, http://almende.com
+ * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+ *
+ * @license
+ * vis.js is dual licensed under both
+ *
+ *   1. The Apache 2.0 License
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   and
+ *
+ *   2. The MIT License
+ *      http://opensource.org/licenses/MIT
+ *
+ * vis.js may be distributed under either license.
+ */
+!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t=t||self).vis=t.vis||{})}(this,(function(t){"use strict";var e="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function i(){throw new Error("Dynamic requires are not currently supported by @rollup/plugin-commonjs")}function n(t,e){return t(e={exports:{}},e.exports),e.exports}function o(t){return t&&t.default||t}var s=n((function(t,e){t.exports=function(){var e,n;function o(){return e.apply(null,arguments)}function s(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function r(t){return null!=t&&"[object Object]"===Object.prototype.toString.call(t)}function a(t){return void 0===t}function l(t){return"number"==typeof t||"[object Number]"===Object.prototype.toString.call(t)}function h(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function d(t,e){var i,n=[];for(i=0;i<t.length;++i)n.push(e(t[i],i));return n}function u(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function c(t,e){for(var i in e)u(e,i)&&(t[i]=e[i]);return u(e,"toString")&&(t.toString=e.toString),u(e,"valueOf")&&(t.valueOf=e.valueOf),t}function p(t,e,i,n){return De(t,e,i,n,!0).utc()}function m(t){return null==t._pf&&(t._pf={empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}),t._pf}function f(t){if(null==t._isValid){var e=m(t),i=n.call(e.parsedDateParts,(function(t){return null!=t})),o=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.weekdayMismatch&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i);if(t._strict&&(o=o&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour),null!=Object.isFrozen&&Object.isFrozen(t))return o;t._isValid=o}return t._isValid}function g(t){var e=p(NaN);return null!=t?c(m(e),t):m(e).userInvalidated=!0,e}n=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,n=0;n<i;n++)if(n in e&&t.call(this,e[n],n,e))return!0;return!1};var v=o.momentProperties=[];function b(t,e){var i,n,o;if(a(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),a(e._i)||(t._i=e._i),a(e._f)||(t._f=e._f),a(e._l)||(t._l=e._l),a(e._strict)||(t._strict=e._strict),a(e._tzm)||(t._tzm=e._tzm),a(e._isUTC)||(t._isUTC=e._isUTC),a(e._offset)||(t._offset=e._offset),a(e._pf)||(t._pf=m(e)),a(e._locale)||(t._locale=e._locale),v.length>0)for(i=0;i<v.length;i++)a(o=e[n=v[i]])||(t[n]=o);return t}var y=!1;function x(t){b(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===y&&(y=!0,o.updateOffset(this),y=!1)}function w(t){return t instanceof x||null!=t&&null!=t._isAMomentObject}function _(t){return t<0?Math.ceil(t)||0:Math.floor(t)}function k(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=_(e)),i}function D(t,e,i){var n,o=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(n=0;n<o;n++)(i&&t[n]!==e[n]||!i&&k(t[n])!==k(e[n]))&&r++;return r+s}function I(t){!1===o.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function C(t,e){var i=!0;return c((function(){if(null!=o.deprecationHandler&&o.deprecationHandler(null,t),i){for(var n,s=[],r=0;r<arguments.length;r++){if(n="","object"==typeof arguments[r]){for(var a in n+="\n["+r+"] ",arguments[0])n+=a+": "+arguments[0][a]+", ";n=n.slice(0,-2)}else n=arguments[r];s.push(n)}I(t+"\nArguments: "+Array.prototype.slice.call(s).join("")+"\n"+(new Error).stack),i=!1}return e.apply(this,arguments)}),e)}var L,G={};function M(t,e){null!=o.deprecationHandler&&o.deprecationHandler(t,e),G[t]||(I(e),G[t]=!0)}function S(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function Q(t,e){var i,n=c({},t);for(i in e)u(e,i)&&(r(t[i])&&r(e[i])?(n[i]={},c(n[i],t[i]),c(n[i],e[i])):null!=e[i]?n[i]=e[i]:delete n[i]);for(i in t)u(t,i)&&!u(e,i)&&r(t[i])&&(n[i]=c({},n[i]));return n}function N(t){null!=t&&this.set(t)}o.suppressDeprecationWarnings=!1,o.deprecationHandler=null,L=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)u(t,e)&&i.push(e);return i};var F={};function Z(t,e){var i=t.toLowerCase();F[i]=F[i+"s"]=F[e]=t}function B(t){return"string"==typeof t?F[t]||F[t.toLowerCase()]:void 0}function O(t){var e,i,n={};for(i in t)u(t,i)&&(e=B(i))&&(n[e]=t[i]);return n}var W={};function A(t,e){W[t]=e}function X(t,e,i){var n=""+Math.abs(t),o=e-n.length;return(t>=0?i?"+":"":"-")+Math.pow(10,Math.max(0,o)).toString().substr(1)+n}var T=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Y=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,E={},R={};function U(t,e,i,n){var o=n;"string"==typeof n&&(o=function(){return this[n]()}),t&&(R[t]=o),e&&(R[e[0]]=function(){return X(o.apply(this,arguments),e[1],e[2])}),i&&(R[i]=function(){return this.localeData().ordinal(o.apply(this,arguments),t)})}function z(t,e){return t.isValid()?(e=V(e,t.localeData()),E[e]=E[e]||function(t){var e,i,n,o=t.match(T);for(e=0,i=o.length;e<i;e++)R[o[e]]?o[e]=R[o[e]]:o[e]=(n=o[e]).match(/\[[\s\S]/)?n.replace(/^\[|\]$/g,""):n.replace(/\\/g,"");return function(e){var n,s="";for(n=0;n<i;n++)s+=S(o[n])?o[n].call(e,t):o[n];return s}}(e),E[e](t)):t.localeData().invalidDate()}function V(t,e){var i=5;function n(t){return e.longDateFormat(t)||t}for(Y.lastIndex=0;i>=0&&Y.test(t);)t=t.replace(Y,n),Y.lastIndex=0,i-=1;return t}var j=/\d/,H=/\d\d/,P=/\d{3}/,J=/\d{4}/,K=/[+-]?\d{6}/,q=/\d\d?/,$=/\d\d\d\d?/,tt=/\d\d\d\d\d\d?/,et=/\d{1,3}/,it=/\d{1,4}/,nt=/[+-]?\d{1,6}/,ot=/\d+/,st=/[+-]?\d+/,rt=/Z|[+-]\d\d:?\d\d/gi,at=/Z|[+-]\d\d(?::?\d\d)?/gi,lt=/[0-9]{0,256}['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFF07\uFF10-\uFFEF]{1,256}|[\u0600-\u06FF\/]{1,256}(\s*?[\u0600-\u06FF]{1,256}){1,2}/i,ht={};function dt(t,e,i){ht[t]=S(e)?e:function(t,n){return t&&i?i:e}}function ut(t,e){return u(ht,t)?ht[t](e._strict,e._locale):new RegExp(ct(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,(function(t,e,i,n,o){return e||i||n||o}))))}function ct(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}var pt={};function mt(t,e){var i,n=e;for("string"==typeof t&&(t=[t]),l(e)&&(n=function(t,i){i[e]=k(t)}),i=0;i<t.length;i++)pt[t[i]]=n}function ft(t,e){mt(t,(function(t,i,n,o){n._w=n._w||{},e(t,n._w,n,o)}))}function gt(t,e,i){null!=e&&u(pt,t)&&pt[t](e,i._a,i,t)}function vt(t){return bt(t)?366:365}function bt(t){return t%4==0&&t%100!=0||t%400==0}U("Y",0,0,(function(){var t=this.year();return t<=9999?""+t:"+"+t})),U(0,["YY",2],0,(function(){return this.year()%100})),U(0,["YYYY",4],0,"year"),U(0,["YYYYY",5],0,"year"),U(0,["YYYYYY",6,!0],0,"year"),Z("year","y"),A("year",1),dt("Y",st),dt("YY",q,H),dt("YYYY",it,J),dt("YYYYY",nt,K),dt("YYYYYY",nt,K),mt(["YYYYY","YYYYYY"],0),mt("YYYY",(function(t,e){e[0]=2===t.length?o.parseTwoDigitYear(t):k(t)})),mt("YY",(function(t,e){e[0]=o.parseTwoDigitYear(t)})),mt("Y",(function(t,e){e[0]=parseInt(t,10)})),o.parseTwoDigitYear=function(t){return k(t)+(k(t)>68?1900:2e3)};var yt,xt=wt("FullYear",!0);function wt(t,e){return function(i){return null!=i?(kt(this,t,i),o.updateOffset(this,e),this):_t(this,t)}}function _t(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function kt(t,e,i){t.isValid()&&!isNaN(i)&&("FullYear"===e&&bt(t.year())&&1===t.month()&&29===t.date()?t._d["set"+(t._isUTC?"UTC":"")+e](i,t.month(),Dt(i,t.month())):t._d["set"+(t._isUTC?"UTC":"")+e](i))}function Dt(t,e){if(isNaN(t)||isNaN(e))return NaN;var i,n=(e%(i=12)+i)%i;return t+=(e-n)/12,1===n?bt(t)?29:28:31-n%7%2}yt=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},U("M",["MM",2],"Mo",(function(){return this.month()+1})),U("MMM",0,0,(function(t){return this.localeData().monthsShort(this,t)})),U("MMMM",0,0,(function(t){return this.localeData().months(this,t)})),Z("month","M"),A("month",8),dt("M",q),dt("MM",q,H),dt("MMM",(function(t,e){return e.monthsShortRegex(t)})),dt("MMMM",(function(t,e){return e.monthsRegex(t)})),mt(["M","MM"],(function(t,e){e[1]=k(t)-1})),mt(["MMM","MMMM"],(function(t,e,i,n){var o=i._locale.monthsParse(t,n,i._strict);null!=o?e[1]=o:m(i).invalidMonth=t}));var It=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,Ct="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),Lt="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_");function Gt(t,e,i){var n,o,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],n=0;n<12;++n)s=p([2e3,n]),this._shortMonthsParse[n]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[n]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?-1!==(o=yt.call(this._shortMonthsParse,r))?o:null:-1!==(o=yt.call(this._longMonthsParse,r))?o:null:"MMM"===e?-1!==(o=yt.call(this._shortMonthsParse,r))||-1!==(o=yt.call(this._longMonthsParse,r))?o:null:-1!==(o=yt.call(this._longMonthsParse,r))||-1!==(o=yt.call(this._shortMonthsParse,r))?o:null}function Mt(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=k(e);else if(!l(e=t.localeData().monthsParse(e)))return t;return i=Math.min(t.date(),Dt(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function St(t){return null!=t?(Mt(this,t),o.updateOffset(this,!0),this):_t(this,"Month")}var Qt=lt,Nt=lt;function Ft(){function t(t,e){return e.length-t.length}var e,i,n=[],o=[],s=[];for(e=0;e<12;e++)i=p([2e3,e]),n.push(this.monthsShort(i,"")),o.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(n.sort(t),o.sort(t),s.sort(t),e=0;e<12;e++)n[e]=ct(n[e]),o[e]=ct(o[e]);for(e=0;e<24;e++)s[e]=ct(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+o.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+n.join("|")+")","i")}function Zt(t,e,i,n,o,s,r){var a;return t<100&&t>=0?(a=new Date(t+400,e,i,n,o,s,r),isFinite(a.getFullYear())&&a.setFullYear(t)):a=new Date(t,e,i,n,o,s,r),a}function Bt(t){var e;if(t<100&&t>=0){var i=Array.prototype.slice.call(arguments);i[0]=t+400,e=new Date(Date.UTC.apply(null,i)),isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t)}else e=new Date(Date.UTC.apply(null,arguments));return e}function Ot(t,e,i){var n=7+e-i;return-(7+Bt(t,0,n).getUTCDay()-e)%7+n-1}function Wt(t,e,i,n,o){var s,r,a=1+7*(e-1)+(7+i-n)%7+Ot(t,n,o);return a<=0?r=vt(s=t-1)+a:a>vt(t)?(s=t+1,r=a-vt(t)):(s=t,r=a),{year:s,dayOfYear:r}}function At(t,e,i){var n,o,s=Ot(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return r<1?n=r+Xt(o=t.year()-1,e,i):r>Xt(t.year(),e,i)?(n=r-Xt(t.year(),e,i),o=t.year()+1):(o=t.year(),n=r),{week:n,year:o}}function Xt(t,e,i){var n=Ot(t,e,i),o=Ot(t+1,e,i);return(vt(t)-n+o)/7}function Tt(t,e){return t.slice(e,7).concat(t.slice(0,e))}U("w",["ww",2],"wo","week"),U("W",["WW",2],"Wo","isoWeek"),Z("week","w"),Z("isoWeek","W"),A("week",5),A("isoWeek",5),dt("w",q),dt("ww",q,H),dt("W",q),dt("WW",q,H),ft(["w","ww","W","WW"],(function(t,e,i,n){e[n.substr(0,1)]=k(t)})),U("d",0,"do","day"),U("dd",0,0,(function(t){return this.localeData().weekdaysMin(this,t)})),U("ddd",0,0,(function(t){return this.localeData().weekdaysShort(this,t)})),U("dddd",0,0,(function(t){return this.localeData().weekdays(this,t)})),U("e",0,0,"weekday"),U("E",0,0,"isoWeekday"),Z("day","d"),Z("weekday","e"),Z("isoWeekday","E"),A("day",11),A("weekday",11),A("isoWeekday",11),dt("d",q),dt("e",q),dt("E",q),dt("dd",(function(t,e){return e.weekdaysMinRegex(t)})),dt("ddd",(function(t,e){return e.weekdaysShortRegex(t)})),dt("dddd",(function(t,e){return e.weekdaysRegex(t)})),ft(["dd","ddd","dddd"],(function(t,e,i,n){var o=i._locale.weekdaysParse(t,n,i._strict);null!=o?e.d=o:m(i).invalidWeekday=t})),ft(["d","e","E"],(function(t,e,i,n){e[n]=k(t)}));var Yt="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Et="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Rt="Su_Mo_Tu_We_Th_Fr_Sa".split("_");function Ut(t,e,i){var n,o,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],n=0;n<7;++n)s=p([2e3,1]).day(n),this._minWeekdaysParse[n]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[n]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[n]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?-1!==(o=yt.call(this._weekdaysParse,r))?o:null:"ddd"===e?-1!==(o=yt.call(this._shortWeekdaysParse,r))?o:null:-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:"dddd"===e?-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._shortWeekdaysParse,r))||-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:"ddd"===e?-1!==(o=yt.call(this._shortWeekdaysParse,r))||-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:-1!==(o=yt.call(this._minWeekdaysParse,r))||-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._shortWeekdaysParse,r))?o:null}var zt=lt,Vt=lt,jt=lt;function Ht(){function t(t,e){return e.length-t.length}var e,i,n,o,s,r=[],a=[],l=[],h=[];for(e=0;e<7;e++)i=p([2e3,1]).day(e),n=this.weekdaysMin(i,""),o=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(n),a.push(o),l.push(s),h.push(n),h.push(o),h.push(s);for(r.sort(t),a.sort(t),l.sort(t),h.sort(t),e=0;e<7;e++)a[e]=ct(a[e]),l[e]=ct(l[e]),h[e]=ct(h[e]);this._weekdaysRegex=new RegExp("^("+h.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function Pt(){return this.hours()%12||12}function Jt(t,e){U(t,0,0,(function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)}))}function Kt(t,e){return e._meridiemParse}U("H",["HH",2],0,"hour"),U("h",["hh",2],0,Pt),U("k",["kk",2],0,(function(){return this.hours()||24})),U("hmm",0,0,(function(){return""+Pt.apply(this)+X(this.minutes(),2)})),U("hmmss",0,0,(function(){return""+Pt.apply(this)+X(this.minutes(),2)+X(this.seconds(),2)})),U("Hmm",0,0,(function(){return""+this.hours()+X(this.minutes(),2)})),U("Hmmss",0,0,(function(){return""+this.hours()+X(this.minutes(),2)+X(this.seconds(),2)})),Jt("a",!0),Jt("A",!1),Z("hour","h"),A("hour",13),dt("a",Kt),dt("A",Kt),dt("H",q),dt("h",q),dt("k",q),dt("HH",q,H),dt("hh",q,H),dt("kk",q,H),dt("hmm",$),dt("hmmss",tt),dt("Hmm",$),dt("Hmmss",tt),mt(["H","HH"],3),mt(["k","kk"],(function(t,e,i){var n=k(t);e[3]=24===n?0:n})),mt(["a","A"],(function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t})),mt(["h","hh"],(function(t,e,i){e[3]=k(t),m(i).bigHour=!0})),mt("hmm",(function(t,e,i){var n=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n)),m(i).bigHour=!0})),mt("hmmss",(function(t,e,i){var n=t.length-4,o=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n,2)),e[5]=k(t.substr(o)),m(i).bigHour=!0})),mt("Hmm",(function(t,e,i){var n=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n))})),mt("Hmmss",(function(t,e,i){var n=t.length-4,o=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n,2)),e[5]=k(t.substr(o))}));var qt,$t=wt("Hours",!0),te={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:Ct,monthsShort:Lt,week:{dow:0,doy:6},weekdays:Yt,weekdaysMin:Rt,weekdaysShort:Et,meridiemParse:/[ap]\.?m?\.?/i},ee={},ie={};function ne(t){return t?t.toLowerCase().replace("_","-"):t}function oe(e){var n=null;if(!ee[e]&&t&&t.exports)try{n=qt._abbr,i(),se(n)}catch(t){}return ee[e]}function se(t,e){var i;return t&&((i=a(e)?ae(t):re(t,e))?qt=i:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+t+" not found. Did you forget to load it?")),qt._abbr}function re(t,e){if(null!==e){var i,n=te;if(e.abbr=t,null!=ee[t])M("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),n=ee[t]._config;else if(null!=e.parentLocale)if(null!=ee[e.parentLocale])n=ee[e.parentLocale]._config;else{if(null==(i=oe(e.parentLocale)))return ie[e.parentLocale]||(ie[e.parentLocale]=[]),ie[e.parentLocale].push({name:t,config:e}),null;n=i._config}return ee[t]=new N(Q(n,e)),ie[t]&&ie[t].forEach((function(t){re(t.name,t.config)})),se(t),ee[t]}return delete ee[t],null}function ae(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return qt;if(!s(t)){if(e=oe(t))return e;t=[t]}return function(t){for(var e,i,n,o,s=0;s<t.length;){for(e=(o=ne(t[s]).split("-")).length,i=(i=ne(t[s+1]))?i.split("-"):null;e>0;){if(n=oe(o.slice(0,e).join("-")))return n;if(i&&i.length>=e&&D(o,i,!0)>=e-1)break;e--}s++}return qt}(t)}function le(t){var e,i=t._a;return i&&-2===m(t).overflow&&(e=i[1]<0||i[1]>11?1:i[2]<1||i[2]>Dt(i[0],i[1])?2:i[3]<0||i[3]>24||24===i[3]&&(0!==i[4]||0!==i[5]||0!==i[6])?3:i[4]<0||i[4]>59?4:i[5]<0||i[5]>59?5:i[6]<0||i[6]>999?6:-1,m(t)._overflowDayOfYear&&(e<0||e>2)&&(e=2),m(t)._overflowWeeks&&-1===e&&(e=7),m(t)._overflowWeekday&&-1===e&&(e=8),m(t).overflow=e),t}function he(t,e,i){return null!=t?t:null!=e?e:i}function de(t){var e,i,n,s,r,a=[];if(!t._d){for(n=function(t){var e=new Date(o.now());return t._useUTC?[e.getUTCFullYear(),e.getUTCMonth(),e.getUTCDate()]:[e.getFullYear(),e.getMonth(),e.getDate()]}(t),t._w&&null==t._a[2]&&null==t._a[1]&&function(t){var e,i,n,o,s,r,a,l;if(null!=(e=t._w).GG||null!=e.W||null!=e.E)s=1,r=4,i=he(e.GG,t._a[0],At(Ie(),1,4).year),n=he(e.W,1),((o=he(e.E,1))<1||o>7)&&(l=!0);else{s=t._locale._week.dow,r=t._locale._week.doy;var h=At(Ie(),s,r);i=he(e.gg,t._a[0],h.year),n=he(e.w,h.week),null!=e.d?((o=e.d)<0||o>6)&&(l=!0):null!=e.e?(o=e.e+s,(e.e<0||e.e>6)&&(l=!0)):o=s}n<1||n>Xt(i,s,r)?m(t)._overflowWeeks=!0:null!=l?m(t)._overflowWeekday=!0:(a=Wt(i,n,o,s,r),t._a[0]=a.year,t._dayOfYear=a.dayOfYear)}(t),null!=t._dayOfYear&&(r=he(t._a[0],n[0]),(t._dayOfYear>vt(r)||0===t._dayOfYear)&&(m(t)._overflowDayOfYear=!0),i=Bt(r,0,t._dayOfYear),t._a[1]=i.getUTCMonth(),t._a[2]=i.getUTCDate()),e=0;e<3&&null==t._a[e];++e)t._a[e]=a[e]=n[e];for(;e<7;e++)t._a[e]=a[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[3]&&0===t._a[4]&&0===t._a[5]&&0===t._a[6]&&(t._nextDay=!0,t._a[3]=0),t._d=(t._useUTC?Bt:Zt).apply(null,a),s=t._useUTC?t._d.getUTCDay():t._d.getDay(),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[3]=24),t._w&&void 0!==t._w.d&&t._w.d!==s&&(m(t).weekdayMismatch=!0)}}var ue=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,ce=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,pe=/Z|[+-]\d\d(?::?\d\d)?/,me=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],fe=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],ge=/^\/?Date\((\-?\d+)/i;function ve(t){var e,i,n,o,s,r,a=t._i,l=ue.exec(a)||ce.exec(a);if(l){for(m(t).iso=!0,e=0,i=me.length;e<i;e++)if(me[e][1].exec(l[1])){o=me[e][0],n=!1!==me[e][2];break}if(null==o)return void(t._isValid=!1);if(l[3]){for(e=0,i=fe.length;e<i;e++)if(fe[e][1].exec(l[3])){s=(l[2]||" ")+fe[e][0];break}if(null==s)return void(t._isValid=!1)}if(!n&&null!=s)return void(t._isValid=!1);if(l[4]){if(!pe.exec(l[4]))return void(t._isValid=!1);r="Z"}t._f=o+(s||"")+(r||""),_e(t)}else t._isValid=!1}var be=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/;function ye(t){var e=parseInt(t,10);return e<=49?2e3+e:e<=999?1900+e:e}var xe={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};function we(t){var e,i,n,o,s,r,a,l=be.exec(t._i.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").replace(/^\s\s*/,"").replace(/\s\s*$/,""));if(l){var h=(e=l[4],i=l[3],n=l[2],o=l[5],s=l[6],r=l[7],a=[ye(e),Lt.indexOf(i),parseInt(n,10),parseInt(o,10),parseInt(s,10)],r&&a.push(parseInt(r,10)),a);if(!function(t,e,i){return!t||Et.indexOf(t)===new Date(e[0],e[1],e[2]).getDay()||(m(i).weekdayMismatch=!0,i._isValid=!1,!1)}(l[1],h,t))return;t._a=h,t._tzm=function(t,e,i){if(t)return xe[t];if(e)return 0;var n=parseInt(i,10),o=n%100;return(n-o)/100*60+o}(l[8],l[9],l[10]),t._d=Bt.apply(null,t._a),t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),m(t).rfc2822=!0}else t._isValid=!1}function _e(t){if(t._f!==o.ISO_8601)if(t._f!==o.RFC_2822){t._a=[],m(t).empty=!0;var e,i,n,s,r,a=""+t._i,l=a.length,h=0;for(n=V(t._f,t._locale).match(T)||[],e=0;e<n.length;e++)s=n[e],(i=(a.match(ut(s,t))||[])[0])&&((r=a.substr(0,a.indexOf(i))).length>0&&m(t).unusedInput.push(r),a=a.slice(a.indexOf(i)+i.length),h+=i.length),R[s]?(i?m(t).empty=!1:m(t).unusedTokens.push(s),gt(s,i,t)):t._strict&&!i&&m(t).unusedTokens.push(s);m(t).charsLeftOver=l-h,a.length>0&&m(t).unusedInput.push(a),t._a[3]<=12&&!0===m(t).bigHour&&t._a[3]>0&&(m(t).bigHour=void 0),m(t).parsedDateParts=t._a.slice(0),m(t).meridiem=t._meridiem,t._a[3]=function(t,e,i){var n;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?((n=t.isPM(i))&&e<12&&(e+=12),n||12!==e||(e=0),e):e}(t._locale,t._a[3],t._meridiem),de(t),le(t)}else we(t);else ve(t)}function ke(t){var e=t._i,i=t._f;return t._locale=t._locale||ae(t._l),null===e||void 0===i&&""===e?g({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),w(e)?new x(le(e)):(h(e)?t._d=e:s(i)?function(t){var e,i,n,o,s;if(0===t._f.length)return m(t).invalidFormat=!0,void(t._d=new Date(NaN));for(o=0;o<t._f.length;o++)s=0,e=b({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[o],_e(e),f(e)&&(s+=m(e).charsLeftOver,s+=10*m(e).unusedTokens.length,m(e).score=s,(null==n||s<n)&&(n=s,i=e));c(t,i||e)}(t):i?_e(t):function(t){var e=t._i;a(e)?t._d=new Date(o.now()):h(e)?t._d=new Date(e.valueOf()):"string"==typeof e?function(t){var e=ge.exec(t._i);null===e?(ve(t),!1===t._isValid&&(delete t._isValid,we(t),!1===t._isValid&&(delete t._isValid,o.createFromInputFallback(t)))):t._d=new Date(+e[1])}(t):s(e)?(t._a=d(e.slice(0),(function(t){return parseInt(t,10)})),de(t)):r(e)?function(t){if(!t._d){var e=O(t._i);t._a=d([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],(function(t){return t&&parseInt(t,10)})),de(t)}}(t):l(e)?t._d=new Date(e):o.createFromInputFallback(t)}(t),f(t)||(t._d=null),t))}function De(t,e,i,n,o){var a,l={};return!0!==i&&!1!==i||(n=i,i=void 0),(r(t)&&function(t){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(t).length;var e;for(e in t)if(t.hasOwnProperty(e))return!1;return!0}(t)||s(t)&&0===t.length)&&(t=void 0),l._isAMomentObject=!0,l._useUTC=l._isUTC=o,l._l=i,l._i=t,l._f=e,l._strict=n,(a=new x(le(ke(l))))._nextDay&&(a.add(1,"d"),a._nextDay=void 0),a}function Ie(t,e,i,n){return De(t,e,i,n,!1)}o.createFromInputFallback=C("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",(function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))})),o.ISO_8601=function(){},o.RFC_2822=function(){};var Ce=C("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",(function(){var t=Ie.apply(null,arguments);return this.isValid()&&t.isValid()?t<this?this:t:g()})),Le=C("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",(function(){var t=Ie.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:g()}));function Ge(t,e){var i,n;if(1===e.length&&s(e[0])&&(e=e[0]),!e.length)return Ie();for(i=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](i)||(i=e[n]);return i}var Me=["year","quarter","month","week","day","hour","minute","second","millisecond"];function Se(t){var e=O(t),i=e.year||0,n=e.quarter||0,o=e.month||0,s=e.week||e.isoWeek||0,r=e.day||0,a=e.hour||0,l=e.minute||0,h=e.second||0,d=e.millisecond||0;this._isValid=function(t){for(var e in t)if(-1===yt.call(Me,e)||null!=t[e]&&isNaN(t[e]))return!1;for(var i=!1,n=0;n<Me.length;++n)if(t[Me[n]]){if(i)return!1;parseFloat(t[Me[n]])!==k(t[Me[n]])&&(i=!0)}return!0}(e),this._milliseconds=+d+1e3*h+6e4*l+1e3*a*60*60,this._days=+r+7*s,this._months=+o+3*n+12*i,this._data={},this._locale=ae(),this._bubble()}function Qe(t){return t instanceof Se}function Ne(t){return t<0?-1*Math.round(-1*t):Math.round(t)}function Fe(t,e){U(t,0,0,(function(){var t=this.utcOffset(),i="+";return t<0&&(t=-t,i="-"),i+X(~~(t/60),2)+e+X(~~t%60,2)}))}Fe("Z",":"),Fe("ZZ",""),dt("Z",at),dt("ZZ",at),mt(["Z","ZZ"],(function(t,e,i){i._useUTC=!0,i._tzm=Be(at,t)}));var Ze=/([\+\-]|\d\d)/gi;function Be(t,e){var i=(e||"").match(t);if(null===i)return null;var n=((i[i.length-1]||[])+"").match(Ze)||["-",0,0],o=60*n[1]+k(n[2]);return 0===o?0:"+"===n[0]?o:-o}function Oe(t,e){var i,n;return e._isUTC?(i=e.clone(),n=(w(t)||h(t)?t.valueOf():Ie(t).valueOf())-i.valueOf(),i._d.setTime(i._d.valueOf()+n),o.updateOffset(i,!1),i):Ie(t).local()}function We(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function Ae(){return!!this.isValid()&&this._isUTC&&0===this._offset}o.updateOffset=function(){};var Xe=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,Te=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;function Ye(t,e){var i,n,o,s,r,a,h=t,d=null;return Qe(t)?h={ms:t._milliseconds,d:t._days,M:t._months}:l(t)?(h={},e?h[e]=t:h.milliseconds=t):(d=Xe.exec(t))?(i="-"===d[1]?-1:1,h={y:0,d:k(d[2])*i,h:k(d[3])*i,m:k(d[4])*i,s:k(d[5])*i,ms:k(Ne(1e3*d[6]))*i}):(d=Te.exec(t))?(i="-"===d[1]?-1:1,h={y:Ee(d[2],i),M:Ee(d[3],i),w:Ee(d[4],i),d:Ee(d[5],i),h:Ee(d[6],i),m:Ee(d[7],i),s:Ee(d[8],i)}):null==h?h={}:"object"==typeof h&&("from"in h||"to"in h)&&(s=Ie(h.from),r=Ie(h.to),o=s.isValid()&&r.isValid()?(r=Oe(r,s),s.isBefore(r)?a=Re(s,r):((a=Re(r,s)).milliseconds=-a.milliseconds,a.months=-a.months),a):{milliseconds:0,months:0},(h={}).ms=o.milliseconds,h.M=o.months),n=new Se(h),Qe(t)&&u(t,"_locale")&&(n._locale=t._locale),n}function Ee(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function Re(t,e){var i={};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function Ue(t,e){return function(i,n){var o;return null===n||isNaN(+n)||(M(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),o=i,i=n,n=o),ze(this,Ye(i="string"==typeof i?+i:i,n),t),this}}function ze(t,e,i,n){var s=e._milliseconds,r=Ne(e._days),a=Ne(e._months);t.isValid()&&(n=null==n||n,a&&Mt(t,_t(t,"Month")+a*i),r&&kt(t,"Date",_t(t,"Date")+r*i),s&&t._d.setTime(t._d.valueOf()+s*i),n&&o.updateOffset(t,r||a))}Ye.fn=Se.prototype,Ye.invalid=function(){return Ye(NaN)};var Ve=Ue(1,"add"),je=Ue(-1,"subtract");function He(t,e){var i=12*(e.year()-t.year())+(e.month()-t.month()),n=t.clone().add(i,"months");return-(i+(e-n<0?(e-n)/(n-t.clone().add(i-1,"months")):(e-n)/(t.clone().add(i+1,"months")-n)))||0}function Pe(t){var e;return void 0===t?this._locale._abbr:(null!=(e=ae(t))&&(this._locale=e),this)}o.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",o.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var Je=C("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",(function(t){return void 0===t?this.localeData():this.locale(t)}));function Ke(){return this._locale}function qe(t,e){return(t%e+e)%e}function $e(t,e,i){return t<100&&t>=0?new Date(t+400,e,i)-126227808e5:new Date(t,e,i).valueOf()}function ti(t,e,i){return t<100&&t>=0?Date.UTC(t+400,e,i)-126227808e5:Date.UTC(t,e,i)}function ei(t,e){U(0,[t,t.length],0,e)}function ii(t,e,i,n,o){var s;return null==t?At(this,n,o).year:(e>(s=Xt(t,n,o))&&(e=s),ni.call(this,t,e,i,n,o))}function ni(t,e,i,n,o){var s=Wt(t,e,i,n,o),r=Bt(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}U(0,["gg",2],0,(function(){return this.weekYear()%100})),U(0,["GG",2],0,(function(){return this.isoWeekYear()%100})),ei("gggg","weekYear"),ei("ggggg","weekYear"),ei("GGGG","isoWeekYear"),ei("GGGGG","isoWeekYear"),Z("weekYear","gg"),Z("isoWeekYear","GG"),A("weekYear",1),A("isoWeekYear",1),dt("G",st),dt("g",st),dt("GG",q,H),dt("gg",q,H),dt("GGGG",it,J),dt("gggg",it,J),dt("GGGGG",nt,K),dt("ggggg",nt,K),ft(["gggg","ggggg","GGGG","GGGGG"],(function(t,e,i,n){e[n.substr(0,2)]=k(t)})),ft(["gg","GG"],(function(t,e,i,n){e[n]=o.parseTwoDigitYear(t)})),U("Q",0,"Qo","quarter"),Z("quarter","Q"),A("quarter",7),dt("Q",j),mt("Q",(function(t,e){e[1]=3*(k(t)-1)})),U("D",["DD",2],"Do","date"),Z("date","D"),A("date",9),dt("D",q),dt("DD",q,H),dt("Do",(function(t,e){return t?e._dayOfMonthOrdinalParse||e._ordinalParse:e._dayOfMonthOrdinalParseLenient})),mt(["D","DD"],2),mt("Do",(function(t,e){e[2]=k(t.match(q)[0])}));var oi=wt("Date",!0);U("DDD",["DDDD",3],"DDDo","dayOfYear"),Z("dayOfYear","DDD"),A("dayOfYear",4),dt("DDD",et),dt("DDDD",P),mt(["DDD","DDDD"],(function(t,e,i){i._dayOfYear=k(t)})),U("m",["mm",2],0,"minute"),Z("minute","m"),A("minute",14),dt("m",q),dt("mm",q,H),mt(["m","mm"],4);var si=wt("Minutes",!1);U("s",["ss",2],0,"second"),Z("second","s"),A("second",15),dt("s",q),dt("ss",q,H),mt(["s","ss"],5);var ri,ai=wt("Seconds",!1);for(U("S",0,0,(function(){return~~(this.millisecond()/100)})),U(0,["SS",2],0,(function(){return~~(this.millisecond()/10)})),U(0,["SSS",3],0,"millisecond"),U(0,["SSSS",4],0,(function(){return 10*this.millisecond()})),U(0,["SSSSS",5],0,(function(){return 100*this.millisecond()})),U(0,["SSSSSS",6],0,(function(){return 1e3*this.millisecond()})),U(0,["SSSSSSS",7],0,(function(){return 1e4*this.millisecond()})),U(0,["SSSSSSSS",8],0,(function(){return 1e5*this.millisecond()})),U(0,["SSSSSSSSS",9],0,(function(){return 1e6*this.millisecond()})),Z("millisecond","ms"),A("millisecond",16),dt("S",et,j),dt("SS",et,H),dt("SSS",et,P),ri="SSSS";ri.length<=9;ri+="S")dt(ri,ot);function li(t,e){e[6]=k(1e3*("0."+t))}for(ri="S";ri.length<=9;ri+="S")mt(ri,li);var hi=wt("Milliseconds",!1);U("z",0,0,"zoneAbbr"),U("zz",0,0,"zoneName");var di=x.prototype;function ui(t){return t}di.add=Ve,di.calendar=function(t,e){var i=t||Ie(),n=Oe(i,this).startOf("day"),s=o.calendarFormat(this,n)||"sameElse",r=e&&(S(e[s])?e[s].call(this,i):e[s]);return this.format(r||this.localeData().calendar(s,this,Ie(i)))},di.clone=function(){return new x(this)},di.diff=function(t,e,i){var n,o,s;if(!this.isValid())return NaN;if(!(n=Oe(t,this)).isValid())return NaN;switch(o=6e4*(n.utcOffset()-this.utcOffset()),e=B(e)){case"year":s=He(this,n)/12;break;case"month":s=He(this,n);break;case"quarter":s=He(this,n)/3;break;case"second":s=(this-n)/1e3;break;case"minute":s=(this-n)/6e4;break;case"hour":s=(this-n)/36e5;break;case"day":s=(this-n-o)/864e5;break;case"week":s=(this-n-o)/6048e5;break;default:s=this-n}return i?s:_(s)},di.endOf=function(t){var e;if(void 0===(t=B(t))||"millisecond"===t||!this.isValid())return this;var i=this._isUTC?ti:$e;switch(t){case"year":e=i(this.year()+1,0,1)-1;break;case"quarter":e=i(this.year(),this.month()-this.month()%3+3,1)-1;break;case"month":e=i(this.year(),this.month()+1,1)-1;break;case"week":e=i(this.year(),this.month(),this.date()-this.weekday()+7)-1;break;case"isoWeek":e=i(this.year(),this.month(),this.date()-(this.isoWeekday()-1)+7)-1;break;case"day":case"date":e=i(this.year(),this.month(),this.date()+1)-1;break;case"hour":e=this._d.valueOf(),e+=36e5-qe(e+(this._isUTC?0:6e4*this.utcOffset()),36e5)-1;break;case"minute":e=this._d.valueOf(),e+=6e4-qe(e,6e4)-1;break;case"second":e=this._d.valueOf(),e+=1e3-qe(e,1e3)-1}return this._d.setTime(e),o.updateOffset(this,!0),this},di.format=function(t){t||(t=this.isUtc()?o.defaultFormatUtc:o.defaultFormat);var e=z(this,t);return this.localeData().postformat(e)},di.from=function(t,e){return this.isValid()&&(w(t)&&t.isValid()||Ie(t).isValid())?Ye({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()},di.fromNow=function(t){return this.from(Ie(),t)},di.to=function(t,e){return this.isValid()&&(w(t)&&t.isValid()||Ie(t).isValid())?Ye({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()},di.toNow=function(t){return this.to(Ie(),t)},di.get=function(t){return S(this[t=B(t)])?this[t]():this},di.invalidAt=function(){return m(this).overflow},di.isAfter=function(t,e){var i=w(t)?t:Ie(t);return!(!this.isValid()||!i.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf())},di.isBefore=function(t,e){var i=w(t)?t:Ie(t);return!(!this.isValid()||!i.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf())},di.isBetween=function(t,e,i,n){var o=w(t)?t:Ie(t),s=w(e)?e:Ie(e);return!!(this.isValid()&&o.isValid()&&s.isValid())&&("("===(n=n||"()")[0]?this.isAfter(o,i):!this.isBefore(o,i))&&(")"===n[1]?this.isBefore(s,i):!this.isAfter(s,i))},di.isSame=function(t,e){var i,n=w(t)?t:Ie(t);return!(!this.isValid()||!n.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()===n.valueOf():(i=n.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf()))},di.isSameOrAfter=function(t,e){return this.isSame(t,e)||this.isAfter(t,e)},di.isSameOrBefore=function(t,e){return this.isSame(t,e)||this.isBefore(t,e)},di.isValid=function(){return f(this)},di.lang=Je,di.locale=Pe,di.localeData=Ke,di.max=Le,di.min=Ce,di.parsingFlags=function(){return c({},m(this))},di.set=function(t,e){if("object"==typeof t)for(var i=function(t){var e=[];for(var i in t)e.push({unit:i,priority:W[i]});return e.sort((function(t,e){return t.priority-e.priority})),e}(t=O(t)),n=0;n<i.length;n++)this[i[n].unit](t[i[n].unit]);else if(S(this[t=B(t)]))return this[t](e);return this},di.startOf=function(t){var e;if(void 0===(t=B(t))||"millisecond"===t||!this.isValid())return this;var i=this._isUTC?ti:$e;switch(t){case"year":e=i(this.year(),0,1);break;case"quarter":e=i(this.year(),this.month()-this.month()%3,1);break;case"month":e=i(this.year(),this.month(),1);break;case"week":e=i(this.year(),this.month(),this.date()-this.weekday());break;case"isoWeek":e=i(this.year(),this.month(),this.date()-(this.isoWeekday()-1));break;case"day":case"date":e=i(this.year(),this.month(),this.date());break;case"hour":e=this._d.valueOf(),e-=qe(e+(this._isUTC?0:6e4*this.utcOffset()),36e5);break;case"minute":e=this._d.valueOf(),e-=qe(e,6e4);break;case"second":e=this._d.valueOf(),e-=qe(e,1e3)}return this._d.setTime(e),o.updateOffset(this,!0),this},di.subtract=je,di.toArray=function(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]},di.toObject=function(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}},di.toDate=function(){return new Date(this.valueOf())},di.toISOString=function(t){if(!this.isValid())return null;var e=!0!==t,i=e?this.clone().utc():this;return i.year()<0||i.year()>9999?z(i,e?"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYYYY-MM-DD[T]HH:mm:ss.SSSZ"):S(Date.prototype.toISOString)?e?this.toDate().toISOString():new Date(this.valueOf()+60*this.utcOffset()*1e3).toISOString().replace("Z",z(i,"Z")):z(i,e?"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYY-MM-DD[T]HH:mm:ss.SSSZ")},di.inspect=function(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var t="moment",e="";this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",e="Z");var i="["+t+'("]',n=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",o=e+'[")]';return this.format(i+n+"-MM-DD[T]HH:mm:ss.SSS"+o)},di.toJSON=function(){return this.isValid()?this.toISOString():null},di.toString=function(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},di.unix=function(){return Math.floor(this.valueOf()/1e3)},di.valueOf=function(){return this._d.valueOf()-6e4*(this._offset||0)},di.creationData=function(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}},di.year=xt,di.isLeapYear=function(){return bt(this.year())},di.weekYear=function(t){return ii.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)},di.isoWeekYear=function(t){return ii.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)},di.quarter=di.quarters=function(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)},di.month=St,di.daysInMonth=function(){return Dt(this.year(),this.month())},di.week=di.weeks=function(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")},di.isoWeek=di.isoWeeks=function(t){var e=At(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")},di.weeksInYear=function(){var t=this.localeData()._week;return Xt(this.year(),t.dow,t.doy)},di.isoWeeksInYear=function(){return Xt(this.year(),1,4)},di.date=oi,di.day=di.days=function(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=function(t,e){return"string"!=typeof t?t:isNaN(t)?"number"==typeof(t=e.weekdaysParse(t))?t:null:parseInt(t,10)}(t,this.localeData()),this.add(t-e,"d")):e},di.weekday=function(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")},di.isoWeekday=function(t){if(!this.isValid())return null!=t?this:NaN;if(null!=t){var e=function(t,e){return"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t}(t,this.localeData());return this.day(this.day()%7?e:e-7)}return this.day()||7},di.dayOfYear=function(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")},di.hour=di.hours=$t,di.minute=di.minutes=si,di.second=di.seconds=ai,di.millisecond=di.milliseconds=hi,di.utcOffset=function(t,e,i){var n,s=this._offset||0;if(!this.isValid())return null!=t?this:NaN;if(null!=t){if("string"==typeof t){if(null===(t=Be(at,t)))return this}else Math.abs(t)<16&&!i&&(t*=60);return!this._isUTC&&e&&(n=We(this)),this._offset=t,this._isUTC=!0,null!=n&&this.add(n,"m"),s!==t&&(!e||this._changeInProgress?ze(this,Ye(t-s,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,o.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?s:We(this)},di.utc=function(t){return this.utcOffset(0,t)},di.local=function(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(We(this),"m")),this},di.parseZone=function(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var t=Be(rt,this._i);null!=t?this.utcOffset(t):this.utcOffset(0,!0)}return this},di.hasAlignedHourOffset=function(t){return!!this.isValid()&&(t=t?Ie(t).utcOffset():0,(this.utcOffset()-t)%60==0)},di.isDST=function(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},di.isLocal=function(){return!!this.isValid()&&!this._isUTC},di.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},di.isUtc=Ae,di.isUTC=Ae,di.zoneAbbr=function(){return this._isUTC?"UTC":""},di.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},di.dates=C("dates accessor is deprecated. Use date instead.",oi),di.months=C("months accessor is deprecated. Use month instead",St),di.years=C("years accessor is deprecated. Use year instead",xt),di.zone=C("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",(function(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()})),di.isDSTShifted=C("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",(function(){if(!a(this._isDSTShifted))return this._isDSTShifted;var t={};if(b(t,this),(t=ke(t))._a){var e=t._isUTC?p(t._a):Ie(t._a);this._isDSTShifted=this.isValid()&&D(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}));var ci=N.prototype;function pi(t,e,i,n){var o=ae(),s=p().set(n,e);return o[i](s,t)}function mi(t,e,i){if(l(t)&&(e=t,t=void 0),t=t||"",null!=e)return pi(t,e,i,"month");var n,o=[];for(n=0;n<12;n++)o[n]=pi(t,n,i,"month");return o}function fi(t,e,i,n){"boolean"==typeof t?(l(e)&&(i=e,e=void 0),e=e||""):(i=e=t,t=!1,l(e)&&(i=e,e=void 0),e=e||"");var o,s=ae(),r=t?s._week.dow:0;if(null!=i)return pi(e,(i+r)%7,n,"day");var a=[];for(o=0;o<7;o++)a[o]=pi(e,(o+r)%7,n,"day");return a}ci.calendar=function(t,e,i){var n=this._calendar[t]||this._calendar.sameElse;return S(n)?n.call(e,i):n},ci.longDateFormat=function(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,(function(t){return t.slice(1)})),this._longDateFormat[t])},ci.invalidDate=function(){return this._invalidDate},ci.ordinal=function(t){return this._ordinal.replace("%d",t)},ci.preparse=ui,ci.postformat=ui,ci.relativeTime=function(t,e,i,n){var o=this._relativeTime[i];return S(o)?o(t,e,i,n):o.replace(/%d/i,t)},ci.pastFuture=function(t,e){var i=this._relativeTime[t>0?"future":"past"];return S(i)?i(e):i.replace(/%s/i,e)},ci.set=function(t){var e,i;for(i in t)S(e=t[i])?this[i]=e:this["_"+i]=e;this._config=t,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)},ci.months=function(t,e){return t?s(this._months)?this._months[t.month()]:this._months[(this._months.isFormat||It).test(e)?"format":"standalone"][t.month()]:s(this._months)?this._months:this._months.standalone},ci.monthsShort=function(t,e){return t?s(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[It.test(e)?"format":"standalone"][t.month()]:s(this._monthsShort)?this._monthsShort:this._monthsShort.standalone},ci.monthsParse=function(t,e,i){var n,o,s;if(this._monthsParseExact)return Gt.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),n=0;n<12;n++){if(o=p([2e3,n]),i&&!this._longMonthsParse[n]&&(this._longMonthsParse[n]=new RegExp("^"+this.months(o,"").replace(".","")+"$","i"),this._shortMonthsParse[n]=new RegExp("^"+this.monthsShort(o,"").replace(".","")+"$","i")),i||this._monthsParse[n]||(s="^"+this.months(o,"")+"|^"+this.monthsShort(o,""),this._monthsParse[n]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[n].test(t))return n;if(i&&"MMM"===e&&this._shortMonthsParse[n].test(t))return n;if(!i&&this._monthsParse[n].test(t))return n}},ci.monthsRegex=function(t){return this._monthsParseExact?(u(this,"_monthsRegex")||Ft.call(this),t?this._monthsStrictRegex:this._monthsRegex):(u(this,"_monthsRegex")||(this._monthsRegex=Nt),this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex)},ci.monthsShortRegex=function(t){return this._monthsParseExact?(u(this,"_monthsRegex")||Ft.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):(u(this,"_monthsShortRegex")||(this._monthsShortRegex=Qt),this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex)},ci.week=function(t){return At(t,this._week.dow,this._week.doy).week},ci.firstDayOfYear=function(){return this._week.doy},ci.firstDayOfWeek=function(){return this._week.dow},ci.weekdays=function(t,e){var i=s(this._weekdays)?this._weekdays:this._weekdays[t&&!0!==t&&this._weekdays.isFormat.test(e)?"format":"standalone"];return!0===t?Tt(i,this._week.dow):t?i[t.day()]:i},ci.weekdaysMin=function(t){return!0===t?Tt(this._weekdaysMin,this._week.dow):t?this._weekdaysMin[t.day()]:this._weekdaysMin},ci.weekdaysShort=function(t){return!0===t?Tt(this._weekdaysShort,this._week.dow):t?this._weekdaysShort[t.day()]:this._weekdaysShort},ci.weekdaysParse=function(t,e,i){var n,o,s;if(this._weekdaysParseExact)return Ut.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),n=0;n<7;n++){if(o=p([2e3,1]).day(n),i&&!this._fullWeekdaysParse[n]&&(this._fullWeekdaysParse[n]=new RegExp("^"+this.weekdays(o,"").replace(".","\\.?")+"$","i"),this._shortWeekdaysParse[n]=new RegExp("^"+this.weekdaysShort(o,"").replace(".","\\.?")+"$","i"),this._minWeekdaysParse[n]=new RegExp("^"+this.weekdaysMin(o,"").replace(".","\\.?")+"$","i")),this._weekdaysParse[n]||(s="^"+this.weekdays(o,"")+"|^"+this.weekdaysShort(o,"")+"|^"+this.weekdaysMin(o,""),this._weekdaysParse[n]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[n].test(t))return n;if(i&&"ddd"===e&&this._shortWeekdaysParse[n].test(t))return n;if(i&&"dd"===e&&this._minWeekdaysParse[n].test(t))return n;if(!i&&this._weekdaysParse[n].test(t))return n}},ci.weekdaysRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):(u(this,"_weekdaysRegex")||(this._weekdaysRegex=zt),this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex)},ci.weekdaysShortRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(u(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=Vt),this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)},ci.weekdaysMinRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(u(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=jt),this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)},ci.isPM=function(t){return"p"===(t+"").toLowerCase().charAt(0)},ci.meridiem=function(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"},se("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10;return t+(1===k(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th")}}),o.lang=C("moment.lang is deprecated. Use moment.locale instead.",se),o.langData=C("moment.langData is deprecated. Use moment.localeData instead.",ae);var gi=Math.abs;function vi(t,e,i,n){var o=Ye(e,i);return t._milliseconds+=n*o._milliseconds,t._days+=n*o._days,t._months+=n*o._months,t._bubble()}function bi(t){return t<0?Math.floor(t):Math.ceil(t)}function yi(t){return 4800*t/146097}function xi(t){return 146097*t/4800}function wi(t){return function(){return this.as(t)}}var _i=wi("ms"),ki=wi("s"),Di=wi("m"),Ii=wi("h"),Ci=wi("d"),Li=wi("w"),Gi=wi("M"),Mi=wi("Q"),Si=wi("y");function Qi(t){return function(){return this.isValid()?this._data[t]:NaN}}var Ni=Qi("milliseconds"),Fi=Qi("seconds"),Zi=Qi("minutes"),Bi=Qi("hours"),Oi=Qi("days"),Wi=Qi("months"),Ai=Qi("years"),Xi=Math.round,Ti={ss:44,s:45,m:45,h:22,d:26,M:11};function Yi(t,e,i,n,o){return o.relativeTime(e||1,!!i,t,n)}function Ei(t,e,i){var n=Ye(t).abs(),o=Xi(n.as("s")),s=Xi(n.as("m")),r=Xi(n.as("h")),a=Xi(n.as("d")),l=Xi(n.as("M")),h=Xi(n.as("y")),d=o<=Ti.ss&&["s",o]||o<Ti.s&&["ss",o]||s<=1&&["m"]||s<Ti.m&&["mm",s]||r<=1&&["h"]||r<Ti.h&&["hh",r]||a<=1&&["d"]||a<Ti.d&&["dd",a]||l<=1&&["M"]||l<Ti.M&&["MM",l]||h<=1&&["y"]||["yy",h];return d[2]=e,d[3]=+t>0,d[4]=i,Yi.apply(null,d)}var Ri=Math.abs;function Ui(t){return(t>0)-(t<0)||+t}function zi(){if(!this.isValid())return this.localeData().invalidDate();var t,e,i=Ri(this._milliseconds)/1e3,n=Ri(this._days),o=Ri(this._months);t=_(i/60),e=_(t/60),i%=60,t%=60;var s=_(o/12),r=o%=12,a=n,l=e,h=t,d=i?i.toFixed(3).replace(/\.?0+$/,""):"",u=this.asSeconds();if(!u)return"P0D";var c=u<0?"-":"",p=Ui(this._months)!==Ui(u)?"-":"",m=Ui(this._days)!==Ui(u)?"-":"",f=Ui(this._milliseconds)!==Ui(u)?"-":"";return c+"P"+(s?p+s+"Y":"")+(r?p+r+"M":"")+(a?m+a+"D":"")+(l||h||d?"T":"")+(l?f+l+"H":"")+(h?f+h+"M":"")+(d?f+d+"S":"")}var Vi=Se.prototype;return Vi.isValid=function(){return this._isValid},Vi.abs=function(){var t=this._data;return this._milliseconds=gi(this._milliseconds),this._days=gi(this._days),this._months=gi(this._months),t.milliseconds=gi(t.milliseconds),t.seconds=gi(t.seconds),t.minutes=gi(t.minutes),t.hours=gi(t.hours),t.months=gi(t.months),t.years=gi(t.years),this},Vi.add=function(t,e){return vi(this,t,e,1)},Vi.subtract=function(t,e){return vi(this,t,e,-1)},Vi.as=function(t){if(!this.isValid())return NaN;var e,i,n=this._milliseconds;if("month"===(t=B(t))||"quarter"===t||"year"===t)switch(e=this._days+n/864e5,i=this._months+yi(e),t){case"month":return i;case"quarter":return i/3;case"year":return i/12}else switch(e=this._days+Math.round(xi(this._months)),t){case"week":return e/7+n/6048e5;case"day":return e+n/864e5;case"hour":return 24*e+n/36e5;case"minute":return 1440*e+n/6e4;case"second":return 86400*e+n/1e3;case"millisecond":return Math.floor(864e5*e)+n;default:throw new Error("Unknown unit "+t)}},Vi.asMilliseconds=_i,Vi.asSeconds=ki,Vi.asMinutes=Di,Vi.asHours=Ii,Vi.asDays=Ci,Vi.asWeeks=Li,Vi.asMonths=Gi,Vi.asQuarters=Mi,Vi.asYears=Si,Vi.valueOf=function(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*k(this._months/12):NaN},Vi._bubble=function(){var t,e,i,n,o,s=this._milliseconds,r=this._days,a=this._months,l=this._data;return s>=0&&r>=0&&a>=0||s<=0&&r<=0&&a<=0||(s+=864e5*bi(xi(a)+r),r=0,a=0),l.milliseconds=s%1e3,t=_(s/1e3),l.seconds=t%60,e=_(t/60),l.minutes=e%60,i=_(e/60),l.hours=i%24,r+=_(i/24),o=_(yi(r)),a+=o,r-=bi(xi(o)),n=_(a/12),a%=12,l.days=r,l.months=a,l.years=n,this},Vi.clone=function(){return Ye(this)},Vi.get=function(t){return t=B(t),this.isValid()?this[t+"s"]():NaN},Vi.milliseconds=Ni,Vi.seconds=Fi,Vi.minutes=Zi,Vi.hours=Bi,Vi.days=Oi,Vi.weeks=function(){return _(this.days()/7)},Vi.months=Wi,Vi.years=Ai,Vi.humanize=function(t){if(!this.isValid())return this.localeData().invalidDate();var e=this.localeData(),i=Ei(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)},Vi.toISOString=zi,Vi.toString=zi,Vi.toJSON=zi,Vi.locale=Pe,Vi.localeData=Ke,Vi.toIsoString=C("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",zi),Vi.lang=Je,U("X",0,0,"unix"),U("x",0,0,"valueOf"),dt("x",st),dt("X",/[+-]?\d+(\.\d{1,3})?/),mt("X",(function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))})),mt("x",(function(t,e,i){i._d=new Date(k(t))})),o.version="2.24.0",e=Ie,o.fn=di,o.min=function(){var t=[].slice.call(arguments,0);return Ge("isBefore",t)},o.max=function(){var t=[].slice.call(arguments,0);return Ge("isAfter",t)},o.now=function(){return Date.now?Date.now():+new Date},o.utc=p,o.unix=function(t){return Ie(1e3*t)},o.months=function(t,e){return mi(t,e,"months")},o.isDate=h,o.locale=se,o.invalid=g,o.duration=Ye,o.isMoment=w,o.weekdays=function(t,e,i){return fi(t,e,i,"weekdays")},o.parseZone=function(){return Ie.apply(null,arguments).parseZone()},o.localeData=ae,o.isDuration=Qe,o.monthsShort=function(t,e){return mi(t,e,"monthsShort")},o.weekdaysMin=function(t,e,i){return fi(t,e,i,"weekdaysMin")},o.defineLocale=re,o.updateLocale=function(t,e){if(null!=e){var i,n,o=te;null!=(n=oe(t))&&(o=n._config),(i=new N(e=Q(o,e))).parentLocale=ee[t],ee[t]=i,se(t)}else null!=ee[t]&&(null!=ee[t].parentLocale?ee[t]=ee[t].parentLocale:null!=ee[t]&&delete ee[t]);return ee[t]},o.locales=function(){return L(ee)},o.weekdaysShort=function(t,e,i){return fi(t,e,i,"weekdaysShort")},o.normalizeUnits=B,o.relativeTimeRounding=function(t){return void 0===t?Xi:"function"==typeof t&&(Xi=t,!0)},o.relativeTimeThreshold=function(t,e){return void 0!==Ti[t]&&(void 0===e?Ti[t]:(Ti[t]=e,"s"===t&&(Ti.ss=e-1),!0))},o.calendarFormat=function(t,e){var i=t.diff(e,"days",!0);return i<-6?"sameElse":i<-1?"lastWeek":i<0?"lastDay":i<1?"sameDay":i<2?"nextDay":i<7?"nextWeek":"sameElse"},o.prototype=di,o.HTML5_FMT={DATETIME_LOCAL:"YYYY-MM-DDTHH:mm",DATETIME_LOCAL_SECONDS:"YYYY-MM-DDTHH:mm:ss",DATETIME_LOCAL_MS:"YYYY-MM-DDTHH:mm:ss.SSS",DATE:"YYYY-MM-DD",TIME:"HH:mm",TIME_SECONDS:"HH:mm:ss",TIME_MS:"HH:mm:ss.SSS",WEEK:"GGGG-[W]WW",MONTH:"YYYY-MM"},o}()})),r=(n((function(t,e){(function(t){function e(t,e,i,n){var o={m:["eine Minute","einer Minute"],h:["eine Stunde","einer Stunde"],d:["ein Tag","einem Tag"],dd:[t+" Tage",t+" Tagen"],M:["ein Monat","einem Monat"],MM:[t+" Monate",t+" Monaten"],y:["ein Jahr","einem Jahr"],yy:[t+" Jahre",t+" Jahren"]};return e?o[i][0]:o[i][1]}t.defineLocale("de",{months:"Januar_Februar_März_April_Mai_Juni_Juli_August_September_Oktober_November_Dezember".split("_"),monthsShort:"Jan._Feb._März_Apr._Mai_Juni_Juli_Aug._Sep._Okt._Nov._Dez.".split("_"),monthsParseExact:!0,weekdays:"Sonntag_Montag_Dienstag_Mittwoch_Donnerstag_Freitag_Samstag".split("_"),weekdaysShort:"So._Mo._Di._Mi._Do._Fr._Sa.".split("_"),weekdaysMin:"So_Mo_Di_Mi_Do_Fr_Sa".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D. MMMM YYYY",LLL:"D. MMMM YYYY HH:mm",LLLL:"dddd, D. MMMM YYYY HH:mm"},calendar:{sameDay:"[heute um] LT [Uhr]",sameElse:"L",nextDay:"[morgen um] LT [Uhr]",nextWeek:"dddd [um] LT [Uhr]",lastDay:"[gestern um] LT [Uhr]",lastWeek:"[letzten] dddd [um] LT [Uhr]"},relativeTime:{future:"in %s",past:"vor %s",s:"ein paar Sekunden",ss:"%d Sekunden",m:e,mm:"%d Minuten",h:e,hh:"%d Stunden",d:e,dd:e,M:e,MM:e,y:e,yy:e},dayOfMonthOrdinalParse:/\d{1,2}\./,ordinal:"%d.",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){var e="ene._feb._mar._abr._may._jun._jul._ago._sep._oct._nov._dic.".split("_"),i="ene_feb_mar_abr_may_jun_jul_ago_sep_oct_nov_dic".split("_"),n=[/^ene/i,/^feb/i,/^mar/i,/^abr/i,/^may/i,/^jun/i,/^jul/i,/^ago/i,/^sep/i,/^oct/i,/^nov/i,/^dic/i],o=/^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre|ene\.?|feb\.?|mar\.?|abr\.?|may\.?|jun\.?|jul\.?|ago\.?|sep\.?|oct\.?|nov\.?|dic\.?)/i;t.defineLocale("es",{months:"enero_febrero_marzo_abril_mayo_junio_julio_agosto_septiembre_octubre_noviembre_diciembre".split("_"),monthsShort:function(t,n){return t?/-MMM-/.test(n)?i[t.month()]:e[t.month()]:e},monthsRegex:o,monthsShortRegex:o,monthsStrictRegex:/^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)/i,monthsShortStrictRegex:/^(ene\.?|feb\.?|mar\.?|abr\.?|may\.?|jun\.?|jul\.?|ago\.?|sep\.?|oct\.?|nov\.?|dic\.?)/i,monthsParse:n,longMonthsParse:n,shortMonthsParse:n,weekdays:"domingo_lunes_martes_miércoles_jueves_viernes_sábado".split("_"),weekdaysShort:"dom._lun._mar._mié._jue._vie._sáb.".split("_"),weekdaysMin:"do_lu_ma_mi_ju_vi_sá".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"H:mm",LTS:"H:mm:ss",L:"DD/MM/YYYY",LL:"D [de] MMMM [de] YYYY",LLL:"D [de] MMMM [de] YYYY H:mm",LLLL:"dddd, D [de] MMMM [de] YYYY H:mm"},calendar:{sameDay:function(){return"[hoy a la"+(1!==this.hours()?"s":"")+"] LT"},nextDay:function(){return"[mañana a la"+(1!==this.hours()?"s":"")+"] LT"},nextWeek:function(){return"dddd [a la"+(1!==this.hours()?"s":"")+"] LT"},lastDay:function(){return"[ayer a la"+(1!==this.hours()?"s":"")+"] LT"},lastWeek:function(){return"[el] dddd [pasado a la"+(1!==this.hours()?"s":"")+"] LT"},sameElse:"L"},relativeTime:{future:"en %s",past:"hace %s",s:"unos segundos",ss:"%d segundos",m:"un minuto",mm:"%d minutos",h:"una hora",hh:"%d horas",d:"un día",dd:"%d días",M:"un mes",MM:"%d meses",y:"un año",yy:"%d años"},dayOfMonthOrdinalParse:/\d{1,2}º/,ordinal:"%dº",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("fr",{months:"janvier_février_mars_avril_mai_juin_juillet_août_septembre_octobre_novembre_décembre".split("_"),monthsShort:"janv._févr._mars_avr._mai_juin_juil._août_sept._oct._nov._déc.".split("_"),monthsParseExact:!0,weekdays:"dimanche_lundi_mardi_mercredi_jeudi_vendredi_samedi".split("_"),weekdaysShort:"dim._lun._mar._mer._jeu._ven._sam.".split("_"),weekdaysMin:"di_lu_ma_me_je_ve_sa".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD/MM/YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[Aujourd’hui à] LT",nextDay:"[Demain à] LT",nextWeek:"dddd [à] LT",lastDay:"[Hier à] LT",lastWeek:"dddd [dernier à] LT",sameElse:"L"},relativeTime:{future:"dans %s",past:"il y a %s",s:"quelques secondes",ss:"%d secondes",m:"une minute",mm:"%d minutes",h:"une heure",hh:"%d heures",d:"un jour",dd:"%d jours",M:"un mois",MM:"%d mois",y:"un an",yy:"%d ans"},dayOfMonthOrdinalParse:/\d{1,2}(er|)/,ordinal:function(t,e){switch(e){case"D":return t+(1===t?"er":"");default:case"M":case"Q":case"DDD":case"d":return t+(1===t?"er":"e");case"w":case"W":return t+(1===t?"re":"e")}},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("it",{months:"gennaio_febbraio_marzo_aprile_maggio_giugno_luglio_agosto_settembre_ottobre_novembre_dicembre".split("_"),monthsShort:"gen_feb_mar_apr_mag_giu_lug_ago_set_ott_nov_dic".split("_"),weekdays:"domenica_lunedì_martedì_mercoledì_giovedì_venerdì_sabato".split("_"),weekdaysShort:"dom_lun_mar_mer_gio_ven_sab".split("_"),weekdaysMin:"do_lu_ma_me_gi_ve_sa".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD/MM/YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[Oggi alle] LT",nextDay:"[Domani alle] LT",nextWeek:"dddd [alle] LT",lastDay:"[Ieri alle] LT",lastWeek:function(){switch(this.day()){case 0:return"[la scorsa] dddd [alle] LT";default:return"[lo scorso] dddd [alle] LT"}},sameElse:"L"},relativeTime:{future:function(t){return(/^[0-9].+$/.test(t)?"tra":"in")+" "+t},past:"%s fa",s:"alcuni secondi",ss:"%d secondi",m:"un minuto",mm:"%d minuti",h:"un'ora",hh:"%d ore",d:"un giorno",dd:"%d giorni",M:"un mese",MM:"%d mesi",y:"un anno",yy:"%d anni"},dayOfMonthOrdinalParse:/\d{1,2}º/,ordinal:"%dº",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("ja",{months:"一月_二月_三月_四月_五月_六月_七月_八月_九月_十月_十一月_十二月".split("_"),monthsShort:"1月_2月_3月_4月_5月_6月_7月_8月_9月_10月_11月_12月".split("_"),weekdays:"日曜日_月曜日_火曜日_水曜日_木曜日_金曜日_土曜日".split("_"),weekdaysShort:"日_月_火_水_木_金_土".split("_"),weekdaysMin:"日_月_火_水_木_金_土".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"YYYY/MM/DD",LL:"YYYY年M月D日",LLL:"YYYY年M月D日 HH:mm",LLLL:"YYYY年M月D日 dddd HH:mm",l:"YYYY/MM/DD",ll:"YYYY年M月D日",lll:"YYYY年M月D日 HH:mm",llll:"YYYY年M月D日(ddd) HH:mm"},meridiemParse:/午前|午後/i,isPM:function(t){return"午後"===t},meridiem:function(t,e,i){return t<12?"午前":"午後"},calendar:{sameDay:"[今日] LT",nextDay:"[明日] LT",nextWeek:function(t){return t.week()<this.week()?"[来週]dddd LT":"dddd LT"},lastDay:"[昨日] LT",lastWeek:function(t){return this.week()<t.week()?"[先週]dddd LT":"dddd LT"},sameElse:"L"},dayOfMonthOrdinalParse:/\d{1,2}日/,ordinal:function(t,e){switch(e){case"d":case"D":case"DDD":return t+"日";default:return t}},relativeTime:{future:"%s後",past:"%s前",s:"数秒",ss:"%d秒",m:"1分",mm:"%d分",h:"1時間",hh:"%d時間",d:"1日",dd:"%d日",M:"1ヶ月",MM:"%dヶ月",y:"1年",yy:"%d年"}})})(s)})),n((function(t,e){(function(t){var e="jan._feb._mrt._apr._mei_jun._jul._aug._sep._okt._nov._dec.".split("_"),i="jan_feb_mrt_apr_mei_jun_jul_aug_sep_okt_nov_dec".split("_"),n=[/^jan/i,/^feb/i,/^maart|mrt.?$/i,/^apr/i,/^mei$/i,/^jun[i.]?$/i,/^jul[i.]?$/i,/^aug/i,/^sep/i,/^okt/i,/^nov/i,/^dec/i],o=/^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december|jan\.?|feb\.?|mrt\.?|apr\.?|ju[nl]\.?|aug\.?|sep\.?|okt\.?|nov\.?|dec\.?)/i;t.defineLocale("nl",{months:"januari_februari_maart_april_mei_juni_juli_augustus_september_oktober_november_december".split("_"),monthsShort:function(t,n){return t?/-MMM-/.test(n)?i[t.month()]:e[t.month()]:e},monthsRegex:o,monthsShortRegex:o,monthsStrictRegex:/^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december)/i,monthsShortStrictRegex:/^(jan\.?|feb\.?|mrt\.?|apr\.?|mei|ju[nl]\.?|aug\.?|sep\.?|okt\.?|nov\.?|dec\.?)/i,monthsParse:n,longMonthsParse:n,shortMonthsParse:n,weekdays:"zondag_maandag_dinsdag_woensdag_donderdag_vrijdag_zaterdag".split("_"),weekdaysShort:"zo._ma._di._wo._do._vr._za.".split("_"),weekdaysMin:"zo_ma_di_wo_do_vr_za".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD-MM-YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[vandaag om] LT",nextDay:"[morgen om] LT",nextWeek:"dddd [om] LT",lastDay:"[gisteren om] LT",lastWeek:"[afgelopen] dddd [om] LT",sameElse:"L"},relativeTime:{future:"over %s",past:"%s geleden",s:"een paar seconden",ss:"%d seconden",m:"één minuut",mm:"%d minuten",h:"één uur",hh:"%d uur",d:"één dag",dd:"%d dagen",M:"één maand",MM:"%d maanden",y:"één jaar",yy:"%d jaar"},dayOfMonthOrdinalParse:/\d{1,2}(ste|de)/,ordinal:function(t){return t+(1===t||8===t||t>=20?"ste":"de")},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){var e="styczeń_luty_marzec_kwiecień_maj_czerwiec_lipiec_sierpień_wrzesień_październik_listopad_grudzień".split("_"),i="stycznia_lutego_marca_kwietnia_maja_czerwca_lipca_sierpnia_września_października_listopada_grudnia".split("_");function n(t){return t%10<5&&t%10>1&&~~(t/10)%10!=1}function o(t,e,i){var o=t+" ";switch(i){case"ss":return o+(n(t)?"sekundy":"sekund");case"m":return e?"minuta":"minutę";case"mm":return o+(n(t)?"minuty":"minut");case"h":return e?"godzina":"godzinę";case"hh":return o+(n(t)?"godziny":"godzin");case"MM":return o+(n(t)?"miesiące":"miesięcy");case"yy":return o+(n(t)?"lata":"lat")}}t.defineLocale("pl",{months:function(t,n){return t?""===n?"("+i[t.month()]+"|"+e[t.month()]+")":/D MMMM/.test(n)?i[t.month()]:e[t.month()]:e},monthsShort:"sty_lut_mar_kwi_maj_cze_lip_sie_wrz_paź_lis_gru".split("_"),weekdays:"niedziela_poniedziałek_wtorek_środa_czwartek_piątek_sobota".split("_"),weekdaysShort:"ndz_pon_wt_śr_czw_pt_sob".split("_"),weekdaysMin:"Nd_Pn_Wt_Śr_Cz_Pt_So".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd, D MMMM YYYY HH:mm"},calendar:{sameDay:"[Dziś o] LT",nextDay:"[Jutro o] LT",nextWeek:function(){switch(this.day()){case 0:return"[W niedzielę o] LT";case 2:return"[We wtorek o] LT";case 3:return"[W środę o] LT";case 6:return"[W sobotę o] LT";default:return"[W] dddd [o] LT"}},lastDay:"[Wczoraj o] LT",lastWeek:function(){switch(this.day()){case 0:return"[W zeszłą niedzielę o] LT";case 3:return"[W zeszłą środę o] LT";case 6:return"[W zeszłą sobotę o] LT";default:return"[W zeszły] dddd [o] LT"}},sameElse:"L"},relativeTime:{future:"za %s",past:"%s temu",s:"kilka sekund",ss:o,m:o,mm:o,h:o,hh:o,d:"1 dzień",dd:"%d dni",M:"miesiąc",MM:o,y:"rok",yy:o},dayOfMonthOrdinalParse:/\d{1,2}\./,ordinal:"%d.",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){function e(t,e,i){var n,o;return"m"===i?e?"минута":"минуту":t+" "+(n=+t,o={ss:e?"секунда_секунды_секунд":"секунду_секунды_секунд",mm:e?"минута_минуты_минут":"минуту_минуты_минут",hh:"час_часа_часов",dd:"день_дня_дней",MM:"месяц_месяца_месяцев",yy:"год_года_лет"}[i].split("_"),n%10==1&&n%100!=11?o[0]:n%10>=2&&n%10<=4&&(n%100<10||n%100>=20)?o[1]:o[2])}var i=[/^янв/i,/^фев/i,/^мар/i,/^апр/i,/^ма[йя]/i,/^июн/i,/^июл/i,/^авг/i,/^сен/i,/^окт/i,/^ноя/i,/^дек/i];t.defineLocale("ru",{months:{format:"января_февраля_марта_апреля_мая_июня_июля_августа_сентября_октября_ноября_декабря".split("_"),standalone:"январь_февраль_март_апрель_май_июнь_июль_август_сентябрь_октябрь_ноябрь_декабрь".split("_")},monthsShort:{format:"янв._февр._мар._апр._мая_июня_июля_авг._сент._окт._нояб._дек.".split("_"),standalone:"янв._февр._март_апр._май_июнь_июль_авг._сент._окт._нояб._дек.".split("_")},weekdays:{standalone:"воскресенье_понедельник_вторник_среда_четверг_пятница_суббота".split("_"),format:"воскресенье_понедельник_вторник_среду_четверг_пятницу_субботу".split("_"),isFormat:/\[ ?[Вв] ?(?:прошлую|следующую|эту)? ?\] ?dddd/},weekdaysShort:"вс_пн_вт_ср_чт_пт_сб".split("_"),weekdaysMin:"вс_пн_вт_ср_чт_пт_сб".split("_"),monthsParse:i,longMonthsParse:i,shortMonthsParse:i,monthsRegex:/^(январ[ья]|янв\.?|феврал[ья]|февр?\.?|марта?|мар\.?|апрел[ья]|апр\.?|ма[йя]|июн[ья]|июн\.?|июл[ья]|июл\.?|августа?|авг\.?|сентябр[ья]|сент?\.?|октябр[ья]|окт\.?|ноябр[ья]|нояб?\.?|декабр[ья]|дек\.?)/i,monthsShortRegex:/^(январ[ья]|янв\.?|феврал[ья]|февр?\.?|марта?|мар\.?|апрел[ья]|апр\.?|ма[йя]|июн[ья]|июн\.?|июл[ья]|июл\.?|августа?|авг\.?|сентябр[ья]|сент?\.?|октябр[ья]|окт\.?|ноябр[ья]|нояб?\.?|декабр[ья]|дек\.?)/i,monthsStrictRegex:/^(январ[яь]|феврал[яь]|марта?|апрел[яь]|ма[яй]|июн[яь]|июл[яь]|августа?|сентябр[яь]|октябр[яь]|ноябр[яь]|декабр[яь])/i,monthsShortStrictRegex:/^(янв\.|февр?\.|мар[т.]|апр\.|ма[яй]|июн[ья.]|июл[ья.]|авг\.|сент?\.|окт\.|нояб?\.|дек\.)/i,longDateFormat:{LT:"H:mm",LTS:"H:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY г.",LLL:"D MMMM YYYY г., H:mm",LLLL:"dddd, D MMMM YYYY г., H:mm"},calendar:{sameDay:"[Сегодня, в] LT",nextDay:"[Завтра, в] LT",lastDay:"[Вчера, в] LT",nextWeek:function(t){if(t.week()===this.week())return 2===this.day()?"[Во] dddd, [в] LT":"[В] dddd, [в] LT";switch(this.day()){case 0:return"[В следующее] dddd, [в] LT";case 1:case 2:case 4:return"[В следующий] dddd, [в] LT";case 3:case 5:case 6:return"[В следующую] dddd, [в] LT"}},lastWeek:function(t){if(t.week()===this.week())return 2===this.day()?"[Во] dddd, [в] LT":"[В] dddd, [в] LT";switch(this.day()){case 0:return"[В прошлое] dddd, [в] LT";case 1:case 2:case 4:return"[В прошлый] dddd, [в] LT";case 3:case 5:case 6:return"[В прошлую] dddd, [в] LT"}},sameElse:"L"},relativeTime:{future:"через %s",past:"%s назад",s:"несколько секунд",ss:e,m:e,mm:e,h:"час",hh:e,d:"день",dd:e,M:"месяц",MM:e,y:"год",yy:e},meridiemParse:/ночи|утра|дня|вечера/i,isPM:function(t){return/^(дня|вечера)$/.test(t)},meridiem:function(t,e,i){return t<4?"ночи":t<12?"утра":t<17?"дня":"вечера"},dayOfMonthOrdinalParse:/\d{1,2}-(й|го|я)/,ordinal:function(t,e){switch(e){case"M":case"d":case"DDD":return t+"-й";case"D":return t+"-го";case"w":case"W":return t+"-я";default:return t}},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){function e(t,e,i){var n,o;return"m"===i?e?"хвилина":"хвилину":"h"===i?e?"година":"годину":t+" "+(n=+t,o={ss:e?"секунда_секунди_секунд":"секунду_секунди_секунд",mm:e?"хвилина_хвилини_хвилин":"хвилину_хвилини_хвилин",hh:e?"година_години_годин":"годину_години_годин",dd:"день_дні_днів",MM:"місяць_місяці_місяців",yy:"рік_роки_років"}[i].split("_"),n%10==1&&n%100!=11?o[0]:n%10>=2&&n%10<=4&&(n%100<10||n%100>=20)?o[1]:o[2])}function i(t){return function(){return t+"о"+(11===this.hours()?"б":"")+"] LT"}}t.defineLocale("uk",{months:{format:"січня_лютого_березня_квітня_травня_червня_липня_серпня_вересня_жовтня_листопада_грудня".split("_"),standalone:"січень_лютий_березень_квітень_травень_червень_липень_серпень_вересень_жовтень_листопад_грудень".split("_")},monthsShort:"січ_лют_бер_квіт_трав_черв_лип_серп_вер_жовт_лист_груд".split("_"),weekdays:function(t,e){var i={nominative:"неділя_понеділок_вівторок_середа_четвер_п’ятниця_субота".split("_"),accusative:"неділю_понеділок_вівторок_середу_четвер_п’ятницю_суботу".split("_"),genitive:"неділі_понеділка_вівторка_середи_четверга_п’ятниці_суботи".split("_")};return!0===t?i.nominative.slice(1,7).concat(i.nominative.slice(0,1)):t?i[/(\[[ВвУу]\]) ?dddd/.test(e)?"accusative":/\[?(?:минулої|наступної)? ?\] ?dddd/.test(e)?"genitive":"nominative"][t.day()]:i.nominative},weekdaysShort:"нд_пн_вт_ср_чт_пт_сб".split("_"),weekdaysMin:"нд_пн_вт_ср_чт_пт_сб".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY р.",LLL:"D MMMM YYYY р., HH:mm",LLLL:"dddd, D MMMM YYYY р., HH:mm"},calendar:{sameDay:i("[Сьогодні "),nextDay:i("[Завтра "),lastDay:i("[Вчора "),nextWeek:i("[У] dddd ["),lastWeek:function(){switch(this.day()){case 0:case 3:case 5:case 6:return i("[Минулої] dddd [").call(this);case 1:case 2:case 4:return i("[Минулого] dddd [").call(this)}},sameElse:"L"},relativeTime:{future:"за %s",past:"%s тому",s:"декілька секунд",ss:e,m:e,mm:e,h:"годину",hh:e,d:"день",dd:e,M:"місяць",MM:e,y:"рік",yy:e},meridiemParse:/ночі|ранку|дня|вечора/,isPM:function(t){return/^(дня|вечора)$/.test(t)},meridiem:function(t,e,i){return t<4?"ночі":t<12?"ранку":t<17?"дня":"вечора"},dayOfMonthOrdinalParse:/\d{1,2}-(й|го)/,ordinal:function(t,e){switch(e){case"M":case"d":case"DDD":case"w":case"W":return t+"-й";case"D":return t+"-го";default:return t}},week:{dow:1,doy:7}})})(s)})),function(t){return t&&t.Math==Math&&t}),a=r("object"==typeof globalThis&&globalThis)||r("object"==typeof window&&window)||r("object"==typeof self&&self)||r("object"==typeof e&&e)||Function("return this")(),l=function(t){try{return!!t()}catch(t){return!0}},h=!l((function(){return 7!=Object.defineProperty({},1,{get:function(){return 7}})[1]})),d={}.propertyIsEnumerable,u=Object.getOwnPropertyDescriptor,c={f:u&&!d.call({1:2},1)?function(t){var e=u(this,t);return!!e&&e.enumerable}:d},p=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}},m={}.toString,f=function(t){return m.call(t).slice(8,-1)},g="".split,v=l((function(){return!Object("z").propertyIsEnumerable(0)}))?function(t){return"String"==f(t)?g.call(t,""):Object(t)}:Object,b=function(t){if(null==t)throw TypeError("Can't call method on "+t);return t},y=function(t){return v(b(t))},x=function(t){return"object"==typeof t?null!==t:"function"==typeof t},w=function(t,e){if(!x(t))return t;var i,n;if(e&&"function"==typeof(i=t.toString)&&!x(n=i.call(t)))return n;if("function"==typeof(i=t.valueOf)&&!x(n=i.call(t)))return n;if(!e&&"function"==typeof(i=t.toString)&&!x(n=i.call(t)))return n;throw TypeError("Can't convert object to primitive value")},_={}.hasOwnProperty,k=function(t,e){return _.call(t,e)},D=a.document,I=x(D)&&x(D.createElement),C=function(t){return I?D.createElement(t):{}},L=!h&&!l((function(){return 7!=Object.defineProperty(C("div"),"a",{get:function(){return 7}}).a})),G=Object.getOwnPropertyDescriptor,M={f:h?G:function(t,e){if(t=y(t),e=w(e,!0),L)try{return G(t,e)}catch(t){}if(k(t,e))return p(!c.f.call(t,e),t[e])}},S=/#|\.prototype\./,Q=function(t,e){var i=F[N(t)];return i==B||i!=Z&&("function"==typeof e?l(e):!!e)},N=Q.normalize=function(t){return String(t).replace(S,".").toLowerCase()},F=Q.data={},Z=Q.NATIVE="N",B=Q.POLYFILL="P",O=Q,W={},A=function(t){if("function"!=typeof t)throw TypeError(String(t)+" is not a function");return t},X=function(t,e,i){if(A(t),void 0===e)return t;switch(i){case 0:return function(){return t.call(e)};case 1:return function(i){return t.call(e,i)};case 2:return function(i,n){return t.call(e,i,n)};case 3:return function(i,n,o){return t.call(e,i,n,o)}}return function(){return t.apply(e,arguments)}},T=function(t){if(!x(t))throw TypeError(String(t)+" is not an object");return t},Y=Object.defineProperty,E={f:h?Y:function(t,e,i){if(T(t),e=w(e,!0),T(i),L)try{return Y(t,e,i)}catch(t){}if("get"in i||"set"in i)throw TypeError("Accessors not supported");return"value"in i&&(t[e]=i.value),t}},R=h?function(t,e,i){return E.f(t,e,p(1,i))}:function(t,e,i){return t[e]=i,t},U=M.f,z=function(t){var e=function(e,i,n){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(e);case 2:return new t(e,i)}return new t(e,i,n)}return t.apply(this,arguments)};return e.prototype=t.prototype,e},V=function(t,e){var i,n,o,s,r,l,h,d,u=t.target,c=t.global,p=t.stat,m=t.proto,f=c?a:p?a[u]:(a[u]||{}).prototype,g=c?W:W[u]||(W[u]={}),v=g.prototype;for(o in e)i=!O(c?o:u+(p?".":"#")+o,t.forced)&&f&&k(f,o),r=g[o],i&&(l=t.noTargetGet?(d=U(f,o))&&d.value:f[o]),s=i&&l?l:e[o],i&&typeof r==typeof s||(h=t.bind&&i?X(s,a):t.wrap&&i?z(s):m&&"function"==typeof s?X(Function.call,s):s,(t.sham||s&&s.sham||r&&r.sham)&&R(h,"sham",!0),g[o]=h,m&&(k(W,n=u+"Prototype")||R(W,n,{}),W[n][o]=s,t.real&&v&&!v[o]&&R(v,o,s)))};V({target:"Object",stat:!0,forced:!h,sham:!h},{defineProperty:E.f});var j=n((function(t){var e=W.Object,i=t.exports=function(t,i,n){return e.defineProperty(t,i,n)};e.defineProperty.sham&&(i.sham=!0)})),H=j,P=Math.ceil,J=Math.floor,K=function(t){return isNaN(t=+t)?0:(t>0?J:P)(t)},q=Math.min,$=function(t){return t>0?q(K(t),9007199254740991):0},tt=Math.max,et=Math.min,it=function(t,e){var i=K(t);return i<0?tt(i+e,0):et(i,e)},nt=function(t){return function(e,i,n){var o,s=y(e),r=$(s.length),a=it(n,r);if(t&&i!=i){for(;r>a;)if((o=s[a++])!=o)return!0}else for(;r>a;a++)if((t||a in s)&&s[a]===i)return t||a||0;return!t&&-1}},ot={includes:nt(!0),indexOf:nt(!1)},st={},rt=ot.indexOf,at=function(t,e){var i,n=y(t),o=0,s=[];for(i in n)!k(st,i)&&k(n,i)&&s.push(i);for(;e.length>o;)k(n,i=e[o++])&&(~rt(s,i)||s.push(i));return s},lt=["constructor","hasOwnProperty","isPrototypeOf","propertyIsEnumerable","toLocaleString","toString","valueOf"],ht=Object.keys||function(t){return at(t,lt)},dt=h?Object.defineProperties:function(t,e){T(t);for(var i,n=ht(e),o=n.length,s=0;o>s;)E.f(t,i=n[s++],e[i]);return t};V({target:"Object",stat:!0,forced:!h,sham:!h},{defineProperties:dt});var ut=n((function(t){var e=W.Object,i=t.exports=function(t,i){return e.defineProperties(t,i)};e.defineProperties.sham&&(i.sham=!0)})),ct=function(t){return"function"==typeof t?t:void 0},pt=function(t,e){return arguments.length<2?ct(W[t])||ct(a[t]):W[t]&&W[t][e]||a[t]&&a[t][e]},mt=lt.concat("length","prototype"),ft={f:Object.getOwnPropertyNames||function(t){return at(t,mt)}},gt={f:Object.getOwnPropertySymbols},vt=pt("Reflect","ownKeys")||function(t){var e=ft.f(T(t)),i=gt.f;return i?e.concat(i(t)):e},bt=function(t,e,i){var n=w(e);n in t?E.f(t,n,p(0,i)):t[n]=i};V({target:"Object",stat:!0,sham:!h},{getOwnPropertyDescriptors:function(t){for(var e,i,n=y(t),o=M.f,s=vt(n),r={},a=0;s.length>a;)void 0!==(i=o(n,e=s[a++]))&&bt(r,e,i);return r}});var yt=W.Object.getOwnPropertyDescriptors,xt=M.f,wt=l((function(){xt(1)}));V({target:"Object",stat:!0,forced:!h||wt,sham:!h},{getOwnPropertyDescriptor:function(t,e){return xt(y(t),e)}});var _t,kt=n((function(t){var e=W.Object,i=t.exports=function(t,i){return e.getOwnPropertyDescriptor(t,i)};e.getOwnPropertyDescriptor.sham&&(i.sham=!0)})),Dt=!!Object.getOwnPropertySymbols&&!l((function(){return!String(Symbol())})),It=Dt&&!Symbol.sham&&"symbol"==typeof Symbol.iterator,Ct=Array.isArray||function(t){return"Array"==f(t)},Lt=function(t){return Object(b(t))},Gt=pt("document","documentElement"),Mt=a["__core-js_shared__"]||function(t,e){try{R(a,t,e)}catch(i){a[t]=e}return e}("__core-js_shared__",{}),St=n((function(t){(t.exports=function(t,e){return Mt[t]||(Mt[t]=void 0!==e?e:{})})("versions",[]).push({version:"3.6.4",mode:"pure",copyright:"© 2020 Denis Pushkarev (zloirock.ru)"})})),Qt=0,Nt=Math.random(),Ft=function(t){return"Symbol("+String(void 0===t?"":t)+")_"+(++Qt+Nt).toString(36)},Zt=St("keys"),Bt=function(t){return Zt[t]||(Zt[t]=Ft(t))},Ot=Bt("IE_PROTO"),Wt=function(){},At=function(t){return"<script>"+t+"<\/script>"},Xt=function(){try{_t=document.domain&&new ActiveXObject("htmlfile")}catch(t){}var t,e;Xt=_t?function(t){t.write(At("")),t.close();var e=t.parentWindow.Object;return t=null,e}(_t):((e=C("iframe")).style.display="none",Gt.appendChild(e),e.src=String("javascript:"),(t=e.contentWindow.document).open(),t.write(At("document.F=Object")),t.close(),t.F);for(var i=lt.length;i--;)delete Xt.prototype[lt[i]];return Xt()};st[Ot]=!0;var Tt=Object.create||function(t,e){var i;return null!==t?(Wt.prototype=T(t),i=new Wt,Wt.prototype=null,i[Ot]=t):i=Xt(),void 0===e?i:dt(i,e)},Yt=ft.f,Et={}.toString,Rt="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],Ut={f:function(t){return Rt&&"[object Window]"==Et.call(t)?function(t){try{return Yt(t)}catch(t){return Rt.slice()}}(t):Yt(y(t))}},zt=function(t,e,i,n){n&&n.enumerable?t[e]=i:R(t,e,i)},Vt=St("wks"),jt=a.Symbol,Ht=It?jt:jt&&jt.withoutSetter||Ft,Pt=function(t){return k(Vt,t)||(Dt&&k(jt,t)?Vt[t]=jt[t]:Vt[t]=Ht("Symbol."+t)),Vt[t]},Jt={f:Pt},Kt=E.f,qt=function(t){var e=W.Symbol||(W.Symbol={});k(e,t)||Kt(e,t,{value:Jt.f(t)})},$t={};$t[Pt("toStringTag")]="z";var te="[object z]"===String($t),ee=Pt("toStringTag"),ie="Arguments"==f(function(){return arguments}()),ne=te?f:function(t){var e,i,n;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(i=function(t,e){try{return t[e]}catch(t){}}(e=Object(t),ee))?i:ie?f(e):"Object"==(n=f(e))&&"function"==typeof e.callee?"Arguments":n},oe=te?{}.toString:function(){return"[object "+ne(this)+"]"},se=E.f,re=Pt("toStringTag"),ae=function(t,e,i,n){if(t){var o=i?t:t.prototype;k(o,re)||se(o,re,{configurable:!0,value:e}),n&&!te&&R(o,"toString",oe)}},le=Function.toString;"function"!=typeof Mt.inspectSource&&(Mt.inspectSource=function(t){return le.call(t)});var he,de,ue,ce=Mt.inspectSource,pe=a.WeakMap,me="function"==typeof pe&&/native code/.test(ce(pe)),fe=a.WeakMap;if(me){var ge=new fe,ve=ge.get,be=ge.has,ye=ge.set;he=function(t,e){return ye.call(ge,t,e),e},de=function(t){return ve.call(ge,t)||{}},ue=function(t){return be.call(ge,t)}}else{var xe=Bt("state");st[xe]=!0,he=function(t,e){return R(t,xe,e),e},de=function(t){return k(t,xe)?t[xe]:{}},ue=function(t){return k(t,xe)}}var we={set:he,get:de,has:ue,enforce:function(t){return ue(t)?de(t):he(t,{})},getterFor:function(t){return function(e){var i;if(!x(e)||(i=de(e)).type!==t)throw TypeError("Incompatible receiver, "+t+" required");return i}}},_e=Pt("species"),ke=function(t,e){var i;return Ct(t)&&("function"!=typeof(i=t.constructor)||i!==Array&&!Ct(i.prototype)?x(i)&&null===(i=i[_e])&&(i=void 0):i=void 0),new(void 0===i?Array:i)(0===e?0:e)},De=[].push,Ie=function(t){var e=1==t,i=2==t,n=3==t,o=4==t,s=6==t,r=5==t||s;return function(a,l,h,d){for(var u,c,p=Lt(a),m=v(p),f=X(l,h,3),g=$(m.length),b=0,y=d||ke,x=e?y(a,g):i?y(a,0):void 0;g>b;b++)if((r||b in m)&&(c=f(u=m[b],b,p),t))if(e)x[b]=c;else if(c)switch(t){case 3:return!0;case 5:return u;case 6:return b;case 2:De.call(x,u)}else if(o)return!1;return s?-1:n||o?o:x}},Ce={forEach:Ie(0),map:Ie(1),filter:Ie(2),some:Ie(3),every:Ie(4),find:Ie(5),findIndex:Ie(6)},Le=Ce.forEach,Ge=Bt("hidden"),Me=Pt("toPrimitive"),Se=we.set,Qe=we.getterFor("Symbol"),Ne=Object.prototype,Fe=a.Symbol,Ze=pt("JSON","stringify"),Be=M.f,Oe=E.f,We=Ut.f,Ae=c.f,Xe=St("symbols"),Te=St("op-symbols"),Ye=St("string-to-symbol-registry"),Ee=St("symbol-to-string-registry"),Re=St("wks"),Ue=a.QObject,ze=!Ue||!Ue.prototype||!Ue.prototype.findChild,Ve=h&&l((function(){return 7!=Tt(Oe({},"a",{get:function(){return Oe(this,"a",{value:7}).a}})).a}))?function(t,e,i){var n=Be(Ne,e);n&&delete Ne[e],Oe(t,e,i),n&&t!==Ne&&Oe(Ne,e,n)}:Oe,je=function(t,e){var i=Xe[t]=Tt(Fe.prototype);return Se(i,{type:"Symbol",tag:t,description:e}),h||(i.description=e),i},He=It?function(t){return"symbol"==typeof t}:function(t){return Object(t)instanceof Fe},Pe=function(t,e,i){t===Ne&&Pe(Te,e,i),T(t);var n=w(e,!0);return T(i),k(Xe,n)?(i.enumerable?(k(t,Ge)&&t[Ge][n]&&(t[Ge][n]=!1),i=Tt(i,{enumerable:p(0,!1)})):(k(t,Ge)||Oe(t,Ge,p(1,{})),t[Ge][n]=!0),Ve(t,n,i)):Oe(t,n,i)},Je=function(t,e){T(t);var i=y(e),n=ht(i).concat(ti(i));return Le(n,(function(e){h&&!Ke.call(i,e)||Pe(t,e,i[e])})),t},Ke=function(t){var e=w(t,!0),i=Ae.call(this,e);return!(this===Ne&&k(Xe,e)&&!k(Te,e))&&(!(i||!k(this,e)||!k(Xe,e)||k(this,Ge)&&this[Ge][e])||i)},qe=function(t,e){var i=y(t),n=w(e,!0);if(i!==Ne||!k(Xe,n)||k(Te,n)){var o=Be(i,n);return!o||!k(Xe,n)||k(i,Ge)&&i[Ge][n]||(o.enumerable=!0),o}},$e=function(t){var e=We(y(t)),i=[];return Le(e,(function(t){k(Xe,t)||k(st,t)||i.push(t)})),i},ti=function(t){var e=t===Ne,i=We(e?Te:y(t)),n=[];return Le(i,(function(t){!k(Xe,t)||e&&!k(Ne,t)||n.push(Xe[t])})),n};if(Dt||(zt((Fe=function(){if(this instanceof Fe)throw TypeError("Symbol is not a constructor");var t=arguments.length&&void 0!==arguments[0]?String(arguments[0]):void 0,e=Ft(t),i=function(t){this===Ne&&i.call(Te,t),k(this,Ge)&&k(this[Ge],e)&&(this[Ge][e]=!1),Ve(this,e,p(1,t))};return h&&ze&&Ve(Ne,e,{configurable:!0,set:i}),je(e,t)}).prototype,"toString",(function(){return Qe(this).tag})),zt(Fe,"withoutSetter",(function(t){return je(Ft(t),t)})),c.f=Ke,E.f=Pe,M.f=qe,ft.f=Ut.f=$e,gt.f=ti,Jt.f=function(t){return je(Pt(t),t)},h&&Oe(Fe.prototype,"description",{configurable:!0,get:function(){return Qe(this).description}})),V({global:!0,wrap:!0,forced:!Dt,sham:!Dt},{Symbol:Fe}),Le(ht(Re),(function(t){qt(t)})),V({target:"Symbol",stat:!0,forced:!Dt},{for:function(t){var e=String(t);if(k(Ye,e))return Ye[e];var i=Fe(e);return Ye[e]=i,Ee[i]=e,i},keyFor:function(t){if(!He(t))throw TypeError(t+" is not a symbol");if(k(Ee,t))return Ee[t]},useSetter:function(){ze=!0},useSimple:function(){ze=!1}}),V({target:"Object",stat:!0,forced:!Dt,sham:!h},{create:function(t,e){return void 0===e?Tt(t):Je(Tt(t),e)},defineProperty:Pe,defineProperties:Je,getOwnPropertyDescriptor:qe}),V({target:"Object",stat:!0,forced:!Dt},{getOwnPropertyNames:$e,getOwnPropertySymbols:ti}),V({target:"Object",stat:!0,forced:l((function(){gt.f(1)}))},{getOwnPropertySymbols:function(t){return gt.f(Lt(t))}}),Ze){var ei=!Dt||l((function(){var t=Fe();return"[null]"!=Ze([t])||"{}"!=Ze({a:t})||"{}"!=Ze(Object(t))}));V({target:"JSON",stat:!0,forced:ei},{stringify:function(t,e,i){for(var n,o=[t],s=1;arguments.length>s;)o.push(arguments[s++]);if(n=e,(x(e)||void 0!==t)&&!He(t))return Ct(e)||(e=function(t,e){if("function"==typeof n&&(e=n.call(this,t,e)),!He(e))return e}),o[1]=e,Ze.apply(null,o)}})}Fe.prototype[Me]||R(Fe.prototype,Me,Fe.prototype.valueOf),ae(Fe,"Symbol"),st[Ge]=!0;var ii,ni,oi,si=W.Object.getOwnPropertySymbols,ri={},ai=!l((function(){function t(){}return t.prototype.constructor=null,Object.getPrototypeOf(new t)!==t.prototype})),li=Bt("IE_PROTO"),hi=Object.prototype,di=ai?Object.getPrototypeOf:function(t){return t=Lt(t),k(t,li)?t[li]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?hi:null},ui=(Pt("iterator"),!1);[].keys&&("next"in(oi=[].keys())?(ni=di(di(oi)))!==Object.prototype&&(ii=ni):ui=!0),null==ii&&(ii={});var ci={IteratorPrototype:ii,BUGGY_SAFARI_ITERATORS:ui},pi=ci.IteratorPrototype,mi=function(){return this},fi=Object.setPrototypeOf||("__proto__"in{}?function(){var t,e=!1,i={};try{(t=Object.getOwnPropertyDescriptor(Object.prototype,"__proto__").set).call(i,[]),e=i instanceof Array}catch(t){}return function(i,n){return T(i),function(t){if(!x(t)&&null!==t)throw TypeError("Can't set "+String(t)+" as a prototype")}(n),e?t.call(i,n):i.__proto__=n,i}}():void 0),gi=ci.IteratorPrototype,vi=ci.BUGGY_SAFARI_ITERATORS,bi=Pt("iterator"),yi=function(){return this},xi=function(t,e,i,n,o,s,r){!function(t,e,i){var n=e+" Iterator";t.prototype=Tt(pi,{next:p(1,i)}),ae(t,n,!1,!0),ri[n]=mi}(i,e,n);var a,l,h,d=function(t){if(t===o&&g)return g;if(!vi&&t in m)return m[t];switch(t){case"keys":case"values":case"entries":return function(){return new i(this,t)}}return function(){return new i(this)}},u=e+" Iterator",c=!1,m=t.prototype,f=m[bi]||m["@@iterator"]||o&&m[o],g=!vi&&f||d(o),v="Array"==e&&m.entries||f;if(v&&(a=di(v.call(new t)),gi!==Object.prototype&&a.next&&(ae(a,u,!0,!0),ri[u]=yi)),"values"==o&&f&&"values"!==f.name&&(c=!0,g=function(){return f.call(this)}),r&&m[bi]!==g&&R(m,bi,g),ri[e]=g,o)if(l={values:d("values"),keys:s?g:d("keys"),entries:d("entries")},r)for(h in l)!vi&&!c&&h in m||zt(m,h,l[h]);else V({target:e,proto:!0,forced:vi||c},l);return l},wi=we.set,_i=we.getterFor("Array Iterator");xi(Array,"Array",(function(t,e){wi(this,{type:"Array Iterator",target:y(t),index:0,kind:e})}),(function(){var t=_i(this),e=t.target,i=t.kind,n=t.index++;return!e||n>=e.length?(t.target=void 0,{value:void 0,done:!0}):"keys"==i?{value:n,done:!1}:"values"==i?{value:e[n],done:!1}:{value:[n,e[n]],done:!1}}),"values");ri.Arguments=ri.Array;var ki=Pt("toStringTag");for(var Di in{CSSRuleList:0,CSSStyleDeclaration:0,CSSValueList:0,ClientRectList:0,DOMRectList:0,DOMStringList:0,DOMTokenList:1,DataTransferItemList:0,FileList:0,HTMLAllCollection:0,HTMLCollection:0,HTMLFormElement:0,HTMLSelectElement:0,MediaList:0,MimeTypeArray:0,NamedNodeMap:0,NodeList:1,PaintRequestList:0,Plugin:0,PluginArray:0,SVGLengthList:0,SVGNumberList:0,SVGPathSegList:0,SVGPointList:0,SVGStringList:0,SVGTransformList:0,SourceBufferList:0,StyleSheetList:0,TextTrackCueList:0,TextTrackList:0,TouchList:0}){var Ii=a[Di],Ci=Ii&&Ii.prototype;Ci&&ne(Ci)!==ki&&R(Ci,ki,Di),ri[Di]=ri.Array}var Li=function(t){return function(e,i){var n,o,s=String(b(e)),r=K(i),a=s.length;return r<0||r>=a?t?"":void 0:(n=s.charCodeAt(r))<55296||n>56319||r+1===a||(o=s.charCodeAt(r+1))<56320||o>57343?t?s.charAt(r):n:t?s.slice(r,r+2):o-56320+(n-55296<<10)+65536}},Gi={codeAt:Li(!1),charAt:Li(!0)}.charAt,Mi=we.set,Si=we.getterFor("String Iterator");xi(String,"String",(function(t){Mi(this,{type:"String Iterator",string:String(t),index:0})}),(function(){var t,e=Si(this),i=e.string,n=e.index;return n>=i.length?{value:void 0,done:!0}:(t=Gi(i,n),e.index+=t.length,{value:t,done:!1})}));var Qi=Pt("iterator"),Ni=function(t){if(null!=t)return t[Qi]||t["@@iterator"]||ri[ne(t)]},Fi=function(t){var e=Ni(t);if("function"!=typeof e)throw TypeError(String(t)+" is not iterable");return T(e.call(t))};V({target:"Object",stat:!0,sham:!h},{create:Tt});var Zi=W.Object,Bi=function(t,e){return Zi.create(t,e)},Oi=Bi,Wi=j;var Ai=function(t,e,i){return e in t?Wi(t,e,{value:i,enumerable:!0,configurable:!0,writable:!0}):t[e]=i,t},Xi=l((function(){ht(1)}));V({target:"Object",stat:!0,forced:Xi},{keys:function(t){return ht(Lt(t))}});var Ti,Yi=W.Object.keys,Ei="\t\n\v\f\r                　\u2028\u2029\ufeff",Ri="["+Ei+"]",Ui=RegExp("^"+Ri+Ri+"*"),zi=RegExp(Ri+Ri+"*$"),Vi=function(t){return function(e){var i=String(b(e));return 1&t&&(i=i.replace(Ui,"")),2&t&&(i=i.replace(zi,"")),i}},ji={start:Vi(1),end:Vi(2),trim:Vi(3)},Hi=ji.trim;V({target:"String",proto:!0,forced:(Ti="trim",l((function(){return!!Ei[Ti]()||"​…᠎"!="​…᠎"[Ti]()||Ei[Ti].name!==Ti})))},{trim:function(){return Hi(this)}});var Pi=function(t){return W[t+"Prototype"]},Ji=Pi("String").trim,Ki=String.prototype,qi=function(t){var e=t.trim;return"string"==typeof t||t===Ki||t instanceof String&&e===Ki.trim?Ji:e},$i=function(t,e){var i=[][t];return!!i&&l((function(){i.call(null,e||function(){throw 1},1)}))},tn=Object.defineProperty,en={},nn=function(t){throw t},on=function(t,e){if(k(en,t))return en[t];e||(e={});var i=[][t],n=!!k(e,"ACCESSORS")&&e.ACCESSORS,o=k(e,0)?e[0]:nn,s=k(e,1)?e[1]:void 0;return en[t]=!!i&&!l((function(){if(n&&!h)return!0;var t={length:-1};n?tn(t,1,{enumerable:!0,get:nn}):t[1]=1,i.call(t,o,s)}))},sn=Ce.forEach,rn=$i("forEach"),an=on("forEach"),ln=rn&&an?[].forEach:function(t){return sn(this,t,arguments.length>1?arguments[1]:void 0)};V({target:"Array",proto:!0,forced:[].forEach!=ln},{forEach:ln});var hn,dn,un=Pi("Array").forEach,cn=Array.prototype,pn={DOMTokenList:!0,NodeList:!0},mn=function(t){var e=t.forEach;return t===cn||t instanceof Array&&e===cn.forEach||pn.hasOwnProperty(ne(t))?un:e},fn=pt("navigator","userAgent")||"",gn=a.process,vn=gn&&gn.versions,bn=vn&&vn.v8;bn?dn=(hn=bn.split("."))[0]+hn[1]:fn&&(!(hn=fn.match(/Edge\/(\d+)/))||hn[1]>=74)&&(hn=fn.match(/Chrome\/(\d+)/))&&(dn=hn[1]);var yn=dn&&+dn,xn=Pt("species"),wn=function(t){return yn>=51||!l((function(){var e=[];return(e.constructor={})[xn]=function(){return{foo:1}},1!==e[t](Boolean).foo}))},_n=Ce.map,kn=wn("map"),Dn=on("map");V({target:"Array",proto:!0,forced:!kn||!Dn},{map:function(t){return _n(this,t,arguments.length>1?arguments[1]:void 0)}});var In=Pi("Array").map,Cn=Array.prototype,Ln=function(t){var e=t.map;return t===Cn||t instanceof Array&&e===Cn.map?In:e},Gn=ji.trim,Mn=a.parseInt,Sn=/^[+-]?0[Xx]/,Qn=8!==Mn(Ei+"08")||22!==Mn(Ei+"0x16")?function(t,e){var i=Gn(String(t));return Mn(i,e>>>0||(Sn.test(i)?16:10))}:Mn;V({global:!0,forced:parseInt!=Qn},{parseInt:Qn});var Nn=W.parseInt,Fn=c.f,Zn=function(t){return function(e){for(var i,n=y(e),o=ht(n),s=o.length,r=0,a=[];s>r;)i=o[r++],h&&!Fn.call(n,i)||a.push(t?[i,n[i]]:n[i]);return a}},Bn={entries:Zn(!0),values:Zn(!1)}.values;V({target:"Object",stat:!0},{values:function(t){return Bn(t)}});var On=W.Object.values,Wn=Ce.filter,An=wn("filter"),Xn=on("filter");V({target:"Array",proto:!0,forced:!An||!Xn},{filter:function(t){return Wn(this,t,arguments.length>1?arguments[1]:void 0)}});var Tn=Pi("Array").filter,Yn=Array.prototype,En=function(t){var e=t.filter;return t===Yn||t instanceof Array&&e===Yn.filter?Tn:e},Rn=Pt("isConcatSpreadable"),Un=yn>=51||!l((function(){var t=[];return t[Rn]=!1,t.concat()[0]!==t})),zn=wn("concat"),Vn=function(t){if(!x(t))return!1;var e=t[Rn];return void 0!==e?!!e:Ct(t)};V({target:"Array",proto:!0,forced:!Un||!zn},{concat:function(t){var e,i,n,o,s,r=Lt(this),a=ke(r,0),l=0;for(e=-1,n=arguments.length;e<n;e++)if(s=-1===e?r:arguments[e],Vn(s)){if(l+(o=$(s.length))>9007199254740991)throw TypeError("Maximum allowed index exceeded");for(i=0;i<o;i++,l++)i in s&&bt(a,l,s[i])}else{if(l>=9007199254740991)throw TypeError("Maximum allowed index exceeded");bt(a,l++,s)}return a.length=l,a}});var jn=Pi("Array").concat,Hn=Array.prototype,Pn=function(t){var e=t.concat;return t===Hn||t instanceof Array&&e===Hn.concat?jn:e};V({target:"Array",stat:!0},{isArray:Ct});var Jn=W.Array.isArray,Kn=Jn;var qn=function(t){if(Kn(t)){for(var e=0,i=new Array(t.length);e<t.length;e++)i[e]=t[e];return i}},$n=function(t,e,i,n){try{return n?e(T(i)[0],i[1]):e(i)}catch(e){var o=t.return;throw void 0!==o&&T(o.call(t)),e}},to=Pt("iterator"),eo=Array.prototype,io=function(t){return void 0!==t&&(ri.Array===t||eo[to]===t)},no=Pt("iterator"),oo=!1;try{var so=0,ro={next:function(){return{done:!!so++}},return:function(){oo=!0}};ro[no]=function(){return this},Array.from(ro,(function(){throw 2}))}catch(t){}var ao=!function(t,e){if(!e&&!oo)return!1;var i=!1;try{var n={};n[no]=function(){return{next:function(){return{done:i=!0}}}},t(n)}catch(t){}return i}((function(t){Array.from(t)}));V({target:"Array",stat:!0,forced:ao},{from:function(t){var e,i,n,o,s,r,a=Lt(t),l="function"==typeof this?this:Array,h=arguments.length,d=h>1?arguments[1]:void 0,u=void 0!==d,c=Ni(a),p=0;if(u&&(d=X(d,h>2?arguments[2]:void 0,2)),null==c||l==Array&&io(c))for(i=new l(e=$(a.length));e>p;p++)r=u?d(a[p],p):a[p],bt(i,p,r);else for(s=(o=c.call(a)).next,i=new l;!(n=s.call(o)).done;p++)r=u?$n(o,d,[n.value,p],!0):n.value,bt(i,p,r);return i.length=p,i}});var lo=W.Array.from,ho=Pt("iterator"),uo=function(t){var e=Object(t);return void 0!==e[ho]||"@@iterator"in e||ri.hasOwnProperty(ne(e))};var co=function(t){if(uo(Object(t))||"[object Arguments]"===Object.prototype.toString.call(t))return lo(t)};var po=function(){throw new TypeError("Invalid attempt to spread non-iterable instance")};var mo=function(t){return qn(t)||co(t)||po()},fo=wn("slice"),go=on("slice",{ACCESSORS:!0,0:0,1:2}),vo=Pt("species"),bo=[].slice,yo=Math.max;V({target:"Array",proto:!0,forced:!fo||!go},{slice:function(t,e){var i,n,o,s=y(this),r=$(s.length),a=it(t,r),l=it(void 0===e?r:e,r);if(Ct(s)&&("function"!=typeof(i=s.constructor)||i!==Array&&!Ct(i.prototype)?x(i)&&null===(i=i[vo])&&(i=void 0):i=void 0,i===Array||void 0===i))return bo.call(s,a,l);for(n=new(void 0===i?Array:i)(yo(l-a,0)),o=0;a<l;a++,o++)a in s&&bt(n,o,s[a]);return n.length=o,n}});var xo=Pi("Array").slice,wo=Array.prototype,_o=function(t){var e=t.slice;return t===wo||t instanceof Array&&e===wo.slice?xo:e},ko=l((function(){di(1)}));V({target:"Object",stat:!0,forced:ko,sham:!ai},{getPrototypeOf:function(t){return di(Lt(t))}});var Do=W.Object.getPrototypeOf,Io=Do,Co=ot.indexOf,Lo=[].indexOf,Go=!!Lo&&1/[1].indexOf(1,-0)<0,Mo=$i("indexOf"),So=on("indexOf",{ACCESSORS:!0,1:0});V({target:"Array",proto:!0,forced:Go||!Mo||!So},{indexOf:function(t){return Go?Lo.apply(this,arguments)||0:Co(this,t,arguments.length>1?arguments[1]:void 0)}});var Qo=Pi("Array").indexOf,No=Array.prototype,Fo=function(t){var e=t.indexOf;return t===No||t instanceof Array&&e===No.indexOf?Qo:e},Zo=Jn,Bo=Object.assign,Oo=Object.defineProperty,Wo=!Bo||l((function(){if(h&&1!==Bo({b:1},Bo(Oo({},"a",{enumerable:!0,get:function(){Oo(this,"b",{value:3,enumerable:!1})}}),{b:2})).b)return!0;var t={},e={},i=Symbol();return t[i]=7,"abcdefghijklmnopqrst".split("").forEach((function(t){e[t]=t})),7!=Bo({},t)[i]||"abcdefghijklmnopqrst"!=ht(Bo({},e)).join("")}))?function(t,e){for(var i=Lt(t),n=arguments.length,o=1,s=gt.f,r=c.f;n>o;)for(var a,l=v(arguments[o++]),d=s?ht(l).concat(s(l)):ht(l),u=d.length,p=0;u>p;)a=d[p++],h&&!r.call(l,a)||(i[a]=l[a]);return i}:Bo;V({target:"Object",stat:!0,forced:Object.assign!==Wo},{assign:Wo});var Ao=W.Object.assign;qt("iterator");var Xo=Jt.f("iterator"),To=Xo,Yo=Object.freeze({__proto__:null});qt("asyncIterator");var Eo=Object.freeze({__proto__:null});qt("hasInstance"),qt("isConcatSpreadable"),qt("match"),qt("matchAll"),qt("replace"),qt("search"),qt("species"),qt("split"),qt("toPrimitive"),qt("toStringTag"),qt("unscopables"),ae(Math,"Math",!0),ae(a.JSON,"JSON",!0),o(Yo),o(Eo);var Ro=W.Symbol;qt("asyncDispose"),qt("dispose"),qt("observable"),qt("patternMatch"),qt("replaceAll");var Uo=Ro,zo=n((function(t){function e(i){return t.exports=e="function"==typeof Uo&&"symbol"==typeof To?function(t){return typeof t}:function(t){return t&&"function"==typeof Uo&&t.constructor===Uo&&t!==Uo.prototype?"symbol":typeof t},e(i)}t.exports=e}));var Vo=function(t){if(Kn(t))return t};var jo=function(t,e){if(uo(Object(t))||"[object Arguments]"===Object.prototype.toString.call(t)){var i=[],n=!0,o=!1,s=void 0;try{for(var r,a=Fi(t);!(n=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);n=!0);}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return i}};var Ho=function(){throw new TypeError("Invalid attempt to destructure non-iterable instance")};var Po=function(t,e){return Vo(t)||jo(t,e)||Ho()};V({target:"Date",stat:!0},{now:function(){return(new Date).getTime()}});var Jo=W.Date.now;function Ko(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}function qo(t){var e=function(){for(var t=$o(),e=t(" "),i=t(" "),n=t(" "),o=0;o<arguments.length;o++)(e-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(e+=1),(i-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(i+=1),(n-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(n+=1);return[e,i,n]}(t),i=Po(e,3),n=i[0],o=i[1],s=i[2],r=1,a=function(){var t=2091639*n+2.3283064365386963e-10*r;return n=o,o=s,s=t-(r=0|t)};return a.uint32=function(){return 4294967296*a()},a.fract53=function(){return a()+11102230246251565e-32*(2097152*a()|0)},a.algorithm="Alea",a.seed=t,a.version="0.9",a}function $o(){var t=4022871197;return function(e){for(var i=e.toString(),n=0;n<i.length;n++){var o=.02519603282416938*(t+=i.charCodeAt(n));o-=t=o>>>0,t=(o*=t)>>>0,t+=4294967296*(o-=t)}return 2.3283064365386963e-10*(t>>>0)}}var ts=/^\/?Date\((-?\d+)/i,es=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i,is=/^#?([a-f\d])([a-f\d])([a-f\d])$/i,ns=/^rgb\( *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *\)$/i,os=/^rgba\( *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *([01]|0?\.\d+) *\)$/i;function ss(t){return t instanceof Number||"number"==typeof t}function rs(t){return t instanceof String||"string"==typeof t}function as(t){return"object"===zo(t)&&null!==t}function ls(t,e,i,n){var o=!1;!0===n&&(o=null===e[i]&&void 0!==t[i]),o?delete t[i]:t[i]=e[i]}var hs=Ao;function ds(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];for(var o in e)if(Object.prototype.hasOwnProperty.call(e,o)||!0===i)if("object"===zo(e[o])&&null!==e[o]&&Io(e[o])===Object.prototype)void 0===t[o]?t[o]=ds({},e[o],i):"object"===zo(t[o])&&null!==t[o]&&Io(t[o])===Object.prototype?ds(t[o],e[o],i):ls(t,e,o,n);else if(Zo(e[o])){var s;t[o]=_o(s=e[o]).call(s)}else ls(t,e,o,n);return t}function us(t){var e=zo(t);return"object"===e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Zo(t)?"Array":t instanceof Date?"Date":"Object":"number"===e?"Number":"boolean"===e?"Boolean":"string"===e?"String":void 0===e?"undefined":e}var cs=On;var ps={asBoolean:function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},asNumber:function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},asString:function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},asSize:function(t,e){return"function"==typeof t&&(t=t()),rs(t)?t:ss(t)?t+"px":e||null},asElement:function(t,e){return"function"==typeof t&&(t=t()),t||e||null}};function ms(t){var e;switch(t.length){case 3:case 4:return(e=is.exec(t))?{r:Nn(e[1]+e[1],16),g:Nn(e[2]+e[2],16),b:Nn(e[3]+e[3],16)}:null;case 6:case 7:return(e=es.exec(t))?{r:Nn(e[1],16),g:Nn(e[2],16),b:Nn(e[3],16)}:null;default:return null}}function fs(t,e,i){var n;return"#"+_o(n=((1<<24)+(t<<16)+(e<<8)+i).toString(16)).call(n,1)}function gs(t,e,i){t/=255,e/=255,i/=255;var n=Math.min(t,Math.min(e,i)),o=Math.max(t,Math.max(e,i));return n===o?{h:0,s:0,v:n}:{h:60*((t===n?3:i===n?1:5)-(t===n?e-i:i===n?t-e:i-t)/(o-n))/360,s:(o-n)/o,v:o}}var vs=function(t){var e,i={};return mn(e=t.split(";")).call(e,(function(t){if(""!=qi(t).call(t)){var e,n,o=t.split(":"),s=qi(e=o[0]).call(e),r=qi(n=o[1]).call(n);i[s]=r}})),i},bs=function(t){var e;return Ln(e=Yi(t)).call(e,(function(e){return e+": "+t[e]})).join("; ")};function ys(t,e,i){var n,o,s,r=Math.floor(6*t),a=6*t-r,l=i*(1-e),h=i*(1-a*e),d=i*(1-(1-a)*e);switch(r%6){case 0:n=i,o=d,s=l;break;case 1:n=h,o=i,s=l;break;case 2:n=l,o=i,s=d;break;case 3:n=l,o=h,s=i;break;case 4:n=d,o=l,s=i;break;case 5:n=i,o=l,s=h}return{r:Math.floor(255*n),g:Math.floor(255*o),b:Math.floor(255*s)}}function xs(t,e,i){var n=ys(t,e,i);return fs(n.r,n.g,n.b)}function ws(t){var e=ms(t);if(!e)throw new TypeError("'".concat(t,"' is not a valid color."));return gs(e.r,e.g,e.b)}function _s(t){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t)}function ks(t){return ns.test(t)}function Ds(t){if(null===t||"object"!==zo(t))return null;if(t instanceof Element)return t;var e=Oi(t);for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&"object"==zo(t[i])&&(e[i]=Ds(t[i]));return e}var Is=Object.freeze({__proto__:null,Alea:
+/**
+	 * vis-util
+	 * https://github.com/visjs/vis-util
+	 *
+	 * utilitie collection for visjs
+	 *
+	 * @version 4.0.0
+	 * @date    2020-02-29T15:35:13.178Z
+	 *
+	 * @copyright (c) 2011-2017 Almende B.V, http://almende.com
+	 * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+	 *
+	 * @license
+	 * vis.js is dual licensed under both
+	 *
+	 *   1. The Apache 2.0 License
+	 *      http://www.apache.org/licenses/LICENSE-2.0
+	 *
+	 *   and
+	 *
+	 *   2. The MIT License
+	 *      http://opensource.org/licenses/MIT
+	 *
+	 * vis.js may be distributed under either license.
+	 */
+function(){for(var t=arguments.length,e=new Array(t),i=0;i<t;i++)e[i]=arguments[i];return qo(e.length?e:[Jo()])},HSVToHex:xs,HSVToRGB:ys,RGBToHSV:gs,RGBToHex:fs,addClassName:function(t,e){var i=t.className.split(" "),n=e.split(" ");i=Pn(i).call(i,En(n).call(n,(function(t){return Fo(i).call(i,t)<0}))),t.className=i.join(" ")},addCssText:function(t,e){var i=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=Ko(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=Ko(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},vs(t.style.cssText),{},vs(e));t.style.cssText=bs(i)},addEventListener:function(t,e,i,n){var o;t.addEventListener?(void 0===n&&(n=!1),"mousewheel"===e&&Fo(o=navigator.userAgent).call(o,"Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,n)):t.attachEvent("on"+e,i)},binarySearchCustom:function(t,e,i,n){for(var o=0,s=0,r=t.length-1;s<=r&&o<1e4;){var a=Math.floor((s+r)/2),l=t[a],h=e(void 0===n?l[i]:l[i][n]);if(0==h)return a;-1==h?s=a+1:r=a-1,o++}return-1},binarySearchValue:function(t,e,i,n,o){var s,r,a,l,h=0,d=0,u=t.length-1;for(o=null!=o?o:function(t,e){return t==e?0:t<e?-1:1};d<=u&&h<1e4;){if(l=Math.floor(.5*(u+d)),s=t[Math.max(0,l-1)][i],r=t[l][i],a=t[Math.min(t.length-1,l+1)][i],0==o(r,e))return l;if(o(s,e)<0&&o(r,e)>0)return"before"==n?Math.max(0,l-1):l;if(o(r,e)<0&&o(a,e)>0)return"before"==n?l:Math.min(t.length-1,l+1);o(r,e)<0?d=l+1:u=l-1,h++}return-1},bridgeObject:Ds,copyAndExtendArray:function(t,e){var i;return Pn(i=[]).call(i,mo(t),[e])},copyArray:function(t){return _o(t).call(t)},deepExtend:ds,easingFunctions:{linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return t<.5?2*t*t:(4-2*t)*t-1},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return t<.5?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return t<.5?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return t<.5?16*t*t*t*t*t:1+16*--t*t*t*t*t}},equalArray:function(t,e){if(t.length!==e.length)return!1;for(var i=0,n=t.length;i<n;i++)if(t[i]!=e[i])return!1;return!0},extend:hs,fillIfDefined:function t(e,i){var n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];for(var o in e)if(void 0!==i[o])if(null===i[o]||"object"!==zo(i[o]))ls(e,i,o,n);else{var s=e[o],r=i[o];as(s)&&as(r)&&t(s,r,n)}},forEach:function(t,e){if(Zo(t))for(var i=t.length,n=0;n<i;n++)e(t[n],n,t);else for(var o in t)Object.prototype.hasOwnProperty.call(t,o)&&e(t[o],o,t)},getAbsoluteLeft:function(t){return t.getBoundingClientRect().left},getAbsoluteRight:function(t){return t.getBoundingClientRect().right},getAbsoluteTop:function(t){return t.getBoundingClientRect().top},getScrollBarWidth:function(){var t=document.createElement("p");t.style.width="100%",t.style.height="200px";var e=document.createElement("div");e.style.position="absolute",e.style.top="0px",e.style.left="0px",e.style.visibility="hidden",e.style.width="200px",e.style.height="150px",e.style.overflow="hidden",e.appendChild(t),document.body.appendChild(e);var i=t.offsetWidth;e.style.overflow="scroll";var n=t.offsetWidth;return i==n&&(n=e.clientWidth),document.body.removeChild(e),i-n},getTarget:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.event,e=null;return t&&(t.target?e=t.target:t.srcElement&&(e=t.srcElement)),e instanceof Element&&(null==e.nodeType||3!=e.nodeType||(e=e.parentNode)instanceof Element)?e:null},getType:us,hasParent:function(t,e){for(var i=t;i;){if(i===e)return!0;if(!i.parentNode)return!1;i=i.parentNode}return!1},hexToHSV:ws,hexToRGB:ms,insertSort:function(t,e){for(var i=0;i<t.length;i++){var n=t[i],o=void 0;for(o=i;o>0&&e(n,t[o-1])<0;o--)t[o]=t[o-1];t[o]=n}return t},isDate:function(t){if(t instanceof Date)return!0;if(rs(t)){if(ts.exec(t))return!0;if(!isNaN(Date.parse(t)))return!0}return!1},isNumber:ss,isObject:as,isString:rs,isValidHex:_s,isValidRGB:ks,isValidRGBA:function(t){return os.test(t)},mergeOptions:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},o=function(t){return null!=t},s=function(t){return null!==t&&"object"===zo(t)},r=function(t){for(var e in t)if(Object.prototype.hasOwnProperty.call(t,e))return!1;return!0};if(!s(t))throw new Error("Parameter mergeTarget must be an object");if(!s(e))throw new Error("Parameter options must be an object");if(!o(i))throw new Error("Parameter option must have a value");if(!s(n))throw new Error("Parameter globalOptions must be an object");var a=function(t,e,i){s(t[i])||(t[i]={});var n=e[i],o=t[i];for(var r in n)Object.prototype.hasOwnProperty.call(n,r)&&(o[r]=n[r])},l=e[i],h=s(n)&&!r(n),d=h?n[i]:void 0,u=d?d.enabled:void 0;if(void 0!==l){if("boolean"==typeof l)return s(t[i])||(t[i]={}),void(t[i].enabled=l);if(null===l&&!s(t[i])){if(!o(d))return;t[i]=Oi(d)}if(s(l)){var c=!0;void 0!==l.enabled?c=l.enabled:void 0!==u&&(c=d.enabled),a(t,e,i),t[i].enabled=c}}},option:ps,overrideOpacity:function(t,e){if(-1!==Fo(t).call(t,"rgba"))return t;if(-1!==Fo(t).call(t,"rgb")){var i=t.substr(Fo(t).call(t,"(")+1).replace(")","").split(",");return"rgba("+i[0]+","+i[1]+","+i[2]+","+e+")"}var n=ms(t);return null==n?t:"rgba("+n.r+","+n.g+","+n.b+","+e+")"},parseColor:function(t,e){if(rs(t)){var i=t;if(ks(i)){var n,o=Ln(n=i.substr(4).substr(0,i.length-5).split(",")).call(n,(function(t){return Nn(t)}));i=fs(o[0],o[1],o[2])}if(!0===_s(i)){var s=ws(i),r={h:s.h,s:.8*s.s,v:Math.min(1,1.02*s.v)},a={h:s.h,s:Math.min(1,1.25*s.s),v:.8*s.v},l=xs(a.h,a.s,a.v),h=xs(r.h,r.s,r.v);return{background:i,border:l,highlight:{background:h,border:l},hover:{background:h,border:l}}}return{background:i,border:i,highlight:{background:i,border:i},hover:{background:i,border:i}}}return e?{background:t.background||e.background,border:t.border||e.border,highlight:rs(t.highlight)?{border:t.highlight,background:t.highlight}:{background:t.highlight&&t.highlight.background||e.highlight.background,border:t.highlight&&t.highlight.border||e.highlight.border},hover:rs(t.hover)?{border:t.hover,background:t.hover}:{border:t.hover&&t.hover.border||e.hover.border,background:t.hover&&t.hover.background||e.hover.background}}:{background:t.background||void 0,border:t.border||void 0,highlight:rs(t.highlight)?{border:t.highlight,background:t.highlight}:{background:t.highlight&&t.highlight.background||void 0,border:t.highlight&&t.highlight.border||void 0},hover:rs(t.hover)?{border:t.hover,background:t.hover}:{border:t.hover&&t.hover.border||void 0,background:t.hover&&t.hover.background||void 0}}},preventDefault:function(t){t||(t=window.event),t&&(t.preventDefault?t.preventDefault():t.returnValue=!1)},recursiveDOMDelete:function t(e){if(e)for(;!0===e.hasChildNodes();){var i=e.firstChild;i&&(t(i),e.removeChild(i))}},removeClassName:function(t,e){var i=t.className.split(" "),n=e.split(" ");i=En(i).call(i,(function(t){return Fo(n).call(n,t)<0})),t.className=i.join(" ")},removeCssText:function(t,e){var i=vs(t.style.cssText),n=vs(e);for(var o in n)Object.prototype.hasOwnProperty.call(n,o)&&delete i[o];t.style.cssText=bs(i)},removeEventListener:function(t,e,i,n){var o;t.removeEventListener?(void 0===n&&(n=!1),"mousewheel"===e&&Fo(o=navigator.userAgent).call(o,"Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,n)):t.detachEvent("on"+e,i)},selectiveBridgeObject:function(t,e){if(null!==e&&"object"===zo(e)){for(var i=Oi(e),n=0;n<t.length;n++)Object.prototype.hasOwnProperty.call(e,t[n])&&"object"==zo(e[t[n]])&&(i[t[n]]=Ds(e[t[n]]));return i}return null},selectiveDeepExtend:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Zo(i))throw new TypeError("Arrays are not supported by deepExtend");for(var o=0;o<t.length;o++){var s=t[o];if(Object.prototype.hasOwnProperty.call(i,s))if(i[s]&&i[s].constructor===Object)void 0===e[s]&&(e[s]={}),e[s].constructor===Object?ds(e[s],i[s],!1,n):ls(e,i,s,n);else{if(Zo(i[s]))throw new TypeError("Arrays are not supported by deepExtend");ls(e,i,s,n)}}return e},selectiveExtend:function(t,e){if(!Zo(t))throw new Error("Array with property names expected as first argument");for(var i=arguments.length,n=new Array(i>2?i-2:0),o=2;o<i;o++)n[o-2]=arguments[o];for(var s=0,r=n;s<r.length;s++)for(var a=r[s],l=0;l<t.length;l++){var h=t[l];a&&Object.prototype.hasOwnProperty.call(a,h)&&(e[h]=a[h])}return e},selectiveNotDeepExtend:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Zo(i))throw new TypeError("Arrays are not supported by deepExtend");for(var o in i)if(Object.prototype.hasOwnProperty.call(i,o)&&-1===Fo(t).call(t,o))if(i[o]&&i[o].constructor===Object)void 0===e[o]&&(e[o]={}),e[o].constructor===Object?ds(e[o],i[o]):ls(e,i,o,n);else if(Zo(i[o])){e[o]=[];for(var s=0;s<i[o].length;s++)e[o].push(i[o][s])}else ls(e,i,o,n);return e},throttle:function(t){var e=!1;return function(){e||(e=!0,requestAnimationFrame((function(){e=!1,t()})))}},toArray:cs,topMost:function(t,e){var i;Zo(e)||(e=[e]);var n=!0,o=!1,s=void 0;try{for(var r,a=Fi(t);!(n=(r=a.next()).done);n=!0){var l=r.value;if(l){i=l[e[0]];for(var h=1;h<e.length;h++)i&&(i=i[e[h]]);if(void 0!==i)break}}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return i},updateProperty:function(t,e,i){return t[e]!==i&&(t[e]=i,!0)}});function Cs(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])}function Ls(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}}function Gs(t){Cs(t),Ls(t),Cs(t)}function Ms(t,e,i){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(n)):(n=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(n)),e[t].used.push(n),n}function Ss(t,e,i,n){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElement(t),void 0!==n?i.insertBefore(o,n):i.appendChild(o)):(o=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==n?i.insertBefore(o,n):i.appendChild(o)),e[t].used.push(o),o}function Qs(t,e,i,n,o,s){var r;if("circle"==i.style?((r=Ms("circle",n,o)).setAttributeNS(null,"cx",t),r.setAttributeNS(null,"cy",e),r.setAttributeNS(null,"r",.5*i.size)):((r=Ms("rect",n,o)).setAttributeNS(null,"x",t-.5*i.size),r.setAttributeNS(null,"y",e-.5*i.size),r.setAttributeNS(null,"width",i.size),r.setAttributeNS(null,"height",i.size)),void 0!==i.styles&&r.setAttributeNS(null,"style",i.styles),r.setAttributeNS(null,"class",i.className+" vis-point"),s){var a=Ms("text",n,o);s.xOffset&&(t+=s.xOffset),s.yOffset&&(e+=s.yOffset),s.content&&(a.textContent=s.content),s.className&&a.setAttributeNS(null,"class",s.className+" vis-label"),a.setAttributeNS(null,"x",t),a.setAttributeNS(null,"y",e)}return r}function Ns(t,e,i,n,o,s,r,a){if(0!=n){n<0&&(e-=n*=-1);var l=Ms("rect",s,r);l.setAttributeNS(null,"x",t-.5*i),l.setAttributeNS(null,"y",e),l.setAttributeNS(null,"width",i),l.setAttributeNS(null,"height",n),l.setAttributeNS(null,"class",o),a&&l.setAttributeNS(null,"style",a)}}function Fs(){try{return navigator?navigator.languages&&navigator.languages.length?navigator.languages:navigator.userLanguage||navigator.language||navigator.browserLanguage||"en":"en"}catch(t){return"en"}}var Zs=Object.freeze({__proto__:null,prepareElements:Cs,cleanupElements:Ls,resetElements:Gs,getSVGElement:Ms,getDOMElement:Ss,drawPoint:Qs,drawBar:Ns,getNavigatorLanguage:Fs});var Bs=function(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t},Os=Pi("Array").entries,Ws=Array.prototype,As={DOMTokenList:!0,NodeList:!0},Xs=function(t){var e=t.entries;return t===Ws||t instanceof Array&&e===Ws.entries||As.hasOwnProperty(ne(t))?Os:e},Ts=pt("JSON","stringify"),Ys=/[\uD800-\uDFFF]/g,Es=/^[\uD800-\uDBFF]$/,Rs=/^[\uDC00-\uDFFF]$/,Us=function(t,e,i){var n=i.charAt(e-1),o=i.charAt(e+1);return Es.test(t)&&!Rs.test(o)||Rs.test(t)&&!Es.test(n)?"\\u"+t.charCodeAt(0).toString(16):t},zs=l((function(){return'"\\udf06\\ud834"'!==Ts("\udf06\ud834")||'"\\udead"'!==Ts("\udead")}));Ts&&V({target:"JSON",stat:!0,forced:zs},{stringify:function(t,e,i){var n=Ts.apply(null,arguments);return"string"==typeof n?n.replace(Ys,Us):n}}),W.JSON||(W.JSON={stringify:JSON.stringify});var Vs=function(t,e,i){return W.JSON.stringify.apply(null,arguments)},js=Pi("Array").values,Hs=Array.prototype,Ps={DOMTokenList:!0,NodeList:!0},Js=function(t){var e=t.values;return t===Hs||t instanceof Array&&e===Hs.values||Ps.hasOwnProperty(ne(t))?js:e},Ks=Pi("Array").keys,qs=Array.prototype,$s={DOMTokenList:!0,NodeList:!0},tr=function(t){var e=t.keys;return t===qs||t instanceof Array&&e===qs.keys||$s.hasOwnProperty(ne(t))?Ks:e},er=Ce.some,ir=$i("some"),nr=on("some");V({target:"Array",proto:!0,forced:!ir||!nr},{some:function(t){return er(this,t,arguments.length>1?arguments[1]:void 0)}});var or=Pi("Array").some,sr=Array.prototype,rr=function(t){var e=t.some;return t===sr||t instanceof Array&&e===sr.some?or:e};var ar=function(t,e){return!e||"object"!==zo(e)&&"function"!=typeof e?Bs(t):e},lr=Do;V({target:"Object",stat:!0},{setPrototypeOf:fi});var hr=W.Object.setPrototypeOf,dr=n((function(t){function e(i){return t.exports=e=hr?lr:function(t){return t.__proto__||lr(t)},e(i)}t.exports=e})),ur=Bi,cr=n((function(t){function e(i,n){return t.exports=e=hr||function(t,e){return t.__proto__=e,t},e(i,n)}t.exports=e}));var pr=function(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=ur(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),e&&cr(t,e)},mr=[],fr=mr.sort,gr=l((function(){mr.sort(void 0)})),vr=l((function(){mr.sort(null)})),br=$i("sort");V({target:"Array",proto:!0,forced:gr||!vr||!br},{sort:function(t){return void 0===t?fr.call(Lt(this)):fr.call(Lt(this),A(t))}});var yr=Pi("Array").sort,xr=Array.prototype,wr=function(t){var e=t.sort;return t===xr||t instanceof Array&&e===xr.sort?yr:e},_r=!l((function(){return Object.isExtensible(Object.preventExtensions({}))})),kr=n((function(t){var e=E.f,i=Ft("meta"),n=0,o=Object.isExtensible||function(){return!0},s=function(t){e(t,i,{value:{objectID:"O"+ ++n,weakData:{}}})},r=t.exports={REQUIRED:!1,fastKey:function(t,e){if(!x(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!k(t,i)){if(!o(t))return"F";if(!e)return"E";s(t)}return t[i].objectID},getWeakData:function(t,e){if(!k(t,i)){if(!o(t))return!0;if(!e)return!1;s(t)}return t[i].weakData},onFreeze:function(t){return _r&&r.REQUIRED&&o(t)&&!k(t,i)&&s(t),t}};st[i]=!0})),Dr=(kr.REQUIRED,kr.fastKey,kr.getWeakData,kr.onFreeze,n((function(t){var e=function(t,e){this.stopped=t,this.result=e};(t.exports=function(t,i,n,o,s){var r,a,l,h,d,u,c,p=X(i,n,o?2:1);if(s)r=t;else{if("function"!=typeof(a=Ni(t)))throw TypeError("Target is not iterable");if(io(a)){for(l=0,h=$(t.length);h>l;l++)if((d=o?p(T(c=t[l])[0],c[1]):p(t[l]))&&d instanceof e)return d;return new e(!1)}r=a.call(t)}for(u=r.next;!(c=u.call(r)).done;)if("object"==typeof(d=$n(r,p,c.value,o))&&d&&d instanceof e)return d;return new e(!1)}).stop=function(t){return new e(!0,t)}}))),Ir=function(t,e,i){if(!(t instanceof e))throw TypeError("Incorrect "+(i?i+" ":"")+"invocation");return t},Cr=E.f,Lr=Ce.forEach,Gr=we.set,Mr=we.getterFor,Sr=function(t,e,i){var n,o=-1!==t.indexOf("Map"),s=-1!==t.indexOf("Weak"),r=o?"set":"add",d=a[t],u=d&&d.prototype,c={};if(h&&"function"==typeof d&&(s||u.forEach&&!l((function(){(new d).entries().next()})))){n=e((function(e,i){Gr(Ir(e,n,t),{type:t,collection:new d}),null!=i&&Dr(i,e[r],e,o)}));var p=Mr(t);Lr(["add","clear","delete","forEach","get","has","set","keys","values","entries"],(function(t){var e="add"==t||"set"==t;t in u&&(!s||"clear"!=t)&&R(n.prototype,t,(function(i,n){var o=p(this).collection;if(!e&&s&&!x(i))return"get"==t&&void 0;var r=o[t](0===i?0:i,n);return e?this:r}))})),s||Cr(n.prototype,"size",{configurable:!0,get:function(){return p(this).collection.size}})}else n=i.getConstructor(e,t,o,r),kr.REQUIRED=!0;return ae(n,t,!1,!0),c[t]=n,V({global:!0,forced:!0},c),s||i.setStrong(n,t,o),n},Qr=function(t,e,i){for(var n in e)i&&i.unsafe&&t[n]?t[n]=e[n]:zt(t,n,e[n],i);return t},Nr=Pt("species"),Fr=E.f,Zr=kr.fastKey,Br=we.set,Or=we.getterFor,Wr={getConstructor:function(t,e,i,n){var o=t((function(t,s){Ir(t,o,e),Br(t,{type:e,index:Tt(null),first:void 0,last:void 0,size:0}),h||(t.size=0),null!=s&&Dr(s,t[n],t,i)})),s=Or(e),r=function(t,e,i){var n,o,r=s(t),l=a(t,e);return l?l.value=i:(r.last=l={index:o=Zr(e,!0),key:e,value:i,previous:n=r.last,next:void 0,removed:!1},r.first||(r.first=l),n&&(n.next=l),h?r.size++:t.size++,"F"!==o&&(r.index[o]=l)),t},a=function(t,e){var i,n=s(t),o=Zr(e);if("F"!==o)return n.index[o];for(i=n.first;i;i=i.next)if(i.key==e)return i};return Qr(o.prototype,{clear:function(){for(var t=s(this),e=t.index,i=t.first;i;)i.removed=!0,i.previous&&(i.previous=i.previous.next=void 0),delete e[i.index],i=i.next;t.first=t.last=void 0,h?t.size=0:this.size=0},delete:function(t){var e=s(this),i=a(this,t);if(i){var n=i.next,o=i.previous;delete e.index[i.index],i.removed=!0,o&&(o.next=n),n&&(n.previous=o),e.first==i&&(e.first=n),e.last==i&&(e.last=o),h?e.size--:this.size--}return!!i},forEach:function(t){for(var e,i=s(this),n=X(t,arguments.length>1?arguments[1]:void 0,3);e=e?e.next:i.first;)for(n(e.value,e.key,this);e&&e.removed;)e=e.previous},has:function(t){return!!a(this,t)}}),Qr(o.prototype,i?{get:function(t){var e=a(this,t);return e&&e.value},set:function(t,e){return r(this,0===t?0:t,e)}}:{add:function(t){return r(this,t=0===t?0:t,t)}}),h&&Fr(o.prototype,"size",{get:function(){return s(this).size}}),o},setStrong:function(t,e,i){var n=e+" Iterator",o=Or(e),s=Or(n);xi(t,e,(function(t,e){Br(this,{type:n,target:t,state:o(t),kind:e,last:void 0})}),(function(){for(var t=s(this),e=t.kind,i=t.last;i&&i.removed;)i=i.previous;return t.target&&(t.last=i=i?i.next:t.state.first)?"keys"==e?{value:i.key,done:!1}:"values"==e?{value:i.value,done:!1}:{value:[i.key,i.value],done:!1}:(t.target=void 0,{value:void 0,done:!0})}),i?"entries":"values",!i,!0),function(t){var e=pt(t),i=E.f;h&&e&&!e[Nr]&&i(e,Nr,{configurable:!0,get:function(){return this}})}(e)}},Ar=(Sr("Set",(function(t){return function(){return t(this,arguments.length?arguments[0]:void 0)}}),Wr),W.Set),Xr=(Sr("Map",(function(t){return function(){return t(this,arguments.length?arguments[0]:void 0)}}),Wr),W.Map),Tr=n((function(t){var e=function(t){var e=Object.prototype,i=e.hasOwnProperty,n="function"==typeof Symbol?Symbol:{},o=n.iterator||"@@iterator",s=n.asyncIterator||"@@asyncIterator",r=n.toStringTag||"@@toStringTag";function a(t,e,i,n){var o=e&&e.prototype instanceof d?e:d,s=Object.create(o.prototype),r=new _(n||[]);return s._invoke=function(t,e,i){var n="suspendedStart";return function(o,s){if("executing"===n)throw new Error("Generator is already running");if("completed"===n){if("throw"===o)throw s;return D()}for(i.method=o,i.arg=s;;){var r=i.delegate;if(r){var a=y(r,i);if(a){if(a===h)continue;return a}}if("next"===i.method)i.sent=i._sent=i.arg;else if("throw"===i.method){if("suspendedStart"===n)throw n="completed",i.arg;i.dispatchException(i.arg)}else"return"===i.method&&i.abrupt("return",i.arg);n="executing";var d=l(t,e,i);if("normal"===d.type){if(n=i.done?"completed":"suspendedYield",d.arg===h)continue;return{value:d.arg,done:i.done}}"throw"===d.type&&(n="completed",i.method="throw",i.arg=d.arg)}}}(t,i,r),s}function l(t,e,i){try{return{type:"normal",arg:t.call(e,i)}}catch(t){return{type:"throw",arg:t}}}t.wrap=a;var h={};function d(){}function u(){}function c(){}var p={};p[o]=function(){return this};var m=Object.getPrototypeOf,f=m&&m(m(k([])));f&&f!==e&&i.call(f,o)&&(p=f);var g=c.prototype=d.prototype=Object.create(p);function v(t){["next","throw","return"].forEach((function(e){t[e]=function(t){return this._invoke(e,t)}}))}function b(t,e){var n;this._invoke=function(o,s){function r(){return new e((function(n,r){!function n(o,s,r,a){var h=l(t[o],t,s);if("throw"!==h.type){var d=h.arg,u=d.value;return u&&"object"==typeof u&&i.call(u,"__await")?e.resolve(u.__await).then((function(t){n("next",t,r,a)}),(function(t){n("throw",t,r,a)})):e.resolve(u).then((function(t){d.value=t,r(d)}),(function(t){return n("throw",t,r,a)}))}a(h.arg)}(o,s,n,r)}))}return n=n?n.then(r,r):r()}}function y(t,e){var i=t.iterator[e.method];if(void 0===i){if(e.delegate=null,"throw"===e.method){if(t.iterator.return&&(e.method="return",e.arg=void 0,y(t,e),"throw"===e.method))return h;e.method="throw",e.arg=new TypeError("The iterator does not provide a 'throw' method")}return h}var n=l(i,t.iterator,e.arg);if("throw"===n.type)return e.method="throw",e.arg=n.arg,e.delegate=null,h;var o=n.arg;return o?o.done?(e[t.resultName]=o.value,e.next=t.nextLoc,"return"!==e.method&&(e.method="next",e.arg=void 0),e.delegate=null,h):o:(e.method="throw",e.arg=new TypeError("iterator result is not an object"),e.delegate=null,h)}function x(t){var e={tryLoc:t[0]};1 in t&&(e.catchLoc=t[1]),2 in t&&(e.finallyLoc=t[2],e.afterLoc=t[3]),this.tryEntries.push(e)}function w(t){var e=t.completion||{};e.type="normal",delete e.arg,t.completion=e}function _(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(x,this),this.reset(!0)}function k(t){if(t){var e=t[o];if(e)return e.call(t);if("function"==typeof t.next)return t;if(!isNaN(t.length)){var n=-1,s=function e(){for(;++n<t.length;)if(i.call(t,n))return e.value=t[n],e.done=!1,e;return e.value=void 0,e.done=!0,e};return s.next=s}}return{next:D}}function D(){return{value:void 0,done:!0}}return u.prototype=g.constructor=c,c.constructor=u,c[r]=u.displayName="GeneratorFunction",t.isGeneratorFunction=function(t){var e="function"==typeof t&&t.constructor;return!!e&&(e===u||"GeneratorFunction"===(e.displayName||e.name))},t.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,c):(t.__proto__=c,r in t||(t[r]="GeneratorFunction")),t.prototype=Object.create(g),t},t.awrap=function(t){return{__await:t}},v(b.prototype),b.prototype[s]=function(){return this},t.AsyncIterator=b,t.async=function(e,i,n,o,s){void 0===s&&(s=Promise);var r=new b(a(e,i,n,o),s);return t.isGeneratorFunction(i)?r:r.next().then((function(t){return t.done?t.value:r.next()}))},v(g),g[r]="Generator",g[o]=function(){return this},g.toString=function(){return"[object Generator]"},t.keys=function(t){var e=[];for(var i in t)e.push(i);return e.reverse(),function i(){for(;e.length;){var n=e.pop();if(n in t)return i.value=n,i.done=!1,i}return i.done=!0,i}},t.values=k,_.prototype={constructor:_,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=void 0,this.done=!1,this.delegate=null,this.method="next",this.arg=void 0,this.tryEntries.forEach(w),!t)for(var e in this)"t"===e.charAt(0)&&i.call(this,e)&&!isNaN(+e.slice(1))&&(this[e]=void 0)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(t){if(this.done)throw t;var e=this;function n(i,n){return r.type="throw",r.arg=t,e.next=i,n&&(e.method="next",e.arg=void 0),!!n}for(var o=this.tryEntries.length-1;o>=0;--o){var s=this.tryEntries[o],r=s.completion;if("root"===s.tryLoc)return n("end");if(s.tryLoc<=this.prev){var a=i.call(s,"catchLoc"),l=i.call(s,"finallyLoc");if(a&&l){if(this.prev<s.catchLoc)return n(s.catchLoc,!0);if(this.prev<s.finallyLoc)return n(s.finallyLoc)}else if(a){if(this.prev<s.catchLoc)return n(s.catchLoc,!0)}else{if(!l)throw new Error("try statement without catch or finally");if(this.prev<s.finallyLoc)return n(s.finallyLoc)}}}},abrupt:function(t,e){for(var n=this.tryEntries.length-1;n>=0;--n){var o=this.tryEntries[n];if(o.tryLoc<=this.prev&&i.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var s=o;break}}s&&("break"===t||"continue"===t)&&s.tryLoc<=e&&e<=s.finallyLoc&&(s=null);var r=s?s.completion:{};return r.type=t,r.arg=e,s?(this.method="next",this.next=s.finallyLoc,h):this.complete(r)},complete:function(t,e){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&e&&(this.next=e),h},finish:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e];if(i.finallyLoc===t)return this.complete(i.completion,i.afterLoc),w(i),h}},catch:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e];if(i.tryLoc===t){var n=i.completion;if("throw"===n.type){var o=n.arg;w(i)}return o}}throw new Error("illegal catch attempt")},delegateYield:function(t,e,i){return this.delegate={iterator:k(t),resultName:e,nextLoc:i},"next"===this.method&&(this.arg=void 0),h}},t}(t.exports);try{regeneratorRuntime=e}catch(t){Function("r","regeneratorRuntime = r")(e)}})),Yr=Xo,Er=wn("splice"),Rr=on("splice",{ACCESSORS:!0,0:0,1:2}),Ur=Math.max,zr=Math.min;V({target:"Array",proto:!0,forced:!Er||!Rr},{splice:function(t,e){var i,n,o,s,r,a,l=Lt(this),h=$(l.length),d=it(t,h),u=arguments.length;if(0===u?i=n=0:1===u?(i=0,n=h-d):(i=u-2,n=zr(Ur(K(e),0),h-d)),h+i-n>9007199254740991)throw TypeError("Maximum allowed length exceeded");for(o=ke(l,n),s=0;s<n;s++)(r=d+s)in l&&bt(o,s,l[r]);if(o.length=n,i<n){for(s=d;s<h-n;s++)a=s+i,(r=s+n)in l?l[a]=l[r]:delete l[a];for(s=h;s>h-n+i;s--)delete l[s-1]}else if(i>n)for(s=h-n;s>d;s--)a=s+i-1,(r=s+n-1)in l?l[a]=l[r]:delete l[a];for(s=0;s<i;s++)l[s+d]=arguments[s+2];return l.length=h-n+i,o}});var Vr=Pi("Array").splice,jr=Array.prototype,Hr=function(t){var e=t.splice;return t===jr||t instanceof Array&&e===jr.splice?Vr:e},Pr=[].slice,Jr=/MSIE .\./.test(fn),Kr=function(t){return function(e,i){var n=arguments.length>2,o=n?Pr.call(arguments,2):void 0;return t(n?function(){("function"==typeof e?e:Function(e)).apply(this,o)}:e,i)}};V({global:!0,bind:!0,forced:Jr},{setTimeout:Kr(a.setTimeout),setInterval:Kr(a.setInterval)});var qr=W.setTimeout,$r=function(t,e,i,n,o,s,r,a){for(var l,h=o,d=0,u=!!r&&X(r,a,3);d<n;){if(d in i){if(l=u?u(i[d],d,e):i[d],s>0&&Ct(l))h=$r(t,e,l,$(l.length),h,s-1)-1;else{if(h>=9007199254740991)throw TypeError("Exceed the acceptable array length");t[h]=l}h++}d++}return h},ta=$r;V({target:"Array",proto:!0},{flatMap:function(t){var e,i=Lt(this),n=$(i.length);return A(t),(e=ke(i,0)).length=ta(e,i,i,n,0,1,t,arguments.length>1?arguments[1]:void 0),e}});var ea=Pi("Array").flatMap,ia=Array.prototype,na=function(t){var e=t.flatMap;return t===ia||t instanceof Array&&e===ia.flatMap?ea:e},oa=function(t){return function(e,i,n,o){A(i);var s=Lt(e),r=v(s),a=$(s.length),l=t?a-1:0,h=t?-1:1;if(n<2)for(;;){if(l in r){o=r[l],l+=h;break}if(l+=h,t?l<0:a<=l)throw TypeError("Reduce of empty array with no initial value")}for(;t?l>=0:a>l;l+=h)l in r&&(o=i(o,r[l],l,s));return o}},sa={left:oa(!1),right:oa(!0)}.left,ra=$i("reduce"),aa=on("reduce",{1:0});V({target:"Array",proto:!0,forced:!ra||!aa},{reduce:function(t){return sa(this,t,arguments.length,arguments.length>1?arguments[1]:void 0)}});var la=Pi("Array").reduce,ha=Array.prototype,da=function(t){var e=t.reduce;return t===ha||t instanceof Array&&e===ha.reduce?la:e},ua=[].slice,ca={},pa=function(t,e,i){if(!(e in ca)){for(var n=[],o=0;o<e;o++)n[o]="a["+o+"]";ca[e]=Function("C,a","return new C("+n.join(",")+")")}return ca[e](t,i)},ma=Function.bind||function(t){var e=A(this),i=ua.call(arguments,1),n=function(){var o=i.concat(ua.call(arguments));return this instanceof n?pa(e,o.length,o):e.apply(t,o)};return x(e.prototype)&&(n.prototype=e.prototype),n};V({target:"Function",proto:!0},{bind:ma});var fa=Pi("Function").bind,ga=Function.prototype,va=function(t){var e=t.bind;return t===ga||t instanceof Function&&e===ga.bind?fa:e};var ba=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")};function ya(t,e){for(var i=0;i<e.length;i++){var n=e[i];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Wi(t,n.key,n)}}var xa=function(t,e,i){return e&&ya(t.prototype,e),i&&ya(t,i),t};function wa(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}function _a(t){return new Da(t)}var ka=function(){function t(e,i,n){var o,s,r;ba(this,t),this._source=e,this._transformers=i,this._target=n,this._listeners={add:va(o=this._add).call(o,this),remove:va(s=this._remove).call(s,this),update:va(r=this._update).call(r,this)}}return xa(t,[{key:"all",value:function(){return this._target.update(this._transformItems(this._source.get())),this}},{key:"start",value:function(){return this._source.on("add",this._listeners.add),this._source.on("remove",this._listeners.remove),this._source.on("update",this._listeners.update),this}},{key:"stop",value:function(){return this._source.off("add",this._listeners.add),this._source.off("remove",this._listeners.remove),this._source.off("update",this._listeners.update),this}},{key:"_transformItems",value:function(t){var e;return da(e=this._transformers).call(e,(function(t,e){return e(t)}),t)}},{key:"_add",value:function(t,e){null!=e&&this._target.add(this._transformItems(this._source.get(e.items)))}},{key:"_update",value:function(t,e){null!=e&&this._target.update(this._transformItems(this._source.get(e.items)))}},{key:"_remove",value:function(t,e){null!=e&&this._target.remove(this._transformItems(e.oldData))}}]),t}(),Da=function(){function t(e){ba(this,t),this._source=e,this._transformers=[]}return xa(t,[{key:"filter",value:function(t){return this._transformers.push((function(e){return En(e).call(e,t)})),this}},{key:"map",value:function(t){return this._transformers.push((function(e){return Ln(e).call(e,t)})),this}},{key:"flatMap",value:function(t){return this._transformers.push((function(e){return na(e).call(e,t)})),this}},{key:"to",value:function(t){return new ka(this._source,this._transformers,t)}}]),t}();for(var Ia=function(t,e){return t(e={exports:{}},e.exports),e.exports}((function(t){var e,i,n="undefined"!=typeof crypto&&crypto.getRandomValues&&va(e=crypto.getRandomValues).call(e,crypto)||"undefined"!=typeof msCrypto&&"function"==typeof window.msCrypto.getRandomValues&&va(i=msCrypto.getRandomValues).call(i,msCrypto);if(n){var o=new Uint8Array(16);t.exports=function(){return n(o),o}}else{var s=new Array(16);t.exports=function(){for(var t,e=0;e<16;e++)0==(3&e)&&(t=4294967296*Math.random()),s[e]=t>>>((3&e)<<3)&255;return s}}})),Ca=[],La=0;La<256;++La)Ca[La]=(La+256).toString(16).substr(1);var Ga=function(t,e){var i=e||0,n=Ca;return[n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]]].join("")};var Ma=function(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"===t?new Array(16):null,t=null);var o=(t=t||{}).random||(t.rng||Ia)();if(o[6]=15&o[6]|64,o[8]=63&o[8]|128,e)for(var s=0;s<16;++s)e[n+s]=o[s];return e||Ga(o)},Sa=/^\/?Date\((-?\d+)/i;function Qa(t){return s.isMoment(t)}function Na(t,e){var i;if(void 0!==t){if(null===t)return null;if(!e)return t;if("string"!=typeof e&&!(e instanceof String))throw new Error("Type must be a string");switch(e){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return rs(t)&&!isNaN(Date.parse(t))?s(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(ss(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(Qa(t))return new Date(t.valueOf());if(rs(t))return(i=Sa.exec(t))?new Date(Number(i[1])):s(new Date(t)).toDate();throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"Moment":if(ss(t))return s(t);if(t instanceof Date)return s(t.valueOf());if(Qa(t))return s(t);if(rs(t))return i=Sa.exec(t),s(i?Number(i[1]):t);throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"ISODate":if(ss(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(Qa(t))return t.toDate().toISOString();if(rs(t))return(i=Sa.exec(t))?new Date(Number(i[1])).toISOString():s(t).format();throw new Error("Cannot convert object of type "+us(t)+" to type ISODate");case"ASPDate":if(ss(t))return"/Date("+t+")/";if(t instanceof Date||Qa(t))return"/Date("+t.valueOf()+")/";if(rs(t))return"/Date("+((i=Sa.exec(t))?new Date(Number(i[1])).valueOf():new Date(t).valueOf())+")/";throw new Error("Cannot convert object of type "+us(t)+" to type ASPDate");default:throw new Error("Unknown type ".concat(e))}}}function Fa(t){return"string"==typeof t||"number"==typeof t}var Za=function(){function t(e){ba(this,t),this._queue=[],this._timeout=null,this._extended=null,this.delay=null,this.max=1/0,this.setOptions(e)}return xa(t,[{key:"setOptions",value:function(t){t&&void 0!==t.delay&&(this.delay=t.delay),t&&void 0!==t.max&&(this.max=t.max),this._flushIfNeeded()}},{key:"destroy",value:function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var n=e[i];n.original?t[n.name]=n.original:delete t[n.name]}this._extended=null}}},{key:"replace",value:function(t,e){var i=this,n=t[e];if(!n)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=arguments.length,e=new Array(t),o=0;o<t;o++)e[o]=arguments[o];i.queue({args:e,fn:n,context:this})}}},{key:"queue",value:function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()}},{key:"_flushIfNeeded",value:function(){var t=this;this._queue.length>this.max&&this.flush(),null!=this._timeout&&(clearTimeout(this._timeout),this._timeout=null),this.queue.length>0&&"number"==typeof this.delay&&(this._timeout=qr((function(){t.flush()}),this.delay))}},{key:"flush",value:function(){var t,e;mn(t=Hr(e=this._queue).call(e,0)).call(t,(function(t){t.fn.apply(t.context||t.fn,t.args||[])}))}}],[{key:"extend",value:function(e,i){var n=new t(i);if(void 0!==e.flush)throw new Error("Target object already has a property flush");e.flush=function(){n.flush()};var o=[{name:"flush",original:void 0}];if(i&&i.replace)for(var s=0;s<i.replace.length;s++){var r=i.replace[s];o.push({name:r,original:e[r]}),n.replace(e,r)}return n._extended={object:e,methods:o},n}}]),t}(),Ba=function(){function t(){ba(this,t),this._subscribers={"*":[],add:[],remove:[],update:[]},this.subscribe=t.prototype.on,this.unsubscribe=t.prototype.off}return xa(t,[{key:"_trigger",value:function(t,e,i){var n,o;if("*"===t)throw new Error("Cannot trigger event *");mn(n=Pn(o=[]).call(o,mo(this._subscribers[t]),mo(this._subscribers["*"]))).call(n,(function(n){n(t,e,null!=i?i:null)}))}},{key:"on",value:function(t,e){"function"==typeof e&&this._subscribers[t].push(e)}},{key:"off",value:function(t,e){var i;this._subscribers[t]=En(i=this._subscribers[t]).call(i,(function(t){return t!==e}))}}]),t}(),Oa=function(){function t(e){ba(this,t),this._pairs=e}return xa(t,[{key:Yr,value:Tr.mark((function t(){var e,i,n,o,s,r,a,l;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[0],l=r[1],t.next=9,[a,l];case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"entries",value:Tr.mark((function t(){var e,i,n,o,s,r,a,l;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[0],l=r[1],t.next=9,[a,l];case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"keys",value:Tr.mark((function t(){var e,i,n,o,s,r,a;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,1),a=r[0],t.next=9,a;case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"values",value:Tr.mark((function t(){var e,i,n,o,s,r,a;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[1],t.next=9,a;case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"toIdArray",value:function(){var t;return Ln(t=mo(this._pairs)).call(t,(function(t){return t[0]}))}},{key:"toItemArray",value:function(){var t;return Ln(t=mo(this._pairs)).call(t,(function(t){return t[1]}))}},{key:"toEntryArray",value:function(){return mo(this._pairs)}},{key:"toObjectMap",value:function(){var t=Oi(null),e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this._pairs);!(e=(o=s.next()).done);e=!0){var r=Po(o.value,2),a=r[0],l=r[1];t[a]=l}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}return t}},{key:"toMap",value:function(){return new Xr(this._pairs)}},{key:"toIdSet",value:function(){return new Ar(this.toIdArray())}},{key:"toItemSet",value:function(){return new Ar(this.toItemArray())}},{key:"cache",value:function(){return new t(mo(this._pairs))}},{key:"distinct",value:function(t){var e=new Ar,i=!0,n=!1,o=void 0;try{for(var s,r=Fi(this._pairs);!(i=(s=r.next()).done);i=!0){var a=Po(s.value,2),l=a[0],h=a[1];e.add(t(h,l))}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}return e}},{key:"filter",value:function(e){var i=this._pairs;return new t(Ai({},Yr,Tr.mark((function t(){var n,o,s,r,a,l,h,d;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:n=!0,o=!1,s=void 0,t.prev=3,r=Fi(i);case 5:if(n=(a=r.next()).done){t.next=13;break}if(l=Po(a.value,2),h=l[0],d=l[1],!e(d,h)){t.next=10;break}return t.next=10,[h,d];case 10:n=!0,t.next=5;break;case 13:t.next=19;break;case 15:t.prev=15,t.t0=t.catch(3),o=!0,s=t.t0;case 19:t.prev=19,t.prev=20,n||null==r.return||r.return();case 22:if(t.prev=22,!o){t.next=25;break}throw s;case 25:return t.finish(22);case 26:return t.finish(19);case 27:case"end":return t.stop()}}),t,null,[[3,15,19,27],[20,,22,26]])}))))}},{key:"forEach",value:function(t){var e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this._pairs);!(e=(o=s.next()).done);e=!0){var r=Po(o.value,2),a=r[0];t(r[1],a)}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}}},{key:"map",value:function(e){var i=this._pairs;return new t(Ai({},Yr,Tr.mark((function t(){var n,o,s,r,a,l,h,d;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:n=!0,o=!1,s=void 0,t.prev=3,r=Fi(i);case 5:if(n=(a=r.next()).done){t.next=12;break}return l=Po(a.value,2),h=l[0],d=l[1],t.next=9,[h,e(d,h)];case 9:n=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),o=!0,s=t.t0;case 18:t.prev=18,t.prev=19,n||null==r.return||r.return();case 21:if(t.prev=21,!o){t.next=24;break}throw s;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,null,[[3,14,18,26],[19,,21,25]])}))))}},{key:"max",value:function(t){var e=Fi(this._pairs),i=e.next();if(i.done)return null;for(var n=i.value[1],o=t(i.value[1],i.value[0]);!(i=e.next()).done;){var s=Po(i.value,2),r=s[0],a=s[1],l=t(a,r);l>o&&(o=l,n=a)}return n}},{key:"min",value:function(t){var e=Fi(this._pairs),i=e.next();if(i.done)return null;for(var n=i.value[1],o=t(i.value[1],i.value[0]);!(i=e.next()).done;){var s=Po(i.value,2),r=s[0],a=s[1],l=t(a,r);l<o&&(o=l,n=a)}return n}},{key:"reduce",value:function(t,e){var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(this._pairs);!(i=(s=r.next()).done);i=!0){var a=Po(s.value,2),l=a[0];e=t(e,a[1],l)}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}return e}},{key:"sort",value:function(e){var i=this;return new t(Ai({},Yr,(function(){var t;return Fi(wr(t=mo(i._pairs)).call(t,(function(t,i){var n=Po(t,2),o=n[0],s=n[1],r=Po(i,2),a=r[0],l=r[1];return e(s,l,o,a)})))})))}}]),t}(),Wa=function(){console.warn("Type coercion has been deprecated. Please, use data pipes instead. See https://visjs.github.io/vis-data/data/datapipe.html#TypeCoercion for more details with working migration example.")},Aa=function(t){function e(t,i){var n;if(ba(this,e),n=ar(this,dr(e).call(this)),t&&!Zo(t)&&(i=t,t=[]),n._options=i||{},n._data=new Xr,n.length=0,n._idProp=n._options.fieldId||"id",n._type={},n._options.type){Wa();for(var o=Yi(n._options.type),s=0,r=o.length;s<r;s++){var a=o[s],l=n._options.type[a];n._type[a]="Date"==l||"ISODate"==l||"ASPDate"==l?"Date":l}}return t&&t.length&&n.add(t),n.setOptions(i),n}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&void 0!==t.queue&&(!1===t.queue?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=Za.extend(this,{replace:["add","update","remove"]})),t.queue&&"object"===zo(t.queue)&&this._queue.setOptions(t.queue)))}},{key:"add",value:function(t,e){var i,n=this,o=[];if(Zo(t)){var s=Ln(t).call(t,(function(t){return t[n._idProp]}));if(rr(s).call(s,(function(t){return n._data.has(t)})))throw new Error("A duplicate id was found in the parameter array.");for(var r=0,a=t.length;r<a;r++)i=this._addItem(t[r]),o.push(i)}else{if(!t||"object"!==zo(t))throw new Error("Unknown dataType");i=this._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o}},{key:"update",value:function(t,e){var i=this,n=[],o=[],s=[],r=[],a=this._idProp,l=function(t){var e=t[a];if(null!=e&&i._data.has(e)){var l=t,h=Ao({},i._data.get(e)),d=i._updateItem(l);o.push(d),r.push(l),s.push(h)}else{var u=i._addItem(t);n.push(u)}};if(Zo(t))for(var h=0,d=t.length;h<d;h++)t[h]&&"object"===zo(t[h])?l(t[h]):console.warn("Ignoring input item, which is not an object at index "+h);else{if(!t||"object"!==zo(t))throw new Error("Unknown dataType");l(t)}if(n.length&&this._trigger("add",{items:n},e),o.length){var u={items:o,oldData:s,data:r};this._trigger("update",u,e)}return Pn(n).call(n,o)}},{key:"updateOnly",value:function(t,e){var i,n=this;Zo(t)||(t=[t]);var o=Ln(i=Ln(t).call(t,(function(t){var e=n._data.get(t[n._idProp]);if(null==e)throw new Error("Updating non-existent items is not allowed.");return{oldData:e,update:t}}))).call(i,(function(t){var e=t.oldData,i=t.update,o=e[n._idProp],s=ds(ds({},e),i);return n._data.set(o,s),{id:o,oldData:e,updatedData:s}}));if(o.length){var s={items:Ln(o).call(o,(function(t){return t.id})),oldData:Ln(o).call(o,(function(t){return t.oldData})),data:Ln(o).call(o,(function(t){return t.updatedData}))};return this._trigger("update",s,e),s.items}return[]}},{key:"get",value:function(t,e){var i=void 0,n=void 0,o=void 0;Fa(t)?(i=t,o=e):Zo(t)?(n=t,o=e):o=t;var s=o&&"Object"===o.returnType?"Object":"Array",r=o&&o.type||this._options.type,a=o&&En(o),l=[],h=null,d=null,u=null;if(null!=i)(h=this._getItem(i,r))&&a&&!a(h)&&(h=null);else if(null!=n)for(var c=0,p=n.length;c<p;c++)null==(h=this._getItem(n[c],r))||a&&!a(h)||l.push(h);else for(var m,f=0,g=(d=mo(tr(m=this._data).call(m))).length;f<g;f++)u=d[f],null==(h=this._getItem(u,r))||a&&!a(h)||l.push(h);if(o&&o.order&&null==i&&this._sort(l,o.order),o&&o.fields){var v=o.fields;if(null!=i&&null!=h)h=this._filterFields(h,v);else for(var b=0,y=l.length;b<y;b++)l[b]=this._filterFields(l[b],v)}if("Object"==s){for(var x={},w=0,_=l.length;w<_;w++){var k=l[w];x[k[this._idProp]]=k}return x}return null!=i?h:l}},{key:"getIds",value:function(t){var e,i,n=this._data,o=t&&En(t),s=t&&t.order,r=t&&t.type||this._options.type,a=mo(tr(n).call(n)),l=[];if(o)if(s){i=[];for(var h=0,d=a.length;h<d;h++){var u=a[h];o(e=this._getItem(u,r))&&i.push(e)}this._sort(i,s);for(var c=0,p=i.length;c<p;c++)l.push(i[c][this._idProp])}else for(var m=0,f=a.length;m<f;m++){var g=a[m];o(e=this._getItem(g,r))&&l.push(e[this._idProp])}else if(s){i=[];for(var v=0,b=a.length;v<b;v++){var y=a[v];i.push(n.get(y))}this._sort(i,s);for(var x=0,w=i.length;x<w;x++)l.push(i[x][this._idProp])}else for(var _=0,k=a.length;_<k;_++){var D=a[_];e=n.get(D),l.push(e[this._idProp])}return l}},{key:"getDataSet",value:function(){return this}},{key:"forEach",value:function(t,e){var i=e&&En(e),n=e&&e.type||this._options.type,o=this._data,s=mo(tr(o).call(o));if(e&&e.order)for(var r=this.get(e),a=0,l=r.length;a<l;a++){var h=r[a];t(h,h[this._idProp])}else for(var d=0,u=s.length;d<u;d++){var c=s[d],p=this._getItem(c,n);i&&!i(p)||t(p,c)}}},{key:"map",value:function(t,e){for(var i=e&&En(e),n=e&&e.type||this._options.type,o=[],s=this._data,r=mo(tr(s).call(s)),a=0,l=r.length;a<l;a++){var h=r[a],d=this._getItem(h,n);i&&!i(d)||o.push(t(d,h))}return e&&e.order&&this._sort(o,e.order),o}},{key:"_filterFields",value:function(t,e){var i;return t?da(i=Zo(e)?e:Yi(e)).call(i,(function(e,i){return e[i]=t[i],e}),{}):t}},{key:"_sort",value:function(t,e){if("string"==typeof e){var i=e;wr(t).call(t,(function(t,e){var n=t[i],o=e[i];return n>o?1:n<o?-1:0}))}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");wr(t).call(t,e)}}},{key:"remove",value:function(t,e){for(var i=[],n=[],o=Zo(t)?t:[t],s=0,r=o.length;s<r;s++){var a=this._remove(o[s]);if(a){var l=a[this._idProp];null!=l&&(i.push(l),n.push(a))}}return i.length&&this._trigger("remove",{items:i,oldData:n},e),i}},{key:"_remove",value:function(t){var e;if(Fa(t)?e=t:t&&"object"===zo(t)&&(e=t[this._idProp]),null!=e&&this._data.has(e)){var i=this._data.get(e)||null;return this._data.delete(e),--this.length,i}return null}},{key:"clear",value:function(t){for(var e,i=mo(tr(e=this._data).call(e)),n=[],o=0,s=i.length;o<s;o++)n.push(this._data.get(i[o]));return this._data.clear(),this.length=0,this._trigger("remove",{items:i,oldData:n},t),i}},{key:"max",value:function(t){var e=null,i=null,n=!0,o=!1,s=void 0;try{for(var r,a=Fi(Js(l=this._data).call(l));!(n=(r=a.next()).done);n=!0){var l,h=r.value,d=h[t];"number"==typeof d&&(null==i||d>i)&&(e=h,i=d)}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return e||null}},{key:"min",value:function(t){var e=null,i=null,n=!0,o=!1,s=void 0;try{for(var r,a=Fi(Js(l=this._data).call(l));!(n=(r=a.next()).done);n=!0){var l,h=r.value,d=h[t];"number"==typeof d&&(null==i||d<i)&&(e=h,i=d)}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return e||null}},{key:"distinct",value:function(t){for(var e=this._data,i=mo(tr(e).call(e)),n=[],o=this._options.type&&this._options.type[t]||null,s=0,r=0,a=i.length;r<a;r++){for(var l=i[r],h=e.get(l)[t],d=!1,u=0;u<s;u++)if(n[u]==h){d=!0;break}d||void 0===h||(n[s]=h,s++)}if(o)for(var c=0,p=n.length;c<p;c++)n[c]=Na(n[c],o);return n}},{key:"_addItem",value:function(t){var e=t[this._idProp];if(null!=e){if(this._data.has(e))throw new Error("Cannot add item: item with id "+e+" already exists")}else e=Ma(),t[this._idProp]=e;for(var i={},n=Yi(t),o=0,s=n.length;o<s;o++){var r=n[o],a=this._type[r];i[r]=Na(t[r],a)}return this._data.set(e,i),++this.length,e}},{key:"_getItem",value:function(t,e){var i,n=this._data.get(t);if(!n)return null;var o=Yi(n);if(e){Wa(),i={};for(var s=0,r=o.length;s<r;s++){var a=o[s],l=n[a];i[a]=Na(l,e[a])}}else i=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=wa(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=wa(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},n);return null==i[this._idProp]&&(i[this._idProp]=n.id),i}},{key:"_updateItem",value:function(t){var e=t[this._idProp];if(null==e)throw new Error("Cannot update item: item has no id (item: "+Vs(t)+")");var i=this._data.get(e);if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var n=Yi(t),o=0,s=n.length;o<s;o++){var r=n[o],a=this._type[r];i[r]=Na(t[r],a)}return e}},{key:"stream",value:function(t){if(t){var e=this._data;return new Oa(Ai({},Yr,Tr.mark((function i(){var n,o,s,r,a,l,h;return Tr.wrap((function(i){for(;;)switch(i.prev=i.next){case 0:n=!0,o=!1,s=void 0,i.prev=3,r=Fi(t);case 5:if(n=(a=r.next()).done){i.next=14;break}if(l=a.value,null==(h=e.get(l))){i.next=11;break}return i.next=11,[l,h];case 11:n=!0,i.next=5;break;case 14:i.next=20;break;case 16:i.prev=16,i.t0=i.catch(3),o=!0,s=i.t0;case 20:i.prev=20,i.prev=21,n||null==r.return||r.return();case 23:if(i.prev=23,!o){i.next=26;break}throw s;case 26:return i.finish(23);case 27:return i.finish(20);case 28:case"end":return i.stop()}}),i,null,[[3,16,20,28],[21,,23,27]])}))))}var i;return new Oa(Ai({},Yr,va(i=Xs(this._data)).call(i,this._data)))}}]),e}(Ba),Xa=function(t){function e(t,i){var n,o;return ba(this,e),(o=ar(this,dr(e).call(this))).length=0,o._ids=new Ar,o._options=i||{},o._listener=va(n=o._onEvent).call(n,Bs(o)),o.setData(t),o}return pr(e,t),xa(e,[{key:"setData",value:function(t){if(this._data){this._data.off&&this._data.off("*",this._listener);var e=this._data.getIds({filter:En(this._options)}),i=this._data.get(e);this._ids.clear(),this.length=0,this._trigger("remove",{items:e,oldData:i})}if(null!=t){this._data=t;for(var n=this._data.getIds({filter:En(this._options)}),o=0,s=n.length;o<s;o++){var r=n[o];this._ids.add(r)}this.length=n.length,this._trigger("add",{items:n})}else this._data=new Aa;this._data.on&&this._data.on("*",this._listener)}},{key:"refresh",value:function(){for(var t=this._data.getIds({filter:En(this._options)}),e=mo(this._ids),i={},n=[],o=[],s=[],r=0,a=t.length;r<a;r++){var l=t[r];i[l]=!0,this._ids.has(l)||(n.push(l),this._ids.add(l))}for(var h=0,d=e.length;h<d;h++){var u=e[h],c=this._data.get(u);null==c?console.error("If you see this, report it please."):i[u]||(o.push(u),s.push(c),this._ids.delete(u))}this.length+=n.length-o.length,n.length&&this._trigger("add",{items:n}),o.length&&this._trigger("remove",{items:o,oldData:s})}},{key:"get",value:function(t,e){if(null==this._data)return null;var i,n=null;Fa(t)||Zo(t)?(n=t,i=e):i=t;var o=Ao({},this._options,i),s=En(this._options),r=i&&En(i);return s&&r&&(o.filter=function(t){return s(t)&&r(t)}),null==n?this._data.get(o):this._data.get(n,o)}},{key:"getIds",value:function(t){if(this._data.length){var e,i=En(this._options),n=null!=t?En(t):null;return e=n?i?function(t){return i(t)&&n(t)}:n:i,this._data.getIds({filter:e,order:t&&t.order})}return[]}},{key:"forEach",value:function(t,e){if(this._data){var i,n,o=En(this._options),s=e&&En(e);n=s?o?function(t){return o(t)&&s(t)}:s:o,mn(i=this._data).call(i,t,{filter:n,order:e&&e.order})}}},{key:"map",value:function(t,e){if(this._data){var i,n,o=En(this._options),s=e&&En(e);return n=s?o?function(t){return o(t)&&s(t)}:s:o,Ln(i=this._data).call(i,t,{filter:n,order:e&&e.order})}return[]}},{key:"getDataSet",value:function(){return this._data.getDataSet()}},{key:"stream",value:function(t){var e;return this._data.stream(t||Ai({},Yr,va(e=tr(this._ids)).call(e,this._ids)))}},{key:"dispose",value:function(){var t;(null===(t=this._data)||void 0===t?void 0:t.off)&&this._data.off("*",this._listener);var e="This data view has already been disposed of.";H(this,"_data",{get:function(){throw new Error(e)},set:function(){throw new Error(e)},configurable:!1})}},{key:"_onEvent",value:function(t,e,i){if(e&&e.items&&this._data){var n=e.items,o=[],s=[],r=[],a=[],l=[],h=[];switch(t){case"add":for(var d=0,u=n.length;d<u;d++){var c=n[d];this.get(c)&&(this._ids.add(c),o.push(c))}break;case"update":for(var p=0,m=n.length;p<m;p++){var f=n[p];this.get(f)?this._ids.has(f)?(s.push(f),l.push(e.data[p]),a.push(e.oldData[p])):(this._ids.add(f),o.push(f)):this._ids.has(f)&&(this._ids.delete(f),r.push(f),h.push(e.oldData[p]))}break;case"remove":for(var g=0,v=n.length;g<v;g++){var b=n[g];this._ids.has(b)&&(this._ids.delete(b),r.push(b),h.push(e.oldData[g]))}}this.length+=o.length-r.length,o.length&&this._trigger("add",{items:o},i),s.length&&this._trigger("update",{items:s,oldData:a,data:l},i),r.length&&this._trigger("remove",{items:r,oldData:h},i)}}}]),e}(Ba),Ta="undefined"!=typeof window&&window.moment||s;function Ya(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}var Ea=/^\/?Date\((-?\d+)/i;function Ra(t,e){var i;if(void 0!==t){if(null===t)return null;if(!e)return t;if("string"!=typeof e&&!(e instanceof String))throw new Error("Type must be a string");switch(e){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return rs(t)&&!isNaN(Date.parse(t))?s(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(ss(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(s.isMoment(t))return new Date(t.valueOf());if(rs(t))return(i=Ea.exec(t))?new Date(Number(i[1])):s(new Date(t)).toDate();throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"Moment":if(ss(t))return s(t);if(t instanceof Date)return s(t.valueOf());if(s.isMoment(t))return s(t);if(rs(t))return i=Ea.exec(t),s(i?Number(i[1]):t);throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"ISODate":if(ss(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(s.isMoment(t))return t.toDate().toISOString();if(rs(t))return(i=Ea.exec(t))?new Date(Number(i[1])).toISOString():s(t).format();throw new Error("Cannot convert object of type "+us(t)+" to type ISODate");case"ASPDate":if(ss(t))return"/Date("+t+")/";if(t instanceof Date||s.isMoment(t))return"/Date("+t.valueOf()+")/";if(rs(t))return"/Date("+((i=Ea.exec(t))?new Date(Number(i[1])).valueOf():new Date(t).valueOf())+")/";throw new Error("Cannot convert object of type "+us(t)+" to type ASPDate");default:throw new Error("Unknown type ".concat(e))}}}function Ua(t){var e,i,n,o,s,r,a=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{start:"Date",end:"Date"},l=t._idProp,h=new Aa({fieldId:l}),d=Ln(e=_a(t)).call(e,(function(t){var e;return da(e=Yi(t)).call(e,(function(e,i){return e[i]=Ra(t[i],a[i]),e}),{})})).to(h);return d.all().start(),{add:function(){var e;return(e=t.getDataSet()).add.apply(e,arguments)},remove:function(){var e;return(e=t.getDataSet()).remove.apply(e,arguments)},update:function(){var e;return(e=t.getDataSet()).update.apply(e,arguments)},updateOnly:function(){var e;return(e=t.getDataSet()).updateOnly.apply(e,arguments)},forEach:va(i=mn(h)).call(i,h),get:va(n=h.get).call(n,h),getIds:va(o=h.getIds).call(o,h),off:va(s=h.off).call(s,h),on:va(r=h.on).call(r,h),get length(){return h.length},idProp:l,type:a,rawDS:t,coercedDS:h,dispose:function(){return d.stop()}}}var za=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=Ya(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=Ya(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},Is,{convert:Ra}),Va=ji.trim,ja=a.parseFloat,Ha=1/ja(Ei+"-0")!=-1/0?function(t){var e=Va(String(t)),i=ja(e);return 0===i&&"-"==e.charAt(0)?-0:i}:ja;V({global:!0,forced:parseFloat!=Ha},{parseFloat:Ha});var Pa=W.parseFloat,Ja=function(){function t(e,i){ba(this,t),this.options=null,this.props=null}return xa(t,[{key:"setOptions",value:function(t){t&&za.extend(this.options,t)}},{key:"redraw",value:function(){return!1}},{key:"destroy",value:function(){}},{key:"_isResized",value:function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t}}]),t}();V({target:"String",proto:!0},{repeat:"".repeat||function(t){var e=String(b(this)),i="",n=K(t);if(n<0||n==1/0)throw RangeError("Wrong number of repetitions");for(;n>0;(n>>>=1)&&(e+=e))1&n&&(i+=e);return i}});var Ka=Pi("String").repeat,qa=String.prototype,$a=function(t){var e=t.repeat;return"string"==typeof t||t===qa||t instanceof String&&e===qa.repeat?Ka:e};function tl(t,e,i){if(i&&!Zo(i))return tl(t,e,[i]);if(e.hiddenDates=[],i&&1==Zo(i)){for(var n,o=0;o<i.length;o++)if(void 0===$a(i[o])){var s={};s.start=t(i[o].start).toDate().valueOf(),s.end=t(i[o].end).toDate().valueOf(),e.hiddenDates.push(s)}wr(n=e.hiddenDates).call(n,(function(t,e){return t.start-e.start}))}}function el(t,e,i){if(i&&!Zo(i))return el(t,e,[i]);if(i&&void 0!==e.domProps.centerContainer.width){tl(t,e,i);for(var n=t(e.range.start),o=t(e.range.end),s=(e.range.end-e.range.start)/e.domProps.centerContainer.width,r=0;r<i.length;r++)if(void 0!==$a(i[r])){var a=t(i[r].start),l=t(i[r].end);if("Invalid Date"==a._d)throw new Error("Supplied start date is not valid: ".concat(i[r].start));if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: ".concat(i[r].end));if(l-a>=4*s){var h=0,d=o.clone();switch($a(i[r])){case"daily":a.day()!=l.day()&&(h=1),a.dayOfYear(n.dayOfYear()),a.year(n.year()),a.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-h,"days"),d.add(1,"weeks");break;case"weekly":var u=l.diff(a,"days"),c=a.day();a.date(n.date()),a.month(n.month()),a.year(n.year()),l=a.clone(),a.day(c),l.day(c),l.add(u,"days"),a.subtract(1,"weeks"),l.subtract(1,"weeks"),d.add(1,"weeks");break;case"monthly":a.month()!=l.month()&&(h=1),a.month(n.month()),a.year(n.year()),a.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(h,"months"),d.add(1,"months");break;case"yearly":a.year()!=l.year()&&(h=1),a.year(n.year()),a.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(h,"years"),d.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",$a(i[r]))}for(;a<d;)switch(e.hiddenDates.push({start:a.valueOf(),end:l.valueOf()}),$a(i[r])){case"daily":a.add(1,"days"),l.add(1,"days");break;case"weekly":a.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":a.add(1,"months"),l.add(1,"months");break;case"yearly":a.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",$a(i[r]))}e.hiddenDates.push({start:a.valueOf(),end:l.valueOf()})}}il(e);var p=cl(e.range.start,e.hiddenDates),m=cl(e.range.end,e.hiddenDates),f=e.range.start,g=e.range.end;1==p.hidden&&(f=1==e.range.startToFront?p.startDate-1:p.endDate+1),1==m.hidden&&(g=1==e.range.endToFront?m.startDate-1:m.endDate+1),1!=p.hidden&&1!=m.hidden||e.range._applyRange(f,g)}}function il(t){for(var e,i=t.hiddenDates,n=[],o=0;o<i.length;o++)for(var s=0;s<i.length;s++)o!=s&&1!=i[s].remove&&1!=i[o].remove&&(i[s].start>=i[o].start&&i[s].end<=i[o].end?i[s].remove=!0:i[s].start>=i[o].start&&i[s].start<=i[o].end?(i[o].end=i[s].end,i[s].remove=!0):i[s].end>=i[o].start&&i[s].end<=i[o].end&&(i[o].start=i[s].start,i[s].remove=!0));for(o=0;o<i.length;o++)!0!==i[o].remove&&n.push(i[o]);t.hiddenDates=n,wr(e=t.hiddenDates).call(e,(function(t,e){return t.start-e.start}))}function nl(t,e,i){for(var n=!1,o=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(o>=r&&o<a){n=!0;break}}if(1==n&&o<e._end.valueOf()&&o!=i){var l=t(i),h=t(a);l.year()!=h.year()?e.switchedYear=!0:l.month()!=h.month()?e.switchedMonth=!0:l.dayOfYear()!=h.dayOfYear()&&(e.switchedDay=!0),e.current=h}}function ol(t,e,i){var n;if(0==t.body.hiddenDates.length)return n=t.range.conversion(i),(e.valueOf()-n.offset)*n.scale;var o=cl(e,t.body.hiddenDates);1==o.hidden&&(e=o.startDate);var s=rl(t.body.hiddenDates,t.range.start,t.range.end);if(e<t.range.start){n=t.range.conversion(i,s);var r=al(t.body.hiddenDates,e,n.offset);return e=t.options.moment(e).toDate().valueOf(),e+=r,-(n.offset-e.valueOf())*n.scale}if(e>t.range.end){var a={start:t.range.start,end:e};return e=ll(t.options.moment,t.body.hiddenDates,a,e),n=t.range.conversion(i,s),(e.valueOf()-n.offset)*n.scale}return e=ll(t.options.moment,t.body.hiddenDates,t.range,e),n=t.range.conversion(i,s),(e.valueOf()-n.offset)*n.scale}function sl(t,e,i){if(0==t.body.hiddenDates.length){var n=t.range.conversion(i);return new Date(e/n.scale+n.offset)}var o=rl(t.body.hiddenDates,t.range.start,t.range.end),s=(t.range.end-t.range.start-o)*e/i,r=dl(t.body.hiddenDates,t.range,s);return new Date(r+s+t.range.start)}function rl(t,e,i){for(var n=0,o=0;o<t.length;o++){var s=t[o].start,r=t[o].end;s>=e&&r<i&&(n+=r-s)}return n}function al(t,e,i){for(var n=0,o=0;o<t.length;o++){var s=t[o].start,r=t[o].end;s>=e&&r<=i&&(n+=r-s)}return n}function ll(t,e,i,n){return n=t(n).toDate().valueOf(),n-=hl(t,e,i,n)}function hl(t,e,i,n){var o=0;n=t(n).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&n>=a&&(o+=a-r)}return o}function dl(t,e,i){for(var n=0,o=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,l=t[r].end;if(a>=e.start&&l<e.end){if(o+=a-s,s=l,o>=i)break;n+=l-a}}return n}function ul(t,e,i,n){var o=cl(e,t);return 1==o.hidden?i<0?1==n?o.startDate-(o.endDate-e)-1:o.startDate-1:1==n?o.endDate+(e-o.startDate)+1:o.endDate+1:e}function cl(t,e){for(var i=0;i<e.length;i++){var n=e[i].start,o=e[i].end;if(t>=n&&t<o)return{hidden:!0,startDate:n,endDate:o}}return{hidden:!1,startDate:n,endDate:o}}var pl=Object.freeze({__proto__:null,convertHiddenOptions:tl,updateHiddenDates:el,removeDuplicates:il,printDates:function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},stepOverHiddenDates:nl,toScreen:ol,toTime:sl,getHiddenDurationBetween:rl,getHiddenDurationBeforeStart:al,correctTimeForHidden:ll,getHiddenDurationBefore:hl,getAccumulatedHiddenDuration:dl,snapAwayFromHidden:ul,getIsHidden:cl}),ml=function(t){function e(t,i){var n,o,s,r,a,l,h,d;ba(this,e),d=ar(this,dr(e).call(this));var u=Ta().hours(0).minutes(0).seconds(0).milliseconds(0),c=u.clone().add(-3,"days").valueOf(),p=u.clone().add(3,"days").valueOf();return d.millisecondsPerPixelCache=void 0,void 0===i?(d.start=c,d.end=p):(d.start=i.start||c,d.end=i.end||p),d.rolling=!1,d.body=t,d.deltaDifference=0,d.scaleOffset=0,d.startToFront=!1,d.endToFront=!0,d.defaultOptions={rtl:!1,start:null,end:null,moment:Ta,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10,rollingMode:{follow:!1,offset:.5}},d.options=za.extend({},d.defaultOptions),d.props={touch:{}},d.animationTimer=null,d.body.emitter.on("panstart",va(n=d._onDragStart).call(n,Bs(d))),d.body.emitter.on("panmove",va(o=d._onDrag).call(o,Bs(d))),d.body.emitter.on("panend",va(s=d._onDragEnd).call(s,Bs(d))),d.body.emitter.on("mousewheel",va(r=d._onMouseWheel).call(r,Bs(d))),d.body.emitter.on("touch",va(a=d._onTouch).call(a,Bs(d))),d.body.emitter.on("pinch",va(l=d._onPinch).call(l,Bs(d))),d.body.dom.rollingModeBtn.addEventListener("click",va(h=d.startRolling).call(h,Bs(d))),d.setOptions(i),d}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){if(t){za.selectiveExtend(["animation","direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","zoomFriction","rtl","showCurrentTime","rollingMode","horizontalScroll"],this.options,t),t.rollingMode&&t.rollingMode.follow&&this.startRolling(),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}}},{key:"startRolling",value:function(){var t=this;!function e(){t.stopRolling(),t.rolling=!0;var i=t.end-t.start,n=za.convert(new Date,"Date").valueOf(),o=t.options.rollingMode&&t.options.rollingMode.offset||.5,s=n-i*o,r=n+i*(1-o);t.setRange(s,r,{animation:!1}),(i=1/t.conversion(t.body.domProps.center.width).scale/10)<30&&(i=30),i>1e3&&(i=1e3),t.body.dom.rollingModeBtn.style.visibility="hidden",t.currentTimeTimer=qr(e,i)}()}},{key:"stopRolling",value:function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),this.rolling=!1,this.body.dom.rollingModeBtn.style.visibility="visible")}},{key:"setRange",value:function(t,e,i,n,o){i||(i={}),!0!==i.byUser&&(i.byUser=!1);var s=this,r=null!=t?za.convert(t,"Date").valueOf():null,a=null!=e?za.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),this.millisecondsPerPixelCache=void 0,i.animation){var l,h=this.start,d=this.end,u="object"===zo(i.animation)&&"duration"in i.animation?i.animation.duration:500,c="object"===zo(i.animation)&&"easingFunction"in i.animation?i.animation.easingFunction:"easeInOutQuad",p=za.easingFunctions[c];if(!p)throw new Error(Pn(l="Unknown easing function ".concat(Vs(c),". Choose from: ")).call(l,Yi(za.easingFunctions).join(", ")));var m=Jo(),f=!1;return function t(){if(!s.props.touch.dragging){var e=Jo()-m,l=p(e/u),c=e>u,v=c||null===r?r:h+(r-h)*l,b=c||null===a?a:d+(a-d)*l;g=s._applyRange(v,b),el(s.options.moment,s.body,s.options.hiddenDates),f=f||g;var y={start:new Date(s.start),end:new Date(s.end),byUser:i.byUser,event:i.event};if(o&&o(l,g,c),g&&s.body.emitter.emit("rangechange",y),c){if(f&&(s.body.emitter.emit("rangechanged",y),n))return n()}else s.animationTimer=qr(t,20)}}()}var g=this._applyRange(r,a);if(el(this.options.moment,this.body,this.options.hiddenDates),g){var v={start:new Date(this.start),end:new Date(this.end),byUser:i.byUser,event:i.event};if(this.body.emitter.emit("rangechange",v),clearTimeout(s.timeoutID),s.timeoutID=qr((function(){s.body.emitter.emit("rangechanged",v)}),200),n)return n()}}},{key:"getMillisecondsPerPixel",value:function(){return void 0===this.millisecondsPerPixelCache&&(this.millisecondsPerPixelCache=(this.end-this.start)/this.body.dom.center.clientWidth),this.millisecondsPerPixelCache}},{key:"_cancelAnimation",value:function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)}},{key:"_applyRange",value:function(t,e){var i,n=null!=t?za.convert(t,"Date").valueOf():this.start,o=null!=e?za.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?za.convert(this.options.max,"Date").valueOf():null,r=null!=this.options.min?za.convert(this.options.min,"Date").valueOf():null;if(isNaN(n)||null===n)throw new Error('Invalid start "'.concat(t,'"'));if(isNaN(o)||null===o)throw new Error('Invalid end "'.concat(e,'"'));if(o<n&&(o=n),null!==r&&n<r&&(n+=i=r-n,o+=i,null!=s&&o>s&&(o=s)),null!==s&&o>s&&(n-=i=o-s,o-=i,null!=r&&n<r&&(n=r)),null!==this.options.zoomMin){var a=Pa(this.options.zoomMin);if(a<0&&(a=0),o-n<a){this.end-this.start===a&&n>=this.start-.5&&o<=this.end?(n=this.start,o=this.end):(n-=(i=a-(o-n))/2,o+=i/2)}}if(null!==this.options.zoomMax){var l=Pa(this.options.zoomMax);l<0&&(l=0),o-n>l&&(this.end-this.start===l&&n<this.start&&o>this.end?(n=this.start,o=this.end):(n+=(i=o-n-l)/2,o-=i/2))}var h=this.start!=n||this.end!=o;return n>=this.start&&n<=this.end||o>=this.start&&o<=this.end||this.start>=n&&this.start<=o||this.end>=n&&this.end<=o||this.body.emitter.emit("checkRangedItems"),this.start=n,this.end=o,h}},{key:"getRange",value:function(){return{start:this.start,end:this.end}}},{key:"conversion",value:function(t,i){return e.conversion(this.start,this.end,t,i)}},{key:"_onDragStart",value:function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.stopRolling(),this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))}},{key:"_onDrag",value:function(t){if(t&&this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;fl(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var n=this.props.touch.end-this.props.touch.start;n-=rl(this.body.hiddenDates,this.start,this.end);var o,s="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;o=this.options.rtl?i/s*n:-i/s*n;var r=this.props.touch.start+o,a=this.props.touch.end+o,l=ul(this.body.hiddenDates,r,this.previousDelta-i,!0),h=ul(this.body.hiddenDates,a,this.previousDelta-i,!0);if(l!=r||h!=a)return this.deltaDifference+=i,this.props.touch.start=l,this.props.touch.end=h,void this._onDrag(t);this.previousDelta=i,this._applyRange(r,a);var d=new Date(this.start),u=new Date(this.end);this.body.emitter.emit("rangechange",{start:d,end:u,byUser:!0,event:t}),this.body.emitter.emit("panmove")}}},{key:"_onDragEnd",value:function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0,event:t}))}},{key:"_onMouseWheel",value:function(t){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail?e=-t.detail/3:t.deltaY&&(e=-t.deltaY/3),!(this.options.zoomKey&&!t[this.options.zoomKey]&&this.options.zoomable||!this.options.zoomable&&this.options.moveable)&&this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&e){var i,n,o=this.options.zoomFriction||5;if(i=e<0?1-e/o:1/(1+e/o),this.rolling){var s=this.options.rollingMode&&this.options.rollingMode.offset||.5;n=this.start+(this.end-this.start)*s}else{var r=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center);n=this._pointerToDate(r)}this.zoom(i,n,e,t),t.preventDefault()}}},{key:"_onTouch",value:function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.props.touch.centerDate=null,this.scaleOffset=0,this.deltaDifference=0,za.preventDefault(t)}},{key:"_onPinch",value:function(t){if(this.options.zoomable&&this.options.moveable){za.preventDefault(t),this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center),this.props.touch.centerDate=this._pointerToDate(this.props.touch.center)),this.stopRolling();var e=1/(t.scale+this.scaleOffset),i=this.props.touch.centerDate,n=rl(this.body.hiddenDates,this.start,this.end),o=hl(this.options.moment,this.body.hiddenDates,this,i),s=n-o,r=i-o+(this.props.touch.start-(i-o))*e,a=i+s+(this.props.touch.end-(i+s))*e;this.startToFront=1-e<=0,this.endToFront=e-1<=0;var l=ul(this.body.hiddenDates,r,1-e,!0),h=ul(this.body.hiddenDates,a,e-1,!0);l==r&&h==a||(this.props.touch.start=l,this.props.touch.end=h,this.scaleOffset=1-t.scale,r=l,a=h);var d={animation:!1,byUser:!0,event:t};this.setRange(r,a,d),this.startToFront=!1,this.endToFront=!0}}},{key:"_isInsideRange",value:function(t){var e=t.center?t.center.x:t.clientX,i=this.body.dom.centerContainer.getBoundingClientRect(),n=this.options.rtl?e-i.left:i.right-e,o=this.body.util.toTime(n);return o>=this.start&&o<=this.end}},{key:"_pointerToDate",value:function(t){var e,i=this.options.direction;if(fl(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var n=this.body.domProps.center.height;return e=this.conversion(n),t.y/e.scale+e.offset}},{key:"getPointer",value:function(t,e){var i=e.getBoundingClientRect();return this.options.rtl?{x:i.right-t.x,y:t.y-i.top}:{x:t.x-i.left,y:t.y-i.top}}},{key:"zoom",value:function(t,e,i,n){null==e&&(e=(this.start+this.end)/2);var o=rl(this.body.hiddenDates,this.start,this.end),s=hl(this.options.moment,this.body.hiddenDates,this,e),r=o-s,a=e-s+(this.start-(e-s))*t,l=e+r+(this.end-(e+r))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var h=ul(this.body.hiddenDates,a,i,!0),d=ul(this.body.hiddenDates,l,-i,!0);h==a&&d==l||(a=h,l=d);var u={animation:!1,byUser:!0,event:n};this.setRange(a,l,u),this.startToFront=!1,this.endToFront=!0}},{key:"move",value:function(t){var e=this.end-this.start,i=this.start+e*t,n=this.end+e*t;this.start=i,this.end=n}},{key:"moveTo",value:function(t){var e=(this.start+this.end)/2-t,i=this.start-e,n=this.end-e;this.setRange(i,n,{animation:!1,byUser:!0,event:null})}}],[{key:"conversion",value:function(t,e,i,n){return void 0===n&&(n=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-n)}:{offset:0,scale:1}}}]),e}(Ja);function fl(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'.concat(t,'". Choose "horizontal" or "vertical".'))}var gl=W.setInterval,vl=n((function(t){function e(t){if(t)return function(t){for(var i in e.prototype)t[i]=e.prototype[i];return t}(t)}t.exports=e,e.prototype.on=e.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks["$"+t]=this._callbacks["$"+t]||[]).push(e),this},e.prototype.once=function(t,e){function i(){this.off(t,i),e.apply(this,arguments)}return i.fn=e,this.on(t,i),this},e.prototype.off=e.prototype.removeListener=e.prototype.removeAllListeners=e.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i,n=this._callbacks["$"+t];if(!n)return this;if(1==arguments.length)return delete this._callbacks["$"+t],this;for(var o=0;o<n.length;o++)if((i=n[o])===e||i.fn===e){n.splice(o,1);break}return 0===n.length&&delete this._callbacks["$"+t],this},e.prototype.emit=function(t){this._callbacks=this._callbacks||{};for(var e=new Array(arguments.length-1),i=this._callbacks["$"+t],n=1;n<arguments.length;n++)e[n-1]=arguments[n];if(i){n=0;for(var o=(i=i.slice(0)).length;n<o;++n)i[n].apply(this,e)}return this},e.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks["$"+t]||[]},e.prototype.hasListeners=function(t){return!!this.listeners(t).length}})),bl=n((function(t,e){var i;t.exports=(i=null,function t(e,n){var o=n||{preventDefault:!1};if(e.Manager){var s=e,r=function(e,i){var n=Object.create(o);return i&&s.assign(n,i),t(new s(e,n),n)};return s.assign(r,s),r.Manager=function(e,i){var n=Object.create(o);return i&&s.assign(n,i),t(new s.Manager(e,n),n)},r}var a=Object.create(e),l=e.element;function h(t){return t.match(/[^ ]+/g)}function d(t){if("hammer.input"!==t.type){if(t.srcEvent._handled||(t.srcEvent._handled={}),t.srcEvent._handled[t.type])return;t.srcEvent._handled[t.type]=!0}var e=!1;t.stopPropagation=function(){e=!0};var n=t.srcEvent.stopPropagation.bind(t.srcEvent);"function"==typeof n&&(t.srcEvent.stopPropagation=function(){n(),t.stopPropagation()}),t.firstTarget=i;for(var o=i;o&&!e;){var s=o.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[t.type])for(var l=0;l<r.length&&!e;l++)r[l](t);o=o.parentNode}}return l.hammer||(l.hammer=[]),l.hammer.push(a),e.on("hammer.input",(function(t){!0!==o.preventDefault&&o.preventDefault!==t.pointerType||t.preventDefault(),t.isFirst&&(i=t.target)})),a._handlers={},a.on=function(t,i){return h(t).forEach((function(t){var n=a._handlers[t];n||(a._handlers[t]=n=[],e.on(t,d)),n.push(i)})),a},a.off=function(t,i){return h(t).forEach((function(t){var n=a._handlers[t];n&&((n=i?n.filter((function(t){return t!==i})):[]).length>0?a._handlers[t]=n:(e.off(t,d),delete a._handlers[t]))})),a},a.emit=function(t,n){i=n.target,e.emit(t,n)},a.destroy=function(){var t=e.element.hammer,i=t.indexOf(a);-1!==i&&t.splice(i,1),t.length||delete e.element.hammer,a._handlers={},e.destroy()},a})}));
+/*! Hammer.JS - v2.0.17-rc - 2019-12-16
+	 * http://naver.github.io/egjs
+	 *
+	 * Forked By Naver egjs
+	 * Copyright (c) hammerjs
+	 * Licensed under the MIT license */
+function yl(){return(yl=Object.assign||function(t){for(var e=1;e<arguments.length;e++){var i=arguments[e];for(var n in i)Object.prototype.hasOwnProperty.call(i,n)&&(t[n]=i[n])}return t}).apply(this,arguments)}function xl(t,e){t.prototype=Object.create(e.prototype),t.prototype.constructor=t,t.__proto__=e}function wl(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}var _l,kl="function"!=typeof Object.assign?function(t){if(null==t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var n=arguments[i];if(null!=n)for(var o in n)n.hasOwnProperty(o)&&(e[o]=n[o])}return e}:Object.assign,Dl=["","webkit","Moz","MS","ms","o"],Il="undefined"==typeof document?{style:{}}:document.createElement("div"),Cl=Math.round,Ll=Math.abs,Gl=Date.now;function Ml(t,e){for(var i,n,o=e[0].toUpperCase()+e.slice(1),s=0;s<Dl.length;){if((n=(i=Dl[s])?i+o:e)in t)return n;s++}}_l="undefined"==typeof window?{}:window;var Sl=Ml(Il.style,"touchAction"),Ql=void 0!==Sl;var Nl=function(){if(!Ql)return!1;var t={},e=_l.CSS&&_l.CSS.supports;return["auto","manipulation","pan-y","pan-x","pan-x pan-y","none"].forEach((function(i){return t[i]=!e||_l.CSS.supports("touch-action",i)})),t}(),Fl="ontouchstart"in _l,Zl=void 0!==Ml(_l,"PointerEvent"),Bl=Fl&&/mobile|tablet|ip(ad|hone|od)|android/i.test(navigator.userAgent),Ol=["x","y"],Wl=["clientX","clientY"];function Al(t,e,i){var n;if(t)if(t.forEach)t.forEach(e,i);else if(void 0!==t.length)for(n=0;n<t.length;)e.call(i,t[n],n,t),n++;else for(n in t)t.hasOwnProperty(n)&&e.call(i,t[n],n,t)}function Xl(t,e){return"function"==typeof t?t.apply(e&&e[0]||void 0,e):t}function Tl(t,e){return t.indexOf(e)>-1}var Yl=function(){function t(t,e){this.manager=t,this.set(e)}var e=t.prototype;return e.set=function(t){"compute"===t&&(t=this.compute()),Ql&&this.manager.element.style&&Nl[t]&&(this.manager.element.style[Sl]=t),this.actions=t.toLowerCase().trim()},e.update=function(){this.set(this.manager.options.touchAction)},e.compute=function(){var t=[];return Al(this.manager.recognizers,(function(e){Xl(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))})),function(t){if(Tl(t,"none"))return"none";var e=Tl(t,"pan-x"),i=Tl(t,"pan-y");return e&&i?"none":e||i?e?"pan-x":"pan-y":Tl(t,"manipulation")?"manipulation":"auto"}(t.join(" "))},e.preventDefaults=function(t){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)e.preventDefault();else{var n=this.actions,o=Tl(n,"none")&&!Nl.none,s=Tl(n,"pan-y")&&!Nl["pan-y"],r=Tl(n,"pan-x")&&!Nl["pan-x"];if(o){var a=1===t.pointers.length,l=t.distance<2,h=t.deltaTime<250;if(a&&l&&h)return}if(!r||!s)return o||s&&6&i||r&&24&i?this.preventSrc(e):void 0}},e.preventSrc=function(t){this.manager.session.prevented=!0,t.preventDefault()},t}();function El(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}function Rl(t){var e=t.length;if(1===e)return{x:Cl(t[0].clientX),y:Cl(t[0].clientY)};for(var i=0,n=0,o=0;o<e;)i+=t[o].clientX,n+=t[o].clientY,o++;return{x:Cl(i/e),y:Cl(n/e)}}function Ul(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:Cl(t.pointers[i].clientX),clientY:Cl(t.pointers[i].clientY)},i++;return{timeStamp:Gl(),pointers:e,center:Rl(e),deltaX:t.deltaX,deltaY:t.deltaY}}function zl(t,e,i){i||(i=Ol);var n=e[i[0]]-t[i[0]],o=e[i[1]]-t[i[1]];return Math.sqrt(n*n+o*o)}function Vl(t,e,i){i||(i=Ol);var n=e[i[0]]-t[i[0]],o=e[i[1]]-t[i[1]];return 180*Math.atan2(o,n)/Math.PI}function jl(t,e){return t===e?1:Ll(t)>=Ll(e)?t<0?2:4:e<0?8:16}function Hl(t,e,i){return{x:e/t||0,y:i/t||0}}function Pl(t,e){var i=t.session,n=e.pointers,o=n.length;i.firstInput||(i.firstInput=Ul(e)),o>1&&!i.firstMultiple?i.firstMultiple=Ul(e):1===o&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,l=e.center=Rl(n);e.timeStamp=Gl(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=Vl(a,l),e.distance=zl(a,l),function(t,e){var i=e.center,n=t.offsetDelta||{},o=t.prevDelta||{},s=t.prevInput||{};1!==e.eventType&&4!==s.eventType||(o=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},n=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=o.x+(i.x-n.x),e.deltaY=o.y+(i.y-n.y)}(i,e),e.offsetDirection=jl(e.deltaX,e.deltaY);var h,d,u=Hl(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=u.x,e.overallVelocityY=u.y,e.overallVelocity=Ll(u.x)>Ll(u.y)?u.x:u.y,e.scale=r?(h=r.pointers,zl((d=n)[0],d[1],Wl)/zl(h[0],h[1],Wl)):1,e.rotation=r?function(t,e){return Vl(e[1],e[0],Wl)+Vl(t[1],t[0],Wl)}(r.pointers,n):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,function(t,e){var i,n,o,s,r=t.lastInterval||e,a=e.timeStamp-r.timeStamp;if(8!==e.eventType&&(a>25||void 0===r.velocity)){var l=e.deltaX-r.deltaX,h=e.deltaY-r.deltaY,d=Hl(a,l,h);n=d.x,o=d.y,i=Ll(d.x)>Ll(d.y)?d.x:d.y,s=jl(l,h),t.lastInterval=e}else i=r.velocity,n=r.velocityX,o=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=n,e.velocityY=o,e.direction=s}(i,e);var c,p=t.element,m=e.srcEvent;El(c=m.composedPath?m.composedPath()[0]:m.path?m.path[0]:m.target,p)&&(p=c),e.target=p}function Jl(t,e,i){var n=i.pointers.length,o=i.changedPointers.length,s=1&e&&n-o==0,r=12&e&&n-o==0;i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,Pl(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function Kl(t){return t.trim().split(/\s+/g)}function ql(t,e,i){Al(Kl(e),(function(e){t.addEventListener(e,i,!1)}))}function $l(t,e,i){Al(Kl(e),(function(e){t.removeEventListener(e,i,!1)}))}function th(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||window}var eh=function(){function t(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){Xl(t.options.enable,[t])&&i.handler(e)},this.init()}var e=t.prototype;return e.handler=function(){},e.init=function(){this.evEl&&ql(this.element,this.evEl,this.domHandler),this.evTarget&&ql(this.target,this.evTarget,this.domHandler),this.evWin&&ql(th(this.element),this.evWin,this.domHandler)},e.destroy=function(){this.evEl&&$l(this.element,this.evEl,this.domHandler),this.evTarget&&$l(this.target,this.evTarget,this.domHandler),this.evWin&&$l(th(this.element),this.evWin,this.domHandler)},t}();function ih(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var n=0;n<t.length;){if(i&&t[n][i]==e||!i&&t[n]===e)return n;n++}return-1}var nh={pointerdown:1,pointermove:2,pointerup:4,pointercancel:8,pointerout:8},oh={2:"touch",3:"pen",4:"mouse",5:"kinect"},sh="pointerdown",rh="pointermove pointerup pointercancel";_l.MSPointerEvent&&!_l.PointerEvent&&(sh="MSPointerDown",rh="MSPointerMove MSPointerUp MSPointerCancel");var ah=function(t){function e(){var i,n=e.prototype;return n.evEl=sh,n.evWin=rh,(i=t.apply(this,arguments)||this).store=i.manager.session.pointerEvents=[],i}return xl(e,t),e.prototype.handler=function(t){var e=this.store,i=!1,n=t.type.toLowerCase().replace("ms",""),o=nh[n],s=oh[t.pointerType]||t.pointerType,r="touch"===s,a=ih(e,t.pointerId,"pointerId");1&o&&(0===t.button||r)?a<0&&(e.push(t),a=e.length-1):12&o&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,o,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))},e}(eh);function lh(t){return Array.prototype.slice.call(t,0)}function hh(t,e,i){for(var n=[],o=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];ih(o,r)<0&&n.push(t[s]),o[s]=r,s++}return i&&(n=e?n.sort((function(t,i){return t[e]>i[e]})):n.sort()),n}var dh={touchstart:1,touchmove:2,touchend:4,touchcancel:8},uh=function(t){function e(){var i;return e.prototype.evTarget="touchstart touchmove touchend touchcancel",(i=t.apply(this,arguments)||this).targetIds={},i}return xl(e,t),e.prototype.handler=function(t){var e=dh[t.type],i=ch.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})},e}(eh);function ch(t,e){var i,n,o=lh(t.touches),s=this.targetIds;if(3&e&&1===o.length)return s[o[0].identifier]=!0,[o,o];var r=lh(t.changedTouches),a=[],l=this.target;if(n=o.filter((function(t){return El(t.target,l)})),1===e)for(i=0;i<n.length;)s[n[i].identifier]=!0,i++;for(i=0;i<r.length;)s[r[i].identifier]&&a.push(r[i]),12&e&&delete s[r[i].identifier],i++;return a.length?[hh(n.concat(a),"identifier",!0),a]:void 0}var ph={mousedown:1,mousemove:2,mouseup:4},mh=function(t){function e(){var i,n=e.prototype;return n.evEl="mousedown",n.evWin="mousemove mouseup",(i=t.apply(this,arguments)||this).pressed=!1,i}return xl(e,t),e.prototype.handler=function(t){var e=ph[t.type];1&e&&0===t.button&&(this.pressed=!0),2&e&&1!==t.which&&(e=4),this.pressed&&(4&e&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:"mouse",srcEvent:t}))},e}(eh);function fh(t){var e=t.changedPointers[0];if(e.identifier===this.primaryTouch){var i={x:e.clientX,y:e.clientY},n=this.lastTouches;this.lastTouches.push(i);setTimeout((function(){var t=n.indexOf(i);t>-1&&n.splice(t,1)}),2500)}}function gh(t,e){1&t?(this.primaryTouch=e.changedPointers[0].identifier,fh.call(this,e)):12&t&&fh.call(this,e)}function vh(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,n=0;n<this.lastTouches.length;n++){var o=this.lastTouches[n],s=Math.abs(e-o.x),r=Math.abs(i-o.y);if(s<=25&&r<=25)return!0}return!1}var bh=function(){return function(t){function e(e,i){var n;return(n=t.call(this,e,i)||this).handler=function(t,e,i){var o="touch"===i.pointerType,s="mouse"===i.pointerType;if(!(s&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(o)gh.call(wl(wl(n)),e,i);else if(s&&vh.call(wl(wl(n)),i))return;n.callback(t,e,i)}},n.touch=new uh(n.manager,n.handler),n.mouse=new mh(n.manager,n.handler),n.primaryTouch=null,n.lastTouches=[],n}return xl(e,t),e.prototype.destroy=function(){this.touch.destroy(),this.mouse.destroy()},e}(eh)}();function yh(t,e,i){return!!Array.isArray(t)&&(Al(t,i[e],i),!0)}var xh=1;function wh(t,e){var i=e.manager;return i?i.get(t):t}function _h(t){return 16&t?"cancel":8&t?"end":4&t?"move":2&t?"start":""}var kh=function(){function t(t){void 0===t&&(t={}),this.options=yl({enable:!0},t),this.id=xh++,this.manager=null,this.state=1,this.simultaneous={},this.requireFail=[]}var e=t.prototype;return e.set=function(t){return kl(this.options,t),this.manager&&this.manager.touchAction.update(),this},e.recognizeWith=function(t){if(yh(t,"recognizeWith",this))return this;var e=this.simultaneous;return e[(t=wh(t,this)).id]||(e[t.id]=t,t.recognizeWith(this)),this},e.dropRecognizeWith=function(t){return yh(t,"dropRecognizeWith",this)||(t=wh(t,this),delete this.simultaneous[t.id]),this},e.requireFailure=function(t){if(yh(t,"requireFailure",this))return this;var e=this.requireFail;return-1===ih(e,t=wh(t,this))&&(e.push(t),t.requireFailure(this)),this},e.dropRequireFailure=function(t){if(yh(t,"dropRequireFailure",this))return this;t=wh(t,this);var e=ih(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},e.hasRequireFailures=function(){return this.requireFail.length>0},e.canRecognizeWith=function(t){return!!this.simultaneous[t.id]},e.emit=function(t){var e=this,i=this.state;function n(i){e.manager.emit(i,t)}i<8&&n(e.options.event+_h(i)),n(e.options.event),t.additionalEvent&&n(t.additionalEvent),i>=8&&n(e.options.event+_h(i))},e.tryEmit=function(t){if(this.canEmit())return this.emit(t);this.state=32},e.canEmit=function(){for(var t=0;t<this.requireFail.length;){if(!(33&this.requireFail[t].state))return!1;t++}return!0},e.recognize=function(t){var e=kl({},t);if(!Xl(this.options.enable,[this,e]))return this.reset(),void(this.state=32);56&this.state&&(this.state=1),this.state=this.process(e),30&this.state&&this.tryEmit(e)},e.process=function(t){},e.getTouchAction=function(){},e.reset=function(){},t}(),Dh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},e))||this).pTime=!1,i.pCenter=!1,i._timer=null,i._input=null,i.count=0,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["manipulation"]},i.process=function(t){var e=this,i=this.options,n=t.pointers.length===i.pointers,o=t.distance<i.threshold,s=t.deltaTime<i.time;if(this.reset(),1&t.eventType&&0===this.count)return this.failTimeout();if(o&&s&&n){if(4!==t.eventType)return this.failTimeout();var r=!this.pTime||t.timeStamp-this.pTime<i.interval,a=!this.pCenter||zl(this.pCenter,t.center)<i.posThreshold;if(this.pTime=t.timeStamp,this.pCenter=t.center,a&&r?this.count+=1:this.count=1,this._input=t,0===this.count%i.taps)return this.hasRequireFailures()?(this._timer=setTimeout((function(){e.state=8,e.tryEmit()}),i.interval),2):8}return 32},i.failTimeout=function(){var t=this;return this._timer=setTimeout((function(){t.state=32}),this.options.interval),32},i.reset=function(){clearTimeout(this._timer)},i.emit=function(){8===this.state&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))},e}(kh),Ih=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({pointers:1},e))||this}xl(e,t);var i=e.prototype;return i.attrTest=function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},i.process=function(t){var e=this.state,i=t.eventType,n=6&e,o=this.attrTest(t);return n&&(8&i||!o)?16|e:n||o?4&i?8|e:2&e?4|e:2:32},e}(kh);function Ch(t){return 16===t?"down":8===t?"up":2===t?"left":4===t?"right":""}var Lh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"pan",threshold:10,pointers:1,direction:30},e))||this).pX=null,i.pY=null,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){var t=this.options.direction,e=[];return 6&t&&e.push("pan-y"),24&t&&e.push("pan-x"),e},i.directionTest=function(t){var e=this.options,i=!0,n=t.distance,o=t.direction,s=t.deltaX,r=t.deltaY;return o&e.direction||(6&e.direction?(o=0===s?1:s<0?2:4,i=s!==this.pX,n=Math.abs(t.deltaX)):(o=0===r?1:r<0?8:16,i=r!==this.pY,n=Math.abs(t.deltaY))),t.direction=o,i&&n>e.threshold&&o&e.direction},i.attrTest=function(t){return Ih.prototype.attrTest.call(this,t)&&(2&this.state||!(2&this.state)&&this.directionTest(t))},i.emit=function(e){this.pX=e.deltaX,this.pY=e.deltaY;var i=Ch(e.direction);i&&(e.additionalEvent=this.options.event+i),t.prototype.emit.call(this,e)},e}(Ih),Gh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"swipe",threshold:10,velocity:.3,direction:30,pointers:1},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return Lh.prototype.getTouchAction.call(this)},i.attrTest=function(e){var i,n=this.options.direction;return 30&n?i=e.overallVelocity:6&n?i=e.overallVelocityX:24&n&&(i=e.overallVelocityY),t.prototype.attrTest.call(this,e)&&n&e.offsetDirection&&e.distance>this.options.threshold&&e.maxPointers===this.options.pointers&&Ll(i)>this.options.velocity&&4&e.eventType},i.emit=function(t){var e=Ch(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)},e}(Ih),Mh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"pinch",threshold:0,pointers:2},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["none"]},i.attrTest=function(e){return t.prototype.attrTest.call(this,e)&&(Math.abs(e.scale-1)>this.options.threshold||2&this.state)},i.emit=function(e){if(1!==e.scale){var i=e.scale<1?"in":"out";e.additionalEvent=this.options.event+i}t.prototype.emit.call(this,e)},e}(Ih),Sh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"rotate",threshold:0,pointers:2},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["none"]},i.attrTest=function(e){return t.prototype.attrTest.call(this,e)&&(Math.abs(e.rotation)>this.options.threshold||2&this.state)},e}(Ih),Qh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"press",pointers:1,time:251,threshold:9},e))||this)._timer=null,i._input=null,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["auto"]},i.process=function(t){var e=this,i=this.options,n=t.pointers.length===i.pointers,o=t.distance<i.threshold,s=t.deltaTime>i.time;if(this._input=t,!o||!n||12&t.eventType&&!s)this.reset();else if(1&t.eventType)this.reset(),this._timer=setTimeout((function(){e.state=8,e.tryEmit()}),i.time);else if(4&t.eventType)return 8;return 32},i.reset=function(){clearTimeout(this._timer)},i.emit=function(t){8===this.state&&(t&&4&t.eventType?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=Gl(),this.manager.emit(this.options.event,this._input)))},e}(kh),Nh={domEvents:!1,touchAction:"compute",enable:!0,inputTarget:null,inputClass:null,cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}},Fh=[[Sh,{enable:!1}],[Mh,{enable:!1},["rotate"]],[Gh,{direction:6}],[Lh,{direction:6},["swipe"]],[Dh],[Dh,{event:"doubletap",taps:2},["tap"]],[Qh]];function Zh(t,e){var i,n=t.element;n.style&&(Al(t.options.cssProps,(function(o,s){i=Ml(n.style,s),e?(t.oldCssProps[i]=n.style[i],n.style[i]=o):n.style[i]=t.oldCssProps[i]||""})),e||(t.oldCssProps={}))}var Bh=function(){function t(t,e){var i,n=this;this.options=kl({},Nh,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.oldCssProps={},this.element=t,this.input=new((i=this).options.inputClass||(Zl?ah:Bl?uh:Fl?bh:mh))(i,Jl),this.touchAction=new Yl(this,this.options.touchAction),Zh(this,!0),Al(this.options.recognizers,(function(t){var e=n.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])}),this)}var e=t.prototype;return e.set=function(t){return kl(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},e.stop=function(t){this.session.stopped=t?2:1},e.recognize=function(t){var e=this.session;if(!e.stopped){var i;this.touchAction.preventDefaults(t);var n=this.recognizers,o=e.curRecognizer;(!o||o&&8&o.state)&&(e.curRecognizer=null,o=null);for(var s=0;s<n.length;)i=n[s],2===e.stopped||o&&i!==o&&!i.canRecognizeWith(o)?i.reset():i.recognize(t),!o&&14&i.state&&(e.curRecognizer=i,o=i),s++}},e.get=function(t){if(t instanceof kh)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event===t)return e[i];return null},e.add=function(t){if(yh(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},e.remove=function(t){if(yh(t,"remove",this))return this;var e=this.get(t);if(t){var i=this.recognizers,n=ih(i,e);-1!==n&&(i.splice(n,1),this.touchAction.update())}return this},e.on=function(t,e){if(void 0===t||void 0===e)return this;var i=this.handlers;return Al(Kl(t),(function(t){i[t]=i[t]||[],i[t].push(e)})),this},e.off=function(t,e){if(void 0===t)return this;var i=this.handlers;return Al(Kl(t),(function(t){e?i[t]&&i[t].splice(ih(i[t],e),1):delete i[t]})),this},e.emit=function(t,e){this.options.domEvents&&function(t,e){var i=document.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var n=0;n<i.length;)i[n](e),n++}},e.destroy=function(){this.element&&Zh(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null},t}(),Oh={touchstart:1,touchmove:2,touchend:4,touchcancel:8},Wh=function(t){function e(){var i,n=e.prototype;return n.evTarget="touchstart",n.evWin="touchstart touchmove touchend touchcancel",(i=t.apply(this,arguments)||this).started=!1,i}return xl(e,t),e.prototype.handler=function(t){var e=Oh[t.type];if(1===e&&(this.started=!0),this.started){var i=Ah.call(this,t,e);12&e&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}},e}(eh);function Ah(t,e){var i=lh(t.touches),n=lh(t.changedTouches);return 12&e&&(i=hh(i.concat(n),"identifier",!0)),[i,n]}function Xh(t,e,i){var n="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",o=window.console&&(window.console.warn||window.console.log);return o&&o.call(window.console,n,i),t.apply(this,arguments)}}var Th=Xh((function(t,e,i){for(var n=Object.keys(e),o=0;o<n.length;)(!i||i&&void 0===t[n[o]])&&(t[n[o]]=e[n[o]]),o++;return t}),"extend","Use `assign`."),Yh=Xh((function(t,e){return Th(t,e,!0)}),"merge","Use `assign`.");function Eh(t,e,i){var n,o=e.prototype;(n=t.prototype=Object.create(o)).constructor=t,n._super=o,i&&kl(n,i)}function Rh(t,e){return function(){return t.apply(e,arguments)}}var Uh,zh=function(){var t=function(t,e){return void 0===e&&(e={}),new Bh(t,yl({recognizers:Fh.concat()},e))};return t.VERSION="2.0.17-rc",t.DIRECTION_ALL=30,t.DIRECTION_DOWN=16,t.DIRECTION_LEFT=2,t.DIRECTION_RIGHT=4,t.DIRECTION_UP=8,t.DIRECTION_HORIZONTAL=6,t.DIRECTION_VERTICAL=24,t.DIRECTION_NONE=1,t.DIRECTION_DOWN=16,t.INPUT_START=1,t.INPUT_MOVE=2,t.INPUT_END=4,t.INPUT_CANCEL=8,t.STATE_POSSIBLE=1,t.STATE_BEGAN=2,t.STATE_CHANGED=4,t.STATE_ENDED=8,t.STATE_RECOGNIZED=8,t.STATE_CANCELLED=16,t.STATE_FAILED=32,t.Manager=Bh,t.Input=eh,t.TouchAction=Yl,t.TouchInput=uh,t.MouseInput=mh,t.PointerEventInput=ah,t.TouchMouseInput=bh,t.SingleTouchInput=Wh,t.Recognizer=kh,t.AttrRecognizer=Ih,t.Tap=Dh,t.Pan=Lh,t.Swipe=Gh,t.Pinch=Mh,t.Rotate=Sh,t.Press=Qh,t.on=ql,t.off=$l,t.each=Al,t.merge=Yh,t.extend=Th,t.bindFn=Rh,t.assign=kl,t.inherit=Eh,t.bindFn=Rh,t.prefixed=Ml,t.toArray=lh,t.inArray=ih,t.uniqueArray=hh,t.splitStr=Kl,t.boolOrFn=Xl,t.hasParent=El,t.addEventListeners=ql,t.removeEventListeners=$l,t.defaults=kl({},Nh,{preset:Fh}),t}();"undefined"!=typeof window?Uh=bl(window.Hammer||zh,{preventDefault:"mouse"}):Uh=function(){return{on:t=function(){},off:t,destroy:t,emit:t,get:function(e){return{set:t}}};var t};var Vh=Uh;function jh(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)}var Hh=function(){function t(e,i,n,o,s){ba(this,t),this.moment=s&&s.moment||Ta,this.options=s||{},this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(e,i,n),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Zo(o)?this.hiddenDates=o:this.hiddenDates=null!=o?[o]:[],this.format=t.FORMAT}return xa(t,[{key:"setMoment",value:function(t){this.moment=t,this.current=this.moment(this.current.valueOf()),this._start=this.moment(this._start.valueOf()),this._end=this.moment(this._end.valueOf())}},{key:"setFormat",value:function(e){var i=za.deepExtend({},t.FORMAT);this.format=za.deepExtend(i,e)}},{key:"setRange",value:function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=null!=t?this.moment(t.valueOf()):Jo(),this._end=null!=e?this.moment(e.valueOf()):Jo(),this.autoScale&&this.setMinimumStep(i)}},{key:"start",value:function(){this.current=this._start.clone(),this.roundToMinor()}},{key:"roundToMinor",value:function(){switch("week"==this.scale&&this.current.weekday(0),this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"week":case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step){var t=this.current.clone();switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"week":this.current.subtract(this.current.week()%this.step,"week");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}t.isSame(this.current)||(this.current=this.moment(ul(this.hiddenDates,this.current.valueOf(),-1,!0)))}}},{key:"hasNext",value:function(){return this.current.valueOf()<=this._end.valueOf()}},{key:"next",value:function(){var t=this.current.valueOf();switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.month()<6?this.current.subtract(this.current.hours()%this.step,"hour"):this.current.hours()%this.step!=0&&this.current.add(this.step-this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"week":if(0!==this.current.weekday())this.current.weekday(0),this.current.add(this.step,"week");else if(!1===this.options.showMajorLabels)this.current.add(this.step,"week");else{var e=this.current.clone();e.add(1,"week"),e.isSame(this.current,"month")?this.current.add(this.step,"week"):(this.current.add(this.step,"week"),this.current.date(1))}break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()>0&&this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()>0&&this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()>0&&this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()>0&&this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"week":this.current.week()<this.step&&this.current.week(1);break;case"month":this.current.month()<this.step&&this.current.month(0)}this.current.valueOf()==t&&(this.current=this._end.clone()),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,nl(this.moment,this,t)}},{key:"getCurrent",value:function(){return this.current.clone()}},{key:"setScale",value:function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)}},{key:"setAutoScale",value:function(t){this.autoScale=t}},{key:"setMinimumStep",value:function(t){if(null!=t){31104e9>t&&(this.scale="year",this.step=1e3),15552e9>t&&(this.scale="year",this.step=500),31104e8>t&&(this.scale="year",this.step=100),15552e8>t&&(this.scale="year",this.step=50),31104e7>t&&(this.scale="year",this.step=10),15552e7>t&&(this.scale="year",this.step=5),31104e6>t&&(this.scale="year",this.step=1),7776e6>t&&(this.scale="month",this.step=3),2592e6>t&&(this.scale="month",this.step=1),6048e5>t&&this.options.showWeekScale&&(this.scale="week",this.step=1),1728e5>t&&(this.scale="day",this.step=2),864e5>t&&(this.scale="day",this.step=1),432e5>t&&(this.scale="weekday",this.step=1),144e5>t&&(this.scale="hour",this.step=4),36e5>t&&(this.scale="hour",this.step=1),9e5>t&&(this.scale="minute",this.step=15),6e5>t&&(this.scale="minute",this.step=10),3e5>t&&(this.scale="minute",this.step=5),6e4>t&&(this.scale="minute",this.step=1),15e3>t&&(this.scale="second",this.step=15),1e4>t&&(this.scale="second",this.step=10),5e3>t&&(this.scale="second",this.step=5),1e3>t&&(this.scale="second",this.step=1),200>t&&(this.scale="millisecond",this.step=200),100>t&&(this.scale="millisecond",this.step=100),50>t&&(this.scale="millisecond",this.step=50),10>t&&(this.scale="millisecond",this.step=10),5>t&&(this.scale="millisecond",this.step=5),1>t&&(this.scale="millisecond",this.step=1)}}},{key:"isMajor",value:function(){if(1==this.switchedYear)switch(this.scale){case"year":case"month":case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.scale){case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":case"week":return 1==t.date();case"month":return 0==t.month();case"year":default:return!1}}},{key:"getLabelMinor",value:function(t){if(null==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.minorLabels)return this.format.minorLabels(t,this.scale,this.step);var e=this.format.minorLabels[this.scale];switch(this.scale){case"week":if(1===t.date()&&0!==t.weekday())return"";default:return e&&e.length>0?this.moment(t).format(e):""}}},{key:"getLabelMajor",value:function(t){if(null==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.majorLabels)return this.format.majorLabels(t,this.scale,this.step);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""}},{key:"getClassName",value:function(){var t,e=this.moment,i=this.moment(this.current),n=i.locale?i.locale("en"):i.lang("en"),o=this.step,s=[];function r(t){return t/o%2==0?" vis-even":" vis-odd"}function a(t){return t.isSame(Jo(),"day")?" vis-today":t.isSame(e().add(1,"day"),"day")?" vis-tomorrow":t.isSame(e().add(-1,"day"),"day")?" vis-yesterday":""}function l(t){return t.isSame(Jo(),"week")?" vis-current-week":""}function h(t){return t.isSame(Jo(),"month")?" vis-current-month":""}switch(this.scale){case"millisecond":s.push(a(n)),s.push(r(n.milliseconds()));break;case"second":s.push(a(n)),s.push(r(n.seconds()));break;case"minute":s.push(a(n)),s.push(r(n.minutes()));break;case"hour":s.push(Pn(t="vis-h".concat(n.hours())).call(t,4==this.step?"-h"+(n.hours()+4):"")),s.push(a(n)),s.push(r(n.hours()));break;case"weekday":s.push("vis-".concat(n.format("dddd").toLowerCase())),s.push(a(n)),s.push(l(n)),s.push(r(n.date()));break;case"day":s.push("vis-day".concat(n.date())),s.push("vis-".concat(n.format("MMMM").toLowerCase())),s.push(a(n)),s.push(h(n)),s.push(this.step<=2?a(n):""),s.push(this.step<=2?"vis-".concat(n.format("dddd").toLowerCase()):""),s.push(r(n.date()-1));break;case"week":s.push("vis-week".concat(n.format("w"))),s.push(l(n)),s.push(r(n.week()));break;case"month":s.push("vis-".concat(n.format("MMMM").toLowerCase())),s.push(h(n)),s.push(r(n.month()));break;case"year":s.push("vis-year".concat(n.year())),s.push(function(t){return t.isSame(Jo(),"year")?" vis-current-year":""}(n)),s.push(r(n.year()))}return En(s).call(s,String).join(" ")}}],[{key:"snap",value:function(t,e,i){var n=Ta(t);if("year"==e){var o=n.year()+Math.round(n.month()/12);n.year(Math.round(o/i)*i),n.month(0),n.date(0),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("month"==e)n.date()>15?(n.date(1),n.add(1,"month")):n.date(1),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0);else if("week"==e)n.weekday()>2?(n.weekday(0),n.add(1,"week")):n.weekday(0),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:n.hours(24*Math.round(n.hours()/24));break;default:n.hours(12*Math.round(n.hours()/12))}n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:n.hours(12*Math.round(n.hours()/12));break;default:n.hours(6*Math.round(n.hours()/6))}n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("hour"==e){switch(i){case 4:n.minutes(60*Math.round(n.minutes()/60));break;default:n.minutes(30*Math.round(n.minutes()/30))}n.seconds(0),n.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:n.minutes(5*Math.round(n.minutes()/5)),n.seconds(0);break;case 5:n.seconds(60*Math.round(n.seconds()/60));break;default:n.seconds(30*Math.round(n.seconds()/30))}n.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:n.seconds(5*Math.round(n.seconds()/5)),n.milliseconds(0);break;case 5:n.milliseconds(1e3*Math.round(n.milliseconds()/1e3));break;default:n.milliseconds(500*Math.round(n.milliseconds()/500))}else if("millisecond"==e){var s=i>5?i/2:1;n.milliseconds(Math.round(n.milliseconds()/s)*s)}return n}}]),t}();function Ph(t,e){void 0===e&&(e={});var i=e.insertAt;if(t&&"undefined"!=typeof document){var n=document.head||document.getElementsByTagName("head")[0],o=document.createElement("style");o.type="text/css","top"===i&&n.firstChild?n.insertBefore(o,n.firstChild):n.appendChild(o),o.styleSheet?o.styleSheet.cssText=t:o.appendChild(document.createTextNode(t))}}Hh.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}};Ph(".vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRpbWVheGlzLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxlQUNFLGlCQUFrQixDQUNsQixlQUNGLENBRUEsOEJBQ0UsS0FBTSxDQUNOLE1BQU8sQ0FDUCxVQUNGLENBRUEsOEJBQ0UsaUJBQWtCLENBQ2xCLEtBQU0sQ0FDTixNQUFPLENBQ1AsVUFBVyxDQUNYLFdBQ0YsQ0FFQSx5QkFDRSxpQkFBa0IsQ0FDbEIsYUFBYyxDQUNkLFdBQVksQ0FDWixlQUFnQixDQUNoQixxQkFBc0IsQ0FFdEIsa0JBQ0YsQ0FFQSxxQ0FDRSxpQkFBa0IsQ0FDbEIsY0FBZSxDQUNmLGVBQWdCLENBQ2hCLGFBQWMsQ0FDZCxjQUFlLENBQ2YsaUJBQ0YsQ0FFQSxzQ0FDRSxpQkFBa0IsQ0FDbEIscUJBQ0YsQ0FFQSwwQ0FDRSxpQkFBa0IsQ0FDbEIsc0JBQ0YsQ0FFQSxtQ0FDRSxvQkFDRixDQUVBLG1DQUNFLG9CQUNGIiwiZmlsZSI6InRpbWVheGlzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi52aXMtdGltZS1heGlzIHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBvdmVyZmxvdzogaGlkZGVuO1xufVxuXG4udmlzLXRpbWUtYXhpcy52aXMtZm9yZWdyb3VuZCB7XG4gIHRvcDogMDtcbiAgbGVmdDogMDtcbiAgd2lkdGg6IDEwMCU7XG59XG5cbi52aXMtdGltZS1heGlzLnZpcy1iYWNrZ3JvdW5kIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtdGV4dCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICM0ZDRkNGQ7XG4gIHBhZGRpbmc6IDNweDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcblxuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLXRleHQudmlzLW1lYXN1cmUge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHBhZGRpbmctbGVmdDogMDtcbiAgcGFkZGluZy1yaWdodDogMDtcbiAgbWFyZ2luLWxlZnQ6IDA7XG4gIG1hcmdpbi1yaWdodDogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLWdyaWQudmlzLXZlcnRpY2FsIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICBib3JkZXItbGVmdDogMXB4IHNvbGlkO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLWdyaWQudmlzLXZlcnRpY2FsLXJ0bCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgYm9yZGVyLXJpZ2h0OiAxcHggc29saWQ7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtZ3JpZC52aXMtbWlub3Ige1xuICBib3JkZXItY29sb3I6ICNlNWU1ZTU7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtZ3JpZC52aXMtbWFqb3Ige1xuICBib3JkZXItY29sb3I6ICNiZmJmYmY7XG59XG4iXX0= */");var Jh=function(t){function e(t,i){var n;return ba(this,e),(n=ar(this,dr(e).call(this))).dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},n.props={range:{start:0,end:0,minimumStep:0},lineTop:0},n.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,maxMinorChars:7,format:Hh.FORMAT,moment:Ta,timeAxis:null},n.options=za.extend({},n.defaultOptions),n.body=t,n._create(),n.setOptions(i),n}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&(za.selectiveExtend(["showMinorLabels","showMajorLabels","showWeekScale","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),za.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===zo(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof Ta.locale?Ta.locale(t.locale):Ta.lang(t.locale)))}},{key:"_create",value:function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"}},{key:"destroy",value:function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null}},{key:"redraw",value:function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,n="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,o=e.parentNode!==n;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,l=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height="".concat(this.props.height,"px"),this._repaintLabels(),a?n.insertBefore(e,a):n.appendChild(e),l?this.body.dom.backgroundVertical.insertBefore(i,l):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||o}},{key:"_repaintLabels",value:function(){var t=this.options.orientation.axis,e=za.convert(this.body.range.start,"Number"),i=za.convert(this.body.range.end,"Number"),n=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),o=n-hl(this.options.moment,this.body.hiddenDates,this.body.range,n);o-=this.body.util.toTime(0).valueOf();var s=new Hh(new Date(e),new Date(i),o,this.body.hiddenDates,this.options);s.setMoment(this.options.moment),this.options.format&&s.setFormat(this.options.format),this.options.timeAxis&&s.setScale(this.options.timeAxis),this.step=s;var r,a,l,h,d,u,c=this.dom;c.redundant.lines=c.lines,c.redundant.majorTexts=c.majorTexts,c.redundant.minorTexts=c.minorTexts,c.lines=[],c.majorTexts=[],c.minorTexts=[];var p,m,f,g=0,v=void 0,b=0;for(s.start(),a=s.getCurrent(),h=this.body.util.toScreen(a);s.hasNext()&&b<1e3;){switch(b++,d=s.isMajor(),f=s.getClassName(),r=a,l=h,s.next(),a=s.getCurrent(),p=g,g=(h=this.body.util.toScreen(a))-l,s.scale){case"week":u=!0;break;default:u=g>=.4*p}if(this.options.showMinorLabels&&u){var y=this._repaintMinorText(l,s.getLabelMinor(r),t,f);y.style.width="".concat(g,"px")}d&&this.options.showMajorLabels?(l>0&&(null==v&&(v=l),y=this._repaintMajorText(l,s.getLabelMajor(r),t,f)),m=this._repaintMajorLine(l,g,t,f)):u?m=this._repaintMinorLine(l,g,t,f):m&&(m.style.width="".concat(Nn(m.style.width)+g,"px"))}if(1e3!==b||Kh||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to ".concat(1e3," lines.")),Kh=!0),this.options.showMajorLabels){var x=this.body.util.toTime(0),w=s.getLabelMajor(x),_=w.length*(this.props.majorCharWidth||10)+10;(null==v||_<v)&&this._repaintMajorText(0,w,t,f)}mn(za).call(za,this.dom.redundant,(function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}}))}},{key:"_repaintMinorText",value:function(t,e,i,n){var o=this.dom.redundant.minorTexts.shift();if(!o){var s=document.createTextNode("");(o=document.createElement("div")).appendChild(s),this.dom.foreground.appendChild(o)}this.dom.minorTexts.push(o),o.innerHTML=e;var r="top"==i?this.props.majorLabelHeight:0;return this._setXY(o,t,r),o.className="vis-text vis-minor ".concat(n),o}},{key:"_repaintMajorText",value:function(t,e,i,n){var o=this.dom.redundant.majorTexts.shift();if(!o){var s=document.createElement("div");(o=document.createElement("div")).appendChild(s),this.dom.foreground.appendChild(o)}o.childNodes[0].innerHTML=e,o.className="vis-text vis-major ".concat(n);var r="top"==i?0:this.props.minorLabelHeight;return this._setXY(o,t,r),this.dom.majorTexts.push(o),o}},{key:"_setXY",value:function(t,e,i){var n,o=this.options.rtl?-1*e:e;t.style.transform=Pn(n="translate(".concat(o,"px, ")).call(n,i,"px)")}},{key:"_repaintMinorLine",value:function(t,e,i,n){var o,s=this.dom.redundant.lines.shift();s||(s=document.createElement("div"),this.dom.background.appendChild(s)),this.dom.lines.push(s);var r=this.props;s.style.width="".concat(e,"px"),s.style.height="".concat(r.minorLineHeight,"px");var a="top"==i?r.majorLabelHeight:this.body.domProps.top.height,l=t-r.minorLineWidth/2;return this._setXY(s,l,a),s.className=Pn(o="vis-grid ".concat(this.options.rtl?"vis-vertical-rtl":"vis-vertical"," vis-minor ")).call(o,n),s}},{key:"_repaintMajorLine",value:function(t,e,i,n){var o,s=this.dom.redundant.lines.shift();s||(s=document.createElement("div"),this.dom.background.appendChild(s)),this.dom.lines.push(s);var r=this.props;s.style.width="".concat(e,"px"),s.style.height="".concat(r.majorLineHeight,"px");var a="top"==i?0:this.body.domProps.top.height,l=t-r.majorLineWidth/2;return this._setXY(s,l,a),s.className=Pn(o="vis-grid ".concat(this.options.rtl?"vis-vertical-rtl":"vis-vertical"," vis-major ")).call(o,n),s}},{key:"_calculateCharSize",value:function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth}}]),e}(Ja),Kh=!1,qh=n((function(t,e){t.exports=function(t){var e,i=t&&t.preventDefault||!1,n=t&&t.container||window,o={},s={keydown:{},keyup:{}},r={};for(e=97;e<=122;e++)r[String.fromCharCode(e)]={code:e-97+65,shift:!1};for(e=65;e<=90;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;e<=9;e++)r[""+e]={code:48+e,shift:!1};for(e=1;e<=12;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;e<=9;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r.delete={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){h(t,"keydown")},l=function(t){h(t,"keyup")},h=function(t,e){if(void 0!==s[e][t.keyCode]){for(var n=s[e][t.keyCode],o=0;o<n.length;o++)(void 0===n[o].shift||1==n[o].shift&&1==t.shiftKey||0==n[o].shift&&0==t.shiftKey)&&n[o].fn(t);1==i&&t.preventDefault()}};return o.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},o.bindAll=function(t,e){for(var i in void 0===e&&(e="keydown"),r)r.hasOwnProperty(i)&&o.bind(i,t,e)},o.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},o.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var n=[],o=s[i][r[t].code];if(void 0!==o)for(var a=0;a<o.length;a++)o[a].fn==e&&o[a].shift==r[t].shift||n.push(s[i][r[t].code][a]);s[i][r[t].code]=n}else s[i][r[t].code]=[]},o.reset=function(){s={keydown:{},keyup:{}}},o.destroy=function(){s={keydown:{},keyup:{}},n.removeEventListener("keydown",a,!0),n.removeEventListener("keyup",l,!0)},n.addEventListener("keydown",a,!0),n.addEventListener("keyup",l,!0),o}}));function $h(t){var e,i;this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=Vh(this.dom.overlay),this.hammer.on("tap",va(e=this._onTapOverlay).call(e,this));var n=this,o=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];mn(o).call(o,(function(t){n.hammer.on(t,(function(t){t.stopPropagation()}))})),document&&document.body&&(this.onClick=function(e){(function(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1})(e.target,t)||n.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=qh(),this.escListener=va(i=this.deactivate).call(i,this)}Ph(".vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImFjdGl2YXRvci5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FDRSxpQkFBa0IsQ0FDbEIsS0FBTSxDQUNOLE1BQU8sQ0FDUCxVQUFXLENBQ1gsV0FBWSxDQUdaLFVBQ0YsQ0FFQSxZQUNFLDJCQUNGIiwiZmlsZSI6ImFjdGl2YXRvci5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyIudmlzIC5vdmVybGF5IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG5cbiAgLyogTXVzdCBiZSBkaXNwbGF5ZWQgYWJvdmUgZm9yIGV4YW1wbGUgc2VsZWN0ZWQgVGltZWxpbmUgaXRlbXMgKi9cbiAgei1pbmRleDogMTA7XG59XG5cbi52aXMtYWN0aXZlIHtcbiAgYm94LXNoYWRvdzogMCAwIDEwcHggIzg2ZDVmODtcbn1cbiJdfQ== */"),vl($h.prototype),$h.current=null,$h.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=null,this.hammer.destroy(),this.hammer=null},$h.prototype.activate=function(){var t;$h.current&&$h.current.deactivate(),$h.current=this,this.active=!0,this.dom.overlay.style.display="none",za.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),va(t=this.keycharm).call(t,"esc",this.escListener)},$h.prototype.deactivate=function(){$h.current===this&&($h.current=null),this.active=!1,this.dom.overlay.style.display="",za.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},$h.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()};var td={current:"current",time:"time",deleteSelected:"Delete selected"},ed={current:"attuale",time:"tempo",deleteSelected:"Cancella la selezione"},id={current:"huidige",time:"tijd",deleteSelected:"Selectie verwijderen"},nd={current:"Aktuelle",time:"Zeit",deleteSelected:"Lösche Auswahl"},od={current:"actuel",time:"heure",deleteSelected:"Effacer la selection"},sd={current:"corriente",time:"hora",deleteSelected:"Eliminar selección"},rd={current:"поточний",time:"час",deleteSelected:"Видалити обране"},ad={current:"текущее",time:"время",deleteSelected:"Удалить выбранное"},ld={current:"aktualny",time:"czas",deleteSelected:"Usuń wybrane"},hd={current:"現在",time:"時刻",deleteSelected:"選択されたものを削除"},dd={en:td,en_EN:td,en_US:td,it:ed,it_IT:ed,it_CH:ed,nl:id,nl_NL:id,nl_BE:id,de:nd,de_DE:nd,fr:od,fr_FR:od,fr_CA:od,fr_BE:od,es:sd,es_ES:sd,uk:rd,uk_UA:rd,ru:ad,ru_RU:ad,pl:ld,pl_PL:ld,ja:hd,ja_JP:hd};Ph(".vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-custom-time>.vis-custom-time-marker{background-color:inherit;color:#fff;font-size:12px;white-space:nowrap;padding:3px 5px;top:0;cursor:auto;z-index:inherit}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImN1c3RvbXRpbWUuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlCQUNFLHdCQUF5QixDQUN6QixTQUFVLENBQ1YsV0FBWSxDQUNaLFNBQ0YsQ0FFQSx5Q0FDRSx3QkFBeUIsQ0FDekIsVUFBWSxDQUNaLGNBQWUsQ0FDZixrQkFBbUIsQ0FDbkIsZUFBZ0IsQ0FDaEIsS0FBUSxDQUNSLFdBQWUsQ0FDZixlQUNGIiwiZmlsZSI6ImN1c3RvbXRpbWUuY3NzIiwic291cmNlc0NvbnRlbnQiOlsiLnZpcy1jdXN0b20tdGltZSB7XG4gIGJhY2tncm91bmQtY29sb3I6ICM2RTk0RkY7XG4gIHdpZHRoOiAycHg7XG4gIGN1cnNvcjogbW92ZTtcbiAgei1pbmRleDogMTtcbn1cblxuLnZpcy1jdXN0b20tdGltZSA+IC52aXMtY3VzdG9tLXRpbWUtbWFya2VyIHtcbiAgYmFja2dyb3VuZC1jb2xvcjogaW5oZXJpdDtcbiAgY29sb3I6IHdoaXRlO1xuICBmb250LXNpemU6IDEycHg7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIHBhZGRpbmc6IDNweCA1cHg7XG4gIHRvcDogMHB4O1xuICBjdXJzb3I6IGluaXRpYWw7XG4gIHotaW5kZXg6IGluaGVyaXQ7XG59Il19 */");var ud=function(t){function e(t,i){var n,o;ba(this,e),(o=ar(this,dr(e).call(this))).body=t,o.defaultOptions={moment:Ta,locales:dd,locale:"en",id:void 0,title:void 0},o.options=za.extend({},o.defaultOptions),o.setOptions(i),o.options.locales=za.extend({},dd,o.options.locales);var s=o.defaultOptions.locales[o.defaultOptions.locale];return mn(n=Yi(o.options.locales)).call(n,(function(t){o.options.locales[t]=za.extend({},s,o.options.locales[t])})),i&&null!=i.time?o.customTime=i.time:o.customTime=new Date,o.eventParams={},o._create(),o}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&za.selectiveExtend(["moment","locale","locales","id","title","rtl","snap"],this.options,t)}},{key:"_create",value:function(){var t,e,i,n=document.createElement("div");n["custom-time"]=this,n.className="vis-custom-time ".concat(this.options.id||""),n.style.position="absolute",n.style.top="0px",n.style.height="100%",this.bar=n;var o=document.createElement("div");function s(t){this.body.range._onMouseWheel(t)}o.style.position="relative",o.style.top="0px",this.options.rtl?o.style.right="-10px":o.style.left="-10px",o.style.height="100%",o.style.width="20px",o.addEventListener?(o.addEventListener("mousewheel",va(s).call(s,this),!1),o.addEventListener("DOMMouseScroll",va(s).call(s,this),!1)):o.attachEvent("onmousewheel",va(s).call(s,this)),n.appendChild(o),this.hammer=new Vh(o),this.hammer.on("panstart",va(t=this._onDragStart).call(t,this)),this.hammer.on("panmove",va(e=this._onDrag).call(e,this)),this.hammer.on("panend",va(i=this._onDragEnd).call(i,this)),this.hammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_ALL})}},{key:"destroy",value:function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null}},{key:"redraw",value:function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),i=this.options.locales.en);var n,o=this.options.title;void 0===o?o=(o=Pn(n="".concat(i.time,": ")).call(n,this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"))).charAt(0).toUpperCase()+o.substring(1):"function"==typeof o&&(o=o.call(this,this.customTime));return this.options.rtl?this.bar.style.right="".concat(e,"px"):this.bar.style.left="".concat(e,"px"),this.bar.title=o,!1}},{key:"hide",value:function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)}},{key:"setCustomTime",value:function(t){this.customTime=za.convert(t,"Date"),this.redraw()}},{key:"getCustomTime",value:function(){return new Date(this.customTime.valueOf())}},{key:"setCustomMarker",value:function(t,e){var i,n,o=document.createElement("div");(o.className="vis-custom-time-marker",o.innerHTML=t,o.style.position="absolute",e)&&(o.setAttribute("contenteditable","true"),o.addEventListener("pointerdown",(function(){o.focus()})),o.addEventListener("input",va(i=this._onMarkerChange).call(i,this)),o.title=t,o.addEventListener("blur",va(n=function(t){this.title!=t.target.innerHTML&&(this._onMarkerChanged(t),this.title=t.target.innerHTML)}).call(n,this)));this.bar.appendChild(o)}},{key:"setCustomTitle",value:function(t){this.options.title=t}},{key:"_onDragStart",value:function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()}},{key:"_onDrag",value:function(t){if(this.eventParams.dragging){var e=this.options.rtl?-1*t.deltaX:t.deltaX,i=this.body.util.toScreen(this.eventParams.customTime)+e,n=this.body.util.toTime(i),o=this.body.util.getScale(),s=this.body.util.getStep(),r=this.options.snap,a=r?r(n,o,s):n;this.setCustomTime(a),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation()}}},{key:"_onDragEnd",value:function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation())}},{key:"_onMarkerChange",value:function(t){this.body.emitter.emit("markerchange",{id:this.options.id,title:t.target.innerHTML,event:t}),t.stopPropagation()}},{key:"_onMarkerChanged",value:function(t){this.body.emitter.emit("markerchanged",{id:this.options.id,title:t.target.innerHTML,event:t}),t.stopPropagation()}}],[{key:"customTimeFromTarget",value:function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null}}]),e}(Ja);Ph("\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IiIsImZpbGUiOiJhbmltYXRpb24uY3NzIn0= */");Ph('.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\\26F6"}.vis-rolling-mode-btn:hover{opacity:1}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImN1cnJlbnR0aW1lLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxrQkFDRSx3QkFBeUIsQ0FDekIsU0FBVSxDQUNWLFNBQVUsQ0FDVixtQkFDRixDQUVBLHNCQUNFLFdBQVksQ0FDWixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGNBQWUsQ0FDZixjQUFlLENBQ2YsVUFBWSxDQUNaLFVBQVksQ0FDWixlQUFpQixDQUNqQixpQkFBa0IsQ0FDbEIsa0JBQ0YsQ0FDQSw2QkFDRSxlQUNGLENBRUEsNEJBQ0UsU0FDRiIsImZpbGUiOiJjdXJyZW50dGltZS5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyIudmlzLWN1cnJlbnQtdGltZSB7XG4gIGJhY2tncm91bmQtY29sb3I6ICNGRjdGNkU7XG4gIHdpZHRoOiAycHg7XG4gIHotaW5kZXg6IDE7XG4gIHBvaW50ZXItZXZlbnRzOiBub25lO1xufVxuXG4udmlzLXJvbGxpbmctbW9kZS1idG4ge1xuICBoZWlnaHQ6IDQwcHg7XG4gIHdpZHRoOiA0MHB4O1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHRvcDogN3B4O1xuICByaWdodDogMjBweDtcbiAgYm9yZGVyLXJhZGl1czogNTAlO1xuICBmb250LXNpemU6IDI4cHg7XG4gIGN1cnNvcjogcG9pbnRlcjtcbiAgb3BhY2l0eTogMC44O1xuICBjb2xvcjogd2hpdGU7XG4gIGZvbnQtd2VpZ2h0OiBib2xkO1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG4gIGJhY2tncm91bmQ6ICMzODc2YzI7XG59XG4udmlzLXJvbGxpbmctbW9kZS1idG46YmVmb3JlIHtcbiAgY29udGVudDogXCJcXDI2RjZcIjtcbn1cblxuLnZpcy1yb2xsaW5nLW1vZGUtYnRuOmhvdmVyIHtcbiAgb3BhY2l0eTogMTtcbn0iXX0= */');Ph(".vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content,.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhbmVsLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxXQUNFLGlCQUFrQixDQUVsQixTQUFVLENBQ1YsUUFBUyxDQUVULHFCQUNGLENBRUEsd0dBS0Usa0JBQ0YsQ0FFQSwrREFHRSxzQkFBdUIsQ0FDdkIseUJBQTBCLENBQzFCLGVBQ0YsQ0FFQSxpRkFDRSxXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGlCQUNGLENBRUEsd0NBQ0UsYUFDRixDQU1BLDhGQUNFLGFBQ0YsQ0FFQSxzREFDRSxhQUNGLENBRUEsK0RBR0UsdUJBQXdCLENBQ3hCLHdCQUNGLENBRUEsZ0JBQ0UsZUFDRixDQUVBLHdCQUNFLGlCQUNGLENBRUEsdUJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxVQUFXLENBQ1gsa0NBSUYsQ0FFQSwrQkFDRSxRQUFTLENBQ1QsTUFDRixDQUVBLGtDQUNFLFdBQVksQ0FDWixNQUNGIiwiZmlsZSI6InBhbmVsLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi52aXMtcGFuZWwge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG5cbiAgcGFkZGluZzogMDtcbiAgbWFyZ2luOiAwO1xuXG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtcGFuZWwudmlzLWNlbnRlcixcbi52aXMtcGFuZWwudmlzLWxlZnQsXG4udmlzLXBhbmVsLnZpcy1yaWdodCxcbi52aXMtcGFuZWwudmlzLXRvcCxcbi52aXMtcGFuZWwudmlzLWJvdHRvbSB7XG4gIGJvcmRlcjogMXB4ICNiZmJmYmY7XG59XG5cbi52aXMtcGFuZWwudmlzLWNlbnRlcixcbi52aXMtcGFuZWwudmlzLWxlZnQsXG4udmlzLXBhbmVsLnZpcy1yaWdodCB7XG4gIGJvcmRlci10b3Atc3R5bGU6IHNvbGlkO1xuICBib3JkZXItYm90dG9tLXN0eWxlOiBzb2xpZDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbn1cblxuLnZpcy1sZWZ0LnZpcy1wYW5lbC52aXMtdmVydGljYWwtc2Nyb2xsLCAudmlzLXJpZ2h0LnZpcy1wYW5lbC52aXMtdmVydGljYWwtc2Nyb2xsIHtcbiAgaGVpZ2h0OiAxMDAlO1xuICBvdmVyZmxvdy14OiBoaWRkZW47XG4gIG92ZXJmbG93LXk6IHNjcm9sbDtcbn0gXG5cbi52aXMtbGVmdC52aXMtcGFuZWwudmlzLXZlcnRpY2FsLXNjcm9sbCB7XG4gIGRpcmVjdGlvbjogcnRsO1xufVxuXG4udmlzLWxlZnQudmlzLXBhbmVsLnZpcy12ZXJ0aWNhbC1zY3JvbGwgLnZpcy1jb250ZW50IHtcbiAgZGlyZWN0aW9uOiBsdHI7XG59XG5cbi52aXMtcmlnaHQudmlzLXBhbmVsLnZpcy12ZXJ0aWNhbC1zY3JvbGwge1xuICBkaXJlY3Rpb246IGx0cjtcbn1cblxuLnZpcy1yaWdodC52aXMtcGFuZWwudmlzLXZlcnRpY2FsLXNjcm9sbCAudmlzLWNvbnRlbnQge1xuICBkaXJlY3Rpb246IHJ0bDtcbn1cblxuLnZpcy1wYW5lbC52aXMtY2VudGVyLFxuLnZpcy1wYW5lbC52aXMtdG9wLFxuLnZpcy1wYW5lbC52aXMtYm90dG9tIHtcbiAgYm9yZGVyLWxlZnQtc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmlnaHQtc3R5bGU6IHNvbGlkO1xufVxuXG4udmlzLWJhY2tncm91bmQge1xuICBvdmVyZmxvdzogaGlkZGVuO1xufVxuXG4udmlzLXBhbmVsID4gLnZpcy1jb250ZW50IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xufVxuXG4udmlzLXBhbmVsIC52aXMtc2hhZG93IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB3aWR0aDogMTAwJTtcbiAgaGVpZ2h0OiAxcHg7XG4gIGJveC1zaGFkb3c6IDAgMCAxMHB4IHJnYmEoMCwwLDAsMC44KTtcbiAgLyogVE9ETzogZmluZCBhIG5pY2Ugd2F5IHRvIGVuc3VyZSB2aXMtc2hhZG93cyBhcmUgZHJhd24gb24gdG9wIG9mIGl0ZW1zXG4gIHotaW5kZXg6IDE7XG4gICovXG59XG5cbi52aXMtcGFuZWwgLnZpcy1zaGFkb3cudmlzLXRvcCB7XG4gIHRvcDogLTFweDtcbiAgbGVmdDogMDtcbn1cblxuLnZpcy1wYW5lbCAudmlzLXNoYWRvdy52aXMtYm90dG9tIHtcbiAgYm90dG9tOiAtMXB4O1xuICBsZWZ0OiAwO1xufSJdfQ== */");Ph(".vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhdGhTdHlsZXMuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGtCQUNJLFlBQVksQ0FDWixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFZLENBQ1osY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksWUFBYSxDQUNiLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsY0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFhLENBQ2IsY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksWUFBYSxDQUNiLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsY0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFZLENBQ1osY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksU0FBWSxDQUNaLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsV0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSx3QkFDSSxlQUFnQixDQUNoQixXQUNKLENBR0EsdUJBQ0ksZUFBZ0IsQ0FDaEIsZ0JBQ0osQ0FFQSx5QkFDSSxnQkFBZ0IsQ0FDaEIsY0FDSixDQUdBLHFDQUNJLGdCQUFnQixDQUNoQixlQUFnQixDQUNoQixTQUFhLENBQ2IsY0FDSixDQUdBLDJCQUNJLGdCQUFnQixDQUNoQixjQUFjLENBQ2QsU0FBYSxDQUNiLGNBQ0osQ0FFQSw2QkFDSSxlQUFnQixDQUNoQixXQUNKIiwiZmlsZSI6InBhdGhTdHlsZXMuY3NzIiwic291cmNlc0NvbnRlbnQiOlsiLnZpcy1ncmFwaC1ncm91cDAge1xuICAgIGZpbGw6IzRmODFiZDtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogIzRmODFiZDtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDEge1xuICAgIGZpbGw6I2Y3OTY0NjtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogI2Y3OTY0Njtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDIge1xuICAgIGZpbGw6ICM4YzUxY2Y7XG4gICAgZmlsbC1vcGFjaXR5OjA7XG4gICAgc3Ryb2tlLXdpZHRoOjJweDtcbiAgICBzdHJva2U6ICM4YzUxY2Y7XG59XG5cbi52aXMtZ3JhcGgtZ3JvdXAzIHtcbiAgICBmaWxsOiAjNzVjODQxO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjNzVjODQxO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwNCB7XG4gICAgZmlsbDogI2ZmMDEwMDtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogI2ZmMDEwMDtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDUge1xuICAgIGZpbGw6ICMzN2Q4ZTY7XG4gICAgZmlsbC1vcGFjaXR5OjA7XG4gICAgc3Ryb2tlLXdpZHRoOjJweDtcbiAgICBzdHJva2U6ICMzN2Q4ZTY7XG59XG5cbi52aXMtZ3JhcGgtZ3JvdXA2IHtcbiAgICBmaWxsOiAjMDQyNjYyO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjMDQyNjYyO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwNyB7XG4gICAgZmlsbDojMDBmZjI2O1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjMDBmZjI2O1xufVxuXG4udmlzLWdyYXBoLWdyb3VwOCB7XG4gICAgZmlsbDojZmYwMGZmO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjZmYwMGZmO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwOSB7XG4gICAgZmlsbDogIzhmMzkzODtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogIzhmMzkzODtcbn1cblxuLnZpcy10aW1lbGluZSAudmlzLWZpbGwge1xuICAgIGZpbGwtb3BhY2l0eTowLjE7XG4gICAgc3Ryb2tlOiBub25lO1xufVxuXG5cbi52aXMtdGltZWxpbmUgLnZpcy1iYXIge1xuICAgIGZpbGwtb3BhY2l0eTowLjU7XG4gICAgc3Ryb2tlLXdpZHRoOjFweDtcbn1cblxuLnZpcy10aW1lbGluZSAudmlzLXBvaW50IHtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIGZpbGwtb3BhY2l0eToxLjA7XG59XG5cblxuLnZpcy10aW1lbGluZSAudmlzLWxlZ2VuZC1iYWNrZ3JvdW5kIHtcbiAgICBzdHJva2Utd2lkdGg6MXB4O1xuICAgIGZpbGwtb3BhY2l0eTowLjk7XG4gICAgZmlsbDogI2ZmZmZmZjtcbiAgICBzdHJva2U6ICNjMmMyYzI7XG59XG5cblxuLnZpcy10aW1lbGluZSAudmlzLW91dGxpbmUge1xuICAgIHN0cm9rZS13aWR0aDoxcHg7XG4gICAgZmlsbC1vcGFjaXR5OjE7XG4gICAgZmlsbDogI2ZmZmZmZjtcbiAgICBzdHJva2U6ICNlNWU1ZTU7XG59XG5cbi52aXMtdGltZWxpbmUgLnZpcy1pY29uLWZpbGwge1xuICAgIGZpbGwtb3BhY2l0eTowLjM7XG4gICAgc3Ryb2tlOiBub25lO1xufVxuIl19 */");Ph(".vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}.vis-loading-screen{width:100%;height:100%;position:absolute;top:0;left:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRpbWVsaW5lLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxjQUNFLGlCQUFrQixDQUNsQix3QkFBeUIsQ0FDekIsZUFBZ0IsQ0FDaEIsU0FBVSxDQUNWLFFBQVMsQ0FDVCxxQkFDRixDQUVBLG9CQUNFLFVBQVcsQ0FDWCxXQUFZLENBQ1osaUJBQWtCLENBQ2xCLEtBQU0sQ0FDTixNQUNGIiwiZmlsZSI6InRpbWVsaW5lLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy10aW1lbGluZSB7XG4gIHBvc2l0aW9uOiByZWxhdGl2ZTtcbiAgYm9yZGVyOiAxcHggc29saWQgI2JmYmZiZjtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbiAgcGFkZGluZzogMDtcbiAgbWFyZ2luOiAwO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xufVxuXG4udmlzLWxvYWRpbmctc2NyZWVuIHtcbiAgd2lkdGg6IDEwMCU7XG4gIGhlaWdodDogMTAwJTtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG59Il19 */");Ph(".vis [class*=span]{min-height:0;width:auto}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImJvb3RzdHJhcC5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsbUJBQ0UsWUFBYSxDQUNiLFVBQ0YiLCJmaWxlIjoiYm9vdHN0cmFwLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi8qIG92ZXJyaWRlIHNvbWUgYm9vdHN0cmFwIHN0eWxlcyBzY3Jld2luZyB1cCB0aGUgdGltZWxpbmVzIGNzcyAqL1xuXG4udmlzIFtjbGFzcyo9XCJzcGFuXCJdIHtcbiAgbWluLWhlaWdodDogMDtcbiAgd2lkdGg6IGF1dG87XG59XG4iXX0= */");var cd=function(){function t(){ba(this,t)}return xa(t,[{key:"_create",value:function(t){var e,i,n,o=this;this.dom={},this.dom.container=t,this.dom.container.style.position="relative",this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.rollingModeBtn=document.createElement("div"),this.dom.loadingScreen=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.rollingModeBtn.className="vis-rolling-mode-btn",this.dom.loadingScreen.className="vis-loading-screen",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.rollingModeBtn),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.on("rangechange",(function(){!0===o.initialDrawDone&&o._redraw()})),this.on("rangechanged",(function(){o.initialRangeChangeDone||(o.initialRangeChangeDone=!0)})),this.on("touch",va(e=this._onTouch).call(e,this)),this.on("panmove",va(i=this._onDrag).call(i,this));var s=this;this._origRedraw=va(n=this._redraw).call(n,this),this._redraw=za.throttle(this._origRedraw),this.on("_change",(function(t){s.itemSet&&s.itemSet.initialItemSetDrawn&&t&&1==t.queue?s._redraw():s._origRedraw()})),this.hammer=new Vh(this.dom.root);var r=this.hammer.get("pinch").set({enable:!0});r&&function(t){t.getTouchAction=function(){return["pan-y"]}}(r),this.hammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_ALL}),this.listeners={};var a,l,h=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];function d(t){this.isActive()&&this.emit("mousewheel",t);var e=0,i=0;if("detail"in t&&(i=-1*t.detail),"wheelDelta"in t&&(i=t.wheelDelta),"wheelDeltaY"in t&&(i=t.wheelDeltaY),"wheelDeltaX"in t&&(e=-1*t.wheelDeltaX),"axis"in t&&t.axis===t.HORIZONTAL_AXIS&&(e=-1*i,i=0),"deltaY"in t&&(i=-1*t.deltaY),"deltaX"in t&&(e=t.deltaX),t.deltaMode&&(1===t.deltaMode?(e*=40,i*=40):(e*=40,i*=800)),this.options.preferZoom){if(!this.options.zoomKey||t[this.options.zoomKey])return}else if(this.options.zoomKey&&t[this.options.zoomKey])return;if(this.options.verticalScroll||this.options.horizontalScroll)if(this.options.verticalScroll&&Math.abs(i)>=Math.abs(e)){var n=this.props.scrollTop,o=n+i;if(this.isActive())this._setScrollTop(o)!==n&&(this._redraw(),this.emit("scroll",t),t.preventDefault())}else if(this.options.horizontalScroll){var s=(Math.abs(e)>=Math.abs(i)?e:i)/120*(this.range.end-this.range.start)/20,r=this.range.start+s,a=this.range.end+s,l={animation:!1,byUser:!0,event:t};this.range.setRange(r,a,l),t.preventDefault()}}mn(h).call(h,(function(t){var e=function(e){s.isActive()&&s.emit(t,e)};s.hammer.on(t,e),s.listeners[t]=e})),jh(this.hammer,(function(t){s.emit("touch",t)})),a=this.hammer,(l=function(t){s.emit("release",t)}).inputHandler=function(t){t.isFinal&&l(t)},a.on("hammer.input",l.inputHandler);var u="onwheel"in document.createElement("div")?"wheel":void 0!==document.onmousewheel?"mousewheel":this.dom.centerContainer.addEventListener?"DOMMouseScroll":"onmousewheel";function c(t){if(s.options.verticalScroll)if(s._isProgramaticallyScrolled)s._isProgramaticallyScrolled=!1;else if(t.preventDefault(),s.isActive()){var e=-t.target.scrollTop;s._setScrollTop(e),s._redraw(),s.emit("scrollSide",t)}}this.dom.top.addEventListener,this.dom.bottom.addEventListener,this.dom.centerContainer.addEventListener(u,va(d).call(d,this),!1),this.dom.top.addEventListener(u,va(d).call(d,this),!1),this.dom.bottom.addEventListener(u,va(d).call(d,this),!1),this.dom.left.parentNode.addEventListener("scroll",va(c).call(c,this)),this.dom.right.parentNode.addEventListener("scroll",va(c).call(c,this));var p=!1;function m(t){var e;if(t.preventDefault&&(s.emit("dragover",s.getEventProperties(t)),t.preventDefault()),Fo(e=t.target.className).call(e,"timeline")>-1&&!p)return t.dataTransfer.dropEffect="move",p=!0,!1}function f(t){t.preventDefault&&t.preventDefault(),t.stopPropagation&&t.stopPropagation();try{var e=JSON.parse(t.dataTransfer.getData("text"));if(!e||!e.content)return}catch(t){return!1}return p=!1,t.center={x:t.clientX,y:t.clientY},"item"!==e.target?s.itemSet._onAddItem(t):s.itemSet._onDropObjectOnItem(t),s.emit("drop",s.getEventProperties(t)),!1}if(this.dom.center.addEventListener("dragover",va(m).call(m,this),!1),this.dom.center.addEventListener("drop",va(f).call(f,this),!1),this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,this.initialRangeChangeDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root),t.appendChild(this.dom.loadingScreen)}},{key:"setOptions",value:function(t){var e;if(t){if(za.selectiveExtend(["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","preferZoom","rtl","zoomKey","horizontalScroll","verticalScroll","longSelectPressTime"],this.options,t),this.dom.rollingModeBtn.style.visibility="hidden",this.options.rtl&&(this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"),this.options.verticalScroll&&(this.options.rtl?this.dom.rightContainer.className="vis-panel vis-right vis-vertical-scroll":this.dom.leftContainer.className="vis-panel vis-left vis-vertical-scroll"),"object"!==zo(this.options.orientation)&&(this.options.orientation={item:void 0,axis:void 0}),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===zo(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var i=this.timeAxis2=new Jh(this.body);i.setOptions=function(t){var e=t?za.extend({},t):{};e.orientation="top",Jh.prototype.setOptions.call(i,e)},this.components.push(i)}}else if(this.timeAxis2){var n,o,s=Fo(n=this.components).call(n,this.timeAxis2);if(-1!==s)Hr(o=this.components).call(o,s,1);this.timeAxis2.destroy(),this.timeAxis2=null}"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&tl(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new $h(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),this._initAutoResize()}if(mn(e=this.components).call(e,(function(e){return e.setOptions(t)})),"configure"in t){var r;this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var a=za.deepExtend({},this.options);mn(r=this.components).call(r,(function(t){za.deepExtend(a,t.options)})),this.configurator.setModuleOptions({global:a})}this._redraw()}},{key:"isActive",value:function(){return!this.activator||this.activator.active}},{key:"destroy",value:function(){var t;for(var e in this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator),this.listeners)this.listeners.hasOwnProperty(e)&&delete this.listeners[e];this.listeners=null,this.hammer&&this.hammer.destroy(),this.hammer=null,mn(t=this.components).call(t,(function(t){return t.destroy()})),this.body=null}},{key:"setCustomTime",value:function(t,e){var i,n=En(i=this.customTimes).call(i,(function(t){return e===t.options.id}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));n.length>0&&n[0].setCustomTime(t)}},{key:"getCustomTime",value:function(t){var e,i=En(e=this.customTimes).call(e,(function(e){return e.options.id===t}));if(0===i.length)throw new Error("No custom time bar found with id ".concat(Vs(t)));return i[0].getCustomTime()}},{key:"setCustomTimeMarker",value:function(t,e,i){var n,o=En(n=this.customTimes).call(n,(function(t){return t.options.id===e}));if(0===o.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));o.length>0&&o[0].setCustomMarker(t,i)}},{key:"setCustomTimeTitle",value:function(t,e){var i,n=En(i=this.customTimes).call(i,(function(t){return t.options.id===e}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));if(n.length>0)return n[0].setCustomTitle(t)}},{key:"getEventProperties",value:function(t){return{event:t}}},{key:"addCustomTime",value:function(t,e){var i,n=void 0!==t?za.convert(t,"Date"):new Date;if(rr(i=this.customTimes).call(i,(function(t){return t.options.id===e})))throw new Error("A custom time with id ".concat(Vs(e)," already exists"));var o=new ud(this.body,za.extend({},this.options,{time:n,id:e,snap:this.itemSet.options.snap}));return this.customTimes.push(o),this.components.push(o),this._redraw(),e}},{key:"removeCustomTime",value:function(t){var e,i=this,n=En(e=this.customTimes).call(e,(function(e){return e.options.id===t}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(t)));mn(n).call(n,(function(t){var e,n,o,s;Hr(e=i.customTimes).call(e,Fo(n=i.customTimes).call(n,t),1),Hr(o=i.components).call(o,Fo(s=i.components).call(s,t),1),t.destroy()}))}},{key:"getVisibleItems",value:function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]}},{key:"getVisibleGroups",value:function(){return this.itemSet&&this.itemSet.getVisibleGroups()||[]}},{key:"fit",value:function(t,e){var i=this.getDataRange();if(null!==i.min||null!==i.max){var n=i.max-i.min,o=new Date(i.min.valueOf()-.01*n),s=new Date(i.max.valueOf()+.01*n),r=!t||void 0===t.animation||t.animation;this.range.setRange(o,s,{animation:r},e)}}},{key:"getDataRange",value:function(){throw new Error("Cannot invoke abstract method getDataRange")}},{key:"setWindow",value:function(t,e,i,n){var o,s;"function"==typeof arguments[2]&&(n=arguments[2],i={}),1==arguments.length?(o=void 0===(s=arguments[0]).animation||s.animation,this.range.setRange(s.start,s.end,{animation:o})):2==arguments.length&&"function"==typeof arguments[1]?(s=arguments[0],n=arguments[1],o=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:o},n)):(o=!i||void 0===i.animation||i.animation,this.range.setRange(t,e,{animation:o},n))}},{key:"moveTo",value:function(t,e,i){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.range.end-this.range.start,o=za.convert(t,"Date").valueOf(),s=o-n/2,r=o+n/2,a=!e||void 0===e.animation||e.animation;this.range.setRange(s,r,{animation:a},i)}},{key:"getWindow",value:function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}}},{key:"zoomIn",value:function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.getWindow(),o=n.start.valueOf(),s=n.end.valueOf(),r=s-o,a=r/(1+t),l=(r-a)/2,h=o+l,d=s-l;this.setWindow(h,d,e,i)}}},{key:"zoomOut",value:function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.getWindow(),o=n.start.valueOf(),s=n.end.valueOf(),r=s-o,a=o-r*t/2,l=s+r*t/2;this.setWindow(a,l,e,i)}}},{key:"redraw",value:function(){this._redraw()}},{key:"_redraw",value:function(){var t;this.redrawCount++;var e=this.dom;if(e&&e.container&&0!=e.root.offsetWidth){var i=!1,n=this.options,o=this.props;el(this.options.moment,this.body,this.options.hiddenDates),"top"==n.orientation?(za.addClassName(e.root,"vis-top"),za.removeClassName(e.root,"vis-bottom")):(za.removeClassName(e.root,"vis-top"),za.addClassName(e.root,"vis-bottom")),n.rtl?(za.addClassName(e.root,"vis-rtl"),za.removeClassName(e.root,"vis-ltr")):(za.addClassName(e.root,"vis-ltr"),za.removeClassName(e.root,"vis-rtl")),e.root.style.maxHeight=za.option.asSize(n.maxHeight,""),e.root.style.minHeight=za.option.asSize(n.minHeight,""),e.root.style.width=za.option.asSize(n.width,"");var s=e.root.offsetWidth;o.border.left=1,o.border.right=1,o.border.top=1,o.border.bottom=1,o.center.height=e.center.offsetHeight,o.left.height=e.left.offsetHeight,o.right.height=e.right.offsetHeight,o.top.height=e.top.clientHeight||-o.border.top,o.bottom.height=e.bottom.clientHeight||-o.border.bottom;var r=Math.max(o.left.height,o.center.height,o.right.height),a=o.top.height+r+o.bottom.height+o.border.top+o.border.bottom;e.root.style.height=za.option.asSize(n.height,"".concat(a,"px")),o.root.height=e.root.offsetHeight,o.background.height=o.root.height;var l=o.root.height-o.top.height-o.bottom.height;o.centerContainer.height=l,o.leftContainer.height=l,o.rightContainer.height=o.leftContainer.height,o.root.width=s,o.background.width=o.root.width,this.initialDrawDone||(o.scrollbarWidth=za.getScrollBarWidth());var h=e.leftContainer.clientWidth,d=e.rightContainer.clientWidth;n.verticalScroll?n.rtl?(o.left.width=h||-o.border.left,o.right.width=d+o.scrollbarWidth||-o.border.right):(o.left.width=h+o.scrollbarWidth||-o.border.left,o.right.width=d||-o.border.right):(o.left.width=h||-o.border.left,o.right.width=d||-o.border.right),this._setDOM();var u=this._updateScrollTop();"top"!=n.orientation.item&&(u+=Math.max(o.centerContainer.height-o.center.height-o.border.top-o.border.bottom,0)),e.center.style.transform="translateY(".concat(u,"px)");var c=0==o.scrollTop?"hidden":"",p=o.scrollTop==o.scrollTopMin?"hidden":"";e.shadowTop.style.visibility=c,e.shadowBottom.style.visibility=p,e.shadowTopLeft.style.visibility=c,e.shadowBottomLeft.style.visibility=p,e.shadowTopRight.style.visibility=c,e.shadowBottomRight.style.visibility=p,n.verticalScroll&&(e.rightContainer.className="vis-panel vis-right vis-vertical-scroll",e.leftContainer.className="vis-panel vis-left vis-vertical-scroll",e.shadowTopRight.style.visibility="hidden",e.shadowBottomRight.style.visibility="hidden",e.shadowTopLeft.style.visibility="hidden",e.shadowBottomLeft.style.visibility="hidden",e.left.style.top="0px",e.right.style.top="0px"),(!n.verticalScroll||o.center.height<o.centerContainer.height)&&(e.left.style.top="".concat(u,"px"),e.right.style.top="".concat(u,"px"),e.rightContainer.className=e.rightContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),e.leftContainer.className=e.leftContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),o.left.width=h||-o.border.left,o.right.width=d||-o.border.right,this._setDOM());var m=o.center.height>o.centerContainer.height;this.hammer.get("pan").set({direction:m?Vh.DIRECTION_ALL:Vh.DIRECTION_HORIZONTAL}),this.hammer.get("press").set({time:this.options.longSelectPressTime}),mn(t=this.components).call(t,(function(t){i=t.redraw()||i}));if(i){if(this.redrawCount<5)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.body.emitter.emit("changed")}}},{key:"_setDOM",value:function(){var t=this.props,e=this.dom;t.leftContainer.width=t.left.width,t.rightContainer.width=t.right.width;var i=t.root.width-t.left.width-t.right.width;t.center.width=i,t.centerContainer.width=i,t.top.width=i,t.bottom.width=i,e.background.style.height="".concat(t.background.height,"px"),e.backgroundVertical.style.height="".concat(t.background.height,"px"),e.backgroundHorizontal.style.height="".concat(t.centerContainer.height,"px"),e.centerContainer.style.height="".concat(t.centerContainer.height,"px"),e.leftContainer.style.height="".concat(t.leftContainer.height,"px"),e.rightContainer.style.height="".concat(t.rightContainer.height,"px"),e.background.style.width="".concat(t.background.width,"px"),e.backgroundVertical.style.width="".concat(t.centerContainer.width,"px"),e.backgroundHorizontal.style.width="".concat(t.background.width,"px"),e.centerContainer.style.width="".concat(t.center.width,"px"),e.top.style.width="".concat(t.top.width,"px"),e.bottom.style.width="".concat(t.bottom.width,"px"),e.background.style.left="0",e.background.style.top="0",e.backgroundVertical.style.left="".concat(t.left.width+t.border.left,"px"),e.backgroundVertical.style.top="0",e.backgroundHorizontal.style.left="0",e.backgroundHorizontal.style.top="".concat(t.top.height,"px"),e.centerContainer.style.left="".concat(t.left.width,"px"),e.centerContainer.style.top="".concat(t.top.height,"px"),e.leftContainer.style.left="0",e.leftContainer.style.top="".concat(t.top.height,"px"),e.rightContainer.style.left="".concat(t.left.width+t.center.width,"px"),e.rightContainer.style.top="".concat(t.top.height,"px"),e.top.style.left="".concat(t.left.width,"px"),e.top.style.top="0",e.bottom.style.left="".concat(t.left.width,"px"),e.bottom.style.top="".concat(t.top.height+t.centerContainer.height,"px"),e.center.style.left="0",e.left.style.left="0",e.right.style.left="0"}},{key:"setCurrentTime",value:function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)}},{key:"getCurrentTime",value:function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()}},{key:"_toTime",value:function(t){return sl(this,t,this.props.center.width)}},{key:"_toGlobalTime",value:function(t){return sl(this,t,this.props.root.width)}},{key:"_toScreen",value:function(t){return ol(this,t,this.props.center.width)}},{key:"_toGlobalScreen",value:function(t){return ol(this,t,this.props.root.width)}},{key:"_initAutoResize",value:function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()}},{key:"_startAutoResize",value:function(){var t=this;this._stopAutoResize(),this._onResize=function(){if(1==t.options.autoResize){if(t.dom.root){var e=t.dom.root.offsetHeight,i=t.dom.root.offsetWidth;i==t.props.lastWidth&&e==t.props.lastHeight||(t.props.lastWidth=i,t.props.lastHeight=e,t.props.scrollbarWidth=za.getScrollBarWidth(),t.body.emitter.emit("_change"))}}else t._stopAutoResize()},za.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=gl(this._onResize,1e3)}},{key:"_stopAutoResize",value:function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(za.removeEventListener(window,"resize",this._onResize),this._onResize=null)}},{key:"_onTouch",value:function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop}},{key:"_onPinch",value:function(t){this.touch.allowDragging=!1}},{key:"_onDrag",value:function(t){if(t&&this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),n=this._setScrollTop(this.touch.initialScrollTop+e);this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),n!=i&&this.emit("verticalDrag")}}},{key:"_setScrollTop",value:function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop}},{key:"_updateScrollTop",value:function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),this._isProgramaticallyScrolled=!0,this.props.scrollTop}},{key:"_getScrollTop",value:function(){return this.props.scrollTop}},{key:"_createConfigurator",value:function(){throw new Error("Cannot invoke abstract method _createConfigurator")}}]),t}();vl(cd.prototype);var pd=function(t){function e(t,i){var n,o;ba(this,e),(o=ar(this,dr(e).call(this))).body=t,o.defaultOptions={rtl:!1,showCurrentTime:!0,alignCurrentTime:void 0,moment:Ta,locales:dd,locale:"en"},o.options=za.extend({},o.defaultOptions),o.setOptions(i),o.options.locales=za.extend({},dd,o.options.locales);var s=o.defaultOptions.locales[o.defaultOptions.locale];return mn(n=Yi(o.options.locales)).call(n,(function(t){o.options.locales[t]=za.extend({},s,o.options.locales[t])})),o.offset=0,o._create(),o}return pr(e,t),xa(e,[{key:"_create",value:function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t}},{key:"destroy",value:function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null}},{key:"setOptions",value:function(t){t&&za.selectiveExtend(["rtl","showCurrentTime","alignCurrentTime","moment","locale","locales"],this.options,t)}},{key:"redraw",value:function(){if(this.options.showCurrentTime){var t,e,i=this.body.dom.backgroundVertical;this.bar.parentNode!=i&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),i.appendChild(this.bar),this.start());var n=this.options.moment(Jo()+this.offset);this.options.alignCurrentTime&&(n=n.startOf(this.options.alignCurrentTime));var o=this.body.util.toScreen(n),s=this.options.locales[this.options.locale];s||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),s=this.options.locales.en);var r=Pn(t=Pn(e="".concat(s.current," ")).call(e,s.time,": ")).call(t,n.format("dddd, MMMM Do YYYY, H:mm:ss"));r=r.charAt(0).toUpperCase()+r.substring(1),this.options.rtl?this.bar.style.transform="translateX(".concat(-1*o,"px)"):this.bar.style.transform="translateX(".concat(o,"px)"),this.bar.title=r}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1}},{key:"start",value:function(){var t=this;!function e(){t.stop();var i=1/t.body.range.conversion(t.body.domProps.center.width).scale/10;i<30&&(i=30),i>1e3&&(i=1e3),t.redraw(),t.body.emitter.emit("currentTimeTick"),t.currentTimeTimer=qr(e,i)}()}},{key:"stop",value:function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)}},{key:"setCurrentTime",value:function(t){var e=za.convert(t,"Date").valueOf(),i=Jo();this.offset=e-i,this.redraw()}},{key:"getCurrentTime",value:function(){return new Date(Jo()+this.offset)}}]),e}(Ja),md=Ce.find,fd=!0,gd=on("find");"find"in[]&&Array(1).find((function(){fd=!1})),V({target:"Array",proto:!0,forced:fd||!gd},{find:function(t){return md(this,t,arguments.length>1?arguments[1]:void 0)}});var vd=Pi("Array").find,bd=Array.prototype,yd=function(t){var e=t.find;return t===bd||t instanceof Array&&e===bd.find?vd:e},xd="undefined"!=typeof crypto&&crypto.getRandomValues&&crypto.getRandomValues.bind(crypto)||"undefined"!=typeof msCrypto&&"function"==typeof msCrypto.getRandomValues&&msCrypto.getRandomValues.bind(msCrypto),wd=new Uint8Array(16);function _d(){if(!xd)throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");return xd(wd)}for(var kd=[],Dd=0;Dd<256;++Dd)kd[Dd]=(Dd+256).toString(16).substr(1);function Id(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"===t?new Array(16):null,t=null);var o=(t=t||{}).random||(t.rng||_d)();if(o[6]=15&o[6]|64,o[8]=63&o[8]|128,e)for(var s=0;s<16;++s)e[n+s]=o[s];return e||function(t,e){var i=e||0,n=kd;return[n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]]].join("")}(o)}var Cd=ot.includes,Ld=on("indexOf",{ACCESSORS:!0,1:0});V({target:"Array",proto:!0,forced:!Ld},{includes:function(t){return Cd(this,t,arguments.length>1?arguments[1]:void 0)}});var Gd=Pi("Array").includes,Md=Pt("match"),Sd=function(t){if(function(t){var e;return x(t)&&(void 0!==(e=t[Md])?!!e:"RegExp"==f(t))}(t))throw TypeError("The method doesn't accept regular expressions");return t},Qd=Pt("match");V({target:"String",proto:!0,forced:!function(t){var e=/./;try{"/./"[t](e)}catch(i){try{return e[Qd]=!1,"/./"[t](e)}catch(t){}}return!1}("includes")},{includes:function(t){return!!~String(b(this)).indexOf(Sd(t),arguments.length>1?arguments[1]:void 0)}});var Nd=Pi("String").includes,Fd=Array.prototype,Zd=String.prototype,Bd=function(t){var e=t.includes;return t===Fd||t instanceof Array&&e===Fd.includes?Gd:"string"==typeof t||t===Zd||t instanceof String&&e===Zd.includes?Nd:e};function Od(t){wr(t).call(t,(function(t,e){return t.data.start-e.data.start}))}function Wd(t){wr(t).call(t,(function(t,e){return("end"in t.data?t.data.end:t.data.start)-("end"in e.data?e.data.end:e.data.start)}))}function Ad(t,e,i,n){if(i)for(var o=0;o<t.length;o++)t[o].top=null;for(o=0;o<t.length;o++){var s=t[o];if(s.stack&&null===s.top){s.top=e.axis;var r=!1;do{for(var a=null,l=0,h=t.length;l<h;l++){var d=t[l];if(r=n()||!1)return!0;if(null!==d.top&&d!==s&&d.stack&&Rd(s,d,e.item,d.options.rtl)){a=d;break}}null!=a&&(s.top=a.top+a.height+e.item.vertical)}while(a)}}return r}function Xd(t,e,i){for(var n=0;n<t.length;n++)t[n].top=null;var o=i.height;for(n=0;n<t.length;n++){var s=t[n];if(s.stack&&null===s.top){s.top=s.baseTop;do{for(var r=null,a=0,l=t.length;a<l;a++){var h=t[a];if(null!==h.top&&h!==s&&Rd(s,h,e.item,h.options.rtl)){r=h;break}}null!=r&&(s.top=r.top+r.height+e.item.vertical),s.top+s.height>o&&(o=s.top+s.height)}while(r)}}i.height=o-i.top+.5*e.item.vertical}function Td(t,e,i,n){for(var o=0;o<t.length;o++)if(null==t[o].data.subgroup)t[o].top=e.item.vertical;else if(void 0!==t[o].data.subgroup&&n){var s=0;for(var r in i)i.hasOwnProperty(r)&&1==i[r].visible&&i[r].index<i[t[o].data.subgroup].index&&(s+=i[r].height,i[t[o].data.subgroup].top=s);t[o].top=s+.5*e.item.vertical}n||Yd(t,e,i)}function Yd(t,e,i){for(var n in i)if(i.hasOwnProperty(n)){i[n].top=0;do{var o=null;for(var s in i)if(null!==i[s].top&&s!==n&&i[n].index>i[s].index&&Ud(i[n],i[s])){o=i[s];break}null!=o&&(i[n].top=o.top+o.height)}while(o)}for(var r=0;r<t.length;r++)void 0!==t[r].data.subgroup&&(t[r].top=i[t[r].data.subgroup].top+.5*e.item.vertical)}function Ed(t,e,i){var n=!1,o=[];for(var s in i)i[s].hasOwnProperty("index")?o[i[s].index]=s:o.push(s);for(var r=0;r<o.length;r++)if(s=o[r],i.hasOwnProperty(s)){for(var a in n=n||i[s].stack,i[s].top=0,i)i[a].visible&&i[s].index>i[a].index&&(i[s].top+=i[a].height);for(var l=t[s],h=0;h<l.length;h++)void 0!==l[h].data.subgroup&&(l[h].top=i[l[h].data.subgroup].top+.5*e.item.vertical,i[s].stack&&(l[h].baseTop=l[h].top));n&&i[s].stack&&Xd(t[s],e,i[s])}}function Rd(t,e,i,n){return n?t.right-i.horizontal+.001<e.right+e.width&&t.right+t.width+i.horizontal-.001>e.right&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top:t.left-i.horizontal+.001<e.left+e.width&&t.left+t.width+i.horizontal-.001>e.left&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top}function Ud(t,e){var i=t.start<e.end&&t.end>e.start,n=t.top<e.top+e.height&&t.top+t.height>e.top;return i&&n}var zd=Object.freeze({__proto__:null,orderByStart:Od,orderByEnd:Wd,stack:Ad,substack:Xd,nostack:Td,stackSubgroups:Yd,stackSubgroupsWithInnerStack:Ed,collision:Rd,collisionByTimes:Ud}),Vd="__background__",jd=function(){function t(e,i,n){var o=this;if(ba(this,t),this.groupId=e,this.subgroups={},this.subgroupStack={},this.subgroupStackAll=!1,this.subgroupVisibility={},this.doInnerStack=!1,this.shouldBailStackItems=!1,this.subgroupIndex=0,this.subgroupOrderer=i&&i.subgroupOrder,this.itemSet=n,this.isVisible=null,this.stackDirty=!0,this._disposeCallbacks=[],i&&i.nestedGroups&&(this.nestedGroups=i.nestedGroups,0==i.showNested?this.showNested=!1:this.showNested=!0),i&&i.subgroupStack)if("boolean"==typeof i.subgroupStack)this.doInnerStack=i.subgroupStack,this.subgroupStackAll=i.subgroupStack;else for(var s in i.subgroupStack)this.subgroupStack[s]=i.subgroupStack[s],this.doInnerStack=this.doInnerStack||i.subgroupStack[s];i&&i.heightMode?this.heightMode=i.heightMode:this.heightMode=n.options.groupHeightMode,this.nestedInGroup=null,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.itemsInRange=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var r=function(){o.checkRangedItems=!0};this.itemSet.body.emitter.on("checkRangedItems",r),this._disposeCallbacks.push((function(){o.itemSet.body.emitter.off("checkRangedItems",r)})),this._create(),this.setData(i)}return xa(t,[{key:"_create",value:function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["vis-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.style.position="absolute",this.dom.marker.innerHTML="",this.dom.background.appendChild(this.dom.marker)}},{key:"setData",value:function(t){if(!this.itemSet.groupTouchParams.isDragging){var e,i,n;if(t&&t.subgroupVisibility)for(var o in t.subgroupVisibility)this.subgroupVisibility[o]=t.subgroupVisibility[o];if(this.itemSet.options&&this.itemSet.options.groupTemplate)e=(i=va(n=this.itemSet.options.groupTemplate).call(n,this))(t,this.dom.inner);else e=t&&t.content;if(e instanceof Element){for(;this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else e instanceof Object&&e.isReactComponent||(e instanceof Object?i(t,this.dom.inner):this.dom.inner.innerHTML=null!=e?e:this.groupId||"");this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?za.removeClassName(this.dom.inner,"vis-hidden"):za.addClassName(this.dom.inner,"vis-hidden"),t&&t.nestedGroups?(this.nestedGroups&&this.nestedGroups==t.nestedGroups||(this.nestedGroups=t.nestedGroups),void 0===t.showNested&&void 0!==this.showNested||(0==t.showNested?this.showNested=!1:this.showNested=!0),za.addClassName(this.dom.label,"vis-nesting-group"),this.showNested?(za.removeClassName(this.dom.label,"collapsed"),za.addClassName(this.dom.label,"expanded")):(za.removeClassName(this.dom.label,"expanded"),za.addClassName(this.dom.label,"collapsed"))):this.nestedGroups&&(this.nestedGroups=null,za.removeClassName(this.dom.label,"collapsed"),za.removeClassName(this.dom.label,"expanded"),za.removeClassName(this.dom.label,"vis-nesting-group")),t&&(t.treeLevel||t.nestedInGroup)?(za.addClassName(this.dom.label,"vis-nested-group"),t.treeLevel?za.addClassName(this.dom.label,"vis-group-level-"+t.treeLevel):za.addClassName(this.dom.label,"vis-group-level-unknown-but-gte1")):za.addClassName(this.dom.label,"vis-group-level-0");var s=t&&t.className||null;s!=this.className&&(this.className&&(za.removeClassName(this.dom.label,this.className),za.removeClassName(this.dom.foreground,this.className),za.removeClassName(this.dom.background,this.className),za.removeClassName(this.dom.axis,this.className)),za.addClassName(this.dom.label,s),za.addClassName(this.dom.foreground,s),za.addClassName(this.dom.background,s),za.addClassName(this.dom.axis,s),this.className=s),this.style&&(za.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(za.addCssText(this.dom.label,t.style),this.style=t.style)}}},{key:"getLabelWidth",value:function(){return this.props.label.width}},{key:"_didMarkerHeightChange",value:function(){var t=this.dom.marker.clientHeight;if(t!=this.lastMarkerHeight){this.lastMarkerHeight=t;var e={},i=0;if(mn(za).call(za,this.items,(function(t,n){if(t.dirty=!0,t.displayed){e[n]=t.redraw(!0),i=e[n].length}})),i>0)for(var n=function(t){mn(za).call(za,e,(function(e){e[t]()}))},o=0;o<i;o++)n(o);return!0}return!1}},{key:"_calculateGroupSizeAndPosition",value:function(){var t=this.dom.foreground,e=t.offsetTop,i=t.offsetLeft,n=t.offsetWidth;this.top=e,this.right=i,this.width=n}},{key:"_shouldBailItemsRedraw",value:function(){var t=this,e=this.itemSet.options.onTimeout,i={relativeBailingTime:this.itemSet.itemsSettingTime,bailTimeMs:e&&e.timeoutMs,userBailFunction:e&&e.callback,shouldBailStackItems:this.shouldBailStackItems},n=null;if(!this.itemSet.initialDrawDone){if(i.shouldBailStackItems)return!0;Math.abs(Jo()-new Date(i.relativeBailingTime))>i.bailTimeMs&&(i.userBailFunction&&null==this.itemSet.userContinueNotBail?i.userBailFunction((function(e){t.itemSet.userContinueNotBail=e,n=!e})):n=0==t.itemSet.userContinueNotBail)}return n}},{key:"_redrawItems",value:function(t,e,i,n){var o=this;if(t||this.stackDirty||this.isVisible&&!e){var s,r,a,l,h,d,u={byEnd:En(s=this.orderedItems.byEnd).call(s,(function(t){return!t.isCluster})),byStart:En(r=this.orderedItems.byStart).call(r,(function(t){return!t.isCluster}))},c={byEnd:mo(new Ar(En(a=Ln(l=this.orderedItems.byEnd).call(l,(function(t){return t.cluster}))).call(a,(function(t){return!!t})))),byStart:mo(new Ar(En(h=Ln(d=this.orderedItems.byStart).call(d,(function(t){return t.cluster}))).call(h,(function(t){return!!t}))))},p=function(){var t,e,i,s=o._updateItemsInRange(u,En(t=o.visibleItems).call(t,(function(t){return!t.isCluster})),n),r=o._updateClustersInRange(c,En(e=o.visibleItems).call(e,(function(t){return t.isCluster})),n);return Pn(i=[]).call(i,mo(s),mo(r))},m=function(t){var e={},i=function(i){var n,s=En(n=o.visibleItems).call(n,(function(t){return t.data.subgroup===i}));e[i]=t?wr(s).call(s,(function(e,i){return t(e.data,i.data)})):s};for(var n in o.subgroups)i(n);return e};if("function"==typeof this.itemSet.options.order){var f=this;if(this.doInnerStack&&this.itemSet.options.stackSubgroups){Ed(m(this.itemSet.options.order),i,this.subgroups),this.visibleItems=p(),this._updateSubGroupHeights(i)}else{var g,v,b,y;this.visibleItems=p(),this._updateSubGroupHeights(i);var x=wr(g=En(v=_o(b=this.visibleItems).call(b)).call(v,(function(t){return t.isCluster||!t.isCluster&&!t.cluster}))).call(g,(function(t,e){return f.itemSet.options.order(t.data,e.data)}));this.shouldBailStackItems=Ad(x,i,!0,va(y=this._shouldBailItemsRedraw).call(y,this))}}else{var w;if(this.visibleItems=p(),this._updateSubGroupHeights(i),this.itemSet.options.stack)if(this.doInnerStack&&this.itemSet.options.stackSubgroups)Ed(m(),i,this.subgroups);else this.shouldBailStackItems=Ad(this.visibleItems,i,!0,va(w=this._shouldBailItemsRedraw).call(w,this));else Td(this.visibleItems,i,this.subgroups,this.itemSet.options.stackSubgroups)}for(var _=0;_<this.visibleItems.length;_++)this.visibleItems[_].repositionX(),void 0!==this.subgroupVisibility[this.visibleItems[_].data.subgroup]&&(this.subgroupVisibility[this.visibleItems[_].data.subgroup]||this.visibleItems[_].hide());this.itemSet.options.cluster&&mn(za).call(za,this.items,(function(t){t.cluster&&t.displayed&&t.hide()})),this.shouldBailStackItems&&this.itemSet.body.emitter.emit("destroyTimeline"),this.stackDirty=!1}}},{key:"_didResize",value:function(t,e){t=za.updateProperty(this,"height",e)||t;var i=this.dom.inner.clientWidth,n=this.dom.inner.clientHeight;return t=za.updateProperty(this.props.label,"width",i)||t,t=za.updateProperty(this.props.label,"height",n)||t}},{key:"_applyGroupHeight",value:function(t){this.dom.background.style.height="".concat(t,"px"),this.dom.foreground.style.height="".concat(t,"px"),this.dom.label.style.height="".concat(t,"px")}},{key:"_updateItemsVerticalPosition",value:function(t){for(var e=0,i=this.visibleItems.length;e<i;e++){var n=this.visibleItems[e];n.repositionY(t),this.isVisible||this.groupId==Vd||n.displayed&&n.hide()}}},{key:"redraw",value:function(t,e,i,n){var o,s,r,a,l,h,d,u=this,c=!1,p=this.isVisible,m=[function(){i=u._didMarkerHeightChange.call(u)||i},va(o=this._updateSubGroupHeights).call(o,this,e),va(s=this._calculateGroupSizeAndPosition).call(s,this),function(){var i;u.isVisible=va(i=u._isGroupVisible).call(i,u)(t,e)},function(){var n;va(n=u._redrawItems).call(n,u)(i,p,e,t)},va(r=this._updateSubgroupsSizes).call(r,this),function(){var t;h=va(t=u._calculateHeight).call(t,u)(e)},va(a=this._calculateGroupSizeAndPosition).call(a,this),function(){var t;c=va(t=u._didResize).call(t,u)(c,h)},function(){var t;va(t=u._applyGroupHeight).call(t,u)(h)},function(){var t;va(t=u._updateItemsVerticalPosition).call(t,u)(e)},va(l=function(){return!u.isVisible&&u.height&&(c=!1),c}).call(l,this)];return n?m:(mn(m).call(m,(function(t){d=t()})),d)}},{key:"_updateSubGroupHeights",value:function(t){var e=this;if(Yi(this.subgroups).length>0){var i=this;this._resetSubgroups(),mn(za).call(za,this.visibleItems,(function(n){void 0!==n.data.subgroup&&(i.subgroups[n.data.subgroup].height=Math.max(i.subgroups[n.data.subgroup].height,n.height+t.item.vertical),i.subgroups[n.data.subgroup].visible=void 0===e.subgroupVisibility[n.data.subgroup]||Boolean(e.subgroupVisibility[n.data.subgroup]))}))}}},{key:"_isGroupVisible",value:function(t,e){return this.top<=t.body.domProps.centerContainer.height-t.body.domProps.scrollTop+e.axis&&this.top+this.height+e.axis>=-t.body.domProps.scrollTop}},{key:"_calculateHeight",value:function(t){var e,i;if((i="fixed"===this.heightMode?za.toArray(this.items):this.visibleItems).length>0){var n=i[0].top,o=i[0].top+i[0].height;if(mn(za).call(za,i,(function(t){n=Math.min(n,t.top),o=Math.max(o,t.top+t.height)})),n>t.axis){var s=n-t.axis;o-=s,mn(za).call(za,i,(function(t){t.top-=s}))}e=Math.ceil(o+t.item.vertical/2),"fitItems"!==this.heightMode&&(e=Math.max(e,this.props.label.height))}else e=this.props.label.height;return e}},{key:"show",value:function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)}},{key:"hide",value:function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var n=this.dom.axis;n.parentNode&&n.parentNode.removeChild(n)}},{key:"add",value:function(t){var e;if(this.items[t.id]=t,t.setParent(this),this.stackDirty=!0,void 0!==t.data.subgroup&&(this._addToSubgroup(t),this.orderSubgroups()),!Bd(e=this.visibleItems).call(e,t)){var i=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,i)}}},{key:"_addToSubgroup",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:t.data.subgroup;null!=e&&void 0===this.subgroups[e]&&(this.subgroups[e]={height:0,top:0,start:t.data.start,end:t.data.end||t.data.start,visible:!1,index:this.subgroupIndex,items:[],stack:this.subgroupStackAll||this.subgroupStack[e]||!1},this.subgroupIndex++),new Date(t.data.start)<new Date(this.subgroups[e].start)&&(this.subgroups[e].start=t.data.start);var i=t.data.end||t.data.start;new Date(i)>new Date(this.subgroups[e].end)&&(this.subgroups[e].end=i),this.subgroups[e].items.push(t)}},{key:"_updateSubgroupsSizes",value:function(){var t=this;if(t.subgroups){var e=function(e){var i,n=t.subgroups[e].items[0].data.end||t.subgroups[e].items[0].data.start,o=t.subgroups[e].items[0].data.start,s=n-1;mn(i=t.subgroups[e].items).call(i,(function(t){new Date(t.data.start)<new Date(o)&&(o=t.data.start);var e=t.data.end||t.data.start;new Date(e)>new Date(s)&&(s=e)})),t.subgroups[e].start=o,t.subgroups[e].end=new Date(s-1)};for(var i in t.subgroups)e(i)}}},{key:"orderSubgroups",value:function(){if(void 0!==this.subgroupOrderer){var t=[];if("string"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push({subgroup:e,sortField:this.subgroups[e].items[0].data[this.subgroupOrderer]});wr(t).call(t,(function(t,e){return t.sortField-e.sortField}))}else if("function"==typeof this.subgroupOrderer){for(var i in this.subgroups)t.push(this.subgroups[i].items[0].data);wr(t).call(t,this.subgroupOrderer)}if(t.length>0)for(var n=0;n<t.length;n++)this.subgroups[t[n].subgroup].index=n}}},{key:"_resetSubgroups",value:function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1,this.subgroups[t].height=0)}},{key:"remove",value:function(t){var e,i;delete this.items[t.id],t.setParent(null),this.stackDirty=!0;var n=Fo(e=this.visibleItems).call(e,t);-1!=n&&Hr(i=this.visibleItems).call(i,n,1),void 0!==t.data.subgroup&&(this._removeFromSubgroup(t),this.orderSubgroups())}},{key:"_removeFromSubgroup",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:t.data.subgroup;if(null!=e){var i=this.subgroups[e];if(i){var n,o,s=Fo(n=i.items).call(n,t);if(s>=0)Hr(o=i.items).call(o,s,1),i.items.length?this._updateSubgroupsSizes():delete this.subgroups[e]}}}},{key:"removeFromDataSet",value:function(t){this.itemSet.removeItem(t.id)}},{key:"order",value:function(){for(var t=za.toArray(this.items),e=[],i=[],n=0;n<t.length;n++)void 0!==t[n].data.end&&i.push(t[n]),e.push(t[n]);this.orderedItems={byStart:e,byEnd:i},Od(this.orderedItems.byStart),Wd(this.orderedItems.byEnd)}},{key:"_updateItemsInRange",value:function(t,e,i){var n=[],o={};if(!this.isVisible&&this.groupId!=Vd){for(var s=0;s<e.length;s++){var r=e[s];r.displayed&&r.hide()}return n}var a=(i.end-i.start)/4,l=i.start-a,h=i.end+a;if(e.length>0)for(var d=0;d<e.length;d++)this._checkIfVisibleWithReference(e[d],n,o,i);var u=za.binarySearchCustom(t.byStart,(function(t){return t<l?-1:t<=h?0:1}),"data","start");if(this._traceVisible(u,t.byStart,n,o,(function(t){return t.data.start<l||t.data.start>h})),1==this.checkRangedItems){this.checkRangedItems=!1;for(var c=0;c<t.byEnd.length;c++)this._checkIfVisibleWithReference(t.byEnd[c],n,o,i)}else{var p=za.binarySearchCustom(t.byEnd,(function(t){return t<l?-1:0}),"data","end");this._traceVisible(p,t.byEnd,n,o,(function(t){return t.data.end<l||t.data.start>h}))}for(var m={},f=0,g=0;g<n.length;g++){var v=n[g];if(!v.displayed){m[g]=v.redraw(!0),f=m[g].length}}if(f>0)for(var b=function(t){mn(za).call(za,m,(function(e){e[t]()}))},y=0;y<f;y++)b(y);for(var x=0;x<n.length;x++)n[x].repositionX();return n}},{key:"_traceVisible",value:function(t,e,i,n,o){if(-1!=t){for(var s=t;s>=0;s--){var r=e[s];if(o(r))break;r.isCluster&&!r.hasItems()||r.cluster||void 0===n[r.id]&&(n[r.id]=!0,i.push(r))}for(var a=t+1;a<e.length;a++){var l=e[a];if(o(l))break;l.isCluster&&!l.hasItems()||l.cluster||void 0===n[l.id]&&(n[l.id]=!0,i.push(l))}}}},{key:"_checkIfVisible",value:function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()}},{key:"_checkIfVisibleWithReference",value:function(t,e,i,n){t.isVisible(n)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()}},{key:"_updateClustersInRange",value:function(t,e,i){var n=[],o={};if(e.length>0)for(var s=0;s<e.length;s++)this._checkIfVisibleWithReference(e[s],n,o,i);for(var r=0;r<t.byStart.length;r++)this._checkIfVisibleWithReference(t.byStart[r],n,o,i);for(var a=0;a<t.byEnd.length;a++)this._checkIfVisibleWithReference(t.byEnd[a],n,o,i);for(var l={},h=0,d=0;d<n.length;d++){var u=n[d];if(!u.displayed){l[d]=u.redraw(!0),h=l[d].length}}if(h>0)for(var c=0;c<h;c++)mn(za).call(za,l,(function(t){t[c]()}));for(var p=0;p<n.length;p++)n[p].repositionX();return n}},{key:"changeSubgroup",value:function(t,e,i){this._removeFromSubgroup(t,e),this._addToSubgroup(t,i),this.orderSubgroups()}},{key:"dispose",value:function(){var t;for(this.hide();t=this._disposeCallbacks.pop();)t()}}]),t}(),Hd=function(t){function e(t,i,n){var o;return ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).width=0,o.height=0,o.top=0,o.left=0,o}return pr(e,t),xa(e,[{key:"redraw",value:function(t,e,i){this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var n=0,o=this.visibleItems.length;n<o;n++){this.visibleItems[n].repositionY(e)}return!1}},{key:"show",value:function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)}}]),e}(jd);Ph('.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:none}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-left,.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0}.vis-item.vis-range .vis-drag-right{right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-item.vis-cluster{vertical-align:center;text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-cluster-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item.vis-cluster-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIml0ZW0uY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLFVBQ0UsaUJBQWtCLENBQ2xCLGFBQWMsQ0FDZCxvQkFBcUIsQ0FDckIsZ0JBQWlCLENBQ2pCLHdCQUF5QixDQUN6QixvQkFBcUIsQ0FDckIsU0FFRixDQUVBLHVCQUNFLG9CQUFxQixDQUNyQix3QkFBeUIsQ0FHekIsU0FDRixDQUVBLDJCQUNFLFdBQ0YsQ0FFQSxpQ0FDRSx3QkFDRixDQUVBLGtCQUNFLGlCQUFrQixDQUNsQixrQkFBbUIsQ0FDbkIsaUJBQ0YsQ0FFQSxvQkFDRSxlQUNGLENBRUEsa0JBQ0UsaUJBQWtCLENBQ2xCLFNBQVUsQ0FDVixnQkFBaUIsQ0FDakIsa0JBQW1CLENBQ25CLGlCQUNGLENBRUEsb0JBQ0Usa0JBQW1CLENBQ25CLGlCQUFrQixDQUNsQixxQkFDRixDQUVBLHlCQUNFLFdBQVksQ0FDWixxQ0FBMEMsQ0FDMUMscUJBQXNCLENBQ3RCLFNBQVUsQ0FDVixRQUNGLENBRUEsNkJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxXQUFZLENBQ1osU0FBVSxDQUNWLFFBQVMsQ0FDVCxlQUNGLENBRUEsd0JBQ0Usa0JBQ0YsQ0FFQSxzQ0FDRSxpQkFBa0IsQ0FDbEIsb0JBQ0YsQ0FFQSwyQ0FDRSxpQkFBa0IsQ0FDbEIsb0JBQ0YsQ0FFQSxtQkFDRSxTQUFVLENBQ1YsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixxQkFBc0IsQ0FDdEIsdUJBQ0YsQ0FFQSw0QkFDRSxrQkFBbUIsQ0FDbkIscUJBQXNCLENBQ3RCLFdBQ0YsQ0FFQSxvQ0FDRSxpQkFBa0IsQ0FDbEIsa0JBQW1CLENBQ25CLFVBQVksQ0FDWixXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGtCQUFtQixDQUNuQixXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGNBQWdCLENBQ2hCLGlCQUFtQixDQUNuQixtQkFBcUIsQ0FDckIsc0JBQ0YsQ0FFQSxnREFDRSxpQkFBa0IsQ0FDbEIsS0FBUSxDQUNSLFVBQVcsQ0FDWCxXQUFZLENBQ1oscUJBQXNCLENBQ3RCLGFBQWdCLENBQ2hCLGNBQWUsQ0FFZix3Q0FBMEMsQ0FDMUMscUNBQXVDLENBQ3ZDLG9DQUFzQyxDQUN0QyxtQ0FBcUMsQ0FDckMsZ0NBQ0YsQ0FFQSxzQkFDRSxXQUNGLENBRUEsMEJBQ0UsVUFDRixDQUVBLDREQUNFLGVBQWdCLENBQ2hCLFNBQVUsQ0FDViw0QkFBOEIsQ0FDOUIsY0FBZSxDQUNmLGVBQWlCLENBRWpCLG1DQUFxQyxDQUNyQyxnQ0FBa0MsQ0FDbEMsK0JBQWlDLENBQ2pDLDhCQUFnQyxDQUNoQywyQkFDRixDQUVBLDREQUNFLGNBQ0YsQ0FFQSx3RUFDRSxVQUNGLENBRUEsMkJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxXQUFZLENBQ1osS0FBTSxDQUNOLE1BQVMsQ0FDVCxXQUNGLENBRUEsbUNBT0UsU0FBVSxDQUVWLGVBQ0YsQ0FFQSx1RUFYRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLGFBQWMsQ0FDZCxhQUFjLENBQ2QsV0FBWSxDQUNaLEtBZ0JGLENBVkEsb0NBT0UsVUFBVyxDQUVYLGVBQ0YsQ0FFQSxpR0FFRSxXQUNGLENBRUEsc0JBQ0UscUJBQXNCLENBQ3RCLGlCQUFrQixDQUNsQixrQkFBbUIsQ0FDbkIsaUJBQ0YsQ0FFQSwyQkFDRSxTQUFVLENBQ1YsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixxQkFBc0IsQ0FDdEIsdUJBQ0YsQ0FFQSwwQkFDRSxpQkFBa0IsQ0FDbEIsU0FBVSxDQUNWLGdCQUFpQixDQUNqQixrQkFBbUIsQ0FDbkIsaUJBQ0YiLCJmaWxlIjoiaXRlbS5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyJcbi52aXMtaXRlbSB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICMxQTFBMUE7XG4gIGJvcmRlci1jb2xvcjogIzk3QjBGODtcbiAgYm9yZGVyLXdpZHRoOiAxcHg7XG4gIGJhY2tncm91bmQtY29sb3I6ICNENURERjY7XG4gIGRpc3BsYXk6IGlubGluZS1ibG9jaztcbiAgei1pbmRleDogMTtcbiAgLypvdmVyZmxvdzogaGlkZGVuOyovXG59XG5cbi52aXMtaXRlbS52aXMtc2VsZWN0ZWQge1xuICBib3JkZXItY29sb3I6ICNGRkMyMDA7XG4gIGJhY2tncm91bmQtY29sb3I6ICNGRkY3ODU7XG5cbiAgLyogei1pbmRleCBtdXN0IGJlIGhpZ2hlciB0aGFuIHRoZSB6LWluZGV4IG9mIGN1c3RvbSB0aW1lIGJhciBhbmQgY3VycmVudCB0aW1lIGJhciAqL1xuICB6LWluZGV4OiAyO1xufVxuXG4udmlzLWVkaXRhYmxlLnZpcy1zZWxlY3RlZCB7XG4gIGN1cnNvcjogbW92ZTtcbn1cblxuLnZpcy1pdGVtLnZpcy1wb2ludC52aXMtc2VsZWN0ZWQge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiAjRkZGNzg1O1xufVxuXG4udmlzLWl0ZW0udmlzLWJveCB7XG4gIHRleHQtYWxpZ246IGNlbnRlcjtcbiAgYm9yZGVyLXN0eWxlOiBzb2xpZDtcbiAgYm9yZGVyLXJhZGl1czogMnB4O1xufVxuXG4udmlzLWl0ZW0udmlzLXBvaW50IHtcbiAgYmFja2dyb3VuZDogbm9uZTtcbn1cblxuLnZpcy1pdGVtLnZpcy1kb3Qge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHBhZGRpbmc6IDA7XG4gIGJvcmRlci13aWR0aDogNHB4O1xuICBib3JkZXItc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmFkaXVzOiA0cHg7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2Uge1xuICBib3JkZXItc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmFkaXVzOiAycHg7XG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtaXRlbS52aXMtYmFja2dyb3VuZCB7XG4gIGJvcmRlcjogbm9uZTtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgyMTMsIDIyMSwgMjQ2LCAwLjQpO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG59XG5cbi52aXMtaXRlbSAudmlzLWl0ZW0tb3ZlcmZsb3cge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbn1cblxuLnZpcy1pdGVtLXZpc2libGUtZnJhbWUge1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLWl0ZW0udmlzLXJhbmdlIC52aXMtaXRlbS1jb250ZW50IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG59XG5cbi52aXMtaXRlbS52aXMtYmFja2dyb3VuZCAudmlzLWl0ZW0tY29udGVudCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgZGlzcGxheTogaW5saW5lLWJsb2NrO1xufVxuXG4udmlzLWl0ZW0udmlzLWxpbmUge1xuICBwYWRkaW5nOiAwO1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAwO1xuICBib3JkZXItbGVmdC13aWR0aDogMXB4O1xuICBib3JkZXItbGVmdC1zdHlsZTogc29saWQ7XG59XG5cbi52aXMtaXRlbSAudmlzLWl0ZW0tY29udGVudCB7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG4gIHBhZGRpbmc6IDVweDtcbn1cblxuLnZpcy1pdGVtIC52aXMtb25VcGRhdGVUaW1lLXRvb2x0aXAge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIGJhY2tncm91bmQ6ICM0ZjgxYmQ7XG4gIGNvbG9yOiB3aGl0ZTtcbiAgd2lkdGg6IDIwMHB4O1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIHBhZGRpbmc6IDVweDtcbiAgYm9yZGVyLXJhZGl1czogMXB4O1xuICB0cmFuc2l0aW9uOiAwLjRzO1xuICAtby10cmFuc2l0aW9uOiAwLjRzO1xuICAtbW96LXRyYW5zaXRpb246IDAuNHM7XG4gIC13ZWJraXQtdHJhbnNpdGlvbjogMC40cztcbn1cblxuLnZpcy1pdGVtIC52aXMtZGVsZXRlLCAudmlzLWl0ZW0gLnZpcy1kZWxldGUtcnRsIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDBweDtcbiAgd2lkdGg6IDI0cHg7XG4gIGhlaWdodDogMjRweDtcbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcbiAgcGFkZGluZzogMHB4IDVweDtcbiAgY3Vyc29yOiBwb2ludGVyO1xuXG4gIC13ZWJraXQtdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgLW1vei10cmFuc2l0aW9uOiBiYWNrZ3JvdW5kIDAuMnMgbGluZWFyO1xuICAtbXMtdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgLW8tdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbn1cblxuLnZpcy1pdGVtIC52aXMtZGVsZXRlIHtcbiAgcmlnaHQ6IC0yNHB4O1xufVxuXG4udmlzLWl0ZW0gLnZpcy1kZWxldGUtcnRsIHtcbiAgbGVmdDogLTI0cHg7XG59XG5cbi52aXMtaXRlbSAudmlzLWRlbGV0ZTphZnRlciwgLnZpcy1pdGVtIC52aXMtZGVsZXRlLXJ0bDphZnRlciB7XG4gIGNvbnRlbnQ6IFwiXFwwMEQ3XCI7IC8qIE1VTFRJUExJQ0FUSU9OIFNJR04gKi9cbiAgY29sb3I6IHJlZDtcbiAgZm9udC1mYW1pbHk6IGFyaWFsLCBzYW5zLXNlcmlmO1xuICBmb250LXNpemU6IDIycHg7XG4gIGZvbnQtd2VpZ2h0OiBib2xkO1xuXG4gIC13ZWJraXQtdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIC1tb3otdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIC1tcy10cmFuc2l0aW9uOiBjb2xvciAwLjJzIGxpbmVhcjtcbiAgLW8tdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIHRyYW5zaXRpb246IGNvbG9yIDAuMnMgbGluZWFyO1xufVxuXG4udmlzLWl0ZW0gLnZpcy1kZWxldGU6aG92ZXIsIC52aXMtaXRlbSAudmlzLWRlbGV0ZS1ydGw6aG92ZXIge1xuICBiYWNrZ3JvdW5kOiByZWQ7XG59XG5cbi52aXMtaXRlbSAudmlzLWRlbGV0ZTpob3ZlcjphZnRlciwgLnZpcy1pdGVtIC52aXMtZGVsZXRlLXJ0bDpob3ZlcjphZnRlciB7XG4gIGNvbG9yOiB3aGl0ZTtcbn1cblxuLnZpcy1pdGVtIC52aXMtZHJhZy1jZW50ZXIge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHRvcDogMDtcbiAgbGVmdDogMHB4O1xuICBjdXJzb3I6IG1vdmU7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2UgLnZpcy1kcmFnLWxlZnQge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAyNHB4O1xuICBtYXgtd2lkdGg6IDIwJTtcbiAgbWluLXdpZHRoOiAycHg7XG4gIGhlaWdodDogMTAwJTtcbiAgdG9wOiAwO1xuICBsZWZ0OiAtNHB4O1xuXG4gIGN1cnNvcjogdy1yZXNpemU7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2UgLnZpcy1kcmFnLXJpZ2h0IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB3aWR0aDogMjRweDtcbiAgbWF4LXdpZHRoOiAyMCU7XG4gIG1pbi13aWR0aDogMnB4O1xuICBoZWlnaHQ6IDEwMCU7XG4gIHRvcDogMDtcbiAgcmlnaHQ6IC00cHg7XG5cbiAgY3Vyc29yOiBlLXJlc2l6ZTtcbn1cblxuLnZpcy1yYW5nZS52aXMtaXRlbS52aXMtcmVhZG9ubHkgLnZpcy1kcmFnLWxlZnQsXG4udmlzLXJhbmdlLnZpcy1pdGVtLnZpcy1yZWFkb25seSAudmlzLWRyYWctcmlnaHQge1xuICBjdXJzb3I6IGF1dG87XG59XG5cbi52aXMtaXRlbS52aXMtY2x1c3RlciB7XG4gIHZlcnRpY2FsLWFsaWduOiBjZW50ZXI7XG4gIHRleHQtYWxpZ246IGNlbnRlcjtcbiAgYm9yZGVyLXN0eWxlOiBzb2xpZDtcbiAgYm9yZGVyLXJhZGl1czogMnB4O1xufVxuXG4udmlzLWl0ZW0udmlzLWNsdXN0ZXItbGluZSB7XG4gIHBhZGRpbmc6IDA7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgd2lkdGg6IDA7XG4gIGJvcmRlci1sZWZ0LXdpZHRoOiAxcHg7XG4gIGJvcmRlci1sZWZ0LXN0eWxlOiBzb2xpZDtcbn1cblxuLnZpcy1pdGVtLnZpcy1jbHVzdGVyLWRvdCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgcGFkZGluZzogMDtcbiAgYm9yZGVyLXdpZHRoOiA0cHg7XG4gIGJvcmRlci1zdHlsZTogc29saWQ7XG4gIGJvcmRlci1yYWRpdXM6IDRweDtcbn0iXX0= */');var Pd=function(){function t(e,i,n){var o,s=this;ba(this,t),this.id=null,this.parent=null,this.data=e,this.dom=null,this.conversion=i||{},this.defaultOptions={locales:dd,locale:"en"},this.options=za.extend({},this.defaultOptions,n),this.options.locales=za.extend({},dd,this.options.locales);var r=this.defaultOptions.locales[this.defaultOptions.locale];mn(o=Yi(this.options.locales)).call(o,(function(t){s.options.locales[t]=za.extend({},r,s.options.locales[t])})),this.selected=!1,this.displayed=!1,this.groupShowing=!0,this.selectable=n&&n.selectable||!1,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.setSelectability(e),this.editable=null,this._updateEditStatus()}return xa(t,[{key:"select",value:function(){this.selectable&&(this.selected=!0,this.dirty=!0,this.displayed&&this.redraw())}},{key:"unselect",value:function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()}},{key:"setData",value:function(t){null!=t.group&&this.data.group!=t.group&&null!=this.parent&&this.parent.itemSet._moveToGroup(this,t.group),this.setSelectability(t),this.parent&&(this.parent.stackDirty=!0),null!=t.subgroup&&this.data.subgroup!=t.subgroup&&null!=this.parent&&this.parent.changeSubgroup(this,this.data.subgroup,t.subgroup),this.data=t,this._updateEditStatus(),this.dirty=!0,this.displayed&&this.redraw()}},{key:"setSelectability",value:function(t){t&&(this.selectable=void 0===t.selectable||Boolean(t.selectable))}},{key:"setParent",value:function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t}},{key:"isVisible",value:function(t){return!1}},{key:"show",value:function(){return!1}},{key:"hide",value:function(){return!1}},{key:"redraw",value:function(){}},{key:"repositionX",value:function(){}},{key:"repositionY",value:function(){}},{key:"_repaintDragCenter",value:function(){if(this.selected&&this.editable.updateTime&&!this.dom.dragCenter){var t,e,i=this,n=document.createElement("div");n.className="vis-drag-center",n.dragCenterItem=this,this.hammerDragCenter=new Vh(n),this.hammerDragCenter.on("tap",(function(t){i.parent.itemSet.body.emitter.emit("click",{event:t,item:i.id})})),this.hammerDragCenter.on("doubletap",(function(t){t.stopPropagation(),i.parent.itemSet._onUpdateItem(i),i.parent.itemSet.body.emitter.emit("doubleClick",{event:t,item:i.id})})),this.hammerDragCenter.on("panstart",(function(t){t.stopPropagation(),i.parent.itemSet._onDragStart(t)})),this.hammerDragCenter.on("panmove",va(t=i.parent.itemSet._onDrag).call(t,i.parent.itemSet)),this.hammerDragCenter.on("panend",va(e=i.parent.itemSet._onDragEnd).call(e,i.parent.itemSet)),this.dom.box?this.dom.dragLeft?this.dom.box.insertBefore(n,this.dom.dragLeft):this.dom.box.appendChild(n):this.dom.point&&this.dom.point.appendChild(n),this.dom.dragCenter=n}else!this.selected&&this.dom.dragCenter&&(this.dom.dragCenter.parentNode&&this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter),this.dom.dragCenter=null,this.hammerDragCenter&&(this.hammerDragCenter.destroy(),this.hammerDragCenter=null))}},{key:"_repaintDeleteButton",value:function(t){var e=(this.options.editable.overrideItems||null==this.editable)&&this.options.editable.remove||!this.options.editable.overrideItems&&null!=this.editable&&this.editable.remove;if(this.selected&&e&&!this.dom.deleteButton){var i=this,n=document.createElement("div");this.options.rtl?n.className="vis-delete-rtl":n.className="vis-delete";var o=this.options.locales[this.options.locale];o||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),o=this.options.locales.en),n.title=o.deleteSelected,this.hammerDeleteButton=new Vh(n).on("tap",(function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)})),t.appendChild(n),this.dom.deleteButton=n}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null,this.hammerDeleteButton&&(this.hammerDeleteButton.destroy(),this.hammerDeleteButton=null))}},{key:"_repaintOnItemUpdateTimeTooltip",value:function(t){if(this.options.tooltipOnItemUpdateTime){var e=(this.options.editable.updateTime||!0===this.data.editable)&&!1!==this.data.editable;if(this.selected&&e&&!this.dom.onItemUpdateTimeTooltip){var i=document.createElement("div");i.className="vis-onUpdateTime-tooltip",t.appendChild(i),this.dom.onItemUpdateTimeTooltip=i}else!this.selected&&this.dom.onItemUpdateTimeTooltip&&(this.dom.onItemUpdateTimeTooltip.parentNode&&this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip),this.dom.onItemUpdateTimeTooltip=null);if(this.dom.onItemUpdateTimeTooltip){this.dom.onItemUpdateTimeTooltip.style.visibility=this.parent.itemSet.touchParams.itemIsDragging?"visible":"hidden",this.dom.onItemUpdateTimeTooltip.style.transform="translateX(-50%)",this.dom.onItemUpdateTimeTooltip.style.left="50%";var n,o,s=this.parent.itemSet.body.domProps.scrollTop;if(("top"==this.options.orientation.item?this.top:this.parent.height-this.top-this.height)+this.parent.top-50<-s?(this.dom.onItemUpdateTimeTooltip.style.bottom="",this.dom.onItemUpdateTimeTooltip.style.top="".concat(this.height+2,"px")):(this.dom.onItemUpdateTimeTooltip.style.top="",this.dom.onItemUpdateTimeTooltip.style.bottom="".concat(this.height+2,"px")),this.options.tooltipOnItemUpdateTime&&this.options.tooltipOnItemUpdateTime.template)n=va(o=this.options.tooltipOnItemUpdateTime.template).call(o,this)(this.data);else n="start: ".concat(Ta(this.data.start).format("MM/DD/YYYY hh:mm")),this.data.end&&(n+="<br> end: ".concat(Ta(this.data.end).format("MM/DD/YYYY hh:mm")));this.dom.onItemUpdateTimeTooltip.innerHTML=n}}}},{key:"_getItemData",value:function(){return this.parent.itemSet.itemsData.get(this.id)}},{key:"_updateContents",value:function(t){var e,i,n,o,s,r,a=this._getItemData(),l=(this.dom.box||this.dom.point).getElementsByClassName("vis-item-visible-frame")[0];this.options.visibleFrameTemplate?n=(o=va(s=this.options.visibleFrameTemplate).call(s,this))(a,l):n="";if(l)if(n instanceof Object&&!(n instanceof Element))o(a,l);else if(this._contentToString(this.itemVisibleFrameContent)!==this._contentToString(n)){if(n instanceof Element)l.innerHTML="",l.appendChild(n);else if(null!=n)l.innerHTML=n;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '.concat(this.id));this.itemVisibleFrameContent=n}this.options.template?e=(i=va(r=this.options.template).call(r,this))(a,t,this.data):e=this.data.content;if(e instanceof Object&&!(e instanceof Element))i(a,t);else if(this._contentToString(this.content)!==this._contentToString(e)){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(null!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '.concat(this.id));this.content=e}}},{key:"_updateDataAttributes",value:function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Zo(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=Yi(this.data)}var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(e);!(i=(s=r.next()).done);i=!0){var a=s.value,l=this.data[a];null!=l?t.setAttribute("data-".concat(a),l):t.removeAttribute("data-".concat(a))}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}}}},{key:"_updateStyle",value:function(t){this.style&&(za.removeCssText(t,this.style),this.style=null),this.data.style&&(za.addCssText(t,this.data.style),this.style=this.data.style)}},{key:"_contentToString",value:function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t}},{key:"_updateEditStatus",value:function(){this.options&&("boolean"==typeof this.options.editable?this.editable={updateTime:this.options.editable,updateGroup:this.options.editable,remove:this.options.editable}:"object"===zo(this.options.editable)&&(this.editable={},za.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.options.editable))),this.options&&this.options.editable&&!0===this.options.editable.overrideItems||this.data&&("boolean"==typeof this.data.editable?this.editable={updateTime:this.data.editable,updateGroup:this.data.editable,remove:this.data.editable}:"object"===zo(this.data.editable)&&(this.editable={},za.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.data.editable)))}},{key:"getWidthLeft",value:function(){return 0}},{key:"getWidthRight",value:function(){return 0}},{key:"getTitle",value:function(){var t;return this.options.tooltip&&this.options.tooltip.template?va(t=this.options.tooltip.template).call(t,this)(this._getItemData(),this.data):this.data.title}}]),t}();Pd.prototype.stack=!0;var Jd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={dot:{width:0,height:0},line:{width:0,height:0}},t&&null==t.start)throw new Error('Property "start" missing in item '.concat(t));return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){if(this.cluster)return!1;var e,i=this.data.align||this.options.align,n=this.width*t.getMillisecondsPerPixel();return e="right"==i?this.data.start.getTime()>t.start&&this.data.start.getTime()-n<t.end:"left"==i?this.data.start.getTime()+n>t.start&&this.data.start.getTime()<t.end:this.data.start.getTime()+n/2>t.start&&this.data.start.getTime()-n/2<t.end,e}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-line",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-dot",this.dom.box["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}if(!this.dom.line.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className="vis-item vis-box".concat(e),this.dom.line.className="vis-item vis-line".concat(e),this.dom.dot.className="vis-item vis-dot".concat(e)}}},{key:"_getDomComponentsSizes",value:function(){return{previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},dot:{height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},line:{width:this.dom.line.offsetWidth},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.props.dot.height=t.dot.height,this.props.dot.width=t.dot.width,this.props.line.width=t.line.width,this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.box)}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){a.dirty&&(s=a._getDomComponentsSizes())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){if(this.displayed){var t=this.dom;t.box.remove?t.box.remove():t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.remove?t.line.remove():t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.remove?t.dot.remove():t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}}},{key:"repositionXY",value:function(){var t=this.options.rtl,e=function(t,e,i){var n,o=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e||void 0!==i){var s=o?-1*e:e;t.style.transform=void 0!==i?void 0!==e?Pn(n="translate(".concat(s,"px, ")).call(n,i,"px)"):"translateY(".concat(i,"px)"):"translateX(".concat(s,"px)")}};e(this.dom.box,this.boxX,this.boxY,t),e(this.dom.dot,this.dotX,this.dotY,t),e(this.dom.line,this.lineX,this.lineY,t)}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start),e=void 0===this.data.align?this.options.align:this.data.align,i=this.props.line.width,n=this.props.dot.width;"right"==e?(this.boxX=t-this.width,this.lineX=t-i,this.dotX=t-i/2-n/2):"left"==e?(this.boxX=t,this.lineX=t,this.dotX=t+i/2-n/2):(this.boxX=t-this.width/2,this.lineX=this.options.rtl?t-i:t-i/2,this.dotX=t-n/2),this.options.rtl?this.right=this.boxX:this.left=this.boxX,this.repositionXY()}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.line.style;if("top"==t){var i=this.parent.top+this.top+1;this.boxY=this.top||0,e.height="".concat(i,"px"),e.bottom="",e.top="0"}else{var n=this.parent.itemSet.props.height-this.parent.top-this.parent.height+this.top;this.boxY=this.parent.height-this.top-(this.height||0),e.height="".concat(n,"px"),e.top="",e.bottom="0"}this.dotY=-this.props.dot.height/2,this.repositionXY()}},{key:"getWidthLeft",value:function(){return this.width/2}},{key:"getWidthRight",value:function(){return this.width/2}}]),e}(Pd),Kd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},t&&null==t.start)throw new Error('Property "start" missing in item '.concat(t));return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){if(this.cluster)return!1;var e=this.width*t.getMillisecondsPerPixel();return this.data.start.getTime()+e>t.start&&this.data.start<t.end}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.point=document.createElement("div"),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.point.appendChild(this.dom.content),this.dom.dot=document.createElement("div"),this.dom.point.appendChild(this.dom.dot),this.dom.point["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.point.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.point)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.point.className="vis-item vis-point".concat(e),this.dom.dot.className="vis-item vis-dot".concat(e)}}},{key:"_getDomComponentsSizes",value:function(){return{dot:{width:this.dom.dot.offsetWidth,height:this.dom.dot.offsetHeight},content:{width:this.dom.content.offsetWidth,height:this.dom.content.offsetHeight},point:{width:this.dom.point.offsetWidth,height:this.dom.point.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.dot.width=t.dot.width,this.props.dot.height=t.dot.height,this.props.content.height=t.content.height,this.options.rtl?this.dom.content.style.marginRight="".concat(2*this.props.dot.width,"px"):this.dom.content.style.marginLeft="".concat(2*this.props.dot.width,"px"),this.width=t.point.width,this.height=t.point.height,this.dom.dot.style.top="".concat((this.height-this.props.dot.height)/2,"px");var e=this.props.dot.width,i=this.options.rtl?e/2*-1:e/2;this.dom.dot.style.transform="translateX(".concat(i,"px"),this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.point),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.point)}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){a.dirty&&(s=a._getDomComponentsSizes())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"repositionXY",value:function(){var t=this.options.rtl,e=function(t,e,i){var n,o=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e||void 0!==i){var s=o?-1*e:e;t.style.transform=void 0!==i?void 0!==e?Pn(n="translate(".concat(s,"px, ")).call(n,i,"px)"):"translateY(".concat(i,"px)"):"translateX(".concat(s,"px)")}};e(this.dom.point,this.pointX,this.pointY,t)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start);this.pointX=t,this.options.rtl?this.right=t-this.props.dot.width:this.left=t-this.props.dot.width,this.repositionXY()}},{key:"repositionY",value:function(){var t=this.options.orientation.item;this.pointY="top"==t?this.top:this.parent.height-this.top-this.height,this.repositionXY()}},{key:"getWidthLeft",value:function(){return this.props.dot.width}},{key:"getWidthRight",value:function(){return this.props.dot.width}}]),e}(Pd),qd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={content:{width:0}},o.overflow=!1,t){if(null==t.start)throw new Error('Property "start" missing in item '.concat(t.id));if(null==t.end)throw new Error('Property "end" missing in item '.concat(t.id))}return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){return!this.cluster&&(this.data.start<t.end&&this.data.end>t.start)}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.visibleFrame=document.createElement("div"),this.dom.visibleFrame.className="vis-item-visible-frame",this.dom.box.appendChild(this.dom.visibleFrame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dom.box["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className=this.baseClassName+e,this.dom.content.style.maxWidth="none"}}},{key:"_getDomComponentsSizes",value:function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.frame).overflow,this.whiteSpace="nowrap"!==window.getComputedStyle(this.dom.content).whiteSpace,{content:{width:this.dom.content.offsetWidth},box:{height:this.dom.box.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.content.width=t.content.width,this.height=t.box.height,this.dom.content.style.maxWidth="",this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDeleteButton(this.dom.box),this._repaintDragCenter(),this._repaintDragLeft(),this._repaintDragRight()}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){var t;a.dirty&&(s=va(t=a._getDomComponentsSizes).call(t,a)())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}}},{key:"repositionX",value:function(t){var e,i,n=this.parent.width,o=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end),r=void 0===this.data.align?this.options.align:this.data.align;!1===this.data.limitSize||void 0!==t&&!0!==t||(o<-n&&(o=-n),s>2*n&&(s=2*n));var a=Math.max(Math.round(1e3*(s-o))/1e3,1);switch(this.overflow?(this.options.rtl?this.right=o:this.left=o,this.width=a+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=o:this.left=o,this.width=a,i=Math.min(s-o,this.props.content.width)),this.options.rtl?this.dom.box.style.transform="translateX(".concat(-1*this.right,"px)"):this.dom.box.style.transform="translateX(".concat(this.left,"px)"),this.dom.box.style.width="".concat(a,"px"),this.whiteSpace&&(this.height=this.dom.box.offsetHeight),r){case"left":this.dom.content.style.transform="translateX(0)";break;case"right":if(this.options.rtl){var l=-1*Math.max(a-i,0);this.dom.content.style.transform="translateX(".concat(l,"px)")}else this.dom.content.style.transform="translateX(".concat(Math.max(a-i,0),"px)");break;case"center":if(this.options.rtl){var h=-1*Math.max((a-i)/2,0);this.dom.content.style.transform="translateX(".concat(h,"px)")}else this.dom.content.style.transform="translateX(".concat(Math.max((a-i)/2,0),"px)");break;default:if(e=this.overflow?s>0?Math.max(-o,0):-i:o<0?-o:0,this.options.rtl){var d=-1*e;this.dom.content.style.transform="translateX(".concat(d,"px)")}else this.dom.content.style.transform="translateX(".concat(e,"px)")}}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.box;e.style.top="".concat("top"==t?this.top:this.parent.height-this.top-this.height,"px")}},{key:"_repaintDragLeft",value:function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragLeft||(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)}},{key:"_repaintDragRight",value:function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragRight||(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)}}]),e}(Pd);qd.prototype.baseClassName="vis-item vis-range";var $d=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={content:{width:0}},o.overflow=!1,t){if(null==t.start)throw new Error('Property "start" missing in item '.concat(t.id));if(null==t.end)throw new Error('Property "end" missing in item '.concat(t.id))}return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){return this.data.start<t.end&&this.data.end>t.start}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.background;if(!t)throw new Error("Cannot redraw item: parent has no background container element");t.appendChild(this.dom.box)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var t=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");this.dom.box.className=this.baseClassName+t}}},{key:"_getDomComponentsSizes",value:function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.content).overflow,{content:{width:this.dom.content.offsetWidth}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.content.width=t.content.width,this.height=0,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){var t;a.dirty&&(s=va(t=a._getDomComponentsSizes).call(t,a)())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"repositionY",value:function(t){var e,i=this.options.orientation.item;if(void 0!==this.data.subgroup){var n=this.data.subgroup;this.dom.box.style.height="".concat(this.parent.subgroups[n].height,"px"),this.dom.box.style.top="".concat("top"==i?this.parent.top+this.parent.subgroups[n].top:this.parent.top+this.parent.height-this.parent.subgroups[n].top-this.parent.subgroups[n].height,"px"),this.dom.box.style.bottom=""}else this.parent instanceof Hd?(e=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.bottom="bottom"==i?"0":"",this.dom.box.style.top="top"==i?"0":""):(e=this.parent.height,this.dom.box.style.top="".concat(this.parent.top,"px"),this.dom.box.style.bottom="");this.dom.box.style.height="".concat(e,"px")}}]),e}(Pd);$d.prototype.baseClassName="vis-item vis-background",$d.prototype.stack=!1,$d.prototype.show=qd.prototype.show,$d.prototype.hide=qd.prototype.hide,$d.prototype.repositionX=qd.prototype.repositionX;Ph("div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRvb2x0aXAuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGdCQUNFLGlCQUFrQixDQUNsQixpQkFBa0IsQ0FDbEIsV0FBWSxDQUNaLGtCQUFtQixDQUVuQixtQkFBb0IsQ0FDcEIsY0FBYyxDQUNkLFVBQWEsQ0FDYix3QkFBeUIsQ0FFekIsc0JBQXVCLENBQ3ZCLHlCQUEwQixDQUMxQixpQkFBa0IsQ0FDbEIsd0JBQXlCLENBRXpCLHNDQUEyQyxDQUMzQyxtQkFBb0IsQ0FFcEIsU0FDRiIsImZpbGUiOiJ0b29sdGlwLmNzcyIsInNvdXJjZXNDb250ZW50IjpbImRpdi52aXMtdG9vbHRpcCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICBwYWRkaW5nOiA1cHg7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG5cbiAgZm9udC1mYW1pbHk6IHZlcmRhbmE7XG4gIGZvbnQtc2l6ZToxNHB4O1xuICBjb2xvcjojMDAwMDAwO1xuICBiYWNrZ3JvdW5kLWNvbG9yOiAjZjVmNGVkO1xuXG4gIC1tb3otYm9yZGVyLXJhZGl1czogM3B4O1xuICAtd2Via2l0LWJvcmRlci1yYWRpdXM6IDNweDtcbiAgYm9yZGVyLXJhZGl1czogM3B4O1xuICBib3JkZXI6IDFweCBzb2xpZCAjODA4MDc0O1xuXG4gIGJveC1zaGFkb3c6IDNweCAzcHggMTBweCByZ2JhKDAsIDAsIDAsIDAuMik7XG4gIHBvaW50ZXItZXZlbnRzOiBub25lO1xuXG4gIHotaW5kZXg6IDU7XG59XG4iXX0= */");var tu=function(){function t(e,i){ba(this,t),this.container=e,this.overflowMethod=i||"cap",this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-tooltip",this.container.appendChild(this.frame)}return xa(t,[{key:"setPosition",value:function(t,e){this.x=Nn(t),this.y=Nn(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),!0===t){var e=this.frame.clientHeight,i=this.frame.clientWidth,n=this.frame.parentNode.clientHeight,o=this.frame.parentNode.clientWidth,s=0,r=0;if("flip"==this.overflowMethod||"none"==this.overflowMethod){var a=!1,l=!0;"flip"==this.overflowMethod&&(this.y-e<this.padding&&(l=!1),this.x+i>o-this.padding&&(a=!0)),s=a?this.x-i:this.x,r=l?this.y-e:this.y}else(r=this.y-e)+e+this.padding>n&&(r=n-e-this.padding),r<this.padding&&(r=this.padding),(s=this.x)+i+this.padding>o&&(s=o-i-this.padding),s<this.padding&&(s=this.padding);this.frame.style.left=s+"px",this.frame.style.top=r+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.left="0",this.frame.style.top="0",this.frame.style.visibility="hidden"}},{key:"destroy",value:function(){this.frame.parentNode.removeChild(this.frame)}}]),t}(),eu=Ce.every,iu=$i("every"),nu=on("every");V({target:"Array",proto:!0,forced:!iu||!nu},{every:function(t){return eu(this,t,arguments.length>1?arguments[1]:void 0)}});var ou=Pi("Array").every,su=Array.prototype,ru=function(t){var e=t.every;return t===su||t instanceof Array&&e===su.every?ou:e},au=function(t){function e(t,i,n){var o;ba(this,e);var s=Ao({},{fitOnDoubleClick:!0},n,{editable:!1});if((o=ar(this,dr(e).call(this,t,i,s))).props={content:{width:0,height:0}},!t||null==t.uiItems)throw new Error('Property "uiItems" missing in item '+t.id);return o.id=Id(),o.group=t.group,o._setupRange(),o.emitter=o.data.eventEmitter,o.range=o.data.range,o.attached=!1,o.isCluster=!0,o.data.isCluster=!0,o}return pr(e,t),xa(e,[{key:"hasItems",value:function(){return this.data.uiItems&&this.data.uiItems.length&&this.attached}},{key:"setUiItems",value:function(t){this.detach(),this.data.uiItems=t,this._setupRange(),this.attach()}},{key:"isVisible",value:function(t){var e=this.data.end?this.data.end-this.data.start:0,i=this.width*t.getMillisecondsPerPixel(),n=Math.max(e,this.data.start.getTime()+i);return this.data.start<t.end&&n>t.start&&this.hasItems()}},{key:"getData",value:function(){return{isCluster:!0,id:this.id,items:this.data.items||[],data:this.data}}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a,l,h=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),va(o=function(){this.dirty&&(a=this._getDomComponentsSizes())}).call(o,this),va(s=function(){var t;this.dirty&&va(t=this._updateDomComponentsSizes).call(t,this)(a)}).call(s,this),va(r=this._repaintDomAdditionals).call(r,this)];return t?h:(mn(h).call(h,(function(t){l=t()})),l)}},{key:"show",value:function(){this.displayed||this.redraw()}},{key:"hide",value:function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),this.options.showStipes&&(t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot)),this.displayed=!1}}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start),e=this.data.end?this.conversion.toScreen(this.data.end):0;if(e)this.repositionXWithRanges(t,e);else{var i=void 0===this.data.align?this.options.align:this.data.align;this.repositionXWithoutRanges(t,i)}this.options.showStipes&&(this.dom.line.style.display=this._isStipeVisible()?"block":"none",this.dom.dot.style.display=this._isStipeVisible()?"block":"none",this._isStipeVisible()&&this.repositionStype(t,e))}},{key:"repositionStype",value:function(t,e){this.dom.line.style.display="block",this.dom.dot.style.display="block";var i=this.dom.line.offsetWidth,n=this.dom.dot.offsetWidth;if(e){var o=i+t+(e-t)/2,s=o-n/2,r=this.options.rtl?-1*o:o,a=this.options.rtl?-1*s:s;this.dom.line.style.transform="translateX(".concat(r,"px)"),this.dom.dot.style.transform="translateX(".concat(a,"px)")}else{var l=this.options.rtl?-1*t:t,h=this.options.rtl?-1*(t-n/2):t-n/2;this.dom.line.style.transform="translateX(".concat(l,"px)"),this.dom.dot.style.transform="translateX(".concat(h,"px)")}}},{key:"repositionXWithoutRanges",value:function(t,e){"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px"):(this.left=t,this.dom.box.style.left=this.left+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px")}},{key:"repositionXWithRanges",value:function(t,e){var i=Math.round(Math.max(e-t+.5,1));this.options.rtl?this.right=t:this.left=t,this.width=Math.max(i,this.minWidth||0),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=i+"px"}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.box;if(e.style.top="top"==t?(this.top||0)+"px":(this.parent.height-this.top-this.height||0)+"px",this.options.showStipes){if("top"==t)this.dom.line.style.top="0",this.dom.line.style.height=this.parent.top+this.top+1+"px",this.dom.line.style.bottom="";else{var i=this.parent.itemSet.props.height,n=i-this.parent.top-this.parent.height+this.top;this.dom.line.style.top=i-n+"px",this.dom.line.style.bottom="0"}this.dom.dot.style.top=-this.dom.dot.offsetHeight/2+"px"}}},{key:"getWidthLeft",value:function(){return this.width/2}},{key:"getWidthRight",value:function(){return this.width/2}},{key:"move",value:function(){this.repositionX(),this.repositionY()}},{key:"attach",value:function(){var t,e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this.data.uiItems);!(e=(o=s.next()).done);e=!0){o.value.cluster=this}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}this.data.items=Ln(t=this.data.uiItems).call(t,(function(t){return t.data})),this.attached=!0,this.dirty=!0}},{key:"detach",value:function(){var t=arguments.length>0&&void 0!==arguments[0]&&arguments[0];if(this.hasItems()){var e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this.data.uiItems);!(e=(o=s.next()).done);e=!0){var r=o.value;delete r.cluster}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}this.attached=!1,t&&this.group&&(this.group.remove(this),this.group=null),this.data.items=[],this.dirty=!0}}},{key:"_onDoubleClick",value:function(){this._fit()}},{key:"_setupRange",value:function(){var t,e,i,n=Ln(t=this.data.uiItems).call(t,(function(t){return{start:t.data.start.valueOf(),end:t.data.end?t.data.end.valueOf():t.data.start.valueOf()}}));this.data.min=Math.min.apply(Math,mo(Ln(n).call(n,(function(t){return Math.min(t.start,t.end||t.start)})))),this.data.max=Math.max.apply(Math,mo(Ln(n).call(n,(function(t){return Math.max(t.start,t.end||t.start)}))));var o=Ln(e=this.data.uiItems).call(e,(function(t){return t.center})),s=da(o).call(o,(function(t,e){return t+e}),0)/this.data.uiItems.length;rr(i=this.data.uiItems).call(i,(function(t){return t.data.end}))?(this.data.start=new Date(this.data.min),this.data.end=new Date(this.data.max)):(this.data.start=new Date(s),this.data.end=null)}},{key:"_getUiItems",value:function(){var t,e=this;return this.data.uiItems&&this.data.uiItems.length?En(t=this.data.uiItems).call(t,(function(t){return t.cluster===e})):[]}},{key:"_createDomElement",value:function(){if(!this.dom){var t;if(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.options.showStipes&&(this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-cluster-line",this.dom.line.style.display="none",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-cluster-dot",this.dom.dot.style.display="none"),this.options.fitOnDoubleClick)this.dom.box.ondblclick=va(t=e.prototype._onDoubleClick).call(t,this);this.dom.box["vis-item"]=this,this.dirty=!0}}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}var e=this.parent.dom.background;if(this.options.showStipes){if(!this.dom.line.parentNode){if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.baseClassName+" "+(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+" vis-readonly";this.dom.box.className="vis-item "+t,this.options.showStipes&&(this.dom.line.className="vis-item vis-cluster-line "+(this.selected?" vis-selected":""),this.dom.dot.className="vis-item vis-cluster-dot "+(this.selected?" vis-selected":"")),this.data.end&&(this.dom.content.style.maxWidth="none")}}},{key:"_getDomComponentsSizes",value:function(){var t={previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}};return this.options.showStipes&&(t.dot={height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},t.line={width:this.dom.line.offsetWidth}),t}},{key:"_updateDomComponentsSizes",value:function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.data.end?this.minWidth=t.box.width:this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box)}},{key:"_isStipeVisible",value:function(){return this.minWidth>=this.width||!this.data.end}},{key:"_getFitRange",value:function(){var t=.05*(this.data.max-this.data.min)/2;return{fitStart:this.data.min-t,fitEnd:this.data.max+t}}},{key:"_fit",value:function(){if(this.emitter){var t=this._getFitRange(),e=t.fitStart,i=t.fitEnd,n={start:new Date(e),end:new Date(i),animation:!0};this.emitter.emit("fit",n)}}},{key:"_getItemData",value:function(){return this.data}}]),e}(Pd);au.prototype.baseClassName="vis-item vis-range vis-cluster";var lu="__ungrouped__",hu=function(){function t(e){ba(this,t),this.itemSet=e,this.groups={},this.cache={},this.cache[-1]=[]}return xa(t,[{key:"createClusterItem",value:function(t,e,i){return new au(t,e,i)}},{key:"setItems",value:function(t,e){this.items=t||[],this.dataChanged=!0,this.applyOnChangedLevel=!1,e&&e.applyOnChangedLevel&&(this.applyOnChangedLevel=e.applyOnChangedLevel)}},{key:"updateData",value:function(){this.dataChanged=!0,this.applyOnChangedLevel=!1}},{key:"getClusters",value:function(t,e,i){var n="boolean"==typeof i?{}:i,o=n.maxItems,s=n.clusterCriteria;s||(s=function(){return!0}),o=o||1;var r=-1,a=0;if(e>0){if(e>=1)return[];r=Math.abs(Math.round(Math.log(100/e)/Math.log(2))),a=Math.abs(Math.pow(2,r))}if(this.dataChanged){var l=r!=this.cacheLevel;(!this.applyOnChangedLevel||l)&&(this._dropLevelsCache(),this._filterData())}this.cacheLevel=r;var h=this.cache[r];if(!h){for(var d in h=[],this.groups)if(this.groups.hasOwnProperty(d))for(var u=this.groups[d],c=u.length,p=0;p<c;){for(var m=u[p],f=1,g=p-1;g>=0&&m.center-u[g].center<a/2;)!u[g].cluster&&s(m.data,u[g].data)&&f++,g--;for(var v=p+1;v<u.length&&u[v].center-m.center<a/2;)s(m.data,u[v].data)&&f++,v++;for(var b=h.length-1;b>=0&&m.center-h[b].center<a;)m.group==h[b].group&&s(m.data,h[b].data)&&f++,b--;if(f>o){for(var y=f-o+1,x=[],w=p;x.length<y&&w<u.length;)s(u[w].data,u[w].data)&&x.push(u[w]),w++;var _=this.itemSet.getGroupId(m.data),k=this.itemSet.groups[_]||this.itemSet.groups[lu],D=this._getClusterForItems(x,k,t,i);h.push(D),p+=y}else delete m.cluster,p+=1}this.cache[r]=h}return h}},{key:"_filterData",value:function(){var t={};this.groups=t;for(var e=0,i=On(this.items);e<i.length;e++){var n=i[e],o=n.parent?n.parent.groupId:"",s=t[o];s||(s=[],t[o]=s),s.push(n),n.data.start&&(n.data.end?n.center=(n.data.start.valueOf()+n.data.end.valueOf())/2:n.center=n.data.start.valueOf())}for(var r in t){var a;if(t.hasOwnProperty(r))wr(a=t[r]).call(a,(function(t,e){return t.center-e.center}))}this.dataChanged=!1}},{key:"_getClusterForItems",value:function(t,e,i,n){var o,s,r=Ln(o=i||[]).call(o,(function(t){var e;return{cluster:t,itemsIds:new Ar(Ln(e=t.data.uiItems).call(e,(function(t){return t.id})))}}));if(r.length){var a=!0,l=!1,h=void 0;try{for(var d,u=function(){var e=d.value;if(e.itemsIds.size===t.length&&ru(t).call(t,(function(t){return e.itemsIds.has(t.id)})))return s=e.cluster,"break"},c=Fi(r);!(a=(d=c.next()).done);a=!0){if("break"===u())break}}catch(t){l=!0,h=t}finally{try{a||null==c.return||c.return()}finally{if(l)throw h}}}if(s)return s.setUiItems(t),s.group!==e&&(s.group&&s.group.remove(s),e&&(e.add(s),s.group=e)),s;var p=n.titleTemplate||"",m={toScreen:this.itemSet.body.util.toScreen,toTime:this.itemSet.body.util.toTime},f=p.replace(/{count}/,t.length),g='<div title="'+f+'">'+t.length+"</div>",v=Ao({},n,this.itemSet.options),b={content:g,title:f,group:e,uiItems:t,eventEmitter:this.itemSet.body.emitter,range:this.itemSet.body.range};return s=this.createClusterItem(b,m,v),e&&(e.add(s),s.group=e),s.attach(),s}},{key:"_dropLevelsCache",value:function(){this.cache={},this.cacheLevel=-1,this.cache[this.cacheLevel]=[]}}]),t}();Ph('.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1{background:#f5f5f5}.vis-label.vis-nested-group.vis-group-level-0{background-color:#fff}.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-left:0}.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-right:0}.vis-label.vis-nested-group.vis-group-level-1{background-color:rgba(0,0,0,.05)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-left:15px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-right:15px}.vis-label.vis-nested-group.vis-group-level-2{background-color:rgba(0,0,0,.1)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-left:30px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-right:30px}.vis-label.vis-nested-group.vis-group-level-3{background-color:rgba(0,0,0,.15)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-left:45px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-right:45px}.vis-label.vis-nested-group.vis-group-level-4{background-color:rgba(0,0,0,.2)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-left:60px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-right:60px}.vis-label.vis-nested-group.vis-group-level-5{background-color:rgba(0,0,0,.25)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-left:75px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-right:75px}.vis-label.vis-nested-group.vis-group-level-6{background-color:rgba(0,0,0,.3)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-left:90px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-right:90px}.vis-label.vis-nested-group.vis-group-level-7{background-color:rgba(0,0,0,.35)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-left:105px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-right:105px}.vis-label.vis-nested-group.vis-group-level-8{background-color:rgba(0,0,0,.4)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-left:120px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-right:120px}.vis-label.vis-nested-group.vis-group-level-9{background-color:rgba(0,0,0,.45)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-left:135px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-right:135px}.vis-label.vis-nested-group{background-color:rgba(0,0,0,.5)}.vis-ltr .vis-label.vis-nested-group .vis-inner{padding-left:150px}.vis-rtl .vis-label.vis-nested-group .vis-inner{padding-right:150px}.vis-group-level-unknown-but-gte1{border:1px solid red}.vis-label.vis-nesting-group:before{display:inline-block;width:15px}.vis-label.vis-nesting-group.expanded:before{content:"\\25BC"}.vis-label.vis-nesting-group.collapsed:before{content:"\\25B6"}.vis-rtl .vis-label.vis-nesting-group.collapsed:before{content:"\\25C0"}.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-left:15px}.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-right:15px}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIml0ZW1zZXQuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLGFBQ0UsaUJBQWtCLENBQ2xCLFNBQVUsQ0FDVixRQUFTLENBRVQscUJBQ0YsQ0FFQSwwREFFRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLFdBQVksQ0FDWixnQkFDRixDQUVBLFVBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxRQUFTLENBQ1QsTUFBTyxDQUNQLFNBQ0YsQ0FFQSwyQkFDRSxpQkFBa0IsQ0FDbEIscUJBQXNCLENBQ3RCLCtCQUNGLENBRUEsc0NBQ0Usa0JBQ0YsQ0FFQSxtQkFDRSxjQUNGLENBRUEsNkRBQ0Usa0JBQ0YsQ0FDQSw4Q0FDRSxxQkFDRixDQUNBLGtFQUNFLGNBQ0YsQ0FDQSxrRUFDRSxlQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBR0EsNEJBQ0UsK0JBQ0YsQ0FDQSxnREFDRSxrQkFDRixDQUNBLGdEQUNFLG1CQUNGLENBRUEsa0NBQ0Usb0JBQ0YsQ0FHQSxvQ0FFRSxvQkFBcUIsQ0FDckIsVUFDRixDQUNBLDZDQUNFLGVBQ0YsQ0FDQSw4Q0FDRSxlQUNGLENBQ0EsdURBQ0UsZUFDRixDQUVBLG9FQUNFLGlCQUNGLENBQ0Esb0VBQ0Usa0JBQ0YsQ0FFQSxhQUNFLGlCQUFrQixDQUNsQixLQUFNLENBQ04sTUFBTyxDQUNQLFVBQVcsQ0FDWCxXQUFZLENBQ1osVUFDRiIsImZpbGUiOiJpdGVtc2V0LmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy1pdGVtc2V0IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG5cbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcbn1cblxuLnZpcy1pdGVtc2V0IC52aXMtYmFja2dyb3VuZCxcbi52aXMtaXRlbXNldCAudmlzLWZvcmVncm91bmQge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIG92ZXJmbG93OiB2aXNpYmxlO1xufVxuXG4udmlzLWF4aXMge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDA7XG4gIGxlZnQ6IDA7XG4gIHotaW5kZXg6IDE7XG59XG5cbi52aXMtZm9yZWdyb3VuZCAudmlzLWdyb3VwIHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xuICBib3JkZXItYm90dG9tOiAxcHggc29saWQgI2JmYmZiZjtcbn1cblxuLnZpcy1mb3JlZ3JvdW5kIC52aXMtZ3JvdXA6bGFzdC1jaGlsZCB7XG4gIGJvcmRlci1ib3R0b206IG5vbmU7XG59XG5cbi52aXMtbmVzdGluZy1ncm91cCB7XG4gIGN1cnNvcjogcG9pbnRlcjtcbn1cblxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC11bmtub3duLWJ1dC1ndGUxIHtcbiAgYmFja2dyb3VuZDogI2Y1ZjVmNTtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMCB7XG4gIGJhY2tncm91bmQtY29sb3I6ICNmZmZmZmY7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMCAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogMDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMSB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4wNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTEgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTVweDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMSAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogMTVweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMiB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4xKTtcbn1cbi52aXMtbHRyIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMiAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1sZWZ0OiAzMHB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0yIC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiAzMHB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0zIHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjE1KTtcbn1cbi52aXMtbHRyIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMyAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1sZWZ0OiA0NXB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0zIC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiA0NXB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC00IHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjIpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC00IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDYwcHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTQgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDYwcHg7XG59XG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTUge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuMjUpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC01IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDc1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTUgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDc1cHg7XG59XG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTYge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuMyk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTYgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogOTBweDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtNiAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogOTBweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtNyB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4zNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTcgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTA1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTcgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDEwNXB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjQpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDEyMHB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiAxMjBweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtOSB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC40NSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTkgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTM1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTkgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDEzNXB4O1xufVxuLyogZGVmYXVsdCB0YWtlcyBvdmVyIGJlZ2lubmluZyB3aXRoIGxldmVsLTEwICh0aGF0cyB3aHkgd2UgYWRkIC52aXMtbmVzdGVkLWdyb3VwXG4gIHRvIHRoZSBzZWxlY3RvcnMgYWJvdmUsIHRvIGhhdmUgaGlnaGVyIHNwZWNpZml0eSB0aGFuIHRoZXNlIHJ1bGVzIGZvciB0aGUgZGVmYXVsdHMpICovXG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTUwcHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDE1MHB4O1xufVxuXG4udmlzLWdyb3VwLWxldmVsLXVua25vd24tYnV0LWd0ZTEge1xuICBib3JkZXI6IDFweCBzb2xpZCByZWQ7XG59XG5cbi8qIGV4cGFuZGVkL2NvbGxhcHNlZCBpbmRpY2F0b3JzICovXG4udmlzLWxhYmVsLnZpcy1uZXN0aW5nLWdyb3VwOmJlZm9yZSxcbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXA6YmVmb3JlIHtcbiAgZGlzcGxheTogaW5saW5lLWJsb2NrO1xuICB3aWR0aDogMTVweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXAuZXhwYW5kZWQ6YmVmb3JlIHtcbiAgY29udGVudDogXCJcXDI1QkNcIjtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXAuY29sbGFwc2VkOmJlZm9yZSB7XG4gIGNvbnRlbnQ6IFwiXFwyNUI2XCI7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0aW5nLWdyb3VwLmNvbGxhcHNlZDpiZWZvcmUge1xuICBjb250ZW50OiBcIlxcMjVDMFwiO1xufVxuLyogY29tcGVuc2F0ZSBtaXNzaW5nIGV4cGFuZGVkL2NvbGxhcHNlZCBpbmRpY2F0b3IsIGJ1dCBvbmx5IGF0IGxldmVscyA+IDAgKi9cbi52aXMtbHRyIC52aXMtbGFiZWw6bm90KC52aXMtbmVzdGluZy1ncm91cCk6bm90KC52aXMtZ3JvdXAtbGV2ZWwtMCkge1xuICBwYWRkaW5nLWxlZnQ6IDE1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsOm5vdCgudmlzLW5lc3RpbmctZ3JvdXApOm5vdCgudmlzLWdyb3VwLWxldmVsLTApIHtcbiAgcGFkZGluZy1yaWdodDogMTVweDtcbn1cblxuLnZpcy1vdmVybGF5IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHotaW5kZXg6IDEwO1xufSJdfQ== */');Ph(".vis-labelset{overflow:hidden}.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-group-is-dragging{background:rgba(0,0,0,.1)}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImxhYmVsc2V0LmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxjQUdFLGVBR0YsQ0FFQSx1Q0FQRSxpQkFBa0IsQ0FJbEIscUJBV0YsQ0FSQSx5QkFFRSxNQUFPLENBQ1AsS0FBTSxDQUNOLFVBQVcsQ0FDWCxhQUFjLENBTWQsK0JBSEYsQ0FNQSxtQ0FDRSxjQUNGLENBRUEsdUJBQ0UseUJBQ0YsQ0FFQSxvQ0FDRSxrQkFDRixDQUVBLG9DQUNFLG9CQUFxQixDQUNyQixXQUNGLENBRUEsK0NBQ0UsU0FDRiIsImZpbGUiOiJsYWJlbHNldC5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyJcbi52aXMtbGFiZWxzZXQge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG5cbiAgb3ZlcmZsb3c6IGhpZGRlbjtcblxuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xufVxuXG4udmlzLWxhYmVsc2V0IC52aXMtbGFiZWwge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG4gIGxlZnQ6IDA7XG4gIHRvcDogMDtcbiAgd2lkdGg6IDEwMCU7XG4gIGNvbG9yOiAjNGQ0ZDRkO1xuXG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtbGFiZWxzZXQgLnZpcy1sYWJlbCB7XG4gIGJvcmRlci1ib3R0b206IDFweCBzb2xpZCAjYmZiZmJmO1xufVxuXG4udmlzLWxhYmVsc2V0IC52aXMtbGFiZWwuZHJhZ2dhYmxlIHtcbiAgY3Vyc29yOiBwb2ludGVyO1xufVxuXG4udmlzLWdyb3VwLWlzLWRyYWdnaW5nIHtcbiAgYmFja2dyb3VuZDogcmdiYSgwLCAwLCAwLCAuMSk7XG59XG5cbi52aXMtbGFiZWxzZXQgLnZpcy1sYWJlbDpsYXN0LWNoaWxkIHtcbiAgYm9yZGVyLWJvdHRvbTogbm9uZTtcbn1cblxuLnZpcy1sYWJlbHNldCAudmlzLWxhYmVsIC52aXMtaW5uZXIge1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG4gIHBhZGRpbmc6IDVweDtcbn1cblxuLnZpcy1sYWJlbHNldCAudmlzLWxhYmVsIC52aXMtaW5uZXIudmlzLWhpZGRlbiB7XG4gIHBhZGRpbmc6IDA7XG59XG4iXX0= */");var du="__background__",uu=function(t){function e(t,i){var n;ba(this,e),(n=ar(this,dr(e).call(this))).body=t,n.defaultOptions={type:null,orientation:{item:"bottom"},align:"auto",stack:!0,stackSubgroups:!0,groupOrderSwap:function(t,e,i){var n=e.order;e.order=t.order,t.order=n},groupOrder:"order",selectable:!0,multiselect:!1,longSelectPressTime:251,itemsAlwaysDraggable:{item:!1,range:!1},editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1,overrideItems:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:Hh.snap,onDropObjectOnItem:function(t,e,i){i(e)},onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20},showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip",delay:500},tooltipOnItemUpdateTime:!1},n.options=za.extend({},n.defaultOptions),n.options.rtl=i.rtl,n.options.onTimeout=i.onTimeout,n.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},n.dom={},n.props={},n.hammer=null;var o=Bs(n);return n.itemsData=null,n.groupsData=null,n.itemsSettingTime=null,n.initialItemSetDrawn=!1,n.userContinueNotBail=null,n.sequentialSelection=!1,n.itemListeners={add:function(t,e,i){o._onAdd(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()},update:function(t,e,i){o._onUpdate(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()},remove:function(t,e,i){o._onRemove(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()}},n.groupListeners={add:function(t,e,i){if(o._onAddGroups(e.items),o.groupsData&&o.groupsData.length>0){var n,s=o.groupsData.getDataSet();mn(n=s.get()).call(n,(function(t){if(t.nestedGroups){var e;0!=t.showNested&&(t.showNested=!0);var n=[];mn(e=t.nestedGroups).call(e,(function(e){var i=s.get(e);i&&(i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),n=Pn(n).call(n,i))})),s.update(n,i)}}))}},update:function(t,e,i){o._onUpdateGroups(e.items)},remove:function(t,e,i){o._onRemoveGroups(e.items)}},n.items={},n.groups={},n.groupIds=[],n.selection=[],n.popup=null,n.popupTimer=null,n.touchParams={},n.groupTouchParams={group:null,isDragging:!1},n._create(),n.setOptions(i),n.clusters=[],n}return pr(e,t),xa(e,[{key:"_create",value:function(){var t,e,i,n,o,s,r,a,l,h,d,u,c,p,m,f=this,g=document.createElement("div");g.className="vis-itemset",g["vis-itemset"]=this,this.dom.frame=g;var v=document.createElement("div");v.className="vis-background",g.appendChild(v),this.dom.background=v;var b=document.createElement("div");b.className="vis-foreground",g.appendChild(b),this.dom.foreground=b;var y=document.createElement("div");y.className="vis-axis",this.dom.axis=y;var x=document.createElement("div");x.className="vis-labelset",this.dom.labelSet=x,this._updateUngrouped();var w=new Hd(du,null,this);w.show(),this.groups[du]=w,this.hammer=new Vh(this.body.dom.centerContainer),this.hammer.on("hammer.input",(function(t){t.isFirst&&f._onTouch(t)})),this.hammer.on("panstart",va(t=this._onDragStart).call(t,this)),this.hammer.on("panmove",va(e=this._onDrag).call(e,this)),this.hammer.on("panend",va(i=this._onDragEnd).call(i,this)),this.hammer.get("pan").set({threshold:5,direction:Vh.ALL}),this.hammer.on("tap",va(n=this._onSelectItem).call(n,this)),this.hammer.on("press",va(o=this._onMultiSelectItem).call(o,this)),this.hammer.on("doubletap",va(s=this._onAddItem).call(s,this)),this.options.rtl?this.groupHammer=new Vh(this.body.dom.rightContainer):this.groupHammer=new Vh(this.body.dom.leftContainer),this.groupHammer.on("tap",va(r=this._onGroupClick).call(r,this)),this.groupHammer.on("panstart",va(a=this._onGroupDragStart).call(a,this)),this.groupHammer.on("panmove",va(l=this._onGroupDrag).call(l,this)),this.groupHammer.on("panend",va(h=this._onGroupDragEnd).call(h,this)),this.groupHammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_VERTICAL}),this.body.dom.centerContainer.addEventListener("mouseover",va(d=this._onMouseOver).call(d,this)),this.body.dom.centerContainer.addEventListener("mouseout",va(u=this._onMouseOut).call(u,this)),this.body.dom.centerContainer.addEventListener("mousemove",va(c=this._onMouseMove).call(c,this)),this.body.dom.centerContainer.addEventListener("contextmenu",va(p=this._onDragEnd).call(p,this)),this.body.dom.centerContainer.addEventListener("mousewheel",va(m=this._onMouseWheel).call(m,this)),this.show()}},{key:"setOptions",value:function(t){var e=this;if(t){var i,n;za.selectiveExtend(["type","rtl","align","order","stack","stackSubgroups","selectable","multiselect","sequentialSelection","multiselectPerGroup","longSelectPressTime","groupOrder","dataAttributes","template","groupTemplate","visibleFrameTemplate","hide","snap","groupOrderSwap","showTooltips","tooltip","tooltipOnItemUpdateTime","groupHeightMode","onTimeout"],this.options,t),"itemsAlwaysDraggable"in t&&("boolean"==typeof t.itemsAlwaysDraggable?(this.options.itemsAlwaysDraggable.item=t.itemsAlwaysDraggable,this.options.itemsAlwaysDraggable.range=!1):"object"===zo(t.itemsAlwaysDraggable)&&(za.selectiveExtend(["item","range"],this.options.itemsAlwaysDraggable,t.itemsAlwaysDraggable),this.options.itemsAlwaysDraggable.item||(this.options.itemsAlwaysDraggable.range=!1))),"sequentialSelection"in t&&"boolean"==typeof t.sequentialSelection&&(this.options.sequentialSelection=t.sequentialSelection),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===zo(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===zo(t.margin)&&(za.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===zo(t.margin.item)&&za.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),mn(i=["locale","locales"]).call(i,(function(i){i in t&&(e.options[i]=t[i])})),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable,this.options.editable.overrideItems=!1):"object"===zo(t.editable)&&za.selectiveExtend(["updateTime","updateGroup","add","remove","overrideItems"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===zo(t.groupEditable)&&za.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));mn(n=["onDropObjectOnItem","onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"]).call(n,(function(i){var n=t[i];if(n){var o;if("function"!=typeof n)throw new Error(Pn(o="option ".concat(i," must be a function ")).call(o,i,"(item, callback)"));e.options[i]=n}})),t.cluster?(Ao(this.options,{cluster:t.cluster}),this.clusterGenerator||(this.clusterGenerator=new hu(this)),this.clusterGenerator.setItems(this.items,{applyOnChangedLevel:!1}),this.markDirty({refreshItems:!0,restackGroups:!0}),this.redraw()):this.clusterGenerator?(this._detachAllClusters(),this.clusters=[],this.clusterGenerator=null,this.options.cluster=void 0,this.markDirty({refreshItems:!0,restackGroups:!0}),this.redraw()):this.markDirty()}}},{key:"markDirty",value:function(t){this.groupIds=[],t&&(t.refreshItems&&mn(za).call(za,this.items,(function(t){t.dirty=!0,t.displayed&&t.redraw()})),t.restackGroups&&mn(za).call(za,this.groups,(function(t,e){e!==du&&(t.stackDirty=!0)})))}},{key:"destroy",value:function(){this.clearPopupTimer(),this.hide(),this.setItems(null),this.setGroups(null),this.hammer&&this.hammer.destroy(),this.groupHammer&&this.groupHammer.destroy(),this.hammer=null,this.body=null,this.conversion=null}},{key:"hide",value:function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)}},{key:"show",value:function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||(this.options.rtl?this.body.dom.right.appendChild(this.dom.labelSet):this.body.dom.left.appendChild(this.dom.labelSet))}},{key:"setPopupTimer",value:function(t){if(this.clearPopupTimer(),t){var e=this.options.tooltip.delay||"number"==typeof this.options.tooltip.delay?this.options.tooltip.delay:500;this.popupTimer=qr((function(){t.show()}),e)}}},{key:"clearPopupTimer",value:function(){null!=this.popupTimer&&(clearTimeout(this.popupTimer),this.popupTimer=null)}},{key:"setSelection",value:function(t){var e;null==t&&(t=[]),Zo(t)||(t=[t]);var i=En(e=this.selection).call(e,(function(e){return-1===Fo(t).call(t,e)})),n=!0,o=!1,s=void 0;try{for(var r,a=Fi(i);!(n=(r=a.next()).done);n=!0){var l=r.value,h=this.getItemById(l);h&&h.unselect()}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}this.selection=mo(t);var d=!0,u=!1,c=void 0;try{for(var p,m=Fi(t);!(d=(p=m.next()).done);d=!0){var f=p.value,g=this.getItemById(f);g&&g.select()}}catch(t){u=!0,c=t}finally{try{d||null==m.return||m.return()}finally{if(u)throw c}}}},{key:"getSelection",value:function(){var t;return Pn(t=this.selection).call(t,[])}},{key:"getVisibleItems",value:function(){var t,e,i=this.body.range.getRange();this.options.rtl?(t=this.body.util.toScreen(i.start),e=this.body.util.toScreen(i.end)):(e=this.body.util.toScreen(i.start),t=this.body.util.toScreen(i.end));var n=[];for(var o in this.groups)if(this.groups.hasOwnProperty(o)){var s=this.groups[o],r=s.isVisible?s.visibleItems:[],a=!0,l=!1,h=void 0;try{for(var d,u=Fi(r);!(a=(d=u.next()).done);a=!0){var c=d.value;this.options.rtl?c.right<e&&c.right+c.width>t&&n.push(c.id):c.left<t&&c.left+c.width>e&&n.push(c.id)}}catch(t){l=!0,h=t}finally{try{a||null==u.return||u.return()}finally{if(l)throw h}}}return n}},{key:"getVisibleGroups",value:function(){var t=[];for(var e in this.groups){if(this.groups.hasOwnProperty(e))this.groups[e].isVisible&&t.push(e)}return t}},{key:"getItemById",value:function(t){var e;return this.items[t]||yd(e=this.clusters).call(e,(function(e){return e.id===t}))}},{key:"_deselect",value:function(t){for(var e=this.selection,i=0,n=e.length;i<n;i++)if(e[i]==t){Hr(e).call(e,i,1);break}}},{key:"redraw",value:function(){var t=this,e=this.options.margin,i=this.body.range,n=za.option.asSize,o=this.options,s=o.orientation.item,r=!1,a=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,a.className="vis-itemset",this.options.cluster&&this._clusterItems(),r=this._orderGroups()||r;var l=i.end-i.start,h=l!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth,d=i.start!=this.lastRangeStart,u=o.stack!=this.lastStack,c=o.stackSubgroups!=this.lastStackSubgroups,p=h||d||u||c;this.lastVisibleInterval=l,this.lastRangeStart=i.start,this.lastStack=o.stack,this.lastStackSubgroups=o.stackSubgroups,this.props.lastWidth=this.props.width;var m=this._firstGroup(),f={item:e.item,axis:e.axis},g={item:e.item,axis:e.item.vertical/2},v=0,b=e.axis+e.item.vertical;this.groups[du].redraw(i,g,p);var y={},x=0;return mn(za).call(za,this.groups,(function(t,e){if(e!==du){var n=t==m?f:g;y[e]=t.redraw(i,n,p,!0),x=y[e].length}})),x>0&&function(){for(var e={},i=function(t){mn(za).call(za,y,(function(i,n){e[n]=i[t]()}))},n=0;n<x;n++)i(n);mn(za).call(za,t.groups,(function(t,i){if(i!==du){var n=e[i];r=n||r,v+=t.height}})),v=Math.max(v,b)}(),v=Math.max(v,b),a.style.height=n(v),this.props.width=a.offsetWidth,this.props.height=v,this.dom.axis.style.top=n("top"==s?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",this.hammer.get("press").set({time:this.options.longSelectPressTime}),this.initialItemSetDrawn=!0,r=this._isResized()||r}},{key:"_firstGroup",value:function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t];return this.groups[e]||this.groups.__ungrouped__||null}},{key:"_updateUngrouped",value:function(){var t,e,i=this.groups.__ungrouped__;if(this.groupsData){if(i)for(e in i.dispose(),delete this.groups.__ungrouped__,this.items)if(this.items.hasOwnProperty(e)){(t=this.items[e]).parent&&t.parent.remove(t);var n=this.getGroupId(t.data),o=this.groups[n];o&&o.add(t)||t.hide()}}else if(!i){for(e in i=new jd(null,null,this),this.groups.__ungrouped__=i,this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}}},{key:"getLabelSet",value:function(){return this.dom.labelSet}},{key:"setItems",value:function(t){this.itemsSettingTime=new Date;var e,i=this,n=this.itemsData;if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=Ua(t)}else this.itemsData=null;if(n&&(mn(za).call(za,this.itemListeners,(function(t,e){n.off(e,t)})),n.dispose(),e=n.getIds(),this._onRemove(e)),this.itemsData){var o=this.id;mn(za).call(za,this.itemListeners,(function(t,e){i.itemsData.on(e,t,o)})),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})}},{key:"getItems",value:function(){return null!=this.itemsData?this.itemsData.rawDS:null}},{key:"setGroups",value:function(t){var e,i=this;if(this.groupsData&&(mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.off(e,t)})),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n,o=this.groupsData;this.groupsData instanceof Xa&&(o=this.groupsData.getDataSet()),mn(n=o.get()).call(n,(function(t){var e;t.nestedGroups&&mn(e=t.nestedGroups).call(e,(function(e){var i=o.get(e);i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),o.update(i)}))}));var s=this.id;mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.on(e,t,s)})),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.options.cluster&&(this.clusterGenerator.updateData(),this._clusterItems(),this.markDirty({refreshItems:!0,restackGroups:!0})),this.body.emitter.emit("_change",{queue:!0})}},{key:"getGroups",value:function(){return this.groupsData}},{key:"removeItem",value:function(t){var e=this,i=this.itemsData.get(t);i&&this.options.onRemove(i,(function(i){i&&e.itemsData.remove(t)}))}},{key:"_getType",value:function(t){return t.type||this.options.type||(t.end?"range":"box")}},{key:"getGroupId",value:function(t){return"background"==this._getType(t)&&null==t.group?du:this.groupsData?t.group:"__ungrouped__"}},{key:"_onUpdate",value:function(t){var i=this,n=this;mn(t).call(t,(function(t){var o,s=n.itemsData.get(t),r=n.items[t],a=s?n._getType(s):null,l=e.types[a];if(r&&(l&&r instanceof l?n._updateItem(r,s):(o=r.selected,n._removeItem(r),r=null)),!r&&s){if(!l)throw new TypeError('Unknown item type "'.concat(a,'"'));(r=new l(s,n.conversion,n.options)).id=t,n._addItem(r),o&&(i.selection.push(t),r.select())}})),this._order(),this.options.cluster&&(this.clusterGenerator.setItems(this.items,{applyOnChangedLevel:!1}),this._clusterItems()),this.body.emitter.emit("_change",{queue:!0})}},{key:"_onRemove",value:function(t){var e=0,i=this;mn(t).call(t,(function(t){var n=i.items[t];n&&(e++,i._removeItem(n))})),e&&(this._order(),this.body.emitter.emit("_change",{queue:!0}))}},{key:"_order",value:function(){mn(za).call(za,this.groups,(function(t){t.order()}))}},{key:"_onUpdateGroups",value:function(t){this._onAddGroups(t)}},{key:"_onAddGroups",value:function(t){var e=this;mn(t).call(t,(function(t){var i=e.groupsData.get(t),n=e.groups[t];if(n)n.setData(i);else{if("__ungrouped__"==t||t==du)throw new Error("Illegal group id. ".concat(t," is a reserved id."));var o=Oi(e.options);for(var s in za.extend(o,{height:null}),n=new jd(t,i,e),e.groups[t]=n,e.items)if(e.items.hasOwnProperty(s)){var r=e.items[s];r.data.group==t&&n.add(r)}n.order(),n.show()}})),this.body.emitter.emit("_change",{queue:!0})}},{key:"_onRemoveGroups",value:function(t){var e=this;mn(t).call(t,(function(t){var i=e.groups[t];i&&(i.dispose(),delete e.groups[t])})),this.options.cluster&&(this.clusterGenerator.updateData(),this._clusterItems()),this.markDirty({restackGroups:!!this.options.cluster}),this.body.emitter.emit("_change",{queue:!0})}},{key:"_orderGroups",value:function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder});t=this._orderNestedGroups(t);var e=!za.equalArray(t,this.groupIds);if(e){var i=this.groups;mn(t).call(t,(function(t){i[t].hide()})),mn(t).call(t,(function(t){i[t].show()})),this.groupIds=t}return e}return!1}},{key:"_orderNestedGroups",value:function(t){var e=this;return function t(e,i){var n=[];return mn(i).call(i,(function(i){if(n.push(i),e.groupsData.get(i).nestedGroups){var o,s=Ln(o=e.groupsData.get({filter:function(t){return t.nestedInGroup==i},order:e.options.groupOrder})).call(o,(function(t){return t.id}));n=Pn(n).call(n,t(e,s))}})),n}(this,En(t).call(t,(function(t){return!e.groupsData.get(t).nestedInGroup})))}},{key:"_addItem",value:function(t){this.items[t.id]=t;var e=this.getGroupId(t.data),i=this.groups[e];i?i&&i.data&&i.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1,i&&i.add(t)}},{key:"_updateItem",value:function(t,e){t.setData(e);var i=this.getGroupId(t.data),n=this.groups[i];n?n&&n.data&&n.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1}},{key:"_removeItem",value:function(t){var e,i;t.hide(),delete this.items[t.id];var n=Fo(e=this.selection).call(e,t.id);-1!=n&&Hr(i=this.selection).call(i,n,1),t.parent&&t.parent.remove(t),null!=this.popup&&this.popup.hide()}},{key:"_constructByEndArray",value:function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof qd&&e.push(t[i]);return e}},{key:"_onTouch",value:function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null}},{key:"_getGroupIndex",value:function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e}},{key:"_onDragStart",value:function(t){var e=this;if(!this.touchParams.itemIsDragging){var i,n=this.touchParams.item||null,o=this;if(n&&(n.selected||this.options.itemsAlwaysDraggable.item)){if(this.options.editable.overrideItems&&!this.options.editable.updateTime&&!this.options.editable.updateGroup)return;if(null!=n.editable&&!n.editable.updateTime&&!n.editable.updateGroup&&!this.options.editable.overrideItems)return;var s=this.touchParams.dragLeftItem,r=this.touchParams.dragRightItem;if(this.touchParams.itemIsDragging=!0,this.touchParams.selectedItem=n,s)i={item:s,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(n.data)},this.touchParams.itemProps=[i];else if(r)i={item:r,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(n.data)},this.touchParams.itemProps=[i];else if(this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey))this._onDragStartAddItem(t);else{this.groupIds.length<1&&this.redraw();var a=this._getGroupIndex(n.data.group),l=this.options.itemsAlwaysDraggable.item&&!n.selected?[n.id]:this.getSelection();this.touchParams.itemProps=Ln(l).call(l,(function(i){var n=o.items[i],s=o._getGroupIndex(n.data.group);return{item:n,initialX:t.center.x,groupOffset:a-s,data:e._cloneItemData(n.data)}}))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)}}},{key:"_onDragStartAddItem",value:function(t){var e=this.options.snap||null,i=this.dom.frame.getBoundingClientRect(),n=this.options.rtl?i.right-t.center.x+10:t.center.x-i.left-10,o=this.body.util.toTime(n),s=this.body.util.getScale(),r=this.body.util.getStep(),a=e?e(o,s,r):o,l={type:"range",start:a,end:a,content:"new item"},h=Id();l[this.itemsData.idProp]=h;var d=this.groupFromTarget(t);d&&(l.group=d.groupId);var u=new qd(l,this.conversion,this.options);u.id=h,u.data=this._cloneItemData(l),this._addItem(u),this.touchParams.selectedItem=u;var c={item:u,initialX:t.center.x,data:u.data};this.options.rtl?c.dragLeft=!0:c.dragRight=!0,this.touchParams.itemProps=[c],t.stopPropagation()}},{key:"_onDrag",value:function(t){var e=this;if(null!=this.popup&&this.options.showTooltips&&!this.popup.hidden){var i=this.body.dom.centerContainer,n=i.getBoundingClientRect();this.popup.setPosition(t.center.x-n.left+i.offsetLeft,t.center.y-n.top+i.offsetTop),this.popup.show()}if(this.touchParams.itemProps){var o;t.stopPropagation();var s=this,r=this.options.snap||null,a=this.body.dom.root.offsetLeft,l=this.options.rtl?a+this.body.domProps.right.width:a+this.body.domProps.left.width,h=this.body.util.getScale(),d=this.body.util.getStep(),u=this.touchParams.selectedItem,c=(this.options.editable.overrideItems||null==u.editable)&&this.options.editable.updateGroup||!this.options.editable.overrideItems&&null!=u.editable&&u.editable.updateGroup,p=null;if(c&&u&&null!=u.data.group){var m=s.groupFromTarget(t);m&&(p=this._getGroupIndex(m.groupId))}mn(o=this.touchParams.itemProps).call(o,(function(i){var n,o,a,m,f,g=s.body.util.toTime(t.center.x-l),v=s.body.util.toTime(i.initialX-l);n=e.options.rtl?-(g-v):g-v;var b=e._cloneItemData(i.item.data);if(null==i.item.editable||i.item.editable.updateTime||i.item.editable.updateGroup||s.options.editable.overrideItems){if((e.options.editable.overrideItems||null==u.editable)&&e.options.editable.updateTime||!e.options.editable.overrideItems&&null!=u.editable&&u.editable.updateTime)if(i.dragLeft)e.options.rtl?null!=b.end&&(a=za.convert(i.data.end,"Date"),f=new Date(a.valueOf()+n),b.end=r?r(f,h,d):f):null!=b.start&&(o=za.convert(i.data.start,"Date"),m=new Date(o.valueOf()+n),b.start=r?r(m,h,d):m);else if(i.dragRight)e.options.rtl?null!=b.start&&(o=za.convert(i.data.start,"Date"),m=new Date(o.valueOf()+n),b.start=r?r(m,h,d):m):null!=b.end&&(a=za.convert(i.data.end,"Date"),f=new Date(a.valueOf()+n),b.end=r?r(f,h,d):f);else if(null!=b.start)if(o=za.convert(i.data.start,"Date").valueOf(),m=new Date(o+n),null!=b.end){var y=(a=za.convert(i.data.end,"Date")).valueOf()-o.valueOf();b.start=r?r(m,h,d):m,b.end=new Date(b.start.valueOf()+y)}else b.start=r?r(m,h,d):m;if(c&&!i.dragLeft&&!i.dragRight&&null!=p&&null!=b.group){var x=p-i.groupOffset;x=Math.max(0,x),x=Math.min(s.groupIds.length-1,x),b.group=s.groupIds[x]}b=e._cloneItemData(b),s.options.onMoving(b,(function(t){t&&i.item.setData(e._cloneItemData(t,"Date"))}))}})),this.body.emitter.emit("_change")}}},{key:"_moveToGroup",value:function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var n=t.parent;n.remove(t),n.order(),t.data.group=i.groupId,i.add(t),i.order()}}},{key:"_onDragEnd",value:function(t){var e=this;if(this.touchParams.itemIsDragging=!1,this.touchParams.itemProps){t.stopPropagation();var i=this,n=this.touchParams.itemProps;this.touchParams.itemProps=null,mn(n).call(n,(function(t){var n=t.item.id;if(null!=i.itemsData.get(n)){var o=e._cloneItemData(t.item.data);i.options.onMove(o,(function(o){o?(o[e.itemsData.idProp]=n,e.itemsData.update(o)):(t.item.setData(t.data),i.body.emitter.emit("_change"))}))}else i.options.onAdd(t.item.data,(function(e){i._removeItem(t.item),e&&i.itemsData.add(e),i.body.emitter.emit("_change")}))}))}}},{key:"_onGroupClick",value:function(t){var e=this,i=this.groupFromTarget(t);qr((function(){e.toggleGroupShowNested(i)}),1)}},{key:"toggleGroupShowNested",value:function(t){var e,i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:void 0;if(t&&t.nestedGroups){var n=this.groupsData.getDataSet();t.showNested=null!=i?!!i:!t.showNested;var o=n.get(t.groupId);o.showNested=t.showNested;for(var s=t.nestedGroups,r=s;r.length>0;){var a=r;r=[];for(var l=0;l<a.length;l++){var h=n.get(a[l]);h.nestedGroups&&(r=Pn(r).call(r,h.nestedGroups))}r.length>0&&(s=Pn(s).call(s,r))}var d=Ln(e=n.get(s)).call(e,(function(t){return null==t.visible&&(t.visible=!0),t.visible=!!o.showNested,t}));n.update(Pn(d).call(d,o)),o.showNested?(za.removeClassName(t.dom.label,"collapsed"),za.addClassName(t.dom.label,"expanded")):(za.removeClassName(t.dom.label,"expanded"),za.addClassName(t.dom.label,"collapsed"))}}},{key:"toggleGroupDragClassName",value:function(t){t.dom.label.classList.toggle("vis-group-is-dragging"),t.dom.foreground.classList.toggle("vis-group-is-dragging")}},{key:"_onGroupDragStart",value:function(t){this.groupTouchParams.isDragging||this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.isDragging=!0,this.toggleGroupDragClassName(this.groupTouchParams.group),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))}},{key:"_onGroupDrag",value:function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupsData;this.groupsData instanceof Xa&&(e=this.groupsData.getDataSet());var i=this.groupFromTarget(t);if(i&&i.height!=this.groupTouchParams.group.height){var n=i.top<this.groupTouchParams.group.top,o=t.center?t.center.y:t.clientY,s=i.dom.foreground.getBoundingClientRect(),r=this.groupTouchParams.group.height;if(n){if(s.top+r<o)return}else{var a=i.height;if(s.top+a-r>o)return}}if(i&&i!=this.groupTouchParams.group){var l=e.get(i.groupId),h=e.get(this.groupTouchParams.group.groupId);h&&l&&(this.options.groupOrderSwap(h,l,e),e.update(h),e.update(l));var d=e.getIds({order:this.options.groupOrder});if(!za.equalArray(d,this.groupTouchParams.originalOrder))for(var u=this.groupTouchParams.originalOrder,c=this.groupTouchParams.group.groupId,p=Math.min(u.length,d.length),m=0,f=0,g=0;m<p;){for(;m+f<p&&m+g<p&&d[m+f]==u[m+g];)m++;if(m+f>=p)break;if(d[m+f]==c)f=1;else if(u[m+g]==c)g=1;else{var v=Fo(d).call(d,u[m+g]),b=e.get(d[m+f]),y=e.get(u[m+g]);this.options.groupOrderSwap(b,y,e),e.update(b),e.update(y);var x=d[m+f];d[m+f]=u[m+g],d[v]=x,m++}}}}}},{key:"_onGroupDragEnd",value:function(t){if(this.groupTouchParams.isDragging=!1,this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,n=e.groupsData.getDataSet(),o=za.extend({},n.get(i));e.options.onMoveGroup(o,(function(t){if(t)t[n._idProp]=i,n.update(t);else{var o=n.getIds({order:e.options.groupOrder});if(!za.equalArray(o,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,r=Math.min(s.length,o.length),a=0;a<r;){for(;a<r&&o[a]==s[a];)a++;if(a>=r)break;var l=Fo(o).call(o,s[a]),h=n.get(o[a]),d=n.get(s[a]);e.options.groupOrderSwap(h,d,n),n.update(h),n.update(d);var u=o[a];o[a]=s[a],o[l]=u,a++}}})),e.body.emitter.emit("groupDragged",{groupId:i}),this.toggleGroupDragClassName(this.groupTouchParams.group),this.groupTouchParams.group=null}}},{key:"_onSelectItem",value:function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)this._onMultiSelectItem(t);else{var n=this.getSelection(),o=this.itemFromTarget(t),s=o&&o.selectable?[o.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||n.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}}}},{key:"_onMouseOver",value:function(t){var e=this.itemFromTarget(t);if(e&&e!==this.itemFromRelatedTarget(t)){var i=e.getTitle();if(this.options.showTooltips&&i){null==this.popup&&(this.popup=new tu(this.body.dom.root,this.options.tooltip.overflowMethod||"flip")),this.popup.setText(i);var n=this.body.dom.centerContainer,o=n.getBoundingClientRect();this.popup.setPosition(t.clientX-o.left+n.offsetLeft,t.clientY-o.top+n.offsetTop),this.setPopupTimer(this.popup)}else this.clearPopupTimer(),null!=this.popup&&this.popup.hide();this.body.emitter.emit("itemover",{item:e.id,event:t})}}},{key:"_onMouseOut",value:function(t){var e=this.itemFromTarget(t);e&&(e!==this.itemFromRelatedTarget(t)&&(this.clearPopupTimer(),null!=this.popup&&this.popup.hide(),this.body.emitter.emit("itemout",{item:e.id,event:t})))}},{key:"_onMouseMove",value:function(t){if(this.itemFromTarget(t)&&(null!=this.popupTimer&&this.setPopupTimer(this.popup),this.options.showTooltips&&this.options.tooltip.followMouse&&this.popup&&!this.popup.hidden)){var e=this.body.dom.centerContainer,i=e.getBoundingClientRect();this.popup.setPosition(t.clientX-i.left+e.offsetLeft,t.clientY-i.top+e.offsetTop),this.popup.show()}}},{key:"_onMouseWheel",value:function(t){this.touchParams.itemIsDragging&&this._onDragEnd(t)}},{key:"_onUpdateItem",value:function(t){if(this.options.selectable&&(this.options.editable.updateTime||this.options.editable.updateGroup)){var e=this;if(t){var i=e.itemsData.get(t.id);this.options.onUpdate(i,(function(t){t&&e.itemsData.update(t)}))}}}},{key:"_onDropObjectOnItem",value:function(t){var e=this.itemFromTarget(t),i=JSON.parse(t.dataTransfer.getData("text"));this.options.onDropObjectOnItem(i,e)}},{key:"_onAddItem",value:function(t){if(this.options.selectable&&this.options.editable.add){var e,i,n=this,o=this.options.snap||null,s=this.dom.frame.getBoundingClientRect(),r=this.options.rtl?s.right-t.center.x:t.center.x-s.left,a=this.body.util.toTime(r),l=this.body.util.getScale(),h=this.body.util.getStep();"drop"==t.type?((i=JSON.parse(t.dataTransfer.getData("text"))).content=i.content?i.content:"new item",i.start=i.start?i.start:o?o(a,l,h):a,i.type=i.type||"box",i[this.itemsData.idProp]=i.id||Id(),"range"!=i.type||i.end||(e=this.body.util.toTime(r+this.props.width/5),i.end=o?o(e,l,h):e)):((i={start:o?o(a,l,h):a,content:"new item"})[this.itemsData.idProp]=Id(),"range"===this.options.type&&(e=this.body.util.toTime(r+this.props.width/5),i.end=o?o(e,l,h):e));var d=this.groupFromTarget(t);d&&(i.group=d.groupId),i=this._cloneItemData(i),this.options.onAdd(i,(function(e){e&&(n.itemsData.add(e),"drop"==t.type&&n.setSelection([e.id]))}))}}},{key:"_onMultiSelectItem",value:function(t){var i=this;if(this.options.selectable){var n=this.itemFromTarget(t);if(n){var o=this.options.multiselect?this.getSelection():[];if((t.srcEvent&&t.srcEvent.shiftKey||!1||this.options.sequentialSelection)&&this.options.multiselect){var s=this.itemsData.get(n.id).group,r=void 0;this.options.multiselectPerGroup&&o.length>0&&(r=this.itemsData.get(o[0]).group),this.options.multiselectPerGroup&&null!=r&&r!=s||o.push(n.id);var a=e._getItemRange(this.itemsData.get(o));if(!this.options.multiselectPerGroup||r==s)for(var l in o=[],this.items)if(this.items.hasOwnProperty(l)){var h=this.items[l],d=h.data.start,u=void 0!==h.data.end?h.data.end:d;!(d>=a.min&&u<=a.max)||this.options.multiselectPerGroup&&r!=this.itemsData.get(h.id).group||h instanceof $d||o.push(h.id)}}else{var c=Fo(o).call(o,n.id);-1==c?o.push(n.id):Hr(o).call(o,c,1)}var p=En(o).call(o,(function(t){return i.getItemById(t).selectable}));this.setSelection(p),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}}},{key:"itemFromElement",value:function(t){for(var e=t;e;){if(e.hasOwnProperty("vis-item"))return e["vis-item"];e=e.parentNode}return null}},{key:"itemFromTarget",value:function(t){return this.itemFromElement(t.target)}},{key:"itemFromRelatedTarget",value:function(t){return this.itemFromElement(t.relatedTarget)}},{key:"groupFromTarget",value:function(t){var e=t.center?t.center.y:t.clientY,i=this.groupIds;i.length<=0&&this.groupsData&&(i=this.groupsData.getIds({order:this.options.groupOrder}));for(var n=0;n<i.length;n++){var o=i[n],s=this.groups[o],r=s.dom.foreground,a=r.getBoundingClientRect();if(e>=a.top&&e<a.top+r.offsetHeight)return s;if("top"===this.options.orientation.item){if(n===this.groupIds.length-1&&e>a.top)return s}else if(0===n&&e<a.top+r.offset)return s}return null}},{key:"_cloneItemData",value:function(t,e){var i=za.extend({},t);return e||(e=this.itemsData.type),null!=i.start&&(i.start=za.convert(i.start,e&&e.start||"Date")),null!=i.end&&(i.end=za.convert(i.end,e&&e.end||"Date")),i}},{key:"_clusterItems",value:function(){if(this.options.cluster){var t=this.body.range.conversion(this.body.domProps.center.width).scale,e=this.clusterGenerator.getClusters(this.clusters,t,this.options.cluster);if(this.clusters!=e){if(this._detachAllClusters(),e){var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(e);!(i=(s=r.next()).done);i=!0){s.value.attach()}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}this.clusters=e}this._updateClusters(e)}}}},{key:"_detachAllClusters",value:function(){if(this.options.cluster&&this.clusters&&this.clusters.length){var t=!0,e=!1,i=void 0;try{for(var n,o=Fi(this.clusters);!(t=(n=o.next()).done);t=!0){n.value.detach()}}catch(t){e=!0,i=t}finally{try{t||null==o.return||o.return()}finally{if(e)throw i}}}}},{key:"_updateClusters",value:function(t){if(this.clusters&&this.clusters.length){var e,i=new Ar(Ln(t).call(t,(function(t){return t.id}))),n=En(e=this.clusters).call(e,(function(t){return!i.has(t.id)})),o=!1,s=!0,r=!1,a=void 0;try{for(var l,h=Fi(n);!(s=(l=h.next()).done);s=!0){var d,u,c=l.value,p=Fo(d=this.selection).call(d,c.id);if(-1!==p)c.unselect(),Hr(u=this.selection).call(u,p,1),o=!0}}catch(t){r=!0,a=t}finally{try{s||null==h.return||h.return()}finally{if(r)throw a}}if(o){var m=this.getSelection();this.body.emitter.emit("select",{items:m,event:event})}}this.clusters=t||[]}}],[{key:"_getItemRange",value:function(t){var e=null,i=null;return mn(t).call(t,(function(t){(null==i||t.start<i)&&(i=t.start),null!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)})),{min:i,max:e}}},{key:"itemSetFromTarget",value:function(t){for(var e=t.target;e;){if(e.hasOwnProperty("vis-itemset"))return e["vis-itemset"];e=e.parentNode}return null}}]),e}(Ja);uu.types={background:$d,box:Jd,range:qd,point:Kd},uu.prototype._onAdd=uu.prototype._onUpdate;var cu,pu=!1,mu="background: #FFeeee; color: #dd0000",fu=function(){function t(){ba(this,t)}return xa(t,null,[{key:"validate",value:function(e,i,n){pu=!1,cu=i;var o=i;return void 0!==n&&(o=i[n]),t.parse(e,o,[]),pu}},{key:"parse",value:function(e,i,n){for(var o in e)e.hasOwnProperty(o)&&t.check(o,e,i,n)}},{key:"check",value:function(e,i,n,o){if(void 0!==n[e]||void 0!==n.__any__){var s=e,r=!0;void 0===n[e]&&void 0!==n.__any__&&(s="__any__",r="object"===t.getType(i[e]));var a=n[s];r&&void 0!==a.__type__&&(a=a.__type__),t.checkFields(e,i,n,s,a,o)}else t.getSuggestion(e,n,o)}},{key:"checkFields",value:function(e,i,n,o,s,r){var a=function(i){console.log("%c"+i+t.printLocation(r,e),mu)},l=t.getType(i[e]),h=s[l];void 0!==h?"array"===t.getType(h)&&-1===Fo(h).call(h,i[e])?(a('Invalid option detected in "'+e+'". Allowed values are:'+t.print(h)+' not "'+i[e]+'". '),pu=!0):"object"===l&&"__any__"!==o&&(r=za.copyAndExtendArray(r,e),t.parse(i[e],n[o],r)):void 0===s.any&&(a('Invalid type received for "'+e+'". Expected: '+t.print(Yi(s))+". Received ["+l+'] "'+i[e]+'"'),pu=!0)}},{key:"getType",value:function(t){var e=zo(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Zo(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":!0===t._isAMomentObject?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,n){var o,s=t.findInOptions(e,i,n,!1),r=t.findInOptions(e,cu,[],!0);o=void 0!==s.indexMatch?" in "+t.printLocation(s.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+s.indexMatch+'"?\n\n':r.distance<=4&&s.distance>r.distance?" in "+t.printLocation(s.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(r.path,r.closestMatch,""):s.distance<=8?'. Did you mean "'+s.closestMatch+'"?'+t.printLocation(s.path,e):". Did you mean one of these: "+t.print(Yi(i))+t.printLocation(n,e),console.log('%cUnknown option detected: "'+e+'"'+o,mu),pu=!0}},{key:"findInOptions",value:function(e,i,n){var o=arguments.length>3&&void 0!==arguments[3]&&arguments[3],s=1e9,r="",a=[],l=e.toLowerCase(),h=void 0;for(var d in i){var u=void 0;if(void 0!==i[d].__type__&&!0===o){var c=t.findInOptions(e,i[d],za.copyAndExtendArray(n,d));s>c.distance&&(r=c.closestMatch,a=c.path,s=c.distance,h=c.indexMatch)}else{var p;-1!==Fo(p=d.toLowerCase()).call(p,l)&&(h=d),s>(u=t.levenshteinDistance(e,d))&&(r=d,a=za.copyArray(n),s=u)}}return{closestMatch:r,path:a,distance:s,indexMatch:h}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"Problem value found at: \n",n="\n\n"+i+"options = {\n",o=0;o<t.length;o++){for(var s=0;s<o+1;s++)n+="  ";n+=t[o]+": {\n"}for(var r=0;r<t.length+1;r++)n+="  ";n+=e+"\n";for(var a=0;a<t.length+1;a++){for(var l=0;l<t.length-a;l++)n+="  ";n+="}\n"}return n+"\n\n"}},{key:"print",value:function(t){return Vs(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,n,o=[];for(i=0;i<=e.length;i++)o[i]=[i];for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}(),gu="string",vu="boolean",bu="number",yu="object",xu={configure:{enabled:{boolean:vu},filter:{boolean:vu,function:"function"},container:{dom:"dom"},__type__:{object:yu,boolean:vu,function:"function"}},align:{string:gu},alignCurrentTime:{string:gu,undefined:"undefined"},rtl:{boolean:vu,undefined:"undefined"},rollingMode:{follow:{boolean:vu},offset:{number:bu,undefined:"undefined"},__type__:{object:yu}},onTimeout:{timeoutMs:{number:bu},callback:{function:"function"},__type__:{object:yu}},verticalScroll:{boolean:vu,undefined:"undefined"},horizontalScroll:{boolean:vu,undefined:"undefined"},autoResize:{boolean:vu},throttleRedraw:{number:bu},clickToUse:{boolean:vu},dataAttributes:{string:gu,array:"array"},editable:{add:{boolean:vu,undefined:"undefined"},remove:{boolean:vu,undefined:"undefined"},updateGroup:{boolean:vu,undefined:"undefined"},updateTime:{boolean:vu,undefined:"undefined"},overrideItems:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},end:{number:bu,date:"date",string:gu,moment:"moment"},format:{minorLabels:{millisecond:{string:gu,undefined:"undefined"},second:{string:gu,undefined:"undefined"},minute:{string:gu,undefined:"undefined"},hour:{string:gu,undefined:"undefined"},weekday:{string:gu,undefined:"undefined"},day:{string:gu,undefined:"undefined"},week:{string:gu,undefined:"undefined"},month:{string:gu,undefined:"undefined"},year:{string:gu,undefined:"undefined"},__type__:{object:yu,function:"function"}},majorLabels:{millisecond:{string:gu,undefined:"undefined"},second:{string:gu,undefined:"undefined"},minute:{string:gu,undefined:"undefined"},hour:{string:gu,undefined:"undefined"},weekday:{string:gu,undefined:"undefined"},day:{string:gu,undefined:"undefined"},week:{string:gu,undefined:"undefined"},month:{string:gu,undefined:"undefined"},year:{string:gu,undefined:"undefined"},__type__:{object:yu,function:"function"}},__type__:{object:yu}},moment:{function:"function"},groupHeightMode:{string:gu},groupOrder:{string:gu,function:"function"},groupEditable:{add:{boolean:vu,undefined:"undefined"},remove:{boolean:vu,undefined:"undefined"},order:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},groupOrderSwap:{function:"function"},height:{string:gu,number:bu},hiddenDates:{start:{date:"date",number:bu,string:gu,moment:"moment"},end:{date:"date",number:bu,string:gu,moment:"moment"},repeat:{string:gu},__type__:{object:yu,array:"array"}},itemsAlwaysDraggable:{item:{boolean:vu,undefined:"undefined"},range:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},limitSize:{boolean:vu},locale:{string:gu},locales:{__any__:{any:"any"},__type__:{object:yu}},longSelectPressTime:{number:bu},margin:{axis:{number:bu},item:{horizontal:{number:bu,undefined:"undefined"},vertical:{number:bu,undefined:"undefined"},__type__:{object:yu,number:bu}},__type__:{object:yu,number:bu}},max:{date:"date",number:bu,string:gu,moment:"moment"},maxHeight:{number:bu,string:gu},maxMinorChars:{number:bu},min:{date:"date",number:bu,string:gu,moment:"moment"},minHeight:{number:bu,string:gu},moveable:{boolean:vu},multiselect:{boolean:vu},multiselectPerGroup:{boolean:vu},onAdd:{function:"function"},onDropObjectOnItem:{function:"function"},onUpdate:{function:"function"},onMove:{function:"function"},onMoving:{function:"function"},onRemove:{function:"function"},onAddGroup:{function:"function"},onMoveGroup:{function:"function"},onRemoveGroup:{function:"function"},onInitialDrawComplete:{function:"function"},order:{function:"function"},orientation:{axis:{string:gu,undefined:"undefined"},item:{string:gu,undefined:"undefined"},__type__:{string:gu,object:yu}},selectable:{boolean:vu},sequentialSelection:{boolean:vu},showCurrentTime:{boolean:vu},showMajorLabels:{boolean:vu},showMinorLabels:{boolean:vu},showWeekScale:{boolean:vu},stack:{boolean:vu},stackSubgroups:{boolean:vu},cluster:{maxItems:{number:bu,undefined:"undefined"},titleTemplate:{string:gu,undefined:"undefined"},clusterCriteria:{function:"function",undefined:"undefined"},showStipes:{boolean:vu,undefined:"undefined"},fitOnDoubleClick:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},snap:{function:"function",null:"null"},start:{date:"date",number:bu,string:gu,moment:"moment"},template:{function:"function"},loadingScreenTemplate:{function:"function"},groupTemplate:{function:"function"},visibleFrameTemplate:{string:gu,function:"function"},showTooltips:{boolean:vu},tooltip:{followMouse:{boolean:vu},overflowMethod:{string:["cap","flip","none"]},delay:{number:bu},template:{function:"function"},__type__:{object:yu}},tooltipOnItemUpdateTime:{template:{function:"function"},__type__:{boolean:vu,object:yu}},timeAxis:{scale:{string:gu,undefined:"undefined"},step:{number:bu,undefined:"undefined"},__type__:{object:yu}},type:{string:gu},width:{string:gu,number:bu},preferZoom:{boolean:vu},zoomable:{boolean:vu},zoomKey:{string:["ctrlKey","altKey","shiftKey","metaKey",""]},zoomFriction:{number:bu},zoomMax:{number:bu},zoomMin:{number:bu},__type__:{object:yu}},wu={global:{align:["center","left","right"],alignCurrentTime:["none","year","month","quarter","week","isoWeek","day","date","hour","minute","second"],direction:!1,autoResize:!0,clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},groupHeightMode:["auto","fixed","fitItems"],groupsDraggable:!1,height:"",locale:"",longSelectPressTime:251,margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},preferZoom:!1,selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,stackSubgroups:!0,cluster:!1,start:"",showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip",delay:[500,0,99999,100]},tooltipOnItemUpdateTime:!1,type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","shiftKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};V({target:"Array",proto:!0},{fill:function(t){for(var e=Lt(this),i=$(e.length),n=arguments.length,o=it(n>1?arguments[1]:void 0,i),s=n>2?arguments[2]:void 0,r=void 0===s?i:it(s,i);r>o;)e[o++]=t;return e}});var _u=Pi("Array").fill,ku=Array.prototype,Du=function(t){var e=t.fill;return t===ku||t instanceof Array&&e===ku.fill?_u:e},Iu={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"},Cu=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;ba(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return xa(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){if("string"==typeof t)return Iu[t]}},{key:"setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];if("none"!==t){var i,n=this._isColorString(t);if(void 0!==n&&(t=n),!0===za.isString(t)){if(!0===za.isValidRGB(t)){var o=t.substr(4).substr(0,t.length-5).split(",");i={r:o[0],g:o[1],b:o[2],a:1}}else if(!0===za.isValidRGBA(t)){var s=t.substr(5).substr(0,t.length-6).split(",");i={r:s[0],g:s[1],b:s[2],a:s[3]}}else if(!0===za.isValidHex(t)){var r=za.hexToRGB(t);i={r:r.r,g:r.g,b:r.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var a=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:a}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+Vs(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this,e=!(arguments.length>0&&void 0!==arguments[0])||arguments[0];!0===e&&(this.previousColor=za.extend({},this.color)),!0===this.applied&&this.updateCallback(this.initialColor),this.frame.style.display="none",qr((function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)}),0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];!0===e&&(this.initialColor=za.extend({},t)),this.color=t;var i=za.RGBToHSV(t.r,t.g,t.b),n=2*Math.PI,o=this.r*i.s,s=this.centerCoordinates.x+o*Math.sin(n*i.h),r=this.centerCoordinates.y+o*Math.cos(n*i.h);this.colorPickerSelector.style.left=s-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=r-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=za.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=za.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.color,e=za.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var n=this.colorPickerCanvas.clientWidth,o=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,n,o),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),Du(i).call(i),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){var t,e,i,n;if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var o=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(o.webkitBackingStorePixelRatio||o.mozBackingStorePixelRatio||o.msBackingStorePixelRatio||o.oBackingStorePixelRatio||o.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var s=document.createElement("DIV");s.style.color="red",s.style.fontWeight="bold",s.style.padding="10px",s.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(s)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(t){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(t){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var r=this;this.opacityRange.onchange=function(){r._setOpacity(this.value)},this.opacityRange.oninput=function(){r._setOpacity(this.value)},this.brightnessRange.onchange=function(){r._setBrightness(this.value)},this.brightnessRange.oninput=function(){r._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=va(t=this._hide).call(t,this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=va(e=this._apply).call(e,this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=va(i=this._save).call(i,this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=va(n=this._loadLast).call(n,this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new Vh(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),jh(this.hammer,(function(e){t._moveSelector(e)})),this.hammer.on("tap",(function(e){t._moveSelector(e)})),this.hammer.on("panstart",(function(e){t._moveSelector(e)})),this.hammer.on("panmove",(function(e){t._moveSelector(e)})),this.hammer.on("panend",(function(e){t._moveSelector(e)}))}},{key:"_generateHueCircle",value:function(){if(!1===this.generated){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e,i,n,o,s=this.colorPickerCanvas.clientWidth,r=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,s,r),this.centerCoordinates={x:.5*s,y:.5*r},this.r=.49*s;var a,l=2*Math.PI/360,h=1/this.r;for(n=0;n<360;n++)for(o=0;o<this.r;o++)e=this.centerCoordinates.x+o*Math.sin(l*n),i=this.centerCoordinates.y+o*Math.cos(l*n),a=za.HSVToRGB(n*(1/360),o*h,1),t.fillStyle="rgb("+a.r+","+a.g+","+a.b+")",t.fillRect(e-.5,i-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,s,r)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,n=t.center.y-e.top,o=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,a=n-o,l=Math.atan2(r,a),h=.98*Math.min(Math.sqrt(r*r+a*a),s),d=Math.cos(l)*h+o,u=Math.sin(l)*h+s;this.colorPickerSelector.style.top=d-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var c=l/(2*Math.PI);c=c<0?c+1:c;var p=h/this.r,m=za.RGBToHSV(this.color.r,this.color.g,this.color.b);m.h=c,m.s=p;var f=za.HSVToRGB(m.h,m.s,m.v);f.a=this.color.a,this.color=f,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();Ph('div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper:after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:transparent;width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2,#385380);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(180deg,#3876c2 0,#385380);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#3876c2",endColorstr="#385380",GradientType=0);box-shadow:0 0 1px 0 #111927;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:none}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(180deg,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#9d9d9d",endColorstr="#c8c8c8",GradientType=0)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:transparent;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0) rgba(136,183,213,0) rgba(136,183,213,0) rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0) rgba(194,225,245,0) rgba(194,225,245,0) #f2faff;border-width:12px;margin-top:-12px}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImNvbmZpZ3VyYXRpb24uY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLHNCQUNJLGlCQUFpQixDQUNqQixhQUFhLENBQ2IsVUFBVSxDQUNWLGNBQ0osQ0FFQSw4QkFDSSxhQUFhLENBQ2IsV0FDSixDQUVBLG9DQUNFLFVBQVcsQ0FDWCxVQUFXLENBQ1gsYUFDRixDQUVBLGtEQUNJLGFBQWEsQ0FDYixXQUFXLENBQ1gscUJBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixpQkFBaUIsQ0FDakIsZUFBZSxDQUNmLFNBQVMsQ0FDVCxnQkFDSixDQUVBLHdDQUNJLGFBQWEsQ0FDYixXQUFXLENBQ1gsV0FBVyxDQUNYLHFCQUFzQixDQUN0QixnQkFBZ0IsQ0FDaEIsd0JBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixpQkFBaUIsQ0FDakIsZUFBZSxDQUNmLFNBQVMsQ0FDVCxnQkFBZ0IsQ0FDaEIsY0FBZSxDQUNmLGtCQUNKLENBRUEsOENBQ0ksd0JBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixVQUNKLENBRUEsc0NBQ0ksYUFBYSxDQUNiLFVBQVUsQ0FDVixXQUFXLENBQ1gsV0FBVyxDQUNYLHFCQUFzQixDQUN0QixnQkFDSixDQUdBLG9EQUNJLFNBQVMsQ0FDVCx3QkFBeUIsQ0FDekIsZ0JBQWdCLENBQ2hCLGlCQUNKLENBQ0Esb0RBQ0ksU0FBUyxDQUNULHdCQUF5QixDQUN6QixnQkFBZ0IsQ0FDaEIsaUJBQ0osQ0FDQSxvREFDSSxTQUFTLENBQ1Qsd0JBQXlCLENBQ3pCLGdCQUFnQixDQUNoQixpQkFDSixDQUVBLHdDQUNJLGNBQWMsQ0FDZCxlQUNKLENBRUEsdUNBQ0ksV0FBVyxDQUNYLFdBQVcsQ0FDWCxnQkFDSixDQUVBLHFEQUNJLFdBQ0osQ0FDQSxxREFDSSxXQUNKLENBRUEsNENBQ0ksT0FBTyxDQUNQLFVBQVUsQ0FDVixXQUFXLENBQ1gscUJBQXdCLENBQ3hCLGlCQUFpQixDQUNqQixTQUFXLENBQ1gsUUFBVSxDQUNWLGNBQ0osQ0FFQSw0Q0FDSSxTQUNKLENBR0EsOENBQ0ksaUJBQWlCLENBQ2pCLFFBQVEsQ0FDUixVQUFVLENBRVYsV0FBVyxDQUNYLFFBQVEsQ0FDUixtQkFDSixDQUVBLHlDQUVJLHVCQUF3QixDQUd4QixtQkFBdUIsQ0FDdkIsNEJBQThCLENBRzlCLFdBQVksQ0FDWixXQUNKLENBQ0Esd0VBQ0ksV0FBWSxDQUNaLFVBQVcsQ0FDWCxrQkFBbUIsQ0FDbkIsMERBQStELENBQy9ELHNHQUE0RyxDQUM1RywyREFBaUUsQ0FDakUsd0RBQTRELENBQzVELHlEQUE2RCxDQUM3RCx3REFBK0QsQ0FDL0QsK0dBQW1ILENBRW5ILHFCQUF5QixDQUN6Qix5QkFBbUMsQ0FDbkMsaUJBQ0osQ0FDQSwrREFDSSx1QkFBd0IsQ0FDeEIsd0JBQXlCLENBQ3pCLFdBQVksQ0FDWixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGtCQUFtQixDQUNuQiwyREFBZ0UsQ0FDaEUsdUdBQTZHLENBQzdHLHVEQUFrRSxDQUNsRSx5REFBNkQsQ0FDN0QsMERBQThELENBQzlELG9EQUFnRSxDQUNoRSwrR0FBbUgsQ0FDbkgsNEJBQW1DLENBQ25DLGVBQ0osQ0FDQSwrQ0FDSSxZQUNKLENBQ0EsOEVBQ0ksa0JBQW1CLENBQ25CLDBEQUE4RCxDQUM5RCxzR0FBNEcsQ0FDNUcsMkRBQWlFLENBQ2pFLHdEQUE0RCxDQUM1RCx5REFBNkQsQ0FDN0Qsd0RBQStELENBQy9ELCtHQUNKLENBRUEsMkRBQ0ksV0FBWSxDQUNaLFdBQVksQ0FDWixrQkFBbUIsQ0FDbkIsMERBQStELENBQy9ELHNHQUE0RyxDQUM1RywyREFBaUUsQ0FDakUsd0RBQTRELENBQzVELHlEQUE2RCxDQUM3RCx3REFBK0QsQ0FDL0QsK0dBQW1ILENBRW5ILHFCQUF5QixDQUN6Qix5QkFBbUMsQ0FDbkMsaUJBQ0osQ0FDQSwyREFDSSxXQUFZLENBQ1osV0FBWSxDQUNaLFVBQVcsQ0FFWCxpQkFBa0IsQ0FDbEIsa0JBQ0osQ0FHQSx3REFDSSxzQkFBd0IsQ0FDeEIsbUJBQ0osQ0FFQSxvREFDSSxXQUFZLENBQ1osVUFBVyxDQUdYLHNCQUF1QixDQUd2Qix3QkFBeUIsQ0FDekIsa0JBQW1CLENBR25CLGlCQUNKLENBQ0EseURBQ0ksZUFBZ0IsQ0FDaEIsa0JBQ0osQ0FDQSx5REFDSSxlQUFnQixDQUNoQixrQkFDSixDQUNBLG9EQUNJLFdBQVksQ0FDWixXQUFZLENBQ1osVUFBVyxDQUNYLGlCQUFrQixDQUNsQixrQkFDSixDQUNBLCtEQUNJLGVBQ0osQ0FDQSwrREFDSSxlQUNKLENBRUEseUJBQ0ksaUJBQWtCLENBQ2xCLDZCQUFrQyxDQUNsQyx3QkFBeUIsQ0FDekIsZ0JBQWdCLENBQ2hCLFdBQVcsQ0FDWCxXQUFXLENBQ1gsaUJBQWlCLENBQ2pCLFVBQWMsQ0FDZCxjQUFjLENBQ2QsaUJBQWlCLENBQ2pCLDBDQUE0QyxDQUM1Qyx1Q0FBeUMsQ0FDekMsa0NBQ0osQ0FDQSwrREFDSSxTQUFVLENBQ1YsT0FBUSxDQUNSLHdCQUF5QixDQUN6QixXQUFZLENBQ1osUUFBUyxDQUNULE9BQVEsQ0FDUixpQkFBa0IsQ0FDbEIsbUJBQ0osQ0FFQSwrQkFFSSwyRkFBeUMsQ0FDekMsZ0JBQWlCLENBQ2pCLGVBQ0osQ0FDQSxnQ0FFSSxnRkFBMEIsQ0FDMUIsaUJBQWtCLENBQ2xCLGdCQUNKIiwiZmlsZSI6ImNvbmZpZ3VyYXRpb24uY3NzIiwic291cmNlc0NvbnRlbnQiOlsiZGl2LnZpcy1jb25maWd1cmF0aW9uIHtcbiAgICBwb3NpdGlvbjpyZWxhdGl2ZTtcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIGZsb2F0OmxlZnQ7XG4gICAgZm9udC1zaXplOjEycHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi13cmFwcGVyIHtcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIHdpZHRoOjcwMHB4O1xufVxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24td3JhcHBlcjo6YWZ0ZXIge1xuICBjbGVhcjogYm90aDtcbiAgY29udGVudDogXCJcIjtcbiAgZGlzcGxheTogYmxvY2s7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLW9wdGlvbi1jb250YWluZXJ7XG4gICAgZGlzcGxheTpibG9jaztcbiAgICB3aWR0aDo0OTVweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjZmZmZmZmO1xuICAgIGJvcmRlcjoycHggc29saWQgI2Y3ZjhmYTtcbiAgICBib3JkZXItcmFkaXVzOjRweDtcbiAgICBtYXJnaW4tdG9wOjIwcHg7XG4gICAgbGVmdDoxMHB4O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWJ1dHRvbntcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIHdpZHRoOjQ5NXB4O1xuICAgIGhlaWdodDoyNXB4O1xuICAgIHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7XG4gICAgbGluZS1oZWlnaHQ6MjVweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjZjdmOGZhO1xuICAgIGJvcmRlcjoycHggc29saWQgI2NlY2VkMDtcbiAgICBib3JkZXItcmFkaXVzOjRweDtcbiAgICBtYXJnaW4tdG9wOjIwcHg7XG4gICAgbGVmdDoxMHB4O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG4gICAgY3Vyc29yOiBwb2ludGVyO1xuICAgIG1hcmdpbi1ib3R0b206MzBweDtcbn1cblxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctYnV0dG9uLmhvdmVye1xuICAgIGJhY2tncm91bmQtY29sb3I6ICM0NTg4ZTY7XG4gICAgYm9yZGVyOjJweCBzb2xpZCAjMjE0MzczO1xuICAgIGNvbG9yOiNmZmZmZmY7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWl0ZW17XG4gICAgZGlzcGxheTpibG9jaztcbiAgICBmbG9hdDpsZWZ0O1xuICAgIHdpZHRoOjQ5NXB4O1xuICAgIGhlaWdodDoyNXB4O1xuICAgIHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7XG4gICAgbGluZS1oZWlnaHQ6MjVweDtcbn1cblxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1pdGVtLnZpcy1jb25maWctczJ7XG4gICAgbGVmdDoxMHB4O1xuICAgIGJhY2tncm91bmQtY29sb3I6ICNmN2Y4ZmE7XG4gICAgcGFkZGluZy1sZWZ0OjVweDtcbiAgICBib3JkZXItcmFkaXVzOjNweDtcbn1cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWl0ZW0udmlzLWNvbmZpZy1zM3tcbiAgICBsZWZ0OjIwcHg7XG4gICAgYmFja2dyb3VuZC1jb2xvcjogI2U0ZTlmMDtcbiAgICBwYWRkaW5nLWxlZnQ6NXB4O1xuICAgIGJvcmRlci1yYWRpdXM6M3B4O1xufVxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctaXRlbS52aXMtY29uZmlnLXM0e1xuICAgIGxlZnQ6MzBweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjY2ZkOGU2O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG4gICAgYm9yZGVyLXJhZGl1czozcHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWhlYWRlcntcbiAgICBmb250LXNpemU6MThweDtcbiAgICBmb250LXdlaWdodDogYm9sZDtcbn1cblxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctbGFiZWx7XG4gICAgd2lkdGg6MTIwcHg7XG4gICAgaGVpZ2h0OjI1cHg7XG4gICAgbGluZS1oZWlnaHQ6IDI1cHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWxhYmVsLnZpcy1jb25maWctczN7XG4gICAgd2lkdGg6MTEwcHg7XG59XG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1sYWJlbC52aXMtY29uZmlnLXM0e1xuICAgIHdpZHRoOjEwMHB4O1xufVxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1jb2xvckJsb2Nre1xuICAgIHRvcDoxcHg7XG4gICAgd2lkdGg6MzBweDtcbiAgICBoZWlnaHQ6MTlweDtcbiAgICBib3JkZXI6MXB4IHNvbGlkICM0NDQ0NDQ7XG4gICAgYm9yZGVyLXJhZGl1czoycHg7XG4gICAgcGFkZGluZzowcHg7XG4gICAgbWFyZ2luOjBweDtcbiAgICBjdXJzb3I6cG9pbnRlcjtcbn1cblxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1jaGVja2JveCB7XG4gICAgbGVmdDotNXB4O1xufVxuXG5cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2VpbnB1dHtcbiAgICBwb3NpdGlvbjpyZWxhdGl2ZTtcbiAgICB0b3A6LTVweDtcbiAgICB3aWR0aDo2MHB4O1xuICAgIC8qaGVpZ2h0OjEzcHg7Ki9cbiAgICBwYWRkaW5nOjFweDtcbiAgICBtYXJnaW46MDtcbiAgICBwb2ludGVyLWV2ZW50czpub25lO1xufVxuXG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdle1xuICAgIC8qcmVtb3ZlcyBkZWZhdWx0IHdlYmtpdCBzdHlsZXMqL1xuICAgIC13ZWJraXQtYXBwZWFyYW5jZTogbm9uZTtcblxuICAgIC8qZml4IGZvciBGRiB1bmFibGUgdG8gYXBwbHkgZm9jdXMgc3R5bGUgYnVnICovXG4gICAgYm9yZGVyOiAwcHggc29saWQgd2hpdGU7XG4gICAgYmFja2dyb3VuZC1jb2xvcjpyZ2JhKDAsMCwwLDApO1xuXG4gICAgLypyZXF1aXJlZCBmb3IgcHJvcGVyIHRyYWNrIHNpemluZyBpbiBGRiovXG4gICAgd2lkdGg6IDMwMHB4O1xuICAgIGhlaWdodDoyMHB4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LXdlYmtpdC1zbGlkZXItcnVubmFibGUtdHJhY2sge1xuICAgIHdpZHRoOiAzMDBweDtcbiAgICBoZWlnaHQ6IDVweDtcbiAgICBiYWNrZ3JvdW5kOiAjZGVkZWRlOyAvKiBPbGQgYnJvd3NlcnMgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbW96LWxpbmVhci1ncmFkaWVudCh0b3AsICAjZGVkZWRlIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsI2RlZGVkZSksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgI2RlZGVkZSAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgI2RlZGVkZSAwJSwgI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICNkZWRlZGUgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjZGVkZWRlIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyNkZWRlZGUnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cblxuICAgIGJvcmRlcjogMXB4IHNvbGlkICM5OTk5OTk7XG4gICAgYm94LXNoYWRvdzogI2FhYWFhYSAwcHggMHB4IDNweCAwcHg7XG4gICAgYm9yZGVyLXJhZGl1czogM3B4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LXdlYmtpdC1zbGlkZXItdGh1bWIge1xuICAgIC13ZWJraXQtYXBwZWFyYW5jZTogbm9uZTtcbiAgICBib3JkZXI6IDFweCBzb2xpZCAjMTQzMzRiO1xuICAgIGhlaWdodDogMTdweDtcbiAgICB3aWR0aDogMTdweDtcbiAgICBib3JkZXItcmFkaXVzOiA1MCU7XG4gICAgYmFja2dyb3VuZDogIzM4NzZjMjsgLyogT2xkIGJyb3dzZXJzICovXG4gICAgYmFja2dyb3VuZDogLW1vei1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwgIzM4NTM4MCAxMDAlKTsgLyogRkYzLjYrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1ncmFkaWVudChsaW5lYXIsIGxlZnQgdG9wLCBsZWZ0IGJvdHRvbSwgY29sb3Itc3RvcCgwJSwjMzg3NmMyKSwgY29sb3Itc3RvcCgxMDAlLCMzODUzODApKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwjMzg1MzgwIDEwMCUpOyAvKiBDaHJvbWUxMCssU2FmYXJpNS4xKyAqL1xuICAgIGJhY2tncm91bmQ6IC1vLWxpbmVhci1ncmFkaWVudCh0b3AsICAjMzg3NmMyIDAlLCMzODUzODAgMTAwJSk7IC8qIE9wZXJhIDExLjEwKyAqL1xuICAgIGJhY2tncm91bmQ6IC1tcy1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwjMzg1MzgwIDEwMCUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjMzg3NmMyIDAlLCMzODUzODAgMTAwJSk7IC8qIFczQyAqL1xuICAgIGZpbHRlcjogcHJvZ2lkOkRYSW1hZ2VUcmFuc2Zvcm0uTWljcm9zb2Z0LmdyYWRpZW50KCBzdGFydENvbG9yc3RyPScjMzg3NmMyJywgZW5kQ29sb3JzdHI9JyMzODUzODAnLEdyYWRpZW50VHlwZT0wICk7IC8qIElFNi05ICovXG4gICAgYm94LXNoYWRvdzogIzExMTkyNyAwcHggMHB4IDFweCAwcHg7XG4gICAgbWFyZ2luLXRvcDogLTdweDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Zm9jdXMge1xuICAgIG91dGxpbmU6IG5vbmU7XG59XG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOmZvY3VzOjotd2Via2l0LXNsaWRlci1ydW5uYWJsZS10cmFjayB7XG4gICAgYmFja2dyb3VuZDogIzlkOWQ5ZDsgLyogT2xkIGJyb3dzZXJzICovXG4gICAgYmFja2dyb3VuZDogLW1vei1saW5lYXItZ3JhZGllbnQodG9wLCAjOWQ5ZDlkIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsIzlkOWQ5ZCksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgIzlkOWQ5ZCAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgICM5ZDlkOWQgMCUsI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICM5ZDlkOWQgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjOWQ5ZDlkIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyM5ZDlkOWQnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cbn1cblxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1vei1yYW5nZS10cmFjayB7XG4gICAgd2lkdGg6IDMwMHB4O1xuICAgIGhlaWdodDogMTBweDtcbiAgICBiYWNrZ3JvdW5kOiAjZGVkZWRlOyAvKiBPbGQgYnJvd3NlcnMgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbW96LWxpbmVhci1ncmFkaWVudCh0b3AsICAjZGVkZWRlIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsI2RlZGVkZSksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgI2RlZGVkZSAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgI2RlZGVkZSAwJSwgI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICNkZWRlZGUgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjZGVkZWRlIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyNkZWRlZGUnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cblxuICAgIGJvcmRlcjogMXB4IHNvbGlkICM5OTk5OTk7XG4gICAgYm94LXNoYWRvdzogI2FhYWFhYSAwcHggMHB4IDNweCAwcHg7XG4gICAgYm9yZGVyLXJhZGl1czogM3B4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1vei1yYW5nZS10aHVtYiB7XG4gICAgYm9yZGVyOiBub25lO1xuICAgIGhlaWdodDogMTZweDtcbiAgICB3aWR0aDogMTZweDtcblxuICAgIGJvcmRlci1yYWRpdXM6IDUwJTtcbiAgICBiYWNrZ3JvdW5kOiAgIzM4NTM4MDtcbn1cblxuLypoaWRlIHRoZSBvdXRsaW5lIGJlaGluZCB0aGUgYm9yZGVyKi9cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6LW1vei1mb2N1c3Jpbmd7XG4gICAgb3V0bGluZTogMXB4IHNvbGlkIHdoaXRlO1xuICAgIG91dGxpbmUtb2Zmc2V0OiAtMXB4O1xufVxuXG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOjotbXMtdHJhY2sge1xuICAgIHdpZHRoOiAzMDBweDtcbiAgICBoZWlnaHQ6IDVweDtcblxuICAgIC8qcmVtb3ZlIGJnIGNvbG91ciBmcm9tIHRoZSB0cmFjaywgd2UnbGwgdXNlIG1zLWZpbGwtbG93ZXIgYW5kIG1zLWZpbGwtdXBwZXIgaW5zdGVhZCAqL1xuICAgIGJhY2tncm91bmQ6IHRyYW5zcGFyZW50O1xuXG4gICAgLypsZWF2ZSByb29tIGZvciB0aGUgbGFyZ2VyIHRodW1iIHRvIG92ZXJmbG93IHdpdGggYSB0cmFuc3BhcmVudCBib3JkZXIgKi9cbiAgICBib3JkZXItY29sb3I6IHRyYW5zcGFyZW50O1xuICAgIGJvcmRlci13aWR0aDogNnB4IDA7XG5cbiAgICAvKnJlbW92ZSBkZWZhdWx0IHRpY2sgbWFya3MqL1xuICAgIGNvbG9yOiB0cmFuc3BhcmVudDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Oi1tcy1maWxsLWxvd2VyIHtcbiAgICBiYWNrZ3JvdW5kOiAjNzc3O1xuICAgIGJvcmRlci1yYWRpdXM6IDEwcHg7XG59XG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOjotbXMtZmlsbC11cHBlciB7XG4gICAgYmFja2dyb3VuZDogI2RkZDtcbiAgICBib3JkZXItcmFkaXVzOiAxMHB4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1zLXRodW1iIHtcbiAgICBib3JkZXI6IG5vbmU7XG4gICAgaGVpZ2h0OiAxNnB4O1xuICAgIHdpZHRoOiAxNnB4O1xuICAgIGJvcmRlci1yYWRpdXM6IDUwJTtcbiAgICBiYWNrZ3JvdW5kOiAgIzM4NTM4MDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Zm9jdXM6Oi1tcy1maWxsLWxvd2VyIHtcbiAgICBiYWNrZ3JvdW5kOiAjODg4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTpmb2N1czo6LW1zLWZpbGwtdXBwZXIge1xuICAgIGJhY2tncm91bmQ6ICNjY2M7XG59XG5cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cCB7XG4gICAgcG9zaXRpb246IGFic29sdXRlO1xuICAgIGJhY2tncm91bmQ6IHJnYmEoNTcsIDc2LCA4OSwgMC44NSk7XG4gICAgYm9yZGVyOiAycHggc29saWQgI2YyZmFmZjtcbiAgICBsaW5lLWhlaWdodDozMHB4O1xuICAgIGhlaWdodDozMHB4O1xuICAgIHdpZHRoOjE1MHB4O1xuICAgIHRleHQtYWxpZ246Y2VudGVyO1xuICAgIGNvbG9yOiAjZmZmZmZmO1xuICAgIGZvbnQtc2l6ZToxNHB4O1xuICAgIGJvcmRlci1yYWRpdXM6NHB4O1xuICAgIC13ZWJraXQtdHJhbnNpdGlvbjogb3BhY2l0eSAwLjNzIGVhc2UtaW4tb3V0O1xuICAgIC1tb3otdHJhbnNpdGlvbjogb3BhY2l0eSAwLjNzIGVhc2UtaW4tb3V0O1xuICAgIHRyYW5zaXRpb246IG9wYWNpdHkgMC4zcyBlYXNlLWluLW91dDtcbn1cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cDphZnRlciwgLnZpcy1jb25maWd1cmF0aW9uLXBvcHVwOmJlZm9yZSB7XG4gICAgbGVmdDogMTAwJTtcbiAgICB0b3A6IDUwJTtcbiAgICBib3JkZXI6IHNvbGlkIHRyYW5zcGFyZW50O1xuICAgIGNvbnRlbnQ6IFwiIFwiO1xuICAgIGhlaWdodDogMDtcbiAgICB3aWR0aDogMDtcbiAgICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gICAgcG9pbnRlci1ldmVudHM6IG5vbmU7XG59XG5cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cDphZnRlciB7XG4gICAgYm9yZGVyLWNvbG9yOiByZ2JhKDEzNiwgMTgzLCAyMTMsIDApO1xuICAgIGJvcmRlci1sZWZ0LWNvbG9yOiByZ2JhKDU3LCA3NiwgODksIDAuODUpO1xuICAgIGJvcmRlci13aWR0aDogOHB4O1xuICAgIG1hcmdpbi10b3A6IC04cHg7XG59XG4udmlzLWNvbmZpZ3VyYXRpb24tcG9wdXA6YmVmb3JlIHtcbiAgICBib3JkZXItY29sb3I6IHJnYmEoMTk0LCAyMjUsIDI0NSwgMCk7XG4gICAgYm9yZGVyLWxlZnQtY29sb3I6ICNmMmZhZmY7XG4gICAgYm9yZGVyLXdpZHRoOiAxMnB4O1xuICAgIG1hcmdpbi10b3A6IC0xMnB4O1xufSJdfQ== */');var Lu=function(){function t(e,i,n){var o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;ba(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},za.extend(this.options,this.defaultOptions),this.configureOptions=n,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new Cu(o),this.wrapper=void 0}return xa(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;if("string"==typeof t)this.options.filter=t;else if(t instanceof Array)this.options.filter=t.join();else if("object"===zo(t)){if(null==t)throw new TypeError("options cannot be null");void 0!==t.container&&(this.options.container=t.container),void 0!==En(t)&&(this.options.filter=En(t)),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)}else"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0);!1===En(this.options)&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,!0===this.options.enabled&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){this._clean(),this.changedOptions=[];var t=En(this.options),e=0,i=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,i=!1,"function"==typeof t?i=(i=t(n,[]))||this._handleObject(this.configureOptions[n],[n],!0):!0!==t&&-1===Fo(t).call(t,n)||(i=!0),!1!==i&&(this.allowCreation=!0,e>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),e++);this._makeButton(),this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){if(!0===this.allowCreation){var e=document.createElement("div");e.className="vis-configuration vis-config-item vis-config-s"+t.length;for(var i=arguments.length,n=new Array(i>1?i-1:0),o=1;o<i;o++)n[o-1]=arguments[o];return mn(n).call(n,(function(t){e.appendChild(t)})),this.domElements.push(e),this.domElements.length}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=document.createElement("div");return n.className="vis-configuration vis-config-label vis-config-s"+e.length,n.innerHTML=!0===i?"<i><b>"+t+":</b></i>":t+":",n}},{key:"_makeDropdown",value:function(t,e,i){var n=document.createElement("select");n.className="vis-configuration vis-config-select";var o=0;void 0!==e&&-1!==Fo(t).call(t,e)&&(o=Fo(t).call(t,e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===o&&(r.selected="selected"),r.innerHTML=t[s],n.appendChild(r)}var a=this;n.onchange=function(){a._update(this.value,i)};var l=this._makeLabel(i[i.length-1],i);this._makeItem(i,l,n)}},{key:"_makeRange",value:function(t,e,i){var n=t[0],o=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=o,a.max=s}catch(t){}a.step=r;var l="",h=0;if(void 0!==e){e<0&&1.2*e<o?(a.min=Math.ceil(1.2*e),h=a.min,l="range increased"):e/1.2<o&&(a.min=Math.ceil(e/1.2),h=a.min,l="range increased"),1.2*e>s&&1!==s&&(a.max=Math.ceil(1.2*e),h=a.max,l="range increased"),a.value=e}else a.value=n;var d=document.createElement("input");d.className="vis-configuration vis-config-rangeinput",d.value=Number(a.value);var u=this;a.onchange=function(){d.value=this.value,u._update(Number(this.value),i)},a.oninput=function(){d.value=this.value};var c=this._makeLabel(i[i.length-1],i),p=this._makeItem(i,c,a,d);""!==l&&this.popupHistory[p]!==h&&(this.popupHistory[p]=h,this._setupPopup(l,p))}},{key:"_makeButton",value:function(){var t=this;if(!0===this.options.showButton){var e=document.createElement("div");e.className="vis-configuration vis-config-button",e.innerHTML="generate options",e.onclick=function(){t._printOptions()},e.onmouseover=function(){e.className="vis-configuration vis-config-button hover"},e.onmouseout=function(){e.className="vis-configuration vis-config-button"},this.optionsContainer=document.createElement("div"),this.optionsContainer.className="vis-configuration vis-config-option-container",this.domElements.push(this.optionsContainer),this.domElements.push(e)}}},{key:"_setupPopup",value:function(t,e){var i=this;if(!0===this.initialized&&!0===this.allowCreation&&this.popupCounter<this.popupLimit){var n=document.createElement("div");n.id="vis-configuration-popup",n.className="vis-configuration-popup",n.innerHTML=t,n.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:n,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index].getBoundingClientRect();this.popupDiv.html.style.left=e.left+"px",this.popupDiv.html.style.top=e.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=qr((function(){t.popupDiv.html.style.opacity=0}),1500),this.popupDiv.deleteTimeout=qr((function(){t._removePopup()}),1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var n=document.createElement("input");n.type="checkbox",n.className="vis-configuration vis-config-checkbox",n.checked=t,void 0!==e&&(n.checked=e,e!==t&&("object"===zo(t)?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var o=this;n.onchange=function(){o._update(this.checked,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,n)}},{key:"_makeTextInput",value:function(t,e,i){var n=document.createElement("input");n.type="text",n.className="vis-configuration vis-config-text",n.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var o=this;n.onchange=function(){o._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,n)}},{key:"_makeColorField",value:function(t,e,i){var n=this,o=t[1],s=document.createElement("div");"none"!==(e=void 0===e?o:e)?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?o:e,s.onclick=function(){n._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var n=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback((function(t){var o="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=o,n._update(o,i)})),this.colorPicker.setCloseCallback((function(){e.onclick=function(){n._showColorPicker(t,e,i)}}))}},{key:"_handleObject",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=!1,o=En(this.options),s=!1;for(var r in t)if(t.hasOwnProperty(r)){n=!0;var a=t[r],l=za.copyAndExtendArray(e,r);if("function"==typeof o&&!1===(n=o(r,e))&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,n=this._handleObject(a,l,!0),this.allowCreation=!1===i),!1!==n){s=!0;var h=this._getValue(l);if(a instanceof Array)this._handleArray(a,h,l);else if("string"==typeof a)this._makeTextInput(a,h,l);else if("boolean"==typeof a)this._makeCheckbox(a,h,l);else if(a instanceof Object){var d=!0;if(-1!==Fo(e).call(e,"physics")&&this.moduleOptions.physics.solver!==r&&(d=!1),!0===d)if(void 0!==a.enabled){var u=za.copyAndExtendArray(l,"enabled"),c=this._getValue(u);if(!0===c){var p=this._makeLabel(r,l,!0);this._makeItem(l,p),s=this._handleObject(a,l)||s}else this._makeCheckbox(a,c,l)}else{var m=this._makeLabel(r,l,!0);this._makeItem(l,m),s=this._handleObject(a,l)||s}}else console.error("dont know how to handle",a,r,l)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},n=i;t="false"!==(t="true"===t||t)&&t;for(var o=0;o<e.length;o++)"global"!==e[o]&&(void 0===n[e[o]]&&(n[e[o]]={}),o!==e.length-1?n=n[e[o]]:n[e[o]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+Vs(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}(),Gu=function(t){function e(t,i,n,o){var s,r,a,l,h,d,u,c;if(ba(this,e),(c=ar(this,dr(e).call(this))).initTime=new Date,c.itemsDone=!1,!(Bs(c)instanceof e))throw new SyntaxError("Constructor must be called with the new operator");if(!(Zo(n)||n instanceof Aa||n instanceof Xa)&&n instanceof Object){var p=o;o=n,n=p}o&&o.throttleRedraw&&console.warn('Timeline option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var m=Bs(c);if(c.defaultOptions={autoResize:!0,longSelectPressTime:251,orientation:{axis:"bottom",item:"bottom"},moment:Ta},c.options=za.deepExtend({},c.defaultOptions),c._create(t),!o||o&&void 0===o.rtl){var f;c.dom.root.style.visibility="hidden";for(var g=c.dom.root;!f&&g;)f=window.getComputedStyle(g,null).direction,g=g.parentElement;c.options.rtl=f&&"rtl"==f.toLowerCase()}else c.options.rtl=o.rtl;o&&(o.rollingMode&&(c.options.rollingMode=o.rollingMode),o.onInitialDrawComplete&&(c.options.onInitialDrawComplete=o.onInitialDrawComplete),o.onTimeout&&(c.options.onTimeout=o.onTimeout),o.loadingScreenTemplate&&(c.options.loadingScreenTemplate=o.loadingScreenTemplate));var v=document.createElement("div");if(c.options.loadingScreenTemplate){var b,y=va(b=c.options.loadingScreenTemplate).call(b,Bs(c)),x=y(c.dom.loadingScreen);x instanceof Object&&!(x instanceof Element)?y(v):x instanceof Element?(v.innerHTML="",v.appendChild(x)):null!=x&&(v.innerHTML=x)}return c.dom.loadingScreen.appendChild(v),c.components=[],c.body={dom:c.dom,domProps:c.props,emitter:{on:va(s=c.on).call(s,Bs(c)),off:va(r=c.off).call(r,Bs(c)),emit:va(a=c.emit).call(a,Bs(c))},hiddenDates:[],util:{getScale:function(){return m.timeAxis.step.scale},getStep:function(){return m.timeAxis.step.step},toScreen:va(l=m._toScreen).call(l,m),toGlobalScreen:va(h=m._toGlobalScreen).call(h,m),toTime:va(d=m._toTime).call(d,m),toGlobalTime:va(u=m._toGlobalTime).call(u,m)}},c.range=new ml(c.body,c.options),c.components.push(c.range),c.body.range=c.range,c.timeAxis=new Jh(c.body,c.options),c.timeAxis2=null,c.components.push(c.timeAxis),c.currentTime=new pd(c.body,c.options),c.components.push(c.currentTime),c.itemSet=new uu(c.body,c.options),c.components.push(c.itemSet),c.itemsData=null,c.groupsData=null,c.dom.root.onclick=function(t){m.emit("click",m.getEventProperties(t))},c.dom.root.ondblclick=function(t){m.emit("doubleClick",m.getEventProperties(t))},c.dom.root.oncontextmenu=function(t){m.emit("contextmenu",m.getEventProperties(t))},c.dom.root.onmouseover=function(t){m.emit("mouseOver",m.getEventProperties(t))},window.PointerEvent?(c.dom.root.onpointerdown=function(t){m.emit("mouseDown",m.getEventProperties(t))},c.dom.root.onpointermove=function(t){m.emit("mouseMove",m.getEventProperties(t))},c.dom.root.onpointerup=function(t){m.emit("mouseUp",m.getEventProperties(t))}):(c.dom.root.onmousemove=function(t){m.emit("mouseMove",m.getEventProperties(t))},c.dom.root.onmousedown=function(t){m.emit("mouseDown",m.getEventProperties(t))},c.dom.root.onmouseup=function(t){m.emit("mouseUp",m.getEventProperties(t))}),c.initialFitDone=!1,c.on("changed",(function(){if(null!=m.itemsData){if(!m.initialFitDone&&!m.options.rollingMode)if(m.initialFitDone=!0,null!=m.options.start||null!=m.options.end){if(null==m.options.start||null==m.options.end)var t=m.getItemRange();var e=null!=m.options.start?m.options.start:t.min,i=null!=m.options.end?m.options.end:t.max;m.setWindow(e,i,{animation:!1})}else m.fit({animation:!1});m.initialDrawDone||!m.initialRangeChangeDone&&(m.options.start||m.options.end)&&!m.options.rollingMode||(m.initialDrawDone=!0,m.itemSet.initialDrawDone=!0,m.dom.root.style.visibility="visible",m.dom.loadingScreen.parentNode.removeChild(m.dom.loadingScreen),m.options.onInitialDrawComplete&&qr((function(){return m.options.onInitialDrawComplete()}),0))}})),c.on("destroyTimeline",(function(){m.destroy()})),o&&c.setOptions(o),c.body.emitter.on("fit",(function(t){c._onFit(t),c.redraw()})),n&&c.setGroups(n),i&&c.setItems(i),c._redraw(),c}return pr(e,t),xa(e,[{key:"_createConfigurator",value:function(){return new Lu(this,this.dom.container,wu)}},{key:"redraw",value:function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()}},{key:"setOptions",value:function(t){if(!0===fu.validate(t,xu)&&console.log("%cErrors have been found in the supplied options object.",mu),cd.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var e=this.itemsData;if(e){var i=this.getSelection();this.setItems(null),this.setItems(e.rawDS),this.setSelection(i)}}}},{key:"setItems",value:function(t){var e;this.itemsDone=!1,e=t?Ua(t instanceof Aa||t instanceof Xa?t:new Aa(t)):null,this.itemsData&&this.itemsData.dispose(),this.itemsData=e,this.itemSet&&this.itemSet.setItems(null!=e?e.rawDS:null)}},{key:"setGroups",value:function(t){var e;if(t){var i=function(t){return!1!==t.visible};e=t instanceof Aa||t instanceof Xa?new Xa(t,{filter:i}):new Aa(En(t).call(t,i))}else e=null;this.groupsData instanceof Xa&&this.groupsData.setData(null),this.groupsData=e,this.itemSet.setGroups(e)}},{key:"setData",value:function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)}},{key:"setSelection",value:function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)}},{key:"getSelection",value:function(){return this.itemSet&&this.itemSet.getSelection()||[]}},{key:"focus",value:function(t,e){if(this.itemsData&&null!=t){var i=Zo(t)?t:[t],n=this.itemsData.get(i),o=null,s=null;if(mn(n).call(n,(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===o||e<o)&&(o=e),(null===s||i>s)&&(s=i)})),null!==o&&null!==s){var r=this,a=this.itemSet.items[i[0]],l=-1*this._getScrollTop(),h=null,d=function(){var t=Qu(r,a);t.shouldScroll&&t.itemTop!=h.itemTop&&(r._setScrollTop(-t.scrollOffset),r._redraw())},u=!e||void 0===e.zoom||e.zoom,c=(o+s)/2,p=u?1.1*(s-o):Math.max(this.range.end-this.range.start,1.1*(s-o)),m=!e||void 0===e.animation||e.animation;m||(h={shouldScroll:!1,scrollOffset:-1,itemTop:-1}),this.range.setRange(c-p/2,c+p/2,{animation:m},(function(){d(),qr(d,100)}),(function(t,e,i){var n=Qu(r,a);if(!1!==n&&(h||(h=n),h.itemTop!=n.itemTop||h.shouldScroll)){h.itemTop!=n.itemTop&&n.shouldScroll&&(h=n,l=-1*r._getScrollTop());var o=l,s=h.scrollOffset,d=i?s:o+(s-o)*t;r._setScrollTop(-d),e||r._redraw()}}))}}}},{key:"fit",value:function(t,e){var i,n=!t||void 0===t.animation||t.animation;1===this.itemsData.length&&void 0===this.itemsData.get()[0].end?(i=this.getDataRange(),this.moveTo(i.min.valueOf(),{animation:n},e)):(i=this.getItemRange(),this.range.setRange(i.min,i.max,{animation:n},e))}},{key:"getItemRange",value:function(){var t=this,e=this.getDataRange(),i=null!==e.min?e.min.valueOf():null,n=null!==e.max?e.max.valueOf():null,o=null,s=null;if(null!=i&&null!=n){var r=n-i;r<=0&&(r=10);var a=r/this.props.center.width,l={},h=0;if(mn(za).call(za,this.itemSet.items,(function(t,e){if(t.groupShowing){l[e]=t.redraw(!0),h=l[e].length}})),h>0)for(var d=function(t){mn(za).call(za,l,(function(e){e[t]()}))},u=0;u<h;u++)d(u);if(mn(za).call(za,this.itemSet.items,(function(e){var r,l,h=Mu(e),d=Su(e);t.options.rtl?(r=h-(e.getWidthRight()+10)*a,l=d+(e.getWidthLeft()+10)*a):(r=h-(e.getWidthLeft()+10)*a,l=d+(e.getWidthRight()+10)*a),r<i&&(i=r,o=e),l>n&&(n=l,s=e)})),o&&s){var c=o.getWidthLeft()+10,p=s.getWidthRight()+10,m=this.props.center.width-c-p;m>0&&(this.options.rtl?(i=Mu(o)-p*r/m,n=Su(s)+c*r/m):(i=Mu(o)-c*r/m,n=Su(s)+p*r/m))}}return{min:null!=i?new Date(i):null,max:null!=n?new Date(n):null}}},{key:"getDataRange",value:function(){var t,e=null,i=null;this.itemsData&&mn(t=this.itemsData).call(t,(function(t){var n=za.convert(t.start,"Date").valueOf(),o=za.convert(null!=t.end?t.end:t.start,"Date").valueOf();(null===e||n<e)&&(e=n),(null===i||o>i)&&(i=o)}));return{min:null!=e?new Date(e):null,max:null!=i?new Date(i):null}}},{key:"getEventProperties",value:function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,n=this.dom.centerContainer.getBoundingClientRect(),o=this.options.rtl?n.right-e:e-n.left,s=i-n.top,r=this.itemSet.itemFromTarget(t),a=this.itemSet.groupFromTarget(t),l=ud.customTimeFromTarget(t),h=this.itemSet.options.snap||null,d=this.body.util.getScale(),u=this.body.util.getStep(),c=this._toTime(o),p=h?h(c,d,u):c,m=za.getTarget(t),f=null;return null!=r?f="item":null!=l?f="custom-time":za.hasParent(m,this.timeAxis.dom.foreground)||this.timeAxis2&&za.hasParent(m,this.timeAxis2.dom.foreground)?f="axis":za.hasParent(m,this.itemSet.dom.labelSet)?f="group-label":za.hasParent(m,this.currentTime.bar)?f="current-time":za.hasParent(m,this.dom.center)&&(f="background"),{event:t,item:r?r.id:null,isCluster:!!r&&!!r.isCluster,items:r?r.items||[]:null,group:a?a.groupId:null,customTime:l?l.options.id:null,what:f,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:s,time:c,snappedTime:p}}},{key:"toggleRollingMode",value:function(){this.range.rolling?this.range.stopRolling():(null==this.options.rollingMode&&this.setOptions(this.options),this.range.startRolling())}},{key:"_redraw",value:function(){cd.prototype._redraw.call(this)}},{key:"_onFit",value:function(t){var e=t.start,i=t.end,n=t.animation;i?this.range.setRange(e,i,{animation:n}):this.moveTo(e.valueOf(),{animation:n})}}]),e}(cd);function Mu(t){return za.convert(t.data.start,"Date").valueOf()}function Su(t){var e=null!=t.data.end?t.data.end:t.data.start;return za.convert(e,"Date").valueOf()}function Qu(t,e){if(!e.parent)return!1;var i=t.options.rtl?t.props.rightContainer.height:t.props.leftContainer.height,n=t.props.center.height,o=e.parent,s=o.top,r=!0,a=t.timeAxis.options.orientation.axis,l=function(){return"bottom"==a?o.height-e.top-e.height:e.top},h=-1*t._getScrollTop(),d=s+l(),u=e.height;return d<h?s+i<=s+l()+u&&(s+=l()-t.itemSet.options.margin.item.vertical):d+u>h+i?s+=l()+u-i+t.itemSet.options.margin.item.vertical:r=!1,{shouldScroll:r,scrollOffset:s=Math.min(s,n-i),itemTop:d}}var Nu=function(){function t(e,i,n,o,s,r){var a=arguments.length>6&&void 0!==arguments[6]&&arguments[6],l=arguments.length>7&&void 0!==arguments[7]&&arguments[7];if(ba(this,t),this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=s,this.majorCharHeight=r,this._start=e,this._end=i,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=a,this.autoScaleStart=n,this.autoScaleEnd=o,this.formattingFunction=l,n||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>h.magnitudefactor*h.minorSteps[h.minorStepIdx]*.5?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};n&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}return xa(t,[{key:"setCharHeight",value:function(t){this.majorCharHeight=t}},{key:"setHeight",value:function(t){this.containerHeight=t}},{key:"determineScale",value:function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var n=0;i<0&&(n=i);for(var o=!1,s=n;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){if(this.magnitudefactor*this.minorSteps[r]>=e){o=!0,this.minorStepIdx=r;break}}if(!0===o)break}}},{key:"is_major",value:function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])==0}},{key:"getStep",value:function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]}},{key:"getFirstMajor",value:function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)}},{key:"formatValue",value:function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?"".concat(e):"string"==typeof e?e:t.toPrecision(5)}},{key:"getLines",value:function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,n=this._start+i;this._end-n>1e-5;n+=e)n!=this._start&&t.push({major:this.is_major(n),y:this.convertValue(n),val:this.formatValue(n)});return t}},{key:"followScale",value:function(t){var e=this.minorStepIdx,i=this._start,n=this._end,o=this,s=function(){o.magnitudefactor*=2},r=function(){o.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e||s(),s()):(this.minorStepIdx=2,1==e||r(),r()));for(var a=t.convertValue(0),l=t.getStep()*t.scale,h=!1,d=0;!h&&d++<5;){this.scale=l/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var u=this.containerHeight/this.scale;this._start=i,this._end=this._start+u;var c=this._end*this.scale,p=this.magnitudefactor*this.majorSteps[this.minorStepIdx],m=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var f=a-c;this._end+=f/this.scale,this._start=this._end-u}else this.autoScaleStart?(this._start-=m/this.scale,this._end=this._start+u):(this._start+=p-m/this.scale,this._end=this._start+u);if(!this.autoScaleEnd&&this._end>n+1e-5)r(),h=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),h=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&u<n-i?(s(),h=!1):h=!0}}}},{key:"convertValue",value:function(t){return this.containerHeight-(t-this._start)*this.scale}},{key:"screenToValue",value:function(t){return(this.containerHeight-t)/this.scale+this._start}}]),t}();Ph(".vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px hsla(0,0%,60.4%,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImRhdGFheGlzLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxrRUFDRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLFFBQVMsQ0FDVCx1QkFDRixDQUVBLDZEQUNFLG9CQUNGLENBRUEsNkRBQ0Usb0JBQ0YsQ0FHQSxxQ0FDRSxVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGFBQWMsQ0FDZCxrQkFDRixDQUVBLGlEQUNFLFNBQVUsQ0FDVixRQUFTLENBQ1QsUUFBUyxDQUNULGlCQUFrQixDQUNsQixVQUNGLENBR0EscUNBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxhQUFjLENBQ2Qsa0JBQ0YsQ0FFQSxpREFDRSxTQUFVLENBQ1YsUUFBUyxDQUNULFFBQVMsQ0FDVCxpQkFBa0IsQ0FDbEIsVUFDRixDQUVBLHFDQUNFLGlCQUFrQixDQUNsQixhQUFjLENBQ2Qsa0JBQW1CLENBQ25CLFdBQVksQ0FDWixpQkFDRixDQUVBLGlEQUNFLFNBQVUsQ0FDVixRQUFTLENBQ1QsaUJBQWtCLENBQ2xCLFVBQ0YsQ0FFQSw4Q0FDRSxRQUFTLENBQ1QsaUNBQWtDLENBQ2xDLDhCQUErQixDQUMvQiw2QkFBOEIsQ0FDOUIsNEJBQTZCLENBQzdCLDRCQUE2QixDQUM3QixnQ0FBaUMsQ0FDakMsNkJBQThCLENBQzlCLDRCQUE2QixDQUM3QiwyQkFBNEIsQ0FDNUIsd0JBQ0YsQ0FFQSwrQ0FDRSxRQUFTLENBQ1QscUNBQXNDLENBQ3RDLGtDQUFtQyxDQUNuQyxpQ0FBa0MsQ0FDbEMsZ0NBQWlDLENBQ2pDLDZCQUE4QixDQUM5QiwrQkFBZ0MsQ0FDaEMsNEJBQTZCLENBQzdCLDJCQUE0QixDQUM1QiwwQkFBMkIsQ0FDM0IsdUJBQ0YsQ0FFQSxZQUNFLHNDQUEyQyxDQUMzQyxXQUFZLENBQ1osd0JBQXlCLENBQ3pCLDRDQUNGLENBRUEsaUJBRUUsa0JBQW1CLENBQ25CLG9CQUNGIiwiZmlsZSI6ImRhdGFheGlzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy1wYW5lbC52aXMtYmFja2dyb3VuZC52aXMtaG9yaXpvbnRhbCAudmlzLWdyaWQudmlzLWhvcml6b250YWwge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDA7XG4gIGJvcmRlci1ib3R0b206IDFweCBzb2xpZDtcbn1cblxuLnZpcy1wYW5lbC52aXMtYmFja2dyb3VuZC52aXMtaG9yaXpvbnRhbCAudmlzLWdyaWQudmlzLW1pbm9yIHtcbiAgYm9yZGVyLWNvbG9yOiAjZTVlNWU1O1xufVxuXG4udmlzLXBhbmVsLnZpcy1iYWNrZ3JvdW5kLnZpcy1ob3Jpem9udGFsIC52aXMtZ3JpZC52aXMtbWFqb3Ige1xuICBib3JkZXItY29sb3I6ICNiZmJmYmY7XG59XG5cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLW1ham9yIHtcbiAgd2lkdGg6IDEwMCU7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICM0ZDRkNGQ7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG59XG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy1tYWpvci52aXMtbWVhc3VyZSB7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgYm9yZGVyOiAwO1xuICB2aXNpYmlsaXR5OiBoaWRkZW47XG4gIHdpZHRoOiBhdXRvO1xufVxuXG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy1taW5vciB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgd2lkdGg6IDEwMCU7XG4gIGNvbG9yOiAjYmViZWJlO1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLWRhdGEtYXhpcyAudmlzLXktYXhpcy52aXMtbWlub3IudmlzLW1lYXN1cmUge1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG4gIGJvcmRlcjogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICB3aWR0aDogYXV0bztcbn1cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLXRpdGxlIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICBjb2xvcjogIzRkNGQ0ZDtcbiAgd2hpdGUtc3BhY2U6IG5vd3JhcDtcbiAgYm90dG9tOiAyMHB4O1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG59XG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy10aXRsZS52aXMtbWVhc3VyZSB7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICB3aWR0aDogYXV0bztcbn1cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLXRpdGxlLnZpcy1sZWZ0IHtcbiAgYm90dG9tOiAwO1xuICAtd2Via2l0LXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICAtbW96LXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICAtbXMtdHJhbnNmb3JtLW9yaWdpbjogbGVmdCB0b3A7XG4gIC1vLXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICB0cmFuc2Zvcm0tb3JpZ2luOiBsZWZ0IGJvdHRvbTtcbiAgLXdlYmtpdC10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICAtbW96LXRyYW5zZm9ybTogcm90YXRlKC05MGRlZyk7XG4gIC1tcy10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICAtby10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICB0cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xufVxuXG4udmlzLWRhdGEtYXhpcyAudmlzLXktYXhpcy52aXMtdGl0bGUudmlzLXJpZ2h0IHtcbiAgYm90dG9tOiAwO1xuICAtd2Via2l0LXRyYW5zZm9ybS1vcmlnaW46IHJpZ2h0IGJvdHRvbTtcbiAgLW1vei10cmFuc2Zvcm0tb3JpZ2luOiByaWdodCBib3R0b207XG4gIC1tcy10cmFuc2Zvcm0tb3JpZ2luOiByaWdodCBib3R0b207XG4gIC1vLXRyYW5zZm9ybS1vcmlnaW46IHJpZ2h0IGJvdHRvbTtcbiAgdHJhbnNmb3JtLW9yaWdpbjogcmlnaHQgYm90dG9tO1xuICAtd2Via2l0LXRyYW5zZm9ybTogcm90YXRlKDkwZGVnKTtcbiAgLW1vei10cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7XG4gIC1tcy10cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7XG4gIC1vLXRyYW5zZm9ybTogcm90YXRlKDkwZGVnKTtcbiAgdHJhbnNmb3JtOiByb3RhdGUoOTBkZWcpO1xufVxuXG4udmlzLWxlZ2VuZCB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMjQ3LCAyNTIsIDI1NSwgMC42NSk7XG4gIHBhZGRpbmc6IDVweDtcbiAgYm9yZGVyOiAxcHggc29saWQgI2IzYjNiMztcbiAgYm94LXNoYWRvdzogMnB4IDJweCAxMHB4IHJnYmEoMTU0LCAxNTQsIDE1NCwgMC41NSk7XG59XG5cbi52aXMtbGVnZW5kLXRleHQge1xuICAvKmZvbnQtc2l6ZTogMTBweDsqL1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2tcbn0iXX0= */");var Fu=function(t){function e(t,i,n,o){var s;ba(this,e),(s=ar(this,dr(e).call(this))).id=Id(),s.body=t,s.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return"".concat(Pa(t.toPrecision(3)))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return"".concat(Pa(t.toPrecision(3)))},title:{text:void 0,style:void 0}}},s.linegraphOptions=o,s.linegraphSVG=n,s.props={},s.DOMelements={lines:{},labels:{},title:{}},s.dom={},s.scale=void 0,s.range={start:0,end:0},s.options=za.extend({},s.defaultOptions),s.conversionFactor=1,s.setOptions(i),s.width=Number("".concat(s.options.width).replace("px","")),s.minWidth=s.width,s.height=s.linegraphSVG.getBoundingClientRect().height,s.hidden=!1,s.stepPixels=25,s.zeroCrossing=-1,s.amountOfSteps=-1,s.lineOffset=0,s.master=!0,s.masterAxis=null,s.svgElements={},s.iconsRemoved=!1,s.groups={},s.amountOfGroups=0,s._create(),null==s.scale&&s._redrawLabels(),s.framework={svg:s.svg,svgElements:s.svgElements,options:s.options,groups:s.groups};var r=Bs(s);return s.body.emitter.on("verticalDrag",(function(){r.dom.lineContainer.style.top="".concat(r.body.domProps.scrollTop,"px")})),s}return pr(e,t),xa(e,[{key:"addGroup",value:function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1}},{key:"updateGroup",value:function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e}},{key:"removeGroup",value:function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)}},{key:"setOptions",value:function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);za.selectiveDeepExtend(["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"],this.options,t),this.minWidth=Number("".concat(this.options.width).replace("px","")),!0===e&&this.dom.frame&&(this.hide(),this.show())}}},{key:"_create",value:function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.dom.lineContainer.style.visibility="visible",this.dom.lineContainer.style.display="block",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)}},{key:"_redrawGroupIcons",value:function(){var t;Cs(this.svgElements);var e=this.options.iconWidth,i=11.5;t="left"===this.options.orientation?4:this.width-e-4;var n=Yi(this.groups);wr(n).call(n,(function(t,e){return t<e?-1:1}));var o=!0,s=!1,r=void 0;try{for(var a,l=Fi(n);!(o=(a=l.next()).done);o=!0){var h=a.value;!0!==this.groups[h].visible||void 0!==this.linegraphOptions.visibility[h]&&!0!==this.linegraphOptions.visibility[h]||(this.groups[h].getLegend(e,15,this.framework,t,i),i+=19)}}catch(t){s=!0,r=t}finally{try{o||null==l.return||l.return()}finally{if(s)throw r}}Ls(this.svgElements),this.iconsRemoved=!1}},{key:"_cleanupIcons",value:function(){!1===this.iconsRemoved&&(Cs(this.svgElements),Ls(this.svgElements),this.iconsRemoved=!0)}},{key:"show",value:function(){this.hidden=!1,this.dom.frame.parentNode||("left"===this.options.orientation?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.right.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer),this.dom.lineContainer.style.display="block"}},{key:"hide",value:function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.style.display="none"}},{key:"setRange",value:function(t,e){this.range.start=t,this.range.end=e}},{key:"redraw",value:function(){var t=!1,e=0;for(var i in this.dom.lineContainer.style.top="".concat(this.body.domProps.scrollTop,"px"),this.groups)this.groups.hasOwnProperty(i)&&(!0!==this.groups[i].visible||void 0!==this.linegraphOptions.visibility[i]&&!0!==this.linegraphOptions.visibility[i]||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height="".concat(this.height,"px"),this.width=!0===this.options.visible?Number("".concat(this.options.width).replace("px","")):0;var n=this.props,o=this.dom.frame;o.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels,l=this.body.dom.backgroundHorizontal.offsetWidth;n.minorLabelHeight=r?n.minorCharHeight:0,n.majorLabelHeight=a?n.majorCharHeight:0,n.minorLineWidth=l-this.lineOffset-this.width+2*this.options.minorLinesOffset,n.minorLineHeight=1,n.majorLineWidth=l-this.lineOffset-this.width+2*this.options.majorLinesOffset,n.majorLineHeight=1,"left"===s?(o.style.top="0",o.style.left="0",o.style.bottom="",o.style.width="".concat(this.width,"px"),o.style.height="".concat(this.height,"px"),this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(o.style.top="",o.style.bottom="0",o.style.left="0",o.style.width="".concat(this.width,"px"),o.style.height="".concat(this.height,"px"),this.props.width=this.body.domProps.right.width,this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,!0===this.options.icons?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t}},{key:"_redrawLabels",value:function(){var t=this,e=!1;Cs(this.DOMelements.lines),Cs(this.DOMelements.labels);var i=this.options.orientation,n=null!=this.options[i].range?this.options[i].range:{},o=!0;null!=n.max&&(this.range.end=n.max,o=!1);var s=!0;null!=n.min&&(this.range.start=n.min,s=!1),this.scale=new Nu(this.range.start,this.range.end,s,o,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),!1===this.master&&null!=this.masterAxis?(this.scale.followScale(this.masterAxis.scale),this.dom.lineContainer.style.display="none"):this.dom.lineContainer.style.display="block",this.maxLabelSize=0;var r=this.scale.getLines();mn(r).call(r,(function(e){var n=e.y,o=e.major;t.options.showMinorLabels&&!1===o&&t._redrawLabel(n-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),o&&n>=0&&t._redrawLabel(n-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),!0===t.master&&(o?t._redrawLine(n,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(n,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))}));var a=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(a=this.props.titleCharHeight);var l=!0===this.options.icons?Math.max(this.options.iconWidth,a)+this.options.labelOffsetX+15:a+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-l&&!0===this.options.visible?(this.width=this.maxLabelSize+l,this.options.width="".concat(this.width,"px"),Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-l&&!0===this.options.visible&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+l),this.options.width="".concat(this.width,"px"),Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),this.redraw(),e=!0):(Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),e=!1),e}},{key:"convertValue",value:function(t){return this.scale.convertValue(t)}},{key:"screenToValue",value:function(t){return this.scale.screenToValue(t)}},{key:"_redrawLabel",value:function(t,e,i,n,o){var s=Ss("div",this.DOMelements.labels,this.dom.frame);s.className=n,s.innerHTML=e,"left"===i?(s.style.left="-".concat(this.options.labelOffsetX,"px"),s.style.textAlign="right"):(s.style.right="-".concat(this.options.labelOffsetX,"px"),s.style.textAlign="left"),s.style.top="".concat(t-.5*o+this.options.labelOffsetY,"px"),e+="";var r=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*r&&(this.maxLabelSize=e.length*r)}},{key:"_redrawLine",value:function(t,e,i,n,o){if(!0===this.master){var s=Ss("div",this.DOMelements.lines,this.dom.lineContainer);s.className=i,s.innerHTML="","left"===e?s.style.left="".concat(this.width-n,"px"):s.style.right="".concat(this.width-n,"px"),s.style.width="".concat(o,"px"),s.style.top="".concat(t,"px")}}},{key:"_redrawTitle",value:function(t){if(Cs(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=Ss("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-".concat(t),e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&za.addCssText(e,this.options[t].title.style),"left"===t?e.style.left="".concat(this.props.titleCharHeight,"px"):e.style.right="".concat(this.props.titleCharHeight,"px"),e.style.width="".concat(this.height,"px")}Ls(this.DOMelements.title)}},{key:"_calculateCharSize",value:function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),n=document.createElement("div");n.className="vis-y-axis vis-major vis-measure",n.appendChild(i),this.dom.frame.appendChild(n),this.props.majorCharHeight=n.clientHeight,this.props.majorCharWidth=n.clientWidth,this.dom.frame.removeChild(n)}if(!("titleCharHeight"in this.props)){var o=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(o),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}}}]),e}(Ja);function Zu(t,e){}function Bu(t,e){return{style:(e=void 0===e?{}:e).style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function Ou(t,e){}function Wu(t,e){}function Au(t,e,i,n){this.id=e;this.options=za.selectiveBridgeObject(["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"],i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=n,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible||t.visible}function Xu(t,e,i,n){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=za.extend({},this.defaultOptions),this.linegraphOptions=n,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}Zu.draw=function(t,e,i,n){n=n||0;for(var o=function(t,e){var i=void 0;t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender);e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender);return i}(i,e),s=0;s<t.length;s++)if(o){var r=o(t[s],e);!0!==r&&"object"!==zo(r)||Qs(t[s].screen_x+n,t[s].screen_y,Bu(e,r),i.svgElements,i.svg,t[s].label)}else Qs(t[s].screen_x+n,t[s].screen_y,Bu(e),i.svgElements,i.svg,t[s].label)},Zu.drawIcon=function(t,e,i,n,o,s){var r=.5*o,a=Ms("rect",s.svgElements,s.svg);a.setAttributeNS(null,"x",e),a.setAttributeNS(null,"y",i-r),a.setAttributeNS(null,"width",n),a.setAttributeNS(null,"height",2*r),a.setAttributeNS(null,"class","vis-outline"),Qs(e+.5*n,i,Bu(t),s.svgElements,s.svg)},Ou.drawIcon=function(t,e,i,n,o,s){var r=.5*o,a=Ms("rect",s.svgElements,s.svg);a.setAttributeNS(null,"x",e),a.setAttributeNS(null,"y",i-r),a.setAttributeNS(null,"width",n),a.setAttributeNS(null,"height",2*r),a.setAttributeNS(null,"class","vis-outline");var l=Math.round(.3*n),h=t.options.barChart.width/l,d=Math.round(.4*o),u=Math.round(.75*o),c=Math.round((n-2*l)/3);if(Ns(e+.5*l+c,i+r-d-1,l,d,t.className+" vis-bar",s.svgElements,s.svg,t.style),Ns(e+1.5*l+c+2,i+r-u-1,l,u,t.className+" vis-bar",s.svgElements,s.svg,t.style),1==t.options.drawPoints.enabled){var p={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/h,className:t.className};Qs(e+.5*l+c,i+r-d-1,p,s.svgElements,s.svg),Qs(e+1.5*l+c+2,i+r-u-1,p,s.svgElements,s.svg)}},Ou.draw=function(t,e,i){var n,o,s,r,a,l,h=[],d={},u=0;for(a=0;a<t.length;a++)if("bar"===(r=i.groups[t[a]]).options.style&&!0===r.visible&&(void 0===i.options.groups.visibility[t[a]]||!0===i.options.groups.visibility[t[a]]))for(l=0;l<e[t[a]].length;l++)h.push({screen_x:e[t[a]][l].screen_x,screen_end:e[t[a]][l].screen_end,screen_y:e[t[a]][l].screen_y,x:e[t[a]][l].x,end:e[t[a]][l].end,y:e[t[a]][l].y,groupId:t[a],label:e[t[a]][l].label}),u+=1;if(0!==u)for(wr(h).call(h,(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x})),Ou._getDataIntersections(d,h),a=0;a<h.length;a++){var c=null!=(r=i.groups[h[a].groupId]).options.barChart.minWidth?r.options.barChart.minWidth:.1*r.options.barChart.width,p=0;if(void 0===d[o=h[a].screen_x])a+1<h.length&&(n=Math.abs(h[a+1].screen_x-o)),s=Ou._getSafeDrawData(n,r,c);else{var m=a+(d[o].amount-d[o].resolved);m<h.length&&(n=Math.abs(h[m].screen_x-o)),s=Ou._getSafeDrawData(n,r,c),d[o].resolved+=1,!0===r.options.stack&&!0!==r.options.excludeFromStacking?h[a].screen_y<r.zeroPosition?(p=d[o].accumulatedNegative,d[o].accumulatedNegative+=r.zeroPosition-h[a].screen_y):(p=d[o].accumulatedPositive,d[o].accumulatedPositive+=r.zeroPosition-h[a].screen_y):!0===r.options.barChart.sideBySide&&(s.width=s.width/d[o].amount,s.offset+=d[o].resolved*s.width-.5*s.width*(d[o].amount+1))}var f=s.width,g=h[a].screen_x;if(null!=h[a].screen_end?g+=.5*(f=h[a].screen_end-h[a].screen_x):g+=s.offset,Ns(g,h[a].screen_y-p,f,r.zeroPosition-h[a].screen_y,r.className+" vis-bar",i.svgElements,i.svg,r.style),!0===r.options.drawPoints.enabled){var v={screen_x:h[a].screen_x,screen_y:h[a].screen_y-p,x:h[a].x,y:h[a].y,groupId:h[a].groupId,label:h[a].label};Zu.draw([v],r,i,s.offset)}}},Ou._getDataIntersections=function(t,e){for(var i,n=0;n<e.length;n++)n+1<e.length&&(i=Math.abs(e[n+1].screen_x-e[n].screen_x)),n>0&&(i=Math.min(i,Math.abs(e[n-1].screen_x-e[n].screen_x))),0===i&&(void 0===t[e[n].screen_x]&&(t[e[n].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[n].screen_x].amount+=1)},Ou._getSafeDrawData=function(t,e,i){var n,o;return t<e.options.barChart.width&&t>0?(n=t<i?i:t,o=0,"left"===e.options.barChart.align?o-=.5*t:"right"===e.options.barChart.align&&(o+=.5*t)):(n=e.options.barChart.width,o=0,"left"===e.options.barChart.align?o-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(o+=.5*e.options.barChart.width)),{width:n,offset:o}},Ou.getStackedYRange=function(t,e,i,n,o){if(t.length>0){wr(t).call(t,(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}));var s={};Ou._getDataIntersections(s,t),e[n]=Ou._getStackedYRange(s,t),e[n].yAxisOrientation=o,i.push(n)}},Ou._getStackedYRange=function(t,e){for(var i,n=e[0].screen_y,o=e[0].screen_y,s=0;s<e.length;s++)void 0===t[i=e[s].screen_x]?(n=n>e[s].screen_y?e[s].screen_y:n,o=o<e[s].screen_y?e[s].screen_y:o):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(n=(n=n>t[r].accumulatedNegative?t[r].accumulatedNegative:n)>t[r].accumulatedPositive?t[r].accumulatedPositive:n,o=(o=o<t[r].accumulatedNegative?t[r].accumulatedNegative:o)<t[r].accumulatedPositive?t[r].accumulatedPositive:o);return{min:n,max:o}},Wu.calcPath=function(t,e){if(null!=t&&t.length>0){return 1==e.options.interpolation.enabled?Wu._catmullRom(t,e):Wu._linear(t)}},Wu.drawIcon=function(t,e,i,n,o,s){var r,a,l=.5*o,h=Ms("rect",s.svgElements,s.svg);(h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-l),h.setAttributeNS(null,"width",n),h.setAttributeNS(null,"height",2*l),h.setAttributeNS(null,"class","vis-outline"),(r=Ms("path",s.svgElements,s.svg)).setAttributeNS(null,"class",t.className),void 0!==t.style&&r.setAttributeNS(null,"style",t.style),r.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+n)+","+i),1==t.options.shaded.enabled&&(a=Ms("path",s.svgElements,s.svg),"top"==t.options.shaded.orientation?a.setAttributeNS(null,"d","M"+e+", "+(i-l)+"L"+e+","+i+" L"+(e+n)+","+i+" L"+(e+n)+","+(i-l)):a.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+l)+" L"+(e+n)+","+(i+l)+"L"+(e+n)+","+i),a.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&a.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled)&&Qs(e+.5*n,i,{style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className},s.svgElements,s.svg)},Wu.drawShading=function(t,e,i,n){if(1==e.options.shaded.enabled){var o,s=Number(n.svg.style.height.replace("px","")),r=Ms("path",n.svgElements,n.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var l=0;l="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),o="group"==e.options.shaded.orientation&&null!=i&&null!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+l+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",o)}},Wu.draw=function(t,e,i){if(null!=t&&null!=t){var n=Ms("path",i.svgElements,i.svg);n.setAttributeNS(null,"class",e.className),void 0!==e.style&&n.setAttributeNS(null,"style",e.style);var o="L";1==e.options.interpolation.enabled&&(o="C"),n.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,o,!1))}},Wu.serializePath=function(t,e,i){if(t.length<2)return"";var n,o=e;if(i)for(n=t.length-2;n>0;n--)o+=t[n][0]+","+t[n][1]+" ";else for(n=1;n<t.length;n++)o+=t[n][0]+","+t[n][1]+" ";return o},Wu._catmullRomUniform=function(t){var e,i,n,o,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var l=t.length,h=0;h<l-1;h++)e=0==h?t[0]:t[h-1],i=t[h],n=t[h+1],o=h+2<l?t[h+2]:n,s={screen_x:(-e.screen_x+6*i.screen_x+n.screen_x)*(1/6),screen_y:(-e.screen_y+6*i.screen_y+n.screen_y)*(1/6)},r={screen_x:(i.screen_x+6*n.screen_x-o.screen_x)*(1/6),screen_y:(i.screen_y+6*n.screen_y-o.screen_y)*(1/6)},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([n.screen_x,n.screen_y]);return a},Wu._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var n,o,s,r,a,l,h,d,u,c,p,m,f,g,v,b,y,x,w,_=[];_.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var k=t.length,D=0;D<k-1;D++)n=0==D?t[0]:t[D-1],o=t[D],s=t[D+1],r=D+2<k?t[D+2]:s,h=Math.sqrt(Math.pow(n.screen_x-o.screen_x,2)+Math.pow(n.screen_y-o.screen_y,2)),d=Math.sqrt(Math.pow(o.screen_x-s.screen_x,2)+Math.pow(o.screen_y-s.screen_y,2)),u=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),g=Math.pow(u,i),b=Math.pow(u,2*i),v=Math.pow(d,i),y=Math.pow(d,2*i),w=Math.pow(h,i),c=2*(x=Math.pow(h,2*i))+3*w*v+y,p=2*b+3*g*v+y,(m=3*w*(w+v))>0&&(m=1/m),(f=3*g*(g+v))>0&&(f=1/f),a={screen_x:(-y*n.screen_x+c*o.screen_x+x*s.screen_x)*m,screen_y:(-y*n.screen_y+c*o.screen_y+x*s.screen_y)*m},l={screen_x:(b*o.screen_x+p*s.screen_x-y*r.screen_x)*f,screen_y:(b*o.screen_y+p*s.screen_y-y*r.screen_y)*f},0==a.screen_x&&0==a.screen_y&&(a=o),0==l.screen_x&&0==l.screen_y&&(l=s),_.push([a.screen_x,a.screen_y]),_.push([l.screen_x,l.screen_y]),_.push([s.screen_x,s.screen_y]);return _},Wu._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},Au.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==wr(this.options)&&za.insertSort(this.itemsData,(function(t,e){return t.x>e.x?1:-1}))):this.itemsData=[]},Au.prototype.getItems=function(){return this.itemsData},Au.prototype.setZeroPosition=function(t){this.zeroPosition=t},Au.prototype.setOptions=function(t){if(void 0!==t){za.selectiveDeepExtend(["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"],this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),za.mergeOptions(this.options,t,"interpolation"),za.mergeOptions(this.options,t,"drawPoints"),za.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==zo(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},Au.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible||t.visible,this.style=t.style,this.setOptions(t.options)},Au.prototype.getLegend=function(t,e,i,n,o){null!=i&&null!=i||(i={svg:document.createElementNS("http://www.w3.org/2000/svg","svg"),svgElements:{},options:this.options,groups:[this]});switch(null!=n&&null!=n||(n=0),null!=o&&null!=o||(o=.5*e),this.options.style){case"line":Wu.drawIcon(this,n,o,t,e,i);break;case"points":case"point":Zu.drawIcon(this,n,o,t,e,i);break;case"bar":Ou.drawIcon(this,n,o,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},Au.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,n=0;n<t.length;n++)e=e>t[n].y?t[n].y:e,i=i<t[n].y?t[n].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},Xu.prototype=new Ja,Xu.prototype.clear=function(){this.groups={},this.amountOfGroups=0},Xu.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},Xu.prototype.updateGroup=function(t,e){this.groups[t]=e},Xu.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},Xu.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},Xu.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},Xu.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},Xu.prototype.setOptions=function(t){za.selectiveDeepExtend(["enabled","orientation","icons","left","right"],this.options,t)},Xu.prototype.redraw=function(){var t=0,e=Yi(this.groups);wr(e).call(e,(function(t,e){return t<e?-1:1}));for(var i=0;i<e.length;i++){var n=e[i];1!=this.groups[n].visible||void 0!==this.linegraphOptions.visibility[n]&&1!=this.linegraphOptions.visibility[n]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var o=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+o+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());var s="";for(i=0;i<e.length;i++)n=e[i],1!=this.groups[n].visible||void 0!==this.linegraphOptions.visibility[n]&&1!=this.linegraphOptions.visibility[n]||(s+=this.groups[n].content+"<br />");this.dom.textArea.innerHTML=s,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},Xu.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=Yi(this.groups);wr(t).call(t,(function(t,e){return t<e?-1:1})),Gs(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),n=i,o=this.options.iconSize,s=.75*this.options.iconSize,r=i+.5*s+3;this.svg.style.width=o+5+i+"px";for(var a=0;a<t.length;a++){var l=t[a];1!=this.groups[l].visible||void 0!==this.linegraphOptions.visibility[l]&&1!=this.linegraphOptions.visibility[l]||(this.groups[l].getLegend(o,s,this.framework,n,r),r+=s+this.options.iconSpacing)}}};function Tu(t,e){this.id=Id(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=za.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,n){i._onAdd(e.items)},update:function(t,e,n){i._onUpdate(e.items)},remove:function(t,e,n){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,n){i._onAddGroups(e.items)},update:function(t,e,n){i._onUpdateGroups(e.items)},remove:function(t,e,n){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",(function(){i.lastStart=i.body.range.start,i.svg.style.left=za.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)})),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}Tu.prototype=new Ja,Tu.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new Fu(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new Fu(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new Xu(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new Xu(this.body,this.options.legend,"right",this.options.groups),this.show()},Tu.prototype.setOptions=function(t){if(t){void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&Nn((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),za.selectiveDeepExtend(["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"],this.options,t),za.mergeOptions(this.options,t,"interpolation"),za.mergeOptions(this.options,t,"drawPoints"),za.mergeOptions(this.options,t,"shaded"),za.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==zo(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty("__ungrouped__")&&this.groups.__ungrouped__.setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},Tu.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},Tu.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},Tu.prototype.setItems=function(t){var e,i=this,n=this.itemsData;if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=Ua(t)}else this.itemsData=null;if(n&&(mn(za).call(za,this.itemListeners,(function(t,e){n.off(e,t)})),n.dispose(),e=n.getIds(),this._onRemove(e)),this.itemsData){var o=this.id;mn(za).call(za,this.itemListeners,(function(t,e){i.itemsData.on(e,t,o)})),e=this.itemsData.getIds(),this._onAdd(e)}},Tu.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.off(e,t)})),e=this.groupsData.getIds(),this.groupsData=null;for(var n=0;n<e.length;n++)this._removeGroup(e[n])}if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.id;mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.on(e,t,o)})),e=this.groupsData.getIds(),this._onAddGroups(e)}},Tu.prototype._onUpdate=function(t){this._updateAllGroupData(t)},Tu.prototype._onAdd=function(t){this._onUpdate(t)},Tu.prototype._onRemove=function(t){this._onUpdate(t)},Tu.prototype._onUpdateGroups=function(t){this._updateAllGroupData(null,t)},Tu.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},Tu.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},Tu.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},Tu.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new Au(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},Tu.prototype._updateAllGroupData=function(t,e){if(null!=this.itemsData){var i={},n=this.itemsData.get(),o=this.itemsData.idProp,s={};t&&Ln(t).call(t,(function(t){s[t]=t}));for(var r={},a=0;a<n.length;a++){var l=n[a],h=l.group;null==h&&(h="__ungrouped__"),r.hasOwnProperty(h)?r[h]++:r[h]=1}var d={};if(!e&&t)for(h in this.groups)if(this.groups.hasOwnProperty(h)){var u=(f=this.groups[h]).getItems();i[h]=En(u).call(u,(function(t){return d[t[o]]=t[o],t[o]!==s[t[o]]}));var c=r[h];r[h]-=i[h].length,i[h].length<c&&(i[h][c-1]={})}for(a=0;a<n.length;a++)if(null==(h=(l=n[a]).group)&&(h="__ungrouped__"),e||!t||l[o]===s[l[o]]||!d.hasOwnProperty(l[o])){i.hasOwnProperty(h)||(i[h]=new Array(r[h]));var p=za.bridgeObject(l);p.x=za.convert(l.x,"Date"),p.end=za.convert(l.end,"Date"),p.orginalY=l.y,p.y=Number(l.y),p[o]=l[o];var m=i[h].length-r[h]--;i[h][m]=p}for(h in this.groups)this.groups.hasOwnProperty(h)&&(i.hasOwnProperty(h)||(i[h]=new Array(0)));for(h in i)if(i.hasOwnProperty(h))if(0==i[h].length)this.groups.hasOwnProperty(h)&&this._removeGroup(h);else{var f=void 0;null!=this.groupsData&&(f=this.groupsData.get(h)),null==f&&(f={id:h,content:this.options.defaultGroup+h}),this._updateGroup(f,h),this.groups[h].setItems(i[h])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},Tu.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e,i=this.body.range.end-this.body.range.start,n=i!=this.lastVisibleInterval;(this.lastVisibleInterval=i,1==t)&&(this.svg.style.width=za.option.asSize(3*this.props.width),this.svg.style.left=za.option.asSize(-this.props.width),-1==Fo(e=this.options.height+"").call(e,"%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0));if(1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==n||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,s=this.body.range.end-this.body.range.start;if(0!=this.props.width){var r=o*(this.props.width/s);this.svg.style.left=-this.props.width-r+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},Tu.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}za.insertSort(t,(function(t,e){var i=t.zIndex,n=e.zIndex;return void 0===i&&(i=0),void 0===n&&(n=0),i==n?0:i<n?-1:1}));for(var n=new Array(t.length),o=0;o<t.length;o++)n[o]=t[o].id;return n},Tu.prototype._updateGraph=function(){if(Cs(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},n=this.body.util.toGlobalTime(-this.body.domProps.root.width),o=this.body.util.toGlobalTime(2*this.body.domProps.root.width),s=this._getSortedGroupIds();if(s.length>0){var r={};for(this._getRelevantData(s,r,n,o),this._applySampling(s,r),e=0;e<s.length;e++)this._convertXcoordinates(r[s[e]]);if(this._getYRanges(s,r,i),1==this._updateYAxis(s,i))return Ls(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var a=void 0;for(e=0;e<s.length;e++)t=this.groups[s[e]],!0===this.options.stack&&"line"===this.options.style&&(null!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(null!=a&&(this._stack(r[t.id],r[a.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==a.options.shaded.orientation?(a.options.shaded.orientation="group",a.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=a.id))),a=t)),this._convertYcoordinates(r[s[e]],t);var l={};for(e=0;e<s.length;e++)if("line"===(t=this.groups[s[e]]).options.style&&1==t.options.shaded.enabled){var h=r[s[e]];if(null==h||0==h.length)continue;if(l.hasOwnProperty(s[e])||(l[s[e]]=Wu.calcPath(h,t)),"group"===t.options.shaded.orientation){var d=t.options.shaded.groupId;if(-1===Fo(s).call(s,d)){console.log(t.id+": Unknown shading group target given:"+d);continue}l.hasOwnProperty(d)||(l[d]=Wu.calcPath(r[d],this.groups[d])),Wu.drawShading(l[s[e]],t,l[d],this.framework)}else Wu.drawShading(l[s[e]],t,void 0,this.framework)}for(Ou.draw(s,r,this.framework),e=0;e<s.length;e++)if(t=this.groups[s[e]],r[s[e]].length>0)switch(t.options.style){case"line":l.hasOwnProperty(s[e])||(l[s[e]]=Wu.calcPath(r[s[e]],t)),Wu.draw(l[s[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||Zu.draw(r[s[e]],t,this.framework)}}}return Ls(this.svgElements),!1},Tu.prototype._stack=function(t,e){var i,n,o,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var l=i;l<e.length;l++){if(e[l].x===t[a].x){s=e[l],r=e[l],i=l;break}if(e[l].x>t[a].x){r=e[l],s=0==l?r:e[l-1],i=l;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),n=r.x-s.x,o=r.y-s.y,t[a].y=0==n?t[a].orginalY+r.y:t[a].orginalY+o/n*(t[a].x-s.x)+s.y}},Tu.prototype._getRelevantData=function(t,e,i,n){var o,s,r,a;if(t.length>0)for(s=0;s<t.length;s++){var l=(o=this.groups[t[s]]).getItems();if(1==wr(o.options)){var h=function(t,e){return t.getTime()==e.getTime()?0:t<e?-1:1},d=Math.max(0,za.binarySearchValue(l,i,"x","before",h)),u=Math.min(l.length,za.binarySearchValue(l,n,"x","after",h)+1);u<=0&&(u=l.length);var c=new Array(u-d);for(r=d;r<u;r++)a=o.itemsData[r],c[r-d]=a;e[t[s]]=c}else e[t[s]]=o.itemsData}},Tu.prototype._applySampling=function(t,e){if(t.length>0)for(var i=0;i<t.length;i++)if(1==this.groups[t[i]].options.sampling){var n=e[t[i]];if(n.length>0){var o,s=n.length,r=s/(this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x));o=Math.min(Math.ceil(.2*s),Math.max(1,Math.round(r)));for(var a=new Array(s),l=0;l<s;l+=o){a[Math.round(l/o)]=n[l]}e[t[i]]=Hr(a).call(a,0,Math.round(s/o))}}},Tu.prototype._getYRanges=function(t,e,i){var n,o,s,r,a=[],l=[];if(t.length>0){for(s=0;s<t.length;s++)n=e[t[s]],r=this.groups[t[s]].options,n.length>0&&(o=this.groups[t[s]],!0===r.stack&&"bar"===r.style?"left"===r.yAxisOrientation?a=Pn(a).call(a,n):l=Pn(l).call(l,n):i[t[s]]=o.getYRange(n,t[s]));Ou.getStackedYRange(a,i,t,"__barStackLeft","left"),Ou.getStackedYRange(l,i,t,"__barStackRight","right")}},Tu.prototype._updateYAxis=function(t,e){var i,n,o=!1,s=!1,r=!1,a=1e9,l=1e9,h=-1e9,d=-1e9;if(t.length>0){for(var u=0;u<t.length;u++){var c=this.groups[t[u]];c&&"right"!=c.options.yAxisOrientation?(s=!0,a=1e9,h=-1e9):c&&c.options.yAxisOrientation&&(r=!0,l=1e9,d=-1e9)}for(u=0;u<t.length;u++)e.hasOwnProperty(t[u])&&!0!==e[t[u]].ignore&&(i=e[t[u]].min,n=e[t[u]].max,"right"!=e[t[u]].yAxisOrientation?(s=!0,a=a>i?i:a,h=h<n?n:h):(r=!0,l=l>i?i:l,d=d<n?n:d));1==s&&this.yAxisLeft.setRange(a,h),1==r&&this.yAxisRight.setRange(l,d)}o=this._toggleAxisVisiblity(s,this.yAxisLeft)||o,o=this._toggleAxisVisiblity(r,this.yAxisRight)||o,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(this.yAxisLeft.lineOffset=1==r?this.yAxisRight.width:0,o=this.yAxisLeft.redraw()||o,o=this.yAxisRight.redraw()||o):o=this.yAxisRight.redraw()||o;var p=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"];for(u=0;u<p.length;u++)-1!=Fo(t).call(t,p[u])&&Hr(t).call(t,Fo(t).call(t,p[u]),1);return o},Tu.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},Tu.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y,null!=t[i].end?t[i].screen_end=e(t[i].end)+this.props.width:t[i].screen_end=void 0},Tu.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,n=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var o=0;o<t.length;o++)t[o].screen_y=Math.round(i.convertValue(t[o].y));e.setZeroPosition(Math.min(n,i.convertValue(0)))};var Yu="boolean",Eu={configure:{enabled:{boolean:Yu},filter:{boolean:Yu,function:"function"},container:{dom:"dom"},__type__:{object:"object",boolean:Yu,function:"function"}},alignCurrentTime:{string:"string",undefined:"undefined"},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:"string"},sort:{boolean:Yu},sampling:{boolean:Yu},stack:{boolean:Yu},graphHeight:{string:"string",number:"number"},shaded:{enabled:{boolean:Yu},orientation:{string:["bottom","top","zero","group"]},groupId:{object:"object"},__type__:{boolean:Yu,object:"object"}},style:{string:["line","bar","points"]},barChart:{width:{number:"number"},minWidth:{number:"number"},sideBySide:{boolean:Yu},align:{string:["left","center","right"]},__type__:{object:"object"}},interpolation:{enabled:{boolean:Yu},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:"number"},__type__:{object:"object",boolean:Yu}},drawPoints:{enabled:{boolean:Yu},onRender:{function:"function"},size:{number:"number"},style:{string:["square","circle"]},__type__:{object:"object",boolean:Yu,function:"function"}},dataAxis:{showMinorLabels:{boolean:Yu},showMajorLabels:{boolean:Yu},showWeekScale:{boolean:Yu},icons:{boolean:Yu},width:{string:"string",number:"number"},visible:{boolean:Yu},alignZeros:{boolean:Yu},left:{range:{min:{number:"number",undefined:"undefined"},max:{number:"number",undefined:"undefined"},__type__:{object:"object"}},format:{function:"function"},title:{text:{string:"string",number:"number",undefined:"undefined"},style:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},right:{range:{min:{number:"number",undefined:"undefined"},max:{number:"number",undefined:"undefined"},__type__:{object:"object"}},format:{function:"function"},title:{text:{string:"string",number:"number",undefined:"undefined"},style:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},__type__:{object:"object"}},legend:{enabled:{boolean:Yu},icons:{boolean:Yu},left:{visible:{boolean:Yu},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:"object"}},right:{visible:{boolean:Yu},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:"object"}},__type__:{object:"object",boolean:Yu}},groups:{visibility:{any:"any"},__type__:{object:"object"}},autoResize:{boolean:Yu},throttleRedraw:{number:"number"},clickToUse:{boolean:Yu},end:{number:"number",date:"date",string:"string",moment:"moment"},format:{minorLabels:{millisecond:{string:"string",undefined:"undefined"},second:{string:"string",undefined:"undefined"},minute:{string:"string",undefined:"undefined"},hour:{string:"string",undefined:"undefined"},weekday:{string:"string",undefined:"undefined"},day:{string:"string",undefined:"undefined"},week:{string:"string",undefined:"undefined"},month:{string:"string",undefined:"undefined"},quarter:{string:"string",undefined:"undefined"},year:{string:"string",undefined:"undefined"},__type__:{object:"object"}},majorLabels:{millisecond:{string:"string",undefined:"undefined"},second:{string:"string",undefined:"undefined"},minute:{string:"string",undefined:"undefined"},hour:{string:"string",undefined:"undefined"},weekday:{string:"string",undefined:"undefined"},day:{string:"string",undefined:"undefined"},week:{string:"string",undefined:"undefined"},month:{string:"string",undefined:"undefined"},quarter:{string:"string",undefined:"undefined"},year:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},moment:{function:"function"},height:{string:"string",number:"number"},hiddenDates:{start:{date:"date",number:"number",string:"string",moment:"moment"},end:{date:"date",number:"number",string:"string",moment:"moment"},repeat:{string:"string"},__type__:{object:"object",array:"array"}},locale:{string:"string"},locales:{__any__:{any:"any"},__type__:{object:"object"}},max:{date:"date",number:"number",string:"string",moment:"moment"},maxHeight:{number:"number",string:"string"},maxMinorChars:{number:"number"},min:{date:"date",number:"number",string:"string",moment:"moment"},minHeight:{number:"number",string:"string"},moveable:{boolean:Yu},multiselect:{boolean:Yu},orientation:{string:"string"},showCurrentTime:{boolean:Yu},showMajorLabels:{boolean:Yu},showMinorLabels:{boolean:Yu},showWeekScale:{boolean:Yu},start:{date:"date",number:"number",string:"string",moment:"moment"},timeAxis:{scale:{string:"string",undefined:"undefined"},step:{number:"number",undefined:"undefined"},__type__:{object:"object"}},width:{string:"string",number:"number"},zoomable:{boolean:Yu},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:"number"},zoomMin:{number:"number"},zIndex:{number:"number"},__type__:{object:"object"}},Ru={global:{alignCurrentTime:["none","year","month","quarter","week","isoWeek","day","date","hour","minute","second"],sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",quarter:"[Q]Q",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",quarter:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,showWeekScale:!1,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};function Uu(t,e,i,n){var o,s,r,a,l,h,d;if(!(Zo(i)||i instanceof Aa||i instanceof Xa)&&i instanceof Object){var u=n;n=i,i=u}n&&n.throttleRedraw&&console.warn('Graph2d option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var c=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:Ta,width:null,height:null,maxHeight:null,minHeight:null},this.options=za.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:va(o=this.on).call(o,this),off:va(s=this.off).call(s,this),emit:va(r=this.emit).call(r,this)},hiddenDates:[],util:{toScreen:va(a=c._toScreen).call(a,c),toGlobalScreen:va(l=c._toGlobalScreen).call(l,c),toTime:va(h=c._toTime).call(h,c),toGlobalTime:va(d=c._toGlobalTime).call(d,c)}},this.range=new ml(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new Jh(this.body),this.components.push(this.timeAxis),this.currentTime=new pd(this.body),this.components.push(this.currentTime),this.linegraph=new Tu(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",(function(t){c.emit("click",c.getEventProperties(t))})),this.on("doubletap",(function(t){c.emit("doubleClick",c.getEventProperties(t))})),this.dom.root.oncontextmenu=function(t){c.emit("contextmenu",c.getEventProperties(t))},this.initialFitDone=!1,this.on("changed",(function(){if(null!=c.itemsData){if(!c.initialFitDone&&!c.options.rollingMode)if(c.initialFitDone=!0,null!=c.options.start||null!=c.options.end){if(null==c.options.start||null==c.options.end)var t=c.getItemRange();var e=null!=c.options.start?c.options.start:t.min,i=null!=c.options.end?c.options.end:t.max;c.setWindow(e,i,{animation:!1})}else c.fit({animation:!1});c.initialDrawDone||!c.initialRangeChangeDone&&(c.options.start||c.options.end)&&!c.options.rollingMode||(c.initialDrawDone=!0,c.dom.root.style.visibility="visible",c.dom.loadingScreen.parentNode.removeChild(c.dom.loadingScreen),c.options.onInitialDrawComplete&&qr((function(){return c.options.onInitialDrawComplete()}),0))}})),n&&this.setOptions(n),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}Uu.prototype=new cd,Uu.prototype.setOptions=function(t){!0===fu.validate(t,Eu)&&console.log("%cErrors have been found in the supplied options object.",mu),cd.prototype.setOptions.call(this,t)},Uu.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?Ua(t instanceof Aa||t instanceof Xa?t:new Aa(t)):null,this.itemsData&&this.itemsData.dispose(),this.itemsData=e,this.linegraph&&this.linegraph.setItems(null!=e?e.rawDS:null),i)if(null!=this.options.start||null!=this.options.end){var n=null!=this.options.start?this.options.start:null,o=null!=this.options.end?this.options.end:null;this.setWindow(n,o,{animation:!1})}else this.fit({animation:!1})},Uu.prototype.setGroups=function(t){var e;e=t?t instanceof Aa||t instanceof Xa?t:new Aa(t):null,this.groupsData=e,this.linegraph.setGroups(e)},Uu.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},Uu.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]&&(this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]))},Uu.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var n=0;n<this.linegraph.groups[i].itemsData.length;n++){var o=this.linegraph.groups[i].itemsData[n],s=za.convert(o.x,"Date").valueOf();t=null==t||t>s?s:t,e=null==e||e<s?s:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},Uu.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,n=e-za.getAbsoluteLeft(this.dom.centerContainer),o=i-za.getAbsoluteTop(this.dom.centerContainer),s=this._toTime(n),r=ud.customTimeFromTarget(t),a=za.getTarget(t),l=null;za.hasParent(a,this.timeAxis.dom.foreground)||this.timeAxis2&&za.hasParent(a,this.timeAxis2.dom.foreground)?l="axis":za.hasParent(a,this.linegraph.yAxisLeft.dom.frame)||za.hasParent(a,this.linegraph.yAxisRight.dom.frame)?l="data-axis":za.hasParent(a,this.linegraph.legendLeft.dom.frame)||za.hasParent(a,this.linegraph.legendRight.dom.frame)?l="legend":null!=r?l="custom-time":za.hasParent(a,this.currentTime.bar)?l="current-time":za.hasParent(a,this.dom.center)&&(l="background");var h=[],d=this.linegraph.yAxisLeft,u=this.linegraph.yAxisRight;return!d.hidden&&this.itemsData.length>0&&h.push(d.screenToValue(o)),!u.hidden&&this.itemsData.length>0&&h.push(u.screenToValue(o)),{event:t,customTime:r?r.options.id:null,what:l,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:n,y:o,time:s,value:h}},Uu.prototype._createConfigurator=function(){return new Lu(this,this.dom.container,Ru)};var zu=Fs();Ta.locale(zu);var Vu={Core:cd,DateUtil:pl,Range:ml,stack:zd,TimeStep:Hh,components:{items:{Item:Pd,BackgroundItem:$d,BoxItem:Jd,PointItem:Kd,RangeItem:qd},BackgroundGroup:Hd,Component:Ja,CurrentTime:pd,CustomTime:ud,DataAxis:Fu,DataScale:Nu,GraphGroup:Au,Group:jd,ItemSet:uu,Legend:Xu,LineGraph:Tu,TimeAxis:Jh}};t.DOMutil=Zs,t.DataSet=Aa,t.DataView=Xa,t.Graph2d=Uu,t.Hammer=Vh,t.Queue=Za,t.Timeline=Gu,t.keycharm=qh,t.moment=Ta,t.timeline=Vu,t.util=Is,Object.defineProperty(t,"__esModule",{value:!0})}));
+//# sourceMappingURL=vis-timeline-graph2d.min.js.map
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
new file mode 100644
index 0000000000000..779880dcc11af
--- /dev/null
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
@@ -0,0 +1 @@
+{"version":3,"file":"vis-timeline-graph2d.min.js","sources":["../../node_modules/moment/moment.js","../../node_modules/core-js-pure/internals/global.js","../../node_modules/moment/locale/de.js","../../node_modules/moment/locale/es.js","../../node_modules/moment/locale/fr.js","../../node_modules/moment/locale/it.js","../../node_modules/moment/locale/ja.js","../../node_modules/moment/locale/nl.js","../../node_modules/moment/locale/pl.js","../../node_modules/moment/locale/ru.js","../../node_modules/moment/locale/uk.js","../../node_modules/core-js-pure/internals/fails.js","../../node_modules/core-js-pure/internals/descriptors.js","../../node_modules/core-js-pure/internals/object-property-is-enumerable.js","../../node_modules/core-js-pure/internals/create-property-descriptor.js","../../node_modules/core-js-pure/internals/classof-raw.js","../../node_modules/core-js-pure/internals/indexed-object.js","../../node_modules/core-js-pure/internals/require-object-coercible.js","../../node_modules/core-js-pure/internals/to-indexed-object.js","../../node_modules/core-js-pure/internals/is-object.js","../../node_modules/core-js-pure/internals/to-primitive.js","../../node_modules/core-js-pure/internals/has.js","../../node_modules/core-js-pure/internals/document-create-element.js","../../node_modules/core-js-pure/internals/ie8-dom-define.js","../../node_modules/core-js-pure/internals/object-get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/is-forced.js","../../node_modules/core-js-pure/internals/path.js","../../node_modules/core-js-pure/internals/a-function.js","../../node_modules/core-js-pure/internals/function-bind-context.js","../../node_modules/core-js-pure/internals/an-object.js","../../node_modules/core-js-pure/internals/object-define-property.js","../../node_modules/core-js-pure/internals/create-non-enumerable-property.js","../../node_modules/core-js-pure/internals/export.js","../../node_modules/core-js-pure/modules/es.object.define-property.js","../../node_modules/core-js-pure/es/object/define-property.js","../../node_modules/core-js-pure/stable/object/define-property.js","../../node_modules/core-js-pure/internals/to-integer.js","../../node_modules/core-js-pure/internals/to-length.js","../../node_modules/core-js-pure/internals/to-absolute-index.js","../../node_modules/core-js-pure/internals/array-includes.js","../../node_modules/core-js-pure/internals/hidden-keys.js","../../node_modules/core-js-pure/internals/object-keys-internal.js","../../node_modules/core-js-pure/internals/enum-bug-keys.js","../../node_modules/core-js-pure/internals/object-keys.js","../../node_modules/core-js-pure/internals/object-define-properties.js","../../node_modules/core-js-pure/modules/es.object.define-properties.js","../../node_modules/core-js-pure/es/object/define-properties.js","../../node_modules/core-js-pure/internals/get-built-in.js","../../node_modules/core-js-pure/internals/object-get-own-property-names.js","../../node_modules/core-js-pure/internals/object-get-own-property-symbols.js","../../node_modules/core-js-pure/internals/own-keys.js","../../node_modules/core-js-pure/internals/create-property.js","../../node_modules/core-js-pure/modules/es.object.get-own-property-descriptors.js","../../node_modules/core-js-pure/es/object/get-own-property-descriptors.js","../../node_modules/core-js-pure/modules/es.object.get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/object-create.js","../../node_modules/core-js-pure/es/object/get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/native-symbol.js","../../node_modules/core-js-pure/internals/use-symbol-as-uid.js","../../node_modules/core-js-pure/internals/is-array.js","../../node_modules/core-js-pure/internals/to-object.js","../../node_modules/core-js-pure/internals/html.js","../../node_modules/core-js-pure/internals/shared-store.js","../../node_modules/core-js-pure/internals/set-global.js","../../node_modules/core-js-pure/internals/shared.js","../../node_modules/core-js-pure/internals/uid.js","../../node_modules/core-js-pure/internals/shared-key.js","../../node_modules/core-js-pure/internals/object-get-own-property-names-external.js","../../node_modules/core-js-pure/internals/redefine.js","../../node_modules/core-js-pure/internals/well-known-symbol.js","../../node_modules/core-js-pure/internals/well-known-symbol-wrapped.js","../../node_modules/core-js-pure/internals/define-well-known-symbol.js","../../node_modules/core-js-pure/internals/to-string-tag-support.js","../../node_modules/core-js-pure/internals/classof.js","../../node_modules/core-js-pure/internals/object-to-string.js","../../node_modules/core-js-pure/internals/set-to-string-tag.js","../../node_modules/core-js-pure/internals/inspect-source.js","../../node_modules/core-js-pure/internals/internal-state.js","../../node_modules/core-js-pure/internals/native-weak-map.js","../../node_modules/core-js-pure/internals/array-species-create.js","../../node_modules/core-js-pure/internals/array-iteration.js","../../node_modules/core-js-pure/modules/es.symbol.js","../../node_modules/core-js-pure/es/object/get-own-property-symbols.js","../../node_modules/core-js-pure/internals/iterators-core.js","../../node_modules/core-js-pure/internals/iterators.js","../../node_modules/core-js-pure/internals/correct-prototype-getter.js","../../node_modules/core-js-pure/internals/object-get-prototype-of.js","../../node_modules/core-js-pure/internals/create-iterator-constructor.js","../../node_modules/core-js-pure/internals/object-set-prototype-of.js","../../node_modules/core-js-pure/internals/a-possible-prototype.js","../../node_modules/core-js-pure/internals/define-iterator.js","../../node_modules/core-js-pure/modules/es.array.iterator.js","../../node_modules/core-js-pure/internals/dom-iterables.js","../../node_modules/core-js-pure/modules/web.dom-collections.iterator.js","../../node_modules/core-js-pure/internals/string-multibyte.js","../../node_modules/core-js-pure/modules/es.string.iterator.js","../../node_modules/core-js-pure/internals/get-iterator-method.js","../../node_modules/core-js-pure/internals/get-iterator.js","../../node_modules/core-js-pure/modules/es.object.create.js","../../node_modules/core-js-pure/es/object/create.js","../../node_modules/core-js-pure/stable/object/create.js","../../node_modules/core-js-pure/features/object/define-property.js","../../node_modules/@babel/runtime-corejs3/helpers/defineProperty.js","../../node_modules/core-js-pure/modules/es.object.keys.js","../../node_modules/core-js-pure/es/object/keys.js","../../node_modules/core-js-pure/internals/string-trim-forced.js","../../node_modules/core-js-pure/internals/whitespaces.js","../../node_modules/core-js-pure/internals/string-trim.js","../../node_modules/core-js-pure/modules/es.string.trim.js","../../node_modules/core-js-pure/internals/entry-virtual.js","../../node_modules/core-js-pure/es/string/virtual/trim.js","../../node_modules/core-js-pure/es/instance/trim.js","../../node_modules/core-js-pure/internals/array-method-is-strict.js","../../node_modules/core-js-pure/internals/array-method-uses-to-length.js","../../node_modules/core-js-pure/internals/array-for-each.js","../../node_modules/core-js-pure/modules/es.array.for-each.js","../../node_modules/core-js-pure/es/array/virtual/for-each.js","../../node_modules/core-js-pure/internals/engine-v8-version.js","../../node_modules/core-js-pure/stable/instance/for-each.js","../../node_modules/core-js-pure/internals/engine-user-agent.js","../../node_modules/core-js-pure/internals/array-method-has-species-support.js","../../node_modules/core-js-pure/modules/es.array.map.js","../../node_modules/core-js-pure/es/array/virtual/map.js","../../node_modules/core-js-pure/es/instance/map.js","../../node_modules/core-js-pure/internals/number-parse-int.js","../../node_modules/core-js-pure/modules/es.parse-int.js","../../node_modules/core-js-pure/es/parse-int.js","../../node_modules/core-js-pure/internals/object-to-array.js","../../node_modules/core-js-pure/modules/es.object.values.js","../../node_modules/core-js-pure/es/object/values.js","../../node_modules/core-js-pure/modules/es.array.filter.js","../../node_modules/core-js-pure/es/array/virtual/filter.js","../../node_modules/core-js-pure/es/instance/filter.js","../../node_modules/core-js-pure/modules/es.array.concat.js","../../node_modules/core-js-pure/es/array/virtual/concat.js","../../node_modules/core-js-pure/es/instance/concat.js","../../node_modules/core-js-pure/modules/es.array.is-array.js","../../node_modules/core-js-pure/es/array/is-array.js","../../node_modules/core-js-pure/features/array/is-array.js","../../node_modules/@babel/runtime-corejs3/helpers/arrayWithoutHoles.js","../../node_modules/core-js-pure/internals/call-with-safe-iteration-closing.js","../../node_modules/core-js-pure/internals/is-array-iterator-method.js","../../node_modules/core-js-pure/internals/check-correctness-of-iteration.js","../../node_modules/core-js-pure/modules/es.array.from.js","../../node_modules/core-js-pure/internals/array-from.js","../../node_modules/core-js-pure/es/array/from.js","../../node_modules/core-js-pure/internals/is-iterable.js","../../node_modules/@babel/runtime-corejs3/helpers/iterableToArray.js","../../node_modules/@babel/runtime-corejs3/helpers/nonIterableSpread.js","../../node_modules/@babel/runtime-corejs3/helpers/toConsumableArray.js","../../node_modules/core-js-pure/modules/es.array.slice.js","../../node_modules/core-js-pure/es/array/virtual/slice.js","../../node_modules/core-js-pure/es/instance/slice.js","../../node_modules/core-js-pure/modules/es.object.get-prototype-of.js","../../node_modules/core-js-pure/es/object/get-prototype-of.js","../../node_modules/core-js-pure/stable/object/get-prototype-of.js","../../node_modules/core-js-pure/modules/es.array.index-of.js","../../node_modules/core-js-pure/es/array/virtual/index-of.js","../../node_modules/core-js-pure/es/instance/index-of.js","../../node_modules/core-js-pure/stable/array/is-array.js","../../node_modules/core-js-pure/internals/object-assign.js","../../node_modules/core-js-pure/modules/es.object.assign.js","../../node_modules/core-js-pure/es/object/assign.js","../../node_modules/core-js-pure/modules/es.symbol.iterator.js","../../node_modules/core-js-pure/es/symbol/iterator.js","../../node_modules/core-js-pure/features/symbol/iterator.js","../../node_modules/core-js-pure/modules/es.symbol.async-iterator.js","../../node_modules/core-js-pure/modules/es.symbol.has-instance.js","../../node_modules/core-js-pure/modules/es.symbol.is-concat-spreadable.js","../../node_modules/core-js-pure/modules/es.symbol.match.js","../../node_modules/core-js-pure/modules/es.symbol.match-all.js","../../node_modules/core-js-pure/modules/es.symbol.replace.js","../../node_modules/core-js-pure/modules/es.symbol.search.js","../../node_modules/core-js-pure/modules/es.symbol.species.js","../../node_modules/core-js-pure/modules/es.symbol.split.js","../../node_modules/core-js-pure/modules/es.symbol.to-primitive.js","../../node_modules/core-js-pure/modules/es.symbol.to-string-tag.js","../../node_modules/core-js-pure/modules/es.symbol.unscopables.js","../../node_modules/core-js-pure/modules/es.math.to-string-tag.js","../../node_modules/core-js-pure/modules/es.json.to-string-tag.js","../../node_modules/core-js-pure/es/symbol/index.js","../../node_modules/core-js-pure/modules/esnext.symbol.async-dispose.js","../../node_modules/core-js-pure/modules/esnext.symbol.dispose.js","../../node_modules/core-js-pure/modules/esnext.symbol.observable.js","../../node_modules/core-js-pure/modules/esnext.symbol.pattern-match.js","../../node_modules/core-js-pure/modules/esnext.symbol.replace-all.js","../../node_modules/core-js-pure/features/symbol/index.js","../../node_modules/@babel/runtime-corejs3/helpers/typeof.js","../../node_modules/@babel/runtime-corejs3/helpers/arrayWithHoles.js","../../node_modules/@babel/runtime-corejs3/helpers/iterableToArrayLimit.js","../../node_modules/@babel/runtime-corejs3/helpers/nonIterableRest.js","../../node_modules/@babel/runtime-corejs3/helpers/slicedToArray.js","../../node_modules/core-js-pure/modules/es.date.now.js","../../node_modules/core-js-pure/es/date/now.js","../../node_modules/vis-util/esnext/esm/vis-util.js","../../lib/DOMutil.js","../../node_modules/@babel/runtime-corejs3/helpers/assertThisInitialized.js","../../node_modules/core-js-pure/es/array/virtual/entries.js","../../node_modules/core-js-pure/stable/instance/entries.js","../../node_modules/core-js-pure/modules/es.json.stringify.js","../../node_modules/core-js-pure/es/json/stringify.js","../../node_modules/core-js-pure/es/array/virtual/values.js","../../node_modules/core-js-pure/stable/instance/values.js","../../node_modules/core-js-pure/es/array/virtual/keys.js","../../node_modules/core-js-pure/stable/instance/keys.js","../../node_modules/core-js-pure/modules/es.array.some.js","../../node_modules/core-js-pure/es/array/virtual/some.js","../../node_modules/core-js-pure/es/instance/some.js","../../node_modules/@babel/runtime-corejs3/helpers/possibleConstructorReturn.js","../../node_modules/core-js-pure/features/object/get-prototype-of.js","../../node_modules/core-js-pure/modules/es.object.set-prototype-of.js","../../node_modules/core-js-pure/es/object/set-prototype-of.js","../../node_modules/@babel/runtime-corejs3/helpers/getPrototypeOf.js","../../node_modules/core-js-pure/features/object/create.js","../../node_modules/@babel/runtime-corejs3/helpers/setPrototypeOf.js","../../node_modules/@babel/runtime-corejs3/helpers/inherits.js","../../node_modules/core-js-pure/modules/es.array.sort.js","../../node_modules/core-js-pure/es/array/virtual/sort.js","../../node_modules/core-js-pure/es/instance/sort.js","../../node_modules/core-js-pure/internals/freezing.js","../../node_modules/core-js-pure/internals/internal-metadata.js","../../node_modules/core-js-pure/internals/iterate.js","../../node_modules/core-js-pure/internals/an-instance.js","../../node_modules/core-js-pure/internals/collection.js","../../node_modules/core-js-pure/internals/redefine-all.js","../../node_modules/core-js-pure/internals/set-species.js","../../node_modules/core-js-pure/internals/collection-strong.js","../../node_modules/core-js-pure/modules/es.set.js","../../node_modules/core-js-pure/es/set/index.js","../../node_modules/core-js-pure/modules/es.map.js","../../node_modules/core-js-pure/es/map/index.js","../../node_modules/@babel/runtime-corejs3/node_modules/regenerator-runtime/runtime.js","../../node_modules/core-js-pure/stable/symbol/iterator.js","../../node_modules/core-js-pure/modules/es.array.splice.js","../../node_modules/core-js-pure/es/array/virtual/splice.js","../../node_modules/core-js-pure/es/instance/splice.js","../../node_modules/core-js-pure/modules/web.timers.js","../../node_modules/core-js-pure/stable/set-timeout.js","../../node_modules/core-js-pure/internals/flatten-into-array.js","../../node_modules/core-js-pure/modules/es.array.flat-map.js","../../node_modules/core-js-pure/es/array/virtual/flat-map.js","../../node_modules/core-js-pure/es/instance/flat-map.js","../../node_modules/core-js-pure/internals/array-reduce.js","../../node_modules/core-js-pure/modules/es.array.reduce.js","../../node_modules/core-js-pure/es/array/virtual/reduce.js","../../node_modules/core-js-pure/es/instance/reduce.js","../../node_modules/core-js-pure/internals/function-bind.js","../../node_modules/core-js-pure/modules/es.function.bind.js","../../node_modules/core-js-pure/es/function/virtual/bind.js","../../node_modules/core-js-pure/es/instance/bind.js","../../node_modules/@babel/runtime-corejs3/helpers/classCallCheck.js","../../node_modules/@babel/runtime-corejs3/helpers/createClass.js","../../node_modules/vis-data/esnext/esm/vis-data.js","../../lib/module/moment.js","../../lib/util.js","../../node_modules/core-js-pure/internals/number-parse-float.js","../../node_modules/core-js-pure/modules/es.parse-float.js","../../node_modules/core-js-pure/es/parse-float.js","../../lib/timeline/component/Component.js","../../node_modules/core-js-pure/modules/es.string.repeat.js","../../node_modules/core-js-pure/internals/string-repeat.js","../../node_modules/core-js-pure/es/string/virtual/repeat.js","../../node_modules/core-js-pure/es/instance/repeat.js","../../lib/timeline/DateUtil.js","../../lib/timeline/Range.js","../../node_modules/core-js-pure/stable/set-interval.js","../../node_modules/component-emitter/index.js","../../node_modules/propagating-hammerjs/propagating.js","../../node_modules/@egjs/hammerjs/dist/hammer.esm.js","../../lib/module/hammer.js","../../lib/hammerUtil.js","../../lib/timeline/TimeStep.js","../../node_modules/style-inject/dist/style-inject.es.js","../../lib/timeline/component/TimeAxis.js","../../node_modules/keycharm/keycharm.js","../../lib/shared/Activator.js","../../lib/timeline/locales.js","../../lib/timeline/component/CustomTime.js","../../lib/timeline/Core.js","../../lib/timeline/component/CurrentTime.js","../../node_modules/core-js-pure/modules/es.array.find.js","../../node_modules/core-js-pure/es/array/virtual/find.js","../../node_modules/core-js-pure/es/instance/find.js","../../node_modules/uuid/dist/esm-browser/rng.js","../../node_modules/uuid/dist/esm-browser/bytesToUuid.js","../../node_modules/uuid/dist/esm-browser/v4.js","../../node_modules/core-js-pure/modules/es.array.includes.js","../../node_modules/core-js-pure/es/array/virtual/includes.js","../../node_modules/core-js-pure/internals/is-regexp.js","../../node_modules/core-js-pure/internals/not-a-regexp.js","../../node_modules/core-js-pure/internals/correct-is-regexp-logic.js","../../node_modules/core-js-pure/modules/es.string.includes.js","../../node_modules/core-js-pure/es/string/virtual/includes.js","../../node_modules/core-js-pure/es/instance/includes.js","../../lib/timeline/Stack.js","../../lib/timeline/component/Group.js","../../lib/timeline/component/BackgroundGroup.js","../../lib/timeline/component/item/Item.js","../../lib/timeline/component/item/BoxItem.js","../../lib/timeline/component/item/PointItem.js","../../lib/timeline/component/item/RangeItem.js","../../lib/timeline/component/item/BackgroundItem.js","../../lib/shared/Popup.js","../../node_modules/core-js-pure/modules/es.array.every.js","../../node_modules/core-js-pure/es/array/virtual/every.js","../../node_modules/core-js-pure/es/instance/every.js","../../lib/timeline/component/item/ClusterItem.js","../../lib/timeline/component/ClusterGenerator.js","../../lib/timeline/component/ItemSet.js","../../lib/shared/Validator.js","../../lib/timeline/optionsTimeline.js","../../node_modules/core-js-pure/modules/es.array.fill.js","../../node_modules/core-js-pure/internals/array-fill.js","../../node_modules/core-js-pure/es/array/virtual/fill.js","../../node_modules/core-js-pure/es/instance/fill.js","../../lib/shared/ColorPicker.js","../../lib/shared/Configurator.js","../../lib/timeline/Timeline.js","../../lib/timeline/component/DataScale.js","../../lib/timeline/component/DataAxis.js","../../lib/timeline/component/graph2d_types/points.js","../../lib/timeline/component/graph2d_types/bar.js","../../lib/timeline/component/graph2d_types/line.js","../../lib/timeline/component/GraphGroup.js","../../lib/timeline/component/Legend.js","../../lib/timeline/component/LineGraph.js","../../lib/timeline/optionsGraph2d.js","../../lib/timeline/Graph2d.js","../../lib/entry-standalone.js"],"sourcesContent":["//! moment.js\n\n;(function (global, factory) {\n    typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :\n    typeof define === 'function' && define.amd ? define(factory) :\n    global.moment = factory()\n}(this, (function () { 'use strict';\n\n    var hookCallback;\n\n    function hooks () {\n        return hookCallback.apply(null, arguments);\n    }\n\n    // This is done to register the method called with moment()\n    // without creating circular dependencies.\n    function setHookCallback (callback) {\n        hookCallback = callback;\n    }\n\n    function isArray(input) {\n        return input instanceof Array || Object.prototype.toString.call(input) === '[object Array]';\n    }\n\n    function isObject(input) {\n        // IE8 will treat undefined and null as object if it wasn't for\n        // input != null\n        return input != null && Object.prototype.toString.call(input) === '[object Object]';\n    }\n\n    function isObjectEmpty(obj) {\n        if (Object.getOwnPropertyNames) {\n            return (Object.getOwnPropertyNames(obj).length === 0);\n        } else {\n            var k;\n            for (k in obj) {\n                if (obj.hasOwnProperty(k)) {\n                    return false;\n                }\n            }\n            return true;\n        }\n    }\n\n    function isUndefined(input) {\n        return input === void 0;\n    }\n\n    function isNumber(input) {\n        return typeof input === 'number' || Object.prototype.toString.call(input) === '[object Number]';\n    }\n\n    function isDate(input) {\n        return input instanceof Date || Object.prototype.toString.call(input) === '[object Date]';\n    }\n\n    function map(arr, fn) {\n        var res = [], i;\n        for (i = 0; i < arr.length; ++i) {\n            res.push(fn(arr[i], i));\n        }\n        return res;\n    }\n\n    function hasOwnProp(a, b) {\n        return Object.prototype.hasOwnProperty.call(a, b);\n    }\n\n    function extend(a, b) {\n        for (var i in b) {\n            if (hasOwnProp(b, i)) {\n                a[i] = b[i];\n            }\n        }\n\n        if (hasOwnProp(b, 'toString')) {\n            a.toString = b.toString;\n        }\n\n        if (hasOwnProp(b, 'valueOf')) {\n            a.valueOf = b.valueOf;\n        }\n\n        return a;\n    }\n\n    function createUTC (input, format, locale, strict) {\n        return createLocalOrUTC(input, format, locale, strict, true).utc();\n    }\n\n    function defaultParsingFlags() {\n        // We need to deep clone this object.\n        return {\n            empty           : false,\n            unusedTokens    : [],\n            unusedInput     : [],\n            overflow        : -2,\n            charsLeftOver   : 0,\n            nullInput       : false,\n            invalidMonth    : null,\n            invalidFormat   : false,\n            userInvalidated : false,\n            iso             : false,\n            parsedDateParts : [],\n            meridiem        : null,\n            rfc2822         : false,\n            weekdayMismatch : false\n        };\n    }\n\n    function getParsingFlags(m) {\n        if (m._pf == null) {\n            m._pf = defaultParsingFlags();\n        }\n        return m._pf;\n    }\n\n    var some;\n    if (Array.prototype.some) {\n        some = Array.prototype.some;\n    } else {\n        some = function (fun) {\n            var t = Object(this);\n            var len = t.length >>> 0;\n\n            for (var i = 0; i < len; i++) {\n                if (i in t && fun.call(this, t[i], i, t)) {\n                    return true;\n                }\n            }\n\n            return false;\n        };\n    }\n\n    function isValid(m) {\n        if (m._isValid == null) {\n            var flags = getParsingFlags(m);\n            var parsedParts = some.call(flags.parsedDateParts, function (i) {\n                return i != null;\n            });\n            var isNowValid = !isNaN(m._d.getTime()) &&\n                flags.overflow < 0 &&\n                !flags.empty &&\n                !flags.invalidMonth &&\n                !flags.invalidWeekday &&\n                !flags.weekdayMismatch &&\n                !flags.nullInput &&\n                !flags.invalidFormat &&\n                !flags.userInvalidated &&\n                (!flags.meridiem || (flags.meridiem && parsedParts));\n\n            if (m._strict) {\n                isNowValid = isNowValid &&\n                    flags.charsLeftOver === 0 &&\n                    flags.unusedTokens.length === 0 &&\n                    flags.bigHour === undefined;\n            }\n\n            if (Object.isFrozen == null || !Object.isFrozen(m)) {\n                m._isValid = isNowValid;\n            }\n            else {\n                return isNowValid;\n            }\n        }\n        return m._isValid;\n    }\n\n    function createInvalid (flags) {\n        var m = createUTC(NaN);\n        if (flags != null) {\n            extend(getParsingFlags(m), flags);\n        }\n        else {\n            getParsingFlags(m).userInvalidated = true;\n        }\n\n        return m;\n    }\n\n    // Plugins that add properties should also add the key here (null value),\n    // so we can properly clone ourselves.\n    var momentProperties = hooks.momentProperties = [];\n\n    function copyConfig(to, from) {\n        var i, prop, val;\n\n        if (!isUndefined(from._isAMomentObject)) {\n            to._isAMomentObject = from._isAMomentObject;\n        }\n        if (!isUndefined(from._i)) {\n            to._i = from._i;\n        }\n        if (!isUndefined(from._f)) {\n            to._f = from._f;\n        }\n        if (!isUndefined(from._l)) {\n            to._l = from._l;\n        }\n        if (!isUndefined(from._strict)) {\n            to._strict = from._strict;\n        }\n        if (!isUndefined(from._tzm)) {\n            to._tzm = from._tzm;\n        }\n        if (!isUndefined(from._isUTC)) {\n            to._isUTC = from._isUTC;\n        }\n        if (!isUndefined(from._offset)) {\n            to._offset = from._offset;\n        }\n        if (!isUndefined(from._pf)) {\n            to._pf = getParsingFlags(from);\n        }\n        if (!isUndefined(from._locale)) {\n            to._locale = from._locale;\n        }\n\n        if (momentProperties.length > 0) {\n            for (i = 0; i < momentProperties.length; i++) {\n                prop = momentProperties[i];\n                val = from[prop];\n                if (!isUndefined(val)) {\n                    to[prop] = val;\n                }\n            }\n        }\n\n        return to;\n    }\n\n    var updateInProgress = false;\n\n    // Moment prototype object\n    function Moment(config) {\n        copyConfig(this, config);\n        this._d = new Date(config._d != null ? config._d.getTime() : NaN);\n        if (!this.isValid()) {\n            this._d = new Date(NaN);\n        }\n        // Prevent infinite loop in case updateOffset creates new moment\n        // objects.\n        if (updateInProgress === false) {\n            updateInProgress = true;\n            hooks.updateOffset(this);\n            updateInProgress = false;\n        }\n    }\n\n    function isMoment (obj) {\n        return obj instanceof Moment || (obj != null && obj._isAMomentObject != null);\n    }\n\n    function absFloor (number) {\n        if (number < 0) {\n            // -0 -> 0\n            return Math.ceil(number) || 0;\n        } else {\n            return Math.floor(number);\n        }\n    }\n\n    function toInt(argumentForCoercion) {\n        var coercedNumber = +argumentForCoercion,\n            value = 0;\n\n        if (coercedNumber !== 0 && isFinite(coercedNumber)) {\n            value = absFloor(coercedNumber);\n        }\n\n        return value;\n    }\n\n    // compare two arrays, return the number of differences\n    function compareArrays(array1, array2, dontConvert) {\n        var len = Math.min(array1.length, array2.length),\n            lengthDiff = Math.abs(array1.length - array2.length),\n            diffs = 0,\n            i;\n        for (i = 0; i < len; i++) {\n            if ((dontConvert && array1[i] !== array2[i]) ||\n                (!dontConvert && toInt(array1[i]) !== toInt(array2[i]))) {\n                diffs++;\n            }\n        }\n        return diffs + lengthDiff;\n    }\n\n    function warn(msg) {\n        if (hooks.suppressDeprecationWarnings === false &&\n                (typeof console !==  'undefined') && console.warn) {\n            console.warn('Deprecation warning: ' + msg);\n        }\n    }\n\n    function deprecate(msg, fn) {\n        var firstTime = true;\n\n        return extend(function () {\n            if (hooks.deprecationHandler != null) {\n                hooks.deprecationHandler(null, msg);\n            }\n            if (firstTime) {\n                var args = [];\n                var arg;\n                for (var i = 0; i < arguments.length; i++) {\n                    arg = '';\n                    if (typeof arguments[i] === 'object') {\n                        arg += '\\n[' + i + '] ';\n                        for (var key in arguments[0]) {\n                            arg += key + ': ' + arguments[0][key] + ', ';\n                        }\n                        arg = arg.slice(0, -2); // Remove trailing comma and space\n                    } else {\n                        arg = arguments[i];\n                    }\n                    args.push(arg);\n                }\n                warn(msg + '\\nArguments: ' + Array.prototype.slice.call(args).join('') + '\\n' + (new Error()).stack);\n                firstTime = false;\n            }\n            return fn.apply(this, arguments);\n        }, fn);\n    }\n\n    var deprecations = {};\n\n    function deprecateSimple(name, msg) {\n        if (hooks.deprecationHandler != null) {\n            hooks.deprecationHandler(name, msg);\n        }\n        if (!deprecations[name]) {\n            warn(msg);\n            deprecations[name] = true;\n        }\n    }\n\n    hooks.suppressDeprecationWarnings = false;\n    hooks.deprecationHandler = null;\n\n    function isFunction(input) {\n        return input instanceof Function || Object.prototype.toString.call(input) === '[object Function]';\n    }\n\n    function set (config) {\n        var prop, i;\n        for (i in config) {\n            prop = config[i];\n            if (isFunction(prop)) {\n                this[i] = prop;\n            } else {\n                this['_' + i] = prop;\n            }\n        }\n        this._config = config;\n        // Lenient ordinal parsing accepts just a number in addition to\n        // number + (possibly) stuff coming from _dayOfMonthOrdinalParse.\n        // TODO: Remove \"ordinalParse\" fallback in next major release.\n        this._dayOfMonthOrdinalParseLenient = new RegExp(\n            (this._dayOfMonthOrdinalParse.source || this._ordinalParse.source) +\n                '|' + (/\\d{1,2}/).source);\n    }\n\n    function mergeConfigs(parentConfig, childConfig) {\n        var res = extend({}, parentConfig), prop;\n        for (prop in childConfig) {\n            if (hasOwnProp(childConfig, prop)) {\n                if (isObject(parentConfig[prop]) && isObject(childConfig[prop])) {\n                    res[prop] = {};\n                    extend(res[prop], parentConfig[prop]);\n                    extend(res[prop], childConfig[prop]);\n                } else if (childConfig[prop] != null) {\n                    res[prop] = childConfig[prop];\n                } else {\n                    delete res[prop];\n                }\n            }\n        }\n        for (prop in parentConfig) {\n            if (hasOwnProp(parentConfig, prop) &&\n                    !hasOwnProp(childConfig, prop) &&\n                    isObject(parentConfig[prop])) {\n                // make sure changes to properties don't modify parent config\n                res[prop] = extend({}, res[prop]);\n            }\n        }\n        return res;\n    }\n\n    function Locale(config) {\n        if (config != null) {\n            this.set(config);\n        }\n    }\n\n    var keys;\n\n    if (Object.keys) {\n        keys = Object.keys;\n    } else {\n        keys = function (obj) {\n            var i, res = [];\n            for (i in obj) {\n                if (hasOwnProp(obj, i)) {\n                    res.push(i);\n                }\n            }\n            return res;\n        };\n    }\n\n    var defaultCalendar = {\n        sameDay : '[Today at] LT',\n        nextDay : '[Tomorrow at] LT',\n        nextWeek : 'dddd [at] LT',\n        lastDay : '[Yesterday at] LT',\n        lastWeek : '[Last] dddd [at] LT',\n        sameElse : 'L'\n    };\n\n    function calendar (key, mom, now) {\n        var output = this._calendar[key] || this._calendar['sameElse'];\n        return isFunction(output) ? output.call(mom, now) : output;\n    }\n\n    var defaultLongDateFormat = {\n        LTS  : 'h:mm:ss A',\n        LT   : 'h:mm A',\n        L    : 'MM/DD/YYYY',\n        LL   : 'MMMM D, YYYY',\n        LLL  : 'MMMM D, YYYY h:mm A',\n        LLLL : 'dddd, MMMM D, YYYY h:mm A'\n    };\n\n    function longDateFormat (key) {\n        var format = this._longDateFormat[key],\n            formatUpper = this._longDateFormat[key.toUpperCase()];\n\n        if (format || !formatUpper) {\n            return format;\n        }\n\n        this._longDateFormat[key] = formatUpper.replace(/MMMM|MM|DD|dddd/g, function (val) {\n            return val.slice(1);\n        });\n\n        return this._longDateFormat[key];\n    }\n\n    var defaultInvalidDate = 'Invalid date';\n\n    function invalidDate () {\n        return this._invalidDate;\n    }\n\n    var defaultOrdinal = '%d';\n    var defaultDayOfMonthOrdinalParse = /\\d{1,2}/;\n\n    function ordinal (number) {\n        return this._ordinal.replace('%d', number);\n    }\n\n    var defaultRelativeTime = {\n        future : 'in %s',\n        past   : '%s ago',\n        s  : 'a few seconds',\n        ss : '%d seconds',\n        m  : 'a minute',\n        mm : '%d minutes',\n        h  : 'an hour',\n        hh : '%d hours',\n        d  : 'a day',\n        dd : '%d days',\n        M  : 'a month',\n        MM : '%d months',\n        y  : 'a year',\n        yy : '%d years'\n    };\n\n    function relativeTime (number, withoutSuffix, string, isFuture) {\n        var output = this._relativeTime[string];\n        return (isFunction(output)) ?\n            output(number, withoutSuffix, string, isFuture) :\n            output.replace(/%d/i, number);\n    }\n\n    function pastFuture (diff, output) {\n        var format = this._relativeTime[diff > 0 ? 'future' : 'past'];\n        return isFunction(format) ? format(output) : format.replace(/%s/i, output);\n    }\n\n    var aliases = {};\n\n    function addUnitAlias (unit, shorthand) {\n        var lowerCase = unit.toLowerCase();\n        aliases[lowerCase] = aliases[lowerCase + 's'] = aliases[shorthand] = unit;\n    }\n\n    function normalizeUnits(units) {\n        return typeof units === 'string' ? aliases[units] || aliases[units.toLowerCase()] : undefined;\n    }\n\n    function normalizeObjectUnits(inputObject) {\n        var normalizedInput = {},\n            normalizedProp,\n            prop;\n\n        for (prop in inputObject) {\n            if (hasOwnProp(inputObject, prop)) {\n                normalizedProp = normalizeUnits(prop);\n                if (normalizedProp) {\n                    normalizedInput[normalizedProp] = inputObject[prop];\n                }\n            }\n        }\n\n        return normalizedInput;\n    }\n\n    var priorities = {};\n\n    function addUnitPriority(unit, priority) {\n        priorities[unit] = priority;\n    }\n\n    function getPrioritizedUnits(unitsObj) {\n        var units = [];\n        for (var u in unitsObj) {\n            units.push({unit: u, priority: priorities[u]});\n        }\n        units.sort(function (a, b) {\n            return a.priority - b.priority;\n        });\n        return units;\n    }\n\n    function zeroFill(number, targetLength, forceSign) {\n        var absNumber = '' + Math.abs(number),\n            zerosToFill = targetLength - absNumber.length,\n            sign = number >= 0;\n        return (sign ? (forceSign ? '+' : '') : '-') +\n            Math.pow(10, Math.max(0, zerosToFill)).toString().substr(1) + absNumber;\n    }\n\n    var formattingTokens = /(\\[[^\\[]*\\])|(\\\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g;\n\n    var localFormattingTokens = /(\\[[^\\[]*\\])|(\\\\)?(LTS|LT|LL?L?L?|l{1,4})/g;\n\n    var formatFunctions = {};\n\n    var formatTokenFunctions = {};\n\n    // token:    'M'\n    // padded:   ['MM', 2]\n    // ordinal:  'Mo'\n    // callback: function () { this.month() + 1 }\n    function addFormatToken (token, padded, ordinal, callback) {\n        var func = callback;\n        if (typeof callback === 'string') {\n            func = function () {\n                return this[callback]();\n            };\n        }\n        if (token) {\n            formatTokenFunctions[token] = func;\n        }\n        if (padded) {\n            formatTokenFunctions[padded[0]] = function () {\n                return zeroFill(func.apply(this, arguments), padded[1], padded[2]);\n            };\n        }\n        if (ordinal) {\n            formatTokenFunctions[ordinal] = function () {\n                return this.localeData().ordinal(func.apply(this, arguments), token);\n            };\n        }\n    }\n\n    function removeFormattingTokens(input) {\n        if (input.match(/\\[[\\s\\S]/)) {\n            return input.replace(/^\\[|\\]$/g, '');\n        }\n        return input.replace(/\\\\/g, '');\n    }\n\n    function makeFormatFunction(format) {\n        var array = format.match(formattingTokens), i, length;\n\n        for (i = 0, length = array.length; i < length; i++) {\n            if (formatTokenFunctions[array[i]]) {\n                array[i] = formatTokenFunctions[array[i]];\n            } else {\n                array[i] = removeFormattingTokens(array[i]);\n            }\n        }\n\n        return function (mom) {\n            var output = '', i;\n            for (i = 0; i < length; i++) {\n                output += isFunction(array[i]) ? array[i].call(mom, format) : array[i];\n            }\n            return output;\n        };\n    }\n\n    // format date using native date object\n    function formatMoment(m, format) {\n        if (!m.isValid()) {\n            return m.localeData().invalidDate();\n        }\n\n        format = expandFormat(format, m.localeData());\n        formatFunctions[format] = formatFunctions[format] || makeFormatFunction(format);\n\n        return formatFunctions[format](m);\n    }\n\n    function expandFormat(format, locale) {\n        var i = 5;\n\n        function replaceLongDateFormatTokens(input) {\n            return locale.longDateFormat(input) || input;\n        }\n\n        localFormattingTokens.lastIndex = 0;\n        while (i >= 0 && localFormattingTokens.test(format)) {\n            format = format.replace(localFormattingTokens, replaceLongDateFormatTokens);\n            localFormattingTokens.lastIndex = 0;\n            i -= 1;\n        }\n\n        return format;\n    }\n\n    var match1         = /\\d/;            //       0 - 9\n    var match2         = /\\d\\d/;          //      00 - 99\n    var match3         = /\\d{3}/;         //     000 - 999\n    var match4         = /\\d{4}/;         //    0000 - 9999\n    var match6         = /[+-]?\\d{6}/;    // -999999 - 999999\n    var match1to2      = /\\d\\d?/;         //       0 - 99\n    var match3to4      = /\\d\\d\\d\\d?/;     //     999 - 9999\n    var match5to6      = /\\d\\d\\d\\d\\d\\d?/; //   99999 - 999999\n    var match1to3      = /\\d{1,3}/;       //       0 - 999\n    var match1to4      = /\\d{1,4}/;       //       0 - 9999\n    var match1to6      = /[+-]?\\d{1,6}/;  // -999999 - 999999\n\n    var matchUnsigned  = /\\d+/;           //       0 - inf\n    var matchSigned    = /[+-]?\\d+/;      //    -inf - inf\n\n    var matchOffset    = /Z|[+-]\\d\\d:?\\d\\d/gi; // +00:00 -00:00 +0000 -0000 or Z\n    var matchShortOffset = /Z|[+-]\\d\\d(?::?\\d\\d)?/gi; // +00 -00 +00:00 -00:00 +0000 -0000 or Z\n\n    var matchTimestamp = /[+-]?\\d+(\\.\\d{1,3})?/; // 123456789 123456789.123\n\n    // any word (or two) characters or numbers including two/three word month in arabic.\n    // includes scottish gaelic two word and hyphenated months\n    var matchWord = /[0-9]{0,256}['a-z\\u00A0-\\u05FF\\u0700-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFF07\\uFF10-\\uFFEF]{1,256}|[\\u0600-\\u06FF\\/]{1,256}(\\s*?[\\u0600-\\u06FF]{1,256}){1,2}/i;\n\n    var regexes = {};\n\n    function addRegexToken (token, regex, strictRegex) {\n        regexes[token] = isFunction(regex) ? regex : function (isStrict, localeData) {\n            return (isStrict && strictRegex) ? strictRegex : regex;\n        };\n    }\n\n    function getParseRegexForToken (token, config) {\n        if (!hasOwnProp(regexes, token)) {\n            return new RegExp(unescapeFormat(token));\n        }\n\n        return regexes[token](config._strict, config._locale);\n    }\n\n    // Code from http://stackoverflow.com/questions/3561493/is-there-a-regexp-escape-function-in-javascript\n    function unescapeFormat(s) {\n        return regexEscape(s.replace('\\\\', '').replace(/\\\\(\\[)|\\\\(\\])|\\[([^\\]\\[]*)\\]|\\\\(.)/g, function (matched, p1, p2, p3, p4) {\n            return p1 || p2 || p3 || p4;\n        }));\n    }\n\n    function regexEscape(s) {\n        return s.replace(/[-\\/\\\\^$*+?.()|[\\]{}]/g, '\\\\$&');\n    }\n\n    var tokens = {};\n\n    function addParseToken (token, callback) {\n        var i, func = callback;\n        if (typeof token === 'string') {\n            token = [token];\n        }\n        if (isNumber(callback)) {\n            func = function (input, array) {\n                array[callback] = toInt(input);\n            };\n        }\n        for (i = 0; i < token.length; i++) {\n            tokens[token[i]] = func;\n        }\n    }\n\n    function addWeekParseToken (token, callback) {\n        addParseToken(token, function (input, array, config, token) {\n            config._w = config._w || {};\n            callback(input, config._w, config, token);\n        });\n    }\n\n    function addTimeToArrayFromToken(token, input, config) {\n        if (input != null && hasOwnProp(tokens, token)) {\n            tokens[token](input, config._a, config, token);\n        }\n    }\n\n    var YEAR = 0;\n    var MONTH = 1;\n    var DATE = 2;\n    var HOUR = 3;\n    var MINUTE = 4;\n    var SECOND = 5;\n    var MILLISECOND = 6;\n    var WEEK = 7;\n    var WEEKDAY = 8;\n\n    // FORMATTING\n\n    addFormatToken('Y', 0, 0, function () {\n        var y = this.year();\n        return y <= 9999 ? '' + y : '+' + y;\n    });\n\n    addFormatToken(0, ['YY', 2], 0, function () {\n        return this.year() % 100;\n    });\n\n    addFormatToken(0, ['YYYY',   4],       0, 'year');\n    addFormatToken(0, ['YYYYY',  5],       0, 'year');\n    addFormatToken(0, ['YYYYYY', 6, true], 0, 'year');\n\n    // ALIASES\n\n    addUnitAlias('year', 'y');\n\n    // PRIORITIES\n\n    addUnitPriority('year', 1);\n\n    // PARSING\n\n    addRegexToken('Y',      matchSigned);\n    addRegexToken('YY',     match1to2, match2);\n    addRegexToken('YYYY',   match1to4, match4);\n    addRegexToken('YYYYY',  match1to6, match6);\n    addRegexToken('YYYYYY', match1to6, match6);\n\n    addParseToken(['YYYYY', 'YYYYYY'], YEAR);\n    addParseToken('YYYY', function (input, array) {\n        array[YEAR] = input.length === 2 ? hooks.parseTwoDigitYear(input) : toInt(input);\n    });\n    addParseToken('YY', function (input, array) {\n        array[YEAR] = hooks.parseTwoDigitYear(input);\n    });\n    addParseToken('Y', function (input, array) {\n        array[YEAR] = parseInt(input, 10);\n    });\n\n    // HELPERS\n\n    function daysInYear(year) {\n        return isLeapYear(year) ? 366 : 365;\n    }\n\n    function isLeapYear(year) {\n        return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0;\n    }\n\n    // HOOKS\n\n    hooks.parseTwoDigitYear = function (input) {\n        return toInt(input) + (toInt(input) > 68 ? 1900 : 2000);\n    };\n\n    // MOMENTS\n\n    var getSetYear = makeGetSet('FullYear', true);\n\n    function getIsLeapYear () {\n        return isLeapYear(this.year());\n    }\n\n    function makeGetSet (unit, keepTime) {\n        return function (value) {\n            if (value != null) {\n                set$1(this, unit, value);\n                hooks.updateOffset(this, keepTime);\n                return this;\n            } else {\n                return get(this, unit);\n            }\n        };\n    }\n\n    function get (mom, unit) {\n        return mom.isValid() ?\n            mom._d['get' + (mom._isUTC ? 'UTC' : '') + unit]() : NaN;\n    }\n\n    function set$1 (mom, unit, value) {\n        if (mom.isValid() && !isNaN(value)) {\n            if (unit === 'FullYear' && isLeapYear(mom.year()) && mom.month() === 1 && mom.date() === 29) {\n                mom._d['set' + (mom._isUTC ? 'UTC' : '') + unit](value, mom.month(), daysInMonth(value, mom.month()));\n            }\n            else {\n                mom._d['set' + (mom._isUTC ? 'UTC' : '') + unit](value);\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function stringGet (units) {\n        units = normalizeUnits(units);\n        if (isFunction(this[units])) {\n            return this[units]();\n        }\n        return this;\n    }\n\n\n    function stringSet (units, value) {\n        if (typeof units === 'object') {\n            units = normalizeObjectUnits(units);\n            var prioritized = getPrioritizedUnits(units);\n            for (var i = 0; i < prioritized.length; i++) {\n                this[prioritized[i].unit](units[prioritized[i].unit]);\n            }\n        } else {\n            units = normalizeUnits(units);\n            if (isFunction(this[units])) {\n                return this[units](value);\n            }\n        }\n        return this;\n    }\n\n    function mod(n, x) {\n        return ((n % x) + x) % x;\n    }\n\n    var indexOf;\n\n    if (Array.prototype.indexOf) {\n        indexOf = Array.prototype.indexOf;\n    } else {\n        indexOf = function (o) {\n            // I know\n            var i;\n            for (i = 0; i < this.length; ++i) {\n                if (this[i] === o) {\n                    return i;\n                }\n            }\n            return -1;\n        };\n    }\n\n    function daysInMonth(year, month) {\n        if (isNaN(year) || isNaN(month)) {\n            return NaN;\n        }\n        var modMonth = mod(month, 12);\n        year += (month - modMonth) / 12;\n        return modMonth === 1 ? (isLeapYear(year) ? 29 : 28) : (31 - modMonth % 7 % 2);\n    }\n\n    // FORMATTING\n\n    addFormatToken('M', ['MM', 2], 'Mo', function () {\n        return this.month() + 1;\n    });\n\n    addFormatToken('MMM', 0, 0, function (format) {\n        return this.localeData().monthsShort(this, format);\n    });\n\n    addFormatToken('MMMM', 0, 0, function (format) {\n        return this.localeData().months(this, format);\n    });\n\n    // ALIASES\n\n    addUnitAlias('month', 'M');\n\n    // PRIORITY\n\n    addUnitPriority('month', 8);\n\n    // PARSING\n\n    addRegexToken('M',    match1to2);\n    addRegexToken('MM',   match1to2, match2);\n    addRegexToken('MMM',  function (isStrict, locale) {\n        return locale.monthsShortRegex(isStrict);\n    });\n    addRegexToken('MMMM', function (isStrict, locale) {\n        return locale.monthsRegex(isStrict);\n    });\n\n    addParseToken(['M', 'MM'], function (input, array) {\n        array[MONTH] = toInt(input) - 1;\n    });\n\n    addParseToken(['MMM', 'MMMM'], function (input, array, config, token) {\n        var month = config._locale.monthsParse(input, token, config._strict);\n        // if we didn't find a month name, mark the date as invalid.\n        if (month != null) {\n            array[MONTH] = month;\n        } else {\n            getParsingFlags(config).invalidMonth = input;\n        }\n    });\n\n    // LOCALES\n\n    var MONTHS_IN_FORMAT = /D[oD]?(\\[[^\\[\\]]*\\]|\\s)+MMMM?/;\n    var defaultLocaleMonths = 'January_February_March_April_May_June_July_August_September_October_November_December'.split('_');\n    function localeMonths (m, format) {\n        if (!m) {\n            return isArray(this._months) ? this._months :\n                this._months['standalone'];\n        }\n        return isArray(this._months) ? this._months[m.month()] :\n            this._months[(this._months.isFormat || MONTHS_IN_FORMAT).test(format) ? 'format' : 'standalone'][m.month()];\n    }\n\n    var defaultLocaleMonthsShort = 'Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec'.split('_');\n    function localeMonthsShort (m, format) {\n        if (!m) {\n            return isArray(this._monthsShort) ? this._monthsShort :\n                this._monthsShort['standalone'];\n        }\n        return isArray(this._monthsShort) ? this._monthsShort[m.month()] :\n            this._monthsShort[MONTHS_IN_FORMAT.test(format) ? 'format' : 'standalone'][m.month()];\n    }\n\n    function handleStrictParse(monthName, format, strict) {\n        var i, ii, mom, llc = monthName.toLocaleLowerCase();\n        if (!this._monthsParse) {\n            // this is not used\n            this._monthsParse = [];\n            this._longMonthsParse = [];\n            this._shortMonthsParse = [];\n            for (i = 0; i < 12; ++i) {\n                mom = createUTC([2000, i]);\n                this._shortMonthsParse[i] = this.monthsShort(mom, '').toLocaleLowerCase();\n                this._longMonthsParse[i] = this.months(mom, '').toLocaleLowerCase();\n            }\n        }\n\n        if (strict) {\n            if (format === 'MMM') {\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._longMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        } else {\n            if (format === 'MMM') {\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._longMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._longMonthsParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        }\n    }\n\n    function localeMonthsParse (monthName, format, strict) {\n        var i, mom, regex;\n\n        if (this._monthsParseExact) {\n            return handleStrictParse.call(this, monthName, format, strict);\n        }\n\n        if (!this._monthsParse) {\n            this._monthsParse = [];\n            this._longMonthsParse = [];\n            this._shortMonthsParse = [];\n        }\n\n        // TODO: add sorting\n        // Sorting makes sure if one month (or abbr) is a prefix of another\n        // see sorting in computeMonthsParse\n        for (i = 0; i < 12; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, i]);\n            if (strict && !this._longMonthsParse[i]) {\n                this._longMonthsParse[i] = new RegExp('^' + this.months(mom, '').replace('.', '') + '$', 'i');\n                this._shortMonthsParse[i] = new RegExp('^' + this.monthsShort(mom, '').replace('.', '') + '$', 'i');\n            }\n            if (!strict && !this._monthsParse[i]) {\n                regex = '^' + this.months(mom, '') + '|^' + this.monthsShort(mom, '');\n                this._monthsParse[i] = new RegExp(regex.replace('.', ''), 'i');\n            }\n            // test the regex\n            if (strict && format === 'MMMM' && this._longMonthsParse[i].test(monthName)) {\n                return i;\n            } else if (strict && format === 'MMM' && this._shortMonthsParse[i].test(monthName)) {\n                return i;\n            } else if (!strict && this._monthsParse[i].test(monthName)) {\n                return i;\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function setMonth (mom, value) {\n        var dayOfMonth;\n\n        if (!mom.isValid()) {\n            // No op\n            return mom;\n        }\n\n        if (typeof value === 'string') {\n            if (/^\\d+$/.test(value)) {\n                value = toInt(value);\n            } else {\n                value = mom.localeData().monthsParse(value);\n                // TODO: Another silent failure?\n                if (!isNumber(value)) {\n                    return mom;\n                }\n            }\n        }\n\n        dayOfMonth = Math.min(mom.date(), daysInMonth(mom.year(), value));\n        mom._d['set' + (mom._isUTC ? 'UTC' : '') + 'Month'](value, dayOfMonth);\n        return mom;\n    }\n\n    function getSetMonth (value) {\n        if (value != null) {\n            setMonth(this, value);\n            hooks.updateOffset(this, true);\n            return this;\n        } else {\n            return get(this, 'Month');\n        }\n    }\n\n    function getDaysInMonth () {\n        return daysInMonth(this.year(), this.month());\n    }\n\n    var defaultMonthsShortRegex = matchWord;\n    function monthsShortRegex (isStrict) {\n        if (this._monthsParseExact) {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                computeMonthsParse.call(this);\n            }\n            if (isStrict) {\n                return this._monthsShortStrictRegex;\n            } else {\n                return this._monthsShortRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_monthsShortRegex')) {\n                this._monthsShortRegex = defaultMonthsShortRegex;\n            }\n            return this._monthsShortStrictRegex && isStrict ?\n                this._monthsShortStrictRegex : this._monthsShortRegex;\n        }\n    }\n\n    var defaultMonthsRegex = matchWord;\n    function monthsRegex (isStrict) {\n        if (this._monthsParseExact) {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                computeMonthsParse.call(this);\n            }\n            if (isStrict) {\n                return this._monthsStrictRegex;\n            } else {\n                return this._monthsRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                this._monthsRegex = defaultMonthsRegex;\n            }\n            return this._monthsStrictRegex && isStrict ?\n                this._monthsStrictRegex : this._monthsRegex;\n        }\n    }\n\n    function computeMonthsParse () {\n        function cmpLenRev(a, b) {\n            return b.length - a.length;\n        }\n\n        var shortPieces = [], longPieces = [], mixedPieces = [],\n            i, mom;\n        for (i = 0; i < 12; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, i]);\n            shortPieces.push(this.monthsShort(mom, ''));\n            longPieces.push(this.months(mom, ''));\n            mixedPieces.push(this.months(mom, ''));\n            mixedPieces.push(this.monthsShort(mom, ''));\n        }\n        // Sorting makes sure if one month (or abbr) is a prefix of another it\n        // will match the longer piece.\n        shortPieces.sort(cmpLenRev);\n        longPieces.sort(cmpLenRev);\n        mixedPieces.sort(cmpLenRev);\n        for (i = 0; i < 12; i++) {\n            shortPieces[i] = regexEscape(shortPieces[i]);\n            longPieces[i] = regexEscape(longPieces[i]);\n        }\n        for (i = 0; i < 24; i++) {\n            mixedPieces[i] = regexEscape(mixedPieces[i]);\n        }\n\n        this._monthsRegex = new RegExp('^(' + mixedPieces.join('|') + ')', 'i');\n        this._monthsShortRegex = this._monthsRegex;\n        this._monthsStrictRegex = new RegExp('^(' + longPieces.join('|') + ')', 'i');\n        this._monthsShortStrictRegex = new RegExp('^(' + shortPieces.join('|') + ')', 'i');\n    }\n\n    function createDate (y, m, d, h, M, s, ms) {\n        // can't just apply() to create a date:\n        // https://stackoverflow.com/q/181348\n        var date;\n        // the date constructor remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            date = new Date(y + 400, m, d, h, M, s, ms);\n            if (isFinite(date.getFullYear())) {\n                date.setFullYear(y);\n            }\n        } else {\n            date = new Date(y, m, d, h, M, s, ms);\n        }\n\n        return date;\n    }\n\n    function createUTCDate (y) {\n        var date;\n        // the Date.UTC function remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            var args = Array.prototype.slice.call(arguments);\n            // preserve leap years using a full 400 year cycle, then reset\n            args[0] = y + 400;\n            date = new Date(Date.UTC.apply(null, args));\n            if (isFinite(date.getUTCFullYear())) {\n                date.setUTCFullYear(y);\n            }\n        } else {\n            date = new Date(Date.UTC.apply(null, arguments));\n        }\n\n        return date;\n    }\n\n    // start-of-first-week - start-of-year\n    function firstWeekOffset(year, dow, doy) {\n        var // first-week day -- which january is always in the first week (4 for iso, 1 for other)\n            fwd = 7 + dow - doy,\n            // first-week day local weekday -- which local weekday is fwd\n            fwdlw = (7 + createUTCDate(year, 0, fwd).getUTCDay() - dow) % 7;\n\n        return -fwdlw + fwd - 1;\n    }\n\n    // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_a_date_given_the_year.2C_week_number_and_weekday\n    function dayOfYearFromWeeks(year, week, weekday, dow, doy) {\n        var localWeekday = (7 + weekday - dow) % 7,\n            weekOffset = firstWeekOffset(year, dow, doy),\n            dayOfYear = 1 + 7 * (week - 1) + localWeekday + weekOffset,\n            resYear, resDayOfYear;\n\n        if (dayOfYear <= 0) {\n            resYear = year - 1;\n            resDayOfYear = daysInYear(resYear) + dayOfYear;\n        } else if (dayOfYear > daysInYear(year)) {\n            resYear = year + 1;\n            resDayOfYear = dayOfYear - daysInYear(year);\n        } else {\n            resYear = year;\n            resDayOfYear = dayOfYear;\n        }\n\n        return {\n            year: resYear,\n            dayOfYear: resDayOfYear\n        };\n    }\n\n    function weekOfYear(mom, dow, doy) {\n        var weekOffset = firstWeekOffset(mom.year(), dow, doy),\n            week = Math.floor((mom.dayOfYear() - weekOffset - 1) / 7) + 1,\n            resWeek, resYear;\n\n        if (week < 1) {\n            resYear = mom.year() - 1;\n            resWeek = week + weeksInYear(resYear, dow, doy);\n        } else if (week > weeksInYear(mom.year(), dow, doy)) {\n            resWeek = week - weeksInYear(mom.year(), dow, doy);\n            resYear = mom.year() + 1;\n        } else {\n            resYear = mom.year();\n            resWeek = week;\n        }\n\n        return {\n            week: resWeek,\n            year: resYear\n        };\n    }\n\n    function weeksInYear(year, dow, doy) {\n        var weekOffset = firstWeekOffset(year, dow, doy),\n            weekOffsetNext = firstWeekOffset(year + 1, dow, doy);\n        return (daysInYear(year) - weekOffset + weekOffsetNext) / 7;\n    }\n\n    // FORMATTING\n\n    addFormatToken('w', ['ww', 2], 'wo', 'week');\n    addFormatToken('W', ['WW', 2], 'Wo', 'isoWeek');\n\n    // ALIASES\n\n    addUnitAlias('week', 'w');\n    addUnitAlias('isoWeek', 'W');\n\n    // PRIORITIES\n\n    addUnitPriority('week', 5);\n    addUnitPriority('isoWeek', 5);\n\n    // PARSING\n\n    addRegexToken('w',  match1to2);\n    addRegexToken('ww', match1to2, match2);\n    addRegexToken('W',  match1to2);\n    addRegexToken('WW', match1to2, match2);\n\n    addWeekParseToken(['w', 'ww', 'W', 'WW'], function (input, week, config, token) {\n        week[token.substr(0, 1)] = toInt(input);\n    });\n\n    // HELPERS\n\n    // LOCALES\n\n    function localeWeek (mom) {\n        return weekOfYear(mom, this._week.dow, this._week.doy).week;\n    }\n\n    var defaultLocaleWeek = {\n        dow : 0, // Sunday is the first day of the week.\n        doy : 6  // The week that contains Jan 6th is the first week of the year.\n    };\n\n    function localeFirstDayOfWeek () {\n        return this._week.dow;\n    }\n\n    function localeFirstDayOfYear () {\n        return this._week.doy;\n    }\n\n    // MOMENTS\n\n    function getSetWeek (input) {\n        var week = this.localeData().week(this);\n        return input == null ? week : this.add((input - week) * 7, 'd');\n    }\n\n    function getSetISOWeek (input) {\n        var week = weekOfYear(this, 1, 4).week;\n        return input == null ? week : this.add((input - week) * 7, 'd');\n    }\n\n    // FORMATTING\n\n    addFormatToken('d', 0, 'do', 'day');\n\n    addFormatToken('dd', 0, 0, function (format) {\n        return this.localeData().weekdaysMin(this, format);\n    });\n\n    addFormatToken('ddd', 0, 0, function (format) {\n        return this.localeData().weekdaysShort(this, format);\n    });\n\n    addFormatToken('dddd', 0, 0, function (format) {\n        return this.localeData().weekdays(this, format);\n    });\n\n    addFormatToken('e', 0, 0, 'weekday');\n    addFormatToken('E', 0, 0, 'isoWeekday');\n\n    // ALIASES\n\n    addUnitAlias('day', 'd');\n    addUnitAlias('weekday', 'e');\n    addUnitAlias('isoWeekday', 'E');\n\n    // PRIORITY\n    addUnitPriority('day', 11);\n    addUnitPriority('weekday', 11);\n    addUnitPriority('isoWeekday', 11);\n\n    // PARSING\n\n    addRegexToken('d',    match1to2);\n    addRegexToken('e',    match1to2);\n    addRegexToken('E',    match1to2);\n    addRegexToken('dd',   function (isStrict, locale) {\n        return locale.weekdaysMinRegex(isStrict);\n    });\n    addRegexToken('ddd',   function (isStrict, locale) {\n        return locale.weekdaysShortRegex(isStrict);\n    });\n    addRegexToken('dddd',   function (isStrict, locale) {\n        return locale.weekdaysRegex(isStrict);\n    });\n\n    addWeekParseToken(['dd', 'ddd', 'dddd'], function (input, week, config, token) {\n        var weekday = config._locale.weekdaysParse(input, token, config._strict);\n        // if we didn't get a weekday name, mark the date as invalid\n        if (weekday != null) {\n            week.d = weekday;\n        } else {\n            getParsingFlags(config).invalidWeekday = input;\n        }\n    });\n\n    addWeekParseToken(['d', 'e', 'E'], function (input, week, config, token) {\n        week[token] = toInt(input);\n    });\n\n    // HELPERS\n\n    function parseWeekday(input, locale) {\n        if (typeof input !== 'string') {\n            return input;\n        }\n\n        if (!isNaN(input)) {\n            return parseInt(input, 10);\n        }\n\n        input = locale.weekdaysParse(input);\n        if (typeof input === 'number') {\n            return input;\n        }\n\n        return null;\n    }\n\n    function parseIsoWeekday(input, locale) {\n        if (typeof input === 'string') {\n            return locale.weekdaysParse(input) % 7 || 7;\n        }\n        return isNaN(input) ? null : input;\n    }\n\n    // LOCALES\n    function shiftWeekdays (ws, n) {\n        return ws.slice(n, 7).concat(ws.slice(0, n));\n    }\n\n    var defaultLocaleWeekdays = 'Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday'.split('_');\n    function localeWeekdays (m, format) {\n        var weekdays = isArray(this._weekdays) ? this._weekdays :\n            this._weekdays[(m && m !== true && this._weekdays.isFormat.test(format)) ? 'format' : 'standalone'];\n        return (m === true) ? shiftWeekdays(weekdays, this._week.dow)\n            : (m) ? weekdays[m.day()] : weekdays;\n    }\n\n    var defaultLocaleWeekdaysShort = 'Sun_Mon_Tue_Wed_Thu_Fri_Sat'.split('_');\n    function localeWeekdaysShort (m) {\n        return (m === true) ? shiftWeekdays(this._weekdaysShort, this._week.dow)\n            : (m) ? this._weekdaysShort[m.day()] : this._weekdaysShort;\n    }\n\n    var defaultLocaleWeekdaysMin = 'Su_Mo_Tu_We_Th_Fr_Sa'.split('_');\n    function localeWeekdaysMin (m) {\n        return (m === true) ? shiftWeekdays(this._weekdaysMin, this._week.dow)\n            : (m) ? this._weekdaysMin[m.day()] : this._weekdaysMin;\n    }\n\n    function handleStrictParse$1(weekdayName, format, strict) {\n        var i, ii, mom, llc = weekdayName.toLocaleLowerCase();\n        if (!this._weekdaysParse) {\n            this._weekdaysParse = [];\n            this._shortWeekdaysParse = [];\n            this._minWeekdaysParse = [];\n\n            for (i = 0; i < 7; ++i) {\n                mom = createUTC([2000, 1]).day(i);\n                this._minWeekdaysParse[i] = this.weekdaysMin(mom, '').toLocaleLowerCase();\n                this._shortWeekdaysParse[i] = this.weekdaysShort(mom, '').toLocaleLowerCase();\n                this._weekdaysParse[i] = this.weekdays(mom, '').toLocaleLowerCase();\n            }\n        }\n\n        if (strict) {\n            if (format === 'dddd') {\n                ii = indexOf.call(this._weekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else if (format === 'ddd') {\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        } else {\n            if (format === 'dddd') {\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else if (format === 'ddd') {\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        }\n    }\n\n    function localeWeekdaysParse (weekdayName, format, strict) {\n        var i, mom, regex;\n\n        if (this._weekdaysParseExact) {\n            return handleStrictParse$1.call(this, weekdayName, format, strict);\n        }\n\n        if (!this._weekdaysParse) {\n            this._weekdaysParse = [];\n            this._minWeekdaysParse = [];\n            this._shortWeekdaysParse = [];\n            this._fullWeekdaysParse = [];\n        }\n\n        for (i = 0; i < 7; i++) {\n            // make the regex if we don't have it already\n\n            mom = createUTC([2000, 1]).day(i);\n            if (strict && !this._fullWeekdaysParse[i]) {\n                this._fullWeekdaysParse[i] = new RegExp('^' + this.weekdays(mom, '').replace('.', '\\\\.?') + '$', 'i');\n                this._shortWeekdaysParse[i] = new RegExp('^' + this.weekdaysShort(mom, '').replace('.', '\\\\.?') + '$', 'i');\n                this._minWeekdaysParse[i] = new RegExp('^' + this.weekdaysMin(mom, '').replace('.', '\\\\.?') + '$', 'i');\n            }\n            if (!this._weekdaysParse[i]) {\n                regex = '^' + this.weekdays(mom, '') + '|^' + this.weekdaysShort(mom, '') + '|^' + this.weekdaysMin(mom, '');\n                this._weekdaysParse[i] = new RegExp(regex.replace('.', ''), 'i');\n            }\n            // test the regex\n            if (strict && format === 'dddd' && this._fullWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (strict && format === 'ddd' && this._shortWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (strict && format === 'dd' && this._minWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (!strict && this._weekdaysParse[i].test(weekdayName)) {\n                return i;\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function getSetDayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        var day = this._isUTC ? this._d.getUTCDay() : this._d.getDay();\n        if (input != null) {\n            input = parseWeekday(input, this.localeData());\n            return this.add(input - day, 'd');\n        } else {\n            return day;\n        }\n    }\n\n    function getSetLocaleDayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        var weekday = (this.day() + 7 - this.localeData()._week.dow) % 7;\n        return input == null ? weekday : this.add(input - weekday, 'd');\n    }\n\n    function getSetISODayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n\n        // behaves the same as moment#day except\n        // as a getter, returns 7 instead of 0 (1-7 range instead of 0-6)\n        // as a setter, sunday should belong to the previous week.\n\n        if (input != null) {\n            var weekday = parseIsoWeekday(input, this.localeData());\n            return this.day(this.day() % 7 ? weekday : weekday - 7);\n        } else {\n            return this.day() || 7;\n        }\n    }\n\n    var defaultWeekdaysRegex = matchWord;\n    function weekdaysRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysStrictRegex;\n            } else {\n                return this._weekdaysRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                this._weekdaysRegex = defaultWeekdaysRegex;\n            }\n            return this._weekdaysStrictRegex && isStrict ?\n                this._weekdaysStrictRegex : this._weekdaysRegex;\n        }\n    }\n\n    var defaultWeekdaysShortRegex = matchWord;\n    function weekdaysShortRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysShortStrictRegex;\n            } else {\n                return this._weekdaysShortRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysShortRegex')) {\n                this._weekdaysShortRegex = defaultWeekdaysShortRegex;\n            }\n            return this._weekdaysShortStrictRegex && isStrict ?\n                this._weekdaysShortStrictRegex : this._weekdaysShortRegex;\n        }\n    }\n\n    var defaultWeekdaysMinRegex = matchWord;\n    function weekdaysMinRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysMinStrictRegex;\n            } else {\n                return this._weekdaysMinRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysMinRegex')) {\n                this._weekdaysMinRegex = defaultWeekdaysMinRegex;\n            }\n            return this._weekdaysMinStrictRegex && isStrict ?\n                this._weekdaysMinStrictRegex : this._weekdaysMinRegex;\n        }\n    }\n\n\n    function computeWeekdaysParse () {\n        function cmpLenRev(a, b) {\n            return b.length - a.length;\n        }\n\n        var minPieces = [], shortPieces = [], longPieces = [], mixedPieces = [],\n            i, mom, minp, shortp, longp;\n        for (i = 0; i < 7; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, 1]).day(i);\n            minp = this.weekdaysMin(mom, '');\n            shortp = this.weekdaysShort(mom, '');\n            longp = this.weekdays(mom, '');\n            minPieces.push(minp);\n            shortPieces.push(shortp);\n            longPieces.push(longp);\n            mixedPieces.push(minp);\n            mixedPieces.push(shortp);\n            mixedPieces.push(longp);\n        }\n        // Sorting makes sure if one weekday (or abbr) is a prefix of another it\n        // will match the longer piece.\n        minPieces.sort(cmpLenRev);\n        shortPieces.sort(cmpLenRev);\n        longPieces.sort(cmpLenRev);\n        mixedPieces.sort(cmpLenRev);\n        for (i = 0; i < 7; i++) {\n            shortPieces[i] = regexEscape(shortPieces[i]);\n            longPieces[i] = regexEscape(longPieces[i]);\n            mixedPieces[i] = regexEscape(mixedPieces[i]);\n        }\n\n        this._weekdaysRegex = new RegExp('^(' + mixedPieces.join('|') + ')', 'i');\n        this._weekdaysShortRegex = this._weekdaysRegex;\n        this._weekdaysMinRegex = this._weekdaysRegex;\n\n        this._weekdaysStrictRegex = new RegExp('^(' + longPieces.join('|') + ')', 'i');\n        this._weekdaysShortStrictRegex = new RegExp('^(' + shortPieces.join('|') + ')', 'i');\n        this._weekdaysMinStrictRegex = new RegExp('^(' + minPieces.join('|') + ')', 'i');\n    }\n\n    // FORMATTING\n\n    function hFormat() {\n        return this.hours() % 12 || 12;\n    }\n\n    function kFormat() {\n        return this.hours() || 24;\n    }\n\n    addFormatToken('H', ['HH', 2], 0, 'hour');\n    addFormatToken('h', ['hh', 2], 0, hFormat);\n    addFormatToken('k', ['kk', 2], 0, kFormat);\n\n    addFormatToken('hmm', 0, 0, function () {\n        return '' + hFormat.apply(this) + zeroFill(this.minutes(), 2);\n    });\n\n    addFormatToken('hmmss', 0, 0, function () {\n        return '' + hFormat.apply(this) + zeroFill(this.minutes(), 2) +\n            zeroFill(this.seconds(), 2);\n    });\n\n    addFormatToken('Hmm', 0, 0, function () {\n        return '' + this.hours() + zeroFill(this.minutes(), 2);\n    });\n\n    addFormatToken('Hmmss', 0, 0, function () {\n        return '' + this.hours() + zeroFill(this.minutes(), 2) +\n            zeroFill(this.seconds(), 2);\n    });\n\n    function meridiem (token, lowercase) {\n        addFormatToken(token, 0, 0, function () {\n            return this.localeData().meridiem(this.hours(), this.minutes(), lowercase);\n        });\n    }\n\n    meridiem('a', true);\n    meridiem('A', false);\n\n    // ALIASES\n\n    addUnitAlias('hour', 'h');\n\n    // PRIORITY\n    addUnitPriority('hour', 13);\n\n    // PARSING\n\n    function matchMeridiem (isStrict, locale) {\n        return locale._meridiemParse;\n    }\n\n    addRegexToken('a',  matchMeridiem);\n    addRegexToken('A',  matchMeridiem);\n    addRegexToken('H',  match1to2);\n    addRegexToken('h',  match1to2);\n    addRegexToken('k',  match1to2);\n    addRegexToken('HH', match1to2, match2);\n    addRegexToken('hh', match1to2, match2);\n    addRegexToken('kk', match1to2, match2);\n\n    addRegexToken('hmm', match3to4);\n    addRegexToken('hmmss', match5to6);\n    addRegexToken('Hmm', match3to4);\n    addRegexToken('Hmmss', match5to6);\n\n    addParseToken(['H', 'HH'], HOUR);\n    addParseToken(['k', 'kk'], function (input, array, config) {\n        var kInput = toInt(input);\n        array[HOUR] = kInput === 24 ? 0 : kInput;\n    });\n    addParseToken(['a', 'A'], function (input, array, config) {\n        config._isPm = config._locale.isPM(input);\n        config._meridiem = input;\n    });\n    addParseToken(['h', 'hh'], function (input, array, config) {\n        array[HOUR] = toInt(input);\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('hmm', function (input, array, config) {\n        var pos = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos));\n        array[MINUTE] = toInt(input.substr(pos));\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('hmmss', function (input, array, config) {\n        var pos1 = input.length - 4;\n        var pos2 = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos1));\n        array[MINUTE] = toInt(input.substr(pos1, 2));\n        array[SECOND] = toInt(input.substr(pos2));\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('Hmm', function (input, array, config) {\n        var pos = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos));\n        array[MINUTE] = toInt(input.substr(pos));\n    });\n    addParseToken('Hmmss', function (input, array, config) {\n        var pos1 = input.length - 4;\n        var pos2 = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos1));\n        array[MINUTE] = toInt(input.substr(pos1, 2));\n        array[SECOND] = toInt(input.substr(pos2));\n    });\n\n    // LOCALES\n\n    function localeIsPM (input) {\n        // IE8 Quirks Mode & IE7 Standards Mode do not allow accessing strings like arrays\n        // Using charAt should be more compatible.\n        return ((input + '').toLowerCase().charAt(0) === 'p');\n    }\n\n    var defaultLocaleMeridiemParse = /[ap]\\.?m?\\.?/i;\n    function localeMeridiem (hours, minutes, isLower) {\n        if (hours > 11) {\n            return isLower ? 'pm' : 'PM';\n        } else {\n            return isLower ? 'am' : 'AM';\n        }\n    }\n\n\n    // MOMENTS\n\n    // Setting the hour should keep the time, because the user explicitly\n    // specified which hour they want. So trying to maintain the same hour (in\n    // a new timezone) makes sense. Adding/subtracting hours does not follow\n    // this rule.\n    var getSetHour = makeGetSet('Hours', true);\n\n    var baseConfig = {\n        calendar: defaultCalendar,\n        longDateFormat: defaultLongDateFormat,\n        invalidDate: defaultInvalidDate,\n        ordinal: defaultOrdinal,\n        dayOfMonthOrdinalParse: defaultDayOfMonthOrdinalParse,\n        relativeTime: defaultRelativeTime,\n\n        months: defaultLocaleMonths,\n        monthsShort: defaultLocaleMonthsShort,\n\n        week: defaultLocaleWeek,\n\n        weekdays: defaultLocaleWeekdays,\n        weekdaysMin: defaultLocaleWeekdaysMin,\n        weekdaysShort: defaultLocaleWeekdaysShort,\n\n        meridiemParse: defaultLocaleMeridiemParse\n    };\n\n    // internal storage for locale config files\n    var locales = {};\n    var localeFamilies = {};\n    var globalLocale;\n\n    function normalizeLocale(key) {\n        return key ? key.toLowerCase().replace('_', '-') : key;\n    }\n\n    // pick the locale from the array\n    // try ['en-au', 'en-gb'] as 'en-au', 'en-gb', 'en', as in move through the list trying each\n    // substring from most specific to least, but move to the next array item if it's a more specific variant than the current root\n    function chooseLocale(names) {\n        var i = 0, j, next, locale, split;\n\n        while (i < names.length) {\n            split = normalizeLocale(names[i]).split('-');\n            j = split.length;\n            next = normalizeLocale(names[i + 1]);\n            next = next ? next.split('-') : null;\n            while (j > 0) {\n                locale = loadLocale(split.slice(0, j).join('-'));\n                if (locale) {\n                    return locale;\n                }\n                if (next && next.length >= j && compareArrays(split, next, true) >= j - 1) {\n                    //the next array item is better than a shallower substring of this one\n                    break;\n                }\n                j--;\n            }\n            i++;\n        }\n        return globalLocale;\n    }\n\n    function loadLocale(name) {\n        var oldLocale = null;\n        // TODO: Find a better way to register and load all the locales in Node\n        if (!locales[name] && (typeof module !== 'undefined') &&\n                module && module.exports) {\n            try {\n                oldLocale = globalLocale._abbr;\n                var aliasedRequire = require;\n                aliasedRequire('./locale/' + name);\n                getSetGlobalLocale(oldLocale);\n            } catch (e) {}\n        }\n        return locales[name];\n    }\n\n    // This function will load locale and then set the global locale.  If\n    // no arguments are passed in, it will simply return the current global\n    // locale key.\n    function getSetGlobalLocale (key, values) {\n        var data;\n        if (key) {\n            if (isUndefined(values)) {\n                data = getLocale(key);\n            }\n            else {\n                data = defineLocale(key, values);\n            }\n\n            if (data) {\n                // moment.duration._locale = moment._locale = data;\n                globalLocale = data;\n            }\n            else {\n                if ((typeof console !==  'undefined') && console.warn) {\n                    //warn user if arguments are passed but the locale could not be set\n                    console.warn('Locale ' + key +  ' not found. Did you forget to load it?');\n                }\n            }\n        }\n\n        return globalLocale._abbr;\n    }\n\n    function defineLocale (name, config) {\n        if (config !== null) {\n            var locale, parentConfig = baseConfig;\n            config.abbr = name;\n            if (locales[name] != null) {\n                deprecateSimple('defineLocaleOverride',\n                        'use moment.updateLocale(localeName, config) to change ' +\n                        'an existing locale. moment.defineLocale(localeName, ' +\n                        'config) should only be used for creating a new locale ' +\n                        'See http://momentjs.com/guides/#/warnings/define-locale/ for more info.');\n                parentConfig = locales[name]._config;\n            } else if (config.parentLocale != null) {\n                if (locales[config.parentLocale] != null) {\n                    parentConfig = locales[config.parentLocale]._config;\n                } else {\n                    locale = loadLocale(config.parentLocale);\n                    if (locale != null) {\n                        parentConfig = locale._config;\n                    } else {\n                        if (!localeFamilies[config.parentLocale]) {\n                            localeFamilies[config.parentLocale] = [];\n                        }\n                        localeFamilies[config.parentLocale].push({\n                            name: name,\n                            config: config\n                        });\n                        return null;\n                    }\n                }\n            }\n            locales[name] = new Locale(mergeConfigs(parentConfig, config));\n\n            if (localeFamilies[name]) {\n                localeFamilies[name].forEach(function (x) {\n                    defineLocale(x.name, x.config);\n                });\n            }\n\n            // backwards compat for now: also set the locale\n            // make sure we set the locale AFTER all child locales have been\n            // created, so we won't end up with the child locale set.\n            getSetGlobalLocale(name);\n\n\n            return locales[name];\n        } else {\n            // useful for testing\n            delete locales[name];\n            return null;\n        }\n    }\n\n    function updateLocale(name, config) {\n        if (config != null) {\n            var locale, tmpLocale, parentConfig = baseConfig;\n            // MERGE\n            tmpLocale = loadLocale(name);\n            if (tmpLocale != null) {\n                parentConfig = tmpLocale._config;\n            }\n            config = mergeConfigs(parentConfig, config);\n            locale = new Locale(config);\n            locale.parentLocale = locales[name];\n            locales[name] = locale;\n\n            // backwards compat for now: also set the locale\n            getSetGlobalLocale(name);\n        } else {\n            // pass null for config to unupdate, useful for tests\n            if (locales[name] != null) {\n                if (locales[name].parentLocale != null) {\n                    locales[name] = locales[name].parentLocale;\n                } else if (locales[name] != null) {\n                    delete locales[name];\n                }\n            }\n        }\n        return locales[name];\n    }\n\n    // returns locale data\n    function getLocale (key) {\n        var locale;\n\n        if (key && key._locale && key._locale._abbr) {\n            key = key._locale._abbr;\n        }\n\n        if (!key) {\n            return globalLocale;\n        }\n\n        if (!isArray(key)) {\n            //short-circuit everything else\n            locale = loadLocale(key);\n            if (locale) {\n                return locale;\n            }\n            key = [key];\n        }\n\n        return chooseLocale(key);\n    }\n\n    function listLocales() {\n        return keys(locales);\n    }\n\n    function checkOverflow (m) {\n        var overflow;\n        var a = m._a;\n\n        if (a && getParsingFlags(m).overflow === -2) {\n            overflow =\n                a[MONTH]       < 0 || a[MONTH]       > 11  ? MONTH :\n                a[DATE]        < 1 || a[DATE]        > daysInMonth(a[YEAR], a[MONTH]) ? DATE :\n                a[HOUR]        < 0 || a[HOUR]        > 24 || (a[HOUR] === 24 && (a[MINUTE] !== 0 || a[SECOND] !== 0 || a[MILLISECOND] !== 0)) ? HOUR :\n                a[MINUTE]      < 0 || a[MINUTE]      > 59  ? MINUTE :\n                a[SECOND]      < 0 || a[SECOND]      > 59  ? SECOND :\n                a[MILLISECOND] < 0 || a[MILLISECOND] > 999 ? MILLISECOND :\n                -1;\n\n            if (getParsingFlags(m)._overflowDayOfYear && (overflow < YEAR || overflow > DATE)) {\n                overflow = DATE;\n            }\n            if (getParsingFlags(m)._overflowWeeks && overflow === -1) {\n                overflow = WEEK;\n            }\n            if (getParsingFlags(m)._overflowWeekday && overflow === -1) {\n                overflow = WEEKDAY;\n            }\n\n            getParsingFlags(m).overflow = overflow;\n        }\n\n        return m;\n    }\n\n    // Pick the first defined of two or three arguments.\n    function defaults(a, b, c) {\n        if (a != null) {\n            return a;\n        }\n        if (b != null) {\n            return b;\n        }\n        return c;\n    }\n\n    function currentDateArray(config) {\n        // hooks is actually the exported moment object\n        var nowValue = new Date(hooks.now());\n        if (config._useUTC) {\n            return [nowValue.getUTCFullYear(), nowValue.getUTCMonth(), nowValue.getUTCDate()];\n        }\n        return [nowValue.getFullYear(), nowValue.getMonth(), nowValue.getDate()];\n    }\n\n    // convert an array to a date.\n    // the array should mirror the parameters below\n    // note: all values past the year are optional and will default to the lowest possible value.\n    // [year, month, day , hour, minute, second, millisecond]\n    function configFromArray (config) {\n        var i, date, input = [], currentDate, expectedWeekday, yearToUse;\n\n        if (config._d) {\n            return;\n        }\n\n        currentDate = currentDateArray(config);\n\n        //compute day of the year from weeks and weekdays\n        if (config._w && config._a[DATE] == null && config._a[MONTH] == null) {\n            dayOfYearFromWeekInfo(config);\n        }\n\n        //if the day of the year is set, figure out what it is\n        if (config._dayOfYear != null) {\n            yearToUse = defaults(config._a[YEAR], currentDate[YEAR]);\n\n            if (config._dayOfYear > daysInYear(yearToUse) || config._dayOfYear === 0) {\n                getParsingFlags(config)._overflowDayOfYear = true;\n            }\n\n            date = createUTCDate(yearToUse, 0, config._dayOfYear);\n            config._a[MONTH] = date.getUTCMonth();\n            config._a[DATE] = date.getUTCDate();\n        }\n\n        // Default to current date.\n        // * if no year, month, day of month are given, default to today\n        // * if day of month is given, default month and year\n        // * if month is given, default only year\n        // * if year is given, don't default anything\n        for (i = 0; i < 3 && config._a[i] == null; ++i) {\n            config._a[i] = input[i] = currentDate[i];\n        }\n\n        // Zero out whatever was not defaulted, including time\n        for (; i < 7; i++) {\n            config._a[i] = input[i] = (config._a[i] == null) ? (i === 2 ? 1 : 0) : config._a[i];\n        }\n\n        // Check for 24:00:00.000\n        if (config._a[HOUR] === 24 &&\n                config._a[MINUTE] === 0 &&\n                config._a[SECOND] === 0 &&\n                config._a[MILLISECOND] === 0) {\n            config._nextDay = true;\n            config._a[HOUR] = 0;\n        }\n\n        config._d = (config._useUTC ? createUTCDate : createDate).apply(null, input);\n        expectedWeekday = config._useUTC ? config._d.getUTCDay() : config._d.getDay();\n\n        // Apply timezone offset from input. The actual utcOffset can be changed\n        // with parseZone.\n        if (config._tzm != null) {\n            config._d.setUTCMinutes(config._d.getUTCMinutes() - config._tzm);\n        }\n\n        if (config._nextDay) {\n            config._a[HOUR] = 24;\n        }\n\n        // check for mismatching day of week\n        if (config._w && typeof config._w.d !== 'undefined' && config._w.d !== expectedWeekday) {\n            getParsingFlags(config).weekdayMismatch = true;\n        }\n    }\n\n    function dayOfYearFromWeekInfo(config) {\n        var w, weekYear, week, weekday, dow, doy, temp, weekdayOverflow;\n\n        w = config._w;\n        if (w.GG != null || w.W != null || w.E != null) {\n            dow = 1;\n            doy = 4;\n\n            // TODO: We need to take the current isoWeekYear, but that depends on\n            // how we interpret now (local, utc, fixed offset). So create\n            // a now version of current config (take local/utc/offset flags, and\n            // create now).\n            weekYear = defaults(w.GG, config._a[YEAR], weekOfYear(createLocal(), 1, 4).year);\n            week = defaults(w.W, 1);\n            weekday = defaults(w.E, 1);\n            if (weekday < 1 || weekday > 7) {\n                weekdayOverflow = true;\n            }\n        } else {\n            dow = config._locale._week.dow;\n            doy = config._locale._week.doy;\n\n            var curWeek = weekOfYear(createLocal(), dow, doy);\n\n            weekYear = defaults(w.gg, config._a[YEAR], curWeek.year);\n\n            // Default to current week.\n            week = defaults(w.w, curWeek.week);\n\n            if (w.d != null) {\n                // weekday -- low day numbers are considered next week\n                weekday = w.d;\n                if (weekday < 0 || weekday > 6) {\n                    weekdayOverflow = true;\n                }\n            } else if (w.e != null) {\n                // local weekday -- counting starts from beginning of week\n                weekday = w.e + dow;\n                if (w.e < 0 || w.e > 6) {\n                    weekdayOverflow = true;\n                }\n            } else {\n                // default to beginning of week\n                weekday = dow;\n            }\n        }\n        if (week < 1 || week > weeksInYear(weekYear, dow, doy)) {\n            getParsingFlags(config)._overflowWeeks = true;\n        } else if (weekdayOverflow != null) {\n            getParsingFlags(config)._overflowWeekday = true;\n        } else {\n            temp = dayOfYearFromWeeks(weekYear, week, weekday, dow, doy);\n            config._a[YEAR] = temp.year;\n            config._dayOfYear = temp.dayOfYear;\n        }\n    }\n\n    // iso 8601 regex\n    // 0000-00-00 0000-W00 or 0000-W00-0 + T + 00 or 00:00 or 00:00:00 or 00:00:00.000 + +00:00 or +0000 or +00)\n    var extendedIsoRegex = /^\\s*((?:[+-]\\d{6}|\\d{4})-(?:\\d\\d-\\d\\d|W\\d\\d-\\d|W\\d\\d|\\d\\d\\d|\\d\\d))(?:(T| )(\\d\\d(?::\\d\\d(?::\\d\\d(?:[.,]\\d+)?)?)?)([\\+\\-]\\d\\d(?::?\\d\\d)?|\\s*Z)?)?$/;\n    var basicIsoRegex = /^\\s*((?:[+-]\\d{6}|\\d{4})(?:\\d\\d\\d\\d|W\\d\\d\\d|W\\d\\d|\\d\\d\\d|\\d\\d))(?:(T| )(\\d\\d(?:\\d\\d(?:\\d\\d(?:[.,]\\d+)?)?)?)([\\+\\-]\\d\\d(?::?\\d\\d)?|\\s*Z)?)?$/;\n\n    var tzRegex = /Z|[+-]\\d\\d(?::?\\d\\d)?/;\n\n    var isoDates = [\n        ['YYYYYY-MM-DD', /[+-]\\d{6}-\\d\\d-\\d\\d/],\n        ['YYYY-MM-DD', /\\d{4}-\\d\\d-\\d\\d/],\n        ['GGGG-[W]WW-E', /\\d{4}-W\\d\\d-\\d/],\n        ['GGGG-[W]WW', /\\d{4}-W\\d\\d/, false],\n        ['YYYY-DDD', /\\d{4}-\\d{3}/],\n        ['YYYY-MM', /\\d{4}-\\d\\d/, false],\n        ['YYYYYYMMDD', /[+-]\\d{10}/],\n        ['YYYYMMDD', /\\d{8}/],\n        // YYYYMM is NOT allowed by the standard\n        ['GGGG[W]WWE', /\\d{4}W\\d{3}/],\n        ['GGGG[W]WW', /\\d{4}W\\d{2}/, false],\n        ['YYYYDDD', /\\d{7}/]\n    ];\n\n    // iso time formats and regexes\n    var isoTimes = [\n        ['HH:mm:ss.SSSS', /\\d\\d:\\d\\d:\\d\\d\\.\\d+/],\n        ['HH:mm:ss,SSSS', /\\d\\d:\\d\\d:\\d\\d,\\d+/],\n        ['HH:mm:ss', /\\d\\d:\\d\\d:\\d\\d/],\n        ['HH:mm', /\\d\\d:\\d\\d/],\n        ['HHmmss.SSSS', /\\d\\d\\d\\d\\d\\d\\.\\d+/],\n        ['HHmmss,SSSS', /\\d\\d\\d\\d\\d\\d,\\d+/],\n        ['HHmmss', /\\d\\d\\d\\d\\d\\d/],\n        ['HHmm', /\\d\\d\\d\\d/],\n        ['HH', /\\d\\d/]\n    ];\n\n    var aspNetJsonRegex = /^\\/?Date\\((\\-?\\d+)/i;\n\n    // date from iso format\n    function configFromISO(config) {\n        var i, l,\n            string = config._i,\n            match = extendedIsoRegex.exec(string) || basicIsoRegex.exec(string),\n            allowTime, dateFormat, timeFormat, tzFormat;\n\n        if (match) {\n            getParsingFlags(config).iso = true;\n\n            for (i = 0, l = isoDates.length; i < l; i++) {\n                if (isoDates[i][1].exec(match[1])) {\n                    dateFormat = isoDates[i][0];\n                    allowTime = isoDates[i][2] !== false;\n                    break;\n                }\n            }\n            if (dateFormat == null) {\n                config._isValid = false;\n                return;\n            }\n            if (match[3]) {\n                for (i = 0, l = isoTimes.length; i < l; i++) {\n                    if (isoTimes[i][1].exec(match[3])) {\n                        // match[2] should be 'T' or space\n                        timeFormat = (match[2] || ' ') + isoTimes[i][0];\n                        break;\n                    }\n                }\n                if (timeFormat == null) {\n                    config._isValid = false;\n                    return;\n                }\n            }\n            if (!allowTime && timeFormat != null) {\n                config._isValid = false;\n                return;\n            }\n            if (match[4]) {\n                if (tzRegex.exec(match[4])) {\n                    tzFormat = 'Z';\n                } else {\n                    config._isValid = false;\n                    return;\n                }\n            }\n            config._f = dateFormat + (timeFormat || '') + (tzFormat || '');\n            configFromStringAndFormat(config);\n        } else {\n            config._isValid = false;\n        }\n    }\n\n    // RFC 2822 regex: For details see https://tools.ietf.org/html/rfc2822#section-3.3\n    var rfc2822 = /^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\\s)?(\\d{1,2})\\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\\s(\\d{2,4})\\s(\\d\\d):(\\d\\d)(?::(\\d\\d))?\\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\\d{4}))$/;\n\n    function extractFromRFC2822Strings(yearStr, monthStr, dayStr, hourStr, minuteStr, secondStr) {\n        var result = [\n            untruncateYear(yearStr),\n            defaultLocaleMonthsShort.indexOf(monthStr),\n            parseInt(dayStr, 10),\n            parseInt(hourStr, 10),\n            parseInt(minuteStr, 10)\n        ];\n\n        if (secondStr) {\n            result.push(parseInt(secondStr, 10));\n        }\n\n        return result;\n    }\n\n    function untruncateYear(yearStr) {\n        var year = parseInt(yearStr, 10);\n        if (year <= 49) {\n            return 2000 + year;\n        } else if (year <= 999) {\n            return 1900 + year;\n        }\n        return year;\n    }\n\n    function preprocessRFC2822(s) {\n        // Remove comments and folding whitespace and replace multiple-spaces with a single space\n        return s.replace(/\\([^)]*\\)|[\\n\\t]/g, ' ').replace(/(\\s\\s+)/g, ' ').replace(/^\\s\\s*/, '').replace(/\\s\\s*$/, '');\n    }\n\n    function checkWeekday(weekdayStr, parsedInput, config) {\n        if (weekdayStr) {\n            // TODO: Replace the vanilla JS Date object with an indepentent day-of-week check.\n            var weekdayProvided = defaultLocaleWeekdaysShort.indexOf(weekdayStr),\n                weekdayActual = new Date(parsedInput[0], parsedInput[1], parsedInput[2]).getDay();\n            if (weekdayProvided !== weekdayActual) {\n                getParsingFlags(config).weekdayMismatch = true;\n                config._isValid = false;\n                return false;\n            }\n        }\n        return true;\n    }\n\n    var obsOffsets = {\n        UT: 0,\n        GMT: 0,\n        EDT: -4 * 60,\n        EST: -5 * 60,\n        CDT: -5 * 60,\n        CST: -6 * 60,\n        MDT: -6 * 60,\n        MST: -7 * 60,\n        PDT: -7 * 60,\n        PST: -8 * 60\n    };\n\n    function calculateOffset(obsOffset, militaryOffset, numOffset) {\n        if (obsOffset) {\n            return obsOffsets[obsOffset];\n        } else if (militaryOffset) {\n            // the only allowed military tz is Z\n            return 0;\n        } else {\n            var hm = parseInt(numOffset, 10);\n            var m = hm % 100, h = (hm - m) / 100;\n            return h * 60 + m;\n        }\n    }\n\n    // date and time from ref 2822 format\n    function configFromRFC2822(config) {\n        var match = rfc2822.exec(preprocessRFC2822(config._i));\n        if (match) {\n            var parsedArray = extractFromRFC2822Strings(match[4], match[3], match[2], match[5], match[6], match[7]);\n            if (!checkWeekday(match[1], parsedArray, config)) {\n                return;\n            }\n\n            config._a = parsedArray;\n            config._tzm = calculateOffset(match[8], match[9], match[10]);\n\n            config._d = createUTCDate.apply(null, config._a);\n            config._d.setUTCMinutes(config._d.getUTCMinutes() - config._tzm);\n\n            getParsingFlags(config).rfc2822 = true;\n        } else {\n            config._isValid = false;\n        }\n    }\n\n    // date from iso format or fallback\n    function configFromString(config) {\n        var matched = aspNetJsonRegex.exec(config._i);\n\n        if (matched !== null) {\n            config._d = new Date(+matched[1]);\n            return;\n        }\n\n        configFromISO(config);\n        if (config._isValid === false) {\n            delete config._isValid;\n        } else {\n            return;\n        }\n\n        configFromRFC2822(config);\n        if (config._isValid === false) {\n            delete config._isValid;\n        } else {\n            return;\n        }\n\n        // Final attempt, use Input Fallback\n        hooks.createFromInputFallback(config);\n    }\n\n    hooks.createFromInputFallback = deprecate(\n        'value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), ' +\n        'which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are ' +\n        'discouraged and will be removed in an upcoming major release. Please refer to ' +\n        'http://momentjs.com/guides/#/warnings/js-date/ for more info.',\n        function (config) {\n            config._d = new Date(config._i + (config._useUTC ? ' UTC' : ''));\n        }\n    );\n\n    // constant that refers to the ISO standard\n    hooks.ISO_8601 = function () {};\n\n    // constant that refers to the RFC 2822 form\n    hooks.RFC_2822 = function () {};\n\n    // date from string and format string\n    function configFromStringAndFormat(config) {\n        // TODO: Move this to another part of the creation flow to prevent circular deps\n        if (config._f === hooks.ISO_8601) {\n            configFromISO(config);\n            return;\n        }\n        if (config._f === hooks.RFC_2822) {\n            configFromRFC2822(config);\n            return;\n        }\n        config._a = [];\n        getParsingFlags(config).empty = true;\n\n        // This array is used to make a Date, either with `new Date` or `Date.UTC`\n        var string = '' + config._i,\n            i, parsedInput, tokens, token, skipped,\n            stringLength = string.length,\n            totalParsedInputLength = 0;\n\n        tokens = expandFormat(config._f, config._locale).match(formattingTokens) || [];\n\n        for (i = 0; i < tokens.length; i++) {\n            token = tokens[i];\n            parsedInput = (string.match(getParseRegexForToken(token, config)) || [])[0];\n            // console.log('token', token, 'parsedInput', parsedInput,\n            //         'regex', getParseRegexForToken(token, config));\n            if (parsedInput) {\n                skipped = string.substr(0, string.indexOf(parsedInput));\n                if (skipped.length > 0) {\n                    getParsingFlags(config).unusedInput.push(skipped);\n                }\n                string = string.slice(string.indexOf(parsedInput) + parsedInput.length);\n                totalParsedInputLength += parsedInput.length;\n            }\n            // don't parse if it's not a known token\n            if (formatTokenFunctions[token]) {\n                if (parsedInput) {\n                    getParsingFlags(config).empty = false;\n                }\n                else {\n                    getParsingFlags(config).unusedTokens.push(token);\n                }\n                addTimeToArrayFromToken(token, parsedInput, config);\n            }\n            else if (config._strict && !parsedInput) {\n                getParsingFlags(config).unusedTokens.push(token);\n            }\n        }\n\n        // add remaining unparsed input length to the string\n        getParsingFlags(config).charsLeftOver = stringLength - totalParsedInputLength;\n        if (string.length > 0) {\n            getParsingFlags(config).unusedInput.push(string);\n        }\n\n        // clear _12h flag if hour is <= 12\n        if (config._a[HOUR] <= 12 &&\n            getParsingFlags(config).bigHour === true &&\n            config._a[HOUR] > 0) {\n            getParsingFlags(config).bigHour = undefined;\n        }\n\n        getParsingFlags(config).parsedDateParts = config._a.slice(0);\n        getParsingFlags(config).meridiem = config._meridiem;\n        // handle meridiem\n        config._a[HOUR] = meridiemFixWrap(config._locale, config._a[HOUR], config._meridiem);\n\n        configFromArray(config);\n        checkOverflow(config);\n    }\n\n\n    function meridiemFixWrap (locale, hour, meridiem) {\n        var isPm;\n\n        if (meridiem == null) {\n            // nothing to do\n            return hour;\n        }\n        if (locale.meridiemHour != null) {\n            return locale.meridiemHour(hour, meridiem);\n        } else if (locale.isPM != null) {\n            // Fallback\n            isPm = locale.isPM(meridiem);\n            if (isPm && hour < 12) {\n                hour += 12;\n            }\n            if (!isPm && hour === 12) {\n                hour = 0;\n            }\n            return hour;\n        } else {\n            // this is not supposed to happen\n            return hour;\n        }\n    }\n\n    // date from string and array of format strings\n    function configFromStringAndArray(config) {\n        var tempConfig,\n            bestMoment,\n\n            scoreToBeat,\n            i,\n            currentScore;\n\n        if (config._f.length === 0) {\n            getParsingFlags(config).invalidFormat = true;\n            config._d = new Date(NaN);\n            return;\n        }\n\n        for (i = 0; i < config._f.length; i++) {\n            currentScore = 0;\n            tempConfig = copyConfig({}, config);\n            if (config._useUTC != null) {\n                tempConfig._useUTC = config._useUTC;\n            }\n            tempConfig._f = config._f[i];\n            configFromStringAndFormat(tempConfig);\n\n            if (!isValid(tempConfig)) {\n                continue;\n            }\n\n            // if there is any input that was not parsed add a penalty for that format\n            currentScore += getParsingFlags(tempConfig).charsLeftOver;\n\n            //or tokens\n            currentScore += getParsingFlags(tempConfig).unusedTokens.length * 10;\n\n            getParsingFlags(tempConfig).score = currentScore;\n\n            if (scoreToBeat == null || currentScore < scoreToBeat) {\n                scoreToBeat = currentScore;\n                bestMoment = tempConfig;\n            }\n        }\n\n        extend(config, bestMoment || tempConfig);\n    }\n\n    function configFromObject(config) {\n        if (config._d) {\n            return;\n        }\n\n        var i = normalizeObjectUnits(config._i);\n        config._a = map([i.year, i.month, i.day || i.date, i.hour, i.minute, i.second, i.millisecond], function (obj) {\n            return obj && parseInt(obj, 10);\n        });\n\n        configFromArray(config);\n    }\n\n    function createFromConfig (config) {\n        var res = new Moment(checkOverflow(prepareConfig(config)));\n        if (res._nextDay) {\n            // Adding is smart enough around DST\n            res.add(1, 'd');\n            res._nextDay = undefined;\n        }\n\n        return res;\n    }\n\n    function prepareConfig (config) {\n        var input = config._i,\n            format = config._f;\n\n        config._locale = config._locale || getLocale(config._l);\n\n        if (input === null || (format === undefined && input === '')) {\n            return createInvalid({nullInput: true});\n        }\n\n        if (typeof input === 'string') {\n            config._i = input = config._locale.preparse(input);\n        }\n\n        if (isMoment(input)) {\n            return new Moment(checkOverflow(input));\n        } else if (isDate(input)) {\n            config._d = input;\n        } else if (isArray(format)) {\n            configFromStringAndArray(config);\n        } else if (format) {\n            configFromStringAndFormat(config);\n        }  else {\n            configFromInput(config);\n        }\n\n        if (!isValid(config)) {\n            config._d = null;\n        }\n\n        return config;\n    }\n\n    function configFromInput(config) {\n        var input = config._i;\n        if (isUndefined(input)) {\n            config._d = new Date(hooks.now());\n        } else if (isDate(input)) {\n            config._d = new Date(input.valueOf());\n        } else if (typeof input === 'string') {\n            configFromString(config);\n        } else if (isArray(input)) {\n            config._a = map(input.slice(0), function (obj) {\n                return parseInt(obj, 10);\n            });\n            configFromArray(config);\n        } else if (isObject(input)) {\n            configFromObject(config);\n        } else if (isNumber(input)) {\n            // from milliseconds\n            config._d = new Date(input);\n        } else {\n            hooks.createFromInputFallback(config);\n        }\n    }\n\n    function createLocalOrUTC (input, format, locale, strict, isUTC) {\n        var c = {};\n\n        if (locale === true || locale === false) {\n            strict = locale;\n            locale = undefined;\n        }\n\n        if ((isObject(input) && isObjectEmpty(input)) ||\n                (isArray(input) && input.length === 0)) {\n            input = undefined;\n        }\n        // object construction must be done this way.\n        // https://github.com/moment/moment/issues/1423\n        c._isAMomentObject = true;\n        c._useUTC = c._isUTC = isUTC;\n        c._l = locale;\n        c._i = input;\n        c._f = format;\n        c._strict = strict;\n\n        return createFromConfig(c);\n    }\n\n    function createLocal (input, format, locale, strict) {\n        return createLocalOrUTC(input, format, locale, strict, false);\n    }\n\n    var prototypeMin = deprecate(\n        'moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/',\n        function () {\n            var other = createLocal.apply(null, arguments);\n            if (this.isValid() && other.isValid()) {\n                return other < this ? this : other;\n            } else {\n                return createInvalid();\n            }\n        }\n    );\n\n    var prototypeMax = deprecate(\n        'moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/',\n        function () {\n            var other = createLocal.apply(null, arguments);\n            if (this.isValid() && other.isValid()) {\n                return other > this ? this : other;\n            } else {\n                return createInvalid();\n            }\n        }\n    );\n\n    // Pick a moment m from moments so that m[fn](other) is true for all\n    // other. This relies on the function fn to be transitive.\n    //\n    // moments should either be an array of moment objects or an array, whose\n    // first element is an array of moment objects.\n    function pickBy(fn, moments) {\n        var res, i;\n        if (moments.length === 1 && isArray(moments[0])) {\n            moments = moments[0];\n        }\n        if (!moments.length) {\n            return createLocal();\n        }\n        res = moments[0];\n        for (i = 1; i < moments.length; ++i) {\n            if (!moments[i].isValid() || moments[i][fn](res)) {\n                res = moments[i];\n            }\n        }\n        return res;\n    }\n\n    // TODO: Use [].sort instead?\n    function min () {\n        var args = [].slice.call(arguments, 0);\n\n        return pickBy('isBefore', args);\n    }\n\n    function max () {\n        var args = [].slice.call(arguments, 0);\n\n        return pickBy('isAfter', args);\n    }\n\n    var now = function () {\n        return Date.now ? Date.now() : +(new Date());\n    };\n\n    var ordering = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second', 'millisecond'];\n\n    function isDurationValid(m) {\n        for (var key in m) {\n            if (!(indexOf.call(ordering, key) !== -1 && (m[key] == null || !isNaN(m[key])))) {\n                return false;\n            }\n        }\n\n        var unitHasDecimal = false;\n        for (var i = 0; i < ordering.length; ++i) {\n            if (m[ordering[i]]) {\n                if (unitHasDecimal) {\n                    return false; // only allow non-integers for smallest unit\n                }\n                if (parseFloat(m[ordering[i]]) !== toInt(m[ordering[i]])) {\n                    unitHasDecimal = true;\n                }\n            }\n        }\n\n        return true;\n    }\n\n    function isValid$1() {\n        return this._isValid;\n    }\n\n    function createInvalid$1() {\n        return createDuration(NaN);\n    }\n\n    function Duration (duration) {\n        var normalizedInput = normalizeObjectUnits(duration),\n            years = normalizedInput.year || 0,\n            quarters = normalizedInput.quarter || 0,\n            months = normalizedInput.month || 0,\n            weeks = normalizedInput.week || normalizedInput.isoWeek || 0,\n            days = normalizedInput.day || 0,\n            hours = normalizedInput.hour || 0,\n            minutes = normalizedInput.minute || 0,\n            seconds = normalizedInput.second || 0,\n            milliseconds = normalizedInput.millisecond || 0;\n\n        this._isValid = isDurationValid(normalizedInput);\n\n        // representation for dateAddRemove\n        this._milliseconds = +milliseconds +\n            seconds * 1e3 + // 1000\n            minutes * 6e4 + // 1000 * 60\n            hours * 1000 * 60 * 60; //using 1000 * 60 * 60 instead of 36e5 to avoid floating point rounding errors https://github.com/moment/moment/issues/2978\n        // Because of dateAddRemove treats 24 hours as different from a\n        // day when working around DST, we need to store them separately\n        this._days = +days +\n            weeks * 7;\n        // It is impossible to translate months into days without knowing\n        // which months you are are talking about, so we have to store\n        // it separately.\n        this._months = +months +\n            quarters * 3 +\n            years * 12;\n\n        this._data = {};\n\n        this._locale = getLocale();\n\n        this._bubble();\n    }\n\n    function isDuration (obj) {\n        return obj instanceof Duration;\n    }\n\n    function absRound (number) {\n        if (number < 0) {\n            return Math.round(-1 * number) * -1;\n        } else {\n            return Math.round(number);\n        }\n    }\n\n    // FORMATTING\n\n    function offset (token, separator) {\n        addFormatToken(token, 0, 0, function () {\n            var offset = this.utcOffset();\n            var sign = '+';\n            if (offset < 0) {\n                offset = -offset;\n                sign = '-';\n            }\n            return sign + zeroFill(~~(offset / 60), 2) + separator + zeroFill(~~(offset) % 60, 2);\n        });\n    }\n\n    offset('Z', ':');\n    offset('ZZ', '');\n\n    // PARSING\n\n    addRegexToken('Z',  matchShortOffset);\n    addRegexToken('ZZ', matchShortOffset);\n    addParseToken(['Z', 'ZZ'], function (input, array, config) {\n        config._useUTC = true;\n        config._tzm = offsetFromString(matchShortOffset, input);\n    });\n\n    // HELPERS\n\n    // timezone chunker\n    // '+10:00' > ['10',  '00']\n    // '-1530'  > ['-15', '30']\n    var chunkOffset = /([\\+\\-]|\\d\\d)/gi;\n\n    function offsetFromString(matcher, string) {\n        var matches = (string || '').match(matcher);\n\n        if (matches === null) {\n            return null;\n        }\n\n        var chunk   = matches[matches.length - 1] || [];\n        var parts   = (chunk + '').match(chunkOffset) || ['-', 0, 0];\n        var minutes = +(parts[1] * 60) + toInt(parts[2]);\n\n        return minutes === 0 ?\n          0 :\n          parts[0] === '+' ? minutes : -minutes;\n    }\n\n    // Return a moment from input, that is local/utc/zone equivalent to model.\n    function cloneWithOffset(input, model) {\n        var res, diff;\n        if (model._isUTC) {\n            res = model.clone();\n            diff = (isMoment(input) || isDate(input) ? input.valueOf() : createLocal(input).valueOf()) - res.valueOf();\n            // Use low-level api, because this fn is low-level api.\n            res._d.setTime(res._d.valueOf() + diff);\n            hooks.updateOffset(res, false);\n            return res;\n        } else {\n            return createLocal(input).local();\n        }\n    }\n\n    function getDateOffset (m) {\n        // On Firefox.24 Date#getTimezoneOffset returns a floating point.\n        // https://github.com/moment/moment/pull/1871\n        return -Math.round(m._d.getTimezoneOffset() / 15) * 15;\n    }\n\n    // HOOKS\n\n    // This function will be called whenever a moment is mutated.\n    // It is intended to keep the offset in sync with the timezone.\n    hooks.updateOffset = function () {};\n\n    // MOMENTS\n\n    // keepLocalTime = true means only change the timezone, without\n    // affecting the local hour. So 5:31:26 +0300 --[utcOffset(2, true)]-->\n    // 5:31:26 +0200 It is possible that 5:31:26 doesn't exist with offset\n    // +0200, so we adjust the time as needed, to be valid.\n    //\n    // Keeping the time actually adds/subtracts (one hour)\n    // from the actual represented time. That is why we call updateOffset\n    // a second time. In case it wants us to change the offset again\n    // _changeInProgress == true case, then we have to adjust, because\n    // there is no such time in the given timezone.\n    function getSetOffset (input, keepLocalTime, keepMinutes) {\n        var offset = this._offset || 0,\n            localAdjust;\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        if (input != null) {\n            if (typeof input === 'string') {\n                input = offsetFromString(matchShortOffset, input);\n                if (input === null) {\n                    return this;\n                }\n            } else if (Math.abs(input) < 16 && !keepMinutes) {\n                input = input * 60;\n            }\n            if (!this._isUTC && keepLocalTime) {\n                localAdjust = getDateOffset(this);\n            }\n            this._offset = input;\n            this._isUTC = true;\n            if (localAdjust != null) {\n                this.add(localAdjust, 'm');\n            }\n            if (offset !== input) {\n                if (!keepLocalTime || this._changeInProgress) {\n                    addSubtract(this, createDuration(input - offset, 'm'), 1, false);\n                } else if (!this._changeInProgress) {\n                    this._changeInProgress = true;\n                    hooks.updateOffset(this, true);\n                    this._changeInProgress = null;\n                }\n            }\n            return this;\n        } else {\n            return this._isUTC ? offset : getDateOffset(this);\n        }\n    }\n\n    function getSetZone (input, keepLocalTime) {\n        if (input != null) {\n            if (typeof input !== 'string') {\n                input = -input;\n            }\n\n            this.utcOffset(input, keepLocalTime);\n\n            return this;\n        } else {\n            return -this.utcOffset();\n        }\n    }\n\n    function setOffsetToUTC (keepLocalTime) {\n        return this.utcOffset(0, keepLocalTime);\n    }\n\n    function setOffsetToLocal (keepLocalTime) {\n        if (this._isUTC) {\n            this.utcOffset(0, keepLocalTime);\n            this._isUTC = false;\n\n            if (keepLocalTime) {\n                this.subtract(getDateOffset(this), 'm');\n            }\n        }\n        return this;\n    }\n\n    function setOffsetToParsedOffset () {\n        if (this._tzm != null) {\n            this.utcOffset(this._tzm, false, true);\n        } else if (typeof this._i === 'string') {\n            var tZone = offsetFromString(matchOffset, this._i);\n            if (tZone != null) {\n                this.utcOffset(tZone);\n            }\n            else {\n                this.utcOffset(0, true);\n            }\n        }\n        return this;\n    }\n\n    function hasAlignedHourOffset (input) {\n        if (!this.isValid()) {\n            return false;\n        }\n        input = input ? createLocal(input).utcOffset() : 0;\n\n        return (this.utcOffset() - input) % 60 === 0;\n    }\n\n    function isDaylightSavingTime () {\n        return (\n            this.utcOffset() > this.clone().month(0).utcOffset() ||\n            this.utcOffset() > this.clone().month(5).utcOffset()\n        );\n    }\n\n    function isDaylightSavingTimeShifted () {\n        if (!isUndefined(this._isDSTShifted)) {\n            return this._isDSTShifted;\n        }\n\n        var c = {};\n\n        copyConfig(c, this);\n        c = prepareConfig(c);\n\n        if (c._a) {\n            var other = c._isUTC ? createUTC(c._a) : createLocal(c._a);\n            this._isDSTShifted = this.isValid() &&\n                compareArrays(c._a, other.toArray()) > 0;\n        } else {\n            this._isDSTShifted = false;\n        }\n\n        return this._isDSTShifted;\n    }\n\n    function isLocal () {\n        return this.isValid() ? !this._isUTC : false;\n    }\n\n    function isUtcOffset () {\n        return this.isValid() ? this._isUTC : false;\n    }\n\n    function isUtc () {\n        return this.isValid() ? this._isUTC && this._offset === 0 : false;\n    }\n\n    // ASP.NET json date format regex\n    var aspNetRegex = /^(\\-|\\+)?(?:(\\d*)[. ])?(\\d+)\\:(\\d+)(?:\\:(\\d+)(\\.\\d*)?)?$/;\n\n    // from http://docs.closure-library.googlecode.com/git/closure_goog_date_date.js.source.html\n    // somewhat more in line with 4.4.3.2 2004 spec, but allows decimal anywhere\n    // and further modified to allow for strings containing both week and day\n    var isoRegex = /^(-|\\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;\n\n    function createDuration (input, key) {\n        var duration = input,\n            // matching against regexp is expensive, do it on demand\n            match = null,\n            sign,\n            ret,\n            diffRes;\n\n        if (isDuration(input)) {\n            duration = {\n                ms : input._milliseconds,\n                d  : input._days,\n                M  : input._months\n            };\n        } else if (isNumber(input)) {\n            duration = {};\n            if (key) {\n                duration[key] = input;\n            } else {\n                duration.milliseconds = input;\n            }\n        } else if (!!(match = aspNetRegex.exec(input))) {\n            sign = (match[1] === '-') ? -1 : 1;\n            duration = {\n                y  : 0,\n                d  : toInt(match[DATE])                         * sign,\n                h  : toInt(match[HOUR])                         * sign,\n                m  : toInt(match[MINUTE])                       * sign,\n                s  : toInt(match[SECOND])                       * sign,\n                ms : toInt(absRound(match[MILLISECOND] * 1000)) * sign // the millisecond decimal point is included in the match\n            };\n        } else if (!!(match = isoRegex.exec(input))) {\n            sign = (match[1] === '-') ? -1 : 1;\n            duration = {\n                y : parseIso(match[2], sign),\n                M : parseIso(match[3], sign),\n                w : parseIso(match[4], sign),\n                d : parseIso(match[5], sign),\n                h : parseIso(match[6], sign),\n                m : parseIso(match[7], sign),\n                s : parseIso(match[8], sign)\n            };\n        } else if (duration == null) {// checks for null or undefined\n            duration = {};\n        } else if (typeof duration === 'object' && ('from' in duration || 'to' in duration)) {\n            diffRes = momentsDifference(createLocal(duration.from), createLocal(duration.to));\n\n            duration = {};\n            duration.ms = diffRes.milliseconds;\n            duration.M = diffRes.months;\n        }\n\n        ret = new Duration(duration);\n\n        if (isDuration(input) && hasOwnProp(input, '_locale')) {\n            ret._locale = input._locale;\n        }\n\n        return ret;\n    }\n\n    createDuration.fn = Duration.prototype;\n    createDuration.invalid = createInvalid$1;\n\n    function parseIso (inp, sign) {\n        // We'd normally use ~~inp for this, but unfortunately it also\n        // converts floats to ints.\n        // inp may be undefined, so careful calling replace on it.\n        var res = inp && parseFloat(inp.replace(',', '.'));\n        // apply sign while we're at it\n        return (isNaN(res) ? 0 : res) * sign;\n    }\n\n    function positiveMomentsDifference(base, other) {\n        var res = {};\n\n        res.months = other.month() - base.month() +\n            (other.year() - base.year()) * 12;\n        if (base.clone().add(res.months, 'M').isAfter(other)) {\n            --res.months;\n        }\n\n        res.milliseconds = +other - +(base.clone().add(res.months, 'M'));\n\n        return res;\n    }\n\n    function momentsDifference(base, other) {\n        var res;\n        if (!(base.isValid() && other.isValid())) {\n            return {milliseconds: 0, months: 0};\n        }\n\n        other = cloneWithOffset(other, base);\n        if (base.isBefore(other)) {\n            res = positiveMomentsDifference(base, other);\n        } else {\n            res = positiveMomentsDifference(other, base);\n            res.milliseconds = -res.milliseconds;\n            res.months = -res.months;\n        }\n\n        return res;\n    }\n\n    // TODO: remove 'name' arg after deprecation is removed\n    function createAdder(direction, name) {\n        return function (val, period) {\n            var dur, tmp;\n            //invert the arguments, but complain about it\n            if (period !== null && !isNaN(+period)) {\n                deprecateSimple(name, 'moment().' + name  + '(period, number) is deprecated. Please use moment().' + name + '(number, period). ' +\n                'See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info.');\n                tmp = val; val = period; period = tmp;\n            }\n\n            val = typeof val === 'string' ? +val : val;\n            dur = createDuration(val, period);\n            addSubtract(this, dur, direction);\n            return this;\n        };\n    }\n\n    function addSubtract (mom, duration, isAdding, updateOffset) {\n        var milliseconds = duration._milliseconds,\n            days = absRound(duration._days),\n            months = absRound(duration._months);\n\n        if (!mom.isValid()) {\n            // No op\n            return;\n        }\n\n        updateOffset = updateOffset == null ? true : updateOffset;\n\n        if (months) {\n            setMonth(mom, get(mom, 'Month') + months * isAdding);\n        }\n        if (days) {\n            set$1(mom, 'Date', get(mom, 'Date') + days * isAdding);\n        }\n        if (milliseconds) {\n            mom._d.setTime(mom._d.valueOf() + milliseconds * isAdding);\n        }\n        if (updateOffset) {\n            hooks.updateOffset(mom, days || months);\n        }\n    }\n\n    var add      = createAdder(1, 'add');\n    var subtract = createAdder(-1, 'subtract');\n\n    function getCalendarFormat(myMoment, now) {\n        var diff = myMoment.diff(now, 'days', true);\n        return diff < -6 ? 'sameElse' :\n                diff < -1 ? 'lastWeek' :\n                diff < 0 ? 'lastDay' :\n                diff < 1 ? 'sameDay' :\n                diff < 2 ? 'nextDay' :\n                diff < 7 ? 'nextWeek' : 'sameElse';\n    }\n\n    function calendar$1 (time, formats) {\n        // We want to compare the start of today, vs this.\n        // Getting start-of-today depends on whether we're local/utc/offset or not.\n        var now = time || createLocal(),\n            sod = cloneWithOffset(now, this).startOf('day'),\n            format = hooks.calendarFormat(this, sod) || 'sameElse';\n\n        var output = formats && (isFunction(formats[format]) ? formats[format].call(this, now) : formats[format]);\n\n        return this.format(output || this.localeData().calendar(format, this, createLocal(now)));\n    }\n\n    function clone () {\n        return new Moment(this);\n    }\n\n    function isAfter (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input);\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() > localInput.valueOf();\n        } else {\n            return localInput.valueOf() < this.clone().startOf(units).valueOf();\n        }\n    }\n\n    function isBefore (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input);\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() < localInput.valueOf();\n        } else {\n            return this.clone().endOf(units).valueOf() < localInput.valueOf();\n        }\n    }\n\n    function isBetween (from, to, units, inclusivity) {\n        var localFrom = isMoment(from) ? from : createLocal(from),\n            localTo = isMoment(to) ? to : createLocal(to);\n        if (!(this.isValid() && localFrom.isValid() && localTo.isValid())) {\n            return false;\n        }\n        inclusivity = inclusivity || '()';\n        return (inclusivity[0] === '(' ? this.isAfter(localFrom, units) : !this.isBefore(localFrom, units)) &&\n            (inclusivity[1] === ')' ? this.isBefore(localTo, units) : !this.isAfter(localTo, units));\n    }\n\n    function isSame (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input),\n            inputMs;\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() === localInput.valueOf();\n        } else {\n            inputMs = localInput.valueOf();\n            return this.clone().startOf(units).valueOf() <= inputMs && inputMs <= this.clone().endOf(units).valueOf();\n        }\n    }\n\n    function isSameOrAfter (input, units) {\n        return this.isSame(input, units) || this.isAfter(input, units);\n    }\n\n    function isSameOrBefore (input, units) {\n        return this.isSame(input, units) || this.isBefore(input, units);\n    }\n\n    function diff (input, units, asFloat) {\n        var that,\n            zoneDelta,\n            output;\n\n        if (!this.isValid()) {\n            return NaN;\n        }\n\n        that = cloneWithOffset(input, this);\n\n        if (!that.isValid()) {\n            return NaN;\n        }\n\n        zoneDelta = (that.utcOffset() - this.utcOffset()) * 6e4;\n\n        units = normalizeUnits(units);\n\n        switch (units) {\n            case 'year': output = monthDiff(this, that) / 12; break;\n            case 'month': output = monthDiff(this, that); break;\n            case 'quarter': output = monthDiff(this, that) / 3; break;\n            case 'second': output = (this - that) / 1e3; break; // 1000\n            case 'minute': output = (this - that) / 6e4; break; // 1000 * 60\n            case 'hour': output = (this - that) / 36e5; break; // 1000 * 60 * 60\n            case 'day': output = (this - that - zoneDelta) / 864e5; break; // 1000 * 60 * 60 * 24, negate dst\n            case 'week': output = (this - that - zoneDelta) / 6048e5; break; // 1000 * 60 * 60 * 24 * 7, negate dst\n            default: output = this - that;\n        }\n\n        return asFloat ? output : absFloor(output);\n    }\n\n    function monthDiff (a, b) {\n        // difference in months\n        var wholeMonthDiff = ((b.year() - a.year()) * 12) + (b.month() - a.month()),\n            // b is in (anchor - 1 month, anchor + 1 month)\n            anchor = a.clone().add(wholeMonthDiff, 'months'),\n            anchor2, adjust;\n\n        if (b - anchor < 0) {\n            anchor2 = a.clone().add(wholeMonthDiff - 1, 'months');\n            // linear across the month\n            adjust = (b - anchor) / (anchor - anchor2);\n        } else {\n            anchor2 = a.clone().add(wholeMonthDiff + 1, 'months');\n            // linear across the month\n            adjust = (b - anchor) / (anchor2 - anchor);\n        }\n\n        //check for negative zero, return zero if negative zero\n        return -(wholeMonthDiff + adjust) || 0;\n    }\n\n    hooks.defaultFormat = 'YYYY-MM-DDTHH:mm:ssZ';\n    hooks.defaultFormatUtc = 'YYYY-MM-DDTHH:mm:ss[Z]';\n\n    function toString () {\n        return this.clone().locale('en').format('ddd MMM DD YYYY HH:mm:ss [GMT]ZZ');\n    }\n\n    function toISOString(keepOffset) {\n        if (!this.isValid()) {\n            return null;\n        }\n        var utc = keepOffset !== true;\n        var m = utc ? this.clone().utc() : this;\n        if (m.year() < 0 || m.year() > 9999) {\n            return formatMoment(m, utc ? 'YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]' : 'YYYYYY-MM-DD[T]HH:mm:ss.SSSZ');\n        }\n        if (isFunction(Date.prototype.toISOString)) {\n            // native implementation is ~50x faster, use it when we can\n            if (utc) {\n                return this.toDate().toISOString();\n            } else {\n                return new Date(this.valueOf() + this.utcOffset() * 60 * 1000).toISOString().replace('Z', formatMoment(m, 'Z'));\n            }\n        }\n        return formatMoment(m, utc ? 'YYYY-MM-DD[T]HH:mm:ss.SSS[Z]' : 'YYYY-MM-DD[T]HH:mm:ss.SSSZ');\n    }\n\n    /**\n     * Return a human readable representation of a moment that can\n     * also be evaluated to get a new moment which is the same\n     *\n     * @link https://nodejs.org/dist/latest/docs/api/util.html#util_custom_inspect_function_on_objects\n     */\n    function inspect () {\n        if (!this.isValid()) {\n            return 'moment.invalid(/* ' + this._i + ' */)';\n        }\n        var func = 'moment';\n        var zone = '';\n        if (!this.isLocal()) {\n            func = this.utcOffset() === 0 ? 'moment.utc' : 'moment.parseZone';\n            zone = 'Z';\n        }\n        var prefix = '[' + func + '(\"]';\n        var year = (0 <= this.year() && this.year() <= 9999) ? 'YYYY' : 'YYYYYY';\n        var datetime = '-MM-DD[T]HH:mm:ss.SSS';\n        var suffix = zone + '[\")]';\n\n        return this.format(prefix + year + datetime + suffix);\n    }\n\n    function format (inputString) {\n        if (!inputString) {\n            inputString = this.isUtc() ? hooks.defaultFormatUtc : hooks.defaultFormat;\n        }\n        var output = formatMoment(this, inputString);\n        return this.localeData().postformat(output);\n    }\n\n    function from (time, withoutSuffix) {\n        if (this.isValid() &&\n                ((isMoment(time) && time.isValid()) ||\n                 createLocal(time).isValid())) {\n            return createDuration({to: this, from: time}).locale(this.locale()).humanize(!withoutSuffix);\n        } else {\n            return this.localeData().invalidDate();\n        }\n    }\n\n    function fromNow (withoutSuffix) {\n        return this.from(createLocal(), withoutSuffix);\n    }\n\n    function to (time, withoutSuffix) {\n        if (this.isValid() &&\n                ((isMoment(time) && time.isValid()) ||\n                 createLocal(time).isValid())) {\n            return createDuration({from: this, to: time}).locale(this.locale()).humanize(!withoutSuffix);\n        } else {\n            return this.localeData().invalidDate();\n        }\n    }\n\n    function toNow (withoutSuffix) {\n        return this.to(createLocal(), withoutSuffix);\n    }\n\n    // If passed a locale key, it will set the locale for this\n    // instance.  Otherwise, it will return the locale configuration\n    // variables for this instance.\n    function locale (key) {\n        var newLocaleData;\n\n        if (key === undefined) {\n            return this._locale._abbr;\n        } else {\n            newLocaleData = getLocale(key);\n            if (newLocaleData != null) {\n                this._locale = newLocaleData;\n            }\n            return this;\n        }\n    }\n\n    var lang = deprecate(\n        'moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.',\n        function (key) {\n            if (key === undefined) {\n                return this.localeData();\n            } else {\n                return this.locale(key);\n            }\n        }\n    );\n\n    function localeData () {\n        return this._locale;\n    }\n\n    var MS_PER_SECOND = 1000;\n    var MS_PER_MINUTE = 60 * MS_PER_SECOND;\n    var MS_PER_HOUR = 60 * MS_PER_MINUTE;\n    var MS_PER_400_YEARS = (365 * 400 + 97) * 24 * MS_PER_HOUR;\n\n    // actual modulo - handles negative numbers (for dates before 1970):\n    function mod$1(dividend, divisor) {\n        return (dividend % divisor + divisor) % divisor;\n    }\n\n    function localStartOfDate(y, m, d) {\n        // the date constructor remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            return new Date(y + 400, m, d) - MS_PER_400_YEARS;\n        } else {\n            return new Date(y, m, d).valueOf();\n        }\n    }\n\n    function utcStartOfDate(y, m, d) {\n        // Date.UTC remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            return Date.UTC(y + 400, m, d) - MS_PER_400_YEARS;\n        } else {\n            return Date.UTC(y, m, d);\n        }\n    }\n\n    function startOf (units) {\n        var time;\n        units = normalizeUnits(units);\n        if (units === undefined || units === 'millisecond' || !this.isValid()) {\n            return this;\n        }\n\n        var startOfDate = this._isUTC ? utcStartOfDate : localStartOfDate;\n\n        switch (units) {\n            case 'year':\n                time = startOfDate(this.year(), 0, 1);\n                break;\n            case 'quarter':\n                time = startOfDate(this.year(), this.month() - this.month() % 3, 1);\n                break;\n            case 'month':\n                time = startOfDate(this.year(), this.month(), 1);\n                break;\n            case 'week':\n                time = startOfDate(this.year(), this.month(), this.date() - this.weekday());\n                break;\n            case 'isoWeek':\n                time = startOfDate(this.year(), this.month(), this.date() - (this.isoWeekday() - 1));\n                break;\n            case 'day':\n            case 'date':\n                time = startOfDate(this.year(), this.month(), this.date());\n                break;\n            case 'hour':\n                time = this._d.valueOf();\n                time -= mod$1(time + (this._isUTC ? 0 : this.utcOffset() * MS_PER_MINUTE), MS_PER_HOUR);\n                break;\n            case 'minute':\n                time = this._d.valueOf();\n                time -= mod$1(time, MS_PER_MINUTE);\n                break;\n            case 'second':\n                time = this._d.valueOf();\n                time -= mod$1(time, MS_PER_SECOND);\n                break;\n        }\n\n        this._d.setTime(time);\n        hooks.updateOffset(this, true);\n        return this;\n    }\n\n    function endOf (units) {\n        var time;\n        units = normalizeUnits(units);\n        if (units === undefined || units === 'millisecond' || !this.isValid()) {\n            return this;\n        }\n\n        var startOfDate = this._isUTC ? utcStartOfDate : localStartOfDate;\n\n        switch (units) {\n            case 'year':\n                time = startOfDate(this.year() + 1, 0, 1) - 1;\n                break;\n            case 'quarter':\n                time = startOfDate(this.year(), this.month() - this.month() % 3 + 3, 1) - 1;\n                break;\n            case 'month':\n                time = startOfDate(this.year(), this.month() + 1, 1) - 1;\n                break;\n            case 'week':\n                time = startOfDate(this.year(), this.month(), this.date() - this.weekday() + 7) - 1;\n                break;\n            case 'isoWeek':\n                time = startOfDate(this.year(), this.month(), this.date() - (this.isoWeekday() - 1) + 7) - 1;\n                break;\n            case 'day':\n            case 'date':\n                time = startOfDate(this.year(), this.month(), this.date() + 1) - 1;\n                break;\n            case 'hour':\n                time = this._d.valueOf();\n                time += MS_PER_HOUR - mod$1(time + (this._isUTC ? 0 : this.utcOffset() * MS_PER_MINUTE), MS_PER_HOUR) - 1;\n                break;\n            case 'minute':\n                time = this._d.valueOf();\n                time += MS_PER_MINUTE - mod$1(time, MS_PER_MINUTE) - 1;\n                break;\n            case 'second':\n                time = this._d.valueOf();\n                time += MS_PER_SECOND - mod$1(time, MS_PER_SECOND) - 1;\n                break;\n        }\n\n        this._d.setTime(time);\n        hooks.updateOffset(this, true);\n        return this;\n    }\n\n    function valueOf () {\n        return this._d.valueOf() - ((this._offset || 0) * 60000);\n    }\n\n    function unix () {\n        return Math.floor(this.valueOf() / 1000);\n    }\n\n    function toDate () {\n        return new Date(this.valueOf());\n    }\n\n    function toArray () {\n        var m = this;\n        return [m.year(), m.month(), m.date(), m.hour(), m.minute(), m.second(), m.millisecond()];\n    }\n\n    function toObject () {\n        var m = this;\n        return {\n            years: m.year(),\n            months: m.month(),\n            date: m.date(),\n            hours: m.hours(),\n            minutes: m.minutes(),\n            seconds: m.seconds(),\n            milliseconds: m.milliseconds()\n        };\n    }\n\n    function toJSON () {\n        // new Date(NaN).toJSON() === null\n        return this.isValid() ? this.toISOString() : null;\n    }\n\n    function isValid$2 () {\n        return isValid(this);\n    }\n\n    function parsingFlags () {\n        return extend({}, getParsingFlags(this));\n    }\n\n    function invalidAt () {\n        return getParsingFlags(this).overflow;\n    }\n\n    function creationData() {\n        return {\n            input: this._i,\n            format: this._f,\n            locale: this._locale,\n            isUTC: this._isUTC,\n            strict: this._strict\n        };\n    }\n\n    // FORMATTING\n\n    addFormatToken(0, ['gg', 2], 0, function () {\n        return this.weekYear() % 100;\n    });\n\n    addFormatToken(0, ['GG', 2], 0, function () {\n        return this.isoWeekYear() % 100;\n    });\n\n    function addWeekYearFormatToken (token, getter) {\n        addFormatToken(0, [token, token.length], 0, getter);\n    }\n\n    addWeekYearFormatToken('gggg',     'weekYear');\n    addWeekYearFormatToken('ggggg',    'weekYear');\n    addWeekYearFormatToken('GGGG',  'isoWeekYear');\n    addWeekYearFormatToken('GGGGG', 'isoWeekYear');\n\n    // ALIASES\n\n    addUnitAlias('weekYear', 'gg');\n    addUnitAlias('isoWeekYear', 'GG');\n\n    // PRIORITY\n\n    addUnitPriority('weekYear', 1);\n    addUnitPriority('isoWeekYear', 1);\n\n\n    // PARSING\n\n    addRegexToken('G',      matchSigned);\n    addRegexToken('g',      matchSigned);\n    addRegexToken('GG',     match1to2, match2);\n    addRegexToken('gg',     match1to2, match2);\n    addRegexToken('GGGG',   match1to4, match4);\n    addRegexToken('gggg',   match1to4, match4);\n    addRegexToken('GGGGG',  match1to6, match6);\n    addRegexToken('ggggg',  match1to6, match6);\n\n    addWeekParseToken(['gggg', 'ggggg', 'GGGG', 'GGGGG'], function (input, week, config, token) {\n        week[token.substr(0, 2)] = toInt(input);\n    });\n\n    addWeekParseToken(['gg', 'GG'], function (input, week, config, token) {\n        week[token] = hooks.parseTwoDigitYear(input);\n    });\n\n    // MOMENTS\n\n    function getSetWeekYear (input) {\n        return getSetWeekYearHelper.call(this,\n                input,\n                this.week(),\n                this.weekday(),\n                this.localeData()._week.dow,\n                this.localeData()._week.doy);\n    }\n\n    function getSetISOWeekYear (input) {\n        return getSetWeekYearHelper.call(this,\n                input, this.isoWeek(), this.isoWeekday(), 1, 4);\n    }\n\n    function getISOWeeksInYear () {\n        return weeksInYear(this.year(), 1, 4);\n    }\n\n    function getWeeksInYear () {\n        var weekInfo = this.localeData()._week;\n        return weeksInYear(this.year(), weekInfo.dow, weekInfo.doy);\n    }\n\n    function getSetWeekYearHelper(input, week, weekday, dow, doy) {\n        var weeksTarget;\n        if (input == null) {\n            return weekOfYear(this, dow, doy).year;\n        } else {\n            weeksTarget = weeksInYear(input, dow, doy);\n            if (week > weeksTarget) {\n                week = weeksTarget;\n            }\n            return setWeekAll.call(this, input, week, weekday, dow, doy);\n        }\n    }\n\n    function setWeekAll(weekYear, week, weekday, dow, doy) {\n        var dayOfYearData = dayOfYearFromWeeks(weekYear, week, weekday, dow, doy),\n            date = createUTCDate(dayOfYearData.year, 0, dayOfYearData.dayOfYear);\n\n        this.year(date.getUTCFullYear());\n        this.month(date.getUTCMonth());\n        this.date(date.getUTCDate());\n        return this;\n    }\n\n    // FORMATTING\n\n    addFormatToken('Q', 0, 'Qo', 'quarter');\n\n    // ALIASES\n\n    addUnitAlias('quarter', 'Q');\n\n    // PRIORITY\n\n    addUnitPriority('quarter', 7);\n\n    // PARSING\n\n    addRegexToken('Q', match1);\n    addParseToken('Q', function (input, array) {\n        array[MONTH] = (toInt(input) - 1) * 3;\n    });\n\n    // MOMENTS\n\n    function getSetQuarter (input) {\n        return input == null ? Math.ceil((this.month() + 1) / 3) : this.month((input - 1) * 3 + this.month() % 3);\n    }\n\n    // FORMATTING\n\n    addFormatToken('D', ['DD', 2], 'Do', 'date');\n\n    // ALIASES\n\n    addUnitAlias('date', 'D');\n\n    // PRIORITY\n    addUnitPriority('date', 9);\n\n    // PARSING\n\n    addRegexToken('D',  match1to2);\n    addRegexToken('DD', match1to2, match2);\n    addRegexToken('Do', function (isStrict, locale) {\n        // TODO: Remove \"ordinalParse\" fallback in next major release.\n        return isStrict ?\n          (locale._dayOfMonthOrdinalParse || locale._ordinalParse) :\n          locale._dayOfMonthOrdinalParseLenient;\n    });\n\n    addParseToken(['D', 'DD'], DATE);\n    addParseToken('Do', function (input, array) {\n        array[DATE] = toInt(input.match(match1to2)[0]);\n    });\n\n    // MOMENTS\n\n    var getSetDayOfMonth = makeGetSet('Date', true);\n\n    // FORMATTING\n\n    addFormatToken('DDD', ['DDDD', 3], 'DDDo', 'dayOfYear');\n\n    // ALIASES\n\n    addUnitAlias('dayOfYear', 'DDD');\n\n    // PRIORITY\n    addUnitPriority('dayOfYear', 4);\n\n    // PARSING\n\n    addRegexToken('DDD',  match1to3);\n    addRegexToken('DDDD', match3);\n    addParseToken(['DDD', 'DDDD'], function (input, array, config) {\n        config._dayOfYear = toInt(input);\n    });\n\n    // HELPERS\n\n    // MOMENTS\n\n    function getSetDayOfYear (input) {\n        var dayOfYear = Math.round((this.clone().startOf('day') - this.clone().startOf('year')) / 864e5) + 1;\n        return input == null ? dayOfYear : this.add((input - dayOfYear), 'd');\n    }\n\n    // FORMATTING\n\n    addFormatToken('m', ['mm', 2], 0, 'minute');\n\n    // ALIASES\n\n    addUnitAlias('minute', 'm');\n\n    // PRIORITY\n\n    addUnitPriority('minute', 14);\n\n    // PARSING\n\n    addRegexToken('m',  match1to2);\n    addRegexToken('mm', match1to2, match2);\n    addParseToken(['m', 'mm'], MINUTE);\n\n    // MOMENTS\n\n    var getSetMinute = makeGetSet('Minutes', false);\n\n    // FORMATTING\n\n    addFormatToken('s', ['ss', 2], 0, 'second');\n\n    // ALIASES\n\n    addUnitAlias('second', 's');\n\n    // PRIORITY\n\n    addUnitPriority('second', 15);\n\n    // PARSING\n\n    addRegexToken('s',  match1to2);\n    addRegexToken('ss', match1to2, match2);\n    addParseToken(['s', 'ss'], SECOND);\n\n    // MOMENTS\n\n    var getSetSecond = makeGetSet('Seconds', false);\n\n    // FORMATTING\n\n    addFormatToken('S', 0, 0, function () {\n        return ~~(this.millisecond() / 100);\n    });\n\n    addFormatToken(0, ['SS', 2], 0, function () {\n        return ~~(this.millisecond() / 10);\n    });\n\n    addFormatToken(0, ['SSS', 3], 0, 'millisecond');\n    addFormatToken(0, ['SSSS', 4], 0, function () {\n        return this.millisecond() * 10;\n    });\n    addFormatToken(0, ['SSSSS', 5], 0, function () {\n        return this.millisecond() * 100;\n    });\n    addFormatToken(0, ['SSSSSS', 6], 0, function () {\n        return this.millisecond() * 1000;\n    });\n    addFormatToken(0, ['SSSSSSS', 7], 0, function () {\n        return this.millisecond() * 10000;\n    });\n    addFormatToken(0, ['SSSSSSSS', 8], 0, function () {\n        return this.millisecond() * 100000;\n    });\n    addFormatToken(0, ['SSSSSSSSS', 9], 0, function () {\n        return this.millisecond() * 1000000;\n    });\n\n\n    // ALIASES\n\n    addUnitAlias('millisecond', 'ms');\n\n    // PRIORITY\n\n    addUnitPriority('millisecond', 16);\n\n    // PARSING\n\n    addRegexToken('S',    match1to3, match1);\n    addRegexToken('SS',   match1to3, match2);\n    addRegexToken('SSS',  match1to3, match3);\n\n    var token;\n    for (token = 'SSSS'; token.length <= 9; token += 'S') {\n        addRegexToken(token, matchUnsigned);\n    }\n\n    function parseMs(input, array) {\n        array[MILLISECOND] = toInt(('0.' + input) * 1000);\n    }\n\n    for (token = 'S'; token.length <= 9; token += 'S') {\n        addParseToken(token, parseMs);\n    }\n    // MOMENTS\n\n    var getSetMillisecond = makeGetSet('Milliseconds', false);\n\n    // FORMATTING\n\n    addFormatToken('z',  0, 0, 'zoneAbbr');\n    addFormatToken('zz', 0, 0, 'zoneName');\n\n    // MOMENTS\n\n    function getZoneAbbr () {\n        return this._isUTC ? 'UTC' : '';\n    }\n\n    function getZoneName () {\n        return this._isUTC ? 'Coordinated Universal Time' : '';\n    }\n\n    var proto = Moment.prototype;\n\n    proto.add               = add;\n    proto.calendar          = calendar$1;\n    proto.clone             = clone;\n    proto.diff              = diff;\n    proto.endOf             = endOf;\n    proto.format            = format;\n    proto.from              = from;\n    proto.fromNow           = fromNow;\n    proto.to                = to;\n    proto.toNow             = toNow;\n    proto.get               = stringGet;\n    proto.invalidAt         = invalidAt;\n    proto.isAfter           = isAfter;\n    proto.isBefore          = isBefore;\n    proto.isBetween         = isBetween;\n    proto.isSame            = isSame;\n    proto.isSameOrAfter     = isSameOrAfter;\n    proto.isSameOrBefore    = isSameOrBefore;\n    proto.isValid           = isValid$2;\n    proto.lang              = lang;\n    proto.locale            = locale;\n    proto.localeData        = localeData;\n    proto.max               = prototypeMax;\n    proto.min               = prototypeMin;\n    proto.parsingFlags      = parsingFlags;\n    proto.set               = stringSet;\n    proto.startOf           = startOf;\n    proto.subtract          = subtract;\n    proto.toArray           = toArray;\n    proto.toObject          = toObject;\n    proto.toDate            = toDate;\n    proto.toISOString       = toISOString;\n    proto.inspect           = inspect;\n    proto.toJSON            = toJSON;\n    proto.toString          = toString;\n    proto.unix              = unix;\n    proto.valueOf           = valueOf;\n    proto.creationData      = creationData;\n    proto.year       = getSetYear;\n    proto.isLeapYear = getIsLeapYear;\n    proto.weekYear    = getSetWeekYear;\n    proto.isoWeekYear = getSetISOWeekYear;\n    proto.quarter = proto.quarters = getSetQuarter;\n    proto.month       = getSetMonth;\n    proto.daysInMonth = getDaysInMonth;\n    proto.week           = proto.weeks        = getSetWeek;\n    proto.isoWeek        = proto.isoWeeks     = getSetISOWeek;\n    proto.weeksInYear    = getWeeksInYear;\n    proto.isoWeeksInYear = getISOWeeksInYear;\n    proto.date       = getSetDayOfMonth;\n    proto.day        = proto.days             = getSetDayOfWeek;\n    proto.weekday    = getSetLocaleDayOfWeek;\n    proto.isoWeekday = getSetISODayOfWeek;\n    proto.dayOfYear  = getSetDayOfYear;\n    proto.hour = proto.hours = getSetHour;\n    proto.minute = proto.minutes = getSetMinute;\n    proto.second = proto.seconds = getSetSecond;\n    proto.millisecond = proto.milliseconds = getSetMillisecond;\n    proto.utcOffset            = getSetOffset;\n    proto.utc                  = setOffsetToUTC;\n    proto.local                = setOffsetToLocal;\n    proto.parseZone            = setOffsetToParsedOffset;\n    proto.hasAlignedHourOffset = hasAlignedHourOffset;\n    proto.isDST                = isDaylightSavingTime;\n    proto.isLocal              = isLocal;\n    proto.isUtcOffset          = isUtcOffset;\n    proto.isUtc                = isUtc;\n    proto.isUTC                = isUtc;\n    proto.zoneAbbr = getZoneAbbr;\n    proto.zoneName = getZoneName;\n    proto.dates  = deprecate('dates accessor is deprecated. Use date instead.', getSetDayOfMonth);\n    proto.months = deprecate('months accessor is deprecated. Use month instead', getSetMonth);\n    proto.years  = deprecate('years accessor is deprecated. Use year instead', getSetYear);\n    proto.zone   = deprecate('moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/', getSetZone);\n    proto.isDSTShifted = deprecate('isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information', isDaylightSavingTimeShifted);\n\n    function createUnix (input) {\n        return createLocal(input * 1000);\n    }\n\n    function createInZone () {\n        return createLocal.apply(null, arguments).parseZone();\n    }\n\n    function preParsePostFormat (string) {\n        return string;\n    }\n\n    var proto$1 = Locale.prototype;\n\n    proto$1.calendar        = calendar;\n    proto$1.longDateFormat  = longDateFormat;\n    proto$1.invalidDate     = invalidDate;\n    proto$1.ordinal         = ordinal;\n    proto$1.preparse        = preParsePostFormat;\n    proto$1.postformat      = preParsePostFormat;\n    proto$1.relativeTime    = relativeTime;\n    proto$1.pastFuture      = pastFuture;\n    proto$1.set             = set;\n\n    proto$1.months            =        localeMonths;\n    proto$1.monthsShort       =        localeMonthsShort;\n    proto$1.monthsParse       =        localeMonthsParse;\n    proto$1.monthsRegex       = monthsRegex;\n    proto$1.monthsShortRegex  = monthsShortRegex;\n    proto$1.week = localeWeek;\n    proto$1.firstDayOfYear = localeFirstDayOfYear;\n    proto$1.firstDayOfWeek = localeFirstDayOfWeek;\n\n    proto$1.weekdays       =        localeWeekdays;\n    proto$1.weekdaysMin    =        localeWeekdaysMin;\n    proto$1.weekdaysShort  =        localeWeekdaysShort;\n    proto$1.weekdaysParse  =        localeWeekdaysParse;\n\n    proto$1.weekdaysRegex       =        weekdaysRegex;\n    proto$1.weekdaysShortRegex  =        weekdaysShortRegex;\n    proto$1.weekdaysMinRegex    =        weekdaysMinRegex;\n\n    proto$1.isPM = localeIsPM;\n    proto$1.meridiem = localeMeridiem;\n\n    function get$1 (format, index, field, setter) {\n        var locale = getLocale();\n        var utc = createUTC().set(setter, index);\n        return locale[field](utc, format);\n    }\n\n    function listMonthsImpl (format, index, field) {\n        if (isNumber(format)) {\n            index = format;\n            format = undefined;\n        }\n\n        format = format || '';\n\n        if (index != null) {\n            return get$1(format, index, field, 'month');\n        }\n\n        var i;\n        var out = [];\n        for (i = 0; i < 12; i++) {\n            out[i] = get$1(format, i, field, 'month');\n        }\n        return out;\n    }\n\n    // ()\n    // (5)\n    // (fmt, 5)\n    // (fmt)\n    // (true)\n    // (true, 5)\n    // (true, fmt, 5)\n    // (true, fmt)\n    function listWeekdaysImpl (localeSorted, format, index, field) {\n        if (typeof localeSorted === 'boolean') {\n            if (isNumber(format)) {\n                index = format;\n                format = undefined;\n            }\n\n            format = format || '';\n        } else {\n            format = localeSorted;\n            index = format;\n            localeSorted = false;\n\n            if (isNumber(format)) {\n                index = format;\n                format = undefined;\n            }\n\n            format = format || '';\n        }\n\n        var locale = getLocale(),\n            shift = localeSorted ? locale._week.dow : 0;\n\n        if (index != null) {\n            return get$1(format, (index + shift) % 7, field, 'day');\n        }\n\n        var i;\n        var out = [];\n        for (i = 0; i < 7; i++) {\n            out[i] = get$1(format, (i + shift) % 7, field, 'day');\n        }\n        return out;\n    }\n\n    function listMonths (format, index) {\n        return listMonthsImpl(format, index, 'months');\n    }\n\n    function listMonthsShort (format, index) {\n        return listMonthsImpl(format, index, 'monthsShort');\n    }\n\n    function listWeekdays (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdays');\n    }\n\n    function listWeekdaysShort (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdaysShort');\n    }\n\n    function listWeekdaysMin (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdaysMin');\n    }\n\n    getSetGlobalLocale('en', {\n        dayOfMonthOrdinalParse: /\\d{1,2}(th|st|nd|rd)/,\n        ordinal : function (number) {\n            var b = number % 10,\n                output = (toInt(number % 100 / 10) === 1) ? 'th' :\n                (b === 1) ? 'st' :\n                (b === 2) ? 'nd' :\n                (b === 3) ? 'rd' : 'th';\n            return number + output;\n        }\n    });\n\n    // Side effect imports\n\n    hooks.lang = deprecate('moment.lang is deprecated. Use moment.locale instead.', getSetGlobalLocale);\n    hooks.langData = deprecate('moment.langData is deprecated. Use moment.localeData instead.', getLocale);\n\n    var mathAbs = Math.abs;\n\n    function abs () {\n        var data           = this._data;\n\n        this._milliseconds = mathAbs(this._milliseconds);\n        this._days         = mathAbs(this._days);\n        this._months       = mathAbs(this._months);\n\n        data.milliseconds  = mathAbs(data.milliseconds);\n        data.seconds       = mathAbs(data.seconds);\n        data.minutes       = mathAbs(data.minutes);\n        data.hours         = mathAbs(data.hours);\n        data.months        = mathAbs(data.months);\n        data.years         = mathAbs(data.years);\n\n        return this;\n    }\n\n    function addSubtract$1 (duration, input, value, direction) {\n        var other = createDuration(input, value);\n\n        duration._milliseconds += direction * other._milliseconds;\n        duration._days         += direction * other._days;\n        duration._months       += direction * other._months;\n\n        return duration._bubble();\n    }\n\n    // supports only 2.0-style add(1, 's') or add(duration)\n    function add$1 (input, value) {\n        return addSubtract$1(this, input, value, 1);\n    }\n\n    // supports only 2.0-style subtract(1, 's') or subtract(duration)\n    function subtract$1 (input, value) {\n        return addSubtract$1(this, input, value, -1);\n    }\n\n    function absCeil (number) {\n        if (number < 0) {\n            return Math.floor(number);\n        } else {\n            return Math.ceil(number);\n        }\n    }\n\n    function bubble () {\n        var milliseconds = this._milliseconds;\n        var days         = this._days;\n        var months       = this._months;\n        var data         = this._data;\n        var seconds, minutes, hours, years, monthsFromDays;\n\n        // if we have a mix of positive and negative values, bubble down first\n        // check: https://github.com/moment/moment/issues/2166\n        if (!((milliseconds >= 0 && days >= 0 && months >= 0) ||\n                (milliseconds <= 0 && days <= 0 && months <= 0))) {\n            milliseconds += absCeil(monthsToDays(months) + days) * 864e5;\n            days = 0;\n            months = 0;\n        }\n\n        // The following code bubbles up values, see the tests for\n        // examples of what that means.\n        data.milliseconds = milliseconds % 1000;\n\n        seconds           = absFloor(milliseconds / 1000);\n        data.seconds      = seconds % 60;\n\n        minutes           = absFloor(seconds / 60);\n        data.minutes      = minutes % 60;\n\n        hours             = absFloor(minutes / 60);\n        data.hours        = hours % 24;\n\n        days += absFloor(hours / 24);\n\n        // convert days to months\n        monthsFromDays = absFloor(daysToMonths(days));\n        months += monthsFromDays;\n        days -= absCeil(monthsToDays(monthsFromDays));\n\n        // 12 months -> 1 year\n        years = absFloor(months / 12);\n        months %= 12;\n\n        data.days   = days;\n        data.months = months;\n        data.years  = years;\n\n        return this;\n    }\n\n    function daysToMonths (days) {\n        // 400 years have 146097 days (taking into account leap year rules)\n        // 400 years have 12 months === 4800\n        return days * 4800 / 146097;\n    }\n\n    function monthsToDays (months) {\n        // the reverse of daysToMonths\n        return months * 146097 / 4800;\n    }\n\n    function as (units) {\n        if (!this.isValid()) {\n            return NaN;\n        }\n        var days;\n        var months;\n        var milliseconds = this._milliseconds;\n\n        units = normalizeUnits(units);\n\n        if (units === 'month' || units === 'quarter' || units === 'year') {\n            days = this._days + milliseconds / 864e5;\n            months = this._months + daysToMonths(days);\n            switch (units) {\n                case 'month':   return months;\n                case 'quarter': return months / 3;\n                case 'year':    return months / 12;\n            }\n        } else {\n            // handle milliseconds separately because of floating point math errors (issue #1867)\n            days = this._days + Math.round(monthsToDays(this._months));\n            switch (units) {\n                case 'week'   : return days / 7     + milliseconds / 6048e5;\n                case 'day'    : return days         + milliseconds / 864e5;\n                case 'hour'   : return days * 24    + milliseconds / 36e5;\n                case 'minute' : return days * 1440  + milliseconds / 6e4;\n                case 'second' : return days * 86400 + milliseconds / 1000;\n                // Math.floor prevents floating point math errors here\n                case 'millisecond': return Math.floor(days * 864e5) + milliseconds;\n                default: throw new Error('Unknown unit ' + units);\n            }\n        }\n    }\n\n    // TODO: Use this.as('ms')?\n    function valueOf$1 () {\n        if (!this.isValid()) {\n            return NaN;\n        }\n        return (\n            this._milliseconds +\n            this._days * 864e5 +\n            (this._months % 12) * 2592e6 +\n            toInt(this._months / 12) * 31536e6\n        );\n    }\n\n    function makeAs (alias) {\n        return function () {\n            return this.as(alias);\n        };\n    }\n\n    var asMilliseconds = makeAs('ms');\n    var asSeconds      = makeAs('s');\n    var asMinutes      = makeAs('m');\n    var asHours        = makeAs('h');\n    var asDays         = makeAs('d');\n    var asWeeks        = makeAs('w');\n    var asMonths       = makeAs('M');\n    var asQuarters     = makeAs('Q');\n    var asYears        = makeAs('y');\n\n    function clone$1 () {\n        return createDuration(this);\n    }\n\n    function get$2 (units) {\n        units = normalizeUnits(units);\n        return this.isValid() ? this[units + 's']() : NaN;\n    }\n\n    function makeGetter(name) {\n        return function () {\n            return this.isValid() ? this._data[name] : NaN;\n        };\n    }\n\n    var milliseconds = makeGetter('milliseconds');\n    var seconds      = makeGetter('seconds');\n    var minutes      = makeGetter('minutes');\n    var hours        = makeGetter('hours');\n    var days         = makeGetter('days');\n    var months       = makeGetter('months');\n    var years        = makeGetter('years');\n\n    function weeks () {\n        return absFloor(this.days() / 7);\n    }\n\n    var round = Math.round;\n    var thresholds = {\n        ss: 44,         // a few seconds to seconds\n        s : 45,         // seconds to minute\n        m : 45,         // minutes to hour\n        h : 22,         // hours to day\n        d : 26,         // days to month\n        M : 11          // months to year\n    };\n\n    // helper function for moment.fn.from, moment.fn.fromNow, and moment.duration.fn.humanize\n    function substituteTimeAgo(string, number, withoutSuffix, isFuture, locale) {\n        return locale.relativeTime(number || 1, !!withoutSuffix, string, isFuture);\n    }\n\n    function relativeTime$1 (posNegDuration, withoutSuffix, locale) {\n        var duration = createDuration(posNegDuration).abs();\n        var seconds  = round(duration.as('s'));\n        var minutes  = round(duration.as('m'));\n        var hours    = round(duration.as('h'));\n        var days     = round(duration.as('d'));\n        var months   = round(duration.as('M'));\n        var years    = round(duration.as('y'));\n\n        var a = seconds <= thresholds.ss && ['s', seconds]  ||\n                seconds < thresholds.s   && ['ss', seconds] ||\n                minutes <= 1             && ['m']           ||\n                minutes < thresholds.m   && ['mm', minutes] ||\n                hours   <= 1             && ['h']           ||\n                hours   < thresholds.h   && ['hh', hours]   ||\n                days    <= 1             && ['d']           ||\n                days    < thresholds.d   && ['dd', days]    ||\n                months  <= 1             && ['M']           ||\n                months  < thresholds.M   && ['MM', months]  ||\n                years   <= 1             && ['y']           || ['yy', years];\n\n        a[2] = withoutSuffix;\n        a[3] = +posNegDuration > 0;\n        a[4] = locale;\n        return substituteTimeAgo.apply(null, a);\n    }\n\n    // This function allows you to set the rounding function for relative time strings\n    function getSetRelativeTimeRounding (roundingFunction) {\n        if (roundingFunction === undefined) {\n            return round;\n        }\n        if (typeof(roundingFunction) === 'function') {\n            round = roundingFunction;\n            return true;\n        }\n        return false;\n    }\n\n    // This function allows you to set a threshold for relative time strings\n    function getSetRelativeTimeThreshold (threshold, limit) {\n        if (thresholds[threshold] === undefined) {\n            return false;\n        }\n        if (limit === undefined) {\n            return thresholds[threshold];\n        }\n        thresholds[threshold] = limit;\n        if (threshold === 's') {\n            thresholds.ss = limit - 1;\n        }\n        return true;\n    }\n\n    function humanize (withSuffix) {\n        if (!this.isValid()) {\n            return this.localeData().invalidDate();\n        }\n\n        var locale = this.localeData();\n        var output = relativeTime$1(this, !withSuffix, locale);\n\n        if (withSuffix) {\n            output = locale.pastFuture(+this, output);\n        }\n\n        return locale.postformat(output);\n    }\n\n    var abs$1 = Math.abs;\n\n    function sign(x) {\n        return ((x > 0) - (x < 0)) || +x;\n    }\n\n    function toISOString$1() {\n        // for ISO strings we do not use the normal bubbling rules:\n        //  * milliseconds bubble up until they become hours\n        //  * days do not bubble at all\n        //  * months bubble up until they become years\n        // This is because there is no context-free conversion between hours and days\n        // (think of clock changes)\n        // and also not between days and months (28-31 days per month)\n        if (!this.isValid()) {\n            return this.localeData().invalidDate();\n        }\n\n        var seconds = abs$1(this._milliseconds) / 1000;\n        var days         = abs$1(this._days);\n        var months       = abs$1(this._months);\n        var minutes, hours, years;\n\n        // 3600 seconds -> 60 minutes -> 1 hour\n        minutes           = absFloor(seconds / 60);\n        hours             = absFloor(minutes / 60);\n        seconds %= 60;\n        minutes %= 60;\n\n        // 12 months -> 1 year\n        years  = absFloor(months / 12);\n        months %= 12;\n\n\n        // inspired by https://github.com/dordille/moment-isoduration/blob/master/moment.isoduration.js\n        var Y = years;\n        var M = months;\n        var D = days;\n        var h = hours;\n        var m = minutes;\n        var s = seconds ? seconds.toFixed(3).replace(/\\.?0+$/, '') : '';\n        var total = this.asSeconds();\n\n        if (!total) {\n            // this is the same as C#'s (Noda) and python (isodate)...\n            // but not other JS (goog.date)\n            return 'P0D';\n        }\n\n        var totalSign = total < 0 ? '-' : '';\n        var ymSign = sign(this._months) !== sign(total) ? '-' : '';\n        var daysSign = sign(this._days) !== sign(total) ? '-' : '';\n        var hmsSign = sign(this._milliseconds) !== sign(total) ? '-' : '';\n\n        return totalSign + 'P' +\n            (Y ? ymSign + Y + 'Y' : '') +\n            (M ? ymSign + M + 'M' : '') +\n            (D ? daysSign + D + 'D' : '') +\n            ((h || m || s) ? 'T' : '') +\n            (h ? hmsSign + h + 'H' : '') +\n            (m ? hmsSign + m + 'M' : '') +\n            (s ? hmsSign + s + 'S' : '');\n    }\n\n    var proto$2 = Duration.prototype;\n\n    proto$2.isValid        = isValid$1;\n    proto$2.abs            = abs;\n    proto$2.add            = add$1;\n    proto$2.subtract       = subtract$1;\n    proto$2.as             = as;\n    proto$2.asMilliseconds = asMilliseconds;\n    proto$2.asSeconds      = asSeconds;\n    proto$2.asMinutes      = asMinutes;\n    proto$2.asHours        = asHours;\n    proto$2.asDays         = asDays;\n    proto$2.asWeeks        = asWeeks;\n    proto$2.asMonths       = asMonths;\n    proto$2.asQuarters     = asQuarters;\n    proto$2.asYears        = asYears;\n    proto$2.valueOf        = valueOf$1;\n    proto$2._bubble        = bubble;\n    proto$2.clone          = clone$1;\n    proto$2.get            = get$2;\n    proto$2.milliseconds   = milliseconds;\n    proto$2.seconds        = seconds;\n    proto$2.minutes        = minutes;\n    proto$2.hours          = hours;\n    proto$2.days           = days;\n    proto$2.weeks          = weeks;\n    proto$2.months         = months;\n    proto$2.years          = years;\n    proto$2.humanize       = humanize;\n    proto$2.toISOString    = toISOString$1;\n    proto$2.toString       = toISOString$1;\n    proto$2.toJSON         = toISOString$1;\n    proto$2.locale         = locale;\n    proto$2.localeData     = localeData;\n\n    proto$2.toIsoString = deprecate('toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)', toISOString$1);\n    proto$2.lang = lang;\n\n    // Side effect imports\n\n    // FORMATTING\n\n    addFormatToken('X', 0, 0, 'unix');\n    addFormatToken('x', 0, 0, 'valueOf');\n\n    // PARSING\n\n    addRegexToken('x', matchSigned);\n    addRegexToken('X', matchTimestamp);\n    addParseToken('X', function (input, array, config) {\n        config._d = new Date(parseFloat(input, 10) * 1000);\n    });\n    addParseToken('x', function (input, array, config) {\n        config._d = new Date(toInt(input));\n    });\n\n    // Side effect imports\n\n\n    hooks.version = '2.24.0';\n\n    setHookCallback(createLocal);\n\n    hooks.fn                    = proto;\n    hooks.min                   = min;\n    hooks.max                   = max;\n    hooks.now                   = now;\n    hooks.utc                   = createUTC;\n    hooks.unix                  = createUnix;\n    hooks.months                = listMonths;\n    hooks.isDate                = isDate;\n    hooks.locale                = getSetGlobalLocale;\n    hooks.invalid               = createInvalid;\n    hooks.duration              = createDuration;\n    hooks.isMoment              = isMoment;\n    hooks.weekdays              = listWeekdays;\n    hooks.parseZone             = createInZone;\n    hooks.localeData            = getLocale;\n    hooks.isDuration            = isDuration;\n    hooks.monthsShort           = listMonthsShort;\n    hooks.weekdaysMin           = listWeekdaysMin;\n    hooks.defineLocale          = defineLocale;\n    hooks.updateLocale          = updateLocale;\n    hooks.locales               = listLocales;\n    hooks.weekdaysShort         = listWeekdaysShort;\n    hooks.normalizeUnits        = normalizeUnits;\n    hooks.relativeTimeRounding  = getSetRelativeTimeRounding;\n    hooks.relativeTimeThreshold = getSetRelativeTimeThreshold;\n    hooks.calendarFormat        = getCalendarFormat;\n    hooks.prototype             = proto;\n\n    // currently HTML5 input type only supports 24-hour formats\n    hooks.HTML5_FMT = {\n        DATETIME_LOCAL: 'YYYY-MM-DDTHH:mm',             // <input type=\"datetime-local\" />\n        DATETIME_LOCAL_SECONDS: 'YYYY-MM-DDTHH:mm:ss',  // <input type=\"datetime-local\" step=\"1\" />\n        DATETIME_LOCAL_MS: 'YYYY-MM-DDTHH:mm:ss.SSS',   // <input type=\"datetime-local\" step=\"0.001\" />\n        DATE: 'YYYY-MM-DD',                             // <input type=\"date\" />\n        TIME: 'HH:mm',                                  // <input type=\"time\" />\n        TIME_SECONDS: 'HH:mm:ss',                       // <input type=\"time\" step=\"1\" />\n        TIME_MS: 'HH:mm:ss.SSS',                        // <input type=\"time\" step=\"0.001\" />\n        WEEK: 'GGGG-[W]WW',                             // <input type=\"week\" />\n        MONTH: 'YYYY-MM'                                // <input type=\"month\" />\n    };\n\n    return hooks;\n\n})));\n","var check = function (it) {\n  return it && it.Math == Math && it;\n};\n\n// https://github.com/zloirock/core-js/issues/86#issuecomment-115759028\nmodule.exports =\n  // eslint-disable-next-line no-undef\n  check(typeof globalThis == 'object' && globalThis) ||\n  check(typeof window == 'object' && window) ||\n  check(typeof self == 'object' && self) ||\n  check(typeof global == 'object' && global) ||\n  // eslint-disable-next-line no-new-func\n  Function('return this')();\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function processRelativeTime(number, withoutSuffix, key, isFuture) {\n        var format = {\n            'm': ['eine Minute', 'einer Minute'],\n            'h': ['eine Stunde', 'einer Stunde'],\n            'd': ['ein Tag', 'einem Tag'],\n            'dd': [number + ' Tage', number + ' Tagen'],\n            'M': ['ein Monat', 'einem Monat'],\n            'MM': [number + ' Monate', number + ' Monaten'],\n            'y': ['ein Jahr', 'einem Jahr'],\n            'yy': [number + ' Jahre', number + ' Jahren']\n        };\n        return withoutSuffix ? format[key][0] : format[key][1];\n    }\n\n    var de = moment.defineLocale('de', {\n        months : 'Januar_Februar_März_April_Mai_Juni_Juli_August_September_Oktober_November_Dezember'.split('_'),\n        monthsShort : 'Jan._Feb._März_Apr._Mai_Juni_Juli_Aug._Sep._Okt._Nov._Dez.'.split('_'),\n        monthsParseExact : true,\n        weekdays : 'Sonntag_Montag_Dienstag_Mittwoch_Donnerstag_Freitag_Samstag'.split('_'),\n        weekdaysShort : 'So._Mo._Di._Mi._Do._Fr._Sa.'.split('_'),\n        weekdaysMin : 'So_Mo_Di_Mi_Do_Fr_Sa'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT: 'HH:mm',\n            LTS: 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D. MMMM YYYY',\n            LLL : 'D. MMMM YYYY HH:mm',\n            LLLL : 'dddd, D. MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[heute um] LT [Uhr]',\n            sameElse: 'L',\n            nextDay: '[morgen um] LT [Uhr]',\n            nextWeek: 'dddd [um] LT [Uhr]',\n            lastDay: '[gestern um] LT [Uhr]',\n            lastWeek: '[letzten] dddd [um] LT [Uhr]'\n        },\n        relativeTime : {\n            future : 'in %s',\n            past : 'vor %s',\n            s : 'ein paar Sekunden',\n            ss : '%d Sekunden',\n            m : processRelativeTime,\n            mm : '%d Minuten',\n            h : processRelativeTime,\n            hh : '%d Stunden',\n            d : processRelativeTime,\n            dd : processRelativeTime,\n            M : processRelativeTime,\n            MM : processRelativeTime,\n            y : processRelativeTime,\n            yy : processRelativeTime\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}\\./,\n        ordinal : '%d.',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return de;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsShortDot = 'ene._feb._mar._abr._may._jun._jul._ago._sep._oct._nov._dic.'.split('_'),\n        monthsShort = 'ene_feb_mar_abr_may_jun_jul_ago_sep_oct_nov_dic'.split('_');\n\n    var monthsParse = [/^ene/i, /^feb/i, /^mar/i, /^abr/i, /^may/i, /^jun/i, /^jul/i, /^ago/i, /^sep/i, /^oct/i, /^nov/i, /^dic/i];\n    var monthsRegex = /^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre|ene\\.?|feb\\.?|mar\\.?|abr\\.?|may\\.?|jun\\.?|jul\\.?|ago\\.?|sep\\.?|oct\\.?|nov\\.?|dic\\.?)/i;\n\n    var es = moment.defineLocale('es', {\n        months : 'enero_febrero_marzo_abril_mayo_junio_julio_agosto_septiembre_octubre_noviembre_diciembre'.split('_'),\n        monthsShort : function (m, format) {\n            if (!m) {\n                return monthsShortDot;\n            } else if (/-MMM-/.test(format)) {\n                return monthsShort[m.month()];\n            } else {\n                return monthsShortDot[m.month()];\n            }\n        },\n        monthsRegex : monthsRegex,\n        monthsShortRegex : monthsRegex,\n        monthsStrictRegex : /^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)/i,\n        monthsShortStrictRegex : /^(ene\\.?|feb\\.?|mar\\.?|abr\\.?|may\\.?|jun\\.?|jul\\.?|ago\\.?|sep\\.?|oct\\.?|nov\\.?|dic\\.?)/i,\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n        weekdays : 'domingo_lunes_martes_miércoles_jueves_viernes_sábado'.split('_'),\n        weekdaysShort : 'dom._lun._mar._mié._jue._vie._sáb.'.split('_'),\n        weekdaysMin : 'do_lu_ma_mi_ju_vi_sá'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'H:mm',\n            LTS : 'H:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D [de] MMMM [de] YYYY',\n            LLL : 'D [de] MMMM [de] YYYY H:mm',\n            LLLL : 'dddd, D [de] MMMM [de] YYYY H:mm'\n        },\n        calendar : {\n            sameDay : function () {\n                return '[hoy a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            nextDay : function () {\n                return '[mañana a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            nextWeek : function () {\n                return 'dddd [a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            lastDay : function () {\n                return '[ayer a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            lastWeek : function () {\n                return '[el] dddd [pasado a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            sameElse : 'L'\n        },\n        relativeTime : {\n            future : 'en %s',\n            past : 'hace %s',\n            s : 'unos segundos',\n            ss : '%d segundos',\n            m : 'un minuto',\n            mm : '%d minutos',\n            h : 'una hora',\n            hh : '%d horas',\n            d : 'un día',\n            dd : '%d días',\n            M : 'un mes',\n            MM : '%d meses',\n            y : 'un año',\n            yy : '%d años'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}º/,\n        ordinal : '%dº',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return es;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var fr = moment.defineLocale('fr', {\n        months : 'janvier_février_mars_avril_mai_juin_juillet_août_septembre_octobre_novembre_décembre'.split('_'),\n        monthsShort : 'janv._févr._mars_avr._mai_juin_juil._août_sept._oct._nov._déc.'.split('_'),\n        monthsParseExact : true,\n        weekdays : 'dimanche_lundi_mardi_mercredi_jeudi_vendredi_samedi'.split('_'),\n        weekdaysShort : 'dim._lun._mar._mer._jeu._ven._sam.'.split('_'),\n        weekdaysMin : 'di_lu_ma_me_je_ve_sa'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay : '[Aujourd’hui à] LT',\n            nextDay : '[Demain à] LT',\n            nextWeek : 'dddd [à] LT',\n            lastDay : '[Hier à] LT',\n            lastWeek : 'dddd [dernier à] LT',\n            sameElse : 'L'\n        },\n        relativeTime : {\n            future : 'dans %s',\n            past : 'il y a %s',\n            s : 'quelques secondes',\n            ss : '%d secondes',\n            m : 'une minute',\n            mm : '%d minutes',\n            h : 'une heure',\n            hh : '%d heures',\n            d : 'un jour',\n            dd : '%d jours',\n            M : 'un mois',\n            MM : '%d mois',\n            y : 'un an',\n            yy : '%d ans'\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}(er|)/,\n        ordinal : function (number, period) {\n            switch (period) {\n                // TODO: Return 'e' when day of month > 1. Move this case inside\n                // block for masculine words below.\n                // See https://github.com/moment/moment/issues/3375\n                case 'D':\n                    return number + (number === 1 ? 'er' : '');\n\n                // Words with masculine grammatical gender: mois, trimestre, jour\n                default:\n                case 'M':\n                case 'Q':\n                case 'DDD':\n                case 'd':\n                    return number + (number === 1 ? 'er' : 'e');\n\n                // Words with feminine grammatical gender: semaine\n                case 'w':\n                case 'W':\n                    return number + (number === 1 ? 're' : 'e');\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return fr;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var it = moment.defineLocale('it', {\n        months : 'gennaio_febbraio_marzo_aprile_maggio_giugno_luglio_agosto_settembre_ottobre_novembre_dicembre'.split('_'),\n        monthsShort : 'gen_feb_mar_apr_mag_giu_lug_ago_set_ott_nov_dic'.split('_'),\n        weekdays : 'domenica_lunedì_martedì_mercoledì_giovedì_venerdì_sabato'.split('_'),\n        weekdaysShort : 'dom_lun_mar_mer_gio_ven_sab'.split('_'),\n        weekdaysMin : 'do_lu_ma_me_gi_ve_sa'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[Oggi alle] LT',\n            nextDay: '[Domani alle] LT',\n            nextWeek: 'dddd [alle] LT',\n            lastDay: '[Ieri alle] LT',\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[la scorsa] dddd [alle] LT';\n                    default:\n                        return '[lo scorso] dddd [alle] LT';\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : function (s) {\n                return ((/^[0-9].+$/).test(s) ? 'tra' : 'in') + ' ' + s;\n            },\n            past : '%s fa',\n            s : 'alcuni secondi',\n            ss : '%d secondi',\n            m : 'un minuto',\n            mm : '%d minuti',\n            h : 'un\\'ora',\n            hh : '%d ore',\n            d : 'un giorno',\n            dd : '%d giorni',\n            M : 'un mese',\n            MM : '%d mesi',\n            y : 'un anno',\n            yy : '%d anni'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}º/,\n        ordinal: '%dº',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return it;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var ja = moment.defineLocale('ja', {\n        months : '一月_二月_三月_四月_五月_六月_七月_八月_九月_十月_十一月_十二月'.split('_'),\n        monthsShort : '1月_2月_3月_4月_5月_6月_7月_8月_9月_10月_11月_12月'.split('_'),\n        weekdays : '日曜日_月曜日_火曜日_水曜日_木曜日_金曜日_土曜日'.split('_'),\n        weekdaysShort : '日_月_火_水_木_金_土'.split('_'),\n        weekdaysMin : '日_月_火_水_木_金_土'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'YYYY/MM/DD',\n            LL : 'YYYY年M月D日',\n            LLL : 'YYYY年M月D日 HH:mm',\n            LLLL : 'YYYY年M月D日 dddd HH:mm',\n            l : 'YYYY/MM/DD',\n            ll : 'YYYY年M月D日',\n            lll : 'YYYY年M月D日 HH:mm',\n            llll : 'YYYY年M月D日(ddd) HH:mm'\n        },\n        meridiemParse: /午前|午後/i,\n        isPM : function (input) {\n            return input === '午後';\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 12) {\n                return '午前';\n            } else {\n                return '午後';\n            }\n        },\n        calendar : {\n            sameDay : '[今日] LT',\n            nextDay : '[明日] LT',\n            nextWeek : function (now) {\n                if (now.week() < this.week()) {\n                    return '[来週]dddd LT';\n                } else {\n                    return 'dddd LT';\n                }\n            },\n            lastDay : '[昨日] LT',\n            lastWeek : function (now) {\n                if (this.week() < now.week()) {\n                    return '[先週]dddd LT';\n                } else {\n                    return 'dddd LT';\n                }\n            },\n            sameElse : 'L'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}日/,\n        ordinal : function (number, period) {\n            switch (period) {\n                case 'd':\n                case 'D':\n                case 'DDD':\n                    return number + '日';\n                default:\n                    return number;\n            }\n        },\n        relativeTime : {\n            future : '%s後',\n            past : '%s前',\n            s : '数秒',\n            ss : '%d秒',\n            m : '1分',\n            mm : '%d分',\n            h : '1時間',\n            hh : '%d時間',\n            d : '1日',\n            dd : '%d日',\n            M : '1ヶ月',\n            MM : '%dヶ月',\n            y : '1年',\n            yy : '%d年'\n        }\n    });\n\n    return ja;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsShortWithDots = 'jan._feb._mrt._apr._mei_jun._jul._aug._sep._okt._nov._dec.'.split('_'),\n        monthsShortWithoutDots = 'jan_feb_mrt_apr_mei_jun_jul_aug_sep_okt_nov_dec'.split('_');\n\n    var monthsParse = [/^jan/i, /^feb/i, /^maart|mrt.?$/i, /^apr/i, /^mei$/i, /^jun[i.]?$/i, /^jul[i.]?$/i, /^aug/i, /^sep/i, /^okt/i, /^nov/i, /^dec/i];\n    var monthsRegex = /^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december|jan\\.?|feb\\.?|mrt\\.?|apr\\.?|ju[nl]\\.?|aug\\.?|sep\\.?|okt\\.?|nov\\.?|dec\\.?)/i;\n\n    var nl = moment.defineLocale('nl', {\n        months : 'januari_februari_maart_april_mei_juni_juli_augustus_september_oktober_november_december'.split('_'),\n        monthsShort : function (m, format) {\n            if (!m) {\n                return monthsShortWithDots;\n            } else if (/-MMM-/.test(format)) {\n                return monthsShortWithoutDots[m.month()];\n            } else {\n                return monthsShortWithDots[m.month()];\n            }\n        },\n\n        monthsRegex: monthsRegex,\n        monthsShortRegex: monthsRegex,\n        monthsStrictRegex: /^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december)/i,\n        monthsShortStrictRegex: /^(jan\\.?|feb\\.?|mrt\\.?|apr\\.?|mei|ju[nl]\\.?|aug\\.?|sep\\.?|okt\\.?|nov\\.?|dec\\.?)/i,\n\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n\n        weekdays : 'zondag_maandag_dinsdag_woensdag_donderdag_vrijdag_zaterdag'.split('_'),\n        weekdaysShort : 'zo._ma._di._wo._do._vr._za.'.split('_'),\n        weekdaysMin : 'zo_ma_di_wo_do_vr_za'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD-MM-YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[vandaag om] LT',\n            nextDay: '[morgen om] LT',\n            nextWeek: 'dddd [om] LT',\n            lastDay: '[gisteren om] LT',\n            lastWeek: '[afgelopen] dddd [om] LT',\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'over %s',\n            past : '%s geleden',\n            s : 'een paar seconden',\n            ss : '%d seconden',\n            m : 'één minuut',\n            mm : '%d minuten',\n            h : 'één uur',\n            hh : '%d uur',\n            d : 'één dag',\n            dd : '%d dagen',\n            M : 'één maand',\n            MM : '%d maanden',\n            y : 'één jaar',\n            yy : '%d jaar'\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}(ste|de)/,\n        ordinal : function (number) {\n            return number + ((number === 1 || number === 8 || number >= 20) ? 'ste' : 'de');\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return nl;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsNominative = 'styczeń_luty_marzec_kwiecień_maj_czerwiec_lipiec_sierpień_wrzesień_październik_listopad_grudzień'.split('_'),\n        monthsSubjective = 'stycznia_lutego_marca_kwietnia_maja_czerwca_lipca_sierpnia_września_października_listopada_grudnia'.split('_');\n    function plural(n) {\n        return (n % 10 < 5) && (n % 10 > 1) && ((~~(n / 10) % 10) !== 1);\n    }\n    function translate(number, withoutSuffix, key) {\n        var result = number + ' ';\n        switch (key) {\n            case 'ss':\n                return result + (plural(number) ? 'sekundy' : 'sekund');\n            case 'm':\n                return withoutSuffix ? 'minuta' : 'minutę';\n            case 'mm':\n                return result + (plural(number) ? 'minuty' : 'minut');\n            case 'h':\n                return withoutSuffix  ? 'godzina'  : 'godzinę';\n            case 'hh':\n                return result + (plural(number) ? 'godziny' : 'godzin');\n            case 'MM':\n                return result + (plural(number) ? 'miesiące' : 'miesięcy');\n            case 'yy':\n                return result + (plural(number) ? 'lata' : 'lat');\n        }\n    }\n\n    var pl = moment.defineLocale('pl', {\n        months : function (momentToFormat, format) {\n            if (!momentToFormat) {\n                return monthsNominative;\n            } else if (format === '') {\n                // Hack: if format empty we know this is used to generate\n                // RegExp by moment. Give then back both valid forms of months\n                // in RegExp ready format.\n                return '(' + monthsSubjective[momentToFormat.month()] + '|' + monthsNominative[momentToFormat.month()] + ')';\n            } else if (/D MMMM/.test(format)) {\n                return monthsSubjective[momentToFormat.month()];\n            } else {\n                return monthsNominative[momentToFormat.month()];\n            }\n        },\n        monthsShort : 'sty_lut_mar_kwi_maj_cze_lip_sie_wrz_paź_lis_gru'.split('_'),\n        weekdays : 'niedziela_poniedziałek_wtorek_środa_czwartek_piątek_sobota'.split('_'),\n        weekdaysShort : 'ndz_pon_wt_śr_czw_pt_sob'.split('_'),\n        weekdaysMin : 'Nd_Pn_Wt_Śr_Cz_Pt_So'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd, D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[Dziś o] LT',\n            nextDay: '[Jutro o] LT',\n            nextWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[W niedzielę o] LT';\n\n                    case 2:\n                        return '[We wtorek o] LT';\n\n                    case 3:\n                        return '[W środę o] LT';\n\n                    case 6:\n                        return '[W sobotę o] LT';\n\n                    default:\n                        return '[W] dddd [o] LT';\n                }\n            },\n            lastDay: '[Wczoraj o] LT',\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[W zeszłą niedzielę o] LT';\n                    case 3:\n                        return '[W zeszłą środę o] LT';\n                    case 6:\n                        return '[W zeszłą sobotę o] LT';\n                    default:\n                        return '[W zeszły] dddd [o] LT';\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'za %s',\n            past : '%s temu',\n            s : 'kilka sekund',\n            ss : translate,\n            m : translate,\n            mm : translate,\n            h : translate,\n            hh : translate,\n            d : '1 dzień',\n            dd : '%d dni',\n            M : 'miesiąc',\n            MM : translate,\n            y : 'rok',\n            yy : translate\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}\\./,\n        ordinal : '%d.',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return pl;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function plural(word, num) {\n        var forms = word.split('_');\n        return num % 10 === 1 && num % 100 !== 11 ? forms[0] : (num % 10 >= 2 && num % 10 <= 4 && (num % 100 < 10 || num % 100 >= 20) ? forms[1] : forms[2]);\n    }\n    function relativeTimeWithPlural(number, withoutSuffix, key) {\n        var format = {\n            'ss': withoutSuffix ? 'секунда_секунды_секунд' : 'секунду_секунды_секунд',\n            'mm': withoutSuffix ? 'минута_минуты_минут' : 'минуту_минуты_минут',\n            'hh': 'час_часа_часов',\n            'dd': 'день_дня_дней',\n            'MM': 'месяц_месяца_месяцев',\n            'yy': 'год_года_лет'\n        };\n        if (key === 'm') {\n            return withoutSuffix ? 'минута' : 'минуту';\n        }\n        else {\n            return number + ' ' + plural(format[key], +number);\n        }\n    }\n    var monthsParse = [/^янв/i, /^фев/i, /^мар/i, /^апр/i, /^ма[йя]/i, /^июн/i, /^июл/i, /^авг/i, /^сен/i, /^окт/i, /^ноя/i, /^дек/i];\n\n    // http://new.gramota.ru/spravka/rules/139-prop : § 103\n    // Сокращения месяцев: http://new.gramota.ru/spravka/buro/search-answer?s=242637\n    // CLDR data:          http://www.unicode.org/cldr/charts/28/summary/ru.html#1753\n    var ru = moment.defineLocale('ru', {\n        months : {\n            format: 'января_февраля_марта_апреля_мая_июня_июля_августа_сентября_октября_ноября_декабря'.split('_'),\n            standalone: 'январь_февраль_март_апрель_май_июнь_июль_август_сентябрь_октябрь_ноябрь_декабрь'.split('_')\n        },\n        monthsShort : {\n            // по CLDR именно \"июл.\" и \"июн.\", но какой смысл менять букву на точку ?\n            format: 'янв._февр._мар._апр._мая_июня_июля_авг._сент._окт._нояб._дек.'.split('_'),\n            standalone: 'янв._февр._март_апр._май_июнь_июль_авг._сент._окт._нояб._дек.'.split('_')\n        },\n        weekdays : {\n            standalone: 'воскресенье_понедельник_вторник_среда_четверг_пятница_суббота'.split('_'),\n            format: 'воскресенье_понедельник_вторник_среду_четверг_пятницу_субботу'.split('_'),\n            isFormat: /\\[ ?[Вв] ?(?:прошлую|следующую|эту)? ?\\] ?dddd/\n        },\n        weekdaysShort : 'вс_пн_вт_ср_чт_пт_сб'.split('_'),\n        weekdaysMin : 'вс_пн_вт_ср_чт_пт_сб'.split('_'),\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n\n        // полные названия с падежами, по три буквы, для некоторых, по 4 буквы, сокращения с точкой и без точки\n        monthsRegex: /^(январ[ья]|янв\\.?|феврал[ья]|февр?\\.?|марта?|мар\\.?|апрел[ья]|апр\\.?|ма[йя]|июн[ья]|июн\\.?|июл[ья]|июл\\.?|августа?|авг\\.?|сентябр[ья]|сент?\\.?|октябр[ья]|окт\\.?|ноябр[ья]|нояб?\\.?|декабр[ья]|дек\\.?)/i,\n\n        // копия предыдущего\n        monthsShortRegex: /^(январ[ья]|янв\\.?|феврал[ья]|февр?\\.?|марта?|мар\\.?|апрел[ья]|апр\\.?|ма[йя]|июн[ья]|июн\\.?|июл[ья]|июл\\.?|августа?|авг\\.?|сентябр[ья]|сент?\\.?|октябр[ья]|окт\\.?|ноябр[ья]|нояб?\\.?|декабр[ья]|дек\\.?)/i,\n\n        // полные названия с падежами\n        monthsStrictRegex: /^(январ[яь]|феврал[яь]|марта?|апрел[яь]|ма[яй]|июн[яь]|июл[яь]|августа?|сентябр[яь]|октябр[яь]|ноябр[яь]|декабр[яь])/i,\n\n        // Выражение, которое соотвествует только сокращённым формам\n        monthsShortStrictRegex: /^(янв\\.|февр?\\.|мар[т.]|апр\\.|ма[яй]|июн[ья.]|июл[ья.]|авг\\.|сент?\\.|окт\\.|нояб?\\.|дек\\.)/i,\n        longDateFormat : {\n            LT : 'H:mm',\n            LTS : 'H:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY г.',\n            LLL : 'D MMMM YYYY г., H:mm',\n            LLLL : 'dddd, D MMMM YYYY г., H:mm'\n        },\n        calendar : {\n            sameDay: '[Сегодня, в] LT',\n            nextDay: '[Завтра, в] LT',\n            lastDay: '[Вчера, в] LT',\n            nextWeek: function (now) {\n                if (now.week() !== this.week()) {\n                    switch (this.day()) {\n                        case 0:\n                            return '[В следующее] dddd, [в] LT';\n                        case 1:\n                        case 2:\n                        case 4:\n                            return '[В следующий] dddd, [в] LT';\n                        case 3:\n                        case 5:\n                        case 6:\n                            return '[В следующую] dddd, [в] LT';\n                    }\n                } else {\n                    if (this.day() === 2) {\n                        return '[Во] dddd, [в] LT';\n                    } else {\n                        return '[В] dddd, [в] LT';\n                    }\n                }\n            },\n            lastWeek: function (now) {\n                if (now.week() !== this.week()) {\n                    switch (this.day()) {\n                        case 0:\n                            return '[В прошлое] dddd, [в] LT';\n                        case 1:\n                        case 2:\n                        case 4:\n                            return '[В прошлый] dddd, [в] LT';\n                        case 3:\n                        case 5:\n                        case 6:\n                            return '[В прошлую] dddd, [в] LT';\n                    }\n                } else {\n                    if (this.day() === 2) {\n                        return '[Во] dddd, [в] LT';\n                    } else {\n                        return '[В] dddd, [в] LT';\n                    }\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'через %s',\n            past : '%s назад',\n            s : 'несколько секунд',\n            ss : relativeTimeWithPlural,\n            m : relativeTimeWithPlural,\n            mm : relativeTimeWithPlural,\n            h : 'час',\n            hh : relativeTimeWithPlural,\n            d : 'день',\n            dd : relativeTimeWithPlural,\n            M : 'месяц',\n            MM : relativeTimeWithPlural,\n            y : 'год',\n            yy : relativeTimeWithPlural\n        },\n        meridiemParse: /ночи|утра|дня|вечера/i,\n        isPM : function (input) {\n            return /^(дня|вечера)$/.test(input);\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 4) {\n                return 'ночи';\n            } else if (hour < 12) {\n                return 'утра';\n            } else if (hour < 17) {\n                return 'дня';\n            } else {\n                return 'вечера';\n            }\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}-(й|го|я)/,\n        ordinal: function (number, period) {\n            switch (period) {\n                case 'M':\n                case 'd':\n                case 'DDD':\n                    return number + '-й';\n                case 'D':\n                    return number + '-го';\n                case 'w':\n                case 'W':\n                    return number + '-я';\n                default:\n                    return number;\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return ru;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function plural(word, num) {\n        var forms = word.split('_');\n        return num % 10 === 1 && num % 100 !== 11 ? forms[0] : (num % 10 >= 2 && num % 10 <= 4 && (num % 100 < 10 || num % 100 >= 20) ? forms[1] : forms[2]);\n    }\n    function relativeTimeWithPlural(number, withoutSuffix, key) {\n        var format = {\n            'ss': withoutSuffix ? 'секунда_секунди_секунд' : 'секунду_секунди_секунд',\n            'mm': withoutSuffix ? 'хвилина_хвилини_хвилин' : 'хвилину_хвилини_хвилин',\n            'hh': withoutSuffix ? 'година_години_годин' : 'годину_години_годин',\n            'dd': 'день_дні_днів',\n            'MM': 'місяць_місяці_місяців',\n            'yy': 'рік_роки_років'\n        };\n        if (key === 'm') {\n            return withoutSuffix ? 'хвилина' : 'хвилину';\n        }\n        else if (key === 'h') {\n            return withoutSuffix ? 'година' : 'годину';\n        }\n        else {\n            return number + ' ' + plural(format[key], +number);\n        }\n    }\n    function weekdaysCaseReplace(m, format) {\n        var weekdays = {\n            'nominative': 'неділя_понеділок_вівторок_середа_четвер_п’ятниця_субота'.split('_'),\n            'accusative': 'неділю_понеділок_вівторок_середу_четвер_п’ятницю_суботу'.split('_'),\n            'genitive': 'неділі_понеділка_вівторка_середи_четверга_п’ятниці_суботи'.split('_')\n        };\n\n        if (m === true) {\n            return weekdays['nominative'].slice(1, 7).concat(weekdays['nominative'].slice(0, 1));\n        }\n        if (!m) {\n            return weekdays['nominative'];\n        }\n\n        var nounCase = (/(\\[[ВвУу]\\]) ?dddd/).test(format) ?\n            'accusative' :\n            ((/\\[?(?:минулої|наступної)? ?\\] ?dddd/).test(format) ?\n                'genitive' :\n                'nominative');\n        return weekdays[nounCase][m.day()];\n    }\n    function processHoursFunction(str) {\n        return function () {\n            return str + 'о' + (this.hours() === 11 ? 'б' : '') + '] LT';\n        };\n    }\n\n    var uk = moment.defineLocale('uk', {\n        months : {\n            'format': 'січня_лютого_березня_квітня_травня_червня_липня_серпня_вересня_жовтня_листопада_грудня'.split('_'),\n            'standalone': 'січень_лютий_березень_квітень_травень_червень_липень_серпень_вересень_жовтень_листопад_грудень'.split('_')\n        },\n        monthsShort : 'січ_лют_бер_квіт_трав_черв_лип_серп_вер_жовт_лист_груд'.split('_'),\n        weekdays : weekdaysCaseReplace,\n        weekdaysShort : 'нд_пн_вт_ср_чт_пт_сб'.split('_'),\n        weekdaysMin : 'нд_пн_вт_ср_чт_пт_сб'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY р.',\n            LLL : 'D MMMM YYYY р., HH:mm',\n            LLLL : 'dddd, D MMMM YYYY р., HH:mm'\n        },\n        calendar : {\n            sameDay: processHoursFunction('[Сьогодні '),\n            nextDay: processHoursFunction('[Завтра '),\n            lastDay: processHoursFunction('[Вчора '),\n            nextWeek: processHoursFunction('[У] dddd ['),\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                    case 3:\n                    case 5:\n                    case 6:\n                        return processHoursFunction('[Минулої] dddd [').call(this);\n                    case 1:\n                    case 2:\n                    case 4:\n                        return processHoursFunction('[Минулого] dddd [').call(this);\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'за %s',\n            past : '%s тому',\n            s : 'декілька секунд',\n            ss : relativeTimeWithPlural,\n            m : relativeTimeWithPlural,\n            mm : relativeTimeWithPlural,\n            h : 'годину',\n            hh : relativeTimeWithPlural,\n            d : 'день',\n            dd : relativeTimeWithPlural,\n            M : 'місяць',\n            MM : relativeTimeWithPlural,\n            y : 'рік',\n            yy : relativeTimeWithPlural\n        },\n        // M. E.: those two are virtually unused but a user might want to implement them for his/her website for some reason\n        meridiemParse: /ночі|ранку|дня|вечора/,\n        isPM: function (input) {\n            return /^(дня|вечора)$/.test(input);\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 4) {\n                return 'ночі';\n            } else if (hour < 12) {\n                return 'ранку';\n            } else if (hour < 17) {\n                return 'дня';\n            } else {\n                return 'вечора';\n            }\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}-(й|го)/,\n        ordinal: function (number, period) {\n            switch (period) {\n                case 'M':\n                case 'd':\n                case 'DDD':\n                case 'w':\n                case 'W':\n                    return number + '-й';\n                case 'D':\n                    return number + '-го';\n                default:\n                    return number;\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 7  // The week that contains Jan 7th is the first week of the year.\n        }\n    });\n\n    return uk;\n\n})));\n","module.exports = function (exec) {\n  try {\n    return !!exec();\n  } catch (error) {\n    return true;\n  }\n};\n","var fails = require('../internals/fails');\n\n// Thank's IE8 for his funny defineProperty\nmodule.exports = !fails(function () {\n  return Object.defineProperty({}, 1, { get: function () { return 7; } })[1] != 7;\n});\n","'use strict';\nvar nativePropertyIsEnumerable = {}.propertyIsEnumerable;\nvar getOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;\n\n// Nashorn ~ JDK8 bug\nvar NASHORN_BUG = getOwnPropertyDescriptor && !nativePropertyIsEnumerable.call({ 1: 2 }, 1);\n\n// `Object.prototype.propertyIsEnumerable` method implementation\n// https://tc39.github.io/ecma262/#sec-object.prototype.propertyisenumerable\nexports.f = NASHORN_BUG ? function propertyIsEnumerable(V) {\n  var descriptor = getOwnPropertyDescriptor(this, V);\n  return !!descriptor && descriptor.enumerable;\n} : nativePropertyIsEnumerable;\n","module.exports = function (bitmap, value) {\n  return {\n    enumerable: !(bitmap & 1),\n    configurable: !(bitmap & 2),\n    writable: !(bitmap & 4),\n    value: value\n  };\n};\n","var toString = {}.toString;\n\nmodule.exports = function (it) {\n  return toString.call(it).slice(8, -1);\n};\n","var fails = require('../internals/fails');\nvar classof = require('../internals/classof-raw');\n\nvar split = ''.split;\n\n// fallback for non-array-like ES3 and non-enumerable old V8 strings\nmodule.exports = fails(function () {\n  // throws an error in rhino, see https://github.com/mozilla/rhino/issues/346\n  // eslint-disable-next-line no-prototype-builtins\n  return !Object('z').propertyIsEnumerable(0);\n}) ? function (it) {\n  return classof(it) == 'String' ? split.call(it, '') : Object(it);\n} : Object;\n","// `RequireObjectCoercible` abstract operation\n// https://tc39.github.io/ecma262/#sec-requireobjectcoercible\nmodule.exports = function (it) {\n  if (it == undefined) throw TypeError(\"Can't call method on \" + it);\n  return it;\n};\n","// toObject with fallback for non-array-like ES3 strings\nvar IndexedObject = require('../internals/indexed-object');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\nmodule.exports = function (it) {\n  return IndexedObject(requireObjectCoercible(it));\n};\n","module.exports = function (it) {\n  return typeof it === 'object' ? it !== null : typeof it === 'function';\n};\n","var isObject = require('../internals/is-object');\n\n// `ToPrimitive` abstract operation\n// https://tc39.github.io/ecma262/#sec-toprimitive\n// instead of the ES6 spec version, we didn't implement @@toPrimitive case\n// and the second argument - flag - preferred type is a string\nmodule.exports = function (input, PREFERRED_STRING) {\n  if (!isObject(input)) return input;\n  var fn, val;\n  if (PREFERRED_STRING && typeof (fn = input.toString) == 'function' && !isObject(val = fn.call(input))) return val;\n  if (typeof (fn = input.valueOf) == 'function' && !isObject(val = fn.call(input))) return val;\n  if (!PREFERRED_STRING && typeof (fn = input.toString) == 'function' && !isObject(val = fn.call(input))) return val;\n  throw TypeError(\"Can't convert object to primitive value\");\n};\n","var hasOwnProperty = {}.hasOwnProperty;\n\nmodule.exports = function (it, key) {\n  return hasOwnProperty.call(it, key);\n};\n","var global = require('../internals/global');\nvar isObject = require('../internals/is-object');\n\nvar document = global.document;\n// typeof document.createElement is 'object' in old IE\nvar EXISTS = isObject(document) && isObject(document.createElement);\n\nmodule.exports = function (it) {\n  return EXISTS ? document.createElement(it) : {};\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar createElement = require('../internals/document-create-element');\n\n// Thank's IE8 for his funny defineProperty\nmodule.exports = !DESCRIPTORS && !fails(function () {\n  return Object.defineProperty(createElement('div'), 'a', {\n    get: function () { return 7; }\n  }).a != 7;\n});\n","var DESCRIPTORS = require('../internals/descriptors');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar toPrimitive = require('../internals/to-primitive');\nvar has = require('../internals/has');\nvar IE8_DOM_DEFINE = require('../internals/ie8-dom-define');\n\nvar nativeGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;\n\n// `Object.getOwnPropertyDescriptor` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptor\nexports.f = DESCRIPTORS ? nativeGetOwnPropertyDescriptor : function getOwnPropertyDescriptor(O, P) {\n  O = toIndexedObject(O);\n  P = toPrimitive(P, true);\n  if (IE8_DOM_DEFINE) try {\n    return nativeGetOwnPropertyDescriptor(O, P);\n  } catch (error) { /* empty */ }\n  if (has(O, P)) return createPropertyDescriptor(!propertyIsEnumerableModule.f.call(O, P), O[P]);\n};\n","var fails = require('../internals/fails');\n\nvar replacement = /#|\\.prototype\\./;\n\nvar isForced = function (feature, detection) {\n  var value = data[normalize(feature)];\n  return value == POLYFILL ? true\n    : value == NATIVE ? false\n    : typeof detection == 'function' ? fails(detection)\n    : !!detection;\n};\n\nvar normalize = isForced.normalize = function (string) {\n  return String(string).replace(replacement, '.').toLowerCase();\n};\n\nvar data = isForced.data = {};\nvar NATIVE = isForced.NATIVE = 'N';\nvar POLYFILL = isForced.POLYFILL = 'P';\n\nmodule.exports = isForced;\n","module.exports = {};\n","module.exports = function (it) {\n  if (typeof it != 'function') {\n    throw TypeError(String(it) + ' is not a function');\n  } return it;\n};\n","var aFunction = require('../internals/a-function');\n\n// optional / simple context binding\nmodule.exports = function (fn, that, length) {\n  aFunction(fn);\n  if (that === undefined) return fn;\n  switch (length) {\n    case 0: return function () {\n      return fn.call(that);\n    };\n    case 1: return function (a) {\n      return fn.call(that, a);\n    };\n    case 2: return function (a, b) {\n      return fn.call(that, a, b);\n    };\n    case 3: return function (a, b, c) {\n      return fn.call(that, a, b, c);\n    };\n  }\n  return function (/* ...args */) {\n    return fn.apply(that, arguments);\n  };\n};\n","var isObject = require('../internals/is-object');\n\nmodule.exports = function (it) {\n  if (!isObject(it)) {\n    throw TypeError(String(it) + ' is not an object');\n  } return it;\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar IE8_DOM_DEFINE = require('../internals/ie8-dom-define');\nvar anObject = require('../internals/an-object');\nvar toPrimitive = require('../internals/to-primitive');\n\nvar nativeDefineProperty = Object.defineProperty;\n\n// `Object.defineProperty` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperty\nexports.f = DESCRIPTORS ? nativeDefineProperty : function defineProperty(O, P, Attributes) {\n  anObject(O);\n  P = toPrimitive(P, true);\n  anObject(Attributes);\n  if (IE8_DOM_DEFINE) try {\n    return nativeDefineProperty(O, P, Attributes);\n  } catch (error) { /* empty */ }\n  if ('get' in Attributes || 'set' in Attributes) throw TypeError('Accessors not supported');\n  if ('value' in Attributes) O[P] = Attributes.value;\n  return O;\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar definePropertyModule = require('../internals/object-define-property');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\n\nmodule.exports = DESCRIPTORS ? function (object, key, value) {\n  return definePropertyModule.f(object, key, createPropertyDescriptor(1, value));\n} : function (object, key, value) {\n  object[key] = value;\n  return object;\n};\n","'use strict';\nvar global = require('../internals/global');\nvar getOwnPropertyDescriptor = require('../internals/object-get-own-property-descriptor').f;\nvar isForced = require('../internals/is-forced');\nvar path = require('../internals/path');\nvar bind = require('../internals/function-bind-context');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\n\nvar wrapConstructor = function (NativeConstructor) {\n  var Wrapper = function (a, b, c) {\n    if (this instanceof NativeConstructor) {\n      switch (arguments.length) {\n        case 0: return new NativeConstructor();\n        case 1: return new NativeConstructor(a);\n        case 2: return new NativeConstructor(a, b);\n      } return new NativeConstructor(a, b, c);\n    } return NativeConstructor.apply(this, arguments);\n  };\n  Wrapper.prototype = NativeConstructor.prototype;\n  return Wrapper;\n};\n\n/*\n  options.target      - name of the target object\n  options.global      - target is the global object\n  options.stat        - export as static methods of target\n  options.proto       - export as prototype methods of target\n  options.real        - real prototype method for the `pure` version\n  options.forced      - export even if the native feature is available\n  options.bind        - bind methods to the target, required for the `pure` version\n  options.wrap        - wrap constructors to preventing global pollution, required for the `pure` version\n  options.unsafe      - use the simple assignment of property instead of delete + defineProperty\n  options.sham        - add a flag to not completely full polyfills\n  options.enumerable  - export as enumerable property\n  options.noTargetGet - prevent calling a getter on target\n*/\nmodule.exports = function (options, source) {\n  var TARGET = options.target;\n  var GLOBAL = options.global;\n  var STATIC = options.stat;\n  var PROTO = options.proto;\n\n  var nativeSource = GLOBAL ? global : STATIC ? global[TARGET] : (global[TARGET] || {}).prototype;\n\n  var target = GLOBAL ? path : path[TARGET] || (path[TARGET] = {});\n  var targetPrototype = target.prototype;\n\n  var FORCED, USE_NATIVE, VIRTUAL_PROTOTYPE;\n  var key, sourceProperty, targetProperty, nativeProperty, resultProperty, descriptor;\n\n  for (key in source) {\n    FORCED = isForced(GLOBAL ? key : TARGET + (STATIC ? '.' : '#') + key, options.forced);\n    // contains in native\n    USE_NATIVE = !FORCED && nativeSource && has(nativeSource, key);\n\n    targetProperty = target[key];\n\n    if (USE_NATIVE) if (options.noTargetGet) {\n      descriptor = getOwnPropertyDescriptor(nativeSource, key);\n      nativeProperty = descriptor && descriptor.value;\n    } else nativeProperty = nativeSource[key];\n\n    // export native or implementation\n    sourceProperty = (USE_NATIVE && nativeProperty) ? nativeProperty : source[key];\n\n    if (USE_NATIVE && typeof targetProperty === typeof sourceProperty) continue;\n\n    // bind timers to global for call from export context\n    if (options.bind && USE_NATIVE) resultProperty = bind(sourceProperty, global);\n    // wrap global constructors for prevent changs in this version\n    else if (options.wrap && USE_NATIVE) resultProperty = wrapConstructor(sourceProperty);\n    // make static versions for prototype methods\n    else if (PROTO && typeof sourceProperty == 'function') resultProperty = bind(Function.call, sourceProperty);\n    // default case\n    else resultProperty = sourceProperty;\n\n    // add a flag to not completely full polyfills\n    if (options.sham || (sourceProperty && sourceProperty.sham) || (targetProperty && targetProperty.sham)) {\n      createNonEnumerableProperty(resultProperty, 'sham', true);\n    }\n\n    target[key] = resultProperty;\n\n    if (PROTO) {\n      VIRTUAL_PROTOTYPE = TARGET + 'Prototype';\n      if (!has(path, VIRTUAL_PROTOTYPE)) {\n        createNonEnumerableProperty(path, VIRTUAL_PROTOTYPE, {});\n      }\n      // export virtual prototype methods\n      path[VIRTUAL_PROTOTYPE][key] = sourceProperty;\n      // export real prototype methods\n      if (options.real && targetPrototype && !targetPrototype[key]) {\n        createNonEnumerableProperty(targetPrototype, key, sourceProperty);\n      }\n    }\n  }\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar objectDefinePropertyModile = require('../internals/object-define-property');\n\n// `Object.defineProperty` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperty\n$({ target: 'Object', stat: true, forced: !DESCRIPTORS, sham: !DESCRIPTORS }, {\n  defineProperty: objectDefinePropertyModile.f\n});\n","require('../../modules/es.object.define-property');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar defineProperty = module.exports = function defineProperty(it, key, desc) {\n  return Object.defineProperty(it, key, desc);\n};\n\nif (Object.defineProperty.sham) defineProperty.sham = true;\n","var parent = require('../../es/object/define-property');\n\nmodule.exports = parent;\n","var ceil = Math.ceil;\nvar floor = Math.floor;\n\n// `ToInteger` abstract operation\n// https://tc39.github.io/ecma262/#sec-tointeger\nmodule.exports = function (argument) {\n  return isNaN(argument = +argument) ? 0 : (argument > 0 ? floor : ceil)(argument);\n};\n","var toInteger = require('../internals/to-integer');\n\nvar min = Math.min;\n\n// `ToLength` abstract operation\n// https://tc39.github.io/ecma262/#sec-tolength\nmodule.exports = function (argument) {\n  return argument > 0 ? min(toInteger(argument), 0x1FFFFFFFFFFFFF) : 0; // 2 ** 53 - 1 == 9007199254740991\n};\n","var toInteger = require('../internals/to-integer');\n\nvar max = Math.max;\nvar min = Math.min;\n\n// Helper for a popular repeating case of the spec:\n// Let integer be ? ToInteger(index).\n// If integer < 0, let result be max((length + integer), 0); else let result be min(integer, length).\nmodule.exports = function (index, length) {\n  var integer = toInteger(index);\n  return integer < 0 ? max(integer + length, 0) : min(integer, length);\n};\n","var toIndexedObject = require('../internals/to-indexed-object');\nvar toLength = require('../internals/to-length');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\n\n// `Array.prototype.{ indexOf, includes }` methods implementation\nvar createMethod = function (IS_INCLUDES) {\n  return function ($this, el, fromIndex) {\n    var O = toIndexedObject($this);\n    var length = toLength(O.length);\n    var index = toAbsoluteIndex(fromIndex, length);\n    var value;\n    // Array#includes uses SameValueZero equality algorithm\n    // eslint-disable-next-line no-self-compare\n    if (IS_INCLUDES && el != el) while (length > index) {\n      value = O[index++];\n      // eslint-disable-next-line no-self-compare\n      if (value != value) return true;\n    // Array#indexOf ignores holes, Array#includes - not\n    } else for (;length > index; index++) {\n      if ((IS_INCLUDES || index in O) && O[index] === el) return IS_INCLUDES || index || 0;\n    } return !IS_INCLUDES && -1;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.includes` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.includes\n  includes: createMethod(true),\n  // `Array.prototype.indexOf` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.indexof\n  indexOf: createMethod(false)\n};\n","module.exports = {};\n","var has = require('../internals/has');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar indexOf = require('../internals/array-includes').indexOf;\nvar hiddenKeys = require('../internals/hidden-keys');\n\nmodule.exports = function (object, names) {\n  var O = toIndexedObject(object);\n  var i = 0;\n  var result = [];\n  var key;\n  for (key in O) !has(hiddenKeys, key) && has(O, key) && result.push(key);\n  // Don't enum bug & hidden keys\n  while (names.length > i) if (has(O, key = names[i++])) {\n    ~indexOf(result, key) || result.push(key);\n  }\n  return result;\n};\n","// IE8- don't enum bug keys\nmodule.exports = [\n  'constructor',\n  'hasOwnProperty',\n  'isPrototypeOf',\n  'propertyIsEnumerable',\n  'toLocaleString',\n  'toString',\n  'valueOf'\n];\n","var internalObjectKeys = require('../internals/object-keys-internal');\nvar enumBugKeys = require('../internals/enum-bug-keys');\n\n// `Object.keys` method\n// https://tc39.github.io/ecma262/#sec-object.keys\nmodule.exports = Object.keys || function keys(O) {\n  return internalObjectKeys(O, enumBugKeys);\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar definePropertyModule = require('../internals/object-define-property');\nvar anObject = require('../internals/an-object');\nvar objectKeys = require('../internals/object-keys');\n\n// `Object.defineProperties` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperties\nmodule.exports = DESCRIPTORS ? Object.defineProperties : function defineProperties(O, Properties) {\n  anObject(O);\n  var keys = objectKeys(Properties);\n  var length = keys.length;\n  var index = 0;\n  var key;\n  while (length > index) definePropertyModule.f(O, key = keys[index++], Properties[key]);\n  return O;\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar defineProperties = require('../internals/object-define-properties');\n\n// `Object.defineProperties` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperties\n$({ target: 'Object', stat: true, forced: !DESCRIPTORS, sham: !DESCRIPTORS }, {\n  defineProperties: defineProperties\n});\n","require('../../modules/es.object.define-properties');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar defineProperties = module.exports = function defineProperties(T, D) {\n  return Object.defineProperties(T, D);\n};\n\nif (Object.defineProperties.sham) defineProperties.sham = true;\n","var path = require('../internals/path');\nvar global = require('../internals/global');\n\nvar aFunction = function (variable) {\n  return typeof variable == 'function' ? variable : undefined;\n};\n\nmodule.exports = function (namespace, method) {\n  return arguments.length < 2 ? aFunction(path[namespace]) || aFunction(global[namespace])\n    : path[namespace] && path[namespace][method] || global[namespace] && global[namespace][method];\n};\n","var internalObjectKeys = require('../internals/object-keys-internal');\nvar enumBugKeys = require('../internals/enum-bug-keys');\n\nvar hiddenKeys = enumBugKeys.concat('length', 'prototype');\n\n// `Object.getOwnPropertyNames` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertynames\nexports.f = Object.getOwnPropertyNames || function getOwnPropertyNames(O) {\n  return internalObjectKeys(O, hiddenKeys);\n};\n","exports.f = Object.getOwnPropertySymbols;\n","var getBuiltIn = require('../internals/get-built-in');\nvar getOwnPropertyNamesModule = require('../internals/object-get-own-property-names');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar anObject = require('../internals/an-object');\n\n// all object keys, includes non-enumerable and symbols\nmodule.exports = getBuiltIn('Reflect', 'ownKeys') || function ownKeys(it) {\n  var keys = getOwnPropertyNamesModule.f(anObject(it));\n  var getOwnPropertySymbols = getOwnPropertySymbolsModule.f;\n  return getOwnPropertySymbols ? keys.concat(getOwnPropertySymbols(it)) : keys;\n};\n","'use strict';\nvar toPrimitive = require('../internals/to-primitive');\nvar definePropertyModule = require('../internals/object-define-property');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\n\nmodule.exports = function (object, key, value) {\n  var propertyKey = toPrimitive(key);\n  if (propertyKey in object) definePropertyModule.f(object, propertyKey, createPropertyDescriptor(0, value));\n  else object[propertyKey] = value;\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar ownKeys = require('../internals/own-keys');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar getOwnPropertyDescriptorModule = require('../internals/object-get-own-property-descriptor');\nvar createProperty = require('../internals/create-property');\n\n// `Object.getOwnPropertyDescriptors` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptors\n$({ target: 'Object', stat: true, sham: !DESCRIPTORS }, {\n  getOwnPropertyDescriptors: function getOwnPropertyDescriptors(object) {\n    var O = toIndexedObject(object);\n    var getOwnPropertyDescriptor = getOwnPropertyDescriptorModule.f;\n    var keys = ownKeys(O);\n    var result = {};\n    var index = 0;\n    var key, descriptor;\n    while (keys.length > index) {\n      descriptor = getOwnPropertyDescriptor(O, key = keys[index++]);\n      if (descriptor !== undefined) createProperty(result, key, descriptor);\n    }\n    return result;\n  }\n});\n","require('../../modules/es.object.get-own-property-descriptors');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getOwnPropertyDescriptors;\n","var $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar nativeGetOwnPropertyDescriptor = require('../internals/object-get-own-property-descriptor').f;\nvar DESCRIPTORS = require('../internals/descriptors');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeGetOwnPropertyDescriptor(1); });\nvar FORCED = !DESCRIPTORS || FAILS_ON_PRIMITIVES;\n\n// `Object.getOwnPropertyDescriptor` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptor\n$({ target: 'Object', stat: true, forced: FORCED, sham: !DESCRIPTORS }, {\n  getOwnPropertyDescriptor: function getOwnPropertyDescriptor(it, key) {\n    return nativeGetOwnPropertyDescriptor(toIndexedObject(it), key);\n  }\n});\n","var anObject = require('../internals/an-object');\nvar defineProperties = require('../internals/object-define-properties');\nvar enumBugKeys = require('../internals/enum-bug-keys');\nvar hiddenKeys = require('../internals/hidden-keys');\nvar html = require('../internals/html');\nvar documentCreateElement = require('../internals/document-create-element');\nvar sharedKey = require('../internals/shared-key');\n\nvar GT = '>';\nvar LT = '<';\nvar PROTOTYPE = 'prototype';\nvar SCRIPT = 'script';\nvar IE_PROTO = sharedKey('IE_PROTO');\n\nvar EmptyConstructor = function () { /* empty */ };\n\nvar scriptTag = function (content) {\n  return LT + SCRIPT + GT + content + LT + '/' + SCRIPT + GT;\n};\n\n// Create object with fake `null` prototype: use ActiveX Object with cleared prototype\nvar NullProtoObjectViaActiveX = function (activeXDocument) {\n  activeXDocument.write(scriptTag(''));\n  activeXDocument.close();\n  var temp = activeXDocument.parentWindow.Object;\n  activeXDocument = null; // avoid memory leak\n  return temp;\n};\n\n// Create object with fake `null` prototype: use iframe Object with cleared prototype\nvar NullProtoObjectViaIFrame = function () {\n  // Thrash, waste and sodomy: IE GC bug\n  var iframe = documentCreateElement('iframe');\n  var JS = 'java' + SCRIPT + ':';\n  var iframeDocument;\n  iframe.style.display = 'none';\n  html.appendChild(iframe);\n  // https://github.com/zloirock/core-js/issues/475\n  iframe.src = String(JS);\n  iframeDocument = iframe.contentWindow.document;\n  iframeDocument.open();\n  iframeDocument.write(scriptTag('document.F=Object'));\n  iframeDocument.close();\n  return iframeDocument.F;\n};\n\n// Check for document.domain and active x support\n// No need to use active x approach when document.domain is not set\n// see https://github.com/es-shims/es5-shim/issues/150\n// variation of https://github.com/kitcambridge/es5-shim/commit/4f738ac066346\n// avoid IE GC bug\nvar activeXDocument;\nvar NullProtoObject = function () {\n  try {\n    /* global ActiveXObject */\n    activeXDocument = document.domain && new ActiveXObject('htmlfile');\n  } catch (error) { /* ignore */ }\n  NullProtoObject = activeXDocument ? NullProtoObjectViaActiveX(activeXDocument) : NullProtoObjectViaIFrame();\n  var length = enumBugKeys.length;\n  while (length--) delete NullProtoObject[PROTOTYPE][enumBugKeys[length]];\n  return NullProtoObject();\n};\n\nhiddenKeys[IE_PROTO] = true;\n\n// `Object.create` method\n// https://tc39.github.io/ecma262/#sec-object.create\nmodule.exports = Object.create || function create(O, Properties) {\n  var result;\n  if (O !== null) {\n    EmptyConstructor[PROTOTYPE] = anObject(O);\n    result = new EmptyConstructor();\n    EmptyConstructor[PROTOTYPE] = null;\n    // add \"__proto__\" for Object.getPrototypeOf polyfill\n    result[IE_PROTO] = O;\n  } else result = NullProtoObject();\n  return Properties === undefined ? result : defineProperties(result, Properties);\n};\n","require('../../modules/es.object.get-own-property-descriptor');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar getOwnPropertyDescriptor = module.exports = function getOwnPropertyDescriptor(it, key) {\n  return Object.getOwnPropertyDescriptor(it, key);\n};\n\nif (Object.getOwnPropertyDescriptor.sham) getOwnPropertyDescriptor.sham = true;\n","var fails = require('../internals/fails');\n\nmodule.exports = !!Object.getOwnPropertySymbols && !fails(function () {\n  // Chrome 38 Symbol has incorrect toString conversion\n  // eslint-disable-next-line no-undef\n  return !String(Symbol());\n});\n","var NATIVE_SYMBOL = require('../internals/native-symbol');\n\nmodule.exports = NATIVE_SYMBOL\n  // eslint-disable-next-line no-undef\n  && !Symbol.sham\n  // eslint-disable-next-line no-undef\n  && typeof Symbol.iterator == 'symbol';\n","var classof = require('../internals/classof-raw');\n\n// `IsArray` abstract operation\n// https://tc39.github.io/ecma262/#sec-isarray\nmodule.exports = Array.isArray || function isArray(arg) {\n  return classof(arg) == 'Array';\n};\n","var requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `ToObject` abstract operation\n// https://tc39.github.io/ecma262/#sec-toobject\nmodule.exports = function (argument) {\n  return Object(requireObjectCoercible(argument));\n};\n","var getBuiltIn = require('../internals/get-built-in');\n\nmodule.exports = getBuiltIn('document', 'documentElement');\n","var global = require('../internals/global');\nvar setGlobal = require('../internals/set-global');\n\nvar SHARED = '__core-js_shared__';\nvar store = global[SHARED] || setGlobal(SHARED, {});\n\nmodule.exports = store;\n","var global = require('../internals/global');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\n\nmodule.exports = function (key, value) {\n  try {\n    createNonEnumerableProperty(global, key, value);\n  } catch (error) {\n    global[key] = value;\n  } return value;\n};\n","var IS_PURE = require('../internals/is-pure');\nvar store = require('../internals/shared-store');\n\n(module.exports = function (key, value) {\n  return store[key] || (store[key] = value !== undefined ? value : {});\n})('versions', []).push({\n  version: '3.6.4',\n  mode: IS_PURE ? 'pure' : 'global',\n  copyright: '© 2020 Denis Pushkarev (zloirock.ru)'\n});\n","var id = 0;\nvar postfix = Math.random();\n\nmodule.exports = function (key) {\n  return 'Symbol(' + String(key === undefined ? '' : key) + ')_' + (++id + postfix).toString(36);\n};\n","var shared = require('../internals/shared');\nvar uid = require('../internals/uid');\n\nvar keys = shared('keys');\n\nmodule.exports = function (key) {\n  return keys[key] || (keys[key] = uid(key));\n};\n","var toIndexedObject = require('../internals/to-indexed-object');\nvar nativeGetOwnPropertyNames = require('../internals/object-get-own-property-names').f;\n\nvar toString = {}.toString;\n\nvar windowNames = typeof window == 'object' && window && Object.getOwnPropertyNames\n  ? Object.getOwnPropertyNames(window) : [];\n\nvar getWindowNames = function (it) {\n  try {\n    return nativeGetOwnPropertyNames(it);\n  } catch (error) {\n    return windowNames.slice();\n  }\n};\n\n// fallback for IE11 buggy Object.getOwnPropertyNames with iframe and window\nmodule.exports.f = function getOwnPropertyNames(it) {\n  return windowNames && toString.call(it) == '[object Window]'\n    ? getWindowNames(it)\n    : nativeGetOwnPropertyNames(toIndexedObject(it));\n};\n","var createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\n\nmodule.exports = function (target, key, value, options) {\n  if (options && options.enumerable) target[key] = value;\n  else createNonEnumerableProperty(target, key, value);\n};\n","var global = require('../internals/global');\nvar shared = require('../internals/shared');\nvar has = require('../internals/has');\nvar uid = require('../internals/uid');\nvar NATIVE_SYMBOL = require('../internals/native-symbol');\nvar USE_SYMBOL_AS_UID = require('../internals/use-symbol-as-uid');\n\nvar WellKnownSymbolsStore = shared('wks');\nvar Symbol = global.Symbol;\nvar createWellKnownSymbol = USE_SYMBOL_AS_UID ? Symbol : Symbol && Symbol.withoutSetter || uid;\n\nmodule.exports = function (name) {\n  if (!has(WellKnownSymbolsStore, name)) {\n    if (NATIVE_SYMBOL && has(Symbol, name)) WellKnownSymbolsStore[name] = Symbol[name];\n    else WellKnownSymbolsStore[name] = createWellKnownSymbol('Symbol.' + name);\n  } return WellKnownSymbolsStore[name];\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nexports.f = wellKnownSymbol;\n","var path = require('../internals/path');\nvar has = require('../internals/has');\nvar wrappedWellKnownSymbolModule = require('../internals/well-known-symbol-wrapped');\nvar defineProperty = require('../internals/object-define-property').f;\n\nmodule.exports = function (NAME) {\n  var Symbol = path.Symbol || (path.Symbol = {});\n  if (!has(Symbol, NAME)) defineProperty(Symbol, NAME, {\n    value: wrappedWellKnownSymbolModule.f(NAME)\n  });\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\nvar test = {};\n\ntest[TO_STRING_TAG] = 'z';\n\nmodule.exports = String(test) === '[object z]';\n","var TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar classofRaw = require('../internals/classof-raw');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n// ES3 wrong here\nvar CORRECT_ARGUMENTS = classofRaw(function () { return arguments; }()) == 'Arguments';\n\n// fallback for IE11 Script Access Denied error\nvar tryGet = function (it, key) {\n  try {\n    return it[key];\n  } catch (error) { /* empty */ }\n};\n\n// getting tag from ES6+ `Object.prototype.toString`\nmodule.exports = TO_STRING_TAG_SUPPORT ? classofRaw : function (it) {\n  var O, tag, result;\n  return it === undefined ? 'Undefined' : it === null ? 'Null'\n    // @@toStringTag case\n    : typeof (tag = tryGet(O = Object(it), TO_STRING_TAG)) == 'string' ? tag\n    // builtinTag case\n    : CORRECT_ARGUMENTS ? classofRaw(O)\n    // ES3 arguments fallback\n    : (result = classofRaw(O)) == 'Object' && typeof O.callee == 'function' ? 'Arguments' : result;\n};\n","'use strict';\nvar TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar classof = require('../internals/classof');\n\n// `Object.prototype.toString` method implementation\n// https://tc39.github.io/ecma262/#sec-object.prototype.tostring\nmodule.exports = TO_STRING_TAG_SUPPORT ? {}.toString : function toString() {\n  return '[object ' + classof(this) + ']';\n};\n","var TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar defineProperty = require('../internals/object-define-property').f;\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\nvar toString = require('../internals/object-to-string');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n\nmodule.exports = function (it, TAG, STATIC, SET_METHOD) {\n  if (it) {\n    var target = STATIC ? it : it.prototype;\n    if (!has(target, TO_STRING_TAG)) {\n      defineProperty(target, TO_STRING_TAG, { configurable: true, value: TAG });\n    }\n    if (SET_METHOD && !TO_STRING_TAG_SUPPORT) {\n      createNonEnumerableProperty(target, 'toString', toString);\n    }\n  }\n};\n","var store = require('../internals/shared-store');\n\nvar functionToString = Function.toString;\n\n// this helper broken in `3.4.1-3.4.4`, so we can't use `shared` helper\nif (typeof store.inspectSource != 'function') {\n  store.inspectSource = function (it) {\n    return functionToString.call(it);\n  };\n}\n\nmodule.exports = store.inspectSource;\n","var NATIVE_WEAK_MAP = require('../internals/native-weak-map');\nvar global = require('../internals/global');\nvar isObject = require('../internals/is-object');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar objectHas = require('../internals/has');\nvar sharedKey = require('../internals/shared-key');\nvar hiddenKeys = require('../internals/hidden-keys');\n\nvar WeakMap = global.WeakMap;\nvar set, get, has;\n\nvar enforce = function (it) {\n  return has(it) ? get(it) : set(it, {});\n};\n\nvar getterFor = function (TYPE) {\n  return function (it) {\n    var state;\n    if (!isObject(it) || (state = get(it)).type !== TYPE) {\n      throw TypeError('Incompatible receiver, ' + TYPE + ' required');\n    } return state;\n  };\n};\n\nif (NATIVE_WEAK_MAP) {\n  var store = new WeakMap();\n  var wmget = store.get;\n  var wmhas = store.has;\n  var wmset = store.set;\n  set = function (it, metadata) {\n    wmset.call(store, it, metadata);\n    return metadata;\n  };\n  get = function (it) {\n    return wmget.call(store, it) || {};\n  };\n  has = function (it) {\n    return wmhas.call(store, it);\n  };\n} else {\n  var STATE = sharedKey('state');\n  hiddenKeys[STATE] = true;\n  set = function (it, metadata) {\n    createNonEnumerableProperty(it, STATE, metadata);\n    return metadata;\n  };\n  get = function (it) {\n    return objectHas(it, STATE) ? it[STATE] : {};\n  };\n  has = function (it) {\n    return objectHas(it, STATE);\n  };\n}\n\nmodule.exports = {\n  set: set,\n  get: get,\n  has: has,\n  enforce: enforce,\n  getterFor: getterFor\n};\n","var global = require('../internals/global');\nvar inspectSource = require('../internals/inspect-source');\n\nvar WeakMap = global.WeakMap;\n\nmodule.exports = typeof WeakMap === 'function' && /native code/.test(inspectSource(WeakMap));\n","var isObject = require('../internals/is-object');\nvar isArray = require('../internals/is-array');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar SPECIES = wellKnownSymbol('species');\n\n// `ArraySpeciesCreate` abstract operation\n// https://tc39.github.io/ecma262/#sec-arrayspeciescreate\nmodule.exports = function (originalArray, length) {\n  var C;\n  if (isArray(originalArray)) {\n    C = originalArray.constructor;\n    // cross-realm fallback\n    if (typeof C == 'function' && (C === Array || isArray(C.prototype))) C = undefined;\n    else if (isObject(C)) {\n      C = C[SPECIES];\n      if (C === null) C = undefined;\n    }\n  } return new (C === undefined ? Array : C)(length === 0 ? 0 : length);\n};\n","var bind = require('../internals/function-bind-context');\nvar IndexedObject = require('../internals/indexed-object');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar arraySpeciesCreate = require('../internals/array-species-create');\n\nvar push = [].push;\n\n// `Array.prototype.{ forEach, map, filter, some, every, find, findIndex }` methods implementation\nvar createMethod = function (TYPE) {\n  var IS_MAP = TYPE == 1;\n  var IS_FILTER = TYPE == 2;\n  var IS_SOME = TYPE == 3;\n  var IS_EVERY = TYPE == 4;\n  var IS_FIND_INDEX = TYPE == 6;\n  var NO_HOLES = TYPE == 5 || IS_FIND_INDEX;\n  return function ($this, callbackfn, that, specificCreate) {\n    var O = toObject($this);\n    var self = IndexedObject(O);\n    var boundFunction = bind(callbackfn, that, 3);\n    var length = toLength(self.length);\n    var index = 0;\n    var create = specificCreate || arraySpeciesCreate;\n    var target = IS_MAP ? create($this, length) : IS_FILTER ? create($this, 0) : undefined;\n    var value, result;\n    for (;length > index; index++) if (NO_HOLES || index in self) {\n      value = self[index];\n      result = boundFunction(value, index, O);\n      if (TYPE) {\n        if (IS_MAP) target[index] = result; // map\n        else if (result) switch (TYPE) {\n          case 3: return true;              // some\n          case 5: return value;             // find\n          case 6: return index;             // findIndex\n          case 2: push.call(target, value); // filter\n        } else if (IS_EVERY) return false;  // every\n      }\n    }\n    return IS_FIND_INDEX ? -1 : IS_SOME || IS_EVERY ? IS_EVERY : target;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.forEach` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.foreach\n  forEach: createMethod(0),\n  // `Array.prototype.map` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.map\n  map: createMethod(1),\n  // `Array.prototype.filter` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.filter\n  filter: createMethod(2),\n  // `Array.prototype.some` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.some\n  some: createMethod(3),\n  // `Array.prototype.every` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.every\n  every: createMethod(4),\n  // `Array.prototype.find` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.find\n  find: createMethod(5),\n  // `Array.prototype.findIndex` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.findIndex\n  findIndex: createMethod(6)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar global = require('../internals/global');\nvar getBuiltIn = require('../internals/get-built-in');\nvar IS_PURE = require('../internals/is-pure');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar NATIVE_SYMBOL = require('../internals/native-symbol');\nvar USE_SYMBOL_AS_UID = require('../internals/use-symbol-as-uid');\nvar fails = require('../internals/fails');\nvar has = require('../internals/has');\nvar isArray = require('../internals/is-array');\nvar isObject = require('../internals/is-object');\nvar anObject = require('../internals/an-object');\nvar toObject = require('../internals/to-object');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar toPrimitive = require('../internals/to-primitive');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar nativeObjectCreate = require('../internals/object-create');\nvar objectKeys = require('../internals/object-keys');\nvar getOwnPropertyNamesModule = require('../internals/object-get-own-property-names');\nvar getOwnPropertyNamesExternal = require('../internals/object-get-own-property-names-external');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar getOwnPropertyDescriptorModule = require('../internals/object-get-own-property-descriptor');\nvar definePropertyModule = require('../internals/object-define-property');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar redefine = require('../internals/redefine');\nvar shared = require('../internals/shared');\nvar sharedKey = require('../internals/shared-key');\nvar hiddenKeys = require('../internals/hidden-keys');\nvar uid = require('../internals/uid');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar wrappedWellKnownSymbolModule = require('../internals/well-known-symbol-wrapped');\nvar defineWellKnownSymbol = require('../internals/define-well-known-symbol');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar InternalStateModule = require('../internals/internal-state');\nvar $forEach = require('../internals/array-iteration').forEach;\n\nvar HIDDEN = sharedKey('hidden');\nvar SYMBOL = 'Symbol';\nvar PROTOTYPE = 'prototype';\nvar TO_PRIMITIVE = wellKnownSymbol('toPrimitive');\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(SYMBOL);\nvar ObjectPrototype = Object[PROTOTYPE];\nvar $Symbol = global.Symbol;\nvar $stringify = getBuiltIn('JSON', 'stringify');\nvar nativeGetOwnPropertyDescriptor = getOwnPropertyDescriptorModule.f;\nvar nativeDefineProperty = definePropertyModule.f;\nvar nativeGetOwnPropertyNames = getOwnPropertyNamesExternal.f;\nvar nativePropertyIsEnumerable = propertyIsEnumerableModule.f;\nvar AllSymbols = shared('symbols');\nvar ObjectPrototypeSymbols = shared('op-symbols');\nvar StringToSymbolRegistry = shared('string-to-symbol-registry');\nvar SymbolToStringRegistry = shared('symbol-to-string-registry');\nvar WellKnownSymbolsStore = shared('wks');\nvar QObject = global.QObject;\n// Don't use setters in Qt Script, https://github.com/zloirock/core-js/issues/173\nvar USE_SETTER = !QObject || !QObject[PROTOTYPE] || !QObject[PROTOTYPE].findChild;\n\n// fallback for old Android, https://code.google.com/p/v8/issues/detail?id=687\nvar setSymbolDescriptor = DESCRIPTORS && fails(function () {\n  return nativeObjectCreate(nativeDefineProperty({}, 'a', {\n    get: function () { return nativeDefineProperty(this, 'a', { value: 7 }).a; }\n  })).a != 7;\n}) ? function (O, P, Attributes) {\n  var ObjectPrototypeDescriptor = nativeGetOwnPropertyDescriptor(ObjectPrototype, P);\n  if (ObjectPrototypeDescriptor) delete ObjectPrototype[P];\n  nativeDefineProperty(O, P, Attributes);\n  if (ObjectPrototypeDescriptor && O !== ObjectPrototype) {\n    nativeDefineProperty(ObjectPrototype, P, ObjectPrototypeDescriptor);\n  }\n} : nativeDefineProperty;\n\nvar wrap = function (tag, description) {\n  var symbol = AllSymbols[tag] = nativeObjectCreate($Symbol[PROTOTYPE]);\n  setInternalState(symbol, {\n    type: SYMBOL,\n    tag: tag,\n    description: description\n  });\n  if (!DESCRIPTORS) symbol.description = description;\n  return symbol;\n};\n\nvar isSymbol = USE_SYMBOL_AS_UID ? function (it) {\n  return typeof it == 'symbol';\n} : function (it) {\n  return Object(it) instanceof $Symbol;\n};\n\nvar $defineProperty = function defineProperty(O, P, Attributes) {\n  if (O === ObjectPrototype) $defineProperty(ObjectPrototypeSymbols, P, Attributes);\n  anObject(O);\n  var key = toPrimitive(P, true);\n  anObject(Attributes);\n  if (has(AllSymbols, key)) {\n    if (!Attributes.enumerable) {\n      if (!has(O, HIDDEN)) nativeDefineProperty(O, HIDDEN, createPropertyDescriptor(1, {}));\n      O[HIDDEN][key] = true;\n    } else {\n      if (has(O, HIDDEN) && O[HIDDEN][key]) O[HIDDEN][key] = false;\n      Attributes = nativeObjectCreate(Attributes, { enumerable: createPropertyDescriptor(0, false) });\n    } return setSymbolDescriptor(O, key, Attributes);\n  } return nativeDefineProperty(O, key, Attributes);\n};\n\nvar $defineProperties = function defineProperties(O, Properties) {\n  anObject(O);\n  var properties = toIndexedObject(Properties);\n  var keys = objectKeys(properties).concat($getOwnPropertySymbols(properties));\n  $forEach(keys, function (key) {\n    if (!DESCRIPTORS || $propertyIsEnumerable.call(properties, key)) $defineProperty(O, key, properties[key]);\n  });\n  return O;\n};\n\nvar $create = function create(O, Properties) {\n  return Properties === undefined ? nativeObjectCreate(O) : $defineProperties(nativeObjectCreate(O), Properties);\n};\n\nvar $propertyIsEnumerable = function propertyIsEnumerable(V) {\n  var P = toPrimitive(V, true);\n  var enumerable = nativePropertyIsEnumerable.call(this, P);\n  if (this === ObjectPrototype && has(AllSymbols, P) && !has(ObjectPrototypeSymbols, P)) return false;\n  return enumerable || !has(this, P) || !has(AllSymbols, P) || has(this, HIDDEN) && this[HIDDEN][P] ? enumerable : true;\n};\n\nvar $getOwnPropertyDescriptor = function getOwnPropertyDescriptor(O, P) {\n  var it = toIndexedObject(O);\n  var key = toPrimitive(P, true);\n  if (it === ObjectPrototype && has(AllSymbols, key) && !has(ObjectPrototypeSymbols, key)) return;\n  var descriptor = nativeGetOwnPropertyDescriptor(it, key);\n  if (descriptor && has(AllSymbols, key) && !(has(it, HIDDEN) && it[HIDDEN][key])) {\n    descriptor.enumerable = true;\n  }\n  return descriptor;\n};\n\nvar $getOwnPropertyNames = function getOwnPropertyNames(O) {\n  var names = nativeGetOwnPropertyNames(toIndexedObject(O));\n  var result = [];\n  $forEach(names, function (key) {\n    if (!has(AllSymbols, key) && !has(hiddenKeys, key)) result.push(key);\n  });\n  return result;\n};\n\nvar $getOwnPropertySymbols = function getOwnPropertySymbols(O) {\n  var IS_OBJECT_PROTOTYPE = O === ObjectPrototype;\n  var names = nativeGetOwnPropertyNames(IS_OBJECT_PROTOTYPE ? ObjectPrototypeSymbols : toIndexedObject(O));\n  var result = [];\n  $forEach(names, function (key) {\n    if (has(AllSymbols, key) && (!IS_OBJECT_PROTOTYPE || has(ObjectPrototype, key))) {\n      result.push(AllSymbols[key]);\n    }\n  });\n  return result;\n};\n\n// `Symbol` constructor\n// https://tc39.github.io/ecma262/#sec-symbol-constructor\nif (!NATIVE_SYMBOL) {\n  $Symbol = function Symbol() {\n    if (this instanceof $Symbol) throw TypeError('Symbol is not a constructor');\n    var description = !arguments.length || arguments[0] === undefined ? undefined : String(arguments[0]);\n    var tag = uid(description);\n    var setter = function (value) {\n      if (this === ObjectPrototype) setter.call(ObjectPrototypeSymbols, value);\n      if (has(this, HIDDEN) && has(this[HIDDEN], tag)) this[HIDDEN][tag] = false;\n      setSymbolDescriptor(this, tag, createPropertyDescriptor(1, value));\n    };\n    if (DESCRIPTORS && USE_SETTER) setSymbolDescriptor(ObjectPrototype, tag, { configurable: true, set: setter });\n    return wrap(tag, description);\n  };\n\n  redefine($Symbol[PROTOTYPE], 'toString', function toString() {\n    return getInternalState(this).tag;\n  });\n\n  redefine($Symbol, 'withoutSetter', function (description) {\n    return wrap(uid(description), description);\n  });\n\n  propertyIsEnumerableModule.f = $propertyIsEnumerable;\n  definePropertyModule.f = $defineProperty;\n  getOwnPropertyDescriptorModule.f = $getOwnPropertyDescriptor;\n  getOwnPropertyNamesModule.f = getOwnPropertyNamesExternal.f = $getOwnPropertyNames;\n  getOwnPropertySymbolsModule.f = $getOwnPropertySymbols;\n\n  wrappedWellKnownSymbolModule.f = function (name) {\n    return wrap(wellKnownSymbol(name), name);\n  };\n\n  if (DESCRIPTORS) {\n    // https://github.com/tc39/proposal-Symbol-description\n    nativeDefineProperty($Symbol[PROTOTYPE], 'description', {\n      configurable: true,\n      get: function description() {\n        return getInternalState(this).description;\n      }\n    });\n    if (!IS_PURE) {\n      redefine(ObjectPrototype, 'propertyIsEnumerable', $propertyIsEnumerable, { unsafe: true });\n    }\n  }\n}\n\n$({ global: true, wrap: true, forced: !NATIVE_SYMBOL, sham: !NATIVE_SYMBOL }, {\n  Symbol: $Symbol\n});\n\n$forEach(objectKeys(WellKnownSymbolsStore), function (name) {\n  defineWellKnownSymbol(name);\n});\n\n$({ target: SYMBOL, stat: true, forced: !NATIVE_SYMBOL }, {\n  // `Symbol.for` method\n  // https://tc39.github.io/ecma262/#sec-symbol.for\n  'for': function (key) {\n    var string = String(key);\n    if (has(StringToSymbolRegistry, string)) return StringToSymbolRegistry[string];\n    var symbol = $Symbol(string);\n    StringToSymbolRegistry[string] = symbol;\n    SymbolToStringRegistry[symbol] = string;\n    return symbol;\n  },\n  // `Symbol.keyFor` method\n  // https://tc39.github.io/ecma262/#sec-symbol.keyfor\n  keyFor: function keyFor(sym) {\n    if (!isSymbol(sym)) throw TypeError(sym + ' is not a symbol');\n    if (has(SymbolToStringRegistry, sym)) return SymbolToStringRegistry[sym];\n  },\n  useSetter: function () { USE_SETTER = true; },\n  useSimple: function () { USE_SETTER = false; }\n});\n\n$({ target: 'Object', stat: true, forced: !NATIVE_SYMBOL, sham: !DESCRIPTORS }, {\n  // `Object.create` method\n  // https://tc39.github.io/ecma262/#sec-object.create\n  create: $create,\n  // `Object.defineProperty` method\n  // https://tc39.github.io/ecma262/#sec-object.defineproperty\n  defineProperty: $defineProperty,\n  // `Object.defineProperties` method\n  // https://tc39.github.io/ecma262/#sec-object.defineproperties\n  defineProperties: $defineProperties,\n  // `Object.getOwnPropertyDescriptor` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptors\n  getOwnPropertyDescriptor: $getOwnPropertyDescriptor\n});\n\n$({ target: 'Object', stat: true, forced: !NATIVE_SYMBOL }, {\n  // `Object.getOwnPropertyNames` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertynames\n  getOwnPropertyNames: $getOwnPropertyNames,\n  // `Object.getOwnPropertySymbols` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertysymbols\n  getOwnPropertySymbols: $getOwnPropertySymbols\n});\n\n// Chrome 38 and 39 `Object.getOwnPropertySymbols` fails on primitives\n// https://bugs.chromium.org/p/v8/issues/detail?id=3443\n$({ target: 'Object', stat: true, forced: fails(function () { getOwnPropertySymbolsModule.f(1); }) }, {\n  getOwnPropertySymbols: function getOwnPropertySymbols(it) {\n    return getOwnPropertySymbolsModule.f(toObject(it));\n  }\n});\n\n// `JSON.stringify` method behavior with symbols\n// https://tc39.github.io/ecma262/#sec-json.stringify\nif ($stringify) {\n  var FORCED_JSON_STRINGIFY = !NATIVE_SYMBOL || fails(function () {\n    var symbol = $Symbol();\n    // MS Edge converts symbol values to JSON as {}\n    return $stringify([symbol]) != '[null]'\n      // WebKit converts symbol values to JSON as null\n      || $stringify({ a: symbol }) != '{}'\n      // V8 throws on boxed symbols\n      || $stringify(Object(symbol)) != '{}';\n  });\n\n  $({ target: 'JSON', stat: true, forced: FORCED_JSON_STRINGIFY }, {\n    // eslint-disable-next-line no-unused-vars\n    stringify: function stringify(it, replacer, space) {\n      var args = [it];\n      var index = 1;\n      var $replacer;\n      while (arguments.length > index) args.push(arguments[index++]);\n      $replacer = replacer;\n      if (!isObject(replacer) && it === undefined || isSymbol(it)) return; // IE8 returns string on undefined\n      if (!isArray(replacer)) replacer = function (key, value) {\n        if (typeof $replacer == 'function') value = $replacer.call(this, key, value);\n        if (!isSymbol(value)) return value;\n      };\n      args[1] = replacer;\n      return $stringify.apply(null, args);\n    }\n  });\n}\n\n// `Symbol.prototype[@@toPrimitive]` method\n// https://tc39.github.io/ecma262/#sec-symbol.prototype-@@toprimitive\nif (!$Symbol[PROTOTYPE][TO_PRIMITIVE]) {\n  createNonEnumerableProperty($Symbol[PROTOTYPE], TO_PRIMITIVE, $Symbol[PROTOTYPE].valueOf);\n}\n// `Symbol.prototype[@@toStringTag]` property\n// https://tc39.github.io/ecma262/#sec-symbol.prototype-@@tostringtag\nsetToStringTag($Symbol, SYMBOL);\n\nhiddenKeys[HIDDEN] = true;\n","require('../../modules/es.symbol');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getOwnPropertySymbols;\n","'use strict';\nvar getPrototypeOf = require('../internals/object-get-prototype-of');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar IS_PURE = require('../internals/is-pure');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar BUGGY_SAFARI_ITERATORS = false;\n\nvar returnThis = function () { return this; };\n\n// `%IteratorPrototype%` object\n// https://tc39.github.io/ecma262/#sec-%iteratorprototype%-object\nvar IteratorPrototype, PrototypeOfArrayIteratorPrototype, arrayIterator;\n\nif ([].keys) {\n  arrayIterator = [].keys();\n  // Safari 8 has buggy iterators w/o `next`\n  if (!('next' in arrayIterator)) BUGGY_SAFARI_ITERATORS = true;\n  else {\n    PrototypeOfArrayIteratorPrototype = getPrototypeOf(getPrototypeOf(arrayIterator));\n    if (PrototypeOfArrayIteratorPrototype !== Object.prototype) IteratorPrototype = PrototypeOfArrayIteratorPrototype;\n  }\n}\n\nif (IteratorPrototype == undefined) IteratorPrototype = {};\n\n// 25.1.2.1.1 %IteratorPrototype%[@@iterator]()\nif (!IS_PURE && !has(IteratorPrototype, ITERATOR)) {\n  createNonEnumerableProperty(IteratorPrototype, ITERATOR, returnThis);\n}\n\nmodule.exports = {\n  IteratorPrototype: IteratorPrototype,\n  BUGGY_SAFARI_ITERATORS: BUGGY_SAFARI_ITERATORS\n};\n","module.exports = {};\n","var fails = require('../internals/fails');\n\nmodule.exports = !fails(function () {\n  function F() { /* empty */ }\n  F.prototype.constructor = null;\n  return Object.getPrototypeOf(new F()) !== F.prototype;\n});\n","var has = require('../internals/has');\nvar toObject = require('../internals/to-object');\nvar sharedKey = require('../internals/shared-key');\nvar CORRECT_PROTOTYPE_GETTER = require('../internals/correct-prototype-getter');\n\nvar IE_PROTO = sharedKey('IE_PROTO');\nvar ObjectPrototype = Object.prototype;\n\n// `Object.getPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.getprototypeof\nmodule.exports = CORRECT_PROTOTYPE_GETTER ? Object.getPrototypeOf : function (O) {\n  O = toObject(O);\n  if (has(O, IE_PROTO)) return O[IE_PROTO];\n  if (typeof O.constructor == 'function' && O instanceof O.constructor) {\n    return O.constructor.prototype;\n  } return O instanceof Object ? ObjectPrototype : null;\n};\n","'use strict';\nvar IteratorPrototype = require('../internals/iterators-core').IteratorPrototype;\nvar create = require('../internals/object-create');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar Iterators = require('../internals/iterators');\n\nvar returnThis = function () { return this; };\n\nmodule.exports = function (IteratorConstructor, NAME, next) {\n  var TO_STRING_TAG = NAME + ' Iterator';\n  IteratorConstructor.prototype = create(IteratorPrototype, { next: createPropertyDescriptor(1, next) });\n  setToStringTag(IteratorConstructor, TO_STRING_TAG, false, true);\n  Iterators[TO_STRING_TAG] = returnThis;\n  return IteratorConstructor;\n};\n","var anObject = require('../internals/an-object');\nvar aPossiblePrototype = require('../internals/a-possible-prototype');\n\n// `Object.setPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.setprototypeof\n// Works with __proto__ only. Old v8 can't work with null proto objects.\n/* eslint-disable no-proto */\nmodule.exports = Object.setPrototypeOf || ('__proto__' in {} ? function () {\n  var CORRECT_SETTER = false;\n  var test = {};\n  var setter;\n  try {\n    setter = Object.getOwnPropertyDescriptor(Object.prototype, '__proto__').set;\n    setter.call(test, []);\n    CORRECT_SETTER = test instanceof Array;\n  } catch (error) { /* empty */ }\n  return function setPrototypeOf(O, proto) {\n    anObject(O);\n    aPossiblePrototype(proto);\n    if (CORRECT_SETTER) setter.call(O, proto);\n    else O.__proto__ = proto;\n    return O;\n  };\n}() : undefined);\n","var isObject = require('../internals/is-object');\n\nmodule.exports = function (it) {\n  if (!isObject(it) && it !== null) {\n    throw TypeError(\"Can't set \" + String(it) + ' as a prototype');\n  } return it;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar createIteratorConstructor = require('../internals/create-iterator-constructor');\nvar getPrototypeOf = require('../internals/object-get-prototype-of');\nvar setPrototypeOf = require('../internals/object-set-prototype-of');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar redefine = require('../internals/redefine');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar IS_PURE = require('../internals/is-pure');\nvar Iterators = require('../internals/iterators');\nvar IteratorsCore = require('../internals/iterators-core');\n\nvar IteratorPrototype = IteratorsCore.IteratorPrototype;\nvar BUGGY_SAFARI_ITERATORS = IteratorsCore.BUGGY_SAFARI_ITERATORS;\nvar ITERATOR = wellKnownSymbol('iterator');\nvar KEYS = 'keys';\nvar VALUES = 'values';\nvar ENTRIES = 'entries';\n\nvar returnThis = function () { return this; };\n\nmodule.exports = function (Iterable, NAME, IteratorConstructor, next, DEFAULT, IS_SET, FORCED) {\n  createIteratorConstructor(IteratorConstructor, NAME, next);\n\n  var getIterationMethod = function (KIND) {\n    if (KIND === DEFAULT && defaultIterator) return defaultIterator;\n    if (!BUGGY_SAFARI_ITERATORS && KIND in IterablePrototype) return IterablePrototype[KIND];\n    switch (KIND) {\n      case KEYS: return function keys() { return new IteratorConstructor(this, KIND); };\n      case VALUES: return function values() { return new IteratorConstructor(this, KIND); };\n      case ENTRIES: return function entries() { return new IteratorConstructor(this, KIND); };\n    } return function () { return new IteratorConstructor(this); };\n  };\n\n  var TO_STRING_TAG = NAME + ' Iterator';\n  var INCORRECT_VALUES_NAME = false;\n  var IterablePrototype = Iterable.prototype;\n  var nativeIterator = IterablePrototype[ITERATOR]\n    || IterablePrototype['@@iterator']\n    || DEFAULT && IterablePrototype[DEFAULT];\n  var defaultIterator = !BUGGY_SAFARI_ITERATORS && nativeIterator || getIterationMethod(DEFAULT);\n  var anyNativeIterator = NAME == 'Array' ? IterablePrototype.entries || nativeIterator : nativeIterator;\n  var CurrentIteratorPrototype, methods, KEY;\n\n  // fix native\n  if (anyNativeIterator) {\n    CurrentIteratorPrototype = getPrototypeOf(anyNativeIterator.call(new Iterable()));\n    if (IteratorPrototype !== Object.prototype && CurrentIteratorPrototype.next) {\n      if (!IS_PURE && getPrototypeOf(CurrentIteratorPrototype) !== IteratorPrototype) {\n        if (setPrototypeOf) {\n          setPrototypeOf(CurrentIteratorPrototype, IteratorPrototype);\n        } else if (typeof CurrentIteratorPrototype[ITERATOR] != 'function') {\n          createNonEnumerableProperty(CurrentIteratorPrototype, ITERATOR, returnThis);\n        }\n      }\n      // Set @@toStringTag to native iterators\n      setToStringTag(CurrentIteratorPrototype, TO_STRING_TAG, true, true);\n      if (IS_PURE) Iterators[TO_STRING_TAG] = returnThis;\n    }\n  }\n\n  // fix Array#{values, @@iterator}.name in V8 / FF\n  if (DEFAULT == VALUES && nativeIterator && nativeIterator.name !== VALUES) {\n    INCORRECT_VALUES_NAME = true;\n    defaultIterator = function values() { return nativeIterator.call(this); };\n  }\n\n  // define iterator\n  if ((!IS_PURE || FORCED) && IterablePrototype[ITERATOR] !== defaultIterator) {\n    createNonEnumerableProperty(IterablePrototype, ITERATOR, defaultIterator);\n  }\n  Iterators[NAME] = defaultIterator;\n\n  // export additional methods\n  if (DEFAULT) {\n    methods = {\n      values: getIterationMethod(VALUES),\n      keys: IS_SET ? defaultIterator : getIterationMethod(KEYS),\n      entries: getIterationMethod(ENTRIES)\n    };\n    if (FORCED) for (KEY in methods) {\n      if (BUGGY_SAFARI_ITERATORS || INCORRECT_VALUES_NAME || !(KEY in IterablePrototype)) {\n        redefine(IterablePrototype, KEY, methods[KEY]);\n      }\n    } else $({ target: NAME, proto: true, forced: BUGGY_SAFARI_ITERATORS || INCORRECT_VALUES_NAME }, methods);\n  }\n\n  return methods;\n};\n","'use strict';\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar Iterators = require('../internals/iterators');\nvar InternalStateModule = require('../internals/internal-state');\nvar defineIterator = require('../internals/define-iterator');\n\nvar ARRAY_ITERATOR = 'Array Iterator';\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(ARRAY_ITERATOR);\n\n// `Array.prototype.entries` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.entries\n// `Array.prototype.keys` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.keys\n// `Array.prototype.values` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.values\n// `Array.prototype[@@iterator]` method\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@iterator\n// `CreateArrayIterator` internal method\n// https://tc39.github.io/ecma262/#sec-createarrayiterator\nmodule.exports = defineIterator(Array, 'Array', function (iterated, kind) {\n  setInternalState(this, {\n    type: ARRAY_ITERATOR,\n    target: toIndexedObject(iterated), // target\n    index: 0,                          // next index\n    kind: kind                         // kind\n  });\n// `%ArrayIteratorPrototype%.next` method\n// https://tc39.github.io/ecma262/#sec-%arrayiteratorprototype%.next\n}, function () {\n  var state = getInternalState(this);\n  var target = state.target;\n  var kind = state.kind;\n  var index = state.index++;\n  if (!target || index >= target.length) {\n    state.target = undefined;\n    return { value: undefined, done: true };\n  }\n  if (kind == 'keys') return { value: index, done: false };\n  if (kind == 'values') return { value: target[index], done: false };\n  return { value: [index, target[index]], done: false };\n}, 'values');\n\n// argumentsList[@@iterator] is %ArrayProto_values%\n// https://tc39.github.io/ecma262/#sec-createunmappedargumentsobject\n// https://tc39.github.io/ecma262/#sec-createmappedargumentsobject\nIterators.Arguments = Iterators.Array;\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('keys');\naddToUnscopables('values');\naddToUnscopables('entries');\n","// iterable DOM collections\n// flag - `iterable` interface - 'entries', 'keys', 'values', 'forEach' methods\nmodule.exports = {\n  CSSRuleList: 0,\n  CSSStyleDeclaration: 0,\n  CSSValueList: 0,\n  ClientRectList: 0,\n  DOMRectList: 0,\n  DOMStringList: 0,\n  DOMTokenList: 1,\n  DataTransferItemList: 0,\n  FileList: 0,\n  HTMLAllCollection: 0,\n  HTMLCollection: 0,\n  HTMLFormElement: 0,\n  HTMLSelectElement: 0,\n  MediaList: 0,\n  MimeTypeArray: 0,\n  NamedNodeMap: 0,\n  NodeList: 1,\n  PaintRequestList: 0,\n  Plugin: 0,\n  PluginArray: 0,\n  SVGLengthList: 0,\n  SVGNumberList: 0,\n  SVGPathSegList: 0,\n  SVGPointList: 0,\n  SVGStringList: 0,\n  SVGTransformList: 0,\n  SourceBufferList: 0,\n  StyleSheetList: 0,\n  TextTrackCueList: 0,\n  TextTrackList: 0,\n  TouchList: 0\n};\n","require('./es.array.iterator');\nvar DOMIterables = require('../internals/dom-iterables');\nvar global = require('../internals/global');\nvar classof = require('../internals/classof');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar Iterators = require('../internals/iterators');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n\nfor (var COLLECTION_NAME in DOMIterables) {\n  var Collection = global[COLLECTION_NAME];\n  var CollectionPrototype = Collection && Collection.prototype;\n  if (CollectionPrototype && classof(CollectionPrototype) !== TO_STRING_TAG) {\n    createNonEnumerableProperty(CollectionPrototype, TO_STRING_TAG, COLLECTION_NAME);\n  }\n  Iterators[COLLECTION_NAME] = Iterators.Array;\n}\n","var toInteger = require('../internals/to-integer');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `String.prototype.{ codePointAt, at }` methods implementation\nvar createMethod = function (CONVERT_TO_STRING) {\n  return function ($this, pos) {\n    var S = String(requireObjectCoercible($this));\n    var position = toInteger(pos);\n    var size = S.length;\n    var first, second;\n    if (position < 0 || position >= size) return CONVERT_TO_STRING ? '' : undefined;\n    first = S.charCodeAt(position);\n    return first < 0xD800 || first > 0xDBFF || position + 1 === size\n      || (second = S.charCodeAt(position + 1)) < 0xDC00 || second > 0xDFFF\n        ? CONVERT_TO_STRING ? S.charAt(position) : first\n        : CONVERT_TO_STRING ? S.slice(position, position + 2) : (first - 0xD800 << 10) + (second - 0xDC00) + 0x10000;\n  };\n};\n\nmodule.exports = {\n  // `String.prototype.codePointAt` method\n  // https://tc39.github.io/ecma262/#sec-string.prototype.codepointat\n  codeAt: createMethod(false),\n  // `String.prototype.at` method\n  // https://github.com/mathiasbynens/String.prototype.at\n  charAt: createMethod(true)\n};\n","'use strict';\nvar charAt = require('../internals/string-multibyte').charAt;\nvar InternalStateModule = require('../internals/internal-state');\nvar defineIterator = require('../internals/define-iterator');\n\nvar STRING_ITERATOR = 'String Iterator';\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(STRING_ITERATOR);\n\n// `String.prototype[@@iterator]` method\n// https://tc39.github.io/ecma262/#sec-string.prototype-@@iterator\ndefineIterator(String, 'String', function (iterated) {\n  setInternalState(this, {\n    type: STRING_ITERATOR,\n    string: String(iterated),\n    index: 0\n  });\n// `%StringIteratorPrototype%.next` method\n// https://tc39.github.io/ecma262/#sec-%stringiteratorprototype%.next\n}, function next() {\n  var state = getInternalState(this);\n  var string = state.string;\n  var index = state.index;\n  var point;\n  if (index >= string.length) return { value: undefined, done: true };\n  point = charAt(string, index);\n  state.index += point.length;\n  return { value: point, done: false };\n});\n","var classof = require('../internals/classof');\nvar Iterators = require('../internals/iterators');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar ITERATOR = wellKnownSymbol('iterator');\n\nmodule.exports = function (it) {\n  if (it != undefined) return it[ITERATOR]\n    || it['@@iterator']\n    || Iterators[classof(it)];\n};\n","var anObject = require('../internals/an-object');\nvar getIteratorMethod = require('../internals/get-iterator-method');\n\nmodule.exports = function (it) {\n  var iteratorMethod = getIteratorMethod(it);\n  if (typeof iteratorMethod != 'function') {\n    throw TypeError(String(it) + ' is not iterable');\n  } return anObject(iteratorMethod.call(it));\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar create = require('../internals/object-create');\n\n// `Object.create` method\n// https://tc39.github.io/ecma262/#sec-object.create\n$({ target: 'Object', stat: true, sham: !DESCRIPTORS }, {\n  create: create\n});\n","require('../../modules/es.object.create');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nmodule.exports = function create(P, D) {\n  return Object.create(P, D);\n};\n","var parent = require('../../es/object/create');\n\nmodule.exports = parent;\n","var parent = require('../../es/object/define-property');\n\nmodule.exports = parent;\n","var _Object$defineProperty = require(\"../core-js/object/define-property\");\n\nfunction _defineProperty(obj, key, value) {\n  if (key in obj) {\n    _Object$defineProperty(obj, key, {\n      value: value,\n      enumerable: true,\n      configurable: true,\n      writable: true\n    });\n  } else {\n    obj[key] = value;\n  }\n\n  return obj;\n}\n\nmodule.exports = _defineProperty;","var $ = require('../internals/export');\nvar toObject = require('../internals/to-object');\nvar nativeKeys = require('../internals/object-keys');\nvar fails = require('../internals/fails');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeKeys(1); });\n\n// `Object.keys` method\n// https://tc39.github.io/ecma262/#sec-object.keys\n$({ target: 'Object', stat: true, forced: FAILS_ON_PRIMITIVES }, {\n  keys: function keys(it) {\n    return nativeKeys(toObject(it));\n  }\n});\n","require('../../modules/es.object.keys');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.keys;\n","var fails = require('../internals/fails');\nvar whitespaces = require('../internals/whitespaces');\n\nvar non = '\\u200B\\u0085\\u180E';\n\n// check that a method works with the correct list\n// of whitespaces and has a correct name\nmodule.exports = function (METHOD_NAME) {\n  return fails(function () {\n    return !!whitespaces[METHOD_NAME]() || non[METHOD_NAME]() != non || whitespaces[METHOD_NAME].name !== METHOD_NAME;\n  });\n};\n","// a string of all valid unicode whitespaces\n// eslint-disable-next-line max-len\nmodule.exports = '\\u0009\\u000A\\u000B\\u000C\\u000D\\u0020\\u00A0\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200A\\u202F\\u205F\\u3000\\u2028\\u2029\\uFEFF';\n","var requireObjectCoercible = require('../internals/require-object-coercible');\nvar whitespaces = require('../internals/whitespaces');\n\nvar whitespace = '[' + whitespaces + ']';\nvar ltrim = RegExp('^' + whitespace + whitespace + '*');\nvar rtrim = RegExp(whitespace + whitespace + '*$');\n\n// `String.prototype.{ trim, trimStart, trimEnd, trimLeft, trimRight }` methods implementation\nvar createMethod = function (TYPE) {\n  return function ($this) {\n    var string = String(requireObjectCoercible($this));\n    if (TYPE & 1) string = string.replace(ltrim, '');\n    if (TYPE & 2) string = string.replace(rtrim, '');\n    return string;\n  };\n};\n\nmodule.exports = {\n  // `String.prototype.{ trimLeft, trimStart }` methods\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trimstart\n  start: createMethod(1),\n  // `String.prototype.{ trimRight, trimEnd }` methods\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trimend\n  end: createMethod(2),\n  // `String.prototype.trim` method\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trim\n  trim: createMethod(3)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $trim = require('../internals/string-trim').trim;\nvar forcedStringTrimMethod = require('../internals/string-trim-forced');\n\n// `String.prototype.trim` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.trim\n$({ target: 'String', proto: true, forced: forcedStringTrimMethod('trim') }, {\n  trim: function trim() {\n    return $trim(this);\n  }\n});\n","var path = require('../internals/path');\n\nmodule.exports = function (CONSTRUCTOR) {\n  return path[CONSTRUCTOR + 'Prototype'];\n};\n","require('../../../modules/es.string.trim');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').trim;\n","var trim = require('../string/virtual/trim');\n\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.trim;\n  return typeof it === 'string' || it === StringPrototype\n    || (it instanceof String && own === StringPrototype.trim) ? trim : own;\n};\n","'use strict';\nvar fails = require('../internals/fails');\n\nmodule.exports = function (METHOD_NAME, argument) {\n  var method = [][METHOD_NAME];\n  return !!method && fails(function () {\n    // eslint-disable-next-line no-useless-call,no-throw-literal\n    method.call(null, argument || function () { throw 1; }, 1);\n  });\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar has = require('../internals/has');\n\nvar defineProperty = Object.defineProperty;\nvar cache = {};\n\nvar thrower = function (it) { throw it; };\n\nmodule.exports = function (METHOD_NAME, options) {\n  if (has(cache, METHOD_NAME)) return cache[METHOD_NAME];\n  if (!options) options = {};\n  var method = [][METHOD_NAME];\n  var ACCESSORS = has(options, 'ACCESSORS') ? options.ACCESSORS : false;\n  var argument0 = has(options, 0) ? options[0] : thrower;\n  var argument1 = has(options, 1) ? options[1] : undefined;\n\n  return cache[METHOD_NAME] = !!method && !fails(function () {\n    if (ACCESSORS && !DESCRIPTORS) return true;\n    var O = { length: -1 };\n\n    if (ACCESSORS) defineProperty(O, 1, { enumerable: true, get: thrower });\n    else O[1] = 1;\n\n    method.call(O, argument0, argument1);\n  });\n};\n","'use strict';\nvar $forEach = require('../internals/array-iteration').forEach;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('forEach');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('forEach');\n\n// `Array.prototype.forEach` method implementation\n// https://tc39.github.io/ecma262/#sec-array.prototype.foreach\nmodule.exports = (!STRICT_METHOD || !USES_TO_LENGTH) ? function forEach(callbackfn /* , thisArg */) {\n  return $forEach(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n} : [].forEach;\n","'use strict';\nvar $ = require('../internals/export');\nvar forEach = require('../internals/array-for-each');\n\n// `Array.prototype.forEach` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.foreach\n$({ target: 'Array', proto: true, forced: [].forEach != forEach }, {\n  forEach: forEach\n});\n","require('../../../modules/es.array.for-each');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').forEach;\n","var global = require('../internals/global');\nvar userAgent = require('../internals/engine-user-agent');\n\nvar process = global.process;\nvar versions = process && process.versions;\nvar v8 = versions && versions.v8;\nvar match, version;\n\nif (v8) {\n  match = v8.split('.');\n  version = match[0] + match[1];\n} else if (userAgent) {\n  match = userAgent.match(/Edge\\/(\\d+)/);\n  if (!match || match[1] >= 74) {\n    match = userAgent.match(/Chrome\\/(\\d+)/);\n    if (match) version = match[1];\n  }\n}\n\nmodule.exports = version && +version;\n","require('../../modules/web.dom-collections.iterator');\nvar forEach = require('../array/virtual/for-each');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.forEach;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.forEach)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? forEach : own;\n};\n","var getBuiltIn = require('../internals/get-built-in');\n\nmodule.exports = getBuiltIn('navigator', 'userAgent') || '';\n","var fails = require('../internals/fails');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar V8_VERSION = require('../internals/engine-v8-version');\n\nvar SPECIES = wellKnownSymbol('species');\n\nmodule.exports = function (METHOD_NAME) {\n  // We can't use this feature detection in V8 since it causes\n  // deoptimization and serious performance degradation\n  // https://github.com/zloirock/core-js/issues/677\n  return V8_VERSION >= 51 || !fails(function () {\n    var array = [];\n    var constructor = array.constructor = {};\n    constructor[SPECIES] = function () {\n      return { foo: 1 };\n    };\n    return array[METHOD_NAME](Boolean).foo !== 1;\n  });\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $map = require('../internals/array-iteration').map;\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('map');\n// FF49- issue\nvar USES_TO_LENGTH = arrayMethodUsesToLength('map');\n\n// `Array.prototype.map` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.map\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  map: function map(callbackfn /* , thisArg */) {\n    return $map(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.map');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').map;\n","var map = require('../array/virtual/map');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.map;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.map) ? map : own;\n};\n","var global = require('../internals/global');\nvar trim = require('../internals/string-trim').trim;\nvar whitespaces = require('../internals/whitespaces');\n\nvar $parseInt = global.parseInt;\nvar hex = /^[+-]?0[Xx]/;\nvar FORCED = $parseInt(whitespaces + '08') !== 8 || $parseInt(whitespaces + '0x16') !== 22;\n\n// `parseInt` method\n// https://tc39.github.io/ecma262/#sec-parseint-string-radix\nmodule.exports = FORCED ? function parseInt(string, radix) {\n  var S = trim(String(string));\n  return $parseInt(S, (radix >>> 0) || (hex.test(S) ? 16 : 10));\n} : $parseInt;\n","var $ = require('../internals/export');\nvar parseIntImplementation = require('../internals/number-parse-int');\n\n// `parseInt` method\n// https://tc39.github.io/ecma262/#sec-parseint-string-radix\n$({ global: true, forced: parseInt != parseIntImplementation }, {\n  parseInt: parseIntImplementation\n});\n","require('../modules/es.parse-int');\nvar path = require('../internals/path');\n\nmodule.exports = path.parseInt;\n","var DESCRIPTORS = require('../internals/descriptors');\nvar objectKeys = require('../internals/object-keys');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar propertyIsEnumerable = require('../internals/object-property-is-enumerable').f;\n\n// `Object.{ entries, values }` methods implementation\nvar createMethod = function (TO_ENTRIES) {\n  return function (it) {\n    var O = toIndexedObject(it);\n    var keys = objectKeys(O);\n    var length = keys.length;\n    var i = 0;\n    var result = [];\n    var key;\n    while (length > i) {\n      key = keys[i++];\n      if (!DESCRIPTORS || propertyIsEnumerable.call(O, key)) {\n        result.push(TO_ENTRIES ? [key, O[key]] : O[key]);\n      }\n    }\n    return result;\n  };\n};\n\nmodule.exports = {\n  // `Object.entries` method\n  // https://tc39.github.io/ecma262/#sec-object.entries\n  entries: createMethod(true),\n  // `Object.values` method\n  // https://tc39.github.io/ecma262/#sec-object.values\n  values: createMethod(false)\n};\n","var $ = require('../internals/export');\nvar $values = require('../internals/object-to-array').values;\n\n// `Object.values` method\n// https://tc39.github.io/ecma262/#sec-object.values\n$({ target: 'Object', stat: true }, {\n  values: function values(O) {\n    return $values(O);\n  }\n});\n","require('../../modules/es.object.values');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.values;\n","'use strict';\nvar $ = require('../internals/export');\nvar $filter = require('../internals/array-iteration').filter;\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('filter');\n// Edge 14- issue\nvar USES_TO_LENGTH = arrayMethodUsesToLength('filter');\n\n// `Array.prototype.filter` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.filter\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  filter: function filter(callbackfn /* , thisArg */) {\n    return $filter(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.filter');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').filter;\n","var filter = require('../array/virtual/filter');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.filter;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.filter) ? filter : own;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar isArray = require('../internals/is-array');\nvar isObject = require('../internals/is-object');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar createProperty = require('../internals/create-property');\nvar arraySpeciesCreate = require('../internals/array-species-create');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar V8_VERSION = require('../internals/engine-v8-version');\n\nvar IS_CONCAT_SPREADABLE = wellKnownSymbol('isConcatSpreadable');\nvar MAX_SAFE_INTEGER = 0x1FFFFFFFFFFFFF;\nvar MAXIMUM_ALLOWED_INDEX_EXCEEDED = 'Maximum allowed index exceeded';\n\n// We can't use this feature detection in V8 since it causes\n// deoptimization and serious performance degradation\n// https://github.com/zloirock/core-js/issues/679\nvar IS_CONCAT_SPREADABLE_SUPPORT = V8_VERSION >= 51 || !fails(function () {\n  var array = [];\n  array[IS_CONCAT_SPREADABLE] = false;\n  return array.concat()[0] !== array;\n});\n\nvar SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('concat');\n\nvar isConcatSpreadable = function (O) {\n  if (!isObject(O)) return false;\n  var spreadable = O[IS_CONCAT_SPREADABLE];\n  return spreadable !== undefined ? !!spreadable : isArray(O);\n};\n\nvar FORCED = !IS_CONCAT_SPREADABLE_SUPPORT || !SPECIES_SUPPORT;\n\n// `Array.prototype.concat` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.concat\n// with adding support of @@isConcatSpreadable and @@species\n$({ target: 'Array', proto: true, forced: FORCED }, {\n  concat: function concat(arg) { // eslint-disable-line no-unused-vars\n    var O = toObject(this);\n    var A = arraySpeciesCreate(O, 0);\n    var n = 0;\n    var i, k, length, len, E;\n    for (i = -1, length = arguments.length; i < length; i++) {\n      E = i === -1 ? O : arguments[i];\n      if (isConcatSpreadable(E)) {\n        len = toLength(E.length);\n        if (n + len > MAX_SAFE_INTEGER) throw TypeError(MAXIMUM_ALLOWED_INDEX_EXCEEDED);\n        for (k = 0; k < len; k++, n++) if (k in E) createProperty(A, n, E[k]);\n      } else {\n        if (n >= MAX_SAFE_INTEGER) throw TypeError(MAXIMUM_ALLOWED_INDEX_EXCEEDED);\n        createProperty(A, n++, E);\n      }\n    }\n    A.length = n;\n    return A;\n  }\n});\n","require('../../../modules/es.array.concat');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').concat;\n","var concat = require('../array/virtual/concat');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.concat;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.concat) ? concat : own;\n};\n","var $ = require('../internals/export');\nvar isArray = require('../internals/is-array');\n\n// `Array.isArray` method\n// https://tc39.github.io/ecma262/#sec-array.isarray\n$({ target: 'Array', stat: true }, {\n  isArray: isArray\n});\n","require('../../modules/es.array.is-array');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Array.isArray;\n","var parent = require('../../es/array/is-array');\n\nmodule.exports = parent;\n","var _Array$isArray = require(\"../core-js/array/is-array\");\n\nfunction _arrayWithoutHoles(arr) {\n  if (_Array$isArray(arr)) {\n    for (var i = 0, arr2 = new Array(arr.length); i < arr.length; i++) {\n      arr2[i] = arr[i];\n    }\n\n    return arr2;\n  }\n}\n\nmodule.exports = _arrayWithoutHoles;","var anObject = require('../internals/an-object');\n\n// call something on iterator step with safe closing on error\nmodule.exports = function (iterator, fn, value, ENTRIES) {\n  try {\n    return ENTRIES ? fn(anObject(value)[0], value[1]) : fn(value);\n  // 7.4.6 IteratorClose(iterator, completion)\n  } catch (error) {\n    var returnMethod = iterator['return'];\n    if (returnMethod !== undefined) anObject(returnMethod.call(iterator));\n    throw error;\n  }\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\nvar Iterators = require('../internals/iterators');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar ArrayPrototype = Array.prototype;\n\n// check on default Array iterator\nmodule.exports = function (it) {\n  return it !== undefined && (Iterators.Array === it || ArrayPrototype[ITERATOR] === it);\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar SAFE_CLOSING = false;\n\ntry {\n  var called = 0;\n  var iteratorWithReturn = {\n    next: function () {\n      return { done: !!called++ };\n    },\n    'return': function () {\n      SAFE_CLOSING = true;\n    }\n  };\n  iteratorWithReturn[ITERATOR] = function () {\n    return this;\n  };\n  // eslint-disable-next-line no-throw-literal\n  Array.from(iteratorWithReturn, function () { throw 2; });\n} catch (error) { /* empty */ }\n\nmodule.exports = function (exec, SKIP_CLOSING) {\n  if (!SKIP_CLOSING && !SAFE_CLOSING) return false;\n  var ITERATION_SUPPORT = false;\n  try {\n    var object = {};\n    object[ITERATOR] = function () {\n      return {\n        next: function () {\n          return { done: ITERATION_SUPPORT = true };\n        }\n      };\n    };\n    exec(object);\n  } catch (error) { /* empty */ }\n  return ITERATION_SUPPORT;\n};\n","var $ = require('../internals/export');\nvar from = require('../internals/array-from');\nvar checkCorrectnessOfIteration = require('../internals/check-correctness-of-iteration');\n\nvar INCORRECT_ITERATION = !checkCorrectnessOfIteration(function (iterable) {\n  Array.from(iterable);\n});\n\n// `Array.from` method\n// https://tc39.github.io/ecma262/#sec-array.from\n$({ target: 'Array', stat: true, forced: INCORRECT_ITERATION }, {\n  from: from\n});\n","'use strict';\nvar bind = require('../internals/function-bind-context');\nvar toObject = require('../internals/to-object');\nvar callWithSafeIterationClosing = require('../internals/call-with-safe-iteration-closing');\nvar isArrayIteratorMethod = require('../internals/is-array-iterator-method');\nvar toLength = require('../internals/to-length');\nvar createProperty = require('../internals/create-property');\nvar getIteratorMethod = require('../internals/get-iterator-method');\n\n// `Array.from` method implementation\n// https://tc39.github.io/ecma262/#sec-array.from\nmodule.exports = function from(arrayLike /* , mapfn = undefined, thisArg = undefined */) {\n  var O = toObject(arrayLike);\n  var C = typeof this == 'function' ? this : Array;\n  var argumentsLength = arguments.length;\n  var mapfn = argumentsLength > 1 ? arguments[1] : undefined;\n  var mapping = mapfn !== undefined;\n  var iteratorMethod = getIteratorMethod(O);\n  var index = 0;\n  var length, result, step, iterator, next, value;\n  if (mapping) mapfn = bind(mapfn, argumentsLength > 2 ? arguments[2] : undefined, 2);\n  // if the target is not iterable or it's an array with the default iterator - use a simple case\n  if (iteratorMethod != undefined && !(C == Array && isArrayIteratorMethod(iteratorMethod))) {\n    iterator = iteratorMethod.call(O);\n    next = iterator.next;\n    result = new C();\n    for (;!(step = next.call(iterator)).done; index++) {\n      value = mapping ? callWithSafeIterationClosing(iterator, mapfn, [step.value, index], true) : step.value;\n      createProperty(result, index, value);\n    }\n  } else {\n    length = toLength(O.length);\n    result = new C(length);\n    for (;length > index; index++) {\n      value = mapping ? mapfn(O[index], index) : O[index];\n      createProperty(result, index, value);\n    }\n  }\n  result.length = index;\n  return result;\n};\n","require('../../modules/es.string.iterator');\nrequire('../../modules/es.array.from');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Array.from;\n","var classof = require('../internals/classof');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar Iterators = require('../internals/iterators');\n\nvar ITERATOR = wellKnownSymbol('iterator');\n\nmodule.exports = function (it) {\n  var O = Object(it);\n  return O[ITERATOR] !== undefined\n    || '@@iterator' in O\n    // eslint-disable-next-line no-prototype-builtins\n    || Iterators.hasOwnProperty(classof(O));\n};\n","var _Array$from = require(\"../core-js/array/from\");\n\nvar _isIterable = require(\"../core-js/is-iterable\");\n\nfunction _iterableToArray(iter) {\n  if (_isIterable(Object(iter)) || Object.prototype.toString.call(iter) === \"[object Arguments]\") return _Array$from(iter);\n}\n\nmodule.exports = _iterableToArray;","function _nonIterableSpread() {\n  throw new TypeError(\"Invalid attempt to spread non-iterable instance\");\n}\n\nmodule.exports = _nonIterableSpread;","var arrayWithoutHoles = require(\"./arrayWithoutHoles\");\n\nvar iterableToArray = require(\"./iterableToArray\");\n\nvar nonIterableSpread = require(\"./nonIterableSpread\");\n\nfunction _toConsumableArray(arr) {\n  return arrayWithoutHoles(arr) || iterableToArray(arr) || nonIterableSpread();\n}\n\nmodule.exports = _toConsumableArray;","'use strict';\nvar $ = require('../internals/export');\nvar isObject = require('../internals/is-object');\nvar isArray = require('../internals/is-array');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toLength = require('../internals/to-length');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar createProperty = require('../internals/create-property');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('slice');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('slice', { ACCESSORS: true, 0: 0, 1: 2 });\n\nvar SPECIES = wellKnownSymbol('species');\nvar nativeSlice = [].slice;\nvar max = Math.max;\n\n// `Array.prototype.slice` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.slice\n// fallback for not array-like ES3 strings and DOM objects\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  slice: function slice(start, end) {\n    var O = toIndexedObject(this);\n    var length = toLength(O.length);\n    var k = toAbsoluteIndex(start, length);\n    var fin = toAbsoluteIndex(end === undefined ? length : end, length);\n    // inline `ArraySpeciesCreate` for usage native `Array#slice` where it's possible\n    var Constructor, result, n;\n    if (isArray(O)) {\n      Constructor = O.constructor;\n      // cross-realm fallback\n      if (typeof Constructor == 'function' && (Constructor === Array || isArray(Constructor.prototype))) {\n        Constructor = undefined;\n      } else if (isObject(Constructor)) {\n        Constructor = Constructor[SPECIES];\n        if (Constructor === null) Constructor = undefined;\n      }\n      if (Constructor === Array || Constructor === undefined) {\n        return nativeSlice.call(O, k, fin);\n      }\n    }\n    result = new (Constructor === undefined ? Array : Constructor)(max(fin - k, 0));\n    for (n = 0; k < fin; k++, n++) if (k in O) createProperty(result, n, O[k]);\n    result.length = n;\n    return result;\n  }\n});\n","require('../../../modules/es.array.slice');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').slice;\n","var slice = require('../array/virtual/slice');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.slice;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.slice) ? slice : own;\n};\n","var $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar toObject = require('../internals/to-object');\nvar nativeGetPrototypeOf = require('../internals/object-get-prototype-of');\nvar CORRECT_PROTOTYPE_GETTER = require('../internals/correct-prototype-getter');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeGetPrototypeOf(1); });\n\n// `Object.getPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.getprototypeof\n$({ target: 'Object', stat: true, forced: FAILS_ON_PRIMITIVES, sham: !CORRECT_PROTOTYPE_GETTER }, {\n  getPrototypeOf: function getPrototypeOf(it) {\n    return nativeGetPrototypeOf(toObject(it));\n  }\n});\n\n","require('../../modules/es.object.get-prototype-of');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getPrototypeOf;\n","var parent = require('../../es/object/get-prototype-of');\n\nmodule.exports = parent;\n","'use strict';\nvar $ = require('../internals/export');\nvar $indexOf = require('../internals/array-includes').indexOf;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar nativeIndexOf = [].indexOf;\n\nvar NEGATIVE_ZERO = !!nativeIndexOf && 1 / [1].indexOf(1, -0) < 0;\nvar STRICT_METHOD = arrayMethodIsStrict('indexOf');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('indexOf', { ACCESSORS: true, 1: 0 });\n\n// `Array.prototype.indexOf` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.indexof\n$({ target: 'Array', proto: true, forced: NEGATIVE_ZERO || !STRICT_METHOD || !USES_TO_LENGTH }, {\n  indexOf: function indexOf(searchElement /* , fromIndex = 0 */) {\n    return NEGATIVE_ZERO\n      // convert -0 to +0\n      ? nativeIndexOf.apply(this, arguments) || 0\n      : $indexOf(this, searchElement, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.index-of');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').indexOf;\n","var indexOf = require('../array/virtual/index-of');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.indexOf;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.indexOf) ? indexOf : own;\n};\n","var parent = require('../../es/array/is-array');\n\nmodule.exports = parent;\n","'use strict';\nvar DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar objectKeys = require('../internals/object-keys');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar toObject = require('../internals/to-object');\nvar IndexedObject = require('../internals/indexed-object');\n\nvar nativeAssign = Object.assign;\nvar defineProperty = Object.defineProperty;\n\n// `Object.assign` method\n// https://tc39.github.io/ecma262/#sec-object.assign\nmodule.exports = !nativeAssign || fails(function () {\n  // should have correct order of operations (Edge bug)\n  if (DESCRIPTORS && nativeAssign({ b: 1 }, nativeAssign(defineProperty({}, 'a', {\n    enumerable: true,\n    get: function () {\n      defineProperty(this, 'b', {\n        value: 3,\n        enumerable: false\n      });\n    }\n  }), { b: 2 })).b !== 1) return true;\n  // should work with symbols and should have deterministic property order (V8 bug)\n  var A = {};\n  var B = {};\n  // eslint-disable-next-line no-undef\n  var symbol = Symbol();\n  var alphabet = 'abcdefghijklmnopqrst';\n  A[symbol] = 7;\n  alphabet.split('').forEach(function (chr) { B[chr] = chr; });\n  return nativeAssign({}, A)[symbol] != 7 || objectKeys(nativeAssign({}, B)).join('') != alphabet;\n}) ? function assign(target, source) { // eslint-disable-line no-unused-vars\n  var T = toObject(target);\n  var argumentsLength = arguments.length;\n  var index = 1;\n  var getOwnPropertySymbols = getOwnPropertySymbolsModule.f;\n  var propertyIsEnumerable = propertyIsEnumerableModule.f;\n  while (argumentsLength > index) {\n    var S = IndexedObject(arguments[index++]);\n    var keys = getOwnPropertySymbols ? objectKeys(S).concat(getOwnPropertySymbols(S)) : objectKeys(S);\n    var length = keys.length;\n    var j = 0;\n    var key;\n    while (length > j) {\n      key = keys[j++];\n      if (!DESCRIPTORS || propertyIsEnumerable.call(S, key)) T[key] = S[key];\n    }\n  } return T;\n} : nativeAssign;\n","var $ = require('../internals/export');\nvar assign = require('../internals/object-assign');\n\n// `Object.assign` method\n// https://tc39.github.io/ecma262/#sec-object.assign\n$({ target: 'Object', stat: true, forced: Object.assign !== assign }, {\n  assign: assign\n});\n","require('../../modules/es.object.assign');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.assign;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.iterator` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.iterator\ndefineWellKnownSymbol('iterator');\n","require('../../modules/es.symbol.iterator');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar WrappedWellKnownSymbolModule = require('../../internals/well-known-symbol-wrapped');\n\nmodule.exports = WrappedWellKnownSymbolModule.f('iterator');\n","var parent = require('../../es/symbol/iterator');\n\nmodule.exports = parent;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.asyncIterator` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.asynciterator\ndefineWellKnownSymbol('asyncIterator');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.hasInstance` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.hasinstance\ndefineWellKnownSymbol('hasInstance');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.isConcatSpreadable` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.isconcatspreadable\ndefineWellKnownSymbol('isConcatSpreadable');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.match` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.match\ndefineWellKnownSymbol('match');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.matchAll` well-known symbol\ndefineWellKnownSymbol('matchAll');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.replace` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.replace\ndefineWellKnownSymbol('replace');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.search` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.search\ndefineWellKnownSymbol('search');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.species` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.species\ndefineWellKnownSymbol('species');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.split` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.split\ndefineWellKnownSymbol('split');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.toPrimitive` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.toprimitive\ndefineWellKnownSymbol('toPrimitive');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.toStringTag` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.tostringtag\ndefineWellKnownSymbol('toStringTag');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.unscopables` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.unscopables\ndefineWellKnownSymbol('unscopables');\n","var setToStringTag = require('../internals/set-to-string-tag');\n\n// Math[@@toStringTag] property\n// https://tc39.github.io/ecma262/#sec-math-@@tostringtag\nsetToStringTag(Math, 'Math', true);\n","var global = require('../internals/global');\nvar setToStringTag = require('../internals/set-to-string-tag');\n\n// JSON[@@toStringTag] property\n// https://tc39.github.io/ecma262/#sec-json-@@tostringtag\nsetToStringTag(global.JSON, 'JSON', true);\n","require('../../modules/es.array.concat');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.symbol');\nrequire('../../modules/es.symbol.async-iterator');\nrequire('../../modules/es.symbol.description');\nrequire('../../modules/es.symbol.has-instance');\nrequire('../../modules/es.symbol.is-concat-spreadable');\nrequire('../../modules/es.symbol.iterator');\nrequire('../../modules/es.symbol.match');\nrequire('../../modules/es.symbol.match-all');\nrequire('../../modules/es.symbol.replace');\nrequire('../../modules/es.symbol.search');\nrequire('../../modules/es.symbol.species');\nrequire('../../modules/es.symbol.split');\nrequire('../../modules/es.symbol.to-primitive');\nrequire('../../modules/es.symbol.to-string-tag');\nrequire('../../modules/es.symbol.unscopables');\nrequire('../../modules/es.math.to-string-tag');\nrequire('../../modules/es.json.to-string-tag');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Symbol;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.asyncDispose` well-known symbol\n// https://github.com/tc39/proposal-using-statement\ndefineWellKnownSymbol('asyncDispose');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.dispose` well-known symbol\n// https://github.com/tc39/proposal-using-statement\ndefineWellKnownSymbol('dispose');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.observable` well-known symbol\n// https://github.com/tc39/proposal-observable\ndefineWellKnownSymbol('observable');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.patternMatch` well-known symbol\n// https://github.com/tc39/proposal-pattern-matching\ndefineWellKnownSymbol('patternMatch');\n","// TODO: remove from `core-js@4`\nvar defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\ndefineWellKnownSymbol('replaceAll');\n","var parent = require('../../es/symbol');\nrequire('../../modules/esnext.symbol.async-dispose');\nrequire('../../modules/esnext.symbol.dispose');\nrequire('../../modules/esnext.symbol.observable');\nrequire('../../modules/esnext.symbol.pattern-match');\n// TODO: Remove from `core-js@4`\nrequire('../../modules/esnext.symbol.replace-all');\n\nmodule.exports = parent;\n","var _Symbol$iterator = require(\"../core-js/symbol/iterator\");\n\nvar _Symbol = require(\"../core-js/symbol\");\n\nfunction _typeof(obj) {\n  \"@babel/helpers - typeof\";\n\n  if (typeof _Symbol === \"function\" && typeof _Symbol$iterator === \"symbol\") {\n    module.exports = _typeof = function _typeof(obj) {\n      return typeof obj;\n    };\n  } else {\n    module.exports = _typeof = function _typeof(obj) {\n      return obj && typeof _Symbol === \"function\" && obj.constructor === _Symbol && obj !== _Symbol.prototype ? \"symbol\" : typeof obj;\n    };\n  }\n\n  return _typeof(obj);\n}\n\nmodule.exports = _typeof;","var _Array$isArray = require(\"../core-js/array/is-array\");\n\nfunction _arrayWithHoles(arr) {\n  if (_Array$isArray(arr)) return arr;\n}\n\nmodule.exports = _arrayWithHoles;","var _getIterator = require(\"../core-js/get-iterator\");\n\nvar _isIterable = require(\"../core-js/is-iterable\");\n\nfunction _iterableToArrayLimit(arr, i) {\n  if (!(_isIterable(Object(arr)) || Object.prototype.toString.call(arr) === \"[object Arguments]\")) {\n    return;\n  }\n\n  var _arr = [];\n  var _n = true;\n  var _d = false;\n  var _e = undefined;\n\n  try {\n    for (var _i = _getIterator(arr), _s; !(_n = (_s = _i.next()).done); _n = true) {\n      _arr.push(_s.value);\n\n      if (i && _arr.length === i) break;\n    }\n  } catch (err) {\n    _d = true;\n    _e = err;\n  } finally {\n    try {\n      if (!_n && _i[\"return\"] != null) _i[\"return\"]();\n    } finally {\n      if (_d) throw _e;\n    }\n  }\n\n  return _arr;\n}\n\nmodule.exports = _iterableToArrayLimit;","function _nonIterableRest() {\n  throw new TypeError(\"Invalid attempt to destructure non-iterable instance\");\n}\n\nmodule.exports = _nonIterableRest;","var arrayWithHoles = require(\"./arrayWithHoles\");\n\nvar iterableToArrayLimit = require(\"./iterableToArrayLimit\");\n\nvar nonIterableRest = require(\"./nonIterableRest\");\n\nfunction _slicedToArray(arr, i) {\n  return arrayWithHoles(arr) || iterableToArrayLimit(arr, i) || nonIterableRest();\n}\n\nmodule.exports = _slicedToArray;","var $ = require('../internals/export');\n\n// `Date.now` method\n// https://tc39.github.io/ecma262/#sec-date.now\n$({ target: 'Date', stat: true }, {\n  now: function now() {\n    return new Date().getTime();\n  }\n});\n","require('../../modules/es.date.now');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Date.now;\n","/**\n * vis-util\n * https://github.com/visjs/vis-util\n *\n * utilitie collection for visjs\n *\n * @version 4.0.0\n * @date    2020-02-29T15:35:13.178Z\n *\n * @copyright (c) 2011-2017 Almende B.V, http://almende.com\n * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs\n *\n * @license\n * vis.js is dual licensed under both\n *\n *   1. The Apache 2.0 License\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *   and\n *\n *   2. The MIT License\n *      http://opensource.org/licenses/MIT\n *\n * vis.js may be distributed under either license.\n */\n\n/**\r\n * Seedable, fast and reasonably good (not crypto but more than okay for our\r\n * needs) random number generator.\r\n *\r\n * @remarks\r\n * Adapted from {@link https://web.archive.org/web/20110429100736/http://baagoe.com:80/en/RandomMusings/javascript}.\r\n * Original algorithm created by Johannes Baagøe \\<baagoe\\@baagoe.com\\> in 2010.\r\n */\r\n/**\r\n * Create a seeded pseudo random generator based on Alea by Johannes Baagøe.\r\n *\r\n * @param seed - All supplied arguments will be used as a seed. In case nothing\r\n * is supplied the current time will be used to seed the generator.\r\n *\r\n * @returns A ready to use seeded generator.\r\n */\r\nfunction Alea(...seed) {\r\n    return AleaImplementation(seed.length ? seed : [Date.now()]);\r\n}\r\n/**\r\n * An implementation of [[Alea]] without user input validation.\r\n *\r\n * @param seed - The data that will be used to seed the generator.\r\n *\r\n * @returns A ready to use seeded generator.\r\n */\r\nfunction AleaImplementation(seed) {\r\n    let [s0, s1, s2] = mashSeed(seed);\r\n    let c = 1;\r\n    const random = () => {\r\n        const t = 2091639 * s0 + c * 2.3283064365386963e-10; // 2^-32\r\n        s0 = s1;\r\n        s1 = s2;\r\n        return (s2 = t - (c = t | 0));\r\n    };\r\n    random.uint32 = () => random() * 0x100000000; // 2^32\r\n    random.fract53 = () => random() + ((random() * 0x200000) | 0) * 1.1102230246251565e-16; // 2^-53\r\n    random.algorithm = \"Alea\";\r\n    random.seed = seed;\r\n    random.version = \"0.9\";\r\n    return random;\r\n}\r\n/**\r\n * Turn arbitrary data into values [[AleaImplementation]] can use to generate\r\n * random numbers.\r\n *\r\n * @param seed - Arbitrary data that will be used as the seed.\r\n *\r\n * @returns Three numbers to use as initial values for [[AleaImplementation]].\r\n */\r\nfunction mashSeed(...seed) {\r\n    const mash = Mash();\r\n    let s0 = mash(\" \");\r\n    let s1 = mash(\" \");\r\n    let s2 = mash(\" \");\r\n    for (let i = 0; i < seed.length; i++) {\r\n        s0 -= mash(seed[i]);\r\n        if (s0 < 0) {\r\n            s0 += 1;\r\n        }\r\n        s1 -= mash(seed[i]);\r\n        if (s1 < 0) {\r\n            s1 += 1;\r\n        }\r\n        s2 -= mash(seed[i]);\r\n        if (s2 < 0) {\r\n            s2 += 1;\r\n        }\r\n    }\r\n    return [s0, s1, s2];\r\n}\r\n/**\r\n * Create a new mash function.\r\n *\r\n * @returns A nonpure function that takes arbitrary [[Mashable]] data and turns\r\n * them into numbers.\r\n */\r\nfunction Mash() {\r\n    let n = 0xefc8249d;\r\n    return function (data) {\r\n        const string = data.toString();\r\n        for (let i = 0; i < string.length; i++) {\r\n            n += string.charCodeAt(i);\r\n            let h = 0.02519603282416938 * n;\r\n            n = h >>> 0;\r\n            h -= n;\r\n            h *= n;\r\n            n = h >>> 0;\r\n            h -= n;\r\n            n += h * 0x100000000; // 2^32\r\n        }\r\n        return (n >>> 0) * 2.3283064365386963e-10; // 2^-32\r\n    };\r\n}\n\n// utility functions\r\n// parse ASP.Net Date pattern,\r\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\r\n// code from http://momentjs.com/\r\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\r\n// Color REs\r\nconst fullHexRE = /^#?([a-f\\d]{2})([a-f\\d]{2})([a-f\\d]{2})$/i;\r\nconst shortHexRE = /^#?([a-f\\d])([a-f\\d])([a-f\\d])$/i;\r\nconst rgbRE = /^rgb\\( *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *\\)$/i;\r\nconst rgbaRE = /^rgba\\( *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *([01]|0?\\.\\d+) *\\)$/i;\r\n/**\r\n * Test whether given object is a number.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if number, false otherwise.\r\n */\r\nfunction isNumber(value) {\r\n    return value instanceof Number || typeof value === \"number\";\r\n}\r\n/**\r\n * Remove everything in the DOM object.\r\n *\r\n * @param DOMobject - Node whose child nodes will be recursively deleted.\r\n */\r\nfunction recursiveDOMDelete(DOMobject) {\r\n    if (DOMobject) {\r\n        while (DOMobject.hasChildNodes() === true) {\r\n            const child = DOMobject.firstChild;\r\n            if (child) {\r\n                recursiveDOMDelete(child);\r\n                DOMobject.removeChild(child);\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Test whether given object is a string.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if string, false otherwise.\r\n */\r\nfunction isString(value) {\r\n    return value instanceof String || typeof value === \"string\";\r\n}\r\n/**\r\n * Test whether given object is a object (not primitive or null).\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if not null object, false otherwise.\r\n */\r\nfunction isObject(value) {\r\n    return typeof value === \"object\" && value !== null;\r\n}\r\n/**\r\n * Test whether given object is a Date, or a String containing a Date.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if Date instance or string date representation, false otherwise.\r\n */\r\nfunction isDate(value) {\r\n    if (value instanceof Date) {\r\n        return true;\r\n    }\r\n    else if (isString(value)) {\r\n        // test whether this string contains a date\r\n        const match = ASPDateRegex.exec(value);\r\n        if (match) {\r\n            return true;\r\n        }\r\n        else if (!isNaN(Date.parse(value))) {\r\n            return true;\r\n        }\r\n    }\r\n    return false;\r\n}\r\n/**\r\n * Copy property from b to a if property present in a.\r\n * If property in b explicitly set to null, delete it if `allowDeletion` set.\r\n *\r\n * Internal helper routine, should not be exported. Not added to `exports` for that reason.\r\n *\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param prop - Name of property to copy from b to a.\r\n * @param allowDeletion - If true, delete property in a if explicitly set to null in b.\r\n */\r\nfunction copyOrDelete(a, b, prop, allowDeletion) {\r\n    let doDeletion = false;\r\n    if (allowDeletion === true) {\r\n        doDeletion = b[prop] === null && a[prop] !== undefined;\r\n    }\r\n    if (doDeletion) {\r\n        delete a[prop];\r\n    }\r\n    else {\r\n        a[prop] = b[prop]; // Remember, this is a reference copy!\r\n    }\r\n}\r\n/**\r\n * Fill an object with a possibly partially defined other object.\r\n *\r\n * Only copies values for the properties already present in a.\r\n * That means an object is not created on a property if only the b object has it.\r\n *\r\n * @param a - The object that will have it's properties updated.\r\n * @param b - The object with property updates.\r\n * @param allowDeletion - If true, delete properties in a that are explicitly set to null in b.\r\n */\r\nfunction fillIfDefined(a, b, allowDeletion = false) {\r\n    // NOTE: iteration of properties of a\r\n    // NOTE: prototype properties iterated over as well\r\n    for (const prop in a) {\r\n        if (b[prop] !== undefined) {\r\n            if (b[prop] === null || typeof b[prop] !== \"object\") {\r\n                // Note: typeof null === 'object'\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n            else {\r\n                const aProp = a[prop];\r\n                const bProp = b[prop];\r\n                if (isObject(aProp) && isObject(bProp)) {\r\n                    fillIfDefined(aProp, bProp, allowDeletion);\r\n                }\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Copy the values of all of the enumerable own properties from one or more source objects to a\r\n * target object. Returns the target object.\r\n *\r\n * @param target - The target object to copy to.\r\n * @param source - The source object from which to copy properties.\r\n *\r\n * @returns The target object.\r\n */\r\nconst extend = Object.assign;\r\n/**\r\n * Extend object a with selected properties of object b or a series of objects.\r\n *\r\n * @remarks\r\n * Only properties with defined values are copied.\r\n *\r\n * @param props - Properties to be copied to a.\r\n * @param a - The target.\r\n * @param others - The sources.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveExtend(props, a, ...others) {\r\n    if (!Array.isArray(props)) {\r\n        throw new Error(\"Array with property names expected as first argument\");\r\n    }\r\n    for (const other of others) {\r\n        for (let p = 0; p < props.length; p++) {\r\n            const prop = props[p];\r\n            if (other && Object.prototype.hasOwnProperty.call(other, prop)) {\r\n                a[prop] = other[prop];\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Extend object a with selected properties of object b.\r\n * Only properties with defined values are copied.\r\n *\r\n * @remarks\r\n * Previous version of this routine implied that multiple source objects could\r\n * be used; however, the implementation was **wrong**. Since multiple (\\>1)\r\n * sources weren't used anywhere in the `vis.js` code, this has been removed\r\n *\r\n * @param props - Names of first-level properties to copy over.\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param allowDeletion - If true, delete property in a if explicitly set to null in b.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveDeepExtend(props, a, b, allowDeletion = false) {\r\n    // TODO: add support for Arrays to deepExtend\r\n    if (Array.isArray(b)) {\r\n        throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n    }\r\n    for (let p = 0; p < props.length; p++) {\r\n        const prop = props[p];\r\n        if (Object.prototype.hasOwnProperty.call(b, prop)) {\r\n            if (b[prop] && b[prop].constructor === Object) {\r\n                if (a[prop] === undefined) {\r\n                    a[prop] = {};\r\n                }\r\n                if (a[prop].constructor === Object) {\r\n                    deepExtend(a[prop], b[prop], false, allowDeletion);\r\n                }\r\n                else {\r\n                    copyOrDelete(a, b, prop, allowDeletion);\r\n                }\r\n            }\r\n            else if (Array.isArray(b[prop])) {\r\n                throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Extend object `a` with properties of object `b`, ignoring properties which\r\n * are explicitly specified to be excluded.\r\n *\r\n * @remarks\r\n * The properties of `b` are considered for copying. Properties which are\r\n * themselves objects are are also extended. Only properties with defined\r\n * values are copied.\r\n *\r\n * @param propsToExclude - Names of properties which should *not* be copied.\r\n * @param a - Object to extend.\r\n * @param b - Object to take properties from for extension.\r\n * @param allowDeletion - If true, delete properties in a that are explicitly\r\n * set to null in b.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveNotDeepExtend(propsToExclude, a, b, allowDeletion = false) {\r\n    // TODO: add support for Arrays to deepExtend\r\n    // NOTE: array properties have an else-below; apparently, there is a problem here.\r\n    if (Array.isArray(b)) {\r\n        throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n    }\r\n    for (const prop in b) {\r\n        if (!Object.prototype.hasOwnProperty.call(b, prop)) {\r\n            continue;\r\n        } // Handle local properties only\r\n        if (propsToExclude.indexOf(prop) !== -1) {\r\n            continue;\r\n        } // In exclusion list, skip\r\n        if (b[prop] && b[prop].constructor === Object) {\r\n            if (a[prop] === undefined) {\r\n                a[prop] = {};\r\n            }\r\n            if (a[prop].constructor === Object) {\r\n                deepExtend(a[prop], b[prop]); // NOTE: allowDeletion not propagated!\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n        else if (Array.isArray(b[prop])) {\r\n            a[prop] = [];\r\n            for (let i = 0; i < b[prop].length; i++) {\r\n                a[prop].push(b[prop][i]);\r\n            }\r\n        }\r\n        else {\r\n            copyOrDelete(a, b, prop, allowDeletion);\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Deep extend an object a with the properties of object b.\r\n *\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param protoExtend - If true, the prototype values will also be extended.\r\n * (That is the options objects that inherit from others will also get the\r\n * inherited options).\r\n * @param allowDeletion - If true, the values of fields that are null will be deleted.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction deepExtend(a, b, protoExtend = false, allowDeletion = false) {\r\n    for (const prop in b) {\r\n        if (Object.prototype.hasOwnProperty.call(b, prop) || protoExtend === true) {\r\n            if (typeof b[prop] === \"object\" &&\r\n                b[prop] !== null &&\r\n                Object.getPrototypeOf(b[prop]) === Object.prototype) {\r\n                if (a[prop] === undefined) {\r\n                    a[prop] = deepExtend({}, b[prop], protoExtend); // NOTE: allowDeletion not propagated!\r\n                }\r\n                else if (typeof a[prop] === \"object\" &&\r\n                    a[prop] !== null &&\r\n                    Object.getPrototypeOf(a[prop]) === Object.prototype) {\r\n                    deepExtend(a[prop], b[prop], protoExtend); // NOTE: allowDeletion not propagated!\r\n                }\r\n                else {\r\n                    copyOrDelete(a, b, prop, allowDeletion);\r\n                }\r\n            }\r\n            else if (Array.isArray(b[prop])) {\r\n                a[prop] = b[prop].slice();\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Test whether all elements in two arrays are equal.\r\n *\r\n * @param a - First array.\r\n * @param b - Second array.\r\n *\r\n * @returns True if both arrays have the same length and same elements (1 = '1').\r\n */\r\nfunction equalArray(a, b) {\r\n    if (a.length !== b.length) {\r\n        return false;\r\n    }\r\n    for (let i = 0, len = a.length; i < len; i++) {\r\n        if (a[i] != b[i]) {\r\n            return false;\r\n        }\r\n    }\r\n    return true;\r\n}\r\n/**\r\n * Get the type of an object, for example exports.getType([]) returns 'Array'.\r\n *\r\n * @param object - Input value of unknown type.\r\n *\r\n * @returns Detected type.\r\n */\r\nfunction getType(object) {\r\n    const type = typeof object;\r\n    if (type === \"object\") {\r\n        if (object === null) {\r\n            return \"null\";\r\n        }\r\n        if (object instanceof Boolean) {\r\n            return \"Boolean\";\r\n        }\r\n        if (object instanceof Number) {\r\n            return \"Number\";\r\n        }\r\n        if (object instanceof String) {\r\n            return \"String\";\r\n        }\r\n        if (Array.isArray(object)) {\r\n            return \"Array\";\r\n        }\r\n        if (object instanceof Date) {\r\n            return \"Date\";\r\n        }\r\n        return \"Object\";\r\n    }\r\n    if (type === \"number\") {\r\n        return \"Number\";\r\n    }\r\n    if (type === \"boolean\") {\r\n        return \"Boolean\";\r\n    }\r\n    if (type === \"string\") {\r\n        return \"String\";\r\n    }\r\n    if (type === undefined) {\r\n        return \"undefined\";\r\n    }\r\n    return type;\r\n}\r\n/**\r\n * Used to extend an array and copy it. This is used to propagate paths recursively.\r\n *\r\n * @param arr - First part.\r\n * @param newValue - The value to be aadded into the array.\r\n *\r\n * @returns A new array with all items from arr and newValue (which is last).\r\n */\r\nfunction copyAndExtendArray(arr, newValue) {\r\n    return [...arr, newValue];\r\n}\r\n/**\r\n * Used to extend an array and copy it. This is used to propagate paths recursively.\r\n *\r\n * @param arr - The array to be copied.\r\n *\r\n * @returns Shallow copy of arr.\r\n */\r\nfunction copyArray(arr) {\r\n    return arr.slice();\r\n}\r\n/**\r\n * Retrieve the absolute left value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute left position of this element in the browser page.\r\n */\r\nfunction getAbsoluteLeft(elem) {\r\n    return elem.getBoundingClientRect().left;\r\n}\r\n/**\r\n * Retrieve the absolute right value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute right position of this element in the browser page.\r\n */\r\nfunction getAbsoluteRight(elem) {\r\n    return elem.getBoundingClientRect().right;\r\n}\r\n/**\r\n * Retrieve the absolute top value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute top position of this element in the browser page.\r\n */\r\nfunction getAbsoluteTop(elem) {\r\n    return elem.getBoundingClientRect().top;\r\n}\r\n/**\r\n * Add a className to the given elements style.\r\n *\r\n * @param elem - The element to which the classes will be added.\r\n * @param classNames - Space separated list of classes.\r\n */\r\nfunction addClassName(elem, classNames) {\r\n    let classes = elem.className.split(\" \");\r\n    const newClasses = classNames.split(\" \");\r\n    classes = classes.concat(newClasses.filter(function (className) {\r\n        return classes.indexOf(className) < 0;\r\n    }));\r\n    elem.className = classes.join(\" \");\r\n}\r\n/**\r\n * Remove a className from the given elements style.\r\n *\r\n * @param elem - The element from which the classes will be removed.\r\n * @param classNames - Space separated list of classes.\r\n */\r\nfunction removeClassName(elem, classNames) {\r\n    let classes = elem.className.split(\" \");\r\n    const oldClasses = classNames.split(\" \");\r\n    classes = classes.filter(function (className) {\r\n        return oldClasses.indexOf(className) < 0;\r\n    });\r\n    elem.className = classes.join(\" \");\r\n}\r\n/**\r\n * For each method for both arrays and objects.\r\n * In case of an array, the built-in Array.forEach() is applied (**No, it's not!**).\r\n * In case of an Object, the method loops over all properties of the object.\r\n *\r\n * @param object - An Object or Array to be iterated over.\r\n * @param callback - Array.forEach-like callback.\r\n */\r\nfunction forEach(object, callback) {\r\n    if (Array.isArray(object)) {\r\n        // array\r\n        const len = object.length;\r\n        for (let i = 0; i < len; i++) {\r\n            callback(object[i], i, object);\r\n        }\r\n    }\r\n    else {\r\n        // object\r\n        for (const key in object) {\r\n            if (Object.prototype.hasOwnProperty.call(object, key)) {\r\n                callback(object[key], key, object);\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert an object into an array: all objects properties are put into the array. The resulting array is unordered.\r\n *\r\n * @param o - Object that contains the properties and methods.\r\n *\r\n * @returns An array of unordered values.\r\n */\r\nconst toArray = Object.values;\r\n/**\r\n * Update a property in an object.\r\n *\r\n * @param object - The object whose property will be updated.\r\n * @param key - Name of the property to be updated.\r\n * @param value - The new value to be assigned.\r\n *\r\n * @returns Whether the value was updated (true) or already strictly the same in the original object (false).\r\n */\r\nfunction updateProperty(object, key, value) {\r\n    if (object[key] !== value) {\r\n        object[key] = value;\r\n        return true;\r\n    }\r\n    else {\r\n        return false;\r\n    }\r\n}\r\n/**\r\n * Throttle the given function to be only executed once per animation frame.\r\n *\r\n * @param fn - The original function.\r\n *\r\n * @returns The throttled function.\r\n */\r\nfunction throttle(fn) {\r\n    let scheduled = false;\r\n    return () => {\r\n        if (!scheduled) {\r\n            scheduled = true;\r\n            requestAnimationFrame(() => {\r\n                scheduled = false;\r\n                fn();\r\n            });\r\n        }\r\n    };\r\n}\r\n/**\r\n * Add and event listener. Works for all browsers.\r\n *\r\n * @param element - The element to bind the event listener to.\r\n * @param action - Same as Element.addEventListener(action, —, —).\r\n * @param listener - Same as Element.addEventListener(—, listener, —).\r\n * @param useCapture - Same as Element.addEventListener(—, —, useCapture).\r\n */\r\nfunction addEventListener(element, action, listener, useCapture) {\r\n    if (element.addEventListener) {\r\n        if (useCapture === undefined) {\r\n            useCapture = false;\r\n        }\r\n        if (action === \"mousewheel\" &&\r\n            navigator.userAgent.indexOf(\"Firefox\") >= 0) {\r\n            action = \"DOMMouseScroll\"; // For Firefox\r\n        }\r\n        element.addEventListener(action, listener, useCapture);\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        element.attachEvent(\"on\" + action, listener); // IE browsers\r\n    }\r\n}\r\n/**\r\n * Remove an event listener from an element.\r\n *\r\n * @param element - The element to bind the event listener to.\r\n * @param action - Same as Element.removeEventListener(action, —, —).\r\n * @param listener - Same as Element.removeEventListener(—, listener, —).\r\n * @param useCapture - Same as Element.removeEventListener(—, —, useCapture).\r\n */\r\nfunction removeEventListener(element, action, listener, useCapture) {\r\n    if (element.removeEventListener) {\r\n        // non-IE browsers\r\n        if (useCapture === undefined) {\r\n            useCapture = false;\r\n        }\r\n        if (action === \"mousewheel\" &&\r\n            navigator.userAgent.indexOf(\"Firefox\") >= 0) {\r\n            action = \"DOMMouseScroll\"; // For Firefox\r\n        }\r\n        element.removeEventListener(action, listener, useCapture);\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        element.detachEvent(\"on\" + action, listener); // IE browsers\r\n    }\r\n}\r\n/**\r\n * Cancels the event's default action if it is cancelable, without stopping further propagation of the event.\r\n *\r\n * @param event - The event whose default action should be prevented.\r\n */\r\nfunction preventDefault(event) {\r\n    if (!event) {\r\n        event = window.event;\r\n    }\r\n    if (!event) ;\r\n    else if (event.preventDefault) {\r\n        event.preventDefault(); // non-IE browsers\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        event.returnValue = false; // IE browsers\r\n    }\r\n}\r\n/**\r\n * Get HTML element which is the target of the event.\r\n *\r\n * @param event - The event.\r\n *\r\n * @returns The element or null if not obtainable.\r\n */\r\nfunction getTarget(event = window.event) {\r\n    // code from http://www.quirksmode.org/js/events_properties.html\r\n    // @TODO: EventTarget can be almost anything, is it okay to return only Elements?\r\n    let target = null;\r\n    if (!event) ;\r\n    else if (event.target) {\r\n        target = event.target;\r\n    }\r\n    else if (event.srcElement) {\r\n        target = event.srcElement;\r\n    }\r\n    if (!(target instanceof Element)) {\r\n        return null;\r\n    }\r\n    if (target.nodeType != null && target.nodeType == 3) {\r\n        // defeat Safari bug\r\n        target = target.parentNode;\r\n        if (!(target instanceof Element)) {\r\n            return null;\r\n        }\r\n    }\r\n    return target;\r\n}\r\n/**\r\n * Check if given element contains given parent somewhere in the DOM tree.\r\n *\r\n * @param element - The element to be tested.\r\n * @param parent - The ancestor (not necessarily parent) of the element.\r\n *\r\n * @returns True if parent is an ancestor of the element, false otherwise.\r\n */\r\nfunction hasParent(element, parent) {\r\n    let elem = element;\r\n    while (elem) {\r\n        if (elem === parent) {\r\n            return true;\r\n        }\r\n        else if (elem.parentNode) {\r\n            elem = elem.parentNode;\r\n        }\r\n        else {\r\n            return false;\r\n        }\r\n    }\r\n    return false;\r\n}\r\nconst option = {\r\n    /**\r\n     * Convert a value into a boolean.\r\n     *\r\n     * @param value - Value to be converted intoboolean, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding boolean value, if none then the default value, if none then null.\r\n     */\r\n    asBoolean(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return value != false;\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a number.\r\n     *\r\n     * @param value - Value to be converted intonumber, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding **boxed** number value, if none then the default value, if none then null.\r\n     */\r\n    asNumber(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return Number(value) || defaultValue || null;\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a string.\r\n     *\r\n     * @param value - Value to be converted intostring, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding **boxed** string value, if none then the default value, if none then null.\r\n     */\r\n    asString(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return String(value);\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a size.\r\n     *\r\n     * @param value - Value to be converted intosize, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding string value (number + 'px'), if none then the default value, if none then null.\r\n     */\r\n    asSize(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (isString(value)) {\r\n            return value;\r\n        }\r\n        else if (isNumber(value)) {\r\n            return value + \"px\";\r\n        }\r\n        else {\r\n            return defaultValue || null;\r\n        }\r\n    },\r\n    /**\r\n     * Convert a value into a DOM Element.\r\n     *\r\n     * @param value - Value to be converted into DOM Element, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns The DOM Element, if none then the default value, if none then null.\r\n     */\r\n    asElement(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        return value || defaultValue || null;\r\n    }\r\n};\r\n/**\r\n * Convert hex color string into RGB color object.\r\n *\r\n * @remarks\r\n * {@link http://stackoverflow.com/questions/5623838/rgb-to-hex-and-hex-to-rgb}\r\n *\r\n * @param hex - Hex color string (3 or 6 digits, with or without #).\r\n *\r\n * @returns RGB color object.\r\n */\r\nfunction hexToRGB(hex) {\r\n    let result;\r\n    switch (hex.length) {\r\n        case 3:\r\n        case 4:\r\n            result = shortHexRE.exec(hex);\r\n            return result\r\n                ? {\r\n                    r: parseInt(result[1] + result[1], 16),\r\n                    g: parseInt(result[2] + result[2], 16),\r\n                    b: parseInt(result[3] + result[3], 16)\r\n                }\r\n                : null;\r\n        case 6:\r\n        case 7:\r\n            result = fullHexRE.exec(hex);\r\n            return result\r\n                ? {\r\n                    r: parseInt(result[1], 16),\r\n                    g: parseInt(result[2], 16),\r\n                    b: parseInt(result[3], 16)\r\n                }\r\n                : null;\r\n        default:\r\n            return null;\r\n    }\r\n}\r\n/**\r\n * This function takes string color in hex or RGB format and adds the opacity, RGBA is passed through unchanged.\r\n *\r\n * @param color - The color string (hex, RGB, RGBA).\r\n * @param opacity - The new opacity.\r\n *\r\n * @returns RGBA string, for example 'rgba(255, 0, 127, 0.3)'.\r\n */\r\nfunction overrideOpacity(color, opacity) {\r\n    if (color.indexOf(\"rgba\") !== -1) {\r\n        return color;\r\n    }\r\n    else if (color.indexOf(\"rgb\") !== -1) {\r\n        const rgb = color\r\n            .substr(color.indexOf(\"(\") + 1)\r\n            .replace(\")\", \"\")\r\n            .split(\",\");\r\n        return \"rgba(\" + rgb[0] + \",\" + rgb[1] + \",\" + rgb[2] + \",\" + opacity + \")\";\r\n    }\r\n    else {\r\n        const rgb = hexToRGB(color);\r\n        if (rgb == null) {\r\n            return color;\r\n        }\r\n        else {\r\n            return \"rgba(\" + rgb.r + \",\" + rgb.g + \",\" + rgb.b + \",\" + opacity + \")\";\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert RGB \\<0, 255\\> into hex color string.\r\n *\r\n * @param red - Red channel.\r\n * @param green - Green channel.\r\n * @param blue - Blue channel.\r\n *\r\n * @returns Hex color string (for example: '#0acdc0').\r\n */\r\nfunction RGBToHex(red, green, blue) {\r\n    return (\"#\" + ((1 << 24) + (red << 16) + (green << 8) + blue).toString(16).slice(1));\r\n}\r\n/**\r\n * Parse a color property into an object with border, background, and highlight colors.\r\n *\r\n * @param inputColor - Shorthand color string or input color object.\r\n * @param defaultColor - Full color object to fill in missing values in inputColor.\r\n *\r\n * @returns Color object.\r\n */\r\nfunction parseColor(inputColor, defaultColor) {\r\n    if (isString(inputColor)) {\r\n        let colorStr = inputColor;\r\n        if (isValidRGB(colorStr)) {\r\n            const rgb = colorStr\r\n                .substr(4)\r\n                .substr(0, colorStr.length - 5)\r\n                .split(\",\")\r\n                .map(function (value) {\r\n                return parseInt(value);\r\n            });\r\n            colorStr = RGBToHex(rgb[0], rgb[1], rgb[2]);\r\n        }\r\n        if (isValidHex(colorStr) === true) {\r\n            const hsv = hexToHSV(colorStr);\r\n            const lighterColorHSV = {\r\n                h: hsv.h,\r\n                s: hsv.s * 0.8,\r\n                v: Math.min(1, hsv.v * 1.02)\r\n            };\r\n            const darkerColorHSV = {\r\n                h: hsv.h,\r\n                s: Math.min(1, hsv.s * 1.25),\r\n                v: hsv.v * 0.8\r\n            };\r\n            const darkerColorHex = HSVToHex(darkerColorHSV.h, darkerColorHSV.s, darkerColorHSV.v);\r\n            const lighterColorHex = HSVToHex(lighterColorHSV.h, lighterColorHSV.s, lighterColorHSV.v);\r\n            return {\r\n                background: colorStr,\r\n                border: darkerColorHex,\r\n                highlight: {\r\n                    background: lighterColorHex,\r\n                    border: darkerColorHex\r\n                },\r\n                hover: {\r\n                    background: lighterColorHex,\r\n                    border: darkerColorHex\r\n                }\r\n            };\r\n        }\r\n        else {\r\n            return {\r\n                background: colorStr,\r\n                border: colorStr,\r\n                highlight: {\r\n                    background: colorStr,\r\n                    border: colorStr\r\n                },\r\n                hover: {\r\n                    background: colorStr,\r\n                    border: colorStr\r\n                }\r\n            };\r\n        }\r\n    }\r\n    else {\r\n        if (defaultColor) {\r\n            const color = {\r\n                background: inputColor.background || defaultColor.background,\r\n                border: inputColor.border || defaultColor.border,\r\n                highlight: isString(inputColor.highlight)\r\n                    ? {\r\n                        border: inputColor.highlight,\r\n                        background: inputColor.highlight\r\n                    }\r\n                    : {\r\n                        background: (inputColor.highlight && inputColor.highlight.background) ||\r\n                            defaultColor.highlight.background,\r\n                        border: (inputColor.highlight && inputColor.highlight.border) ||\r\n                            defaultColor.highlight.border\r\n                    },\r\n                hover: isString(inputColor.hover)\r\n                    ? {\r\n                        border: inputColor.hover,\r\n                        background: inputColor.hover\r\n                    }\r\n                    : {\r\n                        border: (inputColor.hover && inputColor.hover.border) ||\r\n                            defaultColor.hover.border,\r\n                        background: (inputColor.hover && inputColor.hover.background) ||\r\n                            defaultColor.hover.background\r\n                    }\r\n            };\r\n            return color;\r\n        }\r\n        else {\r\n            const color = {\r\n                background: inputColor.background || undefined,\r\n                border: inputColor.border || undefined,\r\n                highlight: isString(inputColor.highlight)\r\n                    ? {\r\n                        border: inputColor.highlight,\r\n                        background: inputColor.highlight\r\n                    }\r\n                    : {\r\n                        background: (inputColor.highlight && inputColor.highlight.background) ||\r\n                            undefined,\r\n                        border: (inputColor.highlight && inputColor.highlight.border) ||\r\n                            undefined\r\n                    },\r\n                hover: isString(inputColor.hover)\r\n                    ? {\r\n                        border: inputColor.hover,\r\n                        background: inputColor.hover\r\n                    }\r\n                    : {\r\n                        border: (inputColor.hover && inputColor.hover.border) || undefined,\r\n                        background: (inputColor.hover && inputColor.hover.background) || undefined\r\n                    }\r\n            };\r\n            return color;\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert RGB \\<0, 255\\> into HSV object.\r\n *\r\n * @remarks\r\n * {@link http://www.javascripter.net/faq/rgb2hsv.htm}\r\n *\r\n * @param red - Red channel.\r\n * @param green - Green channel.\r\n * @param blue - Blue channel.\r\n *\r\n * @returns HSV color object.\r\n */\r\nfunction RGBToHSV(red, green, blue) {\r\n    red = red / 255;\r\n    green = green / 255;\r\n    blue = blue / 255;\r\n    const minRGB = Math.min(red, Math.min(green, blue));\r\n    const maxRGB = Math.max(red, Math.max(green, blue));\r\n    // Black-gray-white\r\n    if (minRGB === maxRGB) {\r\n        return { h: 0, s: 0, v: minRGB };\r\n    }\r\n    // Colors other than black-gray-white:\r\n    const d = red === minRGB ? green - blue : blue === minRGB ? red - green : blue - red;\r\n    const h = red === minRGB ? 3 : blue === minRGB ? 1 : 5;\r\n    const hue = (60 * (h - d / (maxRGB - minRGB))) / 360;\r\n    const saturation = (maxRGB - minRGB) / maxRGB;\r\n    const value = maxRGB;\r\n    return { h: hue, s: saturation, v: value };\r\n}\r\nconst cssUtil = {\r\n    // split a string with css styles into an object with key/values\r\n    split(cssText) {\r\n        const styles = {};\r\n        cssText.split(\";\").forEach((style) => {\r\n            if (style.trim() != \"\") {\r\n                const parts = style.split(\":\");\r\n                const key = parts[0].trim();\r\n                const value = parts[1].trim();\r\n                styles[key] = value;\r\n            }\r\n        });\r\n        return styles;\r\n    },\r\n    // build a css text string from an object with key/values\r\n    join(styles) {\r\n        return Object.keys(styles)\r\n            .map(function (key) {\r\n            return key + \": \" + styles[key];\r\n        })\r\n            .join(\"; \");\r\n    }\r\n};\r\n/**\r\n * Append a string with css styles to an element.\r\n *\r\n * @param element - The element that will receive new styles.\r\n * @param cssText - The styles to be appended.\r\n */\r\nfunction addCssText(element, cssText) {\r\n    const currentStyles = cssUtil.split(element.style.cssText);\r\n    const newStyles = cssUtil.split(cssText);\r\n    const styles = {\r\n        ...currentStyles,\r\n        ...newStyles\r\n    };\r\n    element.style.cssText = cssUtil.join(styles);\r\n}\r\n/**\r\n * Remove a string with css styles from an element.\r\n *\r\n * @param element - The element from which styles should be removed.\r\n * @param cssText - The styles to be removed.\r\n */\r\nfunction removeCssText(element, cssText) {\r\n    const styles = cssUtil.split(element.style.cssText);\r\n    const removeStyles = cssUtil.split(cssText);\r\n    for (const key in removeStyles) {\r\n        if (Object.prototype.hasOwnProperty.call(removeStyles, key)) {\r\n            delete styles[key];\r\n        }\r\n    }\r\n    element.style.cssText = cssUtil.join(styles);\r\n}\r\n/**\r\n * Convert HSV \\<0, 1\\> into RGB color object.\r\n *\r\n * @remarks\r\n * {@link https://gist.github.com/mjijackson/5311256}\r\n *\r\n * @param h - Hue.\r\n * @param s - Saturation.\r\n * @param v - Value.\r\n *\r\n * @returns RGB color object.\r\n */\r\nfunction HSVToRGB(h, s, v) {\r\n    let r;\r\n    let g;\r\n    let b;\r\n    const i = Math.floor(h * 6);\r\n    const f = h * 6 - i;\r\n    const p = v * (1 - s);\r\n    const q = v * (1 - f * s);\r\n    const t = v * (1 - (1 - f) * s);\r\n    switch (i % 6) {\r\n        case 0:\r\n            (r = v), (g = t), (b = p);\r\n            break;\r\n        case 1:\r\n            (r = q), (g = v), (b = p);\r\n            break;\r\n        case 2:\r\n            (r = p), (g = v), (b = t);\r\n            break;\r\n        case 3:\r\n            (r = p), (g = q), (b = v);\r\n            break;\r\n        case 4:\r\n            (r = t), (g = p), (b = v);\r\n            break;\r\n        case 5:\r\n            (r = v), (g = p), (b = q);\r\n            break;\r\n    }\r\n    return {\r\n        r: Math.floor(r * 255),\r\n        g: Math.floor(g * 255),\r\n        b: Math.floor(b * 255)\r\n    };\r\n}\r\n/**\r\n * Convert HSV \\<0, 1\\> into hex color string.\r\n *\r\n * @param h - Hue.\r\n * @param s - Saturation.\r\n * @param v - Value.\r\n *\r\n * @returns Hex color string.\r\n */\r\nfunction HSVToHex(h, s, v) {\r\n    const rgb = HSVToRGB(h, s, v);\r\n    return RGBToHex(rgb.r, rgb.g, rgb.b);\r\n}\r\n/**\r\n * Convert hex color string into HSV \\<0, 1\\>.\r\n *\r\n * @param hex - Hex color string.\r\n *\r\n * @returns HSV color object.\r\n */\r\nfunction hexToHSV(hex) {\r\n    const rgb = hexToRGB(hex);\r\n    if (!rgb) {\r\n        throw new TypeError(`'${hex}' is not a valid color.`);\r\n    }\r\n    return RGBToHSV(rgb.r, rgb.g, rgb.b);\r\n}\r\n/**\r\n * Validate hex color string.\r\n *\r\n * @param hex - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidHex(hex) {\r\n    const isOk = /(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(hex);\r\n    return isOk;\r\n}\r\n/**\r\n * Validate RGB color string.\r\n *\r\n * @param rgb - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidRGB(rgb) {\r\n    return rgbRE.test(rgb);\r\n}\r\n/**\r\n * Validate RGBA color string.\r\n *\r\n * @param rgba - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidRGBA(rgba) {\r\n    return rgbaRE.test(rgba);\r\n}\r\n/**\r\n * This recursively redirects the prototype of JSON objects to the referenceObject.\r\n * This is used for default options.\r\n *\r\n * @param fields - Names of properties to be bridged.\r\n * @param referenceObject - The original object.\r\n *\r\n * @returns A new object inheriting from the referenceObject.\r\n */\r\nfunction selectiveBridgeObject(fields, referenceObject) {\r\n    if (referenceObject !== null && typeof referenceObject === \"object\") {\r\n        // !!! typeof null === 'object'\r\n        const objectTo = Object.create(referenceObject);\r\n        for (let i = 0; i < fields.length; i++) {\r\n            if (Object.prototype.hasOwnProperty.call(referenceObject, fields[i])) {\r\n                if (typeof referenceObject[fields[i]] == \"object\") {\r\n                    objectTo[fields[i]] = bridgeObject(referenceObject[fields[i]]);\r\n                }\r\n            }\r\n        }\r\n        return objectTo;\r\n    }\r\n    else {\r\n        return null;\r\n    }\r\n}\r\n/**\r\n * This recursively redirects the prototype of JSON objects to the referenceObject.\r\n * This is used for default options.\r\n *\r\n * @param referenceObject - The original object.\r\n *\r\n * @returns The Element if the referenceObject is an Element, or a new object inheriting from the referenceObject.\r\n */\r\nfunction bridgeObject(referenceObject) {\r\n    if (referenceObject === null || typeof referenceObject !== \"object\") {\r\n        return null;\r\n    }\r\n    if (referenceObject instanceof Element) {\r\n        // Avoid bridging DOM objects\r\n        return referenceObject;\r\n    }\r\n    const objectTo = Object.create(referenceObject);\r\n    for (const i in referenceObject) {\r\n        if (Object.prototype.hasOwnProperty.call(referenceObject, i)) {\r\n            if (typeof referenceObject[i] == \"object\") {\r\n                objectTo[i] = bridgeObject(referenceObject[i]);\r\n            }\r\n        }\r\n    }\r\n    return objectTo;\r\n}\r\n/**\r\n * This method provides a stable sort implementation, very fast for presorted data.\r\n *\r\n * @param a - The array to be sorted (in-place).\r\n * @param compare - An order comparator.\r\n *\r\n * @returns The argument a.\r\n */\r\nfunction insertSort(a, compare) {\r\n    for (let i = 0; i < a.length; i++) {\r\n        const k = a[i];\r\n        let j;\r\n        for (j = i; j > 0 && compare(k, a[j - 1]) < 0; j--) {\r\n            a[j] = a[j - 1];\r\n        }\r\n        a[j] = k;\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * This is used to set the options of subobjects in the options object.\r\n *\r\n * A requirement of these subobjects is that they have an 'enabled' element\r\n * which is optional for the user but mandatory for the program.\r\n *\r\n * The added value here of the merge is that option 'enabled' is set as required.\r\n *\r\n * @param mergeTarget - Either this.options or the options used for the groups.\r\n * @param options - Options.\r\n * @param option - Option key in the options argument.\r\n * @param globalOptions - Global options, passed in to determine value of option 'enabled'.\r\n */\r\nfunction mergeOptions(mergeTarget, options, option, globalOptions = {}) {\r\n    // Local helpers\r\n    const isPresent = function (obj) {\r\n        return obj !== null && obj !== undefined;\r\n    };\r\n    const isObject = function (obj) {\r\n        return obj !== null && typeof obj === \"object\";\r\n    };\r\n    // https://stackoverflow.com/a/34491287/1223531\r\n    const isEmpty = function (obj) {\r\n        for (const x in obj) {\r\n            if (Object.prototype.hasOwnProperty.call(obj, x)) {\r\n                return false;\r\n            }\r\n        }\r\n        return true;\r\n    };\r\n    // Guards\r\n    if (!isObject(mergeTarget)) {\r\n        throw new Error(\"Parameter mergeTarget must be an object\");\r\n    }\r\n    if (!isObject(options)) {\r\n        throw new Error(\"Parameter options must be an object\");\r\n    }\r\n    if (!isPresent(option)) {\r\n        throw new Error(\"Parameter option must have a value\");\r\n    }\r\n    if (!isObject(globalOptions)) {\r\n        throw new Error(\"Parameter globalOptions must be an object\");\r\n    }\r\n    //\r\n    // Actual merge routine, separated from main logic\r\n    // Only a single level of options is merged. Deeper levels are ref'd. This may actually be an issue.\r\n    //\r\n    const doMerge = function (target, options, option) {\r\n        if (!isObject(target[option])) {\r\n            target[option] = {};\r\n        }\r\n        const src = options[option];\r\n        const dst = target[option];\r\n        for (const prop in src) {\r\n            if (Object.prototype.hasOwnProperty.call(src, prop)) {\r\n                dst[prop] = src[prop];\r\n            }\r\n        }\r\n    };\r\n    // Local initialization\r\n    const srcOption = options[option];\r\n    const globalPassed = isObject(globalOptions) && !isEmpty(globalOptions);\r\n    const globalOption = globalPassed ? globalOptions[option] : undefined;\r\n    const globalEnabled = globalOption ? globalOption.enabled : undefined;\r\n    /////////////////////////////////////////\r\n    // Main routine\r\n    /////////////////////////////////////////\r\n    if (srcOption === undefined) {\r\n        return; // Nothing to do\r\n    }\r\n    if (typeof srcOption === \"boolean\") {\r\n        if (!isObject(mergeTarget[option])) {\r\n            mergeTarget[option] = {};\r\n        }\r\n        mergeTarget[option].enabled = srcOption;\r\n        return;\r\n    }\r\n    if (srcOption === null && !isObject(mergeTarget[option])) {\r\n        // If possible, explicit copy from globals\r\n        if (isPresent(globalOption)) {\r\n            mergeTarget[option] = Object.create(globalOption);\r\n        }\r\n        else {\r\n            return; // Nothing to do\r\n        }\r\n    }\r\n    if (!isObject(srcOption)) {\r\n        return;\r\n    }\r\n    //\r\n    // Ensure that 'enabled' is properly set. It is required internally\r\n    // Note that the value from options will always overwrite the existing value\r\n    //\r\n    let enabled = true; // default value\r\n    if (srcOption.enabled !== undefined) {\r\n        enabled = srcOption.enabled;\r\n    }\r\n    else {\r\n        // Take from globals, if present\r\n        if (globalEnabled !== undefined) {\r\n            enabled = globalOption.enabled;\r\n        }\r\n    }\r\n    doMerge(mergeTarget, options, option);\r\n    mergeTarget[option].enabled = enabled;\r\n}\r\n/**\r\n * This function does a binary search for a visible item in a sorted list. If we find a visible item, the code that uses\r\n * this function will then iterate in both directions over this sorted list to find all visible items.\r\n *\r\n * @param orderedItems - Items ordered by start.\r\n * @param comparator - -1 is lower, 0 is equal, 1 is higher.\r\n * @param field - Property name on an item (That is item[field]).\r\n * @param field2 - Second property name on an item (That is item[field][field2]).\r\n *\r\n * @returns Index of the found item or -1 if nothing was found.\r\n */\r\nfunction binarySearchCustom(orderedItems, comparator, field, field2) {\r\n    const maxIterations = 10000;\r\n    let iteration = 0;\r\n    let low = 0;\r\n    let high = orderedItems.length - 1;\r\n    while (low <= high && iteration < maxIterations) {\r\n        const middle = Math.floor((low + high) / 2);\r\n        const item = orderedItems[middle];\r\n        const value = field2 === undefined ? item[field] : item[field][field2];\r\n        const searchResult = comparator(value);\r\n        if (searchResult == 0) {\r\n            // jihaa, found a visible item!\r\n            return middle;\r\n        }\r\n        else if (searchResult == -1) {\r\n            // it is too small --> increase low\r\n            low = middle + 1;\r\n        }\r\n        else {\r\n            // it is too big --> decrease high\r\n            high = middle - 1;\r\n        }\r\n        iteration++;\r\n    }\r\n    return -1;\r\n}\r\n/**\r\n * This function does a binary search for a specific value in a sorted array.\r\n * If it does not exist but is in between of two values, we return either the\r\n * one before or the one after, depending on user input If it is found, we\r\n * return the index, else -1.\r\n *\r\n * @param orderedItems - Sorted array.\r\n * @param target - The searched value.\r\n * @param field - Name of the property in items to be searched.\r\n * @param sidePreference - If the target is between two values, should the index of the before or the after be returned?\r\n * @param comparator - An optional comparator, returning -1, 0, 1 for \\<, ===, \\>.\r\n *\r\n * @returns The index of found value or -1 if nothing was found.\r\n */\r\nfunction binarySearchValue(orderedItems, target, field, sidePreference, comparator) {\r\n    const maxIterations = 10000;\r\n    let iteration = 0;\r\n    let low = 0;\r\n    let high = orderedItems.length - 1;\r\n    let prevValue;\r\n    let value;\r\n    let nextValue;\r\n    let middle;\r\n    comparator =\r\n        comparator != undefined\r\n            ? comparator\r\n            : function (a, b) {\r\n                return a == b ? 0 : a < b ? -1 : 1;\r\n            };\r\n    while (low <= high && iteration < maxIterations) {\r\n        // get a new guess\r\n        middle = Math.floor(0.5 * (high + low));\r\n        prevValue = orderedItems[Math.max(0, middle - 1)][field];\r\n        value = orderedItems[middle][field];\r\n        nextValue =\r\n            orderedItems[Math.min(orderedItems.length - 1, middle + 1)][field];\r\n        if (comparator(value, target) == 0) {\r\n            // we found the target\r\n            return middle;\r\n        }\r\n        else if (comparator(prevValue, target) < 0 &&\r\n            comparator(value, target) > 0) {\r\n            // target is in between of the previous and the current\r\n            return sidePreference == \"before\" ? Math.max(0, middle - 1) : middle;\r\n        }\r\n        else if (comparator(value, target) < 0 &&\r\n            comparator(nextValue, target) > 0) {\r\n            // target is in between of the current and the next\r\n            return sidePreference == \"before\"\r\n                ? middle\r\n                : Math.min(orderedItems.length - 1, middle + 1);\r\n        }\r\n        else {\r\n            // didnt find the target, we need to change our boundaries.\r\n            if (comparator(value, target) < 0) {\r\n                // it is too small --> increase low\r\n                low = middle + 1;\r\n            }\r\n            else {\r\n                // it is too big --> decrease high\r\n                high = middle - 1;\r\n            }\r\n        }\r\n        iteration++;\r\n    }\r\n    // didnt find anything. Return -1.\r\n    return -1;\r\n}\r\n/*\r\n * Easing Functions.\r\n * Only considering the t value for the range [0, 1] => [0, 1].\r\n *\r\n * Inspiration: from http://gizma.com/easing/\r\n * https://gist.github.com/gre/1650294\r\n */\r\nconst easingFunctions = {\r\n    /**\r\n     * Provides no easing and no acceleration.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    linear(t) {\r\n        return t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuad(t) {\r\n        return t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuad(t) {\r\n        return t * (2 - t);\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuad(t) {\r\n        return t < 0.5 ? 2 * t * t : -1 + (4 - 2 * t) * t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInCubic(t) {\r\n        return t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutCubic(t) {\r\n        return --t * t * t + 1;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutCubic(t) {\r\n        return t < 0.5 ? 4 * t * t * t : (t - 1) * (2 * t - 2) * (2 * t - 2) + 1;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuart(t) {\r\n        return t * t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuart(t) {\r\n        return 1 - --t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuart(t) {\r\n        return t < 0.5 ? 8 * t * t * t * t : 1 - 8 * --t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuint(t) {\r\n        return t * t * t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuint(t) {\r\n        return 1 + --t * t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuint(t) {\r\n        return t < 0.5 ? 16 * t * t * t * t * t : 1 + 16 * --t * t * t * t * t;\r\n    }\r\n};\r\n/**\r\n * Experimentaly compute the width of the scrollbar for this browser.\r\n *\r\n * @returns The width in pixels.\r\n */\r\nfunction getScrollBarWidth() {\r\n    const inner = document.createElement(\"p\");\r\n    inner.style.width = \"100%\";\r\n    inner.style.height = \"200px\";\r\n    const outer = document.createElement(\"div\");\r\n    outer.style.position = \"absolute\";\r\n    outer.style.top = \"0px\";\r\n    outer.style.left = \"0px\";\r\n    outer.style.visibility = \"hidden\";\r\n    outer.style.width = \"200px\";\r\n    outer.style.height = \"150px\";\r\n    outer.style.overflow = \"hidden\";\r\n    outer.appendChild(inner);\r\n    document.body.appendChild(outer);\r\n    const w1 = inner.offsetWidth;\r\n    outer.style.overflow = \"scroll\";\r\n    let w2 = inner.offsetWidth;\r\n    if (w1 == w2) {\r\n        w2 = outer.clientWidth;\r\n    }\r\n    document.body.removeChild(outer);\r\n    return w1 - w2;\r\n}\r\n// @TODO: This doesn't work properly.\r\n// It works only for single property objects,\r\n// otherwise it combines all of the types in a union.\r\n// export function topMost<K1 extends string, V1> (\r\n//   pile: Record<K1, undefined | V1>[],\r\n//   accessors: K1 | [K1]\r\n// ): undefined | V1\r\n// export function topMost<K1 extends string, K2 extends string, V1, V2> (\r\n//   pile: Record<K1, undefined | V1 | Record<K2, undefined | V2>>[],\r\n//   accessors: [K1, K2]\r\n// ): undefined | V1 | V2\r\n// export function topMost<K1 extends string, K2 extends string, K3 extends string, V1, V2, V3> (\r\n//   pile: Record<K1, undefined | V1 | Record<K2, undefined | V2 | Record<K3, undefined | V3>>>[],\r\n//   accessors: [K1, K2, K3]\r\n// ): undefined | V1 | V2 | V3\r\n/**\r\n * Get the top most property value from a pile of objects.\r\n *\r\n * @param pile - Array of objects, no required format.\r\n * @param accessors - Array of property names.\r\n * For example `object['foo']['bar']` → `['foo', 'bar']`.\r\n *\r\n * @returns Value of the property with given accessors path from the first pile item where it's not undefined.\r\n */\r\nfunction topMost(pile, accessors) {\r\n    let candidate;\r\n    if (!Array.isArray(accessors)) {\r\n        accessors = [accessors];\r\n    }\r\n    for (const member of pile) {\r\n        if (member) {\r\n            candidate = member[accessors[0]];\r\n            for (let i = 1; i < accessors.length; i++) {\r\n                if (candidate) {\r\n                    candidate = candidate[accessors[i]];\r\n                }\r\n            }\r\n            if (typeof candidate !== \"undefined\") {\r\n                break;\r\n            }\r\n        }\r\n    }\r\n    return candidate;\r\n}\n\nexport { Alea, HSVToHex, HSVToRGB, RGBToHSV, RGBToHex, addClassName, addCssText, addEventListener, binarySearchCustom, binarySearchValue, bridgeObject, copyAndExtendArray, copyArray, deepExtend, easingFunctions, equalArray, extend, fillIfDefined, forEach, getAbsoluteLeft, getAbsoluteRight, getAbsoluteTop, getScrollBarWidth, getTarget, getType, hasParent, hexToHSV, hexToRGB, insertSort, isDate, isNumber, isObject, isString, isValidHex, isValidRGB, isValidRGBA, mergeOptions, option, overrideOpacity, parseColor, preventDefault, recursiveDOMDelete, removeClassName, removeCssText, removeEventListener, selectiveBridgeObject, selectiveDeepExtend, selectiveExtend, selectiveNotDeepExtend, throttle, toArray, topMost, updateProperty };\n//# sourceMappingURL=vis-util.js.map\n","// DOM utility methods\n\n/**\n * this prepares the JSON container for allocating SVG elements\n * @param {Object} JSONcontainer\n * @private\n */\nexport function prepareElements(JSONcontainer) {\n  // cleanup the redundant svgElements;\n  for (var elementType in JSONcontainer) {\n    if (JSONcontainer.hasOwnProperty(elementType)) {\n      JSONcontainer[elementType].redundant = JSONcontainer[elementType].used;\n      JSONcontainer[elementType].used = [];\n    }\n  }\n}\n\n/**\n * this cleans up all the unused SVG elements. By asking for the parentNode, we only need to supply the JSON container from\n * which to remove the redundant elements.\n *\n * @param {Object} JSONcontainer\n * @private\n */\nexport function cleanupElements(JSONcontainer) {\n  // cleanup the redundant svgElements;\n  for (var elementType in JSONcontainer) {\n    if (JSONcontainer.hasOwnProperty(elementType)) {\n      if (JSONcontainer[elementType].redundant) {\n        for (var i = 0; i < JSONcontainer[elementType].redundant.length; i++) {\n          JSONcontainer[elementType].redundant[i].parentNode.removeChild(JSONcontainer[elementType].redundant[i]);\n        }\n        JSONcontainer[elementType].redundant = [];\n      }\n    }\n  }\n}\n\n/**\n * Ensures that all elements are removed first up so they can be recreated cleanly\n * @param {Object} JSONcontainer\n */\nexport function resetElements(JSONcontainer) {\n  prepareElements(JSONcontainer);\n  cleanupElements(JSONcontainer);\n  prepareElements(JSONcontainer);\n}\n\n/**\n * Allocate or generate an SVG element if needed. Store a reference to it in the JSON container and draw it in the svgContainer\n * the JSON container and the SVG container have to be supplied so other svg containers (like the legend) can use this.\n *\n * @param {string} elementType\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @returns {Element}\n * @private\n */\nexport function getSVGElement(elementType, JSONcontainer, svgContainer) {\n  var element;\n  // allocate SVG element, if it doesnt yet exist, create one.\n  if (JSONcontainer.hasOwnProperty(elementType)) { // this element has been created before\n    // check if there is an redundant element\n    if (JSONcontainer[elementType].redundant.length > 0) {\n      element = JSONcontainer[elementType].redundant[0];\n      JSONcontainer[elementType].redundant.shift();\n    }\n    else {\n      // create a new element and add it to the SVG\n      element = document.createElementNS('http://www.w3.org/2000/svg', elementType);\n      svgContainer.appendChild(element);\n    }\n  }\n  else {\n    // create a new element and add it to the SVG, also create a new object in the svgElements to keep track of it.\n    element = document.createElementNS('http://www.w3.org/2000/svg', elementType);\n    JSONcontainer[elementType] = {used: [], redundant: []};\n    svgContainer.appendChild(element);\n  }\n  JSONcontainer[elementType].used.push(element);\n  return element;\n}\n\n\n/**\n * Allocate or generate an SVG element if needed. Store a reference to it in the JSON container and draw it in the svgContainer\n * the JSON container and the SVG container have to be supplied so other svg containers (like the legend) can use this.\n *\n * @param {string} elementType\n * @param {Object} JSONcontainer\n * @param {Element} DOMContainer\n * @param {Element} insertBefore\n * @returns {*}\n */\nexport function getDOMElement(elementType, JSONcontainer, DOMContainer, insertBefore) {\n  var element;\n  // allocate DOM element, if it doesnt yet exist, create one.\n  if (JSONcontainer.hasOwnProperty(elementType)) { // this element has been created before\n    // check if there is an redundant element\n    if (JSONcontainer[elementType].redundant.length > 0) {\n      element = JSONcontainer[elementType].redundant[0];\n      JSONcontainer[elementType].redundant.shift();\n    }\n    else {\n      // create a new element and add it to the SVG\n      element = document.createElement(elementType);\n      if (insertBefore !== undefined) {\n        DOMContainer.insertBefore(element, insertBefore);\n      }\n      else {\n        DOMContainer.appendChild(element);\n      }\n    }\n  }\n  else {\n    // create a new element and add it to the SVG, also create a new object in the svgElements to keep track of it.\n    element = document.createElement(elementType);\n    JSONcontainer[elementType] = {used: [], redundant: []};\n    if (insertBefore !== undefined) {\n      DOMContainer.insertBefore(element, insertBefore);\n    }\n    else {\n      DOMContainer.appendChild(element);\n    }\n  }\n  JSONcontainer[elementType].used.push(element);\n  return element;\n}\n\n\n\n\n/**\n * Draw a point object. This is a separate function because it can also be called by the legend.\n * The reason the JSONcontainer and the target SVG svgContainer have to be supplied is so the legend can use these functions\n * as well.\n *\n * @param {number} x\n * @param {number} y\n * @param {Object} groupTemplate: A template containing the necessary information to draw the datapoint e.g., {style: 'circle', size: 5, className: 'className' }\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @param {Object} labelObj\n * @returns {vis.PointItem}\n */\nexport function drawPoint(x, y, groupTemplate, JSONcontainer, svgContainer, labelObj) {\n  var point;\n  if (groupTemplate.style == 'circle') {\n    point = getSVGElement('circle', JSONcontainer, svgContainer);\n    point.setAttributeNS(null, \"cx\", x);\n    point.setAttributeNS(null, \"cy\", y);\n    point.setAttributeNS(null, \"r\", 0.5 * groupTemplate.size);\n  }\n  else {\n    point = getSVGElement('rect', JSONcontainer, svgContainer);\n    point.setAttributeNS(null, \"x\", x - 0.5 * groupTemplate.size);\n    point.setAttributeNS(null, \"y\", y - 0.5 * groupTemplate.size);\n    point.setAttributeNS(null, \"width\", groupTemplate.size);\n    point.setAttributeNS(null, \"height\", groupTemplate.size);\n  }\n\n  if (groupTemplate.styles !== undefined) {\n    point.setAttributeNS(null, \"style\", groupTemplate.styles);\n  }\n  point.setAttributeNS(null, \"class\", groupTemplate.className + \" vis-point\");\n  //handle label\n\n\n  if (labelObj) {\n    var label = getSVGElement('text', JSONcontainer, svgContainer);\n    if (labelObj.xOffset) {\n      x = x + labelObj.xOffset;\n    }\n\n    if (labelObj.yOffset) {\n      y = y + labelObj.yOffset;\n    }\n    if (labelObj.content) {\n      label.textContent = labelObj.content;\n    }\n\n    if (labelObj.className) {\n      label.setAttributeNS(null, \"class\", labelObj.className  + \" vis-label\");\n    }\n    label.setAttributeNS(null, \"x\", x);\n    label.setAttributeNS(null, \"y\", y);\n  }\n\n  return point;\n}\n\n/**\n * draw a bar SVG element centered on the X coordinate\n *\n * @param {number} x\n * @param {number} y\n * @param {number} width\n * @param {number} height\n * @param {string} className\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @param {string} style\n */\nexport function drawBar (x, y, width, height, className, JSONcontainer, svgContainer, style) {\n  if (height != 0) {\n    if (height < 0) {\n      height *= -1;\n      y -= height;\n    }\n    var rect = getSVGElement('rect',JSONcontainer, svgContainer);\n    rect.setAttributeNS(null, \"x\", x - 0.5 * width);\n    rect.setAttributeNS(null, \"y\", y);\n    rect.setAttributeNS(null, \"width\", width);\n    rect.setAttributeNS(null, \"height\", height);\n    rect.setAttributeNS(null, \"class\", className);\n    if (style) {\n      rect.setAttributeNS(null, \"style\", style);\n    }\n  }\n}\n\n/**\n * get default language\n * @returns {string}\n */\nexport function getNavigatorLanguage() {\n  try {\n    if (!navigator) return 'en';\n    if (navigator.languages && navigator.languages.length) {\n      return navigator.languages;\n    } else {\n      return navigator.userLanguage || navigator.language || navigator.browserLanguage || 'en';\n    }\n  } \n  catch(error) {\n    return 'en';\n  }\n}\n","function _assertThisInitialized(self) {\n  if (self === void 0) {\n    throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");\n  }\n\n  return self;\n}\n\nmodule.exports = _assertThisInitialized;","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').entries;\n","require('../../modules/web.dom-collections.iterator');\nvar entries = require('../array/virtual/entries');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.entries;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.entries)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? entries : own;\n};\n","var $ = require('../internals/export');\nvar getBuiltIn = require('../internals/get-built-in');\nvar fails = require('../internals/fails');\n\nvar $stringify = getBuiltIn('JSON', 'stringify');\nvar re = /[\\uD800-\\uDFFF]/g;\nvar low = /^[\\uD800-\\uDBFF]$/;\nvar hi = /^[\\uDC00-\\uDFFF]$/;\n\nvar fix = function (match, offset, string) {\n  var prev = string.charAt(offset - 1);\n  var next = string.charAt(offset + 1);\n  if ((low.test(match) && !hi.test(next)) || (hi.test(match) && !low.test(prev))) {\n    return '\\\\u' + match.charCodeAt(0).toString(16);\n  } return match;\n};\n\nvar FORCED = fails(function () {\n  return $stringify('\\uDF06\\uD834') !== '\"\\\\udf06\\\\ud834\"'\n    || $stringify('\\uDEAD') !== '\"\\\\udead\"';\n});\n\nif ($stringify) {\n  // https://github.com/tc39/proposal-well-formed-stringify\n  $({ target: 'JSON', stat: true, forced: FORCED }, {\n    // eslint-disable-next-line no-unused-vars\n    stringify: function stringify(it, replacer, space) {\n      var result = $stringify.apply(null, arguments);\n      return typeof result == 'string' ? result.replace(re, fix) : result;\n    }\n  });\n}\n","require('../../modules/es.json.stringify');\nvar core = require('../../internals/path');\n\nif (!core.JSON) core.JSON = { stringify: JSON.stringify };\n\n// eslint-disable-next-line no-unused-vars\nmodule.exports = function stringify(it, replacer, space) {\n  return core.JSON.stringify.apply(null, arguments);\n};\n","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').values;\n","require('../../modules/web.dom-collections.iterator');\nvar values = require('../array/virtual/values');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.values;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.values)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? values : own;\n};\n","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').keys;\n","require('../../modules/web.dom-collections.iterator');\nvar keys = require('../array/virtual/keys');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.keys;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.keys)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? keys : own;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $some = require('../internals/array-iteration').some;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('some');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('some');\n\n// `Array.prototype.some` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.some\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  some: function some(callbackfn /* , thisArg */) {\n    return $some(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.some');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').some;\n","var some = require('../array/virtual/some');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.some;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.some) ? some : own;\n};\n","var _typeof = require(\"../helpers/typeof\");\n\nvar assertThisInitialized = require(\"./assertThisInitialized\");\n\nfunction _possibleConstructorReturn(self, call) {\n  if (call && (_typeof(call) === \"object\" || typeof call === \"function\")) {\n    return call;\n  }\n\n  return assertThisInitialized(self);\n}\n\nmodule.exports = _possibleConstructorReturn;","var parent = require('../../es/object/get-prototype-of');\n\nmodule.exports = parent;\n","var $ = require('../internals/export');\nvar setPrototypeOf = require('../internals/object-set-prototype-of');\n\n// `Object.setPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.setprototypeof\n$({ target: 'Object', stat: true }, {\n  setPrototypeOf: setPrototypeOf\n});\n","require('../../modules/es.object.set-prototype-of');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.setPrototypeOf;\n","var _Object$getPrototypeOf = require(\"../core-js/object/get-prototype-of\");\n\nvar _Object$setPrototypeOf = require(\"../core-js/object/set-prototype-of\");\n\nfunction _getPrototypeOf(o) {\n  module.exports = _getPrototypeOf = _Object$setPrototypeOf ? _Object$getPrototypeOf : function _getPrototypeOf(o) {\n    return o.__proto__ || _Object$getPrototypeOf(o);\n  };\n  return _getPrototypeOf(o);\n}\n\nmodule.exports = _getPrototypeOf;","var parent = require('../../es/object/create');\n\nmodule.exports = parent;\n","var _Object$setPrototypeOf = require(\"../core-js/object/set-prototype-of\");\n\nfunction _setPrototypeOf(o, p) {\n  module.exports = _setPrototypeOf = _Object$setPrototypeOf || function _setPrototypeOf(o, p) {\n    o.__proto__ = p;\n    return o;\n  };\n\n  return _setPrototypeOf(o, p);\n}\n\nmodule.exports = _setPrototypeOf;","var _Object$create = require(\"../core-js/object/create\");\n\nvar setPrototypeOf = require(\"./setPrototypeOf\");\n\nfunction _inherits(subClass, superClass) {\n  if (typeof superClass !== \"function\" && superClass !== null) {\n    throw new TypeError(\"Super expression must either be null or a function\");\n  }\n\n  subClass.prototype = _Object$create(superClass && superClass.prototype, {\n    constructor: {\n      value: subClass,\n      writable: true,\n      configurable: true\n    }\n  });\n  if (superClass) setPrototypeOf(subClass, superClass);\n}\n\nmodule.exports = _inherits;","'use strict';\nvar $ = require('../internals/export');\nvar aFunction = require('../internals/a-function');\nvar toObject = require('../internals/to-object');\nvar fails = require('../internals/fails');\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\n\nvar test = [];\nvar nativeSort = test.sort;\n\n// IE8-\nvar FAILS_ON_UNDEFINED = fails(function () {\n  test.sort(undefined);\n});\n// V8 bug\nvar FAILS_ON_NULL = fails(function () {\n  test.sort(null);\n});\n// Old WebKit\nvar STRICT_METHOD = arrayMethodIsStrict('sort');\n\nvar FORCED = FAILS_ON_UNDEFINED || !FAILS_ON_NULL || !STRICT_METHOD;\n\n// `Array.prototype.sort` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.sort\n$({ target: 'Array', proto: true, forced: FORCED }, {\n  sort: function sort(comparefn) {\n    return comparefn === undefined\n      ? nativeSort.call(toObject(this))\n      : nativeSort.call(toObject(this), aFunction(comparefn));\n  }\n});\n","require('../../../modules/es.array.sort');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').sort;\n","var sort = require('../array/virtual/sort');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.sort;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.sort) ? sort : own;\n};\n","var fails = require('../internals/fails');\n\nmodule.exports = !fails(function () {\n  return Object.isExtensible(Object.preventExtensions({}));\n});\n","var hiddenKeys = require('../internals/hidden-keys');\nvar isObject = require('../internals/is-object');\nvar has = require('../internals/has');\nvar defineProperty = require('../internals/object-define-property').f;\nvar uid = require('../internals/uid');\nvar FREEZING = require('../internals/freezing');\n\nvar METADATA = uid('meta');\nvar id = 0;\n\nvar isExtensible = Object.isExtensible || function () {\n  return true;\n};\n\nvar setMetadata = function (it) {\n  defineProperty(it, METADATA, { value: {\n    objectID: 'O' + ++id, // object ID\n    weakData: {}          // weak collections IDs\n  } });\n};\n\nvar fastKey = function (it, create) {\n  // return a primitive with prefix\n  if (!isObject(it)) return typeof it == 'symbol' ? it : (typeof it == 'string' ? 'S' : 'P') + it;\n  if (!has(it, METADATA)) {\n    // can't set metadata to uncaught frozen object\n    if (!isExtensible(it)) return 'F';\n    // not necessary to add metadata\n    if (!create) return 'E';\n    // add missing metadata\n    setMetadata(it);\n  // return object ID\n  } return it[METADATA].objectID;\n};\n\nvar getWeakData = function (it, create) {\n  if (!has(it, METADATA)) {\n    // can't set metadata to uncaught frozen object\n    if (!isExtensible(it)) return true;\n    // not necessary to add metadata\n    if (!create) return false;\n    // add missing metadata\n    setMetadata(it);\n  // return the store of weak collections IDs\n  } return it[METADATA].weakData;\n};\n\n// add metadata on freeze-family methods calling\nvar onFreeze = function (it) {\n  if (FREEZING && meta.REQUIRED && isExtensible(it) && !has(it, METADATA)) setMetadata(it);\n  return it;\n};\n\nvar meta = module.exports = {\n  REQUIRED: false,\n  fastKey: fastKey,\n  getWeakData: getWeakData,\n  onFreeze: onFreeze\n};\n\nhiddenKeys[METADATA] = true;\n","var anObject = require('../internals/an-object');\nvar isArrayIteratorMethod = require('../internals/is-array-iterator-method');\nvar toLength = require('../internals/to-length');\nvar bind = require('../internals/function-bind-context');\nvar getIteratorMethod = require('../internals/get-iterator-method');\nvar callWithSafeIterationClosing = require('../internals/call-with-safe-iteration-closing');\n\nvar Result = function (stopped, result) {\n  this.stopped = stopped;\n  this.result = result;\n};\n\nvar iterate = module.exports = function (iterable, fn, that, AS_ENTRIES, IS_ITERATOR) {\n  var boundFunction = bind(fn, that, AS_ENTRIES ? 2 : 1);\n  var iterator, iterFn, index, length, result, next, step;\n\n  if (IS_ITERATOR) {\n    iterator = iterable;\n  } else {\n    iterFn = getIteratorMethod(iterable);\n    if (typeof iterFn != 'function') throw TypeError('Target is not iterable');\n    // optimisation for array iterators\n    if (isArrayIteratorMethod(iterFn)) {\n      for (index = 0, length = toLength(iterable.length); length > index; index++) {\n        result = AS_ENTRIES\n          ? boundFunction(anObject(step = iterable[index])[0], step[1])\n          : boundFunction(iterable[index]);\n        if (result && result instanceof Result) return result;\n      } return new Result(false);\n    }\n    iterator = iterFn.call(iterable);\n  }\n\n  next = iterator.next;\n  while (!(step = next.call(iterator)).done) {\n    result = callWithSafeIterationClosing(iterator, boundFunction, step.value, AS_ENTRIES);\n    if (typeof result == 'object' && result && result instanceof Result) return result;\n  } return new Result(false);\n};\n\niterate.stop = function (result) {\n  return new Result(true, result);\n};\n","module.exports = function (it, Constructor, name) {\n  if (!(it instanceof Constructor)) {\n    throw TypeError('Incorrect ' + (name ? name + ' ' : '') + 'invocation');\n  } return it;\n};\n","'use strict';\nvar $ = require('./export');\nvar global = require('../internals/global');\nvar InternalMetadataModule = require('../internals/internal-metadata');\nvar fails = require('../internals/fails');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar iterate = require('../internals/iterate');\nvar anInstance = require('../internals/an-instance');\nvar isObject = require('../internals/is-object');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar defineProperty = require('../internals/object-define-property').f;\nvar forEach = require('../internals/array-iteration').forEach;\nvar DESCRIPTORS = require('../internals/descriptors');\nvar InternalStateModule = require('../internals/internal-state');\n\nvar setInternalState = InternalStateModule.set;\nvar internalStateGetterFor = InternalStateModule.getterFor;\n\nmodule.exports = function (CONSTRUCTOR_NAME, wrapper, common) {\n  var IS_MAP = CONSTRUCTOR_NAME.indexOf('Map') !== -1;\n  var IS_WEAK = CONSTRUCTOR_NAME.indexOf('Weak') !== -1;\n  var ADDER = IS_MAP ? 'set' : 'add';\n  var NativeConstructor = global[CONSTRUCTOR_NAME];\n  var NativePrototype = NativeConstructor && NativeConstructor.prototype;\n  var exported = {};\n  var Constructor;\n\n  if (!DESCRIPTORS || typeof NativeConstructor != 'function'\n    || !(IS_WEAK || NativePrototype.forEach && !fails(function () { new NativeConstructor().entries().next(); }))\n  ) {\n    // create collection constructor\n    Constructor = common.getConstructor(wrapper, CONSTRUCTOR_NAME, IS_MAP, ADDER);\n    InternalMetadataModule.REQUIRED = true;\n  } else {\n    Constructor = wrapper(function (target, iterable) {\n      setInternalState(anInstance(target, Constructor, CONSTRUCTOR_NAME), {\n        type: CONSTRUCTOR_NAME,\n        collection: new NativeConstructor()\n      });\n      if (iterable != undefined) iterate(iterable, target[ADDER], target, IS_MAP);\n    });\n\n    var getInternalState = internalStateGetterFor(CONSTRUCTOR_NAME);\n\n    forEach(['add', 'clear', 'delete', 'forEach', 'get', 'has', 'set', 'keys', 'values', 'entries'], function (KEY) {\n      var IS_ADDER = KEY == 'add' || KEY == 'set';\n      if (KEY in NativePrototype && !(IS_WEAK && KEY == 'clear')) {\n        createNonEnumerableProperty(Constructor.prototype, KEY, function (a, b) {\n          var collection = getInternalState(this).collection;\n          if (!IS_ADDER && IS_WEAK && !isObject(a)) return KEY == 'get' ? undefined : false;\n          var result = collection[KEY](a === 0 ? 0 : a, b);\n          return IS_ADDER ? this : result;\n        });\n      }\n    });\n\n    IS_WEAK || defineProperty(Constructor.prototype, 'size', {\n      configurable: true,\n      get: function () {\n        return getInternalState(this).collection.size;\n      }\n    });\n  }\n\n  setToStringTag(Constructor, CONSTRUCTOR_NAME, false, true);\n\n  exported[CONSTRUCTOR_NAME] = Constructor;\n  $({ global: true, forced: true }, exported);\n\n  if (!IS_WEAK) common.setStrong(Constructor, CONSTRUCTOR_NAME, IS_MAP);\n\n  return Constructor;\n};\n","var redefine = require('../internals/redefine');\n\nmodule.exports = function (target, src, options) {\n  for (var key in src) {\n    if (options && options.unsafe && target[key]) target[key] = src[key];\n    else redefine(target, key, src[key], options);\n  } return target;\n};\n","'use strict';\nvar getBuiltIn = require('../internals/get-built-in');\nvar definePropertyModule = require('../internals/object-define-property');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar DESCRIPTORS = require('../internals/descriptors');\n\nvar SPECIES = wellKnownSymbol('species');\n\nmodule.exports = function (CONSTRUCTOR_NAME) {\n  var Constructor = getBuiltIn(CONSTRUCTOR_NAME);\n  var defineProperty = definePropertyModule.f;\n\n  if (DESCRIPTORS && Constructor && !Constructor[SPECIES]) {\n    defineProperty(Constructor, SPECIES, {\n      configurable: true,\n      get: function () { return this; }\n    });\n  }\n};\n","'use strict';\nvar defineProperty = require('../internals/object-define-property').f;\nvar create = require('../internals/object-create');\nvar redefineAll = require('../internals/redefine-all');\nvar bind = require('../internals/function-bind-context');\nvar anInstance = require('../internals/an-instance');\nvar iterate = require('../internals/iterate');\nvar defineIterator = require('../internals/define-iterator');\nvar setSpecies = require('../internals/set-species');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar fastKey = require('../internals/internal-metadata').fastKey;\nvar InternalStateModule = require('../internals/internal-state');\n\nvar setInternalState = InternalStateModule.set;\nvar internalStateGetterFor = InternalStateModule.getterFor;\n\nmodule.exports = {\n  getConstructor: function (wrapper, CONSTRUCTOR_NAME, IS_MAP, ADDER) {\n    var C = wrapper(function (that, iterable) {\n      anInstance(that, C, CONSTRUCTOR_NAME);\n      setInternalState(that, {\n        type: CONSTRUCTOR_NAME,\n        index: create(null),\n        first: undefined,\n        last: undefined,\n        size: 0\n      });\n      if (!DESCRIPTORS) that.size = 0;\n      if (iterable != undefined) iterate(iterable, that[ADDER], that, IS_MAP);\n    });\n\n    var getInternalState = internalStateGetterFor(CONSTRUCTOR_NAME);\n\n    var define = function (that, key, value) {\n      var state = getInternalState(that);\n      var entry = getEntry(that, key);\n      var previous, index;\n      // change existing entry\n      if (entry) {\n        entry.value = value;\n      // create new entry\n      } else {\n        state.last = entry = {\n          index: index = fastKey(key, true),\n          key: key,\n          value: value,\n          previous: previous = state.last,\n          next: undefined,\n          removed: false\n        };\n        if (!state.first) state.first = entry;\n        if (previous) previous.next = entry;\n        if (DESCRIPTORS) state.size++;\n        else that.size++;\n        // add to index\n        if (index !== 'F') state.index[index] = entry;\n      } return that;\n    };\n\n    var getEntry = function (that, key) {\n      var state = getInternalState(that);\n      // fast case\n      var index = fastKey(key);\n      var entry;\n      if (index !== 'F') return state.index[index];\n      // frozen object case\n      for (entry = state.first; entry; entry = entry.next) {\n        if (entry.key == key) return entry;\n      }\n    };\n\n    redefineAll(C.prototype, {\n      // 23.1.3.1 Map.prototype.clear()\n      // 23.2.3.2 Set.prototype.clear()\n      clear: function clear() {\n        var that = this;\n        var state = getInternalState(that);\n        var data = state.index;\n        var entry = state.first;\n        while (entry) {\n          entry.removed = true;\n          if (entry.previous) entry.previous = entry.previous.next = undefined;\n          delete data[entry.index];\n          entry = entry.next;\n        }\n        state.first = state.last = undefined;\n        if (DESCRIPTORS) state.size = 0;\n        else that.size = 0;\n      },\n      // 23.1.3.3 Map.prototype.delete(key)\n      // 23.2.3.4 Set.prototype.delete(value)\n      'delete': function (key) {\n        var that = this;\n        var state = getInternalState(that);\n        var entry = getEntry(that, key);\n        if (entry) {\n          var next = entry.next;\n          var prev = entry.previous;\n          delete state.index[entry.index];\n          entry.removed = true;\n          if (prev) prev.next = next;\n          if (next) next.previous = prev;\n          if (state.first == entry) state.first = next;\n          if (state.last == entry) state.last = prev;\n          if (DESCRIPTORS) state.size--;\n          else that.size--;\n        } return !!entry;\n      },\n      // 23.2.3.6 Set.prototype.forEach(callbackfn, thisArg = undefined)\n      // 23.1.3.5 Map.prototype.forEach(callbackfn, thisArg = undefined)\n      forEach: function forEach(callbackfn /* , that = undefined */) {\n        var state = getInternalState(this);\n        var boundFunction = bind(callbackfn, arguments.length > 1 ? arguments[1] : undefined, 3);\n        var entry;\n        while (entry = entry ? entry.next : state.first) {\n          boundFunction(entry.value, entry.key, this);\n          // revert to the last existing entry\n          while (entry && entry.removed) entry = entry.previous;\n        }\n      },\n      // 23.1.3.7 Map.prototype.has(key)\n      // 23.2.3.7 Set.prototype.has(value)\n      has: function has(key) {\n        return !!getEntry(this, key);\n      }\n    });\n\n    redefineAll(C.prototype, IS_MAP ? {\n      // 23.1.3.6 Map.prototype.get(key)\n      get: function get(key) {\n        var entry = getEntry(this, key);\n        return entry && entry.value;\n      },\n      // 23.1.3.9 Map.prototype.set(key, value)\n      set: function set(key, value) {\n        return define(this, key === 0 ? 0 : key, value);\n      }\n    } : {\n      // 23.2.3.1 Set.prototype.add(value)\n      add: function add(value) {\n        return define(this, value = value === 0 ? 0 : value, value);\n      }\n    });\n    if (DESCRIPTORS) defineProperty(C.prototype, 'size', {\n      get: function () {\n        return getInternalState(this).size;\n      }\n    });\n    return C;\n  },\n  setStrong: function (C, CONSTRUCTOR_NAME, IS_MAP) {\n    var ITERATOR_NAME = CONSTRUCTOR_NAME + ' Iterator';\n    var getInternalCollectionState = internalStateGetterFor(CONSTRUCTOR_NAME);\n    var getInternalIteratorState = internalStateGetterFor(ITERATOR_NAME);\n    // add .keys, .values, .entries, [@@iterator]\n    // 23.1.3.4, 23.1.3.8, 23.1.3.11, 23.1.3.12, 23.2.3.5, 23.2.3.8, 23.2.3.10, 23.2.3.11\n    defineIterator(C, CONSTRUCTOR_NAME, function (iterated, kind) {\n      setInternalState(this, {\n        type: ITERATOR_NAME,\n        target: iterated,\n        state: getInternalCollectionState(iterated),\n        kind: kind,\n        last: undefined\n      });\n    }, function () {\n      var state = getInternalIteratorState(this);\n      var kind = state.kind;\n      var entry = state.last;\n      // revert to the last existing entry\n      while (entry && entry.removed) entry = entry.previous;\n      // get next entry\n      if (!state.target || !(state.last = entry = entry ? entry.next : state.state.first)) {\n        // or finish the iteration\n        state.target = undefined;\n        return { value: undefined, done: true };\n      }\n      // return step by kind\n      if (kind == 'keys') return { value: entry.key, done: false };\n      if (kind == 'values') return { value: entry.value, done: false };\n      return { value: [entry.key, entry.value], done: false };\n    }, IS_MAP ? 'entries' : 'values', !IS_MAP, true);\n\n    // add [@@species], 23.1.2.2, 23.2.2.2\n    setSpecies(CONSTRUCTOR_NAME);\n  }\n};\n","'use strict';\nvar collection = require('../internals/collection');\nvar collectionStrong = require('../internals/collection-strong');\n\n// `Set` constructor\n// https://tc39.github.io/ecma262/#sec-set-objects\nmodule.exports = collection('Set', function (init) {\n  return function Set() { return init(this, arguments.length ? arguments[0] : undefined); };\n}, collectionStrong);\n","require('../../modules/es.set');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Set;\n","'use strict';\nvar collection = require('../internals/collection');\nvar collectionStrong = require('../internals/collection-strong');\n\n// `Map` constructor\n// https://tc39.github.io/ecma262/#sec-map-objects\nmodule.exports = collection('Map', function (init) {\n  return function Map() { return init(this, arguments.length ? arguments[0] : undefined); };\n}, collectionStrong);\n","require('../../modules/es.map');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Map;\n","/**\n * Copyright (c) 2014-present, Facebook, Inc.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\nvar runtime = (function (exports) {\n  \"use strict\";\n\n  var Op = Object.prototype;\n  var hasOwn = Op.hasOwnProperty;\n  var undefined; // More compressible than void 0.\n  var $Symbol = typeof Symbol === \"function\" ? Symbol : {};\n  var iteratorSymbol = $Symbol.iterator || \"@@iterator\";\n  var asyncIteratorSymbol = $Symbol.asyncIterator || \"@@asyncIterator\";\n  var toStringTagSymbol = $Symbol.toStringTag || \"@@toStringTag\";\n\n  function wrap(innerFn, outerFn, self, tryLocsList) {\n    // If outerFn provided and outerFn.prototype is a Generator, then outerFn.prototype instanceof Generator.\n    var protoGenerator = outerFn && outerFn.prototype instanceof Generator ? outerFn : Generator;\n    var generator = Object.create(protoGenerator.prototype);\n    var context = new Context(tryLocsList || []);\n\n    // The ._invoke method unifies the implementations of the .next,\n    // .throw, and .return methods.\n    generator._invoke = makeInvokeMethod(innerFn, self, context);\n\n    return generator;\n  }\n  exports.wrap = wrap;\n\n  // Try/catch helper to minimize deoptimizations. Returns a completion\n  // record like context.tryEntries[i].completion. This interface could\n  // have been (and was previously) designed to take a closure to be\n  // invoked without arguments, but in all the cases we care about we\n  // already have an existing method we want to call, so there's no need\n  // to create a new function object. We can even get away with assuming\n  // the method takes exactly one argument, since that happens to be true\n  // in every case, so we don't have to touch the arguments object. The\n  // only additional allocation required is the completion record, which\n  // has a stable shape and so hopefully should be cheap to allocate.\n  function tryCatch(fn, obj, arg) {\n    try {\n      return { type: \"normal\", arg: fn.call(obj, arg) };\n    } catch (err) {\n      return { type: \"throw\", arg: err };\n    }\n  }\n\n  var GenStateSuspendedStart = \"suspendedStart\";\n  var GenStateSuspendedYield = \"suspendedYield\";\n  var GenStateExecuting = \"executing\";\n  var GenStateCompleted = \"completed\";\n\n  // Returning this object from the innerFn has the same effect as\n  // breaking out of the dispatch switch statement.\n  var ContinueSentinel = {};\n\n  // Dummy constructor functions that we use as the .constructor and\n  // .constructor.prototype properties for functions that return Generator\n  // objects. For full spec compliance, you may wish to configure your\n  // minifier not to mangle the names of these two functions.\n  function Generator() {}\n  function GeneratorFunction() {}\n  function GeneratorFunctionPrototype() {}\n\n  // This is a polyfill for %IteratorPrototype% for environments that\n  // don't natively support it.\n  var IteratorPrototype = {};\n  IteratorPrototype[iteratorSymbol] = function () {\n    return this;\n  };\n\n  var getProto = Object.getPrototypeOf;\n  var NativeIteratorPrototype = getProto && getProto(getProto(values([])));\n  if (NativeIteratorPrototype &&\n      NativeIteratorPrototype !== Op &&\n      hasOwn.call(NativeIteratorPrototype, iteratorSymbol)) {\n    // This environment has a native %IteratorPrototype%; use it instead\n    // of the polyfill.\n    IteratorPrototype = NativeIteratorPrototype;\n  }\n\n  var Gp = GeneratorFunctionPrototype.prototype =\n    Generator.prototype = Object.create(IteratorPrototype);\n  GeneratorFunction.prototype = Gp.constructor = GeneratorFunctionPrototype;\n  GeneratorFunctionPrototype.constructor = GeneratorFunction;\n  GeneratorFunctionPrototype[toStringTagSymbol] =\n    GeneratorFunction.displayName = \"GeneratorFunction\";\n\n  // Helper for defining the .next, .throw, and .return methods of the\n  // Iterator interface in terms of a single ._invoke method.\n  function defineIteratorMethods(prototype) {\n    [\"next\", \"throw\", \"return\"].forEach(function(method) {\n      prototype[method] = function(arg) {\n        return this._invoke(method, arg);\n      };\n    });\n  }\n\n  exports.isGeneratorFunction = function(genFun) {\n    var ctor = typeof genFun === \"function\" && genFun.constructor;\n    return ctor\n      ? ctor === GeneratorFunction ||\n        // For the native GeneratorFunction constructor, the best we can\n        // do is to check its .name property.\n        (ctor.displayName || ctor.name) === \"GeneratorFunction\"\n      : false;\n  };\n\n  exports.mark = function(genFun) {\n    if (Object.setPrototypeOf) {\n      Object.setPrototypeOf(genFun, GeneratorFunctionPrototype);\n    } else {\n      genFun.__proto__ = GeneratorFunctionPrototype;\n      if (!(toStringTagSymbol in genFun)) {\n        genFun[toStringTagSymbol] = \"GeneratorFunction\";\n      }\n    }\n    genFun.prototype = Object.create(Gp);\n    return genFun;\n  };\n\n  // Within the body of any async function, `await x` is transformed to\n  // `yield regeneratorRuntime.awrap(x)`, so that the runtime can test\n  // `hasOwn.call(value, \"__await\")` to determine if the yielded value is\n  // meant to be awaited.\n  exports.awrap = function(arg) {\n    return { __await: arg };\n  };\n\n  function AsyncIterator(generator, PromiseImpl) {\n    function invoke(method, arg, resolve, reject) {\n      var record = tryCatch(generator[method], generator, arg);\n      if (record.type === \"throw\") {\n        reject(record.arg);\n      } else {\n        var result = record.arg;\n        var value = result.value;\n        if (value &&\n            typeof value === \"object\" &&\n            hasOwn.call(value, \"__await\")) {\n          return PromiseImpl.resolve(value.__await).then(function(value) {\n            invoke(\"next\", value, resolve, reject);\n          }, function(err) {\n            invoke(\"throw\", err, resolve, reject);\n          });\n        }\n\n        return PromiseImpl.resolve(value).then(function(unwrapped) {\n          // When a yielded Promise is resolved, its final value becomes\n          // the .value of the Promise<{value,done}> result for the\n          // current iteration.\n          result.value = unwrapped;\n          resolve(result);\n        }, function(error) {\n          // If a rejected Promise was yielded, throw the rejection back\n          // into the async generator function so it can be handled there.\n          return invoke(\"throw\", error, resolve, reject);\n        });\n      }\n    }\n\n    var previousPromise;\n\n    function enqueue(method, arg) {\n      function callInvokeWithMethodAndArg() {\n        return new PromiseImpl(function(resolve, reject) {\n          invoke(method, arg, resolve, reject);\n        });\n      }\n\n      return previousPromise =\n        // If enqueue has been called before, then we want to wait until\n        // all previous Promises have been resolved before calling invoke,\n        // so that results are always delivered in the correct order. If\n        // enqueue has not been called before, then it is important to\n        // call invoke immediately, without waiting on a callback to fire,\n        // so that the async generator function has the opportunity to do\n        // any necessary setup in a predictable way. This predictability\n        // is why the Promise constructor synchronously invokes its\n        // executor callback, and why async functions synchronously\n        // execute code before the first await. Since we implement simple\n        // async functions in terms of async generators, it is especially\n        // important to get this right, even though it requires care.\n        previousPromise ? previousPromise.then(\n          callInvokeWithMethodAndArg,\n          // Avoid propagating failures to Promises returned by later\n          // invocations of the iterator.\n          callInvokeWithMethodAndArg\n        ) : callInvokeWithMethodAndArg();\n    }\n\n    // Define the unified helper method that is used to implement .next,\n    // .throw, and .return (see defineIteratorMethods).\n    this._invoke = enqueue;\n  }\n\n  defineIteratorMethods(AsyncIterator.prototype);\n  AsyncIterator.prototype[asyncIteratorSymbol] = function () {\n    return this;\n  };\n  exports.AsyncIterator = AsyncIterator;\n\n  // Note that simple async functions are implemented on top of\n  // AsyncIterator objects; they just return a Promise for the value of\n  // the final result produced by the iterator.\n  exports.async = function(innerFn, outerFn, self, tryLocsList, PromiseImpl) {\n    if (PromiseImpl === void 0) PromiseImpl = Promise;\n\n    var iter = new AsyncIterator(\n      wrap(innerFn, outerFn, self, tryLocsList),\n      PromiseImpl\n    );\n\n    return exports.isGeneratorFunction(outerFn)\n      ? iter // If outerFn is a generator, return the full iterator.\n      : iter.next().then(function(result) {\n          return result.done ? result.value : iter.next();\n        });\n  };\n\n  function makeInvokeMethod(innerFn, self, context) {\n    var state = GenStateSuspendedStart;\n\n    return function invoke(method, arg) {\n      if (state === GenStateExecuting) {\n        throw new Error(\"Generator is already running\");\n      }\n\n      if (state === GenStateCompleted) {\n        if (method === \"throw\") {\n          throw arg;\n        }\n\n        // Be forgiving, per 25.3.3.3.3 of the spec:\n        // https://people.mozilla.org/~jorendorff/es6-draft.html#sec-generatorresume\n        return doneResult();\n      }\n\n      context.method = method;\n      context.arg = arg;\n\n      while (true) {\n        var delegate = context.delegate;\n        if (delegate) {\n          var delegateResult = maybeInvokeDelegate(delegate, context);\n          if (delegateResult) {\n            if (delegateResult === ContinueSentinel) continue;\n            return delegateResult;\n          }\n        }\n\n        if (context.method === \"next\") {\n          // Setting context._sent for legacy support of Babel's\n          // function.sent implementation.\n          context.sent = context._sent = context.arg;\n\n        } else if (context.method === \"throw\") {\n          if (state === GenStateSuspendedStart) {\n            state = GenStateCompleted;\n            throw context.arg;\n          }\n\n          context.dispatchException(context.arg);\n\n        } else if (context.method === \"return\") {\n          context.abrupt(\"return\", context.arg);\n        }\n\n        state = GenStateExecuting;\n\n        var record = tryCatch(innerFn, self, context);\n        if (record.type === \"normal\") {\n          // If an exception is thrown from innerFn, we leave state ===\n          // GenStateExecuting and loop back for another invocation.\n          state = context.done\n            ? GenStateCompleted\n            : GenStateSuspendedYield;\n\n          if (record.arg === ContinueSentinel) {\n            continue;\n          }\n\n          return {\n            value: record.arg,\n            done: context.done\n          };\n\n        } else if (record.type === \"throw\") {\n          state = GenStateCompleted;\n          // Dispatch the exception by looping back around to the\n          // context.dispatchException(context.arg) call above.\n          context.method = \"throw\";\n          context.arg = record.arg;\n        }\n      }\n    };\n  }\n\n  // Call delegate.iterator[context.method](context.arg) and handle the\n  // result, either by returning a { value, done } result from the\n  // delegate iterator, or by modifying context.method and context.arg,\n  // setting context.delegate to null, and returning the ContinueSentinel.\n  function maybeInvokeDelegate(delegate, context) {\n    var method = delegate.iterator[context.method];\n    if (method === undefined) {\n      // A .throw or .return when the delegate iterator has no .throw\n      // method always terminates the yield* loop.\n      context.delegate = null;\n\n      if (context.method === \"throw\") {\n        // Note: [\"return\"] must be used for ES3 parsing compatibility.\n        if (delegate.iterator[\"return\"]) {\n          // If the delegate iterator has a return method, give it a\n          // chance to clean up.\n          context.method = \"return\";\n          context.arg = undefined;\n          maybeInvokeDelegate(delegate, context);\n\n          if (context.method === \"throw\") {\n            // If maybeInvokeDelegate(context) changed context.method from\n            // \"return\" to \"throw\", let that override the TypeError below.\n            return ContinueSentinel;\n          }\n        }\n\n        context.method = \"throw\";\n        context.arg = new TypeError(\n          \"The iterator does not provide a 'throw' method\");\n      }\n\n      return ContinueSentinel;\n    }\n\n    var record = tryCatch(method, delegate.iterator, context.arg);\n\n    if (record.type === \"throw\") {\n      context.method = \"throw\";\n      context.arg = record.arg;\n      context.delegate = null;\n      return ContinueSentinel;\n    }\n\n    var info = record.arg;\n\n    if (! info) {\n      context.method = \"throw\";\n      context.arg = new TypeError(\"iterator result is not an object\");\n      context.delegate = null;\n      return ContinueSentinel;\n    }\n\n    if (info.done) {\n      // Assign the result of the finished delegate to the temporary\n      // variable specified by delegate.resultName (see delegateYield).\n      context[delegate.resultName] = info.value;\n\n      // Resume execution at the desired location (see delegateYield).\n      context.next = delegate.nextLoc;\n\n      // If context.method was \"throw\" but the delegate handled the\n      // exception, let the outer generator proceed normally. If\n      // context.method was \"next\", forget context.arg since it has been\n      // \"consumed\" by the delegate iterator. If context.method was\n      // \"return\", allow the original .return call to continue in the\n      // outer generator.\n      if (context.method !== \"return\") {\n        context.method = \"next\";\n        context.arg = undefined;\n      }\n\n    } else {\n      // Re-yield the result returned by the delegate method.\n      return info;\n    }\n\n    // The delegate iterator is finished, so forget it and continue with\n    // the outer generator.\n    context.delegate = null;\n    return ContinueSentinel;\n  }\n\n  // Define Generator.prototype.{next,throw,return} in terms of the\n  // unified ._invoke helper method.\n  defineIteratorMethods(Gp);\n\n  Gp[toStringTagSymbol] = \"Generator\";\n\n  // A Generator should always return itself as the iterator object when the\n  // @@iterator function is called on it. Some browsers' implementations of the\n  // iterator prototype chain incorrectly implement this, causing the Generator\n  // object to not be returned from this call. This ensures that doesn't happen.\n  // See https://github.com/facebook/regenerator/issues/274 for more details.\n  Gp[iteratorSymbol] = function() {\n    return this;\n  };\n\n  Gp.toString = function() {\n    return \"[object Generator]\";\n  };\n\n  function pushTryEntry(locs) {\n    var entry = { tryLoc: locs[0] };\n\n    if (1 in locs) {\n      entry.catchLoc = locs[1];\n    }\n\n    if (2 in locs) {\n      entry.finallyLoc = locs[2];\n      entry.afterLoc = locs[3];\n    }\n\n    this.tryEntries.push(entry);\n  }\n\n  function resetTryEntry(entry) {\n    var record = entry.completion || {};\n    record.type = \"normal\";\n    delete record.arg;\n    entry.completion = record;\n  }\n\n  function Context(tryLocsList) {\n    // The root entry object (effectively a try statement without a catch\n    // or a finally block) gives us a place to store values thrown from\n    // locations where there is no enclosing try statement.\n    this.tryEntries = [{ tryLoc: \"root\" }];\n    tryLocsList.forEach(pushTryEntry, this);\n    this.reset(true);\n  }\n\n  exports.keys = function(object) {\n    var keys = [];\n    for (var key in object) {\n      keys.push(key);\n    }\n    keys.reverse();\n\n    // Rather than returning an object with a next method, we keep\n    // things simple and return the next function itself.\n    return function next() {\n      while (keys.length) {\n        var key = keys.pop();\n        if (key in object) {\n          next.value = key;\n          next.done = false;\n          return next;\n        }\n      }\n\n      // To avoid creating an additional object, we just hang the .value\n      // and .done properties off the next function object itself. This\n      // also ensures that the minifier will not anonymize the function.\n      next.done = true;\n      return next;\n    };\n  };\n\n  function values(iterable) {\n    if (iterable) {\n      var iteratorMethod = iterable[iteratorSymbol];\n      if (iteratorMethod) {\n        return iteratorMethod.call(iterable);\n      }\n\n      if (typeof iterable.next === \"function\") {\n        return iterable;\n      }\n\n      if (!isNaN(iterable.length)) {\n        var i = -1, next = function next() {\n          while (++i < iterable.length) {\n            if (hasOwn.call(iterable, i)) {\n              next.value = iterable[i];\n              next.done = false;\n              return next;\n            }\n          }\n\n          next.value = undefined;\n          next.done = true;\n\n          return next;\n        };\n\n        return next.next = next;\n      }\n    }\n\n    // Return an iterator with no values.\n    return { next: doneResult };\n  }\n  exports.values = values;\n\n  function doneResult() {\n    return { value: undefined, done: true };\n  }\n\n  Context.prototype = {\n    constructor: Context,\n\n    reset: function(skipTempReset) {\n      this.prev = 0;\n      this.next = 0;\n      // Resetting context._sent for legacy support of Babel's\n      // function.sent implementation.\n      this.sent = this._sent = undefined;\n      this.done = false;\n      this.delegate = null;\n\n      this.method = \"next\";\n      this.arg = undefined;\n\n      this.tryEntries.forEach(resetTryEntry);\n\n      if (!skipTempReset) {\n        for (var name in this) {\n          // Not sure about the optimal order of these conditions:\n          if (name.charAt(0) === \"t\" &&\n              hasOwn.call(this, name) &&\n              !isNaN(+name.slice(1))) {\n            this[name] = undefined;\n          }\n        }\n      }\n    },\n\n    stop: function() {\n      this.done = true;\n\n      var rootEntry = this.tryEntries[0];\n      var rootRecord = rootEntry.completion;\n      if (rootRecord.type === \"throw\") {\n        throw rootRecord.arg;\n      }\n\n      return this.rval;\n    },\n\n    dispatchException: function(exception) {\n      if (this.done) {\n        throw exception;\n      }\n\n      var context = this;\n      function handle(loc, caught) {\n        record.type = \"throw\";\n        record.arg = exception;\n        context.next = loc;\n\n        if (caught) {\n          // If the dispatched exception was caught by a catch block,\n          // then let that catch block handle the exception normally.\n          context.method = \"next\";\n          context.arg = undefined;\n        }\n\n        return !! caught;\n      }\n\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        var record = entry.completion;\n\n        if (entry.tryLoc === \"root\") {\n          // Exception thrown outside of any try block that could handle\n          // it, so set the completion value of the entire function to\n          // throw the exception.\n          return handle(\"end\");\n        }\n\n        if (entry.tryLoc <= this.prev) {\n          var hasCatch = hasOwn.call(entry, \"catchLoc\");\n          var hasFinally = hasOwn.call(entry, \"finallyLoc\");\n\n          if (hasCatch && hasFinally) {\n            if (this.prev < entry.catchLoc) {\n              return handle(entry.catchLoc, true);\n            } else if (this.prev < entry.finallyLoc) {\n              return handle(entry.finallyLoc);\n            }\n\n          } else if (hasCatch) {\n            if (this.prev < entry.catchLoc) {\n              return handle(entry.catchLoc, true);\n            }\n\n          } else if (hasFinally) {\n            if (this.prev < entry.finallyLoc) {\n              return handle(entry.finallyLoc);\n            }\n\n          } else {\n            throw new Error(\"try statement without catch or finally\");\n          }\n        }\n      }\n    },\n\n    abrupt: function(type, arg) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.tryLoc <= this.prev &&\n            hasOwn.call(entry, \"finallyLoc\") &&\n            this.prev < entry.finallyLoc) {\n          var finallyEntry = entry;\n          break;\n        }\n      }\n\n      if (finallyEntry &&\n          (type === \"break\" ||\n           type === \"continue\") &&\n          finallyEntry.tryLoc <= arg &&\n          arg <= finallyEntry.finallyLoc) {\n        // Ignore the finally entry if control is not jumping to a\n        // location outside the try/catch block.\n        finallyEntry = null;\n      }\n\n      var record = finallyEntry ? finallyEntry.completion : {};\n      record.type = type;\n      record.arg = arg;\n\n      if (finallyEntry) {\n        this.method = \"next\";\n        this.next = finallyEntry.finallyLoc;\n        return ContinueSentinel;\n      }\n\n      return this.complete(record);\n    },\n\n    complete: function(record, afterLoc) {\n      if (record.type === \"throw\") {\n        throw record.arg;\n      }\n\n      if (record.type === \"break\" ||\n          record.type === \"continue\") {\n        this.next = record.arg;\n      } else if (record.type === \"return\") {\n        this.rval = this.arg = record.arg;\n        this.method = \"return\";\n        this.next = \"end\";\n      } else if (record.type === \"normal\" && afterLoc) {\n        this.next = afterLoc;\n      }\n\n      return ContinueSentinel;\n    },\n\n    finish: function(finallyLoc) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.finallyLoc === finallyLoc) {\n          this.complete(entry.completion, entry.afterLoc);\n          resetTryEntry(entry);\n          return ContinueSentinel;\n        }\n      }\n    },\n\n    \"catch\": function(tryLoc) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.tryLoc === tryLoc) {\n          var record = entry.completion;\n          if (record.type === \"throw\") {\n            var thrown = record.arg;\n            resetTryEntry(entry);\n          }\n          return thrown;\n        }\n      }\n\n      // The context.catch method must only be called with a location\n      // argument that corresponds to a known catch block.\n      throw new Error(\"illegal catch attempt\");\n    },\n\n    delegateYield: function(iterable, resultName, nextLoc) {\n      this.delegate = {\n        iterator: values(iterable),\n        resultName: resultName,\n        nextLoc: nextLoc\n      };\n\n      if (this.method === \"next\") {\n        // Deliberately forget the last sent value so that we don't\n        // accidentally pass it on to the delegate.\n        this.arg = undefined;\n      }\n\n      return ContinueSentinel;\n    }\n  };\n\n  // Regardless of whether this script is executing as a CommonJS module\n  // or not, return the runtime object so that we can declare the variable\n  // regeneratorRuntime in the outer scope, which allows this module to be\n  // injected easily by `bin/regenerator --include-runtime script.js`.\n  return exports;\n\n}(\n  // If this script is executing as a CommonJS module, use module.exports\n  // as the regeneratorRuntime namespace. Otherwise create a new empty\n  // object. Either way, the resulting object will be used to initialize\n  // the regeneratorRuntime variable at the top of this file.\n  typeof module === \"object\" ? module.exports : {}\n));\n\ntry {\n  regeneratorRuntime = runtime;\n} catch (accidentalStrictMode) {\n  // This module should not be running in strict mode, so the above\n  // assignment should always work unless something is misconfigured. Just\n  // in case runtime.js accidentally runs in strict mode, we can escape\n  // strict mode using a global Function call. This could conceivably fail\n  // if a Content Security Policy forbids using Function, but in that case\n  // the proper solution is to fix the accidental strict mode problem. If\n  // you've misconfigured your bundler to force strict mode and applied a\n  // CSP to forbid Function, and you're not willing to fix either of those\n  // problems, please detail your unique predicament in a GitHub issue.\n  Function(\"r\", \"regeneratorRuntime = r\")(runtime);\n}\n","var parent = require('../../es/symbol/iterator');\n\nmodule.exports = parent;\n","'use strict';\nvar $ = require('../internals/export');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toInteger = require('../internals/to-integer');\nvar toLength = require('../internals/to-length');\nvar toObject = require('../internals/to-object');\nvar arraySpeciesCreate = require('../internals/array-species-create');\nvar createProperty = require('../internals/create-property');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('splice');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('splice', { ACCESSORS: true, 0: 0, 1: 2 });\n\nvar max = Math.max;\nvar min = Math.min;\nvar MAX_SAFE_INTEGER = 0x1FFFFFFFFFFFFF;\nvar MAXIMUM_ALLOWED_LENGTH_EXCEEDED = 'Maximum allowed length exceeded';\n\n// `Array.prototype.splice` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.splice\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  splice: function splice(start, deleteCount /* , ...items */) {\n    var O = toObject(this);\n    var len = toLength(O.length);\n    var actualStart = toAbsoluteIndex(start, len);\n    var argumentsLength = arguments.length;\n    var insertCount, actualDeleteCount, A, k, from, to;\n    if (argumentsLength === 0) {\n      insertCount = actualDeleteCount = 0;\n    } else if (argumentsLength === 1) {\n      insertCount = 0;\n      actualDeleteCount = len - actualStart;\n    } else {\n      insertCount = argumentsLength - 2;\n      actualDeleteCount = min(max(toInteger(deleteCount), 0), len - actualStart);\n    }\n    if (len + insertCount - actualDeleteCount > MAX_SAFE_INTEGER) {\n      throw TypeError(MAXIMUM_ALLOWED_LENGTH_EXCEEDED);\n    }\n    A = arraySpeciesCreate(O, actualDeleteCount);\n    for (k = 0; k < actualDeleteCount; k++) {\n      from = actualStart + k;\n      if (from in O) createProperty(A, k, O[from]);\n    }\n    A.length = actualDeleteCount;\n    if (insertCount < actualDeleteCount) {\n      for (k = actualStart; k < len - actualDeleteCount; k++) {\n        from = k + actualDeleteCount;\n        to = k + insertCount;\n        if (from in O) O[to] = O[from];\n        else delete O[to];\n      }\n      for (k = len; k > len - actualDeleteCount + insertCount; k--) delete O[k - 1];\n    } else if (insertCount > actualDeleteCount) {\n      for (k = len - actualDeleteCount; k > actualStart; k--) {\n        from = k + actualDeleteCount - 1;\n        to = k + insertCount - 1;\n        if (from in O) O[to] = O[from];\n        else delete O[to];\n      }\n    }\n    for (k = 0; k < insertCount; k++) {\n      O[k + actualStart] = arguments[k + 2];\n    }\n    O.length = len - actualDeleteCount + insertCount;\n    return A;\n  }\n});\n","require('../../../modules/es.array.splice');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').splice;\n","var splice = require('../array/virtual/splice');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.splice;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.splice) ? splice : own;\n};\n","var $ = require('../internals/export');\nvar global = require('../internals/global');\nvar userAgent = require('../internals/engine-user-agent');\n\nvar slice = [].slice;\nvar MSIE = /MSIE .\\./.test(userAgent); // <- dirty ie9- check\n\nvar wrap = function (scheduler) {\n  return function (handler, timeout /* , ...arguments */) {\n    var boundArgs = arguments.length > 2;\n    var args = boundArgs ? slice.call(arguments, 2) : undefined;\n    return scheduler(boundArgs ? function () {\n      // eslint-disable-next-line no-new-func\n      (typeof handler == 'function' ? handler : Function(handler)).apply(this, args);\n    } : handler, timeout);\n  };\n};\n\n// ie9- setTimeout & setInterval additional parameters fix\n// https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#timers\n$({ global: true, bind: true, forced: MSIE }, {\n  // `setTimeout` method\n  // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-settimeout\n  setTimeout: wrap(global.setTimeout),\n  // `setInterval` method\n  // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-setinterval\n  setInterval: wrap(global.setInterval)\n});\n","require('../modules/web.timers');\nvar path = require('../internals/path');\n\nmodule.exports = path.setTimeout;\n","'use strict';\nvar isArray = require('../internals/is-array');\nvar toLength = require('../internals/to-length');\nvar bind = require('../internals/function-bind-context');\n\n// `FlattenIntoArray` abstract operation\n// https://tc39.github.io/proposal-flatMap/#sec-FlattenIntoArray\nvar flattenIntoArray = function (target, original, source, sourceLen, start, depth, mapper, thisArg) {\n  var targetIndex = start;\n  var sourceIndex = 0;\n  var mapFn = mapper ? bind(mapper, thisArg, 3) : false;\n  var element;\n\n  while (sourceIndex < sourceLen) {\n    if (sourceIndex in source) {\n      element = mapFn ? mapFn(source[sourceIndex], sourceIndex, original) : source[sourceIndex];\n\n      if (depth > 0 && isArray(element)) {\n        targetIndex = flattenIntoArray(target, original, element, toLength(element.length), targetIndex, depth - 1) - 1;\n      } else {\n        if (targetIndex >= 0x1FFFFFFFFFFFFF) throw TypeError('Exceed the acceptable array length');\n        target[targetIndex] = element;\n      }\n\n      targetIndex++;\n    }\n    sourceIndex++;\n  }\n  return targetIndex;\n};\n\nmodule.exports = flattenIntoArray;\n","'use strict';\nvar $ = require('../internals/export');\nvar flattenIntoArray = require('../internals/flatten-into-array');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar aFunction = require('../internals/a-function');\nvar arraySpeciesCreate = require('../internals/array-species-create');\n\n// `Array.prototype.flatMap` method\n// https://github.com/tc39/proposal-flatMap\n$({ target: 'Array', proto: true }, {\n  flatMap: function flatMap(callbackfn /* , thisArg */) {\n    var O = toObject(this);\n    var sourceLen = toLength(O.length);\n    var A;\n    aFunction(callbackfn);\n    A = arraySpeciesCreate(O, 0);\n    A.length = flattenIntoArray(A, O, O, sourceLen, 0, 1, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n    return A;\n  }\n});\n","require('../../../modules/es.array.flat-map');\nrequire('../../../modules/es.array.unscopables.flat-map');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').flatMap;\n","var flatMap = require('../array/virtual/flat-map');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.flatMap;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.flatMap) ? flatMap : own;\n};\n","var aFunction = require('../internals/a-function');\nvar toObject = require('../internals/to-object');\nvar IndexedObject = require('../internals/indexed-object');\nvar toLength = require('../internals/to-length');\n\n// `Array.prototype.{ reduce, reduceRight }` methods implementation\nvar createMethod = function (IS_RIGHT) {\n  return function (that, callbackfn, argumentsLength, memo) {\n    aFunction(callbackfn);\n    var O = toObject(that);\n    var self = IndexedObject(O);\n    var length = toLength(O.length);\n    var index = IS_RIGHT ? length - 1 : 0;\n    var i = IS_RIGHT ? -1 : 1;\n    if (argumentsLength < 2) while (true) {\n      if (index in self) {\n        memo = self[index];\n        index += i;\n        break;\n      }\n      index += i;\n      if (IS_RIGHT ? index < 0 : length <= index) {\n        throw TypeError('Reduce of empty array with no initial value');\n      }\n    }\n    for (;IS_RIGHT ? index >= 0 : length > index; index += i) if (index in self) {\n      memo = callbackfn(memo, self[index], index, O);\n    }\n    return memo;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.reduce` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.reduce\n  left: createMethod(false),\n  // `Array.prototype.reduceRight` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.reduceright\n  right: createMethod(true)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $reduce = require('../internals/array-reduce').left;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('reduce');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('reduce', { 1: 0 });\n\n// `Array.prototype.reduce` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.reduce\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  reduce: function reduce(callbackfn /* , initialValue */) {\n    return $reduce(this, callbackfn, arguments.length, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.reduce');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').reduce;\n","var reduce = require('../array/virtual/reduce');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.reduce;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.reduce) ? reduce : own;\n};\n","'use strict';\nvar aFunction = require('../internals/a-function');\nvar isObject = require('../internals/is-object');\n\nvar slice = [].slice;\nvar factories = {};\n\nvar construct = function (C, argsLength, args) {\n  if (!(argsLength in factories)) {\n    for (var list = [], i = 0; i < argsLength; i++) list[i] = 'a[' + i + ']';\n    // eslint-disable-next-line no-new-func\n    factories[argsLength] = Function('C,a', 'return new C(' + list.join(',') + ')');\n  } return factories[argsLength](C, args);\n};\n\n// `Function.prototype.bind` method implementation\n// https://tc39.github.io/ecma262/#sec-function.prototype.bind\nmodule.exports = Function.bind || function bind(that /* , ...args */) {\n  var fn = aFunction(this);\n  var partArgs = slice.call(arguments, 1);\n  var boundFunction = function bound(/* args... */) {\n    var args = partArgs.concat(slice.call(arguments));\n    return this instanceof boundFunction ? construct(fn, args.length, args) : fn.apply(that, args);\n  };\n  if (isObject(fn.prototype)) boundFunction.prototype = fn.prototype;\n  return boundFunction;\n};\n","var $ = require('../internals/export');\nvar bind = require('../internals/function-bind');\n\n// `Function.prototype.bind` method\n// https://tc39.github.io/ecma262/#sec-function.prototype.bind\n$({ target: 'Function', proto: true }, {\n  bind: bind\n});\n","require('../../../modules/es.function.bind');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Function').bind;\n","var bind = require('../function/virtual/bind');\n\nvar FunctionPrototype = Function.prototype;\n\nmodule.exports = function (it) {\n  var own = it.bind;\n  return it === FunctionPrototype || (it instanceof Function && own === FunctionPrototype.bind) ? bind : own;\n};\n","function _classCallCheck(instance, Constructor) {\n  if (!(instance instanceof Constructor)) {\n    throw new TypeError(\"Cannot call a class as a function\");\n  }\n}\n\nmodule.exports = _classCallCheck;","var _Object$defineProperty = require(\"../core-js/object/define-property\");\n\nfunction _defineProperties(target, props) {\n  for (var i = 0; i < props.length; i++) {\n    var descriptor = props[i];\n    descriptor.enumerable = descriptor.enumerable || false;\n    descriptor.configurable = true;\n    if (\"value\" in descriptor) descriptor.writable = true;\n\n    _Object$defineProperty(target, descriptor.key, descriptor);\n  }\n}\n\nfunction _createClass(Constructor, protoProps, staticProps) {\n  if (protoProps) _defineProperties(Constructor.prototype, protoProps);\n  if (staticProps) _defineProperties(Constructor, staticProps);\n  return Constructor;\n}\n\nmodule.exports = _createClass;","/**\n * vis-data\n * http://visjs.org/\n *\n * Manage unstructured data using DataSet. Add, update, and remove data, and listen for changes in the data.\n *\n * @version 6.5.0\n * @date    2020-03-07T21:44:22.378Z\n *\n * @copyright (c) 2011-2017 Almende B.V, http://almende.com\n * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs\n *\n * @license\n * vis.js is dual licensed under both\n *\n *   1. The Apache 2.0 License\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *   and\n *\n *   2. The MIT License\n *      http://opensource.org/licenses/MIT\n *\n * vis.js may be distributed under either license.\n */\n\nimport { isNumber, isString, getType, deepExtend } from 'vis-util/esnext';\nimport moment from 'moment';\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\n/**\r\n * Create new data pipe.\r\n *\r\n * @param from - The source data set or data view.\r\n *\r\n * @remarks\r\n * Example usage:\r\n * ```typescript\r\n * interface AppItem {\r\n *   whoami: string;\r\n *   appData: unknown;\r\n *   visData: VisItem;\r\n * }\r\n * interface VisItem {\r\n *   id: number;\r\n *   label: string;\r\n *   color: string;\r\n *   x: number;\r\n *   y: number;\r\n * }\r\n *\r\n * const ds1 = new DataSet<AppItem, \"whoami\">([], { fieldId: \"whoami\" });\r\n * const ds2 = new DataSet<VisItem, \"id\">();\r\n *\r\n * const pipe = createNewDataPipeFrom(ds1)\r\n *   .filter((item): boolean => item.enabled === true)\r\n *   .map<VisItem, \"id\">((item): VisItem => item.visData)\r\n *   .to(ds2);\r\n *\r\n * pipe.start();\r\n * ```\r\n *\r\n * @returns A factory whose methods can be used to configure the pipe.\r\n */\r\nfunction createNewDataPipeFrom(from) {\r\n    return new DataPipeUnderConstruction(from);\r\n}\r\n/**\r\n * Internal implementation of the pipe. This should be accessible only through\r\n * `createNewDataPipeFrom` from the outside.\r\n *\r\n * @typeparam SI - Source item type.\r\n * @typeparam SP - Source item type's id property name.\r\n * @typeparam TI - Target item type.\r\n * @typeparam TP - Target item type's id property name.\r\n */\r\nclass SimpleDataPipe {\r\n    /**\r\n     * Create a new data pipe.\r\n     *\r\n     * @param _source - The data set or data view that will be observed.\r\n     * @param _transformers - An array of transforming functions to be used to\r\n     * filter or transform the items in the pipe.\r\n     * @param _target - The data set or data view that will receive the items.\r\n     */\r\n    constructor(_source, _transformers, _target) {\r\n        this._source = _source;\r\n        this._transformers = _transformers;\r\n        this._target = _target;\r\n        /**\r\n         * Bound listeners for use with `DataInterface['on' | 'off']`.\r\n         */\r\n        this._listeners = {\r\n            add: this._add.bind(this),\r\n            remove: this._remove.bind(this),\r\n            update: this._update.bind(this)\r\n        };\r\n    }\r\n    /** @inheritdoc */\r\n    all() {\r\n        this._target.update(this._transformItems(this._source.get()));\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    start() {\r\n        this._source.on(\"add\", this._listeners.add);\r\n        this._source.on(\"remove\", this._listeners.remove);\r\n        this._source.on(\"update\", this._listeners.update);\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    stop() {\r\n        this._source.off(\"add\", this._listeners.add);\r\n        this._source.off(\"remove\", this._listeners.remove);\r\n        this._source.off(\"update\", this._listeners.update);\r\n        return this;\r\n    }\r\n    /**\r\n     * Apply the transformers to the items.\r\n     *\r\n     * @param items - The items to be transformed.\r\n     *\r\n     * @returns The transformed items.\r\n     */\r\n    _transformItems(items) {\r\n        return this._transformers.reduce((items, transform) => {\r\n            return transform(items);\r\n        }, items);\r\n    }\r\n    /**\r\n     * Handle an add event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the ids of the added items.\r\n     */\r\n    _add(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.add(this._transformItems(this._source.get(payload.items)));\r\n    }\r\n    /**\r\n     * Handle an update event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the ids of the updated items.\r\n     */\r\n    _update(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.update(this._transformItems(this._source.get(payload.items)));\r\n    }\r\n    /**\r\n     * Handle a remove event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the data of the removed items.\r\n     */\r\n    _remove(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.remove(this._transformItems(payload.oldData));\r\n    }\r\n}\r\n/**\r\n * Internal implementation of the pipe factory. This should be accessible\r\n * only through `createNewDataPipeFrom` from the outside.\r\n *\r\n * @typeparam TI - Target item type.\r\n * @typeparam TP - Target item type's id property name.\r\n */\r\nclass DataPipeUnderConstruction {\r\n    /**\r\n     * Create a new data pipe factory. This is an internal constructor that\r\n     * should never be called from outside of this file.\r\n     *\r\n     * @param _source - The source data set or data view for this pipe.\r\n     */\r\n    constructor(_source) {\r\n        this._source = _source;\r\n        /**\r\n         * Array transformers used to transform items within the pipe. This is typed\r\n         * as any for the sake of simplicity.\r\n         */\r\n        this._transformers = [];\r\n    }\r\n    /**\r\n     * Filter the items.\r\n     *\r\n     * @param callback - A filtering function that returns true if given item\r\n     * should be piped and false if not.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    filter(callback) {\r\n        this._transformers.push((input) => input.filter(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Map each source item to a new type.\r\n     *\r\n     * @param callback - A mapping function that takes a source item and returns\r\n     * corresponding mapped item.\r\n     *\r\n     * @typeparam TI - Target item type.\r\n     * @typeparam TP - Target item type's id property name.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    map(callback) {\r\n        this._transformers.push((input) => input.map(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Map each source item to zero or more items of a new type.\r\n     *\r\n     * @param callback - A mapping function that takes a source item and returns\r\n     * an array of corresponding mapped items.\r\n     *\r\n     * @typeparam TI - Target item type.\r\n     * @typeparam TP - Target item type's id property name.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    flatMap(callback) {\r\n        this._transformers.push((input) => input.flatMap(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Connect this pipe to given data set.\r\n     *\r\n     * @param target - The data set that will receive the items from this pipe.\r\n     *\r\n     * @returns The pipe connected between given data sets and performing\r\n     * configured transformation on the processed items.\r\n     */\r\n    to(target) {\r\n        return new SimpleDataPipe(this._source, this._transformers, target);\r\n    }\r\n}\n\nfunction createCommonjsModule(fn, module) {\n\treturn module = { exports: {} }, fn(module, module.exports), module.exports;\n}\n\nvar rngBrowser = createCommonjsModule(function (module) {\n// Unique ID creation requires a high quality random # generator.  In the\n// browser this is a little complicated due to unknown quality of Math.random()\n// and inconsistent support for the `crypto` API.  We do the best we can via\n// feature-detection\n\n// getRandomValues needs to be invoked in a context where \"this\" is a Crypto\n// implementation. Also, find the complete implementation of crypto on IE11.\nvar getRandomValues = (typeof(crypto) != 'undefined' && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) ||\n                      (typeof(msCrypto) != 'undefined' && typeof window.msCrypto.getRandomValues == 'function' && msCrypto.getRandomValues.bind(msCrypto));\n\nif (getRandomValues) {\n  // WHATWG crypto RNG - http://wiki.whatwg.org/wiki/Crypto\n  var rnds8 = new Uint8Array(16); // eslint-disable-line no-undef\n\n  module.exports = function whatwgRNG() {\n    getRandomValues(rnds8);\n    return rnds8;\n  };\n} else {\n  // Math.random()-based (RNG)\n  //\n  // If all else fails, use Math.random().  It's fast, but is of unspecified\n  // quality.\n  var rnds = new Array(16);\n\n  module.exports = function mathRNG() {\n    for (var i = 0, r; i < 16; i++) {\n      if ((i & 0x03) === 0) r = Math.random() * 0x100000000;\n      rnds[i] = r >>> ((i & 0x03) << 3) & 0xff;\n    }\n\n    return rnds;\n  };\n}\n});\n\n/**\n * Convert array of 16 byte values to UUID string format of the form:\n * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n */\nvar byteToHex = [];\nfor (var i = 0; i < 256; ++i) {\n  byteToHex[i] = (i + 0x100).toString(16).substr(1);\n}\n\nfunction bytesToUuid(buf, offset) {\n  var i = offset || 0;\n  var bth = byteToHex;\n  // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4\n  return ([\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]]\n  ]).join('');\n}\n\nvar bytesToUuid_1 = bytesToUuid;\n\nfunction v4(options, buf, offset) {\n  var i = buf && offset || 0;\n\n  if (typeof(options) == 'string') {\n    buf = options === 'binary' ? new Array(16) : null;\n    options = null;\n  }\n  options = options || {};\n\n  var rnds = options.random || (options.rng || rngBrowser)();\n\n  // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`\n  rnds[6] = (rnds[6] & 0x0f) | 0x40;\n  rnds[8] = (rnds[8] & 0x3f) | 0x80;\n\n  // Copy bytes to buffer, if provided\n  if (buf) {\n    for (var ii = 0; ii < 16; ++ii) {\n      buf[i + ii] = rnds[ii];\n    }\n  }\n\n  return buf || bytesToUuid_1(rnds);\n}\n\nvar v4_1 = v4;\n\n// utility functions\r\n// parse ASP.Net Date pattern,\r\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\r\n// code from http://momentjs.com/\r\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\r\n/**\r\n * Test whether given object is a Moment date.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if Moment instance, false otherwise.\r\n */\r\nfunction isMoment(value) {\r\n    return moment.isMoment(value);\r\n}\r\n/**\r\n * Convert an object into another type\r\n *\r\n * @param object - Value of unknown type.\r\n * @param type - Name of the desired type.\r\n *\r\n * @returns Object in the desired type.\r\n * @throws Error\r\n */\r\nfunction convert(object, type) {\r\n    let match;\r\n    if (object === undefined) {\r\n        return undefined;\r\n    }\r\n    if (object === null) {\r\n        return null;\r\n    }\r\n    if (!type) {\r\n        return object;\r\n    }\r\n    if (!(typeof type === \"string\") && !(type instanceof String)) {\r\n        throw new Error(\"Type must be a string\");\r\n    }\r\n    //noinspection FallthroughInSwitchStatementJS\r\n    switch (type) {\r\n        case \"boolean\":\r\n        case \"Boolean\":\r\n            return Boolean(object);\r\n        case \"number\":\r\n        case \"Number\":\r\n            if (isString(object) && !isNaN(Date.parse(object))) {\r\n                return moment(object).valueOf();\r\n            }\r\n            else {\r\n                // @TODO: I don't think that Number and String constructors are a good idea.\r\n                // This could also fail if the object doesn't have valueOf method or if it's redefined.\r\n                // For example: Object.create(null) or { valueOf: 7 }.\r\n                return Number(object.valueOf());\r\n            }\r\n        case \"string\":\r\n        case \"String\":\r\n            return String(object);\r\n        case \"Date\":\r\n            if (isNumber(object)) {\r\n                return new Date(object);\r\n            }\r\n            if (object instanceof Date) {\r\n                return new Date(object.valueOf());\r\n            }\r\n            else if (isMoment(object)) {\r\n                return new Date(object.valueOf());\r\n            }\r\n            if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return new Date(Number(match[1])); // parse number\r\n                }\r\n                else {\r\n                    return moment(new Date(object)).toDate(); // parse string\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" + getType(object) + \" to type Date\");\r\n            }\r\n        case \"Moment\":\r\n            if (isNumber(object)) {\r\n                return moment(object);\r\n            }\r\n            if (object instanceof Date) {\r\n                return moment(object.valueOf());\r\n            }\r\n            else if (isMoment(object)) {\r\n                return moment(object);\r\n            }\r\n            if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return moment(Number(match[1])); // parse number\r\n                }\r\n                else {\r\n                    return moment(object); // parse string\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" + getType(object) + \" to type Date\");\r\n            }\r\n        case \"ISODate\":\r\n            if (isNumber(object)) {\r\n                return new Date(object);\r\n            }\r\n            else if (object instanceof Date) {\r\n                return object.toISOString();\r\n            }\r\n            else if (isMoment(object)) {\r\n                return object.toDate().toISOString();\r\n            }\r\n            else if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return new Date(Number(match[1])).toISOString(); // parse number\r\n                }\r\n                else {\r\n                    return moment(object).format(); // ISO 8601\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" +\r\n                    getType(object) +\r\n                    \" to type ISODate\");\r\n            }\r\n        case \"ASPDate\":\r\n            if (isNumber(object)) {\r\n                return \"/Date(\" + object + \")/\";\r\n            }\r\n            else if (object instanceof Date || isMoment(object)) {\r\n                return \"/Date(\" + object.valueOf() + \")/\";\r\n            }\r\n            else if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                let value;\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    value = new Date(Number(match[1])).valueOf(); // parse number\r\n                }\r\n                else {\r\n                    value = new Date(object).valueOf(); // parse string\r\n                }\r\n                return \"/Date(\" + value + \")/\";\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" +\r\n                    getType(object) +\r\n                    \" to type ASPDate\");\r\n            }\r\n        default:\r\n            const never = type;\r\n            throw new Error(`Unknown type ${never}`);\r\n    }\r\n}\n\n/**\r\n * Determine whether a value can be used as an id.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if the value is valid id, false otherwise.\r\n */\r\nfunction isId(value) {\r\n    return typeof value === \"string\" || typeof value === \"number\";\r\n}\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\n/**\r\n * A queue.\r\n *\r\n * @typeParam T - The type of method names to be replaced by queued versions.\r\n */\r\nclass Queue {\r\n    /**\r\n     * Construct a new Queue.\r\n     *\r\n     * @param options - Queue configuration.\r\n     */\r\n    constructor(options) {\r\n        this._queue = [];\r\n        this._timeout = null;\r\n        this._extended = null;\r\n        // options\r\n        this.delay = null;\r\n        this.max = Infinity;\r\n        this.setOptions(options);\r\n    }\r\n    /**\r\n     * Update the configuration of the queue.\r\n     *\r\n     * @param options - Queue configuration.\r\n     */\r\n    setOptions(options) {\r\n        if (options && typeof options.delay !== \"undefined\") {\r\n            this.delay = options.delay;\r\n        }\r\n        if (options && typeof options.max !== \"undefined\") {\r\n            this.max = options.max;\r\n        }\r\n        this._flushIfNeeded();\r\n    }\r\n    /**\r\n     * Extend an object with queuing functionality.\r\n     * The object will be extended with a function flush, and the methods provided in options.replace will be replaced with queued ones.\r\n     *\r\n     * @param object - The object to be extended.\r\n     * @param options - Additional options.\r\n     *\r\n     * @returns The created queue.\r\n     */\r\n    static extend(object, options) {\r\n        const queue = new Queue(options);\r\n        if (object.flush !== undefined) {\r\n            throw new Error(\"Target object already has a property flush\");\r\n        }\r\n        object.flush = () => {\r\n            queue.flush();\r\n        };\r\n        const methods = [\r\n            {\r\n                name: \"flush\",\r\n                original: undefined\r\n            }\r\n        ];\r\n        if (options && options.replace) {\r\n            for (let i = 0; i < options.replace.length; i++) {\r\n                const name = options.replace[i];\r\n                methods.push({\r\n                    name: name,\r\n                    // @TODO: better solution?\r\n                    original: object[name]\r\n                });\r\n                // @TODO: better solution?\r\n                queue.replace(object, name);\r\n            }\r\n        }\r\n        queue._extended = {\r\n            object: object,\r\n            methods: methods\r\n        };\r\n        return queue;\r\n    }\r\n    /**\r\n     * Destroy the queue. The queue will first flush all queued actions, and in case it has extended an object, will restore the original object.\r\n     */\r\n    destroy() {\r\n        this.flush();\r\n        if (this._extended) {\r\n            const object = this._extended.object;\r\n            const methods = this._extended.methods;\r\n            for (let i = 0; i < methods.length; i++) {\r\n                const method = methods[i];\r\n                if (method.original) {\r\n                    // @TODO: better solution?\r\n                    object[method.name] = method.original;\r\n                }\r\n                else {\r\n                    // @TODO: better solution?\r\n                    delete object[method.name];\r\n                }\r\n            }\r\n            this._extended = null;\r\n        }\r\n    }\r\n    /**\r\n     * Replace a method on an object with a queued version.\r\n     *\r\n     * @param object - Object having the method.\r\n     * @param method - The method name.\r\n     */\r\n    replace(object, method) {\r\n        /* eslint-disable-next-line @typescript-eslint/no-this-alias */\r\n        const me = this;\r\n        const original = object[method];\r\n        if (!original) {\r\n            throw new Error(\"Method \" + method + \" undefined\");\r\n        }\r\n        object[method] = function (...args) {\r\n            // add this call to the queue\r\n            me.queue({\r\n                args: args,\r\n                fn: original,\r\n                context: this\r\n            });\r\n        };\r\n    }\r\n    /**\r\n     * Queue a call.\r\n     *\r\n     * @param entry - The function or entry to be queued.\r\n     */\r\n    queue(entry) {\r\n        if (typeof entry === \"function\") {\r\n            this._queue.push({ fn: entry });\r\n        }\r\n        else {\r\n            this._queue.push(entry);\r\n        }\r\n        this._flushIfNeeded();\r\n    }\r\n    /**\r\n     * Check whether the queue needs to be flushed.\r\n     */\r\n    _flushIfNeeded() {\r\n        // flush when the maximum is exceeded.\r\n        if (this._queue.length > this.max) {\r\n            this.flush();\r\n        }\r\n        // flush after a period of inactivity when a delay is configured\r\n        if (this._timeout != null) {\r\n            clearTimeout(this._timeout);\r\n            this._timeout = null;\r\n        }\r\n        if (this.queue.length > 0 && typeof this.delay === \"number\") {\r\n            this._timeout = setTimeout(() => {\r\n                this.flush();\r\n            }, this.delay);\r\n        }\r\n    }\r\n    /**\r\n     * Flush all queued calls\r\n     */\r\n    flush() {\r\n        this._queue.splice(0).forEach((entry) => {\r\n            entry.fn.apply(entry.context || entry.fn, entry.args || []);\r\n        });\r\n    }\r\n}\n\n/* eslint-disable @typescript-eslint/member-ordering */\r\n/**\r\n * [[DataSet]] code that can be reused in [[DataView]] or other similar implementations of [[DataInterface]].\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataSetPart {\r\n    constructor() {\r\n        this._subscribers = {\r\n            \"*\": [],\r\n            add: [],\r\n            remove: [],\r\n            update: []\r\n        };\r\n        /**\r\n         * @deprecated Use on instead (PS: DataView.subscribe === DataView.on).\r\n         */\r\n        this.subscribe = DataSetPart.prototype.on;\r\n        /**\r\n         * @deprecated Use off instead (PS: DataView.unsubscribe === DataView.off).\r\n         */\r\n        this.unsubscribe = DataSetPart.prototype.off;\r\n    }\r\n    /**\r\n     * Trigger an event\r\n     *\r\n     * @param event - Event name.\r\n     * @param payload - Event payload.\r\n     * @param senderId - Id of the sender.\r\n     */\r\n    _trigger(event, payload, senderId) {\r\n        if (event === \"*\") {\r\n            throw new Error(\"Cannot trigger event *\");\r\n        }\r\n        [...this._subscribers[event], ...this._subscribers[\"*\"]].forEach((subscriber) => {\r\n            subscriber(event, payload, senderId != null ? senderId : null);\r\n        });\r\n    }\r\n    /**\r\n     * Subscribe to an event, add an event listener.\r\n     *\r\n     * @remarks Non-function callbacks are ignored.\r\n     *\r\n     * @param event - Event name.\r\n     * @param callback - Callback method.\r\n     */\r\n    on(event, callback) {\r\n        if (typeof callback === \"function\") {\r\n            this._subscribers[event].push(callback);\r\n        }\r\n        // @TODO: Maybe throw for invalid callbacks?\r\n    }\r\n    /**\r\n     * Unsubscribe from an event, remove an event listener.\r\n     *\r\n     * @remarks If the same callback was subscribed more than once **all** occurences will be removed.\r\n     *\r\n     * @param event - Event name.\r\n     * @param callback - Callback method.\r\n     */\r\n    off(event, callback) {\r\n        this._subscribers[event] = this._subscribers[event].filter((subscriber) => subscriber !== callback);\r\n    }\r\n}\n\n/**\r\n * Data stream\r\n *\r\n * @remarks\r\n * [[DataStream]] offers an always up to date stream of items from a [[DataSet]] or [[DataView]].\r\n * That means that the stream is evaluated at the time of iteration, conversion to another data type or when [[cache]] is called, not when the [[DataStream]] was created.\r\n * Multiple invocations of for example [[toItemArray]] may yield different results (if the data source like for example [[DataSet]] gets modified).\r\n *\r\n * @typeparam Item - The item type this stream is going to work with.\r\n */\r\nclass DataStream {\r\n    /**\r\n     * Create a new data stream.\r\n     *\r\n     * @param _pairs - The id, item pairs.\r\n     */\r\n    constructor(_pairs) {\r\n        this._pairs = _pairs;\r\n    }\r\n    /**\r\n     * Return an iterable of key, value pairs for every entry in the stream.\r\n     */\r\n    *[Symbol.iterator]() {\r\n        for (const [id, item] of this._pairs) {\r\n            yield [id, item];\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of key, value pairs for every entry in the stream.\r\n     */\r\n    *entries() {\r\n        for (const [id, item] of this._pairs) {\r\n            yield [id, item];\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of keys in the stream.\r\n     */\r\n    *keys() {\r\n        for (const [id] of this._pairs) {\r\n            yield id;\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of values in the stream.\r\n     */\r\n    *values() {\r\n        for (const [, item] of this._pairs) {\r\n            yield item;\r\n        }\r\n    }\r\n    /**\r\n     * Return an array containing all the ids in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all ids from this stream.\r\n     */\r\n    toIdArray() {\r\n        return [...this._pairs].map((pair) => pair[0]);\r\n    }\r\n    /**\r\n     * Return an array containing all the items in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all items from this stream.\r\n     */\r\n    toItemArray() {\r\n        return [...this._pairs].map((pair) => pair[1]);\r\n    }\r\n    /**\r\n     * Return an array containing all the entries in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all entries from this stream.\r\n     */\r\n    toEntryArray() {\r\n        return [...this._pairs];\r\n    }\r\n    /**\r\n     * Return an object map containing all the items in this stream accessible by ids.\r\n     *\r\n     * @remarks\r\n     * In case of duplicate ids (coerced to string so `7 == '7'`) the last encoutered appears in the returned object.\r\n     *\r\n     * @returns The object map of all id → item pairs from this stream.\r\n     */\r\n    toObjectMap() {\r\n        const map = Object.create(null);\r\n        for (const [id, item] of this._pairs) {\r\n            map[id] = item;\r\n        }\r\n        return map;\r\n    }\r\n    /**\r\n     * Return a map containing all the items in this stream accessible by ids.\r\n     *\r\n     * @returns The map of all id → item pairs from this stream.\r\n     */\r\n    toMap() {\r\n        return new Map(this._pairs);\r\n    }\r\n    /**\r\n     * Return a set containing all the (unique) ids in this stream.\r\n     *\r\n     * @returns The set of all ids from this stream.\r\n     */\r\n    toIdSet() {\r\n        return new Set(this.toIdArray());\r\n    }\r\n    /**\r\n     * Return a set containing all the (unique) items in this stream.\r\n     *\r\n     * @returns The set of all items from this stream.\r\n     */\r\n    toItemSet() {\r\n        return new Set(this.toItemArray());\r\n    }\r\n    /**\r\n     * Cache the items from this stream.\r\n     *\r\n     * @remarks\r\n     * This method allows for items to be fetched immediatelly and used (possibly multiple times) later.\r\n     * It can also be used to optimize performance as [[DataStream]] would otherwise reevaluate everything upon each iteration.\r\n     *\r\n     * ## Example\r\n     * ```javascript\r\n     * const ds = new DataSet([…])\r\n     *\r\n     * const cachedStream = ds.stream()\r\n     *   .filter(…)\r\n     *   .sort(…)\r\n     *   .map(…)\r\n     *   .cached(…) // Data are fetched, processed and cached here.\r\n     *\r\n     * ds.clear()\r\n     * chachedStream // Still has all the items.\r\n     * ```\r\n     *\r\n     * @returns A new [[DataStream]] with cached items (detached from the original [[DataSet]]).\r\n     */\r\n    cache() {\r\n        return new DataStream([...this._pairs]);\r\n    }\r\n    /**\r\n     * Get the distinct values of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @typeparam T - The type of the distinct value.\r\n     *\r\n     * @returns A set of all distinct properties.\r\n     */\r\n    distinct(callback) {\r\n        const set = new Set();\r\n        for (const [id, item] of this._pairs) {\r\n            set.add(callback(item, id));\r\n        }\r\n        return set;\r\n    }\r\n    /**\r\n     * Filter the items of the stream.\r\n     *\r\n     * @param callback - The function that decides whether an item will be included.\r\n     *\r\n     * @returns A new data stream with the filtered items.\r\n     */\r\n    filter(callback) {\r\n        const pairs = this._pairs;\r\n        return new DataStream({\r\n            *[Symbol.iterator]() {\r\n                for (const [id, item] of pairs) {\r\n                    if (callback(item, id)) {\r\n                        yield [id, item];\r\n                    }\r\n                }\r\n            }\r\n        });\r\n    }\r\n    /**\r\n     * Execute a callback for each item of the stream.\r\n     *\r\n     * @param callback - The function that will be invoked for each item.\r\n     */\r\n    forEach(callback) {\r\n        for (const [id, item] of this._pairs) {\r\n            callback(item, id);\r\n        }\r\n    }\r\n    /**\r\n     * Map the items into a different type.\r\n     *\r\n     * @param callback - The function that does the conversion.\r\n     *\r\n     * @typeparam Mapped - The type of the item after mapping.\r\n     *\r\n     * @returns A new data stream with the mapped items.\r\n     */\r\n    map(callback) {\r\n        const pairs = this._pairs;\r\n        return new DataStream({\r\n            *[Symbol.iterator]() {\r\n                for (const [id, item] of pairs) {\r\n                    yield [id, callback(item, id)];\r\n                }\r\n            }\r\n        });\r\n    }\r\n    /**\r\n     * Get the item with the maximum value of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @returns The item with the maximum if found otherwise null.\r\n     */\r\n    max(callback) {\r\n        const iter = this._pairs[Symbol.iterator]();\r\n        let curr = iter.next();\r\n        if (curr.done) {\r\n            return null;\r\n        }\r\n        let maxItem = curr.value[1];\r\n        let maxValue = callback(curr.value[1], curr.value[0]);\r\n        while (!(curr = iter.next()).done) {\r\n            const [id, item] = curr.value;\r\n            const value = callback(item, id);\r\n            if (value > maxValue) {\r\n                maxValue = value;\r\n                maxItem = item;\r\n            }\r\n        }\r\n        return maxItem;\r\n    }\r\n    /**\r\n     * Get the item with the minimum value of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @returns The item with the minimum if found otherwise null.\r\n     */\r\n    min(callback) {\r\n        const iter = this._pairs[Symbol.iterator]();\r\n        let curr = iter.next();\r\n        if (curr.done) {\r\n            return null;\r\n        }\r\n        let minItem = curr.value[1];\r\n        let minValue = callback(curr.value[1], curr.value[0]);\r\n        while (!(curr = iter.next()).done) {\r\n            const [id, item] = curr.value;\r\n            const value = callback(item, id);\r\n            if (value < minValue) {\r\n                minValue = value;\r\n                minItem = item;\r\n            }\r\n        }\r\n        return minItem;\r\n    }\r\n    /**\r\n     * Reduce the items into a single value.\r\n     *\r\n     * @param callback - The function that does the reduction.\r\n     * @param accumulator - The initial value of the accumulator.\r\n     *\r\n     * @typeparam T - The type of the accumulated value.\r\n     *\r\n     * @returns The reduced value.\r\n     */\r\n    reduce(callback, accumulator) {\r\n        for (const [id, item] of this._pairs) {\r\n            accumulator = callback(accumulator, item, id);\r\n        }\r\n        return accumulator;\r\n    }\r\n    /**\r\n     * Sort the items.\r\n     *\r\n     * @param callback - Item comparator.\r\n     *\r\n     * @returns A new stream with sorted items.\r\n     */\r\n    sort(callback) {\r\n        return new DataStream({\r\n            [Symbol.iterator]: () => [...this._pairs]\r\n                .sort(([idA, itemA], [idB, itemB]) => callback(itemA, itemB, idA, idB))[Symbol.iterator]()\r\n        });\r\n    }\r\n}\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\nconst warnTypeCorectionDeprecation = () => {\r\n    console.warn(\"Type coercion has been deprecated. \" +\r\n        \"Please, use data pipes instead. \" +\r\n        \"See https://visjs.github.io/vis-data/data/datapipe.html#TypeCoercion for more details with working migration example.\");\r\n};\r\n/**\r\n * # DataSet\r\n *\r\n * Vis.js comes with a flexible DataSet, which can be used to hold and manipulate unstructured data and listen for changes in the data. The DataSet is key/value based. Data items can be added, updated and removed from the DataSet, and one can subscribe to changes in the DataSet. The data in the DataSet can be filtered and ordered, and fields (like dates) can be converted to a specific type. Data can be normalized when appending it to the DataSet as well.\r\n *\r\n * ## Example\r\n *\r\n * The following example shows how to use a DataSet.\r\n *\r\n * ```javascript\r\n * // create a DataSet\r\n * var options = {};\r\n * var data = new vis.DataSet(options);\r\n *\r\n * // add items\r\n * // note that the data items can contain different properties and data formats\r\n * data.add([\r\n *   {id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true},\r\n *   {id: 2, text: 'item 2', date: '2013-06-23', group: 2},\r\n *   {id: 3, text: 'item 3', date: '2013-06-25', group: 2},\r\n *   {id: 4, text: 'item 4'}\r\n * ]);\r\n *\r\n * // subscribe to any change in the DataSet\r\n * data.on('*', function (event, properties, senderId) {\r\n *   console.log('event', event, properties);\r\n * });\r\n *\r\n * // update an existing item\r\n * data.update({id: 2, group: 1});\r\n *\r\n * // remove an item\r\n * data.remove(4);\r\n *\r\n * // get all ids\r\n * var ids = data.getIds();\r\n * console.log('ids', ids);\r\n *\r\n * // get a specific item\r\n * var item1 = data.get(1);\r\n * console.log('item1', item1);\r\n *\r\n * // retrieve a filtered subset of the data\r\n * var items = data.get({\r\n *   filter: function (item) {\r\n *     return item.group == 1;\r\n *   }\r\n * });\r\n * console.log('filtered items', items);\r\n * ```\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataSet extends DataSetPart {\r\n    /**\r\n     * Construct a new DataSet.\r\n     *\r\n     * @param data - Initial data or options.\r\n     * @param options - Options (type error if data is also options).\r\n     */\r\n    constructor(data, options) {\r\n        super();\r\n        // correctly read optional arguments\r\n        if (data && !Array.isArray(data)) {\r\n            options = data;\r\n            data = [];\r\n        }\r\n        this._options = options || {};\r\n        this._data = new Map(); // map with data indexed by id\r\n        this.length = 0; // number of items in the DataSet\r\n        this._idProp = this._options.fieldId || \"id\"; // name of the field containing id\r\n        this._type = {}; // internal field types (NOTE: this can differ from this._options.type)\r\n        // all variants of a Date are internally stored as Date, so we can convert\r\n        // from everything to everything (also from ISODate to Number for example)\r\n        if (this._options.type) {\r\n            warnTypeCorectionDeprecation();\r\n            const fields = Object.keys(this._options.type);\r\n            for (let i = 0, len = fields.length; i < len; i++) {\r\n                const field = fields[i];\r\n                const value = this._options.type[field];\r\n                if (value == \"Date\" || value == \"ISODate\" || value == \"ASPDate\") {\r\n                    this._type[field] = \"Date\";\r\n                }\r\n                else {\r\n                    this._type[field] = value;\r\n                }\r\n            }\r\n        }\r\n        // add initial data when provided\r\n        if (data && data.length) {\r\n            this.add(data);\r\n        }\r\n        this.setOptions(options);\r\n    }\r\n    /**\r\n     * Set new options.\r\n     *\r\n     * @param options - The new options.\r\n     */\r\n    setOptions(options) {\r\n        if (options && options.queue !== undefined) {\r\n            if (options.queue === false) {\r\n                // delete queue if loaded\r\n                if (this._queue) {\r\n                    this._queue.destroy();\r\n                    delete this._queue;\r\n                }\r\n            }\r\n            else {\r\n                // create queue and update its options\r\n                if (!this._queue) {\r\n                    this._queue = Queue.extend(this, {\r\n                        replace: [\"add\", \"update\", \"remove\"]\r\n                    });\r\n                }\r\n                if (options.queue && typeof options.queue === \"object\") {\r\n                    this._queue.setOptions(options.queue);\r\n                }\r\n            }\r\n        }\r\n    }\r\n    /**\r\n     * Add a data item or an array with items.\r\n     *\r\n     * After the items are added to the DataSet, the DataSet will trigger an event `add`. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet()\r\n     *\r\n     * // add items\r\n     * const ids = data.add([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { text: 'item without an id' }\r\n     * ])\r\n     *\r\n     * console.log(ids) // [1, 2, '<UUIDv4>']\r\n     * ```\r\n     *\r\n     * @param data - Items to be added (ids will be generated if missing).\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns addedIds - Array with the ids (generated if not present) of the added items.\r\n     *\r\n     * @throws When an item with the same id as any of the added items already exists.\r\n     */\r\n    add(data, senderId) {\r\n        const addedIds = [];\r\n        let id;\r\n        if (Array.isArray(data)) {\r\n            // Array\r\n            const idsToAdd = data.map(d => d[this._idProp]);\r\n            if (idsToAdd.some(id => this._data.has(id))) {\r\n                throw new Error(\"A duplicate id was found in the parameter array.\");\r\n            }\r\n            for (let i = 0, len = data.length; i < len; i++) {\r\n                id = this._addItem(data[i]);\r\n                addedIds.push(id);\r\n            }\r\n        }\r\n        else if (data && typeof data === \"object\") {\r\n            // Single item\r\n            id = this._addItem(data);\r\n            addedIds.push(id);\r\n        }\r\n        else {\r\n            throw new Error(\"Unknown dataType\");\r\n        }\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        return addedIds;\r\n    }\r\n    /**\r\n     * Update existing items. When an item does not exist, it will be created.\r\n     *\r\n     * @remarks\r\n     * The provided properties will be merged in the existing item. When an item does not exist, it will be created.\r\n     *\r\n     * After the items are updated, the DataSet will trigger an event `add` for the added items, and an event `update`. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' }\r\n     * ])\r\n     *\r\n     * // update items\r\n     * const ids = data.update([\r\n     *   { id: 2, text: 'item 2 (updated)' },\r\n     *   { id: 4, text: 'item 4 (new)' }\r\n     * ])\r\n     *\r\n     * console.log(ids) // [2, 4]\r\n     * ```\r\n     *\r\n     * ## Warning for TypeScript users\r\n     * This method may introduce partial items into the data set. Use add or updateOnly instead for better type safety.\r\n     *\r\n     * @param data - Items to be updated (if the id is already present) or added (if the id is missing).\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns updatedIds - The ids of the added (these may be newly generated if there was no id in the item from the data) or updated items.\r\n     *\r\n     * @throws When the supplied data is neither an item nor an array of items.\r\n     */\r\n    update(data, senderId) {\r\n        const addedIds = [];\r\n        const updatedIds = [];\r\n        const oldData = [];\r\n        const updatedData = [];\r\n        const idProp = this._idProp;\r\n        const addOrUpdate = (item) => {\r\n            const origId = item[idProp];\r\n            if (origId != null && this._data.has(origId)) {\r\n                const fullItem = item; // it has an id, therefore it is a fullitem\r\n                const oldItem = Object.assign({}, this._data.get(origId));\r\n                // update item\r\n                const id = this._updateItem(fullItem);\r\n                updatedIds.push(id);\r\n                updatedData.push(fullItem);\r\n                oldData.push(oldItem);\r\n            }\r\n            else {\r\n                // add new item\r\n                const id = this._addItem(item);\r\n                addedIds.push(id);\r\n            }\r\n        };\r\n        if (Array.isArray(data)) {\r\n            // Array\r\n            for (let i = 0, len = data.length; i < len; i++) {\r\n                if (data[i] && typeof data[i] === \"object\") {\r\n                    addOrUpdate(data[i]);\r\n                }\r\n                else {\r\n                    console.warn(\"Ignoring input item, which is not an object at index \" + i);\r\n                }\r\n            }\r\n        }\r\n        else if (data && typeof data === \"object\") {\r\n            // Single item\r\n            addOrUpdate(data);\r\n        }\r\n        else {\r\n            throw new Error(\"Unknown dataType\");\r\n        }\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        if (updatedIds.length) {\r\n            const props = { items: updatedIds, oldData: oldData, data: updatedData };\r\n            // TODO: remove deprecated property 'data' some day\r\n            //Object.defineProperty(props, 'data', {\r\n            //  'get': (function() {\r\n            //    console.warn('Property data is deprecated. Use DataSet.get(ids) to retrieve the new data, use the oldData property on this object to get the old data');\r\n            //    return updatedData;\r\n            //  }).bind(this)\r\n            //});\r\n            this._trigger(\"update\", props, senderId);\r\n        }\r\n        return addedIds.concat(updatedIds);\r\n    }\r\n    /**\r\n     * Update existing items. When an item does not exist, an error will be thrown.\r\n     *\r\n     * @remarks\r\n     * The provided properties will be deeply merged into the existing item.\r\n     * When an item does not exist (id not present in the data set or absent), an error will be thrown and nothing will be changed.\r\n     *\r\n     * After the items are updated, the DataSet will trigger an event `update`.\r\n     * When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' },\r\n     * ])\r\n     *\r\n     * // update items\r\n     * const ids = data.update([\r\n     *   { id: 2, text: 'item 2 (updated)' }, // works\r\n     *   // { id: 4, text: 'item 4 (new)' }, // would throw\r\n     *   // { text: 'item 4 (new)' }, // would also throw\r\n     * ])\r\n     *\r\n     * console.log(ids) // [2]\r\n     * ```\r\n     *\r\n     * @param data - Updates (the id and optionally other props) to the items in this data set.\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns updatedIds - The ids of the updated items.\r\n     *\r\n     * @throws When the supplied data is neither an item nor an array of items, when the ids are missing.\r\n     */\r\n    updateOnly(data, senderId) {\r\n        if (!Array.isArray(data)) {\r\n            data = [data];\r\n        }\r\n        const updateEventData = data\r\n            .map((update) => {\r\n            const oldData = this._data.get(update[this._idProp]);\r\n            if (oldData == null) {\r\n                throw new Error(\"Updating non-existent items is not allowed.\");\r\n            }\r\n            return { oldData, update };\r\n        })\r\n            .map(({ oldData, update }) => {\r\n            const id = oldData[this._idProp];\r\n            const updatedData = deepExtend(deepExtend({}, oldData), update);\r\n            this._data.set(id, updatedData);\r\n            return {\r\n                id,\r\n                oldData: oldData,\r\n                updatedData\r\n            };\r\n        });\r\n        if (updateEventData.length) {\r\n            const props = {\r\n                items: updateEventData.map((value) => value.id),\r\n                oldData: updateEventData.map((value) => value.oldData),\r\n                data: updateEventData.map((value) => value.updatedData)\r\n            };\r\n            // TODO: remove deprecated property 'data' some day\r\n            //Object.defineProperty(props, 'data', {\r\n            //  'get': (function() {\r\n            //    console.warn('Property data is deprecated. Use DataSet.get(ids) to retrieve the new data, use the oldData property on this object to get the old data');\r\n            //    return updatedData;\r\n            //  }).bind(this)\r\n            //});\r\n            this._trigger(\"update\", props, senderId);\r\n            return props.items;\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    get(first, second) {\r\n        // @TODO: Woudn't it be better to split this into multiple methods?\r\n        // parse the arguments\r\n        let id = undefined;\r\n        let ids = undefined;\r\n        let options = undefined;\r\n        if (isId(first)) {\r\n            // get(id [, options])\r\n            id = first;\r\n            options = second;\r\n        }\r\n        else if (Array.isArray(first)) {\r\n            // get(ids [, options])\r\n            ids = first;\r\n            options = second;\r\n        }\r\n        else {\r\n            // get([, options])\r\n            options = first;\r\n        }\r\n        // determine the return type\r\n        const returnType = options && options.returnType === \"Object\" ? \"Object\" : \"Array\";\r\n        // @TODO: WTF is this? Or am I missing something?\r\n        // var returnType\r\n        // if (options && options.returnType) {\r\n        //   var allowedValues = ['Array', 'Object']\r\n        //   returnType =\r\n        //     allowedValues.indexOf(options.returnType) == -1\r\n        //       ? 'Array'\r\n        //       : options.returnType\r\n        // } else {\r\n        //   returnType = 'Array'\r\n        // }\r\n        // build options\r\n        const type = (options && options.type) || this._options.type;\r\n        const filter = options && options.filter;\r\n        const items = [];\r\n        let item = null;\r\n        let itemIds = null;\r\n        let itemId = null;\r\n        // convert items\r\n        if (id != null) {\r\n            // return a single item\r\n            item = this._getItem(id, type);\r\n            if (item && filter && !filter(item)) {\r\n                item = null;\r\n            }\r\n        }\r\n        else if (ids != null) {\r\n            // return a subset of items\r\n            for (let i = 0, len = ids.length; i < len; i++) {\r\n                item = this._getItem(ids[i], type);\r\n                if (item != null && (!filter || filter(item))) {\r\n                    items.push(item);\r\n                }\r\n            }\r\n        }\r\n        else {\r\n            // return all items\r\n            itemIds = [...this._data.keys()];\r\n            for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                itemId = itemIds[i];\r\n                item = this._getItem(itemId, type);\r\n                if (item != null && (!filter || filter(item))) {\r\n                    items.push(item);\r\n                }\r\n            }\r\n        }\r\n        // order the results\r\n        if (options && options.order && id == undefined) {\r\n            this._sort(items, options.order);\r\n        }\r\n        // filter fields of the items\r\n        if (options && options.fields) {\r\n            const fields = options.fields;\r\n            if (id != undefined && item != null) {\r\n                item = this._filterFields(item, fields);\r\n            }\r\n            else {\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    items[i] = this._filterFields(items[i], fields);\r\n                }\r\n            }\r\n        }\r\n        // return the results\r\n        if (returnType == \"Object\") {\r\n            const result = {};\r\n            for (let i = 0, len = items.length; i < len; i++) {\r\n                const resultant = items[i];\r\n                // @TODO: Shoudn't this be this._fieldId?\r\n                // result[resultant.id] = resultant\r\n                const id = resultant[this._idProp];\r\n                result[id] = resultant;\r\n            }\r\n            return result;\r\n        }\r\n        else {\r\n            if (id != null) {\r\n                // a single item\r\n                return item;\r\n            }\r\n            else {\r\n                // just return our array\r\n                return items;\r\n            }\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getIds(options) {\r\n        const data = this._data;\r\n        const filter = options && options.filter;\r\n        const order = options && options.order;\r\n        const type = (options && options.type) || this._options.type;\r\n        const itemIds = [...data.keys()];\r\n        const ids = [];\r\n        let item;\r\n        let items;\r\n        if (filter) {\r\n            // get filtered items\r\n            if (order) {\r\n                // create ordered list\r\n                items = [];\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = this._getItem(id, type);\r\n                    if (filter(item)) {\r\n                        items.push(item);\r\n                    }\r\n                }\r\n                this._sort(items, order);\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    ids.push(items[i][this._idProp]);\r\n                }\r\n            }\r\n            else {\r\n                // create unordered list\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = this._getItem(id, type);\r\n                    if (filter(item)) {\r\n                        ids.push(item[this._idProp]);\r\n                    }\r\n                }\r\n            }\r\n        }\r\n        else {\r\n            // get all items\r\n            if (order) {\r\n                // create an ordered list\r\n                items = [];\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    items.push(data.get(id));\r\n                }\r\n                this._sort(items, order);\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    ids.push(items[i][this._idProp]);\r\n                }\r\n            }\r\n            else {\r\n                // create unordered list\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = data.get(id);\r\n                    ids.push(item[this._idProp]);\r\n                }\r\n            }\r\n        }\r\n        return ids;\r\n    }\r\n    /** @inheritdoc */\r\n    getDataSet() {\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    forEach(callback, options) {\r\n        const filter = options && options.filter;\r\n        const type = (options && options.type) || this._options.type;\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        if (options && options.order) {\r\n            // execute forEach on ordered list\r\n            const items = this.get(options);\r\n            for (let i = 0, len = items.length; i < len; i++) {\r\n                const item = items[i];\r\n                const id = item[this._idProp];\r\n                callback(item, id);\r\n            }\r\n        }\r\n        else {\r\n            // unordered\r\n            for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                const id = itemIds[i];\r\n                const item = this._getItem(id, type);\r\n                if (!filter || filter(item)) {\r\n                    callback(item, id);\r\n                }\r\n            }\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    map(callback, options) {\r\n        const filter = options && options.filter;\r\n        const type = (options && options.type) || this._options.type;\r\n        const mappedItems = [];\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        // convert and filter items\r\n        for (let i = 0, len = itemIds.length; i < len; i++) {\r\n            const id = itemIds[i];\r\n            const item = this._getItem(id, type);\r\n            if (!filter || filter(item)) {\r\n                mappedItems.push(callback(item, id));\r\n            }\r\n        }\r\n        // order items\r\n        if (options && options.order) {\r\n            this._sort(mappedItems, options.order);\r\n        }\r\n        return mappedItems;\r\n    }\r\n    /**\r\n     * Filter the fields of an item.\r\n     *\r\n     * @param item - The item whose fields should be filtered.\r\n     * @param fields - The names of the fields that will be kept.\r\n     *\r\n     * @typeParam K - Field name type.\r\n     *\r\n     * @returns The item without any additional fields.\r\n     */\r\n    _filterFields(item, fields) {\r\n        if (!item) {\r\n            // item is null\r\n            return item;\r\n        }\r\n        return (Array.isArray(fields)\r\n            ? // Use the supplied array\r\n                fields\r\n            : // Use the keys of the supplied object\r\n                Object.keys(fields)).reduce((filteredItem, field) => {\r\n            filteredItem[field] = item[field];\r\n            return filteredItem;\r\n        }, {});\r\n    }\r\n    /**\r\n     * Sort the provided array with items.\r\n     *\r\n     * @param items - Items to be sorted in place.\r\n     * @param order - A field name or custom sort function.\r\n     *\r\n     * @typeParam T - The type of the items in the items array.\r\n     */\r\n    _sort(items, order) {\r\n        if (typeof order === \"string\") {\r\n            // order by provided field name\r\n            const name = order; // field name\r\n            items.sort((a, b) => {\r\n                // @TODO: How to treat missing properties?\r\n                const av = a[name];\r\n                const bv = b[name];\r\n                return av > bv ? 1 : av < bv ? -1 : 0;\r\n            });\r\n        }\r\n        else if (typeof order === \"function\") {\r\n            // order by sort function\r\n            items.sort(order);\r\n        }\r\n        else {\r\n            // TODO: extend order by an Object {field:string, direction:string}\r\n            //       where direction can be 'asc' or 'desc'\r\n            throw new TypeError(\"Order must be a function or a string\");\r\n        }\r\n    }\r\n    /**\r\n     * Remove an item or multiple items by “reference” (only the id is used) or by id.\r\n     *\r\n     * The method ignores removal of non-existing items, and returns an array containing the ids of the items which are actually removed from the DataSet.\r\n     *\r\n     * After the items are removed, the DataSet will trigger an event `remove` for the removed items. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' }\r\n     * ])\r\n     *\r\n     * // remove items\r\n     * const ids = data.remove([2, { id: 3 }, 4])\r\n     *\r\n     * console.log(ids) // [2, 3]\r\n     * ```\r\n     *\r\n     * @param id - One or more items or ids of items to be removed.\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns The ids of the removed items.\r\n     */\r\n    remove(id, senderId) {\r\n        const removedIds = [];\r\n        const removedItems = [];\r\n        // force everything to be an array for simplicity\r\n        const ids = Array.isArray(id) ? id : [id];\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            const item = this._remove(ids[i]);\r\n            if (item) {\r\n                const itemId = item[this._idProp];\r\n                if (itemId != null) {\r\n                    removedIds.push(itemId);\r\n                    removedItems.push(item);\r\n                }\r\n            }\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems }, senderId);\r\n        }\r\n        return removedIds;\r\n    }\r\n    /**\r\n     * Remove an item by its id or reference.\r\n     *\r\n     * @param id - Id of an item or the item itself.\r\n     *\r\n     * @returns The removed item if removed, null otherwise.\r\n     */\r\n    _remove(id) {\r\n        // @TODO: It origianlly returned the item although the docs say id.\r\n        // The code expects the item, so probably an error in the docs.\r\n        let ident;\r\n        // confirm the id to use based on the args type\r\n        if (isId(id)) {\r\n            ident = id;\r\n        }\r\n        else if (id && typeof id === \"object\") {\r\n            ident = id[this._idProp]; // look for the identifier field using ._idProp\r\n        }\r\n        // do the removing if the item is found\r\n        if (ident != null && this._data.has(ident)) {\r\n            const item = this._data.get(ident) || null;\r\n            this._data.delete(ident);\r\n            --this.length;\r\n            return item;\r\n        }\r\n        return null;\r\n    }\r\n    /**\r\n     * Clear the entire data set.\r\n     *\r\n     * After the items are removed, the [[DataSet]] will trigger an event `remove` for all removed items. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns removedIds - The ids of all removed items.\r\n     */\r\n    clear(senderId) {\r\n        const ids = [...this._data.keys()];\r\n        const items = [];\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            items.push(this._data.get(ids[i]));\r\n        }\r\n        this._data.clear();\r\n        this.length = 0;\r\n        this._trigger(\"remove\", { items: ids, oldData: items }, senderId);\r\n        return ids;\r\n    }\r\n    /**\r\n     * Find the item with maximum value of a specified field.\r\n     *\r\n     * @param field - Name of the property that should be searched for max value.\r\n     *\r\n     * @returns Item containing max value, or null if no items.\r\n     */\r\n    max(field) {\r\n        let max = null;\r\n        let maxField = null;\r\n        for (const item of this._data.values()) {\r\n            const itemField = item[field];\r\n            if (typeof itemField === \"number\" &&\r\n                (maxField == null || itemField > maxField)) {\r\n                max = item;\r\n                maxField = itemField;\r\n            }\r\n        }\r\n        return max || null;\r\n    }\r\n    /**\r\n     * Find the item with minimum value of a specified field.\r\n     *\r\n     * @param field - Name of the property that should be searched for min value.\r\n     *\r\n     * @returns Item containing min value, or null if no items.\r\n     */\r\n    min(field) {\r\n        let min = null;\r\n        let minField = null;\r\n        for (const item of this._data.values()) {\r\n            const itemField = item[field];\r\n            if (typeof itemField === \"number\" &&\r\n                (minField == null || itemField < minField)) {\r\n                min = item;\r\n                minField = itemField;\r\n            }\r\n        }\r\n        return min || null;\r\n    }\r\n    /**\r\n     * Find all distinct values of a specified field\r\n     *\r\n     * @param prop - The property name whose distinct values should be returned.\r\n     *\r\n     * @returns Unordered array containing all distinct values. Items without specified property are ignored.\r\n     */\r\n    distinct(prop) {\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        const values = [];\r\n        const fieldType = (this._options.type && this._options.type[prop]) || null;\r\n        let count = 0;\r\n        for (let i = 0, len = itemIds.length; i < len; i++) {\r\n            const id = itemIds[i];\r\n            const item = data.get(id);\r\n            const value = item[prop];\r\n            let exists = false;\r\n            for (let j = 0; j < count; j++) {\r\n                if (values[j] == value) {\r\n                    exists = true;\r\n                    break;\r\n                }\r\n            }\r\n            if (!exists && value !== undefined) {\r\n                values[count] = value;\r\n                count++;\r\n            }\r\n        }\r\n        if (fieldType) {\r\n            for (let i = 0, len = values.length; i < len; i++) {\r\n                values[i] = convert(values[i], fieldType);\r\n            }\r\n        }\r\n        return values;\r\n    }\r\n    /**\r\n     * Add a single item. Will fail when an item with the same id already exists.\r\n     *\r\n     * @param item - A new item to be added.\r\n     *\r\n     * @returns Added item's id. An id is generated when it is not present in the item.\r\n     */\r\n    _addItem(item) {\r\n        let id = item[this._idProp];\r\n        if (id != null) {\r\n            // check whether this id is already taken\r\n            if (this._data.has(id)) {\r\n                // item already exists\r\n                throw new Error(\"Cannot add item: item with id \" + id + \" already exists\");\r\n            }\r\n        }\r\n        else {\r\n            // generate an id\r\n            id = v4_1();\r\n            item[this._idProp] = id;\r\n        }\r\n        const d = {};\r\n        const fields = Object.keys(item);\r\n        for (let i = 0, len = fields.length; i < len; i++) {\r\n            const field = fields[i];\r\n            const fieldType = this._type[field]; // type may be undefined\r\n            d[field] = convert(item[field], fieldType);\r\n        }\r\n        this._data.set(id, d);\r\n        ++this.length;\r\n        return id;\r\n    }\r\n    /**\r\n     * Get an item. Fields can be converted to a specific type\r\n     *\r\n     * @param id - Id of the requested item.\r\n     * @param types - Property name to type name object map of type converstions.\r\n     *\r\n     * @returns The item, optionally after type conversion.\r\n     */\r\n    _getItem(id, types) {\r\n        // @TODO: I have no idea how to type this.\r\n        // get the item from the dataset\r\n        const raw = this._data.get(id);\r\n        if (!raw) {\r\n            return null;\r\n        }\r\n        // convert the items field types\r\n        let converted;\r\n        const fields = Object.keys(raw);\r\n        if (types) {\r\n            warnTypeCorectionDeprecation();\r\n            converted = {};\r\n            for (let i = 0, len = fields.length; i < len; i++) {\r\n                const field = fields[i];\r\n                const value = raw[field];\r\n                converted[field] = convert(value, types[field]);\r\n            }\r\n        }\r\n        else {\r\n            // no field types specified, no converting needed\r\n            converted = { ...raw };\r\n        }\r\n        if (converted[this._idProp] == null) {\r\n            converted[this._idProp] = raw.id;\r\n        }\r\n        return converted;\r\n    }\r\n    /**\r\n     * Update a single item: merge with existing item.\r\n     * Will fail when the item has no id, or when there does not exist an item with the same id.\r\n     *\r\n     * @param item - The new item\r\n     *\r\n     * @returns The id of the updated item.\r\n     */\r\n    _updateItem(item) {\r\n        const id = item[this._idProp];\r\n        if (id == null) {\r\n            throw new Error(\"Cannot update item: item has no id (item: \" +\r\n                JSON.stringify(item) +\r\n                \")\");\r\n        }\r\n        const d = this._data.get(id);\r\n        if (!d) {\r\n            // item doesn't exist\r\n            throw new Error(\"Cannot update item: no item with id \" + id + \" found\");\r\n        }\r\n        // merge with current item\r\n        const fields = Object.keys(item);\r\n        for (let i = 0, len = fields.length; i < len; i++) {\r\n            const field = fields[i];\r\n            const fieldType = this._type[field]; // type may be undefined\r\n            d[field] = convert(item[field], fieldType);\r\n        }\r\n        return id;\r\n    }\r\n    /** @inheritdoc */\r\n    stream(ids) {\r\n        if (ids) {\r\n            const data = this._data;\r\n            return new DataStream({\r\n                *[Symbol.iterator]() {\r\n                    for (const id of ids) {\r\n                        const item = data.get(id);\r\n                        if (item != null) {\r\n                            yield [id, item];\r\n                        }\r\n                    }\r\n                }\r\n            });\r\n        }\r\n        else {\r\n            return new DataStream({\r\n                [Symbol.iterator]: this._data.entries.bind(this._data)\r\n            });\r\n        }\r\n    }\r\n}\n\n/**\r\n * DataView\r\n *\r\n * A DataView offers a filtered and/or formatted view on a DataSet. One can subscribe to changes in a DataView, and easily get filtered or formatted data without having to specify filters and field types all the time.\r\n *\r\n * ## Example\r\n * ```javascript\r\n * // create a DataSet\r\n * var data = new vis.DataSet();\r\n * data.add([\r\n *   {id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true},\r\n *   {id: 2, text: 'item 2', date: '2013-06-23', group: 2},\r\n *   {id: 3, text: 'item 3', date: '2013-06-25', group: 2},\r\n *   {id: 4, text: 'item 4'}\r\n * ]);\r\n *\r\n * // create a DataView\r\n * // the view will only contain items having a property group with value 1,\r\n * // and will only output fields id, text, and date.\r\n * var view = new vis.DataView(data, {\r\n *   filter: function (item) {\r\n *     return (item.group == 1);\r\n *   },\r\n *   fields: ['id', 'text', 'date']\r\n * });\r\n *\r\n * // subscribe to any change in the DataView\r\n * view.on('*', function (event, properties, senderId) {\r\n *   console.log('event', event, properties);\r\n * });\r\n *\r\n * // update an item in the data set\r\n * data.update({id: 2, group: 1});\r\n *\r\n * // get all ids in the view\r\n * var ids = view.getIds();\r\n * console.log('ids', ids); // will output [1, 2]\r\n *\r\n * // get all items in the view\r\n * var items = view.get();\r\n * ```\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataView extends DataSetPart {\r\n    /**\r\n     * Create a DataView.\r\n     *\r\n     * @param data - The instance containing data (directly or indirectly).\r\n     * @param options - Options to configure this data view.\r\n     */\r\n    constructor(data, options) {\r\n        super();\r\n        /** @inheritdoc */\r\n        this.length = 0;\r\n        this._ids = new Set(); // ids of the items currently in memory (just contains a boolean true)\r\n        this._options = options || {};\r\n        this._listener = this._onEvent.bind(this);\r\n        this.setData(data);\r\n    }\r\n    // TODO: implement a function .config() to dynamically update things like configured filter\r\n    // and trigger changes accordingly\r\n    /**\r\n     * Set a data source for the view.\r\n     *\r\n     * @param data - The instance containing data (directly or indirectly).\r\n     *\r\n     * @remarks\r\n     * Note that when the data view is bound to a data set it won't be garbage\r\n     * collected unless the data set is too. Use `dataView.setData(null)` or\r\n     * `dataView.dispose()` to enable garbage collection before you lose the last\r\n     * reference.\r\n     */\r\n    setData(data) {\r\n        if (this._data) {\r\n            // unsubscribe from current dataset\r\n            if (this._data.off) {\r\n                this._data.off(\"*\", this._listener);\r\n            }\r\n            // trigger a remove of all items in memory\r\n            const ids = this._data.getIds({ filter: this._options.filter });\r\n            const items = this._data.get(ids);\r\n            this._ids.clear();\r\n            this.length = 0;\r\n            this._trigger(\"remove\", { items: ids, oldData: items });\r\n        }\r\n        if (data != null) {\r\n            this._data = data;\r\n            // trigger an add of all added items\r\n            const ids = this._data.getIds({ filter: this._options.filter });\r\n            for (let i = 0, len = ids.length; i < len; i++) {\r\n                const id = ids[i];\r\n                this._ids.add(id);\r\n            }\r\n            this.length = ids.length;\r\n            this._trigger(\"add\", { items: ids });\r\n        }\r\n        else {\r\n            this._data = new DataSet();\r\n        }\r\n        // subscribe to new dataset\r\n        if (this._data.on) {\r\n            this._data.on(\"*\", this._listener);\r\n        }\r\n    }\r\n    /**\r\n     * Refresh the DataView.\r\n     * Useful when the DataView has a filter function containing a variable parameter.\r\n     */\r\n    refresh() {\r\n        const ids = this._data.getIds({\r\n            filter: this._options.filter\r\n        });\r\n        const oldIds = [...this._ids];\r\n        const newIds = {};\r\n        const addedIds = [];\r\n        const removedIds = [];\r\n        const removedItems = [];\r\n        // check for additions\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            const id = ids[i];\r\n            newIds[id] = true;\r\n            if (!this._ids.has(id)) {\r\n                addedIds.push(id);\r\n                this._ids.add(id);\r\n            }\r\n        }\r\n        // check for removals\r\n        for (let i = 0, len = oldIds.length; i < len; i++) {\r\n            const id = oldIds[i];\r\n            const item = this._data.get(id);\r\n            if (item == null) {\r\n                // @TODO: Investigate.\r\n                // Doesn't happen during tests or examples.\r\n                // Is it really impossible or could it eventually happen?\r\n                // How to handle it if it does? The types guarantee non-nullable items.\r\n                console.error(\"If you see this, report it please.\");\r\n            }\r\n            else if (!newIds[id]) {\r\n                removedIds.push(id);\r\n                removedItems.push(item);\r\n                this._ids.delete(id);\r\n            }\r\n        }\r\n        this.length += addedIds.length - removedIds.length;\r\n        // trigger events\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds });\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems });\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    get(first, second) {\r\n        if (this._data == null) {\r\n            return null;\r\n        }\r\n        // parse the arguments\r\n        let ids = null;\r\n        let options;\r\n        if (isId(first) || Array.isArray(first)) {\r\n            ids = first;\r\n            options = second;\r\n        }\r\n        else {\r\n            options = first;\r\n        }\r\n        // extend the options with the default options and provided options\r\n        const viewOptions = Object.assign({}, this._options, options);\r\n        // create a combined filter method when needed\r\n        const thisFilter = this._options.filter;\r\n        const optionsFilter = options && options.filter;\r\n        if (thisFilter && optionsFilter) {\r\n            viewOptions.filter = (item) => {\r\n                return thisFilter(item) && optionsFilter(item);\r\n            };\r\n        }\r\n        if (ids == null) {\r\n            return this._data.get(viewOptions);\r\n        }\r\n        else {\r\n            return this._data.get(ids, viewOptions);\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getIds(options) {\r\n        if (this._data.length) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options != null ? options.filter : null;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = (item) => {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            return this._data.getIds({\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    forEach(callback, options) {\r\n        if (this._data) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options && options.filter;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = function (item) {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            this._data.forEach(callback, {\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    map(callback, options) {\r\n        if (this._data) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options && options.filter;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = (item) => {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            return this._data.map(callback, {\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getDataSet() {\r\n        return this._data.getDataSet();\r\n    }\r\n    /** @inheritdoc */\r\n    stream(ids) {\r\n        return this._data.stream(ids || {\r\n            [Symbol.iterator]: this._ids.keys.bind(this._ids)\r\n        });\r\n    }\r\n    /**\r\n     * Render the instance unusable prior to garbage collection.\r\n     *\r\n     * @remarks\r\n     * The intention of this method is to help discover scenarios where the data\r\n     * view is being used when the programmer thinks it has been garbage collected\r\n     * already. It's stricter version of `dataView.setData(null)`.\r\n     */\r\n    dispose() {\r\n        var _a;\r\n        if ((_a = this._data) === null || _a === void 0 ? void 0 : _a.off) {\r\n            this._data.off(\"*\", this._listener);\r\n        }\r\n        const message = \"This data view has already been disposed of.\";\r\n        Object.defineProperty(this, \"_data\", {\r\n            get: () => {\r\n                throw new Error(message);\r\n            },\r\n            set: () => {\r\n                throw new Error(message);\r\n            },\r\n            configurable: false\r\n        });\r\n    }\r\n    /**\r\n     * Event listener. Will propagate all events from the connected data set to the subscribers of the DataView, but will filter the items and only trigger when there are changes in the filtered data set.\r\n     *\r\n     * @param event - The name of the event.\r\n     * @param params - Parameters of the event.\r\n     * @param senderId - Id supplied by the sender.\r\n     */\r\n    _onEvent(event, params, senderId) {\r\n        if (!params || !params.items || !this._data) {\r\n            return;\r\n        }\r\n        const ids = params.items;\r\n        const addedIds = [];\r\n        const updatedIds = [];\r\n        const removedIds = [];\r\n        const oldItems = [];\r\n        const updatedItems = [];\r\n        const removedItems = [];\r\n        switch (event) {\r\n            case \"add\":\r\n                // filter the ids of the added items\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    const item = this.get(id);\r\n                    if (item) {\r\n                        this._ids.add(id);\r\n                        addedIds.push(id);\r\n                    }\r\n                }\r\n                break;\r\n            case \"update\":\r\n                // determine the event from the views viewpoint: an updated\r\n                // item can be added, updated, or removed from this view.\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    const item = this.get(id);\r\n                    if (item) {\r\n                        if (this._ids.has(id)) {\r\n                            updatedIds.push(id);\r\n                            updatedItems.push(params.data[i]);\r\n                            oldItems.push(params.oldData[i]);\r\n                        }\r\n                        else {\r\n                            this._ids.add(id);\r\n                            addedIds.push(id);\r\n                        }\r\n                    }\r\n                    else {\r\n                        if (this._ids.has(id)) {\r\n                            this._ids.delete(id);\r\n                            removedIds.push(id);\r\n                            removedItems.push(params.oldData[i]);\r\n                        }\r\n                    }\r\n                }\r\n                break;\r\n            case \"remove\":\r\n                // filter the ids of the removed items\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    if (this._ids.has(id)) {\r\n                        this._ids.delete(id);\r\n                        removedIds.push(id);\r\n                        removedItems.push(params.oldData[i]);\r\n                    }\r\n                }\r\n                break;\r\n        }\r\n        this.length += addedIds.length - removedIds.length;\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        if (updatedIds.length) {\r\n            this._trigger(\"update\", { items: updatedIds, oldData: oldItems, data: updatedItems }, senderId);\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems }, senderId);\r\n        }\r\n    }\r\n}\n\nexport { DataSet, DataStream, DataView, Queue, createNewDataPipeFrom };\n//# sourceMappingURL=vis-data.js.map\n","// first check if moment.js is already loaded in the browser window, if so,\n// use this instance. Else, load via commonjs.\n//\n// Note: This doesn't work in ESM.\nmodule.exports = (typeof window !== 'undefined') && window['moment'] || require('moment');\n","// utility functions\n\nexport * from \"vis-util/esnext\";\nimport * as util from \"vis-util/esnext\";\nimport { getType, isNumber, isString } from \"vis-util/esnext\";\nimport { DataSet, createNewDataPipeFrom } from \"vis-data/esnext\";\n\nimport moment from \"moment\";\n\n// parse ASP.Net Date pattern,\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\n// code from http://momentjs.com/\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\n\n/**\n * Convert an object into another type\n *\n * @param object - Value of unknown type.\n * @param type - Name of the desired type.\n *\n * @returns Object in the desired type.\n * @throws Error\n */\nexport function convert(object, type) {\n  let match;\n\n  if (object === undefined) {\n    return undefined;\n  }\n  if (object === null) {\n    return null;\n  }\n\n  if (!type) {\n    return object;\n  }\n  if (!(typeof type === \"string\") && !(type instanceof String)) {\n    throw new Error(\"Type must be a string\");\n  }\n\n  //noinspection FallthroughInSwitchStatementJS\n  switch (type) {\n    case \"boolean\":\n    case \"Boolean\":\n      return Boolean(object);\n\n    case \"number\":\n    case \"Number\":\n      if (isString(object) && !isNaN(Date.parse(object))) {\n        return moment(object).valueOf();\n      } else {\n        // @TODO: I don't think that Number and String constructors are a good idea.\n        // This could also fail if the object doesn't have valueOf method or if it's redefined.\n        // For example: Object.create(null) or { valueOf: 7 }.\n        return Number(object.valueOf());\n      }\n    case \"string\":\n    case \"String\":\n      return String(object);\n\n    case \"Date\":\n      if (isNumber(object)) {\n        return new Date(object);\n      }\n      if (object instanceof Date) {\n        return new Date(object.valueOf());\n      } else if (moment.isMoment(object)) {\n        return new Date(object.valueOf());\n      }\n      if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return new Date(Number(match[1])); // parse number\n        } else {\n          return moment(new Date(object)).toDate(); // parse string\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" + getType(object) + \" to type Date\"\n        );\n      }\n\n    case \"Moment\":\n      if (isNumber(object)) {\n        return moment(object);\n      }\n      if (object instanceof Date) {\n        return moment(object.valueOf());\n      } else if (moment.isMoment(object)) {\n        return moment(object);\n      }\n      if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return moment(Number(match[1])); // parse number\n        } else {\n          return moment(object); // parse string\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" + getType(object) + \" to type Date\"\n        );\n      }\n\n    case \"ISODate\":\n      if (isNumber(object)) {\n        return new Date(object);\n      } else if (object instanceof Date) {\n        return object.toISOString();\n      } else if (moment.isMoment(object)) {\n        return object.toDate().toISOString();\n      } else if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return new Date(Number(match[1])).toISOString(); // parse number\n        } else {\n          return moment(object).format(); // ISO 8601\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" +\n            getType(object) +\n            \" to type ISODate\"\n        );\n      }\n\n    case \"ASPDate\":\n      if (isNumber(object)) {\n        return \"/Date(\" + object + \")/\";\n      } else if (object instanceof Date || moment.isMoment(object)) {\n        return \"/Date(\" + object.valueOf() + \")/\";\n      } else if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        let value;\n        if (match) {\n          // object is an ASP date\n          value = new Date(Number(match[1])).valueOf(); // parse number\n        } else {\n          value = new Date(object).valueOf(); // parse string\n        }\n        return \"/Date(\" + value + \")/\";\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" +\n            getType(object) +\n            \" to type ASPDate\"\n        );\n      }\n\n    default:\n      throw new Error(`Unknown type ${type}`);\n  }\n}\n\n/**\n * Create a Data Set like wrapper to seamlessly coerce data types.\n *\n * @param rawDS - The Data Set with raw uncoerced data.\n * @param type - A record assigning a data type to property name.\n *\n * @remarks\n * The write operations (`add`, `remove`, `update` and `updateOnly`) write into\n * the raw (uncoerced) data set. These values are then picked up by a pipe\n * which coerces the values using the [[convert]] function and feeds them into\n * the coerced data set. When querying (`forEach`, `get`, `getIds`, `off` and\n * `on`) the values are then fetched from the coerced data set and already have\n * the required data types. The values are coerced only once when inserted and\n * then the same value is returned each time until it is updated or deleted.\n *\n * For example: `typeCoercedDataSet.add({ id: 7, start: \"2020-01-21\" })` would\n * result in `typeCoercedDataSet.get(7)` returning `{ id: 7, start: moment(new\n * Date(\"2020-01-21\")).toDate() }`.\n *\n * Use the dispose method prior to throwing a reference to this away. Otherwise\n * the pipe connecting the two Data Sets will keep the unaccessible coerced\n * Data Set alive and updated as long as the raw Data Set exists.\n *\n * @returns A Data Set like object that saves data into the raw Data Set and\n * retrieves them from the coerced Data Set.\n */\nexport function typeCoerceDataSet(\n  rawDS,\n  type = { start: \"Date\", end: \"Date\" }\n) {\n  const idProp = rawDS._idProp;\n  const coercedDS = new DataSet({ fieldId: idProp });\n\n  const pipe = createNewDataPipeFrom(rawDS)\n    .map(item =>\n      Object.keys(item).reduce((acc, key) => {\n        acc[key] = convert(item[key], type[key]);\n        return acc;\n      }, {})\n    )\n    .to(coercedDS);\n\n  pipe.all().start();\n\n  return {\n    // Write only.\n    add: (...args) => rawDS.getDataSet().add(...args),\n    remove: (...args) => rawDS.getDataSet().remove(...args),\n    update: (...args) => rawDS.getDataSet().update(...args),\n    updateOnly: (...args) => rawDS.getDataSet().updateOnly(...args),\n\n    // Read only.\n    forEach: coercedDS.forEach.bind(coercedDS),\n    get: coercedDS.get.bind(coercedDS),\n    getIds: coercedDS.getIds.bind(coercedDS),\n    off: coercedDS.off.bind(coercedDS),\n    on: coercedDS.on.bind(coercedDS),\n\n    get length() {\n      return coercedDS.length;\n    },\n\n    // Non standard.\n    idProp,\n    type,\n\n    rawDS,\n    coercedDS,\n    dispose: () => pipe.stop()\n  };\n}\n\nexport default {\n  ...util,\n  convert\n};\n","var global = require('../internals/global');\nvar trim = require('../internals/string-trim').trim;\nvar whitespaces = require('../internals/whitespaces');\n\nvar $parseFloat = global.parseFloat;\nvar FORCED = 1 / $parseFloat(whitespaces + '-0') !== -Infinity;\n\n// `parseFloat` method\n// https://tc39.github.io/ecma262/#sec-parsefloat-string\nmodule.exports = FORCED ? function parseFloat(string) {\n  var trimmedString = trim(String(string));\n  var result = $parseFloat(trimmedString);\n  return result === 0 && trimmedString.charAt(0) == '-' ? -0 : result;\n} : $parseFloat;\n","var $ = require('../internals/export');\nvar parseFloatImplementation = require('../internals/number-parse-float');\n\n// `parseFloat` method\n// https://tc39.github.io/ecma262/#sec-parsefloat-string\n$({ global: true, forced: parseFloat != parseFloatImplementation }, {\n  parseFloat: parseFloatImplementation\n});\n","require('../modules/es.parse-float');\nvar path = require('../internals/path');\n\nmodule.exports = path.parseFloat;\n","import util from '../../util';\n\n/** Prototype for visual components */\nexport default class Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} [body]\n * @param {Object} [options]\n */\n  constructor(body, options) {  // eslint-disable-line no-unused-vars\n    this.options = null;\n    this.props = null;\n  }\n\n  /**\n   * Set options for the component. The new options will be merged into the\n   * current options.\n   * @param {Object} options\n   */\n  setOptions(options) {\n    if (options) {\n      util.extend(this.options, options);\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    // should be implemented by the component\n    return false;\n  }\n\n  /**\n   * Destroy the component. Cleanup DOM and event listeners\n   */\n  destroy() {\n    // should be implemented by the component\n  }\n\n  /**\n   * Test whether the component is resized since the last time _isResized() was\n   * called.\n   * @return {Boolean} Returns true if the component is resized\n   * @protected\n   */\n  _isResized() {\n    const resized = (\n      this.props._previousWidth !== this.props.width ||\n      this.props._previousHeight !== this.props.height\n    );\n\n    this.props._previousWidth = this.props.width;\n    this.props._previousHeight = this.props.height;\n\n    return resized;\n  }\n}\n","var $ = require('../internals/export');\nvar repeat = require('../internals/string-repeat');\n\n// `String.prototype.repeat` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.repeat\n$({ target: 'String', proto: true }, {\n  repeat: repeat\n});\n","'use strict';\nvar toInteger = require('../internals/to-integer');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `String.prototype.repeat` method implementation\n// https://tc39.github.io/ecma262/#sec-string.prototype.repeat\nmodule.exports = ''.repeat || function repeat(count) {\n  var str = String(requireObjectCoercible(this));\n  var result = '';\n  var n = toInteger(count);\n  if (n < 0 || n == Infinity) throw RangeError('Wrong number of repetitions');\n  for (;n > 0; (n >>>= 1) && (str += str)) if (n & 1) result += str;\n  return result;\n};\n","require('../../../modules/es.string.repeat');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').repeat;\n","var repeat = require('../string/virtual/repeat');\n\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.repeat;\n  return typeof it === 'string' || it === StringPrototype\n    || (it instanceof String && own === StringPrototype.repeat) ? repeat : own;\n};\n","\n/**\n * used in Core to convert the options into a volatile variable\n * \n * @param {function} moment\n * @param {Object} body\n * @param {Array | Object} hiddenDates\n * @returns {number}\n */\nexport function convertHiddenOptions(moment, body, hiddenDates) {\n  if (hiddenDates && !Array.isArray(hiddenDates)) {\n    return convertHiddenOptions(moment, body, [hiddenDates])\n  }\n\n  body.hiddenDates = [];\n  if (hiddenDates) {\n    if (Array.isArray(hiddenDates) == true) {\n      for (let i = 0; i < hiddenDates.length; i++) {\n        if (hiddenDates[i].repeat === undefined) {\n          const dateItem = {};\n          dateItem.start = moment(hiddenDates[i].start).toDate().valueOf();\n          dateItem.end = moment(hiddenDates[i].end).toDate().valueOf();\n          body.hiddenDates.push(dateItem);\n        }\n      }\n      body.hiddenDates.sort((a, b) => a.start - b.start); // sort by start time\n    }\n  }\n}\n\n/**\n * create new entrees for the repeating hidden dates\n *\n * @param {function} moment\n * @param {Object} body\n * @param {Array | Object} hiddenDates\n * @returns {null}\n */\nexport function updateHiddenDates(moment, body, hiddenDates) {\n  if (hiddenDates && !Array.isArray(hiddenDates)) {\n    return updateHiddenDates(moment, body, [hiddenDates])\n  }\n\n  if (hiddenDates && body.domProps.centerContainer.width !== undefined) {\n    convertHiddenOptions(moment, body, hiddenDates);\n\n    const start = moment(body.range.start);\n    const end = moment(body.range.end);\n\n    const totalRange = (body.range.end - body.range.start);\n    const pixelTime = totalRange / body.domProps.centerContainer.width;\n\n    for (let i = 0; i < hiddenDates.length; i++) {\n      if (hiddenDates[i].repeat !== undefined) {\n        const startDate = moment(hiddenDates[i].start);\n        let endDate = moment(hiddenDates[i].end);\n\n        if (startDate._d == \"Invalid Date\") {\n          throw new Error(`Supplied start date is not valid: ${hiddenDates[i].start}`);\n        }\n        if (endDate._d == \"Invalid Date\") {\n          throw new Error(`Supplied end date is not valid: ${hiddenDates[i].end}`);\n        }\n\n        const duration = endDate - startDate;\n        if (duration >= 4 * pixelTime) {\n\n          let offset = 0;\n          const runUntil = end.clone();\n          switch (hiddenDates[i].repeat) {\n            case \"daily\": // case of time\n              if (startDate.day() != endDate.day()) {\n                offset = 1;\n              }\n              startDate.dayOfYear(start.dayOfYear());\n              startDate.year(start.year());\n              startDate.subtract(7,'days');\n\n              endDate.dayOfYear(start.dayOfYear());\n              endDate.year(start.year());\n              endDate.subtract(7 - offset,'days');\n\n              runUntil.add(1, 'weeks');\n              break;\n            case \"weekly\": {\n              const dayOffset = endDate.diff(startDate,'days');\n              const day = startDate.day();\n\n              // set the start date to the range.start\n              startDate.date(start.date());\n              startDate.month(start.month());\n              startDate.year(start.year());\n              endDate = startDate.clone();\n\n              // force\n              startDate.day(day);\n              endDate.day(day);\n              endDate.add(dayOffset,'days');\n\n              startDate.subtract(1,'weeks');\n              endDate.subtract(1,'weeks');\n\n              runUntil.add(1, 'weeks');\n              break;\n            }\n            case \"monthly\":\n              if (startDate.month() != endDate.month()) {\n                offset = 1;\n              }\n              startDate.month(start.month());\n              startDate.year(start.year());\n              startDate.subtract(1,'months');\n\n              endDate.month(start.month());\n              endDate.year(start.year());\n              endDate.subtract(1,'months');\n              endDate.add(offset,'months');\n\n              runUntil.add(1, 'months');\n              break;\n            case \"yearly\":\n              if (startDate.year() != endDate.year()) {\n                offset = 1;\n              }\n              startDate.year(start.year());\n              startDate.subtract(1,'years');\n              endDate.year(start.year());\n              endDate.subtract(1,'years');\n              endDate.add(offset,'years');\n\n              runUntil.add(1, 'years');\n              break;\n            default:\n              console.log(\"Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:\", hiddenDates[i].repeat);\n              return;\n          }\n          while (startDate < runUntil) {\n            body.hiddenDates.push({start: startDate.valueOf(), end: endDate.valueOf()});\n            switch (hiddenDates[i].repeat) {\n              case \"daily\":\n                startDate.add(1, 'days');\n                endDate.add(1, 'days');\n                break;\n              case \"weekly\":\n                startDate.add(1, 'weeks');\n                endDate.add(1, 'weeks');\n                break;\n              case \"monthly\":\n                startDate.add(1, 'months');\n                endDate.add(1, 'months');\n                break;\n              case \"yearly\":\n                startDate.add(1, 'y');\n                endDate.add(1, 'y');\n                break;\n              default:\n                console.log(\"Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:\", hiddenDates[i].repeat);\n                return;\n            }\n          }\n          body.hiddenDates.push({start: startDate.valueOf(), end: endDate.valueOf()});\n        }\n      }\n    }\n    // remove duplicates, merge where possible\n    removeDuplicates(body);\n    // ensure the new positions are not on hidden dates\n    const startHidden = getIsHidden(body.range.start, body.hiddenDates);\n    const endHidden = getIsHidden(body.range.end,body.hiddenDates);\n    let rangeStart = body.range.start;\n    let rangeEnd = body.range.end;\n    if (startHidden.hidden == true) {rangeStart = body.range.startToFront == true ? startHidden.startDate - 1 : startHidden.endDate + 1;}\n    if (endHidden.hidden == true)   {rangeEnd   = body.range.endToFront == true ?   endHidden.startDate - 1   : endHidden.endDate + 1;}\n    if (startHidden.hidden == true || endHidden.hidden == true) {\n      body.range._applyRange(rangeStart, rangeEnd);\n    }\n  }\n\n}\n\n/**\n * remove duplicates from the hidden dates list. Duplicates are evil. They mess everything up.\n * Scales with N^2\n *\n * @param {Object} body\n */\nexport function removeDuplicates(body) {\n  const hiddenDates = body.hiddenDates;\n  const safeDates = [];\n  for (var i = 0; i < hiddenDates.length; i++) {\n    for (let j = 0; j < hiddenDates.length; j++) {\n      if (i != j && hiddenDates[j].remove != true && hiddenDates[i].remove != true) {\n        // j inside i\n        if (hiddenDates[j].start >= hiddenDates[i].start && hiddenDates[j].end <= hiddenDates[i].end) {\n          hiddenDates[j].remove = true;\n        }\n        // j start inside i\n        else if (hiddenDates[j].start >= hiddenDates[i].start && hiddenDates[j].start <= hiddenDates[i].end) {\n          hiddenDates[i].end = hiddenDates[j].end;\n          hiddenDates[j].remove = true;\n        }\n        // j end inside i\n        else if (hiddenDates[j].end >= hiddenDates[i].start && hiddenDates[j].end <= hiddenDates[i].end) {\n          hiddenDates[i].start = hiddenDates[j].start;\n          hiddenDates[j].remove = true;\n        }\n      }\n    }\n  }\n\n  for (i = 0; i < hiddenDates.length; i++) {\n    if (hiddenDates[i].remove !== true) {\n      safeDates.push(hiddenDates[i]);\n    }\n  }\n\n  body.hiddenDates = safeDates;\n  body.hiddenDates.sort((a, b) => a.start - b.start); // sort by start time\n}\n\n/**\n * Prints dates to console\n * @param {array} dates\n */\nexport function printDates(dates) {\n  for (let i =0; i < dates.length; i++) {\n    console.log(i, new Date(dates[i].start),new Date(dates[i].end), dates[i].start, dates[i].end, dates[i].remove);\n  }\n}\n\n/**\n * Used in TimeStep to avoid the hidden times.\n * @param {function} moment\n * @param {TimeStep} timeStep\n * @param {Date} previousTime\n */\nexport function stepOverHiddenDates(moment, timeStep, previousTime) {\n  let stepInHidden = false;\n  const currentValue = timeStep.current.valueOf();\n  for (let i = 0; i < timeStep.hiddenDates.length; i++) {\n    const startDate = timeStep.hiddenDates[i].start;\n    var endDate = timeStep.hiddenDates[i].end;\n    if (currentValue >= startDate && currentValue < endDate) {\n      stepInHidden = true;\n      break;\n    }\n  }\n\n  if (stepInHidden == true && currentValue < timeStep._end.valueOf() && currentValue != previousTime) {\n    const prevValue = moment(previousTime);\n    const newValue = moment(endDate);\n    //check if the next step should be major\n    if (prevValue.year() != newValue.year()) {timeStep.switchedYear = true;}\n    else if (prevValue.month() != newValue.month()) {timeStep.switchedMonth = true;}\n    else if (prevValue.dayOfYear() != newValue.dayOfYear()) {timeStep.switchedDay = true;}\n\n    timeStep.current = newValue;\n  }\n}\n\n///**\n// * Used in TimeStep to avoid the hidden times.\n// * @param timeStep\n// * @param previousTime\n// */\n//checkFirstStep = function(timeStep) {\n//  var stepInHidden = false;\n//  var currentValue = timeStep.current.valueOf();\n//  for (var i = 0; i < timeStep.hiddenDates.length; i++) {\n//    var startDate = timeStep.hiddenDates[i].start;\n//    var endDate = timeStep.hiddenDates[i].end;\n//    if (currentValue >= startDate && currentValue < endDate) {\n//      stepInHidden = true;\n//      break;\n//    }\n//  }\n//\n//  if (stepInHidden == true && currentValue <= timeStep._end.valueOf()) {\n//    var newValue = moment(endDate);\n//    timeStep.current = newValue.toDate();\n//  }\n//};\n\n/**\n * replaces the Core toScreen methods\n *\n * @param {timeline.Core} Core\n * @param {Date} time\n * @param {number} width\n * @returns {number}\n */\nexport function toScreen(Core, time, width) {\n  let conversion;\n  if (Core.body.hiddenDates.length == 0) {\n      conversion = Core.range.conversion(width);\n      return (time.valueOf() - conversion.offset) * conversion.scale;\n    } else {\n      const hidden = getIsHidden(time, Core.body.hiddenDates);\n      if (hidden.hidden == true) {\n        time = hidden.startDate;\n      }\n\n      const duration = getHiddenDurationBetween(Core.body.hiddenDates, Core.range.start, Core.range.end);\n      if (time < Core.range.start) {\n        conversion = Core.range.conversion(width, duration);\n        const hiddenBeforeStart = getHiddenDurationBeforeStart(Core.body.hiddenDates, time, conversion.offset);\n        time = Core.options.moment(time).toDate().valueOf();\n        time = time + hiddenBeforeStart;\n        return -(conversion.offset - time.valueOf()) * conversion.scale;\n        \n      } else if (time > Core.range.end) {\n        const rangeAfterEnd = {start: Core.range.start, end: time};\n        time = correctTimeForHidden(Core.options.moment, Core.body.hiddenDates, rangeAfterEnd, time);\n        conversion = Core.range.conversion(width, duration);\n        return (time.valueOf() - conversion.offset) * conversion.scale;\n\n      } else {\n        time = correctTimeForHidden(Core.options.moment, Core.body.hiddenDates, Core.range, time);\n        conversion = Core.range.conversion(width, duration);\n        return (time.valueOf() - conversion.offset) * conversion.scale;\n      }\n    }\n  }\n\n/**\n * Replaces the core toTime methods\n *\n * @param {timeline.Core} Core\n * @param {number} x\n * @param {number} width\n * @returns {Date}\n */\nexport function toTime(Core, x, width) {\n  if (Core.body.hiddenDates.length == 0) {\n    const conversion = Core.range.conversion(width);\n    return new Date(x / conversion.scale + conversion.offset);\n  }\n  else {\n    const hiddenDuration = getHiddenDurationBetween(Core.body.hiddenDates, Core.range.start, Core.range.end);\n    const totalDuration = Core.range.end - Core.range.start - hiddenDuration;\n    const partialDuration = totalDuration * x / width;\n    const accumulatedHiddenDuration = getAccumulatedHiddenDuration(Core.body.hiddenDates, Core.range, partialDuration);\n\n    return new Date(accumulatedHiddenDuration + partialDuration + Core.range.start);\n  }\n}\n\n/**\n * Support function\n *\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {number} start\n * @param {number} end\n * @returns {number}\n */\nexport function getHiddenDurationBetween(hiddenDates, start, end) {\n  let duration = 0;\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= start && endDate < end) {\n      duration += endDate - startDate;\n    }\n  }\n  return duration;\n}\n\n/**\n * Support function\n *\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {number} start\n * @param {number} end\n * @returns {number}\n */\nexport function getHiddenDurationBeforeStart(hiddenDates, start, end) {\n  let duration = 0;\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n\n    if (startDate >= start && endDate <= end) {\n      duration += endDate - startDate;\n    }\n  }\n  return duration;\n}\n\n/**\n * Support function\n * @param {function} moment\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {Date} time\n * @returns {number}\n */\nexport function correctTimeForHidden(moment, hiddenDates, range, time) {\n  time = moment(time).toDate().valueOf();\n  time -= getHiddenDurationBefore(moment, hiddenDates,range,time);\n  return time;\n}\n\n/**\n * Support function\n * @param {function} moment\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {Date} time\n * @returns {number}\n */\nexport function getHiddenDurationBefore(moment, hiddenDates, range, time) {\n  let timeOffset = 0;\n  time = moment(time).toDate().valueOf();\n\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= range.start && endDate < range.end) {\n      if (time >= endDate) {\n        timeOffset += (endDate - startDate);\n      }\n    }\n  }\n  return timeOffset;\n}\n\n/**\n * sum the duration from start to finish, including the hidden duration,\n * until the required amount has been reached, return the accumulated hidden duration\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {number} [requiredDuration=0]\n * @returns {number}\n */\nexport function getAccumulatedHiddenDuration(hiddenDates, range, requiredDuration) {\n  let hiddenDuration = 0;\n  let duration = 0;\n  let previousPoint = range.start;\n  //printDates(hiddenDates)\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= range.start && endDate < range.end) {\n      duration += startDate - previousPoint;\n      previousPoint = endDate;\n      if (duration >= requiredDuration) {\n        break;\n      }\n      else {\n        hiddenDuration += endDate - startDate;\n      }\n    }\n  }\n\n  return hiddenDuration;\n}\n\n/**\n * used to step over to either side of a hidden block. Correction is disabled on tablets, might be set to true\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {Date} time\n * @param {number} direction\n * @param {boolean} correctionEnabled\n * @returns {Date|number}\n */\nexport function snapAwayFromHidden(hiddenDates, time, direction, correctionEnabled) {\n  const isHidden = getIsHidden(time, hiddenDates);\n  if (isHidden.hidden == true) {\n    if (direction < 0) {\n      if (correctionEnabled == true) {\n        return isHidden.startDate - (isHidden.endDate - time) - 1;\n      }\n      else {\n        return isHidden.startDate - 1;\n      }\n    }\n    else {\n      if (correctionEnabled == true) {\n        return isHidden.endDate + (time - isHidden.startDate) + 1;\n      }\n      else {\n        return isHidden.endDate + 1;\n      }\n    }\n  }\n  else {\n    return time;\n  }\n\n}\n\n/**\n * Check if a time is hidden\n *\n * @param {Date} time\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @returns {{hidden: boolean, startDate: Window.start, endDate: *}}\n */\nexport function getIsHidden(time, hiddenDates) {\n  for (let i = 0; i < hiddenDates.length; i++) {\n    var startDate = hiddenDates[i].start;\n    var endDate = hiddenDates[i].end;\n\n    if (time >= startDate && time < endDate) { // if the start is entering a hidden zone\n      return {hidden: true, startDate, endDate};\n    }\n  }\n  return {hidden: false, startDate, endDate};\n}\n","import util from '../util';\nimport moment from '../module/moment';\nimport Component from './component/Component';\nimport * as DateUtil  from './DateUtil';\n\n/**\n * A Range controls a numeric range with a start and end value.\n * The Range adjusts the range based on mouse events or programmatic changes,\n * and triggers events when the range is changing or has been changed.\n */\nexport default class Range extends Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter}} body\n * @param {Object} [options]    See description at Range.setOptions\n * @constructor Range\n * @extends Component\n */\n  constructor(body, options) {\n    super();\n    const now = moment().hours(0).minutes(0).seconds(0).milliseconds(0);\n    const start = now.clone().add(-3, 'days').valueOf();\n    const end = now.clone().add(3, 'days').valueOf(); \n    this.millisecondsPerPixelCache = undefined;\n    \n    if(options === undefined) {\n      this.start = start;\n      this.end = end;\n    } else {\n      this.start = options.start || start;\n      this.end = options.end || end\n    }\n\n    this.rolling = false;\n\n    this.body = body;\n    this.deltaDifference = 0;\n    this.scaleOffset = 0;\n    this.startToFront = false;\n    this.endToFront = true;\n\n    // default options\n    this.defaultOptions = {\n      rtl: false,\n      start: null,\n      end: null,\n      moment,\n      direction: 'horizontal', // 'horizontal' or 'vertical'\n      moveable: true,\n      zoomable: true,\n      min: null,\n      max: null,\n      zoomMin: 10,                                // milliseconds\n      zoomMax: 1000 * 60 * 60 * 24 * 365 * 10000,  // milliseconds\n      rollingMode: {\n        follow: false,\n        offset: 0.5\n      }\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.props = {\n      touch: {}\n    };\n    this.animationTimer = null;\n\n    // drag listeners for dragging\n    this.body.emitter.on('panstart', this._onDragStart.bind(this));\n    this.body.emitter.on('panmove',  this._onDrag.bind(this));\n    this.body.emitter.on('panend',   this._onDragEnd.bind(this));\n\n    // mouse wheel for zooming\n    this.body.emitter.on('mousewheel', this._onMouseWheel.bind(this));\n\n    // pinch to zoom\n    this.body.emitter.on('touch', this._onTouch.bind(this));\n    this.body.emitter.on('pinch', this._onPinch.bind(this));\n\n    // on click of rolling mode button\n    this.body.dom.rollingModeBtn.addEventListener('click', this.startRolling.bind(this));\n\n    this.setOptions(options);\n  }\n\n  /**\n   * Set options for the range controller\n   * @param {Object} options      Available options:\n   *                              {number | Date | String} start  Start date for the range\n   *                              {number | Date | String} end    End date for the range\n   *                              {number} min    Minimum value for start\n   *                              {number} max    Maximum value for end\n   *                              {number} zoomMin    Set a minimum value for\n   *                                                  (end - start).\n   *                              {number} zoomMax    Set a maximum value for\n   *                                                  (end - start).\n   *                              {boolean} moveable Enable moving of the range\n   *                                                 by dragging. True by default\n   *                              {boolean} zoomable Enable zooming of the range\n   *                                                 by pinching/scrolling. True by default\n   */\n  setOptions(options) {\n    if (options) {\n      // copy the options that we know\n      const fields = [\n        'animation', 'direction', 'min', 'max', 'zoomMin', 'zoomMax', 'moveable', 'zoomable',\n        'moment', 'activate', 'hiddenDates', 'zoomKey', 'zoomFriction', 'rtl', 'showCurrentTime', 'rollingMode', 'horizontalScroll'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n\n      if (options.rollingMode && options.rollingMode.follow) {\n        this.startRolling();\n      }\n      if ('start' in options || 'end' in options) {\n        // apply a new range. both start and end are optional\n        this.setRange(options.start, options.end);\n      }\n    }\n  }\n\n  /**\n   * Start auto refreshing the current time bar\n   */\n  startRolling() {\n    const me = this;\n\n    /**\n     *  Updates the current time.\n     */\n    function update () {\n      me.stopRolling();\n      me.rolling = true;\n\n\n      let interval = me.end - me.start;\n      const t = util.convert(new Date(), 'Date').valueOf();\n      const rollingModeOffset = me.options.rollingMode && me.options.rollingMode.offset || 0.5\n\n      const start = t - interval * (rollingModeOffset);\n      const end = t + interval * (1 - rollingModeOffset);\n\n      const options = {\n        animation: false\n      };\n      me.setRange(start, end, options);\n\n      // determine interval to refresh\n      const scale = me.conversion(me.body.domProps.center.width).scale;\n      interval = 1 / scale / 10;\n      if (interval < 30)   interval = 30;\n      if (interval > 1000) interval = 1000;\n\n      me.body.dom.rollingModeBtn.style.visibility = \"hidden\";\n      // start a renderTimer to adjust for the new time\n      me.currentTimeTimer = setTimeout(update, interval);\n    }\n\n    update();\n  }\n\n  /**\n   * Stop auto refreshing the current time bar\n   */\n  stopRolling() {\n    if (this.currentTimeTimer !== undefined) {\n      clearTimeout(this.currentTimeTimer);\n      this.rolling = false;\n      this.body.dom.rollingModeBtn.style.visibility = \"visible\";\n    }\n  }\n\n  /**\n   * Set a new start and end range\n   * @param {Date | number | string} start\n   * @param {Date | number | string} end\n   * @param {Object} options      Available options:\n   *                              {boolean | {duration: number, easingFunction: string}} [animation=false]\n   *                                    If true, the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                              {boolean} [byUser=false]\n   *                              {Event}  event  Mouse event\n   * @param {Function} callback     a callback function to be executed at the end of this function  \n   * @param {Function} frameCallback    a callback function executed each frame of the range animation.\n   *                                    The callback will be passed three parameters:\n   *                                    {number} easeCoefficient    an easing coefficent\n   *                                    {boolean} willDraw          If true the caller will redraw after the callback completes\n   *                                    {boolean} done              If true then animation is ending after the current frame\n   * @return {void}\n   */\n  setRange(start, end, options, callback, frameCallback) {\n    if (!options) {\n      options = {};\n    }\n    if (options.byUser !== true) {\n      options.byUser = false;\n    }\n    const me = this;\n    const finalStart = start != undefined ? util.convert(start, 'Date').valueOf() : null;\n    const finalEnd   = end != undefined   ? util.convert(end, 'Date').valueOf()   : null;\n    this._cancelAnimation();\n    this.millisecondsPerPixelCache = undefined;\n\n    if (options.animation) { // true or an Object\n      const initStart = this.start;\n      const initEnd = this.end;\n      const duration = (typeof options.animation === 'object' && 'duration' in options.animation) ? options.animation.duration : 500;\n      const easingName = (typeof options.animation === 'object' && 'easingFunction' in options.animation) ? options.animation.easingFunction : 'easeInOutQuad';\n      const easingFunction = util.easingFunctions[easingName];\n      if (!easingFunction) {\n        throw new Error(`Unknown easing function ${JSON.stringify(easingName)}. Choose from: ${Object.keys(util.easingFunctions).join(', ')}`);\n      }\n\n      const initTime = Date.now();\n      let anyChanged = false;\n\n      const next = () => {\n        if (!me.props.touch.dragging) {\n          const now = Date.now();\n          const time = now - initTime;\n          const ease = easingFunction(time / duration);\n          const done = time > duration;\n          const s = (done || finalStart === null) ? finalStart : initStart + (finalStart - initStart) * ease;\n          const e = (done || finalEnd   === null) ? finalEnd   : initEnd   + (finalEnd   - initEnd)   * ease;\n\n          changed = me._applyRange(s, e);\n          DateUtil.updateHiddenDates(me.options.moment, me.body, me.options.hiddenDates);\n          anyChanged = anyChanged || changed;\n\n          const params = {\n            start: new Date(me.start), \n            end: new Date(me.end), \n            byUser: options.byUser,\n            event: options.event\n          };\n\n          if (frameCallback) { frameCallback(ease, changed, done); }\n\n          if (changed) {          \n            me.body.emitter.emit('rangechange', params);\n          }\n\n          if (done) {\n            if (anyChanged) {\n              me.body.emitter.emit('rangechanged', params);\n              if (callback) { return callback() }\n            }\n          }\n          else {\n            // animate with as high as possible frame rate, leave 20 ms in between\n            // each to prevent the browser from blocking\n            me.animationTimer = setTimeout(next, 20);\n          }\n        }\n      };\n\n      return next();\n    }\n    else {\n      var changed = this._applyRange(finalStart, finalEnd);\n      DateUtil.updateHiddenDates(this.options.moment, this.body, this.options.hiddenDates);\n      if (changed) {\n        const params = {\n          start: new Date(this.start), \n          end: new Date(this.end), \n          byUser: options.byUser, \n          event: options.event\n        };\n\n        this.body.emitter.emit('rangechange', params);\n        clearTimeout( me.timeoutID );\n        me.timeoutID = setTimeout( () => {\n          me.body.emitter.emit('rangechanged', params);\n        }, 200 );\n        if (callback) { return callback() }\n      }\n    }\n  }\n\n  /**\n   * Get the number of milliseconds per pixel.\n   *\n   * @returns {undefined|number}\n   */\n  getMillisecondsPerPixel() {\n    if (this.millisecondsPerPixelCache === undefined) {\n      this.millisecondsPerPixelCache = (this.end - this.start) / this.body.dom.center.clientWidth;\n    }\n    return this.millisecondsPerPixelCache;\n  }\n\n  /**\n   * Stop an animation\n   * @private\n   */\n  _cancelAnimation() {\n    if (this.animationTimer) {\n      clearTimeout(this.animationTimer);\n      this.animationTimer = null;\n    }\n  }\n\n  /**\n   * Set a new start and end range. This method is the same as setRange, but\n   * does not trigger a range change and range changed event, and it returns\n   * true when the range is changed\n   * @param {number} [start]\n   * @param {number} [end]\n   * @return {boolean} changed\n   * @private\n   */\n  _applyRange(start, end) {\n    let newStart = (start != null) ? util.convert(start, 'Date').valueOf() : this.start;\n    let newEnd   = (end != null)   ? util.convert(end, 'Date').valueOf()   : this.end;\n    const max = (this.options.max != null) ? util.convert(this.options.max, 'Date').valueOf() : null;\n    const min = (this.options.min != null) ? util.convert(this.options.min, 'Date').valueOf() : null;\n    let diff;\n\n    // check for valid number\n    if (isNaN(newStart) || newStart === null) {\n      throw new Error(`Invalid start \"${start}\"`);\n    }\n    if (isNaN(newEnd) || newEnd === null) {\n      throw new Error(`Invalid end \"${end}\"`);\n    }\n\n    // prevent end < start\n    if (newEnd < newStart) {\n      newEnd = newStart;\n    }\n\n    // prevent start < min\n    if (min !== null) {\n      if (newStart < min) {\n        diff = (min - newStart);\n        newStart += diff;\n        newEnd += diff;\n\n        // prevent end > max\n        if (max != null) {\n          if (newEnd > max) {\n            newEnd = max;\n          }\n        }\n      }\n    }\n\n    // prevent end > max\n    if (max !== null) {\n      if (newEnd > max) {\n        diff = (newEnd - max);\n        newStart -= diff;\n        newEnd -= diff;\n\n        // prevent start < min\n        if (min != null) {\n          if (newStart < min) {\n            newStart = min;\n          }\n        }\n      }\n    }\n\n    // prevent (end-start) < zoomMin\n    if (this.options.zoomMin !== null) {\n      let zoomMin = parseFloat(this.options.zoomMin);\n      if (zoomMin < 0) {\n        zoomMin = 0;\n      }\n      if ((newEnd - newStart) < zoomMin) {\n        // compensate for a scale of 0.5 ms\n        const compensation = 0.5;\n        if ((this.end - this.start) === zoomMin && newStart >= this.start - compensation && newEnd <= this.end) {\n          // ignore this action, we are already zoomed to the minimum\n          newStart = this.start;\n          newEnd = this.end;\n        }\n        else {\n          // zoom to the minimum\n          diff = (zoomMin - (newEnd - newStart));\n          newStart -= diff / 2;\n          newEnd += diff / 2;\n        }\n      }\n    }\n\n    // prevent (end-start) > zoomMax\n    if (this.options.zoomMax !== null) {\n      let zoomMax = parseFloat(this.options.zoomMax);\n      if (zoomMax < 0) {\n        zoomMax = 0;\n      }\n\n      if ((newEnd - newStart) > zoomMax) {\n        if ((this.end - this.start) === zoomMax && newStart < this.start && newEnd > this.end) {\n          // ignore this action, we are already zoomed to the maximum\n          newStart = this.start;\n          newEnd = this.end;\n        }\n        else {\n          // zoom to the maximum\n          diff = ((newEnd - newStart) - zoomMax);\n          newStart += diff / 2;\n          newEnd -= diff / 2;\n        }\n      }\n    }\n\n    const changed = (this.start != newStart || this.end != newEnd);\n\n    // if the new range does NOT overlap with the old range, emit checkRangedItems to avoid not showing ranged items (ranged meaning has end time, not necessarily of type Range)\n    if (!((newStart >= this.start && newStart   <= this.end) || (newEnd   >= this.start && newEnd   <= this.end)) &&\n        !((this.start >= newStart && this.start <= newEnd)   || (this.end >= newStart   && this.end <= newEnd) )) {\n      this.body.emitter.emit('checkRangedItems');\n    }\n\n    this.start = newStart;\n    this.end = newEnd;\n    return changed;\n  }\n\n  /**\n   * Retrieve the current range.\n   * @return {Object} An object with start and end properties\n   */\n  getRange() {\n    return {\n      start: this.start,\n      end: this.end\n    };\n  }\n\n  /**\n   * Calculate the conversion offset and scale for current range, based on\n   * the provided width\n   * @param {number} width\n   * @param {number} [totalHidden=0]\n   * @returns {{offset: number, scale: number}} conversion\n   */\n  conversion(width, totalHidden) {\n    return Range.conversion(this.start, this.end, width, totalHidden);\n  }\n\n  /**\n   * Static method to calculate the conversion offset and scale for a range,\n   * based on the provided start, end, and width\n   * @param {number} start\n   * @param {number} end\n   * @param {number} width\n   * @param {number} [totalHidden=0]\n   * @returns {{offset: number, scale: number}} conversion\n   */\n  static conversion(start, end, width, totalHidden) {\n    if (totalHidden === undefined) {\n      totalHidden = 0;\n    }\n    if (width != 0 && (end - start != 0)) {\n      return {\n        offset: start,\n        scale: width / (end - start - totalHidden)\n      }\n    }\n    else {\n      return {\n        offset: 0,\n        scale: 1\n      };\n    }\n  }\n\n  /**\n   * Start dragging horizontally or vertically\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    this.deltaDifference = 0;\n    this.previousDelta = 0;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // only start dragging when the mouse is inside the current range\n    if (!this._isInsideRange(event)) return;\n\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    this.stopRolling();\n\n    this.props.touch.start = this.start;\n    this.props.touch.end = this.end;\n    this.props.touch.dragging = true;\n\n    if (this.body.dom.root) {\n      this.body.dom.root.style.cursor = 'move';\n    }\n  }\n\n  /**\n   * Perform dragging operation\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!event) return;\n\n    if (!this.props.touch.dragging) return;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // TODO: this may be redundant in hammerjs2\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    const direction = this.options.direction;\n    validateDirection(direction);\n    let delta = (direction == 'horizontal') ? event.deltaX : event.deltaY;\n    delta -= this.deltaDifference;\n    let interval = (this.props.touch.end - this.props.touch.start);\n\n    // normalize dragging speed if cutout is in between.\n    const duration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    interval -= duration;\n\n    const width = (direction == 'horizontal') ? this.body.domProps.center.width : this.body.domProps.center.height;\n    let diffRange;\n    if (this.options.rtl) {\n      diffRange = delta / width * interval;\n    } else {\n      diffRange = -delta / width * interval;\n    }\n\n    const newStart = this.props.touch.start + diffRange;\n    const newEnd = this.props.touch.end + diffRange;\n\n    // snapping times away from hidden zones\n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, this.previousDelta-delta, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, this.previousDelta-delta, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      this.deltaDifference += delta;\n      this.props.touch.start = safeStart;\n      this.props.touch.end = safeEnd;\n      this._onDrag(event);\n      return;\n    }\n\n    this.previousDelta = delta;\n    this._applyRange(newStart, newEnd);\n\n\n    const startDate = new Date(this.start);\n    const endDate = new Date(this.end);\n\n    // fire a rangechange event\n    this.body.emitter.emit('rangechange', {\n      start: startDate,\n      end:   endDate,\n      byUser: true,\n      event\n    });\n\n    // fire a panmove event\n    this.body.emitter.emit('panmove');\n  }\n\n  /**\n   * Stop dragging operation\n   * @param {event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    if (!this.props.touch.dragging) return;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // TODO: this may be redundant in hammerjs2\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    this.props.touch.dragging = false;\n    if (this.body.dom.root) {\n      this.body.dom.root.style.cursor = 'auto';\n    }\n\n    // fire a rangechanged event\n    this.body.emitter.emit('rangechanged', {\n      start: new Date(this.start),\n      end:   new Date(this.end),\n      byUser: true,\n      event\n    });\n  }\n\n  /**\n   * Event handler for mouse wheel event, used to zoom\n   * Code from http://adomas.org/javascript-mouse-wheel/\n   * @param {Event} event\n   * @private\n   */\n  _onMouseWheel(event) {\n    // retrieve delta\n    let delta = 0;\n    if (event.wheelDelta) { /* IE/Opera. */\n      delta = event.wheelDelta / 120;\n    } else if (event.detail) { /* Mozilla case. */\n      // In Mozilla, sign of delta is different than in IE.\n      // Also, delta is multiple of 3.\n      delta = -event.detail / 3;\n    } else if (event.deltaY) {\n      delta = -event.deltaY / 3;\n    } \n\n    // don't allow zoom when the according key is pressed and the zoomKey option or not zoomable but movable\n    if ((this.options.zoomKey && !event[this.options.zoomKey] && this.options.zoomable) \n      || (!this.options.zoomable && this.options.moveable)) {\n      return;\n    }\n\n    // only allow zooming when configured as zoomable and moveable\n    if (!(this.options.zoomable && this.options.moveable)) return;\n    \n    // only zoom when the mouse is inside the current range\n    if (!this._isInsideRange(event)) return;\n\n    // If delta is nonzero, handle it.\n    // Basically, delta is now positive if wheel was scrolled up,\n    // and negative, if wheel was scrolled down.\n    if (delta) {\n      // perform the zoom action. Delta is normally 1 or -1\n\n      // adjust a negative delta such that zooming in with delta 0.1\n      // equals zooming out with a delta -0.1\n\n      const zoomFriction = this.options.zoomFriction || 5;\n      let scale;\n      if (delta < 0) {\n        scale = 1 - (delta / zoomFriction);\n      }\n      else {\n        scale = 1 / (1 + (delta / zoomFriction)) ;\n      }\n\n      // calculate center, the date to zoom around\n      let pointerDate;\n      if (this.rolling) {\n        const rollingModeOffset = this.options.rollingMode && this.options.rollingMode.offset || 0.5;\n        pointerDate = this.start + ((this.end - this.start) * rollingModeOffset);\n      } else {\n        const pointer = this.getPointer({x: event.clientX, y: event.clientY}, this.body.dom.center);\n        pointerDate = this._pointerToDate(pointer);\n      }\n      this.zoom(scale, pointerDate, delta, event);\n\n      // Prevent default actions caused by mouse wheel\n      // (else the page and timeline both scroll)\n      event.preventDefault();\n    }\n  }\n\n  /**\n   * Start of a touch gesture\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {  // eslint-disable-line no-unused-vars\n    this.props.touch.start = this.start;\n    this.props.touch.end = this.end;\n    this.props.touch.allowDragging = true;\n    this.props.touch.center = null;\n    this.props.touch.centerDate = null;\n    this.scaleOffset = 0;\n    this.deltaDifference = 0;\n    // Disable the browser default handling of this event.\n    util.preventDefault(event);\n  }\n\n  /**\n   * Handle pinch event\n   * @param {Event} event\n   * @private\n   */\n  _onPinch(event) {\n    // only allow zooming when configured as zoomable and moveable\n    if (!(this.options.zoomable && this.options.moveable)) return;\n\n    // Disable the browser default handling of this event.\n    util.preventDefault(event);\n\n    this.props.touch.allowDragging = false;\n\n    if (!this.props.touch.center) {\n      this.props.touch.center = this.getPointer(event.center, this.body.dom.center);\n      this.props.touch.centerDate = this._pointerToDate(this.props.touch.center);\n    }\n\n    this.stopRolling();\n    const scale = 1 / (event.scale + this.scaleOffset);\n    const centerDate = this.props.touch.centerDate;\n\n    const hiddenDuration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    const hiddenDurationBefore = DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this, centerDate);\n    const hiddenDurationAfter = hiddenDuration - hiddenDurationBefore;\n\n    // calculate new start and end\n    let newStart = (centerDate - hiddenDurationBefore) + (this.props.touch.start - (centerDate - hiddenDurationBefore)) * scale;\n    let newEnd = (centerDate + hiddenDurationAfter) + (this.props.touch.end - (centerDate + hiddenDurationAfter)) * scale;\n\n    // snapping times away from hidden zones\n    this.startToFront = 1 - scale <= 0; // used to do the right auto correction with periodic hidden times\n    this.endToFront = scale - 1 <= 0;   // used to do the right auto correction with periodic hidden times\n    \n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, 1 - scale, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, scale - 1, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      this.props.touch.start = safeStart;\n      this.props.touch.end = safeEnd;\n      this.scaleOffset = 1 - event.scale;\n      newStart = safeStart;\n      newEnd = safeEnd;\n    }\n    \n    const options = {\n      animation: false,\n      byUser: true,\n      event\n    };\n    this.setRange(newStart, newEnd, options);\n\n    this.startToFront = false; // revert to default\n    this.endToFront = true; // revert to default\n  }\n\n  /**\n   * Test whether the mouse from a mouse event is inside the visible window,\n   * between the current start and end date\n   * @param {Object} event\n   * @return {boolean} Returns true when inside the visible window\n   * @private\n   */\n  _isInsideRange(event) {\n    // calculate the time where the mouse is, check whether inside\n    // and no scroll action should happen.\n    const clientX = event.center ? event.center.x : event.clientX;\n    const centerContainerRect = this.body.dom.centerContainer.getBoundingClientRect();\n    const x = this.options.rtl ? clientX - centerContainerRect.left : centerContainerRect.right - clientX;\n    const time = this.body.util.toTime(x);\n\n    return time >= this.start && time <= this.end;\n  }\n\n  /**\n   * Helper function to calculate the center date for zooming\n   * @param {{x: number, y: number}} pointer\n   * @return {number} date\n   * @private\n   */\n  _pointerToDate(pointer) {\n    let conversion;\n    const direction = this.options.direction;\n\n    validateDirection(direction);\n\n    if (direction == 'horizontal') {\n      return this.body.util.toTime(pointer.x).valueOf();\n    }\n    else {\n      const height = this.body.domProps.center.height;\n      conversion = this.conversion(height);\n      return pointer.y / conversion.scale + conversion.offset;\n    }\n  }\n\n  /**\n   * Get the pointer location relative to the location of the dom element\n   * @param {{x: number, y: number}} touch\n   * @param {Element} element   HTML DOM element\n   * @return {{x: number, y: number}} pointer\n   * @private\n   */\n  getPointer(touch, element) {\n    const elementRect = element.getBoundingClientRect();\n    if (this.options.rtl) {\n      return {\n        x: elementRect.right - touch.x,\n        y: touch.y - elementRect.top\n      };\n    } else {\n      return {\n        x: touch.x - elementRect.left,\n        y: touch.y - elementRect.top\n      };\n    }\n  }\n\n  /**\n   * Zoom the range the given scale in or out. Start and end date will\n   * be adjusted, and the timeline will be redrawn. You can optionally give a\n   * date around which to zoom.\n   * For example, try scale = 0.9 or 1.1\n   * @param {number} scale      Scaling factor. Values above 1 will zoom out,\n   *                            values below 1 will zoom in.\n   * @param {number} [center]   Value representing a date around which will\n   *                            be zoomed.\n   * @param {number} delta\n   * @param {Event} event\n   */\n  zoom(scale, center, delta, event) {\n    // if centerDate is not provided, take it half between start Date and end Date\n    if (center == null) {\n      center = (this.start + this.end) / 2;\n    }\n\n    const hiddenDuration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    const hiddenDurationBefore = DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this, center);\n    const hiddenDurationAfter = hiddenDuration - hiddenDurationBefore;\n\n    // calculate new start and end\n    let newStart = (center-hiddenDurationBefore) + (this.start - (center-hiddenDurationBefore)) * scale;\n    let newEnd   = (center+hiddenDurationAfter) + (this.end - (center+hiddenDurationAfter)) * scale;\n\n    // snapping times away from hidden zones\n    this.startToFront = delta > 0 ? false : true; // used to do the right autocorrection with periodic hidden times\n    this.endToFront = -delta  > 0 ? false : true; // used to do the right autocorrection with periodic hidden times\n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, delta, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, -delta, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      newStart = safeStart;\n      newEnd = safeEnd;\n    }\n\n    const options = {\n      animation: false,\n      byUser: true,\n      event\n    };\n    this.setRange(newStart, newEnd, options);\n\n    this.startToFront = false; // revert to default\n    this.endToFront = true; // revert to default\n  }\n\n  /**\n   * Move the range with a given delta to the left or right. Start and end\n   * value will be adjusted. For example, try delta = 0.1 or -0.1\n   * @param {number}  delta     Moving amount. Positive value will move right,\n   *                            negative value will move left\n   */\n  move(delta) {\n    // zoom start Date and end Date relative to the centerDate\n    const diff = (this.end - this.start);\n\n    // apply new values\n    const newStart = this.start + diff * delta;\n    const newEnd = this.end + diff * delta;\n\n    // TODO: reckon with min and max range\n\n    this.start = newStart;\n    this.end = newEnd;\n  }\n\n  /**\n   * Move the range to a new center point\n   * @param {number} moveTo      New center point of the range\n   */\n  moveTo(moveTo) {\n    const center = (this.start + this.end) / 2;\n\n    const diff = center - moveTo;\n\n    // calculate new start and end\n    const newStart = this.start - diff;\n    const newEnd = this.end - diff;\n\n    const options = {\n      animation: false,\n      byUser: true,\n      event: null\n    };\n    this.setRange(newStart, newEnd, options);\n  }\n}\n\n/**\n * Test whether direction has a valid value\n * @param {string} direction    'horizontal' or 'vertical'\n */\nfunction validateDirection (direction) {\n  if (direction != 'horizontal' && direction != 'vertical') {\n    throw new TypeError(`Unknown direction \"${direction}\". Choose \"horizontal\" or \"vertical\".`);\n  }\n}\n","require('../modules/web.timers');\nvar path = require('../internals/path');\n\nmodule.exports = path.setInterval;\n","\r\n/**\r\n * Expose `Emitter`.\r\n */\r\n\r\nif (typeof module !== 'undefined') {\r\n  module.exports = Emitter;\r\n}\r\n\r\n/**\r\n * Initialize a new `Emitter`.\r\n *\r\n * @api public\r\n */\r\n\r\nfunction Emitter(obj) {\r\n  if (obj) return mixin(obj);\r\n};\r\n\r\n/**\r\n * Mixin the emitter properties.\r\n *\r\n * @param {Object} obj\r\n * @return {Object}\r\n * @api private\r\n */\r\n\r\nfunction mixin(obj) {\r\n  for (var key in Emitter.prototype) {\r\n    obj[key] = Emitter.prototype[key];\r\n  }\r\n  return obj;\r\n}\r\n\r\n/**\r\n * Listen on the given `event` with `fn`.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.on =\r\nEmitter.prototype.addEventListener = function(event, fn){\r\n  this._callbacks = this._callbacks || {};\r\n  (this._callbacks['$' + event] = this._callbacks['$' + event] || [])\r\n    .push(fn);\r\n  return this;\r\n};\r\n\r\n/**\r\n * Adds an `event` listener that will be invoked a single\r\n * time then automatically removed.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.once = function(event, fn){\r\n  function on() {\r\n    this.off(event, on);\r\n    fn.apply(this, arguments);\r\n  }\r\n\r\n  on.fn = fn;\r\n  this.on(event, on);\r\n  return this;\r\n};\r\n\r\n/**\r\n * Remove the given callback for `event` or all\r\n * registered callbacks.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.off =\r\nEmitter.prototype.removeListener =\r\nEmitter.prototype.removeAllListeners =\r\nEmitter.prototype.removeEventListener = function(event, fn){\r\n  this._callbacks = this._callbacks || {};\r\n\r\n  // all\r\n  if (0 == arguments.length) {\r\n    this._callbacks = {};\r\n    return this;\r\n  }\r\n\r\n  // specific event\r\n  var callbacks = this._callbacks['$' + event];\r\n  if (!callbacks) return this;\r\n\r\n  // remove all handlers\r\n  if (1 == arguments.length) {\r\n    delete this._callbacks['$' + event];\r\n    return this;\r\n  }\r\n\r\n  // remove specific handler\r\n  var cb;\r\n  for (var i = 0; i < callbacks.length; i++) {\r\n    cb = callbacks[i];\r\n    if (cb === fn || cb.fn === fn) {\r\n      callbacks.splice(i, 1);\r\n      break;\r\n    }\r\n  }\r\n\r\n  // Remove event specific arrays for event types that no\r\n  // one is subscribed for to avoid memory leak.\r\n  if (callbacks.length === 0) {\r\n    delete this._callbacks['$' + event];\r\n  }\r\n\r\n  return this;\r\n};\r\n\r\n/**\r\n * Emit `event` with the given args.\r\n *\r\n * @param {String} event\r\n * @param {Mixed} ...\r\n * @return {Emitter}\r\n */\r\n\r\nEmitter.prototype.emit = function(event){\r\n  this._callbacks = this._callbacks || {};\r\n\r\n  var args = new Array(arguments.length - 1)\r\n    , callbacks = this._callbacks['$' + event];\r\n\r\n  for (var i = 1; i < arguments.length; i++) {\r\n    args[i - 1] = arguments[i];\r\n  }\r\n\r\n  if (callbacks) {\r\n    callbacks = callbacks.slice(0);\r\n    for (var i = 0, len = callbacks.length; i < len; ++i) {\r\n      callbacks[i].apply(this, args);\r\n    }\r\n  }\r\n\r\n  return this;\r\n};\r\n\r\n/**\r\n * Return array of callbacks for `event`.\r\n *\r\n * @param {String} event\r\n * @return {Array}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.listeners = function(event){\r\n  this._callbacks = this._callbacks || {};\r\n  return this._callbacks['$' + event] || [];\r\n};\r\n\r\n/**\r\n * Check if this emitter has `event` handlers.\r\n *\r\n * @param {String} event\r\n * @return {Boolean}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.hasListeners = function(event){\r\n  return !! this.listeners(event).length;\r\n};\r\n","'use strict';\n\n(function (factory) {\n  if (typeof define === 'function' && define.amd) {\n    // AMD. Register as an anonymous module.\n    define([], factory);\n  } else if (typeof exports === 'object') {\n    // Node. Does not work with strict CommonJS, but\n    // only CommonJS-like environments that support module.exports,\n    // like Node.\n    module.exports = factory();\n  } else {\n    // Browser globals (root is window)\n    window.propagating = factory();\n  }\n}(function () {\n  var _firstTarget = null; // singleton, will contain the target element where the touch event started\n\n  /**\n   * Extend an Hammer.js instance with event propagation.\n   *\n   * Features:\n   * - Events emitted by hammer will propagate in order from child to parent\n   *   elements.\n   * - Events are extended with a function `event.stopPropagation()` to stop\n   *   propagation to parent elements.\n   * - An option `preventDefault` to stop all default browser behavior.\n   *\n   * Usage:\n   *   var hammer = propagatingHammer(new Hammer(element));\n   *   var hammer = propagatingHammer(new Hammer(element), {preventDefault: true});\n   *\n   * @param {Hammer.Manager} hammer   An hammer instance.\n   * @param {Object} [options]        Available options:\n   *                                  - `preventDefault: true | false | 'mouse' | 'touch' | 'pen'`.\n   *                                    Enforce preventing the default browser behavior.\n   *                                    Cannot be set to `false`.\n   * @return {Hammer.Manager} Returns the same hammer instance with extended\n   *                          functionality\n   */\n  return function propagating(hammer, options) {\n    var _options = options || {\n      preventDefault: false\n    };\n\n    if (hammer.Manager) {\n      // This looks like the Hammer constructor.\n      // Overload the constructors with our own.\n      var Hammer = hammer;\n\n      var PropagatingHammer = function(element, options) {\n        var o = Object.create(_options);\n        if (options) Hammer.assign(o, options);\n        return propagating(new Hammer(element, o), o);\n      };\n      Hammer.assign(PropagatingHammer, Hammer);\n\n      PropagatingHammer.Manager = function (element, options) {\n        var o = Object.create(_options);\n        if (options) Hammer.assign(o, options);\n        return propagating(new Hammer.Manager(element, o), o);\n      };\n\n      return PropagatingHammer;\n    }\n\n    // create a wrapper object which will override the functions\n    // `on`, `off`, `destroy`, and `emit` of the hammer instance\n    var wrapper = Object.create(hammer);\n\n    // attach to DOM element\n    var element = hammer.element;\n\n    if(!element.hammer) element.hammer = [];\n    element.hammer.push(wrapper);\n\n    // register an event to catch the start of a gesture and store the\n    // target in a singleton\n    hammer.on('hammer.input', function (event) {\n      if (_options.preventDefault === true || (_options.preventDefault === event.pointerType)) {\n        event.preventDefault();\n      }\n      if (event.isFirst) {\n        _firstTarget = event.target;\n      }\n    });\n\n    /** @type {Object.<String, Array.<function>>} */\n    wrapper._handlers = {};\n\n    /**\n     * Register a handler for one or multiple events\n     * @param {String} events    A space separated string with events\n     * @param {function} handler A callback function, called as handler(event)\n     * @returns {Hammer.Manager} Returns the hammer instance\n     */\n    wrapper.on = function (events, handler) {\n      // register the handler\n      split(events).forEach(function (event) {\n        var _handlers = wrapper._handlers[event];\n        if (!_handlers) {\n          wrapper._handlers[event] = _handlers = [];\n\n          // register the static, propagated handler\n          hammer.on(event, propagatedHandler);\n        }\n        _handlers.push(handler);\n      });\n\n      return wrapper;\n    };\n\n    /**\n     * Unregister a handler for one or multiple events\n     * @param {String} events      A space separated string with events\n     * @param {function} [handler] Optional. The registered handler. If not\n     *                             provided, all handlers for given events\n     *                             are removed.\n     * @returns {Hammer.Manager}   Returns the hammer instance\n     */\n    wrapper.off = function (events, handler) {\n      // unregister the handler\n      split(events).forEach(function (event) {\n        var _handlers = wrapper._handlers[event];\n        if (_handlers) {\n          _handlers = handler ? _handlers.filter(function (h) {\n            return h !== handler;\n          }) : [];\n\n          if (_handlers.length > 0) {\n            wrapper._handlers[event] = _handlers;\n          }\n          else {\n            // remove static, propagated handler\n            hammer.off(event, propagatedHandler);\n            delete wrapper._handlers[event];\n          }\n        }\n      });\n\n      return wrapper;\n    };\n\n    /**\n     * Emit to the event listeners\n     * @param {string} eventType\n     * @param {Event} event\n     */\n    wrapper.emit = function(eventType, event) {\n      _firstTarget = event.target;\n      hammer.emit(eventType, event);\n    };\n\n    wrapper.destroy = function () {\n      // Detach from DOM element\n      var hammers = hammer.element.hammer;\n      var idx = hammers.indexOf(wrapper);\n      if(idx !== -1) hammers.splice(idx,1);\n      if(!hammers.length) delete hammer.element.hammer;\n\n      // clear all handlers\n      wrapper._handlers = {};\n\n      // call original hammer destroy\n      hammer.destroy();\n    };\n\n    // split a string with space separated words\n    function split(events) {\n      return events.match(/[^ ]+/g);\n    }\n\n    /**\n     * A static event handler, applying event propagation.\n     * @param {Object} event\n     */\n    function propagatedHandler(event) {\n      // let only a single hammer instance handle this event\n      if (event.type !== 'hammer.input') {\n        // it is possible that the same srcEvent is used with multiple hammer events,\n        // we keep track on which events are handled in an object _handled\n        if (!event.srcEvent._handled) {\n          event.srcEvent._handled = {};\n        }\n\n        if (event.srcEvent._handled[event.type]) {\n          return;\n        }\n        else {\n          event.srcEvent._handled[event.type] = true;\n        }\n      }\n\n      // attach a stopPropagation function to the event\n      var stopped = false;\n      event.stopPropagation = function () {\n        stopped = true;\n      };\n\n      //wrap the srcEvent's stopPropagation to also stop hammer propagation:\n      var srcStop = event.srcEvent.stopPropagation.bind(event.srcEvent);\n      if(typeof srcStop == \"function\") {\n        event.srcEvent.stopPropagation = function(){\n          srcStop();\n          event.stopPropagation();\n        }\n      }\n\n      // attach firstTarget property to the event\n      event.firstTarget = _firstTarget;\n\n      // propagate over all elements (until stopped)\n      var elem = _firstTarget;\n      while (elem && !stopped) {\n        var elemHammer = elem.hammer;\n        if(elemHammer){\n          var _handlers;\n          for(var k = 0; k < elemHammer.length; k++){\n            _handlers = elemHammer[k]._handlers[event.type];\n            if(_handlers) for (var i = 0; i < _handlers.length && !stopped; i++) {\n              _handlers[i](event);\n            }\n          }\n        }\n        elem = elem.parentNode;\n      }\n    }\n\n    return wrapper;\n  };\n}));\n","/*! Hammer.JS - v2.0.17-rc - 2019-12-16\n * http://naver.github.io/egjs\n *\n * Forked By Naver egjs\n * Copyright (c) hammerjs\n * Licensed under the MIT license */\nfunction _extends() {\n  _extends = Object.assign || function (target) {\n    for (var i = 1; i < arguments.length; i++) {\n      var source = arguments[i];\n\n      for (var key in source) {\n        if (Object.prototype.hasOwnProperty.call(source, key)) {\n          target[key] = source[key];\n        }\n      }\n    }\n\n    return target;\n  };\n\n  return _extends.apply(this, arguments);\n}\n\nfunction _inheritsLoose(subClass, superClass) {\n  subClass.prototype = Object.create(superClass.prototype);\n  subClass.prototype.constructor = subClass;\n  subClass.__proto__ = superClass;\n}\n\nfunction _assertThisInitialized(self) {\n  if (self === void 0) {\n    throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");\n  }\n\n  return self;\n}\n\n/**\n * @private\n * extend object.\n * means that properties in dest will be overwritten by the ones in src.\n * @param {Object} target\n * @param {...Object} objects_to_assign\n * @returns {Object} target\n */\nvar assign;\n\nif (typeof Object.assign !== 'function') {\n  assign = function assign(target) {\n    if (target === undefined || target === null) {\n      throw new TypeError('Cannot convert undefined or null to object');\n    }\n\n    var output = Object(target);\n\n    for (var index = 1; index < arguments.length; index++) {\n      var source = arguments[index];\n\n      if (source !== undefined && source !== null) {\n        for (var nextKey in source) {\n          if (source.hasOwnProperty(nextKey)) {\n            output[nextKey] = source[nextKey];\n          }\n        }\n      }\n    }\n\n    return output;\n  };\n} else {\n  assign = Object.assign;\n}\n\nvar assign$1 = assign;\n\nvar VENDOR_PREFIXES = ['', 'webkit', 'Moz', 'MS', 'ms', 'o'];\nvar TEST_ELEMENT = typeof document === \"undefined\" ? {\n  style: {}\n} : document.createElement('div');\nvar TYPE_FUNCTION = 'function';\nvar round = Math.round,\n    abs = Math.abs;\nvar now = Date.now;\n\n/**\n * @private\n * get the prefixed property\n * @param {Object} obj\n * @param {String} property\n * @returns {String|Undefined} prefixed\n */\n\nfunction prefixed(obj, property) {\n  var prefix;\n  var prop;\n  var camelProp = property[0].toUpperCase() + property.slice(1);\n  var i = 0;\n\n  while (i < VENDOR_PREFIXES.length) {\n    prefix = VENDOR_PREFIXES[i];\n    prop = prefix ? prefix + camelProp : property;\n\n    if (prop in obj) {\n      return prop;\n    }\n\n    i++;\n  }\n\n  return undefined;\n}\n\n/* eslint-disable no-new-func, no-nested-ternary */\nvar win;\n\nif (typeof window === \"undefined\") {\n  // window is undefined in node.js\n  win = {};\n} else {\n  win = window;\n}\n\nvar PREFIXED_TOUCH_ACTION = prefixed(TEST_ELEMENT.style, 'touchAction');\nvar NATIVE_TOUCH_ACTION = PREFIXED_TOUCH_ACTION !== undefined;\nfunction getTouchActionProps() {\n  if (!NATIVE_TOUCH_ACTION) {\n    return false;\n  }\n\n  var touchMap = {};\n  var cssSupports = win.CSS && win.CSS.supports;\n  ['auto', 'manipulation', 'pan-y', 'pan-x', 'pan-x pan-y', 'none'].forEach(function (val) {\n    // If css.supports is not supported but there is native touch-action assume it supports\n    // all values. This is the case for IE 10 and 11.\n    return touchMap[val] = cssSupports ? win.CSS.supports('touch-action', val) : true;\n  });\n  return touchMap;\n}\n\nvar TOUCH_ACTION_COMPUTE = 'compute';\nvar TOUCH_ACTION_AUTO = 'auto';\nvar TOUCH_ACTION_MANIPULATION = 'manipulation'; // not implemented\n\nvar TOUCH_ACTION_NONE = 'none';\nvar TOUCH_ACTION_PAN_X = 'pan-x';\nvar TOUCH_ACTION_PAN_Y = 'pan-y';\nvar TOUCH_ACTION_MAP = getTouchActionProps();\n\nvar MOBILE_REGEX = /mobile|tablet|ip(ad|hone|od)|android/i;\nvar SUPPORT_TOUCH = 'ontouchstart' in win;\nvar SUPPORT_POINTER_EVENTS = prefixed(win, 'PointerEvent') !== undefined;\nvar SUPPORT_ONLY_TOUCH = SUPPORT_TOUCH && MOBILE_REGEX.test(navigator.userAgent);\nvar INPUT_TYPE_TOUCH = 'touch';\nvar INPUT_TYPE_PEN = 'pen';\nvar INPUT_TYPE_MOUSE = 'mouse';\nvar INPUT_TYPE_KINECT = 'kinect';\nvar COMPUTE_INTERVAL = 25;\nvar INPUT_START = 1;\nvar INPUT_MOVE = 2;\nvar INPUT_END = 4;\nvar INPUT_CANCEL = 8;\nvar DIRECTION_NONE = 1;\nvar DIRECTION_LEFT = 2;\nvar DIRECTION_RIGHT = 4;\nvar DIRECTION_UP = 8;\nvar DIRECTION_DOWN = 16;\nvar DIRECTION_HORIZONTAL = DIRECTION_LEFT | DIRECTION_RIGHT;\nvar DIRECTION_VERTICAL = DIRECTION_UP | DIRECTION_DOWN;\nvar DIRECTION_ALL = DIRECTION_HORIZONTAL | DIRECTION_VERTICAL;\nvar PROPS_XY = ['x', 'y'];\nvar PROPS_CLIENT_XY = ['clientX', 'clientY'];\n\n/**\n * @private\n * walk objects and arrays\n * @param {Object} obj\n * @param {Function} iterator\n * @param {Object} context\n */\nfunction each(obj, iterator, context) {\n  var i;\n\n  if (!obj) {\n    return;\n  }\n\n  if (obj.forEach) {\n    obj.forEach(iterator, context);\n  } else if (obj.length !== undefined) {\n    i = 0;\n\n    while (i < obj.length) {\n      iterator.call(context, obj[i], i, obj);\n      i++;\n    }\n  } else {\n    for (i in obj) {\n      obj.hasOwnProperty(i) && iterator.call(context, obj[i], i, obj);\n    }\n  }\n}\n\n/**\n * @private\n * let a boolean value also be a function that must return a boolean\n * this first item in args will be used as the context\n * @param {Boolean|Function} val\n * @param {Array} [args]\n * @returns {Boolean}\n */\n\nfunction boolOrFn(val, args) {\n  if (typeof val === TYPE_FUNCTION) {\n    return val.apply(args ? args[0] || undefined : undefined, args);\n  }\n\n  return val;\n}\n\n/**\n * @private\n * small indexOf wrapper\n * @param {String} str\n * @param {String} find\n * @returns {Boolean} found\n */\nfunction inStr(str, find) {\n  return str.indexOf(find) > -1;\n}\n\n/**\n * @private\n * when the touchActions are collected they are not a valid value, so we need to clean things up. *\n * @param {String} actions\n * @returns {*}\n */\n\nfunction cleanTouchActions(actions) {\n  // none\n  if (inStr(actions, TOUCH_ACTION_NONE)) {\n    return TOUCH_ACTION_NONE;\n  }\n\n  var hasPanX = inStr(actions, TOUCH_ACTION_PAN_X);\n  var hasPanY = inStr(actions, TOUCH_ACTION_PAN_Y); // if both pan-x and pan-y are set (different recognizers\n  // for different directions, e.g. horizontal pan but vertical swipe?)\n  // we need none (as otherwise with pan-x pan-y combined none of these\n  // recognizers will work, since the browser would handle all panning\n\n  if (hasPanX && hasPanY) {\n    return TOUCH_ACTION_NONE;\n  } // pan-x OR pan-y\n\n\n  if (hasPanX || hasPanY) {\n    return hasPanX ? TOUCH_ACTION_PAN_X : TOUCH_ACTION_PAN_Y;\n  } // manipulation\n\n\n  if (inStr(actions, TOUCH_ACTION_MANIPULATION)) {\n    return TOUCH_ACTION_MANIPULATION;\n  }\n\n  return TOUCH_ACTION_AUTO;\n}\n\n/**\n * @private\n * Touch Action\n * sets the touchAction property or uses the js alternative\n * @param {Manager} manager\n * @param {String} value\n * @constructor\n */\n\nvar TouchAction =\n/*#__PURE__*/\nfunction () {\n  function TouchAction(manager, value) {\n    this.manager = manager;\n    this.set(value);\n  }\n  /**\n   * @private\n   * set the touchAction value on the element or enable the polyfill\n   * @param {String} value\n   */\n\n\n  var _proto = TouchAction.prototype;\n\n  _proto.set = function set(value) {\n    // find out the touch-action by the event handlers\n    if (value === TOUCH_ACTION_COMPUTE) {\n      value = this.compute();\n    }\n\n    if (NATIVE_TOUCH_ACTION && this.manager.element.style && TOUCH_ACTION_MAP[value]) {\n      this.manager.element.style[PREFIXED_TOUCH_ACTION] = value;\n    }\n\n    this.actions = value.toLowerCase().trim();\n  };\n  /**\n   * @private\n   * just re-set the touchAction value\n   */\n\n\n  _proto.update = function update() {\n    this.set(this.manager.options.touchAction);\n  };\n  /**\n   * @private\n   * compute the value for the touchAction property based on the recognizer's settings\n   * @returns {String} value\n   */\n\n\n  _proto.compute = function compute() {\n    var actions = [];\n    each(this.manager.recognizers, function (recognizer) {\n      if (boolOrFn(recognizer.options.enable, [recognizer])) {\n        actions = actions.concat(recognizer.getTouchAction());\n      }\n    });\n    return cleanTouchActions(actions.join(' '));\n  };\n  /**\n   * @private\n   * this method is called on each input cycle and provides the preventing of the browser behavior\n   * @param {Object} input\n   */\n\n\n  _proto.preventDefaults = function preventDefaults(input) {\n    var srcEvent = input.srcEvent;\n    var direction = input.offsetDirection; // if the touch action did prevented once this session\n\n    if (this.manager.session.prevented) {\n      srcEvent.preventDefault();\n      return;\n    }\n\n    var actions = this.actions;\n    var hasNone = inStr(actions, TOUCH_ACTION_NONE) && !TOUCH_ACTION_MAP[TOUCH_ACTION_NONE];\n    var hasPanY = inStr(actions, TOUCH_ACTION_PAN_Y) && !TOUCH_ACTION_MAP[TOUCH_ACTION_PAN_Y];\n    var hasPanX = inStr(actions, TOUCH_ACTION_PAN_X) && !TOUCH_ACTION_MAP[TOUCH_ACTION_PAN_X];\n\n    if (hasNone) {\n      // do not prevent defaults if this is a tap gesture\n      var isTapPointer = input.pointers.length === 1;\n      var isTapMovement = input.distance < 2;\n      var isTapTouchTime = input.deltaTime < 250;\n\n      if (isTapPointer && isTapMovement && isTapTouchTime) {\n        return;\n      }\n    }\n\n    if (hasPanX && hasPanY) {\n      // `pan-x pan-y` means browser handles all scrolling/panning, do not prevent\n      return;\n    }\n\n    if (hasNone || hasPanY && direction & DIRECTION_HORIZONTAL || hasPanX && direction & DIRECTION_VERTICAL) {\n      return this.preventSrc(srcEvent);\n    }\n  };\n  /**\n   * @private\n   * call preventDefault to prevent the browser's default behavior (scrolling in most cases)\n   * @param {Object} srcEvent\n   */\n\n\n  _proto.preventSrc = function preventSrc(srcEvent) {\n    this.manager.session.prevented = true;\n    srcEvent.preventDefault();\n  };\n\n  return TouchAction;\n}();\n\n/**\n * @private\n * find if a node is in the given parent\n * @method hasParent\n * @param {HTMLElement} node\n * @param {HTMLElement} parent\n * @return {Boolean} found\n */\nfunction hasParent(node, parent) {\n  while (node) {\n    if (node === parent) {\n      return true;\n    }\n\n    node = node.parentNode;\n  }\n\n  return false;\n}\n\n/**\n * @private\n * get the center of all the pointers\n * @param {Array} pointers\n * @return {Object} center contains `x` and `y` properties\n */\n\nfunction getCenter(pointers) {\n  var pointersLength = pointers.length; // no need to loop when only one touch\n\n  if (pointersLength === 1) {\n    return {\n      x: round(pointers[0].clientX),\n      y: round(pointers[0].clientY)\n    };\n  }\n\n  var x = 0;\n  var y = 0;\n  var i = 0;\n\n  while (i < pointersLength) {\n    x += pointers[i].clientX;\n    y += pointers[i].clientY;\n    i++;\n  }\n\n  return {\n    x: round(x / pointersLength),\n    y: round(y / pointersLength)\n  };\n}\n\n/**\n * @private\n * create a simple clone from the input used for storage of firstInput and firstMultiple\n * @param {Object} input\n * @returns {Object} clonedInputData\n */\n\nfunction simpleCloneInputData(input) {\n  // make a simple copy of the pointers because we will get a reference if we don't\n  // we only need clientXY for the calculations\n  var pointers = [];\n  var i = 0;\n\n  while (i < input.pointers.length) {\n    pointers[i] = {\n      clientX: round(input.pointers[i].clientX),\n      clientY: round(input.pointers[i].clientY)\n    };\n    i++;\n  }\n\n  return {\n    timeStamp: now(),\n    pointers: pointers,\n    center: getCenter(pointers),\n    deltaX: input.deltaX,\n    deltaY: input.deltaY\n  };\n}\n\n/**\n * @private\n * calculate the absolute distance between two points\n * @param {Object} p1 {x, y}\n * @param {Object} p2 {x, y}\n * @param {Array} [props] containing x and y keys\n * @return {Number} distance\n */\n\nfunction getDistance(p1, p2, props) {\n  if (!props) {\n    props = PROPS_XY;\n  }\n\n  var x = p2[props[0]] - p1[props[0]];\n  var y = p2[props[1]] - p1[props[1]];\n  return Math.sqrt(x * x + y * y);\n}\n\n/**\n * @private\n * calculate the angle between two coordinates\n * @param {Object} p1\n * @param {Object} p2\n * @param {Array} [props] containing x and y keys\n * @return {Number} angle\n */\n\nfunction getAngle(p1, p2, props) {\n  if (!props) {\n    props = PROPS_XY;\n  }\n\n  var x = p2[props[0]] - p1[props[0]];\n  var y = p2[props[1]] - p1[props[1]];\n  return Math.atan2(y, x) * 180 / Math.PI;\n}\n\n/**\n * @private\n * get the direction between two points\n * @param {Number} x\n * @param {Number} y\n * @return {Number} direction\n */\n\nfunction getDirection(x, y) {\n  if (x === y) {\n    return DIRECTION_NONE;\n  }\n\n  if (abs(x) >= abs(y)) {\n    return x < 0 ? DIRECTION_LEFT : DIRECTION_RIGHT;\n  }\n\n  return y < 0 ? DIRECTION_UP : DIRECTION_DOWN;\n}\n\nfunction computeDeltaXY(session, input) {\n  var center = input.center; // let { offsetDelta:offset = {}, prevDelta = {}, prevInput = {} } = session;\n  // jscs throwing error on defalut destructured values and without defaults tests fail\n\n  var offset = session.offsetDelta || {};\n  var prevDelta = session.prevDelta || {};\n  var prevInput = session.prevInput || {};\n\n  if (input.eventType === INPUT_START || prevInput.eventType === INPUT_END) {\n    prevDelta = session.prevDelta = {\n      x: prevInput.deltaX || 0,\n      y: prevInput.deltaY || 0\n    };\n    offset = session.offsetDelta = {\n      x: center.x,\n      y: center.y\n    };\n  }\n\n  input.deltaX = prevDelta.x + (center.x - offset.x);\n  input.deltaY = prevDelta.y + (center.y - offset.y);\n}\n\n/**\n * @private\n * calculate the velocity between two points. unit is in px per ms.\n * @param {Number} deltaTime\n * @param {Number} x\n * @param {Number} y\n * @return {Object} velocity `x` and `y`\n */\nfunction getVelocity(deltaTime, x, y) {\n  return {\n    x: x / deltaTime || 0,\n    y: y / deltaTime || 0\n  };\n}\n\n/**\n * @private\n * calculate the scale factor between two pointersets\n * no scale is 1, and goes down to 0 when pinched together, and bigger when pinched out\n * @param {Array} start array of pointers\n * @param {Array} end array of pointers\n * @return {Number} scale\n */\n\nfunction getScale(start, end) {\n  return getDistance(end[0], end[1], PROPS_CLIENT_XY) / getDistance(start[0], start[1], PROPS_CLIENT_XY);\n}\n\n/**\n * @private\n * calculate the rotation degrees between two pointersets\n * @param {Array} start array of pointers\n * @param {Array} end array of pointers\n * @return {Number} rotation\n */\n\nfunction getRotation(start, end) {\n  return getAngle(end[1], end[0], PROPS_CLIENT_XY) + getAngle(start[1], start[0], PROPS_CLIENT_XY);\n}\n\n/**\n * @private\n * velocity is calculated every x ms\n * @param {Object} session\n * @param {Object} input\n */\n\nfunction computeIntervalInputData(session, input) {\n  var last = session.lastInterval || input;\n  var deltaTime = input.timeStamp - last.timeStamp;\n  var velocity;\n  var velocityX;\n  var velocityY;\n  var direction;\n\n  if (input.eventType !== INPUT_CANCEL && (deltaTime > COMPUTE_INTERVAL || last.velocity === undefined)) {\n    var deltaX = input.deltaX - last.deltaX;\n    var deltaY = input.deltaY - last.deltaY;\n    var v = getVelocity(deltaTime, deltaX, deltaY);\n    velocityX = v.x;\n    velocityY = v.y;\n    velocity = abs(v.x) > abs(v.y) ? v.x : v.y;\n    direction = getDirection(deltaX, deltaY);\n    session.lastInterval = input;\n  } else {\n    // use latest velocity info if it doesn't overtake a minimum period\n    velocity = last.velocity;\n    velocityX = last.velocityX;\n    velocityY = last.velocityY;\n    direction = last.direction;\n  }\n\n  input.velocity = velocity;\n  input.velocityX = velocityX;\n  input.velocityY = velocityY;\n  input.direction = direction;\n}\n\n/**\n* @private\n * extend the data with some usable properties like scale, rotate, velocity etc\n * @param {Object} manager\n * @param {Object} input\n */\n\nfunction computeInputData(manager, input) {\n  var session = manager.session;\n  var pointers = input.pointers;\n  var pointersLength = pointers.length; // store the first input to calculate the distance and direction\n\n  if (!session.firstInput) {\n    session.firstInput = simpleCloneInputData(input);\n  } // to compute scale and rotation we need to store the multiple touches\n\n\n  if (pointersLength > 1 && !session.firstMultiple) {\n    session.firstMultiple = simpleCloneInputData(input);\n  } else if (pointersLength === 1) {\n    session.firstMultiple = false;\n  }\n\n  var firstInput = session.firstInput,\n      firstMultiple = session.firstMultiple;\n  var offsetCenter = firstMultiple ? firstMultiple.center : firstInput.center;\n  var center = input.center = getCenter(pointers);\n  input.timeStamp = now();\n  input.deltaTime = input.timeStamp - firstInput.timeStamp;\n  input.angle = getAngle(offsetCenter, center);\n  input.distance = getDistance(offsetCenter, center);\n  computeDeltaXY(session, input);\n  input.offsetDirection = getDirection(input.deltaX, input.deltaY);\n  var overallVelocity = getVelocity(input.deltaTime, input.deltaX, input.deltaY);\n  input.overallVelocityX = overallVelocity.x;\n  input.overallVelocityY = overallVelocity.y;\n  input.overallVelocity = abs(overallVelocity.x) > abs(overallVelocity.y) ? overallVelocity.x : overallVelocity.y;\n  input.scale = firstMultiple ? getScale(firstMultiple.pointers, pointers) : 1;\n  input.rotation = firstMultiple ? getRotation(firstMultiple.pointers, pointers) : 0;\n  input.maxPointers = !session.prevInput ? input.pointers.length : input.pointers.length > session.prevInput.maxPointers ? input.pointers.length : session.prevInput.maxPointers;\n  computeIntervalInputData(session, input); // find the correct target\n\n  var target = manager.element;\n  var srcEvent = input.srcEvent;\n  var srcEventTarget;\n\n  if (srcEvent.composedPath) {\n    srcEventTarget = srcEvent.composedPath()[0];\n  } else if (srcEvent.path) {\n    srcEventTarget = srcEvent.path[0];\n  } else {\n    srcEventTarget = srcEvent.target;\n  }\n\n  if (hasParent(srcEventTarget, target)) {\n    target = srcEventTarget;\n  }\n\n  input.target = target;\n}\n\n/**\n * @private\n * handle input events\n * @param {Manager} manager\n * @param {String} eventType\n * @param {Object} input\n */\n\nfunction inputHandler(manager, eventType, input) {\n  var pointersLen = input.pointers.length;\n  var changedPointersLen = input.changedPointers.length;\n  var isFirst = eventType & INPUT_START && pointersLen - changedPointersLen === 0;\n  var isFinal = eventType & (INPUT_END | INPUT_CANCEL) && pointersLen - changedPointersLen === 0;\n  input.isFirst = !!isFirst;\n  input.isFinal = !!isFinal;\n\n  if (isFirst) {\n    manager.session = {};\n  } // source event is the normalized value of the domEvents\n  // like 'touchstart, mouseup, pointerdown'\n\n\n  input.eventType = eventType; // compute scale, rotation etc\n\n  computeInputData(manager, input); // emit secret event\n\n  manager.emit('hammer.input', input);\n  manager.recognize(input);\n  manager.session.prevInput = input;\n}\n\n/**\n * @private\n * split string on whitespace\n * @param {String} str\n * @returns {Array} words\n */\nfunction splitStr(str) {\n  return str.trim().split(/\\s+/g);\n}\n\n/**\n * @private\n * addEventListener with multiple events at once\n * @param {EventTarget} target\n * @param {String} types\n * @param {Function} handler\n */\n\nfunction addEventListeners(target, types, handler) {\n  each(splitStr(types), function (type) {\n    target.addEventListener(type, handler, false);\n  });\n}\n\n/**\n * @private\n * removeEventListener with multiple events at once\n * @param {EventTarget} target\n * @param {String} types\n * @param {Function} handler\n */\n\nfunction removeEventListeners(target, types, handler) {\n  each(splitStr(types), function (type) {\n    target.removeEventListener(type, handler, false);\n  });\n}\n\n/**\n * @private\n * get the window object of an element\n * @param {HTMLElement} element\n * @returns {DocumentView|Window}\n */\nfunction getWindowForElement(element) {\n  var doc = element.ownerDocument || element;\n  return doc.defaultView || doc.parentWindow || window;\n}\n\n/**\n * @private\n * create new input type manager\n * @param {Manager} manager\n * @param {Function} callback\n * @returns {Input}\n * @constructor\n */\n\nvar Input =\n/*#__PURE__*/\nfunction () {\n  function Input(manager, callback) {\n    var self = this;\n    this.manager = manager;\n    this.callback = callback;\n    this.element = manager.element;\n    this.target = manager.options.inputTarget; // smaller wrapper around the handler, for the scope and the enabled state of the manager,\n    // so when disabled the input events are completely bypassed.\n\n    this.domHandler = function (ev) {\n      if (boolOrFn(manager.options.enable, [manager])) {\n        self.handler(ev);\n      }\n    };\n\n    this.init();\n  }\n  /**\n   * @private\n   * should handle the inputEvent data and trigger the callback\n   * @virtual\n   */\n\n\n  var _proto = Input.prototype;\n\n  _proto.handler = function handler() {};\n  /**\n   * @private\n   * bind the events\n   */\n\n\n  _proto.init = function init() {\n    this.evEl && addEventListeners(this.element, this.evEl, this.domHandler);\n    this.evTarget && addEventListeners(this.target, this.evTarget, this.domHandler);\n    this.evWin && addEventListeners(getWindowForElement(this.element), this.evWin, this.domHandler);\n  };\n  /**\n   * @private\n   * unbind the events\n   */\n\n\n  _proto.destroy = function destroy() {\n    this.evEl && removeEventListeners(this.element, this.evEl, this.domHandler);\n    this.evTarget && removeEventListeners(this.target, this.evTarget, this.domHandler);\n    this.evWin && removeEventListeners(getWindowForElement(this.element), this.evWin, this.domHandler);\n  };\n\n  return Input;\n}();\n\n/**\n * @private\n * find if a array contains the object using indexOf or a simple polyFill\n * @param {Array} src\n * @param {String} find\n * @param {String} [findByKey]\n * @return {Boolean|Number} false when not found, or the index\n */\nfunction inArray(src, find, findByKey) {\n  if (src.indexOf && !findByKey) {\n    return src.indexOf(find);\n  } else {\n    var i = 0;\n\n    while (i < src.length) {\n      if (findByKey && src[i][findByKey] == find || !findByKey && src[i] === find) {\n        // do not use === here, test fails\n        return i;\n      }\n\n      i++;\n    }\n\n    return -1;\n  }\n}\n\nvar POINTER_INPUT_MAP = {\n  pointerdown: INPUT_START,\n  pointermove: INPUT_MOVE,\n  pointerup: INPUT_END,\n  pointercancel: INPUT_CANCEL,\n  pointerout: INPUT_CANCEL\n}; // in IE10 the pointer types is defined as an enum\n\nvar IE10_POINTER_TYPE_ENUM = {\n  2: INPUT_TYPE_TOUCH,\n  3: INPUT_TYPE_PEN,\n  4: INPUT_TYPE_MOUSE,\n  5: INPUT_TYPE_KINECT // see https://twitter.com/jacobrossi/status/480596438489890816\n\n};\nvar POINTER_ELEMENT_EVENTS = 'pointerdown';\nvar POINTER_WINDOW_EVENTS = 'pointermove pointerup pointercancel'; // IE10 has prefixed support, and case-sensitive\n\nif (win.MSPointerEvent && !win.PointerEvent) {\n  POINTER_ELEMENT_EVENTS = 'MSPointerDown';\n  POINTER_WINDOW_EVENTS = 'MSPointerMove MSPointerUp MSPointerCancel';\n}\n/**\n * @private\n * Pointer events input\n * @constructor\n * @extends Input\n */\n\n\nvar PointerEventInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(PointerEventInput, _Input);\n\n  function PointerEventInput() {\n    var _this;\n\n    var proto = PointerEventInput.prototype;\n    proto.evEl = POINTER_ELEMENT_EVENTS;\n    proto.evWin = POINTER_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.store = _this.manager.session.pointerEvents = [];\n    return _this;\n  }\n  /**\n   * @private\n   * handle mouse events\n   * @param {Object} ev\n   */\n\n\n  var _proto = PointerEventInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var store = this.store;\n    var removePointer = false;\n    var eventTypeNormalized = ev.type.toLowerCase().replace('ms', '');\n    var eventType = POINTER_INPUT_MAP[eventTypeNormalized];\n    var pointerType = IE10_POINTER_TYPE_ENUM[ev.pointerType] || ev.pointerType;\n    var isTouch = pointerType === INPUT_TYPE_TOUCH; // get index of the event in the store\n\n    var storeIndex = inArray(store, ev.pointerId, 'pointerId'); // start and mouse must be down\n\n    if (eventType & INPUT_START && (ev.button === 0 || isTouch)) {\n      if (storeIndex < 0) {\n        store.push(ev);\n        storeIndex = store.length - 1;\n      }\n    } else if (eventType & (INPUT_END | INPUT_CANCEL)) {\n      removePointer = true;\n    } // it not found, so the pointer hasn't been down (so it's probably a hover)\n\n\n    if (storeIndex < 0) {\n      return;\n    } // update the event in the store\n\n\n    store[storeIndex] = ev;\n    this.callback(this.manager, eventType, {\n      pointers: store,\n      changedPointers: [ev],\n      pointerType: pointerType,\n      srcEvent: ev\n    });\n\n    if (removePointer) {\n      // remove from the store\n      store.splice(storeIndex, 1);\n    }\n  };\n\n  return PointerEventInput;\n}(Input);\n\n/**\n * @private\n * convert array-like objects to real arrays\n * @param {Object} obj\n * @returns {Array}\n */\nfunction toArray(obj) {\n  return Array.prototype.slice.call(obj, 0);\n}\n\n/**\n * @private\n * unique array with objects based on a key (like 'id') or just by the array's value\n * @param {Array} src [{id:1},{id:2},{id:1}]\n * @param {String} [key]\n * @param {Boolean} [sort=False]\n * @returns {Array} [{id:1},{id:2}]\n */\n\nfunction uniqueArray(src, key, sort) {\n  var results = [];\n  var values = [];\n  var i = 0;\n\n  while (i < src.length) {\n    var val = key ? src[i][key] : src[i];\n\n    if (inArray(values, val) < 0) {\n      results.push(src[i]);\n    }\n\n    values[i] = val;\n    i++;\n  }\n\n  if (sort) {\n    if (!key) {\n      results = results.sort();\n    } else {\n      results = results.sort(function (a, b) {\n        return a[key] > b[key];\n      });\n    }\n  }\n\n  return results;\n}\n\nvar TOUCH_INPUT_MAP = {\n  touchstart: INPUT_START,\n  touchmove: INPUT_MOVE,\n  touchend: INPUT_END,\n  touchcancel: INPUT_CANCEL\n};\nvar TOUCH_TARGET_EVENTS = 'touchstart touchmove touchend touchcancel';\n/**\n * @private\n * Multi-user touch events input\n * @constructor\n * @extends Input\n */\n\nvar TouchInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(TouchInput, _Input);\n\n  function TouchInput() {\n    var _this;\n\n    TouchInput.prototype.evTarget = TOUCH_TARGET_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.targetIds = {}; // this.evTarget = TOUCH_TARGET_EVENTS;\n\n    return _this;\n  }\n\n  var _proto = TouchInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var type = TOUCH_INPUT_MAP[ev.type];\n    var touches = getTouches.call(this, ev, type);\n\n    if (!touches) {\n      return;\n    }\n\n    this.callback(this.manager, type, {\n      pointers: touches[0],\n      changedPointers: touches[1],\n      pointerType: INPUT_TYPE_TOUCH,\n      srcEvent: ev\n    });\n  };\n\n  return TouchInput;\n}(Input);\n\nfunction getTouches(ev, type) {\n  var allTouches = toArray(ev.touches);\n  var targetIds = this.targetIds; // when there is only one touch, the process can be simplified\n\n  if (type & (INPUT_START | INPUT_MOVE) && allTouches.length === 1) {\n    targetIds[allTouches[0].identifier] = true;\n    return [allTouches, allTouches];\n  }\n\n  var i;\n  var targetTouches;\n  var changedTouches = toArray(ev.changedTouches);\n  var changedTargetTouches = [];\n  var target = this.target; // get target touches from touches\n\n  targetTouches = allTouches.filter(function (touch) {\n    return hasParent(touch.target, target);\n  }); // collect touches\n\n  if (type === INPUT_START) {\n    i = 0;\n\n    while (i < targetTouches.length) {\n      targetIds[targetTouches[i].identifier] = true;\n      i++;\n    }\n  } // filter changed touches to only contain touches that exist in the collected target ids\n\n\n  i = 0;\n\n  while (i < changedTouches.length) {\n    if (targetIds[changedTouches[i].identifier]) {\n      changedTargetTouches.push(changedTouches[i]);\n    } // cleanup removed touches\n\n\n    if (type & (INPUT_END | INPUT_CANCEL)) {\n      delete targetIds[changedTouches[i].identifier];\n    }\n\n    i++;\n  }\n\n  if (!changedTargetTouches.length) {\n    return;\n  }\n\n  return [// merge targetTouches with changedTargetTouches so it contains ALL touches, including 'end' and 'cancel'\n  uniqueArray(targetTouches.concat(changedTargetTouches), 'identifier', true), changedTargetTouches];\n}\n\nvar MOUSE_INPUT_MAP = {\n  mousedown: INPUT_START,\n  mousemove: INPUT_MOVE,\n  mouseup: INPUT_END\n};\nvar MOUSE_ELEMENT_EVENTS = 'mousedown';\nvar MOUSE_WINDOW_EVENTS = 'mousemove mouseup';\n/**\n * @private\n * Mouse events input\n * @constructor\n * @extends Input\n */\n\nvar MouseInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(MouseInput, _Input);\n\n  function MouseInput() {\n    var _this;\n\n    var proto = MouseInput.prototype;\n    proto.evEl = MOUSE_ELEMENT_EVENTS;\n    proto.evWin = MOUSE_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.pressed = false; // mousedown state\n\n    return _this;\n  }\n  /**\n   * @private\n   * handle mouse events\n   * @param {Object} ev\n   */\n\n\n  var _proto = MouseInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var eventType = MOUSE_INPUT_MAP[ev.type]; // on start we want to have the left mouse button down\n\n    if (eventType & INPUT_START && ev.button === 0) {\n      this.pressed = true;\n    }\n\n    if (eventType & INPUT_MOVE && ev.which !== 1) {\n      eventType = INPUT_END;\n    } // mouse must be down\n\n\n    if (!this.pressed) {\n      return;\n    }\n\n    if (eventType & INPUT_END) {\n      this.pressed = false;\n    }\n\n    this.callback(this.manager, eventType, {\n      pointers: [ev],\n      changedPointers: [ev],\n      pointerType: INPUT_TYPE_MOUSE,\n      srcEvent: ev\n    });\n  };\n\n  return MouseInput;\n}(Input);\n\n/**\n * @private\n * Combined touch and mouse input\n *\n * Touch has a higher priority then mouse, and while touching no mouse events are allowed.\n * This because touch devices also emit mouse events while doing a touch.\n *\n * @constructor\n * @extends Input\n */\n\nvar DEDUP_TIMEOUT = 2500;\nvar DEDUP_DISTANCE = 25;\n\nfunction setLastTouch(eventData) {\n  var _eventData$changedPoi = eventData.changedPointers,\n      touch = _eventData$changedPoi[0];\n\n  if (touch.identifier === this.primaryTouch) {\n    var lastTouch = {\n      x: touch.clientX,\n      y: touch.clientY\n    };\n    var lts = this.lastTouches;\n    this.lastTouches.push(lastTouch);\n\n    var removeLastTouch = function removeLastTouch() {\n      var i = lts.indexOf(lastTouch);\n\n      if (i > -1) {\n        lts.splice(i, 1);\n      }\n    };\n\n    setTimeout(removeLastTouch, DEDUP_TIMEOUT);\n  }\n}\n\nfunction recordTouches(eventType, eventData) {\n  if (eventType & INPUT_START) {\n    this.primaryTouch = eventData.changedPointers[0].identifier;\n    setLastTouch.call(this, eventData);\n  } else if (eventType & (INPUT_END | INPUT_CANCEL)) {\n    setLastTouch.call(this, eventData);\n  }\n}\n\nfunction isSyntheticEvent(eventData) {\n  var x = eventData.srcEvent.clientX;\n  var y = eventData.srcEvent.clientY;\n\n  for (var i = 0; i < this.lastTouches.length; i++) {\n    var t = this.lastTouches[i];\n    var dx = Math.abs(x - t.x);\n    var dy = Math.abs(y - t.y);\n\n    if (dx <= DEDUP_DISTANCE && dy <= DEDUP_DISTANCE) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nvar TouchMouseInput =\n/*#__PURE__*/\nfunction () {\n  var TouchMouseInput =\n  /*#__PURE__*/\n  function (_Input) {\n    _inheritsLoose(TouchMouseInput, _Input);\n\n    function TouchMouseInput(_manager, callback) {\n      var _this;\n\n      _this = _Input.call(this, _manager, callback) || this;\n\n      _this.handler = function (manager, inputEvent, inputData) {\n        var isTouch = inputData.pointerType === INPUT_TYPE_TOUCH;\n        var isMouse = inputData.pointerType === INPUT_TYPE_MOUSE;\n\n        if (isMouse && inputData.sourceCapabilities && inputData.sourceCapabilities.firesTouchEvents) {\n          return;\n        } // when we're in a touch event, record touches to  de-dupe synthetic mouse event\n\n\n        if (isTouch) {\n          recordTouches.call(_assertThisInitialized(_assertThisInitialized(_this)), inputEvent, inputData);\n        } else if (isMouse && isSyntheticEvent.call(_assertThisInitialized(_assertThisInitialized(_this)), inputData)) {\n          return;\n        }\n\n        _this.callback(manager, inputEvent, inputData);\n      };\n\n      _this.touch = new TouchInput(_this.manager, _this.handler);\n      _this.mouse = new MouseInput(_this.manager, _this.handler);\n      _this.primaryTouch = null;\n      _this.lastTouches = [];\n      return _this;\n    }\n    /**\n     * @private\n     * handle mouse and touch events\n     * @param {Hammer} manager\n     * @param {String} inputEvent\n     * @param {Object} inputData\n     */\n\n\n    var _proto = TouchMouseInput.prototype;\n\n    /**\n     * @private\n     * remove the event listeners\n     */\n    _proto.destroy = function destroy() {\n      this.touch.destroy();\n      this.mouse.destroy();\n    };\n\n    return TouchMouseInput;\n  }(Input);\n\n  return TouchMouseInput;\n}();\n\n/**\n * @private\n * create new input type manager\n * called by the Manager constructor\n * @param {Hammer} manager\n * @returns {Input}\n */\n\nfunction createInputInstance(manager) {\n  var Type; // let inputClass = manager.options.inputClass;\n\n  var inputClass = manager.options.inputClass;\n\n  if (inputClass) {\n    Type = inputClass;\n  } else if (SUPPORT_POINTER_EVENTS) {\n    Type = PointerEventInput;\n  } else if (SUPPORT_ONLY_TOUCH) {\n    Type = TouchInput;\n  } else if (!SUPPORT_TOUCH) {\n    Type = MouseInput;\n  } else {\n    Type = TouchMouseInput;\n  }\n\n  return new Type(manager, inputHandler);\n}\n\n/**\n * @private\n * if the argument is an array, we want to execute the fn on each entry\n * if it aint an array we don't want to do a thing.\n * this is used by all the methods that accept a single and array argument.\n * @param {*|Array} arg\n * @param {String} fn\n * @param {Object} [context]\n * @returns {Boolean}\n */\n\nfunction invokeArrayArg(arg, fn, context) {\n  if (Array.isArray(arg)) {\n    each(arg, context[fn], context);\n    return true;\n  }\n\n  return false;\n}\n\nvar STATE_POSSIBLE = 1;\nvar STATE_BEGAN = 2;\nvar STATE_CHANGED = 4;\nvar STATE_ENDED = 8;\nvar STATE_RECOGNIZED = STATE_ENDED;\nvar STATE_CANCELLED = 16;\nvar STATE_FAILED = 32;\n\n/**\n * @private\n * get a unique id\n * @returns {number} uniqueId\n */\nvar _uniqueId = 1;\nfunction uniqueId() {\n  return _uniqueId++;\n}\n\n/**\n * @private\n * get a recognizer by name if it is bound to a manager\n * @param {Recognizer|String} otherRecognizer\n * @param {Recognizer} recognizer\n * @returns {Recognizer}\n */\nfunction getRecognizerByNameIfManager(otherRecognizer, recognizer) {\n  var manager = recognizer.manager;\n\n  if (manager) {\n    return manager.get(otherRecognizer);\n  }\n\n  return otherRecognizer;\n}\n\n/**\n * @private\n * get a usable string, used as event postfix\n * @param {constant} state\n * @returns {String} state\n */\n\nfunction stateStr(state) {\n  if (state & STATE_CANCELLED) {\n    return 'cancel';\n  } else if (state & STATE_ENDED) {\n    return 'end';\n  } else if (state & STATE_CHANGED) {\n    return 'move';\n  } else if (state & STATE_BEGAN) {\n    return 'start';\n  }\n\n  return '';\n}\n\n/**\n * @private\n * Recognizer flow explained; *\n * All recognizers have the initial state of POSSIBLE when a input session starts.\n * The definition of a input session is from the first input until the last input, with all it's movement in it. *\n * Example session for mouse-input: mousedown -> mousemove -> mouseup\n *\n * On each recognizing cycle (see Manager.recognize) the .recognize() method is executed\n * which determines with state it should be.\n *\n * If the recognizer has the state FAILED, CANCELLED or RECOGNIZED (equals ENDED), it is reset to\n * POSSIBLE to give it another change on the next cycle.\n *\n *               Possible\n *                  |\n *            +-----+---------------+\n *            |                     |\n *      +-----+-----+               |\n *      |           |               |\n *   Failed      Cancelled          |\n *                          +-------+------+\n *                          |              |\n *                      Recognized       Began\n *                                         |\n *                                      Changed\n *                                         |\n *                                  Ended/Recognized\n */\n\n/**\n * @private\n * Recognizer\n * Every recognizer needs to extend from this class.\n * @constructor\n * @param {Object} options\n */\n\nvar Recognizer =\n/*#__PURE__*/\nfunction () {\n  function Recognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    this.options = _extends({\n      enable: true\n    }, options);\n    this.id = uniqueId();\n    this.manager = null; // default is enable true\n\n    this.state = STATE_POSSIBLE;\n    this.simultaneous = {};\n    this.requireFail = [];\n  }\n  /**\n   * @private\n   * set options\n   * @param {Object} options\n   * @return {Recognizer}\n   */\n\n\n  var _proto = Recognizer.prototype;\n\n  _proto.set = function set(options) {\n    assign$1(this.options, options); // also update the touchAction, in case something changed about the directions/enabled state\n\n    this.manager && this.manager.touchAction.update();\n    return this;\n  };\n  /**\n   * @private\n   * recognize simultaneous with an other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.recognizeWith = function recognizeWith(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'recognizeWith', this)) {\n      return this;\n    }\n\n    var simultaneous = this.simultaneous;\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n\n    if (!simultaneous[otherRecognizer.id]) {\n      simultaneous[otherRecognizer.id] = otherRecognizer;\n      otherRecognizer.recognizeWith(this);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * drop the simultaneous link. it doesnt remove the link on the other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.dropRecognizeWith = function dropRecognizeWith(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'dropRecognizeWith', this)) {\n      return this;\n    }\n\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n    delete this.simultaneous[otherRecognizer.id];\n    return this;\n  };\n  /**\n   * @private\n   * recognizer can only run when an other is failing\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.requireFailure = function requireFailure(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'requireFailure', this)) {\n      return this;\n    }\n\n    var requireFail = this.requireFail;\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n\n    if (inArray(requireFail, otherRecognizer) === -1) {\n      requireFail.push(otherRecognizer);\n      otherRecognizer.requireFailure(this);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * drop the requireFailure link. it does not remove the link on the other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.dropRequireFailure = function dropRequireFailure(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'dropRequireFailure', this)) {\n      return this;\n    }\n\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n    var index = inArray(this.requireFail, otherRecognizer);\n\n    if (index > -1) {\n      this.requireFail.splice(index, 1);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * has require failures boolean\n   * @returns {boolean}\n   */\n\n\n  _proto.hasRequireFailures = function hasRequireFailures() {\n    return this.requireFail.length > 0;\n  };\n  /**\n   * @private\n   * if the recognizer can recognize simultaneous with an other recognizer\n   * @param {Recognizer} otherRecognizer\n   * @returns {Boolean}\n   */\n\n\n  _proto.canRecognizeWith = function canRecognizeWith(otherRecognizer) {\n    return !!this.simultaneous[otherRecognizer.id];\n  };\n  /**\n   * @private\n   * You should use `tryEmit` instead of `emit` directly to check\n   * that all the needed recognizers has failed before emitting.\n   * @param {Object} input\n   */\n\n\n  _proto.emit = function emit(input) {\n    var self = this;\n    var state = this.state;\n\n    function emit(event) {\n      self.manager.emit(event, input);\n    } // 'panstart' and 'panmove'\n\n\n    if (state < STATE_ENDED) {\n      emit(self.options.event + stateStr(state));\n    }\n\n    emit(self.options.event); // simple 'eventName' events\n\n    if (input.additionalEvent) {\n      // additional event(panleft, panright, pinchin, pinchout...)\n      emit(input.additionalEvent);\n    } // panend and pancancel\n\n\n    if (state >= STATE_ENDED) {\n      emit(self.options.event + stateStr(state));\n    }\n  };\n  /**\n   * @private\n   * Check that all the require failure recognizers has failed,\n   * if true, it emits a gesture event,\n   * otherwise, setup the state to FAILED.\n   * @param {Object} input\n   */\n\n\n  _proto.tryEmit = function tryEmit(input) {\n    if (this.canEmit()) {\n      return this.emit(input);\n    } // it's failing anyway\n\n\n    this.state = STATE_FAILED;\n  };\n  /**\n   * @private\n   * can we emit?\n   * @returns {boolean}\n   */\n\n\n  _proto.canEmit = function canEmit() {\n    var i = 0;\n\n    while (i < this.requireFail.length) {\n      if (!(this.requireFail[i].state & (STATE_FAILED | STATE_POSSIBLE))) {\n        return false;\n      }\n\n      i++;\n    }\n\n    return true;\n  };\n  /**\n   * @private\n   * update the recognizer\n   * @param {Object} inputData\n   */\n\n\n  _proto.recognize = function recognize(inputData) {\n    // make a new copy of the inputData\n    // so we can change the inputData without messing up the other recognizers\n    var inputDataClone = assign$1({}, inputData); // is is enabled and allow recognizing?\n\n    if (!boolOrFn(this.options.enable, [this, inputDataClone])) {\n      this.reset();\n      this.state = STATE_FAILED;\n      return;\n    } // reset when we've reached the end\n\n\n    if (this.state & (STATE_RECOGNIZED | STATE_CANCELLED | STATE_FAILED)) {\n      this.state = STATE_POSSIBLE;\n    }\n\n    this.state = this.process(inputDataClone); // the recognizer has recognized a gesture\n    // so trigger an event\n\n    if (this.state & (STATE_BEGAN | STATE_CHANGED | STATE_ENDED | STATE_CANCELLED)) {\n      this.tryEmit(inputDataClone);\n    }\n  };\n  /**\n   * @private\n   * return the state of the recognizer\n   * the actual recognizing happens in this method\n   * @virtual\n   * @param {Object} inputData\n   * @returns {constant} STATE\n   */\n\n  /* jshint ignore:start */\n\n\n  _proto.process = function process(inputData) {};\n  /* jshint ignore:end */\n\n  /**\n   * @private\n   * return the preferred touch-action\n   * @virtual\n   * @returns {Array}\n   */\n\n\n  _proto.getTouchAction = function getTouchAction() {};\n  /**\n   * @private\n   * called when the gesture isn't allowed to recognize\n   * like when another is being recognized or it is disabled\n   * @virtual\n   */\n\n\n  _proto.reset = function reset() {};\n\n  return Recognizer;\n}();\n\n/**\n * @private\n * A tap is recognized when the pointer is doing a small tap/click. Multiple taps are recognized if they occur\n * between the given interval and position. The delay option can be used to recognize multi-taps without firing\n * a single tap.\n *\n * The eventData from the emitted event contains the property `tapCount`, which contains the amount of\n * multi-taps being recognized.\n * @constructor\n * @extends Recognizer\n */\n\nvar TapRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(TapRecognizer, _Recognizer);\n\n  function TapRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _Recognizer.call(this, _extends({\n      event: 'tap',\n      pointers: 1,\n      taps: 1,\n      interval: 300,\n      // max time between the multi-tap taps\n      time: 250,\n      // max time of the pointer to be down (like finger on the screen)\n      threshold: 9,\n      // a minimal movement is ok, but keep it low\n      posThreshold: 10\n    }, options)) || this; // previous time and center,\n    // used for tap counting\n\n    _this.pTime = false;\n    _this.pCenter = false;\n    _this._timer = null;\n    _this._input = null;\n    _this.count = 0;\n    return _this;\n  }\n\n  var _proto = TapRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_MANIPULATION];\n  };\n\n  _proto.process = function process(input) {\n    var _this2 = this;\n\n    var options = this.options;\n    var validPointers = input.pointers.length === options.pointers;\n    var validMovement = input.distance < options.threshold;\n    var validTouchTime = input.deltaTime < options.time;\n    this.reset();\n\n    if (input.eventType & INPUT_START && this.count === 0) {\n      return this.failTimeout();\n    } // we only allow little movement\n    // and we've reached an end event, so a tap is possible\n\n\n    if (validMovement && validTouchTime && validPointers) {\n      if (input.eventType !== INPUT_END) {\n        return this.failTimeout();\n      }\n\n      var validInterval = this.pTime ? input.timeStamp - this.pTime < options.interval : true;\n      var validMultiTap = !this.pCenter || getDistance(this.pCenter, input.center) < options.posThreshold;\n      this.pTime = input.timeStamp;\n      this.pCenter = input.center;\n\n      if (!validMultiTap || !validInterval) {\n        this.count = 1;\n      } else {\n        this.count += 1;\n      }\n\n      this._input = input; // if tap count matches we have recognized it,\n      // else it has began recognizing...\n\n      var tapCount = this.count % options.taps;\n\n      if (tapCount === 0) {\n        // no failing requirements, immediately trigger the tap event\n        // or wait as long as the multitap interval to trigger\n        if (!this.hasRequireFailures()) {\n          return STATE_RECOGNIZED;\n        } else {\n          this._timer = setTimeout(function () {\n            _this2.state = STATE_RECOGNIZED;\n\n            _this2.tryEmit();\n          }, options.interval);\n          return STATE_BEGAN;\n        }\n      }\n    }\n\n    return STATE_FAILED;\n  };\n\n  _proto.failTimeout = function failTimeout() {\n    var _this3 = this;\n\n    this._timer = setTimeout(function () {\n      _this3.state = STATE_FAILED;\n    }, this.options.interval);\n    return STATE_FAILED;\n  };\n\n  _proto.reset = function reset() {\n    clearTimeout(this._timer);\n  };\n\n  _proto.emit = function emit() {\n    if (this.state === STATE_RECOGNIZED) {\n      this._input.tapCount = this.count;\n      this.manager.emit(this.options.event, this._input);\n    }\n  };\n\n  return TapRecognizer;\n}(Recognizer);\n\n/**\n * @private\n * This recognizer is just used as a base for the simple attribute recognizers.\n * @constructor\n * @extends Recognizer\n */\n\nvar AttrRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(AttrRecognizer, _Recognizer);\n\n  function AttrRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _Recognizer.call(this, _extends({\n      pointers: 1\n    }, options)) || this;\n  }\n  /**\n   * @private\n   * Used to check if it the recognizer receives valid input, like input.distance > 10.\n   * @memberof AttrRecognizer\n   * @param {Object} input\n   * @returns {Boolean} recognized\n   */\n\n\n  var _proto = AttrRecognizer.prototype;\n\n  _proto.attrTest = function attrTest(input) {\n    var optionPointers = this.options.pointers;\n    return optionPointers === 0 || input.pointers.length === optionPointers;\n  };\n  /**\n   * @private\n   * Process the input and return the state for the recognizer\n   * @memberof AttrRecognizer\n   * @param {Object} input\n   * @returns {*} State\n   */\n\n\n  _proto.process = function process(input) {\n    var state = this.state;\n    var eventType = input.eventType;\n    var isRecognized = state & (STATE_BEGAN | STATE_CHANGED);\n    var isValid = this.attrTest(input); // on cancel input and we've recognized before, return STATE_CANCELLED\n\n    if (isRecognized && (eventType & INPUT_CANCEL || !isValid)) {\n      return state | STATE_CANCELLED;\n    } else if (isRecognized || isValid) {\n      if (eventType & INPUT_END) {\n        return state | STATE_ENDED;\n      } else if (!(state & STATE_BEGAN)) {\n        return STATE_BEGAN;\n      }\n\n      return state | STATE_CHANGED;\n    }\n\n    return STATE_FAILED;\n  };\n\n  return AttrRecognizer;\n}(Recognizer);\n\n/**\n * @private\n * direction cons to string\n * @param {constant} direction\n * @returns {String}\n */\n\nfunction directionStr(direction) {\n  if (direction === DIRECTION_DOWN) {\n    return 'down';\n  } else if (direction === DIRECTION_UP) {\n    return 'up';\n  } else if (direction === DIRECTION_LEFT) {\n    return 'left';\n  } else if (direction === DIRECTION_RIGHT) {\n    return 'right';\n  }\n\n  return '';\n}\n\n/**\n * @private\n * Pan\n * Recognized when the pointer is down and moved in the allowed direction.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar PanRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(PanRecognizer, _AttrRecognizer);\n\n  function PanRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _AttrRecognizer.call(this, _extends({\n      event: 'pan',\n      threshold: 10,\n      pointers: 1,\n      direction: DIRECTION_ALL\n    }, options)) || this;\n    _this.pX = null;\n    _this.pY = null;\n    return _this;\n  }\n\n  var _proto = PanRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    var direction = this.options.direction;\n    var actions = [];\n\n    if (direction & DIRECTION_HORIZONTAL) {\n      actions.push(TOUCH_ACTION_PAN_Y);\n    }\n\n    if (direction & DIRECTION_VERTICAL) {\n      actions.push(TOUCH_ACTION_PAN_X);\n    }\n\n    return actions;\n  };\n\n  _proto.directionTest = function directionTest(input) {\n    var options = this.options;\n    var hasMoved = true;\n    var distance = input.distance;\n    var direction = input.direction;\n    var x = input.deltaX;\n    var y = input.deltaY; // lock to axis?\n\n    if (!(direction & options.direction)) {\n      if (options.direction & DIRECTION_HORIZONTAL) {\n        direction = x === 0 ? DIRECTION_NONE : x < 0 ? DIRECTION_LEFT : DIRECTION_RIGHT;\n        hasMoved = x !== this.pX;\n        distance = Math.abs(input.deltaX);\n      } else {\n        direction = y === 0 ? DIRECTION_NONE : y < 0 ? DIRECTION_UP : DIRECTION_DOWN;\n        hasMoved = y !== this.pY;\n        distance = Math.abs(input.deltaY);\n      }\n    }\n\n    input.direction = direction;\n    return hasMoved && distance > options.threshold && direction & options.direction;\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return AttrRecognizer.prototype.attrTest.call(this, input) && ( // replace with a super call\n    this.state & STATE_BEGAN || !(this.state & STATE_BEGAN) && this.directionTest(input));\n  };\n\n  _proto.emit = function emit(input) {\n    this.pX = input.deltaX;\n    this.pY = input.deltaY;\n    var direction = directionStr(input.direction);\n\n    if (direction) {\n      input.additionalEvent = this.options.event + direction;\n    }\n\n    _AttrRecognizer.prototype.emit.call(this, input);\n  };\n\n  return PanRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Swipe\n * Recognized when the pointer is moving fast (velocity), with enough distance in the allowed direction.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar SwipeRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(SwipeRecognizer, _AttrRecognizer);\n\n  function SwipeRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'swipe',\n      threshold: 10,\n      velocity: 0.3,\n      direction: DIRECTION_HORIZONTAL | DIRECTION_VERTICAL,\n      pointers: 1\n    }, options)) || this;\n  }\n\n  var _proto = SwipeRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return PanRecognizer.prototype.getTouchAction.call(this);\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    var direction = this.options.direction;\n    var velocity;\n\n    if (direction & (DIRECTION_HORIZONTAL | DIRECTION_VERTICAL)) {\n      velocity = input.overallVelocity;\n    } else if (direction & DIRECTION_HORIZONTAL) {\n      velocity = input.overallVelocityX;\n    } else if (direction & DIRECTION_VERTICAL) {\n      velocity = input.overallVelocityY;\n    }\n\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && direction & input.offsetDirection && input.distance > this.options.threshold && input.maxPointers === this.options.pointers && abs(velocity) > this.options.velocity && input.eventType & INPUT_END;\n  };\n\n  _proto.emit = function emit(input) {\n    var direction = directionStr(input.offsetDirection);\n\n    if (direction) {\n      this.manager.emit(this.options.event + direction, input);\n    }\n\n    this.manager.emit(this.options.event, input);\n  };\n\n  return SwipeRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Pinch\n * Recognized when two or more pointers are moving toward (zoom-in) or away from each other (zoom-out).\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar PinchRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(PinchRecognizer, _AttrRecognizer);\n\n  function PinchRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'pinch',\n      threshold: 0,\n      pointers: 2\n    }, options)) || this;\n  }\n\n  var _proto = PinchRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_NONE];\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && (Math.abs(input.scale - 1) > this.options.threshold || this.state & STATE_BEGAN);\n  };\n\n  _proto.emit = function emit(input) {\n    if (input.scale !== 1) {\n      var inOut = input.scale < 1 ? 'in' : 'out';\n      input.additionalEvent = this.options.event + inOut;\n    }\n\n    _AttrRecognizer.prototype.emit.call(this, input);\n  };\n\n  return PinchRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Rotate\n * Recognized when two or more pointer are moving in a circular motion.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar RotateRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(RotateRecognizer, _AttrRecognizer);\n\n  function RotateRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'rotate',\n      threshold: 0,\n      pointers: 2\n    }, options)) || this;\n  }\n\n  var _proto = RotateRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_NONE];\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && (Math.abs(input.rotation) > this.options.threshold || this.state & STATE_BEGAN);\n  };\n\n  return RotateRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Press\n * Recognized when the pointer is down for x ms without any movement.\n * @constructor\n * @extends Recognizer\n */\n\nvar PressRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(PressRecognizer, _Recognizer);\n\n  function PressRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _Recognizer.call(this, _extends({\n      event: 'press',\n      pointers: 1,\n      time: 251,\n      // minimal time of the pointer to be pressed\n      threshold: 9\n    }, options)) || this;\n    _this._timer = null;\n    _this._input = null;\n    return _this;\n  }\n\n  var _proto = PressRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_AUTO];\n  };\n\n  _proto.process = function process(input) {\n    var _this2 = this;\n\n    var options = this.options;\n    var validPointers = input.pointers.length === options.pointers;\n    var validMovement = input.distance < options.threshold;\n    var validTime = input.deltaTime > options.time;\n    this._input = input; // we only allow little movement\n    // and we've reached an end event, so a tap is possible\n\n    if (!validMovement || !validPointers || input.eventType & (INPUT_END | INPUT_CANCEL) && !validTime) {\n      this.reset();\n    } else if (input.eventType & INPUT_START) {\n      this.reset();\n      this._timer = setTimeout(function () {\n        _this2.state = STATE_RECOGNIZED;\n\n        _this2.tryEmit();\n      }, options.time);\n    } else if (input.eventType & INPUT_END) {\n      return STATE_RECOGNIZED;\n    }\n\n    return STATE_FAILED;\n  };\n\n  _proto.reset = function reset() {\n    clearTimeout(this._timer);\n  };\n\n  _proto.emit = function emit(input) {\n    if (this.state !== STATE_RECOGNIZED) {\n      return;\n    }\n\n    if (input && input.eventType & INPUT_END) {\n      this.manager.emit(this.options.event + \"up\", input);\n    } else {\n      this._input.timeStamp = now();\n      this.manager.emit(this.options.event, this._input);\n    }\n  };\n\n  return PressRecognizer;\n}(Recognizer);\n\nvar defaults = {\n  /**\n   * @private\n   * set if DOM events are being triggered.\n   * But this is slower and unused by simple implementations, so disabled by default.\n   * @type {Boolean}\n   * @default false\n   */\n  domEvents: false,\n\n  /**\n   * @private\n   * The value for the touchAction property/fallback.\n   * When set to `compute` it will magically set the correct value based on the added recognizers.\n   * @type {String}\n   * @default compute\n   */\n  touchAction: TOUCH_ACTION_COMPUTE,\n\n  /**\n   * @private\n   * @type {Boolean}\n   * @default true\n   */\n  enable: true,\n\n  /**\n   * @private\n   * EXPERIMENTAL FEATURE -- can be removed/changed\n   * Change the parent input target element.\n   * If Null, then it is being set the to main element.\n   * @type {Null|EventTarget}\n   * @default null\n   */\n  inputTarget: null,\n\n  /**\n   * @private\n   * force an input class\n   * @type {Null|Function}\n   * @default null\n   */\n  inputClass: null,\n\n  /**\n   * @private\n   * Some CSS properties can be used to improve the working of Hammer.\n   * Add them to this method and they will be set when creating a new Manager.\n   * @namespace\n   */\n  cssProps: {\n    /**\n     * @private\n     * Disables text selection to improve the dragging gesture. Mainly for desktop browsers.\n     * @type {String}\n     * @default 'none'\n     */\n    userSelect: \"none\",\n\n    /**\n     * @private\n     * Disable the Windows Phone grippers when pressing an element.\n     * @type {String}\n     * @default 'none'\n     */\n    touchSelect: \"none\",\n\n    /**\n     * @private\n     * Disables the default callout shown when you touch and hold a touch target.\n     * On iOS, when you touch and hold a touch target such as a link, Safari displays\n     * a callout containing information about the link. This property allows you to disable that callout.\n     * @type {String}\n     * @default 'none'\n     */\n    touchCallout: \"none\",\n\n    /**\n     * @private\n     * Specifies whether zooming is enabled. Used by IE10>\n     * @type {String}\n     * @default 'none'\n     */\n    contentZooming: \"none\",\n\n    /**\n     * @private\n     * Specifies that an entire element should be draggable instead of its contents. Mainly for desktop browsers.\n     * @type {String}\n     * @default 'none'\n     */\n    userDrag: \"none\",\n\n    /**\n     * @private\n     * Overrides the highlight color shown when the user taps a link or a JavaScript\n     * clickable element in iOS. This property obeys the alpha value, if specified.\n     * @type {String}\n     * @default 'rgba(0,0,0,0)'\n     */\n    tapHighlightColor: \"rgba(0,0,0,0)\"\n  }\n};\n/**\n * @private\n * Default recognizer setup when calling `Hammer()`\n * When creating a new Manager these will be skipped.\n * This is separated with other defaults because of tree-shaking.\n * @type {Array}\n */\n\nvar preset = [[RotateRecognizer, {\n  enable: false\n}], [PinchRecognizer, {\n  enable: false\n}, ['rotate']], [SwipeRecognizer, {\n  direction: DIRECTION_HORIZONTAL\n}], [PanRecognizer, {\n  direction: DIRECTION_HORIZONTAL\n}, ['swipe']], [TapRecognizer], [TapRecognizer, {\n  event: 'doubletap',\n  taps: 2\n}, ['tap']], [PressRecognizer]];\n\nvar STOP = 1;\nvar FORCED_STOP = 2;\n/**\n * @private\n * add/remove the css properties as defined in manager.options.cssProps\n * @param {Manager} manager\n * @param {Boolean} add\n */\n\nfunction toggleCssProps(manager, add) {\n  var element = manager.element;\n\n  if (!element.style) {\n    return;\n  }\n\n  var prop;\n  each(manager.options.cssProps, function (value, name) {\n    prop = prefixed(element.style, name);\n\n    if (add) {\n      manager.oldCssProps[prop] = element.style[prop];\n      element.style[prop] = value;\n    } else {\n      element.style[prop] = manager.oldCssProps[prop] || \"\";\n    }\n  });\n\n  if (!add) {\n    manager.oldCssProps = {};\n  }\n}\n/**\n * @private\n * trigger dom event\n * @param {String} event\n * @param {Object} data\n */\n\n\nfunction triggerDomEvent(event, data) {\n  var gestureEvent = document.createEvent(\"Event\");\n  gestureEvent.initEvent(event, true, true);\n  gestureEvent.gesture = data;\n  data.target.dispatchEvent(gestureEvent);\n}\n/**\n* @private\n * Manager\n * @param {HTMLElement} element\n * @param {Object} [options]\n * @constructor\n */\n\n\nvar Manager =\n/*#__PURE__*/\nfunction () {\n  function Manager(element, options) {\n    var _this = this;\n\n    this.options = assign$1({}, defaults, options || {});\n    this.options.inputTarget = this.options.inputTarget || element;\n    this.handlers = {};\n    this.session = {};\n    this.recognizers = [];\n    this.oldCssProps = {};\n    this.element = element;\n    this.input = createInputInstance(this);\n    this.touchAction = new TouchAction(this, this.options.touchAction);\n    toggleCssProps(this, true);\n    each(this.options.recognizers, function (item) {\n      var recognizer = _this.add(new item[0](item[1]));\n\n      item[2] && recognizer.recognizeWith(item[2]);\n      item[3] && recognizer.requireFailure(item[3]);\n    }, this);\n  }\n  /**\n   * @private\n   * set options\n   * @param {Object} options\n   * @returns {Manager}\n   */\n\n\n  var _proto = Manager.prototype;\n\n  _proto.set = function set(options) {\n    assign$1(this.options, options); // Options that need a little more setup\n\n    if (options.touchAction) {\n      this.touchAction.update();\n    }\n\n    if (options.inputTarget) {\n      // Clean up existing event listeners and reinitialize\n      this.input.destroy();\n      this.input.target = options.inputTarget;\n      this.input.init();\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * stop recognizing for this session.\n   * This session will be discarded, when a new [input]start event is fired.\n   * When forced, the recognizer cycle is stopped immediately.\n   * @param {Boolean} [force]\n   */\n\n\n  _proto.stop = function stop(force) {\n    this.session.stopped = force ? FORCED_STOP : STOP;\n  };\n  /**\n   * @private\n   * run the recognizers!\n   * called by the inputHandler function on every movement of the pointers (touches)\n   * it walks through all the recognizers and tries to detect the gesture that is being made\n   * @param {Object} inputData\n   */\n\n\n  _proto.recognize = function recognize(inputData) {\n    var session = this.session;\n\n    if (session.stopped) {\n      return;\n    } // run the touch-action polyfill\n\n\n    this.touchAction.preventDefaults(inputData);\n    var recognizer;\n    var recognizers = this.recognizers; // this holds the recognizer that is being recognized.\n    // so the recognizer's state needs to be BEGAN, CHANGED, ENDED or RECOGNIZED\n    // if no recognizer is detecting a thing, it is set to `null`\n\n    var curRecognizer = session.curRecognizer; // reset when the last recognizer is recognized\n    // or when we're in a new session\n\n    if (!curRecognizer || curRecognizer && curRecognizer.state & STATE_RECOGNIZED) {\n      session.curRecognizer = null;\n      curRecognizer = null;\n    }\n\n    var i = 0;\n\n    while (i < recognizers.length) {\n      recognizer = recognizers[i]; // find out if we are allowed try to recognize the input for this one.\n      // 1.   allow if the session is NOT forced stopped (see the .stop() method)\n      // 2.   allow if we still haven't recognized a gesture in this session, or the this recognizer is the one\n      //      that is being recognized.\n      // 3.   allow if the recognizer is allowed to run simultaneous with the current recognized recognizer.\n      //      this can be setup with the `recognizeWith()` method on the recognizer.\n\n      if (session.stopped !== FORCED_STOP && ( // 1\n      !curRecognizer || recognizer === curRecognizer || // 2\n      recognizer.canRecognizeWith(curRecognizer))) {\n        // 3\n        recognizer.recognize(inputData);\n      } else {\n        recognizer.reset();\n      } // if the recognizer has been recognizing the input as a valid gesture, we want to store this one as the\n      // current active recognizer. but only if we don't already have an active recognizer\n\n\n      if (!curRecognizer && recognizer.state & (STATE_BEGAN | STATE_CHANGED | STATE_ENDED)) {\n        session.curRecognizer = recognizer;\n        curRecognizer = recognizer;\n      }\n\n      i++;\n    }\n  };\n  /**\n   * @private\n   * get a recognizer by its event name.\n   * @param {Recognizer|String} recognizer\n   * @returns {Recognizer|Null}\n   */\n\n\n  _proto.get = function get(recognizer) {\n    if (recognizer instanceof Recognizer) {\n      return recognizer;\n    }\n\n    var recognizers = this.recognizers;\n\n    for (var i = 0; i < recognizers.length; i++) {\n      if (recognizers[i].options.event === recognizer) {\n        return recognizers[i];\n      }\n    }\n\n    return null;\n  };\n  /**\n   * @private add a recognizer to the manager\n   * existing recognizers with the same event name will be removed\n   * @param {Recognizer} recognizer\n   * @returns {Recognizer|Manager}\n   */\n\n\n  _proto.add = function add(recognizer) {\n    if (invokeArrayArg(recognizer, \"add\", this)) {\n      return this;\n    } // remove existing\n\n\n    var existing = this.get(recognizer.options.event);\n\n    if (existing) {\n      this.remove(existing);\n    }\n\n    this.recognizers.push(recognizer);\n    recognizer.manager = this;\n    this.touchAction.update();\n    return recognizer;\n  };\n  /**\n   * @private\n   * remove a recognizer by name or instance\n   * @param {Recognizer|String} recognizer\n   * @returns {Manager}\n   */\n\n\n  _proto.remove = function remove(recognizer) {\n    if (invokeArrayArg(recognizer, \"remove\", this)) {\n      return this;\n    }\n\n    var targetRecognizer = this.get(recognizer); // let's make sure this recognizer exists\n\n    if (recognizer) {\n      var recognizers = this.recognizers;\n      var index = inArray(recognizers, targetRecognizer);\n\n      if (index !== -1) {\n        recognizers.splice(index, 1);\n        this.touchAction.update();\n      }\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * bind event\n   * @param {String} events\n   * @param {Function} handler\n   * @returns {EventEmitter} this\n   */\n\n\n  _proto.on = function on(events, handler) {\n    if (events === undefined || handler === undefined) {\n      return this;\n    }\n\n    var handlers = this.handlers;\n    each(splitStr(events), function (event) {\n      handlers[event] = handlers[event] || [];\n      handlers[event].push(handler);\n    });\n    return this;\n  };\n  /**\n   * @private unbind event, leave emit blank to remove all handlers\n   * @param {String} events\n   * @param {Function} [handler]\n   * @returns {EventEmitter} this\n   */\n\n\n  _proto.off = function off(events, handler) {\n    if (events === undefined) {\n      return this;\n    }\n\n    var handlers = this.handlers;\n    each(splitStr(events), function (event) {\n      if (!handler) {\n        delete handlers[event];\n      } else {\n        handlers[event] && handlers[event].splice(inArray(handlers[event], handler), 1);\n      }\n    });\n    return this;\n  };\n  /**\n   * @private emit event to the listeners\n   * @param {String} event\n   * @param {Object} data\n   */\n\n\n  _proto.emit = function emit(event, data) {\n    // we also want to trigger dom events\n    if (this.options.domEvents) {\n      triggerDomEvent(event, data);\n    } // no handlers, so skip it all\n\n\n    var handlers = this.handlers[event] && this.handlers[event].slice();\n\n    if (!handlers || !handlers.length) {\n      return;\n    }\n\n    data.type = event;\n\n    data.preventDefault = function () {\n      data.srcEvent.preventDefault();\n    };\n\n    var i = 0;\n\n    while (i < handlers.length) {\n      handlers[i](data);\n      i++;\n    }\n  };\n  /**\n   * @private\n   * destroy the manager and unbinds all events\n   * it doesn't unbind dom events, that is the user own responsibility\n   */\n\n\n  _proto.destroy = function destroy() {\n    this.element && toggleCssProps(this, false);\n    this.handlers = {};\n    this.session = {};\n    this.input.destroy();\n    this.element = null;\n  };\n\n  return Manager;\n}();\n\nvar SINGLE_TOUCH_INPUT_MAP = {\n  touchstart: INPUT_START,\n  touchmove: INPUT_MOVE,\n  touchend: INPUT_END,\n  touchcancel: INPUT_CANCEL\n};\nvar SINGLE_TOUCH_TARGET_EVENTS = 'touchstart';\nvar SINGLE_TOUCH_WINDOW_EVENTS = 'touchstart touchmove touchend touchcancel';\n/**\n * @private\n * Touch events input\n * @constructor\n * @extends Input\n */\n\nvar SingleTouchInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(SingleTouchInput, _Input);\n\n  function SingleTouchInput() {\n    var _this;\n\n    var proto = SingleTouchInput.prototype;\n    proto.evTarget = SINGLE_TOUCH_TARGET_EVENTS;\n    proto.evWin = SINGLE_TOUCH_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.started = false;\n    return _this;\n  }\n\n  var _proto = SingleTouchInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var type = SINGLE_TOUCH_INPUT_MAP[ev.type]; // should we handle the touch events?\n\n    if (type === INPUT_START) {\n      this.started = true;\n    }\n\n    if (!this.started) {\n      return;\n    }\n\n    var touches = normalizeSingleTouches.call(this, ev, type); // when done, reset the started state\n\n    if (type & (INPUT_END | INPUT_CANCEL) && touches[0].length - touches[1].length === 0) {\n      this.started = false;\n    }\n\n    this.callback(this.manager, type, {\n      pointers: touches[0],\n      changedPointers: touches[1],\n      pointerType: INPUT_TYPE_TOUCH,\n      srcEvent: ev\n    });\n  };\n\n  return SingleTouchInput;\n}(Input);\n\nfunction normalizeSingleTouches(ev, type) {\n  var all = toArray(ev.touches);\n  var changed = toArray(ev.changedTouches);\n\n  if (type & (INPUT_END | INPUT_CANCEL)) {\n    all = uniqueArray(all.concat(changed), 'identifier', true);\n  }\n\n  return [all, changed];\n}\n\n/**\n * @private\n * wrap a method with a deprecation warning and stack trace\n * @param {Function} method\n * @param {String} name\n * @param {String} message\n * @returns {Function} A new function wrapping the supplied method.\n */\nfunction deprecate(method, name, message) {\n  var deprecationMessage = \"DEPRECATED METHOD: \" + name + \"\\n\" + message + \" AT \\n\";\n  return function () {\n    var e = new Error('get-stack-trace');\n    var stack = e && e.stack ? e.stack.replace(/^[^\\(]+?[\\n$]/gm, '').replace(/^\\s+at\\s+/gm, '').replace(/^Object.<anonymous>\\s*\\(/gm, '{anonymous}()@') : 'Unknown Stack Trace';\n    var log = window.console && (window.console.warn || window.console.log);\n\n    if (log) {\n      log.call(window.console, deprecationMessage, stack);\n    }\n\n    return method.apply(this, arguments);\n  };\n}\n\n/**\n * @private\n * extend object.\n * means that properties in dest will be overwritten by the ones in src.\n * @param {Object} dest\n * @param {Object} src\n * @param {Boolean} [merge=false]\n * @returns {Object} dest\n */\n\nvar extend = deprecate(function (dest, src, merge) {\n  var keys = Object.keys(src);\n  var i = 0;\n\n  while (i < keys.length) {\n    if (!merge || merge && dest[keys[i]] === undefined) {\n      dest[keys[i]] = src[keys[i]];\n    }\n\n    i++;\n  }\n\n  return dest;\n}, 'extend', 'Use `assign`.');\n\n/**\n * @private\n * merge the values from src in the dest.\n * means that properties that exist in dest will not be overwritten by src\n * @param {Object} dest\n * @param {Object} src\n * @returns {Object} dest\n */\n\nvar merge = deprecate(function (dest, src) {\n  return extend(dest, src, true);\n}, 'merge', 'Use `assign`.');\n\n/**\n * @private\n * simple class inheritance\n * @param {Function} child\n * @param {Function} base\n * @param {Object} [properties]\n */\n\nfunction inherit(child, base, properties) {\n  var baseP = base.prototype;\n  var childP;\n  childP = child.prototype = Object.create(baseP);\n  childP.constructor = child;\n  childP._super = baseP;\n\n  if (properties) {\n    assign$1(childP, properties);\n  }\n}\n\n/**\n * @private\n * simple function bind\n * @param {Function} fn\n * @param {Object} context\n * @returns {Function}\n */\nfunction bindFn(fn, context) {\n  return function boundFn() {\n    return fn.apply(context, arguments);\n  };\n}\n\n/**\n * @private\n * Simple way to create a manager with a default set of recognizers.\n * @param {HTMLElement} element\n * @param {Object} [options]\n * @constructor\n */\n\nvar Hammer =\n/*#__PURE__*/\nfunction () {\n  var Hammer =\n  /**\n    * @private\n    * @const {string}\n    */\n  function Hammer(element, options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return new Manager(element, _extends({\n      recognizers: preset.concat()\n    }, options));\n  };\n\n  Hammer.VERSION = \"2.0.17-rc\";\n  Hammer.DIRECTION_ALL = DIRECTION_ALL;\n  Hammer.DIRECTION_DOWN = DIRECTION_DOWN;\n  Hammer.DIRECTION_LEFT = DIRECTION_LEFT;\n  Hammer.DIRECTION_RIGHT = DIRECTION_RIGHT;\n  Hammer.DIRECTION_UP = DIRECTION_UP;\n  Hammer.DIRECTION_HORIZONTAL = DIRECTION_HORIZONTAL;\n  Hammer.DIRECTION_VERTICAL = DIRECTION_VERTICAL;\n  Hammer.DIRECTION_NONE = DIRECTION_NONE;\n  Hammer.DIRECTION_DOWN = DIRECTION_DOWN;\n  Hammer.INPUT_START = INPUT_START;\n  Hammer.INPUT_MOVE = INPUT_MOVE;\n  Hammer.INPUT_END = INPUT_END;\n  Hammer.INPUT_CANCEL = INPUT_CANCEL;\n  Hammer.STATE_POSSIBLE = STATE_POSSIBLE;\n  Hammer.STATE_BEGAN = STATE_BEGAN;\n  Hammer.STATE_CHANGED = STATE_CHANGED;\n  Hammer.STATE_ENDED = STATE_ENDED;\n  Hammer.STATE_RECOGNIZED = STATE_RECOGNIZED;\n  Hammer.STATE_CANCELLED = STATE_CANCELLED;\n  Hammer.STATE_FAILED = STATE_FAILED;\n  Hammer.Manager = Manager;\n  Hammer.Input = Input;\n  Hammer.TouchAction = TouchAction;\n  Hammer.TouchInput = TouchInput;\n  Hammer.MouseInput = MouseInput;\n  Hammer.PointerEventInput = PointerEventInput;\n  Hammer.TouchMouseInput = TouchMouseInput;\n  Hammer.SingleTouchInput = SingleTouchInput;\n  Hammer.Recognizer = Recognizer;\n  Hammer.AttrRecognizer = AttrRecognizer;\n  Hammer.Tap = TapRecognizer;\n  Hammer.Pan = PanRecognizer;\n  Hammer.Swipe = SwipeRecognizer;\n  Hammer.Pinch = PinchRecognizer;\n  Hammer.Rotate = RotateRecognizer;\n  Hammer.Press = PressRecognizer;\n  Hammer.on = addEventListeners;\n  Hammer.off = removeEventListeners;\n  Hammer.each = each;\n  Hammer.merge = merge;\n  Hammer.extend = extend;\n  Hammer.bindFn = bindFn;\n  Hammer.assign = assign$1;\n  Hammer.inherit = inherit;\n  Hammer.bindFn = bindFn;\n  Hammer.prefixed = prefixed;\n  Hammer.toArray = toArray;\n  Hammer.inArray = inArray;\n  Hammer.uniqueArray = uniqueArray;\n  Hammer.splitStr = splitStr;\n  Hammer.boolOrFn = boolOrFn;\n  Hammer.hasParent = hasParent;\n  Hammer.addEventListeners = addEventListeners;\n  Hammer.removeEventListeners = removeEventListeners;\n  Hammer.defaults = assign$1({}, defaults, {\n    preset: preset\n  });\n  return Hammer;\n}();\n\n//  style loader but by script tag, not by the loader.\n\nvar defaults$1 = Hammer.defaults;\n\nexport default Hammer;\nexport { INPUT_START, INPUT_MOVE, INPUT_END, INPUT_CANCEL, STATE_POSSIBLE, STATE_BEGAN, STATE_CHANGED, STATE_ENDED, STATE_RECOGNIZED, STATE_CANCELLED, STATE_FAILED, DIRECTION_NONE, DIRECTION_LEFT, DIRECTION_RIGHT, DIRECTION_UP, DIRECTION_DOWN, DIRECTION_HORIZONTAL, DIRECTION_VERTICAL, DIRECTION_ALL, Manager, Input, TouchAction, TouchInput, MouseInput, PointerEventInput, TouchMouseInput, SingleTouchInput, Recognizer, AttrRecognizer, TapRecognizer as Tap, PanRecognizer as Pan, SwipeRecognizer as Swipe, PinchRecognizer as Pinch, RotateRecognizer as Rotate, PressRecognizer as Press, addEventListeners as on, removeEventListeners as off, each, merge, extend, assign$1 as assign, inherit, bindFn, prefixed, toArray, inArray, uniqueArray, splitStr, boolOrFn, hasParent, addEventListeners, removeEventListeners, defaults$1 as defaults };\n//# sourceMappingURL=hammer.esm.js.map\n","import PropagatingHammer from 'propagating-hammerjs';\nimport Hammer from '@egjs/hammerjs';\n\n/**\n * Setup a mock hammer.js object, for unit testing.\n *\n * Inspiration: https://github.com/uber/deck.gl/pull/658\n *\n * @returns {{on: noop, off: noop, destroy: noop, emit: noop, get: get}}\n */\nfunction hammerMock() {\n  const noop = () => {};\n\n  return {\n    on: noop,\n    off: noop,\n    destroy: noop,\n    emit: noop,\n\n    get(m) {\t//eslint-disable-line no-unused-vars\n      return {\n        set: noop\n      };\n    }\n  };\n}\n\nlet modifiedHammer\n\nif (typeof window !== 'undefined') {\n  const OurHammer = window['Hammer'] || Hammer;\n  modifiedHammer = PropagatingHammer(OurHammer, {\n    preventDefault: 'mouse'\n  });\n} else {\n  modifiedHammer = () => // hammer.js is only available in a browser, not in node.js. Replacing it with a mock object.\n  hammerMock()\n}\n\nexport default modifiedHammer;\n","/**\n * Register a touch event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\nexport function onTouch (hammer, callback) {\n  callback.inputHandler = function (event) {\n    if (event.isFirst) {\n      callback(event);\n    }\n  };\n\n  hammer.on('hammer.input', callback.inputHandler);\n}\n\n/**\n * Register a release event, taking place after a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n * @returns {*}\n */\nexport function onRelease (hammer, callback) {\n  callback.inputHandler = function (event) {\n    if (event.isFinal) {\n      callback(event);\n    }\n  };\n\n  return hammer.on('hammer.input', callback.inputHandler);\n}\n\n\n/**\n * Unregister a touch event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\nexport function offTouch (hammer, callback) {\n  hammer.off('hammer.input', callback.inputHandler);\n}\n\n/**\n * Unregister a release event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\n\nexport const offRelease = offTouch;\n\n/**\n * Hack the PinchRecognizer such that it doesn't prevent default behavior\n * for vertical panning.\n *\n * Yeah ... this is quite a hack ... see https://github.com/hammerjs/hammer.js/issues/932\n *\n * @param {Hammer.Pinch} pinchRecognizer\n * @return {Hammer.Pinch} returns the pinchRecognizer\n */\nexport function disablePreventDefaultVertically (pinchRecognizer) {\n  const TOUCH_ACTION_PAN_Y = 'pan-y';\n\n  pinchRecognizer.getTouchAction = function() {\n    // default method returns [TOUCH_ACTION_NONE]\n    return [TOUCH_ACTION_PAN_Y];\n  };\n\n  return pinchRecognizer;\n}\n","import moment from '../module/moment';\nimport * as DateUtil  from './DateUtil';\nimport util from '../util';\n\n/**\n * The class TimeStep is an iterator for dates. You provide a start date and an\n * end date. The class itself determines the best scale (step size) based on the\n * provided start Date, end Date, and minimumStep.\n *\n * If minimumStep is provided, the step size is chosen as close as possible\n * to the minimumStep but larger than minimumStep. If minimumStep is not\n * provided, the scale is set to 1 DAY.\n * The minimumStep should correspond with the onscreen size of about 6 characters\n *\n * Alternatively, you can set a scale by hand.\n * After creation, you can initialize the class by executing first(). Then you\n * can iterate from the start date to the end date via next(). You can check if\n * the end date is reached with the function hasNext(). After each step, you can\n * retrieve the current date via getCurrent().\n * The TimeStep has scales ranging from milliseconds, seconds, minutes, hours,\n * days, to years.\n *\n * Version: 1.2\n *\n */\nclass TimeStep {\n  /**\n    * @param {Date} [start]         The start date, for example new Date(2010, 9, 21)\n    *                               or new Date(2010, 9, 21, 23, 45, 00)\n    * @param {Date} [end]           The end date\n    * @param {number} [minimumStep] Optional. Minimum step size in milliseconds\n    * @param {Date|Array.<Date>} [hiddenDates] Optional.\n    * @param {{showMajorLabels: boolean, showWeekScale: boolean}} [options] Optional.\n    * @constructor  TimeStep\n    */\n  constructor(start, end, minimumStep, hiddenDates, options) {\n    this.moment = (options && options.moment) || moment;\n    this.options = options ? options : {};\n\n    // variables\n    this.current = this.moment();\n    this._start = this.moment();\n    this._end = this.moment();\n\n    this.autoScale  = true;\n    this.scale = 'day';\n    this.step = 1;\n\n    // initialize the range\n    this.setRange(start, end, minimumStep);\n\n    // hidden Dates options\n    this.switchedDay = false;\n    this.switchedMonth = false;\n    this.switchedYear = false;\n    if (Array.isArray(hiddenDates)) {\n      this.hiddenDates = hiddenDates;\n    }\n    else if (hiddenDates != undefined) {\n      this.hiddenDates = [hiddenDates];\n    }\n    else {\n      this.hiddenDates = [];\n    }\n\n    this.format = TimeStep.FORMAT; // default formatting\n  }\n\n  /**\n   * Set custom constructor function for moment. Can be used to set dates\n   * to UTC or to set a utcOffset.\n   * @param {function} moment\n   */\n  setMoment(moment) {\n    this.moment = moment;\n\n    // update the date properties, can have a new utcOffset\n    this.current = this.moment(this.current.valueOf());\n    this._start = this.moment(this._start.valueOf());\n    this._end = this.moment(this._end.valueOf());\n  }\n\n  /**\n   * Set custom formatting for the minor an major labels of the TimeStep.\n   * Both `minorLabels` and `majorLabels` are an Object with properties:\n   * 'millisecond', 'second', 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'.\n   * @param {{minorLabels: Object, majorLabels: Object}} format\n   */\n  setFormat(format) {\n    const defaultFormat = util.deepExtend({}, TimeStep.FORMAT);\n    this.format = util.deepExtend(defaultFormat, format);\n  }\n\n  /**\n   * Set a new range\n   * If minimumStep is provided, the step size is chosen as close as possible\n   * to the minimumStep but larger than minimumStep. If minimumStep is not\n   * provided, the scale is set to 1 DAY.\n   * The minimumStep should correspond with the onscreen size of about 6 characters\n   * @param {Date} [start]      The start date and time.\n   * @param {Date} [end]        The end date and time.\n   * @param {int} [minimumStep] Optional. Minimum step size in milliseconds\n   */\n  setRange(start, end, minimumStep) {\n    if (!(start instanceof Date) || !(end instanceof Date)) {\n      throw  \"No legal start or end date in method setRange\";\n    }\n\n    this._start = (start != undefined) ? this.moment(start.valueOf()) : Date.now();\n    this._end = (end != undefined) ? this.moment(end.valueOf()) : Date.now();\n\n    if (this.autoScale) {\n      this.setMinimumStep(minimumStep);\n    }\n  }\n\n  /**\n   * Set the range iterator to the start date.\n   */\n  start() {\n    this.current = this._start.clone();\n    this.roundToMinor();\n  }\n\n  /**\n   * Round the current date to the first minor date value\n   * This must be executed once when the current date is set to start Date\n   */\n  roundToMinor() {\n    // round to floor\n    // to prevent year & month scales rounding down to the first day of week we perform this separately\n    if (this.scale == 'week') {\n      this.current.weekday(0);\n    }\n    // IMPORTANT: we have no breaks in this switch! (this is no bug)\n    // noinspection FallThroughInSwitchStatementJS\n    switch (this.scale) {\n      case 'year':\n        this.current.year(this.step * Math.floor(this.current.year() / this.step));\n        this.current.month(0);\n      case 'month':        this.current.date(1);          // eslint-disable-line no-fallthrough\n      case 'week':                                        // eslint-disable-line no-fallthrough\n      case 'day':                                         // eslint-disable-line no-fallthrough\n      case 'weekday':      this.current.hours(0);         // eslint-disable-line no-fallthrough\n      case 'hour':         this.current.minutes(0);       // eslint-disable-line no-fallthrough\n      case 'minute':       this.current.seconds(0);       // eslint-disable-line no-fallthrough\n      case 'second':       this.current.milliseconds(0);  // eslint-disable-line no-fallthrough\n      //case 'millisecond': // nothing to do for milliseconds\n    }\n\n    if (this.step != 1) {\n      // round down to the first minor value that is a multiple of the current step size\n      let  priorCurrent = this.current.clone();\n      switch (this.scale) {        \n        case 'millisecond':  this.current.subtract(this.current.milliseconds() % this.step, 'milliseconds');  break;\n        case 'second':       this.current.subtract(this.current.seconds() % this.step, 'seconds'); break;\n        case 'minute':       this.current.subtract(this.current.minutes() % this.step, 'minutes'); break;\n        case 'hour':         this.current.subtract(this.current.hours() % this.step, 'hours'); break;\n        case 'weekday':      // intentional fall through\n        case 'day':          this.current.subtract((this.current.date() - 1) % this.step, 'day'); break;\n        case 'week':         this.current.subtract(this.current.week() % this.step, 'week'); break;\n        case 'month':        this.current.subtract(this.current.month() % this.step, 'month');  break;\n        case 'year':         this.current.subtract(this.current.year() % this.step, 'year'); break;\n        default: break;\n      }\n      if (!priorCurrent.isSame(this.current)) {\n          this.current = this.moment(DateUtil.snapAwayFromHidden(this.hiddenDates, this.current.valueOf(), -1, true));\n      }\n    }\n  }\n\n  /**\n   * Check if the there is a next step\n   * @return {boolean}  true if the current date has not passed the end date\n   */\n  hasNext() {\n    return (this.current.valueOf() <= this._end.valueOf());\n  }\n\n  /**\n   * Do the next step\n   */\n  next() {\n    const prev = this.current.valueOf();\n\n    // Two cases, needed to prevent issues with switching daylight savings\n    // (end of March and end of October)\n    switch (this.scale) {\n      case 'millisecond':  this.current.add(this.step, 'millisecond'); break;\n      case 'second':       this.current.add(this.step, 'second'); break;\n      case 'minute':       this.current.add(this.step, 'minute'); break;\n      case 'hour':\n        this.current.add(this.step, 'hour');\n\n        if (this.current.month() < 6) {\n          this.current.subtract(this.current.hours() % this.step, 'hour');\n        } else {\n          if (this.current.hours() % this.step !== 0) {\n            this.current.add(this.step - this.current.hours() % this.step, 'hour');\n          }\n        }\n        break;\n      case 'weekday':      // intentional fall through\n      case 'day':          this.current.add(this.step, 'day'); break;\n      case 'week':\n        if (this.current.weekday() !== 0){ // we had a month break not correlating with a week's start before\n          this.current.weekday(0); // switch back to week cycles\n          this.current.add(this.step, 'week');\n        } else if(this.options.showMajorLabels === false) {\n          this.current.add(this.step, 'week'); // the default case\n        } else { // first day of the week\n          const nextWeek = this.current.clone();\n          nextWeek.add(1, 'week');\n          if(nextWeek.isSame(this.current, 'month')){ // is the first day of the next week in the same month?\n            this.current.add(this.step, 'week'); // the default case\n          } else { // inject a step at each first day of the month\n            this.current.add(this.step, 'week');\n            this.current.date(1);\n          }\n        }\n        break;\n      case 'month':        this.current.add(this.step, 'month'); break;\n      case 'year':         this.current.add(this.step, 'year'); break;\n      default: break;\n    }\n\n    if (this.step != 1) {\n      // round down to the correct major value\n      switch (this.scale) {\n        case 'millisecond':  if(this.current.milliseconds() > 0 && this.current.milliseconds() < this.step) this.current.milliseconds(0);  break;\n        case 'second':       if(this.current.seconds() > 0 && this.current.seconds() < this.step) this.current.seconds(0);  break;\n        case 'minute':       if(this.current.minutes() > 0 && this.current.minutes() < this.step) this.current.minutes(0); break;\n        case 'hour':         if(this.current.hours() > 0 && this.current.hours() < this.step) this.current.hours(0);  break;\n        case 'weekday':      // intentional fall through\n        case 'day':          if(this.current.date() < this.step+1) this.current.date(1); break;\n        case 'week':         if(this.current.week() < this.step) this.current.week(1); break; // week numbering starts at 1, not 0\n        case 'month':        if(this.current.month() < this.step) this.current.month(0);  break;\n        case 'year':         break; // nothing to do for year\n        default:             break;\n      }\n    }\n\n    // safety mechanism: if current time is still unchanged, move to the end\n    if (this.current.valueOf() == prev) {\n      this.current = this._end.clone();\n    }\n\n    // Reset switches for year, month and day. Will get set to true where appropriate in DateUtil.stepOverHiddenDates\n    this.switchedDay = false;\n    this.switchedMonth = false;\n    this.switchedYear = false;\n\n    DateUtil.stepOverHiddenDates(this.moment, this, prev);\n  }\n\n  /**\n   * Get the current datetime\n   * @return {Moment}  current The current date\n   */\n  getCurrent() {\n    return this.current.clone();\n  }\n\n  /**\n   * Set a custom scale. Autoscaling will be disabled.\n   * For example setScale('minute', 5) will result\n   * in minor steps of 5 minutes, and major steps of an hour.\n   *\n   * @param {{scale: string, step: number}} params\n   *                               An object containing two properties:\n   *                               - A string 'scale'. Choose from 'millisecond', 'second',\n   *                                 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'.\n   *                               - A number 'step'. A step size, by default 1.\n   *                                 Choose for example 1, 2, 5, or 10.\n   */\n  setScale(params) {\n    if (params && typeof params.scale == 'string') {\n      this.scale = params.scale;\n      this.step = params.step > 0 ? params.step : 1;\n      this.autoScale = false;\n    }\n  }\n\n  /**\n   * Enable or disable autoscaling\n   * @param {boolean} enable  If true, autoascaling is set true\n   */\n  setAutoScale(enable) {\n    this.autoScale = enable;\n  }\n\n  /**\n   * Automatically determine the scale that bests fits the provided minimum step\n   * @param {number} [minimumStep]  The minimum step size in milliseconds\n   */\n  setMinimumStep(minimumStep) {\n    if (minimumStep == undefined) {\n      return;\n    }\n\n    //var b = asc + ds;\n\n    const stepYear       = (1000 * 60 * 60 * 24 * 30 * 12);\n    const stepMonth      = (1000 * 60 * 60 * 24 * 30);\n    const stepDay        = (1000 * 60 * 60 * 24);\n    const stepHour       = (1000 * 60 * 60);\n    const stepMinute     = (1000 * 60);\n    const stepSecond     = (1000);\n    const stepMillisecond= (1);\n\n    // find the smallest step that is larger than the provided minimumStep\n    if (stepYear*1000 > minimumStep)        {this.scale = 'year';        this.step = 1000;}\n    if (stepYear*500 > minimumStep)         {this.scale = 'year';        this.step = 500;}\n    if (stepYear*100 > minimumStep)         {this.scale = 'year';        this.step = 100;}\n    if (stepYear*50 > minimumStep)          {this.scale = 'year';        this.step = 50;}\n    if (stepYear*10 > minimumStep)          {this.scale = 'year';        this.step = 10;}\n    if (stepYear*5 > minimumStep)           {this.scale = 'year';        this.step = 5;}\n    if (stepYear > minimumStep)             {this.scale = 'year';        this.step = 1;}\n    if (stepMonth*3 > minimumStep)          {this.scale = 'month';       this.step = 3;}\n    if (stepMonth > minimumStep)            {this.scale = 'month';       this.step = 1;}\n    if (stepDay*7 > minimumStep && this.options.showWeekScale)            {this.scale = 'week';        this.step = 1;}\n    if (stepDay*2 > minimumStep)            {this.scale = 'day';         this.step = 2;}\n    if (stepDay > minimumStep)              {this.scale = 'day';         this.step = 1;}\n    if (stepDay/2 > minimumStep)            {this.scale = 'weekday';     this.step = 1;}\n    if (stepHour*4 > minimumStep)           {this.scale = 'hour';        this.step = 4;}\n    if (stepHour > minimumStep)             {this.scale = 'hour';        this.step = 1;}\n    if (stepMinute*15 > minimumStep)        {this.scale = 'minute';      this.step = 15;}\n    if (stepMinute*10 > minimumStep)        {this.scale = 'minute';      this.step = 10;}\n    if (stepMinute*5 > minimumStep)         {this.scale = 'minute';      this.step = 5;}\n    if (stepMinute > minimumStep)           {this.scale = 'minute';      this.step = 1;}\n    if (stepSecond*15 > minimumStep)        {this.scale = 'second';      this.step = 15;}\n    if (stepSecond*10 > minimumStep)        {this.scale = 'second';      this.step = 10;}\n    if (stepSecond*5 > minimumStep)         {this.scale = 'second';      this.step = 5;}\n    if (stepSecond > minimumStep)           {this.scale = 'second';      this.step = 1;}\n    if (stepMillisecond*200 > minimumStep)  {this.scale = 'millisecond'; this.step = 200;}\n    if (stepMillisecond*100 > minimumStep)  {this.scale = 'millisecond'; this.step = 100;}\n    if (stepMillisecond*50 > minimumStep)   {this.scale = 'millisecond'; this.step = 50;}\n    if (stepMillisecond*10 > minimumStep)   {this.scale = 'millisecond'; this.step = 10;}\n    if (stepMillisecond*5 > minimumStep)    {this.scale = 'millisecond'; this.step = 5;}\n    if (stepMillisecond > minimumStep)      {this.scale = 'millisecond'; this.step = 1;}\n  }\n\n  /**\n   * Snap a date to a rounded value.\n   * The snap intervals are dependent on the current scale and step.\n   * Static function\n   * @param {Date} date    the date to be snapped.\n   * @param {string} scale Current scale, can be 'millisecond', 'second',\n   *                       'minute', 'hour', 'weekday, 'day', 'week', 'month', 'year'.\n   * @param {number} step  Current step (1, 2, 4, 5, ...\n   * @return {Date} snappedDate\n   */\n  static snap(date, scale, step) {\n    const clone = moment(date);\n\n    if (scale == 'year') {\n      const year = clone.year() + Math.round(clone.month() / 12);\n      clone.year(Math.round(year / step) * step);\n      clone.month(0);\n      clone.date(0);\n      clone.hours(0);\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'month') {\n      if (clone.date() > 15) {\n        clone.date(1);\n        clone.add(1, 'month');\n        // important: first set Date to 1, after that change the month.\n      }\n      else {\n        clone.date(1);\n      }\n\n      clone.hours(0);\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'week') {\n        if (clone.weekday() > 2) { // doing it the momentjs locale aware way\n            clone.weekday(0);\n            clone.add(1, 'week');\n        }\n        else {\n            clone.weekday(0);\n        }\n\n        clone.hours(0);\n        clone.minutes(0);\n        clone.seconds(0);\n        clone.milliseconds(0);\n    }\n    else if (scale == 'day') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 5:\n        case 2:\n          clone.hours(Math.round(clone.hours() / 24) * 24); break;\n        default:\n          clone.hours(Math.round(clone.hours() / 12) * 12); break;\n      }\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'weekday') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 5:\n        case 2:\n          clone.hours(Math.round(clone.hours() / 12) * 12); break;\n        default:\n          clone.hours(Math.round(clone.hours() / 6) * 6); break;\n      }\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'hour') {\n      switch (step) {\n        case 4:\n          clone.minutes(Math.round(clone.minutes() / 60) * 60); break;\n        default:\n          clone.minutes(Math.round(clone.minutes() / 30) * 30); break;\n      }\n      clone.seconds(0);\n      clone.milliseconds(0);\n    } else if (scale == 'minute') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 15:\n        case 10:\n          clone.minutes(Math.round(clone.minutes() / 5) * 5);\n          clone.seconds(0);\n          break;\n        case 5:\n          clone.seconds(Math.round(clone.seconds() / 60) * 60); break;\n        default:\n          clone.seconds(Math.round(clone.seconds() / 30) * 30); break;\n      }\n      clone.milliseconds(0);\n    }\n    else if (scale == 'second') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 15:\n        case 10:\n          clone.seconds(Math.round(clone.seconds() / 5) * 5);\n          clone.milliseconds(0);\n          break;\n        case 5:\n          clone.milliseconds(Math.round(clone.milliseconds() / 1000) * 1000); break;\n        default:\n          clone.milliseconds(Math.round(clone.milliseconds() / 500) * 500); break;\n      }\n    }\n    else if (scale == 'millisecond') {\n      const _step = step > 5 ? step / 2 : 1;\n      clone.milliseconds(Math.round(clone.milliseconds() / _step) * _step);\n    }\n\n    return clone;\n  }\n\n  /**\n   * Check if the current value is a major value (for example when the step\n   * is DAY, a major value is each first day of the MONTH)\n   * @return {boolean} true if current date is major, else false.\n   */\n  isMajor() {\n    if (this.switchedYear == true) {\n      switch (this.scale) {\n        case 'year':\n        case 'month':\n        case 'week':\n        case 'weekday':\n        case 'day':\n        case 'hour':\n        case 'minute':\n        case 'second':\n        case 'millisecond':\n          return true;\n        default:\n          return false;\n      }\n    }\n    else if (this.switchedMonth == true) {\n      switch (this.scale) {\n        case 'week':\n        case 'weekday':\n        case 'day':\n        case 'hour':\n        case 'minute':\n        case 'second':\n        case 'millisecond':\n          return true;\n        default:\n          return false;\n      }\n    }\n    else if (this.switchedDay == true) {\n      switch (this.scale) {\n        case 'millisecond':\n        case 'second':\n        case 'minute':\n        case 'hour':\n          return true;\n        default:\n          return false;\n      }\n    }\n\n    const date = this.moment(this.current);\n    switch (this.scale) {\n      case 'millisecond':\n        return (date.milliseconds() == 0);\n      case 'second':\n        return (date.seconds() == 0);\n      case 'minute':\n        return (date.hours() == 0) && (date.minutes() == 0);\n      case 'hour':\n        return (date.hours() == 0);\n      case 'weekday': // intentional fall through\n      case 'day':\n        return (date.date() == 1);\n      case 'week':\n        return (date.date() == 1);\n      case 'month':\n        return (date.month() == 0);\n      case 'year':\n        return false;\n      default:\n        return false;\n    }\n  }\n\n  /**\n   * Returns formatted text for the minor axislabel, depending on the current\n   * date and the scale. For example when scale is MINUTE, the current time is\n   * formatted as \"hh:mm\".\n   * @param {Date} [date=this.current] custom date. if not provided, current date is taken\n   * @returns {String}\n   */\n  getLabelMinor(date) {\n    if (date == undefined) {\n      date = this.current;\n    }\n    if (date instanceof Date) {\n      date = this.moment(date)\n    }\n\n    if (typeof(this.format.minorLabels) === \"function\") {\n      return this.format.minorLabels(date, this.scale, this.step);\n    }\n\n    const format = this.format.minorLabels[this.scale];\n    // noinspection FallThroughInSwitchStatementJS\n    switch (this.scale) {\n      case 'week':\n        // Don't draw the minor label if this date is the first day of a month AND if it's NOT the start of the week.\n        // The 'date' variable may actually be the 'next' step when called from TimeAxis' _repaintLabels.\n        if(date.date() === 1 && date.weekday() !== 0){\n            return \"\";\n        }\n      default: // eslint-disable-line no-fallthrough\n        return (format && format.length > 0) ? this.moment(date).format(format) : '';\n    }\n  }\n\n  /**\n   * Returns formatted text for the major axis label, depending on the current\n   * date and the scale. For example when scale is MINUTE, the major scale is\n   * hours, and the hour will be formatted as \"hh\".\n   * @param {Date} [date=this.current] custom date. if not provided, current date is taken\n   * @returns {String}\n   */\n  getLabelMajor(date) {\n    if (date == undefined) {\n      date = this.current;\n    }\n    if (date instanceof Date) {\n      date = this.moment(date)\n    }\n\n    if (typeof(this.format.majorLabels) === \"function\") {\n      return this.format.majorLabels(date, this.scale, this.step);\n    }\n\n    const format = this.format.majorLabels[this.scale];\n    return (format && format.length > 0) ? this.moment(date).format(format) : '';\n  }\n\n  /**\n   * get class name\n   * @return {string} class name\n   */\n  getClassName() {\n    const _moment = this.moment;\n    const m = this.moment(this.current);\n    const current = m.locale ? m.locale('en') : m.lang('en'); // old versions of moment have .lang() function\n    const step = this.step;\n    const classNames = [];\n\n    /**\n     *\n     * @param {number} value\n     * @returns {String}\n     */\n    function even(value) {\n      return (value / step % 2 == 0) ? ' vis-even' : ' vis-odd';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function today(date) {\n      if (date.isSame(Date.now(), 'day')) {\n        return ' vis-today';\n      }\n      if (date.isSame(_moment().add(1, 'day'), 'day')) {\n        return ' vis-tomorrow';\n      }\n      if (date.isSame(_moment().add(-1, 'day'), 'day')) {\n        return ' vis-yesterday';\n      }\n      return '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentWeek(date) {\n      return date.isSame(Date.now(), 'week') ? ' vis-current-week' : '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentMonth(date) {\n      return date.isSame(Date.now(), 'month') ? ' vis-current-month' : '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentYear(date) {\n      return date.isSame(Date.now(), 'year') ? ' vis-current-year' : '';\n    }\n\n    switch (this.scale) {\n      case 'millisecond':\n        classNames.push(today(current));\n        classNames.push(even(current.milliseconds()));\n        break;\n      case 'second':\n        classNames.push(today(current));\n        classNames.push(even(current.seconds()));\n        break;\n      case 'minute':\n        classNames.push(today(current));\n        classNames.push(even(current.minutes()));\n        break;\n      case 'hour':\n        classNames.push(`vis-h${current.hours()}${this.step == 4 ? '-h' + (current.hours() + 4) : ''}`);\n        classNames.push(today(current));\n        classNames.push(even(current.hours()));\n        break;\n      case 'weekday':\n        classNames.push(`vis-${current.format('dddd').toLowerCase()}`);\n        classNames.push(today(current));\n        classNames.push(currentWeek(current));\n        classNames.push(even(current.date()));\n        break;\n      case 'day':\n        classNames.push(`vis-day${current.date()}`);\n        classNames.push(`vis-${current.format('MMMM').toLowerCase()}`);\n        classNames.push(today(current));\n        classNames.push(currentMonth(current));\n        classNames.push(this.step <= 2 ? today(current) : '');\n        classNames.push(this.step <= 2 ? `vis-${current.format('dddd').toLowerCase()}` : '');\n        classNames.push(even(current.date() - 1));\n        break;\n      case 'week':\n        classNames.push(`vis-week${current.format('w')}`);\n        classNames.push(currentWeek(current));\n        classNames.push(even(current.week()));\n        break;\n      case 'month':\n        classNames.push(`vis-${current.format('MMMM').toLowerCase()}`);\n        classNames.push(currentMonth(current));\n        classNames.push(even(current.month()));\n        break;\n      case 'year':\n        classNames.push(`vis-year${current.year()}`);\n        classNames.push(currentYear(current));\n        classNames.push(even(current.year()));\n        break;\n    }\n    return classNames.filter(String).join(\" \");\n  }\n}\n\n// Time formatting\nTimeStep.FORMAT = {\n  minorLabels: {\n    millisecond:'SSS',\n    second:     's',\n    minute:     'HH:mm',\n    hour:       'HH:mm',\n    weekday:    'ddd D',\n    day:        'D',\n    week:       'w',\n    month:      'MMM',\n    year:       'YYYY'\n  },\n  majorLabels: {\n    millisecond:'HH:mm:ss',\n    second:     'D MMMM HH:mm',\n    minute:     'ddd D MMMM',\n    hour:       'ddd D MMMM',\n    weekday:    'MMMM YYYY',\n    day:        'MMMM YYYY',\n    week:       'MMMM YYYY',\n    month:      'YYYY',\n    year:       ''\n  }\n};\n\nexport default TimeStep;\n","function styleInject(css, ref) {\n  if ( ref === void 0 ) ref = {};\n  var insertAt = ref.insertAt;\n\n  if (!css || typeof document === 'undefined') { return; }\n\n  var head = document.head || document.getElementsByTagName('head')[0];\n  var style = document.createElement('style');\n  style.type = 'text/css';\n\n  if (insertAt === 'top') {\n    if (head.firstChild) {\n      head.insertBefore(style, head.firstChild);\n    } else {\n      head.appendChild(style);\n    }\n  } else {\n    head.appendChild(style);\n  }\n\n  if (style.styleSheet) {\n    style.styleSheet.cssText = css;\n  } else {\n    style.appendChild(document.createTextNode(css));\n  }\n}\n\nexport default styleInject;\n","import util from '../../util';\nimport Component from'./Component';\nimport TimeStep from '../TimeStep';\nimport * as DateUtil  from '../DateUtil';\nimport moment from '../../module/moment';\n\nimport './css/timeaxis.css';\n\n/** A horizontal time axis */\nclass TimeAxis extends Component {\n/**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} body\n * @param {Object} [options]        See TimeAxis.setOptions for the available\n *                                  options.\n * @constructor TimeAxis\n * @extends Component\n */\n  constructor(body, options) {\n    super();\n    this.dom = {\n      foreground: null,\n      lines: [],\n      majorTexts: [],\n      minorTexts: [],\n      redundant: {\n        lines: [],\n        majorTexts: [],\n        minorTexts: []\n      }\n    };\n    this.props = {\n      range: {\n        start: 0,\n        end: 0,\n        minimumStep: 0\n      },\n      lineTop: 0\n    };\n\n    this.defaultOptions = {\n      orientation: {\n        axis: 'bottom'\n      },  // axis orientation: 'top' or 'bottom'\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      maxMinorChars: 7,\n      format: TimeStep.FORMAT,\n      moment,\n      timeAxis: null\n    };\n    this.options = util.extend({}, this.defaultOptions);\n\n    this.body = body;\n\n    // create the HTML DOM\n    this._create();\n\n    this.setOptions(options);\n  }\n\n  /**\n   * Set options for the TimeAxis.\n   * Parameters will be merged in current options.\n   * @param {Object} options  Available options:\n   *                          {string} [orientation.axis]\n   *                          {boolean} [showMinorLabels]\n   *                          {boolean} [showMajorLabels]\n   *                          {boolean} [showWeekScale]\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend([\n        'showMinorLabels',\n        'showMajorLabels',\n        'showWeekScale',\n        'maxMinorChars',\n        'hiddenDates',\n        'timeAxis',\n        'moment',\n        'rtl'\n      ], this.options, options);\n\n      // deep copy the format options\n      util.selectiveDeepExtend(['format'], this.options, options);\n\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation.axis = options.orientation;\n        }\n        else if (typeof options.orientation === 'object' && 'axis' in options.orientation) {\n          this.options.orientation.axis = options.orientation.axis;\n        }\n      }\n\n      // apply locale to moment.js\n      // TODO: not so nice, this is applied globally to moment.js\n      if ('locale' in options) {\n        if (typeof moment.locale === 'function') {\n          // moment.js 2.8.1+\n          moment.locale(options.locale);\n        }\n        else {\n          moment.lang(options.locale);\n        }\n      }\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the TimeAxis\n   */\n  _create() {\n    this.dom.foreground = document.createElement('div');\n    this.dom.background = document.createElement('div');\n\n    this.dom.foreground.className = 'vis-time-axis vis-foreground';\n    this.dom.background.className = 'vis-time-axis vis-background';\n  }\n\n  /**\n   * Destroy the TimeAxis\n   */\n  destroy() {\n    // remove from DOM\n    if (this.dom.foreground.parentNode) {\n      this.dom.foreground.parentNode.removeChild(this.dom.foreground);\n    }\n    if (this.dom.background.parentNode) {\n      this.dom.background.parentNode.removeChild(this.dom.background);\n    }\n\n    this.body = null;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const props = this.props;\n    const foreground = this.dom.foreground;\n    const background = this.dom.background;\n\n    // determine the correct parent DOM element (depending on option orientation)\n    const parent = (this.options.orientation.axis == 'top') ? this.body.dom.top : this.body.dom.bottom;\n    const parentChanged = (foreground.parentNode !== parent);\n\n    // calculate character width and height\n    this._calculateCharSize();\n\n    // TODO: recalculate sizes only needed when parent is resized or options is changed\n    const showMinorLabels = this.options.showMinorLabels && this.options.orientation.axis !== 'none';\n    const showMajorLabels = this.options.showMajorLabels && this.options.orientation.axis !== 'none';\n\n    // determine the width and height of the elemens for the axis\n    props.minorLabelHeight = showMinorLabels ? props.minorCharHeight : 0;\n    props.majorLabelHeight = showMajorLabels ? props.majorCharHeight : 0;\n    props.height = props.minorLabelHeight + props.majorLabelHeight;\n    props.width = foreground.offsetWidth;\n\n    props.minorLineHeight = this.body.domProps.root.height - props.majorLabelHeight -\n        (this.options.orientation.axis == 'top' ? this.body.domProps.bottom.height : this.body.domProps.top.height);\n    props.minorLineWidth = 1; // TODO: really calculate width\n    props.majorLineHeight = props.minorLineHeight + props.majorLabelHeight;\n    props.majorLineWidth = 1; // TODO: really calculate width\n\n    //  take foreground and background offline while updating (is almost twice as fast)\n    const foregroundNextSibling = foreground.nextSibling;\n    const backgroundNextSibling = background.nextSibling;\n    foreground.parentNode && foreground.parentNode.removeChild(foreground);\n    background.parentNode && background.parentNode.removeChild(background);\n\n    foreground.style.height = `${this.props.height}px`;\n\n    this._repaintLabels();\n\n    // put DOM online again (at the same place)\n    if (foregroundNextSibling) {\n      parent.insertBefore(foreground, foregroundNextSibling);\n    }\n    else {\n      parent.appendChild(foreground)\n    }\n    if (backgroundNextSibling) {\n      this.body.dom.backgroundVertical.insertBefore(background, backgroundNextSibling);\n    }\n    else {\n      this.body.dom.backgroundVertical.appendChild(background)\n    }\n    return this._isResized() || parentChanged;\n  }\n\n  /**\n   * Repaint major and minor text labels and vertical grid lines\n   * @private\n   */\n  _repaintLabels() {\n    const orientation = this.options.orientation.axis;\n\n    // calculate range and step (step such that we have space for 7 characters per label)\n    const start = util.convert(this.body.range.start, 'Number');\n    const end = util.convert(this.body.range.end, 'Number');\n    const timeLabelsize = this.body.util.toTime((this.props.minorCharWidth || 10) * this.options.maxMinorChars).valueOf();\n    let minimumStep = timeLabelsize - DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this.body.range, timeLabelsize);\n    minimumStep -= this.body.util.toTime(0).valueOf();\n\n    const step = new TimeStep(new Date(start), new Date(end), minimumStep, this.body.hiddenDates, this.options);\n    step.setMoment(this.options.moment);\n    if (this.options.format) {\n      step.setFormat(this.options.format);\n    }\n    if (this.options.timeAxis) {\n      step.setScale(this.options.timeAxis);\n    }\n    this.step = step;\n\n    // Move all DOM elements to a \"redundant\" list, where they\n    // can be picked for re-use, and clear the lists with lines and texts.\n    // At the end of the function _repaintLabels, left over elements will be cleaned up\n    const dom = this.dom;\n    dom.redundant.lines = dom.lines;\n    dom.redundant.majorTexts = dom.majorTexts;\n    dom.redundant.minorTexts = dom.minorTexts;\n    dom.lines = [];\n    dom.majorTexts = [];\n    dom.minorTexts = [];\n\n    let current;\n    let next;\n    let x;\n    let xNext;\n    let isMajor;\n    let showMinorGrid;\n    let width = 0;\n    let prevWidth;\n    let line;\n    let xFirstMajorLabel = undefined;\n    let count = 0;\n    const MAX = 1000;\n    let className;\n\n    step.start();\n    next = step.getCurrent();\n    xNext = this.body.util.toScreen(next);\n    while (step.hasNext() && count < MAX) {\n      count++;\n\n      isMajor = step.isMajor();\n      className = step.getClassName();\n\n      current = next;\n      x = xNext;\n\n      step.next();\n      next = step.getCurrent();\n      xNext = this.body.util.toScreen(next);\n\n      prevWidth = width;\n      width = xNext - x;\n      switch (step.scale) {\n        case 'week':         showMinorGrid = true; break;\n        default:             showMinorGrid = (width >= prevWidth * 0.4); break; // prevent displaying of the 31th of the month on a scale of 5 days\n      }\n\n      if (this.options.showMinorLabels && showMinorGrid) {\n        var label = this._repaintMinorText(x, step.getLabelMinor(current), orientation, className);\n        label.style.width = `${width}px`; // set width to prevent overflow\n      }\n\n      if (isMajor && this.options.showMajorLabels) {\n        if (x > 0) {\n          if (xFirstMajorLabel == undefined) {\n            xFirstMajorLabel = x;\n          }\n          label = this._repaintMajorText(x, step.getLabelMajor(current), orientation, className);\n        }\n        line = this._repaintMajorLine(x, width, orientation, className);\n      }\n      else { // minor line\n        if (showMinorGrid) {\n          line = this._repaintMinorLine(x, width, orientation, className);\n        }\n        else {\n          if (line) {\n            // adjust the width of the previous grid\n            line.style.width = `${parseInt(line.style.width) + width}px`;\n          }\n        }\n      }\n    }\n\n    if (count === MAX && !warnedForOverflow) {\n        console.warn(`Something is wrong with the Timeline scale. Limited drawing of grid lines to ${MAX} lines.`);\n        warnedForOverflow = true;\n    }\n\n    // create a major label on the left when needed\n    if (this.options.showMajorLabels) {\n      const leftTime = this.body.util.toTime(0); // upper bound estimation\n      const leftText = step.getLabelMajor(leftTime);\n      const widthText = leftText.length * (this.props.majorCharWidth || 10) + 10;\n\n      if (xFirstMajorLabel == undefined || widthText < xFirstMajorLabel) {\n        this._repaintMajorText(0, leftText, orientation, className);\n      }\n    }\n\n    // Cleanup leftover DOM elements from the redundant list\n    util.forEach(this.dom.redundant, arr => {\n      while (arr.length) {\n        const elem = arr.pop();\n        if (elem && elem.parentNode) {\n          elem.parentNode.removeChild(elem);\n        }\n      }\n    });\n  }\n\n  /**\n   * Create a minor label for the axis at position x\n   * @param {number} x\n   * @param {string} text\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the HTML element of the created label\n   * @private\n   */\n  _repaintMinorText(x, text, orientation, className) {\n    // reuse redundant label\n    let label = this.dom.redundant.minorTexts.shift();\n\n    if (!label) {\n      // create new label\n      const content = document.createTextNode('');\n      label = document.createElement('div');\n      label.appendChild(content);\n      this.dom.foreground.appendChild(label);\n    }\n    this.dom.minorTexts.push(label);\n    label.innerHTML = text;\n\n\n    let y = (orientation == 'top') ? this.props.majorLabelHeight : 0;\n    this._setXY(label, x, y);\n\n    label.className = `vis-text vis-minor ${className}`;\n    //label.title = title;  // TODO: this is a heavy operation\n\n    return label;\n  }\n\n  /**\n   * Create a Major label for the axis at position x\n   * @param {number} x\n   * @param {string} text\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the HTML element of the created label\n   * @private\n   */\n  _repaintMajorText(x, text, orientation, className) {\n    // reuse redundant label\n    let label = this.dom.redundant.majorTexts.shift();\n\n    if (!label) {\n      // create label\n      const content = document.createElement('div');\n      label = document.createElement('div');\n      label.appendChild(content);\n      this.dom.foreground.appendChild(label);\n    }\n\n    label.childNodes[0].innerHTML = text;\n    label.className = `vis-text vis-major ${className}`;\n    //label.title = title; // TODO: this is a heavy operation\n\n    let y = (orientation == 'top') ? 0 : this.props.minorLabelHeight;\n    this._setXY(label, x, y);\n\n    this.dom.majorTexts.push(label);\n    return label;\n  }\n\n  /**\n   * sets xy\n   * @param {string} label\n   * @param {number} x\n   * @param {number} y\n   * @private\n   */\n  _setXY(label, x, y) {\n    // If rtl is true, inverse x.\n    const directionX = this.options.rtl ? (x * -1) : x;\n    label.style.transform = `translate(${directionX}px, ${y}px)`;\n  }\n\n  /**\n   * Create a minor line for the axis at position x\n   * @param {number} left\n   * @param {number} width\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the created line\n   * @private\n   */\n  _repaintMinorLine(left, width, orientation, className) {\n    // reuse redundant line\n    let line = this.dom.redundant.lines.shift();\n    if (!line) {\n      // create vertical line\n      line = document.createElement('div');\n      this.dom.background.appendChild(line);\n    }\n    this.dom.lines.push(line);\n\n    const props = this.props;\n    \n    line.style.width = `${width}px`;\n    line.style.height = `${props.minorLineHeight}px`;\n\n    let y = (orientation == 'top') ? props.majorLabelHeight : this.body.domProps.top.height;\n    let x = left - props.minorLineWidth / 2;\n\n    this._setXY(line, x, y);\n    line.className = `vis-grid ${this.options.rtl ?  'vis-vertical-rtl' : 'vis-vertical'} vis-minor ${className}`;\n\n    return line;\n  }\n\n  /**\n   * Create a Major line for the axis at position x\n   * @param {number} left\n   * @param {number} width\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the created line\n   * @private\n   */\n  _repaintMajorLine(left, width, orientation, className) {\n    // reuse redundant line\n    let line = this.dom.redundant.lines.shift();\n    if (!line) {\n      // create vertical line\n      line = document.createElement('div');\n      this.dom.background.appendChild(line);\n    }\n    this.dom.lines.push(line);\n\n    const props = this.props;\n    \n    line.style.width = `${width}px`;\n    line.style.height = `${props.majorLineHeight}px`;\n\n    let y = (orientation == 'top') ? 0 : this.body.domProps.top.height;\n    let x = left - props.majorLineWidth / 2;\n\n    this._setXY(line, x, y);\n    line.className = `vis-grid ${this.options.rtl ?  'vis-vertical-rtl' : 'vis-vertical'} vis-major ${className}`;\n\n    return line;\n  }\n\n  /**\n   * Determine the size of text on the axis (both major and minor axis).\n   * The size is calculated only once and then cached in this.props.\n   * @private\n   */\n  _calculateCharSize() {\n    // Note: We calculate char size with every redraw. Size may change, for\n    // example when any of the timelines parents had display:none for example.\n\n    // determine the char width and height on the minor axis\n    if (!this.dom.measureCharMinor) {\n      this.dom.measureCharMinor = document.createElement('DIV');\n      this.dom.measureCharMinor.className = 'vis-text vis-minor vis-measure';\n      this.dom.measureCharMinor.style.position = 'absolute';\n\n      this.dom.measureCharMinor.appendChild(document.createTextNode('0'));\n      this.dom.foreground.appendChild(this.dom.measureCharMinor);\n    }\n    this.props.minorCharHeight = this.dom.measureCharMinor.clientHeight;\n    this.props.minorCharWidth = this.dom.measureCharMinor.clientWidth;\n\n    // determine the char width and height on the major axis\n    if (!this.dom.measureCharMajor) {\n      this.dom.measureCharMajor = document.createElement('DIV');\n      this.dom.measureCharMajor.className = 'vis-text vis-major vis-measure';\n      this.dom.measureCharMajor.style.position = 'absolute';\n\n      this.dom.measureCharMajor.appendChild(document.createTextNode('0'));\n      this.dom.foreground.appendChild(this.dom.measureCharMajor);\n    }\n    this.props.majorCharHeight = this.dom.measureCharMajor.clientHeight;\n    this.props.majorCharWidth = this.dom.measureCharMajor.clientWidth;\n  }\n}\n\n\nvar warnedForOverflow = false;\n\nexport default TimeAxis;\n","\"use strict\";\r\n/**\r\n * Created by Alex on 11/6/2014.\r\n */\r\n\r\n// https://github.com/umdjs/umd/blob/master/returnExports.js#L40-L60\r\n// if the module has no dependencies, the above pattern can be simplified to\r\n(function (root, factory) {\r\n  if (typeof define === 'function' && define.amd) {\r\n    // AMD. Register as an anonymous module.\r\n    define([], factory);\r\n  } else if (typeof exports === 'object') {\r\n    // Node. Does not work with strict CommonJS, but\r\n    // only CommonJS-like environments that support module.exports,\r\n    // like Node.\r\n    module.exports = factory();\r\n  } else {\r\n    // Browser globals (root is window)\r\n    root.keycharm = factory();\r\n  }\r\n}(this, function () {\r\n\r\n  function keycharm(options) {\r\n    var preventDefault = options && options.preventDefault || false;\r\n\r\n    var container = options && options.container || window;\r\n\r\n    var _exportFunctions = {};\r\n    var _bound = {keydown:{}, keyup:{}};\r\n    var _keys = {};\r\n    var i;\r\n\r\n    // a - z\r\n    for (i = 97; i <= 122; i++) {_keys[String.fromCharCode(i)] = {code:65 + (i - 97), shift: false};}\r\n    // A - Z\r\n    for (i = 65; i <= 90; i++) {_keys[String.fromCharCode(i)] = {code:i, shift: true};}\r\n    // 0 - 9\r\n    for (i = 0;  i <= 9;   i++) {_keys['' + i] = {code:48 + i, shift: false};}\r\n    // F1 - F12\r\n    for (i = 1;  i <= 12;   i++) {_keys['F' + i] = {code:111 + i, shift: false};}\r\n    // num0 - num9\r\n    for (i = 0;  i <= 9;   i++) {_keys['num' + i] = {code:96 + i, shift: false};}\r\n\r\n    // numpad misc\r\n    _keys['num*'] = {code:106, shift: false};\r\n    _keys['num+'] = {code:107, shift: false};\r\n    _keys['num-'] = {code:109, shift: false};\r\n    _keys['num/'] = {code:111, shift: false};\r\n    _keys['num.'] = {code:110, shift: false};\r\n    // arrows\r\n    _keys['left']  = {code:37, shift: false};\r\n    _keys['up']    = {code:38, shift: false};\r\n    _keys['right'] = {code:39, shift: false};\r\n    _keys['down']  = {code:40, shift: false};\r\n    // extra keys\r\n    _keys['space'] = {code:32, shift: false};\r\n    _keys['enter'] = {code:13, shift: false};\r\n    _keys['shift'] = {code:16, shift: undefined};\r\n    _keys['esc']   = {code:27, shift: false};\r\n    _keys['backspace'] = {code:8, shift: false};\r\n    _keys['tab']       = {code:9, shift: false};\r\n    _keys['ctrl']      = {code:17, shift: false};\r\n    _keys['alt']       = {code:18, shift: false};\r\n    _keys['delete']    = {code:46, shift: false};\r\n    _keys['pageup']    = {code:33, shift: false};\r\n    _keys['pagedown']  = {code:34, shift: false};\r\n    // symbols\r\n    _keys['=']     = {code:187, shift: false};\r\n    _keys['-']     = {code:189, shift: false};\r\n    _keys[']']     = {code:221, shift: false};\r\n    _keys['[']     = {code:219, shift: false};\r\n\r\n\r\n\r\n    var down = function(event) {handleEvent(event,'keydown');};\r\n    var up = function(event) {handleEvent(event,'keyup');};\r\n\r\n    // handle the actualy bound key with the event\r\n    var handleEvent = function(event,type) {\r\n      if (_bound[type][event.keyCode] !== undefined) {\r\n        var bound = _bound[type][event.keyCode];\r\n        for (var i = 0; i < bound.length; i++) {\r\n          if (bound[i].shift === undefined) {\r\n            bound[i].fn(event);\r\n          }\r\n          else if (bound[i].shift == true && event.shiftKey == true) {\r\n            bound[i].fn(event);\r\n          }\r\n          else if (bound[i].shift == false && event.shiftKey == false) {\r\n            bound[i].fn(event);\r\n          }\r\n        }\r\n\r\n        if (preventDefault == true) {\r\n          event.preventDefault();\r\n        }\r\n      }\r\n    };\r\n\r\n    // bind a key to a callback\r\n    _exportFunctions.bind = function(key, callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      if (_keys[key] === undefined) {\r\n        throw new Error(\"unsupported key: \" + key);\r\n      }\r\n      if (_bound[type][_keys[key].code] === undefined) {\r\n        _bound[type][_keys[key].code] = [];\r\n      }\r\n      _bound[type][_keys[key].code].push({fn:callback, shift:_keys[key].shift});\r\n    };\r\n\r\n\r\n    // bind all keys to a call back (demo purposes)\r\n    _exportFunctions.bindAll = function(callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      for (var key in _keys) {\r\n        if (_keys.hasOwnProperty(key)) {\r\n          _exportFunctions.bind(key,callback,type);\r\n        }\r\n      }\r\n    };\r\n\r\n    // get the key label from an event\r\n    _exportFunctions.getKey = function(event) {\r\n      for (var key in _keys) {\r\n        if (_keys.hasOwnProperty(key)) {\r\n          if (event.shiftKey == true && _keys[key].shift == true && event.keyCode == _keys[key].code) {\r\n            return key;\r\n          }\r\n          else if (event.shiftKey == false && _keys[key].shift == false && event.keyCode == _keys[key].code) {\r\n            return key;\r\n          }\r\n          else if (event.keyCode == _keys[key].code && key == 'shift') {\r\n            return key;\r\n          }\r\n        }\r\n      }\r\n      return \"unknown key, currently not supported\";\r\n    };\r\n\r\n    // unbind either a specific callback from a key or all of them (by leaving callback undefined)\r\n    _exportFunctions.unbind = function(key, callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      if (_keys[key] === undefined) {\r\n        throw new Error(\"unsupported key: \" + key);\r\n      }\r\n      if (callback !== undefined) {\r\n        var newBindings = [];\r\n        var bound = _bound[type][_keys[key].code];\r\n        if (bound !== undefined) {\r\n          for (var i = 0; i < bound.length; i++) {\r\n            if (!(bound[i].fn == callback && bound[i].shift == _keys[key].shift)) {\r\n              newBindings.push(_bound[type][_keys[key].code][i]);\r\n            }\r\n          }\r\n        }\r\n        _bound[type][_keys[key].code] = newBindings;\r\n      }\r\n      else {\r\n        _bound[type][_keys[key].code] = [];\r\n      }\r\n    };\r\n\r\n    // reset all bound variables.\r\n    _exportFunctions.reset = function() {\r\n      _bound = {keydown:{}, keyup:{}};\r\n    };\r\n\r\n    // unbind all listeners and reset all variables.\r\n    _exportFunctions.destroy = function() {\r\n      _bound = {keydown:{}, keyup:{}};\r\n      container.removeEventListener('keydown', down, true);\r\n      container.removeEventListener('keyup', up, true);\r\n    };\r\n\r\n    // create listeners.\r\n    container.addEventListener('keydown',down,true);\r\n    container.addEventListener('keyup',up,true);\r\n\r\n    // return the public functions.\r\n    return _exportFunctions;\r\n  }\r\n\r\n  return keycharm;\r\n}));\r\n\r\n\r\n","import keycharm from 'keycharm';\nimport Emitter from 'component-emitter';\nimport Hammer from '../module/hammer';\nimport util from '../util';\n\nimport './activator.css';\n\n/**\n * Turn an element into an clickToUse element.\n * When not active, the element has a transparent overlay. When the overlay is\n * clicked, the mode is changed to active.\n * When active, the element is displayed with a blue border around it, and\n * the interactive contents of the element can be used. When clicked outside\n * the element, the elements mode is changed to inactive.\n * @param {Element} container\n * @constructor Activator\n */\nfunction Activator(container) {\n  this.active = false;\n\n  this.dom = {\n    container: container\n  };\n\n  this.dom.overlay = document.createElement('div');\n  this.dom.overlay.className = 'vis-overlay';\n\n  this.dom.container.appendChild(this.dom.overlay);\n\n  this.hammer = Hammer(this.dom.overlay);\n  this.hammer.on('tap', this._onTapOverlay.bind(this));\n\n  // block all touch events (except tap)\n  var me = this;\n  var events = [\n    'tap', 'doubletap', 'press',\n    'pinch',\n    'pan', 'panstart', 'panmove', 'panend'\n  ];\n  events.forEach(function (event) {\n    me.hammer.on(event, function (event) {\n      event.stopPropagation();\n    });\n  });\n\n  // attach a click event to the window, in order to deactivate when clicking outside the timeline\n  if (document && document.body) {\n    this.onClick = function (event) {\n      if (!_hasParent(event.target, container)) {\n        me.deactivate();\n      }\n    };\n    document.body.addEventListener('click', this.onClick);\n  }\n\n  if (this.keycharm !== undefined) {\n    this.keycharm.destroy();\n  }\n  this.keycharm = keycharm();\n\n  // keycharm listener only bounded when active)\n  this.escListener = this.deactivate.bind(this);\n}\n\n// turn into an event emitter\nEmitter(Activator.prototype);\n\n// The currently active activator\nActivator.current = null;\n\n/**\n * Destroy the activator. Cleans up all created DOM and event listeners\n */\nActivator.prototype.destroy = function () {\n  this.deactivate();\n\n  // remove dom\n  this.dom.overlay.parentNode.removeChild(this.dom.overlay);\n\n  // remove global event listener\n  if (this.onClick) {\n    document.body.removeEventListener('click', this.onClick);\n  }\n  // remove keycharm\n  if (this.keycharm !== undefined) {\n    this.keycharm.destroy();\n  }\n  this.keycharm = null;\n  // cleanup hammer instances\n  this.hammer.destroy();\n  this.hammer = null;\n  // FIXME: cleaning up hammer instances doesn't work (Timeline not removed from memory)\n};\n\n/**\n * Activate the element\n * Overlay is hidden, element is decorated with a blue shadow border\n */\nActivator.prototype.activate = function () {\n  // we allow only one active activator at a time\n  if (Activator.current) {\n    Activator.current.deactivate();\n  }\n  Activator.current = this;\n\n  this.active = true;\n  this.dom.overlay.style.display = 'none';\n  util.addClassName(this.dom.container, 'vis-active');\n\n  this.emit('change');\n  this.emit('activate');\n\n  // ugly hack: bind ESC after emitting the events, as the Network rebinds all\n  // keyboard events on a 'change' event\n  this.keycharm.bind('esc', this.escListener);\n};\n\n/**\n * Deactivate the element\n * Overlay is displayed on top of the element\n */\nActivator.prototype.deactivate = function () {\n  if (Activator.current === this) {\n    Activator.current = null;\n  }\n\n  this.active = false;\n  this.dom.overlay.style.display = '';\n  util.removeClassName(this.dom.container, 'vis-active');\n  this.keycharm.unbind('esc', this.escListener);\n\n  this.emit('change');\n  this.emit('deactivate');\n};\n\n/**\n * Handle a tap event: activate the container\n * @param {Event}  event   The event\n * @private\n */\nActivator.prototype._onTapOverlay = function (event) {\n  // activate the container\n  this.activate();\n  event.stopPropagation();\n};\n\n/**\n * Test whether the element has the requested parent element somewhere in\n * its chain of parent nodes.\n * @param {HTMLElement} element\n * @param {HTMLElement} parent\n * @returns {boolean} Returns true when the parent is found somewhere in the\n *                    chain of parent nodes.\n * @private\n */\nfunction _hasParent(element, parent) {\n  while (element) {\n    if (element === parent) {\n      return true\n    }\n    element = element.parentNode;\n  }\n  return false;\n}\n\nexport default Activator;\n","/*\n * IMPORTANT: Locales for Moment has to be imported in the legacy and standalone\n * entry points. For the peer build it's users responsibility to do so.\n */\n\n// English\nconst en = {\n  current: 'current',\n  time: 'time',\n  deleteSelected: 'Delete selected',\n};\nconst en_EN = en;\nconst en_US = en;\n\n// Italiano\nconst it = {\n  current: 'attuale',\n  time: 'tempo',\n  deleteSelected: 'Cancella la selezione',\n};\nconst it_IT = it;\nconst it_CH = it;\n\n// Dutch\nconst nl = {\n  current: 'huidige',\n  time: 'tijd',\n  deleteSelected: 'Selectie verwijderen'\n};\nconst nl_NL = nl;\nconst nl_BE = nl;\n\n// German\nconst de = {\n  current: 'Aktuelle',\n  time: 'Zeit',\n  deleteSelected: 'L\\u00f6sche Auswahl',\n};\nconst de_DE = de;\n\n// French\nconst fr = {\n  current: 'actuel',\n  time: 'heure',\n  deleteSelected: 'Effacer la selection',\n};\nconst fr_FR = fr;\nconst fr_CA = fr;\nconst fr_BE = fr;\n\n// Espanol\nconst es = {\n  current: 'corriente',\n  time: 'hora',\n  deleteSelected: 'Eliminar selecci\\u00f3n',\n};\nconst es_ES = es;\n\n// Ukrainian\nconst uk = {\n  current: 'поточний',\n  time: 'час',\n  deleteSelected: 'Видалити обране',\n};\nconst uk_UA = uk;\n\n// Russian\nconst ru = {\n  current: 'текущее',\n  time: 'время',\n  deleteSelected: 'Удалить выбранное',\n};\nconst ru_RU = ru;\n\n// Polish\nconst pl = {\n  current: 'aktualny',\n  time: 'czas',\n  deleteSelected: 'Usuń wybrane',\n};\nconst pl_PL = pl;\n\n// Japanese\nconst ja = {\n  current: '現在',\n  time: '時刻',\n  deleteSelected: '選択されたものを削除',\n};\nconst ja_JP = ja;\n\nconst locales = {\n  en,\n  en_EN,\n  en_US,\n  it,\n  it_IT,\n  it_CH,\n  nl,\n  nl_NL,\n  nl_BE,\n  de,\n  de_DE,\n  fr,\n  fr_FR,\n  fr_CA,\n  fr_BE,\n  es,\n  es_ES,\n  uk,\n  uk_UA,\n  ru,\n  ru_RU,\n  pl,\n  pl_PL,\n  ja,\n  ja_JP,\n}\n\nexport default locales\n","import Hammer from '../../module/hammer';\nimport util from '../../util';\nimport Component from './Component';\nimport moment from '../../module/moment';\nimport locales from '../locales';\n\nimport './css/customtime.css';\n\n/** A custom time bar */\nclass CustomTime extends Component {\n /**\n * @param {{range: Range, dom: Object}} body\n * @param {Object} [options]        Available parameters:\n *                                  {number | string} id\n *                                  {string} locales\n *                                  {string} locale\n * @constructor CustomTime\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n\n    // default options\n    this.defaultOptions = {\n      moment,\n      locales,\n      locale: 'en',\n      id: undefined,\n      title: undefined\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.setOptions(options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n\n    if (options && options.time != null) {\n      this.customTime = options.time;\n    } else {\n      this.customTime = new Date();\n    }\n\n    this.eventParams = {}; // stores state parameters while dragging the bar\n\n    // create the DOM\n    this._create();\n  }\n\n  /**\n   * Set options for the component. Options will be merged in current options.\n   * @param {Object} options  Available parameters:\n   *                                  {number | string} id\n   *                                  {string} locales\n   *                                  {string} locale\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend(['moment', 'locale', 'locales', 'id', 'title', 'rtl', 'snap'], this.options, options);\n    }\n  }\n\n  /**\n   * Create the DOM for the custom time\n   * @private\n   */\n  _create() {\n    const bar = document.createElement('div');\n    bar['custom-time'] = this;\n    bar.className = `vis-custom-time ${this.options.id || ''}`;\n    bar.style.position = 'absolute';\n    bar.style.top = '0px';\n    bar.style.height = '100%';\n    this.bar = bar;\n\n    const drag = document.createElement('div');\n    drag.style.position = 'relative';\n    drag.style.top = '0px';\n    if(this.options.rtl) {\n      drag.style.right = '-10px';\n    } else  {\n       drag.style.left = '-10px';\n    }\n    drag.style.height = '100%';\n    drag.style.width = '20px';\n\n    /**\n     *\n     * @param {WheelEvent} e\n     */\n    function onMouseWheel (e) {\n      this.body.range._onMouseWheel(e);\n    }\n\n    if (drag.addEventListener) {\n      // IE9, Chrome, Safari, Opera\n      drag.addEventListener(\"mousewheel\", onMouseWheel.bind(this), false);\n      // Firefox\n      drag.addEventListener(\"DOMMouseScroll\", onMouseWheel.bind(this), false);\n    } else {\n      // IE 6/7/8\n      drag.attachEvent(\"onmousewheel\", onMouseWheel.bind(this));\n    }\n\n    bar.appendChild(drag);\n    // attach event listeners\n    this.hammer = new Hammer(drag);\n    this.hammer.on('panstart', this._onDragStart.bind(this));\n    this.hammer.on('panmove',  this._onDrag.bind(this));\n    this.hammer.on('panend',   this._onDragEnd.bind(this));\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_ALL});\n  }\n\n  /**\n   * Destroy the CustomTime bar\n   */\n  destroy() {\n    this.hide();\n\n    this.hammer.destroy();\n    this.hammer = null;\n\n    this.body = null;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const parent = this.body.dom.backgroundVertical;\n    if (this.bar.parentNode != parent) {\n      // attach to the dom\n      if (this.bar.parentNode) {\n        this.bar.parentNode.removeChild(this.bar);\n      }\n      parent.appendChild(this.bar);\n    }\n\n    const x = this.body.util.toScreen(this.customTime);\n\n    let locale = this.options.locales[this.options.locale];\n    if (!locale) {\n      if (!this.warned) {\n        console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n        this.warned = true;\n      }\n      locale = this.options.locales['en']; // fall back on english when not available\n    }\n\n    let title = this.options.title;\n    // To hide the title completely use empty string ''.\n    if (title === undefined) {\n      title = `${locale.time}: ${this.options.moment(this.customTime).format('dddd, MMMM Do YYYY, H:mm:ss')}`;\n      title = title.charAt(0).toUpperCase() + title.substring(1);\n    } else if (typeof title === \"function\") {\n      title = title.call(this, this.customTime);\n    }\n\n    this.options.rtl ? this.bar.style.right = `${x}px` : this.bar.style.left = `${x}px`;\n    this.bar.title = title;\n\n    return false;\n  }\n\n  /**\n   * Remove the CustomTime from the DOM\n   */\n  hide() {\n    // remove the line from the DOM\n    if (this.bar.parentNode) {\n      this.bar.parentNode.removeChild(this.bar);\n    }\n  }\n\n  /**\n   * Set custom time.\n   * @param {Date | number | string} time\n   */\n  setCustomTime(time) {\n    this.customTime = util.convert(time, 'Date');\n    this.redraw();\n  }\n\n  /**\n   * Retrieve the current custom time.\n   * @return {Date} customTime\n   */\n  getCustomTime() {\n    return new Date(this.customTime.valueOf());\n  }\n\n  /**\n   * Set custom marker.\n   * @param {string} [title] Title of the custom marker\n   * @param {boolean} [editable] Make the custom marker editable.\n   */\n  setCustomMarker(title, editable) {\n    const marker = document.createElement('div');\n    marker.className = `vis-custom-time-marker`;\n    marker.innerHTML = title;\n    marker.style.position = 'absolute';\n\n    if (editable) {\n      marker.setAttribute('contenteditable', 'true');\n      marker.addEventListener('pointerdown', function () {\n        marker.focus();\n      });\n      marker.addEventListener('input', this._onMarkerChange.bind(this));\n      // The editable div element has no change event, so here emulates the change event.\n      marker.title = title;\n      marker.addEventListener('blur', function (event) {\n        if (this.title != event.target.innerHTML) {\n          this._onMarkerChanged(event);\n          this.title = event.target.innerHTML;\n        }\n      }.bind(this));\n    }\n\n    this.bar.appendChild(marker);\n  }\n\n  /**\n    * Set custom title.\n    * @param {Date | number | string} title\n    */\n  setCustomTitle(title) {\n    this.options.title = title;\n  }\n\n  /**\n   * Start moving horizontally\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    this.eventParams.dragging = true;\n    this.eventParams.customTime = this.customTime;\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform moving operating.\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!this.eventParams.dragging) return;\n\n    let deltaX = this.options.rtl ? (-1) * event.deltaX : event.deltaX;\n\n    const x = this.body.util.toScreen(this.eventParams.customTime) + deltaX;\n    const time = this.body.util.toTime(x);\n\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const snap = this.options.snap;\n\n    const snappedTime = snap ? snap(time, scale, step) : time;\n\n    this.setCustomTime(snappedTime);\n\n    // fire a timechange event\n    this.body.emitter.emit('timechange', {\n      id: this.options.id,\n      time: new Date(this.customTime.valueOf()),\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Stop moving operating.\n   * @param {Event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    if (!this.eventParams.dragging) return;\n\n    // fire a timechanged event\n    this.body.emitter.emit('timechanged', {\n      id: this.options.id,\n      time: new Date(this.customTime.valueOf()),\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform input operating.\n   * @param {Event} event\n   * @private\n   */\n  _onMarkerChange(event) {\n    this.body.emitter.emit('markerchange', {\n      id: this.options.id,\n      title: event.target.innerHTML,\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform change operating.\n   * @param {Event} event\n   * @private\n   */\n  _onMarkerChanged(event) {\n    this.body.emitter.emit('markerchanged', {\n      id: this.options.id,\n      title: event.target.innerHTML,\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Find a custom time from an event target:\n   * searches for the attribute 'custom-time' in the event target's element tree\n   * @param {Event} event\n   * @return {CustomTime | null} customTime\n   */\n  static customTimeFromTarget(event) {\n    let target = event.target;\n    while (target) {\n      if (target.hasOwnProperty('custom-time')) {\n        return target['custom-time'];\n      }\n      target = target.parentNode;\n    }\n\n    return null;\n  }\n}\n\nexport default CustomTime;\n","import Emitter from 'component-emitter';\nimport Hammer from '../module/hammer';\nimport * as hammerUtil from '../hammerUtil';\nimport util from '../util';\nimport TimeAxis from './component/TimeAxis';\nimport Activator from '../shared/Activator';\nimport * as DateUtil  from './DateUtil';\nimport CustomTime from './component/CustomTime';\n\nimport './component/css/animation.css';\nimport './component/css/currenttime.css';\nimport './component/css/panel.css';\nimport './component/css/pathStyles.css';\nimport './component/css/timeline.css';\nimport '../shared/bootstrap.css';\n\n/**\n * Create a timeline visualization\n * @constructor Core\n */\nclass Core {\n  /**\n   * Create the main DOM for the Core: a root panel containing left, right,\n   * top, bottom, content, and background panel.\n   * @param {Element} container  The container element where the Core will\n   *                             be attached.\n   * @protected\n   */\n  _create(container) {\n    this.dom = {};\n\n    this.dom.container = container;\n    this.dom.container.style.position = 'relative';\n\n    this.dom.root                 = document.createElement('div');\n    this.dom.background           = document.createElement('div');\n    this.dom.backgroundVertical   = document.createElement('div');\n    this.dom.backgroundHorizontal = document.createElement('div');\n    this.dom.centerContainer      = document.createElement('div');\n    this.dom.leftContainer        = document.createElement('div');\n    this.dom.rightContainer       = document.createElement('div');\n    this.dom.center               = document.createElement('div');\n    this.dom.left                 = document.createElement('div');\n    this.dom.right                = document.createElement('div');\n    this.dom.top                  = document.createElement('div');\n    this.dom.bottom               = document.createElement('div');\n    this.dom.shadowTop            = document.createElement('div');\n    this.dom.shadowBottom         = document.createElement('div');\n    this.dom.shadowTopLeft        = document.createElement('div');\n    this.dom.shadowBottomLeft     = document.createElement('div');\n    this.dom.shadowTopRight       = document.createElement('div');\n    this.dom.shadowBottomRight    = document.createElement('div');\n    this.dom.rollingModeBtn       = document.createElement('div');\n    this.dom.loadingScreen        = document.createElement('div');\n\n    this.dom.root.className                 = 'vis-timeline';\n    this.dom.background.className           = 'vis-panel vis-background';\n    this.dom.backgroundVertical.className   = 'vis-panel vis-background vis-vertical';\n    this.dom.backgroundHorizontal.className = 'vis-panel vis-background vis-horizontal';\n    this.dom.centerContainer.className      = 'vis-panel vis-center';\n    this.dom.leftContainer.className        = 'vis-panel vis-left';\n    this.dom.rightContainer.className       = 'vis-panel vis-right';\n    this.dom.top.className                  = 'vis-panel vis-top';\n    this.dom.bottom.className               = 'vis-panel vis-bottom';\n    this.dom.left.className                 = 'vis-content';\n    this.dom.center.className               = 'vis-content';\n    this.dom.right.className                = 'vis-content';\n    this.dom.shadowTop.className            = 'vis-shadow vis-top';\n    this.dom.shadowBottom.className         = 'vis-shadow vis-bottom';\n    this.dom.shadowTopLeft.className        = 'vis-shadow vis-top';\n    this.dom.shadowBottomLeft.className     = 'vis-shadow vis-bottom';\n    this.dom.shadowTopRight.className       = 'vis-shadow vis-top';\n    this.dom.shadowBottomRight.className    = 'vis-shadow vis-bottom';\n    this.dom.rollingModeBtn.className       = 'vis-rolling-mode-btn';\n    this.dom.loadingScreen.className        = 'vis-loading-screen';\n\n    this.dom.root.appendChild(this.dom.background);\n    this.dom.root.appendChild(this.dom.backgroundVertical);\n    this.dom.root.appendChild(this.dom.backgroundHorizontal);\n    this.dom.root.appendChild(this.dom.centerContainer);\n    this.dom.root.appendChild(this.dom.leftContainer);\n    this.dom.root.appendChild(this.dom.rightContainer);\n    this.dom.root.appendChild(this.dom.top);\n    this.dom.root.appendChild(this.dom.bottom);\n    this.dom.root.appendChild(this.dom.rollingModeBtn);\n\n    this.dom.centerContainer.appendChild(this.dom.center);\n    this.dom.leftContainer.appendChild(this.dom.left);\n    this.dom.rightContainer.appendChild(this.dom.right);\n    this.dom.centerContainer.appendChild(this.dom.shadowTop);\n    this.dom.centerContainer.appendChild(this.dom.shadowBottom);\n    this.dom.leftContainer.appendChild(this.dom.shadowTopLeft);\n    this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft);\n    this.dom.rightContainer.appendChild(this.dom.shadowTopRight);\n    this.dom.rightContainer.appendChild(this.dom.shadowBottomRight);\n    \n    // size properties of each of the panels\n    this.props = {\n      root: {},\n      background: {},\n      centerContainer: {},\n      leftContainer: {},\n      rightContainer: {},\n      center: {},\n      left: {},\n      right: {},\n      top: {},\n      bottom: {},\n      border: {},\n      scrollTop: 0,\n      scrollTopMin: 0\n    };\n\n    this.on('rangechange', () => {\n      if (this.initialDrawDone === true) {\n        this._redraw();\n      }\n    });\n    this.on('rangechanged', () => {\n      if (!this.initialRangeChangeDone) {\n        this.initialRangeChangeDone = true;\n      }\n    });\n    this.on('touch',       this._onTouch.bind(this));\n    this.on('panmove',         this._onDrag.bind(this));\n\n    const me = this;\n    this._origRedraw = this._redraw.bind(this);\n    this._redraw = util.throttle(this._origRedraw);\n\n    this.on('_change', properties => {\n      if (me.itemSet && me.itemSet.initialItemSetDrawn && properties && properties.queue == true) {\n        me._redraw()\n      } else {\n        me._origRedraw();\n      }\n    });\n\n    // create event listeners for all interesting events, these events will be\n    // emitted via emitter\n    this.hammer = new Hammer(this.dom.root);\n    const pinchRecognizer = this.hammer.get('pinch').set({enable: true});\n    pinchRecognizer && hammerUtil.disablePreventDefaultVertically(pinchRecognizer);\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_ALL});\n    this.listeners = {};\n\n    const events = [\n      'tap', 'doubletap', 'press',\n      'pinch',\n      'pan', 'panstart', 'panmove', 'panend'\n        // TODO: cleanup\n      //'touch', 'pinch',\n      //'tap', 'doubletap', 'hold',\n      //'dragstart', 'drag', 'dragend',\n      //'mousewheel', 'DOMMouseScroll' // DOMMouseScroll is needed for Firefox\n    ];\n    events.forEach(type => {\n      const listener = event => {\n        if (me.isActive()) {\n          me.emit(type, event);\n        }\n      };\n      me.hammer.on(type, listener);\n      me.listeners[type] = listener;\n    });\n\n    // emulate a touch event (emitted before the start of a pan, pinch, tap, or press)\n    hammerUtil.onTouch(this.hammer, event => {\n      me.emit('touch', event);\n    });\n\n    // emulate a release event (emitted after a pan, pinch, tap, or press)\n    hammerUtil.onRelease(this.hammer, event => {\n      me.emit('release', event);\n    });\n\n    /**\n     *\n     * @param {WheelEvent} event\n     */\n    function onMouseWheel(event) {\n\n      // Reasonable default wheel deltas\n      const LINE_HEIGHT = 40;\n      const PAGE_HEIGHT = 800;\n\n      if (this.isActive()) {\n        this.emit('mousewheel', event);\n      }\n\n      // deltaX and deltaY normalization from jquery.mousewheel.js\n      let deltaX = 0;\n      let deltaY = 0;\n\n      // Old school scrollwheel delta\n      if ( 'detail'      in event ) { deltaY = event.detail * -1;      }\n      if ( 'wheelDelta'  in event ) { deltaY = event.wheelDelta;       }\n      if ( 'wheelDeltaY' in event ) { deltaY = event.wheelDeltaY;      }\n      if ( 'wheelDeltaX' in event ) { deltaX = event.wheelDeltaX * -1; }\n\n      // Firefox < 17 horizontal scrolling related to DOMMouseScroll event\n      if ( 'axis' in event && event.axis === event.HORIZONTAL_AXIS ) {\n        deltaX = deltaY * -1;\n        deltaY = 0;\n      }\n\n      // New school wheel delta (wheel event)\n      if ( 'deltaY' in event ) {\n        deltaY = event.deltaY * -1;\n      }\n      if ( 'deltaX' in event ) {\n        deltaX = event.deltaX;\n      }\n\n      // Normalize deltas\n      if (event.deltaMode) {\n          if (event.deltaMode === 1) {   // delta in LINE units\n            deltaX *= LINE_HEIGHT;\n            deltaY *= LINE_HEIGHT;\n          } else {                       // delta in PAGE units\n            deltaX *= LINE_HEIGHT;\n            deltaY *= PAGE_HEIGHT;\n          }\n        }\n        // Prevent scrolling when zooming (no zoom key, or pressing zoom key)\n      if (this.options.preferZoom) {\n        if (!this.options.zoomKey || event[this.options.zoomKey]) return;\n      } else {\n        if (this.options.zoomKey && event[this.options.zoomKey]) return\n      }\n      // Don't preventDefault if you can't scroll\n      if (!this.options.verticalScroll && !this.options.horizontalScroll) return;\n\n      if (this.options.verticalScroll && Math.abs(deltaY) >= Math.abs(deltaX)) {\n        const current = this.props.scrollTop;\n        const adjusted = current + deltaY;\n\n        if (this.isActive()) {\n          const newScrollTop = this._setScrollTop(adjusted);\n\n          if (newScrollTop !== current) {\n            this._redraw();\n            this.emit('scroll', event);\n\n            // Prevent default actions caused by mouse wheel\n            // (else the page and timeline both scroll)\n            event.preventDefault();\n          }\n        }\n      } else if (this.options.horizontalScroll) {\n        const delta = Math.abs(deltaX) >= Math.abs(deltaY) ? deltaX : deltaY;\n\n        // calculate a single scroll jump relative to the range scale\n        const diff = (delta / 120) * (this.range.end - this.range.start) / 20;\n        // calculate new start and end\n        const newStart = this.range.start + diff;\n        const newEnd = this.range.end + diff;\n\n        const options = {\n          animation: false,\n          byUser: true,\n          event\n        };\n        this.range.setRange(newStart, newEnd, options);\n\n        event.preventDefault();\n      }\n    }\n\n    // Add modern wheel event listener\n    const wheelType = \"onwheel\" in document.createElement(\"div\") ? \"wheel\" : // Modern browsers support \"wheel\"\n    document.onmousewheel !== undefined ? \"mousewheel\" : // Webkit and IE support at least \"mousewheel\"\n    \n    // DOMMouseScroll - Older Firefox versions use \"DOMMouseScroll\"\n    // onmousewheel - All the use \"onmousewheel\"\n    this.dom.centerContainer.addEventListener ? \"DOMMouseScroll\" :  \"onmousewheel\"\n    this.dom.top.addEventListener ? \"DOMMouseScroll\" : \"onmousewheel\"\n    this.dom.bottom.addEventListener ? \"DOMMouseScroll\" : \"onmousewheel\"\n    this.dom.centerContainer.addEventListener(wheelType, onMouseWheel.bind(this), false);\n    this.dom.top.addEventListener(wheelType, onMouseWheel.bind(this), false);\n    this.dom.bottom.addEventListener(wheelType, onMouseWheel.bind(this), false);\n\n\n    /**\n     *\n     * @param {scroll} event\n     */\n    function onMouseScrollSide(event) {\n      if (!me.options.verticalScroll) return;\n      \n      if (me._isProgramaticallyScrolled) {\n        me._isProgramaticallyScrolled = false;\n        return;\n      }\n      \n      event.preventDefault();\n      if (me.isActive()) {\n        const adjusted = -event.target.scrollTop;\n        me._setScrollTop(adjusted);\n        me._redraw();\n        me.emit('scrollSide', event);\n      }\n    }\n\n    this.dom.left.parentNode.addEventListener('scroll', onMouseScrollSide.bind(this));\n    this.dom.right.parentNode.addEventListener('scroll', onMouseScrollSide.bind(this));\n\n    let itemAddedToTimeline = false;\n\n    /**\n     *\n     * @param {dragover} event\n     * @returns {boolean}\n     */\n    function handleDragOver(event) {\n      if (event.preventDefault) {\n        me.emit('dragover', me.getEventProperties(event));\n        event.preventDefault(); // Necessary. Allows us to drop.\n      }\n\n      // make sure your target is a timeline element\n      if (!(event.target.className.indexOf(\"timeline\") > -1)) return;\n\n      // make sure only one item is added every time you're over the timeline\n      if (itemAddedToTimeline) return;\n\n      event.dataTransfer.dropEffect = 'move';  \n      itemAddedToTimeline = true;\n      return false;\n    }\n\n    /**\n     *\n     * @param {drop} event\n     * @returns {boolean}\n     */\n    function handleDrop(event) {\n      // prevent redirect to blank page - Firefox\n      if(event.preventDefault) { event.preventDefault(); }\n      if(event.stopPropagation) { event.stopPropagation(); }\n      // return when dropping non-timeline items\n      try {\n        var itemData = JSON.parse(event.dataTransfer.getData(\"text\"))\n        if (!itemData || !itemData.content) return\n      } catch (err) {\n        return false;\n      }\n\n      itemAddedToTimeline = false;\n      event.center = {\n        x: event.clientX,\n        y: event.clientY\n      };\n\n      if (itemData.target !== 'item') {\n        me.itemSet._onAddItem(event);\n      } else {\n        me.itemSet._onDropObjectOnItem(event);\n      }\n      me.emit('drop', me.getEventProperties(event))\n      return false;\n    }\n\n    this.dom.center.addEventListener('dragover', handleDragOver.bind(this), false);\n    this.dom.center.addEventListener('drop', handleDrop.bind(this), false);\n\n    this.customTimes = [];\n\n    // store state information needed for touch events\n    this.touch = {};\n\n    this.redrawCount = 0;\n    this.initialDrawDone = false;\n    this.initialRangeChangeDone = false;\n\n    // attach the root panel to the provided container\n    if (!container) throw new Error('No container provided');\n    container.appendChild(this.dom.root);\n    container.appendChild(this.dom.loadingScreen);\n  }\n\n  /**\n   * Set options. Options will be passed to all components loaded in the Timeline.\n   * @param {Object} [options]\n   *                           {String} orientation\n   *                              Vertical orientation for the Timeline,\n   *                              can be 'bottom' (default) or 'top'.\n   *                           {string | number} width\n   *                              Width for the timeline, a number in pixels or\n   *                              a css string like '1000px' or '75%'. '100%' by default.\n   *                           {string | number} height\n   *                              Fixed height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'. If undefined,\n   *                              The Timeline will automatically size such that\n   *                              its contents fit.\n   *                           {string | number} minHeight\n   *                              Minimum height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'.\n   *                           {string | number} maxHeight\n   *                              Maximum height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'.\n   *                           {number | Date | string} start\n   *                              Start date for the visible window\n   *                           {number | Date | string} end\n   *                              End date for the visible window\n   */\n  setOptions(options) {\n    if (options) {\n      // copy the known options\n      const fields = [\n        'width', 'height', 'minHeight', 'maxHeight', 'autoResize',\n        'start', 'end', 'clickToUse', 'dataAttributes', 'hiddenDates',\n        'locale', 'locales', 'moment', 'preferZoom', 'rtl', 'zoomKey',\n        'horizontalScroll', 'verticalScroll', 'longSelectPressTime'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n      this.dom.rollingModeBtn.style.visibility = 'hidden';\n\n      if (this.options.rtl) {\n        this.dom.container.style.direction = \"rtl\";\n        this.dom.backgroundVertical.className = 'vis-panel vis-background vis-vertical-rtl';    \n      }\n\n      if (this.options.verticalScroll) {\n        if (this.options.rtl) {\n          this.dom.rightContainer.className = 'vis-panel vis-right vis-vertical-scroll';\n        } else {\n          this.dom.leftContainer.className = 'vis-panel vis-left vis-vertical-scroll';\n        }\n      }\n\n      if (typeof this.options.orientation !== 'object') {\n        this.options.orientation = {item:undefined,axis:undefined};\n      }\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation = {\n            item: options.orientation,\n            axis: options.orientation\n          };\n        }\n        else if (typeof options.orientation === 'object') {\n          if ('item' in options.orientation) {\n            this.options.orientation.item = options.orientation.item;\n          }\n          if ('axis' in options.orientation) {\n            this.options.orientation.axis = options.orientation.axis;\n          }\n        }\n      }\n\n      if (this.options.orientation.axis === 'both') {\n        if (!this.timeAxis2) {\n          const timeAxis2 = this.timeAxis2 = new TimeAxis(this.body);\n          timeAxis2.setOptions = options => {\n            const _options = options ? util.extend({}, options) : {};\n            _options.orientation = 'top'; // override the orientation option, always top\n            TimeAxis.prototype.setOptions.call(timeAxis2, _options);\n          };\n          this.components.push(timeAxis2);\n        }\n      }\n      else {\n        if (this.timeAxis2) {\n          const index = this.components.indexOf(this.timeAxis2);\n          if (index !== -1) {\n            this.components.splice(index, 1);\n          }\n          this.timeAxis2.destroy();\n          this.timeAxis2 = null;\n        }\n      }\n\n      // if the graph2d's drawPoints is a function delegate the callback to the onRender property\n      if (typeof options.drawPoints == 'function') {\n        options.drawPoints = {\n            onRender: options.drawPoints\n        };\n      }\n\n      if ('hiddenDates' in this.options) {\n        DateUtil.convertHiddenOptions(this.options.moment, this.body, this.options.hiddenDates);\n      }\n\n      if ('clickToUse' in options) {\n        if (options.clickToUse) {\n          if (!this.activator) {\n            this.activator = new Activator(this.dom.root);\n          }\n        }\n        else {\n          if (this.activator) {\n            this.activator.destroy();\n            delete this.activator;\n          }\n        }\n      }\n\n      // enable/disable autoResize\n      this._initAutoResize();\n    }\n\n    // propagate options to all components\n    this.components.forEach(component => component.setOptions(options));\n\n    // enable/disable configure\n    if ('configure' in options) {\n      if (!this.configurator) {\n        this.configurator = this._createConfigurator();\n      }\n\n      this.configurator.setOptions(options.configure);\n\n      // collect the settings of all components, and pass them to the configuration system\n      const appliedOptions = util.deepExtend({}, this.options);\n      this.components.forEach(component => {\n        util.deepExtend(appliedOptions, component.options);\n      });\n      this.configurator.setModuleOptions({global: appliedOptions});\n    }\n\n    this._redraw();\n  }\n\n  /**\n   * Returns true when the Timeline is active.\n   * @returns {boolean}\n   */\n  isActive() {\n    return !this.activator || this.activator.active;\n  }\n\n  /**\n   * Destroy the Core, clean up all DOM elements and event listeners.\n   */\n  destroy() {\n    // unbind datasets\n    this.setItems(null);\n    this.setGroups(null);\n\n    // remove all event listeners\n    this.off();\n\n    // stop checking for changed size\n    this._stopAutoResize();\n\n    // remove from DOM\n    if (this.dom.root.parentNode) {\n      this.dom.root.parentNode.removeChild(this.dom.root);\n    }\n    this.dom = null;\n\n    // remove Activator\n    if (this.activator) {\n      this.activator.destroy();\n      delete this.activator;\n    }\n\n    // cleanup hammer touch events\n    for (const event in this.listeners) {\n      if (this.listeners.hasOwnProperty(event)) {\n        delete this.listeners[event];\n      }\n    }\n    this.listeners = null;\n    this.hammer && this.hammer.destroy();\n    this.hammer = null;\n\n    // give all components the opportunity to cleanup\n    this.components.forEach(component => component.destroy());\n\n    this.body = null;\n  }\n\n  /**\n   * Set a custom time bar\n   * @param {Date} time\n   * @param {number} [id=undefined] Optional id of the custom time bar to be adjusted.\n   */\n  setCustomTime(time, id) {\n    const customTimes = this.customTimes.filter(component => id === component.options.id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n\n    if (customTimes.length > 0) {\n      customTimes[0].setCustomTime(time);\n    }\n  }\n\n  /**\n   * Retrieve the current custom time.\n   * @param {number} [id=undefined]    Id of the custom time bar.\n   * @return {Date | undefined} customTime\n   */\n  getCustomTime(id) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    return customTimes[0].getCustomTime();\n  }\n\n  /**\n   * Set a custom marker for the custom time bar.\n   * @param {string} [title] Title of the custom marker.\n   * @param {number} [id=undefined] Id of the custom marker.\n   * @param {boolean} [editable=false] Make the custom marker editable.\n   */\n  setCustomTimeMarker(title, id, editable) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    if (customTimes.length > 0) {\n      customTimes[0].setCustomMarker(title, editable);\n    }\n  }\n\n  /**\n   * Set a custom title for the custom time bar.\n   * @param {string} [title] Custom title\n   * @param {number} [id=undefined]    Id of the custom time bar.\n   * @returns {*}\n   */\n  setCustomTimeTitle(title, id) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    if (customTimes.length > 0) {\n      return customTimes[0].setCustomTitle(title);\n    }\n  }\n\n  /**\n   * Retrieve meta information from an event.\n   * Should be overridden by classes extending Core\n   * @param {Event} event\n   * @return {Object} An object with related information.\n   */\n  getEventProperties(event) {\n    return { event };\n  }\n\n  /**\n   * Add custom vertical bar\n   * @param {Date | string | number} [time]  A Date, unix timestamp, or\n   *                                         ISO date string. Time point where\n   *                                         the new bar should be placed.\n   *                                         If not provided, `new Date()` will\n   *                                         be used.\n   * @param {number | string} [id=undefined] Id of the new bar. Optional\n   * @return {number | string}               Returns the id of the new bar\n   */\n  addCustomTime(time, id) {\n    const timestamp = time !== undefined\n        ? util.convert(time, 'Date')\n        : new Date();\n\n    const exists = this.customTimes.some(customTime => customTime.options.id === id);\n    if (exists) {\n      throw new Error(`A custom time with id ${JSON.stringify(id)} already exists`);\n    }\n\n    const customTime = new CustomTime(this.body, util.extend({}, this.options, {\n      time : timestamp,\n      id,\n      snap: this.itemSet.options.snap\n    }));\n\n    this.customTimes.push(customTime);\n    this.components.push(customTime);\n    this._redraw();\n\n    return id;\n  }\n\n  /**\n   * Remove previously added custom bar\n   * @param {int} id ID of the custom bar to be removed\n   * [at]returns {boolean} True if the bar exists and is removed, false otherwise\n   */\n  removeCustomTime(id) {\n    const customTimes = this.customTimes.filter(bar => bar.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n\n    customTimes.forEach(customTime => {\n      this.customTimes.splice(this.customTimes.indexOf(customTime), 1);\n      this.components.splice(this.components.indexOf(customTime), 1);\n      customTime.destroy();\n    })\n  }\n\n  /**\n   * Get the id's of the currently visible items.\n   * @returns {Array} The ids of the visible items\n   */\n  getVisibleItems() {\n    return this.itemSet && this.itemSet.getVisibleItems() || [];\n  }\n\n  /**\n   * Get the id's of the currently visible groups.\n   * @returns {Array} The ids of the visible groups\n   */\n  getVisibleGroups() {\n    return this.itemSet && this.itemSet.getVisibleGroups() || [];\n  }\n\n  /**\n   * Set Core window such that it fits all items\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  fit(options, callback) {\n    const range = this.getDataRange();\n\n    // skip range set if there is no min and max date\n    if (range.min === null && range.max === null) {\n      return;\n    }\n\n    // apply a margin of 1% left and right of the data\n    const interval = range.max - range.min;\n    const min = new Date(range.min.valueOf() - interval * 0.01);\n    const max = new Date(range.max.valueOf() + interval * 0.01);\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n    this.range.setRange(min, max, { animation }, callback);\n  }\n\n  /**\n   * Calculate the data range of the items start and end dates\n   * [at]returns {{min: [Date], max: [Date]}}\n   * @protected\n   */\n  getDataRange() {\n    // must be implemented by Timeline and Graph2d\n    throw new Error('Cannot invoke abstract method getDataRange');\n  }\n\n  /**\n   * Set the visible window. Both parameters are optional, you can change only\n   * start or only end. Syntax:\n   *\n   *     TimeLine.setWindow(start, end)\n   *     TimeLine.setWindow(start, end, options)\n   *     TimeLine.setWindow(range)\n   *\n   * Where start and end can be a Date, number, or string, and range is an\n   * object with properties start and end.\n   *\n   * @param {Date | number | string | Object} [start] Start date of visible window\n   * @param {Date | number | string} [end]            End date of visible window\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  setWindow(start, end, options, callback) {\n    if (typeof arguments[2] == \"function\") {\n      callback = arguments[2];\n      options = {};\n    }\n    let animation;\n    let range;\n    if (arguments.length == 1) {\n      range = arguments[0];\n      animation = (range.animation !== undefined) ? range.animation : true;\n      this.range.setRange(range.start, range.end, { animation });\n    }\n    else if (arguments.length == 2 && typeof arguments[1] == \"function\") {\n      range = arguments[0];\n      callback = arguments[1];\n      animation = (range.animation !== undefined) ? range.animation : true;\n      this.range.setRange(range.start, range.end, { animation }, callback);\n    }\n    else {\n      animation = (options && options.animation !== undefined) ? options.animation : true;\n      this.range.setRange(start, end, { animation }, callback);\n    }\n  }\n\n  /**\n   * Move the window such that given time is centered on screen.\n   * @param {Date | number | string} time\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  moveTo(time, options, callback) {\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const interval = this.range.end - this.range.start;\n    const t = util.convert(time, 'Date').valueOf();\n\n    const start = t - interval / 2;\n    const end = t + interval / 2;\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n\n    this.range.setRange(start, end, { animation }, callback);\n  }\n\n  /**\n   * Get the visible window\n   * @return {{start: Date, end: Date}}   Visible range\n   */\n  getWindow() {\n    const range = this.range.getRange();\n    return {\n      start: new Date(range.start),\n      end: new Date(range.end)\n    };\n  }\n\n  /**\n   * Zoom in the window such that given time is centered on screen.\n   * @param {number} percentage - must be between [0..1]\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  zoomIn(percentage, options, callback) {\n    if (!percentage || percentage < 0 || percentage > 1) return;\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const range = this.getWindow();\n    const start = range.start.valueOf();\n    const end = range.end.valueOf();\n    const interval = end - start;\n    const newInterval = interval / (1 + percentage);\n    const distance = (interval - newInterval) / 2;\n    const newStart = start + distance;\n    const newEnd = end - distance;\n\n    this.setWindow(newStart, newEnd, options, callback);\n  }\n\n  /**\n   * Zoom out the window such that given time is centered on screen.\n   * @param {number} percentage - must be between [0..1]\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  zoomOut(percentage, options, callback) {\n    if (!percentage || percentage < 0 || percentage > 1) return\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const range = this.getWindow();\n    const start = range.start.valueOf();\n    const end = range.end.valueOf();\n    const interval = end - start;\n    const newStart = start - interval * percentage / 2;\n    const newEnd = end + interval * percentage / 2;\n\n    this.setWindow(newStart, newEnd, options, callback);\n  }\n\n  /**\n   * Force a redraw. Can be overridden by implementations of Core\n   *\n   * Note: this function will be overridden on construction with a trottled version\n   */\n  redraw() {\n    this._redraw();\n  }\n\n  /**\n   * Redraw for internal use. Redraws all components. See also the public\n   * method redraw.\n   * @protected\n   */\n  _redraw() {\n    this.redrawCount++;\n    const dom = this.dom;\n    \n    if (!dom || !dom.container || dom.root.offsetWidth == 0) return; // when destroyed, or invisible\n    \n    let resized = false;\n    const options = this.options;\n    const props = this.props;\n\n    DateUtil.updateHiddenDates(this.options.moment, this.body, this.options.hiddenDates);\n\n    // update class names\n    if (options.orientation == 'top') {\n      util.addClassName(dom.root, 'vis-top');\n      util.removeClassName(dom.root, 'vis-bottom');\n    }\n    else {\n      util.removeClassName(dom.root, 'vis-top');\n      util.addClassName(dom.root, 'vis-bottom');\n    }\n\n    if (options.rtl) {\n      util.addClassName(dom.root, 'vis-rtl');\n      util.removeClassName(dom.root, 'vis-ltr');\n    } else {\n      util.addClassName(dom.root, 'vis-ltr');\n      util.removeClassName(dom.root, 'vis-rtl');\n    }\n\n    // update root width and height options\n    dom.root.style.maxHeight = util.option.asSize(options.maxHeight, '');\n    dom.root.style.minHeight = util.option.asSize(options.minHeight, '');\n    dom.root.style.width = util.option.asSize(options.width, '');\n    const rootOffsetWidth = dom.root.offsetWidth;\n\n    // calculate border widths\n    props.border.left = 1\n    props.border.right = 1\n    props.border.top = 1\n    props.border.bottom = 1\n\n    // calculate the heights. If any of the side panels is empty, we set the height to\n    // minus the border width, such that the border will be invisible\n    props.center.height = dom.center.offsetHeight;\n    props.left.height   = dom.left.offsetHeight;\n    props.right.height  = dom.right.offsetHeight;\n    props.top.height    = dom.top.clientHeight    || -props.border.top;\n    props.bottom.height = dom.bottom.clientHeight || -props.border.bottom;\n\n    // TODO: compensate borders when any of the panels is empty.\n\n    // apply auto height\n    // TODO: only calculate autoHeight when needed (else we cause an extra reflow/repaint of the DOM)\n    const contentHeight = Math.max(props.left.height, props.center.height, props.right.height);\n    const autoHeight = props.top.height + contentHeight + props.bottom.height + props.border.top + props.border.bottom;\n    dom.root.style.height = util.option.asSize(options.height, `${autoHeight}px`);\n\n    // calculate heights of the content panels\n    props.root.height = dom.root.offsetHeight;\n    props.background.height = props.root.height;\n    const containerHeight = props.root.height - props.top.height - props.bottom.height;\n    props.centerContainer.height  = containerHeight;\n    props.leftContainer.height    = containerHeight;\n    props.rightContainer.height   = props.leftContainer.height;\n\n    // calculate the widths of the panels\n    props.root.width = rootOffsetWidth;\n    props.background.width = props.root.width;\n\n    if (!this.initialDrawDone) {\n      props.scrollbarWidth = util.getScrollBarWidth();\n    }\n\n    const leftContainerClientWidth = dom.leftContainer.clientWidth;\n    const rightContainerClientWidth = dom.rightContainer.clientWidth;\n\n    if (options.verticalScroll) {\n      if (options.rtl) {\n        props.left.width = leftContainerClientWidth || -props.border.left;\n        props.right.width = rightContainerClientWidth + props.scrollbarWidth || -props.border.right;\n      } else {\n        props.left.width = leftContainerClientWidth + props.scrollbarWidth || -props.border.left;\n        props.right.width = rightContainerClientWidth || -props.border.right;\n      }\n    } else {\n      props.left.width = leftContainerClientWidth  || -props.border.left;\n      props.right.width = rightContainerClientWidth || -props.border.right;\n    }\n\n    this._setDOM();\n\n    // update the scrollTop, feasible range for the offset can be changed\n    // when the height of the Core or of the contents of the center changed\n    let offset = this._updateScrollTop();\n\n    // reposition the scrollable contents\n    if (options.orientation.item != 'top') {\n      offset += Math.max(props.centerContainer.height - props.center.height -\n        props.border.top - props.border.bottom, 0);\n    }\n    dom.center.style.transform  = `translateY(${offset}px)`;\n\n    // show shadows when vertical scrolling is available\n    const visibilityTop = props.scrollTop == 0 ? 'hidden' : '';\n    const visibilityBottom = props.scrollTop == props.scrollTopMin ? 'hidden' : '';\n    dom.shadowTop.style.visibility          = visibilityTop;\n    dom.shadowBottom.style.visibility       = visibilityBottom;\n    dom.shadowTopLeft.style.visibility      = visibilityTop;\n    dom.shadowBottomLeft.style.visibility   = visibilityBottom;\n    dom.shadowTopRight.style.visibility     = visibilityTop;\n    dom.shadowBottomRight.style.visibility  = visibilityBottom;\n\n    if (options.verticalScroll) {\n      dom.rightContainer.className = 'vis-panel vis-right vis-vertical-scroll';\n      dom.leftContainer.className = 'vis-panel vis-left vis-vertical-scroll';\n\n      dom.shadowTopRight.style.visibility     = \"hidden\";\n      dom.shadowBottomRight.style.visibility  = \"hidden\";\n      dom.shadowTopLeft.style.visibility      = \"hidden\";\n      dom.shadowBottomLeft.style.visibility   = \"hidden\";\n\n      dom.left.style.top    = '0px';\n      dom.right.style.top   = '0px';\n    }\n\n    if (!options.verticalScroll ||  props.center.height < props.centerContainer.height) {\n      dom.left.style.top    = `${offset}px`;\n      dom.right.style.top   = `${offset}px`;\n      dom.rightContainer.className = dom.rightContainer.className.replace(new RegExp('(?:^|\\\\s)'+ 'vis-vertical-scroll' + '(?:\\\\s|$)'), ' ');\n      dom.leftContainer.className = dom.leftContainer.className.replace(new RegExp('(?:^|\\\\s)'+ 'vis-vertical-scroll' + '(?:\\\\s|$)'), ' ');\n      props.left.width = leftContainerClientWidth  || -props.border.left;\n      props.right.width = rightContainerClientWidth || -props.border.right;\n      this._setDOM();\n    }\n\n    // enable/disable vertical panning\n    const contentsOverflow = props.center.height > props.centerContainer.height;\n    this.hammer.get('pan').set({\n      direction: contentsOverflow ? Hammer.DIRECTION_ALL : Hammer.DIRECTION_HORIZONTAL\n    });\n\n    // set the long press time\n    this.hammer.get('press').set({\n      time: this.options.longSelectPressTime\n    });\n\n    // redraw all components\n    this.components.forEach(component => {\n      resized = component.redraw() || resized;\n    });\n    const MAX_REDRAW = 5;\n    if (resized) {\n      if (this.redrawCount < MAX_REDRAW) {\n        this.body.emitter.emit('_change');\n        return;\n      }\n      else {\n        console.log('WARNING: infinite loop in redraw?');\n      }\n    } else {\n      this.redrawCount = 0;\n    }\n\n    //Emit public 'changed' event for UI updates, see issue #1592\n    this.body.emitter.emit(\"changed\");\n  }\n\n  /**\n   * sets the basic DOM components needed for the timeline\\graph2d\n   */\n  _setDOM() {\n    const props = this.props;\n    const dom = this.dom;\n\n    props.leftContainer.width = props.left.width;\n    props.rightContainer.width = props.right.width;\n    const centerWidth = props.root.width - props.left.width - props.right.width;\n    props.center.width          = centerWidth;\n    props.centerContainer.width = centerWidth;\n    props.top.width             = centerWidth;\n    props.bottom.width          = centerWidth;\n\n    // resize the panels\n    dom.background.style.height           = `${props.background.height}px`;\n    dom.backgroundVertical.style.height   = `${props.background.height}px`;\n    dom.backgroundHorizontal.style.height = `${props.centerContainer.height}px`;\n    dom.centerContainer.style.height      = `${props.centerContainer.height}px`;\n    dom.leftContainer.style.height        = `${props.leftContainer.height}px`;\n    dom.rightContainer.style.height       = `${props.rightContainer.height}px`;\n\n    dom.background.style.width            = `${props.background.width}px`;\n    dom.backgroundVertical.style.width    = `${props.centerContainer.width}px`;\n    dom.backgroundHorizontal.style.width  = `${props.background.width}px`;\n    dom.centerContainer.style.width       = `${props.center.width}px`;\n    dom.top.style.width                   = `${props.top.width}px`;\n    dom.bottom.style.width                = `${props.bottom.width}px`;\n\n    // reposition the panels\n    dom.background.style.left           = '0';\n    dom.background.style.top            = '0';\n    dom.backgroundVertical.style.left   = `${props.left.width + props.border.left}px`;\n    dom.backgroundVertical.style.top    = '0';\n    dom.backgroundHorizontal.style.left = '0';\n    dom.backgroundHorizontal.style.top  = `${props.top.height}px`;\n    dom.centerContainer.style.left      = `${props.left.width}px`;\n    dom.centerContainer.style.top       = `${props.top.height}px`;\n    dom.leftContainer.style.left        = '0';\n    dom.leftContainer.style.top         = `${props.top.height}px`;\n    dom.rightContainer.style.left       = `${props.left.width + props.center.width}px`;\n    dom.rightContainer.style.top        = `${props.top.height}px`;\n    dom.top.style.left                  = `${props.left.width}px`;\n    dom.top.style.top                   = '0';\n    dom.bottom.style.left               = `${props.left.width}px`;\n    dom.bottom.style.top                = `${props.top.height + props.centerContainer.height}px`;\n    dom.center.style.left               = '0';\n    dom.left.style.left                 = '0';\n    dom.right.style.left                = '0';\n  }\n\n  /**\n   * Set a current time. This can be used for example to ensure that a client's\n   * time is synchronized with a shared server time.\n   * Only applicable when option `showCurrentTime` is true.\n   * @param {Date | string | number} time     A Date, unix timestamp, or\n   *                                          ISO date string.\n   */\n  setCurrentTime(time) {\n    if (!this.currentTime) {\n      throw new Error('Option showCurrentTime must be true');\n    }\n\n    this.currentTime.setCurrentTime(time);\n  }\n\n  /**\n   * Get the current time.\n   * Only applicable when option `showCurrentTime` is true.\n   * @return {Date} Returns the current time.\n   */\n  getCurrentTime() {\n    if (!this.currentTime) {\n      throw new Error('Option showCurrentTime must be true');\n    }\n\n    return this.currentTime.getCurrentTime();\n  }\n\n  /**\n   * Convert a position on screen (pixels) to a datetime\n   * @param {int}     x    Position on the screen in pixels\n   * @return {Date}   time The datetime the corresponds with given position x\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toTime(x) {\n    return DateUtil.toTime(this, x, this.props.center.width);\n  }\n\n  /**\n   * Convert a position on the global screen (pixels) to a datetime\n   * @param {int}     x    Position on the screen in pixels\n   * @return {Date}   time The datetime the corresponds with given position x\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toGlobalTime(x) {\n    return DateUtil.toTime(this, x, this.props.root.width);\n    //var conversion = this.range.conversion(this.props.root.width);\n    //return new Date(x / conversion.scale + conversion.offset);\n  }\n\n  /**\n   * Convert a datetime (Date object) into a position on the screen\n   * @param {Date}   time A date\n   * @return {int}   x    The position on the screen in pixels which corresponds\n   *                      with the given date.\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toScreen(time) {\n    return DateUtil.toScreen(this, time, this.props.center.width);\n  }\n\n  /**\n   * Convert a datetime (Date object) into a position on the root\n   * This is used to get the pixel density estimate for the screen, not the center panel\n   * @param {Date}   time A date\n   * @return {int}   x    The position on root in pixels which corresponds\n   *                      with the given date.\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toGlobalScreen(time) {\n    return DateUtil.toScreen(this, time, this.props.root.width);\n    //var conversion = this.range.conversion(this.props.root.width);\n    //return (time.valueOf() - conversion.offset) * conversion.scale;\n  }\n\n  /**\n   * Initialize watching when option autoResize is true\n   * @private\n   */\n  _initAutoResize() {\n    if (this.options.autoResize == true) {\n      this._startAutoResize();\n    }\n    else {\n      this._stopAutoResize();\n    }\n  }\n\n  /**\n   * Watch for changes in the size of the container. On resize, the Panel will\n   * automatically redraw itself.\n   * @private\n   */\n  _startAutoResize() {\n    const me = this;\n\n    this._stopAutoResize();\n\n    this._onResize = () => {\n      if (me.options.autoResize != true) {\n        // stop watching when the option autoResize is changed to false\n        me._stopAutoResize();\n        return;\n      }\n\n      if (me.dom.root) {\n        const rootOffsetHeight = me.dom.root.offsetHeight;\n        const rootOffsetWidth = me.dom.root.offsetWidth;\n        // check whether the frame is resized\n        // Note: we compare offsetWidth here, not clientWidth. For some reason,\n        // IE does not restore the clientWidth from 0 to the actual width after\n        // changing the timeline's container display style from none to visible\n        if ((rootOffsetWidth != me.props.lastWidth) ||\n          (rootOffsetHeight != me.props.lastHeight)) {\n          me.props.lastWidth = rootOffsetWidth;\n          me.props.lastHeight = rootOffsetHeight;\n          me.props.scrollbarWidth = util.getScrollBarWidth();\n\n          me.body.emitter.emit('_change');\n        }\n      }\n    };\n\n    // add event listener to window resize\n    util.addEventListener(window, 'resize', this._onResize);\n\n    //Prevent initial unnecessary redraw\n    if (me.dom.root) {\n        me.props.lastWidth = me.dom.root.offsetWidth;\n        me.props.lastHeight = me.dom.root.offsetHeight;\n    }\n\n    this.watchTimer = setInterval(this._onResize, 1000);\n  }\n\n  /**\n   * Stop watching for a resize of the frame.\n   * @private\n   */\n  _stopAutoResize() {\n    if (this.watchTimer) {\n      clearInterval(this.watchTimer);\n      this.watchTimer = undefined;\n    }\n\n    // remove event listener on window.resize\n    if (this._onResize) {\n      util.removeEventListener(window, 'resize', this._onResize);\n      this._onResize = null;\n    }\n  }\n\n  /**\n   * Start moving the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {  // eslint-disable-line no-unused-vars\n    this.touch.allowDragging = true;\n    this.touch.initialScrollTop = this.props.scrollTop;\n  }\n\n  /**\n   * Start moving the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onPinch(event) {  // eslint-disable-line no-unused-vars\n    this.touch.allowDragging = false;\n  }\n\n  /**\n   * Move the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!event) return\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.touch.allowDragging) return;\n\n    const delta = event.deltaY;\n\n    const oldScrollTop = this._getScrollTop();\n    const newScrollTop = this._setScrollTop(this.touch.initialScrollTop + delta);\n\n    if (this.options.verticalScroll) {\n      this.dom.left.parentNode.scrollTop = -this.props.scrollTop;\n      this.dom.right.parentNode.scrollTop = -this.props.scrollTop;\n    }\n\n    if (newScrollTop != oldScrollTop) {\n      this.emit(\"verticalDrag\");\n    }\n  }\n\n  /**\n   * Apply a scrollTop\n   * @param {number} scrollTop\n   * @returns {number} scrollTop  Returns the applied scrollTop\n   * @private\n   */\n  _setScrollTop(scrollTop) {\n    this.props.scrollTop = scrollTop;\n    this._updateScrollTop();\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Update the current scrollTop when the height of  the containers has been changed\n   * @returns {number} scrollTop  Returns the applied scrollTop\n   * @private\n   */\n  _updateScrollTop() {\n    // recalculate the scrollTopMin\n    const scrollTopMin = Math.min(this.props.centerContainer.height - this.props.center.height, 0); // is negative or zero\n    if (scrollTopMin != this.props.scrollTopMin) {\n      // in case of bottom orientation, change the scrollTop such that the contents\n      // do not move relative to the time axis at the bottom\n      if (this.options.orientation.item != 'top') {\n        this.props.scrollTop += (scrollTopMin - this.props.scrollTopMin);\n      }\n      this.props.scrollTopMin = scrollTopMin;\n    }\n\n    // limit the scrollTop to the feasible scroll range\n    if (this.props.scrollTop > 0) this.props.scrollTop = 0;\n    if (this.props.scrollTop < scrollTopMin) this.props.scrollTop = scrollTopMin;\n\n    if (this.options.verticalScroll) {\n      this.dom.left.parentNode.scrollTop = -this.props.scrollTop;\n      this.dom.right.parentNode.scrollTop = -this.props.scrollTop;\n    }\n    this._isProgramaticallyScrolled = true;\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Get the current scrollTop\n   * @returns {number} scrollTop\n   * @private\n   */\n  _getScrollTop() {\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Load a configurator\n   * [at]returns {Object}\n   * @private\n   */\n  _createConfigurator() {\n    throw new Error('Cannot invoke abstract method _createConfigurator');\n  }\n}\n\n// turn Core into an event emitter\nEmitter(Core.prototype);\n\nexport default Core;\n","import util from '../../util';\nimport Component from './Component';\nimport moment from '../../module/moment';\nimport locales from '../locales';\n\n/**\n * A current time bar\n */\nclass CurrentTime extends Component {\n/**\n * @param {{range: Range, dom: Object, domProps: Object}} body\n * @param {Object} [options]        Available parameters:\n *                                  {Boolean} [showCurrentTime]\n *                                  {String}  [alignCurrentTime]\n * @constructor CurrentTime\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n\n    // default options\n    this.defaultOptions = {\n      rtl: false,\n      showCurrentTime: true,\n      alignCurrentTime: undefined,\n\n      moment,\n      locales,\n      locale: 'en'\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.setOptions(options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n    this.offset = 0;\n\n    this._create();\n\n  }\n\n  /**\n   * Create the HTML DOM for the current time bar\n   * @private\n   */\n  _create() {\n    const bar = document.createElement('div');\n    bar.className = 'vis-current-time';\n    bar.style.position = 'absolute';\n    bar.style.top = '0px';\n    bar.style.height = '100%';\n\n    this.bar = bar;\n  }\n\n  /**\n   * Destroy the CurrentTime bar\n   */\n  destroy() {\n    this.options.showCurrentTime = false;\n    this.redraw(); // will remove the bar from the DOM and stop refreshing\n\n    this.body = null;\n  }\n\n  /**\n   * Set options for the component. Options will be merged in current options.\n   * @param {Object} options  Available parameters:\n   *                          {boolean} [showCurrentTime]\n   *                          {String}  [alignCurrentTime]\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend(['rtl', 'showCurrentTime', 'alignCurrentTime', 'moment', 'locale', 'locales'], this.options, options);\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    if (this.options.showCurrentTime) {\n      const parent = this.body.dom.backgroundVertical;\n      if (this.bar.parentNode != parent) {\n        // attach to the dom\n        if (this.bar.parentNode) {\n          this.bar.parentNode.removeChild(this.bar);\n        }\n        parent.appendChild(this.bar);\n\n        this.start();\n      }\n\n      let now = this.options.moment(Date.now() + this.offset);\n\n      if (this.options.alignCurrentTime) {\n        now = now.startOf(this.options.alignCurrentTime);\n      }\n\n      const x = this.body.util.toScreen(now);\n\n      let locale = this.options.locales[this.options.locale];\n      if (!locale) {\n        if (!this.warned) {\n          console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n          this.warned = true;\n        }\n        locale = this.options.locales['en']; // fall back on english when not available\n      }\n      let title = `${locale.current} ${locale.time}: ${now.format('dddd, MMMM Do YYYY, H:mm:ss')}`;\n      title = title.charAt(0).toUpperCase() + title.substring(1);\n\n      if (this.options.rtl) {\n        this.bar.style.transform = `translateX(${x * -1}px)`;\n      } else {\n        this.bar.style.transform = `translateX(${x}px)`;\n      }\n      this.bar.title = title;\n    }\n    else {\n      // remove the line from the DOM\n      if (this.bar.parentNode) {\n        this.bar.parentNode.removeChild(this.bar);\n      }\n      this.stop();\n    }\n\n    return false;\n  }\n\n  /**\n   * Start auto refreshing the current time bar\n   */\n  start() {\n    const me = this;\n\n      /**\n       *  Updates the current time.\n       */\n      function update () {\n      me.stop();\n\n      // determine interval to refresh\n      const scale = me.body.range.conversion(me.body.domProps.center.width).scale;\n      let interval = 1 / scale / 10;\n      if (interval < 30)   interval = 30;\n      if (interval > 1000) interval = 1000;\n\n      me.redraw();\n      me.body.emitter.emit('currentTimeTick');\n\n      // start a renderTimer to adjust for the new time\n      me.currentTimeTimer = setTimeout(update, interval);\n    }\n\n    update();\n  }\n\n  /**\n   * Stop auto refreshing the current time bar\n   */\n  stop() {\n    if (this.currentTimeTimer !== undefined) {\n      clearTimeout(this.currentTimeTimer);\n      delete this.currentTimeTimer;\n    }\n  }\n\n  /**\n   * Set a current time. This can be used for example to ensure that a client's\n   * time is synchronized with a shared server time.\n   * @param {Date | string | number} time     A Date, unix timestamp, or\n   *                                          ISO date string.\n   */\n  setCurrentTime(time) {\n    const t = util.convert(time, 'Date').valueOf();\n    const now = Date.now();\n    this.offset = t - now;\n    this.redraw();\n  }\n\n  /**\n   * Get the current time.\n   * @return {Date} Returns the current time.\n   */\n  getCurrentTime() {\n    return new Date(Date.now() + this.offset);\n  }\n}\n\nexport default CurrentTime;\n","'use strict';\nvar $ = require('../internals/export');\nvar $find = require('../internals/array-iteration').find;\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar FIND = 'find';\nvar SKIPS_HOLES = true;\n\nvar USES_TO_LENGTH = arrayMethodUsesToLength(FIND);\n\n// Shouldn't skip holes\nif (FIND in []) Array(1)[FIND](function () { SKIPS_HOLES = false; });\n\n// `Array.prototype.find` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.find\n$({ target: 'Array', proto: true, forced: SKIPS_HOLES || !USES_TO_LENGTH }, {\n  find: function find(callbackfn /* , that = undefined */) {\n    return $find(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables(FIND);\n","require('../../../modules/es.array.find');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').find;\n","var find = require('../array/virtual/find');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.find;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.find) ? find : own;\n};\n","// Unique ID creation requires a high quality random # generator. In the browser we therefore\n// require the crypto API and do not support built-in fallback to lower quality random number\n// generators (like Math.random()).\n// getRandomValues needs to be invoked in a context where \"this\" is a Crypto implementation. Also,\n// find the complete implementation of crypto (msCrypto) on IE11.\nvar getRandomValues = typeof crypto != 'undefined' && crypto.getRandomValues && crypto.getRandomValues.bind(crypto) || typeof msCrypto != 'undefined' && typeof msCrypto.getRandomValues == 'function' && msCrypto.getRandomValues.bind(msCrypto);\nvar rnds8 = new Uint8Array(16); // eslint-disable-line no-undef\n\nexport default function rng() {\n  if (!getRandomValues) {\n    throw new Error('crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported');\n  }\n\n  return getRandomValues(rnds8);\n}","/**\n * Convert array of 16 byte values to UUID string format of the form:\n * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n */\nvar byteToHex = [];\n\nfor (var i = 0; i < 256; ++i) {\n  byteToHex[i] = (i + 0x100).toString(16).substr(1);\n}\n\nfunction bytesToUuid(buf, offset) {\n  var i = offset || 0;\n  var bth = byteToHex; // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4\n\n  return [bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]]].join('');\n}\n\nexport default bytesToUuid;","import rng from './rng.js';\nimport bytesToUuid from './bytesToUuid.js';\n\nfunction v4(options, buf, offset) {\n  var i = buf && offset || 0;\n\n  if (typeof options == 'string') {\n    buf = options === 'binary' ? new Array(16) : null;\n    options = null;\n  }\n\n  options = options || {};\n  var rnds = options.random || (options.rng || rng)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`\n\n  rnds[6] = rnds[6] & 0x0f | 0x40;\n  rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided\n\n  if (buf) {\n    for (var ii = 0; ii < 16; ++ii) {\n      buf[i + ii] = rnds[ii];\n    }\n  }\n\n  return buf || bytesToUuid(rnds);\n}\n\nexport default v4;","'use strict';\nvar $ = require('../internals/export');\nvar $includes = require('../internals/array-includes').includes;\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar USES_TO_LENGTH = arrayMethodUsesToLength('indexOf', { ACCESSORS: true, 1: 0 });\n\n// `Array.prototype.includes` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.includes\n$({ target: 'Array', proto: true, forced: !USES_TO_LENGTH }, {\n  includes: function includes(el /* , fromIndex = 0 */) {\n    return $includes(this, el, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('includes');\n","require('../../../modules/es.array.includes');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').includes;\n","var isObject = require('../internals/is-object');\nvar classof = require('../internals/classof-raw');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar MATCH = wellKnownSymbol('match');\n\n// `IsRegExp` abstract operation\n// https://tc39.github.io/ecma262/#sec-isregexp\nmodule.exports = function (it) {\n  var isRegExp;\n  return isObject(it) && ((isRegExp = it[MATCH]) !== undefined ? !!isRegExp : classof(it) == 'RegExp');\n};\n","var isRegExp = require('../internals/is-regexp');\n\nmodule.exports = function (it) {\n  if (isRegExp(it)) {\n    throw TypeError(\"The method doesn't accept regular expressions\");\n  } return it;\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar MATCH = wellKnownSymbol('match');\n\nmodule.exports = function (METHOD_NAME) {\n  var regexp = /./;\n  try {\n    '/./'[METHOD_NAME](regexp);\n  } catch (e) {\n    try {\n      regexp[MATCH] = false;\n      return '/./'[METHOD_NAME](regexp);\n    } catch (f) { /* empty */ }\n  } return false;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar notARegExp = require('../internals/not-a-regexp');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\nvar correctIsRegExpLogic = require('../internals/correct-is-regexp-logic');\n\n// `String.prototype.includes` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.includes\n$({ target: 'String', proto: true, forced: !correctIsRegExpLogic('includes') }, {\n  includes: function includes(searchString /* , position = 0 */) {\n    return !!~String(requireObjectCoercible(this))\n      .indexOf(notARegExp(searchString), arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.string.includes');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').includes;\n","var arrayIncludes = require('../array/virtual/includes');\nvar stringIncludes = require('../string/virtual/includes');\n\nvar ArrayPrototype = Array.prototype;\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.includes;\n  if (it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.includes)) return arrayIncludes;\n  if (typeof it === 'string' || it === StringPrototype || (it instanceof String && own === StringPrototype.includes)) {\n    return stringIncludes;\n  } return own;\n};\n","// Utility functions for ordering and stacking of items\nconst EPSILON = 0.001; // used when checking collisions, to prevent round-off errors\n\n/**\n * Order items by their start data\n * @param {Item[]} items\n */\nexport function orderByStart(items) {\n  items.sort((a, b) => a.data.start - b.data.start);\n}\n\n/**\n * Order items by their end date. If they have no end date, their start date\n * is used.\n * @param {Item[]} items\n */\nexport function orderByEnd(items) {\n  items.sort((a, b) => {\n    const aTime = ('end' in a.data) ? a.data.end : a.data.start;\n    const bTime = ('end' in b.data) ? b.data.end : b.data.start;\n\n    return aTime - bTime;\n  });\n}\n\n/**\n * Adjust vertical positions of the items such that they don't overlap each\n * other.\n * @param {Item[]} items\n *            All visible items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {boolean} [force=false]\n *            If true, all items will be repositioned. If false (default), only\n *            items having a top===null will be re-stacked\n * @param {function} shouldBailItemsRedrawFunction\n *            bailing function\n * @return {boolean} shouldBail\n */\nexport function stack(items, margin, force, shouldBailItemsRedrawFunction) {\n  if (force) {\n    // reset top position of all items\n    for (var i = 0; i < items.length; i++) {\n      items[i].top = null;\n    }\n  }\n\n  // calculate new, non-overlapping positions\n  for (var i = 0; i < items.length; i++) {  // eslint-disable-line no-redeclare\n    const item = items[i];\n    if (item.stack && item.top === null) {\n      // initialize top position\n      item.top = margin.axis;\n      var shouldBail = false;\n\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (let j = 0, jj = items.length; j < jj; j++) {\n          const other = items[j];\n          shouldBail = shouldBailItemsRedrawFunction() || false;\n\n          if (shouldBail) { return true; }\n\n          if (other.top !== null && other !== item && other.stack && collision(item, other, margin.item, other.options.rtl)) {\n            collidingItem = other;\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the items above the colliding element\n          item.top = collidingItem.top + collidingItem.height + margin.item.vertical;\n        }\n      } while (collidingItem);\n    }\n  }\n  return shouldBail;\n}\n\n/**\n * Adjust vertical positions of the items within a single subgroup such that they\n * don't overlap each other.\n * @param {Item[]} items\n *            All items withina subgroup\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroup} subgroup\n *            The subgroup that is being stacked\n */\nexport function substack(items, margin, subgroup) {\n  for (var i = 0; i < items.length; i++) {\n    items[i].top = null;\n  }\n\n  // Set the initial height\n  let subgroupHeight = subgroup.height;\n\n  // calculate new, non-overlapping positions\n  for (i = 0; i < items.length; i++) {\n    const item = items[i];\n\n    if (item.stack && item.top === null) {\n      // initialize top position\n      item.top = item.baseTop;//margin.axis + item.baseTop;\n\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (let j = 0, jj = items.length; j < jj; j++) {\n          const other = items[j];\n          if (other.top !== null && other !== item /*&& other.stack*/ && collision(item, other, margin.item, other.options.rtl)) {\n            collidingItem = other;\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the items above the colliding element\n          item.top = collidingItem.top + collidingItem.height + margin.item.vertical;// + item.baseTop;\n        }\n\n        if (item.top + item.height > subgroupHeight) {\n          subgroupHeight = item.top + item.height;\n        }\n      } while (collidingItem);\n    }\n  }\n\n  // Set the new height\n  subgroup.height = subgroupHeight - subgroup.top + 0.5 * margin.item.vertical;\n}\n\n/**\n * Adjust vertical positions of the items without stacking them\n * @param {Item[]} items\n *            All visible items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n * @param {boolean} isStackSubgroups\n */\nexport function nostack(items, margin, subgroups, isStackSubgroups) {\n  for (let i = 0; i < items.length; i++) {\n    if (items[i].data.subgroup == undefined) {\n      items[i].top = margin.item.vertical;\n    } else if (items[i].data.subgroup !== undefined && isStackSubgroups) {\n      let newTop = 0;\n      for (const subgroup in subgroups) {\n        if (subgroups.hasOwnProperty(subgroup)) {\n          if (subgroups[subgroup].visible == true && subgroups[subgroup].index < subgroups[items[i].data.subgroup].index) {\n            newTop += subgroups[subgroup].height;\n            subgroups[items[i].data.subgroup].top = newTop;\n          }\n        }\n      }\n      items[i].top = newTop + 0.5 * margin.item.vertical;\n    }\n  }\n  if (!isStackSubgroups) {\n    stackSubgroups(items, margin, subgroups)\n  }\n}\n\n/**\n * Adjust vertical positions of the subgroups such that they don't overlap each\n * other.\n * @param {Array.<timeline.Item>} items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n */\nexport function stackSubgroups(items, margin, subgroups) {\n  for (const subgroup in subgroups) {\n    if (subgroups.hasOwnProperty(subgroup)) {\n\n\n      subgroups[subgroup].top = 0;\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (const otherSubgroup in subgroups) {\n          if (subgroups[otherSubgroup].top !== null && otherSubgroup !== subgroup && subgroups[subgroup].index > subgroups[otherSubgroup].index && collisionByTimes(subgroups[subgroup], subgroups[otherSubgroup])) {\n            collidingItem = subgroups[otherSubgroup];\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the subgroups above the colliding element\n          subgroups[subgroup].top = collidingItem.top + collidingItem.height;\n        }\n      } while (collidingItem);\n    }\n  }\n  for (let i = 0; i < items.length; i++) {\n    if (items[i].data.subgroup !== undefined) {\n      items[i].top = subgroups[items[i].data.subgroup].top + 0.5 * margin.item.vertical;\n    }\n  }\n}\n\n/**\n * Adjust vertical positions of the subgroups such that they don't overlap each\n * other, then stacks the contents of each subgroup individually.\n * @param {Item[]} subgroupItems\n *            All the items in a subgroup\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n */\nexport function stackSubgroupsWithInnerStack(subgroupItems, margin, subgroups) {\n  let doSubStack = false;\n\n  // Run subgroups in their order (if any)\n  const subgroupOrder = [];\n\n  for(var subgroup in subgroups) {\n    if (subgroups[subgroup].hasOwnProperty(\"index\")) {\n      subgroupOrder[subgroups[subgroup].index] = subgroup;\n    }\n    else {\n      subgroupOrder.push(subgroup);\n    }\n  }\n\n  for(let j = 0; j < subgroupOrder.length; j++) {\n    subgroup = subgroupOrder[j];\n    if (subgroups.hasOwnProperty(subgroup)) {\n\n      doSubStack = doSubStack || subgroups[subgroup].stack;\n      subgroups[subgroup].top = 0;\n\n      for (const otherSubgroup in subgroups) {\n        if (subgroups[otherSubgroup].visible && subgroups[subgroup].index > subgroups[otherSubgroup].index) {\n          subgroups[subgroup].top += subgroups[otherSubgroup].height;\n        }\n      }\n\n      const items = subgroupItems[subgroup];\n      for(let i = 0; i < items.length; i++) {\n        if (items[i].data.subgroup !== undefined) {\n          items[i].top = subgroups[items[i].data.subgroup].top + 0.5 * margin.item.vertical;\n\n          if (subgroups[subgroup].stack) {\n            items[i].baseTop = items[i].top;\n          }\n        }\n      }\n\n      if (doSubStack && subgroups[subgroup].stack) {\n        substack(subgroupItems[subgroup], margin, subgroups[subgroup]);\n      }\n    }\n  }\n}\n\n/**\n * Test if the two provided items collide\n * The items must have parameters left, width, top, and height.\n * @param {Item} a          The first item\n * @param {Item} b          The second item\n * @param {{horizontal: number, vertical: number}} margin\n *                          An object containing a horizontal and vertical\n *                          minimum required margin.\n * @param {boolean} rtl\n * @return {boolean}        true if a and b collide, else false\n */\nexport function collision(a, b, margin, rtl) {\n  if (rtl) {\n    return  ((a.right - margin.horizontal + EPSILON)  < (b.right + b.width) &&\n    (a.right + a.width + margin.horizontal - EPSILON) > b.right &&\n    (a.top - margin.vertical + EPSILON)              < (b.top + b.height) &&\n    (a.top + a.height + margin.vertical - EPSILON)   > b.top);\n  } else {\n    return ((a.left - margin.horizontal + EPSILON)   < (b.left + b.width) &&\n    (a.left + a.width + margin.horizontal - EPSILON) > b.left &&\n    (a.top - margin.vertical + EPSILON)              < (b.top + b.height) &&\n    (a.top + a.height + margin.vertical - EPSILON)   > b.top);\n  }\n}\n\n/**\n * Test if the two provided objects collide\n * The objects must have parameters start, end, top, and height.\n * @param {Object} a          The first Object\n * @param {Object} b          The second Object\n * @return {boolean}        true if a and b collide, else false\n */\nexport function collisionByTimes(a, b) {\n\n  // Check for overlap by time and height. Abutting is OK and\n  // not considered a collision while overlap is considered a collision.\n  const timeOverlap = a.start < b.end && a.end > b.start;\n  const heightOverlap = a.top < (b.top + b.height) && (a.top + a.height) > b.top;\n  return timeOverlap && heightOverlap;\n}\n","import util from '../../util';\nimport * as stack from '../Stack';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n\n/**\n * @constructor Group\n */\nclass Group {\n  /**\n * @param {number | string} groupId\n * @param {Object} data\n * @param {ItemSet} itemSet\n * @constructor Group\n */\n  constructor(groupId, data, itemSet) {\n    this.groupId = groupId;\n    this.subgroups = {};\n    this.subgroupStack = {};\n    this.subgroupStackAll = false;\n    this.subgroupVisibility = {};\n    this.doInnerStack = false;\n    this.shouldBailStackItems = false;\n    this.subgroupIndex = 0;\n    this.subgroupOrderer = data && data.subgroupOrder;\n    this.itemSet = itemSet;\n    this.isVisible = null;\n    this.stackDirty = true; // if true, items will be restacked on next redraw\n\n    // This is a stack of functions (`() => void`) that will be executed before\n    // the instance is disposed off (method `dispose`). Anything that needs to\n    // be manually disposed off before garbage collection happens (or so that\n    // garbage collection can happen) should be added to this stack.\n    this._disposeCallbacks = [];\n\n    if (data && data.nestedGroups) {\n      this.nestedGroups = data.nestedGroups;\n      if (data.showNested == false) {\n        this.showNested = false;\n      } else {\n        this.showNested = true;\n      }\n    }\n\n    if (data && data.subgroupStack) {\n      if (typeof data.subgroupStack === \"boolean\") {\n        this.doInnerStack = data.subgroupStack;\n        this.subgroupStackAll = data.subgroupStack;\n      }\n      else {\n        // We might be doing stacking on specific sub groups, but only\n        // if at least one is set to do stacking\n        for(const key in data.subgroupStack) {\n          this.subgroupStack[key] = data.subgroupStack[key];\n          this.doInnerStack = this.doInnerStack || data.subgroupStack[key];\n        }\n      }\n    }\n\n    if (data && data.heightMode) {\n      this.heightMode = data.heightMode;\n    } else {\n      this.heightMode = itemSet.options.groupHeightMode;\n    }\n\n    this.nestedInGroup = null;\n\n    this.dom = {};\n    this.props = {\n      label: {\n        width: 0,\n        height: 0\n      }\n    };\n    this.className = null;\n\n    this.items = {};        // items filtered by groupId of this group\n    this.visibleItems = []; // items currently visible in window\n    this.itemsInRange = []; // items currently in range\n    this.orderedItems = {\n      byStart: [],\n      byEnd: []\n    };\n    this.checkRangedItems = false; // needed to refresh the ranged items if the window is programatically changed with NO overlap.\n\n    const handleCheckRangedItems = () => {\n      this.checkRangedItems = true;\n    };\n    this.itemSet.body.emitter.on(\"checkRangedItems\", handleCheckRangedItems);\n    this._disposeCallbacks.push(() => {\n      this.itemSet.body.emitter.off(\"checkRangedItems\", handleCheckRangedItems);\n    });\n\n    this._create();\n\n    this.setData(data);\n  }\n\n  /**\n   * Create DOM elements for the group\n   * @private\n   */\n  _create() {\n    const label = document.createElement('div');\n    if (this.itemSet.options.groupEditable.order) {\n      label.className = 'vis-label draggable';\n    } else {\n      label.className = 'vis-label';\n    }\n    this.dom.label = label;\n\n    const inner = document.createElement('div');\n    inner.className = 'vis-inner';\n    label.appendChild(inner);\n    this.dom.inner = inner;\n\n    const foreground = document.createElement('div');\n    foreground.className = 'vis-group';\n    foreground['vis-group'] = this;\n    this.dom.foreground = foreground;\n\n    this.dom.background = document.createElement('div');\n    this.dom.background.className = 'vis-group';\n\n    this.dom.axis = document.createElement('div');\n    this.dom.axis.className = 'vis-group';\n\n    // create a hidden marker to detect when the Timelines container is attached\n    // to the DOM, or the style of a parent of the Timeline is changed from\n    // display:none is changed to visible.\n    this.dom.marker = document.createElement('div');\n    this.dom.marker.style.visibility = 'hidden';\n    this.dom.marker.style.position = 'absolute';\n    this.dom.marker.innerHTML = '';\n    this.dom.background.appendChild(this.dom.marker);\n  }\n\n  /**\n   * Set the group data for this group\n   * @param {Object} data   Group data, can contain properties content and className\n   */\n  setData(data) {\n    if (this.itemSet.groupTouchParams.isDragging) return;\n\n    // update contents\n    let content;\n    let templateFunction;\n\n    if (data && data.subgroupVisibility) {\n      for (const key in data.subgroupVisibility) {\n        this.subgroupVisibility[key] = data.subgroupVisibility[key];\n      }\n    }\n\n    if (this.itemSet.options && this.itemSet.options.groupTemplate) {\n      templateFunction = this.itemSet.options.groupTemplate.bind(this);\n      content = templateFunction(data, this.dom.inner);\n    } else {\n      content = data && data.content;\n    }\n\n    if (content instanceof Element) {\n      while (this.dom.inner.firstChild) {\n        this.dom.inner.removeChild(this.dom.inner.firstChild);\n      }\n      this.dom.inner.appendChild(content);\n    } else if (content instanceof Object && content.isReactComponent) {\n      // Do nothing. Component was rendered into the node be ReactDOM.render.\n      // That branch is necessary for evasion of a second call templateFunction.\n      // Supports only React < 16(due to the asynchronous nature of React 16).\n    } else if (content instanceof Object) {\n      templateFunction(data, this.dom.inner);\n    } else if (content !== undefined && content !== null) {\n      this.dom.inner.innerHTML = content;\n    } else {\n      this.dom.inner.innerHTML = this.groupId || ''; // groupId can be null\n    }\n\n    // update title\n    this.dom.label.title = data && data.title || '';\n    if (!this.dom.inner.firstChild) {\n      util.addClassName(this.dom.inner, 'vis-hidden');\n    }\n    else {\n      util.removeClassName(this.dom.inner, 'vis-hidden');\n    }\n\n    if (data && data.nestedGroups) {\n      if (!this.nestedGroups || this.nestedGroups != data.nestedGroups) {\n        this.nestedGroups = data.nestedGroups;\n      }\n\n      if (data.showNested !== undefined || this.showNested === undefined) {\n        if (data.showNested == false) {\n          this.showNested = false;\n        } else {\n          this.showNested = true;\n        }\n      }\n\n      util.addClassName(this.dom.label, 'vis-nesting-group');\n      if (this.showNested) {\n        util.removeClassName(this.dom.label, 'collapsed');\n        util.addClassName(this.dom.label, 'expanded');\n      } else {\n        util.removeClassName(this.dom.label, 'expanded');\n        util.addClassName(this.dom.label, 'collapsed');\n      }\n    } else if (this.nestedGroups) {\n      this.nestedGroups = null;\n      util.removeClassName(this.dom.label, 'collapsed');\n      util.removeClassName(this.dom.label, 'expanded');\n      util.removeClassName(this.dom.label, 'vis-nesting-group');\n    }\n\n    if (data && (data.treeLevel|| data.nestedInGroup)) {\n      util.addClassName(this.dom.label, 'vis-nested-group');\n      if (data.treeLevel) {\n        util.addClassName(this.dom.label, 'vis-group-level-' + data.treeLevel);\n      } else {\n        // Nesting level is unknown, but we're sure it's at least 1\n        util.addClassName(this.dom.label, 'vis-group-level-unknown-but-gte1');\n      }\n    } else {\n      util.addClassName(this.dom.label, 'vis-group-level-0');\n    }\n    \n    // update className\n    const className = data && data.className || null;\n    if (className != this.className) {\n      if (this.className) {\n        util.removeClassName(this.dom.label, this.className);\n        util.removeClassName(this.dom.foreground, this.className);\n        util.removeClassName(this.dom.background, this.className);\n        util.removeClassName(this.dom.axis, this.className);\n      }\n      util.addClassName(this.dom.label, className);\n      util.addClassName(this.dom.foreground, className);\n      util.addClassName(this.dom.background, className);\n      util.addClassName(this.dom.axis, className);\n      this.className = className;\n    }\n\n    // update style\n    if (this.style) {\n      util.removeCssText(this.dom.label, this.style);\n      this.style = null;\n    }\n    if (data && data.style) {\n      util.addCssText(this.dom.label, data.style);\n      this.style = data.style;\n    }\n  }\n\n  /**\n   * Get the width of the group label\n   * @return {number} width\n   */\n  getLabelWidth() {\n    return this.props.label.width;\n  }\n\n  /**\n   * check if group has had an initial height hange\n   * @returns {boolean} \n   */\n  _didMarkerHeightChange() {\n    const markerHeight = this.dom.marker.clientHeight;\n    if (markerHeight != this.lastMarkerHeight) {\n      this.lastMarkerHeight = markerHeight;\n      const redrawQueue = {};\n      let redrawQueueLength = 0;\n\n      util.forEach(this.items, (item, key) => {\n        item.dirty = true;\n        if (item.displayed) {\n          const returnQueue = true;\n          redrawQueue[key] = item.redraw(returnQueue);\n          redrawQueueLength = redrawQueue[key].length;\n        }\n      })\n\n      const needRedraw = redrawQueueLength > 0;\n      if (needRedraw) {\n        // redraw all regular items\n        for (let i = 0; i < redrawQueueLength; i++) {\n          util.forEach(redrawQueue, fns => {\n            fns[i]();\n          });\n        }\n      }\n      return true;\n    } else {\n      return false;\n    }\n  }\n\n  /**\n   * calculate group dimentions and position\n   * @param {number} pixels\n   */\n  _calculateGroupSizeAndPosition() {\n    const { offsetTop, offsetLeft, offsetWidth } = this.dom.foreground;\n    this.top = offsetTop;\n    this.right = offsetLeft;\n    this.width = offsetWidth;\n  }\n\n  /**\n   * checks if should bail redraw of items\n   * @returns {boolean} should bail \n   */\n  _shouldBailItemsRedraw() {\n    const me = this;\n    const timeoutOptions = this.itemSet.options.onTimeout;\n    const bailOptions = {\n      relativeBailingTime: this.itemSet.itemsSettingTime,\n      bailTimeMs: timeoutOptions && timeoutOptions.timeoutMs,\n      userBailFunction: timeoutOptions && timeoutOptions.callback,\n      shouldBailStackItems: this.shouldBailStackItems\n    };\n    let bail = null;\n    if (!this.itemSet.initialDrawDone) {\n      if (bailOptions.shouldBailStackItems) { return true; }\n      if (Math.abs(Date.now() - new Date(bailOptions.relativeBailingTime)) > bailOptions.bailTimeMs) {\n        if (bailOptions.userBailFunction && this.itemSet.userContinueNotBail == null) {\n          bailOptions.userBailFunction(didUserContinue => {\n            me.itemSet.userContinueNotBail = didUserContinue;\n            bail = !didUserContinue;\n          })\n        } else if (me.itemSet.userContinueNotBail == false) {\n          bail = true;\n        } else {\n          bail = false;\n        }\n      }\n    }\n\n    return bail;\n  }\n\n  /**\n   * redraws items\n   * @param {boolean} forceRestack\n   * @param {boolean} lastIsVisible\n   * @param {number} margin\n   * @param {object} range\n   * @private\n   */\n  _redrawItems(forceRestack, lastIsVisible, margin, range) {\n    const restack = forceRestack || this.stackDirty || this.isVisible && !lastIsVisible;\n\n    // if restacking, reposition visible items vertically\n    if (restack) {\n      const orderedItems = {\n        byEnd: this.orderedItems.byEnd.filter(item => !item.isCluster),\n        byStart: this.orderedItems.byStart.filter(item => !item.isCluster)\n      }\n\n      const orderedClusters = {\n        byEnd: [...new Set(this.orderedItems.byEnd.map(item => item.cluster).filter(item => !!item))],\n        byStart: [...new Set(this.orderedItems.byStart.map(item => item.cluster).filter(item => !!item))],\n      }\n\n     /**\n     * Get all visible items in range\n     * @return {array} items\n     */\n      const getVisibleItems = () => {\n        const visibleItems = this._updateItemsInRange(orderedItems, this.visibleItems.filter(item => !item.isCluster), range);\n        const visibleClusters = this._updateClustersInRange(orderedClusters, this.visibleItems.filter(item => item.isCluster), range);\n        return [...visibleItems, ...visibleClusters];\n      }\n\n      /**\n       * Get visible items grouped by subgroup\n       * @param {function} orderFn An optional function to order items inside the subgroups\n       * @return {Object}\n       */\n      const getVisibleItemsGroupedBySubgroup = orderFn => {\n        let visibleSubgroupsItems = {};\n        for (const subgroup in this.subgroups) {\n          const items = this.visibleItems.filter(item => item.data.subgroup === subgroup);\n          visibleSubgroupsItems[subgroup] = orderFn ? items.sort((a, b) => orderFn(a.data, b.data)) : items;\n        }\n        return visibleSubgroupsItems;\n      };\n\n      if (typeof this.itemSet.options.order === 'function') {\n        // a custom order function\n        //show all items\n        const me = this;\n        if (this.doInnerStack && this.itemSet.options.stackSubgroups) {\n          // Order the items within each subgroup\n          const visibleSubgroupsItems = getVisibleItemsGroupedBySubgroup(this.itemSet.options.order);\n          stack.stackSubgroupsWithInnerStack(visibleSubgroupsItems, margin, this.subgroups);\n          this.visibleItems = getVisibleItems();\n          this._updateSubGroupHeights(margin);\n        }\n        else {\n          this.visibleItems = getVisibleItems();\n          this._updateSubGroupHeights(margin);\n          // order all items and force a restacking\n           // order all items outside clusters and force a restacking\n          const customOrderedItems = this.visibleItems\n                                  .slice()\n                                  .filter(item => item.isCluster || (!item.isCluster && !item.cluster))\n                                  .sort((a, b) => {\n                                      return me.itemSet.options.order(a.data, b.data);\n                                  });\n          this.shouldBailStackItems = stack.stack(customOrderedItems, margin, true, this._shouldBailItemsRedraw.bind(this));\n        }\n      } else {\n        // no custom order function, lazy stacking\n        this.visibleItems = getVisibleItems();\n        this._updateSubGroupHeights(margin);\n\n        if (this.itemSet.options.stack) {\n          if (this.doInnerStack && this.itemSet.options.stackSubgroups) {\n            const visibleSubgroupsItems = getVisibleItemsGroupedBySubgroup();\n            stack.stackSubgroupsWithInnerStack(visibleSubgroupsItems, margin, this.subgroups);\n          }\n          else {\n            // TODO: ugly way to access options...\n            this.shouldBailStackItems = stack.stack(this.visibleItems, margin, true, this._shouldBailItemsRedraw.bind(this));\n          }\n        } else {\n          // no stacking\n          stack.nostack(this.visibleItems, margin, this.subgroups, this.itemSet.options.stackSubgroups);\n        }\n      }\n\n      for (let i = 0; i < this.visibleItems.length; i++) {\n        this.visibleItems[i].repositionX();\n        if (this.subgroupVisibility[this.visibleItems[i].data.subgroup] !== undefined) {\n          if (!this.subgroupVisibility[this.visibleItems[i].data.subgroup]) {\n            this.visibleItems[i].hide();\n          }\n        }\n      }\n\n      if (this.itemSet.options.cluster) {\n        util.forEach(this.items, item => {\n          if (item.cluster && item.displayed) {\n            item.hide();\n          }\n        });\n      }\n\n      if (this.shouldBailStackItems) {\n        this.itemSet.body.emitter.emit('destroyTimeline')\n      }\n      this.stackDirty = false;\n    }\n  }\n\n  /**\n   * check if group resized\n   * @param {boolean} resized\n   * @param {number} height\n   * @return {boolean} did resize\n   */\n  _didResize(resized, height) {\n    resized = util.updateProperty(this, 'height', height) || resized;\n    // recalculate size of label\n    const labelWidth = this.dom.inner.clientWidth;\n    const labelHeight = this.dom.inner.clientHeight;\n    resized = util.updateProperty(this.props.label, 'width', labelWidth) || resized;\n    resized = util.updateProperty(this.props.label, 'height', labelHeight) || resized;\n    return resized;\n  }\n\n  /**\n   * apply group height\n   * @param {number} height\n   */\n  _applyGroupHeight(height) {\n    this.dom.background.style.height = `${height}px`;\n    this.dom.foreground.style.height = `${height}px`;\n    this.dom.label.style.height = `${height}px`;\n  }\n\n  /**\n   * update vertical position of items after they are re-stacked and the height of the group is calculated\n   * @param {number} margin\n   */\n  _updateItemsVerticalPosition(margin) {\n    for (let i = 0, ii = this.visibleItems.length; i < ii; i++) {\n      const item = this.visibleItems[i];\n      item.repositionY(margin);\n      if (!this.isVisible && this.groupId != ReservedGroupIds.BACKGROUND) {\n        if (item.displayed) item.hide();\n      }\n    }\n  }\n\n  /**\n   * Repaint this group\n   * @param {{start: number, end: number}} range\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @param {boolean} [forceRestack=false]  Force restacking of all items\n   * @param {boolean} [returnQueue=false]  return the queue or if the group resized\n   * @return {boolean} Returns true if the group is resized or the redraw queue if returnQueue=true\n   */\n  redraw(range, margin, forceRestack, returnQueue) {\n    let resized = false;\n    const lastIsVisible = this.isVisible;\n    let height;\n\n    const queue = [\n      () => {\n        forceRestack = this._didMarkerHeightChange.call(this) || forceRestack;\n      },\n      \n      // recalculate the height of the subgroups\n      this._updateSubGroupHeights.bind(this, margin),\n\n      // calculate actual size and position\n      this._calculateGroupSizeAndPosition.bind(this),\n\n      () => {\n        this.isVisible = this._isGroupVisible.bind(this)(range, margin);\n      },\n      \n      () => {\n        this._redrawItems.bind(this)(forceRestack, lastIsVisible, margin, range)\n      },\n\n      // update subgroups\n      this._updateSubgroupsSizes.bind(this),\n\n      () => {\n        height = this._calculateHeight.bind(this)(margin);\n      },\n\n      // calculate actual size and position again\n      this._calculateGroupSizeAndPosition.bind(this),\n\n      () => {\n        resized = this._didResize.bind(this)(resized, height)\n      },\n\n      () => {\n        this._applyGroupHeight.bind(this)(height)\n      },\n\n      () => {\n        this._updateItemsVerticalPosition.bind(this)(margin)\n      },\n\n      (() => {\n        if (!this.isVisible && this.height) {\n          resized = false;\n        }\n        return resized\n      }).bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * recalculate the height of the subgroups\n   *\n   * @param {{item: timeline.Item}} margin\n   * @private\n   */\n  _updateSubGroupHeights(margin) {\n    if (Object.keys(this.subgroups).length > 0) {\n      const me = this;\n\n      this._resetSubgroups();\n\n      util.forEach(this.visibleItems, item => {\n        if (item.data.subgroup !== undefined) {\n          me.subgroups[item.data.subgroup].height = Math.max(me.subgroups[item.data.subgroup].height, item.height + margin.item.vertical);\n          me.subgroups[item.data.subgroup].visible = typeof this.subgroupVisibility[item.data.subgroup] === 'undefined' ? true : Boolean(this.subgroupVisibility[item.data.subgroup]);\n        }\n      });\n    }\n  }\n\n  /**\n   * check if group is visible\n   *\n   * @param {timeline.Range} range\n   * @param {{axis: timeline.DataAxis}} margin\n   * @returns {boolean} is visible\n   * @private\n   */\n  _isGroupVisible(range, margin) {\n    return (this.top <= range.body.domProps.centerContainer.height - range.body.domProps.scrollTop + margin.axis)\n    && (this.top + this.height + margin.axis >= - range.body.domProps.scrollTop);\n  }\n\n  /**\n   * recalculate the height of the group\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @returns {number} Returns the height\n   * @private\n   */\n  _calculateHeight(margin) {\n    // recalculate the height of the group\n    let height;\n\n    let items;\n\n    if (this.heightMode === 'fixed') {\n      items = util.toArray(this.items);\n    } else {\n      // default or 'auto'\n      items = this.visibleItems;\n    }\n\n    if (items.length > 0) {\n      let min = items[0].top;\n      let max = items[0].top + items[0].height;\n      util.forEach(items, item => {\n        min = Math.min(min, item.top);\n        max = Math.max(max, (item.top + item.height));\n      });\n      if (min > margin.axis) {\n        // there is an empty gap between the lowest item and the axis\n        const offset = min - margin.axis;\n        max -= offset;\n        util.forEach(items, item => {\n          item.top -= offset;\n        });\n      }\n      height = Math.ceil(max + margin.item.vertical / 2);\n      if (this.heightMode !== \"fitItems\") {\n        height = Math.max(height, this.props.label.height);\n      }\n    }\n    else {\n      height = 0 || this.props.label.height;\n    }\n    return height;\n  }\n\n  /**\n   * Show this group: attach to the DOM\n   */\n  show() {\n    if (!this.dom.label.parentNode) {\n      this.itemSet.dom.labelSet.appendChild(this.dom.label);\n    }\n\n    if (!this.dom.foreground.parentNode) {\n      this.itemSet.dom.foreground.appendChild(this.dom.foreground);\n    }\n\n    if (!this.dom.background.parentNode) {\n      this.itemSet.dom.background.appendChild(this.dom.background);\n    }\n\n    if (!this.dom.axis.parentNode) {\n      this.itemSet.dom.axis.appendChild(this.dom.axis);\n    }\n  }\n\n  /**\n   * Hide this group: remove from the DOM\n   */\n  hide() {\n    const label = this.dom.label;\n    if (label.parentNode) {\n      label.parentNode.removeChild(label);\n    }\n\n    const foreground = this.dom.foreground;\n    if (foreground.parentNode) {\n      foreground.parentNode.removeChild(foreground);\n    }\n\n    const background = this.dom.background;\n    if (background.parentNode) {\n      background.parentNode.removeChild(background);\n    }\n\n    const axis = this.dom.axis;\n    if (axis.parentNode) {\n      axis.parentNode.removeChild(axis);\n    }\n  }\n\n  /**\n   * Add an item to the group\n   * @param {Item} item\n   */\n  add(item) {\n    this.items[item.id] = item;\n    item.setParent(this);\n    this.stackDirty = true;\n    // add to\n    if (item.data.subgroup !== undefined) {\n      this._addToSubgroup(item);\n      this.orderSubgroups();\n    }\n\n    if (!this.visibleItems.includes(item)) {\n      const range = this.itemSet.body.range; // TODO: not nice accessing the range like this\n      this._checkIfVisible(item, this.visibleItems, range);\n    }\n  }\n\n  /**\n   * add item to subgroup\n   * @param {object} item\n   * @param {string} subgroupId\n   */\n  _addToSubgroup(item, subgroupId=item.data.subgroup) {\n    if (subgroupId != undefined && this.subgroups[subgroupId] === undefined) {\n      this.subgroups[subgroupId] = {\n        height: 0,\n        top: 0,\n        start: item.data.start,\n        end: item.data.end || item.data.start,\n        visible: false,\n        index: this.subgroupIndex,\n        items: [],\n        stack: this.subgroupStackAll || this.subgroupStack[subgroupId] || false\n      };\n      this.subgroupIndex++;\n    }\n\n\n    if (new Date(item.data.start) < new Date(this.subgroups[subgroupId].start)) {\n      this.subgroups[subgroupId].start = item.data.start;\n    }\n\n    const itemEnd = item.data.end || item.data.start;\n    if (new Date(itemEnd) > new Date(this.subgroups[subgroupId].end)) {\n      this.subgroups[subgroupId].end = itemEnd;\n    }\n\n    this.subgroups[subgroupId].items.push(item);\n  }\n\n  /**\n   * update subgroup sizes\n   */\n  _updateSubgroupsSizes() {\n    const me = this;\n    if (me.subgroups) {\n      for (const subgroup in me.subgroups) {\n        const initialEnd = me.subgroups[subgroup].items[0].data.end || me.subgroups[subgroup].items[0].data.start;\n        let newStart = me.subgroups[subgroup].items[0].data.start;\n        let newEnd = initialEnd - 1;\n\n        me.subgroups[subgroup].items.forEach(item => {\n          if (new Date(item.data.start) < new Date(newStart)) {\n            newStart = item.data.start;\n          }\n\n          const itemEnd = item.data.end || item.data.start;\n          if (new Date(itemEnd) > new Date(newEnd)) {\n            newEnd = itemEnd;\n          }\n        })\n\n        me.subgroups[subgroup].start = newStart;\n        me.subgroups[subgroup].end = new Date(newEnd - 1) // -1 to compensate for colliding end to start subgroups;\n\n      }\n    }\n  }\n\n  /**\n   * order subgroups\n   */\n  orderSubgroups() {\n    if (this.subgroupOrderer !== undefined) {\n      const sortArray = [];\n      if (typeof this.subgroupOrderer == 'string') {\n        for (const subgroup in this.subgroups) {\n          sortArray.push({subgroup, sortField: this.subgroups[subgroup].items[0].data[this.subgroupOrderer]})\n        }\n        sortArray.sort((a, b) => a.sortField - b.sortField)\n      }\n      else if (typeof this.subgroupOrderer == 'function') {\n        for (const subgroup in this.subgroups) {\n          sortArray.push(this.subgroups[subgroup].items[0].data);\n        }\n        sortArray.sort(this.subgroupOrderer);\n      }\n\n      if (sortArray.length > 0) {\n        for (let i = 0; i < sortArray.length; i++) {\n          this.subgroups[sortArray[i].subgroup].index = i;\n        }\n      }\n    }\n  }\n\n  /**\n   * add item to subgroup\n   */\n  _resetSubgroups() {\n    for (const subgroup in this.subgroups) {\n      if (this.subgroups.hasOwnProperty(subgroup)) {\n        this.subgroups[subgroup].visible = false;\n        this.subgroups[subgroup].height = 0;\n      }\n    }\n  }\n\n  /**\n   * Remove an item from the group\n   * @param {Item} item\n   */\n  remove(item) {\n    delete this.items[item.id];\n    item.setParent(null);\n    this.stackDirty = true;\n\n    // remove from visible items\n    const index = this.visibleItems.indexOf(item);\n    if (index != -1) this.visibleItems.splice(index, 1);\n\n    if(item.data.subgroup !== undefined){\n      this._removeFromSubgroup(item);\n      this.orderSubgroups();\n    }\n  }\n\n  /**\n   * remove item from subgroup\n   * @param {object} item\n   * @param {string} subgroupId\n   */\n  _removeFromSubgroup(item, subgroupId=item.data.subgroup) {\n    if (subgroupId != undefined) {\n      const subgroup = this.subgroups[subgroupId];\n      if (subgroup){\n        const itemIndex = subgroup.items.indexOf(item);\n        //  Check the item is actually in this subgroup. How should items not in the group be handled?\n        if (itemIndex >= 0) {\n          subgroup.items.splice(itemIndex,1);\n          if (!subgroup.items.length){\n            delete this.subgroups[subgroupId];\n          } else {\n            this._updateSubgroupsSizes();\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * Remove an item from the corresponding DataSet\n   * @param {Item} item\n   */\n  removeFromDataSet(item) {\n    this.itemSet.removeItem(item.id);\n  }\n\n  /**\n   * Reorder the items\n   */\n  order() {\n    const array = util.toArray(this.items);\n    const startArray = [];\n    const endArray = [];\n\n    for (let i = 0; i < array.length; i++) {\n      if (array[i].data.end !== undefined) {\n        endArray.push(array[i]);\n      }\n      startArray.push(array[i]);\n    }\n    this.orderedItems = {\n      byStart: startArray,\n      byEnd: endArray\n    };\n\n    stack.orderByStart(this.orderedItems.byStart);\n    stack.orderByEnd(this.orderedItems.byEnd);\n  }\n\n  /**\n   * Update the visible items\n   * @param {{byStart: Item[], byEnd: Item[]}} orderedItems   All items ordered by start date and by end date\n   * @param {Item[]} oldVisibleItems                          The previously visible items.\n   * @param {{start: number, end: number}} range              Visible range\n   * @return {Item[]} visibleItems                            The new visible items.\n   * @private\n   */\n  _updateItemsInRange(orderedItems, oldVisibleItems, range) {\n    const visibleItems = [];\n    const visibleItemsLookup = {}; // we keep this to quickly look up if an item already exists in the list without using indexOf on visibleItems\n\n    if (!this.isVisible && this.groupId != ReservedGroupIds.BACKGROUND) {\n      for (let i = 0; i < oldVisibleItems.length; i++) {\n        var item = oldVisibleItems[i];\n        if (item.displayed) item.hide();\n      }\n      return visibleItems;\n    } \n\n    const interval = (range.end - range.start) / 4;\n    const lowerBound = range.start - interval;\n    const upperBound = range.end + interval;\n\n    // this function is used to do the binary search for items having start date only.\n    const startSearchFunction = value => {\n      if      (value < lowerBound)  {return -1;}\n      else if (value <= upperBound) {return  0;}\n      else                          {return  1;}\n    };\n\n    // this function is used to do the binary search for items having start and end dates (range).\n    const endSearchFunction = value => {\n      if      (value < lowerBound)  {return -1;}\n      else                          {return  0;}\n    }\n\n    // first check if the items that were in view previously are still in view.\n    // IMPORTANT: this handles the case for the items with startdate before the window and enddate after the window!\n    // also cleans up invisible items.\n    if (oldVisibleItems.length > 0) {\n      for (let i = 0; i < oldVisibleItems.length; i++) {\n        this._checkIfVisibleWithReference(oldVisibleItems[i], visibleItems, visibleItemsLookup, range);\n      }\n    }\n\n    // we do a binary search for the items that have only start values.\n    const initialPosByStart = util.binarySearchCustom(orderedItems.byStart, startSearchFunction, 'data','start');\n\n    // trace the visible items from the inital start pos both ways until an invisible item is found, we only look at the start values.\n    this._traceVisible(initialPosByStart, orderedItems.byStart, visibleItems, visibleItemsLookup, item => item.data.start < lowerBound || item.data.start > upperBound);\n\n    // if the window has changed programmatically without overlapping the old window, the ranged items with start < lowerBound and end > upperbound are not shown.\n    // We therefore have to brute force check all items in the byEnd list\n    if (this.checkRangedItems == true) {\n      this.checkRangedItems = false;\n      for (let i = 0; i < orderedItems.byEnd.length; i++) {\n        this._checkIfVisibleWithReference(orderedItems.byEnd[i], visibleItems, visibleItemsLookup, range);\n      }\n    }\n    else {\n      // we do a binary search for the items that have defined end times.\n      const initialPosByEnd = util.binarySearchCustom(orderedItems.byEnd, endSearchFunction, 'data','end');\n\n      // trace the visible items from the inital start pos both ways until an invisible item is found, we only look at the end values.\n      this._traceVisible(initialPosByEnd, orderedItems.byEnd, visibleItems, visibleItemsLookup, item => item.data.end < lowerBound || item.data.start > upperBound);\n    }\n\n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n\n    for (let i = 0; i < visibleItems.length; i++) {\n      const item = visibleItems[i];\n      if (!item.displayed) {\n        const returnQueue = true;\n        redrawQueue[i] = item.redraw(returnQueue);\n        redrawQueueLength = redrawQueue[i].length;\n      }\n    }\n\n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      // redraw all regular items\n      for (let j = 0; j < redrawQueueLength; j++) {\n        util.forEach(redrawQueue, fns => {\n          fns[j]();\n        });\n      }\n    }\n\n    for (let i = 0; i < visibleItems.length; i++) {\n      visibleItems[i].repositionX();\n    }\n\n    return visibleItems;\n  }\n\n  /**\n   * trace visible items in group\n   * @param {number} initialPos\n   * @param {array} items\n   * @param {aray} visibleItems\n   * @param {object} visibleItemsLookup\n   * @param {function} breakCondition\n   */\n  _traceVisible(initialPos, items, visibleItems, visibleItemsLookup, breakCondition) {\n    if (initialPos != -1) {\n      for (let i = initialPos; i >= 0; i--) {\n        let item = items[i];\n        if (breakCondition(item)) {\n          break;\n        }\n        else {\n          if (!(item.isCluster  && !item.hasItems()) && !item.cluster) {\n            if (visibleItemsLookup[item.id] === undefined) {\n              visibleItemsLookup[item.id] = true;\n              visibleItems.push(item);\n            }\n          }\n        }\n      }\n\n      for (let i = initialPos + 1; i < items.length; i++) {\n        let item = items[i];\n        if (breakCondition(item)) {\n          break;\n        }\n        else {\n          if (!(item.isCluster && !item.hasItems()) && !item.cluster) {\n            if (visibleItemsLookup[item.id] === undefined) {\n              visibleItemsLookup[item.id] = true;\n              visibleItems.push(item);\n            }\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * this function is very similar to the _checkIfInvisible() but it does not\n   * return booleans, hides the item if it should not be seen and always adds to\n   * the visibleItems.\n   * this one is for brute forcing and hiding.\n   *\n   * @param {Item} item\n   * @param {Array} visibleItems\n   * @param {{start:number, end:number}} range\n   * @private\n   */\n  _checkIfVisible(item, visibleItems, range) {\n      if (item.isVisible(range)) {\n        if (!item.displayed) item.show();\n        // reposition item horizontally\n        item.repositionX();\n        visibleItems.push(item);\n      }\n      else {\n        if (item.displayed) item.hide();\n      }\n  }\n\n  /**\n   * this function is very similar to the _checkIfInvisible() but it does not\n   * return booleans, hides the item if it should not be seen and always adds to\n   * the visibleItems.\n   * this one is for brute forcing and hiding.\n   *\n   * @param {Item} item\n   * @param {Array.<timeline.Item>} visibleItems\n   * @param {Object<number, boolean>} visibleItemsLookup\n   * @param {{start:number, end:number}} range\n   * @private\n   */\n  _checkIfVisibleWithReference(item, visibleItems, visibleItemsLookup, range) {\n    if (item.isVisible(range)) {\n      if (visibleItemsLookup[item.id] === undefined) {\n        visibleItemsLookup[item.id] = true;\n        visibleItems.push(item);\n      }\n    }\n    else {\n      if (item.displayed) item.hide();\n    }\n  }\n\n  /**\n   * Update the visible items\n   * @param {array} orderedClusters \n   * @param {array} oldVisibleClusters                         \n   * @param {{start: number, end: number}} range             \n   * @return {Item[]} visibleItems                            \n   * @private\n   */\n  _updateClustersInRange(orderedClusters, oldVisibleClusters, range) {\n    // Clusters can overlap each other so we cannot use binary search here\n    const visibleClusters = [];\n    const visibleClustersLookup = {}; // we keep this to quickly look up if an item already exists in the list without using indexOf on visibleItems\n  \n    if (oldVisibleClusters.length > 0) {\n      for (let i = 0; i < oldVisibleClusters.length; i++) {\n        this._checkIfVisibleWithReference(oldVisibleClusters[i], visibleClusters, visibleClustersLookup, range);\n      }\n    }\n  \n    for (let i = 0; i < orderedClusters.byStart.length; i++) {\n      this._checkIfVisibleWithReference(orderedClusters.byStart[i], visibleClusters, visibleClustersLookup, range);\n    }\n  \n    for (let i = 0; i < orderedClusters.byEnd.length; i++) {\n      this._checkIfVisibleWithReference(orderedClusters.byEnd[i], visibleClusters, visibleClustersLookup, range);\n    }\n  \n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n  \n    for (let i = 0; i < visibleClusters.length; i++) {\n      const item = visibleClusters[i];\n      if (!item.displayed) {\n        const returnQueue = true;\n        redrawQueue[i] = item.redraw(returnQueue);\n        redrawQueueLength = redrawQueue[i].length;\n      }\n    }\n  \n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      // redraw all regular items\n      for (var j = 0; j < redrawQueueLength; j++) {\n        util.forEach(redrawQueue, function (fns) {\n          fns[j]();\n        });\n      }\n    }\n  \n    for (let i = 0; i < visibleClusters.length; i++) {\n      visibleClusters[i].repositionX();\n    }\n    \n    return visibleClusters;\n  }\n\n  /**\n   * change item subgroup\n   * @param {object} item\n   * @param {string} oldSubgroup\n   * @param {string} newSubgroup\n   */\n  changeSubgroup(item, oldSubgroup, newSubgroup) {\n    this._removeFromSubgroup(item, oldSubgroup);\n    this._addToSubgroup(item, newSubgroup);\n    this.orderSubgroups();\n  }\n\n  /**\n   * Call this method before you lose the last reference to an instance of this.\n   * It will remove listeners etc.\n   */\n  dispose() {\n    this.hide();\n\n    let disposeCallback;\n    while ((disposeCallback = this._disposeCallbacks.pop())) {\n      disposeCallback();\n    }\n  }\n}\n\nexport default Group;\n","import Group from './Group'\n\n/**\n * @constructor BackgroundGroup\n * @extends Group\n */\nclass BackgroundGroup extends Group {\n  /**\n * @param {number | string} groupId\n * @param {Object} data\n * @param {ItemSet} itemSet\n */\n  constructor(groupId, data, itemSet) {\n    super(groupId, data, itemSet)\n    // Group.call(this, groupId, data, itemSet);\n\n    this.width = 0;\n    this.height = 0;\n    this.top = 0;\n    this.left = 0;\n  }\n\n  /**\n   * Repaint this group\n   * @param {{start: number, end: number}} range\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @param {boolean} [forceRestack=false]  Force restacking of all items\n   * @return {boolean} Returns true if the group is resized\n   */\n  redraw(range, margin, forceRestack) {  // eslint-disable-line no-unused-vars\n    const resized = false;\n\n    this.visibleItems = this._updateItemsInRange(this.orderedItems, this.visibleItems, range);\n\n    // calculate actual size\n    this.width = this.dom.background.offsetWidth;\n\n    // apply new height (just always zero for BackgroundGroup\n    this.dom.background.style.height  = '0';\n\n    // update vertical position of items after they are re-stacked and the height of the group is calculated\n    for (let i = 0, ii = this.visibleItems.length; i < ii; i++) {\n      const item = this.visibleItems[i];\n      item.repositionY(margin);\n    }\n\n    return resized;\n  }\n\n  /**\n   * Show this group: attach to the DOM\n   */\n  show() {\n    if (!this.dom.background.parentNode) {\n      this.itemSet.dom.background.appendChild(this.dom.background);\n    }\n  }\n}\n\nexport default BackgroundGroup;","import Hammer from '../../../module/hammer';\nimport util from '../../../util';\nimport moment from '../../../module/moment';\nimport locales from '../../locales';\n\nimport '../css/item.css';\n\n/**\n * Item\n */\nclass Item {\n  /**\n * @constructor Item\n * @param {Object} data             Object containing (optional) parameters type,\n *                                  start, end, content, group, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} options          Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    this.id = null;\n    this.parent = null;\n    this.data = data;\n    this.dom = null;\n    this.conversion = conversion || {};\n    this.defaultOptions = {\n      locales,\n      locale: 'en'\n    };\n    this.options = util.extend({}, this.defaultOptions, options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n    this.selected = false;\n    this.displayed = false;\n    this.groupShowing = true;\n    this.selectable = (options && options.selectable) || false;\n    this.dirty = true;\n\n    this.top = null;\n    this.right = null;\n    this.left = null;\n    this.width = null;\n    this.height = null;\n\n    this.setSelectability(data);\n\n    this.editable = null;\n    this._updateEditStatus();\n  }\n\n  /**\n   * Select current item\n   */\n  select() {\n    if (this.selectable) {\n      this.selected = true;\n      this.dirty = true;\n      if (this.displayed) this.redraw();\n    }\n  }\n\n  /**\n   * Unselect current item\n   */\n  unselect() {\n    this.selected = false;\n    this.dirty = true;\n    if (this.displayed) this.redraw();\n  }\n\n  /**\n   * Set data for the item. Existing data will be updated. The id should not\n   * be changed. When the item is displayed, it will be redrawn immediately.\n   * @param {Object} data\n   */\n  setData(data) {\n    const groupChanged = data.group != undefined && this.data.group != data.group;\n    if (groupChanged && this.parent != null) {\n      this.parent.itemSet._moveToGroup(this, data.group);\n    }\n\n    this.setSelectability(data);\n\n    if (this.parent) {\n      this.parent.stackDirty = true;\n    }\n    \n    const subGroupChanged = data.subgroup != undefined && this.data.subgroup != data.subgroup;\n    if (subGroupChanged && this.parent != null) {\n      this.parent.changeSubgroup(this, this.data.subgroup, data.subgroup);\n    }\n\n    this.data = data;\n    this._updateEditStatus();\n    this.dirty = true;\n    if (this.displayed) this.redraw();\n  }\n\n  /**\n   * Set whether the item can be selected.\n   * Can only be set/unset if the timeline's `selectable` configuration option is `true`.\n   * @param {Object} data `data` from `constructor` and `setData`\n   */\n  setSelectability(data) {\n    if (data) {\n      this.selectable = typeof data.selectable === 'undefined' ? true : Boolean(data.selectable);\n    }\n  }\n\n  /**\n   * Set a parent for the item\n   * @param {Group} parent\n   */\n  setParent(parent) {\n    if (this.displayed) {\n      this.hide();\n      this.parent = parent;\n      if (this.parent) {\n        this.show();\n      }\n    }\n    else {\n      this.parent = parent;\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {  // eslint-disable-line no-unused-vars\n    return false;\n  }\n\n  /**\n   * Show the Item in the DOM (when not already visible)\n   * @return {Boolean} changed\n   */\n  show() {\n    return false;\n  }\n\n  /**\n   * Hide the Item from the DOM (when visible)\n   * @return {Boolean} changed\n   */\n  hide() {\n    return false;\n  }\n\n  /**\n   * Repaint the item\n   */\n  redraw() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Reposition the Item horizontally\n   */\n  repositionX() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Reposition the Item vertically\n   */\n  repositionY() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Repaint a drag area on the center of the item when the item is selected\n   * @protected\n   */\n  _repaintDragCenter() {\n    if (this.selected && this.editable.updateTime && !this.dom.dragCenter) {\n      const me = this;\n      // create and show drag area\n      const dragCenter = document.createElement('div');\n      dragCenter.className = 'vis-drag-center';\n      dragCenter.dragCenterItem = this;\n      this.hammerDragCenter = new Hammer(dragCenter);\n\n      this.hammerDragCenter.on('tap', event => {\n        me.parent.itemSet.body.emitter.emit('click',  {\n          event,\n          item: me.id\n        });\n      });\n      this.hammerDragCenter.on('doubletap', event => {\n        event.stopPropagation();\n        me.parent.itemSet._onUpdateItem(me);\n        me.parent.itemSet.body.emitter.emit('doubleClick', {\n          event,\n          item: me.id\n        });\n      });\n      this.hammerDragCenter.on('panstart', event => {\n        // do not allow this event to propagate to the Range\n        event.stopPropagation();\n        me.parent.itemSet._onDragStart(event);\n      })\n      this.hammerDragCenter.on('panmove',  me.parent.itemSet._onDrag.bind(me.parent.itemSet));\n      this.hammerDragCenter.on('panend',   me.parent.itemSet._onDragEnd.bind(me.parent.itemSet));\n\n      if (this.dom.box) {\n        if (this.dom.dragLeft) {\n          this.dom.box.insertBefore(dragCenter, this.dom.dragLeft);\n        }\n        else {\n          this.dom.box.appendChild(dragCenter);\n        }\n      }\n      else if (this.dom.point) {\n        this.dom.point.appendChild(dragCenter);\n      }\n      \n      this.dom.dragCenter = dragCenter;\n    }\n    else if (!this.selected && this.dom.dragCenter) {\n      // delete drag area\n      if (this.dom.dragCenter.parentNode) {\n        this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter);\n      }\n      this.dom.dragCenter = null;\n      \n      if (this.hammerDragCenter) {\n        this.hammerDragCenter.destroy();\n        this.hammerDragCenter = null;\n      }\n    }\n  }\n\n  /**\n   * Repaint a delete button on the top right of the item when the item is selected\n   * @param {HTMLElement} anchor\n   * @protected\n   */\n  _repaintDeleteButton(anchor) {\n    const editable = ((this.options.editable.overrideItems || this.editable == null) && this.options.editable.remove) ||\n                   (!this.options.editable.overrideItems && this.editable != null && this.editable.remove);\n\n    if (this.selected && editable && !this.dom.deleteButton) {\n      // create and show button\n      const me = this;\n\n      const deleteButton = document.createElement('div');\n\n      if (this.options.rtl) {\n        deleteButton.className = 'vis-delete-rtl';\n      } else {\n        deleteButton.className = 'vis-delete';\n      }\n      let optionsLocale = this.options.locales[this.options.locale];\n      if (!optionsLocale) {\n        if (!this.warned) {\n          console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n          this.warned = true;\n        }\n        optionsLocale = this.options.locales['en']; // fall back on english when not available\n      }\n      deleteButton.title = optionsLocale.deleteSelected;\n\n      // TODO: be able to destroy the delete button\n      this.hammerDeleteButton = new Hammer(deleteButton).on('tap', event => {\n        event.stopPropagation();\n        me.parent.removeFromDataSet(me);\n      });\n\n      anchor.appendChild(deleteButton);\n      this.dom.deleteButton = deleteButton;\n    }\n    else if (!this.selected && this.dom.deleteButton) {\n      // remove button\n      if (this.dom.deleteButton.parentNode) {\n        this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton);\n      }\n      this.dom.deleteButton = null;\n\n      if (this.hammerDeleteButton) {\n        this.hammerDeleteButton.destroy();\n        this.hammerDeleteButton = null;\n      }\n    }\n  }\n\n  /**\n   * Repaint a onChange tooltip on the top right of the item when the item is selected\n   * @param {HTMLElement} anchor\n   * @protected\n   */\n  _repaintOnItemUpdateTimeTooltip(anchor) {\n    if (!this.options.tooltipOnItemUpdateTime) return;\n\n    const editable = (this.options.editable.updateTime || \n                    this.data.editable === true) &&\n                   this.data.editable !== false;\n\n    if (this.selected && editable && !this.dom.onItemUpdateTimeTooltip) {\n      const onItemUpdateTimeTooltip = document.createElement('div');\n\n      onItemUpdateTimeTooltip.className = 'vis-onUpdateTime-tooltip';\n      anchor.appendChild(onItemUpdateTimeTooltip);\n      this.dom.onItemUpdateTimeTooltip = onItemUpdateTimeTooltip;\n\n    } else if (!this.selected && this.dom.onItemUpdateTimeTooltip) {\n      // remove button\n      if (this.dom.onItemUpdateTimeTooltip.parentNode) {\n        this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip);\n      }\n      this.dom.onItemUpdateTimeTooltip = null;\n    }\n\n    // position onChange tooltip\n    if (this.dom.onItemUpdateTimeTooltip) {\n\n      // only show when editing\n      this.dom.onItemUpdateTimeTooltip.style.visibility = this.parent.itemSet.touchParams.itemIsDragging ? 'visible' : 'hidden';\n      \n      // position relative to item's content\n      this.dom.onItemUpdateTimeTooltip.style.transform = 'translateX(-50%)';\n      this.dom.onItemUpdateTimeTooltip.style.left = '50%';\n\n      // position above or below the item depending on the item's position in the window\n      const tooltipOffset = 50; // TODO: should be tooltip height (depends on template)\n      const scrollTop = this.parent.itemSet.body.domProps.scrollTop;\n\n        // TODO: this.top for orientation:true is actually the items distance from the bottom... \n        // (should be this.bottom)\n      let itemDistanceFromTop; \n      if (this.options.orientation.item == 'top') {\n        itemDistanceFromTop = this.top;\n      } else {\n        itemDistanceFromTop = (this.parent.height - this.top - this.height)\n      }\n      const isCloseToTop = itemDistanceFromTop + this.parent.top - tooltipOffset < -scrollTop;\n\n      if (isCloseToTop) {\n        this.dom.onItemUpdateTimeTooltip.style.bottom = \"\";\n        this.dom.onItemUpdateTimeTooltip.style.top = `${this.height + 2}px`;\n      } else {\n        this.dom.onItemUpdateTimeTooltip.style.top = \"\";\n        this.dom.onItemUpdateTimeTooltip.style.bottom = `${this.height + 2}px`;\n      }\n      \n      // handle tooltip content\n      let content;\n      let templateFunction;\n\n      if (this.options.tooltipOnItemUpdateTime && this.options.tooltipOnItemUpdateTime.template) {\n        templateFunction = this.options.tooltipOnItemUpdateTime.template.bind(this);\n        content = templateFunction(this.data);\n      } else {\n        content = `start: ${moment(this.data.start).format('MM/DD/YYYY hh:mm')}`;\n        if (this.data.end) { \n          content += `<br> end: ${moment(this.data.end).format('MM/DD/YYYY hh:mm')}`;\n        }\n      }\n      this.dom.onItemUpdateTimeTooltip.innerHTML = content;\n    }\n  }\n\n   /**\n   * get item data\n   * @return {object}\n   * @private\n   */\n  _getItemData() {\n    return this.parent.itemSet.itemsData.get(this.id);\n  }\n\n  /**\n   * Set HTML contents for the item\n   * @param {Element} element   HTML element to fill with the contents\n   * @private\n   */\n  _updateContents(element) {\n    let content;\n    let changed;\n    let templateFunction;\n    let itemVisibleFrameContent;\n    let visibleFrameTemplateFunction; \n    const itemData = this._getItemData(); // get a clone of the data from the dataset\n\n    const frameElement = this.dom.box || this.dom.point;\n    const itemVisibleFrameContentElement = frameElement.getElementsByClassName('vis-item-visible-frame')[0];\n\n    if (this.options.visibleFrameTemplate) {\n      visibleFrameTemplateFunction = this.options.visibleFrameTemplate.bind(this);\n      itemVisibleFrameContent = visibleFrameTemplateFunction(itemData, itemVisibleFrameContentElement);\n    } else {\n      itemVisibleFrameContent = '';\n    }\n    \n    if (itemVisibleFrameContentElement) {\n      if ((itemVisibleFrameContent instanceof Object) && !(itemVisibleFrameContent instanceof Element)) {\n        visibleFrameTemplateFunction(itemData, itemVisibleFrameContentElement)\n      } else {\n         changed = this._contentToString(this.itemVisibleFrameContent) !== this._contentToString(itemVisibleFrameContent);\n         if (changed) {\n          // only replace the content when changed\n          if (itemVisibleFrameContent instanceof Element) {\n            itemVisibleFrameContentElement.innerHTML = '';\n            itemVisibleFrameContentElement.appendChild(itemVisibleFrameContent);\n          }\n          else if (itemVisibleFrameContent != undefined) {\n            itemVisibleFrameContentElement.innerHTML = itemVisibleFrameContent;\n          }\n          else {\n            if (!(this.data.type == 'background' && this.data.content === undefined)) {\n              throw new Error(`Property \"content\" missing in item ${this.id}`);\n            }\n          }\n\n          this.itemVisibleFrameContent = itemVisibleFrameContent;\n         }\n      }\n    }\n\n    if (this.options.template) {\n      templateFunction = this.options.template.bind(this);\n      content = templateFunction(itemData, element, this.data);\n    } else {\n      content = this.data.content;\n    }\n\n    if ((content instanceof Object) && !(content instanceof Element)) {\n      templateFunction(itemData, element)\n    } else {\n      changed = this._contentToString(this.content) !== this._contentToString(content);\n      if (changed) {\n        // only replace the content when changed\n        if (content instanceof Element) {\n          element.innerHTML = '';\n          element.appendChild(content);\n        }\n        else if (content != undefined) {\n          element.innerHTML = content;\n        }\n        else {\n          if (!(this.data.type == 'background' && this.data.content === undefined)) {\n            throw new Error(`Property \"content\" missing in item ${this.id}`);\n          }\n        }\n        this.content = content;\n      }\n    }\n  }\n\n  /**\n   * Process dataAttributes timeline option and set as data- attributes on dom.content\n   * @param {Element} element   HTML element to which the attributes will be attached\n   * @private\n   */\n  _updateDataAttributes(element) {\n   if (this.options.dataAttributes && this.options.dataAttributes.length > 0) {\n     let attributes = [];\n\n     if (Array.isArray(this.options.dataAttributes)) {\n       attributes = this.options.dataAttributes;\n     }\n     else if (this.options.dataAttributes == 'all') {\n       attributes = Object.keys(this.data);\n     }\n     else {\n       return;\n     }\n\n     for (const name of attributes) {\n       const value = this.data[name];\n\n       if (value != null) {\n         element.setAttribute(`data-${name}`, value);\n       }\n       else {\n         element.removeAttribute(`data-${name}`);\n       }\n     }\n   }\n }\n\n  /**\n   * Update custom styles of the element\n   * @param {Element} element\n   * @private\n   */\n  _updateStyle(element) {\n    // remove old styles\n    if (this.style) {\n      util.removeCssText(element, this.style);\n      this.style = null;\n    }\n\n    // append new styles\n    if (this.data.style) {\n      util.addCssText(element, this.data.style);\n      this.style = this.data.style;\n    }\n  }\n\n  /**\n   * Stringify the items contents\n   * @param {string | Element | undefined} content\n   * @returns {string | undefined}\n   * @private\n   */\n  _contentToString(content) {\n    if (typeof content === 'string') return content;\n    if (content && 'outerHTML' in content) return content.outerHTML;\n    return content;\n  }\n\n  /**\n   * Update the editability of this item.\n   */\n  _updateEditStatus() {\n    if (this.options) {\n      if(typeof this.options.editable === 'boolean') {\n        this.editable = {\n          updateTime: this.options.editable,\n          updateGroup: this.options.editable,\n          remove: this.options.editable\n        };\n      } else if(typeof this.options.editable === 'object') {\n          this.editable = {};\n          util.selectiveExtend(['updateTime', 'updateGroup', 'remove'], this.editable, this.options.editable);\n      }\n    }\n    // Item data overrides, except if options.editable.overrideItems is set.\n    if (!this.options || !(this.options.editable) || (this.options.editable.overrideItems !== true)) {\n      if (this.data) {\n        if (typeof this.data.editable === 'boolean') {\n          this.editable = {\n            updateTime: this.data.editable,\n            updateGroup: this.data.editable,\n            remove: this.data.editable\n          }\n        } else if (typeof this.data.editable === 'object') {\n          // TODO: in timeline.js 5.0, we should change this to not reset options from the timeline configuration.\n          // Basically just remove the next line...\n          this.editable = {};\n          util.selectiveExtend(['updateTime', 'updateGroup', 'remove'], this.editable, this.data.editable);\n        }\n      }\n    }\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return 0;\n  }\n\n  /**\n   * Return the width of the item right from the max of its start and end date\n   * @return {number}\n   */\n  getWidthRight() {\n    return 0;\n  }\n\n  /**\n   * Return the title of the item\n   * @return {string | undefined}\n   */\n  getTitle() {\n    if (this.options.tooltip && this.options.tooltip.template) {\n      const templateFunction = this.options.tooltip.template.bind(this);\n      return templateFunction(this._getItemData(), this.data);\n    }\n\n    return this.data.title;\n  }\n}\n\nItem.prototype.stack = true;\n\nexport default Item;\n","import Item from './Item';\n\n/**\n * @constructor BoxItem\n * @extends Item\n */\nclass BoxItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      dot: {\n        width: 0,\n        height: 0\n      },\n      line: {\n        width: 0,\n        height: 0\n      }\n    };\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {{start: number, end: number}} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    let isVisible;\n    const align = this.data.align || this.options.align;\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n\n    if (align == 'right') {\n      isVisible = (this.data.start.getTime() > range.start) && (this.data.start.getTime() - widthInMs < range.end);\n    }\n    else if (align == 'left') {\n      isVisible = (this.data.start.getTime() + widthInMs > range.start) && (this.data.start.getTime() < range.end);\n    }\n    else {\n      // default or 'center'\n      isVisible = (this.data.start.getTime() + widthInMs / 2 > range.start ) && (this.data.start.getTime() - widthInMs/2 < range.end);\n    }\n    return isVisible;\n  }\n\n   /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // create main box\n      this.dom.box = document.createElement('DIV');\n\n      // contents box (inside the background box). used for making margins\n      this.dom.content = document.createElement('DIV');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.box.appendChild(this.dom.content);\n\n      // line to axis\n      this.dom.line = document.createElement('DIV');\n      this.dom.line.className = 'vis-line';\n\n      // dot on axis\n      this.dom.dot = document.createElement('DIV');\n      this.dom.dot.className = 'vis-dot';\n\n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) throw new Error('Cannot redraw item: parent has no foreground container element');\n      foreground.appendChild(this.dom.box);\n    }\n    if (!this.dom.line.parentNode) {\n      var background = this.parent.dom.background;\n      if (!background) throw new Error('Cannot redraw item: parent has no background container element');\n      background.appendChild(this.dom.line);\n    }\n    if (!this.dom.dot.parentNode) {\n      const axis = this.parent.dom.axis;\n      if (!background) throw new Error('Cannot redraw item: parent has no axis container element');\n      axis.appendChild(this.dom.dot);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM element\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n\n      // update class\n      const className = (this.data.className ? ' ' + this.data.className : '') +\n        (this.selected ? ' vis-selected' : '') +\n        (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.box.className = `vis-item vis-box${className}`;\n      this.dom.line.className = `vis-item vis-line${className}`;\n      this.dom.dot.className = `vis-item vis-dot${className}`;\n    }\n  }\n\n  /**\n   * get DOM components sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    return {\n      previous: {\n        right: this.dom.box.style.right,\n        left: this.dom.box.style.left\n      },\n      dot: {\n        height: this.dom.dot.offsetHeight,\n        width: this.dom.dot.offsetWidth\n      },\n      line: {\n        width: this.dom.line.offsetWidth\n      },\n      box: {\n        width: this.dom.box.offsetWidth,\n        height: this.dom.box.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    if (this.options.rtl) {\n      this.dom.box.style.right = \"0px\";\n    } else {\n      this.dom.box.style.left = \"0px\";\n    }\n\n    // recalculate size\n    this.props.dot.height = sizes.dot.height;\n    this.props.dot.width = sizes.dot.width;\n    this.props.line.width = sizes.line.width;\n    this.width = sizes.box.width;\n    this.height = sizes.box.height;\n\n    // restore previous position\n    if (this.options.rtl) {\n      this.dom.box.style.right = sizes.previous.right;\n    } else {\n      this.dom.box.style.left = sizes.previous.left;\n    }\n\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n    this._repaintDragCenter();\n    this._repaintDeleteButton(this.dom.box);\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      const dom = this.dom;\n\n      if (dom.box.remove) dom.box.remove();\n      else if (dom.box.parentNode) dom.box.parentNode.removeChild(dom.box); // IE11\n\n      if (dom.line.remove) dom.line.remove();\n      else if (dom.line.parentNode) dom.line.parentNode.removeChild(dom.line); // IE11\n      \n      if (dom.dot.remove) dom.dot.remove();\n      else if (dom.dot.parentNode) dom.dot.parentNode.removeChild(dom.dot); // IE11\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item XY\n   */\n  repositionXY() {\n    const rtl = this.options.rtl;\n\n    const repositionXY = (element, x, y, rtl = false) => {\n      if (x === undefined && y === undefined) return;\n      // If rtl invert the number.\n      const directionX = rtl ? (x * -1) : x;\n\n      //no y. translate x\n      if (y === undefined) {\n        element.style.transform = `translateX(${directionX}px)`;\n        return;\n      }\n\n      //no x. translate y\n      if (x === undefined) {\n        element.style.transform = `translateY(${y}px)`;\n        return;\n      }\n\n      element.style.transform = `translate(${directionX}px, ${y}px)`;\n    }\n    repositionXY(this.dom.box, this.boxX, this.boxY, rtl);\n    repositionXY(this.dom.dot, this.dotX, this.dotY, rtl);\n    repositionXY(this.dom.line, this.lineX, this.lineY, rtl);\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @Override\n   */\n  repositionX() {\n    const start = this.conversion.toScreen(this.data.start);\n    const align = this.data.align === undefined ? this.options.align : this.data.align\n    const lineWidth = this.props.line.width;\n    const dotWidth = this.props.dot.width;\n    \n    if (align == 'right') {\n      // calculate right position of the box\n      this.boxX = start - this.width;\n      this.lineX = start - lineWidth;\n      this.dotX = start - lineWidth / 2 - dotWidth / 2;\n    }\n    else if (align == 'left') {\n      // calculate left position of the box\n      this.boxX = start;\n      this.lineX = start;\n      this.dotX = start + lineWidth / 2 - dotWidth / 2;\n    }\n    else {\n      // default or 'center'\n      this.boxX = start - this.width / 2;\n      this.lineX = this.options.rtl ? start - lineWidth : start - lineWidth / 2;\n      this.dotX = start - dotWidth / 2;\n    }\n\n    if (this.options.rtl)\n      this.right = this.boxX;\n    else\n      this.left = this.boxX;\n\n    this.repositionXY();\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    const lineStyle = this.dom.line.style;\n\n    if (orientation == 'top') {\n      const lineHeight = this.parent.top + this.top + 1;\n\n      this.boxY = this.top || 0;\n      lineStyle.height = `${lineHeight}px`;\n      lineStyle.bottom = '';\n      lineStyle.top = '0';\n    }\n    else { // orientation 'bottom'\n      const itemSetHeight = this.parent.itemSet.props.height; // TODO: this is nasty\n      const lineHeight = itemSetHeight - this.parent.top - this.parent.height + this.top;\n\n      this.boxY = this.parent.height - this.top - (this.height || 0);\n      lineStyle.height = `${lineHeight}px`;\n      lineStyle.top = '';\n      lineStyle.bottom = '0';\n    }\n\n    this.dotY = -this.props.dot.height / 2;\n\n    this.repositionXY();\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.width / 2;\n  }\n\n  /**\n   * Return the width of the item right from its start date\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.width / 2;\n  }\n}\n\nexport default BoxItem;\n","import Item from './Item';\n\n/**\n * @constructor PointItem\n * @extends Item\n */\nclass PointItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      dot: {\n        top: 0,\n        width: 0,\n        height: 0\n      },\n      content: {\n        height: 0,\n        marginLeft: 0,\n        marginRight: 0\n      }\n    };\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {{start: number, end: number}} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n    \n    return (this.data.start.getTime() + widthInMs > range.start ) && (this.data.start < range.end);\n  }\n\n  /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // background box\n      this.dom.point = document.createElement('div');\n      // className is updated in redraw()\n\n      // contents box, right from the dot\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.point.appendChild(this.dom.content);\n\n      // dot at start\n      this.dom.dot = document.createElement('div');\n      this.dom.point.appendChild(this.dom.dot);\n\n      // attach this item as attribute\n      this.dom.point['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.point.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n      foreground.appendChild(this.dom.point);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.point);\n      this._updateStyle(this.dom.point);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n      // update class\n      const className = (this.data.className ? ' ' + this.data.className : '') +\n          (this.selected ? ' vis-selected' : '') +\n          (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.point.className  = `vis-item vis-point${className}`;\n      this.dom.dot.className  = `vis-item vis-dot${className}`;\n    }\n  }\n\n  /**\n   * get DOM component sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    return {\n      dot:  {\n        width: this.dom.dot.offsetWidth,\n        height: this.dom.dot.offsetHeight\n      },\n      content: {\n        width: this.dom.content.offsetWidth,\n        height: this.dom.content.offsetHeight\n      },\n      point: {\n        width: this.dom.point.offsetWidth,\n        height: this.dom.point.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {array} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    // recalculate size of dot and contents\n    this.props.dot.width = sizes.dot.width;\n    this.props.dot.height = sizes.dot.height;\n    this.props.content.height = sizes.content.height;\n\n    // resize contents\n    if (this.options.rtl) {\n      this.dom.content.style.marginRight = `${2 * this.props.dot.width}px`;\n    } else {\n      this.dom.content.style.marginLeft = `${2 * this.props.dot.width}px`;\n    }\n    //this.dom.content.style.marginRight = ... + 'px'; // TODO: margin right\n\n    // recalculate size\n    this.width = sizes.point.width;\n    this.height = sizes.point.height;\n\n    // reposition the dot\n    this.dom.dot.style.top = `${(this.height - this.props.dot.height) / 2}px`;\n    \n    const dotWidth = this.props.dot.width;\n    const translateX = this.options.rtl ? (dotWidth / 2) * -1 : dotWidth / 2;\n    this.dom.dot.style.transform = `translateX(${translateX}px`;\n    this.dirty = false;\n  }\n\n  /**\n   * Repain DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.point);\n    this._repaintDragCenter();\n    this._repaintDeleteButton(this.dom.point);\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  \n  /**\n   * Reposition XY\n   */\n  repositionXY() {\n    const rtl = this.options.rtl;\n    \n    const repositionXY = (element, x, y, rtl = false) => {\n      if (x === undefined && y === undefined) return;\n      // If rtl invert the number.\n      const directionX = rtl ? (x * -1) : x;\n\n      //no y. translate x\n      if (y === undefined) {\n        element.style.transform = `translateX(${directionX}px)`;\n        return;\n      }\n\n      //no x. translate y\n      if (x === undefined) {\n        element.style.transform = `translateY(${y}px)`;\n        return;\n      }\n\n      element.style.transform = `translate(${directionX}px, ${y}px)`;\n    }\n    repositionXY(this.dom.point, this.pointX, this.pointY, rtl);\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      if (this.dom.point.parentNode) {\n        this.dom.point.parentNode.removeChild(this.dom.point);\n      }\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @Override\n   */\n  repositionX() {\n    const start = this.conversion.toScreen(this.data.start);\n\n    this.pointX = start;\n    if (this.options.rtl) {\n      this.right = start - this.props.dot.width;\n    } else {\n      this.left = start - this.props.dot.width;\n    }\n\n    this.repositionXY();\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    if (orientation == 'top') {\n      this.pointY = this.top;\n    }\n    else {\n      this.pointY = this.parent.height - this.top - this.height\n    }\n\n    this.repositionXY();\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.props.dot.width;\n  }\n\n  /**\n   * Return the width of the item right from  its start date\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.props.dot.width;\n  }\n}\n\nexport default PointItem;\n","import Item from './Item';\n\n/**\n * @constructor RangeItem\n * @extends Item\n */\nclass RangeItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start, end\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      content: {\n        width: 0\n      }\n    };\n    this.overflow = false; // if contents can overflow (css styling), this flag is set to true\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data.id}`);\n      }\n      if (data.end == undefined) {\n        throw new Error(`Property \"end\" missing in item ${data.id}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   *\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    return (this.data.start < range.end) && (this.data.end > range.start);\n  }\n\n  /**\n   * create DOM elements\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n        // background box\n      this.dom.box = document.createElement('div');\n      // className is updated in redraw()\n\n      // frame box (to prevent the item contents from overflowing)\n      this.dom.frame = document.createElement('div');\n      this.dom.frame.className = 'vis-item-overflow';\n      this.dom.box.appendChild(this.dom.frame);\n    \n      // visible frame box (showing the frame that is always visible)\n      this.dom.visibleFrame = document.createElement('div');\n      this.dom.visibleFrame.className = 'vis-item-visible-frame';\n      this.dom.box.appendChild(this.dom.visibleFrame);\n\n      // contents box\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.frame.appendChild(this.dom.content);\n\n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n\n  }\n\n  /**\n   * append element to DOM\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n      foreground.appendChild(this.dom.box);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // update dirty DOM. An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n\n      // update class\n      const className = (this.data.className ? (' ' + this.data.className) : '') +\n          (this.selected ? ' vis-selected' : '') + \n          (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.box.className = this.baseClassName + className;\n\n      // turn off max-width to be able to calculate the real width\n      // this causes an extra browser repaint/reflow, but so be it\n      this.dom.content.style.maxWidth = 'none';\n    }\n  }\n\n  /**\n   * get DOM component sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    // determine from css whether this box has overflow\n    this.overflow = window.getComputedStyle(this.dom.frame).overflow !== 'hidden';\n    this.whiteSpace = window.getComputedStyle(this.dom.content).whiteSpace !== 'nowrap';\n    return {\n      content: {\n        width: this.dom.content.offsetWidth,\n      },\n      box: {\n        height: this.dom.box.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM component sizes\n   * @param {array} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    this.props.content.width = sizes.content.width;\n    this.height = sizes.box.height;\n    this.dom.content.style.maxWidth = '';\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additional components\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n    this._repaintDeleteButton(this.dom.box);\n    this._repaintDragCenter();\n    this._repaintDragLeft();\n    this._repaintDragRight();\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM \n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes.bind(this)();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      const box = this.dom.box;\n\n      if (box.parentNode) {\n        box.parentNode.removeChild(box);\n      }\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @param {boolean} [limitSize=true] If true (default), the width of the range\n   *                                   item will be limited, as the browser cannot\n   *                                   display very wide divs. This means though\n   *                                   that the applied left and width may\n   *                                   not correspond to the ranges start and end\n   * @Override\n   */\n  repositionX(limitSize) {\n    const parentWidth = this.parent.width;\n    let start = this.conversion.toScreen(this.data.start);\n    let end = this.conversion.toScreen(this.data.end);\n    const align = this.data.align === undefined ? this.options.align : this.data.align;\n    let contentStartPosition;\n    let contentWidth;\n\n    // limit the width of the range, as browsers cannot draw very wide divs\n    // unless limitSize: false is explicitly set in item data\n    if (this.data.limitSize !== false && (limitSize === undefined || limitSize === true)) {\n      if (start < -parentWidth) {\n        start = -parentWidth;\n      }\n      if (end > 2 * parentWidth) {\n        end = 2 * parentWidth;\n      }\n    }\n\n    //round to 3 decimals to compensate floating-point values rounding\n    const boxWidth = Math.max(Math.round((end - start) * 1000) / 1000, 1);\n\n    if (this.overflow) {\n      if (this.options.rtl) {\n        this.right = start;\n      } else {\n        this.left = start;\n      }\n      this.width = boxWidth + this.props.content.width;\n      contentWidth = this.props.content.width;\n\n      // Note: The calculation of width is an optimistic calculation, giving\n      //       a width which will not change when moving the Timeline\n      //       So no re-stacking needed, which is nicer for the eye;\n    }\n    else {\n      if (this.options.rtl) {\n        this.right = start;\n      } else {\n        this.left = start;\n      }\n      this.width = boxWidth;\n      contentWidth = Math.min(end - start, this.props.content.width);\n    }\n\n    if (this.options.rtl) {\n      this.dom.box.style.transform = `translateX(${this.right * -1}px)`;\n    } else {\n      this.dom.box.style.transform = `translateX(${this.left}px)`;\n    }\n    this.dom.box.style.width = `${boxWidth}px`;\n    if (this.whiteSpace) {\n        this.height = this.dom.box.offsetHeight;\n    }\n\n    switch (align) {\n      case 'left':\n        this.dom.content.style.transform = 'translateX(0)';\n        break;\n\n      case 'right':\n        if (this.options.rtl) {\n          const translateX = Math.max((boxWidth - contentWidth), 0) * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${Math.max((boxWidth - contentWidth), 0)}px)`;\n        }\n        break;\n\n      case 'center':\n        if (this.options.rtl) {\n          const translateX = Math.max((boxWidth - contentWidth) / 2, 0) * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${Math.max((boxWidth - contentWidth) / 2, 0)}px)`;\n        }\n        \n        break;\n\n      default: // 'auto'\n        // when range exceeds left of the window, position the contents at the left of the visible area\n        if (this.overflow) {\n          if (end > 0) {\n            contentStartPosition = Math.max(-start, 0);\n          }\n          else {\n            contentStartPosition = -contentWidth; // ensure it's not visible anymore\n          }\n        }\n        else {\n          if (start < 0) {\n            contentStartPosition = -start;\n          }\n          else {\n            contentStartPosition = 0;\n          }\n        }\n        if (this.options.rtl) {\n          const translateX = contentStartPosition * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${contentStartPosition}px)`;\n          // this.dom.content.style.width = `calc(100% - ${contentStartPosition}px)`;\n        }\n    }\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    const box = this.dom.box;\n\n    if (orientation == 'top') {\n      box.style.top = `${this.top}px`;\n    }\n    else {\n      box.style.top = `${this.parent.height - this.top - this.height}px`;\n    }\n  }\n\n  /**\n   * Repaint a drag area on the left side of the range when the range is selected\n   * @protected\n   */\n  _repaintDragLeft() {\n    if ((this.selected || this.options.itemsAlwaysDraggable.range) && this.editable.updateTime && !this.dom.dragLeft) {\n      // create and show drag area\n      const dragLeft = document.createElement('div');\n      dragLeft.className = 'vis-drag-left';\n      dragLeft.dragLeftItem = this;\n\n      this.dom.box.appendChild(dragLeft);\n      this.dom.dragLeft = dragLeft;\n    }\n    else if (!this.selected && !this.options.itemsAlwaysDraggable.range && this.dom.dragLeft) {\n      // delete drag area\n      if (this.dom.dragLeft.parentNode) {\n        this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft);\n      }\n      this.dom.dragLeft = null;\n    }\n  }\n\n  /**\n   * Repaint a drag area on the right side of the range when the range is selected\n   * @protected\n   */\n  _repaintDragRight() {\n    if ((this.selected || this.options.itemsAlwaysDraggable.range) && this.editable.updateTime && !this.dom.dragRight) {\n      // create and show drag area\n      const dragRight = document.createElement('div');\n      dragRight.className = 'vis-drag-right';\n      dragRight.dragRightItem = this;\n\n      this.dom.box.appendChild(dragRight);\n      this.dom.dragRight = dragRight;\n    }\n    else if (!this.selected && !this.options.itemsAlwaysDraggable.range && this.dom.dragRight) {\n      // delete drag area\n      if (this.dom.dragRight.parentNode) {\n        this.dom.dragRight.parentNode.removeChild(this.dom.dragRight);\n      }\n      this.dom.dragRight = null;\n    }\n  }\n}\n\nRangeItem.prototype.baseClassName = 'vis-item vis-range';\n\nexport default RangeItem;\n","import Item from './Item';\nimport BackgroundGroup from '../BackgroundGroup';\nimport RangeItem from './RangeItem';\n\n/**\n * @constructor BackgroundItem\n * @extends Item\n */\nclass BackgroundItem extends Item {\n  /**\n * @constructor BackgroundItem\n * @param {Object} data             Object containing parameters start, end\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe options\n * // TODO: implement support for the BackgroundItem just having a start, then being displayed as a sort of an annotation\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      content: {\n        width: 0\n      }\n    };\n    this.overflow = false; // if contents can overflow (css styling), this flag is set to true\n\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data.id}`);\n      }\n      if (data.end == undefined) {\n        throw new Error(`Property \"end\" missing in item ${data.id}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    // determine visibility\n    return (this.data.start < range.end) && (this.data.end > range.start); \n  }\n\n  /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // background box\n      this.dom.box = document.createElement('div');\n      // className is updated in redraw()\n\n      // frame box (to prevent the item contents from overflowing\n      this.dom.frame = document.createElement('div');\n      this.dom.frame.className = 'vis-item-overflow';\n      this.dom.box.appendChild(this.dom.frame);\n\n      // contents box\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.frame.appendChild(this.dom.content);\n\n      // Note: we do NOT attach this item as attribute to the DOM,\n      //       such that background items cannot be selected\n      //this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const background = this.parent.dom.background;\n      if (!background) {\n        throw new Error('Cannot redraw item: parent has no background container element');\n      }\n      background.appendChild(this.dom.box);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update DOM Dirty components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // update dirty DOM. An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.content);\n      this._updateStyle(this.dom.box);\n\n      // update class\n      const className = (this.data.className ? (' ' + this.data.className) : '') +\n          (this.selected ? ' vis-selected' : '');\n      this.dom.box.className = this.baseClassName + className;\n    }\n  }\n\n  /**\n   * get DOM components sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    // determine from css whether this box has overflow\n    this.overflow = window.getComputedStyle(this.dom.content).overflow !== 'hidden';\n    return {\n      content: {\n        width: this.dom.content.offsetWidth\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    // recalculate size\n    this.props.content.width = sizes.content.width;\n    this.height = 0; // set height zero, so this item will be ignored when stacking items\n\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw result or the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes.bind(this)();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY(margin) {  // eslint-disable-line no-unused-vars\n    let height;\n    const orientation = this.options.orientation.item;\n\n    // special positioning for subgroups\n    if (this.data.subgroup !== undefined) {\n      // TODO: instead of calculating the top position of the subgroups here for every BackgroundItem, calculate the top of the subgroup once in Itemset\n      const itemSubgroup = this.data.subgroup;\n\n      this.dom.box.style.height = `${this.parent.subgroups[itemSubgroup].height}px`;\n\n      if (orientation == 'top') { \n        this.dom.box.style.top = `${this.parent.top + this.parent.subgroups[itemSubgroup].top}px`;\n      } else {\n        this.dom.box.style.top = `${this.parent.top + this.parent.height - this.parent.subgroups[itemSubgroup].top - this.parent.subgroups[itemSubgroup].height}px`;\n      }\n      this.dom.box.style.bottom = '';\n    }\n    // and in the case of no subgroups:\n    else {\n      // we want backgrounds with groups to only show in groups.\n      if (this.parent instanceof BackgroundGroup) {\n        // if the item is not in a group:\n        height = Math.max(this.parent.height,\n            this.parent.itemSet.body.domProps.center.height,\n            this.parent.itemSet.body.domProps.centerContainer.height);\n        this.dom.box.style.bottom = orientation == 'bottom' ? '0' : '';\n        this.dom.box.style.top = orientation == 'top' ? '0' : '';\n      }\n      else {\n        height = this.parent.height;\n        // same alignment for items when orientation is top or bottom\n        this.dom.box.style.top = `${this.parent.top}px`;\n        this.dom.box.style.bottom = '';\n      }\n    }\n    this.dom.box.style.height = `${height}px`;\n  }\n}\n\nBackgroundItem.prototype.baseClassName = 'vis-item vis-background';\n\nBackgroundItem.prototype.stack = false;\n\n/**\n * Show the item in the DOM (when not already visible). The items DOM will\n * be created when needed.\n */\nBackgroundItem.prototype.show = RangeItem.prototype.show;\n\n/**\n * Hide the item from the DOM (when visible)\n * @return {Boolean} changed\n */\nBackgroundItem.prototype.hide = RangeItem.prototype.hide;\n\n/**\n * Reposition the item horizontally\n * @Override\n */\nBackgroundItem.prototype.repositionX = RangeItem.prototype.repositionX;\n\nexport default BackgroundItem;\n","import './tooltip.css';\n\n/**\n * Popup is a class to create a popup window with some text\n */\nclass Popup {\n  /**\n   * @param {Element} container       The container object.\n   * @param {string}  overflowMethod  How the popup should act to overflowing ('flip', 'cap' or 'none')\n   */\n  constructor(container, overflowMethod) {\n    this.container = container;\n    this.overflowMethod = overflowMethod || 'cap';\n\n    this.x = 0;\n    this.y = 0;\n    this.padding = 5;\n    this.hidden = false;\n\n    // create the frame\n    this.frame = document.createElement('div');\n    this.frame.className = 'vis-tooltip';\n    this.container.appendChild(this.frame);\n  }\n\n  /**\n   * @param {number} x   Horizontal position of the popup window\n   * @param {number} y   Vertical position of the popup window\n   */\n  setPosition(x, y) {\n    this.x = parseInt(x);\n    this.y = parseInt(y);\n  }\n\n  /**\n   * Set the content for the popup window. This can be HTML code or text.\n   * @param {string | Element} content\n   */\n  setText(content) {\n    if (content instanceof Element) {\n      this.frame.innerHTML = '';\n      this.frame.appendChild(content);\n    }\n    else {\n      this.frame.innerHTML = content; // string containing text or HTML\n    }\n  }\n\n  /**\n   * Show the popup window\n   * @param {boolean} [doShow]    Show or hide the window\n   */\n  show(doShow) {\n    if (doShow === undefined) {\n      doShow = true;\n    }\n\n    if (doShow === true) {\n      var height = this.frame.clientHeight;\n      var width = this.frame.clientWidth;\n      var maxHeight = this.frame.parentNode.clientHeight;\n      var maxWidth = this.frame.parentNode.clientWidth;\n\n      var left = 0, top = 0;\n\n      if (this.overflowMethod == 'flip' || this.overflowMethod == 'none') {\n        let isLeft = false, isTop = true; // Where around the position it's located\n\n        if (this.overflowMethod == 'flip') {\n          if (this.y - height < this.padding) {\n            isTop = false;\n          }\n\n          if (this.x + width > maxWidth - this.padding) {\n            isLeft = true;\n          }\n        }\n\n        if (isLeft) {\n          left = this.x - width;\n        } else {\n          left = this.x;\n        }\n\n        if (isTop) {\n          top = this.y - height;\n        } else {\n          top = this.y;\n        }\n      } else { // this.overflowMethod == 'cap'\n        top = (this.y - height);\n        if (top + height + this.padding > maxHeight) {\n          top = maxHeight - height - this.padding;\n        }\n        if (top < this.padding) {\n          top = this.padding;\n        }\n\n        left = this.x;\n        if (left + width + this.padding > maxWidth) {\n          left = maxWidth - width - this.padding;\n        }\n        if (left < this.padding) {\n          left = this.padding;\n        }\n      }\n\n      this.frame.style.left = left + \"px\";\n      this.frame.style.top = top + \"px\";\n      this.frame.style.visibility = \"visible\";\n      this.hidden = false;\n    }\n    else {\n      this.hide();\n    }\n  }\n\n  /**\n   * Hide the popup window\n   */\n  hide() {\n    this.hidden = true;\n    this.frame.style.left = \"0\";\n    this.frame.style.top = \"0\";\n    this.frame.style.visibility = \"hidden\";\n  }\n\n  /**\n   * Remove the popup window\n   */\n  destroy() {\n    this.frame.parentNode.removeChild(this.frame); // Remove element from DOM\n  }\n}\n\nexport default Popup;\n","'use strict';\nvar $ = require('../internals/export');\nvar $every = require('../internals/array-iteration').every;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('every');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('every');\n\n// `Array.prototype.every` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.every\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  every: function every(callbackfn /* , thisArg */) {\n    return $every(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.every');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').every;\n","var every = require('../array/virtual/every');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.every;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.every) ? every : own;\n};\n","import { v4 as randomUUID } from \"uuid\";\nimport Item from './Item';\n\n/**\n * ClusterItem\n */\nclass ClusterItem extends Item {\n  /**\n * @constructor Item\n * @param {Object} data             Object containing (optional) parameters type,\n *                                  start, end, content, group, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} options          Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    const modifiedOptions = Object.assign({}, {fitOnDoubleClick: true}, options, {editable: false});\n    super(data, conversion, modifiedOptions);\n\n    this.props = {\n      content: {\n        width: 0,\n        height: 0\n      },\n    };\n    \n    if (!data || data.uiItems == undefined) {\n      throw new Error('Property \"uiItems\" missing in item ' + data.id);\n    }\n  \n    this.id = randomUUID();\n    this.group = data.group;\n    this._setupRange();\n  \n    this.emitter = this.data.eventEmitter;\n    this.range = this.data.range;\n    this.attached = false;\n    this.isCluster = true;\n    this.data.isCluster = true;\n  }\n\n  /**\n   * check if there are items\n   * @return {boolean}\n   */\n  hasItems() {\n    return this.data.uiItems && this.data.uiItems.length && this.attached;\n  }\n  \n  /**\n   * set UI items\n   * @param {array} items\n   */\n  setUiItems(items) {\n    this.detach();\n  \n    this.data.uiItems = items;\n  \n    this._setupRange();\n  \n    this.attach();\n  }\n  \n  /**\n   * check is visible\n   * @param {object} range\n   * @return {boolean}\n   */\n  isVisible(range) {\n    const rangeWidth = this.data.end ? this.data.end - this.data.start : 0;\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n    const end = Math.max(rangeWidth, this.data.start.getTime() + widthInMs);\n    return (this.data.start < range.end) && (end > range.start) && this.hasItems();\n  }\n  \n  /**\n   * get cluster data\n   * @return {object}\n   */\n  getData() {\n    return {\n      isCluster: true,\n      id: this.id,\n      items: this.data.items || [],\n      data: this.data\n    }\n  }\n  \n  /**\n   * redraw cluster item\n   * @param {boolean} returnQueue\n   * @return {boolean}\n   */\n  redraw (returnQueue) {\n    var sizes\n    var queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n  \n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n  \n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n  \n      (function () {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      }).bind(this),\n  \n      (function () {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      }).bind(this),\n  \n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n  \n    if (returnQueue) {\n      return queue;\n    } else {\n      var result;\n      queue.forEach(function (fn) {\n        result = fn();\n      });\n      return result;\n    }\n  }\n  \n  /**\n   * show cluster item\n   */\n  show() {\n    if (!this.displayed) {\n      this.redraw();\n    }\n  }\n  \n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      var dom = this.dom;\n      if (dom.box.parentNode) {\n        dom.box.parentNode.removeChild(dom.box);\n      }\n\n      if (this.options.showStipes) {\n        if (dom.line.parentNode)  {\n          dom.line.parentNode.removeChild(dom.line);\n        }\n        if (dom.dot.parentNode)  {\n          dom.dot.parentNode.removeChild(dom.dot);\n        }\n      }\n      this.displayed = false;\n    }\n  }\n  \n  /**\n   * reposition item x axis\n   */\n  repositionX() {\n    let start = this.conversion.toScreen(this.data.start);\n    let end = this.data.end ? this.conversion.toScreen(this.data.end) : 0;\n    if (end) {\n      this.repositionXWithRanges(start, end);\n    } else {\n      let align = this.data.align === undefined ? this.options.align : this.data.align;\n      this.repositionXWithoutRanges(start, align);\n    }\n  \n    if (this.options.showStipes) {\n      this.dom.line.style.display = this._isStipeVisible() ? 'block' : 'none';\n      this.dom.dot.style.display = this._isStipeVisible() ? 'block' : 'none';\n\n      if (this._isStipeVisible()) {\n        this.repositionStype(start, end);\n      }\n    }\n  }\n\n  /**\n   * reposition item stype\n   * @param {date} start\n   * @param {date} end\n   */\n  repositionStype(start, end) {\n    this.dom.line.style.display = 'block';\n    this.dom.dot.style.display = 'block';\n    const lineOffsetWidth = this.dom.line.offsetWidth;\n    const dotOffsetWidth = this.dom.dot.offsetWidth;\n\n    if (end) {\n      const lineOffset = lineOffsetWidth + start + (end - start) / 2;\n      const dotOffset = lineOffset - dotOffsetWidth / 2;\n      const lineOffsetDirection = this.options.rtl ? lineOffset * -1 : lineOffset;\n      const dotOffsetDirection = this.options.rtl ? dotOffset * -1 : dotOffset;\n\n      this.dom.line.style.transform = `translateX(${lineOffsetDirection}px)`;\n      this.dom.dot.style.transform = `translateX(${dotOffsetDirection}px)`;\n    } else {\n      const lineOffsetDirection = this.options.rtl ? (start * -1) : start;\n      const dotOffsetDirection = this.options.rtl ? ((start - dotOffsetWidth / 2) * -1) : (start - dotOffsetWidth / 2);\n\n      this.dom.line.style.transform = `translateX(${lineOffsetDirection}px)`;\n      this.dom.dot.style.transform = `translateX(${dotOffsetDirection}px)`;\n    }\n  }\n  \n  /**\n   * reposition x without ranges\n   * @param {date} start\n   * @param {string} align\n   */\n  repositionXWithoutRanges(start, align) {\n    // calculate left position of the box\n    if (align == 'right') {\n      if (this.options.rtl) {\n        this.right = start - this.width;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start - this.width;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    } else if (align == 'left') {\n      if (this.options.rtl) {\n        this.right = start;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    } else {\n      // default or 'center'\n      if (this.options.rtl) {\n        this.right = start - this.width / 2;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start - this.width / 2;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    }\n  }\n  \n  /**\n   * reposition x with ranges\n   * @param {date} start\n   * @param {date} end\n   */\n  repositionXWithRanges(start, end) {\n    let boxWidth = Math.round(Math.max(end - start + 0.5, 1));\n  \n    if (this.options.rtl) {\n      this.right = start;\n    } else {\n      this.left = start;\n    }\n  \n    this.width = Math.max(boxWidth, this.minWidth || 0);\n  \n    if (this.options.rtl) {\n      this.dom.box.style.right = this.right + 'px';\n    } else {\n      this.dom.box.style.left = this.left + 'px';\n    }\n  \n    this.dom.box.style.width = boxWidth + 'px';\n  }\n  \n  /**\n   * reposition item y axis\n   */\n  repositionY() {\n    var orientation = this.options.orientation.item;\n    var box = this.dom.box;\n    if (orientation == 'top') {\n      box.style.top = (this.top || 0) + 'px';\n    } else { \n      // orientation 'bottom'\n      box.style.top = (this.parent.height - this.top - this.height || 0) + 'px';\n    }\n  \n    if (this.options.showStipes) {\n      if (orientation == 'top') {\n        this.dom.line.style.top    = '0';\n        this.dom.line.style.height = (this.parent.top + this.top + 1) + 'px';\n        this.dom.line.style.bottom = '';\n      }\n      else { \n        // orientation 'bottom'\n        var itemSetHeight = this.parent.itemSet.props.height;\n        var lineHeight = itemSetHeight - this.parent.top - this.parent.height + this.top;\n        this.dom.line.style.top    = (itemSetHeight - lineHeight) + 'px';\n        this.dom.line.style.bottom = '0';\n      }\n    \n      this.dom.dot.style.top = (-this.dom.dot.offsetHeight / 2) + 'px';\n    }\n  }\n\n  /**\n   * get width left\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.width / 2;\n  }\n  \n  /**\n   * get width right\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.width / 2;\n  }\n  \n  /**\n   * move cluster item\n   */\n  move() {\n    this.repositionX();\n    this.repositionY();\n  }\n  \n  /**\n   * attach\n   */\n  attach() {\n    for (let item of this.data.uiItems) {\n      item.cluster = this;\n    }\n  \n    this.data.items = this.data.uiItems.map(item => item.data);\n\n    this.attached = true;\n    this.dirty = true;\n  }\n  \n  /**\n   * detach\n   * @param {boolean} detachFromParent\n   * @return {void}\n   */\n  detach(detachFromParent = false) {\n    if (!this.hasItems()) {\n      return;\n    }\n  \n    for (let item of this.data.uiItems) {\n      delete item.cluster;\n    }\n  \n    this.attached = false;\n  \n    if (detachFromParent && this.group) {\n      this.group.remove(this);\n      this.group = null;\n    }\n  \n    this.data.items = [];\n    this.dirty = true;\n  }\n  \n  /**\n   * handle on double click\n   */\n  _onDoubleClick() {\n   this._fit();\n  }\n  \n  /**\n   * set range\n   */\n  _setupRange() {\n    const stats = this.data.uiItems.map(item => ({\n      start: item.data.start.valueOf(),\n      end: item.data.end ? item.data.end.valueOf() : item.data.start.valueOf(),\n    }));\n  \n    this.data.min = Math.min(...stats.map(s => Math.min(s.start, s.end || s.start)));\n    this.data.max = Math.max(...stats.map(s => Math.max(s.start, s.end || s.start)));\n    const centers = this.data.uiItems.map(item => item.center);\n    const avg = centers.reduce((sum, value) => sum + value, 0) / this.data.uiItems.length;\n  \n    if (this.data.uiItems.some(item => item.data.end)) {\n      // contains ranges\n      this.data.start = new Date(this.data.min);\n      this.data.end = new Date(this.data.max);\n    } else {\n      this.data.start = new Date(avg);\n      this.data.end = null;\n    }\n  }\n  \n  /**\n   * get UI items\n   * @return {array}\n   */\n  _getUiItems() {\n    if (this.data.uiItems && this.data.uiItems.length) {\n      return this.data.uiItems.filter(item => item.cluster === this);\n    }\n  \n    return [];\n  }\n  \n  /**\n   * create DOM element\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n  \n      // create main box\n      this.dom.box = document.createElement('DIV');\n  \n      // contents box (inside the background box). used for making margins\n      this.dom.content = document.createElement('DIV');\n      this.dom.content.className = 'vis-item-content';\n      \n      this.dom.box.appendChild(this.dom.content);\n  \n      if (this.options.showStipes) {\n        // line to axis\n        this.dom.line = document.createElement('DIV');\n        this.dom.line.className = 'vis-cluster-line';\n        this.dom.line.style.display = 'none';\n  \n        // dot on axis\n        this.dom.dot = document.createElement('DIV');\n        this.dom.dot.className = 'vis-cluster-dot';\n        this.dom.dot.style.display = 'none';\n      }\n  \n      if (this.options.fitOnDoubleClick) {\n        this.dom.box.ondblclick = ClusterItem.prototype._onDoubleClick.bind(this);\n      }\n      \n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n  \n      this.dirty = true;\n    }\n  }\n  \n  /**\n   * append element to DOM\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n  \n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n  \n      foreground.appendChild(this.dom.box);\n    }\n  \n    const background = this.parent.dom.background;\n  \n    if (this.options.showStipes) {\n      if (!this.dom.line.parentNode) {\n        if (!background) throw new Error('Cannot redraw item: parent has no background container element');\n        background.appendChild(this.dom.line);\n      }\n    \n      if (!this.dom.dot.parentNode) {\n        var axis = this.parent.dom.axis;\n        if (!background) throw new Error('Cannot redraw item: parent has no axis container element');\n        axis.appendChild(this.dom.dot);\n      }\n    }\n  \n    this.displayed = true;\n  }\n  \n  /**\n   * update dirty DOM components\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n  \n      // update class\n      const className = this.baseClassName + ' ' + (this.data.className ? ' ' + this.data.className : '') +\n        (this.selected ? ' vis-selected' : '') + ' vis-readonly';\n      this.dom.box.className = 'vis-item ' + className;\n  \n      if (this.options.showStipes) {\n        this.dom.line.className = 'vis-item vis-cluster-line ' +  (this.selected ? ' vis-selected' : '');\n        this.dom.dot.className  = 'vis-item vis-cluster-dot ' +  (this.selected ? ' vis-selected' : '');\n      }\n  \n      if (this.data.end) {\n        // turn off max-width to be able to calculate the real width\n        // this causes an extra browser repaint/reflow, but so be it\n        this.dom.content.style.maxWidth = 'none';\n      }\n    }\n  }\n  \n  /**\n   * get DOM components sizes\n   * @return {object}\n   */\n  _getDomComponentsSizes() {\n    const sizes = {\n      previous: {\n        right: this.dom.box.style.right,\n        left: this.dom.box.style.left\n      },\n      box: {\n        width: this.dom.box.offsetWidth,\n        height: this.dom.box.offsetHeight\n      },\n    };\n\n    if (this.options.showStipes) {\n      sizes.dot = {\n        height: this.dom.dot.offsetHeight,\n        width: this.dom.dot.offsetWidth\n      };\n      sizes.line = {\n        width: this.dom.line.offsetWidth\n      };\n    }\n\n    return sizes;\n  }\n  \n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   */\n  _updateDomComponentsSizes(sizes) {\n    if (this.options.rtl) {\n      this.dom.box.style.right = \"0px\";\n    } else {\n      this.dom.box.style.left = \"0px\";\n    }\n  \n    // recalculate size\n    if (!this.data.end) {\n      this.width = sizes.box.width;\n    } else {\n      this.minWidth = sizes.box.width;\n    }\n  \n    this.height = sizes.box.height;\n  \n    // restore previous position\n    if (this.options.rtl) {\n      this.dom.box.style.right = sizes.previous.right;\n    } else {\n      this.dom.box.style.left = sizes.previous.left;\n    }\n  \n    this.dirty = false;\n  }\n  \n  /**\n   * repaint DOM additional components\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n  }\n  \n  /**\n   * check is stripe visible\n   * @return {number}\n   * @private\n   */\n  _isStipeVisible() {\n    return this.minWidth >= this.width || !this.data.end;\n  }\n  \n  /**\n   * get fit range\n   * @return {object}\n   * @private\n   */\n  _getFitRange() {\n    const offset = 0.05*(this.data.max - this.data.min) / 2;\n      return {\n        fitStart: this.data.min - offset,\n        fitEnd: this.data.max + offset\n      };\n  }\n  \n   /**\n   * fit\n   * @private\n   */\n  _fit() {\n    if (this.emitter) {\n      const {fitStart, fitEnd} = this._getFitRange();\n  \n      const fitArgs = {\n        start: new Date(fitStart),\n        end: new Date(fitEnd),\n        animation: true\n      };\n  \n      this.emitter.emit('fit', fitArgs);\n    }\n  }\n\n   /**\n   * get item data\n   * @return {object}\n   * @private\n   */\n  _getItemData() {\n    return this.data;\n  }\n}\n\nClusterItem.prototype.baseClassName = 'vis-item vis-range vis-cluster';\nexport default ClusterItem;\n","import ClusterItem from './item/ClusterItem';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n/**\n * An Cluster generator generates cluster items\n */\nexport default class ClusterGenerator {\n    /**\n     * @param {ItemSet} itemSet itemsSet instance\n     * @constructor ClusterGenerator\n    */\n    constructor(itemSet) {\n        this.itemSet = itemSet;\n        this.groups = {};\n        this.cache = {};\n        this.cache[-1] = [];\n    }\n\n    /**\n     * @param {Object} itemData             Object containing parameters start content, className.\n     * @param {{toScreen: function, toTime: function}} conversion\n     *                                  Conversion functions from time to screen and vice versa\n     * @param {Object} [options]        Configuration options\n     * @return {Object} newItem\n    */\n    createClusterItem(itemData, conversion, options) {\n        const newItem = new ClusterItem(itemData, conversion, options);\n        return newItem;\n    }\n\n    /**\n     * Set the items to be clustered.\n     * This will clear cached clusters.\n     * @param {Item[]} items\n     * @param {Object} [options]  Available options:\n     *                            {boolean} applyOnChangedLevel\n     *                                If true (default), the changed data is applied\n     *                                as soon the cluster level changes. If false,\n     *                                The changed data is applied immediately\n     */\n    setItems(items, options) {\n        this.items = items || [];\n        this.dataChanged = true;\n        this.applyOnChangedLevel = false;\n\n        if (options && options.applyOnChangedLevel) {\n            this.applyOnChangedLevel = options.applyOnChangedLevel;\n        }\n    }\n\n    /**\n     * Update the current data set: clear cache, and recalculate the clustering for\n     * the current level\n     */\n    updateData() {\n        this.dataChanged = true;\n        this.applyOnChangedLevel = false;\n    }\n\n    /**\n     * Cluster the items which are too close together\n     * @param {array} oldClusters \n     * @param {number} scale      The scale of the current window : (windowWidth / (endDate - startDate)) \n     * @param {{maxItems: number, clusterCriteria: function, titleTemplate: string}} options             \n     * @return {array} clusters\n    */\n    getClusters(oldClusters, scale, options) {\n        let { maxItems, clusterCriteria } = typeof options === \"boolean\" ? {} : options;\n    \n        if (!clusterCriteria) {\n            clusterCriteria = () => true;\n        }\n\n        maxItems = maxItems || 1;\n\n        let level = -1;\n        let granularity = 2;\n        let timeWindow = 0;\n\n        if (scale > 0) {\n            if (scale >= 1) {\n                return [];\n            }\n\n            level = Math.abs(Math.round(Math.log(100 / scale) / Math.log(granularity)));\n            timeWindow = Math.abs(Math.pow(granularity, level));\n        }\n\n        // clear the cache when and re-generate groups the data when needed.\n        if (this.dataChanged) {\n            const levelChanged = (level != this.cacheLevel);\n            const applyDataNow = this.applyOnChangedLevel ? levelChanged : true;\n            if (applyDataNow) {\n                this._dropLevelsCache();\n                this._filterData();\n            }\n        }\n\n        this.cacheLevel = level;\n        let clusters = this.cache[level];\n        if (!clusters) {\n            clusters = [];\n            for (let groupName in this.groups) {\n                if (this.groups.hasOwnProperty(groupName)) {\n                    const items = this.groups[groupName];\n                    const iMax = items.length;\n                    let i = 0;\n                    while (i < iMax) {\n                        // find all items around current item, within the timeWindow\n                        let item = items[i];\n                        let neighbors = 1; // start at 1, to include itself)\n\n                        // loop through items left from the current item\n                        let j = i - 1;\n                        while (j >= 0 && (item.center - items[j].center) < timeWindow / 2) {\n                            if (!items[j].cluster && clusterCriteria(item.data, items[j].data)) {\n                                neighbors++;\n                            }\n                            j--;\n                        }\n\n                        // loop through items right from the current item\n                        let k = i + 1;\n                        while (k < items.length && (items[k].center - item.center) < timeWindow / 2) {\n                            if (clusterCriteria(item.data, items[k].data)) {\n                                neighbors++;\n                            }\n                            k++;\n                        }\n\n                        // loop through the created clusters\n                        let l = clusters.length - 1;\n                        while (l >= 0 && (item.center - clusters[l].center) < timeWindow) {\n                            if (item.group == clusters[l].group && clusterCriteria(item.data, clusters[l].data)) {\n                                neighbors++;\n                            }\n                            l--;\n                        }\n\n                        // aggregate until the number of items is within maxItems\n                        if (neighbors > maxItems) {\n                            // too busy in this window.\n                            const num = neighbors - maxItems + 1;\n                            const clusterItems = [];\n\n                            // append the items to the cluster,\n                            // and calculate the average start for the cluster\n                            let m = i;\n                            while (clusterItems.length < num && m < items.length) {\n                                if (clusterCriteria(items[m].data, items[m].data)) {\n                                    clusterItems.push(items[m]);\n                                }\n                                m++;\n                            }\n\n                            const groupId = this.itemSet.getGroupId(item.data);\n                            const group = this.itemSet.groups[groupId] || this.itemSet.groups[ReservedGroupIds.UNGROUPED];\n                            let cluster = this._getClusterForItems(clusterItems, group, oldClusters, options);\n                            clusters.push(cluster);\n\n                            i += num;\n                        } else {\n                            delete item.cluster;\n                            i += 1;\n                        }\n                    }\n                }\n            }\n\n            this.cache[level] = clusters;\n        }\n\n        return clusters;\n    }\n\n    /**\n     * Filter the items per group.\n     * @private\n     */\n    _filterData() {\n        // filter per group\n        const groups = {};\n        this.groups = groups;\n\n        // split the items per group\n        for (const item of Object.values(this.items)) {\n            // put the item in the correct group\n            const groupName = item.parent ? item.parent.groupId : '';\n            let group = groups[groupName];\n            if (!group) {\n                group = [];\n                groups[groupName] = group;\n            }\n            group.push(item);\n\n            // calculate the center of the item\n            if (item.data.start) {\n                if (item.data.end) {\n                    // range\n                    item.center = (item.data.start.valueOf() + item.data.end.valueOf()) / 2;\n                } else {\n                    // box, dot\n                    item.center = item.data.start.valueOf();\n                }\n            }\n        }\n\n        // sort the items per group\n        for (let currentGroupName in groups) {\n            if (groups.hasOwnProperty(currentGroupName)) {\n                groups[currentGroupName].sort((a, b) => a.center - b.center);\n            }\n        }\n\n        this.dataChanged = false;\n    }\n\n    /**\n     * Create new cluster or return existing\n     * @private\n     * @param {array} clusterItems    \n     * @param {object} group \n     * @param {array} oldClusters \n     * @param {object} options \n     * @returns {object} cluster\n     */\n    _getClusterForItems(clusterItems, group, oldClusters, options) {\n        const oldClustersLookup = (oldClusters || []).map(cluster => ({\n            cluster,\n            itemsIds: new Set(cluster.data.uiItems.map(item => item.id))\n        }));\n        let cluster;\n        if (oldClustersLookup.length) {\n            for (let oldClusterData of oldClustersLookup) {\n                if (oldClusterData.itemsIds.size === clusterItems.length \n                    && clusterItems.every(clusterItem => oldClusterData.itemsIds.has(clusterItem.id))) {\n                    cluster = oldClusterData.cluster;\n                    break;\n                }\n            }\n        }\n\n        if (cluster) {\n            cluster.setUiItems(clusterItems);\n            if (cluster.group !== group) {\n                if (cluster.group) {\n                    cluster.group.remove(cluster);    \n                }\n\n                if (group) {\n                    group.add(cluster);    \n                    cluster.group = group;\n                }\n            }\n            return cluster;\n        }\n\n        let titleTemplate = options.titleTemplate || '';\n        const conversion = {\n            toScreen: this.itemSet.body.util.toScreen,\n            toTime: this.itemSet.body.util.toTime\n        };\n\n        const title = titleTemplate.replace(/{count}/, clusterItems.length);\n        const clusterContent = '<div title=\"' + title + '\">' + clusterItems.length + '</div>';\n        const clusterOptions = Object.assign({}, options, this.itemSet.options);\n        const data = {\n            'content': clusterContent,\n            'title': title,\n            'group': group,\n            'uiItems': clusterItems,\n            'eventEmitter': this.itemSet.body.emitter,\n            'range': this.itemSet.body.range\n        };\n        cluster = this.createClusterItem(data,\n        conversion,\n        clusterOptions);\n\n        if (group) {\n            group.add(cluster);\n            cluster.group = group;\n        }\n\n        cluster.attach();\n\n        return cluster;\n    }\n\n    /**\n     * Drop cache\n     * @private\n     */\n    _dropLevelsCache() {\n        this.cache = {};\n        this.cacheLevel = -1;\n        this.cache[this.cacheLevel] = [];\n    }\n}\n","import { v4 as randomUUID } from \"uuid\";\nimport Hammer from '../../module/hammer';\nimport util, { typeCoerceDataSet } from '../../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport TimeStep from '../TimeStep';\nimport Component from './Component';\nimport Group from './Group';\nimport BackgroundGroup from './BackgroundGroup';\nimport BoxItem from './item/BoxItem';\nimport PointItem from './item/PointItem';\nimport RangeItem from './item/RangeItem';\nimport BackgroundItem from './item/BackgroundItem';\nimport Popup from '../../shared/Popup';\nimport ClusterGenerator from './ClusterGenerator';\n\nimport './css/itemset.css';\nimport './css/labelset.css';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n/**\n * An ItemSet holds a set of items and ranges which can be displayed in a\n * range. The width is determined by the parent of the ItemSet, and the height\n * is determined by the size of the items.\n */\nclass ItemSet extends Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} body\n * @param {Object} [options]      See ItemSet.setOptions for the available options.\n * @constructor ItemSet\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n    this.defaultOptions = {\n      type: null,  // 'box', 'point', 'range', 'background'\n      orientation: {\n        item: 'bottom'   // item orientation: 'top' or 'bottom'\n      },\n      align: 'auto', // alignment of box items\n      stack: true,\n      stackSubgroups: true,\n      groupOrderSwap(fromGroup, toGroup, groups) {  // eslint-disable-line no-unused-vars\n        const targetOrder = toGroup.order;\n        toGroup.order = fromGroup.order;\n        fromGroup.order = targetOrder;\n      },\n      groupOrder: 'order',\n\n      selectable: true,\n      multiselect: false,\n      longSelectPressTime: 251,\n      itemsAlwaysDraggable: {\n        item: false,\n        range: false,\n      },\n\n      editable: {\n        updateTime: false,\n        updateGroup: false,\n        add: false,\n        remove: false,\n        overrideItems: false\n      },\n\n      groupEditable: {\n        order: false,\n        add: false,\n        remove: false\n      },\n\n      snap: TimeStep.snap,\n\n      // Only called when `objectData.target === 'item'.\n      onDropObjectOnItem(objectData, item, callback) {\n        callback(item)\n      },\n      onAdd(item, callback) {\n        callback(item);\n      },\n      onUpdate(item, callback) {\n        callback(item);\n      },\n      onMove(item, callback) {\n        callback(item);\n      },\n      onRemove(item, callback) {\n        callback(item);\n      },\n      onMoving(item, callback) {\n        callback(item);\n      },\n      onAddGroup(item, callback) {\n        callback(item);\n      },\n      onMoveGroup(item, callback) {\n        callback(item);\n      },\n      onRemoveGroup(item, callback) {\n        callback(item);\n      },\n\n      margin: {\n        item: {\n          horizontal: 10,\n          vertical: 10\n        },\n        axis: 20\n      },\n\n      showTooltips: true,\n\n      tooltip: {\n        followMouse: false,\n        overflowMethod: 'flip',\n        delay: 500\n      },\n\n      tooltipOnItemUpdateTime: false\n    };\n\n    // options is shared by this ItemSet and all its items\n    this.options = util.extend({}, this.defaultOptions);\n    this.options.rtl = options.rtl;\n    this.options.onTimeout = options.onTimeout;\n\n    this.conversion = {\n      toScreen: body.util.toScreen,\n      toTime: body.util.toTime\n    };\n    this.dom = {};\n    this.props = {};\n    this.hammer = null;\n    \n    const me = this;\n    this.itemsData = null;    // DataSet\n    this.groupsData = null;   // DataSet\n    this.itemsSettingTime = null;\n    this.initialItemSetDrawn = false;\n    this.userContinueNotBail = null;  \n\n    this.sequentialSelection = false;\n    \n    // listeners for the DataSet of the items\n    this.itemListeners = {\n      'add'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onAdd(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      },\n      'update'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onUpdate(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      },\n      'remove'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onRemove(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      }\n    };\n\n    // listeners for the DataSet of the groups\n    this.groupListeners = {\n      'add'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onAddGroups(params.items);\n        \n        if (me.groupsData && me.groupsData.length > 0) {\n            const groupsData = me.groupsData.getDataSet();\n            groupsData.get().forEach(groupData => {\n            if (groupData.nestedGroups) {\n              if (groupData.showNested != false) {\n                groupData.showNested = true;\n              }\n              let updatedGroups = [];\n              groupData.nestedGroups.forEach(nestedGroupId => {\n                const updatedNestedGroup = groupsData.get(nestedGroupId);\n                if (!updatedNestedGroup) { return; }\n                updatedNestedGroup.nestedInGroup = groupData.id;\n                if (groupData.showNested == false) {\n                  updatedNestedGroup.visible = false;\n                }\n                updatedGroups = updatedGroups.concat(updatedNestedGroup);\n              });\n              groupsData.update(updatedGroups, senderId);\n            }\n          });\n        }\n      },\n      'update'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onUpdateGroups(params.items);\n      },\n      'remove'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onRemoveGroups(params.items);\n      }\n    };\n\n    this.items = {};      // object with an Item for every data item\n    this.groups = {};     // Group object for every group\n    this.groupIds = [];\n\n    this.selection = [];  // list with the ids of all selected nodes\n\n    this.popup = null;\n    this.popupTimer = null;\n\n    this.touchParams = {}; // stores properties while dragging\n    this.groupTouchParams = {\n      group: null,\n      isDragging: false\n    };\n  \n    // create the HTML DOM\n    this._create();\n\n    this.setOptions(options);\n    this.clusters = [];\n  }\n\n  /**\n   * Create the HTML DOM for the ItemSet\n   */\n  _create() {\n    const frame = document.createElement('div');\n    frame.className = 'vis-itemset';\n    frame['vis-itemset'] = this;\n    this.dom.frame = frame;\n\n    // create background panel\n    const background = document.createElement('div');\n    background.className = 'vis-background';\n    frame.appendChild(background);\n    this.dom.background = background;\n\n    // create foreground panel\n    const foreground = document.createElement('div');\n    foreground.className = 'vis-foreground';\n    frame.appendChild(foreground);\n    this.dom.foreground = foreground;\n\n    // create axis panel\n    const axis = document.createElement('div');\n    axis.className = 'vis-axis';\n    this.dom.axis = axis;\n\n    // create labelset\n    const labelSet = document.createElement('div');\n    labelSet.className = 'vis-labelset';\n    this.dom.labelSet = labelSet;\n\n    // create ungrouped Group\n    this._updateUngrouped();\n\n    // create background Group\n    const backgroundGroup = new BackgroundGroup(BACKGROUND, null, this);\n    backgroundGroup.show();\n    this.groups[BACKGROUND] = backgroundGroup;\n\n    // attach event listeners\n    // Note: we bind to the centerContainer for the case where the height\n    //       of the center container is larger than of the ItemSet, so we\n    //       can click in the empty area to create a new item or deselect an item.\n    this.hammer = new Hammer(this.body.dom.centerContainer);\n\n    // drag items when selected\n    this.hammer.on('hammer.input', event => {\n      if (event.isFirst) {\n        this._onTouch(event);\n      }\n    });\n    this.hammer.on('panstart', this._onDragStart.bind(this));\n    this.hammer.on('panmove',  this._onDrag.bind(this));\n    this.hammer.on('panend',   this._onDragEnd.bind(this));\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.ALL});\n\n    // single select (or unselect) when tapping an item\n    this.hammer.on('tap',  this._onSelectItem.bind(this));\n\n    // multi select when holding mouse/touch, or on ctrl+click\n    this.hammer.on('press', this._onMultiSelectItem.bind(this));\n\n    // add item on doubletap\n    this.hammer.on('doubletap', this._onAddItem.bind(this));\n\n    if (this.options.rtl) {\n      this.groupHammer = new Hammer(this.body.dom.rightContainer);\n    } else {\n      this.groupHammer = new Hammer(this.body.dom.leftContainer);\n    }\n    \n    this.groupHammer.on('tap',      this._onGroupClick.bind(this));\n    this.groupHammer.on('panstart', this._onGroupDragStart.bind(this));\n    this.groupHammer.on('panmove',  this._onGroupDrag.bind(this));\n    this.groupHammer.on('panend',   this._onGroupDragEnd.bind(this));\n    this.groupHammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_VERTICAL});\n    \n    this.body.dom.centerContainer.addEventListener('mouseover', this._onMouseOver.bind(this));\n    this.body.dom.centerContainer.addEventListener('mouseout', this._onMouseOut.bind(this));\n    this.body.dom.centerContainer.addEventListener('mousemove', this._onMouseMove.bind(this));\n    // right-click on timeline \n    this.body.dom.centerContainer.addEventListener('contextmenu', this._onDragEnd.bind(this));\n\n    this.body.dom.centerContainer.addEventListener('mousewheel', this._onMouseWheel.bind(this));\n\n    // attach to the DOM\n    this.show();\n  }\n\n  /**\n   * Set options for the ItemSet. Existing options will be extended/overwritten.\n   * @param {Object} [options] The following options are available:\n   *                           {string} type\n   *                              Default type for the items. Choose from 'box'\n   *                              (default), 'point', 'range', or 'background'.\n   *                              The default style can be overwritten by\n   *                              individual items.\n   *                           {string} align\n   *                              Alignment for the items, only applicable for\n   *                              BoxItem. Choose 'center' (default), 'left', or\n   *                              'right'.\n   *                           {string} orientation.item\n   *                              Orientation of the item set. Choose 'top' or\n   *                              'bottom' (default).\n   *                           {Function} groupOrder\n   *                              A sorting function for ordering groups\n   *                           {boolean} stack\n   *                              If true (default), items will be stacked on\n   *                              top of each other.\n   *                           {number} margin.axis\n   *                              Margin between the axis and the items in pixels.\n   *                              Default is 20.\n   *                           {number} margin.item.horizontal\n   *                              Horizontal margin between items in pixels.\n   *                              Default is 10.\n   *                           {number} margin.item.vertical\n   *                              Vertical Margin between items in pixels.\n   *                              Default is 10.\n   *                           {number} margin.item\n   *                              Margin between items in pixels in both horizontal\n   *                              and vertical direction. Default is 10.\n   *                           {number} margin\n   *                              Set margin for both axis and items in pixels.\n   *                           {boolean} selectable\n   *                              If true (default), items can be selected.\n   *                           {boolean} multiselect\n   *                              If true, multiple items can be selected.\n   *                              False by default.\n   *                           {boolean} editable\n   *                              Set all editable options to true or false\n   *                           {boolean} editable.updateTime\n   *                              Allow dragging an item to an other moment in time\n   *                           {boolean} editable.updateGroup\n   *                              Allow dragging an item to an other group\n   *                           {boolean} editable.add\n   *                              Allow creating new items on double tap\n   *                           {boolean} editable.remove\n   *                              Allow removing items by clicking the delete button\n   *                              top right of a selected item.\n   *                           {Function(item: Item, callback: Function)} onAdd\n   *                              Callback function triggered when an item is about to be added:\n   *                              when the user double taps an empty space in the Timeline.\n   *                           {Function(item: Item, callback: Function)} onUpdate\n   *                              Callback function fired when an item is about to be updated.\n   *                              This function typically has to show a dialog where the user\n   *                              change the item. If not implemented, nothing happens.\n   *                           {Function(item: Item, callback: Function)} onMove\n   *                              Fired when an item has been moved. If not implemented,\n   *                              the move action will be accepted.\n   *                           {Function(item: Item, callback: Function)} onRemove\n   *                              Fired when an item is about to be deleted.\n   *                              If not implemented, the item will be always removed.\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      const fields = [\n        'type', 'rtl', 'align', 'order', 'stack', 'stackSubgroups', 'selectable', 'multiselect', 'sequentialSelection',\n        'multiselectPerGroup', 'longSelectPressTime', 'groupOrder', 'dataAttributes', 'template', 'groupTemplate', 'visibleFrameTemplate',\n        'hide', 'snap', 'groupOrderSwap', 'showTooltips', 'tooltip', 'tooltipOnItemUpdateTime', 'groupHeightMode', 'onTimeout'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n\n      if ('itemsAlwaysDraggable' in options) {\n        if (typeof options.itemsAlwaysDraggable === 'boolean') {\n          this.options.itemsAlwaysDraggable.item = options.itemsAlwaysDraggable;\n          this.options.itemsAlwaysDraggable.range = false;\n        }\n        else if (typeof options.itemsAlwaysDraggable === 'object') {\n          util.selectiveExtend(['item', 'range'], this.options.itemsAlwaysDraggable, options.itemsAlwaysDraggable);\n          // only allow range always draggable when item is always draggable as well\n          if (! this.options.itemsAlwaysDraggable.item) {\n            this.options.itemsAlwaysDraggable.range = false;\n          }\n        }\n      }\n\n      if ('sequentialSelection' in options) {\n        if (typeof options.sequentialSelection === 'boolean') {\n          this.options.sequentialSelection = options.sequentialSelection;\n        }\n      }\n\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation.item = options.orientation === 'top' ? 'top' : 'bottom';\n        }\n        else if (typeof options.orientation === 'object' && 'item' in options.orientation) {\n          this.options.orientation.item = options.orientation.item;\n        }\n      }\n\n      if ('margin' in options) {\n        if (typeof options.margin === 'number') {\n          this.options.margin.axis = options.margin;\n          this.options.margin.item.horizontal = options.margin;\n          this.options.margin.item.vertical = options.margin;\n        }\n        else if (typeof options.margin === 'object') {\n          util.selectiveExtend(['axis'], this.options.margin, options.margin);\n          if ('item' in options.margin) {\n            if (typeof options.margin.item === 'number') {\n              this.options.margin.item.horizontal = options.margin.item;\n              this.options.margin.item.vertical = options.margin.item;\n            }\n            else if (typeof options.margin.item === 'object') {\n              util.selectiveExtend(['horizontal', 'vertical'], this.options.margin.item, options.margin.item);\n            }\n          }\n        }\n      }\n\n      ['locale', 'locales'].forEach(key => {\n        if (key in options) {\n          this.options[key] = options[key];\n        }\n      });\n\n      if ('editable' in options) {\n        if (typeof options.editable === 'boolean') {\n          this.options.editable.updateTime    = options.editable;\n          this.options.editable.updateGroup   = options.editable;\n          this.options.editable.add           = options.editable;\n          this.options.editable.remove        = options.editable;\n          this.options.editable.overrideItems = false;\n        }\n        else if (typeof options.editable === 'object') {\n          util.selectiveExtend(['updateTime', 'updateGroup', 'add', 'remove', 'overrideItems'], this.options.editable, options.editable);\n        }\n      }\n\n      if ('groupEditable' in options) {\n        if (typeof options.groupEditable === 'boolean') {\n          this.options.groupEditable.order  = options.groupEditable;\n          this.options.groupEditable.add    = options.groupEditable;\n          this.options.groupEditable.remove = options.groupEditable;\n        }\n        else if (typeof options.groupEditable === 'object') {\n          util.selectiveExtend(['order', 'add', 'remove'], this.options.groupEditable, options.groupEditable);\n        }\n      }\n\n      // callback functions\n      const addCallback = name => {\n        const fn = options[name];\n        if (fn) {\n          if (!(typeof fn === 'function')) {\n            throw new Error(`option ${name} must be a function ${name}(item, callback)`);\n          }\n          this.options[name] = fn;\n        }\n      };\n      ['onDropObjectOnItem', 'onAdd', 'onUpdate', 'onRemove', 'onMove', 'onMoving', 'onAddGroup', 'onMoveGroup', 'onRemoveGroup'].forEach(addCallback);\n\n      if (options.cluster) {\n        Object.assign(this.options, {\n          cluster: options.cluster\n        });\n        if (!this.clusterGenerator) {\n          this.clusterGenerator = new ClusterGenerator(this);\n        } \n        this.clusterGenerator.setItems(this.items, { applyOnChangedLevel: false });\n        this.markDirty({ refreshItems: true, restackGroups: true });\n\n        this.redraw();\n      } else if (this.clusterGenerator) {\n        this._detachAllClusters();\n        this.clusters = [];\n        this.clusterGenerator = null;\n        this.options.cluster = undefined;\n        this.markDirty({ refreshItems: true, restackGroups: true });\n\n        this.redraw();\n      } else {\n        // force the itemSet to refresh: options like orientation and margins may be changed\n        this.markDirty();\n      }\n    }\n  }\n\n  /**\n   * Mark the ItemSet dirty so it will refresh everything with next redraw.\n   * Optionally, all items can be marked as dirty and be refreshed.\n   * @param {{refreshItems: boolean}} [options]\n   */\n  markDirty(options) {\n    this.groupIds = [];\n\n    if (options) {\n      if (options.refreshItems) {\n        util.forEach(this.items, item => {\n          item.dirty = true;\n          if (item.displayed) item.redraw();\n        });\n      }\n      \n      if (options.restackGroups) {\n        util.forEach(this.groups, (group, key) => {\n          if (key === BACKGROUND) return;\n          group.stackDirty = true;\n        });\n      }\n    }\n  }\n\n  /**\n   * Destroy the ItemSet\n   */\n  destroy() {\n    this.clearPopupTimer();\n    this.hide();\n    this.setItems(null);\n    this.setGroups(null);\n\n    this.hammer && this.hammer.destroy();\n    this.groupHammer && this.groupHammer.destroy();\n    this.hammer = null;\n\n    this.body = null;\n    this.conversion = null;\n  }\n\n  /**\n   * Hide the component from the DOM\n   */\n  hide() {\n    // remove the frame containing the items\n    if (this.dom.frame.parentNode) {\n      this.dom.frame.parentNode.removeChild(this.dom.frame);\n    }\n\n    // remove the axis with dots\n    if (this.dom.axis.parentNode) {\n      this.dom.axis.parentNode.removeChild(this.dom.axis);\n    }\n\n    // remove the labelset containing all group labels\n    if (this.dom.labelSet.parentNode) {\n      this.dom.labelSet.parentNode.removeChild(this.dom.labelSet);\n    }\n  }\n\n  /**\n   * Show the component in the DOM (when not already visible).\n   */\n  show() {\n    // show frame containing the items\n    if (!this.dom.frame.parentNode) {\n      this.body.dom.center.appendChild(this.dom.frame);\n    }\n\n    // show axis with dots\n    if (!this.dom.axis.parentNode) {\n      this.body.dom.backgroundVertical.appendChild(this.dom.axis);\n    }\n\n    // show labelset containing labels\n    if (!this.dom.labelSet.parentNode) {\n      if (this.options.rtl) {\n        this.body.dom.right.appendChild(this.dom.labelSet);\n      } else {\n        this.body.dom.left.appendChild(this.dom.labelSet);\n      }\n    }\n  }\n\n  /**\n   * Activates the popup timer to show the given popup after a fixed time.\n   * @param {Popup} popup\n   */\n  setPopupTimer(popup) {\n    this.clearPopupTimer();\n    if (popup) {\n      const delay = this.options.tooltip.delay || typeof this.options.tooltip.delay === 'number' ?\n            this.options.tooltip.delay :\n            500;\n      this.popupTimer = setTimeout(\n        function () {\n          popup.show()\n        }, delay);\n    }\n  }\n\n  /**\n   * Clears the popup timer for the tooltip.\n   */\n  clearPopupTimer() {\n    if (this.popupTimer != null) {\n        clearTimeout(this.popupTimer);\n        this.popupTimer = null;\n    }\n  }\n  \n  /**\n   * Set selected items by their id. Replaces the current selection\n   * Unknown id's are silently ignored.\n   * @param {string[] | string} [ids] An array with zero or more id's of the items to be\n   *                                  selected, or a single item id. If ids is undefined\n   *                                  or an empty array, all items will be unselected.\n   */\n  setSelection(ids) {\n    if (ids == undefined) { \n      ids = [];\n    }\n    \n    if (!Array.isArray(ids)) {\n      ids = [ids];\n    }\n  \n\n    const idsToDeselect = this.selection.filter(id => ids.indexOf(id) === -1);\n\n    // unselect currently selected items\n    for (let selectedId of idsToDeselect) {\n      const item = this.getItemById(selectedId);\n      if (item) {\n        item.unselect();\n      }\n    }\n  \n    // select items\n    this.selection = [ ...ids ];\n    for (let id of ids) {\n      const item = this.getItemById(id);\n      if (item) {\n        item.select();\n      }\n    }\n  }\n\n  /**\n   * Get the selected items by their id\n   * @return {Array} ids  The ids of the selected items\n   */\n  getSelection() {\n    return this.selection.concat([]);\n  }\n\n  /**\n   * Get the id's of the currently visible items.\n   * @returns {Array} The ids of the visible items\n   */\n  getVisibleItems() {\n    const range = this.body.range.getRange();\n    let right;\n    let left;\n\n    if (this.options.rtl) { \n      right  = this.body.util.toScreen(range.start);\n      left = this.body.util.toScreen(range.end);\n    } else {\n      left  = this.body.util.toScreen(range.start);\n      right = this.body.util.toScreen(range.end);\n    }\n\n    const ids = [];\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        const group = this.groups[groupId];\n        const rawVisibleItems = group.isVisible ? group.visibleItems : [];\n\n        // filter the \"raw\" set with visibleItems into a set which is really\n        // visible by pixels\n        for (const item of rawVisibleItems) {\n          // TODO: also check whether visible vertically\n          if (this.options.rtl) { \n            if ((item.right < left) && (item.right + item.width > right)) {\n              ids.push(item.id);\n            }\n          } else {\n            if ((item.left < right) && (item.left + item.width > left)) {\n              ids.push(item.id);\n            }\n          }\n        }\n      }\n    }\n\n    return ids;\n  }\n\n  /**\n   * Get the id's of the currently visible groups.\n   * @returns {Array} The ids of the visible groups\n   */\n  getVisibleGroups() {\n    const ids = [];\n\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        const group = this.groups[groupId];\n        if (group.isVisible) {\n          ids.push(groupId)\n        }\n      }\n    }\n\n    return ids;\n  }\n  \n  /**\n   * get item by id\n   * @param {string} id\n   * @return {object} item\n   */\n  getItemById(id) {\n    return this.items[id] || this.clusters.find(cluster => cluster.id === id);\n  } \n\n  /**\n   * Deselect a selected item\n   * @param {string | number} id\n   * @private\n   */\n  _deselect(id) {\n    const selection = this.selection;\n    for (let i = 0, ii = selection.length; i < ii; i++) {\n      if (selection[i] == id) { // non-strict comparison!\n        selection.splice(i, 1);\n        break;\n      }\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const margin = this.options.margin;\n    const range = this.body.range;\n    const asSize = util.option.asSize;\n    const options = this.options;\n    const orientation = options.orientation.item;\n    let resized = false;\n    const frame = this.dom.frame;\n\n    // recalculate absolute position (before redrawing groups)\n    this.props.top = this.body.domProps.top.height + this.body.domProps.border.top;\n\n    if (this.options.rtl) {\n      this.props.right = this.body.domProps.right.width + this.body.domProps.border.right;\n    } else {\n      this.props.left = this.body.domProps.left.width + this.body.domProps.border.left;\n    }\n\n    // update class name\n    frame.className = 'vis-itemset';\n\n    if (this.options.cluster) {\n      this._clusterItems();\n    }\n\n    // reorder the groups (if needed)\n    resized = this._orderGroups() || resized;\n\n    // check whether zoomed (in that case we need to re-stack everything)\n    // TODO: would be nicer to get this as a trigger from Range\n    const visibleInterval = range.end - range.start;\n    const zoomed = (visibleInterval != this.lastVisibleInterval) || (this.props.width != this.props.lastWidth);\n    const scrolled = range.start != this.lastRangeStart;\n    const changedStackOption = options.stack != this.lastStack;\n    const changedStackSubgroupsOption = options.stackSubgroups != this.lastStackSubgroups;\n    const forceRestack = (zoomed || scrolled || changedStackOption || changedStackSubgroupsOption);\n    this.lastVisibleInterval = visibleInterval;\n    this.lastRangeStart = range.start;\n    this.lastStack = options.stack;\n    this.lastStackSubgroups = options.stackSubgroups;\n\n    this.props.lastWidth = this.props.width;\n    const firstGroup = this._firstGroup();\n    const firstMargin = {\n      item: margin.item,\n      axis: margin.axis\n    };\n    const nonFirstMargin = {\n      item: margin.item,\n      axis: margin.item.vertical / 2\n    };\n    let height = 0;\n    const minHeight = margin.axis + margin.item.vertical;\n\n    // redraw the background group\n    this.groups[BACKGROUND].redraw(range, nonFirstMargin, forceRestack);\n\n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n\n    // collect redraw functions\n    util.forEach(this.groups, (group, key) => {\n      if (key === BACKGROUND) return;\n      const groupMargin = group == firstGroup ? firstMargin : nonFirstMargin;\n      const returnQueue = true;\n      redrawQueue[key] = group.redraw(range, groupMargin, forceRestack, returnQueue);\n      redrawQueueLength = redrawQueue[key].length;\n    });\n\n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      const redrawResults = {};\n\n      for (let i = 0; i < redrawQueueLength; i++) {\n        util.forEach(redrawQueue, (fns, key) => {\n          redrawResults[key] = fns[i]();\n        });\n      }\n\n      // redraw all regular groups\n      util.forEach(this.groups, (group, key) => {\n        if (key === BACKGROUND) return;\n        const groupResized = redrawResults[key];\n        resized = groupResized || resized;\n        height += group.height;\n      });\n      height = Math.max(height, minHeight);\n    }\n\n    height = Math.max(height, minHeight);\n\n    // update frame height\n    frame.style.height  = asSize(height);\n\n    // calculate actual size\n    this.props.width = frame.offsetWidth;\n    this.props.height = height;\n\n    // reposition axis\n    this.dom.axis.style.top = asSize((orientation == 'top') ?\n        (this.body.domProps.top.height + this.body.domProps.border.top) :\n        (this.body.domProps.top.height + this.body.domProps.centerContainer.height));\n    if (this.options.rtl) {\n      this.dom.axis.style.right = '0';\n    } else {\n      this.dom.axis.style.left = '0';\n    }\n\n    this.hammer.get('press').set({time: this.options.longSelectPressTime});\n\n    this.initialItemSetDrawn = true;\n    // check if this component is resized\n    resized = this._isResized() || resized;\n\n    return resized;\n  }\n\n  /**\n   * Get the first group, aligned with the axis\n   * @return {Group | null} firstGroup\n   * @private\n   */\n  _firstGroup() {\n    const firstGroupIndex = (this.options.orientation.item == 'top') ? 0 : (this.groupIds.length - 1);\n    const firstGroupId = this.groupIds[firstGroupIndex];\n    const firstGroup = this.groups[firstGroupId] || this.groups[UNGROUPED];\n\n    return firstGroup || null;\n  }\n\n  /**\n   * Create or delete the group holding all ungrouped items. This group is used when\n   * there are no groups specified.\n   * @protected\n   */\n  _updateUngrouped() {\n    let ungrouped = this.groups[UNGROUPED];\n    let item;\n    let itemId;\n\n    if (this.groupsData) {\n      // remove the group holding all ungrouped items\n      if (ungrouped) {\n        ungrouped.dispose();\n        delete this.groups[UNGROUPED];\n\n        for (itemId in this.items) {\n          if (this.items.hasOwnProperty(itemId)) {\n            item = this.items[itemId];\n            item.parent && item.parent.remove(item);\n            const groupId = this.getGroupId(item.data);\n            const group = this.groups[groupId];\n            group && group.add(item) || item.hide();\n          }\n        }\n      }\n    }\n    else {\n      // create a group holding all (unfiltered) items\n      if (!ungrouped) {\n        const id = null;\n        const data = null;\n        ungrouped = new Group(id, data, this);\n        this.groups[UNGROUPED] = ungrouped;\n\n        for (itemId in this.items) {\n          if (this.items.hasOwnProperty(itemId)) {\n            item = this.items[itemId];\n            ungrouped.add(item);\n          }\n        }\n\n        ungrouped.show();\n      }\n    }\n  }\n\n  /**\n   * Get the element for the labelset\n   * @return {HTMLElement} labelSet\n   */\n  getLabelSet() {\n    return this.dom.labelSet;\n  }\n\n  /**\n   * Set items\n   * @param {vis.DataSet | null} items\n   */\n  setItems(items) {\n    this.itemsSettingTime = new Date();\n    const me = this;\n    let ids;\n    const oldItemsData = this.itemsData;\n\n    // replace the dataset\n    if (!items) {\n      this.itemsData = null;\n    }\n    else if (items instanceof DataSet || items instanceof DataView) {\n      this.itemsData = typeCoerceDataSet(items);\n    }\n    else {\n      throw new TypeError('Data must be an instance of DataSet or DataView');\n    }\n\n    if (oldItemsData) {\n      // unsubscribe from old dataset\n      util.forEach(this.itemListeners, (callback, event) => {\n        oldItemsData.off(event, callback);\n      });\n\n      // stop maintaining a coerced version of the old data set\n      oldItemsData.dispose()\n\n      // remove all drawn items\n      ids = oldItemsData.getIds();\n      this._onRemove(ids);\n    }\n\n    if (this.itemsData) {\n      // subscribe to new dataset\n      const id = this.id;\n      util.forEach(this.itemListeners, (callback, event) => {\n        me.itemsData.on(event, callback, id);\n      });\n\n      // add all new items\n      ids = this.itemsData.getIds();\n      this._onAdd(ids);\n\n      // update the group holding all ungrouped items\n      this._updateUngrouped();\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Get the current items\n   * @returns {vis.DataSet | null}\n   */\n  getItems() {\n    return this.itemsData != null ? this.itemsData.rawDS : null;\n  }\n\n  /**\n   * Set groups\n   * @param {vis.DataSet} groups\n   */\n  setGroups(groups) {\n    const me = this;\n    let ids;\n\n    // unsubscribe from current dataset\n    if (this.groupsData) {\n      util.forEach(this.groupListeners, (callback, event) => {\n        me.groupsData.off(event, callback);\n      });\n\n      // remove all drawn groups\n      ids = this.groupsData.getIds();\n      this.groupsData = null;\n      this._onRemoveGroups(ids); // note: this will cause a redraw\n    }\n\n    // replace the dataset\n    if (!groups) {\n      this.groupsData = null;\n    }\n    else if (groups instanceof DataSet || groups instanceof DataView) {\n      this.groupsData = groups;\n    }\n    else {\n      throw new TypeError('Data must be an instance of DataSet or DataView');\n    }\n\n    if (this.groupsData) {\n      // go over all groups nesting\n      let groupsData = this.groupsData;\n      if (this.groupsData instanceof DataView) {\n        groupsData = this.groupsData.getDataSet()\n      }\n\n      groupsData.get().forEach(group => {\n        if (group.nestedGroups) {\n          group.nestedGroups.forEach(nestedGroupId => {\n            const updatedNestedGroup = groupsData.get(nestedGroupId);\n            updatedNestedGroup.nestedInGroup = group.id;\n            if (group.showNested == false) {\n              updatedNestedGroup.visible = false;\n            }\n            groupsData.update(updatedNestedGroup);\n          })\n        }\n      });\n\n      // subscribe to new dataset\n      const id = this.id;\n      util.forEach(this.groupListeners, (callback, event) => {\n        me.groupsData.on(event, callback, id);\n      });\n\n      // draw all ms\n      ids = this.groupsData.getIds();\n      this._onAddGroups(ids);\n    }\n\n    // update the group holding all ungrouped items\n    this._updateUngrouped();\n\n    // update the order of all items in each group\n    this._order();\n\n    if (this.options.cluster) {\n      this.clusterGenerator.updateData();\n      this._clusterItems();\n      this.markDirty({ refreshItems: true, restackGroups: true });\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Get the current groups\n   * @returns {vis.DataSet | null} groups\n   */\n  getGroups() {\n    return this.groupsData;\n  }\n\n  /**\n   * Remove an item by its id\n   * @param {string | number} id\n   */\n  removeItem(id) {\n    const item = this.itemsData.get(id);\n\n    if (item) {\n      // confirm deletion\n      this.options.onRemove(item, item => {\n        if (item) {\n          // remove by id here, it is possible that an item has no id defined\n          // itself, so better not delete by the item itself\n          this.itemsData.remove(id);\n        }\n      });\n    }\n  }\n\n  /**\n   * Get the time of an item based on it's data and options.type\n   * @param {Object} itemData\n   * @returns {string} Returns the type\n   * @private\n   */\n  _getType(itemData) {\n    return itemData.type || this.options.type || (itemData.end ? 'range' : 'box');\n  }\n\n  /**\n   * Get the group id for an item\n   * @param {Object} itemData\n   * @returns {string} Returns the groupId\n   * @private\n   */\n  getGroupId(itemData) {\n    const type = this._getType(itemData);\n    if (type == 'background' && itemData.group == undefined) {\n     return BACKGROUND;\n    }\n    else {\n      return this.groupsData ? itemData.group : UNGROUPED;\n    }\n  }\n\n  /**\n   * Handle updated items\n   * @param {number[]} ids\n   * @protected\n   */\n  _onUpdate(ids) {\n    const me = this;\n\n    ids.forEach(id => {\n      const itemData = me.itemsData.get(id);\n      let item = me.items[id];\n      const type = itemData ? me._getType(itemData) : null;\n\n      const constructor = ItemSet.types[type];\n      let selected;\n\n      if (item) {\n        // update item   \t\n        if (!constructor || !(item instanceof constructor)) {\n          // item type has changed, delete the item and recreate it\n          selected = item.selected; // preserve selection of this item\n          me._removeItem(item);\n          item = null;\n        }\n        else {\n          me._updateItem(item, itemData);\n        }\n      }\n\n      if (!item && itemData) {\n        // create item\n        if (constructor) {\n          item = new constructor(itemData, me.conversion, me.options);\n          item.id = id; // TODO: not so nice setting id afterwards\n\n          me._addItem(item);\n          if (selected) {\n            this.selection.push(id);\n            item.select();\n          }\n        }\n        else {\n          throw new TypeError(`Unknown item type \"${type}\"`);\n        }\n      }\n    });\n\n    this._order();\n    \n    if (this.options.cluster) {\n      this.clusterGenerator.setItems(this.items, { applyOnChangedLevel: false });\n      this._clusterItems();\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Handle removed items\n   * @param {number[]} ids\n   * @protected\n   */\n  _onRemove(ids) {\n    let count = 0;\n    const me = this;\n    ids.forEach(id => {\n      const item = me.items[id];\n      if (item) {\n        count++;\n        me._removeItem(item);\n      }\n    });\n\n    if (count) {\n      // update order\n      this._order();\n      this.body.emitter.emit('_change', {queue: true});\n    }\n  }\n\n  /**\n   * Update the order of item in all groups\n   * @private\n   */\n  _order() {\n    // reorder the items in all groups\n    // TODO: optimization: only reorder groups affected by the changed items\n    util.forEach(this.groups, group => {\n      group.order();\n    });\n  }\n\n  /**\n   * Handle updated groups\n   * @param {number[]} ids\n   * @private\n   */\n  _onUpdateGroups(ids) {\n    this._onAddGroups(ids);\n  }\n\n  /**\n   * Handle changed groups (added or updated)\n   * @param {number[]} ids\n   * @private\n   */\n  _onAddGroups(ids) {\n    const me = this;\n\n    ids.forEach(id => {\n      const groupData = me.groupsData.get(id);\n      let group = me.groups[id];\n\n      if (!group) {\n        // check for reserved ids\n        if (id == UNGROUPED || id == BACKGROUND) {\n          throw new Error(`Illegal group id. ${id} is a reserved id.`);\n        }\n\n        const groupOptions = Object.create(me.options);\n        util.extend(groupOptions, {\n          height: null\n        });\n\n        group = new Group(id, groupData, me);\n        me.groups[id] = group;\n\n        // add items with this groupId to the new group\n        for (const itemId in me.items) {\n          if (me.items.hasOwnProperty(itemId)) {\n            const item = me.items[itemId];\n            if (item.data.group == id) {\n              group.add(item);\n            }\n          }\n        }\n\n        group.order();\n        group.show();\n      }\n      else {\n        // update group\n        group.setData(groupData);\n      }\n    });\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Handle removed groups\n   * @param {number[]} ids\n   * @private\n   */\n  _onRemoveGroups(ids) {\n    ids.forEach(id => {\n      const group = this.groups[id];\n\n      if (group) {\n        group.dispose();\n        delete this.groups[id];\n      }\n    });\n\n    if (this.options.cluster) {\n      this.clusterGenerator.updateData();\n      this._clusterItems();\n    } \n\n    this.markDirty({ restackGroups: !!this.options.cluster });\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Reorder the groups if needed\n   * @return {boolean} changed\n   * @private\n   */\n  _orderGroups() {\n    if (this.groupsData) {\n      // reorder the groups\n      let groupIds = this.groupsData.getIds({\n        order: this.options.groupOrder\n      });\n\n      groupIds = this._orderNestedGroups(groupIds);\n\n      const changed = !util.equalArray(groupIds, this.groupIds);\n      if (changed) {\n        // hide all groups, removes them from the DOM\n        const groups = this.groups;\n        groupIds.forEach(groupId => {\n          groups[groupId].hide();\n        });\n\n        // show the groups again, attach them to the DOM in correct order\n        groupIds.forEach(groupId => {\n          groups[groupId].show();\n        });\n\n        this.groupIds = groupIds;\n      }\n\n      return changed;\n    }\n    else {\n      return false;\n    }\n  }\n\n  /**\n   * Reorder the nested groups\n   *\n   * @param {Array.<number>} groupIds\n   * @returns {Array.<number>}\n   * @private\n   */\n  _orderNestedGroups(groupIds) {\n    /**\n     * Recursively order nested groups\n     *\n     * @param {ItemSet} t\n     * @param {Array.<number>} groupIds\n     * @returns {Array.<number>}\n     * @private\n     */\n    function getOrderedNestedGroups(t, groupIds) {\n      let result = [];\n      groupIds.forEach(groupId => {\n        result.push(groupId);\n        const groupData = t.groupsData.get(groupId);\n        if (groupData.nestedGroups) {\n          const nestedGroupIds = t.groupsData.get({\n            filter(nestedGroup) {\n              return nestedGroup.nestedInGroup == groupId;\n            },\n            order: t.options.groupOrder\n          }).map(nestedGroup => nestedGroup.id);\n          result = result.concat(getOrderedNestedGroups(t, nestedGroupIds));\n        }\n      });\n\n      return result;\n    }\n\n    const topGroupIds = groupIds.filter(groupId => !this.groupsData.get(groupId).nestedInGroup);\n\n    return getOrderedNestedGroups(this, topGroupIds);\n  }\n\n  /**\n   * Add a new item\n   * @param {Item} item\n   * @private\n   */\n  _addItem(item) {\n    this.items[item.id] = item;\n\n    // add to group\n    const groupId = this.getGroupId(item.data);\n    const group = this.groups[groupId];\n\n    if (!group) {\n      item.groupShowing = false;\n    } else if (group && group.data && group.data.showNested) {\n      item.groupShowing = true;\n    }\n\n    if (group) group.add(item);\n  }\n\n  /**\n   * Update an existing item\n   * @param {Item} item\n   * @param {Object} itemData\n   * @private\n   */\n  _updateItem(item, itemData) {\n    // update the items data (will redraw the item when displayed)\n    item.setData(itemData);\n\n    const groupId = this.getGroupId(item.data);\n    const group = this.groups[groupId];\n    if (!group) {\n      item.groupShowing = false;\n    } else if (group && group.data && group.data.showNested) {\n      item.groupShowing = true;\n    }\n  }\n\n  /**\n   * Delete an item from the ItemSet: remove it from the DOM, from the map\n   * with items, and from the map with visible items, and from the selection\n   * @param {Item} item\n   * @private\n   */\n  _removeItem(item) {\n    // remove from DOM\n    item.hide();\n\n    // remove from items\n    delete this.items[item.id];\n\n    // remove from selection\n    const index = this.selection.indexOf(item.id);\n    if (index != -1) this.selection.splice(index, 1);\n\n    // remove from group\n    item.parent && item.parent.remove(item);\n\n    // remove Tooltip from DOM\n    if (this.popup != null) {\n      this.popup.hide();\n    }\n  }\n\n  /**\n   * Create an array containing all items being a range (having an end date)\n   * @param {Array.<Object>} array\n   * @returns {Array}\n   * @private\n   */\n  _constructByEndArray(array) {\n    const endArray = [];\n\n    for (let i = 0; i < array.length; i++) {\n      if (array[i] instanceof RangeItem) {\n        endArray.push(array[i]);\n      }\n    }\n    return endArray;\n  }\n\n  /**\n   * Register the clicked item on touch, before dragStart is initiated.\n   *\n   * dragStart is initiated from a mousemove event, AFTER the mouse/touch is\n   * already moving. Therefore, the mouse/touch can sometimes be above an other\n   * DOM element than the item itself.\n   *\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {\n    // store the touched item, used in _onDragStart\n    this.touchParams.item = this.itemFromTarget(event);\n    this.touchParams.dragLeftItem = event.target.dragLeftItem || false;\n    this.touchParams.dragRightItem = event.target.dragRightItem || false;\n    this.touchParams.itemProps = null;\n  }\n\n  /**\n   * Given an group id, returns the index it has.\n   *\n   * @param {number} groupId\n   * @returns {number} index / groupId\n   * @private\n   */\n  _getGroupIndex(groupId) {\n      for (let i = 0; i < this.groupIds.length; i++) {\n          if (groupId == this.groupIds[i])\n              return i;\n      }\n  }\n\n  /**\n   * Start dragging the selected events\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    if (this.touchParams.itemIsDragging) { return; }\n    const item = this.touchParams.item || null;\n    const me = this;\n    let props;\n\n    if (item && (item.selected || this.options.itemsAlwaysDraggable.item)) {\n\n      if (this.options.editable.overrideItems &&\n          !this.options.editable.updateTime &&\n          !this.options.editable.updateGroup) {\n        return;\n      }\n\n      // override options.editable\n      if ((item.editable != null && !item.editable.updateTime && !item.editable.updateGroup)\n          && !this.options.editable.overrideItems) {\n        return;\n      }\n\n      const dragLeftItem = this.touchParams.dragLeftItem;\n      const dragRightItem = this.touchParams.dragRightItem;\n      this.touchParams.itemIsDragging = true;\n      this.touchParams.selectedItem = item;\n\n      if (dragLeftItem) {\n        props = {\n          item: dragLeftItem,\n          initialX: event.center.x,\n          dragLeft:  true,\n          data: this._cloneItemData(item.data)\n        };\n\n        this.touchParams.itemProps = [props];\n      } else if (dragRightItem) {\n        props = {\n          item: dragRightItem,\n          initialX: event.center.x,\n          dragRight: true,\n          data: this._cloneItemData(item.data)\n        };\n\n        this.touchParams.itemProps = [props];\n      } else if (this.options.editable.add && (event.srcEvent.ctrlKey || event.srcEvent.metaKey)) {\n        // create a new range item when dragging with ctrl key down\n        this._onDragStartAddItem(event);\n      } else {\n        if(this.groupIds.length < 1) {\n          // Mitigates a race condition if _onDragStart() is\n          // called after markDirty() without redraw() being called between.\n          this.redraw();\n        }\n        \n        const baseGroupIndex = this._getGroupIndex(item.data.group);\n\n        const itemsToDrag = (this.options.itemsAlwaysDraggable.item && !item.selected) ? [item.id] : this.getSelection();\n\n        this.touchParams.itemProps = itemsToDrag.map(id => {\n          const item = me.items[id];\n          const groupIndex = me._getGroupIndex(item.data.group);\n          return {\n            item,\n            initialX: event.center.x,\n            groupOffset: baseGroupIndex-groupIndex,\n            data: this._cloneItemData(item.data)\n          };\n        });\n      }\n\n      event.stopPropagation();\n    } else if (this.options.editable.add && (event.srcEvent.ctrlKey || event.srcEvent.metaKey)) {\n      // create a new range item when dragging with ctrl key down\n      this._onDragStartAddItem(event);\n    }\n  }\n\n  /**\n   * Start creating a new range item by dragging.\n   * @param {Event} event\n   * @private\n   */\n  _onDragStartAddItem(event) {\n    const snap = this.options.snap || null;\n    const frameRect = this.dom.frame.getBoundingClientRect()\n\n    // plus (if rtl) 10 to compensate for the drag starting as soon as you've moved 10px\n    const x = this.options.rtl ? frameRect.right - event.center.x  + 10 : event.center.x - frameRect.left - 10;\n\n    const time = this.body.util.toTime(x);\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const start = snap ? snap(time, scale, step) : time;\n    const end = start;\n\n    const itemData = {\n      type: 'range',\n      start,\n      end,\n      content: 'new item'\n    };\n\n    const id = randomUUID();\n    itemData[this.itemsData.idProp] = id;\n\n    const group = this.groupFromTarget(event);\n    if (group) {\n      itemData.group = group.groupId;\n    }\n    const newItem = new RangeItem(itemData, this.conversion, this.options);\n    newItem.id = id; // TODO: not so nice setting id afterwards\n    newItem.data = this._cloneItemData(itemData);\n    this._addItem(newItem);\n    this.touchParams.selectedItem = newItem;\n    \n    const props = {\n      item: newItem,\n      initialX: event.center.x,\n      data: newItem.data\n    };\n\n    if (this.options.rtl) {\n      props.dragLeft = true;\n    } else {\n      props.dragRight = true;\n    }\n    this.touchParams.itemProps = [props];\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Drag selected items\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (this.popup != null && this.options.showTooltips && !this.popup.hidden) {\n      // this.popup.hide();\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.center.x - containerRect.left + container.offsetLeft,\n        event.center.y - containerRect.top + container.offsetTop\n      );\n      this.popup.show(); // redraw\n    }\n    \n    if (this.touchParams.itemProps) {\n      event.stopPropagation();\n\n      const me = this;\n      const snap = this.options.snap || null;\n      const domRootOffsetLeft = this.body.dom.root.offsetLeft;\n      const xOffset = this.options.rtl ? domRootOffsetLeft + this.body.domProps.right.width : domRootOffsetLeft + this.body.domProps.left.width;\n      const scale = this.body.util.getScale();\n      const step = this.body.util.getStep();\n\n      //only calculate the new group for the item that's actually dragged\n      const selectedItem = this.touchParams.selectedItem;\n      const updateGroupAllowed = ((this.options.editable.overrideItems || selectedItem.editable == null) && this.options.editable.updateGroup) ||\n                               (!this.options.editable.overrideItems && selectedItem.editable != null && selectedItem.editable.updateGroup);\n      let newGroupBase = null;\n      if (updateGroupAllowed && selectedItem) {\n        if (selectedItem.data.group != undefined) {\n          // drag from one group to another\n          const group = me.groupFromTarget(event);\n          if (group) {\n            //we know the offset for all items, so the new group for all items\n            //will be relative to this one.\n            newGroupBase = this._getGroupIndex(group.groupId);\n          }\n        }\n      }\n\n      // move\n      this.touchParams.itemProps.forEach(props => {\n        const current = me.body.util.toTime(event.center.x - xOffset);\n        const initial = me.body.util.toTime(props.initialX - xOffset);\n        let offset;\n        let initialStart;\n        let initialEnd;\n        let start;\n        let end;\n\n        if (this.options.rtl) {\n          offset = -(current - initial); // ms\n        } else {\n          offset = (current - initial); // ms\n        }\n\n        let itemData = this._cloneItemData(props.item.data); // clone the data\n        if (props.item.editable != null\n          && !props.item.editable.updateTime\n          && !props.item.editable.updateGroup\n          && !me.options.editable.overrideItems) {\n          return;\n        }\n\n        const updateTimeAllowed = ((this.options.editable.overrideItems || selectedItem.editable == null) && this.options.editable.updateTime) ||\n                                 (!this.options.editable.overrideItems && selectedItem.editable != null && selectedItem.editable.updateTime);\n        if (updateTimeAllowed) {\n          if (props.dragLeft) {\n            // drag left side of a range item\n            if (this.options.rtl) {\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                end = new Date(initialEnd.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.end = snap ? snap(end, scale, step) : end;\n              }\n            } else {\n              if (itemData.start != undefined) {\n                initialStart = util.convert(props.data.start, 'Date');\n                start = new Date(initialStart.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            }\n          }\n          else if (props.dragRight) {\n            // drag right side of a range item\n            if (this.options.rtl) {\n              if (itemData.start != undefined) {\n                initialStart = util.convert(props.data.start, 'Date');\n                start = new Date(initialStart.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            } else {\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                end = new Date(initialEnd.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.end = snap ? snap(end, scale, step) : end;\n              }\n            }\n          }\n          else {\n            // drag both start and end\n            if (itemData.start != undefined) {\n\n              initialStart = util.convert(props.data.start, 'Date').valueOf();\n              start = new Date(initialStart + offset);\n\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                const duration  = initialEnd.valueOf() - initialStart.valueOf();\n\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n                itemData.end   = new Date(itemData.start.valueOf() + duration);\n              }\n              else {\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            }\n          }\n        }\n\n        if (updateGroupAllowed && (!props.dragLeft && !props.dragRight) && newGroupBase!=null) {\n          if (itemData.group != undefined) {\n            let newOffset = newGroupBase - props.groupOffset;\n\n            //make sure we stay in bounds\n            newOffset = Math.max(0, newOffset);\n            newOffset = Math.min(me.groupIds.length-1, newOffset);\n            itemData.group = me.groupIds[newOffset];\n          }\n        }\n\n        // confirm moving the item\n        itemData = this._cloneItemData(itemData);  // convert start and end to the correct type\n        me.options.onMoving(itemData, itemData => {\n          if (itemData) {\n            props.item.setData(this._cloneItemData(itemData, 'Date'));\n          }\n        });\n      });\n      \n      this.body.emitter.emit('_change');\n    }\n  }\n\n  /**\n   * Move an item to another group\n   * @param {Item} item\n   * @param {string | number} groupId\n   * @private\n   */\n  _moveToGroup(item, groupId) {\n    const group = this.groups[groupId];\n    if (group && group.groupId != item.data.group) {\n      const oldGroup = item.parent;\n      oldGroup.remove(item);\n      oldGroup.order();\n      \n      item.data.group = group.groupId;\n      \n      group.add(item);\n      group.order();\n    }\n  }\n\n  /**\n   * End of dragging selected items\n   * @param {Event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    this.touchParams.itemIsDragging = false;\n    if (this.touchParams.itemProps) {\n      event.stopPropagation();\n\n      const me = this;\n      const itemProps = this.touchParams.itemProps;\n      this.touchParams.itemProps = null;\n\n      itemProps.forEach(props => {\n        const id = props.item.id;\n        const exists = me.itemsData.get(id) != null;\n\n        if (!exists) {\n          // add a new item\n          me.options.onAdd(props.item.data, itemData => {\n            me._removeItem(props.item); // remove temporary item\n            if (itemData) {\n              me.itemsData.add(itemData);\n            }\n\n            // force re-stacking of all items next redraw\n            me.body.emitter.emit('_change');\n          });\n        }\n        else {\n          // update existing item\n          const itemData = this._cloneItemData(props.item.data); // convert start and end to the correct type\n          me.options.onMove(itemData, itemData => {\n            if (itemData) {\n              // apply changes\n              itemData[this.itemsData.idProp] = id; // ensure the item contains its id (can be undefined)\n              this.itemsData.update(itemData);\n            }\n            else {\n              // restore original values\n              props.item.setData(props.data);\n\n              me.body.emitter.emit('_change');\n            }\n          });\n        }\n      });\n    }\n  }\n\n  /**\n   * On group click\n   * @param {Event} event\n   * @private\n   */\n  _onGroupClick(event) {\n    const group = this.groupFromTarget(event);\n    setTimeout(() => {\n      this.toggleGroupShowNested(group);\n    }, 1)\n  }\n  \n  /**\n   * Toggle show nested\n   * @param {object} group\n   * @param {boolean} force\n   */\n  toggleGroupShowNested(group, force = undefined) {\n\n    if (!group || !group.nestedGroups) return;\n\n    const groupsData = this.groupsData.getDataSet();\n\n    if (force != undefined) {\n      group.showNested = !!force;\n    } else {\n      group.showNested = !group.showNested;\n    }\n\n    let nestingGroup = groupsData.get(group.groupId);\n    nestingGroup.showNested = group.showNested;\n\n    let fullNestedGroups = group.nestedGroups;\n    let nextLevel = fullNestedGroups;\n    while (nextLevel.length > 0) {\n      let current = nextLevel;\n      nextLevel = [];\n      for (let i = 0; i < current.length; i++) {\n        let node = groupsData.get(current[i]);\n        if (node.nestedGroups) {\n          nextLevel = nextLevel.concat(node.nestedGroups);\n        }\n      }\n      if (nextLevel.length > 0) {\n        fullNestedGroups = fullNestedGroups.concat(nextLevel);\n      }\n    }\n    let nestedGroups = groupsData.get(fullNestedGroups).map(function (nestedGroup) {\n        if (nestedGroup.visible == undefined) {\n          nestedGroup.visible = true;\n        }\n        nestedGroup.visible = !!nestingGroup.showNested;\n        return nestedGroup;\n      });\n\n    groupsData.update(nestedGroups.concat(nestingGroup));\n\n    if (nestingGroup.showNested) {\n      util.removeClassName(group.dom.label, 'collapsed');\n      util.addClassName(group.dom.label, 'expanded');\n    } else {\n      util.removeClassName(group.dom.label, 'expanded');\n      util.addClassName(group.dom.label, 'collapsed');\n    }\n  }\n  \n  /**\n   * Toggle group drag classname\n   * @param {object} group\n   */\n  toggleGroupDragClassName(group) {\n    group.dom.label.classList.toggle('vis-group-is-dragging');\n    group.dom.foreground.classList.toggle('vis-group-is-dragging');\n  }\n  \n  /**\n   * on drag start\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onGroupDragStart(event) {\n    if (this.groupTouchParams.isDragging) return;\n\n    if (this.options.groupEditable.order) {\n      this.groupTouchParams.group = this.groupFromTarget(event);\n      \n      if (this.groupTouchParams.group) {\n        event.stopPropagation();      \n        \n        this.groupTouchParams.isDragging = true;\n        this.toggleGroupDragClassName(this.groupTouchParams.group);\n        \n        this.groupTouchParams.originalOrder = this.groupsData.getIds({\n          order: this.options.groupOrder\n        });\n      }\n    }\n  }\n\n  /**\n   * on drag\n   * @param {Event} event\n   * @return {void}\n   * @private\n   */\n  _onGroupDrag(event) {\n      if (this.options.groupEditable.order && this.groupTouchParams.group) {\n          event.stopPropagation();\n          \n      let groupsData = this.groupsData;\n      if (this.groupsData instanceof DataView) {\n        groupsData = this.groupsData.getDataSet()\n      }\n          // drag from one group to another\n          const group = this.groupFromTarget(event);\n          \n          // try to avoid toggling when groups differ in height\n          if (group && group.height != this.groupTouchParams.group.height) {\n              const movingUp = (group.top < this.groupTouchParams.group.top);\n              const clientY = event.center ? event.center.y : event.clientY;\n              const targetGroup = group.dom.foreground.getBoundingClientRect()\n              const draggedGroupHeight = this.groupTouchParams.group.height;\n              if (movingUp) {\n                  // skip swapping the groups when the dragged group is not below clientY afterwards\n                  if (targetGroup.top + draggedGroupHeight < clientY) {\n                      return;\n                  }\n              } else {\n                  const targetGroupHeight = group.height;\n                  // skip swapping the groups when the dragged group is not below clientY afterwards\n                  if (targetGroup.top + targetGroupHeight - draggedGroupHeight > clientY) {\n                      return;\n                  }\n              }\n          }\n          \n          if (group && group != this.groupTouchParams.group) {\n              const targetGroup = groupsData.get(group.groupId);\n              const draggedGroup = groupsData.get(this.groupTouchParams.group.groupId);\n              \n              // switch groups\n              if (draggedGroup && targetGroup) {\n                  this.options.groupOrderSwap(draggedGroup, targetGroup, groupsData);\n                  groupsData.update(draggedGroup);\n                  groupsData.update(targetGroup);\n              }\n              \n              // fetch current order of groups\n              const newOrder = groupsData.getIds({\n                order: this.options.groupOrder\n              });\n\n              \n              // in case of changes since _onGroupDragStart\n              if (!util.equalArray(newOrder, this.groupTouchParams.originalOrder)) {\n                  const origOrder = this.groupTouchParams.originalOrder;\n                  const draggedId = this.groupTouchParams.group.groupId;\n                  const numGroups = Math.min(origOrder.length, newOrder.length);\n                  let curPos = 0;\n                  let newOffset = 0;\n                  let orgOffset = 0;\n                  while (curPos < numGroups) {\n                      // as long as the groups are where they should be step down along the groups order\n                      while ((curPos+newOffset) < numGroups \n                          && (curPos+orgOffset) < numGroups \n                          && newOrder[curPos+newOffset] == origOrder[curPos+orgOffset]) {\n                          curPos++;\n                      }\n                      \n                      // all ok\n                      if (curPos+newOffset >= numGroups) {\n                          break;\n                      }\n                      \n                      // not all ok\n                      // if dragged group was move upwards everything below should have an offset\n                      if (newOrder[curPos+newOffset] == draggedId) {\n                          newOffset = 1;\n\n                      }\n                      // if dragged group was move downwards everything above should have an offset\n                      else if (origOrder[curPos+orgOffset] == draggedId) {\n                          orgOffset = 1;\n\n                      } \n                      // found a group (apart from dragged group) that has the wrong position -> switch with the \n                      // group at the position where other one should be, fix index arrays and continue\n                      else {\n                          const slippedPosition = newOrder.indexOf(origOrder[curPos+orgOffset]);\n                          const switchGroup = groupsData.get(newOrder[curPos+newOffset]);\n                          const shouldBeGroup = groupsData.get(origOrder[curPos+orgOffset]);\n                          this.options.groupOrderSwap(switchGroup, shouldBeGroup, groupsData);\n                          groupsData.update(switchGroup);\n                          groupsData.update(shouldBeGroup);\n                          \n                          const switchGroupId = newOrder[curPos+newOffset];\n                          newOrder[curPos+newOffset] = origOrder[curPos+orgOffset];\n                          newOrder[slippedPosition] = switchGroupId;\n                          \n                          curPos++;\n                      }\n                  }\n              }\n              \n          }\n      }\n  }\n\n  /**\n   * on drag end\n   * @param {Event} event\n   * @return {void}\n   * @private\n   */\n  _onGroupDragEnd(event) {\n    this.groupTouchParams.isDragging = false;\n\n    if (this.options.groupEditable.order && this.groupTouchParams.group) {\n      event.stopPropagation();\n          \n      // update existing group\n      const me = this;\n      const id = me.groupTouchParams.group.groupId;\n      const dataset = me.groupsData.getDataSet();\n      const groupData = util.extend({}, dataset.get(id)); // clone the data\n      me.options.onMoveGroup(groupData, groupData => {\n        if (groupData) {\n          // apply changes\n          groupData[dataset._idProp] = id; // ensure the group contains its id (can be undefined)\n          dataset.update(groupData);\n        }\n        else {\n\n          // fetch current order of groups\n          const newOrder = dataset.getIds({\n              order: me.options.groupOrder\n          });\n\n          // restore original order\n          if (!util.equalArray(newOrder, me.groupTouchParams.originalOrder)) {\n            const origOrder = me.groupTouchParams.originalOrder;\n            const numGroups = Math.min(origOrder.length, newOrder.length);\n            let curPos = 0;\n            while (curPos < numGroups) {\n              // as long as the groups are where they should be step down along the groups order\n              while (curPos < numGroups && newOrder[curPos] == origOrder[curPos]) {\n                curPos++;\n              }\n\n              // all ok\n              if (curPos >= numGroups) {\n                break;\n              }\n\n              // found a group that has the wrong position -> switch with the\n              // group at the position where other one should be, fix index arrays and continue\n              const slippedPosition = newOrder.indexOf(origOrder[curPos]);\n              const switchGroup = dataset.get(newOrder[curPos]);\n              const shouldBeGroup = dataset.get(origOrder[curPos]);\n              me.options.groupOrderSwap(switchGroup, shouldBeGroup, dataset);\n              dataset.update(switchGroup);\n              dataset.update(shouldBeGroup);\n\n              const switchGroupId = newOrder[curPos];\n              newOrder[curPos] = origOrder[curPos];\n              newOrder[slippedPosition] = switchGroupId;\n\n              curPos++;\n            }\n          }\n        }\n      });\n\n      me.body.emitter.emit('groupDragged', { groupId: id });\n      this.toggleGroupDragClassName(this.groupTouchParams.group);\n      this.groupTouchParams.group = null;\n    }\n  }\n\n  /**\n   * Handle selecting/deselecting an item when tapping it\n   * @param {Event} event\n   * @private\n   */\n  _onSelectItem(event) {\n    if (!this.options.selectable) return;\n\n    const ctrlKey  = event.srcEvent && (event.srcEvent.ctrlKey || event.srcEvent.metaKey);\n    const shiftKey = event.srcEvent && event.srcEvent.shiftKey;\n    if (ctrlKey || shiftKey) {\n      this._onMultiSelectItem(event);\n      return;\n    }\n\n    const oldSelection = this.getSelection();\n\n    const item = this.itemFromTarget(event);\n    const selection = item && item.selectable ? [item.id] : [];\n    this.setSelection(selection);\n\n    const newSelection = this.getSelection();\n\n    // emit a select event,\n    // except when old selection is empty and new selection is still empty\n    if (newSelection.length > 0 || oldSelection.length > 0) {\n      this.body.emitter.emit('select', {\n        items: newSelection,\n        event\n      });\n    }\n  }\n\n  /**\n   * Handle hovering an item\n   * @param {Event} event\n   * @private\n   */\n  _onMouseOver(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    // Item we just left\n    const related = this.itemFromRelatedTarget(event);\n    if (item === related) {\n      // We haven't changed item, just element in the item\n      return;\n    }\n\n    const title = item.getTitle();\n    if (this.options.showTooltips && title) {\n      if (this.popup == null) {\n        this.popup = new Popup(this.body.dom.root,\n            this.options.tooltip.overflowMethod || 'flip');\n      }\n\n      this.popup.setText(title);\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.clientX - containerRect.left + container.offsetLeft,\n        event.clientY - containerRect.top + container.offsetTop\n      );\n      this.setPopupTimer(this.popup);\n    } else {\n      // Hovering over item without a title, hide popup\n      // Needed instead of _just_ in _onMouseOut due to #2572\n      this.clearPopupTimer();\n      if (this.popup != null) {\n        this.popup.hide();\n      }\n    }\n\n    this.body.emitter.emit('itemover', {\n      item: item.id,\n      event\n    });\n  }\n\n  /**\n   * on mouse start\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onMouseOut(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    // Item we are going to\n    const related = this.itemFromRelatedTarget(event);\n    if (item === related) {\n      // We aren't changing item, just element in the item\n      return;\n    }\n\n    this.clearPopupTimer();\n    if (this.popup != null) {\n      this.popup.hide();\n    }\n\n    this.body.emitter.emit('itemout', {\n      item: item.id,\n      event\n    });\n  }\n\n  /**\n   * on mouse move\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onMouseMove(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    if (this.popupTimer != null) {\n      // restart timer\n      this.setPopupTimer(this.popup);\n    }\n    \n    if (this.options.showTooltips && this.options.tooltip.followMouse && this.popup && !this.popup.hidden) {\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.clientX - containerRect.left + container.offsetLeft,\n        event.clientY - containerRect.top + container.offsetTop\n      );\n      this.popup.show(); // Redraw\n    }\n  }\n\n  /**\n   * Handle mousewheel\n   * @param {Event}  event   The event\n   * @private\n   */\n  _onMouseWheel(event) {\n    if (this.touchParams.itemIsDragging) {\n      this._onDragEnd(event);\n    }\n  }\n\n  /**\n   * Handle updates of an item on double tap\n   * @param {timeline.Item}  item   The item\n   * @private\n   */\n  _onUpdateItem(item) {\n    if (!this.options.selectable) return;\n    if (!this.options.editable.updateTime && !this.options.editable.updateGroup) return;\n\n    const me = this;\n   \n    if (item) {\n      // execute async handler to update the item (or cancel it)\n      const itemData = me.itemsData.get(item.id); // get a clone of the data from the dataset\n      this.options.onUpdate(itemData, itemData => {\n        if (itemData) {\n          me.itemsData.update(itemData);\n        }\n      });\n    }\n  }\n\n  /**\n   * Handle drop event of data on item\n   * Only called when `objectData.target === 'item'.\n   * @param {Event} event The event \n   * @private\n   */\n  _onDropObjectOnItem(event) {\n    const item = this.itemFromTarget(event);\n    const objectData = JSON.parse(event.dataTransfer.getData(\"text\"));\n    this.options.onDropObjectOnItem(objectData, item)\n  }\n\n  /**\n   * Handle creation of an item on double tap or drop of a drag event\n   * @param {Event} event   The event\n   * @private\n   */\n  _onAddItem(event) {\n    if (!this.options.selectable) return;\n    if (!this.options.editable.add) return;\n\n    const me = this;\n    const snap = this.options.snap || null;\n\n    // add item\n    const frameRect = this.dom.frame.getBoundingClientRect()\n    const x = this.options.rtl ? frameRect.right - event.center.x : event.center.x - frameRect.left;\n    const start = this.body.util.toTime(x);\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    let end;\n\n    let newItemData;\n    if (event.type == 'drop') {\n      newItemData = JSON.parse(event.dataTransfer.getData(\"text\"));\n      newItemData.content = newItemData.content ? newItemData.content : 'new item';\n      newItemData.start = newItemData.start ? newItemData.start : (snap ? snap(start, scale, step) : start);\n      newItemData.type = newItemData.type || 'box';\n      newItemData[this.itemsData.idProp] = newItemData.id || randomUUID();\n\n      if (newItemData.type == 'range' && !newItemData.end) {\n        end = this.body.util.toTime(x + this.props.width / 5);\n        newItemData.end = snap ? snap(end, scale, step) : end;\n      }\n    } else {\n      newItemData = {\n        start: snap ? snap(start, scale, step) : start,\n        content: 'new item'\n      };\n      newItemData[this.itemsData.idProp] = randomUUID();\n\n      // when default type is a range, add a default end date to the new item\n      if (this.options.type === 'range') {\n        end = this.body.util.toTime(x + this.props.width / 5);\n        newItemData.end = snap ? snap(end, scale, step) : end;\n      }\n    }\n\n    const group = this.groupFromTarget(event);\n    if (group) {\n      newItemData.group = group.groupId;\n    }\n\n    // execute async handler to customize (or cancel) adding an item\n    newItemData = this._cloneItemData(newItemData);     // convert start and end to the correct type\n    this.options.onAdd(newItemData, item => {\n      if (item) {\n        me.itemsData.add(item);\n        if (event.type == 'drop') {\n          me.setSelection([item.id]);\n        }\n        // TODO: need to trigger a redraw?\n      }\n    });\n  }\n\n  /**\n   * Handle selecting/deselecting multiple items when holding an item\n   * @param {Event} event\n   * @private\n   */\n  _onMultiSelectItem(event) {\n    if (!this.options.selectable) return;\n\n    const item = this.itemFromTarget(event);\n\n    if (item) {\n      // multi select items (if allowed)\n\n      let selection = this.options.multiselect\n        ? this.getSelection() // take current selection\n        : [];                 // deselect current selection\n\n      const shiftKey = event.srcEvent && event.srcEvent.shiftKey || false;\n\n      if ((shiftKey || this.options.sequentialSelection) && this.options.multiselect) {\n        // select all items between the old selection and the tapped item\n        const itemGroup = this.itemsData.get(item.id).group;\n\n        // when filtering get the group of the last selected item\n        let lastSelectedGroup = undefined;\n        if (this.options.multiselectPerGroup) {\n          if (selection.length > 0) {\n            lastSelectedGroup = this.itemsData.get(selection[0]).group;\n          }\n        }\n\n        // determine the selection range\n        if (!this.options.multiselectPerGroup || lastSelectedGroup == undefined || lastSelectedGroup == itemGroup) {\n          selection.push(item.id);\n        }\n        const range = ItemSet._getItemRange(this.itemsData.get(selection));\n        \n        if (!this.options.multiselectPerGroup || lastSelectedGroup == itemGroup) {\n          // select all items within the selection range\n          selection = [];\n          for (const id in this.items) {\n            if (this.items.hasOwnProperty(id)) {\n              const _item = this.items[id];\n              const start = _item.data.start;\n              const end = (_item.data.end !== undefined) ? _item.data.end : start;\n\n              if (start >= range.min &&\n                  end <= range.max &&\n                  (!this.options.multiselectPerGroup || lastSelectedGroup == this.itemsData.get(_item.id).group) &&\n                  !(_item instanceof BackgroundItem)) {\n                selection.push(_item.id); // do not use id but item.id, id itself is stringified\n              }\n            }\n          }\n        }\n      }\n      else {\n        // add/remove this item from the current selection\n        const index = selection.indexOf(item.id);\n        if (index == -1) {\n          // item is not yet selected -> select it\n          selection.push(item.id);\n        }\n        else {\n          // item is already selected -> deselect it\n          selection.splice(index, 1);\n        }\n      }\n\n      const filteredSelection = selection.filter(item => this.getItemById(item).selectable);\n\n      this.setSelection(filteredSelection);\n\n      this.body.emitter.emit('select', {\n        items: this.getSelection(),\n        event\n      });\n    }\n  }\n\n  /**\n   * Calculate the time range of a list of items\n   * @param {Array.<Object>} itemsData\n   * @return {{min: Date, max: Date}} Returns the range of the provided items\n   * @private\n   */\n  static _getItemRange(itemsData) {\n    let max = null;\n    let min = null;\n\n    itemsData.forEach(data => {\n      if (min == null || data.start < min) {\n        min = data.start;\n      }\n\n      if (data.end != undefined) {\n        if (max == null || data.end > max) {\n          max = data.end;\n        }\n      }\n      else {\n        if (max == null || data.start > max) {\n          max = data.start;\n        }\n      }\n    });\n\n    return {\n      min,\n      max\n    }\n  }\n\n  /**\n   * Find an item from an element:\n   * searches for the attribute 'vis-item' in the element's tree\n   * @param {HTMLElement} element\n   * @return {Item | null} item\n   */\n  itemFromElement(element) {\n    let cur = element;\n    while (cur) {\n      if (cur.hasOwnProperty('vis-item')) {\n        return cur['vis-item'];\n      }\n      cur = cur.parentNode;\n    }\n\n    return null;\n  }\n\n  /**\n   * Find an item from an event target:\n   * searches for the attribute 'vis-item' in the event target's element tree\n   * @param {Event} event\n   * @return {Item | null} item\n   */\n  itemFromTarget(event) {\n    return this.itemFromElement(event.target);\n  }\n\n  /**\n   * Find an item from an event's related target:\n   * searches for the attribute 'vis-item' in the related target's element tree\n   * @param {Event} event\n   * @return {Item | null} item\n   */\n  itemFromRelatedTarget(event) {\n    return this.itemFromElement(event.relatedTarget);\n  }\n\n  /**\n   * Find the Group from an event target:\n   * searches for the attribute 'vis-group' in the event target's element tree\n   * @param {Event} event\n   * @return {Group | null} group\n   */\n  groupFromTarget(event) {\n    const clientY = event.center ? event.center.y : event.clientY;\n    let groupIds = this.groupIds;\n    \n    if (groupIds.length <= 0 && this.groupsData) {\n      groupIds = this.groupsData.getIds({\n        order: this.options.groupOrder\n      });\n    }\n    \n    for (let i = 0; i < groupIds.length; i++) {\n      const groupId = groupIds[i];\n      const group = this.groups[groupId];\n      const foreground = group.dom.foreground;\n      const foregroundRect = foreground.getBoundingClientRect()\n      if (clientY >= foregroundRect.top && clientY < foregroundRect.top + foreground.offsetHeight) {\n        return group;\n      }\n\n      if (this.options.orientation.item === 'top') {\n        if (i === this.groupIds.length - 1 && clientY > foregroundRect.top) {\n          return group;\n        }\n      }\n      else {\n        if (i === 0 && clientY < foregroundRect.top + foreground.offset) {\n          return group;\n        }\n      }\n    }\n\n    return null;\n  }\n\n  /**\n   * Find the ItemSet from an event target:\n   * searches for the attribute 'vis-itemset' in the event target's element tree\n   * @param {Event} event\n   * @return {ItemSet | null} item\n   */\n  static itemSetFromTarget(event) {\n    let target = event.target;\n    while (target) {\n      if (target.hasOwnProperty('vis-itemset')) {\n        return target['vis-itemset'];\n      }\n      target = target.parentNode;\n    }\n\n    return null;\n  }\n\n  /**\n   * Clone the data of an item, and \"normalize\" it: convert the start and end date\n   * to the type (Date, Moment, ...) configured in the DataSet. If not configured,\n   * start and end are converted to Date.\n   * @param {Object} itemData, typically `item.data`\n   * @param {string} [type]  Optional Date type. If not provided, the type from the DataSet is taken\n   * @return {Object} The cloned object\n   * @private\n   */\n  _cloneItemData(itemData, type) {\n    const clone = util.extend({}, itemData);\n\n    if (!type) {\n      // convert start and end date to the type (Date, Moment, ...) configured in the DataSet\n      type = this.itemsData.type;\n    }\n\n    if (clone.start != undefined) {\n      clone.start = util.convert(clone.start, type && type.start || 'Date');\n    }\n    if (clone.end != undefined) {\n      clone.end = util.convert(clone.end , type && type.end || 'Date');\n    }\n\n    return clone;\n  }\n\n  /**\n   * cluster items\n   * @return {void}   \n   * @private\n   */\n  _clusterItems() {\n    if (!this.options.cluster) {\n      return;\n    }\n\n    const { scale } = this.body.range.conversion(this.body.domProps.center.width);\n    const clusters = this.clusterGenerator.getClusters(this.clusters, scale, this.options.cluster);\n\n    if (this.clusters != clusters) {\n      this._detachAllClusters();\n\n      if (clusters) {\n        for (let cluster of clusters) {\n          cluster.attach();\n        }\n        this.clusters = clusters;\n      }\n\n      this._updateClusters(clusters);\n    }\n  }\n\n  /**\n   * detach all cluster items\n   * @private\n   */\n  _detachAllClusters() {\n    if (this.options.cluster) {\n      if (this.clusters && this.clusters.length) {\n        for (let cluster of this.clusters) {\n          cluster.detach();\n        }\n      }\n    }\n  }\n\n  /**\n   * update clusters\n   * @param {array} clusters\n   * @private\n   */\n  _updateClusters(clusters) {\n    if (this.clusters && this.clusters.length) {\n      const newClustersIds = new Set(clusters.map(cluster => cluster.id));\n      const clustersToUnselect = this.clusters.filter(cluster => !newClustersIds.has(cluster.id));\n      let selectionChanged = false;\n      for (let cluster of clustersToUnselect) {\n        const selectedIdx = this.selection.indexOf(cluster.id);\n        if (selectedIdx !== -1) {\n          cluster.unselect();\n          this.selection.splice(selectedIdx, 1);\n          selectionChanged = true;\n        }\n      }\n\n      if (selectionChanged) {\n        const newSelection = this.getSelection();\n        this.body.emitter.emit('select', {\n          items: newSelection,\n          event: event\n        });\n      }\n    }\n\n    this.clusters = clusters || [];\n  }\n}\n\n// available item types will be registered here\nItemSet.types = {\n  background: BackgroundItem,\n  box: BoxItem,\n  range: RangeItem,\n  point: PointItem\n};\n\n/**\n * Handle added items\n * @param {number[]} ids\n * @protected\n */\nItemSet.prototype._onAdd = ItemSet.prototype._onUpdate;\n\nexport default ItemSet;\n","import util from '../util';\n\nlet errorFound = false;\nlet allOptions;\nlet printStyle = 'background: #FFeeee; color: #dd0000';\n/**\n *  Used to validate options.\n */\nclass Validator {\n  /**\n   * @ignore\n   */\n  constructor() {\n  }\n\n  /**\n   * Main function to be called\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {Object} subObject\n   * @returns {boolean}\n   * @static\n   */\n  static validate(options, referenceOptions, subObject) {\n    errorFound = false;\n    allOptions = referenceOptions;\n    let usedOptions = referenceOptions;\n    if (subObject !== undefined) {\n      usedOptions = referenceOptions[subObject];\n    }\n    Validator.parse(options, usedOptions, []);\n    return errorFound;\n  }\n\n\n  /**\n   * Will traverse an object recursively and check every value\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {array} path    | where to look for the actual option\n   * @static\n   */\n  static parse(options, referenceOptions, path) {\n    for (let option in options) {\n      if (options.hasOwnProperty(option)) {\n        Validator.check(option, options, referenceOptions, path);\n      }\n    }\n  }\n\n\n  /**\n   * Check every value. If the value is an object, call the parse function on that object.\n   * @param {string} option\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {array} path    | where to look for the actual option\n   * @static\n   */\n  static check(option, options, referenceOptions, path) {\n    if (referenceOptions[option] === undefined && referenceOptions.__any__ === undefined) {\n      Validator.getSuggestion(option, referenceOptions, path);\n      return;\n    }\n\n    let referenceOption = option;\n    let is_object = true;\n\n    if (referenceOptions[option] === undefined && referenceOptions.__any__ !== undefined) {\n      // NOTE: This only triggers if the __any__ is in the top level of the options object.\n      //       THAT'S A REALLY BAD PLACE TO ALLOW IT!!!!\n      // TODO: Examine if needed, remove if possible\n\n      // __any__ is a wildcard. Any value is accepted and will be further analysed by reference.\n      referenceOption = '__any__';\n\n      // if the any-subgroup is not a predefined object in the configurator,\n      // we do not look deeper into the object.\n      is_object = (Validator.getType(options[option]) === 'object');\n    }\n    else {\n      // Since all options in the reference are objects, we can check whether \n      // they are supposed to be the object to look for the __type__ field.\n      // if this is an object, we check if the correct type has been supplied to account for shorthand options.\n    }\n\n    let refOptionObj = referenceOptions[referenceOption];\n    if (is_object && refOptionObj.__type__ !== undefined) {\n      refOptionObj = refOptionObj.__type__;\n    }\n\n    Validator.checkFields(option, options, referenceOptions, referenceOption, refOptionObj, path);\n  }\n\n  /**\n   *\n   * @param {string}  option           | the option property\n   * @param {Object}  options          | The supplied options object\n   * @param {Object}  referenceOptions | The reference options containing all options and their allowed formats\n   * @param {string}  referenceOption  | Usually this is the same as option, except when handling an __any__ tag.\n   * @param {string}  refOptionObj     | This is the type object from the reference options\n   * @param {Array}   path             | where in the object is the option\n   * @static\n   */\n  static checkFields(option, options, referenceOptions, referenceOption, refOptionObj, path) {\n    let log = function(message) {\n      console.log('%c' + message + Validator.printLocation(path, option), printStyle);\n    };\n\n    let optionType = Validator.getType(options[option]);\n    let refOptionType = refOptionObj[optionType];\n\n    if (refOptionType !== undefined) {\n      // if the type is correct, we check if it is supposed to be one of a few select values\n      if (Validator.getType(refOptionType) === 'array' && refOptionType.indexOf(options[option]) === -1) {\n        log('Invalid option detected in \"' + option + '\".' +\n          ' Allowed values are:' + Validator.print(refOptionType) +\n          ' not \"' + options[option] + '\". ');\n        errorFound = true;\n      }\n      else if (optionType === 'object' && referenceOption !== \"__any__\") {\n        path = util.copyAndExtendArray(path, option);\n        Validator.parse(options[option], referenceOptions[referenceOption], path);\n      }\n    }\n    else if (refOptionObj['any'] === undefined) {\n      // type of the field is incorrect and the field cannot be any\n      log('Invalid type received for \"' + option +\n        '\". Expected: ' + Validator.print(Object.keys(refOptionObj)) +\n        '. Received ['  + optionType + '] \"' + options[option] + '\"');\n      errorFound = true;\n    }\n  }\n\n  /**\n   *\n   * @param {Object|boolean|number|string|Array.<number>|Date|Node|Moment|undefined|null} object\n   * @returns {string}\n   * @static\n   */\n  static getType(object) {\n    var type = typeof object;\n\n    if (type === 'object') {\n      if (object === null) {\n        return 'null';\n      }\n      if (object instanceof Boolean) {\n        return 'boolean';\n      }\n      if (object instanceof Number) {\n        return 'number';\n      }\n      if (object instanceof String) {\n        return 'string';\n      }\n      if (Array.isArray(object)) {\n        return 'array';\n      }\n      if (object instanceof Date) {\n        return 'date';\n      }\n      if (object.nodeType !== undefined) {\n        return 'dom';\n      }\n      if (object._isAMomentObject === true) {\n        return 'moment';\n      }\n      return 'object';\n    }\n    else if (type === 'number') {\n      return 'number';\n    }\n    else if (type === 'boolean') {\n      return 'boolean';\n    }\n    else if (type === 'string') {\n      return 'string';\n    }\n    else if (type === undefined) {\n      return 'undefined';\n    }\n    return type;\n  }\n\n  /**\n   * @param {string} option\n   * @param {Object} options\n   * @param {Array.<string>} path\n   * @static\n   */\n  static getSuggestion(option, options, path) {\n    let localSearch = Validator.findInOptions(option,options,path,false);\n    let globalSearch = Validator.findInOptions(option,allOptions,[],true);\n\n    let localSearchThreshold = 8;\n    let globalSearchThreshold = 4;\n\n    let msg;\n    if (localSearch.indexMatch !== undefined) {\n      msg = ' in ' + Validator.printLocation(localSearch.path, option,'') +\n        'Perhaps it was incomplete? Did you mean: \"' + localSearch.indexMatch + '\"?\\n\\n';\n    }\n    else if (globalSearch.distance <= globalSearchThreshold && localSearch.distance > globalSearch.distance) {\n      msg = ' in ' + Validator.printLocation(localSearch.path, option,'') +\n        'Perhaps it was misplaced? Matching option found at: ' +\n        Validator.printLocation(globalSearch.path, globalSearch.closestMatch,'');\n    }\n    else if (localSearch.distance <= localSearchThreshold) {\n      msg = '. Did you mean \"' + localSearch.closestMatch + '\"?' +\n        Validator.printLocation(localSearch.path, option);\n    }\n    else {\n      msg = '. Did you mean one of these: ' + Validator.print(Object.keys(options)) +\n      Validator.printLocation(path, option);\n    }\n\n    console.log('%cUnknown option detected: \"' + option + '\"' + msg, printStyle);\n    errorFound = true;\n  }\n\n  /**\n   * traverse the options in search for a match.\n   * @param {string} option\n   * @param {Object} options\n   * @param {Array} path    | where to look for the actual option\n   * @param {boolean} [recursive=false]\n   * @returns {{closestMatch: string, path: Array, distance: number}}\n   * @static\n   */\n  static findInOptions(option, options, path, recursive = false) {\n    let min = 1e9;\n    let closestMatch = '';\n    let closestMatchPath = [];\n    let lowerCaseOption = option.toLowerCase();\n    let indexMatch = undefined;\n    for (let op in options) {  // eslint-disable-line guard-for-in\n      let distance;\n      if (options[op].__type__ !== undefined && recursive === true) {\n        let result = Validator.findInOptions(option, options[op], util.copyAndExtendArray(path,op));\n        if (min > result.distance) {\n          closestMatch = result.closestMatch;\n          closestMatchPath = result.path;\n          min = result.distance;\n          indexMatch = result.indexMatch;\n        }\n      }\n      else {\n        if (op.toLowerCase().indexOf(lowerCaseOption) !== -1) {\n          indexMatch = op;\n        }\n        distance = Validator.levenshteinDistance(option, op);\n        if (min > distance) {\n          closestMatch = op;\n          closestMatchPath = util.copyArray(path);\n          min = distance;\n        }\n      }\n    }\n    return {closestMatch:closestMatch, path:closestMatchPath, distance:min, indexMatch: indexMatch};\n  }\n\n  /**\n   * @param {Array.<string>} path\n   * @param {Object} option\n   * @param {string} prefix\n   * @returns {String}\n   * @static\n   */\n  static printLocation(path, option, prefix = 'Problem value found at: \\n') {\n    let str = '\\n\\n' + prefix + 'options = {\\n';\n    for (let i = 0; i < path.length; i++) {\n      for (let j = 0; j < i + 1; j++) {\n        str += '  ';\n      }\n      str += path[i] + ': {\\n'\n    }\n    for (let j = 0; j < path.length + 1; j++) {\n      str += '  ';\n    }\n    str += option + '\\n';\n    for (let i = 0; i < path.length + 1; i++) {\n      for (let j = 0; j < path.length - i; j++) {\n        str += '  ';\n      }\n      str += '}\\n'\n    }\n    return str + '\\n\\n';\n  }\n\n  /**\n   * @param {Object} options\n   * @returns {String}\n   * @static\n   */\n  static print(options) {\n    return JSON.stringify(options).replace(/(\\\")|(\\[)|(\\])|(,\"__type__\")/g, \"\").replace(/(\\,)/g, ', ')\n  }\n\n\n  /**\n   *  Compute the edit distance between the two given strings\n   * http://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#JavaScript\n   *\n   * Copyright (c) 2011 Andrei Mackenzie\n   *\n   * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n   *\n   * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n   *\n   * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n   *\n   * @param {string} a\n   * @param {string} b\n   * @returns {Array.<Array.<number>>}}\n   * @static\n   */\n  static levenshteinDistance(a, b) {\n    if (a.length === 0) return b.length;\n    if (b.length === 0) return a.length;\n\n    var matrix = [];\n\n    // increment along the first column of each row\n    var i;\n    for (i = 0; i <= b.length; i++) {\n      matrix[i] = [i];\n    }\n\n    // increment each column in the first row\n    var j;\n    for (j = 0; j <= a.length; j++) {\n      matrix[0][j] = j;\n    }\n\n    // Fill in the rest of the matrix\n    for (i = 1; i <= b.length; i++) {\n      for (j = 1; j <= a.length; j++) {\n        if (b.charAt(i - 1) == a.charAt(j - 1)) {\n          matrix[i][j] = matrix[i - 1][j - 1];\n        } else {\n          matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, // substitution\n            Math.min(matrix[i][j - 1] + 1, // insertion\n              matrix[i - 1][j] + 1)); // deletion\n        }\n      }\n    }\n\n    return matrix[b.length][a.length];\n  }\n}\n\n\nexport {\n  Validator,\n  printStyle\n};\n","/**\n * This object contains all possible options. It will check if the types are correct, if required if the option is one\n * of the allowed values.\n *\n * __any__ means that the name of the property does not matter.\n * __type__ is a required field for all objects and contains the allowed types of all objects\n */\nlet string = 'string';\nlet bool = 'boolean';\nlet number = 'number';\nlet array = 'array';\nlet date = 'date';\nlet object = 'object'; // should only be in a __type__ property\nlet dom = 'dom';\nlet moment = 'moment';\nlet any = 'any';\n\nlet allOptions = {\n  configure: {\n    enabled: { 'boolean': bool},\n    filter: { 'boolean': bool,'function': 'function'},\n    container: {dom},\n    __type__: {object, 'boolean': bool,'function': 'function'}\n  },\n\n  //globals :\n  align: {string},\n  alignCurrentTime: {string, 'undefined': 'undefined'},\n  rtl: { 'boolean': bool, 'undefined': 'undefined'},\n  rollingMode: {\n    follow: { 'boolean': bool },\n    offset: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  onTimeout: {\n    timeoutMs: {number},\n    callback: {'function': 'function'},\n    __type__: {object}\n  },\n  verticalScroll: { 'boolean': bool, 'undefined': 'undefined'},\n  horizontalScroll: { 'boolean': bool, 'undefined': 'undefined'},\n  autoResize: { 'boolean': bool},\n  throttleRedraw: {number}, // TODO: DEPRICATED see https://github.com/almende/vis/issues/2511\n  clickToUse: { 'boolean': bool},\n  dataAttributes: {string, array},\n  editable: {\n    add: { 'boolean': bool, 'undefined': 'undefined'},\n    remove: { 'boolean': bool, 'undefined': 'undefined'},\n    updateGroup: { 'boolean': bool, 'undefined': 'undefined'},\n    updateTime: { 'boolean': bool, 'undefined': 'undefined'},\n    overrideItems: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  end: {number, date, string, moment},\n  format: {\n    minorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object, 'function': 'function'}\n    },\n    majorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object, 'function': 'function'}\n    },\n    __type__: {object}\n  },\n  moment: {'function': 'function'},\n  groupHeightMode: {string},\n  groupOrder: {string, 'function': 'function'},\n  groupEditable: {\n    add: { 'boolean': bool, 'undefined': 'undefined'},\n    remove: { 'boolean': bool, 'undefined': 'undefined'},\n    order: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  groupOrderSwap: {'function': 'function'},\n  height: {string, number},\n  hiddenDates: {\n    start: {date, number, string, moment},\n    end: {date, number, string, moment},\n    repeat: {string},\n    __type__: {object, array}\n  },\n  itemsAlwaysDraggable: {\n    item: { 'boolean': bool, 'undefined': 'undefined'},\n    range: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  limitSize: {'boolean': bool},\n  locale:{string},\n  locales:{\n    __any__: {any},\n    __type__: {object}\n  },\n  longSelectPressTime: {number},\n  margin: {\n    axis: {number},\n    item: {\n      horizontal: {number,'undefined': 'undefined'},\n      vertical: {number,'undefined': 'undefined'},\n      __type__: {object,number}\n    },\n    __type__: {object,number}\n  },\n  max: {date, number, string, moment},\n  maxHeight: {number, string},\n  maxMinorChars: {number},\n  min: {date, number, string, moment},\n  minHeight: {number, string},\n  moveable: { 'boolean': bool},\n  multiselect: { 'boolean': bool},\n  multiselectPerGroup: { 'boolean': bool},\n  onAdd: {'function': 'function'},\n  onDropObjectOnItem: {'function': 'function'},\n  onUpdate: {'function': 'function'},\n  onMove: {'function': 'function'},\n  onMoving: {'function': 'function'},\n  onRemove: {'function': 'function'},\n  onAddGroup: {'function': 'function'},\n  onMoveGroup: {'function': 'function'},\n  onRemoveGroup: {'function': 'function'},\n  onInitialDrawComplete: {'function': 'function'},\n  order: {'function': 'function'},\n  orientation: {\n    axis: {string,'undefined': 'undefined'},\n    item: {string,'undefined': 'undefined'},\n    __type__: {string, object}\n  },\n  selectable: { 'boolean': bool},\n  sequentialSelection: { 'boolean': bool },\n  showCurrentTime: { 'boolean': bool},\n  showMajorLabels: { 'boolean': bool},\n  showMinorLabels: { 'boolean': bool},\n  showWeekScale: { 'boolean': bool},\n  stack: { 'boolean': bool},\n  stackSubgroups: { 'boolean': bool},\n  cluster: { \n    maxItems: {'number': number, 'undefined': 'undefined'},\n    titleTemplate: {'string': string, 'undefined': 'undefined'},\n    clusterCriteria: { 'function': 'function', 'undefined': 'undefined'},\n    showStipes: {'boolean': bool, 'undefined': 'undefined'},\n    fitOnDoubleClick: {'boolean': bool, 'undefined': 'undefined'},\n    __type__: {'boolean': bool, object}\n  },\n  snap: {'function': 'function', 'null': 'null'},\n  start: {date, number, string, moment},\n  template: {'function': 'function'},\n  loadingScreenTemplate: {'function': 'function'},\n  groupTemplate: {'function': 'function'},\n  visibleFrameTemplate: {string, 'function': 'function'},\n  showTooltips: { 'boolean': bool},\n  tooltip: {\n    followMouse: { 'boolean': bool },\n    overflowMethod: { 'string': ['cap', 'flip', 'none'] },\n    delay: {number},\n    template: {'function': 'function'},\n    __type__: {object}\n  },\n  tooltipOnItemUpdateTime: {\n    template: {'function': 'function'},\n    __type__: { 'boolean': bool, object}\n  },\n  timeAxis: {\n    scale: {string,'undefined': 'undefined'},\n    step: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  type: {string},\n  width: {string, number},\n  preferZoom: { 'boolean': bool},\n  zoomable: { 'boolean': bool},\n  zoomKey: {string: ['ctrlKey', 'altKey', 'shiftKey', 'metaKey', '']},\n  zoomFriction: {number},\n  zoomMax: {number},\n  zoomMin: {number},\n\n  __type__: {object}\n};\n\nlet configureOptions = {\n  global: {\n    align:  ['center', 'left', 'right'],\n    alignCurrentTime: ['none', 'year', 'month', 'quarter', 'week', 'isoWeek', 'day', 'date', 'hour', 'minute', 'second'],\n    direction:  false,\n    autoResize: true,\n    clickToUse: false,\n    // dataAttributes: ['all'], // FIXME: can be 'all' or string[]\n      editable: {\n      add: false,\n      remove: false,\n      updateGroup: false,\n      updateTime: false\n    },\n    end: '',\n    format: {\n      minorLabels: {\n        millisecond:'SSS',\n        second:     's',\n        minute:     'HH:mm',\n        hour:       'HH:mm',\n        weekday:    'ddd D',\n        day:        'D',\n        week:       'w',\n        month:      'MMM',\n        year:       'YYYY'\n      },\n      majorLabels: {\n        millisecond:'HH:mm:ss',\n        second:     'D MMMM HH:mm',\n        minute:     'ddd D MMMM',\n        hour:       'ddd D MMMM',\n        weekday:    'MMMM YYYY',\n        day:        'MMMM YYYY',\n        week:       'MMMM YYYY',\n        month:      'YYYY',\n        year:       ''\n      }\n    },\n    groupHeightMode: ['auto', 'fixed', 'fitItems'],\n    //groupOrder: {string, 'function': 'function'},\n    groupsDraggable: false,\n    height: '',\n    //hiddenDates: {object, array},\n    locale: '',\n    longSelectPressTime: 251,\n    margin: {\n      axis: [20, 0, 100, 1],\n      item: {\n        horizontal: [10, 0, 100, 1],\n        vertical: [10, 0, 100, 1]\n      }\n    },\n    max: '',\n    maxHeight: '',\n    maxMinorChars: [7, 0, 20, 1],\n    min: '',\n    minHeight: '',\n    moveable: false,\n    multiselect: false,\n    multiselectPerGroup: false,\n    //onAdd: {'function': 'function'},\n    //onUpdate: {'function': 'function'},\n    //onMove: {'function': 'function'},\n    //onMoving: {'function': 'function'},\n    //onRename: {'function': 'function'},\n    //order: {'function': 'function'},\n    orientation: {\n      axis: ['both', 'bottom', 'top'],\n      item: ['bottom', 'top']\n    },\n    preferZoom: false,\n    selectable: true,\n    showCurrentTime: false,\n    showMajorLabels: true,\n    showMinorLabels: true,\n    stack: true,\n    stackSubgroups: true,\n    cluster: false,\n    //snap: {'function': 'function', nada},\n    start: '',\n    //template: {'function': 'function'},\n    //timeAxis: {\n    //  scale: ['millisecond', 'second', 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'],\n    //  step: [1, 1, 10, 1]\n    //},\n    showTooltips: true,\n    tooltip: {\n      followMouse: false,\n      overflowMethod: 'flip',\n      delay: [500, 0, 99999, 100],\n    },\n    tooltipOnItemUpdateTime: false,\n    type: ['box', 'point', 'range', 'background'],\n    width: '100%',\n    zoomable: true,\n    zoomKey: ['ctrlKey', 'altKey', 'shiftKey', 'metaKey', ''],\n    zoomMax: [315360000000000, 10, 315360000000000, 1],\n    zoomMin: [10, 10, 315360000000000, 1]\n  }\n};\n\nexport {allOptions, configureOptions};\n","var $ = require('../internals/export');\nvar fill = require('../internals/array-fill');\nvar addToUnscopables = require('../internals/add-to-unscopables');\n\n// `Array.prototype.fill` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.fill\n$({ target: 'Array', proto: true }, {\n  fill: fill\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('fill');\n","'use strict';\nvar toObject = require('../internals/to-object');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toLength = require('../internals/to-length');\n\n// `Array.prototype.fill` method implementation\n// https://tc39.github.io/ecma262/#sec-array.prototype.fill\nmodule.exports = function fill(value /* , start = 0, end = @length */) {\n  var O = toObject(this);\n  var length = toLength(O.length);\n  var argumentsLength = arguments.length;\n  var index = toAbsoluteIndex(argumentsLength > 1 ? arguments[1] : undefined, length);\n  var end = argumentsLength > 2 ? arguments[2] : undefined;\n  var endPos = end === undefined ? length : toAbsoluteIndex(end, length);\n  while (endPos > index) O[index++] = value;\n  return O;\n};\n","require('../../../modules/es.array.fill');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').fill;\n","var fill = require('../array/virtual/fill');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.fill;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.fill) ? fill : own;\n};\n","import Hammer from '../module/hammer';\nimport * as hammerUtil from '../hammerUtil';\nimport util from '../util';\n\n\nvar htmlColors = {black: '#000000', navy: '#000080', darkblue: '#00008B', mediumblue: '#0000CD', blue: '#0000FF', darkgreen: '#006400', green: '#008000', teal: '#008080', darkcyan: '#008B8B', deepskyblue: '#00BFFF', darkturquoise: '#00CED1', mediumspringgreen: '#00FA9A', lime: '#00FF00', springgreen: '#00FF7F', aqua: '#00FFFF', cyan: '#00FFFF', midnightblue: '#191970', dodgerblue: '#1E90FF', lightseagreen: '#20B2AA', forestgreen: '#228B22', seagreen: '#2E8B57', darkslategray: '#2F4F4F', limegreen: '#32CD32', mediumseagreen: '#3CB371', turquoise: '#40E0D0', royalblue: '#4169E1', steelblue: '#4682B4', darkslateblue: '#483D8B', mediumturquoise: '#48D1CC', indigo: '#4B0082', darkolivegreen: '#556B2F', cadetblue: '#5F9EA0', cornflowerblue: '#6495ED', mediumaquamarine: '#66CDAA', dimgray: '#696969', slateblue: '#6A5ACD', olivedrab: '#6B8E23', slategray: '#708090', lightslategray: '#778899', mediumslateblue: '#7B68EE', lawngreen: '#7CFC00', chartreuse: '#7FFF00', aquamarine: '#7FFFD4', maroon: '#800000', purple: '#800080', olive: '#808000', gray: '#808080', skyblue: '#87CEEB', lightskyblue: '#87CEFA', blueviolet: '#8A2BE2', darkred: '#8B0000', darkmagenta: '#8B008B', saddlebrown: '#8B4513', darkseagreen: '#8FBC8F', lightgreen: '#90EE90', mediumpurple: '#9370D8', darkviolet: '#9400D3', palegreen: '#98FB98', darkorchid: '#9932CC', yellowgreen: '#9ACD32', sienna: '#A0522D', brown: '#A52A2A', darkgray: '#A9A9A9', lightblue: '#ADD8E6', greenyellow: '#ADFF2F', paleturquoise: '#AFEEEE', lightsteelblue: '#B0C4DE', powderblue: '#B0E0E6', firebrick: '#B22222', darkgoldenrod: '#B8860B', mediumorchid: '#BA55D3', rosybrown: '#BC8F8F', darkkhaki: '#BDB76B', silver: '#C0C0C0', mediumvioletred: '#C71585', indianred: '#CD5C5C', peru: '#CD853F', chocolate: '#D2691E', tan: '#D2B48C', lightgrey: '#D3D3D3', palevioletred: '#D87093', thistle: '#D8BFD8', orchid: '#DA70D6', goldenrod: '#DAA520', crimson: '#DC143C', gainsboro: '#DCDCDC', plum: '#DDA0DD', burlywood: '#DEB887', lightcyan: '#E0FFFF', lavender: '#E6E6FA', darksalmon: '#E9967A', violet: '#EE82EE', palegoldenrod: '#EEE8AA', lightcoral: '#F08080', khaki: '#F0E68C', aliceblue: '#F0F8FF', honeydew: '#F0FFF0', azure: '#F0FFFF', sandybrown: '#F4A460', wheat: '#F5DEB3', beige: '#F5F5DC', whitesmoke: '#F5F5F5', mintcream: '#F5FFFA', ghostwhite: '#F8F8FF', salmon: '#FA8072', antiquewhite: '#FAEBD7', linen: '#FAF0E6', lightgoldenrodyellow: '#FAFAD2', oldlace: '#FDF5E6', red: '#FF0000', fuchsia: '#FF00FF', magenta: '#FF00FF', deeppink: '#FF1493', orangered: '#FF4500', tomato: '#FF6347', hotpink: '#FF69B4', coral: '#FF7F50', darkorange: '#FF8C00', lightsalmon: '#FFA07A', orange: '#FFA500', lightpink: '#FFB6C1', pink: '#FFC0CB', gold: '#FFD700', peachpuff: '#FFDAB9', navajowhite: '#FFDEAD', moccasin: '#FFE4B5', bisque: '#FFE4C4', mistyrose: '#FFE4E1', blanchedalmond: '#FFEBCD', papayawhip: '#FFEFD5', lavenderblush: '#FFF0F5', seashell: '#FFF5EE', cornsilk: '#FFF8DC', lemonchiffon: '#FFFACD', floralwhite: '#FFFAF0', snow: '#FFFAFA', yellow: '#FFFF00', lightyellow: '#FFFFE0', ivory: '#FFFFF0', white: '#FFFFFF'};\n\n/**\n * @param {number} [pixelRatio=1]\n */\nclass ColorPicker {\n  /**\n   * @param {number} [pixelRatio=1]\n   */\n  constructor(pixelRatio = 1) {\n    this.pixelRatio = pixelRatio;\n    this.generated = false;\n    this.centerCoordinates = {x:289/2, y:289/2};\n    this.r = 289 * 0.49;\n    this.color = {r:255,g:255,b:255,a:1.0};\n    this.hueCircle = undefined;\n    this.initialColor = {r:255,g:255,b:255,a:1.0};\n    this.previousColor= undefined;\n    this.applied = false;\n\n    // bound by\n    this.updateCallback = () => {};\n    this.closeCallback = () => {};\n\n    // create all DOM elements\n    this._create();\n  }\n\n\n  /**\n   * this inserts the colorPicker into a div from the DOM\n   * @param {Element} container\n   */\n  insertTo(container) {\n    if (this.hammer !== undefined) {\n      this.hammer.destroy();\n      this.hammer = undefined;\n    }\n    this.container = container;\n    this.container.appendChild(this.frame);\n    this._bindHammer();\n\n    this._setSize();\n  }\n\n  /**\n   * the callback is executed on apply and save. Bind it to the application\n   * @param {function} callback\n   */\n  setUpdateCallback(callback) {\n    if (typeof callback === 'function') {\n      this.updateCallback = callback;\n    }\n    else {\n      throw new Error(\"Function attempted to set as colorPicker update callback is not a function.\");\n    }\n  }\n\n  /**\n   * the callback is executed on apply and save. Bind it to the application\n   * @param {function} callback\n   */\n  setCloseCallback(callback) {\n    if (typeof callback === 'function') {\n      this.closeCallback = callback;\n    }\n    else {\n      throw new Error(\"Function attempted to set as colorPicker closing callback is not a function.\");\n    }\n  }\n\n  /**\n   *\n   * @param {string} color\n   * @returns {String}\n   * @private\n   */\n  _isColorString(color) {\n    if (typeof color === 'string') {\n      return htmlColors[color];\n    }\n  }\n\n\n  /**\n   * Set the color of the colorPicker\n   * Supported formats:\n   * 'red'                   --> HTML color string\n   * '#ffffff'               --> hex string\n   * 'rgb(255,255,255)'      --> rgb string\n   * 'rgba(255,255,255,1.0)' --> rgba string\n   * {r:255,g:255,b:255}     --> rgb object\n   * {r:255,g:255,b:255,a:1.0} --> rgba object\n   * @param {string|Object} color\n   * @param {boolean} [setInitial=true]\n   */\n  setColor(color, setInitial = true) {\n    if (color === 'none') {\n      return;\n    }\n\n    let rgba;\n\n    // if a html color shorthand is used, convert to hex\n    var htmlColor = this._isColorString(color);\n    if (htmlColor !== undefined) {\n      color = htmlColor;\n    }\n\n    // check format\n    if (util.isString(color) === true) {\n      if (util.isValidRGB(color) === true) {\n        let rgbaArray = color.substr(4).substr(0, color.length - 5).split(',');\n        rgba = {r:rgbaArray[0], g:rgbaArray[1], b:rgbaArray[2], a:1.0};\n      }\n      else if (util.isValidRGBA(color) === true) {\n        let rgbaArray = color.substr(5).substr(0, color.length - 6).split(',');\n        rgba = {r:rgbaArray[0], g:rgbaArray[1], b:rgbaArray[2], a:rgbaArray[3]};\n      }\n      else if (util.isValidHex(color) === true) {\n        let rgbObj = util.hexToRGB(color);\n        rgba = {r:rgbObj.r, g:rgbObj.g, b:rgbObj.b, a:1.0};\n      }\n    }\n    else {\n      if (color instanceof Object) {\n        if (color.r !== undefined && color.g !== undefined && color.b !== undefined) {\n          let alpha = color.a !== undefined ? color.a : '1.0';\n          rgba = {r:color.r, g:color.g, b:color.b, a:alpha};\n        }\n      }\n    }\n\n    // set color\n    if (rgba === undefined) {\n      throw new Error(\"Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: \" + JSON.stringify(color));\n    }\n    else {\n      this._setColor(rgba, setInitial);\n    }\n  }\n\n\n  /**\n   * this shows the color picker.\n   * The hue circle is constructed once and stored.\n   */\n  show() {\n    if (this.closeCallback !== undefined) {\n      this.closeCallback();\n      this.closeCallback = undefined;\n    }\n\n    this.applied = false;\n    this.frame.style.display = 'block';\n    this._generateHueCircle();\n  }\n\n  // ------------------------------------------ PRIVATE ----------------------------- //\n\n  /**\n   * Hide the picker. Is called by the cancel button.\n   * Optional boolean to store the previous color for easy access later on.\n   * @param {boolean} [storePrevious=true]\n   * @private\n   */\n  _hide(storePrevious = true) {\n    // store the previous color for next time;\n    if (storePrevious === true) {\n      this.previousColor = util.extend({}, this.color);\n    }\n\n    if (this.applied === true) {\n      this.updateCallback(this.initialColor);\n    }\n\n    this.frame.style.display = 'none';\n\n    // call the closing callback, restoring the onclick method.\n    // this is in a setTimeout because it will trigger the show again before the click is done.\n    setTimeout(() => {\n      if (this.closeCallback !== undefined) {\n        this.closeCallback();\n        this.closeCallback = undefined;\n      }\n    },0);\n  }\n\n\n  /**\n   * bound to the save button. Saves and hides.\n   * @private\n   */\n  _save() {\n    this.updateCallback(this.color);\n    this.applied = false;\n    this._hide();\n  }\n\n\n  /**\n   * Bound to apply button. Saves but does not close. Is undone by the cancel button.\n   * @private\n   */\n  _apply() {\n    this.applied = true;\n    this.updateCallback(this.color);\n    this._updatePicker(this.color);\n  }\n\n\n  /**\n   * load the color from the previous session.\n   * @private\n   */\n  _loadLast() {\n    if (this.previousColor !== undefined) {\n      this.setColor(this.previousColor, false);\n    }\n    else {\n      alert(\"There is no last color to load...\");\n    }\n  }\n\n\n  /**\n   * set the color, place the picker\n   * @param {Object} rgba\n   * @param {boolean} [setInitial=true]\n   * @private\n   */\n  _setColor(rgba, setInitial = true) {\n    // store the initial color\n    if (setInitial === true) {\n      this.initialColor = util.extend({}, rgba);\n    }\n\n    this.color = rgba;\n    let hsv = util.RGBToHSV(rgba.r, rgba.g, rgba.b);\n\n    let angleConvert = 2 * Math.PI;\n    let radius = this.r * hsv.s;\n    let x = this.centerCoordinates.x + radius * Math.sin(angleConvert * hsv.h);\n    let y = this.centerCoordinates.y + radius * Math.cos(angleConvert * hsv.h);\n\n    this.colorPickerSelector.style.left = x - 0.5 * this.colorPickerSelector.clientWidth + 'px';\n    this.colorPickerSelector.style.top = y - 0.5 * this.colorPickerSelector.clientHeight + 'px';\n\n    this._updatePicker(rgba);\n  }\n\n\n  /**\n   * bound to opacity control\n   * @param {number} value\n   * @private\n   */\n  _setOpacity(value) {\n    this.color.a = value / 100;\n    this._updatePicker(this.color);\n  }\n\n\n  /**\n   * bound to brightness control\n   * @param {number} value\n   * @private\n   */\n  _setBrightness(value) {\n    let hsv = util.RGBToHSV(this.color.r, this.color.g, this.color.b);\n    hsv.v = value / 100;\n    let rgba = util.HSVToRGB(hsv.h, hsv.s, hsv.v);\n    rgba['a'] = this.color.a;\n    this.color = rgba;\n    this._updatePicker();\n  }\n\n\n  /**\n   * update the color picker. A black circle overlays the hue circle to mimic the brightness decreasing.\n   * @param {Object} rgba\n   * @private\n   */\n  _updatePicker(rgba = this.color) {\n    let hsv = util.RGBToHSV(rgba.r, rgba.g, rgba.b);\n    let ctx = this.colorPickerCanvas.getContext('2d');\n    if (this.pixelRation === undefined) {\n      this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n      ctx.mozBackingStorePixelRatio ||\n      ctx.msBackingStorePixelRatio ||\n      ctx.oBackingStorePixelRatio ||\n      ctx.backingStorePixelRatio || 1);\n    }\n    ctx.setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n\n    // clear the canvas\n    let w = this.colorPickerCanvas.clientWidth;\n    let h = this.colorPickerCanvas.clientHeight;\n    ctx.clearRect(0, 0, w, h);\n\n    ctx.putImageData(this.hueCircle, 0,0);\n    ctx.fillStyle = 'rgba(0,0,0,' + (1- hsv.v) + ')';\n    ctx.circle(this.centerCoordinates.x, this.centerCoordinates.y, this.r);\n    ctx.fill();\n\n    this.brightnessRange.value = 100 * hsv.v;\n    this.opacityRange.value    = 100 * rgba.a;\n\n    this.initialColorDiv.style.backgroundColor = 'rgba(' + this.initialColor.r + ',' + this.initialColor.g + ',' + this.initialColor.b + ',' + this.initialColor.a + ')';\n    this.newColorDiv.style.backgroundColor = 'rgba(' + this.color.r + ',' + this.color.g + ',' + this.color.b + ',' + this.color.a + ')';\n  }\n\n\n  /**\n   * used by create to set the size of the canvas.\n   * @private\n   */\n  _setSize() {\n    this.colorPickerCanvas.style.width = '100%';\n    this.colorPickerCanvas.style.height = '100%';\n\n    this.colorPickerCanvas.width = 289 * this.pixelRatio;\n    this.colorPickerCanvas.height = 289 * this.pixelRatio;\n  }\n\n\n  /**\n   * create all dom elements\n   * TODO: cleanup, lots of similar dom elements\n   * @private\n   */\n  _create() {\n    this.frame = document.createElement('div');\n    this.frame.className = 'vis-color-picker';\n\n    this.colorPickerDiv = document.createElement('div');\n    this.colorPickerSelector = document.createElement('div');\n    this.colorPickerSelector.className = 'vis-selector';\n    this.colorPickerDiv.appendChild(this.colorPickerSelector);\n\n    this.colorPickerCanvas = document.createElement('canvas');\n    this.colorPickerDiv.appendChild(this.colorPickerCanvas);\n\n    if (!this.colorPickerCanvas.getContext) {\n      let noCanvas = document.createElement( 'DIV' );\n      noCanvas.style.color = 'red';\n      noCanvas.style.fontWeight =  'bold' ;\n      noCanvas.style.padding =  '10px';\n      noCanvas.innerHTML =  'Error: your browser does not support HTML canvas';\n      this.colorPickerCanvas.appendChild(noCanvas);\n    }\n    else {\n      let ctx = this.colorPickerCanvas.getContext(\"2d\");\n      this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n      ctx.mozBackingStorePixelRatio ||\n      ctx.msBackingStorePixelRatio ||\n      ctx.oBackingStorePixelRatio ||\n      ctx.backingStorePixelRatio || 1);\n      this.colorPickerCanvas.getContext(\"2d\").setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n    }\n\n    this.colorPickerDiv.className = 'vis-color';\n\n    this.opacityDiv = document.createElement('div');\n    this.opacityDiv.className = 'vis-opacity';\n\n    this.brightnessDiv = document.createElement('div');\n    this.brightnessDiv.className = 'vis-brightness';\n\n    this.arrowDiv = document.createElement('div');\n    this.arrowDiv.className = 'vis-arrow';\n\n    this.opacityRange = document.createElement('input');\n    try {\n      this.opacityRange.type = 'range'; // Not supported on IE9\n      this.opacityRange.min = '0';\n      this.opacityRange.max = '100';\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    this.opacityRange.value = '100';\n    this.opacityRange.className = 'vis-range';\n\n    this.brightnessRange = document.createElement('input');\n    try {\n      this.brightnessRange.type = 'range'; // Not supported on IE9\n      this.brightnessRange.min = '0';\n      this.brightnessRange.max = '100';\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    this.brightnessRange.value = '100';\n    this.brightnessRange.className = 'vis-range';\n\n    this.opacityDiv.appendChild(this.opacityRange);\n    this.brightnessDiv.appendChild(this.brightnessRange);\n\n    var me = this;\n    this.opacityRange.onchange = function () {me._setOpacity(this.value);};\n    this.opacityRange.oninput  = function () {me._setOpacity(this.value);};\n    this.brightnessRange.onchange = function () {me._setBrightness(this.value);};\n    this.brightnessRange.oninput  = function () {me._setBrightness(this.value);};\n\n    this.brightnessLabel = document.createElement(\"div\");\n    this.brightnessLabel.className = \"vis-label vis-brightness\";\n    this.brightnessLabel.innerHTML = 'brightness:';\n\n    this.opacityLabel = document.createElement(\"div\");\n    this.opacityLabel.className = \"vis-label vis-opacity\";\n    this.opacityLabel.innerHTML = 'opacity:';\n\n    this.newColorDiv = document.createElement(\"div\");\n    this.newColorDiv.className = \"vis-new-color\";\n    this.newColorDiv.innerHTML = 'new';\n\n    this.initialColorDiv = document.createElement(\"div\");\n    this.initialColorDiv.className = \"vis-initial-color\";\n    this.initialColorDiv.innerHTML = 'initial';\n\n    this.cancelButton = document.createElement(\"div\");\n    this.cancelButton.className = \"vis-button vis-cancel\";\n    this.cancelButton.innerHTML = 'cancel';\n    this.cancelButton.onclick = this._hide.bind(this, false);\n\n    this.applyButton = document.createElement(\"div\");\n    this.applyButton.className = \"vis-button vis-apply\";\n    this.applyButton.innerHTML = 'apply';\n    this.applyButton.onclick = this._apply.bind(this);\n\n    this.saveButton = document.createElement(\"div\");\n    this.saveButton.className = \"vis-button vis-save\";\n    this.saveButton.innerHTML = 'save';\n    this.saveButton.onclick = this._save.bind(this);\n\n    this.loadButton = document.createElement(\"div\");\n    this.loadButton.className = \"vis-button vis-load\";\n    this.loadButton.innerHTML = 'load last';\n    this.loadButton.onclick = this._loadLast.bind(this);\n\n    this.frame.appendChild(this.colorPickerDiv);\n    this.frame.appendChild(this.arrowDiv);\n    this.frame.appendChild(this.brightnessLabel);\n    this.frame.appendChild(this.brightnessDiv);\n    this.frame.appendChild(this.opacityLabel);\n    this.frame.appendChild(this.opacityDiv);\n    this.frame.appendChild(this.newColorDiv);\n    this.frame.appendChild(this.initialColorDiv);\n\n    this.frame.appendChild(this.cancelButton);\n    this.frame.appendChild(this.applyButton);\n    this.frame.appendChild(this.saveButton);\n    this.frame.appendChild(this.loadButton);\n  }\n\n\n  /**\n   * bind hammer to the color picker\n   * @private\n   */\n  _bindHammer() {\n    this.drag = {};\n    this.pinch = {};\n    this.hammer = new Hammer(this.colorPickerCanvas);\n    this.hammer.get('pinch').set({enable: true});\n\n    hammerUtil.onTouch(this.hammer, (event) => {this._moveSelector(event)});\n    this.hammer.on('tap',       (event) => {this._moveSelector(event)});\n    this.hammer.on('panstart',  (event) => {this._moveSelector(event)});\n    this.hammer.on('panmove',   (event) => {this._moveSelector(event)});\n    this.hammer.on('panend',    (event) => {this._moveSelector(event)});\n  }\n\n\n  /**\n   * generate the hue circle. This is relatively heavy (200ms) and is done only once on the first time it is shown.\n   * @private\n   */\n  _generateHueCircle() {\n    if (this.generated === false) {\n      let ctx = this.colorPickerCanvas.getContext('2d');\n      if (this.pixelRation === undefined) {\n        this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n        ctx.mozBackingStorePixelRatio ||\n        ctx.msBackingStorePixelRatio ||\n        ctx.oBackingStorePixelRatio ||\n        ctx.backingStorePixelRatio || 1);\n      }\n      ctx.setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n\n      // clear the canvas\n      let w = this.colorPickerCanvas.clientWidth;\n      let h = this.colorPickerCanvas.clientHeight;\n      ctx.clearRect(0, 0, w, h);\n\n\n      // draw hue circle\n      let x, y, hue, sat;\n      this.centerCoordinates = {x: w * 0.5, y: h * 0.5};\n      this.r = 0.49 * w;\n      let angleConvert = (2 * Math.PI) / 360;\n      let hfac = 1 / 360;\n      let sfac = 1 / this.r;\n      let rgb;\n      for (hue = 0; hue < 360; hue++) {\n        for (sat = 0; sat < this.r; sat++) {\n          x = this.centerCoordinates.x + sat * Math.sin(angleConvert * hue);\n          y = this.centerCoordinates.y + sat * Math.cos(angleConvert * hue);\n          rgb = util.HSVToRGB(hue * hfac, sat * sfac, 1);\n          ctx.fillStyle = 'rgb(' + rgb.r + ',' + rgb.g + ',' + rgb.b + ')';\n          ctx.fillRect(x - 0.5, y - 0.5, 2, 2);\n        }\n      }\n      ctx.strokeStyle = 'rgba(0,0,0,1)';\n      ctx.circle(this.centerCoordinates.x, this.centerCoordinates.y, this.r);\n      ctx.stroke();\n\n      this.hueCircle = ctx.getImageData(0,0,w,h);\n    }\n    this.generated = true;\n  }\n\n\n  /**\n   * move the selector. This is called by hammer functions.\n   *\n   * @param {Event}  event   The event\n   * @private\n   */\n  _moveSelector(event) {\n    let rect = this.colorPickerDiv.getBoundingClientRect();\n    let left = event.center.x - rect.left;\n    let top = event.center.y - rect.top;\n\n    let centerY = 0.5 * this.colorPickerDiv.clientHeight;\n    let centerX = 0.5 * this.colorPickerDiv.clientWidth;\n\n    let x = left - centerX;\n    let y = top - centerY;\n\n    let angle = Math.atan2(x,y);\n    let radius = 0.98 * Math.min(Math.sqrt(x * x + y * y), centerX);\n\n    let newTop = Math.cos(angle) * radius + centerY;\n    let newLeft = Math.sin(angle) * radius + centerX;\n\n    this.colorPickerSelector.style.top = newTop - 0.5 * this.colorPickerSelector.clientHeight + 'px';\n    this.colorPickerSelector.style.left = newLeft - 0.5 * this.colorPickerSelector.clientWidth + 'px';\n\n    // set color\n    let h = angle / (2 * Math.PI);\n    h = h < 0 ? h + 1 : h;\n    let s = radius / this.r;\n    let hsv = util.RGBToHSV(this.color.r, this.color.g, this.color.b);\n    hsv.h = h;\n    hsv.s = s;\n    let rgba = util.HSVToRGB(hsv.h, hsv.s, hsv.v);\n    rgba['a'] = this.color.a;\n    this.color = rgba;\n\n    // update previews\n    this.initialColorDiv.style.backgroundColor = 'rgba(' + this.initialColor.r + ',' + this.initialColor.g + ',' + this.initialColor.b + ',' + this.initialColor.a + ')';\n    this.newColorDiv.style.backgroundColor = 'rgba(' + this.color.r + ',' + this.color.g + ',' + this.color.b + ',' + this.color.a + ')';\n  }\n}\n\nexport default ColorPicker;\n","import util from '../util';\nimport ColorPicker from './ColorPicker';\n\nimport './configuration.css';\n\n/**\n * The way this works is for all properties of this.possible options, you can supply the property name in any form to list the options.\n * Boolean options are recognised as Boolean\n * Number options should be written as array: [default value, min value, max value, stepsize]\n * Colors should be written as array: ['color', '#ffffff']\n * Strings with should be written as array: [option1, option2, option3, ..]\n *\n * The options are matched with their counterparts in each of the modules and the values used in the configuration are\n */\nclass Configurator {\n  /**\n   * @param {Object} parentModule        | the location where parentModule.setOptions() can be called\n   * @param {Object} defaultContainer    | the default container of the module\n   * @param {Object} configureOptions    | the fully configured and predefined options set found in allOptions.js\n   * @param {number} pixelRatio          | canvas pixel ratio\n   */\n  constructor(parentModule, defaultContainer, configureOptions, pixelRatio = 1) {\n    this.parent = parentModule;\n    this.changedOptions = [];\n    this.container = defaultContainer;\n    this.allowCreation = false;\n\n    this.options = {};\n    this.initialized = false;\n    this.popupCounter = 0;\n    this.defaultOptions = {\n      enabled: false,\n      filter: true,\n      container: undefined,\n      showButton: true\n    };\n    util.extend(this.options, this.defaultOptions);\n\n    this.configureOptions = configureOptions;\n    this.moduleOptions = {};\n    this.domElements = [];\n    this.popupDiv = {};\n    this.popupLimit = 5;\n    this.popupHistory = {};\n    this.colorPicker = new ColorPicker(pixelRatio);\n    this.wrapper = undefined;\n  }\n\n\n  /**\n   * refresh all options.\n   * Because all modules parse their options by themselves, we just use their options. We copy them here.\n   *\n   * @param {Object} options\n   */\n  setOptions(options) {\n    if (options !== undefined) {\n      // reset the popup history because the indices may have been changed.\n      this.popupHistory = {};\n      this._removePopup();\n\n      let enabled = true;\n      if (typeof options === 'string') {\n        this.options.filter = options;\n      }\n      else if (options instanceof Array) {\n        this.options.filter = options.join();\n      }\n      else if (typeof options === 'object') {\n        if (options == null) {\n          throw new TypeError('options cannot be null');\n        }\n        if (options.container !== undefined) {\n          this.options.container = options.container;\n        }\n        if (options.filter !== undefined) {\n          this.options.filter = options.filter;\n        }\n        if (options.showButton !== undefined) {\n          this.options.showButton = options.showButton;\n        }\n        if (options.enabled !== undefined) {\n          enabled = options.enabled;\n        }\n      }\n      else if (typeof options === 'boolean') {\n        this.options.filter = true;\n        enabled = options;\n      }\n      else if (typeof options === 'function') {\n        this.options.filter = options;\n        enabled = true;\n      }\n      if (this.options.filter === false) {\n        enabled = false;\n      }\n\n      this.options.enabled = enabled;\n    }\n    this._clean();\n  }\n\n  /**\n   *\n   * @param {Object} moduleOptions\n   */\n  setModuleOptions(moduleOptions) {\n    this.moduleOptions = moduleOptions;\n    if (this.options.enabled === true) {\n      this._clean();\n      if (this.options.container !== undefined) {\n        this.container = this.options.container;\n      }\n      this._create();\n    }\n  }\n\n  /**\n   * Create all DOM elements\n   * @private\n   */\n  _create() {\n    this._clean();\n    this.changedOptions = [];\n\n    let filter = this.options.filter;\n    let counter = 0;\n    let show = false;\n    for (let option in this.configureOptions) {\n      if (this.configureOptions.hasOwnProperty(option)) {\n        this.allowCreation = false;\n        show = false;\n        if (typeof filter === 'function') {\n          show = filter(option,[]);\n          show = show || this._handleObject(this.configureOptions[option], [option], true);\n        }\n        else if (filter === true || filter.indexOf(option) !== -1) {\n          show = true;\n        }\n\n        if (show !== false) {\n          this.allowCreation = true;\n\n          // linebreak between categories\n          if (counter > 0) {\n            this._makeItem([]);\n          }\n          // a header for the category\n          this._makeHeader(option);\n\n          // get the sub options\n          this._handleObject(this.configureOptions[option], [option]);\n        }\n        counter++;\n      }\n    }\n    this._makeButton();\n    this._push();\n    //~ this.colorPicker.insertTo(this.container);\n  }\n\n\n  /**\n   * draw all DOM elements on the screen\n   * @private\n   */\n  _push() {\n    this.wrapper = document.createElement('div');\n    this.wrapper.className = 'vis-configuration-wrapper';\n    this.container.appendChild(this.wrapper);\n    for (var i = 0; i < this.domElements.length; i++) {\n      this.wrapper.appendChild(this.domElements[i]);\n    }\n\n    this._showPopupIfNeeded()\n  }\n\n\n  /**\n   * delete all DOM elements\n   * @private\n   */\n  _clean() {\n    for (var i = 0; i < this.domElements.length; i++) {\n      this.wrapper.removeChild(this.domElements[i]);\n    }\n\n    if (this.wrapper !== undefined) {\n      this.container.removeChild(this.wrapper);\n      this.wrapper = undefined;\n    }\n    this.domElements = [];\n\n    this._removePopup();\n  }\n\n\n  /**\n   * get the value from the actualOptions if it exists\n   * @param {array} path    | where to look for the actual option\n   * @returns {*}\n   * @private\n   */\n  _getValue(path) {\n    let base = this.moduleOptions;\n    for (let i = 0; i < path.length; i++) {\n      if (base[path[i]] !== undefined) {\n        base = base[path[i]];\n      }\n      else {\n        base = undefined;\n        break;\n      }\n    }\n    return base;\n  }\n\n\n  /**\n   * all option elements are wrapped in an item\n   * @param {Array} path    | where to look for the actual option\n   * @param {Array.<Element>} domElements\n   * @returns {number}\n   * @private\n   */\n  _makeItem(path, ...domElements) {\n    if (this.allowCreation === true) {\n      let item = document.createElement('div');\n      item.className = 'vis-configuration vis-config-item vis-config-s' + path.length;\n      domElements.forEach((element) => {\n        item.appendChild(element);\n      });\n      this.domElements.push(item);\n      return this.domElements.length;\n    }\n    return 0;\n  }\n\n\n  /**\n   * header for major subjects\n   * @param {string} name\n   * @private\n   */\n  _makeHeader(name) {\n    let div = document.createElement('div');\n    div.className = 'vis-configuration vis-config-header';\n    div.innerHTML = name;\n    this._makeItem([],div);\n  }\n\n\n  /**\n   * make a label, if it is an object label, it gets different styling.\n   * @param {string} name\n   * @param {array} path    | where to look for the actual option\n   * @param {string} objectLabel\n   * @returns {HTMLElement}\n   * @private\n   */\n  _makeLabel(name, path, objectLabel = false) {\n    let div = document.createElement('div');\n    div.className = 'vis-configuration vis-config-label vis-config-s' + path.length;\n    if (objectLabel === true) {\n      div.innerHTML = '<i><b>' + name + ':</b></i>';\n    }\n    else {\n      div.innerHTML = name + ':';\n    }\n    return div;\n  }\n\n\n  /**\n   * make a dropdown list for multiple possible string optoins\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeDropdown(arr, value, path) {\n    let select = document.createElement('select');\n    select.className = 'vis-configuration vis-config-select';\n    let selectedValue = 0;\n    if (value !== undefined) {\n      if (arr.indexOf(value) !== -1) {\n        selectedValue = arr.indexOf(value);\n      }\n    }\n\n    for (let i = 0; i < arr.length; i++) {\n      let option = document.createElement('option');\n      option.value = arr[i];\n      if (i === selectedValue) {\n        option.selected = 'selected';\n      }\n      option.innerHTML = arr[i];\n      select.appendChild(option);\n    }\n\n    let me = this;\n    select.onchange = function () {me._update(this.value, path);};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, select);\n  }\n\n\n  /**\n   * make a range object for numeric options\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeRange(arr, value, path) {\n    let defaultValue = arr[0];\n    let min = arr[1];\n    let max = arr[2];\n    let step = arr[3];\n    let range = document.createElement('input');\n    range.className = 'vis-configuration vis-config-range';\n    try {\n      range.type = 'range'; // not supported on IE9\n      range.min = min;\n      range.max = max;\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    range.step = step;\n\n    // set up the popup settings in case they are needed.\n    let popupString = '';\n    let popupValue = 0;\n\n    if (value !== undefined) {\n      let factor = 1.20;\n      if (value < 0 && value * factor < min) {\n        range.min = Math.ceil(value * factor);\n        popupValue = range.min;\n        popupString = 'range increased';\n      }\n      else if (value / factor < min) {\n        range.min = Math.ceil(value / factor);\n        popupValue = range.min;\n        popupString = 'range increased';\n      }\n      if (value * factor > max && max !== 1) {\n        range.max = Math.ceil(value * factor);\n        popupValue = range.max;\n        popupString = 'range increased';\n      }\n      range.value = value;\n    }\n    else {\n      range.value = defaultValue;\n    }\n\n    let input = document.createElement('input');\n    input.className = 'vis-configuration vis-config-rangeinput';\n    input.value = Number(range.value);\n\n    var me = this;\n    range.onchange = function () {input.value = this.value; me._update(Number(this.value), path);};\n    range.oninput  = function () {input.value = this.value; };\n\n    let label = this._makeLabel(path[path.length-1], path);\n    let itemIndex = this._makeItem(path, label, range, input);\n\n    // if a popup is needed AND it has not been shown for this value, show it.\n    if (popupString !== '' && this.popupHistory[itemIndex] !== popupValue) {\n      this.popupHistory[itemIndex] = popupValue;\n      this._setupPopup(popupString, itemIndex);\n    }\n  }\n\n  /**\n   * make a button object\n   * @private\n   */\n  _makeButton() {\n    if (this.options.showButton === true) {\n      let generateButton = document.createElement('div');\n      generateButton.className = 'vis-configuration vis-config-button';\n      generateButton.innerHTML = 'generate options';\n      generateButton.onclick =     () => {this._printOptions();};\n      generateButton.onmouseover = () => {generateButton.className = 'vis-configuration vis-config-button hover';};\n      generateButton.onmouseout =  () => {generateButton.className = 'vis-configuration vis-config-button';};\n\n      this.optionsContainer = document.createElement('div');\n      this.optionsContainer.className = 'vis-configuration vis-config-option-container';\n\n      this.domElements.push(this.optionsContainer);\n      this.domElements.push(generateButton);\n    }\n  }\n\n\n  /**\n   * prepare the popup\n   * @param {string} string\n   * @param {number} index\n   * @private\n   */\n  _setupPopup(string, index) {\n    if (this.initialized === true && this.allowCreation === true && this.popupCounter < this.popupLimit) {\n      let div = document.createElement(\"div\");\n      div.id = \"vis-configuration-popup\";\n      div.className = \"vis-configuration-popup\";\n      div.innerHTML = string;\n      div.onclick = () => {this._removePopup()};\n      this.popupCounter += 1;\n      this.popupDiv = {html:div, index:index};\n    }\n  }\n\n\n  /**\n   * remove the popup from the dom\n   * @private\n   */\n  _removePopup() {\n    if (this.popupDiv.html !== undefined) {\n      this.popupDiv.html.parentNode.removeChild(this.popupDiv.html);\n      clearTimeout(this.popupDiv.hideTimeout);\n      clearTimeout(this.popupDiv.deleteTimeout);\n      this.popupDiv = {};\n    }\n  }\n\n\n  /**\n   * Show the popup if it is needed.\n   * @private\n   */\n  _showPopupIfNeeded() {\n    if (this.popupDiv.html !== undefined) {\n      let correspondingElement = this.domElements[this.popupDiv.index];\n      let rect = correspondingElement.getBoundingClientRect();\n      this.popupDiv.html.style.left = rect.left + \"px\";\n      this.popupDiv.html.style.top = rect.top - 30 + \"px\"; // 30 is the height;\n      document.body.appendChild(this.popupDiv.html)\n      this.popupDiv.hideTimeout = setTimeout(() => {\n        this.popupDiv.html.style.opacity = 0;\n      },1500);\n      this.popupDiv.deleteTimeout = setTimeout(() => {\n        this._removePopup();\n      },1800)\n    }\n  }\n\n  /**\n   * make a checkbox for boolean options.\n   * @param {number} defaultValue\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeCheckbox(defaultValue, value, path) {\n    var checkbox = document.createElement('input');\n    checkbox.type = 'checkbox';\n    checkbox.className = 'vis-configuration vis-config-checkbox';\n    checkbox.checked = defaultValue;\n    if (value !== undefined) {\n      checkbox.checked = value;\n      if (value !== defaultValue) {\n        if (typeof defaultValue === 'object') {\n          if (value !== defaultValue.enabled) {\n            this.changedOptions.push({path:path, value:value});\n          }\n        }\n        else {\n          this.changedOptions.push({path:path, value:value});\n        }\n      }\n    }\n\n    let me = this;\n    checkbox.onchange = function() {me._update(this.checked, path)};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, checkbox);\n  }\n\n  /**\n   * make a text input field for string options.\n   * @param {number} defaultValue\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeTextInput(defaultValue, value, path) {\n    var checkbox = document.createElement('input');\n    checkbox.type = 'text';\n    checkbox.className = 'vis-configuration vis-config-text';\n    checkbox.value = value;\n    if (value !== defaultValue) {\n      this.changedOptions.push({path:path, value:value});\n    }\n\n    let me = this;\n    checkbox.onchange = function() {me._update(this.value, path)};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, checkbox);\n  }\n\n\n  /**\n   * make a color field with a color picker for color fields\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeColorField(arr, value, path) {\n    let defaultColor = arr[1];\n    let div = document.createElement('div');\n    value = value === undefined ? defaultColor : value;\n\n    if (value !== 'none') {\n      div.className = 'vis-configuration vis-config-colorBlock';\n      div.style.backgroundColor = value;\n    }\n    else {\n      div.className = 'vis-configuration vis-config-colorBlock none';\n    }\n\n    value = value === undefined ? defaultColor : value;\n    div.onclick = () => {\n      this._showColorPicker(value,div,path);\n    };\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path,label, div);\n  }\n\n\n  /**\n   * used by the color buttons to call the color picker.\n   * @param {number} value\n   * @param {HTMLElement} div\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _showColorPicker(value, div, path) {\n    // clear the callback from this div\n    div.onclick = function() {};\n\n    this.colorPicker.insertTo(div);\n    this.colorPicker.show();\n\n    this.colorPicker.setColor(value);\n    this.colorPicker.setUpdateCallback((color) => {\n      let colorString = 'rgba(' + color.r + ',' + color.g + ',' + color.b + ',' + color.a + ')';\n      div.style.backgroundColor = colorString;\n      this._update(colorString,path);\n    });\n\n    // on close of the colorpicker, restore the callback.\n    this.colorPicker.setCloseCallback(() => {\n      div.onclick = () => {\n        this._showColorPicker(value,div,path);\n      };\n    });\n  }\n\n\n  /**\n   * parse an object and draw the correct items\n   * @param {Object} obj\n   * @param {array} [path=[]]    | where to look for the actual option\n   * @param {boolean} [checkOnly=false]\n   * @returns {boolean}\n   * @private\n   */\n  _handleObject(obj, path = [], checkOnly = false) {\n    let show = false;\n    let filter = this.options.filter;\n    let visibleInSet = false;\n    for (let subObj in obj) {\n      if (obj.hasOwnProperty(subObj)) {\n        show = true;\n        let item = obj[subObj];\n        let newPath = util.copyAndExtendArray(path, subObj);\n        if (typeof filter === 'function') {\n          show = filter(subObj,path);\n\n          // if needed we must go deeper into the object.\n          if (show === false) {\n            if (!(item instanceof Array) && typeof item !== 'string' && typeof item !== 'boolean' && item instanceof Object) {\n              this.allowCreation = false;\n              show = this._handleObject(item, newPath, true);\n              this.allowCreation = checkOnly === false;\n            }\n          }\n        }\n\n        if (show !== false) {\n          visibleInSet = true;\n          let value = this._getValue(newPath);\n\n          if (item instanceof Array) {\n            this._handleArray(item, value, newPath);\n          }\n          else if (typeof item === 'string') {\n            this._makeTextInput(item, value, newPath);\n          }\n          else if (typeof item === 'boolean') {\n            this._makeCheckbox(item, value, newPath);\n          }\n          else if (item instanceof Object) {\n            // collapse the physics options that are not enabled\n            let draw = true;\n            if (path.indexOf('physics') !== -1) {\n              if (this.moduleOptions.physics.solver !== subObj) {\n                draw = false;\n              }\n            }\n\n            if (draw === true) {\n              // initially collapse options with an disabled enabled option.\n              if (item.enabled !== undefined) {\n                let enabledPath = util.copyAndExtendArray(newPath, 'enabled');\n                let enabledValue = this._getValue(enabledPath);\n                if (enabledValue === true) {\n                  let label = this._makeLabel(subObj, newPath, true);\n                  this._makeItem(newPath, label);\n                  visibleInSet = this._handleObject(item, newPath) || visibleInSet;\n                }\n                else {\n                  this._makeCheckbox(item, enabledValue, newPath);\n                }\n              }\n              else {\n                let label = this._makeLabel(subObj, newPath, true);\n                this._makeItem(newPath, label);\n                visibleInSet = this._handleObject(item, newPath) || visibleInSet;\n              }\n            }\n          }\n          else {\n            console.error('dont know how to handle', item, subObj, newPath);\n          }\n        }\n      }\n    }\n    return visibleInSet;\n  }\n\n\n  /**\n   * handle the array type of option\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _handleArray(arr, value, path) {\n    if (typeof arr[0] === 'string' && arr[0] === 'color') {\n      this._makeColorField(arr, value, path);\n      if (arr[1] !== value) {this.changedOptions.push({path:path, value:value});}\n    }\n    else if (typeof arr[0] === 'string') {\n      this._makeDropdown(arr, value, path);\n      if (arr[0] !== value) {this.changedOptions.push({path:path, value:value});}\n    }\n    else if (typeof arr[0] === 'number') {\n      this._makeRange(arr, value, path);\n      if (arr[0] !== value) {this.changedOptions.push({path:path, value:Number(value)});}\n    }\n  }\n\n\n\n  /**\n   * called to update the network with the new settings.\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _update(value, path) {\n    let options = this._constructOptions(value,path);\n\n    if (this.parent.body && this.parent.body.emitter && this.parent.body.emitter.emit) {\n      this.parent.body.emitter.emit(\"configChange\", options);\n    }\n    this.initialized = true;\n    this.parent.setOptions(options);\n  }\n\n\n  /**\n   *\n   * @param {string|Boolean} value\n   * @param {Array.<string>} path\n   * @param {{}} optionsObj\n   * @returns {{}}\n   * @private\n   */\n  _constructOptions(value, path, optionsObj = {}) {\n    let pointer = optionsObj;\n\n    // when dropdown boxes can be string or boolean, we typecast it into correct types\n    value = value === 'true'  ? true  : value;\n    value = value === 'false' ? false : value;\n\n    for (let i = 0; i < path.length; i++) {\n      if (path[i] !== 'global') {\n        if (pointer[path[i]] === undefined) {\n          pointer[path[i]] = {};\n        }\n        if (i !== path.length - 1) {\n          pointer = pointer[path[i]];\n        }\n        else {\n          pointer[path[i]] = value;\n        }\n      }\n    }\n    return optionsObj;\n  }\n\n  /**\n   * @private\n   */\n  _printOptions() {\n    let options = this.getOptions();\n    this.optionsContainer.innerHTML = '<pre>var options = ' + JSON.stringify(options, null, 2) + '</pre>';\n  }\n\n  /**\n   *\n   * @returns {{}} options\n   */\n  getOptions() {\n    let options = {};\n    for (var i = 0; i < this.changedOptions.length; i++) {\n      this._constructOptions(this.changedOptions[i].value, this.changedOptions[i].path, options)\n    }\n    return options;\n  }\n}\n\n\nexport default Configurator;\n","import moment from '../module/moment';\nimport util, { typeCoerceDataSet } from '../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Range from './Range';\nimport Core from './Core';\nimport TimeAxis from './component/TimeAxis';\nimport CurrentTime from './component/CurrentTime';\nimport CustomTime from './component/CustomTime';\nimport ItemSet from './component/ItemSet';\n\nimport { Validator } from '../shared/Validator';\nimport { printStyle } from '../shared/Validator';\nimport { allOptions } from './optionsTimeline';\nimport { configureOptions } from './optionsTimeline';\n\nimport Configurator from '../shared/Configurator';\n\n/**\n * Create a timeline visualization\n * @extends Core\n */\nexport default class Timeline extends Core {\n  /**\n * @param {HTMLElement} container\n * @param {vis.DataSet | vis.DataView | Array} [items]\n * @param {vis.DataSet | vis.DataView | Array} [groups]\n * @param {Object} [options]  See Timeline.setOptions for the available options.\n * @constructor Timeline\n */\n  constructor(container, items, groups, options) {\n    super()\n    this.initTime = new Date();\n    this.itemsDone = false;\n\n    if (!(this instanceof Timeline)) {\n      throw new SyntaxError('Constructor must be called with the new operator');\n    }\n\n    // if the third element is options, the forth is groups (optionally);\n    if (!(Array.isArray(groups) || groups instanceof DataSet || groups instanceof DataView) && groups instanceof Object) {\n      const forthArgument = options;\n      options = groups;\n      groups = forthArgument;\n    }\n\n    // TODO: REMOVE THIS in the next MAJOR release\n    // see https://github.com/almende/vis/issues/2511\n    if (options && options.throttleRedraw) {\n      console.warn(\"Timeline option \\\"throttleRedraw\\\" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.\");\n    }\n\n    const me = this;\n    this.defaultOptions = {\n      autoResize: true,\n      longSelectPressTime: 251,\n      orientation: {\n        axis: 'bottom',   // axis orientation: 'bottom', 'top', or 'both'\n        item: 'bottom'    // not relevant\n      },\n      moment,\n    };\n    this.options = util.deepExtend({}, this.defaultOptions);\n\n    // Create the DOM, props, and emitter\n    this._create(container);\n    if (!options || (options && typeof options.rtl == \"undefined\")) {\n      this.dom.root.style.visibility = 'hidden';\n      let directionFromDom;\n      let domNode = this.dom.root;\n      while (!directionFromDom && domNode) {\n        directionFromDom = window.getComputedStyle(domNode, null).direction;\n        domNode = domNode.parentElement;\n      }\n      this.options.rtl = (directionFromDom && (directionFromDom.toLowerCase() == \"rtl\"));\n    } else {\n      this.options.rtl = options.rtl;\n    }\n\n    if (options) {\n      if (options.rollingMode) { this.options.rollingMode = options.rollingMode; }\n      if (options.onInitialDrawComplete) { this.options.onInitialDrawComplete = options.onInitialDrawComplete; }\n      if (options.onTimeout) { this.options.onTimeout = options.onTimeout; }\n      if (options.loadingScreenTemplate) { this.options.loadingScreenTemplate = options.loadingScreenTemplate; }\n    }\n\n    // Prepare loading screen\n    const loadingScreenFragment = document.createElement('div');\n    if (this.options.loadingScreenTemplate) {\n      const templateFunction = this.options.loadingScreenTemplate.bind(this);\n      const loadingScreen = templateFunction(this.dom.loadingScreen);\n      if ((loadingScreen instanceof Object) && !(loadingScreen instanceof Element)) {\n        templateFunction(loadingScreenFragment)\n      } else {\n        if (loadingScreen instanceof Element) {\n          loadingScreenFragment.innerHTML = '';\n          loadingScreenFragment.appendChild(loadingScreen);\n        }\n        else if (loadingScreen != undefined) {\n          loadingScreenFragment.innerHTML = loadingScreen;\n        }\n      }\n    }\n    this.dom.loadingScreen.appendChild(loadingScreenFragment);\n\n    // all components listed here will be repainted automatically\n    this.components = [];\n\n    this.body = {\n      dom: this.dom,\n      domProps: this.props,\n      emitter: {\n        on: this.on.bind(this),\n        off: this.off.bind(this),\n        emit: this.emit.bind(this)\n      },\n      hiddenDates: [],\n      util: {\n        getScale() {\n          return me.timeAxis.step.scale;\n        },\n        getStep() {\n          return me.timeAxis.step.step;\n        },\n\n        toScreen: me._toScreen.bind(me),\n        toGlobalScreen: me._toGlobalScreen.bind(me), // this refers to the root.width\n        toTime: me._toTime.bind(me),\n        toGlobalTime : me._toGlobalTime.bind(me)\n      }\n    };\n\n    // range\n    this.range = new Range(this.body, this.options);\n    this.components.push(this.range);\n    this.body.range = this.range;\n\n    // time axis\n    this.timeAxis = new TimeAxis(this.body, this.options);\n    this.timeAxis2 = null; // used in case of orientation option 'both'\n    this.components.push(this.timeAxis);\n\n    // current time bar\n    this.currentTime = new CurrentTime(this.body, this.options);\n    this.components.push(this.currentTime);\n\n    // item set\n    this.itemSet = new ItemSet(this.body, this.options);\n    this.components.push(this.itemSet);\n\n    this.itemsData = null;      // DataSet\n    this.groupsData = null;     // DataSet\n\n    this.dom.root.onclick = event => {\n      me.emit('click', me.getEventProperties(event))\n    };\n    this.dom.root.ondblclick = event => {\n      me.emit('doubleClick', me.getEventProperties(event))\n    };\n    this.dom.root.oncontextmenu = event => {\n      me.emit('contextmenu', me.getEventProperties(event))\n    };\n    this.dom.root.onmouseover = event => {\n      me.emit('mouseOver', me.getEventProperties(event))\n    };\n    if(window.PointerEvent) {\n      this.dom.root.onpointerdown = event => {\n        me.emit('mouseDown', me.getEventProperties(event))\n      };\n      this.dom.root.onpointermove = event => {\n        me.emit('mouseMove', me.getEventProperties(event))\n      };\n      this.dom.root.onpointerup = event => {\n        me.emit('mouseUp', me.getEventProperties(event))\n      };\n    } else {\n      this.dom.root.onmousemove = event => {\n        me.emit('mouseMove', me.getEventProperties(event))\n      };\n      this.dom.root.onmousedown = event => {\n        me.emit('mouseDown', me.getEventProperties(event))\n      };\n      this.dom.root.onmouseup = event => {\n        me.emit('mouseUp', me.getEventProperties(event))\n      };\n    }\n\n    //Single time autoscale/fit\n    this.initialFitDone = false;\n    this.on('changed', () => {\n      if (me.itemsData == null) return;\n      if (!me.initialFitDone && !me.options.rollingMode) {\n        me.initialFitDone = true;\n        if (me.options.start != undefined || me.options.end != undefined) {\n          if (me.options.start == undefined || me.options.end == undefined) {\n            var range = me.getItemRange();\n          }\n\n          const start = me.options.start != undefined ? me.options.start : range.min;\n          const end   = me.options.end   != undefined ? me.options.end   : range.max;\n          me.setWindow(start, end, {animation: false});\n        } else {\n          me.fit({animation: false});\n        }\n      }\n\n      if (!me.initialDrawDone && (me.initialRangeChangeDone || (!me.options.start && !me.options.end) \n        || me.options.rollingMode)) {\n        me.initialDrawDone = true;\n        me.itemSet.initialDrawDone = true;\n        me.dom.root.style.visibility = 'visible';\n        me.dom.loadingScreen.parentNode.removeChild(me.dom.loadingScreen);\n        if (me.options.onInitialDrawComplete) {\n          setTimeout(() => {\n            return me.options.onInitialDrawComplete();\n          }, 0)\n        }\n      }\n    });\n\n    this.on('destroyTimeline', () => {\n      me.destroy()\n    });\n\n    // apply options\n    if (options) {\n      this.setOptions(options);\n    }\n\n    this.body.emitter.on('fit', (args) => {\n      this._onFit(args);\n      this.redraw();\n    });\n\n    // IMPORTANT: THIS HAPPENS BEFORE SET ITEMS!\n    if (groups) {\n      this.setGroups(groups);\n    }\n\n    // create itemset\n    if (items) {\n      this.setItems(items);\n    }\n\n    // draw for the first time\n    this._redraw();\n  }\n\n  /**\n   * Load a configurator\n   * @return {Object}\n   * @private\n   */\n  _createConfigurator() {\n    return new Configurator(this, this.dom.container, configureOptions);\n  }\n\n  /**\n   * Force a redraw. The size of all items will be recalculated.\n   * Can be useful to manually redraw when option autoResize=false and the window\n   * has been resized, or when the items CSS has been changed.\n   *\n   * Note: this function will be overridden on construction with a trottled version\n   */\n  redraw() {\n    this.itemSet && this.itemSet.markDirty({refreshItems: true});\n    this._redraw();\n  }\n\n  /**\n   * Remove an item from the group\n   * @param {object} options\n   */\n  setOptions(options) {\n    // validate options\n    let errorFound = Validator.validate(options, allOptions);\n\n    if (errorFound === true) {\n      console.log('%cErrors have been found in the supplied options object.', printStyle);\n    }\n\n    Core.prototype.setOptions.call(this, options);\n\n    if ('type' in options) {\n      if (options.type !== this.options.type) {\n        this.options.type = options.type;\n\n        // force recreation of all items\n        const itemsData = this.itemsData;\n        if (itemsData) {\n          const selection = this.getSelection();\n          this.setItems(null);            // remove all\n          this.setItems(itemsData.rawDS); // add all\n          this.setSelection(selection);   // restore selection\n        }\n      }\n    }\n  }\n\n  /**\n   * Set items\n   * @param {vis.DataSet | Array | null} items\n   */\n  setItems(items) {\n    this.itemsDone = false;\n    \n    // convert to type DataSet when needed\n    let newDataSet;\n    if (!items) {\n      newDataSet = null;\n    }\n    else if (items instanceof DataSet || items instanceof DataView) {\n      newDataSet = typeCoerceDataSet(items);\n    }\n    else {\n      // turn an array into a dataset\n      newDataSet = typeCoerceDataSet(new DataSet(items));\n    }\n\n    // set items\n    if (this.itemsData) {\n      // stop maintaining a coerced version of the old data set\n      this.itemsData.dispose();\n    }\n    this.itemsData = newDataSet;\n    this.itemSet && this.itemSet.setItems(newDataSet != null ? newDataSet.rawDS : null);\n  }\n\n  /**\n   * Set groups\n   * @param {vis.DataSet | Array} groups\n   */\n  setGroups(groups) {\n    // convert to type DataSet when needed\n    let newDataSet;\n    if (!groups) {\n      newDataSet = null;\n    }\n    else {\n      const filter = group => group.visible !== false;\n      if (groups instanceof DataSet || groups instanceof DataView) {\n        newDataSet = new DataView(groups,{filter});\n      }\n      else {\n        // turn an array into a dataset\n        newDataSet = new DataSet(groups.filter(filter));\n      }\n    }\n\n    // This looks weird but it's necessary to prevent memory leaks.\n    //\n    // The problem is that the DataView will exist as long as the DataSet it's\n    // connected to. This will force it to swap the groups DataSet for it's own\n    // DataSet. In this arrangement it will become unreferenced from the outside\n    // and garbage collected.\n    //\n    // IMPORTANT NOTE: If `this.groupsData` is a DataView was created in this\n    // method. Even if the original is a DataView already a new one has been\n    // created and assigned to `this.groupsData`. In case this changes in the\n    // future it will be necessary to rework this!!!!\n    if (this.groupsData instanceof DataView) {\n      this.groupsData.setData(null);\n    }\n    this.groupsData = newDataSet;\n    this.itemSet.setGroups(newDataSet);\n  }\n\n  /**\n   * Set both items and groups in one go\n   * @param {{items: (Array | vis.DataSet), groups: (Array | vis.DataSet)}} data\n   */\n  setData(data) {\n    if (data && data.groups) {\n      this.setGroups(data.groups);\n    }\n\n    if (data && data.items) {\n      this.setItems(data.items);\n    }\n  }\n\n  /**\n   * Set selected items by their id. Replaces the current selection\n   * Unknown id's are silently ignored.\n   * @param {string[] | string} [ids]  An array with zero or more id's of the items to be\n   *                                selected. If ids is an empty array, all items will be\n   *                                unselected.\n   * @param {Object} [options]      Available options:\n   *                                `focus: boolean`\n   *                                    If true, focus will be set to the selected item(s)\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                                    Only applicable when option focus is true.\n   */\n  setSelection(ids, options) {\n    this.itemSet && this.itemSet.setSelection(ids);\n\n    if (options && options.focus) {\n      this.focus(ids, options);\n    }\n  }\n\n  /**\n   * Get the selected items by their id\n   * @return {Array} ids  The ids of the selected items\n   */\n  getSelection() {\n    return this.itemSet && this.itemSet.getSelection() || [];\n  }\n\n  /**\n   * Adjust the visible window such that the selected item (or multiple items)\n   * are centered on screen.\n   * @param {string | String[]} id     An item id or array with item ids\n   * @param {Object} [options]      Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                                `zoom: boolean`\n   *                                    If true (default), the timeline will\n   *                                    zoom on the element after focus it.\n   */\n  focus(id, options) {\n    if (!this.itemsData || id == undefined) return;\n\n    const ids = Array.isArray(id) ? id : [id];\n\n    // get the specified item(s)\n    const itemsData = this.itemsData.get(ids);\n\n    // calculate minimum start and maximum end of specified items\n    let start = null;\n    let end = null;\n    itemsData.forEach(itemData => {\n      const s = itemData.start.valueOf();\n      const e = 'end' in itemData ? itemData.end.valueOf() : itemData.start.valueOf();\n\n      if (start === null || s < start) {\n        start = s;\n      }\n\n      if (end === null || e > end) {\n        end = e;\n      }\n    });\n\n\n    if (start !== null && end !== null) {\n      const me = this;\n      // Use the first item for the vertical focus\n      const item = this.itemSet.items[ids[0]];\n      let startPos = this._getScrollTop() * -1;\n      let initialVerticalScroll = null;\n\n      // Setup a handler for each frame of the vertical scroll\n      const verticalAnimationFrame = (ease, willDraw, done) => {\n        const verticalScroll = getItemVerticalScroll(me, item);\n\n        if (verticalScroll === false) {\n          return; // We don't need to scroll, so do nothing\n        }\n\n        if(!initialVerticalScroll) {\n          initialVerticalScroll = verticalScroll;\n        }\n\n        if(initialVerticalScroll.itemTop == verticalScroll.itemTop && !initialVerticalScroll.shouldScroll) {\n          return; // We don't need to scroll, so do nothing\n        }\n        else if(initialVerticalScroll.itemTop != verticalScroll.itemTop && verticalScroll.shouldScroll) {\n          // The redraw shifted elements, so reset the animation to correct\n          initialVerticalScroll = verticalScroll;\n          startPos = me._getScrollTop() * -1;\n        }      \n\n        const from = startPos;\n        const to = initialVerticalScroll.scrollOffset;\n        const scrollTop = done ? to : (from + (to - from) * ease);\n\n        me._setScrollTop(-scrollTop);\n\n        if(!willDraw) {\n          me._redraw();\n        }\n      };\n\n      // Enforces the final vertical scroll position\n      const setFinalVerticalPosition = () => {\n        const finalVerticalScroll = getItemVerticalScroll(me, item);\n\n        if (finalVerticalScroll.shouldScroll && finalVerticalScroll.itemTop != initialVerticalScroll.itemTop) {\n          me._setScrollTop(-finalVerticalScroll.scrollOffset);\n          me._redraw();\n        }\n      };\n\n      // Perform one last check at the end to make sure the final vertical\n      // position is correct\n      const finalVerticalCallback = () => {\n        // Double check we ended at the proper scroll position\n        setFinalVerticalPosition();\n\n        // Let the redraw settle and finalize the position.      \n        setTimeout(setFinalVerticalPosition, 100);\n      };\n\n      // calculate the new middle and interval for the window\n      const zoom = options && options.zoom !== undefined ? options.zoom : true;\n      const middle = (start + end) / 2;\n      const interval = zoom ? (end - start) * 1.1 : Math.max(this.range.end - this.range.start, (end - start) * 1.1);\n\n      const animation = options && options.animation !== undefined ? options.animation : true;\n\n      if (!animation) {\n        // We aren't animating so set a default so that the final callback forces the vertical location\n        initialVerticalScroll = { shouldScroll: false, scrollOffset: -1, itemTop: -1 };\n      }\n\n      this.range.setRange(middle - interval / 2, middle + interval / 2, { animation }, finalVerticalCallback, verticalAnimationFrame);  \n    }\n  }\n\n  /**\n   * Set Timeline window such that it fits all items\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback]\n   */\n  fit(options, callback) {\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n    let range;\n\n    if (this.itemsData.length === 1 && this.itemsData.get()[0].end === undefined) {\n      // a single item -> don't fit, just show a range around the item from -4 to +3 days\n      range = this.getDataRange();\n      this.moveTo(range.min.valueOf(), {animation}, callback);\n    }\n    else {\n      // exactly fit the items (plus a small margin)\n      range = this.getItemRange();\n      this.range.setRange(range.min, range.max, { animation }, callback);\n    }\n  }\n\n  /**\n   * Determine the range of the items, taking into account their actual width\n   * and a margin of 10 pixels on both sides.\n   *\n   * @returns {{min: Date, max: Date}}\n   */\n  getItemRange() {\n    // get a rough approximation for the range based on the items start and end dates\n    const range = this.getDataRange();\n    let min = range.min !== null ? range.min.valueOf() : null;\n    let max = range.max !== null ? range.max.valueOf() : null;\n    let minItem = null;\n    let maxItem = null;\n\n    if (min != null && max != null) {\n      let interval = (max - min); // ms\n      if (interval <= 0) {\n        interval = 10;\n      }\n      const factor = interval / this.props.center.width;\n\n      const redrawQueue = {};\n      let redrawQueueLength = 0;\n\n      // collect redraw functions\n      util.forEach(this.itemSet.items, (item, key) => {\n        if (item.groupShowing) {\n          const returnQueue = true;\n          redrawQueue[key] = item.redraw(returnQueue);\n          redrawQueueLength = redrawQueue[key].length;\n        }\n      })\n\n      const needRedraw = redrawQueueLength > 0;\n      if (needRedraw) {\n        // redraw all regular items\n        for (let i = 0; i < redrawQueueLength; i++) {\n          util.forEach(redrawQueue, fns => {\n            fns[i]();\n          });\n        }\n      }\n\n       // calculate the date of the left side and right side of the items given\n      util.forEach(this.itemSet.items, item => {\n        const start = getStart(item);\n        const end = getEnd(item);\n        let startSide;\n        let endSide;\n\n        if (this.options.rtl) {\n          startSide  = start - (item.getWidthRight()  + 10) * factor;\n          endSide = end   + (item.getWidthLeft() + 10) * factor;\n        } else {\n          startSide  = start - (item.getWidthLeft()  + 10) * factor;\n          endSide = end   + (item.getWidthRight() + 10) * factor;\n        }\n\n        if (startSide < min) {\n          min = startSide;\n          minItem = item;\n        }\n        if (endSide > max) {\n          max = endSide;\n          maxItem = item;\n        }\n      });\n\n      if (minItem && maxItem) {\n        const lhs = minItem.getWidthLeft() + 10;\n        const rhs = maxItem.getWidthRight() + 10;\n        const delta = this.props.center.width - lhs - rhs;  // px\n\n        if (delta > 0) {\n          if (this.options.rtl) {\n            min = getStart(minItem) - rhs * interval / delta; // ms\n            max = getEnd(maxItem)   + lhs * interval / delta; // ms\n          } else {\n            min = getStart(minItem) - lhs * interval / delta; // ms\n            max = getEnd(maxItem)   + rhs * interval / delta; // ms\n          }\n        }\n      }\n    }\n\n    return {\n      min: min != null ? new Date(min) : null,\n      max: max != null ? new Date(max) : null\n    }\n  }\n\n  /**\n   * Calculate the data range of the items start and end dates\n   * @returns {{min: Date, max: Date}}\n   */\n  getDataRange() {\n    let min = null;\n    let max = null;\n\n    if (this.itemsData) {\n      this.itemsData.forEach(item => {\n        const start = util.convert(item.start, 'Date').valueOf();\n        const end   = util.convert(item.end != undefined ? item.end : item.start, 'Date').valueOf();\n        if (min === null || start < min) {\n          min = start;\n        }\n        if (max === null || end > max) {\n          max = end;\n        }\n      });\n    }\n\n    return {\n      min: min != null ? new Date(min) : null,\n      max: max != null ? new Date(max) : null\n    }\n  }\n\n  /**\n   * Generate Timeline related information from an event\n   * @param {Event} event\n   * @return {Object} An object with related information, like on which area\n   *                  The event happened, whether clicked on an item, etc.\n   */\n  getEventProperties(event) {\n    const clientX = event.center ? event.center.x : event.clientX;\n    const clientY = event.center ? event.center.y : event.clientY;\n    const centerContainerRect = this.dom.centerContainer.getBoundingClientRect();\n    const x = this.options.rtl ? centerContainerRect.right - clientX : clientX - centerContainerRect.left;\n    const y = clientY - centerContainerRect.top;\n    \n    const item  = this.itemSet.itemFromTarget(event);\n    const group = this.itemSet.groupFromTarget(event);\n    const customTime = CustomTime.customTimeFromTarget(event);\n\n    const snap = this.itemSet.options.snap || null;\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const time = this._toTime(x);\n    const snappedTime = snap ? snap(time, scale, step) : time;\n\n    const element = util.getTarget(event);\n    let what = null;\n    if (item != null)                                                    {what = 'item';}\n    else if (customTime != null)                                         {what = 'custom-time';}\n    else if (util.hasParent(element, this.timeAxis.dom.foreground))      {what = 'axis';}\n    else if (this.timeAxis2 && util.hasParent(element, this.timeAxis2.dom.foreground)) {what = 'axis';}\n    else if (util.hasParent(element, this.itemSet.dom.labelSet))         {what = 'group-label';}\n    else if (util.hasParent(element, this.currentTime.bar))              {what = 'current-time';}\n    else if (util.hasParent(element, this.dom.center))                   {what = 'background';}\n\n    return {\n      event,\n      item: item ? item.id : null,\n      isCluster: item ? !!item.isCluster: false,\n      items: item ? item.items || []: null,\n      group: group ? group.groupId : null,\n      customTime: customTime ? customTime.options.id : null,\n      what,\n      pageX: event.srcEvent ? event.srcEvent.pageX : event.pageX,\n      pageY: event.srcEvent ? event.srcEvent.pageY : event.pageY,\n      x,\n      y,\n      time,\n      snappedTime\n    }\n  }\n\n  /**\n   * Toggle Timeline rolling mode\n   */\n  toggleRollingMode() {\n    if (this.range.rolling) {\n      this.range.stopRolling();\n    } else {\n      if (this.options.rollingMode == undefined) {\n        this.setOptions(this.options)\n      }\n      this.range.startRolling();\n    }\n  }\n\n  /**\n   * redraw\n   * @private\n   */\n  _redraw() {\n    Core.prototype._redraw.call(this);\n  }\n\n  /**\n   * on fit callback\n   * @param {object} args\n   * @private\n   */\n  _onFit(args) {\n    const { start, end, animation } = args;\n    if (!end) {\n      this.moveTo(start.valueOf(), {\n        animation\n      });\n    } else {\n      this.range.setRange(start, end, {\n        animation: animation\n      });\n    }\n  }\n}\n\n/**\n *\n * @param {timeline.Item} item\n * @returns {number}\n */\nfunction getStart(item) {\n  return util.convert(item.data.start, 'Date').valueOf()\n}\n\n/**\n *\n * @param {timeline.Item} item\n * @returns {number}\n */\nfunction getEnd(item) {\n  const end = item.data.end != undefined ? item.data.end : item.data.start;\n  return util.convert(end, 'Date').valueOf();\n}\n\n/**\n * @param {vis.Timeline} timeline\n * @param {timeline.Item} item\n * @return {{shouldScroll: bool, scrollOffset: number, itemTop: number}}\n */\nfunction getItemVerticalScroll(timeline, item) {\n  if (!item.parent) {\n    // The item no longer exists, so ignore this focus.\n    return false;\n  }\n\n  const itemsetHeight = timeline.options.rtl ? timeline.props.rightContainer.height : timeline.props.leftContainer.height;\n  const contentHeight = timeline.props.center.height;\n  \n  const group = item.parent;\n  let offset = group.top;\n  let shouldScroll = true;\n  const orientation = timeline.timeAxis.options.orientation.axis;\n  \n  const itemTop = () => {\n  if (orientation == \"bottom\") {\n      return group.height - item.top - item.height;\n    }\n    else {\n      return item.top;\n    }\n  };\n\n  const currentScrollHeight = timeline._getScrollTop() * -1;\n  const targetOffset = offset + itemTop();\n  const height = item.height;\n\n  if (targetOffset < currentScrollHeight) {\n    if (offset + itemsetHeight <= offset + itemTop() + height) {\n      offset += itemTop() - timeline.itemSet.options.margin.item.vertical;\n    }\n  }\n  else if (targetOffset + height > currentScrollHeight + itemsetHeight) {\n    offset += itemTop() + height - itemsetHeight + timeline.itemSet.options.margin.item.vertical;\n  }\n  else {\n    shouldScroll = false;\n  }\n\n  offset = Math.min(offset, contentHeight - itemsetHeight);\n\n  return { shouldScroll, scrollOffset: offset, itemTop: targetOffset };\n}\n","/** DataScale */\nclass DataScale {\n  /**\n   *\n   * @param {number} start\n   * @param {number} end\n   * @param {boolean} autoScaleStart\n   * @param {boolean} autoScaleEnd\n   * @param {number} containerHeight\n   * @param {number} majorCharHeight\n   * @param {boolean} zeroAlign\n   * @param {function} formattingFunction\n   * @constructor DataScale\n   */\n  constructor(\n    start,\n    end,\n    autoScaleStart,\n    autoScaleEnd,\n    containerHeight,\n    majorCharHeight,\n    zeroAlign = false,\n    formattingFunction=false) {\n    this.majorSteps = [1, 2, 5, 10];\n    this.minorSteps = [0.25, 0.5, 1, 2];\n    this.customLines = null;\n\n    this.containerHeight = containerHeight;\n    this.majorCharHeight = majorCharHeight;\n    this._start = start;\n    this._end = end;\n\n    this.scale = 1;\n    this.minorStepIdx = -1;\n    this.magnitudefactor = 1;\n    this.determineScale();\n\n    this.zeroAlign = zeroAlign;\n    this.autoScaleStart = autoScaleStart;\n    this.autoScaleEnd = autoScaleEnd;\n\n    this.formattingFunction = formattingFunction;\n\n    if (autoScaleStart || autoScaleEnd) {\n      const me = this;\n      const roundToMinor = value => {\n        const rounded = value - (value % (me.magnitudefactor * me.minorSteps[me.minorStepIdx]));\n        if (value % (me.magnitudefactor * me.minorSteps[me.minorStepIdx]) > 0.5 * (me.magnitudefactor * me.minorSteps[me.minorStepIdx])) {\n          return rounded + (me.magnitudefactor * me.minorSteps[me.minorStepIdx]);\n        }\n        else {\n          return rounded;\n        }\n      };\n      if (autoScaleStart) {\n        this._start -= this.magnitudefactor * 2 * this.minorSteps[this.minorStepIdx];\n        this._start = roundToMinor(this._start);\n      }\n\n      if (autoScaleEnd) {\n        this._end += this.magnitudefactor * this.minorSteps[this.minorStepIdx];\n        this._end = roundToMinor(this._end);\n      }\n      this.determineScale();\n    }\n  }\n\n  /**\n   * set chart height\n   * @param {number} majorCharHeight \n   */\n  setCharHeight(majorCharHeight) {\n    this.majorCharHeight = majorCharHeight;\n  }\n\n  /**\n   * set height\n   * @param {number} containerHeight \n   */\n  setHeight(containerHeight) {\n    this.containerHeight = containerHeight;\n  }\n\n  /**\n   * determine scale\n   */\n  determineScale() {\n    const range = this._end - this._start;\n    this.scale = this.containerHeight / range;\n    const minimumStepValue = this.majorCharHeight / this.scale;\n    const orderOfMagnitude = (range > 0)\n        ? Math.round(Math.log(range) / Math.LN10)\n        : 0;\n\n    this.minorStepIdx = -1;\n    this.magnitudefactor = Math.pow(10, orderOfMagnitude);\n\n    let start = 0;\n    if (orderOfMagnitude < 0) {\n      start = orderOfMagnitude;\n    }\n\n    let solutionFound = false;\n    for (let l = start; Math.abs(l) <= Math.abs(orderOfMagnitude); l++) {\n      this.magnitudefactor = Math.pow(10, l);\n      for (let j = 0; j < this.minorSteps.length; j++) {\n        const stepSize = this.magnitudefactor * this.minorSteps[j];\n        if (stepSize >= minimumStepValue) {\n          solutionFound = true;\n          this.minorStepIdx = j;\n          break;\n        }\n      }\n      if (solutionFound === true) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * returns if value is major\n   * @param {number} value\n   * @returns {boolean} \n   */\n  is_major(value) {\n    return (value % (this.magnitudefactor * this.majorSteps[this.minorStepIdx]) === 0);\n  }\n\n  /**\n   * returns step size\n   * @returns {number} \n   */\n  getStep() {\n    return this.magnitudefactor * this.minorSteps[this.minorStepIdx];\n  }\n\n  /**\n   * returns first major\n   * @returns {number} \n   */\n  getFirstMajor() {\n    const majorStep = this.magnitudefactor * this.majorSteps[this.minorStepIdx];\n    return this.convertValue(this._start + ((majorStep - (this._start % majorStep)) % majorStep));\n  }\n\n  /**\n   * returns first major\n   * @param {date} current\n   * @returns {date} formatted date\n   */\n  formatValue(current) {\n    let returnValue = current.toPrecision(5);\n    if (typeof this.formattingFunction === 'function') {\n      returnValue = this.formattingFunction(current);\n    }\n\n    if (typeof returnValue === 'number') {\n      return `${returnValue}`;\n    }\n    else if (typeof returnValue === 'string') {\n      return returnValue;\n    }\n    else {\n      return current.toPrecision(5);\n    }\n\n  }\n\n  /**\n   * returns lines\n   * @returns {object} lines\n   */\n  getLines() {\n    const lines = [];\n    const step = this.getStep();\n    const bottomOffset = (step - (this._start % step)) % step;\n    for (let i = (this._start + bottomOffset); this._end-i > 0.00001; i += step) {\n      if (i != this._start) { //Skip the bottom line\n        lines.push({major: this.is_major(i), y: this.convertValue(i), val: this.formatValue(i)});\n      }\n    }\n    return lines;\n  }\n\n  /**\n   * follow scale\n   * @param {object} other\n   */\n  followScale(other) {\n    const oldStepIdx = this.minorStepIdx;\n    const oldStart = this._start;\n    const oldEnd = this._end;\n\n    const me = this;\n    const increaseMagnitude = () => {\n      me.magnitudefactor *= 2;\n    };\n    const decreaseMagnitude = () => {\n      me.magnitudefactor /= 2;\n    };\n\n    if ((other.minorStepIdx <= 1 && this.minorStepIdx <= 1) || (other.minorStepIdx > 1 && this.minorStepIdx > 1)) {\n      //easy, no need to change stepIdx nor multiplication factor\n    } else if (other.minorStepIdx < this.minorStepIdx) {\n      //I'm 5, they are 4 per major.\n      this.minorStepIdx = 1;\n      if (oldStepIdx == 2) {\n        increaseMagnitude();\n      } else {\n        increaseMagnitude();\n        increaseMagnitude();\n      }\n    } else {\n      //I'm 4, they are 5 per major\n      this.minorStepIdx = 2;\n      if (oldStepIdx == 1) {\n        decreaseMagnitude();\n      } else {\n        decreaseMagnitude();\n        decreaseMagnitude();\n      }\n    }\n\n    //Get masters stats:\n    const otherZero = other.convertValue(0);\n    const otherStep = other.getStep() * other.scale;\n\n    let done = false;\n    let count = 0;\n    //Loop until magnitude is correct for given constrains.\n    while (!done && count++ <5) {\n\n      //Get my stats:\n      this.scale = otherStep / (this.minorSteps[this.minorStepIdx] * this.magnitudefactor);\n      const newRange = this.containerHeight / this.scale;\n\n      //For the case the magnitudefactor has changed:\n      this._start = oldStart;\n      this._end = this._start + newRange;\n\n      const myOriginalZero = this._end * this.scale;\n      const majorStep = this.magnitudefactor * this.majorSteps[this.minorStepIdx];\n      const majorOffset = this.getFirstMajor() - other.getFirstMajor();\n\n      if (this.zeroAlign) {\n        const zeroOffset = otherZero - myOriginalZero;\n        this._end += (zeroOffset / this.scale);\n        this._start = this._end - newRange;\n      } else {\n        if (!this.autoScaleStart) {\n          this._start += majorStep - (majorOffset / this.scale);\n          this._end = this._start + newRange;\n        } else {\n          this._start -= majorOffset / this.scale;\n          this._end = this._start + newRange;\n        }\n      }\n      if (!this.autoScaleEnd && this._end > oldEnd+0.00001) {\n        //Need to decrease magnitude to prevent scale overshoot! (end)\n        decreaseMagnitude();\n        done = false;\n        continue;\n      }\n      if (!this.autoScaleStart && this._start < oldStart-0.00001) {\n        if (this.zeroAlign && oldStart >= 0) {\n          console.warn(\"Can't adhere to given 'min' range, due to zeroalign\");\n        } else {\n          //Need to decrease magnitude to prevent scale overshoot! (start)\n          decreaseMagnitude();\n          done = false;\n          continue;\n        }\n      }\n      if (this.autoScaleStart && this.autoScaleEnd && newRange < (oldEnd-oldStart)){\n        increaseMagnitude();\n        done = false;\n        continue;\n      }\n      done = true;\n    }\n  }\n\n  /**\n   * convert value\n   * @param {number} value\n   * @returns {number} \n   */\n  convertValue(value) {\n    return this.containerHeight - ((value - this._start) * this.scale);\n  }\n\n  /**\n   * returns screen to value\n   * @param {number} pixels\n   * @returns {number} \n   */\n  screenToValue(pixels) {\n    return ((this.containerHeight - pixels) / this.scale) + this._start;\n  }\n}\n\nexport default DataScale;","import { v4 as randomUUID } from \"uuid\";import util from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport Component from './Component';\nimport DataScale from './DataScale';\n\nimport './css/dataaxis.css';\n\n/** A horizontal time axis */\nclass DataAxis extends Component {\n  /**\n * @param {Object} body\n * @param {Object} [options]        See DataAxis.setOptions for the available\n *                                  options.\n * @param {SVGElement} svg\n * @param {timeline.LineGraph.options} linegraphOptions\n * @constructor DataAxis\n * @extends Component\n */\n  constructor(body, options, svg, linegraphOptions) {\n    super()\n    this.id = randomUUID();\n    this.body = body;\n\n    this.defaultOptions = {\n      orientation: 'left',  // supported: 'left', 'right'\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      icons: false,\n      majorLinesOffset: 7,\n      minorLinesOffset: 4,\n      labelOffsetX: 10,\n      labelOffsetY: 2,\n      iconWidth: 20,\n      width: '40px',\n      visible: true,\n      alignZeros: true,\n      left: {\n        range: {min: undefined, max: undefined},\n        format(value) {\n          return `${parseFloat(value.toPrecision(3))}`;\n        },\n        title: {text: undefined, style: undefined}\n      },\n      right: {\n        range: {min: undefined, max: undefined},\n        format(value) {\n          return `${parseFloat(value.toPrecision(3))}`;\n        },\n        title: {text: undefined, style: undefined}\n      }\n    };\n\n    this.linegraphOptions = linegraphOptions;\n    this.linegraphSVG = svg;\n    this.props = {};\n    this.DOMelements = { // dynamic elements\n      lines: {},\n      labels: {},\n      title: {}\n    };\n\n    this.dom = {};\n    this.scale = undefined;\n    this.range = {start: 0, end: 0};\n\n    this.options = util.extend({}, this.defaultOptions);\n    this.conversionFactor = 1;\n\n    this.setOptions(options);\n    this.width = Number((`${this.options.width}`).replace(\"px\", \"\"));\n    this.minWidth = this.width;\n    this.height = this.linegraphSVG.getBoundingClientRect().height;\n    this.hidden = false;\n\n    this.stepPixels = 25;\n    this.zeroCrossing = -1;\n    this.amountOfSteps = -1;\n\n    this.lineOffset = 0;\n    this.master = true;\n    this.masterAxis = null;\n    this.svgElements = {};\n    this.iconsRemoved = false;\n\n    this.groups = {};\n    this.amountOfGroups = 0;\n\n    // create the HTML DOM\n    this._create();\n    if (this.scale == undefined) {\n      this._redrawLabels();\n    }\n    this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n\n    const me = this;\n    this.body.emitter.on(\"verticalDrag\", () => {\n      me.dom.lineContainer.style.top = `${me.body.domProps.scrollTop}px`;\n    });\n  }\n\n  /**\n   * Adds group to data axis\n   * @param {string} label \n   * @param {object} graphOptions\n   */\n  addGroup(label, graphOptions) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.groups[label] = graphOptions;\n    }\n    this.amountOfGroups += 1;\n  }\n\n  /**\n   * updates group of data axis\n   * @param {string} label \n   * @param {object} graphOptions\n   */\n  updateGroup(label, graphOptions) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.amountOfGroups += 1;\n    }\n    this.groups[label] = graphOptions;\n  }\n\n  /**\n   * removes group of data axis\n   * @param {string} label \n   */\n  removeGroup(label) {\n    if (this.groups.hasOwnProperty(label)) {\n      delete this.groups[label];\n      this.amountOfGroups -= 1;\n    }\n  }\n\n  /**\n   * sets options\n   * @param {object} options\n   */\n  setOptions(options) {\n    if (options) {\n      let redraw = false;\n      if (this.options.orientation != options.orientation && options.orientation !== undefined) {\n        redraw = true;\n      }\n      const fields = [\n        'orientation',\n        'showMinorLabels',\n        'showMajorLabels',\n        'icons',\n        'majorLinesOffset',\n        'minorLinesOffset',\n        'labelOffsetX',\n        'labelOffsetY',\n        'iconWidth',\n        'width',\n        'visible',\n        'left',\n        'right',\n        'alignZeros'\n      ];\n      util.selectiveDeepExtend(fields, this.options, options);\n\n      this.minWidth = Number((`${this.options.width}`).replace(\"px\", \"\"));\n      if (redraw === true && this.dom.frame) {\n        this.hide();\n        this.show();\n      }\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  _create() {\n    this.dom.frame = document.createElement('div');\n    this.dom.frame.style.width = this.options.width;\n    this.dom.frame.style.height = this.height;\n\n    this.dom.lineContainer = document.createElement('div');\n    this.dom.lineContainer.style.width = '100%';\n    this.dom.lineContainer.style.height = this.height;\n    this.dom.lineContainer.style.position = 'relative';\n    this.dom.lineContainer.style.visibility = 'visible';\n    this.dom.lineContainer.style.display = 'block';\n\n    // create svg element for graph drawing.\n    this.svg = document.createElementNS('http://www.w3.org/2000/svg', \"svg\");\n    this.svg.style.position = \"absolute\";\n    this.svg.style.top = '0px';\n    this.svg.style.height = '100%';\n    this.svg.style.width = '100%';\n    this.svg.style.display = \"block\";\n    this.dom.frame.appendChild(this.svg);\n  }\n\n  /**\n   * redraws groups icons\n   */\n  _redrawGroupIcons() {\n    DOMutil.prepareElements(this.svgElements);\n\n    let x;\n    const iconWidth = this.options.iconWidth;\n    const iconHeight = 15;\n    const iconOffset = 4;\n    let y = iconOffset + 0.5 * iconHeight;\n\n    if (this.options.orientation === 'left') {\n      x = iconOffset;\n    }\n    else {\n      x = this.width - iconWidth - iconOffset;\n    }\n\n    const groupArray = Object.keys(this.groups);\n    groupArray.sort((a, b) => a < b ? -1 : 1)\n\n    for (const groupId of groupArray) {\n      if (this.groups[groupId].visible === true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] === true)) {\n        this.groups[groupId].getLegend(iconWidth, iconHeight, this.framework, x, y);\n        y += iconHeight + iconOffset;\n      }\n    }\n\n    DOMutil.cleanupElements(this.svgElements);\n    this.iconsRemoved = false;\n  }\n\n  /**\n   * Cleans up icons\n   */\n  _cleanupIcons() {\n    if (this.iconsRemoved === false) {\n      DOMutil.prepareElements(this.svgElements);\n      DOMutil.cleanupElements(this.svgElements);\n      this.iconsRemoved = true;\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  show() {\n    this.hidden = false;\n    if (!this.dom.frame.parentNode) {\n      if (this.options.orientation === 'left') {\n        this.body.dom.left.appendChild(this.dom.frame);\n      }\n      else {\n        this.body.dom.right.appendChild(this.dom.frame);\n      }\n    }\n\n    if (!this.dom.lineContainer.parentNode) {\n      this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer);\n    }\n    this.dom.lineContainer.style.display = 'block';\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  hide() {\n    this.hidden = true;\n    if (this.dom.frame.parentNode) {\n      this.dom.frame.parentNode.removeChild(this.dom.frame);\n    }\n\n    this.dom.lineContainer.style.display = 'none';\n  }\n\n  /**\n   * Set a range (start and end)\n   * @param {number} start\n   * @param {number} end\n   */\n  setRange(start, end) {\n    this.range.start = start;\n    this.range.end = end;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    let resized = false;\n    let activeGroups = 0;\n\n    // Make sure the line container adheres to the vertical scrolling.\n    this.dom.lineContainer.style.top = `${this.body.domProps.scrollTop}px`;\n\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        if (this.groups[groupId].visible === true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] === true)) {\n          activeGroups++;\n        }\n      }\n    }\n    if (this.amountOfGroups === 0 || activeGroups === 0) {\n      this.hide();\n    }\n    else {\n      this.show();\n      this.height = Number(this.linegraphSVG.style.height.replace(\"px\", \"\"));\n\n      // svg offsetheight did not work in firefox and explorer...\n      this.dom.lineContainer.style.height = `${this.height}px`;\n      this.width = this.options.visible === true ? Number((`${this.options.width}`).replace(\"px\", \"\")) : 0;\n\n      const props = this.props;\n      const frame = this.dom.frame;\n\n      // update classname\n      frame.className = 'vis-data-axis';\n\n      // calculate character width and height\n      this._calculateCharSize();\n\n      const orientation = this.options.orientation;\n      const showMinorLabels = this.options.showMinorLabels;\n      const showMajorLabels = this.options.showMajorLabels;\n\n      const backgroundHorizontalOffsetWidth = this.body.dom.backgroundHorizontal.offsetWidth;\n\n      // determine the width and height of the elements for the axis\n      props.minorLabelHeight = showMinorLabels ? props.minorCharHeight : 0;\n      props.majorLabelHeight = showMajorLabels ? props.majorCharHeight : 0;\n\n      props.minorLineWidth = backgroundHorizontalOffsetWidth - this.lineOffset - this.width + 2 * this.options.minorLinesOffset;\n      props.minorLineHeight = 1;\n      props.majorLineWidth = backgroundHorizontalOffsetWidth - this.lineOffset - this.width + 2 * this.options.majorLinesOffset;\n      props.majorLineHeight = 1;\n\n      //  take frame offline while updating (is almost twice as fast)\n      if (orientation === 'left') {\n        frame.style.top = '0';\n        frame.style.left = '0';\n        frame.style.bottom = '';\n        frame.style.width = `${this.width}px`;\n        frame.style.height = `${this.height}px`;\n        this.props.width = this.body.domProps.left.width;\n        this.props.height = this.body.domProps.left.height;\n      }\n      else { // right\n        frame.style.top = '';\n        frame.style.bottom = '0';\n        frame.style.left = '0';\n        frame.style.width = `${this.width}px`;\n        frame.style.height = `${this.height}px`;\n        this.props.width = this.body.domProps.right.width;\n        this.props.height = this.body.domProps.right.height;\n      }\n\n      resized = this._redrawLabels();\n      resized = this._isResized() || resized;\n\n      if (this.options.icons === true) {\n        this._redrawGroupIcons();\n      }\n      else {\n        this._cleanupIcons();\n      }\n\n      this._redrawTitle(orientation);\n    }\n    return resized;\n  }\n\n  /**\n   * Repaint major and minor text labels and vertical grid lines\n   *\n   * @returns {boolean}\n   * @private\n   */\n  _redrawLabels() {\n    let resized = false;\n    DOMutil.prepareElements(this.DOMelements.lines);\n    DOMutil.prepareElements(this.DOMelements.labels);\n    const orientation = this.options['orientation'];\n    const customRange = this.options[orientation].range != undefined ? this.options[orientation].range : {};\n\n    //Override range with manual options:\n    let autoScaleEnd = true;\n    if (customRange.max != undefined) {\n      this.range.end = customRange.max;\n      autoScaleEnd = false;\n    }\n    let autoScaleStart = true;\n    if (customRange.min != undefined) {\n      this.range.start = customRange.min;\n      autoScaleStart = false;\n    }\n\n    this.scale = new DataScale(\n      this.range.start,\n      this.range.end,\n      autoScaleStart,\n      autoScaleEnd,\n      this.dom.frame.offsetHeight,\n      this.props.majorCharHeight,\n      this.options.alignZeros,\n      this.options[orientation].format\n    );\n\n    if (this.master === false && this.masterAxis != undefined) {\n      this.scale.followScale(this.masterAxis.scale);\n      this.dom.lineContainer.style.display = 'none';\n    } else {\n      this.dom.lineContainer.style.display = 'block';\n    }\n\n    //Is updated in side-effect of _redrawLabel():\n    this.maxLabelSize = 0;\n\n    const lines = this.scale.getLines();\n    lines.forEach(\n      line=> {\n        const y = line.y;\n        const isMajor = line.major;\n        if (this.options['showMinorLabels'] && isMajor === false) {\n          this._redrawLabel(y - 2, line.val, orientation, 'vis-y-axis vis-minor', this.props.minorCharHeight);\n        }\n        if (isMajor) {\n          if (y >= 0) {\n            this._redrawLabel(y - 2, line.val, orientation, 'vis-y-axis vis-major', this.props.majorCharHeight);\n          }\n        }\n        if (this.master === true) {\n          if (isMajor) {\n            this._redrawLine(y, orientation, 'vis-grid vis-horizontal vis-major', this.options.majorLinesOffset, this.props.majorLineWidth);\n          }\n          else {\n            this._redrawLine(y, orientation, 'vis-grid vis-horizontal vis-minor', this.options.minorLinesOffset, this.props.minorLineWidth);\n          }\n        }\n      });\n\n    // Note that title is rotated, so we're using the height, not width!\n    let titleWidth = 0;\n    if (this.options[orientation].title !== undefined && this.options[orientation].title.text !== undefined) {\n      titleWidth = this.props.titleCharHeight;\n    }\n    const offset = this.options.icons === true ? Math.max(this.options.iconWidth, titleWidth) + this.options.labelOffsetX + 15 : titleWidth + this.options.labelOffsetX + 15;\n\n    // this will resize the yAxis to accommodate the labels.\n    if (this.maxLabelSize > (this.width - offset) && this.options.visible === true) {\n      this.width = this.maxLabelSize + offset;\n      this.options.width = `${this.width}px`;\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      this.redraw();\n      resized = true;\n    }\n    // this will resize the yAxis if it is too big for the labels.\n    else if (this.maxLabelSize < (this.width - offset) && this.options.visible === true && this.width > this.minWidth) {\n      this.width = Math.max(this.minWidth, this.maxLabelSize + offset);\n      this.options.width = `${this.width}px`;\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      this.redraw();\n      resized = true;\n    }\n    else {\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      resized = false;\n    }\n\n    return resized;\n  }\n\n  /**\n   * converts value\n   * @param {number} value\n   * @returns {number} converted number\n   */\n  convertValue(value) {\n    return this.scale.convertValue(value);\n  }\n\n  /**\n   * converts value\n   * @param {number} x\n   * @returns {number} screen value\n   */\n  screenToValue(x) {\n    return this.scale.screenToValue(x);\n  }\n\n  /**\n   * Create a label for the axis at position x\n   *\n   * @param {number} y\n   * @param {string} text\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   * @param {string} className\n   * @param {number} characterHeight\n   * @private\n   */\n  _redrawLabel(y, text, orientation, className, characterHeight) {\n    // reuse redundant label\n    const label = DOMutil.getDOMElement('div', this.DOMelements.labels, this.dom.frame); //this.dom.redundant.labels.shift();\n    label.className = className;\n    label.innerHTML = text;\n    if (orientation === 'left') {\n      label.style.left = `-${this.options.labelOffsetX}px`;\n      label.style.textAlign = \"right\";\n    }\n    else {\n      label.style.right = `-${this.options.labelOffsetX}px`;\n      label.style.textAlign = \"left\";\n    }\n\n    label.style.top = `${y - 0.5 * characterHeight + this.options.labelOffsetY}px`;\n\n    text += '';\n\n    const largestWidth = Math.max(this.props.majorCharWidth, this.props.minorCharWidth);\n    if (this.maxLabelSize < text.length * largestWidth) {\n      this.maxLabelSize = text.length * largestWidth;\n    }\n  }\n\n  /**\n   * Create a minor line for the axis at position y\n   * @param {number} y\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   * @param {string} className\n   * @param {number} offset\n   * @param {number} width\n   */\n  _redrawLine(y, orientation, className, offset, width) {\n    if (this.master === true) {\n      const line = DOMutil.getDOMElement('div', this.DOMelements.lines, this.dom.lineContainer);  //this.dom.redundant.lines.shift();\n      line.className = className;\n      line.innerHTML = '';\n\n      if (orientation === 'left') {\n        line.style.left = `${this.width - offset}px`;\n      }\n      else {\n        line.style.right = `${this.width - offset}px`;\n      }\n\n      line.style.width = `${width}px`;\n      line.style.top = `${y}px`;\n    }\n  }\n\n  /**\n   * Create a title for the axis\n   * @private\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   */\n  _redrawTitle(orientation) {\n    DOMutil.prepareElements(this.DOMelements.title);\n\n    // Check if the title is defined for this axes\n    if (this.options[orientation].title !== undefined && this.options[orientation].title.text !== undefined) {\n      const title = DOMutil.getDOMElement('div', this.DOMelements.title, this.dom.frame);\n      title.className = `vis-y-axis vis-title vis-${orientation}`;\n      title.innerHTML = this.options[orientation].title.text;\n\n      // Add style - if provided\n      if (this.options[orientation].title.style !== undefined) {\n        util.addCssText(title, this.options[orientation].title.style);\n      }\n\n      if (orientation === 'left') {\n        title.style.left = `${this.props.titleCharHeight}px`;\n      }\n      else {\n        title.style.right = `${this.props.titleCharHeight}px`;\n      }\n\n      title.style.width = `${this.height}px`;\n    }\n\n    // we need to clean up in case we did not use all elements.\n    DOMutil.cleanupElements(this.DOMelements.title);\n  }\n\n  /**\n   * Determine the size of text on the axis (both major and minor axis).\n   * The size is calculated only once and then cached in this.props.\n   * @private\n   */\n  _calculateCharSize() {\n    // determine the char width and height on the minor axis\n    if (!('minorCharHeight' in this.props)) {\n      const textMinor = document.createTextNode('0');\n      const measureCharMinor = document.createElement('div');\n      measureCharMinor.className = 'vis-y-axis vis-minor vis-measure';\n      measureCharMinor.appendChild(textMinor);\n      this.dom.frame.appendChild(measureCharMinor);\n\n      this.props.minorCharHeight = measureCharMinor.clientHeight;\n      this.props.minorCharWidth = measureCharMinor.clientWidth;\n\n      this.dom.frame.removeChild(measureCharMinor);\n    }\n\n    if (!('majorCharHeight' in this.props)) {\n      const textMajor = document.createTextNode('0');\n      const measureCharMajor = document.createElement('div');\n      measureCharMajor.className = 'vis-y-axis vis-major vis-measure';\n      measureCharMajor.appendChild(textMajor);\n      this.dom.frame.appendChild(measureCharMajor);\n\n      this.props.majorCharHeight = measureCharMajor.clientHeight;\n      this.props.majorCharWidth = measureCharMajor.clientWidth;\n\n      this.dom.frame.removeChild(measureCharMajor);\n    }\n\n    if (!('titleCharHeight' in this.props)) {\n      const textTitle = document.createTextNode('0');\n      const measureCharTitle = document.createElement('div');\n      measureCharTitle.className = 'vis-y-axis vis-title vis-measure';\n      measureCharTitle.appendChild(textTitle);\n      this.dom.frame.appendChild(measureCharTitle);\n\n      this.props.titleCharHeight = measureCharTitle.clientHeight;\n      this.props.titleCharWidth = measureCharTitle.clientWidth;\n\n      this.dom.frame.removeChild(measureCharTitle);\n    }\n  }\n}\n\nexport default DataAxis;\n","import * as DOMutil from '../../../DOMutil';\n\n/**\n *\n * @param {number | string} groupId\n * @param {Object} options   // TODO: Describe options\n *\n * @constructor Points\n */\nfunction Points(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\n/**\n * draw the data points\n *\n * @param {Array} dataset\n * @param {GraphGroup} group\n * @param {Object} framework            | SVG DOM element\n * @param {number} [offset]\n */\nPoints.draw = function (dataset, group, framework, offset) {\n  offset = offset || 0;\n  var callback = getCallback(framework, group);\n\n  for (var i = 0; i < dataset.length; i++) {\n    if (!callback) {\n      // draw the point the simple way.\n      DOMutil.drawPoint(dataset[i].screen_x + offset, dataset[i].screen_y, getGroupTemplate(group), framework.svgElements, framework.svg, dataset[i].label);\n    }\n    else {\n      var callbackResult = callback(dataset[i], group); // result might be true, false or an object\n      if (callbackResult === true || typeof callbackResult === 'object') {\n        DOMutil.drawPoint(dataset[i].screen_x + offset, dataset[i].screen_y, getGroupTemplate(group, callbackResult), framework.svgElements, framework.svg, dataset[i].label);\n      }\n    }\n  }\n};\n\nPoints.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n  var fillHeight = iconHeight * 0.5;\n\n  var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n  outline.setAttributeNS(null, \"x\", x);\n  outline.setAttributeNS(null, \"y\", y - fillHeight);\n  outline.setAttributeNS(null, \"width\", iconWidth);\n  outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n  outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n  //Don't call callback on icon\n  DOMutil.drawPoint(x + 0.5 * iconWidth, y, getGroupTemplate(group), framework.svgElements, framework.svg);\n};\n\n/**\n *\n * @param {vis.Group} group\n * @param {any} callbackResult\n * @returns {{style: *, styles: (*|string), size: *, className: *}}\n */\nfunction getGroupTemplate(group, callbackResult) {\n  callbackResult = (typeof callbackResult === 'undefined') ? {} : callbackResult;\n  return {\n    style: callbackResult.style || group.options.drawPoints.style,\n    styles: callbackResult.styles || group.options.drawPoints.styles,\n    size: callbackResult.size || group.options.drawPoints.size,\n    className: callbackResult.className || group.className\n  };\n}\n\n/**\n *\n * @param {Object} framework            | SVG DOM element\n * @param {vis.Group} group\n * @returns {function}\n */\nfunction getCallback(framework, group) {\n  var callback = undefined;\n  // check for the graph2d onRender\n  if (framework.options && framework.options.drawPoints && framework.options.drawPoints.onRender && typeof framework.options.drawPoints.onRender == 'function') {\n    callback = framework.options.drawPoints.onRender;\n  }\n\n  // override it with the group onRender if defined\n  if (group.group.options && group.group.options.drawPoints && group.group.options.drawPoints.onRender && typeof group.group.options.drawPoints.onRender == 'function') {\n    callback = group.group.options.drawPoints.onRender;\n  }\n  return callback;\n}\n\nexport default Points;\n","import * as DOMutil from '../../../DOMutil';\nimport Points from './points';\n\n/**\n *\n * @param {vis.GraphGroup.id} groupId\n * @param {Object} options   // TODO: Describe options\n * @constructor Bargraph\n */\nfunction Bargraph(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\nBargraph.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n  var fillHeight = iconHeight * 0.5;\n  var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n  outline.setAttributeNS(null, \"x\", x);\n  outline.setAttributeNS(null, \"y\", y - fillHeight);\n  outline.setAttributeNS(null, \"width\", iconWidth);\n  outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n  outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n  var barWidth = Math.round(0.3 * iconWidth);\n  var originalWidth = group.options.barChart.width;\n  var scale = originalWidth / barWidth;\n  var bar1Height = Math.round(0.4 * iconHeight);\n  var bar2Height = Math.round(0.75 * iconHeight);\n\n  var offset = Math.round((iconWidth - (2 * barWidth)) / 3);\n\n  DOMutil.drawBar(x + 0.5 * barWidth + offset, y + fillHeight - bar1Height - 1, barWidth, bar1Height, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n  DOMutil.drawBar(x + 1.5 * barWidth + offset + 2, y + fillHeight - bar2Height - 1, barWidth, bar2Height, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n\n  if (group.options.drawPoints.enabled == true) {\n    var groupTemplate = {\n      style: group.options.drawPoints.style,\n      styles: group.options.drawPoints.styles,\n      size: (group.options.drawPoints.size / scale),\n      className: group.className\n    };\n    DOMutil.drawPoint(x + 0.5 * barWidth + offset, y + fillHeight - bar1Height - 1, groupTemplate, framework.svgElements, framework.svg);\n    DOMutil.drawPoint(x + 1.5 * barWidth + offset + 2, y + fillHeight - bar2Height - 1, groupTemplate, framework.svgElements, framework.svg);\n  }\n};\n\n/**\n * draw a bar graph\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {Object} processedGroupData\n * @param {{svg: Object, svgElements: Array.<Object>, options: Object, groups: Array.<vis.Group>}} framework\n */\nBargraph.draw = function (groupIds, processedGroupData, framework) {\n  var combinedData = [];\n  var intersections = {};\n  var coreDistance;\n  var key, drawData;\n  var group;\n  var i, j;\n  var barPoints = 0;\n\n  // combine all barchart data\n  for (i = 0; i < groupIds.length; i++) {\n    group = framework.groups[groupIds[i]];\n    if (group.options.style === 'bar') {\n      if (group.visible === true && (framework.options.groups.visibility[groupIds[i]] === undefined || framework.options.groups.visibility[groupIds[i]] === true)) {\n        for (j = 0; j < processedGroupData[groupIds[i]].length; j++) {\n          combinedData.push({\n            screen_x: processedGroupData[groupIds[i]][j].screen_x,\n            screen_end: processedGroupData[groupIds[i]][j].screen_end,\n            screen_y: processedGroupData[groupIds[i]][j].screen_y,\n            x: processedGroupData[groupIds[i]][j].x,\n            end: processedGroupData[groupIds[i]][j].end,\n            y: processedGroupData[groupIds[i]][j].y,\n            groupId: groupIds[i],\n            label: processedGroupData[groupIds[i]][j].label\n          });\n          barPoints += 1;\n        }\n      }\n    }\n  }\n\n  if (barPoints === 0) {\n    return;\n  }\n\n  // sort by time and by group\n  combinedData.sort(function (a, b) {\n    if (a.screen_x === b.screen_x) {\n      return a.groupId < b.groupId ? -1 : 1;\n    }\n    else {\n      return a.screen_x - b.screen_x;\n    }\n  });\n\n  // get intersections\n  Bargraph._getDataIntersections(intersections, combinedData);\n\n  // plot barchart\n  for (i = 0; i < combinedData.length; i++) {\n    group = framework.groups[combinedData[i].groupId];\n    var minWidth = group.options.barChart.minWidth != undefined ? group.options.barChart.minWidth : 0.1 * group.options.barChart.width;\n\n    key = combinedData[i].screen_x;\n    var heightOffset = 0;\n    if (intersections[key] === undefined) {\n      if (i + 1 < combinedData.length) {\n        coreDistance = Math.abs(combinedData[i + 1].screen_x - key);\n      }\n      drawData = Bargraph._getSafeDrawData(coreDistance, group, minWidth);\n    }\n    else {\n      var nextKey = i + (intersections[key].amount - intersections[key].resolved);\n      if (nextKey < combinedData.length) {\n        coreDistance = Math.abs(combinedData[nextKey].screen_x - key);\n      }\n      drawData = Bargraph._getSafeDrawData(coreDistance, group, minWidth);\n      intersections[key].resolved += 1;\n\n      if (group.options.stack === true && group.options.excludeFromStacking !== true) {\n        if (combinedData[i].screen_y < group.zeroPosition) {\n          heightOffset = intersections[key].accumulatedNegative;\n          intersections[key].accumulatedNegative += group.zeroPosition - combinedData[i].screen_y;\n        }\n        else {\n          heightOffset = intersections[key].accumulatedPositive;\n          intersections[key].accumulatedPositive += group.zeroPosition - combinedData[i].screen_y;\n        }\n      }\n      else if (group.options.barChart.sideBySide === true) {\n        drawData.width = drawData.width / intersections[key].amount;\n        drawData.offset += (intersections[key].resolved) * drawData.width - (0.5 * drawData.width * (intersections[key].amount + 1));\n      }\n    }\n    \n    let dataWidth = drawData.width;\n    let start = combinedData[i].screen_x;\n\n    // are we drawing explicit boxes? (we supplied an end value)\n    if (combinedData[i].screen_end != undefined){\n      dataWidth = combinedData[i].screen_end - combinedData[i].screen_x;\n      start += (dataWidth * 0.5);\n    }\n    else {\n      start += drawData.offset\n    }\n\n    DOMutil.drawBar(start, combinedData[i].screen_y - heightOffset, dataWidth, group.zeroPosition - combinedData[i].screen_y, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n\n    // draw points\n    if (group.options.drawPoints.enabled === true) {\n      let pointData = {\n        screen_x: combinedData[i].screen_x,\n        screen_y: combinedData[i].screen_y - heightOffset,\n        x: combinedData[i].x,\n        y: combinedData[i].y,\n        groupId: combinedData[i].groupId,\n        label: combinedData[i].label\n      };\n      Points.draw([pointData], group, framework, drawData.offset);\n      //DOMutil.drawPoint(combinedData[i].x + drawData.offset, combinedData[i].y, group, framework.svgElements, framework.svg);\n    }\n  }\n};\n\n\n/**\n * Fill the intersections object with counters of how many datapoints share the same x coordinates\n * @param {Object} intersections\n * @param {Array.<Object>} combinedData\n * @private\n */\nBargraph._getDataIntersections = function (intersections, combinedData) {\n  // get intersections\n  var coreDistance;\n  for (var i = 0; i < combinedData.length; i++) {\n    if (i + 1 < combinedData.length) {\n      coreDistance = Math.abs(combinedData[i + 1].screen_x - combinedData[i].screen_x);\n    }\n    if (i > 0) {\n      coreDistance = Math.min(coreDistance, Math.abs(combinedData[i - 1].screen_x - combinedData[i].screen_x));\n    }\n    if (coreDistance === 0) {\n      if (intersections[combinedData[i].screen_x] === undefined) {\n        intersections[combinedData[i].screen_x] = {\n          amount: 0,\n          resolved: 0,\n          accumulatedPositive: 0,\n          accumulatedNegative: 0\n        };\n      }\n      intersections[combinedData[i].screen_x].amount += 1;\n    }\n  }\n};\n\n\n/**\n * Get the width and offset for bargraphs based on the coredistance between datapoints\n *\n * @param {number} coreDistance\n * @param {vis.Group} group\n * @param {number} minWidth\n * @returns {{width: number, offset: number}}\n * @private\n */\nBargraph._getSafeDrawData = function (coreDistance, group, minWidth) {\n  var width, offset;\n  if (coreDistance < group.options.barChart.width && coreDistance > 0) {\n    width = coreDistance < minWidth ? minWidth : coreDistance\n\n    offset = 0; // recalculate offset with the new width;\n    if (group.options.barChart.align === 'left') {\n      offset -= 0.5 * coreDistance;\n    }\n    else if (group.options.barChart.align === 'right') {\n      offset += 0.5 * coreDistance;\n    }\n  }\n  else {\n    // default settings\n    width = group.options.barChart.width;\n    offset = 0;\n    if (group.options.barChart.align === 'left') {\n      offset -= 0.5 * group.options.barChart.width;\n    }\n    else if (group.options.barChart.align === 'right') {\n      offset += 0.5 * group.options.barChart.width;\n    }\n  }\n\n  return {width: width, offset: offset};\n};\n\nBargraph.getStackedYRange = function (combinedData, groupRanges, groupIds, groupLabel, orientation) {\n  if (combinedData.length > 0) {\n    // sort by time and by group\n    combinedData.sort(function (a, b) {\n      if (a.screen_x === b.screen_x) {\n        return a.groupId < b.groupId ? -1 : 1;\n      }\n      else {\n        return a.screen_x - b.screen_x;\n      }\n    });\n    var intersections = {};\n\n    Bargraph._getDataIntersections(intersections, combinedData);\n    groupRanges[groupLabel] = Bargraph._getStackedYRange(intersections, combinedData);\n    groupRanges[groupLabel].yAxisOrientation = orientation;\n    groupIds.push(groupLabel);\n  }\n};\n\nBargraph._getStackedYRange = function (intersections, combinedData) {\n  var key;\n  var yMin = combinedData[0].screen_y;\n  var yMax = combinedData[0].screen_y;\n  for (var i = 0; i < combinedData.length; i++) {\n    key = combinedData[i].screen_x;\n    if (intersections[key] === undefined) {\n      yMin = yMin > combinedData[i].screen_y ? combinedData[i].screen_y : yMin;\n      yMax = yMax < combinedData[i].screen_y ? combinedData[i].screen_y : yMax;\n    }\n    else {\n      if (combinedData[i].screen_y < 0) {\n        intersections[key].accumulatedNegative += combinedData[i].screen_y;\n      }\n      else {\n        intersections[key].accumulatedPositive += combinedData[i].screen_y;\n      }\n    }\n  }\n  for (var xpos in intersections) {\n    if (intersections.hasOwnProperty(xpos)) {\n      yMin = yMin > intersections[xpos].accumulatedNegative ? intersections[xpos].accumulatedNegative : yMin;\n      yMin = yMin > intersections[xpos].accumulatedPositive ? intersections[xpos].accumulatedPositive : yMin;\n      yMax = yMax < intersections[xpos].accumulatedNegative ? intersections[xpos].accumulatedNegative : yMax;\n      yMax = yMax < intersections[xpos].accumulatedPositive ? intersections[xpos].accumulatedPositive : yMax;\n    }\n  }\n\n  return {min: yMin, max: yMax};\n};\n\nexport default Bargraph;","import * as DOMutil from '../../../DOMutil';\n\n/**\n *\n * @param {vis.GraphGroup.id} groupId\n * @param {Object} options   // TODO: Describe options\n * @constructor Line\n */\nfunction Line(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\nLine.calcPath = function (dataset, group) {\n    if (dataset != null) {\n        if (dataset.length > 0) {\n            var d = [];\n\n            // construct path from dataset\n            if (group.options.interpolation.enabled == true) {\n                d = Line._catmullRom(dataset, group);\n            }\n            else {\n                d = Line._linear(dataset);\n            }\n            return d;\n        }\n    }\n};\n\nLine.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n    var fillHeight = iconHeight * 0.5;\n    var path, fillPath;\n\n    var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n    outline.setAttributeNS(null, \"x\", x);\n    outline.setAttributeNS(null, \"y\", y - fillHeight);\n    outline.setAttributeNS(null, \"width\", iconWidth);\n    outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n    outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n    path = DOMutil.getSVGElement(\"path\", framework.svgElements, framework.svg);\n    path.setAttributeNS(null, \"class\", group.className);\n    if (group.style !== undefined) {\n        path.setAttributeNS(null, \"style\", group.style);\n    }\n\n    path.setAttributeNS(null, \"d\", \"M\" + x + \",\" + y + \" L\" + (x + iconWidth) + \",\" + y + \"\");\n    if (group.options.shaded.enabled == true) {\n        fillPath = DOMutil.getSVGElement(\"path\", framework.svgElements, framework.svg);\n        if (group.options.shaded.orientation == 'top') {\n            fillPath.setAttributeNS(null, \"d\", \"M\" + x + \", \" + (y - fillHeight) +\n              \"L\" + x + \",\" + y + \" L\" + (x + iconWidth) + \",\" + y + \" L\" + (x + iconWidth) + \",\" + (y - fillHeight));\n        }\n        else {\n            fillPath.setAttributeNS(null, \"d\", \"M\" + x + \",\" + y + \" \" +\n              \"L\" + x + \",\" + (y + fillHeight) + \" \" +\n              \"L\" + (x + iconWidth) + \",\" + (y + fillHeight) +\n              \"L\" + (x + iconWidth) + \",\" + y);\n        }\n        fillPath.setAttributeNS(null, \"class\", group.className + \" vis-icon-fill\");\n        if (group.options.shaded.style !== undefined && group.options.shaded.style !== \"\") {\n            fillPath.setAttributeNS(null, \"style\", group.options.shaded.style);\n        }\n    }\n\n    if (group.options.drawPoints.enabled == true) {\n        var groupTemplate = {\n            style: group.options.drawPoints.style,\n            styles: group.options.drawPoints.styles,\n            size: group.options.drawPoints.size,\n            className: group.className\n        };\n        DOMutil.drawPoint(x + 0.5 * iconWidth, y, groupTemplate, framework.svgElements, framework.svg);\n    }\n};\n\nLine.drawShading = function (pathArray, group, subPathArray, framework) {\n    // append shading to the path\n    if (group.options.shaded.enabled == true) {\n        var svgHeight = Number(framework.svg.style.height.replace('px',''));\n        var fillPath = DOMutil.getSVGElement('path', framework.svgElements, framework.svg);\n        var type = \"L\";\n        if (group.options.interpolation.enabled == true){\n            type = \"C\";\n        }\n        var dFill;\n        var zero = 0;\n        if (group.options.shaded.orientation == 'top') {\n            zero = 0;\n        }\n        else if (group.options.shaded.orientation == 'bottom') {\n            zero = svgHeight;\n        }\n        else {\n            zero = Math.min(Math.max(0, group.zeroPosition), svgHeight);\n        }\n        if (group.options.shaded.orientation == 'group' && (subPathArray != null && subPathArray != undefined)) {\n            dFill = 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" +\n                    this.serializePath(pathArray,type,false) +\n                    ' L'+ subPathArray[subPathArray.length-1][0]+ \",\" + subPathArray[subPathArray.length-1][1] + \" \" +\n                    this.serializePath(subPathArray,type,true) +\n                    subPathArray[0][0]+ \",\"+subPathArray[0][1] + \" Z\";\n        }\n        else {\n            dFill = 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" +\n                    this.serializePath(pathArray,type,false) +\n                    ' V' + zero + ' H'+ pathArray[0][0] + \" Z\";\n        }\n\n        fillPath.setAttributeNS(null, 'class', group.className + ' vis-fill');\n        if (group.options.shaded.style !== undefined) {\n            fillPath.setAttributeNS(null, 'style', group.options.shaded.style);\n        }\n        fillPath.setAttributeNS(null, 'd', dFill);\n    }\n};\n\n/**\n * draw a line graph\n *\n * @param {Array.<Object>} pathArray\n * @param {vis.Group} group\n * @param {{svg: Object, svgElements: Array.<Object>, options: Object, groups: Array.<vis.Group>}} framework\n */\nLine.draw = function (pathArray, group, framework) {\n    if (pathArray != null && pathArray != undefined) {\n        var path = DOMutil.getSVGElement('path', framework.svgElements, framework.svg);\n        path.setAttributeNS(null, \"class\", group.className);\n        if (group.style !== undefined) {\n            path.setAttributeNS(null, \"style\", group.style);\n        }\n\n        var type = \"L\";\n        if (group.options.interpolation.enabled == true){\n            type = \"C\";\n        }\n        // copy properties to path for drawing.\n        path.setAttributeNS(null, 'd', 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" + this.serializePath(pathArray,type,false));\n    }\n};\n\nLine.serializePath = function(pathArray,type,inverse){\n    if (pathArray.length < 2){\n        //Too little data to create a path.\n        return \"\";\n    }\n    var d = type;\n    var i;\n    if (inverse){\n        for (i = pathArray.length-2; i > 0; i--){\n            d += pathArray[i][0] + \",\" + pathArray[i][1] + \" \";\n        }\n    }\n    else {\n        for (i = 1; i < pathArray.length; i++){\n            d += pathArray[i][0] + \",\" + pathArray[i][1] + \" \";\n        }\n    }\n    return d;\n};\n\n/**\n * This uses an uniform parametrization of the interpolation algorithm:\n * 'On the Parameterization of Catmull-Rom Curves' by Cem Yuksel et al.\n * @param {Array.<Object>} data\n * @returns {string}\n * @private\n */\nLine._catmullRomUniform = function (data) {\n    // catmull rom\n    var p0, p1, p2, p3, bp1, bp2;\n    var d = [];\n    d.push( [ Math.round(data[0].screen_x) , Math.round(data[0].screen_y) ]);\n    var normalization = 1 / 6;\n    var length = data.length;\n    for (var i = 0; i < length - 1; i++) {\n\n        p0 = (i == 0) ? data[0] : data[i - 1];\n        p1 = data[i];\n        p2 = data[i + 1];\n        p3 = (i + 2 < length) ? data[i + 2] : p2;\n\n\n        // Catmull-Rom to Cubic Bezier conversion matrix\n        //    0       1       0       0\n        //  -1/6      1      1/6      0\n        //    0      1/6      1     -1/6\n        //    0       0       1       0\n\n        //    bp0 = { x: p1.x,                               y: p1.y };\n        bp1 = {\n            screen_x: ((-p0.screen_x + 6 * p1.screen_x + p2.screen_x) * normalization),\n            screen_y: ((-p0.screen_y + 6 * p1.screen_y + p2.screen_y) * normalization)\n        };\n        bp2 = {\n            screen_x: (( p1.screen_x + 6 * p2.screen_x - p3.screen_x) * normalization),\n            screen_y: (( p1.screen_y + 6 * p2.screen_y - p3.screen_y) * normalization)\n        };\n        //    bp0 = { x: p2.x,                               y: p2.y };\n\n        d.push( [ bp1.screen_x , bp1.screen_y ]);\n        d.push( [ bp2.screen_x , bp2.screen_y ]);\n        d.push( [ p2.screen_x  , p2.screen_y  ]);\n    }\n\n    return d;\n};\n\n/**\n * This uses either the chordal or centripetal parameterization of the catmull-rom algorithm.\n * By default, the centripetal parameterization is used because this gives the nicest results.\n * These parameterizations are relatively heavy because the distance between 4 points have to be calculated.\n *\n * One optimization can be used to reuse distances since this is a sliding window approach.\n * @param {Array.<Object>} data\n * @param {vis.GraphGroup} group\n * @returns {string}\n * @private\n */\nLine._catmullRom = function (data, group) {\n    var alpha = group.options.interpolation.alpha;\n    if (alpha == 0 || alpha === undefined) {\n        return this._catmullRomUniform(data);\n    }\n    else {\n        var p0, p1, p2, p3, bp1, bp2, d1, d2, d3, A, B, N, M;\n        var d3powA, d2powA, d3pow2A, d2pow2A, d1pow2A, d1powA;\n        var d = [];\n        d.push( [ Math.round(data[0].screen_x) , Math.round(data[0].screen_y) ]);\n        var length = data.length;\n        for (var i = 0; i < length - 1; i++) {\n\n            p0 = (i == 0) ? data[0] : data[i - 1];\n            p1 = data[i];\n            p2 = data[i + 1];\n            p3 = (i + 2 < length) ? data[i + 2] : p2;\n\n            d1 = Math.sqrt(Math.pow(p0.screen_x - p1.screen_x, 2) + Math.pow(p0.screen_y - p1.screen_y, 2));\n            d2 = Math.sqrt(Math.pow(p1.screen_x - p2.screen_x, 2) + Math.pow(p1.screen_y - p2.screen_y, 2));\n            d3 = Math.sqrt(Math.pow(p2.screen_x - p3.screen_x, 2) + Math.pow(p2.screen_y - p3.screen_y, 2));\n\n            // Catmull-Rom to Cubic Bezier conversion matrix\n\n            // A = 2d1^2a + 3d1^a * d2^a + d3^2a\n            // B = 2d3^2a + 3d3^a * d2^a + d2^2a\n\n            // [   0             1            0          0          ]\n            // [   -d2^2a /N     A/N          d1^2a /N   0          ]\n            // [   0             d3^2a /M     B/M        -d2^2a /M  ]\n            // [   0             0            1          0          ]\n\n            d3powA = Math.pow(d3, alpha);\n            d3pow2A = Math.pow(d3, 2 * alpha);\n            d2powA = Math.pow(d2, alpha);\n            d2pow2A = Math.pow(d2, 2 * alpha);\n            d1powA = Math.pow(d1, alpha);\n            d1pow2A = Math.pow(d1, 2 * alpha);\n\n            A = 2 * d1pow2A + 3 * d1powA * d2powA + d2pow2A;\n            B = 2 * d3pow2A + 3 * d3powA * d2powA + d2pow2A;\n            N = 3 * d1powA * (d1powA + d2powA);\n            if (N > 0) {\n                N = 1 / N;\n            }\n            M = 3 * d3powA * (d3powA + d2powA);\n            if (M > 0) {\n                M = 1 / M;\n            }\n\n            bp1 = {\n                screen_x: ((-d2pow2A * p0.screen_x + A * p1.screen_x + d1pow2A * p2.screen_x) * N),\n                screen_y: ((-d2pow2A * p0.screen_y + A * p1.screen_y + d1pow2A * p2.screen_y) * N)\n            };\n\n            bp2 = {\n                screen_x: (( d3pow2A * p1.screen_x + B * p2.screen_x - d2pow2A * p3.screen_x) * M),\n                screen_y: (( d3pow2A * p1.screen_y + B * p2.screen_y - d2pow2A * p3.screen_y) * M)\n            };\n\n            if (bp1.screen_x == 0 && bp1.screen_y == 0) {\n                bp1 = p1;\n            }\n            if (bp2.screen_x == 0 && bp2.screen_y == 0) {\n                bp2 = p2;\n            }\n            d.push( [ bp1.screen_x , bp1.screen_y ]);\n            d.push( [ bp2.screen_x , bp2.screen_y ]);\n            d.push( [ p2.screen_x  , p2.screen_y  ]);\n        }\n\n        return d;\n    }\n};\n\n/**\n * this generates the SVG path for a linear drawing between datapoints.\n * @param {Array.<Object>} data\n * @returns {string}\n * @private\n */\nLine._linear = function (data) {\n    // linear\n    var d = [];\n    for (var i = 0; i < data.length; i++) {\n        d.push([ data[i].screen_x , data[i].screen_y ]);\n    }\n    return d;\n};\n\nexport default Line;\n","import util from '../../util';\nimport Bars from './graph2d_types/bar';\nimport Lines from './graph2d_types/line';\nimport Points from './graph2d_types/points';\n\n/**\n * /**\n * @param {object} group            | the object of the group from the dataset\n * @param {string} groupId          | ID of the group\n * @param {object} options          | the default options\n * @param {array} groupsUsingDefaultStyles  | this array has one entree.\n *                                            It is passed as an array so it is passed by reference.\n *                                            It enumerates through the default styles\n * @constructor GraphGroup\n */\nfunction GraphGroup(group, groupId, options, groupsUsingDefaultStyles) {\n  this.id = groupId;\n  var fields = ['sampling', 'style', 'sort', 'yAxisOrientation', 'barChart', 'drawPoints', 'shaded', 'interpolation', 'zIndex','excludeFromStacking', 'excludeFromLegend'];\n  this.options = util.selectiveBridgeObject(fields, options);\n  this.usingDefaultStyle = group.className === undefined;\n  this.groupsUsingDefaultStyles = groupsUsingDefaultStyles;\n  this.zeroPosition = 0;\n  this.update(group);\n  if (this.usingDefaultStyle == true) {\n    this.groupsUsingDefaultStyles[0] += 1;\n  }\n  this.itemsData = [];\n  this.visible = group.visible === undefined ? true : group.visible;\n}\n\n/**\n * this loads a reference to all items in this group into this group.\n * @param {array} items\n */\nGraphGroup.prototype.setItems = function (items) {\n  if (items != null) {\n    this.itemsData = items;\n    if (this.options.sort == true) {\n      util.insertSort(this.itemsData,function (a, b) {\n        return a.x > b.x ? 1 : -1;\n      });\n    }\n  }\n  else {\n    this.itemsData = [];\n  }\n};\n\nGraphGroup.prototype.getItems = function () {\n  return this.itemsData;\n};\n\n/**\n * this is used for barcharts and shading, this way, we only have to calculate it once.\n * @param {number} pos\n */\nGraphGroup.prototype.setZeroPosition = function (pos) {\n  this.zeroPosition = pos;\n};\n\n/**\n * set the options of the graph group over the default options.\n * @param {Object} options\n */\nGraphGroup.prototype.setOptions = function (options) {\n  if (options !== undefined) {\n    var fields = ['sampling', 'style', 'sort', 'yAxisOrientation', 'barChart', 'zIndex','excludeFromStacking', 'excludeFromLegend'];\n    util.selectiveDeepExtend(fields, this.options, options);\n\n    // if the group's drawPoints is a function delegate the callback to the onRender property\n    if (typeof options.drawPoints == 'function') {\n      options.drawPoints = {\n        onRender: options.drawPoints\n      }\n    }\n\n    util.mergeOptions(this.options, options, 'interpolation');\n    util.mergeOptions(this.options, options, 'drawPoints');\n    util.mergeOptions(this.options, options, 'shaded');\n\n    if (options.interpolation) {\n      if (typeof options.interpolation == 'object') {\n        if (options.interpolation.parametrization) {\n          if (options.interpolation.parametrization == 'uniform') {\n            this.options.interpolation.alpha = 0;\n          }\n          else if (options.interpolation.parametrization == 'chordal') {\n            this.options.interpolation.alpha = 1.0;\n          }\n          else {\n            this.options.interpolation.parametrization = 'centripetal';\n            this.options.interpolation.alpha = 0.5;\n          }\n        }\n      }\n    }\n  }\n};\n\n\n/**\n * this updates the current group class with the latest group dataset entree, used in _updateGroup in linegraph\n * @param {vis.Group} group\n */\nGraphGroup.prototype.update = function (group) {\n  this.group = group;\n  this.content = group.content || 'graph';\n  this.className = group.className || this.className || 'vis-graph-group' + this.groupsUsingDefaultStyles[0] % 10;\n  this.visible = group.visible === undefined ? true : group.visible;\n  this.style = group.style;\n  this.setOptions(group.options);\n};\n\n/**\n * return the legend entree for this group.\n *\n * @param {number} iconWidth\n * @param {number} iconHeight\n * @param {{svg: (*|Element), svgElements: Object, options: Object, groups: Array.<Object>}} framework\n * @param {number} x\n * @param {number} y\n * @returns {{icon: (*|Element), label: (*|string), orientation: *}}\n */\nGraphGroup.prototype.getLegend = function (iconWidth, iconHeight, framework, x, y) {\n  if (framework == undefined || framework == null) {\n    var svg = document.createElementNS('http://www.w3.org/2000/svg', \"svg\");\n    framework = {svg: svg, svgElements:{}, options: this.options, groups: [this]}\n  }\n  if (x == undefined || x == null){\n    x = 0;\n  }\n  if (y == undefined || y == null){\n    y = 0.5 * iconHeight;\n  }\n  switch (this.options.style){\n    case \"line\":\n      Lines.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n    case \"points\": //explicit no break\n    case \"point\":\n      Points.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n    case \"bar\":\n      Bars.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n  }\n  return {icon: framework.svg, label: this.content, orientation: this.options.yAxisOrientation};\n};\n\nGraphGroup.prototype.getYRange = function (groupData) {\n  var yMin = groupData[0].y;\n  var yMax = groupData[0].y;\n  for (var j = 0; j < groupData.length; j++) {\n    yMin = yMin > groupData[j].y ? groupData[j].y : yMin;\n    yMax = yMax < groupData[j].y ? groupData[j].y : yMax;\n  }\n  return {min: yMin, max: yMax, yAxisOrientation: this.options.yAxisOrientation};\n};\n\nexport default GraphGroup;\n","import util from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport Component from './Component';\n\n/**\n * Legend for Graph2d\n *\n * @param {vis.Graph2d.body} body\n * @param {vis.Graph2d.options} options\n * @param {number} side\n * @param {vis.LineGraph.options} linegraphOptions\n * @constructor Legend\n * @extends Component\n */\nfunction Legend(body, options, side, linegraphOptions) {\n  this.body = body;\n  this.defaultOptions = {\n    enabled: false,\n    icons: true,\n    iconSize: 20,\n    iconSpacing: 6,\n    left: {\n      visible: true,\n      position: 'top-left' // top/bottom - left,center,right\n    },\n    right: {\n      visible: true,\n      position: 'top-right' // top/bottom - left,center,right\n    }\n  };\n\n  this.side = side;\n  this.options = util.extend({}, this.defaultOptions);\n  this.linegraphOptions = linegraphOptions;\n\n  this.svgElements = {};\n  this.dom = {};\n  this.groups = {};\n  this.amountOfGroups = 0;\n  this._create();\n  this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n\n  this.setOptions(options);\n}\n\nLegend.prototype = new Component();\n\nLegend.prototype.clear = function() {\n  this.groups = {};\n  this.amountOfGroups = 0;\n};\n\nLegend.prototype.addGroup = function(label, graphOptions) {\n\n  // Include a group only if the group option 'excludeFromLegend: false' is not set.\n  if (graphOptions.options.excludeFromLegend != true) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.groups[label] = graphOptions;\n    }\n    this.amountOfGroups += 1;\n  }\n};\n\nLegend.prototype.updateGroup = function(label, graphOptions) {\n  this.groups[label] = graphOptions;\n};\n\nLegend.prototype.removeGroup = function(label) {\n  if (this.groups.hasOwnProperty(label)) {\n    delete this.groups[label];\n    this.amountOfGroups -= 1;\n  }\n};\n\nLegend.prototype._create = function() {\n  this.dom.frame = document.createElement('div');\n  this.dom.frame.className = 'vis-legend';\n  this.dom.frame.style.position = \"absolute\";\n  this.dom.frame.style.top = \"10px\";\n  this.dom.frame.style.display = \"block\";\n\n  this.dom.textArea = document.createElement('div');\n  this.dom.textArea.className = 'vis-legend-text';\n  this.dom.textArea.style.position = \"relative\";\n  this.dom.textArea.style.top = \"0px\";\n\n  this.svg = document.createElementNS('http://www.w3.org/2000/svg',\"svg\");\n  this.svg.style.position = 'absolute';\n  this.svg.style.top = 0 +'px';\n  this.svg.style.width = this.options.iconSize + 5 + 'px';\n  this.svg.style.height = '100%';\n\n  this.dom.frame.appendChild(this.svg);\n  this.dom.frame.appendChild(this.dom.textArea);\n};\n\n/**\n * Hide the component from the DOM\n */\nLegend.prototype.hide = function() {\n  // remove the frame containing the items\n  if (this.dom.frame.parentNode) {\n    this.dom.frame.parentNode.removeChild(this.dom.frame);\n  }\n};\n\n/**\n * Show the component in the DOM (when not already visible).\n */\nLegend.prototype.show = function() {\n  // show frame containing the items\n  if (!this.dom.frame.parentNode) {\n    this.body.dom.center.appendChild(this.dom.frame);\n  }\n};\n\nLegend.prototype.setOptions = function(options) {\n  var fields = ['enabled','orientation','icons','left','right'];\n  util.selectiveDeepExtend(fields, this.options, options);\n};\n\nLegend.prototype.redraw = function() {\n  var activeGroups = 0;\n  var groupArray = Object.keys(this.groups);\n  groupArray.sort(function (a,b) {\n    return (a < b ? -1 : 1);\n  })\n\n  for (var i = 0; i < groupArray.length; i++) {\n    var groupId = groupArray[i];\n    if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n      activeGroups++;\n    }\n  }\n\n  if (this.options[this.side].visible == false || this.amountOfGroups == 0 || this.options.enabled == false || activeGroups == 0) {\n    this.hide();\n  }\n  else {\n    this.show();\n    if (this.options[this.side].position == 'top-left' || this.options[this.side].position == 'bottom-left') {\n      this.dom.frame.style.left = '4px';\n      this.dom.frame.style.textAlign = \"left\";\n      this.dom.textArea.style.textAlign = \"left\";\n      this.dom.textArea.style.left = (this.options.iconSize + 15) + 'px';\n      this.dom.textArea.style.right = '';\n      this.svg.style.left = 0 +'px';\n      this.svg.style.right = '';\n    }\n    else {\n      this.dom.frame.style.right = '4px';\n      this.dom.frame.style.textAlign = \"right\";\n      this.dom.textArea.style.textAlign = \"right\";\n      this.dom.textArea.style.right = (this.options.iconSize + 15) + 'px';\n      this.dom.textArea.style.left = '';\n      this.svg.style.right = 0 +'px';\n      this.svg.style.left = '';\n    }\n\n    if (this.options[this.side].position == 'top-left' || this.options[this.side].position == 'top-right') {\n      this.dom.frame.style.top = 4 - Number(this.body.dom.center.style.top.replace(\"px\",\"\")) + 'px';\n      this.dom.frame.style.bottom = '';\n    }\n    else {\n      var scrollableHeight = this.body.domProps.center.height - this.body.domProps.centerContainer.height;\n      this.dom.frame.style.bottom = 4 + scrollableHeight + Number(this.body.dom.center.style.top.replace(\"px\",\"\")) + 'px';\n      this.dom.frame.style.top = '';\n    }\n\n    if (this.options.icons == false) {\n      this.dom.frame.style.width = this.dom.textArea.offsetWidth + 10 + 'px';\n      this.dom.textArea.style.right = '';\n      this.dom.textArea.style.left = '';\n      this.svg.style.width = '0px';\n    }\n    else {\n      this.dom.frame.style.width = this.options.iconSize + 15 + this.dom.textArea.offsetWidth + 10 + 'px'\n      this.drawLegendIcons();\n    }\n\n    var content = '';\n    for (i = 0; i < groupArray.length; i++) {\n      groupId = groupArray[i];\n      if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n        content += this.groups[groupId].content + '<br />';\n      }\n    }\n    this.dom.textArea.innerHTML = content;\n    this.dom.textArea.style.lineHeight = ((0.75 * this.options.iconSize) + this.options.iconSpacing) + 'px';\n  }\n};\n\nLegend.prototype.drawLegendIcons = function() {\n  if (this.dom.frame.parentNode) {\n    var groupArray = Object.keys(this.groups);\n    groupArray.sort(function (a,b) {\n      return (a < b ? -1 : 1);\n    });\n\n    // this resets the elements so the order is maintained\n    DOMutil.resetElements(this.svgElements);\n\n    var padding = window.getComputedStyle(this.dom.frame).paddingTop;\n    var iconOffset = Number(padding.replace('px',''));\n    var x = iconOffset;\n    var iconWidth = this.options.iconSize;\n    var iconHeight = 0.75 * this.options.iconSize;\n    var y = iconOffset + 0.5 * iconHeight + 3;\n\n    this.svg.style.width = iconWidth + 5 + iconOffset + 'px';\n\n    for (var i = 0; i < groupArray.length; i++) {\n      var groupId = groupArray[i];\n      if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n        this.groups[groupId].getLegend(iconWidth, iconHeight, this.framework, x, y);\n        y += iconHeight + this.options.iconSpacing;\n      }\n    }\n  }\n};\n\nexport default Legend;\n","import { v4 as randomUUID } from \"uuid\";\nimport util, { typeCoerceDataSet } from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Component from './Component';\nimport DataAxis from './DataAxis';\nimport GraphGroup from './GraphGroup';\nimport Legend from './Legend';\nimport Bars from './graph2d_types/bar';\nimport Lines from './graph2d_types/line';\nimport Points from './graph2d_types/points';\n\nvar UNGROUPED = '__ungrouped__'; // reserved group id for ungrouped items\n\n/**\n * This is the constructor of the LineGraph. It requires a Timeline body and options.\n *\n * @param {vis.Timeline.body} body\n * @param {Object} options\n * @constructor LineGraph\n * @extends Component\n */\nfunction LineGraph(body, options) {\n  this.id = randomUUID();\n  this.body = body;\n\n  this.defaultOptions = {\n    yAxisOrientation: 'left',\n    defaultGroup: 'default',\n    sort: true,\n    sampling: true,\n    stack: false,\n    graphHeight: '400px',\n    shaded: {\n      enabled: false,\n      orientation: 'bottom' // top, bottom, zero\n    },\n    style: 'line', // line, bar\n    barChart: {\n      width: 50,\n      sideBySide: false,\n      align: 'center' // left, center, right\n    },\n    interpolation: {\n      enabled: true,\n      parametrization: 'centripetal', // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n      alpha: 0.5\n    },\n    drawPoints: {\n      enabled: true,\n      size: 6,\n      style: 'square' // square, circle\n    },\n    dataAxis: {}, //Defaults are done on DataAxis level\n    legend: {}, //Defaults are done on Legend level\n    groups: {\n      visibility: {}\n    }\n  };\n\n  // options is shared by this lineGraph and all its items\n  this.options = util.extend({}, this.defaultOptions);\n  this.dom = {};\n  this.props = {};\n  this.hammer = null;\n  this.groups = {};\n  this.abortedGraphUpdate = false;\n  this.updateSVGheight = false;\n  this.updateSVGheightOnResize = false;\n  this.forceGraphUpdate = true;\n\n  var me = this;\n  this.itemsData = null;    // DataSet\n  this.groupsData = null;   // DataSet\n\n  // listeners for the DataSet of the items\n  this.itemListeners = {\n    'add': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onAdd(params.items);\n    },\n    'update': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onUpdate(params.items);\n    },\n    'remove': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onRemove(params.items);\n    }\n  };\n\n  // listeners for the DataSet of the groups\n  this.groupListeners = {\n    'add': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onAddGroups(params.items);\n    },\n    'update': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onUpdateGroups(params.items);\n    },\n    'remove': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onRemoveGroups(params.items);\n    }\n  };\n\n  this.items = {};      // object with an Item for every data item\n  this.selection = [];  // list with the ids of all selected nodes\n  this.lastStart = this.body.range.start;\n  this.touchParams = {}; // stores properties while dragging\n\n  this.svgElements = {};\n  this.setOptions(options);\n  this.groupsUsingDefaultStyles = [0];\n  this.body.emitter.on('rangechanged', function () {\n    me.lastStart = me.body.range.start;\n    me.svg.style.left = util.option.asSize(-me.props.width);\n\n    me.forceGraphUpdate = true;\n    //Is this local redraw necessary? (Core also does a change event!)\n    me.redraw.call(me);\n  });\n\n  // create the HTML DOM\n  this._create();\n  this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n}\n\nLineGraph.prototype = new Component();\n\n/**\n * Create the HTML DOM for the ItemSet\n */\nLineGraph.prototype._create = function () {\n  var frame = document.createElement('div');\n  frame.className = 'vis-line-graph';\n  this.dom.frame = frame;\n\n  // create svg element for graph drawing.\n  this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');\n  this.svg.style.position = 'relative';\n  this.svg.style.height = ('' + this.options.graphHeight).replace('px', '') + 'px';\n  this.svg.style.display = 'block';\n  frame.appendChild(this.svg);\n\n  // data axis\n  this.options.dataAxis.orientation = 'left';\n  this.yAxisLeft = new DataAxis(this.body, this.options.dataAxis, this.svg, this.options.groups);\n\n  this.options.dataAxis.orientation = 'right';\n  this.yAxisRight = new DataAxis(this.body, this.options.dataAxis, this.svg, this.options.groups);\n  delete this.options.dataAxis.orientation;\n\n  // legends\n  this.legendLeft = new Legend(this.body, this.options.legend, 'left', this.options.groups);\n  this.legendRight = new Legend(this.body, this.options.legend, 'right', this.options.groups);\n\n  this.show();\n};\n\n/**\n * set the options of the LineGraph. the mergeOptions is used for subObjects that have an enabled element.\n * @param {object} options\n */\nLineGraph.prototype.setOptions = function (options) {\n  if (options) {\n    var fields = ['sampling', 'defaultGroup', 'stack', 'height', 'graphHeight', 'yAxisOrientation', 'style', 'barChart', 'dataAxis', 'sort', 'groups'];\n    if (options.graphHeight === undefined && options.height !== undefined) {\n      this.updateSVGheight = true;\n      this.updateSVGheightOnResize = true;\n    }\n    else if (this.body.domProps.centerContainer.height !== undefined && options.graphHeight !== undefined) {\n      if (parseInt((options.graphHeight + '').replace(\"px\", '')) < this.body.domProps.centerContainer.height) {\n        this.updateSVGheight = true;\n      }\n    }\n    util.selectiveDeepExtend(fields, this.options, options);\n    util.mergeOptions(this.options, options, 'interpolation');\n    util.mergeOptions(this.options, options, 'drawPoints');\n    util.mergeOptions(this.options, options, 'shaded');\n    util.mergeOptions(this.options, options, 'legend');\n\n    if (options.interpolation) {\n      if (typeof options.interpolation == 'object') {\n        if (options.interpolation.parametrization) {\n          if (options.interpolation.parametrization == 'uniform') {\n            this.options.interpolation.alpha = 0;\n          }\n          else if (options.interpolation.parametrization == 'chordal') {\n            this.options.interpolation.alpha = 1.0;\n          }\n          else {\n            this.options.interpolation.parametrization = 'centripetal';\n            this.options.interpolation.alpha = 0.5;\n          }\n        }\n      }\n    }\n\n    if (this.yAxisLeft) {\n      if (options.dataAxis !== undefined) {\n        this.yAxisLeft.setOptions(this.options.dataAxis);\n        this.yAxisRight.setOptions(this.options.dataAxis);\n      }\n    }\n\n    if (this.legendLeft) {\n      if (options.legend !== undefined) {\n        this.legendLeft.setOptions(this.options.legend);\n        this.legendRight.setOptions(this.options.legend);\n      }\n    }\n\n    if (this.groups.hasOwnProperty(UNGROUPED)) {\n      this.groups[UNGROUPED].setOptions(options);\n    }\n  }\n\n  // this is used to redraw the graph if the visibility of the groups is changed.\n  if (this.dom.frame) { //not on initial run?\n    this.forceGraphUpdate=true;\n    this.body.emitter.emit(\"_change\",{queue: true});\n  }\n};\n\n/**\n * Hide the component from the DOM\n */\nLineGraph.prototype.hide = function () {\n  // remove the frame containing the items\n  if (this.dom.frame.parentNode) {\n    this.dom.frame.parentNode.removeChild(this.dom.frame);\n  }\n};\n\n\n/**\n * Show the component in the DOM (when not already visible).\n */\nLineGraph.prototype.show = function () {\n  // show frame containing the items\n  if (!this.dom.frame.parentNode) {\n    this.body.dom.center.appendChild(this.dom.frame);\n  }\n};\n\n\n/**\n * Set items\n * @param {vis.DataSet | null} items\n */\nLineGraph.prototype.setItems = function (items) {\n  var me = this,\n    ids,\n    oldItemsData = this.itemsData;\n\n  // replace the dataset\n  if (!items) {\n    this.itemsData = null;\n  }\n  else if (items instanceof DataSet || items instanceof DataView) {\n    this.itemsData = typeCoerceDataSet(items);\n  }\n  else {\n    throw new TypeError('Data must be an instance of DataSet or DataView');\n  }\n\n  if (oldItemsData) {\n    // unsubscribe from old dataset\n    util.forEach(this.itemListeners, function (callback, event) {\n      oldItemsData.off(event, callback);\n    });\n\n    // stop maintaining a coerced version of the old data set\n    oldItemsData.dispose()\n\n    // remove all drawn items\n    ids = oldItemsData.getIds();\n    this._onRemove(ids);\n  }\n\n  if (this.itemsData) {\n    // subscribe to new dataset\n    var id = this.id;\n    util.forEach(this.itemListeners, function (callback, event) {\n      me.itemsData.on(event, callback, id);\n    });\n\n    // add all new items\n    ids = this.itemsData.getIds();\n    this._onAdd(ids);\n  }\n};\n\n\n/**\n * Set groups\n * @param {vis.DataSet} groups\n */\nLineGraph.prototype.setGroups = function (groups) {\n  var me = this;\n  var ids;\n\n  // unsubscribe from current dataset\n  if (this.groupsData) {\n    util.forEach(this.groupListeners, function (callback, event) {\n      me.groupsData.off(event, callback);\n    });\n\n    // remove all drawn groups\n    ids = this.groupsData.getIds();\n    this.groupsData = null;\n    for (var i = 0; i < ids.length; i++) {\n      this._removeGroup(ids[i]);\n    }\n  }\n\n  // replace the dataset\n  if (!groups) {\n    this.groupsData = null;\n  }\n  else if (groups instanceof DataSet || groups instanceof DataView) {\n    this.groupsData = groups;\n  }\n  else {\n    throw new TypeError('Data must be an instance of DataSet or DataView');\n  }\n\n  if (this.groupsData) {\n    // subscribe to new dataset\n    var id = this.id;\n    util.forEach(this.groupListeners, function (callback, event) {\n      me.groupsData.on(event, callback, id);\n    });\n\n    // draw all ms\n    ids = this.groupsData.getIds();\n    this._onAddGroups(ids);\n  }\n};\n\nLineGraph.prototype._onUpdate = function (ids) {\n  this._updateAllGroupData(ids);\n};\nLineGraph.prototype._onAdd = function (ids) {\n  this._onUpdate(ids);\n};\nLineGraph.prototype._onRemove = function (ids) {\n  this._onUpdate(ids);\n};\nLineGraph.prototype._onUpdateGroups = function (groupIds) {\n  this._updateAllGroupData(null, groupIds);\n};\nLineGraph.prototype._onAddGroups = function (groupIds) {\n  this._onUpdateGroups(groupIds);\n};\n\n/**\n * this cleans the group out off the legends and the dataaxis, updates the ungrouped and updates the graph\n * @param {Array} groupIds\n * @private\n */\nLineGraph.prototype._onRemoveGroups = function (groupIds) {\n  for (var i = 0; i < groupIds.length; i++) {\n    this._removeGroup(groupIds[i]);\n  }\n  this.forceGraphUpdate = true;\n  this.body.emitter.emit(\"_change\",{queue: true});\n};\n\n/**\n * this cleans the group out off the legends and the dataaxis\n * @param {vis.GraphGroup.id} groupId\n * @private\n */\nLineGraph.prototype._removeGroup = function (groupId) {\n  if (this.groups.hasOwnProperty(groupId)) {\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.removeGroup(groupId);\n      this.legendRight.removeGroup(groupId);\n      this.legendRight.redraw();\n    }\n    else {\n      this.yAxisLeft.removeGroup(groupId);\n      this.legendLeft.removeGroup(groupId);\n      this.legendLeft.redraw();\n    }\n    delete this.groups[groupId];\n  }\n};\n\n/**\n * update a group object with the group dataset entree\n *\n * @param {vis.GraphGroup} group\n * @param {vis.GraphGroup.id} groupId\n * @private\n */\nLineGraph.prototype._updateGroup = function (group, groupId) {\n  if (!this.groups.hasOwnProperty(groupId)) {\n    this.groups[groupId] = new GraphGroup(group, groupId, this.options, this.groupsUsingDefaultStyles);\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.addGroup(groupId, this.groups[groupId]);\n      this.legendRight.addGroup(groupId, this.groups[groupId]);\n    }\n    else {\n      this.yAxisLeft.addGroup(groupId, this.groups[groupId]);\n      this.legendLeft.addGroup(groupId, this.groups[groupId]);\n    }\n  }\n  else {\n    this.groups[groupId].update(group);\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.updateGroup(groupId, this.groups[groupId]);\n      this.legendRight.updateGroup(groupId, this.groups[groupId]);\n      //If yAxisOrientation changed, clean out the group from the other axis.\n      this.yAxisLeft.removeGroup(groupId);\n      this.legendLeft.removeGroup(groupId);\n    }\n    else {\n      this.yAxisLeft.updateGroup(groupId, this.groups[groupId]);\n      this.legendLeft.updateGroup(groupId, this.groups[groupId]);\n      //If yAxisOrientation changed, clean out the group from the other axis.\n      this.yAxisRight.removeGroup(groupId);\n      this.legendRight.removeGroup(groupId);\n    }\n  }\n  this.legendLeft.redraw();\n  this.legendRight.redraw();\n};\n\n\n/**\n * this updates all groups, it is used when there is an update the the itemset.\n *\n * @param  {Array} ids\n * @param  {Array} groupIds\n * @private\n */\nLineGraph.prototype._updateAllGroupData = function (ids, groupIds) {\n  if (this.itemsData != null) {\n    var groupsContent = {};\n    var items = this.itemsData.get();\n    var fieldId = this.itemsData.idProp;\n    var idMap = {};\n    if (ids){\n      ids.map(function (id) {\n        idMap[id] = id;\n      });\n    }\n\n    //pre-Determine array sizes, for more efficient memory claim\n    var groupCounts = {};\n    for (var i = 0; i < items.length; i++) {\n      var item = items[i];\n      var groupId = item.group;\n      if (groupId === null || groupId === undefined) {\n        groupId = UNGROUPED;\n      }\n      groupCounts.hasOwnProperty(groupId) ? groupCounts[groupId]++ : groupCounts[groupId] = 1;\n    }\n\n    //Pre-load arrays from existing groups if items are not changed (not in ids)\n    var existingItemsMap = {};\n    if (!groupIds && ids) {\n      for (groupId in this.groups) {\n        if (this.groups.hasOwnProperty(groupId)) {\n          group = this.groups[groupId];\n          var existing_items = group.getItems();\n\n          groupsContent[groupId] = existing_items.filter(function (item) {\n            existingItemsMap[item[fieldId]] = item[fieldId];\n            return (item[fieldId] !== idMap[item[fieldId]]);\n          });\n          var newLength = groupCounts[groupId];\n          groupCounts[groupId] -= groupsContent[groupId].length;\n          if (groupsContent[groupId].length < newLength) {\n            groupsContent[groupId][newLength - 1] = {};\n          }\n        }\n      }\n    }\n\n    //Now insert data into the arrays.\n    for (i = 0; i < items.length; i++) {\n      item = items[i];\n      groupId = item.group;\n      if (groupId === null || groupId === undefined) {\n        groupId = UNGROUPED;\n      }\n      if (!groupIds && ids && (item[fieldId] !== idMap[item[fieldId]]) && existingItemsMap.hasOwnProperty(item[fieldId])) {\n        continue;\n      }\n      if (!groupsContent.hasOwnProperty(groupId)) {\n        groupsContent[groupId] = new Array(groupCounts[groupId]);\n      }\n      //Copy data (because of unmodifiable DataView input.\n      var extended = util.bridgeObject(item);\n      extended.x = util.convert(item.x, 'Date');\n      extended.end = util.convert(item.end, 'Date');\n      extended.orginalY = item.y; //real Y\n      extended.y = Number(item.y);\n      extended[fieldId] = item[fieldId];\n\n      var index= groupsContent[groupId].length - groupCounts[groupId]--;\n      groupsContent[groupId][index] = extended;\n    }\n\n    //Make sure all groups are present, to allow removal of old groups\n    for (groupId in this.groups){\n      if (this.groups.hasOwnProperty(groupId)){\n        if (!groupsContent.hasOwnProperty(groupId)) {\n          groupsContent[groupId] = new Array(0);\n        }\n      }\n    }\n\n    //Update legendas, style and axis\n    for (groupId in groupsContent) {\n      if (groupsContent.hasOwnProperty(groupId)) {\n        if (groupsContent[groupId].length == 0) {\n          if (this.groups.hasOwnProperty(groupId)) {\n            this._removeGroup(groupId);\n          }\n        } else {\n          var group = undefined;\n          if (this.groupsData != undefined) {\n            group = this.groupsData.get(groupId);\n          }\n          if (group == undefined) {\n            group = {id: groupId, content: this.options.defaultGroup + groupId};\n          }\n          this._updateGroup(group, groupId);\n          this.groups[groupId].setItems(groupsContent[groupId]);\n        }\n      }\n    }\n    this.forceGraphUpdate = true;\n    this.body.emitter.emit(\"_change\",{queue: true});\n  }\n};\n\n/**\n * Redraw the component, mandatory function\n * @return {boolean} Returns true if the component is resized\n */\nLineGraph.prototype.redraw = function () {\n  var resized = false;\n\n  // calculate actual size and position\n  this.props.width = this.dom.frame.offsetWidth;\n  this.props.height = this.body.domProps.centerContainer.height\n    - this.body.domProps.border.top\n    - this.body.domProps.border.bottom;\n\n  // check if this component is resized\n  resized = this._isResized() || resized;\n\n  // check whether zoomed (in that case we need to re-stack everything)\n  var visibleInterval = this.body.range.end - this.body.range.start;\n  var zoomed = (visibleInterval != this.lastVisibleInterval);\n  this.lastVisibleInterval = visibleInterval;\n\n\n  // the svg element is three times as big as the width, this allows for fully dragging left and right\n  // without reloading the graph. the controls for this are bound to events in the constructor\n  if (resized == true) {\n    this.svg.style.width = util.option.asSize(3 * this.props.width);\n    this.svg.style.left = util.option.asSize(-this.props.width);\n\n    // if the height of the graph is set as proportional, change the height of the svg\n    if ((this.options.height + '').indexOf(\"%\") != -1 || this.updateSVGheightOnResize == true) {\n      this.updateSVGheight = true;\n    }\n  }\n\n  // update the height of the graph on each redraw of the graph.\n  if (this.updateSVGheight == true) {\n    if (this.options.graphHeight != this.props.height + 'px') {\n      this.options.graphHeight = this.props.height + 'px';\n      this.svg.style.height = this.props.height + 'px';\n    }\n    this.updateSVGheight = false;\n  }\n  else {\n    this.svg.style.height = ('' + this.options.graphHeight).replace('px', '') + 'px';\n  }\n\n  // zoomed is here to ensure that animations are shown correctly.\n  if (resized == true || zoomed == true || this.abortedGraphUpdate == true || this.forceGraphUpdate == true) {\n    resized = this._updateGraph() || resized;\n    this.forceGraphUpdate = false;\n  }\n  else {\n    // move the whole svg while dragging\n    if (this.lastStart != 0) {\n      var offset = this.body.range.start - this.lastStart;\n      var range = this.body.range.end - this.body.range.start;\n      if (this.props.width != 0) {\n        var rangePerPixelInv = this.props.width / range;\n        var xOffset = offset * rangePerPixelInv;\n        this.svg.style.left = (-this.props.width - xOffset) + 'px';\n      }\n    }\n  }\n  this.legendLeft.redraw();\n  this.legendRight.redraw();\n  return resized;\n};\n\n\nLineGraph.prototype._getSortedGroupIds = function(){\n  // getting group Ids\n  var grouplist = [];\n  for (var groupId in this.groups) {\n    if (this.groups.hasOwnProperty(groupId)) {\n      var group = this.groups[groupId];\n      if (group.visible == true && (this.options.groups.visibility[groupId] === undefined || this.options.groups.visibility[groupId] == true)) {\n        grouplist.push({id:groupId,zIndex:group.options.zIndex});\n      }\n    }\n  }\n  util.insertSort(grouplist,function(a,b){\n    var az = a.zIndex;\n    var bz = b.zIndex;\n    if (az === undefined) az=0;\n    if (bz === undefined) bz=0;\n    return az==bz? 0: (az<bz ? -1: 1);\n  });\n  var groupIds = new Array(grouplist.length);\n  for (var i=0; i< grouplist.length; i++){\n    groupIds[i] = grouplist[i].id;\n  }\n  return groupIds;\n};\n\n/**\n * Update and redraw the graph.\n *\n * @returns {boolean}\n * @private\n */\nLineGraph.prototype._updateGraph = function () {\n  // reset the svg elements\n  DOMutil.prepareElements(this.svgElements);\n  if (this.props.width != 0 && this.itemsData != null) {\n    var group, i;\n    var groupRanges = {};\n    var changeCalled = false;\n    // this is the range of the SVG canvas\n    var minDate = this.body.util.toGlobalTime(-this.body.domProps.root.width);\n    var maxDate = this.body.util.toGlobalTime(2 * this.body.domProps.root.width);\n\n    // getting group Ids\n    var groupIds = this._getSortedGroupIds();\n    if (groupIds.length > 0) {\n      var groupsData = {};\n\n      // fill groups data, this only loads the data we require based on the timewindow\n      this._getRelevantData(groupIds, groupsData, minDate, maxDate);\n\n      // apply sampling, if disabled, it will pass through this function.\n      this._applySampling(groupIds, groupsData);\n\n      // we transform the X coordinates to detect collisions\n      for (i = 0; i < groupIds.length; i++) {\n        this._convertXcoordinates(groupsData[groupIds[i]]);\n      }\n\n      // now all needed data has been collected we start the processing.\n      this._getYRanges(groupIds, groupsData, groupRanges);\n\n      // update the Y axis first, we use this data to draw at the correct Y points\n      changeCalled = this._updateYAxis(groupIds, groupRanges);\n\n      //  at changeCalled, abort this update cycle as the graph needs another update with new Width input from the Redraw container.\n      //  Cleanup SVG elements on abort.\n      if (changeCalled == true) {\n        DOMutil.cleanupElements(this.svgElements);\n        this.abortedGraphUpdate = true;\n        return true;\n      }\n      this.abortedGraphUpdate = false;\n\n      // With the yAxis scaled correctly, use this to get the Y values of the points.\n      var below = undefined;\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (this.options.stack === true && this.options.style === 'line') {\n          if (group.options.excludeFromStacking == undefined || !group.options.excludeFromStacking) {\n            if (below != undefined) {\n              this._stack(groupsData[group.id], groupsData[below.id]);\n              if (group.options.shaded.enabled == true && group.options.shaded.orientation !== \"group\"){\n                if (group.options.shaded.orientation == \"top\" && below.options.shaded.orientation !== \"group\"){\n                  below.options.shaded.orientation=\"group\";\n                  below.options.shaded.groupId=group.id;\n                } else {\n                  group.options.shaded.orientation=\"group\";\n                  group.options.shaded.groupId=below.id;\n                }\n              }\n            }\n            below = group;\n          }\n        }\n        this._convertYcoordinates(groupsData[groupIds[i]], group);\n      }\n\n      //Precalculate paths and draw shading if appropriate. This will make sure the shading is always behind any lines.\n      var paths = {};\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (group.options.style === 'line' && group.options.shaded.enabled == true) {\n          var dataset = groupsData[groupIds[i]];\n          if (dataset == null || dataset.length == 0) {\n            continue;\n          }\n          if (!paths.hasOwnProperty(groupIds[i])) {\n            paths[groupIds[i]] = Lines.calcPath(dataset, group);\n          }\n          if (group.options.shaded.orientation === \"group\") {\n            var subGroupId = group.options.shaded.groupId;\n            if (groupIds.indexOf(subGroupId) === -1) {\n              console.log(group.id + \": Unknown shading group target given:\" + subGroupId);\n              continue;\n            }\n            if (!paths.hasOwnProperty(subGroupId)) {\n              paths[subGroupId] = Lines.calcPath(groupsData[subGroupId], this.groups[subGroupId]);\n            }\n            Lines.drawShading(paths[groupIds[i]], group, paths[subGroupId], this.framework);\n          }\n          else {\n            Lines.drawShading(paths[groupIds[i]], group, undefined, this.framework);\n          }\n        }\n      }\n\n      // draw the groups, calculating paths if still necessary.\n      Bars.draw(groupIds, groupsData, this.framework);\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (groupsData[groupIds[i]].length > 0) {\n          switch (group.options.style) {\n            case \"line\":\n              if (!paths.hasOwnProperty(groupIds[i])) {\n                paths[groupIds[i]] = Lines.calcPath(groupsData[groupIds[i]], group);\n              }\n              Lines.draw(paths[groupIds[i]], group, this.framework);\n            // eslint-disable-line no-fallthrough\n            case \"point\":\n            // eslint-disable-line no-fallthrough\n            case \"points\":\n              if (group.options.style == \"point\" || group.options.style == \"points\" || group.options.drawPoints.enabled == true) {\n                Points.draw(groupsData[groupIds[i]], group, this.framework);\n              }\n              break;\n            case \"bar\":\n            // bar needs to be drawn enmasse\n            // eslint-disable-line no-fallthrough\n            default:\n            //do nothing...\n          }\n        }\n\n      }\n    }\n  }\n\n  // cleanup unused svg elements\n  DOMutil.cleanupElements(this.svgElements);\n  return false;\n};\n\nLineGraph.prototype._stack = function (data, subData) {\n  var index, dx, dy, subPrevPoint, subNextPoint;\n  index = 0;\n  // for each data point we look for a matching on in the set below\n  for (var j = 0; j < data.length; j++) {\n    subPrevPoint = undefined;\n    subNextPoint = undefined;\n    // we look for time matches or a before-after point\n    for (var k = index; k < subData.length; k++) {\n      // if times match exactly\n      if (subData[k].x === data[j].x) {\n        subPrevPoint = subData[k];\n        subNextPoint = subData[k];\n        index = k;\n        break;\n      }\n      else if (subData[k].x > data[j].x) { // overshoot\n        subNextPoint = subData[k];\n        if (k == 0) {\n          subPrevPoint = subNextPoint;\n        }\n        else {\n          subPrevPoint = subData[k - 1];\n        }\n        index = k;\n        break;\n      }\n    }\n    // in case the last data point has been used, we assume it stays like this.\n    if (subNextPoint === undefined) {\n      subPrevPoint = subData[subData.length - 1];\n      subNextPoint = subData[subData.length - 1];\n    }\n    // linear interpolation\n    dx = subNextPoint.x - subPrevPoint.x;\n    dy = subNextPoint.y - subPrevPoint.y;\n    if (dx == 0) {\n      data[j].y = data[j].orginalY + subNextPoint.y;\n    }\n    else {\n      data[j].y = data[j].orginalY + (dy / dx) * (data[j].x - subPrevPoint.x) + subPrevPoint.y; // ax + b where b is data[j].y\n    }\n  }\n}\n\n\n/**\n * first select and preprocess the data from the datasets.\n * the groups have their preselection of data, we now loop over this data to see\n * what data we need to draw. Sorted data is much faster.\n * more optimization is possible by doing the sampling before and using the binary search\n * to find the end date to determine the increment.\n *\n * @param {array}  groupIds\n * @param {object} groupsData\n * @param {date}   minDate\n * @param {date}   maxDate\n * @private\n */\nLineGraph.prototype._getRelevantData = function (groupIds, groupsData, minDate, maxDate) {\n  var group, i, j, item;\n  if (groupIds.length > 0) {\n    for (i = 0; i < groupIds.length; i++) {\n      group = this.groups[groupIds[i]];\n      var itemsData = group.getItems();\n      // optimization for sorted data\n      if (group.options.sort == true) {\n        var dateComparator = function (a, b) {\n          return a.getTime() == b.getTime() ? 0 : a < b ? -1 : 1\n        };\n        var first = Math.max(0, util.binarySearchValue(itemsData, minDate, 'x', 'before', dateComparator));\n        var last = Math.min(itemsData.length, util.binarySearchValue(itemsData, maxDate, 'x', 'after', dateComparator) + 1);\n        if (last <= 0) {\n          last = itemsData.length;\n        }\n        var dataContainer = new Array(last-first);\n        for (j = first; j < last; j++) {\n          item = group.itemsData[j];\n          dataContainer[j-first] = item;\n        }\n        groupsData[groupIds[i]] = dataContainer;\n      }\n      else {\n        // If unsorted data, all data is relevant, just returning entire structure\n        groupsData[groupIds[i]] = group.itemsData;\n      }\n    }\n  }\n};\n\n\n/**\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {vis.DataSet} groupsData\n * @private\n */\nLineGraph.prototype._applySampling = function (groupIds, groupsData) {\n  var group;\n  if (groupIds.length > 0) {\n    for (var i = 0; i < groupIds.length; i++) {\n      group = this.groups[groupIds[i]];\n      if (group.options.sampling == true) {\n        var dataContainer = groupsData[groupIds[i]];\n        if (dataContainer.length > 0) {\n          var increment = 1;\n          var amountOfPoints = dataContainer.length;\n\n          // the global screen is used because changing the width of the yAxis may affect the increment, resulting in an endless loop\n          // of width changing of the yAxis.\n          //TODO: This assumes sorted data, but that's not guaranteed!\n          var xDistance = this.body.util.toGlobalScreen(dataContainer[dataContainer.length - 1].x) - this.body.util.toGlobalScreen(dataContainer[0].x);\n          var pointsPerPixel = amountOfPoints / xDistance;\n          increment = Math.min(Math.ceil(0.2 * amountOfPoints), Math.max(1, Math.round(pointsPerPixel)));\n\n          var sampledData = new Array(amountOfPoints);\n          for (var j = 0; j < amountOfPoints; j += increment) {\n            var idx = Math.round(j/increment);\n            sampledData[idx]=dataContainer[j];\n          }\n          groupsData[groupIds[i]] = sampledData.splice(0,Math.round(amountOfPoints/increment));\n        }\n      }\n    }\n  }\n};\n\n\n/**\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {vis.DataSet} groupsData\n * @param {object} groupRanges  | this is being filled here\n * @private\n */\nLineGraph.prototype._getYRanges = function (groupIds, groupsData, groupRanges) {\n  var groupData, group, i;\n  var combinedDataLeft = [];\n  var combinedDataRight = [];\n  var options;\n  if (groupIds.length > 0) {\n    for (i = 0; i < groupIds.length; i++) {\n      groupData = groupsData[groupIds[i]];\n      options = this.groups[groupIds[i]].options;\n      if (groupData.length > 0) {\n        group = this.groups[groupIds[i]];\n        // if bar graphs are stacked, their range need to be handled differently and accumulated over all groups.\n        if (options.stack === true && options.style === 'bar') {\n          if (options.yAxisOrientation === 'left') {\n            combinedDataLeft = combinedDataLeft.concat(groupData);\n          }\n          else {\n            combinedDataRight = combinedDataRight.concat(groupData);\n          }\n        }\n        else {\n          groupRanges[groupIds[i]] = group.getYRange(groupData, groupIds[i]);\n        }\n      }\n    }\n\n    // if bar graphs are stacked, their range need to be handled differently and accumulated over all groups.\n    Bars.getStackedYRange(combinedDataLeft, groupRanges, groupIds, '__barStackLeft', 'left');\n    Bars.getStackedYRange(combinedDataRight, groupRanges, groupIds, '__barStackRight', 'right');\n  }\n};\n\n\n/**\n * this sets the Y ranges for the Y axis. It also determines which of the axis should be shown or hidden.\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {Object} groupRanges\n * @returns {boolean} resized\n * @private\n */\nLineGraph.prototype._updateYAxis = function (groupIds, groupRanges) {\n  var resized = false;\n  var yAxisLeftUsed = false;\n  var yAxisRightUsed = false;\n  var minLeft = 1e9, minRight = 1e9, maxLeft = -1e9, maxRight = -1e9, minVal, maxVal;\n  // if groups are present\n  if (groupIds.length > 0) {\n    // this is here to make sure that if there are no items in the axis but there are groups, that there is no infinite draw/redraw loop.\n    for (var i = 0; i < groupIds.length; i++) {\n      var group = this.groups[groupIds[i]];\n      if (group && group.options.yAxisOrientation != 'right') {\n        yAxisLeftUsed = true;\n        minLeft = 1e9;\n        maxLeft = -1e9;\n      }\n      else if (group && group.options.yAxisOrientation) {\n        yAxisRightUsed = true;\n        minRight = 1e9;\n        maxRight = -1e9;\n      }\n    }\n\n    // if there are items:\n    for (i = 0; i < groupIds.length; i++) {\n      if (groupRanges.hasOwnProperty(groupIds[i])) {\n        if (groupRanges[groupIds[i]].ignore !== true) {\n          minVal = groupRanges[groupIds[i]].min;\n          maxVal = groupRanges[groupIds[i]].max;\n\n          if (groupRanges[groupIds[i]].yAxisOrientation != 'right') {\n            yAxisLeftUsed = true;\n            minLeft = minLeft > minVal ? minVal : minLeft;\n            maxLeft = maxLeft < maxVal ? maxVal : maxLeft;\n          }\n          else {\n            yAxisRightUsed = true;\n            minRight = minRight > minVal ? minVal : minRight;\n            maxRight = maxRight < maxVal ? maxVal : maxRight;\n          }\n        }\n      }\n    }\n\n    if (yAxisLeftUsed == true) {\n      this.yAxisLeft.setRange(minLeft, maxLeft);\n    }\n    if (yAxisRightUsed == true) {\n      this.yAxisRight.setRange(minRight, maxRight);\n    }\n  }\n  resized = this._toggleAxisVisiblity(yAxisLeftUsed, this.yAxisLeft) || resized;\n  resized = this._toggleAxisVisiblity(yAxisRightUsed, this.yAxisRight) || resized;\n\n  if (yAxisRightUsed == true && yAxisLeftUsed == true) {\n    this.yAxisLeft.drawIcons = true;\n    this.yAxisRight.drawIcons = true;\n  }\n  else {\n    this.yAxisLeft.drawIcons = false;\n    this.yAxisRight.drawIcons = false;\n  }\n  this.yAxisRight.master = !yAxisLeftUsed;\n  this.yAxisRight.masterAxis = this.yAxisLeft;\n\n  if (this.yAxisRight.master == false) {\n    if (yAxisRightUsed == true) {\n      this.yAxisLeft.lineOffset = this.yAxisRight.width;\n    }\n    else {\n      this.yAxisLeft.lineOffset = 0;\n    }\n\n    resized = this.yAxisLeft.redraw() || resized;\n    resized = this.yAxisRight.redraw() || resized;\n  }\n  else {\n    resized = this.yAxisRight.redraw() || resized;\n  }\n\n  // clean the accumulated lists\n  var tempGroups = ['__barStackLeft', '__barStackRight', '__lineStackLeft', '__lineStackRight'];\n  for (i = 0; i < tempGroups.length; i++) {\n    if (groupIds.indexOf(tempGroups[i]) != -1) {\n      groupIds.splice(groupIds.indexOf(tempGroups[i]), 1);\n    }\n  }\n\n  return resized;\n};\n\n\n/**\n * This shows or hides the Y axis if needed. If there is a change, the changed event is emitted by the updateYAxis function\n *\n * @param {boolean} axisUsed\n * @param {vis.DataAxis}  axis\n * @returns {boolean}\n * @private\n */\nLineGraph.prototype._toggleAxisVisiblity = function (axisUsed, axis) {\n  var changed = false;\n  if (axisUsed == false) {\n    if (axis.dom.frame.parentNode && axis.hidden == false) {\n      axis.hide();\n      changed = true;\n    }\n  }\n  else {\n    if (!axis.dom.frame.parentNode && axis.hidden == true) {\n      axis.show();\n      changed = true;\n    }\n  }\n  return changed;\n};\n\n\n/**\n * This uses the DataAxis object to generate the correct X coordinate on the SVG window. It uses the\n * util function toScreen to get the x coordinate from the timestamp. It also pre-filters the data and get the minMax ranges for\n * the yAxis.\n *\n * @param {Array.<Object>} datapoints\n * @private\n */\nLineGraph.prototype._convertXcoordinates = function (datapoints) {\n  var toScreen = this.body.util.toScreen;\n  for (var i = 0; i < datapoints.length; i++) {\n    datapoints[i].screen_x = toScreen(datapoints[i].x) + this.props.width;\n    datapoints[i].screen_y = datapoints[i].y; //starting point for range calculations\n    if (datapoints[i].end != undefined) {\n      datapoints[i].screen_end = toScreen(datapoints[i].end) + this.props.width;\n    }\n    else {\n      datapoints[i].screen_end = undefined;\n    }\n  }\n};\n\n\n/**\n * This uses the DataAxis object to generate the correct X coordinate on the SVG window. It uses the\n * util function toScreen to get the x coordinate from the timestamp. It also pre-filters the data and get the minMax ranges for\n * the yAxis.\n *\n * @param {Array.<Object>} datapoints\n * @param {vis.GraphGroup} group\n * @private\n */\nLineGraph.prototype._convertYcoordinates = function (datapoints, group) {\n  var axis = this.yAxisLeft;\n  var svgHeight = Number(this.svg.style.height.replace('px', ''));\n  if (group.options.yAxisOrientation == 'right') {\n    axis = this.yAxisRight;\n  }\n  for (var i = 0; i < datapoints.length; i++) {\n    datapoints[i].screen_y = Math.round(axis.convertValue(datapoints[i].y));\n  }\n  group.setZeroPosition(Math.min(svgHeight, axis.convertValue(0)));\n};\n\n\nexport default LineGraph;\n","/**\n * This object contains all possible options. It will check if the types are correct, if required if the option is one\n * of the allowed values.\n *\n * __any__ means that the name of the property does not matter.\n * __type__ is a required field for all objects and contains the allowed types of all objects\n */\nlet string = 'string';\nlet bool = 'boolean';\nlet number = 'number';\nlet array = 'array';\nlet date = 'date';\nlet object = 'object'; // should only be in a __type__ property\nlet dom = 'dom';\nlet moment = 'moment';\nlet any = 'any';\n\n\nlet allOptions = {\n  configure: {\n    enabled: {'boolean': bool},\n    filter: {'boolean': bool,'function': 'function'},\n    container: {dom},\n    __type__: {object,'boolean': bool,'function': 'function'}\n  },\n\n  //globals :\n  alignCurrentTime: {string, 'undefined': 'undefined'},  \n  yAxisOrientation: {string:['left','right']},\n  defaultGroup: {string},\n  sort: {'boolean': bool},\n  sampling: {'boolean': bool},\n  stack:{'boolean': bool},\n  graphHeight: {string, number},\n  shaded: {\n    enabled: {'boolean': bool},\n    orientation: {string:['bottom','top','zero','group']}, // top, bottom, zero, group\n    groupId: {object},\n    __type__: {'boolean': bool,object}\n  },\n  style: {string:['line','bar','points']}, // line, bar\n  barChart: {\n    width: {number},\n    minWidth: {number},\n    sideBySide: {'boolean': bool},\n    align: {string:['left','center','right']},\n    __type__: {object}\n  },\n  interpolation: {\n    enabled: {'boolean': bool},\n    parametrization: {string:['centripetal', 'chordal','uniform']}, // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n    alpha: {number},\n    __type__: {object,'boolean': bool}\n  },\n  drawPoints: {\n    enabled: {'boolean': bool},\n    onRender: { 'function': 'function' },\n    size: {number},\n    style: {string:['square','circle']}, // square, circle\n    __type__: {object,'boolean': bool,'function': 'function'}\n  },\n  dataAxis: {\n    showMinorLabels: {'boolean': bool},\n    showMajorLabels: {'boolean': bool},\n    showWeekScale: {'boolean': bool},\n    icons: {'boolean': bool},\n    width: {string, number},\n    visible: {'boolean': bool},\n    alignZeros: {'boolean': bool},\n    left:{\n      range: {min:{number,'undefined': 'undefined'},max:{number,'undefined': 'undefined'},__type__: {object}},\n      format: {'function': 'function'},\n      title: {text:{string,number,'undefined': 'undefined'},style:{string,'undefined': 'undefined'},__type__: {object}},\n      __type__: {object}\n    },\n    right:{\n      range: {min:{number,'undefined': 'undefined'},max:{number,'undefined': 'undefined'},__type__: {object}},\n      format: {'function': 'function'},\n      title: {text:{string,number,'undefined': 'undefined'},style:{string,'undefined': 'undefined'},__type__: {object}},\n      __type__: {object}\n    },\n    __type__: {object}\n  },\n  legend: {\n    enabled: {'boolean': bool},\n    icons: {'boolean': bool},\n    left: {\n      visible: {'boolean': bool},\n      position: {string:['top-right','bottom-right','top-left','bottom-left']},\n      __type__: {object}\n    },\n    right: {\n      visible: {'boolean': bool},\n      position: {string:['top-right','bottom-right','top-left','bottom-left']},\n      __type__: {object}\n    },\n    __type__: {object,'boolean': bool}\n  },\n  groups: {\n    visibility: {any},\n    __type__: {object}\n  },\n\n  autoResize: {'boolean': bool},\n  throttleRedraw: {number}, // TODO: DEPRICATED see https://github.com/almende/vis/issues/2511\n  clickToUse: {'boolean': bool},\n  end: {number, date, string, moment},\n  format: {\n    minorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      quarter: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object}\n    },\n    majorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      quarter: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object}\n    },\n    __type__: {object}\n  },\n  moment: {'function': 'function'},\n  height: {string, number},\n  hiddenDates: {\n    start: {date, number, string, moment},\n    end: {date, number, string, moment},\n    repeat: {string},\n    __type__: {object, array}\n  },\n  locale:{string},\n  locales:{\n    __any__: {any},\n    __type__: {object}\n  },\n  max: {date, number, string, moment},\n  maxHeight: {number, string},\n  maxMinorChars: {number},\n  min: {date, number, string, moment},\n  minHeight: {number, string},\n  moveable: {'boolean': bool},\n  multiselect: {'boolean': bool},\n  orientation: {string},\n  showCurrentTime: {'boolean': bool},\n  showMajorLabels: {'boolean': bool},\n  showMinorLabels: {'boolean': bool},\n  showWeekScale: {'boolean': bool},\n  start: {date, number, string, moment},\n  timeAxis: {\n    scale: {string,'undefined': 'undefined'},\n    step: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  width: {string, number},\n  zoomable: {'boolean': bool},\n  zoomKey: {string: ['ctrlKey', 'altKey', 'metaKey', '']},\n  zoomMax: {number},\n  zoomMin: {number},\n  zIndex: {number},\n  __type__: {object}\n};\n\nlet configureOptions = {\n  global: {\n    alignCurrentTime: ['none', 'year', 'month', 'quarter', 'week', 'isoWeek', 'day', 'date', 'hour', 'minute', 'second'],   \n    //yAxisOrientation: ['left','right'], // TDOO: enable as soon as Grahp2d doesn't crash when changing this on the fly\n    sort: true,\n    sampling: true,\n    stack:false,\n    shaded: {\n      enabled: false,\n      orientation: ['zero','top','bottom','group'] // zero, top, bottom\n    },\n    style: ['line','bar','points'], // line, bar\n    barChart: {\n      width: [50,5,100,5],\n      minWidth: [50,5,100,5],\n      sideBySide: false,\n      align: ['left','center','right'] // left, center, right\n    },\n    interpolation: {\n      enabled: true,\n      parametrization: ['centripetal','chordal','uniform'] // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n    },\n    drawPoints: {\n      enabled: true,\n      size: [6,2,30,1],\n      style: ['square', 'circle'] // square, circle\n    },\n    dataAxis: {\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      icons: false,\n      width: [40,0,200,1],\n      visible: true,\n      alignZeros: true,\n      left:{\n        //range: {min:'undefined': 'undefined'ined,max:'undefined': 'undefined'ined},\n        //format: function (value) {return value;},\n        title: {text:'',style:''}\n      },\n      right:{\n        //range: {min:'undefined': 'undefined'ined,max:'undefined': 'undefined'ined},\n        //format: function (value) {return value;},\n        title: {text:'',style:''}\n      }\n    },\n    legend: {\n      enabled: false,\n      icons: true,\n      left: {\n        visible: true,\n        position: ['top-right','bottom-right','top-left','bottom-left'] // top/bottom - left,right\n      },\n      right: {\n        visible: true,\n        position: ['top-right','bottom-right','top-left','bottom-left'] // top/bottom - left,right\n      }\n    },\n\n    autoResize: true,\n    clickToUse: false,\n    end: '',\n    format: {\n      minorLabels: {\n        millisecond:'SSS',\n        second:     's',\n        minute:     'HH:mm',\n        hour:       'HH:mm',\n        weekday:    'ddd D',\n        day:        'D',\n        week:       'w',\n        month:      'MMM',\n        quarter:    '[Q]Q',\n        year:       'YYYY'\n      },\n      majorLabels: {\n        millisecond:'HH:mm:ss',\n        second:     'D MMMM HH:mm',\n        minute:     'ddd D MMMM',\n        hour:       'ddd D MMMM',\n        weekday:    'MMMM YYYY',\n        day:        'MMMM YYYY',\n        week:       'MMMM YYYY',\n        month:      'YYYY',\n        quarter:    'YYYY',\n        year:       ''\n      }\n    },\n\n    height: '',\n    locale: '',\n    max: '',\n    maxHeight: '',\n    maxMinorChars: [7, 0, 20, 1],\n    min: '',\n    minHeight: '',\n    moveable:true,\n    orientation: ['both', 'bottom', 'top'],\n    showCurrentTime: false,\n    showMajorLabels: true,\n    showMinorLabels: true,\n    showWeekScale: false,\n    start: '',\n    width: '100%',\n    zoomable: true,\n    zoomKey: ['ctrlKey', 'altKey', 'metaKey', ''],\n    zoomMax: [315360000000000, 10, 315360000000000, 1],\n    zoomMin: [10, 10, 315360000000000, 1],\n    zIndex: 0\n  }\n};\n\nexport {allOptions, configureOptions};\n","import moment from '../module/moment';\nimport util, { typeCoerceDataSet } from '../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Range from './Range';\nimport Core from './Core';\nimport TimeAxis from './component/TimeAxis';\nimport CurrentTime from './component/CurrentTime';\nimport CustomTime from './component/CustomTime';\nimport LineGraph from './component/LineGraph';\n\nimport { Validator } from '../shared/Validator';\nimport { printStyle } from '../shared/Validator';\nimport { allOptions } from './optionsGraph2d';\nimport { configureOptions } from './optionsGraph2d';\n\nimport Configurator from '../shared/Configurator';\n\n/**\n * Create a timeline visualization\n * @param {HTMLElement} container\n * @param {vis.DataSet | Array} [items]\n * @param {vis.DataSet | Array | vis.DataView | Object} [groups]\n * @param {Object} [options]  See Graph2d.setOptions for the available options.\n * @constructor Graph2d\n * @extends Core\n */\nfunction Graph2d (container, items, groups, options) {\n  // if the third element is options, the forth is groups (optionally);\n  if (!(Array.isArray(groups) || groups instanceof DataSet || groups instanceof DataView) && groups instanceof Object) {\n    var forthArgument = options;\n    options = groups;\n    groups = forthArgument;\n  }\n\n  // TODO: REMOVE THIS in the next MAJOR release\n  // see https://github.com/almende/vis/issues/2511\n  if (options && options.throttleRedraw) {\n    console.warn(\"Graph2d option \\\"throttleRedraw\\\" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.\");\n  }\n\n  var me = this;\n  this.defaultOptions = {\n    start: null,\n    end:   null,\n\n    autoResize: true,\n\n    orientation: {\n      axis: 'bottom',   // axis orientation: 'bottom', 'top', or 'both'\n      item: 'bottom'    // not relevant for Graph2d\n    },\n\n    moment: moment,\n\n    width: null,\n    height: null,\n    maxHeight: null,\n    minHeight: null\n  };\n  this.options = util.deepExtend({}, this.defaultOptions);\n\n  // Create the DOM, props, and emitter\n  this._create(container);\n\n  // all components listed here will be repainted automatically\n  this.components = [];\n\n  this.body = {\n    dom: this.dom,\n    domProps: this.props,\n    emitter: {\n      on: this.on.bind(this),\n      off: this.off.bind(this),\n      emit: this.emit.bind(this)\n    },\n    hiddenDates: [],\n    util: {\n      toScreen: me._toScreen.bind(me),\n      toGlobalScreen: me._toGlobalScreen.bind(me), // this refers to the root.width\n      toTime: me._toTime.bind(me),\n      toGlobalTime : me._toGlobalTime.bind(me)\n    }\n  };\n\n  // range\n  this.range = new Range(this.body);\n  this.components.push(this.range);\n  this.body.range = this.range;\n\n  // time axis\n  this.timeAxis = new TimeAxis(this.body);\n  this.components.push(this.timeAxis);\n  //this.body.util.snap = this.timeAxis.snap.bind(this.timeAxis);\n\n  // current time bar\n  this.currentTime = new CurrentTime(this.body);\n  this.components.push(this.currentTime);\n\n  // item set\n  this.linegraph = new LineGraph(this.body);\n\n  this.components.push(this.linegraph);\n\n  this.itemsData = null;      // DataSet\n  this.groupsData = null;     // DataSet\n\n\n  this.on('tap', function (event) {\n    me.emit('click', me.getEventProperties(event))\n  });\n  this.on('doubletap', function (event) {\n    me.emit('doubleClick', me.getEventProperties(event))\n  });\n  this.dom.root.oncontextmenu = function (event) {\n    me.emit('contextmenu', me.getEventProperties(event))\n  };\n  \n  //Single time autoscale/fit\n  this.initialFitDone = false;\n  this.on('changed', function (){\n    if (me.itemsData == null) return;\n    if (!me.initialFitDone && !me.options.rollingMode) {\n      me.initialFitDone = true;\n      if (me.options.start != undefined || me.options.end != undefined) {\n        if (me.options.start == undefined || me.options.end == undefined) {\n          var range = me.getItemRange();\n        }\n\n        var start = me.options.start != undefined ? me.options.start : range.min;\n        var end   = me.options.end   != undefined ? me.options.end   : range.max;\n        me.setWindow(start, end, {animation: false});\n      } else {\n        me.fit({animation: false});\n      }\n    }\n\n    if (!me.initialDrawDone && (me.initialRangeChangeDone || (!me.options.start && !me.options.end) \n      || me.options.rollingMode)) {\n      me.initialDrawDone = true;\n      me.dom.root.style.visibility = 'visible';\n      me.dom.loadingScreen.parentNode.removeChild(me.dom.loadingScreen);\n      if (me.options.onInitialDrawComplete) {\n        setTimeout(() => {\n          return me.options.onInitialDrawComplete();\n        }, 0)\n      }\n    }\n  });\n  \n  // apply options\n  if (options) {\n    this.setOptions(options);\n  }\n\n  // IMPORTANT: THIS HAPPENS BEFORE SET ITEMS!\n  if (groups) {\n    this.setGroups(groups);\n  }\n\n  // create itemset\n  if (items) {\n    this.setItems(items);\n  }\n\n  // draw for the first time\n  this._redraw();\n}\n\n// Extend the functionality from Core\nGraph2d.prototype = new Core();\n\nGraph2d.prototype.setOptions = function (options) {\n  // validate options\n  let errorFound = Validator.validate(options, allOptions);\n  if (errorFound === true) {\n    console.log('%cErrors have been found in the supplied options object.', printStyle);\n  }\n\n  Core.prototype.setOptions.call(this, options);\n};\n\n/**\n * Set items\n * @param {vis.DataSet | Array | null} items\n */\nGraph2d.prototype.setItems = function(items) {\n  var initialLoad = (this.itemsData == null);\n\n  // convert to type DataSet when needed\n  var newDataSet;\n  if (!items) {\n    newDataSet = null;\n  }\n  else if (items instanceof DataSet || items instanceof DataView) {\n    newDataSet = typeCoerceDataSet(items);\n  }\n  else {\n    // turn an array into a dataset\n    newDataSet = typeCoerceDataSet(new DataSet(items));\n  }\n\n  // set items\n  if (this.itemsData) {\n    // stop maintaining a coerced version of the old data set\n    this.itemsData.dispose()\n  }\n  this.itemsData = newDataSet;\n  this.linegraph && this.linegraph.setItems(newDataSet != null ? newDataSet.rawDS : null);\n\n  if (initialLoad) {\n    if (this.options.start != undefined || this.options.end != undefined) {\n      var start = this.options.start != undefined ? this.options.start : null;\n      var end   = this.options.end != undefined   ? this.options.end : null;\n      this.setWindow(start, end, {animation: false});\n    }\n    else {\n      this.fit({animation: false});\n    }\n  }\n};\n\n/**\n * Set groups\n * @param {vis.DataSet | Array} groups\n */\nGraph2d.prototype.setGroups = function(groups) {\n  // convert to type DataSet when needed\n  var newDataSet;\n  if (!groups) {\n    newDataSet = null;\n  }\n  else if (groups instanceof DataSet || groups instanceof DataView) {\n    newDataSet = groups;\n  }\n  else {\n    // turn an array into a dataset\n    newDataSet = new DataSet(groups);\n  }\n\n  this.groupsData = newDataSet;\n  this.linegraph.setGroups(newDataSet);\n};\n\n/**\n * Returns an object containing an SVG element with the icon of the group (size determined by iconWidth and iconHeight), the label of the group (content) and the yAxisOrientation of the group (left or right).\n * @param {vis.GraphGroup.id} groupId\n * @param {number} width\n * @param {number} height\n * @returns {{icon: SVGElement, label: string, orientation: string}|string}\n */\nGraph2d.prototype.getLegend = function(groupId, width, height) {\n  if (width  === undefined) {width  = 15;}\n  if (height === undefined) {height = 15;}\n  if (this.linegraph.groups[groupId] !== undefined) {\n    return this.linegraph.groups[groupId].getLegend(width,height);\n  }\n  else {\n    return \"cannot find group:'\" +  groupId + \"'\";\n  }\n};\n\n/**\n * This checks if the visible option of the supplied group (by ID) is true or false.\n * @param {vis.GraphGroup.id} groupId\n * @returns {boolean}\n */\nGraph2d.prototype.isGroupVisible = function(groupId) {\n  if (this.linegraph.groups[groupId] !== undefined) {\n    return (this.linegraph.groups[groupId].visible && (this.linegraph.options.groups.visibility[groupId] === undefined || this.linegraph.options.groups.visibility[groupId] == true));\n  }\n  else {\n    return false;\n  }\n};\n\n\n/**\n * Get the data range of the item set.\n * @returns {{min: Date, max: Date}} range  A range with a start and end Date.\n *                                          When no minimum is found, min==null\n *                                          When no maximum is found, max==null\n */\nGraph2d.prototype.getDataRange = function() {\n  var min = null;\n  var max = null;\n\n  // calculate min from start filed\n  for (var groupId in this.linegraph.groups) {\n    if (this.linegraph.groups.hasOwnProperty(groupId)) {\n      if (this.linegraph.groups[groupId].visible == true) {\n        for (var i = 0; i < this.linegraph.groups[groupId].itemsData.length; i++) {\n          var item = this.linegraph.groups[groupId].itemsData[i];\n          var value = util.convert(item.x, 'Date').valueOf();\n          min = min == null ? value : min > value ? value : min;\n          max = max == null ? value : max < value ? value : max;\n        }\n      }\n    }\n  }\n\n  return {\n    min: (min != null) ? new Date(min) : null,\n    max: (max != null) ? new Date(max) : null\n  };\n};\n\n\n/**\n * Generate Timeline related information from an event\n * @param {Event} event\n * @return {Object} An object with related information, like on which area\n *                  The event happened, whether clicked on an item, etc.\n */\nGraph2d.prototype.getEventProperties = function (event) {\n  var clientX = event.center ? event.center.x : event.clientX;\n  var clientY = event.center ? event.center.y : event.clientY;\n  var x = clientX - util.getAbsoluteLeft(this.dom.centerContainer);\n  var y = clientY - util.getAbsoluteTop(this.dom.centerContainer);\n  var time = this._toTime(x);\n\n  var customTime = CustomTime.customTimeFromTarget(event);\n\n  var element = util.getTarget(event);\n  var what = null;\n  if (util.hasParent(element, this.timeAxis.dom.foreground))              {what = 'axis';}\n  else if (this.timeAxis2 && util.hasParent(element, this.timeAxis2.dom.foreground)) {what = 'axis';}\n  else if (util.hasParent(element, this.linegraph.yAxisLeft.dom.frame))   {what = 'data-axis';}\n  else if (util.hasParent(element, this.linegraph.yAxisRight.dom.frame))  {what = 'data-axis';}\n  else if (util.hasParent(element, this.linegraph.legendLeft.dom.frame))  {what = 'legend';}\n  else if (util.hasParent(element, this.linegraph.legendRight.dom.frame)) {what = 'legend';}\n  else if (customTime != null)                {what = 'custom-time';}\n  else if (util.hasParent(element, this.currentTime.bar))                 {what = 'current-time';}\n  else if (util.hasParent(element, this.dom.center))                      {what = 'background';}\n\n  var value = [];\n  var yAxisLeft = this.linegraph.yAxisLeft;\n  var yAxisRight = this.linegraph.yAxisRight;\n  if (!yAxisLeft.hidden && this.itemsData.length > 0) {\n    value.push(yAxisLeft.screenToValue(y));\n  }\n  if (!yAxisRight.hidden && this.itemsData.length > 0) {\n    value.push(yAxisRight.screenToValue(y));\n  }\n\n  return {\n    event: event,\n    customTime: customTime ? customTime.options.id : null,\n    what: what,\n    pageX: event.srcEvent ? event.srcEvent.pageX : event.pageX,\n    pageY: event.srcEvent ? event.srcEvent.pageY : event.pageY,\n    x: x,\n    y: y,\n    time: time,\n    value: value\n  }\n};\n\n/**\n * Load a configurator\n * @return {Object}\n * @private\n */\nGraph2d.prototype._createConfigurator = function () {\n  return new Configurator(this, this.dom.container, configureOptions);\n};\n\n\nexport default Graph2d;\n","// locales\nimport \"moment/locale/de\";\nimport \"moment/locale/es\";\nimport \"moment/locale/fr\";\nimport \"moment/locale/it\";\nimport \"moment/locale/ja\";\nimport \"moment/locale/nl\";\nimport \"moment/locale/pl\";\nimport \"moment/locale/ru\";\nimport \"moment/locale/uk\";\n\n// utils\nimport * as util from \"vis-util/esnext\";\nimport * as DOMutil from \"./DOMutil\";\n\n// data\nimport { DataSet, DataView, Queue } from \"vis-data/esnext\";\n\n// Timeline\nimport Timeline from \"./timeline/Timeline\";\nimport Graph2d from \"./timeline/Graph2d\";\n\nimport Core from \"./timeline/Core\";\nimport * as DateUtil from \"./timeline/DateUtil\";\nimport Range from \"./timeline/Range\";\nimport * as stack from \"./timeline/Stack\";\nimport TimeStep from \"./timeline/TimeStep\";\n\nimport Item from \"./timeline/component/item/Item\";\nimport BackgroundItem from \"./timeline/component/item/BackgroundItem\";\nimport BoxItem from \"./timeline/component/item/BoxItem\";\nimport PointItem from \"./timeline/component/item/PointItem\";\nimport RangeItem from \"./timeline/component/item/RangeItem\";\n\nimport BackgroundGroup from \"./timeline/component/BackgroundGroup\";\nimport Component from \"./timeline/component/Component\";\nimport CurrentTime from \"./timeline/component/CurrentTime\";\nimport CustomTime from \"./timeline/component/CustomTime\";\nimport DataAxis from \"./timeline/component/DataAxis\";\nimport DataScale from \"./timeline/component/DataScale\";\nimport GraphGroup from \"./timeline/component/GraphGroup\";\nimport Group from \"./timeline/component/Group\";\nimport ItemSet from \"./timeline/component/ItemSet\";\nimport Legend from \"./timeline/component/Legend\";\nimport LineGraph from \"./timeline/component/LineGraph\";\nimport TimeAxis from \"./timeline/component/TimeAxis\";\n\n// bundled external libraries\nimport moment from \"./module/moment\";\nimport Hammer from \"./module/hammer\";\nimport keycharm from \"keycharm\";\n\n// TODO: This should probably be moved somewhere else to ensure that both builds\n// behave the same way.\nconst defaultLanguage = DOMutil.getNavigatorLanguage();\nmoment.locale(defaultLanguage);\n\nconst timeline = {\n  Core,\n  DateUtil,\n  Range,\n  stack,\n  TimeStep,\n\n  components: {\n    items: {\n      Item,\n      BackgroundItem,\n      BoxItem,\n      PointItem,\n      RangeItem\n    },\n\n    BackgroundGroup,\n    Component,\n    CurrentTime,\n    CustomTime,\n    DataAxis,\n    DataScale,\n    GraphGroup,\n    Group,\n    ItemSet,\n    Legend,\n    LineGraph,\n    TimeAxis\n  }\n};\n\nexport {\n  DOMutil,\n  DataSet,\n  DataView,\n  Graph2d,\n  Hammer,\n  Queue,\n  Timeline,\n  keycharm,\n  moment,\n  timeline,\n  util\n};\n"],"names":["module","hookCallback","some","hooks","apply","arguments","isArray","input","Array","Object","prototype","toString","call","isObject","isUndefined","isNumber","isDate","Date","map","arr","fn","i","res","length","push","hasOwnProp","a","b","hasOwnProperty","extend","valueOf","createUTC","format","locale","strict","createLocalOrUTC","utc","getParsingFlags","m","_pf","empty","unusedTokens","unusedInput","overflow","charsLeftOver","nullInput","invalidMonth","invalidFormat","userInvalidated","iso","parsedDateParts","meridiem","rfc2822","weekdayMismatch","isValid","_isValid","flags","parsedParts","isNowValid","isNaN","_d","getTime","invalidWeekday","_strict","undefined","bigHour","isFrozen","createInvalid","NaN","fun","t","this","len","momentProperties","copyConfig","to","from","prop","val","_isAMomentObject","_i","_f","_l","_tzm","_isUTC","_offset","_locale","updateInProgress","Moment","config","updateOffset","isMoment","obj","absFloor","number","Math","ceil","floor","toInt","argumentForCoercion","coercedNumber","value","isFinite","compareArrays","array1","array2","dontConvert","min","lengthDiff","abs","diffs","warn","msg","suppressDeprecationWarnings","console","deprecate","firstTime","deprecationHandler","arg","args","key","slice","join","Error","stack","keys","deprecations","deprecateSimple","name","isFunction","Function","mergeConfigs","parentConfig","childConfig","Locale","set","aliases","addUnitAlias","unit","shorthand","lowerCase","toLowerCase","normalizeUnits","units","normalizeObjectUnits","inputObject","normalizedProp","normalizedInput","priorities","addUnitPriority","priority","zeroFill","targetLength","forceSign","absNumber","zerosToFill","pow","max","substr","formattingTokens","localFormattingTokens","formatFunctions","formatTokenFunctions","addFormatToken","token","padded","ordinal","callback","func","localeData","formatMoment","expandFormat","array","match","replace","mom","output","makeFormatFunction","invalidDate","replaceLongDateFormatTokens","longDateFormat","lastIndex","test","match1","match2","match3","match4","match6","match1to2","match3to4","match5to6","match1to3","match1to4","match1to6","matchUnsigned","matchSigned","matchOffset","matchShortOffset","matchWord","regexes","addRegexToken","regex","strictRegex","isStrict","getParseRegexForToken","RegExp","regexEscape","matched","p1","p2","p3","p4","s","tokens","addParseToken","addWeekParseToken","_w","addTimeToArrayFromToken","_a","daysInYear","year","isLeapYear","y","parseTwoDigitYear","parseInt","indexOf","getSetYear","makeGetSet","keepTime","set$1","get","month","date","daysInMonth","x","modMonth","o","monthsShort","months","monthsShortRegex","monthsRegex","monthsParse","MONTHS_IN_FORMAT","defaultLocaleMonths","split","defaultLocaleMonthsShort","handleStrictParse","monthName","ii","llc","toLocaleLowerCase","_monthsParse","_longMonthsParse","_shortMonthsParse","setMonth","dayOfMonth","getSetMonth","defaultMonthsShortRegex","defaultMonthsRegex","computeMonthsParse","cmpLenRev","shortPieces","longPieces","mixedPieces","sort","_monthsRegex","_monthsShortRegex","_monthsStrictRegex","_monthsShortStrictRegex","createDate","d","h","M","ms","getFullYear","setFullYear","createUTCDate","UTC","getUTCFullYear","setUTCFullYear","firstWeekOffset","dow","doy","fwd","getUTCDay","dayOfYearFromWeeks","week","weekday","resYear","resDayOfYear","dayOfYear","weekOfYear","resWeek","weekOffset","weeksInYear","weekOffsetNext","shiftWeekdays","ws","n","concat","weekdaysMin","weekdaysShort","weekdays","weekdaysMinRegex","weekdaysShortRegex","weekdaysRegex","weekdaysParse","defaultLocaleWeekdays","defaultLocaleWeekdaysShort","defaultLocaleWeekdaysMin","handleStrictParse$1","weekdayName","_weekdaysParse","_shortWeekdaysParse","_minWeekdaysParse","day","defaultWeekdaysRegex","defaultWeekdaysShortRegex","defaultWeekdaysMinRegex","computeWeekdaysParse","minp","shortp","longp","minPieces","_weekdaysRegex","_weekdaysShortRegex","_weekdaysMinRegex","_weekdaysStrictRegex","_weekdaysShortStrictRegex","_weekdaysMinStrictRegex","hFormat","hours","lowercase","minutes","matchMeridiem","_meridiemParse","seconds","kInput","_isPm","isPM","_meridiem","pos","pos1","pos2","globalLocale","getSetHour","baseConfig","calendar","sameDay","nextDay","nextWeek","lastDay","lastWeek","sameElse","LTS","LT","L","LL","LLL","LLLL","dayOfMonthOrdinalParse","relativeTime","future","past","ss","mm","hh","dd","MM","yy","meridiemParse","locales","localeFamilies","normalizeLocale","loadLocale","oldLocale","exports","_abbr","require","getSetGlobalLocale","e","values","data","getLocale","defineLocale","abbr","_config","parentLocale","forEach","names","j","next","chooseLocale","checkOverflow","_overflowDayOfYear","_overflowWeeks","_overflowWeekday","defaults","c","configFromArray","currentDate","expectedWeekday","yearToUse","nowValue","now","_useUTC","getUTCMonth","getUTCDate","getMonth","getDate","currentDateArray","w","weekYear","temp","weekdayOverflow","GG","W","E","createLocal","_week","curWeek","gg","_dayOfYear","dayOfYearFromWeekInfo","_nextDay","getDay","setUTCMinutes","getUTCMinutes","extendedIsoRegex","basicIsoRegex","tzRegex","isoDates","isoTimes","aspNetJsonRegex","configFromISO","l","allowTime","dateFormat","timeFormat","tzFormat","string","exec","configFromStringAndFormat","untruncateYear","yearStr","obsOffsets","UT","GMT","EDT","EST","CDT","CST","MDT","MST","PDT","PST","configFromRFC2822","monthStr","dayStr","hourStr","minuteStr","secondStr","result","parsedArray","weekdayStr","parsedInput","checkWeekday","obsOffset","militaryOffset","numOffset","hm","calculateOffset","ISO_8601","RFC_2822","skipped","stringLength","totalParsedInputLength","hour","isPm","meridiemHour","meridiemFixWrap","prepareConfig","preparse","tempConfig","bestMoment","scoreToBeat","currentScore","score","configFromStringAndArray","createFromInputFallback","configFromString","minute","second","millisecond","configFromObject","configFromInput","isUTC","getOwnPropertyNames","k","isObjectEmpty","add","prototypeMin","other","prototypeMax","pickBy","moments","ordering","Duration","duration","years","quarters","quarter","weeks","isoWeek","days","milliseconds","unitHasDecimal","parseFloat","isDurationValid","_milliseconds","_days","_months","_data","_bubble","isDuration","absRound","round","offset","separator","utcOffset","sign","offsetFromString","chunkOffset","matcher","matches","parts","cloneWithOffset","model","diff","clone","setTime","local","getDateOffset","getTimezoneOffset","isUtc","aspNetRegex","isoRegex","createDuration","ret","diffRes","base","parseIso","isBefore","positiveMomentsDifference","inp","isAfter","createAdder","direction","period","tmp","addSubtract","isAdding","invalid","subtract","monthDiff","wholeMonthDiff","anchor","newLocaleData","defaultFormat","defaultFormatUtc","lang","mod$1","dividend","divisor","localStartOfDate","utcStartOfDate","addWeekYearFormatToken","getter","getSetWeekYearHelper","weeksTarget","setWeekAll","dayOfYearData","isoWeekYear","_dayOfMonthOrdinalParse","_ordinalParse","_dayOfMonthOrdinalParseLenient","getSetDayOfMonth","getSetMinute","getSetSecond","parseMs","getSetMillisecond","proto","preParsePostFormat","time","formats","sod","startOf","calendarFormat","asFloat","that","zoneDelta","endOf","startOfDate","isoWeekday","inputString","postformat","withoutSuffix","humanize","fromNow","toNow","invalidAt","localInput","isBetween","inclusivity","localFrom","localTo","isSame","inputMs","isSameOrAfter","isSameOrBefore","parsingFlags","prioritized","unitsObj","u","getPrioritizedUnits","toArray","toObject","toDate","toISOString","keepOffset","inspect","zone","isLocal","prefix","suffix","toJSON","unix","creationData","isoWeeks","weekInfo","isoWeeksInYear","parseWeekday","parseIsoWeekday","keepLocalTime","keepMinutes","localAdjust","_changeInProgress","parseZone","tZone","hasAlignedHourOffset","isDST","isUtcOffset","zoneAbbr","zoneName","dates","isDSTShifted","_isDSTShifted","proto$1","get$1","index","field","setter","listMonthsImpl","out","listWeekdaysImpl","localeSorted","shift","_calendar","_longDateFormat","formatUpper","toUpperCase","_invalidDate","_ordinal","isFuture","_relativeTime","pastFuture","source","isFormat","_monthsShort","_monthsParseExact","firstDayOfYear","firstDayOfWeek","_weekdays","_weekdaysMin","_weekdaysShort","_weekdaysParseExact","_fullWeekdaysParse","charAt","isLower","langData","mathAbs","addSubtract$1","absCeil","daysToMonths","monthsToDays","makeAs","alias","as","asMilliseconds","asSeconds","asMinutes","asHours","asDays","asWeeks","asMonths","asQuarters","asYears","makeGetter","thresholds","substituteTimeAgo","relativeTime$1","posNegDuration","abs$1","toISOString$1","Y","D","toFixed","total","totalSign","ymSign","daysSign","hmsSign","proto$2","monthsFromDays","withSuffix","toIsoString","version","updateLocale","tmpLocale","relativeTimeRounding","roundingFunction","relativeTimeThreshold","threshold","limit","myMoment","HTML5_FMT","DATETIME_LOCAL","DATETIME_LOCAL_SECONDS","DATETIME_LOCAL_MS","DATE","TIME","TIME_SECONDS","TIME_MS","WEEK","MONTH","factory","check","moment","processRelativeTime","monthsParseExact","weekdaysParseExact","require$$0","monthsShortDot","monthsStrictRegex","monthsShortStrictRegex","longMonthsParse","shortMonthsParse","ll","lll","llll","monthsShortWithDots","monthsShortWithoutDots","monthsNominative","monthsSubjective","plural","translate","momentToFormat","relativeTimeWithPlural","num","forms","standalone","processHoursFunction","str","it","globalThis","window","self","global","error","fails","defineProperty","nativePropertyIsEnumerable","propertyIsEnumerable","getOwnPropertyDescriptor","V","descriptor","enumerable","bitmap","configurable","writable","classof","TypeError","IndexedObject","requireObjectCoercible","PREFERRED_STRING","document","EXISTS","createElement","DESCRIPTORS","nativeGetOwnPropertyDescriptor","O","P","toIndexedObject","toPrimitive","IE8_DOM_DEFINE","has","createPropertyDescriptor","propertyIsEnumerableModule","f","replacement","isForced","feature","detection","normalize","POLYFILL","NATIVE","String","aFunction","nativeDefineProperty","Attributes","anObject","object","definePropertyModule","wrapConstructor","NativeConstructor","Wrapper","options","USE_NATIVE","VIRTUAL_PROTOTYPE","sourceProperty","targetProperty","nativeProperty","resultProperty","TARGET","target","GLOBAL","STATIC","stat","PROTO","nativeSource","path","targetPrototype","forced","noTargetGet","bind","wrap","sham","createNonEnumerableProperty","real","objectDefinePropertyModile","desc","parent","argument","toInteger","integer","createMethod","IS_INCLUDES","$this","el","fromIndex","toLength","toAbsoluteIndex","includes","hiddenKeys","internalObjectKeys","enumBugKeys","defineProperties","Properties","objectKeys","T","variable","namespace","method","getOwnPropertySymbols","getBuiltIn","getOwnPropertyNamesModule","getOwnPropertySymbolsModule","propertyKey","getOwnPropertyDescriptors","getOwnPropertyDescriptorModule","ownKeys","createProperty","FAILS_ON_PRIMITIVES","activeXDocument","Symbol","NATIVE_SYMBOL","iterator","setGlobal","store","mode","copyright","id","postfix","random","shared","uid","IE_PROTO","sharedKey","EmptyConstructor","scriptTag","content","NullProtoObject","domain","ActiveXObject","iframeDocument","iframe","write","close","parentWindow","NullProtoObjectViaActiveX","documentCreateElement","style","display","html","appendChild","src","contentWindow","open","F","create","nativeGetOwnPropertyNames","windowNames","getWindowNames","WellKnownSymbolsStore","createWellKnownSymbol","USE_SYMBOL_AS_UID","withoutSetter","wellKnownSymbol","NAME","wrappedWellKnownSymbolModule","TO_STRING_TAG","CORRECT_ARGUMENTS","classofRaw","TO_STRING_TAG_SUPPORT","tag","tryGet","callee","TAG","SET_METHOD","functionToString","inspectSource","WeakMap","NATIVE_WEAK_MAP","wmget","wmhas","wmset","metadata","STATE","objectHas","enforce","getterFor","TYPE","state","type","SPECIES","originalArray","C","constructor","IS_MAP","IS_FILTER","IS_SOME","IS_EVERY","IS_FIND_INDEX","NO_HOLES","callbackfn","specificCreate","boundFunction","arraySpeciesCreate","filter","every","find","findIndex","$forEach","HIDDEN","TO_PRIMITIVE","setInternalState","InternalStateModule","getInternalState","ObjectPrototype","$Symbol","$stringify","getOwnPropertyNamesExternal","AllSymbols","ObjectPrototypeSymbols","StringToSymbolRegistry","SymbolToStringRegistry","QObject","USE_SETTER","findChild","setSymbolDescriptor","nativeObjectCreate","ObjectPrototypeDescriptor","description","symbol","isSymbol","$defineProperty","$defineProperties","properties","$getOwnPropertySymbols","$propertyIsEnumerable","$getOwnPropertyDescriptor","$getOwnPropertyNames","IS_OBJECT_PROTOTYPE","redefine","defineWellKnownSymbol","keyFor","sym","useSetter","useSimple","FORCED_JSON_STRINGIFY","$","stringify","replacer","space","$replacer","setToStringTag","IteratorPrototype","PrototypeOfArrayIteratorPrototype","arrayIterator","getPrototypeOf","CORRECT_PROTOTYPE_GETTER","BUGGY_SAFARI_ITERATORS","returnThis","setPrototypeOf","CORRECT_SETTER","aPossiblePrototype","__proto__","IteratorsCore","ITERATOR","Iterable","IteratorConstructor","DEFAULT","IS_SET","FORCED","Iterators","createIteratorConstructor","CurrentIteratorPrototype","methods","KEY","getIterationMethod","KIND","defaultIterator","IterablePrototype","INCORRECT_VALUES_NAME","nativeIterator","anyNativeIterator","entries","defineIterator","iterated","kind","done","Arguments","COLLECTION_NAME","CSSRuleList","CSSStyleDeclaration","CSSValueList","ClientRectList","DOMRectList","DOMStringList","DOMTokenList","DataTransferItemList","FileList","HTMLAllCollection","HTMLCollection","HTMLFormElement","HTMLSelectElement","MediaList","MimeTypeArray","NamedNodeMap","NodeList","PaintRequestList","Plugin","PluginArray","SVGLengthList","SVGNumberList","SVGPathSegList","SVGPointList","SVGStringList","SVGTransformList","SourceBufferList","StyleSheetList","TextTrackCueList","TextTrackList","TouchList","Collection","CollectionPrototype","CONVERT_TO_STRING","first","S","position","size","charCodeAt","codeAt","point","iteratorMethod","getIteratorMethod","_Object$defineProperty","nativeKeys","METHOD_NAME","whitespace","whitespaces","ltrim","rtrim","start","end","trim","$trim","CONSTRUCTOR","entryVirtual","StringPrototype","own","cache","thrower","ACCESSORS","argument0","argument1","STRICT_METHOD","arrayMethodIsStrict","USES_TO_LENGTH","arrayMethodUsesToLength","ArrayPrototype","DOMIterables","process","versions","v8","userAgent","V8_VERSION","foo","Boolean","$map","HAS_SPECIES_SUPPORT","arrayMethodHasSpeciesSupport","$parseInt","hex","radix","parseIntImplementation","TO_ENTRIES","$values","$filter","IS_CONCAT_SPREADABLE","IS_CONCAT_SPREADABLE_SUPPORT","SPECIES_SUPPORT","isConcatSpreadable","spreadable","A","_Array$isArray","arr2","ENTRIES","returnMethod","SAFE_CLOSING","called","iteratorWithReturn","INCORRECT_ITERATION","SKIP_CLOSING","ITERATION_SUPPORT","checkCorrectnessOfIteration","iterable","arrayLike","step","argumentsLength","mapfn","mapping","isArrayIteratorMethod","callWithSafeIterationClosing","iter","_isIterable","_Array$from","arrayWithoutHoles","iterableToArray","nonIterableSpread","nativeSlice","Constructor","fin","nativeGetPrototypeOf","$indexOf","nativeIndexOf","NEGATIVE_ZERO","searchElement","nativeAssign","assign","B","chr","WrappedWellKnownSymbolModule","JSON","_typeof","_Symbol","_Symbol$iterator","_arr","_n","_e","_s","_getIterator","err","arrayWithHoles","iterableToArrayLimit","nonIterableRest","AleaImplementation","seed","mash","Mash","s0","s1","s2","mashSeed","uint32","fract53","algorithm","_Date$now","prepareElements","JSONcontainer","elementType","redundant","used","cleanupElements","parentNode","removeChild","resetElements","getSVGElement","svgContainer","element","createElementNS","getDOMElement","DOMContainer","insertBefore","drawPoint","groupTemplate","labelObj","setAttributeNS","styles","className","label","xOffset","yOffset","textContent","drawBar","width","height","rect","getNavigatorLanguage","navigator","languages","userLanguage","language","browserLanguage","ReferenceError","re","low","hi","fix","prev","core","$some","assertThisInitialized","_getPrototypeOf","_Object$setPrototypeOf","_Object$getPrototypeOf","_setPrototypeOf","p","subClass","superClass","_Object$create","nativeSort","FAILS_ON_UNDEFINED","FAILS_ON_NULL","comparefn","isExtensible","preventExtensions","METADATA","setMetadata","objectID","weakData","meta","REQUIRED","fastKey","getWeakData","onFreeze","FREEZING","Result","stopped","AS_ENTRIES","IS_ITERATOR","iterFn","stop","require$$1","internalStateGetterFor","CONSTRUCTOR_NAME","wrapper","common","IS_WEAK","ADDER","NativePrototype","exported","anInstance","collection","iterate","IS_ADDER","getConstructor","InternalMetadataModule","setStrong","unsafe","last","define","previous","entry","getEntry","removed","redefineAll","clear","ITERATOR_NAME","getInternalCollectionState","getInternalIteratorState","setSpecies","init","collectionStrong","Set","Map","runtime","Op","hasOwn","iteratorSymbol","asyncIteratorSymbol","asyncIterator","toStringTagSymbol","toStringTag","innerFn","outerFn","tryLocsList","protoGenerator","Generator","generator","context","Context","_invoke","doneResult","delegate","delegateResult","maybeInvokeDelegate","ContinueSentinel","sent","_sent","dispatchException","abrupt","record","tryCatch","makeInvokeMethod","GeneratorFunction","GeneratorFunctionPrototype","getProto","NativeIteratorPrototype","Gp","defineIteratorMethods","AsyncIterator","PromiseImpl","previousPromise","callInvokeWithMethodAndArg","resolve","reject","invoke","__await","then","unwrapped","info","resultName","nextLoc","pushTryEntry","locs","tryLoc","catchLoc","finallyLoc","afterLoc","tryEntries","resetTryEntry","completion","reset","displayName","isGeneratorFunction","genFun","ctor","mark","awrap","async","Promise","reverse","pop","skipTempReset","rootRecord","rval","exception","handle","loc","caught","hasCatch","hasFinally","finallyEntry","complete","finish","thrown","delegateYield","regeneratorRuntime","accidentalStrictMode","splice","deleteCount","insertCount","actualDeleteCount","actualStart","MSIE","scheduler","handler","timeout","boundArgs","setTimeout","setInterval","flattenIntoArray","original","sourceLen","depth","mapper","thisArg","targetIndex","sourceIndex","mapFn","flatMap","IS_RIGHT","memo","$reduce","left","right","reduce","factories","construct","argsLength","list","partArgs","FunctionPrototype","instance","_defineProperties","props","protoProps","staticProps","createNewDataPipeFrom","DataPipeUnderConstruction","SimpleDataPipe","_source","_transformers","_target","_listeners","_bindInstanceProperty","_add","remove","_remove","update","_update","_transformItems","on","off","items","_reduceInstanceProperty","transform","_name","payload","oldData","_filterInstanceProperty","_mapInstanceProperty","_flatMapInstanceProperty","ASPDateRegex","convert","isString","parse","Number","getType","typeCoerceDataSet","rawDS","idProp","_idProp","coercedDS","DataSet","fieldId","pipe","item","acc","all","getDataSet","updateOnly","getIds","dispose","util","$parseFloat","Infinity","trimmedString","parseFloatImplementation","Component","body","resized","_previousWidth","_previousHeight","repeat","count","RangeError","convertHiddenOptions","hiddenDates","_repeatInstanceProperty","dateItem","updateHiddenDates","domProps","centerContainer","range","pixelTime","startDate","endDate","runUntil","dayOffset","log","removeDuplicates","startHidden","getIsHidden","endHidden","rangeStart","rangeEnd","hidden","startToFront","endToFront","_applyRange","safeDates","stepOverHiddenDates","timeStep","previousTime","stepInHidden","currentValue","current","_end","prevValue","newValue","switchedYear","switchedMonth","switchedDay","toScreen","Core","conversion","scale","getHiddenDurationBetween","hiddenBeforeStart","getHiddenDurationBeforeStart","rangeAfterEnd","correctTimeForHidden","toTime","hiddenDuration","partialDuration","accumulatedHiddenDuration","getAccumulatedHiddenDuration","getHiddenDurationBefore","timeOffset","requiredDuration","previousPoint","snapAwayFromHidden","correctionEnabled","isHidden","Range","millisecondsPerPixelCache","rolling","deltaDifference","scaleOffset","defaultOptions","rtl","moveable","zoomable","zoomMin","zoomMax","rollingMode","follow","_this","touch","animationTimer","emitter","_onDragStart","_onDrag","_onDragEnd","_onMouseWheel","_onTouch","_onPinch","dom","rollingModeBtn","addEventListener","startRolling","setOptions","selectiveExtend","setRange","me","stopRolling","interval","rollingModeOffset","animation","center","visibility","currentTimeTimer","_setTimeout","clearTimeout","frameCallback","byUser","finalStart","finalEnd","_cancelAnimation","initStart","initEnd","easingName","easingFunction","easingFunctions","_JSON$stringify","_Object$keys","initTime","anyChanged","dragging","ease","changed","DateUtil","params","event","emit","timeoutID","clientWidth","newStart","newEnd","_parseFloat","totalHidden","previousDelta","_isInsideRange","allowDragging","root","cursor","validateDirection","delta","deltaX","deltaY","diffRange","safeStart","safeEnd","wheelDelta","detail","zoomKey","pointerDate","zoomFriction","pointer","getPointer","clientX","clientY","_pointerToDate","zoom","preventDefault","centerDate","hiddenDurationBefore","hiddenDurationAfter","centerContainerRect","getBoundingClientRect","elementRect","top","moveTo","Emitter","mixin","_callbacks","once","removeListener","removeAllListeners","removeEventListener","cb","callbacks","listeners","hasListeners","_firstTarget","propagating","hammer","_options","Manager","Hammer","PropagatingHammer","events","propagatedHandler","srcEvent","_handled","stopPropagation","srcStop","firstTarget","elem","elemHammer","_handlers","pointerType","isFirst","eventType","destroy","hammers","idx","assign$1","nextKey","modifiedHammer","noop","onTouch","inputHandler","TimeStep","minimumStep","_start","autoScale","FORMAT","deepExtend","setMinimumStep","roundToMinor","priorCurrent","showMajorLabels","enable","stepYear","stepMonth","stepDay","showWeekScale","stepHour","stepMinute","stepSecond","stepMillisecond","minorLabels","majorLabels","_moment","classNames","even","today","currentWeek","currentMonth","currentYear","_step","styleInject","css","ref","insertAt","head","getElementsByTagName","firstChild","styleSheet","cssText","createTextNode","TimeAxis","foreground","lines","majorTexts","minorTexts","lineTop","orientation","axis","showMinorLabels","maxMinorChars","timeAxis","_create","selectiveDeepExtend","background","bottom","parentChanged","_calculateCharSize","minorLabelHeight","minorCharHeight","majorLabelHeight","majorCharHeight","offsetWidth","minorLineHeight","minorLineWidth","majorLineHeight","majorLineWidth","foregroundNextSibling","nextSibling","backgroundNextSibling","_repaintLabels","backgroundVertical","_isResized","timeLabelsize","minorCharWidth","setMoment","setFormat","setScale","xNext","isMajor","showMinorGrid","prevWidth","line","xFirstMajorLabel","getCurrent","hasNext","getClassName","_repaintMinorText","getLabelMinor","_repaintMajorText","getLabelMajor","_repaintMajorLine","_repaintMinorLine","_parseInt","warnedForOverflow","leftTime","leftText","widthText","majorCharWidth","text","innerHTML","_setXY","childNodes","directionX","measureCharMinor","clientHeight","measureCharMajor","container","_exportFunctions","_bound","keydown","keyup","_keys","fromCharCode","code","down","handleEvent","up","keyCode","bound","shiftKey","bindAll","getKey","unbind","newBindings","Activator","active","overlay","_onTapOverlay","onClick","_hasParent","deactivate","keycharm","escListener","activate","addClassName","removeClassName","en","deleteSelected","nl","de","fr","es","uk","ru","pl","ja","en_EN","en_US","it_IT","it_CH","nl_NL","nl_BE","de_DE","fr_FR","fr_CA","fr_BE","es_ES","uk_UA","ru_RU","pl_PL","ja_JP","CustomTime","title","defaultLocales","customTime","eventParams","bar","drag","onMouseWheel","attachEvent","DIRECTION_ALL","hide","warned","substring","redraw","editable","marker","setAttribute","focus","_onMarkerChange","_onMarkerChanged","getScale","getStep","snap","snappedTime","setCustomTime","backgroundHorizontal","leftContainer","rightContainer","shadowTop","shadowBottom","shadowTopLeft","shadowBottomLeft","shadowTopRight","shadowBottomRight","loadingScreen","border","scrollTop","scrollTopMin","initialDrawDone","_redraw","initialRangeChangeDone","_origRedraw","throttle","itemSet","initialItemSetDrawn","queue","pinchRecognizer","getTouchAction","hammerUtil","isActive","wheelDeltaY","wheelDeltaX","HORIZONTAL_AXIS","deltaMode","preferZoom","verticalScroll","horizontalScroll","adjusted","_setScrollTop","listener","isFinal","wheelType","onmousewheel","onMouseScrollSide","_isProgramaticallyScrolled","itemAddedToTimeline","handleDragOver","getEventProperties","_indexOfInstanceProperty","dataTransfer","dropEffect","handleDrop","itemData","getData","_onAddItem","_onDropObjectOnItem","customTimes","redrawCount","timeAxis2","components","drawPoints","onRender","clickToUse","activator","_initAutoResize","component","configurator","_createConfigurator","configure","appliedOptions","setModuleOptions","setItems","setGroups","_stopAutoResize","getCustomTime","setCustomMarker","setCustomTitle","timestamp","_someInstanceProperty","_this2","getVisibleItems","getVisibleGroups","getDataRange","getRange","percentage","getWindow","newInterval","distance","setWindow","maxHeight","option","asSize","minHeight","rootOffsetWidth","offsetHeight","contentHeight","autoHeight","containerHeight","scrollbarWidth","getScrollBarWidth","leftContainerClientWidth","rightContainerClientWidth","_setDOM","_updateScrollTop","visibilityTop","visibilityBottom","contentsOverflow","DIRECTION_HORIZONTAL","longSelectPressTime","centerWidth","currentTime","setCurrentTime","getCurrentTime","autoResize","_startAutoResize","_onResize","rootOffsetHeight","lastWidth","lastHeight","watchTimer","_setInterval","clearInterval","initialScrollTop","oldScrollTop","_getScrollTop","newScrollTop","CurrentTime","showCurrentTime","alignCurrentTime","$find","SKIPS_HOLES","getRandomValues","crypto","msCrypto","rnds8","Uint8Array","rng","byteToHex","v4","buf","rnds","bth","bytesToUuid","$includes","MATCH","isRegExp","regexp","correctIsRegExpLogic","searchString","notARegExp","arrayIncludes","stringIncludes","orderByStart","orderByEnd","margin","force","shouldBailItemsRedrawFunction","shouldBail","collidingItem","jj","collision","vertical","substack","subgroup","subgroupHeight","baseTop","nostack","subgroups","isStackSubgroups","newTop","visible","stackSubgroups","otherSubgroup","collisionByTimes","stackSubgroupsWithInnerStack","subgroupItems","doSubStack","subgroupOrder","horizontal","timeOverlap","heightOverlap","ReservedGroupIds","Group","groupId","subgroupStack","subgroupStackAll","subgroupVisibility","doInnerStack","shouldBailStackItems","subgroupIndex","subgroupOrderer","isVisible","stackDirty","_disposeCallbacks","nestedGroups","showNested","heightMode","groupHeightMode","nestedInGroup","visibleItems","itemsInRange","orderedItems","byStart","byEnd","checkRangedItems","handleCheckRangedItems","setData","groupEditable","order","inner","groupTouchParams","isDragging","templateFunction","Element","isReactComponent","treeLevel","removeCssText","addCssText","markerHeight","lastMarkerHeight","redrawQueue","redrawQueueLength","dirty","displayed","fns","offsetTop","offsetLeft","timeoutOptions","onTimeout","bailOptions","relativeBailingTime","itemsSettingTime","bailTimeMs","timeoutMs","userBailFunction","bail","userContinueNotBail","didUserContinue","forceRestack","lastIsVisible","isCluster","orderedClusters","cluster","_updateItemsInRange","visibleClusters","_updateClustersInRange","getVisibleItemsGroupedBySubgroup","orderFn","visibleSubgroupsItems","_sortInstanceProperty","_updateSubGroupHeights","customOrderedItems","_shouldBailItemsRedraw","repositionX","updateProperty","labelWidth","labelHeight","repositionY","returnQueue","_this3","_didMarkerHeightChange","_calculateGroupSizeAndPosition","_isGroupVisible","_redrawItems","_updateSubgroupsSizes","_calculateHeight","_didResize","_applyGroupHeight","_updateItemsVerticalPosition","_resetSubgroups","_this4","labelSet","setParent","_addToSubgroup","orderSubgroups","_includesInstanceProperty","_checkIfVisible","subgroupId","itemEnd","initialEnd","sortArray","sortField","_spliceInstanceProperty","_removeFromSubgroup","itemIndex","removeItem","startArray","endArray","oldVisibleItems","visibleItemsLookup","lowerBound","upperBound","_checkIfVisibleWithReference","initialPosByStart","binarySearchCustom","_traceVisible","initialPosByEnd","initialPos","breakCondition","hasItems","show","oldVisibleClusters","visibleClustersLookup","oldSubgroup","newSubgroup","disposeCallback","BackgroundGroup","Item","selected","groupShowing","selectable","setSelectability","_updateEditStatus","group","_moveToGroup","changeSubgroup","updateTime","dragCenter","dragCenterItem","hammerDragCenter","_onUpdateItem","box","dragLeft","overrideItems","deleteButton","optionsLocale","hammerDeleteButton","removeFromDataSet","tooltipOnItemUpdateTime","onItemUpdateTimeTooltip","touchParams","itemIsDragging","template","itemsData","itemVisibleFrameContent","visibleFrameTemplateFunction","_getItemData","itemVisibleFrameContentElement","getElementsByClassName","visibleFrameTemplate","_contentToString","dataAttributes","attributes","removeAttribute","outerHTML","updateGroup","tooltip","BoxItem","dot","align","widthInMs","getMillisecondsPerPixel","_updateContents","_updateDataAttributes","_updateStyle","sizes","_repaintOnItemUpdateTimeTooltip","_repaintDragCenter","_repaintDeleteButton","_createDomElement","_appendDomElement","_updateDirtyDomComponents","_getDomComponentsSizes","_updateDomComponentsSizes","_repaintDomAdditionals","repositionXY","boxX","boxY","dotX","dotY","lineX","lineY","lineWidth","dotWidth","lineStyle","lineHeight","PointItem","marginLeft","marginRight","translateX","pointX","pointY","RangeItem","frame","visibleFrame","baseClassName","maxWidth","getComputedStyle","whiteSpace","_repaintDragLeft","_repaintDragRight","limitSize","contentStartPosition","contentWidth","parentWidth","boxWidth","itemsAlwaysDraggable","dragLeftItem","dragRight","dragRightItem","BackgroundItem","itemSubgroup","Popup","overflowMethod","padding","doShow","isLeft","isTop","$every","ClusterItem","modifiedOptions","_Object$assign","fitOnDoubleClick","uiItems","randomUUID","_setupRange","eventEmitter","attached","detach","attach","rangeWidth","showStipes","repositionXWithRanges","repositionXWithoutRanges","_isStipeVisible","repositionStype","lineOffsetWidth","dotOffsetWidth","lineOffset","dotOffset","lineOffsetDirection","dotOffsetDirection","minWidth","itemSetHeight","detachFromParent","_fit","stats","centers","avg","sum","ondblclick","_onDoubleClick","fitStart","fitEnd","_getFitRange","fitArgs","ClusterGenerator","groups","dataChanged","applyOnChangedLevel","oldClusters","maxItems","clusterCriteria","level","timeWindow","levelChanged","cacheLevel","_dropLevelsCache","_filterData","clusters","groupName","iMax","neighbors","clusterItems","getGroupId","_getClusterForItems","_Object$values2","currentGroupName","oldClustersLookup","itemsIds","oldClusterData","_everyInstanceProperty","clusterItem","setUiItems","titleTemplate","clusterContent","clusterOptions","createClusterItem","BACKGROUND","ItemSet","groupOrderSwap","fromGroup","toGroup","targetOrder","groupOrder","multiselect","onDropObjectOnItem","objectData","onAdd","onUpdate","onMove","onRemove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup","showTooltips","followMouse","delay","groupsData","sequentialSelection","itemListeners","senderId","_onAdd","clusterGenerator","_onUpdate","_onRemove","groupListeners","_onAddGroups","groupData","updatedGroups","nestedGroupId","updatedNestedGroup","_concatInstanceProperty","_onUpdateGroups","_onRemoveGroups","groupIds","selection","popup","popupTimer","_updateUngrouped","backgroundGroup","ALL","_onSelectItem","_onMultiSelectItem","groupHammer","_onGroupClick","_onGroupDragStart","_onGroupDrag","_onGroupDragEnd","DIRECTION_VERTICAL","_onMouseOver","_onMouseOut","_onMouseMove","markDirty","refreshItems","restackGroups","_detachAllClusters","clearPopupTimer","ids","idsToDeselect","selectedId","getItemById","unselect","select","rawVisibleItems","_findInstanceProperty","_clusterItems","_orderGroups","visibleInterval","zoomed","lastVisibleInterval","scrolled","lastRangeStart","changedStackOption","lastStack","changedStackSubgroupsOption","lastStackSubgroups","firstGroup","_firstGroup","firstMargin","nonFirstMargin","groupMargin","redrawResults","groupResized","firstGroupIndex","firstGroupId","itemId","ungrouped","oldItemsData","DataView","_order","updateData","_this5","_getType","types","_updateItem","_removeItem","_addItem","_this6","groupOptions","_this7","_orderNestedGroups","equalArray","getOrderedNestedGroups","nestedGroupIds","nestedGroup","_this8","itemFromTarget","itemProps","selectedItem","initialX","_cloneItemData","ctrlKey","metaKey","_onDragStartAddItem","baseGroupIndex","_getGroupIndex","itemsToDrag","getSelection","groupIndex","groupOffset","_this9","frameRect","groupFromTarget","newItem","containerRect","setPosition","domRootOffsetLeft","updateGroupAllowed","newGroupBase","initialStart","initial","_this10","newOffset","oldGroup","_this11","_this12","toggleGroupShowNested","nestingGroup","fullNestedGroups","nextLevel","node","classList","toggle","toggleGroupDragClassName","originalOrder","movingUp","targetGroup","draggedGroupHeight","targetGroupHeight","draggedGroup","newOrder","origOrder","draggedId","numGroups","curPos","orgOffset","slippedPosition","switchGroup","shouldBeGroup","switchGroupId","dataset","oldSelection","setSelection","newSelection","itemFromRelatedTarget","getTitle","setText","setPopupTimer","newItemData","itemGroup","lastSelectedGroup","multiselectPerGroup","_getItemRange","_item","filteredSelection","_this13","cur","itemFromElement","relatedTarget","foregroundRect","getClusters","_updateClusters","newClustersIds","clustersToUnselect","selectionChanged","selectedIdx","allOptions","errorFound","printStyle","Validator","referenceOptions","subObject","usedOptions","__any__","referenceOption","is_object","refOptionObj","__type__","checkFields","getSuggestion","message","printLocation","optionType","refOptionType","print","copyAndExtendArray","nodeType","localSearch","findInOptions","globalSearch","indexMatch","closestMatch","recursive","closestMatchPath","lowerCaseOption","op","levenshteinDistance","copyArray","matrix","bool","enabled","throttleRedraw","any","onInitialDrawComplete","loadingScreenTemplate","configureOptions","groupsDraggable","fill","endPos","htmlColors","black","navy","darkblue","mediumblue","blue","darkgreen","green","teal","darkcyan","deepskyblue","darkturquoise","mediumspringgreen","lime","springgreen","aqua","cyan","midnightblue","dodgerblue","lightseagreen","forestgreen","seagreen","darkslategray","limegreen","mediumseagreen","turquoise","royalblue","steelblue","darkslateblue","mediumturquoise","indigo","darkolivegreen","cadetblue","cornflowerblue","mediumaquamarine","dimgray","slateblue","olivedrab","slategray","lightslategray","mediumslateblue","lawngreen","chartreuse","aquamarine","maroon","purple","olive","gray","skyblue","lightskyblue","blueviolet","darkred","darkmagenta","saddlebrown","darkseagreen","lightgreen","mediumpurple","darkviolet","palegreen","darkorchid","yellowgreen","sienna","brown","darkgray","lightblue","greenyellow","paleturquoise","lightsteelblue","powderblue","firebrick","darkgoldenrod","mediumorchid","rosybrown","darkkhaki","silver","mediumvioletred","indianred","peru","chocolate","tan","lightgrey","palevioletred","thistle","orchid","goldenrod","crimson","gainsboro","plum","burlywood","lightcyan","lavender","darksalmon","violet","palegoldenrod","lightcoral","khaki","aliceblue","honeydew","azure","sandybrown","wheat","beige","whitesmoke","mintcream","ghostwhite","salmon","antiquewhite","linen","lightgoldenrodyellow","oldlace","red","fuchsia","magenta","deeppink","orangered","tomato","hotpink","coral","darkorange","lightsalmon","orange","lightpink","pink","gold","peachpuff","navajowhite","moccasin","bisque","mistyrose","blanchedalmond","papayawhip","lavenderblush","seashell","cornsilk","lemonchiffon","floralwhite","snow","yellow","lightyellow","ivory","white","ColorPicker","pixelRatio","generated","centerCoordinates","r","color","g","hueCircle","initialColor","previousColor","applied","updateCallback","closeCallback","_bindHammer","_setSize","setInitial","rgba","htmlColor","_isColorString","isValidRGB","rgbaArray","isValidRGBA","isValidHex","rgbObj","hexToRGB","alpha","_setColor","_generateHueCircle","storePrevious","_hide","_updatePicker","setColor","alert","hsv","RGBToHSV","angleConvert","PI","radius","sin","cos","colorPickerSelector","v","HSVToRGB","ctx","colorPickerCanvas","getContext","pixelRation","devicePixelRatio","webkitBackingStorePixelRatio","mozBackingStorePixelRatio","msBackingStorePixelRatio","oBackingStorePixelRatio","backingStorePixelRatio","setTransform","clearRect","putImageData","fillStyle","circle","brightnessRange","opacityRange","initialColorDiv","backgroundColor","newColorDiv","colorPickerDiv","noCanvas","fontWeight","opacityDiv","brightnessDiv","arrowDiv","onchange","_setOpacity","oninput","_setBrightness","brightnessLabel","opacityLabel","cancelButton","onclick","applyButton","_apply","saveButton","_save","loadButton","_loadLast","pinch","_moveSelector","hue","sat","rgb","sfac","fillRect","strokeStyle","stroke","getImageData","centerY","centerX","angle","atan2","sqrt","newLeft","Configurator","parentModule","defaultContainer","changedOptions","allowCreation","initialized","popupCounter","showButton","moduleOptions","domElements","popupDiv","popupLimit","popupHistory","colorPicker","_removePopup","_clean","counter","_handleObject","_makeItem","_makeHeader","_makeButton","_push","_showPopupIfNeeded","div","objectLabel","selectedValue","_makeLabel","defaultValue","popupString","popupValue","_setupPopup","generateButton","_printOptions","onmouseover","onmouseout","optionsContainer","hideTimeout","deleteTimeout","opacity","checkbox","checked","defaultColor","_showColorPicker","insertTo","setUpdateCallback","colorString","setCloseCallback","checkOnly","visibleInSet","subObj","newPath","_getValue","_handleArray","_makeTextInput","_makeCheckbox","draw","physics","solver","enabledPath","enabledValue","_makeColorField","_makeDropdown","_makeRange","_constructOptions","optionsObj","getOptions","Timeline","itemsDone","_assertThisInitialized","SyntaxError","forthArgument","directionFromDom","domNode","parentElement","loadingScreenFragment","_toScreen","toGlobalScreen","_toGlobalScreen","_toTime","toGlobalTime","_toGlobalTime","oncontextmenu","PointerEvent","onpointerdown","onpointermove","onpointerup","onmousemove","onmousedown","onmouseup","initialFitDone","getItemRange","fit","_onFit","validate","newDataSet","startPos","initialVerticalScroll","setFinalVerticalPosition","finalVerticalScroll","getItemVerticalScroll","shouldScroll","itemTop","scrollOffset","middle","willDraw","minItem","maxItem","factor","startSide","endSide","getStart","getEnd","getWidthRight","getWidthLeft","lhs","rhs","customTimeFromTarget","getTarget","what","hasParent","pageX","pageY","timeline","itemsetHeight","currentScrollHeight","targetOffset","DataScale","autoScaleStart","autoScaleEnd","zeroAlign","formattingFunction","majorSteps","minorSteps","customLines","minorStepIdx","magnitudefactor","determineScale","rounded","minimumStepValue","orderOfMagnitude","LN10","solutionFound","majorStep","convertValue","returnValue","toPrecision","bottomOffset","major","is_major","formatValue","oldStepIdx","oldStart","oldEnd","increaseMagnitude","decreaseMagnitude","otherZero","otherStep","newRange","myOriginalZero","majorOffset","getFirstMajor","zeroOffset","pixels","DataAxis","svg","linegraphOptions","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","alignZeros","linegraphSVG","DOMelements","labels","conversionFactor","stepPixels","zeroCrossing","amountOfSteps","master","masterAxis","svgElements","iconsRemoved","amountOfGroups","_redrawLabels","framework","lineContainer","graphOptions","DOMutil","iconOffset","groupArray","getLegend","iconHeight","activeGroups","backgroundHorizontalOffsetWidth","_redrawGroupIcons","_cleanupIcons","_redrawTitle","customRange","followScale","maxLabelSize","getLines","_redrawLabel","_redrawLine","titleWidth","titleCharHeight","screenToValue","characterHeight","textAlign","largestWidth","textMinor","textMajor","textTitle","measureCharTitle","titleCharWidth","Points","getGroupTemplate","callbackResult","Bargraph","Line","GraphGroup","groupsUsingDefaultStyles","selectiveBridgeObject","usingDefaultStyle","zeroPosition","Legend","side","iconSize","iconSpacing","getCallback","screen_x","screen_y","drawIcon","fillHeight","outline","barWidth","barChart","bar1Height","bar2Height","processedGroupData","coreDistance","drawData","combinedData","intersections","barPoints","screen_end","_getDataIntersections","heightOffset","_getSafeDrawData","amount","resolved","excludeFromStacking","accumulatedNegative","accumulatedPositive","sideBySide","dataWidth","pointData","getStackedYRange","groupRanges","groupLabel","_getStackedYRange","yAxisOrientation","yMin","yMax","xpos","calcPath","interpolation","_catmullRom","_linear","fillPath","shaded","drawShading","pathArray","subPathArray","dFill","svgHeight","zero","serializePath","inverse","_catmullRomUniform","p0","bp1","bp2","d1","d2","d3","N","d3powA","d2powA","d3pow2A","d2pow2A","d1pow2A","d1powA","insertSort","getItems","setZeroPosition","mergeOptions","parametrization","Lines","Bars","icon","getYRange","addGroup","excludeFromLegend","removeGroup","textArea","scrollableHeight","drawLegendIcons","paddingTop","LineGraph","defaultGroup","sampling","graphHeight","dataAxis","legend","abortedGraphUpdate","updateSVGheight","updateSVGheightOnResize","forceGraphUpdate","lastStart","yAxisLeft","yAxisRight","legendLeft","legendRight","_removeGroup","_updateAllGroupData","_updateGroup","groupsContent","idMap","groupCounts","existingItemsMap","existing_items","newLength","extended","bridgeObject","orginalY","_updateGraph","_getSortedGroupIds","grouplist","zIndex","az","bz","minDate","maxDate","_getRelevantData","_applySampling","_convertXcoordinates","_getYRanges","_updateYAxis","below","_stack","_convertYcoordinates","paths","subGroupId","subData","dx","dy","subPrevPoint","subNextPoint","dateComparator","binarySearchValue","dataContainer","increment","amountOfPoints","pointsPerPixel","sampledData","combinedDataLeft","combinedDataRight","minVal","maxVal","yAxisLeftUsed","yAxisRightUsed","minLeft","minRight","maxLeft","maxRight","ignore","_toggleAxisVisiblity","drawIcons","tempGroups","axisUsed","datapoints","Graph2d","linegraph","initialLoad","isGroupVisible","getAbsoluteLeft","getAbsoluteTop","defaultLanguage"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;8jBAGmEA,UAG1D,eAEDC,EA6GAC,WA3GKC,WACEF,EAAaG,MAAM,KAAMC,oBAS3BC,EAAQC,UACNA,aAAiBC,OAAmD,mBAA1CC,OAAOC,UAAUC,SAASC,KAAKL,YAG3DM,EAASN,UAGE,MAATA,GAA2D,oBAA1CE,OAAOC,UAAUC,SAASC,KAAKL,YAiBlDO,EAAYP,eACA,IAAVA,WAGFQ,EAASR,SACU,iBAAVA,GAAgE,oBAA1CE,OAAOC,UAAUC,SAASC,KAAKL,YAG9DS,EAAOT,UACLA,aAAiBU,MAAkD,kBAA1CR,OAAOC,UAAUC,SAASC,KAAKL,YAG1DW,EAAIC,EAAKC,OACAC,EAAVC,EAAM,OACLD,EAAI,EAAGA,EAAIF,EAAII,SAAUF,EAC1BC,EAAIE,KAAKJ,EAAGD,EAAIE,GAAIA,WAEjBC,WAGFG,EAAWC,EAAGC,UACZlB,OAAOC,UAAUkB,eAAehB,KAAKc,EAAGC,YAG1CE,EAAOH,EAAGC,OACV,IAAIN,KAAKM,EACNF,EAAWE,EAAGN,KACdK,EAAEL,GAAKM,EAAEN,WAIbI,EAAWE,EAAG,cACdD,EAAEf,SAAWgB,EAAEhB,UAGfc,EAAWE,EAAG,aACdD,EAAEI,QAAUH,EAAEG,SAGXJ,WAGFK,EAAWxB,EAAOyB,EAAQC,EAAQC,UAChCC,GAAiB5B,EAAOyB,EAAQC,EAAQC,GAAQ,GAAME,eAuBxDC,EAAgBC,UACR,MAATA,EAAEC,MACFD,EAAEC,IApBC,CACHC,OAAkB,EAClBC,aAAkB,GAClBC,YAAkB,GAClBC,UAAmB,EACnBC,cAAkB,EAClBC,WAAkB,EAClBC,aAAkB,KAClBC,eAAkB,EAClBC,iBAAkB,EAClBC,KAAkB,EAClBC,gBAAkB,GAClBC,SAAkB,KAClBC,SAAkB,EAClBC,iBAAkB,IAQff,EAAEC,aAqBJe,EAAQhB,MACK,MAAdA,EAAEiB,SAAkB,KAChBC,EAAQnB,EAAgBC,GACxBmB,EAAcvD,EAAKU,KAAK4C,EAAMN,iBAAiB,SAAU7B,UAC7C,MAALA,KAEPqC,GAAcC,MAAMrB,EAAEsB,GAAGC,YACzBL,EAAMb,SAAW,IAChBa,EAAMhB,QACNgB,EAAMV,eACNU,EAAMM,iBACNN,EAAMH,kBACNG,EAAMX,YACNW,EAAMT,gBACNS,EAAMR,mBACLQ,EAAML,UAAaK,EAAML,UAAYM,MAEvCnB,EAAEyB,UACFL,EAAaA,GACe,IAAxBF,EAAMZ,eACwB,IAA9BY,EAAMf,aAAalB,aACDyC,IAAlBR,EAAMS,SAGS,MAAnBxD,OAAOyD,UAAqBzD,OAAOyD,SAAS5B,UAIrCoB,EAHPpB,EAAEiB,SAAWG,SAMdpB,EAAEiB,kBAGJY,EAAeX,OAChBlB,EAAIP,EAAUqC,YACL,MAATZ,EACA3B,EAAOQ,EAAgBC,GAAIkB,GAG3BnB,EAAgBC,GAAGU,iBAAkB,EAGlCV,EA3DPpC,EADAM,MAAME,UAAUR,KACTM,MAAME,UAAUR,KAEhB,SAAUmE,WACTC,EAAI7D,OAAO8D,MACXC,EAAMF,EAAE/C,SAAW,EAEdF,EAAI,EAAGA,EAAImD,EAAKnD,OACjBA,KAAKiD,GAAKD,EAAIzD,KAAK2D,KAAMD,EAAEjD,GAAIA,EAAGiD,UAC3B,SAIR,OAoDXG,EAAmBtE,EAAMsE,iBAAmB,YAEvCC,EAAWC,EAAIC,OAChBvD,EAAGwD,EAAMC,KAERhE,EAAY8D,EAAKG,oBAClBJ,EAAGI,iBAAmBH,EAAKG,kBAE1BjE,EAAY8D,EAAKI,MAClBL,EAAGK,GAAKJ,EAAKI,IAEZlE,EAAY8D,EAAKK,MAClBN,EAAGM,GAAKL,EAAKK,IAEZnE,EAAY8D,EAAKM,MAClBP,EAAGO,GAAKN,EAAKM,IAEZpE,EAAY8D,EAAKb,WAClBY,EAAGZ,QAAUa,EAAKb,SAEjBjD,EAAY8D,EAAKO,QAClBR,EAAGQ,KAAOP,EAAKO,MAEdrE,EAAY8D,EAAKQ,UAClBT,EAAGS,OAASR,EAAKQ,QAEhBtE,EAAY8D,EAAKS,WAClBV,EAAGU,QAAUT,EAAKS,SAEjBvE,EAAY8D,EAAKrC,OAClBoC,EAAGpC,IAAMF,EAAgBuC,IAExB9D,EAAY8D,EAAKU,WAClBX,EAAGW,QAAUV,EAAKU,SAGlBb,EAAiBlD,OAAS,MACrBF,EAAI,EAAGA,EAAIoD,EAAiBlD,OAAQF,IAGhCP,EADLgE,EAAMF,EADNC,EAAOJ,EAAiBpD,OAGpBsD,EAAGE,GAAQC,UAKhBH,MAGPY,GAAmB,WAGdC,EAAOC,GACZf,EAAWH,KAAMkB,QACZ7B,GAAK,IAAI3C,KAAkB,MAAbwE,EAAO7B,GAAa6B,EAAO7B,GAAGC,UAAYO,KACxDG,KAAKjB,iBACDM,GAAK,IAAI3C,KAAKmD,OAIE,IAArBmB,IACAA,GAAmB,EACnBpF,EAAMuF,aAAanB,MACnBgB,GAAmB,YAIlBI,EAAUC,UACRA,aAAeJ,GAAkB,MAAPI,GAAuC,MAAxBA,EAAIb,0BAG/Cc,EAAUC,UACXA,EAAS,EAEFC,KAAKC,KAAKF,IAAW,EAErBC,KAAKE,MAAMH,YAIjBI,EAAMC,OACPC,GAAiBD,EACjBE,EAAQ,SAEU,IAAlBD,GAAuBE,SAASF,KAChCC,EAAQR,EAASO,IAGdC,WAIFE,EAAcC,EAAQC,EAAQC,OAI/BrF,EAHAmD,EAAMuB,KAAKY,IAAIH,EAAOjF,OAAQkF,EAAOlF,QACrCqF,EAAab,KAAKc,IAAIL,EAAOjF,OAASkF,EAAOlF,QAC7CuF,EAAQ,MAEPzF,EAAI,EAAGA,EAAImD,EAAKnD,KACZqF,GAAeF,EAAOnF,KAAOoF,EAAOpF,KACnCqF,GAAeR,EAAMM,EAAOnF,MAAQ6E,EAAMO,EAAOpF,MACnDyF,WAGDA,EAAQF,WAGVG,EAAKC,IACgC,IAAtC7G,EAAM8G,6BACmB,oBAAbC,SAA6BA,QAAQH,MACjDG,QAAQH,KAAK,wBAA0BC,YAItCG,EAAUH,EAAK5F,OAChBgG,GAAY,SAETvF,GAAO,cACsB,MAA5B1B,EAAMkH,oBACNlH,EAAMkH,mBAAmB,KAAML,GAE/BI,EAAW,SAEPE,EADAC,EAAO,GAEFlG,EAAI,EAAGA,EAAIhB,UAAUkB,OAAQF,IAAK,IACvCiG,EAAM,GACsB,iBAAjBjH,UAAUgB,GAAiB,KAE7B,IAAImG,KADTF,GAAO,MAAQjG,EAAI,KACHhB,UAAU,GACtBiH,GAAOE,EAAM,KAAOnH,UAAU,GAAGmH,GAAO,KAE5CF,EAAMA,EAAIG,MAAM,GAAI,QAEpBH,EAAMjH,UAAUgB,GAEpBkG,EAAK/F,KAAK8F,GAEdP,EAAKC,EAAM,gBAAkBxG,MAAME,UAAU+G,MAAM7G,KAAK2G,GAAMG,KAAK,IAAM,MAAQ,IAAIC,OAASC,OAC9FR,GAAY,SAEThG,EAAGhB,MAAMmE,KAAMlE,aACvBe,OAyEHyG,EAtEAC,EAAe,YAEVC,EAAgBC,EAAMhB,GACK,MAA5B7G,EAAMkH,oBACNlH,EAAMkH,mBAAmBW,EAAMhB,GAE9Bc,EAAaE,KACdjB,EAAKC,GACLc,EAAaE,IAAQ,YAOpBC,EAAW1H,UACTA,aAAiB2H,UAAsD,sBAA1CzH,OAAOC,UAAUC,SAASC,KAAKL,YAsB9D4H,EAAaC,EAAcC,OACIxD,EAAhCvD,EAAMO,EAAO,GAAIuG,OAChBvD,KAAQwD,EACL5G,EAAW4G,EAAaxD,KACpBhE,EAASuH,EAAavD,KAAUhE,EAASwH,EAAYxD,KACrDvD,EAAIuD,GAAQ,GACZhD,EAAOP,EAAIuD,GAAOuD,EAAavD,IAC/BhD,EAAOP,EAAIuD,GAAOwD,EAAYxD,KACF,MAArBwD,EAAYxD,GACnBvD,EAAIuD,GAAQwD,EAAYxD,UAEjBvD,EAAIuD,QAIlBA,KAAQuD,EACL3G,EAAW2G,EAAcvD,KACpBpD,EAAW4G,EAAaxD,IACzBhE,EAASuH,EAAavD,MAE1BvD,EAAIuD,GAAQhD,EAAO,GAAIP,EAAIuD,YAG5BvD,WAGFgH,EAAO7C,GACE,MAAVA,QACK8C,IAAI9C,GAtDjBtF,EAAM8G,6BAA8B,EACpC9G,EAAMkH,mBAAqB,KA4DvBQ,EADApH,OAAOoH,KACApH,OAAOoH,KAEP,SAAUjC,OACTvE,EAAGC,EAAM,OACRD,KAAKuE,EACFnE,EAAWmE,EAAKvE,IAChBC,EAAIE,KAAKH,UAGVC,OAoFXkH,EAAU,YAELC,EAAcC,EAAMC,OACrBC,EAAYF,EAAKG,cACrBL,EAAQI,GAAaJ,EAAQI,EAAY,KAAOJ,EAAQG,GAAaD,WAGhEI,EAAeC,SACI,iBAAVA,EAAqBP,EAAQO,IAAUP,EAAQO,EAAMF,oBAAiB7E,WAG/EgF,EAAqBC,OAEtBC,EACArE,EAFAsE,EAAkB,OAIjBtE,KAAQoE,EACLxH,EAAWwH,EAAapE,KACxBqE,EAAiBJ,EAAejE,MAE5BsE,EAAgBD,GAAkBD,EAAYpE,WAKnDsE,MAGPC,EAAa,YAERC,EAAgBX,EAAMY,GAC3BF,EAAWV,GAAQY,WAcdC,EAASzD,EAAQ0D,EAAcC,OAChCC,EAAY,GAAK3D,KAAKc,IAAIf,GAC1B6D,EAAcH,EAAeE,EAAUnI,cAChCuE,GAAU,EACL2D,EAAY,IAAM,GAAM,KACpC1D,KAAK6D,IAAI,GAAI7D,KAAK8D,IAAI,EAAGF,IAAchJ,WAAWmJ,OAAO,GAAKJ,MAGlEK,EAAmB,uLAEnBC,EAAwB,6CAExBC,EAAkB,GAElBC,EAAuB,YAMlBC,EAAgBC,EAAOC,EAAQC,EAASC,OACzCC,EAAOD,EACa,iBAAbA,IACPC,EAAO,kBACIjG,KAAKgG,OAGhBH,IACAF,EAAqBE,GAASI,GAE9BH,IACAH,EAAqBG,EAAO,IAAM,kBACvBd,EAASiB,EAAKpK,MAAMmE,KAAMlE,WAAYgK,EAAO,GAAIA,EAAO,MAGnEC,IACAJ,EAAqBI,GAAW,kBACrB/F,KAAKkG,aAAaH,QAAQE,EAAKpK,MAAMmE,KAAMlE,WAAY+J,cAiCjEM,EAAapI,EAAGN,UAChBM,EAAEgB,WAIPtB,EAAS2I,EAAa3I,EAAQM,EAAEmI,cAChCR,EAAgBjI,GAAUiI,EAAgBjI,aA3BlBA,OACoBX,EAAGE,EARnBhB,EAQxBqK,EAAQ5I,EAAO6I,MAAMd,OAEpB1I,EAAI,EAAGE,EAASqJ,EAAMrJ,OAAQF,EAAIE,EAAQF,IACvC6I,EAAqBU,EAAMvJ,IAC3BuJ,EAAMvJ,GAAK6I,EAAqBU,EAAMvJ,IAEtCuJ,EAAMvJ,IAdcd,EAccqK,EAAMvJ,IAbtCwJ,MAAM,YACLtK,EAAMuK,QAAQ,WAAY,IAE9BvK,EAAMuK,QAAQ,MAAO,WAcrB,SAAUC,OACI1J,EAAb2J,EAAS,OACR3J,EAAI,EAAGA,EAAIE,EAAQF,IACpB2J,GAAU/C,EAAW2C,EAAMvJ,IAAMuJ,EAAMvJ,GAAGT,KAAKmK,EAAK/I,GAAU4I,EAAMvJ,UAEjE2J,GAW0CC,CAAmBjJ,GAEjEiI,EAAgBjI,GAAQM,IANpBA,EAAEmI,aAAaS,uBASrBP,EAAa3I,EAAQC,OACtBZ,EAAI,WAEC8J,EAA4B5K,UAC1B0B,EAAOmJ,eAAe7K,IAAUA,MAG3CyJ,EAAsBqB,UAAY,EAC3BhK,GAAK,GAAK2I,EAAsBsB,KAAKtJ,IACxCA,EAASA,EAAO8I,QAAQd,EAAuBmB,GAC/CnB,EAAsBqB,UAAY,EAClChK,GAAK,SAGFW,MAGPuJ,EAAiB,KACjBC,EAAiB,OACjBC,EAAiB,QACjBC,EAAiB,QACjBC,EAAiB,aACjBC,EAAiB,QACjBC,EAAiB,YACjBC,GAAiB,gBACjBC,GAAiB,UACjBC,GAAiB,UACjBC,GAAiB,eAEjBC,GAAiB,MACjBC,GAAiB,WAEjBC,GAAiB,qBACjBC,GAAmB,0BAMnBC,GAAY,wJAEZC,GAAU,YAELC,GAAepC,EAAOqC,EAAOC,GAClCH,GAAQnC,GAASnC,EAAWwE,GAASA,EAAQ,SAAUE,EAAUlC,UACrDkC,GAAYD,EAAeA,EAAcD,YAIhDG,GAAuBxC,EAAO3E,UAC9BhE,EAAW8K,GAASnC,GAIlBmC,GAAQnC,GAAO3E,EAAO1B,QAAS0B,EAAOH,SAHlC,IAAIuH,OAQRC,GAR8B1C,EAQhBU,QAAQ,KAAM,IAAIA,QAAQ,uCAAuC,SAAUiC,EAASC,EAAIC,EAAIC,EAAIC,UAC1GH,GAAMC,GAAMC,GAAMC,gBAIxBL,GAAYM,UACVA,EAAEtC,QAAQ,yBAA0B,YAG3CuC,GAAS,YAEJC,GAAelD,EAAOG,OACvBlJ,EAAGmJ,EAAOD,MACO,iBAAVH,IACPA,EAAQ,CAACA,IAETrJ,EAASwJ,KACTC,EAAO,SAAUjK,EAAOqK,GACpBA,EAAML,GAAYrE,EAAM3F,KAG3Bc,EAAI,EAAGA,EAAI+I,EAAM7I,OAAQF,IAC1BgM,GAAOjD,EAAM/I,IAAMmJ,WAIlB+C,GAAmBnD,EAAOG,GAC/B+C,GAAclD,GAAO,SAAU7J,EAAOqK,EAAOnF,EAAQ2E,GACjD3E,EAAO+H,GAAK/H,EAAO+H,IAAM,GACzBjD,EAAShK,EAAOkF,EAAO+H,GAAI/H,EAAQ2E,eAIlCqD,GAAwBrD,EAAO7J,EAAOkF,GAC9B,MAATlF,GAAiBkB,EAAW4L,GAAQjD,IACpCiD,GAAOjD,GAAO7J,EAAOkF,EAAOiI,GAAIjI,EAAQ2E,YA0DvCuD,GAAWC,UACTC,GAAWD,GAAQ,IAAM,aAG3BC,GAAWD,UACRA,EAAO,GAAM,GAAKA,EAAO,KAAQ,GAAMA,EAAO,KAAQ,EA/ClEzD,EAAe,IAAK,EAAG,GAAG,eAClB2D,EAAIvJ,KAAKqJ,cACNE,GAAK,KAAO,GAAKA,EAAI,IAAMA,KAGtC3D,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAKqJ,OAAS,OAGzBzD,EAAe,EAAG,CAAC,OAAU,GAAU,EAAG,QAC1CA,EAAe,EAAG,CAAC,QAAU,GAAU,EAAG,QAC1CA,EAAe,EAAG,CAAC,SAAU,GAAG,GAAO,EAAG,QAI1C1B,EAAa,OAAQ,KAIrBY,EAAgB,OAAQ,GAIxBmD,GAAc,IAAUL,IACxBK,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,OAAUR,GAAWN,GACnCc,GAAc,QAAUP,GAAWN,GACnCa,GAAc,SAAUP,GAAWN,GAEnC2B,GAAc,CAAC,QAAS,UAzCb,GA0CXA,GAAc,QAAQ,SAAU/M,EAAOqK,GACnCA,EA3CO,GA2CwB,IAAjBrK,EAAMgB,OAAepB,EAAM4N,kBAAkBxN,GAAS2F,EAAM3F,MAE9E+M,GAAc,MAAM,SAAU/M,EAAOqK,GACjCA,EA9CO,GA8COzK,EAAM4N,kBAAkBxN,MAE1C+M,GAAc,KAAK,SAAU/M,EAAOqK,GAChCA,EAjDO,GAiDOoD,SAASzN,EAAO,OAelCJ,EAAM4N,kBAAoB,SAAUxN,UACzB2F,EAAM3F,IAAU2F,EAAM3F,GAAS,GAAK,KAAO,UAsElD0N,GAjEAC,GAAaC,GAAW,YAAY,YAM/BA,GAAYzF,EAAM0F,UAChB,SAAU/H,UACA,MAATA,GACAgI,GAAM9J,KAAMmE,EAAMrC,GAClBlG,EAAMuF,aAAanB,KAAM6J,GAClB7J,MAEA+J,GAAI/J,KAAMmE,aAKpB4F,GAAKvD,EAAKrC,UACRqC,EAAIzH,UACPyH,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,KAAUtE,aAGpDiK,GAAOtD,EAAKrC,EAAMrC,GACnB0E,EAAIzH,YAAcK,MAAM0C,KACX,aAATqC,GAAuBmF,GAAW9C,EAAI6C,SAA2B,IAAhB7C,EAAIwD,SAAgC,KAAfxD,EAAIyD,OAC1EzD,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,GAAMrC,EAAO0E,EAAIwD,QAASE,GAAYpI,EAAO0E,EAAIwD,UAG5FxD,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,GAAMrC,aAqDpDoI,GAAYb,EAAMW,MACnB5K,MAAMiK,IAASjK,MAAM4K,UACdnK,QAvBCsK,EAyBRC,GAAeJ,GAzBPG,EAyBc,IAxBRA,GAAKA,SAyBvBd,IAASW,EAAQI,GAAY,GACT,IAAbA,EAAkBd,GAAWD,GAAQ,GAAK,GAAO,GAAKe,EAAW,EAAI,EApB5EV,GADAzN,MAAME,UAAUuN,QACNzN,MAAME,UAAUuN,QAEhB,SAAUW,OAEZvN,MACCA,EAAI,EAAGA,EAAIkD,KAAKhD,SAAUF,KACvBkD,KAAKlD,KAAOuN,SACLvN,SAGP,GAehB8I,EAAe,IAAK,CAAC,KAAM,GAAI,MAAM,kBAC1B5F,KAAKgK,QAAU,KAG1BpE,EAAe,MAAO,EAAG,GAAG,SAAUnI,UAC3BuC,KAAKkG,aAAaoE,YAAYtK,KAAMvC,MAG/CmI,EAAe,OAAQ,EAAG,GAAG,SAAUnI,UAC5BuC,KAAKkG,aAAaqE,OAAOvK,KAAMvC,MAK1CyG,EAAa,QAAS,KAItBY,EAAgB,QAAS,GAIzBmD,GAAc,IAAQZ,GACtBY,GAAc,KAAQZ,EAAWJ,GACjCgB,GAAc,OAAQ,SAAUG,EAAU1K,UAC/BA,EAAO8M,iBAAiBpC,MAEnCH,GAAc,QAAQ,SAAUG,EAAU1K,UAC/BA,EAAO+M,YAAYrC,MAG9BW,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,GACxCA,EAlMQ,GAkMO1E,EAAM3F,GAAS,KAGlC+M,GAAc,CAAC,MAAO,SAAS,SAAU/M,EAAOqK,EAAOnF,EAAQ2E,OACvDmE,EAAQ9I,EAAOH,QAAQ2J,YAAY1O,EAAO6J,EAAO3E,EAAO1B,SAE/C,MAATwK,EACA3D,EAzMI,GAyMW2D,EAEflM,EAAgBoD,GAAQ3C,aAAevC,SAM3C2O,GAAmB,gCACnBC,GAAsB,wFAAwFC,MAAM,KAUpHC,GAA2B,kDAAkDD,MAAM,cAU9EE,GAAkBC,EAAWvN,EAAQE,OACtCb,EAAGmO,EAAIzE,EAAK0E,EAAMF,EAAUG,wBAC3BnL,KAAKoL,sBAEDA,aAAe,QACfC,iBAAmB,QACnBC,kBAAoB,GACpBxO,EAAI,EAAGA,EAAI,KAAMA,EAClB0J,EAAMhJ,EAAU,CAAC,IAAMV,SAClBwO,kBAAkBxO,GAAKkD,KAAKsK,YAAY9D,EAAK,IAAI2E,yBACjDE,iBAAiBvO,GAAKkD,KAAKuK,OAAO/D,EAAK,IAAI2E,2BAIpDxN,EACe,QAAXF,GAEe,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,IACvBD,EAAK,MAGT,KADfA,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,IACtBD,EAAK,KAGb,QAAXxN,GAEY,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,MAK3B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,IAF9BD,EAGa,MAGZ,KADZA,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,MAK1B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,IAF/BD,EAGa,cA6C3BM,GAAU/E,EAAK1E,OAChB0J,MAEChF,EAAIzH,iBAEEyH,KAGU,iBAAV1E,KACH,QAAQiF,KAAKjF,GACbA,EAAQH,EAAMG,YAITtF,EAFLsF,EAAQ0E,EAAIN,aAAawE,YAAY5I,WAG1B0E,SAKnBgF,EAAahK,KAAKY,IAAIoE,EAAIyD,OAAQC,GAAY1D,EAAI6C,OAAQvH,IAC1D0E,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAM,SAASiB,EAAO0J,GACpDhF,WAGFiF,GAAa3J,UACL,MAATA,GACAyJ,GAASvL,KAAM8B,GACflG,EAAMuF,aAAanB,MAAM,GAClBA,MAEA+J,GAAI/J,KAAM,aAQrB0L,GAA0B3D,GAoB1B4D,GAAqB5D,YAoBhB6D,cACIC,EAAU1O,EAAGC,UACXA,EAAEJ,OAASG,EAAEH,WAIpBF,EAAG0J,EADHsF,EAAc,GAAIC,EAAa,GAAIC,EAAc,OAEhDlP,EAAI,EAAGA,EAAI,GAAIA,IAEhB0J,EAAMhJ,EAAU,CAAC,IAAMV,IACvBgP,EAAY7O,KAAK+C,KAAKsK,YAAY9D,EAAK,KACvCuF,EAAW9O,KAAK+C,KAAKuK,OAAO/D,EAAK,KACjCwF,EAAY/O,KAAK+C,KAAKuK,OAAO/D,EAAK,KAClCwF,EAAY/O,KAAK+C,KAAKsK,YAAY9D,EAAK,SAI3CsF,EAAYG,KAAKJ,GACjBE,EAAWE,KAAKJ,GAChBG,EAAYC,KAAKJ,GACZ/O,EAAI,EAAGA,EAAI,GAAIA,IAChBgP,EAAYhP,GAAKyL,GAAYuD,EAAYhP,IACzCiP,EAAWjP,GAAKyL,GAAYwD,EAAWjP,QAEtCA,EAAI,EAAGA,EAAI,GAAIA,IAChBkP,EAAYlP,GAAKyL,GAAYyD,EAAYlP,SAGxCoP,aAAe,IAAI5D,OAAO,KAAO0D,EAAY7I,KAAK,KAAO,IAAK,UAC9DgJ,kBAAoBnM,KAAKkM,kBACzBE,mBAAqB,IAAI9D,OAAO,KAAOyD,EAAW5I,KAAK,KAAO,IAAK,UACnEkJ,wBAA0B,IAAI/D,OAAO,KAAOwD,EAAY3I,KAAK,KAAO,IAAK,cAGzEmJ,GAAY/C,EAAGxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,OAG/BzC,SAEAV,EAAI,KAAOA,GAAK,GAEhBU,EAAO,IAAIvN,KAAK6M,EAAI,IAAKxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,GACpC3K,SAASkI,EAAK0C,gBACd1C,EAAK2C,YAAYrD,IAGrBU,EAAO,IAAIvN,KAAK6M,EAAGxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,GAG/BzC,WAGF4C,GAAetD,OAChBU,KAEAV,EAAI,KAAOA,GAAK,EAAG,KACfvG,EAAO/G,MAAME,UAAU+G,MAAM7G,KAAKP,WAEtCkH,EAAK,GAAKuG,EAAI,IACdU,EAAO,IAAIvN,KAAKA,KAAKoQ,IAAIjR,MAAM,KAAMmH,IACjCjB,SAASkI,EAAK8C,mBACd9C,EAAK+C,eAAezD,QAGxBU,EAAO,IAAIvN,KAAKA,KAAKoQ,IAAIjR,MAAM,KAAMC,mBAGlCmO,WAIFgD,GAAgB5D,EAAM6D,EAAKC,OAE5BC,EAAM,EAAIF,EAAMC,UAEP,EAAIN,GAAcxD,EAAM,EAAG+D,GAAKC,YAAcH,GAAO,EAElDE,EAAM,WAIjBE,GAAmBjE,EAAMkE,EAAMC,EAASN,EAAKC,OAI9CM,EAASC,EADTC,EAAY,EAAI,GAAKJ,EAAO,IAFZ,EAAIC,EAAUN,GAAO,EACxBD,GAAgB5D,EAAM6D,EAAKC,UAIxCQ,GAAa,EAEbD,EAAetE,GADfqE,EAAUpE,EAAO,GACoBsE,EAC9BA,EAAYvE,GAAWC,IAC9BoE,EAAUpE,EAAO,EACjBqE,EAAeC,EAAYvE,GAAWC,KAEtCoE,EAAUpE,EACVqE,EAAeC,GAGZ,CACHtE,KAAMoE,EACNE,UAAWD,YAIVE,GAAWpH,EAAK0G,EAAKC,OAGtBU,EAASJ,EAFTK,EAAab,GAAgBzG,EAAI6C,OAAQ6D,EAAKC,GAC9CI,EAAO/L,KAAKE,OAAO8E,EAAImH,YAAcG,EAAa,GAAK,GAAK,SAG5DP,EAAO,EAEPM,EAAUN,EAAOQ,GADjBN,EAAUjH,EAAI6C,OAAS,EACe6D,EAAKC,GACpCI,EAAOQ,GAAYvH,EAAI6C,OAAQ6D,EAAKC,IAC3CU,EAAUN,EAAOQ,GAAYvH,EAAI6C,OAAQ6D,EAAKC,GAC9CM,EAAUjH,EAAI6C,OAAS,IAEvBoE,EAAUjH,EAAI6C,OACdwE,EAAUN,GAGP,CACHA,KAAMM,EACNxE,KAAMoE,YAILM,GAAY1E,EAAM6D,EAAKC,OACxBW,EAAab,GAAgB5D,EAAM6D,EAAKC,GACxCa,EAAiBf,GAAgB5D,EAAO,EAAG6D,EAAKC,UAC5C/D,GAAWC,GAAQyE,EAAaE,GAAkB,WAoJrDC,GAAeC,EAAIC,UACjBD,EAAGhL,MAAMiL,EAAG,GAAGC,OAAOF,EAAGhL,MAAM,EAAGiL,IAhJ7CvI,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,QACrCA,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,WAIrC1B,EAAa,OAAQ,KACrBA,EAAa,UAAW,KAIxBY,EAAgB,OAAQ,GACxBA,EAAgB,UAAW,GAI3BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAE/B+B,GAAkB,CAAC,IAAK,KAAM,IAAK,OAAO,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GACrE0H,EAAK1H,EAAMN,OAAO,EAAG,IAAM5D,EAAM3F,MAsCrC4J,EAAe,IAAK,EAAG,KAAM,OAE7BA,EAAe,KAAM,EAAG,GAAG,SAAUnI,UAC1BuC,KAAKkG,aAAamI,YAAYrO,KAAMvC,MAG/CmI,EAAe,MAAO,EAAG,GAAG,SAAUnI,UAC3BuC,KAAKkG,aAAaoI,cAActO,KAAMvC,MAGjDmI,EAAe,OAAQ,EAAG,GAAG,SAAUnI,UAC5BuC,KAAKkG,aAAaqI,SAASvO,KAAMvC,MAG5CmI,EAAe,IAAK,EAAG,EAAG,WAC1BA,EAAe,IAAK,EAAG,EAAG,cAI1B1B,EAAa,MAAO,KACpBA,EAAa,UAAW,KACxBA,EAAa,aAAc,KAG3BY,EAAgB,MAAO,IACvBA,EAAgB,UAAW,IAC3BA,EAAgB,aAAc,IAI9BmD,GAAc,IAAQZ,GACtBY,GAAc,IAAQZ,GACtBY,GAAc,IAAQZ,GACtBY,GAAc,MAAQ,SAAUG,EAAU1K,UAC/BA,EAAO8Q,iBAAiBpG,MAEnCH,GAAc,OAAS,SAAUG,EAAU1K,UAChCA,EAAO+Q,mBAAmBrG,MAErCH,GAAc,QAAU,SAAUG,EAAU1K,UACjCA,EAAOgR,cAActG,MAGhCY,GAAkB,CAAC,KAAM,MAAO,SAAS,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,OAChE2H,EAAUtM,EAAOH,QAAQ4N,cAAc3S,EAAO6J,EAAO3E,EAAO1B,SAEjD,MAAXgO,EACAD,EAAKhB,EAAIiB,EAET1P,EAAgBoD,GAAQ3B,eAAiBvD,KAIjDgN,GAAkB,CAAC,IAAK,IAAK,MAAM,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GAC9D0H,EAAK1H,GAASlE,EAAM3F,UAkCpB4S,GAAwB,2DAA2D/D,MAAM,KAQzFgE,GAA6B,8BAA8BhE,MAAM,KAMjEiE,GAA2B,uBAAuBjE,MAAM,cAMnDkE,GAAoBC,EAAavR,EAAQE,OAC1Cb,EAAGmO,EAAIzE,EAAK0E,EAAM8D,EAAY7D,wBAC7BnL,KAAKiP,wBACDA,eAAiB,QACjBC,oBAAsB,QACtBC,kBAAoB,GAEpBrS,EAAI,EAAGA,EAAI,IAAKA,EACjB0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,QAC1BqS,kBAAkBrS,GAAKkD,KAAKqO,YAAY7H,EAAK,IAAI2E,yBACjD+D,oBAAoBpS,GAAKkD,KAAKsO,cAAc9H,EAAK,IAAI2E,yBACrD8D,eAAenS,GAAKkD,KAAKuO,SAAS/H,EAAK,IAAI2E,2BAIpDxN,EACe,SAAXF,GAEe,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,IACpBD,EAAK,KACN,QAAXxN,GAEQ,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,IACzBD,EAAK,MAGT,KADfA,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IACvBD,EAAK,KAGb,SAAXxN,GAEY,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAK3B,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,MAK7B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IAN/BD,EAOa,KACN,QAAXxN,GAEK,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,MAKhC,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAKxB,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IAN/BD,EAOa,MAGZ,KADZA,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,MAK9B,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAKxB,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,IANjCD,EAOa,SAqFhCoE,GAAuBtH,GAoBvBuH,GAA4BvH,GAoB5BwH,GAA0BxH,YAqBrByH,cACI3D,EAAU1O,EAAGC,UACXA,EAAEJ,OAASG,EAAEH,WAIpBF,EAAG0J,EAAKiJ,EAAMC,EAAQC,EADtBC,EAAY,GAAI9D,EAAc,GAAIC,EAAa,GAAIC,EAAc,OAEhElP,EAAI,EAAGA,EAAI,EAAGA,IAEf0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,GAC/B2S,EAAOzP,KAAKqO,YAAY7H,EAAK,IAC7BkJ,EAAS1P,KAAKsO,cAAc9H,EAAK,IACjCmJ,EAAQ3P,KAAKuO,SAAS/H,EAAK,IAC3BoJ,EAAU3S,KAAKwS,GACf3D,EAAY7O,KAAKyS,GACjB3D,EAAW9O,KAAK0S,GAChB3D,EAAY/O,KAAKwS,GACjBzD,EAAY/O,KAAKyS,GACjB1D,EAAY/O,KAAK0S,OAIrBC,EAAU3D,KAAKJ,GACfC,EAAYG,KAAKJ,GACjBE,EAAWE,KAAKJ,GAChBG,EAAYC,KAAKJ,GACZ/O,EAAI,EAAGA,EAAI,EAAGA,IACfgP,EAAYhP,GAAKyL,GAAYuD,EAAYhP,IACzCiP,EAAWjP,GAAKyL,GAAYwD,EAAWjP,IACvCkP,EAAYlP,GAAKyL,GAAYyD,EAAYlP,SAGxC+S,eAAiB,IAAIvH,OAAO,KAAO0D,EAAY7I,KAAK,KAAO,IAAK,UAChE2M,oBAAsB9P,KAAK6P,oBAC3BE,kBAAoB/P,KAAK6P,oBAEzBG,qBAAuB,IAAI1H,OAAO,KAAOyD,EAAW5I,KAAK,KAAO,IAAK,UACrE8M,0BAA4B,IAAI3H,OAAO,KAAOwD,EAAY3I,KAAK,KAAO,IAAK,UAC3E+M,wBAA0B,IAAI5H,OAAO,KAAOsH,EAAUzM,KAAK,KAAO,IAAK,cAKvEgN,YACEnQ,KAAKoQ,QAAU,IAAM,YA6BvBxR,GAAUiH,EAAOwK,GACtBzK,EAAeC,EAAO,EAAG,GAAG,kBACjB7F,KAAKkG,aAAatH,SAASoB,KAAKoQ,QAASpQ,KAAKsQ,UAAWD,eAgB/DE,GAAenI,EAAU1K,UACvBA,EAAO8S,eAzClB5K,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,QAClCA,EAAe,IAAK,CAAC,KAAM,GAAI,EAAGuK,IAClCvK,EAAe,IAAK,CAAC,KAAM,GAAI,qBALpB5F,KAAKoQ,SAAW,MAO3BxK,EAAe,MAAO,EAAG,GAAG,iBACjB,GAAKuK,GAAQtU,MAAMmE,MAAQgF,EAAShF,KAAKsQ,UAAW,MAG/D1K,EAAe,QAAS,EAAG,GAAG,iBACnB,GAAKuK,GAAQtU,MAAMmE,MAAQgF,EAAShF,KAAKsQ,UAAW,GACvDtL,EAAShF,KAAKyQ,UAAW,MAGjC7K,EAAe,MAAO,EAAG,GAAG,iBACjB,GAAK5F,KAAKoQ,QAAUpL,EAAShF,KAAKsQ,UAAW,MAGxD1K,EAAe,QAAS,EAAG,GAAG,iBACnB,GAAK5F,KAAKoQ,QAAUpL,EAAShF,KAAKsQ,UAAW,GAChDtL,EAAShF,KAAKyQ,UAAW,MASjC7R,GAAS,KAAK,GACdA,GAAS,KAAK,GAIdsF,EAAa,OAAQ,KAGrBY,EAAgB,OAAQ,IAQxBmD,GAAc,IAAMsI,IACpBtI,GAAc,IAAMsI,IACpBtI,GAAc,IAAMZ,GACpBY,GAAc,IAAMZ,GACpBY,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,KAAMZ,EAAWJ,GAE/BgB,GAAc,MAAOX,GACrBW,GAAc,QAASV,IACvBU,GAAc,MAAOX,GACrBW,GAAc,QAASV,IAEvBwB,GAAc,CAAC,IAAK,MA3+BT,GA4+BXA,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,OAC3CwP,EAAS/O,EAAM3F,GACnBqK,EA9+BO,GA8+BkB,KAAXqK,EAAgB,EAAIA,KAEtC3H,GAAc,CAAC,IAAK,MAAM,SAAU/M,EAAOqK,EAAOnF,GAC9CA,EAAOyP,MAAQzP,EAAOH,QAAQ6P,KAAK5U,GACnCkF,EAAO2P,UAAY7U,KAEvB+M,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,GAC/CmF,EAr/BO,GAq/BO1E,EAAM3F,GACpB8B,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,OAAO,SAAU/M,EAAOqK,EAAOnF,OACrC4P,EAAM9U,EAAMgB,OAAS,EACzBqJ,EA1/BO,GA0/BO1E,EAAM3F,EAAMuJ,OAAO,EAAGuL,IACpCzK,EA1/BS,GA0/BO1E,EAAM3F,EAAMuJ,OAAOuL,IACnChT,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,SAAS,SAAU/M,EAAOqK,EAAOnF,OACvC6P,EAAO/U,EAAMgB,OAAS,EACtBgU,EAAOhV,EAAMgB,OAAS,EAC1BqJ,EAjgCO,GAigCO1E,EAAM3F,EAAMuJ,OAAO,EAAGwL,IACpC1K,EAjgCS,GAigCO1E,EAAM3F,EAAMuJ,OAAOwL,EAAM,IACzC1K,EAjgCS,GAigCO1E,EAAM3F,EAAMuJ,OAAOyL,IACnClT,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,OAAO,SAAU/M,EAAOqK,EAAOnF,OACrC4P,EAAM9U,EAAMgB,OAAS,EACzBqJ,EAxgCO,GAwgCO1E,EAAM3F,EAAMuJ,OAAO,EAAGuL,IACpCzK,EAxgCS,GAwgCO1E,EAAM3F,EAAMuJ,OAAOuL,OAEvC/H,GAAc,SAAS,SAAU/M,EAAOqK,EAAOnF,OACvC6P,EAAO/U,EAAMgB,OAAS,EACtBgU,EAAOhV,EAAMgB,OAAS,EAC1BqJ,EA9gCO,GA8gCO1E,EAAM3F,EAAMuJ,OAAO,EAAGwL,IACpC1K,EA9gCS,GA8gCO1E,EAAM3F,EAAMuJ,OAAOwL,EAAM,IACzC1K,EA9gCS,GA8gCO1E,EAAM3F,EAAMuJ,OAAOyL,WAoDnCC,GAzBAC,GAAatH,GAAW,SAAS,GAEjCuH,GAAa,CACbC,SAj2CkB,CAClBC,QAAU,gBACVC,QAAU,mBACVC,SAAW,eACXC,QAAU,oBACVC,SAAW,sBACXC,SAAW,KA41CX7K,eAp1CwB,CACxB8K,IAAO,YACPC,GAAO,SACPC,EAAO,aACPC,GAAO,eACPC,IAAO,sBACPC,KAAO,6BA+0CPrL,YA7zCqB,eA8zCrBZ,QAxzCiB,KAyzCjBkM,uBAxzCgC,UAyzChCC,aAnzCsB,CACtBC,OAAS,QACTC,KAAS,SACTvJ,EAAK,gBACLwJ,GAAK,aACLtU,EAAK,WACLuU,GAAK,aACL9F,EAAK,UACL+F,GAAK,WACLhG,EAAK,QACLiG,GAAK,UACL/F,EAAK,UACLgG,GAAK,YACLlJ,EAAK,SACLmJ,GAAK,YAuyCLnI,OAAQK,GACRN,YAAaQ,GAEbyC,KA9gBoB,CACpBL,IAAM,EACNC,IAAM,GA8gBNoB,SAAUK,GACVP,YAAaS,GACbR,cAAeO,GAEf8D,cAnC6B,iBAuC7BC,GAAU,GACVC,GAAiB,YAGZC,GAAgB7P,UACdA,EAAMA,EAAIqB,cAAciC,QAAQ,IAAK,KAAOtD,WA8B9C8P,GAAWtP,OACZuP,EAAY,SAEXJ,GAAQnP,IACLhI,GAAUA,EAAOwX,YAEjBD,EAAY/B,GAAaiC,MACJC,IAErBC,GAAmBJ,GACrB,MAAOK,WAENT,GAAQnP,YAMV2P,GAAoBnQ,EAAKqQ,OAC1BC,SACAtQ,KAEIsQ,EADAhX,EAAY+W,GACLE,GAAUvQ,GAGVwQ,GAAaxQ,EAAKqQ,IAKzBrC,GAAesC,EAGU,oBAAb5Q,SAA6BA,QAAQH,MAE7CG,QAAQH,KAAK,UAAYS,EAAO,2CAKrCgO,GAAaiC,eAGfO,GAAchQ,EAAMvC,MACV,OAAXA,EAAiB,KACbxD,EAAQmG,EAAesN,MAC3BjQ,EAAOwS,KAAOjQ,EACO,MAAjBmP,GAAQnP,GACRD,EAAgB,uBACR,2OAIRK,EAAe+O,GAAQnP,GAAMkQ,aAC1B,GAA2B,MAAvBzS,EAAO0S,gBACsB,MAAhChB,GAAQ1R,EAAO0S,cACf/P,EAAe+O,GAAQ1R,EAAO0S,cAAcD,YACzC,IAEW,OADdjW,EAASqV,GAAW7R,EAAO0S,sBAIlBf,GAAe3R,EAAO0S,gBACvBf,GAAe3R,EAAO0S,cAAgB,IAE1Cf,GAAe3R,EAAO0S,cAAc3W,KAAK,CACrCwG,KAAMA,EACNvC,OAAQA,IAEL,KATP2C,EAAenG,EAAOiW,eAalCf,GAAQnP,GAAQ,IAAIM,EAAOH,EAAaC,EAAc3C,IAElD2R,GAAepP,IACfoP,GAAepP,GAAMoQ,SAAQ,SAAU1J,GACnCsJ,GAAatJ,EAAE1G,KAAM0G,EAAEjJ,WAO/BkS,GAAmB3P,GAGZmP,GAAQnP,iBAGRmP,GAAQnP,GACR,cAiCN+P,GAAWvQ,OACZvF,KAEAuF,GAAOA,EAAIlC,SAAWkC,EAAIlC,QAAQmS,QAClCjQ,EAAMA,EAAIlC,QAAQmS,QAGjBjQ,SACMgO,OAGNlV,EAAQkH,GAAM,IAEfvF,EAASqV,GAAW9P,UAETvF,EAEXuF,EAAM,CAACA,mBArKO6Q,WACPC,EAAGC,EAAMtW,EAAQmN,EAAxB/N,EAAI,EAEDA,EAAIgX,EAAM9W,QAAQ,KAErB+W,GADAlJ,EAAQiI,GAAgBgB,EAAMhX,IAAI+N,MAAM,MAC9B7N,OAEVgX,GADAA,EAAOlB,GAAgBgB,EAAMhX,EAAI,KACnBkX,EAAKnJ,MAAM,KAAO,KACzBkJ,EAAI,GAAG,IACVrW,EAASqV,GAAWlI,EAAM3H,MAAM,EAAG6Q,GAAG5Q,KAAK,aAEhCzF,KAEPsW,GAAQA,EAAKhX,QAAU+W,GAAK/R,EAAc6I,EAAOmJ,GAAM,IAASD,EAAI,QAIxEA,IAEJjX,WAEGmU,GAmJAgD,CAAahR,YAOfiR,GAAenW,OAChBK,EACAjB,EAAIY,EAAEoL,UAENhM,IAAsC,IAAjCW,EAAgBC,GAAGK,WACxBA,EACIjB,EApwCA,GAowCiB,GAAKA,EApwCtB,GAowCuC,GApwCvC,EAqwCAA,EApwCD,GAowCkB,GAAKA,EApwCvB,GAowCwC+M,GAAY/M,EAtwCpD,GAswC6DA,EArwC5D,IACD,EAqwCCA,EApwCD,GAowCkB,GAAKA,EApwCvB,GAowCwC,IAAmB,KAAZA,EApwC/C,KAowCgF,IAAdA,EAnwChE,IAmwCiG,IAAdA,EAlwCnF,IAkwCyH,IAAnBA,EAjwCjG,IAHP,EAqwCCA,EApwCC,GAowCgB,GAAKA,EApwCrB,GAowCsC,GApwCtC,EAqwCDA,EApwCC,GAowCgB,GAAKA,EApwCrB,GAowCsC,GApwCtC,EAqwCDA,EApwCM,GAowCW,GAAKA,EApwChB,GAowCiC,IApwCjC,GAqwCL,EAEDW,EAAgBC,GAAGoW,qBAAuB/V,EA7wC3C,GA6wC8DA,EA3wC9D,KA4wCCA,EA5wCD,GA8wCCN,EAAgBC,GAAGqW,iBAAgC,IAAdhW,IACrCA,EA1wCD,GA4wCCN,EAAgBC,GAAGsW,mBAAkC,IAAdjW,IACvCA,EA5wCE,GA+wCNN,EAAgBC,GAAGK,SAAWA,GAG3BL,WAIFuW,GAASnX,EAAGC,EAAGmX,UACX,MAALpX,EACOA,EAEF,MAALC,EACOA,EAEJmX,WAgBFC,GAAiBtT,OAClBpE,EAAGmN,EAAkBwK,EAAaC,EAAiBC,EAA1C3Y,EAAQ,OAEjBkF,EAAO7B,QAIXoV,WApBsBvT,OAElB0T,EAAW,IAAIlY,KAAKd,EAAMiZ,cAC1B3T,EAAO4T,QACA,CAACF,EAAS7H,iBAAkB6H,EAASG,cAAeH,EAASI,cAEjE,CAACJ,EAASjI,cAAeiI,EAASK,WAAYL,EAASM,WAchDC,CAAiBjU,GAG3BA,EAAO+H,IAAyB,MAAnB/H,EAAOiI,GA7zCjB,IA6zCyD,MAApBjI,EAAOiI,GA9zC3C,aAy3CmBjI,OACvBkU,EAAGC,EAAU9H,EAAMC,EAASN,EAAKC,EAAKmI,EAAMC,KAGpC,OADZH,EAAIlU,EAAO+H,IACLuM,IAAqB,MAAPJ,EAAEK,GAAoB,MAAPL,EAAEM,EACjCxI,EAAM,EACNC,EAAM,EAMNkI,EAAWf,GAASc,EAAEI,GAAItU,EAAOiI,GAt4C9B,GAs4CwCyE,GAAW+H,KAAe,EAAG,GAAGtM,MAC3EkE,EAAO+G,GAASc,EAAEK,EAAG,KACrBjI,EAAU8G,GAASc,EAAEM,EAAG,IACV,GAAKlI,EAAU,KACzB+H,GAAkB,OAEnB,CACHrI,EAAMhM,EAAOH,QAAQ6U,MAAM1I,IAC3BC,EAAMjM,EAAOH,QAAQ6U,MAAMzI,QAEvB0I,EAAUjI,GAAW+H,KAAezI,EAAKC,GAE7CkI,EAAWf,GAASc,EAAEU,GAAI5U,EAAOiI,GAl5C9B,GAk5CwC0M,EAAQxM,MAGnDkE,EAAO+G,GAASc,EAAEA,EAAGS,EAAQtI,MAElB,MAAP6H,EAAE7I,IAEFiB,EAAU4H,EAAE7I,GACE,GAAKiB,EAAU,KACzB+H,GAAkB,GAER,MAAPH,EAAE/B,GAET7F,EAAU4H,EAAE/B,EAAInG,GACZkI,EAAE/B,EAAI,GAAK+B,EAAE/B,EAAI,KACjBkC,GAAkB,IAItB/H,EAAUN,EAGdK,EAAO,GAAKA,EAAOQ,GAAYsH,EAAUnI,EAAKC,GAC9CrP,EAAgBoD,GAAQkT,gBAAiB,EACf,MAAnBmB,EACPzX,EAAgBoD,GAAQmT,kBAAmB,GAE3CiB,EAAOhI,GAAmB+H,EAAU9H,EAAMC,EAASN,EAAKC,GACxDjM,EAAOiI,GA96CJ,GA86CemM,EAAKjM,KACvBnI,EAAO6U,WAAaT,EAAK3H,WA/GzBqI,CAAsB9U,GAID,MAArBA,EAAO6U,aACPpB,EAAYL,GAASpT,EAAOiI,GAr0CzB,GAq0CmCsL,EAr0CnC,KAu0CCvT,EAAO6U,WAAa3M,GAAWuL,IAAoC,IAAtBzT,EAAO6U,cACpDjY,EAAgBoD,GAAQiT,oBAAqB,GAGjDlK,EAAO4C,GAAc8H,EAAW,EAAGzT,EAAO6U,YAC1C7U,EAAOiI,GA30CH,GA20Cec,EAAK8K,cACxB7T,EAAOiI,GA30CJ,GA20Cec,EAAK+K,cAQtBlY,EAAI,EAAGA,EAAI,GAAqB,MAAhBoE,EAAOiI,GAAGrM,KAAcA,EACzCoE,EAAOiI,GAAGrM,GAAKd,EAAMc,GAAK2X,EAAY3X,QAInCA,EAAI,EAAGA,IACVoE,EAAOiI,GAAGrM,GAAKd,EAAMc,GAAsB,MAAhBoE,EAAOiI,GAAGrM,GAAqB,IAANA,EAAU,EAAI,EAAKoE,EAAOiI,GAAGrM,GAI7D,KAApBoE,EAAOiI,GA51CJ,IA61CuB,IAAtBjI,EAAOiI,GA51CN,IA61CqB,IAAtBjI,EAAOiI,GA51CN,IA61C0B,IAA3BjI,EAAOiI,GA51CD,KA61CVjI,EAAO+U,UAAW,EAClB/U,EAAOiI,GAj2CJ,GAi2Ce,GAGtBjI,EAAO7B,IAAM6B,EAAO4T,QAAUjI,GAAgBP,IAAYzQ,MAAM,KAAMG,GACtE0Y,EAAkBxT,EAAO4T,QAAU5T,EAAO7B,GAAGgO,YAAcnM,EAAO7B,GAAG6W,SAIlD,MAAfhV,EAAON,MACPM,EAAO7B,GAAG8W,cAAcjV,EAAO7B,GAAG+W,gBAAkBlV,EAAON,MAG3DM,EAAO+U,WACP/U,EAAOiI,GA92CJ,GA82Ce,IAIlBjI,EAAO+H,SAA6B,IAAhB/H,EAAO+H,GAAGsD,GAAqBrL,EAAO+H,GAAGsD,IAAMmI,IACnE5W,EAAgBoD,GAAQpC,iBAAkB,QA+D9CuX,GAAmB,mJACnBC,GAAgB,8IAEhBC,GAAU,wBAEVC,GAAW,CACX,CAAC,eAAgB,uBACjB,CAAC,aAAc,mBACf,CAAC,eAAgB,kBACjB,CAAC,aAAc,eAAe,GAC9B,CAAC,WAAY,eACb,CAAC,UAAW,cAAc,GAC1B,CAAC,aAAc,cACf,CAAC,WAAY,UAEZ,aAAc,eACf,CAAC,YAAa,eAAe,GAC7B,CAAC,UAAW,UAIZC,GAAW,CACX,CAAC,gBAAiB,uBAClB,CAAC,gBAAiB,sBAClB,CAAC,WAAY,kBACb,CAAC,QAAS,aACV,CAAC,cAAe,qBAChB,CAAC,cAAe,oBAChB,CAAC,SAAU,gBACX,CAAC,OAAQ,YACT,CAAC,KAAM,SAGPC,GAAkB,+BAGbC,GAAczV,OACfpE,EAAG8Z,EAGHC,EAAWC,EAAYC,EAAYC,EAFnCC,EAAS/V,EAAOT,GAChB6F,EAAQ+P,GAAiBa,KAAKD,IAAWX,GAAcY,KAAKD,MAG5D3Q,EAAO,KACPxI,EAAgBoD,GAAQxC,KAAM,EAEzB5B,EAAI,EAAG8Z,EAAIJ,GAASxZ,OAAQF,EAAI8Z,EAAG9Z,OAChC0Z,GAAS1Z,GAAG,GAAGoa,KAAK5Q,EAAM,IAAK,CAC/BwQ,EAAaN,GAAS1Z,GAAG,GACzB+Z,GAA+B,IAAnBL,GAAS1Z,GAAG,YAId,MAAdga,cACA5V,EAAOlC,UAAW,MAGlBsH,EAAM,GAAI,KACLxJ,EAAI,EAAG8Z,EAAIH,GAASzZ,OAAQF,EAAI8Z,EAAG9Z,OAChC2Z,GAAS3Z,GAAG,GAAGoa,KAAK5Q,EAAM,IAAK,CAE/ByQ,GAAczQ,EAAM,IAAM,KAAOmQ,GAAS3Z,GAAG,YAInC,MAAdia,cACA7V,EAAOlC,UAAW,OAIrB6X,GAA2B,MAAdE,cACd7V,EAAOlC,UAAW,MAGlBsH,EAAM,GAAI,KACNiQ,GAAQW,KAAK5Q,EAAM,gBAGnBpF,EAAOlC,UAAW,GAFlBgY,EAAW,IAMnB9V,EAAOR,GAAKoW,GAAcC,GAAc,KAAOC,GAAY,IAC3DG,GAA0BjW,QAE1BA,EAAOlC,UAAW,MAKtBH,GAAU,mMAkBLuY,GAAeC,OAChBhO,EAAOI,SAAS4N,EAAS,WACzBhO,GAAQ,GACD,IAAOA,EACPA,GAAQ,IACR,KAAOA,EAEXA,MAsBPiO,GAAa,CACbC,GAAI,EACJC,IAAK,EACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,cAiBAC,GAAkB/W,OAxEQmW,EAASa,EAAUC,EAAQC,EAASC,EAAWC,EAC1EC,EAwEAjS,EAAQzH,GAAQqY,KAAuBhW,EAAOT,GA7CzC8F,QAAQ,oBAAqB,KAAKA,QAAQ,WAAY,KAAKA,QAAQ,SAAU,IAAIA,QAAQ,SAAU,QA8CxGD,EAAO,KACHkS,GA3EuBnB,EA2EiB/Q,EAAM,GA3Ed4R,EA2EkB5R,EAAM,GA3Ed6R,EA2EkB7R,EAAM,GA3EhB8R,EA2EoB9R,EAAM,GA3EjB+R,EA2EqB/R,EAAM,GA3EhBgS,EA2EoBhS,EAAM,GA1EpGiS,EAAS,CACTnB,GAAeC,GACfvM,GAAyBpB,QAAQwO,GACjCzO,SAAS0O,EAAQ,IACjB1O,SAAS2O,EAAS,IAClB3O,SAAS4O,EAAW,KAGpBC,GACAC,EAAOtb,KAAKwM,SAAS6O,EAAW,KAG7BC,gBAkBWE,EAAYC,EAAaxX,UACvCuX,GAEsB5J,GAA2BnF,QAAQ+O,KACrC,IAAI/b,KAAKgc,EAAY,GAAIA,EAAY,GAAIA,EAAY,IAAIxC,WAEzEpY,EAAgBoD,GAAQpC,iBAAkB,EAC1CoC,EAAOlC,UAAW,GACX,GAqCN2Z,CAAarS,EAAM,GAAIkS,EAAatX,UAIzCA,EAAOiI,GAAKqP,EACZtX,EAAON,cAvBUgY,EAAWC,EAAgBC,MAC5CF,SACOtB,GAAWsB,GACf,GAAIC,SAEA,MAEHE,EAAKtP,SAASqP,EAAW,IACzB/a,EAAIgb,EAAK,WAAUA,EAAKhb,GAAK,IACtB,GAAKA,EAcFib,CAAgB1S,EAAM,GAAIA,EAAM,GAAIA,EAAM,KAExDpF,EAAO7B,GAAKwN,GAAchR,MAAM,KAAMqF,EAAOiI,IAC7CjI,EAAO7B,GAAG8W,cAAcjV,EAAO7B,GAAG+W,gBAAkBlV,EAAON,MAE3D9C,EAAgBoD,GAAQrC,SAAU,OAElCqC,EAAOlC,UAAW,WAgDjBmY,GAA0BjW,MAE3BA,EAAOR,KAAO9E,EAAMqd,YAIpB/X,EAAOR,KAAO9E,EAAMsd,UAIxBhY,EAAOiI,GAAK,GACZrL,EAAgBoD,GAAQjD,OAAQ,MAI5BnB,EAAG4b,EAAa5P,EAAQjD,EAAOsT,EAD/BlC,EAAS,GAAK/V,EAAOT,GAErB2Y,EAAenC,EAAOja,OACtBqc,EAAyB,MAE7BvQ,EAAS1C,EAAalF,EAAOR,GAAIQ,EAAOH,SAASuF,MAAMd,IAAqB,GAEvE1I,EAAI,EAAGA,EAAIgM,EAAO9L,OAAQF,IAC3B+I,EAAQiD,EAAOhM,IACf4b,GAAezB,EAAO3Q,MAAM+B,GAAsBxC,EAAO3E,KAAY,IAAI,OAIrEiY,EAAUlC,EAAO1R,OAAO,EAAG0R,EAAOvN,QAAQgP,KAC9B1b,OAAS,GACjBc,EAAgBoD,GAAQ/C,YAAYlB,KAAKkc,GAE7ClC,EAASA,EAAO/T,MAAM+T,EAAOvN,QAAQgP,GAAeA,EAAY1b,QAChEqc,GAA0BX,EAAY1b,QAGtC2I,EAAqBE,IACjB6S,EACA5a,EAAgBoD,GAAQjD,OAAQ,EAGhCH,EAAgBoD,GAAQhD,aAAajB,KAAK4I,GAE9CqD,GAAwBrD,EAAO6S,EAAaxX,IAEvCA,EAAO1B,UAAYkZ,GACxB5a,EAAgBoD,GAAQhD,aAAajB,KAAK4I,GAKlD/H,EAAgBoD,GAAQ7C,cAAgB+a,EAAeC,EACnDpC,EAAOja,OAAS,GAChBc,EAAgBoD,GAAQ/C,YAAYlB,KAAKga,GAIzC/V,EAAOiI,GA7sDJ,IA6sDgB,KACiB,IAApCrL,EAAgBoD,GAAQxB,SACxBwB,EAAOiI,GA/sDJ,GA+sDe,IAClBrL,EAAgBoD,GAAQxB,aAAUD,GAGtC3B,EAAgBoD,GAAQvC,gBAAkBuC,EAAOiI,GAAGjG,MAAM,GAC1DpF,EAAgBoD,GAAQtC,SAAWsC,EAAO2P,UAE1C3P,EAAOiI,GAttDA,YA6tDezL,EAAQ4b,EAAM1a,OAChC2a,SAEY,MAAZ3a,EAEO0a,EAEgB,MAAvB5b,EAAO8b,aACA9b,EAAO8b,aAAaF,EAAM1a,GACX,MAAflB,EAAOkT,OAEd2I,EAAO7b,EAAOkT,KAAKhS,KACP0a,EAAO,KACfA,GAAQ,IAEPC,GAAiB,KAATD,IACTA,EAAO,GAEJA,GAGAA,EA5BOG,CAAgBvY,EAAOH,QAASG,EAAOiI,GAttDlD,GAstD4DjI,EAAO2P,WAE1E2D,GAAgBtT,GAChBgT,GAAchT,QA7DV+W,GAAkB/W,QAJlByV,GAAczV,YAmKbwY,GAAexY,OAChBlF,EAAQkF,EAAOT,GACfhD,EAASyD,EAAOR,UAEpBQ,EAAOH,QAAUG,EAAOH,SAAWyS,GAAUtS,EAAOP,IAEtC,OAAV3E,QAA8ByD,IAAXhC,GAAkC,KAAVzB,EACpC4D,EAAc,CAACtB,WAAW,KAGhB,iBAAVtC,IACPkF,EAAOT,GAAKzE,EAAQkF,EAAOH,QAAQ4Y,SAAS3d,IAG5CoF,EAASpF,GACF,IAAIiF,EAAOiT,GAAclY,KACzBS,EAAOT,GACdkF,EAAO7B,GAAKrD,EACLD,EAAQ0B,YAtFWyD,OAC1B0Y,EACAC,EAEAC,EACAhd,EACAid,KAEqB,IAArB7Y,EAAOR,GAAG1D,cACVc,EAAgBoD,GAAQ1C,eAAgB,OACxC0C,EAAO7B,GAAK,IAAI3C,KAAKmD,UAIpB/C,EAAI,EAAGA,EAAIoE,EAAOR,GAAG1D,OAAQF,IAC9Bid,EAAe,EACfH,EAAazZ,EAAW,GAAIe,GACN,MAAlBA,EAAO4T,UACP8E,EAAW9E,QAAU5T,EAAO4T,SAEhC8E,EAAWlZ,GAAKQ,EAAOR,GAAG5D,GAC1Bqa,GAA0ByC,GAErB7a,EAAQ6a,KAKbG,GAAgBjc,EAAgB8b,GAAYvb,cAG5C0b,GAAkE,GAAlDjc,EAAgB8b,GAAY1b,aAAalB,OAEzDc,EAAgB8b,GAAYI,MAAQD,GAEjB,MAAfD,GAAuBC,EAAeD,KACtCA,EAAcC,EACdF,EAAaD,IAIrBtc,EAAO4D,EAAQ2Y,GAAcD,GA8CzBK,CAAyB/Y,GAClBzD,EACP0Z,GAA0BjW,YAYTA,OACjBlF,EAAQkF,EAAOT,GACflE,EAAYP,GACZkF,EAAO7B,GAAK,IAAI3C,KAAKd,EAAMiZ,OACpBpY,EAAOT,GACdkF,EAAO7B,GAAK,IAAI3C,KAAKV,EAAMuB,WACH,iBAAVvB,WAxPIkF,OAClBsH,EAAUkO,GAAgBQ,KAAKhW,EAAOT,IAE1B,OAAZ+H,GAKJmO,GAAczV,IACU,IAApBA,EAAOlC,kBACAkC,EAAOlC,SAKlBiZ,GAAkB/W,IACM,IAApBA,EAAOlC,kBACAkC,EAAOlC,SAMlBpD,EAAMse,wBAAwBhZ,MAnB1BA,EAAO7B,GAAK,IAAI3C,MAAM8L,EAAQ,IAqP9B2R,CAAiBjZ,GACVnF,EAAQC,IACfkF,EAAOiI,GAAKxM,EAAIX,EAAMkH,MAAM,IAAI,SAAU7B,UAC/BoI,SAASpI,EAAK,OAEzBmT,GAAgBtT,IACT5E,EAASN,YAtEEkF,OAClBA,EAAO7B,QAIPvC,EAAI2H,EAAqBvD,EAAOT,IACpCS,EAAOiI,GAAKxM,EAAI,CAACG,EAAEuM,KAAMvM,EAAEkN,MAAOlN,EAAEsS,KAAOtS,EAAEmN,KAAMnN,EAAEwc,KAAMxc,EAAEsd,OAAQtd,EAAEud,OAAQvd,EAAEwd,cAAc,SAAUjZ,UAC9FA,GAAOoI,SAASpI,EAAK,OAGhCmT,GAAgBtT,IA6DZqZ,CAAiBrZ,GACV1E,EAASR,GAEhBkF,EAAO7B,GAAK,IAAI3C,KAAKV,GAErBJ,EAAMse,wBAAwBhZ,GA7B9BsZ,CAAgBtZ,GAGfnC,EAAQmC,KACTA,EAAO7B,GAAK,MAGT6B,aA0BFtD,GAAkB5B,EAAOyB,EAAQC,EAAQC,EAAQ8c,OAlElD1d,EAmEAwX,EAAI,UAEO,IAAX7W,IAA8B,IAAXA,IACnBC,EAASD,EACTA,OAAS+B,IAGRnD,EAASN,aA5iFKqF,MACfnF,OAAOwe,2BAC4C,IAA3Cxe,OAAOwe,oBAAoBrZ,GAAKrE,WAEpC2d,MACCA,KAAKtZ,KACFA,EAAIhE,eAAesd,UACZ,SAGR,EAkiFaC,CAAc5e,IAC7BD,EAAQC,IAA2B,IAAjBA,EAAMgB,UAC7BhB,OAAQyD,GAIZ8U,EAAE/T,kBAAmB,EACrB+T,EAAEO,QAAUP,EAAE1T,OAAS4Z,EACvBlG,EAAE5T,GAAKjD,EACP6W,EAAE9T,GAAKzE,EACPuY,EAAE7T,GAAKjD,EACP8W,EAAE/U,QAAU7B,GArFRZ,EAAM,IAAIkE,EAAOiT,GAAcwF,GAuFXnF,MAtFhB0B,WAEJlZ,EAAI8d,IAAI,EAAG,KACX9d,EAAIkZ,cAAWxW,GAGZ1C,WAmFF4Y,GAAa3Z,EAAOyB,EAAQC,EAAQC,UAClCC,GAAiB5B,EAAOyB,EAAQC,EAAQC,GAAQ,GAxQ3D/B,EAAMse,wBAA0BtX,EAC5B,kVAIA,SAAU1B,GACNA,EAAO7B,GAAK,IAAI3C,KAAKwE,EAAOT,IAAMS,EAAO4T,QAAU,OAAS,QAKpElZ,EAAMqd,SAAW,aAGjBrd,EAAMsd,SAAW,iBA6Pb4B,GAAelY,EACf,sGACA,eACQmY,EAAQpF,GAAY9Z,MAAM,KAAMC,kBAChCkE,KAAKjB,WAAagc,EAAMhc,UACjBgc,EAAQ/a,KAAOA,KAAO+a,EAEtBnb,OAKfob,GAAepY,EACf,sGACA,eACQmY,EAAQpF,GAAY9Z,MAAM,KAAMC,kBAChCkE,KAAKjB,WAAagc,EAAMhc,UACjBgc,EAAQ/a,KAAOA,KAAO+a,EAEtBnb,gBAUVqb,GAAOpe,EAAIqe,OACZne,EAAKD,KACc,IAAnBoe,EAAQle,QAAgBjB,EAAQmf,EAAQ,MACxCA,EAAUA,EAAQ,KAEjBA,EAAQle,cACF2Y,SAEX5Y,EAAMme,EAAQ,GACTpe,EAAI,EAAGA,EAAIoe,EAAQle,SAAUF,EACzBoe,EAAQpe,GAAGiC,YAAamc,EAAQpe,GAAGD,GAAIE,KACxCA,EAAMme,EAAQpe,WAGfC,MAoBPoe,GAAW,CAAC,OAAQ,UAAW,QAAS,OAAQ,MAAO,OAAQ,SAAU,SAAU,wBAgC9EC,GAAUC,OACXzW,EAAkBH,EAAqB4W,GACvCC,EAAQ1W,EAAgByE,MAAQ,EAChCkS,EAAW3W,EAAgB4W,SAAW,EACtCjR,EAAS3F,EAAgBoF,OAAS,EAClCyR,EAAQ7W,EAAgB2I,MAAQ3I,EAAgB8W,SAAW,EAC3DC,EAAO/W,EAAgBwK,KAAO,EAC9BgB,EAAQxL,EAAgB0U,MAAQ,EAChChJ,EAAU1L,EAAgBwV,QAAU,EACpC3J,EAAU7L,EAAgByV,QAAU,EACpCuB,EAAehX,EAAgB0V,aAAe,OAE7Ctb,kBA1CgBjB,OAChB,IAAIkF,KAAOlF,MAC2B,IAAjC2L,GAAQrN,KAAK8e,GAAUlY,IAA0B,MAAVlF,EAAEkF,IAAiB7D,MAAMrB,EAAEkF,WAC7D,UAIX4Y,GAAiB,EACZ/e,EAAI,EAAGA,EAAIqe,GAASne,SAAUF,KAC/BiB,EAAEod,GAASre,IAAK,IACZ+e,SACO,EAEPC,WAAW/d,EAAEod,GAASre,OAAS6E,EAAM5D,EAAEod,GAASre,OAChD+e,GAAiB,UAKtB,EAuBSE,CAAgBnX,QAG3BoX,eAAiBJ,EACR,IAAVnL,EACU,IAAVH,EACQ,IAARF,EAAe,GAAK,QAGnB6L,OAASN,EACF,EAARF,OAICS,SAAW3R,EACD,EAAXgR,EACQ,GAARD,OAECa,MAAQ,QAERpb,QAAUyS,UAEV4I,mBAGAC,GAAYhb,UACVA,aAAe+Z,YAGjBkB,GAAU/a,UACXA,EAAS,GACyB,EAA3BC,KAAK+a,OAAO,EAAIhb,GAEhBC,KAAK+a,MAAMhb,YAMjBib,GAAQ3W,EAAO4W,GACpB7W,EAAeC,EAAO,EAAG,GAAG,eACpB2W,EAASxc,KAAK0c,YACdC,EAAO,WACPH,EAAS,IACTA,GAAUA,EACVG,EAAO,KAEJA,EAAO3X,KAAYwX,EAAS,IAAK,GAAKC,EAAYzX,IAAYwX,EAAU,GAAI,MAI3FA,GAAO,IAAK,KACZA,GAAO,KAAM,IAIbvU,GAAc,IAAMH,IACpBG,GAAc,KAAMH,IACpBiB,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,GAC/CA,EAAO4T,SAAU,EACjB5T,EAAON,KAAOgc,GAAiB9U,GAAkB9L,UAQjD6gB,GAAc,2BAETD,GAAiBE,EAAS7F,OAC3B8F,GAAW9F,GAAU,IAAI3Q,MAAMwW,MAEnB,OAAZC,SACO,SAIPC,IADUD,EAAQA,EAAQ/f,OAAS,IAAM,IACtB,IAAIsJ,MAAMuW,KAAgB,CAAC,IAAK,EAAG,GACtDvM,EAAuB,GAAX0M,EAAM,GAAWrb,EAAMqb,EAAM,WAE1B,IAAZ1M,EACL,EACa,MAAb0M,EAAM,GAAa1M,GAAWA,WAI3B2M,GAAgBjhB,EAAOkhB,OACxBngB,EAAKogB,SACLD,EAAMrc,QACN9D,EAAMmgB,EAAME,QACZD,GAAQ/b,EAASpF,IAAUS,EAAOT,GAASA,EAAMuB,UAAYoY,GAAY3Z,GAAOuB,WAAaR,EAAIQ,UAEjGR,EAAIsC,GAAGge,QAAQtgB,EAAIsC,GAAG9B,UAAY4f,GAClCvhB,EAAMuF,aAAapE,GAAK,GACjBA,GAEA4Y,GAAY3Z,GAAOshB,iBAIzBC,GAAexf,UAGgC,IAA5CyD,KAAK+a,MAAMxe,EAAEsB,GAAGme,oBAAsB,aAqJzCC,aACEzd,KAAKjB,WAAYiB,KAAKa,QAA2B,IAAjBb,KAAKc,QA/IhDlF,EAAMuF,aAAe,iBAmJjBuc,GAAc,2DAKdC,GAAW,+KAENC,GAAgB5hB,EAAOiH,OAIxB0Z,EACAkB,EACAC,EAiFmBC,EAAMhD,EACzBhe,EAvFAse,EAAWrf,EAEXsK,EAAQ,YAKR+V,GAAWrgB,GACXqf,EAAW,CACP3O,GAAK1Q,EAAMggB,cACXzP,EAAKvQ,EAAMigB,MACXxP,EAAKzQ,EAAMkgB,SAER1f,EAASR,IAChBqf,EAAW,GACPpY,EACAoY,EAASpY,GAAOjH,EAEhBqf,EAASO,aAAe5f,IAElBsK,EAAQoX,GAAYxG,KAAKlb,KACnC2gB,EAAqB,MAAbrW,EAAM,IAAe,EAAI,EACjC+U,EAAW,CACP9R,EAAK,EACLgD,EAAK5K,EAAM2E,EA7xEZ,IA6xEmDqW,EAClDnQ,EAAK7K,EAAM2E,EA7xEZ,IA6xEmDqW,EAClD5e,EAAK4D,EAAM2E,EA7xEV,IA6xEiDqW,EAClD9T,EAAKlH,EAAM2E,EA7xEV,IA6xEiDqW,EAClDjQ,GAAK/K,EAAM2a,GAA8B,IAArBhW,EA7xEd,KA6xE4CqW,KAE5CrW,EAAQqX,GAASzG,KAAKlb,KAChC2gB,EAAqB,MAAbrW,EAAM,IAAe,EAAI,EACjC+U,EAAW,CACP9R,EAAIyU,GAAS1X,EAAM,GAAIqW,GACvBlQ,EAAIuR,GAAS1X,EAAM,GAAIqW,GACvBvH,EAAI4I,GAAS1X,EAAM,GAAIqW,GACvBpQ,EAAIyR,GAAS1X,EAAM,GAAIqW,GACvBnQ,EAAIwR,GAAS1X,EAAM,GAAIqW,GACvB5e,EAAIigB,GAAS1X,EAAM,GAAIqW,GACvB9T,EAAImV,GAAS1X,EAAM,GAAIqW,KAER,MAAZtB,EACPA,EAAW,GACgB,iBAAbA,IAA0B,SAAUA,GAAY,OAAQA,KA2CnD0C,EA1CSpI,GAAY0F,EAAShb,MA0CxB0a,EA1C+BpF,GAAY0F,EAASjb,IAA7E0d,EA4CEC,EAAKhf,WAAagc,EAAMhc,WAI9Bgc,EAAQkC,GAAgBlC,EAAOgD,GAC3BA,EAAKE,SAASlD,GACdhe,EAAMmhB,GAA0BH,EAAMhD,KAEtChe,EAAMmhB,GAA0BnD,EAAOgD,IACnCnC,cAAgB7e,EAAI6e,aACxB7e,EAAIwN,QAAUxN,EAAIwN,QAGfxN,GAZI,CAAC6e,aAAc,EAAGrR,OAAQ,IA3CjC8Q,EAAW,IACF3O,GAAKoR,EAAQlC,aACtBP,EAAS5O,EAAIqR,EAAQvT,QAGzBsT,EAAM,IAAIzC,GAASC,GAEfgB,GAAWrgB,IAAUkB,EAAWlB,EAAO,aACvC6hB,EAAI9c,QAAU/E,EAAM+E,SAGjB8c,WAMFG,GAAUG,EAAKxB,OAIhB5f,EAAMohB,GAAOrC,WAAWqC,EAAI5X,QAAQ,IAAK,aAErCnH,MAAMrC,GAAO,EAAIA,GAAO4f,WAG3BuB,GAA0BH,EAAMhD,OACjChe,EAAM,UAEVA,EAAIwN,OAASwQ,EAAM/Q,QAAU+T,EAAK/T,QACC,IAA9B+Q,EAAM1R,OAAS0U,EAAK1U,QACrB0U,EAAKX,QAAQvC,IAAI9d,EAAIwN,OAAQ,KAAK6T,QAAQrD,MACxChe,EAAIwN,OAGVxN,EAAI6e,cAAgBb,GAAUgD,EAAKX,QAAQvC,IAAI9d,EAAIwN,OAAQ,KAEpDxN,WAsBFshB,GAAYC,EAAW7a,UACrB,SAAUlD,EAAKge,OACTC,SAEM,OAAXD,GAAoBnf,OAAOmf,KAC3B/a,EAAgBC,EAAM,YAAcA,EAAQ,uDAAyDA,EAAO,kGAE5G+a,EAAMje,EAAKA,EAAMge,EAAQA,EAASC,GAKtCC,GAAYze,KADN4d,GADNrd,EAAqB,iBAARA,GAAoBA,EAAMA,EACbge,GACHD,GAChBte,eAINye,GAAajY,EAAK6U,EAAUqD,EAAUvd,OACvCya,EAAeP,EAASW,cACxBL,EAAOW,GAASjB,EAASY,OACzB1R,EAAS+R,GAASjB,EAASa,SAE1B1V,EAAIzH,YAKToC,EAA+B,MAAhBA,GAA8BA,EAEzCoJ,GACAgB,GAAS/E,EAAKuD,GAAIvD,EAAK,SAAW+D,EAASmU,GAE3C/C,GACA7R,GAAMtD,EAAK,OAAQuD,GAAIvD,EAAK,QAAUmV,EAAO+C,GAE7C9C,GACApV,EAAInH,GAAGge,QAAQ7W,EAAInH,GAAG9B,UAAYqe,EAAe8C,GAEjDvd,GACAvF,EAAMuF,aAAaqF,EAAKmV,GAAQpR,IApFxCqT,GAAe/gB,GAAKue,GAASjf,UAC7ByhB,GAAee,0BAtVJf,GAAe/d,UA6atBgb,GAAWwD,GAAY,EAAG,OAC1BO,GAAWP,IAAa,EAAG,qBA0HtBQ,GAAW1hB,EAAGC,OAEf0hB,EAA0C,IAAvB1hB,EAAEiM,OAASlM,EAAEkM,SAAiBjM,EAAE4M,QAAU7M,EAAE6M,SAE/D+U,EAAS5hB,EAAEigB,QAAQvC,IAAIiE,EAAgB,kBAclCA,GAXL1hB,EAAI2hB,EAAS,GAGH3hB,EAAI2hB,IAAWA,EAFf5hB,EAAEigB,QAAQvC,IAAIiE,EAAiB,EAAG,YAMlC1hB,EAAI2hB,IAFJ5hB,EAAEigB,QAAQvC,IAAIiE,EAAiB,EAAG,UAETC,MAIF,WA6FhCrhB,GAAQuF,OACT+b,cAEQvf,IAARwD,EACOjD,KAAKe,QAAQmS,OAGC,OADrB8L,EAAgBxL,GAAUvQ,WAEjBlC,QAAUie,GAEZhf,MApGfpE,EAAMqjB,cAAgB,uBACtBrjB,EAAMsjB,iBAAmB,6BAuGrBC,GAAOvc,EACP,mJACA,SAAUK,eACMxD,IAARwD,EACOjD,KAAKkG,aAELlG,KAAKtC,OAAOuF,eAKtBiD,YACElG,KAAKe,iBASPqe,GAAMC,EAAUC,UACbD,EAAWC,EAAUA,GAAWA,WAGnCC,GAAiBhW,EAAGxL,EAAGwO,UAExBhD,EAAI,KAAOA,GAAK,EAET,IAAI7M,KAAK6M,EAAI,IAAKxL,EAAGwO,GAXb,YAaR,IAAI7P,KAAK6M,EAAGxL,EAAGwO,GAAGhP,mBAIxBiiB,GAAejW,EAAGxL,EAAGwO,UAEtBhD,EAAI,KAAOA,GAAK,EAET7M,KAAKoQ,IAAIvD,EAAI,IAAKxL,EAAGwO,GArBb,YAuBR7P,KAAKoQ,IAAIvD,EAAGxL,EAAGwO,YAuKrBkT,GAAwB5Z,EAAO6Z,GACpC9Z,EAAe,EAAG,CAACC,EAAOA,EAAM7I,QAAS,EAAG0iB,YA+DvCC,GAAqB3jB,EAAOuR,EAAMC,EAASN,EAAKC,OACjDyS,SACS,MAAT5jB,EACO4R,GAAW5N,KAAMkN,EAAKC,GAAK9D,MAG9BkE,GADJqS,EAAc7R,GAAY/R,EAAOkR,EAAKC,MAElCI,EAAOqS,GAEJC,GAAWxjB,KAAK2D,KAAMhE,EAAOuR,EAAMC,EAASN,EAAKC,aAIvD0S,GAAWxK,EAAU9H,EAAMC,EAASN,EAAKC,OAC1C2S,EAAgBxS,GAAmB+H,EAAU9H,EAAMC,EAASN,EAAKC,GACjElD,EAAO4C,GAAciT,EAAczW,KAAM,EAAGyW,EAAcnS,uBAEzDtE,KAAKY,EAAK8C,uBACV/C,MAAMC,EAAK8K,oBACX9K,KAAKA,EAAK+K,cACRhV,KA5FX4F,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAKqV,WAAa,OAG7BzP,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAK+f,cAAgB,OAOhCN,GAAuB,OAAY,YACnCA,GAAuB,QAAY,YACnCA,GAAuB,OAAS,eAChCA,GAAuB,QAAS,eAIhCvb,EAAa,WAAY,MACzBA,EAAa,cAAe,MAI5BY,EAAgB,WAAY,GAC5BA,EAAgB,cAAe,GAK/BmD,GAAc,IAAUL,IACxBK,GAAc,IAAUL,IACxBK,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,OAAUR,GAAWN,GACnCc,GAAc,OAAUR,GAAWN,GACnCc,GAAc,QAAUP,GAAWN,GACnCa,GAAc,QAAUP,GAAWN,GAEnC4B,GAAkB,CAAC,OAAQ,QAAS,OAAQ,UAAU,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GACjF0H,EAAK1H,EAAMN,OAAO,EAAG,IAAM5D,EAAM3F,MAGrCgN,GAAkB,CAAC,KAAM,OAAO,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GAC3D0H,EAAK1H,GAASjK,EAAM4N,kBAAkBxN,MAqD1C4J,EAAe,IAAK,EAAG,KAAM,WAI7B1B,EAAa,UAAW,KAIxBY,EAAgB,UAAW,GAI3BmD,GAAc,IAAKjB,GACnB+B,GAAc,KAAK,SAAU/M,EAAOqK,GAChCA,EAz8FQ,GAy8F4B,GAApB1E,EAAM3F,GAAS,MAWnC4J,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,QAIrC1B,EAAa,OAAQ,KAGrBY,EAAgB,OAAQ,GAIxBmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,MAAM,SAAUG,EAAU1K,UAE7B0K,EACJ1K,EAAOsiB,yBAA2BtiB,EAAOuiB,cAC1CviB,EAAOwiB,kCAGbnX,GAAc,CAAC,IAAK,MAv+FT,GAw+FXA,GAAc,MAAM,SAAU/M,EAAOqK,GACjCA,EAz+FO,GAy+FO1E,EAAM3F,EAAMsK,MAAMe,GAAW,WAK3C8Y,GAAmBvW,GAAW,QAAQ,GAI1ChE,EAAe,MAAO,CAAC,OAAQ,GAAI,OAAQ,aAI3C1B,EAAa,YAAa,OAG1BY,EAAgB,YAAa,GAI7BmD,GAAc,MAAQT,IACtBS,GAAc,OAAQf,GACtB6B,GAAc,CAAC,MAAO,SAAS,SAAU/M,EAAOqK,EAAOnF,GACnDA,EAAO6U,WAAapU,EAAM3F,MAc9B4J,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,UAIlC1B,EAAa,SAAU,KAIvBY,EAAgB,SAAU,IAI1BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/B8B,GAAc,CAAC,IAAK,MA1hGP,OA8hGTqX,GAAexW,GAAW,WAAW,GAIzChE,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,UAIlC1B,EAAa,SAAU,KAIvBY,EAAgB,SAAU,IAI1BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/B8B,GAAc,CAAC,IAAK,MA/iGP,OAkmGTlD,GA/CAwa,GAAezW,GAAW,WAAW,OAIzChE,EAAe,IAAK,EAAG,GAAG,oBACZ5F,KAAKsa,cAAgB,QAGnC1U,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,oBAClB5F,KAAKsa,cAAgB,OAGnC1U,EAAe,EAAG,CAAC,MAAO,GAAI,EAAG,eACjCA,EAAe,EAAG,CAAC,OAAQ,GAAI,GAAG,kBACF,GAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,QAAS,GAAI,GAAG,kBACH,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,SAAU,GAAI,GAAG,kBACJ,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,UAAW,GAAI,GAAG,kBACL,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,WAAY,GAAI,GAAG,kBACN,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,YAAa,GAAI,GAAG,kBACP,IAArB5F,KAAKsa,iBAMhBpW,EAAa,cAAe,MAI5BY,EAAgB,cAAe,IAI/BmD,GAAc,IAAQT,GAAWR,GACjCiB,GAAc,KAAQT,GAAWP,GACjCgB,GAAc,MAAQT,GAAWN,GAG5BrB,GAAQ,OAAQA,GAAM7I,QAAU,EAAG6I,IAAS,IAC7CoC,GAAcpC,GAAO8B,aAGhB2Y,GAAQtkB,EAAOqK,GACpBA,EAvmGc,GAumGO1E,EAAuB,KAAhB,KAAO3F,QAGlC6J,GAAQ,IAAKA,GAAM7I,QAAU,EAAG6I,IAAS,IAC1CkD,GAAclD,GAAOya,QAIrBC,GAAoB3W,GAAW,gBAAgB,GAInDhE,EAAe,IAAM,EAAG,EAAG,YAC3BA,EAAe,KAAM,EAAG,EAAG,gBAYvB4a,GAAQvf,EAAO9E,mBAsFVskB,GAAoBxJ,UAClBA,EArFXuJ,GAAM3F,IAAoBA,GAC1B2F,GAAMpP,kBAjuBesP,EAAMC,OAGnB9L,EAAM6L,GAAQ/K,KACdiL,EAAM3D,GAAgBpI,EAAK7U,MAAM6gB,QAAQ,OACzCpjB,EAAS7B,EAAMklB,eAAe9gB,KAAM4gB,IAAQ,WAE5Cna,EAASka,IAAYjd,EAAWid,EAAQljB,IAAWkjB,EAAQljB,GAAQpB,KAAK2D,KAAM6U,GAAO8L,EAAQljB,WAE1FuC,KAAKvC,OAAOgJ,GAAUzG,KAAKkG,aAAakL,SAAS3T,EAAQuC,KAAM2V,GAAYd,MAytBtF2L,GAAMpD,wBArtBK,IAAInc,EAAOjB,OAstBtBwgB,GAAMrD,cAvpBSnhB,EAAOwI,EAAOuc,OACrBC,EACAC,EACAxa,MAECzG,KAAKjB,iBACCc,SAGXmhB,EAAO/D,GAAgBjhB,EAAOgE,OAEpBjB,iBACCc,WAGXohB,EAAoD,KAAvCD,EAAKtE,YAAc1c,KAAK0c,aAErClY,EAAQD,EAAeC,QAGd,OAAQiC,EAASoY,GAAU7e,KAAMghB,GAAQ,aACzC,QAASva,EAASoY,GAAU7e,KAAMghB,aAClC,UAAWva,EAASoY,GAAU7e,KAAMghB,GAAQ,YAC5C,SAAUva,GAAUzG,KAAOghB,GAAQ,cACnC,SAAUva,GAAUzG,KAAOghB,GAAQ,cACnC,OAAQva,GAAUzG,KAAOghB,GAAQ,eACjC,MAAOva,GAAUzG,KAAOghB,EAAOC,GAAa,gBAC5C,OAAQxa,GAAUzG,KAAOghB,EAAOC,GAAa,qBACzCxa,EAASzG,KAAOghB,SAGtBD,EAAUta,EAASnF,EAASmF,IAynBvC+Z,GAAMU,eA5ZU1c,OACRkc,UAEUjhB,KADd+E,EAAQD,EAAeC,KACc,gBAAVA,IAA4BxE,KAAKjB,iBACjDiB,SAGPmhB,EAAcnhB,KAAKa,OAAS2e,GAAiBD,UAEzC/a,OACC,OACDkc,EAAOS,EAAYnhB,KAAKqJ,OAAS,EAAG,EAAG,GAAK,YAE3C,UACDqX,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAUhK,KAAKgK,QAAU,EAAI,EAAG,GAAK,YAEzE,QACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAU,EAAG,GAAK,YAEtD,OACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAASjK,KAAKwN,UAAY,GAAK,YAEjF,UACDkT,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,QAAUjK,KAAKohB,aAAe,GAAK,GAAK,YAE1F,UACA,OACDV,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAAS,GAAK,YAEhE,OACDyW,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GA3GM,KA2GgBtB,GAAMsB,GAAQ1gB,KAAKa,OAAS,EA5G1C,IA4G8Cb,KAAK0c,aA3GrD,MA2GkG,YAEvG,SACDgE,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAhHQ,IAgHgBtB,GAAMsB,EAhHtB,KAgH6C,YAEpD,SACDA,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GArHQ,IAqHgBtB,GAAMsB,EArHtB,KAqH6C,cAIxDrhB,GAAGge,QAAQqD,GAChB9kB,EAAMuF,aAAanB,MAAM,GAClBA,MAgXXwgB,GAAM/iB,gBA/iBW4jB,GACRA,IACDA,EAAcrhB,KAAKyd,QAAU7hB,EAAMsjB,iBAAmBtjB,EAAMqjB,mBAE5DxY,EAASN,EAAanG,KAAMqhB,UACzBrhB,KAAKkG,aAAaob,WAAW7a,IA2iBxC+Z,GAAMngB,cAxiBSqgB,EAAMa,UACbvhB,KAAKjB,YACCqC,EAASsf,IAASA,EAAK3hB,WACxB4W,GAAY+K,GAAM3hB,WAChB6e,GAAe,CAACxd,GAAIJ,KAAMK,KAAMqgB,IAAOhjB,OAAOsC,KAAKtC,UAAU8jB,UAAUD,GAEvEvhB,KAAKkG,aAAaS,eAmiBjC6Z,GAAMiB,iBA/hBYF,UACPvhB,KAAKK,KAAKsV,KAAe4L,IA+hBpCf,GAAMpgB,YA5hBOsgB,EAAMa,UACXvhB,KAAKjB,YACCqC,EAASsf,IAASA,EAAK3hB,WACxB4W,GAAY+K,GAAM3hB,WAChB6e,GAAe,CAACvd,KAAML,KAAMI,GAAIsgB,IAAOhjB,OAAOsC,KAAKtC,UAAU8jB,UAAUD,GAEvEvhB,KAAKkG,aAAaS,eAuhBjC6Z,GAAMkB,eAnhBUH,UACLvhB,KAAKI,GAAGuV,KAAe4L,IAmhBlCf,GAAMzW,aAxiGcvF,UAEZd,EAAW1D,KADfwE,EAAQD,EAAeC,KAEZxE,KAAKwE,KAETxE,MAoiGXwgB,GAAMmB,4BAvUK7jB,EAAgBkC,MAAM5B,UAwUjCoiB,GAAMpC,iBA5tBYpiB,EAAOwI,OACjBod,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YACjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,UAAYqkB,EAAWrkB,UAE5BqkB,EAAWrkB,UAAYyC,KAAKod,QAAQyD,QAAQrc,GAAOjH,YAotBlEijB,GAAMvC,kBAhtBajiB,EAAOwI,OAClBod,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YACjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,UAAYqkB,EAAWrkB,UAE5ByC,KAAKod,QAAQ8D,MAAM1c,GAAOjH,UAAYqkB,EAAWrkB,YAwsBhEijB,GAAMqB,mBApsBcxhB,EAAMD,EAAIoE,EAAOsd,OAC7BC,EAAY3gB,EAASf,GAAQA,EAAOsV,GAAYtV,GAChD2hB,EAAU5gB,EAAShB,GAAMA,EAAKuV,GAAYvV,YACxCJ,KAAKjB,WAAagjB,EAAUhjB,WAAaijB,EAAQjjB,aAI5B,OAD3B+iB,EAAcA,GAAe,MACT,GAAa9hB,KAAKoe,QAAQ2D,EAAWvd,IAAUxE,KAAKie,SAAS8D,EAAWvd,MACpE,MAAnBsd,EAAY,GAAa9hB,KAAKie,SAAS+D,EAASxd,IAAUxE,KAAKoe,QAAQ4D,EAASxd,KA6rBzFgc,GAAMyB,gBA1rBWjmB,EAAOwI,OAEhB0d,EADAN,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YAEjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,YAAcqkB,EAAWrkB,WAErC2kB,EAAUN,EAAWrkB,UACdyC,KAAKod,QAAQyD,QAAQrc,GAAOjH,WAAa2kB,GAAWA,GAAWliB,KAAKod,QAAQ8D,MAAM1c,GAAOjH,aAgrBxGijB,GAAM2B,uBA5qBkBnmB,EAAOwI,UACpBxE,KAAKiiB,OAAOjmB,EAAOwI,IAAUxE,KAAKoe,QAAQpiB,EAAOwI,IA4qB5Dgc,GAAM4B,wBAzqBmBpmB,EAAOwI,UACrBxE,KAAKiiB,OAAOjmB,EAAOwI,IAAUxE,KAAKie,SAASjiB,EAAOwI,IAyqB7Dgc,GAAMzhB,0BAtVKA,EAAQiB,OAuVnBwgB,GAAMrB,KAAoBA,GAC1BqB,GAAM9iB,OAAoBA,GAC1B8iB,GAAMta,WAAoBA,GAC1Bsa,GAAMlb,IAAoB0V,GAC1BwF,GAAMpe,IAAoB0Y,GAC1B0F,GAAM6B,+BAxVK/kB,EAAO,GAAIQ,EAAgBkC,QAyVtCwgB,GAAMxc,aA9iGcQ,EAAO1C,MACF,iBAAV0C,UAEH8d,WApTiBC,OACrB/d,EAAQ,OACP,IAAIge,KAAKD,EACV/d,EAAMvH,KAAK,CAACkH,KAAMqe,EAAGzd,SAAUF,EAAW2d,YAE9Che,EAAMyH,MAAK,SAAU9O,EAAGC,UACbD,EAAE4H,SAAW3H,EAAE2H,YAEnBP,EA4Seie,CADlBje,EAAQC,EAAqBD,IAEpB1H,EAAI,EAAGA,EAAIwlB,EAAYtlB,OAAQF,SAC/BwlB,EAAYxlB,GAAGqH,MAAMK,EAAM8d,EAAYxlB,GAAGqH,eAI/CT,EAAW1D,KADfwE,EAAQD,EAAeC,YAEZxE,KAAKwE,GAAO1C,UAGpB9B,MAkiGXwgB,GAAMK,iBAleYrc,OACVkc,UAEUjhB,KADd+E,EAAQD,EAAeC,KACc,gBAAVA,IAA4BxE,KAAKjB,iBACjDiB,SAGPmhB,EAAcnhB,KAAKa,OAAS2e,GAAiBD,UAEzC/a,OACC,OACDkc,EAAOS,EAAYnhB,KAAKqJ,OAAQ,EAAG,aAElC,UACDqX,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAUhK,KAAKgK,QAAU,EAAG,aAEhE,QACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAS,aAE7C,OACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAASjK,KAAKwN,qBAEhE,UACDkT,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,QAAUjK,KAAKohB,aAAe,cAEhF,UACA,OACDV,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,kBAElD,OACDyW,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,GAAQ1gB,KAAKa,OAAS,EA5D5B,IA4DgCb,KAAK0c,aA3DvC,gBA6DL,SACDgE,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,EAhEN,eAkEP,SACDA,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,EArEN,iBAyEXrhB,GAAGge,QAAQqD,GAChB9kB,EAAMuF,aAAanB,MAAM,GAClBA,MAsbXwgB,GAAM5B,SAAoBA,GAC1B4B,GAAMkC,uBAvXE3kB,EAAIiC,WACD,CAACjC,EAAEsL,OAAQtL,EAAEiM,QAASjM,EAAEkM,OAAQlM,EAAEub,OAAQvb,EAAEqc,SAAUrc,EAAEsc,SAAUtc,EAAEuc,gBAuX/EkG,GAAMmC,wBAnXE5kB,EAAIiC,WACD,CACHsb,MAAOvd,EAAEsL,OACTkB,OAAQxM,EAAEiM,QACVC,KAAMlM,EAAEkM,OACRmG,MAAOrS,EAAEqS,QACTE,QAASvS,EAAEuS,UACXG,QAAS1S,EAAE0S,UACXmL,aAAc7d,EAAE6d,iBA4WxB4E,GAAMoC,yBA7XK,IAAIlmB,KAAKsD,KAAKzC,YA8XzBijB,GAAMqC,qBArnBeC,OACZ9iB,KAAKjB,iBACC,SAEPlB,GAAqB,IAAfilB,EACN/kB,EAAIF,EAAMmC,KAAKod,QAAQvf,MAAQmC,YAC/BjC,EAAEsL,OAAS,GAAKtL,EAAEsL,OAAS,KACpBlD,EAAapI,EAAGF,EAAM,iCAAmC,gCAEhE6F,EAAWhH,KAAKP,UAAU0mB,aAEtBhlB,EACOmC,KAAK4iB,SAASC,cAEd,IAAInmB,KAAKsD,KAAKzC,UAA+B,GAAnByC,KAAK0c,YAAmB,KAAMmG,cAActc,QAAQ,IAAKJ,EAAapI,EAAG,MAG3GoI,EAAapI,EAAGF,EAAM,+BAAiC,+BAqmBlE2iB,GAAMuC,uBA3lBG/iB,KAAKjB,gBACC,qBAAuBiB,KAAKS,GAAK,WAExCwF,EAAO,SACP+c,EAAO,GACNhjB,KAAKijB,YACNhd,EAA4B,IAArBjG,KAAK0c,YAAoB,aAAe,mBAC/CsG,EAAO,SAEPE,EAAS,IAAMjd,EAAO,MACtBoD,EAAQ,GAAKrJ,KAAKqJ,QAAUrJ,KAAKqJ,QAAU,KAAQ,OAAS,SAE5D8Z,EAASH,EAAO,cAEbhjB,KAAKvC,OAAOylB,EAAS7Z,EAHb,wBAG+B8Z,IA8kBlD3C,GAAM4C,yBAzWKpjB,KAAKjB,UAAYiB,KAAK6iB,cAAgB,MA0WjDrC,GAAMpkB,2BA3nBK4D,KAAKod,QAAQ1f,OAAO,MAAMD,OAAO,qCA4nB5C+iB,GAAM6C,uBAtYK7hB,KAAKE,MAAM1B,KAAKzC,UAAY,MAuYvCijB,GAAMjjB,0BA3YKyC,KAAKX,GAAG9B,UAAmC,KAArByC,KAAKc,SAAW,IA4YjD0f,GAAM8C,8BA7VK,CACHtnB,MAAOgE,KAAKS,GACZhD,OAAQuC,KAAKU,GACbhD,OAAQsC,KAAKe,QACb0Z,MAAOza,KAAKa,OACZlD,OAAQqC,KAAKR,UAyVrBghB,GAAMnX,KAAaM,GACnB6W,GAAMlX,6BAtmGKA,GAAWtJ,KAAKqJ,SAumG3BmX,GAAMnL,kBApSmBrZ,UACd2jB,GAAqBtjB,KAAK2D,KACzBhE,EACAgE,KAAKuN,OACLvN,KAAKwN,UACLxN,KAAKkG,aAAa0P,MAAM1I,IACxBlN,KAAKkG,aAAa0P,MAAMzI,MA+RpCqT,GAAMT,qBA5RsB/jB,UACjB2jB,GAAqBtjB,KAAK2D,KACzBhE,EAAOgE,KAAK0b,UAAW1b,KAAKohB,aAAc,EAAG,IA2RzDZ,GAAMhF,QAAUgF,GAAMjF,kBAnOEvf,UACJ,MAATA,EAAgBwF,KAAKC,MAAMzB,KAAKgK,QAAU,GAAK,GAAKhK,KAAKgK,MAAoB,GAAbhO,EAAQ,GAASgE,KAAKgK,QAAU,IAmO3GwW,GAAMxW,MAAcyB,GACpB+U,GAAMtW,8BAx1FKA,GAAYlK,KAAKqJ,OAAQrJ,KAAKgK,UAy1FzCwW,GAAMjT,KAAiBiT,GAAM/E,eAznFRzf,OACbuR,EAAOvN,KAAKkG,aAAaqH,KAAKvN,aAClB,MAAThE,EAAgBuR,EAAOvN,KAAK6a,IAAqB,GAAhB7e,EAAQuR,GAAW,MAwnF/DiT,GAAM9E,QAAiB8E,GAAM+C,kBArnFLvnB,OAChBuR,EAAOK,GAAW5N,KAAM,EAAG,GAAGuN,YAClB,MAATvR,EAAgBuR,EAAOvN,KAAK6a,IAAqB,GAAhB7e,EAAQuR,GAAW,MAonF/DiT,GAAMzS,2BAxREyV,EAAWxjB,KAAKkG,aAAa0P,aAC1B7H,GAAY/N,KAAKqJ,OAAQma,EAAStW,IAAKsW,EAASrW,MAwR3DqT,GAAMiD,iCA7RK1V,GAAY/N,KAAKqJ,OAAQ,EAAG,IA8RvCmX,GAAMvW,KAAakW,GACnBK,GAAMpR,IAAaoR,GAAM7E,cA55EC3f,OACjBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,QAE9BuP,EAAMpP,KAAKa,OAASb,KAAKX,GAAGgO,YAAcrN,KAAKX,GAAG6W,gBACzC,MAATla,GACAA,WAjKcA,EAAO0B,SACJ,iBAAV1B,EACAA,EAGNoD,MAAMpD,GAKU,iBADrBA,EAAQ0B,EAAOiR,cAAc3S,IAElBA,EAGJ,KARIyN,SAASzN,EAAO,IA2Jf0nB,CAAa1nB,EAAOgE,KAAKkG,cAC1BlG,KAAK6a,IAAI7e,EAAQoT,EAAK,MAEtBA,GAo5EfoR,GAAMhT,iBAh5E0BxR,OACvBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,QAE9B2N,GAAWxN,KAAKoP,MAAQ,EAAIpP,KAAKkG,aAAa0P,MAAM1I,KAAO,SAC/C,MAATlR,EAAgBwR,EAAUxN,KAAK6a,IAAI7e,EAAQwR,EAAS,MA44E/DgT,GAAMY,oBAz4EuBplB,OACpBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,OAOrB,MAAT7D,EAAe,KACXwR,WAzKaxR,EAAO0B,SACP,iBAAV1B,EACA0B,EAAOiR,cAAc3S,GAAS,GAAK,EAEvCoD,MAAMpD,GAAS,KAAOA,EAqKX2nB,CAAgB3nB,EAAOgE,KAAKkG,qBACnClG,KAAKoP,IAAIpP,KAAKoP,MAAQ,EAAI5B,EAAUA,EAAU,UAE9CxN,KAAKoP,OAAS,GA63E7BoR,GAAM7S,mBApLoB3R,OAClB2R,EAAYnM,KAAK+a,OAAOvc,KAAKod,QAAQyD,QAAQ,OAAS7gB,KAAKod,QAAQyD,QAAQ,SAAW,OAAS,SACnF,MAAT7kB,EAAgB2R,EAAY3N,KAAK6a,IAAK7e,EAAQ2R,EAAY,MAmLrE6S,GAAMlH,KAAOkH,GAAMpQ,MAAQc,GAC3BsP,GAAMpG,OAASoG,GAAMlQ,QAAU8P,GAC/BI,GAAMnG,OAASmG,GAAM/P,QAAU4P,GAC/BG,GAAMlG,YAAckG,GAAM5E,aAAe2E,GACzCC,GAAM9D,mBAxkCiB1gB,EAAO4nB,EAAeC,OAErCC,EADAtH,EAASxc,KAAKc,SAAW,MAExBd,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,OAErB,MAAT7D,EAAe,IACM,iBAAVA,MAEO,QADdA,EAAQ4gB,GAAiB9U,GAAkB9L,WAEhCgE,UAEJwB,KAAKc,IAAItG,GAAS,KAAO6nB,IAChC7nB,GAAgB,WAEfgE,KAAKa,QAAU+iB,IAChBE,EAAcvG,GAAcvd,YAE3Bc,QAAU9E,OACV6E,QAAS,EACK,MAAfijB,QACKjJ,IAAIiJ,EAAa,KAEtBtH,IAAWxgB,KACN4nB,GAAiB5jB,KAAK+jB,kBACvBtF,GAAYze,KAAM4d,GAAe5hB,EAAQwgB,EAAQ,KAAM,GAAG,GAClDxc,KAAK+jB,yBACRA,mBAAoB,EACzBnoB,EAAMuF,aAAanB,MAAM,QACpB+jB,kBAAoB,OAG1B/jB,YAEAA,KAAKa,OAAS2b,EAASe,GAAcvd,OAuiCpDwgB,GAAM3iB,aArhCmB+lB,UACd5jB,KAAK0c,UAAU,EAAGkH,IAqhC7BpD,GAAMlD,eAlhCqBsG,UACnB5jB,KAAKa,cACA6b,UAAU,EAAGkH,QACb/iB,QAAS,EAEV+iB,QACKhF,SAASrB,GAAcvd,MAAO,MAGpCA,MA0gCXwgB,GAAMwD,wBAtgCe,MAAbhkB,KAAKY,UACA8b,UAAU1c,KAAKY,MAAM,GAAO,QAC9B,GAAuB,iBAAZZ,KAAKS,GAAiB,KAChCwjB,EAAQrH,GAAiB/U,GAAa7H,KAAKS,IAClC,MAATwjB,OACKvH,UAAUuH,QAGVvH,UAAU,GAAG,UAGnB1c,MA4/BXwgB,GAAM0D,8BAz/ByBloB,WACtBgE,KAAKjB,YAGV/C,EAAQA,EAAQ2Z,GAAY3Z,GAAO0gB,YAAc,GAEzC1c,KAAK0c,YAAc1gB,GAAS,IAAO,IAo/B/CwkB,GAAM2D,wBA/+BEnkB,KAAK0c,YAAc1c,KAAKod,QAAQpT,MAAM,GAAG0S,aACzC1c,KAAK0c,YAAc1c,KAAKod,QAAQpT,MAAM,GAAG0S,aA++BjD8D,GAAMyC,2BAr9BKjjB,KAAKjB,YAAaiB,KAAKa,QAs9BlC2f,GAAM4D,+BAl9BKpkB,KAAKjB,WAAYiB,KAAKa,QAm9BjC2f,GAAM/C,MAAuBA,GAC7B+C,GAAM/F,MAAuBgD,GAC7B+C,GAAM6D,2BA7EKrkB,KAAKa,OAAS,MAAQ,IA8EjC2f,GAAM8D,2BA1EKtkB,KAAKa,OAAS,6BAA+B,IA2ExD2f,GAAM+D,MAAS3hB,EAAU,kDAAmDud,IAC5EK,GAAMjW,OAAS3H,EAAU,mDAAoD6I,IAC7E+U,GAAMlF,MAAS1Y,EAAU,iDAAkD+G,IAC3E6W,GAAMwC,KAASpgB,EAAU,qHAjjCJ5G,EAAO4nB,UACX,MAAT5nB,GACqB,iBAAVA,IACPA,GAASA,QAGR0gB,UAAU1gB,EAAO4nB,GAEf5jB,OAECA,KAAK0c,eAwiCrB8D,GAAMgE,aAAe5hB,EAAU,0HAp/BtBrG,EAAYyD,KAAKykB,sBACXzkB,KAAKykB,kBAGZlQ,EAAI,MAERpU,EAAWoU,EAAGvU,OACduU,EAAImF,GAAcnF,IAEZpL,GAAI,KACF4R,EAAQxG,EAAE1T,OAASrD,EAAU+W,EAAEpL,IAAMwM,GAAYpB,EAAEpL,SAClDsb,cAAgBzkB,KAAKjB,WACtBiD,EAAcuS,EAAEpL,GAAI4R,EAAM2H,WAAa,YAEtC+B,eAAgB,SAGlBzkB,KAAKykB,qBAi/BZC,GAAU3gB,EAAO5H,mBAiCZwoB,GAAOlnB,EAAQmnB,EAAOC,EAAOC,OAC9BpnB,EAAS8V,KACT3V,EAAML,IAAYwG,IAAI8gB,EAAQF,UAC3BlnB,EAAOmnB,GAAOhnB,EAAKJ,YAGrBsnB,GAAgBtnB,EAAQmnB,EAAOC,MAChCroB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,GAEN,MAATmnB,SACOD,GAAMlnB,EAAQmnB,EAAOC,EAAO,aAGnC/nB,EACAkoB,EAAM,OACLloB,EAAI,EAAGA,EAAI,GAAIA,IAChBkoB,EAAIloB,GAAK6nB,GAAMlnB,EAAQX,EAAG+nB,EAAO,gBAE9BG,WAWFC,GAAkBC,EAAcznB,EAAQmnB,EAAOC,GACxB,kBAAjBK,GACH1oB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,KAGnBmnB,EADAnnB,EAASynB,EAETA,GAAe,EAEX1oB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,QAUnBX,EAPAY,EAAS8V,KACT2R,EAAQD,EAAexnB,EAAOkY,MAAM1I,IAAM,KAEjC,MAAT0X,SACOD,GAAMlnB,GAASmnB,EAAQO,GAAS,EAAGN,EAAO,WAIjDG,EAAM,OACLloB,EAAI,EAAGA,EAAI,EAAGA,IACfkoB,EAAIloB,GAAK6nB,GAAMlnB,GAASX,EAAIqoB,GAAS,EAAGN,EAAO,cAE5CG,EAlGXN,GAAQtT,kBAzgHWnO,EAAKuD,EAAKqO,OACrBpO,EAASzG,KAAKolB,UAAUniB,IAAQjD,KAAKolB,UAAL,gBAC7B1hB,EAAW+C,GAAUA,EAAOpK,KAAKmK,EAAKqO,GAAOpO,GAwgHxDie,GAAQ7d,wBA5/GiB5D,OACjBxF,EAASuC,KAAKqlB,gBAAgBpiB,GAC9BqiB,EAActlB,KAAKqlB,gBAAgBpiB,EAAIsiB,sBAEvC9nB,IAAW6nB,EACJ7nB,QAGN4nB,gBAAgBpiB,GAAOqiB,EAAY/e,QAAQ,oBAAoB,SAAUhG,UACnEA,EAAI2C,MAAM,MAGdlD,KAAKqlB,gBAAgBpiB,KAi/GhCyhB,GAAQ/d,8BA3+GG3G,KAAKwlB,cA4+GhBd,GAAQ3e,iBAt+GUxE,UACPvB,KAAKylB,SAASlf,QAAQ,KAAMhF,IAs+GvCmjB,GAAQ/K,SAAkB8G,GAC1BiE,GAAQpD,WAAkBb,GAC1BiE,GAAQxS,sBAp9Ge3Q,EAAQggB,EAAetK,EAAQyO,OAC9Cjf,EAASzG,KAAK2lB,cAAc1O,UACxBvT,EAAW+C,GACfA,EAAOlF,EAAQggB,EAAetK,EAAQyO,GACtCjf,EAAOF,QAAQ,MAAOhF,IAi9G9BmjB,GAAQkB,oBA98GazI,EAAM1W,OACnBhJ,EAASuC,KAAK2lB,cAAcxI,EAAO,EAAI,SAAW,eAC/CzZ,EAAWjG,GAAUA,EAAOgJ,GAAUhJ,EAAO8I,QAAQ,MAAOE,IA68GvEie,GAAQ1gB,aA7lHM9C,OACNZ,EAAMxD,MACLA,KAAKoE,EAEFwC,EADJpD,EAAOY,EAAOpE,SAELA,GAAKwD,OAEL,IAAMxD,GAAKwD,OAGnBqT,QAAUzS,OAIVgf,+BAAiC,IAAI5X,QACrCtI,KAAKggB,wBAAwB6F,QAAU7lB,KAAKigB,cAAc4F,QACvD,IAAO,UAAWA,SA+kH9BnB,GAAQna,gBAxhGexM,EAAGN,UACjBM,EAIEhC,EAAQiE,KAAKkc,SAAWlc,KAAKkc,QAAQne,EAAEiM,SAC1ChK,KAAKkc,SAASlc,KAAKkc,QAAQ4J,UAAYnb,IAAkB5D,KAAKtJ,GAAU,SAAW,cAAcM,EAAEiM,SAJ5FjO,EAAQiE,KAAKkc,SAAWlc,KAAKkc,QAChClc,KAAKkc,QAAL,YAshGZwI,GAAQpa,qBA/gGoBvM,EAAGN,UACtBM,EAIEhC,EAAQiE,KAAK+lB,cAAgB/lB,KAAK+lB,aAAahoB,EAAEiM,SACpDhK,KAAK+lB,aAAapb,GAAiB5D,KAAKtJ,GAAU,SAAW,cAAcM,EAAEiM,SAJtEjO,EAAQiE,KAAK+lB,cAAgB/lB,KAAK+lB,aACrC/lB,KAAK+lB,aAAL,YA6gGZrB,GAAQha,qBA99FoBM,EAAWvN,EAAQE,OACvCb,EAAG0J,EAAK0B,KAERlI,KAAKgmB,yBACEjb,GAAkB1O,KAAK2D,KAAMgL,EAAWvN,EAAQE,OAGtDqC,KAAKoL,oBACDA,aAAe,QACfC,iBAAmB,QACnBC,kBAAoB,IAMxBxO,EAAI,EAAGA,EAAI,GAAIA,IAAK,IAErB0J,EAAMhJ,EAAU,CAAC,IAAMV,IACnBa,IAAWqC,KAAKqL,iBAAiBvO,UAC5BuO,iBAAiBvO,GAAK,IAAIwL,OAAO,IAAMtI,KAAKuK,OAAO/D,EAAK,IAAID,QAAQ,IAAK,IAAM,IAAK,UACpF+E,kBAAkBxO,GAAK,IAAIwL,OAAO,IAAMtI,KAAKsK,YAAY9D,EAAK,IAAID,QAAQ,IAAK,IAAM,IAAK,MAE9F5I,GAAWqC,KAAKoL,aAAatO,KAC9BoL,EAAQ,IAAMlI,KAAKuK,OAAO/D,EAAK,IAAM,KAAOxG,KAAKsK,YAAY9D,EAAK,SAC7D4E,aAAatO,GAAK,IAAIwL,OAAOJ,EAAM3B,QAAQ,IAAK,IAAK,MAG1D5I,GAAqB,SAAXF,GAAqBuC,KAAKqL,iBAAiBvO,GAAGiK,KAAKiE,UACtDlO,EACJ,GAAIa,GAAqB,QAAXF,GAAoBuC,KAAKsL,kBAAkBxO,GAAGiK,KAAKiE,UAC7DlO,EACJ,IAAKa,GAAUqC,KAAKoL,aAAatO,GAAGiK,KAAKiE,UACrClO,IA87FnB4nB,GAAQja,qBA33FcrC,UACdpI,KAAKgmB,mBACA9oB,EAAW8C,KAAM,iBAClB4L,GAAmBvP,KAAK2D,MAExBoI,EACOpI,KAAKoM,mBAELpM,KAAKkM,eAGXhP,EAAW8C,KAAM,uBACbkM,aAAeP,IAEjB3L,KAAKoM,oBAAsBhE,EAC9BpI,KAAKoM,mBAAqBpM,KAAKkM,eA62F3CwY,GAAQla,0BAh5FmBpC,UACnBpI,KAAKgmB,mBACA9oB,EAAW8C,KAAM,iBAClB4L,GAAmBvP,KAAK2D,MAExBoI,EACOpI,KAAKqM,wBAELrM,KAAKmM,oBAGXjP,EAAW8C,KAAM,4BACbmM,kBAAoBT,IAEtB1L,KAAKqM,yBAA2BjE,EACnCpI,KAAKqM,wBAA0BrM,KAAKmM,oBAk4FhDuY,GAAQnX,cAxsFa/G,UACVoH,GAAWpH,EAAKxG,KAAK4V,MAAM1I,IAAKlN,KAAK4V,MAAMzI,KAAKI,MAwsF3DmX,GAAQuB,iCA3rFGjmB,KAAK4V,MAAMzI,KA4rFtBuX,GAAQwB,iCAhsFGlmB,KAAK4V,MAAM1I,KAksFtBwX,GAAQnW,kBAplFiBxQ,EAAGN,OACpB8Q,EAAWxS,EAAQiE,KAAKmmB,WAAanmB,KAAKmmB,UAC1CnmB,KAAKmmB,UAAWpoB,IAAW,IAANA,GAAciC,KAAKmmB,UAAUL,SAAS/e,KAAKtJ,GAAW,SAAW,qBAC5E,IAANM,EAAckQ,GAAcM,EAAUvO,KAAK4V,MAAM1I,KAClDnP,EAAKwQ,EAASxQ,EAAEqR,OAASb,GAilFpCmW,GAAQrW,qBAvkFoBtQ,UACV,IAANA,EAAckQ,GAAcjO,KAAKomB,aAAcpmB,KAAK4V,MAAM1I,KAC3DnP,EAAKiC,KAAKomB,aAAaroB,EAAEqR,OAASpP,KAAKomB,cAskFlD1B,GAAQpW,uBA9kFsBvQ,UACZ,IAANA,EAAckQ,GAAcjO,KAAKqmB,eAAgBrmB,KAAK4V,MAAM1I,KAC7DnP,EAAKiC,KAAKqmB,eAAetoB,EAAEqR,OAASpP,KAAKqmB,gBA6kFpD3B,GAAQ/V,uBApgFsBK,EAAavR,EAAQE,OAC3Cb,EAAG0J,EAAK0B,KAERlI,KAAKsmB,2BACEvX,GAAoB1S,KAAK2D,KAAMgP,EAAavR,EAAQE,OAG1DqC,KAAKiP,sBACDA,eAAiB,QACjBE,kBAAoB,QACpBD,oBAAsB,QACtBqX,mBAAqB,IAGzBzpB,EAAI,EAAGA,EAAI,EAAGA,IAAK,IAGpB0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,GAC3Ba,IAAWqC,KAAKumB,mBAAmBzpB,UAC9BypB,mBAAmBzpB,GAAK,IAAIwL,OAAO,IAAMtI,KAAKuO,SAAS/H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,UAC5F2I,oBAAoBpS,GAAK,IAAIwL,OAAO,IAAMtI,KAAKsO,cAAc9H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,UAClG4I,kBAAkBrS,GAAK,IAAIwL,OAAO,IAAMtI,KAAKqO,YAAY7H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,MAElGvG,KAAKiP,eAAenS,KACrBoL,EAAQ,IAAMlI,KAAKuO,SAAS/H,EAAK,IAAM,KAAOxG,KAAKsO,cAAc9H,EAAK,IAAM,KAAOxG,KAAKqO,YAAY7H,EAAK,SACpGyI,eAAenS,GAAK,IAAIwL,OAAOJ,EAAM3B,QAAQ,IAAK,IAAK,MAG5D5I,GAAqB,SAAXF,GAAqBuC,KAAKumB,mBAAmBzpB,GAAGiK,KAAKiI,UACxDlS,EACJ,GAAIa,GAAqB,QAAXF,GAAoBuC,KAAKkP,oBAAoBpS,GAAGiK,KAAKiI,UAC/DlS,EACJ,GAAIa,GAAqB,OAAXF,GAAmBuC,KAAKmP,kBAAkBrS,GAAGiK,KAAKiI,UAC5DlS,EACJ,IAAKa,GAAUqC,KAAKiP,eAAenS,GAAGiK,KAAKiI,UACvClS,IAm+EnB4nB,GAAQhW,uBAr7EgBtG,UAChBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKgQ,qBAELhQ,KAAK6P,iBAGX3S,EAAW8C,KAAM,yBACb6P,eAAiBR,IAEnBrP,KAAKgQ,sBAAwB5H,EAChCpI,KAAKgQ,qBAAuBhQ,KAAK6P,iBAu6E7C6U,GAAQjW,4BAl6EqBrG,UACrBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKiQ,0BAELjQ,KAAK8P,sBAGX5S,EAAW8C,KAAM,8BACb8P,oBAAsBR,IAExBtP,KAAKiQ,2BAA6B7H,EACrCpI,KAAKiQ,0BAA4BjQ,KAAK8P,sBAo5ElD4U,GAAQlW,0BA/4EmBpG,UACnBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKkQ,wBAELlQ,KAAK+P,oBAGX7S,EAAW8C,KAAM,4BACb+P,kBAAoBR,IAEtBvP,KAAKkQ,yBAA2B9H,EACnCpI,KAAKkQ,wBAA0BlQ,KAAK+P,oBAk4EhD2U,GAAQ9T,cAtuEa5U,SAGgC,OAAxCA,EAAQ,IAAIsI,cAAckiB,OAAO,IAouE9C9B,GAAQ9lB,kBAhuEiBwR,EAAOE,EAASmW,UACjCrW,EAAQ,GACDqW,EAAU,KAAO,KAEjBA,EAAU,KAAO,MAwzEhCrT,GAAmB,KAAM,CACrBnB,uBAAwB,uBACxBlM,QAAU,SAAUxE,OACZnE,EAAImE,EAAS,UAKVA,GAJoC,IAA7BI,EAAMJ,EAAS,IAAM,IAAa,KACrC,IAANnE,EAAW,KACL,IAANA,EAAW,KACL,IAANA,EAAW,KAAO,SAO/BxB,EAAMujB,KAAOvc,EAAU,wDAAyDwQ,IAChFxX,EAAM8qB,SAAW9jB,EAAU,gEAAiE4Q,QAExFmT,GAAUnlB,KAAKc,aAmBVskB,GAAevL,EAAUrf,EAAO8F,EAAOwc,OACxCvD,EAAQ6C,GAAe5hB,EAAO8F,UAElCuZ,EAASW,eAAiBsC,EAAYvD,EAAMiB,cAC5CX,EAASY,OAAiBqC,EAAYvD,EAAMkB,MAC5CZ,EAASa,SAAiBoC,EAAYvD,EAAMmB,QAErCb,EAASe,mBAaXyK,GAAStlB,UACVA,EAAS,EACFC,KAAKE,MAAMH,GAEXC,KAAKC,KAAKF,YAmDhBulB,GAAcnL,UAGL,KAAPA,EAAc,gBAGhBoL,GAAcxc,UAEH,OAATA,EAAkB,cAkDpByc,GAAQC,UACN,kBACIjnB,KAAKknB,GAAGD,QAInBE,GAAiBH,GAAO,MACxBI,GAAiBJ,GAAO,KACxBK,GAAiBL,GAAO,KACxBM,GAAiBN,GAAO,KACxBO,GAAiBP,GAAO,KACxBQ,GAAiBR,GAAO,KACxBS,GAAiBT,GAAO,KACxBU,GAAiBV,GAAO,KACxBW,GAAiBX,GAAO,cAWnBY,GAAWnkB,UACT,kBACIzD,KAAKjB,UAAYiB,KAAKmc,MAAM1Y,GAAQ5D,SAI/C+b,GAAegM,GAAW,gBAC1BnX,GAAemX,GAAW,WAC1BtX,GAAesX,GAAW,WAC1BxX,GAAewX,GAAW,SAC1BjM,GAAeiM,GAAW,QAC1Brd,GAAeqd,GAAW,UAC1BtM,GAAesM,GAAW,SAM1BrL,GAAQ/a,KAAK+a,MACbsL,GAAa,CACbxV,GAAI,GACJxJ,EAAI,GACJ9K,EAAI,GACJyO,EAAI,GACJD,EAAI,GACJE,EAAI,aAICqb,GAAkB7Q,EAAQ1V,EAAQggB,EAAemE,EAAUhoB,UACzDA,EAAOwU,aAAa3Q,GAAU,IAAKggB,EAAetK,EAAQyO,YAG5DqC,GAAgBC,EAAgBzG,EAAe7jB,OAChD2d,EAAWuC,GAAeoK,GAAgB1lB,MAC1CmO,EAAW8L,GAAMlB,EAAS6L,GAAG,MAC7B5W,EAAWiM,GAAMlB,EAAS6L,GAAG,MAC7B9W,EAAWmM,GAAMlB,EAAS6L,GAAG,MAC7BvL,EAAWY,GAAMlB,EAAS6L,GAAG,MAC7B3c,EAAWgS,GAAMlB,EAAS6L,GAAG,MAC7B5L,EAAWiB,GAAMlB,EAAS6L,GAAG,MAE7B/pB,EAAIsT,GAAWoX,GAAWxV,IAAM,CAAC,IAAK5B,IAClCA,EAAUoX,GAAWhf,GAAO,CAAC,KAAM4H,IACnCH,GAAW,GAAiB,CAAC,MAC7BA,EAAUuX,GAAW9pB,GAAO,CAAC,KAAMuS,IACnCF,GAAW,GAAiB,CAAC,MAC7BA,EAAUyX,GAAWrb,GAAO,CAAC,KAAM4D,IACnCuL,GAAW,GAAiB,CAAC,MAC7BA,EAAUkM,GAAWtb,GAAO,CAAC,KAAMoP,IACnCpR,GAAW,GAAiB,CAAC,MAC7BA,EAAUsd,GAAWpb,GAAO,CAAC,KAAMlC,IACnC+Q,GAAW,GAAiB,CAAC,MAAkB,CAAC,KAAMA,UAE9Dne,EAAE,GAAKokB,EACPpkB,EAAE,IAAM6qB,EAAiB,EACzB7qB,EAAE,GAAKO,EACAoqB,GAAkBjsB,MAAM,KAAMsB,OA6CrC8qB,GAAQzmB,KAAKc,aAERqa,GAAKxS,UACDA,EAAI,IAAMA,EAAI,KAAQA,WAG1B+d,SAQAloB,KAAKjB,iBACCiB,KAAKkG,aAAaS,kBAMzB2J,EAASF,EAHTK,EAAUwX,GAAMjoB,KAAKgc,eAAiB,IACtCL,EAAesM,GAAMjoB,KAAKic,OAC1B1R,EAAe0d,GAAMjoB,KAAKkc,SAI9B5L,EAAoBhP,EAASmP,EAAU,IACvCL,EAAoB9O,EAASgP,EAAU,IACvCG,GAAW,GACXH,GAAW,OAQP6X,EALK7mB,EAASiJ,EAAS,IAMvBkC,EALJlC,GAAU,GAMN6d,EAAIzM,EACJnP,EAAI4D,EACJrS,EAAIuS,EACJzH,EAAI4H,EAAUA,EAAQ4X,QAAQ,GAAG9hB,QAAQ,SAAU,IAAM,GACzD+hB,EAAQtoB,KAAKonB,gBAEZkB,QAGM,UAGPC,EAAYD,EAAQ,EAAI,IAAM,GAC9BE,EAAS7L,GAAK3c,KAAKkc,WAAaS,GAAK2L,GAAS,IAAM,GACpDG,EAAW9L,GAAK3c,KAAKic,SAAWU,GAAK2L,GAAS,IAAM,GACpDI,EAAU/L,GAAK3c,KAAKgc,iBAAmBW,GAAK2L,GAAS,IAAM,UAExDC,EAAY,KACdJ,EAAIK,EAASL,EAAI,IAAM,KACvB1b,EAAI+b,EAAS/b,EAAI,IAAM,KACvB2b,EAAIK,EAAWL,EAAI,IAAM,KACxB5b,GAAKzO,GAAK8K,EAAK,IAAM,KACtB2D,EAAIkc,EAAUlc,EAAI,IAAM,KACxBzO,EAAI2qB,EAAU3qB,EAAI,IAAM,KACxB8K,EAAI6f,EAAU7f,EAAI,IAAM,QAG7B8f,GAAUvN,GAASjf,iBAEvBwsB,GAAQ5pB,0BA3tDGiB,KAAKhB,UA4tDhB2pB,GAAQrmB,mBAvVAiR,EAAiBvT,KAAKmc,kBAErBH,cAAgB2K,GAAQ3mB,KAAKgc,oBAC7BC,MAAgB0K,GAAQ3mB,KAAKic,YAC7BC,QAAgByK,GAAQ3mB,KAAKkc,SAElC3I,EAAKqI,aAAgB+K,GAAQpT,EAAKqI,cAClCrI,EAAK9C,QAAgBkW,GAAQpT,EAAK9C,SAClC8C,EAAKjD,QAAgBqW,GAAQpT,EAAKjD,SAClCiD,EAAKnD,MAAgBuW,GAAQpT,EAAKnD,OAClCmD,EAAKhJ,OAAgBoc,GAAQpT,EAAKhJ,QAClCgJ,EAAK+H,MAAgBqL,GAAQpT,EAAK+H,OAE3Btb,MA2UX2oB,GAAQ9N,aA7TQ7e,EAAO8F,UACZ8kB,GAAc5mB,KAAMhE,EAAO8F,EAAO,IA6T7C6mB,GAAQ/J,kBAzTa5iB,EAAO8F,UACjB8kB,GAAc5mB,KAAMhE,EAAO8F,GAAQ,IAyT9C6mB,GAAQzB,YApPK1iB,OACJxE,KAAKjB,iBACCc,QAEP8b,EACApR,EACAqR,EAAe5b,KAAKgc,iBAIV,WAFdxX,EAAQD,EAAeC,KAEY,YAAVA,GAAiC,SAAVA,SAC5CmX,EAAO3b,KAAKic,MAAQL,EAAe,MACnCrR,EAASvK,KAAKkc,QAAU4K,GAAanL,GAC7BnX,OACC,eAAkB+F,MAClB,iBAAkBA,EAAS,MAC3B,cAAkBA,EAAS,eAIpCoR,EAAO3b,KAAKic,MAAQza,KAAK+a,MAAMwK,GAAa/mB,KAAKkc,UACzC1X,OACC,cAAkBmX,EAAO,EAAQC,EAAe,WAChD,aAAkBD,EAAeC,EAAe,UAChD,cAAyB,GAAPD,EAAeC,EAAe,SAChD,gBAAyB,KAAPD,EAAeC,EAAe,QAChD,gBAAyB,MAAPD,EAAeC,EAAe,QAEhD,qBAAsBpa,KAAKE,MAAa,MAAPia,GAAgBC,gBACvC,IAAIxY,MAAM,gBAAkBoB,KAwNvDmkB,GAAQxB,eAAiBA,GACzBwB,GAAQvB,UAAiBA,GACzBuB,GAAQtB,UAAiBA,GACzBsB,GAAQrB,QAAiBA,GACzBqB,GAAQpB,OAAiBA,GACzBoB,GAAQnB,QAAiBA,GACzBmB,GAAQlB,SAAiBA,GACzBkB,GAAQjB,WAAiBA,GACzBiB,GAAQhB,QAAiBA,GACzBgB,GAAQprB,0BA1NCyC,KAAKjB,UAINiB,KAAKgc,cACQ,MAAbhc,KAAKic,MACJjc,KAAKkc,QAAU,GAAM,OACK,QAA3Bva,EAAM3B,KAAKkc,QAAU,IANdrc,KA0Nf8oB,GAAQvM,uBApTA3L,EAASH,EAASF,EAAOkL,EAAOsN,EAJhChN,EAAe5b,KAAKgc,cACpBL,EAAe3b,KAAKic,MACpB1R,EAAevK,KAAKkc,QACpB3I,EAAevT,KAAKmc,aAKjBP,GAAgB,GAAKD,GAAQ,GAAKpR,GAAU,GAC1CqR,GAAgB,GAAKD,GAAQ,GAAKpR,GAAU,IACjDqR,GAAuD,MAAvCiL,GAAQE,GAAaxc,GAAUoR,GAC/CA,EAAO,EACPpR,EAAS,GAKbgJ,EAAKqI,aAAeA,EAAe,IAEnCnL,EAAoBnP,EAASsa,EAAe,KAC5CrI,EAAK9C,QAAeA,EAAU,GAE9BH,EAAoBhP,EAASmP,EAAU,IACvC8C,EAAKjD,QAAeA,EAAU,GAE9BF,EAAoB9O,EAASgP,EAAU,IACvCiD,EAAKnD,MAAeA,EAAQ,GAE5BuL,GAAQra,EAAS8O,EAAQ,IAGzBwY,EAAiBtnB,EAASwlB,GAAanL,IACvCpR,GAAUqe,EACVjN,GAAQkL,GAAQE,GAAa6B,IAG7BtN,EAAQha,EAASiJ,EAAS,IAC1BA,GAAU,GAEVgJ,EAAKoI,KAASA,EACdpI,EAAKhJ,OAASA,EACdgJ,EAAK+H,MAASA,EAEPtb,MA8QX2oB,GAAQvL,wBAhMGQ,GAAe5d,OAiM1B2oB,GAAQ5e,aA9LQvF,UACZA,EAAQD,EAAeC,GAChBxE,KAAKjB,UAAYiB,KAAKwE,EAAQ,OAAS3E,KA6LlD8oB,GAAQ/M,aAAiBA,GACzB+M,GAAQlY,QAAiBA,GACzBkY,GAAQrY,QAAiBA,GACzBqY,GAAQvY,MAAiBA,GACzBuY,GAAQhN,KAAiBA,GACzBgN,GAAQlN,wBAhLGna,EAAStB,KAAK2b,OAAS,IAiLlCgN,GAAQpe,OAAiBA,GACzBoe,GAAQrN,MAAiBA,GACzBqN,GAAQnH,kBA3GWqH,OACV7oB,KAAKjB,iBACCiB,KAAKkG,aAAaS,kBAGzBjJ,EAASsC,KAAKkG,aACdO,EAASshB,GAAe/nB,MAAO6oB,EAAYnrB,UAE3CmrB,IACApiB,EAAS/I,EAAOkoB,YAAY5lB,KAAMyG,IAG/B/I,EAAO4jB,WAAW7a,IAgG7BkiB,GAAQ9F,YAAiBqF,GACzBS,GAAQvsB,SAAiB8rB,GACzBS,GAAQvF,OAAiB8E,GACzBS,GAAQjrB,OAAiBA,GACzBirB,GAAQziB,WAAiBA,GAEzByiB,GAAQG,YAAclmB,EAAU,sFAAuFslB,IACvHS,GAAQxJ,KAAOA,GAMfvZ,EAAe,IAAK,EAAG,EAAG,QAC1BA,EAAe,IAAK,EAAG,EAAG,WAI1BqC,GAAc,IAAKL,IACnBK,GAAc,IAlzHO,wBAmzHrBc,GAAc,KAAK,SAAU/M,EAAOqK,EAAOnF,GACvCA,EAAO7B,GAAK,IAAI3C,KAA6B,IAAxBof,WAAW9f,EAAO,QAE3C+M,GAAc,KAAK,SAAU/M,EAAOqK,EAAOnF,GACvCA,EAAO7B,GAAK,IAAI3C,KAAKiF,EAAM3F,OAM/BJ,EAAMmtB,QAAU,SAz7IZrtB,EA27IYia,GAEhB/Z,EAAMiB,GAAwB2jB,GAC9B5kB,EAAMwG,mBAj0DEY,EAAO,GAAGE,MAAM7G,KAAKP,UAAW,UAE7Bmf,GAAO,WAAYjY,IAg0D9BpH,EAAM0J,mBA5zDEtC,EAAO,GAAGE,MAAM7G,KAAKP,UAAW,UAE7Bmf,GAAO,UAAWjY,IA2zD7BpH,EAAMiZ,IAxzDI,kBACCnY,KAAKmY,IAAMnY,KAAKmY,OAAU,IAAInY,MAwzDzCd,EAAMiC,IAAwBL,EAC9B5B,EAAMynB,cAnjBernB,UACV2Z,GAAoB,IAAR3Z,IAmjBvBJ,EAAM2O,gBAjce9M,EAAQmnB,UAClBG,GAAetnB,EAAQmnB,EAAO,WAiczChpB,EAAMa,OAAwBA,EAC9Bb,EAAM8B,OAAwB0V,GAC9BxX,EAAM+iB,QAAwB/e,EAC9BhE,EAAMyf,SAAwBuC,GAC9BhiB,EAAMwF,SAAwBA,EAC9BxF,EAAM2S,kBA/biB2W,EAAcznB,EAAQmnB,UAClCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,aA+bzDhpB,EAAMooB,4BAtjBKrO,GAAY9Z,MAAM,KAAMC,WAAWkoB,aAujB9CpoB,EAAMsK,WAAwBsN,GAC9B5X,EAAMygB,WAAwBA,GAC9BzgB,EAAM0O,qBAvcoB7M,EAAQmnB,UACvBG,GAAetnB,EAAQmnB,EAAO,gBAuczChpB,EAAMyS,qBA5boB6W,EAAcznB,EAAQmnB,UACrCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,gBA4bzDhpB,EAAM6X,aAAwBA,GAC9B7X,EAAMotB,sBA9kFgBvlB,EAAMvC,MACV,MAAVA,EAAgB,KACZxD,EAAQurB,EAAWplB,EAAesN,GAGrB,OADjB8X,EAAYlW,GAAWtP,MAEnBI,EAAeolB,EAAUtV,UAG7BjW,EAAS,IAAIqG,EADb7C,EAAS0C,EAAaC,EAAc3C,KAE7B0S,aAAehB,GAAQnP,GAC9BmP,GAAQnP,GAAQ/F,EAGhB0V,GAAmB3P,QAGE,MAAjBmP,GAAQnP,KAC0B,MAA9BmP,GAAQnP,GAAMmQ,aACdhB,GAAQnP,GAAQmP,GAAQnP,GAAMmQ,aACN,MAAjBhB,GAAQnP,WACRmP,GAAQnP,WAIpBmP,GAAQnP,IAsjFnB7H,EAAMgX,0BA1hFKtP,EAAKsP,KA2hFhBhX,EAAM0S,uBApcsB4W,EAAcznB,EAAQmnB,UACvCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,kBAoczDhpB,EAAM2I,eAAwBA,EAC9B3I,EAAMstB,8BA/L+BC,eACR1pB,IAArB0pB,EACO5M,GAEsB,mBAAtB4M,IACP5M,GAAQ4M,GACD,IA0LfvtB,EAAMwtB,+BApLgCC,EAAWC,eACf7pB,IAA1BooB,GAAWwB,UAGD5pB,IAAV6pB,EACOzB,GAAWwB,IAEtBxB,GAAWwB,GAAaC,EACN,MAAdD,IACAxB,GAAWxV,GAAKiX,EAAQ,IAErB,KA0KX1tB,EAAMklB,wBA73CqByI,EAAU1U,OAC7BsI,EAAOoM,EAASpM,KAAKtI,EAAK,QAAQ,UAC/BsI,GAAQ,EAAI,WACXA,GAAQ,EAAI,WACZA,EAAO,EAAI,UACXA,EAAO,EAAI,UACXA,EAAO,EAAI,UACXA,EAAO,EAAI,WAAa,YAu3CpCvhB,EAAMO,UAAwBqkB,GAG9B5kB,EAAM4tB,UAAY,CACdC,eAAgB,mBAChBC,uBAAwB,sBACxBC,kBAAmB,0BACnBC,KAAM,aACNC,KAAM,QACNC,aAAc,WACdC,QAAS,eACTC,KAAM,aACNC,MAAO,WAGJruB,EAp/IyEsuB,MCHhFC,qBCOK,SAAUC,YAGNC,EAAoB9oB,EAAQggB,EAAete,EAAKyiB,OACjDjoB,EAAS,GACJ,CAAC,cAAe,kBAChB,CAAC,cAAe,kBAChB,CAAC,UAAW,gBACX,CAAC8D,EAAS,QAASA,EAAS,YAC7B,CAAC,YAAa,kBACb,CAACA,EAAS,UAAWA,EAAS,cAC/B,CAAC,WAAY,iBACZ,CAACA,EAAS,SAAUA,EAAS,mBAEhCggB,EAAgB9jB,EAAOwF,GAAK,GAAKxF,EAAOwF,GAAK,GAG/CmnB,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,qFAAqFM,MAAM,KACpGP,YAAc,6DAA6DO,MAAM,KACjFyf,kBAAmB,EACnB/b,SAAW,8DAA8D1D,MAAM,KAC/EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAI,QACJD,IAAK,WACLE,EAAI,aACJC,GAAK,eACLC,IAAM,qBACNC,KAAO,4BAEXZ,SAAW,CACPC,QAAS,sBACTK,SAAU,IACVJ,QAAS,uBACTC,SAAU,qBACVC,QAAS,wBACTC,SAAU,gCAEdS,aAAe,CACXC,OAAS,QACTC,KAAO,SACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAIssB,EACJ/X,GAAK,aACL9F,EAAI6d,EACJ9X,GAAK,aACLhG,EAAI8d,EACJ7X,GAAK6X,EACL5d,EAAI4d,EACJ5X,GAAK4X,EACL9gB,EAAI8gB,EACJ3X,GAAK2X,GAETpY,uBAAwB,YACxBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAhEwB+c,CAAQM,wBCGzC,SAAUJ,OAGXK,EAAiB,8DAA8D5f,MAAM,KACrFP,EAAc,kDAAkDO,MAAM,KAEtEH,EAAc,CAAC,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,SAClHD,EAAc,mLAET2f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,2FAA2FM,MAAM,KAC1GP,YAAc,SAAUvM,EAAGN,UAClBM,EAEM,QAAQgJ,KAAKtJ,GACb6M,EAAYvM,EAAEiM,SAEdygB,EAAe1sB,EAAEiM,SAJjBygB,GAOfhgB,YAAcA,EACdD,iBAAmBC,EACnBigB,kBAAoB,+FACpBC,uBAAyB,0FACzBjgB,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EACnB6D,SAAW,uDAAuD1D,MAAM,KACxEyD,cAAgB,qCAAqCzD,MAAM,KAC3DwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,OACLD,IAAM,UACNE,EAAI,aACJC,GAAK,wBACLC,IAAM,6BACNC,KAAO,oCAEXZ,SAAW,CACPC,QAAU,iBACC,aAAiC,IAAjBrR,KAAKoQ,QAAiB,IAAM,IAAM,QAE7DkB,QAAU,iBACC,gBAAoC,IAAjBtR,KAAKoQ,QAAiB,IAAM,IAAM,QAEhEmB,SAAW,iBACA,cAAkC,IAAjBvR,KAAKoQ,QAAiB,IAAM,IAAM,QAE9DoB,QAAU,iBACC,cAAkC,IAAjBxR,KAAKoQ,QAAiB,IAAM,IAAM,QAE9DqB,SAAW,iBACA,0BAA8C,IAAjBzR,KAAKoQ,QAAiB,IAAM,IAAM,QAE1EsB,SAAW,KAEfQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,gBACJwJ,GAAK,cACLtU,EAAI,YACJuU,GAAK,aACL9F,EAAI,WACJ+F,GAAK,WACLhG,EAAI,SACJiG,GAAK,UACL/F,EAAI,SACJgG,GAAK,WACLlJ,EAAI,SACJmJ,GAAK,WAETT,uBAAyB,WACzBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAhFwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,uFAAuFM,MAAM,KACtGP,YAAc,iEAAiEO,MAAM,KACrFyf,kBAAmB,EACnB/b,SAAW,sDAAsD1D,MAAM,KACvEyD,cAAgB,qCAAqCzD,MAAM,KAC3DwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAU,qBACVC,QAAU,gBACVC,SAAW,cACXC,QAAU,cACVC,SAAW,sBACXC,SAAW,KAEfQ,aAAe,CACXC,OAAS,UACTC,KAAO,YACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAI,aACJuU,GAAK,aACL9F,EAAI,YACJ+F,GAAK,YACLhG,EAAI,UACJiG,GAAK,WACL/F,EAAI,UACJgG,GAAK,UACLlJ,EAAI,QACJmJ,GAAK,UAETT,uBAAwB,eACxBlM,QAAU,SAAUxE,EAAQgd,UAChBA,OAIC,WACMhd,GAAqB,IAAXA,EAAe,KAAO,gBAItC,QACA,QACA,UACA,WACMA,GAAqB,IAAXA,EAAe,KAAO,SAGtC,QACA,WACMA,GAAqB,IAAXA,EAAe,KAAO,OAGnDgM,KAAO,CACHL,IAAM,EACNC,IAAM,MAvEwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,gGAAgGM,MAAM,KAC/GP,YAAc,kDAAkDO,MAAM,KACtE0D,SAAW,2DAA2D1D,MAAM,KAC5EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAS,iBACTC,QAAS,mBACTC,SAAU,iBACVC,QAAS,iBACTC,SAAU,kBACEzR,KAAKoP,YACJ,QACM,2CAEA,+BAGnBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,SAAUtJ,UACN,YAAa9B,KAAK8B,GAAK,MAAQ,MAAQ,IAAMA,GAE1DuJ,KAAO,QACPvJ,EAAI,iBACJwJ,GAAK,aACLtU,EAAI,YACJuU,GAAK,YACL9F,EAAI,SACJ+F,GAAK,SACLhG,EAAI,YACJiG,GAAK,YACL/F,EAAI,UACJgG,GAAK,UACLlJ,EAAI,UACJmJ,GAAK,WAETT,uBAAyB,WACzBlM,QAAS,MACTwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAzDwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,wCAAwCM,MAAM,KACvDP,YAAc,yCAAyCO,MAAM,KAC7D0D,SAAW,8BAA8B1D,MAAM,KAC/CyD,cAAgB,gBAAgBzD,MAAM,KACtCwD,YAAc,gBAAgBxD,MAAM,KACpChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,YACLC,IAAM,kBACNC,KAAO,uBACP4E,EAAI,aACJkU,GAAK,YACLC,IAAM,kBACNC,KAAO,wBAEXrY,cAAe,SACf/B,KAAO,SAAU5U,SACI,OAAVA,GAEX4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,GACA,KAEA,MAGflI,SAAW,CACPC,QAAU,UACVC,QAAU,UACVC,SAAW,SAAUsD,UACbA,EAAItH,OAASvN,KAAKuN,OACX,cAEA,WAGfiE,QAAU,UACVC,SAAW,SAAUoD,UACb7U,KAAKuN,OAASsH,EAAItH,OACX,cAEA,WAGfmE,SAAW,KAEfO,uBAAyB,WACzBlM,QAAU,SAAUxE,EAAQgd,UAChBA,OACC,QACA,QACA,aACMhd,EAAS,mBAETA,IAGnB2Q,aAAe,CACXC,OAAS,MACTC,KAAO,MACPvJ,EAAI,KACJwJ,GAAK,MACLtU,EAAI,KACJuU,GAAK,MACL9F,EAAI,MACJ+F,GAAK,OACLhG,EAAI,KACJiG,GAAK,MACL/F,EAAI,MACJgG,GAAK,OACLlJ,EAAI,KACJmJ,GAAK,UAhFyBwX,CAAQM,wBCGzC,SAAUJ,OAGXa,EAAsB,6DAA6DpgB,MAAM,KACzFqgB,EAAyB,kDAAkDrgB,MAAM,KAEjFH,EAAc,CAAC,QAAS,QAAS,iBAAkB,QAAS,SAAU,cAAe,cAAe,QAAS,QAAS,QAAS,QAAS,SACxID,EAAc,qKAET2f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,0FAA0FM,MAAM,KACzGP,YAAc,SAAUvM,EAAGN,UAClBM,EAEM,QAAQgJ,KAAKtJ,GACbytB,EAAuBntB,EAAEiM,SAEzBihB,EAAoBltB,EAAEiM,SAJtBihB,GAQfxgB,YAAaA,EACbD,iBAAkBC,EAClBigB,kBAAmB,4FACnBC,uBAAwB,mFAExBjgB,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EAEnB6D,SAAW,6DAA6D1D,MAAM,KAC9EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAS,kBACTC,QAAS,iBACTC,SAAU,eACVC,QAAS,mBACTC,SAAU,2BACVC,SAAU,KAEdQ,aAAe,CACXC,OAAS,UACTC,KAAO,aACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAI,aACJuU,GAAK,aACL9F,EAAI,UACJ+F,GAAK,SACLhG,EAAI,UACJiG,GAAK,WACL/F,EAAI,YACJgG,GAAK,aACLlJ,EAAI,WACJmJ,GAAK,WAETT,uBAAwB,kBACxBlM,QAAU,SAAUxE,UACTA,GAAsB,IAAXA,GAA2B,IAAXA,GAAgBA,GAAU,GAAM,MAAQ,OAE9EgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA3EwB+c,CAAQM,wBCGzC,SAAUJ,OAGXe,EAAmB,mGAAmGtgB,MAAM,KAC5HugB,EAAmB,qGAAqGvgB,MAAM,cACzHwgB,EAAOld,UACJA,EAAI,GAAK,GAAOA,EAAI,GAAK,MAAWA,EAAI,IAAM,IAAQ,WAEzDmd,EAAU/pB,EAAQggB,EAAete,OAClCsV,EAAShX,EAAS,WACd0B,OACC,YACMsV,GAAU8S,EAAO9pB,GAAU,UAAY,cAC7C,WACMggB,EAAgB,SAAW,aACjC,YACMhJ,GAAU8S,EAAO9pB,GAAU,SAAW,aAC5C,WACMggB,EAAiB,UAAa,cACpC,YACMhJ,GAAU8S,EAAO9pB,GAAU,UAAY,cAC7C,YACMgX,GAAU8S,EAAO9pB,GAAU,WAAa,gBAC9C,YACMgX,GAAU8S,EAAO9pB,GAAU,OAAS,QAI9C6oB,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,SAAUghB,EAAgB9tB,UAC1B8tB,EAEiB,KAAX9tB,EAIA,IAAM2tB,EAAiBG,EAAevhB,SAAW,IAAMmhB,EAAiBI,EAAevhB,SAAW,IAClG,SAASjD,KAAKtJ,GACd2tB,EAAiBG,EAAevhB,SAEhCmhB,EAAiBI,EAAevhB,SAThCmhB,GAYf7gB,YAAc,kDAAkDO,MAAM,KACtE0D,SAAW,6DAA6D1D,MAAM,KAC9EyD,cAAgB,2BAA2BzD,MAAM,KACjDwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,2BAEXZ,SAAW,CACPC,QAAS,cACTC,QAAS,eACTC,SAAU,kBACEvR,KAAKoP,YACJ,QACM,0BAEN,QACM,wBAEN,QACM,sBAEN,QACM,gCAGA,oBAGnBoC,QAAS,iBACTC,SAAU,kBACEzR,KAAKoP,YACJ,QACM,iCACN,QACM,6BACN,QACM,uCAEA,2BAGnBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,eACJwJ,GAAKiZ,EACLvtB,EAAIutB,EACJhZ,GAAKgZ,EACL9e,EAAI8e,EACJ/Y,GAAK+Y,EACL/e,EAAI,UACJiG,GAAK,SACL/F,EAAI,UACJgG,GAAK6Y,EACL/hB,EAAI,MACJmJ,GAAK4Y,GAETrZ,uBAAwB,YACxBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAlHwB+c,CAAQM,wBCGzC,SAAUJ,YAONoB,EAAuBjqB,EAAQggB,EAAete,OAJjCwoB,EACdC,QAYQ,MAARzoB,EACOse,EAAgB,SAAW,SAG3BhgB,EAAS,KAjBFkqB,GAiB6BlqB,EAhB3CmqB,EAIS,IACHnK,EAAgB,yBAA2B,4BAC3CA,EAAgB,sBAAwB,yBACxC,oBACA,mBACA,0BACA,gBAM8Bte,GAhBvB4H,MAAM,KAChB4gB,EAAM,IAAO,GAAKA,EAAM,KAAQ,GAAKC,EAAM,GAAMD,EAAM,IAAM,GAAKA,EAAM,IAAM,IAAMA,EAAM,IAAM,IAAMA,EAAM,KAAO,IAAMC,EAAM,GAAKA,EAAM,QAkBjJhhB,EAAc,CAAC,QAAS,QAAS,QAAS,QAAS,WAAY,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,SAKhH0f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,CACL9M,OAAQ,oFAAoFoN,MAAM,KAClG8gB,WAAY,kFAAkF9gB,MAAM,MAExGP,YAAc,CAEV7M,OAAQ,gEAAgEoN,MAAM,KAC9E8gB,WAAY,gEAAgE9gB,MAAM,MAEtF0D,SAAW,CACPod,WAAY,gEAAgE9gB,MAAM,KAClFpN,OAAQ,gEAAgEoN,MAAM,KAC9Eib,SAAU,kDAEdxX,cAAgB,uBAAuBzD,MAAM,KAC7CwD,YAAc,uBAAuBxD,MAAM,KAC3CH,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EAGnBD,YAAa,2MAGbD,iBAAkB,2MAGlBkgB,kBAAmB,wHAGnBC,uBAAwB,6FACxB9jB,eAAiB,CACb+K,GAAK,OACLD,IAAM,UACNE,EAAI,aACJC,GAAK,iBACLC,IAAM,uBACNC,KAAO,8BAEXZ,SAAW,CACPC,QAAS,kBACTC,QAAS,iBACTE,QAAS,gBACTD,SAAU,SAAUsD,MACZA,EAAItH,SAAWvN,KAAKuN,cAcD,IAAfvN,KAAKoP,MACE,oBAEA,0BAhBHpP,KAAKoP,YACJ,QACM,kCACN,OACA,OACA,QACM,kCACN,OACA,OACA,QACM,+BAUvBqC,SAAU,SAAUoD,MACZA,EAAItH,SAAWvN,KAAKuN,cAcD,IAAfvN,KAAKoP,MACE,oBAEA,0BAhBHpP,KAAKoP,YACJ,QACM,gCACN,OACA,OACA,QACM,gCACN,OACA,OACA,QACM,6BAUvBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,WACTC,KAAO,WACPvJ,EAAI,mBACJwJ,GAAKmZ,EACLztB,EAAIytB,EACJlZ,GAAKkZ,EACLhf,EAAI,MACJ+F,GAAKiZ,EACLjf,EAAI,OACJiG,GAAKgZ,EACL/e,EAAI,QACJgG,GAAK+Y,EACLjiB,EAAI,MACJmJ,GAAK8Y,GAET7Y,cAAe,wBACf/B,KAAO,SAAU5U,SACN,iBAAiB+K,KAAK/K,IAEjC4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,EACA,OACAA,EAAO,GACP,OACAA,EAAO,GACP,MAEA,UAGfrH,uBAAwB,mBACxBlM,QAAS,SAAUxE,EAAQgd,UACfA,OACC,QACA,QACA,aACMhd,EAAS,SACf,WACMA,EAAS,UACf,QACA,WACMA,EAAS,oBAETA,IAGnBgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA1KwB+c,CAAQM,wBCGzC,SAAUJ,YAONoB,EAAuBjqB,EAAQggB,EAAete,OAJjCwoB,EACdC,QAYQ,MAARzoB,EACOse,EAAgB,UAAY,UAEtB,MAARte,EACEse,EAAgB,SAAW,SAG3BhgB,EAAS,KApBFkqB,GAoB6BlqB,EAnB3CmqB,EAIS,IACHnK,EAAgB,yBAA2B,4BAC3CA,EAAgB,yBAA2B,4BAC3CA,EAAgB,sBAAwB,yBACxC,mBACA,2BACA,kBAS8Bte,GAnBvB4H,MAAM,KAChB4gB,EAAM,IAAO,GAAKA,EAAM,KAAQ,GAAKC,EAAM,GAAMD,EAAM,IAAM,GAAKA,EAAM,IAAM,IAAMA,EAAM,IAAM,IAAMA,EAAM,KAAO,IAAMC,EAAM,GAAKA,EAAM,aA0C5IE,EAAqBC,UACnB,kBACIA,EAAM,KAAwB,KAAjB7rB,KAAKoQ,QAAiB,IAAM,IAAM,QAIrDga,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,QACK,yFAAyFM,MAAM,gBAC3F,iGAAiGA,MAAM,MAEzHP,YAAc,yDAAyDO,MAAM,KAC7E0D,kBAjCyBxQ,EAAGN,OACxB8Q,EAAW,YACG,0DAA0D1D,MAAM,gBAChE,0DAA0DA,MAAM,cAClE,4DAA4DA,MAAM,aAGxE,IAAN9M,EACOwQ,EAAQ,WAAerL,MAAM,EAAG,GAAGkL,OAAOG,EAAQ,WAAerL,MAAM,EAAG,IAEhFnF,EASEwQ,EALS,qBAAsBxH,KAAKtJ,GACvC,aACE,sCAAuCsJ,KAAKtJ,GAC1C,WACA,cACkBM,EAAEqR,OARjBb,EAAQ,YAuBnBD,cAAgB,uBAAuBzD,MAAM,KAC7CwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,iBACLC,IAAM,wBACNC,KAAO,+BAEXZ,SAAW,CACPC,QAASua,EAAqB,cAC9Bta,QAASsa,EAAqB,YAC9Bpa,QAASoa,EAAqB,WAC9Bra,SAAUqa,EAAqB,cAC/Bna,SAAU,kBACEzR,KAAKoP,YACJ,OACA,OACA,OACA,SACMwc,EAAqB,oBAAoBvvB,KAAK2D,WACpD,OACA,OACA,SACM4rB,EAAqB,qBAAqBvvB,KAAK2D,QAGlE0R,SAAU,KAEdQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,kBACJwJ,GAAKmZ,EACLztB,EAAIytB,EACJlZ,GAAKkZ,EACLhf,EAAI,SACJ+F,GAAKiZ,EACLjf,EAAI,OACJiG,GAAKgZ,EACL/e,EAAI,SACJgG,GAAK+Y,EACLjiB,EAAI,MACJmJ,GAAK8Y,GAGT7Y,cAAe,wBACf/B,KAAM,SAAU5U,SACL,iBAAiB+K,KAAK/K,IAEjC4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,EACA,OACAA,EAAO,GACP,QACAA,EAAO,GACP,MAEA,UAGfrH,uBAAwB,iBACxBlM,QAAS,SAAUxE,EAAQgd,UACfA,OACC,QACA,QACA,UACA,QACA,WACMhd,EAAS,SACf,WACMA,EAAS,qBAETA,IAGnBgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA9IwB+c,CAAQM,MTJtC,SAAUsB,UACbA,GAAMA,EAAGtqB,MAAQA,MAAQsqB,MAMhC3B,EAA2B,iBAAd4B,YAA0BA,aACvC5B,EAAuB,iBAAV6B,QAAsBA,SACnC7B,EAAqB,iBAAR8B,MAAoBA,OACjC9B,EAAuB,iBAAV+B,GAAsBA,IAEnCvoB,SAAS,cAATA,KUZe,SAAUuT,eAEdA,IACT,MAAOiV,UACA,OCDOC,GAAM,kBACwD,GAAvElwB,OAAOmwB,eAAe,GAAI,EAAG,CAAEtiB,IAAK,kBAAqB,KAAQ,MCHtEuiB,EAA6B,GAAGC,qBAChCC,EAA2BtwB,OAAOswB,8BAGpBA,IAA6BF,EAA2BjwB,KAAK,GAAK,GAAK,GAI/D,SAA8BowB,OAClDC,EAAaF,EAAyBxsB,KAAMysB,WACvCC,GAAcA,EAAWC,YAChCL,KCZa,SAAUM,EAAQ9qB,SAC1B,CACL6qB,aAAuB,EAATC,GACdC,eAAyB,EAATD,GAChBE,WAAqB,EAATF,GACZ9qB,MAAOA,ICLP1F,EAAW,GAAGA,WAED,SAAU0vB,UAClB1vB,EAASC,KAAKyvB,GAAI5oB,MAAM,GAAI,ICAjC2H,EAAQ,GAAGA,QAGEuhB,GAAM,kBAGblwB,OAAO,KAAKqwB,qBAAqB,MACtC,SAAUT,SACS,UAAfiB,EAAQjB,GAAkBjhB,EAAMxO,KAAKyvB,EAAI,IAAM5vB,OAAO4vB,IAC3D5vB,SCVa,SAAU4vB,MACfrsB,MAANqsB,EAAiB,MAAMkB,UAAU,wBAA0BlB,UACxDA,KCAQ,SAAUA,UAClBmB,EAAcC,EAAuBpB,OCL7B,SAAUA,SACJ,iBAAPA,EAAyB,OAAPA,EAA4B,mBAAPA,KCKtC,SAAU9vB,EAAOmxB,OAC3B7wB,EAASN,GAAQ,OAAOA,MACzBa,EAAI0D,KACJ4sB,GAAoD,mBAAxBtwB,EAAKb,EAAMI,YAA4BE,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,KAC3E,mBAAvB1D,EAAKb,EAAMuB,WAA2BjB,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,MACpF4sB,GAAoD,mBAAxBtwB,EAAKb,EAAMI,YAA4BE,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,QACzGysB,UAAU,4CCZd3vB,EAAiB,GAAGA,iBAEP,SAAUyuB,EAAI7oB,UACtB5F,EAAehB,KAAKyvB,EAAI7oB,ICA7BmqB,EAAWlB,EAAOkB,SAElBC,EAAS/wB,EAAS8wB,IAAa9wB,EAAS8wB,EAASE,iBAEpC,SAAUxB,UAClBuB,EAASD,EAASE,cAAcxB,GAAM,OCH7ByB,IAAgBnB,GAAM,kBAG9B,GAFDlwB,OAAOmwB,eAAeiB,EAAc,OAAQ,IAAK,CACtDvjB,IAAK,kBAAqB,KACzB5M,KCADqwB,EAAiCtxB,OAAOswB,8BAIhCe,EAAcC,EAAiC,SAAkCC,EAAGC,MAC9FD,EAAIE,EAAgBF,GACpBC,EAAIE,EAAYF,GAAG,GACfG,EAAgB,WACXL,EAA+BC,EAAGC,GACzC,MAAOvB,OACL2B,EAAIL,EAAGC,GAAI,OAAOK,GAA0BC,EAA2BC,EAAE5xB,KAAKoxB,EAAGC,GAAID,EAAEC,MChBzFQ,EAAc,kBAEdC,EAAW,SAAUC,EAASC,OAC5BvsB,EAAQyR,EAAK+a,EAAUF,WACpBtsB,GAASysB,GACZzsB,GAAS0sB,IACW,mBAAbH,EAA0BjC,EAAMiC,KACrCA,IAGJC,EAAYH,EAASG,UAAY,SAAUrX,UACtCwX,OAAOxX,GAAQ1Q,QAAQ2nB,EAAa,KAAK5pB,eAG9CiP,EAAO4a,EAAS5a,KAAO,GACvBib,EAASL,EAASK,OAAS,IAC3BD,EAAWJ,EAASI,SAAW,MAElBJ,ICpBA,KCAA,SAAUrC,MACR,mBAANA,QACHkB,UAAUyB,OAAO3C,GAAM,6BACtBA,KCAM,SAAUjvB,EAAImkB,EAAMhkB,MACnC0xB,EAAU7xB,QACG4C,IAATuhB,EAAoB,OAAOnkB,SACvBG,QACD,SAAU,kBACNH,EAAGR,KAAK2kB,SAEZ,SAAU,SAAU7jB,UAChBN,EAAGR,KAAK2kB,EAAM7jB,SAElB,SAAU,SAAUA,EAAGC,UACnBP,EAAGR,KAAK2kB,EAAM7jB,EAAGC,SAErB,SAAU,SAAUD,EAAGC,EAAGmX,UACtB1X,EAAGR,KAAK2kB,EAAM7jB,EAAGC,EAAGmX,WAGxB,kBACE1X,EAAGhB,MAAMmlB,EAAMllB,eCnBT,SAAUgwB,OACpBxvB,EAASwvB,SACNkB,UAAUyB,OAAO3C,GAAM,4BACtBA,GCAP6C,EAAuBzyB,OAAOmwB,oBAItBkB,EAAcoB,EAAuB,SAAwBlB,EAAGC,EAAGkB,MAC7EC,EAASpB,GACTC,EAAIE,EAAYF,GAAG,GACnBmB,EAASD,GACLf,EAAgB,WACXc,EAAqBlB,EAAGC,EAAGkB,GAClC,MAAOzC,OACL,QAASyC,GAAc,QAASA,EAAY,MAAM5B,UAAU,iCAC5D,UAAW4B,IAAYnB,EAAEC,GAAKkB,EAAW9sB,OACtC2rB,MCdQF,EAAc,SAAUuB,EAAQ7rB,EAAKnB,UAC7CitB,EAAqBd,EAAEa,EAAQ7rB,EAAK8qB,EAAyB,EAAGjsB,KACrE,SAAUgtB,EAAQ7rB,EAAKnB,UACzBgtB,EAAO7rB,GAAOnB,EACPgtB,GCNLtC,EAA2BhC,EAA2DyD,EAOtFe,EAAkB,SAAUC,OAC1BC,EAAU,SAAU/xB,EAAGC,EAAGmX,MACxBvU,gBAAgBivB,EAAmB,QAC7BnzB,UAAUkB,aACX,SAAU,IAAIiyB,OACd,SAAU,IAAIA,EAAkB9xB,QAChC,SAAU,IAAI8xB,EAAkB9xB,EAAGC,UACjC,IAAI6xB,EAAkB9xB,EAAGC,EAAGmX,UAC9B0a,EAAkBpzB,MAAMmE,KAAMlE,mBAEzCozB,EAAQ/yB,UAAY8yB,EAAkB9yB,UAC/B+yB,KAiBQ,SAAUC,EAAStJ,OAWtBuJ,EAAYC,EACpBpsB,EAAKqsB,EAAgBC,EAAgBC,EAAgBC,EAAgB/C,EAXrEgD,EAASP,EAAQQ,OACjBC,EAAST,EAAQjD,OACjB2D,EAASV,EAAQW,KACjBC,EAAQZ,EAAQ3O,MAEhBwP,EAAeJ,EAAS1D,EAAS2D,EAAS3D,EAAOwD,IAAWxD,EAAOwD,IAAW,IAAIvzB,UAElFwzB,EAASC,EAASK,EAAOA,EAAKP,KAAYO,EAAKP,GAAU,IACzDQ,EAAkBP,EAAOxzB,cAKxB8G,KAAO4iB,EAGVuJ,GAFSjB,EAASyB,EAAS3sB,EAAMysB,GAAUG,EAAS,IAAM,KAAO5sB,EAAKksB,EAAQgB,SAEtDH,GAAgBlC,EAAIkC,EAAc/sB,GAE1DssB,EAAiBI,EAAO1sB,GAEpBmsB,IAEFI,EAFkBL,EAAQiB,aAC1B1D,EAAaF,EAAyBwD,EAAc/sB,KACrBypB,EAAW5qB,MACpBkuB,EAAa/sB,IAGrCqsB,EAAkBF,GAAcI,EAAkBA,EAAiB3J,EAAO5iB,GAEtEmsB,UAAqBG,UAA0BD,IAGnBG,EAA5BN,EAAQkB,MAAQjB,EAA6BiB,EAAKf,EAAgBpD,GAE7DiD,EAAQmB,MAAQlB,EAA6BJ,EAAgBM,GAE7DS,GAAkC,mBAAlBT,EAA+Ce,EAAK1sB,SAAStH,KAAMizB,GAEtEA,GAGlBH,EAAQoB,MAASjB,GAAkBA,EAAeiB,MAAUhB,GAAkBA,EAAegB,OAC/FC,EAA4Bf,EAAgB,QAAQ,GAGtDE,EAAO1sB,GAAOwsB,EAEVM,IAEGjC,EAAImC,EADTZ,EAAoBK,EAAS,cAE3Bc,EAA4BP,EAAMZ,EAAmB,IAGvDY,EAAKZ,GAAmBpsB,GAAOqsB,EAE3BH,EAAQsB,MAAQP,IAAoBA,EAAgBjtB,IACtDutB,EAA4BN,EAAiBjtB,EAAKqsB,QCvFxD,CAAEK,OAAQ,SAAUG,MAAM,EAAMK,QAAS5C,EAAagD,MAAOhD,GAAe,CAC5ElB,eAAgBqE,EAA2BzC,6BCJzC/xB,EAAS+zB,EAAK/zB,OAEdmwB,EAAiB5wB,UAAiB,SAAwBqwB,EAAI7oB,EAAK0tB,UAC9Dz0B,EAAOmwB,eAAeP,EAAI7oB,EAAK0tB,IAGpCz0B,EAAOmwB,eAAekE,OAAMlE,EAAekE,MAAO,QCPrCK,ECFbnvB,EAAOD,KAAKC,KACZC,EAAQF,KAAKE,QAIA,SAAUmvB,UAClBzxB,MAAMyxB,GAAYA,GAAY,GAAKA,EAAW,EAAInvB,EAAQD,GAAMovB,ICJrEzuB,EAAMZ,KAAKY,MAIE,SAAUyuB,UAClBA,EAAW,EAAIzuB,EAAI0uB,EAAUD,GAAW,kBAAoB,GCLjEvrB,GAAM9D,KAAK8D,IACXlD,GAAMZ,KAAKY,OAKE,SAAUwiB,EAAO5nB,OAC5B+zB,EAAUD,EAAUlM,UACjBmM,EAAU,EAAIzrB,GAAIyrB,EAAU/zB,EAAQ,GAAKoF,GAAI2uB,EAAS/zB,ICL3Dg0B,GAAe,SAAUC,UACpB,SAAUC,EAAOC,EAAIC,OAItBtvB,EAHA2rB,EAAIE,EAAgBuD,GACpBl0B,EAASq0B,EAAS5D,EAAEzwB,QACpB4nB,EAAQ0M,GAAgBF,EAAWp0B,MAInCi0B,GAAeE,GAAMA,GAAI,KAAOn0B,EAAS4nB,OAC3C9iB,EAAQ2rB,EAAE7I,OAEG9iB,EAAO,OAAO,OAEtB,KAAM9E,EAAS4nB,EAAOA,QACtBqM,GAAerM,KAAS6I,IAAMA,EAAE7I,KAAWuM,EAAI,OAAOF,GAAerM,GAAS,SAC3EqM,IAAgB,OAIb,CAGfM,SAAUP,IAAa,GAGvBtnB,QAASsnB,IAAa,OC9BP,GCEbtnB,GAAU8gB,GAAuC9gB,WAGpC,SAAUolB,EAAQhb,OAI7B7Q,EAHAwqB,EAAIE,EAAgBmB,GACpBhyB,EAAI,EACJyb,EAAS,OAERtV,KAAOwqB,GAAIK,EAAI0D,GAAYvuB,IAAQ6qB,EAAIL,EAAGxqB,IAAQsV,EAAOtb,KAAKgG,QAE5D6Q,EAAM9W,OAASF,GAAOgxB,EAAIL,EAAGxqB,EAAM6Q,EAAMhX,SAC7C4M,GAAQ6O,EAAQtV,IAAQsV,EAAOtb,KAAKgG,WAEhCsV,MCdQ,CACf,cACA,iBACA,gBACA,uBACA,iBACA,WACA,cCHerc,OAAOoH,MAAQ,SAAcmqB,UACrCgE,GAAmBhE,EAAGiE,QCCdnE,EAAcrxB,OAAOy1B,iBAAmB,SAA0BlE,EAAGmE,GACpF/C,EAASpB,WAILxqB,EAHAK,EAAOuuB,GAAWD,GAClB50B,EAASsG,EAAKtG,OACd4nB,EAAQ,EAEL5nB,EAAS4nB,GAAOmK,EAAqBd,EAAER,EAAGxqB,EAAMK,EAAKshB,KAAUgN,EAAW3uB,WAC1EwqB,KCRP,CAAEkC,OAAQ,SAAUG,MAAM,EAAMK,QAAS5C,EAAagD,MAAOhD,GAAe,CAC5EoE,iBAAkBA,+BCJhBz1B,EAAS+zB,EAAK/zB,OAEdy1B,EAAmBl2B,UAAiB,SAA0Bq2B,EAAG1J,UAC5DlsB,EAAOy1B,iBAAiBG,EAAG1J,IAGhClsB,EAAOy1B,iBAAiBpB,OAAMoB,EAAiBpB,MAAO,MCNtD7B,GAAY,SAAUqD,SACE,mBAAZA,EAAyBA,OAAWtyB,MAGnC,SAAUuyB,EAAWC,UAC7Bn2B,UAAUkB,OAAS,EAAI0xB,GAAUuB,EAAK+B,KAAetD,GAAUxC,EAAO8F,IACzE/B,EAAK+B,IAAc/B,EAAK+B,GAAWC,IAAW/F,EAAO8F,IAAc9F,EAAO8F,GAAWC,ICNvFT,GAAaE,GAAYtjB,OAAO,SAAU,mBAIlClS,OAAOwe,qBAAuB,SAA6B+S,UAC9DgE,GAAmBhE,EAAG+D,YCRnBt1B,OAAOg2B,0BCMFC,GAAW,UAAW,YAAc,SAAiBrG,OAChExoB,EAAO8uB,GAA0BnE,EAAEY,EAAS/C,IAC5CoG,EAAwBG,GAA4BpE,SACjDiE,EAAwB5uB,EAAK8K,OAAO8jB,EAAsBpG,IAAOxoB,MCJzD,SAAUwrB,EAAQ7rB,EAAKnB,OAClCwwB,EAAc1E,EAAY3qB,GAC1BqvB,KAAexD,EAAQC,EAAqBd,EAAEa,EAAQwD,EAAavE,EAAyB,EAAGjsB,IAC9FgtB,EAAOwD,GAAexwB,KCC3B,CAAE6tB,OAAQ,SAAUG,MAAM,EAAMS,MAAOhD,GAAe,CACtDgF,0BAA2B,SAAmCzD,WAMxD7rB,EAAKypB,EALLe,EAAIE,EAAgBmB,GACpBtC,EAA2BgG,EAA+BvE,EAC1D3qB,EAAOmvB,GAAQhF,GACflV,EAAS,GACTqM,EAAQ,EAELthB,EAAKtG,OAAS4nB,QAEAnlB,KADnBitB,EAAaF,EAAyBiB,EAAGxqB,EAAMK,EAAKshB,QACtB8N,GAAena,EAAQtV,EAAKypB,UAErDnU,KClBX,OAAiB0X,EAAK/zB,OAAOq2B,0BCAzB/E,GAAiChD,EAA2DyD,EAG5F0E,GAAsBvG,GAAM,WAAcoB,GAA+B,QAK3E,CAAEmC,OAAQ,SAAUG,MAAM,EAAMK,QAJpB5C,GAAeoF,GAIqBpC,MAAOhD,GAAe,CACtEf,yBAA0B,SAAkCV,EAAI7oB,UACvDuqB,GAA+BG,EAAgB7B,GAAK7oB,UCsC3D2vB,yBChDA12B,EAAS+zB,EAAK/zB,OAEdswB,EAA2B/wB,UAAiB,SAAkCqwB,EAAI7oB,UAC7E/G,EAAOswB,yBAAyBV,EAAI7oB,IAGzC/G,EAAOswB,yBAAyB+D,OAAM/D,EAAyB+D,MAAO,WCPvDr0B,OAAOg2B,wBAA0B9F,GAAM,kBAGhDqC,OAAOoE,gBCHAC,KAEXD,OAAOtC,MAEkB,iBAAnBsC,OAAOE,YCFF92B,MAAMF,SAAW,SAAiBgH,SAC1B,SAAhBgqB,EAAQhqB,OCDA,SAAU8tB,UAClB30B,OAAOgxB,EAAuB2D,QCHtBsB,GAAW,WAAY,sBCE5BjG,EADC,uBCAI,SAAUjpB,EAAKnB,OAE5B0uB,EAA4BtE,EAAQjpB,EAAKnB,GACzC,MAAOqqB,GACPD,EAAOjpB,GAAOnB,SACPA,EDJmBkxB,CADjB,qBACmC,uBED/Cv3B,UAAiB,SAAUwH,EAAKnB,UACxBmxB,GAAMhwB,KAASgwB,GAAMhwB,QAAiBxD,IAAVqC,EAAsBA,EAAQ,MAChE,WAAY,IAAI7E,KAAK,CACtB8rB,QAAS,QACTmK,KAAgB,OAChBC,UAAW,4CCRTC,GAAK,EACLC,GAAU7xB,KAAK8xB,YAEF,SAAUrwB,SAClB,UAAYwrB,YAAehvB,IAARwD,EAAoB,GAAKA,GAAO,QAAUmwB,GAAKC,IAASj3B,SAAS,KCDzFkH,GAAOiwB,GAAO,WAED,SAAUtwB,UAClBK,GAAKL,KAASK,GAAKL,GAAOuwB,GAAIvwB,KXMnCwwB,GAAWC,GAAU,YAErBC,GAAmB,aAEnBC,GAAY,SAAUC,SACjBjiB,WAAmBiiB,EARnB,cA2CLC,GAAkB,eAGlBlB,GAAkBxF,SAAS2G,QAAU,IAAIC,cAAc,YACvD,MAAO7H,IA1BoB,IAIzB8H,EAFAC,EAyBJJ,GAAkBlB,GApCY,SAAUA,GACxCA,EAAgBuB,MAAMP,GAAU,KAChChB,EAAgBwB,YACZ9e,EAAOsd,EAAgByB,aAAan4B,cACxC02B,EAAkB,KACXtd,EA+B6Bgf,CAA0B1B,MAzB1DsB,EAASK,EAAsB,WAG5BC,MAAMC,QAAU,OACvBC,GAAKC,YAAYT,GAEjBA,EAAOU,IAAMnG,OALJ,gBAMTwF,EAAiBC,EAAOW,cAAczH,UACvB0H,OACfb,EAAeE,MAAMP,GAAU,sBAC/BK,EAAeG,QACRH,EAAec,WAelB/3B,EAAS00B,GAAY10B,OAClBA,YAAiB82B,GAAe,UAAYpC,GAAY10B,WACxD82B,MAGTtC,GAAWiC,KAAY,EAIvB,OAAiBv3B,OAAO84B,QAAU,SAAgBvH,EAAGmE,OAC/CrZ,SACM,OAANkV,GACFkG,GAAgB,UAAc9E,EAASpB,GACvClV,EAAS,IAAIob,GACbA,GAAgB,UAAc,KAE9Bpb,EAAOkb,IAAYhG,GACdlV,EAASub,UACMr0B,IAAfmyB,EAA2BrZ,EAASoZ,GAAiBpZ,EAAQqZ,IY3ElEqD,GAA4BzK,GAAsDyD,EAElF7xB,GAAW,GAAGA,SAEd84B,GAA+B,iBAAVlJ,QAAsBA,QAAU9vB,OAAOwe,oBAC5Dxe,OAAOwe,oBAAoBsR,QAAU,SAWtB,SAA6BF,UACvCoJ,IAAoC,mBAArB94B,GAASC,KAAKyvB,GAVjB,SAAUA,cAEpBmJ,GAA0BnJ,GACjC,MAAOK,UACA+I,GAAYhyB,SAOjBiyB,CAAerJ,GACfmJ,GAA0BtH,EAAgB7B,SClB/B,SAAU6D,EAAQ1sB,EAAKnB,EAAOqtB,GACzCA,GAAWA,EAAQxC,WAAYgD,EAAO1sB,GAAOnB,EAC5C0uB,EAA4Bb,EAAQ1sB,EAAKnB,ICG5CszB,GAAwB7B,GAAO,OAC/BV,GAAS3G,EAAO2G,OAChBwC,GAAwBC,GAAoBzC,GAASA,IAAUA,GAAO0C,eAAiB/B,MAE1E,SAAU/vB,UACpBqqB,EAAIsH,GAAuB3xB,KAC1BqvB,IAAiBhF,EAAI+E,GAAQpvB,GAAO2xB,GAAsB3xB,GAAQovB,GAAOpvB,GACxE2xB,GAAsB3xB,GAAQ4xB,GAAsB,UAAY5xB,IAC9D2xB,GAAsB3xB,UCbrB+xB,ICCRnJ,GAAiB7B,EAA+CyD,KAEnD,SAAUwH,OACrB5C,EAAS5C,EAAK4C,SAAW5C,EAAK4C,OAAS,IACtC/E,EAAI+E,EAAQ4C,IAAOpJ,GAAewG,EAAQ4C,EAAM,CACnD3zB,MAAO4zB,GAA6BzH,EAAEwH,MCLtC1uB,GAAO,GAEXA,GAHoByuB,GAAgB,gBAGd,IAEtB,OAAkC,eAAjB/G,OAAO1nB,ICHpB4uB,GAAgBH,GAAgB,eAEhCI,GAAuE,aAAnDC,EAAW,kBAAqB/5B,UAArB,OAUlBg6B,GAAwBD,EAAa,SAAU/J,OAC1D2B,EAAGsI,EAAKxd,cACE9Y,IAAPqsB,EAAmB,YAAqB,OAAPA,EAAc,OAEM,iBAAhDiK,EAXD,SAAUjK,EAAI7oB,cAEhB6oB,EAAG7oB,GACV,MAAOkpB,KAQS6J,CAAOvI,EAAIvxB,OAAO4vB,GAAK6J,KAA8BI,EAEnEH,GAAoBC,EAAWpI,GAEH,WAA3BlV,EAASsd,EAAWpI,KAAsC,mBAAZA,EAAEwI,OAAuB,YAAc1d,MClB3Eud,GAAwB,GAAG15B,SAAW,iBAC9C,WAAa2wB,GAAQ/sB,MAAQ,KCNlCqsB,GAAiB7B,EAA+CyD,EAMhE0H,GAAgBH,GAAgB,kBAEnB,SAAU1J,EAAIoK,EAAKrG,EAAQsG,MACtCrK,EAAI,KACF6D,EAASE,EAAS/D,EAAKA,EAAG3vB,UACzB2xB,EAAI6B,EAAQgG,KACftJ,GAAesD,EAAQgG,GAAe,CAAE9I,cAAc,EAAM/qB,MAAOo0B,IAEjEC,IAAeL,IACjBtF,EAA4Bb,EAAQ,WAAYvzB,MCdlDg6B,GAAmBzyB,SAASvH,SAGE,mBAAvB62B,GAAMoD,gBACfpD,GAAMoD,cAAgB,SAAUvK,UACvBsK,GAAiB/5B,KAAKyvB,KAIjC,ICFI9nB,GAAK+F,GAAK+jB,MDEGmF,GAAMoD,cERnBC,GAAUpK,EAAOoK,WAEe,mBAAZA,IAA0B,cAAcvvB,KAAKsvB,GAAcC,KDG/EA,GAAUpK,EAAOoK,QAgBrB,GAAIC,GAAiB,KACftD,GAAQ,IAAIqD,GACZE,GAAQvD,GAAMlpB,IACd0sB,GAAQxD,GAAMnF,IACd4I,GAAQzD,GAAMjvB,IAClBA,GAAM,SAAU8nB,EAAI6K,UAClBD,GAAMr6B,KAAK42B,GAAOnH,EAAI6K,GACfA,GAET5sB,GAAM,SAAU+hB,UACP0K,GAAMn6B,KAAK42B,GAAOnH,IAAO,IAElCgC,GAAM,SAAUhC,UACP2K,GAAMp6B,KAAK42B,GAAOnH,QAEtB,KACD8K,GAAQlD,GAAU,SACtBlC,GAAWoF,KAAS,EACpB5yB,GAAM,SAAU8nB,EAAI6K,UAClBnG,EAA4B1E,EAAI8K,GAAOD,GAChCA,GAET5sB,GAAM,SAAU+hB,UACP+K,EAAU/K,EAAI8K,IAAS9K,EAAG8K,IAAS,IAE5C9I,GAAM,SAAUhC,UACP+K,EAAU/K,EAAI8K,KAIzB,OAAiB,CACf5yB,IAAKA,GACL+F,IAAKA,GACL+jB,IAAKA,GACLgJ,QA/CY,SAAUhL,UACfgC,GAAIhC,GAAM/hB,GAAI+hB,GAAM9nB,GAAI8nB,EAAI,KA+CnCiL,UA5Cc,SAAUC,UACjB,SAAUlL,OACXmL,MACC36B,EAASwvB,KAAQmL,EAAQltB,GAAI+hB,IAAKoL,OAASF,QACxChK,UAAU,0BAA4BgK,EAAO,oBAC5CC,KEhBTE,GAAU3B,GAAgB,cAIb,SAAU4B,EAAep6B,OACpCq6B,SACAt7B,GAAQq7B,KAGM,mBAFhBC,EAAID,EAAcE,cAEaD,IAAMp7B,QAASF,GAAQs7B,EAAEl7B,WAC/CG,EAAS+6B,IAEN,QADVA,EAAIA,EAAEF,OACUE,OAAI53B,GAH+C43B,OAAI53B,GAKlE,SAAWA,IAAN43B,EAAkBp7B,MAAQo7B,GAAc,IAAXr6B,EAAe,EAAIA,ICZ5DC,GAAO,GAAGA,KAGV+zB,GAAe,SAAUgG,OACvBO,EAAiB,GAARP,EACTQ,EAAoB,GAARR,EACZS,EAAkB,GAART,EACVU,EAAmB,GAARV,EACXW,EAAwB,GAARX,EAChBY,EAAmB,GAARZ,GAAaW,SACrB,SAAUzG,EAAO2G,EAAY7W,EAAM8W,WAQpCh2B,EAAOyW,EAPPkV,EAAI9K,GAASuO,GACbjF,EAAOgB,EAAcQ,GACrBsK,EAAgB1H,EAAKwH,EAAY7W,EAAM,GACvChkB,EAASq0B,EAASpF,EAAKjvB,QACvB4nB,EAAQ,EACRoQ,EAAS8C,GAAkBE,GAC3BrI,EAAS4H,EAASvC,EAAO9D,EAAOl0B,GAAUw6B,EAAYxC,EAAO9D,EAAO,QAAKzxB,EAEvEzC,EAAS4nB,EAAOA,IAAS,IAAIgT,GAAYhT,KAASqH,KAEtD1T,EAASwf,EADTj2B,EAAQmqB,EAAKrH,GACiBA,EAAO6I,GACjCuJ,MACEO,EAAQ5H,EAAO/K,GAASrM,OACvB,GAAIA,EAAQ,OAAQye,QAClB,SAAU,OACV,SAAUl1B,OACV,SAAU8iB,OACV,EAAG3nB,GAAKZ,KAAKszB,EAAQ7tB,QACrB,GAAI41B,EAAU,OAAO,SAGzBC,GAAiB,EAAIF,GAAWC,EAAWA,EAAW/H,OAIhD,CAGf9b,QAASmd,GAAa,GAGtBr0B,IAAKq0B,GAAa,GAGlBiH,OAAQjH,GAAa,GAGrBr1B,KAAMq1B,GAAa,GAGnBkH,MAAOlH,GAAa,GAGpBmH,KAAMnH,GAAa,GAGnBoH,UAAWpH,GAAa,IC3BtBqH,GAAW7N,GAAwC3W,QAEnDykB,GAAS5E,GAAU,UAGnB6E,GAAe/C,GAAgB,eAC/BgD,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAJ9B,UAKT4B,GAAkBz8B,OAAM,UACxB08B,GAAU1M,EAAO2G,OACjBgG,GAAa1G,GAAW,OAAQ,aAChC3E,GAAiCgF,EAA+BvE,EAChEU,GAAuBI,EAAqBd,EAC5CgH,GAA4B6D,GAA4B7K,EACxD3B,GAA6B0B,EAA2BC,EACxD8K,GAAaxF,GAAO,WACpByF,GAAyBzF,GAAO,cAChC0F,GAAyB1F,GAAO,6BAChC2F,GAAyB3F,GAAO,6BAChC6B,GAAwB7B,GAAO,OAC/B4F,GAAUjN,EAAOiN,QAEjBC,IAAcD,KAAYA,GAAO,YAAgBA,GAAO,UAAYE,UAGpEC,GAAsB/L,GAAenB,GAAM,kBAGpC,GAFFmN,GAAmB5K,GAAqB,GAAI,IAAK,CACtD5kB,IAAK,kBAAqB4kB,GAAqB3uB,KAAM,IAAK,CAAE8B,MAAO,IAAK3E,MACtEA,KACD,SAAUswB,EAAGC,EAAGkB,OACf4K,EAA4BhM,GAA+BmL,GAAiBjL,GAC5E8L,UAAkCb,GAAgBjL,GACtDiB,GAAqBlB,EAAGC,EAAGkB,GACvB4K,GAA6B/L,IAAMkL,IACrChK,GAAqBgK,GAAiBjL,EAAG8L,IAEzC7K,GAEA2B,GAAO,SAAUyF,EAAK0D,OACpBC,EAASX,GAAWhD,GAAOwD,GAAmBX,GAAO,kBACzDJ,GAAiBkB,EAAQ,CACvBxC,KAtCS,SAuCTnB,IAAKA,EACL0D,YAAaA,IAEVlM,IAAamM,EAAOD,YAAcA,GAChCC,GAGLC,GAAWrE,GAAoB,SAAUxJ,SACvB,iBAANA,GACZ,SAAUA,UACL5vB,OAAO4vB,aAAe8M,IAG3BgB,GAAkB,SAAwBnM,EAAGC,EAAGkB,GAC9CnB,IAAMkL,IAAiBiB,GAAgBZ,GAAwBtL,EAAGkB,GACtEC,EAASpB,OACLxqB,EAAM2qB,EAAYF,GAAG,UACzBmB,EAASD,GACLd,EAAIiL,GAAY91B,IACb2rB,EAAWjC,YAIVmB,EAAIL,EAAG6K,KAAW7K,EAAE6K,IAAQr1B,KAAMwqB,EAAE6K,IAAQr1B,IAAO,GACvD2rB,EAAa2K,GAAmB3K,EAAY,CAAEjC,WAAYoB,EAAyB,GAAG,OAJjFD,EAAIL,EAAG6K,KAAS3J,GAAqBlB,EAAG6K,GAAQvK,EAAyB,EAAG,KACjFN,EAAE6K,IAAQr1B,IAAO,GAIVq2B,GAAoB7L,EAAGxqB,EAAK2rB,IAC9BD,GAAqBlB,EAAGxqB,EAAK2rB,IAGpCiL,GAAoB,SAA0BpM,EAAGmE,GACnD/C,EAASpB,OACLqM,EAAanM,EAAgBiE,GAC7BtuB,EAAOuuB,GAAWiI,GAAY1rB,OAAO2rB,GAAuBD,WAChEzB,GAAS/0B,GAAM,SAAUL,GAClBsqB,IAAeyM,GAAsB39B,KAAKy9B,EAAY72B,IAAM22B,GAAgBnM,EAAGxqB,EAAK62B,EAAW72B,OAE/FwqB,GAOLuM,GAAwB,SAA8BvN,OACpDiB,EAAIE,EAAYnB,GAAG,GACnBE,EAAaL,GAA2BjwB,KAAK2D,KAAM0tB,WACnD1tB,OAAS24B,IAAmB7K,EAAIiL,GAAYrL,KAAOI,EAAIkL,GAAwBtL,QAC5Ef,IAAemB,EAAI9tB,KAAM0tB,KAAOI,EAAIiL,GAAYrL,IAAMI,EAAI9tB,KAAMs4B,KAAWt4B,KAAKs4B,IAAQ5K,KAAKf,IAGlGsN,GAA4B,SAAkCxM,EAAGC,OAC/D5B,EAAK6B,EAAgBF,GACrBxqB,EAAM2qB,EAAYF,GAAG,MACrB5B,IAAO6M,KAAmB7K,EAAIiL,GAAY91B,IAAS6qB,EAAIkL,GAAwB/1B,QAC/EypB,EAAac,GAA+B1B,EAAI7oB,UAChDypB,IAAcoB,EAAIiL,GAAY91B,IAAU6qB,EAAIhC,EAAIwM,KAAWxM,EAAGwM,IAAQr1B,KACxEypB,EAAWC,YAAa,GAEnBD,IAGLwN,GAAuB,SAA6BzM,OAClD3Z,EAAQmhB,GAA0BtH,EAAgBF,IAClDlV,EAAS,UACb8f,GAASvkB,GAAO,SAAU7Q,GACnB6qB,EAAIiL,GAAY91B,IAAS6qB,EAAI0D,GAAYvuB,IAAMsV,EAAOtb,KAAKgG,MAE3DsV,GAGLwhB,GAAyB,SAA+BtM,OACtD0M,EAAsB1M,IAAMkL,GAC5B7kB,EAAQmhB,GAA0BkF,EAAsBnB,GAAyBrL,EAAgBF,IACjGlV,EAAS,UACb8f,GAASvkB,GAAO,SAAU7Q,IACpB6qB,EAAIiL,GAAY91B,IAAUk3B,IAAuBrM,EAAI6K,GAAiB11B,IACxEsV,EAAOtb,KAAK87B,GAAW91B,OAGpBsV,GAkHT,GA7GKua,KAcHsH,IAbAxB,GAAU,cACJ54B,gBAAgB44B,GAAS,MAAM5L,UAAU,mCACzCyM,EAAe39B,UAAUkB,aAA2ByC,IAAjB3D,UAAU,GAA+B2yB,OAAO3yB,UAAU,SAA7B2D,EAChEs2B,EAAMvC,GAAIiG,GACV3U,EAAS,SAAUhjB,GACjB9B,OAAS24B,IAAiB7T,EAAOzoB,KAAK28B,GAAwBl3B,GAC9DgsB,EAAI9tB,KAAMs4B,KAAWxK,EAAI9tB,KAAKs4B,IAASvC,KAAM/1B,KAAKs4B,IAAQvC,IAAO,GACrEuD,GAAoBt5B,KAAM+1B,EAAKhI,EAAyB,EAAGjsB,YAEzDyrB,GAAe6L,IAAYE,GAAoBX,GAAiB5C,EAAK,CAAElJ,cAAc,EAAM7oB,IAAK8gB,IAC7FwL,GAAKyF,EAAK0D,KAGH,UAAa,YAAY,kBAChCf,GAAiB14B,MAAM+1B,OAGhCqE,GAASxB,GAAS,iBAAiB,SAAUa,UACpCnJ,GAAKkD,GAAIiG,GAAcA,MAGhCzL,EAA2BC,EAAI+L,GAC/BjL,EAAqBd,EAAI2L,GACzBpH,EAA+BvE,EAAIgM,GACnC7H,GAA0BnE,EAAI6K,GAA4B7K,EAAIiM,GAC9D7H,GAA4BpE,EAAI8L,GAEhCrE,GAA6BzH,EAAI,SAAUxqB,UAClC6sB,GAAKkF,GAAgB/xB,GAAOA,IAGjC8pB,GAEFoB,GAAqBiK,GAAO,UAAa,cAAe,CACtD/L,cAAc,EACd9iB,IAAK,kBACI2uB,GAAiB14B,MAAMy5B,kBASpC,CAAEvN,QAAQ,EAAMoE,MAAM,EAAMH,QAAS2C,GAAevC,MAAOuC,IAAiB,CAC5ED,OAAQ+F,KAGVP,GAASxG,GAAWuD,KAAwB,SAAU3xB,GACpD42B,GAAsB52B,QAGtB,CAAEksB,OAjLS,SAiLOG,MAAM,EAAMK,QAAS2C,IAAiB,KAGjD,SAAU7vB,OACXgU,EAASwX,OAAOxrB,MAChB6qB,EAAImL,GAAwBhiB,GAAS,OAAOgiB,GAAuBhiB,OACnEyiB,EAASd,GAAQ3hB,UACrBgiB,GAAuBhiB,GAAUyiB,EACjCR,GAAuBQ,GAAUziB,EAC1ByiB,GAITY,OAAQ,SAAgBC,OACjBZ,GAASY,GAAM,MAAMvN,UAAUuN,EAAM,uBACtCzM,EAAIoL,GAAwBqB,GAAM,OAAOrB,GAAuBqB,IAEtEC,UAAW,WAAcpB,IAAa,GACtCqB,UAAW,WAAcrB,IAAa,OAGtC,CAAEzJ,OAAQ,SAAUG,MAAM,EAAMK,QAAS2C,GAAevC,MAAOhD,GAAe,CAG9EyH,OA3HY,SAAgBvH,EAAGmE,eACTnyB,IAAfmyB,EAA2B2H,GAAmB9L,GAAKoM,GAAkBN,GAAmB9L,GAAImE,IA6HnGvF,eAAgBuN,GAGhBjI,iBAAkBkI,GAGlBrN,yBAA0ByN,OAG1B,CAAEtK,OAAQ,SAAUG,MAAM,EAAMK,QAAS2C,IAAiB,CAG1DpY,oBAAqBwf,GAGrBhI,sBAAuB6H,OAKvB,CAAEpK,OAAQ,SAAUG,MAAM,EAAMK,OAAQ/D,GAAM,WAAciG,GAA4BpE,EAAE,OAAU,CACpGiE,sBAAuB,SAA+BpG,UAC7CuG,GAA4BpE,EAAEtL,GAASmJ,OAM9C+M,GAAY,KACV6B,IAAyB5H,IAAiB1G,GAAM,eAC9CsN,EAASd,WAEkB,UAAxBC,GAAW,CAACa,KAEe,MAA7Bb,GAAW,CAAE17B,EAAGu8B,KAEc,MAA9Bb,GAAW38B,OAAOw9B,OAGzBiB,EAAE,CAAEhL,OAAQ,OAAQG,MAAM,EAAMK,OAAQuK,IAAyB,CAE/DE,UAAW,SAAmB9O,EAAI+O,EAAUC,WAGtCC,EAFA/3B,EAAO,CAAC8oB,GACRlH,EAAQ,EAEL9oB,UAAUkB,OAAS4nB,GAAO5hB,EAAK/F,KAAKnB,UAAU8oB,SACrDmW,EAAYF,GACPv+B,EAASu+B,SAAoBp7B,IAAPqsB,KAAoB6N,GAAS7N,UACnD/vB,GAAQ8+B,KAAWA,EAAW,SAAU53B,EAAKnB,MACxB,mBAAbi5B,IAAyBj5B,EAAQi5B,EAAU1+B,KAAK2D,KAAMiD,EAAKnB,KACjE63B,GAAS73B,GAAQ,OAAOA,IAE/BkB,EAAK,GAAK63B,EACHhC,GAAWh9B,MAAM,KAAMmH,MAO/B41B,GAAO,UAAYL,KACtB/H,EAA4BoI,GAAO,UAAaL,GAAcK,GAAO,UAAYr7B,SAInFy9B,GAAepC,GA7QF,UA+QbpH,GAAW8G,KAAU,ECnTrB,ICWI2C,GAAmBC,GAAmCC,MDXzClL,EAAK/zB,OAAOg2B,yBEHZ,OCEC9F,GAAM,oBACb2I,YACTA,EAAE54B,UAAUm7B,YAAc,KACnBp7B,OAAOk/B,eAAe,IAAIrG,KAASA,EAAE54B,aCA1Cs3B,GAAWC,GAAU,YACrBiF,GAAkBz8B,OAAOC,aAIZk/B,GAA2Bn/B,OAAOk/B,eAAiB,SAAU3N,UAC5EA,EAAI9K,GAAS8K,GACTK,EAAIL,EAAGgG,IAAkBhG,EAAEgG,IACH,mBAAjBhG,EAAE6J,aAA6B7J,aAAaA,EAAE6J,YAChD7J,EAAE6J,YAAYn7B,UACdsxB,aAAavxB,OAASy8B,GAAkB,MHP/C2C,IADW9F,GAAgB,aACF,GAQzB,GAAGlyB,OAGC,SAFN63B,GAAgB,GAAG73B,SAIjB43B,GAAoCE,GAAeA,GAAeD,QACxBj/B,OAAOC,YAAW8+B,GAAoBC,IAHlDI,IAAyB,GAOlC77B,MAArBw7B,KAAgCA,GAAoB,IAOxD,OAAiB,CACfA,kBAAmBA,GACnBK,uBAAwBA,IIlCtBL,GAAoBzQ,GAAuCyQ,kBAM3DM,GAAa,kBAAqBv7B,SCArB9D,OAAOs/B,iBAAmB,aAAe,GAAK,eAGzD1W,EAFA2W,GAAiB,EACjB10B,EAAO,QAGT+d,EAAS5oB,OAAOswB,yBAAyBtwB,OAAOC,UAAW,aAAa6H,KACjE3H,KAAK0K,EAAM,IAClB00B,EAAiB10B,aAAgB9K,MACjC,MAAOkwB,WACF,SAAwBsB,EAAGjN,UAChCqO,EAASpB,GCfI,SAAU3B,OACpBxvB,EAASwvB,IAAc,OAAPA,QACbkB,UAAU,aAAeyB,OAAO3C,GAAM,mBDc5C4P,CAAmBlb,GACfib,EAAgB3W,EAAOzoB,KAAKoxB,EAAGjN,GAC9BiN,EAAEkO,UAAYnb,EACZiN,GAdoD,QAgBzDhuB,GEVFw7B,GAAoBW,GAAcX,kBAClCK,GAAyBM,GAAcN,uBACvCO,GAAWrG,GAAgB,YAK3B+F,GAAa,kBAAqBv7B,SAErB,SAAU87B,EAAUrG,EAAMsG,EAAqB/nB,EAAMgoB,EAASC,EAAQC,IHbtE,SAAUH,EAAqBtG,EAAMzhB,OAChD2hB,EAAgBF,EAAO,YAC3BsG,EAAoB5/B,UAAY64B,GAAOiG,GAAmB,CAAEjnB,KAAM+Z,EAAyB,EAAG/Z,KAC9FgnB,GAAee,EAAqBpG,GAAe,GAAO,GAC1DwG,GAAUxG,GAAiB4F,GGU3Ba,CAA0BL,EAAqBtG,EAAMzhB,OAoBjDqoB,EAA0BC,EAASC,EAlBnCC,EAAqB,SAAUC,MAC7BA,IAAST,GAAWU,EAAiB,OAAOA,MAC3CpB,IAA0BmB,KAAQE,EAAmB,OAAOA,EAAkBF,UAC3EA,OAZD,WACE,aACC,iBAaa,kBAA4B,IAAIV,EAAoB/7B,KAAMy8B,WACxE,kBAAqB,IAAIV,EAAoB/7B,QAGpD21B,EAAgBF,EAAO,YACvBmH,GAAwB,EACxBD,EAAoBb,EAAS3/B,UAC7B0gC,EAAiBF,EAAkBd,KAClCc,EAAkB,eAClBX,GAAWW,EAAkBX,GAC9BU,GAAmBpB,IAA0BuB,GAAkBL,EAAmBR,GAClFc,EAA4B,SAARrH,GAAkBkH,EAAkBI,SAA4BF,KAIpFC,IACFT,EAA2BjB,GAAe0B,EAAkBzgC,KAAK,IAAIy/B,IACjEb,KAAsB/+B,OAAOC,WAAakgC,EAAyBroB,OASrEgnB,GAAeqB,EAA0B1G,GAAe,GAAM,GACjDwG,GAAUxG,GAAiB4F,KAzCjC,UA8CPS,GAAqBa,GA9Cd,WA8CgCA,EAAep5B,OACxDm5B,GAAwB,EACxBF,EAAkB,kBAA2BG,EAAexgC,KAAK2D,QAI/D,GAAwB28B,EAAkBd,MAAca,GAC1DlM,EAA4BmM,EAAmBd,GAAUa,GAE3DP,GAAU1G,GAAQiH,EAGdV,KACFM,EAAU,CACRhpB,OAAQkpB,EA5DD,UA6DPl5B,KAAM24B,EAASS,EAAkBF,EA9D5B,QA+DLO,QAASP,EA7DD,YA+DNN,EAAQ,IAAKK,KAAOD,GAClBhB,KAA0BsB,GAA2BL,KAAOI,GAC9DvC,GAASuC,EAAmBJ,EAAKD,EAAQC,SAEtC5B,EAAE,CAAEhL,OAAQ8F,EAAMjV,OAAO,EAAM2P,OAAQmL,IAA0BsB,GAAyBN,UAG5FA,GChFL9D,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAFtB,kBAcJiG,GAAe/gC,MAAO,SAAS,SAAUghC,EAAUC,GAClE1E,GAAiBx4B,KAAM,CACrBk3B,KAhBiB,iBAiBjBvH,OAAQhC,EAAgBsP,GACxBrY,MAAO,EACPsY,KAAMA,OAIP,eACGjG,EAAQyB,GAAiB14B,MACzB2vB,EAASsH,EAAMtH,OACfuN,EAAOjG,EAAMiG,KACbtY,EAAQqS,EAAMrS,eACb+K,GAAU/K,GAAS+K,EAAO3yB,QAC7Bi6B,EAAMtH,YAASlwB,EACR,CAAEqC,WAAOrC,EAAW09B,MAAM,IAEvB,QAARD,EAAuB,CAAEp7B,MAAO8iB,EAAOuY,MAAM,GACrC,UAARD,EAAyB,CAAEp7B,MAAO6tB,EAAO/K,GAAQuY,MAAM,GACpD,CAAEr7B,MAAO,CAAC8iB,EAAO+K,EAAO/K,IAASuY,MAAM,KAC7C,aAKOC,UAAYjB,GAAUlgC,MC7ChC,ICMI05B,GAAgBH,GAAgB,eAEpC,IAAK,IAAI6H,KDRQ,CACfC,YAAa,EACbC,oBAAqB,EACrBC,aAAc,EACdC,eAAgB,EAChBC,YAAa,EACbC,cAAe,EACfC,aAAc,EACdC,qBAAsB,EACtBC,SAAU,EACVC,kBAAmB,EACnBC,eAAgB,EAChBC,gBAAiB,EACjBC,kBAAmB,EACnBC,UAAW,EACXC,cAAe,EACfC,aAAc,EACdC,SAAU,EACVC,iBAAkB,EAClBC,OAAQ,EACRC,YAAa,EACbC,cAAe,EACfC,cAAe,EACfC,eAAgB,EAChBC,aAAc,EACdC,cAAe,EACfC,iBAAkB,EAClBC,iBAAkB,EAClBC,eAAgB,EAChBC,iBAAkB,EAClBC,cAAe,EACfC,UAAW,GCvB6B,KACpCC,GAAanT,EAAOmR,IACpBiC,GAAsBD,IAAcA,GAAWljC,UAC/CmjC,IAAuBvS,GAAQuS,MAAyB3J,IAC1DnF,EAA4B8O,GAAqB3J,GAAe0H,IAElElB,GAAUkB,IAAmBlB,GAAUlgC,MCZzC,IAAI+0B,GAAe,SAAUuO,UACpB,SAAUrO,EAAOpgB,OAIlB0uB,EAAOnlB,EAHPolB,EAAIhR,OAAOvB,EAAuBgE,IAClCwO,EAAW5O,EAAUhgB,GACrB6uB,EAAOF,EAAEziC,cAET0iC,EAAW,GAAKA,GAAYC,EAAaJ,EAAoB,QAAK9/B,GACtE+/B,EAAQC,EAAEG,WAAWF,IACN,OAAUF,EAAQ,OAAUE,EAAW,IAAMC,IACtDtlB,EAASolB,EAAEG,WAAWF,EAAW,IAAM,OAAUrlB,EAAS,MAC1DklB,EAAoBE,EAAEjZ,OAAOkZ,GAAYF,EACzCD,EAAoBE,EAAEv8B,MAAMw8B,EAAUA,EAAW,GAA+BrlB,EAAS,OAAlCmlB,EAAQ,OAAU,IAA0B,QCdzGhZ,GDkBa,CAGfqZ,OAAQ7O,IAAa,GAGrBxK,OAAQwK,IAAa,ICxB+BxK,OAKlDgS,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAFrB,mBAMtBiG,GAAevO,OAAQ,UAAU,SAAUwO,GACzCzE,GAAiBx4B,KAAM,CACrBk3B,KARkB,kBASlBjgB,OAAQwX,OAAOwO,GACfrY,MAAO,OAIR,eAIGkb,EAHA7I,EAAQyB,GAAiB14B,MACzBiX,EAASggB,EAAMhgB,OACf2N,EAAQqS,EAAMrS,aAEdA,GAAS3N,EAAOja,OAAe,CAAE8E,WAAOrC,EAAW09B,MAAM,IAC7D2C,EAAQtZ,GAAOvP,EAAQ2N,GACvBqS,EAAMrS,OAASkb,EAAM9iC,OACd,CAAE8E,MAAOg+B,EAAO3C,MAAM,OCvB/B,IAAItB,GAAWrG,GAAgB,eAEd,SAAU1J,MACfrsB,MAANqsB,EAAiB,OAAOA,EAAG+P,KAC1B/P,EAAG,eACHqQ,GAAUpP,GAAQjB,QCNR,SAAUA,OACrBiU,EAAiBC,GAAkBlU,MACV,mBAAlBiU,QACH/S,UAAUyB,OAAO3C,GAAM,2BACtB+C,EAASkR,EAAe1jC,KAAKyvB,OCDtC,CAAE6D,OAAQ,SAAUG,MAAM,EAAMS,MAAOhD,GAAe,CACtDyH,OAAQA,KCJV,IAAI94B,GAAS+zB,EAAK/zB,UAED,SAAgBwxB,EAAGtF,UAC3BlsB,GAAO84B,OAAOtH,EAAGtF,OCJTwI,MCAAA,ECejB,OAfA,SAAyBvvB,EAAK4B,EAAKnB,UAC7BmB,KAAO5B,EACT4+B,GAAuB5+B,EAAK4B,EAAK,CAC/BnB,MAAOA,EACP6qB,YAAY,EACZE,cAAc,EACdC,UAAU,IAGZzrB,EAAI4B,GAAOnB,EAGNT,GCTLsxB,GAAsBvG,GAAM,WAAc8T,GAAW,QAIvD,CAAEvQ,OAAQ,SAAUG,MAAM,EAAMK,OAAQwC,IAAuB,CAC/DrvB,KAAM,SAAcwoB,UACXoU,GAAWvd,GAASmJ,OCR/B,ICI2BqU,MDJVlQ,EAAK/zB,OAAOoH,QEDZ,gDCCb88B,GAAa,IAAMC,GAAc,IACjCC,GAAQh4B,OAAO,IAAM83B,GAAaA,GAAa,KAC/CG,GAAQj4B,OAAO83B,GAAaA,GAAa,MAGzCpP,GAAe,SAAUgG,UACpB,SAAU9F,OACXja,EAASwX,OAAOvB,EAAuBgE,WAChC,EAAP8F,IAAU/f,EAASA,EAAO1Q,QAAQ+5B,GAAO,KAClC,EAAPtJ,IAAU/f,EAASA,EAAO1Q,QAAQg6B,GAAO,KACtCtpB,OAIM,CAGfupB,MAAOxP,GAAa,GAGpByP,IAAKzP,GAAa,GAGlB0P,KAAM1P,GAAa,ICxBjB2P,GAAQnW,GAAoCkW,OAK9C,CAAE/Q,OAAQ,SAAUnP,OAAO,EAAM2P,QHARgQ,GGAuC,OHCzD/T,GAAM,mBACFiU,GAAYF,OANf,OAAA,MAMqCA,OAAyBE,GAAYF,IAAa18B,OAAS08B,QGF7B,CAC3EO,KAAM,kBACGC,GAAM3gC,SCPjB,OAAiB,SAAU4gC,UAClB3Q,EAAK2Q,EAAc,iBCAXC,GAAa,UAAUH,KCDpCI,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAG4U,WACQ,iBAAP5U,GAAmBA,IAAOgV,IAClChV,aAAc2C,QAAUsS,IAAQD,GAAgBJ,KAAQA,GAAOK,MCJtD,SAAUZ,EAAatP,OAClCoB,EAAS,GAAGkO,WACPlO,GAAU7F,GAAM,WAEvB6F,EAAO51B,KAAK,KAAMw0B,GAAY,iBAAoB,GAAM,OCHxDxE,GAAiBnwB,OAAOmwB,eACxB2U,GAAQ,GAERC,GAAU,SAAUnV,SAAYA,MAEnB,SAAUqU,EAAahR,MAClCrB,EAAIkT,GAAOb,GAAc,OAAOa,GAAMb,GACrChR,IAASA,EAAU,QACpB8C,EAAS,GAAGkO,GACZe,IAAYpT,EAAIqB,EAAS,cAAeA,EAAQ+R,UAChDC,EAAYrT,EAAIqB,EAAS,GAAKA,EAAQ,GAAK8R,GAC3CG,EAAYtT,EAAIqB,EAAS,GAAKA,EAAQ,QAAK1vB,SAExCuhC,GAAMb,KAAiBlO,IAAW7F,GAAM,cACzC8U,IAAc3T,EAAa,OAAO,MAClCE,EAAI,CAAEzwB,QAAS,GAEfkkC,EAAW7U,GAAeoB,EAAG,EAAG,CAAEd,YAAY,EAAM5iB,IAAKk3B,KACxDxT,EAAE,GAAK,EAEZwE,EAAO51B,KAAKoxB,EAAG0T,EAAWC,OCvB1B/I,GAAW7N,GAAwC3W,QAInDwtB,GAAgBC,GAAoB,WACpCC,GAAiBC,GAAwB,cAI1BH,IAAkBE,GAEjC,GAAG1tB,QAFgD,SAAiBgkB,UAC/DQ,GAASr4B,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCLxE,CAAEkwB,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ,GAAGtc,SAAWA,IAAW,CACjEA,QAASA,KCJX,ICGIvN,GAAOyiB,MDHM8X,GAAa,SAAShtB,QEAnC4tB,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGjY,eACNiY,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe5tB,SAE1E6tB,GAAarkC,eAAe0vB,GAAQjB,IAAOjY,GAAUktB,MCZ3C5O,GAAW,YAAa,cAAgB,GFCrDwP,GAAUzV,EAAOyV,QACjBC,GAAWD,IAAWA,GAAQC,SAC9BC,GAAKD,IAAYA,GAASC,GAG1BA,GAEF9Y,IADAziB,GAAQu7B,GAAGh3B,MAAM,MACD,GAAKvE,GAAM,GAClBw7B,OACTx7B,GAAQw7B,GAAUx7B,MAAM,iBACVA,GAAM,IAAM,MACxBA,GAAQw7B,GAAUx7B,MAAM,oBACbyiB,GAAUziB,GAAM,IAI/B,OAAiByiB,KAAYA,GGfzBoO,GAAU3B,GAAgB,cAEb,SAAU2K,UAIlB4B,IAAc,KAAO3V,GAAM,eAC5B/lB,EAAQ,UACMA,EAAMixB,YAAc,IAC1BH,IAAW,iBACd,CAAE6K,IAAK,IAE2B,IAApC37B,EAAM85B,GAAa8B,SAASD,QCdnCE,GAAO1X,GAAwC7tB,IAI/CwlC,GAAsBC,GAA6B,OAEnDb,GAAiBC,GAAwB,SAK3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnF5kC,IAAK,SAAak7B,UACTqK,GAAKliC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCZxE,OAAiBohC,GAAa,SAASlkC,ICDnC8kC,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGnvB,WACNmvB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe9kC,IAAOA,GAAMokC,GCL1FL,GAAOlW,GAAoCkW,KAG3C2B,GAAYnW,EAAOziB,SACnB64B,GAAM,iBACqC,IAAlCD,GAAUhC,GAAc,OAAmD,KAApCgC,GAAUhC,GAAc,QAIlD,SAAkBppB,EAAQsrB,OAC9C9C,EAAIiB,GAAKjS,OAAOxX,WACborB,GAAU5C,EAAI8C,IAAU,IAAOD,GAAIv7B,KAAK04B,GAAK,GAAK,MACvD4C,KCRF,CAAEnW,QAAQ,EAAMiE,OAAQ1mB,UAAY+4B,IAA0B,CAC9D/4B,SAAU+4B,KCHZ,OAAiBvS,EAAKxmB,SCAlB8iB,GAAuB/B,EAAsDyD,EAG7E+C,GAAe,SAAUyR,UACpB,SAAU3W,WAMX7oB,EALAwqB,EAAIE,EAAgB7B,GACpBxoB,EAAOuuB,GAAWpE,GAClBzwB,EAASsG,EAAKtG,OACdF,EAAI,EACJyb,EAAS,GAENvb,EAASF,GACdmG,EAAMK,EAAKxG,KACNywB,IAAehB,GAAqBlwB,KAAKoxB,EAAGxqB,IAC/CsV,EAAOtb,KAAKwlC,EAAa,CAACx/B,EAAKwqB,EAAExqB,IAAQwqB,EAAExqB,WAGxCsV,ICnBPmqB,GDuBa,CAGf3F,QAAS/L,IAAa,GAGtB1d,OAAQ0d,IAAa,IC7B+B1d,SAIpD,CAAEqc,OAAQ,SAAUG,MAAM,GAAQ,CAClCxc,OAAQ,SAAgBma,UACfiV,GAAQjV,MCJnB,OAAiBwC,EAAK/zB,OAAOoX,OCDzBqvB,GAAUnY,GAAwCyN,OAIlDkK,GAAsBC,GAA6B,UAEnDb,GAAiBC,GAAwB,YAK3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnFtJ,OAAQ,SAAgBJ,UACf8K,GAAQ3iC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCZ3E,OAAiBohC,GAAa,SAAS5I,OCDnCwJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmM,cACNnM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAexJ,OAAUA,GAAS8I,GCOhG6B,GAAuBpN,GAAgB,sBAOvCqN,GAA+Bd,IAAc,KAAO3V,GAAM,eACxD/lB,EAAQ,UACZA,EAAMu8B,KAAwB,EACvBv8B,EAAM+H,SAAS,KAAO/H,KAG3By8B,GAAkBV,GAA6B,UAE/CW,GAAqB,SAAUtV,OAC5BnxB,EAASmxB,GAAI,OAAO,MACrBuV,EAAavV,EAAEmV,gBACGnjC,IAAfujC,IAA6BA,EAAajnC,GAAQ0xB,MAQzD,CAAEkC,OAAQ,QAASnP,OAAO,EAAM2P,QALpB0S,KAAiCC,IAKK,CAClD10B,OAAQ,SAAgBrL,OAIlBjG,EAAG6d,EAAG3d,EAAQiD,EAAKyV,EAHnB+X,EAAI9K,GAAS3iB,MACbijC,EAAIjL,GAAmBvK,EAAG,GAC1Btf,EAAI,MAEHrR,GAAK,EAAGE,EAASlB,UAAUkB,OAAQF,EAAIE,EAAQF,OAClD4Y,GAAW,IAAP5Y,EAAW2wB,EAAI3xB,UAAUgB,GACzBimC,GAAmBrtB,GAAI,IAErBvH,GADJlO,EAAMoxB,EAAS3b,EAAE1Y,SAlCF,iBAmCiB,MAAMgwB,UAlCT,sCAmCxBrS,EAAI,EAAGA,EAAI1a,EAAK0a,IAAKxM,IAASwM,KAAKjF,GAAGgd,GAAeuQ,EAAG90B,EAAGuH,EAAEiF,QAC7D,IACDxM,GAtCW,iBAsCY,MAAM6e,UArCJ,kCAsC7B0F,GAAeuQ,EAAG90B,IAAKuH,UAG3ButB,EAAEjmC,OAASmR,EACJ80B,KCtDX,OAAiBpC,GAAa,SAASzyB,OCDnCqzB,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG1d,cACN0d,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAerzB,OAAUA,GAAS2yB,KCDlG,CAAEpR,OAAQ,QAASG,MAAM,GAAQ,CACjC/zB,QAASA,KCHX,OAAiBk0B,EAAKh0B,MAAMF,WCDX60B,GCUjB,OAVA,SAA4Bh0B,MACtBsmC,GAAetmC,GAAM,KAClB,IAAIE,EAAI,EAAGqmC,EAAO,IAAIlnC,MAAMW,EAAII,QAASF,EAAIF,EAAII,OAAQF,IAC5DqmC,EAAKrmC,GAAKF,EAAIE,UAGTqmC,OCLM,SAAUpQ,EAAUl2B,EAAIiF,EAAOshC,cAErCA,EAAUvmC,EAAGgyB,EAAS/sB,GAAO,GAAIA,EAAM,IAAMjF,EAAGiF,GAEvD,MAAOqqB,OACHkX,EAAetQ,EAAQ,kBACNtzB,IAAjB4jC,GAA4BxU,EAASwU,EAAahnC,KAAK02B,IACrD5G,ICPN0P,GAAWrG,GAAgB,YAC3BiM,GAAiBxlC,MAAME,aAGV,SAAU2vB,eACXrsB,IAAPqsB,IAAqBqQ,GAAUlgC,QAAU6vB,GAAM2V,GAAe5F,MAAc/P,ICNjF+P,GAAWrG,GAAgB,YAC3B8N,IAAe,EAEnB,QACMC,GAAS,EACTC,GAAqB,CACvBxvB,KAAM,iBACG,CAAEmpB,OAAQoG,cAET,WACRD,IAAe,IAGnBE,GAAmB3H,IAAY,kBACtB77B,MAGT/D,MAAMoE,KAAKmjC,IAAoB,iBAAoB,KACnD,MAAOrX,IAET,IClBIsX,IDkBa,SAAUvsB,EAAMwsB,OAC1BA,IAAiBJ,GAAc,OAAO,MACvCK,GAAoB,UAElB7U,EAAS,GACbA,EAAO+M,IAAY,iBACV,CACL7nB,KAAM,iBACG,CAAEmpB,KAAMwG,GAAoB,MAIzCzsB,EAAK4X,GACL,MAAO3C,WACFwX,EChCkBC,EAA4B,SAAUC,GAC/D5nC,MAAMoE,KAAKwjC,QAKX,CAAElU,OAAQ,QAASG,MAAM,EAAMK,OAAQsT,IAAuB,CAC9DpjC,KCAe,SAAcyjC,OAQzB9mC,EAAQub,EAAQwrB,EAAMhR,EAAU/e,EAAMlS,EAPtC2rB,EAAI9K,GAASmhB,GACbzM,EAAmB,mBAARr3B,KAAqBA,KAAO/D,MACvC+nC,EAAkBloC,UAAUkB,OAC5BinC,EAAQD,EAAkB,EAAIloC,UAAU,QAAK2D,EAC7CykC,OAAoBzkC,IAAVwkC,EACVlE,EAAiBC,GAAkBvS,GACnC7I,EAAQ,KAERsf,IAASD,EAAQ5T,EAAK4T,EAAOD,EAAkB,EAAIloC,UAAU,QAAK2D,EAAW,IAE3DA,MAAlBsgC,GAAiC1I,GAAKp7B,OAASkoC,GAAsBpE,OAUvExnB,EAAS,IAAI8e,EADbr6B,EAASq0B,EAAS5D,EAAEzwB,SAEdA,EAAS4nB,EAAOA,IACpB9iB,EAAQoiC,EAAUD,EAAMxW,EAAE7I,GAAQA,GAAS6I,EAAE7I,GAC7C8N,GAAena,EAAQqM,EAAO9iB,YAXhCkS,GADA+e,EAAWgN,EAAe1jC,KAAKoxB,IACfzZ,KAChBuE,EAAS,IAAI8e,IACL0M,EAAO/vB,EAAK3X,KAAK02B,IAAWoK,KAAMvY,IACxC9iB,EAAQoiC,EAAUE,GAA6BrR,EAAUkR,EAAO,CAACF,EAAKjiC,MAAO8iB,IAAQ,GAAQmf,EAAKjiC,MAClG4wB,GAAena,EAAQqM,EAAO9iB,UAUlCyW,EAAOvb,OAAS4nB,EACTrM,KCnCT,OAAiB0X,EAAKh0B,MAAMoE,KCAxBw7B,GAAWrG,GAAgB,eAEd,SAAU1J,OACrB2B,EAAIvxB,OAAO4vB,eACQrsB,IAAhBguB,EAAEoO,KACJ,eAAgBpO,GAEhB0O,GAAU9+B,eAAe0vB,GAAQU,KCHxC,OAJA,SAA0B4W,MACpBC,GAAYpoC,OAAOmoC,KAAmD,uBAAzCnoC,OAAOC,UAAUC,SAASC,KAAKgoC,GAAgC,OAAOE,GAAYF,ICDrH,OAJA,iBACQ,IAAIrX,UAAU,oDCStB,OAJA,SAA4BpwB,UACnB4nC,GAAkB5nC,IAAQ6nC,GAAgB7nC,IAAQ8nC,MCKvDvC,GAAsBC,GAA6B,SACnDb,GAAiBC,GAAwB,QAAS,CAAEN,WAAW,IAAS,IAAM,IAE9E/J,GAAU3B,GAAgB,WAC1BmP,GAAc,GAAGzhC,MACjBoC,GAAM9D,KAAK8D,MAKb,CAAEqqB,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnFr+B,MAAO,SAAes9B,EAAOC,OAMvBmE,EAAarsB,EAAQpK,EALrBsf,EAAIE,EAAgB3tB,MACpBhD,EAASq0B,EAAS5D,EAAEzwB,QACpB2d,EAAI2W,GAAgBkP,EAAOxjC,GAC3B6nC,EAAMvT,QAAwB7xB,IAARghC,EAAoBzjC,EAASyjC,EAAKzjC,MAGxDjB,GAAQ0xB,KAGgB,mBAF1BmX,EAAcnX,EAAE6J,cAEyBsN,IAAgB3oC,QAASF,GAAQ6oC,EAAYzoC,WAE3EG,EAASsoC,IAEE,QADpBA,EAAcA,EAAYzN,OACAyN,OAAcnlC,GAHxCmlC,OAAcnlC,EAKZmlC,IAAgB3oC,YAAyBwD,IAAhBmlC,UACpBD,GAAYtoC,KAAKoxB,EAAG9S,EAAGkqB,OAGlCtsB,EAAS,SAAqB9Y,IAAhBmlC,EAA4B3oC,MAAQ2oC,GAAat/B,GAAIu/B,EAAMlqB,EAAG,IACvExM,EAAI,EAAGwM,EAAIkqB,EAAKlqB,IAAKxM,IAASwM,KAAK8S,GAAGiF,GAAena,EAAQpK,EAAGsf,EAAE9S,WACvEpC,EAAOvb,OAASmR,EACToK,KC3CX,OAAiBsoB,GAAa,SAAS39B,MCDnCu+B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG5oB,aACN4oB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAev+B,MAASA,GAAQ69B,GCA9FpO,GAAsBvG,GAAM,WAAc0Y,GAAqB,QAIjE,CAAEnV,OAAQ,SAAUG,MAAM,EAAMK,OAAQwC,GAAqBpC,MAAO8K,IAA4B,CAChGD,eAAgB,SAAwBtP,UAC/BgZ,GAAqBniB,GAASmJ,OCTzC,OAAiBmE,EAAK/zB,OAAOk/B,kBCDZxK,GCAbmU,GAAWva,GAAuC9gB,QAIlDs7B,GAAgB,GAAGt7B,QAEnBu7B,KAAkBD,IAAiB,EAAI,CAAC,GAAGt7B,QAAQ,GAAI,GAAK,EAC5D23B,GAAgBC,GAAoB,WACpCC,GAAiBC,GAAwB,UAAW,CAAEN,WAAW,IAAS,MAI5E,CAAEvR,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ8U,KAAkB5D,KAAkBE,IAAkB,CAC9F73B,QAAS,SAAiBw7B,UACjBD,GAEHD,GAAcnpC,MAAMmE,KAAMlE,YAAc,EACxCipC,GAAS/kC,KAAMklC,EAAeppC,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MChB5E,OAAiBohC,GAAa,SAASn3B,QCDnC+3B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGpiB,eACNoiB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe/3B,QAAWA,GAAUq3B,MCJrFnQ,GCObuU,GAAejpC,OAAOkpC,OACtB/Y,GAAiBnwB,OAAOmwB,mBAIV8Y,IAAgB/Y,GAAM,cAElCmB,GAQiB,IARF4X,GAAa,CAAE/nC,EAAG,GAAK+nC,GAAa9Y,GAAe,GAAI,IAAK,CAC7EM,YAAY,EACZ5iB,IAAK,WACHsiB,GAAersB,KAAM,IAAK,CACxB8B,MAAO,EACP6qB,YAAY,OAGd,CAAEvvB,EAAG,KAAMA,EAAS,OAAO,MAE3B6lC,EAAI,GACJoC,EAAI,GAEJ3L,EAAS7G,gBAEboQ,EAAEvJ,GAAU,EADG,uBAEN7uB,MAAM,IAAIgJ,SAAQ,SAAUyxB,GAAOD,EAAEC,GAAOA,KACf,GAA/BH,GAAa,GAAIlC,GAAGvJ,IAHZ,wBAG4B7H,GAAWsT,GAAa,GAAIE,IAAIliC,KAAK,OAC7E,SAAgBwsB,EAAQ9J,WACvBiM,EAAInP,GAASgN,GACbqU,EAAkBloC,UAAUkB,OAC5B4nB,EAAQ,EACRsN,EAAwBG,GAA4BpE,EACpD1B,EAAuByB,EAA2BC,EAC/C+V,EAAkBpf,WAKnB3hB,EAJAw8B,EAAIxS,EAAcnxB,UAAU8oB,MAC5BthB,EAAO4uB,EAAwBL,GAAW4N,GAAGrxB,OAAO8jB,EAAsBuN,IAAM5N,GAAW4N,GAC3FziC,EAASsG,EAAKtG,OACd+W,EAAI,EAED/W,EAAS+W,GACd9Q,EAAMK,EAAKyQ,KACNwZ,IAAehB,EAAqBlwB,KAAKojC,EAAGx8B,KAAM6uB,EAAE7uB,GAAOw8B,EAAEx8B,WAE7D6uB,GACPqT,KC9CF,CAAExV,OAAQ,SAAUG,MAAM,EAAMK,OAAQj0B,OAAOkpC,SAAWA,IAAU,CACpEA,OAAQA,KCHV,OAAiBnV,EAAK/zB,OAAOkpC,OCC7B/K,GAAsB,YCCtB,OAAiBkL,GAA6BtX,EAAE,eCH/B2C,sCCEjByJ,GAAsB,wDCAtBA,GAAsB,eCAtBA,GAAsB,sBCAtBA,GAAsB,SCDtBA,GAAsB,YCCtBA,GAAsB,WCAtBA,GAAsB,UCAtBA,GAAsB,WCAtBA,GAAsB,SCAtBA,GAAsB,eCAtBA,GAAsB,eCAtBA,GAAsB,eCAtBW,GAAex5B,KAAM,QAAQ,GCC7Bw5B,GAAe9O,EAAOsZ,KAAM,QAAQ,eCgBpC,OAAiBvV,EAAK4C,OCjBtBwH,GAAsB,gBCAtBA,GAAsB,WCAtBA,GAAsB,cCAtBA,GAAsB,gBCDtBA,GAAsB,cCKtB,OAAiBzJ,8BCJR6U,EAAQpkC,UAIb5F,UAAiBgqC,EADI,mBAAZC,IAAsD,iBAArBC,GACf,SAAiBtkC,iBAC5BA,GAGW,SAAiBA,UACnCA,GAA0B,mBAAZqkC,IAA0BrkC,EAAIi2B,cAAgBoO,IAAWrkC,IAAQqkC,GAAQvpC,UAAY,gBAAkBkF,GAIzHokC,EAAQpkC,GAGjB5F,UAAiBgqC,KCdjB,OAJA,SAAyB7oC,MACnBsmC,GAAetmC,GAAM,OAAOA,GC+BlC,OA9BA,SAA+BA,EAAKE,MAC5BwnC,GAAYpoC,OAAOU,KAAiD,uBAAxCV,OAAOC,UAAUC,SAASC,KAAKO,QAI7DgpC,EAAO,GACPC,GAAK,EACLxmC,GAAK,EACLymC,OAAKrmC,UAGF,IAA4BsmC,EAAxBtlC,EAAKulC,GAAappC,KAAYipC,GAAME,EAAKtlC,EAAGuT,QAAQmpB,QAC3DyI,EAAK3oC,KAAK8oC,EAAGjkC,QAEThF,GAAK8oC,EAAK5oC,SAAWF,GAHyC+oC,GAAK,IAKzE,MAAOI,GACP5mC,GAAK,EACLymC,EAAKG,cAGEJ,GAAsB,MAAhBplC,EAAE,QAAoBA,EAAE,oBAE/BpB,EAAI,MAAMymC,UAIXF,IC3BT,OAJA,iBACQ,IAAI5Y,UAAU,yDCStB,OAJA,SAAwBpwB,EAAKE,UACpBopC,GAAetpC,IAAQupC,GAAqBvpC,EAAKE,IAAMspC,QCH9D,CAAEzW,OAAQ,OAAQG,MAAM,GAAQ,CAChCjb,IAAK,kBACI,IAAInY,MAAO4C,aCHtB,OAAiB2wB,EAAKvzB,KAAKmY,6IC4C3B,SAASwxB,GAAmBC,SA+B5B,mBACQC,EAAOC,KAETC,EAAKF,EAAK,KACVG,EAAKH,EAAK,KACVI,EAAKJ,EAAK,KAELzpC,EAAI,EAAGA,EAAIhB,UAAKkB,OAAQF,KAC/B2pC,GAAMF,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP2pC,GAAM,IAERC,GAAMH,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP4pC,GAAM,IAERC,GAAMJ,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP6pC,GAAM,SAIH,CAACF,EAAIC,EAAIC,GApDGC,CAASN,aAAvBG,OAAIC,OAAIC,OACTpyB,EAAI,EAEF+e,EAAc,eACZvzB,EAAI,QAAU0mC,EAAS,uBAAJlyB,SACzBkyB,EAAKC,EACLA,EAAKC,EACGA,EAAK5mC,GAAKwU,EAAQ,EAAJxU,WAGxBuzB,EAAOuT,OAAS,kBAAyB,WAAXvT,KAE9BA,EAAOwT,QAAU,kBACfxT,IAAyC,uBAAjB,QAAXA,IAAuB,IAEtCA,EAAOyT,UAAY,OACnBzT,EAAOgT,KAAOA,EACdhT,EAAOvK,QAAU,MAEVuK,EA+CT,SAASkT,SACHr4B,EAAI,kBAED,SAASoF,WACR0D,EAAS1D,EAAKnX,WACXU,EAAI,EAAGA,EAAIma,EAAOja,OAAQF,IAAK,KAElC0P,EAAI,oBADR2B,GAAK8I,EAAO2oB,WAAW9iC,IAGvB0P,GADA2B,EAAI3B,IAAM,EAGV2B,GADA3B,GAAK2B,KACK,EAEVA,GAAS,YADT3B,GAAK2B,UAGY,wBAAXA,IAAM;;;;;;;;;;;;;;;;;;;;;;;;;;sCA7FMm4B,2BAAAA,yBACfD,GAAmBC,EAAKtpC,OAASspC,EAAO,CAACU,ytSC9B3C,SAASC,GAAgBC,OAEzB,IAAIC,KAAeD,EAClBA,EAAc7pC,eAAe8pC,KAC/BD,EAAcC,GAAaC,UAAYF,EAAcC,GAAaE,KAClEH,EAAcC,GAAaE,KAAO,IAYjC,SAASC,GAAgBJ,OAEzB,IAAIC,KAAeD,KAClBA,EAAc7pC,eAAe8pC,IAC3BD,EAAcC,GAAaC,UAAW,KACnC,IAAItqC,EAAI,EAAGA,EAAIoqC,EAAcC,GAAaC,UAAUpqC,OAAQF,IAC/DoqC,EAAcC,GAAaC,UAAUtqC,GAAGyqC,WAAWC,YAAYN,EAAcC,GAAaC,UAAUtqC,IAEtGoqC,EAAcC,GAAaC,UAAY,IAUxC,SAASK,GAAcP,GAC5BD,GAAgBC,GAChBI,GAAgBJ,GAChBD,GAAgBC,GAaX,SAASQ,GAAcP,EAAaD,EAAeS,OACpDC,SAEAV,EAAc7pC,eAAe8pC,GAE3BD,EAAcC,GAAaC,UAAUpqC,OAAS,GAChD4qC,EAAUV,EAAcC,GAAaC,UAAU,GAC/CF,EAAcC,GAAaC,UAAUjiB,UAIrCyiB,EAAUxa,SAASya,gBAAgB,6BAA8BV,GACjEQ,EAAahT,YAAYiT,KAK3BA,EAAUxa,SAASya,gBAAgB,6BAA8BV,GACjED,EAAcC,GAAe,CAACE,KAAM,GAAID,UAAW,IACnDO,EAAahT,YAAYiT,IAE3BV,EAAcC,GAAaE,KAAKpqC,KAAK2qC,GAC9BA,EAcF,SAASE,GAAcX,EAAaD,EAAea,EAAcC,OAClEJ,SAEAV,EAAc7pC,eAAe8pC,GAE3BD,EAAcC,GAAaC,UAAUpqC,OAAS,GAChD4qC,EAAUV,EAAcC,GAAaC,UAAU,GAC/CF,EAAcC,GAAaC,UAAUjiB,UAIrCyiB,EAAUxa,SAASE,cAAc6Z,QACZ1nC,IAAjBuoC,EACFD,EAAaC,aAAaJ,EAASI,GAGnCD,EAAapT,YAAYiT,KAM7BA,EAAUxa,SAASE,cAAc6Z,GACjCD,EAAcC,GAAe,CAACE,KAAM,GAAID,UAAW,SAC9B3nC,IAAjBuoC,EACFD,EAAaC,aAAaJ,EAASI,GAGnCD,EAAapT,YAAYiT,IAG7BV,EAAcC,GAAaE,KAAKpqC,KAAK2qC,GAC9BA,EAmBF,SAASK,GAAU99B,EAAGZ,EAAG2+B,EAAehB,EAAeS,EAAcQ,OACtErI,KACuB,UAAvBoI,EAAc1T,QAChBsL,EAAQ4H,GAAc,SAAUR,EAAeS,IACzCS,eAAe,KAAM,KAAMj+B,GACjC21B,EAAMsI,eAAe,KAAM,KAAM7+B,GACjCu2B,EAAMsI,eAAe,KAAM,IAAK,GAAMF,EAAcvI,SAGpDG,EAAQ4H,GAAc,OAAQR,EAAeS,IACvCS,eAAe,KAAM,IAAKj+B,EAAI,GAAM+9B,EAAcvI,MACxDG,EAAMsI,eAAe,KAAM,IAAK7+B,EAAI,GAAM2+B,EAAcvI,MACxDG,EAAMsI,eAAe,KAAM,QAASF,EAAcvI,MAClDG,EAAMsI,eAAe,KAAM,SAAUF,EAAcvI,YAGxBlgC,IAAzByoC,EAAcG,QAChBvI,EAAMsI,eAAe,KAAM,QAASF,EAAcG,QAEpDvI,EAAMsI,eAAe,KAAM,QAASF,EAAcI,UAAY,cAI1DH,EAAU,KACRI,EAAQb,GAAc,OAAQR,EAAeS,GAC7CQ,EAASK,UACXr+B,GAAQg+B,EAASK,SAGfL,EAASM,UACXl/B,GAAQ4+B,EAASM,SAEfN,EAAStU,UACX0U,EAAMG,YAAcP,EAAStU,SAG3BsU,EAASG,WACXC,EAAMH,eAAe,KAAM,QAASD,EAASG,UAAa,cAE5DC,EAAMH,eAAe,KAAM,IAAKj+B,GAChCo+B,EAAMH,eAAe,KAAM,IAAK7+B,UAG3Bu2B,EAeF,SAAS6I,GAASx+B,EAAGZ,EAAGq/B,EAAOC,EAAQP,EAAWpB,EAAeS,EAAcnT,MACtE,GAAVqU,EAAa,CACXA,EAAS,IAEXt/B,GADAs/B,IAAW,OAGTC,EAAOpB,GAAc,OAAOR,EAAeS,GAC/CmB,EAAKV,eAAe,KAAM,IAAKj+B,EAAI,GAAMy+B,GACzCE,EAAKV,eAAe,KAAM,IAAK7+B,GAC/Bu/B,EAAKV,eAAe,KAAM,QAASQ,GACnCE,EAAKV,eAAe,KAAM,SAAUS,GACpCC,EAAKV,eAAe,KAAM,QAASE,GAC/B9T,GACFsU,EAAKV,eAAe,KAAM,QAAS5T,IASlC,SAASuU,gBAEPC,UACDA,UAAUC,WAAaD,UAAUC,UAAUjsC,OACtCgsC,UAAUC,UAEVD,UAAUE,cAAgBF,UAAUG,UAAYH,UAAUI,iBAAmB,KAJ/D,KAOzB,MAAMjd,SACG,sLCnOX,OARA,SAAgCF,WACjB,IAATA,QACI,IAAIod,eAAe,oEAGpBpd,MCFQ4U,GAAa,SAAS9D,QCAnC0E,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGiR,eACNjR,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe1E,SAE1E2E,GAAarkC,eAAe0vB,GAAQjB,IAAOiR,GAAUgE,GCVxDlI,GAAa1G,GAAW,OAAQ,aAChCmX,GAAK,mBACLC,GAAM,oBACNC,GAAK,oBAELC,GAAM,SAAUnjC,EAAOkW,EAAQvF,OAC7ByyB,EAAOzyB,EAAOuP,OAAOhK,EAAS,GAC9BxI,EAAOiD,EAAOuP,OAAOhK,EAAS,UAC7B+sB,GAAIxiC,KAAKT,KAAWkjC,GAAGziC,KAAKiN,IAAWw1B,GAAGziC,KAAKT,KAAWijC,GAAIxiC,KAAK2iC,GAC/D,MAAQpjC,EAAMs5B,WAAW,GAAGxjC,SAAS,IACrCkK,GAGP41B,GAAS9P,GAAM,iBACqB,qBAA/ByM,GAAW,iBACY,cAAzBA,GAAW,aAGdA,IAEF8B,EAAE,CAAEhL,OAAQ,OAAQG,MAAM,EAAMK,OAAQ+L,IAAU,CAEhDtB,UAAW,SAAmB9O,EAAI+O,EAAUC,OACtCviB,EAASsgB,GAAWh9B,MAAM,KAAMC,iBACZ,iBAAVyc,EAAqBA,EAAOhS,QAAQ+iC,GAAIG,IAAOlxB,KCzB9DoxB,EAAKnE,OAAMmE,EAAKnE,KAAO,CAAE5K,UAAW4K,KAAK5K,YAG9C,OAAiB,SAAmB9O,EAAI+O,EAAUC,UACzC6O,EAAKnE,KAAK5K,UAAU/+B,MAAM,KAAMC,eCJxB+kC,GAAa,SAASvtB,OCAnCmuB,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGxY,cACNwY,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAenuB,QAE1EouB,GAAarkC,eAAe0vB,GAAQjB,IAAOxY,GAASytB,MCX1CF,GAAa,SAASv9B,KCAnCm+B,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGxoB,YACNwoB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAen+B,MAE1Eo+B,GAAarkC,eAAe0vB,GAAQjB,IAAOxoB,GAAOy9B,GCZrD6I,GAAQpf,GAAwC7uB,KAIhD0lC,GAAgBC,GAAoB,QACpCC,GAAiBC,GAAwB,UAI3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7E5lC,KAAM,SAAck8B,UACX+R,GAAM5pC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCVzE,OAAiBohC,GAAa,SAASllC,KCDnC8lC,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGnwB,YACNmwB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe9lC,KAAQA,GAAOolC,GCMhG,OARA,SAAoC9U,EAAM5vB,UACpCA,GAA2B,WAAlBopC,GAAQppC,IAAsC,mBAATA,EAI3CwtC,GAAsB5d,GAHpB5vB,MCJMu0B,KCGf,CAAEjB,OAAQ,SAAUG,MAAM,GAAQ,CAClC0L,eAAgBA,KCHlB,OAAiBvL,EAAK/zB,OAAOs/B,0CCCpBsO,EAAgBz/B,UACvB5O,UAAiBquC,EAAkBC,GAAyBC,GAAyB,SAAyB3/B,UACrGA,EAAEsxB,WAAaqO,GAAuB3/B,IAExCy/B,EAAgBz/B,GAGzB5O,UAAiBquC,QCTAlZ,8BCARqZ,EAAgB5/B,EAAG6/B,UAC1BzuC,UAAiBwuC,EAAkBF,IAA0B,SAAyB1/B,EAAG6/B,UACvF7/B,EAAEsxB,UAAYuO,EACP7/B,GAGF4/B,EAAgB5/B,EAAG6/B,GAG5BzuC,UAAiBwuC,KCQjB,OAfA,SAAmBE,EAAUC,MACD,mBAAfA,GAA4C,OAAfA,QAChC,IAAIpd,UAAU,sDAGtBmd,EAAShuC,UAAYkuC,GAAeD,GAAcA,EAAWjuC,UAAW,CACtEm7B,YAAa,CACXx1B,MAAOqoC,EACPrd,UAAU,EACVD,cAAc,KAGdud,GAAY5O,GAAe2O,EAAUC,ICTvCrjC,GAAO,GACPujC,GAAavjC,GAAKkF,KAGlBs+B,GAAqBne,GAAM,WAC7BrlB,GAAKkF,UAAKxM,MAGR+qC,GAAgBpe,GAAM,WACxBrlB,GAAKkF,KAAK,SAGRo1B,GAAgBC,GAAoB,UAMtC,CAAE3R,OAAQ,QAASnP,OAAO,EAAM2P,OAJrBoa,KAAuBC,KAAkBnJ,IAIF,CAClDp1B,KAAM,SAAcw+B,eACGhrC,IAAdgrC,EACHH,GAAWjuC,KAAKsmB,GAAS3iB,OACzBsqC,GAAWjuC,KAAKsmB,GAAS3iB,MAAO0uB,EAAU+b,OC1BlD,OAAiB5J,GAAa,SAAS50B,KCDnCw1B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG7f,YACN6f,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAex1B,KAAQA,GAAO80B,OCJ9E3U,GAAM,kBACflwB,OAAOwuC,aAAaxuC,OAAOyuC,kBAAkB,8BCAlDte,EAAiB7B,EAA+CyD,EAIhE2c,EAAWpX,GAAI,QACfJ,EAAK,EAELsX,EAAexuC,OAAOwuC,cAAgB,kBACjC,GAGLG,EAAc,SAAU/e,GAC1BO,EAAeP,EAAI8e,EAAU,CAAE9oC,MAAO,CACpCgpC,SAAU,OAAQ1X,EAClB2X,SAAU,OAoCVC,EAAOvvC,UAAiB,CAC1BwvC,UAAU,EACVC,QAlCY,SAAUpf,EAAIkJ,OAErB14B,EAASwvB,GAAK,MAAoB,iBAANA,EAAiBA,GAAmB,iBAANA,EAAiB,IAAM,KAAOA,MACxFgC,EAAIhC,EAAI8e,GAAW,KAEjBF,EAAa5e,GAAK,MAAO,QAEzBkJ,EAAQ,MAAO,IAEpB6V,EAAY/e,UAELA,EAAG8e,GAAUE,UAwBtBK,YArBgB,SAAUrf,EAAIkJ,OACzBlH,EAAIhC,EAAI8e,GAAW,KAEjBF,EAAa5e,GAAK,OAAO,MAEzBkJ,EAAQ,OAAO,EAEpB6V,EAAY/e,UAELA,EAAG8e,GAAUG,UAatBK,SATa,SAAUtf,UACnBuf,IAAYL,EAAKC,UAAYP,EAAa5e,KAAQgC,EAAIhC,EAAI8e,IAAWC,EAAY/e,GAC9EA,IAUT0F,GAAWoZ,IAAY,8ECrDnBU,EAAS,SAAUC,EAAShzB,QACzBgzB,QAAUA,OACVhzB,OAASA,IAGF9c,UAAiB,SAAUooC,EAAUhnC,EAAImkB,EAAMwqB,EAAYC,OAEnE1Y,EAAU2Y,EAAQ9mB,EAAO5nB,EAAQub,EAAQvE,EAAM+vB,EAD/ChM,EAAgB1H,EAAKxzB,EAAImkB,EAAMwqB,EAAa,EAAI,MAGhDC,EACF1Y,EAAW8Q,MACN,IAEgB,mBADrB6H,EAAS1L,GAAkB6D,IACM,MAAM7W,UAAU,6BAE7CmX,GAAsBuH,GAAS,KAC5B9mB,EAAQ,EAAG5nB,EAASq0B,EAASwS,EAAS7mC,QAASA,EAAS4nB,EAAOA,QAClErM,EAASizB,EACLzT,EAAclJ,EAASkV,EAAOF,EAASjf,IAAQ,GAAImf,EAAK,IACxDhM,EAAc8L,EAASjf,MACbrM,aAAkB+yB,EAAQ,OAAO/yB,SACxC,IAAI+yB,GAAO,GAEtBvY,EAAW2Y,EAAOrvC,KAAKwnC,OAGzB7vB,EAAO+e,EAAS/e,OACP+vB,EAAO/vB,EAAK3X,KAAK02B,IAAWoK,SAEd,iBADrB5kB,EAAS6rB,GAA6BrR,EAAUgF,EAAegM,EAAKjiC,MAAO0pC,KAC1CjzB,GAAUA,aAAkB+yB,EAAQ,OAAO/yB,SACrE,IAAI+yB,GAAO,KAGdK,KAAO,SAAUpzB,UAChB,IAAI+yB,GAAO,EAAM/yB,WCzCT,SAAUuT,EAAI8Y,EAAanhC,QACpCqoB,aAAc8Y,SACZ5X,UAAU,cAAgBvpB,EAAOA,EAAO,IAAM,IAAM,qBACnDqoB,GCOPO,GAAiB7B,EAA+CyD,EAChEpa,GAAU+3B,GAAwC/3B,QAIlD2kB,GAAmBC,GAAoBz0B,IACvC6nC,GAAyBpT,GAAoB1B,aAEhC,SAAU+U,EAAkBC,EAASC,OAOhDpH,EANArN,GAA8C,IAArCuU,EAAiBpiC,QAAQ,OAClCuiC,GAAgD,IAAtCH,EAAiBpiC,QAAQ,QACnCwiC,EAAQ3U,EAAS,MAAQ,MACzBtI,EAAoB/C,EAAO4f,GAC3BK,EAAkBld,GAAqBA,EAAkB9yB,UACzDiwC,EAAW,MAGV7e,GAA2C,mBAArB0B,IACpBgd,GAAWE,EAAgBt4B,UAAYuY,GAAM,gBAAkB6C,GAAoB8N,UAAU/oB,WAK7F,CACL4wB,EAAcmH,GAAQ,SAAUpc,EAAQkU,GACtCrL,GAAiB6T,GAAW1c,EAAQiV,EAAakH,GAAmB,CAClE5U,KAAM4U,EACNQ,WAAY,IAAIrd,IAEFxvB,MAAZokC,GAAuB0I,GAAQ1I,EAAUlU,EAAOuc,GAAQvc,EAAQ4H,UAGlEmB,EAAmBmT,GAAuBC,GAE9Cj4B,GAAQ,CAAC,MAAO,QAAS,SAAU,UAAW,MAAO,MAAO,MAAO,OAAQ,SAAU,YAAY,SAAU0oB,OACrGiQ,EAAkB,OAAPjQ,GAAuB,OAAPA,EAC3BA,KAAO4P,KAAqBF,GAAkB,SAAP1P,IACzC/L,EAA4BoU,EAAYzoC,UAAWogC,GAAK,SAAUp/B,EAAGC,OAC/DkvC,EAAa5T,EAAiB14B,MAAMssC,eACnCE,GAAYP,IAAY3vC,EAASa,GAAI,MAAc,OAAPo/B,QAAe98B,MAC5D8Y,EAAS+zB,EAAW/P,GAAW,IAANp/B,EAAU,EAAIA,EAAGC,UACvCovC,EAAWxsC,KAAOuY,QAK/B0zB,GAAW5f,GAAeuY,EAAYzoC,UAAW,OAAQ,CACvD0wB,cAAc,EACd9iB,IAAK,kBACI2uB,EAAiB14B,MAAMssC,WAAW3M,aA5B7CiF,EAAcoH,EAAOS,eAAeV,EAASD,EAAkBvU,EAAQ2U,GACvEQ,GAAuBzB,UAAW,SAgCpCjQ,GAAe4J,EAAakH,GAAkB,GAAO,GAErDM,EAASN,GAAoBlH,EAC7BjK,EAAE,CAAEzO,QAAQ,EAAMiE,QAAQ,GAAQic,GAE7BH,GAASD,EAAOW,UAAU/H,EAAakH,EAAkBvU,GAEvDqN,MCrEQ,SAAUjV,EAAQiF,EAAKzF,OACjC,IAAIlsB,KAAO2xB,EACVzF,GAAWA,EAAQyd,QAAUjd,EAAO1sB,GAAM0sB,EAAO1sB,GAAO2xB,EAAI3xB,GAC3Dm3B,GAASzK,EAAQ1sB,EAAK2xB,EAAI3xB,GAAMksB,UAC9BQ,GCAPwH,GAAU3B,GAAgB,WCL1BnJ,GAAiB7B,EAA+CyD,EAShEid,GAAUU,GAA0CV,QAGpD1S,GAAmBC,GAAoBz0B,IACvC6nC,GAAyBpT,GAAoB1B,aAEhC,CACf0V,eAAgB,SAAUV,EAASD,EAAkBvU,EAAQ2U,OACvD7U,EAAI0U,GAAQ,SAAU/qB,EAAM6iB,GAC9BwI,GAAWrrB,EAAMqW,EAAGyU,GACpBtT,GAAiBxX,EAAM,CACrBkW,KAAM4U,EACNlnB,MAAOoQ,GAAO,MACdwK,WAAO//B,EACPotC,UAAMptC,EACNkgC,KAAM,IAEHpS,IAAavM,EAAK2e,KAAO,GACdlgC,MAAZokC,GAAuB0I,GAAQ1I,EAAU7iB,EAAKkrB,GAAQlrB,EAAMuW,MAG9DmB,EAAmBmT,GAAuBC,GAE1CgB,EAAS,SAAU9rB,EAAM/d,EAAKnB,OAG5BirC,EAAUnoB,EAFVqS,EAAQyB,EAAiB1X,GACzBgsB,EAAQC,EAASjsB,EAAM/d,UAGvB+pC,EACFA,EAAMlrC,MAAQA,GAGdm1B,EAAM4V,KAAOG,EAAQ,CACnBpoB,MAAOA,EAAQsmB,GAAQjoC,GAAK,GAC5BA,IAAKA,EACLnB,MAAOA,EACPirC,SAAUA,EAAW9V,EAAM4V,KAC3B74B,UAAMvU,EACNytC,SAAS,GAENjW,EAAMuI,QAAOvI,EAAMuI,MAAQwN,GAC5BD,IAAUA,EAAS/4B,KAAOg5B,GAC1Bzf,EAAa0J,EAAM0I,OAClB3e,EAAK2e,OAEI,MAAV/a,IAAeqS,EAAMrS,MAAMA,GAASooB,IACjChsB,GAGPisB,EAAW,SAAUjsB,EAAM/d,OAIzB+pC,EAHA/V,EAAQyB,EAAiB1X,GAEzB4D,EAAQsmB,GAAQjoC,MAEN,MAAV2hB,EAAe,OAAOqS,EAAMrS,MAAMA,OAEjCooB,EAAQ/V,EAAMuI,MAAOwN,EAAOA,EAAQA,EAAMh5B,QACzCg5B,EAAM/pC,KAAOA,EAAK,OAAO+pC,UAIjCG,GAAY9V,EAAEl7B,UAAW,CAGvBixC,MAAO,mBAEDnW,EAAQyB,EADD14B,MAEPuT,EAAO0jB,EAAMrS,MACbooB,EAAQ/V,EAAMuI,MACXwN,GACLA,EAAME,SAAU,EACZF,EAAMD,WAAUC,EAAMD,SAAWC,EAAMD,SAAS/4B,UAAOvU,UACpD8T,EAAKy5B,EAAMpoB,OAClBooB,EAAQA,EAAMh5B,KAEhBijB,EAAMuI,MAAQvI,EAAM4V,UAAOptC,EACvB8tB,EAAa0J,EAAM0I,KAAO,EAXnB3/B,KAYD2/B,KAAO,UAIT,SAAU18B,OAEdg0B,EAAQyB,EADD14B,MAEPgtC,EAAQC,EAFDjtC,KAEgBiD,MACvB+pC,EAAO,KACLh5B,EAAOg5B,EAAMh5B,KACb01B,EAAOsD,EAAMD,gBACV9V,EAAMrS,MAAMooB,EAAMpoB,OACzBooB,EAAME,SAAU,EACZxD,IAAMA,EAAK11B,KAAOA,GAClBA,IAAMA,EAAK+4B,SAAWrD,GACtBzS,EAAMuI,OAASwN,IAAO/V,EAAMuI,MAAQxrB,GACpCijB,EAAM4V,MAAQG,IAAO/V,EAAM4V,KAAOnD,GAClCnc,EAAa0J,EAAM0I,OAZd3/B,KAaC2/B,eACDqN,GAIbn5B,QAAS,SAAiBgkB,WAGpBmV,EAFA/V,EAAQyB,EAAiB14B,MACzB+3B,EAAgB1H,EAAKwH,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,EAAW,GAE/EutC,EAAQA,EAAQA,EAAMh5B,KAAOijB,EAAMuI,WACxCzH,EAAciV,EAAMlrC,MAAOkrC,EAAM/pC,IAAKjD,MAE/BgtC,GAASA,EAAME,SAASF,EAAQA,EAAMD,UAKjDjf,IAAK,SAAa7qB,WACPgqC,EAASjtC,KAAMiD,MAI5BkqC,GAAY9V,EAAEl7B,UAAWo7B,EAAS,CAEhCxtB,IAAK,SAAa9G,OACZ+pC,EAAQC,EAASjtC,KAAMiD,UACpB+pC,GAASA,EAAMlrC,OAGxBkC,IAAK,SAAaf,EAAKnB,UACdgrC,EAAO9sC,KAAc,IAARiD,EAAY,EAAIA,EAAKnB,KAEzC,CAEF+Y,IAAK,SAAa/Y,UACTgrC,EAAO9sC,KAAM8B,EAAkB,IAAVA,EAAc,EAAIA,EAAOA,MAGrDyrB,GAAalB,GAAegL,EAAEl7B,UAAW,OAAQ,CACnD4N,IAAK,kBACI2uB,EAAiB14B,MAAM2/B,QAG3BtI,GAETsV,UAAW,SAAUtV,EAAGyU,EAAkBvU,OACpC8V,EAAgBvB,EAAmB,YACnCwB,EAA6BzB,GAAuBC,GACpDyB,EAA2B1B,GAAuBwB,GAGtDrQ,GAAe3F,EAAGyU,GAAkB,SAAU7O,EAAUC,GACtD1E,GAAiBx4B,KAAM,CACrBk3B,KAAMmW,EACN1d,OAAQsN,EACRhG,MAAOqW,EAA2BrQ,GAClCC,KAAMA,EACN2P,UAAMptC,OAEP,mBACGw3B,EAAQsW,EAAyBvtC,MACjCk9B,EAAOjG,EAAMiG,KACb8P,EAAQ/V,EAAM4V,KAEXG,GAASA,EAAME,SAASF,EAAQA,EAAMD,gBAExC9V,EAAMtH,SAAYsH,EAAM4V,KAAOG,EAAQA,EAAQA,EAAMh5B,KAAOijB,EAAMA,MAAMuI,OAMjE,QAARtC,EAAuB,CAAEp7B,MAAOkrC,EAAM/pC,IAAKk6B,MAAM,GACzC,UAARD,EAAyB,CAAEp7B,MAAOkrC,EAAMlrC,MAAOq7B,MAAM,GAClD,CAAEr7B,MAAO,CAACkrC,EAAM/pC,IAAK+pC,EAAMlrC,OAAQq7B,MAAM,IAN9ClG,EAAMtH,YAASlwB,EACR,CAAEqC,WAAOrC,EAAW09B,MAAM,MAMlC5F,EAAS,UAAY,UAAWA,GAAQ,GD5K9B,SAAUuU,OACrBlH,EAAczS,GAAW2Z,GACzBzf,EAAiB0C,EAAqBd,EAEtCV,GAAeqX,IAAgBA,EAAYzN,KAC7C9K,EAAeuY,EAAazN,GAAS,CACnCtK,cAAc,EACd9iB,IAAK,kBAAqB/J,QCwK5BwtC,CAAW1B,SCjLEQ,GAAW,OAAO,SAAUmB,UACpC,kBAAwBA,EAAKztC,KAAMlE,UAAUkB,OAASlB,UAAU,QAAK2D,MAC3EiuC,ICFczd,EAAK0d,SCALrB,GAAW,OAAO,SAAUmB,UACpC,kBAAwBA,EAAKztC,KAAMlE,UAAUkB,OAASlB,UAAU,QAAK2D,MAC3EiuC,ICFczd,EAAK2d,2BCClBC,EAAW,SAAU56B,OAGnB66B,EAAK5xC,OAAOC,UACZ4xC,EAASD,EAAGzwC,eAEZu7B,EAA4B,mBAAX/F,OAAwBA,OAAS,GAClDmb,EAAiBpV,EAAQ7F,UAAY,aACrCkb,EAAsBrV,EAAQsV,eAAiB,kBAC/CC,EAAoBvV,EAAQwV,aAAe,yBAEtC9d,EAAK+d,EAASC,EAASriB,EAAMsiB,OAEhCC,EAAiBF,GAAWA,EAAQnyC,qBAAqBsyC,EAAYH,EAAUG,EAC/EC,EAAYxyC,OAAO84B,OAAOwZ,EAAeryC,WACzCwyC,EAAU,IAAIC,EAAQL,GAAe,WAIzCG,EAAUG,iBAqMcR,EAASpiB,EAAM0iB,OACnC1X,EA9KuB,wBAgLpB,SAAgBhF,EAAQlvB,MA9KT,cA+KhBk0B,QACI,IAAI7zB,MAAM,mCA/KE,cAkLhB6zB,EAA6B,IAChB,UAAXhF,QACIlvB,SAKD+rC,QAGTH,EAAQ1c,OAASA,EACjB0c,EAAQ5rC,IAAMA,IAED,KACPgsC,EAAWJ,EAAQI,YACnBA,EAAU,KACRC,EAAiBC,EAAoBF,EAAUJ,MAC/CK,EAAgB,IACdA,IAAmBE,EAAkB,gBAClCF,MAIY,SAAnBL,EAAQ1c,OAGV0c,EAAQQ,KAAOR,EAAQS,MAAQT,EAAQ5rC,SAElC,GAAuB,UAAnB4rC,EAAQ1c,OAAoB,IAjNhB,mBAkNjBgF,QACFA,EAhNc,YAiNR0X,EAAQ5rC,IAGhB4rC,EAAQU,kBAAkBV,EAAQ5rC,SAEN,WAAnB4rC,EAAQ1c,QACjB0c,EAAQW,OAAO,SAAUX,EAAQ5rC,KAGnCk0B,EA3NkB,gBA6NdsY,EAASC,EAASnB,EAASpiB,EAAM0iB,MACjB,WAAhBY,EAAOrY,KAAmB,IAG5BD,EAAQ0X,EAAQxR,KAhOA,YAFK,iBAsOjBoS,EAAOxsC,MAAQmsC,iBAIZ,CACLptC,MAAOytC,EAAOxsC,IACdo6B,KAAMwR,EAAQxR,MAGS,UAAhBoS,EAAOrY,OAChBD,EA9OgB,YAiPhB0X,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAMwsC,EAAOxsC,OA7QP0sC,CAAiBpB,EAASpiB,EAAM0iB,GAE7CD,WAcAc,EAAS3yC,EAAIwE,EAAK0B,aAEhB,CAAEm0B,KAAM,SAAUn0B,IAAKlG,EAAGR,KAAKgF,EAAK0B,IAC3C,MAAOkjC,SACA,CAAE/O,KAAM,QAASn0B,IAAKkjC,IAhBjChzB,EAAQqd,KAAOA,MA2BX4e,EAAmB,YAMdT,cACAiB,cACAC,SAIL1U,EAAoB,GACxBA,EAAkB+S,GAAkB,kBAC3BhuC,UAGL4vC,EAAW1zC,OAAOk/B,eAClByU,EAA0BD,GAAYA,EAASA,EAASt8B,EAAO,MAC/Du8B,GACAA,IAA4B/B,GAC5BC,EAAO1xC,KAAKwzC,EAAyB7B,KAGvC/S,EAAoB4U,OAGlBC,EAAKH,EAA2BxzC,UAClCsyC,EAAUtyC,UAAYD,OAAO84B,OAAOiG,YAQ7B8U,EAAsB5zC,IAC5B,OAAQ,QAAS,UAAU0X,SAAQ,SAASoe,GAC3C91B,EAAU81B,GAAU,SAASlvB,UACpB/C,KAAK6uC,QAAQ5c,EAAQlvB,gBAoCzBitC,EAActB,EAAWuB,OAgC5BC,OAgCCrB,iBA9BY5c,EAAQlvB,YACdotC,WACA,IAAIF,GAAY,SAASG,EAASC,aAnCpCC,EAAOre,EAAQlvB,EAAKqtC,EAASC,OAChCd,EAASC,EAASd,EAAUzc,GAASyc,EAAW3rC,MAChC,UAAhBwsC,EAAOrY,KAEJ,KACD3e,EAASg3B,EAAOxsC,IAChBjB,EAAQyW,EAAOzW,aACfA,GACiB,iBAAVA,GACPisC,EAAO1xC,KAAKyF,EAAO,WACdmuC,EAAYG,QAAQtuC,EAAMyuC,SAASC,MAAK,SAAS1uC,GACtDwuC,EAAO,OAAQxuC,EAAOsuC,EAASC,MAC9B,SAASpK,GACVqK,EAAO,QAASrK,EAAKmK,EAASC,MAI3BJ,EAAYG,QAAQtuC,GAAO0uC,MAAK,SAASC,GAI9Cl4B,EAAOzW,MAAQ2uC,EACfL,EAAQ73B,MACP,SAAS4T,UAGHmkB,EAAO,QAASnkB,EAAOikB,EAASC,MAvBzCA,EAAOd,EAAOxsC,KAiCZutC,CAAOre,EAAQlvB,EAAKqtC,EAASC,aAI1BH,EAaLA,EAAkBA,EAAgBM,KAChCL,EAGAA,GACEA,cAkHDlB,EAAoBF,EAAUJ,OACjC1c,EAAS8c,EAAShc,SAAS4b,EAAQ1c,gBAtSrCxyB,IAuSEwyB,EAAsB,IAGxB0c,EAAQI,SAAW,KAEI,UAAnBJ,EAAQ1c,OAAoB,IAE1B8c,EAAShc,SAAT,SAGF4b,EAAQ1c,OAAS,SACjB0c,EAAQ5rC,SAlTZtD,EAmTIwvC,EAAoBF,EAAUJ,GAEP,UAAnBA,EAAQ1c,eAGHid,EAIXP,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAM,IAAIiqB,UAChB,yDAGGkiB,MAGLK,EAASC,EAASvd,EAAQ8c,EAAShc,SAAU4b,EAAQ5rC,QAErC,UAAhBwsC,EAAOrY,YACTyX,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAMwsC,EAAOxsC,IACrB4rC,EAAQI,SAAW,KACZG,MAGLwB,EAAOnB,EAAOxsC,WAEZ2tC,EAOFA,EAAKvT,MAGPwR,EAAQI,EAAS4B,YAAcD,EAAK5uC,MAGpC6sC,EAAQ36B,KAAO+6B,EAAS6B,QAQD,WAAnBjC,EAAQ1c,SACV0c,EAAQ1c,OAAS,OACjB0c,EAAQ5rC,SAtWVtD,GAgXFkvC,EAAQI,SAAW,KACZG,GANEwB,GA3BP/B,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAM,IAAIiqB,UAAU,oCAC5B2hB,EAAQI,SAAW,KACZG,YAoDF2B,EAAaC,OAChB9D,EAAQ,CAAE+D,OAAQD,EAAK,IAEvB,KAAKA,IACP9D,EAAMgE,SAAWF,EAAK,IAGpB,KAAKA,IACP9D,EAAMiE,WAAaH,EAAK,GACxB9D,EAAMkE,SAAWJ,EAAK,SAGnBK,WAAWl0C,KAAK+vC,YAGdoE,EAAcpE,OACjBuC,EAASvC,EAAMqE,YAAc,GACjC9B,EAAOrY,KAAO,gBACPqY,EAAOxsC,IACdiqC,EAAMqE,WAAa9B,WAGZX,EAAQL,QAIV4C,WAAa,CAAC,CAAEJ,OAAQ,SAC7BxC,EAAY16B,QAAQg9B,EAAc7wC,WAC7BsxC,OAAM,YA8BJh+B,EAAOuwB,MACVA,EAAU,KACR9D,EAAiB8D,EAASmK,MAC1BjO,SACKA,EAAe1jC,KAAKwnC,MAGA,mBAAlBA,EAAS7vB,YACX6vB,MAGJzkC,MAAMykC,EAAS7mC,QAAS,KACvBF,GAAK,EAAGkX,EAAO,SAASA,WACjBlX,EAAI+mC,EAAS7mC,WAChB+wC,EAAO1xC,KAAKwnC,EAAU/mC,UACxBkX,EAAKlS,MAAQ+hC,EAAS/mC,GACtBkX,EAAKmpB,MAAO,EACLnpB,SAIXA,EAAKlS,WAtdTrC,EAudIuU,EAAKmpB,MAAO,EAELnpB,UAGFA,EAAKA,KAAOA,SAKhB,CAAEA,KAAM86B,YAIRA,UACA,CAAEhtC,WAtePrC,EAseyB09B,MAAM,UA5ZnCuS,EAAkBvzC,UAAY2zC,EAAGxY,YAAcqY,EAC/CA,EAA2BrY,YAAcoY,EACzCC,EAA2BxB,GACzBuB,EAAkB6B,YAAc,oBAYlCt+B,EAAQu+B,oBAAsB,SAASC,OACjCC,EAAyB,mBAAXD,GAAyBA,EAAOna,oBAC3Coa,IACHA,IAAShC,GAG2B,uBAAnCgC,EAAKH,aAAeG,EAAKjuC,QAIhCwP,EAAQ0+B,KAAO,SAASF,UAClBv1C,OAAOs/B,eACTt/B,OAAOs/B,eAAeiW,EAAQ9B,IAE9B8B,EAAO9V,UAAYgU,EACbxB,KAAqBsD,IACzBA,EAAOtD,GAAqB,sBAGhCsD,EAAOt1C,UAAYD,OAAO84B,OAAO8a,GAC1B2B,GAOTx+B,EAAQ2+B,MAAQ,SAAS7uC,SAChB,CAAEwtC,QAASxtC,IAsEpBgtC,EAAsBC,EAAc7zC,WACpC6zC,EAAc7zC,UAAU8xC,GAAuB,kBACtCjuC,MAETiT,EAAQ+8B,cAAgBA,EAKxB/8B,EAAQ4+B,MAAQ,SAASxD,EAASC,EAASriB,EAAMsiB,EAAa0B,QACxC,IAAhBA,IAAwBA,EAAc6B,aAEtCzN,EAAO,IAAI2L,EACb1f,EAAK+d,EAASC,EAASriB,EAAMsiB,GAC7B0B,UAGKh9B,EAAQu+B,oBAAoBlD,GAC/BjK,EACAA,EAAKrwB,OAAOw8B,MAAK,SAASj4B,UACjBA,EAAO4kB,KAAO5kB,EAAOzW,MAAQuiC,EAAKrwB,WAuKjD+7B,EAAsBD,GAEtBA,EAAG3B,GAAqB,YAOxB2B,EAAG9B,GAAkB,kBACZhuC,MAGT8vC,EAAG1zC,SAAW,iBACL,sBAkCT6W,EAAQ3P,KAAO,SAASwrB,OAClBxrB,EAAO,OACN,IAAIL,KAAO6rB,EACdxrB,EAAKrG,KAAKgG,UAEZK,EAAKyuC,UAIE,SAAS/9B,SACP1Q,EAAKtG,QAAQ,KACdiG,EAAMK,EAAK0uC,SACX/uC,KAAO6rB,SACT9a,EAAKlS,MAAQmB,EACb+Q,EAAKmpB,MAAO,EACLnpB,SAOXA,EAAKmpB,MAAO,EACLnpB,IAsCXf,EAAQK,OAASA,EAMjBs7B,EAAQzyC,UAAY,CAClBm7B,YAAasX,EAEb0C,MAAO,SAASW,WACTvI,KAAO,OACP11B,KAAO,OAGPm7B,KAAOnvC,KAAKovC,WAjfjB3vC,OAkfK09B,MAAO,OACP4R,SAAW,UAEX9c,OAAS,YACTlvB,SAtfLtD,OAwfK0xC,WAAWt9B,QAAQu9B,IAEnBa,MACE,IAAIxuC,KAAQzD,KAEQ,MAAnByD,EAAK+iB,OAAO,IACZunB,EAAO1xC,KAAK2D,KAAMyD,KACjBrE,OAAOqE,EAAKP,MAAM,WAChBO,QAhgBXhE,IAsgBFksC,KAAM,gBACCxO,MAAO,MAGR+U,EADYlyC,KAAKmxC,WAAW,GACLE,cACH,UAApBa,EAAWhb,WACPgb,EAAWnvC,WAGZ/C,KAAKmyC,MAGd9C,kBAAmB,SAAS+C,MACtBpyC,KAAKm9B,WACDiV,MAGJzD,EAAU3uC,cACLqyC,EAAOC,EAAKC,UACnBhD,EAAOrY,KAAO,QACdqY,EAAOxsC,IAAMqvC,EACbzD,EAAQ36B,KAAOs+B,EAEXC,IAGF5D,EAAQ1c,OAAS,OACjB0c,EAAQ5rC,SAjiBZtD,KAoiBY8yC,MAGP,IAAIz1C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,GACxByyC,EAASvC,EAAMqE,cAEE,SAAjBrE,EAAM+D,cAIDsB,EAAO,UAGZrF,EAAM+D,QAAU/wC,KAAK0pC,KAAM,KACzB8I,EAAWzE,EAAO1xC,KAAK2wC,EAAO,YAC9ByF,EAAa1E,EAAO1xC,KAAK2wC,EAAO,iBAEhCwF,GAAYC,EAAY,IACtBzyC,KAAK0pC,KAAOsD,EAAMgE,gBACbqB,EAAOrF,EAAMgE,UAAU,GACzB,GAAIhxC,KAAK0pC,KAAOsD,EAAMiE,kBACpBoB,EAAOrF,EAAMiE,iBAGjB,GAAIuB,MACLxyC,KAAK0pC,KAAOsD,EAAMgE,gBACbqB,EAAOrF,EAAMgE,UAAU,OAG3B,CAAA,IAAIyB,QAMH,IAAIrvC,MAAM,6CALZpD,KAAK0pC,KAAOsD,EAAMiE,kBACboB,EAAOrF,EAAMiE,gBAU9B3B,OAAQ,SAASpY,EAAMn0B,OAChB,IAAIjG,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAM+D,QAAU/wC,KAAK0pC,MACrBqE,EAAO1xC,KAAK2wC,EAAO,eACnBhtC,KAAK0pC,KAAOsD,EAAMiE,WAAY,KAC5ByB,EAAe1F,SAKnB0F,IACU,UAATxb,GACS,aAATA,IACDwb,EAAa3B,QAAUhuC,GACvBA,GAAO2vC,EAAazB,aAGtByB,EAAe,UAGbnD,EAASmD,EAAeA,EAAarB,WAAa,UACtD9B,EAAOrY,KAAOA,EACdqY,EAAOxsC,IAAMA,EAET2vC,QACGzgB,OAAS,YACTje,KAAO0+B,EAAazB,WAClB/B,GAGFlvC,KAAK2yC,SAASpD,IAGvBoD,SAAU,SAASpD,EAAQ2B,MACL,UAAhB3B,EAAOrY,WACHqY,EAAOxsC,UAGK,UAAhBwsC,EAAOrY,MACS,aAAhBqY,EAAOrY,UACJljB,KAAOu7B,EAAOxsC,IACM,WAAhBwsC,EAAOrY,WACXib,KAAOnyC,KAAK+C,IAAMwsC,EAAOxsC,SACzBkvB,OAAS,cACTje,KAAO,OACa,WAAhBu7B,EAAOrY,MAAqBga,SAChCl9B,KAAOk9B,GAGPhC,GAGT0D,OAAQ,SAAS3B,OACV,IAAIn0C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAMiE,aAAeA,cAClB0B,SAAS3F,EAAMqE,WAAYrE,EAAMkE,UACtCE,EAAcpE,GACPkC,UAKJ,SAAS6B,OACX,IAAIj0C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAM+D,SAAWA,EAAQ,KACvBxB,EAASvC,EAAMqE,cACC,UAAhB9B,EAAOrY,KAAkB,KACvB2b,EAAStD,EAAOxsC,IACpBquC,EAAcpE,UAET6F,SAML,IAAIzvC,MAAM,0BAGlB0vC,cAAe,SAASjP,EAAU8M,EAAYC,eACvC7B,SAAW,CACdhc,SAAUzf,EAAOuwB,GACjB8M,WAAYA,EACZC,QAASA,GAGS,SAAhB5wC,KAAKiyB,cAGFlvB,SA1qBPtD,GA6qBOyvC,IAQJj8B,EA1rBM,CAisBgBxX,EAAOwX,aAIpC8/B,mBAAqBlF,EACrB,MAAOmF,GAUPrvC,SAAS,IAAK,yBAAdA,CAAwCkqC,UCrtBzBjd,GCSbuR,GAAsBC,GAA6B,UACnDb,GAAiBC,GAAwB,SAAU,CAAEN,WAAW,IAAS,IAAM,IAE/E57B,GAAM9D,KAAK8D,IACXlD,GAAMZ,KAAKY,MAOb,CAAEutB,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnF0R,OAAQ,SAAgBzS,EAAO0S,OAKzBC,EAAaC,EAAmBnQ,EAAGtoB,EAAGta,EAAMD,EAJ5CqtB,EAAI9K,GAAS3iB,MACbC,EAAMoxB,EAAS5D,EAAEzwB,QACjBq2C,EAAc/hB,GAAgBkP,EAAOvgC,GACrC+jC,EAAkBloC,UAAUkB,UAER,IAApBgnC,EACFmP,EAAcC,EAAoB,EACL,IAApBpP,GACTmP,EAAc,EACdC,EAAoBnzC,EAAMozC,IAE1BF,EAAcnP,EAAkB,EAChCoP,EAAoBhxC,GAAIkD,GAAIwrB,EAAUoiB,GAAc,GAAIjzC,EAAMozC,IAE5DpzC,EAAMkzC,EAAcC,EAtBL,uBAuBXpmB,UAtB0B,uCAwBlCiW,EAAIjL,GAAmBvK,EAAG2lB,GACrBz4B,EAAI,EAAGA,EAAIy4B,EAAmBz4B,KACjCta,EAAOgzC,EAAc14B,KACT8S,GAAGiF,GAAeuQ,EAAGtoB,EAAG8S,EAAEptB,OAExC4iC,EAAEjmC,OAASo2C,EACPD,EAAcC,EAAmB,KAC9Bz4B,EAAI04B,EAAa14B,EAAI1a,EAAMmzC,EAAmBz4B,IAEjDva,EAAKua,EAAIw4B,GADT9yC,EAAOsa,EAAIy4B,KAEC3lB,EAAGA,EAAErtB,GAAMqtB,EAAEptB,UACbotB,EAAErtB,OAEXua,EAAI1a,EAAK0a,EAAI1a,EAAMmzC,EAAoBD,EAAax4B,WAAY8S,EAAE9S,EAAI,QACtE,GAAIw4B,EAAcC,MAClBz4B,EAAI1a,EAAMmzC,EAAmBz4B,EAAI04B,EAAa14B,IAEjDva,EAAKua,EAAIw4B,EAAc,GADvB9yC,EAAOsa,EAAIy4B,EAAoB,KAEnB3lB,EAAGA,EAAErtB,GAAMqtB,EAAEptB,UACbotB,EAAErtB,OAGbua,EAAI,EAAGA,EAAIw4B,EAAax4B,IAC3B8S,EAAE9S,EAAI04B,GAAev3C,UAAU6e,EAAI,UAErC8S,EAAEzwB,OAASiD,EAAMmzC,EAAoBD,EAC9BlQ,KChEX,OAAiBpC,GAAa,SAASoS,OCDnCxR,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmnB,cACNnnB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAewR,OAAUA,GAASlS,GCFhG79B,GAAQ,GAAGA,MACXowC,GAAO,WAAWvsC,KAAK+6B,IAEvBxR,GAAO,SAAUijB,UACZ,SAAUC,EAASC,OACpBC,EAAY53C,UAAUkB,OAAS,EAC/BgG,EAAO0wC,EAAYxwC,GAAM7G,KAAKP,UAAW,QAAK2D,SAC3C8zC,EAAUG,EAAY,YAER,mBAAXF,EAAwBA,EAAU7vC,SAAS6vC,IAAU33C,MAAMmE,KAAMgD,IACvEwwC,EAASC,OAMf,CAAEvnB,QAAQ,EAAMmE,MAAM,EAAMF,OAAQmjB,IAAQ,CAG5CK,WAAYrjB,GAAKpE,EAAOynB,YAGxBC,YAAatjB,GAAKpE,EAAO0nB,eCvB3B,OAAiB3jB,EAAK0jB,WCIlBE,GAAmB,SAAUlkB,EAAQmkB,EAAUjuB,EAAQkuB,EAAWvT,EAAOwT,EAAOC,EAAQC,WAItFtM,EAHAuM,EAAc3T,EACd4T,EAAc,EACdC,IAAQJ,GAAS5jB,EAAK4jB,EAAQC,EAAS,GAGpCE,EAAcL,GAAW,IAC1BK,KAAevuB,EAAQ,IACzB+hB,EAAUyM,EAAQA,EAAMxuB,EAAOuuB,GAAcA,EAAaN,GAAYjuB,EAAOuuB,GAEzEJ,EAAQ,GAAKj4C,GAAQ6rC,GACvBuM,EAAcN,GAAiBlkB,EAAQmkB,EAAUlM,EAASvW,EAASuW,EAAQ5qC,QAASm3C,EAAaH,EAAQ,GAAK,MACzG,IACDG,GAAe,iBAAkB,MAAMnnB,UAAU,sCACrD2C,EAAOwkB,GAAevM,EAGxBuM,IAEFC,WAEKD,MAGQN,KCrBf,CAAElkB,OAAQ,QAASnP,OAAO,GAAQ,CAClC8zB,QAAS,SAAiBzc,OAGpBoL,EAFAxV,EAAI9K,GAAS3iB,MACb+zC,EAAY1iB,EAAS5D,EAAEzwB,eAE3B0xB,EAAUmJ,IACVoL,EAAIjL,GAAmBvK,EAAG,IACxBzwB,OAAS62C,GAAiB5Q,EAAGxV,EAAGA,EAAGsmB,EAAW,EAAG,EAAGlc,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,GACjGwjC,KCdX,OAAiBpC,GAAa,SAASyT,QCFnC7S,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGwoB,eACNxoB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe6S,QAAWA,GAAUvT,GCAlG/P,GAAe,SAAUujB,UACpB,SAAUvzB,EAAM6W,EAAYmM,EAAiBwQ,GAClD9lB,EAAUmJ,OACNpK,EAAI9K,GAAS3B,GACbiL,EAAOgB,EAAcQ,GACrBzwB,EAASq0B,EAAS5D,EAAEzwB,QACpB4nB,EAAQ2vB,EAAWv3C,EAAS,EAAI,EAChCF,EAAIy3C,GAAY,EAAI,KACpBvQ,EAAkB,EAAG,OAAa,IAChCpf,KAASqH,EAAM,CACjBuoB,EAAOvoB,EAAKrH,GACZA,GAAS9nB,WAGX8nB,GAAS9nB,EACLy3C,EAAW3vB,EAAQ,EAAI5nB,GAAU4nB,QAC7BoI,UAAU,oDAGdunB,EAAW3vB,GAAS,EAAI5nB,EAAS4nB,EAAOA,GAAS9nB,EAAO8nB,KAASqH,IACrEuoB,EAAO3c,EAAW2c,EAAMvoB,EAAKrH,GAAQA,EAAO6I,WAEvC+mB,IC1BPC,GD8Ba,CAGfC,KAAM1jB,IAAa,GAGnB2jB,MAAO3jB,IAAa,ICpC6B0jB,KAI/CrT,GAAgBC,GAAoB,UACpCC,GAAiBC,GAAwB,SAAU,GAAK,MAI1D,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7EqT,OAAQ,SAAgB/c,UACf4c,GAAQz0C,KAAM63B,EAAY/7B,UAAUkB,OAAQlB,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCV7F,OAAiBohC,GAAa,SAAS+T,OCDnCnT,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG8oB,cACN9oB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAemT,OAAUA,GAAS7T,GCFhG79B,GAAQ,GAAGA,MACX2xC,GAAY,GAEZC,GAAY,SAAUzd,EAAG0d,EAAY/xC,QACjC+xC,KAAcF,IAAY,KACzB,IAAIG,EAAO,GAAIl4C,EAAI,EAAGA,EAAIi4C,EAAYj4C,IAAKk4C,EAAKl4C,GAAK,KAAOA,EAAI,IAErE+3C,GAAUE,GAAcpxC,SAAS,MAAO,gBAAkBqxC,EAAK7xC,KAAK,KAAO,YACpE0xC,GAAUE,GAAY1d,EAAGr0B,OAKnBW,SAAS0sB,MAAQ,SAAcrP,OAC1CnkB,EAAK6xB,EAAU1uB,MACfi1C,EAAW/xC,GAAM7G,KAAKP,UAAW,GACjCi8B,EAAgB,eACd/0B,EAAOiyC,EAAS7mC,OAAOlL,GAAM7G,KAAKP,mBAC/BkE,gBAAgB+3B,EAAgB+c,GAAUj4C,EAAImG,EAAKhG,OAAQgG,GAAQnG,EAAGhB,MAAMmlB,EAAMhe,WAEvF1G,EAASO,EAAGV,aAAY47B,EAAc57B,UAAYU,EAAGV,WAClD47B,KCpBP,CAAEpI,OAAQ,WAAYnP,OAAO,GAAQ,CACrC6P,KAAMA,KCHR,OAAiBwQ,GAAa,YAAYxQ,KCDtC6kB,GAAoBvxC,SAASxH,aAEhB,SAAU2vB,OACrBiV,EAAMjV,EAAGuE,YACNvE,IAAOopB,IAAsBppB,aAAcnoB,UAAYo9B,IAAQmU,GAAkB7kB,KAAQA,GAAO0Q,GCAzG,OANA,SAAyBoU,EAAUvQ,QAC3BuQ,aAAoBvQ,SAClB,IAAI5X,UAAU,sCCAxB,SAASooB,GAAkBzlB,EAAQ0lB,OAC5B,IAAIv4C,EAAI,EAAGA,EAAIu4C,EAAMr4C,OAAQF,IAAK,KACjC4vB,EAAa2oB,EAAMv4C,GACvB4vB,EAAWC,WAAaD,EAAWC,aAAc,EACjDD,EAAWG,cAAe,EACtB,UAAWH,IAAYA,EAAWI,UAAW,GAEjDmT,GAAuBtQ,EAAQjD,EAAWzpB,IAAKypB,IAUnD,OANA,SAAsBkY,EAAa0Q,EAAYC,UACzCD,GAAYF,GAAkBxQ,EAAYzoC,UAAWm5C,GACrDC,GAAaH,GAAkBxQ,EAAa2Q,GACzC3Q,qJCwDO4Q,GAGdn1C,UACO,IAAIo1C,GAA0Bp1C,OAcjCq1C,yBAwBeC,EACAC,EACAC,6BAFAF,QAAAA,OACAC,cAAAA,OACAC,QAAAA,OAjBFC,WAAqC,CACpDj7B,IAAKk7B,UAAKC,aAAUh2C,MACpBi2C,OAAQF,UAAKG,gBAAal2C,MAC1Bm2C,OAAQJ,UAAKK,gBAAap2C,4DAmBrB61C,QAAQM,OAAOn2C,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,QAC/C/J,iDAKF21C,QAAQW,GAAG,MAAOt2C,KAAK81C,WAAWj7B,UAClC86B,QAAQW,GAAG,SAAUt2C,KAAK81C,WAAWG,aACrCN,QAAQW,GAAG,SAAUt2C,KAAK81C,WAAWK,QAEnCn2C,gDAKF21C,QAAQY,IAAI,MAAOv2C,KAAK81C,WAAWj7B,UACnC86B,QAAQY,IAAI,SAAUv2C,KAAK81C,WAAWG,aACtCN,QAAQY,IAAI,SAAUv2C,KAAK81C,WAAWK,QAEpCn2C,6CAUew2C,gBACfC,UAAKb,uBAAqB,SAACY,EAAOE,UAChCA,EAAUF,KAChBA,gCAUHG,EACAC,GAEe,MAAXA,QAICf,QAAQh7B,IAAI7a,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,IAAI6sC,EAAQJ,yCAU/DG,EACAC,GAEe,MAAXA,QAICf,QAAQM,OAAOn2C,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,IAAI6sC,EAAQJ,yCAUlEG,EACAC,GAEe,MAAXA,QAICf,QAAQI,OAAOj2C,KAAKq2C,gBAAgBO,EAAQC,mBAW/CpB,yBAgBgCE,mBAAAA,QAAAA,OARnBC,cAAoC,6CAmBnD5vC,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqB86C,GAAA96C,QAAAA,EAAagK,MACpDhG,iCAePgG,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqB+6C,GAAA/6C,QAAAA,EAAUgK,MAChDhG,qCAeRgG,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqBg7C,GAAAh7C,QAAAA,EAAcgK,MACpDhG,gCAWA2vB,UACD,IAAI+lB,GAAe11C,KAAK21C,QAAS31C,KAAK41C,cAAejmB,8vsBCjS5B,oBAAX3D,QAA2BA,OAAM,QAAcxB,2ICQxE,IAAMysB,GAAe,qBAWd,SAASC,GAAQpoB,EAAQoI,OAC1B5wB,UAEW7G,IAAXqvB,MAGW,OAAXA,SACK,SAGJoI,SACIpI,KAEa,iBAAToI,KAAwBA,aAAgBzI,cAC7C,IAAIrrB,MAAM,gCAIV8zB,OACD,cACA,iBACI+K,QAAQnT,OAEZ,aACA,gBACCqoB,GAASroB,KAAY1vB,MAAM1C,KAAK06C,MAAMtoB,IACjC1E,EAAO0E,GAAQvxB,UAKf85C,OAAOvoB,EAAOvxB,eAEpB,aACA,gBACIkxB,OAAOK,OAEX,UACCtyB,GAASsyB,UACJ,IAAIpyB,KAAKoyB,MAEdA,aAAkBpyB,YACb,IAAIA,KAAKoyB,EAAOvxB,WAClB,GAAI6sB,EAAOhpB,SAAS0tB,UAClB,IAAIpyB,KAAKoyB,EAAOvxB,cAErB45C,GAASroB,UACXxoB,EAAQ2wC,GAAa//B,KAAK4X,IAGjB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAEtB8jB,EAAO,IAAI1tB,KAAKoyB,IAASlM,eAG5B,IAAIxf,MACR,iCAAmCk0C,GAAQxoB,GAAU,qBAItD,YACCtyB,GAASsyB,UACJ1E,EAAO0E,MAEZA,aAAkBpyB,YACb0tB,EAAO0E,EAAOvxB,WAChB,GAAI6sB,EAAOhpB,SAAS0tB,UAClB1E,EAAO0E,MAEZqoB,GAASroB,UACXxoB,EAAQ2wC,GAAa//B,KAAK4X,GAGjB1E,EAFL9jB,EAEY+wC,OAAO/wC,EAAM,IAEbwoB,SAGV,IAAI1rB,MACR,iCAAmCk0C,GAAQxoB,GAAU,qBAItD,aACCtyB,GAASsyB,UACJ,IAAIpyB,KAAKoyB,GACX,GAAIA,aAAkBpyB,YACpBoyB,EAAOjM,cACT,GAAIuH,EAAOhpB,SAAS0tB,UAClBA,EAAOlM,SAASC,cAClB,GAAIs0B,GAASroB,UAClBxoB,EAAQ2wC,GAAa//B,KAAK4X,IAGjB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAAKuc,cAE3BuH,EAAO0E,GAAQrxB,eAGlB,IAAI2F,MACR,iCACEk0C,GAAQxoB,GACR,wBAIH,aACCtyB,GAASsyB,SACJ,SAAWA,EAAS,KACtB,GAAIA,aAAkBpyB,MAAQ0tB,EAAOhpB,SAAS0tB,SAC5C,SAAWA,EAAOvxB,UAAY,KAChC,GAAI45C,GAASroB,SASX,WARPxoB,EAAQ2wC,GAAa//B,KAAK4X,IAIhB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAAK/I,UAE3B,IAAIb,KAAKoyB,GAAQvxB,WAED,WAEpB,IAAI6F,MACR,iCACEk0C,GAAQxoB,GACR,kCAKA,IAAI1rB,6BAAsB8zB,MA8B/B,SAASqgB,GACdC,mBACAtgB,yDAAO,CAAEsJ,MAAO,OAAQC,IAAK,QAEvBgX,EAASD,EAAME,QACfC,EAAY,IAAIC,GAAQ,CAAEC,QAASJ,IAEnCK,EAAOf,KAAAvB,GAAsBgC,YAC5B,SAAAO,gBACHtB,QAAYsB,YAAa,SAACC,EAAK/0C,UAC7B+0C,EAAI/0C,GAAOi0C,GAAQa,EAAK90C,GAAMi0B,EAAKj0B,IAC5B+0C,IACN,OAEJ53C,GAAGu3C,UAENG,EAAKG,MAAMzX,QAEJ,CAEL3lB,IAAK,0BAAa28B,EAAMU,cAAar9B,wBACrCo7B,OAAQ,0BAAauB,EAAMU,cAAajC,2BACxCE,OAAQ,0BAAaqB,EAAMU,cAAa/B,2BACxCgC,WAAY,0BAAaX,EAAMU,cAAaC,+BAG5CtkC,QAASkiC,QAAA4B,WAAuBA,GAChC5tC,IAAKgsC,KAAA4B,EAAU5tC,YAAS4tC,GACxBS,OAAQrC,KAAA4B,EAAUS,eAAYT,GAC9BpB,IAAKR,KAAA4B,EAAUpB,YAASoB,GACxBrB,GAAIP,KAAA4B,EAAUrB,WAAQqB,uBAGbA,EAAU36C,QAInBy6C,OAAAA,EACAvgB,KAAAA,EAEAsgB,MAAAA,EACAG,UAAAA,EACAU,QAAS,kBAAMP,EAAKnM,uRAKnB2M,IACHpB,QAAAA,KCtOExW,GAAOlW,GAAoCkW,KAG3C6X,GAAcrsB,EAAOpQ,cACZ,EAAIy8B,GAAYlY,GAAc,QAAWmY,EAAAA,EAI5B,SAAoBvhC,OACxCwhC,EAAgB/X,GAAKjS,OAAOxX,IAC5BsB,EAASggC,GAAYE,UACP,IAAXlgC,GAA2C,KAA3BkgC,EAAcjyB,OAAO,IAAa,EAAIjO,GAC3DggC,KCRF,CAAErsB,QAAQ,EAAMiE,OAAQrU,YAAc48B,IAA4B,CAClE58B,WAAY48B,KCHd,OAAiBzoB,EAAKnU,WCAD68B,yBAKPC,EAAMzpB,mBACXA,QAAU,UACVkmB,MAAQ,mDAQJlmB,GACLA,GACFmpB,GAAKh7C,OAAO0C,KAAKmvB,QAASA,2CAUrB,8EAiBD0pB,EACJ74C,KAAKq1C,MAAMyD,iBAAmB94C,KAAKq1C,MAAMzM,OACzC5oC,KAAKq1C,MAAM0D,kBAAoB/4C,KAAKq1C,MAAMxM,mBAGvCwM,MAAMyD,eAAiB94C,KAAKq1C,MAAMzM,WAClCyM,MAAM0D,gBAAkB/4C,KAAKq1C,MAAMxM,OAEjCgQ,aClDT,CAAElpB,OAAQ,SAAUnP,OAAO,GAAQ,CACnCw4B,OCAe,GAAGA,QAAU,SAAgBC,OACxCptB,EAAM4C,OAAOvB,EAAuBltB,OACpCuY,EAAS,GACTpK,EAAI2iB,EAAUmoB,MACd9qC,EAAI,GAAKA,GAAKqqC,EAAAA,EAAU,MAAMU,WAAW,oCACvC/qC,EAAI,GAAIA,KAAO,KAAO0d,GAAOA,GAAc,EAAJ1d,IAAOoK,GAAUsT,UACvDtT,KCTT,OAAiBsoB,GAAa,UAAUmY,OCDpClY,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAGktB,aACQ,iBAAPltB,GAAmBA,IAAOgV,IAClChV,aAAc2C,QAAUsS,IAAQD,GAAgBkY,OAAUA,GAASjY,GCEpE,SAASoY,GAAqB/uB,EAAQwuB,EAAMQ,MAC7CA,IAAgBlW,GAAckW,UACzBD,GAAqB/uB,EAAQwuB,EAAM,CAACQ,OAG7CR,EAAKQ,YAAc,GACfA,GACgC,GAA9BlW,GAAckW,GAAsB,WAC7Bt8C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,YACR2C,IAA1B45C,GAAAD,EAAYt8C,IAAyB,KACjCw8C,EAAW,GACjBA,EAAS9Y,MAAQpW,EAAOgvB,EAAYt8C,GAAG0jC,OAAO5d,SAASrlB,UACvD+7C,EAAS7Y,IAAMrW,EAAOgvB,EAAYt8C,GAAG2jC,KAAK7d,SAASrlB,UACnDq7C,EAAKQ,YAAYn8C,KAAKq8C,QAG1BV,EAAKQ,qBAAiB,SAACj8C,EAAGC,UAAMD,EAAEqjC,MAAQpjC,EAAEojC,UAa3C,SAAS+Y,GAAkBnvB,EAAQwuB,EAAMQ,MAC1CA,IAAgBlW,GAAckW,UACzBG,GAAkBnvB,EAAQwuB,EAAM,CAACQ,OAGtCA,QAAuD35C,IAAxCm5C,EAAKY,SAASC,gBAAgB7Q,MAAqB,CACpEuQ,GAAqB/uB,EAAQwuB,EAAMQ,WAE7B5Y,EAAQpW,EAAOwuB,EAAKc,MAAMlZ,OAC1BC,EAAMrW,EAAOwuB,EAAKc,MAAMjZ,KAGxBkZ,GADcf,EAAKc,MAAMjZ,IAAMmY,EAAKc,MAAMlZ,OACjBoY,EAAKY,SAASC,gBAAgB7Q,MAEpD9rC,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,YACR2C,IAA1B45C,GAAAD,EAAYt8C,IAAyB,KACjC88C,EAAYxvB,EAAOgvB,EAAYt8C,GAAG0jC,OACpCqZ,EAAUzvB,EAAOgvB,EAAYt8C,GAAG2jC,QAEhB,gBAAhBmZ,EAAUv6C,SACN,IAAI+D,kDAA2Cg2C,EAAYt8C,GAAG0jC,WAEpD,gBAAdqZ,EAAQx6C,SACJ,IAAI+D,gDAAyCg2C,EAAYt8C,GAAG2jC,SAGnDoZ,EAAUD,GACX,EAAID,EAAW,KAEzBn9B,EAAS,EACPs9B,EAAWrZ,EAAIrjB,kBACbg8B,EAAYt8C,SACb,QACC88C,EAAUxqC,OAASyqC,EAAQzqC,QAC7BoN,EAAS,GAEXo9B,EAAUjsC,UAAU6yB,EAAM7yB,aAC1BisC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,QAErBi7B,EAAQlsC,UAAU6yB,EAAM7yB,aACxBksC,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAIpC,EAAO,QAE5Bs9B,EAASj/B,IAAI,EAAG,mBAEb,aACGk/B,EAAYF,EAAQ18B,KAAKy8B,EAAU,QACnCxqC,EAAMwqC,EAAUxqC,MAGtBwqC,EAAU3vC,KAAKu2B,EAAMv2B,QACrB2vC,EAAU5vC,MAAMw2B,EAAMx2B,SACtB4vC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBwwC,EAAUD,EAAUx8B,QAGpBw8B,EAAUxqC,IAAIA,GACdyqC,EAAQzqC,IAAIA,GACZyqC,EAAQh/B,IAAIk/B,EAAU,QAEtBH,EAAUh7B,SAAS,EAAE,SACrBi7B,EAAQj7B,SAAS,EAAE,SAEnBk7B,EAASj/B,IAAI,EAAG,mBAGb,UACC++B,EAAU5vC,SAAW6vC,EAAQ7vC,UAC/BwS,EAAS,GAEXo9B,EAAU5vC,MAAMw2B,EAAMx2B,SACtB4vC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,UAErBi7B,EAAQ7vC,MAAMw2B,EAAMx2B,SACpB6vC,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAE,UACnBi7B,EAAQh/B,IAAI2B,EAAO,UAEnBs9B,EAASj/B,IAAI,EAAG,oBAEb,SACC++B,EAAUvwC,QAAUwwC,EAAQxwC,SAC9BmT,EAAS,GAEXo9B,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,SACrBi7B,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAE,SACnBi7B,EAAQh/B,IAAI2B,EAAO,SAEnBs9B,EAASj/B,IAAI,EAAG,mCAGhBlY,QAAQq3C,IAAI,8EAA4EZ,EAAYt8C,UAGjG88C,EAAYE,UACjBlB,EAAKQ,YAAYn8C,KAAK,CAACujC,MAAOoZ,EAAUr8C,UAAWkjC,IAAKoZ,EAAQt8C,eACxD67C,EAAYt8C,SACb,QACH88C,EAAU/+B,IAAI,EAAG,QACjBg/B,EAAQh/B,IAAI,EAAG,kBAEZ,SACH++B,EAAU/+B,IAAI,EAAG,SACjBg/B,EAAQh/B,IAAI,EAAG,mBAEZ,UACH++B,EAAU/+B,IAAI,EAAG,UACjBg/B,EAAQh/B,IAAI,EAAG,oBAEZ,SACH++B,EAAU/+B,IAAI,EAAG,KACjBg/B,EAAQh/B,IAAI,EAAG,+BAGflY,QAAQq3C,IAAI,8EAA4EZ,EAAYt8C,KAI1G87C,EAAKQ,YAAYn8C,KAAK,CAACujC,MAAOoZ,EAAUr8C,UAAWkjC,IAAKoZ,EAAQt8C,aAKtE08C,GAAiBrB,OAEXsB,EAAcC,GAAYvB,EAAKc,MAAMlZ,MAAOoY,EAAKQ,aACjDgB,EAAYD,GAAYvB,EAAKc,MAAMjZ,IAAImY,EAAKQ,aAC9CiB,EAAazB,EAAKc,MAAMlZ,MACxB8Z,EAAW1B,EAAKc,MAAMjZ,IACA,GAAtByZ,EAAYK,SAAiBF,EAAwC,GAA3BzB,EAAKc,MAAMc,aAAuBN,EAAYN,UAAY,EAAIM,EAAYL,QAAU,GAC1G,GAApBO,EAAUG,SAAmBD,EAAsC,GAAzB1B,EAAKc,MAAMe,WAAuBL,EAAUR,UAAY,EAAMQ,EAAUP,QAAU,GACtG,GAAtBK,EAAYK,QAAsC,GAApBH,EAAUG,QAC1C3B,EAAKc,MAAMgB,YAAYL,EAAYC,IAYlC,SAASL,GAAiBrB,aACzBQ,EAAcR,EAAKQ,YACnBuB,EAAY,GACT79C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,QACjC,IAAIiX,EAAI,EAAGA,EAAIqlC,EAAYp8C,OAAQ+W,IAClCjX,GAAKiX,GAA8B,GAAzBqlC,EAAYrlC,GAAGkiC,QAA2C,GAAzBmD,EAAYt8C,GAAGm5C,SAExDmD,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG2jC,IACvF2Y,EAAYrlC,GAAGkiC,QAAS,EAGjBmD,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG2jC,KAC9F2Y,EAAYt8C,GAAG2jC,IAAM2Y,EAAYrlC,GAAG0sB,IACpC2Y,EAAYrlC,GAAGkiC,QAAS,GAGjBmD,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG2jC,MAC1F2Y,EAAYt8C,GAAG0jC,MAAQ4Y,EAAYrlC,GAAGysB,MACtC4Y,EAAYrlC,GAAGkiC,QAAS,QAM3Bn5C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,KACJ,IAA1Bs8C,EAAYt8C,GAAGm5C,QACjB0E,EAAU19C,KAAKm8C,EAAYt8C,IAI/B87C,EAAKQ,YAAcuB,OACnB/B,EAAKQ,qBAAiB,SAACj8C,EAAGC,UAAMD,EAAEqjC,MAAQpjC,EAAEojC,SAmBvC,SAASoa,GAAoBxwB,EAAQywB,EAAUC,WAChDC,GAAe,EACbC,EAAeH,EAASI,QAAQ19C,UAC7BT,EAAI,EAAGA,EAAI+9C,EAASzB,YAAYp8C,OAAQF,IAAK,KAC9C88C,EAAYiB,EAASzB,YAAYt8C,GAAG0jC,MACtCqZ,EAAUgB,EAASzB,YAAYt8C,GAAG2jC,OAClCua,GAAgBpB,GAAaoB,EAAenB,EAAS,CACvDkB,GAAe,YAKC,GAAhBA,GAAwBC,EAAeH,EAASK,KAAK39C,WAAay9C,GAAgBF,EAAc,KAC5FK,EAAY/wB,EAAO0wB,GACnBM,EAAWhxB,EAAOyvB,GAEpBsB,EAAU9xC,QAAU+xC,EAAS/xC,OAASwxC,EAASQ,cAAe,EACzDF,EAAUnxC,SAAWoxC,EAASpxC,QAAU6wC,EAASS,eAAgB,EACjEH,EAAUxtC,aAAeytC,EAASztC,cAAcktC,EAASU,aAAc,GAEhFV,EAASI,QAAUG,GAmChB,SAASI,GAASC,EAAM/6B,EAAMkoB,OAC/B8S,KACgC,GAAhCD,EAAK7C,KAAKQ,YAAYp8C,cACtB0+C,EAAaD,EAAK/B,MAAMgC,WAAW9S,IAC3BloB,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,UAEnDpB,EAASJ,GAAYz5B,EAAM+6B,EAAK7C,KAAKQ,aACtB,GAAjBmB,EAAOA,SACT75B,EAAO65B,EAAOX,eAGVv+B,EAAWugC,GAAyBH,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAMlZ,MAAOib,EAAK/B,MAAMjZ,QAC1F/f,EAAO+6B,EAAK/B,MAAMlZ,MAAO,CAC3Bkb,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,OACpCwgC,EAAoBC,GAA6BL,EAAK7C,KAAKQ,YAAa14B,EAAMg7B,EAAWl/B,eAC/FkE,EAAO+6B,EAAKtsB,QAAQ/E,OAAO1J,GAAMkC,SAASrlB,UAC1CmjB,GAAcm7B,IACLH,EAAWl/B,OAASkE,EAAKnjB,WAAam+C,EAAWC,MAErD,GAAIj7B,EAAO+6B,EAAK/B,MAAMjZ,IAAK,KAC1Bsb,EAAgB,CAACvb,MAAOib,EAAK/B,MAAMlZ,MAAOC,IAAK/f,UACrDA,EAAOs7B,GAAqBP,EAAKtsB,QAAQ/E,OAAQqxB,EAAK7C,KAAKQ,YAAa2C,EAAer7B,GACvFg7B,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,IAClCqF,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,aAGzDj7B,EAAOs7B,GAAqBP,EAAKtsB,QAAQ/E,OAAQqxB,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAOh5B,GACpFg7B,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,IAClCqF,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,MAa1D,SAASM,GAAOR,EAAMtxC,EAAGy+B,MACM,GAAhC6S,EAAK7C,KAAKQ,YAAYp8C,OAAa,KAC/B0+C,EAAaD,EAAK/B,MAAMgC,WAAW9S,UAClC,IAAIlsC,KAAKyN,EAAIuxC,EAAWC,MAAQD,EAAWl/B,YAG5C0/B,EAAiBN,GAAyBH,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAMlZ,MAAOib,EAAK/B,MAAMjZ,KAE9F0b,GADgBV,EAAK/B,MAAMjZ,IAAMgb,EAAK/B,MAAMlZ,MAAQ0b,GAClB/xC,EAAIy+B,EACtCwT,EAA4BC,GAA6BZ,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAOyC,UAE3F,IAAIz/C,KAAK0/C,EAA4BD,EAAkBV,EAAK/B,MAAMlZ,OAYtE,SAASob,GAAyBxC,EAAa5Y,EAAOC,WACvDplB,EAAW,EACNve,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAapZ,GAASqZ,EAAUpZ,IAClCplB,GAAYw+B,EAAUD,UAGnBv+B,EAWF,SAASygC,GAA6B1C,EAAa5Y,EAAOC,WAC3DplB,EAAW,EACNve,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAapZ,GAASqZ,GAAWpZ,IACnCplB,GAAYw+B,EAAUD,UAGnBv+B,EAWF,SAAS2gC,GAAqB5xB,EAAQgvB,EAAaM,EAAOh5B,UAC/DA,EAAO0J,EAAO1J,GAAMkC,SAASrlB,UAC7BmjB,GAAQ47B,GAAwBlyB,EAAQgvB,EAAYM,EAAMh5B,GAYrD,SAAS47B,GAAwBlyB,EAAQgvB,EAAaM,EAAOh5B,OAC9D67B,EAAa,EACjB77B,EAAO0J,EAAO1J,GAAMkC,SAASrlB,cAExB,IAAIT,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAaF,EAAMlZ,OAASqZ,EAAUH,EAAMjZ,KAC1C/f,GAAQm5B,IACV0C,GAAe1C,EAAUD,UAIxB2C,EAWF,SAASF,GAA6BjD,EAAaM,EAAO8C,WAC3DN,EAAiB,EACjB7gC,EAAW,EACXohC,EAAgB/C,EAAMlZ,MAEjB1jC,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,OAE3BmZ,GAAaF,EAAMlZ,OAASqZ,EAAUH,EAAMjZ,IAAK,IACnDplB,GAAYu+B,EAAY6C,EACxBA,EAAgB5C,EACZx+B,GAAYmhC,QAIdN,GAAkBrC,EAAUD,UAK3BsC,EAWF,SAASQ,GAAmBtD,EAAa14B,EAAMpC,EAAWq+B,OACzDC,EAAWzC,GAAYz5B,EAAM04B,UACZ,GAAnBwD,EAASrC,OACPj8B,EAAY,EACW,GAArBq+B,EACKC,EAAShD,WAAagD,EAAS/C,QAAUn5B,GAAQ,EAGjDk8B,EAAShD,UAAY,EAIL,GAArB+C,EACKC,EAAS/C,SAAWn5B,EAAOk8B,EAAShD,WAAa,EAGjDgD,EAAS/C,QAAU,EAKvBn5B,EAYJ,SAASy5B,GAAYz5B,EAAM04B,OAC3B,IAAIt8C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACvC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,OAEzB/f,GAAQk5B,GAAal5B,EAAOm5B,QACvB,CAACU,QAAQ,EAAMX,UAAAA,EAAWC,QAAAA,SAG9B,CAACU,QAAQ,EAAOX,UAAAA,EAAWC,QAAAA,oHA9R7B,SAAoBt1B,OACpB,IAAIznB,EAAG,EAAGA,EAAIynB,EAAMvnB,OAAQF,IAC/B6F,QAAQq3C,IAAIl9C,EAAG,IAAIJ,KAAK6nB,EAAMznB,GAAG0jC,OAAO,IAAI9jC,KAAK6nB,EAAMznB,GAAG2jC,KAAMlc,EAAMznB,GAAG0jC,MAAOjc,EAAMznB,GAAG2jC,IAAKlc,EAAMznB,GAAGm5C,4OCxNtF4G,0BAOPjE,EAAMzpB,kEAEVta,EAAMuV,KAASha,MAAM,GAAGE,QAAQ,GAAGG,QAAQ,GAAGmL,aAAa,GAC3D4kB,EAAQ3rB,EAAIuI,QAAQvC,KAAK,EAAG,QAAQtd,UACpCkjC,EAAM5rB,EAAIuI,QAAQvC,IAAI,EAAG,QAAQtd,mBAClCu/C,+BAA4Br9C,OAElBA,IAAZ0vB,KACIqR,MAAQA,IACRC,IAAMA,MAEND,MAAQrR,EAAQqR,OAASA,IACzBC,IAAMtR,EAAQsR,KAAOA,KAGvBsc,SAAU,IAEVnE,KAAOA,IACPoE,gBAAkB,IAClBC,YAAc,IACdzC,cAAe,IACfC,YAAa,IAGbyC,eAAiB,CACpBC,KAAK,EACL3c,MAAO,KACPC,IAAK,KACLrW,OAAAA,GACA9L,UAAW,aACX8+B,UAAU,EACVC,UAAU,EACVj7C,IAAK,KACLkD,IAAK,KACLg4C,QAAS,GACTC,QAAS,SACTC,YAAa,CACXC,QAAQ,EACRjhC,OAAQ,OAGP2S,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/B7H,MAAQ,CACXsI,MAAO,MAEJC,eAAiB,OAGjBhF,KAAKiF,QAAQvH,GAAG,WAAYP,OAAK+H,+BACjClF,KAAKiF,QAAQvH,GAAG,UAAYP,OAAKgI,0BACjCnF,KAAKiF,QAAQvH,GAAG,SAAYP,OAAKiI,6BAGjCpF,KAAKiF,QAAQvH,GAAG,aAAcP,OAAKkI,gCAGnCrF,KAAKiF,QAAQvH,GAAG,QAASP,OAAKmI,2BAC9BtF,KAAKiF,QAAQvH,GAAG,QAASP,OAAKoI,2BAG9BvF,KAAKwF,IAAIC,eAAeC,iBAAiB,QAASvI,OAAKwI,+BAEvDC,WAAWrvB,2DAmBPA,MACLA,EAAS,CAMXmpB,GAAKmG,gBAJU,CACb,YAAa,YAAa,MAAO,MAAO,UAAW,UAAW,WAAY,WAC1E,SAAU,WAAY,cAAe,UAAW,eAAgB,MAAO,kBAAmB,cAAe,oBAE9Ez+C,KAAKmvB,QAASA,GAEvCA,EAAQquB,aAAeruB,EAAQquB,YAAYC,aACxCc,gBAEH,UAAWpvB,GAAW,QAASA,SAE5BuvB,SAASvvB,EAAQqR,MAAOrR,EAAQsR,iDASnCke,EAAK3+C,eAKFm2C,IACPwI,EAAGC,cACHD,EAAG5B,SAAU,MAGT8B,EAAWF,EAAGle,IAAMke,EAAGne,MACrBzgC,EAAIu4C,GAAKpB,QAAQ,IAAIx6C,KAAQ,QAAQa,UACrCuhD,EAAoBH,EAAGxvB,QAAQquB,aAAemB,EAAGxvB,QAAQquB,YAAYhhC,QAAU,GAE/EgkB,EAAQzgC,EAAI8+C,EAAYC,EACxBre,EAAM1gC,EAAI8+C,GAAY,EAAIC,GAKhCH,EAAGD,SAASle,EAAOC,EAHH,CACdse,WAAW,KAMbF,EAAW,EADGF,EAAGjD,WAAWiD,EAAG/F,KAAKY,SAASwF,OAAOpW,OAAO+S,MACpC,IACR,KAAMkD,EAAW,IAC5BA,EAAW,MAAMA,EAAW,KAEhCF,EAAG/F,KAAKwF,IAAIC,eAAe7pB,MAAMyqB,WAAa,SAE9CN,EAAGO,iBAAmBC,GAAWhJ,EAAQ0I,GAG3C1I,8CAO8B12C,IAA1BO,KAAKk/C,mBACPE,aAAap/C,KAAKk/C,uBACbnC,SAAU,OACVnE,KAAKwF,IAAIC,eAAe7pB,MAAMyqB,WAAa,4CAyB3Cze,EAAOC,EAAKtR,EAASnpB,EAAUq5C,GACjClwB,IACHA,EAAU,KAEW,IAAnBA,EAAQmwB,SACVnwB,EAAQmwB,QAAS,OAEbX,EAAK3+C,KACLu/C,EAAsB9/C,MAAT+gC,EAAqB8X,GAAKpB,QAAQ1W,EAAO,QAAQjjC,UAAY,KAC1EiiD,EAAoB//C,MAAPghC,EAAqB6X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAAc,aAC3EkiD,wBACA3C,+BAA4Br9C,EAE7B0vB,EAAQ4vB,UAAW,OACfW,EAAY1/C,KAAKwgC,MACjBmf,EAAU3/C,KAAKygC,IACfplB,EAAyC,WAA7BoqB,GAAOtW,EAAQ4vB,YAA0B,aAAc5vB,EAAQ4vB,UAAa5vB,EAAQ4vB,UAAU1jC,SAAW,IACrHukC,EAA2C,WAA7Bna,GAAOtW,EAAQ4vB,YAA0B,mBAAoB5vB,EAAQ4vB,UAAa5vB,EAAQ4vB,UAAUc,eAAiB,gBACnIA,EAAiBvH,GAAKwH,gBAAgBF,OACvCC,QACG,IAAIz8C,6CAAiC28C,GAAeH,8BAA6BI,GAAY1H,GAAKwH,iBAAiB38C,KAAK,YAG1H88C,EAAWjZ,KACbkZ,GAAa,SAEJ,SAAPlsC,QACC2qC,EAAGtJ,MAAMsI,MAAMwC,SAAU,KAEtBz/B,EADMsmB,KACOiZ,EACbG,EAAOP,EAAen/B,EAAOrF,GAC7B8hB,EAAOzc,EAAOrF,EACdxS,EAAKs0B,GAAuB,OAAfoiB,EAAuBA,EAAaG,GAAaH,EAAaG,GAAaU,EACxF/sC,EAAK8pB,GAAuB,OAAfqiB,EAAuBA,EAAaG,GAAaH,EAAaG,GAAaS,EAE9FC,EAAU1B,EAAGjE,YAAY7xC,EAAGwK,GAC5BitC,GAA2B3B,EAAGxvB,QAAQ/E,OAAQu0B,EAAG/F,KAAM+F,EAAGxvB,QAAQiqB,aAClE8G,EAAaA,GAAcG,MAErBE,EAAS,CACb/f,MAAO,IAAI9jC,KAAKiiD,EAAGne,OACnBC,IAAK,IAAI/jC,KAAKiiD,EAAGle,KACjB6e,OAAQnwB,EAAQmwB,OAChBkB,MAAOrxB,EAAQqxB,UAGbnB,GAAiBA,EAAce,EAAMC,EAASljB,GAE9CkjB,GACF1B,EAAG/F,KAAKiF,QAAQ4C,KAAK,cAAeF,GAGlCpjB,MACE+iB,IACFvB,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgBF,GACjCv6C,UAAmBA,SAMzB24C,EAAGf,eAAiBuB,GAAWnrC,EAAM,KAKpCA,OAGHqsC,EAAUrgD,KAAK06C,YAAY6E,EAAYC,MAC3Cc,GAA2BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aACpEiH,EAAS,KACLE,EAAS,CACb/f,MAAO,IAAI9jC,KAAKsD,KAAKwgC,OACrBC,IAAK,IAAI/jC,KAAKsD,KAAKygC,KACnB6e,OAAQnwB,EAAQmwB,OAChBkB,MAAOrxB,EAAQqxB,eAGZ5H,KAAKiF,QAAQ4C,KAAK,cAAeF,GACtCnB,aAAcT,EAAG+B,WACjB/B,EAAG+B,UAAYvB,IAAY,WACzBR,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgBF,KACpC,KACCv6C,SAAmBA,mEAWYvG,IAAnCO,KAAK88C,iCACFA,2BAA6B98C,KAAKygC,IAAMzgC,KAAKwgC,OAASxgC,KAAK44C,KAAKwF,IAAIY,OAAO2B,aAE3E3gD,KAAK88C,qEAQR98C,KAAK49C,iBACPwB,aAAap/C,KAAK49C,qBACbA,eAAiB,0CAadpd,EAAOC,OAKbtjB,EAJAyjC,EAAqB,MAATpgB,EAAiB8X,GAAKpB,QAAQ1W,EAAO,QAAQjjC,UAAYyC,KAAKwgC,MAC1EqgB,EAAmB,MAAPpgB,EAAiB6X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAAcyC,KAAKygC,IACxEn7B,EAA2B,MAApBtF,KAAKmvB,QAAQ7pB,IAAegzC,GAAKpB,QAAQl3C,KAAKmvB,QAAQ7pB,IAAK,QAAQ/H,UAAY,KACtF6E,EAA2B,MAApBpC,KAAKmvB,QAAQ/sB,IAAek2C,GAAKpB,QAAQl3C,KAAKmvB,QAAQ/sB,IAAK,QAAQ7E,UAAY,QAIxF6B,MAAMwhD,IAA0B,OAAbA,QACf,IAAIx9C,+BAAwBo9B,WAEhCphC,MAAMyhD,IAAsB,OAAXA,QACb,IAAIz9C,6BAAsBq9B,WAI9BogB,EAASD,IACXC,EAASD,GAIC,OAARx+C,GACEw+C,EAAWx+C,IAEbw+C,GADAzjC,EAAQ/a,EAAMw+C,EAEdC,GAAU1jC,EAGC,MAAP7X,GACEu7C,EAASv7C,IACXu7C,EAASv7C,IAOL,OAARA,GACEu7C,EAASv7C,IAEXs7C,GADAzjC,EAAQ0jC,EAASv7C,EAEjBu7C,GAAU1jC,EAGC,MAAP/a,GACEw+C,EAAWx+C,IACbw+C,EAAWx+C,IAOU,OAAzBpC,KAAKmvB,QAAQmuB,QAAkB,KAC7BA,EAAUwD,GAAW9gD,KAAKmvB,QAAQmuB,YAClCA,EAAU,IACZA,EAAU,GAEPuD,EAASD,EAAYtD,EAAS,CAG5Bt9C,KAAKygC,IAAMzgC,KAAKwgC,QAAW8c,GAAWsD,GAAY5gD,KAAKwgC,MADvC,IAC+DqgB,GAAU7gD,KAAKygC,KAEjGmgB,EAAW5gD,KAAKwgC,MAChBqgB,EAAS7gD,KAAKygC,MAKdmgB,IADAzjC,EAAQmgC,GAAWuD,EAASD,IACT,EACnBC,GAAU1jC,EAAO,OAMM,OAAzBnd,KAAKmvB,QAAQouB,QAAkB,KAC7BA,EAAUuD,GAAW9gD,KAAKmvB,QAAQouB,SAClCA,EAAU,IACZA,EAAU,GAGPsD,EAASD,EAAYrD,IACnBv9C,KAAKygC,IAAMzgC,KAAKwgC,QAAW+c,GAAWqD,EAAW5gD,KAAKwgC,OAASqgB,EAAS7gD,KAAKygC,KAEhFmgB,EAAW5gD,KAAKwgC,MAChBqgB,EAAS7gD,KAAKygC,MAKdmgB,IADAzjC,EAAS0jC,EAASD,EAAYrD,GACX,EACnBsD,GAAU1jC,EAAO,QAKjBkjC,EAAWrgD,KAAKwgC,OAASogB,GAAY5gD,KAAKygC,KAAOogB,SAGhDD,GAAY5gD,KAAKwgC,OAASogB,GAAc5gD,KAAKygC,KAASogB,GAAY7gD,KAAKwgC,OAASqgB,GAAY7gD,KAAKygC,KACjGzgC,KAAKwgC,OAASogB,GAAY5gD,KAAKwgC,OAASqgB,GAAc7gD,KAAKygC,KAAOmgB,GAAc5gD,KAAKygC,KAAOogB,QAC5FjI,KAAKiF,QAAQ4C,KAAK,yBAGpBjgB,MAAQogB,OACRngB,IAAMogB,EACJR,2CAQA,CACL7f,MAAOxgC,KAAKwgC,MACZC,IAAKzgC,KAAKygC,wCAWHmI,EAAOmY,UACTlE,EAAMnB,WAAW17C,KAAKwgC,MAAOxgC,KAAKygC,IAAKmI,EAAOmY,wCAmC1CP,QACNxD,gBAAkB,OAClBgE,cAAgB,EAGhBhhD,KAAKmvB,QAAQiuB,UAGbp9C,KAAKihD,eAAeT,IAIpBxgD,KAAKq1C,MAAMsI,MAAMuD,qBAEjBtC,mBAEAvJ,MAAMsI,MAAMnd,MAAQxgC,KAAKwgC,WACzB6U,MAAMsI,MAAMld,IAAMzgC,KAAKygC,SACvB4U,MAAMsI,MAAMwC,UAAW,EAExBngD,KAAK44C,KAAKwF,IAAI+C,YACXvI,KAAKwF,IAAI+C,KAAK3sB,MAAM4sB,OAAS,yCAS9BZ,MACDA,GAEAxgD,KAAKq1C,MAAMsI,MAAMwC,UAGjBngD,KAAKmvB,QAAQiuB,UAKbp9C,KAAKq1C,MAAMsI,MAAMuD,mBAEhB5iC,EAAYte,KAAKmvB,QAAQ7Q,UAC/B+iC,GAAkB/iC,OACdgjC,EAAsB,cAAbhjC,EAA6BkiC,EAAMe,OAASf,EAAMgB,OAC/DF,GAASthD,KAAKg9C,oBACV6B,EAAY7+C,KAAKq1C,MAAMsI,MAAMld,IAAMzgC,KAAKq1C,MAAMsI,MAAMnd,MAIxDqe,GADiByB,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,SAIvFghB,EADE7Y,EAAsB,cAAbtqB,EAA6Bte,KAAK44C,KAAKY,SAASwF,OAAOpW,MAAQ5oC,KAAK44C,KAAKY,SAASwF,OAAOnW,OAGtG4Y,EADEzhD,KAAKmvB,QAAQguB,IACHmE,EAAQ1Y,EAAQiW,GAEfyC,EAAQ1Y,EAAQiW,MAGzB+B,EAAW5gD,KAAKq1C,MAAMsI,MAAMnd,MAAQihB,EACpCZ,EAAS7gD,KAAKq1C,MAAMsI,MAAMld,IAAMghB,EAGhCC,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAU5gD,KAAKghD,cAAcM,GAAO,GACnGK,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,EAAQ7gD,KAAKghD,cAAcM,GAAO,MACjGI,GAAad,GAAYe,GAAWd,cACjC7D,iBAAmBsE,OACnBjM,MAAMsI,MAAMnd,MAAQkhB,OACpBrM,MAAMsI,MAAMld,IAAMkhB,YAClB5D,QAAQyC,QAIVQ,cAAgBM,OAChB5G,YAAYkG,EAAUC,OAGrBjH,EAAY,IAAIl9C,KAAKsD,KAAKwgC,OAC1BqZ,EAAU,IAAIn9C,KAAKsD,KAAKygC,UAGzBmY,KAAKiF,QAAQ4C,KAAK,cAAe,CACpCjgB,MAAOoZ,EACPnZ,IAAOoZ,EACPyF,QAAQ,EACRkB,MAAAA,SAIG5H,KAAKiF,QAAQ4C,KAAK,+CAQdD,GACJxgD,KAAKq1C,MAAMsI,MAAMwC,UAGjBngD,KAAKmvB,QAAQiuB,UAKbp9C,KAAKq1C,MAAMsI,MAAMuD,qBAEjB7L,MAAMsI,MAAMwC,UAAW,EACxBngD,KAAK44C,KAAKwF,IAAI+C,YACXvI,KAAKwF,IAAI+C,KAAK3sB,MAAM4sB,OAAS,aAI/BxI,KAAKiF,QAAQ4C,KAAK,eAAgB,CACrCjgB,MAAO,IAAI9jC,KAAKsD,KAAKwgC,OACrBC,IAAO,IAAI/jC,KAAKsD,KAAKygC,KACrB6e,QAAQ,EACRkB,MAAAA,2CAUUA,OAERc,EAAQ,KACRd,EAAMoB,WACRN,EAAQd,EAAMoB,WAAa,IAClBpB,EAAMqB,OAGfP,GAASd,EAAMqB,OAAS,EACfrB,EAAMgB,SACfF,GAASd,EAAMgB,OAAS,KAIrBxhD,KAAKmvB,QAAQ2yB,UAAYtB,EAAMxgD,KAAKmvB,QAAQ2yB,UAAY9hD,KAAKmvB,QAAQkuB,WACnEr9C,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,WAKvCp9C,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,UAGvCp9C,KAAKihD,eAAeT,IAKrBc,EAAO,KAOL3F,EASAoG,EAVEC,EAAehiD,KAAKmvB,QAAQ6yB,cAAgB,KAGhDrG,EADE2F,EAAQ,EACF,EAAKA,EAAQU,EAGb,GAAK,EAAKV,EAAQU,GAKxBhiD,KAAK+8C,QAAS,KACV+B,EAAoB9+C,KAAKmvB,QAAQquB,aAAex9C,KAAKmvB,QAAQquB,YAAYhhC,QAAU,GACzFulC,EAAc/hD,KAAKwgC,OAAUxgC,KAAKygC,IAAMzgC,KAAKwgC,OAASse,MACjD,KACCmD,EAAUjiD,KAAKkiD,WAAW,CAAC/3C,EAAGq2C,EAAM2B,QAAS54C,EAAGi3C,EAAM4B,SAAUpiD,KAAK44C,KAAKwF,IAAIY,QACpF+C,EAAc/hD,KAAKqiD,eAAeJ,QAE/BK,KAAK3G,EAAOoG,EAAaT,EAAOd,GAIrCA,EAAM+B,mDASD/B,QACFnL,MAAMsI,MAAMnd,MAAQxgC,KAAKwgC,WACzB6U,MAAMsI,MAAMld,IAAMzgC,KAAKygC,SACvB4U,MAAMsI,MAAMuD,eAAgB,OAC5B7L,MAAMsI,MAAMqB,OAAS,UACrB3J,MAAMsI,MAAM6E,WAAa,UACzBvF,YAAc,OACdD,gBAAkB,EAEvB1E,GAAKiK,eAAe/B,oCAQbA,MAEDxgD,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,UAG5C9E,GAAKiK,eAAe/B,QAEfnL,MAAMsI,MAAMuD,eAAgB,EAE5BlhD,KAAKq1C,MAAMsI,MAAMqB,cACf3J,MAAMsI,MAAMqB,OAASh/C,KAAKkiD,WAAW1B,EAAMxB,OAAQh/C,KAAK44C,KAAKwF,IAAIY,aACjE3J,MAAMsI,MAAM6E,WAAaxiD,KAAKqiD,eAAeriD,KAAKq1C,MAAMsI,MAAMqB,cAGhEJ,kBACCjD,EAAQ,GAAK6E,EAAM7E,MAAQ37C,KAAKi9C,aAChCuF,EAAaxiD,KAAKq1C,MAAMsI,MAAM6E,WAE9BtG,EAAiBoE,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,KAC3FgiB,EAAuBnC,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAMwiD,GAC1GE,EAAsBxG,EAAiBuG,EAGzC7B,EAAY4B,EAAaC,GAAyBziD,KAAKq1C,MAAMsI,MAAMnd,OAASgiB,EAAaC,IAAyB9G,EAClHkF,EAAU2B,EAAaE,GAAwB1iD,KAAKq1C,MAAMsI,MAAMld,KAAO+hB,EAAaE,IAAwB/G,OAG3GnB,aAAe,EAAImB,GAAS,OAC5BlB,WAAakB,EAAQ,GAAK,MAEzB+F,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAU,EAAIjF,GAAO,GACpFgG,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,EAAQlF,EAAQ,GAAG,GAClF+F,GAAad,GAAYe,GAAWd,SACjCxL,MAAMsI,MAAMnd,MAAQkhB,OACpBrM,MAAMsI,MAAMld,IAAMkhB,OAClB1E,YAAc,EAAIuD,EAAM7E,MAC7BiF,EAAWc,EACXb,EAASc,OAGLxyB,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9B,SAASkC,EAAUC,EAAQ1xB,QAE3BqrB,cAAe,OACfC,YAAa,0CAUL+F,OAGP2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDQ,EAAsB3iD,KAAK44C,KAAKwF,IAAI3E,gBAAgBmJ,wBACpDz4C,EAAInK,KAAKmvB,QAAQguB,IAAMgF,EAAUQ,EAAoBjO,KAAOiO,EAAoBhO,MAAQwN,EACxFzhC,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,UAE5BuW,GAAQ1gB,KAAKwgC,OAAS9f,GAAQ1gB,KAAKygC,2CAS7BwhB,OACTvG,EACEp9B,EAAYte,KAAKmvB,QAAQ7Q,aAE/B+iC,GAAkB/iC,GAED,cAAbA,SACKte,KAAK44C,KAAKN,KAAK2D,OAAOgG,EAAQ93C,GAAG5M,cAGlCsrC,EAAS7oC,KAAK44C,KAAKY,SAASwF,OAAOnW,cACzC6S,EAAa17C,KAAK07C,WAAW7S,GACtBoZ,EAAQ14C,EAAImyC,EAAWC,MAAQD,EAAWl/B,0CAW1CmhC,EAAO/V,OACVib,EAAcjb,EAAQgb,+BACxB5iD,KAAKmvB,QAAQguB,IACR,CACLhzC,EAAG04C,EAAYlO,MAAQgJ,EAAMxzC,EAC7BZ,EAAGo0C,EAAMp0C,EAAIs5C,EAAYC,KAGpB,CACL34C,EAAGwzC,EAAMxzC,EAAI04C,EAAYnO,KACzBnrC,EAAGo0C,EAAMp0C,EAAIs5C,EAAYC,kCAiB1BnH,EAAOqD,EAAQsC,EAAOd,GAEX,MAAVxB,IACFA,GAAUh/C,KAAKwgC,MAAQxgC,KAAKygC,KAAO,OAG/Byb,EAAiBoE,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,KAC3FgiB,EAAuBnC,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAMg/C,GAC1G0D,EAAsBxG,EAAiBuG,EAGzC7B,EAAY5B,EAAOyD,GAAyBziD,KAAKwgC,OAASwe,EAAOyD,IAAyB9G,EAC1FkF,EAAY7B,EAAO0D,GAAwB1iD,KAAKygC,KAAOue,EAAO0D,IAAwB/G,OAGrFnB,eAAe8G,EAAQ,QACvB7G,cAAc6G,EAAS,OACtBI,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAUU,GAAO,GAChFK,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,GAASS,GAAO,GAC/EI,GAAad,GAAYe,GAAWd,IACtCD,EAAWc,EACXb,EAASc,OAGLxyB,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9B,SAASkC,EAAUC,EAAQ1xB,QAE3BqrB,cAAe,OACfC,YAAa,+BASf6G,OAEGnkC,EAAQnd,KAAKygC,IAAMzgC,KAAKwgC,MAGxBogB,EAAW5gD,KAAKwgC,MAAQrjB,EAAOmkC,EAC/BT,EAAS7gD,KAAKygC,IAAMtjB,EAAOmkC,OAI5B9gB,MAAQogB,OACRngB,IAAMogB,iCAONkC,OAGC5lC,GAFUnd,KAAKwgC,MAAQxgC,KAAKygC,KAAO,EAEnBsiB,EAGhBnC,EAAW5gD,KAAKwgC,MAAQrjB,EACxB0jC,EAAS7gD,KAAKygC,IAAMtjB,OAOrBuhC,SAASkC,EAAUC,EALR,CACd9B,WAAW,EACXO,QAAQ,EACRkB,MAAO,4CA/aOhgB,EAAOC,EAAKmI,EAAOmY,eACfthD,IAAhBshD,IACFA,EAAc,GAEH,GAATnY,GAAenI,EAAMD,GAAS,EACzB,CACLhkB,OAAQgkB,EACRmb,MAAO/S,GAASnI,EAAMD,EAAQugB,IAIzB,CACLvkC,OAAQ,EACRm/B,MAAO,UAtcoBhD,IAk3BnC,SAAS0I,GAAmB/iC,MACT,cAAbA,GAA0C,YAAbA,QACzB,IAAI0O,uCAAgC1O,4CC33B9C,OAAiB2R,EAAK2jB,uCCYboP,EAAQ3hD,MACXA,EAAK,gBAWIA,OACR,IAAI4B,KAAO+/C,EAAQ7mD,UACtBkF,EAAI4B,GAAO+/C,EAAQ7mD,UAAU8G,UAExB5B,EAfS4hD,CAAM5hD,GAVtB5F,UAAiBunD,EAqCnBA,EAAQ7mD,UAAUm6C,GAClB0M,EAAQ7mD,UAAUmiD,iBAAmB,SAASkC,EAAO3jD,eAC9CqmD,WAAaljD,KAAKkjD,YAAc,IACpCljD,KAAKkjD,WAAW,IAAM1C,GAASxgD,KAAKkjD,WAAW,IAAM1C,IAAU,IAC7DvjD,KAAKJ,GACDmD,MAaTgjD,EAAQ7mD,UAAUgnD,KAAO,SAAS3C,EAAO3jD,YAC9By5C,SACFC,IAAIiK,EAAOlK,GAChBz5C,EAAGhB,MAAMmE,KAAMlE,kBAGjBw6C,EAAGz5C,GAAKA,OACHy5C,GAAGkK,EAAOlK,GACRt2C,MAaTgjD,EAAQ7mD,UAAUo6C,IAClByM,EAAQ7mD,UAAUinD,eAClBJ,EAAQ7mD,UAAUknD,mBAClBL,EAAQ7mD,UAAUmnD,oBAAsB,SAAS9C,EAAO3jD,WACjDqmD,WAAaljD,KAAKkjD,YAAc,GAGjC,GAAKpnD,UAAUkB,mBACZkmD,WAAa,GACXljD,SAcLujD,EAVAC,EAAYxjD,KAAKkjD,WAAW,IAAM1C,OACjCgD,EAAW,OAAOxjD,QAGnB,GAAKlE,UAAUkB,qBACVgD,KAAKkjD,WAAW,IAAM1C,GACtBxgD,SAKJ,IAAIlD,EAAI,EAAGA,EAAI0mD,EAAUxmD,OAAQF,QACpCymD,EAAKC,EAAU1mD,MACJD,GAAM0mD,EAAG1mD,KAAOA,EAAI,CAC7B2mD,EAAUvQ,OAAOn2C,EAAG,gBAOC,IAArB0mD,EAAUxmD,eACLgD,KAAKkjD,WAAW,IAAM1C,GAGxBxgD,MAWTgjD,EAAQ7mD,UAAUskD,KAAO,SAASD,QAC3B0C,WAAaljD,KAAKkjD,YAAc,WAEjClgD,EAAO,IAAI/G,MAAMH,UAAUkB,OAAS,GACpCwmD,EAAYxjD,KAAKkjD,WAAW,IAAM1C,GAE7B1jD,EAAI,EAAGA,EAAIhB,UAAUkB,OAAQF,IACpCkG,EAAKlG,EAAI,GAAKhB,UAAUgB,MAGtB0mD,EAEG,CAAI1mD,EAAI,MAAR,IAAWmD,GADhBujD,EAAYA,EAAUtgD,MAAM,IACIlG,OAAQF,EAAImD,IAAOnD,EACjD0mD,EAAU1mD,GAAGjB,MAAMmE,KAAMgD,UAItBhD,MAWTgjD,EAAQ7mD,UAAUsnD,UAAY,SAASjD,eAChC0C,WAAaljD,KAAKkjD,YAAc,GAC9BljD,KAAKkjD,WAAW,IAAM1C,IAAU,IAWzCwC,EAAQ7mD,UAAUunD,aAAe,SAASlD,WAC9BxgD,KAAKyjD,UAAUjD,GAAOxjD,mCC7J5B2mD,EANFloD,WAMEkoD,EAAe,KAwBZ,SAASC,EAAYC,EAAQ10B,OAC9B20B,EAAW30B,GAAW,CACxBozB,gBAAgB,MAGdsB,EAAOE,QAAS,KAGdC,EAASH,EAETI,EAAoB,SAASrc,EAASzY,OACpC9kB,EAAInO,OAAO84B,OAAO8uB,UAClB30B,GAAS60B,EAAO5e,OAAO/6B,EAAG8kB,GACvBy0B,EAAY,IAAII,EAAOpc,EAASv9B,GAAIA,WAE7C25C,EAAO5e,OAAO6e,EAAmBD,GAEjCC,EAAkBF,QAAU,SAAUnc,EAASzY,OACzC9kB,EAAInO,OAAO84B,OAAO8uB,UAClB30B,GAAS60B,EAAO5e,OAAO/6B,EAAG8kB,GACvBy0B,EAAY,IAAII,EAAOD,QAAQnc,EAASv9B,GAAIA,IAG9C45C,MAKLlY,EAAU7vC,OAAO84B,OAAO6uB,GAGxBjc,EAAUic,EAAOjc,iBAiGZ/8B,EAAMq5C,UACNA,EAAO59C,MAAM,mBAOb69C,EAAkB3D,MAEN,iBAAfA,EAAMtpB,KAAyB,IAG5BspB,EAAM4D,SAASC,WAClB7D,EAAM4D,SAASC,SAAW,IAGxB7D,EAAM4D,SAASC,SAAS7D,EAAMtpB,aAIhCspB,EAAM4D,SAASC,SAAS7D,EAAMtpB,OAAQ,MAKtCqU,GAAU,EACdiV,EAAM8D,gBAAkB,WACtB/Y,GAAU,OAIRgZ,EAAU/D,EAAM4D,SAASE,gBAAgBj0B,KAAKmwB,EAAM4D,UACnC,mBAAXG,IACR/D,EAAM4D,SAASE,gBAAkB,WAC/BC,IACA/D,EAAM8D,oBAKV9D,EAAMgE,YAAcb,UAGhBc,EAAOd,EACJc,IAASlZ,GAAS,KACnBmZ,EAAaD,EAAKZ,UACnBa,UACGC,EACIhqC,EAAI,EAAGA,EAAI+pC,EAAW1nD,OAAQ2d,OACpCgqC,EAAYD,EAAW/pC,GAAGgqC,UAAUnE,EAAMtpB,MAC5B,IAAK,IAAIp6B,EAAI,EAAGA,EAAI6nD,EAAU3nD,SAAWuuC,EAASzuC,IAC9D6nD,EAAU7nD,GAAG0jD,GAInBiE,EAAOA,EAAKld,mBAvJZK,EAAQic,SAAQjc,EAAQic,OAAS,IACrCjc,EAAQic,OAAO5mD,KAAK8uC,GAIpB8X,EAAOvN,GAAG,gBAAgB,SAAUkK,IACF,IAA5BsD,EAASvB,gBAA4BuB,EAASvB,iBAAmB/B,EAAMoE,aACzEpE,EAAM+B,iBAEJ/B,EAAMqE,UACRlB,EAAenD,EAAM7wB,WAKzBoc,EAAQ4Y,UAAY,GAQpB5Y,EAAQuK,GAAK,SAAU4N,EAAQ1Q,UAE7B3oC,EAAMq5C,GAAQrwC,SAAQ,SAAU2sC,OAC1BmE,EAAY5Y,EAAQ4Y,UAAUnE,GAC7BmE,IACH5Y,EAAQ4Y,UAAUnE,GAASmE,EAAY,GAGvCd,EAAOvN,GAAGkK,EAAO2D,IAEnBQ,EAAU1nD,KAAKu2C,MAGVzH,GAWTA,EAAQwK,IAAM,SAAU2N,EAAQ1Q,UAE9B3oC,EAAMq5C,GAAQrwC,SAAQ,SAAU2sC,OAC1BmE,EAAY5Y,EAAQ4Y,UAAUnE,GAC9BmE,KACFA,EAAYnR,EAAUmR,EAAU1sB,QAAO,SAAUzrB,UACxCA,IAAMgnC,KACV,IAESx2C,OAAS,EACrB+uC,EAAQ4Y,UAAUnE,GAASmE,GAI3Bd,EAAOtN,IAAIiK,EAAO2D,UACXpY,EAAQ4Y,UAAUnE,QAKxBzU,GAQTA,EAAQ0U,KAAO,SAASqE,EAAWtE,GACjCmD,EAAenD,EAAM7wB,OACrBk0B,EAAOpD,KAAKqE,EAAWtE,IAGzBzU,EAAQgZ,QAAU,eAEZC,EAAUnB,EAAOjc,QAAQic,OACzBoB,EAAMD,EAAQt7C,QAAQqiC,IACd,IAATkZ,GAAYD,EAAQ/R,OAAOgS,EAAI,GAC9BD,EAAQhoD,eAAe6mD,EAAOjc,QAAQic,OAG1C9X,EAAQ4Y,UAAY,GAGpBd,EAAOkB,WAgEFhZ;;;;;;;6aCpMX,OAAAmZ,GAvBA,mBAAWhpD,OAAPkpC,OACO,SAAAzV,MACHA,MAAAA,QACI,IAAA3C,UAAN,sDAGEvmB,EAASvK,OAAbyzB,GACS/K,EAAT,EAAoBA,EAAQ9oB,UAA5BkB,OAA8C4nB,IAAS,KAC/CiB,EAAS/pB,UAAf8oB,MACIiB,MAAAA,MACG,IAALs/B,KAAAt/B,EACMA,EAAAA,eAAJs/B,KACE1+C,EAAAA,GAAkBof,EAAlBpf,WAKRA,GAGOvK,OAATkpC,utnBCFEggB,ygCAEkB,oBAAXp5B,OAETo5B,GAAiBnB,GADCj4B,OAAM,QAAcg4B,GACQ,CAC5CzB,eAAgB,UAGlB6C,GAAiB,iBAtBV,CACL9O,GAHI+O,EAAO,aAIX9O,IAAK8O,EACLN,QAASM,EACT5E,KAAM4E,EAENt7C,aAAIhM,SACK,CACLiG,IAAKqhD,KAXb,IACQA,UA4BOD,GClCR,SAASE,GAASzB,EAAQ79C,GAC/BA,EAASu/C,aAAe,SAAU/E,GAC5BA,EAAMqE,SACR7+C,EAASw6C,IAIbqD,EAAOvN,GAAG,eAAgBtwC,EAASu/C,kBCa/BC,yBAUQhlB,EAAOC,EAAKglB,EAAarM,EAAajqB,mBAC3C/E,OAAU+E,GAAWA,EAAQ/E,QAAWA,QACxC+E,QAAUA,GAAoB,QAG9B8rB,QAAUj7C,KAAKoqB,cACfs7B,OAAS1lD,KAAKoqB,cACd8wB,KAAOl7C,KAAKoqB,cAEZu7B,WAAa,OACbhK,MAAQ,WACR5X,KAAO,OAGP2a,SAASle,EAAOC,EAAKglB,QAGrBlK,aAAc,OACdD,eAAgB,OAChBD,cAAe,EAChBnY,GAAckW,QACXA,YAAcA,OAGdA,YADiB35C,MAAf25C,EACY,CAACA,GAGD,QAGhB37C,OAAS+nD,EAASI,oDAQfx7B,QACHA,OAASA,OAGT6wB,QAAUj7C,KAAKoqB,OAAOpqB,KAAKi7C,QAAQ19C,gBACnCmoD,OAAS1lD,KAAKoqB,OAAOpqB,KAAK0lD,OAAOnoD,gBACjC29C,KAAOl7C,KAAKoqB,OAAOpqB,KAAKk7C,KAAK39C,6CAS1BE,OACFwhB,EAAgBq5B,GAAKuN,WAAW,GAAIL,EAASI,aAC9CnoD,OAAS66C,GAAKuN,WAAW5mC,EAAexhB,oCAatC+iC,EAAOC,EAAKglB,QACbjlB,aAAiB9jC,MAAW+jC,aAAe/jC,WACxC,qDAGJgpD,OAAmBjmD,MAAT+gC,EAAsBxgC,KAAKoqB,OAAOoW,EAAMjjC,WAAaypC,UAC/DkU,KAAez7C,MAAPghC,EAAoBzgC,KAAKoqB,OAAOqW,EAAIljC,WAAaypC,KAE1DhnC,KAAK2lD,gBACFG,eAAeL,wCAQjBxK,QAAUj7C,KAAK0lD,OAAOtoC,aACtB2oC,6DAUa,QAAd/lD,KAAK27C,YACFV,QAAQztC,QAAQ,GAIfxN,KAAK27C,WACN,YACEV,QAAQ5xC,KAAKrJ,KAAK+jC,KAAOviC,KAAKE,MAAM1B,KAAKi7C,QAAQ5xC,OAASrJ,KAAK+jC,YAC/DkX,QAAQjxC,MAAM,OAChB,aAAqBixC,QAAQhxC,KAAK,OAClC,WACA,UACA,eAAqBgxC,QAAQ7qC,MAAM,OACnC,YAAqB6qC,QAAQ3qC,QAAQ,OACrC,cAAqB2qC,QAAQxqC,QAAQ,OACrC,cAAqBwqC,QAAQr/B,aAAa,MAIhC,GAAb5b,KAAK+jC,KAAW,KAEbiiB,EAAehmD,KAAKi7C,QAAQ79B,eACzBpd,KAAK27C,WACN,mBAAqBV,QAAQr8B,SAAS5e,KAAKi7C,QAAQr/B,eAAiB5b,KAAK+jC,KAAM,0BAC/E,cAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQxqC,UAAYzQ,KAAK+jC,KAAM,qBAC1E,cAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ3qC,UAAYtQ,KAAK+jC,KAAM,qBAC1E,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,mBACxE,cACA,WAAqBkX,QAAQr8B,UAAU5e,KAAKi7C,QAAQhxC,OAAS,GAAKjK,KAAK+jC,KAAM,iBAC7E,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ1tC,OAASvN,KAAK+jC,KAAM,kBACvE,aAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQjxC,QAAUhK,KAAK+jC,KAAM,mBACxE,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ5xC,OAASrJ,KAAK+jC,KAAM,QAGzEiiB,EAAa/jC,OAAOjiB,KAAKi7C,gBACrBA,QAAUj7C,KAAKoqB,OAAOk2B,GAA4BtgD,KAAKo5C,YAAap5C,KAAKi7C,QAAQ19C,WAAY,GAAG,+CAUnGyC,KAAKi7C,QAAQ19C,WAAayC,KAAKk7C,KAAK39C,6CAOtCmsC,EAAO1pC,KAAKi7C,QAAQ19C,iBAIlByC,KAAK27C,WACN,mBAAqBV,QAAQpgC,IAAI7a,KAAK+jC,KAAM,yBAC5C,cAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,oBAC5C,cAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,oBAC5C,YACEkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,QAExB/jC,KAAKi7C,QAAQjxC,QAAU,OACpBixC,QAAQr8B,SAAS5e,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,QAEpD/jC,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,MAAS,QAClCkX,QAAQpgC,IAAI7a,KAAK+jC,KAAO/jC,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,kBAIhE,cACA,WAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,iBAC5C,UAC4B,IAA3B/jC,KAAKi7C,QAAQztC,eACVytC,QAAQztC,QAAQ,QAChBytC,QAAQpgC,IAAI7a,KAAK+jC,KAAM,aACvB,IAAoC,IAAjC/jC,KAAKmvB,QAAQ82B,qBAChBhL,QAAQpgC,IAAI7a,KAAK+jC,KAAM,YACvB,KACCxyB,EAAWvR,KAAKi7C,QAAQ79B,QAC9B7L,EAASsJ,IAAI,EAAG,QACbtJ,EAAS0Q,OAAOjiB,KAAKi7C,QAAS,cAC1BA,QAAQpgC,IAAI7a,KAAK+jC,KAAM,cAEvBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,aACvBkX,QAAQhxC,KAAK,cAInB,aAAqBgxC,QAAQpgC,IAAI7a,KAAK+jC,KAAM,mBAC5C,YAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,WAIlC,GAAb/jC,KAAK+jC,YAEC/jC,KAAK27C,WACN,cAAmB37C,KAAKi7C,QAAQr/B,eAAiB,GAAK5b,KAAKi7C,QAAQr/B,eAAiB5b,KAAK+jC,MAAM/jC,KAAKi7C,QAAQr/B,aAAa,aACzH,SAAmB5b,KAAKi7C,QAAQxqC,UAAY,GAAKzQ,KAAKi7C,QAAQxqC,UAAYzQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQxqC,QAAQ,aAC1G,SAAmBzQ,KAAKi7C,QAAQ3qC,UAAY,GAAKtQ,KAAKi7C,QAAQ3qC,UAAYtQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ3qC,QAAQ,aAC1G,OAAmBtQ,KAAKi7C,QAAQ7qC,QAAU,GAAKpQ,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ7qC,MAAM,aACpG,cACA,MAAmBpQ,KAAKi7C,QAAQhxC,OAASjK,KAAK+jC,KAAK,GAAG/jC,KAAKi7C,QAAQhxC,KAAK,aACxE,OAAmBjK,KAAKi7C,QAAQ1tC,OAASvN,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ1tC,KAAK,aACtE,QAAmBvN,KAAKi7C,QAAQjxC,QAAUhK,KAAK+jC,MAAM/jC,KAAKi7C,QAAQjxC,MAAM,GAO7EhK,KAAKi7C,QAAQ19C,WAAamsC,SACvBuR,QAAUj7C,KAAKk7C,KAAK99B,cAItBm+B,aAAc,OACdD,eAAgB,OAChBD,cAAe,EAEpBiF,GAA6BtgD,KAAKoqB,OAAQpqB,KAAM0pC,+CAQzC1pC,KAAKi7C,QAAQ79B,yCAebmjC,GACHA,GAAiC,iBAAhBA,EAAO5E,aACrBA,MAAQ4E,EAAO5E,WACf5X,KAAOwc,EAAOxc,KAAO,EAAIwc,EAAOxc,KAAO,OACvC4hB,WAAY,wCAQRO,QACNP,UAAYO,yCAOJT,MACMhmD,MAAfgmD,GAeAU,QAAgBV,SAA0B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAeV,SAA2B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAeV,SAA2B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAcV,SAA4B9J,MAAQ,YAAoB5X,KAAO,IAC7EoiB,QAAcV,SAA4B9J,MAAQ,YAAoB5X,KAAO,IAC7EoiB,QAAaV,SAA6B9J,MAAQ,YAAoB5X,KAAO,GAdzD,QAeT0hB,SAA+B9J,MAAQ,YAAoB5X,KAAO,GAC7EqiB,OAAcX,SAA4B9J,MAAQ,aAAoB5X,KAAO,GAfzD,OAgBR0hB,SAA8B9J,MAAQ,aAAoB5X,KAAO,GAC7EsiB,OAAYZ,GAAezlD,KAAKmvB,QAAQm3B,qBAAgC3K,MAAQ,YAAoB5X,KAAO,GAC3GsiB,OAAYZ,SAA8B9J,MAAQ,WAAoB5X,KAAO,GAjBzD,MAkBV0hB,SAAgC9J,MAAQ,WAAoB5X,KAAO,GAC7EsiB,MAAYZ,SAA8B9J,MAAQ,eAAoB5X,KAAO,GAC7EwiB,MAAad,SAA6B9J,MAAQ,YAAoB5X,KAAO,GAnBzD,KAoBT0hB,SAA+B9J,MAAQ,YAAoB5X,KAAO,GAC7EyiB,IAAgBf,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7EyiB,IAAgBf,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7EyiB,IAAef,SAA2B9J,MAAQ,cAAoB5X,KAAO,GAtBzD,IAuBP0hB,SAA6B9J,MAAQ,cAAoB5X,KAAO,GAC7E0iB,KAAgBhB,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7E0iB,IAAgBhB,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7E0iB,IAAehB,SAA2B9J,MAAQ,cAAoB5X,KAAO,GAzBzD,IA0BP0hB,SAA6B9J,MAAQ,cAAoB5X,KAAO,GAC7E2iB,IAAsBjB,SAAoB9J,MAAQ,mBAAoB5X,KAAO,KAC7E2iB,IAAsBjB,SAAoB9J,MAAQ,mBAAoB5X,KAAO,KAC7E2iB,GAAqBjB,SAAqB9J,MAAQ,mBAAoB5X,KAAO,IAC7E2iB,GAAqBjB,SAAqB9J,MAAQ,mBAAoB5X,KAAO,IAC7E2iB,EAAoBjB,SAAsB9J,MAAQ,mBAAoB5X,KAAO,GA9BzD,EA+BF0hB,SAAwB9J,MAAQ,mBAAoB5X,KAAO,yCAqIxD,GAArB/jC,KAAKq7C,oBACCr7C,KAAK27C,WACN,WACA,YACA,WACA,cACA,UACA,WACA,aACA,aACA,qBACI,iBAEA,OAGR,GAA0B,GAAtB37C,KAAKs7C,qBACJt7C,KAAK27C,WACN,WACA,cACA,UACA,WACA,aACA,aACA,qBACI,iBAEA,OAGR,GAAwB,GAApB37C,KAAKu7C,mBACJv7C,KAAK27C,WACN,kBACA,aACA,aACA,cACI,iBAEA,MAIP1xC,EAAOjK,KAAKoqB,OAAOpqB,KAAKi7C,gBACtBj7C,KAAK27C,WACN,qBAC4B,GAAvB1xC,EAAK2R,mBACV,gBACuB,GAAlB3R,EAAKwG,cACV,gBACqB,GAAhBxG,EAAKmG,SAAoC,GAAlBnG,EAAKqG,cACjC,cACqB,GAAhBrG,EAAKmG,YACV,cACA,UAEA,cACoB,GAAfnG,EAAKA,WACV,eACqB,GAAhBA,EAAKD,YACV,sBAGI,yCAWCC,MACAxK,MAARwK,IACFA,EAAOjK,KAAKi7C,SAEVhxC,aAAgBvN,OAClBuN,EAAOjK,KAAKoqB,OAAOngB,IAGmB,mBAA7BjK,KAAKvC,OAAOkpD,mBACd3mD,KAAKvC,OAAOkpD,YAAY18C,EAAMjK,KAAK27C,MAAO37C,KAAK+jC,UAGlDtmC,EAASuC,KAAKvC,OAAOkpD,YAAY3mD,KAAK27C,cAEpC37C,KAAK27C,WACN,UAGgB,IAAhB1xC,EAAKA,QAAmC,IAAnBA,EAAKuD,gBAClB,kBAGH/P,GAAUA,EAAOT,OAAS,EAAKgD,KAAKoqB,OAAOngB,GAAMxM,OAAOA,GAAU,0CAWlEwM,MACAxK,MAARwK,IACFA,EAAOjK,KAAKi7C,SAEVhxC,aAAgBvN,OAClBuN,EAAOjK,KAAKoqB,OAAOngB,IAGmB,mBAA7BjK,KAAKvC,OAAOmpD,mBACd5mD,KAAKvC,OAAOmpD,YAAY38C,EAAMjK,KAAK27C,MAAO37C,KAAK+jC,UAGlDtmC,EAASuC,KAAKvC,OAAOmpD,YAAY5mD,KAAK27C,cACpCl+C,GAAUA,EAAOT,OAAS,EAAKgD,KAAKoqB,OAAOngB,GAAMxM,OAAOA,GAAU,gDAQpEopD,EAAU7mD,KAAKoqB,OACfrsB,EAAIiC,KAAKoqB,OAAOpqB,KAAKi7C,SACrBA,EAAUl9C,EAAEL,OAASK,EAAEL,OAAO,MAAQK,EAAEohB,KAAK,MAC7C4kB,EAAO/jC,KAAK+jC,KACZ+iB,EAAa,YAOVC,EAAKjlD,UACJA,EAAQiiC,EAAO,GAAK,EAAK,YAAc,oBAQxCijB,EAAM/8C,UACTA,EAAKgY,OAAO+kB,KAAY,OACnB,aAEL/8B,EAAKgY,OAAO4kC,IAAUhsC,IAAI,EAAG,OAAQ,OAChC,gBAEL5Q,EAAKgY,OAAO4kC,IAAUhsC,KAAK,EAAG,OAAQ,OACjC,iBAEF,YAQAosC,EAAYh9C,UACZA,EAAKgY,OAAO+kB,KAAY,QAAU,oBAAsB,YAQxDkgB,EAAaj9C,UACbA,EAAKgY,OAAO+kB,KAAY,SAAW,qBAAuB,UAY3DhnC,KAAK27C,WACN,cACHmL,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQr/B,2BAE1B,SACHkrC,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQxqC,sBAE1B,SACHq2C,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ3qC,sBAE1B,OACHw2C,EAAW7pD,yBAAag+C,EAAQ7qC,iBAAuB,GAAbpQ,KAAK+jC,KAAY,MAAQkX,EAAQ7qC,QAAU,GAAK,KAC1F02C,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ7qC,oBAE1B,UACH02C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAKgqD,EAAYhM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQhxC,mBAE1B,MACH68C,EAAW7pD,sBAAeg+C,EAAQhxC,SAClC68C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAKiqD,EAAajM,IAC7B6L,EAAW7pD,KAAK+C,KAAK+jC,MAAQ,EAAIijB,EAAM/L,GAAW,IAClD6L,EAAW7pD,KAAK+C,KAAK+jC,MAAQ,gBAAWkX,EAAQx9C,OAAO,QAAQ6G,eAAkB,IACjFwiD,EAAW7pD,KAAK8pD,EAAK9L,EAAQhxC,OAAS,cAEnC,OACH68C,EAAW7pD,uBAAgBg+C,EAAQx9C,OAAO,OAC1CqpD,EAAW7pD,KAAKgqD,EAAYhM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ1tC,mBAE1B,QACHu5C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAKiqD,EAAajM,IAC7B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQjxC,oBAE1B,OACH88C,EAAW7pD,uBAAgBg+C,EAAQ5xC,SACnCy9C,EAAW7pD,cAjDMgN,UACZA,EAAKgY,OAAO+kB,KAAY,QAAU,oBAAsB,GAgD7CmgB,CAAYlM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ5xC,gBAG1BytC,GAAAgQ,QAAAA,EAAkBr4B,QAAQtrB,KAAK,oCApW5B8G,EAAM0xC,EAAO5X,OACjB3mB,EAAQgN,GAAOngB,MAER,QAAT0xC,EAAiB,KACbtyC,EAAO+T,EAAM/T,OAAS7H,KAAK+a,MAAMa,EAAMpT,QAAU,IACvDoT,EAAM/T,KAAK7H,KAAK+a,MAAMlT,EAAO06B,GAAQA,GACrC3mB,EAAMpT,MAAM,GACZoT,EAAMnT,KAAK,GACXmT,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,SAAT+/B,EACHv+B,EAAMnT,OAAS,IACjBmT,EAAMnT,KAAK,GACXmT,EAAMvC,IAAI,EAAG,UAIbuC,EAAMnT,KAAK,GAGbmT,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,QAAT+/B,EACDv+B,EAAM5P,UAAY,GAClB4P,EAAM5P,QAAQ,GACd4P,EAAMvC,IAAI,EAAG,SAGbuC,EAAM5P,QAAQ,GAGlB4P,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAElB,GAAa,OAAT+/B,EAAgB,QAEf5X,QACD,OACA,EACH3mB,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,mBAEvCgN,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,KAE3CgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,WAAT+/B,EAAoB,QAEnB5X,QACD,OACA,EACH3mB,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,mBAEvCgN,EAAMhN,MAAsC,EAAhC5O,KAAK+a,MAAMa,EAAMhN,QAAU,IAE3CgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,QAAT+/B,EAAiB,QAChB5X,QACD,EACH3mB,EAAM9M,QAA2C,GAAnC9O,KAAK+a,MAAMa,EAAM9M,UAAY,mBAE3C8M,EAAM9M,QAA2C,GAAnC9O,KAAK+a,MAAMa,EAAM9M,UAAY,KAE/C8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QACd,GAAa,UAAT+/B,EAAmB,QAEpB5X,QACD,QACA,GACH3mB,EAAM9M,QAA0C,EAAlC9O,KAAK+a,MAAMa,EAAM9M,UAAY,IAC3C8M,EAAM3M,QAAQ,cAEX,EACH2M,EAAM3M,QAA2C,GAAnCjP,KAAK+a,MAAMa,EAAM3M,UAAY,mBAE3C2M,EAAM3M,QAA2C,GAAnCjP,KAAK+a,MAAMa,EAAM3M,UAAY,KAE/C2M,EAAMxB,aAAa,QAEhB,GAAa,UAAT+/B,SAEC5X,QACD,QACA,GACH3mB,EAAM3M,QAA0C,EAAlCjP,KAAK+a,MAAMa,EAAM3M,UAAY,IAC3C2M,EAAMxB,aAAa,cAEhB,EACHwB,EAAMxB,aAAuD,IAA1Cpa,KAAK+a,MAAMa,EAAMxB,eAAiB,oBAErDwB,EAAMxB,aAAsD,IAAzCpa,KAAK+a,MAAMa,EAAMxB,eAAiB,WAGtD,GAAa,eAAT+/B,EAAwB,KACzByL,EAAQrjB,EAAO,EAAIA,EAAO,EAAI,EACpC3mB,EAAMxB,aAAapa,KAAK+a,MAAMa,EAAMxB,eAAiBwrC,GAASA,UAGzDhqC,WC/cX,SAASiqC,GAAYC,EAAKC,QACX,IAARA,IAAiBA,EAAM,QACxBC,EAAWD,EAAIC,YAEdF,GAA2B,oBAAbl6B,cAEfq6B,EAAOr6B,SAASq6B,MAAQr6B,SAASs6B,qBAAqB,QAAQ,GAC9DlzB,EAAQpH,SAASE,cAAc,SACnCkH,EAAM0C,KAAO,WAEI,QAAbswB,GACEC,EAAKE,WACPF,EAAKzf,aAAaxT,EAAOizB,EAAKE,YAKhCF,EAAK9yB,YAAYH,GAGfA,EAAMozB,WACRpzB,EAAMozB,WAAWC,QAAUP,EAE3B9yB,EAAMG,YAAYvH,SAAS06B,eAAeR,KDkrB9C9B,GAASI,OAAS,CAChBe,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZX,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZX,KAAY,smFErtBV0+C,0BAQQnP,EAAMzpB,yDAEXivB,IAAM,CACT4J,WAAY,KACZC,MAAO,GACPC,WAAY,GACZC,WAAY,GACZ/gB,UAAW,CACT6gB,MAAO,GACPC,WAAY,GACZC,WAAY,OAGX9S,MAAQ,CACXqE,MAAO,CACLlZ,MAAO,EACPC,IAAK,EACLglB,YAAa,GAEf2C,QAAS,KAGNlL,eAAiB,CACpBmL,YAAa,CACXC,KAAM,UAERC,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfkC,cAAe,EACf/qD,OAAQ+nD,GAASI,OACjBx7B,OAAAA,GACAq+B,SAAU,QAEPt5B,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAE/BtE,KAAOA,IAGP8P,YAEAlK,WAAWrvB,2DAYPA,GACLA,IAEFmpB,GAAKmG,gBAAgB,CACnB,kBACA,kBACA,gBACA,gBACA,cACA,WACA,SACA,OACCz+C,KAAKmvB,QAASA,GAGjBmpB,GAAKqQ,oBAAoB,CAAC,UAAW3oD,KAAKmvB,QAASA,GAE/C,gBAAiBA,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAEF,WAA/B5iB,GAAOtW,EAAQk5B,cAA4B,SAAUl5B,EAAQk5B,mBAC/Dl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAAYC,OAMpD,WAAYn5B,IACe,mBAAlB/E,GAAO1sB,OAEhB0sB,GAAO1sB,OAAOyxB,EAAQzxB,QAGtB0sB,GAAOjL,KAAKgQ,EAAQzxB,iDAUrB0gD,IAAI4J,WAAa56B,SAASE,cAAc,YACxC8wB,IAAIwK,WAAax7B,SAASE,cAAc,YAExC8wB,IAAI4J,WAAW1f,UAAY,oCAC3B8V,IAAIwK,WAAWtgB,UAAY,iEAQ5BtoC,KAAKo+C,IAAI4J,WAAWzgB,iBACjB6W,IAAI4J,WAAWzgB,WAAWC,YAAYxnC,KAAKo+C,IAAI4J,YAElDhoD,KAAKo+C,IAAIwK,WAAWrhB,iBACjB6W,IAAIwK,WAAWrhB,WAAWC,YAAYxnC,KAAKo+C,IAAIwK,iBAGjDhQ,KAAO,0CAQNvD,EAAQr1C,KAAKq1C,MACb2S,EAAahoD,KAAKo+C,IAAI4J,WACtBY,EAAa5oD,KAAKo+C,IAAIwK,WAGtBh4B,EAA2C,OAAjC5wB,KAAKmvB,QAAQk5B,YAAYC,KAAiBtoD,KAAK44C,KAAKwF,IAAI0E,IAAM9iD,KAAK44C,KAAKwF,IAAIyK,OACtFC,EAAiBd,EAAWzgB,aAAe3W,OAG5Cm4B,yBAGCR,EAAkBvoD,KAAKmvB,QAAQo5B,iBAAqD,SAAlCvoD,KAAKmvB,QAAQk5B,YAAYC,KAC3ErC,EAAkBjmD,KAAKmvB,QAAQ82B,iBAAqD,SAAlCjmD,KAAKmvB,QAAQk5B,YAAYC,KAGjFjT,EAAM2T,iBAAmBT,EAAkBlT,EAAM4T,gBAAkB,EACnE5T,EAAM6T,iBAAmBjD,EAAkB5Q,EAAM8T,gBAAkB,EACnE9T,EAAMxM,OAASwM,EAAM2T,iBAAmB3T,EAAM6T,iBAC9C7T,EAAMzM,MAAQof,EAAWoB,YAEzB/T,EAAMgU,gBAAkBrpD,KAAK44C,KAAKY,SAAS2H,KAAKtY,OAASwM,EAAM6T,kBACzB,OAAjClpD,KAAKmvB,QAAQk5B,YAAYC,KAAgBtoD,KAAK44C,KAAKY,SAASqP,OAAOhgB,OAAS7oC,KAAK44C,KAAKY,SAASsJ,IAAIja,QACxGwM,EAAMiU,eAAiB,EACvBjU,EAAMkU,gBAAkBlU,EAAMgU,gBAAkBhU,EAAM6T,iBACtD7T,EAAMmU,eAAiB,MAGjBC,EAAwBzB,EAAW0B,YACnCC,EAAwBf,EAAWc,mBACzC1B,EAAWzgB,YAAcygB,EAAWzgB,WAAWC,YAAYwgB,GAC3DY,EAAWrhB,YAAcqhB,EAAWrhB,WAAWC,YAAYohB,GAE3DZ,EAAWxzB,MAAMqU,iBAAY7oC,KAAKq1C,MAAMxM,kBAEnC+gB,iBAGDH,EACF74B,EAAOoX,aAAaggB,EAAYyB,GAGhC74B,EAAO+D,YAAYqzB,GAEjB2B,OACG/Q,KAAKwF,IAAIyL,mBAAmB7hB,aAAa4gB,EAAYe,QAGrD/Q,KAAKwF,IAAIyL,mBAAmBl1B,YAAYi0B,GAExC5oD,KAAK8pD,cAAgBhB,+CAQtBT,EAAcroD,KAAKmvB,QAAQk5B,YAAYC,KAGvC9nB,EAAQ8X,GAAKpB,QAAQl3C,KAAK44C,KAAKc,MAAMlZ,MAAO,UAC5CC,EAAM6X,GAAKpB,QAAQl3C,KAAK44C,KAAKc,MAAMjZ,IAAK,UACxCspB,EAAgB/pD,KAAK44C,KAAKN,KAAK2D,QAAQj8C,KAAKq1C,MAAM2U,gBAAkB,IAAMhqD,KAAKmvB,QAAQq5B,eAAejrD,UACxGkoD,EAAcsE,EAAgBzJ,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAK44C,KAAKc,MAAOqQ,GAChItE,GAAezlD,KAAK44C,KAAKN,KAAK2D,OAAO,GAAG1+C,cAElCwmC,EAAO,IAAIyhB,GAAS,IAAI9oD,KAAK8jC,GAAQ,IAAI9jC,KAAK+jC,GAAMglB,EAAazlD,KAAK44C,KAAKQ,YAAap5C,KAAKmvB,SACnG4U,EAAKkmB,UAAUjqD,KAAKmvB,QAAQ/E,QACxBpqB,KAAKmvB,QAAQ1xB,QACfsmC,EAAKmmB,UAAUlqD,KAAKmvB,QAAQ1xB,QAE1BuC,KAAKmvB,QAAQs5B,UACf1kB,EAAKomB,SAASnqD,KAAKmvB,QAAQs5B,eAExB1kB,KAAOA,MAaRkX,EACAjnC,EACA7J,EACAigD,EACAC,EACAC,EAbElM,EAAMp+C,KAAKo+C,IACjBA,EAAIhX,UAAU6gB,MAAQ7J,EAAI6J,MAC1B7J,EAAIhX,UAAU8gB,WAAa9J,EAAI8J,WAC/B9J,EAAIhX,UAAU+gB,WAAa/J,EAAI+J,WAC/B/J,EAAI6J,MAAQ,GACZ7J,EAAI8J,WAAa,GACjB9J,EAAI+J,WAAa,OASboC,EACAC,EAIAliB,EANAM,EAAQ,EAGR6hB,OAAmBhrD,EACnBw5C,EAAQ,MAIZlV,EAAKvD,QACLxsB,EAAO+vB,EAAK2mB,aACZN,EAAQpqD,KAAK44C,KAAKN,KAAKkD,SAASxnC,GACzB+vB,EAAK4mB,WAAa1R,EANb,KAM0B,QACpCA,IAEAoR,EAAUtmB,EAAKsmB,UACf/hB,EAAYvE,EAAK6mB,eAEjB3P,EAAUjnC,EACV7J,EAAIigD,EAEJrmB,EAAK/vB,OACLA,EAAO+vB,EAAK2mB,aAGZH,EAAY3hB,EACZA,GAHAwhB,EAAQpqD,KAAK44C,KAAKN,KAAKkD,SAASxnC,IAGhB7J,EACR45B,EAAK4X,WACN,OAAgB2O,GAAgB,gBAChBA,EAAiB1hB,GAAqB,GAAZ2hB,KAG7CvqD,KAAKmvB,QAAQo5B,iBAAmB+B,EAAe,KAC7C/hB,EAAQvoC,KAAK6qD,kBAAkB1gD,EAAG45B,EAAK+mB,cAAc7P,GAAUoN,EAAa/f,GAChFC,EAAM/T,MAAMoU,gBAAWA,QAGrByhB,GAAWrqD,KAAKmvB,QAAQ82B,iBACtB97C,EAAI,IACkB1K,MAApBgrD,IACFA,EAAmBtgD,GAErBo+B,EAAQvoC,KAAK+qD,kBAAkB5gD,EAAG45B,EAAKinB,cAAc/P,GAAUoN,EAAa/f,IAE9EkiB,EAAOxqD,KAAKirD,kBAAkB9gD,EAAGy+B,EAAOyf,EAAa/f,IAGjDgiB,EACFE,EAAOxqD,KAAKkrD,kBAAkB/gD,EAAGy+B,EAAOyf,EAAa/f,GAGjDkiB,IAEFA,EAAKh2B,MAAMoU,gBAAWuiB,GAASX,EAAKh2B,MAAMoU,OAASA,YA/C/C,MAqDRqQ,GAAkBmS,KAClBzoD,QAAQH,4FAtDA,gBAuDR4oD,IAAoB,GAIpBprD,KAAKmvB,QAAQ82B,gBAAiB,KAC1BoF,EAAWrrD,KAAK44C,KAAKN,KAAK2D,OAAO,GACjCqP,EAAWvnB,EAAKinB,cAAcK,GAC9BE,EAAYD,EAAStuD,QAAUgD,KAAKq1C,MAAMmW,gBAAkB,IAAM,IAEhD/rD,MAApBgrD,GAAiCc,EAAYd,SAC1CM,kBAAkB,EAAGO,EAAUjD,EAAa/f,MAKrDgQ,SAAAA,GAAat4C,KAAKo+C,IAAIhX,WAAW,SAAAxqC,QACxBA,EAAII,QAAQ,KACXynD,EAAO7nD,EAAIo1C,MACbyS,GAAQA,EAAKld,YACfkd,EAAKld,WAAWC,YAAYid,iDAelBt6C,EAAGshD,EAAMpD,EAAa/f,OAElCC,EAAQvoC,KAAKo+C,IAAIhX,UAAU+gB,WAAWhjC,YAErCojB,EAAO,KAEJ1U,EAAUzG,SAAS06B,eAAe,KACxCvf,EAAQnb,SAASE,cAAc,QACzBqH,YAAYd,QACbuqB,IAAI4J,WAAWrzB,YAAY4T,QAE7B6V,IAAI+J,WAAWlrD,KAAKsrC,GACzBA,EAAMmjB,UAAYD,MAGdliD,EAAoB,OAAf8+C,EAAwBroD,KAAKq1C,MAAM6T,iBAAmB,cAC1DyC,OAAOpjB,EAAOp+B,EAAGZ,GAEtBg/B,EAAMD,uCAAkCA,GAGjCC,4CAYSp+B,EAAGshD,EAAMpD,EAAa/f,OAElCC,EAAQvoC,KAAKo+C,IAAIhX,UAAU8gB,WAAW/iC,YAErCojB,EAAO,KAEJ1U,EAAUzG,SAASE,cAAc,QACvCib,EAAQnb,SAASE,cAAc,QACzBqH,YAAYd,QACbuqB,IAAI4J,WAAWrzB,YAAY4T,GAGlCA,EAAMqjB,WAAW,GAAGF,UAAYD,EAChCljB,EAAMD,uCAAkCA,OAGpC/+B,EAAoB,OAAf8+C,EAAwB,EAAIroD,KAAKq1C,MAAM2T,6BAC3C2C,OAAOpjB,EAAOp+B,EAAGZ,QAEjB60C,IAAI8J,WAAWjrD,KAAKsrC,GAClBA,iCAUFA,EAAOp+B,EAAGZ,SAETsiD,EAAa7rD,KAAKmvB,QAAQguB,KAAY,EAALhzC,EAAUA,EACjDo+B,EAAM/T,MAAMkiB,mCAAyBmV,kBAAiBtiD,mDAYtCmrC,EAAM9L,EAAOyf,EAAa/f,SAEtCkiB,EAAOxqD,KAAKo+C,IAAIhX,UAAU6gB,MAAM9iC,QAC/BqlC,IAEHA,EAAOp9B,SAASE,cAAc,YACzB8wB,IAAIwK,WAAWj0B,YAAY61B,SAE7BpM,IAAI6J,MAAMhrD,KAAKutD,OAEdnV,EAAQr1C,KAAKq1C,MAEnBmV,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMqU,iBAAYwM,EAAMgU,0BAEzB9/C,EAAoB,OAAf8+C,EAAwBhT,EAAM6T,iBAAmBlpD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAC7E1+B,EAAIuqC,EAAOW,EAAMiU,eAAiB,cAEjCqC,OAAOnB,EAAMrgD,EAAGZ,GACrBihD,EAAKliB,kCAAwBtoC,KAAKmvB,QAAQguB,IAAO,mBAAqB,sCAA4B7U,GAE3FkiB,4CAYS9V,EAAM9L,EAAOyf,EAAa/f,SAEtCkiB,EAAOxqD,KAAKo+C,IAAIhX,UAAU6gB,MAAM9iC,QAC/BqlC,IAEHA,EAAOp9B,SAASE,cAAc,YACzB8wB,IAAIwK,WAAWj0B,YAAY61B,SAE7BpM,IAAI6J,MAAMhrD,KAAKutD,OAEdnV,EAAQr1C,KAAKq1C,MAEnBmV,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMqU,iBAAYwM,EAAMkU,0BAEzBhgD,EAAoB,OAAf8+C,EAAwB,EAAIroD,KAAK44C,KAAKY,SAASsJ,IAAIja,OACxD1+B,EAAIuqC,EAAOW,EAAMmU,eAAiB,cAEjCmC,OAAOnB,EAAMrgD,EAAGZ,GACrBihD,EAAKliB,kCAAwBtoC,KAAKmvB,QAAQguB,IAAO,mBAAqB,sCAA4B7U,GAE3FkiB,+CAaFxqD,KAAKo+C,IAAI0N,wBACP1N,IAAI0N,iBAAmB1+B,SAASE,cAAc,YAC9C8wB,IAAI0N,iBAAiBxjB,UAAY,sCACjC8V,IAAI0N,iBAAiBt3B,MAAMkL,SAAW,gBAEtC0e,IAAI0N,iBAAiBn3B,YAAYvH,SAAS06B,eAAe,WACzD1J,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI0N,wBAEtCzW,MAAM4T,gBAAkBjpD,KAAKo+C,IAAI0N,iBAAiBC,kBAClD1W,MAAM2U,eAAiBhqD,KAAKo+C,IAAI0N,iBAAiBnL,YAGjD3gD,KAAKo+C,IAAI4N,wBACP5N,IAAI4N,iBAAmB5+B,SAASE,cAAc,YAC9C8wB,IAAI4N,iBAAiB1jB,UAAY,sCACjC8V,IAAI4N,iBAAiBx3B,MAAMkL,SAAW,gBAEtC0e,IAAI4N,iBAAiBr3B,YAAYvH,SAAS06B,eAAe,WACzD1J,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI4N,wBAEtC3W,MAAM8T,gBAAkBnpD,KAAKo+C,IAAI4N,iBAAiBD,kBAClD1W,MAAMmW,eAAiBxrD,KAAKo+C,IAAI4N,iBAAiBrL,mBAtenChI,IA2enByS,IAAoB,sBCrepB3vD,mBAOgB0zB,OAQZryB,EAPAylD,EAAiBpzB,GAAWA,EAAQozB,iBAAkB,EAEtD0J,EAAY98B,GAAWA,EAAQ88B,WAAajgC,OAE5CkgC,EAAmB,GACnBC,EAAS,CAACC,QAAQ,GAAIC,MAAM,IAC5BC,EAAQ,OAIPxvD,EAAI,GAAIA,GAAK,IAAKA,IAAMwvD,EAAM79B,OAAO89B,aAAazvD,IAAM,CAAC0vD,KAAW1vD,EAAI,GAAV,GAAeqoB,OAAO,OAEpFroB,EAAI,GAAIA,GAAK,GAAIA,IAAMwvD,EAAM79B,OAAO89B,aAAazvD,IAAM,CAAC0vD,KAAK1vD,EAAGqoB,OAAO,OAEvEroB,EAAI,EAAIA,GAAK,EAAKA,IAAMwvD,EAAM,GAAKxvD,GAAK,CAAC0vD,KAAK,GAAK1vD,EAAGqoB,OAAO,OAE7DroB,EAAI,EAAIA,GAAK,GAAMA,IAAMwvD,EAAM,IAAMxvD,GAAK,CAAC0vD,KAAK,IAAM1vD,EAAGqoB,OAAO,OAEhEroB,EAAI,EAAIA,GAAK,EAAKA,IAAMwvD,EAAM,MAAQxvD,GAAK,CAAC0vD,KAAK,GAAK1vD,EAAGqoB,OAAO,GAGrEmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAElCmnC,EAAK,KAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,GAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,KAAY,CAACE,KAAK,GAAIrnC,OAAO,GAElCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,WAAO1lB,GAClC6sD,EAAK,IAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,UAAgB,CAACE,KAAK,EAAGrnC,OAAO,GACrCmnC,EAAK,IAAgB,CAACE,KAAK,EAAGrnC,OAAO,GACrCmnC,EAAK,KAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,IAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,OAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,OAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,SAAgB,CAACE,KAAK,GAAIrnC,OAAO,GAEtCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,OAI/BsnC,EAAO,SAASjM,GAAQkM,EAAYlM,EAAM,YAC1CmM,EAAK,SAASnM,GAAQkM,EAAYlM,EAAM,UAGxCkM,EAAc,SAASlM,EAAMtpB,WACKz3B,IAAhC0sD,EAAOj1B,GAAMspB,EAAMoM,SAAwB,SACzCC,EAAQV,EAAOj1B,GAAMspB,EAAMoM,SACtB9vD,EAAI,EAAGA,EAAI+vD,EAAM7vD,OAAQF,UACT2C,IAAnBotD,EAAM/vD,GAAGqoB,OAGc,GAAlB0nC,EAAM/vD,GAAGqoB,OAAmC,GAAlBq7B,EAAMsM,UAGd,GAAlBD,EAAM/vD,GAAGqoB,OAAoC,GAAlBq7B,EAAMsM,WALxCD,EAAM/vD,GAAGD,GAAG2jD,GAUM,GAAlB+B,GACF/B,EAAM+B,0BAMZ2J,EAAiB77B,KAAO,SAASptB,EAAK+C,EAAUkxB,WACjCz3B,IAATy3B,IACFA,EAAO,gBAEUz3B,IAAf6sD,EAAMrpD,SACF,IAAIG,MAAM,oBAAsBH,QAEFxD,IAAlC0sD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,QAC1BL,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQ,IAElCL,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAMvvD,KAAK,CAACJ,GAAGmJ,EAAUmf,MAAMmnC,EAAMrpD,GAAKkiB,SAKpE+mC,EAAiBa,QAAU,SAAS/mD,EAAUkxB,OAIvC,IAAIj0B,UAHIxD,IAATy3B,IACFA,EAAO,WAEOo1B,EACVA,EAAMjvD,eAAe4F,IACvBipD,EAAiB77B,KAAKptB,EAAI+C,EAASkxB,IAMzCg1B,EAAiBc,OAAS,SAASxM,OAC5B,IAAIv9C,KAAOqpD,KACVA,EAAMjvD,eAAe4F,GAAM,IACP,GAAlBu9C,EAAMsM,UAAwC,GAApBR,EAAMrpD,GAAKkiB,OAAiBq7B,EAAMoM,SAAWN,EAAMrpD,GAAKupD,YAC7EvpD,EAEJ,GAAsB,GAAlBu9C,EAAMsM,UAAyC,GAApBR,EAAMrpD,GAAKkiB,OAAkBq7B,EAAMoM,SAAWN,EAAMrpD,GAAKupD,YACpFvpD,EAEJ,GAAIu9C,EAAMoM,SAAWN,EAAMrpD,GAAKupD,MAAe,SAAPvpD,SACpCA,QAIN,wCAITipD,EAAiBe,OAAS,SAAShqD,EAAK+C,EAAUkxB,WACnCz3B,IAATy3B,IACFA,EAAO,gBAEUz3B,IAAf6sD,EAAMrpD,SACF,IAAIG,MAAM,oBAAsBH,WAEvBxD,IAAbuG,EAAwB,KACtBknD,EAAc,GACdL,EAAQV,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,cACtB/sD,IAAVotD,MACG,IAAI/vD,EAAI,EAAGA,EAAI+vD,EAAM7vD,OAAQF,IAC1B+vD,EAAM/vD,GAAGD,IAAMmJ,GAAY6mD,EAAM/vD,GAAGqoB,OAASmnC,EAAMrpD,GAAKkiB,OAC5D+nC,EAAYjwD,KAAKkvD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAM1vD,IAIrDqvD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQU,OAGhCf,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQ,IAKpCN,EAAiB5a,MAAQ,WACvB6a,EAAS,CAACC,QAAQ,GAAIC,MAAM,KAI9BH,EAAiBnH,QAAU,WACzBoH,EAAS,CAACC,QAAQ,GAAIC,MAAM,IAC5BJ,EAAU3I,oBAAoB,UAAWmJ,GAAM,GAC/CR,EAAU3I,oBAAoB,QAASqJ,GAAI,IAI7CV,EAAU3N,iBAAiB,UAAUmO,GAAK,GAC1CR,EAAU3N,iBAAiB,QAAQqO,GAAG,GAG/BT,MCzKX,SAASiB,GAAUlB,gBACZmB,QAAS,OAEThP,IAAM,CACT6N,UAAWA,QAGR7N,IAAIiP,QAAUjgC,SAASE,cAAc,YACrC8wB,IAAIiP,QAAQ/kB,UAAY,mBAExB8V,IAAI6N,UAAUt3B,YAAY30B,KAAKo+C,IAAIiP,cAEnCxJ,OAASG,GAAOhkD,KAAKo+C,IAAIiP,cACzBxJ,OAAOvN,GAAG,MAAOP,UAAKuX,sBAAmBttD,WAG1C2+C,EAAK3+C,KACLkkD,EAAS,CACX,MAAO,YAAa,QACpB,QACA,MAAO,WAAY,UAAW,aAEhCA,QAAAA,GAAe,SAAU1D,GACvB7B,EAAGkF,OAAOvN,GAAGkK,GAAO,SAAUA,GAC5BA,EAAM8D,wBAKNl3B,UAAYA,SAASwrB,YAClB2U,QAAU,SAAU/M,IA4G7B,SAAoB5Y,EAAShX,QACpBgX,GAAS,IACVA,IAAYhX,SACP,EAETgX,EAAUA,EAAQL,kBAEb,GAlHEimB,CAAWhN,EAAM7wB,OAAQs8B,IAC5BtN,EAAG8O,cAGPrgC,SAASwrB,KAAK0F,iBAAiB,QAASt+C,KAAKutD,eAGzB9tD,IAAlBO,KAAK0tD,eACFA,SAAS3I,eAEX2I,SAAWA,UAGXC,YAAc5X,UAAK0X,mBAAgBztD,kwBAIlCmtD,GAAUhxD,WAGlBgxD,GAAUlS,QAAU,KAKpBkS,GAAUhxD,UAAU4oD,QAAU,gBACvB0I,kBAGArP,IAAIiP,QAAQ9lB,WAAWC,YAAYxnC,KAAKo+C,IAAIiP,SAG7CrtD,KAAKutD,SACPngC,SAASwrB,KAAK0K,oBAAoB,QAAStjD,KAAKutD,cAG5B9tD,IAAlBO,KAAK0tD,eACFA,SAAS3I,eAEX2I,SAAW,UAEX7J,OAAOkB,eACPlB,OAAS,MAQhBsJ,GAAUhxD,UAAUyxD,SAAW,iBAEzBT,GAAUlS,SACZkS,GAAUlS,QAAQwS,aAEpBN,GAAUlS,QAAUj7C,UAEfotD,QAAS,OACThP,IAAIiP,QAAQ74B,MAAMC,QAAU,OACjC6jB,GAAKuV,aAAa7tD,KAAKo+C,IAAI6N,UAAW,mBAEjCxL,KAAK,eACLA,KAAK,sBAILiN,iBAAc,MAAO1tD,KAAK2tD,cAOjCR,GAAUhxD,UAAUsxD,WAAa,WAC3BN,GAAUlS,UAAYj7C,OACxBmtD,GAAUlS,QAAU,WAGjBmS,QAAS,OACThP,IAAIiP,QAAQ74B,MAAMC,QAAU,GACjC6jB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI6N,UAAW,mBACpCyB,SAAST,OAAO,MAAOjtD,KAAK2tD,kBAE5BlN,KAAK,eACLA,KAAK,eAQZ0M,GAAUhxD,UAAUmxD,cAAgB,SAAU9M,QAEvCoN,WACLpN,EAAM8D,mBCzIR,IAAMyJ,GAAK,CACT9S,QAAS,UACTv6B,KAAM,OACNstC,eAAgB,mBAMZliC,GAAK,CACTmvB,QAAS,UACTv6B,KAAM,QACNstC,eAAgB,yBAMZC,GAAK,CACThT,QAAS,UACTv6B,KAAM,OACNstC,eAAgB,wBAMZE,GAAK,CACTjT,QAAS,WACTv6B,KAAM,OACNstC,eAAgB,kBAKZG,GAAK,CACTlT,QAAS,SACTv6B,KAAM,QACNstC,eAAgB,wBAOZI,GAAK,CACTnT,QAAS,YACTv6B,KAAM,OACNstC,eAAgB,sBAKZK,GAAK,CACTpT,QAAS,WACTv6B,KAAM,MACNstC,eAAgB,mBAKZM,GAAK,CACTrT,QAAS,UACTv6B,KAAM,QACNstC,eAAgB,qBAKZO,GAAK,CACTtT,QAAS,WACTv6B,KAAM,OACNstC,eAAgB,gBAKZQ,GAAK,CACTvT,QAAS,KACTv6B,KAAM,KACNstC,eAAgB,cAIZp7C,GAAU,CACdm7C,GAAAA,GACAU,MAjFYV,GAkFZW,MAjFYX,GAkFZjiC,GAAAA,GACA6iC,MA3EY7iC,GA4EZ8iC,MA3EY9iC,GA4EZmiC,GAAAA,GACAY,MArEYZ,GAsEZa,MArEYb,GAsEZC,GAAAA,GACAa,MA/DYb,GAgEZC,GAAAA,GACAa,MAzDYb,GA0DZc,MAzDYd,GA0DZe,MAzDYf,GA0DZC,GAAAA,GACAe,MAnDYf,GAoDZC,GAAAA,GACAe,MA7CYf,GA8CZC,GAAAA,GACAe,MAvCYf,GAwCZC,GAAAA,GACAe,MAjCYf,GAkCZC,GAAAA,GACAe,MA3BYf,yiCC/ERgB,0BAUQ5W,EAAMzpB,oDAEXypB,KAAOA,IAGPsE,eAAiB,CACpB9yB,OAAAA,GACAxX,QAAAA,GACAlV,OAAQ,KACR01B,QAAI3zB,EACJgwD,WAAOhwD,KAEJ0vB,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/BsB,WAAWrvB,KACXA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS8qC,EAAKvuB,QAAQvc,aACvD88C,EAAiBhS,EAAKR,eAAetqC,QAAQ8qC,EAAKR,eAAex/C,uBAC3DggD,EAAKvuB,QAAQvc,kBAAiB,SAAAlV,KACnCyxB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,OAIrByxB,GAA2B,MAAhBA,EAAQzO,OAChBivC,WAAaxgC,EAAQzO,OAErBivC,WAAa,IAAIjzD,OAGnBkzD,YAAc,KAGdlH,kEAUIv5B,GACLA,GAEFmpB,GAAKmG,gBAAgB,CAAC,SAAU,SAAU,UAAW,KAAM,QAAS,MAAO,QAASz+C,KAAKmvB,QAASA,+CAS9F0gC,EAAMziC,SAASE,cAAc,OACnCuiC,EAAI,eAAiB7vD,KACrB6vD,EAAIvnB,oCAA+BtoC,KAAKmvB,QAAQiE,IAAM,IACtDy8B,EAAIr7B,MAAMkL,SAAW,WACrBmwB,EAAIr7B,MAAMsuB,IAAM,MAChB+M,EAAIr7B,MAAMqU,OAAS,YACdgnB,IAAMA,MAELC,EAAO1iC,SAASE,cAAc,gBAe3ByiC,EAAc18C,QAChBulC,KAAKc,MAAMuE,cAAc5qC,GAfhCy8C,EAAKt7B,MAAMkL,SAAW,WACtBowB,EAAKt7B,MAAMsuB,IAAM,MACd9iD,KAAKmvB,QAAQguB,IACd2S,EAAKt7B,MAAMmgB,MAAQ,QAElBmb,EAAKt7B,MAAMkgB,KAAO,QAErBob,EAAKt7B,MAAMqU,OAAS,OACpBinB,EAAKt7B,MAAMoU,MAAQ,OAUfknB,EAAKxR,kBAEPwR,EAAKxR,iBAAiB,aAAcvI,GAAAga,QAAAA,EAAkB/vD,OAAO,GAE7D8vD,EAAKxR,iBAAiB,iBAAkBvI,GAAAga,QAAAA,EAAkB/vD,OAAO,IAGjE8vD,EAAKE,YAAY,eAAgBja,GAAAga,QAAAA,EAAkB/vD,OAGrD6vD,EAAIl7B,YAAYm7B,QAEXjM,OAAS,IAAIG,GAAO8L,QACpBjM,OAAOvN,GAAG,WAAYP,UAAK+H,qBAAkB99C,YAC7C6jD,OAAOvN,GAAG,UAAYP,UAAKgI,gBAAa/9C,YACxC6jD,OAAOvN,GAAG,SAAYP,UAAKiI,mBAAgBh+C,YAC3C6jD,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOiM,uDAOtDC,YAEArM,OAAOkB,eACPlB,OAAS,UAETjL,KAAO,0CAQNhoB,EAAS5wB,KAAK44C,KAAKwF,IAAIyL,mBACzB7pD,KAAK6vD,IAAItoB,YAAc3W,IAErB5wB,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,KAEvCj/B,EAAO+D,YAAY30B,KAAK6vD,UAGpB1lD,EAAInK,KAAK44C,KAAKN,KAAKkD,SAASx7C,KAAK2vD,YAEnCjyD,EAASsC,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QAC1CA,IACEsC,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhBzyD,EAASsC,KAAKmvB,QAAQvc,QAAb,UAGP68C,EAAQzvD,KAAKmvB,QAAQsgC,WAEXhwD,IAAVgwD,EAEFA,GADAA,iBAAW/xD,EAAOgjB,mBAAS1gB,KAAKmvB,QAAQ/E,OAAOpqB,KAAK2vD,YAAYlyD,OAAO,iCACzD+oB,OAAO,GAAGjB,cAAgBkqC,EAAMW,UAAU,GAC9B,mBAAVX,IAChBA,EAAQA,EAAMpzD,KAAK2D,KAAMA,KAAK2vD,yBAG3BxgC,QAAQguB,IAAMn9C,KAAK6vD,IAAIr7B,MAAMmgB,gBAAWxqC,QAAQnK,KAAK6vD,IAAIr7B,MAAMkgB,eAAUvqC,aACzE0lD,IAAIJ,MAAQA,GAEV,iCAQHzvD,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,2CAQ3BnvC,QACPivC,WAAarX,GAAKpB,QAAQx2B,EAAM,aAChC2vC,wDAQE,IAAI3zD,KAAKsD,KAAK2vD,WAAWpyD,mDAQlBkyD,EAAOa,WACfC,EAASnjC,SAASE,cAAc,QACtCijC,EAAOjoB,mCACPioB,EAAO7E,UAAY+D,EACnBc,EAAO/7B,MAAMkL,SAAW,WAEpB4wB,KACFC,EAAOC,aAAa,kBAAmB,QACvCD,EAAOjS,iBAAiB,eAAe,WACrCiS,EAAOE,WAETF,EAAOjS,iBAAiB,QAASvI,UAAK2a,wBAAqB1wD,OAE3DuwD,EAAOd,MAAQA,EACfc,EAAOjS,iBAAiB,OAAQvI,cAAUyK,GACpCxgD,KAAKyvD,OAASjP,EAAM7wB,OAAO+7B,iBACxBiF,iBAAiBnQ,QACjBiP,MAAQjP,EAAM7wB,OAAO+7B,oBAEvB1rD,aAGJ6vD,IAAIl7B,YAAY47B,0CAORd,QACRtgC,QAAQsgC,MAAQA,uCAQVjP,QACNoP,YAAYzP,UAAW,OACvByP,YAAYD,WAAa3vD,KAAK2vD,WAEnCnP,EAAM8D,kDAQA9D,MACDxgD,KAAK4vD,YAAYzP,cAElBoB,EAASvhD,KAAKmvB,QAAQguB,KAAQ,EAAKqD,EAAMe,OAASf,EAAMe,OAEtDp3C,EAAInK,KAAK44C,KAAKN,KAAKkD,SAASx7C,KAAK4vD,YAAYD,YAAcpO,EAC3D7gC,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAE7BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBC,EAAO9wD,KAAKmvB,QAAQ2hC,KAEpBC,EAAcD,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,OAEhDswC,cAAcD,QAGdnY,KAAKiF,QAAQ4C,KAAK,aAAc,CACnCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjB1S,KAAM,IAAIhkB,KAAKsD,KAAK2vD,WAAWpyD,WAC/BijD,MAAAA,IAGFA,EAAM8D,sDAQG9D,GACJxgD,KAAK4vD,YAAYzP,gBAGjBvH,KAAKiF,QAAQ4C,KAAK,cAAe,CACpCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjB1S,KAAM,IAAIhkB,KAAKsD,KAAK2vD,WAAWpyD,WAC/BijD,MAAAA,IAGFA,EAAM8D,2DAQQ9D,QACT5H,KAAKiF,QAAQ4C,KAAK,eAAgB,CACrCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjBq8B,MAAOjP,EAAM7wB,OAAO+7B,UACpBlL,MAAAA,IAGFA,EAAM8D,2DAQS9D,QACV5H,KAAKiF,QAAQ4C,KAAK,gBAAiB,CACtCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjBq8B,MAAOjP,EAAM7wB,OAAO+7B,UACpBlL,MAAAA,IAGFA,EAAM8D,iEASoB9D,WACtB7wB,EAAS6wB,EAAM7wB,OACZA,GAAQ,IACTA,EAAOtyB,eAAe,sBACjBsyB,EAAO,eAEhBA,EAASA,EAAO4X,kBAGX,YA9UcoR,klWCWnB8C,iFAQIwQ,yBACD7N,IAAM,QAENA,IAAI6N,UAAYA,OAChB7N,IAAI6N,UAAUz3B,MAAMkL,SAAW,gBAE/B0e,IAAI+C,KAAuB/zB,SAASE,cAAc,YAClD8wB,IAAIwK,WAAuBx7B,SAASE,cAAc,YAClD8wB,IAAIyL,mBAAuBz8B,SAASE,cAAc,YAClD8wB,IAAI6S,qBAAuB7jC,SAASE,cAAc,YAClD8wB,IAAI3E,gBAAuBrsB,SAASE,cAAc,YAClD8wB,IAAI8S,cAAuB9jC,SAASE,cAAc,YAClD8wB,IAAI+S,eAAuB/jC,SAASE,cAAc,YAClD8wB,IAAIY,OAAuB5xB,SAASE,cAAc,YAClD8wB,IAAI1J,KAAuBtnB,SAASE,cAAc,YAClD8wB,IAAIzJ,MAAuBvnB,SAASE,cAAc,YAClD8wB,IAAI0E,IAAuB11B,SAASE,cAAc,YAClD8wB,IAAIyK,OAAuBz7B,SAASE,cAAc,YAClD8wB,IAAIgT,UAAuBhkC,SAASE,cAAc,YAClD8wB,IAAIiT,aAAuBjkC,SAASE,cAAc,YAClD8wB,IAAIkT,cAAuBlkC,SAASE,cAAc,YAClD8wB,IAAImT,iBAAuBnkC,SAASE,cAAc,YAClD8wB,IAAIoT,eAAuBpkC,SAASE,cAAc,YAClD8wB,IAAIqT,kBAAuBrkC,SAASE,cAAc,YAClD8wB,IAAIC,eAAuBjxB,SAASE,cAAc,YAClD8wB,IAAIsT,cAAuBtkC,SAASE,cAAc,YAElD8wB,IAAI+C,KAAK7Y,UAA4B,oBACrC8V,IAAIwK,WAAWtgB,UAAsB,gCACrC8V,IAAIyL,mBAAmBvhB,UAAc,6CACrC8V,IAAI6S,qBAAqB3oB,UAAY,+CACrC8V,IAAI3E,gBAAgBnR,UAAiB,4BACrC8V,IAAI8S,cAAc5oB,UAAmB,0BACrC8V,IAAI+S,eAAe7oB,UAAkB,2BACrC8V,IAAI0E,IAAIxa,UAA6B,yBACrC8V,IAAIyK,OAAOvgB,UAA0B,4BACrC8V,IAAI1J,KAAKpM,UAA4B,mBACrC8V,IAAIY,OAAO1W,UAA0B,mBACrC8V,IAAIzJ,MAAMrM,UAA2B,mBACrC8V,IAAIgT,UAAU9oB,UAAuB,0BACrC8V,IAAIiT,aAAa/oB,UAAoB,6BACrC8V,IAAIkT,cAAchpB,UAAmB,0BACrC8V,IAAImT,iBAAiBjpB,UAAgB,6BACrC8V,IAAIoT,eAAelpB,UAAkB,0BACrC8V,IAAIqT,kBAAkBnpB,UAAe,6BACrC8V,IAAIC,eAAe/V,UAAkB,4BACrC8V,IAAIsT,cAAcppB,UAAmB,0BAErC8V,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIwK,iBAC9BxK,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIyL,yBAC9BzL,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI6S,2BAC9B7S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI3E,sBAC9B2E,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI8S,oBAC9B9S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI+S,qBAC9B/S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI0E,UAC9B1E,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIyK,aAC9BzK,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIC,qBAE9BD,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIY,aACzCZ,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAI1J,WACvC0J,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIzJ,YACxCyJ,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIgT,gBACzChT,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIiT,mBACzCjT,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAIkT,oBACvClT,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAImT,uBACvCnT,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIoT,qBACxCpT,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIqT,wBAGxCpc,MAAQ,CACX8L,KAAM,GACNyH,WAAY,GACZnP,gBAAiB,GACjByX,cAAe,GACfC,eAAgB,GAChBnS,OAAQ,GACRtK,KAAM,GACNC,MAAO,GACPmO,IAAK,GACL+F,OAAQ,GACR8I,OAAQ,GACRC,UAAW,EACXC,aAAc,QAGXvb,GAAG,eAAe,YACQ,IAAzBoH,EAAKoU,iBACPpU,EAAKqU,kBAGJzb,GAAG,gBAAgB,WACjBoH,EAAKsU,yBACRtU,EAAKsU,wBAAyB,WAG7B1b,GAAG,QAAeP,UAAKmI,iBAAcl+C,YACrCs2C,GAAG,UAAmBP,UAAKgI,gBAAa/9C,WAEvC2+C,EAAK3+C,UACNiyD,YAAclc,UAAKgc,gBAAa/xD,WAChC+xD,QAAUzZ,GAAK4Z,SAASlyD,KAAKiyD,kBAE7B3b,GAAG,WAAW,SAAAxc,GACb6kB,EAAGwT,SAAWxT,EAAGwT,QAAQC,qBAAuBt4B,GAAkC,GAApBA,EAAWu4B,MAC3E1T,EAAGoT,UAEHpT,EAAGsT,sBAMFpO,OAAS,IAAIG,GAAOhkD,KAAKo+C,IAAI+C,UAC5BmR,EAAkBtyD,KAAK6jD,OAAO95C,IAAI,SAAS/F,IAAI,CAACkiD,QAAQ,IAC9DoM,GRpFG,SAA0CA,GAG/CA,EAAgBC,eAAiB,iBAExB,CAJkB,UQmFNC,CAA2CF,QACzDzO,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOiM,qBACtDxM,UAAY,OR3HMI,EAAQ79C,EQ6HzBk+C,EAAS,CACb,MAAO,YAAa,QACpB,QACA,MAAO,WAAY,UAAW,mBA+BvB6L,EAAavP,GAMhBxgD,KAAKyyD,iBACFhS,KAAK,aAAcD,OAItBe,EAAS,EACTC,EAAS,KAGR,WAAiBhB,IAAUgB,GAAyB,EAAhBhB,EAAMqB,QAC1C,eAAiBrB,IAAUgB,EAAShB,EAAMoB,YAC1C,gBAAiBpB,IAAUgB,EAAShB,EAAMkS,aAC1C,gBAAiBlS,IAAUe,GAA8B,EAArBf,EAAMmS,aAG1C,SAAUnS,GAASA,EAAM8H,OAAS9H,EAAMoS,kBAC3CrR,GAAmB,EAAVC,EACTA,EAAS,GAIN,WAAYhB,IACfgB,GAAyB,EAAhBhB,EAAMgB,QAEZ,WAAYhB,IACfe,EAASf,EAAMe,QAIbf,EAAMqS,YACkB,IAApBrS,EAAMqS,WACRtR,GAlCc,GAmCdC,GAnCc,KAqCdD,GArCc,GAsCdC,GArCc,MAyChBxhD,KAAKmvB,QAAQ2jC,gBACV9yD,KAAKmvB,QAAQ2yB,SAAWtB,EAAMxgD,KAAKmvB,QAAQ2yB,SAAU,eAEtD9hD,KAAKmvB,QAAQ2yB,SAAWtB,EAAMxgD,KAAKmvB,QAAQ2yB,SAAU,UAGtD9hD,KAAKmvB,QAAQ4jC,gBAAmB/yD,KAAKmvB,QAAQ6jC,oBAE9ChzD,KAAKmvB,QAAQ4jC,gBAAkBvxD,KAAKc,IAAIk/C,IAAWhgD,KAAKc,IAAIi/C,GAAS,KACjEtG,EAAUj7C,KAAKq1C,MAAMuc,UACrBqB,EAAWhY,EAAUuG,KAEvBxhD,KAAKyyD,WACczyD,KAAKkzD,cAAcD,KAEnBhY,SACd8W,eACAtR,KAAK,SAAUD,GAIpBA,EAAM+B,uBAGL,GAAIviD,KAAKmvB,QAAQ6jC,iBAAkB,KAIlC71C,GAHQ3b,KAAKc,IAAIi/C,IAAW//C,KAAKc,IAAIk/C,GAAUD,EAASC,GAGxC,KAAQxhD,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,OAAS,GAE7DogB,EAAW5gD,KAAK05C,MAAMlZ,MAAQrjB,EAC9B0jC,EAAS7gD,KAAK05C,MAAMjZ,IAAMtjB,EAE1BgS,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9G,MAAMgF,SAASkC,EAAUC,EAAQ1xB,GAEtCqxB,EAAM+B,qBA7GV2B,QAAAA,GAAe,SAAAhtB,OACPi8B,EAAW,SAAA3S,GACX7B,EAAG8T,YACL9T,EAAG8B,KAAKvpB,EAAMspB,IAGlB7B,EAAGkF,OAAOvN,GAAGpf,EAAMi8B,GACnBxU,EAAG8E,UAAUvsB,GAAQi8B,KAIvBX,GAAmBxyD,KAAK6jD,QAAQ,SAAArD,GAC9B7B,EAAG8B,KAAK,QAASD,MRnJIqD,EQuJF7jD,KAAK6jD,QRvJK79C,EQuJG,SAAAw6C,GAChC7B,EAAG8B,KAAK,UAAWD,KRvJd+E,aAAe,SAAU/E,GAC5BA,EAAM4S,SACRptD,EAASw6C,IAINqD,EAAOvN,GAAG,eAAgBtwC,EAASu/C,kBQkPlC8N,EAAY,YAAajmC,SAASE,cAAc,OAAS,aACrC7tB,IAA1B2tB,SAASkmC,aAA6B,kBAIjClV,IAAI3E,gBAAgB6E,iBAAmB,iBAAoB,wBAYvDiV,EAAkB/S,MACpB7B,EAAGxvB,QAAQ4jC,kBAEZpU,EAAG6U,2BACL7U,EAAG6U,4BAA6B,UAIlChT,EAAM+B,iBACF5D,EAAG8T,WAAY,KACXQ,GAAYzS,EAAM7wB,OAAOiiC,UAC/BjT,EAAGuU,cAAcD,GACjBtU,EAAGoT,UACHpT,EAAG8B,KAAK,aAAcD,SAxBrBpC,IAAI0E,IAAIxE,sBACRF,IAAIyK,OAAOvK,sBACXF,IAAI3E,gBAAgB6E,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QACzEo+C,IAAI0E,IAAIxE,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QAC7Do+C,IAAIyK,OAAOvK,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QAwBhEo+C,IAAI1J,KAAKnN,WAAW+W,iBAAiB,SAAUvI,GAAAwd,QAAAA,EAAuBvzD,YACtEo+C,IAAIzJ,MAAMpN,WAAW+W,iBAAiB,SAAUvI,GAAAwd,QAAAA,EAAuBvzD,WAExEyzD,GAAsB,WAOjBC,EAAelT,YAClBA,EAAM+B,iBACR5D,EAAG8B,KAAK,WAAY9B,EAAGgV,mBAAmBnT,IAC1CA,EAAM+B,kBAIFqR,KAAApT,EAAM7wB,OAAO2Y,kBAAkB,aAAe,IAGhDmrB,SAEJjT,EAAMqT,aAAaC,WAAa,OAChCL,GAAsB,GACf,WAQAM,EAAWvT,GAEfA,EAAM+B,gBAAkB/B,EAAM+B,iBAC9B/B,EAAM8D,iBAAmB9D,EAAM8D,0BAG5B0P,EAAWxuB,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,aAChDD,IAAaA,EAASngC,QAAS,OACpC,MAAOoS,UACA,SAGTwtB,GAAsB,EACtBjT,EAAMxB,OAAS,CACb70C,EAAGq2C,EAAM2B,QACT54C,EAAGi3C,EAAM4B,SAGa,SAApB4R,EAASrkC,OACXgvB,EAAGwT,QAAQ+B,WAAW1T,GAEtB7B,EAAGwT,QAAQgC,oBAAoB3T,GAEjC7B,EAAG8B,KAAK,OAAQ9B,EAAGgV,mBAAmBnT,KAC/B,UAGJpC,IAAIY,OAAOV,iBAAiB,WAAYvI,GAAA2d,QAAAA,EAAoB1zD,OAAO,QACnEo+C,IAAIY,OAAOV,iBAAiB,OAAQvI,GAAAge,QAAAA,EAAgB/zD,OAAO,QAE3Do0D,YAAc,QAGdzW,MAAQ,QAER0W,YAAc,OACdvC,iBAAkB,OAClBE,wBAAyB,GAGzB/F,EAAW,MAAM,IAAI7oD,MAAM,yBAChC6oD,EAAUt3B,YAAY30B,KAAKo+C,IAAI+C,MAC/B8K,EAAUt3B,YAAY30B,KAAKo+C,IAAIsT,kDA4BtBviC,YACLA,EAAS,IAQXmpB,GAAKmG,gBANU,CACb,QAAS,SAAU,YAAa,YAAa,aAC7C,QAAS,MAAO,aAAc,iBAAkB,cAChD,SAAU,UAAW,SAAU,aAAc,MAAO,UACpD,mBAAoB,iBAAkB,uBAEXz+C,KAAKmvB,QAASA,QACtCivB,IAAIC,eAAe7pB,MAAMyqB,WAAa,SAEvCj/C,KAAKmvB,QAAQguB,WACViB,IAAI6N,UAAUz3B,MAAMlW,UAAY,WAChC8/B,IAAIyL,mBAAmBvhB,UAAY,6CAGtCtoC,KAAKmvB,QAAQ4jC,iBACX/yD,KAAKmvB,QAAQguB,SACViB,IAAI+S,eAAe7oB,UAAY,+CAE/B8V,IAAI8S,cAAc5oB,UAAY,0CAIC,WAApC7C,GAAOzlC,KAAKmvB,QAAQk5B,oBACjBl5B,QAAQk5B,YAAc,CAACtQ,UAAKt4C,EAAU6oD,UAAK7oD,IAE9C,gBAAiB0vB,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAc,CACzBtQ,KAAM5oB,EAAQk5B,YACdC,KAAMn5B,EAAQk5B,aAGsB,WAA/B5iB,GAAOtW,EAAQk5B,eAClB,SAAUl5B,EAAQk5B,mBACfl5B,QAAQk5B,YAAYtQ,KAAO5oB,EAAQk5B,YAAYtQ,MAElD,SAAU5oB,EAAQk5B,mBACfl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAAYC,QAKpB,SAAlCtoD,KAAKmvB,QAAQk5B,YAAYC,UACtBtoD,KAAKs0D,UAAW,KACbA,EAAYt0D,KAAKs0D,UAAY,IAAIvM,GAAS/nD,KAAK44C,MACrD0b,EAAU9V,WAAa,SAAArvB,OACf20B,EAAW30B,EAAUmpB,GAAKh7C,OAAO,GAAI6xB,GAAW,GACtD20B,EAASuE,YAAc,MACvBN,GAAS5rD,UAAUqiD,WAAWniD,KAAKi4D,EAAWxQ,SAE3CyQ,WAAWt3D,KAAKq3D,YAInBt0D,KAAKs0D,UAAW,SACZ1vC,EAAQgvC,UAAKW,mBAAmBv0D,KAAKs0D,eAC5B,IAAX1vC,YACG2vC,mBAAkB3vC,EAAO,QAE3B0vC,UAAUvP,eACVuP,UAAY,KAKY,mBAAtBnlC,EAAQqlC,aACjBrlC,EAAQqlC,WAAa,CACjBC,SAAUtlC,EAAQqlC,aAIpB,gBAAiBx0D,KAAKmvB,SACxBmxB,GAA8BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aAGzE,eAAgBjqB,IACdA,EAAQulC,WACL10D,KAAK20D,iBACHA,UAAY,IAAIxH,GAAUntD,KAAKo+C,IAAI+C,OAItCnhD,KAAK20D,iBACFA,UAAU5P,iBACR/kD,KAAK20D,iBAMbC,+BAIFL,oBAAmB,SAAAM,UAAaA,EAAUrW,WAAWrvB,MAGtD,cAAeA,EAAS,OACrBnvB,KAAK80D,oBACHA,aAAe90D,KAAK+0D,4BAGtBD,aAAatW,WAAWrvB,EAAQ6lC,eAG/BC,EAAiB3c,GAAKuN,WAAW,GAAI7lD,KAAKmvB,mBAC3ColC,oBAAmB,SAAAM,GACtBvc,GAAKuN,WAAWoP,EAAgBJ,EAAU1lC,iBAEvC2lC,aAAaI,iBAAiB,CAAChpC,OAAQ+oC,SAGzClD,oDAQG/xD,KAAK20D,WAAa30D,KAAK20D,UAAUvH,mDA8BpC,IAAM5M,UAtBN2U,SAAS,WACTC,UAAU,WAGV7e,WAGA8e,kBAGDr1D,KAAKo+C,IAAI+C,KAAK5Z,iBACX6W,IAAI+C,KAAK5Z,WAAWC,YAAYxnC,KAAKo+C,IAAI+C,WAE3C/C,IAAM,KAGPp+C,KAAK20D,iBACFA,UAAU5P,iBACR/kD,KAAK20D,WAIM30D,KAAKyjD,UACnBzjD,KAAKyjD,UAAUpmD,eAAemjD,WACzBxgD,KAAKyjD,UAAUjD,QAGrBiD,UAAY,UACZI,QAAU7jD,KAAK6jD,OAAOkB,eACtBlB,OAAS,eAGT0Q,oBAAmB,SAAAM,UAAaA,EAAU9P,kBAE1CnM,KAAO,2CAQAl4B,EAAM0S,SACZghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAazhC,IAAOyhC,EAAU1lC,QAAQiE,SAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,KAGjEghC,EAAYp3D,OAAS,GACvBo3D,EAAY,GAAGpD,cAActwC,yCASnB0S,SACNghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,YAE9DghC,EAAY,GAAGkB,4DASJ7F,EAAOr8B,EAAIk9B,SACvB8D,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,KAEjEghC,EAAYp3D,OAAS,GACvBo3D,EAAY,GAAGmB,gBAAgB9F,EAAOa,8CAUvBb,EAAOr8B,SAClBghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,QAEjEghC,EAAYp3D,OAAS,SAChBo3D,EAAY,GAAGoB,eAAe/F,8CAUtBjP,SACV,CAAEA,MAAAA,yCAaG9/B,EAAM0S,SACZqiC,OAAqBh2D,IAATihB,EACZ43B,GAAKpB,QAAQx2B,EAAM,QACnB,IAAIhkB,QAEKg5D,UAAKtB,qBAAiB,SAAAzE,UAAcA,EAAWxgC,QAAQiE,KAAOA,WAErE,IAAIhwB,sCAA+B28C,GAAe3sB,2BAGpDu8B,EAAa,IAAIH,GAAWxvD,KAAK44C,KAAMN,GAAKh7C,OAAO,GAAI0C,KAAKmvB,QAAS,CACzEzO,KAAO+0C,EACPriC,GAAAA,EACA09B,KAAM9wD,KAAKmyD,QAAQhjC,QAAQ2hC,oBAGxBsD,YAAYn3D,KAAK0yD,QACjB4E,WAAWt3D,KAAK0yD,QAChBoC,UAEE3+B,2CAQQA,gBACTghC,EAActd,UAAKsd,qBAAmB,SAAAvE,UAAOA,EAAI1gC,QAAQiE,KAAOA,QAE3C,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,QAGrEghC,QAAAA,GAAoB,SAAAzE,oBAClBgG,EAAKvB,oBAAmBR,KAAA+B,EAAKvB,oBAAoBzE,GAAa,QAC9DgG,EAAKpB,mBAAkBX,KAAA+B,EAAKpB,mBAAmB5E,GAAa,GAC5DA,EAAW5K,8DASN/kD,KAAKmyD,SAAWnyD,KAAKmyD,QAAQyD,mBAAqB,qDAQlD51D,KAAKmyD,SAAWnyD,KAAKmyD,QAAQ0D,oBAAsB,+BAcxD1mC,EAASnpB,OACL0zC,EAAQ15C,KAAK81D,kBAGD,OAAdpc,EAAMt3C,KAA8B,OAAds3C,EAAMp0C,SAK1Bu5C,EAAWnF,EAAMp0C,IAAMo0C,EAAMt3C,IAC7BA,EAAM,IAAI1F,KAAKg9C,EAAMt3C,IAAI7E,UAAuB,IAAXshD,GACrCv5C,EAAM,IAAI5I,KAAKg9C,EAAMp0C,IAAI/H,UAAuB,IAAXshD,GACrCE,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eACpErF,MAAMgF,SAASt8C,EAAKkD,EAAK,CAAEy5C,UAAAA,GAAa/4C,iDAUvC,IAAI5C,MAAM,gFAyBRo9B,EAAOC,EAAKtR,EAASnpB,OAKzB+4C,EACArF,EALuB,mBAAhB59C,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,IAIY,GAApBrzB,UAAUkB,QAEZ+hD,OAAiCt/C,KADjCi6C,EAAQ59C,UAAU,IACCijD,WAA2BrF,EAAMqF,eAC/CrF,MAAMgF,SAAShF,EAAMlZ,MAAOkZ,EAAMjZ,IAAK,CAAEse,UAAAA,KAEnB,GAApBjjD,UAAUkB,QAAsC,mBAAhBlB,UAAU,IACjD49C,EAAQ59C,UAAU,GAClBkK,EAAWlK,UAAU,GACrBijD,OAAiCt/C,IAApBi6C,EAAMqF,WAA2BrF,EAAMqF,eAC/CrF,MAAMgF,SAAShF,EAAMlZ,MAAOkZ,EAAMjZ,IAAK,CAAEse,UAAAA,GAAa/4C,KAG3D+4C,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eAC9DrF,MAAMgF,SAASle,EAAOC,EAAK,CAAEse,UAAAA,GAAa/4C,mCAgB5C0a,EAAMyO,EAASnpB,GACO,mBAAhBlK,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAEN0vB,EAAW7+C,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,MACvCzgC,EAAIu4C,GAAKpB,QAAQx2B,EAAM,QAAQnjB,UAE/BijC,EAAQzgC,EAAI8+C,EAAW,EACvBpe,EAAM1gC,EAAI8+C,EAAW,EACrBE,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eAEpErF,MAAMgF,SAASle,EAAOC,EAAK,CAAEse,UAAAA,GAAa/4C,2CAQzC0zC,EAAQ15C,KAAK05C,MAAMqc,iBAClB,CACLv1B,MAAO,IAAI9jC,KAAKg9C,EAAMlZ,OACtBC,IAAK,IAAI/jC,KAAKg9C,EAAMjZ,qCAgBjBu1B,EAAY7mC,EAASnpB,SACrBgwD,GAAcA,EAAa,GAAKA,EAAa,IACvB,mBAAhBl6D,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAENuqB,EAAQ15C,KAAKi2D,YACbz1B,EAAQkZ,EAAMlZ,MAAMjjC,UACpBkjC,EAAMiZ,EAAMjZ,IAAIljC,UAChBshD,EAAWpe,EAAMD,EACjB01B,EAAcrX,GAAY,EAAImX,GAC9BG,GAAYtX,EAAWqX,GAAe,EACtCtV,EAAWpgB,EAAQ21B,EACnBtV,EAASpgB,EAAM01B,OAEhBC,UAAUxV,EAAUC,EAAQ1xB,EAASnpB,oCAepCgwD,EAAY7mC,EAASnpB,SACtBgwD,GAAcA,EAAa,GAAKA,EAAa,IACvB,mBAAhBl6D,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAENuqB,EAAQ15C,KAAKi2D,YACbz1B,EAAQkZ,EAAMlZ,MAAMjjC,UACpBkjC,EAAMiZ,EAAMjZ,IAAIljC,UAChBshD,EAAWpe,EAAMD,EACjBogB,EAAWpgB,EAAQqe,EAAWmX,EAAa,EAC3CnV,EAASpgB,EAAMoe,EAAWmX,EAAa,OAExCI,UAAUxV,EAAUC,EAAQ1xB,EAASnpB,0CASrC+rD,uDASAsC,kBACCjW,EAAMp+C,KAAKo+C,OAEZA,GAAQA,EAAI6N,WAAqC,GAAxB7N,EAAI+C,KAAKiI,iBAEnCvQ,GAAU,EACR1pB,EAAUnvB,KAAKmvB,QACfkmB,EAAQr1C,KAAKq1C,MAEnBiL,GAA2BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aAG7C,OAAvBjqB,EAAQk5B,aACV/P,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,gBAG/B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,WAC/B7I,GAAKuV,aAAazP,EAAI+C,KAAM,eAG1BhyB,EAAQguB,KACV7E,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,aAE/B7I,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,YAIjC/C,EAAI+C,KAAK3sB,MAAM6hC,UAAY/d,GAAKge,OAAOC,OAAOpnC,EAAQknC,UAAW,IACjEjY,EAAI+C,KAAK3sB,MAAMgiC,UAAYle,GAAKge,OAAOC,OAAOpnC,EAAQqnC,UAAW,IACjEpY,EAAI+C,KAAK3sB,MAAMoU,MAAQ0P,GAAKge,OAAOC,OAAOpnC,EAAQyZ,MAAO,QACnD6tB,EAAkBrY,EAAI+C,KAAKiI,YAGjC/T,EAAMsc,OAAOjd,KAAO,EACpBW,EAAMsc,OAAOhd,MAAQ,EACrBU,EAAMsc,OAAO7O,IAAM,EACnBzN,EAAMsc,OAAO9I,OAAS,EAItBxT,EAAM2J,OAAOnW,OAASuV,EAAIY,OAAO0X,aACjCrhB,EAAMX,KAAK7L,OAAWuV,EAAI1J,KAAKgiB,aAC/BrhB,EAAMV,MAAM9L,OAAUuV,EAAIzJ,MAAM+hB,aAChCrhB,EAAMyN,IAAIja,OAAYuV,EAAI0E,IAAIiJ,eAAoB1W,EAAMsc,OAAO7O,IAC/DzN,EAAMwT,OAAOhgB,OAASuV,EAAIyK,OAAOkD,eAAiB1W,EAAMsc,OAAO9I,WAMzD8N,EAAgBn1D,KAAK8D,IAAI+vC,EAAMX,KAAK7L,OAAQwM,EAAM2J,OAAOnW,OAAQwM,EAAMV,MAAM9L,QAC7E+tB,EAAavhB,EAAMyN,IAAIja,OAAS8tB,EAAgBthB,EAAMwT,OAAOhgB,OAASwM,EAAMsc,OAAO7O,IAAMzN,EAAMsc,OAAO9I,OAC5GzK,EAAI+C,KAAK3sB,MAAMqU,OAASyP,GAAKge,OAAOC,OAAOpnC,EAAQ0Z,iBAAW+tB,SAG9DvhB,EAAM8L,KAAKtY,OAASuV,EAAI+C,KAAKuV,aAC7BrhB,EAAMuT,WAAW/f,OAASwM,EAAM8L,KAAKtY,WAC/BguB,EAAkBxhB,EAAM8L,KAAKtY,OAASwM,EAAMyN,IAAIja,OAASwM,EAAMwT,OAAOhgB,OAC5EwM,EAAMoE,gBAAgB5Q,OAAUguB,EAChCxhB,EAAM6b,cAAcroB,OAAYguB,EAChCxhB,EAAM8b,eAAetoB,OAAWwM,EAAM6b,cAAcroB,OAGpDwM,EAAM8L,KAAKvY,MAAQ6tB,EACnBphB,EAAMuT,WAAWhgB,MAAQyM,EAAM8L,KAAKvY,MAE/B5oC,KAAK8xD,kBACRzc,EAAMyhB,eAAiBxe,GAAKye,yBAGxBC,EAA2B5Y,EAAI8S,cAAcvQ,YAC7CsW,EAA4B7Y,EAAI+S,eAAexQ,YAEjDxxB,EAAQ4jC,eACN5jC,EAAQguB,KACV9H,EAAMX,KAAK9L,MAAQouB,IAA6B3hB,EAAMsc,OAAOjd,KAC7DW,EAAMV,MAAM/L,MAAQquB,EAA4B5hB,EAAMyhB,iBAAmBzhB,EAAMsc,OAAOhd,QAEtFU,EAAMX,KAAK9L,MAAQouB,EAA2B3hB,EAAMyhB,iBAAmBzhB,EAAMsc,OAAOjd,KACpFW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,QAGjEU,EAAMX,KAAK9L,MAAQouB,IAA8B3hB,EAAMsc,OAAOjd,KAC9DW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,YAG5DuiB,cAID16C,EAASxc,KAAKm3D,mBAGc,OAA5BhoC,EAAQk5B,YAAYtQ,OACtBv7B,GAAUhb,KAAK8D,IAAI+vC,EAAMoE,gBAAgB5Q,OAASwM,EAAM2J,OAAOnW,OAC7DwM,EAAMsc,OAAO7O,IAAMzN,EAAMsc,OAAO9I,OAAQ,IAE5CzK,EAAIY,OAAOxqB,MAAMkiB,+BAA2Bl6B,aAGtC46C,EAAmC,GAAnB/hB,EAAMuc,UAAiB,SAAW,GAClDyF,EAAmBhiB,EAAMuc,WAAavc,EAAMwc,aAAe,SAAW,GAC5EzT,EAAIgT,UAAU58B,MAAMyqB,WAAsBmY,EAC1ChZ,EAAIiT,aAAa78B,MAAMyqB,WAAmBoY,EAC1CjZ,EAAIkT,cAAc98B,MAAMyqB,WAAkBmY,EAC1ChZ,EAAImT,iBAAiB/8B,MAAMyqB,WAAeoY,EAC1CjZ,EAAIoT,eAAeh9B,MAAMyqB,WAAiBmY,EAC1ChZ,EAAIqT,kBAAkBj9B,MAAMyqB,WAAcoY,EAEtCloC,EAAQ4jC,iBACV3U,EAAI+S,eAAe7oB,UAAY,0CAC/B8V,EAAI8S,cAAc5oB,UAAY,yCAE9B8V,EAAIoT,eAAeh9B,MAAMyqB,WAAiB,SAC1Cb,EAAIqT,kBAAkBj9B,MAAMyqB,WAAc,SAC1Cb,EAAIkT,cAAc98B,MAAMyqB,WAAkB,SAC1Cb,EAAImT,iBAAiB/8B,MAAMyqB,WAAe,SAE1Cb,EAAI1J,KAAKlgB,MAAMsuB,IAAS,MACxB1E,EAAIzJ,MAAMngB,MAAMsuB,IAAQ,SAGrB3zB,EAAQ4jC,gBAAmB1d,EAAM2J,OAAOnW,OAASwM,EAAMoE,gBAAgB5Q,UAC1EuV,EAAI1J,KAAKlgB,MAAMsuB,cAAYtmC,QAC3B4hC,EAAIzJ,MAAMngB,MAAMsuB,cAAWtmC,QAC3B4hC,EAAI+S,eAAe7oB,UAAY8V,EAAI+S,eAAe7oB,UAAU/hC,QAAQ,IAAI+B,OAAO,yCAAmD,KAClI81C,EAAI8S,cAAc5oB,UAAY8V,EAAI8S,cAAc5oB,UAAU/hC,QAAQ,IAAI+B,OAAO,yCAAmD,KAChI+sC,EAAMX,KAAK9L,MAAQouB,IAA8B3hB,EAAMsc,OAAOjd,KAC9DW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,WAC1DuiB,eAIDI,EAAmBjiB,EAAM2J,OAAOnW,OAASwM,EAAMoE,gBAAgB5Q,YAChEgb,OAAO95C,IAAI,OAAO/F,IAAI,CACzBsa,UAAWg5C,EAAmBtT,GAAOiM,cAAgBjM,GAAOuT,4BAIzD1T,OAAO95C,IAAI,SAAS/F,IAAI,CAC3B0c,KAAM1gB,KAAKmvB,QAAQqoC,gCAIhBjD,oBAAmB,SAAAM,GACtBhc,EAAUgc,EAAUxE,UAAYxX,QAG9BA,EAAS,IACP74C,KAAKq0D,YAFQ,mBAGVzb,KAAKiF,QAAQ4C,KAAK,WAIvB99C,QAAQq3C,IAAI,+CAGTqa,YAAc,OAIhBzb,KAAKiF,QAAQ4C,KAAK,kDAOjBpL,EAAQr1C,KAAKq1C,MACb+I,EAAMp+C,KAAKo+C,IAEjB/I,EAAM6b,cAActoB,MAAQyM,EAAMX,KAAK9L,MACvCyM,EAAM8b,eAAevoB,MAAQyM,EAAMV,MAAM/L,UACnC6uB,EAAcpiB,EAAM8L,KAAKvY,MAAQyM,EAAMX,KAAK9L,MAAQyM,EAAMV,MAAM/L,MACtEyM,EAAM2J,OAAOpW,MAAiB6uB,EAC9BpiB,EAAMoE,gBAAgB7Q,MAAQ6uB,EAC9BpiB,EAAMyN,IAAIla,MAAoB6uB,EAC9BpiB,EAAMwT,OAAOjgB,MAAiB6uB,EAG9BrZ,EAAIwK,WAAWp0B,MAAMqU,iBAAsBwM,EAAMuT,WAAW/f,aAC5DuV,EAAIyL,mBAAmBr1B,MAAMqU,iBAAcwM,EAAMuT,WAAW/f,aAC5DuV,EAAI6S,qBAAqBz8B,MAAMqU,iBAAYwM,EAAMoE,gBAAgB5Q,aACjEuV,EAAI3E,gBAAgBjlB,MAAMqU,iBAAiBwM,EAAMoE,gBAAgB5Q,aACjEuV,EAAI8S,cAAc18B,MAAMqU,iBAAmBwM,EAAM6b,cAAcroB,aAC/DuV,EAAI+S,eAAe38B,MAAMqU,iBAAkBwM,EAAM8b,eAAetoB,aAEhEuV,EAAIwK,WAAWp0B,MAAMoU,gBAAsByM,EAAMuT,WAAWhgB,YAC5DwV,EAAIyL,mBAAmBr1B,MAAMoU,gBAAcyM,EAAMoE,gBAAgB7Q,YACjEwV,EAAI6S,qBAAqBz8B,MAAMoU,gBAAYyM,EAAMuT,WAAWhgB,YAC5DwV,EAAI3E,gBAAgBjlB,MAAMoU,gBAAiByM,EAAM2J,OAAOpW,YACxDwV,EAAI0E,IAAItuB,MAAMoU,gBAA6ByM,EAAMyN,IAAIla,YACrDwV,EAAIyK,OAAOr0B,MAAMoU,gBAA0ByM,EAAMwT,OAAOjgB,YAGxDwV,EAAIwK,WAAWp0B,MAAMkgB,KAAiB,IACtC0J,EAAIwK,WAAWp0B,MAAMsuB,IAAiB,IACtC1E,EAAIyL,mBAAmBr1B,MAAMkgB,eAAYW,EAAMX,KAAK9L,MAAQyM,EAAMsc,OAAOjd,WACzE0J,EAAIyL,mBAAmBr1B,MAAMsuB,IAAS,IACtC1E,EAAI6S,qBAAqBz8B,MAAMkgB,KAAO,IACtC0J,EAAI6S,qBAAqBz8B,MAAMsuB,cAAUzN,EAAMyN,IAAIja,aACnDuV,EAAI3E,gBAAgBjlB,MAAMkgB,eAAeW,EAAMX,KAAK9L,YACpDwV,EAAI3E,gBAAgBjlB,MAAMsuB,cAAezN,EAAMyN,IAAIja,aACnDuV,EAAI8S,cAAc18B,MAAMkgB,KAAc,IACtC0J,EAAI8S,cAAc18B,MAAMsuB,cAAiBzN,EAAMyN,IAAIja,aACnDuV,EAAI+S,eAAe38B,MAAMkgB,eAAgBW,EAAMX,KAAK9L,MAAQyM,EAAM2J,OAAOpW,YACzEwV,EAAI+S,eAAe38B,MAAMsuB,cAAgBzN,EAAMyN,IAAIja,aACnDuV,EAAI0E,IAAItuB,MAAMkgB,eAA2BW,EAAMX,KAAK9L,YACpDwV,EAAI0E,IAAItuB,MAAMsuB,IAAwB,IACtC1E,EAAIyK,OAAOr0B,MAAMkgB,eAAwBW,EAAMX,KAAK9L,YACpDwV,EAAIyK,OAAOr0B,MAAMsuB,cAAwBzN,EAAMyN,IAAIja,OAASwM,EAAMoE,gBAAgB5Q,aAClFuV,EAAIY,OAAOxqB,MAAMkgB,KAAqB,IACtC0J,EAAI1J,KAAKlgB,MAAMkgB,KAAuB,IACtC0J,EAAIzJ,MAAMngB,MAAMkgB,KAAsB,2CAUzBh0B,OACR1gB,KAAK03D,kBACF,IAAIt0D,MAAM,4CAGbs0D,YAAYC,eAAej3C,gDAS3B1gB,KAAK03D,kBACF,IAAIt0D,MAAM,8CAGXpD,KAAK03D,YAAYE,iDAUlBztD,UACCm2C,GAAgBtgD,KAAMmK,EAAGnK,KAAKq1C,MAAM2J,OAAOpW,6CAUtCz+B,UACLm2C,GAAgBtgD,KAAMmK,EAAGnK,KAAKq1C,MAAM8L,KAAKvY,yCAaxCloB,UACD4/B,GAAkBtgD,KAAM0gB,EAAM1gB,KAAKq1C,MAAM2J,OAAOpW,+CAYzCloB,UACP4/B,GAAkBtgD,KAAM0gB,EAAM1gB,KAAKq1C,MAAM8L,KAAKvY,iDAUtB,GAA3B5oC,KAAKmvB,QAAQ0oC,gBACVC,wBAGAzC,iEAUD1W,EAAK3+C,UAENq1D,uBAEA0C,UAAY,cACc,GAAzBpZ,EAAGxvB,QAAQ0oC,eAMXlZ,EAAGP,IAAI+C,KAAM,KACT6W,EAAmBrZ,EAAGP,IAAI+C,KAAKuV,aAC/BD,EAAkB9X,EAAGP,IAAI+C,KAAKiI,YAK/BqN,GAAmB9X,EAAGtJ,MAAM4iB,WAC9BD,GAAoBrZ,EAAGtJ,MAAM6iB,aAC9BvZ,EAAGtJ,MAAM4iB,UAAYxB,EACrB9X,EAAGtJ,MAAM6iB,WAAaF,EACtBrZ,EAAGtJ,MAAMyhB,eAAiBxe,GAAKye,oBAE/BpY,EAAG/F,KAAKiF,QAAQ4C,KAAK,kBAjBvB9B,EAAG0W,mBAuBP/c,GAAKgG,iBAAiBtyB,OAAQ,SAAUhsB,KAAK+3D,WAGzCpZ,EAAGP,IAAI+C,OACPxC,EAAGtJ,MAAM4iB,UAAYtZ,EAAGP,IAAI+C,KAAKiI,YACjCzK,EAAGtJ,MAAM6iB,WAAavZ,EAAGP,IAAI+C,KAAKuV,mBAGjCyB,WAAaC,GAAYp4D,KAAK+3D,UAAW,+CAQ1C/3D,KAAKm4D,aACPE,cAAcr4D,KAAKm4D,iBACdA,gBAAa14D,GAIhBO,KAAK+3D,YACPzf,GAAKgL,oBAAoBt3B,OAAQ,SAAUhsB,KAAK+3D,gBAC3CA,UAAY,uCASZvX,QACF7C,MAAMuD,eAAgB,OACtBvD,MAAM2a,iBAAmBt4D,KAAKq1C,MAAMuc,2CAQlCpR,QACF7C,MAAMuD,eAAgB,kCAQrBV,MACDA,GAGAxgD,KAAK29C,MAAMuD,mBAEVI,EAAQd,EAAMgB,OAEd+W,EAAev4D,KAAKw4D,gBACpBC,EAAez4D,KAAKkzD,cAAclzD,KAAK29C,MAAM2a,iBAAmBhX,GAElEthD,KAAKmvB,QAAQ4jC,sBACV3U,IAAI1J,KAAKnN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,eAC5CxT,IAAIzJ,MAAMpN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,WAGhD6G,GAAgBF,QACb9X,KAAK,uDAUAmR,eACPvc,MAAMuc,UAAYA,OAClBuF,mBACEn3D,KAAKq1C,MAAMuc,yDAUZC,EAAerwD,KAAKY,IAAIpC,KAAKq1C,MAAMoE,gBAAgB5Q,OAAS7oC,KAAKq1C,MAAM2J,OAAOnW,OAAQ,UACxFgpB,GAAgB7xD,KAAKq1C,MAAMwc,eAGQ,OAAjC7xD,KAAKmvB,QAAQk5B,YAAYtQ,YACtB1C,MAAMuc,WAAcC,EAAe7xD,KAAKq1C,MAAMwc,mBAEhDxc,MAAMwc,aAAeA,GAIxB7xD,KAAKq1C,MAAMuc,UAAY,IAAG5xD,KAAKq1C,MAAMuc,UAAY,GACjD5xD,KAAKq1C,MAAMuc,UAAYC,IAAc7xD,KAAKq1C,MAAMuc,UAAYC,GAE5D7xD,KAAKmvB,QAAQ4jC,sBACV3U,IAAI1J,KAAKnN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,eAC5CxT,IAAIzJ,MAAMpN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,gBAE/C4B,4BAA6B,EAC3BxzD,KAAKq1C,MAAMuc,yDASX5xD,KAAKq1C,MAAMuc,8DASZ,IAAIxuD,MAAM,iEAKZq4C,GAAKt/C,eCz2CPu8D,0BASQ9f,EAAMzpB,oDAEXypB,KAAOA,IAGPsE,eAAiB,CACpBC,KAAK,EACLwb,iBAAiB,EACjBC,sBAAkBn5D,EAElB2qB,OAAAA,GACAxX,QAAAA,GACAlV,OAAQ,QAELyxB,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/BsB,WAAWrvB,KACXA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS8qC,EAAKvuB,QAAQvc,aACvD88C,EAAiBhS,EAAKR,eAAetqC,QAAQ8qC,EAAKR,eAAex/C,uBAC3DggD,EAAKvuB,QAAQvc,kBAAiB,SAAAlV,KACnCyxB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,SAGpB8e,OAAS,IAETksC,qEASCmH,EAAMziC,SAASE,cAAc,OACnCuiC,EAAIvnB,UAAY,mBAChBunB,EAAIr7B,MAAMkL,SAAW,WACrBmwB,EAAIr7B,MAAMsuB,IAAM,MAChB+M,EAAIr7B,MAAMqU,OAAS,YAEdgnB,IAAMA,yCAON1gC,QAAQwpC,iBAAkB,OAC1BtI,cAEAzX,KAAO,wCASHzpB,GACLA,GAEFmpB,GAAKmG,gBAAgB,CAAC,MAAO,kBAAmB,mBAAoB,SAAU,SAAU,WAAYz+C,KAAKmvB,QAASA,uCAShHnvB,KAAKmvB,QAAQwpC,gBAAiB,SAC1B/nC,EAAS5wB,KAAK44C,KAAKwF,IAAIyL,mBACzB7pD,KAAK6vD,IAAItoB,YAAc3W,IAErB5wB,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,KAEvCj/B,EAAO+D,YAAY30B,KAAK6vD,UAEnBrvB,aAGH3rB,EAAM7U,KAAKmvB,QAAQ/E,OAAO4c,KAAahnC,KAAKwc,QAE5Cxc,KAAKmvB,QAAQypC,mBACf/jD,EAAMA,EAAIgM,QAAQ7gB,KAAKmvB,QAAQypC,uBAG3BzuD,EAAInK,KAAK44C,KAAKN,KAAKkD,SAAS3mC,GAE9BnX,EAASsC,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QAC1CA,IACEsC,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhBzyD,EAASsC,KAAKmvB,QAAQvc,QAAb,QAEP68C,sBAAW/xD,EAAOu9C,qBAAWv9C,EAAOgjB,mBAAS7L,EAAIpX,OAAO,gCAC5DgyD,EAAQA,EAAMjpC,OAAO,GAAGjB,cAAgBkqC,EAAMW,UAAU,GAEpDpwD,KAAKmvB,QAAQguB,SACV0S,IAAIr7B,MAAMkiB,gCAA+B,EAALvsC,cAEpC0lD,IAAIr7B,MAAMkiB,+BAA0BvsC,cAEtC0lD,IAAIJ,MAAQA,OAIbzvD,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,UAElClkB,cAGA,sCAODgT,EAAK3+C,eAKAm2C,IACTwI,EAAGhT,WAICkT,EAAW,EADDF,EAAG/F,KAAKc,MAAMgC,WAAWiD,EAAG/F,KAAKY,SAASwF,OAAOpW,OAAO+S,MAC3C,GACvBkD,EAAW,KAAMA,EAAW,IAC5BA,EAAW,MAAMA,EAAW,KAEhCF,EAAG0R,SACH1R,EAAG/F,KAAKiF,QAAQ4C,KAAK,mBAGrB9B,EAAGO,iBAAmBC,GAAWhJ,EAAQ0I,GAG3C1I,uCAO8B12C,IAA1BO,KAAKk/C,mBACPE,aAAap/C,KAAKk/C,yBACXl/C,KAAKk/C,yDAUDx+B,OACP3gB,EAAIu4C,GAAKpB,QAAQx2B,EAAM,QAAQnjB,UAC/BsX,EAAMmyB,UACPxqB,OAASzc,EAAI8U,OACbw7C,yDAQE,IAAI3zD,KAAKsqC,KAAahnC,KAAKwc,eA3LZm8B,ICNtBkgB,GAAQruC,GAAwC2N,KAKhD2gC,IAAc,EAEdv3B,GAAiBC,GAHV,QAAA,QAMC,IAAIvlC,MAAM,GAAN,MAAe,WAAc68D,IAAc,OAIzD,CAAEnpC,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ2oC,KAAgBv3B,IAAkB,CAC1EpJ,KAAM,SAAcN,UACXghC,GAAM74D,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCfzE,OAAiBohC,GAAa,SAAS1I,KCDnCsJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGqM,YACNrM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAetJ,KAAQA,GAAO4I,GCD5Fg4B,GAAmC,oBAAVC,QAAyBA,OAAOD,iBAAmBC,OAAOD,gBAAgB1oC,KAAK2oC,SAA8B,oBAAZC,UAA8D,mBAA5BA,SAASF,iBAAiCE,SAASF,gBAAgB1oC,KAAK4oC,UACpOC,GAAQ,IAAIC,WAAW,IAEZ,SAASC,SACjBL,SACG,IAAI31D,MAAM,mHAGX21D,GAAgBG,ICPzB,IAFA,IAAIG,GAAY,GAEPv8D,GAAI,EAAGA,GAAI,MAAOA,GACzBu8D,GAAUv8D,KAAMA,GAAI,KAAOV,SAAS,IAAImJ,OAAO,GCJjD,SAAS+zD,GAAGnqC,EAASoqC,EAAK/8C,OACpB1f,EAAIy8D,GAAO/8C,GAAU,EAEH,iBAAX2S,IACToqC,EAAkB,WAAZpqC,EAAuB,IAAIlzB,MAAM,IAAM,KAC7CkzB,EAAU,UAIRqqC,GADJrqC,EAAUA,GAAW,IACFmE,SAAWnE,EAAQiqC,KAAOA,SAE7CI,EAAK,GAAe,GAAVA,EAAK,GAAY,GAC3BA,EAAK,GAAe,GAAVA,EAAK,GAAY,IAEvBD,MACG,IAAItuD,EAAK,EAAGA,EAAK,KAAMA,EAC1BsuD,EAAIz8D,EAAImO,GAAMuuD,EAAKvuD,UAIhBsuD,GDbT,SAAqBA,EAAK/8C,OACpB1f,EAAI0f,GAAU,EACdi9C,EAAMJ,SAEH,CAACI,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,OAAOqG,KAAK,ICSnQu2D,CAAYF,GCrB5B,IAAIG,GAAYnvC,GAAuC+G,SAInDgQ,GAAiBC,GAAwB,UAAW,CAAEN,WAAW,IAAS,MAI5E,CAAEvR,OAAQ,QAASnP,OAAO,EAAM2P,QAASoR,IAAkB,CAC3DhQ,SAAU,SAAkBJ,UACnBwoC,GAAU35D,KAAMmxB,EAAIr1B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCTrE,OAAiBohC,GAAa,SAAStP,SCCnCqoC,GAAQpkC,GAAgB,YCFX,SAAU1J,MDMV,SAAUA,OACrB+tC,SACGv9D,EAASwvB,UAAmCrsB,KAA1Bo6D,EAAW/tC,EAAG8tC,OAA0BC,EAA0B,UAAf9sC,EAAQjB,ICPhF+tC,CAAS/tC,SACLkB,UAAU,wDACTlB,GCHP8tC,GAAQpkC,GAAgB,WCM1B,CAAE7F,OAAQ,SAAUnP,OAAO,EAAM2P,QDJlB,SAAUgQ,OACrB25B,EAAS,cAEL35B,GAAa25B,GACnB,MAAOzmD,cAELymD,EAAOF,KAAS,EACT,MAAMz5B,GAAa25B,GAC1B,MAAO7rC,YACF,ECLiC8rC,CAAqB,aAAe,CAC9ExoC,SAAU,SAAkByoC,YAChBvrC,OAAOvB,EAAuBltB,OACrC0J,QAAQuwD,GAAWD,GAAel+D,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCR/E,OAAiBohC,GAAa,UAAUtP,SCApCkQ,GAAiBxlC,MAAME,UACvB2kC,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAGyF,gBACTzF,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAelQ,SAAkB2oC,GAC5E,iBAAPpuC,GAAmBA,IAAOgV,IAAoBhV,aAAc2C,QAAUsS,IAAQD,GAAgBvP,SAChG4oC,GACAp5B,GCJJ,SAASq5B,GAAa5jB,MAC3BA,QAAAA,GAAW,SAACr5C,EAAGC,UAAMD,EAAEoW,KAAKitB,MAAQpjC,EAAEmW,KAAKitB,SAQtC,SAAS65B,GAAW7jB,MACzBA,QAAAA,GAAW,SAACr5C,EAAGC,UACE,QAASD,EAAEoW,KAAQpW,EAAEoW,KAAKktB,IAAMtjC,EAAEoW,KAAKitB,QACvC,QAASpjC,EAAEmW,KAAQnW,EAAEmW,KAAKktB,IAAMrjC,EAAEmW,KAAKitB,UAoBnD,SAASn9B,GAAMmzC,EAAO8jB,EAAQC,EAAOC,MACtCD,MAEG,IAAIz9D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAChC05C,EAAM15C,GAAGgmD,IAAM,SAKVhmD,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC/Bi7C,EAAOvB,EAAM15C,MACfi7C,EAAK10C,OAAsB,OAAb00C,EAAK+K,IAAc,CAEnC/K,EAAK+K,IAAMwX,EAAOhS,SACdmS,GAAa,IAEd,SAGGC,EAAgB,KACX3mD,EAAI,EAAG4mD,EAAKnkB,EAAMx5C,OAAQ+W,EAAI4mD,EAAI5mD,IAAK,KACxCgH,EAAQy7B,EAAMziC,MACpB0mD,EAAaD,MAAmC,SAEvB,KAEP,OAAdz/C,EAAM+nC,KAAgB/nC,IAAUg9B,GAAQh9B,EAAM1X,OAASu3D,GAAU7iB,EAAMh9B,EAAOu/C,EAAOviB,KAAMh9B,EAAMoU,QAAQguB,KAAM,CACjHud,EAAgB3/C,SAKC,MAAjB2/C,IAEF3iB,EAAK+K,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,OAASyxB,EAAOviB,KAAK8iB,gBAE7DH,WAGND,EAaF,SAASK,GAAStkB,EAAO8jB,EAAQS,OACjC,IAAIj+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAChC05C,EAAM15C,GAAGgmD,IAAM,SAIbkY,EAAiBD,EAASlyB,WAGzB/rC,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC3Bi7C,EAAOvB,EAAM15C,MAEfi7C,EAAK10C,OAAsB,OAAb00C,EAAK+K,IAAc,CAEnC/K,EAAK+K,IAAM/K,EAAKkjB,UAEb,SAGGP,EAAgB,KACX3mD,EAAI,EAAG4mD,EAAKnkB,EAAMx5C,OAAQ+W,EAAI4mD,EAAI5mD,IAAK,KACxCgH,EAAQy7B,EAAMziC,MACF,OAAdgH,EAAM+nC,KAAgB/nC,IAAUg9B,GAA2B6iB,GAAU7iB,EAAMh9B,EAAOu/C,EAAOviB,KAAMh9B,EAAMoU,QAAQguB,KAAM,CACrHud,EAAgB3/C,SAKC,MAAjB2/C,IAEF3iB,EAAK+K,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,OAASyxB,EAAOviB,KAAK8iB,UAGhE9iB,EAAK+K,IAAM/K,EAAKlP,OAASmyB,IAC3BA,EAAiBjjB,EAAK+K,IAAM/K,EAAKlP,cAE5B6xB,IAKbK,EAASlyB,OAASmyB,EAAiBD,EAASjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,SAa/D,SAASK,GAAQ1kB,EAAO8jB,EAAQa,EAAWC,OAC3C,IAAIt+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,OACF2C,MAA1B+2C,EAAM15C,GAAGyW,KAAKwnD,SAChBvkB,EAAM15C,GAAGgmD,IAAMwX,EAAOviB,KAAK8iB,cACtB,QAA+Bp7D,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,UAA0BK,EAAkB,KAC/DC,EAAS,MACR,IAAMN,KAAYI,EACjBA,EAAU99D,eAAe09D,IACQ,GAA/BI,EAAUJ,GAAUO,SAAmBH,EAAUJ,GAAUn2C,MAAQu2C,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUn2C,QACvGy2C,GAAUF,EAAUJ,GAAUlyB,OAC9BsyB,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAMuY,GAI9C7kB,EAAM15C,GAAGgmD,IAAMuY,EAAS,GAAMf,EAAOviB,KAAK8iB,SAGzCO,GACHG,GAAe/kB,EAAO8jB,EAAQa,GAY3B,SAASI,GAAe/kB,EAAO8jB,EAAQa,OACvC,IAAMJ,KAAYI,KACjBA,EAAU99D,eAAe09D,GAAW,CAGtCI,EAAUJ,GAAUjY,IAAM,IACvB,KAGG4X,EAAgB,SACf,IAAMc,KAAiBL,KACW,OAAjCA,EAAUK,GAAe1Y,KAAgB0Y,IAAkBT,GAAYI,EAAUJ,GAAUn2C,MAAQu2C,EAAUK,GAAe52C,OAAS62C,GAAiBN,EAAUJ,GAAWI,EAAUK,IAAiB,CACxMd,EAAgBS,EAAUK,SAKT,MAAjBd,IAEFS,EAAUJ,GAAUjY,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,cAEvD6xB,OAGR,IAAI59D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,SACD2C,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,WAChBvkB,EAAM15C,GAAGgmD,IAAMqY,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,UAexE,SAASa,GAA6BC,EAAerB,EAAQa,OAC9DS,GAAa,EAGXC,EAAgB,OAElB,IAAId,KAAYI,EACdA,EAAUJ,GAAU19D,eAAe,SACrCw+D,EAAcV,EAAUJ,GAAUn2C,OAASm2C,EAG3Cc,EAAc5+D,KAAK89D,OAInB,IAAIhnD,EAAI,EAAGA,EAAI8nD,EAAc7+D,OAAQ+W,OACvCgnD,EAAWc,EAAc9nD,GACrBonD,EAAU99D,eAAe09D,GAAW,KAKjC,IAAMS,KAHXI,EAAaA,GAAcT,EAAUJ,GAAU13D,MAC/C83D,EAAUJ,GAAUjY,IAAM,EAEEqY,EACtBA,EAAUK,GAAeF,SAAWH,EAAUJ,GAAUn2C,MAAQu2C,EAAUK,GAAe52C,QAC3Fu2C,EAAUJ,GAAUjY,KAAOqY,EAAUK,GAAe3yB,gBAIlD2N,EAAQmlB,EAAcZ,GACpBj+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,SACA2C,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,WAChBvkB,EAAM15C,GAAGgmD,IAAMqY,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,SAErEM,EAAUJ,GAAU13D,QACtBmzC,EAAM15C,GAAGm+D,QAAUzkB,EAAM15C,GAAGgmD,MAK9B8Y,GAAcT,EAAUJ,GAAU13D,OACpCy3D,GAASa,EAAcZ,GAAWT,EAAQa,EAAUJ,KAiBrD,SAASH,GAAUz9D,EAAGC,EAAGk9D,EAAQnd,UAClCA,EACQhgD,EAAEw3C,MAAQ2lB,EAAOwB,WAlRf,KAkRyC1+D,EAAEu3C,MAAQv3C,EAAEwrC,OAChEzrC,EAAEw3C,MAAQx3C,EAAEyrC,MAAQ0xB,EAAOwB,WAnRhB,KAmRwC1+D,EAAEu3C,OACrDx3C,EAAE2lD,IAAMwX,EAAOO,SApRJ,KAoRwCz9D,EAAE0lD,IAAM1lD,EAAEyrC,QAC7D1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAASyxB,EAAOO,SArRf,KAqRuCz9D,EAAE0lD,IAE5C3lD,EAAEu3C,KAAO4lB,EAAOwB,WAvRb,KAuRwC1+D,EAAEs3C,KAAOt3C,EAAEwrC,OAC9DzrC,EAAEu3C,KAAOv3C,EAAEyrC,MAAQ0xB,EAAOwB,WAxRf,KAwRuC1+D,EAAEs3C,MACpDv3C,EAAE2lD,IAAMwX,EAAOO,SAzRJ,KAyRwCz9D,EAAE0lD,IAAM1lD,EAAEyrC,QAC7D1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAASyxB,EAAOO,SA1Rf,KA0RuCz9D,EAAE0lD,IAWlD,SAAS2Y,GAAiBt+D,EAAGC,OAI5B2+D,EAAc5+D,EAAEqjC,MAAQpjC,EAAEqjC,KAAOtjC,EAAEsjC,IAAMrjC,EAAEojC,MAC3Cw7B,EAAgB7+D,EAAE2lD,IAAO1lD,EAAE0lD,IAAM1lD,EAAEyrC,QAAY1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAAUzrC,EAAE0lD,WACpEiZ,GAAeC,0LCtSXC,GAFM,iBAWbC,yBAOQC,EAAS5oD,EAAM4+C,iCACpBgK,QAAUA,OACVhB,UAAY,QACZiB,cAAgB,QAChBC,kBAAmB,OACnBC,mBAAqB,QACrBC,cAAe,OACfC,sBAAuB,OACvBC,cAAgB,OAChBC,gBAAkBnpD,GAAQA,EAAKsoD,mBAC/B1J,QAAUA,OACVwK,UAAY,UACZC,YAAa,OAMbC,kBAAoB,GAErBtpD,GAAQA,EAAKupD,oBACVA,aAAevpD,EAAKupD,aACF,GAAnBvpD,EAAKwpD,gBACFA,YAAa,OAEbA,YAAa,GAIlBxpD,GAAQA,EAAK6oD,iBACmB,kBAAvB7oD,EAAK6oD,mBACTG,aAAehpD,EAAK6oD,mBACpBC,iBAAmB9oD,EAAK6oD,uBAKzB,IAAMn5D,KAAOsQ,EAAK6oD,mBACfA,cAAcn5D,GAAOsQ,EAAK6oD,cAAcn5D,QACxCs5D,aAAev8D,KAAKu8D,cAAgBhpD,EAAK6oD,cAAcn5D,GAK9DsQ,GAAQA,EAAKypD,gBACVA,WAAazpD,EAAKypD,gBAElBA,WAAa7K,EAAQhjC,QAAQ8tC,qBAG/BC,cAAgB,UAEhB9e,IAAM,QACN/I,MAAQ,CACX9M,MAAO,CACLK,MAAO,EACPC,OAAQ,SAGPP,UAAY,UAEZkO,MAAQ,QACR2mB,aAAe,QACfC,aAAe,QACfC,aAAe,CAClBC,QAAS,GACTC,MAAO,SAEJC,kBAAmB,MAElBC,EAAyB,WAC7B/f,EAAK8f,kBAAmB,QAErBrL,QAAQvZ,KAAKiF,QAAQvH,GAAG,mBAAoBmnB,QAC5CZ,kBAAkB5/D,MAAK,WAC1BygD,EAAKyU,QAAQvZ,KAAKiF,QAAQtH,IAAI,mBAAoBknB,WAG/C/U,eAEAgV,QAAQnqD,oDAQPg1B,EAAQnb,SAASE,cAAc,OACjCttB,KAAKmyD,QAAQhjC,QAAQwuC,cAAcC,MACrCr1B,EAAMD,UAAY,sBAElBC,EAAMD,UAAY,iBAEf8V,IAAI7V,MAAQA,MAEXs1B,EAAQzwC,SAASE,cAAc,OACrCuwC,EAAMv1B,UAAY,YAClBC,EAAM5T,YAAYkpC,QACbzf,IAAIyf,MAAQA,MAEX7V,EAAa56B,SAASE,cAAc,OAC1C06B,EAAW1f,UAAY,YACvB0f,EAAW,aAAehoD,UACrBo+C,IAAI4J,WAAaA,OAEjB5J,IAAIwK,WAAax7B,SAASE,cAAc,YACxC8wB,IAAIwK,WAAWtgB,UAAY,iBAE3B8V,IAAIkK,KAAOl7B,SAASE,cAAc,YAClC8wB,IAAIkK,KAAKhgB,UAAY,iBAKrB8V,IAAImS,OAASnjC,SAASE,cAAc,YACpC8wB,IAAImS,OAAO/7B,MAAMyqB,WAAa,cAC9Bb,IAAImS,OAAO/7B,MAAMkL,SAAW,gBAC5B0e,IAAImS,OAAO7E,UAAY,QACvBtN,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAImS,wCAOnCh9C,OACFvT,KAAKmyD,QAAQ2L,iBAAiBC,gBAG9BlqC,EACAmqC,OAEAzqD,GAAQA,EAAK+oD,uBACV,IAAMr5D,KAAOsQ,EAAK+oD,wBAChBA,mBAAmBr5D,GAAOsQ,EAAK+oD,mBAAmBr5D,MAIvDjD,KAAKmyD,QAAQhjC,SAAWnvB,KAAKmyD,QAAQhjC,QAAQ+Y,cAE/CrU,GADAmqC,EAAmBjoB,UAAKoc,QAAQhjC,QAAQ+Y,sBAAmBloC,OAChCuT,EAAMvT,KAAKo+C,IAAIyf,YAE1ChqC,EAAUtgB,GAAQA,EAAKsgB,WAGrBA,aAAmBoqC,QAAS,MACvBj+D,KAAKo+C,IAAIyf,MAAMlW,iBACfvJ,IAAIyf,MAAMr2B,YAAYxnC,KAAKo+C,IAAIyf,MAAMlW,iBAEvCvJ,IAAIyf,MAAMlpC,YAAYd,QAClBA,aAAmB33B,QAAU23B,EAAQqqC,mBAIrCrqC,aAAmB33B,OAC5B8hE,EAAiBzqD,EAAMvT,KAAKo+C,IAAIyf,YAE3Bzf,IAAIyf,MAAMnS,UADN73B,MAAAA,EACkBA,EAEA7zB,KAAKm8D,SAAW,SAIxC/d,IAAI7V,MAAMknB,MAAQl8C,GAAQA,EAAKk8C,OAAS,GACxCzvD,KAAKo+C,IAAIyf,MAAMlW,WAIlBrP,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIyf,MAAO,cAHrCvlB,GAAKuV,aAAa7tD,KAAKo+C,IAAIyf,MAAO,cAMhCtqD,GAAQA,EAAKupD,cACV98D,KAAK88D,cAAgB98D,KAAK88D,cAAgBvpD,EAAKupD,oBAC7CA,aAAevpD,EAAKupD,mBAGHr9D,IAApB8T,EAAKwpD,iBAAgDt9D,IAApBO,KAAK+8D,aACjB,GAAnBxpD,EAAKwpD,gBACFA,YAAa,OAEbA,YAAa,GAItBzkB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,qBAC9BvoC,KAAK+8D,YACPzkB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,aACrC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,cAElC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,YACrC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,eAE3BvoC,KAAK88D,oBACTA,aAAe,KACpBxkB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,aACrC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,YACrC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,sBAGnCh1B,IAASA,EAAK4qD,WAAY5qD,EAAK2pD,gBACjC5kB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,oBAC9Bh1B,EAAK4qD,UACP7lB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,mBAAqBh1B,EAAK4qD,WAG5D7lB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,qCAGpC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,yBAI9BD,EAAY/0B,GAAQA,EAAK+0B,WAAa,KACxCA,GAAatoC,KAAKsoC,YAChBtoC,KAAKsoC,YACPgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAOvoC,KAAKsoC,WAC1CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI4J,WAAYhoD,KAAKsoC,WAC/CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIwK,WAAY5oD,KAAKsoC,WAC/CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIkK,KAAMtoD,KAAKsoC,YAE3CgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAOD,GAClCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAI4J,WAAY1f,GACvCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAIwK,WAAYtgB,GACvCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAIkK,KAAMhgB,QAC5BA,UAAYA,GAIftoC,KAAKw0B,QACP8jB,GAAK8lB,cAAcp+D,KAAKo+C,IAAI7V,MAAOvoC,KAAKw0B,YACnCA,MAAQ,MAEXjhB,GAAQA,EAAKihB,QACf8jB,GAAK+lB,WAAWr+D,KAAKo+C,IAAI7V,MAAOh1B,EAAKihB,YAChCA,MAAQjhB,EAAKihB,uDASbx0B,KAAKq1C,MAAM9M,MAAMK,2DAQlB01B,EAAet+D,KAAKo+C,IAAImS,OAAOxE,gBACjCuS,GAAgBt+D,KAAKu+D,iBAAkB,MACpCA,iBAAmBD,MAClBE,EAAc,GAChBC,EAAoB,QAExBnmB,SAAAA,GAAat4C,KAAKw2C,OAAO,SAACuB,EAAM90C,MAC9B80C,EAAK2mB,OAAQ,EACT3mB,EAAK4mB,UAAW,CAElBH,EAAYv7D,GAAO80C,EAAKsY,QADJ,GAEpBoO,EAAoBD,EAAYv7D,GAAKjG,WAItByhE,EAAoB,qBAG5B3hE,MACPw7C,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI9hE,SAFCA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,UAMJ,SAEA,iEASsCkD,KAAKo+C,IAAI4J,WAAhD6W,IAAAA,UAAWC,IAAAA,WAAY1V,IAAAA,iBAC1BtG,IAAM+b,OACNlqB,MAAQmqB,OACRl2B,MAAQwgB,uDAQPzK,EAAK3+C,KACL++D,EAAiB/+D,KAAKmyD,QAAQhjC,QAAQ6vC,UACtCC,EAAc,CAClBC,oBAAqBl/D,KAAKmyD,QAAQgN,iBAClCC,WAAYL,GAAkBA,EAAeM,UAC7CC,iBAAkBP,GAAkBA,EAAe/4D,SACnDw2D,qBAAsBx8D,KAAKw8D,sBAEzB+C,EAAO,SACNv/D,KAAKmyD,QAAQL,gBAAiB,IAC7BmN,EAAYzC,4BAA+B,EAC3Ch7D,KAAKc,IAAI0kC,KAAa,IAAItqC,KAAKuiE,EAAYC,sBAAwBD,EAAYG,aAC7EH,EAAYK,kBAAwD,MAApCt/D,KAAKmyD,QAAQqN,oBAC/CP,EAAYK,kBAAiB,SAAAG,GAC3B9gB,EAAGwT,QAAQqN,oBAAsBC,EACjCF,GAAQE,KAGVF,EAD2C,GAAlC5gB,EAAGwT,QAAQqN,4BAQnBD,uCAWIG,EAAcC,EAAerF,EAAQ5gB,iBAChCgmB,GAAgB1/D,KAAK48D,YAAc58D,KAAK28D,YAAcgD,EAGzD,iBACLtC,EAAe,CACnBE,MAAOzmB,UAAKumB,aAAaE,eAAa,SAAAxlB,UAASA,EAAK6nB,aACpDtC,QAASxmB,UAAKumB,aAAaC,iBAAe,SAAAvlB,UAASA,EAAK6nB,cAGpDC,EAAkB,CACtBtC,SAAW,OAAQzmB,eAAKumB,aAAaE,eAAU,SAAAxlB,UAAQA,EAAK+nB,oBAAgB,SAAA/nB,WAAUA,OACtFulB,WAAa,OAAQxmB,eAAKumB,aAAaC,iBAAY,SAAAvlB,UAAQA,EAAK+nB,oBAAgB,SAAA/nB,WAAUA,QAOtF6d,EAAkB,qBAChBuH,EAAexH,EAAKoK,oBAAoB1C,EAAcvmB,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAASA,EAAK6nB,aAAYlmB,GACzGsmB,EAAkBrK,EAAKsK,uBAAuBJ,EAAiB/oB,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAAQA,EAAK6nB,aAAYlmB,6BAC5GyjB,MAAiB6C,KAQxBE,EAAmC,SAAAC,OACnCC,EAAwB,cACjBrF,SACHvkB,EAAQM,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAAQA,EAAKxkC,KAAKwnD,WAAaA,KACtEqF,EAAsBrF,GAAYoF,EAAUE,GAAA7pB,QAAAA,GAAW,SAACr5C,EAAGC,UAAM+iE,EAAQhjE,EAAEoW,KAAMnW,EAAEmW,SAASijC,OAFzF,IAAMukB,KAAYpF,EAAKwF,YAAjBJ,UAIJqF,MAGiC,mBAA/BpgE,KAAKmyD,QAAQhjC,QAAQyuC,MAAsB,KAG9Cjf,EAAK3+C,QACPA,KAAKu8D,cAAgBv8D,KAAKmyD,QAAQhjC,QAAQosC,eAAgB,CAG5Dl4D,GAD8B68D,EAAiClgE,KAAKmyD,QAAQhjC,QAAQyuC,OAC1BtD,EAAQt6D,KAAKm7D,gBAClEgC,aAAevH,SACf0K,uBAAuBhG,OAEzB,kBACE6C,aAAevH,SACf0K,uBAAuBhG,OAGtBiG,EAAqBF,oBAAKlD,+BAEA,SAAAplB,UAAQA,EAAK6nB,YAAe7nB,EAAK6nB,YAAc7nB,EAAK+nB,oBACtD,SAAC3iE,EAAGC,UACCuhD,EAAGwT,QAAQhjC,QAAQyuC,MAAMzgE,EAAEoW,KAAMnW,EAAEmW,cAEjEipD,qBAAuBn5D,GAAYk9D,EAAoBjG,GAAQ,EAAMvkB,UAAKyqB,+BAA4BxgE,YAExG,eAEAm9D,aAAevH,SACf0K,uBAAuBhG,GAExBt6D,KAAKmyD,QAAQhjC,QAAQ9rB,SACnBrD,KAAKu8D,cAAgBv8D,KAAKmyD,QAAQhjC,QAAQosC,eAE5Cl4D,GAD8B68D,IAC4B5F,EAAQt6D,KAAKm7D,qBAIlEqB,qBAAuBn5D,GAAYrD,KAAKm9D,aAAc7C,GAAQ,EAAMvkB,UAAKyqB,+BAA4BxgE,YAI5GqD,GAAcrD,KAAKm9D,aAAc7C,EAAQt6D,KAAKm7D,UAAWn7D,KAAKmyD,QAAQhjC,QAAQosC,oBAI7E,IAAIz+D,EAAI,EAAGA,EAAIkD,KAAKm9D,aAAangE,OAAQF,SACvCqgE,aAAargE,GAAG2jE,mBAC+ChhE,IAAhEO,KAAKs8D,mBAAmBt8D,KAAKm9D,aAAargE,GAAGyW,KAAKwnD,YAC/C/6D,KAAKs8D,mBAAmBt8D,KAAKm9D,aAAargE,GAAGyW,KAAKwnD,gBAChDoC,aAAargE,GAAGozD,QAKvBlwD,KAAKmyD,QAAQhjC,QAAQ2wC,YACvBxnB,SAAAA,GAAat4C,KAAKw2C,OAAO,SAAAuB,GACnBA,EAAK+nB,SAAW/nB,EAAK4mB,WACvB5mB,EAAKmY,UAKPlwD,KAAKw8D,2BACFrK,QAAQvZ,KAAKiF,QAAQ4C,KAAK,wBAE5Bmc,YAAa,sCAUX/jB,EAAShQ,GAClBgQ,EAAUP,GAAKooB,eAAe1gE,KAAM,SAAU6oC,IAAWgQ,MAEnD8nB,EAAa3gE,KAAKo+C,IAAIyf,MAAMld,YAC5BigB,EAAc5gE,KAAKo+C,IAAIyf,MAAM9R,oBACnClT,EAAUP,GAAKooB,eAAe1gE,KAAKq1C,MAAM9M,MAAO,QAASo4B,IAAe9nB,EACxEA,EAAUP,GAAKooB,eAAe1gE,KAAKq1C,MAAM9M,MAAO,SAAUq4B,IAAgB/nB,4CAQ1DhQ,QACXuV,IAAIwK,WAAWp0B,MAAMqU,iBAAYA,aACjCuV,IAAI4J,WAAWxzB,MAAMqU,iBAAYA,aACjCuV,IAAI7V,MAAM/T,MAAMqU,iBAAYA,6DAONyxB,OACtB,IAAIx9D,EAAI,EAAGmO,EAAKjL,KAAKm9D,aAAangE,OAAQF,EAAImO,EAAInO,IAAK,KACpDi7C,EAAO/3C,KAAKm9D,aAAargE,GAC/Bi7C,EAAK8oB,YAAYvG,GACZt6D,KAAK28D,WAAa38D,KAAKm8D,SAAWF,IACjClkB,EAAK4mB,WAAW5mB,EAAKmY,uCAaxBxW,EAAO4gB,EAAQoF,EAAcoB,iBAG9Bj4B,EAsDEtwB,SAxDFsgC,GAAU,EACR8mB,EAAgB3/D,KAAK28D,UAGrBtK,EAAQ,CACZ,WACEqN,EAAeqB,EAAKC,uBAAuB3kE,KAAK0kE,IAASrB,aAItDY,+BAA4BtgE,KAAMs6D,aAGlC2G,uCAAoCjhE,MAEzC,iBACE+gE,EAAKpE,UAAY5mB,KAAAgrB,EAAKG,wBAAqBH,EAA1BhrB,CAAgC2D,EAAO4gB,IAG1D,sBACEyG,EAAKI,qBAAkBJ,GAAMrB,EAAcC,EAAerF,EAAQ5gB,cAI/D0nB,8BAA2BphE,MAEhC,iBACE6oC,EAASkN,KAAAgrB,EAAKM,yBAAsBN,EAA3BhrB,CAAiCukB,cAIvC2G,uCAAoCjhE,MAEzC,iBACE64C,EAAU9C,KAAAgrB,EAAKO,mBAAgBP,EAArBhrB,CAA2B8C,EAAShQ,IAGhD,sBACEk4B,EAAKQ,0BAAuBR,GAAMl4B,IAGpC,sBACEk4B,EAAKS,qCAAkCT,GAAMzG,IAG/CvkB,KAAC,kBACMgrB,EAAKpE,WAAaoE,EAAKl4B,SAC1BgQ,GAAU,GAELA,WACD74C,cAGN8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,kDAUY+hD,iBACjBta,GAAYhgD,KAAKm7D,WAAWn+D,OAAS,EAAG,KACpC2hD,EAAK3+C,UAENyhE,qBAELnpB,SAAAA,GAAat4C,KAAKm9D,cAAc,SAAAplB,QACHt4C,IAAvBs4C,EAAKxkC,KAAKwnD,WACZpc,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUlyB,OAASrnC,KAAK8D,IAAIq5C,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUlyB,OAAQkP,EAAKlP,OAASyxB,EAAOviB,KAAK8iB,UACtHlc,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUO,aAAiE,IAAhDoG,EAAKpF,mBAAmBvkB,EAAKxkC,KAAKwnD,WAAmC94B,QAAQy/B,EAAKpF,mBAAmBvkB,EAAKxkC,KAAKwnD,wDAczJrhB,EAAO4gB,UACbt6D,KAAK8iD,KAAOpJ,EAAMd,KAAKY,SAASC,gBAAgB5Q,OAAS6Q,EAAMd,KAAKY,SAASoY,UAAY0I,EAAOhS,MACpGtoD,KAAK8iD,IAAM9iD,KAAK6oC,OAASyxB,EAAOhS,OAAU5O,EAAMd,KAAKY,SAASoY,mDASnD0I,OAEXzxB,EAEA2N,MAGFA,EADsB,UAApBx2C,KAAKg9D,WACC1kB,GAAK51B,QAAQ1iB,KAAKw2C,OAGlBx2C,KAAKm9D,cAGLngE,OAAS,EAAG,KAChBoF,EAAMo0C,EAAM,GAAGsM,IACfx9C,EAAMkxC,EAAM,GAAGsM,IAAMtM,EAAM,GAAG3N,aAClCyP,SAAAA,GAAa9B,GAAO,SAAAuB,GAClB31C,EAAMZ,KAAKY,IAAIA,EAAK21C,EAAK+K,KACzBx9C,EAAM9D,KAAK8D,IAAIA,EAAMyyC,EAAK+K,IAAM/K,EAAKlP,WAEnCzmC,EAAMk4D,EAAOhS,KAAM,KAEf9rC,EAASpa,EAAMk4D,EAAOhS,KAC5BhjD,GAAOkX,KACP87B,SAAAA,GAAa9B,GAAO,SAAAuB,GAClBA,EAAK+K,KAAOtmC,KAGhBqsB,EAASrnC,KAAKC,KAAK6D,EAAMg1D,EAAOviB,KAAK8iB,SAAW,GACxB,aAApB76D,KAAKg9D,aACPn0B,EAASrnC,KAAK8D,IAAIujC,EAAQ7oC,KAAKq1C,MAAM9M,MAAMM,cAI7CA,EAAc7oC,KAAKq1C,MAAM9M,MAAMM,cAE1BA,iCAOF7oC,KAAKo+C,IAAI7V,MAAMhB,iBACb4qB,QAAQ/T,IAAIujB,SAAShtC,YAAY30B,KAAKo+C,IAAI7V,OAG5CvoC,KAAKo+C,IAAI4J,WAAWzgB,iBAClB4qB,QAAQ/T,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI4J,YAG9ChoD,KAAKo+C,IAAIwK,WAAWrhB,iBAClB4qB,QAAQ/T,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAIwK,YAG9C5oD,KAAKo+C,IAAIkK,KAAK/gB,iBACZ4qB,QAAQ/T,IAAIkK,KAAK3zB,YAAY30B,KAAKo+C,IAAIkK,yCAQvC/f,EAAQvoC,KAAKo+C,IAAI7V,MACnBA,EAAMhB,YACRgB,EAAMhB,WAAWC,YAAYe,OAGzByf,EAAahoD,KAAKo+C,IAAI4J,WACxBA,EAAWzgB,YACbygB,EAAWzgB,WAAWC,YAAYwgB,OAG9BY,EAAa5oD,KAAKo+C,IAAIwK,WACxBA,EAAWrhB,YACbqhB,EAAWrhB,WAAWC,YAAYohB,OAG9BN,EAAOtoD,KAAKo+C,IAAIkK,KAClBA,EAAK/gB,YACP+gB,EAAK/gB,WAAWC,YAAY8gB,+BAQ5BvQ,iBACGvB,MAAMuB,EAAK3kB,IAAM2kB,EACtBA,EAAK6pB,UAAU5hE,WACV48D,YAAa,OAESn9D,IAAvBs4C,EAAKxkC,KAAKwnD,gBACP8G,eAAe9pB,QACf+pB,mBAGFC,UAAK5E,qBAAsBplB,GAAO,KAC/B2B,EAAQ15C,KAAKmyD,QAAQvZ,KAAKc,WAC3BsoB,gBAAgBjqB,EAAM/3C,KAAKm9D,aAAczjB,2CASnC3B,OAAMkqB,yDAAWlqB,EAAKxkC,KAAKwnD,SACtBt7D,MAAdwiE,QAA0DxiE,IAA/BO,KAAKm7D,UAAU8G,UACvC9G,UAAU8G,GAAc,CAC3Bp5B,OAAQ,EACRia,IAAK,EACLtiB,MAAOuX,EAAKxkC,KAAKitB,MACjBC,IAAKsX,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MAChC86B,SAAS,EACT12C,MAAO5kB,KAAKy8D,cACZjmB,MAAO,GACPnzC,MAAOrD,KAAKq8D,kBAAoBr8D,KAAKo8D,cAAc6F,KAAe,QAE/DxF,iBAIH,IAAI//D,KAAKq7C,EAAKxkC,KAAKitB,OAAS,IAAI9jC,KAAKsD,KAAKm7D,UAAU8G,GAAYzhC,cAC7D26B,UAAU8G,GAAYzhC,MAAQuX,EAAKxkC,KAAKitB,WAGzC0hC,EAAUnqB,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MACvC,IAAI9jC,KAAKwlE,GAAW,IAAIxlE,KAAKsD,KAAKm7D,UAAU8G,GAAYxhC,YACrD06B,UAAU8G,GAAYxhC,IAAMyhC,QAG9B/G,UAAU8G,GAAYzrB,MAAMv5C,KAAK86C,uDAOhC4G,EAAK3+C,QACP2+C,EAAGwc,UAAW,gBACLJ,SACHoH,EAAaxjB,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKktB,KAAOke,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKitB,MAChGogB,EAAWjC,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKitB,MAChDqgB,EAASshB,EAAa,OAE1BxjB,EAAGwc,UAAUJ,GAAUvkB,eAAc,SAAAuB,GAC/B,IAAIr7C,KAAKq7C,EAAKxkC,KAAKitB,OAAS,IAAI9jC,KAAKkkD,KACvCA,EAAW7I,EAAKxkC,KAAKitB,WAGjB0hC,EAAUnqB,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MACvC,IAAI9jC,KAAKwlE,GAAW,IAAIxlE,KAAKmkD,KAC/BA,EAASqhB,MAIbvjB,EAAGwc,UAAUJ,GAAUv6B,MAAQogB,EAC/BjC,EAAGwc,UAAUJ,GAAUt6B,IAAM,IAAI/jC,KAAKmkD,EAAS,QAjB5C,IAAMka,KAAYpc,EAAGwc,YAAfJ,qDA2BgBt7D,IAAzBO,KAAK08D,gBAA+B,KAChC0F,EAAY,MACiB,iBAAxBpiE,KAAK08D,gBAA6B,KACtC,IAAM3B,KAAY/6D,KAAKm7D,UAC1BiH,EAAUnlE,KAAK,CAAC89D,SAAAA,EAAUsH,UAAWriE,KAAKm7D,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKvT,KAAK08D,sBAEnF0F,QAAAA,GAAe,SAACjlE,EAAGC,UAAMD,EAAEklE,UAAYjlE,EAAEilE,kBAEtC,GAAmC,mBAAxBriE,KAAK08D,gBAA+B,KAC7C,IAAM3B,KAAY/6D,KAAKm7D,UAC1BiH,EAAUnlE,KAAK+C,KAAKm7D,UAAUJ,GAAUvkB,MAAM,GAAGjjC,SAEnD6uD,QAAAA,EAAepiE,KAAK08D,oBAGlB0F,EAAUplE,OAAS,MAChB,IAAIF,EAAI,EAAGA,EAAIslE,EAAUplE,OAAQF,SAC/Bq+D,UAAUiH,EAAUtlE,GAAGi+D,UAAUn2C,MAAQ9nB,iDAU/C,IAAMi+D,KAAY/6D,KAAKm7D,UACtBn7D,KAAKm7D,UAAU99D,eAAe09D,UAC3BI,UAAUJ,GAAUO,SAAU,OAC9BH,UAAUJ,GAAUlyB,OAAS,kCASjCkP,kBACE/3C,KAAKw2C,MAAMuB,EAAK3kB,IACvB2kB,EAAK6pB,UAAU,WACVhF,YAAa,MAGZh4C,EAAQgvC,UAAKuJ,qBAAqBplB,IAC1B,GAAVnzB,GAAa09C,UAAKnF,qBAAoBv4C,EAAO,QAEvBnlB,IAAvBs4C,EAAKxkC,KAAKwnD,gBACNwH,oBAAoBxqB,QACpB+pB,8DASW/pB,OAAMkqB,yDAAWlqB,EAAKxkC,KAAKwnD,YAC3Bt7D,MAAdwiE,EAAyB,KACrBlH,EAAW/6D,KAAKm7D,UAAU8G,MAC5BlH,EAAS,SACLyH,EAAY5O,KAAAmH,EAASvkB,cAAcuB,MAErCyqB,GAAa,OACfzH,EAASvkB,cAAagsB,EAAU,GAC3BzH,EAASvkB,MAAMx5C,YAGbokE,+BAFEphE,KAAKm7D,UAAU8G,+CAadlqB,QACXoa,QAAQsQ,WAAW1qB,EAAK3kB,4CAOvB/sB,EAAQiyC,GAAK51B,QAAQ1iB,KAAKw2C,OAC1BksB,EAAa,GACbC,EAAW,GAER7lE,EAAI,EAAGA,EAAIuJ,EAAMrJ,OAAQF,SACN2C,IAAtB4G,EAAMvJ,GAAGyW,KAAKktB,KAChBkiC,EAAS1lE,KAAKoJ,EAAMvJ,IAEtB4lE,EAAWzlE,KAAKoJ,EAAMvJ,SAEnBugE,aAAe,CAClBC,QAASoF,EACTnF,MAAOoF,GAGTt/D,GAAmBrD,KAAKq9D,aAAaC,SACrCj6D,GAAiBrD,KAAKq9D,aAAaE,mDAWjBF,EAAcuF,EAAiBlpB,OAC3CyjB,EAAe,GACf0F,EAAqB,OAEtB7iE,KAAK28D,WAAa38D,KAAKm8D,SAAWF,GAA6B,KAC7D,IAAIn/D,EAAI,EAAGA,EAAI8lE,EAAgB5lE,OAAQF,IAAK,KAC3Ci7C,EAAO6qB,EAAgB9lE,GACvBi7C,EAAK4mB,WAAW5mB,EAAKmY,cAEpBiN,MAGHte,GAAYnF,EAAMjZ,IAAMiZ,EAAMlZ,OAAS,EACvCsiC,EAAappB,EAAMlZ,MAAQqe,EAC3BkkB,EAAarpB,EAAMjZ,IAAMoe,KAkB3B+jB,EAAgB5lE,OAAS,MACtB,IAAIF,EAAI,EAAGA,EAAI8lE,EAAgB5lE,OAAQF,SACrCkmE,6BAA6BJ,EAAgB9lE,GAAIqgE,EAAc0F,EAAoBnpB,OAKtFupB,EAAoB3qB,GAAK4qB,mBAAmB7F,EAAaC,SAtBnC,SAAAx7D,UACjBA,EAAQghE,GAAsB,EAC9BhhE,GAASihE,EAAqB,EACA,IAmBoD,OAAO,iBAG/FI,cAAcF,EAAmB5F,EAAaC,QAASH,EAAc0F,GAAoB,SAAA9qB,UAAQA,EAAKxkC,KAAKitB,MAAQsiC,GAAc/qB,EAAKxkC,KAAKitB,MAAQuiC,KAI3H,GAAzB/iE,KAAKw9D,iBAA0B,MAC5BA,kBAAmB,MACnB,IAAI1gE,EAAI,EAAGA,EAAIugE,EAAaE,MAAMvgE,OAAQF,SACxCkmE,6BAA6B3F,EAAaE,MAAMzgE,GAAIqgE,EAAc0F,EAAoBnpB,OAG1F,KAEG0pB,EAAkB9qB,GAAK4qB,mBAAmB7F,EAAaE,OA9BrC,SAAAz7D,UACfA,EAAQghE,GAAsB,EACA,IA4BgD,OAAO,YAGzFK,cAAcC,EAAiB/F,EAAaE,MAAOJ,EAAc0F,GAAoB,SAAA9qB,UAAQA,EAAKxkC,KAAKktB,IAAMqiC,GAAc/qB,EAAKxkC,KAAKitB,MAAQuiC,aAG9IvE,EAAc,GAChBC,EAAoB,EAEf3hE,EAAI,EAAGA,EAAIqgE,EAAangE,OAAQF,IAAK,KACtCi7C,EAAOolB,EAAargE,OACrBi7C,EAAK4mB,UAAW,CAEnBH,EAAY1hE,GAAKi7C,EAAKsY,QADF,GAEpBoO,EAAoBD,EAAY1hE,GAAGE,WAIpByhE,EAAoB,qBAG5B1qD,MACPukC,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI7qD,SAFCA,EAAI,EAAGA,EAAI0qD,EAAmB1qD,MAA9BA,OAON,IAAIjX,EAAI,EAAGA,EAAIqgE,EAAangE,OAAQF,IACvCqgE,EAAargE,GAAG2jE,qBAGXtD,wCAWKkG,EAAY7sB,EAAO2mB,EAAc0F,EAAoBS,OAC9C,GAAfD,EAAkB,KACf,IAAIvmE,EAAIumE,EAAYvmE,GAAK,EAAGA,IAAK,KAChCi7C,EAAOvB,EAAM15C,MACbwmE,EAAevrB,SAIXA,EAAK6nB,YAAe7nB,EAAKwrB,YAAgBxrB,EAAK+nB,cACdrgE,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,QAMrB,IAAIj7C,EAAIumE,EAAa,EAAGvmE,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC9Ci7C,EAAOvB,EAAM15C,MACbwmE,EAAevrB,SAIXA,EAAK6nB,YAAc7nB,EAAKwrB,YAAgBxrB,EAAK+nB,cACbrgE,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,8CAmBdA,EAAMolB,EAAczjB,GAC5B3B,EAAK4kB,UAAUjjB,IACZ3B,EAAK4mB,WAAW5mB,EAAKyrB,OAE1BzrB,EAAK0oB,cACLtD,EAAalgE,KAAK86C,IAGdA,EAAK4mB,WAAW5mB,EAAKmY,4DAgBFnY,EAAMolB,EAAc0F,EAAoBnpB,GAC/D3B,EAAK4kB,UAAUjjB,QACmBj6C,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,IAIhBA,EAAK4mB,WAAW5mB,EAAKmY,sDAYN2P,EAAiB4D,EAAoB/pB,OAEpDsmB,EAAkB,GAClB0D,EAAwB,MAE1BD,EAAmBzmE,OAAS,MACzB,IAAIF,EAAI,EAAGA,EAAI2mE,EAAmBzmE,OAAQF,SACxCkmE,6BAA6BS,EAAmB3mE,GAAIkjE,EAAiB0D,EAAuBhqB,OAIhG,IAAI58C,EAAI,EAAGA,EAAI+iE,EAAgBvC,QAAQtgE,OAAQF,SAC7CkmE,6BAA6BnD,EAAgBvC,QAAQxgE,GAAIkjE,EAAiB0D,EAAuBhqB,OAGnG,IAAI58C,EAAI,EAAGA,EAAI+iE,EAAgBtC,MAAMvgE,OAAQF,SAC3CkmE,6BAA6BnD,EAAgBtC,MAAMzgE,GAAIkjE,EAAiB0D,EAAuBhqB,WAGhG8kB,EAAc,GAChBC,EAAoB,EAEf3hE,EAAI,EAAGA,EAAIkjE,EAAgBhjE,OAAQF,IAAK,KACzCi7C,EAAOioB,EAAgBljE,OACxBi7C,EAAK4mB,UAAW,CAEnBH,EAAY1hE,GAAKi7C,EAAKsY,QADF,GAEpBoO,EAAoBD,EAAY1hE,GAAGE,WAIpByhE,EAAoB,MAGhC,IAAI1qD,EAAI,EAAGA,EAAI0qD,EAAmB1qD,OACrCukC,SAAAA,GAAakmB,GAAa,SAAUI,GAClCA,EAAI7qD,YAKL,IAAIjX,EAAI,EAAGA,EAAIkjE,EAAgBhjE,OAAQF,IAC1CkjE,EAAgBljE,GAAG2jE,qBAGdT,yCASMjoB,EAAM4rB,EAAaC,QAC3BrB,oBAAoBxqB,EAAM4rB,QAC1B9B,eAAe9pB,EAAM6rB,QACrB9B,uDAUD+B,WAFC3T,OAGG2T,EAAkB7jE,KAAK68D,kBAAkB7qB,OAC/C6xB,aChoCAC,0BAMQ3H,EAAS5oD,EAAM4+C,sDACnBgK,EAAS5oD,EAAM4+C,KAGhBvpB,MAAQ,IACRC,OAAS,IACTia,IAAM,IACNpO,KAAO,sDAUPgF,EAAO4gB,EAAQoF,QAGfvC,aAAen9D,KAAK+/D,oBAAoB//D,KAAKq9D,aAAcr9D,KAAKm9D,aAAczjB,QAG9E9Q,MAAQ5oC,KAAKo+C,IAAIwK,WAAWQ,iBAG5BhL,IAAIwK,WAAWp0B,MAAMqU,OAAU,QAG/B,IAAI/rC,EAAI,EAAGmO,EAAKjL,KAAKm9D,aAAangE,OAAQF,EAAImO,EAAInO,IAAK,CAC7CkD,KAAKm9D,aAAargE,GAC1B+jE,YAAYvG,UAbH,iCAuBXt6D,KAAKo+C,IAAIwK,WAAWrhB,iBAClB4qB,QAAQ/T,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAIwK,mBAhDzBsT,imVCIxB6H,yBAUQxwD,EAAMmoC,EAAYvsB,gCACvBiE,GAAK,UACLxC,OAAS,UACTrd,KAAOA,OACP6qC,IAAM,UACN1C,WAAaA,GAAc,QAC3BwB,eAAiB,CACpBtqC,QAAAA,GACAlV,OAAQ,WAELyxB,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,eAAgB/tB,QAC/CA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS5S,KAAKmvB,QAAQvc,aACvD88C,EAAiB1vD,KAAKk9C,eAAetqC,QAAQ5S,KAAKk9C,eAAex/C,gBAC3DsC,KAAKmvB,QAAQvc,kBAAiB,SAAAlV,GACxCggD,EAAKvuB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,YAGpBsmE,UAAW,OACXrF,WAAY,OACZsF,cAAe,OACfC,WAAc/0C,GAAWA,EAAQ+0C,aAAe,OAChDxF,OAAQ,OAER5b,IAAM,UACNnO,MAAQ,UACRD,KAAO,UACP9L,MAAQ,UACRC,OAAS,UAETs7B,iBAAiB5wD,QAEjB+8C,SAAW,UACX8T,gEAODpkE,KAAKkkE,kBACFF,UAAW,OACXtF,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,kDAQtB2T,UAAW,OACXtF,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,yCAQnB98C,GAC6B9T,MAAd8T,EAAK8wD,OAAsBrkE,KAAKuT,KAAK8wD,OAAS9wD,EAAK8wD,OACrC,MAAfrkE,KAAK4wB,aAClBA,OAAOuhC,QAAQmS,aAAatkE,KAAMuT,EAAK8wD,YAGzCF,iBAAiB5wD,GAElBvT,KAAK4wB,cACFA,OAAOgsC,YAAa,GAGcn9D,MAAjB8T,EAAKwnD,UAAyB/6D,KAAKuT,KAAKwnD,UAAYxnD,EAAKwnD,UAC3C,MAAf/6D,KAAK4wB,aACrBA,OAAO2zC,eAAevkE,KAAMA,KAAKuT,KAAKwnD,SAAUxnD,EAAKwnD,eAGvDxnD,KAAOA,OACP6wD,yBACA1F,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,kDAQV98C,GACXA,SACG2wD,gBAAwC,IAApB3wD,EAAK2wD,YAAoCjiC,QAAQ1uB,EAAK2wD,+CAQzEtzC,GACJ5wB,KAAK2+D,gBACFzO,YACAt/B,OAASA,EACV5wB,KAAK4wB,aACF4yC,aAIF5yC,OAASA,oCASR8oB,UACD,wCAQA,wCAQA,kKA6BH15C,KAAKgkE,UAAYhkE,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAIqmB,WAAY,SAC/D9lB,EAAK3+C,KAELykE,EAAar3C,SAASE,cAAc,OAC1Cm3C,EAAWn8B,UAAY,kBACvBm8B,EAAWC,eAAiB1kE,UACvB2kE,iBAAmB,IAAI3gB,GAAOygB,QAE9BE,iBAAiBruB,GAAG,OAAO,SAAAkK,GAC9B7B,EAAG/tB,OAAOuhC,QAAQvZ,KAAKiF,QAAQ4C,KAAK,QAAU,CAC5CD,MAAAA,EACAzI,KAAM4G,EAAGvrB,aAGRuxC,iBAAiBruB,GAAG,aAAa,SAAAkK,GACpCA,EAAM8D,kBACN3F,EAAG/tB,OAAOuhC,QAAQyS,cAAcjmB,GAChCA,EAAG/tB,OAAOuhC,QAAQvZ,KAAKiF,QAAQ4C,KAAK,cAAe,CACjDD,MAAAA,EACAzI,KAAM4G,EAAGvrB,aAGRuxC,iBAAiBruB,GAAG,YAAY,SAAAkK,GAEnCA,EAAM8D,kBACN3F,EAAG/tB,OAAOuhC,QAAQrU,aAAa0C,WAE5BmkB,iBAAiBruB,GAAG,UAAYP,KAAA4I,EAAG/tB,OAAOuhC,QAAQpU,gBAAaY,EAAG/tB,OAAOuhC,eACzEwS,iBAAiBruB,GAAG,SAAYP,KAAA4I,EAAG/tB,OAAOuhC,QAAQnU,mBAAgBW,EAAG/tB,OAAOuhC,UAE7EnyD,KAAKo+C,IAAIymB,IACP7kE,KAAKo+C,IAAI0mB,cACN1mB,IAAIymB,IAAI78B,aAAay8B,EAAYzkE,KAAKo+C,IAAI0mB,eAG1C1mB,IAAIymB,IAAIlwC,YAAY8vC,GAGpBzkE,KAAKo+C,IAAIte,YACXse,IAAIte,MAAMnL,YAAY8vC,QAGxBrmB,IAAIqmB,WAAaA,OAEdzkE,KAAKgkE,UAAYhkE,KAAKo+C,IAAIqmB,aAE9BzkE,KAAKo+C,IAAIqmB,WAAWl9B,iBACjB6W,IAAIqmB,WAAWl9B,WAAWC,YAAYxnC,KAAKo+C,IAAIqmB,iBAEjDrmB,IAAIqmB,WAAa,KAElBzkE,KAAK2kE,wBACFA,iBAAiB5f,eACjB4f,iBAAmB,oDAUT5lD,OACbuxC,GAAatwD,KAAKmvB,QAAQmhC,SAASyU,eAAkC,MAAjB/kE,KAAKswD,WAAqBtwD,KAAKmvB,QAAQmhC,SAASra,SACzFj2C,KAAKmvB,QAAQmhC,SAASyU,eAAkC,MAAjB/kE,KAAKswD,UAAoBtwD,KAAKswD,SAASra,UAE3Fj2C,KAAKgkE,UAAY1T,IAAatwD,KAAKo+C,IAAI4mB,aAAc,KAEjDrmB,EAAK3+C,KAELglE,EAAe53C,SAASE,cAAc,OAExCttB,KAAKmvB,QAAQguB,IACf6nB,EAAa18B,UAAY,iBAEzB08B,EAAa18B,UAAY,iBAEvB28B,EAAgBjlE,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QACjDunE,IACEjlE,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhB8U,EAAgBjlE,KAAKmvB,QAAQvc,QAAb,IAElBoyD,EAAavV,MAAQwV,EAAcjX,oBAG9BkX,mBAAqB,IAAIlhB,GAAOghB,GAAc1uB,GAAG,OAAO,SAAAkK,GAC3DA,EAAM8D,kBACN3F,EAAG/tB,OAAOu0C,kBAAkBxmB,MAG9B5/B,EAAO4V,YAAYqwC,QACd5mB,IAAI4mB,aAAeA,OAEhBhlE,KAAKgkE,UAAYhkE,KAAKo+C,IAAI4mB,eAE9BhlE,KAAKo+C,IAAI4mB,aAAaz9B,iBACnB6W,IAAI4mB,aAAaz9B,WAAWC,YAAYxnC,KAAKo+C,IAAI4mB,mBAEnD5mB,IAAI4mB,aAAe,KAEpBhlE,KAAKklE,0BACFA,mBAAmBngB,eACnBmgB,mBAAqB,+DAUAnmD,MACzB/e,KAAKmvB,QAAQi2C,6BAEZ9U,GAAYtwD,KAAKmvB,QAAQmhC,SAASkU,aACD,IAAvBxkE,KAAKuT,KAAK+8C,YACY,IAAvBtwD,KAAKuT,KAAK+8C,YAErBtwD,KAAKgkE,UAAY1T,IAAatwD,KAAKo+C,IAAIinB,wBAAyB,KAC5DA,EAA0Bj4C,SAASE,cAAc,OAEvD+3C,EAAwB/8B,UAAY,2BACpCvpB,EAAO4V,YAAY0wC,QACdjnB,IAAIinB,wBAA0BA,OAEzBrlE,KAAKgkE,UAAYhkE,KAAKo+C,IAAIinB,0BAEhCrlE,KAAKo+C,IAAIinB,wBAAwB99B,iBAC9B6W,IAAIinB,wBAAwB99B,WAAWC,YAAYxnC,KAAKo+C,IAAIinB,8BAE9DjnB,IAAIinB,wBAA0B,SAIjCrlE,KAAKo+C,IAAIinB,wBAAyB,MAG/BjnB,IAAIinB,wBAAwB7wC,MAAMyqB,WAAaj/C,KAAK4wB,OAAOuhC,QAAQmT,YAAYC,eAAiB,UAAY,cAG5GnnB,IAAIinB,wBAAwB7wC,MAAMkiB,UAAY,wBAC9C0H,IAAIinB,wBAAwB7wC,MAAMkgB,KAAO,UAyB1C7gB,IArBE+9B,EAAY5xD,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASoY,cAKf,OAAjC5xD,KAAKmvB,QAAQk5B,YAAYtQ,KACL/3C,KAAK8iD,IAEJ9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,QAEnB7oC,KAAK4wB,OAAOkyB,IAXjC,IAWwD8O,QAGvExT,IAAIinB,wBAAwB7wC,MAAMq0B,OAAS,QAC3CzK,IAAIinB,wBAAwB7wC,MAAMsuB,cAAS9iD,KAAK6oC,OAAS,eAEzDuV,IAAIinB,wBAAwB7wC,MAAMsuB,IAAM,QACxC1E,IAAIinB,wBAAwB7wC,MAAMq0B,iBAAY7oD,KAAK6oC,OAAS,SAO/D7oC,KAAKmvB,QAAQi2C,yBAA2BplE,KAAKmvB,QAAQi2C,wBAAwBI,SAE/E3xC,EADmBkiB,UAAK5mB,QAAQi2C,wBAAwBI,iBAAcxlE,KAC5Dg+D,CAAiBh+D,KAAKuT,WAEhCsgB,mBAAoBzJ,GAAOpqB,KAAKuT,KAAKitB,OAAO/iC,OAAO,qBAC/CuC,KAAKuT,KAAKktB,MACZ5M,uBAAwBzJ,GAAOpqB,KAAKuT,KAAKktB,KAAKhjC,OAAO,2BAGpD2gD,IAAIinB,wBAAwB3Z,UAAY73B,kDAUxC7zB,KAAK4wB,OAAOuhC,QAAQsT,UAAU17D,IAAI/J,KAAKozB,4CAQhCwU,OACV/T,EAEAmqC,EACA0H,EACAC,MACE3R,EAAWh0D,KAAK4lE,eAGhBC,GADe7lE,KAAKo+C,IAAIymB,KAAO7kE,KAAKo+C,IAAIte,OACMgmC,uBAAuB,0BAA0B,GAEjG9lE,KAAKmvB,QAAQ42C,qBAEfL,GADAC,EAA+B5vB,UAAK5mB,QAAQ42C,6BAA0B/lE,OACfg0D,EAAU6R,GAEjEH,EAA0B,MAGxBG,KACGH,aAAmCxpE,UAAawpE,aAAmCzH,SACtF0H,EAA6B3R,EAAU6R,WAE5B7lE,KAAKgmE,iBAAiBhmE,KAAK0lE,2BAA6B1lE,KAAKgmE,iBAAiBN,GAC3E,IAERA,aAAmCzH,QACrC4H,EAA+Bna,UAAY,GAC3Cma,EAA+BlxC,YAAY+wC,QAExC,GAA+BjmE,MAA3BimE,EACPG,EAA+Bna,UAAYga,UAGnB,cAAlB1lE,KAAKuT,KAAK2jB,WAA8Cz3B,IAAtBO,KAAKuT,KAAKsgB,cAC1C,IAAIzwB,mDAA4CpD,KAAKozB,UAI1DsyC,wBAA0BA,EAKjC1lE,KAAKmvB,QAAQq2C,SAEf3xC,GADAmqC,EAAmBjoB,UAAK5mB,QAAQq2C,iBAAcxlE,OACnBg0D,EAAUpsB,EAAS5nC,KAAKuT,MAEnDsgB,EAAU7zB,KAAKuT,KAAKsgB,WAGjBA,aAAmB33B,UAAa23B,aAAmBoqC,SACtDD,EAAiBhK,EAAUpsB,WAEjB5nC,KAAKgmE,iBAAiBhmE,KAAK6zB,WAAa7zB,KAAKgmE,iBAAiBnyC,GAC3D,IAEPA,aAAmBoqC,QACrBr2B,EAAQ8jB,UAAY,GACpB9jB,EAAQjT,YAAYd,QAEjB,GAAep0B,MAAXo0B,EACP+T,EAAQ8jB,UAAY73B,UAGI,cAAlB7zB,KAAKuT,KAAK2jB,WAA8Cz3B,IAAtBO,KAAKuT,KAAKsgB,cAC1C,IAAIzwB,mDAA4CpD,KAAKozB,UAG1DS,QAAUA,iDAUC+T,MACjB5nC,KAAKmvB,QAAQ82C,gBAAkBjmE,KAAKmvB,QAAQ82C,eAAejpE,OAAS,EAAG,KACrEkpE,EAAa,MAEbhjC,GAAcljC,KAAKmvB,QAAQ82C,gBAC7BC,EAAalmE,KAAKmvB,QAAQ82C,mBAEvB,CAAA,GAAmC,OAA/BjmE,KAAKmvB,QAAQ82C,sBACpBC,EAAalmB,GAAYhgD,KAAKuT,gDAMb2yD,+BAAY,KAApBziE,UACH3B,EAAQ9B,KAAKuT,KAAK9P,GAEX,MAAT3B,EACF8lC,EAAQ4oB,4BAAqB/sD,GAAQ3B,GAGrC8lC,EAAQu+B,+BAAwB1iE,8HAW1BmkC,GAEP5nC,KAAKw0B,QACP8jB,GAAK8lB,cAAcx2B,EAAS5nC,KAAKw0B,YAC5BA,MAAQ,MAIXx0B,KAAKuT,KAAKihB,QACZ8jB,GAAK+lB,WAAWz2B,EAAS5nC,KAAKuT,KAAKihB,YAC9BA,MAAQx0B,KAAKuT,KAAKihB,gDAUVX,SACQ,iBAAZA,EAA6BA,EACpCA,GAAW,cAAeA,EAAgBA,EAAQuyC,UAC/CvyC,8CAOH7zB,KAAKmvB,UAC6B,kBAA1BnvB,KAAKmvB,QAAQmhC,cAChBA,SAAW,CACdkU,WAAYxkE,KAAKmvB,QAAQmhC,SACzB+V,YAAarmE,KAAKmvB,QAAQmhC,SAC1Bra,OAAQj2C,KAAKmvB,QAAQmhC,UAEkB,WAAjC7qB,GAAOzlC,KAAKmvB,QAAQmhC,iBACrBA,SAAW,GAChBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,UAAWz+C,KAAKswD,SAAUtwD,KAAKmvB,QAAQmhC,YAI3FtwD,KAAKmvB,SAAanvB,KAAKmvB,QAAQmhC,WAAsD,IAAxCtwD,KAAKmvB,QAAQmhC,SAASyU,eAClE/kE,KAAKuT,OAC2B,kBAAvBvT,KAAKuT,KAAK+8C,cACdA,SAAW,CACdkU,WAAYxkE,KAAKuT,KAAK+8C,SACtB+V,YAAarmE,KAAKuT,KAAK+8C,SACvBra,OAAQj2C,KAAKuT,KAAK+8C,UAEmB,WAA9B7qB,GAAOzlC,KAAKuT,KAAK+8C,iBAGrBA,SAAW,GAChBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,UAAWz+C,KAAKswD,SAAUtwD,KAAKuT,KAAK+8C,0DAWtF,iDAQA,kDAQHtwD,KAAKmvB,QAAQm3C,SAAWtmE,KAAKmvB,QAAQm3C,QAAQd,SACtBzvB,UAAK5mB,QAAQm3C,QAAQd,iBAAcxlE,KACrDg+D,CAAiBh+D,KAAK4lE,eAAgB5lE,KAAKuT,MAG7CvT,KAAKuT,KAAKk8C,eAIrBsU,GAAK5nE,UAAUkH,OAAQ,MCrkBjBkjE,0BASQhzD,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXmxB,IAAK,CACH59B,MAAO,EACPC,OAAQ,GAEV2hB,KAAM,CACJ5hB,MAAO,EACPC,OAAQ,IAIRt1B,GACgB9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,kEAUhDmmC,MACJ15C,KAAK8/D,eACA,MAGLnD,EACE8J,EAAQzmE,KAAKuT,KAAKkzD,OAASzmE,KAAKmvB,QAAQs3C,MACxCC,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,iCAGnChK,EADW,SAAT8J,EACWzmE,KAAKuT,KAAKitB,MAAMlhC,UAAYo6C,EAAMlZ,OAAWxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMjZ,IAExF,QAATgmC,EACMzmE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMlZ,OAAWxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYo6C,EAAMjZ,IAI3FzgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAY,EAAIhtB,EAAMlZ,OAAYxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAU,EAAIhtB,EAAMjZ,IAEtHk8B,8CAQF38D,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAGjC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIvqB,cAG7BuqB,IAAIoM,KAAOp9B,SAASE,cAAc,YAClC8wB,IAAIoM,KAAKliB,UAAY,gBAGrB8V,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIooB,IAAIl+B,UAAY,eAGpB8V,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,EAAY,MAAM,IAAI5kD,MAAM,kEACjC4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,SAE7B7kE,KAAKo+C,IAAIoM,KAAKjjB,WAAY,KACzBqhB,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,eAC5BA,EAAY,MAAM,IAAIxlD,MAAM,kEACjCwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIoM,UAE7BxqD,KAAKo+C,IAAIooB,IAAIj/B,WAAY,KACtB+gB,EAAOtoD,KAAK4wB,OAAOwtB,IAAIkK,SACxBM,EAAY,MAAM,IAAIxlD,MAAM,4DACjCklD,EAAK3zB,YAAY30B,KAAKo+C,IAAIooB,UAEvB7H,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAErBvU,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAGtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAClEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC3BlS,IAAIymB,IAAIv8B,oCAA+BA,QACvC8V,IAAIoM,KAAKliB,qCAAgCA,QACzC8V,IAAIooB,IAAIl+B,oCAA+BA,2DAUvC,CACLyE,SAAU,CACR4H,MAAO30C,KAAKo+C,IAAIymB,IAAIrwC,MAAMmgB,MAC1BD,KAAM10C,KAAKo+C,IAAIymB,IAAIrwC,MAAMkgB,MAE3B8xB,IAAK,CACH39B,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,aACrB9tB,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,aAEtBoB,KAAM,CACJ5hB,MAAO5oC,KAAKo+C,IAAIoM,KAAKpB,aAEvByb,IAAK,CACHj8B,MAAO5oC,KAAKo+C,IAAIymB,IAAIzb,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,iEAUDqQ,GACpB/mE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ,WAEtByJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO,WAIvBW,MAAMmxB,IAAI39B,OAASk+B,EAAMP,IAAI39B,YAC7BwM,MAAMmxB,IAAI59B,MAAQm+B,EAAMP,IAAI59B,WAC5ByM,MAAMmV,KAAK5hB,MAAQm+B,EAAMvc,KAAK5hB,WAC9BA,MAAQm+B,EAAMlC,IAAIj8B,WAClBC,OAASk+B,EAAMlC,IAAIh8B,OAGpB7oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQoyB,EAAMh6B,SAAS4H,WAErCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAOqyB,EAAMh6B,SAAS2H,UAGtCgqB,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIymB,UACzCoC,0BACAC,qBAAqBlnE,KAAKo+C,IAAIymB,oCAQ9B/D,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,WACM21D,EAAK+I,QACPqI,EAAQpR,EAAK2R,2BAIjB,iBACM3R,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,gCAUNuoD,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,qCAQjB9gE,KAAK2+D,UAAW,KACZvgB,EAAMp+C,KAAKo+C,IAEbA,EAAIymB,IAAI5uB,OAAQmI,EAAIymB,IAAI5uB,SACnBmI,EAAIymB,IAAIt9B,YAAY6W,EAAIymB,IAAIt9B,WAAWC,YAAY4W,EAAIymB,KAE5DzmB,EAAIoM,KAAKvU,OAAQmI,EAAIoM,KAAKvU,SACrBmI,EAAIoM,KAAKjjB,YAAY6W,EAAIoM,KAAKjjB,WAAWC,YAAY4W,EAAIoM,MAE9DpM,EAAIooB,IAAIvwB,OAAQmI,EAAIooB,IAAIvwB,SACnBmI,EAAIooB,IAAIj/B,YAAY6W,EAAIooB,IAAIj/B,WAAWC,YAAY4W,EAAIooB,UAE3D7H,WAAY,8CAQbxhB,EAAMn9C,KAAKmvB,QAAQguB,IAEnBsqB,EAAe,SAAC7/B,EAASz9B,EAAGZ,SAAG4zC,kEACzB19C,IAAN0K,QAAyB1K,IAAN8J,OAEjBsiD,EAAa1O,GAAY,EAALhzC,EAAUA,EAcpCy9B,EAAQpT,MAAMkiB,eAXJj3C,IAAN8J,OAMM9J,IAAN0K,2BAKmC0hD,kBAAiBtiD,8BAJdA,8BANAsiD,WAY5C4b,EAAaznE,KAAKo+C,IAAIymB,IAAK7kE,KAAK0nE,KAAM1nE,KAAK2nE,KAAMxqB,GACjDsqB,EAAaznE,KAAKo+C,IAAIooB,IAAKxmE,KAAK4nE,KAAM5nE,KAAK6nE,KAAM1qB,GACjDsqB,EAAaznE,KAAKo+C,IAAIoM,KAAMxqD,KAAK8nE,MAAO9nE,KAAK+nE,MAAO5qB,6CAQ9C3c,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CimC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,MACvEuB,EAAYhoE,KAAKq1C,MAAMmV,KAAK5hB,MAC5Bq/B,EAAWjoE,KAAKq1C,MAAMmxB,IAAI59B,MAEnB,SAAT69B,QAEGiB,KAAOlnC,EAAQxgC,KAAK4oC,WACpBk/B,MAAQtnC,EAAQwnC,OAChBJ,KAAOpnC,EAAQwnC,EAAY,EAAIC,EAAW,GAE/B,QAATxB,QAEFiB,KAAOlnC,OACPsnC,MAAQtnC,OACRonC,KAAOpnC,EAAQwnC,EAAY,EAAIC,EAAW,SAI1CP,KAAOlnC,EAAQxgC,KAAK4oC,MAAQ,OAC5Bk/B,MAAQ9nE,KAAKmvB,QAAQguB,IAAM3c,EAAQwnC,EAAYxnC,EAAQwnC,EAAY,OACnEJ,KAAOpnC,EAAQynC,EAAW,GAG7BjoE,KAAKmvB,QAAQguB,IACfn9C,KAAK20C,MAAQ30C,KAAK0nE,KAElB1nE,KAAK00C,KAAO10C,KAAK0nE,UAEdD,yDAQCpf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvCmwB,EAAYloE,KAAKo+C,IAAIoM,KAAKh2B,SAEb,OAAf6zB,EAAsB,KAClB8f,EAAanoE,KAAK4wB,OAAOkyB,IAAM9iD,KAAK8iD,IAAM,OAE3C6kB,KAAO3nE,KAAK8iD,KAAO,EACxBolB,EAAUr/B,iBAAYs/B,QACtBD,EAAUrf,OAAS,GACnBqf,EAAUplB,IAAM,QAEb,KAEGqlB,EADgBnoE,KAAK4wB,OAAOuhC,QAAQ9c,MAAMxM,OACb7oC,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,SAE1E6kB,KAAO3nE,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,KAAO9iD,KAAK6oC,QAAU,GAC5Dq/B,EAAUr/B,iBAAYs/B,QACtBD,EAAUplB,IAAM,GAChBolB,EAAUrf,OAAS,SAGhBgf,MAAQ7nE,KAAKq1C,MAAMmxB,IAAI39B,OAAS,OAEhC4+B,6DAQEznE,KAAK4oC,MAAQ,iDAQb5oC,KAAK4oC,MAAQ,SAvYFm7B,ICAhBqE,0BASQ70D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXmxB,IAAK,CACH1jB,IAAK,EACLla,MAAO,EACPC,OAAQ,GAEVhV,QAAS,CACPgV,OAAQ,EACRw/B,WAAY,EACZC,YAAa,IAIb/0D,GACgB9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,kEAUhDmmC,MACJ15C,KAAK8/D,eACA,MAGH4G,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,iCAE7B3mE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMlZ,OAAYxgC,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,gDAQrFzgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIte,MAAQ1S,SAASE,cAAc,YAInC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIte,MAAMnL,YAAY30B,KAAKo+C,IAAIvqB,cAG/BuqB,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIte,MAAMnL,YAAY30B,KAAKo+C,IAAIooB,UAG/BpoB,IAAIte,MAAM,YAAc9/B,UAExB0+D,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIte,MAAMyH,WAAY,KACxBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAElB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIte,YAE7B6+B,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIte,YAC/BgnC,aAAa9mE,KAAKo+C,IAAIte,WAErBwwB,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAEtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAChEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC7BlS,IAAIte,MAAMwI,sCAAkCA,QAC5C8V,IAAIooB,IAAIl+B,oCAAgCA,2DAUxC,CACLk+B,IAAM,CACJ59B,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,cAEvB7iC,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,YACxBvgB,OAAQ7oC,KAAKo+C,IAAIvqB,QAAQ6iC,cAE3B52B,MAAO,CACL8I,MAAO5oC,KAAKo+C,IAAIte,MAAMspB,YACtBvgB,OAAQ7oC,KAAKo+C,IAAIte,MAAM42B,iEAUHqQ,QAEnB1xB,MAAMmxB,IAAI59B,MAAQm+B,EAAMP,IAAI59B,WAC5ByM,MAAMmxB,IAAI39B,OAASk+B,EAAMP,IAAI39B,YAC7BwM,MAAMxhB,QAAQgV,OAASk+B,EAAMlzC,QAAQgV,OAGtC7oC,KAAKmvB,QAAQguB,SACViB,IAAIvqB,QAAQW,MAAM8zC,sBAAiB,EAAItoE,KAAKq1C,MAAMmxB,IAAI59B,iBAEtDwV,IAAIvqB,QAAQW,MAAM6zC,qBAAgB,EAAIroE,KAAKq1C,MAAMmxB,IAAI59B,iBAKvDA,MAAQm+B,EAAMjnC,MAAM8I,WACpBC,OAASk+B,EAAMjnC,MAAM+I,YAGrBuV,IAAIooB,IAAIhyC,MAAMsuB,eAAU9iD,KAAK6oC,OAAS7oC,KAAKq1C,MAAMmxB,IAAI39B,QAAU,YAE9Do/B,EAAWjoE,KAAKq1C,MAAMmxB,IAAI59B,MAC1B2/B,EAAavoE,KAAKmvB,QAAQguB,IAAO8qB,EAAW,GAAM,EAAIA,EAAW,OAClE7pB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0B6xB,aACxC7J,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIte,YACzCmnC,0BACAC,qBAAqBlnE,KAAKo+C,IAAIte,sCAQ9BghC,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,WACM21D,EAAK+I,QACPqI,EAAQpR,EAAK2R,2BAIjB,iBACM3R,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,8CASH4kC,EAAMn9C,KAAKmvB,QAAQguB,IAEnBsqB,EAAe,SAAC7/B,EAASz9B,EAAGZ,SAAG4zC,kEACzB19C,IAAN0K,QAAyB1K,IAAN8J,OAEjBsiD,EAAa1O,GAAY,EAALhzC,EAAUA,EAcpCy9B,EAAQpT,MAAMkiB,eAXJj3C,IAAN8J,OAMM9J,IAAN0K,2BAKmC0hD,kBAAiBtiD,8BAJdA,8BANAsiD,WAY5C4b,EAAaznE,KAAKo+C,IAAIte,MAAO9/B,KAAKwoE,OAAQxoE,KAAKyoE,OAAQtrB,gCASpD2jB,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,kCAQjB9gE,KAAK2+D,YACH3+D,KAAKo+C,IAAIte,MAAMyH,iBACZ6W,IAAIte,MAAMyH,WAAWC,YAAYxnC,KAAKo+C,IAAIte,YAG5C6+B,WAAY,6CASbn+B,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,YAE5CgoC,OAAShoC,EACVxgC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,EAAQxgC,KAAKq1C,MAAMmxB,IAAI59B,WAE/B8L,KAAOlU,EAAQxgC,KAAKq1C,MAAMmxB,IAAI59B,WAGhC6+B,yDAQCpf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,UAEtC0wB,OADY,OAAfpgB,EACYroD,KAAK8iD,IAGL9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,YAGhD4+B,6DAQEznE,KAAKq1C,MAAMmxB,IAAI59B,qDAQf5oC,KAAKq1C,MAAMmxB,IAAI59B,aArUFm7B,ICAlB2E,0BASQn1D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,MAGNxqC,UAAW,EAEZmV,EAAM,IACU9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,EAAK6f,QAE3C3zB,MAAZ8T,EAAKktB,UACD,IAAIr9B,+CAAwCmQ,EAAK6f,mEAWnDsmB,UACJ15C,KAAK8/D,UAID9/D,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASzgC,KAAKuT,KAAKktB,IAAMiZ,EAAMlZ,mDAQ1DxgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAIjC8wB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,yBACtB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIuqB,YAG7BvqB,IAAIwqB,aAAex7C,SAASE,cAAc,YAC1C8wB,IAAIwqB,aAAatgC,UAAY,8BAC7B8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIwqB,mBAG7BxqB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIvqB,cAG/BuqB,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDAUV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAElB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,UAE7BlG,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAErBvU,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAGtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAa,IAAMtoC,KAAKuT,KAAK+0B,UAAa,KAClEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC7BlS,IAAIymB,IAAIv8B,UAAYtoC,KAAK6oE,cAAgBvgC,OAIzC8V,IAAIvqB,QAAQW,MAAMs0C,SAAW,qEAW/B1qE,SAAgE,WAArD4tB,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIuqB,OAAOvqE,cACnD4qE,WAAsE,WAAzDh9C,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIvqB,SAASm1C,WACrD,CACLn1C,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,aAE1Byb,IAAK,CACHh8B,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,iEAUDqQ,QACnB1xB,MAAMxhB,QAAQ+U,MAAQm+B,EAAMlzC,QAAQ+U,WACpCC,OAASk+B,EAAMlC,IAAIh8B,YACnBuV,IAAIvqB,QAAQW,MAAMs0C,SAAW,QAC7BpK,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIymB,UACzCqC,qBAAqBlnE,KAAKo+C,IAAIymB,UAC9BoC,0BACAgC,wBACAC,mDAQApI,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,iBACM21D,EAAK+I,QACPqI,EAAQhxB,KAAA4f,EAAK2R,+BAA4B3R,EAAjC5f,KAIZ,iBACM4f,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,gCAUNuoD,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,qCAQjB9gE,KAAK2+D,UAAW,KACZkG,EAAM7kE,KAAKo+C,IAAIymB,IAEjBA,EAAIt9B,YACNs9B,EAAIt9B,WAAWC,YAAYq9B,QAGxBlG,WAAY,uCAaTwK,OAKNC,EACAC,EALEC,EAActpE,KAAK4wB,OAAOgY,MAC5BpI,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CC,EAAMzgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKktB,KACvCgmC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,OAMjD,IAAxBzmE,KAAKuT,KAAK41D,gBAAsC1pE,IAAd0pE,IAAyC,IAAdA,IAC3D3oC,GAAS8oC,IACX9oC,GAAS8oC,GAEP7oC,EAAM,EAAI6oC,IACZ7oC,EAAM,EAAI6oC,QAKRC,EAAW/nE,KAAK8D,IAAI9D,KAAK+a,MAAsB,KAAfkkB,EAAMD,IAAiB,IAAM,UAE/DxgC,KAAK5B,UACH4B,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAEToI,MAAQ2gC,EAAWvpE,KAAKq1C,MAAMxhB,QAAQ+U,MAC3CygC,EAAerpE,KAAKq1C,MAAMxhB,QAAQ+U,QAO9B5oC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAEToI,MAAQ2gC,EACbF,EAAe7nE,KAAKY,IAAIq+B,EAAMD,EAAOxgC,KAAKq1C,MAAMxhB,QAAQ+U,QAGtD5oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMkiB,gCAAwC,EAAd12C,KAAK20C,kBAE7CyJ,IAAIymB,IAAIrwC,MAAMkiB,+BAA0B12C,KAAK00C,iBAE/C0J,IAAIymB,IAAIrwC,MAAMoU,gBAAW2gC,QAC1BvpE,KAAKgpE,kBACAngC,OAAS7oC,KAAKo+C,IAAIymB,IAAInO,cAGvB+P,OACD,YACEroB,IAAIvqB,QAAQW,MAAMkiB,UAAY,0BAGhC,WACC12C,KAAKmvB,QAAQguB,IAAK,KACdorB,GAAuD,EAA1C/mE,KAAK8D,IAAKikE,EAAWF,EAAe,QAClDjrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0Bl1C,KAAK8D,IAAKikE,EAAWF,EAAe,oBAIpF,YACCrpE,KAAKmvB,QAAQguB,IAAK,KACdorB,GAA2D,EAA9C/mE,KAAK8D,KAAKikE,EAAWF,GAAgB,EAAG,QACtDjrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0Bl1C,KAAK8D,KAAKikE,EAAWF,GAAgB,EAAG,2BASvFD,EAFAppE,KAAK5B,SACHqiC,EAAM,EACej/B,KAAK8D,KAAKk7B,EAAO,IAGhB6oC,EAItB7oC,EAAQ,GACcA,EAGD,EAGvBxgC,KAAKmvB,QAAQguB,IAAK,KACdorB,GAAqC,EAAxBa,OACdhrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B0yB,oDAWjD/gB,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvC8sB,EAAM7kE,KAAKo+C,IAAIymB,IAGnBA,EAAIrwC,MAAMsuB,cADO,OAAfuF,EACiBroD,KAAK8iD,IAGL9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,4DASrD7oC,KAAKgkE,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAU15C,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAI0mB,SAAU,KAE1GA,EAAW13C,SAASE,cAAc,OACxCw3C,EAASx8B,UAAY,gBACrBw8B,EAAS2E,aAAezpE,UAEnBo+C,IAAIymB,IAAIlwC,YAAYmwC,QACpB1mB,IAAI0mB,SAAWA,OAEZ9kE,KAAKgkE,UAAahkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAS15C,KAAKo+C,IAAI0mB,WAE1E9kE,KAAKo+C,IAAI0mB,SAASv9B,iBACf6W,IAAI0mB,SAASv9B,WAAWC,YAAYxnC,KAAKo+C,IAAI0mB,eAE/C1mB,IAAI0mB,SAAW,sDASjB9kE,KAAKgkE,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAU15C,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAIsrB,UAAW,KAE3GA,EAAYt8C,SAASE,cAAc,OACzCo8C,EAAUphC,UAAY,iBACtBohC,EAAUC,cAAgB3pE,UAErBo+C,IAAIymB,IAAIlwC,YAAY+0C,QACpBtrB,IAAIsrB,UAAYA,OAEb1pE,KAAKgkE,UAAahkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAS15C,KAAKo+C,IAAIsrB,YAE1E1pE,KAAKo+C,IAAIsrB,UAAUniC,iBAChB6W,IAAIsrB,UAAUniC,WAAWC,YAAYxnC,KAAKo+C,IAAIsrB,gBAEhDtrB,IAAIsrB,UAAY,aA5ZH3F,IAiaxB2E,GAAUvsE,UAAU0sE,cAAgB,yBC/Z9Be,0BAWQr2D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,MAGNxqC,UAAW,EAGZmV,EAAM,IACU9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,EAAK6f,QAE3C3zB,MAAZ8T,EAAKktB,UACD,IAAIr9B,+CAAwCmQ,EAAK6f,mEAUnDsmB,UAEA15C,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASzgC,KAAKuT,KAAKktB,IAAMiZ,EAAMlZ,kDAQ1DxgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAIjC8wB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,yBACtB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIuqB,YAG7BvqB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIvqB,cAM/B6qC,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBqhB,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,eAC9BA,QACG,IAAIxlD,MAAM,kEAElBwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIymB,UAE7BlG,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIvqB,cAC/BizC,aAAa9mE,KAAKo+C,IAAIymB,SAGrBv8B,GAAatoC,KAAKuT,KAAK+0B,UAAa,IAAMtoC,KAAKuT,KAAK+0B,UAAa,KAClEtoC,KAAKgkE,SAAW,gBAAkB,SAClC5lB,IAAIymB,IAAIv8B,UAAYtoC,KAAK6oE,cAAgBvgC,gEAW3ClqC,SAAkE,WAAvD4tB,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIvqB,SAASz1B,SACnD,CACLy1B,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,gEAUJ2d,QAEnB1xB,MAAMxhB,QAAQ+U,MAAQm+B,EAAMlzC,QAAQ+U,WACpCC,OAAS,OAET61B,OAAQ,mFAeRoC,eACDiG,EA6BExuD,SA5BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,MAE5B+1C,UAAKsxB,kCAA+BrnE,MAEpC,iBACM21D,EAAK+I,QACPqI,EAAQhxB,KAAA4f,EAAK2R,+BAA4B3R,EAAjC5f,KAIZ,iBACM4f,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,uCAQC+hD,OACNzxB,EACEwf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,aAGlBt4C,IAAvBO,KAAKuT,KAAKwnD,SAAwB,KAE9B8O,EAAe7pE,KAAKuT,KAAKwnD,cAE1B3c,IAAIymB,IAAIrwC,MAAMqU,iBAAY7oC,KAAK4wB,OAAOuqC,UAAU0O,GAAchhC,kBAG5DuV,IAAIymB,IAAIrwC,MAAMsuB,cADF,OAAfuF,EAC0BroD,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOuqC,UAAU0O,GAAc/mB,IAEtD9iD,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK4wB,OAAOuqC,UAAU0O,GAAc/mB,IAAM9iD,KAAK4wB,OAAOuqC,UAAU0O,GAAchhC,kBAE9IuV,IAAIymB,IAAIrwC,MAAMq0B,OAAS,QAKxB7oD,KAAK4wB,kBAAkBkzC,IAEzBj7B,EAASrnC,KAAK8D,IAAItF,KAAK4wB,OAAOiY,OAC1B7oC,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASwF,OAAOnW,OACzC7oC,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASC,gBAAgB5Q,aACjDuV,IAAIymB,IAAIrwC,MAAMq0B,OAAwB,UAAfR,EAA0B,IAAM,QACvDjK,IAAIymB,IAAIrwC,MAAMsuB,IAAqB,OAAfuF,EAAuB,IAAM,KAGtDxf,EAAS7oC,KAAK4wB,OAAOiY,YAEhBuV,IAAIymB,IAAIrwC,MAAMsuB,cAAS9iD,KAAK4wB,OAAOkyB,eACnC1E,IAAIymB,IAAIrwC,MAAMq0B,OAAS,SAG3BzK,IAAIymB,IAAIrwC,MAAMqU,iBAAYA,eArONk7B,IAyO7B6F,GAAeztE,UAAU0sE,cAAgB,0BAEzCe,GAAeztE,UAAUkH,OAAQ,EAMjCumE,GAAeztE,UAAUqnE,KAAOkF,GAAUvsE,UAAUqnE,KAMpDoG,GAAeztE,UAAU+zD,KAAOwY,GAAUvsE,UAAU+zD,KAMpD0Z,GAAeztE,UAAUskE,YAAciI,GAAUvsE,UAAUskE,qzCChQrDqJ,yBAKQ7d,EAAW8d,mBAChB9d,UAAYA,OACZ8d,eAAiBA,GAAkB,WAEnC5/D,EAAI,OACJZ,EAAI,OACJygE,QAAU,OACVzvB,QAAS,OAGTouB,MAAQv7C,SAASE,cAAc,YAC/Bq7C,MAAMrgC,UAAY,mBAClB2jB,UAAUt3B,YAAY30B,KAAK2oE,sDAOtBx+D,EAAGZ,QACRY,EAAIghD,GAAShhD,QACbZ,EAAI4hD,GAAS5hD,mCAOZsqB,GACFA,aAAmBoqC,cAChB0K,MAAMjd,UAAY,QAClBid,MAAMh0C,YAAYd,SAGlB80C,MAAMjd,UAAY73B,+BAQtBo2C,WACYxqE,IAAXwqE,IACFA,GAAS,IAGI,IAAXA,EAAiB,KACfphC,EAAS7oC,KAAK2oE,MAAM5c,aACpBnjB,EAAQ5oC,KAAK2oE,MAAMhoB,YACnB0V,EAAYr2D,KAAK2oE,MAAMphC,WAAWwkB,aAClC+c,EAAW9oE,KAAK2oE,MAAMphC,WAAWoZ,YAEjCjM,EAAO,EAAGoO,EAAM,KAEO,QAAvB9iD,KAAK+pE,gBAAmD,QAAvB/pE,KAAK+pE,eAA0B,KAC9DG,GAAS,EAAOC,GAAQ,EAED,QAAvBnqE,KAAK+pE,iBACH/pE,KAAKuJ,EAAIs/B,EAAS7oC,KAAKgqE,UACzBG,GAAQ,GAGNnqE,KAAKmK,EAAIy+B,EAAQkgC,EAAW9oE,KAAKgqE,UACnCE,GAAS,IAKXx1B,EADEw1B,EACKlqE,KAAKmK,EAAIy+B,EAET5oC,KAAKmK,EAIZ24C,EADEqnB,EACInqE,KAAKuJ,EAAIs/B,EAET7oC,KAAKuJ,OAGbu5C,EAAO9iD,KAAKuJ,EAAIs/B,GACNA,EAAS7oC,KAAKgqE,QAAU3T,IAChCvT,EAAMuT,EAAYxtB,EAAS7oC,KAAKgqE,SAE9BlnB,EAAM9iD,KAAKgqE,UACblnB,EAAM9iD,KAAKgqE,UAGbt1B,EAAO10C,KAAKmK,GACDy+B,EAAQ5oC,KAAKgqE,QAAUlB,IAChCp0B,EAAOo0B,EAAWlgC,EAAQ5oC,KAAKgqE,SAE7Bt1B,EAAO10C,KAAKgqE,UACdt1B,EAAO10C,KAAKgqE,cAIXrB,MAAMn0C,MAAMkgB,KAAOA,EAAO,UAC1Bi0B,MAAMn0C,MAAMsuB,IAAMA,EAAM,UACxB6lB,MAAMn0C,MAAMyqB,WAAa,eACzB1E,QAAS,YAGT2V,2CAQF3V,QAAS,OACTouB,MAAMn0C,MAAMkgB,KAAO,SACnBi0B,MAAMn0C,MAAMsuB,IAAM,SAClB6lB,MAAMn0C,MAAMyqB,WAAa,gDAOzB0pB,MAAMphC,WAAWC,YAAYxnC,KAAK2oE,gBCjIvCyB,GAAS5/C,GAAwC0N,MAIjDmJ,GAAgBC,GAAoB,SACpCC,GAAiBC,GAAwB,WAI3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7ErJ,MAAO,SAAeL,UACbuyC,GAAOpqE,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCV1E,OAAiBohC,GAAa,SAAS3I,MCDnCuJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGoM,aACNpM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAevJ,MAASA,GAAQ6I,GCA5FspC,0BAUQ92D,EAAMmoC,EAAYvsB,wBACtBm7C,EAAkBC,GAAc,GAAI,CAACC,kBAAkB,GAAOr7C,EAAS,CAACmhC,UAAU,kCAClF/8C,EAAMmoC,EAAY4uB,KAEnBj1B,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,EACPC,OAAQ,KAIPt1B,GAAwB9T,MAAhB8T,EAAKk3D,cACV,IAAIrnE,MAAM,sCAAwCmQ,EAAK6f,aAG1DA,GAAKs3C,OACLrG,MAAQ9wD,EAAK8wD,QACbsG,gBAEA9sB,QAAUH,EAAKnqC,KAAKq3D,eACpBlxB,MAAQgE,EAAKnqC,KAAKmmC,QAClBmxB,UAAW,IACXjL,WAAY,IACZrsD,KAAKqsD,WAAY,iEAQf5/D,KAAKuT,KAAKk3D,SAAWzqE,KAAKuT,KAAKk3D,QAAQztE,QAAUgD,KAAK6qE,4CAOpDr0B,QACJs0B,cAEAv3D,KAAKk3D,QAAUj0B,OAEfm0B,mBAEAI,2CAQGrxB,OACFsxB,EAAahrE,KAAKuT,KAAKktB,IAAMzgC,KAAKuT,KAAKktB,IAAMzgC,KAAKuT,KAAKitB,MAAQ,EAC/DkmC,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,0BAC/BlmC,EAAMj/B,KAAK8D,IAAI0lE,EAAYhrE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,UACrD1mE,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASA,EAAMiZ,EAAMlZ,OAAUxgC,KAAKujE,mDAQ7D,CACL3D,WAAW,EACXxsC,GAAIpzB,KAAKozB,GACTojB,MAAOx2C,KAAKuT,KAAKijC,OAAS,GAC1BjjC,KAAMvT,KAAKuT,qCASPutD,mBACFiG,EA8BExuD,EA7BF85C,EAAQ,WAEL8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC+1C,KAAC,WACK/1C,KAAK0+D,QACPqI,EAAQ/mE,KAAKsnE,mCAETtnE,MAER+1C,KAAC,iBACK/1C,KAAK0+D,iBACF6I,kCAA+BvnE,MAAM+mE,YAEtC/mE,gBAGHwnE,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAUx1D,GACtB0b,EAAS1b,OAEJ0b,kCAQJvY,KAAK2+D,gBACHtO,2CAQHrwD,KAAK2+D,UAAW,KACdvgB,EAAMp+C,KAAKo+C,IACXA,EAAIymB,IAAIt9B,YACV6W,EAAIymB,IAAIt9B,WAAWC,YAAY4W,EAAIymB,KAGjC7kE,KAAKmvB,QAAQ87C,aACX7sB,EAAIoM,KAAKjjB,YACX6W,EAAIoM,KAAKjjB,WAAWC,YAAY4W,EAAIoM,MAElCpM,EAAIooB,IAAIj/B,YACV6W,EAAIooB,IAAIj/B,WAAWC,YAAY4W,EAAIooB,WAGlC7H,WAAY,6CAQfn+B,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CC,EAAMzgC,KAAKuT,KAAKktB,IAAMzgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKktB,KAAO,KAChEA,OACGyqC,sBAAsB1qC,EAAOC,OAC7B,KACDgmC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,WACtE0E,yBAAyB3qC,EAAOimC,GAGnCzmE,KAAKmvB,QAAQ87C,kBACV7sB,IAAIoM,KAAKh2B,MAAMC,QAAUz0B,KAAKorE,kBAAoB,QAAU,YAC5DhtB,IAAIooB,IAAIhyC,MAAMC,QAAUz0B,KAAKorE,kBAAoB,QAAU,OAE5DprE,KAAKorE,wBACFC,gBAAgB7qC,EAAOC,4CAUlBD,EAAOC,QAChB2d,IAAIoM,KAAKh2B,MAAMC,QAAU,aACzB2pB,IAAIooB,IAAIhyC,MAAMC,QAAU,YACvB62C,EAAkBtrE,KAAKo+C,IAAIoM,KAAKpB,YAChCmiB,EAAiBvrE,KAAKo+C,IAAIooB,IAAIpd,eAEhC3oB,EAAK,KACD+qC,EAAaF,EAAkB9qC,GAASC,EAAMD,GAAS,EACvDirC,EAAYD,EAAaD,EAAiB,EAC1CG,EAAsB1rE,KAAKmvB,QAAQguB,KAAoB,EAAdquB,EAAkBA,EAC3DG,EAAqB3rE,KAAKmvB,QAAQguB,KAAmB,EAAbsuB,EAAiBA,OAE1DrtB,IAAIoM,KAAKh2B,MAAMkiB,+BAA0Bg1B,cACzCttB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0Bi1B,aACxC,KACCD,EAAsB1rE,KAAKmvB,QAAQguB,KAAgB,EAAT3c,EAAcA,EACxDmrC,EAAqB3rE,KAAKmvB,QAAQguB,KAAuC,GAA/B3c,EAAQ+qC,EAAiB,GAAY/qC,EAAQ+qC,EAAiB,OAEzGntB,IAAIoM,KAAKh2B,MAAMkiB,+BAA0Bg1B,cACzCttB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0Bi1B,2DASxBnrC,EAAOimC,GAEjB,SAATA,EACEzmE,KAAKmvB,QAAQguB,UACVxI,MAAQnU,EAAQxgC,KAAK4oC,WAGrBwV,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,EAAQxgC,KAAK4oC,WAGpBwV,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,MAEtB,QAAT+xB,EACLzmE,KAAKmvB,QAAQguB,UACVxI,MAAQnU,OAGR4d,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,OAGP4d,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,MAIpC10C,KAAKmvB,QAAQguB,UACVxI,MAAQnU,EAAQxgC,KAAK4oC,MAAQ,OAG7BwV,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,EAAQxgC,KAAK4oC,MAAQ,OAG5BwV,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,oDAUtBlU,EAAOC,OACvB8oC,EAAW/nE,KAAK+a,MAAM/a,KAAK8D,IAAIm7B,EAAMD,EAAQ,GAAK,IAElDxgC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAGToI,MAAQpnC,KAAK8D,IAAIikE,EAAUvpE,KAAK4rE,UAAY,GAE7C5rE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,UAEnCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,UAGnC0J,IAAIymB,IAAIrwC,MAAMoU,MAAQ2gC,EAAW,+CAOlClhB,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvC8sB,EAAM7kE,KAAKo+C,IAAIymB,OAEjBA,EAAIrwC,MAAMsuB,IADO,OAAfuF,GACeroD,KAAK8iD,KAAO,GAAK,MAGjB9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,QAAU,GAAK,KAGnE7oC,KAAKmvB,QAAQ87C,WAAY,IACR,OAAf5iB,OACGjK,IAAIoM,KAAKh2B,MAAMsuB,IAAS,SACxB1E,IAAIoM,KAAKh2B,MAAMqU,OAAU7oC,KAAK4wB,OAAOkyB,IAAM9iD,KAAK8iD,IAAM,EAAK,UAC3D1E,IAAIoM,KAAKh2B,MAAMq0B,OAAS,OAE1B,KAECgjB,EAAgB7rE,KAAK4wB,OAAOuhC,QAAQ9c,MAAMxM,OAC1Cs/B,EAAa0D,EAAgB7rE,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,SACxE1E,IAAIoM,KAAKh2B,MAAMsuB,IAAU+oB,EAAgB1D,EAAc,UACvD/pB,IAAIoM,KAAKh2B,MAAMq0B,OAAS,SAG1BzK,IAAIooB,IAAIhyC,MAAMsuB,KAAQ9iD,KAAKo+C,IAAIooB,IAAI9P,aAAe,EAAK,oDASvD12D,KAAK4oC,MAAQ,iDAQb5oC,KAAK4oC,MAAQ,sCAOf63B,mBACAI,2FAOY7gE,KAAKuT,KAAKk3D,qCAAS,SAC7B3K,QAAU9/D,6FAGZuT,KAAKijC,MAAQO,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAKxkC,aAEhDs3D,UAAW,OACXnM,OAAQ,uCAQRoN,6DACA9rE,KAAKujE,sDAIOvjE,KAAKuT,KAAKk3D,qCAAS,KAA3B1yB,iBACAA,EAAK+nB,gGAGT+K,UAAW,EAEZiB,GAAoB9rE,KAAKqkE,aACtBA,MAAMpuB,OAAOj2C,WACbqkE,MAAQ,WAGV9wD,KAAKijC,MAAQ,QACbkoB,OAAQ,iDAOTqN,uDAOEC,EAAQj1B,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,SAAS,CAC3CvX,MAAOuX,EAAKxkC,KAAKitB,MAAMjjC,UACvBkjC,IAAKsX,EAAKxkC,KAAKktB,IAAMsX,EAAKxkC,KAAKktB,IAAIljC,UAAYw6C,EAAKxkC,KAAKitB,MAAMjjC,mBAG5DgW,KAAKnR,IAAMZ,KAAKY,UAALZ,QAAYu1C,GAAAi1B,QAAAA,GAAU,SAAAnjE,UAAKrH,KAAKY,IAAIyG,EAAE23B,MAAO33B,EAAE43B,KAAO53B,EAAE23B,iBACnEjtB,KAAKjO,IAAM9D,KAAK8D,UAAL9D,QAAYu1C,GAAAi1B,QAAAA,GAAU,SAAAnjE,UAAKrH,KAAK8D,IAAIuD,EAAE23B,MAAO33B,EAAE43B,KAAO53B,EAAE23B,gBAClEyrC,EAAUl1B,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAKiH,UAC7CktB,EAAMz1B,GAAAw1B,QAAAA,GAAe,SAACE,EAAKrqE,UAAUqqE,EAAMrqE,IAAO,GAAK9B,KAAKuT,KAAKk3D,QAAQztE,OAE3E04D,UAAKniD,KAAKk3D,iBAAa,SAAA1yB,UAAQA,EAAKxkC,KAAKktB,aAEtCltB,KAAKitB,MAAQ,IAAI9jC,KAAKsD,KAAKuT,KAAKnR,UAChCmR,KAAKktB,IAAM,IAAI/jC,KAAKsD,KAAKuT,KAAKjO,YAE9BiO,KAAKitB,MAAQ,IAAI9jC,KAAKwvE,QACtB34D,KAAKktB,IAAM,gEASdzgC,KAAKuT,KAAKk3D,SAAWzqE,KAAKuT,KAAKk3D,QAAQztE,OAClC85C,UAAKvjC,KAAKk3D,iBAAe,SAAA1yB,UAAQA,EAAK+nB,UAAYnK,KAGpD,mDAOF31D,KAAKo+C,IAAK,eAERA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAGjC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBAExB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIvqB,SAE9B7zB,KAAKmvB,QAAQ87C,kBAEV7sB,IAAIoM,KAAOp9B,SAASE,cAAc,YAClC8wB,IAAIoM,KAAKliB,UAAY,wBACrB8V,IAAIoM,KAAKh2B,MAAMC,QAAU,YAGzB2pB,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIooB,IAAIl+B,UAAY,uBACpB8V,IAAIooB,IAAIhyC,MAAMC,QAAU,QAG3Bz0B,KAAKmvB,QAAQq7C,sBACVpsB,IAAIymB,IAAIuH,WAAar2B,KAAAs0B,EAAYluE,UAAUkwE,uBAAoBrsE,WAIjEo+C,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDAQV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAGbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAGlB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,SAG5Bjc,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,cAE/B5oD,KAAKmvB,QAAQ87C,WAAY,KACtBjrE,KAAKo+C,IAAIoM,KAAKjjB,WAAY,KACxBqhB,EAAY,MAAM,IAAIxlD,MAAM,kEACjCwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIoM,UAG7BxqD,KAAKo+C,IAAIooB,IAAIj/B,WAAY,KACxB+gB,EAAOtoD,KAAK4wB,OAAOwtB,IAAIkK,SACtBM,EAAY,MAAM,IAAIxlD,MAAM,4DACjCklD,EAAK3zB,YAAY30B,KAAKo+C,IAAIooB,WAIzB7H,WAAY,yDAWb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAGrBv8B,EAAYtoC,KAAK6oE,cAAgB,KAAO7oE,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAC7FtoC,KAAKgkE,SAAW,gBAAkB,IAAM,qBACtC5lB,IAAIymB,IAAIv8B,UAAY,YAAcA,EAEnCtoC,KAAKmvB,QAAQ87C,kBACV7sB,IAAIoM,KAAKliB,UAAY,8BAAiCtoC,KAAKgkE,SAAW,gBAAkB,SACxF5lB,IAAIooB,IAAIl+B,UAAa,6BAAgCtoC,KAAKgkE,SAAW,gBAAkB,KAG1FhkE,KAAKuT,KAAKktB,WAGP2d,IAAIvqB,QAAQW,MAAMs0C,SAAW,8DAUhC/B,EAAQ,CACZh6B,SAAU,CACR4H,MAAO30C,KAAKo+C,IAAIymB,IAAIrwC,MAAMmgB,MAC1BD,KAAM10C,KAAKo+C,IAAIymB,IAAIrwC,MAAMkgB,MAE3BmwB,IAAK,CACHj8B,MAAO5oC,KAAKo+C,IAAIymB,IAAIzb,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,sBAIrB12D,KAAKmvB,QAAQ87C,aACflE,EAAMP,IAAM,CACV39B,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,aACrB9tB,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,aAEtB2d,EAAMvc,KAAO,CACX5hB,MAAO5oC,KAAKo+C,IAAIoM,KAAKpB,cAIlB2d,oDAOiBA,GACpB/mE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ,WAEtByJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO,MAIvB10C,KAAKuT,KAAKktB,SAGRmrC,SAAW7E,EAAMlC,IAAIj8B,WAFrBA,MAAQm+B,EAAMlC,IAAIj8B,WAKpBC,OAASk+B,EAAMlC,IAAIh8B,OAGpB7oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQoyB,EAAMh6B,SAAS4H,WAErCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAOqyB,EAAMh6B,SAAS2H,UAGtCgqB,OAAQ,wDAORsI,gCAAgChnE,KAAKo+C,IAAIymB,sDASvC7kE,KAAK4rE,UAAY5rE,KAAK4oC,QAAU5oC,KAAKuT,KAAKktB,+CAS3CjkB,EAAS,KAAMxc,KAAKuT,KAAKjO,IAAMtF,KAAKuT,KAAKnR,KAAO,QAC7C,CACLkqE,SAAUtsE,KAAKuT,KAAKnR,IAAMoa,EAC1B+vD,OAAQvsE,KAAKuT,KAAKjO,IAAMkX,qCASxBxc,KAAK69C,QAAS,OACW79C,KAAKwsE,eAAzBF,IAAAA,SAAUC,IAAAA,OAEXE,EAAU,CACdjsC,MAAO,IAAI9jC,KAAK4vE,GAChB7rC,IAAK,IAAI/jC,KAAK6vE,GACdxtB,WAAW,QAGRlB,QAAQ4C,KAAK,MAAOgsB,kDAUpBzsE,KAAKuT,YA5nBUwwD,IAgoB1BsG,GAAYluE,UAAU0sE,cAAgB,iCCpoBtC,IAGa5M,GAHK,gBAWGyQ,yBAKLva,mBACHA,QAAUA,OACVwa,OAAS,QACT3rC,MAAQ,QACRA,OAAO,GAAK,wDAUHgzB,EAAUtY,EAAYvsB,UACpB,IAAIk7C,GAAYrW,EAAUtY,EAAYvsB,oCAcjDqnB,EAAOrnB,QACPqnB,MAAQA,GAAS,QACjBo2B,aAAc,OACdC,qBAAsB,EAEvB19C,GAAWA,EAAQ09C,2BACdA,oBAAsB19C,EAAQ09C,+DASlCD,aAAc,OACdC,qBAAsB,sCAUnBC,EAAanxB,EAAOxsB,SAC2B,kBAAZA,EAAwB,GAAKA,EAAlE49C,IAAAA,SAAUC,IAAAA,gBAEXA,IACDA,EAAkB,kBAAM,IAG5BD,EAAWA,GAAY,MAEnBE,GAAS,EAETC,EAAa,KAEbvxB,EAAQ,EAAG,IACPA,GAAS,QACF,GAGXsxB,EAAQzrE,KAAKc,IAAId,KAAK+a,MAAM/a,KAAKw4C,IAAI,IAAM2B,GAASn6C,KAAKw4C,IAR3C,KASdkzB,EAAa1rE,KAAKc,IAAId,KAAK6D,IATb,EAS8B4nE,OAI5CjtE,KAAK4sE,YAAa,KACZO,EAAgBF,GAASjtE,KAAKotE,aACfptE,KAAK6sE,qBAAsBM,UAEvCE,wBACAC,oBAIRF,WAAaH,MACdM,EAAWvtE,KAAKghC,MAAMisC,OACrBM,EAAU,KAEN,IAAIC,KADTD,EAAW,GACWvtE,KAAK2sE,UACnB3sE,KAAK2sE,OAAOtvE,eAAemwE,WACrBh3B,EAAQx2C,KAAK2sE,OAAOa,GACpBC,EAAOj3B,EAAMx5C,OACfF,EAAI,EACDA,EAAI2wE,GAAM,SAET11B,EAAOvB,EAAM15C,GACb4wE,EAAY,EAGZ35D,EAAIjX,EAAI,EACLiX,GAAK,GAAMgkC,EAAKiH,OAASxI,EAAMziC,GAAGirC,OAAUkuB,EAAa,IACvD12B,EAAMziC,GAAG+rD,SAAWkN,EAAgBj1B,EAAKxkC,KAAMijC,EAAMziC,GAAGR,OACzDm6D,IAEJ35D,YAIA4G,EAAI7d,EAAI,EACL6d,EAAI67B,EAAMx5C,QAAWw5C,EAAM77B,GAAGqkC,OAASjH,EAAKiH,OAAUkuB,EAAa,GAClEF,EAAgBj1B,EAAKxkC,KAAMijC,EAAM77B,GAAGpH,OACpCm6D,IAEJ/yD,YAIA/D,EAAI22D,EAASvwE,OAAS,EACnB4Z,GAAK,GAAMmhC,EAAKiH,OAASuuB,EAAS32D,GAAGooC,OAAUkuB,GAC9Cn1B,EAAKssB,OAASkJ,EAAS32D,GAAGytD,OAAS2I,EAAgBj1B,EAAKxkC,KAAMg6D,EAAS32D,GAAGrD,OAC1Em6D,IAEJ92D,OAIA82D,EAAYX,EAAU,SAEhBthD,EAAMiiD,EAAYX,EAAW,EAC7BY,EAAe,GAIjB5vE,EAAIjB,EACD6wE,EAAa3wE,OAASyuB,GAAO1tB,EAAIy4C,EAAMx5C,QACtCgwE,EAAgBx2B,EAAMz4C,GAAGwV,KAAMijC,EAAMz4C,GAAGwV,OACxCo6D,EAAa1wE,KAAKu5C,EAAMz4C,IAE5BA,QAGEo+D,EAAUn8D,KAAKmyD,QAAQyb,WAAW71B,EAAKxkC,MACvC8wD,EAAQrkE,KAAKmyD,QAAQwa,OAAOxQ,IAAYn8D,KAAKmyD,QAAQwa,OAAO1Q,IAC9D6D,EAAU9/D,KAAK6tE,oBAAoBF,EAActJ,EAAOyI,EAAa39C,GACzEo+C,EAAStwE,KAAK6iE,GAEdhjE,GAAK2uB,cAEEssB,EAAK+nB,QACZhjE,GAAK,OAMhBkkC,MAAMisC,GAASM,SAGjBA,4CASDZ,EAAS,QACVA,OAASA,gBAGKmB,GAAc9tE,KAAKw2C,sBAAQ,KAAnCuB,OAEDy1B,EAAYz1B,EAAKnnB,OAASmnB,EAAKnnB,OAAOurC,QAAU,GAClDkI,EAAQsI,EAAOa,GACdnJ,IACDA,EAAQ,GACRsI,EAAOa,GAAanJ,GAExBA,EAAMpnE,KAAK86C,GAGPA,EAAKxkC,KAAKitB,QACNuX,EAAKxkC,KAAKktB,IAEVsX,EAAKiH,QAAUjH,EAAKxkC,KAAKitB,MAAMjjC,UAAYw6C,EAAKxkC,KAAKktB,IAAIljC,WAAa,EAGtEw6C,EAAKiH,OAASjH,EAAKxkC,KAAKitB,MAAMjjC,eAMrC,IAAIwwE,KAAoBpB,EAAQ,UAC7BA,EAAOtvE,eAAe0wE,QACtBpB,EAAOoB,YAAuB,SAAC5wE,EAAGC,UAAMD,EAAE6hD,OAAS5hD,EAAE4hD,eAIxD4tB,aAAc,8CAYHe,EAActJ,EAAOyI,EAAa39C,SAK9C2wC,EAJEkO,EAAoBj3B,KAAC+1B,GAAe,YAAQ,SAAAhN,eAAY,CAC1DA,QAAAA,EACAmO,SAAU,OAAQl3B,KAAA+oB,EAAQvsD,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAK3kB,cAGxD46C,EAAkBhxE,OAAQ,uDACjBkxE,aACDA,EAAeD,SAAStuC,OAASguC,EAAa3wE,QAC3CmxE,GAAAR,QAAAA,GAAmB,SAAAS,UAAeF,EAAeD,SAASngD,IAAIsgD,EAAYh7C,cAC7E0sC,EAAUoO,EAAepO,sBAHNkO,+BAAmB,kBAItC,6FAKRlO,SACAA,EAAQuO,WAAWV,GACf7N,EAAQuE,QAAUA,IACdvE,EAAQuE,OACRvE,EAAQuE,MAAMpuB,OAAO6pB,GAGrBuE,IACAA,EAAMxpD,IAAIilD,GACVA,EAAQuE,MAAQA,IAGjBvE,MAGPwO,EAAgBn/C,EAAQm/C,eAAiB,GACvC5yB,EAAa,CACfF,SAAUx7C,KAAKmyD,QAAQvZ,KAAKN,KAAKkD,SACjCS,OAAQj8C,KAAKmyD,QAAQvZ,KAAKN,KAAK2D,QAG7BwT,EAAQ6e,EAAc/nE,QAAQ,UAAWonE,EAAa3wE,QACtDuxE,EAAiB,eAAiB9e,EAAQ,KAAOke,EAAa3wE,OAAS,SACvEwxE,EAAiBjE,GAAc,GAAIp7C,EAASnvB,KAAKmyD,QAAQhjC,SACzD5b,EAAO,SACEg7D,QACF9e,QACA4U,UACEsJ,eACK3tE,KAAKmyD,QAAQvZ,KAAKiF,cACzB79C,KAAKmyD,QAAQvZ,KAAKc,cAE/BomB,EAAU9/D,KAAKyuE,kBAAkBl7D,EACjCmoC,EACA8yB,GAEInK,IACAA,EAAMxpD,IAAIilD,GACVA,EAAQuE,MAAQA,GAGpBvE,EAAQiL,SAEDjL,kDAQF9+B,MAAQ,QACRosC,YAAc,OACdpsC,MAAMhhC,KAAKotE,YAAc,ssaC3RtC,IACMsB,GAAa,iBAYbC,0BAOQ/1B,EAAMzpB,kDAEXypB,KAAOA,IACPsE,eAAiB,CACpBhmB,KAAM,KACNmxB,YAAa,CACXtQ,KAAM,UAER0uB,MAAO,OACPpjE,OAAO,EACPk4D,gBAAgB,EAChBqT,wBAAeC,EAAWC,EAASnC,OAC3BoC,EAAcD,EAAQlR,MAC5BkR,EAAQlR,MAAQiR,EAAUjR,MAC1BiR,EAAUjR,MAAQmR,GAEpBC,WAAY,QAEZ9K,YAAY,EACZ+K,aAAa,EACbzX,oBAAqB,IACrBgS,qBAAsB,CACpBzxB,MAAM,EACN2B,OAAO,GAGT4W,SAAU,CACRkU,YAAY,EACZ6B,aAAa,EACbxrD,KAAK,EACLo7B,QAAQ,EACR8uB,eAAe,GAGjBpH,cAAe,CACbC,OAAO,EACP/iD,KAAK,EACLo7B,QAAQ,GAGV6a,KAAMtL,GAASsL,KAGfoe,4BAAmBC,EAAYp3B,EAAM/xC,GACnCA,EAAS+xC,IAEXq3B,eAAMr3B,EAAM/xC,GACVA,EAAS+xC,IAEXs3B,kBAASt3B,EAAM/xC,GACbA,EAAS+xC,IAEXu3B,gBAAOv3B,EAAM/xC,GACXA,EAAS+xC,IAEXw3B,kBAASx3B,EAAM/xC,GACbA,EAAS+xC,IAEXy3B,kBAASz3B,EAAM/xC,GACbA,EAAS+xC,IAEX03B,oBAAW13B,EAAM/xC,GACfA,EAAS+xC,IAEX23B,qBAAY33B,EAAM/xC,GAChBA,EAAS+xC,IAEX43B,uBAAc53B,EAAM/xC,GAClBA,EAAS+xC,IAGXuiB,OAAQ,CACNviB,KAAM,CACJ+jB,WAAY,GACZjB,SAAU,IAEZvS,KAAM,IAGRsnB,cAAc,EAEdtJ,QAAS,CACPuJ,aAAa,EACb9F,eAAgB,OAChB+F,MAAO,KAGT1K,yBAAyB,KAItBj2C,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/B/tB,QAAQguB,IAAMhuB,EAAQguB,MACtBhuB,QAAQ6vC,UAAY7vC,EAAQ6vC,YAE5BtjB,WAAa,CAChBF,SAAU5C,EAAKN,KAAKkD,SACpBS,OAAQrD,EAAKN,KAAK2D,UAEfmC,IAAM,KACN/I,MAAQ,KACRwO,OAAS,SAERlF,iBACD8mB,UAAY,OACZsK,WAAa,OACb5Q,iBAAmB,OACnB/M,qBAAsB,IACtBoN,oBAAsB,OAEtBwQ,qBAAsB,IAGtBC,cAAgB,cACbzvB,EAAOD,EAAQ2vB,GACnBvxB,EAAGwxB,OAAO5vB,EAAO/J,OACbmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,0BAEI7P,EAAOD,EAAQ2vB,GACtBvxB,EAAG0xB,UAAU9vB,EAAO/J,OAChBmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,0BAEI7P,EAAOD,EAAQ2vB,GACtBvxB,EAAG2xB,UAAU/vB,EAAO/J,OAChBmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,aAKFkgB,eAAiB,cACd/vB,EAAOD,EAAQ2vB,MACnBvxB,EAAG6xB,aAAajwB,EAAO/J,OAEnBmI,EAAGoxB,YAAcpxB,EAAGoxB,WAAW/yE,OAAS,EAAG,OACrC+yE,EAAapxB,EAAGoxB,WAAW73B,kBACjC63B,EAAWhmE,eAAc,SAAA0mE,MACrBA,EAAU3T,aAAc,OACE,GAAxB2T,EAAU1T,aACZ0T,EAAU1T,YAAa,OAErB2T,EAAgB,QACpBD,EAAU3T,sBAAqB,SAAA6T,OACvBC,EAAqBb,EAAWhmE,IAAI4mE,GACrCC,IACLA,EAAmB1T,cAAgBuT,EAAUr9C,GACjB,GAAxBq9C,EAAU1T,aACZ6T,EAAmBtV,SAAU,GAE/BoV,EAAgBG,GAAAH,QAAAA,EAAqBE,OAEvCb,EAAW55B,OAAOu6B,EAAeR,yBAKhC1vB,EAAOD,EAAQ2vB,GACtBvxB,EAAGmyB,gBAAgBvwB,EAAO/J,wBAEnBgK,EAAOD,EAAQ2vB,GACtBvxB,EAAGoyB,gBAAgBxwB,EAAO/J,WAIzBA,MAAQ,KACRm2B,OAAS,KACTqE,SAAW,KAEXC,UAAY,KAEZC,MAAQ,OACRC,WAAa,OAEb7L,YAAc,KACdxH,iBAAmB,CACtBuG,MAAO,KACPtG,YAAY,KAITrV,YAEAlK,WAAWrvB,KACXo+C,SAAW,mGAOV5E,EAAQv7C,SAASE,cAAc,OACrCq7C,EAAMrgC,UAAY,cAClBqgC,EAAM,eAAiB3oE,UAClBo+C,IAAIuqB,MAAQA,MAGX/f,EAAax7B,SAASE,cAAc,OAC1Cs7B,EAAWtgB,UAAY,iBACvBqgC,EAAMh0C,YAAYi0B,QACbxK,IAAIwK,WAAaA,MAGhBZ,EAAa56B,SAASE,cAAc,OAC1C06B,EAAW1f,UAAY,iBACvBqgC,EAAMh0C,YAAYqzB,QACb5J,IAAI4J,WAAaA,MAGhBM,EAAOl7B,SAASE,cAAc,OACpCg7B,EAAKhgB,UAAY,gBACZ8V,IAAIkK,KAAOA,MAGVqZ,EAAWv0C,SAASE,cAAc,OACxCq0C,EAASr5B,UAAY,oBAChB8V,IAAIujB,SAAWA,OAGfyP,uBAGCC,EAAkB,IAAIvN,GAAgB4K,GAAY,KAAM1uE,MAC9DqxE,EAAgB7N,YACXmJ,OAAO+B,IAAc2C,OAMrBxtB,OAAS,IAAIG,GAAOhkD,KAAK44C,KAAKwF,IAAI3E,sBAGlCoK,OAAOvN,GAAG,gBAAgB,SAAAkK,GACzBA,EAAMqE,SACR8Q,EAAKzX,SAASsC,WAGbqD,OAAOvN,GAAG,WAAYP,UAAK+H,qBAAkB99C,YAC7C6jD,OAAOvN,GAAG,UAAYP,UAAKgI,gBAAa/9C,YACxC6jD,OAAOvN,GAAG,SAAYP,UAAKiI,mBAAgBh+C,YAC3C6jD,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOstB,WAGtDztB,OAAOvN,GAAG,MAAQP,UAAKw7B,sBAAmBvxE,YAG1C6jD,OAAOvN,GAAG,QAASP,UAAKy7B,2BAAwBxxE,YAGhD6jD,OAAOvN,GAAG,YAAaP,UAAKme,mBAAgBl0D,OAE7CA,KAAKmvB,QAAQguB,SACVs0B,YAAc,IAAIztB,GAAOhkD,KAAK44C,KAAKwF,IAAI+S,qBAEvCsgB,YAAc,IAAIztB,GAAOhkD,KAAK44C,KAAKwF,IAAI8S,oBAGzCugB,YAAYn7B,GAAG,MAAYP,UAAK27B,sBAAmB1xE,YACnDyxE,YAAYn7B,GAAG,WAAYP,UAAK47B,0BAAuB3xE,YACvDyxE,YAAYn7B,GAAG,UAAYP,UAAK67B,qBAAkB5xE,YAClDyxE,YAAYn7B,GAAG,SAAYP,UAAK87B,wBAAqB7xE,YACrDyxE,YAAY1nE,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAO8tB,0BAE3Dl5B,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,YAAavI,UAAKg8B,qBAAkB/xE,YAC9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,WAAYvI,UAAKi8B,oBAAiBhyE,YAC5E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,YAAavI,UAAKk8B,qBAAkBjyE,YAE9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,cAAevI,UAAKiI,mBAAgBh+C,YAE9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,aAAcvI,UAAKkI,sBAAmBj+C,YAGhFwjE,0CAmEIr0C,iBACLA,EAAS,SAOXmpB,GAAKmG,gBALU,CACb,OAAQ,MAAO,QAAS,QAAS,QAAS,iBAAkB,aAAc,cAAe,sBACzF,sBAAuB,sBAAuB,aAAc,iBAAkB,WAAY,gBAAiB,uBAC3G,OAAQ,OAAQ,iBAAkB,eAAgB,UAAW,0BAA2B,kBAAmB,aAEhFz+C,KAAKmvB,QAASA,GAEvC,yBAA0BA,IACgB,kBAAjCA,EAAQq6C,2BACZr6C,QAAQq6C,qBAAqBzxB,KAAO5oB,EAAQq6C,0BAC5Cr6C,QAAQq6C,qBAAqB9vB,OAAQ,GAEK,WAAxCjU,GAAOtW,EAAQq6C,wBACtBlxB,GAAKmG,gBAAgB,CAAC,OAAQ,SAAUz+C,KAAKmvB,QAAQq6C,qBAAsBr6C,EAAQq6C,sBAE7ExpE,KAAKmvB,QAAQq6C,qBAAqBzxB,YACjC5oB,QAAQq6C,qBAAqB9vB,OAAQ,KAK5C,wBAAyBvqB,GACgB,kBAAhCA,EAAQ6gD,2BACZ7gD,QAAQ6gD,oBAAsB7gD,EAAQ6gD,qBAI3C,gBAAiB7gD,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAYtQ,KAA+B,QAAxB5oB,EAAQk5B,YAAwB,MAAQ,SAElC,WAA/B5iB,GAAOtW,EAAQk5B,cAA4B,SAAUl5B,EAAQk5B,mBAC/Dl5B,QAAQk5B,YAAYtQ,KAAO5oB,EAAQk5B,YAAYtQ,OAIpD,WAAY5oB,IACgB,iBAAnBA,EAAQmrC,aACZnrC,QAAQmrC,OAAOhS,KAAOn5B,EAAQmrC,YAC9BnrC,QAAQmrC,OAAOviB,KAAK+jB,WAAa3sC,EAAQmrC,YACzCnrC,QAAQmrC,OAAOviB,KAAK8iB,SAAW1rC,EAAQmrC,QAEX,WAA1B70B,GAAOtW,EAAQmrC,UACtBhiB,GAAKmG,gBAAgB,CAAC,QAASz+C,KAAKmvB,QAAQmrC,OAAQnrC,EAAQmrC,QACxD,SAAUnrC,EAAQmrC,SACe,iBAAxBnrC,EAAQmrC,OAAOviB,WACnB5oB,QAAQmrC,OAAOviB,KAAK+jB,WAAa3sC,EAAQmrC,OAAOviB,UAChD5oB,QAAQmrC,OAAOviB,KAAK8iB,SAAW1rC,EAAQmrC,OAAOviB,MAEb,WAA/BtS,GAAOtW,EAAQmrC,OAAOviB,OAC7BO,GAAKmG,gBAAgB,CAAC,aAAc,YAAaz+C,KAAKmvB,QAAQmrC,OAAOviB,KAAM5oB,EAAQmrC,OAAOviB,eAMjG,SAAU,oBAAmB,SAAA90C,GACxBA,KAAOksB,IACT4xC,EAAK5xC,QAAQlsB,GAAOksB,EAAQlsB,OAI5B,aAAcksB,IACgB,kBAArBA,EAAQmhC,eACZnhC,QAAQmhC,SAASkU,WAAgBr1C,EAAQmhC,cACzCnhC,QAAQmhC,SAAS+V,YAAgBl3C,EAAQmhC,cACzCnhC,QAAQmhC,SAASz1C,IAAgBsU,EAAQmhC,cACzCnhC,QAAQmhC,SAASra,OAAgB9mB,EAAQmhC,cACzCnhC,QAAQmhC,SAASyU,eAAgB,GAEH,WAA5Bt/B,GAAOtW,EAAQmhC,WACtBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,MAAO,SAAU,iBAAkBz+C,KAAKmvB,QAAQmhC,SAAUnhC,EAAQmhC,WAIrH,kBAAmBnhC,IACgB,kBAA1BA,EAAQwuC,oBACZxuC,QAAQwuC,cAAcC,MAASzuC,EAAQwuC,mBACvCxuC,QAAQwuC,cAAc9iD,IAASsU,EAAQwuC,mBACvCxuC,QAAQwuC,cAAc1nB,OAAS9mB,EAAQwuC,eAEJ,WAAjCl4B,GAAOtW,EAAQwuC,gBACtBrlB,GAAKmG,gBAAgB,CAAC,QAAS,MAAO,UAAWz+C,KAAKmvB,QAAQwuC,cAAexuC,EAAQwuC,sBAcxF,qBAAsB,QAAS,WAAY,WAAY,SAAU,WAAY,aAAc,cAAe,0BATvF,SAAAl6D,OACZ5G,EAAKsyB,EAAQ1rB,MACf5G,EAAI,UACc,mBAAPA,QACL,IAAIuG,4BAAgBK,kCAA2BA,uBAEvDs9D,EAAK5xC,QAAQ1rB,GAAQ5G,MAKrBsyB,EAAQ2wC,YACI9/D,KAAKmvB,QAAS,CAC1B2wC,QAAS3wC,EAAQ2wC,UAEd9/D,KAAKowE,wBACHA,iBAAmB,IAAI1D,GAAiB1sE,YAE1CowE,iBAAiBjb,SAASn1D,KAAKw2C,MAAO,CAAEq2B,qBAAqB,SAC7DqF,UAAU,CAAEC,cAAc,EAAMC,eAAe,SAE/C/hB,UACIrwD,KAAKowE,uBACTiC,0BACA9E,SAAW,QACX6C,iBAAmB,UACnBjhD,QAAQ2wC,aAAUrgE,OAClByyE,UAAU,CAAEC,cAAc,EAAMC,eAAe,SAE/C/hB,eAGA6hB,+CAUD/iD,QACH6hD,SAAW,GAEZ7hD,IACEA,EAAQgjD,iBACV75B,SAAAA,GAAat4C,KAAKw2C,OAAO,SAAAuB,GACvBA,EAAK2mB,OAAQ,EACT3mB,EAAK4mB,WAAW5mB,EAAKsY,YAIzBlhC,EAAQijD,kBACV95B,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAACtI,EAAOphE,GAC5BA,IAAQyrE,KACZrK,EAAMzH,YAAa,8CAUpB0V,uBACApiB,YACAiF,SAAS,WACTC,UAAU,WAEVvR,QAAU7jD,KAAK6jD,OAAOkB,eACtB0sB,aAAezxE,KAAKyxE,YAAY1sB,eAChClB,OAAS,UAETjL,KAAO,UACP8C,WAAa,oCAQd17C,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,OAI7C3oE,KAAKo+C,IAAIkK,KAAK/gB,iBACX6W,IAAIkK,KAAK/gB,WAAWC,YAAYxnC,KAAKo+C,IAAIkK,MAI5CtoD,KAAKo+C,IAAIujB,SAASp6B,iBACf6W,IAAIujB,SAASp6B,WAAWC,YAAYxnC,KAAKo+C,IAAIujB,yCAS/C3hE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,OAIvC3oE,KAAKo+C,IAAIkK,KAAK/gB,iBACZqR,KAAKwF,IAAIyL,mBAAmBl1B,YAAY30B,KAAKo+C,IAAIkK,MAInDtoD,KAAKo+C,IAAIujB,SAASp6B,aACjBvnC,KAAKmvB,QAAQguB,SACVvE,KAAKwF,IAAIzJ,MAAMhgB,YAAY30B,KAAKo+C,IAAIujB,eAEpC/oB,KAAKwF,IAAI1J,KAAK/f,YAAY30B,KAAKo+C,IAAIujB,iDAShCuP,WACPoB,kBACDpB,EAAO,KACHpB,EAAQ9vE,KAAKmvB,QAAQm3C,QAAQwJ,OAA+C,iBAA/B9vE,KAAKmvB,QAAQm3C,QAAQwJ,MAClE9vE,KAAKmvB,QAAQm3C,QAAQwJ,MACrB,SACDqB,WAAahyB,IAChB,WACE+xB,EAAM1N,SACLsM,8CAQgB,MAAnB9vE,KAAKmxE,aACL/xB,aAAap/C,KAAKmxE,iBACbA,WAAa,2CAWXoB,SACA9yE,MAAP8yE,IACFA,EAAM,IAGHrvC,GAAcqvC,KACjBA,EAAM,CAACA,QAIHC,EAAgB17B,UAAKm6B,mBAAiB,SAAA79C,UAA2B,IAArBwgC,GAAA2e,QAAAA,EAAYn/C,4CAGvCo/C,+BAAe,KAA7BC,UACD16B,EAAO/3C,KAAK0yE,YAAYD,GAC1B16B,GACFA,EAAK46B,mGAKJ1B,aAAiBsB,6CACPA,+BAAK,KAAXn/C,UACD2kB,EAAO/3C,KAAK0yE,YAAYt/C,GAC1B2kB,GACFA,EAAK66B,iJAUF/B,UAAKI,kBAAiB,kDASzBt8B,EACAD,EAFEgF,EAAQ15C,KAAK44C,KAAKc,MAAMqc,WAI1B/1D,KAAKmvB,QAAQguB,KACfxI,EAAS30C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMlZ,OACvCkU,EAAO10C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMjZ,OAErCiU,EAAQ10C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMlZ,OACtCmU,EAAQ30C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMjZ,UAGlC8xC,EAAM,OACP,IAAMpW,KAAWn8D,KAAK2sE,UACrB3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KACjCkI,EAAQrkE,KAAK2sE,OAAOxQ,GACpB0W,EAAkBxO,EAAM1H,UAAY0H,EAAMlH,aAAe,yCAI5C0V,+BAAiB,KAAzB96B,UAEL/3C,KAAKmvB,QAAQguB,IACVpF,EAAKpD,MAAQD,GAAUqD,EAAKpD,MAAQoD,EAAKnP,MAAQ+L,GACpD49B,EAAIt1E,KAAK86C,EAAK3kB,IAGX2kB,EAAKrD,KAAOC,GAAWoD,EAAKrD,KAAOqD,EAAKnP,MAAQ8L,GACnD69B,EAAIt1E,KAAK86C,EAAK3kB,+FAOjBm/C,iDAQDA,EAAM,OAEP,IAAMpW,KAAWn8D,KAAK2sE,OAAQ,IAC7B3sE,KAAK2sE,OAAOtvE,eAAe8+D,GACfn8D,KAAK2sE,OAAOxQ,GAChBQ,WACR4V,EAAIt1E,KAAKk/D,UAKRoW,sCAQGn/C,gBACHpzB,KAAKw2C,MAAMpjB,IAAO0/C,UAAKvF,kBAAc,SAAAzN,UAAWA,EAAQ1sC,KAAOA,uCAQ9DA,WACF69C,EAAYjxE,KAAKixE,UACdn0E,EAAI,EAAGmO,EAAKgmE,EAAUj0E,OAAQF,EAAImO,EAAInO,OACzCm0E,EAAUn0E,IAAMs2B,EAAI,IACtB69C,QAAAA,EAAiBn0E,EAAG,sDAWlBw9D,EAASt6D,KAAKmvB,QAAQmrC,OACtB5gB,EAAQ15C,KAAK44C,KAAKc,MAClB6c,EAASje,GAAKge,OAAOC,OACrBpnC,EAAUnvB,KAAKmvB,QACfk5B,EAAcl5B,EAAQk5B,YAAYtQ,KACpCc,GAAU,EACR8vB,EAAQ3oE,KAAKo+C,IAAIuqB,WAGlBtzB,MAAMyN,IAAM9iD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAEvE9iD,KAAKmvB,QAAQguB,SACV9H,MAAMV,MAAQ30C,KAAK44C,KAAKY,SAAS7E,MAAM/L,MAAQ5oC,KAAK44C,KAAKY,SAASmY,OAAOhd,WAEzEU,MAAMX,KAAO10C,KAAK44C,KAAKY,SAAS9E,KAAK9L,MAAQ5oC,KAAK44C,KAAKY,SAASmY,OAAOjd,KAI9Ei0B,EAAMrgC,UAAY,cAEdtoC,KAAKmvB,QAAQ2wC,cACViT,gBAIPl6B,EAAU74C,KAAKgzE,gBAAkBn6B,MAI3Bo6B,EAAkBv5B,EAAMjZ,IAAMiZ,EAAMlZ,MACpC0yC,EAAUD,GAAmBjzE,KAAKmzE,qBAAyBnzE,KAAKq1C,MAAMzM,OAAS5oC,KAAKq1C,MAAM4iB,UAC1Fmb,EAAW15B,EAAMlZ,OAASxgC,KAAKqzE,eAC/BC,EAAqBnkD,EAAQ9rB,OAASrD,KAAKuzE,UAC3CC,EAA8BrkD,EAAQosC,gBAAkBv7D,KAAKyzE,mBAC7D/T,EAAgBwT,GAAUE,GAAYE,GAAsBE,OAC7DL,oBAAsBF,OACtBI,eAAiB35B,EAAMlZ,WACvB+yC,UAAYpkD,EAAQ9rB,WACpBowE,mBAAqBtkD,EAAQosC,oBAE7BlmB,MAAM4iB,UAAYj4D,KAAKq1C,MAAMzM,UAC5B8qC,EAAa1zE,KAAK2zE,cAClBC,EAAc,CAClB77B,KAAMuiB,EAAOviB,KACbuQ,KAAMgS,EAAOhS,MAETurB,EAAiB,CACrB97B,KAAMuiB,EAAOviB,KACbuQ,KAAMgS,EAAOviB,KAAK8iB,SAAW,GAE3BhyB,EAAS,EACP2tB,EAAY8D,EAAOhS,KAAOgS,EAAOviB,KAAK8iB,cAGvC8R,OAAO+B,IAAYre,OAAO3W,EAAOm6B,EAAgBnU,OAEhDlB,EAAc,GAChBC,EAAoB,YAGxBnmB,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAACtI,EAAOphE,MAC5BA,IAAQyrE,QACNoF,EAAczP,GAASqP,EAAaE,EAAcC,EAExDrV,EAAYv7D,GAAOohE,EAAMhU,OAAO3W,EAAOo6B,EAAapU,GADhC,GAEpBjB,EAAoBD,EAAYv7D,GAAKjG,WAGpByhE,EAAoB,sBAE/BsV,EAAgB,cAEbj3E,MACPw7C,SAAAA,GAAakmB,GAAa,SAACI,EAAK37D,GAC9B8wE,EAAc9wE,GAAO27D,EAAI9hE,SAFpBA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,MAOTw7C,SAAAA,GAAaopB,EAAKiL,QAAQ,SAACtI,EAAOphE,MAC5BA,IAAQyrE,QACNsF,EAAeD,EAAc9wE,GACnC41C,EAAUm7B,GAAgBn7B,EAC1BhQ,GAAUw7B,EAAMx7B,WAElBA,EAASrnC,KAAK8D,IAAIujC,EAAQ2tB,MAG5B3tB,EAASrnC,KAAK8D,IAAIujC,EAAQ2tB,GAG1BmS,EAAMn0C,MAAMqU,OAAU0tB,EAAO1tB,QAGxBwM,MAAMzM,MAAQ+/B,EAAMvf,iBACpB/T,MAAMxM,OAASA,OAGfuV,IAAIkK,KAAK9zB,MAAMsuB,IAAMyT,EAAuB,OAAflO,EAC7BroD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAC1D9iD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,QACpE7oC,KAAKmvB,QAAQguB,SACViB,IAAIkK,KAAK9zB,MAAMmgB,MAAQ,SAEvByJ,IAAIkK,KAAK9zB,MAAMkgB,KAAO,SAGxBmP,OAAO95C,IAAI,SAAS/F,IAAI,CAAC0c,KAAM1gB,KAAKmvB,QAAQqoC,2BAE5CpF,qBAAsB,EAE3BvZ,EAAU74C,KAAK8pD,cAAgBjR,4CAWzBo7B,EAAoD,OAAjCj0E,KAAKmvB,QAAQk5B,YAAYtQ,KAAiB,EAAK/3C,KAAKgxE,SAASh0E,OAAS,EACzFk3E,EAAel0E,KAAKgxE,SAASiD,UAChBj0E,KAAK2sE,OAAOuH,IAAiBl0E,KAAK2sE,OAAL,eAE3B,oDAUjB50B,EACAo8B,EAFAC,EAAYp0E,KAAK2sE,OAAL,iBAIZ3sE,KAAK+vE,eAEHqE,MAIGD,KAHLC,EAAU/7B,iBACHr4C,KAAK2sE,OAAL,cAEQ3sE,KAAKw2C,SACdx2C,KAAKw2C,MAAMn5C,eAAe82E,GAAS,EACrCp8B,EAAO/3C,KAAKw2C,MAAM29B,IACbvjD,QAAUmnB,EAAKnnB,OAAOqlB,OAAO8B,OAC5BokB,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GAC1BkI,GAASA,EAAMxpD,IAAIk9B,IAASA,EAAKmY,iBAOlCkkB,EAAW,KAMTD,KAHLC,EAAY,IAAIlY,GAFL,KACE,KACmBl8D,WAC3B2sE,OAAL,cAAyByH,EAEVp0E,KAAKw2C,MACdx2C,KAAKw2C,MAAMn5C,eAAe82E,KAC5Bp8B,EAAO/3C,KAAKw2C,MAAM29B,GAClBC,EAAUv5D,IAAIk9B,IAIlBq8B,EAAU5Q,qDAUPxjE,KAAKo+C,IAAIujB,0CAOTnrB,QACF2oB,iBAAmB,IAAIziE,SAExB61E,EADE5zB,EAAK3+C,KAELq0E,EAAer0E,KAAKylE,aAGrBjvB,EAGA,CAAA,KAAIA,aAAiBoB,IAAWpB,aAAiB89B,UAI9C,IAAItnD,UAAU,wDAHfy4C,UAAYluB,GAAkBf,aAH9BivB,UAAY,QASf4O,OAEF/7B,SAAAA,GAAat4C,KAAKiwE,eAAe,SAACjqE,EAAUw6C,GAC1C6zB,EAAa99B,IAAIiK,EAAOx6C,MAI1BquE,EAAah8B,UAGbk6B,EAAM8B,EAAaj8B,cACdk4B,UAAUiC,IAGbvyE,KAAKylE,UAAW,KAEZryC,EAAKpzB,KAAKozB,MAChBklB,SAAAA,GAAat4C,KAAKiwE,eAAe,SAACjqE,EAAUw6C,GAC1C7B,EAAG8mB,UAAUnvB,GAAGkK,EAAOx6C,EAAUotB,MAInCm/C,EAAMvyE,KAAKylE,UAAUrtB,cAChB+3B,OAAOoC,QAGPnB,wBAGFx4B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,8CAQjB,MAAlBryD,KAAKylE,UAAoBzlE,KAAKylE,UAAUjuB,MAAQ,uCAO/Cm1B,OAEJ4F,EADE5zB,EAAK3+C,QAIPA,KAAK+vE,gBACPz3B,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAACvqE,EAAUw6C,GAC3C7B,EAAGoxB,WAAWx5B,IAAIiK,EAAOx6C,MAI3BusE,EAAMvyE,KAAK+vE,WAAW33B,cACjB23B,WAAa,UACbgB,gBAAgBwB,IAIlB5F,EAGA,CAAA,KAAIA,aAAkB/0B,IAAW+0B,aAAkB2H,UAIhD,IAAItnD,UAAU,wDAHf+iD,WAAapD,YAHboD,WAAa,QAShB/vE,KAAK+vE,WAAY,OAEfA,EAAa/vE,KAAK+vE,WAClB/vE,KAAK+vE,sBAAsBuE,KAC7BvE,EAAa/vE,KAAK+vE,WAAW73B,mBAG/B63B,EAAWhmE,eAAc,SAAAs6D,SACnBA,EAAMvH,mBACRuH,EAAMvH,sBAAqB,SAAA6T,OACnBC,EAAqBb,EAAWhmE,IAAI4mE,GAC1CC,EAAmB1T,cAAgBmH,EAAMjxC,GACjB,GAApBixC,EAAMtH,aACR6T,EAAmBtV,SAAU,GAE/ByU,EAAW55B,OAAOy6B,aAMlBx9C,EAAKpzB,KAAKozB,MAChBklB,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAACvqE,EAAUw6C,GAC3C7B,EAAGoxB,WAAWz5B,GAAGkK,EAAOx6C,EAAUotB,MAIpCm/C,EAAMvyE,KAAK+vE,WAAW33B,cACjBo4B,aAAa+B,QAIfnB,wBAGAmD,SAEDv0E,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBoE,kBACjBzB,qBACAb,UAAU,CAAEC,cAAc,EAAMC,eAAe,UAGjDx5B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,+CAQnCryD,KAAK+vE,8CAOH38C,cACH2kB,EAAO/3C,KAAKylE,UAAU17D,IAAIqpB,GAE5B2kB,QAEG5oB,QAAQogD,SAASx3B,GAAM,SAAAA,GACtBA,GAGF08B,EAAKhP,UAAUxvB,OAAO7iB,uCAYrB4gC,UACAA,EAAS98B,MAAQl3B,KAAKmvB,QAAQ+H,OAAS88B,EAASvzB,IAAM,QAAU,0CAS9DuzB,SAEG,cADCh0D,KAAK00E,SAAS1gB,IACmBv0D,MAAlBu0D,EAASqQ,MAC7BqK,GAGC1uE,KAAK+vE,WAAa/b,EAASqQ,MA1lCtB,kDAmmCNkO,cACF5zB,EAAK3+C,QAEXuyE,QAAAA,GAAY,SAAAn/C,OAMN4wC,EALEhQ,EAAWrV,EAAG8mB,UAAU17D,IAAIqpB,GAC9B2kB,EAAO4G,EAAGnI,MAAMpjB,GACd8D,EAAO88B,EAAWrV,EAAG+1B,SAAS1gB,GAAY,KAE1C18B,EAAcq3C,EAAQgG,MAAMz9C,MAG9B6gB,IAEGzgB,GAAiBygB,aAAgBzgB,EAOpCqnB,EAAGi2B,YAAY78B,EAAMic,IALrBgQ,EAAWjsB,EAAKisB,SAChBrlB,EAAGk2B,YAAY98B,GACfA,EAAO,QAONA,GAAQic,EAAU,KAEjB18B,QAWI,IAAItK,uCAAgCkK,SAV1C6gB,EAAO,IAAIzgB,EAAY08B,EAAUrV,EAAGjD,WAAYiD,EAAGxvB,UAC9CiE,GAAKA,EAEVurB,EAAGm2B,SAAS/8B,GACRisB,IACF+Q,EAAK9D,UAAUh0E,KAAKm2B,GACpB2kB,EAAK66B,mBASR2B,SAEDv0E,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBjb,SAASn1D,KAAKw2C,MAAO,CAAEq2B,qBAAqB,SAC7DkG,sBAGFn6B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,sCAQlCkgB,OACJt5B,EAAQ,EACN0F,EAAK3+C,QACXuyE,QAAAA,GAAY,SAAAn/C,OACJ2kB,EAAO4G,EAAGnI,MAAMpjB,GAClB2kB,IACFkB,IACA0F,EAAGk2B,YAAY98B,OAIfkB,SAEGs7B,cACA37B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,yCAW5C/Z,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAAAtI,GACxBA,EAAMzG,mDASM2U,QACT/B,aAAa+B,wCAQPA,OACL5zB,EAAK3+C,QAEXuyE,QAAAA,GAAY,SAAAn/C,OACJq9C,EAAY9xB,EAAGoxB,WAAWhmE,IAAIqpB,GAChCixC,EAAQ1lB,EAAGguB,OAAOv5C,MAEjBixC,EA6BHA,EAAM3G,QAAQ+S,OA7BJ,IA/sCA,iBAitCNr9C,GAAmBA,GAAMs7C,SACrB,IAAItrE,kCAA2BgwB,6BAGjC4hD,EAAe3qC,GAAcsU,EAAGxvB,aASjC,IAAMglD,KARX77B,GAAKh7C,OAAO03E,EAAc,CACxBnsC,OAAQ,OAGVw7B,EAAQ,IAAInI,GAAM9oC,EAAIq9C,EAAW9xB,GACjCA,EAAGguB,OAAOv5C,GAAMixC,EAGK1lB,EAAGnI,SAClBmI,EAAGnI,MAAMn5C,eAAe82E,GAAS,KAC7Bp8B,EAAO4G,EAAGnI,MAAM29B,GAClBp8B,EAAKxkC,KAAK8wD,OAASjxC,GACrBixC,EAAMxpD,IAAIk9B,GAKhBssB,EAAMzG,QACNyG,EAAMb,gBAQL5qB,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,4CAQ5BkgB,iBACdA,QAAAA,GAAY,SAAAn/C,OACJixC,EAAQ4Q,EAAKtI,OAAOv5C,GAEtBixC,IACFA,EAAMhsB,iBACC48B,EAAKtI,OAAOv5C,OAInBpzB,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBoE,kBACjBzB,sBAGFb,UAAU,CAAEE,gBAAiBpyE,KAAKmvB,QAAQ2wC,eAC1ClnB,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,8CAStCryD,KAAK+vE,WAAY,KAEfiB,EAAWhxE,KAAK+vE,WAAW33B,OAAO,CACpCwlB,MAAO59D,KAAKmvB,QAAQ6/C,aAGtBgC,EAAWhxE,KAAKk1E,mBAAmBlE,OAE7B3wB,GAAW/H,GAAK68B,WAAWnE,EAAUhxE,KAAKgxE,aAC5C3wB,EAAS,KAELssB,EAAS3sE,KAAK2sE,UACpBqE,QAAAA,GAAiB,SAAA7U,GACfwQ,EAAOxQ,GAASjM,aAIlB8gB,QAAAA,GAAiB,SAAA7U,GACfwQ,EAAOxQ,GAASqH,eAGbwN,SAAWA,SAGX3wB,SAGA,6CAWQ2wB,8BASRoE,EAAuBr1E,EAAGixE,OAC7Bz4D,EAAS,aACby4D,QAAAA,GAAiB,SAAA7U,MACf5jD,EAAOtb,KAAKk/D,GACMp8D,EAAEgwE,WAAWhmE,IAAIoyD,GACrBW,aAAc,OACpBuY,EAAiBt+B,KAAAh3C,EAAEgwE,WAAWhmE,IAAI,CACtCkuB,gBAAOq9C,UACEA,EAAYpY,eAAiBf,GAEtCyB,MAAO79D,EAAEovB,QAAQ6/C,sBACZ,SAAAsG,UAAeA,EAAYliD,MAClC7a,EAASs4D,GAAAt4D,QAAAA,EAAc68D,EAAuBr1E,EAAGs1E,QAI9C98D,EAKF68D,CAAuBp1E,KAFV82C,GAAAk6B,QAAAA,GAAgB,SAAA7U,UAAYoZ,EAAKxF,WAAWhmE,IAAIoyD,GAASe,mDAUtEnlB,QACFvB,MAAMuB,EAAK3kB,IAAM2kB,MAGhBokB,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GAErBkI,EAEMA,GAASA,EAAM9wD,MAAQ8wD,EAAM9wD,KAAKwpD,aAC3ChlB,EAAKksB,cAAe,GAFpBlsB,EAAKksB,cAAe,EAKlBI,GAAOA,EAAMxpD,IAAIk9B,uCASXA,EAAMic,GAEhBjc,EAAK2lB,QAAQ1J,OAEPmI,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GACrBkI,EAEMA,GAASA,EAAM9wD,MAAQ8wD,EAAM9wD,KAAKwpD,aAC3ChlB,EAAKksB,cAAe,GAFpBlsB,EAAKksB,cAAe,sCAYZlsB,WAEVA,EAAKmY,cAGElwD,KAAKw2C,MAAMuB,EAAK3kB,QAGjBxO,EAAQgvC,UAAKqd,kBAAkBl5B,EAAK3kB,KAC5B,GAAVxO,GAAa09C,UAAK2O,kBAAiBrsD,EAAO,GAG9CmzB,EAAKnnB,QAAUmnB,EAAKnnB,OAAOqlB,OAAO8B,GAGhB,MAAd/3C,KAAKkxE,YACFA,MAAMhhB,oDAUM7pD,WACbs8D,EAAW,GAER7lE,EAAI,EAAGA,EAAIuJ,EAAMrJ,OAAQF,IAC5BuJ,EAAMvJ,aAAc4rE,IACtB/F,EAAS1lE,KAAKoJ,EAAMvJ,WAGjB6lE,mCAaAniB,QAEF8kB,YAAYvtB,KAAO/3C,KAAKw1E,eAAeh1B,QACvC8kB,YAAYmE,aAAejpB,EAAM7wB,OAAO85C,eAAgB,OACxDnE,YAAYqE,cAAgBnpB,EAAM7wB,OAAOg6C,gBAAiB,OAC1DrE,YAAYmQ,UAAY,4CAUhBtZ,OACN,IAAIr/D,EAAI,EAAGA,EAAIkD,KAAKgxE,SAASh0E,OAAQF,OAClCq/D,GAAWn8D,KAAKgxE,SAASl0E,GACzB,OAAOA,uCASN0jD,kBACPxgD,KAAKslE,YAAYC,oBAGjBlwB,EAFE0C,EAAO/3C,KAAKslE,YAAYvtB,MAAQ,KAChC4G,EAAK3+C,QAGP+3C,IAASA,EAAKisB,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqBzxB,MAAO,IAEjE/3C,KAAKmvB,QAAQmhC,SAASyU,gBACrB/kE,KAAKmvB,QAAQmhC,SAASkU,aACtBxkE,KAAKmvB,QAAQmhC,SAAS+V,sBAKL,MAAjBtuB,EAAKuY,WAAqBvY,EAAKuY,SAASkU,aAAezsB,EAAKuY,SAAS+V,cAClErmE,KAAKmvB,QAAQmhC,SAASyU,yBAIxB0E,EAAezpE,KAAKslE,YAAYmE,aAChCE,EAAgB3pE,KAAKslE,YAAYqE,sBAClCrE,YAAYC,gBAAiB,OAC7BD,YAAYoQ,aAAe39B,EAE5B0xB,EACFp0B,EAAQ,CACN0C,KAAM0xB,EACNkM,SAAUn1B,EAAMxB,OAAO70C,EACvB26D,UAAW,EACXvxD,KAAMvT,KAAK41E,eAAe79B,EAAKxkC,YAG5B+xD,YAAYmQ,UAAY,CAACpgC,QACzB,GAAIs0B,EACTt0B,EAAQ,CACN0C,KAAM4xB,EACNgM,SAAUn1B,EAAMxB,OAAO70C,EACvBu/D,WAAW,EACXn2D,KAAMvT,KAAK41E,eAAe79B,EAAKxkC,YAG5B+xD,YAAYmQ,UAAY,CAACpgC,QACzB,GAAIr1C,KAAKmvB,QAAQmhC,SAASz1C,MAAQ2lC,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,cAE3EC,oBAAoBv1B,OACpB,CACFxgD,KAAKgxE,SAASh0E,OAAS,QAGnBqzD,aAGD2lB,EAAiBh2E,KAAKi2E,eAAel+B,EAAKxkC,KAAK8wD,OAE/C6R,EAAel2E,KAAKmvB,QAAQq6C,qBAAqBzxB,OAASA,EAAKisB,SAAY,CAACjsB,EAAK3kB,IAAMpzB,KAAKm2E,oBAE7F7Q,YAAYmQ,UAAY1+B,GAAAm/B,QAAAA,GAAgB,SAAA9iD,OACrC2kB,EAAO4G,EAAGnI,MAAMpjB,GAChBgjD,EAAaz3B,EAAGs3B,eAAel+B,EAAKxkC,KAAK8wD,aACxC,CACLtsB,KAAAA,EACA49B,SAAUn1B,EAAMxB,OAAO70C,EACvBksE,YAAaL,EAAeI,EAC5B7iE,KAAM+iE,EAAKV,eAAe79B,EAAKxkC,UAKrCitC,EAAM8D,uBACGtkD,KAAKmvB,QAAQmhC,SAASz1C,MAAQ2lC,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,eAE3EC,oBAAoBv1B,gDASTA,OACZsQ,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAC5BylB,EAAYv2E,KAAKo+C,IAAIuqB,MAAM/lB,wBAG3Bz4C,EAAInK,KAAKmvB,QAAQguB,IAAMo5B,EAAU5hC,MAAQ6L,EAAMxB,OAAO70C,EAAK,GAAKq2C,EAAMxB,OAAO70C,EAAIosE,EAAU7hC,KAAO,GAElGh0B,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAC7BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBrwB,EAAQswB,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,EAGzCszC,EAAW,CACf98B,KAAM,QACNsJ,MAAAA,EACAC,IALUD,EAMV3M,QAAS,YAGLT,EAAKs3C,KACX1W,EAASh0D,KAAKylE,UAAUhuB,QAAUrkB,MAE5BixC,EAAQrkE,KAAKw2E,gBAAgBh2B,GAC/B6jB,IACFrQ,EAASqQ,MAAQA,EAAMlI,aAEnBsa,EAAU,IAAI/N,GAAU1U,EAAUh0D,KAAK07C,WAAY17C,KAAKmvB,SAC9DsnD,EAAQrjD,GAAKA,EACbqjD,EAAQljE,KAAOvT,KAAK41E,eAAe5hB,QAC9B8gB,SAAS2B,QACTnR,YAAYoQ,aAAee,MAE1BphC,EAAQ,CACZ0C,KAAM0+B,EACNd,SAAUn1B,EAAMxB,OAAO70C,EACvBoJ,KAAMkjE,EAAQljE,MAGZvT,KAAKmvB,QAAQguB,IACf9H,EAAMyvB,UAAW,EAEjBzvB,EAAMq0B,WAAY,OAEfpE,YAAYmQ,UAAY,CAACpgC,GAE9BmL,EAAM8D,kDAQA9D,iBACY,MAAdxgD,KAAKkxE,OAAiBlxE,KAAKmvB,QAAQygD,eAAiB5vE,KAAKkxE,MAAM32B,OAAQ,KAEnE0R,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAMxB,OAAO70C,EAAIusE,EAAchiC,KAAOuX,EAAU6S,WAChDte,EAAMxB,OAAOz1C,EAAImtE,EAAc5zB,IAAMmJ,EAAU4S,gBAE5CqS,MAAM1N,UAGTxjE,KAAKslE,YAAYmQ,UAAW,OAC9Bj1B,EAAM8D,sBAEA3F,EAAK3+C,KACL8wD,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAC5B8lB,EAAoB52E,KAAK44C,KAAKwF,IAAI+C,KAAK2d,WACvCt2B,EAAUxoC,KAAKmvB,QAAQguB,IAAMy5B,EAAoB52E,KAAK44C,KAAKY,SAAS7E,MAAM/L,MAAQguC,EAAoB52E,KAAK44C,KAAKY,SAAS9E,KAAK9L,MAC9H+S,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UAGtB6kB,EAAe11E,KAAKslE,YAAYoQ,aAChCmB,GAAuB72E,KAAKmvB,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,WAAqBtwD,KAAKmvB,QAAQmhC,SAAS+V,cACjGrmE,KAAKmvB,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,UAAoBolB,EAAaplB,SAAS+V,YACrHyQ,EAAe,QACfD,GAAsBnB,GACOj2E,MAA3Bi2E,EAAaniE,KAAK8wD,MAAoB,KAElCA,EAAQ1lB,EAAG63B,gBAAgBh2B,GAC7B6jB,IAGFyS,EAAe92E,KAAKi2E,eAAe5R,EAAMlI,oBAM1CmJ,YAAYmQ,mBAAkB,SAAApgC,OAG7B74B,EACAu6D,EACA5U,EACA3hC,EACAC,EANEwa,EAAU0D,EAAG/F,KAAKN,KAAK2D,OAAOuE,EAAMxB,OAAO70C,EAAIq+B,GAC/CwuC,EAAUr4B,EAAG/F,KAAKN,KAAK2D,OAAO5G,EAAMsgC,SAAWntC,GAQnDhsB,EADEy6D,EAAK9nD,QAAQguB,MACJlC,EAAU+7B,GAEX/7B,EAAU+7B,MAGlBhjB,EAAWijB,EAAKrB,eAAevgC,EAAM0C,KAAKxkC,SACnB,MAAvB8hC,EAAM0C,KAAKuY,UACTjb,EAAM0C,KAAKuY,SAASkU,YACpBnvB,EAAM0C,KAAKuY,SAAS+V,aACpB1nB,EAAGxvB,QAAQmhC,SAASyU,mBAIEkS,EAAK9nD,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,WAAqB2mB,EAAK9nD,QAAQmhC,SAASkU,aAChGyS,EAAK9nD,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,UAAoBolB,EAAaplB,SAASkU,cAEnHnvB,EAAMyvB,SAEJmS,EAAK9nD,QAAQguB,IACK19C,MAAhBu0D,EAASvzB,MACX0hC,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,QAC1CA,EAAM,IAAI/jC,KAAKylE,EAAW5kE,UAAYif,GAEtCw3C,EAASvzB,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,GAG3BhhC,MAAlBu0D,EAASxzB,QACXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAC9CA,EAAQ,IAAI9jC,KAAKq6E,EAAax5E,UAAYif,GAE1Cw3C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,QAIpD,GAAI6U,EAAMq0B,UAETuN,EAAK9nD,QAAQguB,IACO19C,MAAlBu0D,EAASxzB,QACXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAC9CA,EAAQ,IAAI9jC,KAAKq6E,EAAax5E,UAAYif,GAE1Cw3C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,GAGjC/gC,MAAhBu0D,EAASvzB,MACX0hC,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,QAC1CA,EAAM,IAAI/jC,KAAKylE,EAAW5kE,UAAYif,GAEtCw3C,EAASvzB,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,WAM7BhhC,MAAlBu0D,EAASxzB,SAEXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAAQjjC,UACtDijC,EAAQ,IAAI9jC,KAAKq6E,EAAev6D,GAEZ/c,MAAhBu0D,EAASvzB,IAAkB,KAEvBplB,GADN8mD,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,SACbljC,UAAYw5E,EAAax5E,UAGtDy2D,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EACnDwzB,EAASvzB,IAAQ,IAAI/jC,KAAKs3D,EAASxzB,MAAMjjC,UAAY8d,QAIrD24C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,KAMvDq2C,IAAwBxhC,EAAMyvB,WAAazvB,EAAMq0B,WAA4B,MAAdoN,GAC3Cr3E,MAAlBu0D,EAASqQ,MAAoB,KAC3B6S,EAAYJ,EAAezhC,EAAMghC,YAGrCa,EAAY11E,KAAK8D,IAAI,EAAG4xE,GACxBA,EAAY11E,KAAKY,IAAIu8C,EAAGqyB,SAASh0E,OAAO,EAAGk6E,GAC3CljB,EAASqQ,MAAQ1lB,EAAGqyB,SAASkG,GAKjCljB,EAAWijB,EAAKrB,eAAe5hB,GAC/BrV,EAAGxvB,QAAQqgD,SAASxb,GAAU,SAAAA,GACxBA,GACF3e,EAAM0C,KAAK2lB,QAAQuZ,EAAKrB,eAAe5hB,EAAU,qBAKlDpb,KAAKiF,QAAQ4C,KAAK,iDAUd1I,EAAMokB,OACXkI,EAAQrkE,KAAK2sE,OAAOxQ,MACtBkI,GAASA,EAAMlI,SAAWpkB,EAAKxkC,KAAK8wD,MAAO,KACvC8S,EAAWp/B,EAAKnnB,OACtBumD,EAASlhC,OAAO8B,GAChBo/B,EAASvZ,QAET7lB,EAAKxkC,KAAK8wD,MAAQA,EAAMlI,QAExBkI,EAAMxpD,IAAIk9B,GACVssB,EAAMzG,4CASCpd,sBACJ8kB,YAAYC,gBAAiB,EAC9BvlE,KAAKslE,YAAYmQ,UAAW,CAC9Bj1B,EAAM8D,sBAEA3F,EAAK3+C,KACLy1E,EAAYz1E,KAAKslE,YAAYmQ,eAC9BnQ,YAAYmQ,UAAY,QAE7BA,QAAAA,GAAkB,SAAApgC,OACVjiB,EAAKiiB,EAAM0C,KAAK3kB,MACiB,MAAxBurB,EAAG8mB,UAAU17D,IAAIqpB,GAc3B,KAEG4gC,EAAWojB,EAAKxB,eAAevgC,EAAM0C,KAAKxkC,MAChDorC,EAAGxvB,QAAQmgD,OAAOtb,GAAU,SAAAA,GACtBA,GAEFA,EAASojB,EAAK3R,UAAUhuB,QAAUrkB,EAClCgkD,EAAK3R,UAAUtvB,OAAO6d,KAItB3e,EAAM0C,KAAK2lB,QAAQroB,EAAM9hC,MAEzBorC,EAAG/F,KAAKiF,QAAQ4C,KAAK,oBAvBzB9B,EAAGxvB,QAAQigD,MAAM/5B,EAAM0C,KAAKxkC,MAAM,SAAAygD,GAChCrV,EAAGk2B,YAAYx/B,EAAM0C,MACjBic,GACFrV,EAAG8mB,UAAU5qD,IAAIm5C,GAInBrV,EAAG/F,KAAKiF,QAAQ4C,KAAK,wDA6BjBD,cACN6jB,EAAQrkE,KAAKw2E,gBAAgBh2B,OACxB,WACT62B,EAAKC,sBAAsBjT,KAC1B,iDAQiBA,SAAO9J,8DAAQ96D,KAE9B4kE,GAAUA,EAAMvH,kBAEfiT,EAAa/vE,KAAK+vE,WAAW73B,aAGjCmsB,EAAMtH,WADKt9D,MAAT86D,IACmBA,GAED8J,EAAMtH,eAGxBwa,EAAexH,EAAWhmE,IAAIs6D,EAAMlI,SACxCob,EAAaxa,WAAasH,EAAMtH,mBAE5Bya,EAAmBnT,EAAMvH,aACzB2a,EAAYD,EACTC,EAAUz6E,OAAS,GAAG,KACvBi+C,EAAUw8B,EACdA,EAAY,OACP,IAAI36E,EAAI,EAAGA,EAAIm+C,EAAQj+C,OAAQF,IAAK,KACnC46E,EAAO3H,EAAWhmE,IAAIkxC,EAAQn+C,IAC9B46E,EAAK5a,eACP2a,EAAY5G,GAAA4G,QAAAA,EAAiBC,EAAK5a,eAGlC2a,EAAUz6E,OAAS,IACrBw6E,EAAmB3G,GAAA2G,QAAAA,EAAwBC,QAG3C3a,EAAe/lB,KAAAg5B,EAAWhmE,IAAIytE,YAAsB,SAAUlC,UACnC71E,MAAvB61E,EAAYha,UACdga,EAAYha,SAAU,GAExBga,EAAYha,UAAYic,EAAaxa,WAC9BuY,KAGXvF,EAAW55B,OAAO06B,GAAA/T,QAAAA,EAAoBya,IAElCA,EAAaxa,YACfzkB,GAAKwV,gBAAgBuW,EAAMjmB,IAAI7V,MAAO,aACtC+P,GAAKuV,aAAawW,EAAMjmB,IAAI7V,MAAO,cAEnC+P,GAAKwV,gBAAgBuW,EAAMjmB,IAAI7V,MAAO,YACtC+P,GAAKuV,aAAawW,EAAMjmB,IAAI7V,MAAO,gEAQd87B,GACvBA,EAAMjmB,IAAI7V,MAAMovC,UAAUC,OAAO,yBACjCvT,EAAMjmB,IAAI4J,WAAW2vB,UAAUC,OAAO,mEAStBp3B,GACZxgD,KAAK89D,iBAAiBC,YAEtB/9D,KAAKmvB,QAAQwuC,cAAcC,aACxBE,iBAAiBuG,MAAQrkE,KAAKw2E,gBAAgBh2B,GAE/CxgD,KAAK89D,iBAAiBuG,QACxB7jB,EAAM8D,uBAEDwZ,iBAAiBC,YAAa,OAC9B8Z,yBAAyB73E,KAAK89D,iBAAiBuG,YAE/CvG,iBAAiBga,cAAgB93E,KAAK+vE,WAAW33B,OAAO,CAC3DwlB,MAAO59D,KAAKmvB,QAAQ6/C,oDAYfxuB,MACLxgD,KAAKmvB,QAAQwuC,cAAcC,OAAS59D,KAAK89D,iBAAiBuG,MAAO,CACjE7jB,EAAM8D,sBAENyrB,EAAa/vE,KAAK+vE,WAClB/vE,KAAK+vE,sBAAsBuE,KAC7BvE,EAAa/vE,KAAK+vE,WAAW73B,kBAGrBmsB,EAAQrkE,KAAKw2E,gBAAgBh2B,MAG/B6jB,GAASA,EAAMx7B,QAAU7oC,KAAK89D,iBAAiBuG,MAAMx7B,OAAQ,KACvDkvC,EAAY1T,EAAMvhB,IAAM9iD,KAAK89D,iBAAiBuG,MAAMvhB,IACpDV,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChD41B,EAAc3T,EAAMjmB,IAAI4J,WAAWpF,wBACnCq1B,EAAqBj4E,KAAK89D,iBAAiBuG,MAAMx7B,UACnDkvC,MAEIC,EAAYl1B,IAAMm1B,EAAqB71B,aAGxC,KACG81B,EAAoB7T,EAAMx7B,UAE5BmvC,EAAYl1B,IAAMo1B,EAAoBD,EAAqB71B,aAMnEiiB,GAASA,GAASrkE,KAAK89D,iBAAiBuG,MAAO,KACzC2T,EAAcjI,EAAWhmE,IAAIs6D,EAAMlI,SACnCgc,EAAepI,EAAWhmE,IAAI/J,KAAK89D,iBAAiBuG,MAAMlI,SAG5Dgc,GAAgBH,SACX7oD,QAAQy/C,eAAeuJ,EAAcH,EAAajI,GACvDA,EAAW55B,OAAOgiC,GAClBpI,EAAW55B,OAAO6hC,QAIhBI,EAAWrI,EAAW33B,OAAO,CACjCwlB,MAAO59D,KAAKmvB,QAAQ6/C,iBAKjB12B,GAAK68B,WAAWiD,EAAUp4E,KAAK89D,iBAAiBga,uBAC3CO,EAAYr4E,KAAK89D,iBAAiBga,cAClCQ,EAAYt4E,KAAK89D,iBAAiBuG,MAAMlI,QACxCoc,EAAY/2E,KAAKY,IAAIi2E,EAAUr7E,OAAQo7E,EAASp7E,QAClDw7E,EAAS,EACTtB,EAAY,EACZuB,EAAY,EACTD,EAASD,GAAW,MAEfC,EAAOtB,EAAaqB,GACpBC,EAAOC,EAAaF,GACrBH,EAASI,EAAOtB,IAAcmB,EAAUG,EAAOC,IAClDD,OAIAA,EAAOtB,GAAaqB,WAMpBH,EAASI,EAAOtB,IAAcoB,EAC9BpB,EAAY,OAIX,GAAImB,EAAUG,EAAOC,IAAcH,EACpCG,EAAY,MAKX,KACKC,EAAkB9kB,GAAAwkB,QAAAA,EAAiBC,EAAUG,EAAOC,IACpDE,EAAc5I,EAAWhmE,IAAIquE,EAASI,EAAOtB,IAC7C0B,EAAgB7I,EAAWhmE,IAAIsuE,EAAUG,EAAOC,SACjDtpD,QAAQy/C,eAAe+J,EAAaC,EAAe7I,GACxDA,EAAW55B,OAAOwiC,GAClB5I,EAAW55B,OAAOyiC,OAEZC,EAAgBT,EAASI,EAAOtB,GACtCkB,EAASI,EAAOtB,GAAamB,EAAUG,EAAOC,GAC9CL,EAASM,GAAmBG,EAE5BL,gDAeRh4B,WACTsd,iBAAiBC,YAAa,EAE/B/9D,KAAKmvB,QAAQwuC,cAAcC,OAAS59D,KAAK89D,iBAAiBuG,MAAO,CACnE7jB,EAAM8D,sBAGA3F,EAAK3+C,KACLozB,EAAKurB,EAAGmf,iBAAiBuG,MAAMlI,QAC/B2c,EAAUn6B,EAAGoxB,WAAW73B,aACxBu4B,EAAYn4B,GAAKh7C,OAAO,GAAIw7E,EAAQ/uE,IAAIqpB,IAC9CurB,EAAGxvB,QAAQugD,YAAYe,GAAW,SAAAA,MAC5BA,EAEFA,EAAUqI,EAAQphC,SAAWtkB,EAC7B0lD,EAAQ3iC,OAAOs6B,OAEZ,KAGG2H,EAAWU,EAAQ1gC,OAAO,CAC5BwlB,MAAOjf,EAAGxvB,QAAQ6/C,iBAIjB12B,GAAK68B,WAAWiD,EAAUz5B,EAAGmf,iBAAiBga,uBAC3CO,EAAY15B,EAAGmf,iBAAiBga,cAChCS,EAAY/2E,KAAKY,IAAIi2E,EAAUr7E,OAAQo7E,EAASp7E,QAClDw7E,EAAS,EACNA,EAASD,GAAW,MAElBC,EAASD,GAAaH,EAASI,IAAWH,EAAUG,IACzDA,OAIEA,GAAUD,YAMRG,EAAkB9kB,GAAAwkB,QAAAA,EAAiBC,EAAUG,IAC7CG,EAAcG,EAAQ/uE,IAAIquE,EAASI,IACnCI,EAAgBE,EAAQ/uE,IAAIsuE,EAAUG,IAC5C75B,EAAGxvB,QAAQy/C,eAAe+J,EAAaC,EAAeE,GACtDA,EAAQ3iC,OAAOwiC,GACfG,EAAQ3iC,OAAOyiC,OAETC,EAAgBT,EAASI,GAC/BJ,EAASI,GAAUH,EAAUG,GAC7BJ,EAASM,GAAmBG,EAE5BL,SAMR75B,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgB,CAAE0b,QAAS/oC,SAC3CykD,yBAAyB73E,KAAK89D,iBAAiBuG,YAC/CvG,iBAAiBuG,MAAQ,4CASpB7jB,MACPxgD,KAAKmvB,QAAQ+0C,gBAEZ2R,EAAWr1B,EAAM4D,WAAa5D,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,SACvEhpB,EAAWtM,EAAM4D,UAAY5D,EAAM4D,SAAS0I,YAC9C+oB,GAAW/oB,OACR0kB,mBAAmBhxB,YAIpBu4B,EAAe/4E,KAAKm2E,eAEpBp+B,EAAO/3C,KAAKw1E,eAAeh1B,GAC3BywB,EAAYl5B,GAAQA,EAAKmsB,WAAa,CAACnsB,EAAK3kB,IAAM,QACnD4lD,aAAa/H,OAEZgI,EAAej5E,KAAKm2E,gBAItB8C,EAAaj8E,OAAS,GAAK+7E,EAAa/7E,OAAS,SAC9C47C,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOyiC,EACPz4B,MAAAA,2CAUOA,OACLzI,EAAO/3C,KAAKw1E,eAAeh1B,MAC5BzI,GAIDA,IADY/3C,KAAKk5E,sBAAsB14B,QAMrCiP,EAAQ1X,EAAKohC,cACfn5E,KAAKmvB,QAAQygD,cAAgBngB,EAAO,CACpB,MAAdzvD,KAAKkxE,aACFA,MAAQ,IAAIpH,GAAM9pE,KAAK44C,KAAKwF,IAAI+C,KACjCnhD,KAAKmvB,QAAQm3C,QAAQyD,gBAAkB,cAGxCmH,MAAMkI,QAAQ3pB,OACbxD,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAM2B,QAAUu0B,EAAchiC,KAAOuX,EAAU6S,WAC/Cte,EAAM4B,QAAUs0B,EAAc5zB,IAAMmJ,EAAU4S,gBAE3Cwa,cAAcr5E,KAAKkxE,iBAInBoB,kBACa,MAAdtyE,KAAKkxE,YACFA,MAAMhhB,YAIVtX,KAAKiF,QAAQ4C,KAAK,WAAY,CACjC1I,KAAMA,EAAK3kB,GACXotB,MAAAA,yCAUQA,OACJzI,EAAO/3C,KAAKw1E,eAAeh1B,GAC5BzI,IAIDA,IADY/3C,KAAKk5E,sBAAsB14B,UAMtC8xB,kBACa,MAAdtyE,KAAKkxE,YACFA,MAAMhhB,YAGRtX,KAAKiF,QAAQ4C,KAAK,UAAW,CAChC1I,KAAMA,EAAK3kB,GACXotB,MAAAA,2CAUSA,MACExgD,KAAKw1E,eAAeh1B,KAGV,MAAnBxgD,KAAKmxE,iBAEFkI,cAAcr5E,KAAKkxE,OAGtBlxE,KAAKmvB,QAAQygD,cAAgB5vE,KAAKmvB,QAAQm3C,QAAQuJ,aAAe7vE,KAAKkxE,QAAUlxE,KAAKkxE,MAAM32B,QAAQ,KAC/F0R,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAM2B,QAAUu0B,EAAchiC,KAAOuX,EAAU6S,WAC/Cte,EAAM4B,QAAUs0B,EAAc5zB,IAAMmJ,EAAU4S,gBAE3CqS,MAAM1N,8CASDhjB,GACRxgD,KAAKslE,YAAYC,qBACdvnB,WAAWwC,yCASNzI,MACP/3C,KAAKmvB,QAAQ+0C,aACblkE,KAAKmvB,QAAQmhC,SAASkU,YAAexkE,KAAKmvB,QAAQmhC,SAAS+V,kBAE1D1nB,EAAK3+C,QAEP+3C,EAAM,KAEFic,EAAWrV,EAAG8mB,UAAU17D,IAAIguC,EAAK3kB,SAClCjE,QAAQkgD,SAASrb,GAAU,SAAAA,GAC1BA,GACFrV,EAAG8mB,UAAUtvB,OAAO6d,oDAYRxT,OACZzI,EAAO/3C,KAAKw1E,eAAeh1B,GAC3B2uB,EAAa3pC,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,cACpD9kC,QAAQ+/C,mBAAmBC,EAAYp3B,sCAQnCyI,MACJxgD,KAAKmvB,QAAQ+0C,YACblkE,KAAKmvB,QAAQmhC,SAASz1C,SAWvB4lB,EAEA64C,EAXE36B,EAAK3+C,KACL8wD,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAG5BylB,EAAYv2E,KAAKo+C,IAAIuqB,MAAM/lB,wBAC3Bz4C,EAAInK,KAAKmvB,QAAQguB,IAAMo5B,EAAU5hC,MAAQ6L,EAAMxB,OAAO70C,EAAIq2C,EAAMxB,OAAO70C,EAAIosE,EAAU7hC,KACrFlU,EAAQxgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAC9BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UAIV,QAAdrQ,EAAMtpB,OACRoiD,EAAc9zC,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,UACxCpgC,QAAUylD,EAAYzlD,QAAUylD,EAAYzlD,QAAU,WAClEylD,EAAY94C,MAAQ84C,EAAY94C,MAAQ84C,EAAY94C,MAASswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EAC/F84C,EAAYpiD,KAAOoiD,EAAYpiD,MAAQ,MACvCoiD,EAAYt5E,KAAKylE,UAAUhuB,QAAU6hC,EAAYlmD,IAAMs3C,KAE/B,SAApB4O,EAAYpiD,MAAoBoiD,EAAY74C,MAC9CA,EAAMzgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,EAAInK,KAAKq1C,MAAMzM,MAAQ,GACnD0wC,EAAY74C,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,MAGpD64C,EAAc,CACZ94C,MAAOswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EACzC3M,QAAS,aAEC7zB,KAAKylE,UAAUhuB,QAAUizB,KAGX,UAAtB1qE,KAAKmvB,QAAQ+H,OACfuJ,EAAMzgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,EAAInK,KAAKq1C,MAAMzM,MAAQ,GACnD0wC,EAAY74C,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,QAIhD4jC,EAAQrkE,KAAKw2E,gBAAgBh2B,GAC/B6jB,IACFiV,EAAYjV,MAAQA,EAAMlI,SAI5Bmd,EAAct5E,KAAK41E,eAAe0D,QAC7BnqD,QAAQigD,MAAMkK,GAAa,SAAAvhC,GAC1BA,IACF4G,EAAG8mB,UAAU5qD,IAAIk9B,GACC,QAAdyI,EAAMtpB,MACRynB,EAAGq6B,aAAa,CAACjhC,EAAK3kB,qDAYXotB,iBACZxgD,KAAKmvB,QAAQ+0C,gBAEZnsB,EAAO/3C,KAAKw1E,eAAeh1B,MAE7BzI,EAAM,KAGJk5B,EAAYjxE,KAAKmvB,QAAQ8/C,YACzBjvE,KAAKm2E,eACL,OAEa31B,EAAM4D,UAAY5D,EAAM4D,SAAS0I,WAAY,GAE7C9sD,KAAKmvB,QAAQ6gD,sBAAwBhwE,KAAKmvB,QAAQ8/C,YAAa,KAExEsK,EAAYv5E,KAAKylE,UAAU17D,IAAIguC,EAAK3kB,IAAIixC,MAG1CmV,OAAoB/5E,EACpBO,KAAKmvB,QAAQsqD,qBACXxI,EAAUj0E,OAAS,IACrBw8E,EAAoBx5E,KAAKylE,UAAU17D,IAAIknE,EAAU,IAAI5M,OAKpDrkE,KAAKmvB,QAAQsqD,qBAA4Ch6E,MAArB+5E,GAAkCA,GAAqBD,GAC9FtI,EAAUh0E,KAAK86C,EAAK3kB,QAEhBsmB,EAAQi1B,EAAQ+K,cAAc15E,KAAKylE,UAAU17D,IAAIknE,QAElDjxE,KAAKmvB,QAAQsqD,qBAAuBD,GAAqBD,MAGvD,IAAMnmD,KADX69C,EAAY,GACKjxE,KAAKw2C,SAChBx2C,KAAKw2C,MAAMn5C,eAAe+1B,GAAK,KAC3BumD,EAAQ35E,KAAKw2C,MAAMpjB,GACnBoN,EAAQm5C,EAAMpmE,KAAKitB,MACnBC,OAA0BhhC,IAAnBk6E,EAAMpmE,KAAKktB,IAAqBk5C,EAAMpmE,KAAKktB,IAAMD,IAE1DA,GAASkZ,EAAMt3C,KACfq+B,GAAOiZ,EAAMp0C,MACXtF,KAAKmvB,QAAQsqD,qBAAuBD,GAAqBx5E,KAAKylE,UAAU17D,IAAI4vE,EAAMvmD,IAAIixC,OACtFsV,aAAiB/P,IACrBqH,EAAUh0E,KAAK08E,EAAMvmD,SAM1B,KAEGxO,EAAQgvC,GAAAqd,QAAAA,EAAkBl5B,EAAK3kB,KACvB,GAAVxO,EAEFqsD,EAAUh0E,KAAK86C,EAAK3kB,OAIpB69C,QAAAA,EAAiBrsD,EAAO,OAItBg1D,EAAoB9iC,GAAAm6B,QAAAA,GAAiB,SAAAl5B,UAAQ8hC,EAAKnH,YAAY36B,GAAMmsB,mBAErE8U,aAAaY,QAEbhhC,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOx2C,KAAKm2E,eACZ31B,MAAAA,8CA4CU5Y,WACVkyC,EAAMlyC,EACHkyC,GAAK,IACNA,EAAIz8E,eAAe,mBACdy8E,EAAI,YAEbA,EAAMA,EAAIvyC,kBAGL,4CASMiZ,UACNxgD,KAAK+5E,gBAAgBv5B,EAAM7wB,sDASd6wB,UACbxgD,KAAK+5E,gBAAgBv5B,EAAMw5B,uDASpBx5B,OACR4B,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAClD4uB,EAAWhxE,KAAKgxE,SAEhBA,EAASh0E,QAAU,GAAKgD,KAAK+vE,aAC/BiB,EAAWhxE,KAAK+vE,WAAW33B,OAAO,CAChCwlB,MAAO59D,KAAKmvB,QAAQ6/C,kBAInB,IAAIlyE,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KAClCq/D,EAAU6U,EAASl0E,GACnBunE,EAAQrkE,KAAK2sE,OAAOxQ,GACpBnU,EAAaqc,EAAMjmB,IAAI4J,WACvBiyB,EAAiBjyB,EAAWpF,2BAC9BR,GAAW63B,EAAen3B,KAAOV,EAAU63B,EAAen3B,IAAMkF,EAAW0O,oBACtE2N,KAG6B,QAAlCrkE,KAAKmvB,QAAQk5B,YAAYtQ,SACvBj7C,IAAMkD,KAAKgxE,SAASh0E,OAAS,GAAKolD,EAAU63B,EAAen3B,WACtDuhB,UAIC,IAANvnE,GAAWslD,EAAU63B,EAAen3B,IAAMkF,EAAWxrC,cAChD6nD,SAKN,4CA8BMrQ,EAAU98B,OACjB9Z,EAAQk7B,GAAKh7C,OAAO,GAAI02D,UAEzB98B,IAEHA,EAAOl3B,KAAKylE,UAAUvuC,MAGLz3B,MAAf2d,EAAMojB,QACRpjB,EAAMojB,MAAQ8X,GAAKpB,QAAQ95B,EAAMojB,MAAOtJ,GAAQA,EAAKsJ,OAAS,SAE/C/gC,MAAb2d,EAAMqjB,MACRrjB,EAAMqjB,IAAM6X,GAAKpB,QAAQ95B,EAAMqjB,IAAMvJ,GAAQA,EAAKuJ,KAAO,SAGpDrjB,6CASFpd,KAAKmvB,QAAQ2wC,aAIVnkB,EAAU37C,KAAK44C,KAAKc,MAAMgC,WAAW17C,KAAK44C,KAAKY,SAASwF,OAAOpW,OAA/D+S,MACF4xB,EAAWvtE,KAAKowE,iBAAiB8J,YAAYl6E,KAAKutE,SAAU5xB,EAAO37C,KAAKmvB,QAAQ2wC,YAElF9/D,KAAKutE,UAAYA,EAAU,SACxB8E,qBAED9E,EAAU,2CACQA,+BAAU,SACpBxC,iGAELwC,SAAWA,OAGb4M,gBAAgB5M,qDASnBvtE,KAAKmvB,QAAQ2wC,SACX9/D,KAAKutE,UAAYvtE,KAAKutE,SAASvwE,OAAQ,2CACrBgD,KAAKutE,sCAAU,SACzBzC,sIAWAyC,MACVvtE,KAAKutE,UAAYvtE,KAAKutE,SAASvwE,OAAQ,OACnCo9E,EAAiB,OAAQrjC,GAAAw2B,QAAAA,GAAa,SAAAzN,UAAWA,EAAQ1sC,OACzDinD,EAAqBvjC,UAAKy2B,kBAAgB,SAAAzN,UAAYsa,EAAetsD,IAAIgyC,EAAQ1sC,OACnFknD,GAAmB,wCACHD,+BAAoB,SAA/Bva,UACDya,EAAc3mB,UAAKqd,kBAAkBnR,EAAQ1sC,QAC9B,IAAjBmnD,EACFza,EAAQ6S,qBACH1B,kBAAiBsJ,EAAa,GACnCD,GAAmB,wFAInBA,EAAkB,KACdrB,EAAej5E,KAAKm2E,oBACrBv9B,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOyiC,EACPz4B,MAAOA,cAKR+sB,SAAWA,GAAY,2CA3NT9H,OACfngE,EAAM,KACNlD,EAAM,eAEVqjE,QAAAA,GAAkB,SAAAlyD,IACL,MAAPnR,GAAemR,EAAKitB,MAAQp+B,KAC9BA,EAAMmR,EAAKitB,OAGG/gC,MAAZ8T,EAAKktB,KACI,MAAPn7B,GAAeiO,EAAKktB,IAAMn7B,KAC5BA,EAAMiO,EAAKktB,MAIF,MAAPn7B,GAAeiO,EAAKitB,MAAQl7B,KAC9BA,EAAMiO,EAAKitB,UAKV,CACLp+B,IAAAA,EACAkD,IAAAA,6CAwFqBk7C,WACnB7wB,EAAS6wB,EAAM7wB,OACZA,GAAQ,IACTA,EAAOtyB,eAAe,sBACjBsyB,EAAO,eAEhBA,EAASA,EAAO4X,kBAGX,YA9/EWoR,IAsmFtBg2B,GAAQgG,MAAQ,CACd/rB,WAAYghB,GACZ/E,IAAK0B,GACL7sB,MAAOgvB,GACP5oC,MAAOsoC,IAQTuG,GAAQxyE,UAAUg0E,OAASxB,GAAQxyE,UAAUk0E,UChpF7C,IACImK,GADAC,IAAa,EAEbC,GAAa,sCAIXC,uFAeYxrD,EAASyrD,EAAkBC,GACzCJ,IAAa,EACbD,GAAaI,MACTE,EAAcF,cACAn7E,IAAdo7E,IACFC,EAAcF,EAAiBC,IAEjCF,EAAUvjC,MAAMjoB,EAAS2rD,EAAa,IAC/BL,iCAWItrD,EAASyrD,EAAkB3qD,OACjC,IAAIqmC,KAAUnnC,EACbA,EAAQ9xB,eAAei5D,IACzBqkB,EAAUxwD,MAAMmsC,EAAQnnC,EAASyrD,EAAkB3qD,iCAc5CqmC,EAAQnnC,EAASyrD,EAAkB3qD,WACbxwB,IAA7Bm7E,EAAiBtkB,SAAsD72D,IAA7Bm7E,EAAiBG,aAK3DC,EAAkB1kB,EAClB2kB,GAAY,OAEiBx7E,IAA7Bm7E,EAAiBtkB,SAAsD72D,IAA7Bm7E,EAAiBG,UAM7DC,EAAkB,UAIlBC,EAAoD,WAAvCN,EAAUrjC,QAAQnoB,EAAQmnC,SAQrC4kB,EAAeN,EAAiBI,GAChCC,QAAuCx7E,IAA1By7E,EAAaC,WAC5BD,EAAeA,EAAaC,UAG9BR,EAAUS,YAAY9kB,EAAQnnC,EAASyrD,EAAkBI,EAAiBE,EAAcjrD,QA9BtF0qD,EAAUU,cAAc/kB,EAAQskB,EAAkB3qD,uCA2CnCqmC,EAAQnnC,EAASyrD,EAAkBI,EAAiBE,EAAcjrD,OAC/E+pB,EAAM,SAASshC,GACjB34E,QAAQq3C,IAAI,KAAOshC,EAAUX,EAAUY,cAActrD,EAAMqmC,GAASokB,KAGlEc,EAAab,EAAUrjC,QAAQnoB,EAAQmnC,IACvCmlB,EAAgBP,EAAaM,QAEX/7E,IAAlBg8E,EAEuC,UAArCd,EAAUrjC,QAAQmkC,KAA0E,IAA5C7nB,GAAA6nB,QAAAA,EAAsBtsD,EAAQmnC,KAChFtc,EAAI,+BAAiCsc,EAAS,yBACnBqkB,EAAUe,MAAMD,GACzC,SAAWtsD,EAAQmnC,GAAU,OAC/BmkB,IAAa,GAES,WAAfe,GAA+C,YAApBR,IAClC/qD,EAAOqoB,GAAKqjC,mBAAmB1rD,EAAMqmC,GACrCqkB,EAAUvjC,MAAMjoB,EAAQmnC,GAASskB,EAAiBI,GAAkB/qD,SAGvCxwB,IAAxBy7E,EAAY,MAEnBlhC,EAAI,8BAAgCsc,EAClC,gBAAkBqkB,EAAUe,MAAM17B,GAAYk7B,IAC9C,eAAkBM,EAAa,MAAQrsD,EAAQmnC,GAAU,KAC3DmkB,IAAa,mCAUF3rD,OACToI,KAAcpI,SAEL,WAAToI,EACa,OAAXpI,EACK,OAELA,aAAkBmT,QACb,UAELnT,aAAkBuoB,OACb,SAELvoB,aAAkBL,OACb,SAELyU,GAAcpU,GACT,QAELA,aAAkBpyB,KACb,YAEe+C,IAApBqvB,EAAO8sD,SACF,OAEuB,IAA5B9sD,EAAOtuB,iBACF,SAEF,SAES,WAAT02B,EACA,SAES,YAATA,EACA,UAES,WAATA,EACA,cAESz3B,IAATy3B,EACA,YAEFA,wCASYo/B,EAAQnnC,EAASc,OAOhCxtB,EANAo5E,EAAclB,EAAUmB,cAAcxlB,EAAOnnC,EAAQc,GAAK,GAC1D8rD,EAAepB,EAAUmB,cAAcxlB,EAAOkkB,GAAW,IAAG,GAO9D/3E,OAD6BhD,IAA3Bo8E,EAAYG,WACR,OAASrB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,EAAO,IAC9D,6CAA+CulB,EAAYG,WAAa,SAEnED,EAAa5lB,UAPM,GAO+B0lB,EAAY1lB,SAAW4lB,EAAa5lB,SACvF,OAASwkB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,EAAO,IAC9D,uDACAqkB,EAAUY,cAAcQ,EAAa9rD,KAAM8rD,EAAaE,aAAa,IAEhEJ,EAAY1lB,UAbM,EAcnB,mBAAqB0lB,EAAYI,aAAe,KACpDtB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,GAGtC,gCAAkCqkB,EAAUe,MAAM17B,GAAY7wB,IACpEwrD,EAAUY,cAActrD,EAAMqmC,GAGhC3zD,QAAQq3C,IAAI,+BAAiCsc,EAAS,IAAM7zD,EAAKi4E,IACjED,IAAa,wCAYMnkB,EAAQnnC,EAASc,OAAMisD,0DACtC95E,EAAM,IACN65E,EAAe,GACfE,EAAmB,GACnBC,EAAkB9lB,EAAOhyD,cACzB03E,OAAav8E,MACZ,IAAI48E,KAAMltD,EAAS,KAClBgnC,iBACyB12D,IAAzB0vB,EAAQktD,GAAIlB,WAAwC,IAAde,EAAoB,KACxD3jE,EAASoiE,EAAUmB,cAAcxlB,EAAQnnC,EAAQktD,GAAK/jC,GAAKqjC,mBAAmB1rD,EAAKosD,IACnFj6E,EAAMmW,EAAO49C,WACf8lB,EAAe1jE,EAAO0jE,aACtBE,EAAmB5jE,EAAO0X,KAC1B7tB,EAAMmW,EAAO49C,SACb6lB,EAAazjE,EAAOyjE,gBAGnB,QACgD,IAA/CpoB,KAAAyoB,EAAG/3E,sBAAsB83E,KAC3BJ,EAAaK,GAGXj6E,GADJ+zD,EAAWwkB,EAAU2B,oBAAoBhmB,EAAQ+lB,MAE/CJ,EAAeI,EACfF,EAAmB7jC,GAAKikC,UAAUtsD,GAClC7tB,EAAM+zD,UAIL,CAAC8lB,aAAaA,EAAchsD,KAAKksD,EAAkBhmB,SAAS/zD,EAAK45E,WAAYA,yCAUjE/rD,EAAMqmC,WAAQpzC,yDAAS,6BACtC2I,EAAM,OAAS3I,EAAS,gBACnBpmB,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IAAK,KAC/B,IAAIiX,EAAI,EAAGA,EAAIjX,EAAI,EAAGiX,IACzB8X,GAAO,KAETA,GAAOoE,EAAKnzB,GAAK,YAEd,IAAIiX,EAAI,EAAGA,EAAIkc,EAAKjzB,OAAS,EAAG+W,IACnC8X,GAAO,KAETA,GAAOyqC,EAAS,SACX,IAAIx5D,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAS,EAAGF,IAAK,KACnC,IAAIiX,EAAI,EAAGA,EAAIkc,EAAKjzB,OAASF,EAAGiX,IACnC8X,GAAO,KAETA,GAAO,aAEFA,EAAM,qCAQFsD,UACJ4wB,GAAe5wB,GAAS5oB,QAAQ,gCAAiC,IAAIA,QAAQ,QAAS,kDAqBpEpJ,EAAGC,MACX,IAAbD,EAAEH,OAAc,OAAOI,EAAEJ,UACZ,IAAbI,EAAEJ,OAAc,OAAOG,EAAEH,WAKzBF,EAMAiX,EATAyoE,EAAS,OAIR1/E,EAAI,EAAGA,GAAKM,EAAEJ,OAAQF,IACzB0/E,EAAO1/E,GAAK,CAACA,OAKViX,EAAI,EAAGA,GAAK5W,EAAEH,OAAQ+W,IACzByoE,EAAO,GAAGzoE,GAAKA,MAIZjX,EAAI,EAAGA,GAAKM,EAAEJ,OAAQF,QACpBiX,EAAI,EAAGA,GAAK5W,EAAEH,OAAQ+W,IACrB3W,EAAEopB,OAAO1pB,EAAI,IAAMK,EAAEqpB,OAAOzS,EAAI,GAClCyoE,EAAO1/E,GAAGiX,GAAKyoE,EAAO1/E,EAAI,GAAGiX,EAAI,GAEjCyoE,EAAO1/E,GAAGiX,GAAKvS,KAAKY,IAAIo6E,EAAO1/E,EAAI,GAAGiX,EAAI,GAAK,EAC7CvS,KAAKY,IAAIo6E,EAAO1/E,GAAGiX,EAAI,GAAK,EAC1ByoE,EAAO1/E,EAAI,GAAGiX,GAAK,WAKtByoE,EAAOp/E,EAAEJ,QAAQG,EAAEH,iBCrV1Bia,GAAS,SACTwlE,GAAO,UACPl7E,GAAS,SAGTutB,GAAS,SAKT0rD,GAAa,CACfxlB,UAAW,CACT0nB,QAAS,SAAaD,IACtBxkD,OAAQ,SAAawkD,YAAiB,YACtCxwB,UAAW,CAAC7N,IARN,OASN+8B,SAAU,CAACrsD,OAAAA,WAAmB2tD,YAAiB,aAIjDhW,MAAO,CAACxvD,OAAAA,IACR2hD,iBAAkB,CAAC3hD,OAAAA,aAAqB,aACxCkmC,IAAK,SAAas/B,aAAmB,aACrCj/B,YAAa,CACXC,OAAQ,SAAag/B,IACrBjgE,OAAQ,CAACjb,OAAAA,aAAoB,aAC7B45E,SAAU,CAACrsD,OAAAA,KAEbkwC,UAAW,CACTK,UAAW,CAAC99D,OAAAA,IACZyE,SAAU,UAAa,YACvBm1E,SAAU,CAACrsD,OAAAA,KAEbikC,eAAgB,SAAa0pB,aAAmB,aAChDzpB,iBAAkB,SAAaypB,aAAmB,aAClD5kB,WAAY,SAAa4kB,IACzBE,eAAgB,CAACp7E,OAAAA,IACjBmzD,WAAY,SAAa+nB,IACzBxW,eAAgB,CAAChvD,OAAAA,GAAQ5Q,MAlCf,SAmCViqD,SAAU,CACRz1C,IAAK,SAAa4hE,aAAmB,aACrCxmC,OAAQ,SAAawmC,aAAmB,aACxCpW,YAAa,SAAaoW,aAAmB,aAC7CjY,WAAY,SAAaiY,aAAmB,aAC5C1X,cAAe,SAAa0X,aAAmB,aAC/CtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B2R,IAAK,CAACl/B,OAAAA,GAAQ0I,KA1CL,OA0CWgN,OAAAA,GAAQmT,OAvCjB,UAwCX3sB,OAAQ,CACNkpD,YAAa,CACXrsC,YAAa,CAACrD,OAAAA,aAAoB,aAClCoD,OAAQ,CAACpD,OAAAA,aAAoB,aAC7BmD,OAAQ,CAACnD,OAAAA,aAAoB,aAC7BqC,KAAM,CAACrC,OAAAA,aAAoB,aAC3BzJ,QAAS,CAACyJ,OAAAA,aAAoB,aAC9B7H,IAAK,CAAC6H,OAAAA,aAAoB,aAC1B1J,KAAM,CAAC0J,OAAAA,aAAoB,aAC3BjN,MAAO,CAACiN,OAAAA,aAAoB,aAC5B5N,KAAM,CAAC4N,OAAAA,aAAoB,aAC3BkkE,SAAU,CAACrsD,OAAAA,YAAoB,aAEjC83B,YAAa,CACXtsC,YAAa,CAACrD,OAAAA,aAAoB,aAClCoD,OAAQ,CAACpD,OAAAA,aAAoB,aAC7BmD,OAAQ,CAACnD,OAAAA,aAAoB,aAC7BqC,KAAM,CAACrC,OAAAA,aAAoB,aAC3BzJ,QAAS,CAACyJ,OAAAA,aAAoB,aAC9B7H,IAAK,CAAC6H,OAAAA,aAAoB,aAC1B1J,KAAM,CAAC0J,OAAAA,aAAoB,aAC3BjN,MAAO,CAACiN,OAAAA,aAAoB,aAC5B5N,KAAM,CAAC4N,OAAAA,aAAoB,aAC3BkkE,SAAU,CAACrsD,OAAAA,YAAoB,aAEjCqsD,SAAU,CAACrsD,OAAAA,KAEb1E,OAAQ,UAAa,YACrB6yC,gBAAiB,CAAChmD,OAAAA,IAClB+3D,WAAY,CAAC/3D,OAAAA,YAAoB,YACjC0mD,cAAe,CACb9iD,IAAK,SAAa4hE,aAAmB,aACrCxmC,OAAQ,SAAawmC,aAAmB,aACxC7e,MAAO,SAAa6e,aAAmB,aACvCtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B8/C,eAAgB,UAAa,YAC7B/lC,OAAQ,CAAC5xB,OAAAA,GAAQ1V,OAAAA,IACjB63C,YAAa,CACX5Y,MAAO,CAACv2B,KAlFD,OAkFO1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OA/ErB,UAgFTqW,IAAK,CAACx2B,KAnFC,OAmFK1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAhFnB,UAiFT4uB,OAAQ,CAAC/hC,OAAAA,IACTkkE,SAAU,CAACrsD,OAAAA,GAAQzoB,MAtFX,UAwFVmjE,qBAAsB,CACpBzxB,KAAM,SAAa0kC,aAAmB,aACtC/iC,MAAO,SAAa+iC,aAAmB,aACvCtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/Bq6C,UAAW,SAAYsT,IACvB/+E,OAAO,CAACuZ,OAAAA,IACRrE,QAAQ,CACNmoE,QAAS,CAAC6B,IA3FJ,OA4FNzB,SAAU,CAACrsD,OAAAA,KAEb0oC,oBAAqB,CAACj2D,OAAAA,IACtB+4D,OAAQ,CACNhS,KAAM,CAAC/mD,OAAAA,IACPw2C,KAAM,CACJ+jB,WAAY,CAACv6D,OAAAA,aAAoB,aACjCs5D,SAAU,CAACt5D,OAAAA,aAAoB,aAC/B45E,SAAU,CAACrsD,OAAAA,GAAOvtB,OAAAA,KAEpB45E,SAAU,CAACrsD,OAAAA,GAAOvtB,OAAAA,KAEpB+D,IAAK,CAAC2E,KA5GG,OA4GG1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAzGjB,UA0GXisC,UAAW,CAAC90D,OAAAA,GAAQ0V,OAAAA,IACpBuxC,cAAe,CAACjnD,OAAAA,IAChBa,IAAK,CAAC6H,KA/GG,OA+GG1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OA5GjB,UA6GXosC,UAAW,CAACj1D,OAAAA,GAAQ0V,OAAAA,IACpBmmC,SAAU,SAAaq/B,IACvBxN,YAAa,SAAawN,IAC1BhD,oBAAqB,SAAagD,IAClCrN,MAAO,UAAa,YACpBF,mBAAoB,UAAa,YACjCG,SAAU,UAAa,YACvBC,OAAQ,UAAa,YACrBE,SAAU,UAAa,YACvBD,SAAU,UAAa,YACvBE,WAAY,UAAa,YACzBC,YAAa,UAAa,YAC1BC,cAAe,UAAa,YAC5BkN,sBAAuB,UAAa,YACpCjf,MAAO,UAAa,YACpBvV,YAAa,CACXC,KAAM,CAACrxC,OAAAA,aAAoB,aAC3B8gC,KAAM,CAAC9gC,OAAAA,aAAoB,aAC3BkkE,SAAU,CAAClkE,OAAAA,GAAQ6X,OAAAA,KAErBo1C,WAAY,SAAauY,IACzBzM,oBAAqB,SAAayM,IAClC9jB,gBAAiB,SAAa8jB,IAC9Bx2B,gBAAiB,SAAaw2B,IAC9Bl0B,gBAAiB,SAAak0B,IAC9Bn2B,cAAe,SAAam2B,IAC5Bp5E,MAAO,SAAao5E,IACpBlhB,eAAgB,SAAakhB,IAC7B3c,QAAS,CACPiN,SAAU,QAAWxrE,aAAqB,aAC1C+sE,cAAe,QAAWr3D,aAAqB,aAC/C+1D,gBAAiB,UAAc,qBAAyB,aACxD/B,WAAY,SAAYwR,aAAmB,aAC3CjS,iBAAkB,SAAYiS,aAAmB,aACjDtB,SAAU,SAAYsB,GAAM3tD,OAAAA,KAE9BgiC,KAAM,UAAa,gBAAoB,QACvCtwB,MAAO,CAACv2B,KArJC,OAqJK1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAlJnB,UAmJXo7C,SAAU,UAAa,YACvBsX,sBAAuB,UAAa,YACpC50C,cAAe,UAAa,YAC5B69B,qBAAsB,CAAC9uD,OAAAA,YAAoB,YAC3C24D,aAAc,SAAa6M,IAC3BnW,QAAS,CACPuJ,YAAa,SAAa4M,IAC1B1S,eAAgB,QAAY,CAAC,MAAO,OAAQ,SAC5C+F,MAAO,CAACvuE,OAAAA,IACRikE,SAAU,UAAa,YACvB2V,SAAU,CAACrsD,OAAAA,KAEbs2C,wBAAyB,CACvBI,SAAU,UAAa,YACvB2V,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B25B,SAAU,CACR9M,MAAO,CAAC1kC,OAAAA,aAAoB,aAC5B8sB,KAAM,CAACxiC,OAAAA,aAAoB,aAC3B45E,SAAU,CAACrsD,OAAAA,KAEboI,KAAM,CAACjgB,OAAAA,IACP2xB,MAAO,CAAC3xB,OAAAA,GAAQ1V,OAAAA,IAChBuxD,WAAY,SAAa2pB,IACzBp/B,SAAU,SAAao/B,IACvB36B,QAAS,CAAC7qC,OAAQ,CAAC,UAAW,SAAU,WAAY,UAAW,KAC/D+qC,aAAc,CAACzgD,OAAAA,IACfg8C,QAAS,CAACh8C,OAAAA,IACV+7C,QAAS,CAAC/7C,OAAAA,IAEV45E,SAAU,CAACrsD,OAAAA,KAGTiuD,GAAmB,CACrB7wD,OAAQ,CACNu6C,MAAQ,CAAC,SAAU,OAAQ,SAC3B7N,iBAAkB,CAAC,OAAQ,OAAQ,QAAS,UAAW,OAAQ,UAAW,MAAO,OAAQ,OAAQ,SAAU,UAC3Gt6C,WAAY,EACZu5C,YAAY,EACZnD,YAAY,EAEVpE,SAAU,CACVz1C,KAAK,EACLo7B,QAAQ,EACRowB,aAAa,EACb7B,YAAY,GAEd/jC,IAAK,GACLhjC,OAAQ,CACNkpD,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZX,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZX,KAAY,KAGhB4zD,gBAAiB,CAAC,OAAQ,QAAS,YAEnC+f,iBAAiB,EACjBn0C,OAAQ,GAERnrC,OAAQ,GACR85D,oBAAqB,IACrB8C,OAAQ,CACNhS,KAAM,CAAC,GAAI,EAAG,IAAK,GACnBvQ,KAAM,CACJ+jB,WAAY,CAAC,GAAI,EAAG,IAAK,GACzBjB,SAAU,CAAC,GAAI,EAAG,IAAK,KAG3Bv1D,IAAK,GACL+wD,UAAW,GACX7N,cAAe,CAAC,EAAG,EAAG,GAAI,GAC1BpmD,IAAK,GACLo0D,UAAW,GACXpZ,UAAU,EACV6xB,aAAa,EACbwK,qBAAqB,EAOrBpxB,YAAa,CACXC,KAAM,CAAC,OAAQ,SAAU,OACzBvQ,KAAM,CAAC,SAAU,QAEnB+a,YAAY,EACZoR,YAAY,EACZvL,iBAAiB,EACjB1S,iBAAiB,EACjBsC,iBAAiB,EACjBllD,OAAO,EACPk4D,gBAAgB,EAChBuE,SAAS,EAETt/B,MAAO,GAMPovC,cAAc,EACdtJ,QAAS,CACPuJ,aAAa,EACb9F,eAAgB,OAChB+F,MAAO,CAAC,IAAK,EAAG,MAAO,MAEzB1K,yBAAyB,EACzBluC,KAAM,CAAC,MAAO,QAAS,QAAS,cAChC0R,MAAO,OACPyU,UAAU,EACVyE,QAAS,CAAC,UAAW,SAAU,WAAY,UAAW,IACtDvE,QAAS,CAAC,SAAiB,GAAI,SAAiB,GAChDD,QAAS,CAAC,GAAI,GAAI,SAAiB,OC9RrC,CAAE3tB,OAAQ,QAASnP,OAAO,GAAQ,CAClCy8D,KCAe,SAAcn7E,WACzB2rB,EAAI9K,GAAS3iB,MACbhD,EAASq0B,EAAS5D,EAAEzwB,QACpBgnC,EAAkBloC,UAAUkB,OAC5B4nB,EAAQ0M,GAAgB0S,EAAkB,EAAIloC,UAAU,QAAK2D,EAAWzC,GACxEyjC,EAAMuD,EAAkB,EAAIloC,UAAU,QAAK2D,EAC3Cy9E,OAAiBz9E,IAARghC,EAAoBzjC,EAASs0B,GAAgBmP,EAAKzjC,GACxDkgF,EAASt4D,GAAO6I,EAAE7I,KAAW9iB,SAC7B2rB,KCZT,OAAiBoT,GAAa,SAASo8C,KCDnCx7C,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmxD,YACNnxD,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAew7C,KAAQA,GAAOl8C,GCD5Fo8C,GAAa,CAACC,MAAO,UAAWC,KAAM,UAAWC,SAAU,UAAWC,WAAY,UAAWC,KAAM,UAAWC,UAAW,UAAWC,MAAO,UAAWC,KAAM,UAAWC,SAAU,UAAWC,YAAa,UAAWC,cAAe,UAAWC,kBAAmB,UAAWC,KAAM,UAAWC,YAAa,UAAWC,KAAM,UAAWC,KAAM,UAAWC,aAAc,UAAWC,WAAY,UAAWC,cAAe,UAAWC,YAAa,UAAWC,SAAU,UAAWC,cAAe,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,UAAW,UAAWC,UAAW,UAAWC,UAAW,UAAWC,cAAe,UAAWC,gBAAiB,UAAWC,OAAQ,UAAWC,eAAgB,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,iBAAkB,UAAWC,QAAS,UAAWC,UAAW,UAAWC,UAAW,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,gBAAiB,UAAWC,UAAW,UAAWC,WAAY,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,OAAQ,UAAWC,MAAO,UAAWC,KAAM,UAAWC,QAAS,UAAWC,aAAc,UAAWC,WAAY,UAAWC,QAAS,UAAWC,YAAa,UAAWC,YAAa,UAAWC,aAAc,UAAWC,WAAY,UAAWC,aAAc,UAAWC,WAAY,UAAWC,UAAW,UAAWC,WAAY,UAAWC,YAAa,UAAWC,OAAQ,UAAWC,MAAO,UAAWC,SAAU,UAAWC,UAAW,UAAWC,YAAa,UAAWC,cAAe,UAAWC,eAAgB,UAAWC,WAAY,UAAWC,UAAW,UAAWC,cAAe,UAAWC,aAAc,UAAWC,UAAW,UAAWC,UAAW,UAAWC,OAAQ,UAAWC,gBAAiB,UAAWC,UAAW,UAAWC,KAAM,UAAWC,UAAW,UAAWC,IAAK,UAAWC,UAAW,UAAWC,cAAe,UAAWC,QAAS,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,QAAS,UAAWC,UAAW,UAAWC,KAAM,UAAWC,UAAW,UAAWC,UAAW,UAAWC,SAAU,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,cAAe,UAAWC,WAAY,UAAWC,MAAO,UAAWC,UAAW,UAAWC,SAAU,UAAWC,MAAO,UAAWC,WAAY,UAAWC,MAAO,UAAWC,MAAO,UAAWC,WAAY,UAAWC,UAAW,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,aAAc,UAAWC,MAAO,UAAWC,qBAAsB,UAAWC,QAAS,UAAWC,IAAK,UAAWC,QAAS,UAAWC,QAAS,UAAWC,SAAU,UAAWC,UAAW,UAAWC,OAAQ,UAAWC,QAAS,UAAWC,MAAO,UAAWC,WAAY,UAAWC,YAAa,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,KAAM,UAAWC,KAAM,UAAWC,UAAW,UAAWC,YAAa,UAAWC,SAAU,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,WAAY,UAAWC,cAAe,UAAWC,SAAU,UAAWC,SAAU,UAAWC,aAAc,UAAWC,YAAa,UAAWC,KAAM,UAAWC,OAAQ,UAAWC,YAAa,UAAWC,MAAO,UAAWC,MAAO,WAK1/FC,+BAIQC,yDAAa,kBAClBA,WAAaA,OACbC,WAAY,OACZC,kBAAoB,CAACh8E,EAAE,MAAOZ,EAAE,YAChC68E,EAAI,IAAM,SACVC,MAAQ,CAACD,EAAE,IAAIE,EAAE,IAAIlpF,EAAE,IAAID,EAAE,QAC7BopF,eAAY9mF,OACZ+mF,aAAe,CAACJ,EAAE,IAAIE,EAAE,IAAIlpF,EAAE,IAAID,EAAE,QACpCspF,mBAAehnF,OACfinF,SAAU,OAGVC,eAAiB,kBACjBC,cAAgB,kBAGhBl+B,sDAQEuD,QACaxsD,IAAhBO,KAAK6jD,cACFA,OAAOkB,eACPlB,YAASpkD,QAEXwsD,UAAYA,OACZA,UAAUt3B,YAAY30B,KAAK2oE,YAC3Bke,mBAEAC,qDAOW9gF,MACQ,mBAAbA,QAIH,IAAI5C,MAAM,oFAHXujF,eAAiB3gF,2CAWTA,MACS,mBAAbA,QAIH,IAAI5C,MAAM,qFAHXwjF,cAAgB5gF,yCAaVqgF,MACQ,iBAAVA,SACFlJ,GAAWkJ,oCAiBbA,OAAOU,gEACA,SAAVV,OAIAW,EAGAC,EAAYjnF,KAAKknF,eAAeb,WAClB5mF,IAAdwnF,IACFZ,EAAQY,IAImB,IAAzB3uC,GAAKnB,SAASkvC,QACe,IAA3B/tC,GAAK6uC,WAAWd,GAAiB,KAC/Be,EAAYf,EAAM9gF,OAAO,GAAGA,OAAO,EAAG8gF,EAAMrpF,OAAS,GAAG6N,MAAM,KAClEm8E,EAAO,CAACZ,EAAEgB,EAAU,GAAId,EAAEc,EAAU,GAAIhqF,EAAEgqF,EAAU,GAAIjqF,EAAE,QAEvD,IAAgC,IAA5Bm7C,GAAK+uC,YAAYhB,GAAiB,KACrCe,EAAYf,EAAM9gF,OAAO,GAAGA,OAAO,EAAG8gF,EAAMrpF,OAAS,GAAG6N,MAAM,KAClEm8E,EAAO,CAACZ,EAAEgB,EAAU,GAAId,EAAEc,EAAU,GAAIhqF,EAAEgqF,EAAU,GAAIjqF,EAAEiqF,EAAU,SAEjE,IAA+B,IAA3B9uC,GAAKgvC,WAAWjB,GAAiB,KACpCkB,EAASjvC,GAAKkvC,SAASnB,GAC3BW,EAAO,CAACZ,EAAEmB,EAAOnB,EAAGE,EAAEiB,EAAOjB,EAAGlpF,EAAEmqF,EAAOnqF,EAAGD,EAAE,YAI5CkpF,aAAiBnqF,aACHuD,IAAZ4mF,EAAMD,QAA+B3mF,IAAZ4mF,EAAMC,QAA+B7mF,IAAZ4mF,EAAMjpF,EAAiB,KACvEqqF,OAAoBhoF,IAAZ4mF,EAAMlpF,EAAkBkpF,EAAMlpF,EAAI,MAC9C6pF,EAAO,CAACZ,EAAEC,EAAMD,EAAGE,EAAED,EAAMC,EAAGlpF,EAAEipF,EAAMjpF,EAAGD,EAAEsqF,WAMpChoF,IAATunF,QACI,IAAI5jF,MAAM,gIAAkI28C,GAAesmC,SAG5JqB,UAAUV,EAAMD,wCAUItnF,IAAvBO,KAAK4mF,qBACFA,qBACAA,mBAAgBnnF,QAGlBinF,SAAU,OACV/d,MAAMn0C,MAAMC,QAAU,aACtBkzD,gEAWDC,8DAEkB,IAAlBA,SACGnB,cAAgBnuC,GAAKh7C,OAAO,GAAI0C,KAAKqmF,SAGvB,IAAjBrmF,KAAK0mF,cACFC,eAAe3mF,KAAKwmF,mBAGtB7d,MAAMn0C,MAAMC,QAAU,WAIhB,gBACkBh1B,IAAvBi+C,EAAKkpC,gBACPlpC,EAAKkpC,gBACLlpC,EAAKkpC,mBAAgBnnF,KAEvB,wCASGknF,eAAe3mF,KAAKqmF,YACpBK,SAAU,OACVmB,8CASAnB,SAAU,OACVC,eAAe3mF,KAAKqmF,YACpByB,cAAc9nF,KAAKqmF,gDASG5mF,IAAvBO,KAAKymF,mBACFsB,SAAS/nF,KAAKymF,eAAe,GAGlCuB,MAAM,uEAWAhB,OAAMD,8DAEK,IAAfA,SACGP,aAAeluC,GAAKh7C,OAAO,GAAI0pF,SAGjCX,MAAQW,MACTiB,EAAM3vC,GAAK4vC,SAASlB,EAAKZ,EAAGY,EAAKV,EAAGU,EAAK5pF,GAEzC+qF,EAAe,EAAI3mF,KAAK4mF,GACxBC,EAASroF,KAAKomF,EAAI6B,EAAIp/E,EACtBsB,EAAInK,KAAKmmF,kBAAkBh8E,EAAIk+E,EAAS7mF,KAAK8mF,IAAIH,EAAeF,EAAIz7E,GACpEjD,EAAIvJ,KAAKmmF,kBAAkB58E,EAAI8+E,EAAS7mF,KAAK+mF,IAAIJ,EAAeF,EAAIz7E,QAEnEg8E,oBAAoBh0D,MAAMkgB,KAAOvqC,EAAI,GAAMnK,KAAKwoF,oBAAoB7nC,YAAc,UAClF6nC,oBAAoBh0D,MAAMsuB,IAAMv5C,EAAI,GAAMvJ,KAAKwoF,oBAAoBz8B,aAAe,UAElF+7B,cAAcd,uCASTllF,QACLukF,MAAMlpF,EAAI2E,EAAQ,SAClBgmF,cAAc9nF,KAAKqmF,8CASXvkF,OACTmmF,EAAM3vC,GAAK4vC,SAASloF,KAAKqmF,MAAMD,EAAGpmF,KAAKqmF,MAAMC,EAAGtmF,KAAKqmF,MAAMjpF,GAC/D6qF,EAAIQ,EAAI3mF,EAAQ,QACZklF,EAAO1uC,GAAKowC,SAAST,EAAIz7E,EAAGy7E,EAAIp/E,EAAGo/E,EAAIQ,GAC3CzB,EAAI,EAAQhnF,KAAKqmF,MAAMlpF,OAClBkpF,MAAQW,OACRc,4DASOd,yDAAOhnF,KAAKqmF,MACpB4B,EAAM3vC,GAAK4vC,SAASlB,EAAKZ,EAAGY,EAAKV,EAAGU,EAAK5pF,GACzCurF,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACnBppF,IAArBO,KAAK8oF,mBACF7C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,IAEhCT,EAAIU,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OAGxD7wE,EAAIpV,KAAK4oF,kBAAkBjoC,YAC3Bn0C,EAAIxM,KAAK4oF,kBAAkB78B,aAC/B48B,EAAIW,UAAU,EAAG,EAAGl0E,EAAG5I,GAEvBm8E,EAAIY,aAAavpF,KAAKumF,UAAW,EAAE,GACnCoC,EAAIa,UAAY,eAAiB,EAAGvB,EAAIQ,GAAK,IAC7CE,EAAIc,OAAOzpF,KAAKmmF,kBAAkBh8E,EAAGnK,KAAKmmF,kBAAkB58E,EAAGvJ,KAAKomF,MACpEuC,QAAAA,QAEKe,gBAAgB5nF,MAAQ,IAAMmmF,EAAIQ,OAClCkB,aAAa7nF,MAAW,IAAMklF,EAAK7pF,OAEnCysF,gBAAgBp1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKwmF,aAAaJ,EAAI,IAAMpmF,KAAKwmF,aAAaF,EAAI,IAAMtmF,KAAKwmF,aAAappF,EAAI,IAAM4C,KAAKwmF,aAAarpF,EAAI,SAC5J2sF,YAAYt1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKqmF,MAAMD,EAAI,IAAMpmF,KAAKqmF,MAAMC,EAAI,IAAMtmF,KAAKqmF,MAAMjpF,EAAI,IAAM4C,KAAKqmF,MAAMlpF,EAAI,4CAS5HyrF,kBAAkBp0D,MAAMoU,MAAQ,YAChCggD,kBAAkBp0D,MAAMqU,OAAS,YAEjC+/C,kBAAkBhgD,MAAQ,IAAM5oC,KAAKimF,gBACrC2C,kBAAkB//C,OAAS,IAAM7oC,KAAKimF,iEAUtCtd,MAAQv7C,SAASE,cAAc,YAC/Bq7C,MAAMrgC,UAAY,wBAElByhD,eAAiB38D,SAASE,cAAc,YACxCk7D,oBAAsBp7D,SAASE,cAAc,YAC7Ck7D,oBAAoBlgD,UAAY,oBAChCyhD,eAAep1D,YAAY30B,KAAKwoF,0BAEhCI,kBAAoBx7D,SAASE,cAAc,eAC3Cy8D,eAAep1D,YAAY30B,KAAK4oF,mBAEhC5oF,KAAK4oF,kBAAkBC,WAQvB,KACCF,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACvC5C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,QACzBR,kBAAkBC,WAAW,MAAMQ,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OAf1D,KAClC+D,EAAW58D,SAASE,cAAe,OACvC08D,EAASx1D,MAAM6xD,MAAQ,MACvB2D,EAASx1D,MAAMy1D,WAAc,OAC7BD,EAASx1D,MAAMw1C,QAAW,OAC1BggB,EAASt+B,UAAa,wDACjBk9B,kBAAkBj0D,YAAYq1D,QAYhCD,eAAezhD,UAAY,iBAE3B4hD,WAAa98D,SAASE,cAAc,YACpC48D,WAAW5hD,UAAY,mBAEvB6hD,cAAgB/8D,SAASE,cAAc,YACvC68D,cAAc7hD,UAAY,sBAE1B8hD,SAAWh9D,SAASE,cAAc,YAClC88D,SAAS9hD,UAAY,iBAErBqhD,aAAev8D,SAASE,cAAc,kBAEpCq8D,aAAazyD,KAAO,aACpByyD,aAAavnF,IAAM,SACnBunF,aAAarkF,IAAM,YAGnB2gC,SACF0jD,aAAa7nF,MAAQ,WACrB6nF,aAAarhD,UAAY,iBAEzBohD,gBAAkBt8D,SAASE,cAAc,kBAEvCo8D,gBAAgBxyD,KAAO,aACvBwyD,gBAAgBtnF,IAAM,SACtBsnF,gBAAgBpkF,IAAM,YAGtB2gC,SACFyjD,gBAAgB5nF,MAAQ,WACxB4nF,gBAAgBphD,UAAY,iBAE5B4hD,WAAWv1D,YAAY30B,KAAK2pF,mBAC5BQ,cAAcx1D,YAAY30B,KAAK0pF,qBAEhC/qC,EAAK3+C,UACJ2pF,aAAaU,SAAW,WAAa1rC,EAAG2rC,YAAYtqF,KAAK8B,aACzD6nF,aAAaY,QAAW,WAAa5rC,EAAG2rC,YAAYtqF,KAAK8B,aACzD4nF,gBAAgBW,SAAW,WAAa1rC,EAAG6rC,eAAexqF,KAAK8B,aAC/D4nF,gBAAgBa,QAAW,WAAa5rC,EAAG6rC,eAAexqF,KAAK8B,aAE/D2oF,gBAAkBr9D,SAASE,cAAc,YACzCm9D,gBAAgBniD,UAAY,gCAC5BmiD,gBAAgB/+B,UAAY,mBAE5Bg/B,aAAet9D,SAASE,cAAc,YACtCo9D,aAAapiD,UAAY,6BACzBoiD,aAAah/B,UAAY,gBAEzBo+B,YAAc18D,SAASE,cAAc,YACrCw8D,YAAYxhD,UAAY,qBACxBwhD,YAAYp+B,UAAY,WAExBk+B,gBAAkBx8D,SAASE,cAAc,YACzCs8D,gBAAgBthD,UAAY,yBAC5BshD,gBAAgBl+B,UAAY,eAE5Bi/B,aAAev9D,SAASE,cAAc,YACtCq9D,aAAariD,UAAY,6BACzBqiD,aAAaj/B,UAAY,cACzBi/B,aAAaC,QAAU70C,UAAK8xC,cAAW7nF,MAAM,QAE7C6qF,YAAcz9D,SAASE,cAAc,YACrCu9D,YAAYviD,UAAY,4BACxBuiD,YAAYn/B,UAAY,aACxBm/B,YAAYD,QAAU70C,UAAK+0C,eAAY9qF,WAEvC+qF,WAAa39D,SAASE,cAAc,YACpCy9D,WAAWziD,UAAY,2BACvByiD,WAAWr/B,UAAY,YACvBq/B,WAAWH,QAAU70C,UAAKi1C,cAAWhrF,WAErCirF,WAAa79D,SAASE,cAAc,YACpC29D,WAAW3iD,UAAY,2BACvB2iD,WAAWv/B,UAAY,iBACvBu/B,WAAWL,QAAU70C,UAAKm1C,kBAAelrF,WAEzC2oE,MAAMh0C,YAAY30B,KAAK+pF,qBACvBphB,MAAMh0C,YAAY30B,KAAKoqF,eACvBzhB,MAAMh0C,YAAY30B,KAAKyqF,sBACvB9hB,MAAMh0C,YAAY30B,KAAKmqF,oBACvBxhB,MAAMh0C,YAAY30B,KAAK0qF,mBACvB/hB,MAAMh0C,YAAY30B,KAAKkqF,iBACvBvhB,MAAMh0C,YAAY30B,KAAK8pF,kBACvBnhB,MAAMh0C,YAAY30B,KAAK4pF,sBAEvBjhB,MAAMh0C,YAAY30B,KAAK2qF,mBACvBhiB,MAAMh0C,YAAY30B,KAAK6qF,kBACvBliB,MAAMh0C,YAAY30B,KAAK+qF,iBACvBpiB,MAAMh0C,YAAY30B,KAAKirF,kEASvBn7B,KAAO,QACPq7B,MAAQ,QACRtnC,OAAS,IAAIG,GAAOhkD,KAAK4oF,wBACzB/kC,OAAO95C,IAAI,SAAS/F,IAAI,CAACkiD,QAAQ,IAEtCsM,GAAmBxyD,KAAK6jD,QAAQ,SAACrD,GAAWmV,EAAKy1B,cAAc5qC,WAC1DqD,OAAOvN,GAAG,OAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,YAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,WAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,UAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,uDASpC,IAAnBxgD,KAAKkmF,UAAqB,KACxByC,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACnBppF,IAArBO,KAAK8oF,mBACF7C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,IAEhCT,EAAIU,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OASxD97E,EAAGZ,EAAG8hF,EAAKC,EANXl2E,EAAIpV,KAAK4oF,kBAAkBjoC,YAC3Bn0C,EAAIxM,KAAK4oF,kBAAkB78B,aAC/B48B,EAAIW,UAAU,EAAG,EAAGl0E,EAAG5I,QAKlB25E,kBAAoB,CAACh8E,EAAO,GAAJiL,EAAS7L,EAAO,GAAJiD,QACpC45E,EAAI,IAAOhxE,MAIZm2E,EAHApD,EAAgB,EAAI3mF,KAAK4mF,GAAM,IAE/BoD,EAAO,EAAIxrF,KAAKomF,MAEfiF,EAAM,EAAGA,EAAM,IAAKA,QAClBC,EAAM,EAAGA,EAAMtrF,KAAKomF,EAAGkF,IAC1BnhF,EAAInK,KAAKmmF,kBAAkBh8E,EAAImhF,EAAM9pF,KAAK8mF,IAAIH,EAAekD,GAC7D9hF,EAAIvJ,KAAKmmF,kBAAkB58E,EAAI+hF,EAAM9pF,KAAK+mF,IAAIJ,EAAekD,GAC7DE,EAAMjzC,GAAKowC,SAAS2C,GAPb,EAAI,KAOqBC,EAAME,EAAM,GAC5C7C,EAAIa,UAAY,OAAS+B,EAAInF,EAAI,IAAMmF,EAAIjF,EAAI,IAAMiF,EAAInuF,EAAI,IAC7DurF,EAAI8C,SAASthF,EAAI,GAAKZ,EAAI,GAAK,EAAG,GAGtCo/E,EAAI+C,YAAc,gBAClB/C,EAAIc,OAAOzpF,KAAKmmF,kBAAkBh8E,EAAGnK,KAAKmmF,kBAAkB58E,EAAGvJ,KAAKomF,GACpEuC,EAAIgD,cAECpF,UAAYoC,EAAIiD,aAAa,EAAE,EAAEx2E,EAAE5I,QAErC05E,WAAY,wCAUL1lC,OACR1X,EAAO9oC,KAAK+pF,eAAennC,wBAC3BlO,EAAO8L,EAAMxB,OAAO70C,EAAI2+B,EAAK4L,KAC7BoO,EAAMtC,EAAMxB,OAAOz1C,EAAIu/B,EAAKga,IAE5B+oC,EAAU,GAAM7rF,KAAK+pF,eAAeh+B,aACpC+/B,EAAU,GAAM9rF,KAAK+pF,eAAeppC,YAEpCx2C,EAAIuqC,EAAOo3C,EACXviF,EAAIu5C,EAAM+oC,EAEVE,EAAQvqF,KAAKwqF,MAAM7hF,EAAEZ,GACrB8+E,EAAS,IAAO7mF,KAAKY,IAAIZ,KAAKyqF,KAAK9hF,EAAIA,EAAIZ,EAAIA,GAAIuiF,GAEnDzwB,EAAS75D,KAAK+mF,IAAIwD,GAAS1D,EAASwD,EACpCK,EAAU1qF,KAAK8mF,IAAIyD,GAAS1D,EAASyD,OAEpCtD,oBAAoBh0D,MAAMsuB,IAAMuY,EAAS,GAAMr7D,KAAKwoF,oBAAoBz8B,aAAe,UACvFy8B,oBAAoBh0D,MAAMkgB,KAAOw3C,EAAU,GAAMlsF,KAAKwoF,oBAAoB7nC,YAAc,SAGzFn0C,EAAIu/E,GAAS,EAAIvqF,KAAK4mF,IAC1B57E,EAAIA,EAAI,EAAIA,EAAI,EAAIA,MAChB3D,EAAIw/E,EAASroF,KAAKomF,EAClB6B,EAAM3vC,GAAK4vC,SAASloF,KAAKqmF,MAAMD,EAAGpmF,KAAKqmF,MAAMC,EAAGtmF,KAAKqmF,MAAMjpF,GAC/D6qF,EAAIz7E,EAAIA,EACRy7E,EAAIp/E,EAAIA,MACJm+E,EAAO1uC,GAAKowC,SAAST,EAAIz7E,EAAGy7E,EAAIp/E,EAAGo/E,EAAIQ,GAC3CzB,EAAI,EAAQhnF,KAAKqmF,MAAMlpF,OAClBkpF,MAAQW,OAGR4C,gBAAgBp1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKwmF,aAAaJ,EAAI,IAAMpmF,KAAKwmF,aAAaF,EAAI,IAAMtmF,KAAKwmF,aAAappF,EAAI,IAAM4C,KAAKwmF,aAAarpF,EAAI,SAC5J2sF,YAAYt1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKqmF,MAAMD,EAAI,IAAMpmF,KAAKqmF,MAAMC,EAAI,IAAMtmF,KAAKqmF,MAAMjpF,EAAI,IAAM4C,KAAKqmF,MAAMlpF,EAAI,ylqBCxiB/HgvF,yBAOQC,EAAcC,EAAkBtP,OAAkBkJ,yDAAa,kBACpEr1D,OAASw7D,OACTE,eAAiB,QACjBrgC,UAAYogC,OACZE,eAAgB,OAEhBp9D,QAAU,QACVq9D,aAAc,OACdC,aAAe,OACfvvC,eAAiB,CACpBw/B,SAAS,EACTzkD,QAAQ,EACRg0B,eAAWxsD,EACXitF,YAAY,GAEdp0C,GAAKh7C,OAAO0C,KAAKmvB,QAASnvB,KAAKk9C,qBAE1B6/B,iBAAmBA,OACnB4P,cAAgB,QAChBC,YAAc,QACdC,SAAW,QACXC,WAAa,OACbC,aAAe,QACfC,YAAc,IAAIhH,GAAYC,QAC9Bl6C,aAAUtsC,gDAUN0vB,WACO1vB,IAAZ0vB,EAAuB,MAEpB49D,aAAe,QACfE,mBAEDvQ,GAAU,KACS,iBAAZvtD,OACJA,QAAQ8I,OAAS9I,OAEnB,GAAIA,aAAmBlzB,WACrBkzB,QAAQ8I,OAAS9I,EAAQhsB,YAE3B,GAAuB,WAAnBsiC,GAAOtW,GAAsB,IACrB,MAAXA,QACI,IAAInC,UAAU,+BAEIvtB,IAAtB0vB,EAAQ88B,iBACL98B,QAAQ88B,UAAY98B,EAAQ88B,gBAEZxsD,IAAnBq3C,GAAA3nB,UACGA,QAAQ8I,UAAS9I,SAEG1vB,IAAvB0vB,EAAQu9D,kBACLv9D,QAAQu9D,WAAav9D,EAAQu9D,iBAEZjtF,IAApB0vB,EAAQutD,UACVA,EAAUvtD,EAAQutD,aAGM,kBAAZvtD,QACTA,QAAQ8I,QAAS,EACtBykD,EAAUvtD,GAEgB,mBAAZA,SACTA,QAAQ8I,OAAS9I,EACtButD,GAAU,IAEgB,IAAxB5lC,QAAK3nB,WACPutD,GAAU,QAGPvtD,QAAQutD,QAAUA,OAEpBwQ,kDAOUP,QACVA,cAAgBA,GACQ,IAAzB3sF,KAAKmvB,QAAQutD,eACVwQ,cAC0BztF,IAA3BO,KAAKmvB,QAAQ88B,iBACVA,UAAYjsD,KAAKmvB,QAAQ88B,gBAE3BvD,kDASFwkC,cACAZ,eAAiB,OAElBr0D,KAASj4B,KAAKmvB,SACdg+D,EAAU,EACV3pB,GAAO,MACN,IAAIlN,KAAUt2D,KAAK+8E,iBAClB/8E,KAAK+8E,iBAAiB1/E,eAAei5D,UAClCi2B,eAAgB,EACrB/oB,GAAO,EACe,mBAAXvrC,EAETurC,GADAA,EAAOvrC,EAAOq+B,EAAO,MACNt2D,KAAKotF,cAAcptF,KAAK+8E,iBAAiBzmB,GAAS,CAACA,IAAS,IAEzD,IAAXr+B,IAA+C,IAA5B27B,GAAA37B,QAAAA,EAAeq+B,KACzCkN,GAAO,IAGI,IAATA,SACG+oB,eAAgB,EAGjBY,EAAU,QACPE,UAAU,SAGZC,YAAYh3B,QAGZ82B,cAAcptF,KAAK+8E,iBAAiBzmB,GAAS,CAACA,KAErD62B,UAGCI,mBACAC,6CAUAzhD,QAAU3e,SAASE,cAAc,YACjCye,QAAQzD,UAAY,iCACpB2jB,UAAUt3B,YAAY30B,KAAK+rC,aAC3B,IAAIjvC,EAAI,EAAGA,EAAIkD,KAAK4sF,YAAY5vF,OAAQF,SACtCivC,QAAQpX,YAAY30B,KAAK4sF,YAAY9vF,SAGvC2wF,0DASA,IAAI3wF,EAAI,EAAGA,EAAIkD,KAAK4sF,YAAY5vF,OAAQF,SACtCivC,QAAQvE,YAAYxnC,KAAK4sF,YAAY9vF,SAGvB2C,IAAjBO,KAAK+rC,eACFkgB,UAAUzkB,YAAYxnC,KAAK+rC,cAC3BA,aAAUtsC,QAEZmtF,YAAc,QAEdK,iDAUGh9D,WACJlS,EAAO/d,KAAK2sF,cACP7vF,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IAAK,SACd2C,IAAlBse,EAAKkS,EAAKnzB,IAGT,CACHihB,OAAOte,QAHPse,EAAOA,EAAKkS,EAAKnzB,WAOdihB,oCAWCkS,OACmB,IAAvBjwB,KAAKusF,cAAwB,KAC3Bx0C,EAAO3qB,SAASE,cAAc,OAClCyqB,EAAKzP,UAAY,iDAAmDrY,EAAKjzB,kCAH1D4vF,mCAAAA,8BAIfA,QAAAA,GAAoB,SAAChlD,GACnBmQ,EAAKpjB,YAAYiT,WAEdglD,YAAY3vF,KAAK86C,GACf/3C,KAAK4sF,YAAY5vF,cAEnB,sCASGyG,OACNiqF,EAAMtgE,SAASE,cAAc,OACjCogE,EAAIplD,UAAY,sCAChBolD,EAAIhiC,UAAYjoD,OACX4pF,UAAU,GAAGK,sCAYTjqF,EAAMwsB,OAAM09D,0DACjBD,EAAMtgE,SAASE,cAAc,cACjCogE,EAAIplD,UAAY,kDAAoDrY,EAAKjzB,OAEvE0wF,EAAIhiC,WADc,IAAhBiiC,EACc,SAAWlqF,EAAO,YAGlBA,EAAO,IAElBiqF,wCAWK9wF,EAAKkF,EAAOmuB,OACpB2iD,EAASxlD,SAASE,cAAc,UACpCslD,EAAOtqC,UAAY,0CACfslD,EAAgB,OACNnuF,IAAVqC,IAC0B,IAAxB8xD,GAAAh3D,QAAAA,EAAYkF,KACd8rF,EAAgBh6B,GAAAh3D,QAAAA,EAAYkF,QAI3B,IAAIhF,EAAI,EAAGA,EAAIF,EAAII,OAAQF,IAAK,KAC/Bw5D,EAASlpC,SAASE,cAAc,UACpCgpC,EAAOx0D,MAAQlF,EAAIE,GACfA,IAAM8wF,IACRt3B,EAAO0N,SAAW,YAEpB1N,EAAO5K,UAAY9uD,EAAIE,GACvB81E,EAAOj+C,YAAY2hC,OAGjB3X,EAAK3+C,KACT4yE,EAAOyX,SAAW,WAAa1rC,EAAGvI,QAAQp2C,KAAK8B,MAAOmuB,QAElDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOqqC,sCAWnBh2E,EAAKkF,EAAOmuB,OACjB69D,EAAelxF,EAAI,GACnBwF,EAAMxF,EAAI,GACV0I,EAAM1I,EAAI,GACVmnC,EAAOnnC,EAAI,GACX88C,EAAQtsB,SAASE,cAAc,SACnCosB,EAAMpR,UAAY,yCAEhBoR,EAAMxiB,KAAO,QACbwiB,EAAMt3C,IAAMA,EACZs3C,EAAMp0C,IAAMA,QAGP2gC,IACPyT,EAAM3V,KAAOA,MAGTgqD,EAAc,GACdC,EAAa,UAEHvuF,IAAVqC,EAAqB,CAEnBA,EAAQ,GADC,IACIA,EAAiBM,GAChCs3C,EAAMt3C,IAAMZ,KAAKC,KAFN,IAEWK,GACtBksF,EAAat0C,EAAMt3C,IACnB2rF,EAAc,mBAEPjsF,EANI,IAMaM,IACxBs3C,EAAMt3C,IAAMZ,KAAKC,KAAKK,EAPX,KAQXksF,EAAat0C,EAAMt3C,IACnB2rF,EAAc,mBATH,IAWTjsF,EAAiBwD,GAAe,IAARA,IAC1Bo0C,EAAMp0C,IAAM9D,KAAKC,KAZN,IAYWK,GACtBksF,EAAat0C,EAAMp0C,IACnByoF,EAAc,mBAEhBr0C,EAAM53C,MAAQA,OAGd43C,EAAM53C,MAAQgsF,MAGZ9xF,EAAQoxB,SAASE,cAAc,SACnCtxB,EAAMssC,UAAY,0CAClBtsC,EAAM8F,MAAQu1C,OAAOqC,EAAM53C,WAEvB68C,EAAK3+C,KACT05C,EAAM2wC,SAAW,WAAaruF,EAAM8F,MAAQ9B,KAAK8B,MAAO68C,EAAGvI,QAAQiB,OAAOr3C,KAAK8B,OAAQmuB,IACvFypB,EAAM6wC,QAAW,WAAavuF,EAAM8F,MAAQ9B,KAAK8B,WAE7CymC,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,GAC7CuyC,EAAYxiE,KAAKqtF,UAAUp9D,EAAMsY,EAAOmR,EAAO19C,GAG/B,KAAhB+xF,GAAsB/tF,KAAK+sF,aAAavqB,KAAewrB,SACpDjB,aAAavqB,GAAawrB,OAC1BC,YAAYF,EAAavrB,yDASA,IAA5BxiE,KAAKmvB,QAAQu9D,WAAqB,KAChCwB,EAAiB9gE,SAASE,cAAc,OAC5C4gE,EAAe5lD,UAAY,sCAC3B4lD,EAAexiC,UAAY,mBAC3BwiC,EAAetD,QAAc,WAAOltC,EAAKywC,iBACzCD,EAAeE,YAAc,WAAOF,EAAe5lD,UAAY,6CAC/D4lD,EAAeG,WAAc,WAAOH,EAAe5lD,UAAY,4CAE1DgmD,iBAAmBlhE,SAASE,cAAc,YAC1CghE,iBAAiBhmD,UAAY,qDAE7BskD,YAAY3vF,KAAK+C,KAAKsuF,uBACtB1B,YAAY3vF,KAAKixF,wCAWdj3E,EAAQ2N,kBACO,IAArB5kB,KAAKwsF,cAA+C,IAAvBxsF,KAAKusF,eAA0BvsF,KAAKysF,aAAezsF,KAAK8sF,WAAY,KAC/FY,EAAMtgE,SAASE,cAAc,OACjCogE,EAAIt6D,GAAK,0BACTs6D,EAAIplD,UAAY,0BAChBolD,EAAIhiC,UAAYz0C,EAChBy2E,EAAI9C,QAAU,WAAOj1B,EAAKs3B,qBACrBR,cAAgB,OAChBI,SAAW,CAACn4D,KAAKg5D,EAAK9oE,MAAMA,gDAURnlB,IAAvBO,KAAK6sF,SAASn4D,YACXm4D,SAASn4D,KAAK6S,WAAWC,YAAYxnC,KAAK6sF,SAASn4D,MACxD0qB,aAAap/C,KAAK6sF,SAAS0B,aAC3BnvC,aAAap/C,KAAK6sF,SAAS2B,oBACtB3B,SAAW,oEAUSptF,IAAvBO,KAAK6sF,SAASn4D,KAAoB,KAEhCoU,EADuB9oC,KAAK4sF,YAAY5sF,KAAK6sF,SAASjoE,OAC1Bg+B,6BAC3BiqC,SAASn4D,KAAKF,MAAMkgB,KAAO5L,EAAK4L,KAAO,UACvCm4C,SAASn4D,KAAKF,MAAMsuB,IAAMha,EAAKga,IAAM,GAAK,KAC/C11B,SAASwrB,KAAKjkB,YAAY30B,KAAK6sF,SAASn4D,WACnCm4D,SAAS0B,YAAcpvC,IAAW,WACrC4hB,EAAK8rB,SAASn4D,KAAKF,MAAMi6D,QAAU,IACnC,WACG5B,SAAS2B,cAAgBrvC,IAAW,WACvC4hB,EAAKksB,iBACL,6CAWQa,EAAchsF,EAAOmuB,OAC7By+D,EAAWthE,SAASE,cAAc,SACtCohE,EAASx3D,KAAO,WAChBw3D,EAASpmD,UAAY,wCACrBomD,EAASC,QAAUb,OACLruF,IAAVqC,IACF4sF,EAASC,QAAU7sF,EACfA,IAAUgsF,IACgB,WAAxBroD,GAAOqoD,GACLhsF,IAAUgsF,EAAapR,cACpB4P,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,SAIxCwqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,UAK7C68C,EAAK3+C,KACT0uF,EAASrE,SAAW,WAAY1rC,EAAGvI,QAAQp2C,KAAK2uF,QAAS1+D,QAErDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOmmD,0CAUfZ,EAAchsF,EAAOmuB,OAC9By+D,EAAWthE,SAASE,cAAc,SACtCohE,EAASx3D,KAAO,OAChBw3D,EAASpmD,UAAY,oCACrBomD,EAAS5sF,MAAQA,EACbA,IAAUgsF,QACPxB,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,QAGzC68C,EAAK3+C,KACT0uF,EAASrE,SAAW,WAAY1rC,EAAGvI,QAAQp2C,KAAK8B,MAAOmuB,QAEnDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOmmD,2CAWd9xF,EAAKkF,EAAOmuB,cACtB2+D,EAAehyF,EAAI,GACnB8wF,EAAMtgE,SAASE,cAAc,OAGnB,UAFdxrB,OAAkBrC,IAAVqC,EAAsB8sF,EAAe9sF,IAG3C4rF,EAAIplD,UAAY,0CAChBolD,EAAIl5D,MAAMq1D,gBAAkB/nF,GAG5B4rF,EAAIplD,UAAY,+CAGlBxmC,OAAkBrC,IAAVqC,EAAsB8sF,EAAe9sF,EAC7C4rF,EAAI9C,QAAU,WACZlpB,EAAKmtB,iBAAiB/sF,EAAM4rF,EAAIz9D,QAG9BsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAKsY,EAAOmlD,4CAWZ5rF,EAAO4rF,EAAKz9D,cAE3By9D,EAAI9C,QAAU,kBAEToC,YAAY8B,SAASpB,QACrBV,YAAYxpB,YAEZwpB,YAAYjF,SAASjmF,QACrBkrF,YAAY+B,mBAAkB,SAAC1I,OAC9B2I,EAAc,QAAU3I,EAAMD,EAAI,IAAMC,EAAMC,EAAI,IAAMD,EAAMjpF,EAAI,IAAMipF,EAAMlpF,EAAI,IACtFuwF,EAAIl5D,MAAMq1D,gBAAkBmF,EAC5Bva,EAAKr+B,QAAQ44C,EAAY/+D,WAItB+8D,YAAYiC,kBAAiB,WAChCvB,EAAI9C,QAAU,WACZnW,EAAKoa,iBAAiB/sF,EAAM4rF,EAAIz9D,6CAcxB5uB,OAAK4uB,yDAAO,GAAIi/D,0DACxB1rB,GAAO,EACPvrC,KAASj4B,KAAKmvB,SACdggE,GAAe,MACd,IAAIC,KAAU/tF,KACbA,EAAIhE,eAAe+xF,GAAS,CAC9B5rB,GAAO,MACHzrB,EAAO12C,EAAI+tF,GACXC,EAAU/2C,GAAKqjC,mBAAmB1rD,EAAMm/D,MACtB,mBAAXn3D,IAII,KAHburC,EAAOvrC,EAAOm3D,EAAOn/D,OAIb8nB,aAAgB97C,QAA0B,iBAAT87C,GAAqC,kBAATA,GAAsBA,aAAgB77C,cAClGqwF,eAAgB,EACrB/oB,EAAOxjE,KAAKotF,cAAcr1C,EAAMs3C,GAAS,QACpC9C,eAA8B,IAAd2C,IAKd,IAAT1rB,EAAgB,CAClB2rB,GAAe,MACXrtF,EAAQ9B,KAAKsvF,UAAUD,MAEvBt3C,aAAgB97C,WACbszF,aAAax3C,EAAMj2C,EAAOutF,QAE5B,GAAoB,iBAATt3C,OACTy3C,eAAez3C,EAAMj2C,EAAOutF,QAE9B,GAAoB,kBAATt3C,OACT03C,cAAc13C,EAAMj2C,EAAOutF,QAE7B,GAAIt3C,aAAgB77C,OAAQ,KAE3BwzF,GAAO,MACsB,IAA7B97B,GAAA3jC,QAAAA,EAAa,YACXjwB,KAAK2sF,cAAcgD,QAAQC,SAAWR,IACxCM,GAAO,IAIE,IAATA,UAEmBjwF,IAAjBs4C,EAAK2kC,QAAuB,KAC1BmT,EAAcv3C,GAAKqjC,mBAAmB0T,EAAS,WAC/CS,EAAe9vF,KAAKsvF,UAAUO,OACb,IAAjBC,EAAuB,KACrBvnD,EAAQvoC,KAAK6tF,WAAWuB,EAAQC,GAAS,QACxChC,UAAUgC,EAAS9mD,GACxB4mD,EAAenvF,KAAKotF,cAAcr1C,EAAMs3C,IAAYF,YAG/CM,cAAc13C,EAAM+3C,EAAcT,OAGtC,KACC9mD,EAAQvoC,KAAK6tF,WAAWuB,EAAQC,GAAS,QACxChC,UAAUgC,EAAS9mD,GACxB4mD,EAAenvF,KAAKotF,cAAcr1C,EAAMs3C,IAAYF,QAKxDxsF,QAAQwpB,MAAM,0BAA2B4rB,EAAMq3C,EAAQC,WAKxDF,uCAWIvyF,EAAKkF,EAAOmuB,GACD,iBAAXrzB,EAAI,IAA8B,UAAXA,EAAI,SAC/BmzF,gBAAgBnzF,EAAKkF,EAAOmuB,GAC7BrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,KAEzC,iBAAXlF,EAAI,SACbozF,cAAcpzF,EAAKkF,EAAOmuB,GAC3BrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,KAEzC,iBAAXlF,EAAI,UACbqzF,WAAWrzF,EAAKkF,EAAOmuB,GACxBrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMu1C,OAAOv1C,sCAYrEA,EAAOmuB,OACTd,EAAUnvB,KAAKkwF,kBAAkBpuF,EAAMmuB,GAEvCjwB,KAAK4wB,OAAOgoB,MAAQ54C,KAAK4wB,OAAOgoB,KAAKiF,SAAW79C,KAAK4wB,OAAOgoB,KAAKiF,QAAQ4C,WACtE7vB,OAAOgoB,KAAKiF,QAAQ4C,KAAK,eAAgBtxB,QAE3Cq9D,aAAc,OACd57D,OAAO4tB,WAAWrvB,6CAYPrtB,EAAOmuB,OAAMkgE,yDAAa,GACtCluC,EAAUkuC,EAIdruF,EAAkB,WADlBA,EAAkB,SAAVA,GAA4BA,IACAA,MAE/B,IAAIhF,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IACf,WAAZmzB,EAAKnzB,UACkB2C,IAArBwiD,EAAQhyB,EAAKnzB,MACfmlD,EAAQhyB,EAAKnzB,IAAM,IAEjBA,IAAMmzB,EAAKjzB,OAAS,EACtBilD,EAAUA,EAAQhyB,EAAKnzB,IAGvBmlD,EAAQhyB,EAAKnzB,IAAMgF,UAIlBquF,8CAOHhhE,EAAUnvB,KAAKowF,kBACd9B,iBAAiB5iC,UAAY,sBAAwB3L,GAAe5wB,EAAS,KAAM,GAAK,sDAQzFA,EAAU,GACLryB,EAAI,EAAGA,EAAIkD,KAAKssF,eAAetvF,OAAQF,SACzCozF,kBAAkBlwF,KAAKssF,eAAexvF,GAAGgF,MAAO9B,KAAKssF,eAAexvF,GAAGmzB,KAAMd,UAE7EA,WC9sBUkhE,0BAQPpkC,EAAWzV,EAAOm2B,EAAQx9C,mEAE/B8wB,SAAW,IAAIvjD,OACf4zF,WAAY,IAEXC,gBAAgBF,SACd,IAAIG,YAAY,yDAIlBttD,GAAcypC,IAAWA,aAAkB/0B,IAAW+0B,aAAkB2H,KAAa3H,aAAkBzwE,OAAQ,KAC7Gu0F,EAAgBthE,EACtBA,EAAUw9C,EACVA,EAAS8jB,EAKPthE,GAAWA,EAAQwtD,gBACrBh6E,QAAQH,KAAK,6HAGTm8C,aACDzB,eAAiB,CACpB2a,YAAY,EACZL,oBAAqB,IACrBnP,YAAa,CACXC,KAAM,SACNvQ,KAAM,UAER3tB,OAAAA,MAEG+E,QAAUmpB,GAAKuN,WAAW,GAAInI,EAAKR,kBAGnCwL,QAAQuD,IACR98B,GAAYA,QAAiC,IAAfA,EAAQguB,IAAqB,KAE1DuzC,IADCtyC,IAAI+C,KAAK3sB,MAAMyqB,WAAa,iBAE7B0xC,EAAUjzC,EAAKU,IAAI+C,MACfuvC,GAAoBC,GAC1BD,EAAmB1kE,OAAO+8C,iBAAiB4nB,EAAS,MAAMryE,UAC1DqyE,EAAUA,EAAQC,gBAEfzhE,QAAQguB,IAAOuzC,GAAuD,OAAlCA,EAAiBpsF,qBAErD6qB,QAAQguB,IAAMhuB,EAAQguB,IAGzBhuB,IACEA,EAAQquB,gBAAoBruB,QAAQquB,YAAcruB,EAAQquB,aAC1DruB,EAAQ0tD,0BAA8B1tD,QAAQ0tD,sBAAwB1tD,EAAQ0tD,uBAC9E1tD,EAAQ6vC,cAAkB7vC,QAAQ6vC,UAAY7vC,EAAQ6vC,WACtD7vC,EAAQ2tD,0BAA8B3tD,QAAQ2tD,sBAAwB3tD,EAAQ2tD,4BAI9E+T,EAAwBzjE,SAASE,cAAc,UACjDowB,EAAKvuB,QAAQ2tD,sBAAuB,OAChC9e,EAAmBjoB,OAAK5mB,QAAQ2tD,qCAChCprB,EAAgBsM,EAAiBtgB,EAAKU,IAAIsT,eAC3CA,aAAyBx1D,UAAaw1D,aAAyBuM,SAClED,EAAiB6yB,GAEbn/B,aAAyBuM,SAC3B4yB,EAAsBnlC,UAAY,GAClCmlC,EAAsBl8D,YAAY+8B,IAEVjyD,MAAjBiyD,IACPm/B,EAAsBnlC,UAAYgG,YAInCtT,IAAIsT,cAAc/8B,YAAYk8D,KAG9Bt8B,WAAa,KAEb3b,KAAO,CACVwF,IAAKV,EAAKU,IACV5E,SAAUkE,EAAKrI,MACfwI,QAAS,CACPvH,GAAIP,OAAKO,kBACTC,IAAKR,OAAKQ,mBACVkK,KAAM1K,OAAK0K,qBAEbrH,YAAa,GACbd,KAAM,CACJsY,2BACSjS,EAAG8J,SAAS1kB,KAAK4X,OAE1BkV,0BACSlS,EAAG8J,SAAS1kB,KAAKA,MAG1ByX,SAAUzF,KAAA4I,EAAGmyC,kBAAenyC,GAC5BoyC,eAAgBh7C,KAAA4I,EAAGqyC,wBAAqBryC,GACxC1C,OAAQlG,KAAA4I,EAAGsyC,gBAAatyC,GACxBuyC,aAAen7C,KAAA4I,EAAGwyC,sBAAmBxyC,OAKpCjF,MAAQ,IAAImD,GAAMa,EAAK9E,KAAM8E,EAAKvuB,WAClColC,WAAWt3D,KAAKygD,EAAKhE,SACrBd,KAAKc,MAAQgE,EAAKhE,QAGlB+O,SAAW,IAAIV,GAASrK,EAAK9E,KAAM8E,EAAKvuB,WACxCmlC,UAAY,OACZC,WAAWt3D,KAAKygD,EAAK+K,YAGrBiP,YAAc,IAAIgB,GAAYhb,EAAK9E,KAAM8E,EAAKvuB,WAC9ColC,WAAWt3D,KAAKygD,EAAKga,eAGrBvF,QAAU,IAAIwc,GAAQjxB,EAAK9E,KAAM8E,EAAKvuB,WACtColC,WAAWt3D,KAAKygD,EAAKyU,WAErBsT,UAAY,OACZsK,WAAa,OAEb3xB,IAAI+C,KAAKypC,QAAU,SAAApqC,GACtB7B,EAAG8B,KAAK,QAAS9B,EAAGgV,mBAAmBnT,OAEpCpC,IAAI+C,KAAKirB,WAAa,SAAA5rB,GACzB7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,OAE1CpC,IAAI+C,KAAKiwC,cAAgB,SAAA5wC,GAC5B7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,OAE1CpC,IAAI+C,KAAKitC,YAAc,SAAA5tC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,KAE1Cx0B,OAAOqlE,gBACHjzC,IAAI+C,KAAKmwC,cAAgB,SAAA9wC,GAC5B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKowC,cAAgB,SAAA/wC,GAC5B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKqwC,YAAc,SAAAhxC,GAC1B7B,EAAG8B,KAAK,UAAW9B,EAAGgV,mBAAmBnT,SAGtCpC,IAAI+C,KAAKswC,YAAc,SAAAjxC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKuwC,YAAc,SAAAlxC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKwwC,UAAY,SAAAnxC,GACxB7B,EAAG8B,KAAK,UAAW9B,EAAGgV,mBAAmBnT,QAKxCoxC,gBAAiB,IACjBt7C,GAAG,WAAW,cACG,MAAhBqI,EAAG8mB,eACF9mB,EAAGizC,iBAAmBjzC,EAAGxvB,QAAQquB,eACpCmB,EAAGizC,gBAAiB,EACInyF,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,IAAkB,IACxChhC,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,QAC1CiZ,EAAQiF,EAAGkzC,mBAGXrxD,EAA4B/gC,MAApBk/C,EAAGxvB,QAAQqR,MAAqBme,EAAGxvB,QAAQqR,MAAQkZ,EAAMt3C,IACjEq+B,EAA4BhhC,MAApBk/C,EAAGxvB,QAAQsR,IAAqBke,EAAGxvB,QAAQsR,IAAQiZ,EAAMp0C,IACvEq5C,EAAGyX,UAAU51B,EAAOC,EAAK,CAACse,WAAW,SAErCJ,EAAGmzC,IAAI,CAAC/yC,WAAW,IAIlBJ,EAAGmT,kBAAoBnT,EAAGqT,yBAA4BrT,EAAGxvB,QAAQqR,OAAUme,EAAGxvB,QAAQsR,OACtFke,EAAGxvB,QAAQquB,cACdmB,EAAGmT,iBAAkB,EACrBnT,EAAGwT,QAAQL,iBAAkB,EAC7BnT,EAAGP,IAAI+C,KAAK3sB,MAAMyqB,WAAa,UAC/BN,EAAGP,IAAIsT,cAAcnqB,WAAWC,YAAYmX,EAAGP,IAAIsT,eAC/C/S,EAAGxvB,QAAQ0tD,2BACF,kBACFl+B,EAAGxvB,QAAQ0tD,0BACjB,UAKJvmC,GAAG,mBAAmB,WACzBqI,EAAGoG,aAID51B,KACGqvB,WAAWrvB,KAGbypB,KAAKiF,QAAQvH,GAAG,OAAO,SAACtzC,KACtB+uF,OAAO/uF,KACPqtD,YAIHsc,KACGvX,UAAUuX,GAIbn2B,KACG2e,SAAS3e,KAIXub,oFASE,IAAIo6B,GAAansF,KAAMA,KAAKo+C,IAAI6N,UAAW8wB,0CAW7C5qB,SAAWnyD,KAAKmyD,QAAQ+f,UAAU,CAACC,cAAc,SACjDpgB,6CAOI5iC,OAIU,IAFFwrD,GAAUqX,SAAS7iE,EAASqrD,KAG3C73E,QAAQq3C,IAAI,2DAA4D0gC,IAG1Ej/B,GAAKt/C,UAAUqiD,WAAWniD,KAAK2D,KAAMmvB,GAEjC,SAAUA,GACRA,EAAQ+H,OAASl3B,KAAKmvB,QAAQ+H,KAAM,MACjC/H,QAAQ+H,KAAO/H,EAAQ+H,SAGtBuuC,EAAYzlE,KAAKylE,aACnBA,EAAW,KACPwL,EAAYjxE,KAAKm2E,oBAClBhhB,SAAS,WACTA,SAASsQ,EAAUjuB,YACnBwhC,aAAa/H,sCAUjBz6B,OAIHy7C,OAHC3B,WAAY,EAQf2B,EAJGz7C,EAIUe,GADNf,aAAiBoB,IAAWpB,aAAiB89B,GACrB99B,EAIA,IAAIoB,GAAQpB,IAP9B,KAWXx2C,KAAKylE,gBAEFA,UAAUptB,eAEZotB,UAAYwsB,OACZ9/B,SAAWnyD,KAAKmyD,QAAQgD,SAAuB,MAAd88B,EAAqBA,EAAWz6C,MAAQ,wCAOtEm1B,OAEJslB,KACCtlB,EAGA,KACG10C,EAAS,SAAAosC,UAA2B,IAAlBA,EAAM/I,SAE5B22B,EADEtlB,aAAkB/0B,IAAW+0B,aAAkB2H,GACpC,IAAIA,GAAS3H,EAAO,CAAC10C,OAAAA,IAIrB,IAAI2f,GAAQd,GAAA61B,QAAAA,EAAc10C,SATzCg6D,EAAa,KAwBXjyF,KAAK+vE,sBAAsBuE,SACxBvE,WAAWrS,QAAQ,WAErBqS,WAAakiB,OACb9/B,QAAQiD,UAAU68B,mCAOjB1+E,GACFA,GAAQA,EAAKo5D,aACVvX,UAAU7hD,EAAKo5D,QAGlBp5D,GAAQA,EAAKijC,YACV2e,SAAS5hD,EAAKijC,4CAqBV+7B,EAAKpjD,QACXgjC,SAAWnyD,KAAKmyD,QAAQ6mB,aAAazG,GAEtCpjD,GAAWA,EAAQshC,YAChBA,MAAM8hB,EAAKpjD,iDASXnvB,KAAKmyD,SAAWnyD,KAAKmyD,QAAQgkB,gBAAkB,iCAkBlD/iD,EAAIjE,MACHnvB,KAAKylE,WAAmBhmE,MAAN2zB,OAEjBm/C,EAAMrvC,GAAc9P,GAAMA,EAAK,CAACA,GAGhCqyC,EAAYzlE,KAAKylE,UAAU17D,IAAIwoE,GAGjC/xC,EAAQ,KACRC,EAAM,WACVglC,QAAAA,GAAkB,SAAAzR,OACVnrD,EAAImrD,EAASxzB,MAAMjjC,UACnB8V,EAAI,QAAS2gD,EAAWA,EAASvzB,IAAIljC,UAAYy2D,EAASxzB,MAAMjjC,WAExD,OAAVijC,GAAkB33B,EAAI23B,KACxBA,EAAQ33B,IAGE,OAAR43B,GAAgBptB,EAAIotB,KACtBA,EAAMptB,MAKI,OAAVmtB,GAA0B,OAARC,EAAc,KAC5Bke,EAAK3+C,KAEL+3C,EAAO/3C,KAAKmyD,QAAQ3b,MAAM+7B,EAAI,IAChC2f,GAAmC,EAAxBlyF,KAAKw4D,gBAChB25B,EAAwB,KAmCtBC,EAA2B,eACzBC,EAAsBC,GAAsB3zC,EAAI5G,GAElDs6C,EAAoBE,cAAgBF,EAAoBG,SAAWL,EAAsBK,UAC3F7zC,EAAGuU,eAAem/B,EAAoBI,cACtC9zC,EAAGoT,YAeDzP,GAAOnzB,QAA4B1vB,IAAjB0vB,EAAQmzB,MAAqBnzB,EAAQmzB,KACvDowC,GAAUlyD,EAAQC,GAAO,EACzBoe,EAAWyD,EAAuB,KAAf7hB,EAAMD,GAAeh/B,KAAK8D,IAAItF,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,MAAuB,KAAfC,EAAMD,IAE3Fue,GAAY5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA0B5vB,EAAQ4vB,UAElEA,IAEHozC,EAAwB,CAAEI,cAAc,EAAOE,cAAe,EAAGD,SAAU,SAGxE94C,MAAMgF,SAASg0C,EAAS7zC,EAAW,EAAG6zC,EAAS7zC,EAAW,EAAG,CAAEE,UAAAA,IApBtC,WAE5BqzC,OAGWA,EAA0B,QAhDR,SAAChyC,EAAMuyC,EAAUx1D,OACxC41B,EAAiBu/B,GAAsB3zC,EAAI5G,OAE1B,IAAnBgb,IAIAo/B,IACFA,EAAwBp/B,GAGvBo/B,EAAsBK,SAAWz/B,EAAey/B,SAAYL,EAAsBI,eAG7EJ,EAAsBK,SAAWz/B,EAAey/B,SAAWz/B,EAAew/B,eAEhFJ,EAAwBp/B,EACxBm/B,GAAiC,EAAtBvzC,EAAG6Z,qBAGVn4D,EAAO6xF,EACP9xF,EAAK+xF,EAAsBM,aAC3B7gC,EAAYz0B,EAAO/8B,EAAMC,GAAQD,EAAKC,GAAQ+/C,EAEpDzB,EAAGuU,eAAetB,GAEd+gC,GACFh0C,EAAGoT,4CAmDP5iC,EAASnpB,OAEP0zC,EADEqF,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,UAG3C,IAA1B/+C,KAAKylE,UAAUzoE,aAAgDyC,IAAhCO,KAAKylE,UAAU17D,MAAM,GAAG02B,KAEzDiZ,EAAQ15C,KAAK81D,oBACR/S,OAAOrJ,EAAMt3C,IAAI7E,UAAW,CAACwhD,UAAAA,GAAY/4C,KAI9C0zC,EAAQ15C,KAAK6xF,oBACRn4C,MAAMgF,SAAShF,EAAMt3C,IAAKs3C,EAAMp0C,IAAK,CAAEy5C,UAAAA,GAAa/4C,sDAYrD0zC,EAAQ15C,KAAK81D,eACf1zD,EAAoB,OAAds3C,EAAMt3C,IAAes3C,EAAMt3C,IAAI7E,UAAY,KACjD+H,EAAoB,OAAdo0C,EAAMp0C,IAAeo0C,EAAMp0C,IAAI/H,UAAY,KACjDq1F,EAAU,KACVC,EAAU,QAEH,MAAPzwF,GAAsB,MAAPkD,EAAa,KAC1Bu5C,EAAYv5C,EAAMlD,EAClBy8C,GAAY,IACdA,EAAW,QAEPi0C,EAASj0C,EAAW7+C,KAAKq1C,MAAM2J,OAAOpW,MAEtC41B,EAAc,GAChBC,EAAoB,QAGxBnmB,SAAAA,GAAat4C,KAAKmyD,QAAQ3b,OAAO,SAACuB,EAAM90C,MAClC80C,EAAKksB,aAAc,CAErBzF,EAAYv7D,GAAO80C,EAAKsY,QADJ,GAEpBoO,EAAoBD,EAAYv7D,GAAKjG,WAItByhE,EAAoB,qBAG5B3hE,MACPw7C,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI9hE,SAFCA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,SAQXw7C,SAAAA,GAAat4C,KAAKmyD,QAAQ3b,OAAO,SAAAuB,OAG3Bg7C,EACAC,EAHExyD,EAAQyyD,GAASl7C,GACjBtX,EAAMyyD,GAAOn7C,GAIf4d,EAAKxmC,QAAQguB,KACf41C,EAAavyD,GAASuX,EAAKo7C,gBAAmB,IAAML,EACpDE,EAAUvyD,GAASsX,EAAKq7C,eAAiB,IAAMN,IAE/CC,EAAavyD,GAASuX,EAAKq7C,eAAkB,IAAMN,EACnDE,EAAUvyD,GAASsX,EAAKo7C,gBAAkB,IAAML,GAG9CC,EAAY3wF,IACdA,EAAM2wF,EACNH,EAAU76C,GAERi7C,EAAU1tF,IACZA,EAAM0tF,EACNH,EAAU96C,MAIV66C,GAAWC,EAAS,KAChBQ,EAAMT,EAAQQ,eAAiB,GAC/BE,EAAMT,EAAQM,gBAAkB,GAChC7xC,EAAQthD,KAAKq1C,MAAM2J,OAAOpW,MAAQyqD,EAAMC,EAE1ChyC,EAAQ,IACNthD,KAAKmvB,QAAQguB,KACf/6C,EAAM6wF,GAASL,GAAWU,EAAMz0C,EAAWyC,EAC3Ch8C,EAAM4tF,GAAOL,GAAaQ,EAAMx0C,EAAWyC,IAE3Cl/C,EAAM6wF,GAASL,GAAWS,EAAMx0C,EAAWyC,EAC3Ch8C,EAAM4tF,GAAOL,GAAaS,EAAMz0C,EAAWyC,WAM5C,CACLl/C,IAAY,MAAPA,EAAc,IAAI1F,KAAK0F,GAAO,KACnCkD,IAAY,MAAPA,EAAc,IAAI5I,KAAK4I,GAAO,mDASjClD,EAAM,KACNkD,EAAM,KAENtF,KAAKylE,qBACFA,mBAAkB,SAAA1tB,OACfvX,EAAQ8X,GAAKpB,QAAQa,EAAKvX,MAAO,QAAQjjC,UACzCkjC,EAAQ6X,GAAKpB,QAAoBz3C,MAAZs4C,EAAKtX,IAAmBsX,EAAKtX,IAAMsX,EAAKvX,MAAO,QAAQjjC,WACtE,OAAR6E,GAAgBo+B,EAAQp+B,KAC1BA,EAAMo+B,IAEI,OAARl7B,GAAgBm7B,EAAMn7B,KACxBA,EAAMm7B,YAKL,CACLr+B,IAAY,MAAPA,EAAc,IAAI1F,KAAK0F,GAAO,KACnCkD,IAAY,MAAPA,EAAc,IAAI5I,KAAK4I,GAAO,iDAUpBk7C,OACX2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDC,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChDO,EAAsB3iD,KAAKo+C,IAAI3E,gBAAgBmJ,wBAC/Cz4C,EAAInK,KAAKmvB,QAAQguB,IAAMwF,EAAoBhO,MAAQwN,EAAUA,EAAUQ,EAAoBjO,KAC3FnrC,EAAI64C,EAAUO,EAAoBG,IAElC/K,EAAQ/3C,KAAKmyD,QAAQqjB,eAAeh1B,GACpC6jB,EAAQrkE,KAAKmyD,QAAQqkB,gBAAgBh2B,GACrCmP,EAAaH,GAAW+jC,qBAAqB/yC,GAE7CsQ,EAAO9wD,KAAKmyD,QAAQhjC,QAAQ2hC,MAAQ,KACpCnV,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBnwC,EAAO1gB,KAAKixF,QAAQ9mF,GACpB4mD,EAAcD,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,EAE/CknB,EAAU0Q,GAAKk7C,UAAUhzC,GAC3BizC,EAAO,YACC,MAAR17C,EAAkE07C,EAAO,OACtD,MAAd9jC,EAA6D8jC,EAAO,cACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKyoD,SAASrK,IAAI4J,aAC1ChoD,KAAKs0D,WAAahc,GAAKo7C,UAAU9rD,EAAS5nC,KAAKs0D,UAAUlW,IAAI4J,YADAyrC,EAAO,OAEpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmyD,QAAQ/T,IAAIujB,UAAoB8xB,EAAO,cACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAK03D,YAAY7H,KAAoB4jC,EAAO,eACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKo+C,IAAIY,UAA4By0C,EAAO,cAEtE,CACLjzC,MAAAA,EACAzI,KAAMA,EAAOA,EAAK3kB,GAAK,KACvBwsC,YAAW7nB,KAASA,EAAK6nB,UACzBppB,MAAOuB,EAAOA,EAAKvB,OAAS,GAAI,KAChC6tB,MAAOA,EAAQA,EAAMlI,QAAU,KAC/BxM,WAAYA,EAAaA,EAAWxgC,QAAQiE,GAAK,KACjDqgE,KAAAA,EACAE,MAAOnzC,EAAM4D,SAAW5D,EAAM4D,SAASuvC,MAAQnzC,EAAMmzC,MACrDC,MAAOpzC,EAAM4D,SAAW5D,EAAM4D,SAASwvC,MAAQpzC,EAAMozC,MACrDzpF,EAAAA,EACAZ,EAAAA,EACAmX,KAAAA,EACAqwC,YAAAA,+CAQE/wD,KAAK05C,MAAMqD,aACRrD,MAAMkF,eAEqBn/C,MAA5BO,KAAKmvB,QAAQquB,kBACVgB,WAAWx+C,KAAKmvB,cAElBuqB,MAAM6E,kDASb9C,GAAKt/C,UAAU41D,QAAQ11D,KAAK2D,qCAQvBgD,OACGw9B,EAA0Bx9B,EAA1Bw9B,MAAOC,EAAmBz9B,EAAnBy9B,IAAKse,EAAc/7C,EAAd+7C,UACfte,OAKEiZ,MAAMgF,SAASle,EAAOC,EAAK,CAC9Bse,UAAWA,SALRgE,OAAOviB,EAAMjjC,UAAW,CAC3BwhD,UAAAA,WA7tB8BtD,IA4uBtC,SAASw3C,GAASl7C,UACTO,GAAKpB,QAAQa,EAAKxkC,KAAKitB,MAAO,QAAQjjC,UAQ/C,SAAS21F,GAAOn7C,OACRtX,EAAuBhhC,MAAjBs4C,EAAKxkC,KAAKktB,IAAmBsX,EAAKxkC,KAAKktB,IAAMsX,EAAKxkC,KAAKitB,aAC5D8X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAQnC,SAAS+0F,GAAsBuB,EAAU97C,OAClCA,EAAKnnB,cAED,MAGHkjE,EAAgBD,EAAS1kE,QAAQguB,IAAM02C,EAASx+C,MAAM8b,eAAetoB,OAASgrD,EAASx+C,MAAM6b,cAAcroB,OAC3G8tB,EAAgBk9B,EAASx+C,MAAM2J,OAAOnW,OAEtCw7B,EAAQtsB,EAAKnnB,OACfpU,EAAS6nD,EAAMvhB,IACfyvC,GAAe,EACblqC,EAAcwrC,EAASprC,SAASt5B,QAAQk5B,YAAYC,KAEpDkqC,EAAU,iBACG,UAAfnqC,EACOgc,EAAMx7B,OAASkP,EAAK+K,IAAM/K,EAAKlP,OAG/BkP,EAAK+K,KAIVixC,GAAkD,EAA5BF,EAASr7B,gBAC/Bw7B,EAAex3E,EAASg2E,IACxB3pD,EAASkP,EAAKlP,cAEhBmrD,EAAeD,EACbv3E,EAASs3E,GAAiBt3E,EAASg2E,IAAY3pD,IACjDrsB,GAAUg2E,IAAYqB,EAAS1hC,QAAQhjC,QAAQmrC,OAAOviB,KAAK8iB,UAGtDm5B,EAAenrD,EAASkrD,EAAsBD,EACrDt3E,GAAUg2E,IAAY3pD,EAASirD,EAAgBD,EAAS1hC,QAAQhjC,QAAQmrC,OAAOviB,KAAK8iB,SAGpF03B,GAAe,EAKV,CAAEA,aAAAA,EAAcE,aAFvBj2E,EAAShb,KAAKY,IAAIoa,EAAQm6C,EAAgBm9B,GAEGtB,QAASwB,OC7zBlDC,yBAcFzzD,EACAC,EACAyzD,EACAC,EACAt9B,EACA1N,OACAirC,0DACAC,6EACKC,WAAa,CAAC,EAAG,EAAG,EAAG,SACvBC,WAAa,CAAC,IAAM,GAAK,EAAG,QAC5BC,YAAc,UAEd39B,gBAAkBA,OAClB1N,gBAAkBA,OAClBzD,OAASllB,OACT0a,KAAOza,OAEPkb,MAAQ,OACR84C,cAAgB,OAChBC,gBAAkB,OAClBC,sBAEAP,UAAYA,OACZF,eAAiBA,OACjBC,aAAeA,OAEfE,mBAAqBA,EAEtBH,GAAkBC,EAAc,KAC5Bx1C,EAAK3+C,KACL+lD,EAAe,SAAAjkD,OACb8yF,EAAU9yF,EAASA,GAAS68C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,sBACpE3yF,GAAS68C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,eAAwB91C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,cAA7C,GAC3DG,EAAWj2C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,cAGjDG,GAGPV,SACGxuC,QAAiC,EAAvB1lD,KAAK00F,gBAAsB10F,KAAKu0F,WAAWv0F,KAAKy0F,mBAC1D/uC,OAASK,EAAa/lD,KAAK0lD,SAG9ByuC,SACGj5C,MAAQl7C,KAAK00F,gBAAkB10F,KAAKu0F,WAAWv0F,KAAKy0F,mBACpDv5C,KAAO6K,EAAa/lD,KAAKk7C,YAE3By5C,mEAQKxrC,QACPA,gBAAkBA,oCAOf0N,QACHA,gBAAkBA,+CAOjBnd,EAAQ15C,KAAKk7C,KAAOl7C,KAAK0lD,YAC1B/J,MAAQ37C,KAAK62D,gBAAkBnd,MAC9Bm7C,EAAmB70F,KAAKmpD,gBAAkBnpD,KAAK27C,MAC/Cm5C,EAAoBp7C,EAAQ,EAC5Bl4C,KAAK+a,MAAM/a,KAAKw4C,IAAIN,GAASl4C,KAAKuzF,MAClC,OAEDN,cAAgB,OAChBC,gBAAkBlzF,KAAK6D,IAAI,GAAIyvF,OAEhCt0D,EAAQ,EACRs0D,EAAmB,IACrBt0D,EAAQs0D,WAGNE,GAAgB,EACXp+E,EAAI4pB,EAAOh/B,KAAKc,IAAIsU,IAAMpV,KAAKc,IAAIwyF,GAAmBl+E,IAAK,MAC7D89E,gBAAkBlzF,KAAK6D,IAAI,GAAIuR,OAC/B,IAAI7C,EAAI,EAAGA,EAAI/T,KAAKu0F,WAAWv3F,OAAQ+W,IAAK,IAC9B/T,KAAK00F,gBAAkB10F,KAAKu0F,WAAWxgF,IACxC8gF,EAAkB,CAChCG,GAAgB,OACXP,aAAe1gF,aAIF,IAAlBihF,0CAWClzF,UACCA,GAAS9B,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,gBAAmB,2CAQzEz0F,KAAK00F,gBAAkB10F,KAAKu0F,WAAWv0F,KAAKy0F,0DAQ7CQ,EAAYj1F,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,qBACvDz0F,KAAKk1F,aAAal1F,KAAK0lD,QAAWuvC,EAAaj1F,KAAK0lD,OAASuvC,GAAcA,uCAQxEh6C,OACNk6C,EAAcl6C,EAAQm6C,YAAY,SACC,mBAA5Bp1F,KAAKq0F,qBACdc,EAAcn1F,KAAKq0F,mBAAmBp5C,IAGb,iBAAhBk6C,YACCA,GAEoB,iBAAhBA,EACPA,EAGAl6C,EAAQm6C,YAAY,8CAUvBntC,EAAQ,GACRlkB,EAAO/jC,KAAK6wD,UACZwkC,GAAgBtxD,EAAQ/jC,KAAK0lD,OAAS3hB,GAASA,EAC5CjnC,EAAKkD,KAAK0lD,OAAS2vC,EAAer1F,KAAKk7C,KAAKp+C,EAAI,KAASA,GAAKinC,EACjEjnC,GAAKkD,KAAK0lD,QACZuC,EAAMhrD,KAAK,CAACq4F,MAAOt1F,KAAKu1F,SAASz4F,GAAIyM,EAAGvJ,KAAKk1F,aAAap4F,GAAIyD,IAAKP,KAAKw1F,YAAY14F,YAGjFmrD,sCAOGltC,OACJ06E,EAAaz1F,KAAKy0F,aAClBiB,EAAW11F,KAAK0lD,OAChBiwC,EAAS31F,KAAKk7C,KAEdyD,EAAK3+C,KACL41F,EAAoB,WACxBj3C,EAAG+1C,iBAAmB,GAElBmB,EAAoB,WACxBl3C,EAAG+1C,iBAAmB,GAGnB35E,EAAM05E,cAAgB,GAAKz0F,KAAKy0F,cAAgB,GAAO15E,EAAM05E,aAAe,GAAKz0F,KAAKy0F,aAAe,IAE/F15E,EAAM05E,aAAez0F,KAAKy0F,mBAE9BA,aAAe,EACF,GAAdgB,GAGFG,IAFAA,WAOGnB,aAAe,EACF,GAAdgB,GAGFI,IAFAA,cAQEC,EAAY/6E,EAAMm6E,aAAa,GAC/Ba,EAAYh7E,EAAM81C,UAAY91C,EAAM4gC,MAEtCxe,GAAO,EACP8b,EAAQ,GAEJ9b,GAAQ8b,IAAS,GAAG,MAGrB0C,MAAQo6C,GAAa/1F,KAAKu0F,WAAWv0F,KAAKy0F,cAAgBz0F,KAAK00F,qBAC9DsB,EAAWh2F,KAAK62D,gBAAkB72D,KAAK27C,WAGxC+J,OAASgwC,OACTx6C,KAAOl7C,KAAK0lD,OAASswC,MAEpBC,EAAiBj2F,KAAKk7C,KAAOl7C,KAAK27C,MAClCs5C,EAAYj1F,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,cACxDyB,EAAcl2F,KAAKm2F,gBAAkBp7E,EAAMo7E,mBAE7Cn2F,KAAKo0F,UAAW,KACZgC,EAAaN,EAAYG,OAC1B/6C,MAASk7C,EAAap2F,KAAK27C,WAC3B+J,OAAS1lD,KAAKk7C,KAAO86C,OAErBh2F,KAAKk0F,qBAIHxuC,QAAUwwC,EAAcl2F,KAAK27C,WAC7BT,KAAOl7C,KAAK0lD,OAASswC,SAJrBtwC,QAAUuvC,EAAaiB,EAAcl2F,KAAK27C,WAC1CT,KAAOl7C,KAAK0lD,OAASswC,OAMzBh2F,KAAKm0F,cAAgBn0F,KAAKk7C,KAAOy6C,EAAO,KAE3CE,IACA14D,GAAO,WAGJn9B,KAAKk0F,gBAAkBl0F,KAAK0lD,OAASgwC,EAAS,KAAS,MACtD11F,KAAKo0F,WAAasB,GAAY,GAE3B,CAELG,IACA14D,GAAO,WAJPx6B,QAAQH,KAAK,uDAQbxC,KAAKk0F,gBAAkBl0F,KAAKm0F,cAAgB6B,EAAYL,EAAOD,GACjEE,IACAz4D,GAAO,GAGTA,GAAO,yCASEr7B,UACJ9B,KAAK62D,iBAAoB/0D,EAAQ9B,KAAK0lD,QAAU1lD,KAAK27C,4CAQhD06C,UACHr2F,KAAK62D,gBAAkBw/B,GAAUr2F,KAAK27C,MAAS37C,KAAK0lD,4+LCjS3D4wC,0BAUQ19C,EAAMzpB,EAASonE,EAAKC,kDAEzBpjE,GAAKs3C,OACL9xB,KAAOA,IAEPsE,eAAiB,CACpBmL,YAAa,OACbE,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfmwC,OAAO,EACPC,iBAAkB,EAClBC,iBAAkB,EAClBC,aAAc,GACdC,aAAc,EACdC,UAAW,GACXluD,MAAO,OACP0yB,SAAS,EACTy7B,YAAY,EACZriD,KAAM,CACJgF,MAAO,CAACt3C,SAAK3C,EAAW6F,SAAK7F,GAC7BhC,gBAAOqE,mBACKg/C,GAAWh/C,EAAMszF,YAAY,MAEzC3lC,MAAO,CAAChE,UAAMhsD,EAAW+0B,WAAO/0B,IAElCk1C,MAAO,CACL+E,MAAO,CAACt3C,SAAK3C,EAAW6F,SAAK7F,GAC7BhC,gBAAOqE,mBACKg/C,GAAWh/C,EAAMszF,YAAY,MAEzC3lC,MAAO,CAAChE,UAAMhsD,EAAW+0B,WAAO/0B,OAI/B+2F,iBAAmBA,IACnBQ,aAAeT,IACflhD,MAAQ,KACR4hD,YAAc,CACjBhvC,MAAO,GACPivC,OAAQ,GACRznC,MAAO,MAGJrR,IAAM,KACNzC,WAAQl8C,IACRi6C,MAAQ,CAAClZ,MAAO,EAAGC,IAAK,KAExBtR,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/Bi6C,iBAAmB,IAEnB34C,WAAWrvB,KACXyZ,MAAQyO,OAAO,UAAIqG,EAAKvuB,QAAQyZ,OAASriC,QAAQ,KAAM,OACvDqlE,SAAWluB,EAAK9U,QAChBC,OAAS6U,EAAKs5C,aAAap0C,wBAAwB/Z,SACnD0R,QAAS,IAET68C,WAAa,KACbC,cAAgB,IAChBC,eAAiB,IAEjB9rB,WAAa,IACb+rB,QAAS,IACTC,WAAa,OACbC,YAAc,KACdC,cAAe,IAEf/qB,OAAS,KACTgrB,eAAiB,IAGjBjvC,UACajpD,MAAdi+C,EAAK/B,SACFi8C,kBAEFC,UAAY,CAACtB,IAAK74C,EAAK64C,IAAKkB,YAAa/5C,EAAK+5C,YAAatoE,QAASuuB,EAAKvuB,QAASw9C,OAAQjvB,EAAKivB,YAE9FhuB,iBACD/F,KAAKiF,QAAQvH,GAAG,gBAAgB,WACnCqI,EAAGP,IAAI05C,cAActjE,MAAMsuB,cAASnE,EAAG/F,KAAKY,SAASoY,yEAShDrpB,EAAOwvD,GACT/3F,KAAK2sE,OAAOtvE,eAAekrC,UACzBokC,OAAOpkC,GAASwvD,QAElBJ,gBAAkB,sCAQbpvD,EAAOwvD,GACZ/3F,KAAK2sE,OAAOtvE,eAAekrC,UACzBovD,gBAAkB,QAEpBhrB,OAAOpkC,GAASwvD,sCAOXxvD,GACNvoC,KAAK2sE,OAAOtvE,eAAekrC,YACtBvoC,KAAK2sE,OAAOpkC,QACdovD,gBAAkB,sCAQhBxoE,MACLA,EAAS,KACPkhC,GAAS,EACTrwD,KAAKmvB,QAAQk5B,aAAel5B,EAAQk5B,kBAAuC5oD,IAAxB0vB,EAAQk5B,cAC7DgI,GAAS,GAkBX/X,GAAKqQ,oBAhBU,CACb,cACA,kBACA,kBACA,QACA,mBACA,mBACA,eACA,eACA,YACA,QACA,UACA,OACA,QACA,cAE+B3oD,KAAKmvB,QAASA,QAE1Cy8C,SAAWv0B,OAAO,UAAIr3C,KAAKmvB,QAAQyZ,OAASriC,QAAQ,KAAM,MAChD,IAAX8pD,GAAmBrwD,KAAKo+C,IAAIuqB,aACzBzY,YACAsT,gDASJplB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKmvB,QAAQyZ,WACrCwV,IAAIuqB,MAAMn0C,MAAMqU,OAAS7oC,KAAK6oC,YAE9BuV,IAAI05C,cAAgB1qE,SAASE,cAAc,YAC3C8wB,IAAI05C,cAActjE,MAAMoU,MAAQ,YAChCwV,IAAI05C,cAActjE,MAAMqU,OAAS7oC,KAAK6oC,YACtCuV,IAAI05C,cAActjE,MAAMkL,SAAW,gBACnC0e,IAAI05C,cAActjE,MAAMyqB,WAAa,eACrCb,IAAI05C,cAActjE,MAAMC,QAAU,aAGlC8hE,IAAMnpE,SAASya,gBAAgB,6BAA8B,YAC7D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMsuB,IAAM,WAChByzC,IAAI/hE,MAAMqU,OAAS,YACnB0tD,IAAI/hE,MAAMoU,MAAQ,YAClB2tD,IAAI/hE,MAAMC,QAAU,aACpB2pB,IAAIuqB,MAAMh0C,YAAY30B,KAAKu2F,qDAS5BpsF,EAFJ6tF,GAAwBh4F,KAAKy3F,iBAGvBX,EAAY92F,KAAKmvB,QAAQ2nE,UAG3BvtF,EAAI0uF,KAGN9tF,EAD+B,SAA7BnK,KAAKmvB,QAAQk5B,YAHE,EAObroD,KAAK4oC,MAAQkuD,EAPA,MAUboB,EAAal4C,GAAYhgD,KAAK2sE,WACpCurB,QAAAA,GAAgB,SAAC/6F,EAAGC,UAAMD,EAAIC,GAAK,EAAI,+CAEjB86F,+BAAY,KAAvB/7B,WAC4B,IAAjCn8D,KAAK2sE,OAAOxQ,GAASb,cAAmE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,KAAwE,IAA9Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,UACnIwQ,OAAOxQ,GAASg8B,UAAUrB,EAhBhB,GAgBuC92F,KAAK63F,UAAW1tF,EAAGZ,GACzEA,GAAK6uF,uFAITJ,GAAwBh4F,KAAKy3F,kBACxBC,cAAe,2CAOM,IAAtB13F,KAAK03F,eACPM,GAAwBh4F,KAAKy3F,aAC7BO,GAAwBh4F,KAAKy3F,kBACxBC,cAAe,uCAQjBn9C,QAAS,EACTv6C,KAAKo+C,IAAIuqB,MAAMphC,aACe,SAA7BvnC,KAAKmvB,QAAQk5B,iBACVzP,KAAKwF,IAAI1J,KAAK/f,YAAY30B,KAAKo+C,IAAIuqB,YAGnC/vB,KAAKwF,IAAIzJ,MAAMhgB,YAAY30B,KAAKo+C,IAAIuqB,QAIxC3oE,KAAKo+C,IAAI05C,cAAcvwD,iBACrBqR,KAAKwF,IAAI6S,qBAAqBt8B,YAAY30B,KAAKo+C,IAAI05C,oBAErD15C,IAAI05C,cAActjE,MAAMC,QAAU,4CAOlC8lB,QAAS,EACVv6C,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,YAG5CvqB,IAAI05C,cAActjE,MAAMC,QAAU,wCAQhC+L,EAAOC,QACTiZ,MAAMlZ,MAAQA,OACdkZ,MAAMjZ,IAAMA,uCAQboY,GAAU,EACVw/C,EAAe,MAKd,IAAMl8B,UAFN/d,IAAI05C,cAActjE,MAAMsuB,cAAS9iD,KAAK44C,KAAKY,SAASoY,gBAEnC5xD,KAAK2sE,OACrB3sE,KAAK2sE,OAAOtvE,eAAe8+D,MACQ,IAAjCn8D,KAAK2sE,OAAOxQ,GAASb,cAAmE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,KAAwE,IAA9Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,IACxIk8B,QAIsB,IAAxBr4F,KAAK23F,gBAAyC,IAAjBU,OAC1BnoC,WAEF,MACEsT,YACA36B,OAASwO,OAAOr3C,KAAKg3F,aAAaxiE,MAAMqU,OAAOtiC,QAAQ,KAAM,UAG7D63C,IAAI05C,cAActjE,MAAMqU,iBAAY7oC,KAAK6oC,kBACzCD,OAAiC,IAAzB5oC,KAAKmvB,QAAQmsC,QAAmBjkB,OAAO,UAAIr3C,KAAKmvB,QAAQyZ,OAASriC,QAAQ,KAAM,KAAO,MAE7F8uC,EAAQr1C,KAAKq1C,MACbszB,EAAQ3oE,KAAKo+C,IAAIuqB,MAGvBA,EAAMrgC,UAAY,qBAGbygB,yBAECV,EAAcroD,KAAKmvB,QAAQk5B,YAC3BE,EAAkBvoD,KAAKmvB,QAAQo5B,gBAC/BtC,EAAkBjmD,KAAKmvB,QAAQ82B,gBAE/BqyC,EAAkCt4F,KAAK44C,KAAKwF,IAAI6S,qBAAqB7H,YAG3E/T,EAAM2T,iBAAmBT,EAAkBlT,EAAM4T,gBAAkB,EACnE5T,EAAM6T,iBAAmBjD,EAAkB5Q,EAAM8T,gBAAkB,EAEnE9T,EAAMiU,eAAiBgvC,EAAkCt4F,KAAKwrE,WAAaxrE,KAAK4oC,MAAQ,EAAI5oC,KAAKmvB,QAAQwnE,iBACzGthD,EAAMgU,gBAAkB,EACxBhU,EAAMmU,eAAiB8uC,EAAkCt4F,KAAKwrE,WAAaxrE,KAAK4oC,MAAQ,EAAI5oC,KAAKmvB,QAAQunE,iBACzGrhD,EAAMkU,gBAAkB,EAGJ,SAAhBlB,GACFsgB,EAAMn0C,MAAMsuB,IAAM,IAClB6lB,EAAMn0C,MAAMkgB,KAAO,IACnBi0B,EAAMn0C,MAAMq0B,OAAS,GACrB8f,EAAMn0C,MAAMoU,gBAAW5oC,KAAK4oC,YAC5B+/B,EAAMn0C,MAAMqU,iBAAY7oC,KAAK6oC,kBACxBwM,MAAMzM,MAAQ5oC,KAAK44C,KAAKY,SAAS9E,KAAK9L,WACtCyM,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAAS9E,KAAK7L,SAG5C8/B,EAAMn0C,MAAMsuB,IAAM,GAClB6lB,EAAMn0C,MAAMq0B,OAAS,IACrB8f,EAAMn0C,MAAMkgB,KAAO,IACnBi0B,EAAMn0C,MAAMoU,gBAAW5oC,KAAK4oC,YAC5B+/B,EAAMn0C,MAAMqU,iBAAY7oC,KAAK6oC,kBACxBwM,MAAMzM,MAAQ5oC,KAAK44C,KAAKY,SAAS7E,MAAM/L,WACvCyM,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAAS7E,MAAM9L,QAG/CgQ,EAAU74C,KAAK43F,gBACf/+C,EAAU74C,KAAK8pD,cAAgBjR,GAEJ,IAAvB74C,KAAKmvB,QAAQsnE,WACV8B,yBAGAC,qBAGFC,aAAapwC,UAEbxP,qDAUHA,GAAU,EACdm/C,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,YACnC7uC,EAAcroD,KAAKmvB,QAAL,YACdupE,EAAiDj5F,MAAnCO,KAAKmvB,QAAQk5B,GAAa3O,MAAqB15C,KAAKmvB,QAAQk5B,GAAa3O,MAAQ,GAGjGy6C,GAAe,EACI10F,MAAnBi5F,EAAYpzF,WACTo0C,MAAMjZ,IAAMi4D,EAAYpzF,IAC7B6uF,GAAe,OAEbD,GAAiB,EACEz0F,MAAnBi5F,EAAYt2F,WACTs3C,MAAMlZ,MAAQk4D,EAAYt2F,IAC/B8xF,GAAiB,QAGdv4C,MAAQ,IAAIs4C,GACfj0F,KAAK05C,MAAMlZ,MACXxgC,KAAK05C,MAAMjZ,IACXyzD,EACAC,EACAn0F,KAAKo+C,IAAIuqB,MAAMjS,aACf12D,KAAKq1C,MAAM8T,gBACXnpD,KAAKmvB,QAAQ4nE,WACb/2F,KAAKmvB,QAAQk5B,GAAa5qD,SAGR,IAAhBuC,KAAKu3F,QAAuC93F,MAAnBO,KAAKw3F,iBAC3B77C,MAAMg9C,YAAY34F,KAAKw3F,WAAW77C,YAClCyC,IAAI05C,cAActjE,MAAMC,QAAU,aAElC2pB,IAAI05C,cAActjE,MAAMC,QAAU,aAIpCmkE,aAAe,MAEd3wC,EAAQjoD,KAAK27C,MAAMk9C,cACzB5wC,QAAAA,GACE,SAAAuC,OACQjhD,EAAIihD,EAAKjhD,EACT8gD,EAAUG,EAAK8qC,MACjB3/B,EAAKxmC,QAAL,kBAA+C,IAAZk7B,GACrCsL,EAAKmjC,aAAavvF,EAAI,EAAGihD,EAAKjqD,IAAK8nD,EAAa,uBAAwBsN,EAAKtgB,MAAM4T,iBAEjFoB,GACE9gD,GAAK,GACPosD,EAAKmjC,aAAavvF,EAAI,EAAGihD,EAAKjqD,IAAK8nD,EAAa,uBAAwBsN,EAAKtgB,MAAM8T,kBAGnE,IAAhBwM,EAAK4hC,SACHltC,EACFsL,EAAKojC,YAAYxvF,EAAG8+C,EAAa,oCAAqCsN,EAAKxmC,QAAQunE,iBAAkB/gC,EAAKtgB,MAAMmU,gBAGhHmM,EAAKojC,YAAYxvF,EAAG8+C,EAAa,oCAAqCsN,EAAKxmC,QAAQwnE,iBAAkBhhC,EAAKtgB,MAAMiU,wBAMpH0vC,EAAa,OACuBv5F,IAApCO,KAAKmvB,QAAQk5B,GAAaoH,YAAgEhwD,IAAzCO,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,OACnFutC,EAAah5F,KAAKq1C,MAAM4jD,qBAEpBz8E,GAAgC,IAAvBxc,KAAKmvB,QAAQsnE,MAAiBj1F,KAAK8D,IAAItF,KAAKmvB,QAAQ2nE,UAAWkC,GAAch5F,KAAKmvB,QAAQynE,aAAe,GAAKoC,EAAah5F,KAAKmvB,QAAQynE,aAAe,UAGlK52F,KAAK44F,aAAgB54F,KAAK4oC,MAAQpsB,IAAoC,IAAzBxc,KAAKmvB,QAAQmsC,cACvD1yB,MAAQ5oC,KAAK44F,aAAep8E,OAC5B2S,QAAQyZ,gBAAW5oC,KAAK4oC,YAC7BovD,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,aACpC7mC,SACLxX,GAAU,GAGH74C,KAAK44F,aAAgB54F,KAAK4oC,MAAQpsB,IAAoC,IAAzBxc,KAAKmvB,QAAQmsC,SAAoBt7D,KAAK4oC,MAAQ5oC,KAAK4rE,eAClGhjC,MAAQpnC,KAAK8D,IAAItF,KAAK4rE,SAAU5rE,KAAK44F,aAAep8E,QACpD2S,QAAQyZ,gBAAW5oC,KAAK4oC,YAC7BovD,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,aACpC7mC,SACLxX,GAAU,IAGVm/C,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,QACzCr+C,GAAU,GAGLA,uCAQI/2C,UACJ9B,KAAK27C,MAAMu5C,aAAapzF,yCAQnBqI,UACLnK,KAAK27C,MAAMu9C,cAAc/uF,wCAarBZ,EAAGkiD,EAAMpD,EAAa/f,EAAW6wD,OAEtC5wD,EAAQyvD,GAAsB,MAAOh4F,KAAKi3F,YAAYC,OAAQl3F,KAAKo+C,IAAIuqB,OAC7EpgC,EAAMD,UAAYA,EAClBC,EAAMmjB,UAAYD,EACE,SAAhBpD,GACF9f,EAAM/T,MAAMkgB,gBAAW10C,KAAKmvB,QAAQynE,mBACpCruD,EAAM/T,MAAM4kE,UAAY,UAGxB7wD,EAAM/T,MAAMmgB,iBAAY30C,KAAKmvB,QAAQynE,mBACrCruD,EAAM/T,MAAM4kE,UAAY,QAG1B7wD,EAAM/T,MAAMsuB,cAASv5C,EAAI,GAAM4vF,EAAkBn5F,KAAKmvB,QAAQ0nE,mBAE9DprC,GAAQ,OAEF4tC,EAAe73F,KAAK8D,IAAItF,KAAKq1C,MAAMmW,eAAgBxrD,KAAKq1C,MAAM2U,gBAChEhqD,KAAK44F,aAAentC,EAAKzuD,OAASq8F,SAC/BT,aAAentC,EAAKzuD,OAASq8F,uCAY1B9vF,EAAG8+C,EAAa/f,EAAW9rB,EAAQosB,OACzB,IAAhB5oC,KAAKu3F,OAAiB,KAClB/sC,EAAOwtC,GAAsB,MAAOh4F,KAAKi3F,YAAYhvC,MAAOjoD,KAAKo+C,IAAI05C,eAC3EttC,EAAKliB,UAAYA,EACjBkiB,EAAKkB,UAAY,GAEG,SAAhBrD,EACFmC,EAAKh2B,MAAMkgB,eAAU10C,KAAK4oC,MAAQpsB,QAGlCguC,EAAKh2B,MAAMmgB,gBAAW30C,KAAK4oC,MAAQpsB,QAGrCguC,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMsuB,cAASv5C,8CASX8+C,MACX2vC,GAAwBh4F,KAAKi3F,YAAYxnC,YAGDhwD,IAApCO,KAAKmvB,QAAQk5B,GAAaoH,YAAgEhwD,IAAzCO,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,KAAoB,KACjGgE,EAAQuoC,GAAsB,MAAOh4F,KAAKi3F,YAAYxnC,MAAOzvD,KAAKo+C,IAAIuqB,OAC5ElZ,EAAMnnB,6CAAwC+f,GAC9CoH,EAAM/D,UAAY1rD,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,UAGJhsD,IAA1CO,KAAKmvB,QAAQk5B,GAAaoH,MAAMj7B,OAClC8jB,GAAK+lB,WAAW5O,EAAOzvD,KAAKmvB,QAAQk5B,GAAaoH,MAAMj7B,OAGrC,SAAhB6zB,EACFoH,EAAMj7B,MAAMkgB,eAAU10C,KAAKq1C,MAAM4jD,sBAGjCxpC,EAAMj7B,MAAMmgB,gBAAW30C,KAAKq1C,MAAM4jD,sBAGpCxpC,EAAMj7B,MAAMoU,gBAAW5oC,KAAK6oC,aAI9BmvD,GAAwBh4F,KAAKi3F,YAAYxnC,yDAUnC,oBAAqBzvD,KAAKq1C,OAAQ,KAChCikD,EAAYlsE,SAAS06B,eAAe,KACpCgE,EAAmB1+B,SAASE,cAAc,OAChDw+B,EAAiBxjB,UAAY,mCAC7BwjB,EAAiBn3B,YAAY2kE,QACxBl7C,IAAIuqB,MAAMh0C,YAAYm3B,QAEtBzW,MAAM4T,gBAAkB6C,EAAiBC,kBACzC1W,MAAM2U,eAAiB8B,EAAiBnL,iBAExCvC,IAAIuqB,MAAMnhC,YAAYskB,QAGvB,oBAAqB9rD,KAAKq1C,OAAQ,KAChCkkD,EAAYnsE,SAAS06B,eAAe,KACpCkE,EAAmB5+B,SAASE,cAAc,OAChD0+B,EAAiB1jB,UAAY,mCAC7B0jB,EAAiBr3B,YAAY4kE,QACxBn7C,IAAIuqB,MAAMh0C,YAAYq3B,QAEtB3W,MAAM8T,gBAAkB6C,EAAiBD,kBACzC1W,MAAMmW,eAAiBQ,EAAiBrL,iBAExCvC,IAAIuqB,MAAMnhC,YAAYwkB,QAGvB,oBAAqBhsD,KAAKq1C,OAAQ,KAChCmkD,EAAYpsE,SAAS06B,eAAe,KACpC2xC,EAAmBrsE,SAASE,cAAc,OAChDmsE,EAAiBnxD,UAAY,mCAC7BmxD,EAAiB9kE,YAAY6kE,QACxBp7C,IAAIuqB,MAAMh0C,YAAY8kE,QAEtBpkD,MAAM4jD,gBAAkBQ,EAAiB1tC,kBACzC1W,MAAMqkD,eAAiBD,EAAiB94C,iBAExCvC,IAAIuqB,MAAMnhC,YAAYiyD,WA5mBV9gD,ICCvB,SAASghD,GAAOx9B,EAAShtC,IAiDzB,SAASyqE,GAAiBv1B,EAAOw1B,SAExB,CACLrlE,OAFFqlE,OAA4C,IAAnBA,EAAkC,GAAKA,GAExCrlE,OAAS6vC,EAAMl1C,QAAQqlC,WAAWhgC,MACxD6T,OAAQwxD,EAAexxD,QAAUg8B,EAAMl1C,QAAQqlC,WAAWnsB,OAC1D1I,KAAMk6D,EAAel6D,MAAQ0kC,EAAMl1C,QAAQqlC,WAAW70B,KACtD2I,UAAWuxD,EAAevxD,WAAa+7B,EAAM/7B,WCvDjD,SAASwxD,GAAS39B,EAAShtC,ICD3B,SAAS4qE,GAAK59B,EAAShtC,ICOvB,SAAS6qE,GAAW31B,EAAOlI,EAAShtC,EAAS8qE,QACtC7mE,GAAK+oC,OAELhtC,QAAUmpB,GAAK4hD,sBADP,CAAC,WAAY,QAAS,OAAQ,mBAAoB,WAAY,aAAc,SAAU,gBAAiB,SAAS,sBAAuB,qBAClG/qE,QAC7CgrE,uBAAwC16F,IAApB4kE,EAAM/7B,eAC1B2xD,yBAA2BA,OAC3BG,aAAe,OACfjkD,OAAOkuB,GACkB,GAA1BrkE,KAAKm6F,yBACFF,yBAAyB,IAAM,QAEjCx0B,UAAY,QACZnK,aAA4B77D,IAAlB4kE,EAAM/I,SAA+B+I,EAAM/I,QCb5D,SAAS++B,GAAOzhD,EAAMzpB,EAASmrE,EAAM9D,QAC9B59C,KAAOA,OACPsE,eAAiB,CACpBw/B,SAAS,EACT+Z,OAAO,EACP8D,SAAU,GACVC,YAAa,EACb9lD,KAAM,CACJ4mB,SAAS,EACT57B,SAAU,YAEZiV,MAAO,CACL2mB,SAAS,EACT57B,SAAU,mBAIT46D,KAAOA,OACPnrE,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,qBAC/Bs5C,iBAAmBA,OAEnBiB,YAAc,QACdr5C,IAAM,QACNuuB,OAAS,QACTgrB,eAAiB,OACjBjvC,eACAmvC,UAAY,CAACtB,IAAKv2F,KAAKu2F,IAAKkB,YAAaz3F,KAAKy3F,YAAatoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ3sE,KAAK2sE,aAE/FnuB,WAAWrvB,GJtBlBwqE,GAAOjK,KAAO,SAAU5W,EAASzU,EAAOwzB,EAAWr7E,GACjDA,EAASA,GAAU,UACfxW,EAoDN,SAAqB6xF,EAAWxzB,OAC1Br+D,OAAWvG,EAEXo4F,EAAU1oE,SAAW0oE,EAAU1oE,QAAQqlC,YAAcqjC,EAAU1oE,QAAQqlC,WAAWC,UAA4D,mBAAzCojC,EAAU1oE,QAAQqlC,WAAWC,WACpIzuD,EAAW6xF,EAAU1oE,QAAQqlC,WAAWC,UAItC4P,EAAMA,MAAMl1C,SAAWk1C,EAAMA,MAAMl1C,QAAQqlC,YAAc6P,EAAMA,MAAMl1C,QAAQqlC,WAAWC,UAA8D,mBAA3C4P,EAAMA,MAAMl1C,QAAQqlC,WAAWC,WAC5IzuD,EAAWq+D,EAAMA,MAAMl1C,QAAQqlC,WAAWC,iBAErCzuD,EA/DQy0F,CAAY5C,EAAWxzB,GAE7BvnE,EAAI,EAAGA,EAAIg8E,EAAQ97E,OAAQF,OAC7BkJ,EAIA,KACC6zF,EAAiB7zF,EAAS8yE,EAAQh8E,GAAIunE,IACnB,IAAnBw1B,GAAqD,WAA1Bp0D,GAAOo0D,IACpC7B,GAAkBlf,EAAQh8E,GAAG49F,SAAWl+E,EAAQs8D,EAAQh8E,GAAG69F,SAAUf,GAAiBv1B,EAAOw1B,GAAiBhC,EAAUJ,YAAaI,EAAUtB,IAAKzd,EAAQh8E,GAAGyrC,YALjKyvD,GAAkBlf,EAAQh8E,GAAG49F,SAAWl+E,EAAQs8D,EAAQh8E,GAAG69F,SAAUf,GAAiBv1B,GAAQwzB,EAAUJ,YAAaI,EAAUtB,IAAKzd,EAAQh8E,GAAGyrC,QAWrJoxD,GAAOiB,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAC1DgD,EAA0B,GAAbzC,EAEb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,eAGtC4vD,GAAkB7tF,EAAI,GAAM2sF,EAAWvtF,EAAGqwF,GAAiBv1B,GAAQwzB,EAAUJ,YAAaI,EAAUtB,MCrCtGuD,GAASc,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAC5DgD,EAA0B,GAAbzC,EACb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,mBAElC2yD,EAAWv5F,KAAK+a,MAAM,GAAMu6E,GAE5Bn7C,EADgB0oB,EAAMl1C,QAAQ6rE,SAASpyD,MACfmyD,EACxBE,EAAaz5F,KAAK+a,MAAM,GAAM67E,GAC9B8C,EAAa15F,KAAK+a,MAAM,IAAO67E,GAE/B57E,EAAShb,KAAK+a,OAAOu6E,EAAa,EAAIiE,GAAa,MAEvD/C,GAAgB7tF,EAAI,GAAM4wF,EAAWv+E,EAAQjT,EAAIsxF,EAAaI,EAAa,EAAGF,EAAUE,EAAY52B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,OAC9KwjE,GAAgB7tF,EAAI,IAAM4wF,EAAWv+E,EAAS,EAAGjT,EAAIsxF,EAAaK,EAAa,EAAGH,EAAUG,EAAY72B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,OAE1I,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,QAAiB,KACxCx0C,EAAgB,CAClB1T,MAAO6vC,EAAMl1C,QAAQqlC,WAAWhgC,MAChC6T,OAAQg8B,EAAMl1C,QAAQqlC,WAAWnsB,OACjC1I,KAAO0kC,EAAMl1C,QAAQqlC,WAAW70B,KAAOgc,EACvCrT,UAAW+7B,EAAM/7B,WAEnB0vD,GAAkB7tF,EAAI,GAAM4wF,EAAWv+E,EAAQjT,EAAIsxF,EAAaI,EAAa,EAAG/yD,EAAe2vD,EAAUJ,YAAaI,EAAUtB,KAChIyB,GAAkB7tF,EAAI,IAAM4wF,EAAWv+E,EAAS,EAAGjT,EAAIsxF,EAAaK,EAAa,EAAGhzD,EAAe2vD,EAAUJ,YAAaI,EAAUtB,OAWxIuD,GAASpK,KAAO,SAAU1e,EAAUmqB,EAAoBtD,OAGlDuD,EACAn4F,EAAKo4F,EACLh3B,EACAvnE,EAAGiX,EALHunF,EAAe,GACfC,EAAgB,GAKhBC,EAAY,MAGX1+F,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEH,SAD5BunE,EAAQwzB,EAAUlrB,OAAOqE,EAASl0E,KACxBqyB,QAAQqF,QACM,IAAlB6vC,EAAM/I,eAA0E77D,IAArDo4F,EAAU1oE,QAAQw9C,OAAO1tB,WAAW+xB,EAASl0E,MAA0E,IAArD+6F,EAAU1oE,QAAQw9C,OAAO1tB,WAAW+xB,EAASl0E,SACvIiX,EAAI,EAAGA,EAAIonF,EAAmBnqB,EAASl0E,IAAIE,OAAQ+W,IACtDunF,EAAar+F,KAAK,CAChBy9F,SAAUS,EAAmBnqB,EAASl0E,IAAIiX,GAAG2mF,SAC7Ce,WAAYN,EAAmBnqB,EAASl0E,IAAIiX,GAAG0nF,WAC/Cd,SAAUQ,EAAmBnqB,EAASl0E,IAAIiX,GAAG4mF,SAC7CxwF,EAAGgxF,EAAmBnqB,EAASl0E,IAAIiX,GAAG5J,EACtCs2B,IAAK06D,EAAmBnqB,EAASl0E,IAAIiX,GAAG0sB,IACxCl3B,EAAG4xF,EAAmBnqB,EAASl0E,IAAIiX,GAAGxK,EACtC4yD,QAAS6U,EAASl0E,GAClByrC,MAAO4yD,EAAmBnqB,EAASl0E,IAAIiX,GAAGw0B,QAE5CizD,GAAa,KAMH,IAAdA,SAKJF,QAAAA,GAAkB,SAAUn+F,EAAGC,UACzBD,EAAEu9F,WAAat9F,EAAEs9F,SACZv9F,EAAEg/D,QAAU/+D,EAAE++D,SAAW,EAAI,EAG7Bh/D,EAAEu9F,SAAWt9F,EAAEs9F,YAK1BZ,GAAS4B,sBAAsBH,EAAeD,GAGzCx+F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,IAAK,KAEpC8uE,EAA8CnsE,OADlD4kE,EAAQwzB,EAAUlrB,OAAO2uB,EAAax+F,GAAGq/D,UACpBhtC,QAAQ6rE,SAASpvB,SAAwBvH,EAAMl1C,QAAQ6rE,SAASpvB,SAAW,GAAMvH,EAAMl1C,QAAQ6rE,SAASpyD,MAGzH+yD,EAAe,UACQl8F,IAAvB87F,EAFJt4F,EAAMq4F,EAAax+F,GAAG49F,UAGhB59F,EAAI,EAAIw+F,EAAat+F,SACvBo+F,EAAe55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWz3F,IAEzDo4F,EAAWvB,GAAS8B,iBAAiBR,EAAc/2B,EAAOuH,OAEvD,KACCzmB,EAAUroD,GAAKy+F,EAAct4F,GAAK44F,OAASN,EAAct4F,GAAK64F,UAC9D32C,EAAUm2C,EAAat+F,SACzBo+F,EAAe55F,KAAKc,IAAIg5F,EAAan2C,GAASu1C,SAAWz3F,IAE3Do4F,EAAWvB,GAAS8B,iBAAiBR,EAAc/2B,EAAOuH,GAC1D2vB,EAAct4F,GAAK64F,UAAY,GAEH,IAAxBz3B,EAAMl1C,QAAQ9rB,QAAwD,IAAtCghE,EAAMl1C,QAAQ4sE,oBAC5CT,EAAax+F,GAAG69F,SAAWt2B,EAAM+1B,cACnCuB,EAAeJ,EAAct4F,GAAK+4F,oBAClCT,EAAct4F,GAAK+4F,qBAAuB33B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,WAG/EgB,EAAeJ,EAAct4F,GAAKg5F,oBAClCV,EAAct4F,GAAKg5F,qBAAuB53B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,WAGpC,IAAtCt2B,EAAMl1C,QAAQ6rE,SAASkB,aAC9Bb,EAASzyD,MAAQyyD,EAASzyD,MAAQ2yD,EAAct4F,GAAK44F,OACrDR,EAAS7+E,QAAW++E,EAAct4F,GAAK64F,SAAYT,EAASzyD,MAAS,GAAMyyD,EAASzyD,OAAS2yD,EAAct4F,GAAK44F,OAAS,QAIzHM,EAAYd,EAASzyD,MACrBpI,EAAQ86D,EAAax+F,GAAG49F,YAGMj7F,MAA9B67F,EAAax+F,GAAG2+F,WAElBj7D,GAAsB,IADtB27D,EAAYb,EAAax+F,GAAG2+F,WAAaH,EAAax+F,GAAG49F,UAIzDl6D,GAAS66D,EAAS7+E,OAGpBw7E,GAAgBx3D,EAAO86D,EAAax+F,GAAG69F,SAAWgB,EAAcQ,EAAW93B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,SAAUt2B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,QAG3J,IAArC6vC,EAAMl1C,QAAQqlC,WAAWkoB,QAAkB,KACzC0f,EAAY,CACd1B,SAAUY,EAAax+F,GAAG49F,SAC1BC,SAAUW,EAAax+F,GAAG69F,SAAWgB,EACrCxxF,EAAGmxF,EAAax+F,GAAGqN,EACnBZ,EAAG+xF,EAAax+F,GAAGyM,EACnB4yD,QAASm/B,EAAax+F,GAAGq/D,QACzB5zB,MAAO+yD,EAAax+F,GAAGyrC,OAEzBoxD,GAAOjK,KAAK,CAAC0M,GAAY/3B,EAAOwzB,EAAWwD,EAAS7+E,WAa1Ds9E,GAAS4B,sBAAwB,SAAUH,EAAeD,WAEpDF,EACKt+F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,IACnCA,EAAI,EAAIw+F,EAAat+F,SACvBo+F,EAAe55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWY,EAAax+F,GAAG49F,WAErE59F,EAAI,IACNs+F,EAAe55F,KAAKY,IAAIg5F,EAAc55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWY,EAAax+F,GAAG49F,YAE3E,IAAjBU,SAC8C37F,IAA5C87F,EAAcD,EAAax+F,GAAG49F,YAChCa,EAAcD,EAAax+F,GAAG49F,UAAY,CACxCmB,OAAQ,EACRC,SAAU,EACVG,oBAAqB,EACrBD,oBAAqB,IAGzBT,EAAcD,EAAax+F,GAAG49F,UAAUmB,QAAU,IAexD/B,GAAS8B,iBAAmB,SAAUR,EAAc/2B,EAAOuH,OACrDhjC,EAAOpsB,SACP4+E,EAAe/2B,EAAMl1C,QAAQ6rE,SAASpyD,OAASwyD,EAAe,GAChExyD,EAAQwyD,EAAexvB,EAAWA,EAAWwvB,EAE7C5+E,EAAS,EAC4B,SAAjC6nD,EAAMl1C,QAAQ6rE,SAASv0B,MACzBjqD,GAAU,GAAM4+E,EAEwB,UAAjC/2B,EAAMl1C,QAAQ6rE,SAASv0B,QAC9BjqD,GAAU,GAAM4+E,KAKlBxyD,EAAQy7B,EAAMl1C,QAAQ6rE,SAASpyD,MAC/BpsB,EAAS,EAC4B,SAAjC6nD,EAAMl1C,QAAQ6rE,SAASv0B,MACzBjqD,GAAU,GAAM6nD,EAAMl1C,QAAQ6rE,SAASpyD,MAEC,UAAjCy7B,EAAMl1C,QAAQ6rE,SAASv0B,QAC9BjqD,GAAU,GAAM6nD,EAAMl1C,QAAQ6rE,SAASpyD,QAIpC,CAACA,MAAOA,EAAOpsB,OAAQA,IAGhCs9E,GAASuC,iBAAmB,SAAUf,EAAcgB,EAAatrB,EAAUurB,EAAYl0C,MACjFizC,EAAat+F,OAAS,EAAG,IAE3Bs+F,QAAAA,GAAkB,SAAUn+F,EAAGC,UACzBD,EAAEu9F,WAAat9F,EAAEs9F,SACZv9F,EAAEg/D,QAAU/+D,EAAE++D,SAAW,EAAI,EAG7Bh/D,EAAEu9F,SAAWt9F,EAAEs9F,gBAGtBa,EAAgB,GAEpBzB,GAAS4B,sBAAsBH,EAAeD,GAC9CgB,EAAYC,GAAczC,GAAS0C,kBAAkBjB,EAAeD,GACpEgB,EAAYC,GAAYE,iBAAmBp0C,EAC3C2oB,EAAS/zE,KAAKs/F,KAIlBzC,GAAS0C,kBAAoB,SAAUjB,EAAeD,WAChDr4F,EACAy5F,EAAOpB,EAAa,GAAGX,SACvBgC,EAAOrB,EAAa,GAAGX,SAClB79F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,SAEZ2C,IAAvB87F,EADJt4F,EAAMq4F,EAAax+F,GAAG49F,WAEpBgC,EAAOA,EAAOpB,EAAax+F,GAAG69F,SAAWW,EAAax+F,GAAG69F,SAAW+B,EACpEC,EAAOA,EAAOrB,EAAax+F,GAAG69F,SAAWW,EAAax+F,GAAG69F,SAAWgC,GAGhErB,EAAax+F,GAAG69F,SAAW,EAC7BY,EAAct4F,GAAK+4F,qBAAuBV,EAAax+F,GAAG69F,SAG1DY,EAAct4F,GAAKg5F,qBAAuBX,EAAax+F,GAAG69F,aAI3D,IAAIiC,KAAQrB,EACXA,EAAcl+F,eAAeu/F,KAE/BF,GADAA,EAAOA,EAAOnB,EAAcqB,GAAMZ,oBAAsBT,EAAcqB,GAAMZ,oBAAsBU,GACpFnB,EAAcqB,GAAMX,oBAAsBV,EAAcqB,GAAMX,oBAAsBS,EAElGC,GADAA,EAAOA,EAAOpB,EAAcqB,GAAMZ,oBAAsBT,EAAcqB,GAAMZ,oBAAsBW,GACpFpB,EAAcqB,GAAMX,oBAAsBV,EAAcqB,GAAMX,oBAAsBU,SAI/F,CAACv6F,IAAKs6F,EAAMp3F,IAAKq3F,IChR1B5C,GAAK8C,SAAW,SAAU/jB,EAASzU,MAChB,MAAXyU,GACIA,EAAQ97E,OAAS,EAAG,QAIuB,GAAvCqnE,EAAMl1C,QAAQ2tE,cAAcpgB,QACxBqd,GAAKgD,YAAYjkB,EAASzU,GAG1B01B,GAAKiD,QAAQlkB,KAOjCihB,GAAKa,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAEtD5nE,EAAMgtE,EADNpC,EAA0B,GAAbzC,EAGb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,MAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,gBAEtCnY,EAAO+nE,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,MACjEnuD,eAAe,KAAM,QAASi8B,EAAM/7B,gBACrB7oC,IAAhB4kE,EAAM7vC,OACNvE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM7vC,OAG7CvE,EAAKmY,eAAe,KAAM,IAAK,IAAMj+B,EAAI,IAAMZ,EAAI,MAAQY,EAAI2sF,GAAa,IAAMvtF,GAC9C,GAAhC86D,EAAMl1C,QAAQ+tE,OAAOxgB,UACrBugB,EAAWjF,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAClC,OAApClyB,EAAMl1C,QAAQ+tE,OAAO70C,YACrB40C,EAAS70D,eAAe,KAAM,IAAK,IAAMj+B,EAAI,MAAQZ,EAAIsxF,GACvD,IAAM1wF,EAAI,IAAMZ,EAAI,MAAQY,EAAI2sF,GAAa,IAAMvtF,EAAI,MAAQY,EAAI2sF,GAAa,KAAOvtF,EAAIsxF,IAG7FoC,EAAS70D,eAAe,KAAM,IAAK,IAAMj+B,EAAI,IAAMZ,EAAI,KAC/CY,EAAI,KAAOZ,EAAIsxF,GAAc,MAC5B1wF,EAAI2sF,GAAa,KAAOvtF,EAAIsxF,GACnC,KAAO1wF,EAAI2sF,GAAa,IAAMvtF,GAEpC0zF,EAAS70D,eAAe,KAAM,QAASi8B,EAAM/7B,UAAY,uBACtB7oC,IAA/B4kE,EAAMl1C,QAAQ+tE,OAAO1oE,OAAsD,KAA/B6vC,EAAMl1C,QAAQ+tE,OAAO1oE,OACjEyoE,EAAS70D,eAAe,KAAM,QAASi8B,EAAMl1C,QAAQ+tE,OAAO1oE,QAI5B,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,UAOzBsb,GAAkB7tF,EAAI,GAAM2sF,EAAWvtF,EANnB,CAChBirB,MAAO6vC,EAAMl1C,QAAQqlC,WAAWhgC,MAChC6T,OAAQg8B,EAAMl1C,QAAQqlC,WAAWnsB,OACjC1I,KAAM0kC,EAAMl1C,QAAQqlC,WAAW70B,KAC/B2I,UAAW+7B,EAAM/7B,WAEoCuvD,EAAUJ,YAAaI,EAAUtB,MAIlGwD,GAAKoD,YAAc,SAAUC,EAAW/4B,EAAOg5B,EAAcxF,MAErB,GAAhCxzB,EAAMl1C,QAAQ+tE,OAAOxgB,QAAiB,KAOlC4gB,EANAC,EAAYlmD,OAAOwgD,EAAUtB,IAAI/hE,MAAMqU,OAAOtiC,QAAQ,KAAK,KAC3D02F,EAAWjF,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC1Er/D,EAAO,IACgC,GAAvCmtC,EAAMl1C,QAAQ2tE,cAAcpgB,UAC5BxlD,EAAO,SAGPsmE,EAAO,EAEPA,EADoC,OAApCn5B,EAAMl1C,QAAQ+tE,OAAO70C,YACd,EAEkC,UAApCgc,EAAMl1C,QAAQ+tE,OAAO70C,YACnBk1C,EAGA/7F,KAAKY,IAAIZ,KAAK8D,IAAI,EAAG++D,EAAM+1B,cAAemD,GAGjDD,EADoC,SAApCj5B,EAAMl1C,QAAQ+tE,OAAO70C,aAA2C,MAAhBg1C,GAAwC59F,MAAhB49F,EAChE,IAAMD,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAC7Cp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,GAClC,KAAMmmE,EAAaA,EAAargG,OAAO,GAAG,GAAI,IAAMqgG,EAAaA,EAAargG,OAAO,GAAG,GAAK,IAC7FgD,KAAKy9F,cAAcJ,EAAanmE,GAAK,GACrCmmE,EAAa,GAAG,GAAI,IAAIA,EAAa,GAAG,GAAK,KAG7C,IAAMD,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAC7Cp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,GAClC,KAAOsmE,EAAO,KAAMJ,EAAU,GAAG,GAAK,KAGlDH,EAAS70D,eAAe,KAAM,QAASi8B,EAAM/7B,UAAY,kBACtB7oC,IAA/B4kE,EAAMl1C,QAAQ+tE,OAAO1oE,OACrByoE,EAAS70D,eAAe,KAAM,QAASi8B,EAAMl1C,QAAQ+tE,OAAO1oE,OAEhEyoE,EAAS70D,eAAe,KAAM,IAAKk1D,KAW3CvD,GAAKrK,KAAO,SAAU0N,EAAW/4B,EAAOwzB,MACnB,MAAbuF,GAAkC39F,MAAb29F,EAAwB,KACzCntE,EAAO+nE,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC1EtmE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM/7B,gBACrB7oC,IAAhB4kE,EAAM7vC,OACNvE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM7vC,WAGzC0C,EAAO,IACgC,GAAvCmtC,EAAMl1C,QAAQ2tE,cAAcpgB,UAC5BxlD,EAAO,KAGXjH,EAAKmY,eAAe,KAAM,IAAK,IAAMg1D,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAAMp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,MAI5H6iE,GAAK0D,cAAgB,SAASL,EAAUlmE,EAAKwmE,MACrCN,EAAUpgG,OAAS,QAEZ,OAGPF,EADAyP,EAAI2qB,KAEJwmE,MACK5gG,EAAIsgG,EAAUpgG,OAAO,EAAGF,EAAI,EAAGA,IAChCyP,GAAK6wF,EAAUtgG,GAAG,GAAK,IAAMsgG,EAAUtgG,GAAG,GAAK,aAI9CA,EAAI,EAAGA,EAAIsgG,EAAUpgG,OAAQF,IAC9ByP,GAAK6wF,EAAUtgG,GAAG,GAAK,IAAMsgG,EAAUtgG,GAAG,GAAK,WAGhDyP,GAUXwtF,GAAK4D,mBAAqB,SAAUpqF,OAE5BqqF,EAAIn1F,EAAIC,EAAIC,EAAIk1F,EAAKC,EACrBvxF,EAAI,GACRA,EAAEtP,KAAM,CAAEuE,KAAK+a,MAAMhJ,EAAK,GAAGmnF,UAAYl5F,KAAK+a,MAAMhJ,EAAK,GAAGonF,oBAExD39F,EAASuW,EAAKvW,OACTF,EAAI,EAAGA,EAAIE,EAAS,EAAGF,IAE5B8gG,EAAW,GAAL9gG,EAAUyW,EAAK,GAAKA,EAAKzW,EAAI,GACnC2L,EAAK8K,EAAKzW,GACV4L,EAAK6K,EAAKzW,EAAI,GACd6L,EAAM7L,EAAI,EAAIE,EAAUuW,EAAKzW,EAAI,GAAK4L,EAUtCm1F,EAAM,CACFnD,WAAakD,EAAGlD,SAAW,EAAIjyF,EAAGiyF,SAAWhyF,EAAGgyF,WAlBpC,EAAI,GAmBhBC,WAAaiD,EAAGjD,SAAW,EAAIlyF,EAAGkyF,SAAWjyF,EAAGiyF,WAnBpC,EAAI,IAqBpBmD,EAAM,CACFpD,UAAajyF,EAAGiyF,SAAW,EAAIhyF,EAAGgyF,SAAW/xF,EAAG+xF,WAtBpC,EAAI,GAuBhBC,UAAalyF,EAAGkyF,SAAW,EAAIjyF,EAAGiyF,SAAWhyF,EAAGgyF,WAvBpC,EAAI,IA2BpBpuF,EAAEtP,KAAM,CAAE4gG,EAAInD,SAAWmD,EAAIlD,WAC7BpuF,EAAEtP,KAAM,CAAE6gG,EAAIpD,SAAWoD,EAAInD,WAC7BpuF,EAAEtP,KAAM,CAAEyL,EAAGgyF,SAAYhyF,EAAGiyF,kBAGzBpuF,GAcXwtF,GAAKgD,YAAc,SAAUxpF,EAAM8wD,OAC3BojB,EAAQpjB,EAAMl1C,QAAQ2tE,cAAcrV,SAC3B,GAATA,QAAwBhoF,IAAVgoF,SACPznF,KAAK29F,mBAAmBpqF,OAG3BqqF,EAAIn1F,EAAIC,EAAIC,EAAIk1F,EAAKC,EAAKC,EAAIC,EAAIC,EAAIh7D,EAAGoC,EAAG64D,EAAGzxF,EAC/C0xF,EAAQC,EAAQC,EAASC,EAASC,EAASC,EAC3CjyF,EAAI,GACRA,EAAEtP,KAAM,CAAEuE,KAAK+a,MAAMhJ,EAAK,GAAGmnF,UAAYl5F,KAAK+a,MAAMhJ,EAAK,GAAGonF,oBACxD39F,EAASuW,EAAKvW,OACTF,EAAI,EAAGA,EAAIE,EAAS,EAAGF,IAE5B8gG,EAAW,GAAL9gG,EAAUyW,EAAK,GAAKA,EAAKzW,EAAI,GACnC2L,EAAK8K,EAAKzW,GACV4L,EAAK6K,EAAKzW,EAAI,GACd6L,EAAM7L,EAAI,EAAIE,EAAUuW,EAAKzW,EAAI,GAAK4L,EAEtCq1F,EAAKv8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIu4F,EAAGlD,SAAWjyF,EAAGiyF,SAAU,GAAKl5F,KAAK6D,IAAIu4F,EAAGjD,SAAWlyF,EAAGkyF,SAAU,IAC5FqD,EAAKx8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIoD,EAAGiyF,SAAWhyF,EAAGgyF,SAAU,GAAKl5F,KAAK6D,IAAIoD,EAAGkyF,SAAWjyF,EAAGiyF,SAAU,IAC5FsD,EAAKz8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIqD,EAAGgyF,SAAW/xF,EAAG+xF,SAAU,GAAKl5F,KAAK6D,IAAIqD,EAAGiyF,SAAWhyF,EAAGgyF,SAAU,IAY5FwD,EAAS38F,KAAK6D,IAAI44F,EAAIxW,GACtB4W,EAAU78F,KAAK6D,IAAI44F,EAAI,EAAIxW,GAC3B2W,EAAS58F,KAAK6D,IAAI24F,EAAIvW,GACtB6W,EAAU98F,KAAK6D,IAAI24F,EAAI,EAAIvW,GAC3B+W,EAASh9F,KAAK6D,IAAI04F,EAAItW,GAGtBxkD,EAAI,GAFJs7D,EAAU/8F,KAAK6D,IAAI04F,EAAI,EAAItW,IAET,EAAI+W,EAASJ,EAASE,EACxCj5D,EAAI,EAAIg5D,EAAU,EAAIF,EAASC,EAASE,GACxCJ,EAAI,EAAIM,GAAUA,EAASJ,IACnB,IACJF,EAAI,EAAIA,IAEZzxF,EAAI,EAAI0xF,GAAUA,EAASC,IACnB,IACJ3xF,EAAI,EAAIA,GAGZoxF,EAAM,CACFnD,WAAa4D,EAAUV,EAAGlD,SAAWz3D,EAAIx6B,EAAGiyF,SAAW6D,EAAU71F,EAAGgyF,UAAYwD,EAChFvD,WAAa2D,EAAUV,EAAGjD,SAAW13D,EAAIx6B,EAAGkyF,SAAW4D,EAAU71F,EAAGiyF,UAAYuD,GAGpFJ,EAAM,CACFpD,UAAa2D,EAAU51F,EAAGiyF,SAAWr1D,EAAI38B,EAAGgyF,SAAW4D,EAAU31F,EAAG+xF,UAAYjuF,EAChFkuF,UAAa0D,EAAU51F,EAAGkyF,SAAWt1D,EAAI38B,EAAGiyF,SAAW2D,EAAU31F,EAAGgyF,UAAYluF,GAGhE,GAAhBoxF,EAAInD,UAAiC,GAAhBmD,EAAIlD,WACzBkD,EAAMp1F,GAEU,GAAhBq1F,EAAIpD,UAAiC,GAAhBoD,EAAInD,WACzBmD,EAAMp1F,GAEV6D,EAAEtP,KAAM,CAAE4gG,EAAInD,SAAWmD,EAAIlD,WAC7BpuF,EAAEtP,KAAM,CAAE6gG,EAAIpD,SAAWoD,EAAInD,WAC7BpuF,EAAEtP,KAAM,CAAEyL,EAAGgyF,SAAYhyF,EAAGiyF,kBAGzBpuF,GAUfwtF,GAAKiD,QAAU,SAAUzpF,WAEjBhH,EAAI,GACCzP,EAAI,EAAGA,EAAIyW,EAAKvW,OAAQF,IAC7ByP,EAAEtP,KAAK,CAAEsW,EAAKzW,GAAG49F,SAAWnnF,EAAKzW,GAAG69F,kBAEjCpuF,GC/QXytF,GAAW79F,UAAUg5D,SAAW,SAAU3e,GAC3B,MAATA,QACGivB,UAAYjvB,EACQ,GAArB6pB,QAAKlxC,UACPmpB,GAAKmmD,WAAWz+F,KAAKylE,WAAU,SAAUtoE,EAAGC,UACnCD,EAAEgN,EAAI/M,EAAE+M,EAAI,GAAK,WAKvBs7D,UAAY,IAIrBu0B,GAAW79F,UAAUuiG,SAAW,kBACvB1+F,KAAKylE,WAOdu0B,GAAW79F,UAAUwiG,gBAAkB,SAAU7tF,QAC1CspF,aAAetpF,GAOtBkpF,GAAW79F,UAAUqiD,WAAa,SAAUrvB,WAC1B1vB,IAAZ0vB,EAAuB,CAEzBmpB,GAAKqQ,oBADQ,CAAC,WAAY,QAAS,OAAQ,mBAAoB,WAAY,SAAS,sBAAuB,qBAC1E3oD,KAAKmvB,QAASA,GAGd,mBAAtBA,EAAQqlC,aACjBrlC,EAAQqlC,WAAa,CACnBC,SAAUtlC,EAAQqlC,aAItBlc,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,iBACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,cACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UAErCA,EAAQ2tE,eAC0B,UAAhCr3D,GAAOtW,EAAQ2tE,gBACb3tE,EAAQ2tE,cAAc+B,kBACqB,WAAzC1vE,EAAQ2tE,cAAc+B,qBACnB1vE,QAAQ2tE,cAAcrV,MAAQ,EAEa,WAAzCt4D,EAAQ2tE,cAAc+B,qBACxB1vE,QAAQ2tE,cAAcrV,MAAQ,QAG9Bt4D,QAAQ2tE,cAAc+B,gBAAkB,mBACxC1vE,QAAQ2tE,cAAcrV,MAAQ,OAa/CuS,GAAW79F,UAAUg6C,OAAS,SAAUkuB,QACjCA,MAAQA,OACRxwC,QAAUwwC,EAAMxwC,SAAW,aAC3ByU,UAAY+7B,EAAM/7B,WAAatoC,KAAKsoC,WAAa,kBAAoBtoC,KAAKi6F,yBAAyB,GAAK,QACxG3+B,aAA4B77D,IAAlB4kE,EAAM/I,SAA+B+I,EAAM/I,aACrD9mC,MAAQ6vC,EAAM7vC,WACdgqB,WAAW6lB,EAAMl1C,UAaxB6qE,GAAW79F,UAAUg8F,UAAY,SAAUrB,EAAWsB,EAAYP,EAAW1tF,EAAGZ,GAC7D9J,MAAbo4F,GAAuC,MAAbA,IAE5BA,EAAY,CAACtB,IADHnpE,SAASya,gBAAgB,6BAA8B,OAC1C4vD,YAAY,GAAItoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ,CAAC3sE,eAEhEP,MAAL0K,GAAuB,MAALA,IACpBA,EAAI,GAEG1K,MAAL8J,GAAuB,MAALA,IACpBA,EAAI,GAAM6uF,GAEJp4F,KAAKmvB,QAAQqF,WACd,OACHsqE,GAAMlE,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,aAE/C,aACA,QACH8B,GAAOiB,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,aAEhD,MACHkH,GAAKnE,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,SAG9C,CAACmH,KAAMnH,EAAUtB,IAAKhuD,MAAOvoC,KAAK6zB,QAASw0B,YAAaroD,KAAKmvB,QAAQstE,mBAG9EzC,GAAW79F,UAAU8iG,UAAY,SAAUxuB,WACrCisB,EAAOjsB,EAAU,GAAGlnE,EACpBozF,EAAOlsB,EAAU,GAAGlnE,EACfwK,EAAI,EAAGA,EAAI08D,EAAUzzE,OAAQ+W,IACpC2oF,EAAOA,EAAOjsB,EAAU18D,GAAGxK,EAAIknE,EAAU18D,GAAGxK,EAAImzF,EAChDC,EAAOA,EAAOlsB,EAAU18D,GAAGxK,EAAIknE,EAAU18D,GAAGxK,EAAIozF,QAE3C,CAACv6F,IAAKs6F,EAAMp3F,IAAKq3F,EAAMF,iBAAkBz8F,KAAKmvB,QAAQstE,mBC/G/DpC,GAAOl+F,UAAY,IAAIw8C,GAEvB0hD,GAAOl+F,UAAUixC,MAAQ,gBAClBu/B,OAAS,QACTgrB,eAAiB,GAGxB0C,GAAOl+F,UAAU+iG,SAAW,SAAS32D,EAAOwvD,GAGI,GAA1CA,EAAa5oE,QAAQgwE,oBAClBn/F,KAAK2sE,OAAOtvE,eAAekrC,UACzBokC,OAAOpkC,GAASwvD,QAElBJ,gBAAkB,IAI3B0C,GAAOl+F,UAAUkqE,YAAc,SAAS99B,EAAOwvD,QACxCprB,OAAOpkC,GAASwvD,GAGvBsC,GAAOl+F,UAAUijG,YAAc,SAAS72D,GAClCvoC,KAAK2sE,OAAOtvE,eAAekrC,YACtBvoC,KAAK2sE,OAAOpkC,QACdovD,gBAAkB,IAI3B0C,GAAOl+F,UAAUusD,QAAU,gBACpBtK,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,kBACtB8V,IAAIuqB,MAAMn0C,MAAMkL,SAAW,gBAC3B0e,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,YACtB1E,IAAIuqB,MAAMn0C,MAAMC,QAAU,aAE1B2pB,IAAIihD,SAAWjyE,SAASE,cAAc,YACtC8wB,IAAIihD,SAAS/2D,UAAY,uBACzB8V,IAAIihD,SAAS7qE,MAAMkL,SAAW,gBAC9B0e,IAAIihD,SAAS7qE,MAAMsuB,IAAM,WAEzByzC,IAAMnpE,SAASya,gBAAgB,6BAA6B,YAC5D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMsuB,IAAM,WAChByzC,IAAI/hE,MAAMoU,MAAQ5oC,KAAKmvB,QAAQorE,SAAW,EAAI,UAC9ChE,IAAI/hE,MAAMqU,OAAS,YAEnBuV,IAAIuqB,MAAMh0C,YAAY30B,KAAKu2F,UAC3Bn4C,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIihD,WAMtChF,GAAOl+F,UAAU+zD,KAAO,WAElBlwD,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,QAOnD0xB,GAAOl+F,UAAUqnE,KAAO,WAEjBxjE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,QAI9C0xB,GAAOl+F,UAAUqiD,WAAa,SAASrvB,GAErCmpB,GAAKqQ,oBADQ,CAAC,UAAU,cAAc,QAAQ,OAAO,SACpB3oD,KAAKmvB,QAASA,IAGjDkrE,GAAOl+F,UAAUk0D,OAAS,eACpBgoC,EAAe,EACfH,EAAal4C,GAAYhgD,KAAK2sE,WAClCurB,QAAAA,GAAgB,SAAU/6F,EAAEC,UAClBD,EAAIC,GAAK,EAAI,SAGlB,IAAIN,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IAAK,KACtCq/D,EAAU+7B,EAAWp7F,GACW,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,IACvIk8B,OAImC,GAAnCr4F,KAAKmvB,QAAQnvB,KAAKs6F,MAAMh/B,SAA2C,GAAvBt7D,KAAK23F,gBAA+C,GAAxB33F,KAAKmvB,QAAQutD,SAAoC,GAAhB2b,OACtGnoC,WAEF,SACEsT,OACmC,YAApCxjE,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,UAA8D,eAApC1/B,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,eACvE0e,IAAIuqB,MAAMn0C,MAAMkgB,KAAO,WACvB0J,IAAIuqB,MAAMn0C,MAAM4kE,UAAY,YAC5Bh7C,IAAIihD,SAAS7qE,MAAM4kE,UAAY,YAC/Bh7C,IAAIihD,SAAS7qE,MAAMkgB,KAAQ10C,KAAKmvB,QAAQorE,SAAW,GAAM,UACzDn8C,IAAIihD,SAAS7qE,MAAMmgB,MAAQ,QAC3B4hD,IAAI/hE,MAAMkgB,KAAO,WACjB6hD,IAAI/hE,MAAMmgB,MAAQ,UAGlByJ,IAAIuqB,MAAMn0C,MAAMmgB,MAAQ,WACxByJ,IAAIuqB,MAAMn0C,MAAM4kE,UAAY,aAC5Bh7C,IAAIihD,SAAS7qE,MAAM4kE,UAAY,aAC/Bh7C,IAAIihD,SAAS7qE,MAAMmgB,MAAS30C,KAAKmvB,QAAQorE,SAAW,GAAM,UAC1Dn8C,IAAIihD,SAAS7qE,MAAMkgB,KAAO,QAC1B6hD,IAAI/hE,MAAMmgB,MAAQ,WAClB4hD,IAAI/hE,MAAMkgB,KAAO,IAGgB,YAApC10C,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,UAA8D,aAApC1/B,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,cACvE0e,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,EAAIzL,OAAOr3C,KAAK44C,KAAKwF,IAAIY,OAAOxqB,MAAMsuB,IAAIv8C,QAAQ,KAAK,KAAO,UACpF63C,IAAIuqB,MAAMn0C,MAAMq0B,OAAS,OAE3B,KACCy2C,EAAmBt/F,KAAK44C,KAAKY,SAASwF,OAAOnW,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,YACxFuV,IAAIuqB,MAAMn0C,MAAMq0B,OAAS,EAAIy2C,EAAmBjoD,OAAOr3C,KAAK44C,KAAKwF,IAAIY,OAAOxqB,MAAMsuB,IAAIv8C,QAAQ,KAAK,KAAO,UAC1G63C,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,GAGH,GAAtB9iD,KAAKmvB,QAAQsnE,YACVr4C,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKo+C,IAAIihD,SAASj2C,YAAc,GAAK,UAC7DhL,IAAIihD,SAAS7qE,MAAMmgB,MAAQ,QAC3ByJ,IAAIihD,SAAS7qE,MAAMkgB,KAAO,QAC1B6hD,IAAI/hE,MAAMoU,MAAQ,aAGlBwV,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKmvB,QAAQorE,SAAW,GAAKv6F,KAAKo+C,IAAIihD,SAASj2C,YAAc,GAAK,UAC1Fm2C,uBAGH1rE,EAAU,OACT/2B,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IACjCq/D,EAAU+7B,EAAWp7F,GACe,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,KACvItoC,GAAW7zB,KAAK2sE,OAAOxQ,GAAStoC,QAAU,eAGzCuqB,IAAIihD,SAAS3zC,UAAY73B,OACzBuqB,IAAIihD,SAAS7qE,MAAM2zC,WAAe,IAAOnoE,KAAKmvB,QAAQorE,SAAYv6F,KAAKmvB,QAAQqrE,YAAe,OAIvGH,GAAOl+F,UAAUojG,gBAAkB,cAC7Bv/F,KAAKo+C,IAAIuqB,MAAMphC,WAAY,KACzB2wD,EAAal4C,GAAYhgD,KAAK2sE,WAClCurB,QAAAA,GAAgB,SAAU/6F,EAAEC,UAClBD,EAAIC,GAAK,EAAI,KAIvB46F,GAAsBh4F,KAAKy3F,iBAEvBztB,EAAUh+C,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIuqB,OAAO62B,WAClDvH,EAAa5gD,OAAO2yB,EAAQzjE,QAAQ,KAAK,KACzC4D,EAAI8tF,EACJnB,EAAY92F,KAAKmvB,QAAQorE,SACzBnC,EAAa,IAAOp4F,KAAKmvB,QAAQorE,SACjChxF,EAAI0uF,EAAa,GAAMG,EAAa,OAEnC7B,IAAI/hE,MAAMoU,MAAQkuD,EAAY,EAAImB,EAAa,SAE/C,IAAIn7F,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IAAK,KACtCq/D,EAAU+7B,EAAWp7F,GACW,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,UAClIwQ,OAAOxQ,GAASg8B,UAAUrB,EAAWsB,EAAYp4F,KAAK63F,UAAW1tF,EAAGZ,GACzEA,GAAK6uF,EAAap4F,KAAKmvB,QAAQqrE,gBChMvC,SAASiF,GAAU7mD,EAAMzpB,QAClBiE,GAAKs3C,UACL9xB,KAAOA,OAEPsE,eAAiB,CACpBu/C,iBAAkB,OAClBiD,aAAc,UACdzzF,MAAM,EACN0zF,UAAU,EACVt8F,OAAO,EACPu8F,YAAa,QACb1C,OAAQ,CACNxgB,SAAS,EACTr0B,YAAa,UAEf7zB,MAAO,OACPwmE,SAAU,CACRpyD,MAAO,GACPszD,YAAY,EACZz1B,MAAO,UAETq2B,cAAe,CACbpgB,SAAS,EACTmiB,gBAAiB,cACjBpX,MAAO,IAETjzB,WAAY,CACVkoB,SAAS,EACT/8C,KAAM,EACNnL,MAAO,UAETqrE,SAAU,GACVC,OAAQ,GACRnzB,OAAQ,CACN1tB,WAAY,UAKX9vB,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,qBAC/BkB,IAAM,QACN/I,MAAQ,QACRwO,OAAS,UACT8oB,OAAS,QACTozB,oBAAqB,OACrBC,iBAAkB,OAClBC,yBAA0B,OAC1BC,kBAAmB,MAEpBvhD,EAAK3+C,UACJylE,UAAY,UACZsK,WAAa,UAGbE,cAAgB,KACZ,SAAUzvB,EAAOD,EAAQ2vB,GAC9BvxB,EAAGwxB,OAAO5vB,EAAO/J,eAET,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAG0xB,UAAU9vB,EAAO/J,eAEZ,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAG2xB,UAAU/vB,EAAO/J,cAKnB+5B,eAAiB,KACb,SAAU/vB,EAAOD,EAAQ2vB,GAC9BvxB,EAAG6xB,aAAajwB,EAAO/J,eAEf,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAGmyB,gBAAgBvwB,EAAO/J,eAElB,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAGoyB,gBAAgBxwB,EAAO/J,cAIzBA,MAAQ,QACRy6B,UAAY,QACZkvB,UAAYngG,KAAK44C,KAAKc,MAAMlZ,WAC5B8kC,YAAc,QAEdmyB,YAAc,QACdj5C,WAAWrvB,QACX8qE,yBAA2B,CAAC,QAC5BrhD,KAAKiF,QAAQvH,GAAG,gBAAgB,WACnCqI,EAAGwhD,UAAYxhD,EAAG/F,KAAKc,MAAMlZ,MAC7Bme,EAAG43C,IAAI/hE,MAAMkgB,KAAO4D,GAAKge,OAAOC,QAAQ5X,EAAGtJ,MAAMzM,OAEjD+V,EAAGuhD,kBAAmB,EAEtBvhD,EAAG0R,OAAOh0D,KAAKsiD,WAIZ+J,eACAmvC,UAAY,CAACtB,IAAKv2F,KAAKu2F,IAAKkB,YAAaz3F,KAAKy3F,YAAatoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ3sE,KAAK2sE,QAGtG8yB,GAAUtjG,UAAY,IAAIw8C,GAK1B8mD,GAAUtjG,UAAUusD,QAAU,eACxBigB,EAAQv7C,SAASE,cAAc,OACnCq7C,EAAMrgC,UAAY,sBACb8V,IAAIuqB,MAAQA,OAGZ4tB,IAAMnpE,SAASya,gBAAgB,6BAA8B,YAC7D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMqU,QAAU,GAAK7oC,KAAKmvB,QAAQywE,aAAar5F,QAAQ,KAAM,IAAM,UACvEgwF,IAAI/hE,MAAMC,QAAU,QACzBk0C,EAAMh0C,YAAY30B,KAAKu2F,UAGlBpnE,QAAQ0wE,SAASx3C,YAAc,YAC/B+3C,UAAY,IAAI9J,GAASt2F,KAAK44C,KAAM54C,KAAKmvB,QAAQ0wE,SAAU7/F,KAAKu2F,IAAKv2F,KAAKmvB,QAAQw9C,aAElFx9C,QAAQ0wE,SAASx3C,YAAc,aAC/Bg4C,WAAa,IAAI/J,GAASt2F,KAAK44C,KAAM54C,KAAKmvB,QAAQ0wE,SAAU7/F,KAAKu2F,IAAKv2F,KAAKmvB,QAAQw9C,eACjF3sE,KAAKmvB,QAAQ0wE,SAASx3C,iBAGxBi4C,WAAa,IAAIjG,GAAOr6F,KAAK44C,KAAM54C,KAAKmvB,QAAQ2wE,OAAQ,OAAQ9/F,KAAKmvB,QAAQw9C,aAC7E4zB,YAAc,IAAIlG,GAAOr6F,KAAK44C,KAAM54C,KAAKmvB,QAAQ2wE,OAAQ,QAAS9/F,KAAKmvB,QAAQw9C,aAE/EnJ,QAOPi8B,GAAUtjG,UAAUqiD,WAAa,SAAUrvB,MACrCA,EAAS,MAEiB1vB,IAAxB0vB,EAAQywE,kBAAgDngG,IAAnB0vB,EAAQ0Z,aAC1Cm3D,iBAAkB,OAClBC,yBAA0B,QAEsBxgG,IAA9CO,KAAK44C,KAAKY,SAASC,gBAAgB5Q,aAAgDppC,IAAxB0vB,EAAQywE,aACtEz0C,IAAUh8B,EAAQywE,YAAc,IAAIr5F,QAAQ,KAAM,KAAOvG,KAAK44C,KAAKY,SAASC,gBAAgB5Q,cACzFm3D,iBAAkB,GAG3B1nD,GAAKqQ,oBAVQ,CAAC,WAAY,eAAgB,QAAS,SAAU,cAAe,mBAAoB,QAAS,WAAY,WAAY,OAAQ,UAUxG3oD,KAAKmvB,QAASA,GAC/CmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,iBACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,cACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UAErCA,EAAQ2tE,eAC0B,UAAhCr3D,GAAOtW,EAAQ2tE,gBACb3tE,EAAQ2tE,cAAc+B,kBACqB,WAAzC1vE,EAAQ2tE,cAAc+B,qBACnB1vE,QAAQ2tE,cAAcrV,MAAQ,EAEa,WAAzCt4D,EAAQ2tE,cAAc+B,qBACxB1vE,QAAQ2tE,cAAcrV,MAAQ,QAG9Bt4D,QAAQ2tE,cAAc+B,gBAAkB,mBACxC1vE,QAAQ2tE,cAAcrV,MAAQ,KAMvCznF,KAAKogG,gBACkB3gG,IAArB0vB,EAAQ0wE,gBACLO,UAAU5hD,WAAWx+C,KAAKmvB,QAAQ0wE,eAClCQ,WAAW7hD,WAAWx+C,KAAKmvB,QAAQ0wE,WAIxC7/F,KAAKsgG,iBACgB7gG,IAAnB0vB,EAAQ2wE,cACLQ,WAAW9hD,WAAWx+C,KAAKmvB,QAAQ2wE,aACnCS,YAAY/hD,WAAWx+C,KAAKmvB,QAAQ2wE,SAIzC9/F,KAAK2sE,OAAOtvE,eApMJ,uBAqMLsvE,OAAL,cAAuBnuB,WAAWrvB,GAKlCnvB,KAAKo+C,IAAIuqB,aACNu3B,kBAAiB,OACjBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,MAO7CotC,GAAUtjG,UAAU+zD,KAAO,WAErBlwD,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,QAQnD82B,GAAUtjG,UAAUqnE,KAAO,WAEpBxjE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,QAS9C82B,GAAUtjG,UAAUg5D,SAAW,SAAU3e,OAErC+7B,EADE5zB,EAAK3+C,KAEPq0E,EAAer0E,KAAKylE,aAGjBjvB,EAGA,CAAA,KAAIA,aAAiBoB,IAAWpB,aAAiB89B,UAI9C,IAAItnD,UAAU,wDAHfy4C,UAAYluB,GAAkBf,aAH9BivB,UAAY,QASf4O,OAEF/7B,SAAAA,GAAat4C,KAAKiwE,eAAe,SAAUjqE,EAAUw6C,GACnD6zB,EAAa99B,IAAIiK,EAAOx6C,MAI1BquE,EAAah8B,UAGbk6B,EAAM8B,EAAaj8B,cACdk4B,UAAUiC,IAGbvyE,KAAKylE,UAAW,KAEdryC,EAAKpzB,KAAKozB,MACdklB,SAAAA,GAAat4C,KAAKiwE,eAAe,SAAUjqE,EAAUw6C,GACnD7B,EAAG8mB,UAAUnvB,GAAGkK,EAAOx6C,EAAUotB,MAInCm/C,EAAMvyE,KAAKylE,UAAUrtB,cAChB+3B,OAAOoC,KAShBktB,GAAUtjG,UAAUi5D,UAAY,SAAUuX,OAEpC4F,EADA5zB,EAAK3+C,QAILA,KAAK+vE,WAAY,IACnBz3B,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAAUvqE,EAAUw6C,GACpD7B,EAAGoxB,WAAWx5B,IAAIiK,EAAOx6C,MAI3BusE,EAAMvyE,KAAK+vE,WAAW33B,cACjB23B,WAAa,SACb,IAAIjzE,EAAI,EAAGA,EAAIy1E,EAAIv1E,OAAQF,SACzB0jG,aAAajuB,EAAIz1E,OAKrB6vE,EAGA,CAAA,KAAIA,aAAkB/0B,IAAW+0B,aAAkB2H,UAIhD,IAAItnD,UAAU,wDAHf+iD,WAAapD,YAHboD,WAAa,QAShB/vE,KAAK+vE,WAAY,KAEf38C,EAAKpzB,KAAKozB,MACdklB,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAAUvqE,EAAUw6C,GACpD7B,EAAGoxB,WAAWz5B,GAAGkK,EAAOx6C,EAAUotB,MAIpCm/C,EAAMvyE,KAAK+vE,WAAW33B,cACjBo4B,aAAa+B,KAItBktB,GAAUtjG,UAAUk0E,UAAY,SAAUkC,QACnCkuB,oBAAoBluB,IAE3BktB,GAAUtjG,UAAUg0E,OAAS,SAAUoC,QAChClC,UAAUkC,IAEjBktB,GAAUtjG,UAAUm0E,UAAY,SAAUiC,QACnClC,UAAUkC,IAEjBktB,GAAUtjG,UAAU20E,gBAAkB,SAAUE,QACzCyvB,oBAAoB,KAAMzvB,IAEjCyuB,GAAUtjG,UAAUq0E,aAAe,SAAUQ,QACtCF,gBAAgBE,IAQvByuB,GAAUtjG,UAAU40E,gBAAkB,SAAUC,OACzC,IAAIl0E,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,SAC9B0jG,aAAaxvB,EAASl0E,SAExBojG,kBAAmB,OACnBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,KAQ3CotC,GAAUtjG,UAAUqkG,aAAe,SAAUrkC,GACvCn8D,KAAK2sE,OAAOtvE,eAAe8+D,KACwB,SAAjDn8D,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWjB,YAAYjjC,QACvBokC,YAAYnB,YAAYjjC,QACxBokC,YAAYlwC,gBAGZ+vC,UAAUhB,YAAYjjC,QACtBmkC,WAAWlB,YAAYjjC,QACvBmkC,WAAWjwC,iBAEXrwD,KAAK2sE,OAAOxQ,KAWvBsjC,GAAUtjG,UAAUukG,aAAe,SAAUr8B,EAAOlI,GAC7Cn8D,KAAK2sE,OAAOtvE,eAAe8+D,SAYzBwQ,OAAOxQ,GAAShmB,OAAOkuB,GACyB,SAAjDrkE,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWh6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAC5CokC,YAAYl6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAE7CikC,UAAUhB,YAAYjjC,QACtBmkC,WAAWlB,YAAYjjC,UAGvBikC,UAAU/5B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAC3CmkC,WAAWj6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAE5CkkC,WAAWjB,YAAYjjC,QACvBokC,YAAYnB,YAAYjjC,WAxB1BwQ,OAAOxQ,GAAW,IAAI69B,GAAW31B,EAAOlI,EAASn8D,KAAKmvB,QAASnvB,KAAKi6F,0BACpB,SAAjDj6F,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWnB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,SACzCokC,YAAYrB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,WAG1CikC,UAAUlB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,SACxCmkC,WAAWpB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,WAoB7CmkC,WAAWjwC,cACXkwC,YAAYlwC,UAWnBovC,GAAUtjG,UAAUskG,oBAAsB,SAAUluB,EAAKvB,MACjC,MAAlBhxE,KAAKylE,UAAmB,KACtBk7B,EAAgB,GAChBnqD,EAAQx2C,KAAKylE,UAAU17D,MACvB8tC,EAAU73C,KAAKylE,UAAUhuB,OACzBmpD,EAAQ,GACRruB,MACFA,QAAAA,GAAQ,SAAUn/C,GAChBwtE,EAAMxtE,GAAMA,aAKZytE,EAAc,GACT/jG,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KACjCi7C,EAAOvB,EAAM15C,GACbq/D,EAAUpkB,EAAKssB,MACflI,MAAAA,IACFA,EAxbQ,iBA0bV0kC,EAAYxjG,eAAe8+D,GAAW0kC,EAAY1kC,KAAa0kC,EAAY1kC,GAAW,MAIpF2kC,EAAmB,OAClB9vB,GAAYuB,MACVpW,KAAWn8D,KAAK2sE,UACf3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KAEnC4kC,GADJ18B,EAAQrkE,KAAK2sE,OAAOxQ,IACOuiC,WAE3BiC,EAAcxkC,GAAWrlB,GAAAiqD,QAAAA,GAAsB,SAAUhpD,UACvD+oD,EAAiB/oD,EAAKF,IAAYE,EAAKF,GAC/BE,EAAKF,KAAa+oD,EAAM7oD,EAAKF,WAEnCmpD,EAAYH,EAAY1kC,GAC5B0kC,EAAY1kC,IAAYwkC,EAAcxkC,GAASn/D,OAC3C2jG,EAAcxkC,GAASn/D,OAASgkG,IAClCL,EAAcxkC,GAAS6kC,EAAY,GAAK,QAO3ClkG,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,OAGxBq/D,OADJA,GADApkB,EAAOvB,EAAM15C,IACEunE,SAEblI,EAvdQ,iBAydL6U,IAAYuB,GAAQx6B,EAAKF,KAAa+oD,EAAM7oD,EAAKF,MAAcipD,EAAiBzjG,eAAe06C,EAAKF,KAGpG8oD,EAActjG,eAAe8+D,KAChCwkC,EAAcxkC,GAAW,IAAIlgE,MAAM4kG,EAAY1kC,SAG7C8kC,EAAW3oD,GAAK4oD,aAAanpD,GACjCkpD,EAAS92F,EAAImuC,GAAKpB,QAAQa,EAAK5tC,EAAG,QAClC82F,EAASxgE,IAAM6X,GAAKpB,QAAQa,EAAKtX,IAAK,QACtCwgE,EAASE,SAAWppD,EAAKxuC,EACzB03F,EAAS13F,EAAI8tC,OAAOU,EAAKxuC,GACzB03F,EAASppD,GAAWE,EAAKF,OAErBjzB,EAAO+7E,EAAcxkC,GAASn/D,OAAS6jG,EAAY1kC,KACvDwkC,EAAcxkC,GAASv3C,GAASq8E,MAI7B9kC,KAAWn8D,KAAK2sE,OACf3sE,KAAK2sE,OAAOtvE,eAAe8+D,KACxBwkC,EAActjG,eAAe8+D,KAChCwkC,EAAcxkC,GAAW,IAAIlgE,MAAM,SAMpCkgE,KAAWwkC,KACVA,EAActjG,eAAe8+D,MACM,GAAjCwkC,EAAcxkC,GAASn/D,OACrBgD,KAAK2sE,OAAOtvE,eAAe8+D,SACxBqkC,aAAarkC,OAEf,KACDkI,OAAQ5kE,EACWA,MAAnBO,KAAK+vE,aACP1L,EAAQrkE,KAAK+vE,WAAWhmE,IAAIoyD,IAEjB18D,MAAT4kE,IACFA,EAAQ,CAACjxC,GAAI+oC,EAAStoC,QAAS7zB,KAAKmvB,QAAQuwE,aAAevjC,SAExDukC,aAAar8B,EAAOlI,QACpBwQ,OAAOxQ,GAAShH,SAASwrC,EAAcxkC,SAI7C+jC,kBAAmB,OACnBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,MAQ7CotC,GAAUtjG,UAAUk0D,OAAS,eACvBxX,GAAU,OAGTxD,MAAMzM,MAAQ5oC,KAAKo+C,IAAIuqB,MAAMvf,iBAC7B/T,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,OACnD7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAC1B9iD,KAAK44C,KAAKY,SAASmY,OAAO9I,OAG9BhQ,EAAU74C,KAAK8pD,cAAgBjR,QAG3Bo6B,EAAkBjzE,KAAK44C,KAAKc,MAAMjZ,IAAMzgC,KAAK44C,KAAKc,MAAMlZ,MACxD0yC,EAAUD,GAAmBjzE,KAAKmzE,0BACjCA,oBAAsBF,EAKZ,GAAXp6B,UACG09C,IAAI/hE,MAAMoU,MAAQ0P,GAAKge,OAAOC,OAAO,EAAIv2D,KAAKq1C,MAAMzM,YACpD2tD,IAAI/hE,MAAMkgB,KAAO4D,GAAKge,OAAOC,QAAQv2D,KAAKq1C,MAAMzM,QAGL,GAA5CgrB,KAAC5zD,KAAKmvB,QAAQ0Z,OAAS,WAAY,MAA8C,GAAhC7oC,KAAKigG,+BACnDD,iBAAkB,OAKC,GAAxBhgG,KAAKggG,iBACHhgG,KAAKmvB,QAAQywE,aAAe5/F,KAAKq1C,MAAMxM,OAAS,YAC7C1Z,QAAQywE,YAAc5/F,KAAKq1C,MAAMxM,OAAS,UAC1C0tD,IAAI/hE,MAAMqU,OAAS7oC,KAAKq1C,MAAMxM,OAAS,WAEzCm3D,iBAAkB,QAGlBzJ,IAAI/hE,MAAMqU,QAAU,GAAK7oC,KAAKmvB,QAAQywE,aAAar5F,QAAQ,KAAM,IAAM,KAI/D,GAAXsyC,GAA6B,GAAVq6B,GAA6C,GAA3BlzE,KAAK+/F,oBAAuD,GAAzB//F,KAAKkgG,iBAC/ErnD,EAAU74C,KAAKohG,gBAAkBvoD,OAC5BqnD,kBAAmB,UAIF,GAAlBlgG,KAAKmgG,UAAgB,KACnB3jF,EAASxc,KAAK44C,KAAKc,MAAMlZ,MAAQxgC,KAAKmgG,UACtCzmD,EAAQ15C,KAAK44C,KAAKc,MAAMjZ,IAAMzgC,KAAK44C,KAAKc,MAAMlZ,SAC1B,GAApBxgC,KAAKq1C,MAAMzM,MAAY,KAErBJ,EAAUhsB,GADSxc,KAAKq1C,MAAMzM,MAAQ8Q,QAErC68C,IAAI/hE,MAAMkgB,MAAS10C,KAAKq1C,MAAMzM,MAAQJ,EAAW,kBAIvD83D,WAAWjwC,cACXkwC,YAAYlwC,SACVxX,GAIT4mD,GAAUtjG,UAAUklG,mBAAqB,eAEnCC,EAAY,OACX,IAAInlC,KAAWn8D,KAAK2sE,UACnB3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KACnCkI,EAAQrkE,KAAK2sE,OAAOxQ,GACH,GAAjBkI,EAAM/I,cAAgE77D,IAA5CO,KAAKmvB,QAAQw9C,OAAO1tB,WAAWkd,IAAqE,GAA3Cn8D,KAAKmvB,QAAQw9C,OAAO1tB,WAAWkd,IACpHmlC,EAAUrkG,KAAK,CAACm2B,GAAG+oC,EAAQolC,OAAOl9B,EAAMl1C,QAAQoyE,SAItDjpD,GAAKmmD,WAAW6C,GAAU,SAASnkG,EAAEC,OAC/BokG,EAAKrkG,EAAEokG,OACPE,EAAKrkG,EAAEmkG,mBACA9hG,IAAP+hG,IAAkBA,EAAG,QACd/hG,IAAPgiG,IAAkBA,EAAG,GAClBD,GAAIC,EAAI,EAAID,EAAGC,GAAM,EAAG,aAE7BzwB,EAAW,IAAI/0E,MAAMqlG,EAAUtkG,QAC1BF,EAAE,EAAGA,EAAGwkG,EAAUtkG,OAAQF,IACjCk0E,EAASl0E,GAAKwkG,EAAUxkG,GAAGs2B,UAEtB49C,GASTyuB,GAAUtjG,UAAUilG,aAAe,cAEjCpJ,GAAwBh4F,KAAKy3F,aACL,GAApBz3F,KAAKq1C,MAAMzM,OAAgC,MAAlB5oC,KAAKylE,UAAmB,KAC/CpB,EAAOvnE,EACPw/F,EAAc,GAGdoF,EAAU1hG,KAAK44C,KAAKN,KAAK44C,cAAclxF,KAAK44C,KAAKY,SAAS2H,KAAKvY,OAC/D+4D,EAAU3hG,KAAK44C,KAAKN,KAAK44C,aAAa,EAAIlxF,KAAK44C,KAAKY,SAAS2H,KAAKvY,OAGlEooC,EAAWhxE,KAAKqhG,wBAChBrwB,EAASh0E,OAAS,EAAG,KACnB+yE,EAAa,YAGZ6xB,iBAAiB5wB,EAAUjB,EAAY2xB,EAASC,QAGhDE,eAAe7wB,EAAUjB,GAGzBjzE,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,SAC1BglG,qBAAqB/xB,EAAWiB,EAASl0E,aAI3CilG,YAAY/wB,EAAUjB,EAAYusB,GAOnB,GAJLt8F,KAAKgiG,aAAahxB,EAAUsrB,UAKzCtE,GAAwBh4F,KAAKy3F,kBACxBsI,oBAAqB,GACnB,OAEJA,oBAAqB,MAGtBkC,OAAQxiG,MACP3C,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC/BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACF,IAAvBkD,KAAKmvB,QAAQ9rB,OAAyC,SAAvBrD,KAAKmvB,QAAQqF,QACL/0B,MAArC4kE,EAAMl1C,QAAQ4sE,qBAAqC13B,EAAMl1C,QAAQ4sE,sBACtDt8F,MAATwiG,SACGC,OAAOnyB,EAAW1L,EAAMjxC,IAAK28C,EAAWkyB,EAAM7uE,KACf,GAAhCixC,EAAMl1C,QAAQ+tE,OAAOxgB,SAAwD,UAArCrY,EAAMl1C,QAAQ+tE,OAAO70C,cACvB,OAApCgc,EAAMl1C,QAAQ+tE,OAAO70C,aAA6D,UAArC45C,EAAM9yE,QAAQ+tE,OAAO70C,aACpE45C,EAAM9yE,QAAQ+tE,OAAO70C,YAAY,QACjC45C,EAAM9yE,QAAQ+tE,OAAO/gC,QAAQkI,EAAMjxC,KAEnCixC,EAAMl1C,QAAQ+tE,OAAO70C,YAAY,QACjCgc,EAAMl1C,QAAQ+tE,OAAO/gC,QAAQ8lC,EAAM7uE,MAIzC6uE,EAAQ59B,SAGP89B,qBAAqBpyB,EAAWiB,EAASl0E,IAAKunE,OAIjD+9B,EAAQ,OACPtlG,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEH,UAD5BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACnBqyB,QAAQqF,OAAoD,GAAhC6vC,EAAMl1C,QAAQ+tE,OAAOxgB,QAAiB,KACtE5D,EAAU/I,EAAWiB,EAASl0E,OACnB,MAAXg8E,GAAqC,GAAlBA,EAAQ97E,mBAG1BolG,EAAM/kG,eAAe2zE,EAASl0E,MACjCslG,EAAMpxB,EAASl0E,IAAMgiG,GAAMjC,SAAS/jB,EAASzU,IAEN,UAArCA,EAAMl1C,QAAQ+tE,OAAO70C,YAAyB,KAC5Cg6C,EAAah+B,EAAMl1C,QAAQ+tE,OAAO/gC,YACA,IAAlCvI,GAAAod,QAAAA,EAAiBqxB,GAAoB,CACvC1/F,QAAQq3C,IAAIqqB,EAAMjxC,GAAK,wCAA0CivE,YAG9DD,EAAM/kG,eAAeglG,KACxBD,EAAMC,GAAcvD,GAAMjC,SAAS9sB,EAAWsyB,GAAariG,KAAK2sE,OAAO01B,KAEzEvD,GAAM3B,YAAYiF,EAAMpxB,EAASl0E,IAAKunE,EAAO+9B,EAAMC,GAAariG,KAAK63F,gBAGrEiH,GAAM3B,YAAYiF,EAAMpxB,EAASl0E,IAAKunE,OAAO5kE,EAAWO,KAAK63F,eAMnEkH,GAAKrP,KAAK1e,EAAUjB,EAAY/vE,KAAK63F,WAChC/6F,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAC/BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,IACzBizE,EAAWiB,EAASl0E,IAAIE,OAAS,SAC3BqnE,EAAMl1C,QAAQqF,WACf,OACE4tE,EAAM/kG,eAAe2zE,EAASl0E,MACjCslG,EAAMpxB,EAASl0E,IAAMgiG,GAAMjC,SAAS9sB,EAAWiB,EAASl0E,IAAKunE,IAE/Dy6B,GAAMpP,KAAK0S,EAAMpxB,EAASl0E,IAAKunE,EAAOrkE,KAAK63F,eAExC,YAEA,SACwB,SAAvBxzB,EAAMl1C,QAAQqF,OAA2C,UAAvB6vC,EAAMl1C,QAAQqF,OAAyD,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,SAChGid,GAAOjK,KAAK3f,EAAWiB,EAASl0E,IAAKunE,EAAOrkE,KAAK63F,oBAgB/DG,GAAwBh4F,KAAKy3F,cACtB,GAGTgI,GAAUtjG,UAAU+lG,OAAS,SAAU3uF,EAAM+uF,OACvC19E,EAAO29E,EAAIC,EAAIC,EAAcC,EACjC99E,EAAQ,MAEH,IAAI7Q,EAAI,EAAGA,EAAIR,EAAKvW,OAAQ+W,IAAK,CACpC0uF,OAAehjG,EACfijG,OAAejjG,MAEV,IAAIkb,EAAIiK,EAAOjK,EAAI2nF,EAAQtlG,OAAQ2d,IAAK,IAEvC2nF,EAAQ3nF,GAAGxQ,IAAMoJ,EAAKQ,GAAG5J,EAAG,CAC9Bs4F,EAAeH,EAAQ3nF,GACvB+nF,EAAeJ,EAAQ3nF,GACvBiK,EAAQjK,QAGL,GAAI2nF,EAAQ3nF,GAAGxQ,EAAIoJ,EAAKQ,GAAG5J,EAAG,CACjCu4F,EAAeJ,EAAQ3nF,GAErB8nF,EADO,GAAL9nF,EACa+nF,EAGAJ,EAAQ3nF,EAAI,GAE7BiK,EAAQjK,cAKSlb,IAAjBijG,IACFD,EAAeH,EAAQA,EAAQtlG,OAAS,GACxC0lG,EAAeJ,EAAQA,EAAQtlG,OAAS,IAG1CulG,EAAKG,EAAav4F,EAAIs4F,EAAat4F,EACnCq4F,EAAKE,EAAan5F,EAAIk5F,EAAal5F,EAEjCgK,EAAKQ,GAAGxK,EADA,GAANg5F,EACUhvF,EAAKQ,GAAGotF,SAAWuB,EAAan5F,EAGhCgK,EAAKQ,GAAGotF,SAAYqB,EAAKD,GAAOhvF,EAAKQ,GAAG5J,EAAIs4F,EAAat4F,GAAKs4F,EAAal5F,IAmB7Fk2F,GAAUtjG,UAAUylG,iBAAmB,SAAU5wB,EAAUjB,EAAY2xB,EAASC,OAC1Et9B,EAAOvnE,EAAGiX,EAAGgkC,KACbi5B,EAASh0E,OAAS,MACfF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KAEhC2oE,GADJpB,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACP4hG,cAEI,GAAtBr+B,GAAAgE,EAAMl1C,SAAsB,KAC1BwzE,EAAiB,SAAUxlG,EAAGC,UACzBD,EAAEmC,WAAalC,EAAEkC,UAAY,EAAInC,EAAIC,GAAK,EAAI,GAEnDoiC,EAAQh+B,KAAK8D,IAAI,EAAGgzC,GAAKsqD,kBAAkBn9B,EAAWi8B,EAAS,IAAK,SAAUiB,IAC9E91D,EAAOrrC,KAAKY,IAAIqjE,EAAUzoE,OAAQs7C,GAAKsqD,kBAAkBn9B,EAAWk8B,EAAS,IAAK,QAASgB,GAAkB,GAC7G91D,GAAQ,IACVA,EAAO44B,EAAUzoE,YAEf6lG,EAAgB,IAAI5mG,MAAM4wC,EAAKrN,OAC9BzrB,EAAIyrB,EAAOzrB,EAAI84B,EAAM94B,IACxBgkC,EAAOssB,EAAMoB,UAAU1xD,GACvB8uF,EAAc9uF,EAAEyrB,GAASuY,EAE3Bg4B,EAAWiB,EAASl0E,IAAM+lG,OAI1B9yB,EAAWiB,EAASl0E,IAAMunE,EAAMoB,YAaxCg6B,GAAUtjG,UAAU0lG,eAAiB,SAAU7wB,EAAUjB,MAEnDiB,EAASh0E,OAAS,MACf,IAAIF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEL,GADtBkD,KAAK2sE,OAAOqE,EAASl0E,IACnBqyB,QAAQwwE,SAAkB,KAC9BkD,EAAgB9yB,EAAWiB,EAASl0E,OACpC+lG,EAAc7lG,OAAS,EAAG,KACxB8lG,EACAC,EAAiBF,EAAc7lG,OAM/BgmG,EAAiBD,GADL/iG,KAAK44C,KAAKN,KAAKy4C,eAAe8R,EAAcA,EAAc7lG,OAAS,GAAGmN,GAAKnK,KAAK44C,KAAKN,KAAKy4C,eAAe8R,EAAc,GAAG14F,IAE1I24F,EAAYthG,KAAKY,IAAIZ,KAAKC,KAAK,GAAMshG,GAAiBvhG,KAAK8D,IAAI,EAAG9D,KAAK+a,MAAMymF,aAEzEC,EAAc,IAAIhnG,MAAM8mG,GACnBhvF,EAAI,EAAGA,EAAIgvF,EAAgBhvF,GAAK+uF,EAAW,CAElDG,EADUzhG,KAAK+a,MAAMxI,EAAE+uF,IACND,EAAc9uF,GAEjCg8D,EAAWiB,EAASl0E,IAAMwlE,GAAA2gC,QAAAA,EAAmB,EAAEzhG,KAAK+a,MAAMwmF,EAAeD,OAenFrD,GAAUtjG,UAAU4lG,YAAc,SAAU/wB,EAAUjB,EAAYusB,OAC5D7rB,EAAWpM,EAAOvnE,EAGlBqyB,EAFA+zE,EAAmB,GACnBC,EAAoB,MAEpBnyB,EAASh0E,OAAS,EAAG,KAClBF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC/B2zE,EAAYV,EAAWiB,EAASl0E,IAChCqyB,EAAUnvB,KAAK2sE,OAAOqE,EAASl0E,IAAIqyB,QAC/BshD,EAAUzzE,OAAS,IACrBqnE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KAEP,IAAlBqyB,EAAQ9rB,OAAoC,QAAlB8rB,EAAQqF,MACH,SAA7BrF,EAAQstE,iBACVyG,EAAmBryB,GAAAqyB,QAAAA,EAAwBzyB,GAG3C0yB,EAAoBtyB,GAAAsyB,QAAAA,EAAyB1yB,GAI/C6rB,EAAYtrB,EAASl0E,IAAMunE,EAAM46B,UAAUxuB,EAAWO,EAASl0E,KAMrEiiG,GAAK1C,iBAAiB6G,EAAkB5G,EAAatrB,EAAU,iBAAkB,QACjF+tB,GAAK1C,iBAAiB8G,EAAmB7G,EAAatrB,EAAU,kBAAmB,WAYvFyuB,GAAUtjG,UAAU6lG,aAAe,SAAUhxB,EAAUsrB,OAIe8G,EAAQC,EAHxExqD,GAAU,EACVyqD,GAAgB,EAChBC,GAAiB,EACjBC,EAAU,IAAKC,EAAW,IAAKC,GAAW,IAAKC,GAAY,OAE3D3yB,EAASh0E,OAAS,EAAG,KAElB,IAAIF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KACpCunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,IAC7BunE,GAA2C,SAAlCA,EAAMl1C,QAAQstE,kBACzB6G,GAAgB,EAChBE,EAAU,IACVE,GAAW,KAEJr/B,GAASA,EAAMl1C,QAAQstE,mBAC9B8G,GAAiB,EACjBE,EAAW,IACXE,GAAY,SAKX7mG,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC3Bw/F,EAAYj/F,eAAe2zE,EAASl0E,MACE,IAApCw/F,EAAYtrB,EAASl0E,IAAI8mG,SAC3BR,EAAS9G,EAAYtrB,EAASl0E,IAAIsF,IAClCihG,EAAS/G,EAAYtrB,EAASl0E,IAAIwI,IAEe,SAA7Cg3F,EAAYtrB,EAASl0E,IAAI2/F,kBAC3B6G,GAAgB,EAChBE,EAAUA,EAAUJ,EAASA,EAASI,EACtCE,EAAUA,EAAUL,EAASA,EAASK,IAGtCH,GAAiB,EACjBE,EAAWA,EAAWL,EAASA,EAASK,EACxCE,EAAWA,EAAWN,EAASA,EAASM,IAM3B,GAAjBL,QACGlD,UAAU1hD,SAAS8kD,EAASE,GAEb,GAAlBH,QACGlD,WAAW3hD,SAAS+kD,EAAUE,GAGvC9qD,EAAU74C,KAAK6jG,qBAAqBP,EAAetjG,KAAKogG,YAAcvnD,EACtEA,EAAU74C,KAAK6jG,qBAAqBN,EAAgBvjG,KAAKqgG,aAAexnD,EAElD,GAAlB0qD,GAA2C,GAAjBD,QACvBlD,UAAU0D,WAAY,OACtBzD,WAAWyD,WAAY,SAGvB1D,UAAU0D,WAAY,OACtBzD,WAAWyD,WAAY,QAEzBzD,WAAW9I,QAAU+L,OACrBjD,WAAW7I,WAAax3F,KAAKogG,UAEJ,GAA1BpgG,KAAKqgG,WAAW9I,aAEX6I,UAAU50B,WADK,GAAlB+3B,EAC0BvjG,KAAKqgG,WAAWz3D,MAGhB,EAG9BiQ,EAAU74C,KAAKogG,UAAU/vC,UAAYxX,EACrCA,EAAU74C,KAAKqgG,WAAWhwC,UAAYxX,GAGtCA,EAAU74C,KAAKqgG,WAAWhwC,UAAYxX,MAIpCkrD,EAAa,CAAC,iBAAkB,kBAAmB,kBAAmB,wBACrEjnG,EAAI,EAAGA,EAAIinG,EAAW/mG,OAAQF,KACO,GAApC82D,GAAAod,QAAAA,EAAiB+yB,EAAWjnG,QAC9Bk0E,QAAAA,EAAgBpd,GAAAod,QAAAA,EAAiB+yB,EAAWjnG,IAAK,UAI9C+7C,GAYT4mD,GAAUtjG,UAAU0nG,qBAAuB,SAAUG,EAAU17C,OACzDjI,GAAU,SACE,GAAZ2jD,EACE17C,EAAKlK,IAAIuqB,MAAMphC,YAA6B,GAAf+gB,EAAK/N,SACpC+N,EAAK4H,OACL7P,GAAU,GAIPiI,EAAKlK,IAAIuqB,MAAMphC,YAA6B,GAAf+gB,EAAK/N,SACrC+N,EAAKkb,OACLnjB,GAAU,GAGPA,GAYTo/C,GAAUtjG,UAAU2lG,qBAAuB,SAAUmC,WAC/CzoD,EAAWx7C,KAAK44C,KAAKN,KAAKkD,SACrB1+C,EAAI,EAAGA,EAAImnG,EAAWjnG,OAAQF,IACrCmnG,EAAWnnG,GAAG49F,SAAWl/C,EAASyoD,EAAWnnG,GAAGqN,GAAKnK,KAAKq1C,MAAMzM,MAChEq7D,EAAWnnG,GAAG69F,SAAWsJ,EAAWnnG,GAAGyM,EACd9J,MAArBwkG,EAAWnnG,GAAG2jC,IAChBwjE,EAAWnnG,GAAG2+F,WAAajgD,EAASyoD,EAAWnnG,GAAG2jC,KAAOzgC,KAAKq1C,MAAMzM,MAGpEq7D,EAAWnnG,GAAG2+F,gBAAah8F,GAejCggG,GAAUtjG,UAAUgmG,qBAAuB,SAAU8B,EAAY5/B,OAC3D/b,EAAOtoD,KAAKogG,UACZ7C,EAAYlmD,OAAOr3C,KAAKu2F,IAAI/hE,MAAMqU,OAAOtiC,QAAQ,KAAM,KACrB,SAAlC89D,EAAMl1C,QAAQstE,mBAChBn0C,EAAOtoD,KAAKqgG,gBAET,IAAIvjG,EAAI,EAAGA,EAAImnG,EAAWjnG,OAAQF,IACrCmnG,EAAWnnG,GAAG69F,SAAWn5F,KAAK+a,MAAM+rC,EAAK4sC,aAAa+O,EAAWnnG,GAAGyM,IAEtE86D,EAAMs6B,gBAAgBn9F,KAAKY,IAAIm7F,EAAWj1C,EAAK4sC,aAAa,MCvkC9D,IACIzY,GAAO,UAUPjC,GAAa,CACfxlB,UAAW,CACT0nB,QAAS,SAAYD,IACrBxkD,OAAQ,SAAYwkD,YAAiB,YACrCxwB,UAAW,CAAC7N,IATN,OAUN+8B,SAAU,CAACrsD,OAXF,iBAWoB2tD,YAAiB,aAIhD7jB,iBAAkB,CAAC3hD,OApBR,mBAoB6B,aACxCwlF,iBAAkB,CAACxlF,OAAO,CAAC,OAAO,UAClCyoF,aAAc,CAACzoF,OAtBJ,UAuBXhL,KAAM,SAAYwwE,IAClBkjB,SAAU,SAAYljB,IACtBp5E,MAAM,SAAYo5E,IAClBmjB,YAAa,CAAC3oF,OA1BH,SA0BW1V,OAxBX,UAyBX27F,OAAQ,CACNxgB,QAAS,SAAYD,IACrBp0B,YAAa,CAACpxC,OAAO,CAAC,SAAS,MAAM,OAAO,UAC5CklD,QAAS,CAACrtC,OAzBD,UA0BTqsD,SAAU,SAAYsB,GAAK3tD,OA1BlB,WA4BX0F,MAAO,CAACvd,OAAO,CAAC,OAAO,MAAM,WAC7B+jF,SAAU,CACRpyD,MAAO,CAACrnC,OAjCC,UAkCTqqE,SAAU,CAACrqE,OAlCF,UAmCT26F,WAAY,SAAYzf,IACxBhW,MAAO,CAACxvD,OAAO,CAAC,OAAO,SAAS,UAChCkkE,SAAU,CAACrsD,OAlCF,WAoCXguE,cAAe,CACbpgB,QAAS,SAAYD,IACrBoiB,gBAAiB,CAAC5nF,OAAO,CAAC,cAAe,UAAU,YACnDwwE,MAAO,CAAClmF,OA1CC,UA2CT45E,SAAU,CAACrsD,OAxCF,iBAwCoB2tD,KAE/BjoB,WAAY,CACVkoB,QAAS,SAAYD,IACrBhoB,SAAU,UAAc,YACxB90B,KAAM,CAACp+B,OAhDE,UAiDTizB,MAAO,CAACvd,OAAO,CAAC,SAAS,WACzBkkE,SAAU,CAACrsD,OA/CF,iBA+CoB2tD,YAAiB,aAEhDojB,SAAU,CACRt3C,gBAAiB,SAAYk0B,IAC7Bx2B,gBAAiB,SAAYw2B,IAC7Bn2B,cAAe,SAAYm2B,IAC3Bga,MAAO,SAAYha,IACnB7zC,MAAO,CAAC3xB,OA3DC,SA2DO1V,OAzDP,UA0DT+5D,QAAS,SAAYmhB,IACrBsa,WAAY,SAAYta,IACxB/nC,KAAK,CACHgF,MAAO,CAACt3C,IAAI,CAACb,OA7DN,mBA6D0B,aAAa+D,IAAI,CAAC/D,OA7D5C,mBA6DgE,aAAa45E,SAAU,CAACrsD,OA1DxF,WA2DPrxB,OAAQ,UAAa,YACrBgyD,MAAO,CAAChE,KAAK,CAACx0C,OAjEP,SAiEc1V,OA/Dd,mBA+DkC,aAAaizB,MAAM,CAACvd,OAjEtD,mBAiE0E,aAAakkE,SAAU,CAACrsD,OA5DlG,WA6DPqsD,SAAU,CAACrsD,OA7DJ,WA+DT6lB,MAAM,CACJ+E,MAAO,CAACt3C,IAAI,CAACb,OAnEN,mBAmE0B,aAAa+D,IAAI,CAAC/D,OAnE5C,mBAmEgE,aAAa45E,SAAU,CAACrsD,OAhExF,WAiEPrxB,OAAQ,UAAa,YACrBgyD,MAAO,CAAChE,KAAK,CAACx0C,OAvEP,SAuEc1V,OArEd,mBAqEkC,aAAaizB,MAAM,CAACvd,OAvEtD,mBAuE0E,aAAakkE,SAAU,CAACrsD,OAlElG,WAmEPqsD,SAAU,CAACrsD,OAnEJ,WAqETqsD,SAAU,CAACrsD,OArEF,WAuEXgxE,OAAQ,CACNpjB,QAAS,SAAYD,IACrBga,MAAO,SAAYha,IACnB/nC,KAAM,CACJ4mB,QAAS,SAAYmhB,IACrB/8C,SAAU,CAACzoB,OAAO,CAAC,YAAY,eAAe,WAAW,gBACzDkkE,SAAU,CAACrsD,OA7EJ,WA+ET6lB,MAAO,CACL2mB,QAAS,SAAYmhB,IACrB/8C,SAAU,CAACzoB,OAAO,CAAC,YAAY,eAAe,WAAW,gBACzDkkE,SAAU,CAACrsD,OAlFJ,WAoFTqsD,SAAU,CAACrsD,OApFF,iBAoFoB2tD,KAE/B9P,OAAQ,CACN1tB,WAAY,CAAC29B,IApFP,OAqFNzB,SAAU,CAACrsD,OAxFF,WA2FX+oC,WAAY,SAAY4kB,IACxBE,eAAgB,CAACp7E,OA/FN,UAgGXmzD,WAAY,SAAY+nB,IACxBh8C,IAAK,CAACl/B,OAjGK,SAiGG0I,KA/FL,OA+FWgN,OAnGT,SAmGiBmT,OA5FjB,UA6FX3sB,OAAQ,CACNkpD,YAAa,CACXrsC,YAAa,CAACrD,OAtGP,mBAsG2B,aAClCoD,OAAQ,CAACpD,OAvGF,mBAuGsB,aAC7BmD,OAAQ,CAACnD,OAxGF,mBAwGsB,aAC7BqC,KAAM,CAACrC,OAzGA,mBAyGoB,aAC3BzJ,QAAS,CAACyJ,OA1GH,mBA0GuB,aAC9B7H,IAAK,CAAC6H,OA3GC,mBA2GmB,aAC1B1J,KAAM,CAAC0J,OA5GA,mBA4GoB,aAC3BjN,MAAO,CAACiN,OA7GD,mBA6GqB,aAC5BuE,QAAS,CAACvE,OA9GH,mBA8GuB,aAC9B5N,KAAM,CAAC4N,OA/GA,mBA+GoB,aAC3BkkE,SAAU,CAACrsD,OA3GJ,WA6GT83B,YAAa,CACXtsC,YAAa,CAACrD,OAnHP,mBAmH2B,aAClCoD,OAAQ,CAACpD,OApHF,mBAoHsB,aAC7BmD,OAAQ,CAACnD,OArHF,mBAqHsB,aAC7BqC,KAAM,CAACrC,OAtHA,mBAsHoB,aAC3BzJ,QAAS,CAACyJ,OAvHH,mBAuHuB,aAC9B7H,IAAK,CAAC6H,OAxHC,mBAwHmB,aAC1B1J,KAAM,CAAC0J,OAzHA,mBAyHoB,aAC3BjN,MAAO,CAACiN,OA1HD,mBA0HqB,aAC5BuE,QAAS,CAACvE,OA3HH,mBA2HuB,aAC9B5N,KAAM,CAAC4N,OA5HA,mBA4HoB,aAC3BkkE,SAAU,CAACrsD,OAxHJ,WA0HTqsD,SAAU,CAACrsD,OA1HF,WA4HX1E,OAAQ,UAAa,YACrBye,OAAQ,CAAC5xB,OAlIE,SAkIM1V,OAhIN,UAiIX63C,YAAa,CACX5Y,MAAO,CAACv2B,KAhID,OAgIO1I,OAlIL,SAkIa0V,OApIb,SAoIqBmT,OA7HrB,UA8HTqW,IAAK,CAACx2B,KAjIC,OAiIK1I,OAnIH,SAmIW0V,OArIX,SAqImBmT,OA9HnB,UA+HT4uB,OAAQ,CAAC/hC,OAtIA,UAuITkkE,SAAU,CAACrsD,OAlIF,SAkIUzoB,MApIX,UAsIV3I,OAAO,CAACuZ,OAzIG,UA0IXrE,QAAQ,CACNmoE,QAAS,CAAC6B,IAnIJ,OAoINzB,SAAU,CAACrsD,OAvIF,WAyIXxpB,IAAK,CAAC2E,KA1IG,OA0IG1I,OA5ID,SA4IS0V,OA9IT,SA8IiBmT,OAvIjB,UAwIXisC,UAAW,CAAC90D,OA7ID,SA6IS0V,OA/IT,UAgJXuxC,cAAe,CAACjnD,OA9IL,UA+IXa,IAAK,CAAC6H,KA7IG,OA6IG1I,OA/ID,SA+IS0V,OAjJT,SAiJiBmT,OA1IjB,UA2IXosC,UAAW,CAACj1D,OAhJD,SAgJS0V,OAlJT,UAmJXmmC,SAAU,SAAYq/B,IACtBxN,YAAa,SAAYwN,IACzBp0B,YAAa,CAACpxC,OArJH,UAsJX0hD,gBAAiB,SAAY8jB,IAC7Bx2B,gBAAiB,SAAYw2B,IAC7Bl0B,gBAAiB,SAAYk0B,IAC7Bn2B,cAAe,SAAYm2B,IAC3Bj8C,MAAO,CAACv2B,KAtJC,OAsJK1I,OAxJH,SAwJW0V,OA1JX,SA0JmBmT,OAnJnB,UAoJXq+B,SAAU,CACR9M,MAAO,CAAC1kC,OA5JC,mBA4JmB,aAC5B8sB,KAAM,CAACxiC,OA3JE,mBA2JkB,aAC3B45E,SAAU,CAACrsD,OAzJF,WA2JX8Z,MAAO,CAAC3xB,OAhKG,SAgKK1V,OA9JL,UA+JX87C,SAAU,SAAYo/B,IACtB36B,QAAS,CAAC7qC,OAAQ,CAAC,UAAW,SAAU,UAAW,KACnDsmC,QAAS,CAACh8C,OAjKC,UAkKX+7C,QAAS,CAAC/7C,OAlKC,UAmKXggG,OAAQ,CAAChgG,OAnKE,UAoKX45E,SAAU,CAACrsD,OAjKA,WAoKTiuD,GAAmB,CACrB7wD,OAAQ,CACN0sC,iBAAkB,CAAC,OAAQ,OAAQ,QAAS,UAAW,OAAQ,UAAW,MAAO,OAAQ,OAAQ,SAAU,UAE3G3sD,MAAM,EACN0zF,UAAU,EACVt8F,OAAM,EACN65F,OAAQ,CACNxgB,SAAS,EACTr0B,YAAa,CAAC,OAAO,MAAM,SAAS,UAEtC7zB,MAAO,CAAC,OAAO,MAAM,UACrBwmE,SAAU,CACRpyD,MAAO,CAAC,GAAG,EAAE,IAAI,GACjBgjC,SAAU,CAAC,GAAG,EAAE,IAAI,GACpBswB,YAAY,EACZz1B,MAAO,CAAC,OAAO,SAAS,UAE1Bq2B,cAAe,CACbpgB,SAAS,EACTmiB,gBAAiB,CAAC,cAAc,UAAU,YAE5CrqC,WAAY,CACVkoB,SAAS,EACT/8C,KAAM,CAAC,EAAE,EAAE,GAAG,GACdnL,MAAO,CAAC,SAAU,WAEpBqrE,SAAU,CACRt3C,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfmwC,OAAO,EACP7tD,MAAO,CAAC,GAAG,EAAE,IAAI,GACjB0yB,SAAS,EACTy7B,YAAY,EACZriD,KAAK,CAGH+a,MAAO,CAAChE,KAAK,GAAGj3B,MAAM,KAExBmgB,MAAM,CAGJ8a,MAAO,CAAChE,KAAK,GAAGj3B,MAAM,MAG1BsrE,OAAQ,CACNpjB,SAAS,EACT+Z,OAAO,EACP/hD,KAAM,CACJ4mB,SAAS,EACT57B,SAAU,CAAC,YAAY,eAAe,WAAW,gBAEnDiV,MAAO,CACL2mB,SAAS,EACT57B,SAAU,CAAC,YAAY,eAAe,WAAW,iBAIrDm4B,YAAY,EACZnD,YAAY,EACZj0B,IAAK,GACLhjC,OAAQ,CACNkpD,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZwR,QAAY,OACZnS,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZwR,QAAY,OACZnS,KAAY,KAIhBw/B,OAAQ,GACRnrC,OAAQ,GACR4H,IAAK,GACL+wD,UAAW,GACX7N,cAAe,CAAC,EAAG,EAAG,GAAI,GAC1BpmD,IAAK,GACLo0D,UAAW,GACXpZ,UAAS,EACTiL,YAAa,CAAC,OAAQ,SAAU,OAChCsQ,iBAAiB,EACjB1S,iBAAiB,EACjBsC,iBAAiB,EACjBjC,eAAe,EACf9lB,MAAO,GACPoI,MAAO,OACPyU,UAAU,EACVyE,QAAS,CAAC,UAAW,SAAU,UAAW,IAC1CvE,QAAS,CAAC,SAAiB,GAAI,SAAiB,GAChDD,QAAS,CAAC,GAAI,GAAI,SAAiB,GACnCikD,OAAQ,ICjQZ,SAAS2C,GAASj4C,EAAWzV,EAAOm2B,EAAQx9C,0BAEpC+T,GAAcypC,IAAWA,aAAkB/0B,IAAW+0B,aAAkB2H,KAAa3H,aAAkBzwE,OAAQ,KAC/Gu0F,EAAgBthE,EACpBA,EAAUw9C,EACVA,EAAS8jB,EAKPthE,GAAWA,EAAQwtD,gBACrBh6E,QAAQH,KAAK,4HAGXm8C,EAAK3+C,UACJk9C,eAAiB,CACpB1c,MAAO,KACPC,IAAO,KAEPo3B,YAAY,EAEZxP,YAAa,CACXC,KAAM,SACNvQ,KAAM,UAGR3tB,OAAQA,GAERwe,MAAO,KACPC,OAAQ,KACRwtB,UAAW,KACXG,UAAW,WAERrnC,QAAUmpB,GAAKuN,WAAW,GAAI7lD,KAAKk9C,qBAGnCwL,QAAQuD,QAGRsI,WAAa,QAEb3b,KAAO,CACVwF,IAAKp+C,KAAKo+C,IACV5E,SAAUx5C,KAAKq1C,MACfwI,QAAS,CACPvH,GAAIP,UAAKO,WAAQt2C,MACjBu2C,IAAKR,UAAKQ,YAASv2C,MACnBygD,KAAM1K,UAAK0K,aAAUzgD,OAEvBo5C,YAAa,GACbd,KAAM,CACJkD,SAAUzF,KAAA4I,EAAGmyC,kBAAenyC,GAC5BoyC,eAAgBh7C,KAAA4I,EAAGqyC,wBAAqBryC,GACxC1C,OAAQlG,KAAA4I,EAAGsyC,gBAAatyC,GACxBuyC,aAAen7C,KAAA4I,EAAGwyC,sBAAmBxyC,UAKpCjF,MAAQ,IAAImD,GAAM78C,KAAK44C,WACvB2b,WAAWt3D,KAAK+C,KAAK05C,YACrBd,KAAKc,MAAQ15C,KAAK05C,WAGlB+O,SAAW,IAAIV,GAAS/nD,KAAK44C,WAC7B2b,WAAWt3D,KAAK+C,KAAKyoD,eAIrBiP,YAAc,IAAIgB,GAAY14D,KAAK44C,WACnC2b,WAAWt3D,KAAK+C,KAAK03D,kBAGrBysC,UAAY,IAAI1E,GAAUz/F,KAAK44C,WAE/B2b,WAAWt3D,KAAK+C,KAAKmkG,gBAErB1+B,UAAY,UACZsK,WAAa,UAGbz5B,GAAG,OAAO,SAAUkK,GACvB7B,EAAG8B,KAAK,QAAS9B,EAAGgV,mBAAmBnT,YAEpClK,GAAG,aAAa,SAAUkK,GAC7B7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,YAE1CpC,IAAI+C,KAAKiwC,cAAgB,SAAU5wC,GACtC7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,UAI1CoxC,gBAAiB,OACjBt7C,GAAG,WAAW,cACG,MAAhBqI,EAAG8mB,eACF9mB,EAAGizC,iBAAmBjzC,EAAGxvB,QAAQquB,eACpCmB,EAAGizC,gBAAiB,EACInyF,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,IAAkB,IACxChhC,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,QAC1CiZ,EAAQiF,EAAGkzC,mBAGbrxD,EAA4B/gC,MAApBk/C,EAAGxvB,QAAQqR,MAAqBme,EAAGxvB,QAAQqR,MAAQkZ,EAAMt3C,IACjEq+B,EAA4BhhC,MAApBk/C,EAAGxvB,QAAQsR,IAAqBke,EAAGxvB,QAAQsR,IAAQiZ,EAAMp0C,IACrEq5C,EAAGyX,UAAU51B,EAAOC,EAAK,CAACse,WAAW,SAErCJ,EAAGmzC,IAAI,CAAC/yC,WAAW,IAIlBJ,EAAGmT,kBAAoBnT,EAAGqT,yBAA4BrT,EAAGxvB,QAAQqR,OAAUme,EAAGxvB,QAAQsR,OACtFke,EAAGxvB,QAAQquB,cACdmB,EAAGmT,iBAAkB,EACrBnT,EAAGP,IAAI+C,KAAK3sB,MAAMyqB,WAAa,UAC/BN,EAAGP,IAAIsT,cAAcnqB,WAAWC,YAAYmX,EAAGP,IAAIsT,eAC/C/S,EAAGxvB,QAAQ0tD,2BACF,kBACFl+B,EAAGxvB,QAAQ0tD,0BACjB,QAML1tD,QACGqvB,WAAWrvB,GAIdw9C,QACGvX,UAAUuX,GAIbn2B,QACG2e,SAAS3e,QAIXub,UAIPmyC,GAAQ/nG,UAAY,IAAIs/C,GAExByoD,GAAQ/nG,UAAUqiD,WAAa,SAAUrvB,IAGpB,IADFwrD,GAAUqX,SAAS7iE,EAASqrD,KAE3C73E,QAAQq3C,IAAI,2DAA4D0gC,IAG1Ej/B,GAAKt/C,UAAUqiD,WAAWniD,KAAK2D,KAAMmvB,IAOvC+0E,GAAQ/nG,UAAUg5D,SAAW,SAAS3e,OAIhCy7C,EAHAmS,EAAiC,MAAlBpkG,KAAKylE,aAQtBwsB,EAJGz7C,EAIUe,GADNf,aAAiBoB,IAAWpB,aAAiB89B,GACrB99B,EAIA,IAAIoB,GAAQpB,IAP9B,KAWXx2C,KAAKylE,gBAEFA,UAAUptB,eAEZotB,UAAYwsB,OACZkS,WAAankG,KAAKmkG,UAAUhvC,SAAuB,MAAd88B,EAAqBA,EAAWz6C,MAAQ,MAE9E4sD,KACwB3kG,MAAtBO,KAAKmvB,QAAQqR,OAA0C/gC,MAApBO,KAAKmvB,QAAQsR,IAAkB,KAChED,EAA8B/gC,MAAtBO,KAAKmvB,QAAQqR,MAAqBxgC,KAAKmvB,QAAQqR,MAAQ,KAC/DC,EAA4BhhC,MAApBO,KAAKmvB,QAAQsR,IAAqBzgC,KAAKmvB,QAAQsR,IAAM,UAC5D21B,UAAU51B,EAAOC,EAAK,CAACse,WAAW,cAGlC+yC,IAAI,CAAC/yC,WAAW,KAS3BmlD,GAAQ/nG,UAAUi5D,UAAY,SAASuX,OAEjCslB,EAKFA,EAJGtlB,EAGIA,aAAkB/0B,IAAW+0B,aAAkB2H,GACzC3H,EAIA,IAAI/0B,GAAQ+0B,GAPZ,UAUVoD,WAAakiB,OACbkS,UAAU/uC,UAAU68B,IAU3BiS,GAAQ/nG,UAAUg8F,UAAY,SAASh8B,EAASvzB,EAAOC,eACtCppC,IAAXmpC,IAAuBA,EAAS,SACrBnpC,IAAXopC,IAAuBA,EAAS,SACGppC,IAAnCO,KAAKmkG,UAAUx3B,OAAOxQ,GACjBn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASg8B,UAAUvvD,EAAMC,GAG/C,sBAAyBszB,EAAU,KAS9C+nC,GAAQ/nG,UAAUkoG,eAAiB,SAASloC,eACH18D,IAAnCO,KAAKmkG,UAAUx3B,OAAOxQ,KAChBn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASb,eAAkE77D,IAAtDO,KAAKmkG,UAAUh1E,QAAQw9C,OAAO1tB,WAAWkd,IAA+E,GAArDn8D,KAAKmkG,UAAUh1E,QAAQw9C,OAAO1tB,WAAWkd,MAcnK+nC,GAAQ/nG,UAAU25D,aAAe,eAC3B1zD,EAAM,KACNkD,EAAM,SAGL,IAAI62D,KAAWn8D,KAAKmkG,UAAUx3B,UAC7B3sE,KAAKmkG,UAAUx3B,OAAOtvE,eAAe8+D,IACO,GAA1Cn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASb,YAC5B,IAAIx+D,EAAI,EAAGA,EAAIkD,KAAKmkG,UAAUx3B,OAAOxQ,GAASsJ,UAAUzoE,OAAQF,IAAK,KACpEi7C,EAAO/3C,KAAKmkG,UAAUx3B,OAAOxQ,GAASsJ,UAAU3oE,GAChDgF,EAAQw2C,GAAKpB,QAAQa,EAAK5tC,EAAG,QAAQ5M,UACzC6E,EAAa,MAAPA,GAAsBA,EAAMN,EAAdA,EAA8BM,EAClDkD,EAAa,MAAPA,GAAsBA,EAAMxD,EAAdA,EAA8BwD,QAMnD,CACLlD,IAAa,MAAPA,EAAe,IAAI1F,KAAK0F,GAAO,KACrCkD,IAAa,MAAPA,EAAe,IAAI5I,KAAK4I,GAAO,OAWzC4+F,GAAQ/nG,UAAUw3D,mBAAqB,SAAUnT,OAC3C2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDC,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChDj4C,EAAIg4C,EAAU7J,GAAKgsD,gBAAgBtkG,KAAKo+C,IAAI3E,iBAC5ClwC,EAAI64C,EAAU9J,GAAKisD,eAAevkG,KAAKo+C,IAAI3E,iBAC3C/4B,EAAO1gB,KAAKixF,QAAQ9mF,GAEpBwlD,EAAaH,GAAW+jC,qBAAqB/yC,GAE7C5Y,EAAU0Q,GAAKk7C,UAAUhzC,GACzBizC,EAAO,KACPn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKyoD,SAASrK,IAAI4J,aACrChoD,KAAKs0D,WAAahc,GAAKo7C,UAAU9rD,EAAS5nC,KAAKs0D,UAAUlW,IAAI4J,YADGyrC,EAAO,OAEvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU/D,UAAUhiD,IAAIuqB,QACrDrwB,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU9D,WAAWjiD,IAAIuqB,OADU8qB,EAAO,YAEvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU7D,WAAWliD,IAAIuqB,QACtDrwB,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU5D,YAAYniD,IAAIuqB,OADS8qB,EAAO,SAEzD,MAAd9jC,EAAoC8jC,EAAO,cAC3Cn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAK03D,YAAY7H,KAAuB4jC,EAAO,eACvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKo+C,IAAIY,UAA+By0C,EAAO,kBAE5E3xF,EAAQ,GACRs+F,EAAYpgG,KAAKmkG,UAAU/D,UAC3BC,EAAargG,KAAKmkG,UAAU9D,kBAC3BD,EAAU7lD,QAAUv6C,KAAKylE,UAAUzoE,OAAS,GAC/C8E,EAAM7E,KAAKmjG,EAAUlH,cAAc3vF,KAEhC82F,EAAW9lD,QAAUv6C,KAAKylE,UAAUzoE,OAAS,GAChD8E,EAAM7E,KAAKojG,EAAWnH,cAAc3vF,IAG/B,CACLi3C,MAAOA,EACPmP,WAAYA,EAAaA,EAAWxgC,QAAQiE,GAAK,KACjDqgE,KAAMA,EACNE,MAAOnzC,EAAM4D,SAAW5D,EAAM4D,SAASuvC,MAAQnzC,EAAMmzC,MACrDC,MAAOpzC,EAAM4D,SAAW5D,EAAM4D,SAASwvC,MAAQpzC,EAAMozC,MACrDzpF,EAAGA,EACHZ,EAAGA,EACHmX,KAAMA,EACN5e,MAAOA,IASXoiG,GAAQ/nG,UAAU44D,oBAAsB,kBAC/B,IAAIo3B,GAAansF,KAAMA,KAAKo+C,IAAI6N,UAAW8wB,KCtTpD,IAAMynB,GAAkBxM,QACjBt6F,OAAO8mG,QAER3Q,GAAW,CACfp4C,KAAAA,GACA6E,SAAAA,GACAzD,MAAAA,GACAx5C,MAAAA,GACAmiD,SAAAA,GAEA+O,WAAY,CACV/d,MAAO,CACLutB,KAAAA,GACA6F,eAAAA,GACArD,QAAAA,GACA6B,UAAAA,GACAM,UAAAA,IAGF5E,gBAAAA,GACAnrB,UAAAA,GACA+f,YAAAA,GACAlJ,WAAAA,GACA8mC,SAAAA,GACArC,UAAAA,GACA+F,WAAAA,GACA99B,MAAAA,GACAyS,QAAAA,GACA0rB,OAAAA,GACAoF,UAAAA,GACA13C,SAAAA"}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis.min.css b/core/src/main/resources/org/apache/spark/ui/static/vis.min.css
deleted file mode 100644
index 40d182cfde231..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis.min.css
+++ /dev/null
@@ -1 +0,0 @@
-.vis-background,.vis-labelset,.vis-timeline{overflow:hidden}.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}.vis [class*=span]{min-height:0;width:auto}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper::after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}.vis-panel,.vis-timeline{padding:0;box-sizing:border-box}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:rgba(0,0,0,0);width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8', GradientType=0 );border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2 0,#385380 100%);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(to bottom,#3876c2 0,#385380 100%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#3876c2', endColorstr='#385380', GradientType=0 );box-shadow:#111927 0 0 1px 0;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:0}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(to bottom,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#9d9d9d', endColorstr='#c8c8c8', GradientType=0 )}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8', GradientType=0 );border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:#fff solid 1px;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:0 0;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0);border-left-color:rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0);border-left-color:#f2faff;border-width:12px;margin-top:-12px}.vis-timeline{position:relative;border:1px solid #bfbfbf;margin:0}.vis-panel{position:absolute;margin:0}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-itemset,.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-itemset{padding:0;margin:0}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-item{position:absolute;color:#1A1A1A;border-color:#97B0F8;border-width:1px;background-color:#D5DDF6;display:inline-block}.vis-item.vis-point.vis-selected,.vis-item.vis-selected{background-color:#FFF785}.vis-item.vis-selected{border-color:#FFC200;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:0 0}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item .vis-delete,.vis-item .vis-delete-rtl{background:url(img/timeline/delete.png) center no-repeat;height:24px;top:-4px;cursor:pointer}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-delete{position:absolute;width:24px;right:-24px}.vis-item .vis-delete-rtl{position:absolute;width:24px;left:-24px}.vis-item.vis-range .vis-drag-left{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-current-time{background-color:#FF7F6E;width:2px;z-index:1}.vis-custom-time{background-color:#6E94FF;width:2px;cursor:move;z-index:1}div.vis-network div.vis-close,div.vis-network div.vis-edit-mode div.vis-button,div.vis-network div.vis-manipulation div.vis-button{cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-webkit-touch-callout:none;-khtml-user-select:none}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px rgba(154,154,154,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}div.vis-network div.vis-manipulation{border-width:0;border-bottom:1px;border-style:solid;border-color:#d6d9d8;background:#fff;background:-moz-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#fff),color-stop(48%,#fcfcfc),color-stop(50%,#fafafa),color-stop(100%,#fcfcfc));background:-webkit-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:-o-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:-ms-linear-gradient(top,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);background:linear-gradient(to bottom,#fff 0,#fcfcfc 48%,#fafafa 50%,#fcfcfc 100%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#ffffff', endColorstr='#fcfcfc', GradientType=0 );padding-top:4px;position:absolute;left:0;top:0;width:100%;height:28px}div.vis-network div.vis-edit-mode{position:absolute;left:0;top:5px;height:30px}div.vis-network div.vis-close{position:absolute;right:0;top:0;width:30px;height:30px;background-position:20px 3px;background-repeat:no-repeat;background-image:url(img/network/cross.png);user-select:none}div.vis-network div.vis-close:hover{opacity:.6}div.vis-network div.vis-edit-mode div.vis-button,div.vis-network div.vis-manipulation div.vis-button{float:left;font-family:verdana;font-size:12px;-moz-border-radius:15px;border-radius:15px;display:inline-block;background-position:0 0;background-repeat:no-repeat;height:24px;margin-left:10px;padding:0 8px;user-select:none}div.vis-network div.vis-manipulation div.vis-button:hover{box-shadow:1px 1px 8px rgba(0,0,0,.2)}div.vis-network div.vis-manipulation div.vis-button:active{box-shadow:1px 1px 8px rgba(0,0,0,.5)}div.vis-network div.vis-manipulation div.vis-button.vis-back{background-image:url(img/network/backIcon.png)}div.vis-network div.vis-manipulation div.vis-button.vis-none:hover{box-shadow:1px 1px 8px transparent;cursor:default}div.vis-network div.vis-manipulation div.vis-button.vis-none:active{box-shadow:1px 1px 8px transparent}div.vis-network div.vis-manipulation div.vis-button.vis-none{padding:0}div.vis-network div.vis-manipulation div.notification{margin:2px;font-weight:700}div.vis-network div.vis-manipulation div.vis-button.vis-add{background-image:url(img/network/addNodeIcon.png)}div.vis-network div.vis-edit-mode div.vis-button.vis-edit,div.vis-network div.vis-manipulation div.vis-button.vis-edit{background-image:url(img/network/editIcon.png)}div.vis-network div.vis-edit-mode div.vis-button.vis-edit.vis-edit-mode{background-color:#fcfcfc;border:1px solid #ccc}div.vis-network div.vis-manipulation div.vis-button.vis-connect{background-image:url(img/network/connectIcon.png)}div.vis-network div.vis-manipulation div.vis-button.vis-delete{background-image:url(img/network/deleteIcon.png)}div.vis-network div.vis-edit-mode div.vis-label,div.vis-network div.vis-manipulation div.vis-label{margin:0 0 0 23px;line-height:25px}div.vis-network div.vis-manipulation div.vis-separator-line{float:left;display:inline-block;width:1px;height:21px;background-color:#bdbdbd;margin:0 7px 0 15px}div.vis-network-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none}div.vis-network div.vis-navigation div.vis-button{width:34px;height:34px;-moz-border-radius:17px;border-radius:17px;position:absolute;display:inline-block;background-position:2px 2px;background-repeat:no-repeat;cursor:pointer;-webkit-touch-callout:none;-webkit-user-select:none;-khtml-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}div.vis-network div.vis-navigation div.vis-button:hover{box-shadow:0 0 3px 3px rgba(56,207,21,.3)}div.vis-network div.vis-navigation div.vis-button:active{box-shadow:0 0 1px 3px rgba(56,207,21,.95)}div.vis-network div.vis-navigation div.vis-button.vis-up{background-image:url(img/network/upArrow.png);bottom:50px;left:55px}div.vis-network div.vis-navigation div.vis-button.vis-down{background-image:url(img/network/downArrow.png);bottom:10px;left:55px}div.vis-network div.vis-navigation div.vis-button.vis-left{background-image:url(img/network/leftArrow.png);bottom:10px;left:15px}div.vis-network div.vis-navigation div.vis-button.vis-right{background-image:url(img/network/rightArrow.png);bottom:10px;left:95px}div.vis-network div.vis-navigation div.vis-button.vis-zoomIn{background-image:url(img/network/plus.png);bottom:10px;right:15px}div.vis-network div.vis-navigation div.vis-button.vis-zoomOut{background-image:url(img/network/minus.png);bottom:10px;right:55px}div.vis-network div.vis-navigation div.vis-button.vis-zoomExtends{background-image:url(img/network/zoomExtends.png);bottom:50px;right:15px}div.vis-color-picker{position:absolute;top:0;left:30px;margin-top:-140px;margin-left:30px;width:310px;height:444px;z-index:1;padding:10px;border-radius:15px;background-color:#fff;display:none;box-shadow:rgba(0,0,0,.5) 0 0 10px 0}div.vis-color-picker div.vis-arrow{position:absolute;top:147px;left:5px}div.vis-color-picker div.vis-arrow::after,div.vis-color-picker div.vis-arrow::before{right:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}div.vis-color-picker div.vis-arrow:after{border-color:rgba(255,255,255,0);border-right-color:#fff;border-width:30px;margin-top:-30px}div.vis-color-picker div.vis-color{position:absolute;width:289px;height:289px;cursor:pointer}div.vis-color-picker div.vis-brightness{position:absolute;top:313px}div.vis-color-picker div.vis-opacity{position:absolute;top:350px}div.vis-color-picker div.vis-selector{position:absolute;top:137px;left:137px;width:15px;height:15px;border-radius:15px;border:1px solid #fff;background:#4c4c4c;background:-moz-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#4c4c4c),color-stop(12%,#595959),color-stop(25%,#666),color-stop(39%,#474747),color-stop(50%,#2c2c2c),color-stop(51%,#000),color-stop(60%,#111),color-stop(76%,#2b2b2b),color-stop(91%,#1c1c1c),color-stop(100%,#131313));background:-webkit-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:-o-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:-ms-linear-gradient(top,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);background:linear-gradient(to bottom,#4c4c4c 0,#595959 12%,#666 25%,#474747 39%,#2c2c2c 50%,#000 51%,#111 60%,#2b2b2b 76%,#1c1c1c 91%,#131313 100%);filter:progid:DXImageTransform.Microsoft.gradient( startColorstr='#4c4c4c', endColorstr='#131313', GradientType=0 )}div.vis-color-picker div.vis-initial-color,div.vis-color-picker div.vis-new-color{width:140px;height:20px;top:380px;font-size:10px;color:rgba(0,0,0,.4);line-height:20px;position:absolute;vertical-align:middle}div.vis-color-picker div.vis-new-color{border:1px solid rgba(0,0,0,.1);border-radius:5px;left:159px;text-align:right;padding-right:2px}div.vis-color-picker div.vis-initial-color{border:1px solid rgba(0,0,0,.1);border-radius:5px;left:10px;text-align:left;padding-left:2px}div.vis-color-picker div.vis-label{position:absolute;width:300px;left:10px}div.vis-color-picker div.vis-label.vis-brightness{top:300px}div.vis-color-picker div.vis-label.vis-opacity{top:338px}div.vis-color-picker div.vis-button{position:absolute;width:68px;height:25px;border-radius:10px;vertical-align:middle;text-align:center;line-height:25px;top:410px;border:2px solid #d9d9d9;background-color:#f7f7f7;cursor:pointer}div.vis-color-picker div.vis-button.vis-cancel{left:5px}div.vis-color-picker div.vis-button.vis-load{left:82px}div.vis-color-picker div.vis-button.vis-apply{left:159px}div.vis-color-picker div.vis-button.vis-save{left:236px}div.vis-color-picker input.vis-range{width:290px;height:20px}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis.min.js b/core/src/main/resources/org/apache/spark/ui/static/vis.min.js
deleted file mode 100644
index 92b8ed75d85fc..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis.min.js
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * vis.js
- * https://github.com/almende/vis
- *
- * A dynamic, browser-based visualization library.
- *
- * @version 4.16.1
- * @date    2016-04-18
- *
- * @license
- * Copyright (C) 2011-2016 Almende B.V, http://almende.com
- *
- * Vis.js is dual licensed under both
- *
- * * The Apache 2.0 License
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * and
- *
- * * The MIT License
- *   http://opensource.org/licenses/MIT
- *
- * Vis.js may be distributed under either license.
- */
-"use strict";!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.vis=e():t.vis=e()}(this,function(){return function(t){function e(o){if(i[o])return i[o].exports;var n=i[o]={exports:{},id:o,loaded:!1};return t[o].call(n.exports,n,n.exports,e),n.loaded=!0,n.exports}var i={};return e.m=t,e.c=i,e.p="",e(0)}([function(t,e,i){var o=i(1);o.extend(e,i(7)),o.extend(e,i(24)),o.extend(e,i(60))},function(t,e,i){var o="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},n=i(2),s=i(6);e.isNumber=function(t){return t instanceof Number||"number"==typeof t},e.recursiveDOMDelete=function(t){if(t)for(;t.hasChildNodes()===!0;)e.recursiveDOMDelete(t.firstChild),t.removeChild(t.firstChild)},e.giveRange=function(t,e,i,o){if(e==t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)},e.isString=function(t){return t instanceof String||"string"==typeof t},e.isDate=function(t){if(t instanceof Date)return!0;if(e.isString(t)){var i=r.exec(t);if(i)return!0;if(!isNaN(Date.parse(t)))return!0}return!1},e.randomUUID=function(){return s.v4()},e.assignAllKeys=function(t,e){for(var i in t)t.hasOwnProperty(i)&&"object"!==o(t[i])&&(t[i]=e)},e.fillIfDefined=function(t,i){var n=arguments.length<=2||void 0===arguments[2]?!1:arguments[2];for(var s in t)void 0!==i[s]&&("object"!==o(i[s])?void 0!==i[s]&&null!==i[s]||void 0===t[s]||n!==!0?t[s]=i[s]:delete t[s]:"object"===o(t[s])&&e.fillIfDefined(t[s],i[s],n))},e.protoExtend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)t[n]=o[n]}return t},e.extend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)o.hasOwnProperty(n)&&(t[n]=o[n])}return t},e.selectiveExtend=function(t,e,i){if(!Array.isArray(t))throw new Error("Array with property names expected as first argument");for(var o=2;o<arguments.length;o++)for(var n=arguments[o],s=0;s<t.length;s++){var r=t[s];n.hasOwnProperty(r)&&(e[r]=n[r])}return e},e.selectiveDeepExtend=function(t,i,o){var n=arguments.length<=3||void 0===arguments[3]?!1:arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var s=2;s<arguments.length;s++)for(var r=arguments[s],a=0;a<t.length;a++){var h=t[a];if(r.hasOwnProperty(h))if(o[h]&&o[h].constructor===Object)void 0===i[h]&&(i[h]={}),i[h].constructor===Object?e.deepExtend(i[h],o[h],!1,n):null===o[h]&&void 0!==i[h]&&n===!0?delete i[h]:i[h]=o[h];else{if(Array.isArray(o[h]))throw new TypeError("Arrays are not supported by deepExtend");null===o[h]&&void 0!==i[h]&&n===!0?delete i[h]:i[h]=o[h]}}return i},e.selectiveNotDeepExtend=function(t,i,o){var n=arguments.length<=3||void 0===arguments[3]?!1:arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var s in o)if(o.hasOwnProperty(s)&&-1==t.indexOf(s))if(o[s]&&o[s].constructor===Object)void 0===i[s]&&(i[s]={}),i[s].constructor===Object?e.deepExtend(i[s],o[s]):null===o[s]&&void 0!==i[s]&&n===!0?delete i[s]:i[s]=o[s];else if(Array.isArray(o[s])){i[s]=[];for(var r=0;r<o[s].length;r++)i[s].push(o[s][r])}else null===o[s]&&void 0!==i[s]&&n===!0?delete i[s]:i[s]=o[s];return i},e.deepExtend=function(t,i,o,n){for(var s in i)if(i.hasOwnProperty(s)||o===!0)if(i[s]&&i[s].constructor===Object)void 0===t[s]&&(t[s]={}),t[s].constructor===Object?e.deepExtend(t[s],i[s],o):null===i[s]&&void 0!==t[s]&&n===!0?delete t[s]:t[s]=i[s];else if(Array.isArray(i[s])){t[s]=[];for(var r=0;r<i[s].length;r++)t[s].push(i[s][r])}else null===i[s]&&void 0!==t[s]&&n===!0?delete t[s]:t[s]=i[s];return t},e.equalArray=function(t,e){if(t.length!=e.length)return!1;for(var i=0,o=t.length;o>i;i++)if(t[i]!=e[i])return!1;return!0},e.convert=function(t,i){var o;if(void 0!==t){if(null===t)return null;if(!i)return t;if("string"!=typeof i&&!(i instanceof String))throw new Error("Type must be a string");switch(i){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(n.isMoment(t))return new Date(t.valueOf());if(e.isString(t))return o=r.exec(t),o?new Date(Number(o[1])):n(t).toDate();throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"Moment":if(e.isNumber(t))return n(t);if(t instanceof Date)return n(t.valueOf());if(n.isMoment(t))return n(t);if(e.isString(t))return o=r.exec(t),n(o?Number(o[1]):t);throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"ISODate":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(n.isMoment(t))return t.toDate().toISOString();if(e.isString(t))return o=r.exec(t),o?new Date(Number(o[1])).toISOString():new Date(t).toISOString();throw new Error("Cannot convert object of type "+e.getType(t)+" to type ISODate");case"ASPDate":if(e.isNumber(t))return"/Date("+t+")/";if(t instanceof Date)return"/Date("+t.valueOf()+")/";if(e.isString(t)){o=r.exec(t);var s;return s=o?new Date(Number(o[1])).valueOf():new Date(t).valueOf(),"/Date("+s+")/"}throw new Error("Cannot convert object of type "+e.getType(t)+" to type ASPDate");default:throw new Error('Unknown type "'+i+'"')}}};var r=/^\/?Date\((\-?\d+)/i;e.getType=function(t){var e="undefined"==typeof t?"undefined":o(t);return"object"==e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Array.isArray(t)?"Array":t instanceof Date?"Date":"Object":"number"==e?"Number":"boolean"==e?"Boolean":"string"==e?"String":void 0===e?"undefined":e},e.copyAndExtendArray=function(t,e){for(var i=[],o=0;o<t.length;o++)i.push(t[o]);return i.push(e),i},e.copyArray=function(t){for(var e=[],i=0;i<t.length;i++)e.push(t[i]);return e},e.getAbsoluteLeft=function(t){return t.getBoundingClientRect().left},e.getAbsoluteRight=function(t){return t.getBoundingClientRect().right},e.getAbsoluteTop=function(t){return t.getBoundingClientRect().top},e.addClassName=function(t,e){var i=t.className.split(" ");-1==i.indexOf(e)&&(i.push(e),t.className=i.join(" "))},e.removeClassName=function(t,e){var i=t.className.split(" "),o=i.indexOf(e);-1!=o&&(i.splice(o,1),t.className=i.join(" "))},e.forEach=function(t,e){var i,o;if(Array.isArray(t))for(i=0,o=t.length;o>i;i++)e(t[i],i,t);else for(i in t)t.hasOwnProperty(i)&&e(t[i],i,t)},e.toArray=function(t){var e=[];for(var i in t)t.hasOwnProperty(i)&&e.push(t[i]);return e},e.updateProperty=function(t,e,i){return t[e]!==i?(t[e]=i,!0):!1},e.throttle=function(t,e){var i=null,o=!1;return function n(){i?o=!0:(o=!1,t(),i=setTimeout(function(){i=null,o&&n()},e))}},e.addEventListener=function(t,e,i,o){t.addEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,o)):t.attachEvent("on"+e,i)},e.removeEventListener=function(t,e,i,o){t.removeEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,o)):t.detachEvent("on"+e,i)},e.preventDefault=function(t){t||(t=window.event),t.preventDefault?t.preventDefault():t.returnValue=!1},e.getTarget=function(t){t||(t=window.event);var e;return t.target?e=t.target:t.srcElement&&(e=t.srcElement),void 0!=e.nodeType&&3==e.nodeType&&(e=e.parentNode),e},e.hasParent=function(t,e){for(var i=t;i;){if(i===e)return!0;i=i.parentNode}return!1},e.option={},e.option.asBoolean=function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},e.option.asNumber=function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},e.option.asString=function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},e.option.asSize=function(t,i){return"function"==typeof t&&(t=t()),e.isString(t)?t:e.isNumber(t)?t+"px":i||null},e.option.asElement=function(t,e){return"function"==typeof t&&(t=t()),t||e||null},e.hexToRGB=function(t){var e=/^#?([a-f\d])([a-f\d])([a-f\d])$/i;t=t.replace(e,function(t,e,i,o){return e+e+i+i+o+o});var i=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(t);return i?{r:parseInt(i[1],16),g:parseInt(i[2],16),b:parseInt(i[3],16)}:null},e.overrideOpacity=function(t,i){if(-1!=t.indexOf("rgba"))return t;if(-1!=t.indexOf("rgb")){var o=t.substr(t.indexOf("(")+1).replace(")","").split(",");return"rgba("+o[0]+","+o[1]+","+o[2]+","+i+")"}var o=e.hexToRGB(t);return null==o?t:"rgba("+o.r+","+o.g+","+o.b+","+i+")"},e.RGBToHex=function(t,e,i){return"#"+((1<<24)+(t<<16)+(e<<8)+i).toString(16).slice(1)},e.parseColor=function(t){var i;if(e.isString(t)===!0){if(e.isValidRGB(t)===!0){var o=t.substr(4).substr(0,t.length-5).split(",").map(function(t){return parseInt(t)});t=e.RGBToHex(o[0],o[1],o[2])}if(e.isValidHex(t)===!0){var n=e.hexToHSV(t),s={h:n.h,s:.8*n.s,v:Math.min(1,1.02*n.v)},r={h:n.h,s:Math.min(1,1.25*n.s),v:.8*n.v},a=e.HSVToHex(r.h,r.s,r.v),h=e.HSVToHex(s.h,s.s,s.v);i={background:t,border:a,highlight:{background:h,border:a},hover:{background:h,border:a}}}else i={background:t,border:t,highlight:{background:t,border:t},hover:{background:t,border:t}}}else i={},i.background=t.background||void 0,i.border=t.border||void 0,e.isString(t.highlight)?i.highlight={border:t.highlight,background:t.highlight}:(i.highlight={},i.highlight.background=t.highlight&&t.highlight.background||void 0,i.highlight.border=t.highlight&&t.highlight.border||void 0),e.isString(t.hover)?i.hover={border:t.hover,background:t.hover}:(i.hover={},i.hover.background=t.hover&&t.hover.background||void 0,i.hover.border=t.hover&&t.hover.border||void 0);return i},e.RGBToHSV=function(t,e,i){t/=255,e/=255,i/=255;var o=Math.min(t,Math.min(e,i)),n=Math.max(t,Math.max(e,i));if(o==n)return{h:0,s:0,v:o};var s=t==o?e-i:i==o?t-e:i-t,r=t==o?3:i==o?1:5,a=60*(r-s/(n-o))/360,h=(n-o)/n,d=n;return{h:a,s:h,v:d}};var a={split:function(t){var e={};return t.split(";").forEach(function(t){if(""!=t.trim()){var i=t.split(":"),o=i[0].trim(),n=i[1].trim();e[o]=n}}),e},join:function(t){return Object.keys(t).map(function(e){return e+": "+t[e]}).join("; ")}};e.addCssText=function(t,i){var o=a.split(t.style.cssText),n=a.split(i),s=e.extend(o,n);t.style.cssText=a.join(s)},e.removeCssText=function(t,e){var i=a.split(t.style.cssText),o=a.split(e);for(var n in o)o.hasOwnProperty(n)&&delete i[n];t.style.cssText=a.join(i)},e.HSVToRGB=function(t,e,i){var o,n,s,r=Math.floor(6*t),a=6*t-r,h=i*(1-e),d=i*(1-a*e),l=i*(1-(1-a)*e);switch(r%6){case 0:o=i,n=l,s=h;break;case 1:o=d,n=i,s=h;break;case 2:o=h,n=i,s=l;break;case 3:o=h,n=d,s=i;break;case 4:o=l,n=h,s=i;break;case 5:o=i,n=h,s=d}return{r:Math.floor(255*o),g:Math.floor(255*n),b:Math.floor(255*s)}},e.HSVToHex=function(t,i,o){var n=e.HSVToRGB(t,i,o);return e.RGBToHex(n.r,n.g,n.b)},e.hexToHSV=function(t){var i=e.hexToRGB(t);return e.RGBToHSV(i.r,i.g,i.b)},e.isValidHex=function(t){var e=/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t);return e},e.isValidRGB=function(t){t=t.replace(" ","");var e=/rgb\((\d{1,3}),(\d{1,3}),(\d{1,3})\)/i.test(t);return e},e.isValidRGBA=function(t){t=t.replace(" ","");var e=/rgba\((\d{1,3}),(\d{1,3}),(\d{1,3}),(.{1,3})\)/i.test(t);return e},e.selectiveBridgeObject=function(t,i){if("object"==("undefined"==typeof i?"undefined":o(i))){for(var n=Object.create(i),s=0;s<t.length;s++)i.hasOwnProperty(t[s])&&"object"==o(i[t[s]])&&(n[t[s]]=e.bridgeObject(i[t[s]]));return n}return null},e.bridgeObject=function(t){if("object"==("undefined"==typeof t?"undefined":o(t))){var i=Object.create(t);for(var n in t)t.hasOwnProperty(n)&&"object"==o(t[n])&&(i[n]=e.bridgeObject(t[n]));return i}return null},e.insertSort=function(t,e){for(var i=0;i<t.length;i++){for(var o=t[i],n=i;n>0&&e(o,t[n-1])<0;n--)t[n]=t[n-1];t[n]=o}return t},e.mergeOptions=function(t,e,i){var o=(arguments.length<=3||void 0===arguments[3]?!1:arguments[3],arguments.length<=4||void 0===arguments[4]?{}:arguments[4]);if(null===e[i])t[i]=Object.create(o[i]);else if(void 0!==e[i])if("boolean"==typeof e[i])t[i].enabled=e[i];else{void 0===e[i].enabled&&(t[i].enabled=!0);for(var n in e[i])e[i].hasOwnProperty(n)&&(t[i][n]=e[i][n])}},e.binarySearchCustom=function(t,e,i,o){for(var n=1e4,s=0,r=0,a=t.length-1;a>=r&&n>s;){var h=Math.floor((r+a)/2),d=t[h],l=void 0===o?d[i]:d[i][o],c=e(l);if(0==c)return h;-1==c?r=h+1:a=h-1,s++}return-1},e.binarySearchValue=function(t,e,i,o,n){for(var s,r,a,h,d=1e4,l=0,c=0,u=t.length-1,n=void 0!=n?n:function(t,e){return t==e?0:e>t?-1:1};u>=c&&d>l;){if(h=Math.floor(.5*(u+c)),s=t[Math.max(0,h-1)][i],r=t[h][i],a=t[Math.min(t.length-1,h+1)][i],0==n(r,e))return h;if(n(s,e)<0&&n(r,e)>0)return"before"==o?Math.max(0,h-1):h;if(n(r,e)<0&&n(a,e)>0)return"before"==o?h:Math.min(t.length-1,h+1);n(r,e)<0?c=h+1:u=h-1,l++}return-1},e.easingFunctions={linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return.5>t?2*t*t:-1+(4-2*t)*t},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return.5>t?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return.5>t?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return.5>t?16*t*t*t*t*t:1+16*--t*t*t*t*t}}},function(t,e,i){t.exports="undefined"!=typeof window&&window.moment||i(3)},function(t,e,i){(function(t){!function(e,i){t.exports=i()}(this,function(){function e(){return ro.apply(null,arguments)}function i(t){ro=t}function o(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function n(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function s(t,e){var i,o=[];for(i=0;i<t.length;++i)o.push(e(t[i],i));return o}function r(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function a(t,e){for(var i in e)r(e,i)&&(t[i]=e[i]);return r(e,"toString")&&(t.toString=e.toString),r(e,"valueOf")&&(t.valueOf=e.valueOf),t}function h(t,e,i,o){return Lt(t,e,i,o,!0).utc()}function d(){return{empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null}}function l(t){return null==t._pf&&(t._pf=d()),t._pf}function c(t){if(null==t._isValid){var e=l(t),i=ao.call(e.parsedDateParts,function(t){return null!=t});t._isValid=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i),t._strict&&(t._isValid=t._isValid&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour)}return t._isValid}function u(t){var e=h(NaN);return null!=t?a(l(e),t):l(e).userInvalidated=!0,e}function p(t){return void 0===t}function f(t,e){var i,o,n;if(p(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),p(e._i)||(t._i=e._i),p(e._f)||(t._f=e._f),p(e._l)||(t._l=e._l),p(e._strict)||(t._strict=e._strict),p(e._tzm)||(t._tzm=e._tzm),p(e._isUTC)||(t._isUTC=e._isUTC),p(e._offset)||(t._offset=e._offset),p(e._pf)||(t._pf=l(e)),p(e._locale)||(t._locale=e._locale),ho.length>0)for(i in ho)o=ho[i],n=e[o],p(n)||(t[o]=n);return t}function m(t){f(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),lo===!1&&(lo=!0,e.updateOffset(this),lo=!1)}function v(t){return t instanceof m||null!=t&&null!=t._isAMomentObject}function g(t){return 0>t?Math.ceil(t):Math.floor(t)}function y(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=g(e)),i}function b(t,e,i){var o,n=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(o=0;n>o;o++)(i&&t[o]!==e[o]||!i&&y(t[o])!==y(e[o]))&&r++;return r+s}function w(t){e.suppressDeprecationWarnings===!1&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function _(t,i){var o=!0;return a(function(){return null!=e.deprecationHandler&&e.deprecationHandler(null,t),o&&(w(t+"\nArguments: "+Array.prototype.slice.call(arguments).join(", ")+"\n"+(new Error).stack),o=!1),i.apply(this,arguments)},i)}function x(t,i){null!=e.deprecationHandler&&e.deprecationHandler(t,i),co[t]||(w(i),co[t]=!0)}function k(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function O(t){return"[object Object]"===Object.prototype.toString.call(t)}function M(t){var e,i;for(i in t)e=t[i],k(e)?this[i]=e:this["_"+i]=e;this._config=t,this._ordinalParseLenient=new RegExp(this._ordinalParse.source+"|"+/\d{1,2}/.source)}function D(t,e){var i,o=a({},t);for(i in e)r(e,i)&&(O(t[i])&&O(e[i])?(o[i]={},a(o[i],t[i]),a(o[i],e[i])):null!=e[i]?o[i]=e[i]:delete o[i]);return o}function S(t){null!=t&&this.set(t)}function C(t){return t?t.toLowerCase().replace("_","-"):t}function T(t){for(var e,i,o,n,s=0;s<t.length;){for(n=C(t[s]).split("-"),e=n.length,i=C(t[s+1]),i=i?i.split("-"):null;e>0;){if(o=E(n.slice(0,e).join("-")))return o;if(i&&i.length>=e&&b(n,i,!0)>=e-1)break;e--}s++}return null}function E(e){var i=null;if(!mo[e]&&"undefined"!=typeof t&&t&&t.exports)try{i=po._abbr,!function(){var t=new Error('Cannot find module "./locale"');throw t.code="MODULE_NOT_FOUND",t}(),P(i)}catch(o){}return mo[e]}function P(t,e){var i;return t&&(i=p(e)?R(t):I(t,e),i&&(po=i)),po._abbr}function I(t,e){return null!==e?(e.abbr=t,null!=mo[t]?(x("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale"),e=D(mo[t]._config,e)):null!=e.parentLocale&&(null!=mo[e.parentLocale]?e=D(mo[e.parentLocale]._config,e):x("parentLocaleUndefined","specified parentLocale is not defined yet")),mo[t]=new S(e),P(t),mo[t]):(delete mo[t],null)}function N(t,e){if(null!=e){var i;null!=mo[t]&&(e=D(mo[t]._config,e)),i=new S(e),i.parentLocale=mo[t],mo[t]=i,P(t)}else null!=mo[t]&&(null!=mo[t].parentLocale?mo[t]=mo[t].parentLocale:null!=mo[t]&&delete mo[t]);return mo[t]}function R(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return po;if(!o(t)){if(e=E(t))return e;t=[t]}return T(t)}function z(){return uo(mo)}function L(t,e){var i=t.toLowerCase();vo[i]=vo[i+"s"]=vo[e]=t}function A(t){return"string"==typeof t?vo[t]||vo[t.toLowerCase()]:void 0}function B(t){var e,i,o={};for(i in t)r(t,i)&&(e=A(i),e&&(o[e]=t[i]));return o}function F(t,i){return function(o){return null!=o?(H(this,t,o),e.updateOffset(this,i),this):j(this,t)}}function j(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function H(t,e,i){t.isValid()&&t._d["set"+(t._isUTC?"UTC":"")+e](i)}function W(t,e){var i;if("object"==typeof t)for(i in t)this.set(i,t[i]);else if(t=A(t),k(this[t]))return this[t](e);return this}function Y(t,e,i){var o=""+Math.abs(t),n=e-o.length,s=t>=0;return(s?i?"+":"":"-")+Math.pow(10,Math.max(0,n)).toString().substr(1)+o}function G(t,e,i,o){var n=o;"string"==typeof o&&(n=function(){return this[o]()}),t&&(wo[t]=n),e&&(wo[e[0]]=function(){return Y(n.apply(this,arguments),e[1],e[2])}),i&&(wo[i]=function(){return this.localeData().ordinal(n.apply(this,arguments),t)})}function V(t){return t.match(/\[[\s\S]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"")}function U(t){var e,i,o=t.match(go);for(e=0,i=o.length;i>e;e++)wo[o[e]]?o[e]=wo[o[e]]:o[e]=V(o[e]);return function(e){var n,s="";for(n=0;i>n;n++)s+=o[n]instanceof Function?o[n].call(e,t):o[n];return s}}function q(t,e){return t.isValid()?(e=X(e,t.localeData()),bo[e]=bo[e]||U(e),bo[e](t)):t.localeData().invalidDate()}function X(t,e){function i(t){return e.longDateFormat(t)||t}var o=5;for(yo.lastIndex=0;o>=0&&yo.test(t);)t=t.replace(yo,i),yo.lastIndex=0,o-=1;return t}function Z(t,e,i){Bo[t]=k(e)?e:function(t,o){return t&&i?i:e}}function K(t,e){return r(Bo,t)?Bo[t](e._strict,e._locale):new RegExp(J(t))}function J(t){return Q(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(t,e,i,o,n){return e||i||o||n}))}function Q(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}function $(t,e){var i,o=e;for("string"==typeof t&&(t=[t]),"number"==typeof e&&(o=function(t,i){i[e]=y(t)}),i=0;i<t.length;i++)Fo[t[i]]=o}function tt(t,e){$(t,function(t,i,o,n){o._w=o._w||{},e(t,o._w,o,n)})}function et(t,e,i){null!=e&&r(Fo,t)&&Fo[t](e,i._a,i,t)}function it(t,e){return new Date(Date.UTC(t,e+1,0)).getUTCDate()}function ot(t,e){return o(this._months)?this._months[t.month()]:this._months[Zo.test(e)?"format":"standalone"][t.month()]}function nt(t,e){return o(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[Zo.test(e)?"format":"standalone"][t.month()]}function st(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],o=0;12>o;++o)s=h([2e3,o]),this._shortMonthsParse[o]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[o]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?(n=fo.call(this._shortMonthsParse,r),-1!==n?n:null):(n=fo.call(this._longMonthsParse,r),-1!==n?n:null):"MMM"===e?(n=fo.call(this._shortMonthsParse,r),-1!==n?n:(n=fo.call(this._longMonthsParse,r),-1!==n?n:null)):(n=fo.call(this._longMonthsParse,r),-1!==n?n:(n=fo.call(this._shortMonthsParse,r),-1!==n?n:null))}function rt(t,e,i){var o,n,s;if(this._monthsParseExact)return st.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),o=0;12>o;o++){if(n=h([2e3,o]),i&&!this._longMonthsParse[o]&&(this._longMonthsParse[o]=new RegExp("^"+this.months(n,"").replace(".","")+"$","i"),this._shortMonthsParse[o]=new RegExp("^"+this.monthsShort(n,"").replace(".","")+"$","i")),i||this._monthsParse[o]||(s="^"+this.months(n,"")+"|^"+this.monthsShort(n,""),this._monthsParse[o]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[o].test(t))return o;if(i&&"MMM"===e&&this._shortMonthsParse[o].test(t))return o;if(!i&&this._monthsParse[o].test(t))return o}}function at(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=y(e);else if(e=t.localeData().monthsParse(e),"number"!=typeof e)return t;return i=Math.min(t.date(),it(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function ht(t){return null!=t?(at(this,t),e.updateOffset(this,!0),this):j(this,"Month")}function dt(){return it(this.year(),this.month())}function lt(t){return this._monthsParseExact?(r(this,"_monthsRegex")||ut.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex}function ct(t){return this._monthsParseExact?(r(this,"_monthsRegex")||ut.call(this),t?this._monthsStrictRegex:this._monthsRegex):this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex}function ut(){function t(t,e){return e.length-t.length}var e,i,o=[],n=[],s=[];for(e=0;12>e;e++)i=h([2e3,e]),o.push(this.monthsShort(i,"")),n.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(o.sort(t),n.sort(t),s.sort(t),e=0;12>e;e++)o[e]=Q(o[e]),n[e]=Q(n[e]),s[e]=Q(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+n.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+o.join("|")+")","i")}function pt(t){var e,i=t._a;return i&&-2===l(t).overflow&&(e=i[Ho]<0||i[Ho]>11?Ho:i[Wo]<1||i[Wo]>it(i[jo],i[Ho])?Wo:i[Yo]<0||i[Yo]>24||24===i[Yo]&&(0!==i[Go]||0!==i[Vo]||0!==i[Uo])?Yo:i[Go]<0||i[Go]>59?Go:i[Vo]<0||i[Vo]>59?Vo:i[Uo]<0||i[Uo]>999?Uo:-1,l(t)._overflowDayOfYear&&(jo>e||e>Wo)&&(e=Wo),l(t)._overflowWeeks&&-1===e&&(e=qo),l(t)._overflowWeekday&&-1===e&&(e=Xo),l(t).overflow=e),t}function ft(t){var e,i,o,n,s,r,a=t._i,h=tn.exec(a)||en.exec(a);if(h){for(l(t).iso=!0,e=0,i=nn.length;i>e;e++)if(nn[e][1].exec(h[1])){n=nn[e][0],o=nn[e][2]!==!1;break}if(null==n)return void(t._isValid=!1);if(h[3]){for(e=0,i=sn.length;i>e;e++)if(sn[e][1].exec(h[3])){s=(h[2]||" ")+sn[e][0];break}if(null==s)return void(t._isValid=!1)}if(!o&&null!=s)return void(t._isValid=!1);if(h[4]){if(!on.exec(h[4]))return void(t._isValid=!1);r="Z"}t._f=n+(s||"")+(r||""),Tt(t)}else t._isValid=!1}function mt(t){var i=rn.exec(t._i);return null!==i?void(t._d=new Date(+i[1])):(ft(t),void(t._isValid===!1&&(delete t._isValid,e.createFromInputFallback(t))))}function vt(t,e,i,o,n,s,r){var a=new Date(t,e,i,o,n,s,r);return 100>t&&t>=0&&isFinite(a.getFullYear())&&a.setFullYear(t),a}function gt(t){var e=new Date(Date.UTC.apply(null,arguments));return 100>t&&t>=0&&isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t),e}function yt(t){return bt(t)?366:365}function bt(t){return t%4===0&&t%100!==0||t%400===0}function wt(){return bt(this.year())}function _t(t,e,i){var o=7+e-i,n=(7+gt(t,0,o).getUTCDay()-e)%7;return-n+o-1}function xt(t,e,i,o,n){var s,r,a=(7+i-o)%7,h=_t(t,o,n),d=1+7*(e-1)+a+h;return 0>=d?(s=t-1,r=yt(s)+d):d>yt(t)?(s=t+1,r=d-yt(t)):(s=t,r=d),{year:s,dayOfYear:r}}function kt(t,e,i){var o,n,s=_t(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return 1>r?(n=t.year()-1,o=r+Ot(n,e,i)):r>Ot(t.year(),e,i)?(o=r-Ot(t.year(),e,i),n=t.year()+1):(n=t.year(),o=r),{week:o,year:n}}function Ot(t,e,i){var o=_t(t,e,i),n=_t(t+1,e,i);return(yt(t)-o+n)/7}function Mt(t,e,i){return null!=t?t:null!=e?e:i}function Dt(t){var i=new Date(e.now());return t._useUTC?[i.getUTCFullYear(),i.getUTCMonth(),i.getUTCDate()]:[i.getFullYear(),i.getMonth(),i.getDate()]}function St(t){var e,i,o,n,s=[];if(!t._d){for(o=Dt(t),t._w&&null==t._a[Wo]&&null==t._a[Ho]&&Ct(t),t._dayOfYear&&(n=Mt(t._a[jo],o[jo]),t._dayOfYear>yt(n)&&(l(t)._overflowDayOfYear=!0),i=gt(n,0,t._dayOfYear),t._a[Ho]=i.getUTCMonth(),t._a[Wo]=i.getUTCDate()),e=0;3>e&&null==t._a[e];++e)t._a[e]=s[e]=o[e];for(;7>e;e++)t._a[e]=s[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[Yo]&&0===t._a[Go]&&0===t._a[Vo]&&0===t._a[Uo]&&(t._nextDay=!0,t._a[Yo]=0),t._d=(t._useUTC?gt:vt).apply(null,s),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[Yo]=24)}}function Ct(t){var e,i,o,n,s,r,a,h;e=t._w,null!=e.GG||null!=e.W||null!=e.E?(s=1,r=4,i=Mt(e.GG,t._a[jo],kt(At(),1,4).year),o=Mt(e.W,1),n=Mt(e.E,1),(1>n||n>7)&&(h=!0)):(s=t._locale._week.dow,r=t._locale._week.doy,i=Mt(e.gg,t._a[jo],kt(At(),s,r).year),o=Mt(e.w,1),null!=e.d?(n=e.d,(0>n||n>6)&&(h=!0)):null!=e.e?(n=e.e+s,(e.e<0||e.e>6)&&(h=!0)):n=s),1>o||o>Ot(i,s,r)?l(t)._overflowWeeks=!0:null!=h?l(t)._overflowWeekday=!0:(a=xt(i,o,n,s,r),t._a[jo]=a.year,t._dayOfYear=a.dayOfYear)}function Tt(t){if(t._f===e.ISO_8601)return void ft(t);t._a=[],l(t).empty=!0;var i,o,n,s,r,a=""+t._i,h=a.length,d=0;for(n=X(t._f,t._locale).match(go)||[],i=0;i<n.length;i++)s=n[i],o=(a.match(K(s,t))||[])[0],o&&(r=a.substr(0,a.indexOf(o)),r.length>0&&l(t).unusedInput.push(r),a=a.slice(a.indexOf(o)+o.length),d+=o.length),wo[s]?(o?l(t).empty=!1:l(t).unusedTokens.push(s),et(s,o,t)):t._strict&&!o&&l(t).unusedTokens.push(s);l(t).charsLeftOver=h-d,a.length>0&&l(t).unusedInput.push(a),l(t).bigHour===!0&&t._a[Yo]<=12&&t._a[Yo]>0&&(l(t).bigHour=void 0),l(t).parsedDateParts=t._a.slice(0),l(t).meridiem=t._meridiem,t._a[Yo]=Et(t._locale,t._a[Yo],t._meridiem),St(t),pt(t)}function Et(t,e,i){var o;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?(o=t.isPM(i),o&&12>e&&(e+=12),o||12!==e||(e=0),e):e}function Pt(t){var e,i,o,n,s;if(0===t._f.length)return l(t).invalidFormat=!0,void(t._d=new Date(NaN));for(n=0;n<t._f.length;n++)s=0,e=f({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[n],Tt(e),c(e)&&(s+=l(e).charsLeftOver,s+=10*l(e).unusedTokens.length,l(e).score=s,(null==o||o>s)&&(o=s,i=e));a(t,i||e)}function It(t){if(!t._d){var e=B(t._i);t._a=s([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],function(t){return t&&parseInt(t,10)}),St(t)}}function Nt(t){var e=new m(pt(Rt(t)));return e._nextDay&&(e.add(1,"d"),e._nextDay=void 0),e}function Rt(t){var e=t._i,i=t._f;return t._locale=t._locale||R(t._l),null===e||void 0===i&&""===e?u({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),v(e)?new m(pt(e)):(o(i)?Pt(t):i?Tt(t):n(e)?t._d=e:zt(t),c(t)||(t._d=null),t))}function zt(t){var i=t._i;void 0===i?t._d=new Date(e.now()):n(i)?t._d=new Date(i.valueOf()):"string"==typeof i?mt(t):o(i)?(t._a=s(i.slice(0),function(t){return parseInt(t,10)}),St(t)):"object"==typeof i?It(t):"number"==typeof i?t._d=new Date(i):e.createFromInputFallback(t)}function Lt(t,e,i,o,n){var s={};return"boolean"==typeof i&&(o=i,i=void 0),s._isAMomentObject=!0,s._useUTC=s._isUTC=n,s._l=i,s._i=t,s._f=e,s._strict=o,Nt(s)}function At(t,e,i,o){return Lt(t,e,i,o,!1)}function Bt(t,e){var i,n;if(1===e.length&&o(e[0])&&(e=e[0]),!e.length)return At();for(i=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](i)||(i=e[n]);return i}function Ft(){var t=[].slice.call(arguments,0);return Bt("isBefore",t)}function jt(){var t=[].slice.call(arguments,0);return Bt("isAfter",t)}function Ht(t){var e=B(t),i=e.year||0,o=e.quarter||0,n=e.month||0,s=e.week||0,r=e.day||0,a=e.hour||0,h=e.minute||0,d=e.second||0,l=e.millisecond||0;this._milliseconds=+l+1e3*d+6e4*h+1e3*a*60*60,this._days=+r+7*s,this._months=+n+3*o+12*i,this._data={},this._locale=R(),this._bubble()}function Wt(t){return t instanceof Ht}function Yt(t,e){G(t,0,0,function(){var t=this.utcOffset(),i="+";return 0>t&&(t=-t,i="-"),i+Y(~~(t/60),2)+e+Y(~~t%60,2)})}function Gt(t,e){var i=(e||"").match(t)||[],o=i[i.length-1]||[],n=(o+"").match(cn)||["-",0,0],s=+(60*n[1])+y(n[2]);return"+"===n[0]?s:-s}function Vt(t,i){var o,s;return i._isUTC?(o=i.clone(),s=(v(t)||n(t)?t.valueOf():At(t).valueOf())-o.valueOf(),o._d.setTime(o._d.valueOf()+s),e.updateOffset(o,!1),o):At(t).local()}function Ut(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function qt(t,i){var o,n=this._offset||0;return this.isValid()?null!=t?("string"==typeof t?t=Gt(zo,t):Math.abs(t)<16&&(t=60*t),!this._isUTC&&i&&(o=Ut(this)),this._offset=t,this._isUTC=!0,null!=o&&this.add(o,"m"),n!==t&&(!i||this._changeInProgress?le(this,ne(t-n,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,e.updateOffset(this,!0),this._changeInProgress=null)),this):this._isUTC?n:Ut(this):null!=t?this:NaN}function Xt(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()}function Zt(t){return this.utcOffset(0,t)}function Kt(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(Ut(this),"m")),this}function Jt(){return this._tzm?this.utcOffset(this._tzm):"string"==typeof this._i&&this.utcOffset(Gt(Ro,this._i)),this}function Qt(t){return this.isValid()?(t=t?At(t).utcOffset():0,(this.utcOffset()-t)%60===0):!1}function $t(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()}function te(){if(!p(this._isDSTShifted))return this._isDSTShifted;var t={};if(f(t,this),t=Rt(t),t._a){var e=t._isUTC?h(t._a):At(t._a);this._isDSTShifted=this.isValid()&&b(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}function ee(){return this.isValid()?!this._isUTC:!1}function ie(){return this.isValid()?this._isUTC:!1}function oe(){return this.isValid()?this._isUTC&&0===this._offset:!1}function ne(t,e){var i,o,n,s=t,a=null;return Wt(t)?s={ms:t._milliseconds,d:t._days,M:t._months}:"number"==typeof t?(s={},e?s[e]=t:s.milliseconds=t):(a=un.exec(t))?(i="-"===a[1]?-1:1,s={y:0,d:y(a[Wo])*i,h:y(a[Yo])*i,m:y(a[Go])*i,s:y(a[Vo])*i,ms:y(a[Uo])*i}):(a=pn.exec(t))?(i="-"===a[1]?-1:1,s={y:se(a[2],i),M:se(a[3],i),w:se(a[4],i),d:se(a[5],i),h:se(a[6],i),m:se(a[7],i),s:se(a[8],i)}):null==s?s={}:"object"==typeof s&&("from"in s||"to"in s)&&(n=ae(At(s.from),At(s.to)),s={},s.ms=n.milliseconds,s.M=n.months),o=new Ht(s),Wt(t)&&r(t,"_locale")&&(o._locale=t._locale),o}function se(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function re(t,e){var i={milliseconds:0,months:0};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,
-i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function ae(t,e){var i;return t.isValid()&&e.isValid()?(e=Vt(e,t),t.isBefore(e)?i=re(t,e):(i=re(e,t),i.milliseconds=-i.milliseconds,i.months=-i.months),i):{milliseconds:0,months:0}}function he(t){return 0>t?-1*Math.round(-1*t):Math.round(t)}function de(t,e){return function(i,o){var n,s;return null===o||isNaN(+o)||(x(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period)."),s=i,i=o,o=s),i="string"==typeof i?+i:i,n=ne(i,o),le(this,n,t),this}}function le(t,i,o,n){var s=i._milliseconds,r=he(i._days),a=he(i._months);t.isValid()&&(n=null==n?!0:n,s&&t._d.setTime(t._d.valueOf()+s*o),r&&H(t,"Date",j(t,"Date")+r*o),a&&at(t,j(t,"Month")+a*o),n&&e.updateOffset(t,r||a))}function ce(t,e){var i=t||At(),o=Vt(i,this).startOf("day"),n=this.diff(o,"days",!0),s=-6>n?"sameElse":-1>n?"lastWeek":0>n?"lastDay":1>n?"sameDay":2>n?"nextDay":7>n?"nextWeek":"sameElse",r=e&&(k(e[s])?e[s]():e[s]);return this.format(r||this.localeData().calendar(s,this,At(i)))}function ue(){return new m(this)}function pe(t,e){var i=v(t)?t:At(t);return this.isValid()&&i.isValid()?(e=A(p(e)?"millisecond":e),"millisecond"===e?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf()):!1}function fe(t,e){var i=v(t)?t:At(t);return this.isValid()&&i.isValid()?(e=A(p(e)?"millisecond":e),"millisecond"===e?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf()):!1}function me(t,e,i,o){return o=o||"()",("("===o[0]?this.isAfter(t,i):!this.isBefore(t,i))&&(")"===o[1]?this.isBefore(e,i):!this.isAfter(e,i))}function ve(t,e){var i,o=v(t)?t:At(t);return this.isValid()&&o.isValid()?(e=A(e||"millisecond"),"millisecond"===e?this.valueOf()===o.valueOf():(i=o.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf())):!1}function ge(t,e){return this.isSame(t,e)||this.isAfter(t,e)}function ye(t,e){return this.isSame(t,e)||this.isBefore(t,e)}function be(t,e,i){var o,n,s,r;return this.isValid()?(o=Vt(t,this),o.isValid()?(n=6e4*(o.utcOffset()-this.utcOffset()),e=A(e),"year"===e||"month"===e||"quarter"===e?(r=we(this,o),"quarter"===e?r/=3:"year"===e&&(r/=12)):(s=this-o,r="second"===e?s/1e3:"minute"===e?s/6e4:"hour"===e?s/36e5:"day"===e?(s-n)/864e5:"week"===e?(s-n)/6048e5:s),i?r:g(r)):NaN):NaN}function we(t,e){var i,o,n=12*(e.year()-t.year())+(e.month()-t.month()),s=t.clone().add(n,"months");return 0>e-s?(i=t.clone().add(n-1,"months"),o=(e-s)/(s-i)):(i=t.clone().add(n+1,"months"),o=(e-s)/(i-s)),-(n+o)||0}function _e(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")}function xe(){var t=this.clone().utc();return 0<t.year()&&t.year()<=9999?k(Date.prototype.toISOString)?this.toDate().toISOString():q(t,"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]"):q(t,"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]")}function ke(t){t||(t=this.isUtc()?e.defaultFormatUtc:e.defaultFormat);var i=q(this,t);return this.localeData().postformat(i)}function Oe(t,e){return this.isValid()&&(v(t)&&t.isValid()||At(t).isValid())?ne({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function Me(t){return this.from(At(),t)}function De(t,e){return this.isValid()&&(v(t)&&t.isValid()||At(t).isValid())?ne({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function Se(t){return this.to(At(),t)}function Ce(t){var e;return void 0===t?this._locale._abbr:(e=R(t),null!=e&&(this._locale=e),this)}function Te(){return this._locale}function Ee(t){switch(t=A(t)){case"year":this.month(0);case"quarter":case"month":this.date(1);case"week":case"isoWeek":case"day":case"date":this.hours(0);case"hour":this.minutes(0);case"minute":this.seconds(0);case"second":this.milliseconds(0)}return"week"===t&&this.weekday(0),"isoWeek"===t&&this.isoWeekday(1),"quarter"===t&&this.month(3*Math.floor(this.month()/3)),this}function Pe(t){return t=A(t),void 0===t||"millisecond"===t?this:("date"===t&&(t="day"),this.startOf(t).add(1,"isoWeek"===t?"week":t).subtract(1,"ms"))}function Ie(){return this._d.valueOf()-6e4*(this._offset||0)}function Ne(){return Math.floor(this.valueOf()/1e3)}function Re(){return this._offset?new Date(this.valueOf()):this._d}function ze(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]}function Le(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}}function Ae(){return this.isValid()?this.toISOString():null}function Be(){return c(this)}function Fe(){return a({},l(this))}function je(){return l(this).overflow}function He(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}}function We(t,e){G(0,[t,t.length],0,e)}function Ye(t){return qe.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)}function Ge(t){return qe.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)}function Ve(){return Ot(this.year(),1,4)}function Ue(){var t=this.localeData()._week;return Ot(this.year(),t.dow,t.doy)}function qe(t,e,i,o,n){var s;return null==t?kt(this,o,n).year:(s=Ot(t,o,n),e>s&&(e=s),Xe.call(this,t,e,i,o,n))}function Xe(t,e,i,o,n){var s=xt(t,e,i,o,n),r=gt(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}function Ze(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)}function Ke(t){return kt(t,this._week.dow,this._week.doy).week}function Je(){return this._week.dow}function Qe(){return this._week.doy}function $e(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")}function ti(t){var e=kt(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")}function ei(t,e){return"string"!=typeof t?t:isNaN(t)?(t=e.weekdaysParse(t),"number"==typeof t?t:null):parseInt(t,10)}function ii(t,e){return o(this._weekdays)?this._weekdays[t.day()]:this._weekdays[this._weekdays.isFormat.test(e)?"format":"standalone"][t.day()]}function oi(t){return this._weekdaysShort[t.day()]}function ni(t){return this._weekdaysMin[t.day()]}function si(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],o=0;7>o;++o)s=h([2e3,1]).day(o),this._minWeekdaysParse[o]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[o]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[o]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?(n=fo.call(this._weekdaysParse,r),-1!==n?n:null):"ddd"===e?(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:null):(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:null):"dddd"===e?(n=fo.call(this._weekdaysParse,r),-1!==n?n:(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:null))):"ddd"===e?(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:(n=fo.call(this._weekdaysParse,r),-1!==n?n:(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:null))):(n=fo.call(this._minWeekdaysParse,r),-1!==n?n:(n=fo.call(this._weekdaysParse,r),-1!==n?n:(n=fo.call(this._shortWeekdaysParse,r),-1!==n?n:null)))}function ri(t,e,i){var o,n,s;if(this._weekdaysParseExact)return si.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),o=0;7>o;o++){if(n=h([2e3,1]).day(o),i&&!this._fullWeekdaysParse[o]&&(this._fullWeekdaysParse[o]=new RegExp("^"+this.weekdays(n,"").replace(".",".?")+"$","i"),this._shortWeekdaysParse[o]=new RegExp("^"+this.weekdaysShort(n,"").replace(".",".?")+"$","i"),this._minWeekdaysParse[o]=new RegExp("^"+this.weekdaysMin(n,"").replace(".",".?")+"$","i")),this._weekdaysParse[o]||(s="^"+this.weekdays(n,"")+"|^"+this.weekdaysShort(n,"")+"|^"+this.weekdaysMin(n,""),this._weekdaysParse[o]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[o].test(t))return o;if(i&&"ddd"===e&&this._shortWeekdaysParse[o].test(t))return o;if(i&&"dd"===e&&this._minWeekdaysParse[o].test(t))return o;if(!i&&this._weekdaysParse[o].test(t))return o}}function ai(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=ei(t,this.localeData()),this.add(t-e,"d")):e}function hi(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")}function di(t){return this.isValid()?null==t?this.day()||7:this.day(this.day()%7?t:t-7):null!=t?this:NaN}function li(t){return this._weekdaysParseExact?(r(this,"_weekdaysRegex")||pi.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex}function ci(t){return this._weekdaysParseExact?(r(this,"_weekdaysRegex")||pi.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex}function ui(t){return this._weekdaysParseExact?(r(this,"_weekdaysRegex")||pi.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex}function pi(){function t(t,e){return e.length-t.length}var e,i,o,n,s,r=[],a=[],d=[],l=[];for(e=0;7>e;e++)i=h([2e3,1]).day(e),o=this.weekdaysMin(i,""),n=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(o),a.push(n),d.push(s),l.push(o),l.push(n),l.push(s);for(r.sort(t),a.sort(t),d.sort(t),l.sort(t),e=0;7>e;e++)a[e]=Q(a[e]),d[e]=Q(d[e]),l[e]=Q(l[e]);this._weekdaysRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+d.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function fi(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")}function mi(){return this.hours()%12||12}function vi(){return this.hours()||24}function gi(t,e){G(t,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)})}function yi(t,e){return e._meridiemParse}function bi(t){return"p"===(t+"").toLowerCase().charAt(0)}function wi(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"}function _i(t,e){e[Uo]=y(1e3*("0."+t))}function xi(){return this._isUTC?"UTC":""}function ki(){return this._isUTC?"Coordinated Universal Time":""}function Oi(t){return At(1e3*t)}function Mi(){return At.apply(null,arguments).parseZone()}function Di(t,e,i){var o=this._calendar[t];return k(o)?o.call(e,i):o}function Si(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,function(t){return t.slice(1)}),this._longDateFormat[t])}function Ci(){return this._invalidDate}function Ti(t){return this._ordinal.replace("%d",t)}function Ei(t){return t}function Pi(t,e,i,o){var n=this._relativeTime[i];return k(n)?n(t,e,i,o):n.replace(/%d/i,t)}function Ii(t,e){var i=this._relativeTime[t>0?"future":"past"];return k(i)?i(e):i.replace(/%s/i,e)}function Ni(t,e,i,o){var n=R(),s=h().set(o,e);return n[i](s,t)}function Ri(t,e,i){if("number"==typeof t&&(e=t,t=void 0),t=t||"",null!=e)return Ni(t,e,i,"month");var o,n=[];for(o=0;12>o;o++)n[o]=Ni(t,o,i,"month");return n}function zi(t,e,i,o){"boolean"==typeof t?("number"==typeof e&&(i=e,e=void 0),e=e||""):(e=t,i=e,t=!1,"number"==typeof e&&(i=e,e=void 0),e=e||"");var n=R(),s=t?n._week.dow:0;if(null!=i)return Ni(e,(i+s)%7,o,"day");var r,a=[];for(r=0;7>r;r++)a[r]=Ni(e,(r+s)%7,o,"day");return a}function Li(t,e){return Ri(t,e,"months")}function Ai(t,e){return Ri(t,e,"monthsShort")}function Bi(t,e,i){return zi(t,e,i,"weekdays")}function Fi(t,e,i){return zi(t,e,i,"weekdaysShort")}function ji(t,e,i){return zi(t,e,i,"weekdaysMin")}function Hi(){var t=this._data;return this._milliseconds=jn(this._milliseconds),this._days=jn(this._days),this._months=jn(this._months),t.milliseconds=jn(t.milliseconds),t.seconds=jn(t.seconds),t.minutes=jn(t.minutes),t.hours=jn(t.hours),t.months=jn(t.months),t.years=jn(t.years),this}function Wi(t,e,i,o){var n=ne(e,i);return t._milliseconds+=o*n._milliseconds,t._days+=o*n._days,t._months+=o*n._months,t._bubble()}function Yi(t,e){return Wi(this,t,e,1)}function Gi(t,e){return Wi(this,t,e,-1)}function Vi(t){return 0>t?Math.floor(t):Math.ceil(t)}function Ui(){var t,e,i,o,n,s=this._milliseconds,r=this._days,a=this._months,h=this._data;return s>=0&&r>=0&&a>=0||0>=s&&0>=r&&0>=a||(s+=864e5*Vi(Xi(a)+r),r=0,a=0),h.milliseconds=s%1e3,t=g(s/1e3),h.seconds=t%60,e=g(t/60),h.minutes=e%60,i=g(e/60),h.hours=i%24,r+=g(i/24),n=g(qi(r)),a+=n,r-=Vi(Xi(n)),o=g(a/12),a%=12,h.days=r,h.months=a,h.years=o,this}function qi(t){return 4800*t/146097}function Xi(t){return 146097*t/4800}function Zi(t){var e,i,o=this._milliseconds;if(t=A(t),"month"===t||"year"===t)return e=this._days+o/864e5,i=this._months+qi(e),"month"===t?i:i/12;switch(e=this._days+Math.round(Xi(this._months)),t){case"week":return e/7+o/6048e5;case"day":return e+o/864e5;case"hour":return 24*e+o/36e5;case"minute":return 1440*e+o/6e4;case"second":return 86400*e+o/1e3;case"millisecond":return Math.floor(864e5*e)+o;default:throw new Error("Unknown unit "+t)}}function Ki(){return this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*y(this._months/12)}function Ji(t){return function(){return this.as(t)}}function Qi(t){return t=A(t),this[t+"s"]()}function $i(t){return function(){return this._data[t]}}function to(){return g(this.days()/7)}function eo(t,e,i,o,n){return n.relativeTime(e||1,!!i,t,o)}function io(t,e,i){var o=ne(t).abs(),n=is(o.as("s")),s=is(o.as("m")),r=is(o.as("h")),a=is(o.as("d")),h=is(o.as("M")),d=is(o.as("y")),l=n<os.s&&["s",n]||1>=s&&["m"]||s<os.m&&["mm",s]||1>=r&&["h"]||r<os.h&&["hh",r]||1>=a&&["d"]||a<os.d&&["dd",a]||1>=h&&["M"]||h<os.M&&["MM",h]||1>=d&&["y"]||["yy",d];return l[2]=e,l[3]=+t>0,l[4]=i,eo.apply(null,l)}function oo(t,e){return void 0===os[t]?!1:void 0===e?os[t]:(os[t]=e,!0)}function no(t){var e=this.localeData(),i=io(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)}function so(){var t,e,i,o=ns(this._milliseconds)/1e3,n=ns(this._days),s=ns(this._months);t=g(o/60),e=g(t/60),o%=60,t%=60,i=g(s/12),s%=12;var r=i,a=s,h=n,d=e,l=t,c=o,u=this.asSeconds();return u?(0>u?"-":"")+"P"+(r?r+"Y":"")+(a?a+"M":"")+(h?h+"D":"")+(d||l||c?"T":"")+(d?d+"H":"")+(l?l+"M":"")+(c?c+"S":""):"P0D"}var ro,ao;ao=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,o=0;i>o;o++)if(o in e&&t.call(this,e[o],o,e))return!0;return!1};var ho=e.momentProperties=[],lo=!1,co={};e.suppressDeprecationWarnings=!1,e.deprecationHandler=null;var uo;uo=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)r(t,e)&&i.push(e);return i};var po,fo,mo={},vo={},go=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,yo=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,bo={},wo={},_o=/\d/,xo=/\d\d/,ko=/\d{3}/,Oo=/\d{4}/,Mo=/[+-]?\d{6}/,Do=/\d\d?/,So=/\d\d\d\d?/,Co=/\d\d\d\d\d\d?/,To=/\d{1,3}/,Eo=/\d{1,4}/,Po=/[+-]?\d{1,6}/,Io=/\d+/,No=/[+-]?\d+/,Ro=/Z|[+-]\d\d:?\d\d/gi,zo=/Z|[+-]\d\d(?::?\d\d)?/gi,Lo=/[+-]?\d+(\.\d{1,3})?/,Ao=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,Bo={},Fo={},jo=0,Ho=1,Wo=2,Yo=3,Go=4,Vo=5,Uo=6,qo=7,Xo=8;fo=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},G("M",["MM",2],"Mo",function(){return this.month()+1}),G("MMM",0,0,function(t){return this.localeData().monthsShort(this,t)}),G("MMMM",0,0,function(t){return this.localeData().months(this,t)}),L("month","M"),Z("M",Do),Z("MM",Do,xo),Z("MMM",function(t,e){return e.monthsShortRegex(t)}),Z("MMMM",function(t,e){return e.monthsRegex(t)}),$(["M","MM"],function(t,e){e[Ho]=y(t)-1}),$(["MMM","MMMM"],function(t,e,i,o){var n=i._locale.monthsParse(t,o,i._strict);null!=n?e[Ho]=n:l(i).invalidMonth=t});var Zo=/D[oD]?(\[[^\[\]]*\]|\s+)+MMMM?/,Ko="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),Jo="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),Qo=Ao,$o=Ao,tn=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?/,en=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?/,on=/Z|[+-]\d\d(?::?\d\d)?/,nn=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],sn=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],rn=/^\/?Date\((\-?\d+)/i;e.createFromInputFallback=_("moment construction falls back to js Date. This is discouraged and will be removed in upcoming major release. Please refer to https://github.com/moment/moment/issues/1407 for more info.",function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))}),G("Y",0,0,function(){var t=this.year();return 9999>=t?""+t:"+"+t}),G(0,["YY",2],0,function(){return this.year()%100}),G(0,["YYYY",4],0,"year"),G(0,["YYYYY",5],0,"year"),G(0,["YYYYYY",6,!0],0,"year"),L("year","y"),Z("Y",No),Z("YY",Do,xo),Z("YYYY",Eo,Oo),Z("YYYYY",Po,Mo),Z("YYYYYY",Po,Mo),$(["YYYYY","YYYYYY"],jo),$("YYYY",function(t,i){i[jo]=2===t.length?e.parseTwoDigitYear(t):y(t)}),$("YY",function(t,i){i[jo]=e.parseTwoDigitYear(t)}),$("Y",function(t,e){e[jo]=parseInt(t,10)}),e.parseTwoDigitYear=function(t){return y(t)+(y(t)>68?1900:2e3)};var an=F("FullYear",!0);e.ISO_8601=function(){};var hn=_("moment().min is deprecated, use moment.max instead. https://github.com/moment/moment/issues/1548",function(){var t=At.apply(null,arguments);return this.isValid()&&t.isValid()?this>t?this:t:u()}),dn=_("moment().max is deprecated, use moment.min instead. https://github.com/moment/moment/issues/1548",function(){var t=At.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:u()}),ln=function(){return Date.now?Date.now():+new Date};Yt("Z",":"),Yt("ZZ",""),Z("Z",zo),Z("ZZ",zo),$(["Z","ZZ"],function(t,e,i){i._useUTC=!0,i._tzm=Gt(zo,t)});var cn=/([\+\-]|\d\d)/gi;e.updateOffset=function(){};var un=/^(\-)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)\.?(\d{3})?\d*)?$/,pn=/^(-)?P(?:(-?[0-9,.]*)Y)?(?:(-?[0-9,.]*)M)?(?:(-?[0-9,.]*)W)?(?:(-?[0-9,.]*)D)?(?:T(?:(-?[0-9,.]*)H)?(?:(-?[0-9,.]*)M)?(?:(-?[0-9,.]*)S)?)?$/;ne.fn=Ht.prototype;var fn=de(1,"add"),mn=de(-1,"subtract");e.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",e.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var vn=_("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(t){return void 0===t?this.localeData():this.locale(t)});G(0,["gg",2],0,function(){return this.weekYear()%100}),G(0,["GG",2],0,function(){return this.isoWeekYear()%100}),We("gggg","weekYear"),We("ggggg","weekYear"),We("GGGG","isoWeekYear"),We("GGGGG","isoWeekYear"),L("weekYear","gg"),L("isoWeekYear","GG"),Z("G",No),Z("g",No),Z("GG",Do,xo),Z("gg",Do,xo),Z("GGGG",Eo,Oo),Z("gggg",Eo,Oo),Z("GGGGG",Po,Mo),Z("ggggg",Po,Mo),tt(["gggg","ggggg","GGGG","GGGGG"],function(t,e,i,o){e[o.substr(0,2)]=y(t)}),tt(["gg","GG"],function(t,i,o,n){i[n]=e.parseTwoDigitYear(t)}),G("Q",0,"Qo","quarter"),L("quarter","Q"),Z("Q",_o),$("Q",function(t,e){e[Ho]=3*(y(t)-1)}),G("w",["ww",2],"wo","week"),G("W",["WW",2],"Wo","isoWeek"),L("week","w"),L("isoWeek","W"),Z("w",Do),Z("ww",Do,xo),Z("W",Do),Z("WW",Do,xo),tt(["w","ww","W","WW"],function(t,e,i,o){e[o.substr(0,1)]=y(t)});var gn={dow:0,doy:6};G("D",["DD",2],"Do","date"),L("date","D"),Z("D",Do),Z("DD",Do,xo),Z("Do",function(t,e){return t?e._ordinalParse:e._ordinalParseLenient}),$(["D","DD"],Wo),$("Do",function(t,e){e[Wo]=y(t.match(Do)[0],10)});var yn=F("Date",!0);G("d",0,"do","day"),G("dd",0,0,function(t){return this.localeData().weekdaysMin(this,t)}),G("ddd",0,0,function(t){return this.localeData().weekdaysShort(this,t)}),G("dddd",0,0,function(t){return this.localeData().weekdays(this,t)}),G("e",0,0,"weekday"),G("E",0,0,"isoWeekday"),L("day","d"),L("weekday","e"),L("isoWeekday","E"),Z("d",Do),Z("e",Do),Z("E",Do),Z("dd",function(t,e){return e.weekdaysMinRegex(t)}),Z("ddd",function(t,e){return e.weekdaysShortRegex(t)}),Z("dddd",function(t,e){return e.weekdaysRegex(t)}),tt(["dd","ddd","dddd"],function(t,e,i,o){var n=i._locale.weekdaysParse(t,o,i._strict);null!=n?e.d=n:l(i).invalidWeekday=t}),tt(["d","e","E"],function(t,e,i,o){e[o]=y(t)});var bn="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),wn="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),_n="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),xn=Ao,kn=Ao,On=Ao;G("DDD",["DDDD",3],"DDDo","dayOfYear"),L("dayOfYear","DDD"),Z("DDD",To),Z("DDDD",ko),$(["DDD","DDDD"],function(t,e,i){i._dayOfYear=y(t)}),G("H",["HH",2],0,"hour"),G("h",["hh",2],0,mi),G("k",["kk",2],0,vi),G("hmm",0,0,function(){return""+mi.apply(this)+Y(this.minutes(),2)}),G("hmmss",0,0,function(){return""+mi.apply(this)+Y(this.minutes(),2)+Y(this.seconds(),2)}),G("Hmm",0,0,function(){return""+this.hours()+Y(this.minutes(),2)}),G("Hmmss",0,0,function(){return""+this.hours()+Y(this.minutes(),2)+Y(this.seconds(),2)}),gi("a",!0),gi("A",!1),L("hour","h"),Z("a",yi),Z("A",yi),Z("H",Do),Z("h",Do),Z("HH",Do,xo),Z("hh",Do,xo),Z("hmm",So),Z("hmmss",Co),Z("Hmm",So),Z("Hmmss",Co),$(["H","HH"],Yo),$(["a","A"],function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t}),$(["h","hh"],function(t,e,i){e[Yo]=y(t),l(i).bigHour=!0}),$("hmm",function(t,e,i){var o=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o)),l(i).bigHour=!0}),$("hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o,2)),e[Vo]=y(t.substr(n)),l(i).bigHour=!0}),$("Hmm",function(t,e,i){var o=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o))}),$("Hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[Yo]=y(t.substr(0,o)),e[Go]=y(t.substr(o,2)),e[Vo]=y(t.substr(n))});var Mn=/[ap]\.?m?\.?/i,Dn=F("Hours",!0);G("m",["mm",2],0,"minute"),L("minute","m"),Z("m",Do),Z("mm",Do,xo),$(["m","mm"],Go);var Sn=F("Minutes",!1);G("s",["ss",2],0,"second"),L("second","s"),Z("s",Do),Z("ss",Do,xo),$(["s","ss"],Vo);var Cn=F("Seconds",!1);G("S",0,0,function(){return~~(this.millisecond()/100)}),G(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),G(0,["SSS",3],0,"millisecond"),G(0,["SSSS",4],0,function(){return 10*this.millisecond()}),G(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),G(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),G(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),G(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),G(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),L("millisecond","ms"),Z("S",To,_o),Z("SS",To,xo),Z("SSS",To,ko);var Tn;for(Tn="SSSS";Tn.length<=9;Tn+="S")Z(Tn,Io);for(Tn="S";Tn.length<=9;Tn+="S")$(Tn,_i);var En=F("Milliseconds",!1);G("z",0,0,"zoneAbbr"),G("zz",0,0,"zoneName");var Pn=m.prototype;Pn.add=fn,Pn.calendar=ce,Pn.clone=ue,Pn.diff=be,Pn.endOf=Pe,Pn.format=ke,Pn.from=Oe,Pn.fromNow=Me,Pn.to=De,Pn.toNow=Se,Pn.get=W,Pn.invalidAt=je,Pn.isAfter=pe,Pn.isBefore=fe,Pn.isBetween=me,Pn.isSame=ve,Pn.isSameOrAfter=ge,Pn.isSameOrBefore=ye,Pn.isValid=Be,Pn.lang=vn,Pn.locale=Ce,Pn.localeData=Te,Pn.max=dn,Pn.min=hn,Pn.parsingFlags=Fe,Pn.set=W,Pn.startOf=Ee,Pn.subtract=mn,Pn.toArray=ze,Pn.toObject=Le,Pn.toDate=Re,Pn.toISOString=xe,Pn.toJSON=Ae,Pn.toString=_e,Pn.unix=Ne,Pn.valueOf=Ie,Pn.creationData=He,Pn.year=an,Pn.isLeapYear=wt,Pn.weekYear=Ye,Pn.isoWeekYear=Ge,Pn.quarter=Pn.quarters=Ze,Pn.month=ht,Pn.daysInMonth=dt,Pn.week=Pn.weeks=$e,Pn.isoWeek=Pn.isoWeeks=ti,Pn.weeksInYear=Ue,Pn.isoWeeksInYear=Ve,Pn.date=yn,Pn.day=Pn.days=ai,Pn.weekday=hi,Pn.isoWeekday=di,Pn.dayOfYear=fi,Pn.hour=Pn.hours=Dn,Pn.minute=Pn.minutes=Sn,Pn.second=Pn.seconds=Cn,Pn.millisecond=Pn.milliseconds=En,Pn.utcOffset=qt,Pn.utc=Zt,Pn.local=Kt,Pn.parseZone=Jt,Pn.hasAlignedHourOffset=Qt,Pn.isDST=$t,Pn.isDSTShifted=te,Pn.isLocal=ee,Pn.isUtcOffset=ie,Pn.isUtc=oe,Pn.isUTC=oe,Pn.zoneAbbr=xi,Pn.zoneName=ki,Pn.dates=_("dates accessor is deprecated. Use date instead.",yn),Pn.months=_("months accessor is deprecated. Use month instead",ht),Pn.years=_("years accessor is deprecated. Use year instead",an),Pn.zone=_("moment().zone is deprecated, use moment().utcOffset instead. https://github.com/moment/moment/issues/1779",Xt);var In=Pn,Nn={sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},Rn={LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},zn="Invalid date",Ln="%d",An=/\d{1,2}/,Bn={future:"in %s",past:"%s ago",s:"a few seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},Fn=S.prototype;Fn._calendar=Nn,Fn.calendar=Di,Fn._longDateFormat=Rn,Fn.longDateFormat=Si,Fn._invalidDate=zn,Fn.invalidDate=Ci,Fn._ordinal=Ln,Fn.ordinal=Ti,Fn._ordinalParse=An,Fn.preparse=Ei,Fn.postformat=Ei,Fn._relativeTime=Bn,Fn.relativeTime=Pi,Fn.pastFuture=Ii,Fn.set=M,Fn.months=ot,Fn._months=Ko,Fn.monthsShort=nt,Fn._monthsShort=Jo,Fn.monthsParse=rt,Fn._monthsRegex=$o,Fn.monthsRegex=ct,Fn._monthsShortRegex=Qo,Fn.monthsShortRegex=lt,Fn.week=Ke,Fn._week=gn,Fn.firstDayOfYear=Qe,Fn.firstDayOfWeek=Je,Fn.weekdays=ii,Fn._weekdays=bn,Fn.weekdaysMin=ni,Fn._weekdaysMin=_n,Fn.weekdaysShort=oi,Fn._weekdaysShort=wn,Fn.weekdaysParse=ri,Fn._weekdaysRegex=xn,Fn.weekdaysRegex=li,Fn._weekdaysShortRegex=kn,Fn.weekdaysShortRegex=ci,Fn._weekdaysMinRegex=On,Fn.weekdaysMinRegex=ui,Fn.isPM=bi,Fn._meridiemParse=Mn,Fn.meridiem=wi,P("en",{ordinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10,i=1===y(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th";return t+i}}),e.lang=_("moment.lang is deprecated. Use moment.locale instead.",P),e.langData=_("moment.langData is deprecated. Use moment.localeData instead.",R);var jn=Math.abs,Hn=Ji("ms"),Wn=Ji("s"),Yn=Ji("m"),Gn=Ji("h"),Vn=Ji("d"),Un=Ji("w"),qn=Ji("M"),Xn=Ji("y"),Zn=$i("milliseconds"),Kn=$i("seconds"),Jn=$i("minutes"),Qn=$i("hours"),$n=$i("days"),ts=$i("months"),es=$i("years"),is=Math.round,os={s:45,m:45,h:22,d:26,M:11},ns=Math.abs,ss=Ht.prototype;ss.abs=Hi,ss.add=Yi,ss.subtract=Gi,ss.as=Zi,ss.asMilliseconds=Hn,ss.asSeconds=Wn,ss.asMinutes=Yn,ss.asHours=Gn,ss.asDays=Vn,ss.asWeeks=Un,ss.asMonths=qn,ss.asYears=Xn,ss.valueOf=Ki,ss._bubble=Ui,ss.get=Qi,ss.milliseconds=Zn,ss.seconds=Kn,ss.minutes=Jn,ss.hours=Qn,ss.days=$n,ss.weeks=to,ss.months=ts,ss.years=es,ss.humanize=no,ss.toISOString=so,ss.toString=so,ss.toJSON=so,ss.locale=Ce,ss.localeData=Te,ss.toIsoString=_("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",so),ss.lang=vn,G("X",0,0,"unix"),G("x",0,0,"valueOf"),Z("x",No),Z("X",Lo),$("X",function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))}),$("x",function(t,e,i){i._d=new Date(y(t))}),e.version="2.13.0",i(At),e.fn=In,e.min=Ft,e.max=jt,e.now=ln,e.utc=h,e.unix=Oi,e.months=Li,e.isDate=n,e.locale=P,e.invalid=u,e.duration=ne,e.isMoment=v,e.weekdays=Bi,e.parseZone=Mi,e.localeData=R,e.isDuration=Wt,e.monthsShort=Ai,e.weekdaysMin=ji,e.defineLocale=I,e.updateLocale=N,e.locales=z,e.weekdaysShort=Fi,e.normalizeUnits=A,e.relativeTimeThreshold=oo,e.prototype=In;var rs=e;return rs})}).call(e,i(4)(t))},function(t,e){t.exports=function(t){return t.webpackPolyfill||(t.deprecate=function(){},t.paths=[],t.children=[],t.webpackPolyfill=1),t}},function(t,e){function i(t){throw new Error("Cannot find module '"+t+"'.")}i.keys=function(){return[]},i.resolve=i,t.exports=i,i.id=5},function(t,e){(function(e){function i(t,e,i){var o=e&&i||0,n=0;for(e=e||[],t.toLowerCase().replace(/[0-9a-f]{2}/g,function(t){16>n&&(e[o+n++]=c[t])});16>n;)e[o+n++]=0;return e}function o(t,e){var i=e||0,o=l;return o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]}function n(t,e,i){var n=e&&i||0,s=e||[];t=t||{};var r=void 0!==t.clockseq?t.clockseq:m,a=void 0!==t.msecs?t.msecs:(new Date).getTime(),h=void 0!==t.nsecs?t.nsecs:g+1,d=a-v+(h-g)/1e4;if(0>d&&void 0===t.clockseq&&(r=r+1&16383),(0>d||a>v)&&void 0===t.nsecs&&(h=0),h>=1e4)throw new Error("uuid.v1(): Can't create more than 10M uuids/sec");v=a,g=h,m=r,a+=122192928e5;var l=(1e4*(268435455&a)+h)%4294967296;s[n++]=l>>>24&255,s[n++]=l>>>16&255,s[n++]=l>>>8&255,s[n++]=255&l;var c=a/4294967296*1e4&268435455;s[n++]=c>>>8&255,s[n++]=255&c,s[n++]=c>>>24&15|16,s[n++]=c>>>16&255,s[n++]=r>>>8|128,s[n++]=255&r;for(var u=t.node||f,p=0;6>p;p++)s[n+p]=u[p];return e?e:o(s)}function s(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"==t?new Array(16):null,t=null),t=t||{};var s=t.random||(t.rng||r)();if(s[6]=15&s[6]|64,s[8]=63&s[8]|128,e)for(var a=0;16>a;a++)e[n+a]=s[a];return e||o(s)}var r,a="undefined"!=typeof window?window:"undefined"!=typeof e?e:null;if(a&&a.crypto&&crypto.getRandomValues){var h=new Uint8Array(16);r=function(){return crypto.getRandomValues(h),h}}if(!r){var d=new Array(16);r=function(){for(var t,e=0;16>e;e++)0===(3&e)&&(t=4294967296*Math.random()),d[e]=t>>>((3&e)<<3)&255;return d}}for(var l=[],c={},u=0;256>u;u++)l[u]=(u+256).toString(16).substr(1),c[l[u]]=u;var p=r(),f=[1|p[0],p[1],p[2],p[3],p[4],p[5]],m=16383&(p[6]<<8|p[7]),v=0,g=0,y=s;y.v1=n,y.v4=s,y.parse=i,y.unparse=o,t.exports=y}).call(e,function(){return this}())},function(t,e,i){e.util=i(1),e.DOMutil=i(8),e.DataSet=i(9),e.DataView=i(11),e.Queue=i(10),e.Graph3d=i(12),e.graph3d={Camera:i(16),Filter:i(17),Point2d:i(15),Point3d:i(14),Slider:i(18),StepNumber:i(19)},e.moment=i(2),e.Hammer=i(20),e.keycharm=i(23)},function(t,e){e.prepareElements=function(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])},e.cleanupElements=function(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}},e.resetElements=function(t){e.prepareElements(t),e.cleanupElements(t),e.prepareElements(t)},e.getSVGElement=function(t,e,i){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(o)):(o=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(o)),e[t].used.push(o),o},e.getDOMElement=function(t,e,i,o){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElement(t),void 0!==o?i.insertBefore(n,o):i.appendChild(n)):(n=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==o?i.insertBefore(n,o):i.appendChild(n)),e[t].used.push(n),n},e.drawPoint=function(t,i,o,n,s,r){var a;if("circle"==o.style?(a=e.getSVGElement("circle",n,s),a.setAttributeNS(null,"cx",t),a.setAttributeNS(null,"cy",i),a.setAttributeNS(null,"r",.5*o.size)):(a=e.getSVGElement("rect",n,s),a.setAttributeNS(null,"x",t-.5*o.size),a.setAttributeNS(null,"y",i-.5*o.size),a.setAttributeNS(null,"width",o.size),a.setAttributeNS(null,"height",o.size)),void 0!==o.styles&&a.setAttributeNS(null,"style",o.styles),a.setAttributeNS(null,"class",o.className+" vis-point"),r){var h=e.getSVGElement("text",n,s);
-r.xOffset&&(t+=r.xOffset),r.yOffset&&(i+=r.yOffset),r.content&&(h.textContent=r.content),r.className&&h.setAttributeNS(null,"class",r.className+" vis-label"),h.setAttributeNS(null,"x",t),h.setAttributeNS(null,"y",i)}return a},e.drawBar=function(t,i,o,n,s,r,a,h){if(0!=n){0>n&&(n*=-1,i-=n);var d=e.getSVGElement("rect",r,a);d.setAttributeNS(null,"x",t-.5*o),d.setAttributeNS(null,"y",i),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",n),d.setAttributeNS(null,"class",s),h&&d.setAttributeNS(null,"style",h)}}},function(t,e,i){function o(t,e){if(t&&!Array.isArray(t)&&(e=t,t=null),this._options=e||{},this._data={},this.length=0,this._fieldId=this._options.fieldId||"id",this._type={},this._options.type)for(var i=Object.keys(this._options.type),o=0,n=i.length;n>o;o++){var s=i[o],r=this._options.type[s];"Date"==r||"ISODate"==r||"ASPDate"==r?this._type[s]="Date":this._type[s]=r}if(this._options.convert)throw new Error('Option "convert" is deprecated. Use "type" instead.');this._subscribers={},t&&this.add(t),this.setOptions(e)}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=i(10);o.prototype.setOptions=function(t){t&&void 0!==t.queue&&(t.queue===!1?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=r.extend(this,{replace:["add","update","remove"]})),"object"===n(t.queue)&&this._queue.setOptions(t.queue)))},o.prototype.on=function(t,e){var i=this._subscribers[t];i||(i=[],this._subscribers[t]=i),i.push({callback:e})},o.prototype.subscribe=function(){throw new Error("DataSet.subscribe is deprecated. Use DataSet.on instead.")},o.prototype.off=function(t,e){var i=this._subscribers[t];i&&(this._subscribers[t]=i.filter(function(t){return t.callback!=e}))},o.prototype.unsubscribe=function(){throw new Error("DataSet.unsubscribe is deprecated. Use DataSet.off instead.")},o.prototype._trigger=function(t,e,i){if("*"==t)throw new Error("Cannot trigger event *");var o=[];t in this._subscribers&&(o=o.concat(this._subscribers[t])),"*"in this._subscribers&&(o=o.concat(this._subscribers["*"]));for(var n=0,s=o.length;s>n;n++){var r=o[n];r.callback&&r.callback(t,e,i||null)}},o.prototype.add=function(t,e){var i,o=[],n=this;if(Array.isArray(t))for(var s=0,r=t.length;r>s;s++)i=n._addItem(t[s]),o.push(i);else{if(!(t instanceof Object))throw new Error("Unknown dataType");i=n._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o},o.prototype.update=function(t,e){var i=[],o=[],n=[],r=[],a=this,h=a._fieldId,d=function(t){var e=t[h];if(a._data[e]){var d=s.extend({},a._data[e]);e=a._updateItem(t),o.push(e),r.push(t),n.push(d)}else e=a._addItem(t),i.push(e)};if(Array.isArray(t))for(var l=0,c=t.length;c>l;l++)t[l]instanceof Object?d(t[l]):console.warn("Ignoring input item, which is not an object at index "+l);else{if(!(t instanceof Object))throw new Error("Unknown dataType");d(t)}if(i.length&&this._trigger("add",{items:i},e),o.length){var u={items:o,oldData:n,data:r};this._trigger("update",u,e)}return i.concat(o)},o.prototype.get=function(t){var e,i,o,n=this,r=s.getType(arguments[0]);"String"==r||"Number"==r?(e=arguments[0],o=arguments[1]):"Array"==r?(i=arguments[0],o=arguments[1]):o=arguments[0];var a;if(o&&o.returnType){var h=["Array","Object"];a=-1==h.indexOf(o.returnType)?"Array":o.returnType}else a="Array";var d,l,c,u,p,f=o&&o.type||this._options.type,m=o&&o.filter,v=[];if(void 0!=e)d=n._getItem(e,f),d&&m&&!m(d)&&(d=null);else if(void 0!=i)for(u=0,p=i.length;p>u;u++)d=n._getItem(i[u],f),m&&!m(d)||v.push(d);else for(l=Object.keys(this._data),u=0,p=l.length;p>u;u++)c=l[u],d=n._getItem(c,f),m&&!m(d)||v.push(d);if(o&&o.order&&void 0==e&&this._sort(v,o.order),o&&o.fields){var g=o.fields;if(void 0!=e)d=this._filterFields(d,g);else for(u=0,p=v.length;p>u;u++)v[u]=this._filterFields(v[u],g)}if("Object"==a){var y,b={};for(u=0,p=v.length;p>u;u++)y=v[u],b[y.id]=y;return b}return void 0!=e?d:v},o.prototype.getIds=function(t){var e,i,o,n,s,r=this._data,a=t&&t.filter,h=t&&t.order,d=t&&t.type||this._options.type,l=Object.keys(r),c=[];if(a)if(h){for(s=[],e=0,i=l.length;i>e;e++)o=l[e],n=this._getItem(o,d),a(n)&&s.push(n);for(this._sort(s,h),e=0,i=s.length;i>e;e++)c.push(s[e][this._fieldId])}else for(e=0,i=l.length;i>e;e++)o=l[e],n=this._getItem(o,d),a(n)&&c.push(n[this._fieldId]);else if(h){for(s=[],e=0,i=l.length;i>e;e++)o=l[e],s.push(r[o]);for(this._sort(s,h),e=0,i=s.length;i>e;e++)c.push(s[e][this._fieldId])}else for(e=0,i=l.length;i>e;e++)o=l[e],n=r[o],c.push(n[this._fieldId]);return c},o.prototype.getDataSet=function(){return this},o.prototype.forEach=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=this._data,d=Object.keys(h);if(e&&e.order){var l=this.get(e);for(i=0,o=l.length;o>i;i++)n=l[i],s=n[this._fieldId],t(n,s)}else for(i=0,o=d.length;o>i;i++)s=d[i],n=this._getItem(s,a),r&&!r(n)||t(n,s)},o.prototype.map=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=[],d=this._data,l=Object.keys(d);for(i=0,o=l.length;o>i;i++)n=l[i],s=this._getItem(n,a),r&&!r(s)||h.push(t(s,n));return e&&e.order&&this._sort(h,e.order),h},o.prototype._filterFields=function(t,e){if(!t)return t;var i,o,n={},s=Object.keys(t),r=s.length;if(Array.isArray(e))for(i=0;r>i;i++)o=s[i],-1!=e.indexOf(o)&&(n[o]=t[o]);else for(i=0;r>i;i++)o=s[i],e.hasOwnProperty(o)&&(n[e[o]]=t[o]);return n},o.prototype._sort=function(t,e){if(s.isString(e)){var i=e;t.sort(function(t,e){var o=t[i],n=e[i];return o>n?1:n>o?-1:0})}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");t.sort(e)}},o.prototype.remove=function(t,e){var i,o,n,s=[];if(Array.isArray(t))for(i=0,o=t.length;o>i;i++)n=this._remove(t[i]),null!=n&&s.push(n);else n=this._remove(t),null!=n&&s.push(n);return s.length&&this._trigger("remove",{items:s},e),s},o.prototype._remove=function(t){if(s.isNumber(t)||s.isString(t)){if(this._data[t])return delete this._data[t],this.length--,t}else if(t instanceof Object){var e=t[this._fieldId];if(void 0!==e&&this._data[e])return delete this._data[e],this.length--,e}return null},o.prototype.clear=function(t){var e=Object.keys(this._data);return this._data={},this.length=0,this._trigger("remove",{items:e},t),e},o.prototype.max=function(t){var e,i,o=this._data,n=Object.keys(o),s=null,r=null;for(e=0,i=n.length;i>e;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d>r)&&(s=h,r=d)}return s},o.prototype.min=function(t){var e,i,o=this._data,n=Object.keys(o),s=null,r=null;for(e=0,i=n.length;i>e;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||r>d)&&(s=h,r=d)}return s},o.prototype.distinct=function(t){var e,i,o,n=this._data,r=Object.keys(n),a=[],h=this._options.type&&this._options.type[t]||null,d=0;for(e=0,o=r.length;o>e;e++){var l=r[e],c=n[l],u=c[t],p=!1;for(i=0;d>i;i++)if(a[i]==u){p=!0;break}p||void 0===u||(a[d]=u,d++)}if(h)for(e=0,o=a.length;o>e;e++)a[e]=s.convert(a[e],h);return a},o.prototype._addItem=function(t){var e=t[this._fieldId];if(void 0!=e){if(this._data[e])throw new Error("Cannot add item: item with id "+e+" already exists")}else e=s.randomUUID(),t[this._fieldId]=e;var i,o,n={},r=Object.keys(t);for(i=0,o=r.length;o>i;i++){var a=r[i],h=this._type[a];n[a]=s.convert(t[a],h)}return this._data[e]=n,this.length++,e},o.prototype._getItem=function(t,e){var i,o,n,r,a=this._data[t];if(!a)return null;var h={},d=Object.keys(a);if(e)for(n=0,r=d.length;r>n;n++)i=d[n],o=a[i],h[i]=s.convert(o,e[i]);else for(n=0,r=d.length;r>n;n++)i=d[n],o=a[i],h[i]=o;return h},o.prototype._updateItem=function(t){var e=t[this._fieldId];if(void 0==e)throw new Error("Cannot update item: item has no id (item: "+JSON.stringify(t)+")");var i=this._data[e];if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var o=Object.keys(t),n=0,r=o.length;r>n;n++){var a=o[n],h=this._type[a];i[a]=s.convert(t[a],h)}return e},t.exports=o},function(t,e){function i(t){this.delay=null,this.max=1/0,this._queue=[],this._timeout=null,this._extended=null,this.setOptions(t)}i.prototype.setOptions=function(t){t&&"undefined"!=typeof t.delay&&(this.delay=t.delay),t&&"undefined"!=typeof t.max&&(this.max=t.max),this._flushIfNeeded()},i.extend=function(t,e){var o=new i(e);if(void 0!==t.flush)throw new Error("Target object already has a property flush");t.flush=function(){o.flush()};var n=[{name:"flush",original:void 0}];if(e&&e.replace)for(var s=0;s<e.replace.length;s++){var r=e.replace[s];n.push({name:r,original:t[r]}),o.replace(t,r)}return o._extended={object:t,methods:n},o},i.prototype.destroy=function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var o=e[i];o.original?t[o.name]=o.original:delete t[o.name]}this._extended=null}},i.prototype.replace=function(t,e){var i=this,o=t[e];if(!o)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=[],e=0;e<arguments.length;e++)t[e]=arguments[e];i.queue({args:t,fn:o,context:this})}},i.prototype.queue=function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()},i.prototype._flushIfNeeded=function(){if(this._queue.length>this.max&&this.flush(),clearTimeout(this._timeout),this.queue.length>0&&"number"==typeof this.delay){var t=this;this._timeout=setTimeout(function(){t.flush()},this.delay)}},i.prototype.flush=function(){for(;this._queue.length>0;){var t=this._queue.shift();t.fn.apply(t.context||t.fn,t.args||[])}},t.exports=i},function(t,e,i){function o(t,e){this._data=null,this._ids={},this.length=0,this._options=e||{},this._fieldId="id",this._subscribers={};var i=this;this.listener=function(){i._onEvent.apply(i,arguments)},this.setData(t)}var n=i(1),s=i(9);o.prototype.setData=function(t){var e,i,o,n;if(this._data&&(this._data.off&&this._data.off("*",this.listener),e=Object.keys(this._ids),this._ids={},this.length=0,this._trigger("remove",{items:e})),this._data=t,this._data){for(this._fieldId=this._options.fieldId||this._data&&this._data.options&&this._data.options.fieldId||"id",e=this._data.getIds({filter:this._options&&this._options.filter}),o=0,n=e.length;n>o;o++)i=e[o],this._ids[i]=!0;this.length=e.length,this._trigger("add",{items:e}),this._data.on&&this._data.on("*",this.listener)}},o.prototype.refresh=function(){var t,e,i,o=this._data.getIds({filter:this._options&&this._options.filter}),n=Object.keys(this._ids),s={},r=[],a=[];for(e=0,i=o.length;i>e;e++)t=o[e],s[t]=!0,this._ids[t]||(r.push(t),this._ids[t]=!0);for(e=0,i=n.length;i>e;e++)t=n[e],s[t]||(a.push(t),delete this._ids[t]);this.length+=r.length-a.length,r.length&&this._trigger("add",{items:r}),a.length&&this._trigger("remove",{items:a})},o.prototype.get=function(t){var e,i,o,s=this,r=n.getType(arguments[0]);"String"==r||"Number"==r||"Array"==r?(e=arguments[0],i=arguments[1],o=arguments[2]):(i=arguments[0],o=arguments[1]);var a=n.extend({},this._options,i);this._options.filter&&i&&i.filter&&(a.filter=function(t){return s._options.filter(t)&&i.filter(t)});var h=[];return void 0!=e&&h.push(e),h.push(a),h.push(o),this._data&&this._data.get.apply(this._data,h)},o.prototype.getIds=function(t){var e;if(this._data){var i,o=this._options.filter;i=t&&t.filter?o?function(e){return o(e)&&t.filter(e)}:t.filter:o,e=this._data.getIds({filter:i,order:t&&t.order})}else e=[];return e},o.prototype.map=function(t,e){var i=[];if(this._data){var o,n=this._options.filter;o=e&&e.filter?n?function(t){return n(t)&&e.filter(t)}:e.filter:n,i=this._data.map(t,{filter:o,order:e&&e.order})}else i=[];return i},o.prototype.getDataSet=function(){for(var t=this;t instanceof o;)t=t._data;return t||null},o.prototype._onEvent=function(t,e,i){var o,n,s,r,a=e&&e.items,h=this._data,d=[],l=[],c=[],u=[];if(a&&h){switch(t){case"add":for(o=0,n=a.length;n>o;o++)s=a[o],r=this.get(s),r&&(this._ids[s]=!0,l.push(s));break;case"update":for(o=0,n=a.length;n>o;o++)s=a[o],r=this.get(s),r?this._ids[s]?(c.push(s),d.push(e.data[o])):(this._ids[s]=!0,l.push(s)):this._ids[s]&&(delete this._ids[s],u.push(s));break;case"remove":for(o=0,n=a.length;n>o;o++)s=a[o],this._ids[s]&&(delete this._ids[s],u.push(s))}this.length+=l.length-u.length,l.length&&this._trigger("add",{items:l},i),c.length&&this._trigger("update",{items:c,data:d},i),u.length&&this._trigger("remove",{items:u},i)}},o.prototype.on=s.prototype.on,o.prototype.off=s.prototype.off,o.prototype._trigger=s.prototype._trigger,o.prototype.subscribe=o.prototype.on,o.prototype.unsubscribe=o.prototype.off,t.exports=o},function(t,e,i){function o(t,e,i){if(!(this instanceof o))throw new SyntaxError("Constructor must be called with the new operator");this.containerElement=t,this.width="400px",this.height="400px",this.margin=10,this.defaultXCenter="55%",this.defaultYCenter="50%",this.xLabel="x",this.yLabel="y",this.zLabel="z";var n=function(t){return t};this.xValueLabel=n,this.yValueLabel=n,this.zValueLabel=n,this.filterLabel="time",this.legendLabel="value",this.style=o.STYLE.DOT,this.showPerspective=!0,this.showGrid=!0,this.keepAspectRatio=!0,this.showShadow=!1,this.showGrayBottom=!1,this.showTooltip=!1,this.verticalRatio=.5,this.animationInterval=1e3,this.animationPreload=!1,this.camera=new p,this.camera.setArmRotation(1,.5),this.camera.setArmLength(1.7),this.eye=new c(0,0,-1),this.dataTable=null,this.dataPoints=null,this.colX=void 0,this.colY=void 0,this.colZ=void 0,this.colValue=void 0,this.colFilter=void 0,this.xMin=0,this.xStep=void 0,this.xMax=1,this.yMin=0,this.yStep=void 0,this.yMax=1,this.zMin=0,this.zStep=void 0,this.zMax=1,this.valueMin=0,this.valueMax=1,this.xBarWidth=1,this.yBarWidth=1,this.axisColor="#4D4D4D",this.gridColor="#D3D3D3",this.dataColor={fill:"#7DC1FF",stroke:"#3267D2",strokeWidth:1},this.dotSizeRatio=.02,this.create(),this.setOptions(i),e&&this.setData(e)}function n(t){return"clientX"in t?t.clientX:t.targetTouches[0]&&t.targetTouches[0].clientX||0}function s(t){return"clientY"in t?t.clientY:t.targetTouches[0]&&t.targetTouches[0].clientY||0}var r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},a=i(13),h=i(9),d=i(11),l=i(1),c=i(14),u=i(15),p=i(16),f=i(17),m=i(18),v=i(19);a(o.prototype),o.prototype._setScale=function(){this.scale=new c(1/(this.xMax-this.xMin),1/(this.yMax-this.yMin),1/(this.zMax-this.zMin)),this.keepAspectRatio&&(this.scale.x<this.scale.y?this.scale.y=this.scale.x:this.scale.x=this.scale.y),this.scale.z*=this.verticalRatio,this.scale.value=1/(this.valueMax-this.valueMin);var t=(this.xMax+this.xMin)/2*this.scale.x,e=(this.yMax+this.yMin)/2*this.scale.y,i=(this.zMax+this.zMin)/2*this.scale.z;this.camera.setArmLocation(t,e,i)},o.prototype._convert3Dto2D=function(t){var e=this._convertPointToTranslation(t);return this._convertTranslationToScreen(e)},o.prototype._convertPointToTranslation=function(t){var e=t.x*this.scale.x,i=t.y*this.scale.y,o=t.z*this.scale.z,n=this.camera.getCameraLocation().x,s=this.camera.getCameraLocation().y,r=this.camera.getCameraLocation().z,a=Math.sin(this.camera.getCameraRotation().x),h=Math.cos(this.camera.getCameraRotation().x),d=Math.sin(this.camera.getCameraRotation().y),l=Math.cos(this.camera.getCameraRotation().y),u=Math.sin(this.camera.getCameraRotation().z),p=Math.cos(this.camera.getCameraRotation().z),f=l*(u*(i-s)+p*(e-n))-d*(o-r),m=a*(l*(o-r)+d*(u*(i-s)+p*(e-n)))+h*(p*(i-s)-u*(e-n)),v=h*(l*(o-r)+d*(u*(i-s)+p*(e-n)))-a*(p*(i-s)-u*(e-n));return new c(f,m,v)},o.prototype._convertTranslationToScreen=function(t){var e,i,o=this.eye.x,n=this.eye.y,s=this.eye.z,r=t.x,a=t.y,h=t.z;return this.showPerspective?(e=(r-o)*(s/h),i=(a-n)*(s/h)):(e=r*-(s/this.camera.getArmLength()),i=a*-(s/this.camera.getArmLength())),new u(this.xcenter+e*this.frame.canvas.clientWidth,this.ycenter-i*this.frame.canvas.clientWidth)},o.prototype._setBackgroundColor=function(t){var e="white",i="gray",o=1;if("string"==typeof t)e=t,i="none",o=0;else if("object"===("undefined"==typeof t?"undefined":r(t)))void 0!==t.fill&&(e=t.fill),void 0!==t.stroke&&(i=t.stroke),void 0!==t.strokeWidth&&(o=t.strokeWidth);else if(void 0!==t)throw"Unsupported type of backgroundColor";this.frame.style.backgroundColor=e,this.frame.style.borderColor=i,this.frame.style.borderWidth=o+"px",this.frame.style.borderStyle="solid"},o.STYLE={BAR:0,BARCOLOR:1,BARSIZE:2,DOT:3,DOTLINE:4,DOTCOLOR:5,DOTSIZE:6,GRID:7,LINE:8,SURFACE:9},o.prototype._getStyleNumber=function(t){switch(t){case"dot":return o.STYLE.DOT;case"dot-line":return o.STYLE.DOTLINE;case"dot-color":return o.STYLE.DOTCOLOR;case"dot-size":return o.STYLE.DOTSIZE;case"line":return o.STYLE.LINE;case"grid":return o.STYLE.GRID;case"surface":return o.STYLE.SURFACE;case"bar":return o.STYLE.BAR;case"bar-color":return o.STYLE.BARCOLOR;case"bar-size":return o.STYLE.BARSIZE}return-1},o.prototype._determineColumnIndexes=function(t,e){if(this.style===o.STYLE.DOT||this.style===o.STYLE.DOTLINE||this.style===o.STYLE.LINE||this.style===o.STYLE.GRID||this.style===o.STYLE.SURFACE||this.style===o.STYLE.BAR)this.colX=0,this.colY=1,this.colZ=2,this.colValue=void 0,t.getNumberOfColumns()>3&&(this.colFilter=3);else{if(this.style!==o.STYLE.DOTCOLOR&&this.style!==o.STYLE.DOTSIZE&&this.style!==o.STYLE.BARCOLOR&&this.style!==o.STYLE.BARSIZE)throw'Unknown style "'+this.style+'"';this.colX=0,this.colY=1,this.colZ=2,this.colValue=3,t.getNumberOfColumns()>4&&(this.colFilter=4)}},o.prototype.getNumberOfRows=function(t){return t.length},o.prototype.getNumberOfColumns=function(t){var e=0;for(var i in t[0])t[0].hasOwnProperty(i)&&e++;return e},o.prototype.getDistinctValues=function(t,e){for(var i=[],o=0;o<t.length;o++)-1==i.indexOf(t[o][e])&&i.push(t[o][e]);return i},o.prototype.getColumnRange=function(t,e){for(var i={min:t[0][e],max:t[0][e]},o=0;o<t.length;o++)i.min>t[o][e]&&(i.min=t[o][e]),i.max<t[o][e]&&(i.max=t[o][e]);return i},o.prototype._dataInitialize=function(t,e){var i=this;if(this.dataSet&&this.dataSet.off("*",this._onChange),void 0!==t){Array.isArray(t)&&(t=new h(t));var n;if(!(t instanceof h||t instanceof d))throw new Error("Array, DataSet, or DataView expected");if(n=t.get(),0!=n.length){this.dataSet=t,this.dataTable=n,this._onChange=function(){i.setData(i.dataSet)},this.dataSet.on("*",this._onChange),this.colX="x",this.colY="y",this.colZ="z",this.colValue="style",this.colFilter="filter",n[0].hasOwnProperty("filter")&&void 0===this.dataFilter&&(this.dataFilter=new f(t,this.colFilter,this),this.dataFilter.setOnLoadCallback(function(){i.redraw()}));var s=this.style==o.STYLE.BAR||this.style==o.STYLE.BARCOLOR||this.style==o.STYLE.BARSIZE;if(s){if(void 0!==this.defaultXBarWidth)this.xBarWidth=this.defaultXBarWidth;else{var r=this.getDistinctValues(n,this.colX);this.xBarWidth=r[1]-r[0]||1}if(void 0!==this.defaultYBarWidth)this.yBarWidth=this.defaultYBarWidth;else{var a=this.getDistinctValues(n,this.colY);this.yBarWidth=a[1]-a[0]||1}}var l=this.getColumnRange(n,this.colX);s&&(l.min-=this.xBarWidth/2,l.max+=this.xBarWidth/2),this.xMin=void 0!==this.defaultXMin?this.defaultXMin:l.min,this.xMax=void 0!==this.defaultXMax?this.defaultXMax:l.max,this.xMax<=this.xMin&&(this.xMax=this.xMin+1),this.xStep=void 0!==this.defaultXStep?this.defaultXStep:(this.xMax-this.xMin)/5;var c=this.getColumnRange(n,this.colY);s&&(c.min-=this.yBarWidth/2,c.max+=this.yBarWidth/2),this.yMin=void 0!==this.defaultYMin?this.defaultYMin:c.min,this.yMax=void 0!==this.defaultYMax?this.defaultYMax:c.max,this.yMax<=this.yMin&&(this.yMax=this.yMin+1),this.yStep=void 0!==this.defaultYStep?this.defaultYStep:(this.yMax-this.yMin)/5;var u=this.getColumnRange(n,this.colZ);if(this.zMin=void 0!==this.defaultZMin?this.defaultZMin:u.min,this.zMax=void 0!==this.defaultZMax?this.defaultZMax:u.max,this.zMax<=this.zMin&&(this.zMax=this.zMin+1),this.zStep=void 0!==this.defaultZStep?this.defaultZStep:(this.zMax-this.zMin)/5,void 0!==this.colValue){var p=this.getColumnRange(n,this.colValue);this.valueMin=void 0!==this.defaultValueMin?this.defaultValueMin:p.min,this.valueMax=void 0!==this.defaultValueMax?this.defaultValueMax:p.max,this.valueMax<=this.valueMin&&(this.valueMax=this.valueMin+1)}this._setScale()}}},o.prototype._getDataPoints=function(t){var e,i,n,s,r,a,h=[];if(this.style===o.STYLE.GRID||this.style===o.STYLE.SURFACE){var d=[],l=[];for(n=0;n<this.getNumberOfRows(t);n++)e=t[n][this.colX]||0,i=t[n][this.colY]||0,-1===d.indexOf(e)&&d.push(e),-1===l.indexOf(i)&&l.push(i);var u=function(t,e){return t-e};d.sort(u),l.sort(u);var p=[];for(n=0;n<t.length;n++){e=t[n][this.colX]||0,i=t[n][this.colY]||0,s=t[n][this.colZ]||0;var f=d.indexOf(e),m=l.indexOf(i);void 0===p[f]&&(p[f]=[]);var v=new c;v.x=e,v.y=i,v.z=s,r={},r.point=v,r.trans=void 0,r.screen=void 0,r.bottom=new c(e,i,this.zMin),p[f][m]=r,h.push(r)}for(e=0;e<p.length;e++)for(i=0;i<p[e].length;i++)p[e][i]&&(p[e][i].pointRight=e<p.length-1?p[e+1][i]:void 0,p[e][i].pointTop=i<p[e].length-1?p[e][i+1]:void 0,p[e][i].pointCross=e<p.length-1&&i<p[e].length-1?p[e+1][i+1]:void 0)}else for(n=0;n<t.length;n++)a=new c,a.x=t[n][this.colX]||0,a.y=t[n][this.colY]||0,a.z=t[n][this.colZ]||0,void 0!==this.colValue&&(a.value=t[n][this.colValue]||0),r={},r.point=a,r.bottom=new c(a.x,a.y,this.zMin),r.trans=void 0,r.screen=void 0,h.push(r);return h},o.prototype.create=function(){for(;this.containerElement.hasChildNodes();)this.containerElement.removeChild(this.containerElement.firstChild);this.frame=document.createElement("div"),this.frame.style.position="relative",this.frame.style.overflow="hidden",this.frame.canvas=document.createElement("canvas"),this.frame.canvas.style.position="relative",this.frame.appendChild(this.frame.canvas);var t=document.createElement("DIV");t.style.color="red",t.style.fontWeight="bold",t.style.padding="10px",t.innerHTML="Error: your browser does not support HTML canvas",this.frame.canvas.appendChild(t),this.frame.filter=document.createElement("div"),this.frame.filter.style.position="absolute",this.frame.filter.style.bottom="0px",this.frame.filter.style.left="0px",this.frame.filter.style.width="100%",this.frame.appendChild(this.frame.filter);var e=this,i=function(t){e._onMouseDown(t)},o=function(t){e._onTouchStart(t)},n=function(t){e._onWheel(t)},s=function(t){e._onTooltip(t)};l.addEventListener(this.frame.canvas,"keydown",onkeydown),l.addEventListener(this.frame.canvas,"mousedown",i),l.addEventListener(this.frame.canvas,"touchstart",o),l.addEventListener(this.frame.canvas,"mousewheel",n),l.addEventListener(this.frame.canvas,"mousemove",s),this.containerElement.appendChild(this.frame)},o.prototype.setSize=function(t,e){this.frame.style.width=t,this.frame.style.height=e,this._resizeCanvas()},o.prototype._resizeCanvas=function(){this.frame.canvas.style.width="100%",this.frame.canvas.style.height="100%",this.frame.canvas.width=this.frame.canvas.clientWidth,this.frame.canvas.height=this.frame.canvas.clientHeight,this.frame.filter.style.width=this.frame.canvas.clientWidth-20+"px"},o.prototype.animationStart=function(){if(!this.frame.filter||!this.frame.filter.slider)throw"No animation available";this.frame.filter.slider.play()},o.prototype.animationStop=function(){this.frame.filter&&this.frame.filter.slider&&this.frame.filter.slider.stop()},o.prototype._resizeCenter=function(){"%"===this.defaultXCenter.charAt(this.defaultXCenter.length-1)?this.xcenter=parseFloat(this.defaultXCenter)/100*this.frame.canvas.clientWidth:this.xcenter=parseFloat(this.defaultXCenter),"%"===this.defaultYCenter.charAt(this.defaultYCenter.length-1)?this.ycenter=parseFloat(this.defaultYCenter)/100*(this.frame.canvas.clientHeight-this.frame.filter.clientHeight):this.ycenter=parseFloat(this.defaultYCenter)},o.prototype.setCameraPosition=function(t){void 0!==t&&(void 0!==t.horizontal&&void 0!==t.vertical&&this.camera.setArmRotation(t.horizontal,t.vertical),void 0!==t.distance&&this.camera.setArmLength(t.distance),this.redraw())},o.prototype.getCameraPosition=function(){var t=this.camera.getArmRotation();return t.distance=this.camera.getArmLength(),t},o.prototype._readData=function(t){this._dataInitialize(t,this.style),this.dataFilter?this.dataPoints=this.dataFilter._getDataPoints():this.dataPoints=this._getDataPoints(this.dataTable),this._redrawFilter()},o.prototype.setData=function(t){this._readData(t),this.redraw(),this.animationAutoStart&&this.dataFilter&&this.animationStart()},o.prototype.setOptions=function(t){var e=void 0;if(this.animationStop(),void 0!==t){if(void 0!==t.width&&(this.width=t.width),void 0!==t.height&&(this.height=t.height),void 0!==t.xCenter&&(this.defaultXCenter=t.xCenter),void 0!==t.yCenter&&(this.defaultYCenter=t.yCenter),void 0!==t.filterLabel&&(this.filterLabel=t.filterLabel),void 0!==t.legendLabel&&(this.legendLabel=t.legendLabel),void 0!==t.xLabel&&(this.xLabel=t.xLabel),void 0!==t.yLabel&&(this.yLabel=t.yLabel),void 0!==t.zLabel&&(this.zLabel=t.zLabel),void 0!==t.xValueLabel&&(this.xValueLabel=t.xValueLabel),void 0!==t.yValueLabel&&(this.yValueLabel=t.yValueLabel),void 0!==t.zValueLabel&&(this.zValueLabel=t.zValueLabel),void 0!==t.dotSizeRatio&&(this.dotSizeRatio=t.dotSizeRatio),void 0!==t.style){var i=this._getStyleNumber(t.style);-1!==i&&(this.style=i)}void 0!==t.showGrid&&(this.showGrid=t.showGrid),void 0!==t.showPerspective&&(this.showPerspective=t.showPerspective),void 0!==t.showShadow&&(this.showShadow=t.showShadow),void 0!==t.tooltip&&(this.showTooltip=t.tooltip),void 0!==t.showAnimationControls&&(this.showAnimationControls=t.showAnimationControls),void 0!==t.keepAspectRatio&&(this.keepAspectRatio=t.keepAspectRatio),void 0!==t.verticalRatio&&(this.verticalRatio=t.verticalRatio),void 0!==t.animationInterval&&(this.animationInterval=t.animationInterval),void 0!==t.animationPreload&&(this.animationPreload=t.animationPreload),void 0!==t.animationAutoStart&&(this.animationAutoStart=t.animationAutoStart),void 0!==t.xBarWidth&&(this.defaultXBarWidth=t.xBarWidth),void 0!==t.yBarWidth&&(this.defaultYBarWidth=t.yBarWidth),void 0!==t.xMin&&(this.defaultXMin=t.xMin),void 0!==t.xStep&&(this.defaultXStep=t.xStep),void 0!==t.xMax&&(this.defaultXMax=t.xMax),void 0!==t.yMin&&(this.defaultYMin=t.yMin),void 0!==t.yStep&&(this.defaultYStep=t.yStep),void 0!==t.yMax&&(this.defaultYMax=t.yMax),void 0!==t.zMin&&(this.defaultZMin=t.zMin),void 0!==t.zStep&&(this.defaultZStep=t.zStep),void 0!==t.zMax&&(this.defaultZMax=t.zMax),void 0!==t.valueMin&&(this.defaultValueMin=t.valueMin),void 0!==t.valueMax&&(this.defaultValueMax=t.valueMax),void 0!==t.backgroundColor&&this._setBackgroundColor(t.backgroundColor),void 0!==t.cameraPosition&&(e=t.cameraPosition),void 0!==e&&(this.camera.setArmRotation(e.horizontal,e.vertical),this.camera.setArmLength(e.distance)),void 0!==t.axisColor&&(this.axisColor=t.axisColor),void 0!==t.gridColor&&(this.gridColor=t.gridColor),t.dataColor&&("string"==typeof t.dataColor?(this.dataColor.fill=t.dataColor,this.dataColor.stroke=t.dataColor):(t.dataColor.fill&&(this.dataColor.fill=t.dataColor.fill),t.dataColor.stroke&&(this.dataColor.stroke=t.dataColor.stroke),void 0!==t.dataColor.strokeWidth&&(this.dataColor.strokeWidth=t.dataColor.strokeWidth)))}this.setSize(this.width,this.height),this.dataTable&&this.setData(this.dataTable),this.animationAutoStart&&this.dataFilter&&this.animationStart()},o.prototype.redraw=function(){if(void 0===this.dataPoints)throw"Error: graph data not initialized";this._resizeCanvas(),this._resizeCenter(),this._redrawSlider(),this._redrawClear(),this._redrawAxis(),this.style===o.STYLE.GRID||this.style===o.STYLE.SURFACE?this._redrawDataGrid():this.style===o.STYLE.LINE?this._redrawDataLine():this.style===o.STYLE.BAR||this.style===o.STYLE.BARCOLOR||this.style===o.STYLE.BARSIZE?this._redrawDataBar():this._redrawDataDot(),this._redrawInfo(),this._redrawLegend()},o.prototype._redrawClear=function(){var t=this.frame.canvas,e=t.getContext("2d");e.clearRect(0,0,t.width,t.height)},o.prototype._redrawLegend=function(){var t;if(this.style===o.STYLE.DOTCOLOR||this.style===o.STYLE.DOTSIZE){var e,i,n=this.frame.clientWidth*this.dotSizeRatio;this.style===o.STYLE.DOTSIZE?(e=n/2,i=n/2+2*n):(e=20,i=20);var s=Math.max(.25*this.frame.clientHeight,100),r=this.margin,a=this.frame.clientWidth-this.margin,h=a-i,d=r+s}var l=this.frame.canvas,c=l.getContext("2d");if(c.lineWidth=1,c.font="14px arial",this.style===o.STYLE.DOTCOLOR){var u=0,p=s;for(t=u;p>t;t++){var f=(t-u)/(p-u),m=240*f,g=this._hsv2rgb(m,1,1);c.strokeStyle=g,c.beginPath(),c.moveTo(h,r+t),c.lineTo(a,r+t),c.stroke()}c.strokeStyle=this.axisColor,c.strokeRect(h,r,i,s)}if(this.style===o.STYLE.DOTSIZE&&(c.strokeStyle=this.axisColor,c.fillStyle=this.dataColor.fill,c.beginPath(),c.moveTo(h,r),c.lineTo(a,r),c.lineTo(a-i+e,d),c.lineTo(h,d),c.closePath(),c.fill(),c.stroke()),this.style===o.STYLE.DOTCOLOR||this.style===o.STYLE.DOTSIZE){var y=5,b=new v(this.valueMin,this.valueMax,(this.valueMax-this.valueMin)/5,!0);for(b.start(),b.getCurrent()<this.valueMin&&b.next();!b.end();)t=d-(b.getCurrent()-this.valueMin)/(this.valueMax-this.valueMin)*s,c.beginPath(),c.moveTo(h-y,t),c.lineTo(h,t),c.stroke(),c.textAlign="right",c.textBaseline="middle",c.fillStyle=this.axisColor,c.fillText(b.getCurrent(),h-2*y,t),b.next();c.textAlign="right",c.textBaseline="top";var w=this.legendLabel;c.fillText(w,a,d+this.margin)}},o.prototype._redrawFilter=function(){if(this.frame.filter.innerHTML="",this.dataFilter){var t={visible:this.showAnimationControls},e=new m(this.frame.filter,t);this.frame.filter.slider=e,this.frame.filter.style.padding="10px",e.setValues(this.dataFilter.values),e.setPlayInterval(this.animationInterval);var i=this,o=function(){var t=e.getIndex();i.dataFilter.selectValue(t),i.dataPoints=i.dataFilter._getDataPoints(),i.redraw()};e.setOnChangeCallback(o)}else this.frame.filter.slider=void 0},o.prototype._redrawSlider=function(){void 0!==this.frame.filter.slider&&this.frame.filter.slider.redraw()},o.prototype._redrawInfo=function(){if(this.dataFilter){var t=this.frame.canvas,e=t.getContext("2d");e.font="14px arial",e.lineStyle="gray",e.fillStyle="gray",e.textAlign="left",e.textBaseline="top";var i=this.margin,o=this.margin;e.fillText(this.dataFilter.getLabel()+": "+this.dataFilter.getSelectedValue(),i,o)}},o.prototype._redrawAxis=function(){var t,e,i,o,n,s,r,a,h,d,l,u,p,f=this.frame.canvas,m=f.getContext("2d");m.font=24/this.camera.getArmLength()+"px arial";var g=.025/this.scale.x,y=.025/this.scale.y,b=5/this.camera.getArmLength(),w=this.camera.getArmRotation().horizontal;for(m.lineWidth=1,o=void 0===this.defaultXStep,i=new v(this.xMin,this.xMax,this.xStep,o),i.start(),i.getCurrent()<this.xMin&&i.next();!i.end();){var _=i.getCurrent();this.showGrid?(t=this._convert3Dto2D(new c(_,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(_,this.yMax,this.zMin)),m.strokeStyle=this.gridColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()):(t=this._convert3Dto2D(new c(_,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(_,this.yMin+g,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),t=this._convert3Dto2D(new c(_,this.yMax,this.zMin)),e=this._convert3Dto2D(new c(_,this.yMax-g,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()),r=Math.cos(w)>0?this.yMin:this.yMax,n=this._convert3Dto2D(new c(_,r,this.zMin)),Math.cos(2*w)>0?(m.textAlign="center",m.textBaseline="top",n.y+=b):Math.sin(2*w)<0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText("  "+this.xValueLabel(i.getCurrent())+"  ",n.x,n.y),i.next()}for(m.lineWidth=1,o=void 0===this.defaultYStep,i=new v(this.yMin,this.yMax,this.yStep,o),i.start(),i.getCurrent()<this.yMin&&i.next();!i.end();)this.showGrid?(t=this._convert3Dto2D(new c(this.xMin,i.getCurrent(),this.zMin)),e=this._convert3Dto2D(new c(this.xMax,i.getCurrent(),this.zMin)),m.strokeStyle=this.gridColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()):(t=this._convert3Dto2D(new c(this.xMin,i.getCurrent(),this.zMin)),e=this._convert3Dto2D(new c(this.xMin+y,i.getCurrent(),this.zMin)),
-m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),t=this._convert3Dto2D(new c(this.xMax,i.getCurrent(),this.zMin)),e=this._convert3Dto2D(new c(this.xMax-y,i.getCurrent(),this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke()),s=Math.sin(w)>0?this.xMin:this.xMax,n=this._convert3Dto2D(new c(s,i.getCurrent(),this.zMin)),Math.cos(2*w)<0?(m.textAlign="center",m.textBaseline="top",n.y+=b):Math.sin(2*w)>0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText("  "+this.yValueLabel(i.getCurrent())+"  ",n.x,n.y),i.next();for(m.lineWidth=1,o=void 0===this.defaultZStep,i=new v(this.zMin,this.zMax,this.zStep,o),i.start(),i.getCurrent()<this.zMin&&i.next(),s=Math.cos(w)>0?this.xMin:this.xMax,r=Math.sin(w)<0?this.yMin:this.yMax;!i.end();)t=this._convert3Dto2D(new c(s,r,i.getCurrent())),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(t.x-b,t.y),m.stroke(),m.textAlign="right",m.textBaseline="middle",m.fillStyle=this.axisColor,m.fillText(this.zValueLabel(i.getCurrent())+" ",t.x-5,t.y),i.next();m.lineWidth=1,t=this._convert3Dto2D(new c(s,r,this.zMin)),e=this._convert3Dto2D(new c(s,r,this.zMax)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),m.lineWidth=1,u=this._convert3Dto2D(new c(this.xMin,this.yMin,this.zMin)),p=this._convert3Dto2D(new c(this.xMax,this.yMin,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(u.x,u.y),m.lineTo(p.x,p.y),m.stroke(),u=this._convert3Dto2D(new c(this.xMin,this.yMax,this.zMin)),p=this._convert3Dto2D(new c(this.xMax,this.yMax,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(u.x,u.y),m.lineTo(p.x,p.y),m.stroke(),m.lineWidth=1,t=this._convert3Dto2D(new c(this.xMin,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(this.xMin,this.yMax,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke(),t=this._convert3Dto2D(new c(this.xMax,this.yMin,this.zMin)),e=this._convert3Dto2D(new c(this.xMax,this.yMax,this.zMin)),m.strokeStyle=this.axisColor,m.beginPath(),m.moveTo(t.x,t.y),m.lineTo(e.x,e.y),m.stroke();var x=this.xLabel;x.length>0&&(l=.1/this.scale.y,s=(this.xMin+this.xMax)/2,r=Math.cos(w)>0?this.yMin-l:this.yMax+l,n=this._convert3Dto2D(new c(s,r,this.zMin)),Math.cos(2*w)>0?(m.textAlign="center",m.textBaseline="top"):Math.sin(2*w)<0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText(x,n.x,n.y));var k=this.yLabel;k.length>0&&(d=.1/this.scale.x,s=Math.sin(w)>0?this.xMin-d:this.xMax+d,r=(this.yMin+this.yMax)/2,n=this._convert3Dto2D(new c(s,r,this.zMin)),Math.cos(2*w)<0?(m.textAlign="center",m.textBaseline="top"):Math.sin(2*w)>0?(m.textAlign="right",m.textBaseline="middle"):(m.textAlign="left",m.textBaseline="middle"),m.fillStyle=this.axisColor,m.fillText(k,n.x,n.y));var O=this.zLabel;O.length>0&&(h=30,s=Math.cos(w)>0?this.xMin:this.xMax,r=Math.sin(w)<0?this.yMin:this.yMax,a=(this.zMin+this.zMax)/2,n=this._convert3Dto2D(new c(s,r,a)),m.textAlign="right",m.textBaseline="middle",m.fillStyle=this.axisColor,m.fillText(O,n.x-h,n.y))},o.prototype._hsv2rgb=function(t,e,i){var o,n,s,r,a,h;switch(r=i*e,a=Math.floor(t/60),h=r*(1-Math.abs(t/60%2-1)),a){case 0:o=r,n=h,s=0;break;case 1:o=h,n=r,s=0;break;case 2:o=0,n=r,s=h;break;case 3:o=0,n=h,s=r;break;case 4:o=h,n=0,s=r;break;case 5:o=r,n=0,s=h;break;default:o=0,n=0,s=0}return"RGB("+parseInt(255*o)+","+parseInt(255*n)+","+parseInt(255*s)+")"},o.prototype._redrawDataGrid=function(){var t,e,i,n,s,r,a,h,d,l,u,p,f=this.frame.canvas,m=f.getContext("2d");if(m.lineJoin="round",m.lineCap="round",!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(s=0;s<this.dataPoints.length;s++){var v=this._convertPointToTranslation(this.dataPoints[s].point),g=this._convertTranslationToScreen(v);this.dataPoints[s].trans=v,this.dataPoints[s].screen=g;var y=this._convertPointToTranslation(this.dataPoints[s].bottom);this.dataPoints[s].dist=this.showPerspective?y.length():-y.z}var b=function(t,e){return e.dist-t.dist};if(this.dataPoints.sort(b),this.style===o.STYLE.SURFACE){for(s=0;s<this.dataPoints.length;s++)if(t=this.dataPoints[s],e=this.dataPoints[s].pointRight,i=this.dataPoints[s].pointTop,n=this.dataPoints[s].pointCross,void 0!==t&&void 0!==e&&void 0!==i&&void 0!==n){if(this.showGrayBottom||this.showShadow){var w=c.subtract(n.trans,t.trans),_=c.subtract(i.trans,e.trans),x=c.crossProduct(w,_),k=x.length();r=x.z>0}else r=!0;r?(p=(t.point.z+e.point.z+i.point.z+n.point.z)/4,d=240*(1-(p-this.zMin)*this.scale.z/this.verticalRatio),l=1,this.showShadow?(u=Math.min(1+x.x/k/2,1),a=this._hsv2rgb(d,l,u),h=a):(u=1,a=this._hsv2rgb(d,l,u),h=this.axisColor)):(a="gray",h=this.axisColor),m.lineWidth=this._getStrokeWidth(t),m.fillStyle=a,m.strokeStyle=h,m.beginPath(),m.moveTo(t.screen.x,t.screen.y),m.lineTo(e.screen.x,e.screen.y),m.lineTo(n.screen.x,n.screen.y),m.lineTo(i.screen.x,i.screen.y),m.closePath(),m.fill(),m.stroke()}}else for(s=0;s<this.dataPoints.length;s++)t=this.dataPoints[s],e=this.dataPoints[s].pointRight,i=this.dataPoints[s].pointTop,void 0!==t&&void 0!==e&&(p=(t.point.z+e.point.z)/2,d=240*(1-(p-this.zMin)*this.scale.z/this.verticalRatio),m.lineWidth=2*this._getStrokeWidth(t),m.strokeStyle=this._hsv2rgb(d,1,1),m.beginPath(),m.moveTo(t.screen.x,t.screen.y),m.lineTo(e.screen.x,e.screen.y),m.stroke()),void 0!==t&&void 0!==i&&(p=(t.point.z+i.point.z)/2,d=240*(1-(p-this.zMin)*this.scale.z/this.verticalRatio),m.lineWidth=2*this._getStrokeWidth(t),m.strokeStyle=this._hsv2rgb(d,1,1),m.beginPath(),m.moveTo(t.screen.x,t.screen.y),m.lineTo(i.screen.x,i.screen.y),m.stroke())}},o.prototype._getStrokeWidth=function(t){return void 0!==t?this.showPerspective?1/-t.trans.z*this.dataColor.strokeWidth:-(this.eye.z/this.camera.getArmLength())*this.dataColor.strokeWidth:this.dataColor.strokeWidth},o.prototype._redrawDataDot=function(){var t,e=this.frame.canvas,i=e.getContext("2d");if(!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(t=0;t<this.dataPoints.length;t++){var n=this._convertPointToTranslation(this.dataPoints[t].point),s=this._convertTranslationToScreen(n);this.dataPoints[t].trans=n,this.dataPoints[t].screen=s;var r=this._convertPointToTranslation(this.dataPoints[t].bottom);this.dataPoints[t].dist=this.showPerspective?r.length():-r.z}var a=function(t,e){return e.dist-t.dist};this.dataPoints.sort(a);var h=this.frame.clientWidth*this.dotSizeRatio;for(t=0;t<this.dataPoints.length;t++){var d=this.dataPoints[t];if(this.style===o.STYLE.DOTLINE){var l=this._convert3Dto2D(d.bottom);i.lineWidth=1,i.strokeStyle=this.gridColor,i.beginPath(),i.moveTo(l.x,l.y),i.lineTo(d.screen.x,d.screen.y),i.stroke()}var c;c=this.style===o.STYLE.DOTSIZE?h/2+2*h*(d.point.value-this.valueMin)/(this.valueMax-this.valueMin):h;var u;u=this.showPerspective?c/-d.trans.z:c*-(this.eye.z/this.camera.getArmLength()),0>u&&(u=0);var p,f,m;this.style===o.STYLE.DOTCOLOR?(p=240*(1-(d.point.value-this.valueMin)*this.scale.value),f=this._hsv2rgb(p,1,1),m=this._hsv2rgb(p,1,.8)):this.style===o.STYLE.DOTSIZE?(f=this.dataColor.fill,m=this.dataColor.stroke):(p=240*(1-(d.point.z-this.zMin)*this.scale.z/this.verticalRatio),f=this._hsv2rgb(p,1,1),m=this._hsv2rgb(p,1,.8)),i.lineWidth=this._getStrokeWidth(d),i.strokeStyle=m,i.fillStyle=f,i.beginPath(),i.arc(d.screen.x,d.screen.y,u,0,2*Math.PI,!0),i.fill(),i.stroke()}}},o.prototype._redrawDataBar=function(){var t,e,i,n,s=this.frame.canvas,r=s.getContext("2d");if(!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(t=0;t<this.dataPoints.length;t++){var a=this._convertPointToTranslation(this.dataPoints[t].point),h=this._convertTranslationToScreen(a);this.dataPoints[t].trans=a,this.dataPoints[t].screen=h;var d=this._convertPointToTranslation(this.dataPoints[t].bottom);this.dataPoints[t].dist=this.showPerspective?d.length():-d.z}var l=function(t,e){return e.dist-t.dist};this.dataPoints.sort(l),r.lineJoin="round",r.lineCap="round";var u=this.xBarWidth/2,p=this.yBarWidth/2;for(t=0;t<this.dataPoints.length;t++){var f,m,v,g=this.dataPoints[t];this.style===o.STYLE.BARCOLOR?(f=240*(1-(g.point.value-this.valueMin)*this.scale.value),m=this._hsv2rgb(f,1,1),v=this._hsv2rgb(f,1,.8)):this.style===o.STYLE.BARSIZE?(m=this.dataColor.fill,v=this.dataColor.stroke):(f=240*(1-(g.point.z-this.zMin)*this.scale.z/this.verticalRatio),m=this._hsv2rgb(f,1,1),v=this._hsv2rgb(f,1,.8)),this.style===o.STYLE.BARSIZE&&(u=this.xBarWidth/2*((g.point.value-this.valueMin)/(this.valueMax-this.valueMin)*.8+.2),p=this.yBarWidth/2*((g.point.value-this.valueMin)/(this.valueMax-this.valueMin)*.8+.2));var y=this,b=g.point,w=[{point:new c(b.x-u,b.y-p,b.z)},{point:new c(b.x+u,b.y-p,b.z)},{point:new c(b.x+u,b.y+p,b.z)},{point:new c(b.x-u,b.y+p,b.z)}],_=[{point:new c(b.x-u,b.y-p,this.zMin)},{point:new c(b.x+u,b.y-p,this.zMin)},{point:new c(b.x+u,b.y+p,this.zMin)},{point:new c(b.x-u,b.y+p,this.zMin)}];w.forEach(function(t){t.screen=y._convert3Dto2D(t.point)}),_.forEach(function(t){t.screen=y._convert3Dto2D(t.point)});var x=[{corners:w,center:c.avg(_[0].point,_[2].point)},{corners:[w[0],w[1],_[1],_[0]],center:c.avg(_[1].point,_[0].point)},{corners:[w[1],w[2],_[2],_[1]],center:c.avg(_[2].point,_[1].point)},{corners:[w[2],w[3],_[3],_[2]],center:c.avg(_[3].point,_[2].point)},{corners:[w[3],w[0],_[0],_[3]],center:c.avg(_[0].point,_[3].point)}];for(g.surfaces=x,e=0;e<x.length;e++){i=x[e];var k=this._convertPointToTranslation(i.center);i.dist=this.showPerspective?k.length():-k.z}for(x.sort(function(t,e){var i=e.dist-t.dist;return i?i:t.corners===w?1:e.corners===w?-1:0}),r.lineWidth=this._getStrokeWidth(g),r.strokeStyle=v,r.fillStyle=m,e=2;e<x.length;e++)i=x[e],n=i.corners,r.beginPath(),r.moveTo(n[3].screen.x,n[3].screen.y),r.lineTo(n[0].screen.x,n[0].screen.y),r.lineTo(n[1].screen.x,n[1].screen.y),r.lineTo(n[2].screen.x,n[2].screen.y),r.lineTo(n[3].screen.x,n[3].screen.y),r.fill(),r.stroke()}}},o.prototype._redrawDataLine=function(){var t,e,i=this.frame.canvas,o=i.getContext("2d");if(!(void 0===this.dataPoints||this.dataPoints.length<=0)){for(e=0;e<this.dataPoints.length;e++){var n=this._convertPointToTranslation(this.dataPoints[e].point),s=this._convertTranslationToScreen(n);this.dataPoints[e].trans=n,this.dataPoints[e].screen=s}if(this.dataPoints.length>0){for(t=this.dataPoints[0],o.lineWidth=this._getStrokeWidth(t),o.lineJoin="round",o.lineCap="round",o.strokeStyle=this.dataColor.stroke,o.beginPath(),o.moveTo(t.screen.x,t.screen.y),e=1;e<this.dataPoints.length;e++)t=this.dataPoints[e],o.lineTo(t.screen.x,t.screen.y);o.stroke()}}},o.prototype._onMouseDown=function(t){if(t=t||window.event,this.leftButtonDown&&this._onMouseUp(t),this.leftButtonDown=t.which?1===t.which:1===t.button,this.leftButtonDown||this.touchDown){this.startMouseX=n(t),this.startMouseY=s(t),this.startStart=new Date(this.start),this.startEnd=new Date(this.end),this.startArmRotation=this.camera.getArmRotation(),this.frame.style.cursor="move";var e=this;this.onmousemove=function(t){e._onMouseMove(t)},this.onmouseup=function(t){e._onMouseUp(t)},l.addEventListener(document,"mousemove",e.onmousemove),l.addEventListener(document,"mouseup",e.onmouseup),l.preventDefault(t)}},o.prototype._onMouseMove=function(t){t=t||window.event;var e=parseFloat(n(t))-this.startMouseX,i=parseFloat(s(t))-this.startMouseY,o=this.startArmRotation.horizontal+e/200,r=this.startArmRotation.vertical+i/200,a=4,h=Math.sin(a/360*2*Math.PI);Math.abs(Math.sin(o))<h&&(o=Math.round(o/Math.PI)*Math.PI-.001),Math.abs(Math.cos(o))<h&&(o=(Math.round(o/Math.PI-.5)+.5)*Math.PI-.001),Math.abs(Math.sin(r))<h&&(r=Math.round(r/Math.PI)*Math.PI),Math.abs(Math.cos(r))<h&&(r=(Math.round(r/Math.PI-.5)+.5)*Math.PI),this.camera.setArmRotation(o,r),this.redraw();var d=this.getCameraPosition();this.emit("cameraPositionChange",d),l.preventDefault(t)},o.prototype._onMouseUp=function(t){this.frame.style.cursor="auto",this.leftButtonDown=!1,l.removeEventListener(document,"mousemove",this.onmousemove),l.removeEventListener(document,"mouseup",this.onmouseup),l.preventDefault(t)},o.prototype._onTooltip=function(t){var e=300,i=this.frame.getBoundingClientRect(),o=n(t)-i.left,r=s(t)-i.top;if(this.showTooltip){if(this.tooltipTimeout&&clearTimeout(this.tooltipTimeout),this.leftButtonDown)return void this._hideTooltip();if(this.tooltip&&this.tooltip.dataPoint){var a=this._dataPointFromXY(o,r);a!==this.tooltip.dataPoint&&(a?this._showTooltip(a):this._hideTooltip())}else{var h=this;this.tooltipTimeout=setTimeout(function(){h.tooltipTimeout=null;var t=h._dataPointFromXY(o,r);t&&h._showTooltip(t)},e)}}},o.prototype._onTouchStart=function(t){this.touchDown=!0;var e=this;this.ontouchmove=function(t){e._onTouchMove(t)},this.ontouchend=function(t){e._onTouchEnd(t)},l.addEventListener(document,"touchmove",e.ontouchmove),l.addEventListener(document,"touchend",e.ontouchend),this._onMouseDown(t)},o.prototype._onTouchMove=function(t){this._onMouseMove(t)},o.prototype._onTouchEnd=function(t){this.touchDown=!1,l.removeEventListener(document,"touchmove",this.ontouchmove),l.removeEventListener(document,"touchend",this.ontouchend),this._onMouseUp(t)},o.prototype._onWheel=function(t){t||(t=window.event);var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),e){var i=this.camera.getArmLength(),o=i*(1-e/10);this.camera.setArmLength(o),this.redraw(),this._hideTooltip()}var n=this.getCameraPosition();this.emit("cameraPositionChange",n),l.preventDefault(t)},o.prototype._insideTriangle=function(t,e){function i(t){return t>0?1:0>t?-1:0}var o=e[0],n=e[1],s=e[2],r=i((n.x-o.x)*(t.y-o.y)-(n.y-o.y)*(t.x-o.x)),a=i((s.x-n.x)*(t.y-n.y)-(s.y-n.y)*(t.x-n.x)),h=i((o.x-s.x)*(t.y-s.y)-(o.y-s.y)*(t.x-s.x));return!(0!=r&&0!=a&&r!=a||0!=a&&0!=h&&a!=h||0!=r&&0!=h&&r!=h)},o.prototype._dataPointFromXY=function(t,e){var i,n=100,s=null,r=null,a=null,h=new u(t,e);if(this.style===o.STYLE.BAR||this.style===o.STYLE.BARCOLOR||this.style===o.STYLE.BARSIZE)for(i=this.dataPoints.length-1;i>=0;i--){s=this.dataPoints[i];var d=s.surfaces;if(d)for(var l=d.length-1;l>=0;l--){var c=d[l],p=c.corners,f=[p[0].screen,p[1].screen,p[2].screen],m=[p[2].screen,p[3].screen,p[0].screen];if(this._insideTriangle(h,f)||this._insideTriangle(h,m))return s}}else for(i=0;i<this.dataPoints.length;i++){s=this.dataPoints[i];var v=s.screen;if(v){var g=Math.abs(t-v.x),y=Math.abs(e-v.y),b=Math.sqrt(g*g+y*y);(null===a||a>b)&&n>b&&(a=b,r=s)}}return r},o.prototype._showTooltip=function(t){var e,i,o;this.tooltip?(e=this.tooltip.dom.content,i=this.tooltip.dom.line,o=this.tooltip.dom.dot):(e=document.createElement("div"),e.style.position="absolute",e.style.padding="10px",e.style.border="1px solid #4d4d4d",e.style.color="#1a1a1a",e.style.background="rgba(255,255,255,0.7)",e.style.borderRadius="2px",e.style.boxShadow="5px 5px 10px rgba(128,128,128,0.5)",i=document.createElement("div"),i.style.position="absolute",i.style.height="40px",i.style.width="0",i.style.borderLeft="1px solid #4d4d4d",o=document.createElement("div"),o.style.position="absolute",o.style.height="0",o.style.width="0",o.style.border="5px solid #4d4d4d",o.style.borderRadius="5px",this.tooltip={dataPoint:null,dom:{content:e,line:i,dot:o}}),this._hideTooltip(),this.tooltip.dataPoint=t,"function"==typeof this.showTooltip?e.innerHTML=this.showTooltip(t.point):e.innerHTML="<table><tr><td>"+this.xLabel+":</td><td>"+t.point.x+"</td></tr><tr><td>"+this.yLabel+":</td><td>"+t.point.y+"</td></tr><tr><td>"+this.zLabel+":</td><td>"+t.point.z+"</td></tr></table>",e.style.left="0",e.style.top="0",this.frame.appendChild(e),this.frame.appendChild(i),this.frame.appendChild(o);var n=e.offsetWidth,s=e.offsetHeight,r=i.offsetHeight,a=o.offsetWidth,h=o.offsetHeight,d=t.screen.x-n/2;d=Math.min(Math.max(d,10),this.frame.clientWidth-10-n),i.style.left=t.screen.x+"px",i.style.top=t.screen.y-r+"px",e.style.left=d+"px",e.style.top=t.screen.y-r-s+"px",o.style.left=t.screen.x-a/2+"px",o.style.top=t.screen.y-h/2+"px"},o.prototype._hideTooltip=function(){if(this.tooltip){this.tooltip.dataPoint=null;for(var t in this.tooltip.dom)if(this.tooltip.dom.hasOwnProperty(t)){var e=this.tooltip.dom[t];e&&e.parentNode&&e.parentNode.removeChild(e)}}},t.exports=o},function(t,e){function i(t){return t?o(t):void 0}function o(t){for(var e in i.prototype)t[e]=i.prototype[e];return t}t.exports=i,i.prototype.on=i.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks[t]=this._callbacks[t]||[]).push(e),this},i.prototype.once=function(t,e){function i(){o.off(t,i),e.apply(this,arguments)}var o=this;return this._callbacks=this._callbacks||{},i.fn=e,this.on(t,i),this},i.prototype.off=i.prototype.removeListener=i.prototype.removeAllListeners=i.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i=this._callbacks[t];if(!i)return this;if(1==arguments.length)return delete this._callbacks[t],this;for(var o,n=0;n<i.length;n++)if(o=i[n],o===e||o.fn===e){i.splice(n,1);break}return this},i.prototype.emit=function(t){this._callbacks=this._callbacks||{};var e=[].slice.call(arguments,1),i=this._callbacks[t];if(i){i=i.slice(0);for(var o=0,n=i.length;n>o;++o)i[o].apply(this,e)}return this},i.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks[t]||[]},i.prototype.hasListeners=function(t){return!!this.listeners(t).length}},function(t,e){function i(t,e,i){this.x=void 0!==t?t:0,this.y=void 0!==e?e:0,this.z=void 0!==i?i:0}i.subtract=function(t,e){var o=new i;return o.x=t.x-e.x,o.y=t.y-e.y,o.z=t.z-e.z,o},i.add=function(t,e){var o=new i;return o.x=t.x+e.x,o.y=t.y+e.y,o.z=t.z+e.z,o},i.avg=function(t,e){return new i((t.x+e.x)/2,(t.y+e.y)/2,(t.z+e.z)/2)},i.crossProduct=function(t,e){var o=new i;return o.x=t.y*e.z-t.z*e.y,o.y=t.z*e.x-t.x*e.z,o.z=t.x*e.y-t.y*e.x,o},i.prototype.length=function(){return Math.sqrt(this.x*this.x+this.y*this.y+this.z*this.z)},t.exports=i},function(t,e){function i(t,e){this.x=void 0!==t?t:0,this.y=void 0!==e?e:0}t.exports=i},function(t,e,i){function o(){this.armLocation=new n,this.armRotation={},this.armRotation.horizontal=0,this.armRotation.vertical=0,this.armLength=1.7,this.cameraLocation=new n,this.cameraRotation=new n(.5*Math.PI,0,0),this.calculateCameraOrientation()}var n=i(14);o.prototype.setArmLocation=function(t,e,i){this.armLocation.x=t,this.armLocation.y=e,this.armLocation.z=i,this.calculateCameraOrientation()},o.prototype.setArmRotation=function(t,e){void 0!==t&&(this.armRotation.horizontal=t),void 0!==e&&(this.armRotation.vertical=e,this.armRotation.vertical<0&&(this.armRotation.vertical=0),this.armRotation.vertical>.5*Math.PI&&(this.armRotation.vertical=.5*Math.PI)),void 0===t&&void 0===e||this.calculateCameraOrientation()},o.prototype.getArmRotation=function(){var t={};return t.horizontal=this.armRotation.horizontal,t.vertical=this.armRotation.vertical,t},o.prototype.setArmLength=function(t){void 0!==t&&(this.armLength=t,this.armLength<.71&&(this.armLength=.71),this.armLength>5&&(this.armLength=5),this.calculateCameraOrientation())},o.prototype.getArmLength=function(){return this.armLength},o.prototype.getCameraLocation=function(){return this.cameraLocation},o.prototype.getCameraRotation=function(){return this.cameraRotation},o.prototype.calculateCameraOrientation=function(){this.cameraLocation.x=this.armLocation.x-this.armLength*Math.sin(this.armRotation.horizontal)*Math.cos(this.armRotation.vertical),this.cameraLocation.y=this.armLocation.y-this.armLength*Math.cos(this.armRotation.horizontal)*Math.cos(this.armRotation.vertical),this.cameraLocation.z=this.armLocation.z+this.armLength*Math.sin(this.armRotation.vertical),this.cameraRotation.x=Math.PI/2-this.armRotation.vertical,this.cameraRotation.y=0,this.cameraRotation.z=-this.armRotation.horizontal},t.exports=o},function(t,e,i){function o(t,e,i){this.data=t,this.column=e,this.graph=i,this.index=void 0,this.value=void 0,this.values=i.getDistinctValues(t.get(),this.column),this.values.sort(function(t,e){return t>e?1:e>t?-1:0}),this.values.length>0&&this.selectValue(0),this.dataPoints=[],this.loaded=!1,this.onLoadCallback=void 0,i.animationPreload?(this.loaded=!1,this.loadInBackground()):this.loaded=!0}var n=i(11);o.prototype.isLoaded=function(){return this.loaded},o.prototype.getLoadedProgress=function(){for(var t=this.values.length,e=0;this.dataPoints[e];)e++;return Math.round(e/t*100)},o.prototype.getLabel=function(){return this.graph.filterLabel},o.prototype.getColumn=function(){return this.column},o.prototype.getSelectedValue=function(){return void 0!==this.index?this.values[this.index]:void 0},o.prototype.getValues=function(){return this.values},o.prototype.getValue=function(t){if(t>=this.values.length)throw"Error: index out of range";return this.values[t]},o.prototype._getDataPoints=function(t){if(void 0===t&&(t=this.index),void 0===t)return[];var e;if(this.dataPoints[t])e=this.dataPoints[t];else{var i={};i.column=this.column,i.value=this.values[t];var o=new n(this.data,{filter:function(t){return t[i.column]==i.value}}).get();e=this.graph._getDataPoints(o),this.dataPoints[t]=e}return e},o.prototype.setOnLoadCallback=function(t){this.onLoadCallback=t},o.prototype.selectValue=function(t){if(t>=this.values.length)throw"Error: index out of range";this.index=t,this.value=this.values[t]},o.prototype.loadInBackground=function(t){void 0===t&&(t=0);var e=this.graph.frame;if(t<this.values.length){this._getDataPoints(t);void 0===e.progress&&(e.progress=document.createElement("DIV"),e.progress.style.position="absolute",e.progress.style.color="gray",e.appendChild(e.progress));var i=this.getLoadedProgress();e.progress.innerHTML="Loading animation... "+i+"%",e.progress.style.bottom="60px",e.progress.style.left="10px";var o=this;setTimeout(function(){o.loadInBackground(t+1)},10),this.loaded=!1}else this.loaded=!0,void 0!==e.progress&&(e.removeChild(e.progress),e.progress=void 0),this.onLoadCallback&&this.onLoadCallback()},t.exports=o},function(t,e,i){function o(t,e){if(void 0===t)throw"Error: No container element defined";if(this.container=t,this.visible=e&&void 0!=e.visible?e.visible:!0,this.visible){this.frame=document.createElement("DIV"),this.frame.style.width="100%",this.frame.style.position="relative",this.container.appendChild(this.frame),this.frame.prev=document.createElement("INPUT"),this.frame.prev.type="BUTTON",this.frame.prev.value="Prev",this.frame.appendChild(this.frame.prev),this.frame.play=document.createElement("INPUT"),this.frame.play.type="BUTTON",this.frame.play.value="Play",this.frame.appendChild(this.frame.play),this.frame.next=document.createElement("INPUT"),this.frame.next.type="BUTTON",this.frame.next.value="Next",this.frame.appendChild(this.frame.next),this.frame.bar=document.createElement("INPUT"),this.frame.bar.type="BUTTON",this.frame.bar.style.position="absolute",this.frame.bar.style.border="1px solid red",this.frame.bar.style.width="100px",this.frame.bar.style.height="6px",this.frame.bar.style.borderRadius="2px",this.frame.bar.style.MozBorderRadius="2px",this.frame.bar.style.border="1px solid #7F7F7F",this.frame.bar.style.backgroundColor="#E5E5E5",this.frame.appendChild(this.frame.bar),this.frame.slide=document.createElement("INPUT"),this.frame.slide.type="BUTTON",this.frame.slide.style.margin="0px",this.frame.slide.value=" ",this.frame.slide.style.position="relative",this.frame.slide.style.left="-100px",this.frame.appendChild(this.frame.slide);var i=this;this.frame.slide.onmousedown=function(t){i._onMouseDown(t)},this.frame.prev.onclick=function(t){i.prev(t)},this.frame.play.onclick=function(t){i.togglePlay(t)},this.frame.next.onclick=function(t){i.next(t)}}this.onChangeCallback=void 0,this.values=[],this.index=void 0,this.playTimeout=void 0,this.playInterval=1e3,this.playLoop=!0}var n=i(1);o.prototype.prev=function(){var t=this.getIndex();t>0&&(t--,this.setIndex(t))},o.prototype.next=function(){var t=this.getIndex();t<this.values.length-1&&(t++,this.setIndex(t))},o.prototype.playNext=function(){var t=new Date,e=this.getIndex();e<this.values.length-1?(e++,this.setIndex(e)):this.playLoop&&(e=0,this.setIndex(e));var i=new Date,o=i-t,n=Math.max(this.playInterval-o,0),s=this;this.playTimeout=setTimeout(function(){s.playNext()},n)},o.prototype.togglePlay=function(){void 0===this.playTimeout?this.play():this.stop()},o.prototype.play=function(){this.playTimeout||(this.playNext(),this.frame&&(this.frame.play.value="Stop"))},o.prototype.stop=function(){clearInterval(this.playTimeout),this.playTimeout=void 0,this.frame&&(this.frame.play.value="Play")},o.prototype.setOnChangeCallback=function(t){this.onChangeCallback=t},o.prototype.setPlayInterval=function(t){this.playInterval=t},o.prototype.getPlayInterval=function(t){return this.playInterval},o.prototype.setPlayLoop=function(t){this.playLoop=t},o.prototype.onChange=function(){void 0!==this.onChangeCallback&&this.onChangeCallback()},o.prototype.redraw=function(){if(this.frame){this.frame.bar.style.top=this.frame.clientHeight/2-this.frame.bar.offsetHeight/2+"px",this.frame.bar.style.width=this.frame.clientWidth-this.frame.prev.clientWidth-this.frame.play.clientWidth-this.frame.next.clientWidth-30+"px";var t=this.indexToLeft(this.index);this.frame.slide.style.left=t+"px"}},o.prototype.setValues=function(t){this.values=t,this.values.length>0?this.setIndex(0):this.index=void 0},o.prototype.setIndex=function(t){if(!(t<this.values.length))throw"Error: index out of range";this.index=t,this.redraw(),this.onChange()},o.prototype.getIndex=function(){return this.index},o.prototype.get=function(){return this.values[this.index]},o.prototype._onMouseDown=function(t){var e=t.which?1===t.which:1===t.button;if(e){this.startClientX=t.clientX,this.startSlideX=parseFloat(this.frame.slide.style.left),this.frame.style.cursor="move";var i=this;this.onmousemove=function(t){i._onMouseMove(t)},this.onmouseup=function(t){i._onMouseUp(t)},n.addEventListener(document,"mousemove",this.onmousemove),n.addEventListener(document,"mouseup",this.onmouseup),n.preventDefault(t)}},o.prototype.leftToIndex=function(t){var e=parseFloat(this.frame.bar.style.width)-this.frame.slide.clientWidth-10,i=t-3,o=Math.round(i/e*(this.values.length-1));return 0>o&&(o=0),o>this.values.length-1&&(o=this.values.length-1),o},o.prototype.indexToLeft=function(t){var e=parseFloat(this.frame.bar.style.width)-this.frame.slide.clientWidth-10,i=t/(this.values.length-1)*e,o=i+3;return o},o.prototype._onMouseMove=function(t){var e=t.clientX-this.startClientX,i=this.startSlideX+e,o=this.leftToIndex(i);this.setIndex(o),n.preventDefault()},o.prototype._onMouseUp=function(t){this.frame.style.cursor="auto",n.removeEventListener(document,"mousemove",this.onmousemove),n.removeEventListener(document,"mouseup",this.onmouseup),n.preventDefault()},t.exports=o},function(t,e){function i(t,e,i,o){this._start=0,this._end=0,this._step=1,this.prettyStep=!0,this.precision=5,this._current=0,this.setRange(t,e,i,o)}i.prototype.setRange=function(t,e,i,o){this._start=t?t:0,this._end=e?e:0,this.setStep(i,o)},i.prototype.setStep=function(t,e){void 0===t||0>=t||(void 0!==e&&(this.prettyStep=e),this.prettyStep===!0?this._step=i.calculatePrettyStep(t):this._step=t)},i.calculatePrettyStep=function(t){var e=function(t){return Math.log(t)/Math.LN10},i=Math.pow(10,Math.round(e(t))),o=2*Math.pow(10,Math.round(e(t/2))),n=5*Math.pow(10,Math.round(e(t/5))),s=i;return Math.abs(o-t)<=Math.abs(s-t)&&(s=o),Math.abs(n-t)<=Math.abs(s-t)&&(s=n),0>=s&&(s=1),s},i.prototype.getCurrent=function(){return parseFloat(this._current.toPrecision(this.precision))},i.prototype.getStep=function(){return this._step},i.prototype.start=function(){this._current=this._start-this._start%this._step},i.prototype.next=function(){this._current+=this._step},i.prototype.end=function(){return this._current>this._end},t.exports=i},function(t,e,i){if("undefined"!=typeof window){var o=i(21),n=window.Hammer||i(22);t.exports=o(n,{preventDefault:"mouse"})}else t.exports=function(){throw Error("hammer.js is only available in a browser, not in node.js.")}},function(t,e,i){var o,n,s;!function(i){n=[],o=i,s="function"==typeof o?o.apply(e,n):o,!(void 0!==s&&(t.exports=s))}(function(){var t=null;return function e(i,o){function n(t){return t.match(/[^ ]+/g)}function s(e){if("hammer.input"!==e.type){if(e.srcEvent._handled||(e.srcEvent._handled={}),e.srcEvent._handled[e.type])return;e.srcEvent._handled[e.type]=!0}var i=!1;e.stopPropagation=function(){i=!0};var o=e.srcEvent.stopPropagation.bind(e.srcEvent);"function"==typeof o&&(e.srcEvent.stopPropagation=function(){o(),e.stopPropagation()}),e.firstTarget=t;for(var n=t;n&&!i;){var s=n.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[e.type])for(var h=0;h<r.length&&!i;h++)r[h](e);n=n.parentNode}}var r=o||{preventDefault:!1};if(i.Manager){var a=i,h=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a(t,o),o)};return a.assign(h,a),h.Manager=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a.Manager(t,o),o)},h}var d=Object.create(i),l=i.element;return l.hammer||(l.hammer=[]),l.hammer.push(d),i.on("hammer.input",function(e){r.preventDefault!==!0&&r.preventDefault!==e.pointerType||e.preventDefault(),e.isFirst&&(t=e.target)}),d._handlers={},d.on=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o||(d._handlers[t]=o=[],i.on(t,s)),o.push(e)}),d},d.off=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o&&(o=e?o.filter(function(t){return t!==e}):[],o.length>0?d._handlers[t]=o:(i.off(t,s),delete d._handlers[t]))}),d},d.emit=function(e,o){t=o.target,i.emit(e,o)},d.destroy=function(){var t=i.element.hammer,e=t.indexOf(d);-1!==e&&t.splice(e,1),t.length||delete i.element.hammer,d._handlers={},i.destroy()},d}})},function(t,e,i){var o;!function(n,s,r,a){function h(t,e,i){return setTimeout(p(t,i),e)}function d(t,e,i){return Array.isArray(t)?(l(t,i[e],i),!0):!1}function l(t,e,i){var o;if(t)if(t.forEach)t.forEach(e,i);else if(t.length!==a)for(o=0;o<t.length;)e.call(i,t[o],o,t),o++;else for(o in t)t.hasOwnProperty(o)&&e.call(i,t[o],o,t)}function c(t,e,i){var o="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",s=n.console&&(n.console.warn||n.console.log);return s&&s.call(n.console,o,i),t.apply(this,arguments)}}function u(t,e,i){var o,n=e.prototype;o=t.prototype=Object.create(n),o.constructor=t,o._super=n,i&&ct(o,i)}function p(t,e){return function(){return t.apply(e,arguments)}}function f(t,e){return typeof t==ft?t.apply(e?e[0]||a:a,e):t}function m(t,e){return t===a?e:t}function v(t,e,i){l(w(e),function(e){t.addEventListener(e,i,!1)})}function g(t,e,i){l(w(e),function(e){t.removeEventListener(e,i,!1)})}function y(t,e){for(;t;){if(t==e)return!0;t=t.parentNode}return!1}function b(t,e){return t.indexOf(e)>-1}function w(t){return t.trim().split(/\s+/g)}function _(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var o=0;o<t.length;){if(i&&t[o][i]==e||!i&&t[o]===e)return o;o++}return-1}function x(t){return Array.prototype.slice.call(t,0)}function k(t,e,i){for(var o=[],n=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];_(n,r)<0&&o.push(t[s]),n[s]=r,s++}return i&&(o=e?o.sort(function(t,i){return t[e]>i[e]}):o.sort()),o}function O(t,e){for(var i,o,n=e[0].toUpperCase()+e.slice(1),s=0;s<ut.length;){if(i=ut[s],o=i?i+n:e,o in t)return o;s++}return a}function M(){return wt++}function D(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||n}function S(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){f(t.options.enable,[t])&&i.handler(e)},this.init()}function C(t){var e,i=t.options.inputClass;return new(e=i?i:kt?W:Ot?V:xt?q:H)(t,T)}function T(t,e,i){var o=i.pointers.length,n=i.changedPointers.length,s=e&Et&&o-n===0,r=e&(It|Nt)&&o-n===0;
-i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,E(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function E(t,e){var i=t.session,o=e.pointers,n=o.length;i.firstInput||(i.firstInput=N(e)),n>1&&!i.firstMultiple?i.firstMultiple=N(e):1===n&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,h=e.center=R(o);e.timeStamp=gt(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=B(a,h),e.distance=A(a,h),P(i,e),e.offsetDirection=L(e.deltaX,e.deltaY);var d=z(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=d.x,e.overallVelocityY=d.y,e.overallVelocity=vt(d.x)>vt(d.y)?d.x:d.y,e.scale=r?j(r.pointers,o):1,e.rotation=r?F(r.pointers,o):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,I(i,e);var l=t.element;y(e.srcEvent.target,l)&&(l=e.srcEvent.target),e.target=l}function P(t,e){var i=e.center,o=t.offsetDelta||{},n=t.prevDelta||{},s=t.prevInput||{};e.eventType!==Et&&s.eventType!==It||(n=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},o=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=n.x+(i.x-o.x),e.deltaY=n.y+(i.y-o.y)}function I(t,e){var i,o,n,s,r=t.lastInterval||e,h=e.timeStamp-r.timeStamp;if(e.eventType!=Nt&&(h>Tt||r.velocity===a)){var d=e.deltaX-r.deltaX,l=e.deltaY-r.deltaY,c=z(h,d,l);o=c.x,n=c.y,i=vt(c.x)>vt(c.y)?c.x:c.y,s=L(d,l),t.lastInterval=e}else i=r.velocity,o=r.velocityX,n=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=o,e.velocityY=n,e.direction=s}function N(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:mt(t.pointers[i].clientX),clientY:mt(t.pointers[i].clientY)},i++;return{timeStamp:gt(),pointers:e,center:R(e),deltaX:t.deltaX,deltaY:t.deltaY}}function R(t){var e=t.length;if(1===e)return{x:mt(t[0].clientX),y:mt(t[0].clientY)};for(var i=0,o=0,n=0;e>n;)i+=t[n].clientX,o+=t[n].clientY,n++;return{x:mt(i/e),y:mt(o/e)}}function z(t,e,i){return{x:e/t||0,y:i/t||0}}function L(t,e){return t===e?Rt:vt(t)>=vt(e)?0>t?zt:Lt:0>e?At:Bt}function A(t,e,i){i||(i=Wt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return Math.sqrt(o*o+n*n)}function B(t,e,i){i||(i=Wt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return 180*Math.atan2(n,o)/Math.PI}function F(t,e){return B(e[1],e[0],Yt)+B(t[1],t[0],Yt)}function j(t,e){return A(e[0],e[1],Yt)/A(t[0],t[1],Yt)}function H(){this.evEl=Vt,this.evWin=Ut,this.allow=!0,this.pressed=!1,S.apply(this,arguments)}function W(){this.evEl=Zt,this.evWin=Kt,S.apply(this,arguments),this.store=this.manager.session.pointerEvents=[]}function Y(){this.evTarget=Qt,this.evWin=$t,this.started=!1,S.apply(this,arguments)}function G(t,e){var i=x(t.touches),o=x(t.changedTouches);return e&(It|Nt)&&(i=k(i.concat(o),"identifier",!0)),[i,o]}function V(){this.evTarget=ee,this.targetIds={},S.apply(this,arguments)}function U(t,e){var i=x(t.touches),o=this.targetIds;if(e&(Et|Pt)&&1===i.length)return o[i[0].identifier]=!0,[i,i];var n,s,r=x(t.changedTouches),a=[],h=this.target;if(s=i.filter(function(t){return y(t.target,h)}),e===Et)for(n=0;n<s.length;)o[s[n].identifier]=!0,n++;for(n=0;n<r.length;)o[r[n].identifier]&&a.push(r[n]),e&(It|Nt)&&delete o[r[n].identifier],n++;return a.length?[k(s.concat(a),"identifier",!0),a]:void 0}function q(){S.apply(this,arguments);var t=p(this.handler,this);this.touch=new V(this.manager,t),this.mouse=new H(this.manager,t)}function X(t,e){this.manager=t,this.set(e)}function Z(t){if(b(t,ae))return ae;var e=b(t,he),i=b(t,de);return e&&i?ae:e||i?e?he:de:b(t,re)?re:se}function K(t){this.options=ct({},this.defaults,t||{}),this.id=M(),this.manager=null,this.options.enable=m(this.options.enable,!0),this.state=le,this.simultaneous={},this.requireFail=[]}function J(t){return t&me?"cancel":t&pe?"end":t&ue?"move":t&ce?"start":""}function Q(t){return t==Bt?"down":t==At?"up":t==zt?"left":t==Lt?"right":""}function $(t,e){var i=e.manager;return i?i.get(t):t}function tt(){K.apply(this,arguments)}function et(){tt.apply(this,arguments),this.pX=null,this.pY=null}function it(){tt.apply(this,arguments)}function ot(){K.apply(this,arguments),this._timer=null,this._input=null}function nt(){tt.apply(this,arguments)}function st(){tt.apply(this,arguments)}function rt(){K.apply(this,arguments),this.pTime=!1,this.pCenter=!1,this._timer=null,this._input=null,this.count=0}function at(t,e){return e=e||{},e.recognizers=m(e.recognizers,at.defaults.preset),new ht(t,e)}function ht(t,e){this.options=ct({},at.defaults,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.element=t,this.input=C(this),this.touchAction=new X(this,this.options.touchAction),dt(this,!0),l(this.options.recognizers,function(t){var e=this.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])},this)}function dt(t,e){var i=t.element;i.style&&l(t.options.cssProps,function(t,o){i.style[O(i.style,o)]=e?t:""})}function lt(t,e){var i=s.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}var ct,ut=["","webkit","Moz","MS","ms","o"],pt=s.createElement("div"),ft="function",mt=Math.round,vt=Math.abs,gt=Date.now;ct="function"!=typeof Object.assign?function(t){if(t===a||null===t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var o=arguments[i];if(o!==a&&null!==o)for(var n in o)o.hasOwnProperty(n)&&(e[n]=o[n])}return e}:Object.assign;var yt=c(function(t,e,i){for(var o=Object.keys(e),n=0;n<o.length;)(!i||i&&t[o[n]]===a)&&(t[o[n]]=e[o[n]]),n++;return t},"extend","Use `assign`."),bt=c(function(t,e){return yt(t,e,!0)},"merge","Use `assign`."),wt=1,_t=/mobile|tablet|ip(ad|hone|od)|android/i,xt="ontouchstart"in n,kt=O(n,"PointerEvent")!==a,Ot=xt&&_t.test(navigator.userAgent),Mt="touch",Dt="pen",St="mouse",Ct="kinect",Tt=25,Et=1,Pt=2,It=4,Nt=8,Rt=1,zt=2,Lt=4,At=8,Bt=16,Ft=zt|Lt,jt=At|Bt,Ht=Ft|jt,Wt=["x","y"],Yt=["clientX","clientY"];S.prototype={handler:function(){},init:function(){this.evEl&&v(this.element,this.evEl,this.domHandler),this.evTarget&&v(this.target,this.evTarget,this.domHandler),this.evWin&&v(D(this.element),this.evWin,this.domHandler)},destroy:function(){this.evEl&&g(this.element,this.evEl,this.domHandler),this.evTarget&&g(this.target,this.evTarget,this.domHandler),this.evWin&&g(D(this.element),this.evWin,this.domHandler)}};var Gt={mousedown:Et,mousemove:Pt,mouseup:It},Vt="mousedown",Ut="mousemove mouseup";u(H,S,{handler:function(t){var e=Gt[t.type];e&Et&&0===t.button&&(this.pressed=!0),e&Pt&&1!==t.which&&(e=It),this.pressed&&this.allow&&(e&It&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:St,srcEvent:t}))}});var qt={pointerdown:Et,pointermove:Pt,pointerup:It,pointercancel:Nt,pointerout:Nt},Xt={2:Mt,3:Dt,4:St,5:Ct},Zt="pointerdown",Kt="pointermove pointerup pointercancel";n.MSPointerEvent&&!n.PointerEvent&&(Zt="MSPointerDown",Kt="MSPointerMove MSPointerUp MSPointerCancel"),u(W,S,{handler:function(t){var e=this.store,i=!1,o=t.type.toLowerCase().replace("ms",""),n=qt[o],s=Xt[t.pointerType]||t.pointerType,r=s==Mt,a=_(e,t.pointerId,"pointerId");n&Et&&(0===t.button||r)?0>a&&(e.push(t),a=e.length-1):n&(It|Nt)&&(i=!0),0>a||(e[a]=t,this.callback(this.manager,n,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))}});var Jt={touchstart:Et,touchmove:Pt,touchend:It,touchcancel:Nt},Qt="touchstart",$t="touchstart touchmove touchend touchcancel";u(Y,S,{handler:function(t){var e=Jt[t.type];if(e===Et&&(this.started=!0),this.started){var i=G.call(this,t,e);e&(It|Nt)&&i[0].length-i[1].length===0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:Mt,srcEvent:t})}}});var te={touchstart:Et,touchmove:Pt,touchend:It,touchcancel:Nt},ee="touchstart touchmove touchend touchcancel";u(V,S,{handler:function(t){var e=te[t.type],i=U.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:Mt,srcEvent:t})}}),u(q,S,{handler:function(t,e,i){var o=i.pointerType==Mt,n=i.pointerType==St;if(o)this.mouse.allow=!1;else if(n&&!this.mouse.allow)return;e&(It|Nt)&&(this.mouse.allow=!0),this.callback(t,e,i)},destroy:function(){this.touch.destroy(),this.mouse.destroy()}});var ie=O(pt.style,"touchAction"),oe=ie!==a,ne="compute",se="auto",re="manipulation",ae="none",he="pan-x",de="pan-y";X.prototype={set:function(t){t==ne&&(t=this.compute()),oe&&this.manager.element.style&&(this.manager.element.style[ie]=t),this.actions=t.toLowerCase().trim()},update:function(){this.set(this.manager.options.touchAction)},compute:function(){var t=[];return l(this.manager.recognizers,function(e){f(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))}),Z(t.join(" "))},preventDefaults:function(t){if(!oe){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)return void e.preventDefault();var o=this.actions,n=b(o,ae),s=b(o,de),r=b(o,he);if(n){var a=1===t.pointers.length,h=t.distance<2,d=t.deltaTime<250;if(a&&h&&d)return}if(!r||!s)return n||s&&i&Ft||r&&i&jt?this.preventSrc(e):void 0}},preventSrc:function(t){this.manager.session.prevented=!0,t.preventDefault()}};var le=1,ce=2,ue=4,pe=8,fe=pe,me=16,ve=32;K.prototype={defaults:{},set:function(t){return ct(this.options,t),this.manager&&this.manager.touchAction.update(),this},recognizeWith:function(t){if(d(t,"recognizeWith",this))return this;var e=this.simultaneous;return t=$(t,this),e[t.id]||(e[t.id]=t,t.recognizeWith(this)),this},dropRecognizeWith:function(t){return d(t,"dropRecognizeWith",this)?this:(t=$(t,this),delete this.simultaneous[t.id],this)},requireFailure:function(t){if(d(t,"requireFailure",this))return this;var e=this.requireFail;return t=$(t,this),-1===_(e,t)&&(e.push(t),t.requireFailure(this)),this},dropRequireFailure:function(t){if(d(t,"dropRequireFailure",this))return this;t=$(t,this);var e=_(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},hasRequireFailures:function(){return this.requireFail.length>0},canRecognizeWith:function(t){return!!this.simultaneous[t.id]},emit:function(t){function e(e){i.manager.emit(e,t)}var i=this,o=this.state;pe>o&&e(i.options.event+J(o)),e(i.options.event),t.additionalEvent&&e(t.additionalEvent),o>=pe&&e(i.options.event+J(o))},tryEmit:function(t){return this.canEmit()?this.emit(t):void(this.state=ve)},canEmit:function(){for(var t=0;t<this.requireFail.length;){if(!(this.requireFail[t].state&(ve|le)))return!1;t++}return!0},recognize:function(t){var e=ct({},t);return f(this.options.enable,[this,e])?(this.state&(fe|me|ve)&&(this.state=le),this.state=this.process(e),void(this.state&(ce|ue|pe|me)&&this.tryEmit(e))):(this.reset(),void(this.state=ve))},process:function(t){},getTouchAction:function(){},reset:function(){}},u(tt,K,{defaults:{pointers:1},attrTest:function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},process:function(t){var e=this.state,i=t.eventType,o=e&(ce|ue),n=this.attrTest(t);return o&&(i&Nt||!n)?e|me:o||n?i&It?e|pe:e&ce?e|ue:ce:ve}}),u(et,tt,{defaults:{event:"pan",threshold:10,pointers:1,direction:Ht},getTouchAction:function(){var t=this.options.direction,e=[];return t&Ft&&e.push(de),t&jt&&e.push(he),e},directionTest:function(t){var e=this.options,i=!0,o=t.distance,n=t.direction,s=t.deltaX,r=t.deltaY;return n&e.direction||(e.direction&Ft?(n=0===s?Rt:0>s?zt:Lt,i=s!=this.pX,o=Math.abs(t.deltaX)):(n=0===r?Rt:0>r?At:Bt,i=r!=this.pY,o=Math.abs(t.deltaY))),t.direction=n,i&&o>e.threshold&&n&e.direction},attrTest:function(t){return tt.prototype.attrTest.call(this,t)&&(this.state&ce||!(this.state&ce)&&this.directionTest(t))},emit:function(t){this.pX=t.deltaX,this.pY=t.deltaY;var e=Q(t.direction);e&&(t.additionalEvent=this.options.event+e),this._super.emit.call(this,t)}}),u(it,tt,{defaults:{event:"pinch",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.scale-1)>this.options.threshold||this.state&ce)},emit:function(t){if(1!==t.scale){var e=t.scale<1?"in":"out";t.additionalEvent=this.options.event+e}this._super.emit.call(this,t)}}),u(ot,K,{defaults:{event:"press",pointers:1,time:251,threshold:9},getTouchAction:function(){return[se]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime>e.time;if(this._input=t,!o||!i||t.eventType&(It|Nt)&&!n)this.reset();else if(t.eventType&Et)this.reset(),this._timer=h(function(){this.state=fe,this.tryEmit()},e.time,this);else if(t.eventType&It)return fe;return ve},reset:function(){clearTimeout(this._timer)},emit:function(t){this.state===fe&&(t&&t.eventType&It?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=gt(),this.manager.emit(this.options.event,this._input)))}}),u(nt,tt,{defaults:{event:"rotate",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.rotation)>this.options.threshold||this.state&ce)}}),u(st,tt,{defaults:{event:"swipe",threshold:10,velocity:.3,direction:Ft|jt,pointers:1},getTouchAction:function(){return et.prototype.getTouchAction.call(this)},attrTest:function(t){var e,i=this.options.direction;return i&(Ft|jt)?e=t.overallVelocity:i&Ft?e=t.overallVelocityX:i&jt&&(e=t.overallVelocityY),this._super.attrTest.call(this,t)&&i&t.offsetDirection&&t.distance>this.options.threshold&&t.maxPointers==this.options.pointers&&vt(e)>this.options.velocity&&t.eventType&It},emit:function(t){var e=Q(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)}}),u(rt,K,{defaults:{event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},getTouchAction:function(){return[re]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime<e.time;if(this.reset(),t.eventType&Et&&0===this.count)return this.failTimeout();if(o&&n&&i){if(t.eventType!=It)return this.failTimeout();var s=this.pTime?t.timeStamp-this.pTime<e.interval:!0,r=!this.pCenter||A(this.pCenter,t.center)<e.posThreshold;this.pTime=t.timeStamp,this.pCenter=t.center,r&&s?this.count+=1:this.count=1,this._input=t;var a=this.count%e.taps;if(0===a)return this.hasRequireFailures()?(this._timer=h(function(){this.state=fe,this.tryEmit()},e.interval,this),ce):fe}return ve},failTimeout:function(){return this._timer=h(function(){this.state=ve},this.options.interval,this),ve},reset:function(){clearTimeout(this._timer)},emit:function(){this.state==fe&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))}}),at.VERSION="2.0.6",at.defaults={domEvents:!1,touchAction:ne,enable:!0,inputTarget:null,inputClass:null,preset:[[nt,{enable:!1}],[it,{enable:!1},["rotate"]],[st,{direction:Ft}],[et,{direction:Ft},["swipe"]],[rt],[rt,{event:"doubletap",taps:2},["tap"]],[ot]],cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}};var ge=1,ye=2;ht.prototype={set:function(t){return ct(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},stop:function(t){this.session.stopped=t?ye:ge},recognize:function(t){var e=this.session;if(!e.stopped){this.touchAction.preventDefaults(t);var i,o=this.recognizers,n=e.curRecognizer;(!n||n&&n.state&fe)&&(n=e.curRecognizer=null);for(var s=0;s<o.length;)i=o[s],e.stopped===ye||n&&i!=n&&!i.canRecognizeWith(n)?i.reset():i.recognize(t),!n&&i.state&(ce|ue|pe)&&(n=e.curRecognizer=i),s++}},get:function(t){if(t instanceof K)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event==t)return e[i];return null},add:function(t){if(d(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},remove:function(t){if(d(t,"remove",this))return this;if(t=this.get(t)){var e=this.recognizers,i=_(e,t);-1!==i&&(e.splice(i,1),this.touchAction.update())}return this},on:function(t,e){var i=this.handlers;return l(w(t),function(t){i[t]=i[t]||[],i[t].push(e)}),this},off:function(t,e){var i=this.handlers;return l(w(t),function(t){e?i[t]&&i[t].splice(_(i[t],e),1):delete i[t]}),this},emit:function(t,e){this.options.domEvents&&lt(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var o=0;o<i.length;)i[o](e),o++}},destroy:function(){this.element&&dt(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null}},ct(at,{INPUT_START:Et,INPUT_MOVE:Pt,INPUT_END:It,INPUT_CANCEL:Nt,STATE_POSSIBLE:le,STATE_BEGAN:ce,STATE_CHANGED:ue,STATE_ENDED:pe,STATE_RECOGNIZED:fe,STATE_CANCELLED:me,STATE_FAILED:ve,DIRECTION_NONE:Rt,DIRECTION_LEFT:zt,DIRECTION_RIGHT:Lt,DIRECTION_UP:At,DIRECTION_DOWN:Bt,DIRECTION_HORIZONTAL:Ft,DIRECTION_VERTICAL:jt,DIRECTION_ALL:Ht,Manager:ht,Input:S,TouchAction:X,TouchInput:V,MouseInput:H,PointerEventInput:W,TouchMouseInput:q,SingleTouchInput:Y,Recognizer:K,AttrRecognizer:tt,Tap:rt,Pan:et,Swipe:st,Pinch:it,Rotate:nt,Press:ot,on:v,off:g,each:l,merge:bt,extend:yt,assign:ct,inherit:u,bindFn:p,prefixed:O});var be="undefined"!=typeof n?n:"undefined"!=typeof self?self:{};be.Hammer=at,o=function(){return at}.call(e,i,e,t),!(o!==a&&(t.exports=o))}(window,document,"Hammer")},function(t,e,i){var o,n,s;!function(i,r){n=[],o=r,s="function"==typeof o?o.apply(e,n):o,!(void 0!==s&&(t.exports=s))}(this,function(){function t(t){var e,i=t&&t.preventDefault||!1,o=t&&t.container||window,n={},s={keydown:{},keyup:{}},r={};for(e=97;122>=e;e++)r[String.fromCharCode(e)]={code:65+(e-97),shift:!1};for(e=65;90>=e;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;9>=e;e++)r[""+e]={code:48+e,shift:!1};for(e=1;12>=e;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;9>=e;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r["delete"]={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){d(t,"keydown")},h=function(t){d(t,"keyup")},d=function(t,e){if(void 0!==s[e][t.keyCode]){for(var o=s[e][t.keyCode],n=0;n<o.length;n++)void 0===o[n].shift?o[n].fn(t):1==o[n].shift&&1==t.shiftKey?o[n].fn(t):0==o[n].shift&&0==t.shiftKey&&o[n].fn(t);1==i&&t.preventDefault()}};return n.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},n.bindAll=function(t,e){void 0===e&&(e="keydown");for(var i in r)r.hasOwnProperty(i)&&n.bind(i,t,e)},n.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},n.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var o=[],n=s[i][r[t].code];if(void 0!==n)for(var a=0;a<n.length;a++)n[a].fn==e&&n[a].shift==r[t].shift||o.push(s[i][r[t].code][a]);s[i][r[t].code]=o}else s[i][r[t].code]=[]},n.reset=function(){s={keydown:{},keyup:{}}},n.destroy=function(){s={keydown:{},keyup:{}},o.removeEventListener("keydown",a,!0),o.removeEventListener("keyup",h,!0)},o.addEventListener("keydown",a,!0),o.addEventListener("keyup",h,!0),n}return t})},function(t,e,i){e.util=i(1),e.DOMutil=i(8),e.DataSet=i(9),e.DataView=i(11),e.Queue=i(10),e.Timeline=i(25),e.Graph2d=i(50),e.timeline={Core:i(33),DateUtil:i(32),Range:i(30),stack:i(37),TimeStep:i(35),components:{items:{Item:i(39),BackgroundItem:i(43),BoxItem:i(41),PointItem:i(42),RangeItem:i(38)},BackgroundGroup:i(40),Component:i(31),CurrentTime:i(48),CustomTime:i(46),DataAxis:i(52),DataScale:i(53),GraphGroup:i(54),Group:i(36),ItemSet:i(34),Legend:i(58),LineGraph:i(51),TimeAxis:i(44)}},e.moment=i(2),e.Hammer=i(20),e.keycharm=i(23)},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e,i,o){if(!(this instanceof n))throw new SyntaxError("Constructor must be called with the new operator");if(!(Array.isArray(i)||i instanceof c||i instanceof u)&&i instanceof Object){var s=o;o=i,i=s}var r=this;this.defaultOptions={start:null,end:null,autoResize:!0,throttleRedraw:0,orientation:{axis:"bottom",item:"bottom"},rtl:!1,moment:d,width:null,height:null,maxHeight:null,minHeight:null},this.options=l.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{getScale:function(){return r.timeAxis.step.scale},getStep:function(){return r.timeAxis.step.step},toScreen:r._toScreen.bind(r),toGlobalScreen:r._toGlobalScreen.bind(r),toTime:r._toTime.bind(r),toGlobalTime:r._toGlobalTime.bind(r)}},this.range=new p(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new m(this.body),this.timeAxis2=null,this.components.push(this.timeAxis),this.currentTime=new v(this.body),this.components.push(this.currentTime),this.itemSet=new y(this.body,this.options),this.components.push(this.itemSet),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){r.emit("click",r.getEventProperties(t))}),this.on("doubletap",function(t){r.emit("doubleClick",r.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){r.emit("contextmenu",r.getEventProperties(t))},this.fitDone=!1,this.on("changed",function(){if(null!=this.itemsData&&!r.fitDone)if(r.fitDone=!0,void 0!=r.options.start||void 0!=r.options.end){if(void 0==r.options.start||void 0==r.options.end)var t=r.getItemRange();var e=void 0!=r.options.start?r.options.start:t.min,i=void 0!=r.options.end?r.options.end:t.max;r.setWindow(e,i,{animation:!1})}else r.fit({animation:!1})}),o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var s=i(26),r=o(s),a=i(29),h=o(a),d=(i(13),i(20),i(2)),l=i(1),c=i(9),u=i(11),p=i(30),f=i(33),m=i(44),v=i(48),g=i(46),y=i(34),b=i(29).printStyle,w=i(49).allOptions,_=i(49).configureOptions;n.prototype=new f,n.prototype._createConfigurator=function(){return new r["default"](this,this.dom.container,_)},n.prototype.redraw=function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()},n.prototype.setOptions=function(t){var e=h["default"].validate(t,w);if(e===!0&&console.log("%cErrors have been found in the supplied options object.",b),f.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var i=this.itemsData;if(i){var o=this.getSelection();this.setItems(null),this.setItems(i),this.setSelection(o)}}},n.prototype.setItems=function(t){var e;e=t?t instanceof c||t instanceof u?t:new c(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.itemSet&&this.itemSet.setItems(e)},n.prototype.setGroups=function(t){var e;e=t?t instanceof c||t instanceof u?t:new c(t):null,this.groupsData=e,this.itemSet.setGroups(e)},n.prototype.setData=function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)},n.prototype.setSelection=function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)},n.prototype.getSelection=function(){return this.itemSet&&this.itemSet.getSelection()||[]},n.prototype.focus=function(t,e){if(this.itemsData&&void 0!=t){var i=Array.isArray(t)?t:[t],o=this.itemsData.getDataSet().get(i,{type:{start:"Date",end:"Date"}}),n=null,s=null;if(o.forEach(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===n||n>e)&&(n=e),(null===s||i>s)&&(s=i)}),null!==n&&null!==s){var r=(n+s)/2,a=Math.max(this.range.end-this.range.start,1.1*(s-n)),h=e&&void 0!==e.animation?e.animation:!0;this.range.setRange(r-a/2,r+a/2,h)}}},n.prototype.fit=function(t){var e,i=t&&void 0!==t.animation?t.animation:!0,o=this.itemsData&&this.itemsData.getDataSet();1===o.length&&void 0===o.get()[0].end?(e=this.getDataRange(),this.moveTo(e.min.valueOf(),{animation:i})):(e=this.getItemRange(),this.range.setRange(e.min,e.max,i))},n.prototype.getItemRange=function(){var t=this,e=this.getDataRange(),i=null!==e.min?e.min.valueOf():null,o=null!==e.max?e.max.valueOf():null,n=null,s=null;if(null!=i&&null!=o){var r,a,h,d,c;!function(){var e=function(t){return l.convert(t.data.start,"Date").valueOf()},u=function(t){var e=void 0!=t.data.end?t.data.end:t.data.start;return l.convert(e,"Date").valueOf()};r=o-i,0>=r&&(r=10),a=r/t.props.center.width,l.forEach(t.itemSet.items,function(t){t.show(),t.repositionX();var r=e(t),h=u(t);if(this.options.rtl)var d=r-(t.getWidthRight()+10)*a,l=h+(t.getWidthLeft()+10)*a;else var d=r-(t.getWidthLeft()+10)*a,l=h+(t.getWidthRight()+10)*a;i>d&&(i=d,n=t),l>o&&(o=l,s=t)}.bind(t)),n&&s&&(h=n.getWidthLeft()+10,d=s.getWidthRight()+10,c=t.props.center.width-h-d,c>0&&(t.options.rtl?(i=e(n)-d*r/c,o=u(s)+h*r/c):(i=e(n)-h*r/c,o=u(s)+d*r/c)))}()}return{min:null!=i?new Date(i):null,max:null!=o?new Date(o):null}},n.prototype.getDataRange=function(){var t=null,e=null,i=this.itemsData&&this.itemsData.getDataSet();return i&&i.forEach(function(i){var o=l.convert(i.start,"Date").valueOf(),n=l.convert(void 0!=i.end?i.end:i.start,"Date").valueOf();(null===t||t>o)&&(t=o),(null===e||n>e)&&(e=n)}),{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},n.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY;if(this.options.rtl)var o=l.getAbsoluteRight(this.dom.centerContainer)-e;else var o=e-l.getAbsoluteLeft(this.dom.centerContainer);var n=i-l.getAbsoluteTop(this.dom.centerContainer),s=this.itemSet.itemFromTarget(t),r=this.itemSet.groupFromTarget(t),a=g.customTimeFromTarget(t),h=this.itemSet.options.snap||null,d=this.body.util.getScale(),c=this.body.util.getStep(),u=this._toTime(o),p=h?h(u,d,c):u,f=l.getTarget(t),m=null;return null!=s?m="item":null!=a?m="custom-time":l.hasParent(f,this.timeAxis.dom.foreground)?m="axis":this.timeAxis2&&l.hasParent(f,this.timeAxis2.dom.foreground)?m="axis":l.hasParent(f,this.itemSet.dom.labelSet)?m="group-label":l.hasParent(f,this.currentTime.bar)?m="current-time":l.hasParent(f,this.dom.center)&&(m="background"),{event:t,item:s?s.id:null,group:r?r.groupId:null,what:m,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:u,snappedTime:p}},t.exports=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(27),h=o(a),d=i(1),l=function(){function t(e,i,o){var s=arguments.length<=3||void 0===arguments[3]?1:arguments[3];n(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},d.extend(this.options,this.defaultOptions),this.configureOptions=o,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new h["default"](s),this.wrapper=void 0}return r(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;"string"==typeof t?this.options.filter=t:t instanceof Array?this.options.filter=t.join():"object"===("undefined"==typeof t?"undefined":s(t))?(void 0!==t.container&&(this.options.container=t.container),void 0!==t.filter&&(this.options.filter=t.filter),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)):"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0),this.options.filter===!1&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,this.options.enabled===!0&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){var t=this;this._clean(),this.changedOptions=[];var e=this.options.filter,i=0,o=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,o=!1,"function"==typeof e?(o=e(n,[]),o=o||this._handleObject(this.configureOptions[n],[n],!0)):e!==!0&&-1===e.indexOf(n)||(o=!0),o!==!1&&(this.allowCreation=!0,i>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),i++);this.options.showButton===!0&&!function(){var e=document.createElement("div");e.className="vis-configuration vis-config-button",e.innerHTML="generate options",e.onclick=function(){t._printOptions()},e.onmouseover=function(){e.className="vis-configuration vis-config-button hover"},e.onmouseout=function(){e.className="vis-configuration vis-config-button"},t.optionsContainer=document.createElement("div"),t.optionsContainer.className="vis-configuration vis-config-option-container",t.domElements.push(t.optionsContainer),t.domElements.push(e)}(),this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){var e=arguments,i=this;if(this.allowCreation===!0){var o,n,r,a=function(){var s=document.createElement("div");for(s.className="vis-configuration vis-config-item vis-config-s"+t.length,o=e.length,n=Array(o>1?o-1:0),r=1;o>r;r++)n[r-1]=e[r];return n.forEach(function(t){s.appendChild(t)}),i.domElements.push(s),{v:i.domElements.length}}();if("object"===("undefined"==typeof a?"undefined":s(a)))return a.v}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=document.createElement("div");return o.className="vis-configuration vis-config-label vis-config-s"+e.length,i===!0?o.innerHTML="<i><b>"+t+":</b></i>":o.innerHTML=t+":",o}},{key:"_makeDropdown",value:function(t,e,i){var o=document.createElement("select");o.className="vis-configuration vis-config-select";var n=0;void 0!==e&&-1!==t.indexOf(e)&&(n=t.indexOf(e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===n&&(r.selected="selected"),r.innerHTML=t[s],o.appendChild(r)}var a=this;o.onchange=function(){a._update(this.value,i)};var h=this._makeLabel(i[i.length-1],i);this._makeItem(i,h,o)}},{key:"_makeRange",value:function(t,e,i){
-var o=t[0],n=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=n,a.max=s}catch(h){}a.step=r;var d="",l=0;if(void 0!==e){var c=1.2;0>e&&n>e*c?(a.min=Math.ceil(e*c),l=a.min,d="range increased"):n>e/c&&(a.min=Math.ceil(e/c),l=a.min,d="range increased"),e*c>s&&1!==s&&(a.max=Math.ceil(e*c),l=a.max,d="range increased"),a.value=e}else a.value=o;var u=document.createElement("input");u.className="vis-configuration vis-config-rangeinput",u.value=a.value;var p=this;a.onchange=function(){u.value=this.value,p._update(Number(this.value),i)},a.oninput=function(){u.value=this.value};var f=this._makeLabel(i[i.length-1],i),m=this._makeItem(i,f,a,u);""!==d&&this.popupHistory[m]!==l&&(this.popupHistory[m]=l,this._setupPopup(d,m))}},{key:"_setupPopup",value:function(t,e){var i=this;if(this.initialized===!0&&this.allowCreation===!0&&this.popupCounter<this.popupLimit){var o=document.createElement("div");o.id="vis-configuration-popup",o.className="vis-configuration-popup",o.innerHTML=t,o.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:o,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index],i=e.getBoundingClientRect();this.popupDiv.html.style.left=i.left+"px",this.popupDiv.html.style.top=i.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=setTimeout(function(){t.popupDiv.html.style.opacity=0},1500),this.popupDiv.deleteTimeout=setTimeout(function(){t._removePopup()},1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var o=document.createElement("input");o.type="checkbox",o.className="vis-configuration vis-config-checkbox",o.checked=t,void 0!==e&&(o.checked=e,e!==t&&("object"===("undefined"==typeof t?"undefined":s(t))?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var n=this;o.onchange=function(){n._update(this.checked,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,o)}},{key:"_makeTextInput",value:function(t,e,i){var o=document.createElement("input");o.type="text",o.className="vis-configuration vis-config-text",o.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var n=this;o.onchange=function(){n._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeColorField",value:function(t,e,i){var o=this,n=t[1],s=document.createElement("div");e=void 0===e?n:e,"none"!==e?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?n:e,s.onclick=function(){o._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var o=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback(function(t){var n="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=n,o._update(n,i)}),this.colorPicker.setCloseCallback(function(){e.onclick=function(){o._showColorPicker(t,e,i)}})}},{key:"_handleObject",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?[]:arguments[1],i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=!1,n=this.options.filter,s=!1;for(var r in t)if(t.hasOwnProperty(r)){o=!0;var a=t[r],h=d.copyAndExtendArray(e,r);if("function"==typeof n&&(o=n(r,e),o===!1&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,o=this._handleObject(a,h,!0),this.allowCreation=i===!1)),o!==!1){s=!0;var l=this._getValue(h);if(a instanceof Array)this._handleArray(a,l,h);else if("string"==typeof a)this._makeTextInput(a,l,h);else if("boolean"==typeof a)this._makeCheckbox(a,l,h);else if(a instanceof Object){var c=!0;if(-1!==e.indexOf("physics")&&this.moduleOptions.physics.solver!==r&&(c=!1),c===!0)if(void 0!==a.enabled){var u=d.copyAndExtendArray(h,"enabled"),p=this._getValue(u);if(p===!0){var f=this._makeLabel(r,h,!0);this._makeItem(h,f),s=this._handleObject(a,h)||s}else this._makeCheckbox(a,p,h)}else{var m=this._makeLabel(r,h,!0);this._makeItem(h,m),s=this._handleObject(a,h)||s}}else console.error("dont know how to handle",a,r,h)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?{}:arguments[2],o=i;t="true"===t?!0:t,t="false"===t?!1:t;for(var n=0;n<e.length;n++)"global"!==e[n]&&(void 0===o[e[n]]&&(o[e[n]]={}),n!==e.length-1?o=o[e[n]]:o[e[n]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+JSON.stringify(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}();e["default"]=l},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(20),r=i(28),a=i(1),h=function(){function t(){var e=arguments.length<=0||void 0===arguments[0]?1:arguments[0];o(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return n(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){var e={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"};return"string"==typeof t?e[t]:void 0}},{key:"setColor",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];if("none"!==t){var i=void 0,o=this._isColorString(t);if(void 0!==o&&(t=o),a.isString(t)===!0){if(a.isValidRGB(t)===!0){var n=t.substr(4).substr(0,t.length-5).split(",");i={r:n[0],g:n[1],b:n[2],a:1}}else if(a.isValidRGBA(t)===!0){var s=t.substr(5).substr(0,t.length-6).split(",");i={r:s[0],g:s[1],b:s[2],a:s[3]}}else if(a.isValidHex(t)===!0){var r=a.hexToRGB(t);i={r:r.r,g:r.g,b:r.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var h=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:h}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+JSON.stringify(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?!0:arguments[0];e===!0&&(this.previousColor=a.extend({},this.color)),this.applied===!0&&this.updateCallback(this.initialColor),this.frame.style.display="none",setTimeout(function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)},0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];e===!0&&(this.initialColor=a.extend({},t)),this.color=t;var i=a.RGBToHSV(t.r,t.g,t.b),o=2*Math.PI,n=this.r*i.s,s=this.centerCoordinates.x+n*Math.sin(o*i.h),r=this.centerCoordinates.y+n*Math.cos(o*i.h);this.colorPickerSelector.style.left=s-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=r-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=a.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=a.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length<=0||void 0===arguments[0]?this.color:arguments[0],e=a.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var o=this.colorPickerCanvas.clientWidth,n=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,o,n),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),i.fill(),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var t=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(e)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(i){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(i){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var o=this;this.opacityRange.onchange=function(){o._setOpacity(this.value)},this.opacityRange.oninput=function(){o._setOpacity(this.value)},this.brightnessRange.onchange=function(){o._setBrightness(this.value)},this.brightnessRange.oninput=function(){o._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=this._hide.bind(this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=this._apply.bind(this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=this._save.bind(this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=this._loadLast.bind(this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new s(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),r.onTouch(this.hammer,function(e){t._moveSelector(e)}),this.hammer.on("tap",function(e){t._moveSelector(e)}),this.hammer.on("panstart",function(e){t._moveSelector(e)}),this.hammer.on("panmove",function(e){t._moveSelector(e)}),this.hammer.on("panend",function(e){t._moveSelector(e)})}},{key:"_generateHueCircle",value:function(){if(this.generated===!1){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e=this.colorPickerCanvas.clientWidth,i=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,e,i);var o=void 0,n=void 0,s=void 0,r=void 0;this.centerCoordinates={x:.5*e,y:.5*i},this.r=.49*e;var h=2*Math.PI/360,d=1/360,l=1/this.r,c=void 0;for(s=0;360>s;s++)for(r=0;r<this.r;r++)o=this.centerCoordinates.x+r*Math.sin(h*s),n=this.centerCoordinates.y+r*Math.cos(h*s),c=a.HSVToRGB(s*d,r*l,1),t.fillStyle="rgb("+c.r+","+c.g+","+c.b+")",t.fillRect(o-.5,n-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,e,i)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,o=t.center.y-e.top,n=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,h=o-n,d=Math.atan2(r,h),l=.98*Math.min(Math.sqrt(r*r+h*h),s),c=Math.cos(d)*l+n,u=Math.sin(d)*l+s;this.colorPickerSelector.style.top=c-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var p=d/(2*Math.PI);p=0>p?p+1:p;var f=l/this.r,m=a.RGBToHSV(this.color.r,this.color.g,this.color.b);m.h=p,m.s=f;var v=a.HSVToRGB(m.h,m.s,m.v);v.a=this.color.a,this.color=v,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();e["default"]=h},function(t,e,i){i(20);e.onTouch=function(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)},e.onRelease=function(t,e){return e.inputHandler=function(t){t.isFinal&&e(t)},t.on("hammer.input",e.inputHandler)},e.offTouch=function(t,e){t.off("hammer.input",e.inputHandler)},e.offRelease=e.offTouch,e.disablePreventDefaultVertically=function(t){var e="pan-y";return t.getTouchAction=function(){return[e]},t}},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(1),a=!1,h=void 0,d="background: #FFeeee; color: #dd0000",l=function(){function t(){o(this,t)}return s(t,null,[{key:"validate",value:function(e,i,o){a=!1,h=i;var n=i;return void 0!==o&&(n=i[o]),t.parse(e,n,[]),a}},{key:"parse",value:function(e,i,o){for(var n in e)e.hasOwnProperty(n)&&t.check(n,e,i,o)}},{key:"check",value:function(e,i,o,n){void 0===o[e]&&void 0===o.__any__?t.getSuggestion(e,o,n):void 0===o[e]&&void 0!==o.__any__?"object"===t.getType(i[e])&&void 0!==o.__any__.__type__?t.checkFields(e,i,o,"__any__",o.__any__.__type__,n):t.checkFields(e,i,o,"__any__",o.__any__,n):void 0!==o[e].__type__?t.checkFields(e,i,o,e,o[e].__type__,n):t.checkFields(e,i,o,e,o[e],n)}},{key:"checkFields",value:function(e,i,o,n,s,h){var l=t.getType(i[e]),c=s[l];void 0!==c?"array"===t.getType(c)&&-1===c.indexOf(i[e])?(console.log('%cInvalid option detected in "'+e+'". Allowed values are:'+t.print(c)+' not "'+i[e]+'". '+t.printLocation(h,e),d),a=!0):"object"===l&&"__any__"!==n&&(h=r.copyAndExtendArray(h,e),t.parse(i[e],o[n],h)):void 0===s.any&&(console.log('%cInvalid type received for "'+e+'". Expected: '+t.print(Object.keys(s))+". Received ["+l+'] "'+i[e]+'"'+t.printLocation(h,e),d),a=!0)}},{key:"getType",value:function(t){var e="undefined"==typeof t?"undefined":n(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Array.isArray(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":t._isAMomentObject===!0?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,o){var n=t.findInOptions(e,i,o,!1),s=t.findInOptions(e,h,[],!0),r=8,l=4;void 0!==n.indexMatch?console.log('%cUnknown option detected: "'+e+'" in '+t.printLocation(n.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+n.indexMatch+'"?\n\n',d):s.distance<=l&&n.distance>s.distance?console.log('%cUnknown option detected: "'+e+'" in '+t.printLocation(n.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(s.path,s.closestMatch,""),d):n.distance<=r?console.log('%cUnknown option detected: "'+e+'". Did you mean "'+n.closestMatch+'"?'+t.printLocation(n.path,e),d):console.log('%cUnknown option detected: "'+e+'". Did you mean one of these: '+t.print(Object.keys(i))+t.printLocation(o,e),d),a=!0}},{key:"findInOptions",value:function(e,i,o){var n=arguments.length<=3||void 0===arguments[3]?!1:arguments[3],s=1e9,a="",h=[],d=e.toLowerCase(),l=void 0;for(var c in i){var u=void 0;if(void 0!==i[c].__type__&&n===!0){var p=t.findInOptions(e,i[c],r.copyAndExtendArray(o,c));s>p.distance&&(a=p.closestMatch,h=p.path,s=p.distance,l=p.indexMatch)}else-1!==c.toLowerCase().indexOf(d)&&(l=c),u=t.levenshteinDistance(e,c),s>u&&(a=c,h=r.copyArray(o),s=u)}return{closestMatch:a,path:h,distance:s,indexMatch:l}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length<=2||void 0===arguments[2]?"Problem value found at: \n":arguments[2],o="\n\n"+i+"options = {\n",n=0;n<t.length;n++){for(var s=0;n+1>s;s++)o+="  ";o+=t[n]+": {\n"}for(var r=0;r<t.length+1;r++)o+="  ";o+=e+"\n";for(var a=0;a<t.length+1;a++){for(var h=0;h<t.length-a;h++)o+="  ";o+="}\n"}return o+"\n\n"}},{key:"print",value:function(t){return JSON.stringify(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,o=[];for(i=0;i<=e.length;i++)o[i]=[i];var n;for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}();e["default"]=l,e.printStyle=d},function(t,e,i){function o(t,e){var i=a().hours(0).minutes(0).seconds(0).milliseconds(0);this.start=i.clone().add(-3,"days").valueOf(),this.end=i.clone().add(4,"days").valueOf(),this.body=t,this.deltaDifference=0,this.scaleOffset=0,this.startToFront=!1,this.endToFront=!0,this.defaultOptions={rtl:!1,start:null,end:null,moment:a,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10},this.options=r.extend({},this.defaultOptions),this.props={touch:{}},this.animationTimer=null,this.body.emitter.on("panstart",this._onDragStart.bind(this)),this.body.emitter.on("panmove",this._onDrag.bind(this)),this.body.emitter.on("panend",this._onDragEnd.bind(this)),this.body.emitter.on("mousewheel",this._onMouseWheel.bind(this)),this.body.emitter.on("touch",this._onTouch.bind(this)),this.body.emitter.on("pinch",this._onPinch.bind(this)),this.setOptions(e)}function n(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'+t+'". Choose "horizontal" or "vertical".')}var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=i(1),a=(i(28),i(2)),h=i(31),d=i(32);o.prototype=new h,o.prototype.setOptions=function(t){if(t){var e=["direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","rtl"];r.selectiveExtend(e,this.options,t),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}},o.prototype.setRange=function(t,e,i,o){o!==!0&&(o=!1);var n=void 0!=t?r.convert(t,"Date").valueOf():null,a=void 0!=e?r.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),i){var h=this,l=this.start,c=this.end,u="object"===("undefined"==typeof i?"undefined":s(i))&&"duration"in i?i.duration:500,p="object"===("undefined"==typeof i?"undefined":s(i))&&"easingFunction"in i?i.easingFunction:"easeInOutQuad",f=r.easingFunctions[p];if(!f)throw new Error("Unknown easing function "+JSON.stringify(p)+". Choose from: "+Object.keys(r.easingFunctions).join(", "));var m=(new Date).valueOf(),v=!1,g=function w(){if(!h.props.touch.dragging){var t=(new Date).valueOf(),e=t-m,i=f(e/u),s=e>u,r=s||null===n?n:l+(n-l)*i,p=s||null===a?a:c+(a-c)*i;y=h._applyRange(r,p),d.updateHiddenDates(h.options.moment,h.body,h.options.hiddenDates),v=v||y,y&&h.body.emitter.emit("rangechange",{start:new Date(h.start),end:new Date(h.end),byUser:o}),s?v&&h.body.emitter.emit("rangechanged",{start:new Date(h.start),end:new Date(h.end),byUser:o}):h.animationTimer=setTimeout(w,20)}};return g()}var y=this._applyRange(n,a);if(d.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),y){var b={start:new Date(this.start),end:new Date(this.end),byUser:o};this.body.emitter.emit("rangechange",b),this.body.emitter.emit("rangechanged",b)}},o.prototype._cancelAnimation=function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)},o.prototype._applyRange=function(t,e){var i,o=null!=t?r.convert(t,"Date").valueOf():this.start,n=null!=e?r.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?r.convert(this.options.max,"Date").valueOf():null,a=null!=this.options.min?r.convert(this.options.min,"Date").valueOf():null;if(isNaN(o)||null===o)throw new Error('Invalid start "'+t+'"');if(isNaN(n)||null===n)throw new Error('Invalid end "'+e+'"');if(o>n&&(n=o),null!==a&&a>o&&(i=a-o,o+=i,n+=i,null!=s&&n>s&&(n=s)),null!==s&&n>s&&(i=n-s,o-=i,n-=i,null!=a&&a>o&&(o=a)),null!==this.options.zoomMin){var h=parseFloat(this.options.zoomMin);0>h&&(h=0),h>n-o&&(this.end-this.start===h&&o>this.start&&n<this.end?(o=this.start,n=this.end):(i=h-(n-o),o-=i/2,n+=i/2))}if(null!==this.options.zoomMax){var d=parseFloat(this.options.zoomMax);0>d&&(d=0),n-o>d&&(this.end-this.start===d&&o<this.start&&n>this.end?(o=this.start,n=this.end):(i=n-o-d,o+=i/2,n-=i/2))}var l=this.start!=o||this.end!=n;return o>=this.start&&o<=this.end||n>=this.start&&n<=this.end||this.start>=o&&this.start<=n||this.end>=o&&this.end<=n||this.body.emitter.emit("checkRangedItems"),this.start=o,this.end=n,l},o.prototype.getRange=function(){return{start:this.start,end:this.end}},o.prototype.conversion=function(t,e){return o.conversion(this.start,this.end,t,e)},o.conversion=function(t,e,i,o){return void 0===o&&(o=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-o)}:{offset:0,scale:1}},o.prototype._onDragStart=function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))},o.prototype._onDrag=function(t){if(this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;n(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var o=this.props.touch.end-this.props.touch.start,s=d.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end);o-=s;var r="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;if(this.options.rtl)var a=i/r*o;else var a=-i/r*o;var h=this.props.touch.start+a,l=this.props.touch.end+a,c=d.snapAwayFromHidden(this.body.hiddenDates,h,this.previousDelta-i,!0),u=d.snapAwayFromHidden(this.body.hiddenDates,l,this.previousDelta-i,!0);if(c!=h||u!=l)return this.deltaDifference+=i,this.props.touch.start=c,this.props.touch.end=u,void this._onDrag(t);this.previousDelta=i,this._applyRange(h,l);var p=new Date(this.start),f=new Date(this.end);this.body.emitter.emit("rangechange",{start:p,end:f,byUser:!0})}},o.prototype._onDragEnd=function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0}))},o.prototype._onMouseWheel=function(t){if(this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&(!this.options.zoomKey||t[this.options.zoomKey])){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),e){var i;i=0>e?1-e/5:1/(1+e/5);var o=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center),n=this._pointerToDate(o);this.zoom(i,n,e)}t.preventDefault()}},o.prototype._onTouch=function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.scaleOffset=0,this.deltaDifference=0},o.prototype._onPinch=function(t){if(this.options.zoomable&&this.options.moveable){this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center));var e=1/(t.scale+this.scaleOffset),i=this._pointerToDate(this.props.touch.center),o=d.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,i),s=o-n,r=i-n+(this.props.touch.start-(i-n))*e,a=i+s+(this.props.touch.end-(i+s))*e;
-this.startToFront=0>=1-e,this.endToFront=0>=e-1;var h=d.snapAwayFromHidden(this.body.hiddenDates,r,1-e,!0),l=d.snapAwayFromHidden(this.body.hiddenDates,a,e-1,!0);h==r&&l==a||(this.props.touch.start=h,this.props.touch.end=l,this.scaleOffset=1-t.scale,r=h,a=l),this.setRange(r,a,!1,!0),this.startToFront=!1,this.endToFront=!0}},o.prototype._isInsideRange=function(t){var e=t.center?t.center.x:t.clientX;if(this.options.rtl)var i=e-r.getAbsoluteLeft(this.body.dom.centerContainer);else var i=r.getAbsoluteRight(this.body.dom.centerContainer)-e;var o=this.body.util.toTime(i);return o>=this.start&&o<=this.end},o.prototype._pointerToDate=function(t){var e,i=this.options.direction;if(n(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var o=this.body.domProps.center.height;return e=this.conversion(o),t.y/e.scale+e.offset},o.prototype.getPointer=function(t,e){return this.options.rtl?{x:r.getAbsoluteRight(e)-t.x,y:t.y-r.getAbsoluteTop(e)}:{x:t.x-r.getAbsoluteLeft(e),y:t.y-r.getAbsoluteTop(e)}},o.prototype.zoom=function(t,e,i){null==e&&(e=(this.start+this.end)/2);var o=d.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,e),s=o-n,r=e-n+(this.start-(e-n))*t,a=e+s+(this.end-(e+s))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var h=d.snapAwayFromHidden(this.body.hiddenDates,r,i,!0),l=d.snapAwayFromHidden(this.body.hiddenDates,a,-i,!0);h==r&&l==a||(r=h,a=l),this.setRange(r,a,!1,!0),this.startToFront=!1,this.endToFront=!0},o.prototype.move=function(t){var e=this.end-this.start,i=this.start+e*t,o=this.end+e*t;this.start=i,this.end=o},o.prototype.moveTo=function(t){var e=(this.start+this.end)/2,i=e-t,o=this.start-i,n=this.end-i;this.setRange(o,n)},t.exports=o},function(t,e){function i(t,e){this.options=null,this.props=null}i.prototype.setOptions=function(t){t&&util.extend(this.options,t)},i.prototype.redraw=function(){return!1},i.prototype.destroy=function(){},i.prototype._isResized=function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t},t.exports=i},function(t,e){e.convertHiddenOptions=function(t,i,o){if(o&&!Array.isArray(o))return e.convertHiddenOptions(t,i,[o]);if(i.hiddenDates=[],o&&1==Array.isArray(o)){for(var n=0;n<o.length;n++)if(void 0===o[n].repeat){var s={};s.start=t(o[n].start).toDate().valueOf(),s.end=t(o[n].end).toDate().valueOf(),i.hiddenDates.push(s)}i.hiddenDates.sort(function(t,e){return t.start-e.start})}},e.updateHiddenDates=function(t,i,o){if(o&&!Array.isArray(o))return e.updateHiddenDates(t,i,[o]);if(o&&void 0!==i.domProps.centerContainer.width){e.convertHiddenOptions(t,i,o);for(var n=t(i.range.start),s=t(i.range.end),r=i.range.end-i.range.start,a=r/i.domProps.centerContainer.width,h=0;h<o.length;h++)if(void 0!==o[h].repeat){var d=t(o[h].start),l=t(o[h].end);if("Invalid Date"==d._d)throw new Error("Supplied start date is not valid: "+o[h].start);if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: "+o[h].end);var c=l-d;if(c>=4*a){var u=0,p=s.clone();switch(o[h].repeat){case"daily":d.day()!=l.day()&&(u=1),d.dayOfYear(n.dayOfYear()),d.year(n.year()),d.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-u,"days"),p.add(1,"weeks");break;case"weekly":var f=l.diff(d,"days"),m=d.day();d.date(n.date()),d.month(n.month()),d.year(n.year()),l=d.clone(),d.day(m),l.day(m),l.add(f,"days"),d.subtract(1,"weeks"),l.subtract(1,"weeks"),p.add(1,"weeks");break;case"monthly":d.month()!=l.month()&&(u=1),d.month(n.month()),d.year(n.year()),d.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(u,"months"),p.add(1,"months");break;case"yearly":d.year()!=l.year()&&(u=1),d.year(n.year()),d.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(u,"years"),p.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}for(;p>d;)switch(i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()}),o[h].repeat){case"daily":d.add(1,"days"),l.add(1,"days");break;case"weekly":d.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":d.add(1,"months"),l.add(1,"months");break;case"yearly":d.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()})}}e.removeDuplicates(i);var v=e.isHidden(i.range.start,i.hiddenDates),g=e.isHidden(i.range.end,i.hiddenDates),y=i.range.start,b=i.range.end;1==v.hidden&&(y=1==i.range.startToFront?v.startDate-1:v.endDate+1),1==g.hidden&&(b=1==i.range.endToFront?g.startDate-1:g.endDate+1),1!=v.hidden&&1!=g.hidden||i.range._applyRange(y,b)}},e.removeDuplicates=function(t){for(var e=t.hiddenDates,i=[],o=0;o<e.length;o++)for(var n=0;n<e.length;n++)o!=n&&1!=e[n].remove&&1!=e[o].remove&&(e[n].start>=e[o].start&&e[n].end<=e[o].end?e[n].remove=!0:e[n].start>=e[o].start&&e[n].start<=e[o].end?(e[o].end=e[n].end,e[n].remove=!0):e[n].end>=e[o].start&&e[n].end<=e[o].end&&(e[o].start=e[n].start,e[n].remove=!0));for(var o=0;o<e.length;o++)e[o].remove!==!0&&i.push(e[o]);t.hiddenDates=i,t.hiddenDates.sort(function(t,e){return t.start-e.start})},e.printDates=function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},e.stepOverHiddenDates=function(t,e,i){for(var o=!1,n=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(n>=r&&a>n){o=!0;break}}if(1==o&&n<e._end.valueOf()&&n!=i){var h=t(i),d=t(a);h.year()!=d.year()?e.switchedYear=!0:h.month()!=d.month()?e.switchedMonth=!0:h.dayOfYear()!=d.dayOfYear()&&(e.switchedDay=!0),e.current=d}},e.toScreen=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return(i.valueOf()-n.offset)*n.scale}var s=e.isHidden(i,t.body.hiddenDates);1==s.hidden&&(i=s.startDate);var r=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end);i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,t.range,i);var n=t.range.conversion(o,r);return(i.valueOf()-n.offset)*n.scale},e.toTime=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return new Date(i/n.scale+n.offset)}var s=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end),r=t.range.end-t.range.start-s,a=r*i/o,h=e.getAccumulatedHiddenDuration(t.body.hiddenDates,t.range,a),d=new Date(h+a+t.range.start);return d},e.getHiddenDurationBetween=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&i>r&&(o+=r-s)}return o},e.correctTimeForHidden=function(t,i,o,n){return n=t(n).toDate().valueOf(),n-=e.getHiddenDurationBefore(t,i,o,n)},e.getHiddenDurationBefore=function(t,e,i,o){var n=0;o=t(o).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&o>=a&&(n+=a-r)}return n},e.getAccumulatedHiddenDuration=function(t,e,i){for(var o=0,n=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,h=t[r].end;if(a>=e.start&&h<e.end){if(n+=a-s,s=h,n>=i)break;o+=h-a}}return o},e.snapAwayFromHidden=function(t,i,o,n){var s=e.isHidden(i,t);return 1==s.hidden?0>o?1==n?s.startDate-(s.endDate-i)-1:s.startDate-1:1==n?s.endDate+(i-s.startDate)+1:s.endDate+1:i},e.isHidden=function(t,e){for(var i=0;i<e.length;i++){var o=e[i].start,n=e[i].end;if(t>=o&&n>t)return{hidden:!0,startDate:o,endDate:n}}return{hidden:!1,startDate:o,endDate:n}}},function(t,e,i){function o(){}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(13),r=i(20),a=i(28),h=i(1),d=(i(9),i(11),i(30),i(34),i(44)),l=i(45),c=i(32),u=i(46);s(o.prototype),o.prototype._create=function(t){function e(t){i.isActive()&&i.emit("mousewheel",t)}this.dom={},this.dom.container=t,this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.on("rangechange",function(){this.initialDrawDone===!0&&this._redraw()}.bind(this)),this.on("touch",this._onTouch.bind(this)),this.on("pan",this._onDrag.bind(this));var i=this;this.on("_change",function(t){t&&1==t.queue?i._redrawTimer||(i._redrawTimer=setTimeout(function(){i._redrawTimer=null,i._redraw()},0)):i._redraw()}),this.hammer=new r(this.dom.root);var o=this.hammer.get("pinch").set({enable:!0});a.disablePreventDefaultVertically(o),this.hammer.get("pan").set({threshold:5,direction:r.DIRECTION_HORIZONTAL}),this.listeners={};var n=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];if(n.forEach(function(t){var e=function(e){i.isActive()&&i.emit(t,e)};i.hammer.on(t,e),i.listeners[t]=e}),a.onTouch(this.hammer,function(t){i.emit("touch",t)}.bind(this)),a.onRelease(this.hammer,function(t){i.emit("release",t)}.bind(this)),this.dom.root.addEventListener("mousewheel",e),this.dom.root.addEventListener("DOMMouseScroll",e),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root)},o.prototype.setOptions=function(t){if(t){var e=["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","rtl","throttleRedraw"];if(h.selectiveExtend(e,this.options,t),this.options.rtl){var i=this.dom.leftContainer;this.dom.leftContainer=this.dom.rightContainer,this.dom.rightContainer=i,this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"}if(this.options.orientation={item:void 0,axis:void 0},"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===n(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var o=this.timeAxis2=new d(this.body);o.setOptions=function(t){var e=t?h.extend({},t):{};e.orientation="top",d.prototype.setOptions.call(o,e)},this.components.push(o)}}else if(this.timeAxis2){var s=this.components.indexOf(this.timeAxis2);-1!==s&&this.components.splice(s,1),this.timeAxis2.destroy(),this.timeAxis2=null}if("function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&c.convertHiddenOptions(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new l(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),"showCustomTime"in t)throw new Error("Option `showCustomTime` is deprecated. Create a custom time bar via timeline.addCustomTime(time [, id])");this._initAutoResize()}if(this.components.forEach(function(e){return e.setOptions(t)}),"configure"in t){this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var r=h.deepExtend({},this.options);this.components.forEach(function(t){h.deepExtend(r,t.options)}),this.configurator.setModuleOptions({global:r})}this._origRedraw?this._redraw():(this._origRedraw=this._redraw.bind(this),this._redraw=h.throttle(this._origRedraw,this.options.throttleRedraw))},o.prototype.isActive=function(){return!this.activator||this.activator.active},o.prototype.destroy=function(){this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator);for(var t in this.listeners)this.listeners.hasOwnProperty(t)&&delete this.listeners[t];this.listeners=null,this.hammer=null,this.components.forEach(function(t){return t.destroy()}),this.body=null},o.prototype.setCustomTime=function(t,e){var i=this.customTimes.filter(function(t){return e===t.options.id});if(0===i.length)throw new Error("No custom time bar found with id "+JSON.stringify(e));i.length>0&&i[0].setCustomTime(t)},o.prototype.getCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+JSON.stringify(t));return e[0].getCustomTime()},o.prototype.setCustomTimeTitle=function(t,e){var i=this.customTimes.filter(function(t){return t.options.id===e});if(0===i.length)throw new Error("No custom time bar found with id "+JSON.stringify(e));return i.length>0?i[0].setCustomTitle(t):void 0},o.prototype.getEventProperties=function(t){return{event:t}},o.prototype.addCustomTime=function(t,e){var i=void 0!==t?h.convert(t,"Date").valueOf():new Date,o=this.customTimes.some(function(t){return t.options.id===e});if(o)throw new Error("A custom time with id "+JSON.stringify(e)+" already exists");var n=new u(this.body,h.extend({},this.options,{time:i,id:e}));return this.customTimes.push(n),this.components.push(n),this._redraw(),e},o.prototype.removeCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+JSON.stringify(t));e.forEach(function(t){this.customTimes.splice(this.customTimes.indexOf(t),1),this.components.splice(this.components.indexOf(t),1),t.destroy()}.bind(this))},o.prototype.getVisibleItems=function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]},o.prototype.fit=function(t){var e=this.getDataRange();if(null!==e.min||null!==e.max){var i=e.max-e.min,o=new Date(e.min.valueOf()-.01*i),n=new Date(e.max.valueOf()+.01*i),s=t&&void 0!==t.animation?t.animation:!0;this.range.setRange(o,n,s)}},o.prototype.getDataRange=function(){throw new Error("Cannot invoke abstract method getDataRange")},o.prototype.setWindow=function(t,e,i){var o;if(1==arguments.length){var n=arguments[0];o=void 0!==n.animation?n.animation:!0,this.range.setRange(n.start,n.end,o)}else o=i&&void 0!==i.animation?i.animation:!0,this.range.setRange(t,e,o)},o.prototype.moveTo=function(t,e){var i=this.range.end-this.range.start,o=h.convert(t,"Date").valueOf(),n=o-i/2,s=o+i/2,r=e&&void 0!==e.animation?e.animation:!0;this.range.setRange(n,s,r)},o.prototype.getWindow=function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}},o.prototype.redraw=function(){this._redraw()},o.prototype._redraw=function(){this.redrawCount++;var t=!1,e=this.options,i=this.props,o=this.dom;if(o&&o.container&&0!=o.root.offsetWidth){c.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),"top"==e.orientation?(h.addClassName(o.root,"vis-top"),h.removeClassName(o.root,"vis-bottom")):(h.removeClassName(o.root,"vis-top"),h.addClassName(o.root,"vis-bottom")),o.root.style.maxHeight=h.option.asSize(e.maxHeight,""),o.root.style.minHeight=h.option.asSize(e.minHeight,""),o.root.style.width=h.option.asSize(e.width,""),i.border.left=(o.centerContainer.offsetWidth-o.centerContainer.clientWidth)/2,i.border.right=i.border.left,i.border.top=(o.centerContainer.offsetHeight-o.centerContainer.clientHeight)/2,i.border.bottom=i.border.top;var n=o.root.offsetHeight-o.root.clientHeight,s=o.root.offsetWidth-o.root.clientWidth;0===o.centerContainer.clientHeight&&(i.border.left=i.border.top,i.border.right=i.border.left),0===o.root.clientHeight&&(s=n),i.center.height=o.center.offsetHeight,i.left.height=o.left.offsetHeight,i.right.height=o.right.offsetHeight,i.top.height=o.top.clientHeight||-i.border.top,i.bottom.height=o.bottom.clientHeight||-i.border.bottom;var a=Math.max(i.left.height,i.center.height,i.right.height),d=i.top.height+a+i.bottom.height+n+i.border.top+i.border.bottom;o.root.style.height=h.option.asSize(e.height,d+"px"),i.root.height=o.root.offsetHeight,i.background.height=i.root.height-n;var l=i.root.height-i.top.height-i.bottom.height-n;i.centerContainer.height=l,i.leftContainer.height=l,i.rightContainer.height=i.leftContainer.height,i.root.width=o.root.offsetWidth,i.background.width=i.root.width-s,i.left.width=o.leftContainer.clientWidth||-i.border.left,i.leftContainer.width=i.left.width,i.right.width=o.rightContainer.clientWidth||-i.border.right,i.rightContainer.width=i.right.width;var u=i.root.width-i.left.width-i.right.width-s;i.center.width=u,i.centerContainer.width=u,i.top.width=u,i.bottom.width=u,o.background.style.height=i.background.height+"px",o.backgroundVertical.style.height=i.background.height+"px",o.backgroundHorizontal.style.height=i.centerContainer.height+"px",o.centerContainer.style.height=i.centerContainer.height+"px",o.leftContainer.style.height=i.leftContainer.height+"px",o.rightContainer.style.height=i.rightContainer.height+"px",o.background.style.width=i.background.width+"px",o.backgroundVertical.style.width=i.centerContainer.width+"px",o.backgroundHorizontal.style.width=i.background.width+"px",o.centerContainer.style.width=i.center.width+"px",o.top.style.width=i.top.width+"px",o.bottom.style.width=i.bottom.width+"px",o.background.style.left="0",o.background.style.top="0",o.backgroundVertical.style.left=i.left.width+i.border.left+"px",o.backgroundVertical.style.top="0",o.backgroundHorizontal.style.left="0",o.backgroundHorizontal.style.top=i.top.height+"px",o.centerContainer.style.left=i.left.width+"px",o.centerContainer.style.top=i.top.height+"px",o.leftContainer.style.left="0",o.leftContainer.style.top=i.top.height+"px",o.rightContainer.style.left=i.left.width+i.center.width+"px",o.rightContainer.style.top=i.top.height+"px",o.top.style.left=i.left.width+"px",o.top.style.top="0",o.bottom.style.left=i.left.width+"px",o.bottom.style.top=i.top.height+i.centerContainer.height+"px",this._updateScrollTop();var p=this.props.scrollTop;"top"!=e.orientation.item&&(p+=Math.max(this.props.centerContainer.height-this.props.center.height-this.props.border.top-this.props.border.bottom,0)),o.center.style.left="0",o.center.style.top=p+"px",o.left.style.left="0",o.left.style.top=p+"px",o.right.style.left="0",o.right.style.top=p+"px";var f=0==this.props.scrollTop?"hidden":"",m=this.props.scrollTop==this.props.scrollTopMin?"hidden":"";o.shadowTop.style.visibility=f,o.shadowBottom.style.visibility=m,o.shadowTopLeft.style.visibility=f,o.shadowBottomLeft.style.visibility=m,o.shadowTopRight.style.visibility=f,o.shadowBottomRight.style.visibility=m;var v=this.props.center.height>this.props.centerContainer.height;this.hammer.get("pan").set({direction:v?r.DIRECTION_ALL:r.DIRECTION_HORIZONTAL}),this.components.forEach(function(e){t=e.redraw()||t});var g=5;if(t){if(this.redrawCount<g)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.initialDrawDone=!0,this.body.emitter.emit("changed")}},o.prototype.repaint=function(){throw new Error("Function repaint is deprecated. Use redraw instead.")},o.prototype.setCurrentTime=function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)},o.prototype.getCurrentTime=function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()},o.prototype._toTime=function(t){return c.toTime(this,t,this.props.center.width)},o.prototype._toGlobalTime=function(t){return c.toTime(this,t,this.props.root.width)},o.prototype._toScreen=function(t){return c.toScreen(this,t,this.props.center.width)},o.prototype._toGlobalScreen=function(t){return c.toScreen(this,t,this.props.root.width)},o.prototype._initAutoResize=function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()},o.prototype._startAutoResize=function(){var t=this;this._stopAutoResize(),this._onResize=function(){return 1!=t.options.autoResize?void t._stopAutoResize():void(t.dom.root&&(t.dom.root.offsetWidth==t.props.lastWidth&&t.dom.root.offsetHeight==t.props.lastHeight||(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight,t.body.emitter.emit("_change"))))},h.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=setInterval(this._onResize,1e3)},o.prototype._stopAutoResize=function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(h.removeEventListener(window,"resize",this._onResize),this._onResize=null)},o.prototype._onTouch=function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop},o.prototype._onPinch=function(t){this.touch.allowDragging=!1},o.prototype._onDrag=function(t){if(this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),o=this._setScrollTop(this.touch.initialScrollTop+e);o!=i&&this.emit("verticalDrag")}},o.prototype._setScrollTop=function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop},o.prototype._updateScrollTop=function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.props.scrollTop},o.prototype._getScrollTop=function(){return this.props.scrollTop},o.prototype._createConfigurator=function(){throw new Error("Cannot invoke abstract method _createConfigurator")},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,type:null,orientation:{item:"bottom"},align:"auto",stack:!0,groupOrderSwap:function(t,e,i){var o=e.order;e.order=t.order,t.order=o},groupOrder:"order",selectable:!0,multiselect:!1,itemsAlwaysDraggable:!1,editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:d.snap,onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20}},this.options=r.extend({},this.defaultOptions),this.itemOptions={type:{start:"Date",end:"Date"}},this.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},this.dom={},this.props={},this.hammer=null;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.groups={},this.groupIds=[],this.selection=[],this.stackDirty=!0,this.touchParams={},this.groupTouchParams={},this._create(),this.setOptions(e)}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(20),r=i(1),a=i(9),h=i(11),d=i(35),l=i(31),c=i(36),u=i(40),p=i(41),f=i(42),m=i(38),v=i(43),g="__ungrouped__",y="__background__";o.prototype=new l,o.types={background:v,box:p,range:m,point:f},o.prototype._create=function(){var t=document.createElement("div");t.className="vis-itemset",t["timeline-itemset"]=this,this.dom.frame=t;var e=document.createElement("div");e.className="vis-background",t.appendChild(e),this.dom.background=e;var i=document.createElement("div");i.className="vis-foreground",t.appendChild(i),this.dom.foreground=i;var o=document.createElement("div");o.className="vis-axis",this.dom.axis=o;var n=document.createElement("div");n.className="vis-labelset",this.dom.labelSet=n,this._updateUngrouped();var r=new u(y,null,this);r.show(),this.groups[y]=r,this.hammer=new s(this.body.dom.centerContainer),this.hammer.on("hammer.input",function(t){t.isFirst&&this._onTouch(t)}.bind(this)),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:s.DIRECTION_HORIZONTAL}),this.hammer.on("tap",this._onSelectItem.bind(this)),this.hammer.on("press",this._onMultiSelectItem.bind(this)),this.hammer.on("doubletap",this._onAddItem.bind(this)),this.groupHammer=new s(this.body.dom.leftContainer),this.groupHammer.on("panstart",this._onGroupDragStart.bind(this)),this.groupHammer.on("panmove",this._onGroupDrag.bind(this)),this.groupHammer.on("panend",this._onGroupDragEnd.bind(this)),this.groupHammer.get("pan").set({threshold:5,direction:s.DIRECTION_HORIZONTAL}),this.show()},o.prototype.setOptions=function(t){if(t){var e=["type","rtl","align","order","stack","selectable","multiselect","itemsAlwaysDraggable","multiselectPerGroup","groupOrder","dataAttributes","template","groupTemplate","hide","snap","groupOrderSwap"];r.selectiveExtend(e,this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===n(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===n(t.margin)&&(r.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===n(t.margin.item)&&r.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable):"object"===n(t.editable)&&r.selectiveExtend(["updateTime","updateGroup","add","remove"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===n(t.groupEditable)&&r.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));var i=function(e){var i=t[e];if(i){if(!(i instanceof Function))throw new Error("option "+e+" must be a function "+e+"(item, callback)");this.options[e]=i}}.bind(this);["onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"].forEach(i),this.markDirty()}},o.prototype.markDirty=function(t){this.groupIds=[],this.stackDirty=!0,t&&t.refreshItems&&r.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()})},o.prototype.destroy=function(){this.hide(),this.setItems(null),this.setGroups(null),this.hammer=null,this.body=null,this.conversion=null},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||this.body.dom.left.appendChild(this.dom.labelSet)},o.prototype.setSelection=function(t){var e,i,o,n;for(void 0==t&&(t=[]),Array.isArray(t)||(t=[t]),e=0,i=this.selection.length;i>e;e++)o=this.selection[e],n=this.items[o],n&&n.unselect();for(this.selection=[],e=0,i=t.length;i>e;e++)o=t[e],n=this.items[o],n&&(this.selection.push(o),n.select())},o.prototype.getSelection=function(){return this.selection.concat([])},o.prototype.getVisibleItems=function(){var t=this.body.range.getRange();if(this.options.rtl)var e=this.body.util.toScreen(t.start),i=this.body.util.toScreen(t.end);else var i=this.body.util.toScreen(t.start),e=this.body.util.toScreen(t.end);var o=[];for(var n in this.groups)if(this.groups.hasOwnProperty(n))for(var s=this.groups[n],r=s.visibleItems,a=0;a<r.length;a++){var h=r[a];this.options.rtl?h.right<i&&h.right+h.width>e&&o.push(h.id):h.left<e&&h.left+h.width>i&&o.push(h.id)}return o},o.prototype._deselect=function(t){for(var e=this.selection,i=0,o=e.length;o>i;i++)if(e[i]==t){e.splice(i,1);break}},o.prototype.redraw=function(){var t=this.options.margin,e=this.body.range,i=r.option.asSize,o=this.options,n=o.orientation.item,s=!1,a=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,a.className="vis-itemset",s=this._orderGroups()||s;var h=e.end-e.start,d=h!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth;d&&(this.stackDirty=!0),
-this.lastVisibleInterval=h,this.props.lastWidth=this.props.width;var l=this.stackDirty,c=this._firstGroup(),u={item:t.item,axis:t.axis},p={item:t.item,axis:t.item.vertical/2},f=0,m=t.axis+t.item.vertical;return this.groups[y].redraw(e,p,l),r.forEach(this.groups,function(t){var i=t==c?u:p,o=t.redraw(e,i,l);s=o||s,f+=t.height}),f=Math.max(f,m),this.stackDirty=!1,a.style.height=i(f),this.props.width=a.offsetWidth,this.props.height=f,this.dom.axis.style.top=i("top"==n?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",s=this._isResized()||s},o.prototype._firstGroup=function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t],i=this.groups[e]||this.groups[g];return i||null},o.prototype._updateUngrouped=function(){var t,e,i=this.groups[g];this.groups[y];if(this.groupsData){if(i){i.hide(),delete this.groups[g];for(e in this.items)if(this.items.hasOwnProperty(e)){t=this.items[e],t.parent&&t.parent.remove(t);var o=this._getGroupId(t.data),n=this.groups[o];n&&n.add(t)||t.hide()}}}else if(!i){var s=null,r=null;i=new c(s,r,this),this.groups[g]=i;for(e in this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}},o.prototype.getLabelSet=function(){return this.dom.labelSet},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(r.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;r.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})},o.prototype.getItems=function(){return this.itemsData},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData&&(r.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.id;r.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,o)}),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.body.emitter.emit("_change",{queue:!0})},o.prototype.getGroups=function(){return this.groupsData},o.prototype.removeItem=function(t){var e=this.itemsData.get(t),i=this.itemsData.getDataSet();e&&this.options.onRemove(e,function(e){e&&i.remove(t)})},o.prototype._getType=function(t){return t.type||this.options.type||(t.end?"range":"box")},o.prototype._getGroupId=function(t){var e=this._getType(t);return"background"==e&&void 0==t.group?y:this.groupsData?t.group:g},o.prototype._onUpdate=function(t){var e=this;t.forEach(function(t){var i,n=e.itemsData.get(t,e.itemOptions),s=e.items[t],r=e._getType(n),a=o.types[r];if(s&&(a&&s instanceof a?e._updateItem(s,n):(i=s.selected,e._removeItem(s),s=null)),!s){if(!a)throw"rangeoverflow"==r?new TypeError('Item type "rangeoverflow" is deprecated. Use css styling instead: .vis-item.vis-range .vis-item-content {overflow: visible;}'):new TypeError('Unknown item type "'+r+'"');s=new a(n,e.conversion,e.options),s.id=t,e._addItem(s),i&&(this.selection.push(t),s.select())}}.bind(this)),this._order(),this.stackDirty=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._onAdd=o.prototype._onUpdate,o.prototype._onRemove=function(t){var e=0,i=this;t.forEach(function(t){var o=i.items[t];o&&(e++,i._removeItem(o))}),e&&(this._order(),this.stackDirty=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype._order=function(){r.forEach(this.groups,function(t){t.order()})},o.prototype._onUpdateGroups=function(t){this._onAddGroups(t)},o.prototype._onAddGroups=function(t){var e=this;t.forEach(function(t){var i=e.groupsData.get(t),o=e.groups[t];if(o)o.setData(i);else{if(t==g||t==y)throw new Error("Illegal group id. "+t+" is a reserved id.");var n=Object.create(e.options);r.extend(n,{height:null}),o=new c(t,i,e),e.groups[t]=o;for(var s in e.items)if(e.items.hasOwnProperty(s)){var a=e.items[s];a.data.group==t&&o.add(a)}o.order(),o.show()}}),this.body.emitter.emit("_change",{queue:!0})},o.prototype._onRemoveGroups=function(t){var e=this.groups;t.forEach(function(t){var i=e[t];i&&(i.hide(),delete e[t])}),this.markDirty(),this.body.emitter.emit("_change",{queue:!0})},o.prototype._orderGroups=function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder}),e=!r.equalArray(t,this.groupIds);if(e){var i=this.groups;t.forEach(function(t){i[t].hide()}),t.forEach(function(t){i[t].show()}),this.groupIds=t}return e}return!1},o.prototype._addItem=function(t){this.items[t.id]=t;var e=this._getGroupId(t.data),i=this.groups[e];i&&i.add(t)},o.prototype._updateItem=function(t,e){var i=t.data.group,o=t.data.subgroup;if(t.setData(e),i!=t.data.group||o!=t.data.subgroup){var n=this.groups[i];n&&n.remove(t);var s=this._getGroupId(t.data),r=this.groups[s];r&&r.add(t)}},o.prototype._removeItem=function(t){t.hide(),delete this.items[t.id];var e=this.selection.indexOf(t.id);-1!=e&&this.selection.splice(e,1),t.parent&&t.parent.remove(t)},o.prototype._constructByEndArray=function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof m&&e.push(t[i]);return e},o.prototype._onTouch=function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null},o.prototype._getGroupIndex=function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e},o.prototype._onDragStart=function(t){var e,i=this.touchParams.item||null,o=this;if(i&&(i.selected||this.options.itemsAlwaysDraggable)){if(!this.options.editable.updateTime&&!this.options.editable.updateGroup&&!i.editable)return;if(i.editable===!1)return;var n=this.touchParams.dragLeftItem,s=this.touchParams.dragRightItem;if(n)e={item:n,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(s)e={item:s,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else{this.touchParams.selectedItem=i;var r=this._getGroupIndex(i.data.group),a=this.options.itemsAlwaysDraggable&&!i.selected?[i.id]:this.getSelection();this.touchParams.itemProps=a.map(function(e){var i=o.items[e],n=o._getGroupIndex(i.data.group);return{item:i,initialX:t.center.x,groupOffset:r-n,data:this._cloneItemData(i.data)}}.bind(this))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)},o.prototype._onDragStartAddItem=function(t){var e=this.options.snap||null;if(this.options.rtl)var i=r.getAbsoluteRight(this.dom.frame),o=i-t.center.x+10;else var i=r.getAbsoluteLeft(this.dom.frame),o=t.center.x-i-10;var n=this.body.util.toTime(o),s=this.body.util.getScale(),a=this.body.util.getStep(),h=e?e(n,s,a):n,d=h,l={type:"range",start:h,end:d,content:"new item"},c=r.randomUUID();l[this.itemsData._fieldId]=c;var u=this.groupFromTarget(t);u&&(l.group=u.groupId);var p=new m(l,this.conversion,this.options);p.id=c,p.data=this._cloneItemData(l),this._addItem(p);var f={item:p,initialX:t.center.x,data:p.data};this.options.rtl?f.dragLeft=!0:f.dragRight=!0,this.touchParams.itemProps=[f],t.stopPropagation()},o.prototype._onDrag=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.options.snap||null;if(this.options.rtl)var o=this.body.dom.root.offsetLeft+this.body.domProps.right.width;else var o=this.body.dom.root.offsetLeft+this.body.domProps.left.width;var n=this.body.util.getScale(),s=this.body.util.getStep(),a=this.touchParams.selectedItem,h=e.options.editable.updateGroup,d=null;if(h&&a&&void 0!=a.data.group){var l=e.groupFromTarget(t);l&&(d=this._getGroupIndex(l.groupId))}this.touchParams.itemProps.forEach(function(a){var h=e.body.util.toTime(t.center.x-o),l=e.body.util.toTime(a.initialX-o);if(this.options.rtl)var c=-(h-l);else var c=h-l;var u=this._cloneItemData(a.item.data);if(a.item.editable!==!1){var p=e.options.editable.updateTime||a.item.editable===!0;if(p)if(a.dragLeft){if(this.options.rtl){if(void 0!=u.end){var f=r.convert(a.data.end,"Date"),m=new Date(f.valueOf()+c);u.end=i?i(m,n,s):m}}else if(void 0!=u.start){var v=r.convert(a.data.start,"Date"),g=new Date(v.valueOf()+c);u.start=i?i(g,n,s):g}}else if(a.dragRight){if(this.options.rtl){if(void 0!=u.start){var v=r.convert(a.data.start,"Date"),g=new Date(v.valueOf()+c);u.start=i?i(g,n,s):g}}else if(void 0!=u.end){var f=r.convert(a.data.end,"Date"),m=new Date(f.valueOf()+c);u.end=i?i(m,n,s):m}}else if(void 0!=u.start){var v=r.convert(a.data.start,"Date").valueOf(),g=new Date(v+c);if(void 0!=u.end){var f=r.convert(a.data.end,"Date"),y=f.valueOf()-v.valueOf();u.start=i?i(g,n,s):g,u.end=new Date(u.start.valueOf()+y)}else u.start=i?i(g,n,s):g}var b=e.options.editable.updateGroup||a.item.editable===!0;if(b&&!a.dragLeft&&!a.dragRight&&null!=d&&void 0!=u.group){var w=d-a.groupOffset;w=Math.max(0,w),w=Math.min(e.groupIds.length-1,w),u.group=e.groupIds[w]}u=this._cloneItemData(u),e.options.onMoving(u,function(t){t&&a.item.setData(this._cloneItemData(t,"Date"))}.bind(this))}}.bind(this)),this.stackDirty=!0,this.body.emitter.emit("_change")}},o.prototype._moveToGroup=function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var o=t.parent;o.remove(t),o.order(),i.add(t),i.order(),t.data.group=i.groupId}},o.prototype._onDragEnd=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.itemsData.getDataSet(),o=this.touchParams.itemProps;this.touchParams.itemProps=null,o.forEach(function(t){var o=t.item.id,n=null!=e.itemsData.get(o,e.itemOptions);if(n){var s=this._cloneItemData(t.item.data);e.options.onMove(s,function(n){n?(n[i._fieldId]=o,i.update(n)):(t.item.setData(t.data),e.stackDirty=!0,e.body.emitter.emit("_change"))})}else e.options.onAdd(t.item.data,function(i){e._removeItem(t.item),i&&e.itemsData.getDataSet().add(i),e.stackDirty=!0,e.body.emitter.emit("_change")})}.bind(this))}},o.prototype._onGroupDragStart=function(t){this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))},o.prototype._onGroupDrag=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupFromTarget(t);if(e&&e.height!=this.groupTouchParams.group.height){var i=e.top<this.groupTouchParams.group.top,o=t.center?t.center.y:t.clientY,n=r.getAbsoluteTop(e.dom.foreground),s=this.groupTouchParams.group.height;if(i){if(o>n+s)return}else{var a=e.height;if(n+a-s>o)return}}if(e&&e!=this.groupTouchParams.group){var h=this.groupsData,d=h.get(e.groupId),l=h.get(this.groupTouchParams.group.groupId);l&&d&&(this.options.groupOrderSwap(l,d,this.groupsData),this.groupsData.update(l),this.groupsData.update(d));var c=this.groupsData.getIds({order:this.options.groupOrder});if(!r.equalArray(c,this.groupTouchParams.originalOrder))for(var h=this.groupsData,u=this.groupTouchParams.originalOrder,p=this.groupTouchParams.group.groupId,f=Math.min(u.length,c.length),m=0,v=0,g=0;f>m;){for(;f>m+v&&f>m+g&&c[m+v]==u[m+g];)m++;if(m+v>=f)break;if(c[m+v]!=p)if(u[m+g]!=p){var y=c.indexOf(u[m+g]),b=h.get(c[m+v]),w=h.get(u[m+g]);this.options.groupOrderSwap(b,w,h),h.update(b),h.update(w);var _=c[m+v];c[m+v]=u[m+g],c[y]=_,m++}else g=1;else v=1}}}},o.prototype._onGroupDragEnd=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,o=e.groupsData.getDataSet(),n=r.extend({},o.get(i));e.options.onMoveGroup(n,function(t){if(t)t[o._fieldId]=i,o.update(t);else{var n=o.getIds({order:e.options.groupOrder});if(!r.equalArray(n,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,a=Math.min(s.length,n.length),h=0;a>h;){for(;a>h&&n[h]==s[h];)h++;if(h>=a)break;var d=n.indexOf(s[h]),l=o.get(n[h]),c=o.get(s[h]);e.options.groupOrderSwap(l,c,o),groupsData.update(l),groupsData.update(c);var u=n[h];n[h]=s[h],n[d]=u,h++}}}),e.body.emitter.emit("groupDragged",{groupId:i})}},o.prototype._onSelectItem=function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)return void this._onMultiSelectItem(t);var o=this.getSelection(),n=this.itemFromTarget(t),s=n?[n.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||o.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}},o.prototype._onAddItem=function(t){if(this.options.selectable&&this.options.editable.add){var e=this,i=this.options.snap||null,o=this.itemFromTarget(t);if(o){var n=e.itemsData.get(o.id);this.options.onUpdate(n,function(t){t&&e.itemsData.getDataSet().update(t)})}else{if(this.options.rtl)var s=r.getAbsoluteRight(this.dom.frame),a=s-t.center.x;else var s=r.getAbsoluteLeft(this.dom.frame),a=t.center.x-s;var h=this.body.util.toTime(a),d=this.body.util.getScale(),l=this.body.util.getStep(),c={start:i?i(h,d,l):h,content:"new item"};if("range"===this.options.type){var u=this.body.util.toTime(a+this.props.width/5);c.end=i?i(u,d,l):u}c[this.itemsData._fieldId]=r.randomUUID();var p=this.groupFromTarget(t);p&&(c.group=p.groupId),c=this._cloneItemData(c),this.options.onAdd(c,function(t){t&&e.itemsData.getDataSet().add(t)})}}},o.prototype._onMultiSelectItem=function(t){if(this.options.selectable){var e=this.itemFromTarget(t);if(e){var i=this.options.multiselect?this.getSelection():[],n=t.srcEvent&&t.srcEvent.shiftKey||!1;if(n&&this.options.multiselect){var s=this.itemsData.get(e.id).group,r=void 0;this.options.multiselectPerGroup&&i.length>0&&(r=this.itemsData.get(i[0]).group),this.options.multiselectPerGroup&&void 0!=r&&r!=s||i.push(e.id);var a=o._getItemRange(this.itemsData.get(i,this.itemOptions));if(!this.options.multiselectPerGroup||r==s){i=[];for(var h in this.items)if(this.items.hasOwnProperty(h)){var d=this.items[h],l=d.data.start,c=void 0!==d.data.end?d.data.end:l;!(l>=a.min&&c<=a.max)||this.options.multiselectPerGroup&&r!=this.itemsData.get(d.id).group||d instanceof v||i.push(d.id)}}}else{var u=i.indexOf(e.id);-1==u?i.push(e.id):i.splice(u,1)}this.setSelection(i),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}},o._getItemRange=function(t){var e=null,i=null;return t.forEach(function(t){(null==i||t.start<i)&&(i=t.start),void 0!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)}),{min:i,max:e}},o.prototype.itemFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-item"))return e["timeline-item"];e=e.parentNode}return null},o.prototype.groupFromTarget=function(t){for(var e=t.center?t.center.y:t.clientY,i=0;i<this.groupIds.length;i++){var o=this.groupIds[i],n=this.groups[o],s=n.dom.foreground,a=r.getAbsoluteTop(s);if(e>a&&e<a+s.offsetHeight)return n;if("top"===this.options.orientation.item){if(i===this.groupIds.length-1&&e>a)return n}else if(0===i&&e<a+s.offset)return n}return null},o.itemSetFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-itemset"))return e["timeline-itemset"];e=e.parentNode}return null},o.prototype._cloneItemData=function(t,e){var i=r.extend({},t);return e||(e=this.itemsData.getDataSet()._options.type),void 0!=i.start&&(i.start=r.convert(i.start,e&&e.start||"Date")),void 0!=i.end&&(i.end=r.convert(i.end,e&&e.end||"Date")),i},t.exports=o},function(t,e,i){function o(t,e,i,s){this.moment=n,this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(t,e,i),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Array.isArray(s)?this.hiddenDates=s:void 0!=s?this.hiddenDates=[s]:this.hiddenDates=[],this.format=o.FORMAT}var n=i(2),s=i(32),r=i(1);o.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},o.prototype.setMoment=function(t){this.moment=t,this.current=this.moment(this.current),this._start=this.moment(this._start),this._end=this.moment(this._end)},o.prototype.setFormat=function(t){var e=r.deepExtend({},o.FORMAT);this.format=r.deepExtend(e,t)},o.prototype.setRange=function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=void 0!=t?this.moment(t.valueOf()):new Date,this._end=void 0!=e?this.moment(e.valueOf()):new Date,this.autoScale&&this.setMinimumStep(i)},o.prototype.start=function(){this.current=this._start.clone(),this.roundToMinor()},o.prototype.roundToMinor=function(){switch(this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step)switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}},o.prototype.hasNext=function(){return this.current.valueOf()<=this._end.valueOf()},o.prototype.next=function(){var t=this.current.valueOf();if(this.current.month()<6)switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.subtract(this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}else switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"month":this.current.month()<this.step&&this.current.month(0);break;case"year":}this.current.valueOf()==t&&(this.current=this._end.clone()),s.stepOverHiddenDates(this.moment,this,t)},o.prototype.getCurrent=function(){return this.current},o.prototype.setScale=function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)},o.prototype.setAutoScale=function(t){this.autoScale=t},o.prototype.setMinimumStep=function(t){if(void 0!=t){var e=31104e6,i=2592e6,o=864e5,n=36e5,s=6e4,r=1e3,a=1;1e3*e>t&&(this.scale="year",this.step=1e3),500*e>t&&(this.scale="year",this.step=500),100*e>t&&(this.scale="year",this.step=100),50*e>t&&(this.scale="year",this.step=50),10*e>t&&(this.scale="year",this.step=10),5*e>t&&(this.scale="year",this.step=5),e>t&&(this.scale="year",this.step=1),3*i>t&&(this.scale="month",this.step=3),i>t&&(this.scale="month",this.step=1),5*o>t&&(this.scale="day",this.step=5),2*o>t&&(this.scale="day",this.step=2),o>t&&(this.scale="day",this.step=1),o/2>t&&(this.scale="weekday",this.step=1),4*n>t&&(this.scale="hour",this.step=4),n>t&&(this.scale="hour",this.step=1),15*s>t&&(this.scale="minute",this.step=15),10*s>t&&(this.scale="minute",this.step=10),5*s>t&&(this.scale="minute",this.step=5),s>t&&(this.scale="minute",this.step=1),15*r>t&&(this.scale="second",this.step=15),10*r>t&&(this.scale="second",this.step=10),5*r>t&&(this.scale="second",this.step=5),r>t&&(this.scale="second",this.step=1),200*a>t&&(this.scale="millisecond",this.step=200),100*a>t&&(this.scale="millisecond",this.step=100),50*a>t&&(this.scale="millisecond",this.step=50),10*a>t&&(this.scale="millisecond",this.step=10),5*a>t&&(this.scale="millisecond",this.step=5),a>t&&(this.scale="millisecond",this.step=1)}},o.snap=function(t,e,i){var o=n(t);if("year"==e){var s=o.year()+Math.round(o.month()/12);o.year(Math.round(s/i)*i),o.month(0),o.date(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("month"==e)o.date()>15?(o.date(1),o.add(1,"month")):o.date(1),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:o.hours(24*Math.round(o.hours()/24));break;default:o.hours(12*Math.round(o.hours()/12))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:o.hours(12*Math.round(o.hours()/12));break;default:o.hours(6*Math.round(o.hours()/6))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("hour"==e){switch(i){case 4:o.minutes(60*Math.round(o.minutes()/60));break;default:o.minutes(30*Math.round(o.minutes()/30))}o.seconds(0),o.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:o.minutes(5*Math.round(o.minutes()/5)),o.seconds(0);break;case 5:o.seconds(60*Math.round(o.seconds()/60));break;default:o.seconds(30*Math.round(o.seconds()/30))}o.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:o.seconds(5*Math.round(o.seconds()/5)),o.milliseconds(0);break;case 5:o.milliseconds(1e3*Math.round(o.milliseconds()/1e3));break;default:o.milliseconds(500*Math.round(o.milliseconds()/500))}else if("millisecond"==e){var r=i>5?i/2:1;o.milliseconds(Math.round(o.milliseconds()/r)*r)}return o},o.prototype.isMajor=function(){if(1==this.switchedYear)switch(this.switchedYear=!1,this.scale){case"year":case"month":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.switchedMonth=!1,this.scale){case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.switchedDay=!1,this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":return 1==t.date();case"month":return 0==t.month();case"year":return!1;default:return!1}},o.prototype.getLabelMinor=function(t){void 0==t&&(t=this.current);var e=this.format.minorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getLabelMajor=function(t){void 0==t&&(t=this.current);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getClassName=function(){function t(t){return t/h%2==0?" vis-even":" vis-odd"}function e(t){return t.isSame(new Date,"day")?" vis-today":t.isSame(s().add(1,"day"),"day")?" vis-tomorrow":t.isSame(s().add(-1,"day"),"day")?" vis-yesterday":""}function i(t){return t.isSame(new Date,"week")?" vis-current-week":""}function o(t){return t.isSame(new Date,"month")?" vis-current-month":""}function n(t){return t.isSame(new Date,"year")?" vis-current-year":""}var s=this.moment,r=this.moment(this.current),a=r.locale?r.locale("en"):r.lang("en"),h=this.step;switch(this.scale){case"millisecond":return t(a.milliseconds()).trim();case"second":return t(a.seconds()).trim();case"minute":return t(a.minutes()).trim();case"hour":var d=a.hours();return 4==this.step&&(d=d+"-h"+(d+4)),"vis-h"+d+e(a)+t(a.hours());case"weekday":return"vis-"+a.format("dddd").toLowerCase()+e(a)+i(a)+t(a.date());case"day":var l=a.date(),c=a.format("MMMM").toLowerCase();return"vis-day"+l+" vis-"+c+o(a)+t(l-1);case"month":return"vis-"+a.format("MMMM").toLowerCase()+o(a)+t(a.month());case"year":var u=a.year();return"vis-year"+u+n(a)+t(u);default:return""}},t.exports=o},function(t,e,i){function o(t,e,i){this.groupId=t,this.subgroups={},this.subgroupIndex=0,this.subgroupOrderer=e&&e.subgroupOrder,this.itemSet=i,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var o=this;this.itemSet.body.emitter.on("checkRangedItems",function(){o.checkRangedItems=!0}),this._create(),this.setData(e)}var n=i(1),s=i(37);i(38);o.prototype._create=function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["timeline-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.innerHTML="?",this.dom.background.appendChild(this.dom.marker)},o.prototype.setData=function(t){var e;if(e=this.itemSet.options&&this.itemSet.options.groupTemplate?this.itemSet.options.groupTemplate(t):t&&t.content,e instanceof Element){for(this.dom.inner.appendChild(e);this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else void 0!==e&&null!==e?this.dom.inner.innerHTML=e:this.dom.inner.innerHTML=this.groupId||"";this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?n.removeClassName(this.dom.inner,"vis-hidden"):n.addClassName(this.dom.inner,"vis-hidden");var i=t&&t.className||null;i!=this.className&&(this.className&&(n.removeClassName(this.dom.label,this.className),n.removeClassName(this.dom.foreground,this.className),n.removeClassName(this.dom.background,this.className),n.removeClassName(this.dom.axis,this.className)),n.addClassName(this.dom.label,i),n.addClassName(this.dom.foreground,i),n.addClassName(this.dom.background,i),n.addClassName(this.dom.axis,i),this.className=i),this.style&&(n.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(n.addCssText(this.dom.label,t.style),this.style=t.style)},o.prototype.getLabelWidth=function(){return this.props.label.width},o.prototype.redraw=function(t,e,i){var o=!1,r=this.dom.marker.clientHeight;if(r!=this.lastMarkerHeight&&(this.lastMarkerHeight=r,n.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()}),i=!0),this._calculateSubGroupHeights(),"function"==typeof this.itemSet.options.order){if(i){var a=this,h=!1;n.forEach(this.items,function(t){t.displayed||(t.redraw(),a.visibleItems.push(t)),t.repositionX(h)});var d=this.orderedItems.byStart.slice().sort(function(t,e){return a.itemSet.options.order(t.data,e.data)});s.stack(d,e,!0)}this.visibleItems=this._updateVisibleItems(this.orderedItems,this.visibleItems,t)}else this.visibleItems=this._updateVisibleItems(this.orderedItems,this.visibleItems,t),this.itemSet.options.stack?s.stack(this.visibleItems,e,i):s.nostack(this.visibleItems,e,this.subgroups);var l=this._calculateHeight(e),c=this.dom.foreground;this.top=c.offsetTop,this.right=c.offsetLeft,this.width=c.offsetWidth,o=n.updateProperty(this,"height",l)||o,o=n.updateProperty(this.props.label,"width",this.dom.inner.clientWidth)||o,o=n.updateProperty(this.props.label,"height",this.dom.inner.clientHeight)||o,this.dom.background.style.height=l+"px",this.dom.foreground.style.height=l+"px",this.dom.label.style.height=l+"px";for(var u=0,p=this.visibleItems.length;p>u;u++){var f=this.visibleItems[u];f.repositionY(e)}return o},o.prototype._calculateSubGroupHeights=function(){if(Object.keys(this.subgroups).length>0){var t=this;this.resetSubgroups(),n.forEach(this.visibleItems,function(e){void 0!==e.data.subgroup&&(t.subgroups[e.data.subgroup].height=Math.max(t.subgroups[e.data.subgroup].height,e.height),t.subgroups[e.data.subgroup].visible=!0)})}},o.prototype._calculateHeight=function(t){var e,i=this.visibleItems;if(i.length>0){var o=i[0].top,s=i[0].top+i[0].height;if(n.forEach(i,function(t){o=Math.min(o,t.top),s=Math.max(s,t.top+t.height)}),o>t.axis){var r=o-t.axis;s-=r,n.forEach(i,function(t){t.top-=r})}e=s+t.item.vertical/2}else e=0;return e=Math.max(e,this.props.label.height)},o.prototype.show=function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)},o.prototype.hide=function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var o=this.dom.axis;o.parentNode&&o.parentNode.removeChild(o)},o.prototype.add=function(t){if(this.items[t.id]=t,t.setParent(this),void 0!==t.data.subgroup&&(void 0===this.subgroups[t.data.subgroup]&&(this.subgroups[t.data.subgroup]={height:0,visible:!1,index:this.subgroupIndex,items:[]},this.subgroupIndex++),this.subgroups[t.data.subgroup].items.push(t)),this.orderSubgroups(),-1==this.visibleItems.indexOf(t)){var e=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,e)}},o.prototype.orderSubgroups=function(){if(void 0!==this.subgroupOrderer){var t=[];if("string"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push({subgroup:e,sortField:this.subgroups[e].items[0].data[this.subgroupOrderer]});t.sort(function(t,e){return t.sortField-e.sortField})}else if("function"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push(this.subgroups[e].items[0].data);t.sort(this.subgroupOrderer)}if(t.length>0)for(var i=0;i<t.length;i++)this.subgroups[t[i].subgroup].index=i}},o.prototype.resetSubgroups=function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1)},o.prototype.remove=function(t){delete this.items[t.id],t.setParent(null);var e=this.visibleItems.indexOf(t);if(-1!=e&&this.visibleItems.splice(e,1),void 0!==t.data.subgroup){var i=this.subgroups[t.data.subgroup];if(i){var o=i.items.indexOf(t);i.items.splice(o,1),i.items.length||(delete this.subgroups[t.data.subgroup],this.subgroupIndex--),this.orderSubgroups()}}},o.prototype.removeFromDataSet=function(t){this.itemSet.removeItem(t.id)},o.prototype.order=function(){for(var t=n.toArray(this.items),e=[],i=[],o=0;o<t.length;o++)void 0!==t[o].data.end&&i.push(t[o]),e.push(t[o]);this.orderedItems={byStart:e,byEnd:i},s.orderByStart(this.orderedItems.byStart),s.orderByEnd(this.orderedItems.byEnd)},o.prototype._updateVisibleItems=function(t,e,i){var o,s,r=[],a={},h=(i.end-i.start)/4,d=i.start-h,l=i.end+h,c=function(t){
-return d>t?-1:l>=t?0:1};if(e.length>0)for(s=0;s<e.length;s++)this._checkIfVisibleWithReference(e[s],r,a,i);var u=n.binarySearchCustom(t.byStart,c,"data","start");if(this._traceVisible(u,t.byStart,r,a,function(t){return t.data.start<d||t.data.start>l}),1==this.checkRangedItems)for(this.checkRangedItems=!1,s=0;s<t.byEnd.length;s++)this._checkIfVisibleWithReference(t.byEnd[s],r,a,i);else{var p=n.binarySearchCustom(t.byEnd,c,"data","end");this._traceVisible(p,t.byEnd,r,a,function(t){return t.data.end<d||t.data.end>l})}for(s=0;s<r.length;s++)o=r[s],o.displayed||o.show(),o.repositionX();return r},o.prototype._traceVisible=function(t,e,i,o,n){var s,r;if(-1!=t){for(r=t;r>=0&&(s=e[r],!n(s));r--)void 0===o[s.id]&&(o[s.id]=!0,i.push(s));for(r=t+1;r<e.length&&(s=e[r],!n(s));r++)void 0===o[s.id]&&(o[s.id]=!0,i.push(s))}},o.prototype._checkIfVisible=function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()},o.prototype._checkIfVisibleWithReference=function(t,e,i,o){t.isVisible(o)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()},t.exports=o},function(t,e){var i=.001;e.orderByStart=function(t){t.sort(function(t,e){return t.data.start-e.data.start})},e.orderByEnd=function(t){t.sort(function(t,e){var i="end"in t.data?t.data.end:t.data.start,o="end"in e.data?e.data.end:e.data.start;return i-o})},e.stack=function(t,i,o){var n,s;if(o)for(n=0,s=t.length;s>n;n++)t[n].top=null;for(n=0,s=t.length;s>n;n++){var r=t[n];if(r.stack&&null===r.top){r.top=i.axis;do{for(var a=null,h=0,d=t.length;d>h;h++){var l=t[h];if(null!==l.top&&l!==r&&l.stack&&e.collision(r,l,i.item,l.options.rtl)){a=l;break}}null!=a&&(r.top=a.top+a.height+i.item.vertical)}while(a)}}},e.nostack=function(t,e,i){var o,n,s;for(o=0,n=t.length;n>o;o++)if(void 0!==t[o].data.subgroup){s=e.axis;for(var r in i)i.hasOwnProperty(r)&&1==i[r].visible&&i[r].index<i[t[o].data.subgroup].index&&(s+=i[r].height+e.item.vertical);t[o].top=s}else t[o].top=e.axis},e.collision=function(t,e,o,n){return n?t.right-o.horizontal+i<e.right+e.width&&t.right+t.width+o.horizontal-i>e.right&&t.top-o.vertical+i<e.top+e.height&&t.top+t.height+o.vertical-i>e.top:t.left-o.horizontal+i<e.left+e.width&&t.left+t.width+o.horizontal-i>e.left&&t.top-o.vertical+i<e.top+e.height&&t.top+t.height+o.vertical-i>e.top}},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,this.options=i,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=(i(20),i(39));o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-range",o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.box=document.createElement("div"),t.frame=document.createElement("div"),t.frame.className="vis-item-overflow",t.box.appendChild(t.frame),t.content=document.createElement("div"),t.content.className="vis-item-content",t.frame.appendChild(t.content),t.box["timeline-item"]=this,this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.box.parentNode){var e=this.parent.dom.foreground;if(!e)throw new Error("Cannot redraw item: parent has no foreground container element");e.appendChild(t.box)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.box),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var i=(this.options.editable.updateTime||this.options.editable.updateGroup||this.editable===!0)&&this.editable!==!1,o=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(i?" vis-editable":" vis-readonly");t.box.className=this.baseClassName+o,this.overflow="hidden"!==window.getComputedStyle(t.frame).overflow,this.dom.content.style.maxWidth="none",this.props.content.width=this.dom.content.offsetWidth,this.height=this.dom.box.offsetHeight,this.dom.content.style.maxWidth="",this.dirty=!1}this._repaintDeleteButton(t.box),this._repaintDragLeft(),this._repaintDragRight()},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}},o.prototype.repositionX=function(t){var e,i,o=this.parent.width,n=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end);void 0!==t&&t!==!0||(-o>n&&(n=-o),s>2*o&&(s=2*o));var r=Math.max(s-n,1);switch(this.overflow?(this.options.rtl?this.right=n:this.left=n,this.width=r+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=n:this.left=n,this.width=r,i=Math.min(s-n,this.props.content.width)),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=r+"px",this.options.align){case"left":this.options.rtl?this.dom.content.style.right="0":this.dom.content.style.left="0";break;case"right":this.options.rtl?this.dom.content.style.right=Math.max(r-i,0)+"px":this.dom.content.style.left=Math.max(r-i,0)+"px";break;case"center":this.options.rtl?this.dom.content.style.right=Math.max((r-i)/2,0)+"px":this.dom.content.style.left=Math.max((r-i)/2,0)+"px";break;default:e=this.overflow?s>0?Math.max(-n,0):-i:0>n?-n:0,this.options.rtl?this.dom.content.style.right=e+"px":this.dom.content.style.left=e+"px"}},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box;"top"==t?e.style.top=this.top+"px":e.style.top=this.parent.height-this.top-this.height+"px"},o.prototype._repaintDragLeft=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else!this.selected&&this.dom.dragLeft&&(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)},o.prototype._repaintDragRight=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else!this.selected&&this.dom.dragRight&&(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)},t.exports=o},function(t,e,i){function o(t,e,i){this.id=null,this.parent=null,this.data=t,this.dom=null,this.conversion=e||{},this.options=i||{},this.selected=!1,this.displayed=!1,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.editable=null,this.data&&this.data.hasOwnProperty("editable")&&"boolean"==typeof this.data.editable&&(this.editable=t.editable)}var n=i(20),s=i(1);o.prototype.stack=!0,o.prototype.select=function(){this.selected=!0,this.dirty=!0,this.displayed&&this.redraw()},o.prototype.unselect=function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()},o.prototype.setData=function(t){var e=void 0!=t.group&&this.data.group!=t.group;e&&this.parent.itemSet._moveToGroup(this,t.group),t.hasOwnProperty("editable")&&"boolean"==typeof t.editable&&(this.editable=t.editable),this.data=t,this.dirty=!0,this.displayed&&this.redraw()},o.prototype.setParent=function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t},o.prototype.isVisible=function(t){return!1},o.prototype.show=function(){return!1},o.prototype.hide=function(){return!1},o.prototype.redraw=function(){},o.prototype.repositionX=function(){},o.prototype.repositionY=function(){},o.prototype._repaintDeleteButton=function(t){var e=(this.options.editable.remove||this.data.editable===!0)&&this.data.editable!==!1;if(this.selected&&e&&!this.dom.deleteButton){var i=this,o=document.createElement("div");this.options.rtl?o.className="vis-delete-rtl":o.className="vis-delete",o.title="Delete this item",new n(o).on("tap",function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)}),t.appendChild(o),this.dom.deleteButton=o}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null)},o.prototype._updateContents=function(t){var e;if(this.options.template){var i=this.parent.itemSet.itemsData.get(this.id);e=this.options.template(i)}else e=this.data.content;var o=this._contentToString(this.content)!==this._contentToString(e);if(o){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(void 0!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.content=e}},o.prototype._updateTitle=function(t){null!=this.data.title?t.title=this.data.title||"":t.removeAttribute("vis-title")},o.prototype._updateDataAttributes=function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Array.isArray(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=Object.keys(this.data)}for(var i=0;i<e.length;i++){var o=e[i],n=this.data[o];null!=n?t.setAttribute("data-"+o,n):t.removeAttribute("data-"+o)}}},o.prototype._updateStyle=function(t){this.style&&(s.removeCssText(t,this.style),this.style=null),this.data.style&&(s.addCssText(t,this.data.style),this.style=this.data.style)},o.prototype._contentToString=function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t},o.prototype.getWidthLeft=function(){return 0},o.prototype.getWidthRight=function(){return 0},t.exports=o},function(t,e,i){function o(t,e,i){n.call(this,t,e,i),this.width=0,this.height=0,this.top=0,this.left=0}var n=(i(1),i(36));o.prototype=Object.create(n.prototype),o.prototype.redraw=function(t,e,i){var o=!1;this.visibleItems=this._updateVisibleItems(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var n=0,s=this.visibleItems.length;s>n;n++){var r=this.visibleItems[n];r.repositionY(e)}return o},o.prototype.show=function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{width:0,height:0},line:{width:0,height:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(39);i(1);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=(t.end-t.start)/4;return this.data.start>t.start-e&&this.data.start<t.end+e},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.box=document.createElement("DIV"),t.content=document.createElement("DIV"),t.content.className="vis-item-content",t.box.appendChild(t.content),t.line=document.createElement("DIV"),t.line.className="vis-line",t.dot=document.createElement("DIV"),t.dot.className="vis-dot",t.box["timeline-item"]=this,this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.box.parentNode){var e=this.parent.dom.foreground;if(!e)throw new Error("Cannot redraw item: parent has no foreground container element");e.appendChild(t.box)}if(!t.line.parentNode){var i=this.parent.dom.background;if(!i)throw new Error("Cannot redraw item: parent has no background container element");i.appendChild(t.line)}if(!t.dot.parentNode){var o=this.parent.dom.axis;if(!i)throw new Error("Cannot redraw item: parent has no axis container element");o.appendChild(t.dot)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.box),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var n=(this.options.editable.updateTime||this.options.editable.updateGroup||this.editable===!0)&&this.editable!==!1,s=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(n?" vis-editable":" vis-readonly");t.box.className="vis-item vis-box"+s,t.line.className="vis-item vis-line"+s,t.dot.className="vis-item vis-dot"+s,this.props.dot.height=t.dot.offsetHeight,this.props.dot.width=t.dot.offsetWidth,this.props.line.width=t.line.offsetWidth,this.width=t.box.offsetWidth,this.height=t.box.offsetHeight,this.dirty=!1}this._repaintDeleteButton(t.box)},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start),e=this.options.align;"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width+"px",this.dom.dot.style.left=t-this.props.line.width/2-this.props.dot.width/2+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t+"px",this.dom.dot.style.right=t+this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t+"px",this.dom.dot.style.left=t+this.props.line.width/2-this.props.dot.width/2+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.dot.width/2+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width/2+"px",this.dom.dot.style.left=t-this.props.dot.width/2+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box,i=this.dom.line,o=this.dom.dot;if("top"==t)e.style.top=(this.top||0)+"px",i.style.top="0",i.style.height=this.parent.top+this.top+1+"px",i.style.bottom="";else{var n=this.parent.itemSet.props.height,s=n-this.parent.top-this.parent.height+this.top;e.style.top=(this.parent.height-this.top-this.height||0)+"px",i.style.top=n-s+"px",i.style.bottom="0"}o.style.top=-this.props.dot.height/2+"px"},o.prototype.getWidthLeft=function(){return this.width/2},o.prototype.getWidthRight=function(){return this.width/2},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(39);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=(t.end-t.start)/4;return this.data.start>t.start-e&&this.data.start<t.end+e},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.point=document.createElement("div"),t.content=document.createElement("div"),t.content.className="vis-item-content",t.point.appendChild(t.content),t.dot=document.createElement("div"),t.point.appendChild(t.dot),t.point["timeline-item"]=this,this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.point.parentNode){var e=this.parent.dom.foreground;if(!e)throw new Error("Cannot redraw item: parent has no foreground container element");e.appendChild(t.point)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.point),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var i=(this.options.editable.updateTime||this.options.editable.updateGroup||this.editable===!0)&&this.editable!==!1,o=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(i?" vis-editable":" vis-readonly");t.point.className="vis-item vis-point"+o,t.dot.className="vis-item vis-dot"+o,this.props.dot.width=t.dot.offsetWidth,this.props.dot.height=t.dot.offsetHeight,this.props.content.height=t.content.offsetHeight,this.options.rtl?t.content.style.marginRight=2*this.props.dot.width+"px":t.content.style.marginLeft=2*this.props.dot.width+"px",this.width=t.point.offsetWidth,this.height=t.point.offsetHeight,t.dot.style.top=(this.height-this.props.dot.height)/2+"px",this.options.rtl?t.dot.style.right=this.props.dot.width/2+"px":t.dot.style.left=this.props.dot.width/2+"px",this.dirty=!1}this._repaintDeleteButton(t.point)},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start);this.options.rtl?(this.right=t-this.props.dot.width,this.dom.point.style.right=this.right+"px"):(this.left=t-this.props.dot.width,this.dom.point.style.left=this.left+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.point;"top"==t?e.style.top=this.top+"px":e.style.top=this.parent.height-this.top-this.height+"px"},o.prototype.getWidthLeft=function(){return this.props.dot.width},o.prototype.getWidthRight=function(){return this.props.dot.width},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=(i(20),i(39)),s=i(40),r=i(38);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-background",o.prototype.stack=!1,o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype.redraw=function(){var t=this.dom;if(t||(this.dom={},t=this.dom,t.box=document.createElement("div"),t.frame=document.createElement("div"),t.frame.className="vis-item-overflow",t.box.appendChild(t.frame),t.content=document.createElement("div"),t.content.className="vis-item-content",t.frame.appendChild(t.content),this.dirty=!0),!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!t.box.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(t.box)}if(this.displayed=!0,this.dirty){this._updateContents(this.dom.content),this._updateTitle(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var i=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");t.box.className=this.baseClassName+i,this.overflow="hidden"!==window.getComputedStyle(t.content).overflow,this.props.content.width=this.dom.content.offsetWidth,this.height=0,this.dirty=!1}},o.prototype.show=r.prototype.show,o.prototype.hide=r.prototype.hide,o.prototype.repositionX=r.prototype.repositionX,o.prototype.repositionY=function(t){var e="top"===this.options.orientation.item;this.dom.content.style.top=e?"":"0",this.dom.content.style.bottom=e?"0":"";var i;if(void 0!==this.data.subgroup){var o=this.data.subgroup,n=this.parent.subgroups,r=n[o].index;if(1==e){i=this.parent.subgroups[o].height+t.item.vertical,i+=0==r?t.axis-.5*t.item.vertical:0;var a=this.parent.top;for(var h in n)n.hasOwnProperty(h)&&1==n[h].visible&&n[h].index<r&&(a+=n[h].height+t.item.vertical);a+=0!=r?t.axis-.5*t.item.vertical:0,this.dom.box.style.top=a+"px",this.dom.box.style.bottom=""}else{var a=this.parent.top,d=0;for(var h in n)if(n.hasOwnProperty(h)&&1==n[h].visible){var l=n[h].height+t.item.vertical;d+=l,n[h].index>r&&(a+=l)}i=this.parent.subgroups[o].height+t.item.vertical,this.dom.box.style.top=this.parent.height-d+a+"px",this.dom.box.style.bottom=""}}else this.parent instanceof s?(i=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.top=e?"0":"",this.dom.box.style.bottom=e?"":"0"):(i=this.parent.height,this.dom.box.style.top=this.parent.top+"px",this.dom.box.style.bottom="");this.dom.box.style.height=i+"px"},t.exports=o},function(t,e,i){function o(t,e){this.dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},this.props={range:{start:0,end:0,minimumStep:0},lineTop:0},this.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,maxMinorChars:7,format:a.FORMAT,moment:d,timeAxis:null},this.options=s.extend({},this.defaultOptions),this.body=t,this._create(),this.setOptions(e)}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=i(31),a=i(35),h=i(32),d=i(2);o.prototype=new r,o.prototype.setOptions=function(t){t&&(s.selectiveExtend(["showMinorLabels","showMajorLabels","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),s.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===n(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof d.locale?d.locale(t.locale):d.lang(t.locale)))},o.prototype._create=function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"},o.prototype.destroy=function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null},o.prototype.redraw=function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,o="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,n=e.parentNode!==o;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,h=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height=this.props.height+"px",this._repaintLabels(),a?o.insertBefore(e,a):o.appendChild(e),h?this.body.dom.backgroundVertical.insertBefore(i,h):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||n},o.prototype._repaintLabels=function(){var t=this.options.orientation.axis,e=s.convert(this.body.range.start,"Number"),i=s.convert(this.body.range.end,"Number"),o=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),n=o-h.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this.body.range,o);n-=this.body.util.toTime(0).valueOf();var r=new a(new Date(e),new Date(i),n,this.body.hiddenDates);r.setMoment(this.options.moment),this.options.format&&r.setFormat(this.options.format),this.options.timeAxis&&r.setScale(this.options.timeAxis),this.step=r;var d=this.dom;d.redundant.lines=d.lines,d.redundant.majorTexts=d.majorTexts,d.redundant.minorTexts=d.minorTexts,d.lines=[],d.majorTexts=[],d.minorTexts=[];var c,u,p,f,m,v,g,y,b,w,_=0,x=void 0,k=0,O=1e3;for(r.start(),u=r.getCurrent(),f=this.body.util.toScreen(u);r.hasNext()&&O>k;){k++,m=r.isMajor(),w=r.getClassName(),b=r.getLabelMinor(),c=u,p=f,r.next(),u=r.getCurrent(),v=r.isMajor(),f=this.body.util.toScreen(u),g=_,_=f-p;var M=_>=.4*g;if(this.options.showMinorLabels&&M){var D=this._repaintMinorText(p,b,t,w);D.style.width=_+"px"}m&&this.options.showMajorLabels?(p>0&&(void 0==x&&(x=p),D=this._repaintMajorText(p,r.getLabelMajor(),t,w)),y=this._repaintMajorLine(p,_,t,w)):M?y=this._repaintMinorLine(p,_,t,w):y&&(y.style.width=parseInt(y.style.width)+_+"px")}if(k!==O||l||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to "+O+" lines."),l=!0),this.options.showMajorLabels){var S=this.body.util.toTime(0),C=r.getLabelMajor(S),T=C.length*(this.props.majorCharWidth||10)+10;(void 0==x||x>T)&&this._repaintMajorText(0,C,t,w)}s.forEach(this.dom.redundant,function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}})},o.prototype._repaintMinorText=function(t,e,i,o){var n=this.dom.redundant.minorTexts.shift();if(!n){var s=document.createTextNode("");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.minorTexts.push(n),n.childNodes[0].nodeValue=e,n.style.top="top"==i?this.props.majorLabelHeight+"px":"0",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n.className="vis-text vis-minor "+o,n},o.prototype._repaintMajorText=function(t,e,i,o){var n=this.dom.redundant.majorTexts.shift();if(!n){var s=document.createTextNode(e);n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.majorTexts.push(n),n.childNodes[0].nodeValue=e,n.className="vis-text vis-major "+o,n.style.top="top"==i?"0":this.props.minorLabelHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n},o.prototype._repaintMinorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return"top"==i?n.style.top=s.majorLabelHeight+"px":n.style.top=this.body.domProps.top.height+"px",n.style.height=s.minorLineHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-minor "+o):(n.style.left=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-minor "+o),n.style.width=e+"px",n},o.prototype._repaintMajorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return"top"==i?n.style.top="0":n.style.top=this.body.domProps.top.height+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-major "+o):(n.style.left=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-major "+o),n.style.height=s.majorLineHeight+"px",n.style.width=e+"px",n},o.prototype._calculateCharSize=function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth};var l=!1;t.exports=o},function(t,e,i){function o(t){this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=a(this.dom.overlay),this.hammer.on("tap",this._onTapOverlay.bind(this));var e=this,i=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];i.forEach(function(t){e.hammer.on(t,function(t){t.stopPropagation()})}),document&&document.body&&(this.onClick=function(i){n(i.target,t)||e.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=s(),this.escListener=this.deactivate.bind(this)}function n(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}var s=i(23),r=i(13),a=i(20),h=i(1);r(o.prototype),o.current=null,o.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),this.hammer.destroy(),this.hammer=null},o.prototype.activate=function(){o.current&&o.current.deactivate(),o.current=this,this.active=!0,this.dom.overlay.style.display="none",h.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),this.keycharm.bind("esc",this.escListener)},o.prototype.deactivate=function(){this.active=!1,this.dom.overlay.style.display="",h.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},o.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={moment:a,locales:h,locale:"en",id:void 0,title:void 0},this.options=s.extend({},this.defaultOptions),e&&e.time?this.customTime=e.time:this.customTime=new Date,this.eventParams={},this.setOptions(e),this._create()}var n=i(20),s=i(1),r=i(31),a=i(2),h=i(47);o.prototype=new r,o.prototype.setOptions=function(t){t&&s.selectiveExtend(["moment","locale","locales","id"],this.options,t)},o.prototype._create=function(){var t=document.createElement("div");t["custom-time"]=this,t.className="vis-custom-time "+(this.options.id||""),t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t;var e=document.createElement("div");e.style.position="relative",e.style.top="0px",e.style.left="-10px",e.style.height="100%",e.style.width="20px",t.appendChild(e),this.hammer=new n(e),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:n.DIRECTION_HORIZONTAL})},o.prototype.destroy=function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null},o.prototype.redraw=function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline.html#Localization"),this.warned=!0),i=this.options.locales.en);var o=this.options.title;return void 0===o&&(o=i.time+": "+this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"),o=o.charAt(0).toUpperCase()+o.substring(1)),this.bar.style.left=e+"px",this.bar.title=o,!1},o.prototype.hide=function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)},o.prototype.setCustomTime=function(t){this.customTime=s.convert(t,"Date"),this.redraw()},o.prototype.getCustomTime=function(){return new Date(this.customTime.valueOf())},o.prototype.setCustomTitle=function(t){this.options.title=t},o.prototype._onDragStart=function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()},o.prototype._onDrag=function(t){if(this.eventParams.dragging){var e=this.body.util.toScreen(this.eventParams.customTime)+t.deltaX,i=this.body.util.toTime(e);this.setCustomTime(i),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf())
-}),t.stopPropagation()}},o.prototype._onDragEnd=function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf())}),t.stopPropagation())},o.customTimeFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null},t.exports=o},function(t,e){e.en={current:"current",time:"time"},e.en_EN=e.en,e.en_US=e.en,e.nl={current:"huidige",time:"tijd"},e.nl_NL=e.nl,e.nl_BE=e.nl},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,showCurrentTime:!0,moment:r,locales:a,locale:"en"},this.options=n.extend({},this.defaultOptions),this.offset=0,this._create(),this.setOptions(e)}var n=i(1),s=i(31),r=i(2),a=i(47);o.prototype=new s,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t},o.prototype.destroy=function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null},o.prototype.setOptions=function(t){t&&n.selectiveExtend(["rtl","showCurrentTime","moment","locale","locales"],this.options,t)},o.prototype.redraw=function(){if(this.options.showCurrentTime){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar),this.start());var e=this.options.moment((new Date).valueOf()+this.offset),i=this.body.util.toScreen(e),o=this.options.locales[this.options.locale];o||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),o=this.options.locales.en);var n=o.current+" "+o.time+": "+e.format("dddd, MMMM Do YYYY, H:mm:ss");n=n.charAt(0).toUpperCase()+n.substring(1),this.options.rtl?this.bar.style.right=i+"px":this.bar.style.left=i+"px",this.bar.title=n}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1},o.prototype.start=function(){function t(){e.stop();var i=e.body.range.conversion(e.body.domProps.center.width).scale,o=1/i/10;30>o&&(o=30),o>1e3&&(o=1e3),e.redraw(),e.body.emitter.emit("currentTimeTick"),e.currentTimeTimer=setTimeout(t,o)}var e=this;t()},o.prototype.stop=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)},o.prototype.setCurrentTime=function(t){var e=n.convert(t,"Date").valueOf(),i=(new Date).valueOf();this.offset=e-i,this.redraw()},o.prototype.getCurrentTime=function(){return new Date((new Date).valueOf()+this.offset)},t.exports=o},function(t,e){Object.defineProperty(e,"__esModule",{value:!0});var i="string",o="boolean",n="number",s="array",r="date",a="object",h="dom",d="moment",l="any",c={configure:{enabled:{"boolean":o},filter:{"boolean":o,"function":"function"},container:{dom:h},__type__:{object:a,"boolean":o,"function":"function"}},align:{string:i},rtl:{"boolean":o,undefined:"undefined"},autoResize:{"boolean":o},throttleRedraw:{number:n},clickToUse:{"boolean":o},dataAttributes:{string:i,array:s},editable:{add:{"boolean":o,undefined:"undefined"},remove:{"boolean":o,undefined:"undefined"},updateGroup:{"boolean":o,undefined:"undefined"},updateTime:{"boolean":o,undefined:"undefined"},__type__:{"boolean":o,object:a}},end:{number:n,date:r,string:i,moment:d},format:{minorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},majorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},__type__:{object:a}},moment:{"function":"function"},groupOrder:{string:i,"function":"function"},groupEditable:{add:{"boolean":o,undefined:"undefined"},remove:{"boolean":o,undefined:"undefined"},order:{"boolean":o,undefined:"undefined"},__type__:{"boolean":o,object:a}},groupOrderSwap:{"function":"function"},height:{string:i,number:n},hiddenDates:{start:{date:r,number:n,string:i,moment:d},end:{date:r,number:n,string:i,moment:d},repeat:{string:i},__type__:{object:a,array:s}},itemsAlwaysDraggable:{"boolean":o},locale:{string:i},locales:{__any__:{any:l},__type__:{object:a}},margin:{axis:{number:n},item:{horizontal:{number:n,undefined:"undefined"},vertical:{number:n,undefined:"undefined"},__type__:{object:a,number:n}},__type__:{object:a,number:n}},max:{date:r,number:n,string:i,moment:d},maxHeight:{number:n,string:i},maxMinorChars:{number:n},min:{date:r,number:n,string:i,moment:d},minHeight:{number:n,string:i},moveable:{"boolean":o},multiselect:{"boolean":o},multiselectPerGroup:{"boolean":o},onAdd:{"function":"function"},onUpdate:{"function":"function"},onMove:{"function":"function"},onMoving:{"function":"function"},onRemove:{"function":"function"},onAddGroup:{"function":"function"},onMoveGroup:{"function":"function"},onRemoveGroup:{"function":"function"},order:{"function":"function"},orientation:{axis:{string:i,undefined:"undefined"},item:{string:i,undefined:"undefined"},__type__:{string:i,object:a}},selectable:{"boolean":o},showCurrentTime:{"boolean":o},showMajorLabels:{"boolean":o},showMinorLabels:{"boolean":o},stack:{"boolean":o},snap:{"function":"function","null":"null"},start:{date:r,number:n,string:i,moment:d},template:{"function":"function"},groupTemplate:{"function":"function"},timeAxis:{scale:{string:i,undefined:"undefined"},step:{number:n,undefined:"undefined"},__type__:{object:a}},type:{string:i},width:{string:i,number:n},zoomable:{"boolean":o},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:n},zoomMin:{number:n},__type__:{object:a}},u={global:{align:["center","left","right"],direction:!1,autoResize:!0,throttleRedraw:[10,0,1e3,10],clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},groupsDraggable:!1,height:"",locale:"",margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,start:"",type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};e.allOptions=c,e.configureOptions=u},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e,i,o){if(!(Array.isArray(i)||i instanceof c||i instanceof u)&&i instanceof Object){var n=o;o=i,i=n}var s=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:d,width:null,height:null,maxHeight:null,minHeight:null},this.options=l.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{toScreen:s._toScreen.bind(s),toGlobalScreen:s._toGlobalScreen.bind(s),toTime:s._toTime.bind(s),toGlobalTime:s._toGlobalTime.bind(s)}},this.range=new p(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new m(this.body),this.components.push(this.timeAxis),this.currentTime=new v(this.body),this.components.push(this.currentTime),this.linegraph=new y(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){s.emit("click",s.getEventProperties(t))}),this.on("doubletap",function(t){s.emit("doubleClick",s.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){s.emit("contextmenu",s.getEventProperties(t))},o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var s=i(26),r=o(s),a=i(29),h=o(a),d=(i(13),i(20),i(2)),l=i(1),c=i(9),u=i(11),p=i(30),f=i(33),m=i(44),v=i(48),g=i(46),y=i(51),b=i(29).printStyle,w=i(59).allOptions,_=i(59).configureOptions;n.prototype=new f,n.prototype.setOptions=function(t){var e=h["default"].validate(t,w);e===!0&&console.log("%cErrors have been found in the supplied options object.",b),f.prototype.setOptions.call(this,t)},n.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?t instanceof c||t instanceof u?t:new c(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.linegraph&&this.linegraph.setItems(e),i)if(void 0!=this.options.start||void 0!=this.options.end){var o=void 0!=this.options.start?this.options.start:null,n=void 0!=this.options.end?this.options.end:null;this.setWindow(o,n,{animation:!1})}else this.fit({animation:!1})},n.prototype.setGroups=function(t){var e;e=t?t instanceof c||t instanceof u?t:new c(t):null,this.groupsData=e,this.linegraph.setGroups(e)},n.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},n.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]):!1},n.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var o=0;o<this.linegraph.groups[i].itemsData.length;o++){var n=this.linegraph.groups[i].itemsData[o],s=l.convert(n.x,"Date").valueOf();t=null==t?s:t>s?s:t,e=null==e?s:s>e?s:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},n.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,o=e-l.getAbsoluteLeft(this.dom.centerContainer),n=i-l.getAbsoluteTop(this.dom.centerContainer),s=this._toTime(o),r=g.customTimeFromTarget(t),a=l.getTarget(t),h=null;l.hasParent(a,this.timeAxis.dom.foreground)?h="axis":this.timeAxis2&&l.hasParent(a,this.timeAxis2.dom.foreground)?h="axis":l.hasParent(a,this.linegraph.yAxisLeft.dom.frame)?h="data-axis":l.hasParent(a,this.linegraph.yAxisRight.dom.frame)?h="data-axis":l.hasParent(a,this.linegraph.legendLeft.dom.frame)?h="legend":l.hasParent(a,this.linegraph.legendRight.dom.frame)?h="legend":null!=r?h="custom-time":l.hasParent(a,this.currentTime.bar)?h="current-time":l.hasParent(a,this.dom.center)&&(h="background");var d=[],c=this.linegraph.yAxisLeft,u=this.linegraph.yAxisRight;return c.hidden||d.push(c.screenToValue(n)),u.hidden||d.push(u.screenToValue(n)),{event:t,what:h,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:s,value:d}},n.prototype._createConfigurator=function(){return new r["default"](this,this.dom.container,_)},t.exports=n},function(t,e,i){function o(t,e){this.id=s.randomUUID(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=s.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",function(){i.lastStart=i.body.range.start,i.svg.style.left=s.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)}),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=i(8),a=i(9),h=i(11),d=i(31),l=i(52),c=i(54),u=i(58),p=i(55),f=i(57),m=i(56),v="__ungrouped__";o.prototype=new d,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new l(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new l(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new u(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new u(this.body,this.options.legend,"right",this.options.groups),this.show()},o.prototype.setOptions=function(t){if(t){var e=["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"];void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&parseInt((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),s.selectiveDeepExtend(e,this.options,t),s.mergeOptions(this.options,t,"interpolation"),s.mergeOptions(this.options,t,"drawPoints"),s.mergeOptions(this.options,t,"shaded"),s.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==n(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty(v)&&this.groups[v].setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(s.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;s.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e)}},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){s.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null;for(var o=0;o<e.length;o++)this._removeGroup(e[o])}if(t){if(!(t instanceof a||t instanceof h))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n=this.id;s.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}},o.prototype._onUpdate=function(t){this._updateAllGroupData()},o.prototype._onAdd=function(t){this._onUpdate(t)},o.prototype._onRemove=function(t){this._onUpdate(t)},o.prototype._onUpdateGroups=function(t){this._updateAllGroupData()},o.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},o.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},o.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new c(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},o.prototype._updateAllGroupData=function(){if(null!=this.itemsData){for(var t={},e=this.itemsData.get(),i={},o=0;o<e.length;o++){var n=e[o],r=n.group;null!==r&&void 0!==r||(r=v),i.hasOwnProperty(r)?i[r]++:i[r]=1}for(var o=0;o<e.length;o++){var n=e[o],r=n.group;null!==r&&void 0!==r||(r=v),t.hasOwnProperty(r)||(t[r]=new Array(i[r]));var a=s.bridgeObject(n);a.x=s.convert(n.x,"Date"),a.orginalY=n.y,a.y=Number(n.y);var h=t[r].length-i[r]--;t[r][h]=a}for(var r in this.groups)this.groups.hasOwnProperty(r)&&(t.hasOwnProperty(r)||(t[r]=new Array(0)));for(var r in t)if(t.hasOwnProperty(r))if(0==t[r].length)this.groups.hasOwnProperty(r)&&this._removeGroup(r);else{var d=void 0;void 0!=this.groupsData&&(d=this.groupsData.get(r)),void 0==d&&(d={id:r,content:this.options.defaultGroup+r}),this._updateGroup(d,r),this.groups[r].setItems(t[r])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},o.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e=this.body.range.end-this.body.range.start,i=e!=this.lastVisibleInterval;if(this.lastVisibleInterval=e,1==t&&(this.svg.style.width=s.option.asSize(3*this.props.width),this.svg.style.left=s.option.asSize(-this.props.width),-1==(this.options.height+"").indexOf("%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0)),1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==i||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,n=this.body.range.end-this.body.range.start;if(0!=this.props.width){var r=this.props.width/n,a=o*r;this.svg.style.left=-this.props.width-a+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},o.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}s.insertSort(t,function(t,e){var i=t.zIndex,o=e.zIndex;return void 0===i&&(i=0),void 0===o&&(o=0),i==o?0:o>i?-1:1});for(var o=new Array(t.length),n=0;n<t.length;n++)o[n]=t[n].id;return o},o.prototype._updateGraph=function(){if(r.prepareElements(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},o=!1,n=this.body.util.toGlobalTime(-this.body.domProps.root.width),s=this.body.util.toGlobalTime(2*this.body.domProps.root.width),a=this._getSortedGroupIds();if(a.length>0){var h={};for(this._getRelevantData(a,h,n,s),this._applySampling(a,h),e=0;e<a.length;e++)this._convertXcoordinates(h[a[e]]);if(this._getYRanges(a,h,i),o=this._updateYAxis(a,i),1==o)return r.cleanupElements(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var d=void 0;for(e=0;e<a.length;e++)t=this.groups[a[e]],this.options.stack===!0&&"line"===this.options.style&&(void 0!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(void 0!=d&&(this._stack(h[t.id],h[d.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==d.options.shaded.orientation?(d.options.shaded.orientation="group",d.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=d.id))),d=t)),this._convertYcoordinates(h[a[e]],t);var l={};for(e=0;e<a.length;e++)if(t=this.groups[a[e]],"line"===t.options.style&&1==t.options.shaded.enabled){var c=h[a[e]];if(null==c||0==c.length)continue;if(l.hasOwnProperty(a[e])||(l[a[e]]=f.calcPath(c,t)),"group"===t.options.shaded.orientation){var u=t.options.shaded.groupId;if(-1===a.indexOf(u)){console.log(t.id+": Unknown shading group target given:"+u);continue}l.hasOwnProperty(u)||(l[u]=f.calcPath(h[u],this.groups[u])),f.drawShading(l[a[e]],t,l[u],this.framework)}else f.drawShading(l[a[e]],t,void 0,this.framework)}for(p.draw(a,h,this.framework),e=0;e<a.length;e++)if(t=this.groups[a[e]],h[a[e]].length>0)switch(t.options.style){case"line":l.hasOwnProperty(a[e])||(l[a[e]]=f.calcPath(h[a[e]],t)),f.draw(l[a[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||m.draw(h[a[e]],t,this.framework);break;case"bar":}}}return r.cleanupElements(this.svgElements),!1},o.prototype._stack=function(t,e){var i,o,n,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var h=i;h<e.length;h++){if(e[h].x===t[a].x){s=e[h],r=e[h],i=h;break}if(e[h].x>t[a].x){r=e[h],s=0==h?r:e[h-1],i=h;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),o=r.x-s.x,n=r.y-s.y,0==o?t[a].y=t[a].orginalY+r.y:t[a].y=t[a].orginalY+n/o*(t[a].x-s.x)+s.y}},o.prototype._getRelevantData=function(t,e,i,o){var n,r,a,h;if(t.length>0)for(r=0;r<t.length;r++){n=this.groups[t[r]];var d=n.getItems();if(1==n.options.sort){var l=function(t,e){return t.getTime()==e.getTime()?0:e>t?-1:1},c=Math.max(0,s.binarySearchValue(d,i,"x","before",l)),u=Math.min(d.length,s.binarySearchValue(d,o,"x","after",l)+1);0>=u&&(u=d.length);var p=new Array(u-c);for(a=c;u>a;a++)h=n.itemsData[a],p[a-c]=h;e[t[r]]=p}else e[t[r]]=n.itemsData}},o.prototype._applySampling=function(t,e){var i;if(t.length>0)for(var o=0;o<t.length;o++)if(i=this.groups[t[o]],1==i.options.sampling){var n=e[t[o]];if(n.length>0){var s=1,r=n.length,a=this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x),h=r/a;s=Math.min(Math.ceil(.2*r),Math.max(1,Math.round(h)));for(var d=new Array(r),l=0;r>l;l+=s){var c=Math.round(l/s);d[c]=n[l]}e[t[o]]=d.splice(0,Math.round(r/s))}}},o.prototype._getYRanges=function(t,e,i){var o,n,s,r,a=[],h=[];if(t.length>0){for(s=0;s<t.length;s++)o=e[t[s]],r=this.groups[t[s]].options,o.length>0&&(n=this.groups[t[s]],r.stack===!0&&"bar"===r.style?"left"===r.yAxisOrientation?a=a.concat(n.getItems()):h=h.concat(n.getItems()):i[t[s]]=n.getYRange(o,t[s]));p.getStackedYRange(a,i,t,"__barStackLeft","left"),p.getStackedYRange(h,i,t,"__barStackRight","right")}},o.prototype._updateYAxis=function(t,e){var i,o,n=!1,s=!1,r=!1,a=1e9,h=1e9,d=-1e9,l=-1e9;if(t.length>0){for(var c=0;c<t.length;c++){var u=this.groups[t[c]];u&&"right"!=u.options.yAxisOrientation?(s=!0,a=1e9,d=-1e9):u&&u.options.yAxisOrientation&&(r=!0,h=1e9,l=-1e9)}for(var c=0;c<t.length;c++)e.hasOwnProperty(t[c])&&e[t[c]].ignore!==!0&&(i=e[t[c]].min,o=e[t[c]].max,"right"!=e[t[c]].yAxisOrientation?(s=!0,a=a>i?i:a,d=o>d?o:d):(r=!0,h=h>i?i:h,l=o>l?o:l));1==s&&this.yAxisLeft.setRange(a,d),1==r&&this.yAxisRight.setRange(h,l)}n=this._toggleAxisVisiblity(s,this.yAxisLeft)||n,n=this._toggleAxisVisiblity(r,this.yAxisRight)||n,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(1==r?this.yAxisLeft.lineOffset=this.yAxisRight.width:this.yAxisLeft.lineOffset=0,n=this.yAxisLeft.redraw()||n,n=this.yAxisRight.redraw()||n):n=this.yAxisRight.redraw()||n;for(var p=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"],c=0;c<p.length;c++)-1!=t.indexOf(p[c])&&t.splice(t.indexOf(p[c]),1);return n},o.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},o.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y},o.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,o=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var n=0;n<t.length;n++)t[n].screen_y=Math.round(i.convertValue(t[n].y));e.setZeroPosition(Math.min(o,i.convertValue(0)))},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=n.randomUUID(),this.body=t,this.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}}},this.linegraphOptions=o,this.linegraphSVG=i,this.props={},this.DOMelements={lines:{},labels:{},title:{}},this.dom={},this.scale=void 0,this.range={start:0,end:0},this.options=n.extend({},this.defaultOptions),this.conversionFactor=1,this.setOptions(e),this.width=Number((""+this.options.width).replace("px","")),this.minWidth=this.width,this.height=this.linegraphSVG.getBoundingClientRect().height,this.hidden=!1,this.stepPixels=25,this.zeroCrossing=-1,this.amountOfSteps=-1,this.lineOffset=0,this.master=!0,this.masterAxis=null,this.svgElements={},this.iconsRemoved=!1,this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups};var s=this;this.body.emitter.on("verticalDrag",function(){s.dom.lineContainer.style.top=s.body.domProps.scrollTop+"px"})}var n=i(1),s=i(8),r=i(31),a=i(53);o.prototype=new r,o.prototype.addGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1},o.prototype.updateGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype.setOptions=function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);var i=["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"];n.selectiveDeepExtend(i,this.options,t),this.minWidth=Number((""+this.options.width).replace("px","")),e===!0&&this.dom.frame&&(this.hide(),this.show())}},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)},o.prototype._redrawGroupIcons=function(){s.prepareElements(this.svgElements);var t,e=this.options.iconWidth,i=15,o=4,n=o+.5*i;t="left"===this.options.orientation?o:this.width-e-o;var r=Object.keys(this.groups);r.sort(function(t,e){return e>t?-1:1});for(var a=0;a<r.length;a++){var h=r[a];this.groups[h].visible!==!0||void 0!==this.linegraphOptions.visibility[h]&&this.linegraphOptions.visibility[h]!==!0||(this.groups[h].getLegend(e,i,this.framework,t,n),n+=i+o)}s.cleanupElements(this.svgElements),this.iconsRemoved=!1},o.prototype._cleanupIcons=function(){this.iconsRemoved===!1&&(s.prepareElements(this.svgElements),s.cleanupElements(this.svgElements),this.iconsRemoved=!0)},o.prototype.show=function(){this.hidden=!1,this.dom.frame.parentNode||(this.options.rtl?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.left.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer)},o.prototype.hide=function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.parentNode&&this.dom.lineContainer.parentNode.removeChild(this.dom.lineContainer)},o.prototype.setRange=function(t,e){this.range.start=t,this.range.end=e},o.prototype.redraw=function(){var t=!1,e=0;this.dom.lineContainer.style.top=this.body.domProps.scrollTop+"px";for(var i in this.groups)this.groups.hasOwnProperty(i)&&(this.groups[i].visible!==!0||void 0!==this.linegraphOptions.visibility[i]&&this.linegraphOptions.visibility[i]!==!0||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height=this.height+"px",this.width=this.options.visible===!0?Number((""+this.options.width).replace("px","")):0;var o=this.props,n=this.dom.frame;n.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels;o.minorLabelHeight=r?o.minorCharHeight:0,o.majorLabelHeight=a?o.majorCharHeight:0,o.minorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.minorLinesOffset,o.minorLineHeight=1,o.majorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.majorLinesOffset,o.majorLineHeight=1,"left"===s?(n.style.top="0",n.style.left="0",n.style.bottom="",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(n.style.top="",n.style.bottom="0",n.style.left="0",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.right.width,
-this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,this.options.icons===!0?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t},o.prototype._redrawLabels=function(){var t=this,e=!1;s.prepareElements(this.DOMelements.lines),s.prepareElements(this.DOMelements.labels);var i=this.options.orientation,o=void 0!=this.options[i].range?this.options[i].range:{},n=!0;void 0!=o.max&&(this.range.end=o.max,n=!1);var r=!0;void 0!=o.min&&(this.range.start=o.min,r=!1),this.scale=new a(this.range.start,this.range.end,r,n,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),this.master===!1&&void 0!=this.masterAxis&&this.scale.followScale(this.masterAxis.scale),this.maxLabelSize=0;var h=this.scale.getLines();h.forEach(function(e){var o=e.y,n=e.major;t.options.showMinorLabels&&n===!1&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),n&&o>=0&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),t.master===!0&&(n?t._redrawLine(o,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(o,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))});var d=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(d=this.props.titleCharHeight);var l=this.options.icons===!0?Math.max(this.options.iconWidth,d)+this.options.labelOffsetX+15:d+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-l&&this.options.visible===!0?(this.width=this.maxLabelSize+l,this.options.width=this.width+"px",s.cleanupElements(this.DOMelements.lines),s.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-l&&this.options.visible===!0&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+l),this.options.width=this.width+"px",s.cleanupElements(this.DOMelements.lines),s.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):(s.cleanupElements(this.DOMelements.lines),s.cleanupElements(this.DOMelements.labels),e=!1),e},o.prototype.convertValue=function(t){return this.scale.convertValue(t)},o.prototype.screenToValue=function(t){return this.scale.screenToValue(t)},o.prototype._redrawLabel=function(t,e,i,o,n){var r=s.getDOMElement("div",this.DOMelements.labels,this.dom.frame);r.className=o,r.innerHTML=e,"left"===i?(r.style.left="-"+this.options.labelOffsetX+"px",r.style.textAlign="right"):(r.style.right="-"+this.options.labelOffsetX+"px",r.style.textAlign="left"),r.style.top=t-.5*n+this.options.labelOffsetY+"px",e+="";var a=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*a&&(this.maxLabelSize=e.length*a)},o.prototype._redrawLine=function(t,e,i,o,n){if(this.master===!0){var r=s.getDOMElement("div",this.DOMelements.lines,this.dom.lineContainer);r.className=i,r.innerHTML="","left"===e?r.style.left=this.width-o+"px":r.style.right=this.width-o+"px",r.style.width=n+"px",r.style.top=t+"px"}},o.prototype._redrawTitle=function(t){if(s.prepareElements(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=s.getDOMElement("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-"+t,e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&n.addCssText(e,this.options[t].title.style),"left"===t?e.style.left=this.props.titleCharHeight+"px":e.style.right=this.props.titleCharHeight+"px",e.style.width=this.height+"px"}s.cleanupElements(this.DOMelements.title)},o.prototype._calculateCharSize=function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),o=document.createElement("div");o.className="vis-y-axis vis-major vis-measure",o.appendChild(i),this.dom.frame.appendChild(o),this.props.majorCharHeight=o.clientHeight,this.props.majorCharWidth=o.clientWidth,this.dom.frame.removeChild(o)}if(!("titleCharHeight"in this.props)){var n=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(n),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}},t.exports=o},function(t,e){function i(t,e,i,o,n,s){var r=arguments.length<=6||void 0===arguments[6]?!1:arguments[6],a=arguments.length<=7||void 0===arguments[7]?!1:arguments[7];if(this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=n,this.majorCharHeight=s,this._start=t,this._end=e,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=r,this.autoScaleStart=i,this.autoScaleEnd=o,this.formattingFunction=a,i||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>.5*(h.magnitudefactor*h.minorSteps[h.minorStepIdx])?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};i&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}i.prototype.setCharHeight=function(t){this.majorCharHeight=t},i.prototype.setHeight=function(t){this.containerHeight=t},i.prototype.determineScale=function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var o=0;0>i&&(o=i);for(var n=!1,s=o;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){var a=this.magnitudefactor*this.minorSteps[r];if(a>=e){n=!0,this.minorStepIdx=r;break}}if(n===!0)break}},i.prototype.is_major=function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])===0},i.prototype.getStep=function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]},i.prototype.getFirstMajor=function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)},i.prototype.formatValue=function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?""+e:"string"==typeof e?e:t.toPrecision(5)},i.prototype.getLines=function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,o=this._start+i;this._end-o>1e-5;o+=e)o!=this._start&&t.push({major:this.is_major(o),y:this.convertValue(o),val:this.formatValue(o)});return t},i.prototype.followScale=function(t){var e=this.minorStepIdx,i=this._start,o=this._end,n=this,s=function(){n.magnitudefactor*=2},r=function(){n.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e?s():(s(),s())):(this.minorStepIdx=2,1==e?r():(r(),r())));for(var a=(t.getLines(),t.convertValue(0)),h=t.getStep()*t.scale,d=!1,l=0;!d&&l++<5;){this.scale=h/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var c=this.containerHeight/this.scale;this._start=i,this._end=this._start+c;var u=this._end*this.scale,p=this.magnitudefactor*this.majorSteps[this.minorStepIdx],f=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var m=a-u;this._end+=m/this.scale,this._start=this._end-c}else this.autoScaleStart?(this._start-=f/this.scale,this._end=this._start+c):(this._start+=p-f/this.scale,this._end=this._start+c);if(!this.autoScaleEnd&&this._end>o+1e-5)r(),d=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),d=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&o-i>c?(s(),d=!1):d=!0}}},i.prototype.convertValue=function(t){return this.containerHeight-(t-this._start)*this.scale},i.prototype.screenToValue=function(t){return(this.containerHeight-t)/this.scale+this._start},t.exports=i},function(t,e,i){function o(t,e,i,o){this.id=e;var n=["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"];this.options=s.selectiveBridgeObject(n,i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=o,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible?!0:t.visible}var n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},s=i(1),r=(i(8),i(55)),a=i(57),h=i(56);o.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==this.options.sort&&s.insertSort(this.itemsData,function(t,e){return t.x>e.x?1:-1})):this.itemsData=[]},o.prototype.getItems=function(){return this.itemsData},o.prototype.setZeroPosition=function(t){this.zeroPosition=t},o.prototype.setOptions=function(t){if(void 0!==t){var e=["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"];s.selectiveDeepExtend(e,this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),s.mergeOptions(this.options,t,"interpolation"),s.mergeOptions(this.options,t,"drawPoints"),s.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==n(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},o.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible?!0:t.visible,this.style=t.style,this.setOptions(t.options)},o.prototype.getLegend=function(t,e,i,o,n){if(void 0==i||null==i){var s=document.createElementNS("http://www.w3.org/2000/svg","svg");i={svg:s,svgElements:{},options:this.options,groups:[this]}}switch(void 0!=o&&null!=o||(o=0),void 0!=n&&null!=n||(n=.5*e),this.options.style){case"line":a.drawIcon(this,o,n,t,e,i);break;case"points":case"point":h.drawIcon(this,o,n,t,e,i);break;case"bar":r.drawIcon(this,o,n,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},o.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,o=0;o<t.length;o++)e=e>t[o].y?t[o].y:e,i=i<t[o].y?t[o].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(8),s=i(56);o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,h=n.getSVGElement("rect",r.svgElements,r.svg);h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-a),h.setAttributeNS(null,"width",o),h.setAttributeNS(null,"height",2*a),h.setAttributeNS(null,"class","vis-outline");var d=Math.round(.3*o),l=t.options.barChart.width,c=l/d,u=Math.round(.4*s),p=Math.round(.75*s),f=Math.round((o-2*d)/3);if(n.drawBar(e+.5*d+f,i+a-u-1,d,u,t.className+" vis-bar",r.svgElements,r.svg,t.style),n.drawBar(e+1.5*d+f+2,i+a-p-1,d,p,t.className+" vis-bar",r.svgElements,r.svg,t.style),1==t.options.drawPoints.enabled){var m={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/c,className:t.className};n.drawPoint(e+.5*d+f,i+a-u-1,m,r.svgElements,r.svg),n.drawPoint(e+1.5*d+f+2,i+a-p-1,m,r.svgElements,r.svg)}},o.draw=function(t,e,i){var r,a,h,d,l,c,u=[],p={},f=0;for(l=0;l<t.length;l++)if(d=i.groups[t[l]],"bar"===d.options.style&&d.visible===!0&&(void 0===i.options.groups.visibility[t[l]]||i.options.groups.visibility[t[l]]===!0))for(c=0;c<e[t[l]].length;c++)u.push({screen_x:e[t[l]][c].screen_x,screen_y:e[t[l]][c].screen_y,x:e[t[l]][c].x,y:e[t[l]][c].y,groupId:t[l],label:e[t[l]][c].label}),f+=1;if(0!==f)for(u.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}),o._getDataIntersections(p,u),l=0;l<u.length;l++){d=i.groups[u[l].groupId];var m=void 0!=d.options.barChart.minWidth?d.options.barChart.minWidth:.1*d.options.barChart.width;a=u[l].screen_x;var v=0;if(void 0===p[a])l+1<u.length&&(r=Math.abs(u[l+1].screen_x-a)),h=o._getSafeDrawData(r,d,m);else{var g=l+(p[a].amount-p[a].resolved);l-(p[a].resolved+1);g<u.length&&(r=Math.abs(u[g].screen_x-a)),h=o._getSafeDrawData(r,d,m),p[a].resolved+=1,d.options.stack===!0&&d.options.excludeFromStacking!==!0?u[l].screen_y<d.zeroPosition?(v=p[a].accumulatedNegative,p[a].accumulatedNegative+=d.zeroPosition-u[l].screen_y):(v=p[a].accumulatedPositive,p[a].accumulatedPositive+=d.zeroPosition-u[l].screen_y):d.options.barChart.sideBySide===!0&&(h.width=h.width/p[a].amount,h.offset+=p[a].resolved*h.width-.5*h.width*(p[a].amount+1))}if(n.drawBar(u[l].screen_x+h.offset,u[l].screen_y-v,h.width,d.zeroPosition-u[l].screen_y,d.className+" vis-bar",i.svgElements,i.svg,d.style),d.options.drawPoints.enabled===!0){var y={screen_x:u[l].screen_x,screen_y:u[l].screen_y-v,x:u[l].x,y:u[l].y,groupId:u[l].groupId,label:u[l].label};s.draw([y],d,i,h.offset)}}},o._getDataIntersections=function(t,e){for(var i,o=0;o<e.length;o++)o+1<e.length&&(i=Math.abs(e[o+1].screen_x-e[o].screen_x)),o>0&&(i=Math.min(i,Math.abs(e[o-1].screen_x-e[o].screen_x))),0===i&&(void 0===t[e[o].screen_x]&&(t[e[o].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[o].screen_x].amount+=1)},o._getSafeDrawData=function(t,e,i){var o,n;return t<e.options.barChart.width&&t>0?(o=i>t?i:t,n=0,"left"===e.options.barChart.align?n-=.5*t:"right"===e.options.barChart.align&&(n+=.5*t)):(o=e.options.barChart.width,n=0,"left"===e.options.barChart.align?n-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(n+=.5*e.options.barChart.width)),{width:o,offset:n}},o.getStackedYRange=function(t,e,i,n,s){if(t.length>0){t.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x});var r={};o._getDataIntersections(r,t),e[n]=o._getStackedYRange(r,t),e[n].yAxisOrientation=s,i.push(n)}},o._getStackedYRange=function(t,e){for(var i,o=e[0].screen_y,n=e[0].screen_y,s=0;s<e.length;s++)i=e[s].screen_x,void 0===t[i]?(o=o>e[s].screen_y?e[s].screen_y:o,n=n<e[s].screen_y?e[s].screen_y:n):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(o=o>t[r].accumulatedNegative?t[r].accumulatedNegative:o,o=o>t[r].accumulatedPositive?t[r].accumulatedPositive:o,n=n<t[r].accumulatedNegative?t[r].accumulatedNegative:n,n=n<t[r].accumulatedPositive?t[r].accumulatedPositive:n);return{min:o,max:n}},t.exports=o},function(t,e,i){function o(t,e){}function n(t,e){return e="undefined"==typeof e?{}:e,{style:e.style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function s(t,e){var i=void 0;return t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender),e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender),i}var r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},a=i(8);o.draw=function(t,e,i,o){o=o||0;for(var h=s(i,e),d=0;d<t.length;d++)if(h){var l=h(t[d],e);l!==!0&&"object"!==("undefined"==typeof l?"undefined":r(l))||a.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e,l),i.svgElements,i.svg,t[d].label)}else a.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e),i.svgElements,i.svg,t[d].label)},o.drawIcon=function(t,e,i,o,s,r){var h=.5*s,d=a.getSVGElement("rect",r.svgElements,r.svg);d.setAttributeNS(null,"x",e),d.setAttributeNS(null,"y",i-h),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",2*h),d.setAttributeNS(null,"class","vis-outline"),a.drawPoint(e+.5*o,i,n(t),r.svgElements,r.svg)},t.exports=o},function(t,e,i){function o(t,e){}var n=i(8);o.calcPath=function(t,e){if(null!=t&&t.length>0){var i=[];return i=1==e.options.interpolation.enabled?o._catmullRom(t,e):o._linear(t)}},o.drawIcon=function(t,e,i,o,s,r){var a,h,d=.5*s,l=n.getSVGElement("rect",r.svgElements,r.svg);if(l.setAttributeNS(null,"x",e),l.setAttributeNS(null,"y",i-d),l.setAttributeNS(null,"width",o),l.setAttributeNS(null,"height",2*d),l.setAttributeNS(null,"class","vis-outline"),a=n.getSVGElement("path",r.svgElements,r.svg),a.setAttributeNS(null,"class",t.className),void 0!==t.style&&a.setAttributeNS(null,"style",t.style),a.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+o)+","+i),1==t.options.shaded.enabled&&(h=n.getSVGElement("path",r.svgElements,r.svg),"top"==t.options.shaded.orientation?h.setAttributeNS(null,"d","M"+e+", "+(i-d)+"L"+e+","+i+" L"+(e+o)+","+i+" L"+(e+o)+","+(i-d)):h.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+d)+" L"+(e+o)+","+(i+d)+"L"+(e+o)+","+i),h.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&h.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled){var c={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className};n.drawPoint(e+.5*o,i,c,r.svgElements,r.svg)}},o.drawShading=function(t,e,i,o){if(1==e.options.shaded.enabled){var s=Number(o.svg.style.height.replace("px","")),r=n.getSVGElement("path",o.svgElements,o.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var h,d=0;d="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),h="group"==e.options.shaded.orientation&&null!=i&&void 0!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+d+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",h)}},o.draw=function(t,e,i){if(null!=t&&void 0!=t){var o=n.getSVGElement("path",i.svgElements,i.svg);o.setAttributeNS(null,"class",e.className),void 0!==e.style&&o.setAttributeNS(null,"style",e.style);var s="L";1==e.options.interpolation.enabled&&(s="C"),o.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,s,!1))}},o.serializePath=function(t,e,i){if(t.length<2)return"";var o=e;if(i)for(var n=t.length-2;n>0;n--)o+=t[n][0]+","+t[n][1]+" ";else for(var n=1;n<t.length;n++)o+=t[n][0]+","+t[n][1]+" ";return o},o._catmullRomUniform=function(t){var e,i,o,n,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var h=1/6,d=t.length,l=0;d-1>l;l++)e=0==l?t[0]:t[l-1],i=t[l],o=t[l+1],n=d>l+2?t[l+2]:o,s={screen_x:(-e.screen_x+6*i.screen_x+o.screen_x)*h,screen_y:(-e.screen_y+6*i.screen_y+o.screen_y)*h},r={screen_x:(i.screen_x+6*o.screen_x-n.screen_x)*h,screen_y:(i.screen_y+6*o.screen_y-n.screen_y)*h},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([o.screen_x,o.screen_y]);return a},o._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var o,n,s,r,a,h,d,l,c,u,p,f,m,v,g,y,b,w,_,x=[];x.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var k=t.length,O=0;k-1>O;O++)o=0==O?t[0]:t[O-1],n=t[O],s=t[O+1],r=k>O+2?t[O+2]:s,d=Math.sqrt(Math.pow(o.screen_x-n.screen_x,2)+Math.pow(o.screen_y-n.screen_y,2)),l=Math.sqrt(Math.pow(n.screen_x-s.screen_x,2)+Math.pow(n.screen_y-s.screen_y,2)),c=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),v=Math.pow(c,i),y=Math.pow(c,2*i),g=Math.pow(l,i),b=Math.pow(l,2*i),_=Math.pow(d,i),w=Math.pow(d,2*i),u=2*w+3*_*g+b,p=2*y+3*v*g+b,f=3*_*(_+g),f>0&&(f=1/f),m=3*v*(v+g),m>0&&(m=1/m),a={screen_x:(-b*o.screen_x+u*n.screen_x+w*s.screen_x)*f,screen_y:(-b*o.screen_y+u*n.screen_y+w*s.screen_y)*f},h={screen_x:(y*n.screen_x+p*s.screen_x-b*r.screen_x)*m,screen_y:(y*n.screen_y+p*s.screen_y-b*r.screen_y)*m},0==a.screen_x&&0==a.screen_y&&(a=n),0==h.screen_x&&0==h.screen_y&&(h=s),x.push([a.screen_x,a.screen_y]),x.push([h.screen_x,h.screen_y]),x.push([s.screen_x,s.screen_y]);return x},o._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},t.exports=o},function(t,e,i){function o(t,e,i,o){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=n.extend({},this.defaultOptions),this.linegraphOptions=o,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}var n=i(1),s=i(8),r=i(31);o.prototype=new r,o.prototype.clear=function(){this.groups={},this.amountOfGroups=0},o.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},o.prototype.updateGroup=function(t,e){this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setOptions=function(t){var e=["enabled","orientation","icons","left","right"];n.selectiveDeepExtend(e,this.options,t)},o.prototype.redraw=function(){var t=0,e=Object.keys(this.groups);e.sort(function(t,e){return e>t?-1:1});for(var i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var n=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+n+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());for(var s="",i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||(s+=this.groups[o].content+"<br />")}this.dom.textArea.innerHTML=s,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},o.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=Object.keys(this.groups);t.sort(function(t,e){return e>t?-1:1}),s.resetElements(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),o=i,n=this.options.iconSize,r=.75*this.options.iconSize,a=i+.5*r+3;this.svg.style.width=n+5+i+"px";for(var h=0;h<t.length;h++){var d=t[h];1!=this.groups[d].visible||void 0!==this.linegraphOptions.visibility[d]&&1!=this.linegraphOptions.visibility[d]||(this.groups[d].getLegend(n,r,this.framework,o,a),a+=r+this.options.iconSpacing)}}},t.exports=o},function(t,e){Object.defineProperty(e,"__esModule",{value:!0});var i="string",o="boolean",n="number",s="array",r="date",a="object",h="dom",d="moment",l="any",c={configure:{enabled:{"boolean":o},filter:{"boolean":o,"function":"function"},container:{dom:h},__type__:{object:a,"boolean":o,"function":"function"}},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:i},sort:{"boolean":o},sampling:{"boolean":o},stack:{"boolean":o},graphHeight:{string:i,number:n},shaded:{enabled:{"boolean":o},orientation:{string:["bottom","top","zero","group"]},groupId:{object:a},__type__:{"boolean":o,object:a}},style:{string:["line","bar","points"]},barChart:{width:{number:n},minWidth:{number:n},sideBySide:{"boolean":o},align:{string:["left","center","right"]},__type__:{object:a}},interpolation:{enabled:{"boolean":o},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:n},__type__:{object:a,"boolean":o}},drawPoints:{enabled:{"boolean":o},onRender:{"function":"function"},size:{number:n},style:{string:["square","circle"]},__type__:{object:a,"boolean":o,"function":"function"}},dataAxis:{showMinorLabels:{"boolean":o},showMajorLabels:{"boolean":o},icons:{"boolean":o},width:{string:i,number:n},visible:{"boolean":o},alignZeros:{"boolean":o},left:{range:{min:{number:n},max:{number:n},__type__:{object:a}},format:{"function":"function"},title:{text:{string:i,number:n},style:{string:i},__type__:{object:a}},__type__:{object:a}},right:{range:{min:{number:n},max:{number:n},__type__:{object:a}},format:{"function":"function"},title:{text:{string:i,number:n},style:{string:i},__type__:{object:a}},__type__:{object:a}},__type__:{object:a}},legend:{enabled:{"boolean":o},icons:{"boolean":o},left:{visible:{"boolean":o},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:a}},right:{visible:{"boolean":o},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:a}},__type__:{object:a,"boolean":o}},groups:{visibility:{any:l},__type__:{object:a}},autoResize:{"boolean":o},throttleRedraw:{number:n},clickToUse:{"boolean":o},end:{number:n,date:r,string:i,moment:d},format:{minorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},majorLabels:{millisecond:{string:i,undefined:"undefined"},second:{string:i,undefined:"undefined"},minute:{string:i,undefined:"undefined"},hour:{string:i,undefined:"undefined"},weekday:{string:i,undefined:"undefined"},day:{string:i,undefined:"undefined"},month:{string:i,undefined:"undefined"},year:{string:i,undefined:"undefined"},__type__:{object:a}},__type__:{object:a}},moment:{"function":"function"},height:{string:i,number:n},hiddenDates:{start:{date:r,number:n,string:i,moment:d},end:{date:r,number:n,string:i,moment:d},repeat:{string:i},__type__:{object:a,array:s}},locale:{string:i},locales:{__any__:{any:l},__type__:{object:a}},max:{date:r,number:n,string:i,moment:d},maxHeight:{number:n,string:i},maxMinorChars:{number:n},min:{date:r,number:n,string:i,moment:d},minHeight:{number:n,string:i},moveable:{"boolean":o},multiselect:{"boolean":o},orientation:{string:i},showCurrentTime:{"boolean":o},showMajorLabels:{"boolean":o},showMinorLabels:{"boolean":o},start:{date:r,number:n,string:i,moment:d},timeAxis:{scale:{string:i,undefined:"undefined"},step:{number:n,undefined:"undefined"},__type__:{object:a}},width:{string:i,number:n},zoomable:{"boolean":o},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:n},zoomMin:{number:n},zIndex:{number:n},__type__:{object:a}},u={global:{sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,throttleRedraw:[10,0,1e3,10],clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};e.allOptions=c,e.configureOptions=u},function(t,e,i){e.util=i(1),e.DOMutil=i(8),e.DataSet=i(9),e.DataView=i(11),e.Queue=i(10),e.Network=i(61),e.network={Images:i(62),dotparser:i(118),gephiParser:i(119),allOptions:i(114)},e.network.convertDot=function(t){return e.network.dotparser.DOTToGraph(t)},e.network.convertGephi=function(t,i){return e.network.gephiParser.parseGephi(t,i)},e.moment=i(2),e.Hammer=i(20),e.keycharm=i(23)},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e,i){var o=this;if(!(this instanceof n))throw new SyntaxError("Constructor must be called with the new operator");this.options={},this.defaultOptions={locale:"en",locales:Y,clickToUse:!1},F.extend(this.options,this.defaultOptions),this.body={container:t,nodes:{},nodeIndices:[],edges:{},edgeIndices:[],emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this),once:this.once.bind(this)},eventListeners:{onTap:function(){},onTouch:function(){},onDoubleTap:function(){},onHold:function(){},onDragStart:function(){},onDrag:function(){},onDragEnd:function(){},onMouseWheel:function(){},onPinch:function(){},onMouseMove:function(){},onRelease:function(){},onContext:function(){}},data:{nodes:null,edges:null
-},functions:{createNode:function(){},createEdge:function(){},getPointer:function(){}},modules:{},view:{scale:1,translation:{x:0,y:0}}},this.bindEventListeners(),this.images=new r["default"](function(){return o.body.emitter.emit("_requestRedraw")}),this.groups=new h["default"],this.canvas=new w["default"](this.body),this.selectionHandler=new D["default"](this.body,this.canvas),this.interactionHandler=new O["default"](this.body,this.canvas,this.selectionHandler),this.view=new x["default"](this.body,this.canvas),this.renderer=new y["default"](this.body,this.canvas),this.physics=new f["default"](this.body),this.layoutEngine=new C["default"](this.body),this.clustering=new v["default"](this.body),this.manipulation=new E["default"](this.body,this.canvas,this.selectionHandler),this.nodesHandler=new l["default"](this.body,this.images,this.groups,this.layoutEngine),this.edgesHandler=new u["default"](this.body,this.images,this.groups),this.body.modules.kamadaKawai=new A["default"](this.body,150,.05),this.body.modules.clustering=this.clustering,this.canvas._create(),this.setOptions(i),this.setData(e)}var s=i(62),r=o(s),a=i(63),h=o(a),d=i(64),l=o(d),c=i(84),u=o(c),p=i(93),f=o(p),m=i(102),v=o(m),g=i(105),y=o(g),b=i(106),w=o(b),_=i(107),x=o(_),k=i(108),O=o(k),M=i(111),D=o(M),S=i(112),C=o(S),T=i(113),E=o(T),P=i(26),I=o(P),N=i(29),R=o(N),z=i(114),L=i(115),A=o(L);i(117);var B=i(13),F=i(1),j=(i(9),i(11),i(118)),H=i(119),W=i(45),Y=i(120);B(n.prototype),n.prototype.setOptions=function(t){var e=this;if(void 0!==t){var i=R["default"].validate(t,z.allOptions);i===!0&&console.log("%cErrors have been found in the supplied options object.",N.printStyle);var o=["locale","locales","clickToUse"];if(F.selectiveDeepExtend(o,this.options,t),t=this.layoutEngine.setOptions(t.layout,t),this.canvas.setOptions(t),this.groups.setOptions(t.groups),this.nodesHandler.setOptions(t.nodes),this.edgesHandler.setOptions(t.edges),this.physics.setOptions(t.physics),this.manipulation.setOptions(t.manipulation,t,this.options),this.interactionHandler.setOptions(t.interaction),this.renderer.setOptions(t.interaction),this.selectionHandler.setOptions(t.interaction),void 0!==t.groups&&this.body.emitter.emit("refreshNodes"),"configure"in t&&(this.configurator||(this.configurator=new I["default"](this,this.body.container,z.configureOptions,this.canvas.pixelRatio)),this.configurator.setOptions(t.configure)),this.configurator&&this.configurator.options.enabled===!0){var n={nodes:{},edges:{},layout:{},interaction:{},manipulation:{},physics:{},global:{}};F.deepExtend(n.nodes,this.nodesHandler.options),F.deepExtend(n.edges,this.edgesHandler.options),F.deepExtend(n.layout,this.layoutEngine.options),F.deepExtend(n.interaction,this.selectionHandler.options),F.deepExtend(n.interaction,this.renderer.options),F.deepExtend(n.interaction,this.interactionHandler.options),F.deepExtend(n.manipulation,this.manipulation.options),F.deepExtend(n.physics,this.physics.options),F.deepExtend(n.global,this.canvas.options),F.deepExtend(n.global,this.options),this.configurator.setModuleOptions(n)}void 0!==t.clickToUse?t.clickToUse===!0?void 0===this.activator&&(this.activator=new W(this.canvas.frame),this.activator.on("change",function(){e.body.emitter.emit("activate")})):(void 0!==this.activator&&(this.activator.destroy(),delete this.activator),this.body.emitter.emit("activate")):this.body.emitter.emit("activate"),this.canvas.setSize(),this.body.emitter.emit("startSimulation")}},n.prototype._updateVisibleIndices=function(){var t=this.body.nodes,e=this.body.edges;this.body.nodeIndices=[],this.body.edgeIndices=[];for(var i in t)t.hasOwnProperty(i)&&t[i].options.hidden===!1&&this.body.nodeIndices.push(t[i].id);for(var o in e)e.hasOwnProperty(o)&&e[o].options.hidden===!1&&this.body.edgeIndices.push(e[o].id)},n.prototype.bindEventListeners=function(){var t=this;this.body.emitter.on("_dataChanged",function(){t._updateVisibleIndices(),t.body.emitter.emit("_requestRedraw"),t.body.emitter.emit("_dataUpdated")}),this.body.emitter.on("_dataUpdated",function(){t._updateValueRange(t.body.nodes),t._updateValueRange(t.body.edges),t.body.emitter.emit("startSimulation"),t.body.emitter.emit("_requestRedraw")})},n.prototype.setData=function(t){if(this.body.emitter.emit("resetPhysics"),this.body.emitter.emit("_resetData"),this.selectionHandler.unselectAll(),t&&t.dot&&(t.nodes||t.edges))throw new SyntaxError('Data must contain either parameter "dot" or  parameter pair "nodes" and "edges", but not both.');if(this.setOptions(t&&t.options),t&&t.dot){console.log("The dot property has been depricated. Please use the static convertDot method to convert DOT into vis.network format and use the normal data format with nodes and edges. This converter is used like this: var data = vis.network.convertDot(dotString);");var e=j.DOTToGraph(t.dot);return void this.setData(e)}if(t&&t.gephi){console.log("The gephi property has been depricated. Please use the static convertGephi method to convert gephi into vis.network format and use the normal data format with nodes and edges. This converter is used like this: var data = vis.network.convertGephi(gephiJson);");var i=H.parseGephi(t.gephi);return void this.setData(i)}this.nodesHandler.setData(t&&t.nodes,!0),this.edgesHandler.setData(t&&t.edges,!0),this.body.emitter.emit("_dataChanged"),this.body.emitter.emit("_dataLoaded"),this.body.emitter.emit("initPhysics")},n.prototype.destroy=function(){this.body.emitter.emit("destroy"),this.body.emitter.off(),this.off(),delete this.groups,delete this.canvas,delete this.selectionHandler,delete this.interactionHandler,delete this.view,delete this.renderer,delete this.physics,delete this.layoutEngine,delete this.clustering,delete this.manipulation,delete this.nodesHandler,delete this.edgesHandler,delete this.configurator,delete this.images;for(var t in this.body.nodes)delete this.body.nodes[t];for(var e in this.body.edges)delete this.body.edges[e];F.recursiveDOMDelete(this.body.container)},n.prototype._updateValueRange=function(t){var e,i=void 0,o=void 0,n=0;for(e in t)if(t.hasOwnProperty(e)){var s=t[e].getValue();void 0!==s&&(i=void 0===i?s:Math.min(s,i),o=void 0===o?s:Math.max(s,o),n+=s)}if(void 0!==i&&void 0!==o)for(e in t)t.hasOwnProperty(e)&&t[e].setValueRange(i,o,n)},n.prototype.isActive=function(){return!this.activator||this.activator.active},n.prototype.setSize=function(){return this.canvas.setSize.apply(this.canvas,arguments)},n.prototype.canvasToDOM=function(){return this.canvas.canvasToDOM.apply(this.canvas,arguments)},n.prototype.DOMtoCanvas=function(){return this.canvas.DOMtoCanvas.apply(this.canvas,arguments)},n.prototype.findNode=function(){return this.clustering.findNode.apply(this.clustering,arguments)},n.prototype.isCluster=function(){return this.clustering.isCluster.apply(this.clustering,arguments)},n.prototype.openCluster=function(){return this.clustering.openCluster.apply(this.clustering,arguments)},n.prototype.cluster=function(){return this.clustering.cluster.apply(this.clustering,arguments)},n.prototype.getNodesInCluster=function(){return this.clustering.getNodesInCluster.apply(this.clustering,arguments)},n.prototype.clusterByConnection=function(){return this.clustering.clusterByConnection.apply(this.clustering,arguments)},n.prototype.clusterByHubsize=function(){return this.clustering.clusterByHubsize.apply(this.clustering,arguments)},n.prototype.clusterOutliers=function(){return this.clustering.clusterOutliers.apply(this.clustering,arguments)},n.prototype.getSeed=function(){return this.layoutEngine.getSeed.apply(this.layoutEngine,arguments)},n.prototype.enableEditMode=function(){return this.manipulation.enableEditMode.apply(this.manipulation,arguments)},n.prototype.disableEditMode=function(){return this.manipulation.disableEditMode.apply(this.manipulation,arguments)},n.prototype.addNodeMode=function(){return this.manipulation.addNodeMode.apply(this.manipulation,arguments)},n.prototype.editNode=function(){return this.manipulation.editNode.apply(this.manipulation,arguments)},n.prototype.editNodeMode=function(){return console.log("Deprecated: Please use editNode instead of editNodeMode."),this.manipulation.editNode.apply(this.manipulation,arguments)},n.prototype.addEdgeMode=function(){return this.manipulation.addEdgeMode.apply(this.manipulation,arguments)},n.prototype.editEdgeMode=function(){return this.manipulation.editEdgeMode.apply(this.manipulation,arguments)},n.prototype.deleteSelected=function(){return this.manipulation.deleteSelected.apply(this.manipulation,arguments)},n.prototype.getPositions=function(){return this.nodesHandler.getPositions.apply(this.nodesHandler,arguments)},n.prototype.storePositions=function(){return this.nodesHandler.storePositions.apply(this.nodesHandler,arguments)},n.prototype.moveNode=function(){return this.nodesHandler.moveNode.apply(this.nodesHandler,arguments)},n.prototype.getBoundingBox=function(){return this.nodesHandler.getBoundingBox.apply(this.nodesHandler,arguments)},n.prototype.getConnectedNodes=function(t){return void 0!==this.body.nodes[t]?this.nodesHandler.getConnectedNodes.apply(this.nodesHandler,arguments):this.edgesHandler.getConnectedNodes.apply(this.edgesHandler,arguments)},n.prototype.getConnectedEdges=function(){return this.nodesHandler.getConnectedEdges.apply(this.nodesHandler,arguments)},n.prototype.startSimulation=function(){return this.physics.startSimulation.apply(this.physics,arguments)},n.prototype.stopSimulation=function(){return this.physics.stopSimulation.apply(this.physics,arguments)},n.prototype.stabilize=function(){return this.physics.stabilize.apply(this.physics,arguments)},n.prototype.getSelection=function(){return this.selectionHandler.getSelection.apply(this.selectionHandler,arguments)},n.prototype.setSelection=function(){return this.selectionHandler.setSelection.apply(this.selectionHandler,arguments)},n.prototype.getSelectedNodes=function(){return this.selectionHandler.getSelectedNodes.apply(this.selectionHandler,arguments)},n.prototype.getSelectedEdges=function(){return this.selectionHandler.getSelectedEdges.apply(this.selectionHandler,arguments)},n.prototype.getNodeAt=function(){var t=this.selectionHandler.getNodeAt.apply(this.selectionHandler,arguments);return void 0!==t&&void 0!==t.id?t.id:t},n.prototype.getEdgeAt=function(){var t=this.selectionHandler.getEdgeAt.apply(this.selectionHandler,arguments);return void 0!==t&&void 0!==t.id?t.id:t},n.prototype.selectNodes=function(){return this.selectionHandler.selectNodes.apply(this.selectionHandler,arguments)},n.prototype.selectEdges=function(){return this.selectionHandler.selectEdges.apply(this.selectionHandler,arguments)},n.prototype.unselectAll=function(){this.selectionHandler.unselectAll.apply(this.selectionHandler,arguments),this.redraw()},n.prototype.redraw=function(){return this.renderer.redraw.apply(this.renderer,arguments)},n.prototype.getScale=function(){return this.view.getScale.apply(this.view,arguments)},n.prototype.getViewPosition=function(){return this.view.getViewPosition.apply(this.view,arguments)},n.prototype.fit=function(){return this.view.fit.apply(this.view,arguments)},n.prototype.moveTo=function(){return this.view.moveTo.apply(this.view,arguments)},n.prototype.focus=function(){return this.view.focus.apply(this.view,arguments)},n.prototype.releaseNode=function(){return this.view.releaseNode.apply(this.view,arguments)},n.prototype.getOptionsFromConfigurator=function(){var t={};return this.configurator&&(t=this.configurator.getOptions.apply(this.configurator)),t},t.exports=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e){i(this,t),this.images={},this.imageBroken={},this.callback=e}return o(t,[{key:"_addImageToCache",value:function(t,e){0===e.width&&(document.body.appendChild(e),e.width=e.offsetWidth,e.height=e.offsetHeight,document.body.removeChild(e)),this.images[t]=e}},{key:"_tryloadBrokenUrl",value:function(t,e,i){var o=this;void 0!==t&&void 0!==e&&void 0!==i&&(i.onerror=function(){console.error("Could not load brokenImage:",e),o._addImageToCache(t,new Image)},i.src=e)}},{key:"_redrawWithImage",value:function(t){this.callback&&this.callback(t)}},{key:"load",value:function(t,e,i){var o=this,n=this.images[t];if(n)return n;var s=new Image;return s.onload=function(){o._addImageToCache(t,s),o._redrawWithImage(s)},s.onerror=function(){console.error("Could not load image:",t),o._tryloadBrokenUrl(t,e,s)},s.src=t,s}}]),t}();e["default"]=n},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(1),r=function(){function t(){o(this,t),this.clear(),this.defaultIndex=0,this.groupsArray=[],this.groupIndex=0,this.defaultGroups=[{border:"#2B7CE9",background:"#97C2FC",highlight:{border:"#2B7CE9",background:"#D2E5FF"},hover:{border:"#2B7CE9",background:"#D2E5FF"}},{border:"#FFA500",background:"#FFFF00",highlight:{border:"#FFA500",background:"#FFFFA3"},hover:{border:"#FFA500",background:"#FFFFA3"}},{border:"#FA0A10",background:"#FB7E81",highlight:{border:"#FA0A10",background:"#FFAFB1"},hover:{border:"#FA0A10",background:"#FFAFB1"}},{border:"#41A906",background:"#7BE141",highlight:{border:"#41A906",background:"#A1EC76"},hover:{border:"#41A906",background:"#A1EC76"}},{border:"#E129F0",background:"#EB7DF4",highlight:{border:"#E129F0",background:"#F0B3F5"},hover:{border:"#E129F0",background:"#F0B3F5"}},{border:"#7C29F0",background:"#AD85E4",highlight:{border:"#7C29F0",background:"#D3BDF0"},hover:{border:"#7C29F0",background:"#D3BDF0"}},{border:"#C37F00",background:"#FFA807",highlight:{border:"#C37F00",background:"#FFCA66"},hover:{border:"#C37F00",background:"#FFCA66"}},{border:"#4220FB",background:"#6E6EFD",highlight:{border:"#4220FB",background:"#9B9BFD"},hover:{border:"#4220FB",background:"#9B9BFD"}},{border:"#FD5A77",background:"#FFC0CB",highlight:{border:"#FD5A77",background:"#FFD1D9"},hover:{border:"#FD5A77",background:"#FFD1D9"}},{border:"#4AD63A",background:"#C2FABC",highlight:{border:"#4AD63A",background:"#E6FFE3"},hover:{border:"#4AD63A",background:"#E6FFE3"}},{border:"#990000",background:"#EE0000",highlight:{border:"#BB0000",background:"#FF3333"},hover:{border:"#BB0000",background:"#FF3333"}},{border:"#FF6000",background:"#FF6000",highlight:{border:"#FF6000",background:"#FF6000"},hover:{border:"#FF6000",background:"#FF6000"}},{border:"#97C2FC",background:"#2B7CE9",highlight:{border:"#D2E5FF",background:"#2B7CE9"},hover:{border:"#D2E5FF",background:"#2B7CE9"}},{border:"#399605",background:"#255C03",highlight:{border:"#399605",background:"#255C03"},hover:{border:"#399605",background:"#255C03"}},{border:"#B70054",background:"#FF007E",highlight:{border:"#B70054",background:"#FF007E"},hover:{border:"#B70054",background:"#FF007E"}},{border:"#AD85E4",background:"#7C29F0",highlight:{border:"#D3BDF0",background:"#7C29F0"},hover:{border:"#D3BDF0",background:"#7C29F0"}},{border:"#4557FA",background:"#000EA1",highlight:{border:"#6E6EFD",background:"#000EA1"},hover:{border:"#6E6EFD",background:"#000EA1"}},{border:"#FFC0CB",background:"#FD5A77",highlight:{border:"#FFD1D9",background:"#FD5A77"},hover:{border:"#FFD1D9",background:"#FD5A77"}},{border:"#C2FABC",background:"#74D66A",highlight:{border:"#E6FFE3",background:"#74D66A"},hover:{border:"#E6FFE3",background:"#74D66A"}},{border:"#EE0000",background:"#990000",highlight:{border:"#FF3333",background:"#BB0000"},hover:{border:"#FF3333",background:"#BB0000"}}],this.options={},this.defaultOptions={useDefaultGroups:!0},s.extend(this.options,this.defaultOptions)}return n(t,[{key:"setOptions",value:function(t){var e=["useDefaultGroups"];if(void 0!==t)for(var i in t)if(t.hasOwnProperty(i)&&-1===e.indexOf(i)){var o=t[i];this.add(i,o)}}},{key:"clear",value:function(){this.groups={},this.groupsArray=[]}},{key:"get",value:function(t){var e=this.groups[t];if(void 0===e)if(this.options.useDefaultGroups===!1&&this.groupsArray.length>0){var i=this.groupIndex%this.groupsArray.length;this.groupIndex++,e={},e.color=this.groups[this.groupsArray[i]],this.groups[t]=e}else{var o=this.defaultIndex%this.defaultGroups.length;this.defaultIndex++,e={},e.color=this.defaultGroups[o],this.groups[t]=e}return e}},{key:"add",value:function(t,e){return this.groups[t]=e,this.groupsArray.push(t),e}}]),t}();e["default"]=r},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(65),a=o(r),h=i(66),d=o(h),l=i(1),c=i(9),u=i(11),p=function(){function t(e,i,o,s){var r=this;n(this,t),this.body=e,this.images=i,this.groups=o,this.layoutEngine=s,this.body.functions.createNode=this.create.bind(this),this.nodesListeners={add:function(t,e){r.add(e.items)},update:function(t,e){r.update(e.items,e.data)},remove:function(t,e){r.remove(e.items)}},this.options={},this.defaultOptions={borderWidth:1,borderWidthSelected:2,brokenImage:void 0,color:{border:"#2B7CE9",background:"#97C2FC",highlight:{border:"#2B7CE9",background:"#D2E5FF"},hover:{border:"#2B7CE9",background:"#D2E5FF"}},fixed:{x:!1,y:!1},font:{color:"#343434",size:14,face:"arial",background:"none",strokeWidth:0,strokeColor:"#ffffff",align:"center"},group:void 0,hidden:!1,icon:{face:"FontAwesome",code:void 0,size:50,color:"#2B7CE9"},image:void 0,label:void 0,labelHighlightBold:!0,level:void 0,mass:1,physics:!0,scaling:{min:10,max:30,label:{enabled:!1,min:14,max:30,maxVisible:30,drawThreshold:5},customScalingFunction:function(t,e,i,o){if(e===t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)}},shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:10,x:5,y:5},shape:"ellipse",shapeProperties:{borderDashes:!1,borderRadius:6,interpolation:!0,useImageSize:!1,useBorderWithImage:!1},size:25,title:void 0,value:void 0,x:void 0,y:void 0},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("refreshNodes",this.refresh.bind(this)),this.body.emitter.on("refresh",this.refresh.bind(this)),this.body.emitter.on("destroy",function(){l.forEach(t.nodesListeners,function(e,i){t.body.data.nodes&&t.body.data.nodes.off(i,e)}),delete t.body.functions.createNode,delete t.nodesListeners.add,delete t.nodesListeners.update,delete t.nodesListeners.remove,delete t.nodesListeners})}},{key:"setOptions",value:function(t){if(void 0!==t){if(a["default"].parseOptions(this.options,t),void 0!==t.shape)for(var e in this.body.nodes)this.body.nodes.hasOwnProperty(e)&&this.body.nodes[e].updateShape();if(void 0!==t.font){d["default"].parseOptions(this.options.font,t);for(var i in this.body.nodes)this.body.nodes.hasOwnProperty(i)&&(this.body.nodes[i].updateLabelModule(),this.body.nodes[i]._reset())}if(void 0!==t.size)for(var o in this.body.nodes)this.body.nodes.hasOwnProperty(o)&&this.body.nodes[o]._reset();void 0===t.hidden&&void 0===t.physics||this.body.emitter.emit("_dataChanged")}}},{key:"setData",value:function(t){var e=this,i=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],o=this.body.data.nodes;if(t instanceof c||t instanceof u)this.body.data.nodes=t;else if(Array.isArray(t))this.body.data.nodes=new c,this.body.data.nodes.add(t);else{if(t)throw new TypeError("Array or DataSet expected");this.body.data.nodes=new c}o&&l.forEach(this.nodesListeners,function(t,e){o.off(e,t)}),this.body.nodes={},this.body.data.nodes&&!function(){var t=e;l.forEach(e.nodesListeners,function(e,i){t.body.data.nodes.on(i,e)});var i=e.body.data.nodes.getIds();e.add(i,!0)}(),i===!1&&this.body.emitter.emit("_dataChanged")}},{key:"add",value:function(t){for(var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=void 0,o=[],n=0;n<t.length;n++){i=t[n];var s=this.body.data.nodes.get(i),r=this.create(s);o.push(r),this.body.nodes[i]=r}this.layoutEngine.positionInitially(o),e===!1&&this.body.emitter.emit("_dataChanged")}},{key:"update",value:function(t,e){for(var i=this.body.nodes,o=!1,n=0;n<t.length;n++){var s=t[n],r=i[s],a=e[n];void 0!==r?o=r.setOptions(a):(o=!0,r=this.create(a),i[s]=r)}o===!0?this.body.emitter.emit("_dataChanged"):this.body.emitter.emit("_dataUpdated")}},{key:"remove",value:function(t){for(var e=this.body.nodes,i=0;i<t.length;i++){var o=t[i];delete e[o]}this.body.emitter.emit("_dataChanged")}},{key:"create",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?a["default"]:arguments[1];return new e(t,this.body,this.images,this.groups,this.options)}},{key:"refresh",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!1:arguments[0],e=this.body.nodes;for(var i in e){var o=void 0;e.hasOwnProperty(i)&&(o=e[i]);var n=this.body.data.nodes._data[i];void 0!==o&&void 0!==n&&(t===!0&&o.setOptions({x:null,y:null}),o.setOptions({fixed:!1}),o.setOptions(n))}}},{key:"getPositions",value:function(t){var e={};if(void 0!==t){if(Array.isArray(t)===!0){for(var i=0;i<t.length;i++)if(void 0!==this.body.nodes[t[i]]){var o=this.body.nodes[t[i]];e[t[i]]={x:Math.round(o.x),y:Math.round(o.y)}}}else if(void 0!==this.body.nodes[t]){var n=this.body.nodes[t];e[t]={x:Math.round(n.x),y:Math.round(n.y)}}}else for(var s=0;s<this.body.nodeIndices.length;s++){var r=this.body.nodes[this.body.nodeIndices[s]];e[this.body.nodeIndices[s]]={x:Math.round(r.x),y:Math.round(r.y)}}return e}},{key:"storePositions",value:function(){var t=[],e=this.body.data.nodes.getDataSet();for(var i in e._data)if(e._data.hasOwnProperty(i)){var o=this.body.nodes[i];e._data[i].x==Math.round(o.x)&&e._data[i].y==Math.round(o.y)||t.push({id:o.id,x:Math.round(o.x),y:Math.round(o.y)})}e.update(t)}},{key:"getBoundingBox",value:function(t){return void 0!==this.body.nodes[t]?this.body.nodes[t].shape.boundingBox:void 0}},{key:"getConnectedNodes",value:function(t){var e=[];if(void 0!==this.body.nodes[t])for(var i=this.body.nodes[t],o={},n=0;n<i.edges.length;n++){var s=i.edges[n];s.toId==i.id?void 0===o[s.fromId]&&(e.push(s.fromId),o[s.fromId]=!0):s.fromId==i.id&&void 0===o[s.toId]&&(e.push(s.toId),o[s.toId]=!0)}return e}},{key:"getConnectedEdges",value:function(t){var e=[];if(void 0!==this.body.nodes[t])for(var i=this.body.nodes[t],o=0;o<i.edges.length;o++)e.push(i.edges[o].id);else console.log("NodeId provided for getConnectedEdges does not exist. Provided: ",t);return e}},{key:"moveNode",value:function(t,e,i){var o=this;void 0!==this.body.nodes[t]?(this.body.nodes[t].x=Number(e),this.body.nodes[t].y=Number(i),setTimeout(function(){o.body.emitter.emit("startSimulation")},0)):console.log("Node id supplied to moveNode does not exist. Provided: ",t)}}]),t}();e["default"]=p},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(66),a=o(r),h=i(67),d=o(h),l=i(69),c=o(l),u=i(71),p=o(u),f=i(72),m=o(f),v=i(73),g=o(v),y=i(75),b=o(y),w=i(76),_=o(w),x=i(77),k=o(x),O=i(78),M=o(O),D=i(79),S=o(D),C=i(80),T=o(C),E=i(81),P=o(E),I=i(82),N=o(I),R=i(83),z=o(R),L=i(29),A=(o(L),i(1)),B=function(){function t(e,i,o,s,r){n(this,t),this.options=A.bridgeObject(r),this.globalOptions=r,this.body=i,this.edges=[],this.id=void 0,this.imagelist=o,this.grouplist=s,this.x=void 0,this.y=void 0,this.baseSize=this.options.size,this.baseFontSize=this.options.font.size,this.predefinedPosition=!1,this.selected=!1,this.hover=!1,this.labelModule=new a["default"](this.body,this.options,!1),this.setOptions(e)}return s(t,[{key:"attachEdge",value:function(t){-1===this.edges.indexOf(t)&&this.edges.push(t)}},{key:"detachEdge",value:function(t){var e=this.edges.indexOf(t);-1!=e&&this.edges.splice(e,1)}},{key:"setOptions",value:function(e){var i=this.options.shape;if(e){if(void 0!==e.id&&(this.id=e.id),void 0===this.id)throw"Node must have an id";if(void 0!==e.x&&(null===e.x?(this.x=void 0,this.predefinedPosition=!1):(this.x=parseInt(e.x),this.predefinedPosition=!0)),void 0!==e.y&&(null===e.y?(this.y=void 0,this.predefinedPosition=!1):(this.y=parseInt(e.y),this.predefinedPosition=!0)),void 0!==e.size&&(this.baseSize=e.size),void 0!==e.value&&(e.value=parseFloat(e.value)),"number"==typeof e.group||"string"==typeof e.group&&""!=e.group){var o=this.grouplist.get(e.group);A.deepExtend(this.options,o),this.options.color=A.parseColor(this.options.color)}if(t.parseOptions(this.options,e,!0,this.globalOptions),void 0!==this.options.image){if(!this.imagelist)throw"No imagelist provided";this.imageObj=this.imagelist.load(this.options.image,this.options.brokenImage,this.id)}return this.updateLabelModule(),this.updateShape(i),void 0!==e.hidden||void 0!==e.physics}}},{key:"updateLabelModule",value:function(){void 0!==this.options.label&&null!==this.options.label||(this.options.label=""),this.labelModule.setOptions(this.options,!0),void 0!==this.labelModule.baseSize&&(this.baseFontSize=this.labelModule.baseSize)}},{key:"updateShape",value:function(t){if(t===this.options.shape&&this.shape)this.shape.setOptions(this.options,this.imageObj);else switch(this.options.shape){case"box":this.shape=new d["default"](this.options,this.body,this.labelModule);break;case"circle":this.shape=new c["default"](this.options,this.body,this.labelModule);break;case"circularImage":this.shape=new p["default"](this.options,this.body,this.labelModule,this.imageObj);break;case"database":this.shape=new m["default"](this.options,this.body,this.labelModule);break;case"diamond":this.shape=new g["default"](this.options,this.body,this.labelModule);break;case"dot":this.shape=new b["default"](this.options,this.body,this.labelModule);break;case"ellipse":this.shape=new _["default"](this.options,this.body,this.labelModule);break;case"icon":this.shape=new k["default"](this.options,this.body,this.labelModule);break;case"image":this.shape=new M["default"](this.options,this.body,this.labelModule,this.imageObj);break;case"square":this.shape=new S["default"](this.options,this.body,this.labelModule);break;case"star":this.shape=new T["default"](this.options,this.body,this.labelModule);break;case"text":this.shape=new P["default"](this.options,this.body,this.labelModule);break;case"triangle":this.shape=new N["default"](this.options,this.body,this.labelModule);break;case"triangleDown":this.shape=new z["default"](this.options,this.body,this.labelModule);break;default:this.shape=new _["default"](this.options,this.body,this.labelModule)}this._reset()}},{key:"select",value:function(){this.selected=!0,this._reset()}},{key:"unselect",value:function(){this.selected=!1,this._reset()}},{key:"_reset",value:function(){this.shape.width=void 0,this.shape.height=void 0}},{key:"getTitle",value:function(){return this.options.title}},{key:"distanceToBorder",value:function(t,e){return this.shape.distanceToBorder(t,e)}},{key:"isFixed",value:function(){return this.options.fixed.x&&this.options.fixed.y}},{key:"isSelected",value:function(){return this.selected}},{key:"getValue",value:function(){return this.options.value}},{key:"setValueRange",value:function(t,e,i){if(void 0!==this.options.value){var o=this.options.scaling.customScalingFunction(t,e,i,this.options.value),n=this.options.scaling.max-this.options.scaling.min;if(this.options.scaling.label.enabled===!0){var s=this.options.scaling.label.max-this.options.scaling.label.min;this.options.font.size=this.options.scaling.label.min+o*s}this.options.size=this.options.scaling.min+o*n}else this.options.size=this.baseSize,this.options.font.size=this.baseFontSize;this.updateLabelModule()}},{key:"draw",value:function(t){this.shape.draw(t,this.x,this.y,this.selected,this.hover)}},{key:"updateBoundingBox",value:function(t){this.shape.updateBoundingBox(this.x,this.y,t)}},{key:"resize",value:function(t){this.shape.resize(t,this.selected)}},{key:"isOverlappingWith",value:function(t){return this.shape.left<t.right&&this.shape.left+this.shape.width>t.left&&this.shape.top<t.bottom&&this.shape.top+this.shape.height>t.top}},{key:"isBoundingBoxOverlappingWith",value:function(t){return this.shape.boundingBox.left<t.right&&this.shape.boundingBox.right>t.left&&this.shape.boundingBox.top<t.bottom&&this.shape.boundingBox.bottom>t.top}}],[{key:"parseOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=arguments.length<=3||void 0===arguments[3]?{}:arguments[3],n=["color","font","fixed","shadow"];if(A.selectiveNotDeepExtend(n,t,e,i),A.mergeOptions(t,e,"shadow",i,o),void 0!==e.color&&null!==e.color){var s=A.parseColor(e.color);A.fillIfDefined(t.color,s)}else i===!0&&null===e.color&&(t.color=A.bridgeObject(o.color));void 0!==e.fixed&&null!==e.fixed&&("boolean"==typeof e.fixed?(t.fixed.x=e.fixed,t.fixed.y=e.fixed):(void 0!==e.fixed.x&&"boolean"==typeof e.fixed.x&&(t.fixed.x=e.fixed.x),void 0!==e.fixed.y&&"boolean"==typeof e.fixed.y&&(t.fixed.y=e.fixed.y))),void 0!==e.font&&null!==e.font?a["default"].parseOptions(t.font,e):i===!0&&null===e.font&&(t.font=A.bridgeObject(o.font)),void 0!==e.scaling&&A.mergeOptions(t.scaling,e.scaling,"label",i,o.scaling)}}]),t}();e["default"]=B},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(1),h=function(){function t(e,i){var n=arguments.length<=2||void 0===arguments[2]?!1:arguments[2];o(this,t),this.body=e,this.pointToSelf=!1,this.baseSize=void 0,this.fontOptions={},this.setOptions(i),this.size={top:0,left:0,width:0,height:0,yLine:0},this.isEdgeLabel=n}return r(t,[{key:"setOptions",value:function(e){var i=arguments.length<=1||void 0===arguments[1]?!1:arguments[1];this.nodeOptions=e,this.fontOptions=a.deepExtend({},e.font,!0),void 0!==e.label&&(this.labelDirty=!0),void 0!==e.font&&(t.parseOptions(this.fontOptions,e,i),"string"==typeof e.font?this.baseSize=this.fontOptions.size:"object"===s(e.font)&&void 0!==e.font.size&&(this.baseSize=e.font.size))}},{key:"draw",value:function(t,e,i,o){var n=arguments.length<=4||void 0===arguments[4]?"middle":arguments[4];if(void 0!==this.nodeOptions.label){var s=this.fontOptions.size*this.body.view.scale;this.nodeOptions.label&&s<this.nodeOptions.scaling.label.drawThreshold-1||(this.calculateLabelSize(t,o,e,i,n),
-this._drawBackground(t),this._drawText(t,o,e,i,n))}}},{key:"_drawBackground",value:function(t){if(void 0!==this.fontOptions.background&&"none"!==this.fontOptions.background){t.fillStyle=this.fontOptions.background;var e=2;if(this.isEdgeLabel)switch(this.fontOptions.align){case"middle":t.fillRect(.5*-this.size.width,.5*-this.size.height,this.size.width,this.size.height);break;case"top":t.fillRect(.5*-this.size.width,-(this.size.height+e),this.size.width,this.size.height);break;case"bottom":t.fillRect(.5*-this.size.width,e,this.size.width,this.size.height);break;default:t.fillRect(this.size.left,this.size.top-.5*e,this.size.width,this.size.height)}else t.fillRect(this.size.left,this.size.top-.5*e,this.size.width,this.size.height)}}},{key:"_drawText",value:function(t,e,i,o){var s=arguments.length<=4||void 0===arguments[4]?"middle":arguments[4],r=this.fontOptions.size,a=r*this.body.view.scale;a>=this.nodeOptions.scaling.label.maxVisible&&(r=Number(this.nodeOptions.scaling.label.maxVisible)/this.body.view.scale);var h=this.size.yLine,d=this._getColor(a),l=n(d,2),c=l[0],u=l[1],p=this._setAlignment(t,i,h,s),f=n(p,2);i=f[0],h=f[1],t.font=(e&&this.nodeOptions.labelHighlightBold?"bold ":"")+r+"px "+this.fontOptions.face,t.fillStyle=c,this.isEdgeLabel||"left"!==this.fontOptions.align?t.textAlign="center":(t.textAlign=this.fontOptions.align,i-=.5*this.size.width),this.fontOptions.strokeWidth>0&&(t.lineWidth=this.fontOptions.strokeWidth,t.strokeStyle=u,t.lineJoin="round");for(var m=0;m<this.lineCount;m++)this.fontOptions.strokeWidth>0&&t.strokeText(this.lines[m],i,h),t.fillText(this.lines[m],i,h),h+=r}},{key:"_setAlignment",value:function(t,e,i,o){if(this.isEdgeLabel&&"horizontal"!==this.fontOptions.align&&this.pointToSelf===!1){e=0,i=0;var n=2;"top"===this.fontOptions.align?(t.textBaseline="alphabetic",i-=2*n):"bottom"===this.fontOptions.align?(t.textBaseline="hanging",i+=2*n):t.textBaseline="middle"}else t.textBaseline=o;return[e,i]}},{key:"_getColor",value:function(t){var e=this.fontOptions.color||"#000000",i=this.fontOptions.strokeColor||"#ffffff";if(t<=this.nodeOptions.scaling.label.drawThreshold){var o=Math.max(0,Math.min(1,1-(this.nodeOptions.scaling.label.drawThreshold-t)));e=a.overrideOpacity(e,o),i=a.overrideOpacity(i,o)}return[e,i]}},{key:"getTextSize",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i={width:this._processLabel(t,e),height:this.fontOptions.size*this.lineCount,lineCount:this.lineCount};return i}},{key:"calculateLabelSize",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?0:arguments[2],o=arguments.length<=3||void 0===arguments[3]?0:arguments[3],n=arguments.length<=4||void 0===arguments[4]?"middle":arguments[4];this.labelDirty===!0&&(this.size.width=this._processLabel(t,e)),this.size.height=this.fontOptions.size*this.lineCount,this.size.left=i-.5*this.size.width,this.size.top=o-.5*this.size.height,this.size.yLine=o+.5*(1-this.lineCount)*this.fontOptions.size,"hanging"===n&&(this.size.top+=.5*this.fontOptions.size,this.size.top+=4,this.size.yLine+=4),this.labelDirty=!1}},{key:"_processLabel",value:function(t,e){var i=0,o=[""],n=0;if(void 0!==this.nodeOptions.label){o=String(this.nodeOptions.label).split("\n"),n=o.length,t.font=(e&&this.nodeOptions.labelHighlightBold?"bold ":"")+this.fontOptions.size+"px "+this.fontOptions.face,i=t.measureText(o[0]).width;for(var s=1;n>s;s++){var r=t.measureText(o[s]).width;i=r>i?r:i}}return this.lines=o,this.lineCount=n,i}}],[{key:"parseOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2];if("string"==typeof e.font){var o=e.font.split(" ");t.size=o[0].replace("px",""),t.face=o[1],t.color=o[2]}else"object"===s(e.font)&&a.fillIfDefined(t,e.font,i);t.size=Number(t.size)}}]),t}();e["default"]=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e);this.width=o.width+2*i,this.height=o.height+2*i,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-this.width/2,this.top=i-this.height/2;var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth;t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.lineWidth=o?r:s,t.lineWidth/=this.body.view.scale,t.lineWidth=Math.min(this.width,t.lineWidth),t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background;var a=this.options.shapeProperties.borderRadius;t.roundRect(this.left,this.top,this.width,this.height,a),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),s>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),this.updateBoundingBox(e,i,t,o),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-.5*this.width,this.top=e-.5*this.height;var n=this.options.shapeProperties.borderRadius;this.boundingBox.left=this.left-n,this.boundingBox.top=this.top-n,this.boundingBox.bottom=this.top+this.height+n,this.boundingBox.right=this.left+this.width+n}},{key:"distanceToBorder",value:function(t,e){this.resize(t);var i=this.options.borderWidth;return Math.min(Math.abs(this.width/2/Math.cos(e)),Math.abs(this.height/2/Math.sin(e)))+i}}]),e}(d["default"]);e["default"]=l},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=o,this.labelModule=n,this.setOptions(e),this.top=void 0,this.left=void 0,this.height=void 0,this.width=void 0,this.radius=void 0,this.boundingBox={top:0,left:0,right:0,bottom:0}}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"_distanceToBorder",value:function(t,e){var i=this.options.borderWidth;return this.resize(t),Math.min(Math.abs(this.width/2/Math.cos(e)),Math.abs(this.height/2/Math.sin(e)))+i}},{key:"enableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor=this.options.shadow.color,t.shadowBlur=this.options.shadow.size,t.shadowOffsetX=this.options.shadow.x,t.shadowOffsetY=this.options.shadow.y)}},{key:"disableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor="rgba(0,0,0,0)",t.shadowBlur=0,t.shadowOffsetX=0,t.shadowOffsetY=0)}},{key:"enableBorderDashes",value:function(t){if(this.options.shapeProperties.borderDashes!==!1)if(void 0!==t.setLineDash){var e=this.options.shapeProperties.borderDashes;e===!0&&(e=[5,15]),t.setLineDash(e)}else console.warn("setLineDash is not supported in this browser. The dashed borders cannot be used."),this.options.shapeProperties.borderDashes=!1}},{key:"disableBorderDashes",value:function(t){this.options.shapeProperties.borderDashes!==!1&&(void 0!==t.setLineDash?t.setLineDash([0]):(console.warn("setLineDash is not supported in this browser. The dashed borders cannot be used."),this.options.shapeProperties.borderDashes=!1))}}]),t}();e["default"]=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(70),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e),n=Math.max(o.width,o.height)+2*i;this.options.size=n/2,this.width=n,this.height=n,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-this.width/2,this.top=i-this.height/2,this._drawRawCircle(t,e,i,o,n,this.options.size),this.boundingBox.top=i-this.options.size,this.boundingBox.left=e-this.options.size,this.boundingBox.right=e+this.options.size,this.boundingBox.bottom=i+this.options.size,this.updateBoundingBox(e,i),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e){this.boundingBox.top=e-this.options.size,this.boundingBox.left=t-this.options.size,this.boundingBox.right=t+this.options.size,this.boundingBox.bottom=e+this.options.size}},{key:"distanceToBorder",value:function(t,e){return this.resize(t),.5*this.width}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){n(this,e);var r=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return r.labelOffset=0,r.imageLoaded=!1,r}return r(e,t),a(e,[{key:"setOptions",value:function(t,e){this.options=t,e&&(this.imageObj=e)}},{key:"_resizeImage",value:function(){var t=!1;if(this.imageObj.width&&this.imageObj.height?this.imageLoaded===!1&&(this.imageLoaded=!0,t=!0):this.imageLoaded=!1,!this.width||!this.height||t===!0){var e,i,o;this.imageObj.width&&this.imageObj.height&&(e=0,i=0),this.options.shapeProperties.useImageSize===!1?this.imageObj.width>this.imageObj.height?(o=this.imageObj.width/this.imageObj.height,e=2*this.options.size*o||this.imageObj.width,i=2*this.options.size||this.imageObj.height):(o=this.imageObj.width&&this.imageObj.height?this.imageObj.height/this.imageObj.width:1,e=2*this.options.size,i=2*this.options.size*o):(e=this.imageObj.width,i=this.imageObj.height),this.width=e,this.height=i,this.radius=.5*this.width}}},{key:"_drawRawCircle",value:function(t,e,i,o,n,s){var r=this.options.borderWidth,a=this.options.borderWidthSelected||2*this.options.borderWidth,h=(o?a:r)/this.body.view.scale;t.lineWidth=Math.min(this.width,h),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.circle(e,i,s),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),h>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore()}},{key:"_drawImageAtPosition",value:function(t){if(0!=this.imageObj.width){t.globalAlpha=1,this.enableShadow(t);var e=this.imageObj.width/this.width/this.body.view.scale;if(e>2&&this.options.shapeProperties.interpolation===!0){var i=this.imageObj.width,o=this.imageObj.height,n=document.createElement("canvas");n.width=i,n.height=i;var s=n.getContext("2d");e*=.5,i*=.5,o*=.5,s.drawImage(this.imageObj,0,0,i,o);for(var r=0,a=1;e>2&&4>a;)s.drawImage(n,r,0,i,o,r+i,0,i/2,o/2),r+=i,e*=.5,i*=.5,o*=.5,a+=1;t.drawImage(n,r,0,i,o,this.left,this.top,this.width,this.height)}else t.drawImage(this.imageObj,this.left,this.top,this.width,this.height);this.disableShadow(t)}}},{key:"_drawImageLabel",value:function(t,e,i,o){var n,s=0;if(void 0!==this.height){s=.5*this.height;var r=this.labelModule.getTextSize(t);r.lineCount>=1&&(s+=r.height/2)}n=i+s,this.options.label&&(this.labelOffset=s),this.labelModule.draw(t,e,n,o,"hanging")}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(70),d=o(h),l=function(t){function e(t,i,o,r){n(this,e);var a=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return a.imageObj=r,a._swapToImageResizeWhenImageLoaded=!0,a}return r(e,t),a(e,[{key:"resize",value:function(){if(void 0===this.imageObj.src||void 0===this.imageObj.width||void 0===this.imageObj.height){if(!this.width){var t=2*this.options.size;this.width=t,this.height=t,this._swapToImageResizeWhenImageLoaded=!0,this.radius=.5*this.width}}else this._swapToImageResizeWhenImageLoaded&&(this.width=void 0,this.height=void 0,this._swapToImageResizeWhenImageLoaded=!1),this._resizeImage()}},{key:"draw",value:function(t,e,i,o,n){this.resize(),this.left=e-this.width/2,this.top=i-this.height/2;var s=Math.min(.5*this.height,.5*this.width);this._drawRawCircle(t,e,i,o,n,s),t.save(),t.clip(),this._drawImageAtPosition(t),t.restore(),this._drawImageLabel(t,e,i,o),this.updateBoundingBox(e,i)}},{key:"updateBoundingBox",value:function(t,e){this.boundingBox.top=e-this.options.size,this.boundingBox.left=t-this.options.size,this.boundingBox.right=t+this.options.size,this.boundingBox.bottom=e+this.options.size,this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelOffset)}},{key:"distanceToBorder",value:function(t,e){return this.resize(t),.5*this.width}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e),n=o.width+2*i;this.width=n,this.height=n,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-this.width/2,this.top=i-this.height/2;var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth,a=(o?r:s)/this.body.view.scale;t.lineWidth=Math.min(this.width,a),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.database(e-this.width/2,i-.5*this.height,this.width,this.height),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),a>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),this.updateBoundingBox(e,i,t,o),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-.5*this.width,this.top=e-.5*this.height,this.boundingBox.left=this.left,this.boundingBox.top=this.top,this.boundingBox.bottom=this.top+this.height,this.boundingBox.right=this.left+this.width}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"diamond",4,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_resizeShape",value:function(){if(void 0===this.width){var t=2*this.options.size;this.width=t,this.height=t,this.radius=.5*this.width}}},{key:"_drawShape",value:function(t,e,i,o,n,s,r){this._resizeShape(),this.left=o-this.width/2,this.top=n-this.height/2;var a=this.options.borderWidth,h=this.options.borderWidthSelected||2*this.options.borderWidth,d=(s?h:a)/this.body.view.scale;if(t.lineWidth=Math.min(this.width,d),t.strokeStyle=s?this.options.color.highlight.border:r?this.options.color.hover.border:this.options.color.border,t.fillStyle=s?this.options.color.highlight.background:r?this.options.color.hover.background:this.options.color.background,t[e](o,n,this.options.size),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),d>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),void 0!==this.options.label){var l=n+.5*this.height+3;this.labelModule.draw(t,o,l,s,"hanging")}this.updateBoundingBox(o,n)}},{key:"updateBoundingBox",value:function(t,e){this.boundingBox.top=e-this.options.size,this.boundingBox.left=t-this.options.size,this.boundingBox.right=t+this.options.size,this.boundingBox.bottom=e+this.options.size,void 0!==this.options.label&&this.labelModule.size.width>0&&(this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelModule.size.height+3))}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"circle",2,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this.resize(t),this.options.size}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=this.labelModule.getTextSize(t,e);this.width=1.5*i.width,this.height=2*i.height,this.width<this.height&&(this.width=this.height),this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o),this.left=e-.5*this.width,this.top=i-.5*this.height;var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth,a=(o?r:s)/this.body.view.scale;t.lineWidth=Math.min(this.width,a),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.ellipse(this.left,this.top,this.width,this.height),this.enableShadow(t),t.fill(),this.disableShadow(t),t.save(),a>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),this.updateBoundingBox(e,i,t,o),this.labelModule.draw(t,e,i,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-.5*this.width,this.top=e-.5*this.height,this.boundingBox.left=this.left,this.boundingBox.top=this.top,this.boundingBox.bottom=this.top+this.height,this.boundingBox.right=this.left+this.width}},{key:"distanceToBorder",value:function(t,e){this.resize(t);var i=.5*this.width,o=.5*this.height,n=Math.sin(e)*i,s=Math.cos(e)*o;return i*o/Math.sqrt(n*n+s*s)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){if(void 0===this.width){var e=5,i={width:Number(this.options.icon.size),height:Number(this.options.icon.size)};this.width=i.width+2*e,this.height=i.height+2*e,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){if(this.resize(t),this.options.icon.size=this.options.icon.size||50,this.left=e-.5*this.width,this.top=i-.5*this.height,this._icon(t,e,i,o),void 0!==this.options.label){var s=5;this.labelModule.draw(t,e,i+.5*this.height+s,o)}this.updateBoundingBox(e,i)}},{key:"updateBoundingBox",value:function(t,e){if(this.boundingBox.top=e-.5*this.options.icon.size,this.boundingBox.left=t-.5*this.options.icon.size,this.boundingBox.right=t+.5*this.options.icon.size,this.boundingBox.bottom=e+.5*this.options.icon.size,void 0!==this.options.label&&this.labelModule.size.width>0){var i=5;this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelModule.size.height+i)}}},{key:"_icon",value:function(t,e,i,o){var n=Number(this.options.icon.size);void 0!==this.options.icon.code?(t.font=(o?"bold ":"")+n+"px "+this.options.icon.face,t.fillStyle=this.options.icon.color||"black",t.textAlign="center",t.textBaseline="middle",this.enableShadow(t),t.fillText(this.options.icon.code,e,i),this.disableShadow(t)):console.error("When using the icon shape, you need to define the code in the icon options object. This can be done per node or globally.")}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(70),d=o(h),l=function(t){function e(t,i,o,r){n(this,e);var a=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return a.imageObj=r,a}return r(e,t),a(e,[{key:"resize",value:function(){this._resizeImage()}},{key:"draw",value:function(t,e,i,o,n){if(this.resize(),this.left=e-this.width/2,this.top=i-this.height/2,this.options.shapeProperties.useBorderWithImage===!0){var s=this.options.borderWidth,r=this.options.borderWidthSelected||2*this.options.borderWidth,a=(o?r:s)/this.body.view.scale;t.lineWidth=Math.min(this.width,a),t.beginPath(),t.strokeStyle=o?this.options.color.highlight.border:n?this.options.color.hover.border:this.options.color.border,t.fillStyle=o?this.options.color.highlight.background:n?this.options.color.hover.background:this.options.color.background,t.rect(this.left-.5*t.lineWidth,this.top-.5*t.lineWidth,this.width+t.lineWidth,this.height+t.lineWidth),t.fill(),t.save(),a>0&&(this.enableBorderDashes(t),t.stroke(),this.disableBorderDashes(t)),t.restore(),t.closePath()}this._drawImageAtPosition(t),this._drawImageLabel(t,e,i,o||n),this.updateBoundingBox(e,i)}},{key:"updateBoundingBox",value:function(t,e){this.resize(),this.left=t-this.width/2,this.top=e-this.height/2,this.boundingBox.top=this.top,this.boundingBox.left=this.left,this.boundingBox.right=this.left+this.width,this.boundingBox.bottom=this.top+this.height,void 0!==this.options.label&&this.labelModule.size.width>0&&(this.boundingBox.left=Math.min(this.boundingBox.left,this.labelModule.size.left),this.boundingBox.right=Math.max(this.boundingBox.right,this.labelModule.size.left+this.labelModule.size.width),this.boundingBox.bottom=Math.max(this.boundingBox.bottom,this.boundingBox.bottom+this.labelOffset))}},{key:"distanceToBorder",
-value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"square",2,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"star",4,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(68),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t,e){if(void 0===this.width){var i=5,o=this.labelModule.getTextSize(t,e);this.width=o.width+2*i,this.height=o.height+2*i,this.radius=.5*this.width}}},{key:"draw",value:function(t,e,i,o,n){this.resize(t,o||n),this.left=e-this.width/2,this.top=i-this.height/2,this.enableShadow(t),this.labelModule.draw(t,e,i,o||n),this.disableShadow(t),this.updateBoundingBox(e,i,t,o)}},{key:"updateBoundingBox",value:function(t,e,i,o){this.resize(i,o),this.left=t-this.width/2,this.top=e-this.height/2,this.boundingBox.top=this.top,this.boundingBox.left=this.left,this.boundingBox.right=this.left+this.width,this.boundingBox.bottom=this.top+this.height}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"triangle",3,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(74),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"resize",value:function(t){this._resizeShape()}},{key:"draw",value:function(t,e,i,o,n){this._drawShape(t,"triangleDown",3,e,i,o,n)}},{key:"distanceToBorder",value:function(t,e){return this._distanceToBorder(t,e)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(85),a=o(r),h=i(66),d=o(h),l=i(1),c=i(9),u=i(11),p=function(){function t(e,i,o){var s=this;n(this,t),this.body=e,this.images=i,this.groups=o,this.body.functions.createEdge=this.create.bind(this),this.edgesListeners={add:function(t,e){s.add(e.items)},update:function(t,e){s.update(e.items)},remove:function(t,e){s.remove(e.items)}},this.options={},this.defaultOptions={arrows:{to:{enabled:!1,scaleFactor:1},middle:{enabled:!1,scaleFactor:1},from:{enabled:!1,scaleFactor:1}},arrowStrikethrough:!0,color:{color:"#848484",highlight:"#848484",hover:"#848484",inherit:"from",opacity:1},dashes:!1,font:{color:"#343434",size:14,face:"arial",background:"none",strokeWidth:2,strokeColor:"#ffffff",align:"horizontal"},hidden:!1,hoverWidth:1.5,label:void 0,labelHighlightBold:!0,length:void 0,physics:!0,scaling:{min:1,max:15,label:{enabled:!0,min:14,max:30,maxVisible:30,drawThreshold:5},customScalingFunction:function(t,e,i,o){if(e===t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)}},selectionWidth:1.5,selfReferenceSize:20,shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:10,x:5,y:5},smooth:{enabled:!0,type:"dynamic",forceDirection:"none",roundness:.5},title:void 0,width:1,value:void 0},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("_forceDisableDynamicCurves",function(e){"dynamic"===e&&(e="continuous");var i=!1;for(var o in t.body.edges)if(t.body.edges.hasOwnProperty(o)){var n=t.body.edges[o],s=t.body.data.edges._data[o];if(void 0!==s){var r=s.smooth;void 0!==r&&r.enabled===!0&&"dynamic"===r.type&&(void 0===e?n.setOptions({smooth:!1}):n.setOptions({smooth:{type:e}}),i=!0)}}i===!0&&t.body.emitter.emit("_dataChanged")}),this.body.emitter.on("_dataUpdated",function(){t.reconnectEdges(),t.markAllEdgesAsDirty()}),this.body.emitter.on("refreshEdges",this.refresh.bind(this)),this.body.emitter.on("refresh",this.refresh.bind(this)),this.body.emitter.on("destroy",function(){l.forEach(t.edgesListeners,function(e,i){t.body.data.edges&&t.body.data.edges.off(i,e)}),delete t.body.functions.createEdge,delete t.edgesListeners.add,delete t.edgesListeners.update,delete t.edgesListeners.remove,delete t.edgesListeners})}},{key:"setOptions",value:function(t){if(void 0!==t){a["default"].parseOptions(this.options,t),void 0!==t.color&&this.markAllEdgesAsDirty();var e=!1;if(void 0!==t.smooth)for(var i in this.body.edges)this.body.edges.hasOwnProperty(i)&&(e=this.body.edges[i].updateEdgeType()||e);if(void 0!==t.font){d["default"].parseOptions(this.options.font,t);for(var o in this.body.edges)this.body.edges.hasOwnProperty(o)&&this.body.edges[o].updateLabelModule()}void 0===t.hidden&&void 0===t.physics&&e!==!0||this.body.emitter.emit("_dataChanged")}}},{key:"setData",value:function(t){var e=this,i=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],o=this.body.data.edges;if(t instanceof c||t instanceof u)this.body.data.edges=t;else if(Array.isArray(t))this.body.data.edges=new c,this.body.data.edges.add(t);else{if(t)throw new TypeError("Array or DataSet expected");this.body.data.edges=new c}if(o&&l.forEach(this.edgesListeners,function(t,e){o.off(e,t)}),this.body.edges={},this.body.data.edges){l.forEach(this.edgesListeners,function(t,i){e.body.data.edges.on(i,t)});var n=this.body.data.edges.getIds();this.add(n,!0)}i===!1&&this.body.emitter.emit("_dataChanged")}},{key:"add",value:function(t){for(var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=this.body.edges,o=this.body.data.edges,n=0;n<t.length;n++){var s=t[n],r=i[s];r&&r.disconnect();var a=o.get(s,{showInternalIds:!0});i[s]=this.create(a)}e===!1&&this.body.emitter.emit("_dataChanged")}},{key:"update",value:function(t){for(var e=this.body.edges,i=this.body.data.edges,o=!1,n=0;n<t.length;n++){var s=t[n],r=i.get(s),a=e[s];void 0!==a?(a.disconnect(),o=a.setOptions(r)||o,a.connect()):(this.body.edges[s]=this.create(r),o=!0)}o===!0?this.body.emitter.emit("_dataChanged"):this.body.emitter.emit("_dataUpdated")}},{key:"remove",value:function(t){for(var e=this.body.edges,i=0;i<t.length;i++){var o=t[i],n=e[o];void 0!==n&&(n.cleanup(),n.disconnect(),delete e[o])}this.body.emitter.emit("_dataChanged")}},{key:"refresh",value:function(){var t=this.body.edges;for(var e in t){var i=void 0;t.hasOwnProperty(e)&&(i=t[e]);var o=this.body.data.edges._data[e];void 0!==i&&void 0!==o&&i.setOptions(o)}}},{key:"create",value:function(t){return new a["default"](t,this.body,this.options)}},{key:"markAllEdgesAsDirty",value:function(){for(var t in this.body.edges)this.body.edges[t].edgeType.colorDirty=!0}},{key:"reconnectEdges",value:function(){var t,e=this.body.nodes,i=this.body.edges;for(t in e)e.hasOwnProperty(t)&&(e[t].edges=[]);for(t in i)if(i.hasOwnProperty(t)){var o=i[t];o.from=null,o.to=null,o.connect()}}},{key:"getConnectedNodes",value:function(t){var e=[];if(void 0!==this.body.edges[t]){var i=this.body.edges[t];i.fromId&&e.push(i.fromId),i.toId&&e.push(i.toId)}return e}}]),t}();e["default"]=p},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(66),h=o(a),d=i(86),l=o(d),c=i(90),u=o(c),p=i(91),f=o(p),m=i(92),v=o(m),g=i(1),y=function(){function t(e,i,o){if(n(this,t),void 0===i)throw"No body provided";this.options=g.bridgeObject(o),this.globalOptions=o,this.body=i,this.id=void 0,this.fromId=void 0,this.toId=void 0,this.selected=!1,this.hover=!1,this.labelDirty=!0,this.colorDirty=!0,this.baseWidth=this.options.width,this.baseFontSize=this.options.font.size,this.from=void 0,this.to=void 0,this.edgeType=void 0,this.connected=!1,this.labelModule=new h["default"](this.body,this.options,!0),this.setOptions(e)}return r(t,[{key:"setOptions",value:function(e){if(e){this.colorDirty=!0,t.parseOptions(this.options,e,!0,this.globalOptions),void 0!==e.id&&(this.id=e.id),void 0!==e.from&&(this.fromId=e.from),void 0!==e.to&&(this.toId=e.to),void 0!==e.title&&(this.title=e.title),void 0!==e.value&&(e.value=parseFloat(e.value)),this.updateLabelModule();var i=this.updateEdgeType();return this._setInteractionWidths(),this.connect(),void 0===e.hidden&&void 0===e.physics||(i=!0),i}}},{key:"updateLabelModule",value:function(){this.labelModule.setOptions(this.options,!0),void 0!==this.labelModule.baseSize&&(this.baseFontSize=this.labelModule.baseSize)}},{key:"updateEdgeType",value:function(){var t=!1,e=!0,i=this.options.smooth;return void 0!==this.edgeType&&(this.edgeType instanceof u["default"]&&i.enabled===!0&&"dynamic"===i.type&&(e=!1),this.edgeType instanceof l["default"]&&i.enabled===!0&&"cubicBezier"===i.type&&(e=!1),this.edgeType instanceof f["default"]&&i.enabled===!0&&"dynamic"!==i.type&&"cubicBezier"!==i.type&&(e=!1),this.edgeType instanceof v["default"]&&i.enabled===!1&&(e=!1),e===!0&&(t=this.cleanup())),e===!0?this.options.smooth.enabled===!0?"dynamic"===this.options.smooth.type?(t=!0,this.edgeType=new u["default"](this.options,this.body,this.labelModule)):"cubicBezier"===this.options.smooth.type?this.edgeType=new l["default"](this.options,this.body,this.labelModule):this.edgeType=new f["default"](this.options,this.body,this.labelModule):this.edgeType=new v["default"](this.options,this.body,this.labelModule):this.edgeType.setOptions(this.options),t}},{key:"connect",value:function(){this.disconnect(),this.from=this.body.nodes[this.fromId]||void 0,this.to=this.body.nodes[this.toId]||void 0,this.connected=void 0!==this.from&&void 0!==this.to,this.connected===!0?(this.from.attachEdge(this),this.to.attachEdge(this)):(this.from&&this.from.detachEdge(this),this.to&&this.to.detachEdge(this)),this.edgeType.connect()}},{key:"disconnect",value:function(){this.from&&(this.from.detachEdge(this),this.from=void 0),this.to&&(this.to.detachEdge(this),this.to=void 0),this.connected=!1}},{key:"getTitle",value:function(){return this.title}},{key:"isSelected",value:function(){return this.selected}},{key:"getValue",value:function(){return this.options.value}},{key:"setValueRange",value:function(t,e,i){if(void 0!==this.options.value){var o=this.options.scaling.customScalingFunction(t,e,i,this.options.value),n=this.options.scaling.max-this.options.scaling.min;if(this.options.scaling.label.enabled===!0){var s=this.options.scaling.label.max-this.options.scaling.label.min;this.options.font.size=this.options.scaling.label.min+o*s}this.options.width=this.options.scaling.min+o*n}else this.options.width=this.baseWidth,this.options.font.size=this.baseFontSize;this._setInteractionWidths(),this.updateLabelModule()}},{key:"_setInteractionWidths",value:function(){"function"==typeof this.options.hoverWidth?this.edgeType.hoverWidth=this.options.hoverWidth(this.options.width):this.edgeType.hoverWidth=this.options.hoverWidth+this.options.width,"function"==typeof this.options.selectionWidth?this.edgeType.selectionWidth=this.options.selectionWidth(this.options.width):this.edgeType.selectionWidth=this.options.selectionWidth+this.options.width}},{key:"draw",value:function(t){var e=this.edgeType.getViaNode(),i={};this.edgeType.fromPoint=this.edgeType.from,this.edgeType.toPoint=this.edgeType.to,this.options.arrows.from.enabled===!0&&(i.from=this.edgeType.getArrowData(t,"from",e,this.selected,this.hover),this.options.arrowStrikethrough===!1&&(this.edgeType.fromPoint=i.from.core)),this.options.arrows.to.enabled===!0&&(i.to=this.edgeType.getArrowData(t,"to",e,this.selected,this.hover),this.options.arrowStrikethrough===!1&&(this.edgeType.toPoint=i.to.core)),this.options.arrows.middle.enabled===!0&&(i.middle=this.edgeType.getArrowData(t,"middle",e,this.selected,this.hover)),this.edgeType.drawLine(t,this.selected,this.hover,e),this.drawArrows(t,i),this.drawLabel(t,e)}},{key:"drawArrows",value:function(t,e){this.options.arrows.from.enabled===!0&&this.edgeType.drawArrowHead(t,this.selected,this.hover,e.from),this.options.arrows.middle.enabled===!0&&this.edgeType.drawArrowHead(t,this.selected,this.hover,e.middle),this.options.arrows.to.enabled===!0&&this.edgeType.drawArrowHead(t,this.selected,this.hover,e.to)}},{key:"drawLabel",value:function(t,e){if(void 0!==this.options.label){var i=this.from,o=this.to,n=this.from.selected||this.to.selected||this.selected;if(i.id!=o.id){this.labelModule.pointToSelf=!1;var s=this.edgeType.getPoint(.5,e);t.save(),"horizontal"!==this.options.font.align&&(this.labelModule.calculateLabelSize(t,n,s.x,s.y),t.translate(s.x,this.labelModule.size.yLine),this._rotateForLabelAlignment(t)),this.labelModule.draw(t,s.x,s.y,n),t.restore()}else{this.labelModule.pointToSelf=!0;var r,a,h=this.options.selfReferenceSize;i.shape.width>i.shape.height?(r=i.x+.5*i.shape.width,a=i.y-h):(r=i.x+h,a=i.y-.5*i.shape.height),s=this._pointOnCircle(r,a,h,.125),this.labelModule.draw(t,s.x,s.y,n)}}}},{key:"isOverlappingWith",value:function(t){if(this.connected){var e=10,i=this.from.x,o=this.from.y,n=this.to.x,s=this.to.y,r=t.left,a=t.top,h=this.edgeType.getDistanceToEdge(i,o,n,s,r,a);return e>h}return!1}},{key:"_rotateForLabelAlignment",value:function(t){var e=this.from.y-this.to.y,i=this.from.x-this.to.x,o=Math.atan2(e,i);(-1>o&&0>i||o>0&&0>i)&&(o+=Math.PI),t.rotate(o)}},{key:"_pointOnCircle",value:function(t,e,i,o){var n=2*o*Math.PI;return{x:t+i*Math.cos(n),y:e-i*Math.sin(n)}}},{key:"select",value:function(){this.selected=!0}},{key:"unselect",value:function(){this.selected=!1}},{key:"cleanup",value:function(){return this.edgeType.cleanup()}}],[{key:"parseOptions",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=arguments.length<=3||void 0===arguments[3]?{}:arguments[3],n=["arrowStrikethrough","id","from","hidden","hoverWidth","label","labelHighlightBold","length","line","opacity","physics","scaling","selectionWidth","selfReferenceSize","to","title","value","width"];if(g.selectiveDeepExtend(n,t,e,i),g.mergeOptions(t,e,"smooth",i,o),g.mergeOptions(t,e,"shadow",i,o),void 0!==e.dashes&&null!==e.dashes?t.dashes=e.dashes:i===!0&&null===e.dashes&&(t.dashes=Object.create(o.dashes)),void 0!==e.scaling&&null!==e.scaling?(void 0!==e.scaling.min&&(t.scaling.min=e.scaling.min),void 0!==e.scaling.max&&(t.scaling.max=e.scaling.max),g.mergeOptions(t.scaling,e.scaling,"label",i,o.scaling)):i===!0&&null===e.scaling&&(t.scaling=Object.create(o.scaling)),void 0!==e.arrows&&null!==e.arrows)if("string"==typeof e.arrows){var r=e.arrows.toLowerCase();t.arrows.to.enabled=-1!=r.indexOf("to"),t.arrows.middle.enabled=-1!=r.indexOf("middle"),t.arrows.from.enabled=-1!=r.indexOf("from")}else{if("object"!==s(e.arrows))throw new Error("The arrow newOptions can only be an object or a string. Refer to the documentation. You used:"+JSON.stringify(e.arrows));g.mergeOptions(t.arrows,e.arrows,"to",i,o.arrows),g.mergeOptions(t.arrows,e.arrows,"middle",i,o.arrows),g.mergeOptions(t.arrows,e.arrows,"from",i,o.arrows)}else i===!0&&null===e.arrows&&(t.arrows=Object.create(o.arrows));if(void 0!==e.color&&null!==e.color)if(t.color=g.deepExtend({},t.color,!0),g.isString(e.color))t.color.color=e.color,t.color.highlight=e.color,t.color.hover=e.color,t.color.inherit=!1;else{var a=!1;void 0!==e.color.color&&(t.color.color=e.color.color,a=!0),void 0!==e.color.highlight&&(t.color.highlight=e.color.highlight,a=!0),void 0!==e.color.hover&&(t.color.hover=e.color.hover,a=!0),void 0!==e.color.inherit&&(t.color.inherit=e.color.inherit),void 0!==e.color.opacity&&(t.color.opacity=Math.min(1,Math.max(0,e.color.opacity))),void 0===e.color.inherit&&a===!0&&(t.color.inherit=!1)}else i===!0&&null===e.color&&(t.color=g.bridgeObject(o.color));void 0!==e.font&&null!==e.font?h["default"].parseOptions(t.font,e):i===!0&&null===e.font&&(t.font=g.bridgeObject(o.font))}}]),t}();e["default"]=y},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),h=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),d=i(87),l=o(d),c=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),h(e,[{key:"_line",value:function(t,e){var i=e[0],o=e[1];t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),void 0===e||void 0===i.x?t.lineTo(this.toPoint.x,this.toPoint.y):t.bezierCurveTo(i.x,i.y,o.x,o.y,this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"_getViaCoordinates",value:function(){var t=this.from.x-this.to.x,e=this.from.y-this.to.y,i=void 0,o=void 0,n=void 0,s=void 0,r=this.options.smooth.roundness;return(Math.abs(t)>Math.abs(e)||this.options.smooth.forceDirection===!0||"horizontal"===this.options.smooth.forceDirection)&&"vertical"!==this.options.smooth.forceDirection?(o=this.from.y,s=this.to.y,i=this.from.x-r*t,n=this.to.x+r*t):(o=this.from.y-r*e,s=this.to.y+r*e,i=this.from.x,n=this.to.x),[{x:i,y:o},{x:n,y:s}]}},{key:"getViaNode",value:function(){return this._getViaCoordinates()}},{key:"_findBorderPosition",value:function(t,e){return this._findBorderPositionBezier(t,e)}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){var r=arguments.length<=6||void 0===arguments[6]?this._getViaCoordinates():arguments[6],h=a(r,2),d=h[0],l=h[1];return this._getDistanceToBezierEdge(t,e,i,o,n,s,d,l)}},{key:"getPoint",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this._getViaCoordinates():arguments[1],i=a(e,2),o=i[0],n=i[1],s=t,r=[];r[0]=Math.pow(1-s,3),r[1]=3*s*Math.pow(1-s,2),r[2]=3*Math.pow(s,2)*(1-s),r[3]=Math.pow(s,3);var h=r[0]*this.fromPoint.x+r[1]*o.x+r[2]*n.x+r[3]*this.toPoint.x,d=r[0]*this.fromPoint.y+r[1]*o.y+r[2]*n.y+r[3]*this.toPoint.y;return{x:h,y:d}}}]),e}(l["default"]);e["default"]=c},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(88),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_getDistanceToBezierEdge",value:function(t,e,i,o,n,s,r,a){var h=1e9,d=void 0,l=void 0,c=void 0,u=void 0,p=void 0,f=t,m=e,v=[0,0,0,0];for(l=1;10>l;l++)c=.1*l,v[0]=Math.pow(1-c,3),v[1]=3*c*Math.pow(1-c,2),v[2]=3*Math.pow(c,2)*(1-c),v[3]=Math.pow(c,3),u=v[0]*t+v[1]*r.x+v[2]*a.x+v[3]*i,p=v[0]*e+v[1]*r.y+v[2]*a.y+v[3]*o,l>0&&(d=this._getDistanceToLine(f,m,u,p,n,s),h=h>d?d:h),f=u,m=p;return h}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(89),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_findBorderPositionBezier",value:function(t,e){var i,o,n,s,r,a=arguments.length<=2||void 0===arguments[2]?this._getViaCoordinates():arguments[2],h=10,d=0,l=0,c=1,u=.2,p=this.to,f=!1;for(t.id===this.from.id&&(p=this.from,f=!0);c>=l&&h>d;){var m=.5*(l+c);if(i=this.getPoint(m,a),o=Math.atan2(p.y-i.y,p.x-i.x),n=p.distanceToBorder(e,o),s=Math.sqrt(Math.pow(i.x-p.x,2)+Math.pow(i.y-p.y,2)),r=n-s,Math.abs(r)<u)break;0>r?f===!1?l=m:c=m:f===!1?c=m:l=m,d++}return i.t=m,i}},{key:"_getDistanceToBezierEdge",value:function(t,e,i,o,n,s,r){var a=1e9,h=void 0,d=void 0,l=void 0,c=void 0,u=void 0,p=t,f=e;for(d=1;10>d;d++)l=.1*d,c=Math.pow(1-l,2)*t+2*l*(1-l)*r.x+Math.pow(l,2)*i,u=Math.pow(1-l,2)*e+2*l*(1-l)*r.y+Math.pow(l,2)*o,d>0&&(h=this._getDistanceToLine(p,f,c,u,n,s),a=a>h?h:a),p=c,f=u;return a}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(1),a=function(){function t(e,i,n){o(this,t),this.body=i,this.labelModule=n,this.options={},this.setOptions(e),this.colorDirty=!0,this.color={},this.selectionWidth=2,this.hoverWidth=1.5,this.fromPoint=this.from,this.toPoint=this.to}return s(t,[{key:"connect",value:function(){this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to]}},{key:"cleanup",value:function(){return!1}},{key:"setOptions",value:function(t){this.options=t,this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to],this.id=this.options.id}},{key:"drawLine",value:function(t,e,i,o){t.strokeStyle=this.getColor(t,e,i),t.lineWidth=this.getLineWidth(e,i),this.options.dashes!==!1?this._drawDashedLine(t,o):this._drawLine(t,o)}},{key:"_drawLine",value:function(t,e,i,o){if(this.from!=this.to)this._line(t,e,i,o);else{var s=this._getCircleData(t),r=n(s,3),a=r[0],h=r[1],d=r[2];this._circle(t,a,h,d)}}},{key:"_drawDashedLine",value:function(t,e,i,o){t.lineCap="round";var s=[5,5];if(Array.isArray(this.options.dashes)===!0&&(s=this.options.dashes),void 0!==t.setLineDash){if(t.save(),t.setLineDash(s),t.lineDashOffset=0,this.from!=this.to)this._line(t,e);else{var r=this._getCircleData(t),a=n(r,3),h=a[0],d=a[1],l=a[2];this._circle(t,h,d,l)}t.setLineDash([0]),t.lineDashOffset=0,t.restore()}else{if(this.from!=this.to)t.dashedLine(this.from.x,this.from.y,this.to.x,this.to.y,s);else{var c=this._getCircleData(t),u=n(c,3),p=u[0],f=u[1],m=u[2];this._circle(t,p,f,m)}this.enableShadow(t),t.stroke(),this.disableShadow(t)}}},{key:"findBorderPosition",value:function(t,e,i){return this.from!=this.to?this._findBorderPosition(t,e,i):this._findBorderPositionCircle(t,e,i)}},{key:"findBorderPositions",value:function(t){var e={},i={};if(this.from!=this.to)e=this._findBorderPosition(this.from,t),i=this._findBorderPosition(this.to,t);else{var o=this._getCircleData(t),s=n(o,3),r=s[0],a=s[1];s[2];e=this._findBorderPositionCircle(this.from,t,{x:r,y:a,low:.25,high:.6,direction:-1}),i=this._findBorderPositionCircle(this.from,t,{x:r,y:a,low:.6,high:.8,direction:1})}return{from:e,to:i}}},{key:"_getCircleData",value:function(t){var e=void 0,i=void 0,o=this.from,n=this.options.selfReferenceSize;return void 0!==t&&void 0===o.shape.width&&o.shape.resize(t),o.shape.width>o.shape.height?(e=o.x+.5*o.shape.width,i=o.y-n):(e=o.x+n,i=o.y-.5*o.shape.height),[e,i,n]}},{key:"_pointOnCircle",value:function(t,e,i,o){var n=2*o*Math.PI;return{x:t+i*Math.cos(n),y:e-i*Math.sin(n)}}},{key:"_findBorderPositionCircle",value:function(t,e,i){for(var o=i.x,n=i.y,s=i.low,r=i.high,a=i.direction,h=10,d=0,l=this.options.selfReferenceSize,c=void 0,u=void 0,p=void 0,f=void 0,m=void 0,v=.05,g=.5*(s+r);r>=s&&h>d&&(g=.5*(s+r),c=this._pointOnCircle(o,n,l,g),u=Math.atan2(t.y-c.y,t.x-c.x),p=t.distanceToBorder(e,u),f=Math.sqrt(Math.pow(c.x-t.x,2)+Math.pow(c.y-t.y,2)),
-m=p-f,!(Math.abs(m)<v));)m>0?a>0?s=g:r=g:a>0?r=g:s=g,d++;return c.t=g,c}},{key:"getLineWidth",value:function(t,e){return t===!0?Math.max(this.selectionWidth,.3/this.body.view.scale):e===!0?Math.max(this.hoverWidth,.3/this.body.view.scale):Math.max(this.options.width,.3/this.body.view.scale)}},{key:"getColor",value:function(t,e,i){var o=this.options.color;if(o.inherit!==!1){if("both"===o.inherit&&this.from.id!==this.to.id){var n=t.createLinearGradient(this.from.x,this.from.y,this.to.x,this.to.y),s=void 0,a=void 0;return s=this.from.options.color.highlight.border,a=this.to.options.color.highlight.border,this.from.selected===!1&&this.to.selected===!1?(s=r.overrideOpacity(this.from.options.color.border,this.options.color.opacity),a=r.overrideOpacity(this.to.options.color.border,this.options.color.opacity)):this.from.selected===!0&&this.to.selected===!1?a=this.to.options.color.border:this.from.selected===!1&&this.to.selected===!0&&(s=this.from.options.color.border),n.addColorStop(0,s),n.addColorStop(1,a),n}this.colorDirty===!0&&("to"===o.inherit?(this.color.highlight=this.to.options.color.highlight.border,this.color.hover=this.to.options.color.hover.border,this.color.color=r.overrideOpacity(this.to.options.color.border,o.opacity)):(this.color.highlight=this.from.options.color.highlight.border,this.color.hover=this.from.options.color.hover.border,this.color.color=r.overrideOpacity(this.from.options.color.border,o.opacity)))}else this.colorDirty===!0&&(this.color.highlight=o.highlight,this.color.hover=o.hover,this.color.color=r.overrideOpacity(o.color,o.opacity));return this.colorDirty=!1,e===!0?this.color.highlight:i===!0?this.color.hover:this.color.color}},{key:"_circle",value:function(t,e,i,o){this.enableShadow(t),t.beginPath(),t.arc(e,i,o,0,2*Math.PI,!1),t.stroke(),this.disableShadow(t)}},{key:"getDistanceToEdge",value:function(t,e,i,o,s,r,a){var h=0;if(this.from!=this.to)h=this._getDistanceToEdge(t,e,i,o,s,r,a);else{var d=this._getCircleData(),l=n(d,3),c=l[0],u=l[1],p=l[2],f=c-s,m=u-r;h=Math.abs(Math.sqrt(f*f+m*m)-p)}return this.labelModule.size.left<s&&this.labelModule.size.left+this.labelModule.size.width>s&&this.labelModule.size.top<r&&this.labelModule.size.top+this.labelModule.size.height>r?0:h}},{key:"_getDistanceToLine",value:function(t,e,i,o,n,s){var r=i-t,a=o-e,h=r*r+a*a,d=((n-t)*r+(s-e)*a)/h;d>1?d=1:0>d&&(d=0);var l=t+d*r,c=e+d*a,u=l-n,p=c-s;return Math.sqrt(u*u+p*p)}},{key:"getArrowData",value:function(t,e,i,o,s){var r=void 0,a=void 0,h=void 0,d=void 0,l=void 0,c=void 0,u=this.getLineWidth(o,s);if("from"===e?(h=this.from,d=this.to,l=.1,c=this.options.arrows.from.scaleFactor):"to"===e?(h=this.to,d=this.from,l=-.1,c=this.options.arrows.to.scaleFactor):(h=this.to,d=this.from,c=this.options.arrows.middle.scaleFactor),h!=d)if("middle"!==e)if(this.options.smooth.enabled===!0){a=this.findBorderPosition(h,t,{via:i});var p=this.getPoint(Math.max(0,Math.min(1,a.t+l)),i);r=Math.atan2(a.y-p.y,a.x-p.x)}else r=Math.atan2(h.y-d.y,h.x-d.x),a=this.findBorderPosition(h,t);else r=Math.atan2(h.y-d.y,h.x-d.x),a=this.getPoint(.5,i);else{var f=this._getCircleData(t),m=n(f,3),v=m[0],g=m[1],y=m[2];"from"===e?(a=this.findBorderPosition(this.from,t,{x:v,y:g,low:.25,high:.6,direction:-1}),r=-2*a.t*Math.PI+1.5*Math.PI+.1*Math.PI):"to"===e?(a=this.findBorderPosition(this.from,t,{x:v,y:g,low:.6,high:1,direction:1}),r=-2*a.t*Math.PI+1.5*Math.PI-1.1*Math.PI):(a=this._pointOnCircle(v,g,y,.175),r=3.9269908169872414)}var b=15*c+3*u,w=a.x-.9*b*Math.cos(r),_=a.y-.9*b*Math.sin(r),x={x:w,y:_};return{point:a,core:x,angle:r,length:b}}},{key:"drawArrowHead",value:function(t,e,i,o){t.strokeStyle=this.getColor(t,e,i),t.fillStyle=t.strokeStyle,t.lineWidth=this.getLineWidth(e,i),t.arrow(o.point.x,o.point.y,o.angle,o.length),this.enableShadow(t),t.fill(),this.disableShadow(t)}},{key:"enableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor=this.options.shadow.color,t.shadowBlur=this.options.shadow.size,t.shadowOffsetX=this.options.shadow.x,t.shadowOffsetY=this.options.shadow.y)}},{key:"disableShadow",value:function(t){this.options.shadow.enabled===!0&&(t.shadowColor="rgba(0,0,0,0)",t.shadowBlur=0,t.shadowOffsetX=0,t.shadowOffsetY=0)}}]),t}();e["default"]=a},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(88),d=o(h),l=function(t){function e(t,i,o){n(this,e);var r=s(this,Object.getPrototypeOf(e).call(this,t,i,o));return r._boundFunction=function(){r.positionBezierNode()},r.body.emitter.on("_repositionBezierNodes",r._boundFunction),r}return r(e,t),a(e,[{key:"setOptions",value:function(t){var e=!1;this.options.physics!==t.physics&&(e=!0),this.options=t,this.id=this.options.id,this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to],this.setupSupportNode(),this.connect(),e===!0&&(this.via.setOptions({physics:this.options.physics}),this.positionBezierNode())}},{key:"connect",value:function(){this.from=this.body.nodes[this.options.from],this.to=this.body.nodes[this.options.to],void 0===this.from||void 0===this.to||this.options.physics===!1?this.via.setOptions({physics:!1}):this.from.id===this.to.id?this.via.setOptions({physics:!1}):this.via.setOptions({physics:!0})}},{key:"cleanup",value:function(){return this.body.emitter.off("_repositionBezierNodes",this._boundFunction),void 0!==this.via?(delete this.body.nodes[this.via.id],this.via=void 0,!0):!1}},{key:"setupSupportNode",value:function(){if(void 0===this.via){var t="edgeId:"+this.id,e=this.body.functions.createNode({id:t,shape:"circle",physics:!0,hidden:!0});this.body.nodes[t]=e,this.via=e,this.via.parentEdgeId=this.id,this.positionBezierNode()}}},{key:"positionBezierNode",value:function(){void 0!==this.via&&void 0!==this.from&&void 0!==this.to?(this.via.x=.5*(this.from.x+this.to.x),this.via.y=.5*(this.from.y+this.to.y)):void 0!==this.via&&(this.via.x=0,this.via.y=0)}},{key:"_line",value:function(t,e){t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),void 0===e.x?t.lineTo(this.toPoint.x,this.toPoint.y):t.quadraticCurveTo(e.x,e.y,this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"getViaNode",value:function(){return this.via}},{key:"getPoint",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this.via:arguments[1],i=t,o=Math.pow(1-i,2)*this.fromPoint.x+2*i*(1-i)*e.x+Math.pow(i,2)*this.toPoint.x,n=Math.pow(1-i,2)*this.fromPoint.y+2*i*(1-i)*e.y+Math.pow(i,2)*this.toPoint.y;return{x:o,y:n}}},{key:"_findBorderPosition",value:function(t,e){return this._findBorderPositionBezier(t,e,this.via)}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){return this._getDistanceToBezierEdge(t,e,i,o,n,s,this.via)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(88),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_line",value:function(t,e){t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),void 0===e.x?t.lineTo(this.toPoint.x,this.toPoint.y):t.quadraticCurveTo(e.x,e.y,this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"getViaNode",value:function(){return this._getViaCoordinates()}},{key:"_getViaCoordinates",value:function(){var t=void 0,e=void 0,i=this.options.smooth.roundness,o=this.options.smooth.type,n=Math.abs(this.from.x-this.to.x),s=Math.abs(this.from.y-this.to.y);if("discrete"===o||"diagonalCross"===o)Math.abs(this.from.x-this.to.x)<=Math.abs(this.from.y-this.to.y)?(this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y-i*s):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y-i*s):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y+i*s):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y+i*s)),"discrete"===o&&(t=i*s>n?this.from.x:t)):Math.abs(this.from.x-this.to.x)>Math.abs(this.from.y-this.to.y)&&(this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y-i*n):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y-i*n):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y+i*n):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y+i*n)),"discrete"===o&&(e=i*n>s?this.from.y:e));else if("straightCross"===o)Math.abs(this.from.x-this.to.x)<=Math.abs(this.from.y-this.to.y)?(t=this.from.x,e=this.from.y<this.to.y?this.to.y-(1-i)*s:this.to.y+(1-i)*s):Math.abs(this.from.x-this.to.x)>Math.abs(this.from.y-this.to.y)&&(t=this.from.x<this.to.x?this.to.x-(1-i)*n:this.to.x+(1-i)*n,e=this.from.y);else if("horizontal"===o)t=this.from.x<this.to.x?this.to.x-(1-i)*n:this.to.x+(1-i)*n,e=this.from.y;else if("vertical"===o)t=this.from.x,e=this.from.y<this.to.y?this.to.y-(1-i)*s:this.to.y+(1-i)*s;else if("curvedCW"===o){n=this.to.x-this.from.x,s=this.from.y-this.to.y;var r=Math.sqrt(n*n+s*s),a=Math.PI,h=Math.atan2(s,n),d=(h+(.5*i+.5)*a)%(2*a);t=this.from.x+(.5*i+.5)*r*Math.sin(d),e=this.from.y+(.5*i+.5)*r*Math.cos(d)}else if("curvedCCW"===o){n=this.to.x-this.from.x,s=this.from.y-this.to.y;var l=Math.sqrt(n*n+s*s),c=Math.PI,u=Math.atan2(s,n),p=(u+(.5*-i+.5)*c)%(2*c);t=this.from.x+(.5*i+.5)*l*Math.sin(p),e=this.from.y+(.5*i+.5)*l*Math.cos(p)}else Math.abs(this.from.x-this.to.x)<=Math.abs(this.from.y-this.to.y)?this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y-i*s,t=this.to.x<t?this.to.x:t):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y-i*s,t=this.to.x>t?this.to.x:t):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*s,e=this.from.y+i*s,t=this.to.x<t?this.to.x:t):this.from.x>this.to.x&&(t=this.from.x-i*s,e=this.from.y+i*s,t=this.to.x>t?this.to.x:t)):Math.abs(this.from.x-this.to.x)>Math.abs(this.from.y-this.to.y)&&(this.from.y>=this.to.y?this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y-i*n,e=this.to.y>e?this.to.y:e):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y-i*n,e=this.to.y>e?this.to.y:e):this.from.y<this.to.y&&(this.from.x<=this.to.x?(t=this.from.x+i*n,e=this.from.y+i*n,e=this.to.y<e?this.to.y:e):this.from.x>this.to.x&&(t=this.from.x-i*n,e=this.from.y+i*n,e=this.to.y<e?this.to.y:e)));return{x:t,y:e}}},{key:"_findBorderPosition",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?{}:arguments[2];return this._findBorderPositionBezier(t,e,i.via)}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){var r=arguments.length<=6||void 0===arguments[6]?this._getViaCoordinates():arguments[6];return this._getDistanceToBezierEdge(t,e,i,o,n,s,r)}},{key:"getPoint",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this._getViaCoordinates():arguments[1],i=t,o=Math.pow(1-i,2)*this.fromPoint.x+2*i*(1-i)*e.x+Math.pow(i,2)*this.toPoint.x,n=Math.pow(1-i,2)*this.fromPoint.y+2*i*(1-i)*e.y+Math.pow(i,2)*this.toPoint.y;return{x:o,y:n}}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(89),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_line",value:function(t){t.beginPath(),t.moveTo(this.fromPoint.x,this.fromPoint.y),t.lineTo(this.toPoint.x,this.toPoint.y),this.enableShadow(t),t.stroke(),this.disableShadow(t)}},{key:"getViaNode",value:function(){}},{key:"getPoint",value:function(t){return{x:(1-t)*this.fromPoint.x+t*this.toPoint.x,y:(1-t)*this.fromPoint.y+t*this.toPoint.y}}},{key:"_findBorderPosition",value:function(t,e){var i=this.to,o=this.from;t.id===this.from.id&&(i=this.from,o=this.to);var n=Math.atan2(i.y-o.y,i.x-o.x),s=i.x-o.x,r=i.y-o.y,a=Math.sqrt(s*s+r*r),h=t.distanceToBorder(e,n),d=(a-h)/a,l={};return l.x=(1-d)*o.x+d*i.x,l.y=(1-d)*o.y+d*i.y,l}},{key:"_getDistanceToEdge",value:function(t,e,i,o,n,s){return this._getDistanceToLine(t,e,i,o,n,s)}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(94),a=o(r),h=i(95),d=o(h),l=i(96),c=o(l),u=i(97),p=o(u),f=i(98),m=o(f),v=i(99),g=o(v),y=i(100),b=o(y),w=i(101),_=o(w),x=i(1),k=function(){function t(e){n(this,t),this.body=e,this.physicsBody={physicsNodeIndices:[],physicsEdgeIndices:[],forces:{},velocities:{}},this.physicsEnabled=!0,this.simulationInterval=1e3/60,this.requiresTimeout=!0,this.previousStates={},this.referenceState={},this.freezeCache={},this.renderTimer=void 0,this.adaptiveTimestep=!1,this.adaptiveTimestepEnabled=!1,this.adaptiveCounter=0,this.adaptiveInterval=3,this.stabilized=!1,this.startedStabilization=!1,this.stabilizationIterations=0,this.ready=!1,this.options={},this.defaultOptions={enabled:!0,barnesHut:{theta:.5,gravitationalConstant:-2e3,centralGravity:.3,springLength:95,springConstant:.04,damping:.09,avoidOverlap:0},forceAtlas2Based:{theta:.5,gravitationalConstant:-50,centralGravity:.01,springConstant:.08,springLength:100,damping:.4,avoidOverlap:0},repulsion:{centralGravity:.2,springLength:200,springConstant:.05,nodeDistance:100,damping:.09,avoidOverlap:0},hierarchicalRepulsion:{centralGravity:0,springLength:100,springConstant:.01,nodeDistance:120,damping:.09},maxVelocity:50,minVelocity:.75,solver:"barnesHut",stabilization:{enabled:!0,iterations:1e3,updateInterval:50,onlyDynamicEdges:!1,fit:!0},timestep:.5,adaptiveTimestep:!0},x.extend(this.options,this.defaultOptions),this.timestep=.5,this.layoutFailed=!1,this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("initPhysics",function(){t.initPhysics()}),this.body.emitter.on("_layoutFailed",function(){t.layoutFailed=!0}),this.body.emitter.on("resetPhysics",function(){t.stopSimulation(),t.ready=!1}),this.body.emitter.on("disablePhysics",function(){t.physicsEnabled=!1,t.stopSimulation()}),this.body.emitter.on("restorePhysics",function(){t.setOptions(t.options),t.ready===!0&&t.startSimulation()}),this.body.emitter.on("startSimulation",function(){t.ready===!0&&t.startSimulation()}),this.body.emitter.on("stopSimulation",function(){t.stopSimulation()}),this.body.emitter.on("destroy",function(){t.stopSimulation(!1),t.body.emitter.off()}),this.body.emitter.on("_dataChanged",function(){t.updatePhysicsData()})}},{key:"setOptions",value:function(t){void 0!==t&&(t===!1?(this.options.enabled=!1,this.physicsEnabled=!1,this.stopSimulation()):(this.physicsEnabled=!0,x.selectiveNotDeepExtend(["stabilization"],this.options,t),x.mergeOptions(this.options,t,"stabilization"),void 0===t.enabled&&(this.options.enabled=!0),this.options.enabled===!1&&(this.physicsEnabled=!1,this.stopSimulation()),this.timestep=this.options.timestep)),this.init()}},{key:"init",value:function(){var t;"forceAtlas2Based"===this.options.solver?(t=this.options.forceAtlas2Based,this.nodesSolver=new b["default"](this.body,this.physicsBody,t),this.edgesSolver=new p["default"](this.body,this.physicsBody,t),this.gravitySolver=new _["default"](this.body,this.physicsBody,t)):"repulsion"===this.options.solver?(t=this.options.repulsion,this.nodesSolver=new d["default"](this.body,this.physicsBody,t),this.edgesSolver=new p["default"](this.body,this.physicsBody,t),this.gravitySolver=new g["default"](this.body,this.physicsBody,t)):"hierarchicalRepulsion"===this.options.solver?(t=this.options.hierarchicalRepulsion,this.nodesSolver=new c["default"](this.body,this.physicsBody,t),this.edgesSolver=new m["default"](this.body,this.physicsBody,t),this.gravitySolver=new g["default"](this.body,this.physicsBody,t)):(t=this.options.barnesHut,this.nodesSolver=new a["default"](this.body,this.physicsBody,t),this.edgesSolver=new p["default"](this.body,this.physicsBody,t),this.gravitySolver=new g["default"](this.body,this.physicsBody,t)),this.modelOptions=t}},{key:"initPhysics",value:function(){this.physicsEnabled===!0&&this.options.enabled===!0?this.options.stabilization.enabled===!0?this.stabilize():(this.stabilized=!1,this.ready=!0,this.body.emitter.emit("fit",{},this.layoutFailed),this.startSimulation()):(this.ready=!0,this.body.emitter.emit("fit"))}},{key:"startSimulation",value:function(){this.physicsEnabled===!0&&this.options.enabled===!0?(this.stabilized=!1,this.adaptiveTimestep=!1,this.body.emitter.emit("_resizeNodes"),void 0===this.viewFunction&&(this.viewFunction=this.simulationStep.bind(this),this.body.emitter.on("initRedraw",this.viewFunction),this.body.emitter.emit("_startRendering"))):this.body.emitter.emit("_redraw")}},{key:"stopSimulation",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!0:arguments[0];this.stabilized=!0,t===!0&&this._emitStabilized(),void 0!==this.viewFunction&&(this.body.emitter.off("initRedraw",this.viewFunction),this.viewFunction=void 0,t===!0&&this.body.emitter.emit("_stopRendering"))}},{key:"simulationStep",value:function(){var t=Date.now();this.physicsTick();var e=Date.now()-t;(e<.4*this.simulationInterval||this.runDoubleSpeed===!0)&&this.stabilized===!1&&(this.physicsTick(),this.runDoubleSpeed=!0),this.stabilized===!0&&this.stopSimulation()}},{key:"_emitStabilized",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?this.stabilizationIterations:arguments[0];(this.stabilizationIterations>1||this.startedStabilization===!0)&&setTimeout(function(){t.body.emitter.emit("stabilized",{iterations:e}),t.startedStabilization=!1,t.stabilizationIterations=0},0)}},{key:"physicsTick",value:function(){if(this.startedStabilization===!1&&(this.body.emitter.emit("startStabilizing"),this.startedStabilization=!0),this.stabilized===!1){if(this.adaptiveTimestep===!0&&this.adaptiveTimestepEnabled===!0){var t=1.2;this.adaptiveCounter%this.adaptiveInterval===0?(this.timestep=2*this.timestep,this.calculateForces(),this.moveNodes(),this.revert(),this.timestep=.5*this.timestep,this.calculateForces(),this.moveNodes(),this.calculateForces(),this.moveNodes(),this._evaluateStepQuality()===!0?this.timestep=t*this.timestep:this.timestep/t<this.options.timestep?this.timestep=this.options.timestep:(this.adaptiveCounter=-1,this.timestep=Math.max(this.options.timestep,this.timestep/t))):(this.calculateForces(),this.moveNodes()),this.adaptiveCounter+=1}else this.timestep=this.options.timestep,this.calculateForces(),this.moveNodes();this.stabilized===!0&&this.revert(),this.stabilizationIterations++}}},{key:"updatePhysicsData",value:function(){this.physicsBody.forces={},this.physicsBody.physicsNodeIndices=[],this.physicsBody.physicsEdgeIndices=[];var t=this.body.nodes,e=this.body.edges;for(var i in t)t.hasOwnProperty(i)&&t[i].options.physics===!0&&this.physicsBody.physicsNodeIndices.push(t[i].id);for(var o in e)e.hasOwnProperty(o)&&e[o].options.physics===!0&&this.physicsBody.physicsEdgeIndices.push(e[o].id);for(var n=0;n<this.physicsBody.physicsNodeIndices.length;n++){var s=this.physicsBody.physicsNodeIndices[n];this.physicsBody.forces[s]={x:0,y:0},void 0===this.physicsBody.velocities[s]&&(this.physicsBody.velocities[s]={x:0,y:0})}for(var r in this.physicsBody.velocities)void 0===t[r]&&delete this.physicsBody.velocities[r]}},{key:"revert",value:function(){var t=Object.keys(this.previousStates),e=this.body.nodes,i=this.physicsBody.velocities;this.referenceState={};for(var o=0;o<t.length;o++){var n=t[o];void 0!==e[n]?e[n].options.physics===!0&&(this.referenceState[n]={positions:{x:e[n].x,y:e[n].y}},i[n].x=this.previousStates[n].vx,i[n].y=this.previousStates[n].vy,e[n].x=this.previousStates[n].x,e[n].y=this.previousStates[n].y):delete this.previousStates[n]}}},{key:"_evaluateStepQuality",value:function(){var t=void 0,e=void 0,i=void 0,o=this.body.nodes,n=this.referenceState,s=.3;for(var r in this.referenceState)if(this.referenceState.hasOwnProperty(r)&&void 0!==o[r]&&(t=o[r].x-n[r].positions.x,e=o[r].y-n[r].positions.y,i=Math.sqrt(Math.pow(t,2)+Math.pow(e,2)),i>s))return!1;return!0}},{key:"moveNodes",value:function(){for(var t=this.physicsBody.physicsNodeIndices,e=this.options.maxVelocity?this.options.maxVelocity:1e9,i=0,o=0,n=5,s=0;s<t.length;s++){var r=t[s],a=this._performStep(r,e);i=Math.max(i,a),o+=a}this.adaptiveTimestepEnabled=o/t.length<n,this.stabilized=i<this.options.minVelocity}},{key:"_performStep",value:function(t,e){var i=this.body.nodes[t],o=this.timestep,n=this.physicsBody.forces,s=this.physicsBody.velocities;if(this.previousStates[t]={x:i.x,y:i.y,vx:s[t].x,vy:s[t].y},i.options.fixed.x===!1){var r=this.modelOptions.damping*s[t].x,a=(n[t].x-r)/i.options.mass;s[t].x+=a*o,s[t].x=Math.abs(s[t].x)>e?s[t].x>0?e:-e:s[t].x,i.x+=s[t].x*o}else n[t].x=0,s[t].x=0;if(i.options.fixed.y===!1){var h=this.modelOptions.damping*s[t].y,d=(n[t].y-h)/i.options.mass;s[t].y+=d*o,s[t].y=Math.abs(s[t].y)>e?s[t].y>0?e:-e:s[t].y,i.y+=s[t].y*o}else n[t].y=0,s[t].y=0;var l=Math.sqrt(Math.pow(s[t].x,2)+Math.pow(s[t].y,2));return l}},{key:"calculateForces",value:function(){this.gravitySolver.solve(),this.nodesSolver.solve(),this.edgesSolver.solve()}},{key:"_freezeNodes",value:function(){var t=this.body.nodes;for(var e in t)t.hasOwnProperty(e)&&t[e].x&&t[e].y&&(this.freezeCache[e]={x:t[e].options.fixed.x,y:t[e].options.fixed.y},t[e].options.fixed.x=!0,t[e].options.fixed.y=!0)}},{key:"_restoreFrozenNodes",value:function(){var t=this.body.nodes;for(var e in t)t.hasOwnProperty(e)&&void 0!==this.freezeCache[e]&&(t[e].options.fixed.x=this.freezeCache[e].x,t[e].options.fixed.y=this.freezeCache[e].y);this.freezeCache={}}},{key:"stabilize",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?this.options.stabilization.iterations:arguments[0];return"number"!=typeof e&&(console.log("The stabilize method needs a numeric amount of iterations. Switching to default: ",this.options.stabilization.iterations),e=this.options.stabilization.iterations),0===this.physicsBody.physicsNodeIndices.length?void(this.ready=!0):(this.adaptiveTimestep=this.options.adaptiveTimestep,this.body.emitter.emit("_resizeNodes"),this.stopSimulation(),this.stabilized=!1,this.body.emitter.emit("_blockRedraw"),this.targetIterations=e,this.options.stabilization.onlyDynamicEdges===!0&&this._freezeNodes(),this.stabilizationIterations=0,void setTimeout(function(){return t._stabilizationBatch()},0))}},{key:"_stabilizationBatch",value:function(){this.startedStabilization===!1&&(this.body.emitter.emit("startStabilizing"),this.startedStabilization=!0);for(var t=0;this.stabilized===!1&&t<this.options.stabilization.updateInterval&&this.stabilizationIterations<this.targetIterations;)this.physicsTick(),t++;this.stabilized===!1&&this.stabilizationIterations<this.targetIterations?(this.body.emitter.emit("stabilizationProgress",{iterations:this.stabilizationIterations,total:this.targetIterations}),setTimeout(this._stabilizationBatch.bind(this),0)):this._finalizeStabilization()}},{key:"_finalizeStabilization",value:function(){this.body.emitter.emit("_allowRedraw"),this.options.stabilization.fit===!0&&this.body.emitter.emit("fit"),this.options.stabilization.onlyDynamicEdges===!0&&this._restoreFrozenNodes(),this.body.emitter.emit("stabilizationIterationsDone"),this.body.emitter.emit("_requestRedraw"),this.stabilized===!0?this._emitStabilized():this.startSimulation(),this.ready=!0}},{key:"_drawForces",value:function(t){for(var e=0;e<this.physicsBody.physicsNodeIndices.length;e++){var i=this.body.nodes[this.physicsBody.physicsNodeIndices[e]],o=this.physicsBody.forces[this.physicsBody.physicsNodeIndices[e]],n=20,s=.03,r=Math.sqrt(Math.pow(o.x,2)+Math.pow(o.x,2)),a=Math.min(Math.max(5,r),15),h=3*a,d=x.HSVToHex((180-180*Math.min(1,Math.max(0,s*r)))/360,1,1);t.lineWidth=a,t.strokeStyle=d,t.beginPath(),t.moveTo(i.x,i.y),t.lineTo(i.x+n*o.x,i.y+n*o.y),t.stroke();var l=Math.atan2(o.y,o.x);t.fillStyle=d,t.arrow(i.x+n*o.x+Math.cos(l)*h,i.y+n*o.y+Math.sin(l)*h,l,h),t.fill()}}}]),t}();e["default"]=k},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.barnesHutTree,this.setOptions(n),this.randomSeed=5}return o(t,[{key:"setOptions",value:function(t){this.options=t,this.thetaInversed=1/this.options.theta,this.overlapAvoidanceFactor=1-Math.max(0,Math.min(1,this.options.avoidOverlap))}},{key:"seededRandom",value:function(){var t=1e4*Math.sin(this.randomSeed++);return t-Math.floor(t)}},{key:"solve",value:function(){if(0!==this.options.gravitationalConstant&&this.physicsBody.physicsNodeIndices.length>0){var t=void 0,e=this.body.nodes,i=this.physicsBody.physicsNodeIndices,o=i.length,n=this._formBarnesHutTree(e,i);this.barnesHutTree=n;for(var s=0;o>s;s++)t=e[i[s]],t.options.mass>0&&(this._getForceContribution(n.root.children.NW,t),this._getForceContribution(n.root.children.NE,t),this._getForceContribution(n.root.children.SW,t),this._getForceContribution(n.root.children.SE,t))}}},{key:"_getForceContribution",value:function(t,e){if(t.childrenCount>0){var i=void 0,o=void 0,n=void 0;i=t.centerOfMass.x-e.x,o=t.centerOfMass.y-e.y,n=Math.sqrt(i*i+o*o),n*t.calcSize>this.thetaInversed?this._calculateForces(n,i,o,e,t):4===t.childrenCount?(this._getForceContribution(t.children.NW,e),this._getForceContribution(t.children.NE,e),this._getForceContribution(t.children.SW,e),this._getForceContribution(t.children.SE,e)):t.children.data.id!=e.id&&this._calculateForces(n,i,o,e,t)}}},{key:"_calculateForces",value:function(t,e,i,o,n){0===t&&(t=.1,e=t),this.overlapAvoidanceFactor<1&&(t=Math.max(.1+this.overlapAvoidanceFactor*o.shape.radius,t-o.shape.radius));var s=this.options.gravitationalConstant*n.mass*o.options.mass/Math.pow(t,3),r=e*s,a=i*s;this.physicsBody.forces[o.id].x+=r,this.physicsBody.forces[o.id].y+=a}},{key:"_formBarnesHutTree",value:function(t,e){for(var i=void 0,o=e.length,n=t[e[0]].x,s=t[e[0]].y,r=t[e[0]].x,a=t[e[0]].y,h=1;o>h;h++){var d=t[e[h]].x,l=t[e[h]].y;t[e[h]].options.mass>0&&(n>d&&(n=d),d>r&&(r=d),s>l&&(s=l),l>a&&(a=l))}var c=Math.abs(r-n)-Math.abs(a-s);c>0?(s-=.5*c,a+=.5*c):(n+=.5*c,r-=.5*c);var u=1e-5,p=Math.max(u,Math.abs(r-n)),f=.5*p,m=.5*(n+r),v=.5*(s+a),g={root:{centerOfMass:{x:0,y:0},mass:0,range:{minX:m-f,maxX:m+f,minY:v-f,maxY:v+f},size:p,calcSize:1/p,children:{data:null},maxWidth:0,level:0,childrenCount:4}};this._splitBranch(g.root);for(var y=0;o>y;y++)i=t[e[y]],i.options.mass>0&&this._placeInTree(g.root,i);return g}},{key:"_updateBranchMass",value:function(t,e){var i=t.mass+e.options.mass,o=1/i;t.centerOfMass.x=t.centerOfMass.x*t.mass+e.x*e.options.mass,t.centerOfMass.x*=o,t.centerOfMass.y=t.centerOfMass.y*t.mass+e.y*e.options.mass,t.centerOfMass.y*=o,t.mass=i;var n=Math.max(Math.max(e.height,e.radius),e.width);t.maxWidth=t.maxWidth<n?n:t.maxWidth}},{key:"_placeInTree",value:function(t,e,i){1==i&&void 0!==i||this._updateBranchMass(t,e),t.children.NW.range.maxX>e.x?t.children.NW.range.maxY>e.y?this._placeInRegion(t,e,"NW"):this._placeInRegion(t,e,"SW"):t.children.NW.range.maxY>e.y?this._placeInRegion(t,e,"NE"):this._placeInRegion(t,e,"SE")}},{key:"_placeInRegion",value:function(t,e,i){switch(t.children[i].childrenCount){case 0:t.children[i].children.data=e,t.children[i].childrenCount=1,this._updateBranchMass(t.children[i],e);break;case 1:t.children[i].children.data.x===e.x&&t.children[i].children.data.y===e.y?(e.x+=this.seededRandom(),e.y+=this.seededRandom()):(this._splitBranch(t.children[i]),this._placeInTree(t.children[i],e));break;case 4:this._placeInTree(t.children[i],e)}}},{key:"_splitBranch",value:function(t){var e=null;1===t.childrenCount&&(e=t.children.data,t.mass=0,t.centerOfMass.x=0,t.centerOfMass.y=0),t.childrenCount=4,t.children.data=null,this._insertRegion(t,"NW"),this._insertRegion(t,"NE"),this._insertRegion(t,"SW"),this._insertRegion(t,"SE"),null!=e&&this._placeInTree(t,e)}},{key:"_insertRegion",value:function(t,e){var i=void 0,o=void 0,n=void 0,s=void 0,r=.5*t.size;switch(e){case"NW":i=t.range.minX,o=t.range.minX+r,n=t.range.minY,s=t.range.minY+r;break;case"NE":i=t.range.minX+r,o=t.range.maxX,n=t.range.minY,s=t.range.minY+r;break;case"SW":i=t.range.minX,o=t.range.minX+r,n=t.range.minY+r,s=t.range.maxY;break;case"SE":i=t.range.minX+r,o=t.range.maxX,n=t.range.minY+r,s=t.range.maxY}t.children[e]={centerOfMass:{x:0,y:0},mass:0,range:{minX:i,maxX:o,minY:n,maxY:s},size:.5*t.size,calcSize:2*t.calcSize,children:{data:null},maxWidth:0,level:t.level+1,childrenCount:0}}},{key:"_debug",value:function(t,e){void 0!==this.barnesHutTree&&(t.lineWidth=1,this._drawBranch(this.barnesHutTree.root,t,e))}},{key:"_drawBranch",value:function(t,e,i){void 0===i&&(i="#FF0000"),4===t.childrenCount&&(this._drawBranch(t.children.NW,e),this._drawBranch(t.children.NE,e),this._drawBranch(t.children.SE,e),this._drawBranch(t.children.SW,e)),e.strokeStyle=i,e.beginPath(),e.moveTo(t.range.minX,t.range.minY),e.lineTo(t.range.maxX,t.range.minY),e.stroke(),e.beginPath(),e.moveTo(t.range.maxX,t.range.minY),e.lineTo(t.range.maxX,t.range.maxY),e.stroke(),e.beginPath(),e.moveTo(t.range.maxX,t.range.maxY),
-e.lineTo(t.range.minX,t.range.maxY),e.stroke(),e.beginPath(),e.moveTo(t.range.minX,t.range.maxY),e.lineTo(t.range.minX,t.range.minY),e.stroke()}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t,e,i,o,n,s,r,a,h=this.body.nodes,d=this.physicsBody.physicsNodeIndices,l=this.physicsBody.forces,c=this.options.nodeDistance,u=-2/3/c,p=4/3,f=0;f<d.length-1;f++){r=h[d[f]];for(var m=f+1;m<d.length;m++)a=h[d[m]],t=a.x-r.x,e=a.y-r.y,i=Math.sqrt(t*t+e*e),0===i&&(i=.1*Math.random(),t=i),2*c>i&&(s=.5*c>i?1:u*i+p,s/=i,o=t*s,n=e*s,l[r.id].x-=o,l[r.id].y-=n,l[a.id].x+=o,l[a.id].y+=n)}}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){var t,e,i,o,n,s,r,a,h,d,l=this.body.nodes,c=this.physicsBody.physicsNodeIndices,u=this.physicsBody.forces,p=this.options.nodeDistance;for(h=0;h<c.length-1;h++)for(r=l[c[h]],d=h+1;d<c.length;d++)if(a=l[c[d]],r.level===a.level){t=a.x-r.x,e=a.y-r.y,i=Math.sqrt(t*t+e*e);var f=.05;s=p>i?-Math.pow(f*i,2)+Math.pow(f*p,2):0,0===i?i=.01:s/=i,o=t*s,n=e*s,u[r.id].x-=o,u[r.id].y-=n,u[a.id].x+=o,u[a.id].y+=n}}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t=void 0,e=void 0,i=this.physicsBody.physicsEdgeIndices,o=this.body.edges,n=void 0,s=void 0,r=void 0,a=0;a<i.length;a++)e=o[i[a]],e.connected===!0&&e.toId!==e.fromId&&void 0!==this.body.nodes[e.toId]&&void 0!==this.body.nodes[e.fromId]&&(void 0!==e.edgeType.via?(t=void 0===e.options.length?this.options.springLength:e.options.length,n=e.to,s=e.edgeType.via,r=e.from,this._calculateSpringForce(n,s,.5*t),this._calculateSpringForce(s,r,.5*t)):(t=void 0===e.options.length?1.5*this.options.springLength:e.options.length,this._calculateSpringForce(e.from,e.to,t)))}},{key:"_calculateSpringForce",value:function(t,e,i){var o=t.x-e.x,n=t.y-e.y,s=Math.max(Math.sqrt(o*o+n*n),.01),r=this.options.springConstant*(i-s)/s,a=o*r,h=n*r;void 0!==this.physicsBody.forces[t.id]&&(this.physicsBody.forces[t.id].x+=a,this.physicsBody.forces[t.id].y+=h),void 0!==this.physicsBody.forces[e.id]&&(this.physicsBody.forces[e.id].x-=a,this.physicsBody.forces[e.id].y-=h)}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t,e,i,o,n,s,r,a,h=this.body.edges,d=.5,l=this.physicsBody.physicsEdgeIndices,c=this.physicsBody.physicsNodeIndices,u=this.physicsBody.forces,p=0;p<c.length;p++){var f=c[p];u[f].springFx=0,u[f].springFy=0}for(var m=0;m<l.length;m++)e=h[l[m]],e.connected===!0&&(t=void 0===e.options.length?this.options.springLength:e.options.length,i=e.from.x-e.to.x,o=e.from.y-e.to.y,a=Math.sqrt(i*i+o*o),a=0===a?.01:a,r=this.options.springConstant*(t-a)/a,n=i*r,s=o*r,e.to.level!=e.from.level?(void 0!==u[e.toId]&&(u[e.toId].springFx-=n,u[e.toId].springFy-=s),void 0!==u[e.fromId]&&(u[e.fromId].springFx+=n,u[e.fromId].springFy+=s)):(void 0!==u[e.toId]&&(u[e.toId].x-=d*n,u[e.toId].y-=d*s),void 0!==u[e.fromId]&&(u[e.fromId].x+=d*n,u[e.fromId].y+=d*s)));for(var v,g,r=1,y=0;y<c.length;y++){var b=c[y];v=Math.min(r,Math.max(-r,u[b].springFx)),g=Math.min(r,Math.max(-r,u[b].springFy)),u[b].x+=v,u[b].y+=g}for(var w=0,_=0,x=0;x<c.length;x++){var k=c[x];w+=u[k].x,_+=u[k].y}for(var O=w/c.length,M=_/c.length,D=0;D<c.length;D++){var S=c[D];u[S].x-=O,u[S].y-=M}}}]),t}();e["default"]=n},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e,o,n){i(this,t),this.body=e,this.physicsBody=o,this.setOptions(n)}return o(t,[{key:"setOptions",value:function(t){this.options=t}},{key:"solve",value:function(){for(var t=void 0,e=void 0,i=void 0,o=void 0,n=this.body.nodes,s=this.physicsBody.physicsNodeIndices,r=this.physicsBody.forces,a=0;a<s.length;a++){var h=s[a];o=n[h],t=-o.x,e=-o.y,i=Math.sqrt(t*t+e*e),this._calculateForces(i,t,e,r,o)}}},{key:"_calculateForces",value:function(t,e,i,o,n){var s=0===t?0:this.options.centralGravity/t;o[n.id].x=e*s,o[n.id].y=i*s}}]),t}();e["default"]=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(94),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_calculateForces",value:function(t,e,i,o,n){0===t&&(t=.1*Math.random(),e=t),this.overlapAvoidanceFactor<1&&(t=Math.max(.1+this.overlapAvoidanceFactor*o.shape.radius,t-o.shape.radius));var s=o.edges.length+1,r=this.options.gravitationalConstant*n.mass*o.options.mass*s/Math.pow(t,2),a=e*r,h=i*r;this.physicsBody.forces[o.id].x+=a,this.physicsBody.forces[o.id].y+=h}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(99),d=o(h),l=function(t){function e(t,i,o){return n(this,e),s(this,Object.getPrototypeOf(e).call(this,t,i,o))}return r(e,t),a(e,[{key:"_calculateForces",value:function(t,e,i,o,n){if(t>0){var s=n.edges.length+1,r=this.options.centralGravity*s*n.options.mass;o[n.id].x=e*r,o[n.id].y=i*r}}}]),e}(d["default"]);e["default"]=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(103),h=o(a),d=i(104),l=o(d),c=i(1),u=function(){function t(e){var i=this;n(this,t),this.body=e,this.clusteredNodes={},this.clusteredEdges={},this.options={},this.defaultOptions={},c.extend(this.options,this.defaultOptions),this.body.emitter.on("_resetData",function(){i.clusteredNodes={},i.clusteredEdges={}})}return r(t,[{key:"setOptions",value:function(t){}},{key:"clusterByHubsize",value:function(t,e){void 0===t?t=this._getHubSize():"object"===("undefined"==typeof t?"undefined":s(t))&&(e=this._checkOptions(t),t=this._getHubSize());for(var i=[],o=0;o<this.body.nodeIndices.length;o++){var n=this.body.nodes[this.body.nodeIndices[o]];n.edges.length>=t&&i.push(n.id)}for(var r=0;r<i.length;r++)this.clusterByConnection(i[r],e,!0);this.body.emitter.emit("_dataChanged")}},{key:"cluster",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{}:arguments[0],e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];if(void 0===t.joinCondition)throw new Error("Cannot call clusterByNodeData without a joinCondition function in the options.");t=this._checkOptions(t);for(var i={},o={},n=0;n<this.body.nodeIndices.length;n++){var s=this.body.nodeIndices[n],r=this.body.nodes[s],a=h["default"].cloneOptions(r);if(t.joinCondition(a)===!0){i[s]=this.body.nodes[s];for(var d=0;d<r.edges.length;d++){var l=r.edges[d];void 0===this.clusteredEdges[l.id]&&(o[l.id]=l)}}}this._cluster(i,o,t,e)}},{key:"clusterByEdgeCount",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!0:arguments[2];e=this._checkOptions(e);for(var o=[],n={},s=void 0,r=void 0,a=void 0,d=void 0,l=void 0,c=0;c<this.body.nodeIndices.length;c++){var u={},p={};if(d=this.body.nodeIndices[c],void 0===n[d]){l=0,a=this.body.nodes[d],r=[];for(var f=0;f<a.edges.length;f++)s=a.edges[f],void 0===this.clusteredEdges[s.id]&&(s.toId!==s.fromId&&l++,r.push(s));if(l===t){for(var m=!0,v=0;v<r.length;v++){s=r[v];var g=this._getConnectedId(s,d);if(void 0===e.joinCondition)p[s.id]=s,u[d]=this.body.nodes[d],u[g]=this.body.nodes[g],n[d]=!0;else{var y=h["default"].cloneOptions(this.body.nodes[d]);if(e.joinCondition(y)!==!0){m=!1;break}p[s.id]=s,u[d]=this.body.nodes[d],n[d]=!0}}Object.keys(u).length>0&&Object.keys(p).length>0&&m===!0&&o.push({nodes:u,edges:p})}}}for(var b=0;b<o.length;b++)this._cluster(o[b].nodes,o[b].edges,e,!1);i===!0&&this.body.emitter.emit("_dataChanged")}},{key:"clusterOutliers",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];this.clusterByEdgeCount(1,t,e)}},{key:"clusterBridges",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];this.clusterByEdgeCount(2,t,e)}},{key:"clusterByConnection",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!0:arguments[2];if(void 0===t)throw new Error("No nodeId supplied to clusterByConnection!");if(void 0===this.body.nodes[t])throw new Error("The nodeId given to clusterByConnection does not exist!");var o=this.body.nodes[t];e=this._checkOptions(e,o),void 0===e.clusterNodeProperties.x&&(e.clusterNodeProperties.x=o.x),void 0===e.clusterNodeProperties.y&&(e.clusterNodeProperties.y=o.y),void 0===e.clusterNodeProperties.fixed&&(e.clusterNodeProperties.fixed={},e.clusterNodeProperties.fixed.x=o.options.fixed.x,e.clusterNodeProperties.fixed.y=o.options.fixed.y);var n={},s={},r=o.id,a=h["default"].cloneOptions(o);n[r]=o;for(var d=0;d<o.edges.length;d++){var l=o.edges[d];if(void 0===this.clusteredEdges[l.id]){var c=this._getConnectedId(l,r);if(void 0===this.clusteredNodes[c])if(c!==r)if(void 0===e.joinCondition)s[l.id]=l,n[c]=this.body.nodes[c];else{var u=h["default"].cloneOptions(this.body.nodes[c]);e.joinCondition(a,u)===!0&&(s[l.id]=l,n[c]=this.body.nodes[c])}else s[l.id]=l}}this._cluster(n,s,e,i)}},{key:"_createClusterEdges",value:function(t,e,i,o){for(var n=void 0,s=void 0,r=void 0,a=void 0,d=void 0,l=void 0,u=Object.keys(t),p=[],f=0;f<u.length;f++){s=u[f],r=t[s];for(var m=0;m<r.edges.length;m++)n=r.edges[m],void 0===this.clusteredEdges[n.id]&&(n.toId==n.fromId?e[n.id]=n:n.toId==s?(a=i.id,d=n.fromId,l=d):(a=n.toId,d=i.id,l=a),void 0===t[l]&&p.push({edge:n,fromId:d,toId:a}))}for(var v=0;v<p.length;v++){var g=p[v].edge,y=h["default"].cloneOptions(g,"edge");c.deepExtend(y,o),y.from=p[v].fromId,y.to=p[v].toId,y.id="clusterEdge:"+c.randomUUID();var b=this.body.functions.createEdge(y);b.clusteringEdgeReplacingId=g.id,this.body.edges[b.id]=b,b.connect(),this._backupEdgeOptions(g),g.setOptions({physics:!1,hidden:!0})}}},{key:"_checkOptions",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{}:arguments[0];return void 0===t.clusterEdgeProperties&&(t.clusterEdgeProperties={}),void 0===t.clusterNodeProperties&&(t.clusterNodeProperties={}),t}},{key:"_cluster",value:function(t,e,i){var o=arguments.length<=3||void 0===arguments[3]?!0:arguments[3];if(!(Object.keys(t).length<2)){for(var n in t)if(t.hasOwnProperty(n)&&void 0!==this.clusteredNodes[n])return;var s=c.deepExtend({},i.clusterNodeProperties);if(void 0!==i.processProperties){var r=[];for(var a in t)if(t.hasOwnProperty(a)){var d=h["default"].cloneOptions(t[a]);r.push(d)}var u=[];for(var p in e)if(e.hasOwnProperty(p)&&"clusterEdge:"!==p.substr(0,12)){var f=h["default"].cloneOptions(e[p],"edge");u.push(f)}if(s=i.processProperties(s,r,u),!s)throw new Error("The processProperties function does not return properties!")}void 0===s.id&&(s.id="cluster:"+c.randomUUID());var m=s.id;void 0===s.label&&(s.label="cluster");var v=void 0;void 0===s.x&&(v=this._getClusterPosition(t),s.x=v.x),void 0===s.y&&(void 0===v&&(v=this._getClusterPosition(t)),s.y=v.y),s.id=m;var g=this.body.functions.createNode(s,l["default"]);g.isCluster=!0,g.containedNodes=t,g.containedEdges=e,g.clusterEdgeProperties=i.clusterEdgeProperties,this.body.nodes[s.id]=g,this._createClusterEdges(t,e,s,i.clusterEdgeProperties);for(var y in e)if(e.hasOwnProperty(y)&&void 0!==this.body.edges[y]){var b=this.body.edges[y];this._backupEdgeOptions(b),b.setOptions({physics:!1,hidden:!0})}for(var w in t)t.hasOwnProperty(w)&&(this.clusteredNodes[w]={clusterId:s.id,node:this.body.nodes[w]},this.body.nodes[w].setOptions({hidden:!0,physics:!1}));s.id=void 0,o===!0&&this.body.emitter.emit("_dataChanged")}}},{key:"_backupEdgeOptions",value:function(t){void 0===this.clusteredEdges[t.id]&&(this.clusteredEdges[t.id]={physics:t.options.physics,hidden:t.options.hidden})}},{key:"_restoreEdge",value:function(t){var e=this.clusteredEdges[t.id];void 0!==e&&(t.setOptions({physics:e.physics,hidden:e.hidden}),delete this.clusteredEdges[t.id])}},{key:"isCluster",value:function(t){return void 0!==this.body.nodes[t]?this.body.nodes[t].isCluster===!0:(console.log("Node does not exist."),!1)}},{key:"_getClusterPosition",value:function(t){for(var e=Object.keys(t),i=t[e[0]].x,o=t[e[0]].x,n=t[e[0]].y,s=t[e[0]].y,r=void 0,a=1;a<e.length;a++)r=t[e[a]],i=r.x<i?r.x:i,o=r.x>o?r.x:o,n=r.y<n?r.y:n,s=r.y>s?r.y:s;return{x:.5*(i+o),y:.5*(n+s)}}},{key:"openCluster",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!0:arguments[2];if(void 0===t)throw new Error("No clusterNodeId supplied to openCluster.");if(void 0===this.body.nodes[t])throw new Error("The clusterNodeId supplied to openCluster does not exist.");if(void 0===this.body.nodes[t].containedNodes)return void console.log("The node:"+t+" is not a cluster.");var o=this.body.nodes[t],n=o.containedNodes,s=o.containedEdges;if(void 0!==e&&void 0!==e.releaseFunction&&"function"==typeof e.releaseFunction){var r={},a={x:o.x,y:o.y};for(var d in n)if(n.hasOwnProperty(d)){var l=this.body.nodes[d];r[d]={x:l.x,y:l.y}}var u=e.releaseFunction(a,r);for(var p in n)if(n.hasOwnProperty(p)){var f=this.body.nodes[p];void 0!==u[p]&&(f.x=void 0===u[p].x?o.x:u[p].x,f.y=void 0===u[p].y?o.y:u[p].y)}}else for(var m in n)if(n.hasOwnProperty(m)){var v=this.body.nodes[m];v=n[m],v.options.fixed.x===!1&&(v.x=o.x),v.options.fixed.y===!1&&(v.y=o.y)}for(var g in n)if(n.hasOwnProperty(g)){var y=this.body.nodes[g];y.vx=o.vx,y.vy=o.vy,y.setOptions({hidden:!1,physics:!0}),delete this.clusteredNodes[g]}for(var b=[],w=0;w<o.edges.length;w++)b.push(o.edges[w]);for(var _=0;_<b.length;_++){var x=b[_],k=this._getConnectedId(x,t);if(void 0!==this.clusteredNodes[k]){var O=this.body.nodes[this.clusteredNodes[k].clusterId],M=this.body.edges[x.clusteringEdgeReplacingId];if(void 0!==M){O.containedEdges[M.id]=M,delete s[M.id];var D=M.fromId,S=M.toId;M.toId==k?S=this.clusteredNodes[k].clusterId:D=this.clusteredNodes[k].clusterId;var C=h["default"].cloneOptions(M,"edge");c.deepExtend(C,O.clusterEdgeProperties);var T="clusterEdge:"+c.randomUUID();c.deepExtend(C,{from:D,to:S,hidden:!1,physics:!0,id:T});var E=this.body.functions.createEdge(C);E.clusteringEdgeReplacingId=M.id,this.body.edges[T]=E,this.body.edges[T].connect()}}else{var P=this.body.edges[x.clusteringEdgeReplacingId];void 0!==P&&this._restoreEdge(P)}x.cleanup(),x.disconnect(),delete this.body.edges[x.id]}for(var I in s)s.hasOwnProperty(I)&&this._restoreEdge(s[I]);delete this.body.nodes[t],i===!0&&this.body.emitter.emit("_dataChanged")}},{key:"getNodesInCluster",value:function(t){var e=[];if(this.isCluster(t)===!0){var i=this.body.nodes[t].containedNodes;for(var o in i)i.hasOwnProperty(o)&&e.push(this.body.nodes[o].id)}return e}},{key:"findNode",value:function(t){for(var e=[],i=100,o=0;void 0!==this.clusteredNodes[t]&&i>o;)e.push(this.body.nodes[t].id),t=this.clusteredNodes[t].clusterId,o++;return e.push(this.body.nodes[t].id),e.reverse(),e}},{key:"_getConnectedId",value:function(t,e){return t.toId!=e?t.toId:t.fromId!=e?t.fromId:t.fromId}},{key:"_getHubSize",value:function(){for(var t=0,e=0,i=0,o=0,n=0;n<this.body.nodeIndices.length;n++){var s=this.body.nodes[this.body.nodeIndices[n]];s.edges.length>o&&(o=s.edges.length),t+=s.edges.length,e+=Math.pow(s.edges.length,2),i+=1}t/=i,e/=i;var r=e-Math.pow(t,2),a=Math.sqrt(r),h=Math.floor(t+2*a);return h>o&&(h=o),h}}]),t}();e["default"]=u},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(1),r=function(){function t(){o(this,t)}return n(t,null,[{key:"getRange",value:function(t){var e,i=arguments.length<=1||void 0===arguments[1]?[]:arguments[1],o=1e9,n=-1e9,s=1e9,r=-1e9;if(i.length>0)for(var a=0;a<i.length;a++)e=t[i[a]],s>e.shape.boundingBox.left&&(s=e.shape.boundingBox.left),r<e.shape.boundingBox.right&&(r=e.shape.boundingBox.right),o>e.shape.boundingBox.top&&(o=e.shape.boundingBox.top),n<e.shape.boundingBox.bottom&&(n=e.shape.boundingBox.bottom);return 1e9===s&&-1e9===r&&1e9===o&&-1e9===n&&(o=0,n=0,s=0,r=0),{minX:s,maxX:r,minY:o,maxY:n}}},{key:"getRangeCore",value:function(t){var e,i=arguments.length<=1||void 0===arguments[1]?[]:arguments[1],o=1e9,n=-1e9,s=1e9,r=-1e9;if(i.length>0)for(var a=0;a<i.length;a++)e=t[i[a]],s>e.x&&(s=e.x),r<e.x&&(r=e.x),o>e.y&&(o=e.y),n<e.y&&(n=e.y);return 1e9===s&&-1e9===r&&1e9===o&&-1e9===n&&(o=0,n=0,s=0,r=0),{minX:s,maxX:r,minY:o,maxY:n}}},{key:"findCenter",value:function(t){return{x:.5*(t.maxX+t.minX),y:.5*(t.maxY+t.minY)}}},{key:"cloneOptions",value:function(t,e){var i={};return void 0===e||"node"===e?(s.deepExtend(i,t.options,!0),i.x=t.x,i.y=t.y,i.amountOfConnections=t.edges.length):s.deepExtend(i,t.options,!0),i}}]),t}();e["default"]=r},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t,e){if(!t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!e||"object"!=typeof e&&"function"!=typeof e?t:e}function r(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function, not "+typeof e);t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,enumerable:!1,writable:!0,configurable:!0}}),e&&(Object.setPrototypeOf?Object.setPrototypeOf(t,e):t.__proto__=e)}Object.defineProperty(e,"__esModule",{value:!0});var a=i(65),h=o(a),d=function(t){function e(t,i,o,r,a){n(this,e);var h=s(this,Object.getPrototypeOf(e).call(this,t,i,o,r,a));return h.isCluster=!0,h.containedNodes={},h.containedEdges={},h}return r(e,t),e}(h["default"]);e["default"]=d},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}();"undefined"!=typeof window&&(window.requestAnimationFrame=window.requestAnimationFrame||window.mozRequestAnimationFrame||window.webkitRequestAnimationFrame||window.msRequestAnimationFrame);var s=i(1),r=function(){function t(e,i){o(this,t),this.body=e,this.canvas=i,this.redrawRequested=!1,this.renderTimer=void 0,this.requiresTimeout=!0,this.renderingActive=!1,this.renderRequests=0,this.pixelRatio=void 0,this.allowRedraw=!0,this.dragging=!1,this.options={},this.defaultOptions={hideEdgesOnDrag:!1,hideNodesOnDrag:!1},s.extend(this.options,this.defaultOptions),this._determineBrowserMethod(),this.bindEventListeners()}return n(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("dragStart",function(){t.dragging=!0}),this.body.emitter.on("dragEnd",function(){return t.dragging=!1}),this.body.emitter.on("_resizeNodes",function(){return t._resizeNodes()}),this.body.emitter.on("_redraw",function(){t.renderingActive===!1&&t._redraw()}),this.body.emitter.on("_blockRedraw",function(){t.allowRedraw=!1}),this.body.emitter.on("_allowRedraw",function(){t.allowRedraw=!0,t.redrawRequested=!1}),this.body.emitter.on("_requestRedraw",this._requestRedraw.bind(this)),this.body.emitter.on("_startRendering",function(){t.renderRequests+=1,t.renderingActive=!0,t._startRendering()}),this.body.emitter.on("_stopRendering",function(){t.renderRequests-=1,t.renderingActive=t.renderRequests>0,t.renderTimer=void 0}),this.body.emitter.on("destroy",function(){t.renderRequests=0,t.allowRedraw=!1,t.renderingActive=!1,t.requiresTimeout===!0?clearTimeout(t.renderTimer):cancelAnimationFrame(t.renderTimer),t.body.emitter.off()})}},{key:"setOptions",value:function(t){if(void 0!==t){var e=["hideEdgesOnDrag","hideNodesOnDrag"];s.selectiveDeepExtend(e,this.options,t)}}},{key:"_startRendering",value:function(){this.renderingActive===!0&&void 0===this.renderTimer&&(this.requiresTimeout===!0?this.renderTimer=window.setTimeout(this._renderStep.bind(this),this.simulationInterval):this.renderTimer=window.requestAnimationFrame(this._renderStep.bind(this)))}},{key:"_renderStep",value:function(){this.renderingActive===!0&&(this.renderTimer=void 0,this.requiresTimeout===!0&&this._startRendering(),this._redraw(),this.requiresTimeout===!1&&this._startRendering())}},{key:"redraw",value:function(){this.body.emitter.emit("setSize"),this._redraw()}},{key:"_requestRedraw",value:function(){var t=this;this.redrawRequested!==!0&&this.renderingActive===!1&&this.allowRedraw===!0&&(this.redrawRequested=!0,this.requiresTimeout===!0?window.setTimeout(function(){t._redraw(!1)},0):window.requestAnimationFrame(function(){t._redraw(!1)}))}},{key:"_redraw",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!1:arguments[0];if(this.allowRedraw===!0){this.body.emitter.emit("initRedraw"),this.redrawRequested=!1;var e=this.canvas.frame.canvas.getContext("2d");0!==this.canvas.frame.canvas.width&&0!==this.canvas.frame.canvas.height||this.canvas.setSize(),this.pixelRatio=(window.devicePixelRatio||1)/(e.webkitBackingStorePixelRatio||e.mozBackingStorePixelRatio||e.msBackingStorePixelRatio||e.oBackingStorePixelRatio||e.backingStorePixelRatio||1),e.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var i=this.canvas.frame.canvas.clientWidth,o=this.canvas.frame.canvas.clientHeight;if(e.clearRect(0,0,i,o),0===this.canvas.frame.clientWidth)return;e.save(),e.translate(this.body.view.translation.x,this.body.view.translation.y),e.scale(this.body.view.scale,this.body.view.scale),e.beginPath(),this.body.emitter.emit("beforeDrawing",e),e.closePath(),t===!1&&(this.dragging===!1||this.dragging===!0&&this.options.hideEdgesOnDrag===!1)&&this._drawEdges(e),(this.dragging===!1||this.dragging===!0&&this.options.hideNodesOnDrag===!1)&&this._drawNodes(e,t),e.beginPath(),this.body.emitter.emit("afterDrawing",e),e.closePath(),e.restore(),t===!0&&e.clearRect(0,0,i,o)}}},{key:"_resizeNodes",value:function(){var t=this.canvas.frame.canvas.getContext("2d");void 0===this.pixelRatio&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0),t.save(),t.translate(this.body.view.translation.x,this.body.view.translation.y),t.scale(this.body.view.scale,this.body.view.scale);var e=this.body.nodes,i=void 0;for(var o in e)e.hasOwnProperty(o)&&(i=e[o],i.resize(t),i.updateBoundingBox(t,i.selected));t.restore()}},{key:"_drawNodes",value:function(t){for(var e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=this.body.nodes,o=this.body.nodeIndices,n=void 0,s=[],r=20,a=this.canvas.DOMtoCanvas({x:-r,y:-r}),h=this.canvas.DOMtoCanvas({x:this.canvas.frame.canvas.clientWidth+r,y:this.canvas.frame.canvas.clientHeight+r}),d={top:a.y,left:a.x,bottom:h.y,right:h.x},l=0;l<o.length;l++)n=i[o[l]],n.isSelected()?s.push(o[l]):e===!0?n.draw(t):n.isBoundingBoxOverlappingWith(d)===!0?n.draw(t):n.updateBoundingBox(t,n.selected);for(var c=0;c<s.length;c++)n=i[s[c]],n.draw(t)}},{key:"_drawEdges",value:function(t){for(var e=this.body.edges,i=this.body.edgeIndices,o=void 0,n=0;n<i.length;n++)o=e[i[n]],o.connected===!0&&o.draw(t)}},{key:"_determineBrowserMethod",value:function(){if("undefined"!=typeof window){var t=navigator.userAgent.toLowerCase();this.requiresTimeout=!1,-1!=t.indexOf("msie 9.0")?this.requiresTimeout=!0:-1!=t.indexOf("safari")&&t.indexOf("chrome")<=-1&&(this.requiresTimeout=!0)}else this.requiresTimeout=!0}}]),t}();e["default"]=r},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(20),r=i(28),a=i(1),h=function(){function t(e){o(this,t),this.body=e,this.pixelRatio=1,this.resizeTimer=void 0,this.resizeFunction=this._onResize.bind(this),this.cameraState={},this.initialized=!1,this.options={},this.defaultOptions={autoResize:!0,height:"100%",width:"100%"},a.extend(this.options,this.defaultOptions),this.bindEventListeners()}return n(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.once("resize",function(e){0!==e.width&&(t.body.view.translation.x=.5*e.width),0!==e.height&&(t.body.view.translation.y=.5*e.height)}),this.body.emitter.on("setSize",this.setSize.bind(this)),this.body.emitter.on("destroy",function(){t.hammerFrame.destroy(),t.hammer.destroy(),t._cleanUp()})}},{key:"setOptions",value:function(t){var e=this;if(void 0!==t){var i=["width","height","autoResize"];a.selectiveDeepExtend(i,this.options,t)}this.options.autoResize===!0&&(this._cleanUp(),this.resizeTimer=setInterval(function(){var t=e.setSize();t===!0&&e.body.emitter.emit("_requestRedraw")},1e3),this.resizeFunction=this._onResize.bind(this),a.addEventListener(window,"resize",this.resizeFunction))}},{key:"_cleanUp",value:function(){void 0!==this.resizeTimer&&clearInterval(this.resizeTimer),a.removeEventListener(window,"resize",this.resizeFunction),this.resizeFunction=void 0}},{key:"_onResize",value:function(){this.setSize(),this.body.emitter.emit("_redraw")}},{key:"_getCameraState",value:function(){var t=arguments.length<=0||void 0===arguments[0]?this.pixelRatio:arguments[0];this.initialized===!0&&(this.cameraState.previousWidth=this.frame.canvas.width/t,this.cameraState.previousHeight=this.frame.canvas.height/t,this.cameraState.scale=this.body.view.scale,this.cameraState.position=this.DOMtoCanvas({x:.5*this.frame.canvas.width/t,y:.5*this.frame.canvas.height/t}))}},{key:"_setCameraState",value:function(){if(void 0!==this.cameraState.scale&&0!==this.frame.canvas.clientWidth&&0!==this.frame.canvas.clientHeight&&0!==this.pixelRatio&&this.cameraState.previousWidth>0){var t=this.frame.canvas.width/this.pixelRatio/this.cameraState.previousWidth,e=this.frame.canvas.height/this.pixelRatio/this.cameraState.previousHeight,i=this.cameraState.scale;1!=t&&1!=e?i=.5*this.cameraState.scale*(t+e):1!=t?i=this.cameraState.scale*t:1!=e&&(i=this.cameraState.scale*e),this.body.view.scale=i;var o=this.DOMtoCanvas({x:.5*this.frame.canvas.clientWidth,y:.5*this.frame.canvas.clientHeight}),n={x:o.x-this.cameraState.position.x,y:o.y-this.cameraState.position.y};this.body.view.translation.x+=n.x*this.body.view.scale,this.body.view.translation.y+=n.y*this.body.view.scale}}},{key:"_prepareValue",value:function(t){if("number"==typeof t)return t+"px";if("string"==typeof t){if(-1!==t.indexOf("%")||-1!==t.indexOf("px"))return t;if(-1===t.indexOf("%"))return t+"px"}throw new Error("Could not use the value supplied for width or height:"+t)}},{key:"_create",value:function(){for(;this.body.container.hasChildNodes();)this.body.container.removeChild(this.body.container.firstChild);if(this.frame=document.createElement("div"),this.frame.className="vis-network",this.frame.style.position="relative",this.frame.style.overflow="hidden",this.frame.tabIndex=900,this.frame.canvas=document.createElement("canvas"),this.frame.canvas.style.position="relative",this.frame.appendChild(this.frame.canvas),this.frame.canvas.getContext){var t=this.frame.canvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),
-this.frame.canvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.frame.canvas.appendChild(e)}this.body.container.appendChild(this.frame),this.body.view.scale=1,this.body.view.translation={x:.5*this.frame.canvas.clientWidth,y:.5*this.frame.canvas.clientHeight},this._bindHammer()}},{key:"_bindHammer",value:function(){var t=this;void 0!==this.hammer&&this.hammer.destroy(),this.drag={},this.pinch={},this.hammer=new s(this.frame.canvas),this.hammer.get("pinch").set({enable:!0}),this.hammer.get("pan").set({threshold:5,direction:s.DIRECTION_ALL}),r.onTouch(this.hammer,function(e){t.body.eventListeners.onTouch(e)}),this.hammer.on("tap",function(e){t.body.eventListeners.onTap(e)}),this.hammer.on("doubletap",function(e){t.body.eventListeners.onDoubleTap(e)}),this.hammer.on("press",function(e){t.body.eventListeners.onHold(e)}),this.hammer.on("panstart",function(e){t.body.eventListeners.onDragStart(e)}),this.hammer.on("panmove",function(e){t.body.eventListeners.onDrag(e)}),this.hammer.on("panend",function(e){t.body.eventListeners.onDragEnd(e)}),this.hammer.on("pinch",function(e){t.body.eventListeners.onPinch(e)}),this.frame.canvas.addEventListener("mousewheel",function(e){t.body.eventListeners.onMouseWheel(e)}),this.frame.canvas.addEventListener("DOMMouseScroll",function(e){t.body.eventListeners.onMouseWheel(e)}),this.frame.canvas.addEventListener("mousemove",function(e){t.body.eventListeners.onMouseMove(e)}),this.frame.canvas.addEventListener("contextmenu",function(e){t.body.eventListeners.onContext(e)}),this.hammerFrame=new s(this.frame),r.onRelease(this.hammerFrame,function(e){t.body.eventListeners.onRelease(e)})}},{key:"setSize",value:function(){var t=arguments.length<=0||void 0===arguments[0]?this.options.width:arguments[0],e=arguments.length<=1||void 0===arguments[1]?this.options.height:arguments[1];t=this._prepareValue(t),e=this._prepareValue(e);var i=!1,o=this.frame.canvas.width,n=this.frame.canvas.height,s=this.frame.canvas.getContext("2d"),r=this.pixelRatio;return this.pixelRatio=(window.devicePixelRatio||1)/(s.webkitBackingStorePixelRatio||s.mozBackingStorePixelRatio||s.msBackingStorePixelRatio||s.oBackingStorePixelRatio||s.backingStorePixelRatio||1),t!=this.options.width||e!=this.options.height||this.frame.style.width!=t||this.frame.style.height!=e?(this._getCameraState(r),this.frame.style.width=t,this.frame.style.height=e,this.frame.canvas.style.width="100%",this.frame.canvas.style.height="100%",this.frame.canvas.width=Math.round(this.frame.canvas.clientWidth*this.pixelRatio),this.frame.canvas.height=Math.round(this.frame.canvas.clientHeight*this.pixelRatio),this.options.width=t,this.options.height=e,i=!0):(this.frame.canvas.width==Math.round(this.frame.canvas.clientWidth*this.pixelRatio)&&this.frame.canvas.height==Math.round(this.frame.canvas.clientHeight*this.pixelRatio)||this._getCameraState(r),this.frame.canvas.width!=Math.round(this.frame.canvas.clientWidth*this.pixelRatio)&&(this.frame.canvas.width=Math.round(this.frame.canvas.clientWidth*this.pixelRatio),i=!0),this.frame.canvas.height!=Math.round(this.frame.canvas.clientHeight*this.pixelRatio)&&(this.frame.canvas.height=Math.round(this.frame.canvas.clientHeight*this.pixelRatio),i=!0)),i===!0&&(this.body.emitter.emit("resize",{width:Math.round(this.frame.canvas.width/this.pixelRatio),height:Math.round(this.frame.canvas.height/this.pixelRatio),oldWidth:Math.round(o/this.pixelRatio),oldHeight:Math.round(n/this.pixelRatio)}),this._setCameraState()),this.initialized=!0,i}},{key:"_XconvertDOMtoCanvas",value:function(t){return(t-this.body.view.translation.x)/this.body.view.scale}},{key:"_XconvertCanvasToDOM",value:function(t){return t*this.body.view.scale+this.body.view.translation.x}},{key:"_YconvertDOMtoCanvas",value:function(t){return(t-this.body.view.translation.y)/this.body.view.scale}},{key:"_YconvertCanvasToDOM",value:function(t){return t*this.body.view.scale+this.body.view.translation.y}},{key:"canvasToDOM",value:function(t){return{x:this._XconvertCanvasToDOM(t.x),y:this._YconvertCanvasToDOM(t.y)}}},{key:"DOMtoCanvas",value:function(t){return{x:this._XconvertDOMtoCanvas(t.x),y:this._YconvertDOMtoCanvas(t.y)}}}]),t}();e["default"]=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(103),a=o(r),h=i(1),d=function(){function t(e,i){var o=this;n(this,t),this.body=e,this.canvas=i,this.animationSpeed=1/this.renderRefreshRate,this.animationEasingFunction="easeInOutQuint",this.easingTime=0,this.sourceScale=0,this.targetScale=0,this.sourceTranslation=0,this.targetTranslation=0,this.lockedOnNodeId=void 0,this.lockedOnNodeOffset=void 0,this.touchTime=0,this.viewFunction=void 0,this.body.emitter.on("fit",this.fit.bind(this)),this.body.emitter.on("animationFinished",function(){o.body.emitter.emit("_stopRendering")}),this.body.emitter.on("unlockNode",this.releaseNode.bind(this))}return s(t,[{key:"setOptions",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{}:arguments[0];this.options=t}},{key:"fit",value:function(){var t=arguments.length<=0||void 0===arguments[0]?{nodes:[]}:arguments[0],e=arguments.length<=1||void 0===arguments[1]?!1:arguments[1],i=void 0,o=void 0;if(void 0!==t.nodes&&0!==t.nodes.length||(t.nodes=this.body.nodeIndices),e===!0){var n=0;for(var s in this.body.nodes)if(this.body.nodes.hasOwnProperty(s)){var r=this.body.nodes[s];r.predefinedPosition===!0&&(n+=1)}if(n>.5*this.body.nodeIndices.length)return void this.fit(t,!1);i=a["default"].getRange(this.body.nodes,t.nodes);var h=this.body.nodeIndices.length;o=12.662/(h+7.4147)+.0964822;var d=Math.min(this.canvas.frame.canvas.clientWidth/600,this.canvas.frame.canvas.clientHeight/600);o*=d}else{this.body.emitter.emit("_resizeNodes"),i=a["default"].getRange(this.body.nodes,t.nodes);var l=1.1*Math.abs(i.maxX-i.minX),c=1.1*Math.abs(i.maxY-i.minY),u=this.canvas.frame.canvas.clientWidth/l,p=this.canvas.frame.canvas.clientHeight/c;o=p>=u?u:p}o>1?o=1:0===o&&(o=1);var f=a["default"].findCenter(i),m={position:f,scale:o,animation:t.animation};this.moveTo(m)}},{key:"focus",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?{}:arguments[1];if(void 0!==this.body.nodes[t]){var i={x:this.body.nodes[t].x,y:this.body.nodes[t].y};e.position=i,e.lockedOnNode=t,this.moveTo(e)}else console.log("Node: "+t+" cannot be found.")}},{key:"moveTo",value:function(t){return void 0===t?void(t={}):(void 0===t.offset&&(t.offset={x:0,y:0}),void 0===t.offset.x&&(t.offset.x=0),void 0===t.offset.y&&(t.offset.y=0),void 0===t.scale&&(t.scale=this.body.view.scale),void 0===t.position&&(t.position=this.getViewPosition()),void 0===t.animation&&(t.animation={duration:0}),t.animation===!1&&(t.animation={duration:0}),t.animation===!0&&(t.animation={}),void 0===t.animation.duration&&(t.animation.duration=1e3),void 0===t.animation.easingFunction&&(t.animation.easingFunction="easeInOutQuad"),void this.animateView(t))}},{key:"animateView",value:function(t){if(void 0!==t){this.animationEasingFunction=t.animation.easingFunction,this.releaseNode(),t.locked===!0&&(this.lockedOnNodeId=t.lockedOnNode,this.lockedOnNodeOffset=t.offset),0!=this.easingTime&&this._transitionRedraw(!0),this.sourceScale=this.body.view.scale,this.sourceTranslation=this.body.view.translation,this.targetScale=t.scale,this.body.view.scale=this.targetScale;var e=this.canvas.DOMtoCanvas({x:.5*this.canvas.frame.canvas.clientWidth,y:.5*this.canvas.frame.canvas.clientHeight}),i={x:e.x-t.position.x,y:e.y-t.position.y};this.targetTranslation={x:this.sourceTranslation.x+i.x*this.targetScale+t.offset.x,y:this.sourceTranslation.y+i.y*this.targetScale+t.offset.y},0===t.animation.duration?void 0!=this.lockedOnNodeId?(this.viewFunction=this._lockedRedraw.bind(this),this.body.emitter.on("initRedraw",this.viewFunction)):(this.body.view.scale=this.targetScale,this.body.view.translation=this.targetTranslation,this.body.emitter.emit("_requestRedraw")):(this.animationSpeed=1/(60*t.animation.duration*.001)||1/60,this.animationEasingFunction=t.animation.easingFunction,this.viewFunction=this._transitionRedraw.bind(this),this.body.emitter.on("initRedraw",this.viewFunction),this.body.emitter.emit("_startRendering"))}}},{key:"_lockedRedraw",value:function(){var t={x:this.body.nodes[this.lockedOnNodeId].x,y:this.body.nodes[this.lockedOnNodeId].y},e=this.canvas.DOMtoCanvas({x:.5*this.canvas.frame.canvas.clientWidth,y:.5*this.canvas.frame.canvas.clientHeight}),i={x:e.x-t.x,y:e.y-t.y},o=this.body.view.translation,n={x:o.x+i.x*this.body.view.scale+this.lockedOnNodeOffset.x,y:o.y+i.y*this.body.view.scale+this.lockedOnNodeOffset.y};this.body.view.translation=n}},{key:"releaseNode",value:function(){void 0!==this.lockedOnNodeId&&void 0!==this.viewFunction&&(this.body.emitter.off("initRedraw",this.viewFunction),this.lockedOnNodeId=void 0,this.lockedOnNodeOffset=void 0)}},{key:"_transitionRedraw",value:function(){var t=arguments.length<=0||void 0===arguments[0]?!1:arguments[0];this.easingTime+=this.animationSpeed,this.easingTime=t===!0?1:this.easingTime;var e=h.easingFunctions[this.animationEasingFunction](this.easingTime);this.body.view.scale=this.sourceScale+(this.targetScale-this.sourceScale)*e,this.body.view.translation={x:this.sourceTranslation.x+(this.targetTranslation.x-this.sourceTranslation.x)*e,y:this.sourceTranslation.y+(this.targetTranslation.y-this.sourceTranslation.y)*e},this.easingTime>=1&&(this.body.emitter.off("initRedraw",this.viewFunction),this.easingTime=0,void 0!=this.lockedOnNodeId&&(this.viewFunction=this._lockedRedraw.bind(this),this.body.emitter.on("initRedraw",this.viewFunction)),this.body.emitter.emit("animationFinished"))}},{key:"getScale",value:function(){return this.body.view.scale}},{key:"getViewPosition",value:function(){return this.canvas.DOMtoCanvas({x:.5*this.canvas.frame.canvas.clientWidth,y:.5*this.canvas.frame.canvas.clientHeight})}}]),t}();e["default"]=d},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(109),a=o(r),h=i(110),d=o(h),l=i(1),c=function(){function t(e,i,o){n(this,t),this.body=e,this.canvas=i,this.selectionHandler=o,this.navigationHandler=new a["default"](e,i),this.body.eventListeners.onTap=this.onTap.bind(this),this.body.eventListeners.onTouch=this.onTouch.bind(this),this.body.eventListeners.onDoubleTap=this.onDoubleTap.bind(this),this.body.eventListeners.onHold=this.onHold.bind(this),this.body.eventListeners.onDragStart=this.onDragStart.bind(this),this.body.eventListeners.onDrag=this.onDrag.bind(this),this.body.eventListeners.onDragEnd=this.onDragEnd.bind(this),this.body.eventListeners.onMouseWheel=this.onMouseWheel.bind(this),this.body.eventListeners.onPinch=this.onPinch.bind(this),this.body.eventListeners.onMouseMove=this.onMouseMove.bind(this),this.body.eventListeners.onRelease=this.onRelease.bind(this),this.body.eventListeners.onContext=this.onContext.bind(this),this.touchTime=0,this.drag={},this.pinch={},this.popup=void 0,this.popupObj=void 0,this.popupTimer=void 0,this.body.functions.getPointer=this.getPointer.bind(this),this.options={},this.defaultOptions={dragNodes:!0,dragView:!0,hover:!1,keyboard:{enabled:!1,speed:{x:10,y:10,zoom:.02},bindToWindow:!0},navigationButtons:!1,tooltipDelay:300,zoomView:!0},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return s(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("destroy",function(){clearTimeout(t.popupTimer),delete t.body.functions.getPointer})}},{key:"setOptions",value:function(t){if(void 0!==t){var e=["hideEdgesOnDrag","hideNodesOnDrag","keyboard","multiselect","selectable","selectConnectedEdges"];l.selectiveNotDeepExtend(e,this.options,t),l.mergeOptions(this.options,t,"keyboard"),t.tooltip&&(l.extend(this.options.tooltip,t.tooltip),t.tooltip.color&&(this.options.tooltip.color=l.parseColor(t.tooltip.color)))}this.navigationHandler.setOptions(this.options)}},{key:"getPointer",value:function(t){return{x:t.x-l.getAbsoluteLeft(this.canvas.frame.canvas),y:t.y-l.getAbsoluteTop(this.canvas.frame.canvas)}}},{key:"onTouch",value:function(t){(new Date).valueOf()-this.touchTime>50&&(this.drag.pointer=this.getPointer(t.center),this.drag.pinched=!1,this.pinch.scale=this.body.view.scale,this.touchTime=(new Date).valueOf())}},{key:"onTap",value:function(t){var e=this.getPointer(t.center),i=this.selectionHandler.options.multiselect&&(t.changedPointers[0].ctrlKey||t.changedPointers[0].metaKey);this.checkSelectionChanges(e,t,i),this.selectionHandler._generateClickEvent("click",t,e)}},{key:"onDoubleTap",value:function(t){var e=this.getPointer(t.center);this.selectionHandler._generateClickEvent("doubleClick",t,e)}},{key:"onHold",value:function(t){var e=this.getPointer(t.center),i=this.selectionHandler.options.multiselect;this.checkSelectionChanges(e,t,i),this.selectionHandler._generateClickEvent("click",t,e),this.selectionHandler._generateClickEvent("hold",t,e)}},{key:"onRelease",value:function(t){if((new Date).valueOf()-this.touchTime>10){var e=this.getPointer(t.center);this.selectionHandler._generateClickEvent("release",t,e),this.touchTime=(new Date).valueOf()}}},{key:"onContext",value:function(t){var e=this.getPointer({x:t.clientX,y:t.clientY});this.selectionHandler._generateClickEvent("oncontext",t,e)}},{key:"checkSelectionChanges",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=this.selectionHandler._getSelectedEdgeCount(),n=this.selectionHandler._getSelectedNodeCount(),s=this.selectionHandler.getSelection(),r=void 0;r=i===!0?this.selectionHandler.selectAdditionalOnPoint(t):this.selectionHandler.selectOnPoint(t);var a=this.selectionHandler._getSelectedEdgeCount(),h=this.selectionHandler._getSelectedNodeCount(),d=this.selectionHandler.getSelection(),l=this._determineIfDifferent(s,d),c=l.nodesChanged,u=l.edgesChanged,p=!1;h-n>0?(this.selectionHandler._generateClickEvent("selectNode",e,t),r=!0,p=!0):c===!0&&h>0?(this.selectionHandler._generateClickEvent("deselectNode",e,t,s),this.selectionHandler._generateClickEvent("selectNode",e,t),p=!0,r=!0):0>h-n&&(this.selectionHandler._generateClickEvent("deselectNode",e,t,s),r=!0),a-o>0&&p===!1?(this.selectionHandler._generateClickEvent("selectEdge",e,t),r=!0):a>0&&u===!0?(this.selectionHandler._generateClickEvent("deselectEdge",e,t,s),this.selectionHandler._generateClickEvent("selectEdge",e,t),r=!0):0>a-o&&(this.selectionHandler._generateClickEvent("deselectEdge",e,t,s),r=!0),r===!0&&this.selectionHandler._generateClickEvent("select",e,t)}},{key:"_determineIfDifferent",value:function(t,e){for(var i=!1,o=!1,n=0;n<t.nodes.length;n++)-1===e.nodes.indexOf(t.nodes[n])&&(i=!0);for(var s=0;s<e.nodes.length;s++)-1===t.nodes.indexOf(t.nodes[s])&&(i=!0);for(var r=0;r<t.edges.length;r++)-1===e.edges.indexOf(t.edges[r])&&(o=!0);for(var a=0;a<e.edges.length;a++)-1===t.edges.indexOf(t.edges[a])&&(o=!0);return{nodesChanged:i,edgesChanged:o}}},{key:"onDragStart",value:function(t){void 0===this.drag.pointer&&this.onTouch(t);var e=this.selectionHandler.getNodeAt(this.drag.pointer);if(this.drag.dragging=!0,this.drag.selection=[],this.drag.translation=l.extend({},this.body.view.translation),this.drag.nodeId=void 0,void 0!==e&&this.options.dragNodes===!0){this.drag.nodeId=e.id,e.isSelected()===!1&&(this.selectionHandler.unselectAll(),this.selectionHandler.selectObject(e)),this.selectionHandler._generateClickEvent("dragStart",t,this.drag.pointer);var i=this.selectionHandler.selectionObj.nodes;for(var o in i)if(i.hasOwnProperty(o)){var n=i[o],s={id:n.id,node:n,x:n.x,y:n.y,xFixed:n.options.fixed.x,yFixed:n.options.fixed.y};n.options.fixed.x=!0,n.options.fixed.y=!0,this.drag.selection.push(s)}}else this.selectionHandler._generateClickEvent("dragStart",t,this.drag.pointer,void 0,!0)}},{key:"onDrag",value:function(t){var e=this;if(this.drag.pinched!==!0){this.body.emitter.emit("unlockNode");var i=this.getPointer(t.center),o=this.drag.selection;if(o&&o.length&&this.options.dragNodes===!0)!function(){e.selectionHandler._generateClickEvent("dragging",t,i);var n=i.x-e.drag.pointer.x,s=i.y-e.drag.pointer.y;o.forEach(function(t){var i=t.node;t.xFixed===!1&&(i.x=e.canvas._XconvertDOMtoCanvas(e.canvas._XconvertCanvasToDOM(t.x)+n)),t.yFixed===!1&&(i.y=e.canvas._YconvertDOMtoCanvas(e.canvas._YconvertCanvasToDOM(t.y)+s))}),e.body.emitter.emit("startSimulation")}();else if(this.options.dragView===!0){if(this.selectionHandler._generateClickEvent("dragging",t,i,void 0,!0),void 0===this.drag.pointer)return void this.onDragStart(t);var n=i.x-this.drag.pointer.x,s=i.y-this.drag.pointer.y;this.body.view.translation={x:this.drag.translation.x+n,y:this.drag.translation.y+s},this.body.emitter.emit("_redraw")}}}},{key:"onDragEnd",value:function(t){this.drag.dragging=!1;var e=this.drag.selection;e&&e.length?(e.forEach(function(t){t.node.options.fixed.x=t.xFixed,t.node.options.fixed.y=t.yFixed}),this.selectionHandler._generateClickEvent("dragEnd",t,this.getPointer(t.center)),this.body.emitter.emit("startSimulation")):(this.selectionHandler._generateClickEvent("dragEnd",t,this.getPointer(t.center),void 0,!0),this.body.emitter.emit("_requestRedraw"))}},{key:"onPinch",value:function(t){var e=this.getPointer(t.center);this.drag.pinched=!0,void 0===this.pinch.scale&&(this.pinch.scale=1);var i=this.pinch.scale*t.scale;this.zoom(i,e)}},{key:"zoom",value:function(t,e){if(this.options.zoomView===!0){var i=this.body.view.scale;1e-5>t&&(t=1e-5),t>10&&(t=10);var o=void 0;void 0!==this.drag&&this.drag.dragging===!0&&(o=this.canvas.DOMtoCanvas(this.drag.pointer));var n=this.body.view.translation,s=t/i,r=(1-s)*e.x+n.x*s,a=(1-s)*e.y+n.y*s;if(this.body.view.scale=t,this.body.view.translation={x:r,y:a},void 0!=o){var h=this.canvas.canvasToDOM(o);this.drag.pointer.x=h.x,this.drag.pointer.y=h.y}this.body.emitter.emit("_requestRedraw"),t>i?this.body.emitter.emit("zoom",{direction:"+",scale:this.body.view.scale}):this.body.emitter.emit("zoom",{direction:"-",scale:this.body.view.scale})}}},{key:"onMouseWheel",value:function(t){if(this.options.zoomView===!0){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),0!==e){var i=this.body.view.scale,o=e/10;0>e&&(o/=1-o),i*=1+o;var n=this.getPointer({x:t.clientX,y:t.clientY});this.zoom(i,n)}t.preventDefault()}}},{key:"onMouseMove",value:function(t){var e=this,i=this.getPointer({x:t.clientX,y:t.clientY}),o=!1;if(void 0!==this.popup&&(this.popup.hidden===!1&&this._checkHidePopup(i),this.popup.hidden===!1&&(o=!0,this.popup.setPosition(i.x+3,i.y-5),this.popup.show())),this.options.keyboard.bindToWindow===!1&&this.options.keyboard.enabled===!0&&this.canvas.frame.focus(),o===!1&&(void 0!==this.popupTimer&&(clearInterval(this.popupTimer),this.popupTimer=void 0),this.drag.dragging||(this.popupTimer=setTimeout(function(){return e._checkShowPopup(i)},this.options.tooltipDelay))),this.options.hover===!0){var n=this.selectionHandler.getNodeAt(i);void 0===n&&(n=this.selectionHandler.getEdgeAt(i)),this.selectionHandler.hoverObject(n)}}},{key:"_checkShowPopup",value:function(t){var e=this.canvas._XconvertDOMtoCanvas(t.x),i=this.canvas._YconvertDOMtoCanvas(t.y),o={left:e,top:i,right:e,bottom:i},n=void 0===this.popupObj?void 0:this.popupObj.id,s=!1,r="node";if(void 0===this.popupObj){for(var a=this.body.nodeIndices,h=this.body.nodes,l=void 0,c=[],u=0;u<a.length;u++)l=h[a[u]],l.isOverlappingWith(o)===!0&&void 0!==l.getTitle()&&c.push(a[u]);c.length>0&&(this.popupObj=h[c[c.length-1]],s=!0)}if(void 0===this.popupObj&&s===!1){for(var p=this.body.edgeIndices,f=this.body.edges,m=void 0,v=[],g=0;g<p.length;g++)m=f[p[g]],m.isOverlappingWith(o)===!0&&m.connected===!0&&void 0!==m.getTitle()&&v.push(p[g]);v.length>0&&(this.popupObj=f[v[v.length-1]],r="edge")}void 0!==this.popupObj?this.popupObj.id!==n&&(void 0===this.popup&&(this.popup=new d["default"](this.canvas.frame)),this.popup.popupTargetType=r,this.popup.popupTargetId=this.popupObj.id,this.popup.setPosition(t.x+3,t.y-5),this.popup.setText(this.popupObj.getTitle()),this.popup.show(),this.body.emitter.emit("showPopup",this.popupObj.id)):void 0!==this.popup&&(this.popup.hide(),this.body.emitter.emit("hidePopup"))}},{key:"_checkHidePopup",value:function(t){var e=this.selectionHandler._pointerToPositionObject(t),i=!1;if("node"===this.popup.popupTargetType){if(void 0!==this.body.nodes[this.popup.popupTargetId]&&(i=this.body.nodes[this.popup.popupTargetId].isOverlappingWith(e),i===!0)){var o=this.selectionHandler.getNodeAt(t);i=o.id===this.popup.popupTargetId}}else void 0===this.selectionHandler.getNodeAt(t)&&void 0!==this.body.edges[this.popup.popupTargetId]&&(i=this.body.edges[this.popup.popupTargetId].isOverlappingWith(e));i===!1&&(this.popupObj=void 0,this.popup.hide(),this.body.emitter.emit("hidePopup"))}}]),t}();e["default"]=c},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=(i(1),i(20)),r=i(28),a=i(23),h=function(){function t(e,i){var n=this;o(this,t),this.body=e,this.canvas=i,this.iconsCreated=!1,this.navigationHammers=[],this.boundFunctions={},this.touchTime=0,this.activated=!1,this.body.emitter.on("activate",function(){n.activated=!0,n.configureKeyboardBindings()}),this.body.emitter.on("deactivate",function(){n.activated=!1,n.configureKeyboardBindings()}),this.body.emitter.on("destroy",function(){void 0!==n.keycharm&&n.keycharm.destroy()}),this.options={}}return n(t,[{key:"setOptions",value:function(t){void 0!==t&&(this.options=t,this.create())}},{key:"create",value:function(){this.options.navigationButtons===!0?this.iconsCreated===!1&&this.loadNavigationElements():this.iconsCreated===!0&&this.cleanNavigation(),this.configureKeyboardBindings()}},{key:"cleanNavigation",value:function(){if(0!=this.navigationHammers.length){for(var t=0;t<this.navigationHammers.length;t++)this.navigationHammers[t].destroy();this.navigationHammers=[]}this.navigationDOM&&this.navigationDOM.wrapper&&this.navigationDOM.wrapper.parentNode&&this.navigationDOM.wrapper.parentNode.removeChild(this.navigationDOM.wrapper),this.iconsCreated=!1}},{key:"loadNavigationElements",value:function(){var t=this;this.cleanNavigation(),this.navigationDOM={};var e=["up","down","left","right","zoomIn","zoomOut","zoomExtends"],i=["_moveUp","_moveDown","_moveLeft","_moveRight","_zoomIn","_zoomOut","_fit"];this.navigationDOM.wrapper=document.createElement("div"),this.navigationDOM.wrapper.className="vis-navigation",this.canvas.frame.appendChild(this.navigationDOM.wrapper);for(var o=0;o<e.length;o++){this.navigationDOM[e[o]]=document.createElement("div"),this.navigationDOM[e[o]].className="vis-button vis-"+e[o],this.navigationDOM.wrapper.appendChild(this.navigationDOM[e[o]]);var n=new s(this.navigationDOM[e[o]]);"_fit"===i[o]?r.onTouch(n,this._fit.bind(this)):r.onTouch(n,this.bindToRedraw.bind(this,i[o])),this.navigationHammers.push(n)}var a=new s(this.canvas.frame);r.onRelease(a,function(){t._stopMovement()}),this.navigationHammers.push(a),this.iconsCreated=!0}},{key:"bindToRedraw",value:function(t){void 0===this.boundFunctions[t]&&(this.boundFunctions[t]=this[t].bind(this),this.body.emitter.on("initRedraw",this.boundFunctions[t]),this.body.emitter.emit("_startRendering"))}},{key:"unbindFromRedraw",value:function(t){void 0!==this.boundFunctions[t]&&(this.body.emitter.off("initRedraw",this.boundFunctions[t]),this.body.emitter.emit("_stopRendering"),delete this.boundFunctions[t])}},{key:"_fit",value:function(){(new Date).valueOf()-this.touchTime>700&&(this.body.emitter.emit("fit",{duration:700}),this.touchTime=(new Date).valueOf())}},{key:"_stopMovement",value:function(){for(var t in this.boundFunctions)this.boundFunctions.hasOwnProperty(t)&&(this.body.emitter.off("initRedraw",this.boundFunctions[t]),this.body.emitter.emit("_stopRendering"));this.boundFunctions={}}},{key:"_moveUp",value:function(){this.body.view.translation.y+=this.options.keyboard.speed.y}},{key:"_moveDown",value:function(){this.body.view.translation.y-=this.options.keyboard.speed.y}},{key:"_moveLeft",value:function(){this.body.view.translation.x+=this.options.keyboard.speed.x}},{key:"_moveRight",value:function(){this.body.view.translation.x-=this.options.keyboard.speed.x}},{key:"_zoomIn",value:function(){this.body.view.scale*=1+this.options.keyboard.speed.zoom,this.body.emitter.emit("zoom",{direction:"+",scale:this.body.view.scale})}},{key:"_zoomOut",value:function(){this.body.view.scale/=1+this.options.keyboard.speed.zoom,this.body.emitter.emit("zoom",{direction:"-",scale:this.body.view.scale})}},{key:"configureKeyboardBindings",value:function(){var t=this;void 0!==this.keycharm&&this.keycharm.destroy(),this.options.keyboard.enabled===!0&&(this.options.keyboard.bindToWindow===!0?this.keycharm=a({container:window,preventDefault:!0}):this.keycharm=a({container:this.canvas.frame,preventDefault:!0}),this.keycharm.reset(),this.activated===!0&&(this.keycharm.bind("up",function(){t.bindToRedraw("_moveUp")},"keydown"),this.keycharm.bind("down",function(){t.bindToRedraw("_moveDown")},"keydown"),this.keycharm.bind("left",function(){t.bindToRedraw("_moveLeft")},"keydown"),this.keycharm.bind("right",function(){t.bindToRedraw("_moveRight")},"keydown"),this.keycharm.bind("=",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("num+",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("num-",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("-",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("[",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("]",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("pageup",function(){t.bindToRedraw("_zoomIn")},"keydown"),this.keycharm.bind("pagedown",function(){t.bindToRedraw("_zoomOut")},"keydown"),this.keycharm.bind("up",function(){t.unbindFromRedraw("_moveUp")},"keyup"),this.keycharm.bind("down",function(){t.unbindFromRedraw("_moveDown")},"keyup"),this.keycharm.bind("left",function(){t.unbindFromRedraw("_moveLeft")},"keyup"),this.keycharm.bind("right",function(){t.unbindFromRedraw("_moveRight")},"keyup"),this.keycharm.bind("=",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("num+",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("num-",function(){t.unbindFromRedraw("_zoomOut")},"keyup"),this.keycharm.bind("-",function(){t.unbindFromRedraw("_zoomOut")},"keyup"),this.keycharm.bind("[",function(){t.unbindFromRedraw("_zoomOut")},"keyup"),this.keycharm.bind("]",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("pageup",function(){t.unbindFromRedraw("_zoomIn")},"keyup"),this.keycharm.bind("pagedown",function(){t.unbindFromRedraw("_zoomOut")},"keyup")))}}]),t}();e["default"]=h},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(e){i(this,t),this.container=e,this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-network-tooltip",this.container.appendChild(this.frame)}return o(t,[{key:"setPosition",value:function(t,e){this.x=parseInt(t),this.y=parseInt(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),t===!0){var e=this.frame.clientHeight,i=this.frame.clientWidth,o=this.frame.parentNode.clientHeight,n=this.frame.parentNode.clientWidth,s=this.y-e;s+e+this.padding>o&&(s=o-e-this.padding),s<this.padding&&(s=this.padding);var r=this.x;r+i+this.padding>n&&(r=n-i-this.padding),r<this.padding&&(r=this.padding),this.frame.style.left=r+"px",this.frame.style.top=s+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.visibility="hidden"}}]),t}();e["default"]=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),r=i(65),a=o(r),h=i(85),d=o(h),l=i(1),c=function(){function t(e,i){var o=this;n(this,t),this.body=e,this.canvas=i,this.selectionObj={nodes:[],edges:[]},this.hoverObj={nodes:{},edges:{}},this.options={},this.defaultOptions={multiselect:!1,selectable:!0,selectConnectedEdges:!0,hoverConnectedEdges:!0},l.extend(this.options,this.defaultOptions),this.body.emitter.on("_dataChanged",function(){o.updateSelection()})}return s(t,[{key:"setOptions",value:function(t){if(void 0!==t){var e=["multiselect","hoverConnectedEdges","selectable","selectConnectedEdges"];l.selectiveDeepExtend(e,this.options,t)}}},{key:"selectOnPoint",value:function(t){var e=!1;if(this.options.selectable===!0){var i=this.getNodeAt(t)||this.getEdgeAt(t);this.unselectAll(),void 0!==i&&(e=this.selectObject(i)),this.body.emitter.emit("_requestRedraw")}return e}},{key:"selectAdditionalOnPoint",value:function(t){var e=!1;if(this.options.selectable===!0){var i=this.getNodeAt(t)||this.getEdgeAt(t);void 0!==i&&(e=!0,i.isSelected()===!0?this.deselectObject(i):this.selectObject(i),this.body.emitter.emit("_requestRedraw"))}return e}},{key:"_generateClickEvent",value:function(t,e,i,o){var n=arguments.length<=4||void 0===arguments[4]?!1:arguments[4],s=void 0;s=n===!0?{nodes:[],edges:[]}:this.getSelection(),s.pointer={DOM:{x:i.x,y:i.y},canvas:this.canvas.DOMtoCanvas(i)},s.event=e,void 0!==o&&(s.previousSelection=o),this.body.emitter.emit(t,s)}},{key:"selectObject",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?this.options.selectConnectedEdges:arguments[1];return void 0!==t?(t instanceof a["default"]&&e===!0&&this._selectConnectedEdges(t),t.select(),this._addToSelection(t),!0):!1}},{key:"deselectObject",value:function(t){t.isSelected()===!0&&(t.selected=!1,this._removeFromSelection(t))}},{key:"_getAllNodesOverlappingWith",value:function(t){for(var e=[],i=this.body.nodes,o=0;o<this.body.nodeIndices.length;o++){var n=this.body.nodeIndices[o];i[n].isOverlappingWith(t)&&e.push(n)}return e}},{key:"_pointerToPositionObject",value:function(t){var e=this.canvas.DOMtoCanvas(t);return{left:e.x-1,top:e.y+1,right:e.x+1,bottom:e.y-1}}},{key:"getNodeAt",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1],i=this._pointerToPositionObject(t),o=this._getAllNodesOverlappingWith(i);
-return o.length>0?e===!0?this.body.nodes[o[o.length-1]]:o[o.length-1]:void 0}},{key:"_getEdgesOverlappingWith",value:function(t,e){for(var i=this.body.edges,o=0;o<this.body.edgeIndices.length;o++){var n=this.body.edgeIndices[o];i[n].isOverlappingWith(t)&&e.push(n)}}},{key:"_getAllEdgesOverlappingWith",value:function(t){var e=[];return this._getEdgesOverlappingWith(t,e),e}},{key:"getEdgeAt",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1],i=this._pointerToPositionObject(t),o=this._getAllEdgesOverlappingWith(i);return o.length>0?e===!0?this.body.edges[o[o.length-1]]:o[o.length-1]:void 0}},{key:"_addToSelection",value:function(t){t instanceof a["default"]?this.selectionObj.nodes[t.id]=t:this.selectionObj.edges[t.id]=t}},{key:"_addToHover",value:function(t){t instanceof a["default"]?this.hoverObj.nodes[t.id]=t:this.hoverObj.edges[t.id]=t}},{key:"_removeFromSelection",value:function(t){t instanceof a["default"]?(delete this.selectionObj.nodes[t.id],this._unselectConnectedEdges(t)):delete this.selectionObj.edges[t.id]}},{key:"unselectAll",value:function(){for(var t in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(t)&&this.selectionObj.nodes[t].unselect();for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&this.selectionObj.edges[e].unselect();this.selectionObj={nodes:{},edges:{}}}},{key:"_getSelectedNodeCount",value:function(){var t=0;for(var e in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(e)&&(t+=1);return t}},{key:"_getSelectedNode",value:function(){for(var t in this.selectionObj.nodes)if(this.selectionObj.nodes.hasOwnProperty(t))return this.selectionObj.nodes[t]}},{key:"_getSelectedEdge",value:function(){for(var t in this.selectionObj.edges)if(this.selectionObj.edges.hasOwnProperty(t))return this.selectionObj.edges[t]}},{key:"_getSelectedEdgeCount",value:function(){var t=0;for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&(t+=1);return t}},{key:"_getSelectedObjectCount",value:function(){var t=0;for(var e in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(e)&&(t+=1);for(var i in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(i)&&(t+=1);return t}},{key:"_selectionIsEmpty",value:function(){for(var t in this.selectionObj.nodes)if(this.selectionObj.nodes.hasOwnProperty(t))return!1;for(var e in this.selectionObj.edges)if(this.selectionObj.edges.hasOwnProperty(e))return!1;return!0}},{key:"_clusterInSelection",value:function(){for(var t in this.selectionObj.nodes)if(this.selectionObj.nodes.hasOwnProperty(t)&&this.selectionObj.nodes[t].clusterSize>1)return!0;return!1}},{key:"_selectConnectedEdges",value:function(t){for(var e=0;e<t.edges.length;e++){var i=t.edges[e];i.select(),this._addToSelection(i)}}},{key:"_hoverConnectedEdges",value:function(t){for(var e=0;e<t.edges.length;e++){var i=t.edges[e];i.hover=!0,this._addToHover(i)}}},{key:"_unselectConnectedEdges",value:function(t){for(var e=0;e<t.edges.length;e++){var i=t.edges[e];i.unselect(),this._removeFromSelection(i)}}},{key:"blurObject",value:function(t){t.hover===!0&&(t.hover=!1,t instanceof a["default"]?this.body.emitter.emit("blurNode",{node:t.id}):this.body.emitter.emit("blurEdge",{edge:t.id}))}},{key:"hoverObject",value:function(t){var e=!1;for(var i in this.hoverObj.nodes)this.hoverObj.nodes.hasOwnProperty(i)&&(void 0===t||t instanceof a["default"]&&t.id!=i||t instanceof d["default"])&&(this.blurObject(this.hoverObj.nodes[i]),delete this.hoverObj.nodes[i],e=!0);for(var o in this.hoverObj.edges)this.hoverObj.edges.hasOwnProperty(o)&&(e===!0?(this.hoverObj.edges[o].hover=!1,delete this.hoverObj.edges[o]):void 0===t&&(this.blurObject(this.hoverObj.edges[o]),delete this.hoverObj.edges[o],e=!0));void 0!==t&&(t.hover===!1&&(t.hover=!0,this._addToHover(t),e=!0,t instanceof a["default"]?this.body.emitter.emit("hoverNode",{node:t.id}):this.body.emitter.emit("hoverEdge",{edge:t.id})),t instanceof a["default"]&&this.options.hoverConnectedEdges===!0&&this._hoverConnectedEdges(t)),e===!0&&this.body.emitter.emit("_requestRedraw")}},{key:"getSelection",value:function(){var t=this.getSelectedNodes(),e=this.getSelectedEdges();return{nodes:t,edges:e}}},{key:"getSelectedNodes",value:function(){var t=[];if(this.options.selectable===!0)for(var e in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(e)&&t.push(this.selectionObj.nodes[e].id);return t}},{key:"getSelectedEdges",value:function(){var t=[];if(this.options.selectable===!0)for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&t.push(this.selectionObj.edges[e].id);return t}},{key:"setSelection",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?{}:arguments[1],i=void 0,o=void 0;if(!t||!t.nodes&&!t.edges)throw"Selection must be an object with nodes and/or edges properties";if((e.unselectAll||void 0===e.unselectAll)&&this.unselectAll(),t.nodes)for(i=0;i<t.nodes.length;i++){o=t.nodes[i];var n=this.body.nodes[o];if(!n)throw new RangeError('Node with id "'+o+'" not found');this.selectObject(n,e.highlightEdges)}if(t.edges)for(i=0;i<t.edges.length;i++){o=t.edges[i];var s=this.body.edges[o];if(!s)throw new RangeError('Edge with id "'+o+'" not found');this.selectObject(s)}this.body.emitter.emit("_requestRedraw")}},{key:"selectNodes",value:function(t){var e=arguments.length<=1||void 0===arguments[1]?!0:arguments[1];if(!t||void 0===t.length)throw"Selection must be an array with ids";this.setSelection({nodes:t},{highlightEdges:e})}},{key:"selectEdges",value:function(t){if(!t||void 0===t.length)throw"Selection must be an array with ids";this.setSelection({edges:t})}},{key:"updateSelection",value:function(){for(var t in this.selectionObj.nodes)this.selectionObj.nodes.hasOwnProperty(t)&&(this.body.nodes.hasOwnProperty(t)||delete this.selectionObj.nodes[t]);for(var e in this.selectionObj.edges)this.selectionObj.edges.hasOwnProperty(e)&&(this.body.edges.hasOwnProperty(e)||delete this.selectionObj.edges[e])}}]),t}();e["default"]=c},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol?"symbol":typeof t},a=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),h=i(103),d=o(h),l=i(1),c=function(){function t(e){n(this,t),this.body=e,this.initialRandomSeed=Math.round(1e6*Math.random()),this.randomSeed=this.initialRandomSeed,this.setPhysics=!1,this.options={},this.optionsBackup={physics:{}},this.defaultOptions={randomSeed:void 0,improvedLayout:!0,hierarchical:{enabled:!1,levelSeparation:150,nodeSpacing:100,treeSpacing:200,blockShifting:!0,edgeMinimization:!0,parentCentralization:!0,direction:"UD",sortMethod:"hubsize"}},l.extend(this.options,this.defaultOptions),this.bindEventListeners()}return a(t,[{key:"bindEventListeners",value:function(){var t=this;this.body.emitter.on("_dataChanged",function(){t.setupHierarchicalLayout()}),this.body.emitter.on("_dataLoaded",function(){t.layoutNetwork()}),this.body.emitter.on("_resetHierarchicalLayout",function(){t.setupHierarchicalLayout()})}},{key:"setOptions",value:function(t,e){if(void 0!==t){var i=this.options.hierarchical.enabled;if(l.selectiveDeepExtend(["randomSeed","improvedLayout"],this.options,t),l.mergeOptions(this.options,t,"hierarchical"),void 0!==t.randomSeed&&(this.initialRandomSeed=t.randomSeed),this.options.hierarchical.enabled===!0)return i===!0&&this.body.emitter.emit("refresh",!0),"RL"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?this.options.hierarchical.levelSeparation>0&&(this.options.hierarchical.levelSeparation*=-1):this.options.hierarchical.levelSeparation<0&&(this.options.hierarchical.levelSeparation*=-1),this.body.emitter.emit("_resetHierarchicalLayout"),this.adaptAllOptionsForHierarchicalLayout(e);if(i===!0)return this.body.emitter.emit("refresh"),l.deepExtend(e,this.optionsBackup)}return e}},{key:"adaptAllOptionsForHierarchicalLayout",value:function(t){if(this.options.hierarchical.enabled===!0){void 0===t.physics||t.physics===!0?(t.physics={enabled:void 0===this.optionsBackup.physics.enabled?!0:this.optionsBackup.physics.enabled,solver:"hierarchicalRepulsion"},this.optionsBackup.physics.enabled=void 0===this.optionsBackup.physics.enabled?!0:this.optionsBackup.physics.enabled,this.optionsBackup.physics.solver=this.optionsBackup.physics.solver||"barnesHut"):"object"===r(t.physics)?(this.optionsBackup.physics.enabled=void 0===t.physics.enabled?!0:t.physics.enabled,this.optionsBackup.physics.solver=t.physics.solver||"barnesHut",t.physics.solver="hierarchicalRepulsion"):t.physics!==!1&&(this.optionsBackup.physics.solver="barnesHut",t.physics={solver:"hierarchicalRepulsion"});var e="horizontal";"RL"!==this.options.hierarchical.direction&&"LR"!==this.options.hierarchical.direction||(e="vertical"),void 0===t.edges?(this.optionsBackup.edges={smooth:{enabled:!0,type:"dynamic"}},t.edges={smooth:!1}):void 0===t.edges.smooth?(this.optionsBackup.edges={smooth:{enabled:!0,type:"dynamic"}},t.edges.smooth=!1):"boolean"==typeof t.edges.smooth?(this.optionsBackup.edges={smooth:t.edges.smooth},t.edges.smooth={enabled:t.edges.smooth,type:e}):(void 0!==t.edges.smooth.type&&"dynamic"!==t.edges.smooth.type&&(e=t.edges.smooth.type),this.optionsBackup.edges={smooth:void 0===t.edges.smooth.enabled?!0:t.edges.smooth.enabled,type:void 0===t.edges.smooth.type?"dynamic":t.edges.smooth.type,roundness:void 0===t.edges.smooth.roundness?.5:t.edges.smooth.roundness,forceDirection:void 0===t.edges.smooth.forceDirection?!1:t.edges.smooth.forceDirection},t.edges.smooth={enabled:void 0===t.edges.smooth.enabled?!0:t.edges.smooth.enabled,type:e,roundness:void 0===t.edges.smooth.roundness?.5:t.edges.smooth.roundness,forceDirection:void 0===t.edges.smooth.forceDirection?!1:t.edges.smooth.forceDirection}),this.body.emitter.emit("_forceDisableDynamicCurves",e)}return t}},{key:"seededRandom",value:function(){var t=1e4*Math.sin(this.randomSeed++);return t-Math.floor(t)}},{key:"positionInitially",value:function(t){if(this.options.hierarchical.enabled!==!0){this.randomSeed=this.initialRandomSeed;for(var e=0;e<t.length;e++){var i=t[e],o=1*t.length+10,n=2*Math.PI*this.seededRandom();void 0===i.x&&(i.x=o*Math.cos(n)),void 0===i.y&&(i.y=o*Math.sin(n))}}}},{key:"layoutNetwork",value:function(){if(this.options.hierarchical.enabled!==!0&&this.options.improvedLayout===!0){for(var t=0,e=0;e<this.body.nodeIndices.length;e++){var i=this.body.nodes[this.body.nodeIndices[e]];i.predefinedPosition===!0&&(t+=1)}if(t<.5*this.body.nodeIndices.length){var o=10,n=0,s=100;if(this.body.nodeIndices.length>s){for(var r=this.body.nodeIndices.length;this.body.nodeIndices.length>s;){n+=1;var a=this.body.nodeIndices.length;n%3===0?this.body.modules.clustering.clusterBridges():this.body.modules.clustering.clusterOutliers();var h=this.body.nodeIndices.length;if(a==h&&n%3!==0||n>o)return this._declusterAll(),this.body.emitter.emit("_layoutFailed"),void console.info("This network could not be positioned by this version of the improved layout algorithm. Please disable improvedLayout for better performance.")}this.body.modules.kamadaKawai.setOptions({springLength:Math.max(150,2*r)})}this.body.modules.kamadaKawai.solve(this.body.nodeIndices,this.body.edgeIndices,!0),this._shiftToCenter();for(var d=70,l=0;l<this.body.nodeIndices.length;l++)this.body.nodes[this.body.nodeIndices[l]].x+=(.5-this.seededRandom())*d,this.body.nodes[this.body.nodeIndices[l]].y+=(.5-this.seededRandom())*d;this._declusterAll(),this.body.emitter.emit("_repositionBezierNodes")}}}},{key:"_shiftToCenter",value:function(){for(var t=d["default"].getRangeCore(this.body.nodes,this.body.nodeIndices),e=d["default"].findCenter(t),i=0;i<this.body.nodeIndices.length;i++)this.body.nodes[this.body.nodeIndices[i]].x-=e.x,this.body.nodes[this.body.nodeIndices[i]].y-=e.y}},{key:"_declusterAll",value:function(){for(var t=!0;t===!0;){t=!1;for(var e=0;e<this.body.nodeIndices.length;e++)this.body.nodes[this.body.nodeIndices[e]].isCluster===!0&&(t=!0,this.body.modules.clustering.openCluster(this.body.nodeIndices[e],{},!1));t===!0&&this.body.emitter.emit("_dataChanged")}}},{key:"getSeed",value:function(){return this.initialRandomSeed}},{key:"setupHierarchicalLayout",value:function(){if(this.options.hierarchical.enabled===!0&&this.body.nodeIndices.length>0){var t=void 0,e=void 0,i=!1,o=!0,n=!1;this.hierarchicalLevels={},this.lastNodeOnLevel={},this.hierarchicalChildrenReference={},this.hierarchicalParentReference={},this.hierarchicalTrees={},this.treeIndex=-1,this.distributionOrdering={},this.distributionIndex={},this.distributionOrderingPresence={};for(e in this.body.nodes)this.body.nodes.hasOwnProperty(e)&&(t=this.body.nodes[e],void 0===t.options.x&&void 0===t.options.y&&(o=!1),void 0!==t.options.level?(i=!0,this.hierarchicalLevels[e]=t.options.level):n=!0);if(n===!0&&i===!0)throw new Error("To use the hierarchical layout, nodes require either no predefined levels or levels have to be defined for all nodes.");n===!0&&("hubsize"===this.options.hierarchical.sortMethod?this._determineLevelsByHubsize():"directed"===this.options.hierarchical.sortMethod?this._determineLevelsDirected():"custom"===this.options.hierarchical.sortMethod&&this._determineLevelsCustomCallback());for(var s in this.body.nodes)this.body.nodes.hasOwnProperty(s)&&void 0===this.hierarchicalLevels[s]&&(this.hierarchicalLevels[s]=0);var r=this._getDistribution();this._generateMap(),this._placeNodesByHierarchy(r),this._condenseHierarchy(),this._shiftToCenter()}}},{key:"_condenseHierarchy",value:function(){var t=this,e=!1,i={},o=function(){for(var e=a(),i=0;i<e.length-1;i++){var o=e[i].max-e[i+1].min;n(i+1,o+t.options.hierarchical.treeSpacing)}},n=function(e,i){for(var o in t.hierarchicalTrees)if(t.hierarchicalTrees.hasOwnProperty(o)&&t.hierarchicalTrees[o]===e){var n=t.body.nodes[o],s=t._getPositionForHierarchy(n);t._setPositionForHierarchy(n,s+i,void 0,!0)}},r=function(e){var i=1e9,o=-1e9;for(var n in t.hierarchicalTrees)if(t.hierarchicalTrees.hasOwnProperty(n)&&t.hierarchicalTrees[n]===e){var s=t._getPositionForHierarchy(t.body.nodes[n]);i=Math.min(s,i),o=Math.max(s,o)}return{min:i,max:o}},a=function(){for(var e=[],i=0;i<=t.treeIndex;i++)e.push(r(i));return e},h=function w(e,i){if(i[e.id]=!0,t.hierarchicalChildrenReference[e.id]){var o=t.hierarchicalChildrenReference[e.id];if(o.length>0)for(var n=0;n<o.length;n++)w(t.body.nodes[o[n]],i)}},d=function(e){var i=arguments.length<=1||void 0===arguments[1]?1e9:arguments[1],o=1e9,n=1e9,r=1e9,a=-1e9;for(var h in e)if(e.hasOwnProperty(h)){var d=t.body.nodes[h],l=t.hierarchicalLevels[d.id],c=t._getPositionForHierarchy(d),u=t._getSpaceAroundNode(d,e),p=s(u,2),f=p[0],m=p[1];o=Math.min(f,o),n=Math.min(m,n),i>=l&&(r=Math.min(c,r),a=Math.max(c,a))}return[r,a,o,n]},l=function _(e){var i=t.hierarchicalLevels[e];if(t.hierarchicalChildrenReference[e]){var o=t.hierarchicalChildrenReference[e];if(o.length>0)for(var n=0;n<o.length;n++)i=Math.max(i,_(o[n]))}return i},c=function(t,e){var i=l(t.id),o=l(e.id);return Math.min(i,o)},u=function(e,i){var o=t.hierarchicalParentReference[e.id],n=t.hierarchicalParentReference[i.id];if(void 0===o||void 0===n)return!1;for(var s=0;s<o.length;s++)for(var r=0;r<n.length;r++)if(o[s]==n[r])return!0;return!1},p=function(e,i,o){for(var n=0;n<i.length;n++){var s=i[n],r=t.distributionOrdering[s];if(r.length>1)for(var a=0;a<r.length-1;a++)u(r[a],r[a+1])===!0&&t.hierarchicalTrees[r[a].id]===t.hierarchicalTrees[r[a+1].id]&&e(r[a],r[a+1],o)}},f=function(i,o){var n=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],r=t._getPositionForHierarchy(i),a=t._getPositionForHierarchy(o),l=Math.abs(a-r);if(l>t.options.hierarchical.nodeSpacing){var u={};u[i.id]=!0;var p={};p[o.id]=!0,h(i,u),h(o,p);var f=c(i,o),m=d(u,f),v=s(m,4),g=(v[0],v[1]),y=(v[2],v[3],d(p,f)),b=s(y,4),w=b[0],_=(b[1],b[2]),x=(b[3],Math.abs(g-w));if(x>t.options.hierarchical.nodeSpacing){var k=g-w+t.options.hierarchical.nodeSpacing;k<-_+t.options.hierarchical.nodeSpacing&&(k=-_+t.options.hierarchical.nodeSpacing),0>k&&(t._shiftBlock(o.id,k),e=!0,n===!0&&t._centerParent(o))}}},m=function(o,n){for(var r=n.id,a=n.edges,l=t.hierarchicalLevels[n.id],c=t.options.hierarchical.levelSeparation*t.options.hierarchical.levelSeparation,u={},p=[],f=0;f<a.length;f++){var m=a[f];if(m.toId!=m.fromId){var v=m.toId==r?m.from:m.to;u[a[f].id]=v,t.hierarchicalLevels[v.id]<l&&p.push(m)}}var g=function(e,i){for(var o=0,n=0;n<i.length;n++)if(void 0!==u[i[n].id]){var s=t._getPositionForHierarchy(u[i[n].id])-e;o+=s/Math.sqrt(s*s+c)}return o},y=function(e,i){for(var o=0,n=0;n<i.length;n++)if(void 0!==u[i[n].id]){var s=t._getPositionForHierarchy(u[i[n].id])-e;o-=c*Math.pow(s*s+c,-1.5)}return o},b=function(e,i){for(var o=t._getPositionForHierarchy(n),s={},r=0;e>r;r++){var a=g(o,i),h=y(o,i),d=40,l=Math.max(-d,Math.min(d,Math.round(a/h)));if(o-=l,void 0!==s[o])break;s[o]=r}return o},w=function(o){var r=t._getPositionForHierarchy(n);if(void 0===i[n.id]){var a={};a[n.id]=!0,h(n,a),i[n.id]=a}var l=d(i[n.id]),c=s(l,4),u=(c[0],c[1],c[2]),p=c[3],f=o-r,m=0;f>0?m=Math.min(f,p-t.options.hierarchical.nodeSpacing):0>f&&(m=-Math.min(-f,u-t.options.hierarchical.nodeSpacing)),0!=m&&(t._shiftBlock(n.id,m),e=!0)},_=function(i){var o=t._getPositionForHierarchy(n),r=t._getSpaceAroundNode(n),a=s(r,2),h=a[0],d=a[1],l=i-o,c=o;l>0?c=Math.min(o+(d-t.options.hierarchical.nodeSpacing),i):0>l&&(c=Math.max(o-(h-t.options.hierarchical.nodeSpacing),i)),c!==o&&(t._setPositionForHierarchy(n,c,void 0,!0),e=!0)},x=b(o,p);w(x),x=b(o,a),_(x)},v=function(i){var o=Object.keys(t.distributionOrdering);o=o.reverse();for(var n=0;i>n;n++){e=!1;for(var s=0;s<o.length;s++)for(var r=o[s],a=t.distributionOrdering[r],h=0;h<a.length;h++)m(1e3,a[h]);if(e!==!0)break}},g=function(i){var o=Object.keys(t.distributionOrdering);o=o.reverse();for(var n=0;i>n&&(e=!1,p(f,o,!0),e===!0);n++);},y=function(){for(var e in t.body.nodes)t.body.nodes.hasOwnProperty(e)&&t._centerParent(t.body.nodes[e])},b=function(){var e=Object.keys(t.distributionOrdering);e=e.reverse();for(var i=0;i<e.length;i++)for(var o=e[i],n=t.distributionOrdering[o],s=0;s<n.length;s++)t._centerParent(n[s])};this.options.hierarchical.blockShifting===!0&&(g(5),y()),this.options.hierarchical.edgeMinimization===!0&&v(20),this.options.hierarchical.parentCentralization===!0&&b(),o()}},{key:"_getSpaceAroundNode",value:function(t,e){var i=!0;void 0===e&&(i=!1);var o=this.hierarchicalLevels[t.id];if(void 0!==o){var n=this.distributionIndex[t.id],s=this._getPositionForHierarchy(t),r=1e9,a=1e9;if(0!==n){var h=this.distributionOrdering[o][n-1];if(i===!0&&void 0===e[h.id]||i===!1){var d=this._getPositionForHierarchy(h);r=s-d}}if(n!=this.distributionOrdering[o].length-1){var l=this.distributionOrdering[o][n+1];if(i===!0&&void 0===e[l.id]||i===!1){var c=this._getPositionForHierarchy(l);a=Math.min(a,c-s)}}return[r,a]}return[0,0]}},{key:"_centerParent",value:function(t){if(this.hierarchicalParentReference[t.id])for(var e=this.hierarchicalParentReference[t.id],i=0;i<e.length;i++){var o=e[i],n=this.body.nodes[o];if(this.hierarchicalChildrenReference[o]){var r=1e9,a=-1e9,h=this.hierarchicalChildrenReference[o];if(h.length>0)for(var d=0;d<h.length;d++){var l=this.body.nodes[h[d]];r=Math.min(r,this._getPositionForHierarchy(l)),a=Math.max(a,this._getPositionForHierarchy(l))}var c=this._getPositionForHierarchy(n),u=this._getSpaceAroundNode(n),p=s(u,2),f=p[0],m=p[1],v=.5*(r+a),g=c-v;(0>g&&Math.abs(g)<m-this.options.hierarchical.nodeSpacing||g>0&&Math.abs(g)<f-this.options.hierarchical.nodeSpacing)&&this._setPositionForHierarchy(n,v,void 0,!0)}}}},{key:"_placeNodesByHierarchy",value:function(t){this.positionedNodes={};for(var e in t)if(t.hasOwnProperty(e)){var i=Object.keys(t[e]);i=this._indexArrayToNodes(i),this._sortNodeArray(i);for(var o=0,n=0;n<i.length;n++){var s=i[n];if(void 0===this.positionedNodes[s.id]){var r=this.options.hierarchical.nodeSpacing*o;o>0&&(r=this._getPositionForHierarchy(i[n-1])+this.options.hierarchical.nodeSpacing),this._setPositionForHierarchy(s,r,e),this._validataPositionAndContinue(s,e,r),o++}}}}},{key:"_placeBranchNodes",value:function(t,e){if(void 0!==this.hierarchicalChildrenReference[t]){for(var i=[],o=0;o<this.hierarchicalChildrenReference[t].length;o++)i.push(this.body.nodes[this.hierarchicalChildrenReference[t][o]]);this._sortNodeArray(i);for(var n=0;n<i.length;n++){var s=i[n],r=this.hierarchicalLevels[s.id];if(!(r>e&&void 0===this.positionedNodes[s.id]))return;var a=void 0;a=0===n?this._getPositionForHierarchy(this.body.nodes[t]):this._getPositionForHierarchy(i[n-1])+this.options.hierarchical.nodeSpacing,this._setPositionForHierarchy(s,a,r),this._validataPositionAndContinue(s,r,a)}for(var h=1e9,d=-1e9,l=0;l<i.length;l++){var c=i[l].id;h=Math.min(h,this._getPositionForHierarchy(this.body.nodes[c])),d=Math.max(d,this._getPositionForHierarchy(this.body.nodes[c]))}this._setPositionForHierarchy(this.body.nodes[t],.5*(h+d),e)}}},{key:"_validataPositionAndContinue",value:function(t,e,i){if(void 0!==this.lastNodeOnLevel[e]){var o=this._getPositionForHierarchy(this.body.nodes[this.lastNodeOnLevel[e]]);if(i-o<this.options.hierarchical.nodeSpacing){var n=o+this.options.hierarchical.nodeSpacing-i,s=this._findCommonParent(this.lastNodeOnLevel[e],t.id);this._shiftBlock(s.withChild,n)}}this.lastNodeOnLevel[e]=t.id,this.positionedNodes[t.id]=!0,this._placeBranchNodes(t.id,e)}},{key:"_indexArrayToNodes",value:function(t){for(var e=[],i=0;i<t.length;i++)e.push(this.body.nodes[t[i]]);return e}},{key:"_getDistribution",value:function(){var t={},e=void 0,i=void 0;for(e in this.body.nodes)if(this.body.nodes.hasOwnProperty(e)){i=this.body.nodes[e];var o=void 0===this.hierarchicalLevels[e]?0:this.hierarchicalLevels[e];"UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?(i.y=this.options.hierarchical.levelSeparation*o,i.options.fixed.y=!0):(i.x=this.options.hierarchical.levelSeparation*o,i.options.fixed.x=!0),void 0===t[o]&&(t[o]={}),t[o][e]=i}return t}},{key:"_getHubSize",value:function(){var t=0;for(var e in this.body.nodes)if(this.body.nodes.hasOwnProperty(e)){var i=this.body.nodes[e];void 0===this.hierarchicalLevels[e]&&(t=i.edges.length<t?t:i.edges.length)}return t}},{key:"_determineLevelsByHubsize",value:function(){for(var t=this,e=1,i=function(e,i){void 0===t.hierarchicalLevels[i.id]&&(void 0===t.hierarchicalLevels[e.id]&&(t.hierarchicalLevels[e.id]=0),t.hierarchicalLevels[i.id]=t.hierarchicalLevels[e.id]+1)};e>0&&(e=this._getHubSize(),0!==e);)for(var o in this.body.nodes)if(this.body.nodes.hasOwnProperty(o)){var n=this.body.nodes[o];n.edges.length===e&&this._crawlNetwork(i,o)}}},{key:"_determineLevelsCustomCallback",value:function(){var t=this,e=1e5,i=function(t,e,i){},o=function(o,n,s){var r=t.hierarchicalLevels[o.id];void 0===r&&(t.hierarchicalLevels[o.id]=e);var a=i(d["default"].cloneOptions(o,"node"),d["default"].cloneOptions(n,"node"),d["default"].cloneOptions(s,"edge"));t.hierarchicalLevels[n.id]=t.hierarchicalLevels[o.id]+a};this._crawlNetwork(o),this._setMinLevelToZero()}},{key:"_determineLevelsDirected",value:function(){var t=this,e=1e4,i=function(i,o,n){var s=t.hierarchicalLevels[i.id];void 0===s&&(t.hierarchicalLevels[i.id]=e),n.toId==o.id?t.hierarchicalLevels[o.id]=t.hierarchicalLevels[i.id]+1:t.hierarchicalLevels[o.id]=t.hierarchicalLevels[i.id]-1};this._crawlNetwork(i),this._setMinLevelToZero()}},{key:"_setMinLevelToZero",value:function(){var t=1e9;for(var e in this.body.nodes)this.body.nodes.hasOwnProperty(e)&&void 0!==this.hierarchicalLevels[e]&&(t=Math.min(this.hierarchicalLevels[e],t));for(var i in this.body.nodes)this.body.nodes.hasOwnProperty(i)&&void 0!==this.hierarchicalLevels[i]&&(this.hierarchicalLevels[i]-=t)}},{key:"_generateMap",value:function(){var t=this,e=function(e,i){if(t.hierarchicalLevels[i.id]>t.hierarchicalLevels[e.id]){var o=e.id,n=i.id;void 0===t.hierarchicalChildrenReference[o]&&(t.hierarchicalChildrenReference[o]=[]),t.hierarchicalChildrenReference[o].push(n),void 0===t.hierarchicalParentReference[n]&&(t.hierarchicalParentReference[n]=[]),t.hierarchicalParentReference[n].push(o)}};this._crawlNetwork(e)}},{key:"_crawlNetwork",value:function(){var t=this,e=arguments.length<=0||void 0===arguments[0]?function(){}:arguments[0],i=arguments[1],o={},n=0,s=function d(i,n){if(void 0===o[i.id]){void 0===t.hierarchicalTrees[i.id]&&(t.hierarchicalTrees[i.id]=n,t.treeIndex=Math.max(n,t.treeIndex)),o[i.id]=!0;for(var s=void 0,r=0;r<i.edges.length;r++)i.edges[r].connected===!0&&(s=i.edges[r].toId===i.id?i.edges[r].from:i.edges[r].to,i.id!==s.id&&(e(i,s,i.edges[r]),d(s,n)))}};if(void 0===i)for(var r=0;r<this.body.nodeIndices.length;r++){var a=this.body.nodes[this.body.nodeIndices[r]];void 0===o[a.id]&&(s(a,n),n+=1)}else{var h=this.body.nodes[i];if(void 0===h)return void console.error("Node not found:",i);s(h)}}},{key:"_shiftBlock",value:function(t,e){if("UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?this.body.nodes[t].x+=e:this.body.nodes[t].y+=e,void 0!==this.hierarchicalChildrenReference[t])for(var i=0;i<this.hierarchicalChildrenReference[t].length;i++)this._shiftBlock(this.hierarchicalChildrenReference[t][i],e)}},{key:"_findCommonParent",value:function(t,e){var i=this,o={},n=function r(t,e){if(void 0!==i.hierarchicalParentReference[e])for(var o=0;o<i.hierarchicalParentReference[e].length;o++){var n=i.hierarchicalParentReference[e][o];t[n]=!0,r(t,n)}},s=function a(t,e){if(void 0!==i.hierarchicalParentReference[e])for(var o=0;o<i.hierarchicalParentReference[e].length;o++){var n=i.hierarchicalParentReference[e][o];if(void 0!==t[n])return{foundParent:n,withChild:e};var s=a(t,n);if(null!==s.foundParent)return s}return{foundParent:null,withChild:e}};return n(o,t),s(o,e)}},{key:"_setPositionForHierarchy",value:function(t,e,i){var o=arguments.length<=3||void 0===arguments[3]?!1:arguments[3];o!==!0&&(void 0===this.distributionOrdering[i]&&(this.distributionOrdering[i]=[],this.distributionOrderingPresence[i]={}),void 0===this.distributionOrderingPresence[i][t.id]&&(this.distributionOrdering[i].push(t),this.distributionIndex[t.id]=this.distributionOrdering[i].length-1),this.distributionOrderingPresence[i][t.id]=!0),"UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?t.x=e:t.y=e}},{key:"_getPositionForHierarchy",value:function(t){return"UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?t.x:t.y}},{key:"_sortNodeArray",value:function(t){t.length>1&&("UD"===this.options.hierarchical.direction||"DU"===this.options.hierarchical.direction?t.sort(function(t,e){return t.x-e.x}):t.sort(function(t,e){return t.y-e.y}))}}]),t}();e["default"]=c},function(t,e,i){function o(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var n=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),s=i(1),r=i(20),a=i(28),h=function(){function t(e,i,n){var r=this;o(this,t),this.body=e,this.canvas=i,this.selectionHandler=n,this.editMode=!1,this.manipulationDiv=void 0,this.editModeDiv=void 0,this.closeDiv=void 0,this.manipulationHammers=[],this.temporaryUIFunctions={},this.temporaryEventFunctions=[],this.touchTime=0,this.temporaryIds={nodes:[],edges:[]},this.guiEnabled=!1,this.inMode=!1,this.selectedControlNode=void 0,this.options={},this.defaultOptions={enabled:!1,initiallyActive:!1,addNode:!0,addEdge:!0,editNode:void 0,editEdge:!0,deleteNode:!0,deleteEdge:!0,controlNodeStyle:{shape:"dot",size:6,color:{background:"#ff0000",border:"#3c3c3c",highlight:{background:"#07f968",border:"#3c3c3c"}},borderWidth:2,borderWidthSelected:2}},s.extend(this.options,this.defaultOptions),this.body.emitter.on("destroy",function(){r._clean()}),this.body.emitter.on("_dataChanged",this._restore.bind(this)),this.body.emitter.on("_resetData",this._restore.bind(this))}return n(t,[{key:"_restore",value:function(){this.inMode!==!1&&(this.options.initiallyActive===!0?this.enableEditMode():this.disableEditMode())}},{key:"setOptions",value:function(t,e,i){void 0!==e&&(void 0!==e.locale?this.options.locale=e.locale:this.options.locale=i.locale,void 0!==e.locales?this.options.locales=e.locales:this.options.locales=i.locales),void 0!==t&&("boolean"==typeof t?this.options.enabled=t:(this.options.enabled=!0,s.deepExtend(this.options,t)),this.options.initiallyActive===!0&&(this.editMode=!0),this._setup())}},{key:"toggleEditMode",value:function(){this.editMode===!0?this.disableEditMode():this.enableEditMode()}},{key:"enableEditMode",value:function(){this.editMode=!0,this._clean(),this.guiEnabled===!0&&(this.manipulationDiv.style.display="block",this.closeDiv.style.display="block",this.editModeDiv.style.display="none",this.showManipulatorToolbar())}},{key:"disableEditMode",value:function(){this.editMode=!1,this._clean(),this.guiEnabled===!0&&(this.manipulationDiv.style.display="none",this.closeDiv.style.display="none",this.editModeDiv.style.display="block",this._createEditButton())}},{key:"showManipulatorToolbar",value:function(){if(this._clean(),this.manipulationDOM={},this.guiEnabled===!0){this.editMode=!0,this.manipulationDiv.style.display="block",this.closeDiv.style.display="block";var t=this.selectionHandler._getSelectedNodeCount(),e=this.selectionHandler._getSelectedEdgeCount(),i=t+e,o=this.options.locales[this.options.locale],n=!1;this.options.addNode!==!1&&(this._createAddNodeButton(o),n=!0),this.options.addEdge!==!1&&(n===!0?this._createSeperator(1):n=!0,this._createAddEdgeButton(o)),1===t&&"function"==typeof this.options.editNode?(n===!0?this._createSeperator(2):n=!0,this._createEditNodeButton(o)):1===e&&0===t&&this.options.editEdge!==!1&&(n===!0?this._createSeperator(3):n=!0,this._createEditEdgeButton(o)),0!==i&&(t>0&&this.options.deleteNode!==!1?(n===!0&&this._createSeperator(4),this._createDeleteButton(o)):0===t&&this.options.deleteEdge!==!1&&(n===!0&&this._createSeperator(4),this._createDeleteButton(o))),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this)),this._temporaryBindEvent("select",this.showManipulatorToolbar.bind(this))}this.body.emitter.emit("_redraw")}},{key:"addNodeMode",value:function(){if(this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="addNode",this.guiEnabled===!0){var t=this.options.locales[this.options.locale];this.manipulationDOM={},this._createBackButton(t),this._createSeperator(),this._createDescription(t.addDescription||this.options.locales.en.addDescription),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this))}this._temporaryBindEvent("click",this._performAddNode.bind(this))}},{key:"editNode",value:function(){var t=this;this.editMode!==!0&&this.enableEditMode(),this._clean();var e=this.selectionHandler._getSelectedNode();if(void 0!==e){if(this.inMode="editNode","function"!=typeof this.options.editNode)throw new Error("No function has been configured to handle the editing of nodes.");if(e.isCluster!==!0){var i=s.deepExtend({},e.options,!1);
-if(i.x=e.x,i.y=e.y,2!==this.options.editNode.length)throw new Error("The function for edit does not support two arguments (data, callback)");this.options.editNode(i,function(e){null!==e&&void 0!==e&&"editNode"===t.inMode&&t.body.data.nodes.getDataSet().update(e),t.showManipulatorToolbar()})}else alert(this.options.locales[this.options.locale].editClusterError||this.options.locales.en.editClusterError)}else this.showManipulatorToolbar()}},{key:"addEdgeMode",value:function(){if(this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="addEdge",this.guiEnabled===!0){var t=this.options.locales[this.options.locale];this.manipulationDOM={},this._createBackButton(t),this._createSeperator(),this._createDescription(t.edgeDescription||this.options.locales.en.edgeDescription),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this))}this._temporaryBindUI("onTouch",this._handleConnect.bind(this)),this._temporaryBindUI("onDragEnd",this._finishConnect.bind(this)),this._temporaryBindUI("onDrag",this._dragControlNode.bind(this)),this._temporaryBindUI("onRelease",this._finishConnect.bind(this)),this._temporaryBindUI("onDragStart",function(){}),this._temporaryBindUI("onHold",function(){})}},{key:"editEdgeMode",value:function(){var t=this;if(this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="editEdge",this.guiEnabled===!0){var e=this.options.locales[this.options.locale];this.manipulationDOM={},this._createBackButton(e),this._createSeperator(),this._createDescription(e.editEdgeDescription||this.options.locales.en.editEdgeDescription),this._bindHammerToDiv(this.closeDiv,this.toggleEditMode.bind(this))}this.edgeBeingEditedId=this.selectionHandler.getSelectedEdges()[0],void 0!==this.edgeBeingEditedId?!function(){var e=t.body.edges[t.edgeBeingEditedId],i=t._getNewTargetNode(e.from.x,e.from.y),o=t._getNewTargetNode(e.to.x,e.to.y);t.temporaryIds.nodes.push(i.id),t.temporaryIds.nodes.push(o.id),t.body.nodes[i.id]=i,t.body.nodeIndices.push(i.id),t.body.nodes[o.id]=o,t.body.nodeIndices.push(o.id),t._temporaryBindUI("onTouch",t._controlNodeTouch.bind(t)),t._temporaryBindUI("onTap",function(){}),t._temporaryBindUI("onHold",function(){}),t._temporaryBindUI("onDragStart",t._controlNodeDragStart.bind(t)),t._temporaryBindUI("onDrag",t._controlNodeDrag.bind(t)),t._temporaryBindUI("onDragEnd",t._controlNodeDragEnd.bind(t)),t._temporaryBindUI("onMouseMove",function(){}),t._temporaryBindEvent("beforeDrawing",function(t){var n=e.edgeType.findBorderPositions(t);i.selected===!1&&(i.x=n.from.x,i.y=n.from.y),o.selected===!1&&(o.x=n.to.x,o.y=n.to.y)}),t.body.emitter.emit("_redraw")}():this.showManipulatorToolbar()}},{key:"deleteSelected",value:function(){var t=this;this.editMode!==!0&&this.enableEditMode(),this._clean(),this.inMode="delete";var e=this.selectionHandler.getSelectedNodes(),i=this.selectionHandler.getSelectedEdges(),o=void 0;if(e.length>0){for(var n=0;n<e.length;n++)if(this.body.nodes[e[n]].isCluster===!0)return void alert(this.options.locales[this.options.locale].deleteClusterError||this.options.locales.en.deleteClusterError);"function"==typeof this.options.deleteNode&&(o=this.options.deleteNode)}else i.length>0&&"function"==typeof this.options.deleteEdge&&(o=this.options.deleteEdge);if("function"==typeof o){var s={nodes:e,edges:i};if(2!==o.length)throw new Error("The function for delete does not support two arguments (data, callback)");o(s,function(e){null!==e&&void 0!==e&&"delete"===t.inMode?(t.body.data.edges.getDataSet().remove(e.edges),t.body.data.nodes.getDataSet().remove(e.nodes),t.body.emitter.emit("startSimulation"),t.showManipulatorToolbar()):(t.body.emitter.emit("startSimulation"),t.showManipulatorToolbar())})}else this.body.data.edges.getDataSet().remove(i),this.body.data.nodes.getDataSet().remove(e),this.body.emitter.emit("startSimulation"),this.showManipulatorToolbar()}},{key:"_setup",value:function(){this.options.enabled===!0?(this.guiEnabled=!0,this._createWrappers(),this.editMode===!1?this._createEditButton():this.showManipulatorToolbar()):(this._removeManipulationDOM(),this.guiEnabled=!1)}},{key:"_createWrappers",value:function(){void 0===this.manipulationDiv&&(this.manipulationDiv=document.createElement("div"),this.manipulationDiv.className="vis-manipulation",this.editMode===!0?this.manipulationDiv.style.display="block":this.manipulationDiv.style.display="none",this.canvas.frame.appendChild(this.manipulationDiv)),void 0===this.editModeDiv&&(this.editModeDiv=document.createElement("div"),this.editModeDiv.className="vis-edit-mode",this.editMode===!0?this.editModeDiv.style.display="none":this.editModeDiv.style.display="block",this.canvas.frame.appendChild(this.editModeDiv)),void 0===this.closeDiv&&(this.closeDiv=document.createElement("div"),this.closeDiv.className="vis-close",this.closeDiv.style.display=this.manipulationDiv.style.display,this.canvas.frame.appendChild(this.closeDiv))}},{key:"_getNewTargetNode",value:function(t,e){var i=s.deepExtend({},this.options.controlNodeStyle);i.id="targetNode"+s.randomUUID(),i.hidden=!1,i.physics=!1,i.x=t,i.y=e;var o=this.body.functions.createNode(i);return o.shape.boundingBox={left:t,right:t,top:e,bottom:e},o}},{key:"_createEditButton",value:function(){this._clean(),this.manipulationDOM={},s.recursiveDOMDelete(this.editModeDiv);var t=this.options.locales[this.options.locale],e=this._createButton("editMode","vis-button vis-edit vis-edit-mode",t.edit||this.options.locales.en.edit);this.editModeDiv.appendChild(e),this._bindHammerToDiv(e,this.toggleEditMode.bind(this))}},{key:"_clean",value:function(){this.inMode=!1,this.guiEnabled===!0&&(s.recursiveDOMDelete(this.editModeDiv),s.recursiveDOMDelete(this.manipulationDiv),this._cleanManipulatorHammers()),this._cleanupTemporaryNodesAndEdges(),this._unbindTemporaryUIs(),this._unbindTemporaryEvents(),this.body.emitter.emit("restorePhysics")}},{key:"_cleanManipulatorHammers",value:function(){if(0!=this.manipulationHammers.length){for(var t=0;t<this.manipulationHammers.length;t++)this.manipulationHammers[t].destroy();this.manipulationHammers=[]}}},{key:"_removeManipulationDOM",value:function(){this._clean(),s.recursiveDOMDelete(this.manipulationDiv),s.recursiveDOMDelete(this.editModeDiv),s.recursiveDOMDelete(this.closeDiv),this.manipulationDiv&&this.canvas.frame.removeChild(this.manipulationDiv),this.editModeDiv&&this.canvas.frame.removeChild(this.editModeDiv),this.closeDiv&&this.canvas.frame.removeChild(this.closeDiv),this.manipulationDiv=void 0,this.editModeDiv=void 0,this.closeDiv=void 0}},{key:"_createSeperator",value:function(){var t=arguments.length<=0||void 0===arguments[0]?1:arguments[0];this.manipulationDOM["seperatorLineDiv"+t]=document.createElement("div"),this.manipulationDOM["seperatorLineDiv"+t].className="vis-separator-line",this.manipulationDiv.appendChild(this.manipulationDOM["seperatorLineDiv"+t])}},{key:"_createAddNodeButton",value:function(t){var e=this._createButton("addNode","vis-button vis-add",t.addNode||this.options.locales.en.addNode);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.addNodeMode.bind(this))}},{key:"_createAddEdgeButton",value:function(t){var e=this._createButton("addEdge","vis-button vis-connect",t.addEdge||this.options.locales.en.addEdge);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.addEdgeMode.bind(this))}},{key:"_createEditNodeButton",value:function(t){var e=this._createButton("editNode","vis-button vis-edit",t.editNode||this.options.locales.en.editNode);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.editNode.bind(this))}},{key:"_createEditEdgeButton",value:function(t){var e=this._createButton("editEdge","vis-button vis-edit",t.editEdge||this.options.locales.en.editEdge);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.editEdgeMode.bind(this))}},{key:"_createDeleteButton",value:function(t){if(this.options.rtl)var e="vis-button vis-delete-rtl";else var e="vis-button vis-delete";var i=this._createButton("delete",e,t.del||this.options.locales.en.del);this.manipulationDiv.appendChild(i),this._bindHammerToDiv(i,this.deleteSelected.bind(this))}},{key:"_createBackButton",value:function(t){var e=this._createButton("back","vis-button vis-back",t.back||this.options.locales.en.back);this.manipulationDiv.appendChild(e),this._bindHammerToDiv(e,this.showManipulatorToolbar.bind(this))}},{key:"_createButton",value:function(t,e,i){var o=arguments.length<=3||void 0===arguments[3]?"vis-label":arguments[3];return this.manipulationDOM[t+"Div"]=document.createElement("div"),this.manipulationDOM[t+"Div"].className=e,this.manipulationDOM[t+"Label"]=document.createElement("div"),this.manipulationDOM[t+"Label"].className=o,this.manipulationDOM[t+"Label"].innerHTML=i,this.manipulationDOM[t+"Div"].appendChild(this.manipulationDOM[t+"Label"]),this.manipulationDOM[t+"Div"]}},{key:"_createDescription",value:function(t){this.manipulationDiv.appendChild(this._createButton("description","vis-button vis-none",t))}},{key:"_temporaryBindEvent",value:function(t,e){this.temporaryEventFunctions.push({event:t,boundFunction:e}),this.body.emitter.on(t,e)}},{key:"_temporaryBindUI",value:function(t,e){if(void 0===this.body.eventListeners[t])throw new Error("This UI function does not exist. Typo? You tried: "+t+" possible are: "+JSON.stringify(Object.keys(this.body.eventListeners)));this.temporaryUIFunctions[t]=this.body.eventListeners[t],this.body.eventListeners[t]=e}},{key:"_unbindTemporaryUIs",value:function(){for(var t in this.temporaryUIFunctions)this.temporaryUIFunctions.hasOwnProperty(t)&&(this.body.eventListeners[t]=this.temporaryUIFunctions[t],delete this.temporaryUIFunctions[t]);this.temporaryUIFunctions={}}},{key:"_unbindTemporaryEvents",value:function(){for(var t=0;t<this.temporaryEventFunctions.length;t++){var e=this.temporaryEventFunctions[t].event,i=this.temporaryEventFunctions[t].boundFunction;this.body.emitter.off(e,i)}this.temporaryEventFunctions=[]}},{key:"_bindHammerToDiv",value:function(t,e){var i=new r(t,{});a.onTouch(i,e),this.manipulationHammers.push(i)}},{key:"_cleanupTemporaryNodesAndEdges",value:function(){for(var t=0;t<this.temporaryIds.edges.length;t++){this.body.edges[this.temporaryIds.edges[t]].disconnect(),delete this.body.edges[this.temporaryIds.edges[t]];var e=this.body.edgeIndices.indexOf(this.temporaryIds.edges[t]);-1!==e&&this.body.edgeIndices.splice(e,1)}for(var i=0;i<this.temporaryIds.nodes.length;i++){delete this.body.nodes[this.temporaryIds.nodes[i]];var o=this.body.nodeIndices.indexOf(this.temporaryIds.nodes[i]);-1!==o&&this.body.nodeIndices.splice(o,1)}this.temporaryIds={nodes:[],edges:[]}}},{key:"_controlNodeTouch",value:function(t){this.selectionHandler.unselectAll(),this.lastTouch=this.body.functions.getPointer(t.center),this.lastTouch.translation=s.extend({},this.body.view.translation)}},{key:"_controlNodeDragStart",value:function(t){var e=this.lastTouch,i=this.selectionHandler._pointerToPositionObject(e),o=this.body.nodes[this.temporaryIds.nodes[0]],n=this.body.nodes[this.temporaryIds.nodes[1]],s=this.body.edges[this.edgeBeingEditedId];this.selectedControlNode=void 0;var r=o.isOverlappingWith(i),a=n.isOverlappingWith(i);r===!0?(this.selectedControlNode=o,s.edgeType.from=o):a===!0&&(this.selectedControlNode=n,s.edgeType.to=n),void 0!==this.selectedControlNode&&this.selectionHandler.selectObject(this.selectedControlNode),this.body.emitter.emit("_redraw")}},{key:"_controlNodeDrag",value:function(t){this.body.emitter.emit("disablePhysics");var e=this.body.functions.getPointer(t.center),i=this.canvas.DOMtoCanvas(e);if(void 0!==this.selectedControlNode)this.selectedControlNode.x=i.x,this.selectedControlNode.y=i.y;else{var o=e.x-this.lastTouch.x,n=e.y-this.lastTouch.y;this.body.view.translation={x:this.lastTouch.translation.x+o,y:this.lastTouch.translation.y+n}}this.body.emitter.emit("_redraw")}},{key:"_controlNodeDragEnd",value:function(t){var e=this.body.functions.getPointer(t.center),i=this.selectionHandler._pointerToPositionObject(e),o=this.body.edges[this.edgeBeingEditedId];if(void 0!==this.selectedControlNode){this.selectionHandler.unselectAll();for(var n=this.selectionHandler._getAllNodesOverlappingWith(i),s=void 0,r=n.length-1;r>=0;r--)if(n[r]!==this.selectedControlNode.id){s=this.body.nodes[n[r]];break}if(void 0!==s&&void 0!==this.selectedControlNode)if(s.isCluster===!0)alert(this.options.locales[this.options.locale].createEdgeError||this.options.locales.en.createEdgeError);else{var a=this.body.nodes[this.temporaryIds.nodes[0]];this.selectedControlNode.id===a.id?this._performEditEdge(s.id,o.to.id):this._performEditEdge(o.from.id,s.id)}else o.updateEdgeType(),this.body.emitter.emit("restorePhysics");this.body.emitter.emit("_redraw")}}},{key:"_handleConnect",value:function(t){if((new Date).valueOf()-this.touchTime>100){this.lastTouch=this.body.functions.getPointer(t.center),this.lastTouch.translation=s.extend({},this.body.view.translation);var e=this.lastTouch,i=this.selectionHandler.getNodeAt(e);if(void 0!==i)if(i.isCluster===!0)alert(this.options.locales[this.options.locale].createEdgeError||this.options.locales.en.createEdgeError);else{var o=this._getNewTargetNode(i.x,i.y);this.body.nodes[o.id]=o,this.body.nodeIndices.push(o.id);var n=this.body.functions.createEdge({id:"connectionEdge"+s.randomUUID(),from:i.id,to:o.id,physics:!1,smooth:{enabled:!0,type:"continuous",roundness:.5}});this.body.edges[n.id]=n,this.body.edgeIndices.push(n.id),this.temporaryIds.nodes.push(o.id),this.temporaryIds.edges.push(n.id)}this.touchTime=(new Date).valueOf()}}},{key:"_dragControlNode",value:function(t){var e=this.body.functions.getPointer(t.center);if(void 0!==this.temporaryIds.nodes[0]){var i=this.body.nodes[this.temporaryIds.nodes[0]];i.x=this.canvas._XconvertDOMtoCanvas(e.x),i.y=this.canvas._YconvertDOMtoCanvas(e.y),this.body.emitter.emit("_redraw")}else{var o=e.x-this.lastTouch.x,n=e.y-this.lastTouch.y;this.body.view.translation={x:this.lastTouch.translation.x+o,y:this.lastTouch.translation.y+n}}}},{key:"_finishConnect",value:function(t){var e=this.body.functions.getPointer(t.center),i=this.selectionHandler._pointerToPositionObject(e),o=void 0;void 0!==this.temporaryIds.edges[0]&&(o=this.body.edges[this.temporaryIds.edges[0]].fromId);for(var n=this.selectionHandler._getAllNodesOverlappingWith(i),s=void 0,r=n.length-1;r>=0;r--)if(-1===this.temporaryIds.nodes.indexOf(n[r])){s=this.body.nodes[n[r]];break}this._cleanupTemporaryNodesAndEdges(),void 0!==s&&(s.isCluster===!0?alert(this.options.locales[this.options.locale].createEdgeError||this.options.locales.en.createEdgeError):void 0!==this.body.nodes[o]&&void 0!==this.body.nodes[s.id]&&this._performAddEdge(o,s.id)),this.body.emitter.emit("_redraw")}},{key:"_performAddNode",value:function(t){var e=this,i={id:s.randomUUID(),x:t.pointer.canvas.x,y:t.pointer.canvas.y,label:"new"};if("function"==typeof this.options.addNode){if(2!==this.options.addNode.length)throw new Error("The function for add does not support two arguments (data,callback)");this.options.addNode(i,function(t){null!==t&&void 0!==t&&"addNode"===e.inMode&&(e.body.data.nodes.getDataSet().add(t),e.showManipulatorToolbar())})}else this.body.data.nodes.getDataSet().add(i),this.showManipulatorToolbar()}},{key:"_performAddEdge",value:function(t,e){var i=this,o={from:t,to:e};if("function"==typeof this.options.addEdge){if(2!==this.options.addEdge.length)throw new Error("The function for connect does not support two arguments (data,callback)");this.options.addEdge(o,function(t){null!==t&&void 0!==t&&"addEdge"===i.inMode&&(i.body.data.edges.getDataSet().add(t),i.selectionHandler.unselectAll(),i.showManipulatorToolbar())})}else this.body.data.edges.getDataSet().add(o),this.selectionHandler.unselectAll(),this.showManipulatorToolbar()}},{key:"_performEditEdge",value:function(t,e){var i=this,o={id:this.edgeBeingEditedId,from:t,to:e};if("function"==typeof this.options.editEdge){if(2!==this.options.editEdge.length)throw new Error("The function for edit does not support two arguments (data, callback)");this.options.editEdge(o,function(t){null===t||void 0===t||"editEdge"!==i.inMode?(i.body.edges[o.id].updateEdgeType(),i.body.emitter.emit("_redraw")):(i.body.data.edges.getDataSet().update(t),i.selectionHandler.unselectAll(),i.showManipulatorToolbar())})}else this.body.data.edges.getDataSet().update(o),this.selectionHandler.unselectAll(),this.showManipulatorToolbar()}}]),t}();e["default"]=h},function(t,e){Object.defineProperty(e,"__esModule",{value:!0});var i="string",o="boolean",n="number",s="array",r="object",a="dom",h="any",d={configure:{enabled:{"boolean":o},filter:{"boolean":o,string:i,array:s,"function":"function"},container:{dom:a},showButton:{"boolean":o},__type__:{object:r,"boolean":o,string:i,array:s,"function":"function"}},edges:{arrows:{to:{enabled:{"boolean":o},scaleFactor:{number:n},__type__:{object:r,"boolean":o}},middle:{enabled:{"boolean":o},scaleFactor:{number:n},__type__:{object:r,"boolean":o}},from:{enabled:{"boolean":o},scaleFactor:{number:n},__type__:{object:r,"boolean":o}},__type__:{string:["from","to","middle"],object:r}},arrowStrikethrough:{"boolean":o},color:{color:{string:i},highlight:{string:i},hover:{string:i},inherit:{string:["from","to","both"],"boolean":o},opacity:{number:n},__type__:{object:r,string:i}},dashes:{"boolean":o,array:s},font:{color:{string:i},size:{number:n},face:{string:i},background:{string:i},strokeWidth:{number:n},strokeColor:{string:i},align:{string:["horizontal","top","middle","bottom"]},__type__:{object:r,string:i}},hidden:{"boolean":o},hoverWidth:{"function":"function",number:n},label:{string:i,undefined:"undefined"},labelHighlightBold:{"boolean":o},length:{number:n,undefined:"undefined"},physics:{"boolean":o},scaling:{min:{number:n},max:{number:n},label:{enabled:{"boolean":o},min:{number:n},max:{number:n},maxVisible:{number:n},drawThreshold:{number:n},__type__:{object:r,"boolean":o}},customScalingFunction:{"function":"function"},__type__:{object:r}},selectionWidth:{"function":"function",number:n},selfReferenceSize:{number:n},shadow:{enabled:{"boolean":o},color:{string:i},size:{number:n},x:{number:n},y:{number:n},__type__:{object:r,"boolean":o}},smooth:{enabled:{"boolean":o},type:{string:["dynamic","continuous","discrete","diagonalCross","straightCross","horizontal","vertical","curvedCW","curvedCCW","cubicBezier"]},roundness:{number:n},forceDirection:{string:["horizontal","vertical","none"],"boolean":o},__type__:{object:r,"boolean":o}},title:{string:i,undefined:"undefined"},width:{number:n},value:{number:n,undefined:"undefined"},__type__:{object:r}},groups:{useDefaultGroups:{"boolean":o},__any__:"get from nodes, will be overwritten below",__type__:{object:r}},interaction:{dragNodes:{"boolean":o},dragView:{"boolean":o},hideEdgesOnDrag:{"boolean":o},hideNodesOnDrag:{"boolean":o},hover:{"boolean":o},keyboard:{enabled:{"boolean":o},speed:{x:{number:n},y:{number:n},zoom:{number:n},__type__:{object:r}},bindToWindow:{"boolean":o},__type__:{object:r,"boolean":o}},multiselect:{"boolean":o},navigationButtons:{"boolean":o},selectable:{"boolean":o},selectConnectedEdges:{"boolean":o},hoverConnectedEdges:{"boolean":o},tooltipDelay:{number:n},zoomView:{"boolean":o},__type__:{object:r}},layout:{randomSeed:{undefined:"undefined",number:n},improvedLayout:{"boolean":o},hierarchical:{enabled:{"boolean":o},levelSeparation:{number:n},nodeSpacing:{number:n},treeSpacing:{number:n},blockShifting:{"boolean":o},edgeMinimization:{"boolean":o},parentCentralization:{"boolean":o},direction:{string:["UD","DU","LR","RL"]},sortMethod:{string:["hubsize","directed"]},__type__:{object:r,"boolean":o}},__type__:{object:r}},manipulation:{enabled:{"boolean":o},initiallyActive:{"boolean":o},addNode:{"boolean":o,"function":"function"},addEdge:{"boolean":o,"function":"function"},editNode:{"function":"function"},editEdge:{"boolean":o,"function":"function"},deleteNode:{"boolean":o,"function":"function"},deleteEdge:{"boolean":o,"function":"function"},controlNodeStyle:"get from nodes, will be overwritten below",__type__:{object:r,"boolean":o}},nodes:{borderWidth:{number:n},borderWidthSelected:{number:n,undefined:"undefined"},brokenImage:{string:i,undefined:"undefined"},color:{border:{string:i},background:{string:i},highlight:{border:{string:i},background:{string:i},__type__:{object:r,string:i}},hover:{border:{string:i},background:{string:i},__type__:{object:r,string:i}},__type__:{object:r,string:i}},fixed:{x:{"boolean":o},y:{"boolean":o},__type__:{object:r,"boolean":o}},font:{align:{string:i},color:{string:i},size:{number:n},face:{string:i},background:{string:i},strokeWidth:{number:n},strokeColor:{string:i},__type__:{object:r,string:i}},group:{string:i,number:n,undefined:"undefined"},hidden:{"boolean":o},icon:{face:{string:i},code:{string:i},size:{number:n},color:{string:i},__type__:{object:r}},id:{string:i,number:n},image:{string:i,undefined:"undefined"},label:{string:i,undefined:"undefined"},labelHighlightBold:{"boolean":o},level:{number:n,undefined:"undefined"},mass:{number:n},physics:{"boolean":o},scaling:{min:{number:n},max:{number:n},label:{enabled:{"boolean":o},min:{number:n},max:{number:n},maxVisible:{number:n},drawThreshold:{number:n},__type__:{object:r,"boolean":o}},customScalingFunction:{"function":"function"},__type__:{object:r}},shadow:{enabled:{"boolean":o},color:{string:i},size:{number:n},x:{number:n},y:{number:n},__type__:{object:r,"boolean":o}},shape:{string:["ellipse","circle","database","box","text","image","circularImage","diamond","dot","star","triangle","triangleDown","square","icon"]},shapeProperties:{borderDashes:{"boolean":o,array:s},borderRadius:{number:n},interpolation:{"boolean":o},useImageSize:{"boolean":o},useBorderWithImage:{"boolean":o},__type__:{object:r}},size:{number:n},title:{string:i,undefined:"undefined"},value:{number:n,undefined:"undefined"},x:{number:n},y:{number:n},__type__:{object:r}},physics:{enabled:{"boolean":o},barnesHut:{gravitationalConstant:{number:n},centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},damping:{number:n},avoidOverlap:{number:n},__type__:{object:r}},forceAtlas2Based:{gravitationalConstant:{number:n},centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},damping:{number:n},avoidOverlap:{number:n},__type__:{object:r}},repulsion:{centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},nodeDistance:{number:n},damping:{number:n},__type__:{object:r}},hierarchicalRepulsion:{centralGravity:{number:n},springLength:{number:n},springConstant:{number:n},nodeDistance:{number:n},damping:{number:n},__type__:{object:r}},maxVelocity:{number:n},minVelocity:{number:n},solver:{string:["barnesHut","repulsion","hierarchicalRepulsion","forceAtlas2Based"]},stabilization:{enabled:{"boolean":o},iterations:{number:n},updateInterval:{number:n},onlyDynamicEdges:{"boolean":o},fit:{"boolean":o},__type__:{object:r,"boolean":o}},timestep:{number:n},adaptiveTimestep:{"boolean":o},__type__:{object:r,"boolean":o}},autoResize:{"boolean":o},clickToUse:{"boolean":o},locale:{string:i},locales:{__any__:{any:h},__type__:{object:r}},height:{string:i},width:{string:i},__type__:{object:r}};d.groups.__any__=d.nodes,d.manipulation.controlNodeStyle=d.nodes;var l={nodes:{borderWidth:[1,0,10,1],borderWidthSelected:[2,0,10,1],color:{border:["color","#2B7CE9"],background:["color","#97C2FC"],highlight:{border:["color","#2B7CE9"],background:["color","#D2E5FF"]},hover:{border:["color","#2B7CE9"],background:["color","#D2E5FF"]}},fixed:{x:!1,y:!1},font:{color:["color","#343434"],size:[14,0,100,1],face:["arial","verdana","tahoma"],background:["color","none"],strokeWidth:[0,0,50,1],strokeColor:["color","#ffffff"]},hidden:!1,labelHighlightBold:!0,physics:!0,scaling:{min:[10,0,200,1],max:[30,0,200,1],label:{enabled:!1,min:[14,0,200,1],max:[30,0,200,1],maxVisible:[30,0,200,1],drawThreshold:[5,0,20,1]}},shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:[10,0,20,1],x:[5,-30,30,1],y:[5,-30,30,1]},shape:["ellipse","box","circle","database","diamond","dot","square","star","text","triangle","triangleDown"],shapeProperties:{borderDashes:!1,borderRadius:[6,0,20,1],interpolation:!0,useImageSize:!1},size:[25,0,200,1]},edges:{arrows:{to:{enabled:!1,scaleFactor:[1,0,3,.05]},middle:{enabled:!1,scaleFactor:[1,0,3,.05]},from:{enabled:!1,scaleFactor:[1,0,3,.05]}},arrowStrikethrough:!0,color:{color:["color","#848484"],highlight:["color","#848484"],hover:["color","#848484"],inherit:["from","to","both",!0,!1],opacity:[1,0,1,.05]},dashes:!1,font:{color:["color","#343434"],size:[14,0,100,1],face:["arial","verdana","tahoma"],background:["color","none"],strokeWidth:[2,0,50,1],strokeColor:["color","#ffffff"],align:["horizontal","top","middle","bottom"]},hidden:!1,hoverWidth:[1.5,0,5,.1],labelHighlightBold:!0,physics:!0,scaling:{min:[1,0,100,1],max:[15,0,100,1],label:{enabled:!0,min:[14,0,200,1],max:[30,0,200,1],maxVisible:[30,0,200,1],drawThreshold:[5,0,20,1]}},selectionWidth:[1.5,0,5,.1],selfReferenceSize:[20,0,200,1],shadow:{enabled:!1,color:"rgba(0,0,0,0.5)",size:[10,0,20,1],x:[5,-30,30,1],y:[5,-30,30,1]},smooth:{enabled:!0,type:["dynamic","continuous","discrete","diagonalCross","straightCross","horizontal","vertical","curvedCW","curvedCCW","cubicBezier"],forceDirection:["horizontal","vertical","none"],roundness:[.5,0,1,.05]},width:[1,0,30,1]},layout:{hierarchical:{enabled:!1,levelSeparation:[150,20,500,5],nodeSpacing:[100,20,500,5],treeSpacing:[200,20,500,5],blockShifting:!0,edgeMinimization:!0,parentCentralization:!0,direction:["UD","DU","LR","RL"],sortMethod:["hubsize","directed"]}},interaction:{dragNodes:!0,dragView:!0,hideEdgesOnDrag:!1,hideNodesOnDrag:!1,hover:!1,keyboard:{enabled:!1,speed:{x:[10,0,40,1],y:[10,0,40,1],zoom:[.02,0,.1,.005]},bindToWindow:!0},multiselect:!1,navigationButtons:!1,selectable:!0,selectConnectedEdges:!0,hoverConnectedEdges:!0,tooltipDelay:[300,0,1e3,25],zoomView:!0},manipulation:{enabled:!1,initiallyActive:!1},physics:{enabled:!0,barnesHut:{gravitationalConstant:[-2e3,-3e4,0,50],centralGravity:[.3,0,10,.05],springLength:[95,0,500,5],springConstant:[.04,0,1.2,.005],damping:[.09,0,1,.01],avoidOverlap:[0,0,1,.01]},forceAtlas2Based:{gravitationalConstant:[-50,-500,0,1],centralGravity:[.01,0,1,.005],springLength:[95,0,500,5],springConstant:[.08,0,1.2,.005],damping:[.4,0,1,.01],avoidOverlap:[0,0,1,.01]},repulsion:{centralGravity:[.2,0,10,.05],springLength:[200,0,500,5],springConstant:[.05,0,1.2,.005],nodeDistance:[100,0,500,5],damping:[.09,0,1,.01]},hierarchicalRepulsion:{centralGravity:[.2,0,10,.05],springLength:[100,0,500,5],springConstant:[.01,0,1.2,.005],nodeDistance:[120,0,500,5],damping:[.09,0,1,.01]},maxVelocity:[50,0,150,1],minVelocity:[.1,.01,.5,.01],solver:["barnesHut","forceAtlas2Based","repulsion","hierarchicalRepulsion"],timestep:[.5,.01,1,.01]},global:{locale:["en","nl"]}};e.allOptions=d,e.configureOptions=l},function(t,e,i){function o(t){return t&&t.__esModule?t:{"default":t}}function n(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var s=function(){function t(t,e){var i=[],o=!0,n=!1,s=void 0;try{for(var r,a=t[Symbol.iterator]();!(o=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);o=!0);}catch(h){n=!0,s=h}finally{try{!o&&a["return"]&&a["return"]()}finally{if(n)throw s}}return i}return function(e,i){if(Array.isArray(e))return e;if(Symbol.iterator in Object(e))return t(e,i);throw new TypeError("Invalid attempt to destructure non-iterable instance")}}(),r=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),a=i(116),h=o(a),d=function(){function t(e,i,o){n(this,t),this.body=e,this.springLength=i,this.springConstant=o,this.distanceSolver=new h["default"]}return r(t,[{key:"setOptions",value:function(t){t&&(t.springLength&&(this.springLength=t.springLength),t.springConstant&&(this.springConstant=t.springConstant))}},{key:"solve",value:function(t,e){var i=arguments.length<=2||void 0===arguments[2]?!1:arguments[2],o=this.distanceSolver.getDistances(this.body,t,e);this._createL_matrix(o),this._createK_matrix(o);for(var n=.01,r=1,a=0,h=Math.max(1e3,Math.min(10*this.body.nodeIndices.length,6e3)),d=5,l=1e9,c=0,u=0,p=0,f=0,m=0;l>n&&h>a;){a+=1;var v=this._getHighestEnergyNode(i),g=s(v,4);for(c=g[0],l=g[1],u=g[2],p=g[3],f=l,m=0;f>r&&d>m;){m+=1,this._moveNode(c,u,p);var y=this._getEnergy(c),b=s(y,3);f=b[0],u=b[1],p=b[2]}}}},{key:"_getHighestEnergyNode",value:function(t){for(var e=this.body.nodeIndices,i=this.body.nodes,o=0,n=e[0],r=0,a=0,h=0;h<e.length;h++){var d=e[h];if(i[d].predefinedPosition===!1||i[d].isCluster===!0&&t===!0||i[d].options.fixed.x===!0||i[d].options.fixed.y===!0){var l=this._getEnergy(d),c=s(l,3),u=c[0],p=c[1],f=c[2];u>o&&(o=u,n=d,r=p,a=f)}}return[n,o,r,a]}},{key:"_getEnergy",value:function(t){for(var e=this.body.nodeIndices,i=this.body.nodes,o=i[t].x,n=i[t].y,s=0,r=0,a=0;a<e.length;a++){var h=e[a];if(h!==t){var d=i[h].x,l=i[h].y,c=1/Math.sqrt(Math.pow(o-d,2)+Math.pow(n-l,2));s+=this.K_matrix[t][h]*(o-d-this.L_matrix[t][h]*(o-d)*c),r+=this.K_matrix[t][h]*(n-l-this.L_matrix[t][h]*(n-l)*c)}}var u=Math.sqrt(Math.pow(s,2)+Math.pow(r,2));return[u,s,r]}},{key:"_moveNode",value:function(t,e,i){for(var o=this.body.nodeIndices,n=this.body.nodes,s=0,r=0,a=0,h=n[t].x,d=n[t].y,l=0;l<o.length;l++){var c=o[l];if(c!==t){var u=n[c].x,p=n[c].y,f=1/Math.pow(Math.pow(h-u,2)+Math.pow(d-p,2),1.5);s+=this.K_matrix[t][c]*(1-this.L_matrix[t][c]*Math.pow(d-p,2)*f),r+=this.K_matrix[t][c]*(this.L_matrix[t][c]*(h-u)*(d-p)*f),a+=this.K_matrix[t][c]*(1-this.L_matrix[t][c]*Math.pow(h-u,2)*f)}}var m=s,v=r,g=e,y=a,b=i,w=(g/m+b/v)/(v/m-y/v),_=-(v*w+g)/m;n[t].x+=_,n[t].y+=w}},{key:"_createL_matrix",value:function(t){var e=this.body.nodeIndices,i=this.springLength;this.L_matrix=[];for(var o=0;o<e.length;o++){this.L_matrix[e[o]]={};for(var n=0;n<e.length;n++)this.L_matrix[e[o]][e[n]]=i*t[e[o]][e[n]]}}},{key:"_createK_matrix",value:function(t){var e=this.body.nodeIndices,i=this.springConstant;this.K_matrix=[];for(var o=0;o<e.length;o++){this.K_matrix[e[o]]={};for(var n=0;n<e.length;n++)this.K_matrix[e[o]][e[n]]=i*Math.pow(t[e[o]][e[n]],-2)}}}]),t}();e["default"]=d},function(t,e){function i(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(e,"__esModule",{value:!0});var o=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),Object.defineProperty(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}(),n=function(){function t(){i(this,t)}return o(t,[{key:"getDistances",value:function(t,e,i){for(var o={},n=t.edges,s=0;s<e.length;s++){o[e[s]]={},o[e[s]]={};for(var r=0;r<e.length;r++)o[e[s]][e[r]]=s==r?0:1e9,o[e[s]][e[r]]=s==r?0:1e9}for(var a=0;a<i.length;a++){var h=n[i[a]];h.connected===!0&&void 0!==o[h.fromId]&&void 0!==o[h.toId]&&(o[h.fromId][h.toId]=1,o[h.toId][h.fromId]=1)}for(var d=e.length,l=0;d>l;l++)for(var c=0;d-1>c;c++)for(var u=c+1;d>u;u++)o[e[c]][e[u]]=Math.min(o[e[c]][e[u]],o[e[c]][e[l]]+o[e[l]][e[u]]),o[e[u]][e[c]]=o[e[c]][e[u]];return o}}]),t}();e["default"]=n},function(t,e){"undefined"!=typeof CanvasRenderingContext2D&&(CanvasRenderingContext2D.prototype.circle=function(t,e,i){this.beginPath(),this.arc(t,e,i,0,2*Math.PI,!1),this.closePath()},CanvasRenderingContext2D.prototype.square=function(t,e,i){this.beginPath(),this.rect(t-i,e-i,2*i,2*i),this.closePath()},CanvasRenderingContext2D.prototype.triangle=function(t,e,i){this.beginPath(),i*=1.15,e+=.275*i;var o=2*i,n=o/2,s=Math.sqrt(3)/6*o,r=Math.sqrt(o*o-n*n);this.moveTo(t,e-(r-s)),this.lineTo(t+n,e+s),this.lineTo(t-n,e+s),this.lineTo(t,e-(r-s)),this.closePath()},CanvasRenderingContext2D.prototype.triangleDown=function(t,e,i){this.beginPath(),i*=1.15,e-=.275*i;var o=2*i,n=o/2,s=Math.sqrt(3)/6*o,r=Math.sqrt(o*o-n*n);this.moveTo(t,e+(r-s)),
-this.lineTo(t+n,e-s),this.lineTo(t-n,e-s),this.lineTo(t,e+(r-s)),this.closePath()},CanvasRenderingContext2D.prototype.star=function(t,e,i){this.beginPath(),i*=.82,e+=.1*i;for(var o=0;10>o;o++){var n=o%2===0?1.3*i:.5*i;this.lineTo(t+n*Math.sin(2*o*Math.PI/10),e-n*Math.cos(2*o*Math.PI/10))}this.closePath()},CanvasRenderingContext2D.prototype.diamond=function(t,e,i){this.beginPath(),this.lineTo(t,e+i),this.lineTo(t+i,e),this.lineTo(t,e-i),this.lineTo(t-i,e),this.closePath()},CanvasRenderingContext2D.prototype.roundRect=function(t,e,i,o,n){var s=Math.PI/180;0>i-2*n&&(n=i/2),0>o-2*n&&(n=o/2),this.beginPath(),this.moveTo(t+n,e),this.lineTo(t+i-n,e),this.arc(t+i-n,e+n,n,270*s,360*s,!1),this.lineTo(t+i,e+o-n),this.arc(t+i-n,e+o-n,n,0,90*s,!1),this.lineTo(t+n,e+o),this.arc(t+n,e+o-n,n,90*s,180*s,!1),this.lineTo(t,e+n),this.arc(t+n,e+n,n,180*s,270*s,!1),this.closePath()},CanvasRenderingContext2D.prototype.ellipse=function(t,e,i,o){var n=.5522848,s=i/2*n,r=o/2*n,a=t+i,h=e+o,d=t+i/2,l=e+o/2;this.beginPath(),this.moveTo(t,l),this.bezierCurveTo(t,l-r,d-s,e,d,e),this.bezierCurveTo(d+s,e,a,l-r,a,l),this.bezierCurveTo(a,l+r,d+s,h,d,h),this.bezierCurveTo(d-s,h,t,l+r,t,l),this.closePath()},CanvasRenderingContext2D.prototype.database=function(t,e,i,o){var n=1/3,s=i,r=o*n,a=.5522848,h=s/2*a,d=r/2*a,l=t+s,c=e+r,u=t+s/2,p=e+r/2,f=e+(o-r/2),m=e+o;this.beginPath(),this.moveTo(l,p),this.bezierCurveTo(l,p+d,u+h,c,u,c),this.bezierCurveTo(u-h,c,t,p+d,t,p),this.bezierCurveTo(t,p-d,u-h,e,u,e),this.bezierCurveTo(u+h,e,l,p-d,l,p),this.lineTo(l,f),this.bezierCurveTo(l,f+d,u+h,m,u,m),this.bezierCurveTo(u-h,m,t,f+d,t,f),this.lineTo(t,p)},CanvasRenderingContext2D.prototype.arrow=function(t,e,i,o){var n=t-o*Math.cos(i),s=e-o*Math.sin(i),r=t-.9*o*Math.cos(i),a=e-.9*o*Math.sin(i),h=n+o/3*Math.cos(i+.5*Math.PI),d=s+o/3*Math.sin(i+.5*Math.PI),l=n+o/3*Math.cos(i-.5*Math.PI),c=s+o/3*Math.sin(i-.5*Math.PI);this.beginPath(),this.moveTo(t,e),this.lineTo(h,d),this.lineTo(r,a),this.lineTo(l,c),this.closePath()},CanvasRenderingContext2D.prototype.dashedLine=function(t,e,i,o,n){this.beginPath(),this.moveTo(t,e);for(var s=n.length,r=i-t,a=o-e,h=a/r,d=Math.sqrt(r*r+a*a),l=0,c=!0,u=0,p=n[0];d>=.1;)p=n[l++%s],p>d&&(p=d),u=Math.sqrt(p*p/(1+h*h)),u=0>r?-u:u,t+=u,e+=h*u,c===!0?this.lineTo(t,e):this.moveTo(t,e),d-=p,c=!c})},function(t,e){function i(t){return P=t,p()}function o(){I=0,N=P.charAt(0)}function n(){I++,N=P.charAt(I)}function s(){return P.charAt(I+1)}function r(t){return L.test(t)}function a(t,e){if(t||(t={}),e)for(var i in e)e.hasOwnProperty(i)&&(t[i]=e[i]);return t}function h(t,e,i){for(var o=e.split("."),n=t;o.length;){var s=o.shift();o.length?(n[s]||(n[s]={}),n=n[s]):n[s]=i}}function d(t,e){for(var i,o,n=null,s=[t],r=t;r.parent;)s.push(r.parent),r=r.parent;if(r.nodes)for(i=0,o=r.nodes.length;o>i;i++)if(e.id===r.nodes[i].id){n=r.nodes[i];break}for(n||(n={id:e.id},t.node&&(n.attr=a(n.attr,t.node))),i=s.length-1;i>=0;i--){var h=s[i];h.nodes||(h.nodes=[]),-1===h.nodes.indexOf(n)&&h.nodes.push(n)}e.attr&&(n.attr=a(n.attr,e.attr))}function l(t,e){if(t.edges||(t.edges=[]),t.edges.push(e),t.edge){var i=a({},t.edge);e.attr=a(i,e.attr)}}function c(t,e,i,o,n){var s={from:e,to:i,type:o};return t.edge&&(s.attr=a({},t.edge)),s.attr=a(s.attr||{},n),s}function u(){for(z=T.NULL,R="";" "===N||"	"===N||"\n"===N||"\r"===N;)n();do{var t=!1;if("#"===N){for(var e=I-1;" "===P.charAt(e)||"	"===P.charAt(e);)e--;if("\n"===P.charAt(e)||""===P.charAt(e)){for(;""!=N&&"\n"!=N;)n();t=!0}}if("/"===N&&"/"===s()){for(;""!=N&&"\n"!=N;)n();t=!0}if("/"===N&&"*"===s()){for(;""!=N;){if("*"===N&&"/"===s()){n(),n();break}n()}t=!0}for(;" "===N||"	"===N||"\n"===N||"\r"===N;)n()}while(t);if(""===N)return void(z=T.DELIMITER);var i=N+s();if(E[i])return z=T.DELIMITER,R=i,n(),void n();if(E[N])return z=T.DELIMITER,R=N,void n();if(r(N)||"-"===N){for(R+=N,n();r(N);)R+=N,n();return"false"===R?R=!1:"true"===R?R=!0:isNaN(Number(R))||(R=Number(R)),void(z=T.IDENTIFIER)}if('"'===N){for(n();""!=N&&('"'!=N||'"'===N&&'"'===s());)R+=N,'"'===N&&n(),n();if('"'!=N)throw _('End of string " expected');return n(),void(z=T.IDENTIFIER)}for(z=T.UNKNOWN;""!=N;)R+=N,n();throw new SyntaxError('Syntax error in part "'+x(R,30)+'"')}function p(){var t={};if(o(),u(),"strict"===R&&(t.strict=!0,u()),"graph"!==R&&"digraph"!==R||(t.type=R,u()),z===T.IDENTIFIER&&(t.id=R,u()),"{"!=R)throw _("Angle bracket { expected");if(u(),f(t),"}"!=R)throw _("Angle bracket } expected");if(u(),""!==R)throw _("End of file expected");return u(),delete t.node,delete t.edge,delete t.graph,t}function f(t){for(;""!==R&&"}"!=R;)m(t),";"===R&&u()}function m(t){var e=v(t);if(e)return void b(t,e);var i=g(t);if(!i){if(z!=T.IDENTIFIER)throw _("Identifier expected");var o=R;if(u(),"="===R){if(u(),z!=T.IDENTIFIER)throw _("Identifier expected");t[o]=R,u()}else y(t,o)}}function v(t){var e=null;if("subgraph"===R&&(e={},e.type="subgraph",u(),z===T.IDENTIFIER&&(e.id=R,u())),"{"===R){if(u(),e||(e={}),e.parent=t,e.node=t.node,e.edge=t.edge,e.graph=t.graph,f(e),"}"!=R)throw _("Angle bracket } expected");u(),delete e.node,delete e.edge,delete e.graph,delete e.parent,t.subgraphs||(t.subgraphs=[]),t.subgraphs.push(e)}return e}function g(t){return"node"===R?(u(),t.node=w(),"node"):"edge"===R?(u(),t.edge=w(),"edge"):"graph"===R?(u(),t.graph=w(),"graph"):null}function y(t,e){var i={id:e},o=w();o&&(i.attr=o),d(t,i),b(t,e)}function b(t,e){for(;"->"===R||"--"===R;){var i,o=R;u();var n=v(t);if(n)i=n;else{if(z!=T.IDENTIFIER)throw _("Identifier or subgraph expected");i=R,d(t,{id:i}),u()}var s=w(),r=c(t,e,i,o,s);l(t,r),e=i}}function w(){for(var t=null;"["===R;){for(u(),t={};""!==R&&"]"!=R;){if(z!=T.IDENTIFIER)throw _("Attribute name expected");var e=R;if(u(),"="!=R)throw _("Equal sign = expected");if(u(),z!=T.IDENTIFIER)throw _("Attribute value expected");var i=R;h(t,e,i),u(),","==R&&u()}if("]"!=R)throw _("Bracket ] expected");u()}return t}function _(t){return new SyntaxError(t+', got "'+x(R,30)+'" (char '+I+")")}function x(t,e){return t.length<=e?t:t.substr(0,27)+"..."}function k(t,e,i){Array.isArray(t)?t.forEach(function(t){Array.isArray(e)?e.forEach(function(e){i(t,e)}):i(t,e)}):Array.isArray(e)?e.forEach(function(e){i(t,e)}):i(t,e)}function O(t,e,i){for(var o=e.split("."),n=o.pop(),s=t,r=0;r<o.length;r++){var a=o[r];a in s||(s[a]={}),s=s[a]}return s[n]=i,t}function M(t,e){var i={};for(var o in t)if(t.hasOwnProperty(o)){var n=e[o];Array.isArray(n)?n.forEach(function(e){O(i,e,t[o])}):"string"==typeof n?O(i,n,t[o]):O(i,o,t[o])}return i}function D(t){var e=i(t),o={nodes:[],edges:[],options:{}};if(e.nodes&&e.nodes.forEach(function(t){var e={id:t.id,label:String(t.label||t.id)};a(e,M(t.attr,S)),e.image&&(e.shape="image"),o.nodes.push(e)}),e.edges){var n=function(t){var e={from:t.from,to:t.to};return a(e,M(t.attr,C)),e.arrows="->"===t.type?"to":void 0,e};e.edges.forEach(function(t){var e,i;e=t.from instanceof Object?t.from.nodes:{id:t.from},i=t.to instanceof Object?t.to.nodes:{id:t.to},t.from instanceof Object&&t.from.edges&&t.from.edges.forEach(function(t){var e=n(t);o.edges.push(e)}),k(e,i,function(e,i){var s=c(o,e.id,i.id,t.type,t.attr),r=n(s);o.edges.push(r)}),t.to instanceof Object&&t.to.edges&&t.to.edges.forEach(function(t){var e=n(t);o.edges.push(e)})})}return e.attr&&(o.options=e.attr),o}var S={fontsize:"font.size",fontcolor:"font.color",labelfontcolor:"font.color",fontname:"font.face",color:["color.border","color.background"],fillcolor:"color.background",tooltip:"title",labeltooltip:"title"},C=Object.create(S);C.color="color.color";var T={NULL:0,DELIMITER:1,IDENTIFIER:2,UNKNOWN:3},E={"{":!0,"}":!0,"[":!0,"]":!0,";":!0,"=":!0,",":!0,"->":!0,"--":!0},P="",I=0,N="",R="",z=T.NULL,L=/[a-zA-Z_0-9.:#]/;e.parseDOT=i,e.DOTToGraph=D},function(t,e){function i(t,e){var i=[],o=[],n={edges:{inheritColor:!1},nodes:{fixed:!1,parseColor:!1}};void 0!==e&&(void 0!==e.fixed&&(n.nodes.fixed=e.fixed),void 0!==e.parseColor&&(n.nodes.parseColor=e.parseColor),void 0!==e.inheritColor&&(n.edges.inheritColor=e.inheritColor));for(var s=t.edges,r=t.nodes,a=0;a<s.length;a++){var h={},d=s[a];h.id=d.id,h.from=d.source,h.to=d.target,h.attributes=d.attributes,h.label=d.label,h.title=void 0!==d.attributes?d.attributes.title:void 0,"Directed"===d.type&&(h.arrows="to"),d.color&&n.inheritColor===!1&&(h.color=d.color),i.push(h)}for(var a=0;a<r.length;a++){var l={},c=r[a];l.id=c.id,l.attributes=c.attributes,l.title=c.title,l.x=c.x,l.y=c.y,l.label=c.label,l.title=void 0!==c.attributes?c.attributes.title:void 0,n.nodes.parseColor===!0?l.color=c.color:l.color=void 0!==c.color?{background:c.color,border:c.color,highlight:{background:c.color,border:c.color},hover:{background:c.color,border:c.color}}:void 0,l.size=c.size,l.fixed=n.nodes.fixed&&void 0!==c.x&&void 0!==c.y,o.push(l)}return{nodes:o,edges:i}}e.parseGephi=i},function(t,e){e.en={edit:"Edit",del:"Delete selected",back:"Back",addNode:"Add Node",addEdge:"Add Edge",editNode:"Edit Node",editEdge:"Edit Edge",addDescription:"Click in an empty space to place a new node.",edgeDescription:"Click on a node and drag the edge to another node to connect them.",editEdgeDescription:"Click on the control points and drag them to a node to connect to it.",createEdgeError:"Cannot link edges to a cluster.",deleteClusterError:"Clusters cannot be deleted.",editClusterError:"Clusters cannot be edited."},e.en_EN=e.en,e.en_US=e.en,e.de={edit:"Editieren",del:"Lösche Auswahl",back:"Zurück",addNode:"Knoten hinzufügen",addEdge:"Kante hinzufügen",editNode:"Knoten editieren",editEdge:"Kante editieren",addDescription:"Klicke auf eine freie Stelle, um einen neuen Knoten zu plazieren.",edgeDescription:"Klicke auf einen Knoten und ziehe die Kante zu einem anderen Knoten, um diese zu verbinden.",editEdgeDescription:"Klicke auf die Verbindungspunkte und ziehe diese auf einen Knoten, um sie zu verbinden.",createEdgeError:"Es ist nicht möglich, Kanten mit Clustern zu verbinden.",deleteClusterError:"Cluster können nicht gelöscht werden.",editClusterError:"Cluster können nicht editiert werden."},e.de_DE=e.de,e.es={edit:"Editar",del:"Eliminar selección",back:"Átras",addNode:"Añadir nodo",addEdge:"Añadir arista",editNode:"Editar nodo",editEdge:"Editar arista",addDescription:"Haga clic en un lugar vacío para colocar un nuevo nodo.",edgeDescription:"Haga clic en un nodo y arrastre la arista hacia otro nodo para conectarlos.",editEdgeDescription:"Haga clic en un punto de control y arrastrelo a un nodo para conectarlo.",createEdgeError:"No se puede conectar una arista a un grupo.",deleteClusterError:"No es posible eliminar grupos.",editClusterError:"No es posible editar grupos."},e.es_ES=e.es,e.nl={edit:"Wijzigen",del:"Selectie verwijderen",back:"Terug",addNode:"Node toevoegen",addEdge:"Link toevoegen",editNode:"Node wijzigen",editEdge:"Link wijzigen",addDescription:"Klik op een leeg gebied om een nieuwe node te maken.",edgeDescription:"Klik op een node en sleep de link naar een andere node om ze te verbinden.",editEdgeDescription:"Klik op de verbindingspunten en sleep ze naar een node om daarmee te verbinden.",createEdgeError:"Kan geen link maken naar een cluster.",deleteClusterError:"Clusters kunnen niet worden verwijderd.",editClusterError:"Clusters kunnen niet worden aangepast."},e.nl_NL=e.nl,e.nl_BE=e.nl}])});
-//# sourceMappingURL=vis.map
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 94c45215b5ff2..aaaa21acfec17 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -224,13 +224,14 @@ private[spark] object UIUtils extends Logging {
     <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
     <link rel="stylesheet"
           href={prependBaseUri(request, "/static/bootstrap.min.css")} type="text/css"/>
-    <link rel="stylesheet" href={prependBaseUri(request, "/static/vis.min.css")} type="text/css"/>
+    <link rel="stylesheet"
+          href={prependBaseUri(request, "/static/vis-timeline-graph2d.min.css")} type="text/css"/>
     <link rel="stylesheet" href={prependBaseUri(request, "/static/webui.css")} type="text/css"/>
     <link rel="stylesheet"
           href={prependBaseUri(request, "/static/timeline-view.css")} type="text/css"/>
     <script src={prependBaseUri(request, "/static/sorttable.js")} ></script>
     <script src={prependBaseUri(request, "/static/jquery-3.4.1.min.js")}></script>
-    <script src={prependBaseUri(request, "/static/vis.min.js")}></script>
+    <script src={prependBaseUri(request, "/static/vis-timeline-graph2d.min.js")}></script>
     <script src={prependBaseUri(request, "/static/bootstrap-tooltip.js")}></script>
     <script src={prependBaseUri(request, "/static/initialize-tooltips.js")}></script>
     <script src={prependBaseUri(request, "/static/table.js")}></script>
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 9f0cdeac9ca39..757e03b2b50e5 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -47,7 +47,7 @@ import org.apache.spark.status.api.v1.{JacksonMessageWriter, RDDDataDistribution
 
 private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler {
 
-  private val cssWhiteList = List("bootstrap.min.css", "vis.min.css")
+  private val cssWhiteList = List("bootstrap.min.css", "vis-timeline-graph2d.min.css")
 
   private def isInWhileList(uri: String): Boolean = cssWhiteList.exists(uri.endsWith)
 
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 73f461255de43..76934210aa79e 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -27,8 +27,10 @@ d3.min.js
 dagre-d3.min.js
 graphlib-dot.min.js
 sorttable.js
-vis.min.js
-vis.min.css
+vis-timeline-graph2d.min.js
+vis-timeline-graph2d.min.js.map
+vis-timeline-graph2d.min.css
+vis-timeline-graph2d.min.css.map
 dataTables.bootstrap.css
 dataTables.bootstrap.min.js
 dataTables.rowsGroup.js
diff --git a/licenses-binary/LICENSE-vis-timeline.txt b/licenses-binary/LICENSE-vis-timeline.txt
new file mode 100644
index 0000000000000..d343783c16791
--- /dev/null
+++ b/licenses-binary/LICENSE-vis-timeline.txt
@@ -0,0 +1,23 @@
+vis-timeline and vis-graph2d
+https://visjs.github.io/vis-timeline/
+
+Create a fully customizable, interactive timeline with items and ranges.
+
+@version 7.3.4
+@date    2020-03-18T17:03:58.105Z
+
+@copyright (c) 2011-2017 Almende B.V, http://almende.com
+@copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+
+@license
+vis.js is dual licensed under both
+
+  1. The Apache 2.0 License
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  and
+
+  2. The MIT License
+     http://opensource.org/licenses/MIT
+
+vis.js may be distributed under either license.
diff --git a/licenses-binary/LICENSE-vis.txt b/licenses-binary/LICENSE-vis.txt
deleted file mode 100644
index 18b7323059a41..0000000000000
--- a/licenses-binary/LICENSE-vis.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-vis.js
-https://github.com/almende/vis
-
-A dynamic, browser-based visualization library.
-
-@version 4.16.1
-@date    2016-04-18
-
-@license
-Copyright (C) 2011-2016 Almende B.V, http://almende.com
-
-Vis.js is dual licensed under both
-
-* The Apache 2.0 License
-  http://www.apache.org/licenses/LICENSE-2.0
-
-and
-
-* The MIT License
-  http://opensource.org/licenses/MIT
-
-Vis.js may be distributed under either license.
\ No newline at end of file
diff --git a/licenses/LICENSE-vis-timeline.txt b/licenses/LICENSE-vis-timeline.txt
new file mode 100644
index 0000000000000..d343783c16791
--- /dev/null
+++ b/licenses/LICENSE-vis-timeline.txt
@@ -0,0 +1,23 @@
+vis-timeline and vis-graph2d
+https://visjs.github.io/vis-timeline/
+
+Create a fully customizable, interactive timeline with items and ranges.
+
+@version 7.3.4
+@date    2020-03-18T17:03:58.105Z
+
+@copyright (c) 2011-2017 Almende B.V, http://almende.com
+@copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+
+@license
+vis.js is dual licensed under both
+
+  1. The Apache 2.0 License
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  and
+
+  2. The MIT License
+     http://opensource.org/licenses/MIT
+
+vis.js may be distributed under either license.
diff --git a/licenses/LICENSE-vis.txt b/licenses/LICENSE-vis.txt
deleted file mode 100644
index 18b7323059a41..0000000000000
--- a/licenses/LICENSE-vis.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-vis.js
-https://github.com/almende/vis
-
-A dynamic, browser-based visualization library.
-
-@version 4.16.1
-@date    2016-04-18
-
-@license
-Copyright (C) 2011-2016 Almende B.V, http://almende.com
-
-Vis.js is dual licensed under both
-
-* The Apache 2.0 License
-  http://www.apache.org/licenses/LICENSE-2.0
-
-and
-
-* The MIT License
-  http://opensource.org/licenses/MIT
-
-Vis.js may be distributed under either license.
\ No newline at end of file

From a5a41942138c06cc67ffd1183bbf6da5214f78f0 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 15 Apr 2020 06:19:12 +0000
Subject: [PATCH 0501/1280] [SPARK-31443][SQL] Fix perf regression of
 toJavaDate

### What changes were proposed in this pull request?
Optimise the `toJavaDate()` method of `DateTimeUtils` by:
1. Re-using `rebaseGregorianToJulianDays` optimised by #28067
2. Creating `java.sql.Date` instances from milliseconds in UTC since the epoch instead of date-time fields. This allows to avoid "normalization" inside of  `java.sql.Date`.

Also new benchmark for collecting dates is added to `DateTimeBenchmark`.

### Why are the changes needed?
The changes fix the performance regression of collecting `DATE` values comparing to Spark 2.4 (see `DateTimeBenchmark` in https://github.com/MaxGekk/spark/pull/27):

Spark 2.4.6-SNAPSHOT:
```
To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Date                                  559            603          38          8.9         111.8       1.0X
Collect dates                                      2306           3221        1558          2.2         461.1       0.2X
```
Before the changes:
```
To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Date                                 1052           1130          73          4.8         210.3       1.0X
Collect dates                                      3251           4943        1624          1.5         650.2       0.3X
```
After:
```
To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Date                                  416            419           3         12.0          83.2       1.0X
Collect dates                                      1928           2759        1180          2.6         385.6       0.2X
```

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
- By existing tests suites, in particular, `DateTimeUtilsSuite`, `RebaseDateTimeSuite`, `DateFunctionsSuite`, `DateExpressionsSuite`.
- Re-run `DateTimeBenchmark` in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

Closes #28212 from MaxGekk/optimize-toJavaDate.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 744c2480b580e0b68387328926ef7634cfb93adc)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeUtils.scala     |  11 +-
 .../DateTimeBenchmark-jdk11-results.txt       | 221 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 221 +++++++++---------
 .../benchmark/DateTimeBenchmark.scala         |   5 +
 4 files changed, 236 insertions(+), 222 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index dede92f0df518..8486bba4a25ae 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit._
 
 import scala.util.control.NonFatal
 
+import sun.util.calendar.ZoneInfo
+
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.types.Decimal
@@ -123,8 +125,13 @@ object DateTimeUtils {
    * @return A `java.sql.Date` from number of days since epoch.
    */
   def toJavaDate(daysSinceEpoch: SQLDate): Date = {
-    val localDate = LocalDate.ofEpochDay(daysSinceEpoch)
-    new Date(localDate.getYear - 1900, localDate.getMonthValue - 1, localDate.getDayOfMonth)
+    val rebasedDays = rebaseGregorianToJulianDays(daysSinceEpoch)
+    val localMillis = Math.multiplyExact(rebasedDays, MILLIS_PER_DAY)
+    val timeZoneOffset = TimeZone.getDefault match {
+      case zoneInfo: ZoneInfo => zoneInfo.getOffsetsByWall(localMillis, null)
+      case timeZone: TimeZone => timeZone.getOffset(localMillis - timeZone.getRawOffset)
+    }
+    new Date(localMillis - timeZoneOffset)
   }
 
   /**
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 1323a013332d2..0dfd66e4e98d2 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    440            459          28         22.7          44.0       1.0X
-cast to timestamp wholestage on                     384            407          15         26.1          38.4       1.1X
+cast to timestamp wholestage off                    453            469          22         22.1          45.3       1.0X
+cast to timestamp wholestage on                     378            416          44         26.5          37.8       1.2X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1334           1352          27          7.5         133.4       1.0X
-year of timestamp wholestage on                    1301           1309           9          7.7         130.1       1.0X
+year of timestamp wholestage off                   1381           1388           9          7.2         138.1       1.0X
+year of timestamp wholestage on                    1218           1229          14          8.2         121.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1623           1636          18          6.2         162.3       1.0X
-quarter of timestamp wholestage on                 1592           1603           9          6.3         159.2       1.0X
+quarter of timestamp wholestage off                1621           1627           9          6.2         162.1       1.0X
+quarter of timestamp wholestage on                 1593           1596           2          6.3         159.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1246           1250           5          8.0         124.6       1.0X
-month of timestamp wholestage on                   1277           1290          15          7.8         127.7       1.0X
+month of timestamp wholestage off                  1241           1243           2          8.1         124.1       1.0X
+month of timestamp wholestage on                   1261           1277          10          7.9         126.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1881           1896          20          5.3         188.1       1.0X
-weekofyear of timestamp wholestage on              1865           1879          16          5.4         186.5       1.0X
+weekofyear of timestamp wholestage off             2206           2207           1          4.5         220.6       1.0X
+weekofyear of timestamp wholestage on              1848           1863          13          5.4         184.8       1.2X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1223           1249          37          8.2         122.3       1.0X
-day of timestamp wholestage on                     1261           1276          19          7.9         126.1       1.0X
+day of timestamp wholestage off                    1239           1242           4          8.1         123.9       1.0X
+day of timestamp wholestage on                     1258           1274          11          8.0         125.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1291           1312          29          7.7         129.1       1.0X
-dayofyear of timestamp wholestage on               1284           1290           6          7.8         128.4       1.0X
+dayofyear of timestamp wholestage off              1266           1269           3          7.9         126.6       1.0X
+dayofyear of timestamp wholestage on               1294           1305          13          7.7         129.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1241           1246           7          8.1         124.1       1.0X
-dayofmonth of timestamp wholestage on              1268           1276           6          7.9         126.8       1.0X
+dayofmonth of timestamp wholestage off             1244           1245           2          8.0         124.4       1.0X
+dayofmonth of timestamp wholestage on              1253           1265           6          8.0         125.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1402           1408           9          7.1         140.2       1.0X
-dayofweek of timestamp wholestage on               1414           1423           8          7.1         141.4       1.0X
+dayofweek of timestamp wholestage off              1431           1436           7          7.0         143.1       1.0X
+dayofweek of timestamp wholestage on               1412           1424          12          7.1         141.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1349           1350           2          7.4         134.9       1.0X
-weekday of timestamp wholestage on                 1344           1352           5          7.4         134.4       1.0X
+weekday of timestamp wholestage off                1347           1347           0          7.4         134.7       1.0X
+weekday of timestamp wholestage on                 1347           1358          12          7.4         134.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    974            974           0         10.3          97.4       1.0X
-hour of timestamp wholestage on                    1009           1019          15          9.9         100.9       1.0X
+hour of timestamp wholestage off                   1023           1024           1          9.8         102.3       1.0X
+hour of timestamp wholestage on                     980            988           8         10.2          98.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1024           1028           6          9.8         102.4       1.0X
-minute of timestamp wholestage on                   978            992          11         10.2          97.8       1.0X
+minute of timestamp wholestage off                 1030           1034           6          9.7         103.0       1.0X
+minute of timestamp wholestage on                   986            992           6         10.1          98.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1023           1026           4          9.8         102.3       1.0X
-second of timestamp wholestage on                  1008           1024          14          9.9         100.8       1.0X
+second of timestamp wholestage off                 1031           1032           1          9.7         103.1       1.0X
+second of timestamp wholestage on                   985            991          11         10.2          98.5       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         296            299           5         33.8          29.6       1.0X
-current_date wholestage on                          316            318           2         31.7          31.6       0.9X
+current_date wholestage off                         308            311           3         32.5          30.8       1.0X
+current_date wholestage on                          309            314           6         32.4          30.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    310            345          49         32.2          31.0       1.0X
-current_timestamp wholestage on                     316            332          11         31.7          31.6       1.0X
+current_timestamp wholestage off                    312            332          28         32.0          31.2       1.0X
+current_timestamp wholestage on                     312            328          13         32.1          31.2       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1067           1071           7          9.4         106.7       1.0X
-cast to date wholestage on                         1050           1072          20          9.5         105.0       1.0X
+cast to date wholestage off                        1067           1078          17          9.4         106.7       1.0X
+cast to date wholestage on                         1075           1084           8          9.3         107.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1240           1245           8          8.1         124.0       1.0X
-last_day wholestage on                             1273           1290          14          7.9         127.3       1.0X
+last_day wholestage off                            1246           1249           3          8.0         124.6       1.0X
+last_day wholestage on                             1269           1272           3          7.9         126.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1121           1130          13          8.9         112.1       1.0X
-next_day wholestage on                             1099           1106          11          9.1         109.9       1.0X
+next_day wholestage off                            1116           1124          11          9.0         111.6       1.0X
+next_day wholestage on                             1120           1129           7          8.9         112.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1045           1051           8          9.6         104.5       1.0X
-date_add wholestage on                             1078           1086          16          9.3         107.8       1.0X
+date_add wholestage off                            1054           1064          15          9.5         105.4       1.0X
+date_add wholestage on                             1071           1082          17          9.3         107.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1049           1050           1          9.5         104.9       1.0X
-date_sub wholestage on                             1078           1083           4          9.3         107.8       1.0X
+date_sub wholestage off                            1066           1069           4          9.4         106.6       1.0X
+date_sub wholestage on                             1075           1084          14          9.3         107.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1411           1415           7          7.1         141.1       1.0X
-add_months wholestage on                           1435           1447          13          7.0         143.5       1.0X
+add_months wholestage off                          1411           1417           9          7.1         141.1       1.0X
+add_months wholestage on                           1472           1476           5          6.8         147.2       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5736           5786          71          1.7         573.6       1.0X
-format date wholestage on                          5550           5575          29          1.8         555.0       1.0X
+format date wholestage off                         5778           5801          33          1.7         577.8       1.0X
+format date wholestage on                          5718           5726          11          1.7         571.8       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7378           7379           1          1.4         737.8       1.0X
-from_unixtime wholestage on                        7441           7465          24          1.3         744.1       1.0X
+from_unixtime wholestage off                       7739           7744           8          1.3         773.9       1.0X
+from_unixtime wholestage on                        7742           7758          10          1.3         774.2       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1238           1260          30          8.1         123.8       1.0X
-from_utc_timestamp wholestage on                   1300           1308           5          7.7         130.0       1.0X
+from_utc_timestamp wholestage off                  1236           1243          11          8.1         123.6       1.0X
+from_utc_timestamp wholestage on                   1327           1333           8          7.5         132.7       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1790           1793           5          5.6         179.0       1.0X
-to_utc_timestamp wholestage on                     1737           1748          13          5.8         173.7       1.0X
+to_utc_timestamp wholestage off                    1787           1795          11          5.6         178.7       1.0X
+to_utc_timestamp wholestage on                     1791           1794           2          5.6         179.1       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        347            350           4         28.8          34.7       1.0X
-cast interval wholestage on                         327            329           4         30.6          32.7       1.1X
+cast interval wholestage off                        371            372           2         27.0          37.1       1.0X
+cast interval wholestage on                         340            348          10         29.4          34.0       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1836           1860          33          5.4         183.6       1.0X
-datediff wholestage on                             1811           1826          16          5.5         181.1       1.0X
+datediff wholestage off                            1844           1859          22          5.4         184.4       1.0X
+datediff wholestage on                             1813           1822          10          5.5         181.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5613           5617           6          1.8         561.3       1.0X
-months_between wholestage on                       5590           5624          35          1.8         559.0       1.0X
+months_between wholestage off                      5391           5395           5          1.9         539.1       1.0X
+months_between wholestage on                       5353           5374          12          1.9         535.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2222           2308         122          0.5        2222.2       1.0X
-window wholestage on                              48721          48828         101          0.0       48720.7       0.0X
+window wholestage off                              2601           2709         152          0.4        2601.3       1.0X
+window wholestage on                              45767          45831          46          0.0       45767.4       0.1X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2482           2503          30          4.0         248.2       1.0X
-date_trunc YEAR wholestage on                      2520           2532          12          4.0         252.0       1.0X
+date_trunc YEAR wholestage off                     2494           2506          17          4.0         249.4       1.0X
+date_trunc YEAR wholestage on                      2482           2499          12          4.0         248.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2487           2493           9          4.0         248.7       1.0X
-date_trunc YYYY wholestage on                      2521           2527           8          4.0         252.1       1.0X
+date_trunc YYYY wholestage off                     2485           2504          26          4.0         248.5       1.0X
+date_trunc YYYY wholestage on                      2477           2489           9          4.0         247.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2479           2492          18          4.0         247.9       1.0X
-date_trunc YY wholestage on                        2519           2529           8          4.0         251.9       1.0X
+date_trunc YY wholestage off                       2492           2493           1          4.0         249.2       1.0X
+date_trunc YY wholestage on                        2468           2479           7          4.1         246.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2504           2519          21          4.0         250.4       1.0X
-date_trunc MON wholestage on                       2549           2563          15          3.9         254.9       1.0X
+date_trunc MON wholestage off                      2510           2523          19          4.0         251.0       1.0X
+date_trunc MON wholestage on                       2497           2503           9          4.0         249.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2517           2527          14          4.0         251.7       1.0X
-date_trunc MONTH wholestage on                     2538           2547          11          3.9         253.8       1.0X
+date_trunc MONTH wholestage off                    2509           2526          25          4.0         250.9       1.0X
+date_trunc MONTH wholestage on                     2495           2503           7          4.0         249.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2506           2507           2          4.0         250.6       1.0X
-date_trunc MM wholestage on                        2536           2546          10          3.9         253.6       1.0X
+date_trunc MM wholestage off                       2513           2517           6          4.0         251.3       1.0X
+date_trunc MM wholestage on                        2494           2500           5          4.0         249.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2443           2454          17          4.1         244.3       1.0X
-date_trunc DAY wholestage on                       2468           2479          12          4.1         246.8       1.0X
+date_trunc DAY wholestage off                      2415           2429          19          4.1         241.5       1.0X
+date_trunc DAY wholestage on                       2369           2379           7          4.2         236.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2440           2440           0          4.1         244.0       1.0X
-date_trunc DD wholestage on                        2459           2481          23          4.1         245.9       1.0X
+date_trunc DD wholestage off                       2403           2406           4          4.2         240.3       1.0X
+date_trunc DD wholestage on                        2372           2379           4          4.2         237.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2452           2453           2          4.1         245.2       1.0X
-date_trunc HOUR wholestage on                      2428           2435           5          4.1         242.8       1.0X
+date_trunc HOUR wholestage off                     2394           2409          22          4.2         239.4       1.0X
+date_trunc HOUR wholestage on                      2344           2352           6          4.3         234.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    359            359           1         27.8          35.9       1.0X
-date_trunc MINUTE wholestage on                     382            388           8         26.2          38.2       0.9X
+date_trunc MINUTE wholestage off                    346            347           1         28.9          34.6       1.0X
+date_trunc MINUTE wholestage on                     365            370           5         27.4          36.5       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    382            386           6         26.2          38.2       1.0X
-date_trunc SECOND wholestage on                     379            382           5         26.4          37.9       1.0X
+date_trunc SECOND wholestage off                    377            392          21         26.5          37.7       1.0X
+date_trunc SECOND wholestage on                     361            363           2         27.7          36.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2427           2429           3          4.1         242.7       1.0X
-date_trunc WEEK wholestage on                      2427           2441          12          4.1         242.7       1.0X
+date_trunc WEEK wholestage off                     2428           2436          12          4.1         242.8       1.0X
+date_trunc WEEK wholestage on                      2405           2414           8          4.2         240.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3551           3583          46          2.8         355.1       1.0X
-date_trunc QUARTER wholestage on                   3517           3526           6          2.8         351.7       1.0X
+date_trunc QUARTER wholestage off                  3705           3709           5          2.7         370.5       1.0X
+date_trunc QUARTER wholestage on                   3673           3682           8          2.7         367.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           312            313           1         32.0          31.2       1.0X
-trunc year wholestage on                            319            326           7         31.3          31.9       1.0X
+trunc year wholestage off                           314            316           2         31.9          31.4       1.0X
+trunc year wholestage on                            315            323           6         31.7          31.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           318            318           0         31.5          31.8       1.0X
-trunc yyyy wholestage on                            317            328          17         31.5          31.7       1.0X
+trunc yyyy wholestage off                           309            310           1         32.4          30.9       1.0X
+trunc yyyy wholestage on                            309            332          33         32.4          30.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             315            322          10         31.8          31.5       1.0X
-trunc yy wholestage on                              318            328           9         31.5          31.8       1.0X
+trunc yy wholestage off                             310            311           1         32.3          31.0       1.0X
+trunc yy wholestage on                              307            315           6         32.5          30.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            311            313           2         32.1          31.1       1.0X
-trunc mon wholestage on                             322            326           6         31.0          32.2       1.0X
+trunc mon wholestage off                            308            312           7         32.5          30.8       1.0X
+trunc mon wholestage on                             309            315          11         32.4          30.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          310            318          11         32.2          31.0       1.0X
-trunc month wholestage on                           319            326           8         31.3          31.9       1.0X
+trunc month wholestage off                          307            311           6         32.6          30.7       1.0X
+trunc month wholestage on                           308            312           3         32.4          30.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             311            314           4         32.2          31.1       1.0X
-trunc mm wholestage on                              318            324           7         31.4          31.8       1.0X
+trunc mm wholestage off                             307            308           2         32.6          30.7       1.0X
+trunc mm wholestage on                              307            313           6         32.6          30.7       1.0X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     166            167           2          6.0         165.7       1.0X
-to timestamp str wholestage on                      158            162           4          6.3         158.2       1.0X
+to timestamp str wholestage off                     173            174           1          5.8         173.1       1.0X
+to timestamp str wholestage on                      167            169           1          6.0         167.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1487           1488           1          0.7        1487.1       1.0X
-to_timestamp wholestage on                         1418           1422           4          0.7        1418.4       1.0X
+to_timestamp wholestage off                        1460           1461           2          0.7        1459.7       1.0X
+to_timestamp wholestage on                         1464           1472          13          0.7        1463.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1409           1420          15          0.7        1409.2       1.0X
-to_unix_timestamp wholestage on                    1422           1428           8          0.7        1422.1       1.0X
+to_unix_timestamp wholestage off                   1499           1500           3          0.7        1498.6       1.0X
+to_unix_timestamp wholestage on                    1448           1452           7          0.7        1447.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          207            207           0          4.8         206.9       1.0X
-to date str wholestage on                           209            210           1          4.8         209.4       1.0X
+to date str wholestage off                          215            216           2          4.7         214.9       1.0X
+to date str wholestage on                           213            215           3          4.7         212.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3378           3398          28          0.3        3377.7       1.0X
-to_date wholestage on                              3372           3379           8          0.3        3372.2       1.0X
+to_date wholestage off                             3581           3596          22          0.3        3580.8       1.0X
+to_date wholestage on                              3629           3634           6          0.3        3629.0       1.0X
 
 
 ================================================================================================
@@ -422,9 +422,10 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  427            443          23         11.7          85.4       1.0X
-From java.sql.Timestamp                             361            369          11         13.8          72.3       1.2X
-Collect longs                                      1942           2044         170          2.6         388.3       0.2X
-Collect timestamps                                 1891           2093         192          2.6         378.2       0.2X
+From java.sql.Date                                  435            439           3         11.5          87.1       1.0X
+Collect dates                                      3409           3756         343          1.5         681.9       0.1X
+From java.sql.Timestamp                             350            351           1         14.3          70.1       1.2X
+Collect longs                                      1316           1433         124          3.8         263.2       0.3X
+Collect timestamps                                 1752           2013         231          2.9         350.4       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 77b6f833221e5..d4259a9e6c9b8 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    410            428          25         24.4          41.0       1.0X
-cast to timestamp wholestage on                     364            379          17         27.5          36.4       1.1X
+cast to timestamp wholestage off                    403            433          42         24.8          40.3       1.0X
+cast to timestamp wholestage on                     377            386          14         26.6          37.7       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1308           1318          15          7.6         130.8       1.0X
-year of timestamp wholestage on                    1204           1234          19          8.3         120.4       1.1X
+year of timestamp wholestage off                   1327           1328           1          7.5         132.7       1.0X
+year of timestamp wholestage on                    1341           1368          23          7.5         134.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1439           1442           6          7.0         143.9       1.0X
-quarter of timestamp wholestage on                 1374           1385           9          7.3         137.4       1.0X
+quarter of timestamp wholestage off                1460           1468          11          6.9         146.0       1.0X
+quarter of timestamp wholestage on                 1396           1418          14          7.2         139.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1228           1234           8          8.1         122.8       1.0X
-month of timestamp wholestage on                   1225           1241          25          8.2         122.5       1.0X
+month of timestamp wholestage off                  1232           1239          11          8.1         123.2       1.0X
+month of timestamp wholestage on                   1252           1267          14          8.0         125.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1878           1880           3          5.3         187.8       1.0X
-weekofyear of timestamp wholestage on              1864           1870           6          5.4         186.4       1.0X
+weekofyear of timestamp wholestage off             1852           1853           0          5.4         185.2       1.0X
+weekofyear of timestamp wholestage on              1865           1881          19          5.4         186.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1218           1220           3          8.2         121.8       1.0X
-day of timestamp wholestage on                     1217           1226          10          8.2         121.7       1.0X
+day of timestamp wholestage off                    1224           1226           2          8.2         122.4       1.0X
+day of timestamp wholestage on                     1236           1250           9          8.1         123.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1274           1282          12          7.8         127.4       1.0X
-dayofyear of timestamp wholestage on               1263           1274          14          7.9         126.3       1.0X
+dayofyear of timestamp wholestage off              1276           1276           1          7.8         127.6       1.0X
+dayofyear of timestamp wholestage on               1287           1294           8          7.8         128.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1221           1226           6          8.2         122.1       1.0X
-dayofmonth of timestamp wholestage on              1215           1222           9          8.2         121.5       1.0X
+dayofmonth of timestamp wholestage off             1225           1227           3          8.2         122.5       1.0X
+dayofmonth of timestamp wholestage on              1234           1243          11          8.1         123.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1404           1431          38          7.1         140.4       1.0X
-dayofweek of timestamp wholestage on               1374           1381           8          7.3         137.4       1.0X
+dayofweek of timestamp wholestage off              1398           1407          13          7.2         139.8       1.0X
+dayofweek of timestamp wholestage on               1394           1401           6          7.2         139.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1342           1350          12          7.5         134.2       1.0X
-weekday of timestamp wholestage on                 1311           1317           5          7.6         131.1       1.0X
+weekday of timestamp wholestage off                1339           1345           9          7.5         133.9       1.0X
+weekday of timestamp wholestage on                 1333           1340           9          7.5         133.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    997           1000           4         10.0          99.7       1.0X
-hour of timestamp wholestage on                     959            969           8         10.4          95.9       1.0X
+hour of timestamp wholestage off                    992            996           5         10.1          99.2       1.0X
+hour of timestamp wholestage on                     973            981           7         10.3          97.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  968            970           2         10.3          96.8       1.0X
-minute of timestamp wholestage on                   952            960           8         10.5          95.2       1.0X
+minute of timestamp wholestage off                  988            995          10         10.1          98.8       1.0X
+minute of timestamp wholestage on                   980            990           7         10.2          98.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  971            974           5         10.3          97.1       1.0X
-second of timestamp wholestage on                   952            953           0         10.5          95.2       1.0X
+second of timestamp wholestage off                 1008           1013           8          9.9         100.8       1.0X
+second of timestamp wholestage on                   976            987          13         10.2          97.6       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         277            282           7         36.1          27.7       1.0X
-current_date wholestage on                          283            290          11         35.3          28.3       1.0X
+current_date wholestage off                         287            287           0         34.8          28.7       1.0X
+current_date wholestage on                          301            302           1         33.2          30.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    302            324          31         33.1          30.2       1.0X
-current_timestamp wholestage on                     281            292          13         35.6          28.1       1.1X
+current_timestamp wholestage off                    292            295           5         34.3          29.2       1.0X
+current_timestamp wholestage on                     306            315          10         32.7          30.6       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1064           1066           2          9.4         106.4       1.0X
-cast to date wholestage on                         1002           1010           8         10.0         100.2       1.1X
+cast to date wholestage off                        1062           1071          13          9.4         106.2       1.0X
+cast to date wholestage on                         1030           1039          13          9.7         103.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1247           1250           4          8.0         124.7       1.0X
-last_day wholestage on                             1238           1245           4          8.1         123.8       1.0X
+last_day wholestage off                            1282           1303          29          7.8         128.2       1.0X
+last_day wholestage on                             1257           1262           4          8.0         125.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1112           1113           1          9.0         111.2       1.0X
-next_day wholestage on                             1055           1059           5          9.5         105.5       1.1X
+next_day wholestage off                            1114           1120           9          9.0         111.4       1.0X
+next_day wholestage on                             1075           1085           8          9.3         107.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1049           1051           3          9.5         104.9       1.0X
-date_add wholestage on                             1036           1043           9          9.6         103.6       1.0X
+date_add wholestage off                            1056           1061           7          9.5         105.6       1.0X
+date_add wholestage on                             1057           1063           6          9.5         105.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1043           1063          29          9.6         104.3       1.0X
-date_sub wholestage on                             1036           1041           5          9.6         103.6       1.0X
+date_sub wholestage off                            1069           1080          15          9.4         106.9       1.0X
+date_sub wholestage on                             1056           1064          15          9.5         105.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1371           1373           2          7.3         137.1       1.0X
-add_months wholestage on                           1402           1409          10          7.1         140.2       1.0X
+add_months wholestage off                          1367           1367           1          7.3         136.7       1.0X
+add_months wholestage on                           1432           1438           6          7.0         143.2       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5741           5862         171          1.7         574.1       1.0X
-format date wholestage on                          5589           5600          19          1.8         558.9       1.0X
+format date wholestage off                         5900           5918          25          1.7         590.0       1.0X
+format date wholestage on                          6364           6393          34          1.6         636.4       0.9X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8681           8684           4          1.2         868.1       1.0X
-from_unixtime wholestage on                        8669           8707          33          1.2         866.9       1.0X
+from_unixtime wholestage off                       8740           8753          18          1.1         874.0       1.0X
+from_unixtime wholestage on                        8689           8714          26          1.2         868.9       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1096           1101           7          9.1         109.6       1.0X
-from_utc_timestamp wholestage on                   1108           1116           6          9.0         110.8       1.0X
+from_utc_timestamp wholestage off                  1086           1087           2          9.2         108.6       1.0X
+from_utc_timestamp wholestage on                   1168           1173           5          8.6         116.8       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1626           1638          16          6.1         162.6       1.0X
-to_utc_timestamp wholestage on                     1594           1606          10          6.3         159.4       1.0X
+to_utc_timestamp wholestage off                    1640           1642           3          6.1         164.0       1.0X
+to_utc_timestamp wholestage on                     1600           1612          20          6.2         160.0       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        331            335           6         30.2          33.1       1.0X
-cast interval wholestage on                         298            305           5         33.5          29.8       1.1X
+cast interval wholestage off                        329            330           2         30.4          32.9       1.0X
+cast interval wholestage on                         321            333          17         31.1          32.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1821           1822           2          5.5         182.1       1.0X
-datediff wholestage on                             1759           1763           3          5.7         175.9       1.0X
+datediff wholestage off                            1819           1820           1          5.5         181.9       1.0X
+datediff wholestage on                             1783           1789           6          5.6         178.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      4752           4770          25          2.1         475.2       1.0X
-months_between wholestage on                       4740           4746           5          2.1         474.0       1.0X
+months_between wholestage off                      4791           4793           3          2.1         479.1       1.0X
+months_between wholestage on                       4739           4745           6          2.1         473.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1936           2049         160          0.5        1935.7       1.0X
-window wholestage on                              44821          44850          44          0.0       44820.6       0.0X
+window wholestage off                              1955           2108         217          0.5        1954.9       1.0X
+window wholestage on                              46222          46285          41          0.0       46222.1       0.0X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2382           2384           3          4.2         238.2       1.0X
-date_trunc YEAR wholestage on                      2309           2316           7          4.3         230.9       1.0X
+date_trunc YEAR wholestage off                     2341           2343           3          4.3         234.1       1.0X
+date_trunc YEAR wholestage on                      2288           2298           8          4.4         228.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2368           2375          10          4.2         236.8       1.0X
-date_trunc YYYY wholestage on                      2304           2313           7          4.3         230.4       1.0X
+date_trunc YYYY wholestage off                     2337           2338           2          4.3         233.7       1.0X
+date_trunc YYYY wholestage on                      2286           2295           7          4.4         228.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2381           2396          22          4.2         238.1       1.0X
-date_trunc YY wholestage on                        2303           2305           1          4.3         230.3       1.0X
+date_trunc YY wholestage off                       2339           2340           2          4.3         233.9       1.0X
+date_trunc YY wholestage on                        2288           2292           3          4.4         228.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2325           2333          11          4.3         232.5       1.0X
-date_trunc MON wholestage on                       2293           2301           6          4.4         229.3       1.0X
+date_trunc MON wholestage off                      2308           2309           1          4.3         230.8       1.0X
+date_trunc MON wholestage on                       2348           2357          11          4.3         234.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2335           2335           0          4.3         233.5       1.0X
-date_trunc MONTH wholestage on                     2296           2298           2          4.4         229.6       1.0X
+date_trunc MONTH wholestage off                    2316           2318           3          4.3         231.6       1.0X
+date_trunc MONTH wholestage on                     2346           2356           9          4.3         234.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2328           2329           1          4.3         232.8       1.0X
-date_trunc MM wholestage on                        2290           2295           4          4.4         229.0       1.0X
+date_trunc MM wholestage off                       2313           2315           3          4.3         231.3       1.0X
+date_trunc MM wholestage on                        2348           2355           4          4.3         234.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2139           2140           1          4.7         213.9       1.0X
-date_trunc DAY wholestage on                       2095           2101           7          4.8         209.5       1.0X
+date_trunc DAY wholestage off                      2162           2164           3          4.6         216.2       1.0X
+date_trunc DAY wholestage on                       2177           2183           4          4.6         217.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2143           2145           2          4.7         214.3       1.0X
-date_trunc DD wholestage on                        2087           2092           3          4.8         208.7       1.0X
+date_trunc DD wholestage off                       2162           2169          10          4.6         216.2       1.0X
+date_trunc DD wholestage on                        2178           2181           2          4.6         217.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2201           2207           8          4.5         220.1       1.0X
-date_trunc HOUR wholestage on                      2176           2185           6          4.6         217.6       1.0X
+date_trunc HOUR wholestage off                     2230           2237           9          4.5         223.0       1.0X
+date_trunc HOUR wholestage on                      2242           2251           7          4.5         224.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    356            356           0         28.1          35.6       1.0X
-date_trunc MINUTE wholestage on                     333            344           7         30.1          33.3       1.1X
+date_trunc MINUTE wholestage off                    367            373           8         27.2          36.7       1.0X
+date_trunc MINUTE wholestage on                     343            347           3         29.1          34.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    341            342           2         29.3          34.1       1.0X
-date_trunc SECOND wholestage on                     335            349          18         29.9          33.5       1.0X
+date_trunc SECOND wholestage off                    369            369           1         27.1          36.9       1.0X
+date_trunc SECOND wholestage on                     343            346           2         29.2          34.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2257           2257           1          4.4         225.7       1.0X
-date_trunc WEEK wholestage on                      2202           2211          11          4.5         220.2       1.0X
+date_trunc WEEK wholestage off                     2238           2245          10          4.5         223.8       1.0X
+date_trunc WEEK wholestage on                      2222           2231          10          4.5         222.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3180           3181           2          3.1         318.0       1.0X
-date_trunc QUARTER wholestage on                   3036           3043          12          3.3         303.6       1.0X
+date_trunc QUARTER wholestage off                  3092           3093           2          3.2         309.2       1.0X
+date_trunc QUARTER wholestage on                   3074           3077           4          3.3         307.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           327            328           1         30.6          32.7       1.0X
-trunc year wholestage on                            310            325          13         32.3          31.0       1.1X
+trunc year wholestage off                           320            328          12         31.3          32.0       1.0X
+trunc year wholestage on                            344            365          18         29.1          34.4       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           323            327           6         31.0          32.3       1.0X
-trunc yyyy wholestage on                            304            317          14         32.9          30.4       1.1X
+trunc yyyy wholestage off                           316            318           3         31.6          31.6       1.0X
+trunc yyyy wholestage on                            352            372          21         28.4          35.2       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             324            325           2         30.8          32.4       1.0X
-trunc yy wholestage on                              308            311           3         32.4          30.8       1.1X
+trunc yy wholestage off                             313            314           1         31.9          31.3       1.0X
+trunc yy wholestage on                              346            364          27         28.9          34.6       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            323            323           1         31.0          32.3       1.0X
-trunc mon wholestage on                             303            310           7         33.0          30.3       1.1X
+trunc mon wholestage off                            317            318           1         31.5          31.7       1.0X
+trunc mon wholestage on                             347            367          17         28.8          34.7       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          322            323           0         31.0          32.2       1.0X
-trunc month wholestage on                           307            337          65         32.6          30.7       1.1X
+trunc month wholestage off                          320            321           1         31.2          32.0       1.0X
+trunc month wholestage on                           352            410          77         28.4          35.2       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             324            325           2         30.9          32.4       1.0X
-trunc mm wholestage on                              294            306           7         34.0          29.4       1.1X
+trunc mm wholestage off                             315            318           5         31.8          31.5       1.0X
+trunc mm wholestage on                              353            382          29         28.3          35.3       0.9X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     216            216           1          4.6         215.7       1.0X
-to timestamp str wholestage on                      207            209           3          4.8         207.0       1.0X
+to timestamp str wholestage off                     217            221           5          4.6         217.3       1.0X
+to timestamp str wholestage on                      212            214           2          4.7         212.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1751           1751           1          0.6        1750.6       1.0X
-to_timestamp wholestage on                         1798           1832          31          0.6        1798.4       1.0X
+to_timestamp wholestage off                        1957           1964          11          0.5        1956.6       1.0X
+to_timestamp wholestage on                         1993           1998           9          0.5        1993.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1770           1773           4          0.6        1770.2       1.0X
-to_unix_timestamp wholestage on                    1743           1754          10          0.6        1743.0       1.0X
+to_unix_timestamp wholestage off                   2071           2073           3          0.5        2071.0       1.0X
+to_unix_timestamp wholestage on                    2082           2092          11          0.5        2082.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          260            266           9          3.8         260.1       1.0X
-to date str wholestage on                           257            260           3          3.9         257.3       1.0X
+to date str wholestage off                          262            262           0          3.8         261.7       1.0X
+to date str wholestage on                           258            261           4          3.9         258.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3589           3620          44          0.3        3589.0       1.0X
-to_date wholestage on                              3565           3579          15          0.3        3565.0       1.0X
+to_date wholestage off                             3551           3567          22          0.3        3550.7       1.0X
+to_date wholestage on                              3505           3513           9          0.3        3505.3       1.0X
 
 
 ================================================================================================
@@ -422,9 +422,10 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  416            420           6         12.0          83.2       1.0X
-From java.sql.Timestamp                             375            381           5         13.3          75.1       1.1X
-Collect longs                                      1225           1962        1101          4.1         245.1       0.3X
-Collect timestamps                                 2013           2829        1401          2.5         402.5       0.2X
+From java.sql.Date                                  416            419           3         12.0          83.2       1.0X
+Collect dates                                      1928           2759        1180          2.6         385.6       0.2X
+From java.sql.Timestamp                             383            397          12         13.0          76.7       1.1X
+Collect longs                                      1324           1411          91          3.8         264.9       0.3X
+Collect timestamps                                 1906           2040         185          2.6         381.2       0.2X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 5fa5b9b06a60b..0bb87b19b7a69 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -137,6 +137,11 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
               .map(millis => new java.sql.Date(millis))
               .noop()
           }
+          benchmark.addCase("Collect dates", numIters) { _ =>
+            spark.range(0, rowsNum, 1, 1)
+              .map(millis => new java.sql.Date(millis))
+              .collect()
+          }
           benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
             spark.range(rowsNum)
               .map(millis => new Timestamp(millis))

From 37023273fe0171ab758a81956dcc0ae9f8d2253b Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 15 Apr 2020 16:33:56 +0000
Subject: [PATCH 0502/1280] [SPARK-31423][SQL] Fix rebasing of not-existed
 dates

### What changes were proposed in this pull request?
In the PR, I propose to change rebasing of not-existed dates in the hybrid calendar (Julian + Gregorian since 1582-10-15) in the range (1582-10-04, 1582-10-15). Not existed dates from the range are shifted to the first valid date in the hybrid calendar - 1582-10-15. The changes affect only `rebaseGregorianToJulianDays()` because reverse rebasing from the hybrid dates to Proleptic Gregorian dates does not have such problem.

### Why are the changes needed?
Currently, not-existed dates are shifted by standard difference between Julian and Gregorian calendar on 1582-10-04, for example 1582-10-14 -> 1582-10-24. That's contradict to shifting not existed dates in other cases, for example:
```
scala> sql("select date'1990-9-31'").show
+-----------------+
|DATE '1990-10-01'|
+-----------------+
|       1990-10-01|
+-----------------+
```

### Does this PR introduce any user-facing change?
Yes, this impacts on conversion of Spark SQL `DATE` values to external dates based on non-Proleptic Gregorian calendar. For example, while saving the 1582-10-14 date to ORC files, it will be shifted to the next valid date 1582-10-15.

### How was this patch tested?
- Added tests to `RebaseDateTimeSuite` and to `OrcSourceSuite`
- By existing test suites `DateTimeUtilsSuite`, `DateFunctionsSuite`, `DateExpressionsSuite`, `CollectionExpressionsSuite`, `ParquetIOSuite`.

Closes #28225 from MaxGekk/fix-not-exist-dates.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2b10d70bad30fb7b7c293338c2acc908031af0b8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    | 16 ++++++++++----
 .../catalyst/util/RebaseDateTimeSuite.scala   | 22 +++++++++++++++++++
 .../datasources/orc/OrcSourceSuite.scala      |  8 ++++---
 3 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 50b552ea502d2..6338a595a9b87 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -131,7 +131,8 @@ object RebaseDateTime {
   // The differences in days between Proleptic Gregorian and Julian dates.
   // The diff at the index `i` is applicable for all days in the date interval:
   // [gregJulianDiffSwitchDay(i), gregJulianDiffSwitchDay(i+1))
-  private val gregJulianDiffs = Array(-2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0)
+  private val gregJulianDiffs = Array(
+    -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
   // The sorted days in Proleptic Gregorian calendar when difference in days between
   // Proleptic Gregorian and Julian was changed.
   // The starting point is the `0001-01-01` (-719162 days since the epoch in
@@ -139,13 +140,17 @@ object RebaseDateTime {
   // Rebasing switch days and diffs `gregJulianDiffSwitchDay` and `gregJulianDiffs`
   // was generated by the `localRebaseGregorianToJulianDays` function.
   private val gregJulianDiffSwitchDay = Array(
-    -719162, -682944, -646420, -609896, -536847, -500323, -463799,
-    -390750, -354226, -317702, -244653, -208129, -171605, -141427)
+    -719162, -682944, -646420, -609896, -536847, -500323, -463799, -390750,
+    -354226, -317702, -244653, -208129, -171605, -141436, -141435, -141434,
+    -141433, -141432, -141431, -141430, -141429, -141428, -141427)
 
   // The first days of Common Era (CE) which is mapped to the '0001-01-01' date
   // in Proleptic Gregorian calendar.
   private final val gregorianCommonEraStartDay = gregJulianDiffSwitchDay(0)
 
+  private final val gregorianStartDay = LocalDate.of(1582, 10, 15)
+  private final val julianEndDay = LocalDate.of(1582, 10, 4)
+
   /**
    * Converts the given number of days since the epoch day 1970-01-01 to a local date in Proleptic
    * Gregorian calendar, interprets the result as a local date in Julian calendar, and takes the
@@ -165,7 +170,10 @@ object RebaseDateTime {
    * @return The rebased number of days in Julian calendar.
    */
   private[sql] def localRebaseGregorianToJulianDays(days: Int): Int = {
-    val localDate = LocalDate.ofEpochDay(days)
+    var localDate = LocalDate.ofEpochDay(days)
+    if (localDate.isAfter(julianEndDay) && localDate.isBefore(gregorianStartDay)) {
+      localDate = gregorianStartDay
+    }
     val utcCal = new Calendar.Builder()
       // `gregory` is a hybrid calendar that supports both
       // the Julian and Gregorian calendar systems
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index 1e70e60484850..73ca7614e193e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -364,4 +364,26 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
   }
+
+  test("rebase not-existed dates in the hybrid calendar") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1582-10-04" -> "1582-10-04",
+          "1582-10-05" -> "1582-10-15", "1582-10-06" -> "1582-10-15", "1582-10-07" -> "1582-10-15",
+          "1582-10-08" -> "1582-10-15", "1582-10-09" -> "1582-10-15", "1582-10-11" -> "1582-10-15",
+          "1582-10-12" -> "1582-10-15", "1582-10-13" -> "1582-10-15", "1582-10-14" -> "1582-10-15",
+          "1582-10-15" -> "1582-10-15").foreach { case (hybridDate, gregDate) =>
+          withClue(s"tz = ${zid.getId} hybrid date = $hybridDate greg date = $gregDate") {
+            val date = Date.valueOf(gregDate)
+            val hybridDays = fromJavaDateLegacy(date)
+            val gregorianDays = localDateToDays(LocalDate.parse(hybridDate))
+
+            assert(localRebaseGregorianToJulianDays(gregorianDays) === hybridDays)
+            assert(rebaseGregorianToJulianDays(gregorianDays) === hybridDays)
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 0b7500c012686..f0ce6d503b687 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -493,17 +493,19 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
     }
   }
 
-  test("SPARK-31238: rebasing dates in write") {
+  test("SPARK-31238, SPARK-31423: rebasing dates in write") {
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      Seq("1001-01-01").toDF("dateS")
+      Seq("1001-01-01", "1582-10-10").toDF("dateS")
         .select($"dateS".cast("date").as("date"))
         .write
         .orc(path)
 
       Seq(false, true).foreach { vectorized =>
         withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-          checkAnswer(spark.read.orc(path), Row(Date.valueOf("1001-01-01")))
+          checkAnswer(
+            spark.read.orc(path),
+            Seq(Row(Date.valueOf("1001-01-01")), Row(Date.valueOf("1582-10-15"))))
         }
       }
     }

From d286db145433d6d7610c69980512369f389930ca Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 15 Apr 2020 11:29:55 -0700
Subject: [PATCH 0503/1280] [SPARK-31018][CORE][DOCS] Deprecate support of
 multiple workers on the same host in Standalone

### What changes were proposed in this pull request?

Update the document and shell script to warn user about the deprecation of multiple workers on the same host support.

### Why are the changes needed?

This is a sub-task of [SPARK-30978](https://issues.apache.org/jira/browse/SPARK-30978), which plans to totally remove support of multiple workers in Spark 3.1. This PR makes the first step to deprecate it firstly in Spark 3.0.

### Does this PR introduce any user-facing change?

Yeah, user see warning when they run start worker script.

### How was this patch tested?

Tested manually.

Closes #27768 from Ngone51/deprecate_spark_worker_instances.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit 0d4e4df06105cf2985dde17c1af76093b3ae8c13)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 docs/core-migration-guide.md  | 2 ++
 docs/hardware-provisioning.md | 8 ++++----
 sbin/start-slave.sh           | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index 66a489bcc8be3..cde6e070c5e35 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -38,3 +38,5 @@ license: |
 - Event log file will be written as UTF-8 encoding, and Spark History Server will replay event log files as UTF-8 encoding. Previously Spark wrote the event log file as default charset of driver JVM process, so Spark History Server of Spark 2.x is needed to read the old event log files in case of incompatible encoding.
 
 - A new protocol for fetching shuffle blocks is used. It's recommended that external shuffle services be upgraded when running Spark 3.0 apps. You can still use old external shuffle services by setting the configuration `spark.shuffle.useOldFetchProtocol` to `true`. Otherwise, Spark may run into errors with messages like `IllegalArgumentException: Unexpected message type: <number>`.
+
+- `SPARK_WORKER_INSTANCES` is deprecated in Standalone mode. It's recommended to launch multiple executors in one worker and launch one worker per node instead of launching multiple workers per node and launching one executor per worker.
diff --git a/docs/hardware-provisioning.md b/docs/hardware-provisioning.md
index 4e5d681962c8d..fc87995f9817f 100644
--- a/docs/hardware-provisioning.md
+++ b/docs/hardware-provisioning.md
@@ -63,10 +63,10 @@ Note that memory usage is greatly affected by storage level and serialization fo
 the [tuning guide](tuning.html) for tips on how to reduce it.
 
 Finally, note that the Java VM does not always behave well with more than 200 GiB of RAM. If you
-purchase machines with more RAM than this, you can run _multiple worker JVMs per node_. In
-Spark's [standalone mode](spark-standalone.html), you can set the number of workers per node
-with the `SPARK_WORKER_INSTANCES` variable in `conf/spark-env.sh`, and the number of cores
-per worker with `SPARK_WORKER_CORES`.
+purchase machines with more RAM than this, you can launch multiple executors in a single node. In
+Spark's [standalone mode](spark-standalone.html), a worker is responsible for launching multiple
+executors according to its available memory and cores, and each executor will be launched in a
+separate Java VM.
 
 # Network
 
diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh
index 2cb17a04f67ef..9b3b26b07842b 100755
--- a/sbin/start-slave.sh
+++ b/sbin/start-slave.sh
@@ -22,7 +22,7 @@
 # Environment Variables
 #
 #   SPARK_WORKER_INSTANCES  The number of worker instances to run on this
-#                           slave.  Default is 1.
+#                           slave.  Default is 1. Note it has been deprecate since Spark 3.0.
 #   SPARK_WORKER_PORT       The base port number for the first worker. If set,
 #                           subsequent workers will increment this number.  If
 #                           unset, Spark will find a valid port number, but

From 4476c85775d231c8bb26399284c0baf4292bec7c Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 16 Apr 2020 08:34:26 +0900
Subject: [PATCH 0504/1280] [SPARK-31428][SQL][DOCS] Document Common Table
 Expression in SQL Reference

### What changes were proposed in this pull request?
Document Common Table Expression in SQL Reference

### Why are the changes needed?
Make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes
<img width="1050" alt="Screen Shot 2020-04-13 at 12 06 35 AM" src="https://user-images.githubusercontent.com/13592258/79100257-f61def00-7d1a-11ea-8402-17017059232e.png">

<img width="1050" alt="Screen Shot 2020-04-13 at 12 07 09 AM" src="https://user-images.githubusercontent.com/13592258/79100260-f7e7b280-7d1a-11ea-9408-058c0851f0b6.png">

<img width="1050" alt="Screen Shot 2020-04-13 at 12 07 35 AM" src="https://user-images.githubusercontent.com/13592258/79100262-fa4a0c80-7d1a-11ea-8862-eb1d8960296b.png">

Also link to Select page

<img width="1045" alt="Screen Shot 2020-04-12 at 4 14 30 PM" src="https://user-images.githubusercontent.com/13592258/79082246-217fea00-7cd9-11ea-8d96-1a69769d1e19.png">

### How was this patch tested?
Manually build and check

Closes #28196 from huaxingao/cte.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 92c1b246174948d0c1f4d0833e1ceac265b17be7)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml              |   2 +
 docs/sql-ref-syntax-qry-select-cte.md | 109 +++++++++++++++++++++++++-
 docs/sql-ref-syntax-qry-select.md     |   3 +-
 3 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index badb98d2e593b..7827a0fe7d904 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -166,6 +166,8 @@
                   url: sql-ref-syntax-qry-select-tvf.html
                 - text: Inline Table
                   url: sql-ref-syntax-qry-select-inline-table.html
+                - text: Common Table Expression
+                  url: sql-ref-syntax-qry-select-cte.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-qry-select-cte.md b/docs/sql-ref-syntax-qry-select-cte.md
index 2bd7748291a9f..2146f8e0cd557 100644
--- a/docs/sql-ref-syntax-qry-select-cte.md
+++ b/docs/sql-ref-syntax-qry-select-cte.md
@@ -19,4 +19,111 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+A common table expression (CTE) defines a temporary result set that a user can reference possibly multiple times within the scope of a SQL statement. A CTE is used mainly in a SELECT statement.
+
+### Syntax
+
+{% highlight sql %}
+WITH common_table_expression [ , ... ]
+{% endhighlight %}
+
+While `common_table_expression` is defined as
+{% highlight sql %}
+expression_name [ ( column_name [ , ... ] ) ] [ AS ] ( [ common_table_expression ] query )
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>expression_name</em></code></dt>
+  <dd>
+    Specifies a name for the common table expression.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>query</em></code></dt>
+  <dd>
+    A <a href="sql-ref-syntax-qry-select.html">SELECT</a> statement.
+  </dd>
+</dl>
+
+### Examples
+
+{% highlight sql %}
+-- CTE with multiple column aliases
+WITH t(x, y) AS (SELECT 1, 2)
+SELECT * FROM t WHERE x = 1 AND y = 2;
+  +---+---+
+  |  x|  y|
+  +---+---+
+  |  1|  2|
+  +---+---+
+
+-- CTE in CTE definition
+WITH t as (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+)
+SELECT * FROM t;
+  +---+
+  |  1|
+  +---+
+  |  1|
+  +---+
+
+-- CTE in subquery
+SELECT max(c) FROM (
+    WITH t(c) AS (SELECT 1)
+    SELECT * FROM t
+);
+  +------+
+  |max(c)|
+  +------+
+  |     1|
+  +------+
+
+-- CTE in subquery expression
+SELECT (
+    WITH t AS (SELECT 1)
+    SELECT * FROM t
+);
+  +----------------+
+  |scalarsubquery()|
+  +----------------+
+  |               1|
+  +----------------+
+
+-- CTE in CREATE VIEW statement
+CREATE VIEW v AS
+    WITH t(a, b, c, d) AS (SELECT 1, 2, 3, 4)
+    SELECT * FROM t;
+SELECT * FROM v;
+  +---+---+---+---+
+  |  a|  b|  c|  d|
+  +---+---+---+---+
+  |  1|  2|  3|  4|
+  +---+---+---+---+
+
+-- If name conflict is detected in nested CTE, then AnalysisException is thrown by default.
+-- SET spark.sql.legacy.ctePrecedencePolicy = CORRECTED (which is recommended),
+-- inner CTE definitions take precedence over outer definitions.
+SET spark.sql.legacy.ctePrecedencePolicy = CORRECTED;
+WITH
+    t AS (SELECT 1),
+    t2 AS (
+        WITH t AS (SELECT 2)
+        SELECT * FROM t
+    )
+SELECT * FROM t2;
+  +---+
+  |  2|
+  +---+
+  |  2|
+  +---+
+{% endhighlight %}
+
+### Related Statements
+
+ * [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 94f69d4d733c4..bc2cc0269124e 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -53,7 +53,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 <dl>
   <dt><code><em>with_query</em></code></dt>
   <dd>
-    Specifies the common table expressions (CTEs) before the main query block.
+    Specifies the <a href="sql-ref-syntax-qry-select-cte.html">common table expressions (CTEs)</a> before the main query block.
     These table expressions are allowed to be referenced later in the FROM clause. This is useful to abstract
     out repeated subquery blocks in the FROM clause and improves readability of the query.
   </dd>
@@ -159,3 +159,4 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
  * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
  * [JOIN](sql-ref-syntax-qry-select-join.html)
  * [SET Operators](sql-ref-syntax-qry-select-setops.html)
+ * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)

From e4f153a462985ed1225e0072256181a00b449b2b Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 16 Apr 2020 02:54:38 +0000
Subject: [PATCH 0505/1280] [SPARK-31455][SQL] Fix rebasing of not-existed
 timestamps

### What changes were proposed in this pull request?
In the PR, I propose to change rebasing of not-existed timestamps in the hybrid calendar (Julian + Gregorian since 1582-10-15) in the range [1582-10-05, 1582-10-15). Not existed timestamps from the range are shifted to the first valid date in the hybrid calendar - 1582-10-15. The changes affect only `rebaseGregorianToJulianMicros()` because reverse rebasing from the hybrid timestamps to Proleptic Gregorian timestamps does not have such problem.

The shifting affects only the date part of timestamps while keeping the time part as is. For example:
```
1582-10-10 00:11:22.334455 -> 1582-10-15 00:11:22.334455
```

### Why are the changes needed?
Currently, not-existed timestamps are shifted by standard difference between Julian and Gregorian calendar on 1582-10-04, for example 1582-10-14 00:00:00 -> 1582-10-24 00:00:00. That contradicts to shifting of not existed dates in other cases, for example:
```
scala> sql("select timestamp'1990-9-31 12:12:12'").show
+----------------------------------+
|TIMESTAMP('1990-10-01 12:12:12.0')|
+----------------------------------+
|               1990-10-01 12:12:12|
+----------------------------------+
```

### Does this PR introduce any user-facing change?
Yes, this impacts on conversion of Spark SQL `TIMESTAMP` values to external timestamps based on non-Proleptic Gregorian calendar. For example, while saving the 1582-10-14 12:13:14 date to ORC files, it will be shifted to the next valid date 1582-10-15 12:13:14.

### How was this patch tested?
- Added tests to `RebaseDateTimeSuite` and to `OrcSourceSuite`
- By existing test suites `DateTimeUtilsSuite`, `DateFunctionsSuite`, `DateExpressionsSuite`, `CollectionExpressionsSuite`, `ParquetIOSuite`.

Closes #28227 from MaxGekk/fix-not-exist-timestamps.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c76c31e2c677c571dcfbb2abd0a16dba642adae4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../gregorian-julian-rebase-micros.json       | 2384 ++++++++---------
 .../sql/catalyst/util/RebaseDateTime.scala    |   20 +-
 .../catalyst/util/RebaseDateTimeSuite.scala   |   37 +-
 .../datasources/orc/OrcSourceSuite.scala      |    8 +-
 4 files changed, 1244 insertions(+), 1205 deletions(-)

diff --git a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
index 050aa02918fc3..8d36fbd377c1e 100644
--- a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
+++ b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
@@ -1,2385 +1,2385 @@
 [ {
   "tz" : "Africa/Abidjan",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Accra",
-  "switches" : [ -62135596748, -59006361548, -55850687948, -52695014348, -46383580748, -43227907148, -40072233548, -33760799948, -30605126348, -27449452748, -21138019148, -17982345548, -14826671948, -12219292748, -2208988800 ],
-  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, -52, 0 ]
+  "switches" : [ -62135596748, -59006361548, -55850687948, -52695014348, -46383580748, -43227907148, -40072233548, -33760799948, -30605126348, -27449452748, -21138019148, -17982345548, -14826671948, -12220070348, -12219983948, -12219897548, -12219811148, -12219724748, -12219638348, -12219551948, -12219465548, -12219379148, -12219292748, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ]
 }, {
   "tz" : "Africa/Addis_Ababa",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Algiers",
-  "switches" : [ -62135597532, -59006362332, -55850688732, -52695015132, -46383581532, -43227907932, -40072234332, -33760800732, -30605127132, -27449453532, -21138019932, -17982346332, -14826672732, -12219293532, -2486679072, -2208988800 ],
-  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, -2868, -3039, 0 ]
+  "switches" : [ -62135597532, -59006362332, -55850688732, -52695015132, -46383581532, -43227907932, -40072234332, -33760800732, -30605127132, -27449453532, -21138019932, -17982346332, -14826672732, -12220071132, -12219984732, -12219898332, -12219811932, -12219725532, -12219639132, -12219552732, -12219466332, -12219379932, -12219293532, -2486679072, -2208988800 ],
+  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, 774732, 688332, 601932, 515532, 429132, 342732, 256332, 169932, 83532, -2868, -3039, 0 ]
 }, {
   "tz" : "Africa/Asmara",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Asmera",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Bamako",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Bangui",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Banjul",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Bissau",
-  "switches" : [ -62135593060, -59006357860, -55850684260, -52695010660, -46383577060, -43227903460, -40072229860, -33760796260, -30605122660, -27449449060, -21138015460, -17982341860, -14826668260, -12219289060, -2208988800 ],
-  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, -3740, 0 ]
+  "switches" : [ -62135593060, -59006357860, -55850684260, -52695010660, -46383577060, -43227903460, -40072229860, -33760796260, -30605122660, -27449449060, -21138015460, -17982341860, -14826668260, -12220066660, -12219980260, -12219893860, -12219807460, -12219721060, -12219634660, -12219548260, -12219461860, -12219375460, -12219289060, -2208988800 ],
+  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, 773860, 687460, 601060, 514660, 428260, 341860, 255460, 169060, 82660, -3740, 0 ]
 }, {
   "tz" : "Africa/Blantyre",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Brazzaville",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Bujumbura",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Cairo",
-  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12219300309, -2208988800 ],
-  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 309, 0 ]
+  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12220077909, -12219991509, -12219905109, -12219818709, -12219732309, -12219645909, -12219559509, -12219473109, -12219386709, -12219300309, -2208988800 ],
+  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ]
 }, {
   "tz" : "Africa/Ceuta",
-  "switches" : [ -62135595524, -59006360324, -55850686724, -52695013124, -46383579524, -43227905924, -40072232324, -33760798724, -30605125124, -27449451524, -21138017924, -17982344324, -14826670724, -12219291524, -2208988800 ],
-  "diffs" : [ -177676, -91276, -4876, 81524, 167924, 254324, 340724, 427124, 513524, 599924, 686324, 772724, 859124, -4876, 0 ]
+  "switches" : [ -62135595524, -59006360324, -55850686724, -52695013124, -46383579524, -43227905924, -40072232324, -33760798724, -30605125124, -27449451524, -21138017924, -17982344324, -14826670724, -12220069124, -12219982724, -12219896324, -12219809924, -12219723524, -12219637124, -12219550724, -12219464324, -12219377924, -12219291524, -2208988800 ],
+  "diffs" : [ -177676, -91276, -4876, 81524, 167924, 254324, 340724, 427124, 513524, 599924, 686324, 772724, 859124, 772724, 686324, 599924, 513524, 427124, 340724, 254324, 167924, 81524, -4876, 0 ]
 }, {
   "tz" : "Africa/Conakry",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Dakar",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Dar_es_Salaam",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Djibouti",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Douala",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Freetown",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Gaborone",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Harare",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Johannesburg",
-  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12219299520, -2458173120, -2208988800 ],
-  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, -480, -1800, 0 ]
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12220077120, -12219990720, -12219904320, -12219817920, -12219731520, -12219645120, -12219558720, -12219472320, -12219385920, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, 777120, 690720, 604320, 517920, 431520, 345120, 258720, 172320, 85920, -480, -1800, 0 ]
 }, {
   "tz" : "Africa/Juba",
-  "switches" : [ -62135604388, -59006369188, -55850695588, -52695021988, -46383588388, -43227914788, -40072241188, -33760807588, -30605133988, -27449460388, -21138026788, -17982353188, -14826679588, -12219300388, -2208988800 ],
-  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, -3212, 0 ]
+  "switches" : [ -62135604388, -59006369188, -55850695588, -52695021988, -46383588388, -43227914788, -40072241188, -33760807588, -30605133988, -27449460388, -21138026788, -17982353188, -14826679588, -12220077988, -12219991588, -12219905188, -12219818788, -12219732388, -12219645988, -12219559588, -12219473188, -12219386788, -12219300388, -2208988800 ],
+  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, 774388, 687988, 601588, 515188, 428788, 342388, 255988, 169588, 83188, -3212, 0 ]
 }, {
   "tz" : "Africa/Kampala",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Khartoum",
-  "switches" : [ -62135604608, -59006369408, -55850695808, -52695022208, -46383588608, -43227915008, -40072241408, -33760807808, -30605134208, -27449460608, -21138027008, -17982353408, -14826679808, -12219300608, -2208988800 ],
-  "diffs" : [ -172192, -85792, 608, 87008, 173408, 259808, 346208, 432608, 519008, 605408, 691808, 778208, 864608, 608, 0 ]
+  "switches" : [ -62135604608, -59006369408, -55850695808, -52695022208, -46383588608, -43227915008, -40072241408, -33760807808, -30605134208, -27449460608, -21138027008, -17982353408, -14826679808, -12220078208, -12219991808, -12219905408, -12219819008, -12219732608, -12219646208, -12219559808, -12219473408, -12219387008, -12219300608, -2208988800 ],
+  "diffs" : [ -172192, -85792, 608, 87008, 173408, 259808, 346208, 432608, 519008, 605408, 691808, 778208, 864608, 778208, 691808, 605408, 519008, 432608, 346208, 259808, 173408, 87008, 608, 0 ]
 }, {
   "tz" : "Africa/Kigali",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Kinshasa",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Lagos",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Libreville",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Lome",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Luanda",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Lubumbashi",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Lusaka",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Malabo",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Maputo",
-  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12219300620, -2208988800 ],
-  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 620, 0 ]
+  "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ],
+  "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ]
 }, {
   "tz" : "Africa/Maseru",
-  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12219299520, -2458173120, -2208988800 ],
-  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, -480, -1800, 0 ]
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12220077120, -12219990720, -12219904320, -12219817920, -12219731520, -12219645120, -12219558720, -12219472320, -12219385920, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, 777120, 690720, 604320, 517920, 431520, 345120, 258720, 172320, 85920, -480, -1800, 0 ]
 }, {
   "tz" : "Africa/Mbabane",
-  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12219299520, -2458173120, -2208988800 ],
-  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, -480, -1800, 0 ]
+  "switches" : [ -62135603520, -59006368320, -55850694720, -52695021120, -46383587520, -43227913920, -40072240320, -33760806720, -30605133120, -27449459520, -21138025920, -17982352320, -14826678720, -12220077120, -12219990720, -12219904320, -12219817920, -12219731520, -12219645120, -12219558720, -12219472320, -12219385920, -12219299520, -2458173120, -2208988800 ],
+  "diffs" : [ -173280, -86880, -480, 85920, 172320, 258720, 345120, 431520, 517920, 604320, 690720, 777120, 863520, 777120, 690720, 604320, 517920, 431520, 345120, 258720, 172320, 85920, -480, -1800, 0 ]
 }, {
   "tz" : "Africa/Mogadishu",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Monrovia",
-  "switches" : [ -62135594212, -59006359012, -55850685412, -52695011812, -46383578212, -43227904612, -40072231012, -33760797412, -30605123812, -27449450212, -21138016612, -17982343012, -14826669412, -12219290212, -2208988800 ],
-  "diffs" : [ -175388, -88988, -2588, 83812, 170212, 256612, 343012, 429412, 515812, 602212, 688612, 775012, 861412, -2588, 0 ]
+  "switches" : [ -62135594212, -59006359012, -55850685412, -52695011812, -46383578212, -43227904612, -40072231012, -33760797412, -30605123812, -27449450212, -21138016612, -17982343012, -14826669412, -12220067812, -12219981412, -12219895012, -12219808612, -12219722212, -12219635812, -12219549412, -12219463012, -12219376612, -12219290212, -2208988800 ],
+  "diffs" : [ -175388, -88988, -2588, 83812, 170212, 256612, 343012, 429412, 515812, 602212, 688612, 775012, 861412, 775012, 688612, 602212, 515812, 429412, 343012, 256612, 170212, 83812, -2588, 0 ]
 }, {
   "tz" : "Africa/Nairobi",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Africa/Ndjamena",
-  "switches" : [ -62135600412, -59006365212, -55850691612, -52695018012, -46383584412, -43227910812, -40072237212, -33760803612, -30605130012, -27449456412, -21138022812, -17982349212, -14826675612, -12219296412, -2208988800 ],
-  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 12, 0 ]
+  "switches" : [ -62135600412, -59006365212, -55850691612, -52695018012, -46383584412, -43227910812, -40072237212, -33760803612, -30605130012, -27449456412, -21138022812, -17982349212, -14826675612, -12220074012, -12219987612, -12219901212, -12219814812, -12219728412, -12219642012, -12219555612, -12219469212, -12219382812, -12219296412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ]
 }, {
   "tz" : "Africa/Niamey",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Nouakchott",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Ouagadougou",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Porto-Novo",
-  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12219293616, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Africa/Sao_Tome",
-  "switches" : [ -62135598416, -59006363216, -55850689616, -52695016016, -46383582416, -43227908816, -40072235216, -33760801616, -30605128016, -27449454416, -21138020816, -17982347216, -14826673616, -12219294416, -2713912016, -2208988800 ],
-  "diffs" : [ -171184, -84784, 1616, 88016, 174416, 260816, 347216, 433616, 520016, 606416, 692816, 779216, 865616, 1616, -2205, 0 ]
+  "switches" : [ -62135598416, -59006363216, -55850689616, -52695016016, -46383582416, -43227908816, -40072235216, -33760801616, -30605128016, -27449454416, -21138020816, -17982347216, -14826673616, -12220072016, -12219985616, -12219899216, -12219812816, -12219726416, -12219640016, -12219553616, -12219467216, -12219380816, -12219294416, -2713912016, -2208988800 ],
+  "diffs" : [ -171184, -84784, 1616, 88016, 174416, 260816, 347216, 433616, 520016, 606416, 692816, 779216, 865616, 779216, 692816, 606416, 520016, 433616, 347216, 260816, 174416, 88016, 1616, -2205, 0 ]
 }, {
   "tz" : "Africa/Timbuktu",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Africa/Tripoli",
-  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12219295964, -2208988800 ],
-  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, -4036, 0 ]
+  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12220073564, -12219987164, -12219900764, -12219814364, -12219727964, -12219641564, -12219555164, -12219468764, -12219382364, -12219295964, -2208988800 ],
+  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, 773564, 687164, 600764, 514364, 427964, 341564, 255164, 168764, 82364, -4036, 0 ]
 }, {
   "tz" : "Africa/Tunis",
-  "switches" : [ -62135599244, -59006364044, -55850690444, -52695016844, -46383583244, -43227909644, -40072236044, -33760802444, -30605128844, -27449455244, -21138021644, -17982348044, -14826674444, -12219295244, -2797202444, -2208988800 ],
-  "diffs" : [ -173956, -87556, -1156, 85244, 171644, 258044, 344444, 430844, 517244, 603644, 690044, 776444, 862844, -1156, -3039, 0 ]
+  "switches" : [ -62135599244, -59006364044, -55850690444, -52695016844, -46383583244, -43227909644, -40072236044, -33760802444, -30605128844, -27449455244, -21138021644, -17982348044, -14826674444, -12220072844, -12219986444, -12219900044, -12219813644, -12219727244, -12219640844, -12219554444, -12219468044, -12219381644, -12219295244, -2797202444, -2208988800 ],
+  "diffs" : [ -173956, -87556, -1156, 85244, 171644, 258044, 344444, 430844, 517244, 603644, 690044, 776444, 862844, 776444, 690044, 603644, 517244, 430844, 344444, 258044, 171644, 85244, -1156, -3039, 0 ]
 }, {
   "tz" : "Africa/Windhoek",
-  "switches" : [ -62135600904, -59006365704, -55850692104, -52695018504, -46383584904, -43227911304, -40072237704, -33760804104, -30605130504, -27449456904, -21138023304, -17982349704, -14826676104, -12219296904, -2458170504, -2208988800 ],
-  "diffs" : [ -175896, -89496, -3096, 83304, 169704, 256104, 342504, 428904, 515304, 601704, 688104, 774504, 860904, -3096, -1800, 0 ]
+  "switches" : [ -62135600904, -59006365704, -55850692104, -52695018504, -46383584904, -43227911304, -40072237704, -33760804104, -30605130504, -27449456904, -21138023304, -17982349704, -14826676104, -12220074504, -12219988104, -12219901704, -12219815304, -12219728904, -12219642504, -12219556104, -12219469704, -12219383304, -12219296904, -2458170504, -2208988800 ],
+  "diffs" : [ -175896, -89496, -3096, 83304, 169704, 256104, 342504, 428904, 515304, 601704, 688104, 774504, 860904, 774504, 688104, 601704, 515304, 428904, 342504, 256104, 169704, 83304, -3096, -1800, 0 ]
 }, {
   "tz" : "America/Adak",
-  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12219336802, -3225223727, -2208988800 ],
-  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 80002, -6398, 0 ]
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 857602, 771202, 684802, 598402, 512002, 425602, 339202, 252802, 166402, 80002, -6398, 0 ]
 }, {
   "tz" : "America/Anchorage",
-  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12219343224, -3225223727, -2208988800 ],
-  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 82824, -3576, 0 ]
+  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12220120824, -12220034424, -12219948024, -12219861624, -12219775224, -12219688824, -12219602424, -12219516024, -12219429624, -12219343224, -3225223727, -2208988800 ],
+  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 860424, 774024, 687624, 601224, 514824, 428424, 342024, 255624, 169224, 82824, -3576, 0 ]
 }, {
   "tz" : "America/Anguilla",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Antigua",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Araguaina",
-  "switches" : [ -62135585232, -59006350032, -55850676432, -52695002832, -46383569232, -43227895632, -40072222032, -33760788432, -30605114832, -27449441232, -21138007632, -17982334032, -14826660432, -12219281232, -2208988800 ],
-  "diffs" : [ -173568, -87168, -768, 85632, 172032, 258432, 344832, 431232, 517632, 604032, 690432, 776832, 863232, -768, 0 ]
+  "switches" : [ -62135585232, -59006350032, -55850676432, -52695002832, -46383569232, -43227895632, -40072222032, -33760788432, -30605114832, -27449441232, -21138007632, -17982334032, -14826660432, -12220058832, -12219972432, -12219886032, -12219799632, -12219713232, -12219626832, -12219540432, -12219454032, -12219367632, -12219281232, -2208988800 ],
+  "diffs" : [ -173568, -87168, -768, 85632, 172032, 258432, 344832, 431232, 517632, 604032, 690432, 776832, 863232, 776832, 690432, 604032, 517632, 431232, 344832, 258432, 172032, 85632, -768, 0 ]
 }, {
   "tz" : "America/Argentina/Buenos_Aires",
-  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12219278772, -2372097972, -2208988800 ],
-  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, -3228, -4608, 0 ]
+  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12220056372, -12219969972, -12219883572, -12219797172, -12219710772, -12219624372, -12219537972, -12219451572, -12219365172, -12219278772, -2372097972, -2208988800 ],
+  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, 774372, 687972, 601572, 515172, 428772, 342372, 255972, 169572, 83172, -3228, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Catamarca",
-  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12219277012, -2372096212, -2208988800 ],
-  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, -4988, -4608, 0 ]
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12220054612, -12219968212, -12219881812, -12219795412, -12219709012, -12219622612, -12219536212, -12219449812, -12219363412, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, 772612, 686212, 599812, 513412, 427012, 340612, 254212, 167812, 81412, -4988, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/ComodRivadavia",
-  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12219277012, -2372096212, -2208988800 ],
-  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, -4988, -4608, 0 ]
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12220054612, -12219968212, -12219881812, -12219795412, -12219709012, -12219622612, -12219536212, -12219449812, -12219363412, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, 772612, 686212, 599812, 513412, 427012, 340612, 254212, 167812, 81412, -4988, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Cordoba",
-  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12219277392, -2208988800 ],
-  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, -4608, 0 ]
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, 772992, 686592, 600192, 513792, 427392, 340992, 254592, 168192, 81792, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Jujuy",
-  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12219277128, -2372096328, -2208988800 ],
-  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, -4872, -4608, 0 ]
+  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12220054728, -12219968328, -12219881928, -12219795528, -12219709128, -12219622728, -12219536328, -12219449928, -12219363528, -12219277128, -2372096328, -2208988800 ],
+  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, 772728, 686328, 599928, 513528, 427128, 340728, 254328, 167928, 81528, -4872, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/La_Rioja",
-  "switches" : [ -62135580756, -59006345556, -55850671956, -52694998356, -46383564756, -43227891156, -40072217556, -33760783956, -30605110356, -27449436756, -21138003156, -17982329556, -14826655956, -12219276756, -2372095956, -2208988800 ],
-  "diffs" : [ -178044, -91644, -5244, 81156, 167556, 253956, 340356, 426756, 513156, 599556, 685956, 772356, 858756, -5244, -4608, 0 ]
+  "switches" : [ -62135580756, -59006345556, -55850671956, -52694998356, -46383564756, -43227891156, -40072217556, -33760783956, -30605110356, -27449436756, -21138003156, -17982329556, -14826655956, -12220054356, -12219967956, -12219881556, -12219795156, -12219708756, -12219622356, -12219535956, -12219449556, -12219363156, -12219276756, -2372095956, -2208988800 ],
+  "diffs" : [ -178044, -91644, -5244, 81156, 167556, 253956, 340356, 426756, 513156, 599556, 685956, 772356, 858756, 772356, 685956, 599556, 513156, 426756, 340356, 253956, 167556, 81156, -5244, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Mendoza",
-  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12219276284, -2372095484, -2208988800 ],
-  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, -5716, -4608, 0 ]
+  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12220053884, -12219967484, -12219881084, -12219794684, -12219708284, -12219621884, -12219535484, -12219449084, -12219362684, -12219276284, -2372095484, -2208988800 ],
+  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, 771884, 685484, 599084, 512684, 426284, 339884, 253484, 167084, 80684, -5716, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Rio_Gallegos",
-  "switches" : [ -62135580188, -59006344988, -55850671388, -52694997788, -46383564188, -43227890588, -40072216988, -33760783388, -30605109788, -27449436188, -21138002588, -17982328988, -14826655388, -12219276188, -2372095388, -2208988800 ],
-  "diffs" : [ -178612, -92212, -5812, 80588, 166988, 253388, 339788, 426188, 512588, 598988, 685388, 771788, 858188, -5812, -4608, 0 ]
+  "switches" : [ -62135580188, -59006344988, -55850671388, -52694997788, -46383564188, -43227890588, -40072216988, -33760783388, -30605109788, -27449436188, -21138002588, -17982328988, -14826655388, -12220053788, -12219967388, -12219880988, -12219794588, -12219708188, -12219621788, -12219535388, -12219448988, -12219362588, -12219276188, -2372095388, -2208988800 ],
+  "diffs" : [ -178612, -92212, -5812, 80588, 166988, 253388, 339788, 426188, 512588, 598988, 685388, 771788, 858188, 771788, 685388, 598988, 512588, 426188, 339788, 253388, 166988, 80588, -5812, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Salta",
-  "switches" : [ -62135581100, -59006345900, -55850672300, -52694998700, -46383565100, -43227891500, -40072217900, -33760784300, -30605110700, -27449437100, -21138003500, -17982329900, -14826656300, -12219277100, -2372096300, -2208988800 ],
-  "diffs" : [ -177700, -91300, -4900, 81500, 167900, 254300, 340700, 427100, 513500, 599900, 686300, 772700, 859100, -4900, -4608, 0 ]
+  "switches" : [ -62135581100, -59006345900, -55850672300, -52694998700, -46383565100, -43227891500, -40072217900, -33760784300, -30605110700, -27449437100, -21138003500, -17982329900, -14826656300, -12220054700, -12219968300, -12219881900, -12219795500, -12219709100, -12219622700, -12219536300, -12219449900, -12219363500, -12219277100, -2372096300, -2208988800 ],
+  "diffs" : [ -177700, -91300, -4900, 81500, 167900, 254300, 340700, 427100, 513500, 599900, 686300, 772700, 859100, 772700, 686300, 599900, 513500, 427100, 340700, 254300, 167900, 81500, -4900, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/San_Juan",
-  "switches" : [ -62135580356, -59006345156, -55850671556, -52694997956, -46383564356, -43227890756, -40072217156, -33760783556, -30605109956, -27449436356, -21138002756, -17982329156, -14826655556, -12219276356, -2372095556, -2208988800 ],
-  "diffs" : [ -178444, -92044, -5644, 80756, 167156, 253556, 339956, 426356, 512756, 599156, 685556, 771956, 858356, -5644, -4608, 0 ]
+  "switches" : [ -62135580356, -59006345156, -55850671556, -52694997956, -46383564356, -43227890756, -40072217156, -33760783556, -30605109956, -27449436356, -21138002756, -17982329156, -14826655556, -12220053956, -12219967556, -12219881156, -12219794756, -12219708356, -12219621956, -12219535556, -12219449156, -12219362756, -12219276356, -2372095556, -2208988800 ],
+  "diffs" : [ -178444, -92044, -5644, 80756, 167156, 253556, 339956, 426356, 512756, 599156, 685556, 771956, 858356, 771956, 685556, 599156, 512756, 426356, 339956, 253556, 167156, 80756, -5644, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/San_Luis",
-  "switches" : [ -62135580876, -59006345676, -55850672076, -52694998476, -46383564876, -43227891276, -40072217676, -33760784076, -30605110476, -27449436876, -21138003276, -17982329676, -14826656076, -12219276876, -2372096076, -2208988800 ],
-  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, -5124, -4608, 0 ]
+  "switches" : [ -62135580876, -59006345676, -55850672076, -52694998476, -46383564876, -43227891276, -40072217676, -33760784076, -30605110476, -27449436876, -21138003276, -17982329676, -14826656076, -12220054476, -12219968076, -12219881676, -12219795276, -12219708876, -12219622476, -12219536076, -12219449676, -12219363276, -12219276876, -2372096076, -2208988800 ],
+  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, 772476, 686076, 599676, 513276, 426876, 340476, 254076, 167676, 81276, -5124, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Tucuman",
-  "switches" : [ -62135581148, -59006345948, -55850672348, -52694998748, -46383565148, -43227891548, -40072217948, -33760784348, -30605110748, -27449437148, -21138003548, -17982329948, -14826656348, -12219277148, -2372096348, -2208988800 ],
-  "diffs" : [ -177652, -91252, -4852, 81548, 167948, 254348, 340748, 427148, 513548, 599948, 686348, 772748, 859148, -4852, -4608, 0 ]
+  "switches" : [ -62135581148, -59006345948, -55850672348, -52694998748, -46383565148, -43227891548, -40072217948, -33760784348, -30605110748, -27449437148, -21138003548, -17982329948, -14826656348, -12220054748, -12219968348, -12219881948, -12219795548, -12219709148, -12219622748, -12219536348, -12219449948, -12219363548, -12219277148, -2372096348, -2208988800 ],
+  "diffs" : [ -177652, -91252, -4852, 81548, 167948, 254348, 340748, 427148, 513548, 599948, 686348, 772748, 859148, 772748, 686348, 599948, 513548, 427148, 340748, 254348, 167948, 81548, -4852, -4608, 0 ]
 }, {
   "tz" : "America/Argentina/Ushuaia",
-  "switches" : [ -62135580408, -59006345208, -55850671608, -52694998008, -46383564408, -43227890808, -40072217208, -33760783608, -30605110008, -27449436408, -21138002808, -17982329208, -14826655608, -12219276408, -2372095608, -2208988800 ],
-  "diffs" : [ -178392, -91992, -5592, 80808, 167208, 253608, 340008, 426408, 512808, 599208, 685608, 772008, 858408, -5592, -4608, 0 ]
+  "switches" : [ -62135580408, -59006345208, -55850671608, -52694998008, -46383564408, -43227890808, -40072217208, -33760783608, -30605110008, -27449436408, -21138002808, -17982329208, -14826655608, -12220054008, -12219967608, -12219881208, -12219794808, -12219708408, -12219622008, -12219535608, -12219449208, -12219362808, -12219276408, -2372095608, -2208988800 ],
+  "diffs" : [ -178392, -91992, -5592, 80808, 167208, 253608, 340008, 426408, 512808, 599208, 685608, 772008, 858408, 772008, 685608, 599208, 512808, 426408, 340008, 253608, 167208, 80808, -5592, -4608, 0 ]
 }, {
   "tz" : "America/Aruba",
-  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
-  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
 }, {
   "tz" : "America/Asuncion",
-  "switches" : [ -62135582960, -59006347760, -55850674160, -52695000560, -46383566960, -43227893360, -40072219760, -33760786160, -30605112560, -27449438960, -21138005360, -17982331760, -14826658160, -12219278960, -2208988800 ],
-  "diffs" : [ -172240, -85840, 560, 86960, 173360, 259760, 346160, 432560, 518960, 605360, 691760, 778160, 864560, 560, 0 ]
+  "switches" : [ -62135582960, -59006347760, -55850674160, -52695000560, -46383566960, -43227893360, -40072219760, -33760786160, -30605112560, -27449438960, -21138005360, -17982331760, -14826658160, -12220056560, -12219970160, -12219883760, -12219797360, -12219710960, -12219624560, -12219538160, -12219451760, -12219365360, -12219278960, -2208988800 ],
+  "diffs" : [ -172240, -85840, 560, 86960, 173360, 259760, 346160, 432560, 518960, 605360, 691760, 778160, 864560, 778160, 691760, 605360, 518960, 432560, 346160, 259760, 173360, 86960, 560, 0 ]
 }, {
   "tz" : "America/Atikokan",
-  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12219270812, -2366733212, -2208988800 ],
-  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, -3600, 0 ]
+  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12220048412, -12219962012, -12219875612, -12219789212, -12219702812, -12219616412, -12219530012, -12219443612, -12219357212, -12219270812, -2366733212, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, -3600, 0 ]
 }, {
   "tz" : "America/Atka",
-  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12219336802, -3225223727, -2208988800 ],
-  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 80002, -6398, 0 ]
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 857602, 771202, 684802, 598402, 512002, 425602, 339202, 252802, 166402, 80002, -6398, 0 ]
 }, {
   "tz" : "America/Bahia",
-  "switches" : [ -62135587556, -59006352356, -55850678756, -52695005156, -46383571556, -43227897956, -40072224356, -33760790756, -30605117156, -27449443556, -21138009956, -17982336356, -14826662756, -12219283556, -2208988800 ],
-  "diffs" : [ -171244, -84844, 1556, 87956, 174356, 260756, 347156, 433556, 519956, 606356, 692756, 779156, 865556, 1556, 0 ]
+  "switches" : [ -62135587556, -59006352356, -55850678756, -52695005156, -46383571556, -43227897956, -40072224356, -33760790756, -30605117156, -27449443556, -21138009956, -17982336356, -14826662756, -12220061156, -12219974756, -12219888356, -12219801956, -12219715556, -12219629156, -12219542756, -12219456356, -12219369956, -12219283556, -2208988800 ],
+  "diffs" : [ -171244, -84844, 1556, 87956, 174356, 260756, 347156, 433556, 519956, 606356, 692756, 779156, 865556, 779156, 692756, 606356, 519956, 433556, 347156, 260756, 174356, 87956, 1556, 0 ]
 }, {
   "tz" : "America/Bahia_Banderas",
-  "switches" : [ -62135571540, -59006336340, -55850662740, -52694989140, -46383555540, -43227881940, -40072208340, -33760774740, -30605101140, -27449427540, -21137993940, -17982320340, -14826646740, -12219267540, -2208988800 ],
-  "diffs" : [ -176460, -90060, -3660, 82740, 169140, 255540, 341940, 428340, 514740, 601140, 687540, 773940, 860340, -3660, 0 ]
+  "switches" : [ -62135571540, -59006336340, -55850662740, -52694989140, -46383555540, -43227881940, -40072208340, -33760774740, -30605101140, -27449427540, -21137993940, -17982320340, -14826646740, -12220045140, -12219958740, -12219872340, -12219785940, -12219699540, -12219613140, -12219526740, -12219440340, -12219353940, -12219267540, -2208988800 ],
+  "diffs" : [ -176460, -90060, -3660, 82740, 169140, 255540, 341940, 428340, 514740, 601140, 687540, 773940, 860340, 773940, 687540, 601140, 514740, 428340, 341940, 255540, 169140, 82740, -3660, 0 ]
 }, {
   "tz" : "America/Barbados",
-  "switches" : [ -62135582491, -59006347291, -55850673691, -52695000091, -46383566491, -43227892891, -40072219291, -33760785691, -30605112091, -27449438491, -21138004891, -17982331291, -14826657691, -12219278491, -2208988800 ],
-  "diffs" : [ -172709, -86309, 91, 86491, 172891, 259291, 345691, 432091, 518491, 604891, 691291, 777691, 864091, 91, 0 ]
+  "switches" : [ -62135582491, -59006347291, -55850673691, -52695000091, -46383566491, -43227892891, -40072219291, -33760785691, -30605112091, -27449438491, -21138004891, -17982331291, -14826657691, -12220056091, -12219969691, -12219883291, -12219796891, -12219710491, -12219624091, -12219537691, -12219451291, -12219364891, -12219278491, -2208988800 ],
+  "diffs" : [ -172709, -86309, 91, 86491, 172891, 259291, 345691, 432091, 518491, 604891, 691291, 777691, 864091, 777691, 691291, 604891, 518491, 432091, 345691, 259291, 172891, 86491, 91, 0 ]
 }, {
   "tz" : "America/Belem",
-  "switches" : [ -62135585164, -59006349964, -55850676364, -52695002764, -46383569164, -43227895564, -40072221964, -33760788364, -30605114764, -27449441164, -21138007564, -17982333964, -14826660364, -12219281164, -2208988800 ],
-  "diffs" : [ -173636, -87236, -836, 85564, 171964, 258364, 344764, 431164, 517564, 603964, 690364, 776764, 863164, -836, 0 ]
+  "switches" : [ -62135585164, -59006349964, -55850676364, -52695002764, -46383569164, -43227895564, -40072221964, -33760788364, -30605114764, -27449441164, -21138007564, -17982333964, -14826660364, -12220058764, -12219972364, -12219885964, -12219799564, -12219713164, -12219626764, -12219540364, -12219453964, -12219367564, -12219281164, -2208988800 ],
+  "diffs" : [ -173636, -87236, -836, 85564, 171964, 258364, 344764, 431164, 517564, 603964, 690364, 776764, 863164, 776764, 690364, 603964, 517564, 431164, 344764, 258364, 171964, 85564, -836, 0 ]
 }, {
   "tz" : "America/Belize",
-  "switches" : [ -62135575632, -59006340432, -55850666832, -52694993232, -46383559632, -43227886032, -40072212432, -33760778832, -30605105232, -27449431632, -21137998032, -17982324432, -14826650832, -12219271632, -2208988800 ],
-  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 432, 0 ]
+  "switches" : [ -62135575632, -59006340432, -55850666832, -52694993232, -46383559632, -43227886032, -40072212432, -33760778832, -30605105232, -27449431632, -21137998032, -17982324432, -14826650832, -12220049232, -12219962832, -12219876432, -12219790032, -12219703632, -12219617232, -12219530832, -12219444432, -12219358032, -12219271632, -2208988800 ],
+  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 778032, 691632, 605232, 518832, 432432, 346032, 259632, 173232, 86832, 432, 0 ]
 }, {
   "tz" : "America/Blanc-Sablon",
-  "switches" : [ -62135583092, -59006347892, -55850674292, -52695000692, -46383567092, -43227893492, -40072219892, -33760786292, -30605112692, -27449439092, -21138005492, -17982331892, -14826658292, -12219279092, -2713896692 ],
-  "diffs" : [ -172108, -85708, 692, 87092, 173492, 259892, 346292, 432692, 519092, 605492, 691892, 778292, 864692, 692, 0 ]
+  "switches" : [ -62135583092, -59006347892, -55850674292, -52695000692, -46383567092, -43227893492, -40072219892, -33760786292, -30605112692, -27449439092, -21138005492, -17982331892, -14826658292, -12220056692, -12219970292, -12219883892, -12219797492, -12219711092, -12219624692, -12219538292, -12219451892, -12219365492, -12219279092, -2713896692 ],
+  "diffs" : [ -172108, -85708, 692, 87092, 173492, 259892, 346292, 432692, 519092, 605492, 691892, 778292, 864692, 778292, 691892, 605492, 519092, 432692, 346292, 259892, 173492, 87092, 692, 0 ]
 }, {
   "tz" : "America/Boa_Vista",
-  "switches" : [ -62135582240, -59006347040, -55850673440, -52694999840, -46383566240, -43227892640, -40072219040, -33760785440, -30605111840, -27449438240, -21138004640, -17982331040, -14826657440, -12219278240, -2208988800 ],
-  "diffs" : [ -172960, -86560, -160, 86240, 172640, 259040, 345440, 431840, 518240, 604640, 691040, 777440, 863840, -160, 0 ]
+  "switches" : [ -62135582240, -59006347040, -55850673440, -52694999840, -46383566240, -43227892640, -40072219040, -33760785440, -30605111840, -27449438240, -21138004640, -17982331040, -14826657440, -12220055840, -12219969440, -12219883040, -12219796640, -12219710240, -12219623840, -12219537440, -12219451040, -12219364640, -12219278240, -2208988800 ],
+  "diffs" : [ -172960, -86560, -160, 86240, 172640, 259040, 345440, 431840, 518240, 604640, 691040, 777440, 863840, 777440, 691040, 604640, 518240, 431840, 345440, 259040, 172640, 86240, -160, 0 ]
 }, {
   "tz" : "America/Bogota",
-  "switches" : [ -62135579024, -59006343824, -55850670224, -52694996624, -46383563024, -43227889424, -40072215824, -33760782224, -30605108624, -27449435024, -21138001424, -17982327824, -14826654224, -12219275024, -2208988800 ],
-  "diffs" : [ -172576, -86176, 224, 86624, 173024, 259424, 345824, 432224, 518624, 605024, 691424, 777824, 864224, 224, 0 ]
+  "switches" : [ -62135579024, -59006343824, -55850670224, -52694996624, -46383563024, -43227889424, -40072215824, -33760782224, -30605108624, -27449435024, -21138001424, -17982327824, -14826654224, -12220052624, -12219966224, -12219879824, -12219793424, -12219707024, -12219620624, -12219534224, -12219447824, -12219361424, -12219275024, -2208988800 ],
+  "diffs" : [ -172576, -86176, 224, 86624, 173024, 259424, 345824, 432224, 518624, 605024, 691424, 777824, 864224, 777824, 691424, 605024, 518624, 432224, 345824, 259424, 173024, 86624, 224, 0 ]
 }, {
   "tz" : "America/Boise",
-  "switches" : [ -62135568911, -59006333711, -55850660111, -52694986511, -46383552911, -43227879311, -40072205711, -33760772111, -30605098511, -27449424911, -21137991311, -17982317711, -14826644111, -12219264911, -2717640000, -2208988800 ],
-  "diffs" : [ -175489, -89089, -2689, 83711, 170111, 256511, 342911, 429311, 515711, 602111, 688511, 774911, 861311, -2689, -3600, 0 ]
+  "switches" : [ -62135568911, -59006333711, -55850660111, -52694986511, -46383552911, -43227879311, -40072205711, -33760772111, -30605098511, -27449424911, -21137991311, -17982317711, -14826644111, -12220042511, -12219956111, -12219869711, -12219783311, -12219696911, -12219610511, -12219524111, -12219437711, -12219351311, -12219264911, -2717640000, -2208988800 ],
+  "diffs" : [ -175489, -89089, -2689, 83711, 170111, 256511, 342911, 429311, 515711, 602111, 688511, 774911, 861311, 774911, 688511, 602111, 515711, 429311, 342911, 256511, 170111, 83711, -2689, -3600, 0 ]
 }, {
   "tz" : "America/Buenos_Aires",
-  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12219278772, -2372097972, -2208988800 ],
-  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, -3228, -4608, 0 ]
+  "switches" : [ -62135582772, -59006347572, -55850673972, -52695000372, -46383566772, -43227893172, -40072219572, -33760785972, -30605112372, -27449438772, -21138005172, -17982331572, -14826657972, -12220056372, -12219969972, -12219883572, -12219797172, -12219710772, -12219624372, -12219537972, -12219451572, -12219365172, -12219278772, -2372097972, -2208988800 ],
+  "diffs" : [ -176028, -89628, -3228, 83172, 169572, 255972, 342372, 428772, 515172, 601572, 687972, 774372, 860772, 774372, 687972, 601572, 515172, 428772, 342372, 255972, 169572, 83172, -3228, -4608, 0 ]
 }, {
   "tz" : "America/Cambridge_Bay",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 25200, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 802800, 716400, 630000, 543600, 457200, 370800, 284400, 198000, 111600, 25200, 0 ]
 }, {
   "tz" : "America/Campo_Grande",
-  "switches" : [ -62135583692, -59006348492, -55850674892, -52695001292, -46383567692, -43227894092, -40072220492, -33760786892, -30605113292, -27449439692, -21138006092, -17982332492, -14826658892, -12219279692, -2208988800 ],
-  "diffs" : [ -171508, -85108, 1292, 87692, 174092, 260492, 346892, 433292, 519692, 606092, 692492, 778892, 865292, 1292, 0 ]
+  "switches" : [ -62135583692, -59006348492, -55850674892, -52695001292, -46383567692, -43227894092, -40072220492, -33760786892, -30605113292, -27449439692, -21138006092, -17982332492, -14826658892, -12220057292, -12219970892, -12219884492, -12219798092, -12219711692, -12219625292, -12219538892, -12219452492, -12219366092, -12219279692, -2208988800 ],
+  "diffs" : [ -171508, -85108, 1292, 87692, 174092, 260492, 346892, 433292, 519692, 606092, 692492, 778892, 865292, 778892, 692492, 606092, 519692, 433292, 346892, 260492, 174092, 87692, 1292, 0 ]
 }, {
   "tz" : "America/Cancun",
-  "switches" : [ -62135575976, -59006340776, -55850667176, -52694993576, -46383559976, -43227886376, -40072212776, -33760779176, -30605105576, -27449431976, -21137998376, -17982324776, -14826651176, -12219271976, -2208988800 ],
-  "diffs" : [ -175624, -89224, -2824, 83576, 169976, 256376, 342776, 429176, 515576, 601976, 688376, 774776, 861176, -2824, 0 ]
+  "switches" : [ -62135575976, -59006340776, -55850667176, -52694993576, -46383559976, -43227886376, -40072212776, -33760779176, -30605105576, -27449431976, -21137998376, -17982324776, -14826651176, -12220049576, -12219963176, -12219876776, -12219790376, -12219703976, -12219617576, -12219531176, -12219444776, -12219358376, -12219271976, -2208988800 ],
+  "diffs" : [ -175624, -89224, -2824, 83576, 169976, 256376, 342776, 429176, 515576, 601976, 688376, 774776, 861176, 774776, 688376, 601976, 515576, 429176, 342776, 256376, 169976, 83576, -2824, 0 ]
 }, {
   "tz" : "America/Caracas",
-  "switches" : [ -62135580736, -59006345536, -55850671936, -52694998336, -46383564736, -43227891136, -40072217536, -33760783936, -30605110336, -27449436736, -21138003136, -17982329536, -14826655936, -12219276736, -2524505536, -2208988800 ],
-  "diffs" : [ -174464, -88064, -1664, 84736, 171136, 257536, 343936, 430336, 516736, 603136, 689536, 775936, 862336, -1664, -1660, 0 ]
+  "switches" : [ -62135580736, -59006345536, -55850671936, -52694998336, -46383564736, -43227891136, -40072217536, -33760783936, -30605110336, -27449436736, -21138003136, -17982329536, -14826655936, -12220054336, -12219967936, -12219881536, -12219795136, -12219708736, -12219622336, -12219535936, -12219449536, -12219363136, -12219276736, -2524505536, -2208988800 ],
+  "diffs" : [ -174464, -88064, -1664, 84736, 171136, 257536, 343936, 430336, 516736, 603136, 689536, 775936, 862336, 775936, 689536, 603136, 516736, 430336, 343936, 257536, 171136, 84736, -1664, -1660, 0 ]
 }, {
   "tz" : "America/Catamarca",
-  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12219277012, -2372096212, -2208988800 ],
-  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, -4988, -4608, 0 ]
+  "switches" : [ -62135581012, -59006345812, -55850672212, -52694998612, -46383565012, -43227891412, -40072217812, -33760784212, -30605110612, -27449437012, -21138003412, -17982329812, -14826656212, -12220054612, -12219968212, -12219881812, -12219795412, -12219709012, -12219622612, -12219536212, -12219449812, -12219363412, -12219277012, -2372096212, -2208988800 ],
+  "diffs" : [ -177788, -91388, -4988, 81412, 167812, 254212, 340612, 427012, 513412, 599812, 686212, 772612, 859012, 772612, 686212, 599812, 513412, 427012, 340612, 254212, 167812, 81412, -4988, -4608, 0 ]
 }, {
   "tz" : "America/Cayenne",
-  "switches" : [ -62135584240, -59006349040, -55850675440, -52695001840, -46383568240, -43227894640, -40072221040, -33760787440, -30605113840, -27449440240, -21138006640, -17982333040, -14826659440, -12219280240, -2208988800 ],
-  "diffs" : [ -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, 862240, -1760, 0 ]
+  "switches" : [ -62135584240, -59006349040, -55850675440, -52695001840, -46383568240, -43227894640, -40072221040, -33760787440, -30605113840, -27449440240, -21138006640, -17982333040, -14826659440, -12220057840, -12219971440, -12219885040, -12219798640, -12219712240, -12219625840, -12219539440, -12219453040, -12219366640, -12219280240, -2208988800 ],
+  "diffs" : [ -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, 862240, 775840, 689440, 603040, 516640, 430240, 343840, 257440, 171040, 84640, -1760, 0 ]
 }, {
   "tz" : "America/Cayman",
-  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12219273712, -2524502512, -2208988800 ],
-  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, -1088, -1176, 0 ]
+  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12220051312, -12219964912, -12219878512, -12219792112, -12219705712, -12219619312, -12219532912, -12219446512, -12219360112, -12219273712, -2524502512, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, -1176, 0 ]
 }, {
   "tz" : "America/Chicago",
-  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12219271764, -2717647200 ],
-  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 564, 0 ]
+  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12220049364, -12219962964, -12219876564, -12219790164, -12219703764, -12219617364, -12219530964, -12219444564, -12219358164, -12219271764, -2717647200 ],
+  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 778164, 691764, 605364, 518964, 432564, 346164, 259764, 173364, 86964, 564, 0 ]
 }, {
   "tz" : "America/Chihuahua",
-  "switches" : [ -62135571340, -59006336140, -55850662540, -52694988940, -46383555340, -43227881740, -40072208140, -33760774540, -30605100940, -27449427340, -21137993740, -17982320140, -14826646540, -12219267340, -2208988800 ],
-  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, -260, 0 ]
+  "switches" : [ -62135571340, -59006336140, -55850662540, -52694988940, -46383555340, -43227881740, -40072208140, -33760774540, -30605100940, -27449427340, -21137993740, -17982320140, -14826646540, -12220044940, -12219958540, -12219872140, -12219785740, -12219699340, -12219612940, -12219526540, -12219440140, -12219353740, -12219267340, -2208988800 ],
+  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, 777340, 690940, 604540, 518140, 431740, 345340, 258940, 172540, 86140, -260, 0 ]
 }, {
   "tz" : "America/Coral_Harbour",
-  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12219270812, -2366733212, -2208988800 ],
-  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, -3600, 0 ]
+  "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12220048412, -12219962012, -12219875612, -12219789212, -12219702812, -12219616412, -12219530012, -12219443612, -12219357212, -12219270812, -2366733212, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, -3600, 0 ]
 }, {
   "tz" : "America/Cordoba",
-  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12219277392, -2208988800 ],
-  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, -4608, 0 ]
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, 772992, 686592, 600192, 513792, 427392, 340992, 254592, 168192, 81792, -4608, 0 ]
 }, {
   "tz" : "America/Costa_Rica",
-  "switches" : [ -62135576627, -59006341427, -55850667827, -52694994227, -46383560627, -43227887027, -40072213427, -33760779827, -30605106227, -27449432627, -21137999027, -17982325427, -14826651827, -12219272627, -2208988800 ],
-  "diffs" : [ -171373, -84973, 1427, 87827, 174227, 260627, 347027, 433427, 519827, 606227, 692627, 779027, 865427, 1427, 0 ]
+  "switches" : [ -62135576627, -59006341427, -55850667827, -52694994227, -46383560627, -43227887027, -40072213427, -33760779827, -30605106227, -27449432627, -21137999027, -17982325427, -14826651827, -12220050227, -12219963827, -12219877427, -12219791027, -12219704627, -12219618227, -12219531827, -12219445427, -12219359027, -12219272627, -2208988800 ],
+  "diffs" : [ -171373, -84973, 1427, 87827, 174227, 260627, 347027, 433427, 519827, 606227, 692627, 779027, 865427, 779027, 692627, 606227, 519827, 433427, 347027, 260627, 174227, 87827, 1427, 0 ]
 }, {
   "tz" : "America/Creston",
-  "switches" : [ -62135568836, -59006333636, -55850660036, -52694986436, -46383552836, -43227879236, -40072205636, -33760772036, -30605098436, -27449424836, -21137991236, -17982317636, -14826644036, -12219264836, -2713882436 ],
-  "diffs" : [ -175564, -89164, -2764, 83636, 170036, 256436, 342836, 429236, 515636, 602036, 688436, 774836, 861236, -2764, 0 ]
+  "switches" : [ -62135568836, -59006333636, -55850660036, -52694986436, -46383552836, -43227879236, -40072205636, -33760772036, -30605098436, -27449424836, -21137991236, -17982317636, -14826644036, -12220042436, -12219956036, -12219869636, -12219783236, -12219696836, -12219610436, -12219524036, -12219437636, -12219351236, -12219264836, -2713882436 ],
+  "diffs" : [ -175564, -89164, -2764, 83636, 170036, 256436, 342836, 429236, 515636, 602036, 688436, 774836, 861236, 774836, 688436, 602036, 515636, 429236, 342836, 256436, 170036, 83636, -2764, 0 ]
 }, {
   "tz" : "America/Cuiaba",
-  "switches" : [ -62135583340, -59006348140, -55850674540, -52695000940, -46383567340, -43227893740, -40072220140, -33760786540, -30605112940, -27449439340, -21138005740, -17982332140, -14826658540, -12219279340, -2208988800 ],
-  "diffs" : [ -171860, -85460, 940, 87340, 173740, 260140, 346540, 432940, 519340, 605740, 692140, 778540, 864940, 940, 0 ]
+  "switches" : [ -62135583340, -59006348140, -55850674540, -52695000940, -46383567340, -43227893740, -40072220140, -33760786540, -30605112940, -27449439340, -21138005740, -17982332140, -14826658540, -12220056940, -12219970540, -12219884140, -12219797740, -12219711340, -12219624940, -12219538540, -12219452140, -12219365740, -12219279340, -2208988800 ],
+  "diffs" : [ -171860, -85460, 940, 87340, 173740, 260140, 346540, 432940, 519340, 605740, 692140, 778540, 864940, 778540, 692140, 605740, 519340, 432940, 346540, 260140, 173740, 87340, 940, 0 ]
 }, {
   "tz" : "America/Curacao",
-  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
-  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
 }, {
   "tz" : "America/Danmarkshavn",
-  "switches" : [ -62135592320, -59006357120, -55850683520, -52695009920, -46383576320, -43227902720, -40072229120, -33760795520, -30605121920, -27449448320, -21138014720, -17982341120, -14826667520, -12219288320, -2208988800 ],
-  "diffs" : [ -177280, -90880, -4480, 81920, 168320, 254720, 341120, 427520, 513920, 600320, 686720, 773120, 859520, -4480, 0 ]
+  "switches" : [ -62135592320, -59006357120, -55850683520, -52695009920, -46383576320, -43227902720, -40072229120, -33760795520, -30605121920, -27449448320, -21138014720, -17982341120, -14826667520, -12220065920, -12219979520, -12219893120, -12219806720, -12219720320, -12219633920, -12219547520, -12219461120, -12219374720, -12219288320, -2208988800 ],
+  "diffs" : [ -177280, -90880, -4480, 81920, 168320, 254720, 341120, 427520, 513920, 600320, 686720, 773120, 859520, 773120, 686720, 600320, 513920, 427520, 341120, 254720, 168320, 81920, -4480, 0 ]
 }, {
   "tz" : "America/Dawson",
-  "switches" : [ -62135563340, -59006328140, -55850654540, -52694980940, -46383547340, -43227873740, -40072200140, -33760766540, -30605092940, -27449419340, -21137985740, -17982312140, -14826638540, -12219259340, -2208988800 ],
-  "diffs" : [ -177460, -91060, -4660, 81740, 168140, 254540, 340940, 427340, 513740, 600140, 686540, 772940, 859340, -4660, 0 ]
+  "switches" : [ -62135563340, -59006328140, -55850654540, -52694980940, -46383547340, -43227873740, -40072200140, -33760766540, -30605092940, -27449419340, -21137985740, -17982312140, -14826638540, -12220036940, -12219950540, -12219864140, -12219777740, -12219691340, -12219604940, -12219518540, -12219432140, -12219345740, -12219259340, -2208988800 ],
+  "diffs" : [ -177460, -91060, -4660, 81740, 168140, 254540, 340940, 427340, 513740, 600140, 686540, 772940, 859340, 772940, 686540, 600140, 513740, 427340, 340940, 254540, 168140, 81740, -4660, 0 ]
 }, {
   "tz" : "America/Dawson_Creek",
-  "switches" : [ -62135567944, -59006332744, -55850659144, -52694985544, -46383551944, -43227878344, -40072204744, -33760771144, -30605097544, -27449423944, -21137990344, -17982316744, -14826643144, -12219263944, -2713881544, -2208988800 ],
-  "diffs" : [ -176456, -90056, -3656, 82744, 169144, 255544, 341944, 428344, 514744, 601144, 687544, 773944, 860344, -3656, -3600, 0 ]
+  "switches" : [ -62135567944, -59006332744, -55850659144, -52694985544, -46383551944, -43227878344, -40072204744, -33760771144, -30605097544, -27449423944, -21137990344, -17982316744, -14826643144, -12220041544, -12219955144, -12219868744, -12219782344, -12219695944, -12219609544, -12219523144, -12219436744, -12219350344, -12219263944, -2713881544, -2208988800 ],
+  "diffs" : [ -176456, -90056, -3656, 82744, 169144, 255544, 341944, 428344, 514744, 601144, 687544, 773944, 860344, 773944, 687544, 601144, 514744, 428344, 341944, 255544, 169144, 82744, -3656, -3600, 0 ]
 }, {
   "tz" : "America/Denver",
-  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
-  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
 }, {
   "tz" : "America/Detroit",
-  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12219272869, -2208988800 ],
-  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, -1931, 0 ]
+  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12220050469, -12219964069, -12219877669, -12219791269, -12219704869, -12219618469, -12219532069, -12219445669, -12219359269, -12219272869, -2208988800 ],
+  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, 775669, 689269, 602869, 516469, 430069, 343669, 257269, 170869, 84469, -1931, 0 ]
 }, {
   "tz" : "America/Dominica",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Edmonton",
-  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12219265568, -2208988800 ],
-  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, -2032, 0 ]
+  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12220043168, -12219956768, -12219870368, -12219783968, -12219697568, -12219611168, -12219524768, -12219438368, -12219351968, -12219265568, -2208988800 ],
+  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, 775568, 689168, 602768, 516368, 429968, 343568, 257168, 170768, 84368, -2032, 0 ]
 }, {
   "tz" : "America/Eirunepe",
-  "switches" : [ -62135580032, -59006344832, -55850671232, -52694997632, -46383564032, -43227890432, -40072216832, -33760783232, -30605109632, -27449436032, -21138002432, -17982328832, -14826655232, -12219276032, -2208988800 ],
-  "diffs" : [ -171568, -85168, 1232, 87632, 174032, 260432, 346832, 433232, 519632, 606032, 692432, 778832, 865232, 1232, 0 ]
+  "switches" : [ -62135580032, -59006344832, -55850671232, -52694997632, -46383564032, -43227890432, -40072216832, -33760783232, -30605109632, -27449436032, -21138002432, -17982328832, -14826655232, -12220053632, -12219967232, -12219880832, -12219794432, -12219708032, -12219621632, -12219535232, -12219448832, -12219362432, -12219276032, -2208988800 ],
+  "diffs" : [ -171568, -85168, 1232, 87632, 174032, 260432, 346832, 433232, 519632, 606032, 692432, 778832, 865232, 778832, 692432, 606032, 519632, 433232, 346832, 260432, 174032, 87632, 1232, 0 ]
 }, {
   "tz" : "America/El_Salvador",
-  "switches" : [ -62135575392, -59006340192, -55850666592, -52694992992, -46383559392, -43227885792, -40072212192, -33760778592, -30605104992, -27449431392, -21137997792, -17982324192, -14826650592, -12219271392, -2208988800 ],
-  "diffs" : [ -172608, -86208, 192, 86592, 172992, 259392, 345792, 432192, 518592, 604992, 691392, 777792, 864192, 192, 0 ]
+  "switches" : [ -62135575392, -59006340192, -55850666592, -52694992992, -46383559392, -43227885792, -40072212192, -33760778592, -30605104992, -27449431392, -21137997792, -17982324192, -14826650592, -12220048992, -12219962592, -12219876192, -12219789792, -12219703392, -12219616992, -12219530592, -12219444192, -12219357792, -12219271392, -2208988800 ],
+  "diffs" : [ -172608, -86208, 192, 86592, 172992, 259392, 345792, 432192, 518592, 604992, 691392, 777792, 864192, 777792, 691392, 604992, 518592, 432192, 345792, 259392, 172992, 86592, 192, 0 ]
 }, {
   "tz" : "America/Ensenada",
-  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
-  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
 }, {
   "tz" : "America/Fort_Nelson",
-  "switches" : [ -62135567353, -59006332153, -55850658553, -52694984953, -46383551353, -43227877753, -40072204153, -33760770553, -30605096953, -27449423353, -21137989753, -17982316153, -14826642553, -12219263353, -2713880953, -2208988800 ],
-  "diffs" : [ -177047, -90647, -4247, 82153, 168553, 254953, 341353, 427753, 514153, 600553, 686953, 773353, 859753, -4247, -3600, 0 ]
+  "switches" : [ -62135567353, -59006332153, -55850658553, -52694984953, -46383551353, -43227877753, -40072204153, -33760770553, -30605096953, -27449423353, -21137989753, -17982316153, -14826642553, -12220040953, -12219954553, -12219868153, -12219781753, -12219695353, -12219608953, -12219522553, -12219436153, -12219349753, -12219263353, -2713880953, -2208988800 ],
+  "diffs" : [ -177047, -90647, -4247, 82153, 168553, 254953, 341353, 427753, 514153, 600553, 686953, 773353, 859753, 773353, 686953, 600553, 514153, 427753, 341353, 254953, 168553, 82153, -4247, -3600, 0 ]
 }, {
   "tz" : "America/Fort_Wayne",
-  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
-  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
 }, {
   "tz" : "America/Fortaleza",
-  "switches" : [ -62135587560, -59006352360, -55850678760, -52695005160, -46383571560, -43227897960, -40072224360, -33760790760, -30605117160, -27449443560, -21138009960, -17982336360, -14826662760, -12219283560, -2208988800 ],
-  "diffs" : [ -171240, -84840, 1560, 87960, 174360, 260760, 347160, 433560, 519960, 606360, 692760, 779160, 865560, 1560, 0 ]
+  "switches" : [ -62135587560, -59006352360, -55850678760, -52695005160, -46383571560, -43227897960, -40072224360, -33760790760, -30605117160, -27449443560, -21138009960, -17982336360, -14826662760, -12220061160, -12219974760, -12219888360, -12219801960, -12219715560, -12219629160, -12219542760, -12219456360, -12219369960, -12219283560, -2208988800 ],
+  "diffs" : [ -171240, -84840, 1560, 87960, 174360, 260760, 347160, 433560, 519960, 606360, 692760, 779160, 865560, 779160, 692760, 606360, 519960, 433560, 347160, 260760, 174360, 87960, 1560, 0 ]
 }, {
   "tz" : "America/Glace_Bay",
-  "switches" : [ -62135582412, -59006347212, -55850673612, -52695000012, -46383566412, -43227892812, -40072219212, -33760785612, -30605112012, -27449438412, -21138004812, -17982331212, -14826657612, -12219278412, -2208988800 ],
-  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 12, 0 ]
+  "switches" : [ -62135582412, -59006347212, -55850673612, -52695000012, -46383566412, -43227892812, -40072219212, -33760785612, -30605112012, -27449438412, -21138004812, -17982331212, -14826657612, -12220056012, -12219969612, -12219883212, -12219796812, -12219710412, -12219624012, -12219537612, -12219451212, -12219364812, -12219278412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ]
 }, {
   "tz" : "America/Godthab",
-  "switches" : [ -62135584384, -59006349184, -55850675584, -52695001984, -46383568384, -43227894784, -40072221184, -33760787584, -30605113984, -27449440384, -21138006784, -17982333184, -14826659584, -12219280384, -2208988800 ],
-  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, -1616, 0 ]
+  "switches" : [ -62135584384, -59006349184, -55850675584, -52695001984, -46383568384, -43227894784, -40072221184, -33760787584, -30605113984, -27449440384, -21138006784, -17982333184, -14826659584, -12220057984, -12219971584, -12219885184, -12219798784, -12219712384, -12219625984, -12219539584, -12219453184, -12219366784, -12219280384, -2208988800 ],
+  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, 775984, 689584, 603184, 516784, 430384, 343984, 257584, 171184, 84784, -1616, 0 ]
 }, {
   "tz" : "America/Goose_Bay",
-  "switches" : [ -62135582300, -59006347100, -55850673500, -52694999900, -46383566300, -43227892700, -40072219100, -33760785500, -30605111900, -27449438300, -21138004700, -17982331100, -14826657500, -12219278300, -2713895900, -2208988800 ],
-  "diffs" : [ -172900, -86500, -100, 86300, 172700, 259100, 345500, 431900, 518300, 604700, 691100, 777500, 863900, -100, 1748, 0 ]
+  "switches" : [ -62135582300, -59006347100, -55850673500, -52694999900, -46383566300, -43227892700, -40072219100, -33760785500, -30605111900, -27449438300, -21138004700, -17982331100, -14826657500, -12220055900, -12219969500, -12219883100, -12219796700, -12219710300, -12219623900, -12219537500, -12219451100, -12219364700, -12219278300, -2713895900, -2208988800 ],
+  "diffs" : [ -172900, -86500, -100, 86300, 172700, 259100, 345500, 431900, 518300, 604700, 691100, 777500, 863900, 777500, 691100, 604700, 518300, 431900, 345500, 259100, 172700, 86300, -100, 1748, 0 ]
 }, {
   "tz" : "America/Grand_Turk",
-  "switches" : [ -62135579728, -59006344528, -55850670928, -52694997328, -46383563728, -43227890128, -40072216528, -33760782928, -30605109328, -27449435728, -21138002128, -17982328528, -14826654928, -12219275728, -2524504528, -2208988800 ],
-  "diffs" : [ -171872, -85472, 928, 87328, 173728, 260128, 346528, 432928, 519328, 605728, 692128, 778528, 864928, 928, -430, 0 ]
+  "switches" : [ -62135579728, -59006344528, -55850670928, -52694997328, -46383563728, -43227890128, -40072216528, -33760782928, -30605109328, -27449435728, -21138002128, -17982328528, -14826654928, -12220053328, -12219966928, -12219880528, -12219794128, -12219707728, -12219621328, -12219534928, -12219448528, -12219362128, -12219275728, -2524504528, -2208988800 ],
+  "diffs" : [ -171872, -85472, 928, 87328, 173728, 260128, 346528, 432928, 519328, 605728, 692128, 778528, 864928, 778528, 692128, 605728, 519328, 432928, 346528, 260128, 173728, 87328, 928, -430, 0 ]
 }, {
   "tz" : "America/Grenada",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Guadeloupe",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Guatemala",
-  "switches" : [ -62135575076, -59006339876, -55850666276, -52694992676, -46383559076, -43227885476, -40072211876, -33760778276, -30605104676, -27449431076, -21137997476, -17982323876, -14826650276, -12219271076, -2208988800 ],
-  "diffs" : [ -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, 863876, -124, 0 ]
+  "switches" : [ -62135575076, -59006339876, -55850666276, -52694992676, -46383559076, -43227885476, -40072211876, -33760778276, -30605104676, -27449431076, -21137997476, -17982323876, -14826650276, -12220048676, -12219962276, -12219875876, -12219789476, -12219703076, -12219616676, -12219530276, -12219443876, -12219357476, -12219271076, -2208988800 ],
+  "diffs" : [ -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, 863876, 777476, 691076, 604676, 518276, 431876, 345476, 259076, 172676, 86276, -124, 0 ]
 }, {
   "tz" : "America/Guayaquil",
-  "switches" : [ -62135577640, -59006342440, -55850668840, -52694995240, -46383561640, -43227888040, -40072214440, -33760780840, -30605107240, -27449433640, -21138000040, -17982326440, -14826652840, -12219273640, -2524502440, -2208988800 ],
-  "diffs" : [ -173960, -87560, -1160, 85240, 171640, 258040, 344440, 430840, 517240, 603640, 690040, 776440, 862840, -1160, -840, 0 ]
+  "switches" : [ -62135577640, -59006342440, -55850668840, -52694995240, -46383561640, -43227888040, -40072214440, -33760780840, -30605107240, -27449433640, -21138000040, -17982326440, -14826652840, -12220051240, -12219964840, -12219878440, -12219792040, -12219705640, -12219619240, -12219532840, -12219446440, -12219360040, -12219273640, -2524502440, -2208988800 ],
+  "diffs" : [ -173960, -87560, -1160, 85240, 171640, 258040, 344440, 430840, 517240, 603640, 690040, 776440, 862840, 776440, 690040, 603640, 517240, 430840, 344440, 258040, 171640, 85240, -1160, -840, 0 ]
 }, {
   "tz" : "America/Guyana",
-  "switches" : [ -62135582840, -59006347640, -55850674040, -52695000440, -46383566840, -43227893240, -40072219640, -33760786040, -30605112440, -27449438840, -21138005240, -17982331640, -14826658040, -12219278840, -2208988800 ],
-  "diffs" : [ -172360, -85960, 440, 86840, 173240, 259640, 346040, 432440, 518840, 605240, 691640, 778040, 864440, 440, 0 ]
+  "switches" : [ -62135582840, -59006347640, -55850674040, -52695000440, -46383566840, -43227893240, -40072219640, -33760786040, -30605112440, -27449438840, -21138005240, -17982331640, -14826658040, -12220056440, -12219970040, -12219883640, -12219797240, -12219710840, -12219624440, -12219538040, -12219451640, -12219365240, -12219278840, -2208988800 ],
+  "diffs" : [ -172360, -85960, 440, 86840, 173240, 259640, 346040, 432440, 518840, 605240, 691640, 778040, 864440, 778040, 691640, 605240, 518840, 432440, 346040, 259640, 173240, 86840, 440, 0 ]
 }, {
   "tz" : "America/Halifax",
-  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12219277536, -2208988800 ],
-  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, -864, 0 ]
+  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12220055136, -12219968736, -12219882336, -12219795936, -12219709536, -12219623136, -12219536736, -12219450336, -12219363936, -12219277536, -2208988800 ],
+  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, 776736, 690336, 603936, 517536, 431136, 344736, 258336, 171936, 85536, -864, 0 ]
 }, {
   "tz" : "America/Havana",
-  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12219273032, -2524501832, -2208988800 ],
-  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, -1768, -1776, 0 ]
+  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12220050632, -12219964232, -12219877832, -12219791432, -12219705032, -12219618632, -12219532232, -12219445832, -12219359432, -12219273032, -2524501832, -2208988800 ],
+  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, 775832, 689432, 603032, 516632, 430232, 343832, 257432, 171032, 84632, -1768, -1776, 0 ]
 }, {
   "tz" : "America/Hermosillo",
-  "switches" : [ -62135570168, -59006334968, -55850661368, -52694987768, -46383554168, -43227880568, -40072206968, -33760773368, -30605099768, -27449426168, -21137992568, -17982318968, -14826645368, -12219266168, -2208988800 ],
-  "diffs" : [ -174232, -87832, -1432, 84968, 171368, 257768, 344168, 430568, 516968, 603368, 689768, 776168, 862568, -1432, 0 ]
+  "switches" : [ -62135570168, -59006334968, -55850661368, -52694987768, -46383554168, -43227880568, -40072206968, -33760773368, -30605099768, -27449426168, -21137992568, -17982318968, -14826645368, -12220043768, -12219957368, -12219870968, -12219784568, -12219698168, -12219611768, -12219525368, -12219438968, -12219352568, -12219266168, -2208988800 ],
+  "diffs" : [ -174232, -87832, -1432, 84968, 171368, 257768, 344168, 430568, 516968, 603368, 689768, 776168, 862568, 776168, 689768, 603368, 516968, 430568, 344168, 257768, 171368, 84968, -1432, 0 ]
 }, {
   "tz" : "America/Indiana/Indianapolis",
-  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
-  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
 }, {
   "tz" : "America/Indiana/Knox",
-  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12219272010, -2717647200 ],
-  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 810, 0 ]
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12220049610, -12219963210, -12219876810, -12219790410, -12219704010, -12219617610, -12219531210, -12219444810, -12219358410, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ]
 }, {
   "tz" : "America/Indiana/Marengo",
-  "switches" : [ -62135576077, -59006340877, -55850667277, -52694993677, -46383560077, -43227886477, -40072212877, -33760779277, -30605105677, -27449432077, -21137998477, -17982324877, -14826651277, -12219272077, -2717647200, -2208988800 ],
-  "diffs" : [ -175523, -89123, -2723, 83677, 170077, 256477, 342877, 429277, 515677, 602077, 688477, 774877, 861277, -2723, -3600, 0 ]
+  "switches" : [ -62135576077, -59006340877, -55850667277, -52694993677, -46383560077, -43227886477, -40072212877, -33760779277, -30605105677, -27449432077, -21137998477, -17982324877, -14826651277, -12220049677, -12219963277, -12219876877, -12219790477, -12219704077, -12219617677, -12219531277, -12219444877, -12219358477, -12219272077, -2717647200, -2208988800 ],
+  "diffs" : [ -175523, -89123, -2723, 83677, 170077, 256477, 342877, 429277, 515677, 602077, 688477, 774877, 861277, 774877, 688477, 602077, 515677, 429277, 342877, 256477, 170077, 83677, -2723, -3600, 0 ]
 }, {
   "tz" : "America/Indiana/Petersburg",
-  "switches" : [ -62135575853, -59006340653, -55850667053, -52694993453, -46383559853, -43227886253, -40072212653, -33760779053, -30605105453, -27449431853, -21137998253, -17982324653, -14826651053, -12219271853, -2717647200, -2208988800 ],
-  "diffs" : [ -175747, -89347, -2947, 83453, 169853, 256253, 342653, 429053, 515453, 601853, 688253, 774653, 861053, -2947, -3600, 0 ]
+  "switches" : [ -62135575853, -59006340653, -55850667053, -52694993453, -46383559853, -43227886253, -40072212653, -33760779053, -30605105453, -27449431853, -21137998253, -17982324653, -14826651053, -12220049453, -12219963053, -12219876653, -12219790253, -12219703853, -12219617453, -12219531053, -12219444653, -12219358253, -12219271853, -2717647200, -2208988800 ],
+  "diffs" : [ -175747, -89347, -2947, 83453, 169853, 256253, 342653, 429053, 515453, 601853, 688253, 774653, 861053, 774653, 688253, 601853, 515453, 429053, 342653, 256253, 169853, 83453, -2947, -3600, 0 ]
 }, {
   "tz" : "America/Indiana/Tell_City",
-  "switches" : [ -62135575977, -59006340777, -55850667177, -52694993577, -46383559977, -43227886377, -40072212777, -33760779177, -30605105577, -27449431977, -21137998377, -17982324777, -14826651177, -12219271977, -2717647200 ],
-  "diffs" : [ -172023, -85623, 777, 87177, 173577, 259977, 346377, 432777, 519177, 605577, 691977, 778377, 864777, 777, 0 ]
+  "switches" : [ -62135575977, -59006340777, -55850667177, -52694993577, -46383559977, -43227886377, -40072212777, -33760779177, -30605105577, -27449431977, -21137998377, -17982324777, -14826651177, -12220049577, -12219963177, -12219876777, -12219790377, -12219703977, -12219617577, -12219531177, -12219444777, -12219358377, -12219271977, -2717647200 ],
+  "diffs" : [ -172023, -85623, 777, 87177, 173577, 259977, 346377, 432777, 519177, 605577, 691977, 778377, 864777, 778377, 691977, 605577, 519177, 432777, 346377, 259977, 173577, 87177, 777, 0 ]
 }, {
   "tz" : "America/Indiana/Vevay",
-  "switches" : [ -62135576384, -59006341184, -55850667584, -52694993984, -46383560384, -43227886784, -40072213184, -33760779584, -30605105984, -27449432384, -21137998784, -17982325184, -14826651584, -12219272384, -2717647200, -2208988800 ],
-  "diffs" : [ -175216, -88816, -2416, 83984, 170384, 256784, 343184, 429584, 515984, 602384, 688784, 775184, 861584, -2416, -3600, 0 ]
+  "switches" : [ -62135576384, -59006341184, -55850667584, -52694993984, -46383560384, -43227886784, -40072213184, -33760779584, -30605105984, -27449432384, -21137998784, -17982325184, -14826651584, -12220049984, -12219963584, -12219877184, -12219790784, -12219704384, -12219617984, -12219531584, -12219445184, -12219358784, -12219272384, -2717647200, -2208988800 ],
+  "diffs" : [ -175216, -88816, -2416, 83984, 170384, 256784, 343184, 429584, 515984, 602384, 688784, 775184, 861584, 775184, 688784, 602384, 515984, 429584, 343184, 256784, 170384, 83984, -2416, -3600, 0 ]
 }, {
   "tz" : "America/Indiana/Vincennes",
-  "switches" : [ -62135575793, -59006340593, -55850666993, -52694993393, -46383559793, -43227886193, -40072212593, -33760778993, -30605105393, -27449431793, -21137998193, -17982324593, -14826650993, -12219271793, -2717647200, -2208988800 ],
-  "diffs" : [ -175807, -89407, -3007, 83393, 169793, 256193, 342593, 428993, 515393, 601793, 688193, 774593, 860993, -3007, -3600, 0 ]
+  "switches" : [ -62135575793, -59006340593, -55850666993, -52694993393, -46383559793, -43227886193, -40072212593, -33760778993, -30605105393, -27449431793, -21137998193, -17982324593, -14826650993, -12220049393, -12219962993, -12219876593, -12219790193, -12219703793, -12219617393, -12219530993, -12219444593, -12219358193, -12219271793, -2717647200, -2208988800 ],
+  "diffs" : [ -175807, -89407, -3007, 83393, 169793, 256193, 342593, 428993, 515393, 601793, 688193, 774593, 860993, 774593, 688193, 601793, 515393, 428993, 342593, 256193, 169793, 83393, -3007, -3600, 0 ]
 }, {
   "tz" : "America/Indiana/Winamac",
-  "switches" : [ -62135576015, -59006340815, -55850667215, -52694993615, -46383560015, -43227886415, -40072212815, -33760779215, -30605105615, -27449432015, -21137998415, -17982324815, -14826651215, -12219272015, -2717647200, -2208988800 ],
-  "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, -2785, -3600, 0 ]
+  "switches" : [ -62135576015, -59006340815, -55850667215, -52694993615, -46383560015, -43227886415, -40072212815, -33760779215, -30605105615, -27449432015, -21137998415, -17982324815, -14826651215, -12220049615, -12219963215, -12219876815, -12219790415, -12219704015, -12219617615, -12219531215, -12219444815, -12219358415, -12219272015, -2717647200, -2208988800 ],
+  "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, -3600, 0 ]
 }, {
   "tz" : "America/Indianapolis",
-  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
-  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
 }, {
   "tz" : "America/Inuvik",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 25200, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 802800, 716400, 630000, 543600, 457200, 370800, 284400, 198000, 111600, 25200, 0 ]
 }, {
   "tz" : "America/Iqaluit",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 18000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 795600, 709200, 622800, 536400, 450000, 363600, 277200, 190800, 104400, 18000, 0 ]
 }, {
   "tz" : "America/Jamaica",
-  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12219274370, -2208988800 ],
-  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, -430, 0 ]
+  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12220051970, -12219965570, -12219879170, -12219792770, -12219706370, -12219619970, -12219533570, -12219447170, -12219360770, -12219274370, -2208988800 ],
+  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, 777170, 690770, 604370, 517970, 431570, 345170, 258770, 172370, 85970, -430, 0 ]
 }, {
   "tz" : "America/Jujuy",
-  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12219277128, -2372096328, -2208988800 ],
-  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, -4872, -4608, 0 ]
+  "switches" : [ -62135581128, -59006345928, -55850672328, -52694998728, -46383565128, -43227891528, -40072217928, -33760784328, -30605110728, -27449437128, -21138003528, -17982329928, -14826656328, -12220054728, -12219968328, -12219881928, -12219795528, -12219709128, -12219622728, -12219536328, -12219449928, -12219363528, -12219277128, -2372096328, -2208988800 ],
+  "diffs" : [ -177672, -91272, -4872, 81528, 167928, 254328, 340728, 427128, 513528, 599928, 686328, 772728, 859128, 772728, 686328, 599928, 513528, 427128, 340728, 254328, 167928, 81528, -4872, -4608, 0 ]
 }, {
   "tz" : "America/Juneau",
-  "switches" : [ -62135650939, -59006415739, -55850742139, -52695068539, -46383634939, -43227961339, -40072287739, -33760854139, -30605180539, -27449506939, -21138073339, -17982399739, -14826726139, -12219346939, -3225223727, -2208988800 ],
-  "diffs" : [ -86261, 139, 86539, 172939, 259339, 345739, 432139, 518539, 604939, 691339, 777739, 864139, 950539, 86539, 139, 0 ]
+  "switches" : [ -62135650939, -59006415739, -55850742139, -52695068539, -46383634939, -43227961339, -40072287739, -33760854139, -30605180539, -27449506939, -21138073339, -17982399739, -14826726139, -12220124539, -12220038139, -12219951739, -12219865339, -12219778939, -12219692539, -12219606139, -12219519739, -12219433339, -12219346939, -3225223727, -2208988800 ],
+  "diffs" : [ -86261, 139, 86539, 172939, 259339, 345739, 432139, 518539, 604939, 691339, 777739, 864139, 950539, 864139, 777739, 691339, 604939, 518539, 432139, 345739, 259339, 172939, 86539, 139, 0 ]
 }, {
   "tz" : "America/Kentucky/Louisville",
-  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12219272218, -2717647200, -2208988800 ],
-  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, -2582, -3600, 0 ]
+  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12220049818, -12219963418, -12219877018, -12219790618, -12219704218, -12219617818, -12219531418, -12219445018, -12219358618, -12219272218, -2717647200, -2208988800 ],
+  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, 775018, 688618, 602218, 515818, 429418, 343018, 256618, 170218, 83818, -2582, -3600, 0 ]
 }, {
   "tz" : "America/Kentucky/Monticello",
-  "switches" : [ -62135576436, -59006341236, -55850667636, -52694994036, -46383560436, -43227886836, -40072213236, -33760779636, -30605106036, -27449432436, -21137998836, -17982325236, -14826651636, -12219272436, -2717647200, -2208988800 ],
-  "diffs" : [ -175164, -88764, -2364, 84036, 170436, 256836, 343236, 429636, 516036, 602436, 688836, 775236, 861636, -2364, -3600, 0 ]
+  "switches" : [ -62135576436, -59006341236, -55850667636, -52694994036, -46383560436, -43227886836, -40072213236, -33760779636, -30605106036, -27449432436, -21137998836, -17982325236, -14826651636, -12220050036, -12219963636, -12219877236, -12219790836, -12219704436, -12219618036, -12219531636, -12219445236, -12219358836, -12219272436, -2717647200, -2208988800 ],
+  "diffs" : [ -175164, -88764, -2364, 84036, 170436, 256836, 343236, 429636, 516036, 602436, 688836, 775236, 861636, 775236, 688836, 602436, 516036, 429636, 343236, 256836, 170436, 84036, -2364, -3600, 0 ]
 }, {
   "tz" : "America/Knox_IN",
-  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12219272010, -2717647200 ],
-  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 810, 0 ]
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12220049610, -12219963210, -12219876810, -12219790410, -12219704010, -12219617610, -12219531210, -12219444810, -12219358410, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ]
 }, {
   "tz" : "America/Kralendijk",
-  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
-  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
 }, {
   "tz" : "America/La_Paz",
-  "switches" : [ -62135580444, -59006345244, -55850671644, -52694998044, -46383564444, -43227890844, -40072217244, -33760783644, -30605110044, -27449436444, -21138002844, -17982329244, -14826655644, -12219276444, -2208988800 ],
-  "diffs" : [ -174756, -88356, -1956, 84444, 170844, 257244, 343644, 430044, 516444, 602844, 689244, 775644, 862044, -1956, 0 ]
+  "switches" : [ -62135580444, -59006345244, -55850671644, -52694998044, -46383564444, -43227890844, -40072217244, -33760783644, -30605110044, -27449436444, -21138002844, -17982329244, -14826655644, -12220054044, -12219967644, -12219881244, -12219794844, -12219708444, -12219622044, -12219535644, -12219449244, -12219362844, -12219276444, -2208988800 ],
+  "diffs" : [ -174756, -88356, -1956, 84444, 170844, 257244, 343644, 430044, 516444, 602844, 689244, 775644, 862044, 775644, 689244, 602844, 516444, 430044, 343644, 257244, 170844, 84444, -1956, 0 ]
 }, {
   "tz" : "America/Lima",
-  "switches" : [ -62135578308, -59006343108, -55850669508, -52694995908, -46383562308, -43227888708, -40072215108, -33760781508, -30605107908, -27449434308, -21138000708, -17982327108, -14826653508, -12219274308, -2524503108, -2208988800 ],
-  "diffs" : [ -173292, -86892, -492, 85908, 172308, 258708, 345108, 431508, 517908, 604308, 690708, 777108, 863508, -492, -516, 0 ]
+  "switches" : [ -62135578308, -59006343108, -55850669508, -52694995908, -46383562308, -43227888708, -40072215108, -33760781508, -30605107908, -27449434308, -21138000708, -17982327108, -14826653508, -12220051908, -12219965508, -12219879108, -12219792708, -12219706308, -12219619908, -12219533508, -12219447108, -12219360708, -12219274308, -2524503108, -2208988800 ],
+  "diffs" : [ -173292, -86892, -492, 85908, 172308, 258708, 345108, 431508, 517908, 604308, 690708, 777108, 863508, 777108, 690708, 604308, 517908, 431508, 345108, 258708, 172308, 85908, -492, -516, 0 ]
 }, {
   "tz" : "America/Los_Angeles",
-  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12219264422, -2717640000 ],
-  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 422, 0 ]
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ]
 }, {
   "tz" : "America/Louisville",
-  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12219272218, -2717647200, -2208988800 ],
-  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, -2582, -3600, 0 ]
+  "switches" : [ -62135576218, -59006341018, -55850667418, -52694993818, -46383560218, -43227886618, -40072213018, -33760779418, -30605105818, -27449432218, -21137998618, -17982325018, -14826651418, -12220049818, -12219963418, -12219877018, -12219790618, -12219704218, -12219617818, -12219531418, -12219445018, -12219358618, -12219272218, -2717647200, -2208988800 ],
+  "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, 775018, 688618, 602218, 515818, 429418, 343018, 256618, 170218, 83818, -2582, -3600, 0 ]
 }, {
   "tz" : "America/Lower_Princes",
-  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12219276253, -2208988800 ],
-  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, -2147, 0 ]
+  "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ],
+  "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ]
 }, {
   "tz" : "America/Maceio",
-  "switches" : [ -62135588228, -59006353028, -55850679428, -52695005828, -46383572228, -43227898628, -40072225028, -33760791428, -30605117828, -27449444228, -21138010628, -17982337028, -14826663428, -12219284228, -2208988800 ],
-  "diffs" : [ -170572, -84172, 2228, 88628, 175028, 261428, 347828, 434228, 520628, 607028, 693428, 779828, 866228, 2228, 0 ]
+  "switches" : [ -62135588228, -59006353028, -55850679428, -52695005828, -46383572228, -43227898628, -40072225028, -33760791428, -30605117828, -27449444228, -21138010628, -17982337028, -14826663428, -12220061828, -12219975428, -12219889028, -12219802628, -12219716228, -12219629828, -12219543428, -12219457028, -12219370628, -12219284228, -2208988800 ],
+  "diffs" : [ -170572, -84172, 2228, 88628, 175028, 261428, 347828, 434228, 520628, 607028, 693428, 779828, 866228, 779828, 693428, 607028, 520628, 434228, 347828, 261428, 175028, 88628, 2228, 0 ]
 }, {
   "tz" : "America/Managua",
-  "switches" : [ -62135576092, -59006340892, -55850667292, -52694993692, -46383560092, -43227886492, -40072212892, -33760779292, -30605105692, -27449432092, -21137998492, -17982324892, -14826651292, -12219272092, -2524500892, -2208988800 ],
-  "diffs" : [ -171908, -85508, 892, 87292, 173692, 260092, 346492, 432892, 519292, 605692, 692092, 778492, 864892, 892, 888, 0 ]
+  "switches" : [ -62135576092, -59006340892, -55850667292, -52694993692, -46383560092, -43227886492, -40072212892, -33760779292, -30605105692, -27449432092, -21137998492, -17982324892, -14826651292, -12220049692, -12219963292, -12219876892, -12219790492, -12219704092, -12219617692, -12219531292, -12219444892, -12219358492, -12219272092, -2524500892, -2208988800 ],
+  "diffs" : [ -171908, -85508, 892, 87292, 173692, 260092, 346492, 432892, 519292, 605692, 692092, 778492, 864892, 778492, 692092, 605692, 519292, 432892, 346492, 260092, 173692, 87292, 892, 888, 0 ]
 }, {
   "tz" : "America/Manaus",
-  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12219278396, -2208988800 ],
-  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, -4, 0 ]
+  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12220055996, -12219969596, -12219883196, -12219796796, -12219710396, -12219623996, -12219537596, -12219451196, -12219364796, -12219278396, -2208988800 ],
+  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, 777596, 691196, 604796, 518396, 431996, 345596, 259196, 172796, 86396, -4, 0 ]
 }, {
   "tz" : "America/Marigot",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Martinique",
-  "switches" : [ -62135582140, -59006346940, -55850673340, -52694999740, -46383566140, -43227892540, -40072218940, -33760785340, -30605111740, -27449438140, -21138004540, -17982330940, -14826657340, -12219278140, -2208988800 ],
-  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, -260, 0 ]
+  "switches" : [ -62135582140, -59006346940, -55850673340, -52694999740, -46383566140, -43227892540, -40072218940, -33760785340, -30605111740, -27449438140, -21138004540, -17982330940, -14826657340, -12220055740, -12219969340, -12219882940, -12219796540, -12219710140, -12219623740, -12219537340, -12219450940, -12219364540, -12219278140, -2208988800 ],
+  "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, 777340, 690940, 604540, 518140, 431740, 345340, 258940, 172540, 86140, -260, 0 ]
 }, {
   "tz" : "America/Matamoros",
-  "switches" : [ -62135572800, -59006337600, -55850664000, -52694990400, -46383556800, -43227883200, -40072209600, -33760776000, -30605102400, -27449428800, -21137995200, -17982321600, -14826648000, -12219268800, -2208988800 ],
-  "diffs" : [ -175200, -88800, -2400, 84000, 170400, 256800, 343200, 429600, 516000, 602400, 688800, 775200, 861600, -2400, 0 ]
+  "switches" : [ -62135572800, -59006337600, -55850664000, -52694990400, -46383556800, -43227883200, -40072209600, -33760776000, -30605102400, -27449428800, -21137995200, -17982321600, -14826648000, -12220046400, -12219960000, -12219873600, -12219787200, -12219700800, -12219614400, -12219528000, -12219441600, -12219355200, -12219268800, -2208988800 ],
+  "diffs" : [ -175200, -88800, -2400, 84000, 170400, 256800, 343200, 429600, 516000, 602400, 688800, 775200, 861600, 775200, 688800, 602400, 516000, 429600, 343200, 256800, 170400, 84000, -2400, 0 ]
 }, {
   "tz" : "America/Mazatlan",
-  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12219267260, -2208988800 ],
-  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, -340, 0 ]
+  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12220044860, -12219958460, -12219872060, -12219785660, -12219699260, -12219612860, -12219526460, -12219440060, -12219353660, -12219267260, -2208988800 ],
+  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, 777260, 690860, 604460, 518060, 431660, 345260, 258860, 172460, 86060, -340, 0 ]
 }, {
   "tz" : "America/Mendoza",
-  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12219276284, -2372095484, -2208988800 ],
-  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, -5716, -4608, 0 ]
+  "switches" : [ -62135580284, -59006345084, -55850671484, -52694997884, -46383564284, -43227890684, -40072217084, -33760783484, -30605109884, -27449436284, -21138002684, -17982329084, -14826655484, -12220053884, -12219967484, -12219881084, -12219794684, -12219708284, -12219621884, -12219535484, -12219449084, -12219362684, -12219276284, -2372095484, -2208988800 ],
+  "diffs" : [ -178516, -92116, -5716, 80684, 167084, 253484, 339884, 426284, 512684, 599084, 685484, 771884, 858284, 771884, 685484, 599084, 512684, 426284, 339884, 253484, 167084, 80684, -5716, -4608, 0 ]
 }, {
   "tz" : "America/Menominee",
-  "switches" : [ -62135575773, -59006340573, -55850666973, -52694993373, -46383559773, -43227886173, -40072212573, -33760778973, -30605105373, -27449431773, -21137998173, -17982324573, -14826650973, -12219271773, -2659759773 ],
-  "diffs" : [ -172227, -85827, 573, 86973, 173373, 259773, 346173, 432573, 518973, 605373, 691773, 778173, 864573, 573, 0 ]
+  "switches" : [ -62135575773, -59006340573, -55850666973, -52694993373, -46383559773, -43227886173, -40072212573, -33760778973, -30605105373, -27449431773, -21137998173, -17982324573, -14826650973, -12220049373, -12219962973, -12219876573, -12219790173, -12219703773, -12219617373, -12219530973, -12219444573, -12219358173, -12219271773, -2659759773 ],
+  "diffs" : [ -172227, -85827, 573, 86973, 173373, 259773, 346173, 432573, 518973, 605373, 691773, 778173, 864573, 778173, 691773, 605373, 518973, 432573, 346173, 259773, 173373, 86973, 573, 0 ]
 }, {
   "tz" : "America/Merida",
-  "switches" : [ -62135575292, -59006340092, -55850666492, -52694992892, -46383559292, -43227885692, -40072212092, -33760778492, -30605104892, -27449431292, -21137997692, -17982324092, -14826650492, -12219271292, -2208988800 ],
-  "diffs" : [ -172708, -86308, 92, 86492, 172892, 259292, 345692, 432092, 518492, 604892, 691292, 777692, 864092, 92, 0 ]
+  "switches" : [ -62135575292, -59006340092, -55850666492, -52694992892, -46383559292, -43227885692, -40072212092, -33760778492, -30605104892, -27449431292, -21137997692, -17982324092, -14826650492, -12220048892, -12219962492, -12219876092, -12219789692, -12219703292, -12219616892, -12219530492, -12219444092, -12219357692, -12219271292, -2208988800 ],
+  "diffs" : [ -172708, -86308, 92, 86492, 172892, 259292, 345692, 432092, 518492, 604892, 691292, 777692, 864092, 777692, 691292, 604892, 518492, 432092, 345692, 259292, 172892, 86492, 92, 0 ]
 }, {
   "tz" : "America/Metlakatla",
-  "switches" : [ -62135651622, -59006416422, -55850742822, -52695069222, -46383635622, -43227962022, -40072288422, -33760854822, -30605181222, -27449507622, -21138074022, -17982400422, -14826726822, -12219347622, -3225223727, -2208988800 ],
-  "diffs" : [ -85578, 822, 87222, 173622, 260022, 346422, 432822, 519222, 605622, 692022, 778422, 864822, 951222, 87222, 822, 0 ]
+  "switches" : [ -62135651622, -59006416422, -55850742822, -52695069222, -46383635622, -43227962022, -40072288422, -33760854822, -30605181222, -27449507622, -21138074022, -17982400422, -14826726822, -12220125222, -12220038822, -12219952422, -12219866022, -12219779622, -12219693222, -12219606822, -12219520422, -12219434022, -12219347622, -3225223727, -2208988800 ],
+  "diffs" : [ -85578, 822, 87222, 173622, 260022, 346422, 432822, 519222, 605622, 692022, 778422, 864822, 951222, 864822, 778422, 692022, 605622, 519222, 432822, 346422, 260022, 173622, 87222, 822, 0 ]
 }, {
   "tz" : "America/Mexico_City",
-  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12219269004, -2208988800 ],
-  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, -2196, 0 ]
+  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12220046604, -12219960204, -12219873804, -12219787404, -12219701004, -12219614604, -12219528204, -12219441804, -12219355404, -12219269004, -2208988800 ],
+  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, 775404, 689004, 602604, 516204, 429804, 343404, 257004, 170604, 84204, -2196, 0 ]
 }, {
   "tz" : "America/Miquelon",
-  "switches" : [ -62135583320, -59006348120, -55850674520, -52695000920, -46383567320, -43227893720, -40072220120, -33760786520, -30605112920, -27449439320, -21138005720, -17982332120, -14826658520, -12219279320, -2208988800 ],
-  "diffs" : [ -175480, -89080, -2680, 83720, 170120, 256520, 342920, 429320, 515720, 602120, 688520, 774920, 861320, -2680, 0 ]
+  "switches" : [ -62135583320, -59006348120, -55850674520, -52695000920, -46383567320, -43227893720, -40072220120, -33760786520, -30605112920, -27449439320, -21138005720, -17982332120, -14826658520, -12220056920, -12219970520, -12219884120, -12219797720, -12219711320, -12219624920, -12219538520, -12219452120, -12219365720, -12219279320, -2208988800 ],
+  "diffs" : [ -175480, -89080, -2680, 83720, 170120, 256520, 342920, 429320, 515720, 602120, 688520, 774920, 861320, 774920, 688520, 602120, 515720, 429320, 342920, 256520, 170120, 83720, -2680, 0 ]
 }, {
   "tz" : "America/Moncton",
-  "switches" : [ -62135581252, -59006346052, -55850672452, -52694998852, -46383565252, -43227891652, -40072218052, -33760784452, -30605110852, -27449437252, -21138003652, -17982330052, -14826656452, -12219277252, -2715882052, -2208988800 ],
-  "diffs" : [ -173948, -87548, -1148, 85252, 171652, 258052, 344452, 430852, 517252, 603652, 690052, 776452, 862852, -1148, -3600, 0 ]
+  "switches" : [ -62135581252, -59006346052, -55850672452, -52694998852, -46383565252, -43227891652, -40072218052, -33760784452, -30605110852, -27449437252, -21138003652, -17982330052, -14826656452, -12220054852, -12219968452, -12219882052, -12219795652, -12219709252, -12219622852, -12219536452, -12219450052, -12219363652, -12219277252, -2715882052, -2208988800 ],
+  "diffs" : [ -173948, -87548, -1148, 85252, 171652, 258052, 344452, 430852, 517252, 603652, 690052, 776452, 862852, 776452, 690052, 603652, 517252, 430852, 344452, 258052, 171652, 85252, -1148, -3600, 0 ]
 }, {
   "tz" : "America/Monterrey",
-  "switches" : [ -62135572724, -59006337524, -55850663924, -52694990324, -46383556724, -43227883124, -40072209524, -33760775924, -30605102324, -27449428724, -21137995124, -17982321524, -14826647924, -12219268724, -2208988800 ],
-  "diffs" : [ -175276, -88876, -2476, 83924, 170324, 256724, 343124, 429524, 515924, 602324, 688724, 775124, 861524, -2476, 0 ]
+  "switches" : [ -62135572724, -59006337524, -55850663924, -52694990324, -46383556724, -43227883124, -40072209524, -33760775924, -30605102324, -27449428724, -21137995124, -17982321524, -14826647924, -12220046324, -12219959924, -12219873524, -12219787124, -12219700724, -12219614324, -12219527924, -12219441524, -12219355124, -12219268724, -2208988800 ],
+  "diffs" : [ -175276, -88876, -2476, 83924, 170324, 256724, 343124, 429524, 515924, 602324, 688724, 775124, 861524, 775124, 688724, 602324, 515924, 429524, 343124, 256724, 170324, 83924, -2476, 0 ]
 }, {
   "tz" : "America/Montevideo",
-  "switches" : [ -62135583309, -59006348109, -55850674509, -52695000909, -46383567309, -43227893709, -40072220109, -33760786509, -30605112909, -27449439309, -21138005709, -17982332109, -14826658509, -12219279309, -2208988800 ],
-  "diffs" : [ -175491, -89091, -2691, 83709, 170109, 256509, 342909, 429309, 515709, 602109, 688509, 774909, 861309, -2691, 0 ]
+  "switches" : [ -62135583309, -59006348109, -55850674509, -52695000909, -46383567309, -43227893709, -40072220109, -33760786509, -30605112909, -27449439309, -21138005709, -17982332109, -14826658509, -12220056909, -12219970509, -12219884109, -12219797709, -12219711309, -12219624909, -12219538509, -12219452109, -12219365709, -12219279309, -2208988800 ],
+  "diffs" : [ -175491, -89091, -2691, 83709, 170109, 256509, 342909, 429309, 515709, 602109, 688509, 774909, 861309, 774909, 688509, 602109, 515709, 429309, 342909, 256509, 170109, 83709, -2691, 0 ]
 }, {
   "tz" : "America/Montreal",
-  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12219273748, -2366736148 ],
-  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, -1052, 0 ]
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ]
 }, {
   "tz" : "America/Montserrat",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Nassau",
-  "switches" : [ -62135578230, -59006343030, -55850669430, -52694995830, -46383562230, -43227888630, -40072215030, -33760781430, -30605107830, -27449434230, -21138000630, -17982327030, -14826653430, -12219274230, -2208988800 ],
-  "diffs" : [ -173370, -86970, -570, 85830, 172230, 258630, 345030, 431430, 517830, 604230, 690630, 777030, 863430, -570, 0 ]
+  "switches" : [ -62135578230, -59006343030, -55850669430, -52694995830, -46383562230, -43227888630, -40072215030, -33760781430, -30605107830, -27449434230, -21138000630, -17982327030, -14826653430, -12220051830, -12219965430, -12219879030, -12219792630, -12219706230, -12219619830, -12219533430, -12219447030, -12219360630, -12219274230, -2208988800 ],
+  "diffs" : [ -173370, -86970, -570, 85830, 172230, 258630, 345030, 431430, 517830, 604230, 690630, 777030, 863430, 777030, 690630, 604230, 517830, 431430, 345030, 258630, 172230, 85830, -570, 0 ]
 }, {
   "tz" : "America/New_York",
-  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12219275038, -2717650800 ],
-  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 238, 0 ]
+  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12220052638, -12219966238, -12219879838, -12219793438, -12219707038, -12219620638, -12219534238, -12219447838, -12219361438, -12219275038, -2717650800 ],
+  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 777838, 691438, 605038, 518638, 432238, 345838, 259438, 173038, 86638, 238, 0 ]
 }, {
   "tz" : "America/Nipigon",
-  "switches" : [ -62135575616, -59006340416, -55850666816, -52694993216, -46383559616, -43227886016, -40072212416, -33760778816, -30605105216, -27449431616, -21137998016, -17982324416, -14826650816, -12219271616, -2366734016 ],
-  "diffs" : [ -175984, -89584, -3184, 83216, 169616, 256016, 342416, 428816, 515216, 601616, 688016, 774416, 860816, -3184, 0 ]
+  "switches" : [ -62135575616, -59006340416, -55850666816, -52694993216, -46383559616, -43227886016, -40072212416, -33760778816, -30605105216, -27449431616, -21137998016, -17982324416, -14826650816, -12220049216, -12219962816, -12219876416, -12219790016, -12219703616, -12219617216, -12219530816, -12219444416, -12219358016, -12219271616, -2366734016 ],
+  "diffs" : [ -175984, -89584, -3184, 83216, 169616, 256016, 342416, 428816, 515216, 601616, 688016, 774416, 860816, 774416, 688016, 601616, 515216, 428816, 342416, 256016, 169616, 83216, -3184, 0 ]
 }, {
   "tz" : "America/Nome",
-  "switches" : [ -62135643502, -59006408302, -55850734702, -52695061102, -46383627502, -43227953902, -40072280302, -33760846702, -30605173102, -27449499502, -21138065902, -17982392302, -14826718702, -12219339502, -3225223727, -2208988800 ],
-  "diffs" : [ -93698, -7298, 79102, 165502, 251902, 338302, 424702, 511102, 597502, 683902, 770302, 856702, 943102, 79102, -7298, 0 ]
+  "switches" : [ -62135643502, -59006408302, -55850734702, -52695061102, -46383627502, -43227953902, -40072280302, -33760846702, -30605173102, -27449499502, -21138065902, -17982392302, -14826718702, -12220117102, -12220030702, -12219944302, -12219857902, -12219771502, -12219685102, -12219598702, -12219512302, -12219425902, -12219339502, -3225223727, -2208988800 ],
+  "diffs" : [ -93698, -7298, 79102, 165502, 251902, 338302, 424702, 511102, 597502, 683902, 770302, 856702, 943102, 856702, 770302, 683902, 597502, 511102, 424702, 338302, 251902, 165502, 79102, -7298, 0 ]
 }, {
   "tz" : "America/Noronha",
-  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12219285020, -2208988800 ],
-  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12220062620, -12219976220, -12219889820, -12219803420, -12219717020, -12219630620, -12219544220, -12219457820, -12219371420, -12219285020, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
 }, {
   "tz" : "America/North_Dakota/Beulah",
-  "switches" : [ -62135572373, -59006337173, -55850663573, -52694989973, -46383556373, -43227882773, -40072209173, -33760775573, -30605101973, -27449428373, -21137994773, -17982321173, -14826647573, -12219268373, -2717643600, -2208988800 ],
-  "diffs" : [ -175627, -89227, -2827, 83573, 169973, 256373, 342773, 429173, 515573, 601973, 688373, 774773, 861173, -2827, -3600, 0 ]
+  "switches" : [ -62135572373, -59006337173, -55850663573, -52694989973, -46383556373, -43227882773, -40072209173, -33760775573, -30605101973, -27449428373, -21137994773, -17982321173, -14826647573, -12220045973, -12219959573, -12219873173, -12219786773, -12219700373, -12219613973, -12219527573, -12219441173, -12219354773, -12219268373, -2717643600, -2208988800 ],
+  "diffs" : [ -175627, -89227, -2827, 83573, 169973, 256373, 342773, 429173, 515573, 601973, 688373, 774773, 861173, 774773, 688373, 601973, 515573, 429173, 342773, 256373, 169973, 83573, -2827, -3600, 0 ]
 }, {
   "tz" : "America/North_Dakota/Center",
-  "switches" : [ -62135572488, -59006337288, -55850663688, -52694990088, -46383556488, -43227882888, -40072209288, -33760775688, -30605102088, -27449428488, -21137994888, -17982321288, -14826647688, -12219268488, -2717643600, -2208988800 ],
-  "diffs" : [ -175512, -89112, -2712, 83688, 170088, 256488, 342888, 429288, 515688, 602088, 688488, 774888, 861288, -2712, -3600, 0 ]
+  "switches" : [ -62135572488, -59006337288, -55850663688, -52694990088, -46383556488, -43227882888, -40072209288, -33760775688, -30605102088, -27449428488, -21137994888, -17982321288, -14826647688, -12220046088, -12219959688, -12219873288, -12219786888, -12219700488, -12219614088, -12219527688, -12219441288, -12219354888, -12219268488, -2717643600, -2208988800 ],
+  "diffs" : [ -175512, -89112, -2712, 83688, 170088, 256488, 342888, 429288, 515688, 602088, 688488, 774888, 861288, 774888, 688488, 602088, 515688, 429288, 342888, 256488, 170088, 83688, -2712, -3600, 0 ]
 }, {
   "tz" : "America/North_Dakota/New_Salem",
-  "switches" : [ -62135572461, -59006337261, -55850663661, -52694990061, -46383556461, -43227882861, -40072209261, -33760775661, -30605102061, -27449428461, -21137994861, -17982321261, -14826647661, -12219268461, -2717643600, -2208988800 ],
-  "diffs" : [ -175539, -89139, -2739, 83661, 170061, 256461, 342861, 429261, 515661, 602061, 688461, 774861, 861261, -2739, -3600, 0 ]
+  "switches" : [ -62135572461, -59006337261, -55850663661, -52694990061, -46383556461, -43227882861, -40072209261, -33760775661, -30605102061, -27449428461, -21137994861, -17982321261, -14826647661, -12220046061, -12219959661, -12219873261, -12219786861, -12219700461, -12219614061, -12219527661, -12219441261, -12219354861, -12219268461, -2717643600, -2208988800 ],
+  "diffs" : [ -175539, -89139, -2739, 83661, 170061, 256461, 342861, 429261, 515661, 602061, 688461, 774861, 861261, 774861, 688461, 602061, 515661, 429261, 342861, 256461, 170061, 83661, -2739, -3600, 0 ]
 }, {
   "tz" : "America/Ojinaga",
-  "switches" : [ -62135571740, -59006336540, -55850662940, -52694989340, -46383555740, -43227882140, -40072208540, -33760774940, -30605101340, -27449427740, -21137994140, -17982320540, -14826646940, -12219267740, -2208988800 ],
-  "diffs" : [ -172660, -86260, 140, 86540, 172940, 259340, 345740, 432140, 518540, 604940, 691340, 777740, 864140, 140, 0 ]
+  "switches" : [ -62135571740, -59006336540, -55850662940, -52694989340, -46383555740, -43227882140, -40072208540, -33760774940, -30605101340, -27449427740, -21137994140, -17982320540, -14826646940, -12220045340, -12219958940, -12219872540, -12219786140, -12219699740, -12219613340, -12219526940, -12219440540, -12219354140, -12219267740, -2208988800 ],
+  "diffs" : [ -172660, -86260, 140, 86540, 172940, 259340, 345740, 432140, 518540, 604940, 691340, 777740, 864140, 777740, 691340, 604940, 518540, 432140, 345740, 259340, 172940, 86540, 140, 0 ]
 }, {
   "tz" : "America/Panama",
-  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12219273712, -2524502512, -2208988800 ],
-  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, -1088, -1176, 0 ]
+  "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12220051312, -12219964912, -12219878512, -12219792112, -12219705712, -12219619312, -12219532912, -12219446512, -12219360112, -12219273712, -2524502512, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, -1176, 0 ]
 }, {
   "tz" : "America/Pangnirtung",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 18000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -154800, -68400, 18000, 104400, 190800, 277200, 363600, 450000, 536400, 622800, 709200, 795600, 882000, 795600, 709200, 622800, 536400, 450000, 363600, 277200, 190800, 104400, 18000, 0 ]
 }, {
   "tz" : "America/Paramaribo",
-  "switches" : [ -62135583560, -59006348360, -55850674760, -52695001160, -46383567560, -43227893960, -40072220360, -33760786760, -30605113160, -27449439560, -21138005960, -17982332360, -14826658760, -12219279560, -2208988800 ],
-  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, -2440, 0 ]
+  "switches" : [ -62135583560, -59006348360, -55850674760, -52695001160, -46383567560, -43227893960, -40072220360, -33760786760, -30605113160, -27449439560, -21138005960, -17982332360, -14826658760, -12220057160, -12219970760, -12219884360, -12219797960, -12219711560, -12219625160, -12219538760, -12219452360, -12219365960, -12219279560, -2208988800 ],
+  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, 775160, 688760, 602360, 515960, 429560, 343160, 256760, 170360, 83960, -2440, 0 ]
 }, {
   "tz" : "America/Phoenix",
-  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12219265902, -2717643600 ],
-  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, -1698, 0 ]
+  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12220043502, -12219957102, -12219870702, -12219784302, -12219697902, -12219611502, -12219525102, -12219438702, -12219352302, -12219265902, -2717643600 ],
+  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, 775902, 689502, 603102, 516702, 430302, 343902, 257502, 171102, 84702, -1698, 0 ]
 }, {
   "tz" : "America/Port-au-Prince",
-  "switches" : [ -62135579440, -59006344240, -55850670640, -52694997040, -46383563440, -43227889840, -40072216240, -33760782640, -30605109040, -27449435440, -21138001840, -17982328240, -14826654640, -12219275440, -2524504240, -2208988800 ],
-  "diffs" : [ -172160, -85760, 640, 87040, 173440, 259840, 346240, 432640, 519040, 605440, 691840, 778240, 864640, 640, 660, 0 ]
+  "switches" : [ -62135579440, -59006344240, -55850670640, -52694997040, -46383563440, -43227889840, -40072216240, -33760782640, -30605109040, -27449435440, -21138001840, -17982328240, -14826654640, -12220053040, -12219966640, -12219880240, -12219793840, -12219707440, -12219621040, -12219534640, -12219448240, -12219361840, -12219275440, -2524504240, -2208988800 ],
+  "diffs" : [ -172160, -85760, 640, 87040, 173440, 259840, 346240, 432640, 519040, 605440, 691840, 778240, 864640, 778240, 691840, 605440, 519040, 432640, 346240, 259840, 173440, 87040, 640, 660, 0 ]
 }, {
   "tz" : "America/Port_of_Spain",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Porto_Acre",
-  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12219276528, -2208988800 ],
-  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 1728, 0 ]
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 779328, 692928, 606528, 520128, 433728, 347328, 260928, 174528, 88128, 1728, 0 ]
 }, {
   "tz" : "America/Porto_Velho",
-  "switches" : [ -62135581464, -59006346264, -55850672664, -52694999064, -46383565464, -43227891864, -40072218264, -33760784664, -30605111064, -27449437464, -21138003864, -17982330264, -14826656664, -12219277464, -2208988800 ],
-  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, -936, 0 ]
+  "switches" : [ -62135581464, -59006346264, -55850672664, -52694999064, -46383565464, -43227891864, -40072218264, -33760784664, -30605111064, -27449437464, -21138003864, -17982330264, -14826656664, -12220055064, -12219968664, -12219882264, -12219795864, -12219709464, -12219623064, -12219536664, -12219450264, -12219363864, -12219277464, -2208988800 ],
+  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, 776664, 690264, 603864, 517464, 431064, 344664, 258264, 171864, 85464, -936, 0 ]
 }, {
   "tz" : "America/Puerto_Rico",
-  "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12219276935, -2233035335 ],
-  "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, -1465, 0 ]
+  "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ],
+  "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ]
 }, {
   "tz" : "America/Punta_Arenas",
-  "switches" : [ -62135579780, -59006344580, -55850670980, -52694997380, -46383563780, -43227890180, -40072216580, -33760782980, -30605109380, -27449435780, -21138002180, -17982328580, -14826654980, -12219275780, -2524504580, -2208988800 ],
-  "diffs" : [ -179020, -92620, -6220, 80180, 166580, 252980, 339380, 425780, 512180, 598580, 684980, 771380, 857780, -6220, -6166, 0 ]
+  "switches" : [ -62135579780, -59006344580, -55850670980, -52694997380, -46383563780, -43227890180, -40072216580, -33760782980, -30605109380, -27449435780, -21138002180, -17982328580, -14826654980, -12220053380, -12219966980, -12219880580, -12219794180, -12219707780, -12219621380, -12219534980, -12219448580, -12219362180, -12219275780, -2524504580, -2208988800 ],
+  "diffs" : [ -179020, -92620, -6220, 80180, 166580, 252980, 339380, 425780, 512180, 598580, 684980, 771380, 857780, 771380, 684980, 598580, 512180, 425780, 339380, 252980, 166580, 80180, -6220, -6166, 0 ]
 }, {
   "tz" : "America/Rainy_River",
-  "switches" : [ -62135574104, -59006338904, -55850665304, -52694991704, -46383558104, -43227884504, -40072210904, -33760777304, -30605103704, -27449430104, -21137996504, -17982322904, -14826649304, -12219270104, -2366732504 ],
-  "diffs" : [ -173896, -87496, -1096, 85304, 171704, 258104, 344504, 430904, 517304, 603704, 690104, 776504, 862904, -1096, 0 ]
+  "switches" : [ -62135574104, -59006338904, -55850665304, -52694991704, -46383558104, -43227884504, -40072210904, -33760777304, -30605103704, -27449430104, -21137996504, -17982322904, -14826649304, -12220047704, -12219961304, -12219874904, -12219788504, -12219702104, -12219615704, -12219529304, -12219442904, -12219356504, -12219270104, -2366732504 ],
+  "diffs" : [ -173896, -87496, -1096, 85304, 171704, 258104, 344504, 430904, 517304, 603704, 690104, 776504, 862904, 776504, 690104, 603704, 517304, 430904, 344504, 258104, 171704, 85304, -1096, 0 ]
 }, {
   "tz" : "America/Rankin_Inlet",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 21600, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 799200, 712800, 626400, 540000, 453600, 367200, 280800, 194400, 108000, 21600, 0 ]
 }, {
   "tz" : "America/Recife",
-  "switches" : [ -62135588424, -59006353224, -55850679624, -52695006024, -46383572424, -43227898824, -40072225224, -33760791624, -30605118024, -27449444424, -21138010824, -17982337224, -14826663624, -12219284424, -2208988800 ],
-  "diffs" : [ -170376, -83976, 2424, 88824, 175224, 261624, 348024, 434424, 520824, 607224, 693624, 780024, 866424, 2424, 0 ]
+  "switches" : [ -62135588424, -59006353224, -55850679624, -52695006024, -46383572424, -43227898824, -40072225224, -33760791624, -30605118024, -27449444424, -21138010824, -17982337224, -14826663624, -12220062024, -12219975624, -12219889224, -12219802824, -12219716424, -12219630024, -12219543624, -12219457224, -12219370824, -12219284424, -2208988800 ],
+  "diffs" : [ -170376, -83976, 2424, 88824, 175224, 261624, 348024, 434424, 520824, 607224, 693624, 780024, 866424, 780024, 693624, 607224, 520824, 434424, 348024, 261624, 175224, 88824, 2424, 0 ]
 }, {
   "tz" : "America/Regina",
-  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12219267684, -2208988800 ],
-  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, -3516, 0 ]
+  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12220045284, -12219958884, -12219872484, -12219786084, -12219699684, -12219613284, -12219526884, -12219440484, -12219354084, -12219267684, -2208988800 ],
+  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, 774084, 687684, 601284, 514884, 428484, 342084, 255684, 169284, 82884, -3516, 0 ]
 }, {
   "tz" : "America/Resolute",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 21600, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -151200, -64800, 21600, 108000, 194400, 280800, 367200, 453600, 540000, 626400, 712800, 799200, 885600, 799200, 712800, 626400, 540000, 453600, 367200, 280800, 194400, 108000, 21600, 0 ]
 }, {
   "tz" : "America/Rio_Branco",
-  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12219276528, -2208988800 ],
-  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 1728, 0 ]
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 779328, 692928, 606528, 520128, 433728, 347328, 260928, 174528, 88128, 1728, 0 ]
 }, {
   "tz" : "America/Rosario",
-  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12219277392, -2208988800 ],
-  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, -4608, 0 ]
+  "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ],
+  "diffs" : [ -177408, -91008, -4608, 81792, 168192, 254592, 340992, 427392, 513792, 600192, 686592, 772992, 859392, 772992, 686592, 600192, 513792, 427392, 340992, 254592, 168192, 81792, -4608, 0 ]
 }, {
   "tz" : "America/Santa_Isabel",
-  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
-  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
 }, {
   "tz" : "America/Santarem",
-  "switches" : [ -62135583672, -59006348472, -55850674872, -52695001272, -46383567672, -43227894072, -40072220472, -33760786872, -30605113272, -27449439672, -21138006072, -17982332472, -14826658872, -12219279672, -2208988800 ],
-  "diffs" : [ -175128, -88728, -2328, 84072, 170472, 256872, 343272, 429672, 516072, 602472, 688872, 775272, 861672, -2328, 0 ]
+  "switches" : [ -62135583672, -59006348472, -55850674872, -52695001272, -46383567672, -43227894072, -40072220472, -33760786872, -30605113272, -27449439672, -21138006072, -17982332472, -14826658872, -12220057272, -12219970872, -12219884472, -12219798072, -12219711672, -12219625272, -12219538872, -12219452472, -12219366072, -12219279672, -2208988800 ],
+  "diffs" : [ -175128, -88728, -2328, 84072, 170472, 256872, 343272, 429672, 516072, 602472, 688872, 775272, 861672, 775272, 688872, 602472, 516072, 429672, 343272, 256872, 170472, 84072, -2328, 0 ]
 }, {
   "tz" : "America/Santiago",
-  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12219275834, -2208988800 ],
-  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, -2566, 0 ]
+  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12220053434, -12219967034, -12219880634, -12219794234, -12219707834, -12219621434, -12219535034, -12219448634, -12219362234, -12219275834, -2208988800 ],
+  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, 775034, 688634, 602234, 515834, 429434, 343034, 256634, 170234, 83834, -2566, 0 ]
 }, {
   "tz" : "America/Santo_Domingo",
-  "switches" : [ -62135580024, -59006344824, -55850671224, -52694997624, -46383564024, -43227890424, -40072216824, -33760783224, -30605109624, -27449436024, -21138002424, -17982328824, -14826655224, -12219276024, -2524504824, -2208988800 ],
-  "diffs" : [ -175176, -88776, -2376, 84024, 170424, 256824, 343224, 429624, 516024, 602424, 688824, 775224, 861624, -2376, -2400, 0 ]
+  "switches" : [ -62135580024, -59006344824, -55850671224, -52694997624, -46383564024, -43227890424, -40072216824, -33760783224, -30605109624, -27449436024, -21138002424, -17982328824, -14826655224, -12220053624, -12219967224, -12219880824, -12219794424, -12219708024, -12219621624, -12219535224, -12219448824, -12219362424, -12219276024, -2524504824, -2208988800 ],
+  "diffs" : [ -175176, -88776, -2376, 84024, 170424, 256824, 343224, 429624, 516024, 602424, 688824, 775224, 861624, 775224, 688824, 602424, 516024, 429624, 343224, 256824, 170424, 84024, -2376, -2400, 0 ]
 }, {
   "tz" : "America/Sao_Paulo",
-  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12219281612, -2208988800 ],
-  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, -388, 0 ]
+  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12220059212, -12219972812, -12219886412, -12219800012, -12219713612, -12219627212, -12219540812, -12219454412, -12219368012, -12219281612, -2208988800 ],
+  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, 777212, 690812, 604412, 518012, 431612, 345212, 258812, 172412, 86012, -388, 0 ]
 }, {
   "tz" : "America/Scoresbysund",
-  "switches" : [ -62135591528, -59006356328, -55850682728, -52695009128, -46383575528, -43227901928, -40072228328, -33760794728, -30605121128, -27449447528, -21138013928, -17982340328, -14826666728, -12219287528, -2208988800 ],
-  "diffs" : [ -174472, -88072, -1672, 84728, 171128, 257528, 343928, 430328, 516728, 603128, 689528, 775928, 862328, -1672, 0 ]
+  "switches" : [ -62135591528, -59006356328, -55850682728, -52695009128, -46383575528, -43227901928, -40072228328, -33760794728, -30605121128, -27449447528, -21138013928, -17982340328, -14826666728, -12220065128, -12219978728, -12219892328, -12219805928, -12219719528, -12219633128, -12219546728, -12219460328, -12219373928, -12219287528, -2208988800 ],
+  "diffs" : [ -174472, -88072, -1672, 84728, 171128, 257528, 343928, 430328, 516728, 603128, 689528, 775928, 862328, 775928, 689528, 603128, 516728, 430328, 343928, 257528, 171128, 84728, -1672, 0 ]
 }, {
   "tz" : "America/Shiprock",
-  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
-  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
 }, {
   "tz" : "America/Sitka",
-  "switches" : [ -62135650727, -59006415527, -55850741927, -52695068327, -46383634727, -43227961127, -40072287527, -33760853927, -30605180327, -27449506727, -21138073127, -17982399527, -14826725927, -12219346727, -3225223727, -2208988800 ],
-  "diffs" : [ -86473, -73, 86327, 172727, 259127, 345527, 431927, 518327, 604727, 691127, 777527, 863927, 950327, 86327, -73, 0 ]
+  "switches" : [ -62135650727, -59006415527, -55850741927, -52695068327, -46383634727, -43227961127, -40072287527, -33760853927, -30605180327, -27449506727, -21138073127, -17982399527, -14826725927, -12220124327, -12220037927, -12219951527, -12219865127, -12219778727, -12219692327, -12219605927, -12219519527, -12219433127, -12219346727, -3225223727, -2208988800 ],
+  "diffs" : [ -86473, -73, 86327, 172727, 259127, 345527, 431927, 518327, 604727, 691127, 777527, 863927, 950327, 863927, 777527, 691127, 604727, 518327, 431927, 345527, 259127, 172727, 86327, -73, 0 ]
 }, {
   "tz" : "America/St_Barthelemy",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/St_Johns",
-  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12219280148, -2208988800 ],
-  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, -52, 0 ]
+  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12220057748, -12219971348, -12219884948, -12219798548, -12219712148, -12219625748, -12219539348, -12219452948, -12219366548, -12219280148, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ]
 }, {
   "tz" : "America/St_Kitts",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/St_Lucia",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/St_Thomas",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/St_Vincent",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Swift_Current",
-  "switches" : [ -62135570920, -59006335720, -55850662120, -52694988520, -46383554920, -43227881320, -40072207720, -33760774120, -30605100520, -27449426920, -21137993320, -17982319720, -14826646120, -12219266920, -2208988800 ],
-  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, -4280, 0 ]
+  "switches" : [ -62135570920, -59006335720, -55850662120, -52694988520, -46383554920, -43227881320, -40072207720, -33760774120, -30605100520, -27449426920, -21137993320, -17982319720, -14826646120, -12220044520, -12219958120, -12219871720, -12219785320, -12219698920, -12219612520, -12219526120, -12219439720, -12219353320, -12219266920, -2208988800 ],
+  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, 773320, 686920, 600520, 514120, 427720, 341320, 254920, 168520, 82120, -4280, 0 ]
 }, {
   "tz" : "America/Tegucigalpa",
-  "switches" : [ -62135575868, -59006340668, -55850667068, -52694993468, -46383559868, -43227886268, -40072212668, -33760779068, -30605105468, -27449431868, -21137998268, -17982324668, -14826651068, -12219271868, -2208988800 ],
-  "diffs" : [ -172132, -85732, 668, 87068, 173468, 259868, 346268, 432668, 519068, 605468, 691868, 778268, 864668, 668, 0 ]
+  "switches" : [ -62135575868, -59006340668, -55850667068, -52694993468, -46383559868, -43227886268, -40072212668, -33760779068, -30605105468, -27449431868, -21137998268, -17982324668, -14826651068, -12220049468, -12219963068, -12219876668, -12219790268, -12219703868, -12219617468, -12219531068, -12219444668, -12219358268, -12219271868, -2208988800 ],
+  "diffs" : [ -172132, -85732, 668, 87068, 173468, 259868, 346268, 432668, 519068, 605468, 691868, 778268, 864668, 778268, 691868, 605468, 519068, 432668, 346268, 259868, 173468, 87068, 668, 0 ]
 }, {
   "tz" : "America/Thule",
-  "switches" : [ -62135580292, -59006345092, -55850671492, -52694997892, -46383564292, -43227890692, -40072217092, -33760783492, -30605109892, -27449436292, -21138002692, -17982329092, -14826655492, -12219276292, -2208988800 ],
-  "diffs" : [ -174908, -88508, -2108, 84292, 170692, 257092, 343492, 429892, 516292, 602692, 689092, 775492, 861892, -2108, 0 ]
+  "switches" : [ -62135580292, -59006345092, -55850671492, -52694997892, -46383564292, -43227890692, -40072217092, -33760783492, -30605109892, -27449436292, -21138002692, -17982329092, -14826655492, -12220053892, -12219967492, -12219881092, -12219794692, -12219708292, -12219621892, -12219535492, -12219449092, -12219362692, -12219276292, -2208988800 ],
+  "diffs" : [ -174908, -88508, -2108, 84292, 170692, 257092, 343492, 429892, 516292, 602692, 689092, 775492, 861892, 775492, 689092, 602692, 516292, 429892, 343492, 257092, 170692, 84292, -2108, 0 ]
 }, {
   "tz" : "America/Thunder_Bay",
-  "switches" : [ -62135575380, -59006340180, -55850666580, -52694992980, -46383559380, -43227885780, -40072212180, -33760778580, -30605104980, -27449431380, -21137997780, -17982324180, -14826650580, -12219271380, -2366733780, -2208988800 ],
-  "diffs" : [ -176220, -89820, -3420, 82980, 169380, 255780, 342180, 428580, 514980, 601380, 687780, 774180, 860580, -3420, -3600, 0 ]
+  "switches" : [ -62135575380, -59006340180, -55850666580, -52694992980, -46383559380, -43227885780, -40072212180, -33760778580, -30605104980, -27449431380, -21137997780, -17982324180, -14826650580, -12220048980, -12219962580, -12219876180, -12219789780, -12219703380, -12219616980, -12219530580, -12219444180, -12219357780, -12219271380, -2366733780, -2208988800 ],
+  "diffs" : [ -176220, -89820, -3420, 82980, 169380, 255780, 342180, 428580, 514980, 601380, 687780, 774180, 860580, 774180, 687780, 601380, 514980, 428580, 342180, 255780, 169380, 82980, -3420, -3600, 0 ]
 }, {
   "tz" : "America/Tijuana",
-  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
-  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
 }, {
   "tz" : "America/Toronto",
-  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12219273748, -2366736148 ],
-  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, -1052, 0 ]
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ]
 }, {
   "tz" : "America/Tortola",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Vancouver",
-  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12219263252, -2713880852 ],
-  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, -748, 0 ]
+  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12220040852, -12219954452, -12219868052, -12219781652, -12219695252, -12219608852, -12219522452, -12219436052, -12219349652, -12219263252, -2713880852 ],
+  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, 776852, 690452, 604052, 517652, 431252, 344852, 258452, 172052, 85652, -748, 0 ]
 }, {
   "tz" : "America/Virgin",
-  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12219278036, -2208988800 ],
-  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, -364, 0 ]
+  "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ],
+  "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ]
 }, {
   "tz" : "America/Whitehorse",
-  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12219260388, -2208988800 ],
-  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, -3612, 0 ]
+  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12220037988, -12219951588, -12219865188, -12219778788, -12219692388, -12219605988, -12219519588, -12219433188, -12219346788, -12219260388, -2208988800 ],
+  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, 773988, 687588, 601188, 514788, 428388, 341988, 255588, 169188, 82788, -3612, 0 ]
 }, {
   "tz" : "America/Winnipeg",
-  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12219269484, -2602258284 ],
-  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, -1716, 0 ]
+  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12220047084, -12219960684, -12219874284, -12219787884, -12219701484, -12219615084, -12219528684, -12219442284, -12219355884, -12219269484, -2602258284 ],
+  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, 775884, 689484, 603084, 516684, 430284, 343884, 257484, 171084, 84684, -1716, 0 ]
 }, {
   "tz" : "America/Yakutat",
-  "switches" : [ -62135649665, -59006414465, -55850740865, -52695067265, -46383633665, -43227960065, -40072286465, -33760852865, -30605179265, -27449505665, -21138072065, -17982398465, -14826724865, -12219345665, -3225223727, -2208988800 ],
-  "diffs" : [ -87535, -1135, 85265, 171665, 258065, 344465, 430865, 517265, 603665, 690065, 776465, 862865, 949265, 85265, -1135, 0 ]
+  "switches" : [ -62135649665, -59006414465, -55850740865, -52695067265, -46383633665, -43227960065, -40072286465, -33760852865, -30605179265, -27449505665, -21138072065, -17982398465, -14826724865, -12220123265, -12220036865, -12219950465, -12219864065, -12219777665, -12219691265, -12219604865, -12219518465, -12219432065, -12219345665, -3225223727, -2208988800 ],
+  "diffs" : [ -87535, -1135, 85265, 171665, 258065, 344465, 430865, 517265, 603665, 690065, 776465, 862865, 949265, 862865, 776465, 690065, 603665, 517265, 430865, 344465, 258065, 171665, 85265, -1135, 0 ]
 }, {
   "tz" : "America/Yellowknife",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 25200, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -147600, -61200, 25200, 111600, 198000, 284400, 370800, 457200, 543600, 630000, 716400, 802800, 889200, 802800, 716400, 630000, 543600, 457200, 370800, 284400, 198000, 111600, 25200, 0 ]
 }, {
   "tz" : "Antarctica/Casey",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -201600, -115200, -28800, 57600, 144000, 230400, 316800, 403200, 489600, 576000, 662400, 748800, 835200, -28800, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -201600, -115200, -28800, 57600, 144000, 230400, 316800, 403200, 489600, 576000, 662400, 748800, 835200, 748800, 662400, 576000, 489600, 403200, 316800, 230400, 144000, 57600, -28800, 0 ]
 }, {
   "tz" : "Antarctica/Davis",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -198000, -111600, -25200, 61200, 147600, 234000, 320400, 406800, 493200, 579600, 666000, 752400, 838800, -25200, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -198000, -111600, -25200, 61200, 147600, 234000, 320400, 406800, 493200, 579600, 666000, 752400, 838800, 752400, 666000, 579600, 493200, 406800, 320400, 234000, 147600, 61200, -25200, 0 ]
 }, {
   "tz" : "Antarctica/DumontDUrville",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -208800, -122400, -36000, 50400, 136800, 223200, 309600, 396000, 482400, 568800, 655200, 741600, 828000, -36000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -208800, -122400, -36000, 50400, 136800, 223200, 309600, 396000, 482400, 568800, 655200, 741600, 828000, 741600, 655200, 568800, 482400, 396000, 309600, 223200, 136800, 50400, -36000, 0 ]
 }, {
   "tz" : "Antarctica/Macquarie",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2214259200, -2208988800 ],
-  "diffs" : [ -212400, -126000, -39600, 46800, 133200, 219600, 306000, 392400, 478800, 565200, 651600, 738000, 824400, -39600, -3600, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2214259200, -2208988800 ],
+  "diffs" : [ -212400, -126000, -39600, 46800, 133200, 219600, 306000, 392400, 478800, 565200, 651600, 738000, 824400, 738000, 651600, 565200, 478800, 392400, 306000, 219600, 133200, 46800, -39600, -3600, 0 ]
 }, {
   "tz" : "Antarctica/Mawson",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, -18000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, 759600, 673200, 586800, 500400, 414000, 327600, 241200, 154800, 68400, -18000, 0 ]
 }, {
   "tz" : "Antarctica/McMurdo",
-  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
-  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
 }, {
   "tz" : "Antarctica/Palmer",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 10800, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 788400, 702000, 615600, 529200, 442800, 356400, 270000, 183600, 97200, 10800, 0 ]
 }, {
   "tz" : "Antarctica/Rothera",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 10800, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -162000, -75600, 10800, 97200, 183600, 270000, 356400, 442800, 529200, 615600, 702000, 788400, 874800, 788400, 702000, 615600, 529200, 442800, 356400, 270000, 183600, 97200, 10800, 0 ]
 }, {
   "tz" : "Antarctica/South_Pole",
-  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
-  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
 }, {
   "tz" : "Antarctica/Syowa",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -183600, -97200, -10800, 75600, 162000, 248400, 334800, 421200, 507600, 594000, 680400, 766800, 853200, -10800, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -183600, -97200, -10800, 75600, 162000, 248400, 334800, 421200, 507600, 594000, 680400, 766800, 853200, 766800, 680400, 594000, 507600, 421200, 334800, 248400, 162000, 75600, -10800, 0 ]
 }, {
   "tz" : "Antarctica/Troll",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Antarctica/Vostok",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -194400, -108000, -21600, 64800, 151200, 237600, 324000, 410400, 496800, 583200, 669600, 756000, 842400, -21600, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -194400, -108000, -21600, 64800, 151200, 237600, 324000, 410400, 496800, 583200, 669600, 756000, 842400, 756000, 669600, 583200, 496800, 410400, 324000, 237600, 151200, 64800, -21600, 0 ]
 }, {
   "tz" : "Arctic/Longyearbyen",
-  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12219295380, -2366757780 ],
-  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, -1020, 0 ]
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ]
 }, {
   "tz" : "Asia/Aden",
-  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12219304012, -2208988800 ],
-  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 412, 0 ]
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ]
 }, {
   "tz" : "Asia/Almaty",
-  "switches" : [ -62135615268, -59006380068, -55850706468, -52695032868, -46383599268, -43227925668, -40072252068, -33760818468, -30605144868, -27449471268, -21138037668, -17982364068, -14826690468, -12219311268, -2208988800 ],
-  "diffs" : [ -175932, -89532, -3132, 83268, 169668, 256068, 342468, 428868, 515268, 601668, 688068, 774468, 860868, -3132, 0 ]
+  "switches" : [ -62135615268, -59006380068, -55850706468, -52695032868, -46383599268, -43227925668, -40072252068, -33760818468, -30605144868, -27449471268, -21138037668, -17982364068, -14826690468, -12220088868, -12220002468, -12219916068, -12219829668, -12219743268, -12219656868, -12219570468, -12219484068, -12219397668, -12219311268, -2208988800 ],
+  "diffs" : [ -175932, -89532, -3132, 83268, 169668, 256068, 342468, 428868, 515268, 601668, 688068, 774468, 860868, 774468, 688068, 601668, 515268, 428868, 342468, 256068, 169668, 83268, -3132, 0 ]
 }, {
   "tz" : "Asia/Amman",
-  "switches" : [ -62135605424, -59006370224, -55850696624, -52695023024, -46383589424, -43227915824, -40072242224, -33760808624, -30605135024, -27449461424, -21138027824, -17982354224, -14826680624, -12219301424, -2208988800 ],
-  "diffs" : [ -171376, -84976, 1424, 87824, 174224, 260624, 347024, 433424, 519824, 606224, 692624, 779024, 865424, 1424, 0 ]
+  "switches" : [ -62135605424, -59006370224, -55850696624, -52695023024, -46383589424, -43227915824, -40072242224, -33760808624, -30605135024, -27449461424, -21138027824, -17982354224, -14826680624, -12220079024, -12219992624, -12219906224, -12219819824, -12219733424, -12219647024, -12219560624, -12219474224, -12219387824, -12219301424, -2208988800 ],
+  "diffs" : [ -171376, -84976, 1424, 87824, 174224, 260624, 347024, 433424, 519824, 606224, 692624, 779024, 865424, 779024, 692624, 606224, 519824, 433424, 347024, 260624, 174224, 87824, 1424, 0 ]
 }, {
   "tz" : "Asia/Anadyr",
-  "switches" : [ -62135639396, -59006404196, -55850730596, -52695056996, -46383623396, -43227949796, -40072276196, -33760842596, -30605168996, -27449495396, -21138061796, -17982388196, -14826714596, -12219335396, -2208988800 ],
-  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+  "switches" : [ -62135639396, -59006404196, -55850730596, -52695056996, -46383623396, -43227949796, -40072276196, -33760842596, -30605168996, -27449495396, -21138061796, -17982388196, -14826714596, -12220112996, -12220026596, -12219940196, -12219853796, -12219767396, -12219680996, -12219594596, -12219508196, -12219421796, -12219335396, -2208988800 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
 }, {
   "tz" : "Asia/Aqtau",
-  "switches" : [ -62135608864, -59006373664, -55850700064, -52695026464, -46383592864, -43227919264, -40072245664, -33760812064, -30605138464, -27449464864, -21138031264, -17982357664, -14826684064, -12219304864, -2208988800 ],
-  "diffs" : [ -178736, -92336, -5936, 80464, 166864, 253264, 339664, 426064, 512464, 598864, 685264, 771664, 858064, -5936, 0 ]
+  "switches" : [ -62135608864, -59006373664, -55850700064, -52695026464, -46383592864, -43227919264, -40072245664, -33760812064, -30605138464, -27449464864, -21138031264, -17982357664, -14826684064, -12220082464, -12219996064, -12219909664, -12219823264, -12219736864, -12219650464, -12219564064, -12219477664, -12219391264, -12219304864, -2208988800 ],
+  "diffs" : [ -178736, -92336, -5936, 80464, 166864, 253264, 339664, 426064, 512464, 598864, 685264, 771664, 858064, 771664, 685264, 598864, 512464, 426064, 339664, 253264, 166864, 80464, -5936, 0 ]
 }, {
   "tz" : "Asia/Aqtobe",
-  "switches" : [ -62135610520, -59006375320, -55850701720, -52695028120, -46383594520, -43227920920, -40072247320, -33760813720, -30605140120, -27449466520, -21138032920, -17982359320, -14826685720, -12219306520, -2208988800 ],
-  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, -4280, 0 ]
+  "switches" : [ -62135610520, -59006375320, -55850701720, -52695028120, -46383594520, -43227920920, -40072247320, -33760813720, -30605140120, -27449466520, -21138032920, -17982359320, -14826685720, -12220084120, -12219997720, -12219911320, -12219824920, -12219738520, -12219652120, -12219565720, -12219479320, -12219392920, -12219306520, -2208988800 ],
+  "diffs" : [ -177080, -90680, -4280, 82120, 168520, 254920, 341320, 427720, 514120, 600520, 686920, 773320, 859720, 773320, 686920, 600520, 514120, 427720, 341320, 254920, 168520, 82120, -4280, 0 ]
 }, {
   "tz" : "Asia/Ashgabat",
-  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12219306812, -2208988800 ],
-  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, 0 ]
+  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12220084412, -12219998012, -12219911612, -12219825212, -12219738812, -12219652412, -12219566012, -12219479612, -12219393212, -12219306812, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, 0 ]
 }, {
   "tz" : "Asia/Ashkhabad",
-  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12219306812, -2208988800 ],
-  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, -3988, 0 ]
+  "switches" : [ -62135610812, -59006375612, -55850702012, -52695028412, -46383594812, -43227921212, -40072247612, -33760814012, -30605140412, -27449466812, -21138033212, -17982359612, -14826686012, -12220084412, -12219998012, -12219911612, -12219825212, -12219738812, -12219652412, -12219566012, -12219479612, -12219393212, -12219306812, -2208988800 ],
+  "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, 0 ]
 }, {
   "tz" : "Asia/Atyrau",
-  "switches" : [ -62135609264, -59006374064, -55850700464, -52695026864, -46383593264, -43227919664, -40072246064, -33760812464, -30605138864, -27449465264, -21138031664, -17982358064, -14826684464, -12219305264, -2208988800 ],
-  "diffs" : [ -178336, -91936, -5536, 80864, 167264, 253664, 340064, 426464, 512864, 599264, 685664, 772064, 858464, -5536, 0 ]
+  "switches" : [ -62135609264, -59006374064, -55850700464, -52695026864, -46383593264, -43227919664, -40072246064, -33760812464, -30605138864, -27449465264, -21138031664, -17982358064, -14826684464, -12220082864, -12219996464, -12219910064, -12219823664, -12219737264, -12219650864, -12219564464, -12219478064, -12219391664, -12219305264, -2208988800 ],
+  "diffs" : [ -178336, -91936, -5536, 80864, 167264, 253664, 340064, 426464, 512864, 599264, 685664, 772064, 858464, 772064, 685664, 599264, 512864, 426464, 340064, 253664, 167264, 80864, -5536, 0 ]
 }, {
   "tz" : "Asia/Baghdad",
-  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12219303460, -2524532260, -2208988800 ],
-  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, -140, -144, 0 ]
+  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12220081060, -12219994660, -12219908260, -12219821860, -12219735460, -12219649060, -12219562660, -12219476260, -12219389860, -12219303460, -2524532260, -2208988800 ],
+  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, 777460, 691060, 604660, 518260, 431860, 345460, 259060, 172660, 86260, -140, -144, 0 ]
 }, {
   "tz" : "Asia/Bahrain",
-  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12219305168, -2208988800 ],
-  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 1568, 0 ]
+  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12220082768, -12219996368, -12219909968, -12219823568, -12219737168, -12219650768, -12219564368, -12219477968, -12219391568, -12219305168, -2208988800 ],
+  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 779168, 692768, 606368, 519968, 433568, 347168, 260768, 174368, 87968, 1568, 0 ]
 }, {
   "tz" : "Asia/Baku",
-  "switches" : [ -62135608764, -59006373564, -55850699964, -52695026364, -46383592764, -43227919164, -40072245564, -33760811964, -30605138364, -27449464764, -21138031164, -17982357564, -14826683964, -12219304764, -2208988800 ],
-  "diffs" : [ -175236, -88836, -2436, 83964, 170364, 256764, 343164, 429564, 515964, 602364, 688764, 775164, 861564, -2436, 0 ]
+  "switches" : [ -62135608764, -59006373564, -55850699964, -52695026364, -46383592764, -43227919164, -40072245564, -33760811964, -30605138364, -27449464764, -21138031164, -17982357564, -14826683964, -12220082364, -12219995964, -12219909564, -12219823164, -12219736764, -12219650364, -12219563964, -12219477564, -12219391164, -12219304764, -2208988800 ],
+  "diffs" : [ -175236, -88836, -2436, 83964, 170364, 256764, 343164, 429564, 515964, 602364, 688764, 775164, 861564, 775164, 688764, 602364, 515964, 429564, 343164, 256764, 170364, 83964, -2436, 0 ]
 }, {
   "tz" : "Asia/Bangkok",
-  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12219316924, -2208988800 ],
-  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, -1076, 0 ]
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ]
 }, {
   "tz" : "Asia/Barnaul",
-  "switches" : [ -62135616900, -59006381700, -55850708100, -52695034500, -46383600900, -43227927300, -40072253700, -33760820100, -30605146500, -27449472900, -21138039300, -17982365700, -14826692100, -12219312900, -2208988800 ],
-  "diffs" : [ -177900, -91500, -5100, 81300, 167700, 254100, 340500, 426900, 513300, 599700, 686100, 772500, 858900, -5100, 0 ]
+  "switches" : [ -62135616900, -59006381700, -55850708100, -52695034500, -46383600900, -43227927300, -40072253700, -33760820100, -30605146500, -27449472900, -21138039300, -17982365700, -14826692100, -12220090500, -12220004100, -12219917700, -12219831300, -12219744900, -12219658500, -12219572100, -12219485700, -12219399300, -12219312900, -2208988800 ],
+  "diffs" : [ -177900, -91500, -5100, 81300, 167700, 254100, 340500, 426900, 513300, 599700, 686100, 772500, 858900, 772500, 686100, 599700, 513300, 426900, 340500, 254100, 167700, 81300, -5100, 0 ]
 }, {
   "tz" : "Asia/Beirut",
-  "switches" : [ -62135605320, -59006370120, -55850696520, -52695022920, -46383589320, -43227915720, -40072242120, -33760808520, -30605134920, -27449461320, -21138027720, -17982354120, -14826680520, -12219301320, -2840149320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135605320, -59006370120, -55850696520, -52695022920, -46383589320, -43227915720, -40072242120, -33760808520, -30605134920, -27449461320, -21138027720, -17982354120, -14826680520, -12220078920, -12219992520, -12219906120, -12219819720, -12219733320, -12219646920, -12219560520, -12219474120, -12219387720, -12219301320, -2840149320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Asia/Bishkek",
-  "switches" : [ -62135614704, -59006379504, -55850705904, -52695032304, -46383598704, -43227925104, -40072251504, -33760817904, -30605144304, -27449470704, -21138037104, -17982363504, -14826689904, -12219310704, -2208988800 ],
-  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, -3696, 0 ]
+  "switches" : [ -62135614704, -59006379504, -55850705904, -52695032304, -46383598704, -43227925104, -40072251504, -33760817904, -30605144304, -27449470704, -21138037104, -17982363504, -14826689904, -12220088304, -12220001904, -12219915504, -12219829104, -12219742704, -12219656304, -12219569904, -12219483504, -12219397104, -12219310704, -2208988800 ],
+  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, 773904, 687504, 601104, 514704, 428304, 341904, 255504, 169104, 82704, -3696, 0 ]
 }, {
   "tz" : "Asia/Brunei",
-  "switches" : [ -62135624380, -59006389180, -55850715580, -52695041980, -46383608380, -43227934780, -40072261180, -33760827580, -30605153980, -27449480380, -21138046780, -17982373180, -14826699580, -12219320380, -2208988800 ],
-  "diffs" : [ -174020, -87620, -1220, 85180, 171580, 257980, 344380, 430780, 517180, 603580, 689980, 776380, 862780, -1220, 0 ]
+  "switches" : [ -62135624380, -59006389180, -55850715580, -52695041980, -46383608380, -43227934780, -40072261180, -33760827580, -30605153980, -27449480380, -21138046780, -17982373180, -14826699580, -12220097980, -12220011580, -12219925180, -12219838780, -12219752380, -12219665980, -12219579580, -12219493180, -12219406780, -12219320380, -2208988800 ],
+  "diffs" : [ -174020, -87620, -1220, 85180, 171580, 257980, 344380, 430780, 517180, 603580, 689980, 776380, 862780, 776380, 689980, 603580, 517180, 430780, 344380, 257980, 171580, 85180, -1220, 0 ]
 }, {
   "tz" : "Asia/Calcutta",
-  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12219314008, -3645237208, -3155694800, -2208988800 ],
-  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 1408, 1400, -530, 0 ]
+  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12220091608, -12220005208, -12219918808, -12219832408, -12219746008, -12219659608, -12219573208, -12219486808, -12219400408, -12219314008, -3645237208, -3155694800, -2208988800 ],
+  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 779008, 692608, 606208, 519808, 433408, 347008, 260608, 174208, 87808, 1408, 1400, -530, 0 ]
 }, {
   "tz" : "Asia/Chita",
-  "switches" : [ -62135624032, -59006388832, -55850715232, -52695041632, -46383608032, -43227934432, -40072260832, -33760827232, -30605153632, -27449480032, -21138046432, -17982372832, -14826699232, -12219320032, -2208988800 ],
-  "diffs" : [ -177968, -91568, -5168, 81232, 167632, 254032, 340432, 426832, 513232, 599632, 686032, 772432, 858832, -5168, 0 ]
+  "switches" : [ -62135624032, -59006388832, -55850715232, -52695041632, -46383608032, -43227934432, -40072260832, -33760827232, -30605153632, -27449480032, -21138046432, -17982372832, -14826699232, -12220097632, -12220011232, -12219924832, -12219838432, -12219752032, -12219665632, -12219579232, -12219492832, -12219406432, -12219320032, -2208988800 ],
+  "diffs" : [ -177968, -91568, -5168, 81232, 167632, 254032, 340432, 426832, 513232, 599632, 686032, 772432, 858832, 772432, 686032, 599632, 513232, 426832, 340432, 254032, 167632, 81232, -5168, 0 ]
 }, {
   "tz" : "Asia/Choibalsan",
-  "switches" : [ -62135624280, -59006389080, -55850715480, -52695041880, -46383608280, -43227934680, -40072261080, -33760827480, -30605153880, -27449480280, -21138046680, -17982373080, -14826699480, -12219320280, -2208988800 ],
-  "diffs" : [ -174120, -87720, -1320, 85080, 171480, 257880, 344280, 430680, 517080, 603480, 689880, 776280, 862680, -1320, 0 ]
+  "switches" : [ -62135624280, -59006389080, -55850715480, -52695041880, -46383608280, -43227934680, -40072261080, -33760827480, -30605153880, -27449480280, -21138046680, -17982373080, -14826699480, -12220097880, -12220011480, -12219925080, -12219838680, -12219752280, -12219665880, -12219579480, -12219493080, -12219406680, -12219320280, -2208988800 ],
+  "diffs" : [ -174120, -87720, -1320, 85080, 171480, 257880, 344280, 430680, 517080, 603480, 689880, 776280, 862680, 776280, 689880, 603480, 517080, 430680, 344280, 257880, 171480, 85080, -1320, 0 ]
 }, {
   "tz" : "Asia/Chongqing",
-  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
-  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
 }, {
   "tz" : "Asia/Chungking",
-  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
-  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
 }, {
   "tz" : "Asia/Colombo",
-  "switches" : [ -62135615964, -59006380764, -55850707164, -52695033564, -46383599964, -43227926364, -40072252764, -33760819164, -30605145564, -27449471964, -21138038364, -17982364764, -14826691164, -12219311964, -2840159964, -2208988800 ],
-  "diffs" : [ -173436, -87036, -636, 85764, 172164, 258564, 344964, 431364, 517764, 604164, 690564, 776964, 863364, -636, -628, 0 ]
+  "switches" : [ -62135615964, -59006380764, -55850707164, -52695033564, -46383599964, -43227926364, -40072252764, -33760819164, -30605145564, -27449471964, -21138038364, -17982364764, -14826691164, -12220089564, -12220003164, -12219916764, -12219830364, -12219743964, -12219657564, -12219571164, -12219484764, -12219398364, -12219311964, -2840159964, -2208988800 ],
+  "diffs" : [ -173436, -87036, -636, 85764, 172164, 258564, 344964, 431364, 517764, 604164, 690564, 776964, 863364, 776964, 690564, 604164, 517764, 431364, 344964, 258564, 172164, 85764, -636, -628, 0 ]
 }, {
   "tz" : "Asia/Dacca",
-  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12219314500, -2524543300, -2208988800 ],
-  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 100, -400, 0 ]
+  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12220092100, -12220005700, -12219919300, -12219832900, -12219746500, -12219660100, -12219573700, -12219487300, -12219400900, -12219314500, -2524543300, -2208988800 ],
+  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 777700, 691300, 604900, 518500, 432100, 345700, 259300, 172900, 86500, 100, -400, 0 ]
 }, {
   "tz" : "Asia/Damascus",
-  "switches" : [ -62135605512, -59006370312, -55850696712, -52695023112, -46383589512, -43227915912, -40072242312, -33760808712, -30605135112, -27449461512, -21138027912, -17982354312, -14826680712, -12219301512, -2208988800 ],
-  "diffs" : [ -171288, -84888, 1512, 87912, 174312, 260712, 347112, 433512, 519912, 606312, 692712, 779112, 865512, 1512, 0 ]
+  "switches" : [ -62135605512, -59006370312, -55850696712, -52695023112, -46383589512, -43227915912, -40072242312, -33760808712, -30605135112, -27449461512, -21138027912, -17982354312, -14826680712, -12220079112, -12219992712, -12219906312, -12219819912, -12219733512, -12219647112, -12219560712, -12219474312, -12219387912, -12219301512, -2208988800 ],
+  "diffs" : [ -171288, -84888, 1512, 87912, 174312, 260712, 347112, 433512, 519912, 606312, 692712, 779112, 865512, 779112, 692712, 606312, 519912, 433512, 347112, 260712, 174312, 87912, 1512, 0 ]
 }, {
   "tz" : "Asia/Dhaka",
-  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12219314500, -2524543300, -2208988800 ],
-  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 100, -400, 0 ]
+  "switches" : [ -62135618500, -59006383300, -55850709700, -52695036100, -46383602500, -43227928900, -40072255300, -33760821700, -30605148100, -27449474500, -21138040900, -17982367300, -14826693700, -12220092100, -12220005700, -12219919300, -12219832900, -12219746500, -12219660100, -12219573700, -12219487300, -12219400900, -12219314500, -2524543300, -2208988800 ],
+  "diffs" : [ -172700, -86300, 100, 86500, 172900, 259300, 345700, 432100, 518500, 604900, 691300, 777700, 864100, 777700, 691300, 604900, 518500, 432100, 345700, 259300, 172900, 86500, 100, -400, 0 ]
 }, {
   "tz" : "Asia/Dili",
-  "switches" : [ -62135626940, -59006391740, -55850718140, -52695044540, -46383610940, -43227937340, -40072263740, -33760830140, -30605156540, -27449482940, -21138049340, -17982375740, -14826702140, -12219322940, -2208988800 ],
-  "diffs" : [ -175060, -88660, -2260, 84140, 170540, 256940, 343340, 429740, 516140, 602540, 688940, 775340, 861740, -2260, 0 ]
+  "switches" : [ -62135626940, -59006391740, -55850718140, -52695044540, -46383610940, -43227937340, -40072263740, -33760830140, -30605156540, -27449482940, -21138049340, -17982375740, -14826702140, -12220100540, -12220014140, -12219927740, -12219841340, -12219754940, -12219668540, -12219582140, -12219495740, -12219409340, -12219322940, -2208988800 ],
+  "diffs" : [ -175060, -88660, -2260, 84140, 170540, 256940, 343340, 429740, 516140, 602540, 688940, 775340, 861740, 775340, 688940, 602540, 516140, 429740, 343340, 256940, 170540, 84140, -2260, 0 ]
 }, {
   "tz" : "Asia/Dubai",
-  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12219306072, -2208988800 ],
-  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, -1128, 0 ]
+  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12220083672, -12219997272, -12219910872, -12219824472, -12219738072, -12219651672, -12219565272, -12219478872, -12219392472, -12219306072, -2208988800 ],
+  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, 776472, 690072, 603672, 517272, 430872, 344472, 258072, 171672, 85272, -1128, 0 ]
 }, {
   "tz" : "Asia/Dushanbe",
-  "switches" : [ -62135613312, -59006378112, -55850704512, -52695030912, -46383597312, -43227923712, -40072250112, -33760816512, -30605142912, -27449469312, -21138035712, -17982362112, -14826688512, -12219309312, -2208988800 ],
-  "diffs" : [ -174288, -87888, -1488, 84912, 171312, 257712, 344112, 430512, 516912, 603312, 689712, 776112, 862512, -1488, 0 ]
+  "switches" : [ -62135613312, -59006378112, -55850704512, -52695030912, -46383597312, -43227923712, -40072250112, -33760816512, -30605142912, -27449469312, -21138035712, -17982362112, -14826688512, -12220086912, -12220000512, -12219914112, -12219827712, -12219741312, -12219654912, -12219568512, -12219482112, -12219395712, -12219309312, -2208988800 ],
+  "diffs" : [ -174288, -87888, -1488, 84912, 171312, 257712, 344112, 430512, 516912, 603312, 689712, 776112, 862512, 776112, 689712, 603312, 516912, 430512, 344112, 257712, 171312, 84912, -1488, 0 ]
 }, {
   "tz" : "Asia/Famagusta",
-  "switches" : [ -62135604948, -59006369748, -55850696148, -52695022548, -46383588948, -43227915348, -40072241748, -33760808148, -30605134548, -27449460948, -21138027348, -17982353748, -14826680148, -12219300948, -2208988800 ],
-  "diffs" : [ -171852, -85452, 948, 87348, 173748, 260148, 346548, 432948, 519348, 605748, 692148, 778548, 864948, 948, 0 ]
+  "switches" : [ -62135604948, -59006369748, -55850696148, -52695022548, -46383588948, -43227915348, -40072241748, -33760808148, -30605134548, -27449460948, -21138027348, -17982353748, -14826680148, -12220078548, -12219992148, -12219905748, -12219819348, -12219732948, -12219646548, -12219560148, -12219473748, -12219387348, -12219300948, -2208988800 ],
+  "diffs" : [ -171852, -85452, 948, 87348, 173748, 260148, 346548, 432948, 519348, 605748, 692148, 778548, 864948, 778548, 692148, 605748, 519348, 432948, 346548, 260148, 173748, 87348, 948, 0 ]
 }, {
   "tz" : "Asia/Gaza",
-  "switches" : [ -62135605072, -59006369872, -55850696272, -52695022672, -46383589072, -43227915472, -40072241872, -33760808272, -30605134672, -27449461072, -21138027472, -17982353872, -14826680272, -12219301072, -2208988800 ],
-  "diffs" : [ -171728, -85328, 1072, 87472, 173872, 260272, 346672, 433072, 519472, 605872, 692272, 778672, 865072, 1072, 0 ]
+  "switches" : [ -62135605072, -59006369872, -55850696272, -52695022672, -46383589072, -43227915472, -40072241872, -33760808272, -30605134672, -27449461072, -21138027472, -17982353872, -14826680272, -12220078672, -12219992272, -12219905872, -12219819472, -12219733072, -12219646672, -12219560272, -12219473872, -12219387472, -12219301072, -2208988800 ],
+  "diffs" : [ -171728, -85328, 1072, 87472, 173872, 260272, 346672, 433072, 519472, 605872, 692272, 778672, 865072, 778672, 692272, 605872, 519472, 433072, 346672, 260272, 173872, 87472, 1072, 0 ]
 }, {
   "tz" : "Asia/Harbin",
-  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
-  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
 }, {
   "tz" : "Asia/Hebron",
-  "switches" : [ -62135605223, -59006370023, -55850696423, -52695022823, -46383589223, -43227915623, -40072242023, -33760808423, -30605134823, -27449461223, -21138027623, -17982354023, -14826680423, -12219301223, -2208988800 ],
-  "diffs" : [ -171577, -85177, 1223, 87623, 174023, 260423, 346823, 433223, 519623, 606023, 692423, 778823, 865223, 1223, 0 ]
+  "switches" : [ -62135605223, -59006370023, -55850696423, -52695022823, -46383589223, -43227915623, -40072242023, -33760808423, -30605134823, -27449461223, -21138027623, -17982354023, -14826680423, -12220078823, -12219992423, -12219906023, -12219819623, -12219733223, -12219646823, -12219560423, -12219474023, -12219387623, -12219301223, -2208988800 ],
+  "diffs" : [ -171577, -85177, 1223, 87623, 174023, 260423, 346823, 433223, 519623, 606023, 692423, 778823, 865223, 778823, 692423, 606023, 519623, 433223, 346823, 260423, 174023, 87623, 1223, 0 ]
 }, {
   "tz" : "Asia/Ho_Chi_Minh",
-  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12219318400, -2208988800 ],
-  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 400, 0 ]
+  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12220096000, -12220009600, -12219923200, -12219836800, -12219750400, -12219664000, -12219577600, -12219491200, -12219404800, -12219318400, -2208988800 ],
+  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 778000, 691600, 605200, 518800, 432400, 346000, 259600, 173200, 86800, 400, 0 ]
 }, {
   "tz" : "Asia/Hong_Kong",
-  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12219320202, -2208988800 ],
-  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, -1398, 0 ]
+  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12220097802, -12220011402, -12219925002, -12219838602, -12219752202, -12219665802, -12219579402, -12219493002, -12219406602, -12219320202, -2208988800 ],
+  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, 776202, 689802, 603402, 517002, 430602, 344202, 257802, 171402, 85002, -1398, 0 ]
 }, {
   "tz" : "Asia/Hovd",
-  "switches" : [ -62135618796, -59006383596, -55850709996, -52695036396, -46383602796, -43227929196, -40072255596, -33760821996, -30605148396, -27449474796, -21138041196, -17982367596, -14826693996, -12219314796, -2208988800 ],
-  "diffs" : [ -176004, -89604, -3204, 83196, 169596, 255996, 342396, 428796, 515196, 601596, 687996, 774396, 860796, -3204, 0 ]
+  "switches" : [ -62135618796, -59006383596, -55850709996, -52695036396, -46383602796, -43227929196, -40072255596, -33760821996, -30605148396, -27449474796, -21138041196, -17982367596, -14826693996, -12220092396, -12220005996, -12219919596, -12219833196, -12219746796, -12219660396, -12219573996, -12219487596, -12219401196, -12219314796, -2208988800 ],
+  "diffs" : [ -176004, -89604, -3204, 83196, 169596, 255996, 342396, 428796, 515196, 601596, 687996, 774396, 860796, 774396, 687996, 601596, 515196, 428796, 342396, 255996, 169596, 83196, -3204, 0 ]
 }, {
   "tz" : "Asia/Irkutsk",
-  "switches" : [ -62135621825, -59006386625, -55850713025, -52695039425, -46383605825, -43227932225, -40072258625, -33760825025, -30605151425, -27449477825, -21138044225, -17982370625, -14826697025, -12219317825, -2208988800 ],
-  "diffs" : [ -176575, -90175, -3775, 82625, 169025, 255425, 341825, 428225, 514625, 601025, 687425, 773825, 860225, -3775, 0 ]
+  "switches" : [ -62135621825, -59006386625, -55850713025, -52695039425, -46383605825, -43227932225, -40072258625, -33760825025, -30605151425, -27449477825, -21138044225, -17982370625, -14826697025, -12220095425, -12220009025, -12219922625, -12219836225, -12219749825, -12219663425, -12219577025, -12219490625, -12219404225, -12219317825, -2208988800 ],
+  "diffs" : [ -176575, -90175, -3775, 82625, 169025, 255425, 341825, 428225, 514625, 601025, 687425, 773825, 860225, 773825, 687425, 601025, 514625, 428225, 341825, 255425, 169025, 82625, -3775, 0 ]
 }, {
   "tz" : "Asia/Istanbul",
-  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12219299752, -2840147752, -2208988800 ],
-  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, -3848, -3784, 0 ]
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12220077352, -12219990952, -12219904552, -12219818152, -12219731752, -12219645352, -12219558952, -12219472552, -12219386152, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, 773752, 687352, 600952, 514552, 428152, 341752, 255352, 168952, 82552, -3848, -3784, 0 ]
 }, {
   "tz" : "Asia/Jakarta",
-  "switches" : [ -62135622432, -59006387232, -55850713632, -52695040032, -46383606432, -43227932832, -40072259232, -33760825632, -30605152032, -27449478432, -21138044832, -17982371232, -14826697632, -12219318432, -2208988800 ],
-  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 432, 0 ]
+  "switches" : [ -62135622432, -59006387232, -55850713632, -52695040032, -46383606432, -43227932832, -40072259232, -33760825632, -30605152032, -27449478432, -21138044832, -17982371232, -14826697632, -12220096032, -12220009632, -12219923232, -12219836832, -12219750432, -12219664032, -12219577632, -12219491232, -12219404832, -12219318432, -2208988800 ],
+  "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 778032, 691632, 605232, 518832, 432432, 346032, 259632, 173232, 86832, 432, 0 ]
 }, {
   "tz" : "Asia/Jayapura",
-  "switches" : [ -62135630568, -59006395368, -55850721768, -52695048168, -46383614568, -43227940968, -40072267368, -33760833768, -30605160168, -27449486568, -21138052968, -17982379368, -14826705768, -12219326568, -2208988800 ],
-  "diffs" : [ -171432, -85032, 1368, 87768, 174168, 260568, 346968, 433368, 519768, 606168, 692568, 778968, 865368, 1368, 0 ]
+  "switches" : [ -62135630568, -59006395368, -55850721768, -52695048168, -46383614568, -43227940968, -40072267368, -33760833768, -30605160168, -27449486568, -21138052968, -17982379368, -14826705768, -12220104168, -12220017768, -12219931368, -12219844968, -12219758568, -12219672168, -12219585768, -12219499368, -12219412968, -12219326568, -2208988800 ],
+  "diffs" : [ -171432, -85032, 1368, 87768, 174168, 260568, 346968, 433368, 519768, 606168, 692568, 778968, 865368, 778968, 692568, 606168, 519768, 433368, 346968, 260568, 174168, 87768, 1368, 0 ]
 }, {
   "tz" : "Asia/Jerusalem",
-  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12219301254, -2840149254, -2208988800 ],
-  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 1254, 1240, 0 ]
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 778854, 692454, 606054, 519654, 433254, 346854, 260454, 174054, 87654, 1254, 1240, 0 ]
 }, {
   "tz" : "Asia/Kabul",
-  "switches" : [ -62135613408, -59006378208, -55850704608, -52695031008, -46383597408, -43227923808, -40072250208, -33760816608, -30605143008, -27449469408, -21138035808, -17982362208, -14826688608, -12219309408, -2524538208, -2208988800 ],
-  "diffs" : [ -172392, -85992, 408, 86808, 173208, 259608, 346008, 432408, 518808, 605208, 691608, 778008, 864408, 408, -1800, 0 ]
+  "switches" : [ -62135613408, -59006378208, -55850704608, -52695031008, -46383597408, -43227923808, -40072250208, -33760816608, -30605143008, -27449469408, -21138035808, -17982362208, -14826688608, -12220087008, -12220000608, -12219914208, -12219827808, -12219741408, -12219655008, -12219568608, -12219482208, -12219395808, -12219309408, -2524538208, -2208988800 ],
+  "diffs" : [ -172392, -85992, 408, 86808, 173208, 259608, 346008, 432408, 518808, 605208, 691608, 778008, 864408, 778008, 691608, 605208, 518808, 432408, 346008, 259608, 173208, 86808, 408, -1800, 0 ]
 }, {
   "tz" : "Asia/Kamchatka",
-  "switches" : [ -62135634876, -59006399676, -55850726076, -52695052476, -46383618876, -43227945276, -40072271676, -33760838076, -30605164476, -27449490876, -21138057276, -17982383676, -14826710076, -12219330876, -2208988800 ],
-  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, -5124, 0 ]
+  "switches" : [ -62135634876, -59006399676, -55850726076, -52695052476, -46383618876, -43227945276, -40072271676, -33760838076, -30605164476, -27449490876, -21138057276, -17982383676, -14826710076, -12220108476, -12220022076, -12219935676, -12219849276, -12219762876, -12219676476, -12219590076, -12219503676, -12219417276, -12219330876, -2208988800 ],
+  "diffs" : [ -177924, -91524, -5124, 81276, 167676, 254076, 340476, 426876, 513276, 599676, 686076, 772476, 858876, 772476, 686076, 599676, 513276, 426876, 340476, 254076, 167676, 81276, -5124, 0 ]
 }, {
   "tz" : "Asia/Karachi",
-  "switches" : [ -62135612892, -59006377692, -55850704092, -52695030492, -46383596892, -43227923292, -40072249692, -33760816092, -30605142492, -27449468892, -21138035292, -17982361692, -14826688092, -12219308892, -2208988800 ],
-  "diffs" : [ -174708, -88308, -1908, 84492, 170892, 257292, 343692, 430092, 516492, 602892, 689292, 775692, 862092, -1908, 0 ]
+  "switches" : [ -62135612892, -59006377692, -55850704092, -52695030492, -46383596892, -43227923292, -40072249692, -33760816092, -30605142492, -27449468892, -21138035292, -17982361692, -14826688092, -12220086492, -12220000092, -12219913692, -12219827292, -12219740892, -12219654492, -12219568092, -12219481692, -12219395292, -12219308892, -2208988800 ],
+  "diffs" : [ -174708, -88308, -1908, 84492, 170892, 257292, 343692, 430092, 516492, 602892, 689292, 775692, 862092, 775692, 689292, 602892, 516492, 430092, 343692, 257292, 170892, 84492, -1908, 0 ]
 }, {
   "tz" : "Asia/Kashgar",
-  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12219313820, -2208988800 ],
-  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12220091420, -12220005020, -12219918620, -12219832220, -12219745820, -12219659420, -12219573020, -12219486620, -12219400220, -12219313820, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
 }, {
   "tz" : "Asia/Kathmandu",
-  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12219313276, -2208988800 ],
-  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, -224, 0 ]
+  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12220090876, -12220004476, -12219918076, -12219831676, -12219745276, -12219658876, -12219572476, -12219486076, -12219399676, -12219313276, -2208988800 ],
+  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, 777376, 690976, 604576, 518176, 431776, 345376, 258976, 172576, 86176, -224, 0 ]
 }, {
   "tz" : "Asia/Katmandu",
-  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12219313276, -2208988800 ],
-  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, -224, 0 ]
+  "switches" : [ -62135617276, -59006382076, -55850708476, -52695034876, -46383601276, -43227927676, -40072254076, -33760820476, -30605146876, -27449473276, -21138039676, -17982366076, -14826692476, -12220090876, -12220004476, -12219918076, -12219831676, -12219745276, -12219658876, -12219572476, -12219486076, -12219399676, -12219313276, -2208988800 ],
+  "diffs" : [ -173024, -86624, -224, 86176, 172576, 258976, 345376, 431776, 518176, 604576, 690976, 777376, 863776, 777376, 690976, 604576, 518176, 431776, 345376, 258976, 172576, 86176, -224, 0 ]
 }, {
   "tz" : "Asia/Khandyga",
-  "switches" : [ -62135629333, -59006394133, -55850720533, -52695046933, -46383613333, -43227939733, -40072266133, -33760832533, -30605158933, -27449485333, -21138051733, -17982378133, -14826704533, -12219325333, -2208988800 ],
-  "diffs" : [ -172667, -86267, 133, 86533, 172933, 259333, 345733, 432133, 518533, 604933, 691333, 777733, 864133, 133, 0 ]
+  "switches" : [ -62135629333, -59006394133, -55850720533, -52695046933, -46383613333, -43227939733, -40072266133, -33760832533, -30605158933, -27449485333, -21138051733, -17982378133, -14826704533, -12220102933, -12220016533, -12219930133, -12219843733, -12219757333, -12219670933, -12219584533, -12219498133, -12219411733, -12219325333, -2208988800 ],
+  "diffs" : [ -172667, -86267, 133, 86533, 172933, 259333, 345733, 432133, 518533, 604933, 691333, 777733, 864133, 777733, 691333, 604933, 518533, 432133, 345733, 259333, 172933, 86533, 133, 0 ]
 }, {
   "tz" : "Asia/Kolkata",
-  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12219314008, -3645237208, -3155694800, -2208988800 ],
-  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 1408, 1400, -530, 0 ]
+  "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12220091608, -12220005208, -12219918808, -12219832408, -12219746008, -12219659608, -12219573208, -12219486808, -12219400408, -12219314008, -3645237208, -3155694800, -2208988800 ],
+  "diffs" : [ -171392, -84992, 1408, 87808, 174208, 260608, 347008, 433408, 519808, 606208, 692608, 779008, 865408, 779008, 692608, 606208, 519808, 433408, 347008, 260608, 174208, 87808, 1408, 1400, -530, 0 ]
 }, {
   "tz" : "Asia/Krasnoyarsk",
-  "switches" : [ -62135619086, -59006383886, -55850710286, -52695036686, -46383603086, -43227929486, -40072255886, -33760822286, -30605148686, -27449475086, -21138041486, -17982367886, -14826694286, -12219315086, -2208988800 ],
-  "diffs" : [ -175714, -89314, -2914, 83486, 169886, 256286, 342686, 429086, 515486, 601886, 688286, 774686, 861086, -2914, 0 ]
+  "switches" : [ -62135619086, -59006383886, -55850710286, -52695036686, -46383603086, -43227929486, -40072255886, -33760822286, -30605148686, -27449475086, -21138041486, -17982367886, -14826694286, -12220092686, -12220006286, -12219919886, -12219833486, -12219747086, -12219660686, -12219574286, -12219487886, -12219401486, -12219315086, -2208988800 ],
+  "diffs" : [ -175714, -89314, -2914, 83486, 169886, 256286, 342686, 429086, 515486, 601886, 688286, 774686, 861086, 774686, 688286, 601886, 515486, 429086, 342686, 256286, 169886, 83486, -2914, 0 ]
 }, {
   "tz" : "Asia/Kuala_Lumpur",
-  "switches" : [ -62135621206, -59006386006, -55850712406, -52695038806, -46383605206, -43227931606, -40072258006, -33760824406, -30605150806, -27449477206, -21138043606, -17982370006, -14826696406, -12219317206, -2208988800 ],
-  "diffs" : [ -177194, -90794, -4394, 82006, 168406, 254806, 341206, 427606, 514006, 600406, 686806, 773206, 859606, -4394, 0 ]
+  "switches" : [ -62135621206, -59006386006, -55850712406, -52695038806, -46383605206, -43227931606, -40072258006, -33760824406, -30605150806, -27449477206, -21138043606, -17982370006, -14826696406, -12220094806, -12220008406, -12219922006, -12219835606, -12219749206, -12219662806, -12219576406, -12219490006, -12219403606, -12219317206, -2208988800 ],
+  "diffs" : [ -177194, -90794, -4394, 82006, 168406, 254806, 341206, 427606, 514006, 600406, 686806, 773206, 859606, 773206, 686806, 600406, 514006, 427606, 341206, 254806, 168406, 82006, -4394, 0 ]
 }, {
   "tz" : "Asia/Kuching",
-  "switches" : [ -62135623280, -59006388080, -55850714480, -52695040880, -46383607280, -43227933680, -40072260080, -33760826480, -30605152880, -27449479280, -21138045680, -17982372080, -14826698480, -12219319280, -2208988800 ],
-  "diffs" : [ -175120, -88720, -2320, 84080, 170480, 256880, 343280, 429680, 516080, 602480, 688880, 775280, 861680, -2320, 0 ]
+  "switches" : [ -62135623280, -59006388080, -55850714480, -52695040880, -46383607280, -43227933680, -40072260080, -33760826480, -30605152880, -27449479280, -21138045680, -17982372080, -14826698480, -12220096880, -12220010480, -12219924080, -12219837680, -12219751280, -12219664880, -12219578480, -12219492080, -12219405680, -12219319280, -2208988800 ],
+  "diffs" : [ -175120, -88720, -2320, 84080, 170480, 256880, 343280, 429680, 516080, 602480, 688880, 775280, 861680, 775280, 688880, 602480, 516080, 429680, 343280, 256880, 170480, 84080, -2320, 0 ]
 }, {
   "tz" : "Asia/Kuwait",
-  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12219304012, -2208988800 ],
-  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 412, 0 ]
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ]
 }, {
   "tz" : "Asia/Macao",
-  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12219320050, -2208988800 ],
-  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, -1550, 0 ]
+  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12220097650, -12220011250, -12219924850, -12219838450, -12219752050, -12219665650, -12219579250, -12219492850, -12219406450, -12219320050, -2208988800 ],
+  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, 776050, 689650, 603250, 516850, 430450, 344050, 257650, 171250, 84850, -1550, 0 ]
 }, {
   "tz" : "Asia/Macau",
-  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12219320050, -2208988800 ],
-  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, -1550, 0 ]
+  "switches" : [ -62135624050, -59006388850, -55850715250, -52695041650, -46383608050, -43227934450, -40072260850, -33760827250, -30605153650, -27449480050, -21138046450, -17982372850, -14826699250, -12220097650, -12220011250, -12219924850, -12219838450, -12219752050, -12219665650, -12219579250, -12219492850, -12219406450, -12219320050, -2208988800 ],
+  "diffs" : [ -174350, -87950, -1550, 84850, 171250, 257650, 344050, 430450, 516850, 603250, 689650, 776050, 862450, 776050, 689650, 603250, 516850, 430450, 344050, 257650, 171250, 84850, -1550, 0 ]
 }, {
   "tz" : "Asia/Magadan",
-  "switches" : [ -62135632992, -59006397792, -55850724192, -52695050592, -46383616992, -43227943392, -40072269792, -33760836192, -30605162592, -27449488992, -21138055392, -17982381792, -14826708192, -12219328992, -2208988800 ],
-  "diffs" : [ -176208, -89808, -3408, 82992, 169392, 255792, 342192, 428592, 514992, 601392, 687792, 774192, 860592, -3408, 0 ]
+  "switches" : [ -62135632992, -59006397792, -55850724192, -52695050592, -46383616992, -43227943392, -40072269792, -33760836192, -30605162592, -27449488992, -21138055392, -17982381792, -14826708192, -12220106592, -12220020192, -12219933792, -12219847392, -12219760992, -12219674592, -12219588192, -12219501792, -12219415392, -12219328992, -2208988800 ],
+  "diffs" : [ -176208, -89808, -3408, 82992, 169392, 255792, 342192, 428592, 514992, 601392, 687792, 774192, 860592, 774192, 687792, 601392, 514992, 428592, 342192, 255792, 169392, 82992, -3408, 0 ]
 }, {
   "tz" : "Asia/Makassar",
-  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12219321456, -2208988800 ],
-  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, -144, 0 ]
+  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12220099056, -12220012656, -12219926256, -12219839856, -12219753456, -12219667056, -12219580656, -12219494256, -12219407856, -12219321456, -2208988800 ],
+  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, 777456, 691056, 604656, 518256, 431856, 345456, 259056, 172656, 86256, -144, 0 ]
 }, {
   "tz" : "Asia/Manila",
-  "switches" : [ -62135539440, -59006304240, -55850630640, -52694957040, -46383523440, -43227849840, -40072176240, -33760742640, -30605069040, -27449395440, -21137961840, -17982288240, -14826614640, -12219235440, -3944621040, -2229321840 ],
-  "diffs" : [ -258960, -172560, -86160, 240, 86640, 173040, 259440, 345840, 432240, 518640, 605040, 691440, 777840, -86160, 240, 0 ]
+  "switches" : [ -62135539440, -59006304240, -55850630640, -52694957040, -46383523440, -43227849840, -40072176240, -33760742640, -30605069040, -27449395440, -21137961840, -17982288240, -14826614640, -12220013040, -12219926640, -12219840240, -12219753840, -12219667440, -12219581040, -12219494640, -12219408240, -12219321840, -12219235440, -3944621040, -2229321840 ],
+  "diffs" : [ -258960, -172560, -86160, 240, 86640, 173040, 259440, 345840, 432240, 518640, 605040, 691440, 777840, 691440, 605040, 518640, 432240, 345840, 259440, 173040, 86640, 240, -86160, 240, 0 ]
 }, {
   "tz" : "Asia/Muscat",
-  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12219306072, -2208988800 ],
-  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, -1128, 0 ]
+  "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12220083672, -12219997272, -12219910872, -12219824472, -12219738072, -12219651672, -12219565272, -12219478872, -12219392472, -12219306072, -2208988800 ],
+  "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, 776472, 690072, 603672, 517272, 430872, 344472, 258072, 171672, 85272, -1128, 0 ]
 }, {
   "tz" : "Asia/Nicosia",
-  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12219300808, -2208988800 ],
-  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 808, 0 ]
+  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12220078408, -12219992008, -12219905608, -12219819208, -12219732808, -12219646408, -12219560008, -12219473608, -12219387208, -12219300808, -2208988800 ],
+  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 778408, 692008, 605608, 519208, 432808, 346408, 260008, 173608, 87208, 808, 0 ]
 }, {
   "tz" : "Asia/Novokuznetsk",
-  "switches" : [ -62135617728, -59006382528, -55850708928, -52695035328, -46383601728, -43227928128, -40072254528, -33760820928, -30605147328, -27449473728, -21138040128, -17982366528, -14826692928, -12219313728, -2208988800 ],
-  "diffs" : [ -177072, -90672, -4272, 82128, 168528, 254928, 341328, 427728, 514128, 600528, 686928, 773328, 859728, -4272, 0 ]
+  "switches" : [ -62135617728, -59006382528, -55850708928, -52695035328, -46383601728, -43227928128, -40072254528, -33760820928, -30605147328, -27449473728, -21138040128, -17982366528, -14826692928, -12220091328, -12220004928, -12219918528, -12219832128, -12219745728, -12219659328, -12219572928, -12219486528, -12219400128, -12219313728, -2208988800 ],
+  "diffs" : [ -177072, -90672, -4272, 82128, 168528, 254928, 341328, 427728, 514128, 600528, 686928, 773328, 859728, 773328, 686928, 600528, 514128, 427728, 341328, 254928, 168528, 82128, -4272, 0 ]
 }, {
   "tz" : "Asia/Novosibirsk",
-  "switches" : [ -62135616700, -59006381500, -55850707900, -52695034300, -46383600700, -43227927100, -40072253500, -33760819900, -30605146300, -27449472700, -21138039100, -17982365500, -14826691900, -12219312700, -2208988800 ],
-  "diffs" : [ -178100, -91700, -5300, 81100, 167500, 253900, 340300, 426700, 513100, 599500, 685900, 772300, 858700, -5300, 0 ]
+  "switches" : [ -62135616700, -59006381500, -55850707900, -52695034300, -46383600700, -43227927100, -40072253500, -33760819900, -30605146300, -27449472700, -21138039100, -17982365500, -14826691900, -12220090300, -12220003900, -12219917500, -12219831100, -12219744700, -12219658300, -12219571900, -12219485500, -12219399100, -12219312700, -2208988800 ],
+  "diffs" : [ -178100, -91700, -5300, 81100, 167500, 253900, 340300, 426700, 513100, 599500, 685900, 772300, 858700, 772300, 685900, 599500, 513100, 426700, 340300, 253900, 167500, 81100, -5300, 0 ]
 }, {
   "tz" : "Asia/Omsk",
-  "switches" : [ -62135614410, -59006379210, -55850705610, -52695032010, -46383598410, -43227924810, -40072251210, -33760817610, -30605144010, -27449470410, -21138036810, -17982363210, -14826689610, -12219310410, -2208988800 ],
-  "diffs" : [ -176790, -90390, -3990, 82410, 168810, 255210, 341610, 428010, 514410, 600810, 687210, 773610, 860010, -3990, 0 ]
+  "switches" : [ -62135614410, -59006379210, -55850705610, -52695032010, -46383598410, -43227924810, -40072251210, -33760817610, -30605144010, -27449470410, -21138036810, -17982363210, -14826689610, -12220088010, -12220001610, -12219915210, -12219828810, -12219742410, -12219656010, -12219569610, -12219483210, -12219396810, -12219310410, -2208988800 ],
+  "diffs" : [ -176790, -90390, -3990, 82410, 168810, 255210, 341610, 428010, 514410, 600810, 687210, 773610, 860010, 773610, 687210, 600810, 514410, 428010, 341610, 255210, 168810, 82410, -3990, 0 ]
 }, {
   "tz" : "Asia/Oral",
-  "switches" : [ -62135609124, -59006373924, -55850700324, -52695026724, -46383593124, -43227919524, -40072245924, -33760812324, -30605138724, -27449465124, -21138031524, -17982357924, -14826684324, -12219305124, -2208988800 ],
-  "diffs" : [ -178476, -92076, -5676, 80724, 167124, 253524, 339924, 426324, 512724, 599124, 685524, 771924, 858324, -5676, 0 ]
+  "switches" : [ -62135609124, -59006373924, -55850700324, -52695026724, -46383593124, -43227919524, -40072245924, -33760812324, -30605138724, -27449465124, -21138031524, -17982357924, -14826684324, -12220082724, -12219996324, -12219909924, -12219823524, -12219737124, -12219650724, -12219564324, -12219477924, -12219391524, -12219305124, -2208988800 ],
+  "diffs" : [ -178476, -92076, -5676, 80724, 167124, 253524, 339924, 426324, 512724, 599124, 685524, 771924, 858324, 771924, 685524, 599124, 512724, 426324, 339924, 253524, 167124, 80724, -5676, 0 ]
 }, {
   "tz" : "Asia/Phnom_Penh",
-  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12219316924, -2208988800 ],
-  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, -1076, 0 ]
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ]
 }, {
   "tz" : "Asia/Pontianak",
-  "switches" : [ -62135623040, -59006387840, -55850714240, -52695040640, -46383607040, -43227933440, -40072259840, -33760826240, -30605152640, -27449479040, -21138045440, -17982371840, -14826698240, -12219319040, -2208988800 ],
-  "diffs" : [ -171760, -85360, 1040, 87440, 173840, 260240, 346640, 433040, 519440, 605840, 692240, 778640, 865040, 1040, 0 ]
+  "switches" : [ -62135623040, -59006387840, -55850714240, -52695040640, -46383607040, -43227933440, -40072259840, -33760826240, -30605152640, -27449479040, -21138045440, -17982371840, -14826698240, -12220096640, -12220010240, -12219923840, -12219837440, -12219751040, -12219664640, -12219578240, -12219491840, -12219405440, -12219319040, -2208988800 ],
+  "diffs" : [ -171760, -85360, 1040, 87440, 173840, 260240, 346640, 433040, 519440, 605840, 692240, 778640, 865040, 778640, 692240, 605840, 519440, 433040, 346640, 260240, 173840, 87440, 1040, 0 ]
 }, {
   "tz" : "Asia/Pyongyang",
-  "switches" : [ -62135626980, -59006391780, -55850718180, -52695044580, -46383610980, -43227937380, -40072263780, -33760830180, -30605156580, -27449482980, -21138049380, -17982375780, -14826702180, -12219322980, -2208988800 ],
-  "diffs" : [ -175020, -88620, -2220, 84180, 170580, 256980, 343380, 429780, 516180, 602580, 688980, 775380, 861780, -2220, 0 ]
+  "switches" : [ -62135626980, -59006391780, -55850718180, -52695044580, -46383610980, -43227937380, -40072263780, -33760830180, -30605156580, -27449482980, -21138049380, -17982375780, -14826702180, -12220100580, -12220014180, -12219927780, -12219841380, -12219754980, -12219668580, -12219582180, -12219495780, -12219409380, -12219322980, -2208988800 ],
+  "diffs" : [ -175020, -88620, -2220, 84180, 170580, 256980, 343380, 429780, 516180, 602580, 688980, 775380, 861780, 775380, 688980, 602580, 516180, 429780, 343380, 256980, 170580, 84180, -2220, 0 ]
 }, {
   "tz" : "Asia/Qatar",
-  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12219305168, -2208988800 ],
-  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 1568, 0 ]
+  "switches" : [ -62135609168, -59006373968, -55850700368, -52695026768, -46383593168, -43227919568, -40072245968, -33760812368, -30605138768, -27449465168, -21138031568, -17982357968, -14826684368, -12220082768, -12219996368, -12219909968, -12219823568, -12219737168, -12219650768, -12219564368, -12219477968, -12219391568, -12219305168, -2208988800 ],
+  "diffs" : [ -171232, -84832, 1568, 87968, 174368, 260768, 347168, 433568, 519968, 606368, 692768, 779168, 865568, 779168, 692768, 606368, 519968, 433568, 347168, 260768, 174368, 87968, 1568, 0 ]
 }, {
   "tz" : "Asia/Qostanay",
-  "switches" : [ -62135612068, -59006376868, -55850703268, -52695029668, -46383596068, -43227922468, -40072248868, -33760815268, -30605141668, -27449468068, -21138034468, -17982360868, -14826687268, -12219308068, -2208988800 ],
-  "diffs" : [ -179132, -92732, -6332, 80068, 166468, 252868, 339268, 425668, 512068, 598468, 684868, 771268, 857668, -6332, 0 ]
+  "switches" : [ -62135612068, -59006376868, -55850703268, -52695029668, -46383596068, -43227922468, -40072248868, -33760815268, -30605141668, -27449468068, -21138034468, -17982360868, -14826687268, -12220085668, -12219999268, -12219912868, -12219826468, -12219740068, -12219653668, -12219567268, -12219480868, -12219394468, -12219308068, -2208988800 ],
+  "diffs" : [ -179132, -92732, -6332, 80068, 166468, 252868, 339268, 425668, 512068, 598468, 684868, 771268, 857668, 771268, 684868, 598468, 512068, 425668, 339268, 252868, 166468, 80068, -6332, 0 ]
 }, {
   "tz" : "Asia/Qyzylorda",
-  "switches" : [ -62135612512, -59006377312, -55850703712, -52695030112, -46383596512, -43227922912, -40072249312, -33760815712, -30605142112, -27449468512, -21138034912, -17982361312, -14826687712, -12219308512, -2208988800 ],
-  "diffs" : [ -175088, -88688, -2288, 84112, 170512, 256912, 343312, 429712, 516112, 602512, 688912, 775312, 861712, -2288, 0 ]
+  "switches" : [ -62135612512, -59006377312, -55850703712, -52695030112, -46383596512, -43227922912, -40072249312, -33760815712, -30605142112, -27449468512, -21138034912, -17982361312, -14826687712, -12220086112, -12219999712, -12219913312, -12219826912, -12219740512, -12219654112, -12219567712, -12219481312, -12219394912, -12219308512, -2208988800 ],
+  "diffs" : [ -175088, -88688, -2288, 84112, 170512, 256912, 343312, 429712, 516112, 602512, 688912, 775312, 861712, 775312, 688912, 602512, 516112, 429712, 343312, 256912, 170512, 84112, -2288, 0 ]
 }, {
   "tz" : "Asia/Rangoon",
-  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12219315887, -2208988800 ],
-  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, -313, 0 ]
+  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12220093487, -12220007087, -12219920687, -12219834287, -12219747887, -12219661487, -12219575087, -12219488687, -12219402287, -12219315887, -2208988800 ],
+  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, 777287, 690887, 604487, 518087, 431687, 345287, 258887, 172487, 86087, -313, 0 ]
 }, {
   "tz" : "Asia/Riyadh",
-  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12219304012, -2208988800 ],
-  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 412, 0 ]
+  "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ],
+  "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ]
 }, {
   "tz" : "Asia/Saigon",
-  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12219318400, -2208988800 ],
-  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 400, 0 ]
+  "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12220096000, -12220009600, -12219923200, -12219836800, -12219750400, -12219664000, -12219577600, -12219491200, -12219404800, -12219318400, -2208988800 ],
+  "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 778000, 691600, 605200, 518800, 432400, 346000, 259600, 173200, 86800, 400, 0 ]
 }, {
   "tz" : "Asia/Sakhalin",
-  "switches" : [ -62135631048, -59006395848, -55850722248, -52695048648, -46383615048, -43227941448, -40072267848, -33760834248, -30605160648, -27449487048, -21138053448, -17982379848, -14826706248, -12219327048, -2208988800 ],
-  "diffs" : [ -178152, -91752, -5352, 81048, 167448, 253848, 340248, 426648, 513048, 599448, 685848, 772248, 858648, -5352, 0 ]
+  "switches" : [ -62135631048, -59006395848, -55850722248, -52695048648, -46383615048, -43227941448, -40072267848, -33760834248, -30605160648, -27449487048, -21138053448, -17982379848, -14826706248, -12220104648, -12220018248, -12219931848, -12219845448, -12219759048, -12219672648, -12219586248, -12219499848, -12219413448, -12219327048, -2208988800 ],
+  "diffs" : [ -178152, -91752, -5352, 81048, 167448, 253848, 340248, 426648, 513048, 599448, 685848, 772248, 858648, 772248, 685848, 599448, 513048, 426648, 340248, 253848, 167448, 81048, -5352, 0 ]
 }, {
   "tz" : "Asia/Samarkand",
-  "switches" : [ -62135612873, -59006377673, -55850704073, -52695030473, -46383596873, -43227923273, -40072249673, -33760816073, -30605142473, -27449468873, -21138035273, -17982361673, -14826688073, -12219308873, -2208988800 ],
-  "diffs" : [ -174727, -88327, -1927, 84473, 170873, 257273, 343673, 430073, 516473, 602873, 689273, 775673, 862073, -1927, 0 ]
+  "switches" : [ -62135612873, -59006377673, -55850704073, -52695030473, -46383596873, -43227923273, -40072249673, -33760816073, -30605142473, -27449468873, -21138035273, -17982361673, -14826688073, -12220086473, -12220000073, -12219913673, -12219827273, -12219740873, -12219654473, -12219568073, -12219481673, -12219395273, -12219308873, -2208988800 ],
+  "diffs" : [ -174727, -88327, -1927, 84473, 170873, 257273, 343673, 430073, 516473, 602873, 689273, 775673, 862073, 775673, 689273, 602873, 516473, 430073, 343673, 257273, 170873, 84473, -1927, 0 ]
 }, {
   "tz" : "Asia/Seoul",
-  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12219323272, -2208988800 ],
-  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, -1928, 0 ]
+  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12220100872, -12220014472, -12219928072, -12219841672, -12219755272, -12219668872, -12219582472, -12219496072, -12219409672, -12219323272, -2208988800 ],
+  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, 775672, 689272, 602872, 516472, 430072, 343672, 257272, 170872, 84472, -1928, 0 ]
 }, {
   "tz" : "Asia/Shanghai",
-  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
-  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
 }, {
   "tz" : "Asia/Singapore",
-  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12219317725, -2208988800 ],
-  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, -3875, 0 ]
+  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12220095325, -12220008925, -12219922525, -12219836125, -12219749725, -12219663325, -12219576925, -12219490525, -12219404125, -12219317725, -2208988800 ],
+  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, 773725, 687325, 600925, 514525, 428125, 341725, 255325, 168925, 82525, -3875, 0 ]
 }, {
   "tz" : "Asia/Srednekolymsk",
-  "switches" : [ -62135633692, -59006398492, -55850724892, -52695051292, -46383617692, -43227944092, -40072270492, -33760836892, -30605163292, -27449489692, -21138056092, -17982382492, -14826708892, -12219329692, -2208988800 ],
-  "diffs" : [ -175508, -89108, -2708, 83692, 170092, 256492, 342892, 429292, 515692, 602092, 688492, 774892, 861292, -2708, 0 ]
+  "switches" : [ -62135633692, -59006398492, -55850724892, -52695051292, -46383617692, -43227944092, -40072270492, -33760836892, -30605163292, -27449489692, -21138056092, -17982382492, -14826708892, -12220107292, -12220020892, -12219934492, -12219848092, -12219761692, -12219675292, -12219588892, -12219502492, -12219416092, -12219329692, -2208988800 ],
+  "diffs" : [ -175508, -89108, -2708, 83692, 170092, 256492, 342892, 429292, 515692, 602092, 688492, 774892, 861292, 774892, 688492, 602092, 515692, 429292, 342892, 256492, 170092, 83692, -2708, 0 ]
 }, {
   "tz" : "Asia/Taipei",
-  "switches" : [ -62135625960, -59006390760, -55850717160, -52695043560, -46383609960, -43227936360, -40072262760, -33760829160, -30605155560, -27449481960, -21138048360, -17982374760, -14826701160, -12219321960, -2335248360 ],
-  "diffs" : [ -172440, -86040, 360, 86760, 173160, 259560, 345960, 432360, 518760, 605160, 691560, 777960, 864360, 360, 0 ]
+  "switches" : [ -62135625960, -59006390760, -55850717160, -52695043560, -46383609960, -43227936360, -40072262760, -33760829160, -30605155560, -27449481960, -21138048360, -17982374760, -14826701160, -12220099560, -12220013160, -12219926760, -12219840360, -12219753960, -12219667560, -12219581160, -12219494760, -12219408360, -12219321960, -2335248360 ],
+  "diffs" : [ -172440, -86040, 360, 86760, 173160, 259560, 345960, 432360, 518760, 605160, 691560, 777960, 864360, 777960, 691560, 605160, 518760, 432360, 345960, 259560, 173160, 86760, 360, 0 ]
 }, {
   "tz" : "Asia/Tashkent",
-  "switches" : [ -62135613431, -59006378231, -55850704631, -52695031031, -46383597431, -43227923831, -40072250231, -33760816631, -30605143031, -27449469431, -21138035831, -17982362231, -14826688631, -12219309431, -2208988800 ],
-  "diffs" : [ -174169, -87769, -1369, 85031, 171431, 257831, 344231, 430631, 517031, 603431, 689831, 776231, 862631, -1369, 0 ]
+  "switches" : [ -62135613431, -59006378231, -55850704631, -52695031031, -46383597431, -43227923831, -40072250231, -33760816631, -30605143031, -27449469431, -21138035831, -17982362231, -14826688631, -12220087031, -12220000631, -12219914231, -12219827831, -12219741431, -12219655031, -12219568631, -12219482231, -12219395831, -12219309431, -2208988800 ],
+  "diffs" : [ -174169, -87769, -1369, 85031, 171431, 257831, 344231, 430631, 517031, 603431, 689831, 776231, 862631, 776231, 689831, 603431, 517031, 430631, 344231, 257831, 171431, 85031, -1369, 0 ]
 }, {
   "tz" : "Asia/Tbilisi",
-  "switches" : [ -62135607551, -59006372351, -55850698751, -52695025151, -46383591551, -43227917951, -40072244351, -33760810751, -30605137151, -27449463551, -21138029951, -17982356351, -14826682751, -12219303551, -2208988800 ],
-  "diffs" : [ -176449, -90049, -3649, 82751, 169151, 255551, 341951, 428351, 514751, 601151, 687551, 773951, 860351, -3649, 0 ]
+  "switches" : [ -62135607551, -59006372351, -55850698751, -52695025151, -46383591551, -43227917951, -40072244351, -33760810751, -30605137151, -27449463551, -21138029951, -17982356351, -14826682751, -12220081151, -12219994751, -12219908351, -12219821951, -12219735551, -12219649151, -12219562751, -12219476351, -12219389951, -12219303551, -2208988800 ],
+  "diffs" : [ -176449, -90049, -3649, 82751, 169151, 255551, 341951, 428351, 514751, 601151, 687551, 773951, 860351, 773951, 687551, 601151, 514751, 428351, 341951, 255551, 169151, 82751, -3649, 0 ]
 }, {
   "tz" : "Asia/Tel_Aviv",
-  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12219301254, -2840149254, -2208988800 ],
-  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 1254, 1240, 0 ]
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 778854, 692454, 606054, 519654, 433254, 346854, 260454, 174054, 87654, 1254, 1240, 0 ]
 }, {
   "tz" : "Asia/Thimbu",
-  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12219314316, -2208988800 ],
-  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, -84, 0 ]
+  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12220091916, -12220005516, -12219919116, -12219832716, -12219746316, -12219659916, -12219573516, -12219487116, -12219400716, -12219314316, -2208988800 ],
+  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, 777516, 691116, 604716, 518316, 431916, 345516, 259116, 172716, 86316, -84, 0 ]
 }, {
   "tz" : "Asia/Thimphu",
-  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12219314316, -2208988800 ],
-  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, -84, 0 ]
+  "switches" : [ -62135618316, -59006383116, -55850709516, -52695035916, -46383602316, -43227928716, -40072255116, -33760821516, -30605147916, -27449474316, -21138040716, -17982367116, -14826693516, -12220091916, -12220005516, -12219919116, -12219832716, -12219746316, -12219659916, -12219573516, -12219487116, -12219400716, -12219314316, -2208988800 ],
+  "diffs" : [ -172884, -86484, -84, 86316, 172716, 259116, 345516, 431916, 518316, 604716, 691116, 777516, 863916, 777516, 691116, 604716, 518316, 431916, 345516, 259116, 172716, 86316, -84, 0 ]
 }, {
   "tz" : "Asia/Tokyo",
-  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12219326339, -2587712400 ],
-  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 1139, 0 ]
+  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12220103939, -12220017539, -12219931139, -12219844739, -12219758339, -12219671939, -12219585539, -12219499139, -12219412739, -12219326339, -2587712400 ],
+  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 778739, 692339, 605939, 519539, 433139, 346739, 260339, 173939, 87539, 1139, 0 ]
 }, {
   "tz" : "Asia/Tomsk",
-  "switches" : [ -62135617191, -59006381991, -55850708391, -52695034791, -46383601191, -43227927591, -40072253991, -33760820391, -30605146791, -27449473191, -21138039591, -17982365991, -14826692391, -12219313191, -2208988800 ],
-  "diffs" : [ -177609, -91209, -4809, 81591, 167991, 254391, 340791, 427191, 513591, 599991, 686391, 772791, 859191, -4809, 0 ]
+  "switches" : [ -62135617191, -59006381991, -55850708391, -52695034791, -46383601191, -43227927591, -40072253991, -33760820391, -30605146791, -27449473191, -21138039591, -17982365991, -14826692391, -12220090791, -12220004391, -12219917991, -12219831591, -12219745191, -12219658791, -12219572391, -12219485991, -12219399591, -12219313191, -2208988800 ],
+  "diffs" : [ -177609, -91209, -4809, 81591, 167991, 254391, 340791, 427191, 513591, 599991, 686391, 772791, 859191, 772791, 686391, 599991, 513591, 427191, 340791, 254391, 167991, 81591, -4809, 0 ]
 }, {
   "tz" : "Asia/Ujung_Pandang",
-  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12219321456, -2208988800 ],
-  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, -144, 0 ]
+  "switches" : [ -62135625456, -59006390256, -55850716656, -52695043056, -46383609456, -43227935856, -40072262256, -33760828656, -30605155056, -27449481456, -21138047856, -17982374256, -14826700656, -12220099056, -12220012656, -12219926256, -12219839856, -12219753456, -12219667056, -12219580656, -12219494256, -12219407856, -12219321456, -2208988800 ],
+  "diffs" : [ -172944, -86544, -144, 86256, 172656, 259056, 345456, 431856, 518256, 604656, 691056, 777456, 863856, 777456, 691056, 604656, 518256, 431856, 345456, 259056, 172656, 86256, -144, 0 ]
 }, {
   "tz" : "Asia/Ulaanbaatar",
-  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12219318452, -2208988800 ],
-  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, -3148, 0 ]
+  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12220096052, -12220009652, -12219923252, -12219836852, -12219750452, -12219664052, -12219577652, -12219491252, -12219404852, -12219318452, -2208988800 ],
+  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, 774452, 688052, 601652, 515252, 428852, 342452, 256052, 169652, 83252, -3148, 0 ]
 }, {
   "tz" : "Asia/Ulan_Bator",
-  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12219318452, -2208988800 ],
-  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, -3148, 0 ]
+  "switches" : [ -62135622452, -59006387252, -55850713652, -52695040052, -46383606452, -43227932852, -40072259252, -33760825652, -30605152052, -27449478452, -21138044852, -17982371252, -14826697652, -12220096052, -12220009652, -12219923252, -12219836852, -12219750452, -12219664052, -12219577652, -12219491252, -12219404852, -12219318452, -2208988800 ],
+  "diffs" : [ -175948, -89548, -3148, 83252, 169652, 256052, 342452, 428852, 515252, 601652, 688052, 774452, 860852, 774452, 688052, 601652, 515252, 428852, 342452, 256052, 169652, 83252, -3148, 0 ]
 }, {
   "tz" : "Asia/Urumqi",
-  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12219313820, -2208988800 ],
-  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+  "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12220091420, -12220005020, -12219918620, -12219832220, -12219745820, -12219659420, -12219573020, -12219486620, -12219400220, -12219313820, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
 }, {
   "tz" : "Asia/Ust-Nera",
-  "switches" : [ -62135631174, -59006395974, -55850722374, -52695048774, -46383615174, -43227941574, -40072267974, -33760834374, -30605160774, -27449487174, -21138053574, -17982379974, -14826706374, -12219327174, -2208988800 ],
-  "diffs" : [ -174426, -88026, -1626, 84774, 171174, 257574, 343974, 430374, 516774, 603174, 689574, 775974, 862374, -1626, 0 ]
+  "switches" : [ -62135631174, -59006395974, -55850722374, -52695048774, -46383615174, -43227941574, -40072267974, -33760834374, -30605160774, -27449487174, -21138053574, -17982379974, -14826706374, -12220104774, -12220018374, -12219931974, -12219845574, -12219759174, -12219672774, -12219586374, -12219499974, -12219413574, -12219327174, -2208988800 ],
+  "diffs" : [ -174426, -88026, -1626, 84774, 171174, 257574, 343974, 430374, 516774, 603174, 689574, 775974, 862374, 775974, 689574, 603174, 516774, 430374, 343974, 257574, 171174, 84774, -1626, 0 ]
 }, {
   "tz" : "Asia/Vientiane",
-  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12219316924, -2208988800 ],
-  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, -1076, 0 ]
+  "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ],
+  "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ]
 }, {
   "tz" : "Asia/Vladivostok",
-  "switches" : [ -62135628451, -59006393251, -55850719651, -52695046051, -46383612451, -43227938851, -40072265251, -33760831651, -30605158051, -27449484451, -21138050851, -17982377251, -14826703651, -12219324451, -2208988800 ],
-  "diffs" : [ -177149, -90749, -4349, 82051, 168451, 254851, 341251, 427651, 514051, 600451, 686851, 773251, 859651, -4349, 0 ]
+  "switches" : [ -62135628451, -59006393251, -55850719651, -52695046051, -46383612451, -43227938851, -40072265251, -33760831651, -30605158051, -27449484451, -21138050851, -17982377251, -14826703651, -12220102051, -12220015651, -12219929251, -12219842851, -12219756451, -12219670051, -12219583651, -12219497251, -12219410851, -12219324451, -2208988800 ],
+  "diffs" : [ -177149, -90749, -4349, 82051, 168451, 254851, 341251, 427651, 514051, 600451, 686851, 773251, 859651, 773251, 686851, 600451, 514051, 427651, 341251, 254851, 168451, 82051, -4349, 0 ]
 }, {
   "tz" : "Asia/Yakutsk",
-  "switches" : [ -62135627938, -59006392738, -55850719138, -52695045538, -46383611938, -43227938338, -40072264738, -33760831138, -30605157538, -27449483938, -21138050338, -17982376738, -14826703138, -12219323938, -2208988800 ],
-  "diffs" : [ -174062, -87662, -1262, 85138, 171538, 257938, 344338, 430738, 517138, 603538, 689938, 776338, 862738, -1262, 0 ]
+  "switches" : [ -62135627938, -59006392738, -55850719138, -52695045538, -46383611938, -43227938338, -40072264738, -33760831138, -30605157538, -27449483938, -21138050338, -17982376738, -14826703138, -12220101538, -12220015138, -12219928738, -12219842338, -12219755938, -12219669538, -12219583138, -12219496738, -12219410338, -12219323938, -2208988800 ],
+  "diffs" : [ -174062, -87662, -1262, 85138, 171538, 257938, 344338, 430738, 517138, 603538, 689938, 776338, 862738, 776338, 689938, 603538, 517138, 430738, 344338, 257938, 171538, 85138, -1262, 0 ]
 }, {
   "tz" : "Asia/Yangon",
-  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12219315887, -2208988800 ],
-  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, -313, 0 ]
+  "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12220093487, -12220007087, -12219920687, -12219834287, -12219747887, -12219661487, -12219575087, -12219488687, -12219402287, -12219315887, -2208988800 ],
+  "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, 777287, 690887, 604487, 518087, 431687, 345287, 258887, 172487, 86087, -313, 0 ]
 }, {
   "tz" : "Asia/Yekaterinburg",
-  "switches" : [ -62135611353, -59006376153, -55850702553, -52695028953, -46383595353, -43227921753, -40072248153, -33760814553, -30605140953, -27449467353, -21138033753, -17982360153, -14826686553, -12219307353, -2208988800 ],
-  "diffs" : [ -176247, -89847, -3447, 82953, 169353, 255753, 342153, 428553, 514953, 601353, 687753, 774153, 860553, -3447, 0 ]
+  "switches" : [ -62135611353, -59006376153, -55850702553, -52695028953, -46383595353, -43227921753, -40072248153, -33760814553, -30605140953, -27449467353, -21138033753, -17982360153, -14826686553, -12220084953, -12219998553, -12219912153, -12219825753, -12219739353, -12219652953, -12219566553, -12219480153, -12219393753, -12219307353, -2208988800 ],
+  "diffs" : [ -176247, -89847, -3447, 82953, 169353, 255753, 342153, 428553, 514953, 601353, 687753, 774153, 860553, 774153, 687753, 601353, 514953, 428553, 342153, 255753, 169353, 82953, -3447, 0 ]
 }, {
   "tz" : "Asia/Yerevan",
-  "switches" : [ -62135607480, -59006372280, -55850698680, -52695025080, -46383591480, -43227917880, -40072244280, -33760810680, -30605137080, -27449463480, -21138029880, -17982356280, -14826682680, -12219303480, -2208988800 ],
-  "diffs" : [ -176520, -90120, -3720, 82680, 169080, 255480, 341880, 428280, 514680, 601080, 687480, 773880, 860280, -3720, 0 ]
+  "switches" : [ -62135607480, -59006372280, -55850698680, -52695025080, -46383591480, -43227917880, -40072244280, -33760810680, -30605137080, -27449463480, -21138029880, -17982356280, -14826682680, -12220081080, -12219994680, -12219908280, -12219821880, -12219735480, -12219649080, -12219562680, -12219476280, -12219389880, -12219303480, -2208988800 ],
+  "diffs" : [ -176520, -90120, -3720, 82680, 169080, 255480, 341880, 428280, 514680, 601080, 687480, 773880, 860280, 773880, 687480, 601080, 514680, 428280, 341880, 255480, 169080, 82680, -3720, 0 ]
 }, {
   "tz" : "Atlantic/Azores",
-  "switches" : [ -62135590640, -59006355440, -55850681840, -52695008240, -46383574640, -43227901040, -40072227440, -33760793840, -30605120240, -27449446640, -21138013040, -17982339440, -14826665840, -12219286640, -2713904240, -2208988800 ],
-  "diffs" : [ -175360, -88960, -2560, 83840, 170240, 256640, 343040, 429440, 515840, 602240, 688640, 775040, 861440, -2560, -3272, 0 ]
+  "switches" : [ -62135590640, -59006355440, -55850681840, -52695008240, -46383574640, -43227901040, -40072227440, -33760793840, -30605120240, -27449446640, -21138013040, -17982339440, -14826665840, -12220064240, -12219977840, -12219891440, -12219805040, -12219718640, -12219632240, -12219545840, -12219459440, -12219373040, -12219286640, -2713904240, -2208988800 ],
+  "diffs" : [ -175360, -88960, -2560, 83840, 170240, 256640, 343040, 429440, 515840, 602240, 688640, 775040, 861440, 775040, 688640, 602240, 515840, 429440, 343040, 256640, 170240, 83840, -2560, -3272, 0 ]
 }, {
   "tz" : "Atlantic/Bermuda",
-  "switches" : [ -62135581242, -59006346042, -55850672442, -52694998842, -46383565242, -43227891642, -40072218042, -33760784442, -30605110842, -27449437242, -21138003642, -17982330042, -14826656442, -12219277242, -2208988800 ],
-  "diffs" : [ -173958, -87558, -1158, 85242, 171642, 258042, 344442, 430842, 517242, 603642, 690042, 776442, 862842, -1158, 0 ]
+  "switches" : [ -62135581242, -59006346042, -55850672442, -52694998842, -46383565242, -43227891642, -40072218042, -33760784442, -30605110842, -27449437242, -21138003642, -17982330042, -14826656442, -12220054842, -12219968442, -12219882042, -12219795642, -12219709242, -12219622842, -12219536442, -12219450042, -12219363642, -12219277242, -2208988800 ],
+  "diffs" : [ -173958, -87558, -1158, 85242, 171642, 258042, 344442, 430842, 517242, 603642, 690042, 776442, 862842, 776442, 690042, 603642, 517242, 430842, 344442, 258042, 171642, 85242, -1158, 0 ]
 }, {
   "tz" : "Atlantic/Canary",
-  "switches" : [ -62135593104, -59006357904, -55850684304, -52695010704, -46383577104, -43227903504, -40072229904, -33760796304, -30605122704, -27449449104, -21138015504, -17982341904, -14826668304, -12219289104, -2208988800 ],
-  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, -3696, 0 ]
+  "switches" : [ -62135593104, -59006357904, -55850684304, -52695010704, -46383577104, -43227903504, -40072229904, -33760796304, -30605122704, -27449449104, -21138015504, -17982341904, -14826668304, -12220066704, -12219980304, -12219893904, -12219807504, -12219721104, -12219634704, -12219548304, -12219461904, -12219375504, -12219289104, -2208988800 ],
+  "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, 773904, 687504, 601104, 514704, 428304, 341904, 255504, 169104, 82704, -3696, 0 ]
 }, {
   "tz" : "Atlantic/Cape_Verde",
-  "switches" : [ -62135591156, -59006355956, -55850682356, -52695008756, -46383575156, -43227901556, -40072227956, -33760794356, -30605120756, -27449447156, -21138013556, -17982339956, -14826666356, -12219287156, -2208988800 ],
-  "diffs" : [ -174844, -88444, -2044, 84356, 170756, 257156, 343556, 429956, 516356, 602756, 689156, 775556, 861956, -2044, 0 ]
+  "switches" : [ -62135591156, -59006355956, -55850682356, -52695008756, -46383575156, -43227901556, -40072227956, -33760794356, -30605120756, -27449447156, -21138013556, -17982339956, -14826666356, -12220064756, -12219978356, -12219891956, -12219805556, -12219719156, -12219632756, -12219546356, -12219459956, -12219373556, -12219287156, -2208988800 ],
+  "diffs" : [ -174844, -88444, -2044, 84356, 170756, 257156, 343556, 429956, 516356, 602756, 689156, 775556, 861956, 775556, 689156, 602756, 516356, 429956, 343556, 257156, 170756, 84356, -2044, 0 ]
 }, {
   "tz" : "Atlantic/Faeroe",
-  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12219291176, -2208988800 ],
-  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, -1624, 0 ]
+  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12220068776, -12219982376, -12219895976, -12219809576, -12219723176, -12219636776, -12219550376, -12219463976, -12219377576, -12219291176, -2208988800 ],
+  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, 775976, 689576, 603176, 516776, 430376, 343976, 257576, 171176, 84776, -1624, 0 ]
 }, {
   "tz" : "Atlantic/Faroe",
-  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12219291176, -2208988800 ],
-  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, -1624, 0 ]
+  "switches" : [ -62135595176, -59006359976, -55850686376, -52695012776, -46383579176, -43227905576, -40072231976, -33760798376, -30605124776, -27449451176, -21138017576, -17982343976, -14826670376, -12220068776, -12219982376, -12219895976, -12219809576, -12219723176, -12219636776, -12219550376, -12219463976, -12219377576, -12219291176, -2208988800 ],
+  "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, 775976, 689576, 603176, 516776, 430376, 343976, 257576, 171176, 84776, -1624, 0 ]
 }, {
   "tz" : "Atlantic/Jan_Mayen",
-  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12219295380, -2366757780 ],
-  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, -1020, 0 ]
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ]
 }, {
   "tz" : "Atlantic/Madeira",
-  "switches" : [ -62135592744, -59006357544, -55850683944, -52695010344, -46383576744, -43227903144, -40072229544, -33760795944, -30605122344, -27449448744, -21138015144, -17982341544, -14826667944, -12219288744, -2208988800 ],
-  "diffs" : [ -176856, -90456, -4056, 82344, 168744, 255144, 341544, 427944, 514344, 600744, 687144, 773544, 859944, -4056, 0 ]
+  "switches" : [ -62135592744, -59006357544, -55850683944, -52695010344, -46383576744, -43227903144, -40072229544, -33760795944, -30605122344, -27449448744, -21138015144, -17982341544, -14826667944, -12220066344, -12219979944, -12219893544, -12219807144, -12219720744, -12219634344, -12219547944, -12219461544, -12219375144, -12219288744, -2208988800 ],
+  "diffs" : [ -176856, -90456, -4056, 82344, 168744, 255144, 341544, 427944, 514344, 600744, 687144, 773544, 859944, 773544, 687144, 600744, 514344, 427944, 341544, 255144, 168744, 82344, -4056, 0 ]
 }, {
   "tz" : "Atlantic/Reykjavik",
-  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12219287520, -2208988800 ],
-  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, -5280, 0 ]
+  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12220065120, -12219978720, -12219892320, -12219805920, -12219719520, -12219633120, -12219546720, -12219460320, -12219373920, -12219287520, -2208988800 ],
+  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, 772320, 685920, 599520, 513120, 426720, 340320, 253920, 167520, 81120, -5280, 0 ]
 }, {
   "tz" : "Atlantic/South_Georgia",
-  "switches" : [ -62135588032, -59006352832, -55850679232, -52695005632, -46383572032, -43227898432, -40072224832, -33760791232, -30605117632, -27449444032, -21138010432, -17982336832, -14826663232, -12219284032, -2524512832 ],
-  "diffs" : [ -174368, -87968, -1568, 84832, 171232, 257632, 344032, 430432, 516832, 603232, 689632, 776032, 862432, -1568, 0 ]
+  "switches" : [ -62135588032, -59006352832, -55850679232, -52695005632, -46383572032, -43227898432, -40072224832, -33760791232, -30605117632, -27449444032, -21138010432, -17982336832, -14826663232, -12220061632, -12219975232, -12219888832, -12219802432, -12219716032, -12219629632, -12219543232, -12219456832, -12219370432, -12219284032, -2524512832 ],
+  "diffs" : [ -174368, -87968, -1568, 84832, 171232, 257632, 344032, 430432, 516832, 603232, 689632, 776032, 862432, 776032, 689632, 603232, 516832, 430432, 344032, 257632, 171232, 84832, -1568, 0 ]
 }, {
   "tz" : "Atlantic/St_Helena",
-  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12219291832, -2208988800 ],
-  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, -968, 0 ]
+  "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
+  "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ]
 }, {
   "tz" : "Atlantic/Stanley",
-  "switches" : [ -62135582916, -59006347716, -55850674116, -52695000516, -46383566916, -43227893316, -40072219716, -33760786116, -30605112516, -27449438916, -21138005316, -17982331716, -14826658116, -12219278916, -2208988800 ],
-  "diffs" : [ -175884, -89484, -3084, 83316, 169716, 256116, 342516, 428916, 515316, 601716, 688116, 774516, 860916, -3084, 0 ]
+  "switches" : [ -62135582916, -59006347716, -55850674116, -52695000516, -46383566916, -43227893316, -40072219716, -33760786116, -30605112516, -27449438916, -21138005316, -17982331716, -14826658116, -12220056516, -12219970116, -12219883716, -12219797316, -12219710916, -12219624516, -12219538116, -12219451716, -12219365316, -12219278916, -2208988800 ],
+  "diffs" : [ -175884, -89484, -3084, 83316, 169716, 256116, 342516, 428916, 515316, 601716, 688116, 774516, 860916, 774516, 688116, 601716, 515316, 428916, 342516, 256116, 169716, 83316, -3084, 0 ]
 }, {
   "tz" : "Australia/ACT",
-  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
-  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
 }, {
   "tz" : "Australia/Adelaide",
-  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12219326060, -2364110060, -2230189200 ],
-  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, -940, -1800, 0 ]
+  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12220103660, -12220017260, -12219930860, -12219844460, -12219758060, -12219671660, -12219585260, -12219498860, -12219412460, -12219326060, -2364110060, -2230189200 ],
+  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, 776660, 690260, 603860, 517460, 431060, 344660, 258260, 171860, 85460, -940, -1800, 0 ]
 }, {
   "tz" : "Australia/Brisbane",
-  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12219329528, -2366791928 ],
-  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 728, 0 ]
+  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12220107128, -12220020728, -12219934328, -12219847928, -12219761528, -12219675128, -12219588728, -12219502328, -12219415928, -12219329528, -2366791928 ],
+  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 778328, 691928, 605528, 519128, 432728, 346328, 259928, 173528, 87128, 728, 0 ]
 }, {
   "tz" : "Australia/Broken_Hill",
-  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12219326748, -2364110748, -2314951200, -2230189200 ],
-  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, -252, 1800, -1800, 0 ]
+  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12220104348, -12220017948, -12219931548, -12219845148, -12219758748, -12219672348, -12219585948, -12219499548, -12219413148, -12219326748, -2364110748, -2314951200, -2230189200 ],
+  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, 777348, 690948, 604548, 518148, 431748, 345348, 258948, 172548, 86148, -252, 1800, -1800, 0 ]
 }, {
   "tz" : "Australia/Canberra",
-  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
-  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
 }, {
   "tz" : "Australia/Currie",
-  "switches" : [ -62135631328, -59006396128, -55850722528, -52695048928, -46383615328, -43227941728, -40072268128, -33760834528, -30605160928, -27449487328, -21138053728, -17982380128, -14826706528, -12219327328, -2345794528 ],
-  "diffs" : [ -174272, -87872, -1472, 84928, 171328, 257728, 344128, 430528, 516928, 603328, 689728, 776128, 862528, -1472, 0 ]
+  "switches" : [ -62135631328, -59006396128, -55850722528, -52695048928, -46383615328, -43227941728, -40072268128, -33760834528, -30605160928, -27449487328, -21138053728, -17982380128, -14826706528, -12220104928, -12220018528, -12219932128, -12219845728, -12219759328, -12219672928, -12219586528, -12219500128, -12219413728, -12219327328, -2345794528 ],
+  "diffs" : [ -174272, -87872, -1472, 84928, 171328, 257728, 344128, 430528, 516928, 603328, 689728, 776128, 862528, 776128, 689728, 603328, 516928, 430528, 344128, 257728, 171328, 84928, -1472, 0 ]
 }, {
   "tz" : "Australia/Darwin",
-  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12219324200, -2364108200, -2230189200 ],
-  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, -2800, -1800, 0 ]
+  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12220101800, -12220015400, -12219929000, -12219842600, -12219756200, -12219669800, -12219583400, -12219497000, -12219410600, -12219324200, -2364108200, -2230189200 ],
+  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, 774800, 688400, 602000, 515600, 429200, 342800, 256400, 170000, 83600, -2800, -1800, 0 ]
 }, {
   "tz" : "Australia/Eucla",
-  "switches" : [ -62135627728, -59006392528, -55850718928, -52695045328, -46383611728, -43227938128, -40072264528, -33760830928, -30605157328, -27449483728, -21138050128, -17982376528, -14826702928, -12219323728, -2337928528 ],
-  "diffs" : [ -173372, -86972, -572, 85828, 172228, 258628, 345028, 431428, 517828, 604228, 690628, 777028, 863428, -572, 0 ]
+  "switches" : [ -62135627728, -59006392528, -55850718928, -52695045328, -46383611728, -43227938128, -40072264528, -33760830928, -30605157328, -27449483728, -21138050128, -17982376528, -14826702928, -12220101328, -12220014928, -12219928528, -12219842128, -12219755728, -12219669328, -12219582928, -12219496528, -12219410128, -12219323728, -2337928528 ],
+  "diffs" : [ -173372, -86972, -572, 85828, 172228, 258628, 345028, 431428, 517828, 604228, 690628, 777028, 863428, 777028, 690628, 604228, 517828, 431428, 345028, 258628, 172228, 85828, -572, 0 ]
 }, {
   "tz" : "Australia/Hobart",
-  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12219328156, -2345795356 ],
-  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, -644, 0 ]
+  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12220105756, -12220019356, -12219932956, -12219846556, -12219760156, -12219673756, -12219587356, -12219500956, -12219414556, -12219328156, -2345795356 ],
+  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, 776956, 690556, 604156, 517756, 431356, 344956, 258556, 172156, 85756, -644, 0 ]
 }, {
   "tz" : "Australia/LHI",
-  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12219330980, -2364114980, -2208988800 ],
-  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 380, -1800, 0 ]
+  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12220108580, -12220022180, -12219935780, -12219849380, -12219762980, -12219676580, -12219590180, -12219503780, -12219417380, -12219330980, -2364114980, -2208988800 ],
+  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 777980, 691580, 605180, 518780, 432380, 345980, 259580, 173180, 86780, 380, -1800, 0 ]
 }, {
   "tz" : "Australia/Lindeman",
-  "switches" : [ -62135632556, -59006397356, -55850723756, -52695050156, -46383616556, -43227942956, -40072269356, -33760835756, -30605162156, -27449488556, -21138054956, -17982381356, -14826707756, -12219328556, -2366790956 ],
-  "diffs" : [ -173044, -86644, -244, 86156, 172556, 258956, 345356, 431756, 518156, 604556, 690956, 777356, 863756, -244, 0 ]
+  "switches" : [ -62135632556, -59006397356, -55850723756, -52695050156, -46383616556, -43227942956, -40072269356, -33760835756, -30605162156, -27449488556, -21138054956, -17982381356, -14826707756, -12220106156, -12220019756, -12219933356, -12219846956, -12219760556, -12219674156, -12219587756, -12219501356, -12219414956, -12219328556, -2366790956 ],
+  "diffs" : [ -173044, -86644, -244, 86156, 172556, 258956, 345356, 431756, 518156, 604556, 690956, 777356, 863756, 777356, 690956, 604556, 518156, 431756, 345356, 258956, 172556, 86156, -244, 0 ]
 }, {
   "tz" : "Australia/Lord_Howe",
-  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12219330980, -2364114980, -2208988800 ],
-  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 380, -1800, 0 ]
+  "switches" : [ -62135634980, -59006399780, -55850726180, -52695052580, -46383618980, -43227945380, -40072271780, -33760838180, -30605164580, -27449490980, -21138057380, -17982383780, -14826710180, -12220108580, -12220022180, -12219935780, -12219849380, -12219762980, -12219676580, -12219590180, -12219503780, -12219417380, -12219330980, -2364114980, -2208988800 ],
+  "diffs" : [ -172420, -86020, 380, 86780, 173180, 259580, 345980, 432380, 518780, 605180, 691580, 777980, 864380, 777980, 691580, 605180, 518780, 432380, 345980, 259580, 173180, 86780, 380, -1800, 0 ]
 }, {
   "tz" : "Australia/Melbourne",
-  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12219327592, -2364111592 ],
-  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, -1208, 0 ]
+  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12220105192, -12220018792, -12219932392, -12219845992, -12219759592, -12219673192, -12219586792, -12219500392, -12219413992, -12219327592, -2364111592 ],
+  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, 776392, 689992, 603592, 517192, 430792, 344392, 257992, 171592, 85192, -1208, 0 ]
 }, {
   "tz" : "Australia/NSW",
-  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
-  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
 }, {
   "tz" : "Australia/North",
-  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12219324200, -2364108200, -2230189200 ],
-  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, -2800, -1800, 0 ]
+  "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12220101800, -12220015400, -12219929000, -12219842600, -12219756200, -12219669800, -12219583400, -12219497000, -12219410600, -12219324200, -2364108200, -2230189200 ],
+  "diffs" : [ -175600, -89200, -2800, 83600, 170000, 256400, 342800, 429200, 515600, 602000, 688400, 774800, 861200, 774800, 688400, 602000, 515600, 429200, 342800, 256400, 170000, 83600, -2800, -1800, 0 ]
 }, {
   "tz" : "Australia/Perth",
-  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12219320604, -2337925404 ],
-  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, -996, 0 ]
+  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12220098204, -12220011804, -12219925404, -12219839004, -12219752604, -12219666204, -12219579804, -12219493404, -12219407004, -12219320604, -2337925404 ],
+  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, 776604, 690204, 603804, 517404, 431004, 344604, 258204, 171804, 85404, -996, 0 ]
 }, {
   "tz" : "Australia/Queensland",
-  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12219329528, -2366791928 ],
-  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 728, 0 ]
+  "switches" : [ -62135633528, -59006398328, -55850724728, -52695051128, -46383617528, -43227943928, -40072270328, -33760836728, -30605163128, -27449489528, -21138055928, -17982382328, -14826708728, -12220107128, -12220020728, -12219934328, -12219847928, -12219761528, -12219675128, -12219588728, -12219502328, -12219415928, -12219329528, -2366791928 ],
+  "diffs" : [ -172072, -85672, 728, 87128, 173528, 259928, 346328, 432728, 519128, 605528, 691928, 778328, 864728, 778328, 691928, 605528, 519128, 432728, 346328, 259928, 173528, 87128, 728, 0 ]
 }, {
   "tz" : "Australia/South",
-  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12219326060, -2364110060, -2230189200 ],
-  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, -940, -1800, 0 ]
+  "switches" : [ -62135630060, -59006394860, -55850721260, -52695047660, -46383614060, -43227940460, -40072266860, -33760833260, -30605159660, -27449486060, -21138052460, -17982378860, -14826705260, -12220103660, -12220017260, -12219930860, -12219844460, -12219758060, -12219671660, -12219585260, -12219498860, -12219412460, -12219326060, -2364110060, -2230189200 ],
+  "diffs" : [ -173740, -87340, -940, 85460, 171860, 258260, 344660, 431060, 517460, 603860, 690260, 776660, 863060, 776660, 690260, 603860, 517460, 431060, 344660, 258260, 171860, 85460, -940, -1800, 0 ]
 }, {
   "tz" : "Australia/Sydney",
-  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12219329092, -2364113092 ],
-  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 292, 0 ]
+  "switches" : [ -62135633092, -59006397892, -55850724292, -52695050692, -46383617092, -43227943492, -40072269892, -33760836292, -30605162692, -27449489092, -21138055492, -17982381892, -14826708292, -12220106692, -12220020292, -12219933892, -12219847492, -12219761092, -12219674692, -12219588292, -12219501892, -12219415492, -12219329092, -2364113092 ],
+  "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ]
 }, {
   "tz" : "Australia/Tasmania",
-  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12219328156, -2345795356 ],
-  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, -644, 0 ]
+  "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12220105756, -12220019356, -12219932956, -12219846556, -12219760156, -12219673756, -12219587356, -12219500956, -12219414556, -12219328156, -2345795356 ],
+  "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, 776956, 690556, 604156, 517756, 431356, 344956, 258556, 172156, 85756, -644, 0 ]
 }, {
   "tz" : "Australia/Victoria",
-  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12219327592, -2364111592 ],
-  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, -1208, 0 ]
+  "switches" : [ -62135631592, -59006396392, -55850722792, -52695049192, -46383615592, -43227941992, -40072268392, -33760834792, -30605161192, -27449487592, -21138053992, -17982380392, -14826706792, -12220105192, -12220018792, -12219932392, -12219845992, -12219759592, -12219673192, -12219586792, -12219500392, -12219413992, -12219327592, -2364111592 ],
+  "diffs" : [ -174008, -87608, -1208, 85192, 171592, 257992, 344392, 430792, 517192, 603592, 689992, 776392, 862792, 776392, 689992, 603592, 517192, 430792, 344392, 257992, 171592, 85192, -1208, 0 ]
 }, {
   "tz" : "Australia/West",
-  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12219320604, -2337925404 ],
-  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, -996, 0 ]
+  "switches" : [ -62135624604, -59006389404, -55850715804, -52695042204, -46383608604, -43227935004, -40072261404, -33760827804, -30605154204, -27449480604, -21138047004, -17982373404, -14826699804, -12220098204, -12220011804, -12219925404, -12219839004, -12219752604, -12219666204, -12219579804, -12219493404, -12219407004, -12219320604, -2337925404 ],
+  "diffs" : [ -173796, -87396, -996, 85404, 171804, 258204, 344604, 431004, 517404, 603804, 690204, 776604, 863004, 776604, 690204, 603804, 517404, 431004, 344604, 258204, 171804, 85404, -996, 0 ]
 }, {
   "tz" : "Australia/Yancowinna",
-  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12219326748, -2364110748, -2314951200, -2230189200 ],
-  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, -252, 1800, -1800, 0 ]
+  "switches" : [ -62135630748, -59006395548, -55850721948, -52695048348, -46383614748, -43227941148, -40072267548, -33760833948, -30605160348, -27449486748, -21138053148, -17982379548, -14826705948, -12220104348, -12220017948, -12219931548, -12219845148, -12219758748, -12219672348, -12219585948, -12219499548, -12219413148, -12219326748, -2364110748, -2314951200, -2230189200 ],
+  "diffs" : [ -173052, -86652, -252, 86148, 172548, 258948, 345348, 431748, 518148, 604548, 690948, 777348, 863748, 777348, 690948, 604548, 518148, 431748, 345348, 258948, 172548, 86148, -252, 1800, -1800, 0 ]
 }, {
   "tz" : "Brazil/Acre",
-  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12219276528, -2208988800 ],
-  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 1728, 0 ]
+  "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ],
+  "diffs" : [ -171072, -84672, 1728, 88128, 174528, 260928, 347328, 433728, 520128, 606528, 692928, 779328, 865728, 779328, 692928, 606528, 520128, 433728, 347328, 260928, 174528, 88128, 1728, 0 ]
 }, {
   "tz" : "Brazil/DeNoronha",
-  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12219285020, -2208988800 ],
-  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+  "switches" : [ -62135589020, -59006353820, -55850680220, -52695006620, -46383573020, -43227899420, -40072225820, -33760792220, -30605118620, -27449445020, -21138011420, -17982337820, -14826664220, -12220062620, -12219976220, -12219889820, -12219803420, -12219717020, -12219630620, -12219544220, -12219457820, -12219371420, -12219285020, -2208988800 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
 }, {
   "tz" : "Brazil/East",
-  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12219281612, -2208988800 ],
-  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, -388, 0 ]
+  "switches" : [ -62135585612, -59006350412, -55850676812, -52695003212, -46383569612, -43227896012, -40072222412, -33760788812, -30605115212, -27449441612, -21138008012, -17982334412, -14826660812, -12220059212, -12219972812, -12219886412, -12219800012, -12219713612, -12219627212, -12219540812, -12219454412, -12219368012, -12219281612, -2208988800 ],
+  "diffs" : [ -173188, -86788, -388, 86012, 172412, 258812, 345212, 431612, 518012, 604412, 690812, 777212, 863612, 777212, 690812, 604412, 518012, 431612, 345212, 258812, 172412, 86012, -388, 0 ]
 }, {
   "tz" : "Brazil/West",
-  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12219278396, -2208988800 ],
-  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, -4, 0 ]
+  "switches" : [ -62135582396, -59006347196, -55850673596, -52694999996, -46383566396, -43227892796, -40072219196, -33760785596, -30605111996, -27449438396, -21138004796, -17982331196, -14826657596, -12220055996, -12219969596, -12219883196, -12219796796, -12219710396, -12219623996, -12219537596, -12219451196, -12219364796, -12219278396, -2208988800 ],
+  "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, 777596, 691196, 604796, 518396, 431996, 345596, 259196, 172796, 86396, -4, 0 ]
 }, {
   "tz" : "CET",
-  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12219296400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12220074000, -12219987600, -12219901200, -12219814800, -12219728400, -12219642000, -12219555600, -12219469200, -12219382800, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "CST6CDT",
-  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Canada/Atlantic",
-  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12219277536, -2208988800 ],
-  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, -864, 0 ]
+  "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12220055136, -12219968736, -12219882336, -12219795936, -12219709536, -12219623136, -12219536736, -12219450336, -12219363936, -12219277536, -2208988800 ],
+  "diffs" : [ -173664, -87264, -864, 85536, 171936, 258336, 344736, 431136, 517536, 603936, 690336, 776736, 863136, 776736, 690336, 603936, 517536, 431136, 344736, 258336, 171936, 85536, -864, 0 ]
 }, {
   "tz" : "Canada/Central",
-  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12219269484, -2602258284 ],
-  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, -1716, 0 ]
+  "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12220047084, -12219960684, -12219874284, -12219787884, -12219701484, -12219615084, -12219528684, -12219442284, -12219355884, -12219269484, -2602258284 ],
+  "diffs" : [ -174516, -88116, -1716, 84684, 171084, 257484, 343884, 430284, 516684, 603084, 689484, 775884, 862284, 775884, 689484, 603084, 516684, 430284, 343884, 257484, 171084, 84684, -1716, 0 ]
 }, {
   "tz" : "Canada/Eastern",
-  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12219273748, -2366736148 ],
-  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, -1052, 0 ]
+  "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ],
+  "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ]
 }, {
   "tz" : "Canada/Mountain",
-  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12219265568, -2208988800 ],
-  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, -2032, 0 ]
+  "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12220043168, -12219956768, -12219870368, -12219783968, -12219697568, -12219611168, -12219524768, -12219438368, -12219351968, -12219265568, -2208988800 ],
+  "diffs" : [ -174832, -88432, -2032, 84368, 170768, 257168, 343568, 429968, 516368, 602768, 689168, 775568, 861968, 775568, 689168, 602768, 516368, 429968, 343568, 257168, 170768, 84368, -2032, 0 ]
 }, {
   "tz" : "Canada/Newfoundland",
-  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12219280148, -2208988800 ],
-  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, -52, 0 ]
+  "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12220057748, -12219971348, -12219884948, -12219798548, -12219712148, -12219625748, -12219539348, -12219452948, -12219366548, -12219280148, -2208988800 ],
+  "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ]
 }, {
   "tz" : "Canada/Pacific",
-  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12219263252, -2713880852 ],
-  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, -748, 0 ]
+  "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12220040852, -12219954452, -12219868052, -12219781652, -12219695252, -12219608852, -12219522452, -12219436052, -12219349652, -12219263252, -2713880852 ],
+  "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, 776852, 690452, 604052, 517652, 431252, 344852, 258452, 172052, 85652, -748, 0 ]
 }, {
   "tz" : "Canada/Saskatchewan",
-  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12219267684, -2208988800 ],
-  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, -3516, 0 ]
+  "switches" : [ -62135571684, -59006336484, -55850662884, -52694989284, -46383555684, -43227882084, -40072208484, -33760774884, -30605101284, -27449427684, -21137994084, -17982320484, -14826646884, -12220045284, -12219958884, -12219872484, -12219786084, -12219699684, -12219613284, -12219526884, -12219440484, -12219354084, -12219267684, -2208988800 ],
+  "diffs" : [ -176316, -89916, -3516, 82884, 169284, 255684, 342084, 428484, 514884, 601284, 687684, 774084, 860484, 774084, 687684, 601284, 514884, 428484, 342084, 255684, 169284, 82884, -3516, 0 ]
 }, {
   "tz" : "Canada/Yukon",
-  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12219260388, -2208988800 ],
-  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, -3612, 0 ]
+  "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12220037988, -12219951588, -12219865188, -12219778788, -12219692388, -12219605988, -12219519588, -12219433188, -12219346788, -12219260388, -2208988800 ],
+  "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, 773988, 687588, 601188, 514788, 428388, 341988, 255588, 169188, 82788, -3612, 0 ]
 }, {
   "tz" : "Chile/Continental",
-  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12219275834, -2208988800 ],
-  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, -2566, 0 ]
+  "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12220053434, -12219967034, -12219880634, -12219794234, -12219707834, -12219621434, -12219535034, -12219448634, -12219362234, -12219275834, -2208988800 ],
+  "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, 775034, 688634, 602234, 515834, 429434, 343034, 256634, 170234, 83834, -2566, 0 ]
 }, {
   "tz" : "Chile/EasterIsland",
-  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12219266552, -2208988800 ],
-  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, -4648, 0 ]
+  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12220044152, -12219957752, -12219871352, -12219784952, -12219698552, -12219612152, -12219525752, -12219439352, -12219352952, -12219266552, -2208988800 ],
+  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, 772952, 686552, 600152, 513752, 427352, 340952, 254552, 168152, 81752, -4648, 0 ]
 }, {
   "tz" : "Cuba",
-  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12219273032, -2524501832, -2208988800 ],
-  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, -1768, -1776, 0 ]
+  "switches" : [ -62135577032, -59006341832, -55850668232, -52694994632, -46383561032, -43227887432, -40072213832, -33760780232, -30605106632, -27449433032, -21137999432, -17982325832, -14826652232, -12220050632, -12219964232, -12219877832, -12219791432, -12219705032, -12219618632, -12219532232, -12219445832, -12219359432, -12219273032, -2524501832, -2208988800 ],
+  "diffs" : [ -174568, -88168, -1768, 84632, 171032, 257432, 343832, 430232, 516632, 603032, 689432, 775832, 862232, 775832, 689432, 603032, 516632, 430232, 343832, 257432, 171032, 84632, -1768, -1776, 0 ]
 }, {
   "tz" : "EET",
-  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12219300000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12220077600, -12219991200, -12219904800, -12219818400, -12219732000, -12219645600, -12219559200, -12219472800, -12219386400, -12219300000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "EST5EDT",
-  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Egypt",
-  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12219300309, -2208988800 ],
-  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 309, 0 ]
+  "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12220077909, -12219991509, -12219905109, -12219818709, -12219732309, -12219645909, -12219559509, -12219473109, -12219386709, -12219300309, -2208988800 ],
+  "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ]
 }, {
   "tz" : "Eire",
-  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12219291300, -2821649700, -2208988800 ],
-  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, -1500, -1521, 0 ]
+  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12220068900, -12219982500, -12219896100, -12219809700, -12219723300, -12219636900, -12219550500, -12219464100, -12219377700, -12219291300, -2821649700, -2208988800 ],
+  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, 776100, 689700, 603300, 516900, 430500, 344100, 257700, 171300, 84900, -1500, -1521, 0 ]
 }, {
   "tz" : "Etc/GMT",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+0",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+1",
-  "switches" : [ -62135593200, -59006358000, -55850684400, -52695010800, -46383577200, -43227903600, -40072230000, -33760796400, -30605122800, -27449449200, -21138015600, -17982342000, -14826668400, -12219289200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135593200, -59006358000, -55850684400, -52695010800, -46383577200, -43227903600, -40072230000, -33760796400, -30605122800, -27449449200, -21138015600, -17982342000, -14826668400, -12220066800, -12219980400, -12219894000, -12219807600, -12219721200, -12219634800, -12219548400, -12219462000, -12219375600, -12219289200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+10",
-  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12219256800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12220034400, -12219948000, -12219861600, -12219775200, -12219688800, -12219602400, -12219516000, -12219429600, -12219343200, -12219256800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+11",
-  "switches" : [ -62135557200, -59006322000, -55850648400, -52694974800, -46383541200, -43227867600, -40072194000, -33760760400, -30605086800, -27449413200, -21137979600, -17982306000, -14826632400, -12219253200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135557200, -59006322000, -55850648400, -52694974800, -46383541200, -43227867600, -40072194000, -33760760400, -30605086800, -27449413200, -21137979600, -17982306000, -14826632400, -12220030800, -12219944400, -12219858000, -12219771600, -12219685200, -12219598800, -12219512400, -12219426000, -12219339600, -12219253200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+12",
-  "switches" : [ -62135553600, -59006318400, -55850644800, -52694971200, -46383537600, -43227864000, -40072190400, -33760756800, -30605083200, -27449409600, -21137976000, -17982302400, -14826628800, -12219249600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135553600, -59006318400, -55850644800, -52694971200, -46383537600, -43227864000, -40072190400, -33760756800, -30605083200, -27449409600, -21137976000, -17982302400, -14826628800, -12220027200, -12219940800, -12219854400, -12219768000, -12219681600, -12219595200, -12219508800, -12219422400, -12219336000, -12219249600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+2",
-  "switches" : [ -62135589600, -59006354400, -55850680800, -52695007200, -46383573600, -43227900000, -40072226400, -33760792800, -30605119200, -27449445600, -21138012000, -17982338400, -14826664800, -12219285600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135589600, -59006354400, -55850680800, -52695007200, -46383573600, -43227900000, -40072226400, -33760792800, -30605119200, -27449445600, -21138012000, -17982338400, -14826664800, -12220063200, -12219976800, -12219890400, -12219804000, -12219717600, -12219631200, -12219544800, -12219458400, -12219372000, -12219285600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+3",
-  "switches" : [ -62135586000, -59006350800, -55850677200, -52695003600, -46383570000, -43227896400, -40072222800, -33760789200, -30605115600, -27449442000, -21138008400, -17982334800, -14826661200, -12219282000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135586000, -59006350800, -55850677200, -52695003600, -46383570000, -43227896400, -40072222800, -33760789200, -30605115600, -27449442000, -21138008400, -17982334800, -14826661200, -12220059600, -12219973200, -12219886800, -12219800400, -12219714000, -12219627600, -12219541200, -12219454800, -12219368400, -12219282000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+4",
-  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12219278400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12220056000, -12219969600, -12219883200, -12219796800, -12219710400, -12219624000, -12219537600, -12219451200, -12219364800, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+5",
-  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+6",
-  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+7",
-  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+8",
-  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT+9",
-  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12219260400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12220038000, -12219951600, -12219865200, -12219778800, -12219692400, -12219606000, -12219519600, -12219433200, -12219346800, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-0",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-1",
-  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12219296400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12220074000, -12219987600, -12219901200, -12219814800, -12219728400, -12219642000, -12219555600, -12219469200, -12219382800, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-10",
-  "switches" : [ -62135632800, -59006397600, -55850724000, -52695050400, -46383616800, -43227943200, -40072269600, -33760836000, -30605162400, -27449488800, -21138055200, -17982381600, -14826708000, -12219328800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135632800, -59006397600, -55850724000, -52695050400, -46383616800, -43227943200, -40072269600, -33760836000, -30605162400, -27449488800, -21138055200, -17982381600, -14826708000, -12220106400, -12220020000, -12219933600, -12219847200, -12219760800, -12219674400, -12219588000, -12219501600, -12219415200, -12219328800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-11",
-  "switches" : [ -62135636400, -59006401200, -55850727600, -52695054000, -46383620400, -43227946800, -40072273200, -33760839600, -30605166000, -27449492400, -21138058800, -17982385200, -14826711600, -12219332400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135636400, -59006401200, -55850727600, -52695054000, -46383620400, -43227946800, -40072273200, -33760839600, -30605166000, -27449492400, -21138058800, -17982385200, -14826711600, -12220110000, -12220023600, -12219937200, -12219850800, -12219764400, -12219678000, -12219591600, -12219505200, -12219418800, -12219332400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-12",
-  "switches" : [ -62135640000, -59006404800, -55850731200, -52695057600, -46383624000, -43227950400, -40072276800, -33760843200, -30605169600, -27449496000, -21138062400, -17982388800, -14826715200, -12219336000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135640000, -59006404800, -55850731200, -52695057600, -46383624000, -43227950400, -40072276800, -33760843200, -30605169600, -27449496000, -21138062400, -17982388800, -14826715200, -12220113600, -12220027200, -12219940800, -12219854400, -12219768000, -12219681600, -12219595200, -12219508800, -12219422400, -12219336000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-13",
-  "switches" : [ -62135643600, -59006408400, -55850734800, -52695061200, -46383627600, -43227954000, -40072280400, -33760846800, -30605173200, -27449499600, -21138066000, -17982392400, -14826718800, -12219339600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135643600, -59006408400, -55850734800, -52695061200, -46383627600, -43227954000, -40072280400, -33760846800, -30605173200, -27449499600, -21138066000, -17982392400, -14826718800, -12220117200, -12220030800, -12219944400, -12219858000, -12219771600, -12219685200, -12219598800, -12219512400, -12219426000, -12219339600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-14",
-  "switches" : [ -62135647200, -59006412000, -55850738400, -52695064800, -46383631200, -43227957600, -40072284000, -33760850400, -30605176800, -27449503200, -21138069600, -17982396000, -14826722400, -12219343200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135647200, -59006412000, -55850738400, -52695064800, -46383631200, -43227957600, -40072284000, -33760850400, -30605176800, -27449503200, -21138069600, -17982396000, -14826722400, -12220120800, -12220034400, -12219948000, -12219861600, -12219775200, -12219688800, -12219602400, -12219516000, -12219429600, -12219343200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-2",
-  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12219300000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135604000, -59006368800, -55850695200, -52695021600, -46383588000, -43227914400, -40072240800, -33760807200, -30605133600, -27449460000, -21138026400, -17982352800, -14826679200, -12220077600, -12219991200, -12219904800, -12219818400, -12219732000, -12219645600, -12219559200, -12219472800, -12219386400, -12219300000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-3",
-  "switches" : [ -62135607600, -59006372400, -55850698800, -52695025200, -46383591600, -43227918000, -40072244400, -33760810800, -30605137200, -27449463600, -21138030000, -17982356400, -14826682800, -12219303600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135607600, -59006372400, -55850698800, -52695025200, -46383591600, -43227918000, -40072244400, -33760810800, -30605137200, -27449463600, -21138030000, -17982356400, -14826682800, -12220081200, -12219994800, -12219908400, -12219822000, -12219735600, -12219649200, -12219562800, -12219476400, -12219390000, -12219303600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-4",
-  "switches" : [ -62135611200, -59006376000, -55850702400, -52695028800, -46383595200, -43227921600, -40072248000, -33760814400, -30605140800, -27449467200, -21138033600, -17982360000, -14826686400, -12219307200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135611200, -59006376000, -55850702400, -52695028800, -46383595200, -43227921600, -40072248000, -33760814400, -30605140800, -27449467200, -21138033600, -17982360000, -14826686400, -12220084800, -12219998400, -12219912000, -12219825600, -12219739200, -12219652800, -12219566400, -12219480000, -12219393600, -12219307200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-5",
-  "switches" : [ -62135614800, -59006379600, -55850706000, -52695032400, -46383598800, -43227925200, -40072251600, -33760818000, -30605144400, -27449470800, -21138037200, -17982363600, -14826690000, -12219310800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135614800, -59006379600, -55850706000, -52695032400, -46383598800, -43227925200, -40072251600, -33760818000, -30605144400, -27449470800, -21138037200, -17982363600, -14826690000, -12220088400, -12220002000, -12219915600, -12219829200, -12219742800, -12219656400, -12219570000, -12219483600, -12219397200, -12219310800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-6",
-  "switches" : [ -62135618400, -59006383200, -55850709600, -52695036000, -46383602400, -43227928800, -40072255200, -33760821600, -30605148000, -27449474400, -21138040800, -17982367200, -14826693600, -12219314400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135618400, -59006383200, -55850709600, -52695036000, -46383602400, -43227928800, -40072255200, -33760821600, -30605148000, -27449474400, -21138040800, -17982367200, -14826693600, -12220092000, -12220005600, -12219919200, -12219832800, -12219746400, -12219660000, -12219573600, -12219487200, -12219400800, -12219314400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-7",
-  "switches" : [ -62135622000, -59006386800, -55850713200, -52695039600, -46383606000, -43227932400, -40072258800, -33760825200, -30605151600, -27449478000, -21138044400, -17982370800, -14826697200, -12219318000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135622000, -59006386800, -55850713200, -52695039600, -46383606000, -43227932400, -40072258800, -33760825200, -30605151600, -27449478000, -21138044400, -17982370800, -14826697200, -12220095600, -12220009200, -12219922800, -12219836400, -12219750000, -12219663600, -12219577200, -12219490800, -12219404400, -12219318000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-8",
-  "switches" : [ -62135625600, -59006390400, -55850716800, -52695043200, -46383609600, -43227936000, -40072262400, -33760828800, -30605155200, -27449481600, -21138048000, -17982374400, -14826700800, -12219321600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135625600, -59006390400, -55850716800, -52695043200, -46383609600, -43227936000, -40072262400, -33760828800, -30605155200, -27449481600, -21138048000, -17982374400, -14826700800, -12220099200, -12220012800, -12219926400, -12219840000, -12219753600, -12219667200, -12219580800, -12219494400, -12219408000, -12219321600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT-9",
-  "switches" : [ -62135629200, -59006394000, -55850720400, -52695046800, -46383613200, -43227939600, -40072266000, -33760832400, -30605158800, -27449485200, -21138051600, -17982378000, -14826704400, -12219325200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135629200, -59006394000, -55850720400, -52695046800, -46383613200, -43227939600, -40072266000, -33760832400, -30605158800, -27449485200, -21138051600, -17982378000, -14826704400, -12220102800, -12220016400, -12219930000, -12219843600, -12219757200, -12219670800, -12219584400, -12219498000, -12219411600, -12219325200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/GMT0",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/Greenwich",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/UCT",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/UTC",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/Universal",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Etc/Zulu",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Europe/Amsterdam",
-  "switches" : [ -62135597972, -59006362772, -55850689172, -52695015572, -46383581972, -43227908372, -40072234772, -33760801172, -30605127572, -27449453972, -21138020372, -17982346772, -14826673172, -12219293972, -2208988800 ],
-  "diffs" : [ -175228, -88828, -2428, 83972, 170372, 256772, 343172, 429572, 515972, 602372, 688772, 775172, 861572, -2428, 0 ]
+  "switches" : [ -62135597972, -59006362772, -55850689172, -52695015572, -46383581972, -43227908372, -40072234772, -33760801172, -30605127572, -27449453972, -21138020372, -17982346772, -14826673172, -12220071572, -12219985172, -12219898772, -12219812372, -12219725972, -12219639572, -12219553172, -12219466772, -12219380372, -12219293972, -2208988800 ],
+  "diffs" : [ -175228, -88828, -2428, 83972, 170372, 256772, 343172, 429572, 515972, 602372, 688772, 775172, 861572, 775172, 688772, 602372, 515972, 429572, 343172, 256772, 170372, 83972, -2428, 0 ]
 }, {
   "tz" : "Europe/Andorra",
-  "switches" : [ -62135597164, -59006361964, -55850688364, -52695014764, -46383581164, -43227907564, -40072233964, -33760800364, -30605126764, -27449453164, -21138019564, -17982345964, -14826672364, -12219293164, -2208988800 ],
-  "diffs" : [ -176036, -89636, -3236, 83164, 169564, 255964, 342364, 428764, 515164, 601564, 687964, 774364, 860764, -3236, 0 ]
+  "switches" : [ -62135597164, -59006361964, -55850688364, -52695014764, -46383581164, -43227907564, -40072233964, -33760800364, -30605126764, -27449453164, -21138019564, -17982345964, -14826672364, -12220070764, -12219984364, -12219897964, -12219811564, -12219725164, -12219638764, -12219552364, -12219465964, -12219379564, -12219293164, -2208988800 ],
+  "diffs" : [ -176036, -89636, -3236, 83164, 169564, 255964, 342364, 428764, 515164, 601564, 687964, 774364, 860764, 774364, 687964, 601564, 515164, 428764, 342364, 255964, 169564, 83164, -3236, 0 ]
 }, {
   "tz" : "Europe/Astrakhan",
-  "switches" : [ -62135608332, -59006373132, -55850699532, -52695025932, -46383592332, -43227918732, -40072245132, -33760811532, -30605137932, -27449464332, -21138030732, -17982357132, -14826683532, -12219304332, -2208988800 ],
-  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, -2868, 0 ]
+  "switches" : [ -62135608332, -59006373132, -55850699532, -52695025932, -46383592332, -43227918732, -40072245132, -33760811532, -30605137932, -27449464332, -21138030732, -17982357132, -14826683532, -12220081932, -12219995532, -12219909132, -12219822732, -12219736332, -12219649932, -12219563532, -12219477132, -12219390732, -12219304332, -2208988800 ],
+  "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, 774732, 688332, 601932, 515532, 429132, 342732, 256332, 169932, 83532, -2868, 0 ]
 }, {
   "tz" : "Europe/Athens",
-  "switches" : [ -62135602492, -59006367292, -55850693692, -52695020092, -46383586492, -43227912892, -40072239292, -33760805692, -30605132092, -27449458492, -21138024892, -17982351292, -14826677692, -12219298492, -2208988800 ],
-  "diffs" : [ -174308, -87908, -1508, 84892, 171292, 257692, 344092, 430492, 516892, 603292, 689692, 776092, 862492, -1508, 0 ]
+  "switches" : [ -62135602492, -59006367292, -55850693692, -52695020092, -46383586492, -43227912892, -40072239292, -33760805692, -30605132092, -27449458492, -21138024892, -17982351292, -14826677692, -12220076092, -12219989692, -12219903292, -12219816892, -12219730492, -12219644092, -12219557692, -12219471292, -12219384892, -12219298492, -2208988800 ],
+  "diffs" : [ -174308, -87908, -1508, 84892, 171292, 257692, 344092, 430492, 516892, 603292, 689692, 776092, 862492, 776092, 689692, 603292, 516892, 430492, 344092, 257692, 171292, 84892, -1508, 0 ]
 }, {
   "tz" : "Europe/Belfast",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "Europe/Belgrade",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Europe/Berlin",
-  "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12219296008, -2422054408 ],
-  "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, -392, 0 ]
+  "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ],
+  "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ]
 }, {
   "tz" : "Europe/Bratislava",
-  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12219296264, -2469401864 ],
-  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, -136, 0 ]
+  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12220073864, -12219987464, -12219901064, -12219814664, -12219728264, -12219641864, -12219555464, -12219469064, -12219382664, -12219296264, -2469401864 ],
+  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, 777464, 691064, 604664, 518264, 431864, 345464, 259064, 172664, 86264, -136, 0 ]
 }, {
   "tz" : "Europe/Brussels",
-  "switches" : [ -62135597850, -59006362650, -55850689050, -52695015450, -46383581850, -43227908250, -40072234650, -33760801050, -30605127450, -27449453850, -21138020250, -17982346650, -14826673050, -12219293850, -2450995200, -2208988800 ],
-  "diffs" : [ -175350, -88950, -2550, 83850, 170250, 256650, 343050, 429450, 515850, 602250, 688650, 775050, 861450, -2550, -3600, 0 ]
+  "switches" : [ -62135597850, -59006362650, -55850689050, -52695015450, -46383581850, -43227908250, -40072234650, -33760801050, -30605127450, -27449453850, -21138020250, -17982346650, -14826673050, -12220071450, -12219985050, -12219898650, -12219812250, -12219725850, -12219639450, -12219553050, -12219466650, -12219380250, -12219293850, -2450995200, -2208988800 ],
+  "diffs" : [ -175350, -88950, -2550, 83850, 170250, 256650, 343050, 429450, 515850, 602250, 688650, 775050, 861450, 775050, 688650, 602250, 515850, 429450, 343050, 256650, 170250, 83850, -2550, -3600, 0 ]
 }, {
   "tz" : "Europe/Bucharest",
-  "switches" : [ -62135603064, -59006367864, -55850694264, -52695020664, -46383587064, -43227913464, -40072239864, -33760806264, -30605132664, -27449459064, -21138025464, -17982351864, -14826678264, -12219299064, -2208988800 ],
-  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, -936, 0 ]
+  "switches" : [ -62135603064, -59006367864, -55850694264, -52695020664, -46383587064, -43227913464, -40072239864, -33760806264, -30605132664, -27449459064, -21138025464, -17982351864, -14826678264, -12220076664, -12219990264, -12219903864, -12219817464, -12219731064, -12219644664, -12219558264, -12219471864, -12219385464, -12219299064, -2208988800 ],
+  "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, 776664, 690264, 603864, 517464, 431064, 344664, 258264, 171864, 85464, -936, 0 ]
 }, {
   "tz" : "Europe/Budapest",
-  "switches" : [ -62135601380, -59006366180, -55850692580, -52695018980, -46383585380, -43227911780, -40072238180, -33760804580, -30605130980, -27449457380, -21138023780, -17982350180, -14826676580, -12219297380, -2500938980 ],
-  "diffs" : [ -171820, -85420, 980, 87380, 173780, 260180, 346580, 432980, 519380, 605780, 692180, 778580, 864980, 980, 0 ]
+  "switches" : [ -62135601380, -59006366180, -55850692580, -52695018980, -46383585380, -43227911780, -40072238180, -33760804580, -30605130980, -27449457380, -21138023780, -17982350180, -14826676580, -12220074980, -12219988580, -12219902180, -12219815780, -12219729380, -12219642980, -12219556580, -12219470180, -12219383780, -12219297380, -2500938980 ],
+  "diffs" : [ -171820, -85420, 980, 87380, 173780, 260180, 346580, 432980, 519380, 605780, 692180, 778580, 864980, 778580, 692180, 605780, 519380, 432980, 346580, 260180, 173780, 87380, 980, 0 ]
 }, {
   "tz" : "Europe/Busingen",
-  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12219294848, -3675198848, -2385246586 ],
-  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, -1552, -1814, 0 ]
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12220072448, -12219986048, -12219899648, -12219813248, -12219726848, -12219640448, -12219554048, -12219467648, -12219381248, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, 776048, 689648, 603248, 516848, 430448, 344048, 257648, 171248, 84848, -1552, -1814, 0 ]
 }, {
   "tz" : "Europe/Chisinau",
-  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12219299720, -2840147720, -2208988800 ],
-  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, -280, -300, 0 ]
+  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12220077320, -12219990920, -12219904520, -12219818120, -12219731720, -12219645320, -12219558920, -12219472520, -12219386120, -12219299720, -2840147720, -2208988800 ],
+  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, 777320, 690920, 604520, 518120, 431720, 345320, 258920, 172520, 86120, -280, -300, 0 ]
 }, {
   "tz" : "Europe/Copenhagen",
-  "switches" : [ -62135599820, -59006364620, -55850691020, -52695017420, -46383583820, -43227910220, -40072236620, -33760803020, -30605129420, -27449455820, -21138022220, -17982348620, -14826675020, -12219295820, -2398294220 ],
-  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, -580, 0 ]
+  "switches" : [ -62135599820, -59006364620, -55850691020, -52695017420, -46383583820, -43227910220, -40072236620, -33760803020, -30605129420, -27449455820, -21138022220, -17982348620, -14826675020, -12220073420, -12219987020, -12219900620, -12219814220, -12219727820, -12219641420, -12219555020, -12219468620, -12219382220, -12219295820, -2398294220 ],
+  "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ]
 }, {
   "tz" : "Europe/Dublin",
-  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12219291300, -2821649700, -2208988800 ],
-  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, -1500, -1521, 0 ]
+  "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12220068900, -12219982500, -12219896100, -12219809700, -12219723300, -12219636900, -12219550500, -12219464100, -12219377700, -12219291300, -2821649700, -2208988800 ],
+  "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, 776100, 689700, 603300, 516900, 430500, 344100, 257700, 171300, 84900, -1500, -1521, 0 ]
 }, {
   "tz" : "Europe/Gibraltar",
-  "switches" : [ -62135595516, -59006360316, -55850686716, -52695013116, -46383579516, -43227905916, -40072232316, -33760798716, -30605125116, -27449451516, -21138017916, -17982344316, -14826670716, -12219291516, -2821649916, -2208988800 ],
-  "diffs" : [ -177684, -91284, -4884, 81516, 167916, 254316, 340716, 427116, 513516, 599916, 686316, 772716, 859116, -4884, -3600, 0 ]
+  "switches" : [ -62135595516, -59006360316, -55850686716, -52695013116, -46383579516, -43227905916, -40072232316, -33760798716, -30605125116, -27449451516, -21138017916, -17982344316, -14826670716, -12220069116, -12219982716, -12219896316, -12219809916, -12219723516, -12219637116, -12219550716, -12219464316, -12219377916, -12219291516, -2821649916, -2208988800 ],
+  "diffs" : [ -177684, -91284, -4884, 81516, 167916, 254316, 340716, 427116, 513516, 599916, 686316, 772716, 859116, 772716, 686316, 599916, 513516, 427116, 340716, 254316, 167916, 81516, -4884, -3600, 0 ]
 }, {
   "tz" : "Europe/Guernsey",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "Europe/Helsinki",
-  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12219298789, -2208988800 ],
-  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, -1211, 0 ]
+  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12220076389, -12219989989, -12219903589, -12219817189, -12219730789, -12219644389, -12219557989, -12219471589, -12219385189, -12219298789, -2208988800 ],
+  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, 776389, 689989, 603589, 517189, 430789, 344389, 257989, 171589, 85189, -1211, 0 ]
 }, {
   "tz" : "Europe/Isle_of_Man",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "Europe/Istanbul",
-  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12219299752, -2840147752, -2208988800 ],
-  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, -3848, -3784, 0 ]
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12220077352, -12219990952, -12219904552, -12219818152, -12219731752, -12219645352, -12219558952, -12219472552, -12219386152, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, 773752, 687352, 600952, 514552, 428152, 341752, 255352, 168952, 82552, -3848, -3784, 0 ]
 }, {
   "tz" : "Europe/Jersey",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "Europe/Kaliningrad",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2422056120, -2208988800 ],
-  "diffs" : [ -175080, -88680, -2280, 84120, 170520, 256920, 343320, 429720, 516120, 602520, 688920, 775320, 861720, -2280, -3600, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2422056120, -2208988800 ],
+  "diffs" : [ -175080, -88680, -2280, 84120, 170520, 256920, 343320, 429720, 516120, 602520, 688920, 775320, 861720, 775320, 688920, 602520, 516120, 429720, 343320, 256920, 170520, 84120, -2280, -3600, 0 ]
 }, {
   "tz" : "Europe/Kiev",
-  "switches" : [ -62135604124, -59006368924, -55850695324, -52695021724, -46383588124, -43227914524, -40072240924, -33760807324, -30605133724, -27449460124, -21138026524, -17982352924, -14826679324, -12219300124, -2208988800 ],
-  "diffs" : [ -172676, -86276, 124, 86524, 172924, 259324, 345724, 432124, 518524, 604924, 691324, 777724, 864124, 124, 0 ]
+  "switches" : [ -62135604124, -59006368924, -55850695324, -52695021724, -46383588124, -43227914524, -40072240924, -33760807324, -30605133724, -27449460124, -21138026524, -17982352924, -14826679324, -12220077724, -12219991324, -12219904924, -12219818524, -12219732124, -12219645724, -12219559324, -12219472924, -12219386524, -12219300124, -2208988800 ],
+  "diffs" : [ -172676, -86276, 124, 86524, 172924, 259324, 345724, 432124, 518524, 604924, 691324, 777724, 864124, 777724, 691324, 604924, 518524, 432124, 345724, 259324, 172924, 86524, 124, 0 ]
 }, {
   "tz" : "Europe/Kirov",
-  "switches" : [ -62135608728, -59006373528, -55850699928, -52695026328, -46383592728, -43227919128, -40072245528, -33760811928, -30605138328, -27449464728, -21138031128, -17982357528, -14826683928, -12219304728, -2208988800 ],
-  "diffs" : [ -171672, -85272, 1128, 87528, 173928, 260328, 346728, 433128, 519528, 605928, 692328, 778728, 865128, 1128, 0 ]
+  "switches" : [ -62135608728, -59006373528, -55850699928, -52695026328, -46383592728, -43227919128, -40072245528, -33760811928, -30605138328, -27449464728, -21138031128, -17982357528, -14826683928, -12220082328, -12219995928, -12219909528, -12219823128, -12219736728, -12219650328, -12219563928, -12219477528, -12219391128, -12219304728, -2208988800 ],
+  "diffs" : [ -171672, -85272, 1128, 87528, 173928, 260328, 346728, 433128, 519528, 605928, 692328, 778728, 865128, 778728, 692328, 605928, 519528, 433128, 346728, 260328, 173928, 87528, 1128, 0 ]
 }, {
   "tz" : "Europe/Lisbon",
-  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12219290595, -2208988800 ],
-  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, -2205, 0 ]
+  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12220068195, -12219981795, -12219895395, -12219808995, -12219722595, -12219636195, -12219549795, -12219463395, -12219376995, -12219290595, -2208988800 ],
+  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, 775395, 688995, 602595, 516195, 429795, 343395, 256995, 170595, 84195, -2205, 0 ]
 }, {
   "tz" : "Europe/Ljubljana",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Europe/London",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "Europe/Luxembourg",
-  "switches" : [ -62135598276, -59006363076, -55850689476, -52695015876, -46383582276, -43227908676, -40072235076, -33760801476, -30605127876, -27449454276, -21138020676, -17982347076, -14826673476, -12219294276, -2208988800 ],
-  "diffs" : [ -174924, -88524, -2124, 84276, 170676, 257076, 343476, 429876, 516276, 602676, 689076, 775476, 861876, -2124, 0 ]
+  "switches" : [ -62135598276, -59006363076, -55850689476, -52695015876, -46383582276, -43227908676, -40072235076, -33760801476, -30605127876, -27449454276, -21138020676, -17982347076, -14826673476, -12220071876, -12219985476, -12219899076, -12219812676, -12219726276, -12219639876, -12219553476, -12219467076, -12219380676, -12219294276, -2208988800 ],
+  "diffs" : [ -174924, -88524, -2124, 84276, 170676, 257076, 343476, 429876, 516276, 602676, 689076, 775476, 861876, 775476, 689076, 602676, 516276, 429876, 343476, 257076, 170676, 84276, -2124, 0 ]
 }, {
   "tz" : "Europe/Madrid",
-  "switches" : [ -62135595916, -59006360716, -55850687116, -52695013516, -46383579916, -43227906316, -40072232716, -33760799116, -30605125516, -27449451916, -21138018316, -17982344716, -14826671116, -12219291916, -2208988800 ],
-  "diffs" : [ -177284, -90884, -4484, 81916, 168316, 254716, 341116, 427516, 513916, 600316, 686716, 773116, 859516, -4484, 0 ]
+  "switches" : [ -62135595916, -59006360716, -55850687116, -52695013516, -46383579916, -43227906316, -40072232716, -33760799116, -30605125516, -27449451916, -21138018316, -17982344716, -14826671116, -12220069516, -12219983116, -12219896716, -12219810316, -12219723916, -12219637516, -12219551116, -12219464716, -12219378316, -12219291916, -2208988800 ],
+  "diffs" : [ -177284, -90884, -4484, 81916, 168316, 254716, 341116, 427516, 513916, 600316, 686716, 773116, 859516, 773116, 686716, 600316, 513916, 427516, 341116, 254716, 168316, 81916, -4484, 0 ]
 }, {
   "tz" : "Europe/Malta",
-  "switches" : [ -62135600284, -59006365084, -55850691484, -52695017884, -46383584284, -43227910684, -40072237084, -33760803484, -30605129884, -27449456284, -21138022684, -17982349084, -14826675484, -12219296284, -2403478684 ],
-  "diffs" : [ -172916, -86516, -116, 86284, 172684, 259084, 345484, 431884, 518284, 604684, 691084, 777484, 863884, -116, 0 ]
+  "switches" : [ -62135600284, -59006365084, -55850691484, -52695017884, -46383584284, -43227910684, -40072237084, -33760803484, -30605129884, -27449456284, -21138022684, -17982349084, -14826675484, -12220073884, -12219987484, -12219901084, -12219814684, -12219728284, -12219641884, -12219555484, -12219469084, -12219382684, -12219296284, -2403478684 ],
+  "diffs" : [ -172916, -86516, -116, 86284, 172684, 259084, 345484, 431884, 518284, 604684, 691084, 777484, 863884, 777484, 691084, 604684, 518284, 431884, 345484, 259084, 172684, 86284, -116, 0 ]
 }, {
   "tz" : "Europe/Mariehamn",
-  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12219298789, -2208988800 ],
-  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, -1211, 0 ]
+  "switches" : [ -62135602789, -59006367589, -55850693989, -52695020389, -46383586789, -43227913189, -40072239589, -33760805989, -30605132389, -27449458789, -21138025189, -17982351589, -14826677989, -12220076389, -12219989989, -12219903589, -12219817189, -12219730789, -12219644389, -12219557989, -12219471589, -12219385189, -12219298789, -2208988800 ],
+  "diffs" : [ -174011, -87611, -1211, 85189, 171589, 257989, 344389, 430789, 517189, 603589, 689989, 776389, 862789, 776389, 689989, 603589, 517189, 430789, 344389, 257989, 171589, 85189, -1211, 0 ]
 }, {
   "tz" : "Europe/Minsk",
-  "switches" : [ -62135603416, -59006368216, -55850694616, -52695021016, -46383587416, -43227913816, -40072240216, -33760806616, -30605133016, -27449459416, -21138025816, -17982352216, -14826678616, -12219299416, -2840147416, -2208988800 ],
-  "diffs" : [ -176984, -90584, -4184, 82216, 168616, 255016, 341416, 427816, 514216, 600616, 687016, 773416, 859816, -4184, -4200, 0 ]
+  "switches" : [ -62135603416, -59006368216, -55850694616, -52695021016, -46383587416, -43227913816, -40072240216, -33760806616, -30605133016, -27449459416, -21138025816, -17982352216, -14826678616, -12220077016, -12219990616, -12219904216, -12219817816, -12219731416, -12219645016, -12219558616, -12219472216, -12219385816, -12219299416, -2840147416, -2208988800 ],
+  "diffs" : [ -176984, -90584, -4184, 82216, 168616, 255016, 341416, 427816, 514216, 600616, 687016, 773416, 859816, 773416, 687016, 600616, 514216, 427816, 341416, 255016, 168616, 82216, -4184, -4200, 0 ]
 }, {
   "tz" : "Europe/Monaco",
-  "switches" : [ -62135598572, -59006363372, -55850689772, -52695016172, -46383582572, -43227908972, -40072235372, -33760801772, -30605128172, -27449454572, -21138020972, -17982347372, -14826673772, -12219294572, -2486680172, -2208988800 ],
-  "diffs" : [ -174628, -88228, -1828, 84572, 170972, 257372, 343772, 430172, 516572, 602972, 689372, 775772, 862172, -1828, -3039, 0 ]
+  "switches" : [ -62135598572, -59006363372, -55850689772, -52695016172, -46383582572, -43227908972, -40072235372, -33760801772, -30605128172, -27449454572, -21138020972, -17982347372, -14826673772, -12220072172, -12219985772, -12219899372, -12219812972, -12219726572, -12219640172, -12219553772, -12219467372, -12219380972, -12219294572, -2486680172, -2208988800 ],
+  "diffs" : [ -174628, -88228, -1828, 84572, 170972, 257372, 343772, 430172, 516572, 602972, 689372, 775772, 862172, 775772, 689372, 602972, 516572, 430172, 343772, 257372, 170972, 84572, -1828, -3039, 0 ]
 }, {
   "tz" : "Europe/Moscow",
-  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12219301817, -2208988800 ],
-  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, -1783, 0 ]
+  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12220079417, -12219993017, -12219906617, -12219820217, -12219733817, -12219647417, -12219561017, -12219474617, -12219388217, -12219301817, -2208988800 ],
+  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, 775817, 689417, 603017, 516617, 430217, 343817, 257417, 171017, 84617, -1783, 0 ]
 }, {
   "tz" : "Europe/Nicosia",
-  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12219300808, -2208988800 ],
-  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 808, 0 ]
+  "switches" : [ -62135604808, -59006369608, -55850696008, -52695022408, -46383588808, -43227915208, -40072241608, -33760808008, -30605134408, -27449460808, -21138027208, -17982353608, -14826680008, -12220078408, -12219992008, -12219905608, -12219819208, -12219732808, -12219646408, -12219560008, -12219473608, -12219387208, -12219300808, -2208988800 ],
+  "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 778408, 692008, 605608, 519208, 432808, 346408, 260008, 173608, 87208, 808, 0 ]
 }, {
   "tz" : "Europe/Oslo",
-  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12219295380, -2366757780 ],
-  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, -1020, 0 ]
+  "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ],
+  "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ]
 }, {
   "tz" : "Europe/Paris",
-  "switches" : [ -62135597361, -59006362161, -55850688561, -52695014961, -46383581361, -43227907761, -40072234161, -33760800561, -30605126961, -27449453361, -21138019761, -17982346161, -14826672561, -12219293361, -2208988800 ],
-  "diffs" : [ -175839, -89439, -3039, 83361, 169761, 256161, 342561, 428961, 515361, 601761, 688161, 774561, 860961, -3039, 0 ]
+  "switches" : [ -62135597361, -59006362161, -55850688561, -52695014961, -46383581361, -43227907761, -40072234161, -33760800561, -30605126961, -27449453361, -21138019761, -17982346161, -14826672561, -12220070961, -12219984561, -12219898161, -12219811761, -12219725361, -12219638961, -12219552561, -12219466161, -12219379761, -12219293361, -2208988800 ],
+  "diffs" : [ -175839, -89439, -3039, 83361, 169761, 256161, 342561, 428961, 515361, 601761, 688161, 774561, 860961, 774561, 688161, 601761, 515361, 428961, 342561, 256161, 169761, 83361, -3039, 0 ]
 }, {
   "tz" : "Europe/Podgorica",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Europe/Prague",
-  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12219296264, -2469401864 ],
-  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, -136, 0 ]
+  "switches" : [ -62135600264, -59006365064, -55850691464, -52695017864, -46383584264, -43227910664, -40072237064, -33760803464, -30605129864, -27449456264, -21138022664, -17982349064, -14826675464, -12220073864, -12219987464, -12219901064, -12219814664, -12219728264, -12219641864, -12219555464, -12219469064, -12219382664, -12219296264, -2469401864 ],
+  "diffs" : [ -172936, -86536, -136, 86264, 172664, 259064, 345464, 431864, 518264, 604664, 691064, 777464, 863864, 777464, 691064, 604664, 518264, 431864, 345464, 259064, 172664, 86264, -136, 0 ]
 }, {
   "tz" : "Europe/Riga",
-  "switches" : [ -62135602594, -59006367394, -55850693794, -52695020194, -46383586594, -43227912994, -40072239394, -33760805794, -30605132194, -27449458594, -21138024994, -17982351394, -14826677794, -12219298594, -2208988800 ],
-  "diffs" : [ -174206, -87806, -1406, 84994, 171394, 257794, 344194, 430594, 516994, 603394, 689794, 776194, 862594, -1406, 0 ]
+  "switches" : [ -62135602594, -59006367394, -55850693794, -52695020194, -46383586594, -43227912994, -40072239394, -33760805794, -30605132194, -27449458594, -21138024994, -17982351394, -14826677794, -12220076194, -12219989794, -12219903394, -12219816994, -12219730594, -12219644194, -12219557794, -12219471394, -12219384994, -12219298594, -2208988800 ],
+  "diffs" : [ -174206, -87806, -1406, 84994, 171394, 257794, 344194, 430594, 516994, 603394, 689794, 776194, 862594, 776194, 689794, 603394, 516994, 430594, 344194, 257794, 171394, 84994, -1406, 0 ]
 }, {
   "tz" : "Europe/Rome",
-  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12219295796, -2403565200 ],
-  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12220073396, -12219986996, -12219900596, -12219814196, -12219727796, -12219641396, -12219554996, -12219468596, -12219382196, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
 }, {
   "tz" : "Europe/Samara",
-  "switches" : [ -62135608820, -59006373620, -55850700020, -52695026420, -46383592820, -43227919220, -40072245620, -33760812020, -30605138420, -27449464820, -21138031220, -17982357620, -14826684020, -12219304820, -2208988800 ],
-  "diffs" : [ -175180, -88780, -2380, 84020, 170420, 256820, 343220, 429620, 516020, 602420, 688820, 775220, 861620, -2380, 0 ]
+  "switches" : [ -62135608820, -59006373620, -55850700020, -52695026420, -46383592820, -43227919220, -40072245620, -33760812020, -30605138420, -27449464820, -21138031220, -17982357620, -14826684020, -12220082420, -12219996020, -12219909620, -12219823220, -12219736820, -12219650420, -12219564020, -12219477620, -12219391220, -12219304820, -2208988800 ],
+  "diffs" : [ -175180, -88780, -2380, 84020, 170420, 256820, 343220, 429620, 516020, 602420, 688820, 775220, 861620, 775220, 688820, 602420, 516020, 429620, 343220, 256820, 170420, 84020, -2380, 0 ]
 }, {
   "tz" : "Europe/San_Marino",
-  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12219295796, -2403565200 ],
-  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12220073396, -12219986996, -12219900596, -12219814196, -12219727796, -12219641396, -12219554996, -12219468596, -12219382196, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
 }, {
   "tz" : "Europe/Sarajevo",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Europe/Saratov",
-  "switches" : [ -62135607858, -59006372658, -55850699058, -52695025458, -46383591858, -43227918258, -40072244658, -33760811058, -30605137458, -27449463858, -21138030258, -17982356658, -14826683058, -12219303858, -2208988800 ],
-  "diffs" : [ -176142, -89742, -3342, 83058, 169458, 255858, 342258, 428658, 515058, 601458, 687858, 774258, 860658, -3342, 0 ]
+  "switches" : [ -62135607858, -59006372658, -55850699058, -52695025458, -46383591858, -43227918258, -40072244658, -33760811058, -30605137458, -27449463858, -21138030258, -17982356658, -14826683058, -12220081458, -12219995058, -12219908658, -12219822258, -12219735858, -12219649458, -12219563058, -12219476658, -12219390258, -12219303858, -2208988800 ],
+  "diffs" : [ -176142, -89742, -3342, 83058, 169458, 255858, 342258, 428658, 515058, 601458, 687858, 774258, 860658, 774258, 687858, 601458, 515058, 428658, 342258, 255858, 169458, 83058, -3342, 0 ]
 }, {
   "tz" : "Europe/Simferopol",
-  "switches" : [ -62135604984, -59006369784, -55850696184, -52695022584, -46383588984, -43227915384, -40072241784, -33760808184, -30605134584, -27449460984, -21138027384, -17982353784, -14826680184, -12219300984, -2840148984, -2208988800 ],
-  "diffs" : [ -175416, -89016, -2616, 83784, 170184, 256584, 342984, 429384, 515784, 602184, 688584, 774984, 861384, -2616, -2640, 0 ]
+  "switches" : [ -62135604984, -59006369784, -55850696184, -52695022584, -46383588984, -43227915384, -40072241784, -33760808184, -30605134584, -27449460984, -21138027384, -17982353784, -14826680184, -12220078584, -12219992184, -12219905784, -12219819384, -12219732984, -12219646584, -12219560184, -12219473784, -12219387384, -12219300984, -2840148984, -2208988800 ],
+  "diffs" : [ -175416, -89016, -2616, 83784, 170184, 256584, 342984, 429384, 515784, 602184, 688584, 774984, 861384, 774984, 688584, 602184, 515784, 429384, 342984, 256584, 170184, 83784, -2616, -2640, 0 ]
 }, {
   "tz" : "Europe/Skopje",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Europe/Sofia",
-  "switches" : [ -62135602396, -59006367196, -55850693596, -52695019996, -46383586396, -43227912796, -40072239196, -33760805596, -30605131996, -27449458396, -21138024796, -17982351196, -14826677596, -12219298396, -2840146396, -2369527016 ],
-  "diffs" : [ -174404, -88004, -1604, 84796, 171196, 257596, 343996, 430396, 516796, 603196, 689596, 775996, 862396, -1604, -184, 0 ]
+  "switches" : [ -62135602396, -59006367196, -55850693596, -52695019996, -46383586396, -43227912796, -40072239196, -33760805596, -30605131996, -27449458396, -21138024796, -17982351196, -14826677596, -12220075996, -12219989596, -12219903196, -12219816796, -12219730396, -12219643996, -12219557596, -12219471196, -12219384796, -12219298396, -2840146396, -2369527016 ],
+  "diffs" : [ -174404, -88004, -1604, 84796, 171196, 257596, 343996, 430396, 516796, 603196, 689596, 775996, 862396, 775996, 689596, 603196, 516796, 430396, 343996, 257596, 171196, 84796, -1604, -184, 0 ]
 }, {
   "tz" : "Europe/Stockholm",
-  "switches" : [ -62135601132, -59006365932, -55850692332, -52695018732, -46383585132, -43227911532, -40072237932, -33760804332, -30605130732, -27449457132, -21138023532, -17982349932, -14826676332, -12219297132, -2871681132, -2208992414 ],
-  "diffs" : [ -172068, -85668, 732, 87132, 173532, 259932, 346332, 432732, 519132, 605532, 691932, 778332, 864732, 732, 14, 0 ]
+  "switches" : [ -62135601132, -59006365932, -55850692332, -52695018732, -46383585132, -43227911532, -40072237932, -33760804332, -30605130732, -27449457132, -21138023532, -17982349932, -14826676332, -12220074732, -12219988332, -12219901932, -12219815532, -12219729132, -12219642732, -12219556332, -12219469932, -12219383532, -12219297132, -2871681132, -2208992414 ],
+  "diffs" : [ -172068, -85668, 732, 87132, 173532, 259932, 346332, 432732, 519132, 605532, 691932, 778332, 864732, 778332, 691932, 605532, 519132, 432732, 346332, 259932, 173532, 87132, 732, 14, 0 ]
 }, {
   "tz" : "Europe/Tallinn",
-  "switches" : [ -62135602740, -59006367540, -55850693940, -52695020340, -46383586740, -43227913140, -40072239540, -33760805940, -30605132340, -27449458740, -21138025140, -17982351540, -14826677940, -12219298740, -2208988800 ],
-  "diffs" : [ -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 862740, -1260, 0 ]
+  "switches" : [ -62135602740, -59006367540, -55850693940, -52695020340, -46383586740, -43227913140, -40072239540, -33760805940, -30605132340, -27449458740, -21138025140, -17982351540, -14826677940, -12220076340, -12219989940, -12219903540, -12219817140, -12219730740, -12219644340, -12219557940, -12219471540, -12219385140, -12219298740, -2208988800 ],
+  "diffs" : [ -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 862740, 776340, 689940, 603540, 517140, 430740, 344340, 257940, 171540, 85140, -1260, 0 ]
 }, {
   "tz" : "Europe/Tirane",
-  "switches" : [ -62135601560, -59006366360, -55850692760, -52695019160, -46383585560, -43227911960, -40072238360, -33760804760, -30605131160, -27449457560, -21138023960, -17982350360, -14826676760, -12219297560, -2208988800 ],
-  "diffs" : [ -171640, -85240, 1160, 87560, 173960, 260360, 346760, 433160, 519560, 605960, 692360, 778760, 865160, 1160, 0 ]
+  "switches" : [ -62135601560, -59006366360, -55850692760, -52695019160, -46383585560, -43227911960, -40072238360, -33760804760, -30605131160, -27449457560, -21138023960, -17982350360, -14826676760, -12220075160, -12219988760, -12219902360, -12219815960, -12219729560, -12219643160, -12219556760, -12219470360, -12219383960, -12219297560, -2208988800 ],
+  "diffs" : [ -171640, -85240, 1160, 87560, 173960, 260360, 346760, 433160, 519560, 605960, 692360, 778760, 865160, 778760, 692360, 605960, 519560, 433160, 346760, 260360, 173960, 87560, 1160, 0 ]
 }, {
   "tz" : "Europe/Tiraspol",
-  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12219299720, -2840147720, -2208988800 ],
-  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, -280, -300, 0 ]
+  "switches" : [ -62135603720, -59006368520, -55850694920, -52695021320, -46383587720, -43227914120, -40072240520, -33760806920, -30605133320, -27449459720, -21138026120, -17982352520, -14826678920, -12220077320, -12219990920, -12219904520, -12219818120, -12219731720, -12219645320, -12219558920, -12219472520, -12219386120, -12219299720, -2840147720, -2208988800 ],
+  "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, 777320, 690920, 604520, 518120, 431720, 345320, 258920, 172520, 86120, -280, -300, 0 ]
 }, {
   "tz" : "Europe/Ulyanovsk",
-  "switches" : [ -62135608416, -59006373216, -55850699616, -52695026016, -46383592416, -43227918816, -40072245216, -33760811616, -30605138016, -27449464416, -21138030816, -17982357216, -14826683616, -12219304416, -2208988800 ],
-  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, -2784, 0 ]
+  "switches" : [ -62135608416, -59006373216, -55850699616, -52695026016, -46383592416, -43227918816, -40072245216, -33760811616, -30605138016, -27449464416, -21138030816, -17982357216, -14826683616, -12220082016, -12219995616, -12219909216, -12219822816, -12219736416, -12219650016, -12219563616, -12219477216, -12219390816, -12219304416, -2208988800 ],
+  "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
 }, {
   "tz" : "Europe/Uzhgorod",
-  "switches" : [ -62135602152, -59006366952, -55850693352, -52695019752, -46383586152, -43227912552, -40072238952, -33760805352, -30605131752, -27449458152, -21138024552, -17982350952, -14826677352, -12219298152, -2500939752, -2208988800 ],
-  "diffs" : [ -174648, -88248, -1848, 84552, 170952, 257352, 343752, 430152, 516552, 602952, 689352, 775752, 862152, -1848, -3600, 0 ]
+  "switches" : [ -62135602152, -59006366952, -55850693352, -52695019752, -46383586152, -43227912552, -40072238952, -33760805352, -30605131752, -27449458152, -21138024552, -17982350952, -14826677352, -12220075752, -12219989352, -12219902952, -12219816552, -12219730152, -12219643752, -12219557352, -12219470952, -12219384552, -12219298152, -2500939752, -2208988800 ],
+  "diffs" : [ -174648, -88248, -1848, 84552, 170952, 257352, 343752, 430152, 516552, 602952, 689352, 775752, 862152, 775752, 689352, 602952, 516552, 430152, 343752, 257352, 170952, 84552, -1848, -3600, 0 ]
 }, {
   "tz" : "Europe/Vaduz",
-  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12219294848, -3675198848, -2385246586 ],
-  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, -1552, -1814, 0 ]
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12220072448, -12219986048, -12219899648, -12219813248, -12219726848, -12219640448, -12219554048, -12219467648, -12219381248, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, 776048, 689648, 603248, 516848, 430448, 344048, 257648, 171248, 84848, -1552, -1814, 0 ]
 }, {
   "tz" : "Europe/Vatican",
-  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12219295796, -2403565200 ],
-  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, -604, 0 ]
+  "switches" : [ -62135599796, -59006364596, -55850690996, -52695017396, -46383583796, -43227910196, -40072236596, -33760802996, -30605129396, -27449455796, -21138022196, -17982348596, -14826674996, -12220073396, -12219986996, -12219900596, -12219814196, -12219727796, -12219641396, -12219554996, -12219468596, -12219382196, -12219295796, -2403565200 ],
+  "diffs" : [ -173404, -87004, -604, 85796, 172196, 258596, 344996, 431396, 517796, 604196, 690596, 776996, 863396, 776996, 690596, 604196, 517796, 431396, 344996, 258596, 172196, 85796, -604, 0 ]
 }, {
   "tz" : "Europe/Vienna",
-  "switches" : [ -62135600721, -59006365521, -55850691921, -52695018321, -46383584721, -43227911121, -40072237521, -33760803921, -30605130321, -27449456721, -21138023121, -17982349521, -14826675921, -12219296721, -2422055121 ],
-  "diffs" : [ -172479, -86079, 321, 86721, 173121, 259521, 345921, 432321, 518721, 605121, 691521, 777921, 864321, 321, 0 ]
+  "switches" : [ -62135600721, -59006365521, -55850691921, -52695018321, -46383584721, -43227911121, -40072237521, -33760803921, -30605130321, -27449456721, -21138023121, -17982349521, -14826675921, -12220074321, -12219987921, -12219901521, -12219815121, -12219728721, -12219642321, -12219555921, -12219469521, -12219383121, -12219296721, -2422055121 ],
+  "diffs" : [ -172479, -86079, 321, 86721, 173121, 259521, 345921, 432321, 518721, 605121, 691521, 777921, 864321, 777921, 691521, 605121, 518721, 432321, 345921, 259521, 173121, 86721, 321, 0 ]
 }, {
   "tz" : "Europe/Vilnius",
-  "switches" : [ -62135602876, -59006367676, -55850694076, -52695020476, -46383586876, -43227913276, -40072239676, -33760806076, -30605132476, -27449458876, -21138025276, -17982351676, -14826678076, -12219298876, -2840146876, -2208988800 ],
-  "diffs" : [ -173924, -87524, -1124, 85276, 171676, 258076, 344476, 430876, 517276, 603676, 690076, 776476, 862876, -1124, -2160, 0 ]
+  "switches" : [ -62135602876, -59006367676, -55850694076, -52695020476, -46383586876, -43227913276, -40072239676, -33760806076, -30605132476, -27449458876, -21138025276, -17982351676, -14826678076, -12220076476, -12219990076, -12219903676, -12219817276, -12219730876, -12219644476, -12219558076, -12219471676, -12219385276, -12219298876, -2840146876, -2208988800 ],
+  "diffs" : [ -173924, -87524, -1124, 85276, 171676, 258076, 344476, 430876, 517276, 603676, 690076, 776476, 862876, 776476, 690076, 603676, 517276, 430876, 344476, 258076, 171676, 85276, -1124, -2160, 0 ]
 }, {
   "tz" : "Europe/Volgograd",
-  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12219303460, -2208988800 ],
-  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, -3740, 0 ]
+  "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12220081060, -12219994660, -12219908260, -12219821860, -12219735460, -12219649060, -12219562660, -12219476260, -12219389860, -12219303460, -2208988800 ],
+  "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, 773860, 687460, 601060, 514660, 428260, 341860, 255460, 169060, 82660, -3740, 0 ]
 }, {
   "tz" : "Europe/Warsaw",
-  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12219297840, -2208988800 ],
-  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 1440, 0 ]
+  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12220075440, -12219989040, -12219902640, -12219816240, -12219729840, -12219643440, -12219557040, -12219470640, -12219384240, -12219297840, -2208988800 ],
+  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 779040, 692640, 606240, 519840, 433440, 347040, 260640, 174240, 87840, 1440, 0 ]
 }, {
   "tz" : "Europe/Zagreb",
-  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12219297720, -2713915320 ],
-  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 1320, 0 ]
+  "switches" : [ -62135601720, -59006366520, -55850692920, -52695019320, -46383585720, -43227912120, -40072238520, -33760804920, -30605131320, -27449457720, -21138024120, -17982350520, -14826676920, -12220075320, -12219988920, -12219902520, -12219816120, -12219729720, -12219643320, -12219556920, -12219470520, -12219384120, -12219297720, -2713915320 ],
+  "diffs" : [ -171480, -85080, 1320, 87720, 174120, 260520, 346920, 433320, 519720, 606120, 692520, 778920, 865320, 778920, 692520, 606120, 519720, 433320, 346920, 260520, 174120, 87720, 1320, 0 ]
 }, {
   "tz" : "Europe/Zaporozhye",
-  "switches" : [ -62135605240, -59006370040, -55850696440, -52695022840, -46383589240, -43227915640, -40072242040, -33760808440, -30605134840, -27449461240, -21138027640, -17982354040, -14826680440, -12219301240, -2840149240, -2208988800 ],
-  "diffs" : [ -171560, -85160, 1240, 87640, 174040, 260440, 346840, 433240, 519640, 606040, 692440, 778840, 865240, 1240, 1200, 0 ]
+  "switches" : [ -62135605240, -59006370040, -55850696440, -52695022840, -46383589240, -43227915640, -40072242040, -33760808440, -30605134840, -27449461240, -21138027640, -17982354040, -14826680440, -12220078840, -12219992440, -12219906040, -12219819640, -12219733240, -12219646840, -12219560440, -12219474040, -12219387640, -12219301240, -2840149240, -2208988800 ],
+  "diffs" : [ -171560, -85160, 1240, 87640, 174040, 260440, 346840, 433240, 519640, 606040, 692440, 778840, 865240, 778840, 692440, 606040, 519640, 433240, 346840, 260440, 174040, 87640, 1240, 1200, 0 ]
 }, {
   "tz" : "Europe/Zurich",
-  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12219294848, -3675198848, -2385246586 ],
-  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, -1552, -1814, 0 ]
+  "switches" : [ -62135598848, -59006363648, -55850690048, -52695016448, -46383582848, -43227909248, -40072235648, -33760802048, -30605128448, -27449454848, -21138021248, -17982347648, -14826674048, -12220072448, -12219986048, -12219899648, -12219813248, -12219726848, -12219640448, -12219554048, -12219467648, -12219381248, -12219294848, -3675198848, -2385246586 ],
+  "diffs" : [ -174352, -87952, -1552, 84848, 171248, 257648, 344048, 430448, 516848, 603248, 689648, 776048, 862448, 776048, 689648, 603248, 516848, 430448, 344048, 257648, 171248, 84848, -1552, -1814, 0 ]
 }, {
   "tz" : "GB",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "GB-Eire",
-  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12219292725, -3852662325 ],
-  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, -75, 0 ]
+  "switches" : [ -62135596725, -59006361525, -55850687925, -52695014325, -46383580725, -43227907125, -40072233525, -33760799925, -30605126325, -27449452725, -21138019125, -17982345525, -14826671925, -12220070325, -12219983925, -12219897525, -12219811125, -12219724725, -12219638325, -12219551925, -12219465525, -12219379125, -12219292725, -3852662325 ],
+  "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ]
 }, {
   "tz" : "GMT",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "GMT0",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Greenwich",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Hongkong",
-  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12219320202, -2208988800 ],
-  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, -1398, 0 ]
+  "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12220097802, -12220011402, -12219925002, -12219838602, -12219752202, -12219665802, -12219579402, -12219493002, -12219406602, -12219320202, -2208988800 ],
+  "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, 776202, 689802, 603402, 517002, 430602, 344202, 257802, 171402, 85002, -1398, 0 ]
 }, {
   "tz" : "Iceland",
-  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12219287520, -2208988800 ],
-  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, -5280, 0 ]
+  "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12220065120, -12219978720, -12219892320, -12219805920, -12219719520, -12219633120, -12219546720, -12219460320, -12219373920, -12219287520, -2208988800 ],
+  "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, 772320, 685920, 599520, 513120, 426720, 340320, 253920, 167520, 81120, -5280, 0 ]
 }, {
   "tz" : "Indian/Antananarivo",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Indian/Chagos",
-  "switches" : [ -62135614180, -59006378980, -55850705380, -52695031780, -46383598180, -43227924580, -40072250980, -33760817380, -30605143780, -27449470180, -21138036580, -17982362980, -14826689380, -12219310180, -2208988800 ],
-  "diffs" : [ -177020, -90620, -4220, 82180, 168580, 254980, 341380, 427780, 514180, 600580, 686980, 773380, 859780, -4220, 0 ]
+  "switches" : [ -62135614180, -59006378980, -55850705380, -52695031780, -46383598180, -43227924580, -40072250980, -33760817380, -30605143780, -27449470180, -21138036580, -17982362980, -14826689380, -12220087780, -12220001380, -12219914980, -12219828580, -12219742180, -12219655780, -12219569380, -12219482980, -12219396580, -12219310180, -2208988800 ],
+  "diffs" : [ -177020, -90620, -4220, 82180, 168580, 254980, 341380, 427780, 514180, 600580, 686980, 773380, 859780, 773380, 686980, 600580, 514180, 427780, 341380, 254980, 168580, 82180, -4220, 0 ]
 }, {
   "tz" : "Indian/Christmas",
-  "switches" : [ -62135622172, -59006386972, -55850713372, -52695039772, -46383606172, -43227932572, -40072258972, -33760825372, -30605151772, -27449478172, -21138044572, -17982370972, -14826697372, -12219318172, -2364102172 ],
-  "diffs" : [ -172628, -86228, 172, 86572, 172972, 259372, 345772, 432172, 518572, 604972, 691372, 777772, 864172, 172, 0 ]
+  "switches" : [ -62135622172, -59006386972, -55850713372, -52695039772, -46383606172, -43227932572, -40072258972, -33760825372, -30605151772, -27449478172, -21138044572, -17982370972, -14826697372, -12220095772, -12220009372, -12219922972, -12219836572, -12219750172, -12219663772, -12219577372, -12219490972, -12219404572, -12219318172, -2364102172 ],
+  "diffs" : [ -172628, -86228, 172, 86572, 172972, 259372, 345772, 432172, 518572, 604972, 691372, 777772, 864172, 777772, 691372, 604972, 518572, 432172, 345772, 259372, 172972, 86572, 172, 0 ]
 }, {
   "tz" : "Indian/Cocos",
-  "switches" : [ -62135620060, -59006384860, -55850711260, -52695037660, -46383604060, -43227930460, -40072256860, -33760823260, -30605149660, -27449476060, -21138042460, -17982368860, -14826695260, -12219316060, -2209012060 ],
-  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, -140, 0 ]
+  "switches" : [ -62135620060, -59006384860, -55850711260, -52695037660, -46383604060, -43227930460, -40072256860, -33760823260, -30605149660, -27449476060, -21138042460, -17982368860, -14826695260, -12220093660, -12220007260, -12219920860, -12219834460, -12219748060, -12219661660, -12219575260, -12219488860, -12219402460, -12219316060, -2209012060 ],
+  "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, 777460, 691060, 604660, 518260, 431860, 345460, 259060, 172660, 86260, -140, 0 ]
 }, {
   "tz" : "Indian/Comoro",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Indian/Kerguelen",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800, -2208988800 ],
-  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, -18000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ],
+  "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, 759600, 673200, 586800, 500400, 414000, 327600, 241200, 154800, 68400, -18000, 0 ]
 }, {
   "tz" : "Indian/Mahe",
-  "switches" : [ -62135610108, -59006374908, -55850701308, -52695027708, -46383594108, -43227920508, -40072246908, -33760813308, -30605139708, -27449466108, -21138032508, -17982358908, -14826685308, -12219306108, -2208988800 ],
-  "diffs" : [ -173892, -87492, -1092, 85308, 171708, 258108, 344508, 430908, 517308, 603708, 690108, 776508, 862908, -1092, 0 ]
+  "switches" : [ -62135610108, -59006374908, -55850701308, -52695027708, -46383594108, -43227920508, -40072246908, -33760813308, -30605139708, -27449466108, -21138032508, -17982358908, -14826685308, -12220083708, -12219997308, -12219910908, -12219824508, -12219738108, -12219651708, -12219565308, -12219478908, -12219392508, -12219306108, -2208988800 ],
+  "diffs" : [ -173892, -87492, -1092, 85308, 171708, 258108, 344508, 430908, 517308, 603708, 690108, 776508, 862908, 776508, 690108, 603708, 517308, 430908, 344508, 258108, 171708, 85308, -1092, 0 ]
 }, {
   "tz" : "Indian/Maldives",
-  "switches" : [ -62135614440, -59006379240, -55850705640, -52695032040, -46383598440, -43227924840, -40072251240, -33760817640, -30605144040, -27449470440, -21138036840, -17982363240, -14826689640, -12219310440, -2208988800 ],
-  "diffs" : [ -173160, -86760, -360, 86040, 172440, 258840, 345240, 431640, 518040, 604440, 690840, 777240, 863640, -360, 0 ]
+  "switches" : [ -62135614440, -59006379240, -55850705640, -52695032040, -46383598440, -43227924840, -40072251240, -33760817640, -30605144040, -27449470440, -21138036840, -17982363240, -14826689640, -12220088040, -12220001640, -12219915240, -12219828840, -12219742440, -12219656040, -12219569640, -12219483240, -12219396840, -12219310440, -2208988800 ],
+  "diffs" : [ -173160, -86760, -360, 86040, 172440, 258840, 345240, 431640, 518040, 604440, 690840, 777240, 863640, 777240, 690840, 604440, 518040, 431640, 345240, 258840, 172440, 86040, -360, 0 ]
 }, {
   "tz" : "Indian/Mauritius",
-  "switches" : [ -62135610600, -59006375400, -55850701800, -52695028200, -46383594600, -43227921000, -40072247400, -33760813800, -30605140200, -27449466600, -21138033000, -17982359400, -14826685800, -12219306600, -2208988800 ],
-  "diffs" : [ -173400, -87000, -600, 85800, 172200, 258600, 345000, 431400, 517800, 604200, 690600, 777000, 863400, -600, 0 ]
+  "switches" : [ -62135610600, -59006375400, -55850701800, -52695028200, -46383594600, -43227921000, -40072247400, -33760813800, -30605140200, -27449466600, -21138033000, -17982359400, -14826685800, -12220084200, -12219997800, -12219911400, -12219825000, -12219738600, -12219652200, -12219565800, -12219479400, -12219393000, -12219306600, -2208988800 ],
+  "diffs" : [ -173400, -87000, -600, 85800, 172200, 258600, 345000, 431400, 517800, 604200, 690600, 777000, 863400, 777000, 690600, 604200, 517800, 431400, 345000, 258600, 172200, 85800, -600, 0 ]
 }, {
   "tz" : "Indian/Mayotte",
-  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12219301636, -2208988800 ],
-  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, -1964, 0 ]
+  "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ],
+  "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ]
 }, {
   "tz" : "Indian/Reunion",
-  "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12219306112, -2208988800 ],
-  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, -1088, 0 ]
+  "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12220083712, -12219997312, -12219910912, -12219824512, -12219738112, -12219651712, -12219565312, -12219478912, -12219392512, -12219306112, -2208988800 ],
+  "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, 0 ]
 }, {
   "tz" : "Israel",
-  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12219301254, -2840149254, -2208988800 ],
-  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 1254, 1240, 0 ]
+  "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
+  "diffs" : [ -171546, -85146, 1254, 87654, 174054, 260454, 346854, 433254, 519654, 606054, 692454, 778854, 865254, 778854, 692454, 606054, 519654, 433254, 346854, 260454, 174054, 87654, 1254, 1240, 0 ]
 }, {
   "tz" : "Jamaica",
-  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12219274370, -2208988800 ],
-  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, -430, 0 ]
+  "switches" : [ -62135578370, -59006343170, -55850669570, -52694995970, -46383562370, -43227888770, -40072215170, -33760781570, -30605107970, -27449434370, -21138000770, -17982327170, -14826653570, -12220051970, -12219965570, -12219879170, -12219792770, -12219706370, -12219619970, -12219533570, -12219447170, -12219360770, -12219274370, -2208988800 ],
+  "diffs" : [ -173230, -86830, -430, 85970, 172370, 258770, 345170, 431570, 517970, 604370, 690770, 777170, 863570, 777170, 690770, 604370, 517970, 431570, 345170, 258770, 172370, 85970, -430, 0 ]
 }, {
   "tz" : "Japan",
-  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12219326339, -2587712400 ],
-  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 1139, 0 ]
+  "switches" : [ -62135630339, -59006395139, -55850721539, -52695047939, -46383614339, -43227940739, -40072267139, -33760833539, -30605159939, -27449486339, -21138052739, -17982379139, -14826705539, -12220103939, -12220017539, -12219931139, -12219844739, -12219758339, -12219671939, -12219585539, -12219499139, -12219412739, -12219326339, -2587712400 ],
+  "diffs" : [ -171661, -85261, 1139, 87539, 173939, 260339, 346739, 433139, 519539, 605939, 692339, 778739, 865139, 778739, 692339, 605939, 519539, 433139, 346739, 260339, 173939, 87539, 1139, 0 ]
 }, {
   "tz" : "Kwajalein",
-  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12219332960, -2208988800 ],
-  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, -3040, 0 ]
+  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12220110560, -12220024160, -12219937760, -12219851360, -12219764960, -12219678560, -12219592160, -12219505760, -12219419360, -12219332960, -2208988800 ],
+  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, 774560, 688160, 601760, 515360, 428960, 342560, 256160, 169760, 83360, -3040, 0 ]
 }, {
   "tz" : "Libya",
-  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12219295964, -2208988800 ],
-  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, -4036, 0 ]
+  "switches" : [ -62135599964, -59006364764, -55850691164, -52695017564, -46383583964, -43227910364, -40072236764, -33760803164, -30605129564, -27449455964, -21138022364, -17982348764, -14826675164, -12220073564, -12219987164, -12219900764, -12219814364, -12219727964, -12219641564, -12219555164, -12219468764, -12219382364, -12219295964, -2208988800 ],
+  "diffs" : [ -176836, -90436, -4036, 82364, 168764, 255164, 341564, 427964, 514364, 600764, 687164, 773564, 859964, 773564, 687164, 600764, 514364, 427964, 341564, 255164, 168764, 82364, -4036, 0 ]
 }, {
   "tz" : "MET",
-  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12219296400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135600400, -59006365200, -55850691600, -52695018000, -46383584400, -43227910800, -40072237200, -33760803600, -30605130000, -27449456400, -21138022800, -17982349200, -14826675600, -12220074000, -12219987600, -12219901200, -12219814800, -12219728400, -12219642000, -12219555600, -12219469200, -12219382800, -12219296400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "MST7MDT",
-  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Mexico/BajaNorte",
-  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12219264716, -2208988800 ],
-  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 716, 0 ]
+  "switches" : [ -62135568716, -59006333516, -55850659916, -52694986316, -46383552716, -43227879116, -40072205516, -33760771916, -30605098316, -27449424716, -21137991116, -17982317516, -14826643916, -12220042316, -12219955916, -12219869516, -12219783116, -12219696716, -12219610316, -12219523916, -12219437516, -12219351116, -12219264716, -2208988800 ],
+  "diffs" : [ -172084, -85684, 716, 87116, 173516, 259916, 346316, 432716, 519116, 605516, 691916, 778316, 864716, 778316, 691916, 605516, 519116, 432716, 346316, 259916, 173516, 87116, 716, 0 ]
 }, {
   "tz" : "Mexico/BajaSur",
-  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12219267260, -2208988800 ],
-  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, -340, 0 ]
+  "switches" : [ -62135571260, -59006336060, -55850662460, -52694988860, -46383555260, -43227881660, -40072208060, -33760774460, -30605100860, -27449427260, -21137993660, -17982320060, -14826646460, -12220044860, -12219958460, -12219872060, -12219785660, -12219699260, -12219612860, -12219526460, -12219440060, -12219353660, -12219267260, -2208988800 ],
+  "diffs" : [ -173140, -86740, -340, 86060, 172460, 258860, 345260, 431660, 518060, 604460, 690860, 777260, 863660, 777260, 690860, 604460, 518060, 431660, 345260, 258860, 172460, 86060, -340, 0 ]
 }, {
   "tz" : "Mexico/General",
-  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12219269004, -2208988800 ],
-  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, -2196, 0 ]
+  "switches" : [ -62135573004, -59006337804, -55850664204, -52694990604, -46383557004, -43227883404, -40072209804, -33760776204, -30605102604, -27449429004, -21137995404, -17982321804, -14826648204, -12220046604, -12219960204, -12219873804, -12219787404, -12219701004, -12219614604, -12219528204, -12219441804, -12219355404, -12219269004, -2208988800 ],
+  "diffs" : [ -174996, -88596, -2196, 84204, 170604, 257004, 343404, 429804, 516204, 602604, 689004, 775404, 861804, 775404, 689004, 602604, 516204, 429804, 343404, 257004, 170604, 84204, -2196, 0 ]
 }, {
   "tz" : "NZ",
-  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
-  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
 }, {
   "tz" : "NZ-CHAT",
-  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12219336828, -3192437628, -2208988800 ],
-  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, -1872, -1800, 0 ]
+  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12220114428, -12220028028, -12219941628, -12219855228, -12219768828, -12219682428, -12219596028, -12219509628, -12219423228, -12219336828, -3192437628, -2208988800 ],
+  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, 775728, 689328, 602928, 516528, 430128, 343728, 257328, 170928, 84528, -1872, -1800, 0 ]
 }, {
   "tz" : "Navajo",
-  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
-  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
 }, {
   "tz" : "PRC",
-  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12219321943, -2208988800 ],
-  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 343, 0 ]
+  "switches" : [ -62135625943, -59006390743, -55850717143, -52695043543, -46383609943, -43227936343, -40072262743, -33760829143, -30605155543, -27449481943, -21138048343, -17982374743, -14826701143, -12220099543, -12220013143, -12219926743, -12219840343, -12219753943, -12219667543, -12219581143, -12219494743, -12219408343, -12219321943, -2208988800 ],
+  "diffs" : [ -172457, -86057, 343, 86743, 173143, 259543, 345943, 432343, 518743, 605143, 691543, 777943, 864343, 777943, 691543, 605143, 518743, 432343, 345943, 259543, 173143, 86743, 343, 0 ]
 }, {
   "tz" : "PST8PDT",
-  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Pacific/Apia",
-  "switches" : [ -62135641984, -59006406784, -55850733184, -52695059584, -46383625984, -43227952384, -40072278784, -33760845184, -30605171584, -27449497984, -21138064384, -17982390784, -14826717184, -12219337984, -2445424384, -2208988800 ],
-  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, -1616, -88016, 0 ]
+  "switches" : [ -62135641984, -59006406784, -55850733184, -52695059584, -46383625984, -43227952384, -40072278784, -33760845184, -30605171584, -27449497984, -21138064384, -17982390784, -14826717184, -12220115584, -12220029184, -12219942784, -12219856384, -12219769984, -12219683584, -12219597184, -12219510784, -12219424384, -12219337984, -2445424384, -2208988800 ],
+  "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, 775984, 689584, 603184, 516784, 430384, 343984, 257584, 171184, 84784, -1616, -88016, 0 ]
 }, {
   "tz" : "Pacific/Auckland",
-  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12219334744, -3192435544, -2208988800 ],
-  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, -1256, -1800, 0 ]
+  "switches" : [ -62135638744, -59006403544, -55850729944, -52695056344, -46383622744, -43227949144, -40072275544, -33760841944, -30605168344, -27449494744, -21138061144, -17982387544, -14826713944, -12220112344, -12220025944, -12219939544, -12219853144, -12219766744, -12219680344, -12219593944, -12219507544, -12219421144, -12219334744, -3192435544, -2208988800 ],
+  "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ]
 }, {
   "tz" : "Pacific/Bougainville",
-  "switches" : [ -62135634136, -59006398936, -55850725336, -52695051736, -46383618136, -43227944536, -40072270936, -33760837336, -30605163736, -27449490136, -21138056536, -17982382936, -14826709336, -12219330136, -2840178136, -2366790512, -2208988800 ],
-  "diffs" : [ -175064, -88664, -2264, 84136, 170536, 256936, 343336, 429736, 516136, 602536, 688936, 775336, 861736, -2264, -4288, -3600, 0 ]
+  "switches" : [ -62135634136, -59006398936, -55850725336, -52695051736, -46383618136, -43227944536, -40072270936, -33760837336, -30605163736, -27449490136, -21138056536, -17982382936, -14826709336, -12220107736, -12220021336, -12219934936, -12219848536, -12219762136, -12219675736, -12219589336, -12219502936, -12219416536, -12219330136, -2840178136, -2366790512, -2208988800 ],
+  "diffs" : [ -175064, -88664, -2264, 84136, 170536, 256936, 343336, 429736, 516136, 602536, 688936, 775336, 861736, 775336, 688936, 602536, 516136, 429736, 343336, 256936, 170536, 84136, -2264, -4288, -3600, 0 ]
 }, {
   "tz" : "Pacific/Chatham",
-  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12219336828, -3192437628, -2208988800 ],
-  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, -1872, -1800, 0 ]
+  "switches" : [ -62135640828, -59006405628, -55850732028, -52695058428, -46383624828, -43227951228, -40072277628, -33760844028, -30605170428, -27449496828, -21138063228, -17982389628, -14826716028, -12220114428, -12220028028, -12219941628, -12219855228, -12219768828, -12219682428, -12219596028, -12219509628, -12219423228, -12219336828, -3192437628, -2208988800 ],
+  "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, 775728, 689328, 602928, 516528, 430128, 343728, 257328, 170928, 84528, -1872, -1800, 0 ]
 }, {
   "tz" : "Pacific/Chuuk",
-  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12219242828, -3944628428, -2208988800 ],
-  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, -85972, 428, 0 ]
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ]
 }, {
   "tz" : "Pacific/Easter",
-  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12219266552, -2208988800 ],
-  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, -4648, 0 ]
+  "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12220044152, -12219957752, -12219871352, -12219784952, -12219698552, -12219612152, -12219525752, -12219439352, -12219352952, -12219266552, -2208988800 ],
+  "diffs" : [ -177448, -91048, -4648, 81752, 168152, 254552, 340952, 427352, 513752, 600152, 686552, 772952, 859352, 772952, 686552, 600152, 513752, 427352, 340952, 254552, 168152, 81752, -4648, 0 ]
 }, {
   "tz" : "Pacific/Efate",
-  "switches" : [ -62135637196, -59006401996, -55850728396, -52695054796, -46383621196, -43227947596, -40072273996, -33760840396, -30605166796, -27449493196, -21138059596, -17982385996, -14826712396, -12219333196, -2208988800 ],
-  "diffs" : [ -172004, -85604, 796, 87196, 173596, 259996, 346396, 432796, 519196, 605596, 691996, 778396, 864796, 796, 0 ]
+  "switches" : [ -62135637196, -59006401996, -55850728396, -52695054796, -46383621196, -43227947596, -40072273996, -33760840396, -30605166796, -27449493196, -21138059596, -17982385996, -14826712396, -12220110796, -12220024396, -12219937996, -12219851596, -12219765196, -12219678796, -12219592396, -12219505996, -12219419596, -12219333196, -2208988800 ],
+  "diffs" : [ -172004, -85604, 796, 87196, 173596, 259996, 346396, 432796, 519196, 605596, 691996, 778396, 864796, 778396, 691996, 605596, 519196, 432796, 346396, 259996, 173596, 87196, 796, 0 ]
 }, {
   "tz" : "Pacific/Enderbury",
-  "switches" : [ -62135555740, -59006320540, -55850646940, -52694973340, -46383539740, -43227866140, -40072192540, -33760758940, -30605085340, -27449411740, -21137978140, -17982304540, -14826630940, -12219251740, -2208988800 ],
-  "diffs" : [ -260660, -174260, -87860, -1460, 84940, 171340, 257740, 344140, 430540, 516940, 603340, 689740, 776140, -87860, 0 ]
+  "switches" : [ -62135555740, -59006320540, -55850646940, -52694973340, -46383539740, -43227866140, -40072192540, -33760758940, -30605085340, -27449411740, -21137978140, -17982304540, -14826630940, -12220029340, -12219942940, -12219856540, -12219770140, -12219683740, -12219597340, -12219510940, -12219424540, -12219338140, -12219251740, -2208988800 ],
+  "diffs" : [ -260660, -174260, -87860, -1460, 84940, 171340, 257740, 344140, 430540, 516940, 603340, 689740, 776140, 689740, 603340, 516940, 430540, 344140, 257740, 171340, 84940, -1460, -87860, 0 ]
 }, {
   "tz" : "Pacific/Fakaofo",
-  "switches" : [ -62135555704, -59006320504, -55850646904, -52694973304, -46383539704, -43227866104, -40072192504, -33760758904, -30605085304, -27449411704, -21137978104, -17982304504, -14826630904, -12219251704, -2208988800 ],
-  "diffs" : [ -260696, -174296, -87896, -1496, 84904, 171304, 257704, 344104, 430504, 516904, 603304, 689704, 776104, -87896, 0 ]
+  "switches" : [ -62135555704, -59006320504, -55850646904, -52694973304, -46383539704, -43227866104, -40072192504, -33760758904, -30605085304, -27449411704, -21137978104, -17982304504, -14826630904, -12220029304, -12219942904, -12219856504, -12219770104, -12219683704, -12219597304, -12219510904, -12219424504, -12219338104, -12219251704, -2208988800 ],
+  "diffs" : [ -260696, -174296, -87896, -1496, 84904, 171304, 257704, 344104, 430504, 516904, 603304, 689704, 776104, 689704, 603304, 516904, 430504, 344104, 257704, 171304, 84904, -1496, -87896, 0 ]
 }, {
   "tz" : "Pacific/Fiji",
-  "switches" : [ -62135639744, -59006404544, -55850730944, -52695057344, -46383623744, -43227950144, -40072276544, -33760842944, -30605169344, -27449495744, -21138062144, -17982388544, -14826714944, -12219335744, -2208988800 ],
-  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, -256, 0 ]
+  "switches" : [ -62135639744, -59006404544, -55850730944, -52695057344, -46383623744, -43227950144, -40072276544, -33760842944, -30605169344, -27449495744, -21138062144, -17982388544, -14826714944, -12220113344, -12220026944, -12219940544, -12219854144, -12219767744, -12219681344, -12219594944, -12219508544, -12219422144, -12219335744, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ]
 }, {
   "tz" : "Pacific/Funafuti",
-  "switches" : [ -62135639812, -59006404612, -55850731012, -52695057412, -46383623812, -43227950212, -40072276612, -33760843012, -30605169412, -27449495812, -21138062212, -17982388612, -14826715012, -12219335812, -2208988800 ],
-  "diffs" : [ -172988, -86588, -188, 86212, 172612, 259012, 345412, 431812, 518212, 604612, 691012, 777412, 863812, -188, 0 ]
+  "switches" : [ -62135639812, -59006404612, -55850731012, -52695057412, -46383623812, -43227950212, -40072276612, -33760843012, -30605169412, -27449495812, -21138062212, -17982388612, -14826715012, -12220113412, -12220027012, -12219940612, -12219854212, -12219767812, -12219681412, -12219595012, -12219508612, -12219422212, -12219335812, -2208988800 ],
+  "diffs" : [ -172988, -86588, -188, 86212, 172612, 259012, 345412, 431812, 518212, 604612, 691012, 777412, 863812, 777412, 691012, 604612, 518212, 431812, 345412, 259012, 172612, 86212, -188, 0 ]
 }, {
   "tz" : "Pacific/Galapagos",
-  "switches" : [ -62135575296, -59006340096, -55850666496, -52694992896, -46383559296, -43227885696, -40072212096, -33760778496, -30605104896, -27449431296, -21137997696, -17982324096, -14826650496, -12219271296, -2208988800 ],
-  "diffs" : [ -172704, -86304, 96, 86496, 172896, 259296, 345696, 432096, 518496, 604896, 691296, 777696, 864096, 96, 0 ]
+  "switches" : [ -62135575296, -59006340096, -55850666496, -52694992896, -46383559296, -43227885696, -40072212096, -33760778496, -30605104896, -27449431296, -21137997696, -17982324096, -14826650496, -12220048896, -12219962496, -12219876096, -12219789696, -12219703296, -12219616896, -12219530496, -12219444096, -12219357696, -12219271296, -2208988800 ],
+  "diffs" : [ -172704, -86304, 96, 86496, 172896, 259296, 345696, 432096, 518496, 604896, 691296, 777696, 864096, 777696, 691296, 604896, 518496, 432096, 345696, 259296, 172896, 86496, 96, 0 ]
 }, {
   "tz" : "Pacific/Gambier",
-  "switches" : [ -62135564412, -59006329212, -55850655612, -52694982012, -46383548412, -43227874812, -40072201212, -33760767612, -30605094012, -27449420412, -21137986812, -17982313212, -14826639612, -12219260412, -2208988800 ],
-  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 12, 0 ]
+  "switches" : [ -62135564412, -59006329212, -55850655612, -52694982012, -46383548412, -43227874812, -40072201212, -33760767612, -30605094012, -27449420412, -21137986812, -17982313212, -14826639612, -12220038012, -12219951612, -12219865212, -12219778812, -12219692412, -12219606012, -12219519612, -12219433212, -12219346812, -12219260412, -2208988800 ],
+  "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ]
 }, {
   "tz" : "Pacific/Guadalcanal",
-  "switches" : [ -62135635188, -59006399988, -55850726388, -52695052788, -46383619188, -43227945588, -40072271988, -33760838388, -30605164788, -27449491188, -21138057588, -17982383988, -14826710388, -12219331188, -2208988800 ],
-  "diffs" : [ -174012, -87612, -1212, 85188, 171588, 257988, 344388, 430788, 517188, 603588, 689988, 776388, 862788, -1212, 0 ]
+  "switches" : [ -62135635188, -59006399988, -55850726388, -52695052788, -46383619188, -43227945588, -40072271988, -33760838388, -30605164788, -27449491188, -21138057588, -17982383988, -14826710388, -12220108788, -12220022388, -12219935988, -12219849588, -12219763188, -12219676788, -12219590388, -12219503988, -12219417588, -12219331188, -2208988800 ],
+  "diffs" : [ -174012, -87612, -1212, 85188, 171588, 257988, 344388, 430788, 517188, 603588, 689988, 776388, 862788, 776388, 689988, 603588, 517188, 430788, 344388, 257988, 171588, 85188, -1212, 0 ]
 }, {
   "tz" : "Pacific/Guam",
-  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12219241140, -3944626740, -2208988800 ],
-  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, -87660, -1260, 0 ]
+  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12220018740, -12219932340, -12219845940, -12219759540, -12219673140, -12219586740, -12219500340, -12219413940, -12219327540, -12219241140, -3944626740, -2208988800 ],
+  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 689940, 603540, 517140, 430740, 344340, 257940, 171540, 85140, -1260, -87660, -1260, 0 ]
 }, {
   "tz" : "Pacific/Honolulu",
-  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12219254914, -2334101314, -2208988800 ],
-  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, -1886, -1800, 0 ]
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ]
 }, {
   "tz" : "Pacific/Johnston",
-  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12219254914, -2334101314, -2208988800 ],
-  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, -1886, -1800, 0 ]
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ]
 }, {
   "tz" : "Pacific/Kiritimati",
-  "switches" : [ -62135559040, -59006323840, -55850650240, -52694976640, -46383543040, -43227869440, -40072195840, -33760762240, -30605088640, -27449415040, -21137981440, -17982307840, -14826634240, -12219255040, -2208988800 ],
-  "diffs" : [ -260960, -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, -88160, 0 ]
+  "switches" : [ -62135559040, -59006323840, -55850650240, -52694976640, -46383543040, -43227869440, -40072195840, -33760762240, -30605088640, -27449415040, -21137981440, -17982307840, -14826634240, -12220032640, -12219946240, -12219859840, -12219773440, -12219687040, -12219600640, -12219514240, -12219427840, -12219341440, -12219255040, -2208988800 ],
+  "diffs" : [ -260960, -174560, -88160, -1760, 84640, 171040, 257440, 343840, 430240, 516640, 603040, 689440, 775840, 689440, 603040, 516640, 430240, 343840, 257440, 171040, 84640, -1760, -88160, 0 ]
 }, {
   "tz" : "Pacific/Kosrae",
-  "switches" : [ -62135549516, -59006314316, -55850640716, -52694967116, -46383533516, -43227859916, -40072186316, -33760752716, -30605079116, -27449405516, -21137971916, -17982298316, -14826624716, -12219245516, -3944631116, -2208988800 ],
-  "diffs" : [ -259684, -173284, -86884, -484, 85916, 172316, 258716, 345116, 431516, 517916, 604316, 690716, 777116, -86884, -484, 0 ]
+  "switches" : [ -62135549516, -59006314316, -55850640716, -52694967116, -46383533516, -43227859916, -40072186316, -33760752716, -30605079116, -27449405516, -21137971916, -17982298316, -14826624716, -12220023116, -12219936716, -12219850316, -12219763916, -12219677516, -12219591116, -12219504716, -12219418316, -12219331916, -12219245516, -3944631116, -2208988800 ],
+  "diffs" : [ -259684, -173284, -86884, -484, 85916, 172316, 258716, 345116, 431516, 517916, 604316, 690716, 777116, 690716, 604316, 517916, 431516, 345116, 258716, 172316, 85916, -484, -86884, -484, 0 ]
 }, {
   "tz" : "Pacific/Kwajalein",
-  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12219332960, -2208988800 ],
-  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, -3040, 0 ]
+  "switches" : [ -62135636960, -59006401760, -55850728160, -52695054560, -46383620960, -43227947360, -40072273760, -33760840160, -30605166560, -27449492960, -21138059360, -17982385760, -14826712160, -12220110560, -12220024160, -12219937760, -12219851360, -12219764960, -12219678560, -12219592160, -12219505760, -12219419360, -12219332960, -2208988800 ],
+  "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, 774560, 688160, 601760, 515360, 428960, 342560, 256160, 169760, 83360, -3040, 0 ]
 }, {
   "tz" : "Pacific/Majuro",
-  "switches" : [ -62135637888, -59006402688, -55850729088, -52695055488, -46383621888, -43227948288, -40072274688, -33760841088, -30605167488, -27449493888, -21138060288, -17982386688, -14826713088, -12219333888, -2208988800 ],
-  "diffs" : [ -174912, -88512, -2112, 84288, 170688, 257088, 343488, 429888, 516288, 602688, 689088, 775488, 861888, -2112, 0 ]
+  "switches" : [ -62135637888, -59006402688, -55850729088, -52695055488, -46383621888, -43227948288, -40072274688, -33760841088, -30605167488, -27449493888, -21138060288, -17982386688, -14826713088, -12220111488, -12220025088, -12219938688, -12219852288, -12219765888, -12219679488, -12219593088, -12219506688, -12219420288, -12219333888, -2208988800 ],
+  "diffs" : [ -174912, -88512, -2112, 84288, 170688, 257088, 343488, 429888, 516288, 602688, 689088, 775488, 861888, 775488, 689088, 602688, 516288, 429888, 343488, 257088, 170688, 84288, -2112, 0 ]
 }, {
   "tz" : "Pacific/Marquesas",
-  "switches" : [ -62135563320, -59006328120, -55850654520, -52694980920, -46383547320, -43227873720, -40072200120, -33760766520, -30605092920, -27449419320, -21137985720, -17982312120, -14826638520, -12219259320, -2208988800 ],
-  "diffs" : [ -172080, -85680, 720, 87120, 173520, 259920, 346320, 432720, 519120, 605520, 691920, 778320, 864720, 720, 0 ]
+  "switches" : [ -62135563320, -59006328120, -55850654520, -52694980920, -46383547320, -43227873720, -40072200120, -33760766520, -30605092920, -27449419320, -21137985720, -17982312120, -14826638520, -12220036920, -12219950520, -12219864120, -12219777720, -12219691320, -12219604920, -12219518520, -12219432120, -12219345720, -12219259320, -2208988800 ],
+  "diffs" : [ -172080, -85680, 720, 87120, 173520, 259920, 346320, 432720, 519120, 605520, 691920, 778320, 864720, 778320, 691920, 605520, 519120, 432720, 346320, 259920, 173520, 87120, 720, 0 ]
 }, {
   "tz" : "Pacific/Midway",
-  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
-  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
 }, {
   "tz" : "Pacific/Nauru",
-  "switches" : [ -62135636860, -59006401660, -55850728060, -52695054460, -46383620860, -43227947260, -40072273660, -33760840060, -30605166460, -27449492860, -21138059260, -17982385660, -14826712060, -12219332860, -2208988800 ],
-  "diffs" : [ -175940, -89540, -3140, 83260, 169660, 256060, 342460, 428860, 515260, 601660, 688060, 774460, 860860, -3140, 0 ]
+  "switches" : [ -62135636860, -59006401660, -55850728060, -52695054460, -46383620860, -43227947260, -40072273660, -33760840060, -30605166460, -27449492860, -21138059260, -17982385660, -14826712060, -12220110460, -12220024060, -12219937660, -12219851260, -12219764860, -12219678460, -12219592060, -12219505660, -12219419260, -12219332860, -2208988800 ],
+  "diffs" : [ -175940, -89540, -3140, 83260, 169660, 256060, 342460, 428860, 515260, 601660, 688060, 774460, 860860, 774460, 688060, 601660, 515260, 428860, 342460, 256060, 169660, 83260, -3140, 0 ]
 }, {
   "tz" : "Pacific/Niue",
-  "switches" : [ -62135556020, -59006320820, -55850647220, -52694973620, -46383540020, -43227866420, -40072192820, -33760759220, -30605085620, -27449412020, -21137978420, -17982304820, -14826631220, -12219252020, -2208988800 ],
-  "diffs" : [ -173980, -87580, -1180, 85220, 171620, 258020, 344420, 430820, 517220, 603620, 690020, 776420, 862820, -1180, 0 ]
+  "switches" : [ -62135556020, -59006320820, -55850647220, -52694973620, -46383540020, -43227866420, -40072192820, -33760759220, -30605085620, -27449412020, -21137978420, -17982304820, -14826631220, -12220029620, -12219943220, -12219856820, -12219770420, -12219684020, -12219597620, -12219511220, -12219424820, -12219338420, -12219252020, -2208988800 ],
+  "diffs" : [ -173980, -87580, -1180, 85220, 171620, 258020, 344420, 430820, 517220, 603620, 690020, 776420, 862820, 776420, 690020, 603620, 517220, 430820, 344420, 258020, 171620, 85220, -1180, 0 ]
 }, {
   "tz" : "Pacific/Norfolk",
-  "switches" : [ -62135637112, -59006401912, -55850728312, -52695054712, -46383621112, -43227947512, -40072273912, -33760840312, -30605166712, -27449493112, -21138059512, -17982385912, -14826712312, -12219333112, -2208988800 ],
-  "diffs" : [ -172088, -85688, 712, 87112, 173512, 259912, 346312, 432712, 519112, 605512, 691912, 778312, 864712, 712, 0 ]
+  "switches" : [ -62135637112, -59006401912, -55850728312, -52695054712, -46383621112, -43227947512, -40072273912, -33760840312, -30605166712, -27449493112, -21138059512, -17982385912, -14826712312, -12220110712, -12220024312, -12219937912, -12219851512, -12219765112, -12219678712, -12219592312, -12219505912, -12219419512, -12219333112, -2208988800 ],
+  "diffs" : [ -172088, -85688, 712, 87112, 173512, 259912, 346312, 432712, 519112, 605512, 691912, 778312, 864712, 778312, 691912, 605512, 519112, 432712, 346312, 259912, 173512, 87112, 712, 0 ]
 }, {
   "tz" : "Pacific/Noumea",
-  "switches" : [ -62135636748, -59006401548, -55850727948, -52695054348, -46383620748, -43227947148, -40072273548, -33760839948, -30605166348, -27449492748, -21138059148, -17982385548, -14826711948, -12219332748, -2208988800 ],
-  "diffs" : [ -172452, -86052, 348, 86748, 173148, 259548, 345948, 432348, 518748, 605148, 691548, 777948, 864348, 348, 0 ]
+  "switches" : [ -62135636748, -59006401548, -55850727948, -52695054348, -46383620748, -43227947148, -40072273548, -33760839948, -30605166348, -27449492748, -21138059148, -17982385548, -14826711948, -12220110348, -12220023948, -12219937548, -12219851148, -12219764748, -12219678348, -12219591948, -12219505548, -12219419148, -12219332748, -2208988800 ],
+  "diffs" : [ -172452, -86052, 348, 86748, 173148, 259548, 345948, 432348, 518748, 605148, 691548, 777948, 864348, 777948, 691548, 605148, 518748, 432348, 345948, 259548, 173148, 86748, 348, 0 ]
 }, {
   "tz" : "Pacific/Pago_Pago",
-  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
-  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
 }, {
   "tz" : "Pacific/Palau",
-  "switches" : [ -62135542676, -59006307476, -55850633876, -52694960276, -46383526676, -43227853076, -40072179476, -33760745876, -30605072276, -27449398676, -21137965076, -17982291476, -14826617876, -12219238676, -3944624276, -2208988800 ],
-  "diffs" : [ -259324, -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, -86524, -124, 0 ]
+  "switches" : [ -62135542676, -59006307476, -55850633876, -52694960276, -46383526676, -43227853076, -40072179476, -33760745876, -30605072276, -27449398676, -21137965076, -17982291476, -14826617876, -12220016276, -12219929876, -12219843476, -12219757076, -12219670676, -12219584276, -12219497876, -12219411476, -12219325076, -12219238676, -3944624276, -2208988800 ],
+  "diffs" : [ -259324, -172924, -86524, -124, 86276, 172676, 259076, 345476, 431876, 518276, 604676, 691076, 777476, 691076, 604676, 518276, 431876, 345476, 259076, 172676, 86276, -124, -86524, -124, 0 ]
 }, {
   "tz" : "Pacific/Pitcairn",
-  "switches" : [ -62135565580, -59006330380, -55850656780, -52694983180, -46383549580, -43227875980, -40072202380, -33760768780, -30605095180, -27449421580, -21137987980, -17982314380, -14826640780, -12219261580, -2208988800 ],
-  "diffs" : [ -175220, -88820, -2420, 83980, 170380, 256780, 343180, 429580, 515980, 602380, 688780, 775180, 861580, -2420, 0 ]
+  "switches" : [ -62135565580, -59006330380, -55850656780, -52694983180, -46383549580, -43227875980, -40072202380, -33760768780, -30605095180, -27449421580, -21137987980, -17982314380, -14826640780, -12220039180, -12219952780, -12219866380, -12219779980, -12219693580, -12219607180, -12219520780, -12219434380, -12219347980, -12219261580, -2208988800 ],
+  "diffs" : [ -175220, -88820, -2420, 83980, 170380, 256780, 343180, 429580, 515980, 602380, 688780, 775180, 861580, 775180, 688780, 602380, 515980, 429580, 343180, 256780, 170380, 83980, -2420, 0 ]
 }, {
   "tz" : "Pacific/Pohnpei",
-  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12219244372, -3944629972, -2208988800 ],
-  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, -88028, -1628, 0 ]
+  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12220021972, -12219935572, -12219849172, -12219762772, -12219676372, -12219589972, -12219503572, -12219417172, -12219330772, -12219244372, -3944629972, -2208988800 ],
+  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, 689572, 603172, 516772, 430372, 343972, 257572, 171172, 84772, -1628, -88028, -1628, 0 ]
 }, {
   "tz" : "Pacific/Ponape",
-  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12219244372, -3944629972, -2208988800 ],
-  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, -88028, -1628, 0 ]
+  "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12220021972, -12219935572, -12219849172, -12219762772, -12219676372, -12219589972, -12219503572, -12219417172, -12219330772, -12219244372, -3944629972, -2208988800 ],
+  "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, 689572, 603172, 516772, 430372, 343972, 257572, 171172, 84772, -1628, -88028, -1628, 0 ]
 }, {
   "tz" : "Pacific/Port_Moresby",
-  "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12219328120, -2840176120, -2366790512 ],
-  "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, -680, -688, 0 ]
+  "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ],
+  "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ]
 }, {
   "tz" : "Pacific/Rarotonga",
-  "switches" : [ -62135558456, -59006323256, -55850649656, -52694976056, -46383542456, -43227868856, -40072195256, -33760761656, -30605088056, -27449414456, -21137980856, -17982307256, -14826633656, -12219254456, -2208988800 ],
-  "diffs" : [ -175144, -88744, -2344, 84056, 170456, 256856, 343256, 429656, 516056, 602456, 688856, 775256, 861656, -2344, 0 ]
+  "switches" : [ -62135558456, -59006323256, -55850649656, -52694976056, -46383542456, -43227868856, -40072195256, -33760761656, -30605088056, -27449414456, -21137980856, -17982307256, -14826633656, -12220032056, -12219945656, -12219859256, -12219772856, -12219686456, -12219600056, -12219513656, -12219427256, -12219340856, -12219254456, -2208988800 ],
+  "diffs" : [ -175144, -88744, -2344, 84056, 170456, 256856, 343256, 429656, 516056, 602456, 688856, 775256, 861656, 775256, 688856, 602456, 516056, 429656, 343256, 256856, 170456, 84056, -2344, 0 ]
 }, {
   "tz" : "Pacific/Saipan",
-  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12219241140, -3944626740, -2208988800 ],
-  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, -87660, -1260, 0 ]
+  "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12220018740, -12219932340, -12219845940, -12219759540, -12219673140, -12219586740, -12219500340, -12219413940, -12219327540, -12219241140, -3944626740, -2208988800 ],
+  "diffs" : [ -260460, -174060, -87660, -1260, 85140, 171540, 257940, 344340, 430740, 517140, 603540, 689940, 776340, 689940, 603540, 517140, 430740, 344340, 257940, 171540, 85140, -1260, -87660, -1260, 0 ]
 }, {
   "tz" : "Pacific/Samoa",
-  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
-  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
 }, {
   "tz" : "Pacific/Tahiti",
-  "switches" : [ -62135560904, -59006325704, -55850652104, -52694978504, -46383544904, -43227871304, -40072197704, -33760764104, -30605090504, -27449416904, -21137983304, -17982309704, -14826636104, -12219256904, -2208988800 ],
-  "diffs" : [ -172696, -86296, 104, 86504, 172904, 259304, 345704, 432104, 518504, 604904, 691304, 777704, 864104, 104, 0 ]
+  "switches" : [ -62135560904, -59006325704, -55850652104, -52694978504, -46383544904, -43227871304, -40072197704, -33760764104, -30605090504, -27449416904, -21137983304, -17982309704, -14826636104, -12220034504, -12219948104, -12219861704, -12219775304, -12219688904, -12219602504, -12219516104, -12219429704, -12219343304, -12219256904, -2208988800 ],
+  "diffs" : [ -172696, -86296, 104, 86504, 172904, 259304, 345704, 432104, 518504, 604904, 691304, 777704, 864104, 777704, 691304, 604904, 518504, 432104, 345704, 259304, 172904, 86504, 104, 0 ]
 }, {
   "tz" : "Pacific/Tarawa",
-  "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12219334324, -2208988800 ],
-  "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, -1676, 0 ]
+  "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12220111924, -12220025524, -12219939124, -12219852724, -12219766324, -12219679924, -12219593524, -12219507124, -12219420724, -12219334324, -2208988800 ],
+  "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ]
 }, {
   "tz" : "Pacific/Tongatapu",
-  "switches" : [ -62135641160, -59006405960, -55850732360, -52695058760, -46383625160, -43227951560, -40072277960, -33760844360, -30605170760, -27449497160, -21138063560, -17982389960, -14826716360, -12219337160, -2208988800 ],
-  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, -2440, 0 ]
+  "switches" : [ -62135641160, -59006405960, -55850732360, -52695058760, -46383625160, -43227951560, -40072277960, -33760844360, -30605170760, -27449497160, -21138063560, -17982389960, -14826716360, -12220114760, -12220028360, -12219941960, -12219855560, -12219769160, -12219682760, -12219596360, -12219509960, -12219423560, -12219337160, -2208988800 ],
+  "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, 775160, 688760, 602360, 515960, 429560, 343160, 256760, 170360, 83960, -2440, 0 ]
 }, {
   "tz" : "Pacific/Truk",
-  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12219242828, -3944628428, -2208988800 ],
-  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, -85972, 428, 0 ]
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ]
 }, {
   "tz" : "Pacific/Wake",
-  "switches" : [ -62135636788, -59006401588, -55850727988, -52695054388, -46383620788, -43227947188, -40072273588, -33760839988, -30605166388, -27449492788, -21138059188, -17982385588, -14826711988, -12219332788, -2208988800 ],
-  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, -3212, 0 ]
+  "switches" : [ -62135636788, -59006401588, -55850727988, -52695054388, -46383620788, -43227947188, -40072273588, -33760839988, -30605166388, -27449492788, -21138059188, -17982385588, -14826711988, -12220110388, -12220023988, -12219937588, -12219851188, -12219764788, -12219678388, -12219591988, -12219505588, -12219419188, -12219332788, -2208988800 ],
+  "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, 774388, 687988, 601588, 515188, 428788, 342388, 255988, 169588, 83188, -3212, 0 ]
 }, {
   "tz" : "Pacific/Wallis",
-  "switches" : [ -62135640920, -59006405720, -55850732120, -52695058520, -46383624920, -43227951320, -40072277720, -33760844120, -30605170520, -27449496920, -21138063320, -17982389720, -14826716120, -12219336920, -2208988800 ],
-  "diffs" : [ -171880, -85480, 920, 87320, 173720, 260120, 346520, 432920, 519320, 605720, 692120, 778520, 864920, 920, 0 ]
+  "switches" : [ -62135640920, -59006405720, -55850732120, -52695058520, -46383624920, -43227951320, -40072277720, -33760844120, -30605170520, -27449496920, -21138063320, -17982389720, -14826716120, -12220114520, -12220028120, -12219941720, -12219855320, -12219768920, -12219682520, -12219596120, -12219509720, -12219423320, -12219336920, -2208988800 ],
+  "diffs" : [ -171880, -85480, 920, 87320, 173720, 260120, 346520, 432920, 519320, 605720, 692120, 778520, 864920, 778520, 692120, 605720, 519320, 432920, 346520, 260120, 173720, 87320, 920, 0 ]
 }, {
   "tz" : "Pacific/Yap",
-  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12219242828, -3944628428, -2208988800 ],
-  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, -85972, 428, 0 ]
+  "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ],
+  "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ]
 }, {
   "tz" : "Poland",
-  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12219297840, -2208988800 ],
-  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 1440, 0 ]
+  "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12220075440, -12219989040, -12219902640, -12219816240, -12219729840, -12219643440, -12219557040, -12219470640, -12219384240, -12219297840, -2208988800 ],
+  "diffs" : [ -171360, -84960, 1440, 87840, 174240, 260640, 347040, 433440, 519840, 606240, 692640, 779040, 865440, 779040, 692640, 606240, 519840, 433440, 347040, 260640, 174240, 87840, 1440, 0 ]
 }, {
   "tz" : "Portugal",
-  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12219290595, -2208988800 ],
-  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, -2205, 0 ]
+  "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12220068195, -12219981795, -12219895395, -12219808995, -12219722595, -12219636195, -12219549795, -12219463395, -12219376995, -12219290595, -2208988800 ],
+  "diffs" : [ -175005, -88605, -2205, 84195, 170595, 256995, 343395, 429795, 516195, 602595, 688995, 775395, 861795, 775395, 688995, 602595, 516195, 429795, 343395, 256995, 170595, 84195, -2205, 0 ]
 }, {
   "tz" : "ROK",
-  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12219323272, -2208988800 ],
-  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, -1928, 0 ]
+  "switches" : [ -62135627272, -59006392072, -55850718472, -52695044872, -46383611272, -43227937672, -40072264072, -33760830472, -30605156872, -27449483272, -21138049672, -17982376072, -14826702472, -12220100872, -12220014472, -12219928072, -12219841672, -12219755272, -12219668872, -12219582472, -12219496072, -12219409672, -12219323272, -2208988800 ],
+  "diffs" : [ -174728, -88328, -1928, 84472, 170872, 257272, 343672, 430072, 516472, 602872, 689272, 775672, 862072, 775672, 689272, 602872, 516472, 430072, 343672, 257272, 170872, 84472, -1928, 0 ]
 }, {
   "tz" : "Singapore",
-  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12219317725, -2208988800 ],
-  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, -3875, 0 ]
+  "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12220095325, -12220008925, -12219922525, -12219836125, -12219749725, -12219663325, -12219576925, -12219490525, -12219404125, -12219317725, -2208988800 ],
+  "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, 773725, 687325, 600925, 514525, 428125, 341725, 255325, 168925, 82525, -3875, 0 ]
 }, {
   "tz" : "SystemV/AST4",
-  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12219278400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12220056000, -12219969600, -12219883200, -12219796800, -12219710400, -12219624000, -12219537600, -12219451200, -12219364800, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/AST4ADT",
-  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12219278400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135582400, -59006347200, -55850673600, -52695000000, -46383566400, -43227892800, -40072219200, -33760785600, -30605112000, -27449438400, -21138004800, -17982331200, -14826657600, -12220056000, -12219969600, -12219883200, -12219796800, -12219710400, -12219624000, -12219537600, -12219451200, -12219364800, -12219278400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/CST6",
-  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/CST6CDT",
-  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12219271200 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135575200, -59006340000, -55850666400, -52694992800, -46383559200, -43227885600, -40072212000, -33760778400, -30605104800, -27449431200, -21137997600, -17982324000, -14826650400, -12220048800, -12219962400, -12219876000, -12219789600, -12219703200, -12219616800, -12219530400, -12219444000, -12219357600, -12219271200 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/EST5",
-  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/EST5EDT",
-  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12219274800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135578800, -59006343600, -55850670000, -52694996400, -46383562800, -43227889200, -40072215600, -33760782000, -30605108400, -27449434800, -21138001200, -17982327600, -14826654000, -12220052400, -12219966000, -12219879600, -12219793200, -12219706800, -12219620400, -12219534000, -12219447600, -12219361200, -12219274800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/HST10",
-  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12219256800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135560800, -59006325600, -55850652000, -52694978400, -46383544800, -43227871200, -40072197600, -33760764000, -30605090400, -27449416800, -21137983200, -17982309600, -14826636000, -12220034400, -12219948000, -12219861600, -12219775200, -12219688800, -12219602400, -12219516000, -12219429600, -12219343200, -12219256800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/MST7",
-  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/MST7MDT",
-  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12219267600 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135571600, -59006336400, -55850662800, -52694989200, -46383555600, -43227882000, -40072208400, -33760774800, -30605101200, -27449427600, -21137994000, -17982320400, -14826646800, -12220045200, -12219958800, -12219872400, -12219786000, -12219699600, -12219613200, -12219526800, -12219440400, -12219354000, -12219267600 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/PST8",
-  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/PST8PDT",
-  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12219264000 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135568000, -59006332800, -55850659200, -52694985600, -46383552000, -43227878400, -40072204800, -33760771200, -30605097600, -27449424000, -21137990400, -17982316800, -14826643200, -12220041600, -12219955200, -12219868800, -12219782400, -12219696000, -12219609600, -12219523200, -12219436800, -12219350400, -12219264000 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/YST9",
-  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12219260400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12220038000, -12219951600, -12219865200, -12219778800, -12219692400, -12219606000, -12219519600, -12219433200, -12219346800, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "SystemV/YST9YDT",
-  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12219260400 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135564400, -59006329200, -55850655600, -52694982000, -46383548400, -43227874800, -40072201200, -33760767600, -30605094000, -27449420400, -21137986800, -17982313200, -14826639600, -12220038000, -12219951600, -12219865200, -12219778800, -12219692400, -12219606000, -12219519600, -12219433200, -12219346800, -12219260400 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Turkey",
-  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12219299752, -2840147752, -2208988800 ],
-  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, -3848, -3784, 0 ]
+  "switches" : [ -62135603752, -59006368552, -55850694952, -52695021352, -46383587752, -43227914152, -40072240552, -33760806952, -30605133352, -27449459752, -21138026152, -17982352552, -14826678952, -12220077352, -12219990952, -12219904552, -12219818152, -12219731752, -12219645352, -12219558952, -12219472552, -12219386152, -12219299752, -2840147752, -2208988800 ],
+  "diffs" : [ -176648, -90248, -3848, 82552, 168952, 255352, 341752, 428152, 514552, 600952, 687352, 773752, 860152, 773752, 687352, 600952, 514552, 428152, 341752, 255352, 168952, 82552, -3848, -3784, 0 ]
 }, {
   "tz" : "UCT",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "US/Alaska",
-  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12219343224, -3225223727, -2208988800 ],
-  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 82824, -3576, 0 ]
+  "switches" : [ -62135647224, -59006412024, -55850738424, -52695064824, -46383631224, -43227957624, -40072284024, -33760850424, -30605176824, -27449503224, -21138069624, -17982396024, -14826722424, -12220120824, -12220034424, -12219948024, -12219861624, -12219775224, -12219688824, -12219602424, -12219516024, -12219429624, -12219343224, -3225223727, -2208988800 ],
+  "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 860424, 774024, 687624, 601224, 514824, 428424, 342024, 255624, 169224, 82824, -3576, 0 ]
 }, {
   "tz" : "US/Aleutian",
-  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12219336802, -3225223727, -2208988800 ],
-  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 80002, -6398, 0 ]
+  "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ],
+  "diffs" : [ -92798, -6398, 80002, 166402, 252802, 339202, 425602, 512002, 598402, 684802, 771202, 857602, 944002, 857602, 771202, 684802, 598402, 512002, 425602, 339202, 252802, 166402, 80002, -6398, 0 ]
 }, {
   "tz" : "US/Arizona",
-  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12219265902, -2717643600 ],
-  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, -1698, 0 ]
+  "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12220043502, -12219957102, -12219870702, -12219784302, -12219697902, -12219611502, -12219525102, -12219438702, -12219352302, -12219265902, -2717643600 ],
+  "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, 775902, 689502, 603102, 516702, 430302, 343902, 257502, 171102, 84702, -1698, 0 ]
 }, {
   "tz" : "US/Central",
-  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12219271764, -2717647200 ],
-  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 564, 0 ]
+  "switches" : [ -62135575764, -59006340564, -55850666964, -52694993364, -46383559764, -43227886164, -40072212564, -33760778964, -30605105364, -27449431764, -21137998164, -17982324564, -14826650964, -12220049364, -12219962964, -12219876564, -12219790164, -12219703764, -12219617364, -12219530964, -12219444564, -12219358164, -12219271764, -2717647200 ],
+  "diffs" : [ -172236, -85836, 564, 86964, 173364, 259764, 346164, 432564, 518964, 605364, 691764, 778164, 864564, 778164, 691764, 605364, 518964, 432564, 346164, 259764, 173364, 86964, 564, 0 ]
 }, {
   "tz" : "US/East-Indiana",
-  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12219272122, -2717647200, -2208988800 ],
-  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, -2678, -3600, 0 ]
+  "switches" : [ -62135576122, -59006340922, -55850667322, -52694993722, -46383560122, -43227886522, -40072212922, -33760779322, -30605105722, -27449432122, -21137998522, -17982324922, -14826651322, -12220049722, -12219963322, -12219876922, -12219790522, -12219704122, -12219617722, -12219531322, -12219444922, -12219358522, -12219272122, -2717647200, -2208988800 ],
+  "diffs" : [ -175478, -89078, -2678, 83722, 170122, 256522, 342922, 429322, 515722, 602122, 688522, 774922, 861322, 774922, 688522, 602122, 515722, 429322, 342922, 256522, 170122, 83722, -2678, -3600, 0 ]
 }, {
   "tz" : "US/Eastern",
-  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12219275038, -2717650800 ],
-  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 238, 0 ]
+  "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12220052638, -12219966238, -12219879838, -12219793438, -12219707038, -12219620638, -12219534238, -12219447838, -12219361438, -12219275038, -2717650800 ],
+  "diffs" : [ -172562, -86162, 238, 86638, 173038, 259438, 345838, 432238, 518638, 605038, 691438, 777838, 864238, 777838, 691438, 605038, 518638, 432238, 345838, 259438, 173038, 86638, 238, 0 ]
 }, {
   "tz" : "US/Hawaii",
-  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12219254914, -2334101314, -2208988800 ],
-  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, -1886, -1800, 0 ]
+  "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ],
+  "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ]
 }, {
   "tz" : "US/Indiana-Starke",
-  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12219272010, -2717647200 ],
-  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 810, 0 ]
+  "switches" : [ -62135576010, -59006340810, -55850667210, -52694993610, -46383560010, -43227886410, -40072212810, -33760779210, -30605105610, -27449432010, -21137998410, -17982324810, -14826651210, -12220049610, -12219963210, -12219876810, -12219790410, -12219704010, -12219617610, -12219531210, -12219444810, -12219358410, -12219272010, -2717647200 ],
+  "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ]
 }, {
   "tz" : "US/Michigan",
-  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12219272869, -2208988800 ],
-  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, -1931, 0 ]
+  "switches" : [ -62135576869, -59006341669, -55850668069, -52694994469, -46383560869, -43227887269, -40072213669, -33760780069, -30605106469, -27449432869, -21137999269, -17982325669, -14826652069, -12220050469, -12219964069, -12219877669, -12219791269, -12219704869, -12219618469, -12219532069, -12219445669, -12219359269, -12219272869, -2208988800 ],
+  "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, 775669, 689269, 602869, 516469, 430069, 343669, 257269, 170869, 84469, -1931, 0 ]
 }, {
   "tz" : "US/Mountain",
-  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12219267604, -2717643600 ],
-  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 4, 0 ]
+  "switches" : [ -62135571604, -59006336404, -55850662804, -52694989204, -46383555604, -43227882004, -40072208404, -33760774804, -30605101204, -27449427604, -21137994004, -17982320404, -14826646804, -12220045204, -12219958804, -12219872404, -12219786004, -12219699604, -12219613204, -12219526804, -12219440404, -12219354004, -12219267604, -2717643600 ],
+  "diffs" : [ -172796, -86396, 4, 86404, 172804, 259204, 345604, 432004, 518404, 604804, 691204, 777604, 864004, 777604, 691204, 604804, 518404, 432004, 345604, 259204, 172804, 86404, 4, 0 ]
 }, {
   "tz" : "US/Pacific",
-  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12219264422, -2717640000 ],
-  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 422, 0 ]
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ]
 }, {
   "tz" : "US/Pacific-New",
-  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12219264422, -2717640000 ],
-  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 422, 0 ]
+  "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ],
+  "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ]
 }, {
   "tz" : "US/Samoa",
-  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12219338232, -2445424632, -2208988800 ],
-  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 85032, -1368, 0 ]
+  "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ],
+  "diffs" : [ -87768, -1368, 85032, 171432, 257832, 344232, 430632, 517032, 603432, 689832, 776232, 862632, 949032, 862632, 776232, 689832, 603432, 517032, 430632, 344232, 257832, 171432, 85032, -1368, 0 ]
 }, {
   "tz" : "UTC",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Universal",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "W-SU",
-  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12219301817, -2208988800 ],
-  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, -1783, 0 ]
+  "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12220079417, -12219993017, -12219906617, -12219820217, -12219733817, -12219647417, -12219561017, -12219474617, -12219388217, -12219301817, -2208988800 ],
+  "diffs" : [ -174583, -88183, -1783, 84617, 171017, 257417, 343817, 430217, 516617, 603017, 689417, 775817, 862217, 775817, 689417, 603017, 516617, 430217, 343817, 257417, 171017, 84617, -1783, 0 ]
 }, {
   "tz" : "WET",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 }, {
   "tz" : "Zulu",
-  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12219292800 ],
-  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 0 ]
+  "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ],
+  "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ]
 } ]
\ No newline at end of file
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 6338a595a9b87..95c20a3c6d9d4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.time.{LocalDate, LocalDateTime, ZoneId}
+import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId}
 import java.time.temporal.ChronoField
 import java.util.{Calendar, TimeZone}
 
@@ -148,8 +148,8 @@ object RebaseDateTime {
   // in Proleptic Gregorian calendar.
   private final val gregorianCommonEraStartDay = gregJulianDiffSwitchDay(0)
 
-  private final val gregorianStartDay = LocalDate.of(1582, 10, 15)
-  private final val julianEndDay = LocalDate.of(1582, 10, 4)
+  private final val gregorianStartDate = LocalDate.of(1582, 10, 15)
+  private final val julianEndDate = LocalDate.of(1582, 10, 4)
 
   /**
    * Converts the given number of days since the epoch day 1970-01-01 to a local date in Proleptic
@@ -171,8 +171,8 @@ object RebaseDateTime {
    */
   private[sql] def localRebaseGregorianToJulianDays(days: Int): Int = {
     var localDate = LocalDate.ofEpochDay(days)
-    if (localDate.isAfter(julianEndDay) && localDate.isBefore(gregorianStartDay)) {
-      localDate = gregorianStartDay
+    if (localDate.isAfter(julianEndDate) && localDate.isBefore(gregorianStartDate)) {
+      localDate = gregorianStartDate
     }
     val utcCal = new Calendar.Builder()
       // `gregory` is a hybrid calendar that supports both
@@ -284,6 +284,11 @@ object RebaseDateTime {
    */
   private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json")
 
+  private final val gregorianStartTs = LocalDateTime.of(gregorianStartDate, LocalTime.MIDNIGHT)
+  private final val julianEndTs = LocalDateTime.of(
+    julianEndDate,
+    LocalTime.of(23, 59, 59, 999999999))
+
   /**
    * Converts the given number of microseconds since the epoch '1970-01-01T00:00:00Z', to a local
    * date-time in Proleptic Gregorian calendar with timezone identified by `zoneId`, interprets the
@@ -306,7 +311,10 @@ object RebaseDateTime {
    */
   private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
     val instant = microsToInstant(micros)
-    val ldt = instant.atZone(zoneId).toLocalDateTime
+    var ldt = instant.atZone(zoneId).toLocalDateTime
+    if (ldt.isAfter(julianEndTs) && ldt.isBefore(gregorianStartTs)) {
+      ldt = LocalDateTime.of(gregorianStartDate, ldt.toLocalTime)
+    }
     val cal = new Calendar.Builder()
       // `gregory` is a hybrid calendar that supports both
       // the Julian and Gregorian calendar systems
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index 73ca7614e193e..82ccd97721656 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -373,11 +373,11 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
           "1582-10-05" -> "1582-10-15", "1582-10-06" -> "1582-10-15", "1582-10-07" -> "1582-10-15",
           "1582-10-08" -> "1582-10-15", "1582-10-09" -> "1582-10-15", "1582-10-11" -> "1582-10-15",
           "1582-10-12" -> "1582-10-15", "1582-10-13" -> "1582-10-15", "1582-10-14" -> "1582-10-15",
-          "1582-10-15" -> "1582-10-15").foreach { case (hybridDate, gregDate) =>
-          withClue(s"tz = ${zid.getId} hybrid date = $hybridDate greg date = $gregDate") {
-            val date = Date.valueOf(gregDate)
+          "1582-10-15" -> "1582-10-15").foreach { case (gregDate, hybridDate) =>
+          withClue(s"tz = ${zid.getId} greg date = $gregDate hybrid date = $hybridDate ") {
+            val date = Date.valueOf(hybridDate)
             val hybridDays = fromJavaDateLegacy(date)
-            val gregorianDays = localDateToDays(LocalDate.parse(hybridDate))
+            val gregorianDays = localDateToDays(LocalDate.parse(gregDate))
 
             assert(localRebaseGregorianToJulianDays(gregorianDays) === hybridDays)
             assert(rebaseGregorianToJulianDays(gregorianDays) === hybridDays)
@@ -386,4 +386,33 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
   }
+
+  test("rebase not-existed timestamps in the hybrid calendar") {
+    outstandingZoneIds.foreach { zid =>
+      withDefaultTimeZone(zid) {
+        Seq(
+          "1582-10-04T23:59:59.999999" -> "1582-10-04 23:59:59.999999",
+          "1582-10-05T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
+          "1582-10-06T01:02:03.000001" -> "1582-10-15 01:02:03.000001",
+          "1582-10-07T00:00:00.000000" -> "1582-10-15 00:00:00.000000",
+          "1582-10-08T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
+          "1582-10-09T23:59:59.001001" -> "1582-10-15 23:59:59.001001",
+          "1582-10-10T00:11:22.334455" -> "1582-10-15 00:11:22.334455",
+          "1582-10-11T11:12:13.111111" -> "1582-10-15 11:12:13.111111",
+          "1582-10-12T10:11:12.131415" -> "1582-10-15 10:11:12.131415",
+          "1582-10-13T00:00:00.123321" -> "1582-10-15 00:00:00.123321",
+          "1582-10-14T23:59:59.999999" -> "1582-10-15 23:59:59.999999",
+          "1582-10-15T00:00:00.000000" -> "1582-10-15 00:00:00.000000"
+        ).foreach { case (gregTs, hybridTs) =>
+          withClue(s"tz = ${zid.getId} greg ts = $gregTs hybrid ts = $hybridTs") {
+            val hybridMicros = parseToJulianMicros(hybridTs)
+            val gregorianMicros = parseToGregMicros(gregTs, zid)
+
+            assert(rebaseGregorianToJulianMicros(zid, gregorianMicros) === hybridMicros)
+            assert(rebaseGregorianToJulianMicros(gregorianMicros) === hybridMicros)
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index f0ce6d503b687..806f0d49bcf82 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -521,10 +521,10 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
     }
   }
 
-  test("SPARK-31284: rebasing timestamps in write") {
+  test("SPARK-31284, SPARK-31423: rebasing timestamps in write") {
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      Seq("1001-01-01 01:02:03.123456").toDF("tsS")
+      Seq("1001-01-01 01:02:03.123456", "1582-10-10 11:12:13.654321").toDF("tsS")
         .select($"tsS".cast("timestamp").as("ts"))
         .write
         .orc(path)
@@ -533,7 +533,9 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
         withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
           checkAnswer(
             spark.read.orc(path),
-            Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+            Seq(
+              Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")),
+              Row(java.sql.Timestamp.valueOf("1582-10-15 11:12:13.654321"))))
         }
       }
     }

From 020f3a33dd711d05337bb42d5f65708a4aec2daa Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Thu, 16 Apr 2020 17:52:22 +0900
Subject: [PATCH 0506/1280] [SPARK-30564][SQL] Revert Block.length and use
 comment placeholders in HashAggregateExec
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
SPARK-21870 (cb0cddf#diff-06dc5de6163687b7810aa76e7e152a76R146-R149) caused significant performance regression in cases where the source code size is fairly large as `HashAggregateExec` uses `Block.length` to decide on splitting the code. The change in `length` makes sense as the comment and extra new lines shouldn't be taken into account when deciding on splitting, but the regular expression based approach is very slow and adds a big relative overhead to cases where the execution is quick (small number of rows).
This PR:
- restores `Block.length` to its original form
- places comments in `HashAggragateExec` with `CodegenContext.registerComment` so as to appear only when comments are enabled (`spark.sql.codegen.comments=true`)

Before this PR:
```
deeply nested struct field r/w:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
250 deep x 400 rows (read in-mem)                  1137           1143           8          0.1       11368.3       0.0X
```

After this PR:
```
deeply nested struct field r/w:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
250 deep x 400 rows (read in-mem)                   167            180           7          0.6        1674.3       0.1X
```
### Why are the changes needed?
To fix performance regression.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing UTs.

Closes #28083 from peter-toth/SPARK-30564-use-comment-placeholders.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>

(cherry picked from commit 7ad6ba36f28b7a5ca548950dec6afcd61e5d68b9)

Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../catalyst/expressions/codegen/javaCode.scala    |  8 ++++----
 .../execution/aggregate/HashAggregateExec.scala    | 14 +++++++-------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
index dff258902a0b1..1c59c3c2ad667 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/javaCode.scala
@@ -143,10 +143,10 @@ trait Block extends TreeNode[Block] with JavaCode {
     case _ => code.trim
   }
 
-  def length: Int = {
-    // Returns a code length without comments
-    CodeFormatter.stripExtraNewLinesAndComments(toString).length
-  }
+  // We could remove comments, extra whitespaces and newlines when calculating length as it is used
+  // only for codegen method splitting, but SPARK-30564 showed that this is a performance critical
+  // function so we decided not to do so.
+  def length: Int = toString.length
 
   def isEmpty: Boolean = toString.isEmpty
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index 7a26fd7a8541a..87a408174be92 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -367,10 +367,10 @@ case class HashAggregateExec(
          """.stripMargin
       }
       code"""
-         |// do aggregate for ${aggNames(i)}
-         |// evaluate aggregate function
+         |${ctx.registerComment(s"do aggregate for ${aggNames(i)}")}
+         |${ctx.registerComment("evaluate aggregate function")}
          |${evaluateVariables(bufferEvalsForOneFunc)}
-         |// update aggregation buffers
+         |${ctx.registerComment("update aggregation buffers")}
          |${updates.mkString("\n").trim}
        """.stripMargin
     }
@@ -975,9 +975,9 @@ case class HashAggregateExec(
           CodeGenerator.updateColumn(unsafeRowBuffer, dt, bufferOffset + j, ev, nullable)
         }
         code"""
-           |// evaluate aggregate function for ${aggNames(i)}
+           |${ctx.registerComment(s"evaluate aggregate function for ${aggNames(i)}")}
            |${evaluateVariables(rowBufferEvalsForOneFunc)}
-           |// update unsafe row buffer
+           |${ctx.registerComment("update unsafe row buffer")}
            |${updateRowBuffers.mkString("\n").trim}
          """.stripMargin
       }
@@ -1030,9 +1030,9 @@ case class HashAggregateExec(
                 isVectorized = true)
             }
             code"""
-               |// evaluate aggregate function for ${aggNames(i)}
+               |${ctx.registerComment(s"evaluate aggregate function for ${aggNames(i)}")}
                |${evaluateVariables(fastRowEvalsForOneFunc)}
-               |// update fast row
+               |${ctx.registerComment("update fast row")}
                |${updateRowBuffer.mkString("\n").trim}
              """.stripMargin
           }

From bc6f673da4b36616614c23d8fa69a8e420b966c2 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 17 Apr 2020 02:31:08 +0900
Subject: [PATCH 0507/1280] [SPARK-31420][WEBUI][FOLLOWUP] Make locale of
 timeline-view 'en'
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

<!--
Thanks for sending a pull request!  Here are some tips for you:
  1. If this is your first time, please read our contributor guidelines: https://spark.apache.org/contributing.html
  2. Ensure you have added or run the appropriate tests for your PR: https://spark.apache.org/developer-tools.html
  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][SPARK-XXXX] Your PR title ...'.
  4. Be sure to keep the PR description updated to reflect all changes.
  5. Please write your PR title to summarize what this PR proposes.
  6. If possible, provide a concise example to reproduce the issue for a faster review.
  7. If you want to add a new configuration, please read the guideline first for naming configurations in
     'core/src/main/scala/org/apache/spark/internal/config/ConfigEntry.scala'.
-->

### What changes were proposed in this pull request?
<!--
Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
  1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
  2. If you fix some SQL features, you can provide some references of other DBMSes.
  3. If there is design documentation, please add the link.
  4. If there is a discussion in the mailing list, please add the link.
-->
This change explicitly set locale of timeline view to 'en' to be the same appearance as before upgrading vis-timeline.

### Why are the changes needed?
<!--
Please clarify why the changes are needed. For instance,
  1. If you propose a new API, clarify the use case for a new API.
  2. If you fix a bug, you can clarify why it is a bug.
-->
We upgraded vis-timeline in #28192 and the upgraded version is different from before we used in the notation of dates.
The notation seems to be dependent on locale. The following is appearance in my Japanese environment.
<img width="557" alt="locale-changed" src="https://user-images.githubusercontent.com/4736016/79265314-de886700-7ed0-11ea-8641-fa76b993c0d9.png">

Although the notation is in Japanese, the default format is a little bit unnatural (e.g. 4月9日 05:39 is natural rather than 9 四月 05:39).

I found we can get the same appearance as before by explicitly set locale to 'en'.

### Does this PR introduce any user-facing change?
<!--
If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
If no, write 'No'.
-->
No.

### How was this patch tested?
<!--
If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
If tests were not added, please describe why they were not added and/or why it was difficult to add.
-->
I visited JobsPage, JobPage and StagePage and confirm that timeline view shows dates with 'en' locale.
<img width="735" alt="fix-date-appearance" src="https://user-images.githubusercontent.com/4736016/79267107-8bfc7a00-7ed3-11ea-8a25-f6681d04a83c.png">

NOTE: #28192 will be backported to branch-2.4 and branch-3.0 so this PR should be follow #28214 and #28213 .

Closes #28218 from sarutak/fix-locale-issue.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
(cherry picked from commit df27350142d81a3e8941939870bfc0ab50e37a43)
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 .../main/resources/org/apache/spark/ui/static/timeline-view.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
index a63ee8696f906..5be8cffd1f8db 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -28,6 +28,7 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
     showCurrentTime: false,
     start: startTime,
     zoomable: false,
+    locale: "en",
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }
@@ -110,6 +111,7 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
     showCurrentTime: false,
     start: startTime,
     zoomable: false,
+    locale: "en",
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }
@@ -194,6 +196,7 @@ function drawTaskAssignmentTimeline(groupArray, eventObjArray, minLaunchTime, ma
     start: minLaunchTime,
     end: maxFinishTime,
     zoomable: false,
+    locale: "en",
     moment: function (date) {
       return vis.moment(date).utcOffset(offset);
     }

From 9690d9feb090f6b72fc36d120f1baf3dc3de367e Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Thu, 16 Apr 2020 12:54:10 -0700
Subject: [PATCH 0508/1280] [SPARK-31462][INFRA] The usage of getopts and case
 statement is wrong in do-release.sh

### What changes were proposed in this pull request?

This PR (SPARK-31462) fixes the usage of getopts and case statement in `do-release.sh` and `do-release-docker.sh`.

### Why are the changes needed?

In the current master, do-release.sh contains the following code.
```
while getopts "bn" opt; do
  case $opt in
    b) GIT_BRANCH=$OPTARG ;;
    n) DRY_RUN=1 ;;
    ?) error "Invalid option: $OPTARG" ;;
  esac
done
```
There are 3 wrong usage in getopts and case statement.
1. To set  $OPTARG to an argument passed for the option "b", the parameter for getopts should be "b:".
2. To set $OPTARG to the invalid option name passed, the parameter for getopts starts with ":".
3. It's minor but to match the character "?", it's better to escape like "\\?".

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

I checked that $GIT_BRANCH is set when do-release.sh is launched with -b option.
I also checked that the error message contains invalid option name when do-release.sh is launched with an invalid option.

Closes #28234 from sarutak/fix-do-release.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 2921be7a4e17b45591c25873a55fb286599b0fd7)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/create-release/do-release-docker.sh | 4 ++--
 dev/create-release/do-release.sh        | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index 4a003d7e07de2..2f794c0e0a174 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -54,7 +54,7 @@ WORKDIR=
 IMGTAG=latest
 JAVA=
 RELEASE_STEP=
-while getopts "d:hj:ns:t:" opt; do
+while getopts ":d:hj:ns:t:" opt; do
   case $opt in
     d) WORKDIR="$OPTARG" ;;
     n) DRY_RUN=1 ;;
@@ -62,7 +62,7 @@ while getopts "d:hj:ns:t:" opt; do
     j) JAVA="$OPTARG" ;;
     s) RELEASE_STEP="$OPTARG" ;;
     h) usage ;;
-    ?) error "Invalid option. Run with -h for help." ;;
+    \?) error "Invalid option. Run with -h for help." ;;
   esac
 done
 
diff --git a/dev/create-release/do-release.sh b/dev/create-release/do-release.sh
index f1d4f3ab5ddec..4f18a55a3bceb 100755
--- a/dev/create-release/do-release.sh
+++ b/dev/create-release/do-release.sh
@@ -20,11 +20,11 @@
 SELF=$(cd $(dirname $0) && pwd)
 . "$SELF/release-util.sh"
 
-while getopts "bn" opt; do
+while getopts ":b:n" opt; do
   case $opt in
     b) GIT_BRANCH=$OPTARG ;;
     n) DRY_RUN=1 ;;
-    ?) error "Invalid option: $OPTARG" ;;
+    \?) error "Invalid option: $OPTARG" ;;
   esac
 done
 

From e7fef70fbbea08a38316abdaa9445123bb8c39e2 Mon Sep 17 00:00:00 2001
From: herman <herman@databricks.com>
Date: Thu, 16 Apr 2020 18:47:46 -0700
Subject: [PATCH 0509/1280] [SPARK-31450][SQL] Make ExpressionEncoder
 thread-safe

### What changes were proposed in this pull request?
This PR moves the `ExpressionEncoder.toRow` and `ExpressionEncoder.fromRow` functions into their own function objects(`ExpressionEncoder.Serializer` & `ExpressionEncoder.Deserializer`). This effectively makes the `ExpressionEncoder` stateless, thread-safe and (more) reusable. The function objects are not thread safe, however they are documented as such and should be used in a more limited scope (making it easier to reason about thread safety).

### Why are the changes needed?
ExpressionEncoders are not thread-safe. We had various (nasty) bugs because of this.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Existing tests.

Closes #28223 from hvanhovell/SPARK-31450.

Authored-by: herman <herman@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit fab4ca5156d5e1cc0e976c7c27b28a12fa61eb6d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../ml/source/image/ImageFileFormat.scala     |   4 +-
 .../ml/source/libsvm/LibSVMRelation.scala     |   4 +-
 .../linalg/UDTSerializationBenchmark.scala    |  10 +-
 .../scala/org/apache/spark/sql/Encoder.scala  |   9 +-
 .../catalyst/encoders/ExpressionEncoder.scala | 102 ++++++++++++------
 .../sql/catalyst/expressions/ScalaUDF.scala   |   4 +-
 .../org/apache/spark/sql/HashBenchmark.scala  |   4 +-
 .../spark/sql/UnsafeProjectionBenchmark.scala |   4 +-
 .../encoders/EncoderResolutionSuite.scala     |  28 +++--
 .../encoders/ExpressionEncoderSuite.scala     |  22 ++--
 .../catalyst/encoders/RowEncoderSuite.scala   |  49 +++++----
 .../expressions/HashExpressionsSuite.scala    |   6 +-
 .../expressions/ObjectExpressionsSuite.scala  |   4 +-
 .../GenerateUnsafeRowJoinerSuite.scala        |   4 +-
 .../util/ArrayDataIndexedSeqSuite.scala       |   4 +-
 .../sql/catalyst/util/UnsafeArraySuite.scala  |  74 +++++--------
 .../scala/org/apache/spark/sql/Dataset.scala  |  13 ++-
 .../org/apache/spark/sql/SparkSession.scala   |   9 +-
 .../spark/sql/execution/SparkStrategies.scala |   3 +-
 .../spark/sql/execution/aggregate/udaf.scala  |  15 +--
 .../datasources/DataSourceStrategy.scala      |   4 +-
 .../datasources/jdbc/JdbcUtils.scala          |   4 +-
 .../v2/DescribeNamespaceExec.scala            |   6 +-
 .../datasources/v2/DescribeTableExec.scala    |   6 +-
 .../v2/ShowCurrentNamespaceExec.scala         |  11 +-
 .../datasources/v2/ShowNamespacesExec.scala   |   6 +-
 .../v2/ShowTablePropertiesExec.scala          |   6 +-
 .../datasources/v2/ShowTablesExec.scala       |  12 +--
 .../ContinuousTextSocketSource.scala          |   3 +-
 .../sql/execution/streaming/memory.scala      |   8 +-
 .../sources/ContinuousMemoryStream.scala      |   2 +-
 .../streaming/sources/ForeachBatchSink.scala  |   3 +-
 .../sources/ForeachWriterTable.scala          |   2 +-
 .../execution/streaming/sources/memory.scala  |   4 +-
 .../spark/sql/internal/CatalogImpl.scala      |   3 +-
 .../sql/execution/GroupedIteratorSuite.scala  |  15 ++-
 .../benchmark/UnsafeArrayDataBenchmark.scala  |  29 ++---
 .../binaryfile/BinaryFileFormatSuite.scala    |   2 +-
 .../spark/sql/streaming/StreamTest.scala      |  22 ++--
 39 files changed, 282 insertions(+), 238 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
index c3321447e3c96..4944e0c061091 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/image/ImageFileFormat.scala
@@ -91,8 +91,8 @@ private[image] class ImageFileFormat extends FileFormat with DataSourceRegister
         if (requiredSchema.isEmpty) {
           filteredResult.map(_ => emptyUnsafeRow)
         } else {
-          val converter = RowEncoder(requiredSchema)
-          filteredResult.map(row => converter.toRow(row))
+          val toRow = RowEncoder(requiredSchema).createSerializer()
+          filteredResult.map(row => toRow(row))
         }
       }
     }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
index 6ead4df87fb54..da8f3a24ff27e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
@@ -166,7 +166,7 @@ private[libsvm] class LibSVMFileFormat
             LabeledPoint(label, Vectors.sparse(numFeatures, indices, values))
           }
 
-      val converter = RowEncoder(dataSchema)
+      val toRow = RowEncoder(dataSchema).createSerializer()
       val fullOutput = dataSchema.map { f =>
         AttributeReference(f.name, f.dataType, f.nullable, f.metadata)()
       }
@@ -178,7 +178,7 @@ private[libsvm] class LibSVMFileFormat
 
       points.map { pt =>
         val features = if (isSparse) pt.features.toSparse else pt.features.toDense
-        requiredColumns(converter.toRow(Row(pt.label, features)))
+        requiredColumns(toRow(Row(pt.label, features)))
       }
     }
   }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
index 5f19e466ecad0..3caa8f6d5b1e5 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/UDTSerializationBenchmark.scala
@@ -38,12 +38,14 @@ object UDTSerializationBenchmark extends BenchmarkBase {
       val iters = 1e2.toInt
       val numRows = 1e3.toInt
 
-      val encoder = ExpressionEncoder[Vector].resolveAndBind()
+      val encoder = ExpressionEncoder[Vector]().resolveAndBind()
+      val toRow = encoder.createSerializer()
+      val fromRow = encoder.createDeserializer()
 
       val vectors = (1 to numRows).map { i =>
         Vectors.dense(Array.fill(1e5.toInt)(1.0 * i))
       }.toArray
-      val rows = vectors.map(encoder.toRow)
+      val rows = vectors.map(toRow)
 
       val benchmark = new Benchmark("VectorUDT de/serialization", numRows, iters, output = output)
 
@@ -51,7 +53,7 @@ object UDTSerializationBenchmark extends BenchmarkBase {
         var sum = 0
         var i = 0
         while (i < numRows) {
-          sum += encoder.toRow(vectors(i)).numFields
+          sum += toRow(vectors(i)).numFields
           i += 1
         }
       }
@@ -60,7 +62,7 @@ object UDTSerializationBenchmark extends BenchmarkBase {
         var sum = 0
         var i = 0
         while (i < numRows) {
-          sum += encoder.fromRow(rows(i)).numActives
+          sum += fromRow(rows(i)).numActives
           i += 1
         }
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
index c43a86ad48ec9..ea760d80541c8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
@@ -58,8 +58,7 @@ import org.apache.spark.sql.types._
  * }}}
  *
  * == Implementation ==
- *  - Encoders are not required to be thread-safe and thus they do not need to use locks to guard
- *    against concurrent access if they reuse internal buffers to improve performance.
+ *  - Encoders should be thread-safe.
  *
  * @since 1.6.0
  */
@@ -76,10 +75,4 @@ trait Encoder[T] extends Serializable {
    * A ClassTag that can be used to construct an Array to contain a collection of `T`.
    */
   def clsTag: ClassTag[T]
-
-  /**
-   * Create a copied [[Encoder]]. The implementation may just copy internal reusable fields to speed
-   * up the [[Encoder]] creation.
-   */
-  def makeCopy: Encoder[T]
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
index bd499671d6441..213fbcc989e7c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala
@@ -17,12 +17,15 @@
 
 package org.apache.spark.sql.catalyst.encoders
 
+import java.io.ObjectInputStream
+
 import scala.reflect.ClassTag
 import scala.reflect.runtime.universe.{typeTag, TypeTag}
 
 import org.apache.spark.sql.Encoder
 import org.apache.spark.sql.catalyst.{InternalRow, JavaTypeInference, ScalaReflection}
 import org.apache.spark.sql.catalyst.analysis.{Analyzer, GetColumnByOrdinal, SimpleAnalyzer, UnresolvedAttribute, UnresolvedExtractValue}
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.{Deserializer, Serializer}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.expressions.objects.{AssertNotNull, InitializeJavaBean, Invoke, NewInstance}
@@ -162,6 +165,56 @@ object ExpressionEncoder {
       e4: ExpressionEncoder[T4],
       e5: ExpressionEncoder[T5]): ExpressionEncoder[(T1, T2, T3, T4, T5)] =
     tuple(Seq(e1, e2, e3, e4, e5)).asInstanceOf[ExpressionEncoder[(T1, T2, T3, T4, T5)]]
+
+  private val anyObjectType = ObjectType(classOf[Any])
+
+  /**
+   * Function that deserializes an [[InternalRow]] into an object of type `T`. This class is not
+   * thread-safe.
+   */
+  class Deserializer[T](private val expressions: Seq[Expression])
+    extends (InternalRow => T) with Serializable {
+    @transient
+    private[this] var constructProjection: Projection = _
+
+    override def apply(row: InternalRow): T = try {
+      if (constructProjection == null) {
+        constructProjection = SafeProjection.create(expressions)
+      }
+      constructProjection(row).get(0, anyObjectType).asInstanceOf[T]
+    } catch {
+      case e: Exception =>
+        throw new RuntimeException(s"Error while decoding: $e\n" +
+          s"${expressions.map(_.simpleString(SQLConf.get.maxToStringFields)).mkString("\n")}", e)
+    }
+  }
+
+  /**
+   * Function that serializesa an object of type `T` to an [[InternalRow]]. This class is not
+   * thread-safe. Note that multiple calls to `apply(..)` return the same actual [[InternalRow]]
+   * object.  Thus, the caller should copy the result before making another call if required.
+   */
+  class Serializer[T](private val expressions: Seq[Expression])
+    extends (T => InternalRow) with Serializable {
+    @transient
+    private[this] var inputRow: GenericInternalRow = _
+
+    @transient
+    private[this] var extractProjection: UnsafeProjection = _
+
+    override def apply(t: T): InternalRow = try {
+      if (extractProjection == null) {
+        inputRow = new GenericInternalRow(1)
+        extractProjection = GenerateUnsafeProjection.generate(expressions)
+      }
+      inputRow(0) = t
+      extractProjection(inputRow)
+    } catch {
+      case e: Exception =>
+        throw new RuntimeException(s"Error while encoding: $e\n" +
+          s"${expressions.map(_.simpleString(SQLConf.get.maxToStringFields)).mkString("\n")}", e)
+    }
+  }
 }
 
 /**
@@ -301,25 +354,22 @@ case class ExpressionEncoder[T](
   }
 
   @transient
-  private lazy val extractProjection = GenerateUnsafeProjection.generate({
+  private lazy val optimizedDeserializer: Seq[Expression] = {
     // When using `ExpressionEncoder` directly, we will skip the normal query processing steps
     // (analyzer, optimizer, etc.). Here we apply the ReassignLambdaVariableID rule, as it's
     // important to codegen performance.
-    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(serializer))
+    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(Seq(deserializer)))
     optimizedPlan.asInstanceOf[DummyExpressionHolder].exprs
-  })
-
-  @transient
-  private lazy val inputRow = new GenericInternalRow(1)
+  }
 
   @transient
-  private lazy val constructProjection = SafeProjection.create({
+  private lazy val optimizedSerializer = {
     // When using `ExpressionEncoder` directly, we will skip the normal query processing steps
     // (analyzer, optimizer, etc.). Here we apply the ReassignLambdaVariableID rule, as it's
     // important to codegen performance.
-    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(Seq(deserializer)))
+    val optimizedPlan = ReassignLambdaVariableID.apply(DummyExpressionHolder(serializer))
     optimizedPlan.asInstanceOf[DummyExpressionHolder].exprs
-  })
+  }
 
   /**
    * Returns a new set (with unique ids) of [[NamedExpression]] that represent the serialized form
@@ -331,31 +381,21 @@ case class ExpressionEncoder[T](
   }
 
   /**
-   * Returns an encoded version of `t` as a Spark SQL row.  Note that multiple calls to
-   * toRow are allowed to return the same actual [[InternalRow]] object.  Thus, the caller should
-   * copy the result before making another call if required.
+   * Create a serializer that can convert an object of type `T` to a Spark SQL Row.
+   *
+   * Note that the returned [[Serializer]] is not thread safe. Multiple calls to
+   * `serializer.apply(..)` are allowed to return the same actual [[InternalRow]] object.  Thus,
+   *  the caller should copy the result before making another call if required.
    */
-  def toRow(t: T): InternalRow = try {
-    inputRow(0) = t
-    extractProjection(inputRow)
-  } catch {
-    case e: Exception =>
-      throw new RuntimeException(s"Error while encoding: $e\n" +
-          s"${serializer.map(_.simpleString(SQLConf.get.maxToStringFields)).mkString("\n")}", e)
-  }
+  def createSerializer(): Serializer[T] = new Serializer[T](optimizedSerializer)
 
   /**
-   * Returns an object of type `T`, extracting the required values from the provided row.  Note that
-   * you must `resolveAndBind` an encoder to a specific schema before you can call this
-   * function.
+   * Create a deserializer that can convert a Spark SQL Row into an object of type `T`.
+   *
+   * Note that you must `resolveAndBind` an encoder to a specific schema before you can create a
+   * deserializer.
    */
-  def fromRow(row: InternalRow): T = try {
-    constructProjection(row).get(0, ObjectType(clsTag.runtimeClass)).asInstanceOf[T]
-  } catch {
-    case e: Exception =>
-      throw new RuntimeException(s"Error while decoding: $e\n" +
-        s"${deserializer.simpleString(SQLConf.get.maxToStringFields)}", e)
-  }
+  def createDeserializer(): Deserializer[T] = new Deserializer[T](optimizedDeserializer)
 
   /**
    * The process of resolution to a given schema throws away information about where a given field
@@ -382,8 +422,6 @@ case class ExpressionEncoder[T](
       .map { case(f, a) => s"${f.name}$a: ${f.dataType.simpleString}"}.mkString(", ")
 
   override def toString: String = s"class[$schemaString]"
-
-  override def makeCopy: ExpressionEncoder[T] = copy()
 }
 
 // A dummy logical plan that can hold expressions and go through optimizer rules.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 1ac7ca676a876..e80f03ea84756 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -110,8 +110,8 @@ case class ScalaUDF(
     } else {
       val encoder = inputEncoders(i)
       if (encoder.isDefined && encoder.get.isSerializedAsStructForTopLevel) {
-        val enc = encoder.get.resolveAndBind()
-        row: Any => enc.fromRow(row.asInstanceOf[InternalRow])
+        val fromRow = encoder.get.resolveAndBind().createDeserializer()
+        row: Any => fromRow(row.asInstanceOf[InternalRow])
       } else {
         CatalystTypeConverters.createToScalaConverter(dataType)
       }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala
index 3b4b80daf0843..3f0121bcf4a63 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/HashBenchmark.scala
@@ -41,13 +41,13 @@ object HashBenchmark extends BenchmarkBase {
   def test(name: String, schema: StructType, numRows: Int, iters: Int): Unit = {
     runBenchmark(name) {
       val generator = RandomDataGenerator.forType(schema, nullable = false).get
-      val encoder = RowEncoder(schema)
+      val toRow = RowEncoder(schema).createSerializer()
       val attrs = schema.toAttributes
       val safeProjection = GenerateSafeProjection.generate(attrs, attrs)
 
       val rows = (1 to numRows).map(_ =>
         // The output of encoder is UnsafeRow, use safeProjection to turn in into safe format.
-        safeProjection(encoder.toRow(generator().asInstanceOf[Row])).copy()
+        safeProjection(toRow(generator().asInstanceOf[Row])).copy()
       ).toArray
 
       val benchmark = new Benchmark("Hash For " + name, iters * numRows.toLong, output = output)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
index 42a4cfc91f826..950e313fb727a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/UnsafeProjectionBenchmark.scala
@@ -37,8 +37,8 @@ object UnsafeProjectionBenchmark extends BenchmarkBase {
 
   def generateRows(schema: StructType, numRows: Int): Array[InternalRow] = {
     val generator = RandomDataGenerator.forType(schema, nullable = false).get
-    val encoder = RowEncoder(schema)
-    (1 to numRows).map(_ => encoder.toRow(generator().asInstanceOf[Row]).copy()).toArray
+    val toRow = RowEncoder(schema).createSerializer()
+    (1 to numRows).map(_ => toRow(generator().asInstanceOf[Row]).copy()).toArray
   }
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index 53cb8bce0a52d..48f4ef5051fb3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.TypeTag
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.util.GenericArrayData
 import org.apache.spark.sql.types._
@@ -42,22 +43,29 @@ case class NestedArrayClass(nestedArr: Array[ArrayClass])
 class EncoderResolutionSuite extends PlanTest {
   private val str = UTF8String.fromString("hello")
 
+  def testFromRow[T](
+      encoder: ExpressionEncoder[T],
+      attributes: Seq[Attribute],
+      row: InternalRow): Unit = {
+    encoder.resolveAndBind(attributes).createDeserializer().apply(row)
+  }
+
   test("real type doesn't match encoder schema but they are compatible: product") {
     val encoder = ExpressionEncoder[StringLongClass]
 
     // int type can be up cast to long type
     val attrs1 = Seq('a.string, 'b.int)
-    encoder.resolveAndBind(attrs1).fromRow(InternalRow(str, 1))
+    testFromRow(encoder, attrs1, InternalRow(str, 1))
 
     // int type can be up cast to string type
     val attrs2 = Seq('a.int, 'b.long)
-    encoder.resolveAndBind(attrs2).fromRow(InternalRow(1, 2L))
+    testFromRow(encoder, attrs2, InternalRow(1, 2L))
   }
 
   test("real type doesn't match encoder schema but they are compatible: nested product") {
     val encoder = ExpressionEncoder[ComplexClass]
     val attrs = Seq('a.int, 'b.struct('a.int, 'b.long))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(1, InternalRow(2, 3L)))
+    testFromRow(encoder, attrs, InternalRow(1, InternalRow(2, 3L)))
   }
 
   test("real type doesn't match encoder schema but they are compatible: tupled encoder") {
@@ -65,14 +73,14 @@ class EncoderResolutionSuite extends PlanTest {
       ExpressionEncoder[StringLongClass],
       ExpressionEncoder[Long])
     val attrs = Seq('a.struct('a.string, 'b.byte), 'b.int)
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(InternalRow(str, 1.toByte), 2))
+    testFromRow(encoder, attrs, InternalRow(InternalRow(str, 1.toByte), 2))
   }
 
   test("real type doesn't match encoder schema but they are compatible: primitive array") {
     val encoder = ExpressionEncoder[PrimitiveArrayClass]
     val attrs = Seq('arr.array(IntegerType))
     val array = new GenericArrayData(Array(1, 2, 3))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(array))
+    testFromRow(encoder, attrs, InternalRow(array))
   }
 
   test("the real type is not compatible with encoder schema: primitive array") {
@@ -93,7 +101,7 @@ class EncoderResolutionSuite extends PlanTest {
     val encoder = ExpressionEncoder[ArrayClass]
     val attrs = Seq('arr.array(new StructType().add("a", "int").add("b", "int").add("c", "int")))
     val array = new GenericArrayData(Array(InternalRow(1, 2, 3)))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(array))
+    testFromRow(encoder, attrs, InternalRow(array))
   }
 
   test("real type doesn't match encoder schema but they are compatible: nested array") {
@@ -103,7 +111,7 @@ class EncoderResolutionSuite extends PlanTest {
     val attrs = Seq('nestedArr.array(et))
     val innerArr = new GenericArrayData(Array(InternalRow(1, 2, 3)))
     val outerArr = new GenericArrayData(Array(InternalRow(innerArr)))
-    encoder.resolveAndBind(attrs).fromRow(InternalRow(outerArr))
+    testFromRow(encoder, attrs, InternalRow(outerArr))
   }
 
   test("the real type is not compatible with encoder schema: non-array field") {
@@ -142,14 +150,14 @@ class EncoderResolutionSuite extends PlanTest {
     val attrs = 'a.array(IntegerType) :: Nil
 
     // It should pass analysis
-    val bound = encoder.resolveAndBind(attrs)
+    val fromRow = encoder.resolveAndBind(attrs).createDeserializer()
 
     // If no null values appear, it should work fine
-    bound.fromRow(InternalRow(new GenericArrayData(Array(1, 2))))
+    fromRow(InternalRow(new GenericArrayData(Array(1, 2))))
 
     // If there is null value, it should throw runtime exception
     val e = intercept[RuntimeException] {
-      bound.fromRow(InternalRow(new GenericArrayData(Array(1, null))))
+      fromRow(InternalRow(new GenericArrayData(Array(1, null))))
     }
     assert(e.getMessage.contains("Null value appeared in non-nullable field"))
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
index 1036dc725c205..6a094d4aaddae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala
@@ -369,14 +369,14 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
   }
 
   test("null check for map key: String") {
-    val encoder = ExpressionEncoder[Map[String, Int]]()
-    val e = intercept[RuntimeException](encoder.toRow(Map(("a", 1), (null, 2))))
+    val toRow = ExpressionEncoder[Map[String, Int]]().createSerializer()
+    val e = intercept[RuntimeException](toRow(Map(("a", 1), (null, 2))))
     assert(e.getMessage.contains("Cannot use null as map key"))
   }
 
   test("null check for map key: Integer") {
-    val encoder = ExpressionEncoder[Map[Integer, String]]()
-    val e = intercept[RuntimeException](encoder.toRow(Map((1, "a"), (null, "b"))))
+    val toRow = ExpressionEncoder[Map[Integer, String]]().createSerializer()
+    val e = intercept[RuntimeException](toRow(Map((1, "a"), (null, "b"))))
     assert(e.getMessage.contains("Cannot use null as map key"))
   }
 
@@ -436,10 +436,6 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
   testOverflowingBigNumeric(BigInt("9" * 100), "scala very large big int")
   testOverflowingBigNumeric(new BigInteger("9" * 100), "java very big int")
 
-  encodeDecodeTest("foo" -> 1L, "makeCopy") {
-    Encoders.product[(String, Long)].makeCopy.asInstanceOf[ExpressionEncoder[(String, Long)]]
-  }
-
   private def testOverflowingBigNumeric[T: TypeTag](bigNumeric: T, testName: String): Unit = {
     Seq(true, false).foreach { ansiEnabled =>
       testAndVerifyNotLeakingReflectionObjects(
@@ -450,12 +446,14 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
           // Need to construct Encoder here rather than implicitly resolving it
           // so that SQLConf changes are respected.
           val encoder = ExpressionEncoder[T]()
+          val toRow = encoder.createSerializer()
           if (!ansiEnabled) {
-            val convertedBack = encoder.resolveAndBind().fromRow(encoder.toRow(bigNumeric))
+            val fromRow = encoder.resolveAndBind().createDeserializer()
+            val convertedBack = fromRow(toRow(bigNumeric))
             assert(convertedBack === null)
           } else {
             val e = intercept[RuntimeException] {
-              encoder.toRow(bigNumeric)
+              toRow(bigNumeric)
             }
             assert(e.getMessage.contains("Error while encoding"))
             assert(e.getCause.getClass === classOf[ArithmeticException])
@@ -474,10 +472,10 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes
       // Make sure encoder is serializable.
       ClosureCleaner.clean((s: String) => encoder.getClass.getName)
 
-      val row = encoder.toRow(input)
+      val row = encoder.createSerializer().apply(input)
       val schema = encoder.schema.toAttributes
       val boundEncoder = encoder.resolveAndBind()
-      val convertedBack = try boundEncoder.fromRow(row) catch {
+      val convertedBack = try boundEncoder.createDeserializer().apply(row) catch {
         case e: Exception =>
           fail(
            s"""Exception thrown while decoding
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index 1a1cab823d4f3..c1158e001a780 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.encoders
 import scala.util.Random
 
 import org.apache.spark.sql.{RandomDataGenerator, Row}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.plans.CodegenInterpretedPlanTest
 import org.apache.spark.sql.catalyst.util.{ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.internal.SQLConf
@@ -81,6 +82,18 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
   private val mapOfString = MapType(StringType, StringType)
   private val arrayOfUDT = ArrayType(new ExamplePointUDT, false)
 
+  private def toRow(encoder: ExpressionEncoder[Row], row: Row): InternalRow = {
+    encoder.createSerializer().apply(row)
+  }
+
+  private def fromRow(encoder: ExpressionEncoder[Row], row: InternalRow): Row = {
+    encoder.createDeserializer().apply(row)
+  }
+
+  private def roundTrip(encoder: ExpressionEncoder[Row], row: Row): Row = {
+    fromRow(encoder, toRow(encoder, row))
+  }
+
   encodeDecodeTest(
     new StructType()
       .add("null", NullType)
@@ -144,8 +157,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     val catalystDecimal = Decimal("1234.5678")
 
     val input = Row(100, "test", 0.123, javaDecimal, scalaDecimal, catalystDecimal)
-    val row = encoder.toRow(input)
-    val convertedBack = encoder.fromRow(row)
+    val convertedBack = roundTrip(encoder, input)
     // Decimal will be converted back to Java BigDecimal when decoding.
     assert(convertedBack.getDecimal(3).compareTo(javaDecimal) == 0)
     assert(convertedBack.getDecimal(4).compareTo(scalaDecimal.bigDecimal) == 0)
@@ -157,7 +169,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     val encoder = RowEncoder(schema).resolveAndBind()
     val decimal = Decimal("67123.45")
     val input = Row(decimal)
-    val row = encoder.toRow(input)
+    val row = toRow(encoder, input)
 
     assert(row.toSeq(schema).head == decimal)
   }
@@ -172,7 +184,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
       val encoder = RowEncoder(schema).resolveAndBind()
       intercept[Exception] {
-        encoder.toRow(row)
+        toRow(encoder, row)
       } match {
         case e: ArithmeticException =>
           assert(e.getMessage.contains("cannot be represented as Decimal"))
@@ -184,7 +196,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
 
     withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
       val encoder = RowEncoder(schema).resolveAndBind()
-      assert(encoder.fromRow(encoder.toRow(row)).get(0) == null)
+      assert(roundTrip(encoder, row).get(0) == null)
     }
   }
 
@@ -237,8 +249,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       Array(1.1.toFloat, 123.456.toFloat, Float.MaxValue),
       Array(11.1111, 123456.7890123, Double.MaxValue)
     )
-    val row = encoder.toRow(Row.fromSeq(input))
-    val convertedBack = encoder.fromRow(row)
+    val convertedBack = roundTrip(encoder, Row.fromSeq(input))
     input.zipWithIndex.map { case (array, index) =>
       assert(convertedBack.getSeq(index) === array)
     }
@@ -254,8 +265,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       Array(1, 2, null),
       Array(Array("abc", null), null),
       Array(Seq(Array(0L, null), null), null))
-    val row = encoder.toRow(input)
-    val convertedBack = encoder.fromRow(row)
+    val convertedBack = roundTrip(encoder, input)
     assert(convertedBack.getSeq(0) == Seq(1, 2, null))
     assert(convertedBack.getSeq(1) == Seq(Seq("abc", null), null))
     assert(convertedBack.getSeq(2) == Seq(Seq(Seq(0L, null), null), null))
@@ -264,7 +274,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
   test("RowEncoder should throw RuntimeException if input row object is null") {
     val schema = new StructType().add("int", IntegerType)
     val encoder = RowEncoder(schema)
-    val e = intercept[RuntimeException](encoder.toRow(null))
+    val e = intercept[RuntimeException](toRow(encoder, null))
     assert(e.getMessage.contains("Null value appeared in non-nullable field"))
     assert(e.getMessage.contains("top level Product or row object"))
   }
@@ -273,14 +283,14 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
     val e1 = intercept[RuntimeException] {
       val schema = new StructType().add("a", IntegerType)
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(1.toShort))
+      toRow(encoder, Row(1.toShort))
     }
     assert(e1.getMessage.contains("java.lang.Short is not a valid external type"))
 
     val e2 = intercept[RuntimeException] {
       val schema = new StructType().add("a", StringType)
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(1))
+      toRow(encoder, Row(1))
     }
     assert(e2.getMessage.contains("java.lang.Integer is not a valid external type"))
 
@@ -288,14 +298,14 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val schema = new StructType().add("a",
         new StructType().add("b", IntegerType).add("c", StringType))
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(1 -> "a"))
+      toRow(encoder, Row(1 -> "a"))
     }
     assert(e3.getMessage.contains("scala.Tuple2 is not a valid external type"))
 
     val e4 = intercept[RuntimeException] {
       val schema = new StructType().add("a", ArrayType(TimestampType))
       val encoder = RowEncoder(schema)
-      encoder.toRow(Row(Array("a")))
+      toRow(encoder, Row(Array("a")))
     }
     assert(e4.getMessage.contains("java.lang.String is not a valid external type"))
   }
@@ -313,9 +323,9 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val schema = new StructType().add("t", TimestampType)
       val encoder = RowEncoder(schema).resolveAndBind()
       val instant = java.time.Instant.parse("2019-02-26T16:56:00Z")
-      val row = encoder.toRow(Row(instant))
+      val row = toRow(encoder, Row(instant))
       assert(row.getLong(0) === DateTimeUtils.instantToMicros(instant))
-      val readback = encoder.fromRow(row)
+      val readback = fromRow(encoder, row)
       assert(readback.get(0) === instant)
     }
   }
@@ -325,9 +335,9 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       val schema = new StructType().add("d", DateType)
       val encoder = RowEncoder(schema).resolveAndBind()
       val localDate = java.time.LocalDate.parse("2019-02-27")
-      val row = encoder.toRow(Row(localDate))
+      val row = toRow(encoder, Row(localDate))
       assert(row.getLong(0) === DateTimeUtils.localDateToDays(localDate))
-      val readback = encoder.fromRow(row)
+      val readback = fromRow(encoder, row)
       assert(readback.get(0).equals(localDate))
     }
   }
@@ -374,8 +384,7 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
       try {
         for (_ <- 1 to 5) {
           input = inputGenerator.apply().asInstanceOf[Row]
-          val row = encoder.toRow(input)
-          val convertedBack = encoder.fromRow(row)
+          val convertedBack = roundTrip(encoder, input)
           assert(input == convertedBack)
         }
       } catch {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
index 68da1faaa8f45..af6e5a3f35ee1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HashExpressionsSuite.scala
@@ -699,11 +699,11 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
   private def testHash(inputSchema: StructType): Unit = {
     val inputGenerator = RandomDataGenerator.forType(inputSchema, nullable = false).get
-    val encoder = RowEncoder(inputSchema)
+    val toRow = RowEncoder(inputSchema).createSerializer()
     val seed = scala.util.Random.nextInt()
     test(s"murmur3/xxHash64/hive hash: ${inputSchema.simpleString}") {
       for (_ <- 1 to 10) {
-        val input = encoder.toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
+        val input = toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
         val literals = input.toSeq(inputSchema).zip(inputSchema.map(_.dataType)).map {
           case (value, dt) => Literal.create(value, dt)
         }
@@ -717,7 +717,7 @@ class HashExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     val longSeed = Math.abs(seed).toLong + Integer.MAX_VALUE.toLong
     test(s"SPARK-30633: xxHash64 with long seed: ${inputSchema.simpleString}") {
       for (_ <- 1 to 10) {
-        val input = encoder.toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
+        val input = toRow(inputGenerator.apply().asInstanceOf[Row]).asInstanceOf[UnsafeRow]
         val literals = input.toSeq(inputSchema).zip(inputSchema.map(_.dataType)).map {
           case (value, dt) => Literal.create(value, dt)
         }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
index ef7764dba1e9e..c40149368b055 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
@@ -445,8 +445,8 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     testTypes.foreach { dt =>
       genSchema(dt).map { schema =>
         val row = RandomDataGenerator.randomRow(random, schema)
-        val rowConverter = RowEncoder(schema)
-        val internalRow = rowConverter.toRow(row)
+        val toRow = RowEncoder(schema).createSerializer()
+        val internalRow = toRow(row)
         val lambda = LambdaVariable("dummy", schema(0).dataType, schema(0).nullable, id = 0)
         checkEvaluationWithoutCodegen(lambda, internalRow.get(0, schema(0).dataType), internalRow)
       }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
index fb1ea7b867a6d..dd67a61015e72 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeRowJoinerSuite.scala
@@ -60,8 +60,8 @@ class GenerateUnsafeRowJoinerSuite extends SparkFunSuite {
   test("rows with all empty int arrays") {
     val schema = StructType(Seq(
       StructField("f1", ArrayType(IntegerType)), StructField("f2", ArrayType(IntegerType))))
-    val emptyIntArray =
-      ExpressionEncoder[Array[Int]]().resolveAndBind().toRow(Array.emptyIntArray).getArray(0)
+    val toRow = ExpressionEncoder[Array[Int]]().resolveAndBind().createSerializer()
+    val emptyIntArray = toRow(Array.emptyIntArray).getArray(0)
     val row: UnsafeRow = UnsafeProjection.create(schema).apply(
       InternalRow(emptyIntArray, emptyIntArray))
     testConcat(schema, row, schema, row)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
index da71e3a4d53e2..1e430351b5137 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayDataIndexedSeqSuite.scala
@@ -73,8 +73,8 @@ class ArrayDataIndexedSeqSuite extends SparkFunSuite {
     arrayTypes.foreach { dt =>
       val schema = StructType(StructField("col_1", dt, nullable = false) :: Nil)
       val row = RandomDataGenerator.randomRow(random, schema)
-      val rowConverter = RowEncoder(schema)
-      val internalRow = rowConverter.toRow(row)
+      val toRow = RowEncoder(schema).createSerializer()
+      val internalRow = toRow(row)
 
       val unsafeRowConverter = UnsafeProjection.create(schema)
       val safeRowConverter = SafeProjection.create(schema)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
index e7b1c0810a033..6d8ef68473778 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeArraySuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.util
 
 import java.time.{ZoneId, ZoneOffset}
 
+import scala.reflect.runtime.universe.TypeTag
+
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.serializer.{JavaSerializer, KryoSerializer}
 import org.apache.spark.sql.Row
@@ -70,75 +72,55 @@ class UnsafeArraySuite extends SparkFunSuite {
     arrayData
   }
 
+  private def toUnsafeArray[T : TypeTag](array: Array[T]): ArrayData = {
+    val converted = ExpressionEncoder[Array[T]].createSerializer().apply(array).getArray(0)
+    assert(converted.isInstanceOf[T])
+    assert(converted.numElements == array.length)
+    converted
+  }
+
   test("read array") {
-    val unsafeBoolean = ExpressionEncoder[Array[Boolean]].resolveAndBind().
-      toRow(booleanArray).getArray(0)
-    assert(unsafeBoolean.isInstanceOf[UnsafeArrayData])
-    assert(unsafeBoolean.numElements == booleanArray.length)
+    val unsafeBoolean = toUnsafeArray(booleanArray)
     booleanArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeBoolean.getBoolean(i) == e)
     }
 
-    val unsafeShort = ExpressionEncoder[Array[Short]].resolveAndBind().
-      toRow(shortArray).getArray(0)
-    assert(unsafeShort.isInstanceOf[UnsafeArrayData])
-    assert(unsafeShort.numElements == shortArray.length)
+    val unsafeShort = toUnsafeArray(shortArray)
     shortArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeShort.getShort(i) == e)
     }
 
-    val unsafeInt = ExpressionEncoder[Array[Int]].resolveAndBind().
-      toRow(intArray).getArray(0)
-    assert(unsafeInt.isInstanceOf[UnsafeArrayData])
-    assert(unsafeInt.numElements == intArray.length)
+    val unsafeInt = toUnsafeArray(intArray)
     intArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeInt.getInt(i) == e)
     }
 
-    val unsafeLong = ExpressionEncoder[Array[Long]].resolveAndBind().
-      toRow(longArray).getArray(0)
-    assert(unsafeLong.isInstanceOf[UnsafeArrayData])
-    assert(unsafeLong.numElements == longArray.length)
+    val unsafeLong = toUnsafeArray(longArray)
     longArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeLong.getLong(i) == e)
     }
 
-    val unsafeFloat = ExpressionEncoder[Array[Float]].resolveAndBind().
-      toRow(floatArray).getArray(0)
-    assert(unsafeFloat.isInstanceOf[UnsafeArrayData])
-    assert(unsafeFloat.numElements == floatArray.length)
+    val unsafeFloat = toUnsafeArray(floatArray)
     floatArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeFloat.getFloat(i) == e)
     }
 
-    val unsafeDouble = ExpressionEncoder[Array[Double]].resolveAndBind().
-      toRow(doubleArray).getArray(0)
-    assert(unsafeDouble.isInstanceOf[UnsafeArrayData])
-    assert(unsafeDouble.numElements == doubleArray.length)
+    val unsafeDouble = toUnsafeArray(doubleArray)
     doubleArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeDouble.getDouble(i) == e)
     }
 
-    val unsafeString = ExpressionEncoder[Array[String]].resolveAndBind().
-      toRow(stringArray).getArray(0)
-    assert(unsafeString.isInstanceOf[UnsafeArrayData])
-    assert(unsafeString.numElements == stringArray.length)
+    val unsafeString = toUnsafeArray(stringArray)
     stringArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeString.getUTF8String(i).toString().equals(e))
     }
 
-    val unsafeDate = ExpressionEncoder[Array[Int]].resolveAndBind().
-      toRow(dateArray).getArray(0)
-    assert(unsafeDate.isInstanceOf[UnsafeArrayData])
-    assert(unsafeDate.numElements == dateArray.length)
+    val unsafeDate = toUnsafeArray(dateArray)
     dateArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeDate.get(i, DateType).asInstanceOf[Int] == e)
     }
 
-    val unsafeTimestamp = ExpressionEncoder[Array[Long]].resolveAndBind().
-      toRow(timestampArray).getArray(0)
-    assert(unsafeTimestamp.isInstanceOf[UnsafeArrayData])
-    assert(unsafeTimestamp.numElements == timestampArray.length)
+    val unsafeTimestamp = toUnsafeArray(timestampArray)
     timestampArray.zipWithIndex.map { case (e, i) =>
       assert(unsafeTimestamp.get(i, TimestampType).asInstanceOf[Long] == e)
     }
@@ -149,7 +131,7 @@ class UnsafeArraySuite extends SparkFunSuite {
         "array", ArrayType(DecimalType(decimal.precision, decimal.scale)))
       val encoder = RowEncoder(schema).resolveAndBind()
       val externalRow = Row(decimalArray)
-      val ir = encoder.toRow(externalRow)
+      val ir = encoder.createSerializer().apply(externalRow)
 
       val unsafeDecimal = ir.getArray(0)
       assert(unsafeDecimal.isInstanceOf[UnsafeArrayData])
@@ -162,7 +144,7 @@ class UnsafeArraySuite extends SparkFunSuite {
     val schema = new StructType().add("array", ArrayType(CalendarIntervalType))
     val encoder = RowEncoder(schema).resolveAndBind()
     val externalRow = Row(calenderintervalArray)
-    val ir = encoder.toRow(externalRow)
+    val ir = encoder.createSerializer().apply(externalRow)
     val unsafeCalendar = ir.getArray(0)
     assert(unsafeCalendar.isInstanceOf[UnsafeArrayData])
     assert(unsafeCalendar.numElements == calenderintervalArray.length)
@@ -170,10 +152,7 @@ class UnsafeArraySuite extends SparkFunSuite {
       assert(unsafeCalendar.getInterval(i) == e)
     }
 
-    val unsafeMultiDimInt = ExpressionEncoder[Array[Array[Int]]].resolveAndBind().
-      toRow(intMultiDimArray).getArray(0)
-    assert(unsafeMultiDimInt.isInstanceOf[UnsafeArrayData])
-    assert(unsafeMultiDimInt.numElements == intMultiDimArray.length)
+    val unsafeMultiDimInt = toUnsafeArray(intMultiDimArray)
     intMultiDimArray.zipWithIndex.map { case (a, j) =>
       val u = unsafeMultiDimInt.getArray(j)
       assert(u.isInstanceOf[UnsafeArrayData])
@@ -183,10 +162,7 @@ class UnsafeArraySuite extends SparkFunSuite {
       }
     }
 
-    val unsafeMultiDimDouble = ExpressionEncoder[Array[Array[Double]]].resolveAndBind().
-      toRow(doubleMultiDimArray).getArray(0)
-    assert(unsafeDouble.isInstanceOf[UnsafeArrayData])
-    assert(unsafeMultiDimDouble.numElements == doubleMultiDimArray.length)
+    val unsafeMultiDimDouble = toUnsafeArray(doubleMultiDimArray)
     doubleMultiDimArray.zipWithIndex.map { case (a, j) =>
       val u = unsafeMultiDimDouble.getArray(j)
       assert(u.isInstanceOf[UnsafeArrayData])
@@ -216,11 +192,9 @@ class UnsafeArraySuite extends SparkFunSuite {
   }
 
   test("to primitive array") {
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
-    assert(intEncoder.toRow(intArray).getArray(0).toIntArray.sameElements(intArray))
+    assert(toUnsafeArray(intArray).toIntArray().sameElements(intArray))
 
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
-    assert(doubleEncoder.toRow(doubleArray).getArray(0).toDoubleArray.sameElements(doubleArray))
+    assert(toUnsafeArray(doubleArray).toDoubleArray().sameElements(doubleArray))
   }
 
   test("unsafe java serialization") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 5c3a82afabb73..c17535a43a14b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -2965,9 +2965,8 @@ class Dataset[T] private[sql](
    */
   def toLocalIterator(): java.util.Iterator[T] = {
     withAction("toLocalIterator", queryExecution) { plan =>
-      // `ExpressionEncoder` is not thread-safe, here we create a new encoder.
-      val enc = resolvedEnc.copy()
-      plan.executeToIterator().map(enc.fromRow).asJava
+      val fromRow = resolvedEnc.createDeserializer()
+      plan.executeToIterator().map(fromRow).asJava
     }
   }
 
@@ -3387,9 +3386,10 @@ class Dataset[T] private[sql](
         new JSONOptions(Map.empty[String, String], sessionLocalTimeZone))
 
       new Iterator[String] {
+        private val toRow = exprEnc.createSerializer()
         override def hasNext: Boolean = iter.hasNext
         override def next(): String = {
-          gen.write(exprEnc.toRow(iter.next()))
+          gen.write(toRow(iter.next()))
           gen.flush()
 
           val json = writer.toString
@@ -3621,9 +3621,8 @@ class Dataset[T] private[sql](
    * Collect all elements from a spark plan.
    */
   private def collectFromPlan(plan: SparkPlan): Array[T] = {
-    // `ExpressionEncoder` is not thread-safe, here we create a new encoder.
-    val enc = resolvedEnc.copy()
-    plan.executeCollect().map(enc.fromRow)
+    val fromRow = resolvedEnc.createDeserializer()
+    plan.executeCollect().map(fromRow)
   }
 
   private def sortInternal(global: Boolean, sortExprs: Seq[Column]): Dataset[T] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index bca841c48cacd..731aae882a7b8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -345,7 +345,8 @@ class SparkSession private(
     // TODO: use MutableProjection when rowRDD is another DataFrame and the applied
     // schema differs from the existing schema on any field data type.
     val encoder = RowEncoder(schema)
-    val catalystRows = rowRDD.map(encoder.toRow)
+    val toRow = encoder.createSerializer()
+    val catalystRows = rowRDD.map(toRow)
     internalCreateDataFrame(catalystRows.setName(rowRDD.name), schema)
   }
 
@@ -459,10 +460,10 @@ class SparkSession private(
    * @since 2.0.0
    */
   def createDataset[T : Encoder](data: Seq[T]): Dataset[T] = {
-    // `ExpressionEncoder` is not thread-safe, here we create a new encoder.
-    val enc = encoderFor[T].copy()
+    val enc = encoderFor[T]
+    val toRow = enc.createSerializer()
     val attributes = enc.schema.toAttributes
-    val encoded = data.map(d => enc.toRow(d).copy())
+    val encoded = data.map(d => toRow(d).copy())
     val plan = new LocalRelation(attributes, encoded)
     Dataset[T](self, plan)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index bd2684d92a1d2..12a1a1e7fc16e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -656,7 +656,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
       case MemoryPlan(sink, output) =>
         val encoder = RowEncoder(StructType.fromAttributes(output))
-        LocalTableScanExec(output, sink.allData.map(r => encoder.toRow(r).copy())) :: Nil
+        val toRow = encoder.createSerializer()
+        LocalTableScanExec(output, sink.allData.map(r => toRow(r).copy())) :: Nil
 
       case logical.Distinct(child) =>
         throw new IllegalStateException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
index dfae5c07e0373..544b90a736071 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
@@ -469,14 +469,17 @@ case class ScalaAggregator[IN, BUF, OUT](
   with ImplicitCastInputTypes
   with Logging {
 
-  private[this] lazy val inputEncoder = inputEncoderNR.resolveAndBind()
+  private[this] lazy val inputDeserializer = inputEncoderNR.resolveAndBind().createDeserializer()
   private[this] lazy val bufferEncoder =
     agg.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]].resolveAndBind()
+  private[this] lazy val bufferSerializer = bufferEncoder.createSerializer()
+  private[this] lazy val bufferDeserializer = bufferEncoder.createDeserializer()
   private[this] lazy val outputEncoder = agg.outputEncoder.asInstanceOf[ExpressionEncoder[OUT]]
+  private[this] lazy val outputSerializer = outputEncoder.createSerializer()
 
   def dataType: DataType = outputEncoder.objSerializer.dataType
 
-  def inputTypes: Seq[DataType] = inputEncoder.schema.map(_.dataType)
+  def inputTypes: Seq[DataType] = inputEncoderNR.schema.map(_.dataType)
 
   override lazy val deterministic: Boolean = isDeterministic
 
@@ -491,23 +494,23 @@ case class ScalaAggregator[IN, BUF, OUT](
   def createAggregationBuffer(): BUF = agg.zero
 
   def update(buffer: BUF, input: InternalRow): BUF =
-    agg.reduce(buffer, inputEncoder.fromRow(inputProjection(input)))
+    agg.reduce(buffer, inputDeserializer(inputProjection(input)))
 
   def merge(buffer: BUF, input: BUF): BUF = agg.merge(buffer, input)
 
   def eval(buffer: BUF): Any = {
-    val row = outputEncoder.toRow(agg.finish(buffer))
+    val row = outputSerializer(agg.finish(buffer))
     if (outputEncoder.isSerializedAsStruct) row else row.get(0, dataType)
   }
 
   private[this] lazy val bufferRow = new UnsafeRow(bufferEncoder.namedExpressions.length)
 
   def serialize(agg: BUF): Array[Byte] =
-    bufferEncoder.toRow(agg).asInstanceOf[UnsafeRow].getBytes()
+    bufferSerializer(agg).asInstanceOf[UnsafeRow].getBytes()
 
   def deserialize(storageFormat: Array[Byte]): BUF = {
     bufferRow.pointTo(storageFormat, storageFormat.length)
-    bufferEncoder.fromRow(bufferRow)
+    bufferDeserializer(bufferRow)
   }
 
   override def toString: String = s"""${nodeName}(${children.mkString(",")})"""
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index faf37609ad814..a58038d127818 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -637,9 +637,9 @@ object DataSourceStrategy {
       output: Seq[Attribute],
       rdd: RDD[Row]): RDD[InternalRow] = {
     if (relation.needConversion) {
-      val converters = RowEncoder(StructType.fromAttributes(output))
+      val toRow = RowEncoder(StructType.fromAttributes(output)).createSerializer()
       rdd.mapPartitions { iterator =>
-        iterator.map(converters.toRow)
+        iterator.map(toRow)
       }
     } else {
       rdd.asInstanceOf[RDD[InternalRow]]
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index c1e1aed83bae5..3f44312fd6985 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -331,9 +331,9 @@ object JdbcUtils extends Logging {
   def resultSetToRows(resultSet: ResultSet, schema: StructType): Iterator[Row] = {
     val inputMetrics =
       Option(TaskContext.get()).map(_.taskMetrics().inputMetrics).getOrElse(new InputMetrics)
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val fromRow = RowEncoder(schema).resolveAndBind().createDeserializer()
     val internalRows = resultSetToSparkInternalRows(resultSet, schema, inputMetrics)
-    internalRows.map(encoder.fromRow)
+    internalRows.map(fromRow)
   }
 
   private[spark] def resultSetToSparkInternalRows(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
index 64b98fb83b8fa..b4a14c6face31 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala
@@ -34,7 +34,9 @@ case class DescribeNamespaceExec(
     catalog: SupportsNamespaces,
     namespace: Seq[String],
     isExtended: Boolean) extends V2CommandExec {
-  private val encoder = RowEncoder(StructType.fromAttributes(output)).resolveAndBind()
+  private val toRow = {
+    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
+  }
 
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
@@ -57,6 +59,6 @@ case class DescribeNamespaceExec(
   }
 
   private def toCatalystRow(strs: String*): InternalRow = {
-    encoder.toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
+    toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
index 9c280206c548e..bc6bb175f979e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeTableExec.scala
@@ -31,7 +31,9 @@ case class DescribeTableExec(
     table: Table,
     isExtended: Boolean) extends V2CommandExec {
 
-  private val encoder = RowEncoder(StructType.fromAttributes(output)).resolveAndBind()
+  private val toRow = {
+    RowEncoder(StructType.fromAttributes(output)).resolveAndBind().createSerializer()
+  }
 
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
@@ -85,6 +87,6 @@ case class DescribeTableExec(
   private def emptyRow(): InternalRow = toCatalystRow("", "", "")
 
   private def toCatalystRow(strs: String*): InternalRow = {
-    encoder.toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
+    toRow(new GenericRowWithSchema(strs.toArray, schema)).copy()
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
index 42b80a15080a6..5f7b6f4061467 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCurrentNamespaceExec.scala
@@ -31,10 +31,11 @@ case class ShowCurrentNamespaceExec(
     catalogManager: CatalogManager)
   extends V2CommandExec {
   override protected def run(): Seq[InternalRow] = {
-    val encoder = RowEncoder(schema).resolveAndBind()
-    Seq(encoder
-      .toRow(new GenericRowWithSchema(
-        Array(catalogManager.currentCatalog.name, catalogManager.currentNamespace.quoted), schema))
-      .copy())
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
+    val result = new GenericRowWithSchema(Array[Any](
+      catalogManager.currentCatalog.name,
+      catalogManager.currentNamespace.quoted),
+      schema)
+    Seq(toRow(result).copy())
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
index 6f968481cb7cc..9188f4eb60d56 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowNamespacesExec.scala
@@ -44,13 +44,11 @@ case class ShowNamespacesExec(
     }
 
     val rows = new ArrayBuffer[InternalRow]()
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
     namespaces.map(_.quoted).map { ns =>
       if (pattern.map(StringUtils.filterPattern(Seq(ns), _).nonEmpty).getOrElse(true)) {
-        rows += encoder
-          .toRow(new GenericRowWithSchema(Array(ns), schema))
-          .copy()
+        rows += toRow(new GenericRowWithSchema(Array(ns), schema)).copy()
       }
     }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
index 7905c35f55de0..0bcd7ea541045 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
@@ -32,17 +32,17 @@ case class ShowTablePropertiesExec(
 
   override protected def run(): Seq[InternalRow] = {
     import scala.collection.JavaConverters._
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
     val properties = catalogTable.properties.asScala
     propertyKey match {
       case Some(p) =>
         val propValue = properties
           .getOrElse(p, s"Table ${catalogTable.name} does not have property: $p")
-        Seq(encoder.toRow(new GenericRowWithSchema(Array(p, propValue), schema)).copy())
+        Seq(toRow(new GenericRowWithSchema(Array(p, propValue), schema)).copy())
       case None =>
         properties.keys.map(k =>
-          encoder.toRow(new GenericRowWithSchema(Array(k, properties(k)), schema)).copy()).toSeq
+          toRow(new GenericRowWithSchema(Array(k, properties(k)), schema)).copy()).toSeq
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
index c740e0d370dfd..820f5ae8f1b12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablesExec.scala
@@ -37,17 +37,15 @@ case class ShowTablesExec(
     pattern: Option[String]) extends V2CommandExec with LeafExecNode {
   override protected def run(): Seq[InternalRow] = {
     val rows = new ArrayBuffer[InternalRow]()
-    val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
     val tables = catalog.listTables(namespace.toArray)
     tables.map { table =>
       if (pattern.map(StringUtils.filterPattern(Seq(table.name()), _).nonEmpty).getOrElse(true)) {
-        rows += encoder
-          .toRow(
-            new GenericRowWithSchema(
-              Array(table.namespace().quoted, table.name()),
-              schema))
-          .copy()
+        val result = new GenericRowWithSchema(
+          Array(table.namespace().quoted, table.name()),
+          schema)
+        rows += toRow(result).copy()
       }
     }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
index fc47c5ed3ac00..368dfae0cc95e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/continuous/ContinuousTextSocketSource.scala
@@ -173,6 +173,7 @@ class TextSocketContinuousStream(
       setDaemon(true)
 
       override def run(): Unit = {
+        val toRow = encoder.createSerializer()
         try {
           while (true) {
             val line = reader.readLine()
@@ -187,7 +188,7 @@ class TextSocketContinuousStream(
                 Timestamp.valueOf(
                   TextSocketReader.DATE_FORMAT.format(Calendar.getInstance().getTime()))
               )
-              buckets(currentOffset % numPartitions) += encoder.toRow(newData)
+              buckets(currentOffset % numPartitions) += toRow(newData)
                 .copy().asInstanceOf[UnsafeRow]
             }
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
index ea39c549bd072..e5b9e68d71026 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/memory.scala
@@ -27,7 +27,7 @@ import scala.collection.mutable.ListBuffer
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.encoderFor
+import org.apache.spark.sql.catalyst.encoders.{encoderFor, ExpressionEncoder}
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.util.truncatedString
@@ -57,6 +57,8 @@ abstract class MemoryStreamBase[A : Encoder](sqlContext: SQLContext) extends Spa
   val encoder = encoderFor[A]
   protected val attributes = encoder.schema.toAttributes
 
+  protected lazy val toRow: ExpressionEncoder.Serializer[A] = encoder.createSerializer()
+
   def toDS(): Dataset[A] = {
     Dataset[A](sqlContext.sparkSession, logicalPlan)
   }
@@ -176,7 +178,7 @@ case class MemoryStream[A : Encoder](
 
   def addData(data: TraversableOnce[A]): Offset = {
     val objects = data.toSeq
-    val rows = objects.iterator.map(d => encoder.toRow(d).copy().asInstanceOf[UnsafeRow]).toArray
+    val rows = objects.iterator.map(d => toRow(d).copy().asInstanceOf[UnsafeRow]).toArray
     logDebug(s"Adding: $objects")
     this.synchronized {
       currentOffset = currentOffset + 1
@@ -243,7 +245,7 @@ case class MemoryStream[A : Encoder](
       rows: Seq[UnsafeRow],
       startOrdinal: Int,
       endOrdinal: Int): String = {
-    val fromRow = encoder.resolveAndBind().fromRow _
+    val fromRow = encoder.resolveAndBind().createDeserializer()
     s"MemoryBatch [$startOrdinal, $endOrdinal]: " +
         s"${rows.map(row => fromRow(row)).mkString(", ")}"
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
index f94469385b281..d0cf602c7cca2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ContinuousMemoryStream.scala
@@ -60,7 +60,7 @@ class ContinuousMemoryStream[A : Encoder](id: Int, sqlContext: SQLContext, numPa
     // Distribute data evenly among partition lists.
     data.toSeq.zipWithIndex.map {
       case (item, index) =>
-        records(index % numPartitions) += encoder.toRow(item).copy().asInstanceOf[UnsafeRow]
+        records(index % numPartitions) += toRow(item).copy().asInstanceOf[UnsafeRow]
     }
 
     // The new target offset is the offset where all records in all partitions have been processed.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
index 03c567c58d46a..6d5e7fd5c5cf3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachBatchSink.scala
@@ -30,7 +30,8 @@ class ForeachBatchSink[T](batchWriter: (Dataset[T], Long) => Unit, encoder: Expr
     val resolvedEncoder = encoder.resolveAndBind(
       data.logicalPlan.output,
       data.sparkSession.sessionState.analyzer)
-    val rdd = data.queryExecution.toRdd.map[T](resolvedEncoder.fromRow)(encoder.clsTag)
+    val fromRow = resolvedEncoder.createDeserializer()
+    val rdd = data.queryExecution.toRdd.map[T](fromRow)(encoder.clsTag)
     val ds = data.sparkSession.createDataset(rdd)(encoder)
     batchWriter(ds, batchId)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
index 6e4f40ad080d4..ba54c85d07303 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
@@ -73,7 +73,7 @@ case class ForeachWriterTable[T](
                 val boundEnc = enc.resolveAndBind(
                   inputSchema.toAttributes,
                   SparkSession.getActiveSession.get.sessionState.analyzer)
-                boundEnc.fromRow
+                boundEnc.createDeserializer()
               case Right(func) =>
                 func
             }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
index 2b674070a70ad..deab42bea36ad 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
@@ -172,10 +172,10 @@ class MemoryDataWriter(partition: Int, schema: StructType)
 
   private val data = mutable.Buffer[Row]()
 
-  private val encoder = RowEncoder(schema).resolveAndBind()
+  private val fromRow = RowEncoder(schema).resolveAndBind().createDeserializer()
 
   override def write(row: InternalRow): Unit = {
-    data.append(encoder.fromRow(row))
+    data.append(fromRow(row))
   }
 
   override def commit(): MemoryWriterCommitMessage = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index d3ef03e9b3b74..7ca9fbb40d9f5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -517,7 +517,8 @@ private[sql] object CatalogImpl {
       data: Seq[T],
       sparkSession: SparkSession): Dataset[T] = {
     val enc = ExpressionEncoder[T]()
-    val encoded = data.map(d => enc.toRow(d).copy())
+    val toRow = enc.createSerializer()
+    val encoded = data.map(d => toRow(d).copy())
     val plan = new LocalRelation(enc.schema.toAttributes, encoded)
     val queryExecution = sparkSession.sessionState.executePlan(plan)
     new Dataset[T](queryExecution, enc)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala
index 80340b5552c6d..4b2a2b439c89e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GroupedIteratorSuite.scala
@@ -28,14 +28,16 @@ class GroupedIteratorSuite extends SparkFunSuite {
   test("basic") {
     val schema = new StructType().add("i", IntegerType).add("s", StringType)
     val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = encoder.createSerializer()
+    val fromRow = encoder.createDeserializer()
     val input = Seq(Row(1, "a"), Row(1, "b"), Row(2, "c"))
-    val grouped = GroupedIterator(input.iterator.map(encoder.toRow),
+    val grouped = GroupedIterator(input.iterator.map(toRow),
       Seq('i.int.at(0)), schema.toAttributes)
 
     val result = grouped.map {
       case (key, data) =>
         assert(key.numFields == 1)
-        key.getInt(0) -> data.map(encoder.fromRow).toSeq
+        key.getInt(0) -> data.map(fromRow).toSeq
     }.toSeq
 
     assert(result ==
@@ -46,6 +48,8 @@ class GroupedIteratorSuite extends SparkFunSuite {
   test("group by 2 columns") {
     val schema = new StructType().add("i", IntegerType).add("l", LongType).add("s", StringType)
     val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = encoder.createSerializer()
+    val fromRow = encoder.createDeserializer()
 
     val input = Seq(
       Row(1, 2L, "a"),
@@ -54,13 +58,13 @@ class GroupedIteratorSuite extends SparkFunSuite {
       Row(2, 1L, "d"),
       Row(3, 2L, "e"))
 
-    val grouped = GroupedIterator(input.iterator.map(encoder.toRow),
+    val grouped = GroupedIterator(input.iterator.map(toRow),
       Seq('i.int.at(0), 'l.long.at(1)), schema.toAttributes)
 
     val result = grouped.map {
       case (key, data) =>
         assert(key.numFields == 2)
-        (key.getInt(0), key.getLong(1), data.map(encoder.fromRow).toSeq)
+        (key.getInt(0), key.getLong(1), data.map(fromRow).toSeq)
     }.toSeq
 
     assert(result ==
@@ -73,8 +77,9 @@ class GroupedIteratorSuite extends SparkFunSuite {
   test("do nothing to the value iterator") {
     val schema = new StructType().add("i", IntegerType).add("s", StringType)
     val encoder = RowEncoder(schema).resolveAndBind()
+    val toRow = encoder.createSerializer()
     val input = Seq(Row(1, "a"), Row(1, "b"), Row(2, "c"))
-    val grouped = GroupedIterator(input.iterator.map(encoder.toRow),
+    val grouped = GroupedIterator(input.iterator.map(toRow),
       Seq('i.int.at(0)), schema.toAttributes)
 
     assert(grouped.length == 2)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
index f582d844cdc47..9b0389c6d1ea4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/UnsafeArrayDataBenchmark.scala
@@ -40,13 +40,16 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
     UnsafeArrayData.calculateHeaderPortionInBytes(count)
   }
 
+  private lazy val intEncoder = ExpressionEncoder[Array[Int]]().resolveAndBind()
+
+  private lazy val doubleEncoder = ExpressionEncoder[Array[Double]]().resolveAndBind()
+
   def readUnsafeArray(iters: Int): Unit = {
     val count = 1024 * 1024 * 16
     val rand = new Random(42)
-
+    val intArrayToRow = intEncoder.createSerializer()
     val intPrimitiveArray = Array.fill[Int](count) { rand.nextInt }
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
-    val intUnsafeArray = intEncoder.toRow(intPrimitiveArray).getArray(0)
+    val intUnsafeArray = intArrayToRow(intPrimitiveArray).getArray(0)
     val readIntArray = { i: Int =>
       var n = 0
       while (n < iters) {
@@ -62,8 +65,8 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
     }
 
     val doublePrimitiveArray = Array.fill[Double](count) { rand.nextDouble }
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
-    val doubleUnsafeArray = doubleEncoder.toRow(doublePrimitiveArray).getArray(0)
+    val doubleArrayToRow = doubleEncoder.createSerializer()
+    val doubleUnsafeArray = doubleArrayToRow(doublePrimitiveArray).getArray(0)
     val readDoubleArray = { i: Int =>
       var n = 0
       while (n < iters) {
@@ -90,12 +93,12 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var intTotalLength: Int = 0
     val intPrimitiveArray = Array.fill[Int](count) { rand.nextInt }
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
+    val intArrayToRow = intEncoder.createSerializer()
     val writeIntArray = { i: Int =>
       var len = 0
       var n = 0
       while (n < iters) {
-        len += intEncoder.toRow(intPrimitiveArray).getArray(0).numElements()
+        len += intArrayToRow(intPrimitiveArray).getArray(0).numElements()
         n += 1
       }
       intTotalLength = len
@@ -103,12 +106,12 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var doubleTotalLength: Int = 0
     val doublePrimitiveArray = Array.fill[Double](count) { rand.nextDouble }
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
+    val doubleArrayToRow = doubleEncoder.createSerializer()
     val writeDoubleArray = { i: Int =>
       var len = 0
       var n = 0
       while (n < iters) {
-        len += doubleEncoder.toRow(doublePrimitiveArray).getArray(0).numElements()
+        len += doubleArrayToRow(doublePrimitiveArray).getArray(0).numElements()
         n += 1
       }
       doubleTotalLength = len
@@ -126,8 +129,8 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var intTotalLength: Int = 0
     val intPrimitiveArray = Array.fill[Int](count) { rand.nextInt }
-    val intEncoder = ExpressionEncoder[Array[Int]].resolveAndBind()
-    val intUnsafeArray = intEncoder.toRow(intPrimitiveArray).getArray(0)
+    val intArrayToRow = intEncoder.createSerializer()
+    val intUnsafeArray = intArrayToRow(intPrimitiveArray).getArray(0)
     val readIntArray = { i: Int =>
       var len = 0
       var n = 0
@@ -140,8 +143,8 @@ object UnsafeArrayDataBenchmark extends BenchmarkBase {
 
     var doubleTotalLength: Int = 0
     val doublePrimitiveArray = Array.fill[Double](count) { rand.nextDouble }
-    val doubleEncoder = ExpressionEncoder[Array[Double]].resolveAndBind()
-    val doubleUnsafeArray = doubleEncoder.toRow(doublePrimitiveArray).getArray(0)
+    val doubleArrayToRow = doubleEncoder.createSerializer()
+    val doubleUnsafeArray = doubleArrayToRow(doublePrimitiveArray).getArray(0)
     val readDoubleArray = { i: Int =>
       var len = 0
       var n = 0
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
index 2cd142f913072..8462916daaab8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/binaryfile/BinaryFileFormatSuite.scala
@@ -304,7 +304,7 @@ class BinaryFileFormatSuite extends QueryTest with SharedSparkSession {
     val partitionedFile = mock(classOf[PartitionedFile])
     when(partitionedFile.filePath).thenReturn(file.getPath)
     val encoder = RowEncoder(requiredSchema).resolveAndBind()
-    encoder.fromRow(reader(partitionedFile).next())
+    encoder.createDeserializer().apply(reader(partitionedFile).next())
   }
 
   test("column pruning") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
index 6d5ad873eedea..8d5439534b513 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
@@ -144,16 +144,22 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     }
   }
 
+  private def createToExternalRowConverter[A : Encoder](): A => Row = {
+    val encoder = encoderFor[A]
+    val toInternalRow = encoder.createSerializer()
+    val toExternalRow = RowEncoder(encoder.schema).resolveAndBind().createDeserializer()
+    toExternalRow.compose(toInternalRow)
+  }
+
   /**
    * Checks to make sure that the current data stored in the sink matches the `expectedAnswer`.
    * This operation automatically blocks until all added data has been processed.
    */
   object CheckAnswer {
     def apply[A : Encoder](data: A*): CheckAnswerRows = {
-      val encoder = encoderFor[A]
-      val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
+      val toExternalRow = createToExternalRowConverter[A]()
       CheckAnswerRows(
-        data.map(d => toExternalRow.fromRow(encoder.toRow(d))),
+        data.map(toExternalRow),
         lastOnly = false,
         isSorted = false)
     }
@@ -174,10 +180,9 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     }
 
     def apply[A: Encoder](isSorted: Boolean, data: A*): CheckAnswerRows = {
-      val encoder = encoderFor[A]
-      val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
+      val toExternalRow = createToExternalRowConverter[A]()
       CheckAnswerRows(
-        data.map(d => toExternalRow.fromRow(encoder.toRow(d))),
+        data.map(toExternalRow),
         lastOnly = true,
         isSorted = isSorted)
     }
@@ -215,9 +220,8 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     def apply(): CheckNewAnswerRows = CheckNewAnswerRows(Seq.empty)
 
     def apply[A: Encoder](data: A, moreData: A*): CheckNewAnswerRows = {
-      val encoder = encoderFor[A]
-      val toExternalRow = RowEncoder(encoder.schema).resolveAndBind()
-      CheckNewAnswerRows((data +: moreData).map(d => toExternalRow.fromRow(encoder.toRow(d))))
+      val toExternalRow = createToExternalRowConverter[A]()
+      CheckNewAnswerRows((data +: moreData).map(toExternalRow))
     }
 
     def apply(rows: Row*): CheckNewAnswerRows = CheckNewAnswerRows(rows)

From 33d25ba30b7565d822d252093c5566f95148a483 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 17 Apr 2020 04:48:27 +0000
Subject: [PATCH 0510/1280] [SPARK-31425][SQL][CORE]
 UnsafeKVExternalSorter/VariableLengthRowBasedKeyValueBatch should also
 respect UnsafeAlignedOffset

### What changes were proposed in this pull request?

Make `UnsafeKVExternalSorter` / `VariableLengthRowBasedKeyValueBatch ` also respect `UnsafeAlignedOffset` when reading the record and update some out of date comemnts.

### Why are the changes needed?

Since `BytesToBytesMap` respects `UnsafeAlignedOffset` when writing the record, `UnsafeKVExternalSorter` should also respect `UnsafeAlignedOffset` when reading the record from `BytesToBytesMap` otherwise it will causes data correctness issue.

Unlike `UnsafeKVExternalSorter` may reading records from `BytesToBytesMap`, `VariableLengthRowBasedKeyValueBatch` writes and reads records by itself. Thus, similar to #22053 and [comment](https://github.com/apache/spark/pull/22053#issuecomment-411975239) there, fix for `VariableLengthRowBasedKeyValueBatch` more likely an improvement for the support of SPARC platform.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Manually tested `HashAggregationQueryWithControlledFallbackSuite` with `UAO_SIZE=8`  to simulate SPARC platform. And tests only pass with this fix.

Closes #28195 from Ngone51/fix_uao.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 40f9dbb6284ca6d5664ec0983faba723bc72d7f1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/unsafe/UnsafeAlignedOffset.java     | 14 ++++--
 .../spark/unsafe/map/BytesToBytesMap.java     | 14 +++---
 .../unsafe/sort/UnsafeInMemorySorter.java     |  2 +-
 .../VariableLengthRowBasedKeyValueBatch.java  | 34 +++++++------
 .../sql/execution/UnsafeKVExternalSorter.java | 20 +++++---
 .../execution/AggregationQuerySuite.scala     | 50 +++++++++++--------
 6 files changed, 78 insertions(+), 56 deletions(-)

diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java
index 546e8780a6606..d399e66aa2a5f 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/UnsafeAlignedOffset.java
@@ -28,12 +28,20 @@ public class UnsafeAlignedOffset {
 
   private static final int UAO_SIZE = Platform.unaligned() ? 4 : 8;
 
+  private static int TEST_UAO_SIZE = 0;
+
+  // used for test only
+  public static void setUaoSize(int size) {
+    assert size == 0 || size == 4 || size == 8;
+    TEST_UAO_SIZE = size;
+  }
+
   public static int getUaoSize() {
-    return UAO_SIZE;
+    return TEST_UAO_SIZE == 0 ? UAO_SIZE : TEST_UAO_SIZE;
   }
 
   public static int getSize(Object object, long offset) {
-    switch (UAO_SIZE) {
+    switch (getUaoSize()) {
       case 4:
         return Platform.getInt(object, offset);
       case 8:
@@ -46,7 +54,7 @@ public static int getSize(Object object, long offset) {
   }
 
   public static void putSize(Object object, long offset, int value) {
-    switch (UAO_SIZE) {
+    switch (getUaoSize()) {
       case 4:
         Platform.putInt(object, offset, value);
         break;
diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 7bdd89488d119..b280f7cc35951 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -54,13 +54,13 @@
  * probably be using sorting instead of hashing for better cache locality.
  *
  * The key and values under the hood are stored together, in the following format:
- *   Bytes 0 to 4: len(k) (key length in bytes) + len(v) (value length in bytes) + 4
- *   Bytes 4 to 8: len(k)
- *   Bytes 8 to 8 + len(k): key data
- *   Bytes 8 + len(k) to 8 + len(k) + len(v): value data
- *   Bytes 8 + len(k) + len(v) to 8 + len(k) + len(v) + 8: pointer to next pair
+ *   First uaoSize bytes: len(k) (key length in bytes) + len(v) (value length in bytes) + uaoSize
+ *   Next uaoSize bytes: len(k)
+ *   Next len(k) bytes: key data
+ *   Next len(v) bytes: value data
+ *   Last 8 bytes: pointer to next pair
  *
- * This means that the first four bytes store the entire record (key + value) length. This format
+ * It means first uaoSize bytes store the entire record (key + value + uaoSize) length. This format
  * is compatible with {@link org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter},
  * so we can pass records from this map directly into the sorter to sort records in place.
  */
@@ -707,7 +707,7 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff
       // Here, we'll copy the data into our data pages. Because we only store a relative offset from
       // the key address instead of storing the absolute address of the value, the key and value
       // must be stored in the same memory page.
-      // (8 byte key length) (key) (value) (8 byte pointer to next value)
+      // (total length) (key length) (key) (value) (8 byte pointer to next value)
       int uaoSize = UnsafeAlignedOffset.getUaoSize();
       final long recordLength = (2L * uaoSize) + klen + vlen + 8;
       if (currentPage == null || currentPage.size() - pageCursor < recordLength) {
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
index e14964d68119b..660eb790a550b 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
@@ -235,7 +235,7 @@ public void expandPointerArray(LongArray newArray) {
 
   /**
    * Inserts a record to be sorted. Assumes that the record pointer points to a record length
-   * stored as a 4-byte integer, followed by the record's bytes.
+   * stored as a uaoSize(4 or 8) bytes integer, followed by the record's bytes.
    *
    * @param recordPointer pointer to a record in a data page, encoded by {@link TaskMemoryManager}.
    * @param keyPrefix a user-defined key prefix
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
index c823de4810f2b..4ee913c9bf02d 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/VariableLengthRowBasedKeyValueBatch.java
@@ -19,11 +19,12 @@
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.sql.types.*;
 import org.apache.spark.unsafe.Platform;
+import org.apache.spark.unsafe.UnsafeAlignedOffset;
 
 /**
  * An implementation of `RowBasedKeyValueBatch` in which key-value records have variable lengths.
  *
- *  The format for each record looks like this:
+ *  The format for each record looks like this (in case of uaoSize = 4):
  * [4 bytes total size = (klen + vlen + 4)] [4 bytes key size = klen]
  * [UnsafeRow for key of length klen] [UnsafeRow for Value of length vlen]
  * [8 bytes pointer to next]
@@ -41,7 +42,8 @@ public final class VariableLengthRowBasedKeyValueBatch extends RowBasedKeyValueB
   @Override
   public UnsafeRow appendRow(Object kbase, long koff, int klen,
                              Object vbase, long voff, int vlen) {
-    final long recordLength = 8L + klen + vlen + 8;
+    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final long recordLength = 2 * uaoSize + klen + vlen + 8L;
     // if run out of max supported rows or page size, return null
     if (numRows >= capacity || page == null || page.size() - pageCursor < recordLength) {
       return null;
@@ -49,10 +51,10 @@ public UnsafeRow appendRow(Object kbase, long koff, int klen,
 
     long offset = page.getBaseOffset() + pageCursor;
     final long recordOffset = offset;
-    Platform.putInt(base, offset, klen + vlen + 4);
-    Platform.putInt(base, offset + 4, klen);
+    UnsafeAlignedOffset.putSize(base, offset, klen + vlen + uaoSize);
+    UnsafeAlignedOffset.putSize(base, offset + uaoSize, klen);
 
-    offset += 8;
+    offset += 2 * uaoSize;
     Platform.copyMemory(kbase, koff, base, offset, klen);
     offset += klen;
     Platform.copyMemory(vbase, voff, base, offset, vlen);
@@ -61,11 +63,11 @@ public UnsafeRow appendRow(Object kbase, long koff, int klen,
 
     pageCursor += recordLength;
 
-    keyOffsets[numRows] = recordOffset + 8;
+    keyOffsets[numRows] = recordOffset + 2 * uaoSize;
 
     keyRowId = numRows;
-    keyRow.pointTo(base, recordOffset + 8, klen);
-    valueRow.pointTo(base, recordOffset + 8 + klen, vlen);
+    keyRow.pointTo(base, recordOffset + 2 * uaoSize, klen);
+    valueRow.pointTo(base, recordOffset + 2 * uaoSize + klen, vlen);
     numRows++;
     return valueRow;
   }
@@ -79,7 +81,7 @@ public UnsafeRow getKeyRow(int rowId) {
     assert(rowId < numRows);
     if (keyRowId != rowId) { // if keyRowId == rowId, desired keyRow is already cached
       long offset = keyOffsets[rowId];
-      int klen = Platform.getInt(base, offset - 4);
+      int klen = UnsafeAlignedOffset.getSize(base, offset - UnsafeAlignedOffset.getUaoSize());
       keyRow.pointTo(base, offset, klen);
       // set keyRowId so we can check if desired row is cached
       keyRowId = rowId;
@@ -99,9 +101,10 @@ public UnsafeRow getValueFromKey(int rowId) {
       getKeyRow(rowId);
     }
     assert(rowId >= 0);
+    int uaoSize = UnsafeAlignedOffset.getUaoSize();
     long offset = keyRow.getBaseOffset();
     int klen = keyRow.getSizeInBytes();
-    int vlen = Platform.getInt(base, offset - 8) - klen - 4;
+    int vlen = UnsafeAlignedOffset.getSize(base, offset - uaoSize * 2) - klen - uaoSize;
     valueRow.pointTo(base, offset + klen, vlen);
     return valueRow;
   }
@@ -141,14 +144,15 @@ public boolean next() {
           return false;
         }
 
-        totalLength = Platform.getInt(base, offsetInPage) - 4;
-        currentklen = Platform.getInt(base, offsetInPage + 4);
+        int uaoSize = UnsafeAlignedOffset.getUaoSize();
+        totalLength = UnsafeAlignedOffset.getSize(base, offsetInPage) - uaoSize;
+        currentklen = UnsafeAlignedOffset.getSize(base, offsetInPage + uaoSize);
         currentvlen = totalLength - currentklen;
 
-        key.pointTo(base, offsetInPage + 8, currentklen);
-        value.pointTo(base, offsetInPage + 8 + currentklen, currentvlen);
+        key.pointTo(base, offsetInPage + 2 * uaoSize, currentklen);
+        value.pointTo(base, offsetInPage + 2 * uaoSize + currentklen, currentvlen);
 
-        offsetInPage += 8 + totalLength + 8;
+        offsetInPage += 2 * uaoSize + totalLength + 8;
         recordsInPage -= 1;
         return true;
       }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
index acd54fe25d62d..7a9f61a2cc33d 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
@@ -35,6 +35,7 @@
 import org.apache.spark.storage.BlockManager;
 import org.apache.spark.unsafe.KVIterator;
 import org.apache.spark.unsafe.Platform;
+import org.apache.spark.unsafe.UnsafeAlignedOffset;
 import org.apache.spark.unsafe.array.LongArray;
 import org.apache.spark.unsafe.map.BytesToBytesMap;
 import org.apache.spark.unsafe.memory.MemoryBlock;
@@ -141,9 +142,10 @@ public UnsafeKVExternalSorter(
 
         // Get encoded memory address
         // baseObject + baseOffset point to the beginning of the key data in the map, but that
-        // the KV-pair's length data is stored in the word immediately before that address
+        // the KV-pair's length data is stored at 2 * uaoSize bytes immediately before that address
         MemoryBlock page = loc.getMemoryPage();
-        long address = taskMemoryManager.encodePageNumberAndOffset(page, baseOffset - 8);
+        long address = taskMemoryManager.encodePageNumberAndOffset(page,
+            baseOffset - 2 * UnsafeAlignedOffset.getUaoSize());
 
         // Compute prefix
         row.pointTo(baseObject, baseOffset, loc.getKeyLength());
@@ -262,10 +264,11 @@ public int compare(
         Object baseObj2,
         long baseOff2,
         int baseLen2) {
+      int uaoSize = UnsafeAlignedOffset.getUaoSize();
       // Note that since ordering doesn't need the total length of the record, we just pass 0
       // into the row.
-      row1.pointTo(baseObj1, baseOff1 + 4, 0);
-      row2.pointTo(baseObj2, baseOff2 + 4, 0);
+      row1.pointTo(baseObj1, baseOff1 + uaoSize, 0);
+      row2.pointTo(baseObj2, baseOff2 + uaoSize, 0);
       return ordering.compare(row1, row2);
     }
   }
@@ -289,11 +292,12 @@ public boolean next() throws IOException {
           long recordOffset = underlying.getBaseOffset();
           int recordLen = underlying.getRecordLength();
 
-          // Note that recordLen = keyLen + valueLen + 4 bytes (for the keyLen itself)
+          // Note that recordLen = keyLen + valueLen + uaoSize (for the keyLen itself)
+          int uaoSize = UnsafeAlignedOffset.getUaoSize();
           int keyLen = Platform.getInt(baseObj, recordOffset);
-          int valueLen = recordLen - keyLen - 4;
-          key.pointTo(baseObj, recordOffset + 4, keyLen);
-          value.pointTo(baseObj, recordOffset + 4 + keyLen, valueLen);
+          int valueLen = recordLen - keyLen - uaoSize;
+          key.pointTo(baseObj, recordOffset + uaoSize, keyLen);
+          value.pointTo(baseObj, recordOffset + uaoSize + keyLen, valueLen);
 
           return true;
         } else {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
index f84b854048e8a..ce40a65ed6bac 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.UnsafeAlignedOffset
 
 
 class ScalaAggregateFunction(schema: StructType) extends UserDefinedAggregateFunction {
@@ -1055,30 +1056,35 @@ class HashAggregationQueryWithControlledFallbackSuite extends AggregationQuerySu
     Seq("true", "false").foreach { enableTwoLevelMaps =>
       withSQLConf(SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key ->
         enableTwoLevelMaps) {
-        (1 to 3).foreach { fallbackStartsAt =>
-          withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
-            s"${(fallbackStartsAt - 1).toString}, ${fallbackStartsAt.toString}") {
-            // Create a new df to make sure its physical operator picks up
-            // spark.sql.TungstenAggregate.testFallbackStartsAt.
-            // todo: remove it?
-            val newActual = Dataset.ofRows(spark, actual.logicalPlan)
-
-            QueryTest.getErrorMessageInCheckAnswer(newActual, expectedAnswer) match {
-              case Some(errorMessage) =>
-                val newErrorMessage =
-                  s"""
-                     |The following aggregation query failed when using HashAggregate with
-                     |controlled fallback (it falls back to bytes to bytes map once it has processed
-                     |${fallbackStartsAt - 1} input rows and to sort-based aggregation once it has
-                     |processed $fallbackStartsAt input rows). The query is ${actual.queryExecution}
-                     |
-                    |$errorMessage
-                  """.stripMargin
-
-                fail(newErrorMessage)
-              case None => // Success
+        Seq(4, 8).foreach { uaoSize =>
+          UnsafeAlignedOffset.setUaoSize(uaoSize)
+          (1 to 3).foreach { fallbackStartsAt =>
+            withSQLConf("spark.sql.TungstenAggregate.testFallbackStartsAt" ->
+              s"${(fallbackStartsAt - 1).toString}, ${fallbackStartsAt.toString}") {
+              // Create a new df to make sure its physical operator picks up
+              // spark.sql.TungstenAggregate.testFallbackStartsAt.
+              // todo: remove it?
+              val newActual = Dataset.ofRows(spark, actual.logicalPlan)
+
+              QueryTest.getErrorMessageInCheckAnswer(newActual, expectedAnswer) match {
+                case Some(errorMessage) =>
+                  val newErrorMessage =
+                    s"""
+                       |The following aggregation query failed when using HashAggregate with
+                       |controlled fallback (it falls back to bytes to bytes map once it has
+                       |processed ${fallbackStartsAt - 1} input rows and to sort-based aggregation
+                       |once it has processed $fallbackStartsAt input rows).
+                       |The query is ${actual.queryExecution}
+                       |$errorMessage
+                    """.stripMargin
+
+                  fail(newErrorMessage)
+                case None => // Success
+              }
             }
           }
+          // reset static uaoSize to avoid affect other tests
+          UnsafeAlignedOffset.setUaoSize(0)
         }
       }
     }

From 4a58dde54195fe57b6104586b2e343e92fea24a3 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 17 Apr 2020 13:59:02 +0000
Subject: [PATCH 0511/1280] [SPARK-31469][SQL] Make extract interval field ANSI
 compliance

### What changes were proposed in this pull request?

Currently, we can extract `millennium/century/decade/year/quarter/month/week/day/hour/minute/second(with fractions)//millisecond/microseconds` and `epoch` from interval values

While getting the `millennium/century/decade/year`, it means how many the interval `months` part can be converted to that unit-value. The content of `millennium/century/decade` will overlap `year` and each other.

While getting `month/day` and so on, it means the integral remainder of the previous unit. Here all the units including `year` are individual.

So while extracting `year`, `month`, `day`, `hour`, `minute`, `second`, which are ANSI primary datetime units, the semantic is `extracting`, but others might refer to `transforming`.

While getting epoch we have treat month as 30 days which varies the natural Calendar rules we use.

To avoid ambiguity, I suggest we should only support those extract field defined ANSI with their abbreviations.

### Why are the changes needed?

Extracting `millennium`, `century` etc does not obey the meaning of extracting, and they are not so useful and worth maintaining.

The `extract` is ANSI standard expression and `date_part` is its pg-specific alias function. The current support extract-fields are fully bought from PostgreSQL.

With a look at other systems like Presto/Hive, they don't support those ambiguous fields too.

e.g. Hive 2.2.x also take it from PostgreSQL but without introducing those ambiguous fields https://issues.apache.org/jira/secure/attachment/12828349/HIVE-14579

e.g. presto

```sql
presto> select extract(quater from interval '10-0' year to month);
Query 20200417_094723_00020_m8xq4 failed: line 1:8: Invalid EXTRACT field: quater
select extract(quater from interval '10-0' year to month)

presto> select extract(decade from interval '10-0' year to month);
Query 20200417_094737_00021_m8xq4 failed: line 1:8: Invalid EXTRACT field: decade
select extract(decade from interval '10-0' year to month)

```

### Does this PR introduce any user-facing change?

Yes, as we already have previews versions, this PR will remove support for extracting `millennium/century/decade/quarter/week/millisecond/microseconds` and `epoch` from intervals with `date_part` function

### How was this patch tested?

rm some used tests

Closes #28242 from yaooqinn/SPARK-31469.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 697083c051304a5ac4f9c56a63274f2103caaef4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateTimeConstants.java  |   9 -
 .../expressions/datetimeExpressions.scala     |  19 +-
 .../expressions/intervalExpressions.scala     |  32 ---
 .../sql/catalyst/util/IntervalUtils.scala     |  35 +--
 .../IntervalExpressionsSuite.scala            |  89 -------
 .../catalyst/util/RebaseDateTimeSuite.scala   |   4 +-
 .../resources/sql-tests/inputs/date_part.sql  |  34 ---
 .../resources/sql-tests/inputs/extract.sql    |  58 ++---
 .../sql-tests/results/date_part.sql.out       | 218 +-----------------
 .../sql-tests/results/extract.sql.out         | 162 ++++++-------
 10 files changed, 120 insertions(+), 540 deletions(-)

diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java
index 84a0156ebfb66..0ae238564d591 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/DateTimeConstants.java
@@ -19,15 +19,9 @@
 
 public class DateTimeConstants {
 
-  public static final int YEARS_PER_DECADE = 10;
-  public static final int YEARS_PER_CENTURY = 100;
-  public static final int YEARS_PER_MILLENNIUM = 1000;
-
-  public static final byte MONTHS_PER_QUARTER = 3;
   public static final int MONTHS_PER_YEAR = 12;
 
   public static final byte DAYS_PER_WEEK = 7;
-  public static final long DAYS_PER_MONTH = 30L;
 
   public static final long HOURS_PER_DAY = 24L;
 
@@ -47,9 +41,6 @@ public class DateTimeConstants {
   public static final long MICROS_PER_MINUTE = SECONDS_PER_MINUTE * MICROS_PER_SECOND;
   public static final long MICROS_PER_HOUR = MINUTES_PER_HOUR * MICROS_PER_MINUTE;
   public static final long MICROS_PER_DAY = HOURS_PER_DAY * MICROS_PER_HOUR;
-  public static final long MICROS_PER_MONTH = DAYS_PER_MONTH * MICROS_PER_DAY;
-  /* 365.25 days per year assumes leap year every four years */
-  public static final long MICROS_PER_YEAR = (36525L * MICROS_PER_DAY) / 100;
 
   public static final long NANOS_PER_MICROS = 1000L;
   public static final long NANOS_PER_MILLIS = MICROS_PER_MILLIS * NANOS_PER_MICROS;
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 7ca6ab14ae8c5..391e0d0244990 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -2129,19 +2129,12 @@ object DatePart {
                  "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"),
                  "EPOCH"]
                 Supported string values of `field` for intervals are:
-                 ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"),
-                   "CENTURY", ("CENTURIES", "C", "CENT"),
-                   "DECADE", ("DECADES", "DEC", "DECS"),
-                   "YEAR", ("Y", "YEARS", "YR", "YRS"),
-                   "QUARTER", ("QTR"),
-                   "MONTH", ("MON", "MONS", "MONTHS"),
-                   "DAY", ("D", "DAYS"),
-                   "HOUR", ("H", "HOURS", "HR", "HRS"),
-                   "MINUTE", ("M", "MIN", "MINS", "MINUTES"),
-                   "SECOND", ("S", "SEC", "SECONDS", "SECS"),
-                   "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"),
-                   "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"),
-                   "EPOCH"]
+                 ["YEAR", ("Y", "YEARS", "YR", "YRS"),
+                  "MONTH", ("MON", "MONS", "MONTHS"),
+                  "DAY", ("D", "DAYS"),
+                  "HOUR", ("H", "HOURS", "HR", "HRS"),
+                  "MINUTE", ("M", "MIN", "MINS", "MINUTES"),
+                  "SECOND", ("S", "SEC", "SECONDS", "SECS")]
       * source - a date/timestamp or interval column from where `field` should be extracted
   """,
   examples = """
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
index c09350f33c7f9..1a569a7b89fe1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
@@ -45,21 +45,9 @@ abstract class ExtractIntervalPart(
   }
 }
 
-case class ExtractIntervalMillenniums(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getMillenniums, "getMillenniums")
-
-case class ExtractIntervalCenturies(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getCenturies, "getCenturies")
-
-case class ExtractIntervalDecades(child: Expression)
-  extends ExtractIntervalPart(child, IntegerType, getDecades, "getDecades")
-
 case class ExtractIntervalYears(child: Expression)
   extends ExtractIntervalPart(child, IntegerType, getYears, "getYears")
 
-case class ExtractIntervalQuarters(child: Expression)
-  extends ExtractIntervalPart(child, ByteType, getQuarters, "getQuarters")
-
 case class ExtractIntervalMonths(child: Expression)
   extends ExtractIntervalPart(child, ByteType, getMonths, "getMonths")
 
@@ -75,38 +63,18 @@ case class ExtractIntervalMinutes(child: Expression)
 case class ExtractIntervalSeconds(child: Expression)
   extends ExtractIntervalPart(child, DecimalType(8, 6), getSeconds, "getSeconds")
 
-case class ExtractIntervalMilliseconds(child: Expression)
-  extends ExtractIntervalPart(child, DecimalType(8, 3), getMilliseconds, "getMilliseconds")
-
-case class ExtractIntervalMicroseconds(child: Expression)
-  extends ExtractIntervalPart(child, LongType, getMicroseconds, "getMicroseconds")
-
-// Number of seconds in 10000 years is 315576000001 (30 days per one month)
-// which is 12 digits + 6 digits for the fractional part of seconds.
-case class ExtractIntervalEpoch(child: Expression)
-  extends ExtractIntervalPart(child, DecimalType(18, 6), getEpoch, "getEpoch")
-
 object ExtractIntervalPart {
 
   def parseExtractField(
       extractField: String,
       source: Expression,
       errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match {
-    case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => ExtractIntervalMillenniums(source)
-    case "CENTURY" | "CENTURIES" | "C" | "CENT" => ExtractIntervalCenturies(source)
-    case "DECADE" | "DECADES" | "DEC" | "DECS" => ExtractIntervalDecades(source)
     case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => ExtractIntervalYears(source)
-    case "QUARTER" | "QTR" => ExtractIntervalQuarters(source)
     case "MONTH" | "MON" | "MONS" | "MONTHS" => ExtractIntervalMonths(source)
     case "DAY" | "D" | "DAYS" => ExtractIntervalDays(source)
     case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => ExtractIntervalHours(source)
     case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => ExtractIntervalMinutes(source)
     case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => ExtractIntervalSeconds(source)
-    case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" =>
-      ExtractIntervalMilliseconds(source)
-    case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" =>
-      ExtractIntervalMicroseconds(source)
-    case "EPOCH" => ExtractIntervalEpoch(source)
     case _ => errorHandleFunc
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index ccf8c5ed3603e..3afa0c7c089aa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -50,26 +50,10 @@ object IntervalUtils {
     interval.months / MONTHS_PER_YEAR
   }
 
-  def getMillenniums(interval: CalendarInterval): Int = {
-    getYears(interval) / YEARS_PER_MILLENNIUM
-  }
-
-  def getCenturies(interval: CalendarInterval): Int = {
-    getYears(interval) / YEARS_PER_CENTURY
-  }
-
-  def getDecades(interval: CalendarInterval): Int = {
-    getYears(interval) / YEARS_PER_DECADE
-  }
-
   def getMonths(interval: CalendarInterval): Byte = {
     (interval.months % MONTHS_PER_YEAR).toByte
   }
 
-  def getQuarters(interval: CalendarInterval): Byte = {
-    (getMonths(interval) / MONTHS_PER_QUARTER + 1).toByte
-  }
-
   def getDays(interval: CalendarInterval): Int = {
     interval.days
   }
@@ -82,25 +66,8 @@ object IntervalUtils {
     ((interval.microseconds % MICROS_PER_HOUR) / MICROS_PER_MINUTE).toByte
   }
 
-  def getMicroseconds(interval: CalendarInterval): Long = {
-    interval.microseconds % MICROS_PER_MINUTE
-  }
-
   def getSeconds(interval: CalendarInterval): Decimal = {
-    Decimal(getMicroseconds(interval), 8, 6)
-  }
-
-  def getMilliseconds(interval: CalendarInterval): Decimal = {
-    Decimal(getMicroseconds(interval), 8, 3)
-  }
-
-  // Returns total number of seconds with microseconds fractional part in the given interval.
-  def getEpoch(interval: CalendarInterval): Decimal = {
-    var result = interval.microseconds
-    result += MICROS_PER_DAY * interval.days
-    result += MICROS_PER_YEAR * (interval.months / MONTHS_PER_YEAR)
-    result += MICROS_PER_MONTH * (interval.months % MONTHS_PER_YEAR)
-    Decimal(result, 18, 6)
+    Decimal(interval.microseconds % MICROS_PER_MINUTE, 8, 6)
   }
 
   private def toLongWithRange(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index e591c4984a7bf..a2ee1c6ef2793 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -34,42 +34,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     Literal(stringToInterval( "interval " + s))
   }
 
-  test("millenniums") {
-    checkEvaluation(ExtractIntervalMillenniums("0 years"), 0)
-    checkEvaluation(ExtractIntervalMillenniums("9999 years"), 9)
-    checkEvaluation(ExtractIntervalMillenniums("1000 years"), 1)
-    checkEvaluation(ExtractIntervalMillenniums("-2000 years"), -2)
-    // Microseconds part must not be taken into account
-    checkEvaluation(ExtractIntervalMillenniums("999 years 400 days"), 0)
-    // Millennium must be taken from years and months
-    checkEvaluation(ExtractIntervalMillenniums("999 years 12 months"), 1)
-    checkEvaluation(ExtractIntervalMillenniums("1000 years -1 months"), 0)
-  }
-
-  test("centuries") {
-    checkEvaluation(ExtractIntervalCenturies("0 years"), 0)
-    checkEvaluation(ExtractIntervalCenturies("9999 years"), 99)
-    checkEvaluation(ExtractIntervalCenturies("1000 years"), 10)
-    checkEvaluation(ExtractIntervalCenturies("-2000 years"), -20)
-    // Microseconds part must not be taken into account
-    checkEvaluation(ExtractIntervalCenturies("99 years 400 days"), 0)
-    // Century must be taken from years and months
-    checkEvaluation(ExtractIntervalCenturies("99 years 12 months"), 1)
-    checkEvaluation(ExtractIntervalCenturies("100 years -1 months"), 0)
-  }
-
-  test("decades") {
-    checkEvaluation(ExtractIntervalDecades("0 years"), 0)
-    checkEvaluation(ExtractIntervalDecades("9999 years"), 999)
-    checkEvaluation(ExtractIntervalDecades("1000 years"), 100)
-    checkEvaluation(ExtractIntervalDecades("-2000 years"), -200)
-    // Microseconds part must not be taken into account
-    checkEvaluation(ExtractIntervalDecades("9 years 400 days"), 0)
-    // Decade must be taken from years and months
-    checkEvaluation(ExtractIntervalDecades("9 years 12 months"), 1)
-    checkEvaluation(ExtractIntervalDecades("10 years -1 months"), 0)
-  }
-
   test("years") {
     checkEvaluation(ExtractIntervalYears("0 years"), 0)
     checkEvaluation(ExtractIntervalYears("9999 years"), 9999)
@@ -82,19 +46,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(ExtractIntervalYears("10 years -1 months"), 9)
   }
 
-  test("quarters") {
-    checkEvaluation(ExtractIntervalQuarters("0 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("1 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-1 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("2 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-2 months"), 1.toByte)
-    checkEvaluation(ExtractIntervalQuarters("1 years -1 months"), 4.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-1 years 1 months"), -2.toByte)
-    checkEvaluation(ExtractIntervalQuarters("2 years 3 months"), 2.toByte)
-    checkEvaluation(ExtractIntervalQuarters("-2 years -3 months"), 0.toByte)
-    checkEvaluation(ExtractIntervalQuarters("9999 years"), 1.toByte)
-  }
-
   test("months") {
     checkEvaluation(ExtractIntervalMonths("0 year"), 0.toByte)
     for (m <- -24 to 24) {
@@ -158,46 +109,6 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(ExtractIntervalSeconds("61 seconds 1 microseconds"), Decimal(1000001, 8, 6))
   }
 
-  test("milliseconds") {
-    checkEvaluation(ExtractIntervalMilliseconds("0 milliseconds"), Decimal(0, 8, 3))
-    checkEvaluation(ExtractIntervalMilliseconds("1 milliseconds"), Decimal(1.0, 8, 3))
-    checkEvaluation(ExtractIntervalMilliseconds("-1 milliseconds"), Decimal(-1.0, 8, 3))
-    checkEvaluation(
-      ExtractIntervalMilliseconds("1 second 999 milliseconds"),
-      Decimal(1999.0, 8, 3))
-    checkEvaluation(
-      ExtractIntervalMilliseconds("999 milliseconds 1 microsecond"),
-      Decimal(999.001, 8, 3))
-    checkEvaluation(
-      ExtractIntervalMilliseconds("-1 second -999 milliseconds"),
-      Decimal(-1999.0, 8, 3))
-    // Years and months must not be taken into account
-    checkEvaluation(ExtractIntervalMilliseconds("100 year 1 millisecond"), Decimal(1.0, 8, 3))
-    checkEvaluation(ExtractIntervalMilliseconds(largeInterval), Decimal(59999.999, 8, 3))
-  }
-
-  test("microseconds") {
-    checkEvaluation(ExtractIntervalMicroseconds("0 microseconds"), 0L)
-    checkEvaluation(ExtractIntervalMicroseconds("1 microseconds"), 1L)
-    checkEvaluation(ExtractIntervalMicroseconds("-1 microseconds"), -1L)
-    checkEvaluation(ExtractIntervalMicroseconds("1 second 999 microseconds"), 1000999L)
-    checkEvaluation(ExtractIntervalMicroseconds("999 milliseconds 1 microseconds"), 999001L)
-    checkEvaluation(ExtractIntervalMicroseconds("-1 second -999 microseconds"), -1000999L)
-    // Years and months must not be taken into account
-    checkEvaluation(ExtractIntervalMicroseconds("11 year 1 microseconds"), 1L)
-    checkEvaluation(ExtractIntervalMicroseconds(largeInterval), 59999999L)
-  }
-
-  test("epoch") {
-    checkEvaluation(ExtractIntervalEpoch("0 months"), Decimal(0.0, 18, 6))
-    checkEvaluation(ExtractIntervalEpoch("10000 years"), Decimal(315576000000.0, 18, 6))
-    checkEvaluation(ExtractIntervalEpoch("1 year"), Decimal(31557600.0, 18, 6))
-    checkEvaluation(ExtractIntervalEpoch("-1 year"), Decimal(-31557600.0, 18, 6))
-    checkEvaluation(
-      ExtractIntervalEpoch("1 second 1 millisecond 1 microsecond"),
-      Decimal(1.001001, 18, 6))
-  }
-
   test("multiply") {
     def check(
         interval: String,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index 82ccd97721656..2f957ef3cdfcb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -213,7 +213,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
             val rebased = rebaseGregorianToJulianMicros(zid, micros)
             val rebasedAndOptimized = rebaseGregorianToJulianMicros(micros)
             assert(rebasedAndOptimized === rebased)
-            micros += (MICROS_PER_MONTH * (0.5 + Math.random())).toLong
+            micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
           } while (micros <= end)
         }
       }
@@ -233,7 +233,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
             val rebased = rebaseJulianToGregorianMicros(zid, micros)
             val rebasedAndOptimized = rebaseJulianToGregorianMicros(micros)
             assert(rebasedAndOptimized === rebased)
-            micros += (MICROS_PER_MONTH * (0.5 + Math.random())).toLong
+            micros += (MICROS_PER_DAY * 30 * (0.5 + Math.random())).toLong
           } while (micros <= end)
         }
       }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
index a63cdafb745a0..b36b98b261365 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
@@ -71,30 +71,12 @@ select date_part(null, c) from t;
 
 CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c;
 
-select date_part('millennium', c) from t2;
-select date_part('millennia', c) from t2;
-select date_part('mil', c) from t2;
-select date_part('mils', c) from t2;
-
-select date_part('century', c) from t2;
-select date_part('centuries', c) from t2;
-select date_part('c', c) from t2;
-select date_part('cent', c) from t2;
-
-select date_part('decade', c) from t2;
-select date_part('decades', c) from t2;
-select date_part('dec', c) from t2;
-select date_part('decs', c) from t2;
-
 select date_part('year', c) from t2;
 select date_part('y', c) from t2;
 select date_part('years', c) from t2;
 select date_part('yr', c) from t2;
 select date_part('yrs', c) from t2;
 
-select date_part('quarter', c) from t2;
-select date_part('qtr', c) from t2;
-
 select date_part('month', c) from t2;
 select date_part('mon', c) from t2;
 select date_part('mons', c) from t2;
@@ -122,22 +104,6 @@ select date_part('sec', c) from t2;
 select date_part('seconds', c) from t2;
 select date_part('secs', c) from t2;
 
-select date_part('milliseconds', c) from t2;
-select date_part('msec', c) from t2;
-select date_part('msecs', c) from t2;
-select date_part('millisecon', c) from t2;
-select date_part('mseconds', c) from t2;
-select date_part('ms', c) from t2;
-
-select date_part('microseconds', c) from t2;
-select date_part('usec', c) from t2;
-select date_part('usecs', c) from t2;
-select date_part('useconds', c) from t2;
-select date_part('microsecon', c) from t2;
-select date_part('us', c) from t2;
-
-select date_part('epoch', c) from t2;
-
 select date_part('not_supported', c) from t2;
 
 select date_part(c, c) from t2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
index 4bcc15c875f57..12068d22a4e60 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
@@ -1,19 +1,19 @@
 CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c, interval 10 year 20 month 30 day 40 hour 50 minute 6.7890 second as i;
 
-select extract(millennium from c), extract(millennium from i) from t;
-select extract(millennia from c), extract(millennia from i) from t;
-select extract(mil from c), extract(mil from i) from t;
-select extract(mils from c), extract(mils from i) from t;
+select extract(millennium from c) from t;
+select extract(millennia from c) from t;
+select extract(mil from c) from t;
+select extract(mils from c) from t;
 
-select extract(century from c), extract(century from i) from t;
-select extract(centuries from c), extract(centuries from i) from t;
-select extract(c from c),  extract(c from i) from t;
-select extract(cent from c), extract(cent from i) from t;
+select extract(century from c) from t;
+select extract(centuries from c) from t;
+select extract(c from c) from t;
+select extract(cent from c)from t;
 
-select extract(decade from c), extract(decade from i) from t;
-select extract(decades from c), extract(decades from i) from t;
-select extract(dec from c), extract(dec from i) from t;
-select extract(decs from c), extract(decs from i) from t;
+select extract(decade from c) from t;
+select extract(decades from c) from t;
+select extract(dec from c) from t;
+select extract(decs from c) from t;
 
 select extract(year from c), extract(year from i) from t;
 select extract(y from c), extract(y from i) from t;
@@ -23,8 +23,8 @@ select extract(yrs from c), extract(yrs from i) from t;
 
 select extract(isoyear from c) from t;
 
-select extract(quarter from c), extract(quarter from i) from t;
-select extract(qtr from c), extract(qtr from i) from t;
+select extract(quarter from c) from t;
+select extract(qtr from c) from t;
 
 select extract(month from c), extract(month from i) from t;
 select extract(mon from c), extract(mon from i) from t;
@@ -65,21 +65,21 @@ select extract(sec from c), extract(sec from i) from t;
 select extract(seconds from c), extract(seconds from i) from t;
 select extract(secs from c), extract(secs from i) from t;
 
-select extract(milliseconds from c), extract(milliseconds from i) from t;
-select extract(msec from c), extract(msec from i) from t;
-select extract(msecs from c), extract(msecs from i) from t;
-select extract(millisecon from c), extract(millisecon from i) from t;
-select extract(mseconds from c), extract(mseconds from i) from t;
-select extract(ms from c), extract(ms from i) from t;
-
-select extract(microseconds from c), extract(microseconds from i) from t;
-select extract(usec from c), extract(usec from i) from t;
-select extract(usecs from c), extract(usecs from i) from t;
-select extract(useconds from c), extract(useconds from i) from t;
-select extract(microsecon from c), extract(microsecon from i) from t;
-select extract(us from c), extract(us from i) from t;
-
-select extract(epoch from c), extract(epoch from i) from t;
+select extract(milliseconds from c) from t;
+select extract(msec from c) from t;
+select extract(msecs from c) from t;
+select extract(millisecon from c) from t;
+select extract(mseconds from c) from t;
+select extract(ms from c) from t;
+
+select extract(microseconds from c) from t;
+select extract(usec from c) from t;
+select extract(usecs from c) from t;
+select extract(useconds from c) from t;
+select extract(microsecon from c) from t;
+select extract(us from c) from t;
+
+select extract(epoch from c) from t;
 
 select extract(not_supported from c) from t;
 select extract(not_supported from i) from t;
diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
index b4cceedffd98b..702ac17f7a4e5 100644
--- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 110
+-- Number of queries: 83
 
 
 -- !query
@@ -428,102 +428,6 @@ struct<>
 
 
 
--- !query
-select date_part('millennium', c) from t2
--- !query schema
-struct<date_part('millennium', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('millennia', c) from t2
--- !query schema
-struct<date_part('millennia', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('mil', c) from t2
--- !query schema
-struct<date_part('mil', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('mils', c) from t2
--- !query schema
-struct<date_part('mils', t2.`c`):int>
--- !query output
-1
-
-
--- !query
-select date_part('century', c) from t2
--- !query schema
-struct<date_part('century', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('centuries', c) from t2
--- !query schema
-struct<date_part('centuries', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('c', c) from t2
--- !query schema
-struct<date_part('c', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('cent', c) from t2
--- !query schema
-struct<date_part('cent', t2.`c`):int>
--- !query output
-10
-
-
--- !query
-select date_part('decade', c) from t2
--- !query schema
-struct<date_part('decade', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('decades', c) from t2
--- !query schema
-struct<date_part('decades', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('dec', c) from t2
--- !query schema
-struct<date_part('dec', t2.`c`):int>
--- !query output
-101
-
-
--- !query
-select date_part('decs', c) from t2
--- !query schema
-struct<date_part('decs', t2.`c`):int>
--- !query output
-101
-
-
 -- !query
 select date_part('year', c) from t2
 -- !query schema
@@ -564,22 +468,6 @@ struct<date_part('yrs', t2.`c`):int>
 1010
 
 
--- !query
-select date_part('quarter', c) from t2
--- !query schema
-struct<date_part('quarter', t2.`c`):tinyint>
--- !query output
-4
-
-
--- !query
-select date_part('qtr', c) from t2
--- !query schema
-struct<date_part('qtr', t2.`c`):tinyint>
--- !query output
-4
-
-
 -- !query
 select date_part('month', c) from t2
 -- !query schema
@@ -756,110 +644,6 @@ struct<date_part('secs', t2.`c`):decimal(8,6)>
 5.004003
 
 
--- !query
-select date_part('milliseconds', c) from t2
--- !query schema
-struct<date_part('milliseconds', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('msec', c) from t2
--- !query schema
-struct<date_part('msec', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('msecs', c) from t2
--- !query schema
-struct<date_part('msecs', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('millisecon', c) from t2
--- !query schema
-struct<date_part('millisecon', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('mseconds', c) from t2
--- !query schema
-struct<date_part('mseconds', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('ms', c) from t2
--- !query schema
-struct<date_part('ms', t2.`c`):decimal(8,3)>
--- !query output
-5004.003
-
-
--- !query
-select date_part('microseconds', c) from t2
--- !query schema
-struct<date_part('microseconds', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('usec', c) from t2
--- !query schema
-struct<date_part('usec', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('usecs', c) from t2
--- !query schema
-struct<date_part('usecs', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('useconds', c) from t2
--- !query schema
-struct<date_part('useconds', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('microsecon', c) from t2
--- !query schema
-struct<date_part('microsecon', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('us', c) from t2
--- !query schema
-struct<date_part('us', t2.`c`):bigint>
--- !query output
-5004003
-
-
--- !query
-select date_part('epoch', c) from t2
--- !query schema
-struct<date_part('epoch', t2.`c`):decimal(18,6)>
--- !query output
-31897220765.004003
-
-
 -- !query
 select date_part('not_supported', c) from t2
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 38c8490694088..1f77c67871ae8 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -11,99 +11,99 @@ struct<>
 
 
 -- !query
-select extract(millennium from c), extract(millennium from i) from t
+select extract(millennium from c) from t
 -- !query schema
-struct<date_part('millennium', t.`c`):int,date_part('millennium', t.`i`):int>
+struct<date_part('millennium', t.`c`):int>
 -- !query output
-3	0
+3
 
 
 -- !query
-select extract(millennia from c), extract(millennia from i) from t
+select extract(millennia from c) from t
 -- !query schema
-struct<date_part('millennia', t.`c`):int,date_part('millennia', t.`i`):int>
+struct<date_part('millennia', t.`c`):int>
 -- !query output
-3	0
+3
 
 
 -- !query
-select extract(mil from c), extract(mil from i) from t
+select extract(mil from c) from t
 -- !query schema
-struct<date_part('mil', t.`c`):int,date_part('mil', t.`i`):int>
+struct<date_part('mil', t.`c`):int>
 -- !query output
-3	0
+3
 
 
 -- !query
-select extract(mils from c), extract(mils from i) from t
+select extract(mils from c) from t
 -- !query schema
-struct<date_part('mils', t.`c`):int,date_part('mils', t.`i`):int>
+struct<date_part('mils', t.`c`):int>
 -- !query output
-3	0
+3
 
 
 -- !query
-select extract(century from c), extract(century from i) from t
+select extract(century from c) from t
 -- !query schema
-struct<date_part('century', t.`c`):int,date_part('century', t.`i`):int>
+struct<date_part('century', t.`c`):int>
 -- !query output
-21	0
+21
 
 
 -- !query
-select extract(centuries from c), extract(centuries from i) from t
+select extract(centuries from c) from t
 -- !query schema
-struct<date_part('centuries', t.`c`):int,date_part('centuries', t.`i`):int>
+struct<date_part('centuries', t.`c`):int>
 -- !query output
-21	0
+21
 
 
 -- !query
-select extract(c from c),  extract(c from i) from t
+select extract(c from c) from t
 -- !query schema
-struct<date_part('c', t.`c`):int,date_part('c', t.`i`):int>
+struct<date_part('c', t.`c`):int>
 -- !query output
-21	0
+21
 
 
 -- !query
-select extract(cent from c), extract(cent from i) from t
+select extract(cent from c)from t
 -- !query schema
-struct<date_part('cent', t.`c`):int,date_part('cent', t.`i`):int>
+struct<date_part('cent', t.`c`):int>
 -- !query output
-21	0
+21
 
 
 -- !query
-select extract(decade from c), extract(decade from i) from t
+select extract(decade from c) from t
 -- !query schema
-struct<date_part('decade', t.`c`):int,date_part('decade', t.`i`):int>
+struct<date_part('decade', t.`c`):int>
 -- !query output
-201	1
+201
 
 
 -- !query
-select extract(decades from c), extract(decades from i) from t
+select extract(decades from c) from t
 -- !query schema
-struct<date_part('decades', t.`c`):int,date_part('decades', t.`i`):int>
+struct<date_part('decades', t.`c`):int>
 -- !query output
-201	1
+201
 
 
 -- !query
-select extract(dec from c), extract(dec from i) from t
+select extract(dec from c) from t
 -- !query schema
-struct<date_part('dec', t.`c`):int,date_part('dec', t.`i`):int>
+struct<date_part('dec', t.`c`):int>
 -- !query output
-201	1
+201
 
 
 -- !query
-select extract(decs from c), extract(decs from i) from t
+select extract(decs from c) from t
 -- !query schema
-struct<date_part('decs', t.`c`):int,date_part('decs', t.`i`):int>
+struct<date_part('decs', t.`c`):int>
 -- !query output
-201	1
+201
 
 
 -- !query
@@ -155,19 +155,19 @@ struct<date_part('isoyear', t.`c`):int>
 
 
 -- !query
-select extract(quarter from c), extract(quarter from i) from t
+select extract(quarter from c) from t
 -- !query schema
-struct<date_part('quarter', t.`c`):int,date_part('quarter', t.`i`):tinyint>
+struct<date_part('quarter', t.`c`):int>
 -- !query output
-2	3
+2
 
 
 -- !query
-select extract(qtr from c), extract(qtr from i) from t
+select extract(qtr from c) from t
 -- !query schema
-struct<date_part('qtr', t.`c`):int,date_part('qtr', t.`i`):tinyint>
+struct<date_part('qtr', t.`c`):int>
 -- !query output
-2	3
+2
 
 
 -- !query
@@ -403,107 +403,107 @@ struct<date_part('secs', t.`c`):decimal(8,6),date_part('secs', t.`i`):decimal(8,
 
 
 -- !query
-select extract(milliseconds from c), extract(milliseconds from i) from t
+select extract(milliseconds from c) from t
 -- !query schema
-struct<date_part('milliseconds', t.`c`):decimal(8,3),date_part('milliseconds', t.`i`):decimal(8,3)>
+struct<date_part('milliseconds', t.`c`):decimal(8,3)>
 -- !query output
-9123.456	6789.000
+9123.456
 
 
 -- !query
-select extract(msec from c), extract(msec from i) from t
+select extract(msec from c) from t
 -- !query schema
-struct<date_part('msec', t.`c`):decimal(8,3),date_part('msec', t.`i`):decimal(8,3)>
+struct<date_part('msec', t.`c`):decimal(8,3)>
 -- !query output
-9123.456	6789.000
+9123.456
 
 
 -- !query
-select extract(msecs from c), extract(msecs from i) from t
+select extract(msecs from c) from t
 -- !query schema
-struct<date_part('msecs', t.`c`):decimal(8,3),date_part('msecs', t.`i`):decimal(8,3)>
+struct<date_part('msecs', t.`c`):decimal(8,3)>
 -- !query output
-9123.456	6789.000
+9123.456
 
 
 -- !query
-select extract(millisecon from c), extract(millisecon from i) from t
+select extract(millisecon from c) from t
 -- !query schema
-struct<date_part('millisecon', t.`c`):decimal(8,3),date_part('millisecon', t.`i`):decimal(8,3)>
+struct<date_part('millisecon', t.`c`):decimal(8,3)>
 -- !query output
-9123.456	6789.000
+9123.456
 
 
 -- !query
-select extract(mseconds from c), extract(mseconds from i) from t
+select extract(mseconds from c) from t
 -- !query schema
-struct<date_part('mseconds', t.`c`):decimal(8,3),date_part('mseconds', t.`i`):decimal(8,3)>
+struct<date_part('mseconds', t.`c`):decimal(8,3)>
 -- !query output
-9123.456	6789.000
+9123.456
 
 
 -- !query
-select extract(ms from c), extract(ms from i) from t
+select extract(ms from c) from t
 -- !query schema
-struct<date_part('ms', t.`c`):decimal(8,3),date_part('ms', t.`i`):decimal(8,3)>
+struct<date_part('ms', t.`c`):decimal(8,3)>
 -- !query output
-9123.456	6789.000
+9123.456
 
 
 -- !query
-select extract(microseconds from c), extract(microseconds from i) from t
+select extract(microseconds from c) from t
 -- !query schema
-struct<date_part('microseconds', t.`c`):int,date_part('microseconds', t.`i`):bigint>
+struct<date_part('microseconds', t.`c`):int>
 -- !query output
-9123456	6789000
+9123456
 
 
 -- !query
-select extract(usec from c), extract(usec from i) from t
+select extract(usec from c) from t
 -- !query schema
-struct<date_part('usec', t.`c`):int,date_part('usec', t.`i`):bigint>
+struct<date_part('usec', t.`c`):int>
 -- !query output
-9123456	6789000
+9123456
 
 
 -- !query
-select extract(usecs from c), extract(usecs from i) from t
+select extract(usecs from c) from t
 -- !query schema
-struct<date_part('usecs', t.`c`):int,date_part('usecs', t.`i`):bigint>
+struct<date_part('usecs', t.`c`):int>
 -- !query output
-9123456	6789000
+9123456
 
 
 -- !query
-select extract(useconds from c), extract(useconds from i) from t
+select extract(useconds from c) from t
 -- !query schema
-struct<date_part('useconds', t.`c`):int,date_part('useconds', t.`i`):bigint>
+struct<date_part('useconds', t.`c`):int>
 -- !query output
-9123456	6789000
+9123456
 
 
 -- !query
-select extract(microsecon from c), extract(microsecon from i) from t
+select extract(microsecon from c) from t
 -- !query schema
-struct<date_part('microsecon', t.`c`):int,date_part('microsecon', t.`i`):bigint>
+struct<date_part('microsecon', t.`c`):int>
 -- !query output
-9123456	6789000
+9123456
 
 
 -- !query
-select extract(us from c), extract(us from i) from t
+select extract(us from c) from t
 -- !query schema
-struct<date_part('us', t.`c`):int,date_part('us', t.`i`):bigint>
+struct<date_part('us', t.`c`):int>
 -- !query output
-9123456	6789000
+9123456
 
 
 -- !query
-select extract(epoch from c), extract(epoch from i) from t
+select extract(epoch from c) from t
 -- !query schema
-struct<date_part('epoch', t.`c`):decimal(20,6),date_part('epoch', t.`i`):decimal(18,6)>
+struct<date_part('epoch', t.`c`):decimal(20,6)>
 -- !query output
-1304665689.123456	370608606.789000
+1304665689.123456
 
 
 -- !query

From 6be5bd88ef134fd1c1d0bff9d9784498542dd796 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Fri, 17 Apr 2020 14:11:17 +0000
Subject: [PATCH 0512/1280] [SPARK-31468][SQL] Null types should be implicitly
 casted to Decimal types

### What changes were proposed in this pull request?

This PR intends to fix a bug that occurs when comparing null types to decimal types in master/branch-3.0;
```
scala> Seq(BigDecimal(10)).toDF("v1").selectExpr("v1 = NULL").explain(true)
org.apache.spark.sql.AnalysisException: cannot resolve '(`v1` = NULL)' due to data type mismatch: differing types in '(`v1` = NULL)' (decimal(38,18) and null).; line 1 pos 0;
'Project [(v1#5 = null) AS (v1 = NULL)#7]
+- Project [value#2 AS v1#5]
   +- LocalRelation [value#2]
...
```
The query above passed in v2.4.5.

### Why are the changes needed?

bugfix

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #28241 from maropu/SPARK-31468.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a7fb330ed38e79b91a4dc156b977a5bd36e74010)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/TypeCoercion.scala  | 19 +++++++++++++++----
 .../catalyst/analysis/TypeCoercionSuite.scala | 18 ++++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index eb9a4d4feb783..c6e3f56766a8a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -842,15 +842,26 @@ object TypeCoercion {
    * Casts types according to the expected input types for [[Expression]]s.
    */
   object ImplicitTypeCasts extends TypeCoercionRule {
+
+    private def canHandleTypeCoercion(leftType: DataType, rightType: DataType): Boolean = {
+      (leftType, rightType) match {
+        case (_: DecimalType, NullType) => true
+        case (NullType, _: DecimalType) => true
+        case _ =>
+          // If DecimalType operands are involved except for the two cases above,
+          // DecimalPrecision will handle it.
+          !leftType.isInstanceOf[DecimalType] && !rightType.isInstanceOf[DecimalType] &&
+            leftType != rightType
+      }
+    }
+
     override protected def coerceTypes(
         plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
       // Skip nodes who's children have not been resolved yet.
       case e if !e.childrenResolved => e
 
-      // If DecimalType operands are involved, DecimalPrecision will handle it
-      case b @ BinaryOperator(left, right) if !left.dataType.isInstanceOf[DecimalType] &&
-          !right.dataType.isInstanceOf[DecimalType] &&
-          left.dataType != right.dataType =>
+      case b @ BinaryOperator(left, right)
+          if canHandleTypeCoercion(left.dataType, right.dataType) =>
         findTightestCommonType(left.dataType, right.dataType).map { commonType =>
           if (b.inputType.acceptsType(commonType)) {
             // If the expression accepts the tightest common type, cast to that.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index ab21a9ea5ba18..e37555f1c0ec3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -1541,6 +1541,24 @@ class TypeCoercionSuite extends AnalysisTest {
       Multiply(CaseWhen(Seq((EqualTo(1, 2), Cast(1, DecimalType(34, 24)))),
         Cast(100, DecimalType(34, 24))), Cast(1, IntegerType)))
   }
+
+  test("SPARK-31468: null types should be casted to decimal types in ImplicitTypeCasts") {
+    Seq(AnyTypeBinaryOperator(_, _), NumericTypeBinaryOperator(_, _)).foreach { binaryOp =>
+      // binaryOp(decimal, null) case
+      ruleTest(TypeCoercion.ImplicitTypeCasts,
+        binaryOp(Literal.create(null, DecimalType.SYSTEM_DEFAULT),
+          Literal.create(null, NullType)),
+        binaryOp(Literal.create(null, DecimalType.SYSTEM_DEFAULT),
+          Cast(Literal.create(null, NullType), DecimalType.SYSTEM_DEFAULT)))
+
+      // binaryOp(null, decimal) case
+      ruleTest(TypeCoercion.ImplicitTypeCasts,
+        binaryOp(Literal.create(null, NullType),
+          Literal.create(null, DecimalType.SYSTEM_DEFAULT)),
+        binaryOp(Cast(Literal.create(null, NullType), DecimalType.SYSTEM_DEFAULT),
+          Literal.create(null, DecimalType.SYSTEM_DEFAULT)))
+    }
+  }
 }
 
 

From eae33f97c60961db08bfa8070d91e137aaad2e77 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 17 Apr 2020 13:43:49 -0700
Subject: [PATCH 0513/1280] [SPARK-31446][WEBUI][3.0] Make html elements for a
 paged table possible to have different id attribute

### What changes were proposed in this pull request?

This PR backports #28217 to `branch-3.0`, making each id attribute for page navigations in a page unique.

PagedTable#pageNavigation returns HTML elements representing a page navigation for a paged table.
In the current implementation, the method generates an id and it's used for id attribute for a set of elements for the page navigation.
But some pages have two page navigations so there are two set of elements where corresponding elements have the same id.
For example, there are two form-completedJob-table-page id in JobsPage.

### Why are the changes needed?

Each id attribute should be unique in a page.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

I added a test case for pageNavigation extended.
I also manually tested that there were no warning messages for the uniqueness in JobsPage and JobPage.

Closes #28246 from sarutak/backporting-SPARK-31446-branch-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/ui/PagedTable.scala      | 20 ++++++++-----
 .../org/apache/spark/ui/PagedTableSuite.scala | 29 +++++++++++++++++++
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
index 9a3cd53076c43..8ce2065de4cf8 100644
--- a/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/PagedTable.scala
@@ -115,17 +115,18 @@ private[spark] trait PagedTable[T] {
         _dataSource.pageSize
       }
 
-      val pageNavi = pageNavigation(pageToShow, pageSize, totalPages)
+      val pageNaviTop = pageNavigation(pageToShow, pageSize, totalPages, tableId + "-top")
+      val pageNaviBottom = pageNavigation(pageToShow, pageSize, totalPages, tableId + "-bottom")
 
       <div>
-        {pageNavi}
+        {pageNaviTop}
         <table class={tableCssClass} id={tableId}>
           {headers}
           <tbody>
             {data.map(row)}
           </tbody>
         </table>
-        {pageNavi}
+        {pageNaviBottom}
       </div>
     } catch {
       case e: IndexOutOfBoundsException =>
@@ -171,7 +172,11 @@ private[spark] trait PagedTable[T] {
    * > means jumping to the next page.
    * }}}
    */
-  private[ui] def pageNavigation(page: Int, pageSize: Int, totalPages: Int): Seq[Node] = {
+  private[ui] def pageNavigation(
+      page: Int,
+      pageSize: Int,
+      totalPages: Int,
+      navigationId: String = tableId): Seq[Node] = {
     // A group includes all page numbers will be shown in the page navigation.
     // The size of group is 10 means there are 10 page numbers will be shown.
     // The first group is 1 to 10, the second is 2 to 20, and so on
@@ -214,7 +219,7 @@ private[spark] trait PagedTable[T] {
 
     <div>
       <div>
-        <form id={s"form-$tableId-page"}
+        <form id={s"form-$navigationId-page"}
               method="get"
               action={Unparsed(goButtonFormPath)}
               class="form-inline pull-right"
@@ -223,12 +228,11 @@ private[spark] trait PagedTable[T] {
           <label>{totalPages} Pages. Jump to</label>
           <input type="text"
                  name={pageNumberFormField}
-                 id={s"form-$tableId-page-no"}
+                 id={s"form-$navigationId-page-no"}
                  value={page.toString} class="span1" />
-
           <label>. Show </label>
           <input type="text"
-                 id={s"form-$tableId-page-size"}
+                 id={s"form-$navigationId-page-size"}
                  name={pageSizeFormField}
                  value={pageSize.toString}
                  class="span1" />
diff --git a/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala b/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala
index d18f55474bdb3..4125436907007 100644
--- a/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/PagedTableSuite.scala
@@ -85,6 +85,35 @@ class PagedTableSuite extends SparkFunSuite {
     assert((pagedTable.pageNavigation(93, 10, 97).head \\ "li").map(_.text.trim) ===
       Seq("<<", "<") ++ (91 to 97).map(_.toString) ++ Seq(">"))
   }
+
+  test("pageNavigation with different id") {
+    val pagedTable = new PagedTable[Int] {
+      override def tableId: String = "testTable"
+
+      override def tableCssClass: String = ""
+
+      override def dataSource: PagedDataSource[Int] = null
+
+      override def pageLink(page: Int): String = ""
+
+      override def headers: Seq[Node] = Nil
+
+      override def row(t: Int): Seq[Node] = Nil
+
+      override def pageSizeFormField: String = ""
+
+      override def pageNumberFormField: String = ""
+
+      override def goButtonFormPath: String = ""
+    }
+
+    val defaultIdNavigation = pagedTable.pageNavigation(1, 10, 2).head \\ "form"
+    assert(defaultIdNavigation \@ "id" === "form-testTable-page")
+
+    val customIdNavigation = pagedTable.pageNavigation(1, 10, 2, "customIdTable").head \\ "form"
+    assert(customIdNavigation \@ "id" === "form-customIdTable-page")
+    assert(defaultIdNavigation !== customIdNavigation)
+  }
 }
 
 private[spark] class SeqPagedDataSource[T](seq: Seq[T], pageSize: Int)

From 1139e9b50150e3a99a9c8df0ed57d3fd2b391788 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 18 Apr 2020 09:31:52 +0900
Subject: [PATCH 0514/1280] [SPARK-31390][SQL][DOCS] Document Window Function
 in SQL Syntax Section

### What changes were proposed in this pull request?
Document Window Function in SQL syntax

### Why are the changes needed?
Make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes

<img width="1050" alt="Screen Shot 2020-04-16 at 9 13 34 PM" src="https://user-images.githubusercontent.com/13592258/79531509-7bf5af00-8027-11ea-8291-a91b2e97a1b5.png">

<img width="1050" alt="Screen Shot 2020-04-16 at 9 14 12 PM" src="https://user-images.githubusercontent.com/13592258/79531514-7e580900-8027-11ea-8761-4c5a888c476f.png">

<img width="1050" alt="Screen Shot 2020-04-16 at 9 14 45 PM" src="https://user-images.githubusercontent.com/13592258/79531518-82842680-8027-11ea-876f-6375aa5b5ead.png">

<img width="1050" alt="Screen Shot 2020-04-16 at 9 15 10 PM" src="https://user-images.githubusercontent.com/13592258/79531521-844dea00-8027-11ea-8948-712f054d42ee.png">

<img width="1050" alt="Screen Shot 2020-04-16 at 9 15 25 PM" src="https://user-images.githubusercontent.com/13592258/79531528-8748da80-8027-11ea-9dae-a465286982ac.png">

### How was this patch tested?
Manually build and check

Closes #28220 from huaxingao/sql-win-fun.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 142f43629c42ad750d9b506283191aa830d95c08)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml          |   2 +
 docs/sql-ref-syntax-qry-window.md | 190 +++++++++++++++++++++++++++++-
 2 files changed, 189 insertions(+), 3 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 7827a0fe7d904..5042c25884289 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -168,6 +168,8 @@
                   url: sql-ref-syntax-qry-select-inline-table.html
                 - text: Common Table Expression
                   url: sql-ref-syntax-qry-select-cte.html
+                - text: Window Function
+                  url: sql-ref-syntax-qry-window.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-qry-window.md b/docs/sql-ref-syntax-qry-window.md
index 767f47715e89c..4ec1af70f6079 100644
--- a/docs/sql-ref-syntax-qry-window.md
+++ b/docs/sql-ref-syntax-qry-window.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Windowing Analytic Functions
-displayTitle: Windowing Analytic Functions
+title: Window Functions
+displayTitle: Window Functions
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -19,4 +19,188 @@ license: |
   limitations under the License.
 ---
 
-**This page is under construction**
+### Description
+
+Window functions operate on a group of rows, referred to as a window, and calculate a return value for each row based on the group of rows. Window functions are useful for processing tasks such as calculating a moving average, computing a cumulative statistic, or accessing the value of rows given the relative position of the current row.
+
+### Syntax
+
+{% highlight sql %}
+window_function OVER
+( [  { PARTITION | DISTRIBUTE } BY partition_col_name = partition_col_val ( [ , ... ] ) ]
+  { ORDER | SORT } BY expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ]
+  [ window_frame ] )
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>window_function</em></code></dt>
+  <dd>
+    <ul>
+      <li>Ranking Functions</li>
+      <br>
+      <b>Syntax:</b>
+        <code>
+          RANK | DENSE_RANK | PERCENT_RANK | NTILE | ROW_NUMBER
+        </code>
+    </ul>
+    <ul>
+      <li>Analytic Functions</li>
+      <br>
+      <b>Syntax:</b>
+        <code>
+          CUME_DIST | LAG | LEAD
+        </code>
+    </ul>
+    <ul>
+      <li>Aggregate Functions</li>
+      <br>
+      <b>Syntax:</b>
+        <code>
+          MAX | MIN | COUNT | SUM | AVG | ...
+        </code>
+        <br>
+        Please refer to the <a href="api/sql/">Built-in Functions</a> document for a complete list of Spark aggregate functions.
+    </ul>
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>window_frame</em></code></dt>
+  <dd>
+    Specifies which row to start the window on and where to end it.<br>
+    <b>Syntax:</b><br>
+      <code>{ RANGE | ROWS } { frame_start | BETWEEN frame_start AND frame_end }</code><br>
+      If frame_end is omitted it defaults to CURRENT ROW.<br><br>
+      <ul>
+      <code>frame_start</code> and <code>frame_end</code> have the following syntax<br>
+      <b>Syntax:</b><br>
+        <code>
+          UNBOUNDED PRECEDING | offset PRECEDING | CURRENT ROW | offset FOLLOWING | UNBOUNDED FOLLOWING
+        </code><br>
+        <code>offset:</code>specifies the <code>offset</code> from the position of the current row.
+      </ul>
+  </dd>
+</dl>
+
+### Examples
+
+{% highlight sql %}
+CREATE TABLE employees (name STRING, dept STRING, salary INT, age INT);
+
+INSERT INTO employees VALUES ("Lisa", "Sales", 10000, 35);
+INSERT INTO employees VALUES ("Evan", "Sales", 32000, 38);
+INSERT INTO employees VALUES ("Fred", "Engineering", 21000, 28);
+INSERT INTO employees VALUES ("Alex", "Sales", 30000, 33);
+INSERT INTO employees VALUES ("Tom", "Engineering", 23000, 33);
+INSERT INTO employees VALUES ("Jane", "Marketing", 29000, 28);
+INSERT INTO employees VALUES ("Jeff", "Marketing", 35000, 38);
+INSERT INTO employees VALUES ("Paul", "Engineering", 29000, 23);
+INSERT INTO employees VALUES ("Chloe", "Engineering", 23000, 25);
+
+SELECT * FROM employees;
+  +-----+-----------+------+-----+
+  | name|       dept|salary|  age|
+  +-----+-----------+------+-----+
+  |Chloe|Engineering| 23000|   25|
+  | Fred|Engineering| 21000|   28|
+  | Paul|Engineering| 29000|   23|
+  |Helen|  Marketing| 29000|   40|
+  |  Tom|Engineering| 23000|   33|
+  | Jane|  Marketing| 29000|   28|
+  | Jeff|  Marketing| 35000|   38|
+  | Evan|      Sales| 32000|   38|
+  | Lisa|      Sales| 10000|   35|
+  | Alex|      Sales| 30000|   33|
+  +-----+-----------+------+-----+
+
+SELECT name, dept, RANK() OVER (PARTITION BY dept ORDER BY salary) AS rank FROM employees;
+  +-----+-----------+------+----+
+  | name|       dept|salary|rank|
+  +-----+-----------+------+----+
+  | Lisa|      Sales| 10000|   1|
+  | Alex|      Sales| 30000|   2|
+  | Evan|      Sales| 32000|   3|
+  | Fred|Engineering| 21000|   1|
+  |  Tom|Engineering| 23000|   2|
+  |Chloe|Engineering| 23000|   2|
+  | Paul|Engineering| 29000|   4|
+  |Helen|  Marketing| 29000|   1|
+  | Jane|  Marketing| 29000|   1|
+  | Jeff|  Marketing| 35000|   3|
+  +-----+-----------+------+----+
+
+SELECT name, dept, DENSE_RANK() OVER (PARTITION BY dept ORDER BY salary ROWS BETWEEN
+    UNBOUNDED PRECEDING AND CURRENT ROW) AS dense_rank FROM employees;
+  +-----+-----------+------+----------+
+  | name|       dept|salary|dense_rank|
+  +-----+-----------+------+----------+
+  | Lisa|      Sales| 10000|         1|
+  | Alex|      Sales| 30000|         2|
+  | Evan|      Sales| 32000|         3|
+  | Fred|Engineering| 21000|         1|
+  |  Tom|Engineering| 23000|         2|
+  |Chloe|Engineering| 23000|         2|
+  | Paul|Engineering| 29000|         3|
+  |Helen|  Marketing| 29000|         1|
+  | Jane|  Marketing| 29000|         1|
+  | Jeff|  Marketing| 35000|         2|
+  +-----+-----------+------+----------+
+
+SELECT name, dept, age, CUME_DIST() OVER (PARTITION BY dept ORDER BY age
+    RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_dist FROM employees;
+  +-----+-----------+------+------------------+
+  | name|       dept|age   |         cume_dist|
+  +-----+-----------+------+------------------+
+  | Alex|      Sales|    33|0.3333333333333333|
+  | Lisa|      Sales|    35|0.6666666666666666|
+  | Evan|      Sales|    38|               1.0|
+  | Paul|Engineering|    23|              0.25|
+  |Chloe|Engineering|    25|              0.75|
+  | Fred|Engineering|    28|              0.25|
+  |  Tom|Engineering|    33|               1.0|
+  | Jane|  Marketing|    28|0.3333333333333333|
+  | Jeff|  Marketing|    38|0.6666666666666666|
+  |Helen|  Marketing|    40|               1.0|
+  +-----+-----------+------+------------------+
+
+SELECT name, dept, salary, MIN(salary) OVER (PARTITION BY dept ORDER BY salary) AS min
+    FROM employees;
+  +-----+-----------+------+-----+
+  | name|       dept|salary|  min|
+  +-----+-----------+------+-----+
+  | Lisa|      Sales| 10000|10000|
+  | Alex|      Sales| 30000|10000|
+  | Evan|      Sales| 32000|10000|
+  |Helen|  Marketing| 29000|29000|
+  | Jane|  Marketing| 29000|29000|
+  | Jeff|  Marketing| 35000|29000|
+  | Fred|Engineering| 21000|21000|
+  |  Tom|Engineering| 23000|21000|
+  |Chloe|Engineering| 23000|21000|
+  | Paul|Engineering| 29000|21000|
+  +-----+-----------+------+-----+
+
+SELECT name, salary,
+    LAG(salary) OVER (PARTITION BY dept ORDER BY salary) AS lag,
+    LEAD(salary, 1, 0) OVER (PARTITION BY dept ORDER BY salary) AS lead
+    FROM employees;
+  +-----+-----------+------+-----+-----+
+  | name|       dept|salary|  lag| lead|
+  +-----+-----------+------+-----+-----+
+  | Lisa|      Sales| 10000|NULL |30000|
+  | Alex|      Sales| 30000|10000|32000|
+  | Evan|      Sales| 32000|30000|    0|
+  | Fred|Engineering| 21000| NULL|23000|
+  |Chloe|Engineering| 23000|21000|23000|
+  |  Tom|Engineering| 23000|23000|29000|
+  | Paul|Engineering| 29000|23000|    0|
+  |Helen|  Marketing| 29000| NULL|29000|
+  | Jane|  Marketing| 29000|29000|35000|
+  | Jeff|  Marketing| 35000|29000|    0|
+  +-----+-----------+------+-----+-----+
+{% endhighlight %}
+
+### Related Statements
+
+  * [SELECT](sql-ref-syntax-qry-select.html)

From e1ed80a134a2d613a9221a15847cad3274cd6f2b Mon Sep 17 00:00:00 2001
From: Maryann Xue <maryann.xue@gmail.com>
Date: Sat, 18 Apr 2020 00:08:36 -0700
Subject: [PATCH 0515/1280] [SPARK-31473][SQL] AQE should set active session
 during execution

### What changes were proposed in this pull request?

AQE creates new SparkPlan nodes during execution. This PR makes sure that the active session is set correctly during this process and AQE execution is not disrupted by external session change.

### Why are the changes needed?

To prevent potential errors. If not changed, the physical plans generated by AQE would have the wrong SparkSession or even null SparkSession, which could lead to NPE.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added UT.

Closes #28247 from maryannxue/aqe-activesession.

Authored-by: Maryann Xue <maryann.xue@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6198f384054e7f86521891ceeb1a231f449a16a8)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../execution/adaptive/AdaptiveSparkPlanExec.scala  |  9 +++++++--
 .../execution/adaptive/AdaptiveQueryExecSuite.scala | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index b54a32fa988d1..2b4672407a348 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -139,7 +139,12 @@ case class AdaptiveSparkPlanExec(
   }
 
   private def getFinalPhysicalPlan(): SparkPlan = lock.synchronized {
-    if (!isFinalPlan) {
+    if (isFinalPlan) return currentPhysicalPlan
+
+    // In case of this adaptive plan being executed out of `withActive` scoped functions, e.g.,
+    // `plan.queryExecution.rdd`, we need to set active session here as new plan nodes can be
+    // created in the middle of the execution.
+    context.session.withActive {
       // Subqueries do not have their own execution IDs and therefore rely on the main query to
       // update UI.
       val executionId = Option(context.session.sparkContext.getLocalProperty(
@@ -225,8 +230,8 @@ case class AdaptiveSparkPlanExec(
       isFinalPlan = true
       executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
       logOnLevel(s"Final plan: $currentPhysicalPlan")
+      currentPhysicalPlan
     }
-    currentPhysicalPlan
   }
 
   override def executeCollect(): Array[InternalRow] = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 64dd9aae96e2b..6da510f5f90ba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -23,13 +23,14 @@ import java.net.URI
 import org.apache.log4j.Level
 
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{QueryTest, Row, SparkSession}
 import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.command.DataWritingCommandExec
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight, SortMergeJoinExec}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StructType}
@@ -805,4 +806,14 @@ class AdaptiveQueryExecSuite
       assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec])
     }
   }
+
+  test("AQE should set active session during execution") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      val df = spark.range(10).select(sum('id))
+      assert(df.queryExecution.executedPlan.isInstanceOf[AdaptiveSparkPlanExec])
+      SparkSession.setActiveSession(null)
+      checkAnswer(df, Seq(Row(45)))
+      SparkSession.setActiveSession(spark) // recover the active session.
+    }
+  }
 }

From 29f5962372a1ac1d2101d79bf33c53e9ece43f3b Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sat, 18 Apr 2020 00:32:42 -0700
Subject: [PATCH 0516/1280] [SPARK-31469][SQL][TESTS][FOLLOWUP] Remove
 unsupported fields from ExtractBenchmark

### What changes were proposed in this pull request?

In https://github.com/apache/spark/commit/697083c051304a5ac4f9c56a63274f2103caaef4, we remove  "MILLENNIUM", "CENTURY", "DECADE",  "QUARTER", "MILLISECONDS", "MICROSECONDS", "EPOCH" field for date_part and extract expression, this PR fix the related Benchmark.
### Why are the changes needed?

test fix.

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

passing Jenkins

Closes #28249 from yaooqinn/SPARK-31469-F.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 77cb7cde0d657967f619287a792a848d1f42adf5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../ExtractBenchmark-jdk11-results.txt        | 214 ++++++++----------
 .../benchmarks/ExtractBenchmark-results.txt   | 214 ++++++++----------
 .../benchmark/ExtractBenchmark.scala          |   6 +-
 3 files changed, 201 insertions(+), 233 deletions(-)

diff --git a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
index 8c56c0bcc9cb0..41bb42c504c7b 100644
--- a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
@@ -1,138 +1,124 @@
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   311            331          18         32.2          31.1       1.0X
-MILLENNIUM of timestamp                             873            893          17         11.4          87.3       0.4X
-CENTURY of timestamp                                869            873           5         11.5          86.9       0.4X
-DECADE of timestamp                                 851            872          23         11.7          85.1       0.4X
-YEAR of timestamp                                   841            856          14         11.9          84.1       0.4X
-ISOYEAR of timestamp                                927            938          12         10.8          92.7       0.3X
-QUARTER of timestamp                                959            963           6         10.4          95.9       0.3X
-MONTH of timestamp                                  852            864          18         11.7          85.2       0.4X
-WEEK of timestamp                                  1124           1252         112          8.9         112.4       0.3X
-DAY of timestamp                                    848            867          19         11.8          84.8       0.4X
-DAYOFWEEK of timestamp                              977            987          16         10.2          97.7       0.3X
-DOW of timestamp                                    945            964          18         10.6          94.5       0.3X
-ISODOW of timestamp                                 924            929           5         10.8          92.4       0.3X
-DOY of timestamp                                    852            906          67         11.7          85.2       0.4X
-HOUR of timestamp                                   665            671           5         15.0          66.5       0.5X
-MINUTE of timestamp                                 655            670          15         15.3          65.5       0.5X
-SECOND of timestamp                                 757            763           7         13.2          75.7       0.4X
-MILLISECONDS of timestamp                           745            761          14         13.4          74.5       0.4X
-MICROSECONDS of timestamp                           691            697           7         14.5          69.1       0.5X
-EPOCH of timestamp                                  794            806          12         12.6          79.4       0.4X
+cast to timestamp                                   322            327           4         31.1          32.2       1.0X
+MILLENNIUM of timestamp                             834            841           8         12.0          83.4       0.4X
+CENTURY of timestamp                                828            841          15         12.1          82.8       0.4X
+DECADE of timestamp                                 813            825          13         12.3          81.3       0.4X
+YEAR of timestamp                                   800            817          18         12.5          80.0       0.4X
+ISOYEAR of timestamp                                882            889           6         11.3          88.2       0.4X
+QUARTER of timestamp                                900            908          12         11.1          90.0       0.4X
+MONTH of timestamp                                  809            816           8         12.4          80.9       0.4X
+WEEK of timestamp                                  1119           1123           4          8.9         111.9       0.3X
+DAY of timestamp                                    801            808           7         12.5          80.1       0.4X
+DAYOFWEEK of timestamp                              946            952           5         10.6          94.6       0.3X
+DOW of timestamp                                    939            945          10         10.6          93.9       0.3X
+ISODOW of timestamp                                 869            874           5         11.5          86.9       0.4X
+DOY of timestamp                                    822            835          14         12.2          82.2       0.4X
+HOUR of timestamp                                   633            647          12         15.8          63.3       0.5X
+MINUTE of timestamp                                 635            636           1         15.8          63.5       0.5X
+SECOND of timestamp                                 774            778           4         12.9          77.4       0.4X
+MILLISECONDS of timestamp                           735            743           7         13.6          73.5       0.4X
+MICROSECONDS of timestamp                           652            657           4         15.3          65.2       0.5X
+EPOCH of timestamp                                  794            800           5         12.6          79.4       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   273            274           1         36.6          27.3       1.0X
-MILLENNIUM of timestamp                             860            875          14         11.6          86.0       0.3X
-CENTURY of timestamp                                854            866          11         11.7          85.4       0.3X
-DECADE of timestamp                                 855            863          10         11.7          85.5       0.3X
-YEAR of timestamp                                   833            837           4         12.0          83.3       0.3X
-ISOYEAR of timestamp                               1022           1036          18          9.8         102.2       0.3X
-QUARTER of timestamp                               1067           1132          58          9.4         106.7       0.3X
-MONTH of timestamp                                  855            861           5         11.7          85.5       0.3X
-WEEK of timestamp                                  1312           1325          17          7.6         131.2       0.2X
-DAY of timestamp                                    837            847          10         12.0          83.7       0.3X
-DAYOFWEEK of timestamp                              965            969           4         10.4          96.5       0.3X
-DOW of timestamp                                    981           1038          56         10.2          98.1       0.3X
-ISODOW of timestamp                                 942            959          20         10.6          94.2       0.3X
-DOY of timestamp                                    994           1021          24         10.1          99.4       0.3X
-HOUR of timestamp                                   690            695           7         14.5          69.0       0.4X
-MINUTE of timestamp                                 764            788          21         13.1          76.4       0.4X
-SECOND of timestamp                                 776            795          34         12.9          77.6       0.4X
-MILLISECONDS of timestamp                           863            879          14         11.6          86.3       0.3X
-MICROSECONDS of timestamp                           689            695           7         14.5          68.9       0.4X
-EPOCH of timestamp                                  797            865          69         12.5          79.7       0.3X
+cast to timestamp                                   281            291          11         35.6          28.1       1.0X
+MILLENNIUM of timestamp                             814            825          19         12.3          81.4       0.3X
+CENTURY of timestamp                                842            850           7         11.9          84.2       0.3X
+DECADE of timestamp                                 812            814           3         12.3          81.2       0.3X
+YEAR of timestamp                                   787            798          10         12.7          78.7       0.4X
+ISOYEAR of timestamp                                956            962           6         10.5          95.6       0.3X
+QUARTER of timestamp                                948            954           6         10.6          94.8       0.3X
+MONTH of timestamp                                  797            817          26         12.5          79.7       0.4X
+WEEK of timestamp                                  1126           1134          10          8.9         112.6       0.2X
+DAY of timestamp                                    800            803           2         12.5          80.0       0.4X
+DAYOFWEEK of timestamp                              953            969          16         10.5          95.3       0.3X
+DOW of timestamp                                    978            986           9         10.2          97.8       0.3X
+ISODOW of timestamp                                 907            913           7         11.0          90.7       0.3X
+DOY of timestamp                                    831            844          14         12.0          83.1       0.3X
+HOUR of timestamp                                   646            650           3         15.5          64.6       0.4X
+MINUTE of timestamp                                 639            643           5         15.7          63.9       0.4X
+SECOND of timestamp                                 740            747           6         13.5          74.0       0.4X
+MILLISECONDS of timestamp                           745            752           7         13.4          74.5       0.4X
+MICROSECONDS of timestamp                           653            658           5         15.3          65.3       0.4X
+EPOCH of timestamp                                  724            738          12         13.8          72.4       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        763            774          13         13.1          76.3       1.0X
-MILLENNIUM of date                                  857            911          73         11.7          85.7       0.9X
-CENTURY of date                                     869            884          15         11.5          86.9       0.9X
-DECADE of date                                      860            868           7         11.6          86.0       0.9X
-YEAR of date                                        848            852           7         11.8          84.8       0.9X
-ISOYEAR of date                                    1028           1046          22          9.7         102.8       0.7X
-QUARTER of date                                     979            986           7         10.2          97.9       0.8X
-MONTH of date                                       857            862           4         11.7          85.7       0.9X
-WEEK of date                                       1218           1247          29          8.2         121.8       0.6X
-DAY of date                                         855            868          11         11.7          85.5       0.9X
-DAYOFWEEK of date                                   987           1081          82         10.1          98.7       0.8X
-DOW of date                                         986            996           9         10.1          98.6       0.8X
-ISODOW of date                                      938            947          10         10.7          93.8       0.8X
-DOY of date                                         877            890          11         11.4          87.7       0.9X
-HOUR of date                                       1656           1668          12          6.0         165.6       0.5X
-MINUTE of date                                     1652           1666          15          6.1         165.2       0.5X
-SECOND of date                                     1752           1776          23          5.7         175.2       0.4X
-MILLISECONDS of date                               1757           1766          11          5.7         175.7       0.4X
-MICROSECONDS of date                               1682           1691          14          5.9         168.2       0.5X
-EPOCH of date                                      1706           1721          14          5.9         170.6       0.4X
+cast to date                                        613            621           8         16.3          61.3       1.0X
+MILLENNIUM of date                                  738            741           3         13.6          73.8       0.8X
+CENTURY of date                                     735            737           3         13.6          73.5       0.8X
+DECADE of date                                      722            731           7         13.8          72.2       0.8X
+YEAR of date                                        715            719           4         14.0          71.5       0.9X
+ISOYEAR of date                                     881            883           2         11.4          88.1       0.7X
+QUARTER of date                                     857            867          10         11.7          85.7       0.7X
+MONTH of date                                       719            727           7         13.9          71.9       0.9X
+WEEK of date                                       1038           1043           8          9.6         103.8       0.6X
+DAY of date                                         715            719           4         14.0          71.5       0.9X
+DAYOFWEEK of date                                   861            873          12         11.6          86.1       0.7X
+DOW of date                                         847            865          16         11.8          84.7       0.7X
+ISODOW of date                                      788            796           7         12.7          78.8       0.8X
+DOY of date                                         744            747           4         13.4          74.4       0.8X
+HOUR of date                                       1485           1506          30          6.7         148.5       0.4X
+MINUTE of date                                     1502           1503           1          6.7         150.2       0.4X
+SECOND of date                                     1641           1650           9          6.1         164.1       0.4X
+MILLISECONDS of date                               1624           1639          14          6.2         162.4       0.4X
+MICROSECONDS of date                               1504           1505           2          6.6         150.4       0.4X
+EPOCH of date                                      1733           1748          19          5.8         173.3       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        771            779          10         13.0          77.1       1.0X
-MILLENNIUM of date                                  839            858          22         11.9          83.9       0.9X
-CENTURY of date                                     853            872          22         11.7          85.3       0.9X
-DECADE of date                                      871            878           6         11.5          87.1       0.9X
-YEAR of date                                        853            863           9         11.7          85.3       0.9X
-ISOYEAR of date                                    1011           1021          11          9.9         101.1       0.8X
-QUARTER of date                                     981            988           7         10.2          98.1       0.8X
-MONTH of date                                       859            863           5         11.6          85.9       0.9X
-WEEK of date                                       1148           1159          12          8.7         114.8       0.7X
-DAY of date                                         852            855           2         11.7          85.2       0.9X
-DAYOFWEEK of date                                   937            952          12         10.7          93.7       0.8X
-DOW of date                                         953            956           3         10.5          95.3       0.8X
-ISODOW of date                                      899            916          16         11.1          89.9       0.9X
-DOY of date                                         857            865           7         11.7          85.7       0.9X
-HOUR of date                                       1644           1662          18          6.1         164.4       0.5X
-MINUTE of date                                     1633           1644          10          6.1         163.3       0.5X
-SECOND of date                                     1719           1727           9          5.8         171.9       0.4X
-MILLISECONDS of date                               1721           1733          11          5.8         172.1       0.4X
-MICROSECONDS of date                               1656           1680          34          6.0         165.6       0.5X
-EPOCH of date                                      1721           1736          15          5.8         172.1       0.4X
+cast to date                                        613            627          12         16.3          61.3       1.0X
+MILLENNIUM of date                                  722            734          11         13.9          72.2       0.8X
+CENTURY of date                                     740            746           8         13.5          74.0       0.8X
+DECADE of date                                      723            726           3         13.8          72.3       0.8X
+YEAR of date                                        705            717          11         14.2          70.5       0.9X
+ISOYEAR of date                                     876            885           9         11.4          87.6       0.7X
+QUARTER of date                                     857            877          21         11.7          85.7       0.7X
+MONTH of date                                       714            715           2         14.0          71.4       0.9X
+WEEK of date                                       1040           1043           4          9.6         104.0       0.6X
+DAY of date                                         703            714          10         14.2          70.3       0.9X
+DAYOFWEEK of date                                   861            869           7         11.6          86.1       0.7X
+DOW of date                                         856            863           8         11.7          85.6       0.7X
+ISODOW of date                                      791            797           5         12.6          79.1       0.8X
+DOY of date                                         721            728           6         13.9          72.1       0.9X
+HOUR of date                                       1488           1493           7          6.7         148.8       0.4X
+MINUTE of date                                     1488           1502          12          6.7         148.8       0.4X
+SECOND of date                                     1636           1651          14          6.1         163.6       0.4X
+MILLISECONDS of date                               1644           1665          34          6.1         164.4       0.4X
+MICROSECONDS of date                               1482           1516          42          6.7         148.2       0.4X
+EPOCH of date                                      1752           1790          34          5.7         175.2       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    939            947           9         10.7          93.9       1.0X
-MILLENNIUM of interval                              953            960           6         10.5          95.3       1.0X
-CENTURY of interval                                 971            979          10         10.3          97.1       1.0X
-DECADE of interval                                  962            969           7         10.4          96.2       1.0X
-YEAR of interval                                    965            973          10         10.4          96.5       1.0X
-QUARTER of interval                                 967            979          15         10.3          96.7       1.0X
-MONTH of interval                                   951            979          26         10.5          95.1       1.0X
-DAY of interval                                     945            970          25         10.6          94.5       1.0X
-HOUR of interval                                    963            970           9         10.4          96.3       1.0X
-MINUTE of interval                                  972            991          24         10.3          97.2       1.0X
-SECOND of interval                                 1053           1057           8          9.5         105.3       0.9X
-MILLISECONDS of interval                           1042           1048          11          9.6         104.2       0.9X
-MICROSECONDS of interval                            963            967           4         10.4          96.3       1.0X
-EPOCH of interval                                  1051           1071          22          9.5         105.1       0.9X
+cast to interval                                    921            929           9         10.9          92.1       1.0X
+YEAR of interval                                    927            942          22         10.8          92.7       1.0X
+MONTH of interval                                   956            974          22         10.5          95.6       1.0X
+DAY of interval                                     897            915          16         11.2          89.7       1.0X
+HOUR of interval                                    952            975          35         10.5          95.2       1.0X
+MINUTE of interval                                  938            955          14         10.7          93.8       1.0X
+SECOND of interval                                 1066           1080          24          9.4         106.6       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    944            952           8         10.6          94.4       1.0X
-MILLENNIUM of interval                              955            960           5         10.5          95.5       1.0X
-CENTURY of interval                                 958            972          12         10.4          95.8       1.0X
-DECADE of interval                                  962            967           6         10.4          96.2       1.0X
-YEAR of interval                                    958            978          18         10.4          95.8       1.0X
-QUARTER of interval                                 979            998          19         10.2          97.9       1.0X
-MONTH of interval                                   949            963          12         10.5          94.9       1.0X
-DAY of interval                                     958            971          11         10.4          95.8       1.0X
-HOUR of interval                                    947            962          15         10.6          94.7       1.0X
-MINUTE of interval                                  989           1006          15         10.1          98.9       1.0X
-SECOND of interval                                 1061           1064           4          9.4         106.1       0.9X
-MILLISECONDS of interval                           1042           1061          18          9.6         104.2       0.9X
-MICROSECONDS of interval                            982            992          12         10.2          98.2       1.0X
-EPOCH of interval                                  1047           1057          13          9.5         104.7       0.9X
+cast to interval                                    944            960          21         10.6          94.4       1.0X
+YEAR of interval                                    978           1011          33         10.2          97.8       1.0X
+MONTH of interval                                   961            984          27         10.4          96.1       1.0X
+DAY of interval                                     921            927           7         10.9          92.1       1.0X
+HOUR of interval                                    913            919           8         11.0          91.3       1.0X
+MINUTE of interval                                  920            941          22         10.9          92.0       1.0X
+SECOND of interval                                  994           1012          19         10.1          99.4       0.9X
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-results.txt b/sql/core/benchmarks/ExtractBenchmark-results.txt
index 94cf0bc475582..bc766b47cec21 100644
--- a/sql/core/benchmarks/ExtractBenchmark-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-results.txt
@@ -1,138 +1,124 @@
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   275            307          28         36.3          27.5       1.0X
-MILLENNIUM of timestamp                             911            924          20         11.0          91.1       0.3X
-CENTURY of timestamp                                846            850           5         11.8          84.6       0.3X
-DECADE of timestamp                                 744            761          21         13.4          74.4       0.4X
-YEAR of timestamp                                   750            764          15         13.3          75.0       0.4X
-ISOYEAR of timestamp                                815            828          11         12.3          81.5       0.3X
-QUARTER of timestamp                                882            895          12         11.3          88.2       0.3X
-MONTH of timestamp                                  739            760          19         13.5          73.9       0.4X
-WEEK of timestamp                                  1058           1082          23          9.4         105.8       0.3X
-DAY of timestamp                                    722            730           8         13.8          72.2       0.4X
-DAYOFWEEK of timestamp                              860            907          58         11.6          86.0       0.3X
-DOW of timestamp                                    853            860           7         11.7          85.3       0.3X
-ISODOW of timestamp                                 829            835           8         12.1          82.9       0.3X
-DOY of timestamp                                    757            772          20         13.2          75.7       0.4X
-HOUR of timestamp                                   586            594           7         17.1          58.6       0.5X
-MINUTE of timestamp                                 577            584           8         17.3          57.7       0.5X
-SECOND of timestamp                                 810            827          17         12.4          81.0       0.3X
-MILLISECONDS of timestamp                           687            704          16         14.6          68.7       0.4X
-MICROSECONDS of timestamp                           628            632           6         15.9          62.8       0.4X
-EPOCH of timestamp                                  750            761          12         13.3          75.0       0.4X
+cast to timestamp                                   306            337          27         32.7          30.6       1.0X
+MILLENNIUM of timestamp                            1002           1012          15         10.0         100.2       0.3X
+CENTURY of timestamp                               1055           1064           8          9.5         105.5       0.3X
+DECADE of timestamp                                 940            968          36         10.6          94.0       0.3X
+YEAR of timestamp                                   922            938          27         10.8          92.2       0.3X
+ISOYEAR of timestamp                               1036           1061          23          9.7         103.6       0.3X
+QUARTER of timestamp                               1081           1098          27          9.2         108.1       0.3X
+MONTH of timestamp                                  876            888          17         11.4          87.6       0.3X
+WEEK of timestamp                                  1218           1259          55          8.2         121.8       0.3X
+DAY of timestamp                                    890            911          33         11.2          89.0       0.3X
+DAYOFWEEK of timestamp                             1006           1019          17          9.9         100.6       0.3X
+DOW of timestamp                                    990           1006          22         10.1          99.0       0.3X
+ISODOW of timestamp                                 953            992          35         10.5          95.3       0.3X
+DOY of timestamp                                    883            889           6         11.3          88.3       0.3X
+HOUR of timestamp                                   661            667           6         15.1          66.1       0.5X
+MINUTE of timestamp                                 665            678          12         15.0          66.5       0.5X
+SECOND of timestamp                                 787            791           6         12.7          78.7       0.4X
+MILLISECONDS of timestamp                           800            814          12         12.5          80.0       0.4X
+MICROSECONDS of timestamp                           700            706           8         14.3          70.0       0.4X
+EPOCH of timestamp                                  793            819          40         12.6          79.3       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   238            242           4         42.0          23.8       1.0X
-MILLENNIUM of timestamp                             811            835          25         12.3          81.1       0.3X
-CENTURY of timestamp                                799            823          21         12.5          79.9       0.3X
-DECADE of timestamp                                 733            743          12         13.6          73.3       0.3X
-YEAR of timestamp                                   714            717           3         14.0          71.4       0.3X
-ISOYEAR of timestamp                                874            900          24         11.4          87.4       0.3X
-QUARTER of timestamp                                903            910          11         11.1          90.3       0.3X
-MONTH of timestamp                                  774            808          44         12.9          77.4       0.3X
-WEEK of timestamp                                  1053           1064          12          9.5         105.3       0.2X
-DAY of timestamp                                    755            770          14         13.2          75.5       0.3X
-DAYOFWEEK of timestamp                              831            841           9         12.0          83.1       0.3X
-DOW of timestamp                                    871            898          23         11.5          87.1       0.3X
-ISODOW of timestamp                                 832            887          59         12.0          83.2       0.3X
-DOY of timestamp                                    755            767          13         13.2          75.5       0.3X
-HOUR of timestamp                                   580            597          27         17.3          58.0       0.4X
-MINUTE of timestamp                                 572            589          15         17.5          57.2       0.4X
-SECOND of timestamp                                 716            730          12         14.0          71.6       0.3X
-MILLISECONDS of timestamp                           716            762          57         14.0          71.6       0.3X
-MICROSECONDS of timestamp                           600            610          12         16.7          60.0       0.4X
-EPOCH of timestamp                                  751            779          44         13.3          75.1       0.3X
+cast to timestamp                                   252            259           7         39.7          25.2       1.0X
+MILLENNIUM of timestamp                             917            936          18         10.9          91.7       0.3X
+CENTURY of timestamp                                943            952          12         10.6          94.3       0.3X
+DECADE of timestamp                                 863            867           5         11.6          86.3       0.3X
+YEAR of timestamp                                   839            854          16         11.9          83.9       0.3X
+ISOYEAR of timestamp                               1023           1027           4          9.8         102.3       0.2X
+QUARTER of timestamp                               1052           1060          10          9.5         105.2       0.2X
+MONTH of timestamp                                  835            846          14         12.0          83.5       0.3X
+WEEK of timestamp                                  1182           1198          14          8.5         118.2       0.2X
+DAY of timestamp                                    837            845          10         11.9          83.7       0.3X
+DAYOFWEEK of timestamp                              947            962          14         10.6          94.7       0.3X
+DOW of timestamp                                    946            964          17         10.6          94.6       0.3X
+ISODOW of timestamp                                 934            946          13         10.7          93.4       0.3X
+DOY of timestamp                                    857            859           3         11.7          85.7       0.3X
+HOUR of timestamp                                   660            661           1         15.1          66.0       0.4X
+MINUTE of timestamp                                 649            654           4         15.4          64.9       0.4X
+SECOND of timestamp                                 801            805           5         12.5          80.1       0.3X
+MILLISECONDS of timestamp                           820            825           7         12.2          82.0       0.3X
+MICROSECONDS of timestamp                           700            708          12         14.3          70.0       0.4X
+EPOCH of timestamp                                  789            797          10         12.7          78.9       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        629            634           6         15.9          62.9       1.0X
-MILLENNIUM of date                                  826            912          97         12.1          82.6       0.8X
-CENTURY of date                                     797            840          54         12.6          79.7       0.8X
-DECADE of date                                      734            737           4         13.6          73.4       0.9X
-YEAR of date                                        721            768          78         13.9          72.1       0.9X
-ISOYEAR of date                                     884            892          13         11.3          88.4       0.7X
-QUARTER of date                                     894            949          69         11.2          89.4       0.7X
-MONTH of date                                       726            729           2         13.8          72.6       0.9X
-WEEK of date                                       1013           1023           9          9.9         101.3       0.6X
-DAY of date                                         727            747          26         13.8          72.7       0.9X
-DAYOFWEEK of date                                   828            832           6         12.1          82.8       0.8X
-DOW of date                                         846            850           5         11.8          84.6       0.7X
-ISODOW of date                                      803            810           7         12.5          80.3       0.8X
-DOY of date                                         751            761          11         13.3          75.1       0.8X
-HOUR of date                                       1372           1376           5          7.3         137.2       0.5X
-MINUTE of date                                     1379           1398          20          7.3         137.9       0.5X
-SECOND of date                                     1530           1542          10          6.5         153.0       0.4X
-MILLISECONDS of date                               1532           1541           8          6.5         153.2       0.4X
-MICROSECONDS of date                               1442           1462          23          6.9         144.2       0.4X
-EPOCH of date                                      1573           1594          20          6.4         157.3       0.4X
+cast to date                                        747            751           6         13.4          74.7       1.0X
+MILLENNIUM of date                                  941            947           5         10.6          94.1       0.8X
+CENTURY of date                                     952            961           8         10.5          95.2       0.8X
+DECADE of date                                      864            877          15         11.6          86.4       0.9X
+YEAR of date                                        850            862          11         11.8          85.0       0.9X
+ISOYEAR of date                                    1031           1036           8          9.7         103.1       0.7X
+QUARTER of date                                    1038           1048           9          9.6         103.8       0.7X
+MONTH of date                                       852            865          12         11.7          85.2       0.9X
+WEEK of date                                       1174           1178           5          8.5         117.4       0.6X
+DAY of date                                         830            836           5         12.0          83.0       0.9X
+DAYOFWEEK of date                                   949            972          21         10.5          94.9       0.8X
+DOW of date                                         960            969           8         10.4          96.0       0.8X
+ISODOW of date                                      918            928           9         10.9          91.8       0.8X
+DOY of date                                         858            867          11         11.7          85.8       0.9X
+HOUR of date                                       1584           1591          10          6.3         158.4       0.5X
+MINUTE of date                                     1583           1589           6          6.3         158.3       0.5X
+SECOND of date                                     1721           1728           7          5.8         172.1       0.4X
+MILLISECONDS of date                               1711           1731          17          5.8         171.1       0.4X
+MICROSECONDS of date                               1647           1656           8          6.1         164.7       0.5X
+EPOCH of date                                      1796           1805          10          5.6         179.6       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        622            633          14         16.1          62.2       1.0X
-MILLENNIUM of date                                  793            814          20         12.6          79.3       0.8X
-CENTURY of date                                     798            809          12         12.5          79.8       0.8X
-DECADE of date                                      735            740           6         13.6          73.5       0.8X
-YEAR of date                                        714            732          23         14.0          71.4       0.9X
-ISOYEAR of date                                     881            886           4         11.3          88.1       0.7X
-QUARTER of date                                     916            925          14         10.9          91.6       0.7X
-MONTH of date                                       732            737           5         13.7          73.2       0.8X
-WEEK of date                                       1022           1034          10          9.8         102.2       0.6X
-DAY of date                                         722            745          26         13.8          72.2       0.9X
-DAYOFWEEK of date                                   831            849          17         12.0          83.1       0.7X
-DOW of date                                         853            858           4         11.7          85.3       0.7X
-ISODOW of date                                      824            825           1         12.1          82.4       0.8X
-DOY of date                                         749            752           4         13.3          74.9       0.8X
-HOUR of date                                       1422           1425           3          7.0         142.2       0.4X
-MINUTE of date                                     1379           1394          13          7.3         137.9       0.5X
-SECOND of date                                     1525           1536           9          6.6         152.5       0.4X
-MILLISECONDS of date                               1542           1555          14          6.5         154.2       0.4X
-MICROSECONDS of date                               1446           1449           2          6.9         144.6       0.4X
-EPOCH of date                                      1589           1621          35          6.3         158.9       0.4X
+cast to date                                        731            740           8         13.7          73.1       1.0X
+MILLENNIUM of date                                  939            948          12         10.7          93.9       0.8X
+CENTURY of date                                     929            940          10         10.8          92.9       0.8X
+DECADE of date                                      845            861          15         11.8          84.5       0.9X
+YEAR of date                                        848            853           5         11.8          84.8       0.9X
+ISOYEAR of date                                    1011           1027          14          9.9         101.1       0.7X
+QUARTER of date                                    1038           1050          12          9.6         103.8       0.7X
+MONTH of date                                       847            872          22         11.8          84.7       0.9X
+WEEK of date                                       1143           1152          13          8.7         114.3       0.6X
+DAY of date                                         844            848           5         11.8          84.4       0.9X
+DAYOFWEEK of date                                   947            957           9         10.6          94.7       0.8X
+DOW of date                                         963            972           8         10.4          96.3       0.8X
+ISODOW of date                                      906            919          11         11.0          90.6       0.8X
+DOY of date                                         866            868           3         11.6          86.6       0.8X
+HOUR of date                                       1599           1604           7          6.3         159.9       0.5X
+MINUTE of date                                     1578           1594          13          6.3         157.8       0.5X
+SECOND of date                                     1722           1731          11          5.8         172.2       0.4X
+MILLISECONDS of date                               1699           1730          27          5.9         169.9       0.4X
+MICROSECONDS of date                               1620           1623           5          6.2         162.0       0.5X
+EPOCH of date                                      1742           1767          32          5.7         174.2       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    913            921           8         11.0          91.3       1.0X
-MILLENNIUM of interval                              976            983          12         10.2          97.6       0.9X
-CENTURY of interval                                 976            979           3         10.2          97.6       0.9X
-DECADE of interval                                  975            987          10         10.3          97.5       0.9X
-YEAR of interval                                    964            968           5         10.4          96.4       0.9X
-QUARTER of interval                                 987            997          12         10.1          98.7       0.9X
-MONTH of interval                                   974            983          11         10.3          97.4       0.9X
-DAY of interval                                     959            962           3         10.4          95.9       1.0X
-HOUR of interval                                    972            986          13         10.3          97.2       0.9X
-MINUTE of interval                                  985            988           4         10.2          98.5       0.9X
-SECOND of interval                                 1128           1141          13          8.9         112.8       0.8X
-MILLISECONDS of interval                           1083           1086           3          9.2         108.3       0.8X
-MICROSECONDS of interval                            963            969           5         10.4          96.3       0.9X
-EPOCH of interval                                  1094           1116          23          9.1         109.4       0.8X
+cast to interval                                    920            941          18         10.9          92.0       1.0X
+YEAR of interval                                    955            967          12         10.5          95.5       1.0X
+MONTH of interval                                   954            966          11         10.5          95.4       1.0X
+DAY of interval                                     967            970           4         10.3          96.7       1.0X
+HOUR of interval                                    959            975          21         10.4          95.9       1.0X
+MINUTE of interval                                  966            981          16         10.4          96.6       1.0X
+SECOND of interval                                 1048           1066          16          9.5         104.8       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.3
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    950            971          23         10.5          95.0       1.0X
-MILLENNIUM of interval                              989           1000          11         10.1          98.9       1.0X
-CENTURY of interval                                 984            990           6         10.2          98.4       1.0X
-DECADE of interval                                  962            978          14         10.4          96.2       1.0X
-YEAR of interval                                    945            963          16         10.6          94.5       1.0X
-QUARTER of interval                                 985            998          17         10.2          98.5       1.0X
-MONTH of interval                                   970            976           9         10.3          97.0       1.0X
-DAY of interval                                     953            962           8         10.5          95.3       1.0X
-HOUR of interval                                    942            956          17         10.6          94.2       1.0X
-MINUTE of interval                                  975            993          16         10.3          97.5       1.0X
-SECOND of interval                                 1110           1122          12          9.0         111.0       0.9X
-MILLISECONDS of interval                           1056           1074          22          9.5         105.6       0.9X
-MICROSECONDS of interval                            939            960          36         10.7          93.9       1.0X
-EPOCH of interval                                  1071           1093          21          9.3         107.1       0.9X
+cast to interval                                    928            954          26         10.8          92.8       1.0X
+YEAR of interval                                    955            960           4         10.5          95.5       1.0X
+MONTH of interval                                   956            971          13         10.5          95.6       1.0X
+DAY of interval                                     964           1003          41         10.4          96.4       1.0X
+HOUR of interval                                   1057           1293         248          9.5         105.7       0.9X
+MINUTE of interval                                  975           1091         155         10.3          97.5       1.0X
+SECOND of interval                                 1104           1286         311          9.1         110.4       0.8X
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
index 285211e918d68..6984d9406d402 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
@@ -92,11 +92,7 @@ object ExtractBenchmark extends SqlBasedBenchmark {
       "DAY", "DAYOFWEEK", "DOW", "ISODOW",
       "DOY", "HOUR", "MINUTE", "SECOND",
       "MILLISECONDS", "MICROSECONDS", "EPOCH")
-    val intervalFields = Seq(
-      "MILLENNIUM", "CENTURY", "DECADE", "YEAR",
-      "QUARTER", "MONTH", "DAY",
-      "HOUR", "MINUTE", "SECOND",
-      "MILLISECONDS", "MICROSECONDS", "EPOCH")
+    val intervalFields = Seq("YEAR", "MONTH", "DAY", "HOUR", "MINUTE", "SECOND")
     val settings = Map(
       "timestamp" -> datetimeFields,
       "date" -> datetimeFields,

From c80d5f7aa0fd9e7b37e1bf4175204750098a44a6 Mon Sep 17 00:00:00 2001
From: Sean Owen <srowen@gmail.com>
Date: Fri, 17 Apr 2020 13:25:12 -0500
Subject: [PATCH 0517/1280] Apply appropriate RPC handler to receive,
 receiveStream when auth enabled

---
 .../spark/network/crypto/AuthRpcHandler.java  |  73 +++---------
 .../spark/network/sasl/SaslRpcHandler.java    |  60 ++--------
 .../server/AbstractAuthRpcHandler.java        | 107 ++++++++++++++++++
 .../network/crypto/AuthIntegrationSuite.java  |  12 +-
 .../spark/network/sasl/SparkSaslSuite.java    |   3 +-
 5 files changed, 142 insertions(+), 113 deletions(-)
 create mode 100644 common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java

diff --git a/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java b/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java
index 821cc7a849504..dd31c955350f1 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/crypto/AuthRpcHandler.java
@@ -29,12 +29,11 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.spark.network.client.RpcResponseCallback;
-import org.apache.spark.network.client.StreamCallbackWithID;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.sasl.SecretKeyHolder;
 import org.apache.spark.network.sasl.SaslRpcHandler;
+import org.apache.spark.network.server.AbstractAuthRpcHandler;
 import org.apache.spark.network.server.RpcHandler;
-import org.apache.spark.network.server.StreamManager;
 import org.apache.spark.network.util.TransportConf;
 
 /**
@@ -46,7 +45,7 @@
  * The delegate will only receive messages if the given connection has been successfully
  * authenticated. A connection may be authenticated at most once.
  */
-class AuthRpcHandler extends RpcHandler {
+class AuthRpcHandler extends AbstractAuthRpcHandler {
   private static final Logger LOG = LoggerFactory.getLogger(AuthRpcHandler.class);
 
   /** Transport configuration. */
@@ -55,36 +54,31 @@ class AuthRpcHandler extends RpcHandler {
   /** The client channel. */
   private final Channel channel;
 
-  /**
-   * RpcHandler we will delegate to for authenticated connections. When falling back to SASL
-   * this will be replaced with the SASL RPC handler.
-   */
-  @VisibleForTesting
-  RpcHandler delegate;
-
   /** Class which provides secret keys which are shared by server and client on a per-app basis. */
   private final SecretKeyHolder secretKeyHolder;
 
-  /** Whether auth is done and future calls should be delegated. */
+  /** RPC handler for auth handshake when falling back to SASL auth. */
   @VisibleForTesting
-  boolean doDelegate;
+  SaslRpcHandler saslHandler;
 
   AuthRpcHandler(
       TransportConf conf,
       Channel channel,
       RpcHandler delegate,
       SecretKeyHolder secretKeyHolder) {
+    super(delegate);
     this.conf = conf;
     this.channel = channel;
-    this.delegate = delegate;
     this.secretKeyHolder = secretKeyHolder;
   }
 
   @Override
-  public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) {
-    if (doDelegate) {
-      delegate.receive(client, message, callback);
-      return;
+  protected boolean doAuthChallenge(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
+    if (saslHandler != null) {
+      return saslHandler.doAuthChallenge(client, message, callback);
     }
 
     int position = message.position();
@@ -98,18 +92,17 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
       if (conf.saslFallback()) {
         LOG.warn("Failed to parse new auth challenge, reverting to SASL for client {}.",
           channel.remoteAddress());
-        delegate = new SaslRpcHandler(conf, channel, delegate, secretKeyHolder);
+        saslHandler = new SaslRpcHandler(conf, channel, null, secretKeyHolder);
         message.position(position);
         message.limit(limit);
-        delegate.receive(client, message, callback);
-        doDelegate = true;
+        return saslHandler.doAuthChallenge(client, message, callback);
       } else {
         LOG.debug("Unexpected challenge message from client {}, closing channel.",
           channel.remoteAddress());
         callback.onFailure(new IllegalArgumentException("Unknown challenge message."));
         channel.close();
       }
-      return;
+      return false;
     }
 
     // Here we have the client challenge, so perform the new auth protocol and set up the channel.
@@ -131,7 +124,7 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
       LOG.debug("Authentication failed for client {}, closing channel.", channel.remoteAddress());
       callback.onFailure(new IllegalArgumentException("Authentication failed."));
       channel.close();
-      return;
+      return false;
     } finally {
       if (engine != null) {
         try {
@@ -143,40 +136,6 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
     }
 
     LOG.debug("Authorization successful for client {}.", channel.remoteAddress());
-    doDelegate = true;
-  }
-
-  @Override
-  public void receive(TransportClient client, ByteBuffer message) {
-    delegate.receive(client, message);
-  }
-
-  @Override
-  public StreamCallbackWithID receiveStream(
-      TransportClient client,
-      ByteBuffer message,
-      RpcResponseCallback callback) {
-    return delegate.receiveStream(client, message, callback);
+    return true;
   }
-
-  @Override
-  public StreamManager getStreamManager() {
-    return delegate.getStreamManager();
-  }
-
-  @Override
-  public void channelActive(TransportClient client) {
-    delegate.channelActive(client);
-  }
-
-  @Override
-  public void channelInactive(TransportClient client) {
-    delegate.channelInactive(client);
-  }
-
-  @Override
-  public void exceptionCaught(Throwable cause, TransportClient client) {
-    delegate.exceptionCaught(cause, client);
-  }
-
 }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java b/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java
index 355a3def8cc22..cc9e88fcf98e7 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/sasl/SaslRpcHandler.java
@@ -28,10 +28,9 @@
 import org.slf4j.LoggerFactory;
 
 import org.apache.spark.network.client.RpcResponseCallback;
-import org.apache.spark.network.client.StreamCallbackWithID;
 import org.apache.spark.network.client.TransportClient;
+import org.apache.spark.network.server.AbstractAuthRpcHandler;
 import org.apache.spark.network.server.RpcHandler;
-import org.apache.spark.network.server.StreamManager;
 import org.apache.spark.network.util.JavaUtils;
 import org.apache.spark.network.util.TransportConf;
 
@@ -43,7 +42,7 @@
  * Note that the authentication process consists of multiple challenge-response pairs, each of
  * which are individual RPCs.
  */
-public class SaslRpcHandler extends RpcHandler {
+public class SaslRpcHandler extends AbstractAuthRpcHandler {
   private static final Logger logger = LoggerFactory.getLogger(SaslRpcHandler.class);
 
   /** Transport configuration. */
@@ -52,37 +51,28 @@ public class SaslRpcHandler extends RpcHandler {
   /** The client channel. */
   private final Channel channel;
 
-  /** RpcHandler we will delegate to for authenticated connections. */
-  private final RpcHandler delegate;
-
   /** Class which provides secret keys which are shared by server and client on a per-app basis. */
   private final SecretKeyHolder secretKeyHolder;
 
   private SparkSaslServer saslServer;
-  private boolean isComplete;
-  private boolean isAuthenticated;
 
   public SaslRpcHandler(
       TransportConf conf,
       Channel channel,
       RpcHandler delegate,
       SecretKeyHolder secretKeyHolder) {
+    super(delegate);
     this.conf = conf;
     this.channel = channel;
-    this.delegate = delegate;
     this.secretKeyHolder = secretKeyHolder;
     this.saslServer = null;
-    this.isComplete = false;
-    this.isAuthenticated = false;
   }
 
   @Override
-  public void receive(TransportClient client, ByteBuffer message, RpcResponseCallback callback) {
-    if (isComplete) {
-      // Authentication complete, delegate to base handler.
-      delegate.receive(client, message, callback);
-      return;
-    }
+  public boolean doAuthChallenge(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
     if (saslServer == null || !saslServer.isComplete()) {
       ByteBuf nettyBuf = Unpooled.wrappedBuffer(message);
       SaslMessage saslMessage;
@@ -118,43 +108,21 @@ public void receive(TransportClient client, ByteBuffer message, RpcResponseCallb
       if (!SparkSaslServer.QOP_AUTH_CONF.equals(saslServer.getNegotiatedProperty(Sasl.QOP))) {
         logger.debug("SASL authentication successful for channel {}", client);
         complete(true);
-        return;
+        return true;
       }
 
       logger.debug("Enabling encryption for channel {}", client);
       SaslEncryption.addToChannel(channel, saslServer, conf.maxSaslEncryptedBlockSize());
       complete(false);
-      return;
+      return true;
     }
-  }
-
-  @Override
-  public void receive(TransportClient client, ByteBuffer message) {
-    delegate.receive(client, message);
-  }
-
-  @Override
-  public StreamCallbackWithID receiveStream(
-      TransportClient client,
-      ByteBuffer message,
-      RpcResponseCallback callback) {
-    return delegate.receiveStream(client, message, callback);
-  }
-
-  @Override
-  public StreamManager getStreamManager() {
-    return delegate.getStreamManager();
-  }
-
-  @Override
-  public void channelActive(TransportClient client) {
-    delegate.channelActive(client);
+    return false;
   }
 
   @Override
   public void channelInactive(TransportClient client) {
     try {
-      delegate.channelInactive(client);
+      super.channelInactive(client);
     } finally {
       if (saslServer != null) {
         saslServer.dispose();
@@ -162,11 +130,6 @@ public void channelInactive(TransportClient client) {
     }
   }
 
-  @Override
-  public void exceptionCaught(Throwable cause, TransportClient client) {
-    delegate.exceptionCaught(cause, client);
-  }
-
   private void complete(boolean dispose) {
     if (dispose) {
       try {
@@ -177,7 +140,6 @@ private void complete(boolean dispose) {
     }
 
     saslServer = null;
-    isComplete = true;
   }
 
 }
diff --git a/common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java b/common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java
new file mode 100644
index 0000000000000..92eb886283448
--- /dev/null
+++ b/common/network-common/src/main/java/org/apache/spark/network/server/AbstractAuthRpcHandler.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.network.server;
+
+import java.nio.ByteBuffer;
+
+import org.apache.spark.network.client.RpcResponseCallback;
+import org.apache.spark.network.client.StreamCallbackWithID;
+import org.apache.spark.network.client.TransportClient;
+
+/**
+ * RPC Handler which performs authentication, and when it's successful, delegates further
+ * calls to another RPC handler. The authentication handshake itself should be implemented
+ * by subclasses.
+ */
+public abstract class AbstractAuthRpcHandler extends RpcHandler {
+  /** RpcHandler we will delegate to for authenticated connections. */
+  private final RpcHandler delegate;
+
+  private boolean isAuthenticated;
+
+  protected AbstractAuthRpcHandler(RpcHandler delegate) {
+    this.delegate = delegate;
+  }
+
+  /**
+   * Responds to an authentication challenge.
+   *
+   * @return Whether the client is authenticated.
+   */
+  protected abstract boolean doAuthChallenge(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback);
+
+  @Override
+  public final void receive(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
+    if (isAuthenticated) {
+      delegate.receive(client, message, callback);
+    } else {
+      isAuthenticated = doAuthChallenge(client, message, callback);
+    }
+  }
+
+  @Override
+  public final void receive(TransportClient client, ByteBuffer message) {
+    if (isAuthenticated) {
+      delegate.receive(client, message);
+    } else {
+      throw new SecurityException("Unauthenticated call to receive().");
+    }
+  }
+
+  @Override
+  public final StreamCallbackWithID receiveStream(
+      TransportClient client,
+      ByteBuffer message,
+      RpcResponseCallback callback) {
+    if (isAuthenticated) {
+      return delegate.receiveStream(client, message, callback);
+    } else {
+      throw new SecurityException("Unauthenticated call to receiveStream().");
+    }
+  }
+
+  @Override
+  public StreamManager getStreamManager() {
+    return delegate.getStreamManager();
+  }
+
+  @Override
+  public void channelActive(TransportClient client) {
+    delegate.channelActive(client);
+  }
+
+  @Override
+  public void channelInactive(TransportClient client) {
+    delegate.channelInactive(client);
+  }
+
+  @Override
+  public void exceptionCaught(Throwable cause, TransportClient client) {
+    delegate.exceptionCaught(cause, client);
+  }
+
+  public boolean isAuthenticated() {
+    return isAuthenticated;
+  }
+}
diff --git a/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java b/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java
index 2f9dd629df1ba..a87a6aae21092 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/crypto/AuthIntegrationSuite.java
@@ -34,7 +34,6 @@
 import org.apache.spark.network.client.RpcResponseCallback;
 import org.apache.spark.network.client.TransportClient;
 import org.apache.spark.network.client.TransportClientBootstrap;
-import org.apache.spark.network.sasl.SaslRpcHandler;
 import org.apache.spark.network.sasl.SaslServerBootstrap;
 import org.apache.spark.network.sasl.SecretKeyHolder;
 import org.apache.spark.network.server.RpcHandler;
@@ -65,8 +64,7 @@ public void testNewAuth() throws Exception {
 
     ByteBuffer reply = ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
     assertEquals("Pong", JavaUtils.bytesToString(reply));
-    assertTrue(ctx.authRpcHandler.doDelegate);
-    assertFalse(ctx.authRpcHandler.delegate instanceof SaslRpcHandler);
+    assertNull(ctx.authRpcHandler.saslHandler);
   }
 
   @Test
@@ -78,7 +76,7 @@ public void testAuthFailure() throws Exception {
       ctx.createClient("client");
       fail("Should have failed to create client.");
     } catch (Exception e) {
-      assertFalse(ctx.authRpcHandler.doDelegate);
+      assertFalse(ctx.authRpcHandler.isAuthenticated());
       assertFalse(ctx.serverChannel.isActive());
     }
   }
@@ -91,6 +89,8 @@ public void testSaslServerFallback() throws Exception {
 
     ByteBuffer reply = ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
     assertEquals("Pong", JavaUtils.bytesToString(reply));
+    assertNotNull(ctx.authRpcHandler.saslHandler);
+    assertTrue(ctx.authRpcHandler.isAuthenticated());
   }
 
   @Test
@@ -120,7 +120,7 @@ public void testAuthReplay() throws Exception {
       ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
       fail("Should have failed unencrypted RPC.");
     } catch (Exception e) {
-      assertTrue(ctx.authRpcHandler.doDelegate);
+      assertTrue(ctx.authRpcHandler.isAuthenticated());
     }
   }
 
@@ -151,7 +151,7 @@ public StreamManager getStreamManager() {
       ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), 5000);
       fail("Should have failed unencrypted RPC.");
     } catch (Exception e) {
-      assertTrue(ctx.authRpcHandler.doDelegate);
+      assertTrue(ctx.authRpcHandler.isAuthenticated());
       assertTrue(e.getMessage() + " is not an expected error", e.getMessage().contains("DDDDD"));
       // Verify we receive the complete error message
       int messageStart = e.getMessage().indexOf("DDDDD");
diff --git a/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java b/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
index cf2d72f71e8de..ecaeec98da182 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/sasl/SparkSaslSuite.java
@@ -357,7 +357,8 @@ public void testRpcHandlerDelegate() throws Exception {
   public void testDelegates() throws Exception {
     Method[] rpcHandlerMethods = RpcHandler.class.getDeclaredMethods();
     for (Method m : rpcHandlerMethods) {
-      SaslRpcHandler.class.getDeclaredMethod(m.getName(), m.getParameterTypes());
+      Method delegate = SaslRpcHandler.class.getMethod(m.getName(), m.getParameterTypes());
+      assertNotEquals(delegate.getDeclaringClass(), RpcHandler.class);
     }
   }
 

From 842fbed1d43f2ce5224d1367a7891f636243382e Mon Sep 17 00:00:00 2001
From: ulysses <youxiduo@weidian.com>
Date: Sat, 18 Apr 2020 09:27:44 -0700
Subject: [PATCH 0518/1280] [SPARK-31442][SQL] Print shuffle id at coalesce
 partitions target size

### What changes were proposed in this pull request?

Minor change. Print shuffle id.

### Why are the changes needed?

Make log more clear.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Not need.

Closes #28211 from ulysses-you/print-shuffle-id.

Authored-by: ulysses <youxiduo@weidian.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6c2bf8248a439951fe5157b34ad269cb31df0e16)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/adaptive/ShufflePartitionsUtil.scala  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
index 208cc059d238c..e10ed4f481cf7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/ShufflePartitionsUtil.scala
@@ -68,7 +68,9 @@ object ShufflePartitionsUtil extends Logging {
       math.ceil(totalPostShuffleInputSize / minNumPartitions.toDouble).toLong, 16)
     val targetSize = math.min(maxTargetSize, advisoryTargetSize)
 
-    logInfo(s"advisory target size: $advisoryTargetSize, actual target size $targetSize.")
+    val shuffleIds = mapOutputStatistics.map(_.shuffleId).mkString(", ")
+    logInfo(s"For shuffle($shuffleIds), advisory target size: $advisoryTargetSize, " +
+      s"actual target size $targetSize.")
 
     // Make sure these shuffles have the same number of partitions.
     val distinctNumShufflePartitions =

From 43b511644a0fd0d01f190dc137884fc1081c1933 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sat, 18 Apr 2020 13:37:12 -0700
Subject: [PATCH 0519/1280] [SPARK-31476][SQL] Add an ExpressionInfo entry for
 EXTRACT

### What changes were proposed in this pull request?

This PR intends to add an ExpressionInfo entry for EXTRACT for better documentations.
This PR comes from the comment in https://github.com/apache/spark/pull/21479#discussion_r409900080

### Why are the changes needed?

To make SQL documentations complete.

### Does this PR introduce any user-facing change?

Yes, this PR updates the `Spark SQL, Built-in Functions` page.

### How was this patch tested?

Run the example tests.

Closes #28251 from maropu/AddExtractExpr.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 74aed8cc8b94cb459ff12a6e8f1f15cb7aea8c40)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/FunctionRegistry.scala  |   1 +
 .../expressions/datetimeExpressions.scala     |  85 ++++++++----
 .../sql/catalyst/parser/AstBuilder.scala      |   2 +-
 .../sql-tests/results/extract.sql.out         | 124 +++++++++---------
 .../sql-tests/results/postgreSQL/date.sql.out |  52 ++++----
 5 files changed, 153 insertions(+), 111 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index c11186ebc0745..341a335861e5b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -423,6 +423,7 @@ object FunctionRegistry {
     expression[MakeTimestamp]("make_timestamp"),
     expression[MakeInterval]("make_interval"),
     expression[DatePart]("date_part"),
+    expression[Extract]("extract"),
 
     // collection functions
     expression[CreateArray]("array"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 391e0d0244990..323aa3b005d88 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -2103,6 +2103,31 @@ object DatePart {
   }
 }
 
+object DatePartLike {
+
+  def toEquivalentExpr(field: Expression, source: Expression): Expression = {
+    if (!field.foldable) {
+      throw new AnalysisException("The field parameter needs to be a foldable string value.")
+    }
+    val fieldEval = field.eval()
+    if (fieldEval == null) {
+      Literal(null, DoubleType)
+    } else {
+      val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
+      val errMsg = s"Literals of type '$fieldStr' are currently not supported " +
+        s"for the ${source.dataType.catalogString} type."
+      if (source.dataType == CalendarIntervalType) {
+        ExtractIntervalPart.parseExtractField(
+          fieldStr,
+          source,
+          throw new AnalysisException(errMsg))
+      } else {
+        DatePart.parseExtractField(fieldStr, source, throw new AnalysisException(errMsg))
+      }
+    }
+  }
+}
+
 @ExpressionDescription(
   usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp or interval source.",
   arguments = """
@@ -2156,28 +2181,8 @@ object DatePart {
 case class DatePart(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
 
-  def this(field: Expression, source: Expression) {
-    this(field, source, {
-      if (!field.foldable) {
-        throw new AnalysisException("The field parameter needs to be a foldable string value.")
-      }
-      val fieldEval = field.eval()
-      if (fieldEval == null) {
-        Literal(null, DoubleType)
-      } else {
-        val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
-        val errMsg = s"Literals of type '$fieldStr' are currently not supported " +
-          s"for the ${source.dataType.catalogString} type."
-        if (source.dataType == CalendarIntervalType) {
-          ExtractIntervalPart.parseExtractField(
-            fieldStr,
-            source,
-            throw new AnalysisException(errMsg))
-        } else {
-          DatePart.parseExtractField(fieldStr, source, throw new AnalysisException(errMsg))
-        }
-      }
-    })
+  def this(field: Expression, source: Expression) = {
+    this(field, source, DatePartLike.toEquivalentExpr(field, source))
   }
 
   override def flatArguments: Iterator[Any] = Iterator(field, source)
@@ -2185,6 +2190,42 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
   override def prettyName: String = "date_part"
 }
 
+@ExpressionDescription(
+  usage = "_FUNC_(field FROM source) - Extracts a part of the date/timestamp or interval source.",
+  arguments = """
+    Arguments:
+      * field - selects which part of the source should be extracted and supported string values
+                are the same with the `date_part` fields.
+      * source - a date/timestamp or interval column from where `field` should be extracted
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456');
+       2019
+      > SELECT _FUNC_(week FROM timestamp'2019-08-12 01:00:00.123456');
+       33
+      > SELECT _FUNC_(doy FROM DATE'2019-08-12');
+       224
+      > SELECT _FUNC_(SECONDS FROM timestamp'2019-10-01 00:00:01.000001');
+       1.000001
+      > SELECT _FUNC_(days FROM interval 1 year 10 months 5 days);
+       5
+      > SELECT _FUNC_(seconds FROM interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
+       30.001001
+  """,
+  since = "3.0.0")
+case class Extract(field: Expression, source: Expression, child: Expression)
+  extends RuntimeReplaceable {
+
+  def this(field: Expression, source: Expression) = {
+    this(field, source, DatePartLike.toEquivalentExpr(field, source))
+  }
+
+  override def flatArguments: Iterator[Any] = Iterator(field, source)
+  override def sql: String = s"$prettyName(${field.sql} FROM ${source.sql})"
+  override def prettyName: String = "extract"
+}
+
 /**
  * Returns the interval from startTimestamp to endTimestamp in which the `months` and `day` field
  * is set to 0 and the `microseconds` field is initialized to the microsecond difference
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index b2ef96db0fb26..6c5917053e135 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1550,7 +1550,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitExtract(ctx: ExtractContext): Expression = withOrigin(ctx) {
     val arguments = Seq(Literal(ctx.field.getText), expression(ctx.source))
-    UnresolvedFunction("date_part", arguments, isDistinct = false)
+    UnresolvedFunction("extract", arguments, isDistinct = false)
   }
 
   /**
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 1f77c67871ae8..47cd5180ddebd 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -13,7 +13,7 @@ struct<>
 -- !query
 select extract(millennium from c) from t
 -- !query schema
-struct<date_part('millennium', t.`c`):int>
+struct<extract('millennium' FROM t.`c`):int>
 -- !query output
 3
 
@@ -21,7 +21,7 @@ struct<date_part('millennium', t.`c`):int>
 -- !query
 select extract(millennia from c) from t
 -- !query schema
-struct<date_part('millennia', t.`c`):int>
+struct<extract('millennia' FROM t.`c`):int>
 -- !query output
 3
 
@@ -29,7 +29,7 @@ struct<date_part('millennia', t.`c`):int>
 -- !query
 select extract(mil from c) from t
 -- !query schema
-struct<date_part('mil', t.`c`):int>
+struct<extract('mil' FROM t.`c`):int>
 -- !query output
 3
 
@@ -37,7 +37,7 @@ struct<date_part('mil', t.`c`):int>
 -- !query
 select extract(mils from c) from t
 -- !query schema
-struct<date_part('mils', t.`c`):int>
+struct<extract('mils' FROM t.`c`):int>
 -- !query output
 3
 
@@ -45,7 +45,7 @@ struct<date_part('mils', t.`c`):int>
 -- !query
 select extract(century from c) from t
 -- !query schema
-struct<date_part('century', t.`c`):int>
+struct<extract('century' FROM t.`c`):int>
 -- !query output
 21
 
@@ -53,7 +53,7 @@ struct<date_part('century', t.`c`):int>
 -- !query
 select extract(centuries from c) from t
 -- !query schema
-struct<date_part('centuries', t.`c`):int>
+struct<extract('centuries' FROM t.`c`):int>
 -- !query output
 21
 
@@ -61,7 +61,7 @@ struct<date_part('centuries', t.`c`):int>
 -- !query
 select extract(c from c) from t
 -- !query schema
-struct<date_part('c', t.`c`):int>
+struct<extract('c' FROM t.`c`):int>
 -- !query output
 21
 
@@ -69,7 +69,7 @@ struct<date_part('c', t.`c`):int>
 -- !query
 select extract(cent from c)from t
 -- !query schema
-struct<date_part('cent', t.`c`):int>
+struct<extract('cent' FROM t.`c`):int>
 -- !query output
 21
 
@@ -77,7 +77,7 @@ struct<date_part('cent', t.`c`):int>
 -- !query
 select extract(decade from c) from t
 -- !query schema
-struct<date_part('decade', t.`c`):int>
+struct<extract('decade' FROM t.`c`):int>
 -- !query output
 201
 
@@ -85,7 +85,7 @@ struct<date_part('decade', t.`c`):int>
 -- !query
 select extract(decades from c) from t
 -- !query schema
-struct<date_part('decades', t.`c`):int>
+struct<extract('decades' FROM t.`c`):int>
 -- !query output
 201
 
@@ -93,7 +93,7 @@ struct<date_part('decades', t.`c`):int>
 -- !query
 select extract(dec from c) from t
 -- !query schema
-struct<date_part('dec', t.`c`):int>
+struct<extract('dec' FROM t.`c`):int>
 -- !query output
 201
 
@@ -101,7 +101,7 @@ struct<date_part('dec', t.`c`):int>
 -- !query
 select extract(decs from c) from t
 -- !query schema
-struct<date_part('decs', t.`c`):int>
+struct<extract('decs' FROM t.`c`):int>
 -- !query output
 201
 
@@ -109,7 +109,7 @@ struct<date_part('decs', t.`c`):int>
 -- !query
 select extract(year from c), extract(year from i) from t
 -- !query schema
-struct<date_part('year', t.`c`):int,date_part('year', t.`i`):int>
+struct<extract('year' FROM t.`c`):int,extract('year' FROM t.`i`):int>
 -- !query output
 2011	11
 
@@ -117,7 +117,7 @@ struct<date_part('year', t.`c`):int,date_part('year', t.`i`):int>
 -- !query
 select extract(y from c), extract(y from i) from t
 -- !query schema
-struct<date_part('y', t.`c`):int,date_part('y', t.`i`):int>
+struct<extract('y' FROM t.`c`):int,extract('y' FROM t.`i`):int>
 -- !query output
 2011	11
 
@@ -125,7 +125,7 @@ struct<date_part('y', t.`c`):int,date_part('y', t.`i`):int>
 -- !query
 select extract(years from c), extract(years from i) from t
 -- !query schema
-struct<date_part('years', t.`c`):int,date_part('years', t.`i`):int>
+struct<extract('years' FROM t.`c`):int,extract('years' FROM t.`i`):int>
 -- !query output
 2011	11
 
@@ -133,7 +133,7 @@ struct<date_part('years', t.`c`):int,date_part('years', t.`i`):int>
 -- !query
 select extract(yr from c), extract(yr from i) from t
 -- !query schema
-struct<date_part('yr', t.`c`):int,date_part('yr', t.`i`):int>
+struct<extract('yr' FROM t.`c`):int,extract('yr' FROM t.`i`):int>
 -- !query output
 2011	11
 
@@ -141,7 +141,7 @@ struct<date_part('yr', t.`c`):int,date_part('yr', t.`i`):int>
 -- !query
 select extract(yrs from c), extract(yrs from i) from t
 -- !query schema
-struct<date_part('yrs', t.`c`):int,date_part('yrs', t.`i`):int>
+struct<extract('yrs' FROM t.`c`):int,extract('yrs' FROM t.`i`):int>
 -- !query output
 2011	11
 
@@ -149,7 +149,7 @@ struct<date_part('yrs', t.`c`):int,date_part('yrs', t.`i`):int>
 -- !query
 select extract(isoyear from c) from t
 -- !query schema
-struct<date_part('isoyear', t.`c`):int>
+struct<extract('isoyear' FROM t.`c`):int>
 -- !query output
 2011
 
@@ -157,7 +157,7 @@ struct<date_part('isoyear', t.`c`):int>
 -- !query
 select extract(quarter from c) from t
 -- !query schema
-struct<date_part('quarter', t.`c`):int>
+struct<extract('quarter' FROM t.`c`):int>
 -- !query output
 2
 
@@ -165,7 +165,7 @@ struct<date_part('quarter', t.`c`):int>
 -- !query
 select extract(qtr from c) from t
 -- !query schema
-struct<date_part('qtr', t.`c`):int>
+struct<extract('qtr' FROM t.`c`):int>
 -- !query output
 2
 
@@ -173,7 +173,7 @@ struct<date_part('qtr', t.`c`):int>
 -- !query
 select extract(month from c), extract(month from i) from t
 -- !query schema
-struct<date_part('month', t.`c`):int,date_part('month', t.`i`):tinyint>
+struct<extract('month' FROM t.`c`):int,extract('month' FROM t.`i`):tinyint>
 -- !query output
 5	8
 
@@ -181,7 +181,7 @@ struct<date_part('month', t.`c`):int,date_part('month', t.`i`):tinyint>
 -- !query
 select extract(mon from c), extract(mon from i) from t
 -- !query schema
-struct<date_part('mon', t.`c`):int,date_part('mon', t.`i`):tinyint>
+struct<extract('mon' FROM t.`c`):int,extract('mon' FROM t.`i`):tinyint>
 -- !query output
 5	8
 
@@ -189,7 +189,7 @@ struct<date_part('mon', t.`c`):int,date_part('mon', t.`i`):tinyint>
 -- !query
 select extract(mons from c), extract(mons from i) from t
 -- !query schema
-struct<date_part('mons', t.`c`):int,date_part('mons', t.`i`):tinyint>
+struct<extract('mons' FROM t.`c`):int,extract('mons' FROM t.`i`):tinyint>
 -- !query output
 5	8
 
@@ -197,7 +197,7 @@ struct<date_part('mons', t.`c`):int,date_part('mons', t.`i`):tinyint>
 -- !query
 select extract(months from c), extract(months from i) from t
 -- !query schema
-struct<date_part('months', t.`c`):int,date_part('months', t.`i`):tinyint>
+struct<extract('months' FROM t.`c`):int,extract('months' FROM t.`i`):tinyint>
 -- !query output
 5	8
 
@@ -205,7 +205,7 @@ struct<date_part('months', t.`c`):int,date_part('months', t.`i`):tinyint>
 -- !query
 select extract(week from c) from t
 -- !query schema
-struct<date_part('week', t.`c`):int>
+struct<extract('week' FROM t.`c`):int>
 -- !query output
 18
 
@@ -213,7 +213,7 @@ struct<date_part('week', t.`c`):int>
 -- !query
 select extract(w from c) from t
 -- !query schema
-struct<date_part('w', t.`c`):int>
+struct<extract('w' FROM t.`c`):int>
 -- !query output
 18
 
@@ -221,7 +221,7 @@ struct<date_part('w', t.`c`):int>
 -- !query
 select extract(weeks from c) from t
 -- !query schema
-struct<date_part('weeks', t.`c`):int>
+struct<extract('weeks' FROM t.`c`):int>
 -- !query output
 18
 
@@ -229,7 +229,7 @@ struct<date_part('weeks', t.`c`):int>
 -- !query
 select extract(day from c), extract(day from i) from t
 -- !query schema
-struct<date_part('day', t.`c`):int,date_part('day', t.`i`):int>
+struct<extract('day' FROM t.`c`):int,extract('day' FROM t.`i`):int>
 -- !query output
 6	30
 
@@ -237,7 +237,7 @@ struct<date_part('day', t.`c`):int,date_part('day', t.`i`):int>
 -- !query
 select extract(d from c), extract(d from i) from t
 -- !query schema
-struct<date_part('d', t.`c`):int,date_part('d', t.`i`):int>
+struct<extract('d' FROM t.`c`):int,extract('d' FROM t.`i`):int>
 -- !query output
 6	30
 
@@ -245,7 +245,7 @@ struct<date_part('d', t.`c`):int,date_part('d', t.`i`):int>
 -- !query
 select extract(days from c), extract(days from i) from t
 -- !query schema
-struct<date_part('days', t.`c`):int,date_part('days', t.`i`):int>
+struct<extract('days' FROM t.`c`):int,extract('days' FROM t.`i`):int>
 -- !query output
 6	30
 
@@ -253,7 +253,7 @@ struct<date_part('days', t.`c`):int,date_part('days', t.`i`):int>
 -- !query
 select extract(dayofweek from c) from t
 -- !query schema
-struct<date_part('dayofweek', t.`c`):int>
+struct<extract('dayofweek' FROM t.`c`):int>
 -- !query output
 6
 
@@ -261,7 +261,7 @@ struct<date_part('dayofweek', t.`c`):int>
 -- !query
 select extract(dow from c) from t
 -- !query schema
-struct<date_part('dow', t.`c`):int>
+struct<extract('dow' FROM t.`c`):int>
 -- !query output
 5
 
@@ -269,7 +269,7 @@ struct<date_part('dow', t.`c`):int>
 -- !query
 select extract(isodow from c) from t
 -- !query schema
-struct<date_part('isodow', t.`c`):int>
+struct<extract('isodow' FROM t.`c`):int>
 -- !query output
 5
 
@@ -277,7 +277,7 @@ struct<date_part('isodow', t.`c`):int>
 -- !query
 select extract(doy from c) from t
 -- !query schema
-struct<date_part('doy', t.`c`):int>
+struct<extract('doy' FROM t.`c`):int>
 -- !query output
 126
 
@@ -285,7 +285,7 @@ struct<date_part('doy', t.`c`):int>
 -- !query
 select extract(hour from c), extract(hour from i) from t
 -- !query schema
-struct<date_part('hour', t.`c`):int,date_part('hour', t.`i`):bigint>
+struct<extract('hour' FROM t.`c`):int,extract('hour' FROM t.`i`):bigint>
 -- !query output
 7	40
 
@@ -293,7 +293,7 @@ struct<date_part('hour', t.`c`):int,date_part('hour', t.`i`):bigint>
 -- !query
 select extract(h from c), extract(h from i) from t
 -- !query schema
-struct<date_part('h', t.`c`):int,date_part('h', t.`i`):bigint>
+struct<extract('h' FROM t.`c`):int,extract('h' FROM t.`i`):bigint>
 -- !query output
 7	40
 
@@ -301,7 +301,7 @@ struct<date_part('h', t.`c`):int,date_part('h', t.`i`):bigint>
 -- !query
 select extract(hours from c), extract(hours from i) from t
 -- !query schema
-struct<date_part('hours', t.`c`):int,date_part('hours', t.`i`):bigint>
+struct<extract('hours' FROM t.`c`):int,extract('hours' FROM t.`i`):bigint>
 -- !query output
 7	40
 
@@ -309,7 +309,7 @@ struct<date_part('hours', t.`c`):int,date_part('hours', t.`i`):bigint>
 -- !query
 select extract(hr from c), extract(hr from i) from t
 -- !query schema
-struct<date_part('hr', t.`c`):int,date_part('hr', t.`i`):bigint>
+struct<extract('hr' FROM t.`c`):int,extract('hr' FROM t.`i`):bigint>
 -- !query output
 7	40
 
@@ -317,7 +317,7 @@ struct<date_part('hr', t.`c`):int,date_part('hr', t.`i`):bigint>
 -- !query
 select extract(hrs from c), extract(hrs from i) from t
 -- !query schema
-struct<date_part('hrs', t.`c`):int,date_part('hrs', t.`i`):bigint>
+struct<extract('hrs' FROM t.`c`):int,extract('hrs' FROM t.`i`):bigint>
 -- !query output
 7	40
 
@@ -325,7 +325,7 @@ struct<date_part('hrs', t.`c`):int,date_part('hrs', t.`i`):bigint>
 -- !query
 select extract(minute from c), extract(minute from i) from t
 -- !query schema
-struct<date_part('minute', t.`c`):int,date_part('minute', t.`i`):tinyint>
+struct<extract('minute' FROM t.`c`):int,extract('minute' FROM t.`i`):tinyint>
 -- !query output
 8	50
 
@@ -333,7 +333,7 @@ struct<date_part('minute', t.`c`):int,date_part('minute', t.`i`):tinyint>
 -- !query
 select extract(m from c), extract(m from i) from t
 -- !query schema
-struct<date_part('m', t.`c`):int,date_part('m', t.`i`):tinyint>
+struct<extract('m' FROM t.`c`):int,extract('m' FROM t.`i`):tinyint>
 -- !query output
 8	50
 
@@ -341,7 +341,7 @@ struct<date_part('m', t.`c`):int,date_part('m', t.`i`):tinyint>
 -- !query
 select extract(min from c), extract(min from i) from t
 -- !query schema
-struct<date_part('min', t.`c`):int,date_part('min', t.`i`):tinyint>
+struct<extract('min' FROM t.`c`):int,extract('min' FROM t.`i`):tinyint>
 -- !query output
 8	50
 
@@ -349,7 +349,7 @@ struct<date_part('min', t.`c`):int,date_part('min', t.`i`):tinyint>
 -- !query
 select extract(mins from c), extract(mins from i) from t
 -- !query schema
-struct<date_part('mins', t.`c`):int,date_part('mins', t.`i`):tinyint>
+struct<extract('mins' FROM t.`c`):int,extract('mins' FROM t.`i`):tinyint>
 -- !query output
 8	50
 
@@ -357,7 +357,7 @@ struct<date_part('mins', t.`c`):int,date_part('mins', t.`i`):tinyint>
 -- !query
 select extract(minutes from c), extract(minutes from i) from t
 -- !query schema
-struct<date_part('minutes', t.`c`):int,date_part('minutes', t.`i`):tinyint>
+struct<extract('minutes' FROM t.`c`):int,extract('minutes' FROM t.`i`):tinyint>
 -- !query output
 8	50
 
@@ -365,7 +365,7 @@ struct<date_part('minutes', t.`c`):int,date_part('minutes', t.`i`):tinyint>
 -- !query
 select extract(second from c), extract(second from i) from t
 -- !query schema
-struct<date_part('second', t.`c`):decimal(8,6),date_part('second', t.`i`):decimal(8,6)>
+struct<extract('second' FROM t.`c`):decimal(8,6),extract('second' FROM t.`i`):decimal(8,6)>
 -- !query output
 9.123456	6.789000
 
@@ -373,7 +373,7 @@ struct<date_part('second', t.`c`):decimal(8,6),date_part('second', t.`i`):decima
 -- !query
 select extract(s from c), extract(s from i) from t
 -- !query schema
-struct<date_part('s', t.`c`):decimal(8,6),date_part('s', t.`i`):decimal(8,6)>
+struct<extract('s' FROM t.`c`):decimal(8,6),extract('s' FROM t.`i`):decimal(8,6)>
 -- !query output
 9.123456	6.789000
 
@@ -381,7 +381,7 @@ struct<date_part('s', t.`c`):decimal(8,6),date_part('s', t.`i`):decimal(8,6)>
 -- !query
 select extract(sec from c), extract(sec from i) from t
 -- !query schema
-struct<date_part('sec', t.`c`):decimal(8,6),date_part('sec', t.`i`):decimal(8,6)>
+struct<extract('sec' FROM t.`c`):decimal(8,6),extract('sec' FROM t.`i`):decimal(8,6)>
 -- !query output
 9.123456	6.789000
 
@@ -389,7 +389,7 @@ struct<date_part('sec', t.`c`):decimal(8,6),date_part('sec', t.`i`):decimal(8,6)
 -- !query
 select extract(seconds from c), extract(seconds from i) from t
 -- !query schema
-struct<date_part('seconds', t.`c`):decimal(8,6),date_part('seconds', t.`i`):decimal(8,6)>
+struct<extract('seconds' FROM t.`c`):decimal(8,6),extract('seconds' FROM t.`i`):decimal(8,6)>
 -- !query output
 9.123456	6.789000
 
@@ -397,7 +397,7 @@ struct<date_part('seconds', t.`c`):decimal(8,6),date_part('seconds', t.`i`):deci
 -- !query
 select extract(secs from c), extract(secs from i) from t
 -- !query schema
-struct<date_part('secs', t.`c`):decimal(8,6),date_part('secs', t.`i`):decimal(8,6)>
+struct<extract('secs' FROM t.`c`):decimal(8,6),extract('secs' FROM t.`i`):decimal(8,6)>
 -- !query output
 9.123456	6.789000
 
@@ -405,7 +405,7 @@ struct<date_part('secs', t.`c`):decimal(8,6),date_part('secs', t.`i`):decimal(8,
 -- !query
 select extract(milliseconds from c) from t
 -- !query schema
-struct<date_part('milliseconds', t.`c`):decimal(8,3)>
+struct<extract('milliseconds' FROM t.`c`):decimal(8,3)>
 -- !query output
 9123.456
 
@@ -413,7 +413,7 @@ struct<date_part('milliseconds', t.`c`):decimal(8,3)>
 -- !query
 select extract(msec from c) from t
 -- !query schema
-struct<date_part('msec', t.`c`):decimal(8,3)>
+struct<extract('msec' FROM t.`c`):decimal(8,3)>
 -- !query output
 9123.456
 
@@ -421,7 +421,7 @@ struct<date_part('msec', t.`c`):decimal(8,3)>
 -- !query
 select extract(msecs from c) from t
 -- !query schema
-struct<date_part('msecs', t.`c`):decimal(8,3)>
+struct<extract('msecs' FROM t.`c`):decimal(8,3)>
 -- !query output
 9123.456
 
@@ -429,7 +429,7 @@ struct<date_part('msecs', t.`c`):decimal(8,3)>
 -- !query
 select extract(millisecon from c) from t
 -- !query schema
-struct<date_part('millisecon', t.`c`):decimal(8,3)>
+struct<extract('millisecon' FROM t.`c`):decimal(8,3)>
 -- !query output
 9123.456
 
@@ -437,7 +437,7 @@ struct<date_part('millisecon', t.`c`):decimal(8,3)>
 -- !query
 select extract(mseconds from c) from t
 -- !query schema
-struct<date_part('mseconds', t.`c`):decimal(8,3)>
+struct<extract('mseconds' FROM t.`c`):decimal(8,3)>
 -- !query output
 9123.456
 
@@ -445,7 +445,7 @@ struct<date_part('mseconds', t.`c`):decimal(8,3)>
 -- !query
 select extract(ms from c) from t
 -- !query schema
-struct<date_part('ms', t.`c`):decimal(8,3)>
+struct<extract('ms' FROM t.`c`):decimal(8,3)>
 -- !query output
 9123.456
 
@@ -453,7 +453,7 @@ struct<date_part('ms', t.`c`):decimal(8,3)>
 -- !query
 select extract(microseconds from c) from t
 -- !query schema
-struct<date_part('microseconds', t.`c`):int>
+struct<extract('microseconds' FROM t.`c`):int>
 -- !query output
 9123456
 
@@ -461,7 +461,7 @@ struct<date_part('microseconds', t.`c`):int>
 -- !query
 select extract(usec from c) from t
 -- !query schema
-struct<date_part('usec', t.`c`):int>
+struct<extract('usec' FROM t.`c`):int>
 -- !query output
 9123456
 
@@ -469,7 +469,7 @@ struct<date_part('usec', t.`c`):int>
 -- !query
 select extract(usecs from c) from t
 -- !query schema
-struct<date_part('usecs', t.`c`):int>
+struct<extract('usecs' FROM t.`c`):int>
 -- !query output
 9123456
 
@@ -477,7 +477,7 @@ struct<date_part('usecs', t.`c`):int>
 -- !query
 select extract(useconds from c) from t
 -- !query schema
-struct<date_part('useconds', t.`c`):int>
+struct<extract('useconds' FROM t.`c`):int>
 -- !query output
 9123456
 
@@ -485,7 +485,7 @@ struct<date_part('useconds', t.`c`):int>
 -- !query
 select extract(microsecon from c) from t
 -- !query schema
-struct<date_part('microsecon', t.`c`):int>
+struct<extract('microsecon' FROM t.`c`):int>
 -- !query output
 9123456
 
@@ -493,7 +493,7 @@ struct<date_part('microsecon', t.`c`):int>
 -- !query
 select extract(us from c) from t
 -- !query schema
-struct<date_part('us', t.`c`):int>
+struct<extract('us' FROM t.`c`):int>
 -- !query output
 9123456
 
@@ -501,7 +501,7 @@ struct<date_part('us', t.`c`):int>
 -- !query
 select extract(epoch from c) from t
 -- !query schema
-struct<date_part('epoch', t.`c`):decimal(20,6)>
+struct<extract('epoch' FROM t.`c`):decimal(20,6)>
 -- !query output
 1304665689.123456
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index e5a7c43c0a8ab..0dac13d9deae0 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -574,7 +574,7 @@ struct<Two days:interval>
 -- !query
 SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01')
 -- !query schema
-struct<date_part('EPOCH', DATE '1970-01-01'):decimal(20,6)>
+struct<extract('EPOCH' FROM DATE '1970-01-01'):decimal(20,6)>
 -- !query output
 0.000000
 
@@ -582,7 +582,7 @@ struct<date_part('EPOCH', DATE '1970-01-01'):decimal(20,6)>
 -- !query
 SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01')
 -- !query schema
-struct<date_part('EPOCH', TIMESTAMP '1970-01-01 00:00:00'):decimal(20,6)>
+struct<extract('EPOCH' FROM TIMESTAMP '1970-01-01 00:00:00'):decimal(20,6)>
 -- !query output
 0.000000
 
@@ -590,7 +590,7 @@ struct<date_part('EPOCH', TIMESTAMP '1970-01-01 00:00:00'):decimal(20,6)>
 -- !query
 SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('CENTURY', to_date('0101-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<extract('CENTURY' FROM to_date('0101-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -2
 
@@ -598,7 +598,7 @@ struct<date_part('CENTURY', to_date('0101-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('CENTURY', to_date('0100-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<extract('CENTURY' FROM to_date('0100-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -606,7 +606,7 @@ struct<date_part('CENTURY', to_date('0100-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('CENTURY', to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<extract('CENTURY' FROM to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -614,7 +614,7 @@ struct<date_part('CENTURY', to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM DATE '0001-01-01')
 -- !query schema
-struct<date_part('CENTURY', DATE '0001-01-01'):int>
+struct<extract('CENTURY' FROM DATE '0001-01-01'):int>
 -- !query output
 1
 
@@ -622,7 +622,7 @@ struct<date_part('CENTURY', DATE '0001-01-01'):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD')
 -- !query schema
-struct<date_part('CENTURY', DATE '0001-01-01'):int>
+struct<extract('CENTURY' FROM DATE '0001-01-01'):int>
 -- !query output
 1
 
@@ -630,7 +630,7 @@ struct<date_part('CENTURY', DATE '0001-01-01'):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM DATE '1900-12-31')
 -- !query schema
-struct<date_part('CENTURY', DATE '1900-12-31'):int>
+struct<extract('CENTURY' FROM DATE '1900-12-31'):int>
 -- !query output
 19
 
@@ -638,7 +638,7 @@ struct<date_part('CENTURY', DATE '1900-12-31'):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM DATE '1901-01-01')
 -- !query schema
-struct<date_part('CENTURY', DATE '1901-01-01'):int>
+struct<extract('CENTURY' FROM DATE '1901-01-01'):int>
 -- !query output
 20
 
@@ -646,7 +646,7 @@ struct<date_part('CENTURY', DATE '1901-01-01'):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM DATE '2000-12-31')
 -- !query schema
-struct<date_part('CENTURY', DATE '2000-12-31'):int>
+struct<extract('CENTURY' FROM DATE '2000-12-31'):int>
 -- !query output
 20
 
@@ -654,7 +654,7 @@ struct<date_part('CENTURY', DATE '2000-12-31'):int>
 -- !query
 SELECT EXTRACT(CENTURY FROM DATE '2001-01-01')
 -- !query schema
-struct<date_part('CENTURY', DATE '2001-01-01'):int>
+struct<extract('CENTURY' FROM DATE '2001-01-01'):int>
 -- !query output
 21
 
@@ -670,7 +670,7 @@ true
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('MILLENNIUM', to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<extract('MILLENNIUM' FROM to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -678,7 +678,7 @@ struct<date_part('MILLENNIUM', to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD')
 -- !query schema
-struct<date_part('MILLENNIUM', DATE '0001-01-01'):int>
+struct<extract('MILLENNIUM' FROM DATE '0001-01-01'):int>
 -- !query output
 1
 
@@ -686,7 +686,7 @@ struct<date_part('MILLENNIUM', DATE '0001-01-01'):int>
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31')
 -- !query schema
-struct<date_part('MILLENNIUM', DATE '1000-12-31'):int>
+struct<extract('MILLENNIUM' FROM DATE '1000-12-31'):int>
 -- !query output
 1
 
@@ -694,7 +694,7 @@ struct<date_part('MILLENNIUM', DATE '1000-12-31'):int>
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01')
 -- !query schema
-struct<date_part('MILLENNIUM', DATE '1001-01-01'):int>
+struct<extract('MILLENNIUM' FROM DATE '1001-01-01'):int>
 -- !query output
 2
 
@@ -702,7 +702,7 @@ struct<date_part('MILLENNIUM', DATE '1001-01-01'):int>
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31')
 -- !query schema
-struct<date_part('MILLENNIUM', DATE '2000-12-31'):int>
+struct<extract('MILLENNIUM' FROM DATE '2000-12-31'):int>
 -- !query output
 2
 
@@ -710,7 +710,7 @@ struct<date_part('MILLENNIUM', DATE '2000-12-31'):int>
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01')
 -- !query schema
-struct<date_part('MILLENNIUM', DATE '2001-01-01'):int>
+struct<extract('MILLENNIUM' FROM DATE '2001-01-01'):int>
 -- !query output
 3
 
@@ -718,7 +718,7 @@ struct<date_part('MILLENNIUM', DATE '2001-01-01'):int>
 -- !query
 SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE)
 -- !query schema
-struct<date_part('MILLENNIUM', current_date()):int>
+struct<extract('MILLENNIUM' FROM current_date()):int>
 -- !query output
 3
 
@@ -726,7 +726,7 @@ struct<date_part('MILLENNIUM', current_date()):int>
 -- !query
 SELECT EXTRACT(DECADE FROM DATE '1994-12-25')
 -- !query schema
-struct<date_part('DECADE', DATE '1994-12-25'):int>
+struct<extract('DECADE' FROM DATE '1994-12-25'):int>
 -- !query output
 199
 
@@ -734,7 +734,7 @@ struct<date_part('DECADE', DATE '1994-12-25'):int>
 -- !query
 SELECT EXTRACT(DECADE FROM DATE '0010-01-01')
 -- !query schema
-struct<date_part('DECADE', DATE '0010-01-01'):int>
+struct<extract('DECADE' FROM DATE '0010-01-01'):int>
 -- !query output
 1
 
@@ -742,7 +742,7 @@ struct<date_part('DECADE', DATE '0010-01-01'):int>
 -- !query
 SELECT EXTRACT(DECADE FROM DATE '0009-12-31')
 -- !query schema
-struct<date_part('DECADE', DATE '0009-12-31'):int>
+struct<extract('DECADE' FROM DATE '0009-12-31'):int>
 -- !query output
 0
 
@@ -750,7 +750,7 @@ struct<date_part('DECADE', DATE '0009-12-31'):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', to_date('0001-01-01 BC', 'yyyy-MM-dd G')):int>
+struct<extract('DECADE' FROM to_date('0001-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 0
 
@@ -758,7 +758,7 @@ struct<date_part('DECADE', to_date('0001-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', to_date('0002-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<extract('DECADE' FROM to_date('0002-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -766,7 +766,7 @@ struct<date_part('DECADE', to_date('0002-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', to_date('0011-01-01 BC', 'yyyy-MM-dd G')):int>
+struct<extract('DECADE' FROM to_date('0011-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -1
 
@@ -774,7 +774,7 @@ struct<date_part('DECADE', to_date('0011-01-01 BC', 'yyyy-MM-dd G')):int>
 -- !query
 SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G'))
 -- !query schema
-struct<date_part('DECADE', to_date('0012-12-31 BC', 'yyyy-MM-dd G')):int>
+struct<extract('DECADE' FROM to_date('0012-12-31 BC', 'yyyy-MM-dd G')):int>
 -- !query output
 -2
 
@@ -790,7 +790,7 @@ true
 -- !query
 SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000')
 -- !query schema
-struct<date_part('CENTURY', TIMESTAMP '1970-03-20 04:30:00'):int>
+struct<extract('CENTURY' FROM TIMESTAMP '1970-03-20 04:30:00'):int>
 -- !query output
 20
 

From 59166081daa6fb19c8fcdc2d83bdc19d8a3de1db Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 20 Apr 2020 10:31:44 +0900
Subject: [PATCH 0520/1280] [SPARK-29672][BUILD][PYTHON][FOLLOW-UP] Recover
 PySpark via pip installation with deprecated Python 2, 3.4 and 3.5

### What changes were proposed in this pull request?

The RC fails to install against Python 2.7 via `pip`. We deprecated but didn't remove Python 2, 3.4 and 3.5 support yet. This PR partially reverts the changes from SPARK-29672 to recover Python 2, 3.4 and 3.5 pip installation.

```bash
python2.7 -m pip install https://dist.apache.org/repos/dist/dev/spark/v3.0.0-rc1-bin/pyspark-3.0.0.tar.gz
```
```
...
Collecting https://dist.apache.org/repos/dist/dev/spark/v3.0.0-rc1-bin/pyspark-3.0.0.tar.gz
  Using cached https://dist.apache.org/repos/dist/dev/spark/v3.0.0-rc1-bin/pyspark-3.0.0.tar.gz (203.0 MB)
    ERROR: Command errored out with exit status 1:
     command: /System/Library/Frameworks/Python.framework/Versions/2.7/Resources/Python.app/Contents/MacOS/Python -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/private/var/folders/_1/bzcp960d0hlb988k90654z2w0000gp/T/pip-req-build-sfCnmZ/setup.py'"'"'; __file__='"'"'/private/var/folders/_1/bzcp960d0hlb988k90654z2w0000gp/T/pip-req-build-sfCnmZ/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' egg_info --egg-base /private/var/folders/_1/bzcp960d0hlb988k90654z2w0000gp/T/pip-req-build-sfCnmZ/pip-egg-info
         cwd: /private/var/folders/_1/bzcp960d0hlb988k90654z2w0000gp/T/pip-req-build-sfCnmZ/
    Complete output (6 lines):
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/private/var/folders/_1/bzcp960d0hlb988k90654z2w0000gp/T/pip-req-build-sfCnmZ/setup.py", line 27
        file=sys.stderr)
            ^
    SyntaxError: invalid syntax
    ----------------------------------------
ERROR: Command errored out with exit status 1: python setup.py egg_info Check the logs for full command output.
```

### Why are the changes needed?

To keep the deprecated support instead of removing.

### Does this PR introduce any user-facing change?

No, it's the change in unreleased branches only yet.

### How was this patch tested?

```bash
./build/mvn -DskipTests -Phive -Phive-thriftserver clean package
cd python
python2.7 setup.py sdist
python2.7 -m pip install dist/pyspark-3.1.0.dev0.tar.gz
```

Closes #28243 from HyukjinKwon/SPARK-29672-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit bc212df61033050d4810ae56c4f5c44b616081c8)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/setup.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/python/setup.py b/python/setup.py
index 622e6077dc36e..a1cde31aa0ada 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -16,14 +16,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import glob
 import os
 import sys
 from setuptools import setup
 from shutil import copyfile, copytree, rmtree
 
-if sys.version_info < (3, 6):
-    print("Python versions prior to 3.6 are not supported for pip installed PySpark.",
+if sys.version_info < (2, 7):
+    print("Python versions prior to 2.7 are not supported for pip installed PySpark.",
           file=sys.stderr)
     sys.exit(-1)
 

From 95796ac7dafcfc2a7b57c22f6822d4a5f4d8a1ae Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Mon, 20 Apr 2020 02:59:09 +0000
Subject: [PATCH 0521/1280] [SPARK-31256][SQL] DataFrameNaFunctions.drop should
 work for nested columns

### What changes were proposed in this pull request?

#26700 removed the ability to drop a row whose nested column value is null.

For example, for the following `df`:
```
val schema = new StructType()
  .add("c1", new StructType()
    .add("c1-1", StringType)
    .add("c1-2", StringType))
val data = Seq(Row(Row(null, "a2")), Row(Row("b1", "b2")), Row(null))
val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
df.show
+--------+
|      c1|
+--------+
|  [, a2]|
|[b1, b2]|
|    null|
+--------+
```
In Spark 2.4.4,
```
df.na.drop("any", Seq("c1.c1-1")).show
+--------+
|      c1|
+--------+
|[b1, b2]|
+--------+
```
In Spark 2.4.5 or Spark 3.0.0-preview2, if nested columns are specified, they are ignored.
```
df.na.drop("any", Seq("c1.c1-1")).show
+--------+
|      c1|
+--------+
|  [, a2]|
|[b1, b2]|
|    null|
+--------+
```
### Why are the changes needed?

This seems like a regression.

### Does this PR introduce any user-facing change?

Now, the nested column can be specified:
```
df.na.drop("any", Seq("c1.c1-1")).show
+--------+
|      c1|
+--------+
|[b1, b2]|
+--------+
```

Also, if `*` is specified as a column, it will throw an `AnalysisException` that `*` cannot be resolved, which was the behavior in 2.4.4. Currently, in master, it has no effect.

### How was this patch tested?

Updated existing tests.

Closes #28266 from imback82/SPARK-31256.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d7499aed9cc943304e2ec89379d3651410f6ca90)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/DataFrameNaFunctions.scala      | 12 ++---
 .../spark/sql/DataFrameNaFunctionsSuite.scala | 48 ++++++++++++-------
 2 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
index 953db806258a6..bbf0ac1dd85e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameNaFunctions.scala
@@ -89,7 +89,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    * @since 1.3.1
    */
   def drop(how: String, cols: Seq[String]): DataFrame = {
-    drop0(how, toAttributes(cols))
+    drop0(how, cols.map(df.resolve(_)))
   }
 
   /**
@@ -115,7 +115,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
    * @since 1.3.1
    */
   def drop(minNonNulls: Int, cols: Seq[String]): DataFrame = {
-    drop0(minNonNulls, toAttributes(cols))
+    drop0(minNonNulls, cols.map(df.resolve(_)))
   }
 
   /**
@@ -480,7 +480,7 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
     df.queryExecution.analyzed.output
   }
 
-  private def drop0(how: String, cols: Seq[Attribute]): DataFrame = {
+  private def drop0(how: String, cols: Seq[NamedExpression]): DataFrame = {
     how.toLowerCase(Locale.ROOT) match {
       case "any" => drop0(cols.size, cols)
       case "all" => drop0(1, cols)
@@ -488,12 +488,10 @@ final class DataFrameNaFunctions private[sql](df: DataFrame) {
     }
   }
 
-  private def drop0(minNonNulls: Int, cols: Seq[Attribute]): DataFrame = {
+  private def drop0(minNonNulls: Int, cols: Seq[NamedExpression]): DataFrame = {
     // Filtering condition:
     // only keep the row if it has at least `minNonNulls` non-null and non-NaN values.
-    val predicate = AtLeastNNonNulls(
-      minNonNulls,
-      outputAttributes.filter{ col => cols.exists(_.semanticEquals(col)) })
+    val predicate = AtLeastNNonNulls(minNonNulls, cols)
     df.filter(Column(predicate))
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
index fb1ca69b6f73f..091877f7cac37 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameNaFunctionsSuite.scala
@@ -45,6 +45,16 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {
     ).toDF("int", "long", "short", "byte", "float", "double")
   }
 
+  def createDFWithNestedColumns: DataFrame = {
+    val schema = new StructType()
+      .add("c1", new StructType()
+        .add("c1-1", StringType)
+        .add("c1-2", StringType))
+    val data = Seq(Row(Row(null, "a2")), Row(Row("b1", "b2")), Row(null))
+    spark.createDataFrame(
+      spark.sparkContext.parallelize(data), schema)
+  }
+
   test("drop") {
     val input = createDF()
     val rows = input.collect()
@@ -275,33 +285,35 @@ class DataFrameNaFunctionsSuite extends QueryTest with SharedSparkSession {
     assert(message.contains("Reference 'f2' is ambiguous"))
   }
 
-  test("fill/drop with col(*)") {
+  test("fill with col(*)") {
     val df = createDF()
     // If columns are specified with "*", they are ignored.
     checkAnswer(df.na.fill("new name", Seq("*")), df.collect())
-    checkAnswer(df.na.drop("any", Seq("*")), df.collect())
   }
 
-  test("fill/drop with nested columns") {
-    val schema = new StructType()
-      .add("c1", new StructType()
-        .add("c1-1", StringType)
-        .add("c1-2", StringType))
+  test("drop with col(*)") {
+    val df = createDF()
+    val exception = intercept[AnalysisException] {
+      df.na.drop("any", Seq("*"))
+    }
+    assert(exception.getMessage.contains("Cannot resolve column name \"*\""))
+  }
 
-    val data = Seq(
-      Row(Row(null, "a2")),
-      Row(Row("b1", "b2")),
-      Row(null))
+  test("fill with nested columns") {
+    val df = createDFWithNestedColumns
 
-    val df = spark.createDataFrame(
-      spark.sparkContext.parallelize(data), schema)
+    // Nested columns are ignored for fill().
+    checkAnswer(df.na.fill("a1", Seq("c1.c1-1")), df)
+  }
 
-    checkAnswer(df.select("c1.c1-1"),
-      Row(null) :: Row("b1") :: Row(null) :: Nil)
+  test("drop with nested columns") {
+    val df = createDFWithNestedColumns
 
-    // Nested columns are ignored for fill() and drop().
-    checkAnswer(df.na.fill("a1", Seq("c1.c1-1")), data)
-    checkAnswer(df.na.drop("any", Seq("c1.c1-1")), data)
+    // Rows with the specified nested columns whose null values are dropped.
+    assert(df.count == 3)
+    checkAnswer(
+      df.na.drop("any", Seq("c1.c1-1")),
+      Seq(Row(Row("b1", "b2"))))
   }
 
   test("replace") {

From f367838bb5086a200926158d16bd969358d588ef Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 20 Apr 2020 06:35:16 +0000
Subject: [PATCH 0522/1280] [SPARK-31385][SQL] Restrict micros rebasing via
 switch arrays up to 2037 year

### What changes were proposed in this pull request?
1. Generate rebasing arrays for micros up to 2037 in `RebaseDateTimeSuite.generateRebaseJson()`.
2. Exclude 4 time zones from the black list in `generateRebaseJson()`.
3. Re-generate JSON files with rebasing info - `gregorian-julian-rebase-micros.json` and `julian-gregorian-rebase-micros.json`.

### Why are the changes needed?
1. `sun.util.calendar.ZoneInfo` resolves DST after 2037 year incorrectly. See https://github.com/AdoptOpenJDK/openjdk-jdk8u/blob/aa318070b27849f1fe00d14684b2a40f7b29bf79/jdk/src/share/classes/sun/util/calendar/ZoneInfo.java#L55-L62 . By restricting the rebase arrays to 2037 year, we follow the behaviour of `ZoneInfo` which uses DST of 2037 for all years beyond 2037.
2. To enable optimization of micros rebasing via switch arrays for the time zones:
    - Asia/Tehran
    - Iran
    - Africa/Casablanca
    - Africa/El_Aaiun

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By existing test suites `RebaseDateTimeUtils`, `DateTimeUtilsSuite` and `DateFunctionsSuite`.

Closes #28253 from MaxGekk/fix-4-time-zones-rebasing.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 88d39e5a89cb76286ed8201c4d8b94d8a6199a42)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../gregorian-julian-rebase-micros.json          | 16 ++++++++++++++++
 .../julian-gregorian-rebase-micros.json          | 16 ++++++++++++++++
 .../sql/catalyst/util/RebaseDateTimeSuite.scala  | 12 +++---------
 3 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
index 8d36fbd377c1e..e5f9b0cb7a6f6 100644
--- a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
+++ b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json
@@ -54,6 +54,10 @@
   "tz" : "Africa/Cairo",
   "switches" : [ -62135604309, -59006369109, -55850695509, -52695021909, -46383588309, -43227914709, -40072241109, -33760807509, -30605133909, -27449460309, -21138026709, -17982353109, -14826679509, -12220077909, -12219991509, -12219905109, -12219818709, -12219732309, -12219645909, -12219559509, -12219473109, -12219386709, -12219300309, -2208988800 ],
   "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ]
+}, {
+  "tz" : "Africa/Casablanca",
+  "switches" : [ -62135594980, -59006359780, -55850686180, -52695012580, -46383578980, -43227905380, -40072231780, -33760798180, -30605124580, -27449450980, -21138017380, -17982343780, -14826670180, -12220068580, -12219982180, -12219895780, -12219809380, -12219722980, -12219636580, -12219550180, -12219463780, -12219377380, -12219290980, -2208988800 ],
+  "diffs" : [ -174620, -88220, -1820, 84580, 170980, 257380, 343780, 430180, 516580, 602980, 689380, 775780, 862180, 775780, 689380, 602980, 516580, 430180, 343780, 257380, 170980, 84580, -1820, 0 ]
 }, {
   "tz" : "Africa/Ceuta",
   "switches" : [ -62135595524, -59006360324, -55850686724, -52695013124, -46383579524, -43227905924, -40072232324, -33760798724, -30605125124, -27449451524, -21138017924, -17982344324, -14826670724, -12220069124, -12219982724, -12219896324, -12219809924, -12219723524, -12219637124, -12219550724, -12219464324, -12219377924, -12219291524, -2208988800 ],
@@ -78,6 +82,10 @@
   "tz" : "Africa/Douala",
   "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ],
   "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ]
+}, {
+  "tz" : "Africa/El_Aaiun",
+  "switches" : [ -62135593632, -59006358432, -55850684832, -52695011232, -46383577632, -43227904032, -40072230432, -33760796832, -30605123232, -27449449632, -21138016032, -17982342432, -14826668832, -12220067232, -12219980832, -12219894432, -12219808032, -12219721632, -12219635232, -12219548832, -12219462432, -12219376032, -12219289632, -2208988800 ],
+  "diffs" : [ -175968, -89568, -3168, 83232, 169632, 256032, 342432, 428832, 515232, 601632, 688032, 774432, 860832, 774432, 688032, 601632, 515232, 428832, 342432, 256032, 169632, 83232, -3168, 0 ]
 }, {
   "tz" : "Africa/Freetown",
   "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ],
@@ -1250,6 +1258,10 @@
   "tz" : "Asia/Tbilisi",
   "switches" : [ -62135607551, -59006372351, -55850698751, -52695025151, -46383591551, -43227917951, -40072244351, -33760810751, -30605137151, -27449463551, -21138029951, -17982356351, -14826682751, -12220081151, -12219994751, -12219908351, -12219821951, -12219735551, -12219649151, -12219562751, -12219476351, -12219389951, -12219303551, -2208988800 ],
   "diffs" : [ -176449, -90049, -3649, 82751, 169151, 255551, 341951, 428351, 514751, 601151, 687551, 773951, 860351, 773951, 687551, 601151, 514751, 428351, 341951, 255551, 169151, 82751, -3649, 0 ]
+}, {
+  "tz" : "Asia/Tehran",
+  "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ]
 }, {
   "tz" : "Asia/Tel_Aviv",
   "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
@@ -2002,6 +2014,10 @@
   "tz" : "Indian/Reunion",
   "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12220083712, -12219997312, -12219910912, -12219824512, -12219738112, -12219651712, -12219565312, -12219478912, -12219392512, -12219306112, -2208988800 ],
   "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, 0 ]
+}, {
+  "tz" : "Iran",
+  "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ],
+  "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ]
 }, {
   "tz" : "Israel",
   "switches" : [ -62135605254, -59006370054, -55850696454, -52695022854, -46383589254, -43227915654, -40072242054, -33760808454, -30605134854, -27449461254, -21138027654, -17982354054, -14826680454, -12220078854, -12219992454, -12219906054, -12219819654, -12219733254, -12219646854, -12219560454, -12219474054, -12219387654, -12219301254, -2840149254, -2208988800 ],
diff --git a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
index b733ac43cbf1f..eddc16f9ae767 100644
--- a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
+++ b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json
@@ -54,6 +54,10 @@
   "tz" : "Africa/Cairo",
   "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ],
   "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ]
+}, {
+  "tz" : "Africa/Casablanca",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 174620, 88220, 1820, -84580, -170980, -257380, -343780, -430180, -516580, -602980, -689380, -775780, -862180, 1820, 0 ]
 }, {
   "tz" : "Africa/Ceuta",
   "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
@@ -78,6 +82,10 @@
   "tz" : "Africa/Douala",
   "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ],
   "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ]
+}, {
+  "tz" : "Africa/El_Aaiun",
+  "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
+  "diffs" : [ 175968, 89568, 3168, -83232, -169632, -256032, -342432, -428832, -515232, -601632, -688032, -774432, -860832, 3168, 0 ]
 }, {
   "tz" : "Africa/Freetown",
   "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ],
@@ -1250,6 +1258,10 @@
   "tz" : "Asia/Tbilisi",
   "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
   "diffs" : [ 176449, 90049, 3649, -82751, -169151, -255551, -341951, -428351, -514751, -601151, -687551, -773951, -860351, 3649, 0 ]
+}, {
+  "tz" : "Asia/Tehran",
+  "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ],
+  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ]
 }, {
   "tz" : "Asia/Tel_Aviv",
   "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
@@ -2002,6 +2014,10 @@
   "tz" : "Indian/Reunion",
   "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ],
   "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 0 ]
+}, {
+  "tz" : "Iran",
+  "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ],
+  "diffs" : [ 173056, 86656, 256, -86144, -172544, -258944, -345344, -431744, -518144, -604544, -690944, -777344, -863744, 256, 0 ]
 }, {
   "tz" : "Israel",
   "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2840148014, -2208988800 ],
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index 2f957ef3cdfcb..0111fa0fd8bbd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -256,22 +256,16 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
     case class RebaseRecord(tz: String, switches: Array[Long], diffs: Array[Long])
 
     val result = new ArrayBuffer[RebaseRecord]()
-    // The time zones are excluded because:
-    // 1. Julian to Gregorian rebasing doesn't match to the opposite rebasing from
-    //    Gregorian to Julian rebasing.
-    // 2. Linear searching over switch points might be slow.
-    // 3. Results after the end time point 2100-01-01 are wrong.
-    // See SPARK-31385
-    val blacklist = Set("Asia/Tehran", "Iran", "Africa/Casablanca", "Africa/El_Aaiun")
     ALL_TIMEZONES
-      .filterNot(zid => blacklist.contains(zid.getId))
       .sortBy(_.getId)
       .foreach { zid =>
       withDefaultTimeZone(zid) {
         val start = adjustFunc(instantToMicros(LocalDateTime.of(1, 1, 1, 0, 0, 0)
           .atZone(zid)
           .toInstant))
-        val end = adjustFunc(instantToMicros(LocalDateTime.of(2100, 1, 1, 0, 0, 0)
+        // sun.util.calendar.ZoneInfo resolves DST after 2037 year incorrectly.
+        // See https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8073446
+        val end = adjustFunc(instantToMicros(LocalDateTime.of(2037, 1, 1, 0, 0, 0)
           .atZone(zid)
           .toInstant))
 

From 09b25ca15ebf56ccb716ffde2ecf721c2559407f Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 20 Apr 2020 07:26:38 +0000
Subject: [PATCH 0523/1280] [SPARK-31490][SQL][TESTS] Benchmark conversions
 to/from Java 8 datetime types

### What changes were proposed in this pull request?
- Add benchmark cases for **parallelizing** `java.time.LocalDate` and `java.time.Instant` column values.
- Add benchmark cases for **collecting** `java.time.LocalDate` and `java.time.Instant` column values.

### Why are the changes needed?
- To detect perf regression in the future
- To compare parallelization/collection of Java 8 date-time types with Java 7 date-time types `java.sql.Date` & `java.sql.Timestamp`.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running the modified benchmarks in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

Closes #28263 from MaxGekk/java8-datetime-collect-benchmark.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f1fde0cc2272d5c04cf25960be1ebd46001c5354)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeBenchmark-jdk11-results.txt       | 226 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 226 +++++++++---------
 .../benchmark/DateTimeBenchmark.scala         |  44 ++--
 3 files changed, 259 insertions(+), 237 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 0dfd66e4e98d2..a08fe0031166b 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    453            469          22         22.1          45.3       1.0X
-cast to timestamp wholestage on                     378            416          44         26.5          37.8       1.2X
+cast to timestamp wholestage off                    434            445          17         23.1          43.4       1.0X
+cast to timestamp wholestage on                     388            429          57         25.8          38.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1381           1388           9          7.2         138.1       1.0X
-year of timestamp wholestage on                    1218           1229          14          8.2         121.8       1.1X
+year of timestamp wholestage off                   1363           1374          16          7.3         136.3       1.0X
+year of timestamp wholestage on                    1285           1295           9          7.8         128.5       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1621           1627           9          6.2         162.1       1.0X
-quarter of timestamp wholestage on                 1593           1596           2          6.3         159.3       1.0X
+quarter of timestamp wholestage off                1619           1623           6          6.2         161.9       1.0X
+quarter of timestamp wholestage on                 1590           1606          16          6.3         159.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1241           1243           2          8.1         124.1       1.0X
-month of timestamp wholestage on                   1261           1277          10          7.9         126.1       1.0X
+month of timestamp wholestage off                  1264           1269           8          7.9         126.4       1.0X
+month of timestamp wholestage on                   1277           1290          13          7.8         127.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             2206           2207           1          4.5         220.6       1.0X
-weekofyear of timestamp wholestage on              1848           1863          13          5.4         184.8       1.2X
+weekofyear of timestamp wholestage off             1854           1865          16          5.4         185.4       1.0X
+weekofyear of timestamp wholestage on              1837           1855          24          5.4         183.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1239           1242           4          8.1         123.9       1.0X
-day of timestamp wholestage on                     1258           1274          11          8.0         125.8       1.0X
+day of timestamp wholestage off                    1248           1252           6          8.0         124.8       1.0X
+day of timestamp wholestage on                     1283           1288           7          7.8         128.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1266           1269           3          7.9         126.6       1.0X
-dayofyear of timestamp wholestage on               1294           1305          13          7.7         129.4       1.0X
+dayofyear of timestamp wholestage off              1288           1294           9          7.8         128.8       1.0X
+dayofyear of timestamp wholestage on               1312           1316           5          7.6         131.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1244           1245           2          8.0         124.4       1.0X
-dayofmonth of timestamp wholestage on              1253           1265           6          8.0         125.3       1.0X
+dayofmonth of timestamp wholestage off             1254           1263          13          8.0         125.4       1.0X
+dayofmonth of timestamp wholestage on              1274           1293          17          7.8         127.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1431           1436           7          7.0         143.1       1.0X
-dayofweek of timestamp wholestage on               1412           1424          12          7.1         141.2       1.0X
+dayofweek of timestamp wholestage off              1419           1422           5          7.0         141.9       1.0X
+dayofweek of timestamp wholestage on               1416           1425           8          7.1         141.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1347           1347           0          7.4         134.7       1.0X
-weekday of timestamp wholestage on                 1347           1358          12          7.4         134.7       1.0X
+weekday of timestamp wholestage off                1386           1386           0          7.2         138.6       1.0X
+weekday of timestamp wholestage on                 1348           1354           5          7.4         134.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1023           1024           1          9.8         102.3       1.0X
-hour of timestamp wholestage on                     980            988           8         10.2          98.0       1.0X
+hour of timestamp wholestage off                   1017           1020           4          9.8         101.7       1.0X
+hour of timestamp wholestage on                     996           1006           9         10.0          99.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1030           1034           6          9.7         103.0       1.0X
-minute of timestamp wholestage on                   986            992           6         10.1          98.6       1.0X
+minute of timestamp wholestage off                  994            996           2         10.1          99.4       1.0X
+minute of timestamp wholestage on                   974            991          15         10.3          97.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1031           1032           1          9.7         103.1       1.0X
-second of timestamp wholestage on                   985            991          11         10.2          98.5       1.0X
+second of timestamp wholestage off                 1018           1022           6          9.8         101.8       1.0X
+second of timestamp wholestage on                   976            987          13         10.2          97.6       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         308            311           3         32.5          30.8       1.0X
-current_date wholestage on                          309            314           6         32.4          30.9       1.0X
+current_date wholestage off                         297            299           2         33.7          29.7       1.0X
+current_date wholestage on                          315            322           7         31.8          31.5       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    312            332          28         32.0          31.2       1.0X
-current_timestamp wholestage on                     312            328          13         32.1          31.2       1.0X
+current_timestamp wholestage off                    293            299           9         34.2          29.3       1.0X
+current_timestamp wholestage on                     307            332          23         32.6          30.7       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1067           1078          17          9.4         106.7       1.0X
-cast to date wholestage on                         1075           1084           8          9.3         107.5       1.0X
+cast to date wholestage off                        1083           1087           7          9.2         108.3       1.0X
+cast to date wholestage on                         1057           1078          18          9.5         105.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1246           1249           3          8.0         124.6       1.0X
-last_day wholestage on                             1269           1272           3          7.9         126.9       1.0X
+last_day wholestage off                            1249           1262          18          8.0         124.9       1.0X
+last_day wholestage on                             1288           1297          10          7.8         128.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1116           1124          11          9.0         111.6       1.0X
-next_day wholestage on                             1120           1129           7          8.9         112.0       1.0X
+next_day wholestage off                            1128           1133           7          8.9         112.8       1.0X
+next_day wholestage on                             1106           1115           9          9.0         110.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1054           1064          15          9.5         105.4       1.0X
-date_add wholestage on                             1071           1082          17          9.3         107.1       1.0X
+date_add wholestage off                            1062           1084          31          9.4         106.2       1.0X
+date_add wholestage on                             1079           1093          24          9.3         107.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1066           1069           4          9.4         106.6       1.0X
-date_sub wholestage on                             1075           1084          14          9.3         107.5       1.0X
+date_sub wholestage off                            1077           1081           5          9.3         107.7       1.0X
+date_sub wholestage on                             1086           1105          16          9.2         108.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1411           1417           9          7.1         141.1       1.0X
-add_months wholestage on                           1472           1476           5          6.8         147.2       1.0X
+add_months wholestage off                          1437           1440           4          7.0         143.7       1.0X
+add_months wholestage on                           1452           1459           7          6.9         145.2       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5778           5801          33          1.7         577.8       1.0X
-format date wholestage on                          5718           5726          11          1.7         571.8       1.0X
+format date wholestage off                         5638           5653          21          1.8         563.8       1.0X
+format date wholestage on                          5464           5482          17          1.8         546.4       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7739           7744           8          1.3         773.9       1.0X
-from_unixtime wholestage on                        7742           7758          10          1.3         774.2       1.0X
+from_unixtime wholestage off                       7394           7395           1          1.4         739.4       1.0X
+from_unixtime wholestage on                        7306           7332          27          1.4         730.6       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1236           1243          11          8.1         123.6       1.0X
-from_utc_timestamp wholestage on                   1327           1333           8          7.5         132.7       0.9X
+from_utc_timestamp wholestage off                  1265           1268           4          7.9         126.5       1.0X
+from_utc_timestamp wholestage on                   1288           1300          11          7.8         128.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1787           1795          11          5.6         178.7       1.0X
-to_utc_timestamp wholestage on                     1791           1794           2          5.6         179.1       1.0X
+to_utc_timestamp wholestage off                    1760           1766           8          5.7         176.0       1.0X
+to_utc_timestamp wholestage on                     1732           1743          13          5.8         173.2       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        371            372           2         27.0          37.1       1.0X
-cast interval wholestage on                         340            348          10         29.4          34.0       1.1X
+cast interval wholestage off                        359            360           1         27.8          35.9       1.0X
+cast interval wholestage on                         327            337           9         30.6          32.7       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1844           1859          22          5.4         184.4       1.0X
-datediff wholestage on                             1813           1822          10          5.5         181.3       1.0X
+datediff wholestage off                            1849           1853           5          5.4         184.9       1.0X
+datediff wholestage on                             1819           1826           6          5.5         181.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5391           5395           5          1.9         539.1       1.0X
-months_between wholestage on                       5353           5374          12          1.9         535.3       1.0X
+months_between wholestage off                      6061           6066           7          1.6         606.1       1.0X
+months_between wholestage on                       6022           6034          12          1.7         602.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2601           2709         152          0.4        2601.3       1.0X
-window wholestage on                              45767          45831          46          0.0       45767.4       0.1X
+window wholestage off                              2249           2298          69          0.4        2249.4       1.0X
+window wholestage on                              47857          47942         130          0.0       47857.1       0.0X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2494           2506          17          4.0         249.4       1.0X
-date_trunc YEAR wholestage on                      2482           2499          12          4.0         248.2       1.0X
+date_trunc YEAR wholestage off                     2530           2535           7          4.0         253.0       1.0X
+date_trunc YEAR wholestage on                      2490           2506          18          4.0         249.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2485           2504          26          4.0         248.5       1.0X
-date_trunc YYYY wholestage on                      2477           2489           9          4.0         247.7       1.0X
+date_trunc YYYY wholestage off                     2525           2530           6          4.0         252.5       1.0X
+date_trunc YYYY wholestage on                      2484           2492           7          4.0         248.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2492           2493           1          4.0         249.2       1.0X
-date_trunc YY wholestage on                        2468           2479           7          4.1         246.8       1.0X
+date_trunc YY wholestage off                       2530           2530           0          4.0         253.0       1.0X
+date_trunc YY wholestage on                        2485           2494          11          4.0         248.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2510           2523          19          4.0         251.0       1.0X
-date_trunc MON wholestage on                       2497           2503           9          4.0         249.7       1.0X
+date_trunc MON wholestage off                      2507           2511           5          4.0         250.7       1.0X
+date_trunc MON wholestage on                       2501           2510          10          4.0         250.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2509           2526          25          4.0         250.9       1.0X
-date_trunc MONTH wholestage on                     2495           2503           7          4.0         249.5       1.0X
+date_trunc MONTH wholestage off                    2517           2521           5          4.0         251.7       1.0X
+date_trunc MONTH wholestage on                     2507           2517          15          4.0         250.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2513           2517           6          4.0         251.3       1.0X
-date_trunc MM wholestage on                        2494           2500           5          4.0         249.4       1.0X
+date_trunc MM wholestage off                       2502           2504           4          4.0         250.2       1.0X
+date_trunc MM wholestage on                        2504           2509           4          4.0         250.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2415           2429          19          4.1         241.5       1.0X
-date_trunc DAY wholestage on                       2369           2379           7          4.2         236.9       1.0X
+date_trunc DAY wholestage off                      2343           2344           3          4.3         234.3       1.0X
+date_trunc DAY wholestage on                       2308           2328          22          4.3         230.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2403           2406           4          4.2         240.3       1.0X
-date_trunc DD wholestage on                        2372           2379           4          4.2         237.2       1.0X
+date_trunc DD wholestage off                       2344           2349           7          4.3         234.4       1.0X
+date_trunc DD wholestage on                        2312           2326          16          4.3         231.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2394           2409          22          4.2         239.4       1.0X
-date_trunc HOUR wholestage on                      2344           2352           6          4.3         234.4       1.0X
+date_trunc HOUR wholestage off                     2330           2332           2          4.3         233.0       1.0X
+date_trunc HOUR wholestage on                      2269           2273           4          4.4         226.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    346            347           1         28.9          34.6       1.0X
-date_trunc MINUTE wholestage on                     365            370           5         27.4          36.5       0.9X
+date_trunc MINUTE wholestage off                    396            397           1         25.2          39.6       1.0X
+date_trunc MINUTE wholestage on                     371            379           5         27.0          37.1       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    377            392          21         26.5          37.7       1.0X
-date_trunc SECOND wholestage on                     361            363           2         27.7          36.1       1.0X
+date_trunc SECOND wholestage off                    364            376          18         27.5          36.4       1.0X
+date_trunc SECOND wholestage on                     373            376           2         26.8          37.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2428           2436          12          4.1         242.8       1.0X
-date_trunc WEEK wholestage on                      2405           2414           8          4.2         240.5       1.0X
+date_trunc WEEK wholestage off                     2433           2437           6          4.1         243.3       1.0X
+date_trunc WEEK wholestage on                      2402           2418          21          4.2         240.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3705           3709           5          2.7         370.5       1.0X
-date_trunc QUARTER wholestage on                   3673           3682           8          2.7         367.3       1.0X
+date_trunc QUARTER wholestage off                  3304           3310           9          3.0         330.4       1.0X
+date_trunc QUARTER wholestage on                   3283           3289           5          3.0         328.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           314            316           2         31.9          31.4       1.0X
-trunc year wholestage on                            315            323           6         31.7          31.5       1.0X
+trunc year wholestage off                           323            324           1         30.9          32.3       1.0X
+trunc year wholestage on                            344            349          10         29.1          34.4       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           309            310           1         32.4          30.9       1.0X
-trunc yyyy wholestage on                            309            332          33         32.4          30.9       1.0X
+trunc yyyy wholestage off                           333            337           6         30.0          33.3       1.0X
+trunc yyyy wholestage on                            345            351           7         29.0          34.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             310            311           1         32.3          31.0       1.0X
-trunc yy wholestage on                              307            315           6         32.5          30.7       1.0X
+trunc yy wholestage off                             315            317           3         31.7          31.5       1.0X
+trunc yy wholestage on                              340            345           6         29.4          34.0       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            308            312           7         32.5          30.8       1.0X
-trunc mon wholestage on                             309            315          11         32.4          30.9       1.0X
+trunc mon wholestage off                            314            314           1         31.9          31.4       1.0X
+trunc mon wholestage on                             342            343           1         29.3          34.2       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          307            311           6         32.6          30.7       1.0X
-trunc month wholestage on                           308            312           3         32.4          30.8       1.0X
+trunc month wholestage off                          311            316           8         32.2          31.1       1.0X
+trunc month wholestage on                           341            343           1         29.3          34.1       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             307            308           2         32.6          30.7       1.0X
-trunc mm wholestage on                              307            313           6         32.6          30.7       1.0X
+trunc mm wholestage off                             309            311           3         32.4          30.9       1.0X
+trunc mm wholestage on                              342            347           4         29.3          34.2       0.9X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     173            174           1          5.8         173.1       1.0X
-to timestamp str wholestage on                      167            169           1          6.0         167.5       1.0X
+to timestamp str wholestage off                     175            179           6          5.7         175.3       1.0X
+to timestamp str wholestage on                      170            177          13          5.9         169.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1460           1461           2          0.7        1459.7       1.0X
-to_timestamp wholestage on                         1464           1472          13          0.7        1463.8       1.0X
+to_timestamp wholestage off                        1563           1577          20          0.6        1563.1       1.0X
+to_timestamp wholestage on                         1587           1591           3          0.6        1587.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1499           1500           3          0.7        1498.6       1.0X
-to_unix_timestamp wholestage on                    1448           1452           7          0.7        1447.8       1.0X
+to_unix_timestamp wholestage off                   1646           1647           2          0.6        1646.1       1.0X
+to_unix_timestamp wholestage on                    1602           1609           8          0.6        1602.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          215            216           2          4.7         214.9       1.0X
-to date str wholestage on                           213            215           3          4.7         212.5       1.0X
+to date str wholestage off                          216            217           1          4.6         216.0       1.0X
+to date str wholestage on                           222            226           3          4.5         221.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3581           3596          22          0.3        3580.8       1.0X
-to_date wholestage on                              3629           3634           6          0.3        3629.0       1.0X
+to_date wholestage off                             3800           3806           8          0.3        3800.1       1.0X
+to_date wholestage on                              3796           3808          10          0.3        3795.5       1.0X
 
 
 ================================================================================================
@@ -422,10 +422,14 @@ OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  435            439           3         11.5          87.1       1.0X
-Collect dates                                      3409           3756         343          1.5         681.9       0.1X
-From java.sql.Timestamp                             350            351           1         14.3          70.1       1.2X
-Collect longs                                      1316           1433         124          3.8         263.2       0.3X
-Collect timestamps                                 1752           2013         231          2.9         350.4       0.2X
+From java.sql.Date                                  427            436          12         11.7          85.3       1.0X
+From java.time.LocalDate                            359            363           6         13.9          71.9       1.2X
+Collect java.sql.Date                              2131           2482         403          2.3         426.1       0.2X
+Collect java.time.LocalDate                        1457           1871         495          3.4         291.4       0.3X
+From java.sql.Timestamp                             369            371           1         13.5          73.8       1.2X
+From java.time.Instant                              325            330           6         15.4          65.0       1.3X
+Collect longs                                      1370           1388          20          3.6         274.1       0.3X
+Collect java.sql.Timestamp                         1701           1721          17          2.9         340.2       0.3X
+Collect java.time.Instant                          1817           1959         159          2.8         363.4       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index d4259a9e6c9b8..045daaaad80a1 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,92 +6,92 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    403            433          42         24.8          40.3       1.0X
-cast to timestamp wholestage on                     377            386          14         26.6          37.7       1.1X
+cast to timestamp wholestage off                    422            452          43         23.7          42.2       1.0X
+cast to timestamp wholestage on                     366            382          16         27.3          36.6       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1327           1328           1          7.5         132.7       1.0X
-year of timestamp wholestage on                    1341           1368          23          7.5         134.1       1.0X
+year of timestamp wholestage off                   1298           1324          37          7.7         129.8       1.0X
+year of timestamp wholestage on                    1201           1214          13          8.3         120.1       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1460           1468          11          6.9         146.0       1.0X
-quarter of timestamp wholestage on                 1396           1418          14          7.2         139.6       1.0X
+quarter of timestamp wholestage off                1443           1444           1          6.9         144.3       1.0X
+quarter of timestamp wholestage on                 1401           1415           8          7.1         140.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1232           1239          11          8.1         123.2       1.0X
-month of timestamp wholestage on                   1252           1267          14          8.0         125.2       1.0X
+month of timestamp wholestage off                  1230           1240          13          8.1         123.0       1.0X
+month of timestamp wholestage on                   1252           1262           8          8.0         125.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1852           1853           0          5.4         185.2       1.0X
-weekofyear of timestamp wholestage on              1865           1881          19          5.4         186.5       1.0X
+weekofyear of timestamp wholestage off             2122           2123           2          4.7         212.2       1.0X
+weekofyear of timestamp wholestage on              2398           2413          13          4.2         239.8       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1224           1226           2          8.2         122.4       1.0X
-day of timestamp wholestage on                     1236           1250           9          8.1         123.6       1.0X
+day of timestamp wholestage off                    1225           1226           2          8.2         122.5       1.0X
+day of timestamp wholestage on                     1249           1259           8          8.0         124.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1276           1276           1          7.8         127.6       1.0X
-dayofyear of timestamp wholestage on               1287           1294           8          7.8         128.7       1.0X
+dayofyear of timestamp wholestage off              1274           1279           6          7.8         127.4       1.0X
+dayofyear of timestamp wholestage on               1278           1294          25          7.8         127.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1225           1227           3          8.2         122.5       1.0X
-dayofmonth of timestamp wholestage on              1234           1243          11          8.1         123.4       1.0X
+dayofmonth of timestamp wholestage off             1221           1222           0          8.2         122.1       1.0X
+dayofmonth of timestamp wholestage on              1233           1243           8          8.1         123.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1398           1407          13          7.2         139.8       1.0X
-dayofweek of timestamp wholestage on               1394           1401           6          7.2         139.4       1.0X
+dayofweek of timestamp wholestage off              1411           1422          16          7.1         141.1       1.0X
+dayofweek of timestamp wholestage on               1393           1398           5          7.2         139.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1339           1345           9          7.5         133.9       1.0X
-weekday of timestamp wholestage on                 1333           1340           9          7.5         133.3       1.0X
+weekday of timestamp wholestage off                1351           1352           1          7.4         135.1       1.0X
+weekday of timestamp wholestage on                 1335           1342           5          7.5         133.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    992            996           5         10.1          99.2       1.0X
-hour of timestamp wholestage on                     973            981           7         10.3          97.3       1.0X
+hour of timestamp wholestage off                    998            999           1         10.0          99.8       1.0X
+hour of timestamp wholestage on                     975            989          15         10.3          97.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  988            995          10         10.1          98.8       1.0X
-minute of timestamp wholestage on                   980            990           7         10.2          98.0       1.0X
+minute of timestamp wholestage off                 1005           1012          10         10.0         100.5       1.0X
+minute of timestamp wholestage on                   978           1003          35         10.2          97.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1008           1013           8          9.9         100.8       1.0X
-second of timestamp wholestage on                   976            987          13         10.2          97.6       1.0X
+second of timestamp wholestage off                  963            965           4         10.4          96.3       1.0X
+second of timestamp wholestage on                   977            988          16         10.2          97.7       1.0X
 
 
 ================================================================================================
@@ -102,15 +102,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         287            287           0         34.8          28.7       1.0X
-current_date wholestage on                          301            302           1         33.2          30.1       1.0X
+current_date wholestage off                         289            295           8         34.6          28.9       1.0X
+current_date wholestage on                          304            314           9         32.9          30.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    292            295           5         34.3          29.2       1.0X
-current_timestamp wholestage on                     306            315          10         32.7          30.6       1.0X
+current_timestamp wholestage off                    296            300           7         33.8          29.6       1.0X
+current_timestamp wholestage on                     302            311          11         33.1          30.2       1.0X
 
 
 ================================================================================================
@@ -121,43 +121,43 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1062           1071          13          9.4         106.2       1.0X
-cast to date wholestage on                         1030           1039          13          9.7         103.0       1.0X
+cast to date wholestage off                        1071           1078          10          9.3         107.1       1.0X
+cast to date wholestage on                         1018           1027           7          9.8         101.8       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1282           1303          29          7.8         128.2       1.0X
-last_day wholestage on                             1257           1262           4          8.0         125.7       1.0X
+last_day wholestage off                            1261           1264           5          7.9         126.1       1.0X
+last_day wholestage on                             1259           1271           9          7.9         125.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1114           1120           9          9.0         111.4       1.0X
-next_day wholestage on                             1075           1085           8          9.3         107.5       1.0X
+next_day wholestage off                            1119           1147          41          8.9         111.9       1.0X
+next_day wholestage on                             1076           1082           8          9.3         107.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1056           1061           7          9.5         105.6       1.0X
-date_add wholestage on                             1057           1063           6          9.5         105.7       1.0X
+date_add wholestage off                            1062           1070          12          9.4         106.2       1.0X
+date_add wholestage on                             1058           1060           2          9.5         105.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1069           1080          15          9.4         106.9       1.0X
-date_sub wholestage on                             1056           1064          15          9.5         105.6       1.0X
+date_sub wholestage off                            1048           1080          45          9.5         104.8       1.0X
+date_sub wholestage on                             1060           1066           8          9.4         106.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1367           1367           1          7.3         136.7       1.0X
-add_months wholestage on                           1432           1438           6          7.0         143.2       1.0X
+add_months wholestage off                          1368           1371           4          7.3         136.8       1.0X
+add_months wholestage on                           1423           1429           7          7.0         142.3       1.0X
 
 
 ================================================================================================
@@ -168,8 +168,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5900           5918          25          1.7         590.0       1.0X
-format date wholestage on                          6364           6393          34          1.6         636.4       0.9X
+format date wholestage off                         5865           5898          47          1.7         586.5       1.0X
+format date wholestage on                          5778           5788           8          1.7         577.8       1.0X
 
 
 ================================================================================================
@@ -180,8 +180,8 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8740           8753          18          1.1         874.0       1.0X
-from_unixtime wholestage on                        8689           8714          26          1.2         868.9       1.0X
+from_unixtime wholestage off                       8886           8899          18          1.1         888.6       1.0X
+from_unixtime wholestage on                        8831           8841           7          1.1         883.1       1.0X
 
 
 ================================================================================================
@@ -192,15 +192,15 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1086           1087           2          9.2         108.6       1.0X
-from_utc_timestamp wholestage on                   1168           1173           5          8.6         116.8       0.9X
+from_utc_timestamp wholestage off                  1093           1103          14          9.1         109.3       1.0X
+from_utc_timestamp wholestage on                   1136           1147          11          8.8         113.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1640           1642           3          6.1         164.0       1.0X
-to_utc_timestamp wholestage on                     1600           1612          20          6.2         160.0       1.0X
+to_utc_timestamp wholestage off                    1640           1646           8          6.1         164.0       1.0X
+to_utc_timestamp wholestage on                     1626           1641          12          6.1         162.6       1.0X
 
 
 ================================================================================================
@@ -211,29 +211,29 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        329            330           2         30.4          32.9       1.0X
-cast interval wholestage on                         321            333          17         31.1          32.1       1.0X
+cast interval wholestage off                        338            348          14         29.6          33.8       1.0X
+cast interval wholestage on                         318            323           4         31.4          31.8       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1819           1820           1          5.5         181.9       1.0X
-datediff wholestage on                             1783           1789           6          5.6         178.3       1.0X
+datediff wholestage off                            1826           1829           4          5.5         182.6       1.0X
+datediff wholestage on                             1785           1797          11          5.6         178.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      4791           4793           3          2.1         479.1       1.0X
-months_between wholestage on                       4739           4745           6          2.1         473.9       1.0X
+months_between wholestage off                      4795           4801           9          2.1         479.5       1.0X
+months_between wholestage on                       4752           4766          19          2.1         475.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1955           2108         217          0.5        1954.9       1.0X
-window wholestage on                              46222          46285          41          0.0       46222.1       0.0X
+window wholestage off                              2452           2590         195          0.4        2452.1       1.0X
+window wholestage on                              42821          42877          54          0.0       42820.7       0.1X
 
 
 ================================================================================================
@@ -244,134 +244,134 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2341           2343           3          4.3         234.1       1.0X
-date_trunc YEAR wholestage on                      2288           2298           8          4.4         228.8       1.0X
+date_trunc YEAR wholestage off                     2408           2413           7          4.2         240.8       1.0X
+date_trunc YEAR wholestage on                      2311           2318           6          4.3         231.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2337           2338           2          4.3         233.7       1.0X
-date_trunc YYYY wholestage on                      2286           2295           7          4.4         228.6       1.0X
+date_trunc YYYY wholestage off                     2404           2414          15          4.2         240.4       1.0X
+date_trunc YYYY wholestage on                      2317           2343          17          4.3         231.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2339           2340           2          4.3         233.9       1.0X
-date_trunc YY wholestage on                        2288           2292           3          4.4         228.8       1.0X
+date_trunc YY wholestage off                       2403           2415          17          4.2         240.3       1.0X
+date_trunc YY wholestage on                        2298           2316          13          4.4         229.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2308           2309           1          4.3         230.8       1.0X
-date_trunc MON wholestage on                       2348           2357          11          4.3         234.8       1.0X
+date_trunc MON wholestage off                      2370           2378          10          4.2         237.0       1.0X
+date_trunc MON wholestage on                       2329           2343          13          4.3         232.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2316           2318           3          4.3         231.6       1.0X
-date_trunc MONTH wholestage on                     2346           2356           9          4.3         234.6       1.0X
+date_trunc MONTH wholestage off                    2370           2376           7          4.2         237.0       1.0X
+date_trunc MONTH wholestage on                     2329           2347          15          4.3         232.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2313           2315           3          4.3         231.3       1.0X
-date_trunc MM wholestage on                        2348           2355           4          4.3         234.8       1.0X
+date_trunc MM wholestage off                       2363           2364           2          4.2         236.3       1.0X
+date_trunc MM wholestage on                        2323           2342          15          4.3         232.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2162           2164           3          4.6         216.2       1.0X
-date_trunc DAY wholestage on                       2177           2183           4          4.6         217.7       1.0X
+date_trunc DAY wholestage off                      2222           2226           7          4.5         222.2       1.0X
+date_trunc DAY wholestage on                       2162           2169           6          4.6         216.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2162           2169          10          4.6         216.2       1.0X
-date_trunc DD wholestage on                        2178           2181           2          4.6         217.8       1.0X
+date_trunc DD wholestage off                       2228           2232           5          4.5         222.8       1.0X
+date_trunc DD wholestage on                        2164           2171           6          4.6         216.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2230           2237           9          4.5         223.0       1.0X
-date_trunc HOUR wholestage on                      2242           2251           7          4.5         224.2       1.0X
+date_trunc HOUR wholestage off                     2236           2242           9          4.5         223.6       1.0X
+date_trunc HOUR wholestage on                      2212           2227          16          4.5         221.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    367            373           8         27.2          36.7       1.0X
-date_trunc MINUTE wholestage on                     343            347           3         29.1          34.3       1.1X
+date_trunc MINUTE wholestage off                    368            371           4         27.2          36.8       1.0X
+date_trunc MINUTE wholestage on                     331            335           4         30.2          33.1       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    369            369           1         27.1          36.9       1.0X
-date_trunc SECOND wholestage on                     343            346           2         29.2          34.3       1.1X
+date_trunc SECOND wholestage off                    383            383           0         26.1          38.3       1.0X
+date_trunc SECOND wholestage on                     328            335           8         30.5          32.8       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2238           2245          10          4.5         223.8       1.0X
-date_trunc WEEK wholestage on                      2222           2231          10          4.5         222.2       1.0X
+date_trunc WEEK wholestage off                     2259           2277          26          4.4         225.9       1.0X
+date_trunc WEEK wholestage on                      2201           2227          18          4.5         220.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3092           3093           2          3.2         309.2       1.0X
-date_trunc QUARTER wholestage on                   3074           3077           4          3.3         307.4       1.0X
+date_trunc QUARTER wholestage off                  3168           3183          22          3.2         316.8       1.0X
+date_trunc QUARTER wholestage on                   3130           3139          10          3.2         313.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           320            328          12         31.3          32.0       1.0X
-trunc year wholestage on                            344            365          18         29.1          34.4       0.9X
+trunc year wholestage off                           326            330           5         30.7          32.6       1.0X
+trunc year wholestage on                            295            298           3         33.8          29.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           316            318           3         31.6          31.6       1.0X
-trunc yyyy wholestage on                            352            372          21         28.4          35.2       0.9X
+trunc yyyy wholestage off                           322            323           0         31.0          32.2       1.0X
+trunc yyyy wholestage on                            295            297           2         33.9          29.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             313            314           1         31.9          31.3       1.0X
-trunc yy wholestage on                              346            364          27         28.9          34.6       0.9X
+trunc yy wholestage off                             324            325           2         30.9          32.4       1.0X
+trunc yy wholestage on                              295            300           4         33.9          29.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            317            318           1         31.5          31.7       1.0X
-trunc mon wholestage on                             347            367          17         28.8          34.7       0.9X
+trunc mon wholestage off                            323            336          19         31.0          32.3       1.0X
+trunc mon wholestage on                             294            297           3         34.0          29.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          320            321           1         31.2          32.0       1.0X
-trunc month wholestage on                           352            410          77         28.4          35.2       0.9X
+trunc month wholestage off                          322            322           0         31.1          32.2       1.0X
+trunc month wholestage on                           295            297           4         33.9          29.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             315            318           5         31.8          31.5       1.0X
-trunc mm wholestage on                              353            382          29         28.3          35.3       0.9X
+trunc mm wholestage off                             322            324           2         31.0          32.2       1.0X
+trunc mm wholestage on                              297            306           6         33.7          29.7       1.1X
 
 
 ================================================================================================
@@ -382,36 +382,36 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     217            221           5          4.6         217.3       1.0X
-to timestamp str wholestage on                      212            214           2          4.7         212.3       1.0X
+to timestamp str wholestage off                     214            215           1          4.7         214.1       1.0X
+to timestamp str wholestage on                      212            217           4          4.7         212.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1957           1964          11          0.5        1956.6       1.0X
-to_timestamp wholestage on                         1993           1998           9          0.5        1993.0       1.0X
+to_timestamp wholestage off                        1543           1545           3          0.6        1542.8       1.0X
+to_timestamp wholestage on                         1541           1550          12          0.6        1541.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   2071           2073           3          0.5        2071.0       1.0X
-to_unix_timestamp wholestage on                    2082           2092          11          0.5        2082.4       1.0X
+to_unix_timestamp wholestage off                   1590           1592           2          0.6        1590.4       1.0X
+to_unix_timestamp wholestage on                    1527           1531           4          0.7        1527.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          262            262           0          3.8         261.7       1.0X
-to date str wholestage on                           258            261           4          3.9         258.1       1.0X
+to date str wholestage off                          264            267           4          3.8         264.0       1.0X
+to date str wholestage on                           261            263           3          3.8         261.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3551           3567          22          0.3        3550.7       1.0X
-to_date wholestage on                              3505           3513           9          0.3        3505.3       1.0X
+to_date wholestage off                             3578           3590          16          0.3        3578.2       1.0X
+to_date wholestage on                              3495           3504           9          0.3        3495.4       1.0X
 
 
 ================================================================================================
@@ -422,10 +422,14 @@ OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  416            419           3         12.0          83.2       1.0X
-Collect dates                                      1928           2759        1180          2.6         385.6       0.2X
-From java.sql.Timestamp                             383            397          12         13.0          76.7       1.1X
-Collect longs                                      1324           1411          91          3.8         264.9       0.3X
-Collect timestamps                                 1906           2040         185          2.6         381.2       0.2X
+From java.sql.Date                                  414            424          12         12.1          82.9       1.0X
+From java.time.LocalDate                            350            355           4         14.3          69.9       1.2X
+Collect java.sql.Date                              2064           2709        1101          2.4         412.8       0.2X
+Collect java.time.LocalDate                        1467           1493          31          3.4         293.5       0.3X
+From java.sql.Timestamp                             396            401           4         12.6          79.3       1.0X
+From java.time.Instant                              260            266           7         19.2          52.0       1.6X
+Collect longs                                      1336           1475         178          3.7         267.2       0.3X
+Collect java.sql.Timestamp                         2175           2887        1226          2.3         435.0       0.2X
+Collect java.time.Instant                          1679           1917         228          3.0         335.7       0.2X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 0bb87b19b7a69..0620227687468 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -17,9 +17,11 @@
 
 package org.apache.spark.sql.execution.benchmark
 
-import java.sql.Timestamp
+import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate}
 
 import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
 import org.apache.spark.sql.internal.SQLConf
 
@@ -133,28 +135,40 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           val numIters = 3
           val benchmark = new Benchmark("To/from Java's date-time", rowsNum, output = output)
           benchmark.addCase("From java.sql.Date", numIters) { _ =>
-            spark.range(rowsNum)
-              .map(millis => new java.sql.Date(millis))
-              .noop()
+            spark.range(rowsNum).map(millis => new Date(millis)).noop()
           }
-          benchmark.addCase("Collect dates", numIters) { _ =>
-            spark.range(0, rowsNum, 1, 1)
-              .map(millis => new java.sql.Date(millis))
-              .collect()
+          benchmark.addCase("From java.time.LocalDate", numIters) { _ =>
+            spark.range(rowsNum).map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY)).noop()
+          }
+          benchmark.addCase("Collect java.sql.Date", numIters) { _ =>
+            spark.range(0, rowsNum, 1, 1).map(millis => new Date(millis)).collect()
+          }
+          benchmark.addCase("Collect java.time.LocalDate", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              spark.range(0, rowsNum, 1, 1)
+                .map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY))
+                .collect()
+            }
           }
           benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
-            spark.range(rowsNum)
-              .map(millis => new Timestamp(millis))
-              .noop()
+            spark.range(rowsNum).map(millis => new Timestamp(millis)).noop()
+          }
+          benchmark.addCase("From java.time.Instant", numIters) { _ =>
+            spark.range(rowsNum).map(millis => Instant.ofEpochMilli(millis)).noop()
           }
           benchmark.addCase("Collect longs", numIters) { _ =>
             spark.range(0, rowsNum, 1, 1)
               .collect()
           }
-          benchmark.addCase("Collect timestamps", numIters) { _ =>
-            spark.range(0, rowsNum, 1, 1)
-              .map(millis => new Timestamp(millis))
-              .collect()
+          benchmark.addCase("Collect java.sql.Timestamp", numIters) { _ =>
+            spark.range(0, rowsNum, 1, 1).map(millis => new Timestamp(millis)).collect()
+          }
+          benchmark.addCase("Collect java.time.Instant", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              spark.range(0, rowsNum, 1, 1)
+                .map(millis => Instant.ofEpochMilli(millis))
+                .collect()
+            }
           }
           benchmark.run()
         }

From 2e3216012e8ad85d4cd88671493dd6e4d0e6a668 Mon Sep 17 00:00:00 2001
From: Maryann Xue <maryann.xue@gmail.com>
Date: Mon, 20 Apr 2020 11:55:48 -0700
Subject: [PATCH 0524/1280] [SPARK-31475][SQL] Broadcast stage in AQE did not
 timeout

### What changes were proposed in this pull request?

This PR adds a timeout for the Future of a BroadcastQueryStageExec to make sure it can have the same timeout behavior as a non-AQE broadcast exchange.

### Why are the changes needed?

This is to make the broadcast timeout behavior in AQE consistent with that in non-AQE.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added UT.

Closes #28250 from maryannxue/aqe-broadcast-timeout.

Authored-by: Maryann Xue <maryann.xue@gmail.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 44d370dd4501f0a4abb7194f7cff0d346aac0992)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      |  2 +-
 .../execution/adaptive/QueryStageExec.scala   | 35 ++++++++++++++++---
 .../exchange/BroadcastExchangeExec.scala      |  8 ++---
 .../execution/joins/BroadcastJoinSuite.scala  | 23 ++++++++++--
 4 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 2b4672407a348..0ec8b5f889c58 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -546,7 +546,7 @@ case class AdaptiveSparkPlanExec(
 }
 
 object AdaptiveSparkPlanExec {
-  private val executionContext = ExecutionContext.fromExecutorService(
+  private[adaptive] val executionContext = ExecutionContext.fromExecutorService(
     ThreadUtils.newDaemonCachedThreadPool("QueryStageCreator", 16))
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
index beaa9721fb5e5..f414f854b92ae 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
@@ -17,9 +17,11 @@
 
 package org.apache.spark.sql.execution.adaptive
 
-import scala.concurrent.Future
+import java.util.concurrent.TimeUnit
 
-import org.apache.spark.{FutureAction, MapOutputStatistics}
+import scala.concurrent.{Future, Promise}
+
+import org.apache.spark.{FutureAction, MapOutputStatistics, SparkException}
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -28,6 +30,8 @@ import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.exchange._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.ThreadUtils
 
 /**
  * A query stage is an independent subgraph of the query plan. Query stage materializes its output
@@ -100,8 +104,8 @@ abstract class QueryStageExec extends LeafExecNode {
   override def executeTail(n: Int): Array[InternalRow] = plan.executeTail(n)
   override def executeToIterator(): Iterator[InternalRow] = plan.executeToIterator()
 
-  override def doPrepare(): Unit = plan.prepare()
-  override def doExecute(): RDD[InternalRow] = plan.execute()
+  protected override def doPrepare(): Unit = plan.prepare()
+  protected override def doExecute(): RDD[InternalRow] = plan.execute()
   override def doExecuteBroadcast[T](): Broadcast[T] = plan.executeBroadcast()
   override def doCanonicalize(): SparkPlan = plan.canonicalized
 
@@ -187,8 +191,24 @@ case class BroadcastQueryStageExec(
       throw new IllegalStateException("wrong plan for broadcast stage:\n " + plan.treeString)
   }
 
+  @transient private lazy val materializeWithTimeout = {
+    val broadcastFuture = broadcast.completionFuture
+    val timeout = SQLConf.get.broadcastTimeout
+    val promise = Promise[Any]()
+    val fail = BroadcastQueryStageExec.scheduledExecutor.schedule(new Runnable() {
+      override def run(): Unit = {
+        promise.tryFailure(new SparkException(s"Could not execute broadcast in $timeout secs. " +
+          s"You can increase the timeout for broadcasts via ${SQLConf.BROADCAST_TIMEOUT.key} or " +
+          s"disable broadcast join by setting ${SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key} to -1"))
+      }
+    }, timeout, TimeUnit.SECONDS)
+    broadcastFuture.onComplete(_ => fail.cancel(false))(AdaptiveSparkPlanExec.executionContext)
+    Future.firstCompletedOf(
+      Seq(broadcastFuture, promise.future))(AdaptiveSparkPlanExec.executionContext)
+  }
+
   override def doMaterialize(): Future[Any] = {
-    broadcast.completionFuture
+    materializeWithTimeout
   }
 
   override def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec = {
@@ -204,3 +224,8 @@ case class BroadcastQueryStageExec(
     }
   }
 }
+
+object BroadcastQueryStageExec {
+  private val scheduledExecutor =
+    ThreadUtils.newDaemonSingleThreadScheduledExecutor("BroadcastStageTimeout")
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
index 65e6b7c2f0fba..ba5261c9bb5f8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
@@ -118,7 +118,7 @@ case class BroadcastExchangeExec(
               System.nanoTime() - beforeBroadcast)
             val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
             SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq)
-            promise.success(broadcasted)
+            promise.trySuccess(broadcasted)
             broadcasted
           } catch {
             // SPARK-24294: To bypass scala bug: https://github.com/scala/bug/issues/9554, we throw
@@ -131,14 +131,14 @@ case class BroadcastExchangeExec(
                   s"${SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key} to -1 or increase the spark " +
                   s"driver memory by setting ${SparkLauncher.DRIVER_MEMORY} to a higher value.")
                   .initCause(oe.getCause))
-              promise.failure(ex)
+              promise.tryFailure(ex)
               throw ex
             case e if !NonFatal(e) =>
               val ex = new SparkFatalException(e)
-              promise.failure(ex)
+              promise.tryFailure(ex)
               throw ex
             case e: Throwable =>
-              promise.failure(e)
+              promise.tryFailure(e)
               throw e
           }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 5ce758e1e4eb8..64ecf5deeff4b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{BitwiseAnd, BitwiseOr, Cast, Literal, ShiftLeft}
 import org.apache.spark.sql.catalyst.plans.logical.BROADCAST
 import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AdaptiveTestUtils, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.exchange.EnsureRequirements
 import org.apache.spark.sql.functions._
@@ -39,7 +39,8 @@ import org.apache.spark.sql.types.{LongType, ShortType}
  * unsafe map in [[org.apache.spark.sql.execution.joins.UnsafeHashedRelation]] is not triggered
  * without serializing the hashed relation, which does not happen in local mode.
  */
-class BroadcastJoinSuite extends QueryTest with SQLTestUtils with AdaptiveSparkPlanHelper {
+abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
+  with AdaptiveSparkPlanHelper {
   import testImplicits._
 
   protected var spark: SparkSession = null
@@ -398,4 +399,22 @@ class BroadcastJoinSuite extends QueryTest with SQLTestUtils with AdaptiveSparkP
         }
     }
   }
+
+  test("Broadcast timeout") {
+    val timeout = 5
+    val slowUDF = udf({ x: Int => Thread.sleep(timeout * 10 * 1000); x })
+    val df1 = spark.range(10).select($"id" as 'a)
+    val df2 = spark.range(5).select(slowUDF($"id") as 'a)
+    val testDf = df1.join(broadcast(df2), "a")
+    withSQLConf(SQLConf.BROADCAST_TIMEOUT.key -> timeout.toString) {
+      val e = intercept[Exception] {
+        testDf.collect()
+      }
+      AdaptiveTestUtils.assertExceptionMessage(e, s"Could not execute broadcast in $timeout secs.")
+    }
+  }
 }
+
+class BroadcastJoinSuite extends BroadcastJoinSuiteBase with DisableAdaptiveExecutionSuite
+
+class BroadcastJoinSuiteAE extends BroadcastJoinSuiteBase with EnableAdaptiveExecutionSuite

From 1701f7882aac9e3efaa36c628815edfad09b62fa Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Mon, 20 Apr 2020 13:08:55 -0700
Subject: [PATCH 0525/1280] [SPARK-31234][SQL][FOLLOW-UP] ResetCommand should
 not affect static SQL Configuration

### What changes were proposed in this pull request?
This PR is the follow-up PR of https://github.com/apache/spark/pull/28003

- add a migration guide
- add an end-to-end test case.

### Why are the changes needed?
The original PR made the major behavior change in the user-facing RESET command.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added a new end-to-end test

Closes #28265 from gatorsmile/spark-31234followup.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 6c792a79c10e7b01bd040ef14c848a2a2378e28c)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 docs/core-migration-guide.md                     |  2 +-
 docs/sql-migration-guide.md                      |  4 ++++
 .../spark/sql/internal/StaticSQLConf.scala       |  3 +++
 .../apache/spark/sql/internal/SharedState.scala  |  3 ---
 .../spark/sql/SparkSessionBuilderSuite.scala     | 16 ++++++++++++++++
 .../apache/spark/sql/internal/SQLConfSuite.scala |  2 +-
 6 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index cde6e070c5e35..33406d010bb98 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -25,7 +25,7 @@ license: |
 ## Upgrading from Core 2.4 to 3.0
 
 - The `org.apache.spark.ExecutorPlugin` interface and related configuration has been replaced with
-  `org.apache.spark.plugin.SparkPlugin`, which adds new functionality. Plugins using the old
+  `org.apache.spark.api.plugin.SparkPlugin`, which adds new functionality. Plugins using the old
   interface must be modified to extend the new interfaces. Check the
   [Monitoring](monitoring.html) guide for more details.
 
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index f5c81e9e575f6..8945c136ad937 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -210,6 +210,10 @@ license: |
 
     * The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
 
+## Upgrading from Spark SQL 2.4.5 to 2.4.6
+
+  - In Spark 2.4.6, the `RESET` command does not reset the static SQL configuration values to the default. It only clears the runtime SQL configuration values.
+
 ## Upgrading from Spark SQL 2.4.4 to 2.4.5
 
   - Since Spark 2.4.5, `TRUNCATE TABLE` command tries to set back original permission and ACLs during re-creating the table/partition paths. To restore the behaviour of earlier versions, set `spark.sql.truncateTable.ignorePermissionAcl.enabled` to `true`.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index d20252825c55b..9618ff6062635 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -47,6 +47,9 @@ object StaticSQLConf {
     .internal()
     .version("2.1.0")
     .stringConf
+    // System preserved database should not exists in metastore. However it's hard to guarantee it
+    // for every session, because case-sensitivity differs. Here we always lowercase it to make our
+    // life easier.
     .transform(_.toLowerCase(Locale.ROOT))
     .createWithDefault("global_temp")
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 14b8ea66c8071..47119ab903da7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -153,9 +153,6 @@ private[sql] class SharedState(
    * A manager for global temporary views.
    */
   lazy val globalTempViewManager: GlobalTempViewManager = {
-    // System preserved database should not exists in metastore. However it's hard to guarantee it
-    // for every session, because case-sensitivity differs. Here we always lowercase it to make our
-    // life easier.
     val globalTempDB = conf.get(GLOBAL_TEMP_DATABASE)
     if (externalCatalog.databaseExists(globalTempDB)) {
       throw new SparkException(
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 10b17571d2aaa..f2386413bea18 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -22,6 +22,7 @@ import org.scalatest.BeforeAndAfterEach
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 
 /**
  * Test cases for the builder pattern of [[SparkSession]].
@@ -152,4 +153,19 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
       session.sparkContext.hadoopConfiguration.unset(mySpecialKey)
     }
   }
+
+  test("SPARK-31234: RESET command will not change static sql configs and " +
+    "spark context conf values in SessionState") {
+    val session = SparkSession.builder()
+      .master("local")
+      .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31234")
+      .config("spark.app.name", "test-app-SPARK-31234")
+      .getOrCreate()
+
+    assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
+    assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
+    session.sql("RESET")
+    assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
+    assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index f389465d4b489..feccf52d0d3bc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -116,7 +116,7 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
     }
   }
 
-  test("reset will not change static sql configs and spark core configs") {
+  test("SPARK-31234: reset will not change static sql configs and spark core configs") {
     val conf = spark.sparkContext.getConf.getAll.toMap
     val appName = conf.get("spark.app.name")
     val driverHost = conf.get("spark.driver.host")

From ab93729987084ec55f762639a7e7f7cb8dd275e1 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Mon, 20 Apr 2020 13:38:24 -0700
Subject: [PATCH 0526/1280] [SPARK-31381][SPARK-29245][SQL] Upgrade built-in
 Hive 2.3.6 to 2.3.7

### What changes were proposed in this pull request?

**Hive 2.3.7** fixed these issues:
- HIVE-21508: ClassCastException when initializing HiveMetaStoreClient on JDK10 or newer
- HIVE-21980:Parsing time can be high in case of deeply nested subqueries
- HIVE-22249: Support Parquet through HCatalog

### Why are the changes needed?
Fix CCE during creating HiveMetaStoreClient in JDK11 environment: [SPARK-29245](https://issues.apache.org/jira/browse/SPARK-29245).

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?

- [x] Test Jenkins with Hadoop 2.7 (https://github.com/apache/spark/pull/28148#issuecomment-616757840)
- [x] Test Jenkins with Hadoop 3.2 on JDK11 (https://github.com/apache/spark/pull/28148#issuecomment-616294353)
- [x] Manual test with remote hive metastore.

Hive side:

```
export JAVA_HOME=/usr/lib/jdk1.8.0_221
export PATH=$JAVA_HOME/bin:$PATH
cd /usr/lib/hive-2.3.6 # Start Hive metastore with Hive 2.3.6
bin/schematool -dbType derby -initSchema --verbose
bin/hive --service metastore
```

Spark side:

```
export JAVA_HOME=/usr/lib/jdk-11.0.3
export PATH=$JAVA_HOME/bin:$PATH
build/sbt clean package -Phive -Phadoop-3.2 -Phive-thriftserver
export SPARK_PREPEND_CLASSES=true
bin/spark-sql --conf spark.hadoop.hive.metastore.uris=thrift://localhost:9083
```

Closes #28148 from wangyum/SPARK-31381.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b11e42663be680a0357f2bf7bd8b16afe313eb5e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-2.3       | 26 +++++++++----------
 dev/deps/spark-deps-hadoop-3.2-hive-2.3       | 26 +++++++++----------
 docs/building-spark.md                        |  4 +--
 docs/sql-data-sources-hive-tables.md          |  8 +++---
 docs/sql-migration-guide.md                   |  2 +-
 pom.xml                                       |  4 +--
 .../HiveThriftServer2Suites.scala             |  4 +--
 .../org/apache/spark/sql/hive/HiveUtils.scala |  2 +-
 .../hive/client/IsolatedClientLoader.scala    |  3 ++-
 .../spark/sql/hive/client/package.scala       |  2 +-
 10 files changed, 41 insertions(+), 40 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 79214058b8f03..beb6c83bfa088 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -79,20 +79,20 @@ hadoop-yarn-client/2.7.4//hadoop-yarn-client-2.7.4.jar
 hadoop-yarn-common/2.7.4//hadoop-yarn-common-2.7.4.jar
 hadoop-yarn-server-common/2.7.4//hadoop-yarn-server-common-2.7.4.jar
 hadoop-yarn-server-web-proxy/2.7.4//hadoop-yarn-server-web-proxy-2.7.4.jar
-hive-beeline/2.3.6//hive-beeline-2.3.6.jar
-hive-cli/2.3.6//hive-cli-2.3.6.jar
-hive-common/2.3.6//hive-common-2.3.6.jar
-hive-exec/2.3.6/core/hive-exec-2.3.6-core.jar
-hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
-hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
-hive-metastore/2.3.6//hive-metastore-2.3.6.jar
-hive-serde/2.3.6//hive-serde-2.3.6.jar
-hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
-hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
-hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
-hive-shims/2.3.6//hive-shims-2.3.6.jar
+hive-beeline/2.3.7//hive-beeline-2.3.7.jar
+hive-cli/2.3.7//hive-cli-2.3.7.jar
+hive-common/2.3.7//hive-common-2.3.7.jar
+hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
+hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
+hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
+hive-metastore/2.3.7//hive-metastore-2.3.7.jar
+hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
+hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
+hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
+hive-shims/2.3.7//hive-shims-2.3.7.jar
 hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar
-hive-vector-code-gen/2.3.6//hive-vector-code-gen-2.3.6.jar
+hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index fd4bb7e32a492..69c7cdf8c0ed3 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -78,20 +78,20 @@ hadoop-yarn-common/3.2.0//hadoop-yarn-common-3.2.0.jar
 hadoop-yarn-registry/3.2.0//hadoop-yarn-registry-3.2.0.jar
 hadoop-yarn-server-common/3.2.0//hadoop-yarn-server-common-3.2.0.jar
 hadoop-yarn-server-web-proxy/3.2.0//hadoop-yarn-server-web-proxy-3.2.0.jar
-hive-beeline/2.3.6//hive-beeline-2.3.6.jar
-hive-cli/2.3.6//hive-cli-2.3.6.jar
-hive-common/2.3.6//hive-common-2.3.6.jar
-hive-exec/2.3.6/core/hive-exec-2.3.6-core.jar
-hive-jdbc/2.3.6//hive-jdbc-2.3.6.jar
-hive-llap-common/2.3.6//hive-llap-common-2.3.6.jar
-hive-metastore/2.3.6//hive-metastore-2.3.6.jar
-hive-serde/2.3.6//hive-serde-2.3.6.jar
-hive-shims-0.23/2.3.6//hive-shims-0.23-2.3.6.jar
-hive-shims-common/2.3.6//hive-shims-common-2.3.6.jar
-hive-shims-scheduler/2.3.6//hive-shims-scheduler-2.3.6.jar
-hive-shims/2.3.6//hive-shims-2.3.6.jar
+hive-beeline/2.3.7//hive-beeline-2.3.7.jar
+hive-cli/2.3.7//hive-cli-2.3.7.jar
+hive-common/2.3.7//hive-common-2.3.7.jar
+hive-exec/2.3.7/core/hive-exec-2.3.7-core.jar
+hive-jdbc/2.3.7//hive-jdbc-2.3.7.jar
+hive-llap-common/2.3.7//hive-llap-common-2.3.7.jar
+hive-metastore/2.3.7//hive-metastore-2.3.7.jar
+hive-serde/2.3.7//hive-serde-2.3.7.jar
+hive-shims-0.23/2.3.7//hive-shims-0.23-2.3.7.jar
+hive-shims-common/2.3.7//hive-shims-common-2.3.7.jar
+hive-shims-scheduler/2.3.7//hive-shims-scheduler-2.3.7.jar
+hive-shims/2.3.7//hive-shims-2.3.7.jar
 hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar
-hive-vector-code-gen/2.3.6//hive-vector-code-gen-2.3.6.jar
+hive-vector-code-gen/2.3.7//hive-vector-code-gen-2.3.7.jar
 hk2-api/2.6.1//hk2-api-2.6.1.jar
 hk2-locator/2.6.1//hk2-locator-2.6.1.jar
 hk2-utils/2.6.1//hk2-utils-2.6.1.jar
diff --git a/docs/building-spark.md b/docs/building-spark.md
index 77ab7900dc4a2..3d12a60e2b974 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -83,9 +83,9 @@ Example:
 
 To enable Hive integration for Spark SQL along with its JDBC server and CLI,
 add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options.
-By default Spark will build with Hive 2.3.6.
+By default Spark will build with Hive 2.3.7.
 
-    # With Hive 2.3.6 support
+    # With Hive 2.3.7 support
     ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package
 
 ## Packaging without Hadoop Dependencies for YARN
diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md
index 22514cd082ecf..ae3572c474e1a 100644
--- a/docs/sql-data-sources-hive-tables.md
+++ b/docs/sql-data-sources-hive-tables.md
@@ -127,10 +127,10 @@ The following options can be used to configure the version of Hive that is used
   <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
   <tr>
     <td><code>spark.sql.hive.metastore.version</code></td>
-    <td><code>2.3.6</code></td>
+    <td><code>2.3.7</code></td>
     <td>
       Version of the Hive metastore. Available
-      options are <code>0.12.0</code> through <code>2.3.6</code> and <code>3.0.0</code> through <code>3.1.2</code>.
+      options are <code>0.12.0</code> through <code>2.3.7</code> and <code>3.0.0</code> through <code>3.1.2</code>.
     </td>
     <td>1.4.0</td>
   </tr>
@@ -142,9 +142,9 @@ The following options can be used to configure the version of Hive that is used
       property can be one of three options:
       <ol>
         <li><code>builtin</code></li>
-        Use Hive 2.3.6, which is bundled with the Spark assembly when <code>-Phive</code> is
+        Use Hive 2.3.7, which is bundled with the Spark assembly when <code>-Phive</code> is
         enabled. When this option is chosen, <code>spark.sql.hive.metastore.version</code> must be
-        either <code>2.3.6</code> or not defined.
+        either <code>2.3.7</code> or not defined.
         <li><code>maven</code></li>
         Use Hive jars of specified version downloaded from Maven repositories. This configuration
         is not generally recommended for production deployments.
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 8945c136ad937..854c9eae98a1a 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -787,7 +787,7 @@ Python UDF registration is unchanged.
 Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs.
 Currently, Hive SerDes and UDFs are based on built-in Hive,
 and Spark SQL can be connected to different versions of Hive Metastore
-(from 0.12.0 to 2.3.6 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
+(from 0.12.0 to 2.3.7 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)).
 
 #### Deploying in Existing Hive Warehouses
 {:.no_toc}
diff --git a/pom.xml b/pom.xml
index e5c8724cf6ae4..05479738965d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -127,8 +127,8 @@
     <hive.group>org.apache.hive</hive.group>
     <hive.classifier>core</hive.classifier>
     <!-- Version used in Maven Hive dependency -->
-    <hive.version>2.3.6</hive.version>
-    <hive23.version>2.3.6</hive23.version>
+    <hive.version>2.3.7</hive.version>
+    <hive23.version>2.3.7</hive23.version>
     <!-- Version used for internal directory structure -->
     <hive.version.short>2.3</hive.version.short>
     <!-- note that this should be compatible with Kafka brokers version 0.10 and up -->
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 84eed7b2eda22..28f67cbbf7b46 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -545,7 +545,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
 
       if (HiveUtils.isHive23) {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.6"))
+        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
       } else {
         assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1"))
       }
@@ -562,7 +562,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
 
       if (HiveUtils.isHive23) {
-        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.6"))
+        assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7"))
       } else {
         assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1"))
       }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 3c20e68278e3e..d0f79880e85e9 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -63,7 +63,7 @@ private[spark] object HiveUtils extends Logging {
 
   val HIVE_METASTORE_VERSION = buildConf("spark.sql.hive.metastore.version")
     .doc("Version of the Hive metastore. Available options are " +
-        "<code>0.12.0</code> through <code>2.3.6</code> and " +
+        "<code>0.12.0</code> through <code>2.3.7</code> and " +
         "<code>3.0.0</code> through <code>3.1.2</code>.")
     .version("1.4.0")
     .stringConf
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
index 5da7b70cfc7aa..42a0ec0253b85 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -104,7 +104,8 @@ private[hive] object IsolatedClientLoader extends Logging {
     case "2.0" | "2.0.0" | "2.0.1" => hive.v2_0
     case "2.1" | "2.1.0" | "2.1.1" => hive.v2_1
     case "2.2" | "2.2.0" => hive.v2_2
-    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" => hive.v2_3
+    case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" =>
+      hive.v2_3
     case "3.0" | "3.0.0" => hive.v3_0
     case "3.1" | "3.1.0" | "3.1.1" | "3.1.2" => hive.v3_1
     case version =>
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
index 26319119a596a..8526d86454604 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
@@ -89,7 +89,7 @@ package object client {
 
     // Since HIVE-14496, Hive materialized view need calcite-core.
     // For spark, only VersionsSuite currently creates a hive materialized view for testing.
-    case object v2_3 extends HiveVersion("2.3.6",
+    case object v2_3 extends HiveVersion("2.3.7",
       exclusions = Seq("org.apache.calcite:calcite-druid",
         "org.apache.calcite.avatica:avatica",
         "org.apache.curator:*",

From 9b8fd3425ced975e7e469e5bbbef61dca631b31f Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 21 Apr 2020 10:55:13 +0900
Subject: [PATCH 0527/1280] [SPARK-31429][SQL][DOC] Automatically generates a
 SQL document for built-in functions

This PR intends to add a Python script to generates a SQL document for built-in functions and the document in SQL references.

To make SQL references complete.

Yes;

![a](https://user-images.githubusercontent.com/692303/79406712-c39e1b80-7fd2-11ea-8b85-9f9cbb6efed3.png)
![b](https://user-images.githubusercontent.com/692303/79320526-eb46a280-7f44-11ea-8639-90b1fb2b8848.png)
![c](https://user-images.githubusercontent.com/692303/79320707-3365c500-7f45-11ea-9984-69ffe800fb87.png)

Manually checked and added tests.

Closes #28224 from maropu/SPARK-31429.

Lead-authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Co-authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e42dbe7cd41c3689910165458a92b75c02e70a03)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/.gitignore                               |   2 +-
 docs/_data/menu-sql.yaml                      |   2 +
 docs/configuration.md                         |   4 +-
 docs/sql-ref-functions-builtin.md             |  77 ++++++
 docs/sql-ref-functions.md                     |  12 +
 .../expressions/ExpressionDescription.java    |  14 +-
 .../catalyst/expressions/ExpressionInfo.java  |  27 ++-
 .../catalyst/analysis/FunctionRegistry.scala  |   3 +-
 .../aggregate/ApproximatePercentile.scala     |   1 +
 .../expressions/aggregate/Average.scala       |   1 +
 .../aggregate/CentralMomentAgg.scala          |   6 +
 .../catalyst/expressions/aggregate/Corr.scala |   1 +
 .../expressions/aggregate/Count.scala         |   1 +
 .../expressions/aggregate/CountIf.scala       |   1 +
 .../aggregate/CountMinSketchAgg.scala         |   1 +
 .../expressions/aggregate/Covariance.scala    |   2 +
 .../expressions/aggregate/First.scala         |   1 +
 .../aggregate/HyperLogLogPlusPlus.scala       |   1 +
 .../catalyst/expressions/aggregate/Last.scala |   1 +
 .../catalyst/expressions/aggregate/Max.scala  |   1 +
 .../expressions/aggregate/MaxByAndMinBy.scala |   2 +
 .../catalyst/expressions/aggregate/Min.scala  |   1 +
 .../expressions/aggregate/Percentile.scala    |   1 +
 .../catalyst/expressions/aggregate/Sum.scala  |   1 +
 .../aggregate/UnevaluableAggs.scala           |   2 +
 .../aggregate/bitwiseAggregates.scala         |   2 +
 .../expressions/aggregate/collect.scala       |   2 +
 .../expressions/collectionOperations.scala    |  59 ++++-
 .../expressions/complexTypeCreator.scala      |   1 +
 .../expressions/datetimeExpressions.scala     |  32 +++
 .../expressions/jsonExpressions.scala         |   9 +-
 .../expressions/windowExpressions.scala       |  24 +-
 .../sql/SparkSessionExtensionSuite.scala      |   3 +
 .../scala/org/apache/spark/sql/UDFSuite.scala |  19 ++
 sql/create-docs.sh                            |   3 +
 sql/gen-sql-api-docs.py                       |   5 +-
 sql/gen-sql-config-docs.py                    |   9 +-
 sql/gen-sql-functions-docs.py                 | 228 ++++++++++++++++++
 38 files changed, 523 insertions(+), 39 deletions(-)
 create mode 100644 docs/sql-ref-functions-builtin.md
 create mode 100644 sql/gen-sql-functions-docs.py

diff --git a/docs/.gitignore b/docs/.gitignore
index 2260493b46ab3..9df83f37815b7 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1 +1 @@
-sql-configs.html
+generated-*.html
diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 5042c25884289..a16e1144950ac 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -246,6 +246,8 @@
     - text: Functions
       url: sql-ref-functions.html
       subitems:
+      - text: Built-in Functions
+        url: sql-ref-functions-builtin.html
       - text: Scalar UDFs (User-Defined Functions)
         url: sql-ref-functions-udf-scalar.html
       - text: UDAFs (User-Defined Aggregate Functions)
diff --git a/docs/configuration.md b/docs/configuration.md
index 8e78ef2d0dcda..829d0627be1bc 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2612,10 +2612,10 @@ Spark subsystems.
 
 
 {% for static_file in site.static_files %}
-    {% if static_file.name == 'sql-configs.html' %}
+    {% if static_file.name == 'generated-sql-configuration-table.html' %}
 ### Spark SQL
 
-        {% include_relative sql-configs.html %}
+{% include_relative generated-sql-configuration-table.html %}
         {% break %}
     {% endif %}
 {% endfor %}
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
new file mode 100644
index 0000000000000..1bca68e5f19df
--- /dev/null
+++ b/docs/sql-ref-functions-builtin.md
@@ -0,0 +1,77 @@
+---
+layout: global
+title: Built-in Functions
+displayTitle: Built-in Functions
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+  http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-agg-funcs-table.html' %}
+### Aggregate Functions
+{% include_relative generated-agg-funcs-table.html %}
+#### Examples
+{% include_relative generated-agg-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-window-funcs-table.html' %}
+### Window Functions
+{% include_relative generated-window-funcs-table.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-array-funcs-table.html' %}
+### Array Functions
+{% include_relative generated-array-funcs-table.html %}
+#### Examples
+{% include_relative generated-array-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-map-funcs-table.html' %}
+### Map Functions
+{% include_relative generated-map-funcs-table.html %}
+#### Examples
+{% include_relative generated-map-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-datetime-funcs-table.html' %}
+### Date and Timestamp Functions
+{% include_relative generated-datetime-funcs-table.html %}
+#### Examples
+{% include_relative generated-datetime-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-json-funcs-table.html' %}
+### JSON Functions
+{% include_relative generated-json-funcs-table.html %}
+#### Examples
+{% include_relative generated-agg-funcs-examples.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index e8a0353579301..6368fb705f893 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -22,6 +22,18 @@ license: |
 Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs).
 Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task.
 
+### Built-in Functions
+
+Spark SQL has some categories of frequently-used built-in functions for aggregtion, arrays/maps, date/timestamp, and JSON data.
+This subsection presents the usages and descriptions of these functions.
+
+ * [Aggregate Functions](sql-ref-functions-builtin.html#aggregate-functions)
+ * [Window Functions](sql-ref-functions-builtin.html#window-functions)
+ * [Array Functions](sql-ref-functions-builtin.html#array-functions)
+ * [Map Functions](sql-ref-functions-builtin.html#map-functions)
+ * [Date and Timestamp Functions](sql-ref-functions-builtin.html#date-and-timestamp-functions)
+ * [JSON Functions](sql-ref-functions-builtin.html#json-functions)
+
 ### UDFs (User-Defined Functions)
 
 User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
index acdf6afe10ce5..0f648ab9d7d26 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
@@ -31,7 +31,7 @@
  * `usage()` will be used for the function usage in brief way.
  *
  * These below are concatenated and used for the function usage in verbose way, suppose arguments,
- * examples, note, since and deprecated will be provided.
+ * examples, note, group, since and deprecated will be provided.
  *
  * `arguments()` describes arguments for the expression.
  *
@@ -39,13 +39,16 @@
  *
  * `note()` contains some notes for the expression optionally.
  *
+ * `group()` describes the category that the expression belongs to. The valid value is
+ * "agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs" and "window_funcs".
+ *
  * `since()` contains version information for the expression. Version is specified by,
  * for example, "2.2.0".
  *
  * `deprecated()` contains deprecation information for the expression optionally, for example,
  * "Deprecated since 2.2.0. Use something else instead".
  *
- * The format, in particular for `arguments()`, `examples()`,`note()`, `since()` and
+ * The format, in particular for `arguments()`, `examples()`,`note()`, `group()`, `since()` and
  * `deprecated()`, should strictly be as follows.
  *
  * <pre>
@@ -68,6 +71,7 @@
  *   note = """
  *     ...
  *   """,
+ *   group = "agg_funcs",
  *   since = "3.0.0",
  *   deprecated = """
  *     ...
@@ -78,8 +82,9 @@
  *  We can refer the function name by `_FUNC_`, in `usage()`, `arguments()` and `examples()` as
  *  it is registered in `FunctionRegistry`.
  *
- *  Note that, if `extended()` is defined, `arguments()`, `examples()`, `note()`, `since()` and
- *  `deprecated()` should be not defined together. `extended()` exists for backward compatibility.
+ *  Note that, if `extended()` is defined, `arguments()`, `examples()`, `note()`, `group()`,
+ *  `since()` and `deprecated()` should be not defined together. `extended()` exists
+ *  for backward compatibility.
  *
  *  Note this contents are used in the SparkSQL documentation for built-in functions. The contents
  *  here are considered as a Markdown text and then rendered.
@@ -98,6 +103,7 @@
     String arguments() default "";
     String examples() default "";
     String note() default "";
+    String group() default "";
     String since() default "";
     String deprecated() default "";
 }
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
index 8ee90ed6f4c3b..a500822b21f02 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java
@@ -19,6 +19,10 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * Expression information, will be used to describe a expression.
  */
@@ -31,9 +35,14 @@ public class ExpressionInfo {
     private String arguments;
     private String examples;
     private String note;
+    private String group;
     private String since;
     private String deprecated;
 
+    private static final Set<String> validGroups =
+        new HashSet<>(Arrays.asList("agg_funcs", "array_funcs", "datetime_funcs",
+            "json_funcs", "map_funcs", "window_funcs"));
+
     public String getClassName() {
         return className;
     }
@@ -75,6 +84,10 @@ public String getDeprecated() {
         return deprecated;
     }
 
+    public String getGroup() {
+        return group;
+    }
+
     public String getDb() {
         return db;
     }
@@ -87,6 +100,7 @@ public ExpressionInfo(
             String arguments,
             String examples,
             String note,
+            String group,
             String since,
             String deprecated) {
         assert name != null;
@@ -94,6 +108,7 @@ public ExpressionInfo(
         assert examples != null;
         assert examples.isEmpty() || examples.contains("    Examples:");
         assert note != null;
+        assert group != null;
         assert since != null;
         assert deprecated != null;
 
@@ -104,6 +119,7 @@ public ExpressionInfo(
         this.arguments = arguments;
         this.examples = examples;
         this.note = note;
+        this.group = group;
         this.since = since;
         this.deprecated = deprecated;
 
@@ -120,6 +136,11 @@ public ExpressionInfo(
             }
             this.extended += "\n    Note:\n      " + note.trim() + "\n";
         }
+        if (!group.isEmpty() && !validGroups.contains(group)) {
+            throw new IllegalArgumentException("'group' is malformed in the expression [" +
+                this.name + "]. It should be a value in " + validGroups + "; however, " +
+                "got [" + group + "].");
+        }
         if (!since.isEmpty()) {
             if (Integer.parseInt(since.split("\\.")[0]) < 0) {
                 throw new IllegalArgumentException("'since' is malformed in the expression [" +
@@ -140,11 +161,11 @@ public ExpressionInfo(
     }
 
     public ExpressionInfo(String className, String name) {
-        this(className, null, name, null, "", "", "", "", "");
+        this(className, null, name, null, "", "", "", "", "", "");
     }
 
     public ExpressionInfo(String className, String db, String name) {
-        this(className, db, name, null, "", "", "", "", "");
+        this(className, db, name, null, "", "", "", "", "", "");
     }
 
     /**
@@ -155,7 +176,7 @@ public ExpressionInfo(String className, String db, String name) {
     public ExpressionInfo(String className, String db, String name, String usage, String extended) {
         // `arguments` and `examples` are concatenated for the extended description. So, here
         // simply pass the `extended` as `arguments` and an empty string for `examples`.
-        this(className, db, name, usage, extended, "", "", "", "");
+        this(className, db, name, usage, extended, "", "", "", "", "");
     }
 
     private String replaceFunctionName(String usage) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 341a335861e5b..6cd492f0765c8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -653,7 +653,7 @@ object FunctionRegistry {
     val clazz = scala.reflect.classTag[Cast].runtimeClass
     val usage = "_FUNC_(expr) - Casts the value `expr` to the target data type `_FUNC_`."
     val expressionInfo =
-      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "", "", "")
+      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "", "", "", "")
     (name, (expressionInfo, builder))
   }
 
@@ -673,6 +673,7 @@ object FunctionRegistry {
           df.arguments(),
           df.examples(),
           df.note(),
+          df.group(),
           df.since(),
           df.deprecated())
       } else {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index f8060956df875..d06eeeef23936 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -65,6 +65,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(10.0, 0.5, 100);
        10.0
   """,
+  group = "agg_funcs",
   since = "2.1.0")
 case class ApproximatePercentile(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
index 17f906c698de2..d3ce1f8d331ab 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (NULL) AS tab(col);
        1.5
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Average(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
index bf402807d62d3..53759ca3d9165 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala
@@ -142,6 +142,7 @@ abstract class CentralMomentAgg(child: Expression)
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        0.816496580927726
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
@@ -164,6 +165,7 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        1.0
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class StddevSamp(child: Expression) extends CentralMomentAgg(child) {
@@ -187,6 +189,7 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        0.6666666666666666
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
 
@@ -207,6 +210,7 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (2), (3) AS tab(col);
        1.0
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) {
 
@@ -229,6 +233,7 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (-1000), (-100), (10), (20) AS tab(col);
        -1.1135657469022011
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class Skewness(child: Expression) extends CentralMomentAgg(child) {
 
@@ -251,6 +256,7 @@ case class Skewness(child: Expression) extends CentralMomentAgg(child) {
       > SELECT _FUNC_(col) FROM VALUES (1), (10), (100), (10), (1) as tab(col);
        0.19432323191699075
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class Kurtosis(child: Expression) extends CentralMomentAgg(child) {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
index 91446e05d853d..9ef05bb5d4fec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala
@@ -99,6 +99,7 @@ abstract class PearsonCorrelation(x: Expression, y: Expression)
       > SELECT _FUNC_(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2);
        0.8660254037844387
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class Corr(x: Expression, y: Expression)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
index 2a8edac502c0f..e043c81975066 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(DISTINCT col) FROM VALUES (NULL), (5), (5), (10) AS tab(col);
        2
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 // scalastyle:on line.size.limit
 case class Count(children: Seq[Expression]) extends DeclarativeAggregate {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
index d31355cd022fa..5bb95ead3f715 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountIf.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.types.{AbstractDataType, BooleanType, DataType, Long
       > SELECT _FUNC_(col IS NULL) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col);
        1
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class CountIf(predicate: Expression) extends UnevaluableAggregate with ImplicitCastInputTypes {
   override def prettyName: String = "count_if"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
index 4bd13cf284935..787b21859c6da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala
@@ -44,6 +44,7 @@ import org.apache.spark.util.sketch.CountMinSketch
       `CountMinSketch` before usage. Count-min sketch is a probabilistic data structure used for
       cardinality estimation using sub-linear space.
   """,
+  group = "agg_funcs",
   since = "2.2.0")
 case class CountMinSketchAgg(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
index ac99fa8049f93..f03c2f2710a04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala
@@ -86,6 +86,7 @@ abstract class Covariance(x: Expression, y: Expression)
       > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);
        0.6666666666666666
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) {
   override val evaluateExpression: Expression = {
@@ -102,6 +103,7 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance
       > SELECT _FUNC_(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);
        1.0
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) {
   override val evaluateExpression: Expression = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
index 2c0060c22a865..df806edbfda05 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
@@ -47,6 +47,7 @@ import org.apache.spark.sql.types._
     The function is non-deterministic because its results depends on the order of the rows
     which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class First(child: Expression, ignoreNullsExpr: Expression)
   extends DeclarativeAggregate with ExpectsInputTypes {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
index e3c628e70d11b..aed36902b1567 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
@@ -53,6 +53,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);
        3
   """,
+  group = "agg_funcs",
   since = "1.6.0")
 case class HyperLogLogPlusPlus(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
index 6793ac7632ffd..e55bced192f34 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
@@ -47,6 +47,7 @@ import org.apache.spark.sql.types._
     The function is non-deterministic because its results depends on the order of the rows
     which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class Last(child: Expression, ignoreNullsExpr: Expression)
   extends DeclarativeAggregate with ExpectsInputTypes {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
index 7520db146ba6a..9bba6604c84ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col) FROM VALUES (10), (50), (20) AS tab(col);
        50
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Max(child: Expression) extends DeclarativeAggregate {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
index b69b341b0ee3e..2e202240923c3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
@@ -98,6 +98,7 @@ abstract class MaxMinBy extends DeclarativeAggregate {
       > SELECT _FUNC_(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);
        b
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
   override protected def funcName: String = "max_by"
@@ -116,6 +117,7 @@ case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMin
       > SELECT _FUNC_(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);
        a
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class MinBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
   override protected def funcName: String = "min_by"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
index 106eb968e3917..1d861aa0dd8cf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col) FROM VALUES (10), (-1), (20) AS tab(col);
        -1
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Min(child: Expression) extends DeclarativeAggregate {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
index 0f1c0fb5fcb69..0eba61c741133 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Percentile.scala
@@ -62,6 +62,7 @@ import org.apache.spark.util.collection.OpenHashMap
       > SELECT _FUNC_(col, array(0.25, 0.75)) FROM VALUES (0), (10) AS tab(col);
        [2.5,7.5]
   """,
+  group = "agg_funcs",
   since = "2.1.0")
 case class Percentile(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
index 8bfd889ea0563..d2daaac72fc85 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.types._
       > SELECT _FUNC_(col) FROM VALUES (NULL), (NULL) AS tab(col);
        NULL
   """,
+  group = "agg_funcs",
   since = "1.0.0")
 case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCastInputTypes {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
index a1cd4a77d0445..cb77ded3372a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/UnevaluableAggs.scala
@@ -51,6 +51,7 @@ abstract class UnevaluableBooleanAggBase(arg: Expression)
       > SELECT _FUNC_(col) FROM VALUES (true), (false), (true) AS tab(col);
        false
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BoolAnd(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   override def nodeName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bool_and")
@@ -67,6 +68,7 @@ case class BoolAnd(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
       > SELECT _FUNC_(col) FROM VALUES (false), (false), (NULL) AS tab(col);
        false
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BoolOr(arg: Expression) extends UnevaluableBooleanAggBase(arg) {
   override def nodeName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("bool_or")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
index b77c3bd9cbde4..b4c1b2c708fb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala
@@ -77,6 +77,7 @@ case class BitAndAgg(child: Expression) extends BitAggregate {
       > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col);
        7
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BitOrAgg(child: Expression) extends BitAggregate {
 
@@ -94,6 +95,7 @@ case class BitOrAgg(child: Expression) extends BitAggregate {
       > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col);
        6
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BitXorAgg(child: Expression) extends BitAggregate {
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
index 29f89989b4961..5848aa3f840c6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -96,6 +96,7 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
     The function is non-deterministic because the order of collected results depends
     on the order of the rows which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CollectList(
     child: Expression,
@@ -129,6 +130,7 @@ case class CollectList(
     The function is non-deterministic because the order of collected results depends
     on the order of the rows which may be non-deterministic after a shuffle.
   """,
+  group = "agg_funcs",
   since = "2.0.0")
 case class CollectSet(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 8b61bc4f22b94..4fd68dcfe5156 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -138,7 +138,8 @@ object Size {
     Examples:
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        [1,2]
-  """)
+  """,
+  group = "map_funcs")
 case class MapKeys(child: Expression)
   extends UnaryExpression with ExpectsInputTypes {
 
@@ -169,6 +170,7 @@ case class MapKeys(child: Expression)
       > SELECT _FUNC_(array(1, 2), array(2, 3), array(3, 4));
        [{"0":1,"1":2,"2":3},{"0":2,"1":3,"2":4}]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsInputTypes {
 
@@ -327,7 +329,8 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI
     Examples:
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        ["a","b"]
-  """)
+  """,
+  group = "map_funcs")
 case class MapValues(child: Expression)
   extends UnaryExpression with ExpectsInputTypes {
 
@@ -356,6 +359,7 @@ case class MapValues(child: Expression)
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        [{"key":1,"value":"a"},{"key":2,"value":"b"}]
   """,
+  group = "map_funcs",
   since = "3.0.0")
 case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes {
 
@@ -523,7 +527,9 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp
     Examples:
       > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(3, 'c'));
        {1:"a",2:"b",3:"c"}
-  """, since = "2.4.0")
+  """,
+  group = "map_funcs",
+  since = "2.4.0")
 case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   override def checkInputDataTypes(): TypeCheckResult = {
@@ -641,6 +647,7 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
       > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));
        {1:"a",2:"b"}
   """,
+  group = "map_funcs",
   since = "2.4.0")
 case class MapFromEntries(child: Expression) extends UnaryExpression {
 
@@ -862,7 +869,8 @@ object ArraySortLike {
     Examples:
       > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true);
        [null,"a","b","c","d"]
-  """)
+  """,
+  group = "array_funcs")
 // scalastyle:on line.size.limit
 case class SortArray(base: Expression, ascendingOrder: Expression)
   extends BinaryExpression with ArraySortLike {
@@ -920,6 +928,7 @@ case class SortArray(base: Expression, ascendingOrder: Expression)
   note = """
     The function is non-deterministic.
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
   extends UnaryExpression with ExpectsInputTypes with Stateful with ExpressionWithRandomSeed {
@@ -1002,6 +1011,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None)
       > SELECT _FUNC_(array(2, 1, 4, 3));
        [3,4,1,2]
   """,
+  group = "array_funcs",
   since = "1.5.0",
   note = """
     Reverse logic for arrays is available since 2.4.0.
@@ -1073,7 +1083,8 @@ case class Reverse(child: Expression) extends UnaryExpression with ImplicitCastI
     Examples:
       > SELECT _FUNC_(array(1, 2, 3), 2);
        true
-  """)
+  """,
+  group = "array_funcs")
 case class ArrayContains(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
 
@@ -1169,7 +1180,9 @@ case class ArrayContains(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(array(1, 2, 3), array(3, 4, 5));
        true
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 // scalastyle:off line.size.limit
 case class ArraysOverlap(left: Expression, right: Expression)
   extends BinaryArrayExpressionWithImplicitCast {
@@ -1392,7 +1405,9 @@ case class ArraysOverlap(left: Expression, right: Expression)
        [2,3]
       > SELECT _FUNC_(array(1, 2, 3, 4), -2, 2);
        [3,4]
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 // scalastyle:on line.size.limit
 case class Slice(x: Expression, start: Expression, length: Expression)
   extends TernaryExpression with ImplicitCastInputTypes {
@@ -1505,7 +1520,9 @@ case class Slice(x: Expression, start: Expression, length: Expression)
        hello world
       > SELECT _FUNC_(array('hello', null ,'world'), ' ', ',');
        hello , world
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayJoin(
     array: Expression,
     delimiter: Expression,
@@ -1668,7 +1685,9 @@ case class ArrayJoin(
     Examples:
       > SELECT _FUNC_(array(1, 20, null, 3));
        1
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
   override def nullable: Boolean = true
@@ -1733,7 +1752,9 @@ case class ArrayMin(child: Expression) extends UnaryExpression with ImplicitCast
     Examples:
       > SELECT _FUNC_(array(1, 20, null, 3));
        20
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
   override def nullable: Boolean = true
@@ -1807,6 +1828,7 @@ case class ArrayMax(child: Expression) extends UnaryExpression with ImplicitCast
       > SELECT _FUNC_(array(3, 2, 1), 1);
        3
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayPosition(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
@@ -2021,7 +2043,8 @@ case class ElementAt(left: Expression, right: Expression)
   """,
   note = """
     Concat logic for arrays is available since 2.4.0.
-  """)
+  """,
+  group = "array_funcs")
 case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)
@@ -2220,6 +2243,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
       > SELECT _FUNC_(array(array(1, 2), array(3, 4)));
        [1,2,3,4]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class Flatten(child: Expression) extends UnaryExpression {
 
@@ -2352,6 +2376,7 @@ case class Flatten(child: Expression) extends UnaryExpression {
       > SELECT _FUNC_(to_date('2018-01-01'), to_date('2018-03-01'), interval 1 month);
        [2018-01-01,2018-02-01,2018-03-01]
   """,
+  group = "array_funcs",
   since = "2.4.0"
 )
 case class Sequence(
@@ -2734,6 +2759,7 @@ object Sequence {
       > SELECT _FUNC_('123', 2);
        ["123","123"]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayRepeat(left: Expression, right: Expression)
   extends BinaryExpression with ExpectsInputTypes {
@@ -2854,7 +2880,9 @@ case class ArrayRepeat(left: Expression, right: Expression)
     Examples:
       > SELECT _FUNC_(array(1, 2, 3, null, 3), 3);
        [1,2,null]
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayRemove(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
 
@@ -3049,7 +3077,9 @@ trait ArraySetLike {
     Examples:
       > SELECT _FUNC_(array(1, 2, 3, null, 3));
        [1,2,3,null]
-  """, since = "2.4.0")
+  """,
+  group = "array_funcs",
+  since = "2.4.0")
 case class ArrayDistinct(child: Expression)
   extends UnaryExpression with ArraySetLike with ExpectsInputTypes {
 
@@ -3226,6 +3256,7 @@ object ArrayBinaryLike {
       > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
        [1,2,3,5]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
@@ -3437,6 +3468,7 @@ object ArrayUnion {
       > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
        [1,3]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
@@ -3678,6 +3710,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
       > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5));
        [2]
   """,
+  group = "array_funcs",
   since = "2.4.0")
 case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
   with ComplexTypeMergingExpression {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 6c31511571387..858c91a4d8e86 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -333,6 +333,7 @@ object CreateStruct extends FunctionBuilder {
       "",
       "",
       "",
+      "",
       "")
     ("struct", (info, this))
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 323aa3b005d88..0160827fb20d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -64,6 +64,7 @@ trait TimeZoneAwareExpression extends Expression {
  */
 @ExpressionDescription(
   usage = "_FUNC_() - Returns the current date at the start of query evaluation.",
+  group = "datetime_funcs",
   since = "1.5.0")
 case class CurrentDate(timeZoneId: Option[String] = None)
   extends LeafExpression with TimeZoneAwareExpression with CodegenFallback {
@@ -91,6 +92,7 @@ case class CurrentDate(timeZoneId: Option[String] = None)
  */
 @ExpressionDescription(
   usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.",
+  group = "datetime_funcs",
   since = "1.5.0")
 case class CurrentTimestamp() extends LeafExpression with CodegenFallback {
   override def foldable: Boolean = true
@@ -151,6 +153,7 @@ case class CurrentBatchTimestamp(
       > SELECT _FUNC_('2016-07-30', 1);
        2016-07-31
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DateAdd(startDate: Expression, days: Expression)
   extends BinaryExpression with ExpectsInputTypes {
@@ -186,6 +189,7 @@ case class DateAdd(startDate: Expression, days: Expression)
       > SELECT _FUNC_('2016-07-30', 1);
        2016-07-29
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DateSub(startDate: Expression, days: Expression)
   extends BinaryExpression with ExpectsInputTypes {
@@ -217,6 +221,7 @@ case class DateSub(startDate: Expression, days: Expression)
       > SELECT _FUNC_('2009-07-30 12:58:59');
        12
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Hour(child: Expression, timeZoneId: Option[String] = None)
   extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
@@ -248,6 +253,7 @@ case class Hour(child: Expression, timeZoneId: Option[String] = None)
       > SELECT _FUNC_('2009-07-30 12:58:59');
        58
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Minute(child: Expression, timeZoneId: Option[String] = None)
   extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
@@ -279,6 +285,7 @@ case class Minute(child: Expression, timeZoneId: Option[String] = None)
       > SELECT _FUNC_('2009-07-30 12:58:59');
        59
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Second(child: Expression, timeZoneId: Option[String] = None)
   extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes {
@@ -376,6 +383,7 @@ case class Microseconds(child: Expression, timeZoneId: Option[String] = None)
       > SELECT _FUNC_('2016-04-09');
        100
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -400,6 +408,7 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas
       > SELECT _FUNC_('2016-07-30');
        2016
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Year(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -440,6 +449,7 @@ case class IsoYear(child: Expression) extends UnaryExpression with ImplicitCastI
       > SELECT _FUNC_('2016-08-31');
        3
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -464,6 +474,7 @@ case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastI
       > SELECT _FUNC_('2016-07-30');
        7
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class Month(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
@@ -513,6 +524,7 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa
       > SELECT _FUNC_('2009-07-30');
        5
   """,
+  group = "datetime_funcs",
   since = "2.3.0")
 // scalastyle:on line.size.limit
 case class DayOfWeek(child: Expression) extends DayWeek {
@@ -539,6 +551,7 @@ case class DayOfWeek(child: Expression) extends DayWeek {
       > SELECT _FUNC_('2009-07-30');
        3
   """,
+  group = "datetime_funcs",
   since = "2.4.0")
 // scalastyle:on line.size.limit
 case class WeekDay(child: Expression) extends DayWeek {
@@ -572,6 +585,7 @@ abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes {
       > SELECT _FUNC_('2008-02-20');
        8
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
@@ -609,6 +623,7 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
       > SELECT _FUNC_('2016-04-08', 'y');
        2016
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Option[String] = None)
@@ -691,6 +706,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
       > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd');
        1460098800
   """,
+  group = "datetime_funcs",
   since = "1.6.0")
 // scalastyle:on line.size.limit
 case class ToUnixTimestamp(
@@ -740,6 +756,7 @@ case class ToUnixTimestamp(
       > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd');
        1460041200
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class UnixTimestamp(timeExp: Expression, format: Expression, timeZoneId: Option[String] = None)
@@ -933,6 +950,7 @@ abstract class UnixTime extends ToTimestamp {
       > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss');
        1969-12-31 16:00:00
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[String] = None)
@@ -1053,6 +1071,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
       > SELECT _FUNC_('2009-01-12');
        2009-01-31
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitCastInputTypes {
   override def child: Expression = startDate
@@ -1088,6 +1107,7 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC
       > SELECT _FUNC_('2015-01-14', 'TU');
        2015-01-20
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class NextDay(startDate: Expression, dayOfWeek: Expression)
@@ -1200,6 +1220,7 @@ case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[S
       > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
        2016-08-31 09:00:00
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class FromUTCTimestamp(left: Expression, right: Expression)
@@ -1294,6 +1315,7 @@ case class TimeSub(start: Expression, interval: Expression, timeZoneId: Option[S
       > SELECT _FUNC_('2016-08-31', 1);
        2016-09-30
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class AddMonths(startDate: Expression, numMonths: Expression)
@@ -1343,6 +1365,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression)
       > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30', false);
        3.9495967741935485
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class MonthsBetween(
@@ -1404,6 +1427,7 @@ case class MonthsBetween(
       > SELECT _FUNC_('2016-08-31', 'Asia/Seoul');
        2016-08-30 15:00:00
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class ToUTCTimestamp(left: Expression, right: Expression)
@@ -1475,6 +1499,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
       > SELECT _FUNC_('2016-12-31', 'yyyy-MM-dd');
        2016-12-31
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class ParseToDate(left: Expression, format: Option[Expression], child: Expression)
@@ -1525,6 +1550,7 @@ case class ParseToDate(left: Expression, format: Option[Expression], child: Expr
       > SELECT _FUNC_('2016-12-31', 'yyyy-MM-dd');
        2016-12-31 00:00:00
   """,
+  group = "datetime_funcs",
   since = "2.2.0")
 // scalastyle:on line.size.limit
 case class ParseToTimestamp(left: Expression, format: Option[Expression], child: Expression)
@@ -1656,6 +1682,7 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
       > SELECT _FUNC_('1981-01-19', 'millennium');
        1001-01-01
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 // scalastyle:on line.size.limit
 case class TruncDate(date: Expression, format: Expression)
@@ -1709,6 +1736,7 @@ case class TruncDate(date: Expression, format: Expression)
       > SELECT _FUNC_('CENTURY', '2015-03-05T09:32:05.123456');
        2001-01-01 00:00:00
   """,
+  group = "datetime_funcs",
   since = "2.3.0")
 // scalastyle:on line.size.limit
 case class TruncTimestamp(
@@ -1756,6 +1784,7 @@ case class TruncTimestamp(
       > SELECT _FUNC_('2009-07-30', '2009-07-31');
        -1
   """,
+  group = "datetime_funcs",
   since = "1.5.0")
 case class DateDiff(endDate: Expression, startDate: Expression)
   extends BinaryExpression with ImplicitCastInputTypes {
@@ -1809,6 +1838,7 @@ private case class GetTimestamp(
       > SELECT _FUNC_(2019, 2, 30);
        NULL
   """,
+  group = "datetime_funcs",
   since = "3.0.0")
 case class MakeDate(year: Expression, month: Expression, day: Expression)
   extends TernaryExpression with ImplicitCastInputTypes {
@@ -1870,6 +1900,7 @@ case class MakeDate(year: Expression, month: Expression, day: Expression)
       > SELECT _FUNC_(null, 7, 22, 15, 30, 0);
        NULL
   """,
+  group = "datetime_funcs",
   since = "3.0.0")
 // scalastyle:on line.size.limit
 case class MakeTimestamp(
@@ -2177,6 +2208,7 @@ object DatePartLike {
       > SELECT _FUNC_('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
        30.001001
   """,
+  group = "datetime_funcs",
   since = "3.0.0")
 case class DatePart(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index a63e541a8cad4..38faaf500b9b4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -117,7 +117,8 @@ private[this] object SharedFactory {
     Examples:
       > SELECT _FUNC_('{"a":"b"}', '$.a');
        b
-  """)
+  """,
+  group = "json_funcs")
 case class GetJsonObject(json: Expression, path: Expression)
   extends BinaryExpression with ExpectsInputTypes with CodegenFallback {
 
@@ -340,7 +341,8 @@ case class GetJsonObject(json: Expression, path: Expression)
     Examples:
       > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b');
        1	2
-  """)
+  """,
+  group = "json_funcs")
 // scalastyle:on line.size.limit line.contains.tab
 case class JsonTuple(children: Seq[Expression])
   extends Generator with CodegenFallback {
@@ -508,6 +510,7 @@ case class JsonTuple(children: Seq[Expression])
       > SELECT _FUNC_('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy'));
        {"time":2015-08-26 00:00:00}
   """,
+  group = "json_funcs",
   since = "2.2.0")
 // scalastyle:on line.size.limit
 case class JsonToStructs(
@@ -627,6 +630,7 @@ case class JsonToStructs(
       > SELECT _FUNC_(array((map('a', 1))));
        [{"a":1}]
   """,
+  group = "json_funcs",
   since = "2.2.0")
 // scalastyle:on line.size.limit
 case class StructsToJson(
@@ -736,6 +740,7 @@ case class StructsToJson(
       > SELECT _FUNC_('[{"col":01}]', map('allowNumericLeadingZeros', 'true'));
        array<struct<col:bigint>>
   """,
+  group = "json_funcs",
   since = "2.4.0")
 case class SchemaOfJson(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
index 9070a1ab0b059..c8b6433207355 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala
@@ -426,7 +426,8 @@ abstract class OffsetWindowFunction
       * default - a string expression which is to use when the offset is larger than the window.
           The default value is null.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class Lead(input: Expression, offset: Expression, default: Expression)
     extends OffsetWindowFunction {
 
@@ -459,7 +460,8 @@ case class Lead(input: Expression, offset: Expression, default: Expression)
       * offset - an int expression which is rows to jump back in the partition.
       * default - a string expression which is to use when the offset row does not exist.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class Lag(input: Expression, offset: Expression, default: Expression)
     extends OffsetWindowFunction {
 
@@ -517,7 +519,8 @@ object SizeBasedWindowFunction {
     _FUNC_() - Assigns a unique, sequential number to each row, starting with one,
       according to the ordering of rows within the window partition.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class RowNumber() extends RowNumberLike {
   override val evaluateExpression = rowNumber
   override def prettyName: String = "row_number"
@@ -535,7 +538,8 @@ case class RowNumber() extends RowNumberLike {
   usage = """
     _FUNC_() - Computes the position of a value relative to all values in the partition.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
   override def dataType: DataType = DoubleType
   // The frame for CUME_DIST is Range based instead of Row based, because CUME_DIST must
@@ -574,7 +578,8 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction {
       * buckets - an int expression which is number of buckets to divide the rows in.
           Default value is 1.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction {
   def this() = this(Literal(1))
 
@@ -700,7 +705,8 @@ abstract class RankLike extends AggregateWindowFunction {
           trigger a change in rank. This is an internal parameter and will be assigned by the
           Analyser.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class Rank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): Rank = Rank(order)
@@ -725,7 +731,8 @@ case class Rank(children: Seq[Expression]) extends RankLike {
           trigger a change in rank. This is an internal parameter and will be assigned by the
           Analyser.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class DenseRank(children: Seq[Expression]) extends RankLike {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): DenseRank = DenseRank(order)
@@ -756,7 +763,8 @@ case class DenseRank(children: Seq[Expression]) extends RankLike {
           trigger a change in rank. This is an internal parameter and will be assigned by the
           Analyser.
   """,
-  since = "2.0.0")
+  since = "2.0.0",
+  group = "window_funcs")
 case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction {
   def this() = this(Nil)
   override def withOrder(order: Seq[Expression]): PercentRank = PercentRank(order)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
index 2d48f8df23e65..d9c90c7dbd085 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
@@ -345,6 +345,7 @@ object MyExtensions {
       """
        note
       """,
+      "",
       "3.0.0",
       """
        deprecated
@@ -755,6 +756,7 @@ object MyExtensions2 {
       """
        note
       """,
+      "",
       "3.0.0",
       """
        deprecated
@@ -787,6 +789,7 @@ object MyExtensions2Duplicate {
       """
        note
       """,
+      "",
       "3.0.0",
       """
        deprecated
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index 08f41f6819a0e..92ea0ce246d4e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -21,6 +21,7 @@ import java.math.BigDecimal
 
 import org.apache.spark.sql.api.java._
 import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.{QueryExecution, SimpleMode}
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
@@ -544,6 +545,24 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     assert(info.getExtended.contains("> SELECT upper('SparkSql');"))
   }
 
+  test("group info in ExpressionInfo") {
+    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("sum"))
+    assert(info.getGroup === "agg_funcs")
+
+    Seq("agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs", "window_funcs")
+        .foreach { groupName =>
+      val info = new ExpressionInfo(
+        "testClass", null, "testName", null, "", "", "", groupName, "", "")
+      assert(info.getGroup === groupName)
+    }
+
+    val errMsg = intercept[IllegalArgumentException] {
+      val invalidGroupName = "invalid_group_funcs"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", invalidGroupName, "", "")
+    }.getMessage
+    assert(errMsg.contains("'group' is malformed in the expression [testName]."))
+  }
+
   test("SPARK-28521 error message for CAST(parameter types contains DataType)") {
     val e = intercept[AnalysisException] {
       spark.sql("SELECT CAST(1)")
diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 44aa877332fd5..6614c714e90c7 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -48,6 +48,9 @@ echo "Generating SQL API Markdown files."
 echo "Generating SQL configuration table HTML file."
 "$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
 
+echo "Generating HTML files for SQL function table and examples."
+"$SPARK_HOME/bin/spark-submit" gen-sql-functions-docs.py
+
 echo "Generating HTML files for SQL API documentation."
 mkdocs build --clean
 rm -fr docs
diff --git a/sql/gen-sql-api-docs.py b/sql/gen-sql-api-docs.py
index 4feee7ad52570..3095a51e82d76 100644
--- a/sql/gen-sql-api-docs.py
+++ b/sql/gen-sql-api-docs.py
@@ -20,6 +20,7 @@
 
 from pyspark.java_gateway import launch_gateway
 
+
 ExpressionInfo = namedtuple(
     "ExpressionInfo", "className name usage arguments examples note since deprecated")
 
@@ -159,7 +160,7 @@ def _make_pretty_deprecated(deprecated):
         return "**Deprecated:**\n%s\n" % deprecated
 
 
-def generate_sql_markdown(jvm, path):
+def generate_sql_api_markdown(jvm, path):
     """
     Generates a markdown file after listing the function information. The output file
     is created in `path`.
@@ -223,4 +224,4 @@ def generate_sql_markdown(jvm, path):
     jvm = launch_gateway().jvm
     spark_root_dir = os.path.dirname(os.path.dirname(__file__))
     markdown_file_path = os.path.join(spark_root_dir, "sql/docs/index.md")
-    generate_sql_markdown(jvm, markdown_file_path)
+    generate_sql_api_markdown(jvm, markdown_file_path)
diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py
index 98212ad373370..0043c412fbc16 100644
--- a/sql/gen-sql-config-docs.py
+++ b/sql/gen-sql-config-docs.py
@@ -22,8 +22,10 @@
 
 # To avoid adding a new direct dependency, we import markdown from within mkdocs.
 from mkdocs.structure.pages import markdown
+
 from pyspark.java_gateway import launch_gateway
 
+
 SQLConfEntry = namedtuple(
     "SQLConfEntry", ["name", "default", "description", "version"])
 
@@ -41,7 +43,7 @@ def get_public_sql_configs(jvm):
     return sql_configs
 
 
-def generate_sql_configs_table(sql_configs, path):
+def generate_sql_configs_table_html(sql_configs, path):
     """
     Generates an HTML table at `path` that lists all public SQL
     configuration options.
@@ -116,6 +118,7 @@ def generate_sql_configs_table(sql_configs, path):
     sql_configs = get_public_sql_configs(jvm)
 
     spark_root_dir = os.path.dirname(os.path.dirname(__file__))
-    sql_configs_table_path = os.path.join(spark_root_dir, "docs/sql-configs.html")
+    sql_configs_table_path = os.path.join(
+        spark_root_dir, "docs/generated-sql-configuration-table.html")
 
-    generate_sql_configs_table(sql_configs, path=sql_configs_table_path)
+    generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)
diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py
new file mode 100644
index 0000000000000..7f0b2ae582f56
--- /dev/null
+++ b/sql/gen-sql-functions-docs.py
@@ -0,0 +1,228 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import itertools
+import os
+import re
+from collections import namedtuple
+
+# To avoid adding a new direct dependency, we import markdown from within mkdocs.
+from mkdocs.structure.pages import markdown
+
+from pyspark.java_gateway import launch_gateway
+
+
+ExpressionInfo = namedtuple("ExpressionInfo", "name usage examples group")
+
+groups = {
+    "agg_funcs", "array_funcs", "datetime_funcs",
+    "json_funcs", "map_funcs", "window_funcs",
+}
+
+
+def _list_grouped_function_infos(jvm):
+    """
+    Returns a list of function information grouped by each group value via JVM.
+    Sorts wrapped expression infos in each group by name and returns them.
+    """
+
+    jinfos = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listBuiltinFunctionInfos()
+    infos = []
+
+    for jinfo in filter(lambda x: x.getGroup() in groups, jinfos):
+        name = jinfo.getName()
+        usage = jinfo.getUsage()
+        usage = usage.replace("_FUNC_", name) if usage is not None else usage
+        infos.append(ExpressionInfo(
+            name=name,
+            usage=usage,
+            examples=jinfo.getExamples().replace("_FUNC_", name),
+            group=jinfo.getGroup()))
+
+    # Groups expression info by each group value
+    grouped_infos = itertools.groupby(sorted(infos, key=lambda x: x.group), key=lambda x: x.group)
+    # Then, sort expression infos in each group by name
+    return [(k, sorted(g, key=lambda x: x.name)) for k, g in grouped_infos]
+
+
+# TODO(SPARK-31499): Needs to add a column to describe arguments and their types
+def _make_pretty_usage(infos):
+    """
+    Makes the usage description pretty and returns a formatted string.
+
+    Expected input:
+
+        func(*) - ...
+
+        func(expr[, expr...]) - ...
+
+    Expected output:
+    <table class="table">
+      <thead>
+        <tr>
+          <th style="width:25%">Function</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>func(*)</td>
+          <td>...</td>
+        </tr>
+        <tr>
+          <td>func(expr[, expr...])</td>
+          <td>...</td>
+        </tr>
+      </tbody>
+      ...
+    </table>
+
+    """
+
+    result = []
+    result.append("<table class=\"table\">")
+    result.append("  <thead>")
+    result.append("    <tr>")
+    result.append("      <th style=\"width:25%\">Function</th>")
+    result.append("      <th>Description</th>")
+    result.append("    </tr>")
+    result.append("  </thead>")
+    result.append("  <tbody>")
+
+    for info in infos:
+        # Extracts (signature, description) pairs from `info.usage`, e.g.,
+        # the signature is `func(expr)` and the description is `...` in an usage `func(expr) - ...`.
+        usages = iter(re.split(r"(%s\(.*\)) - " % info.name, info.usage.strip())[1:])
+        for (sig, description) in zip(usages, usages):
+            result.append("    <tr>")
+            result.append("      <td>%s</td>" % sig)
+            result.append("      <td>%s</td>" % description.strip())
+            result.append("    </tr>")
+
+    result.append("  </tbody>")
+    result.append("</table>\n")
+    return "\n".join(result)
+
+
+def _make_pretty_examples(jspark, infos):
+    """
+    Makes the examples description pretty and returns a formatted string if `infos`
+    has any `examples` starting with the example prefix. Otherwise, returns None.
+
+    Expected input:
+
+        Examples:
+          > SELECT func(col)...;
+           ...
+          > SELECT func(col)...;
+           ...
+
+    Expected output:
+    <div class="codehilite"><pre><span></span>
+      <span class="c1">-- func</span>
+      <span class="k">SELECT</span>
+      ...
+    </pre></div>
+    ```
+
+    """
+
+    pretty_output = ""
+    for info in infos:
+        if info.examples.startswith("\n    Examples:"):
+            output = []
+            output.append("-- %s" % info.name)
+            query_examples = filter(lambda x: x.startswith("      > "), info.examples.split("\n"))
+            for query_example in query_examples:
+                query = query_example.lstrip("      > ")
+                print("    %s" % query)
+                query_output = jspark.sql(query).showString(20, 20, False)
+                output.append(query)
+                output.append(query_output)
+            pretty_output += "\n" + "\n".join(output)
+    if pretty_output != "":
+        return markdown.markdown(
+            "```sql%s```" % pretty_output, extensions=['codehilite', 'fenced_code'])
+
+
+def generate_functions_table_html(jvm, html_output_dir):
+    """
+    Generates a HTML file after listing the function information. The output file
+    is created under `html_output_dir`.
+
+    Expected output:
+
+    <table class="table">
+      <thead>
+        <tr>
+          <th style="width:25%">Function</th>
+          <th>Description</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>func(*)</td>
+          <td>...</td>
+        </tr>
+        <tr>
+          <td>func(expr[, expr...])</td>
+          <td>...</td>
+        </tr>
+      </tbody>
+      ...
+    </table>
+
+    """
+    for key, infos in _list_grouped_function_infos(jvm):
+        function_table = _make_pretty_usage(infos)
+        key = key.replace("_", "-")
+        with open("%s/generated-%s-table.html" % (html_output_dir, key), 'w') as table_html:
+            table_html.write(function_table)
+
+
+def generate_functions_examples_html(jvm, jspark, html_output_dir):
+    """
+    Generates a HTML file after listing and executing the function information.
+    The output file is created under `html_output_dir`.
+
+    Expected output:
+
+    <div class="codehilite"><pre><span></span>
+      <span class="c1">-- func</span>
+      <span class="k">SELECT</span>
+      ...
+    </pre></div>
+
+    """
+    print("Running SQL examples to generate formatted output.")
+    for key, infos in _list_grouped_function_infos(jvm):
+        examples = _make_pretty_examples(jspark, infos)
+        key = key.replace("_", "-")
+        if examples is not None:
+            with open("%s/generated-%s-examples.html" % (
+                    html_output_dir, key), 'w') as examples_html:
+                examples_html.write(examples)
+
+
+if __name__ == "__main__":
+    jvm = launch_gateway().jvm
+    jspark = jvm.org.apache.spark.sql.SparkSession.builder().getOrCreate()
+    jspark.sparkContext().setLogLevel("ERROR")  # Make it less noisy.
+    spark_root_dir = os.path.dirname(os.path.dirname(__file__))
+    html_output_dir = os.path.join(spark_root_dir, "docs")
+    generate_functions_table_html(jvm, html_output_dir)
+    generate_functions_examples_html(jvm, jspark, html_output_dir)

From ad2fa2a2d48f12aed741c45db0536ee49fed0275 Mon Sep 17 00:00:00 2001
From: Maryann Xue <maryann.xue@gmail.com>
Date: Tue, 21 Apr 2020 03:56:42 +0000
Subject: [PATCH 0528/1280] [SPARK-31501][SQL] AQE update UI should not cause
 deadlock

### What changes were proposed in this pull request?

This PR makes sure that AQE does not call update UI if the current execution ID does not match the current query. This PR also includes a minor refactoring that moves `getOrCloneSessionWithAqeOff` from `QueryExecution` to `AdaptiveSparkPlanHelper` since that function is not used by `QueryExecution` any more.

### Why are the changes needed?

Without this fix, there could be a potential deadlock.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added UT.

Closes #28275 from maryannxue/aqe-ui-deadlock.

Authored-by: Maryann Xue <maryann.xue@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ae29cf24fc6d27335be411da53bc18a867bf098c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/CacheManager.scala    |  7 ++--
 .../spark/sql/execution/QueryExecution.scala  | 36 ++++++-------------
 .../adaptive/AdaptiveSparkPlanExec.scala      | 11 +++---
 .../adaptive/AdaptiveSparkPlanHelper.scala    | 18 +++++++++-
 .../adaptive/AdaptiveQueryExecSuite.scala     | 30 +++++++++++++++-
 5 files changed, 66 insertions(+), 36 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index ad33ce5ef60ff..52cec8b202885 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, SubqueryExpression}
 import org.apache.spark.sql.catalyst.optimizer.EliminateResolvedHint
 import org.apache.spark.sql.catalyst.plans.logical.{IgnoreCachedData, LogicalPlan, ResolvedHint}
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
 import org.apache.spark.sql.execution.command.CommandUtils
 import org.apache.spark.sql.execution.datasources.{FileIndex, HadoopFsRelation, LogicalRelation}
@@ -45,7 +46,7 @@ case class CachedData(plan: LogicalPlan, cachedRepresentation: InMemoryRelation)
  *
  * Internal to Spark SQL.
  */
-class CacheManager extends Logging {
+class CacheManager extends Logging with AdaptiveSparkPlanHelper {
 
   /**
    * Maintains the list of cached plans as an immutable sequence.  Any updates to the list
@@ -80,7 +81,7 @@ class CacheManager extends Logging {
       logWarning("Asked to cache already cached data.")
     } else {
       // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
-      val sessionWithAqeOff = QueryExecution.getOrCloneSessionWithAqeOff(query.sparkSession)
+      val sessionWithAqeOff = getOrCloneSessionWithAqeOff(query.sparkSession)
       val inMemoryRelation = sessionWithAqeOff.withActive {
         val qe = sessionWithAqeOff.sessionState.executePlan(planToCache)
         InMemoryRelation(
@@ -191,7 +192,7 @@ class CacheManager extends Logging {
     needToRecache.map { cd =>
       cd.cachedRepresentation.cacheBuilder.clearCache()
       // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
-      val sessionWithAqeOff = QueryExecution.getOrCloneSessionWithAqeOff(spark)
+      val sessionWithAqeOff = getOrCloneSessionWithAqeOff(spark)
       val newCache = sessionWithAqeOff.withActive {
         val qe = sessionWithAqeOff.sessionState.executePlan(cd.plan)
         InMemoryRelation(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 20bc289c65586..99bc45fa9e9e8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -112,7 +112,8 @@ class QueryExecution(
   def observedMetrics: Map[String, Row] = CollectMetricsExec.collect(executedPlan)
 
   protected def preparations: Seq[Rule[SparkPlan]] = {
-    QueryExecution.preparations(sparkSession)
+    QueryExecution.preparations(sparkSession,
+      Option(InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession, this))))
   }
 
   private def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
@@ -274,18 +275,15 @@ object QueryExecution {
    * are correct, insert whole stage code gen, and try to reduce the work done by reusing exchanges
    * and subqueries.
    */
-  private[execution] def preparations(sparkSession: SparkSession): Seq[Rule[SparkPlan]] = {
-
-    val sparkSessionWithAqeOff = getOrCloneSessionWithAqeOff(sparkSession)
-
+  private[execution] def preparations(
+      sparkSession: SparkSession,
+      adaptiveExecutionRule: Option[InsertAdaptiveSparkPlan] = None): Seq[Rule[SparkPlan]] = {
+    // `AdaptiveSparkPlanExec` is a leaf node. If inserted, all the following rules will be no-op
+    // as the original plan is hidden behind `AdaptiveSparkPlanExec`.
+    adaptiveExecutionRule.toSeq ++
     Seq(
-      // `AdaptiveSparkPlanExec` is a leaf node. If inserted, all the following rules will be no-op
-      // as the original plan is hidden behind `AdaptiveSparkPlanExec`.
-      InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession)),
-      // If the following rules apply, it means the main query is not AQE-ed, so we make sure the
-      // subqueries are not AQE-ed either.
-      PlanDynamicPruningFilters(sparkSessionWithAqeOff),
-      PlanSubqueries(sparkSessionWithAqeOff),
+      PlanDynamicPruningFilters(sparkSession),
+      PlanSubqueries(sparkSession),
       EnsureRequirements(sparkSession.sessionState.conf),
       ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
         sparkSession.sessionState.columnarRules),
@@ -334,18 +332,4 @@ object QueryExecution {
     val sparkPlan = createSparkPlan(spark, spark.sessionState.planner, plan.clone())
     prepareExecutedPlan(spark, sparkPlan)
   }
-
-  /**
-   * Returns a cloned [[SparkSession]] with adaptive execution disabled, or the original
-   * [[SparkSession]] if its adaptive execution is already disabled.
-   */
-  def getOrCloneSessionWithAqeOff[T](session: SparkSession): SparkSession = {
-    if (!session.sessionState.conf.adaptiveExecutionEnabled) {
-      session
-    } else {
-      val newSession = session.cloneSession()
-      newSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
-      newSession
-    }
-  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 0ec8b5f889c58..d73540cb65da9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -145,10 +145,11 @@ case class AdaptiveSparkPlanExec(
     // `plan.queryExecution.rdd`, we need to set active session here as new plan nodes can be
     // created in the middle of the execution.
     context.session.withActive {
-      // Subqueries do not have their own execution IDs and therefore rely on the main query to
-      // update UI.
-      val executionId = Option(context.session.sparkContext.getLocalProperty(
-        SQLExecution.EXECUTION_ID_KEY)).map(_.toLong)
+      // If the `QueryExecution` does not match the current execution ID, it means the execution ID
+      // belongs to another (parent) query, and we should not call update UI in this query.
+      val executionId =
+        Option(context.session.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY))
+          .map(_.toLong).filter(SQLExecution.getQueryExecution(_) eq context.qe)
       var currentLogicalPlan = currentPhysicalPlan.logicalLink.get
       var result = createQueryStages(currentPhysicalPlan)
       val events = new LinkedBlockingQueue[StageMaterializationEvent]()
@@ -572,7 +573,7 @@ object AdaptiveSparkPlanExec {
 /**
  * The execution context shared between the main query and all sub-queries.
  */
-case class AdaptiveExecutionContext(session: SparkSession) {
+case class AdaptiveExecutionContext(session: SparkSession, qe: QueryExecution) {
 
   /**
    * The subquery-reuse map shared across the entire query.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
index cd87230c593f9..3cf6a13a4a892 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanHelper.scala
@@ -17,7 +17,9 @@
 
 package org.apache.spark.sql.execution.adaptive
 
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.internal.SQLConf
 
 /**
  * This class provides utility methods related to tree traversal of an [[AdaptiveSparkPlanExec]]
@@ -135,4 +137,18 @@ trait AdaptiveSparkPlanHelper {
     case a: AdaptiveSparkPlanExec => a.executedPlan
     case other => other
   }
- }
+
+  /**
+   * Returns a cloned [[SparkSession]] with adaptive execution disabled, or the original
+   * [[SparkSession]] if its adaptive execution is already disabled.
+   */
+  def getOrCloneSessionWithAqeOff[T](session: SparkSession): SparkSession = {
+    if (!session.sessionState.conf.adaptiveExecutionEnabled) {
+      session
+    } else {
+      val newSession = session.cloneSession()
+      newSession.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      newSession
+    }
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 6da510f5f90ba..82258633e604b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -23,7 +23,8 @@ import java.net.URI
 import org.apache.log4j.Level
 
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
-import org.apache.spark.sql.{QueryTest, Row, SparkSession}
+import org.apache.spark.sql.{QueryTest, Row, SparkSession, Strategy}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
 import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.command.DataWritingCommandExec
@@ -816,4 +817,31 @@ class AdaptiveQueryExecSuite
       SparkSession.setActiveSession(spark) // recover the active session.
     }
   }
+
+  test("No deadlock in UI update") {
+    object TestStrategy extends Strategy {
+      def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+        case _: Aggregate =>
+          withSQLConf(
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+            SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+            spark.range(5).rdd
+          }
+          Nil
+        case _ => Nil
+      }
+    }
+
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      try {
+        spark.experimental.extraStrategies = TestStrategy :: Nil
+        val df = spark.range(10).groupBy('id).count()
+        df.collect()
+      } finally {
+        spark.experimental.extraStrategies = Nil
+      }
+    }
+  }
 }

From 2c4765c3477a7e3d880b0ffbe83a9bb976b4b244 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 21 Apr 2020 11:55:33 +0000
Subject: [PATCH 0529/1280] [SPARK-31474][SQL] Consistency between
 dayofweek/dow in extract exprsession and dayofweek function

### What changes were proposed in this pull request?
```sql
spark-sql> SELECT extract(dayofweek from '2009-07-26');
1
spark-sql> SELECT extract(dow from '2009-07-26');
0
spark-sql> SELECT extract(isodow from '2009-07-26');
7
spark-sql> SELECT dayofweek('2009-07-26');
1
spark-sql> SELECT weekday('2009-07-26');
6
```
Currently, there are 4 types of day-of-week range:
1. the function `dayofweek`(2.3.0) and extracting `dayofweek`(2.4.0) result as of Sunday(1) to Saturday(7)
2. extracting `dow`(3.0.0) results as of Sunday(0) to Saturday(6)
3. extracting` isodow` (3.0.0) results as of Monday(1) to Sunday(7)
4. the function `weekday`(2.4.0) results as of Monday(0) to Sunday(6)

Actually, extracting `dayofweek` and `dow` are both derived from PostgreSQL but have different meanings.
https://issues.apache.org/jira/browse/SPARK-23903
https://issues.apache.org/jira/browse/SPARK-28623

In this PR, we make extracting `dow` as same as extracting `dayofweek` and the `dayofweek` function for historical reason and not breaking anything.

Also, add more documentation to the extracting function to make extract field more clear to understand.

### Why are the changes needed?

Consistency insurance

### Does this PR introduce any user-facing change?

yes, doc updated and extract `dow` is as same as `dayofweek`

### How was this patch tested?

1. modified ut
2. local SQL doc verification
#### before
![image](https://user-images.githubusercontent.com/8326978/79601949-3535b100-811c-11ea-957b-a33d68641181.png)

#### after
![image](https://user-images.githubusercontent.com/8326978/79601847-12a39800-811c-11ea-8ff6-aa329255d099.png)

Closes #28248 from yaooqinn/SPARK-31474.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 19854371104451b5c4cb266aaee6b33a5049b145)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/datetimeExpressions.scala     | 74 ++++++++++---------
 .../sql-tests/results/date_part.sql.out       |  2 +-
 .../sql-tests/results/extract.sql.out         |  2 +-
 .../results/postgreSQL/timestamp.sql.out      | 10 +--
 4 files changed, 47 insertions(+), 41 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 0160827fb20d3..a36c1f23934ea 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -17,10 +17,9 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
-import java.sql.Timestamp
 import java.time.{DateTimeException, LocalDate, LocalDateTime, ZoneId}
 import java.time.temporal.IsoFields
-import java.util.{Locale, TimeZone}
+import java.util.Locale
 
 import scala.util.control.NonFatal
 
@@ -2118,8 +2117,7 @@ object DatePart {
     case "MONTH" | "MON" | "MONS" | "MONTHS" => Month(source)
     case "WEEK" | "W" | "WEEKS" => WeekOfYear(source)
     case "DAY" | "D" | "DAYS" => DayOfMonth(source)
-    case "DAYOFWEEK" => DayOfWeek(source)
-    case "DOW" => Subtract(DayOfWeek(source), Literal(1))
+    case "DAYOFWEEK" | "DOW" => DayOfWeek(source)
     case "ISODOW" => Add(WeekDay(source), Literal(1))
     case "DOY" => DayOfYear(source)
     case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source)
@@ -2159,38 +2157,12 @@ object DatePartLike {
   }
 }
 
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(field, source) - Extracts a part of the date/timestamp or interval source.",
   arguments = """
     Arguments:
-      * field - selects which part of the source should be extracted.
-               Supported string values of `field` for dates and timestamps are:
-                ["MILLENNIUM", ("MILLENNIA", "MIL", "MILS"),
-                 "CENTURY", ("CENTURIES", "C", "CENT"),
-                 "DECADE", ("DECADES", "DEC", "DECS"),
-                 "YEAR", ("Y", "YEARS", "YR", "YRS"),
-                 "ISOYEAR",
-                 "QUARTER", ("QTR"),
-                 "MONTH", ("MON", "MONS", "MONTHS"),
-                 "WEEK", ("W", "WEEKS"),
-                 "DAY", ("D", "DAYS"),
-                 "DAYOFWEEK",
-                 "DOW",
-                 "ISODOW",
-                 "DOY",
-                 "HOUR", ("H", "HOURS", "HR", "HRS"),
-                 "MINUTE", ("M", "MIN", "MINS", "MINUTES"),
-                 "SECOND", ("S", "SEC", "SECONDS", "SECS"),
-                 "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS"),
-                 "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US"),
-                 "EPOCH"]
-                Supported string values of `field` for intervals are:
-                 ["YEAR", ("Y", "YEARS", "YR", "YRS"),
-                  "MONTH", ("MON", "MONS", "MONTHS"),
-                  "DAY", ("D", "DAYS"),
-                  "HOUR", ("H", "HOURS", "HR", "HRS"),
-                  "MINUTE", ("M", "MIN", "MINS", "MINUTES"),
-                  "SECOND", ("S", "SEC", "SECONDS", "SECS")]
+      * field - selects which part of the source should be extracted, and supported string values are as same as the fields of the equivalent function `EXTRACT`.
       * source - a date/timestamp or interval column from where `field` should be extracted
   """,
   examples = """
@@ -2208,8 +2180,12 @@ object DatePartLike {
       > SELECT _FUNC_('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
        30.001001
   """,
+  note = """
+    The _FUNC_ function is equivalent to the SQL-standard function `EXTRACT(field FROM source)`
+  """,
   group = "datetime_funcs",
   since = "3.0.0")
+// scalastyle:on line.size.limit
 case class DatePart(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
 
@@ -2222,12 +2198,38 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
   override def prettyName: String = "date_part"
 }
 
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = "_FUNC_(field FROM source) - Extracts a part of the date/timestamp or interval source.",
   arguments = """
     Arguments:
-      * field - selects which part of the source should be extracted and supported string values
-                are the same with the `date_part` fields.
+      * field - selects which part of the source should be extracted
+          - Supported string values of `field` for dates and timestamps are:
+              - "MILLENNIUM", ("MILLENNIA", "MIL", "MILS") - the conventional numbering of millennia
+              - "CENTURY", ("CENTURIES", "C", "CENT") - the conventional numbering of centuries
+              - "DECADE", ("DECADES", "DEC", "DECS") - the year field divided by 10
+              - "YEAR", ("Y", "YEARS", "YR", "YRS") - the year field
+              - "ISOYEAR" - the ISO 8601 week-numbering year that the datetime falls in
+              - "QUARTER", ("QTR") - the quarter (1 - 4) of the year that the datetime falls in
+              - "MONTH", ("MON", "MONS", "MONTHS") - the month field (1 - 12)
+              - "WEEK", ("W", "WEEKS") - the number of the ISO 8601 week-of-week-based-year. A week is considered to start on a Monday and week 1 is the first week with >3 days. In the ISO week-numbering system, it is possible for early-January dates to be part of the 52nd or 53rd week of the previous year, and for late-December dates to be part of the first week of the next year. For example, 2005-01-02 is part of the 53rd week of year 2004, while 2012-12-31 is part of the first week of 2013
+              - "DAY", ("D", "DAYS") - the day of the month field (1 - 31)
+              - "DAYOFWEEK",("DOW") - the day of the week for datetime as Sunday(1) to Saturday(7)
+              - "ISODOW" - ISO 8601 based day of the week for datetime as Monday(1) to Sunday(7)
+              - "DOY" - the day of the year (1 - 365/366)
+              - "HOUR", ("H", "HOURS", "HR", "HRS") - The hour field (0 - 23)
+              - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - the minutes field (0 - 59)
+              - "SECOND", ("S", "SEC", "SECONDS", "SECS") - the seconds field, including fractional parts
+              - "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS") - the seconds field, including fractional parts, multiplied by 1000. Note that this includes full seconds
+              - "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US") - The seconds field, including fractional parts, multiplied by 1000000. Note that this includes full seconds
+              - "EPOCH" - the number of seconds with fractional part in microsecond precision since 1970-01-01 00:00:00 local time (can be negative)
+          - Supported string values of `field` for interval(which consists of `months`, `days`, `microseconds`) are:
+              - "YEAR", ("Y", "YEARS", "YR", "YRS") - the total `months` / 12
+              - "MONTH", ("MON", "MONS", "MONTHS") - the total `months` % 12
+              - "DAY", ("D", "DAYS") - the `days` part of interval
+              - "HOUR", ("H", "HOURS", "HR", "HRS") - how many hours the `microseconds` contains
+              - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - how many minutes left after taking hours from `microseconds`
+              - "SECOND", ("S", "SEC", "SECONDS", "SECS") - how many second with fractions left after taking hours and minutes from `microseconds`
       * source - a date/timestamp or interval column from where `field` should be extracted
   """,
   examples = """
@@ -2245,7 +2247,11 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
       > SELECT _FUNC_(seconds FROM interval 5 hours 30 seconds 1 milliseconds 1 microseconds);
        30.001001
   """,
+  note = """
+    The _FUNC_ function is equivalent to `date_part(field, source)`.
+  """,
   since = "3.0.0")
+// scalastyle:on line.size.limit
 case class Extract(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
 
diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
index 702ac17f7a4e5..028448b4e3a3c 100644
--- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
@@ -255,7 +255,7 @@ select date_part('dow', c) from t
 -- !query schema
 struct<date_part('dow', t.`c`):int>
 -- !query output
-5
+6
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 47cd5180ddebd..e6635e608651e 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -263,7 +263,7 @@ select extract(dow from c) from t
 -- !query schema
 struct<extract('dow' FROM t.`c`):int>
 -- !query output
-5
+6
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
index 75ea3f3c42932..abfce9180ab39 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
@@ -288,11 +288,11 @@ SELECT '' AS `54`, d1 as `timestamp`,
 -- !query schema
 struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int>
 -- !query output
-	1969-12-31 16:00:00	1970	1	3
-	1997-01-02 00:00:00	1997	1	4
-	1997-01-02 03:04:05	1997	1	4
-	1997-02-10 17:32:01	1997	7	1
-	2001-09-22 18:19:20	2001	38	6
+	1969-12-31 16:00:00	1970	1	4
+	1997-01-02 00:00:00	1997	1	5
+	1997-01-02 03:04:05	1997	1	5
+	1997-02-10 17:32:01	1997	7	2
+	2001-09-22 18:19:20	2001	38	7
 
 
 -- !query

From 756e85e781374e2d14284b7df5f0112bb8c5d6d2 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 21 Apr 2020 12:33:58 +0000
Subject: [PATCH 0530/1280] [SPARK-31504][SQL] Formatted Explain should have
 determined order of Output fields

### What changes were proposed in this pull request?

In `verboseStringWithOperatorId`, use `output` (it's `Seq[Attribute]`) instead of `producedAttributes` (it's `AttributeSet`) to generates `"Output"` for the leaf node in order to make `"Output"` determined.

### Why are the changes needed?

Currently, Formatted Explain use `producedAttributes`, the `AttributeSet`,  to generate `"Output"`. As a result, the fields order within `"Output"` can be different from time to time. It's That means, for the same plan, it could have different explain outputs.

### Does this PR introduce any user-facing change?

Yes, user see the determined fields order within formatted explain now.

### How was this patch tested?

Added a regression test.

Closes #28282 from Ngone51/fix_output.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 55b026a783ce3a5aced1f396e5dd03f0cab9356b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/DataSourceScanExec.scala       |  4 ++--
 .../org/apache/spark/sql/execution/SparkPlan.scala     |  2 +-
 .../test/scala/org/apache/spark/sql/ExplainSuite.scala | 10 ++++++++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 8d488d4448506..bd0e1d08f60e6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -76,7 +76,7 @@ trait DataSourceScanExec extends LeafExecNode {
 
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |${ExplainUtils.generateFieldString("Output", producedAttributes)}
+       |${ExplainUtils.generateFieldString("Output", output)}
        |${metadataStr.mkString("\n")}
      """.stripMargin
   }
@@ -378,7 +378,7 @@ case class FileSourceScanExec(
 
     s"""
        |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
-       |${ExplainUtils.generateFieldString("Output", producedAttributes)}
+       |${ExplainUtils.generateFieldString("Output", output)}
        |${metadataStr.mkString("\n")}
      """.stripMargin
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index f5bb554682eab..e1a6495530a30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -515,7 +515,7 @@ trait LeafExecNode extends SparkPlan {
   override def verboseStringWithOperatorId(): String = {
     val argumentString = argString(SQLConf.get.maxToStringFields)
     val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
-    val outputStr = s"${ExplainUtils.generateFieldString("Output", producedAttributes)}"
+    val outputStr = s"${ExplainUtils.generateFieldString("Output", output)}"
 
     if (argumentString.nonEmpty) {
       s"""
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index 1a35e5b729c00..b2047091b1576 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -330,6 +330,16 @@ class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiv
     }.getMessage
     assert(errMsg.contains("Unknown explain mode: unknown"))
   }
+
+  test("SPARK-31504: Output fields in formatted Explain should have determined order") {
+    withTempPath { path =>
+      spark.range(10).selectExpr("id as a", "id as b", "id as c", "id as d", "id as e")
+        .write.mode("overwrite").parquet(path.getAbsolutePath)
+      val df1 = spark.read.parquet(path.getAbsolutePath)
+      val df2 = spark.read.parquet(path.getAbsolutePath)
+      assert(getNormalizedExplain(df1, FormattedMode) === getNormalizedExplain(df2, FormattedMode))
+    }
+  }
 }
 
 case class ExplainSingleData(id: Int)

From 5a70fa8f09bae59b3ad900b9ece8fbc455425b7f Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 22 Apr 2020 00:26:23 +0900
Subject: [PATCH 0531/1280] [SPARK-31361][SQL] Rebase datetime in parquet/avro
 according to file metadata

### What changes were proposed in this pull request?

This PR adds a new parquet/avro file metadata: `org.apache.spark.legacyDatetime`. It indicates that the file was written with the "rebaseInWrite" config enabled, and spark need to do rebase when reading it.

This makes Spark be able to do rebase more smartly:
1. If we don't know which Spark version writes the file, do rebase if the "rebaseInRead" config is true.
2. If the file was written by Spark 2.4 and earlier, then do rebase.
3. If the file was written by Spark 3.0 and later, do rebase if the `org.apache.spark.legacyDatetime` exists in file metadata.

### Why are the changes needed?

It's very easy to have mixed-calendar parquet/avro files: e.g. A user upgrades to Spark 3.0 and writes some parquet files to an existing directory. Then he realizes that the directory contains legacy datetime values before 1582. However, it's too late and he has to find out all the legacy files manually and read them separately.

To support mixed-calendar parquet/avro files, we need to decide to rebase or not based on the file metadata.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Updated test

Closes #28137 from cloud-fan/datetime.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a5ebbacf538cd78f3edc81542a8514c60d340109)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/avro/AvroDeserializer.scala     |  12 +-
 .../spark/sql/avro/AvroFileFormat.scala       |  11 +-
 .../spark/sql/avro/AvroOutputWriter.scala     |  17 ++-
 .../spark/sql/avro/AvroSerializer.scala       |  14 ++-
 .../v2/avro/AvroPartitionReaderFactory.scala  |  10 +-
 .../AvroCatalystDataConversionSuite.scala     |   2 +-
 .../org/apache/spark/sql/avro/AvroSuite.scala | 103 ++++++++++++++----
 .../parquet/VectorizedColumnReader.java       |   6 +-
 .../VectorizedParquetRecordReader.java        |  18 ++-
 .../datasources/DataSourceUtils.scala         |  17 +++
 .../parquet/ParquetFileFormat.scala           |  13 ++-
 .../parquet/ParquetReadSupport.scala          |  15 ++-
 .../parquet/ParquetRecordMaterializer.scala   |  10 +-
 .../parquet/ParquetRowConverter.scala         |  10 +-
 .../parquet/ParquetWriteSupport.scala         |   6 +-
 .../ParquetPartitionReaderFactory.scala       |  26 +++--
 .../scala/org/apache/spark/sql/package.scala  |   6 +
 .../benchmark/DataSourceReadBenchmark.scala   |   6 +-
 .../parquet/ParquetEncodingSuite.scala        |   6 +-
 .../datasources/parquet/ParquetIOSuite.scala  |  89 +++++++++++----
 .../datasources/parquet/ParquetTest.scala     |   7 +-
 21 files changed, 299 insertions(+), 105 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index ce8f9762751a0..27206edb287f9 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -41,12 +41,14 @@ import org.apache.spark.unsafe.types.UTF8String
 /**
  * A deserializer to deserialize data in avro format to data in catalyst format.
  */
-class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType) {
-  private lazy val decimalConversions = new DecimalConversion()
+class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType, rebaseDateTime: Boolean) {
+
+  def this(rootAvroType: Schema, rootCatalystType: DataType) {
+    this(rootAvroType, rootCatalystType,
+      SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ))
+  }
 
-  // Enable rebasing date/timestamp from Julian to Proleptic Gregorian calendar
-  private val rebaseDateTime =
-    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ)
+  private lazy val decimalConversions = new DecimalConversion()
 
   private val converter: Any => Any = rootCatalystType match {
     // A shortcut for empty schema.
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
index 123669ba1376a..e69c95b797c73 100755
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
@@ -34,7 +34,8 @@ import org.apache.spark.TaskContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.execution.datasources.{FileFormat, OutputWriterFactory, PartitionedFile}
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, FileFormat, OutputWriterFactory, PartitionedFile}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.{DataSourceRegister, Filter}
 import org.apache.spark.sql.types._
 import org.apache.spark.util.SerializableConfiguration
@@ -123,8 +124,12 @@ private[sql] class AvroFileFormat extends FileFormat
         reader.sync(file.start)
         val stop = file.start + file.length
 
-        val deserializer =
-          new AvroDeserializer(userProvidedSchema.getOrElse(reader.getSchema), requiredSchema)
+        val rebaseDateTime = DataSourceUtils.needRebaseDateTime(
+          reader.asInstanceOf[DataFileReader[_]].getMetaString).getOrElse {
+          SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ)
+        }
+        val deserializer = new AvroDeserializer(
+          userProvidedSchema.getOrElse(reader.getSchema), requiredSchema, rebaseDateTime)
 
         new Iterator[InternalRow] {
           private[this] var completed = false
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
index 2cfa3a4826ed8..82a568049990e 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
@@ -29,9 +29,10 @@ import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext}
 
 import org.apache.spark.SPARK_VERSION_SHORT
-import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriter
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 // NOTE: This class is instantiated and used on executor side only, no need to be serializable.
@@ -41,16 +42,24 @@ private[avro] class AvroOutputWriter(
     schema: StructType,
     avroSchema: Schema) extends OutputWriter {
 
+  // Whether to rebase datetimes from Gregorian to Julian calendar in write
+  private val rebaseDateTime: Boolean =
+    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE)
+
   // The input rows will never be null.
-  private lazy val serializer = new AvroSerializer(schema, avroSchema, nullable = false)
+  private lazy val serializer =
+    new AvroSerializer(schema, avroSchema, nullable = false, rebaseDateTime)
 
   /**
    * Overrides the couple of methods responsible for generating the output streams / files so
    * that the data can be correctly partitioned
    */
   private val recordWriter: RecordWriter[AvroKey[GenericRecord], NullWritable] = {
-    val sparkVersion = Map(SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT).asJava
-    new SparkAvroKeyOutputFormat(sparkVersion) {
+    val fileMeta = Map(SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT) ++ {
+      if (rebaseDateTime) Some(SPARK_LEGACY_DATETIME -> "") else None
+    }
+
+    new SparkAvroKeyOutputFormat(fileMeta.asJava) {
 
       override def getDefaultWorkFile(context: TaskAttemptContext, extension: String): Path = {
         new Path(path)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index 350d7a3005316..dc232168fd241 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -42,12 +42,16 @@ import org.apache.spark.sql.types._
 /**
  * A serializer to serialize data in catalyst format to data in avro format.
  */
-class AvroSerializer(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean)
-  extends Logging {
+class AvroSerializer(
+    rootCatalystType: DataType,
+    rootAvroType: Schema,
+    nullable: Boolean,
+    rebaseDateTime: Boolean) extends Logging {
 
-  // Whether to rebase datetimes from Gregorian to Julian calendar in write
-  private val rebaseDateTime: Boolean =
-    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE)
+  def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) {
+    this(rootCatalystType, rootAvroType, nullable,
+      SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE))
+  }
 
   def serialize(catalystData: Any): Any = {
     converter.apply(catalystData)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
index 8230dbaf8ea6c..712aec6acbd56 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
@@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.avro.{AvroDeserializer, AvroOptions}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.PartitionReader
-import org.apache.spark.sql.execution.datasources.PartitionedFile
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile}
 import org.apache.spark.sql.execution.datasources.v2.{EmptyPartitionReader, FilePartitionReaderFactory, PartitionReaderWithPartitionValues}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StructType
@@ -88,8 +88,12 @@ case class AvroPartitionReaderFactory(
       reader.sync(partitionedFile.start)
       val stop = partitionedFile.start + partitionedFile.length
 
-      val deserializer =
-        new AvroDeserializer(userProvidedSchema.getOrElse(reader.getSchema), readDataSchema)
+      val rebaseDateTime = DataSourceUtils.needRebaseDateTime(
+        reader.asInstanceOf[DataFileReader[_]].getMetaString).getOrElse {
+        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ)
+      }
+      val deserializer = new AvroDeserializer(
+        userProvidedSchema.getOrElse(reader.getSchema), readDataSchema, rebaseDateTime)
 
       val fileReader = new PartitionReader[InternalRow] {
         private[this] var completed = false
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
index c8a1f670bda9e..64d790bc4acd4 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
@@ -288,7 +288,7 @@ class AvroCatalystDataConversionSuite extends SparkFunSuite
       """.stripMargin
     val avroSchema = new Schema.Parser().parse(jsonFormatSchema)
     val dataType = SchemaConverters.toSqlType(avroSchema).dataType
-    val deserializer = new AvroDeserializer(avroSchema, dataType)
+    val deserializer = new AvroDeserializer(avroSchema, dataType, rebaseDateTime = false)
 
     def checkDeserialization(data: GenericData.Record, expected: Any): Unit = {
       assert(checkResult(
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index a5224fd104ea9..3e754f02911dc 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -84,9 +84,8 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }, new GenericDatumReader[Any]()).getSchema.toString(false)
   }
 
-  private def readResourceAvroFile(name: String): DataFrame = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.format("avro").load(url.toString)
+  private def getResourceAvroFilePath(name: String): String = {
+    Thread.currentThread().getContextClassLoader.getResource(name).toString
   }
 
   test("resolve avro data source") {
@@ -1530,16 +1529,50 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
+    // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
+    def checkReadMixedFiles(fileName: String, dt: String, dataStr: String): Unit = {
+      withTempPaths(2) { paths =>
+        paths.foreach(_.delete())
+        val path2_4 = getResourceAvroFilePath(fileName)
+        val path3_0 = paths(0).getCanonicalPath
+        val path3_0_rebase = paths(1).getCanonicalPath
+        if (dt == "date") {
+          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
+          df.write.format("avro").save(path3_0)
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+            df.write.format("avro").save(path3_0_rebase)
+          }
+          checkAnswer(
+            spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+            1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+        } else {
+          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
+          val avroSchema =
+            s"""
+              |{
+              |  "type" : "record",
+              |  "name" : "test_schema",
+              |  "fields" : [
+              |    {"name": "ts", "type": {"type": "long", "logicalType": "$dt"}}
+              |  ]
+              |}""".stripMargin
+          df.write.format("avro").option("avroSchema", avroSchema).save(path3_0)
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+            df.write.format("avro").option("avroSchema", avroSchema).save(path3_0_rebase)
+          }
+          checkAnswer(
+            spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+            1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+        }
+      }
+    }
+
     withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
-      checkAnswer(
-        readResourceAvroFile("before_1582_date_v2_4.avro"),
-        Row(java.sql.Date.valueOf("1001-01-01")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_micros_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-      checkAnswer(
-        readResourceAvroFile("before_1582_ts_millis_v2_4.avro"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.124")))
+      checkReadMixedFiles("before_1582_date_v2_4.avro", "date", "1001-01-01")
+      checkReadMixedFiles(
+        "before_1582_ts_micros_v2_4.avro", "timestamp-micros", "1001-01-01 01:02:03.123456")
+      checkReadMixedFiles(
+        "before_1582_ts_millis_v2_4.avro", "timestamp-millis", "1001-01-01 01:02:03.124")
     }
   }
 
@@ -1554,10 +1587,18 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
           .write.format("avro")
           .save(path)
       }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+
+      // The file metadata indicates if it needs rebase or not, so we can always get the correct
+      // result regardless of the "rebaseInRead" config.
+      Seq(true, false).foreach { rebase =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+          checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
+        }
       }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "false") {
+
+      // Force to not rebase to prove the written datetime values are rebased and we will get
+      // wrong result if we don't rebase while reading.
+      withSQLConf("spark.test.forceNoRebase" -> "true") {
         checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(nonRebased)))
       }
     }
@@ -1589,12 +1630,20 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
             .format("avro")
             .save(path)
         }
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
-          checkAnswer(
-            spark.read.schema("ts timestamp").format("avro").load(path),
-            Row(Timestamp.valueOf(rebased)))
+
+        // The file metadata indicates if it needs rebase or not, so we can always get the correct
+        // result regardless of the "rebaseInRead" config.
+        Seq(true, false).foreach { rebase =>
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+            checkAnswer(
+              spark.read.schema("ts timestamp").format("avro").load(path),
+              Row(Timestamp.valueOf(rebased)))
+          }
         }
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "false") {
+
+        // Force to not rebase to prove the written datetime values are rebased and we will get
+        // wrong result if we don't rebase while reading.
+        withSQLConf("spark.test.forceNoRebase" -> "true") {
           checkAnswer(
             spark.read.schema("ts timestamp").format("avro").load(path),
             Row(Timestamp.valueOf(nonRebased)))
@@ -1612,10 +1661,18 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
           .write.format("avro")
           .save(path)
       }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
-        checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+
+      // The file metadata indicates if it needs rebase or not, so we can always get the correct
+      // result regardless of the "rebaseInRead" config.
+      Seq(true, false).foreach { rebase =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+          checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
+        }
       }
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "false") {
+
+      // Force to not rebase to prove the written datetime values are rebased and we will get
+      // wrong result if we don't rebase while reading.
+      withSQLConf("spark.test.forceNoRebase" -> "true") {
         checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-07")))
       }
     }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 79edec315ac2f..7d35e96a78991 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -38,7 +38,6 @@
 import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
-import org.apache.spark.sql.internal.SQLConf;
 import org.apache.spark.sql.types.DataTypes;
 import org.apache.spark.sql.types.DecimalType;
 
@@ -109,7 +108,8 @@ public VectorizedColumnReader(
       ColumnDescriptor descriptor,
       OriginalType originalType,
       PageReader pageReader,
-      ZoneId convertTz) throws IOException {
+      ZoneId convertTz,
+      boolean rebaseDateTime) throws IOException {
     this.descriptor = descriptor;
     this.pageReader = pageReader;
     this.convertTz = convertTz;
@@ -132,7 +132,7 @@ public VectorizedColumnReader(
     if (totalValueCount == 0) {
       throw new IOException("totalValueCount == 0");
     }
-    this.rebaseDateTime = SQLConf.get().parquetRebaseDateTimeInReadEnabled();
+    this.rebaseDateTime = rebaseDateTime;
   }
 
   /**
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
index 7306709a79c34..c9590b97ce9cd 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
@@ -86,7 +86,12 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    * The timezone that timestamp INT96 values should be converted to. Null if no conversion. Here to
    * workaround incompatibilities between different engines when writing timestamp values.
    */
-  private ZoneId convertTz = null;
+  private final ZoneId convertTz;
+
+  /**
+   * true if need to rebase date/timestamp from Julian to Proleptic Gregorian calendar.
+   */
+  private final boolean rebaseDateTime;
 
   /**
    * columnBatch object that is used for batch decoding. This is created on first use and triggers
@@ -116,12 +121,19 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
    */
   private final MemoryMode MEMORY_MODE;
 
-  public VectorizedParquetRecordReader(ZoneId convertTz, boolean useOffHeap, int capacity) {
+  public VectorizedParquetRecordReader(
+    ZoneId convertTz, boolean rebaseDateTime, boolean useOffHeap, int capacity) {
     this.convertTz = convertTz;
+    this.rebaseDateTime = rebaseDateTime;
     MEMORY_MODE = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
     this.capacity = capacity;
   }
 
+  // For test only.
+  public VectorizedParquetRecordReader(boolean useOffHeap, int capacity) {
+    this(null, false, useOffHeap, capacity);
+  }
+
   /**
    * Implementation of RecordReader API.
    */
@@ -309,7 +321,7 @@ private void checkEndOfRowGroup() throws IOException {
     for (int i = 0; i < columns.size(); ++i) {
       if (missingColumns[i]) continue;
       columnReaders[i] = new VectorizedColumnReader(columns.get(i), types.get(i).getOriginalType(),
-        pages.getPageReader(columns.get(i)), convertTz);
+        pages.getPageReader(columns.get(i)), convertTz, rebaseDateTime);
     }
     totalCountLoadedSoFar += pages.getRowCount();
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index bd56635084c34..b19de6d3deb89 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -21,8 +21,11 @@ import org.apache.hadoop.fs.Path
 import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.util.Utils
 
 
 object DataSourceUtils {
@@ -64,4 +67,18 @@ object DataSourceUtils {
 
   private[sql] def isDataFile(fileName: String) =
     !(fileName.startsWith("_") || fileName.startsWith("."))
+
+  def needRebaseDateTime(lookupFileMeta: String => String): Option[Boolean] = {
+    if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") {
+      return Some(false)
+    }
+    // If there is no version, we return None and let the caller side to decide.
+    Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version =>
+      // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to
+      // rebase the datetime values.
+      // Files written by Spark 3.0 and latter may also need the rebase if they were written with
+      // the "rebaseInWrite" config enabled.
+      version < "3.0.0" || lookupFileMeta(SPARK_LEGACY_DATETIME) != null
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index 29dbd8dfbca8f..c6d9ddf370e22 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -300,6 +300,11 @@ class ParquetFileFormat
           None
         }
 
+      val rebaseDateTime = DataSourceUtils.needRebaseDateTime(
+        footerFileMetaData.getKeyValueMetaData.get).getOrElse {
+        SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ)
+      }
+
       val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
       val hadoopAttemptContext =
         new TaskAttemptContextImpl(broadcastedHadoopConf.value.value, attemptId)
@@ -312,7 +317,10 @@ class ParquetFileFormat
       val taskContext = Option(TaskContext.get())
       if (enableVectorizedReader) {
         val vectorizedReader = new VectorizedParquetRecordReader(
-          convertTz.orNull, enableOffHeapColumnVector && taskContext.isDefined, capacity)
+          convertTz.orNull,
+          rebaseDateTime,
+          enableOffHeapColumnVector && taskContext.isDefined,
+          capacity)
         val iter = new RecordReaderIterator(vectorizedReader)
         // SPARK-23457 Register a task completion listener before `initialization`.
         taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
@@ -328,7 +336,8 @@ class ParquetFileFormat
       } else {
         logDebug(s"Falling back to parquet-mr")
         // ParquetRecordReader returns InternalRow
-        val readSupport = new ParquetReadSupport(convertTz, enableVectorizedReader = false)
+        val readSupport = new ParquetReadSupport(
+          convertTz, enableVectorizedReader = false, rebaseDateTime)
         val reader = if (pushed.isDefined && enableRecordFilter) {
           val parquetFilter = FilterCompat.get(pushed.get, null)
           new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
index c05ecf16311ab..28165e0bbecde 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
@@ -50,16 +50,18 @@ import org.apache.spark.sql.types._
  * Due to this reason, we no longer rely on [[ReadContext]] to pass requested schema from [[init()]]
  * to [[prepareForRead()]], but use a private `var` for simplicity.
  */
-class ParquetReadSupport(val convertTz: Option[ZoneId],
-    enableVectorizedReader: Boolean)
+class ParquetReadSupport(
+    val convertTz: Option[ZoneId],
+    enableVectorizedReader: Boolean,
+    rebaseDateTime: Boolean)
   extends ReadSupport[InternalRow] with Logging {
   private var catalystRequestedSchema: StructType = _
 
   def this() {
     // We need a zero-arg constructor for SpecificParquetRecordReaderBase.  But that is only
-    // used in the vectorized reader, where we get the convertTz value directly, and the value here
-    // is ignored.
-    this(None, enableVectorizedReader = true)
+    // used in the vectorized reader, where we get the convertTz/rebaseDateTime value directly,
+    // and the values here are ignored.
+    this(None, enableVectorizedReader = true, rebaseDateTime = false)
   }
 
   /**
@@ -127,7 +129,8 @@ class ParquetReadSupport(val convertTz: Option[ZoneId],
       parquetRequestedSchema,
       ParquetReadSupport.expandUDT(catalystRequestedSchema),
       new ParquetToSparkSchemaConverter(conf),
-      convertTz)
+      convertTz,
+      rebaseDateTime)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
index 5622169df1281..ec037130aa7e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
@@ -31,16 +31,20 @@ import org.apache.spark.sql.types.StructType
  * @param parquetSchema Parquet schema of the records to be read
  * @param catalystSchema Catalyst schema of the rows to be constructed
  * @param schemaConverter A Parquet-Catalyst schema converter that helps initializing row converters
+ * @param convertTz the optional time zone to convert to int96 data
+ * @param rebaseDateTime true if need to rebase date/timestamp from Julian to Proleptic Gregorian
+ *                       calendar
  */
 private[parquet] class ParquetRecordMaterializer(
     parquetSchema: MessageType,
     catalystSchema: StructType,
     schemaConverter: ParquetToSparkSchemaConverter,
-    convertTz: Option[ZoneId])
+    convertTz: Option[ZoneId],
+    rebaseDateTime: Boolean)
   extends RecordMaterializer[InternalRow] {
 
-  private val rootConverter =
-    new ParquetRowConverter(schemaConverter, parquetSchema, catalystSchema, convertTz, NoopUpdater)
+  private val rootConverter = new ParquetRowConverter(
+    schemaConverter, parquetSchema, catalystSchema, convertTz, rebaseDateTime, NoopUpdater)
 
   override def getCurrentRecord: InternalRow = rootConverter.currentRecord
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index a5e769c2dc1d2..08fbca2995c04 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -120,7 +120,9 @@ private[parquet] class ParquetPrimitiveConverter(val updater: ParentContainerUpd
  * @param parquetType Parquet schema of Parquet records
  * @param catalystType Spark SQL schema that corresponds to the Parquet record type. User-defined
  *        types should have been expanded.
- * @param convertTz the optional time zone to convert to for int96 data
+ * @param convertTz the optional time zone to convert to int96 data
+ * @param rebaseDateTime true if need to rebase date/timestamp from Julian to Proleptic Gregorian
+ *                       calendar
  * @param updater An updater which propagates converted field values to the parent container
  */
 private[parquet] class ParquetRowConverter(
@@ -128,12 +130,10 @@ private[parquet] class ParquetRowConverter(
     parquetType: GroupType,
     catalystType: StructType,
     convertTz: Option[ZoneId],
+    rebaseDateTime: Boolean,
     updater: ParentContainerUpdater)
   extends ParquetGroupConverter(updater) with Logging {
 
-  // Enable rebasing date/timestamp from Julian to Proleptic Gregorian calendar
-  private val rebaseDateTime = SQLConf.get.parquetRebaseDateTimeInReadEnabled
-
   assert(
     parquetType.getFieldCount <= catalystType.length,
     s"""Field count of the Parquet schema is greater than the field count of the Catalyst schema:
@@ -386,7 +386,7 @@ private[parquet] class ParquetRowConverter(
           }
         }
         new ParquetRowConverter(
-          schemaConverter, parquetType.asGroupType(), t, convertTz, wrappedUpdater)
+          schemaConverter, parquetType.asGroupType(), t, convertTz, rebaseDateTime, wrappedUpdater)
 
       case t =>
         throw new RuntimeException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index 409ea88d6e0c3..e367b9cc774f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -31,7 +31,7 @@ import org.apache.parquet.io.api.{Binary, RecordConsumer}
 
 import org.apache.spark.SPARK_VERSION_SHORT
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SPARK_VERSION_METADATA_KEY
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -103,7 +103,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
     val metadata = Map(
       SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT,
       ParquetReadSupport.SPARK_METADATA_KEY -> schemaString
-    ).asJava
+    ) ++ (if (rebaseDateTime) Some(SPARK_LEGACY_DATETIME -> "") else None)
 
     logInfo(
       s"""Initialized Parquet WriteSupport with Catalyst schema:
@@ -112,7 +112,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
          |$messageType
        """.stripMargin)
 
-    new WriteContext(messageType, metadata)
+    new WriteContext(messageType, metadata.asJava)
   }
 
   override def prepareForWrite(recordConsumer: RecordConsumer): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
index 047bc74a8d81e..1925fa1796d48 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
@@ -33,7 +33,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader}
-import org.apache.spark.sql.execution.datasources.{PartitionedFile, RecordReaderIterator}
+import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedFile, RecordReaderIterator}
 import org.apache.spark.sql.execution.datasources.parquet._
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
@@ -117,7 +117,7 @@ case class ParquetPartitionReaderFactory(
       file: PartitionedFile,
       buildReaderFunc: (
         ParquetInputSplit, InternalRow, TaskAttemptContextImpl, Option[FilterPredicate],
-          Option[ZoneId]) => RecordReader[Void, T]): RecordReader[Void, T] = {
+          Option[ZoneId], Boolean) => RecordReader[Void, T]): RecordReader[Void, T] = {
     val conf = broadcastedConf.value.value
 
     val filePath = new Path(new URI(file.filePath))
@@ -169,8 +169,12 @@ case class ParquetPartitionReaderFactory(
     if (pushed.isDefined) {
       ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get)
     }
-    val reader =
-      buildReaderFunc(split, file.partitionValues, hadoopAttemptContext, pushed, convertTz)
+    val rebaseDatetime = DataSourceUtils.needRebaseDateTime(
+      footerFileMetaData.getKeyValueMetaData.get).getOrElse {
+      SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ)
+    }
+    val reader = buildReaderFunc(
+      split, file.partitionValues, hadoopAttemptContext, pushed, convertTz, rebaseDatetime)
     reader.initialize(split, hadoopAttemptContext)
     reader
   }
@@ -184,11 +188,13 @@ case class ParquetPartitionReaderFactory(
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
-      convertTz: Option[ZoneId]): RecordReader[Void, InternalRow] = {
+      convertTz: Option[ZoneId],
+      needDateTimeRebase: Boolean): RecordReader[Void, InternalRow] = {
     logDebug(s"Falling back to parquet-mr")
     val taskContext = Option(TaskContext.get())
     // ParquetRecordReader returns InternalRow
-    val readSupport = new ParquetReadSupport(convertTz, enableVectorizedReader = false)
+    val readSupport = new ParquetReadSupport(
+      convertTz, enableVectorizedReader = false, needDateTimeRebase)
     val reader = if (pushed.isDefined && enableRecordFilter) {
       val parquetFilter = FilterCompat.get(pushed.get, null)
       new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
@@ -213,10 +219,14 @@ case class ParquetPartitionReaderFactory(
       partitionValues: InternalRow,
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
-      convertTz: Option[ZoneId]): VectorizedParquetRecordReader = {
+      convertTz: Option[ZoneId],
+      rebaseDatetime: Boolean): VectorizedParquetRecordReader = {
     val taskContext = Option(TaskContext.get())
     val vectorizedReader = new VectorizedParquetRecordReader(
-      convertTz.orNull, enableOffHeapColumnVector && taskContext.isDefined, capacity)
+      convertTz.orNull,
+      rebaseDatetime,
+      enableOffHeapColumnVector && taskContext.isDefined,
+      capacity)
     val iter = new RecordReaderIterator(vectorizedReader)
     // SPARK-23457 Register a task completion listener before `initialization`.
     taskContext.foreach(_.addTaskCompletionListener[Unit](_ => iter.close()))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
index 58de6758f4d7b..c0397010acba3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
@@ -54,4 +54,10 @@ package object sql {
    * Note that Hive table property `spark.sql.create.version` also has Spark version.
    */
   private[sql] val SPARK_VERSION_METADATA_KEY = "org.apache.spark.version"
+
+  /**
+   * Parquet/Avro file metadata key to indicate that the file was written with legacy datetime
+   * values.
+   */
+  private[sql] val SPARK_LEGACY_DATETIME = "org.apache.spark.legacyDateTime"
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
index a084bec985510..d29c5e3f88010 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
@@ -169,7 +169,7 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
 
           files.map(_.asInstanceOf[String]).foreach { p =>
             val reader = new VectorizedParquetRecordReader(
-              null, enableOffHeapColumnVector, vectorizedReaderBatchSize)
+              enableOffHeapColumnVector, vectorizedReaderBatchSize)
             try {
               reader.initialize(p, ("id" :: Nil).asJava)
               val batch = reader.resultBatch()
@@ -203,7 +203,7 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
 
           files.map(_.asInstanceOf[String]).foreach { p =>
             val reader = new VectorizedParquetRecordReader(
-              null, enableOffHeapColumnVector, vectorizedReaderBatchSize)
+              enableOffHeapColumnVector, vectorizedReaderBatchSize)
             try {
               reader.initialize(p, ("id" :: Nil).asJava)
               val batch = reader.resultBatch()
@@ -458,7 +458,7 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
           var sum = 0
           files.map(_.asInstanceOf[String]).foreach { p =>
             val reader = new VectorizedParquetRecordReader(
-              null, enableOffHeapColumnVector, vectorizedReaderBatchSize)
+              enableOffHeapColumnVector, vectorizedReaderBatchSize)
             try {
               reader.initialize(p, ("c1" :: "c2" :: Nil).asJava)
               val batch = reader.resultBatch()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
index 6d681afd23b18..fbfedf02dc87e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetEncodingSuite.scala
@@ -42,7 +42,7 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
 
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         reader.initialize(file.asInstanceOf[String], null)
         val batch = reader.resultBatch()
         assert(reader.nextBatch())
@@ -69,7 +69,7 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
 
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         reader.initialize(file.asInstanceOf[String], null)
         val batch = reader.resultBatch()
         assert(reader.nextBatch())
@@ -100,7 +100,7 @@ class ParquetEncodingSuite extends ParquetCompatibilityTest with SharedSparkSess
 
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         reader.initialize(file, null /* set columns to null to project all columns */)
         val column = reader.resultBatch().column(0)
         assert(reader.nextBatch())
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index f901ce16f0965..239db7d4f30d2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.datasources.parquet
 
 import java.sql.{Date, Timestamp}
+import java.time._
 import java.util.Locale
 
 import scala.collection.JavaConverters._
@@ -720,7 +721,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, null)
           val result = mutable.ArrayBuffer.empty[(Int, String)]
@@ -739,7 +740,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, ("_2" :: Nil).asJava)
           val result = mutable.ArrayBuffer.empty[(String)]
@@ -757,7 +758,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, ("_2" :: "_1" :: Nil).asJava)
           val result = mutable.ArrayBuffer.empty[(String, Int)]
@@ -776,7 +777,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       {
         val conf = sqlContext.conf
         val reader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         try {
           reader.initialize(file, List[String]().asJava)
           var result = 0
@@ -817,7 +818,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         val schema = StructType(StructField("pcol", dt) :: Nil)
         val conf = sqlContext.conf
         val vectorizedReader = new VectorizedParquetRecordReader(
-          null, conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
+          conf.offHeapColumnVectorEnabled, conf.parquetVectorizedReaderBatchSize)
         val partitionValues = new GenericInternalRow(Array(v))
         val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
 
@@ -882,19 +883,52 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   }
 
   test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
+    // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
+    def checkReadMixedFiles(fileName: String, dt: String, dataStr: String): Unit = {
+      withTempPaths(2) { paths =>
+        paths.foreach(_.delete())
+        val path2_4 = getResourceParquetFilePath("test-data/" + fileName)
+        val path3_0 = paths(0).getCanonicalPath
+        val path3_0_rebase = paths(1).getCanonicalPath
+        if (dt == "date") {
+          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
+          df.write.parquet(path3_0)
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+            df.write.parquet(path3_0_rebase)
+          }
+          checkAnswer(
+            spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
+            1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+        } else {
+          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
+          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> dt) {
+            df.write.parquet(path3_0)
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+              df.write.parquet(path3_0_rebase)
+            }
+          }
+          checkAnswer(
+            spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
+            1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+        }
+      }
+    }
+
     Seq(false, true).foreach { vectorized =>
       withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
         withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
-          checkAnswer(
-            readResourceParquetFile("test-data/before_1582_date_v2_4.snappy.parquet"),
-            Row(java.sql.Date.valueOf("1001-01-01")))
-          checkAnswer(readResourceParquetFile(
-            "test-data/before_1582_timestamp_micros_v2_4.snappy.parquet"),
-            Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-          checkAnswer(readResourceParquetFile(
-            "test-data/before_1582_timestamp_millis_v2_4.snappy.parquet"),
-            Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123")))
+          checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
+          checkReadMixedFiles(
+            "before_1582_timestamp_micros_v2_4.snappy.parquet",
+            "TIMESTAMP_MICROS",
+            "1001-01-01 01:02:03.123456")
+          checkReadMixedFiles(
+            "before_1582_timestamp_millis_v2_4.snappy.parquet",
+            "TIMESTAMP_MILLIS",
+            "1001-01-01 01:02:03.123")
         }
+
+        // INT96 is a legacy timestamp format and we always rebase the seconds for it.
         checkAnswer(readResourceParquetFile(
           "test-data/before_1582_timestamp_int96_v2_4.snappy.parquet"),
           Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
@@ -918,10 +952,17 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
                 .write
                 .parquet(path)
             }
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
-              checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
+            // The file metadata indicates if it needs rebase or not, so we can always get the
+            // correct result regardless of the "rebaseInRead" config.
+            Seq(true, false).foreach { rebase =>
+              withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
+              }
             }
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "false") {
+
+            // Force to not rebase to prove the written datetime values are rebased and we will get
+            // wrong result if we don't rebase while reading.
+            withSQLConf("spark.test.forceNoRebase" -> "true") {
               checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(nonRebased)))
             }
           }
@@ -939,10 +980,18 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           .write
           .parquet(path)
       }
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
-        checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
+
+      // The file metadata indicates if it needs rebase or not, so we can always get the correct
+      // result regardless of the "rebaseInRead" config.
+      Seq(true, false).foreach { rebase =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+          checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
+        }
       }
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "false") {
+
+      // Force to not rebase to prove the written datetime values are rebased and we will get
+      // wrong result if we don't rebase while reading.
+      withSQLConf("spark.test.forceNoRebase" -> "true") {
         checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-07")))
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index f2dbc536ac566..c833d5f1ab1f6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -156,7 +156,10 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   }
 
   protected def readResourceParquetFile(name: String): DataFrame = {
-    val url = Thread.currentThread().getContextClassLoader.getResource(name)
-    spark.read.parquet(url.toString)
+    spark.read.parquet(getResourceParquetFilePath(name))
+  }
+
+  protected def getResourceParquetFilePath(name: String): String = {
+    Thread.currentThread().getContextClassLoader.getResource(name).toString
   }
 }

From c5eac36ff644ae509dfeb2e888376dfd895ebb5b Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 21 Apr 2020 23:58:45 +0800
Subject: [PATCH 0532/1280] HOT-FIX: fix compilation

---
 .../spark/sql/catalyst/expressions/datetimeExpressions.scala     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index a36c1f23934ea..63d6fccd9dac9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.sql.Timestamp
 import java.time.{DateTimeException, LocalDate, LocalDateTime, ZoneId}
 import java.time.temporal.IsoFields
 import java.util.Locale

From fc4b49c603f9ddef45c6e08144de06f6e4786208 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 21 Apr 2020 11:22:18 -0700
Subject: [PATCH 0533/1280] [SPARK-31503][SQL] fix the SQL string of the TRIM
 functions

### What changes were proposed in this pull request?

override the `sql` method of `StringTrim`, `StringTrimLeft` and `StringTrimRight`, to use the standard SQL syntax.

### Why are the changes needed?

The current implementation is wrong. It gives you a SQL string that returns different result.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

new tests

Closes #28281 from cloud-fan/sql.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b209b5f40677ee7b241362e3805da9897ee942b4)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/stringExpressions.scala       | 30 +++++++++----------
 .../sql-tests/inputs/string-functions.sql     |  1 +
 .../results/postgreSQL/strings.sql.out        |  2 +-
 .../results/string-functions.sql.out          | 26 ++++++++++------
 4 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 50a90ae40497a..3723680ff5db2 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -683,11 +683,22 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi
 
 trait String2TrimExpression extends Expression with ImplicitCastInputTypes {
 
+  protected def srcStr: Expression
+  protected def trimStr: Option[Expression]
+  protected def direction: String
+
+  override def children: Seq[Expression] = srcStr +: trimStr.toSeq
   override def dataType: DataType = StringType
   override def inputTypes: Seq[AbstractDataType] = Seq.fill(children.size)(StringType)
 
   override def nullable: Boolean = children.exists(_.nullable)
   override def foldable: Boolean = children.forall(_.foldable)
+
+  override def sql: String = if (trimStr.isDefined) {
+    s"TRIM($direction ${trimStr.get.sql} FROM ${srcStr.sql})"
+  } else {
+    super.sql
+  }
 }
 
 object StringTrim {
@@ -769,11 +780,8 @@ case class StringTrim(
 
   override def prettyName: String = "trim"
 
-  override def children: Seq[Expression] = if (trimStr.isDefined) {
-    srcStr :: trimStr.get :: Nil
-  } else {
-    srcStr :: Nil
-  }
+  override protected def direction: String = "BOTH"
+
   override def eval(input: InternalRow): Any = {
     val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
     if (srcString == null) {
@@ -865,11 +873,7 @@ case class StringTrimLeft(
 
   override def prettyName: String = "ltrim"
 
-  override def children: Seq[Expression] = if (trimStr.isDefined) {
-    srcStr :: trimStr.get :: Nil
-  } else {
-    srcStr :: Nil
-  }
+  override protected def direction: String = "LEADING"
 
   override def eval(input: InternalRow): Any = {
     val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
@@ -964,11 +968,7 @@ case class StringTrimRight(
 
   override def prettyName: String = "rtrim"
 
-  override def children: Seq[Expression] = if (trimStr.isDefined) {
-    srcStr :: trimStr.get :: Nil
-  } else {
-    srcStr :: Nil
-  }
+  override protected def direction: String = "TRAILING"
 
   override def eval(input: InternalRow): Any = {
     val srcString = srcStr.eval(input).asInstanceOf[UTF8String]
diff --git a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
index fd6cc4d811045..8e33471e8b129 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
@@ -40,6 +40,7 @@ SELECT substring('Spark SQL' from -3);
 SELECT substring('Spark SQL' from 5 for 1);
 
 -- trim
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ");
 SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx');
 SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx');
 SELECT trim(LEADING 'xyz' FROM 'zzzytest');
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
index 5f89c799498ac..e8a3a9b9731a6 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/strings.sql.out
@@ -977,7 +977,7 @@ struct<repeat(Pg, -4):string>
 -- !query
 SELECT trim(binary('\\000') from binary('\\000Tom\\000'))
 -- !query schema
-struct<trim(CAST(CAST(\000Tom\000 AS BINARY) AS STRING), CAST(CAST(\000 AS BINARY) AS STRING)):string>
+struct<TRIM(BOTH CAST(CAST(\000 AS BINARY) AS STRING) FROM CAST(CAST(\000Tom\000 AS BINARY) AS STRING)):string>
 -- !query output
 Tom
 
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 042d332bdb5c2..43c18f5417110 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 33
+-- Number of queries: 34
 
 
 -- !query
@@ -204,10 +204,18 @@ struct<substring(Spark SQL, 5, 1):string>
 k
 
 
+-- !query
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ")
+-- !query schema
+struct<trim( xyz ):string,ltrim( xyz ):string,rtrim( xyz ):string>
+-- !query output
+xyz	xyz 	 xyz
+
+
 -- !query
 SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
 -- !query schema
-struct<trim(yxTomxx, xyz):string,trim(yxTomxx, xyz):string>
+struct<TRIM(BOTH xyz FROM yxTomxx):string,TRIM(BOTH xyz FROM yxTomxx):string>
 -- !query output
 Tom	Tom
 
@@ -215,7 +223,7 @@ Tom	Tom
 -- !query
 SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
 -- !query schema
-struct<trim(xxxbarxxx, x):string,trim(xxxbarxxx, x):string>
+struct<TRIM(BOTH x FROM xxxbarxxx):string,TRIM(BOTH x FROM xxxbarxxx):string>
 -- !query output
 bar	bar
 
@@ -223,7 +231,7 @@ bar	bar
 -- !query
 SELECT trim(LEADING 'xyz' FROM 'zzzytest')
 -- !query schema
-struct<ltrim(zzzytest, xyz):string>
+struct<TRIM(LEADING xyz FROM zzzytest):string>
 -- !query output
 test
 
@@ -231,7 +239,7 @@ test
 -- !query
 SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz')
 -- !query schema
-struct<ltrim(zzzytestxyz, xyz):string>
+struct<TRIM(LEADING xyz FROM zzzytestxyz):string>
 -- !query output
 testxyz
 
@@ -239,7 +247,7 @@ testxyz
 -- !query
 SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
 -- !query schema
-struct<ltrim(xyxXxyLAST WORD, xy):string>
+struct<TRIM(LEADING xy FROM xyxXxyLAST WORD):string>
 -- !query output
 XxyLAST WORD
 
@@ -247,7 +255,7 @@ XxyLAST WORD
 -- !query
 SELECT trim(TRAILING 'xyz' FROM 'testxxzx')
 -- !query schema
-struct<rtrim(testxxzx, xyz):string>
+struct<TRIM(TRAILING xyz FROM testxxzx):string>
 -- !query output
 test
 
@@ -255,7 +263,7 @@ test
 -- !query
 SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx')
 -- !query schema
-struct<rtrim(xyztestxxzx, xyz):string>
+struct<TRIM(TRAILING xyz FROM xyztestxxzx):string>
 -- !query output
 xyztest
 
@@ -263,6 +271,6 @@ xyztest
 -- !query
 SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
 -- !query schema
-struct<rtrim(TURNERyxXxy, xy):string>
+struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
 -- !query output
 TURNERyxX

From 5c396bc22575a805df7e97f587207af7c214df42 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 22 Apr 2020 10:16:39 +0000
Subject: [PATCH 0534/1280] [SPARK-31498][SQL][DOCS] Dump public static sql
 configurations through doc generation

### What changes were proposed in this pull request?

Currently, only the non-static public SQL configurations are dump to public doc, we'd better also add those static public ones as the command `set -v`

This PR force call StaticSQLConf to buildStaticConf.

### Why are the changes needed?

Fix missing SQL configurations in doc

### Does this PR introduce any user-facing change?

NO

### How was this patch tested?

add unit test and verify locally to see if public static SQL conf is in `docs/sql-config.html`

Closes #28274 from yaooqinn/SPARK-31498.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 2c2062ea7c77780ea017cf94e4258a09fe34c6ac)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/configuration.md                         | 27 +++++++-
 .../spark/sql/api/python/PythonSQLUtils.scala | 13 +++-
 .../sql/api/python/PythonSQLUtilsSuite.scala  | 64 +++++++++++++++++++
 sql/create-docs.sh                            |  7 +-
 sql/gen-sql-config-docs.py                    | 21 ++++--
 5 files changed, 119 insertions(+), 13 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala

diff --git a/docs/configuration.md b/docs/configuration.md
index 829d0627be1bc..9e91d08591e45 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2611,11 +2611,32 @@ Please refer to the [Security](security.html) page for available options on how
 Spark subsystems.
 
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'generated-sql-configuration-table.html' %}
 ### Spark SQL
 
-{% include_relative generated-sql-configuration-table.html %}
+#### Runtime SQL Configuration
+
+Runtime SQL configurations are per-session, mutable Spark SQL configurations. They can be set with initial values by the config file
+and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
+Also, they can be set and queried by SET commands and rest to their initial values by RESET command,
+or by `SparkSession.conf`'s setter and getter methods in runtime.
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-runtime-sql-config-table.html' %}
+        {% include_relative generated-runtime-sql-config-table.html %}
+        {% break %}
+    {% endif %}
+{% endfor %}
+
+
+#### Static SQL Configuration
+
+Static SQL configurations are cross-session, immutable Spark SQL configurations. They can be set with final values by the config file
+and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
+External users can query the static sql config values via `SparkSession.conf` or via set command, e.g. `SET spark.sql.extensions;`, but cannot set/unset them.
+
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-static-sql-config-table.html' %}
+        {% include_relative generated-static-sql-config-table.html %}
         {% break %}
     {% endif %}
 {% endfor %}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index 03f5a60aec438..2e5f59edcf1da 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -23,13 +23,13 @@ import java.nio.channels.Channels
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.api.python.PythonRDDServer
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{DataFrame, Dataset, SQLContext}
+import org.apache.spark.sql.{DataFrame, SQLContext}
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.execution.{ExplainMode, QueryExecution}
 import org.apache.spark.sql.execution.arrow.ArrowConverters
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.DataType
 
 private[sql] object PythonSQLUtils {
@@ -43,7 +43,14 @@ private[sql] object PythonSQLUtils {
   def listSQLConfigs(): Array[(String, String, String, String)] = {
     val conf = new SQLConf()
     // Py4J doesn't seem to translate Seq well, so we convert to an Array.
-    conf.getAllDefinedConfs.toArray
+    conf.getAllDefinedConfs.filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+  }
+
+  def listStaticSQLConfigs(): Array[(String, String, String, String)] = {
+    val conf = new SQLConf()
+    // Force to build static SQL configurations
+    StaticSQLConf
+    conf.getAllDefinedConfs.filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
new file mode 100644
index 0000000000000..0d18d123e328a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.python
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
+
+class PythonSQLUtilsSuite extends SparkFunSuite {
+
+  test("listing sql configurations contains runtime ones only") {
+    val configs = PythonSQLUtils.listSQLConfigs()
+
+    // static sql configurations
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listSQLConfigs should contain public dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listSQLConfigs should not contain core configuration")
+  }
+
+  test("listing static sql configurations contains public static ones only") {
+    val configs = PythonSQLUtils.listStaticSQLConfigs()
+
+    // static sql configurations
+    assert(configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
+      "listStaticSQLConfigs should contain public static sql configuration")
+    assert(!configs.exists(entry => entry._1 == StaticSQLConf.DEBUG_MODE.key),
+      "listStaticSQLConfigs should not contain internal static sql configuration")
+
+    // dynamic sql configurations
+    assert(!configs.exists(entry => entry._1 == SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key),
+      "listStaticSQLConfigs should not contain dynamic sql configuration")
+    assert(!configs.exists(entry => entry._1 == SQLConf.ANALYZER_MAX_ITERATIONS.key),
+      "listStaticSQLConfigs should not contain internal dynamic sql configuration")
+
+    // spark core configurations
+    assert(!configs.exists(entry => entry._1 == "spark.master"),
+      "listStaticSQLConfigs should not contain core configuration")
+  }
+}
diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 6614c714e90c7..336afc4fcb9f4 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -45,8 +45,11 @@ mkdir docs
 echo "Generating SQL API Markdown files."
 "$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py
 
-echo "Generating SQL configuration table HTML file."
-"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
+echo "Generating runtime SQL runtime configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py runtime
+
+echo "Generating static SQL configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py static
 
 echo "Generating HTML files for SQL function table and examples."
 "$SPARK_HOME/bin/spark-submit" gen-sql-functions-docs.py
diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py
index 0043c412fbc16..848d2f21f1142 100644
--- a/sql/gen-sql-config-docs.py
+++ b/sql/gen-sql-config-docs.py
@@ -17,6 +17,7 @@
 
 import os
 import re
+import sys
 from collections import namedtuple
 from textwrap import dedent
 
@@ -30,7 +31,11 @@
     "SQLConfEntry", ["name", "default", "description", "version"])
 
 
-def get_public_sql_configs(jvm):
+def get_public_sql_configs(jvm, group):
+    if group == "static":
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listStaticSQLConfigs()
+    else:
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
     sql_configs = [
         SQLConfEntry(
             name=_sql_config._1(),
@@ -38,7 +43,7 @@ def get_public_sql_configs(jvm):
             description=_sql_config._3(),
             version=_sql_config._4()
         )
-        for _sql_config in jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
+        for _sql_config in config_set
     ]
     return sql_configs
 
@@ -114,11 +119,17 @@ def generate_sql_configs_table_html(sql_configs, path):
 
 
 if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: ./bin/spark-submit sql/gen-sql-config-docs.py <static|runtime>")
+        sys.exit(-1)
+    else:
+        group = sys.argv[1]
+
     jvm = launch_gateway().jvm
-    sql_configs = get_public_sql_configs(jvm)
+    sql_configs = get_public_sql_configs(jvm, group)
 
     spark_root_dir = os.path.dirname(os.path.dirname(__file__))
-    sql_configs_table_path = os.path.join(
-        spark_root_dir, "docs/generated-sql-configuration-table.html")
+    sql_configs_table_path = os.path\
+        .join(spark_root_dir, "docs", "generated-" + group + "-sql-config-table.html")
 
     generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)

From 004952d26139cee9c03bf5300fc815940efb79c3 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 22 Apr 2020 10:24:49 +0000
Subject: [PATCH 0535/1280] [SPARK-31507][SQL] Remove uncommon fields support 
 and update some fields with meaningful names for extract function
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

Extracting millennium, century, decade, millisecond, microsecond and epoch from datetime is neither ANSI standard nor quite common in modern SQL platforms. Most of the systems listing below does not support these except PostgreSQL and redshift.

https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF

https://docs.oracle.com/cd/B19306_01/server.102/b14200/functions050.htm

https://prestodb.io/docs/current/functions/datetime.html

https://docs.cloudera.com/documentation/enterprise/5-8-x/topics/impala_datetime_functions.html

https://docs.snowflake.com/en/sql-reference/functions-date-time.html#label-supported-date-time-parts

https://www.postgresql.org/docs/9.1/functions-datetime.html#FUNCTIONS-DATETIME-EXTRACT

This PR removes these extract fields support from extract function for date and timestamp values

`isoyear` is PostgreSQL specific but `yearofweek` is more commonly used across platforms
`isodow` is PostgreSQL specific but `iso` as a suffix is more commonly used across platforms so, `dow_iso` and `dayofweek_iso` is used to replace it.

For historical reasons, we have [`dayofweek`, `dow`] implemented for representing a non-ISO day-of-week and a newly added `isodow` from PostgreSQL for ISO day-of-week. Many other systems only have one week-numbering system support and use either full names or abbreviations. Things in spark become a little bit complicated.
1. because of the existence of `isodow`, so we need to add iso-prefix to `dayofweek` to make a pair for it too. [`dayofweek`, `isodayofweek`, `dow` and `isodow`]
2. because there are rare `iso`-prefixed systems and more systems choose `iso`-suffixed way, so we may result in [`dayofweek`, `dayofweekiso`, `dow`, `dowiso`]
3. `dayofweekiso` looks nice and has use cases in the platforms listed above, e.g. snowflake, but `dowiso` looks weird and no use cases found.
4. with a discussion the community，we have agreed with an underscore before `iso` may look much better because `isodow` is new and there is no standard for `iso` kind of things, so this may be good for us to make it simple and clear for end-users if they are well documented too.

Thus, we finally result in [`dayofweek`, `dow`] for Non-ISO day-of-week system and [`dayofweek_iso`, `dow_iso`] for ISO system

### Why are the changes needed?

Remove some nonstandard and uncommon features as we can add them back if necessary

### Does this PR introduce any user-facing change?

NO, we should target this to 3.0.0 and these are added during 3.0.0

### How was this patch tested?

Remove unused tests

Closes #28284 from yaooqinn/SPARK-31507.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 37d2e037ed804a414ed874c829e0139a277a5ae8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/datetimeExpressions.scala     | 152 +---
 .../sql/catalyst/util/DateTimeUtils.scala     |  39 +-
 .../expressions/DateExpressionsSuite.scala    |  89 +--
 .../ExtractBenchmark-jdk11-results.txt        | 168 ++---
 .../benchmarks/ExtractBenchmark-results.txt   | 180 +++--
 .../resources/sql-tests/inputs/date_part.sql  | 111 ---
 .../resources/sql-tests/inputs/extract.sql    |  92 ++-
 .../sql-tests/inputs/postgreSQL/date.sql      |  76 +-
 .../sql-tests/inputs/postgreSQL/timestamp.sql |  18 +-
 .../sql-tests/results/date_part.sql.out       | 670 ------------------
 .../sql-tests/results/extract.sql.out         | 441 ++++++++----
 .../sql-tests/results/postgreSQL/date.sql.out | 306 +-------
 .../results/postgreSQL/timestamp.sql.out      |  32 +-
 .../benchmark/ExtractBenchmark.scala          |   8 +-
 14 files changed, 576 insertions(+), 1806 deletions(-)
 delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/date_part.sql
 delete mode 100644 sql/core/src/test/resources/sql-tests/results/date_part.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 63d6fccd9dac9..6ca6ab7a3c1ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -334,48 +334,6 @@ case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = No
   }
 }
 
-case class Milliseconds(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with ImplicitCastInputTypes with TimeZoneAwareExpression {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-  // DecimalType is used here to not lose precision while converting microseconds to
-  // the fractional part of milliseconds. Scale 3 is taken to have all microseconds as
-  // the fraction. The precision 8 should cover 2 digits for seconds, 3 digits for
-  // milliseconds and 3 digits for microseconds.
-  override def dataType: DataType = DecimalType(8, 3)
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMilliseconds(timestamp.asInstanceOf[Long], zoneId)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMilliseconds($c, $zid)")
-  }
-}
-
-case class Microseconds(child: Expression, timeZoneId: Option[String] = None)
-  extends UnaryExpression with ImplicitCastInputTypes with TimeZoneAwareExpression {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-  override def dataType: DataType = IntegerType
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getMicroseconds(timestamp.asInstanceOf[Long], zoneId)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMicroseconds($c, $zid)")
-  }
-}
-
 @ExpressionDescription(
   usage = "_FUNC_(date) - Returns the day of year of the date/timestamp.",
   examples = """
@@ -426,19 +384,19 @@ case class Year(child: Expression) extends UnaryExpression with ImplicitCastInpu
   }
 }
 
-case class IsoYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
+case class YearOfWeek(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
 
   override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
 
   override def dataType: DataType = IntegerType
 
   override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getIsoYear(date.asInstanceOf[Int])
+    DateTimeUtils.getWeekBasedYear(date.asInstanceOf[Int])
   }
 
   override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getIsoYear($c)")
+    defineCodeGen(ctx, ev, c => s"$dtu.getWeekBasedYear($c)")
   }
 }
 
@@ -2032,108 +1990,26 @@ case class MakeTimestamp(
   override def prettyName: String = "make_timestamp"
 }
 
-case class Millennium(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getMillennium(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getMillennium($c)")
-  }
-}
-
-case class Century(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getCentury(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getCentury($c)")
-  }
-}
-
-case class Decade(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(DateType)
-
-  override def dataType: DataType = IntegerType
-
-  override protected def nullSafeEval(date: Any): Any = {
-    DateTimeUtils.getDecade(date.asInstanceOf[Int])
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    defineCodeGen(ctx, ev, c => s"$dtu.getDecade($c)")
-  }
-}
-
-case class Epoch(child: Expression, timeZoneId: Option[String] = None)
-    extends UnaryExpression with ImplicitCastInputTypes with TimeZoneAwareExpression {
-
-  override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)
-  // DecimalType is used to not lose precision while converting microseconds to
-  // the fractional part of seconds. Scale 6 is taken to have all microseconds as
-  // the fraction. The precision 20 should cover whole valid range of years [1, 9999]
-  // plus negative years that can be used in some cases though are not officially supported.
-  override def dataType: DataType = DecimalType(20, 6)
-  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
-    copy(timeZoneId = Option(timeZoneId))
-
-  override protected def nullSafeEval(timestamp: Any): Any = {
-    DateTimeUtils.getEpoch(timestamp.asInstanceOf[Long], zoneId)
-  }
-
-  override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
-    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
-    val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
-    defineCodeGen(ctx, ev, c => s"$dtu.getEpoch($c, $zid)")
-  }
-}
-
 object DatePart {
 
   def parseExtractField(
       extractField: String,
       source: Expression,
       errorHandleFunc: => Nothing): Expression = extractField.toUpperCase(Locale.ROOT) match {
-    case "MILLENNIUM" | "MILLENNIA" | "MIL" | "MILS" => Millennium(source)
-    case "CENTURY" | "CENTURIES" | "C" | "CENT" => Century(source)
-    case "DECADE" | "DECADES" | "DEC" | "DECS" => Decade(source)
     case "YEAR" | "Y" | "YEARS" | "YR" | "YRS" => Year(source)
-    case "ISOYEAR" => IsoYear(source)
+    case "YEAROFWEEK" => YearOfWeek(source)
     case "QUARTER" | "QTR" => Quarter(source)
     case "MONTH" | "MON" | "MONS" | "MONTHS" => Month(source)
     case "WEEK" | "W" | "WEEKS" => WeekOfYear(source)
     case "DAY" | "D" | "DAYS" => DayOfMonth(source)
     case "DAYOFWEEK" | "DOW" => DayOfWeek(source)
-    case "ISODOW" => Add(WeekDay(source), Literal(1))
+    case "DAYOFWEEK_ISO" | "DOW_ISO" => Add(WeekDay(source), Literal(1))
     case "DOY" => DayOfYear(source)
     case "HOUR" | "H" | "HOURS" | "HR" | "HRS" => Hour(source)
     case "MINUTE" | "M" | "MIN" | "MINS" | "MINUTES" => Minute(source)
     case "SECOND" | "S" | "SEC" | "SECONDS" | "SECS" => SecondWithFraction(source)
-    case "MILLISECONDS" | "MSEC" | "MSECS" | "MILLISECON" | "MSECONDS" | "MS" =>
-      Milliseconds(source)
-    case "MICROSECONDS" | "USEC" | "USECS" | "USECONDS" | "MICROSECON" | "US" =>
-      Microseconds(source)
-    case "EPOCH" => Epoch(source)
     case _ => errorHandleFunc
   }
-}
-
-object DatePartLike {
 
   def toEquivalentExpr(field: Expression, source: Expression): Expression = {
     if (!field.foldable) {
@@ -2191,7 +2067,7 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
 
   def this(field: Expression, source: Expression) = {
-    this(field, source, DatePartLike.toEquivalentExpr(field, source))
+    this(field, source, DatePart.toEquivalentExpr(field, source))
   }
 
   override def flatArguments: Iterator[Any] = Iterator(field, source)
@@ -2205,26 +2081,20 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
   arguments = """
     Arguments:
       * field - selects which part of the source should be extracted
-          - Supported string values of `field` for dates and timestamps are:
-              - "MILLENNIUM", ("MILLENNIA", "MIL", "MILS") - the conventional numbering of millennia
-              - "CENTURY", ("CENTURIES", "C", "CENT") - the conventional numbering of centuries
-              - "DECADE", ("DECADES", "DEC", "DECS") - the year field divided by 10
+          - Supported string values of `field` for dates and timestamps are(case insensitive):
               - "YEAR", ("Y", "YEARS", "YR", "YRS") - the year field
-              - "ISOYEAR" - the ISO 8601 week-numbering year that the datetime falls in
+              - "YEAROFWEEK" - the ISO 8601 week-numbering year that the datetime falls in. For example, 2005-01-02 is part of the 53rd week of year 2004, so the result is 2004
               - "QUARTER", ("QTR") - the quarter (1 - 4) of the year that the datetime falls in
               - "MONTH", ("MON", "MONS", "MONTHS") - the month field (1 - 12)
               - "WEEK", ("W", "WEEKS") - the number of the ISO 8601 week-of-week-based-year. A week is considered to start on a Monday and week 1 is the first week with >3 days. In the ISO week-numbering system, it is possible for early-January dates to be part of the 52nd or 53rd week of the previous year, and for late-December dates to be part of the first week of the next year. For example, 2005-01-02 is part of the 53rd week of year 2004, while 2012-12-31 is part of the first week of 2013
               - "DAY", ("D", "DAYS") - the day of the month field (1 - 31)
               - "DAYOFWEEK",("DOW") - the day of the week for datetime as Sunday(1) to Saturday(7)
-              - "ISODOW" - ISO 8601 based day of the week for datetime as Monday(1) to Sunday(7)
+              - "DAYOFWEEK_ISO",("DOW_ISO") - ISO 8601 based day of the week for datetime as Monday(1) to Sunday(7)
               - "DOY" - the day of the year (1 - 365/366)
               - "HOUR", ("H", "HOURS", "HR", "HRS") - The hour field (0 - 23)
               - "MINUTE", ("M", "MIN", "MINS", "MINUTES") - the minutes field (0 - 59)
               - "SECOND", ("S", "SEC", "SECONDS", "SECS") - the seconds field, including fractional parts
-              - "MILLISECONDS", ("MSEC", "MSECS", "MILLISECON", "MSECONDS", "MS") - the seconds field, including fractional parts, multiplied by 1000. Note that this includes full seconds
-              - "MICROSECONDS", ("USEC", "USECS", "USECONDS", "MICROSECON", "US") - The seconds field, including fractional parts, multiplied by 1000000. Note that this includes full seconds
-              - "EPOCH" - the number of seconds with fractional part in microsecond precision since 1970-01-01 00:00:00 local time (can be negative)
-          - Supported string values of `field` for interval(which consists of `months`, `days`, `microseconds`) are:
+          - Supported string values of `field` for interval(which consists of `months`, `days`, `microseconds`) are(case insensitive):
               - "YEAR", ("Y", "YEARS", "YR", "YRS") - the total `months` / 12
               - "MONTH", ("MON", "MONS", "MONTHS") - the total `months` % 12
               - "DAY", ("D", "DAYS") - the `days` part of interval
@@ -2257,7 +2127,7 @@ case class Extract(field: Expression, source: Expression, child: Expression)
   extends RuntimeReplaceable {
 
   def this(field: Expression, source: Expression) = {
-    this(field, source, DatePartLike.toEquivalentExpr(field, source))
+    this(field, source, DatePart.toEquivalentExpr(field, source))
   }
 
   override def flatArguments: Iterator[Any] = Iterator(field, source)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 8486bba4a25ae..0e9a15be1a6c6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -526,14 +526,6 @@ object DateTimeUtils {
     Decimal(getMicroseconds(microsec, zoneId), 8, 6)
   }
 
-  /**
-   * Returns seconds, including fractional parts, multiplied by 1000. The timestamp
-   * is expressed in microseconds since the epoch.
-   */
-  def getMilliseconds(timestamp: SQLTimestamp, zoneId: ZoneId): Decimal = {
-    Decimal(getMicroseconds(timestamp, zoneId), 8, 3)
-  }
-
   /**
    * Returns seconds, including fractional parts, multiplied by 1000000. The timestamp
    * is expressed in microseconds since the epoch.
@@ -551,24 +543,6 @@ object DateTimeUtils {
     LocalDate.ofEpochDay(date).getDayOfYear
   }
 
-  private def extractFromYear(date: SQLDate, divider: Int): Int = {
-    val localDate = daysToLocalDate(date)
-    val yearOfEra = localDate.get(ChronoField.YEAR_OF_ERA)
-    var result = yearOfEra / divider
-    if ((yearOfEra % divider) != 0 || yearOfEra <= 1) result += 1
-    if (localDate.get(ChronoField.ERA) == 0) result = -result
-    result
-  }
-
-  /** Returns the millennium for the given date. The date is expressed in days since 1.1.1970. */
-  def getMillennium(date: SQLDate): Int = extractFromYear(date, 1000)
-
-  /** Returns the century for the given date. The date is expressed in days since 1.1.1970. */
-  def getCentury(date: SQLDate): Int = extractFromYear(date, 100)
-
-  /** Returns the decade for the given date. The date is expressed in days since 1.1.1970. */
-  def getDecade(date: SQLDate): Int = Math.floorDiv(getYear(date), 10)
-
   /**
    * Returns the year value for the given date. The date is expressed in days
    * since 1.1.1970.
@@ -581,7 +555,7 @@ object DateTimeUtils {
    * Returns the year which conforms to ISO 8601. Each ISO 8601 week-numbering
    * year begins with the Monday of the week containing the 4th of January.
    */
-  def getIsoYear(date: SQLDate): Int = {
+  def getWeekBasedYear(date: SQLDate): Int = {
     daysToLocalDate(date).get(IsoFields.WEEK_BASED_YEAR)
   }
 
@@ -869,17 +843,6 @@ object DateTimeUtils {
     convertTz(time, getZoneId(timeZone), ZoneOffset.UTC)
   }
 
-  /**
-   * Returns the number of seconds with fractional part in microsecond precision
-   * since 1970-01-01 00:00:00 local time.
-   */
-  def getEpoch(timestamp: SQLTimestamp, zoneId: ZoneId): Decimal = {
-    val offset = SECONDS.toMicros(
-      zoneId.getRules.getOffset(microsToInstant(timestamp)).getTotalSeconds)
-    val sinceEpoch = timestamp + offset
-    Decimal(sinceEpoch, 20, 6)
-  }
-
   def currentTimestamp(): SQLTimestamp = instantToMicros(Instant.now())
 
   def currentDate(zoneId: ZoneId): SQLDate = localDateToDays(LocalDate.now(zoneId))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 7f235d6bb4f30..b3547172fe488 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -1032,94 +1032,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(makeTimestampExpr, Timestamp.valueOf("2019-08-12 00:00:58.000001"))
   }
 
-  test("millennium") {
-    val date = MakeDate(Literal(2019), Literal(1), Literal(1))
-    checkEvaluation(Millennium(date), 3)
-    checkEvaluation(Millennium(date.copy(year = Literal(2001))), 3)
-    checkEvaluation(Millennium(date.copy(year = Literal(2000))), 2)
-    checkEvaluation(Millennium(date.copy(year = Literal(1001), day = Literal(28))), 2)
-    checkEvaluation(Millennium(date.copy(year = Literal(1))), 1)
-    checkEvaluation(Millennium(date.copy(year = Literal(-1))), -1)
-    checkEvaluation(Millennium(date.copy(year = Literal(-100), month = Literal(12))), -1)
-    checkEvaluation(Millennium(date.copy(year = Literal(-2019))), -3)
-  }
-
-  test("century") {
-    val date = MakeDate(Literal(2019), Literal(1), Literal(1))
-    checkEvaluation(Century(date), 21)
-    checkEvaluation(Century(date.copy(year = Literal(2001))), 21)
-    checkEvaluation(Century(date.copy(year = Literal(2000))), 20)
-    checkEvaluation(Century(date.copy(year = Literal(1001), day = Literal(28))), 11)
-    checkEvaluation(Century(date.copy(year = Literal(1))), 1)
-    checkEvaluation(Century(date.copy(year = Literal(-1))), -1)
-    checkEvaluation(Century(date.copy(year = Literal(-100), month = Literal(12))), -2)
-    checkEvaluation(Century(date.copy(year = Literal(-2019))), -21)
-  }
-
-  test("decade") {
-    val date = MakeDate(Literal(2019), Literal(8), Literal(8))
-    checkEvaluation(Decade(date), 201)
-    checkEvaluation(Decade(date.copy(year = Literal(2011))), 201)
-    checkEvaluation(Decade(date.copy(year = Literal(2010))), 201)
-    checkEvaluation(Decade(date.copy(year = Literal(2009))), 200)
-    checkEvaluation(Decade(date.copy(year = Literal(10))), 1)
-    checkEvaluation(Decade(date.copy(year = Literal(1))), 0)
-    checkEvaluation(Decade(date.copy(year = Literal(-1))), -1)
-    checkEvaluation(Decade(date.copy(year = Literal(-10))), -1)
-    checkEvaluation(Decade(date.copy(year = Literal(-11))), -2)
-    checkEvaluation(Decade(date.copy(year = Literal(-2019))), -202)
-  }
-
-  test("milliseconds and microseconds") {
-    outstandingTimezonesIds.foreach { timezone =>
-      var timestamp = MakeTimestamp(Literal(2019), Literal(8), Literal(10),
-        Literal(0), Literal(0), Literal(Decimal(BigDecimal(10.123456789), 8, 6)),
-        Some(Literal(timezone)), Some(timezone))
-      def millis(ts: MakeTimestamp): Milliseconds = Milliseconds(timestamp, Some(timezone))
-      def micros(ts: MakeTimestamp): Microseconds = Microseconds(timestamp, Some(timezone))
-
-      checkEvaluation(millis(timestamp), Decimal(BigDecimal(10123.457), 8, 3))
-      checkEvaluation(
-        millis(timestamp.copy(year = Literal(10))),
-        Decimal(BigDecimal(10123.457), 8, 3))
-
-      checkEvaluation(micros(timestamp), 10123457)
-      checkEvaluation(
-        micros(timestamp.copy(year = Literal(10))),
-        10123457)
-
-      timestamp = timestamp.copy(sec = Literal(Decimal(0.0, 8, 6)))
-      checkEvaluation(millis(timestamp), Decimal(0, 8, 3))
-      checkEvaluation(micros(timestamp), 0)
-
-      timestamp = timestamp.copy(sec = Literal(Decimal(BigDecimal(59.999999), 8, 6)))
-      checkEvaluation(millis(timestamp), Decimal(BigDecimal(59999.999), 8, 3))
-      checkEvaluation(micros(timestamp), 59999999)
-
-      timestamp = timestamp.copy(sec = Literal(Decimal(BigDecimal(60.0), 8, 6)))
-      checkEvaluation(millis(timestamp), Decimal(0, 8, 3))
-      checkEvaluation(micros(timestamp), 0)
-    }
-  }
-
-  test("epoch") {
-    val zoneId = ZoneId.systemDefault()
-    val nanos = 123456000
-    val timestamp = Epoch(MakeTimestamp(
-      Literal(2019), Literal(8), Literal(9), Literal(0), Literal(0),
-      Literal(Decimal(nanos / NANOS_PER_SECOND.toDouble, 8, 6)),
-      Some(Literal(zoneId.getId))))
-    val instant = LocalDateTime.of(2019, 8, 9, 0, 0, 0, nanos)
-      .atZone(zoneId).toInstant
-    val expected = Decimal(BigDecimal(nanos) / NANOS_PER_SECOND +
-      instant.getEpochSecond +
-      zoneId.getRules.getOffset(instant).getTotalSeconds)
-    checkEvaluation(timestamp, expected)
-  }
-
   test("ISO 8601 week-numbering year") {
-    checkEvaluation(IsoYear(MakeDate(Literal(2006), Literal(1), Literal(1))), 2005)
-    checkEvaluation(IsoYear(MakeDate(Literal(2006), Literal(1), Literal(2))), 2006)
+    checkEvaluation(YearOfWeek(MakeDate(Literal(2006), Literal(1), Literal(1))), 2005)
+    checkEvaluation(YearOfWeek(MakeDate(Literal(2006), Literal(1), Literal(2))), 2006)
   }
 
   test("extract the seconds part with fraction from timestamps") {
diff --git a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
index 41bb42c504c7b..77f1bdb5eaee0 100644
--- a/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-jdk11-results.txt
@@ -2,123 +2,103 @@ Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   322            327           4         31.1          32.2       1.0X
-MILLENNIUM of timestamp                             834            841           8         12.0          83.4       0.4X
-CENTURY of timestamp                                828            841          15         12.1          82.8       0.4X
-DECADE of timestamp                                 813            825          13         12.3          81.3       0.4X
-YEAR of timestamp                                   800            817          18         12.5          80.0       0.4X
-ISOYEAR of timestamp                                882            889           6         11.3          88.2       0.4X
-QUARTER of timestamp                                900            908          12         11.1          90.0       0.4X
-MONTH of timestamp                                  809            816           8         12.4          80.9       0.4X
-WEEK of timestamp                                  1119           1123           4          8.9         111.9       0.3X
-DAY of timestamp                                    801            808           7         12.5          80.1       0.4X
-DAYOFWEEK of timestamp                              946            952           5         10.6          94.6       0.3X
-DOW of timestamp                                    939            945          10         10.6          93.9       0.3X
-ISODOW of timestamp                                 869            874           5         11.5          86.9       0.4X
-DOY of timestamp                                    822            835          14         12.2          82.2       0.4X
-HOUR of timestamp                                   633            647          12         15.8          63.3       0.5X
-MINUTE of timestamp                                 635            636           1         15.8          63.5       0.5X
-SECOND of timestamp                                 774            778           4         12.9          77.4       0.4X
-MILLISECONDS of timestamp                           735            743           7         13.6          73.5       0.4X
-MICROSECONDS of timestamp                           652            657           4         15.3          65.2       0.5X
-EPOCH of timestamp                                  794            800           5         12.6          79.4       0.4X
+cast to timestamp                                   333            345          13         30.0          33.3       1.0X
+YEAR of timestamp                                   826            840          16         12.1          82.6       0.4X
+YEAROFWEEK of timestamp                            1161           1168           7          8.6         116.1       0.3X
+QUARTER of timestamp                                961            966           6         10.4          96.1       0.3X
+MONTH of timestamp                                  831            838           6         12.0          83.1       0.4X
+WEEK of timestamp                                  1155           1159           4          8.7         115.5       0.3X
+DAY of timestamp                                    822            828           6         12.2          82.2       0.4X
+DAYOFWEEK of timestamp                              968            972           6         10.3          96.8       0.3X
+DOW of timestamp                                    960            966           9         10.4          96.0       0.3X
+DOW_ISO of timestamp                                961            967           9         10.4          96.1       0.3X
+DAYOFWEEK_ISO of timestamp                          965            968           5         10.4          96.5       0.3X
+DOY of timestamp                                    864            873           9         11.6          86.4       0.4X
+HOUR of timestamp                                   631            647          14         15.9          63.1       0.5X
+MINUTE of timestamp                                 638            651          12         15.7          63.8       0.5X
+SECOND of timestamp                                 758            760           2         13.2          75.8       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   281            291          11         35.6          28.1       1.0X
-MILLENNIUM of timestamp                             814            825          19         12.3          81.4       0.3X
-CENTURY of timestamp                                842            850           7         11.9          84.2       0.3X
-DECADE of timestamp                                 812            814           3         12.3          81.2       0.3X
-YEAR of timestamp                                   787            798          10         12.7          78.7       0.4X
-ISOYEAR of timestamp                                956            962           6         10.5          95.6       0.3X
-QUARTER of timestamp                                948            954           6         10.6          94.8       0.3X
-MONTH of timestamp                                  797            817          26         12.5          79.7       0.4X
-WEEK of timestamp                                  1126           1134          10          8.9         112.6       0.2X
-DAY of timestamp                                    800            803           2         12.5          80.0       0.4X
-DAYOFWEEK of timestamp                              953            969          16         10.5          95.3       0.3X
-DOW of timestamp                                    978            986           9         10.2          97.8       0.3X
-ISODOW of timestamp                                 907            913           7         11.0          90.7       0.3X
-DOY of timestamp                                    831            844          14         12.0          83.1       0.3X
-HOUR of timestamp                                   646            650           3         15.5          64.6       0.4X
-MINUTE of timestamp                                 639            643           5         15.7          63.9       0.4X
-SECOND of timestamp                                 740            747           6         13.5          74.0       0.4X
-MILLISECONDS of timestamp                           745            752           7         13.4          74.5       0.4X
-MICROSECONDS of timestamp                           653            658           5         15.3          65.3       0.4X
-EPOCH of timestamp                                  724            738          12         13.8          72.4       0.4X
+cast to timestamp                                   301            307           8         33.2          30.1       1.0X
+YEAR of timestamp                                   797            805           7         12.6          79.7       0.4X
+YEAROFWEEK of timestamp                            1147           1156           8          8.7         114.7       0.3X
+QUARTER of timestamp                                965            974           9         10.4          96.5       0.3X
+MONTH of timestamp                                  801            815          17         12.5          80.1       0.4X
+WEEK of timestamp                                  1133           1157          21          8.8         113.3       0.3X
+DAY of timestamp                                    825            828           4         12.1          82.5       0.4X
+DAYOFWEEK of timestamp                              929            937           8         10.8          92.9       0.3X
+DOW of timestamp                                    933            940          11         10.7          93.3       0.3X
+DOW_ISO of timestamp                                944            954          11         10.6          94.4       0.3X
+DAYOFWEEK_ISO of timestamp                          948            965          21         10.5          94.8       0.3X
+DOY of timestamp                                    894            898           6         11.2          89.4       0.3X
+HOUR of timestamp                                   678            683           6         14.7          67.8       0.4X
+MINUTE of timestamp                                 669            670           2         15.0          66.9       0.5X
+SECOND of timestamp                                 762            771          12         13.1          76.2       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        613            621           8         16.3          61.3       1.0X
-MILLENNIUM of date                                  738            741           3         13.6          73.8       0.8X
-CENTURY of date                                     735            737           3         13.6          73.5       0.8X
-DECADE of date                                      722            731           7         13.8          72.2       0.8X
-YEAR of date                                        715            719           4         14.0          71.5       0.9X
-ISOYEAR of date                                     881            883           2         11.4          88.1       0.7X
-QUARTER of date                                     857            867          10         11.7          85.7       0.7X
-MONTH of date                                       719            727           7         13.9          71.9       0.9X
-WEEK of date                                       1038           1043           8          9.6         103.8       0.6X
-DAY of date                                         715            719           4         14.0          71.5       0.9X
-DAYOFWEEK of date                                   861            873          12         11.6          86.1       0.7X
-DOW of date                                         847            865          16         11.8          84.7       0.7X
-ISODOW of date                                      788            796           7         12.7          78.8       0.8X
-DOY of date                                         744            747           4         13.4          74.4       0.8X
-HOUR of date                                       1485           1506          30          6.7         148.5       0.4X
-MINUTE of date                                     1502           1503           1          6.7         150.2       0.4X
-SECOND of date                                     1641           1650           9          6.1         164.1       0.4X
-MILLISECONDS of date                               1624           1639          14          6.2         162.4       0.4X
-MICROSECONDS of date                               1504           1505           2          6.6         150.4       0.4X
-EPOCH of date                                      1733           1748          19          5.8         173.3       0.4X
+cast to date                                        737            751          23         13.6          73.7       1.0X
+YEAR of date                                        804            811          11         12.4          80.4       0.9X
+YEAROFWEEK of date                                 1151           1155           4          8.7         115.1       0.6X
+QUARTER of date                                     964            987          26         10.4          96.4       0.8X
+MONTH of date                                       831            835           6         12.0          83.1       0.9X
+WEEK of date                                       1162           1163           2          8.6         116.2       0.6X
+DAY of date                                         826            833           7         12.1          82.6       0.9X
+DAYOFWEEK of date                                   984           1003          26         10.2          98.4       0.7X
+DOW of date                                         995            997           2         10.0          99.5       0.7X
+DOW_ISO of date                                     985            992           8         10.2          98.5       0.7X
+DAYOFWEEK_ISO of date                               985           1001          19         10.1          98.5       0.7X
+DOY of date                                         884            890           8         11.3          88.4       0.8X
+HOUR of date                                       1630           1640          13          6.1         163.0       0.5X
+MINUTE of date                                     1631           1636           6          6.1         163.1       0.5X
+SECOND of date                                     1737           1739           2          5.8         173.7       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        613            627          12         16.3          61.3       1.0X
-MILLENNIUM of date                                  722            734          11         13.9          72.2       0.8X
-CENTURY of date                                     740            746           8         13.5          74.0       0.8X
-DECADE of date                                      723            726           3         13.8          72.3       0.8X
-YEAR of date                                        705            717          11         14.2          70.5       0.9X
-ISOYEAR of date                                     876            885           9         11.4          87.6       0.7X
-QUARTER of date                                     857            877          21         11.7          85.7       0.7X
-MONTH of date                                       714            715           2         14.0          71.4       0.9X
-WEEK of date                                       1040           1043           4          9.6         104.0       0.6X
-DAY of date                                         703            714          10         14.2          70.3       0.9X
-DAYOFWEEK of date                                   861            869           7         11.6          86.1       0.7X
-DOW of date                                         856            863           8         11.7          85.6       0.7X
-ISODOW of date                                      791            797           5         12.6          79.1       0.8X
-DOY of date                                         721            728           6         13.9          72.1       0.9X
-HOUR of date                                       1488           1493           7          6.7         148.8       0.4X
-MINUTE of date                                     1488           1502          12          6.7         148.8       0.4X
-SECOND of date                                     1636           1651          14          6.1         163.6       0.4X
-MILLISECONDS of date                               1644           1665          34          6.1         164.4       0.4X
-MICROSECONDS of date                               1482           1516          42          6.7         148.2       0.4X
-EPOCH of date                                      1752           1790          34          5.7         175.2       0.4X
+cast to date                                        746            753           8         13.4          74.6       1.0X
+YEAR of date                                        841            843           1         11.9          84.1       0.9X
+YEAROFWEEK of date                                 1169           1191          20          8.6         116.9       0.6X
+QUARTER of date                                    1008           1012           4          9.9         100.8       0.7X
+MONTH of date                                       838            844           6         11.9          83.8       0.9X
+WEEK of date                                       1179           1182           3          8.5         117.9       0.6X
+DAY of date                                         849            850           1         11.8          84.9       0.9X
+DAYOFWEEK of date                                   984            988           4         10.2          98.4       0.8X
+DOW of date                                         986            994           7         10.1          98.6       0.8X
+DOW_ISO of date                                     974            992          17         10.3          97.4       0.8X
+DAYOFWEEK_ISO of date                               979            994          14         10.2          97.9       0.8X
+DOY of date                                         874            878           3         11.4          87.4       0.9X
+HOUR of date                                       1644           1649           6          6.1         164.4       0.5X
+MINUTE of date                                     1610           1626          15          6.2         161.0       0.5X
+SECOND of date                                     1768           1789          30          5.7         176.8       0.4X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    921            929           9         10.9          92.1       1.0X
-YEAR of interval                                    927            942          22         10.8          92.7       1.0X
-MONTH of interval                                   956            974          22         10.5          95.6       1.0X
-DAY of interval                                     897            915          16         11.2          89.7       1.0X
-HOUR of interval                                    952            975          35         10.5          95.2       1.0X
-MINUTE of interval                                  938            955          14         10.7          93.8       1.0X
-SECOND of interval                                 1066           1080          24          9.4         106.6       0.9X
+cast to interval                                    989           1003          13         10.1          98.9       1.0X
+YEAR of interval                                    980            999          16         10.2          98.0       1.0X
+MONTH of interval                                   988           1000          15         10.1          98.8       1.0X
+DAY of interval                                     970            983          11         10.3          97.0       1.0X
+HOUR of interval                                    989           1000          13         10.1          98.9       1.0X
+MINUTE of interval                                 1019           1034          13          9.8         101.9       1.0X
+SECOND of interval                                 1084           1095          10          9.2         108.4       0.9X
 
 Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    944            960          21         10.6          94.4       1.0X
-YEAR of interval                                    978           1011          33         10.2          97.8       1.0X
-MONTH of interval                                   961            984          27         10.4          96.1       1.0X
-DAY of interval                                     921            927           7         10.9          92.1       1.0X
-HOUR of interval                                    913            919           8         11.0          91.3       1.0X
-MINUTE of interval                                  920            941          22         10.9          92.0       1.0X
-SECOND of interval                                  994           1012          19         10.1          99.4       0.9X
+cast to interval                                    955            969          15         10.5          95.5       1.0X
+YEAR of interval                                   1004           1007           3         10.0         100.4       1.0X
+MONTH of interval                                   989            990           2         10.1          98.9       1.0X
+DAY of interval                                    1006           1011           6          9.9         100.6       0.9X
+HOUR of interval                                    986            997          10         10.1          98.6       1.0X
+MINUTE of interval                                 1004           1014          15         10.0         100.4       1.0X
+SECOND of interval                                 1083           1088           6          9.2         108.3       0.9X
 
diff --git a/sql/core/benchmarks/ExtractBenchmark-results.txt b/sql/core/benchmarks/ExtractBenchmark-results.txt
index bc766b47cec21..11144a0547e43 100644
--- a/sql/core/benchmarks/ExtractBenchmark-results.txt
+++ b/sql/core/benchmarks/ExtractBenchmark-results.txt
@@ -1,124 +1,104 @@
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for timestamp:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   306            337          27         32.7          30.6       1.0X
-MILLENNIUM of timestamp                            1002           1012          15         10.0         100.2       0.3X
-CENTURY of timestamp                               1055           1064           8          9.5         105.5       0.3X
-DECADE of timestamp                                 940            968          36         10.6          94.0       0.3X
-YEAR of timestamp                                   922            938          27         10.8          92.2       0.3X
-ISOYEAR of timestamp                               1036           1061          23          9.7         103.6       0.3X
-QUARTER of timestamp                               1081           1098          27          9.2         108.1       0.3X
-MONTH of timestamp                                  876            888          17         11.4          87.6       0.3X
-WEEK of timestamp                                  1218           1259          55          8.2         121.8       0.3X
-DAY of timestamp                                    890            911          33         11.2          89.0       0.3X
-DAYOFWEEK of timestamp                             1006           1019          17          9.9         100.6       0.3X
-DOW of timestamp                                    990           1006          22         10.1          99.0       0.3X
-ISODOW of timestamp                                 953            992          35         10.5          95.3       0.3X
-DOY of timestamp                                    883            889           6         11.3          88.3       0.3X
-HOUR of timestamp                                   661            667           6         15.1          66.1       0.5X
-MINUTE of timestamp                                 665            678          12         15.0          66.5       0.5X
-SECOND of timestamp                                 787            791           6         12.7          78.7       0.4X
-MILLISECONDS of timestamp                           800            814          12         12.5          80.0       0.4X
-MICROSECONDS of timestamp                           700            706           8         14.3          70.0       0.4X
-EPOCH of timestamp                                  793            819          40         12.6          79.3       0.4X
+cast to timestamp                                   292            310          16         34.3          29.2       1.0X
+YEAR of timestamp                                   847            875          26         11.8          84.7       0.3X
+YEAROFWEEK of timestamp                             964            981          24         10.4          96.4       0.3X
+QUARTER of timestamp                               1217           1219           2          8.2         121.7       0.2X
+MONTH of timestamp                                  835            844          10         12.0          83.5       0.3X
+WEEK of timestamp                                  1173           1183          15          8.5         117.3       0.2X
+DAY of timestamp                                    851            878          25         11.7          85.1       0.3X
+DAYOFWEEK of timestamp                              946            970          22         10.6          94.6       0.3X
+DOW of timestamp                                    935            959          21         10.7          93.5       0.3X
+DOW_ISO of timestamp                                947            961          13         10.6          94.7       0.3X
+DAYOFWEEK_ISO of timestamp                          965            992          26         10.4          96.5       0.3X
+DOY of timestamp                                    886            904          26         11.3          88.6       0.3X
+HOUR of timestamp                                   697            700           4         14.3          69.7       0.4X
+MINUTE of timestamp                                 654            665          10         15.3          65.4       0.4X
+SECOND of timestamp                                 770            778           8         13.0          77.0       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for timestamp:           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp                                   252            259           7         39.7          25.2       1.0X
-MILLENNIUM of timestamp                             917            936          18         10.9          91.7       0.3X
-CENTURY of timestamp                                943            952          12         10.6          94.3       0.3X
-DECADE of timestamp                                 863            867           5         11.6          86.3       0.3X
-YEAR of timestamp                                   839            854          16         11.9          83.9       0.3X
-ISOYEAR of timestamp                               1023           1027           4          9.8         102.3       0.2X
-QUARTER of timestamp                               1052           1060          10          9.5         105.2       0.2X
-MONTH of timestamp                                  835            846          14         12.0          83.5       0.3X
-WEEK of timestamp                                  1182           1198          14          8.5         118.2       0.2X
-DAY of timestamp                                    837            845          10         11.9          83.7       0.3X
-DAYOFWEEK of timestamp                              947            962          14         10.6          94.7       0.3X
-DOW of timestamp                                    946            964          17         10.6          94.6       0.3X
-ISODOW of timestamp                                 934            946          13         10.7          93.4       0.3X
-DOY of timestamp                                    857            859           3         11.7          85.7       0.3X
-HOUR of timestamp                                   660            661           1         15.1          66.0       0.4X
-MINUTE of timestamp                                 649            654           4         15.4          64.9       0.4X
-SECOND of timestamp                                 801            805           5         12.5          80.1       0.3X
-MILLISECONDS of timestamp                           820            825           7         12.2          82.0       0.3X
-MICROSECONDS of timestamp                           700            708          12         14.3          70.0       0.4X
-EPOCH of timestamp                                  789            797          10         12.7          78.9       0.3X
+cast to timestamp                                   233            243           9         43.0          23.3       1.0X
+YEAR of timestamp                                   810            826          15         12.3          81.0       0.3X
+YEAROFWEEK of timestamp                             996           1019          21         10.0          99.6       0.2X
+QUARTER of timestamp                               1037           1049          11          9.6         103.7       0.2X
+MONTH of timestamp                                  822            852          30         12.2          82.2       0.3X
+WEEK of timestamp                                  1179           1220          35          8.5         117.9       0.2X
+DAY of timestamp                                    822            825           3         12.2          82.2       0.3X
+DAYOFWEEK of timestamp                              937            941           3         10.7          93.7       0.2X
+DOW of timestamp                                    931            970          34         10.7          93.1       0.2X
+DOW_ISO of timestamp                                927            948          22         10.8          92.7       0.3X
+DAYOFWEEK_ISO of timestamp                          896            918          20         11.2          89.6       0.3X
+DOY of timestamp                                    863            891          25         11.6          86.3       0.3X
+HOUR of timestamp                                   639            645           6         15.7          63.9       0.4X
+MINUTE of timestamp                                 639            647          12         15.7          63.9       0.4X
+SECOND of timestamp                                 785            796          11         12.7          78.5       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for date:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        747            751           6         13.4          74.7       1.0X
-MILLENNIUM of date                                  941            947           5         10.6          94.1       0.8X
-CENTURY of date                                     952            961           8         10.5          95.2       0.8X
-DECADE of date                                      864            877          15         11.6          86.4       0.9X
-YEAR of date                                        850            862          11         11.8          85.0       0.9X
-ISOYEAR of date                                    1031           1036           8          9.7         103.1       0.7X
-QUARTER of date                                    1038           1048           9          9.6         103.8       0.7X
-MONTH of date                                       852            865          12         11.7          85.2       0.9X
-WEEK of date                                       1174           1178           5          8.5         117.4       0.6X
-DAY of date                                         830            836           5         12.0          83.0       0.9X
-DAYOFWEEK of date                                   949            972          21         10.5          94.9       0.8X
-DOW of date                                         960            969           8         10.4          96.0       0.8X
-ISODOW of date                                      918            928           9         10.9          91.8       0.8X
-DOY of date                                         858            867          11         11.7          85.8       0.9X
-HOUR of date                                       1584           1591          10          6.3         158.4       0.5X
-MINUTE of date                                     1583           1589           6          6.3         158.3       0.5X
-SECOND of date                                     1721           1728           7          5.8         172.1       0.4X
-MILLISECONDS of date                               1711           1731          17          5.8         171.1       0.4X
-MICROSECONDS of date                               1647           1656           8          6.1         164.7       0.5X
-EPOCH of date                                      1796           1805          10          5.6         179.6       0.4X
+cast to date                                        690            693           3         14.5          69.0       1.0X
+YEAR of date                                        822            841          17         12.2          82.2       0.8X
+YEAROFWEEK of date                                  967            974           7         10.3          96.7       0.7X
+QUARTER of date                                    1034           1044          11          9.7         103.4       0.7X
+MONTH of date                                       831            836           5         12.0          83.1       0.8X
+WEEK of date                                       1152           1177          34          8.7         115.2       0.6X
+DAY of date                                         836            873          34         12.0          83.6       0.8X
+DAYOFWEEK of date                                   960            992          30         10.4          96.0       0.7X
+DOW of date                                        1011           1016           4          9.9         101.1       0.7X
+DOW_ISO of date                                     969            984          16         10.3          96.9       0.7X
+DAYOFWEEK_ISO of date                               967            986          19         10.3          96.7       0.7X
+DOY of date                                         901            953          47         11.1          90.1       0.8X
+HOUR of date                                       1581           1586           5          6.3         158.1       0.4X
+MINUTE of date                                     1570           1584          13          6.4         157.0       0.4X
+SECOND of date                                     1713           1740          27          5.8         171.3       0.4X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for date:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date                                        731            740           8         13.7          73.1       1.0X
-MILLENNIUM of date                                  939            948          12         10.7          93.9       0.8X
-CENTURY of date                                     929            940          10         10.8          92.9       0.8X
-DECADE of date                                      845            861          15         11.8          84.5       0.9X
-YEAR of date                                        848            853           5         11.8          84.8       0.9X
-ISOYEAR of date                                    1011           1027          14          9.9         101.1       0.7X
-QUARTER of date                                    1038           1050          12          9.6         103.8       0.7X
-MONTH of date                                       847            872          22         11.8          84.7       0.9X
-WEEK of date                                       1143           1152          13          8.7         114.3       0.6X
-DAY of date                                         844            848           5         11.8          84.4       0.9X
-DAYOFWEEK of date                                   947            957           9         10.6          94.7       0.8X
-DOW of date                                         963            972           8         10.4          96.3       0.8X
-ISODOW of date                                      906            919          11         11.0          90.6       0.8X
-DOY of date                                         866            868           3         11.6          86.6       0.8X
-HOUR of date                                       1599           1604           7          6.3         159.9       0.5X
-MINUTE of date                                     1578           1594          13          6.3         157.8       0.5X
-SECOND of date                                     1722           1731          11          5.8         172.2       0.4X
-MILLISECONDS of date                               1699           1730          27          5.9         169.9       0.4X
-MICROSECONDS of date                               1620           1623           5          6.2         162.0       0.5X
-EPOCH of date                                      1742           1767          32          5.7         174.2       0.4X
+cast to date                                        756            762           6         13.2          75.6       1.0X
+YEAR of date                                        843            857          13         11.9          84.3       0.9X
+YEAROFWEEK of date                                 1055           1065          16          9.5         105.5       0.7X
+QUARTER of date                                    1066           1073           6          9.4         106.6       0.7X
+MONTH of date                                       856            890          44         11.7          85.6       0.9X
+WEEK of date                                       1155           1204          59          8.7         115.5       0.7X
+DAY of date                                         749            762          12         13.3          74.9       1.0X
+DAYOFWEEK of date                                   850            865          15         11.8          85.0       0.9X
+DOW of date                                         878            893          16         11.4          87.8       0.9X
+DOW_ISO of date                                     865            869           5         11.6          86.5       0.9X
+DAYOFWEEK_ISO of date                               914            967          76         10.9          91.4       0.8X
+DOY of date                                         789            792           4         12.7          78.9       1.0X
+HOUR of date                                       1558           1659         168          6.4         155.8       0.5X
+MINUTE of date                                     1581           1673          80          6.3         158.1       0.5X
+SECOND of date                                     1646           1881         319          6.1         164.6       0.5X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke extract for interval:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    920            941          18         10.9          92.0       1.0X
-YEAR of interval                                    955            967          12         10.5          95.5       1.0X
-MONTH of interval                                   954            966          11         10.5          95.4       1.0X
-DAY of interval                                     967            970           4         10.3          96.7       1.0X
-HOUR of interval                                    959            975          21         10.4          95.9       1.0X
-MINUTE of interval                                  966            981          16         10.4          96.6       1.0X
-SECOND of interval                                 1048           1066          16          9.5         104.8       0.9X
+cast to interval                                    925            941          20         10.8          92.5       1.0X
+YEAR of interval                                    903            919          14         11.1          90.3       1.0X
+MONTH of interval                                   944            958          17         10.6          94.4       1.0X
+DAY of interval                                     917            925           7         10.9          91.7       1.0X
+HOUR of interval                                    925            940          17         10.8          92.5       1.0X
+MINUTE of interval                                  951            962          12         10.5          95.1       1.0X
+SECOND of interval                                 1017           1036          19          9.8         101.7       0.9X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
 Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 Invoke date_part for interval:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to interval                                    928            954          26         10.8          92.8       1.0X
-YEAR of interval                                    955            960           4         10.5          95.5       1.0X
-MONTH of interval                                   956            971          13         10.5          95.6       1.0X
-DAY of interval                                     964           1003          41         10.4          96.4       1.0X
-HOUR of interval                                   1057           1293         248          9.5         105.7       0.9X
-MINUTE of interval                                  975           1091         155         10.3          97.5       1.0X
-SECOND of interval                                 1104           1286         311          9.1         110.4       0.8X
+cast to interval                                    913            920           8         11.0          91.3       1.0X
+YEAR of interval                                    930            935           4         10.8          93.0       1.0X
+MONTH of interval                                   930            943          14         10.7          93.0       1.0X
+DAY of interval                                     933            946          12         10.7          93.3       1.0X
+HOUR of interval                                    951            953           3         10.5          95.1       1.0X
+MINUTE of interval                                  923            958          30         10.8          92.3       1.0X
+SECOND of interval                                  993            995           1         10.1          99.3       0.9X
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql b/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
deleted file mode 100644
index b36b98b261365..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/date_part.sql
+++ /dev/null
@@ -1,111 +0,0 @@
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c;
-
-select date_part('millennium', c) from t;
-select date_part('millennia', c) from t;
-select date_part('mil', c) from t;
-select date_part('mils', c) from t;
-
-select date_part('century', c) from t;
-select date_part('centuries', c) from t;
-select date_part('c', c) from t;
-select date_part('cent', c) from t;
-
-select date_part('decade', c) from t;
-select date_part('decades', c) from t;
-select date_part('dec', c) from t;
-select date_part('decs', c) from t;
-
-select date_part('year', c) from t;
-select date_part('y', c) from t;
-select date_part('years', c) from t;
-select date_part('yr', c) from t;
-select date_part('yrs', c) from t;
-
-select date_part('quarter', c) from t;
-select date_part('qtr', c) from t;
-
-select date_part('month', c) from t;
-select date_part('mon', c) from t;
-select date_part('mons', c) from t;
-select date_part('months', c) from t;
-
-select date_part('week', c) from t;
-select date_part('w', c) from t;
-select date_part('weeks', c) from t;
-
-select date_part('day', c) from t;
-select date_part('d', c) from t;
-select date_part('days', c) from t;
-
-select date_part('dayofweek', c) from t;
-
-select date_part('dow', c) from t;
-
-select date_part('isodow', c) from t;
-
-select date_part('doy', c) from t;
-
-select date_part('hour', c) from t;
-select date_part('h', c) from t;
-select date_part('hours', c) from t;
-select date_part('hr', c) from t;
-select date_part('hrs', c) from t;
-
-select date_part('minute', c) from t;
-select date_part('m', c) from t;
-select date_part('min', c) from t;
-select date_part('mins', c) from t;
-select date_part('minutes', c) from t;
-
-select date_part('second', c) from t;
-select date_part('s', c) from t;
-select date_part('sec', c) from t;
-select date_part('seconds', c) from t;
-select date_part('secs', c) from t;
-
-select date_part('not_supported', c) from t;
-
-select date_part(c, c) from t;
-
-select date_part(null, c) from t;
-
-CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c;
-
-select date_part('year', c) from t2;
-select date_part('y', c) from t2;
-select date_part('years', c) from t2;
-select date_part('yr', c) from t2;
-select date_part('yrs', c) from t2;
-
-select date_part('month', c) from t2;
-select date_part('mon', c) from t2;
-select date_part('mons', c) from t2;
-select date_part('months', c) from t2;
-
-select date_part('day', c) from t2;
-select date_part('d', c) from t2;
-select date_part('days', c) from t2;
-
-select date_part('hour', c) from t2;
-select date_part('h', c) from t2;
-select date_part('hours', c) from t2;
-select date_part('hr', c) from t2;
-select date_part('hrs', c) from t2;
-
-select date_part('minute', c) from t2;
-select date_part('m', c) from t2;
-select date_part('min', c) from t2;
-select date_part('mins', c) from t2;
-select date_part('minutes', c) from t2;
-
-select date_part('second', c) from t2;
-select date_part('s', c) from t2;
-select date_part('sec', c) from t2;
-select date_part('seconds', c) from t2;
-select date_part('secs', c) from t2;
-
-select date_part('not_supported', c) from t2;
-
-select date_part(c, c) from t2;
-
-select date_part(null, c) from t2;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
index 12068d22a4e60..cba7fc22779c5 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
@@ -1,27 +1,12 @@
 CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c, interval 10 year 20 month 30 day 40 hour 50 minute 6.7890 second as i;
 
-select extract(millennium from c) from t;
-select extract(millennia from c) from t;
-select extract(mil from c) from t;
-select extract(mils from c) from t;
-
-select extract(century from c) from t;
-select extract(centuries from c) from t;
-select extract(c from c) from t;
-select extract(cent from c)from t;
-
-select extract(decade from c) from t;
-select extract(decades from c) from t;
-select extract(dec from c) from t;
-select extract(decs from c) from t;
-
 select extract(year from c), extract(year from i) from t;
 select extract(y from c), extract(y from i) from t;
 select extract(years from c), extract(years from i) from t;
 select extract(yr from c), extract(yr from i) from t;
 select extract(yrs from c), extract(yrs from i) from t;
 
-select extract(isoyear from c) from t;
+select extract(yearofweek from c) from t;
 
 select extract(quarter from c) from t;
 select extract(qtr from c) from t;
@@ -40,10 +25,10 @@ select extract(d from c), extract(d from i) from t;
 select extract(days from c), extract(days from i) from t;
 
 select extract(dayofweek from c) from t;
-
 select extract(dow from c) from t;
 
-select extract(isodow from c) from t;
+select extract(dayofweek_iso from c) from t;
+select extract(dow_iso from c) from t;
 
 select extract(doy from c) from t;
 
@@ -65,21 +50,62 @@ select extract(sec from c), extract(sec from i) from t;
 select extract(seconds from c), extract(seconds from i) from t;
 select extract(secs from c), extract(secs from i) from t;
 
-select extract(milliseconds from c) from t;
-select extract(msec from c) from t;
-select extract(msecs from c) from t;
-select extract(millisecon from c) from t;
-select extract(mseconds from c) from t;
-select extract(ms from c) from t;
+select extract(not_supported from c) from t;
+select extract(not_supported from i) from t;
+
+select date_part('year', c), date_part('year', i) from t;
+select date_part('y', c), date_part('y', i) from t;
+select date_part('years', c), date_part('years', i) from t;
+select date_part('yr', c), date_part('yr', i) from t;
+select date_part('yrs', c), date_part('yrs', i) from t;
 
-select extract(microseconds from c) from t;
-select extract(usec from c) from t;
-select extract(usecs from c) from t;
-select extract(useconds from c) from t;
-select extract(microsecon from c) from t;
-select extract(us from c) from t;
+select date_part('yearofweek', c) from t;
 
-select extract(epoch from c) from t;
+select date_part('quarter', c) from t;
+select date_part('qtr', c) from t;
 
-select extract(not_supported from c) from t;
-select extract(not_supported from i) from t;
+select date_part('month', c), date_part('month', i) from t;
+select date_part('mon', c), date_part('mon', i) from t;
+select date_part('mons', c), date_part('mons', i) from t;
+select date_part('months', c), date_part('months', i) from t;
+
+select date_part('week', c) from t;
+select date_part('w', c) from t;
+select date_part('weeks', c) from t;
+
+select date_part('day', c), date_part('day', i) from t;
+select date_part('d', c), date_part('d', i) from t;
+select date_part('days', c), date_part('days', i) from t;
+
+select date_part('dayofweek', c) from t;
+select date_part('dow', c) from t;
+
+select date_part('dayofweek_iso', c) from t;
+select date_part('dow_iso', c) from t;
+
+select date_part('doy', c) from t;
+
+select date_part('hour', c), date_part('hour', i) from t;
+select date_part('h', c), date_part('h', i) from t;
+select date_part('hours', c), date_part('hours', i) from t;
+select date_part('hr', c), date_part('hr', i) from t;
+select date_part('hrs', c), date_part('hrs', i) from t;
+
+select date_part('minute', c), date_part('minute', i) from t;
+select date_part('m', c), date_part('m', i) from t;
+select date_part('min', c), date_part('min', i) from t;
+select date_part('mins', c), date_part('mins', i) from t;
+select date_part('minutes', c), date_part('minutes', i) from t;
+
+select date_part('second', c), date_part('second', i) from t;
+select date_part('s', c), date_part('s', i) from t;
+select date_part('sec', c), date_part('sec', i) from t;
+select date_part('seconds', c), date_part('seconds', i) from t;
+select date_part('secs', c), date_part('secs', i) from t;
+
+select date_part('not_supported', c) from t;
+select date_part(c, c) from t;
+select date_part(null, c) from t;
+
+select date_part(i, i) from t;
+select date_part(null, i) from t;
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql
index 0bab2f884d976..69851080847b9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/date.sql
@@ -230,49 +230,49 @@ SELECT date 'tomorrow' - date 'yesterday' AS `Two days`;
 --
 -- epoch
 --
-SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01');     --  0
-SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01');     --  0
+-- SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01');     --  0
+-- SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01');     --  0
 -- SELECT EXTRACT(EPOCH FROM TIMESTAMPTZ '1970-01-01+00');  --  0
 --
 -- century
 --
-SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')); -- -2
-SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')); -- -1
-SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01');    --  1
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD'); --  1
-SELECT EXTRACT(CENTURY FROM DATE '1900-12-31');    -- 19
-SELECT EXTRACT(CENTURY FROM DATE '1901-01-01');    -- 20
-SELECT EXTRACT(CENTURY FROM DATE '2000-12-31');    -- 20
-SELECT EXTRACT(CENTURY FROM DATE '2001-01-01');    -- 21
-SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True;     -- true
+-- SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G')); -- -2
+-- SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G')); -- -1
+-- SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
+-- SELECT EXTRACT(CENTURY FROM DATE '0001-01-01');    --  1
+-- SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD'); --  1
+-- SELECT EXTRACT(CENTURY FROM DATE '1900-12-31');    -- 19
+-- SELECT EXTRACT(CENTURY FROM DATE '1901-01-01');    -- 20
+-- SELECT EXTRACT(CENTURY FROM DATE '2000-12-31');    -- 20
+-- SELECT EXTRACT(CENTURY FROM DATE '2001-01-01');    -- 21
+-- SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True;     -- true
 --
 -- millennium
 --
-SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
-SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD'); --  1
-SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31');    --  1
-SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01');    --  2
-SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31');    --  2
-SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01');    --  3
+-- SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G')); -- -1
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD'); --  1
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31');    --  1
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01');    --  2
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31');    --  2
+-- SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01');    --  3
 -- next test to be fixed on the turn of the next millennium;-)
-SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE);         --  3
+-- SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE);         --  3
 --
 -- decade
 --
-SELECT EXTRACT(DECADE FROM DATE '1994-12-25');    -- 199
-SELECT EXTRACT(DECADE FROM DATE '0010-01-01');    --   1
-SELECT EXTRACT(DECADE FROM DATE '0009-12-31');    --   0
-SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')); --   0
-SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); --  -1
-SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')); --  -1
-SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')); --  -2
+-- SELECT EXTRACT(DECADE FROM DATE '1994-12-25');    -- 199
+-- SELECT EXTRACT(DECADE FROM DATE '0010-01-01');    --   1
+-- SELECT EXTRACT(DECADE FROM DATE '0009-12-31');    --   0
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G')); --   0
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); --  -1
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G')); --  -1
+-- SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G')); --  -2
 --
 -- some other types:
 --
 -- on a timestamp.
-SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True;       -- true
-SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20
+-- SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True;       -- true
+-- SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20
 -- on an interval
 -- SELECT EXTRACT(CENTURY FROM INTERVAL '100 y');  -- 1
 -- SELECT EXTRACT(CENTURY FROM INTERVAL '99 y');   -- 0
@@ -280,16 +280,16 @@ SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000'); -- 20
 -- SELECT EXTRACT(CENTURY FROM INTERVAL '-100 y'); -- -1
 --
 -- test trunc function!
-SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1001
-SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20'); -- 1001-01-01
-SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1901
-SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20'); -- 1901
-SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10'); -- 2001-01-01
-SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04'); -- 0001-01-01
-SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G')); -- 0100-01-01 BC
-SELECT DATE_TRUNC('DECADE', DATE '1993-12-25'); -- 1990-01-01
-SELECT DATE_TRUNC('DECADE', DATE '0004-12-25'); -- 0001-01-01 BC
-SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); -- 0011-01-01 BC
+-- SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1001
+-- SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20'); -- 1001-01-01
+-- SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000'); -- 1901
+-- SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20'); -- 1901
+-- SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10'); -- 2001-01-01
+-- SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04'); -- 0001-01-01
+-- SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G')); -- 0100-01-01 BC
+-- SELECT DATE_TRUNC('DECADE', DATE '1993-12-25'); -- 1990-01-01
+-- SELECT DATE_TRUNC('DECADE', DATE '0004-12-25'); -- 0001-01-01 BC
+-- SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G')); -- 0011-01-01 BC
 
 -- [SPARK-29006] Support special date/timestamp values `infinity`/`-infinity`
 --
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql
index bf69da295a960..ade29cc41358e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/timestamp.sql
@@ -200,15 +200,15 @@ SELECT '' AS `54`, d1 as `timestamp`,
     date_part( 'minute', d1) AS `minute`, date_part( 'second', d1) AS `second`
     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
 
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
-    date_part( 'usec', d1) AS usec
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
-
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
-    date_part( 'dow', d1) AS dow
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
+-- SELECT '' AS `54`, d1 as `timestamp`,
+--     date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
+--     date_part( 'usec', d1) AS usec
+--     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
+
+-- SELECT '' AS `54`, d1 as `timestamp`,
+--     date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
+--     date_part( 'dow', d1) AS dow
+--     FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01';
 
 -- [SPARK-28137] Data Type Formatting Functions
 -- TO_CHAR()
diff --git a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out b/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
deleted file mode 100644
index 028448b4e3a3c..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/date_part.sql.out
+++ /dev/null
@@ -1,670 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 83
-
-
--- !query
-CREATE TEMPORARY VIEW t AS select '2011-05-06 07:08:09.1234567' as c
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-select date_part('millennium', c) from t
--- !query schema
-struct<date_part('millennium', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('millennia', c) from t
--- !query schema
-struct<date_part('millennia', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('mil', c) from t
--- !query schema
-struct<date_part('mil', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('mils', c) from t
--- !query schema
-struct<date_part('mils', t.`c`):int>
--- !query output
-3
-
-
--- !query
-select date_part('century', c) from t
--- !query schema
-struct<date_part('century', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('centuries', c) from t
--- !query schema
-struct<date_part('centuries', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('c', c) from t
--- !query schema
-struct<date_part('c', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('cent', c) from t
--- !query schema
-struct<date_part('cent', t.`c`):int>
--- !query output
-21
-
-
--- !query
-select date_part('decade', c) from t
--- !query schema
-struct<date_part('decade', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('decades', c) from t
--- !query schema
-struct<date_part('decades', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('dec', c) from t
--- !query schema
-struct<date_part('dec', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('decs', c) from t
--- !query schema
-struct<date_part('decs', t.`c`):int>
--- !query output
-201
-
-
--- !query
-select date_part('year', c) from t
--- !query schema
-struct<date_part('year', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('y', c) from t
--- !query schema
-struct<date_part('y', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('years', c) from t
--- !query schema
-struct<date_part('years', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('yr', c) from t
--- !query schema
-struct<date_part('yr', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('yrs', c) from t
--- !query schema
-struct<date_part('yrs', t.`c`):int>
--- !query output
-2011
-
-
--- !query
-select date_part('quarter', c) from t
--- !query schema
-struct<date_part('quarter', t.`c`):int>
--- !query output
-2
-
-
--- !query
-select date_part('qtr', c) from t
--- !query schema
-struct<date_part('qtr', t.`c`):int>
--- !query output
-2
-
-
--- !query
-select date_part('month', c) from t
--- !query schema
-struct<date_part('month', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('mon', c) from t
--- !query schema
-struct<date_part('mon', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('mons', c) from t
--- !query schema
-struct<date_part('mons', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('months', c) from t
--- !query schema
-struct<date_part('months', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('week', c) from t
--- !query schema
-struct<date_part('week', t.`c`):int>
--- !query output
-18
-
-
--- !query
-select date_part('w', c) from t
--- !query schema
-struct<date_part('w', t.`c`):int>
--- !query output
-18
-
-
--- !query
-select date_part('weeks', c) from t
--- !query schema
-struct<date_part('weeks', t.`c`):int>
--- !query output
-18
-
-
--- !query
-select date_part('day', c) from t
--- !query schema
-struct<date_part('day', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('d', c) from t
--- !query schema
-struct<date_part('d', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('days', c) from t
--- !query schema
-struct<date_part('days', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('dayofweek', c) from t
--- !query schema
-struct<date_part('dayofweek', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('dow', c) from t
--- !query schema
-struct<date_part('dow', t.`c`):int>
--- !query output
-6
-
-
--- !query
-select date_part('isodow', c) from t
--- !query schema
-struct<date_part('isodow', t.`c`):int>
--- !query output
-5
-
-
--- !query
-select date_part('doy', c) from t
--- !query schema
-struct<date_part('doy', t.`c`):int>
--- !query output
-126
-
-
--- !query
-select date_part('hour', c) from t
--- !query schema
-struct<date_part('hour', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('h', c) from t
--- !query schema
-struct<date_part('h', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('hours', c) from t
--- !query schema
-struct<date_part('hours', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('hr', c) from t
--- !query schema
-struct<date_part('hr', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('hrs', c) from t
--- !query schema
-struct<date_part('hrs', t.`c`):int>
--- !query output
-7
-
-
--- !query
-select date_part('minute', c) from t
--- !query schema
-struct<date_part('minute', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('m', c) from t
--- !query schema
-struct<date_part('m', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('min', c) from t
--- !query schema
-struct<date_part('min', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('mins', c) from t
--- !query schema
-struct<date_part('mins', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('minutes', c) from t
--- !query schema
-struct<date_part('minutes', t.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('second', c) from t
--- !query schema
-struct<date_part('second', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('s', c) from t
--- !query schema
-struct<date_part('s', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('sec', c) from t
--- !query schema
-struct<date_part('sec', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('seconds', c) from t
--- !query schema
-struct<date_part('seconds', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('secs', c) from t
--- !query schema
-struct<date_part('secs', t.`c`):decimal(8,6)>
--- !query output
-9.123456
-
-
--- !query
-select date_part('not_supported', c) from t
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
-
-
--- !query
-select date_part(c, c) from t
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-The field parameter needs to be a foldable string value.;; line 1 pos 7
-
-
--- !query
-select date_part(null, c) from t
--- !query schema
-struct<date_part(NULL, t.`c`):double>
--- !query output
-NULL
-
-
--- !query
-CREATE TEMPORARY VIEW t2 AS select interval 1010 year 9 month 8 day 7 hour 6 minute 5 second 4 millisecond 3 microsecond as c
--- !query schema
-struct<>
--- !query output
-
-
-
--- !query
-select date_part('year', c) from t2
--- !query schema
-struct<date_part('year', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('y', c) from t2
--- !query schema
-struct<date_part('y', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('years', c) from t2
--- !query schema
-struct<date_part('years', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('yr', c) from t2
--- !query schema
-struct<date_part('yr', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('yrs', c) from t2
--- !query schema
-struct<date_part('yrs', t2.`c`):int>
--- !query output
-1010
-
-
--- !query
-select date_part('month', c) from t2
--- !query schema
-struct<date_part('month', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('mon', c) from t2
--- !query schema
-struct<date_part('mon', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('mons', c) from t2
--- !query schema
-struct<date_part('mons', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('months', c) from t2
--- !query schema
-struct<date_part('months', t2.`c`):tinyint>
--- !query output
-9
-
-
--- !query
-select date_part('day', c) from t2
--- !query schema
-struct<date_part('day', t2.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('d', c) from t2
--- !query schema
-struct<date_part('d', t2.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('days', c) from t2
--- !query schema
-struct<date_part('days', t2.`c`):int>
--- !query output
-8
-
-
--- !query
-select date_part('hour', c) from t2
--- !query schema
-struct<date_part('hour', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('h', c) from t2
--- !query schema
-struct<date_part('h', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('hours', c) from t2
--- !query schema
-struct<date_part('hours', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('hr', c) from t2
--- !query schema
-struct<date_part('hr', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('hrs', c) from t2
--- !query schema
-struct<date_part('hrs', t2.`c`):bigint>
--- !query output
-7
-
-
--- !query
-select date_part('minute', c) from t2
--- !query schema
-struct<date_part('minute', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('m', c) from t2
--- !query schema
-struct<date_part('m', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('min', c) from t2
--- !query schema
-struct<date_part('min', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('mins', c) from t2
--- !query schema
-struct<date_part('mins', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('minutes', c) from t2
--- !query schema
-struct<date_part('minutes', t2.`c`):tinyint>
--- !query output
-6
-
-
--- !query
-select date_part('second', c) from t2
--- !query schema
-struct<date_part('second', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('s', c) from t2
--- !query schema
-struct<date_part('s', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('sec', c) from t2
--- !query schema
-struct<date_part('sec', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('seconds', c) from t2
--- !query schema
-struct<date_part('seconds', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('secs', c) from t2
--- !query schema
-struct<date_part('secs', t2.`c`):decimal(8,6)>
--- !query output
-5.004003
-
-
--- !query
-select date_part('not_supported', c) from t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
-
-
--- !query
-select date_part(c, c) from t2
--- !query schema
-struct<>
--- !query output
-org.apache.spark.sql.AnalysisException
-The field parameter needs to be a foldable string value.;; line 1 pos 7
-
-
--- !query
-select date_part(null, c) from t2
--- !query schema
-struct<date_part(NULL, t2.`c`):double>
--- !query output
-NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index e6635e608651e..220acfb0bce98 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 65
+-- Number of queries: 84
 
 
 -- !query
@@ -10,102 +10,6 @@ struct<>
 
 
 
--- !query
-select extract(millennium from c) from t
--- !query schema
-struct<extract('millennium' FROM t.`c`):int>
--- !query output
-3
-
-
--- !query
-select extract(millennia from c) from t
--- !query schema
-struct<extract('millennia' FROM t.`c`):int>
--- !query output
-3
-
-
--- !query
-select extract(mil from c) from t
--- !query schema
-struct<extract('mil' FROM t.`c`):int>
--- !query output
-3
-
-
--- !query
-select extract(mils from c) from t
--- !query schema
-struct<extract('mils' FROM t.`c`):int>
--- !query output
-3
-
-
--- !query
-select extract(century from c) from t
--- !query schema
-struct<extract('century' FROM t.`c`):int>
--- !query output
-21
-
-
--- !query
-select extract(centuries from c) from t
--- !query schema
-struct<extract('centuries' FROM t.`c`):int>
--- !query output
-21
-
-
--- !query
-select extract(c from c) from t
--- !query schema
-struct<extract('c' FROM t.`c`):int>
--- !query output
-21
-
-
--- !query
-select extract(cent from c)from t
--- !query schema
-struct<extract('cent' FROM t.`c`):int>
--- !query output
-21
-
-
--- !query
-select extract(decade from c) from t
--- !query schema
-struct<extract('decade' FROM t.`c`):int>
--- !query output
-201
-
-
--- !query
-select extract(decades from c) from t
--- !query schema
-struct<extract('decades' FROM t.`c`):int>
--- !query output
-201
-
-
--- !query
-select extract(dec from c) from t
--- !query schema
-struct<extract('dec' FROM t.`c`):int>
--- !query output
-201
-
-
--- !query
-select extract(decs from c) from t
--- !query schema
-struct<extract('decs' FROM t.`c`):int>
--- !query output
-201
-
-
 -- !query
 select extract(year from c), extract(year from i) from t
 -- !query schema
@@ -147,9 +51,9 @@ struct<extract('yrs' FROM t.`c`):int,extract('yrs' FROM t.`i`):int>
 
 
 -- !query
-select extract(isoyear from c) from t
+select extract(yearofweek from c) from t
 -- !query schema
-struct<extract('isoyear' FROM t.`c`):int>
+struct<extract('yearofweek' FROM t.`c`):int>
 -- !query output
 2011
 
@@ -267,9 +171,17 @@ struct<extract('dow' FROM t.`c`):int>
 
 
 -- !query
-select extract(isodow from c) from t
+select extract(dayofweek_iso from c) from t
+-- !query schema
+struct<extract('dayofweek_iso' FROM t.`c`):int>
+-- !query output
+5
+
+
+-- !query
+select extract(dow_iso from c) from t
 -- !query schema
-struct<extract('isodow' FROM t.`c`):int>
+struct<extract('dow_iso' FROM t.`c`):int>
 -- !query output
 5
 
@@ -403,111 +315,329 @@ struct<extract('secs' FROM t.`c`):decimal(8,6),extract('secs' FROM t.`i`):decima
 
 
 -- !query
-select extract(milliseconds from c) from t
+select extract(not_supported from c) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the string type.;; line 1 pos 7
+
+
+-- !query
+select extract(not_supported from i) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
+
+
+-- !query
+select date_part('year', c), date_part('year', i) from t
 -- !query schema
-struct<extract('milliseconds' FROM t.`c`):decimal(8,3)>
+struct<date_part('year', t.`c`):int,date_part('year', t.`i`):int>
 -- !query output
-9123.456
+2011	11
 
 
 -- !query
-select extract(msec from c) from t
+select date_part('y', c), date_part('y', i) from t
 -- !query schema
-struct<extract('msec' FROM t.`c`):decimal(8,3)>
+struct<date_part('y', t.`c`):int,date_part('y', t.`i`):int>
 -- !query output
-9123.456
+2011	11
 
 
 -- !query
-select extract(msecs from c) from t
+select date_part('years', c), date_part('years', i) from t
 -- !query schema
-struct<extract('msecs' FROM t.`c`):decimal(8,3)>
+struct<date_part('years', t.`c`):int,date_part('years', t.`i`):int>
 -- !query output
-9123.456
+2011	11
 
 
 -- !query
-select extract(millisecon from c) from t
+select date_part('yr', c), date_part('yr', i) from t
 -- !query schema
-struct<extract('millisecon' FROM t.`c`):decimal(8,3)>
+struct<date_part('yr', t.`c`):int,date_part('yr', t.`i`):int>
 -- !query output
-9123.456
+2011	11
 
 
 -- !query
-select extract(mseconds from c) from t
+select date_part('yrs', c), date_part('yrs', i) from t
 -- !query schema
-struct<extract('mseconds' FROM t.`c`):decimal(8,3)>
+struct<date_part('yrs', t.`c`):int,date_part('yrs', t.`i`):int>
 -- !query output
-9123.456
+2011	11
 
 
 -- !query
-select extract(ms from c) from t
+select date_part('yearofweek', c) from t
 -- !query schema
-struct<extract('ms' FROM t.`c`):decimal(8,3)>
+struct<date_part('yearofweek', t.`c`):int>
 -- !query output
-9123.456
+2011
 
 
 -- !query
-select extract(microseconds from c) from t
+select date_part('quarter', c) from t
 -- !query schema
-struct<extract('microseconds' FROM t.`c`):int>
+struct<date_part('quarter', t.`c`):int>
 -- !query output
-9123456
+2
 
 
 -- !query
-select extract(usec from c) from t
+select date_part('qtr', c) from t
 -- !query schema
-struct<extract('usec' FROM t.`c`):int>
+struct<date_part('qtr', t.`c`):int>
 -- !query output
-9123456
+2
 
 
 -- !query
-select extract(usecs from c) from t
+select date_part('month', c), date_part('month', i) from t
 -- !query schema
-struct<extract('usecs' FROM t.`c`):int>
+struct<date_part('month', t.`c`):int,date_part('month', t.`i`):tinyint>
 -- !query output
-9123456
+5	8
 
 
 -- !query
-select extract(useconds from c) from t
+select date_part('mon', c), date_part('mon', i) from t
 -- !query schema
-struct<extract('useconds' FROM t.`c`):int>
+struct<date_part('mon', t.`c`):int,date_part('mon', t.`i`):tinyint>
 -- !query output
-9123456
+5	8
 
 
 -- !query
-select extract(microsecon from c) from t
+select date_part('mons', c), date_part('mons', i) from t
 -- !query schema
-struct<extract('microsecon' FROM t.`c`):int>
+struct<date_part('mons', t.`c`):int,date_part('mons', t.`i`):tinyint>
 -- !query output
-9123456
+5	8
 
 
 -- !query
-select extract(us from c) from t
+select date_part('months', c), date_part('months', i) from t
 -- !query schema
-struct<extract('us' FROM t.`c`):int>
+struct<date_part('months', t.`c`):int,date_part('months', t.`i`):tinyint>
 -- !query output
-9123456
+5	8
 
 
 -- !query
-select extract(epoch from c) from t
+select date_part('week', c) from t
 -- !query schema
-struct<extract('epoch' FROM t.`c`):decimal(20,6)>
+struct<date_part('week', t.`c`):int>
 -- !query output
-1304665689.123456
+18
 
 
 -- !query
-select extract(not_supported from c) from t
+select date_part('w', c) from t
+-- !query schema
+struct<date_part('w', t.`c`):int>
+-- !query output
+18
+
+
+-- !query
+select date_part('weeks', c) from t
+-- !query schema
+struct<date_part('weeks', t.`c`):int>
+-- !query output
+18
+
+
+-- !query
+select date_part('day', c), date_part('day', i) from t
+-- !query schema
+struct<date_part('day', t.`c`):int,date_part('day', t.`i`):int>
+-- !query output
+6	30
+
+
+-- !query
+select date_part('d', c), date_part('d', i) from t
+-- !query schema
+struct<date_part('d', t.`c`):int,date_part('d', t.`i`):int>
+-- !query output
+6	30
+
+
+-- !query
+select date_part('days', c), date_part('days', i) from t
+-- !query schema
+struct<date_part('days', t.`c`):int,date_part('days', t.`i`):int>
+-- !query output
+6	30
+
+
+-- !query
+select date_part('dayofweek', c) from t
+-- !query schema
+struct<date_part('dayofweek', t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select date_part('dow', c) from t
+-- !query schema
+struct<date_part('dow', t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select date_part('dayofweek_iso', c) from t
+-- !query schema
+struct<date_part('dayofweek_iso', t.`c`):int>
+-- !query output
+5
+
+
+-- !query
+select date_part('dow_iso', c) from t
+-- !query schema
+struct<date_part('dow_iso', t.`c`):int>
+-- !query output
+5
+
+
+-- !query
+select date_part('doy', c) from t
+-- !query schema
+struct<date_part('doy', t.`c`):int>
+-- !query output
+126
+
+
+-- !query
+select date_part('hour', c), date_part('hour', i) from t
+-- !query schema
+struct<date_part('hour', t.`c`):int,date_part('hour', t.`i`):bigint>
+-- !query output
+7	40
+
+
+-- !query
+select date_part('h', c), date_part('h', i) from t
+-- !query schema
+struct<date_part('h', t.`c`):int,date_part('h', t.`i`):bigint>
+-- !query output
+7	40
+
+
+-- !query
+select date_part('hours', c), date_part('hours', i) from t
+-- !query schema
+struct<date_part('hours', t.`c`):int,date_part('hours', t.`i`):bigint>
+-- !query output
+7	40
+
+
+-- !query
+select date_part('hr', c), date_part('hr', i) from t
+-- !query schema
+struct<date_part('hr', t.`c`):int,date_part('hr', t.`i`):bigint>
+-- !query output
+7	40
+
+
+-- !query
+select date_part('hrs', c), date_part('hrs', i) from t
+-- !query schema
+struct<date_part('hrs', t.`c`):int,date_part('hrs', t.`i`):bigint>
+-- !query output
+7	40
+
+
+-- !query
+select date_part('minute', c), date_part('minute', i) from t
+-- !query schema
+struct<date_part('minute', t.`c`):int,date_part('minute', t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select date_part('m', c), date_part('m', i) from t
+-- !query schema
+struct<date_part('m', t.`c`):int,date_part('m', t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select date_part('min', c), date_part('min', i) from t
+-- !query schema
+struct<date_part('min', t.`c`):int,date_part('min', t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select date_part('mins', c), date_part('mins', i) from t
+-- !query schema
+struct<date_part('mins', t.`c`):int,date_part('mins', t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select date_part('minutes', c), date_part('minutes', i) from t
+-- !query schema
+struct<date_part('minutes', t.`c`):int,date_part('minutes', t.`i`):tinyint>
+-- !query output
+8	50
+
+
+-- !query
+select date_part('second', c), date_part('second', i) from t
+-- !query schema
+struct<date_part('second', t.`c`):decimal(8,6),date_part('second', t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select date_part('s', c), date_part('s', i) from t
+-- !query schema
+struct<date_part('s', t.`c`):decimal(8,6),date_part('s', t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select date_part('sec', c), date_part('sec', i) from t
+-- !query schema
+struct<date_part('sec', t.`c`):decimal(8,6),date_part('sec', t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select date_part('seconds', c), date_part('seconds', i) from t
+-- !query schema
+struct<date_part('seconds', t.`c`):decimal(8,6),date_part('seconds', t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select date_part('secs', c), date_part('secs', i) from t
+-- !query schema
+struct<date_part('secs', t.`c`):decimal(8,6),date_part('secs', t.`i`):decimal(8,6)>
+-- !query output
+9.123456	6.789000
+
+
+-- !query
+select date_part('not_supported', c) from t
 -- !query schema
 struct<>
 -- !query output
@@ -516,9 +646,34 @@ Literals of type 'not_supported' are currently not supported for the string type
 
 
 -- !query
-select extract(not_supported from i) from t
+select date_part(c, c) from t
 -- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Literals of type 'not_supported' are currently not supported for the interval type.;; line 1 pos 7
+The field parameter needs to be a foldable string value.;; line 1 pos 7
+
+
+-- !query
+select date_part(null, c) from t
+-- !query schema
+struct<date_part(NULL, t.`c`):double>
+-- !query output
+NULL
+
+
+-- !query
+select date_part(i, i) from t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The field parameter needs to be a foldable string value.;; line 1 pos 7
+
+
+-- !query
+select date_part(null, i) from t
+-- !query schema
+struct<date_part(NULL, t.`i`):double>
+-- !query output
+NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index 0dac13d9deae0..1d862ba8a41a8 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 98
+-- Number of queries: 60
 
 
 -- !query
@@ -571,310 +571,6 @@ struct<Two days:interval>
 2 days
 
 
--- !query
-SELECT EXTRACT(EPOCH FROM DATE        '1970-01-01')
--- !query schema
-struct<extract('EPOCH' FROM DATE '1970-01-01'):decimal(20,6)>
--- !query output
-0.000000
-
-
--- !query
-SELECT EXTRACT(EPOCH FROM TIMESTAMP   '1970-01-01')
--- !query schema
-struct<extract('EPOCH' FROM TIMESTAMP '1970-01-01 00:00:00'):decimal(20,6)>
--- !query output
-0.000000
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TO_DATE('0101-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('CENTURY' FROM to_date('0101-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--2
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TO_DATE('0100-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('CENTURY' FROM to_date('0100-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('CENTURY' FROM to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01')
--- !query schema
-struct<extract('CENTURY' FROM DATE '0001-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '0001-01-01 AD')
--- !query schema
-struct<extract('CENTURY' FROM DATE '0001-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '1900-12-31')
--- !query schema
-struct<extract('CENTURY' FROM DATE '1900-12-31'):int>
--- !query output
-19
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '1901-01-01')
--- !query schema
-struct<extract('CENTURY' FROM DATE '1901-01-01'):int>
--- !query output
-20
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '2000-12-31')
--- !query schema
-struct<extract('CENTURY' FROM DATE '2000-12-31'):int>
--- !query output
-20
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM DATE '2001-01-01')
--- !query schema
-struct<extract('CENTURY' FROM DATE '2001-01-01'):int>
--- !query output
-21
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM CURRENT_DATE)>=21 AS True
--- !query schema
-struct<True:boolean>
--- !query output
-true
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM TO_DATE('0001-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('MILLENNIUM' FROM to_date('0001-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '0001-01-01 AD')
--- !query schema
-struct<extract('MILLENNIUM' FROM DATE '0001-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '1000-12-31')
--- !query schema
-struct<extract('MILLENNIUM' FROM DATE '1000-12-31'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '1001-01-01')
--- !query schema
-struct<extract('MILLENNIUM' FROM DATE '1001-01-01'):int>
--- !query output
-2
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '2000-12-31')
--- !query schema
-struct<extract('MILLENNIUM' FROM DATE '2000-12-31'):int>
--- !query output
-2
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM DATE '2001-01-01')
--- !query schema
-struct<extract('MILLENNIUM' FROM DATE '2001-01-01'):int>
--- !query output
-3
-
-
--- !query
-SELECT EXTRACT(MILLENNIUM FROM CURRENT_DATE)
--- !query schema
-struct<extract('MILLENNIUM' FROM current_date()):int>
--- !query output
-3
-
-
--- !query
-SELECT EXTRACT(DECADE FROM DATE '1994-12-25')
--- !query schema
-struct<extract('DECADE' FROM DATE '1994-12-25'):int>
--- !query output
-199
-
-
--- !query
-SELECT EXTRACT(DECADE FROM DATE '0010-01-01')
--- !query schema
-struct<extract('DECADE' FROM DATE '0010-01-01'):int>
--- !query output
-1
-
-
--- !query
-SELECT EXTRACT(DECADE FROM DATE '0009-12-31')
--- !query schema
-struct<extract('DECADE' FROM DATE '0009-12-31'):int>
--- !query output
-0
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0001-01-01 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('DECADE' FROM to_date('0001-01-01 BC', 'yyyy-MM-dd G')):int>
--- !query output
-0
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('DECADE' FROM to_date('0002-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0011-01-01 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('DECADE' FROM to_date('0011-01-01 BC', 'yyyy-MM-dd G')):int>
--- !query output
--1
-
-
--- !query
-SELECT EXTRACT(DECADE FROM TO_DATE('0012-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<extract('DECADE' FROM to_date('0012-12-31 BC', 'yyyy-MM-dd G')):int>
--- !query output
--2
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM NOW())>=21 AS True
--- !query schema
-struct<True:boolean>
--- !query output
-true
-
-
--- !query
-SELECT EXTRACT(CENTURY FROM TIMESTAMP '1970-03-20 04:30:00.00000')
--- !query schema
-struct<extract('CENTURY' FROM TIMESTAMP '1970-03-20 04:30:00'):int>
--- !query output
-20
-
-
--- !query
-SELECT DATE_TRUNC('MILLENNIUM', TIMESTAMP '1970-03-20 04:30:00.00000')
--- !query schema
-struct<date_trunc(MILLENNIUM, TIMESTAMP '1970-03-20 04:30:00'):timestamp>
--- !query output
-1001-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('MILLENNIUM', DATE '1970-03-20')
--- !query schema
-struct<date_trunc(MILLENNIUM, CAST(DATE '1970-03-20' AS TIMESTAMP)):timestamp>
--- !query output
-1001-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', TIMESTAMP '1970-03-20 04:30:00.00000')
--- !query schema
-struct<date_trunc(CENTURY, TIMESTAMP '1970-03-20 04:30:00'):timestamp>
--- !query output
-1901-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', DATE '1970-03-20')
--- !query schema
-struct<date_trunc(CENTURY, CAST(DATE '1970-03-20' AS TIMESTAMP)):timestamp>
--- !query output
-1901-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', DATE '2004-08-10')
--- !query schema
-struct<date_trunc(CENTURY, CAST(DATE '2004-08-10' AS TIMESTAMP)):timestamp>
--- !query output
-2001-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', DATE '0002-02-04')
--- !query schema
-struct<date_trunc(CENTURY, CAST(DATE '0002-02-04' AS TIMESTAMP)):timestamp>
--- !query output
-0001-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('CENTURY', TO_DATE('0055-08-10 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_trunc(CENTURY, CAST(to_date('0055-08-10 BC', 'yyyy-MM-dd G') AS TIMESTAMP)):timestamp>
--- !query output
--0099-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('DECADE', DATE '1993-12-25')
--- !query schema
-struct<date_trunc(DECADE, CAST(DATE '1993-12-25' AS TIMESTAMP)):timestamp>
--- !query output
-1990-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('DECADE', DATE '0004-12-25')
--- !query schema
-struct<date_trunc(DECADE, CAST(DATE '0004-12-25' AS TIMESTAMP)):timestamp>
--- !query output
-0000-01-01 00:00:00
-
-
--- !query
-SELECT DATE_TRUNC('DECADE', TO_DATE('0002-12-31 BC', 'yyyy-MM-dd G'))
--- !query schema
-struct<date_trunc(DECADE, CAST(to_date('0002-12-31 BC', 'yyyy-MM-dd G') AS TIMESTAMP)):timestamp>
--- !query output
--0010-01-01 00:00:00
-
-
 -- !query
 select make_date(2013, 7, 15)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
index abfce9180ab39..5b0b636ff0c29 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 32
+-- Number of queries: 30
 
 
 -- !query
@@ -265,36 +265,6 @@ struct<54:string,timestamp:timestamp,year:int,month:int,day:int,hour:int,minute:
 	2001-09-22 18:19:20	2001	9	22	18	19	20.000000
 
 
--- !query
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'quarter', d1) AS quarter, date_part( 'msec', d1) AS msec,
-    date_part( 'usec', d1) AS usec
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'
--- !query schema
-struct<54:string,timestamp:timestamp,quarter:int,msec:decimal(8,3),usec:int>
--- !query output
-	1969-12-31 16:00:00	4	0.000	0
-	1997-01-02 00:00:00	1	0.000	0
-	1997-01-02 03:04:05	1	5000.000	5000000
-	1997-02-10 17:32:01	1	1000.000	1000000
-	2001-09-22 18:19:20	3	20000.000	20000000
-
-
--- !query
-SELECT '' AS `54`, d1 as `timestamp`,
-    date_part( 'isoyear', d1) AS isoyear, date_part( 'week', d1) AS week,
-    date_part( 'dow', d1) AS dow
-    FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01'
--- !query schema
-struct<54:string,timestamp:timestamp,isoyear:int,week:int,dow:int>
--- !query output
-	1969-12-31 16:00:00	1970	1	4
-	1997-01-02 00:00:00	1997	1	5
-	1997-01-02 03:04:05	1997	1	5
-	1997-02-10 17:32:01	1997	7	2
-	2001-09-22 18:19:20	2001	38	7
-
-
 -- !query
 SELECT make_timestamp(2014,12,28,6,30,45.887)
 -- !query schema
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
index 6984d9406d402..287854dc3646c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ExtractBenchmark.scala
@@ -86,12 +86,8 @@ object ExtractBenchmark extends SqlBasedBenchmark {
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     val N = 10000000L
-    val datetimeFields = Seq(
-      "MILLENNIUM", "CENTURY", "DECADE", "YEAR",
-      "ISOYEAR", "QUARTER", "MONTH", "WEEK",
-      "DAY", "DAYOFWEEK", "DOW", "ISODOW",
-      "DOY", "HOUR", "MINUTE", "SECOND",
-      "MILLISECONDS", "MICROSECONDS", "EPOCH")
+    val datetimeFields = Seq("YEAR", "YEAROFWEEK", "QUARTER", "MONTH", "WEEK", "DAY", "DAYOFWEEK",
+      "DOW", "DOW_ISO", "DAYOFWEEK_ISO", "DOY", "HOUR", "MINUTE", "SECOND")
     val intervalFields = Seq("YEAR", "MONTH", "DAY", "HOUR", "MINUTE", "SECOND")
     val settings = Map(
       "timestamp" -> datetimeFields,

From c9748d4f00c505053c81c8aeb69f7166e92f82a6 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 22 Apr 2020 12:44:06 +0000
Subject: [PATCH 0536/1280] [SPARK-31495][SQL] Support formatted explain for
 AQE

### What changes were proposed in this pull request?

To support formatted explain for AQE.

### Why are the changes needed?

AQE does not support formatted explain yet. It's good to support it for better user experience, debugging, etc.

Before:
```
== Physical Plan ==
AdaptiveSparkPlan (1)
+- * HashAggregate (unknown)
   +- CustomShuffleReader (unknown)
      +- ShuffleQueryStage (unknown)
         +- Exchange (unknown)
            +- * HashAggregate (unknown)
               +- * Project (unknown)
                  +- * BroadcastHashJoin Inner BuildRight (unknown)
                     :- * LocalTableScan (unknown)
                     +- BroadcastQueryStage (unknown)
                        +- BroadcastExchange (unknown)
                           +- LocalTableScan (unknown)

(1) AdaptiveSparkPlan
Output [4]: [k#7, count(v1)#32L, sum(v1)#33L, avg(v2)#34]
Arguments: HashAggregate(keys=[k#7], functions=[count(1), sum(cast(v1#8 as bigint)), avg(cast(v2#19 as bigint))]), AdaptiveExecutionContext(org.apache.spark.sql.SparkSession104ab57b), [PlanAdaptiveSubqueries(Map())], false
```

After:
```
== Physical Plan ==
 AdaptiveSparkPlan (14)
 +- * HashAggregate (13)
    +- CustomShuffleReader (12)
       +- ShuffleQueryStage (11)
          +- Exchange (10)
             +- * HashAggregate (9)
                +- * Project (8)
                   +- * BroadcastHashJoin Inner BuildRight (7)
                      :- * Project (2)
                      :  +- * LocalTableScan (1)
                      +- BroadcastQueryStage (6)
                         +- BroadcastExchange (5)
                            +- * Project (4)
                               +- * LocalTableScan (3)

 (1) LocalTableScan [codegen id : 2]
 Output [2]: [_1#x, _2#x]
 Arguments: [_1#x, _2#x]

 (2) Project [codegen id : 2]
 Output [2]: [_1#x AS k#x, _2#x AS v1#x]
 Input [2]: [_1#x, _2#x]

 (3) LocalTableScan [codegen id : 1]
 Output [2]: [_1#x, _2#x]
 Arguments: [_1#x, _2#x]

 (4) Project [codegen id : 1]
 Output [2]: [_1#x AS k#x, _2#x AS v2#x]
 Input [2]: [_1#x, _2#x]

 (5) BroadcastExchange
 Input [2]: [k#x, v2#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint))), [id=#x]

 (6) BroadcastQueryStage
 Output [2]: [k#x, v2#x]
 Arguments: 0

 (7) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [k#x]
 Right keys [1]: [k#x]
 Join condition: None

 (8) Project [codegen id : 2]
 Output [3]: [k#x, v1#x, v2#x]
 Input [4]: [k#x, v1#x, k#x, v2#x]

 (9) HashAggregate [codegen id : 2]
 Input [3]: [k#x, v1#x, v2#x]
 Keys [1]: [k#x]
 Functions [3]: [partial_count(1), partial_sum(cast(v1#x as bigint)), partial_avg(cast(v2#x as bigint))]
 Aggregate Attributes [4]: [count#xL, sum#xL, sum#x, count#xL]
 Results [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]

 (10) Exchange
 Input [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
 Arguments: hashpartitioning(k#x, 5), true, [id=#x]

 (11) ShuffleQueryStage
 Output [5]: [sum#xL, k#x, sum#x, count#xL, count#xL]
 Arguments: 1

 (12) CustomShuffleReader
 Input [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
 Arguments: coalesced

 (13) HashAggregate [codegen id : 3]
 Input [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
 Keys [1]: [k#x]
 Functions [3]: [count(1), sum(cast(v1#x as bigint)), avg(cast(v2#x as bigint))]
 Aggregate Attributes [3]: [count(1)#xL, sum(cast(v1#x as bigint))#xL, avg(cast(v2#x as bigint))#x]
 Results [4]: [k#x, count(1)#xL AS count(v1)#xL, sum(cast(v1#x as bigint))#xL AS sum(v1)#xL, avg(cast(v2#x as bigint))#x AS avg(v2)#x]

 (14) AdaptiveSparkPlan
 Output [4]: [k#x, count(v1)#xL, sum(v1)#xL, avg(v2)#x]
 Arguments: isFinalPlan=true
```

### Does this PR introduce any user-facing change?

No, this should be new feature along with AQE in Spark 3.0.

### How was this patch tested?

Added a query file: `explain-aqe.sql` and a unit test.

Closes #28271 from Ngone51/support_formatted_explain_for_aqe.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8fbfdb38c0baff7bc5d1ce1e3d6f1df70c25da70)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/ExplainUtils.scala    |  59 +-
 .../adaptive/AdaptiveSparkPlanExec.scala      |   5 +-
 .../adaptive/InsertAdaptiveSparkPlan.scala    |   2 +-
 .../sql-tests/inputs/explain-aqe.sql          |   3 +
 .../resources/sql-tests/inputs/explain.sql    |   1 +
 .../sql-tests/results/explain-aqe.sql.out     | 860 ++++++++++++++++++
 .../sql-tests/results/explain.sql.out         |  10 +-
 .../org/apache/spark/sql/ExplainSuite.scala   |  72 +-
 .../adaptive/AdaptiveQueryExecSuite.scala     |   6 +-
 9 files changed, 981 insertions(+), 37 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index 5d4309357895b..aec1c93072410 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -23,8 +23,9 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.{Expression, PlanExpression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, QueryStageExec}
 
-object ExplainUtils {
+object ExplainUtils extends AdaptiveSparkPlanHelper {
   /**
    * Given a input physical plan, performs the following tasks.
    *   1. Computes the operator id for current operator and records it in the operaror
@@ -144,15 +145,26 @@ object ExplainUtils {
       case p: WholeStageCodegenExec =>
       case p: InputAdapter =>
       case other: QueryPlan[_] =>
-        if (!other.getTagValue(QueryPlan.OP_ID_TAG).isDefined) {
+
+        def setOpId(): Unit = if (other.getTagValue(QueryPlan.OP_ID_TAG).isEmpty) {
           currentOperationID += 1
           other.setTagValue(QueryPlan.OP_ID_TAG, currentOperationID)
           operatorIDs += ((currentOperationID, other))
         }
-        other.innerChildren.foreach { plan =>
-          currentOperationID = generateOperatorIDs(plan,
-            currentOperationID,
-            operatorIDs)
+
+        other match {
+          case p: AdaptiveSparkPlanExec =>
+            currentOperationID =
+              generateOperatorIDs(p.executedPlan, currentOperationID, operatorIDs)
+            setOpId()
+          case p: QueryStageExec =>
+            currentOperationID = generateOperatorIDs(p.plan, currentOperationID, operatorIDs)
+            setOpId()
+          case _ =>
+            setOpId()
+            other.innerChildren.foldLeft(currentOperationID) {
+              (curId, plan) => generateOperatorIDs(plan, curId, operatorIDs)
+            }
         }
     }
     currentOperationID
@@ -163,21 +175,25 @@ object ExplainUtils {
    * whole stage code gen id in the plan via setting a tag.
    */
   private def generateWholeStageCodegenIds(plan: QueryPlan[_]): Unit = {
+    var currentCodegenId = -1
+
+    def setCodegenId(p: QueryPlan[_], children: Seq[QueryPlan[_]]): Unit = {
+      if (currentCodegenId != -1) {
+        p.setTagValue(QueryPlan.CODEGEN_ID_TAG, currentCodegenId)
+      }
+      children.foreach(generateWholeStageCodegenIds)
+    }
+
     // Skip the subqueries as they are not printed as part of main query block.
     if (plan.isInstanceOf[BaseSubqueryExec]) {
       return
     }
-    var currentCodegenId = -1
     plan.foreach {
       case p: WholeStageCodegenExec => currentCodegenId = p.codegenStageId
       case _: InputAdapter => currentCodegenId = -1
-      case other: QueryPlan[_] =>
-        if (currentCodegenId != -1) {
-          other.setTagValue(QueryPlan.CODEGEN_ID_TAG, currentCodegenId)
-        }
-        other.innerChildren.foreach { plan =>
-          generateWholeStageCodegenIds(plan)
-        }
+      case p: AdaptiveSparkPlanExec => setCodegenId(p, Seq(p.executedPlan))
+      case p: QueryStageExec => setCodegenId(p, Seq(p.plan))
+      case other: QueryPlan[_] => setCodegenId(other, other.innerChildren)
     }
   }
 
@@ -232,13 +248,16 @@ object ExplainUtils {
   }
 
   def removeTags(plan: QueryPlan[_]): Unit = {
+    def remove(p: QueryPlan[_], children: Seq[QueryPlan[_]]): Unit = {
+      p.unsetTagValue(QueryPlan.OP_ID_TAG)
+      p.unsetTagValue(QueryPlan.CODEGEN_ID_TAG)
+      children.foreach(removeTags)
+    }
+
     plan foreach {
-      case plan: QueryPlan[_] =>
-        plan.unsetTagValue(QueryPlan.OP_ID_TAG)
-        plan.unsetTagValue(QueryPlan.CODEGEN_ID_TAG)
-        plan.innerChildren.foreach { p =>
-          removeTags(p)
-        }
+      case p: AdaptiveSparkPlanExec => remove(p, Seq(p.executedPlan))
+      case p: QueryStageExec => remove(p, Seq(p.plan))
+      case plan: QueryPlan[_] => remove(plan, plan.innerChildren)
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index d73540cb65da9..f00dce2ca8ae9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -251,10 +251,7 @@ case class AdaptiveSparkPlanExec(
     getFinalPhysicalPlan().execute()
   }
 
-  override def verboseString(maxFields: Int): String = simpleString(maxFields)
-
-  override def simpleString(maxFields: Int): String =
-    s"AdaptiveSparkPlan(isFinalPlan=$isFinalPlan)"
+  protected override def stringArgs: Iterator[Any] = Iterator(s"isFinalPlan=$isFinalPlan")
 
   override def generateTreeString(
       depth: Int,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
index ea586f0bb2561..754225dd3fe95 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/InsertAdaptiveSparkPlan.scala
@@ -122,7 +122,7 @@ case class InsertAdaptiveSparkPlan(
           if !subqueryMap.contains(exprId.id) =>
         val executedPlan = compileSubquery(p)
         verifyAdaptivePlan(executedPlan, p)
-        val subquery = SubqueryExec(s"subquery${exprId.id}", executedPlan)
+        val subquery = SubqueryExec(s"subquery#${exprId.id}", executedPlan)
         subqueryMap.put(exprId.id, subquery)
       case expressions.InSubquery(_, ListQuery(query, _, exprId, _))
           if !subqueryMap.contains(exprId.id) =>
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
new file mode 100644
index 0000000000000..f4afa2b77a9d7
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
@@ -0,0 +1,3 @@
+--IMPORT explain.sql
+
+--SET spark.sql.adaptive.enabled=true
diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain.sql b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
index 497b61c6134a2..80bf258704c70 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/explain.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/explain.sql
@@ -117,3 +117,4 @@ EXPLAIN FORMATTED
 DROP TABLE explain_temp1;
 DROP TABLE explain_temp2;
 DROP TABLE explain_temp3;
+DROP TABLE explain_temp4;
diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
new file mode 100644
index 0000000000000..e1d4fa84c8f54
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
@@ -0,0 +1,860 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 23
+
+
+-- !query
+CREATE table  explain_temp1 (key int, val int) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE table  explain_temp2 (key int, val int) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE table  explain_temp3 (key int, val int) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE table  explain_temp4 (key int, val string) USING PARQUET
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SET spark.sql.codegen.wholeStage = true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.codegen.wholeStage	true
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, max(val) 
+  FROM   explain_temp1 
+  WHERE  key > 0 
+  GROUP  BY key 
+  ORDER  BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- Sort (8)
+   +- Exchange (7)
+      +- HashAggregate (6)
+         +- Exchange (5)
+            +- HashAggregate (4)
+               +- Project (3)
+                  +- Filter (2)
+                     +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+     
+(5) Exchange 
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
+(6) HashAggregate 
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+     
+(7) Exchange 
+Input [2]: [key#x, max(val)#x]
+Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
+      
+(8) Sort 
+Input [2]: [key#x, max(val)#x]
+Arguments: [key#x ASC NULLS FIRST], true, 0
+      
+(9) AdaptiveSparkPlan 
+Output [2]: [key#x, max(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, max(val)
+  FROM explain_temp1
+  WHERE key > 0
+  GROUP BY key
+  HAVING max(val) > 0
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- Project (8)
+   +- Filter (7)
+      +- HashAggregate (6)
+         +- Exchange (5)
+            +- HashAggregate (4)
+               +- Project (3)
+                  +- Filter (2)
+                     +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+     
+(5) Exchange 
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
+(6) HashAggregate 
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
+     
+(7) Filter 
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
+Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
+     
+(8) Project 
+Output [2]: [key#x, max(val)#x]
+Input [3]: [key#x, max(val)#x, max(val#x)#x]
+     
+(9) AdaptiveSparkPlan 
+Output [2]: [key#x, max(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, val FROM explain_temp1 WHERE key > 0
+  UNION 
+  SELECT key, val FROM explain_temp1 WHERE key > 0
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (11)
++- HashAggregate (10)
+   +- Exchange (9)
+      +- HashAggregate (8)
+         +- Union (7)
+            :- Project (3)
+            :  +- Filter (2)
+            :     +- Scan parquet default.explain_temp1 (1)
+            +- Project (6)
+               +- Filter (5)
+                  +- Scan parquet default.explain_temp1 (4)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
+ReadSchema: struct<key:int,val:int>
+     
+(5) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 0))
+     
+(6) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(7) Union 
+      
+(8) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [key#x, val#x]
+     
+(9) Exchange 
+Input [2]: [key#x, val#x]
+Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
+      
+(10) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys [2]: [key#x, val#x]
+Functions: []
+Aggregate Attributes: []
+Results [2]: [key#x, val#x]
+     
+(11) AdaptiveSparkPlan 
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 a, 
+         explain_temp2 b 
+  WHERE  a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- BroadcastHashJoin Inner BuildRight (8)
+   :- Project (3)
+   :  +- Filter (2)
+   :     +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (7)
+      +- Project (6)
+         +- Filter (5)
+            +- Scan parquet default.explain_temp2 (4)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : isnotnull(key#x)
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) Scan parquet default.explain_temp2 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp2]
+PushedFilters: [IsNotNull(key)]
+ReadSchema: struct<key:int,val:int>
+     
+(5) Filter 
+Input [2]: [key#x, val#x]
+Condition : isnotnull(key#x)
+     
+(6) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(7) BroadcastExchange 
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
+(8) BroadcastHashJoin 
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+       
+(9) AdaptiveSparkPlan 
+Output [4]: [key#x, val#x, key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 a 
+         LEFT OUTER JOIN explain_temp2 b 
+                      ON a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (7)
++- BroadcastHashJoin LeftOuter BuildRight (6)
+   :- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (5)
+      +- Project (4)
+         +- Filter (3)
+            +- Scan parquet default.explain_temp2 (2)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Scan parquet default.explain_temp2 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp2]
+PushedFilters: [IsNotNull(key)]
+ReadSchema: struct<key:int,val:int>
+     
+(3) Filter 
+Input [2]: [key#x, val#x]
+Condition : isnotnull(key#x)
+     
+(4) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(5) BroadcastExchange 
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
+(6) BroadcastHashJoin 
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+       
+(7) AdaptiveSparkPlan 
+Output [4]: [key#x, val#x, key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 
+  WHERE  key = (SELECT max(key) 
+                FROM   explain_temp2 
+                WHERE  key = (SELECT max(key) 
+                              FROM   explain_temp3 
+                              WHERE  val > 0) 
+                       AND val = 2) 
+         AND val > 3
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (4)
++- Project (3)
+   +- Filter (2)
+      +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), IsNotNull(val), GreaterThan(val,3)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery subquery#x, [id=#x])) AND (val#x > 3))
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) AdaptiveSparkPlan 
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT * 
+  FROM   explain_temp1 
+  WHERE  key = (SELECT max(key) 
+                FROM   explain_temp2 
+                WHERE  val > 0) 
+         OR
+         key = (SELECT avg(key)
+                FROM   explain_temp3
+                WHERE  val > 0)
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (3)
++- Filter (2)
+   +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : ((key#x = Subquery subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery subquery#x, [id=#x]))
+     
+(3) AdaptiveSparkPlan 
+Output [2]: [key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT (SELECT Avg(key) FROM explain_temp1) + (SELECT Avg(key) FROM explain_temp1)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (3)
++- Project (2)
+   +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output: []
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<>
+     
+(2) Project 
+Output [1]: [(Subquery subquery#x, [id=#x] + Subquery subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
+Input: []
+     
+(3) AdaptiveSparkPlan 
+Output [1]: [(scalarsubquery() + scalarsubquery())#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  WITH cte1 AS (
+    SELECT *
+    FROM explain_temp1 
+    WHERE key > 10
+  )
+  SELECT * FROM cte1 a, cte1 b WHERE a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (9)
++- BroadcastHashJoin Inner BuildRight (8)
+   :- Project (3)
+   :  +- Filter (2)
+   :     +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (7)
+      +- Project (6)
+         +- Filter (5)
+            +- Scan parquet default.explain_temp1 (4)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+     
+(5) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+     
+(6) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(7) BroadcastExchange 
+Input [2]: [key#x, val#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
+(8) BroadcastHashJoin 
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+       
+(9) AdaptiveSparkPlan 
+Output [4]: [key#x, val#x, key#x, val#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  WITH cte1 AS (
+    SELECT key, max(val)
+    FROM explain_temp1 
+    WHERE key > 10
+    GROUP BY key
+  )
+  SELECT * FROM cte1 a, cte1 b WHERE a.key = b.key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (15)
++- BroadcastHashJoin Inner BuildRight (14)
+   :- HashAggregate (6)
+   :  +- Exchange (5)
+   :     +- HashAggregate (4)
+   :        +- Project (3)
+   :           +- Filter (2)
+   :              +- Scan parquet default.explain_temp1 (1)
+   +- BroadcastExchange (13)
+      +- HashAggregate (12)
+         +- Exchange (11)
+            +- HashAggregate (10)
+               +- Project (9)
+                  +- Filter (8)
+                     +- Scan parquet default.explain_temp1 (7)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+     
+(2) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+     
+(3) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(4) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+     
+(5) Exchange 
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
+(6) HashAggregate 
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+     
+(7) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
+ReadSchema: struct<key:int,val:int>
+     
+(8) Filter 
+Input [2]: [key#x, val#x]
+Condition : (isnotnull(key#x) AND (key#x > 10))
+     
+(9) Project 
+Output [2]: [key#x, val#x]
+Input [2]: [key#x, val#x]
+     
+(10) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_max(val#x)]
+Aggregate Attributes [1]: [max#x]
+Results [2]: [key#x, max#x]
+     
+(11) Exchange 
+Input [2]: [key#x, max#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
+(12) HashAggregate 
+Input [2]: [key#x, max#x]
+Keys [1]: [key#x]
+Functions [1]: [max(val#x)]
+Aggregate Attributes [1]: [max(val#x)#x]
+Results [2]: [key#x, max(val#x)#x AS max(val)#x]
+     
+(13) BroadcastExchange 
+Input [2]: [key#x, max(val)#x]
+Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
+      
+(14) BroadcastHashJoin 
+Left keys [1]: [key#x]
+Right keys [1]: [key#x]
+Join condition: None
+       
+(15) AdaptiveSparkPlan 
+Output [4]: [key#x, max(val)#x, key#x, max(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  CREATE VIEW explain_view AS
+    SELECT key, val FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+Execute CreateViewCommand (1)
+   +- CreateViewCommand (2)
+         +- Project (4)
+            +- UnresolvedRelation (3)
+
+
+(1) Execute CreateViewCommand 
+Output: []
+      
+(2) CreateViewCommand 
+Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
+      
+(3) UnresolvedRelation 
+Arguments: [explain_temp1]
+      
+(4) Project 
+Arguments: ['key, 'val]
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT
+    COUNT(val) + SUM(key) as TOTAL,
+    COUNT(key) FILTER (WHERE val > 1)
+  FROM explain_temp1
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (5)
++- HashAggregate (4)
+   +- Exchange (3)
+      +- HashAggregate (2)
+         +- Scan parquet default.explain_temp1 (1)
+
+
+(1) Scan parquet default.explain_temp1 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp1]
+ReadSchema: struct<key:int,val:int>
+     
+(2) HashAggregate 
+Input [2]: [key#x, val#x]
+Keys: []
+Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
+Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
+Results [3]: [count#xL, sum#xL, count#xL]
+     
+(3) Exchange 
+Input [3]: [count#xL, sum#xL, count#xL]
+Arguments: SinglePartition, true, [id=#x]
+      
+(4) HashAggregate 
+Input [3]: [count#xL, sum#xL, count#xL]
+Keys: []
+Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
+Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
+Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
+     
+(5) AdaptiveSparkPlan 
+Output [2]: [TOTAL#xL, count(key) FILTER (WHERE (val > 1))#xL]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, sort_array(collect_set(val))[0]
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (5)
++- ObjectHashAggregate (4)
+   +- Exchange (3)
+      +- ObjectHashAggregate (2)
+         +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+     
+(2) ObjectHashAggregate 
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [buf#x]
+Results [2]: [key#x, buf#x]
+     
+(3) Exchange 
+Input [2]: [key#x, buf#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
+(4) ObjectHashAggregate 
+Input [2]: [key#x, buf#x]
+Keys [1]: [key#x]
+Functions [1]: [collect_set(val#x, 0, 0)]
+Aggregate Attributes [1]: [collect_set(val#x, 0, 0)#x]
+Results [2]: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
+     
+(5) AdaptiveSparkPlan 
+Output [2]: [key#x, sort_array(collect_set(val), true)[0]#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+EXPLAIN FORMATTED
+  SELECT key, MIN(val)
+  FROM explain_temp4
+  GROUP BY key
+-- !query schema
+struct<plan:string>
+-- !query output
+== Physical Plan ==
+AdaptiveSparkPlan (7)
++- SortAggregate (6)
+   +- Sort (5)
+      +- Exchange (4)
+         +- SortAggregate (3)
+            +- Sort (2)
+               +- Scan parquet default.explain_temp4 (1)
+
+
+(1) Scan parquet default.explain_temp4 
+Output [2]: [key#x, val#x]
+Batched: true
+Location [not included in comparison]/{warehouse_dir}/explain_temp4]
+ReadSchema: struct<key:int,val:string>
+     
+(2) Sort 
+Input [2]: [key#x, val#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+      
+(3) SortAggregate 
+Input [2]: [key#x, val#x]
+Keys [1]: [key#x]
+Functions [1]: [partial_min(val#x)]
+Aggregate Attributes [1]: [min#x]
+Results [2]: [key#x, min#x]
+     
+(4) Exchange 
+Input [2]: [key#x, min#x]
+Arguments: hashpartitioning(key#x, 4), true, [id=#x]
+      
+(5) Sort 
+Input [2]: [key#x, min#x]
+Arguments: [key#x ASC NULLS FIRST], false, 0
+      
+(6) SortAggregate 
+Input [2]: [key#x, min#x]
+Keys [1]: [key#x]
+Functions [1]: [min(val#x)]
+Aggregate Attributes [1]: [min(val#x)#x]
+Results [2]: [key#x, min(val#x)#x AS min(val)#x]
+     
+(7) AdaptiveSparkPlan 
+Output [2]: [key#x, min(val)#x]
+Arguments: isFinalPlan=false
+
+
+-- !query
+DROP TABLE explain_temp1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp3
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP TABLE explain_temp4
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index 06226f1274863..1a18d563588c0 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 22
+-- Number of queries: 23
 
 
 -- !query
@@ -1053,3 +1053,11 @@ DROP TABLE explain_temp3
 struct<>
 -- !query output
 
+
+
+-- !query
+DROP TABLE explain_temp4
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index b2047091b1576..a1b6d71fb3803 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -18,16 +18,15 @@
 package org.apache.spark.sql
 
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
+import org.apache.spark.sql.execution.adaptive.{DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.StructType
 
-class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiveExecutionSuite {
-  import testImplicits._
+trait ExplainSuiteHelper extends QueryTest with SharedSparkSession {
 
-  private def getNormalizedExplain(df: DataFrame, mode: ExplainMode): String = {
+  protected def getNormalizedExplain(df: DataFrame, mode: ExplainMode): String = {
     val output = new java.io.ByteArrayOutputStream()
     Console.withOut(output) {
       df.explain(mode.name)
@@ -38,7 +37,7 @@ class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiv
   /**
    * Get the explain from a DataFrame and run the specified action on it.
    */
-  private def withNormalizedExplain(df: DataFrame, mode: ExplainMode)(f: String => Unit) = {
+  protected def withNormalizedExplain(df: DataFrame, mode: ExplainMode)(f: String => Unit) = {
     f(getNormalizedExplain(df, mode))
   }
 
@@ -46,7 +45,7 @@ class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiv
    * Get the explain by running the sql. The explain mode should be part of the
    * sql text itself.
    */
-  private def withNormalizedExplain(queryText: String)(f: String => Unit) = {
+  protected def withNormalizedExplain(queryText: String)(f: String => Unit) = {
     val output = new java.io.ByteArrayOutputStream()
     Console.withOut(output) {
       sql(queryText).show(false)
@@ -58,7 +57,7 @@ class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiv
   /**
    * Runs the plan and makes sure the plans contains all of the keywords.
    */
-  private def checkKeywordsExistsInExplain(
+  protected def checkKeywordsExistsInExplain(
       df: DataFrame, mode: ExplainMode, keywords: String*): Unit = {
     withNormalizedExplain(df, mode) { normalizedOutput =>
       for (key <- keywords) {
@@ -67,9 +66,13 @@ class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiv
     }
   }
 
-  private def checkKeywordsExistsInExplain(df: DataFrame, keywords: String*): Unit = {
+  protected def checkKeywordsExistsInExplain(df: DataFrame, keywords: String*): Unit = {
     checkKeywordsExistsInExplain(df, ExtendedMode, keywords: _*)
   }
+}
+
+class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite {
+  import testImplicits._
 
   test("SPARK-23034 show rdd names in RDD scan nodes (Dataset)") {
     val rddWithName = spark.sparkContext.parallelize(Row(1, "abc") :: Nil).setName("testRdd")
@@ -342,4 +345,57 @@ class ExplainSuite extends QueryTest with SharedSparkSession with DisableAdaptiv
   }
 }
 
+class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuite {
+  import testImplicits._
+
+  test("Explain formatted") {
+    val df1 = Seq((1, 2), (2, 3)).toDF("k", "v1")
+    val df2 = Seq((2, 3), (1, 1)).toDF("k", "v2")
+    val testDf = df1.join(df2, "k").groupBy("k").agg(count("v1"), sum("v1"), avg("v2"))
+    // trigger the final plan for AQE
+    testDf.collect()
+    // whitespace
+    val ws = " "
+    //   == Physical Plan ==
+    //   AdaptiveSparkPlan (14)
+    //   +- * HashAggregate (13)
+    //      +- CustomShuffleReader (12)
+    //         +- ShuffleQueryStage (11)
+    //            +- Exchange (10)
+    //               +- * HashAggregate (9)
+    //                  +- * Project (8)
+    //                     +- * BroadcastHashJoin Inner BuildRight (7)
+    //                        :- * Project (2)
+    //                        :  +- * LocalTableScan (1)
+    //                        +- BroadcastQueryStage (6)
+    //                           +- BroadcastExchange (5)
+    //                              +- * Project (4)
+    //                                 +- * LocalTableScan (3)
+    checkKeywordsExistsInExplain(
+      testDf,
+      FormattedMode,
+      s"""
+         |(6) BroadcastQueryStage$ws
+         |Output [2]: [k#x, v2#x]
+         |Arguments: 0
+         |""".stripMargin,
+      s"""
+         |(11) ShuffleQueryStage$ws
+         |Output [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
+         |Arguments: 1
+         |""".stripMargin,
+      s"""
+         |(12) CustomShuffleReader$ws
+         |Input [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
+         |Arguments: coalesced
+         |""".stripMargin,
+      s"""
+         |(14) AdaptiveSparkPlan$ws
+         |Output [4]: [k#x, count(v1)#xL, sum(v1)#xL, avg(v2)#x]
+         |Arguments: isFinalPlan=true
+         |""".stripMargin
+    )
+  }
+}
+
 case class ExplainSingleData(id: Int)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 82258633e604b..c6caffafa3841 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -53,7 +53,7 @@ class AdaptiveQueryExecSuite
         event match {
           case SparkListenerSQLAdaptiveExecutionUpdate(_, _, sparkPlanInfo) =>
             if (sparkPlanInfo.simpleString.startsWith(
-              "AdaptiveSparkPlan(isFinalPlan=true)")) {
+              "AdaptiveSparkPlan isFinalPlan=true")) {
               finalPlanCnt += 1
             }
           case _ => // ignore other events
@@ -64,14 +64,14 @@ class AdaptiveQueryExecSuite
 
     val dfAdaptive = sql(query)
     val planBefore = dfAdaptive.queryExecution.executedPlan
-    assert(planBefore.toString.startsWith("AdaptiveSparkPlan(isFinalPlan=false)"))
+    assert(planBefore.toString.startsWith("AdaptiveSparkPlan isFinalPlan=false"))
     val result = dfAdaptive.collect()
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
       val df = sql(query)
       QueryTest.sameRows(result.toSeq, df.collect().toSeq)
     }
     val planAfter = dfAdaptive.queryExecution.executedPlan
-    assert(planAfter.toString.startsWith("AdaptiveSparkPlan(isFinalPlan=true)"))
+    assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
 
     spark.sparkContext.listenerBus.waitUntilEmpty()
     assert(finalPlanCnt == 1)

From 6bc6b0d4400f2ba0338770662ebafad8a0de41ac Mon Sep 17 00:00:00 2001
From: Cong Du <asclepius1993@gmail.com>
Date: Wed, 22 Apr 2020 09:44:43 -0500
Subject: [PATCH 0537/1280] [MINOR][DOCS] Fix a typo in
 ContainerPlacementStrategy's class comment

### What changes were proposed in this pull request?
This PR fixes a typo in deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala file.

### Why are the changes needed?
To deliver correct explanation about how the placement policy works.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
UT as specified, although shouldn't influence any functionality since it's in the comment.

Closes #28267 from asclepiusaka/master.

Authored-by: Cong Du <asclepius1993@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 54b97b2e143774a7238fc5a5f63e0d6eec138c41)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../LocalityPreferredContainerPlacementStrategy.scala  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
index 2288bb55d8b47..3e33382e0a395 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/LocalityPreferredContainerPlacementStrategy.scala
@@ -40,7 +40,7 @@ private[yarn] case class ContainerLocalityPreferences(nodes: Array[String], rack
  * and cpus per task is 1, so the required container number is 15,
  * and host ratio is (host1: 30, host2: 30, host3: 20, host4: 10).
  *
- * 1. If requested container number (18) is more than the required container number (15):
+ * 1. If the requested container number (18) is more than the required container number (15):
  *
  * requests for 5 containers with nodes: (host1, host2, host3, host4)
  * requests for 5 containers with nodes: (host1, host2, host3)
@@ -63,16 +63,16 @@ private[yarn] case class ContainerLocalityPreferences(nodes: Array[String], rack
  * follow the method of 1 and 2.
  *
  * 4. If containers exist and some of them can match the requested localities.
- * For example if we have 1 containers on each node (host1: 1, host2: 1: host3: 1, host4: 1),
+ * For example if we have 1 container on each node (host1: 1, host2: 1: host3: 1, host4: 1),
  * and the expected containers on each node would be (host1: 5, host2: 5, host3: 4, host4: 2),
  * so the newly requested containers on each node would be updated to (host1: 4, host2: 4,
  * host3: 3, host4: 1), 12 containers by total.
  *
  *   4.1 If requested container number (18) is more than newly required containers (12). Follow
- *   method 1 with updated ratio 4 : 4 : 3 : 1.
+ *   method 1 with an updated ratio 4 : 4 : 3 : 1.
  *
- *   4.2 If request container number (10) is more than newly required containers (12). Follow
- *   method 2 with updated ratio 4 : 4 : 3 : 1.
+ *   4.2 If request container number (10) is less than newly required containers (12). Follow
+ *   method 2 with an updated ratio 4 : 4 : 3 : 1.
  *
  * 5. If containers exist and existing localities can fully cover the requested localities.
  * For example if we have 5 containers on each node (host1: 5, host2: 5, host3: 5, host4: 5),

From 2baf315c6b64014b692a3b640f51605b1df16665 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 22 Apr 2020 19:57:07 -0700
Subject: [PATCH 0538/1280] [SPARK-31495][SQL][FOLLOW-UP][3.0] Fix test failure
 of explain-aqe.sql

### What changes were proposed in this pull request?

update the `explain-aqe.sql.out`

### Why are the changes needed?

Fix test failure.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Updated test result file.

Closes #28303 from Ngone51/fix_explain_3.0.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../test/resources/sql-tests/results/explain-aqe.sql.out  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
index e1d4fa84c8f54..f00dd81a0c581 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
@@ -314,7 +314,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-       
+     
 (9) AdaptiveSparkPlan 
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
@@ -368,7 +368,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-       
+     
 (7) AdaptiveSparkPlan 
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
@@ -539,7 +539,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-       
+     
 (9) AdaptiveSparkPlan 
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
@@ -649,7 +649,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-       
+     
 (15) AdaptiveSparkPlan 
 Output [4]: [key#x, max(val)#x, key#x, max(val)#x]
 Arguments: isFinalPlan=false

From 14ab957af3d605baadcdbdb26369912d460d6189 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 23 Apr 2020 04:23:33 +0000
Subject: [PATCH 0539/1280] [MINOR][SQL] Add comments for filters values and
 return values of Row.get()/apply()

### What changes were proposed in this pull request?
- Document row field values of `DATE` and `TIMESTAMP` type returned by `Row.get()` and `Row.apply`.
- Refer to `Row.get()` from the description of filter values

### Why are the changes needed?
Reflect current behaviour of Row's method `apply()` and `get()` in comments to inform users about different return types that are depended on the SQL config settings `spark.sql.datetime.java8API.enabled`.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Run `$ ./dev/scalastyle`

Closes #28300 from MaxGekk/doc-filter-date-time.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit e7856a7902e9b0d3d3e75fbbf1fce3557ffffb21)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../src/main/scala/org/apache/spark/sql/Row.scala  | 14 ++++++++++----
 .../org/apache/spark/sql/sources/filters.scala     |  3 ++-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index 9a7e077b658df..4487a2d7f4358 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -165,8 +165,11 @@ trait Row extends Serializable {
    *   StringType -> String
    *   DecimalType -> java.math.BigDecimal
    *
-   *   DateType -> java.sql.Date
-   *   TimestampType -> java.sql.Timestamp
+   *   DateType -> java.sql.Date if spark.sql.datetime.java8API.enabled is false
+   *   DateType -> java.time.LocalDate if spark.sql.datetime.java8API.enabled is true
+   *
+   *   TimestampType -> java.sql.Timestamp if spark.sql.datetime.java8API.enabled is false
+   *   TimestampType -> java.time.Instant if spark.sql.datetime.java8API.enabled is true
    *
    *   BinaryType -> byte array
    *   ArrayType -> scala.collection.Seq (use getList for java.util.List)
@@ -190,8 +193,11 @@ trait Row extends Serializable {
    *   StringType -> String
    *   DecimalType -> java.math.BigDecimal
    *
-   *   DateType -> java.sql.Date
-   *   TimestampType -> java.sql.Timestamp
+   *   DateType -> java.sql.Date if spark.sql.datetime.java8API.enabled is false
+   *   DateType -> java.time.LocalDate if spark.sql.datetime.java8API.enabled is true
+   *
+   *   TimestampType -> java.sql.Timestamp if spark.sql.datetime.java8API.enabled is false
+   *   TimestampType -> java.time.Instant if spark.sql.datetime.java8API.enabled is true
    *
    *   BinaryType -> byte array
    *   ArrayType -> scala.collection.Seq (use getList for java.util.List)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
index 319073e4475be..7533793253513 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
@@ -25,7 +25,8 @@ import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 /**
- * A filter predicate for data sources.
+ * A filter predicate for data sources. Mapping between Spark SQL types and filter value
+ * types follow the convention for return type of [[org.apache.spark.sql.Row#get(int)]].
  *
  * @since 1.3.0
  */

From 5c1e0d0201f40e16ac25dbc804620656345dc675 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 23 Apr 2020 13:33:04 +0900
Subject: [PATCH 0540/1280] [SPARK-31474][SQL][FOLLOWUP] Replace _FUNC_
 placeholder with functionname in the note field of expression info

### What changes were proposed in this pull request?

\_FUNC\_ is used in note() of `ExpressionDescription` since https://github.com/apache/spark/pull/28248, it can be more cases later, we should replace it with function name for documentation

### Why are the changes needed?

doc fix

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

pass Jenkins, and verify locally with Jekyll serve

Closes #28305 from yaooqinn/SPARK-31474-F.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3b5792114aba211f6dd4a89d91f382672096bce3)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/catalyst/expressions/ExpressionDescription.java | 4 ++--
 sql/gen-sql-api-docs.py                                       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
index 0f648ab9d7d26..089fbe53d5171 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
@@ -79,8 +79,8 @@
  * </code>
  * </pre>
  *
- *  We can refer the function name by `_FUNC_`, in `usage()`, `arguments()` and `examples()` as
- *  it is registered in `FunctionRegistry`.
+ *  We can refer the function name by `_FUNC_`, in `usage()`, `arguments()`, `examples()` and
+ *  `note()` as it is registered in `FunctionRegistry`.
  *
  *  Note that, if `extended()` is defined, `arguments()`, `examples()`, `note()`, `group()`,
  *  `since()` and `deprecated()` should be not defined together. `extended()` exists
diff --git a/sql/gen-sql-api-docs.py b/sql/gen-sql-api-docs.py
index 3095a51e82d76..61328997c1c58 100644
--- a/sql/gen-sql-api-docs.py
+++ b/sql/gen-sql-api-docs.py
@@ -43,7 +43,7 @@ def _list_function_infos(jvm):
             usage=usage,
             arguments=jinfo.getArguments().replace("_FUNC_", name),
             examples=jinfo.getExamples().replace("_FUNC_", name),
-            note=jinfo.getNote(),
+            note=jinfo.getNote().replace("_FUNC_", name),
             since=jinfo.getSince(),
             deprecated=jinfo.getDeprecated()))
     return sorted(infos, key=lambda i: i.name)

From 6e76310dd5eaf5da68e3c46bcb536543d1fe27e4 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 23 Apr 2020 04:59:59 +0000
Subject: [PATCH 0541/1280] [SPARK-31476][SQL][FOLLOWUP] Add tests for
 extract('field', source)

### What changes were proposed in this pull request?

SPARK-31476 has supported `extract('field', source)` as side-effect, so this PR intends to add some tests for the function in `SQLQueryTestSuite`.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #28276 from maropu/SPARK-31476-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 820733aee29f001037377d5dd3a3cf0d0ef8232d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../resources/sql-tests/inputs/extract.sql    | 16 ++-
 .../sql-tests/results/extract.sql.out         | 98 ++++++++++++++++++-
 2 files changed, 112 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/inputs/extract.sql b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
index cba7fc22779c5..abb9e82c9ef2e 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/extract.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/extract.sql
@@ -108,4 +108,18 @@ select date_part(c, c) from t;
 select date_part(null, c) from t;
 
 select date_part(i, i) from t;
-select date_part(null, i) from t;
\ No newline at end of file
+select date_part(null, i) from t;
+
+-- In SPARK-31476, we've supported extract('field', source), too
+select extract('year', c) from t;
+select extract('quarter', c) from t;
+select extract('month', c) from t;
+select extract('week', c) from t;
+select extract('day', c) from t;
+select extract('days', c) from t;
+select extract('dayofweek', c) from t;
+select extract('dow', c) from t;
+select extract('doy', c) from t;
+select extract('hour', c) from t;
+select extract('minute', c) from t;
+select extract('second', c) from t;
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index 220acfb0bce98..aea09e4b90f4e 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 84
+-- Number of queries: 96
 
 
 -- !query
@@ -677,3 +677,99 @@ select date_part(null, i) from t
 struct<date_part(NULL, t.`i`):double>
 -- !query output
 NULL
+
+
+-- !query
+select extract('year', c) from t
+-- !query schema
+struct<extract('year' FROM t.`c`):int>
+-- !query output
+2011
+
+
+-- !query
+select extract('quarter', c) from t
+-- !query schema
+struct<extract('quarter' FROM t.`c`):int>
+-- !query output
+2
+
+
+-- !query
+select extract('month', c) from t
+-- !query schema
+struct<extract('month' FROM t.`c`):int>
+-- !query output
+5
+
+
+-- !query
+select extract('week', c) from t
+-- !query schema
+struct<extract('week' FROM t.`c`):int>
+-- !query output
+18
+
+
+-- !query
+select extract('day', c) from t
+-- !query schema
+struct<extract('day' FROM t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select extract('days', c) from t
+-- !query schema
+struct<extract('days' FROM t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select extract('dayofweek', c) from t
+-- !query schema
+struct<extract('dayofweek' FROM t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select extract('dow', c) from t
+-- !query schema
+struct<extract('dow' FROM t.`c`):int>
+-- !query output
+6
+
+
+-- !query
+select extract('doy', c) from t
+-- !query schema
+struct<extract('doy' FROM t.`c`):int>
+-- !query output
+126
+
+
+-- !query
+select extract('hour', c) from t
+-- !query schema
+struct<extract('hour' FROM t.`c`):int>
+-- !query output
+7
+
+
+-- !query
+select extract('minute', c) from t
+-- !query schema
+struct<extract('minute' FROM t.`c`):int>
+-- !query output
+8
+
+
+-- !query
+select extract('second', c) from t
+-- !query schema
+struct<extract('second' FROM t.`c`):decimal(8,6)>
+-- !query output
+9.123456

From ed3e4bd09a6a183c3ba181eea7f2d47bde7fb1db Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 23 Apr 2020 14:12:10 +0900
Subject: [PATCH 0542/1280] [SPARK-31465][SQL][DOCS] Document Literal in SQL
 Reference

### What changes were proposed in this pull request?
Document Literal in SQL Reference

### Why are the changes needed?
Make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes
<img width="1049" alt="Screen Shot 2020-04-22 at 8 50 04 PM" src="https://user-images.githubusercontent.com/13592258/80057912-9ecb0c00-84dc-11ea-881e-1415108d674f.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 50 29 PM" src="https://user-images.githubusercontent.com/13592258/80057917-a12d6600-84dc-11ea-8884-81f2a94644d5.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 50 54 PM" src="https://user-images.githubusercontent.com/13592258/80057922-a4c0ed00-84dc-11ea-9857-75db50f7b054.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 51 15 PM" src="https://user-images.githubusercontent.com/13592258/80057927-a7234700-84dc-11ea-9124-45ae1f6143fd.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 51 44 PM" src="https://user-images.githubusercontent.com/13592258/80057932-ab4f6480-84dc-11ea-8393-cf005af13ce9.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 52 03 PM" src="https://user-images.githubusercontent.com/13592258/80057936-ad192800-84dc-11ea-8d78-9f071a82f1df.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 52 28 PM" src="https://user-images.githubusercontent.com/13592258/80057940-b0141880-84dc-11ea-97a7-f787cad0ee03.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 53 14 PM" src="https://user-images.githubusercontent.com/13592258/80057945-b30f0900-84dc-11ea-985f-c070609e2329.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 53 34 PM" src="https://user-images.githubusercontent.com/13592258/80057949-b5716300-84dc-11ea-9452-3f51137fe03d.png">

<img width="1050" alt="Screen Shot 2020-04-22 at 8 53 56 PM" src="https://user-images.githubusercontent.com/13592258/80057957-b904ea00-84dc-11ea-8b12-a6f00362aa55.png">

<img width="1049" alt="Screen Shot 2020-04-22 at 8 54 12 PM" src="https://user-images.githubusercontent.com/13592258/80057962-bacead80-84dc-11ea-94da-916b1d1c1756.png">

### How was this patch tested?
Manually build and check

Closes #28237 from huaxingao/literal.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 03fe9ee428ebb4544f0f47e861bccea43e0cb325)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml |   2 +
 docs/sql-ref-literals.md | 532 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 534 insertions(+)
 create mode 100644 docs/sql-ref-literals.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index a16e1144950ac..5d8c265bf84fe 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -78,6 +78,8 @@
   subitems:
     - text: Data Types
       url: sql-ref-datatypes.html
+    - text: Literals
+      url: sql-ref-literals.html
     - text: Null Semantics
       url: sql-ref-null-semantics.html
     - text: NaN Semantics
diff --git a/docs/sql-ref-literals.md b/docs/sql-ref-literals.md
new file mode 100644
index 0000000000000..7cf078c1e2c14
--- /dev/null
+++ b/docs/sql-ref-literals.md
@@ -0,0 +1,532 @@
+---
+layout: global
+title: Literals
+displayTitle: Literals
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+A literal (also known as a constant) represents a fixed data value. Spark SQL supports the following literals:
+
+ * [String Literal](#string-literal)
+ * [Binary Literal](#binary-literal)
+ * [Null Literal](#null-literal)
+ * [Boolean Literal](#boolean-literal)
+ * [Numeric Literal](#numeric-literal)
+ * [Datetime Literal](#datetime-literal)
+ * [Interval Literal](#interval-literal)
+
+### String Literal
+
+A string literal is used to specify a character string value.
+
+#### Syntax
+
+{% highlight sql %}
+'c [ ... ]' | "c [ ... ]"
+{% endhighlight %}
+
+#### <em>Parameters</em>
+
+<dl>
+  <dt><code><em>c</em></code></dt>
+  <dd>
+    One character from the character set. Use <code>\</code> to escape special characters (e.g., <code>'</code> or <code>\</code>).
+  </dd>
+</dl>
+
+#### <em>Examples</em>
+
+{% highlight sql %}
+SELECT 'Hello, World!' AS col;
++-------------+
+|          col|
++-------------+
+|Hello, World!|
++-------------+
+
+SELECT "SPARK SQL" AS col;
++---------+
+|      col|
++---------+
+|Spark SQL|
++---------+
+
+SELECT 'it\'s $10.' AS col;
++---------+
+|      col|
++---------+
+|It's $10.|
++---------+
+{% endhighlight %}
+
+### Binary Literal
+
+A binary literal is used to specify a byte sequence value.
+
+#### Syntax
+
+{% highlight sql %}
+X { 'c [ ... ]' | "c [ ... ]" }
+{% endhighlight %}
+
+#### <em>Parameters</em>
+
+<dl>
+  <dt><code><em>c</em></code></dt>
+  <dd>
+    One character from the character set.
+  </dd>
+</dl>
+
+#### <em>Examples</em>
+
+{% highlight sql %}
+SELECT X'123456' AS col;
++----------+
+|       col|
++----------+
+|[12 34 56]|
++----------+
+{% endhighlight %}
+
+### Null Literal
+
+A null literal is used to specify a null value.
+
+#### Syntax
+
+{% highlight sql %}
+NULL
+{% endhighlight %}
+
+#### Examples
+
+{% highlight sql %}
+SELECT NULL AS col;
++----+
+| col|
++----+
+|NULL|
++----+
+{% endhighlight %}
+
+### Boolean Literal
+
+A boolean literal is used to specify a boolean value.
+
+#### Syntax
+
+{% highlight sql %}
+TRUE | FALSE
+{% endhighlight %}
+
+#### Examples
+
+{% highlight sql %}
+SELECT TRUE AS col;
++----+
+| col|
++----+
+|true|
++----+
+{% endhighlight %}
+
+### Numeric Literal
+
+A numeric literal is used to specify a fixed or floating-point number.
+
+#### Integral Literal
+
+#### Syntax
+
+{% highlight sql %}
+[ + | - ] digit [ ... ] [ L | S | Y ]
+{% endhighlight %}
+
+#### Parameters
+
+<dl>
+  <dt><code><em>digit</em></code></dt>
+  <dd>
+    Any numeral from 0 to 9.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>L</em></code></dt>
+  <dd>
+    Case insensitive, indicates <code>BIGINT</code>, which is a 8-byte signed integer number.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>S</em></code></dt>
+  <dd>
+    Case insensitive, indicates <code>SMALLINT</code>, which is a 2-byte signed integer number.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>Y</em></code></dt>
+  <dd>
+    Case insensitive, indicates <code>TINYINT</code>, which is a 1-byte signed integer number.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>default (no postfix)</em></code></dt>
+  <dd>
+    Indicates a 4-byte signed integer number.
+  </dd>
+</dl>
+
+#### Examples
+
+{% highlight sql %}
+SELECT -2147483648 AS col;
++-----------+
+|        col|
++-----------+
+|-2147483648|
++-----------+
+
+SELECT 9223372036854775807l AS col;
++-------------------+
+|                col|
++-------------------+
+|9223372036854775807|
++-------------------+
+
+SELECT -32Y AS col;
++---+
+|col|
++---+
+|-32|
++---+
+
+SELECT 482S AS col;
++---+
+|col|
++---+
+|482|
++---+
+{% endhighlight %}
+
+#### Fractional Literals
+
+#### Syntax
+
+decimal literals:
+{% highlight sql %}
+decimal_digits { [ BD ] | [ exponent BD ] } | digit [ ... ] [ exponent ] BD
+{% endhighlight %}
+
+double literals:
+{% highlight sql %}
+decimal_digits  { D | exponent [ D ] }  | digit [ ... ] { exponent [ D ] | [ exponent ] D }
+{% endhighlight %}
+
+While decimal_digits is defined as
+{% highlight sql %}
+[ + | - ] { digit [ ... ] . [ digit [ ... ] ] | . digit [ ... ] }
+{% endhighlight %}
+
+and exponent is defined as
+{% highlight sql %}
+E [ + | - ] digit [ ... ]
+{% endhighlight %}
+
+#### Parameters
+
+<dl>
+  <dt><code><em>digit</em></code></dt>
+  <dd>
+    Any numeral from 0 to 9.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>D</em></code></dt>
+  <dd>
+    Case insensitive, indicates <code>DOUBLE</code>, which is a 8-byte double-precision floating point number.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>BD</em></code></dt>
+  <dd>
+    Case insensitive, indicates <code>DECIMAL</code>, with the total number of digits as precision and the number of digits to right of decimal point as scale.
+  </dd>
+</dl>
+
+#### Examples
+
+{% highlight sql %}
+SELECT 12.578 AS col;
++------+
+|   col|
++------+
+|12.578|
++------+
+
+SELECT -0.1234567 AS col;
++----------+
+|       col|
++----------+
+|-0.1234567|
++----------+
+
+SELECT -.1234567 AS col;
++----------+
+|       col|
++----------+
+|-0.1234567|
++----------+
+
+SELECT 123. AS col;
++---+
+|col|
++---+
+|123|
++---+
+
+SELECT 123.BD AS col;
++---+
+|col|
++---+
+|123|
++---+
+
+SELECT 5E2 AS col;
++-----+
+|  col|
++-----+
+|500.0|
++-----+
+
+SELECT 5D AS col;
++---+
+|col|
++---+
+|5.0|
++---+
+
+SELECT -5BD AS col;
++---+
+|col|
++---+
+| -5|
++---+
+
+SELECT 12.578e-2d AS col;
++-------+
+|    col|
++-------+
+|0.12578|
++-------+
+
+SELECT -.1234567E+2BD AS col;
++---------+
+|      col|
++---------+
+|-12.34567|
++---------+
+
+SELECT +3.e+3 AS col;
++------+
+|   col|
++------+
+|3000.0|
++------+
+
+SELECT -3.E-3D AS col;
++------+
+|   col|
++------+
+|-0.003|
++------+
+{% endhighlight %}
+
+### Datetime Literal
+
+A Datetime literal is used to specify a datetime value.
+
+#### Date Literal
+
+#### Syntax
+
+{% highlight sql %}
+DATE { 'yyyy' |
+       'yyyy-[m]m' |
+       'yyyy-[m]m-[d]d' |
+       'yyyy-[m]m-[d]d[T]' }
+{% endhighlight %}
+Note: defaults to <code>01</code> if month or day is not specified.
+
+#### Examples
+
+{% highlight sql %}
+SELECT DATE '1997' AS col;
++----------+
+|       col|
++----------+
+|1997-01-01|
++----------+
+
+SELECT DATE '1997-01' AS col;
++----------+
+|       col|
++----------+
+|1997-01-01|
++----------+
+
+SELECT DATE '2011-11-11' AS col;
++----------+
+|       col|
++----------+
+|2011-11-11|
++----------+
+{% endhighlight %}
+
+#### Timestamp Literal
+
+#### Syntax
+
+{% highlight sql %}
+TIMESTAMP { 'yyyy' |
+            'yyyy-[m]m' |
+            'yyyy-[m]m-[d]d' |
+            'yyyy-[m]m-[d]d ' |
+            'yyyy-[m]m-[d]d[T][h]h[:]' |
+            'yyyy-[m]m-[d]d[T][h]h:[m]m[:]' |
+            'yyyy-[m]m-[d]d[T][h]h:[m]m:[s]s[.]' |
+            'yyyy-[m]m-[d]d[T][h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]'}
+{% endhighlight %}
+Note: defaults to <code>00</code> if hour, minute or second is not specified. <br><br>
+`zone_id` should have one of the forms:
+<ul>
+  <li>Z - Zulu time zone UTC+0</li>
+  <li>+|-[h]h:[m]m</li>
+  <li>An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-, and a suffix in the formats:
+    <ul>
+      <li>+|-h[h]</li>
+      <li>+|-hh[:]mm</li>
+      <li>+|-hh:mm:ss</li>
+      <li>+|-hhmmss</li>
+    </ul>
+  </li>
+  <li>Region-based zone IDs in the form <code>area/city</code>, such as <code>Europe/Paris</code></li>
+</ul>
+Note: defaults to the session local timezone (set via <code>spark.sql.session.timeZone</code>) if <code>zone_id</code> is not specified.
+
+#### Examples
+
+{% highlight sql %}
+SELECT TIMESTAMP '1997-01-31 09:26:56.123' AS col;
++-----------------------+
+|                    col|
++-----------------------+
+|1997-01-31 09:26:56.123|
++-----------------------+
+
+SELECT TIMESTAMP '1997-01-31 09:26:56.66666666CST' AS col;
++--------------------------+
+|                      col |
++--------------------------+
+|1997-01-31 07:26:56.666666|
++--------------------------+
+
+SELECT TIMESTAMP '1997-01' AS col;
++-------------------+
+|                col|
++-------------------+
+|1997-01-01 00:00:00|
++-------------------+
+{% endhighlight %}
+
+### Interval Literal
+
+An interval literal is used to specify a fixed period of time.
+
+#### Syntax
+{% highlight sql %}
+{ INTERVAL interval_value interval_unit [ interval_value interval_unit ... ] |
+  INTERVAL 'interval_value interval_unit [ interval_value interval_unit ... ]' |
+  INTERVAL interval_string_value interval_unit TO interval_unit }
+{% endhighlight %}
+
+#### Parameters
+
+<dl>
+  <dt><code><em>interval_value</em></code></dt>
+  <dd>
+    <b>Syntax:</b>
+      <code>
+        [ + | - ] number_value | '[ + | - ] number_value'
+      </code><br>
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>interval_string_value</em></code></dt>
+    <dd>
+      year-month/day-time interval string.
+    </dd>
+</dl>
+<dl>
+  <dt><code><em>interval_unit</em></code></dt>
+  <dd>
+    <b>Syntax:</b><br>
+      <code>
+        YEAR[S] | MONTH[S] | WEEK[S] | DAY[S] | HOUR[S] | MINUTE[S] | SECOND[S] | <br>
+        MILLISECOND[S] | MICROSECOND[S]
+      </code>
+  </dd>
+</dl>
+
+#### Examples
+
+{% highlight sql %}
+SELECT INTERVAL 3 YEAR AS col;
++-------+
+|    col|
++-------+
+|3 years|
++-------+
+
+SELECT INTERVAL -2 HOUR '3' MINUTE AS col;
++--------------------+
+|                 col|
++--------------------+
+|-1 hours -57 minutes|
++--------------------+
+
+SELECT INTERVAL 'INTERVAL 1 YEAR 2 DAYS 3 HOURS';
++----------------------+
+|                   col|
++----------------------+
+|1 years 2 days 3 hours|
++----------------------+
+
+SELECT INTERVAL 1 YEARS 2 MONTH 3 WEEK 4 DAYS 5 HOUR 6 MINUTES 7 SECOND 8
+    MILLISECOND 9 MICROSECONDS AS col;
++-----------------------------------------------------------+
+|                                                        col|
++-----------------------------------------------------------+
+|1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds|
++-----------------------------------------------------------+
+
+SELECT INTERVAL '20 15:40:32.99899999' DAY TO SECOND AS col;
++---------------------------------------------+
+|                                          col|
++---------------------------------------------+
+|20 days 15 hours 40 minutes 32.998999 seconds|
++---------------------------------------------+
+{% endhighlight %}

From 2ebef75ec654bdbb01a4fa6a85225a7503de84b7 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Thu, 23 Apr 2020 14:32:10 +0900
Subject: [PATCH 0543/1280] [SPARK-31515][SQL] Canonicalize Cast should
 consider the value of needTimeZone

### What changes were proposed in this pull request?
Override the canonicalized fields with respect to the result of `needsTimeZone`.

### Why are the changes needed?
The current approach breaks sematic equal of two cast expressions that don't relate with datetime type. If we don't need to use `timeZone` information casting `from` type to `to` type, then the timeZoneId should not influence the canonicalize result.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
New UT added.

Closes #28288 from xuanyuanking/SPARK-31515.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit ca90e1932dcdc43748297c627ec857b6ea97dff7)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/catalyst/expressions/Canonicalize.scala | 10 +++++++++-
 .../apache/spark/sql/catalyst/expressions/Cast.scala  |  4 +++-
 .../sql/catalyst/expressions/CanonicalizeSuite.scala  | 11 ++++++++++-
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
index 4d218b936b3a2..a8031086d82f7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Canonicalize.scala
@@ -27,6 +27,7 @@ package org.apache.spark.sql.catalyst.expressions
  * The following rules are applied:
  *  - Names and nullability hints for [[org.apache.spark.sql.types.DataType]]s are stripped.
  *  - Names for [[GetStructField]] are stripped.
+ *  - TimeZoneId for [[Cast]] and [[AnsiCast]] are stripped if `needsTimeZone` is false.
  *  - Commutative and associative operations ([[Add]] and [[Multiply]]) have their children ordered
  *    by `hashCode`.
  *  - [[EqualTo]] and [[EqualNullSafe]] are reordered by `hashCode`.
@@ -35,7 +36,7 @@ package org.apache.spark.sql.catalyst.expressions
  */
 object Canonicalize {
   def execute(e: Expression): Expression = {
-    expressionReorder(ignoreNamesTypes(e))
+    expressionReorder(ignoreTimeZone(ignoreNamesTypes(e)))
   }
 
   /** Remove names and nullability from types, and names from `GetStructField`. */
@@ -46,6 +47,13 @@ object Canonicalize {
     case _ => e
   }
 
+  /** Remove TimeZoneId for Cast if needsTimeZone return false. */
+  private[expressions] def ignoreTimeZone(e: Expression): Expression = e match {
+    case c: CastBase if c.timeZoneId.nonEmpty && !c.needsTimeZone =>
+      c.withTimeZone(null)
+    case _ => e
+  }
+
   /** Collects adjacent commutative operations. */
   private def gatherCommutative(
       e: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 8d82956cc6f74..fa615d71a61a0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -279,7 +279,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   override lazy val resolved: Boolean =
     childrenResolved && checkInputDataTypes().isSuccess && (!needsTimeZone || timeZoneId.isDefined)
 
-  private[this] def needsTimeZone: Boolean = Cast.needsTimeZone(child.dataType, dataType)
+  def needsTimeZone: Boolean = Cast.needsTimeZone(child.dataType, dataType)
 
   // [[func]] assumes the input is no longer null because eval already does the null check.
   @inline private[this] def buildCast[T](a: Any, func: T => Any): Any = func(a.asInstanceOf[T])
@@ -1708,6 +1708,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
   """)
 case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
   extends CastBase {
+
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
@@ -1724,6 +1725,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
  */
 case class AnsiCast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
   extends CastBase {
+
   override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
     copy(timeZoneId = Option(timeZoneId))
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
index dd719437d618d..a043b4cbed1f1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CanonicalizeSuite.scala
@@ -17,10 +17,12 @@
 
 package org.apache.spark.sql.catalyst.expressions
 
+import java.util.TimeZone
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.plans.logical.Range
-import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
+import org.apache.spark.sql.types.{IntegerType, LongType, StructField, StructType}
 
 class CanonicalizeSuite extends SparkFunSuite {
 
@@ -86,4 +88,11 @@ class CanonicalizeSuite extends SparkFunSuite {
     val subExpr = Subtract(range.output.head, Literal(1))
     assert(addExpr.canonicalized.hashCode() != subExpr.canonicalized.hashCode())
   }
+
+  test("SPARK-31515: Canonicalize Cast should consider the value of needTimeZone") {
+    val literal = Literal(1)
+    val cast = Cast(literal, LongType)
+    val castWithTimeZoneId = Cast(literal, LongType, Some(TimeZone.getDefault.getID))
+    assert(castWithTimeZoneId.semanticEquals(cast))
+  }
 }

From 0f0299749421bc7328c6b962a9305bf460f51ddf Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 23 Apr 2020 15:03:20 +0900
Subject: [PATCH 0544/1280] [SPARK-31465][SQL][DOCS][FOLLOW-UP] Document
 Literal in SQL Reference

### What changes were proposed in this pull request?
Need to address a few more comments

### Why are the changes needed?
Fix a few problems

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
Manually build and check

Closes #28306 from huaxingao/literal-folllowup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit f543d6a1ee76de8cae417ff480fa9c0e0ce5343d)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-literals.md | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/docs/sql-ref-literals.md b/docs/sql-ref-literals.md
index 7cf078c1e2c14..0088f79cb7007 100644
--- a/docs/sql-ref-literals.md
+++ b/docs/sql-ref-literals.md
@@ -437,11 +437,11 @@ SELECT TIMESTAMP '1997-01-31 09:26:56.123' AS col;
 |1997-01-31 09:26:56.123|
 +-----------------------+
 
-SELECT TIMESTAMP '1997-01-31 09:26:56.66666666CST' AS col;
+SELECT TIMESTAMP '1997-01-31 09:26:56.66666666UTC+08:00' AS col;
 +--------------------------+
 |                      col |
 +--------------------------+
-|1997-01-31 07:26:56.666666|
+|1997-01-30 17:26:56.666666|
 +--------------------------+
 
 SELECT TIMESTAMP '1997-01' AS col;
@@ -508,7 +508,7 @@ SELECT INTERVAL -2 HOUR '3' MINUTE AS col;
 |-1 hours -57 minutes|
 +--------------------+
 
-SELECT INTERVAL 'INTERVAL 1 YEAR 2 DAYS 3 HOURS';
+SELECT INTERVAL '1 YEAR 2 DAYS 3 HOURS';
 +----------------------+
 |                   col|
 +----------------------+
@@ -523,6 +523,13 @@ SELECT INTERVAL 1 YEARS 2 MONTH 3 WEEK 4 DAYS 5 HOUR 6 MINUTES 7 SECOND 8
 |1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds|
 +-----------------------------------------------------------+
 
+SELECT INTERVAL '2-3' YEAR TO MONTH AS col;
++----------------+
+|             col|
++----------------+
+|2 years 3 months|
++----------------+
+
 SELECT INTERVAL '20 15:40:32.99899999' DAY TO SECOND AS col;
 +---------------------------------------------+
 |                                          col|

From 6fc37e31240849ab83e81b30ea1202c95a5a43c1 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 23 Apr 2020 08:49:55 +0000
Subject: [PATCH 0545/1280] [SPARK-31344][CORE][3.0] Polish implementation of
 barrier() and allGather()

### What changes were proposed in this pull request?

Backport this in order to backport #28245 more smoothly later.

1. Combine  `BarrierRequestToSync` and `AllGatherRequestToSync` into `RequestToSync`, which is distinguished by `RequestMethod` type.

2. Remove unnecessary Json serialization/deserialization

3. Clean up some codes to make runBarrier() and `BarrierCoordinator` more general

4. Remove unused imports.

### Why are the changes needed?

To make codes simpler for better maintain in the future.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

This is pure code refactor, so should be covered by existed tests.

Closes #28283 from Ngone51/backport-3.0.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/BarrierCoordinator.scala | 114 ++++--------------
 .../org/apache/spark/BarrierTaskContext.scala |  59 ++-------
 .../spark/api/python/PythonRunner.scala       |  17 +--
 .../scheduler/BarrierTaskContextSuite.scala   |   1 -
 python/pyspark/taskcontext.py                 |  15 +--
 5 files changed, 46 insertions(+), 160 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index be5036e82e4b2..5663055129d19 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -17,17 +17,12 @@
 
 package org.apache.spark
 
-import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.{compact, render}
-
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
 import org.apache.spark.scheduler.{LiveListenerBus, SparkListener, SparkListenerStageCompleted}
@@ -107,11 +102,13 @@ private[spark] class BarrierCoordinator(
     // An Array of RPCCallContexts for barrier tasks that have made a blocking runBarrier() call
     private val requesters: ArrayBuffer[RpcCallContext] = new ArrayBuffer[RpcCallContext](numTasks)
 
-    // An Array of allGather messages for barrier tasks that have made a blocking runBarrier() call
-    private val allGatherMessages: ArrayBuffer[String] = new Array[String](numTasks).to[ArrayBuffer]
+    // Messages from each barrier task that have made a blocking runBarrier() call.
+    // The messages will be replied to all tasks once sync finished.
+    private val messages = Array.ofDim[String](numTasks)
 
-    // The blocking requestMethod called by tasks to sync up for this stage attempt
-    private var requestMethodToSync: RequestMethod.Value = RequestMethod.BARRIER
+    // The request method which is called inside this barrier sync. All tasks should make sure
+    // that they're calling the same method within the same barrier sync phase.
+    private var requestMethod: RequestMethod.Value = _
 
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
@@ -140,28 +137,18 @@ private[spark] class BarrierCoordinator(
 
     // Process the global sync request. The barrier() call succeed if collected enough requests
     // within a configured time, otherwise fail all the pending requests.
-    def handleRequest(
-      requester: RpcCallContext,
-      request: RequestToSync
-    ): Unit = synchronized {
+    def handleRequest(requester: RpcCallContext, request: RequestToSync): Unit = synchronized {
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
-      val requestMethod = request.requestMethod
-      val partitionId = request.partitionId
-      val allGatherMessage = request match {
-        case ag: AllGatherRequestToSync => ag.allGatherMessage
-        case _ => ""
-      }
-
-      if (requesters.size == 0) {
-        requestMethodToSync = requestMethod
-      }
+      val curReqMethod = request.requestMethod
 
-      if (requestMethodToSync != requestMethod) {
+      if (requesters.isEmpty) {
+        requestMethod = curReqMethod
+      } else if (requestMethod != curReqMethod) {
         requesters.foreach(
           _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
-            s"`$requestMethod` during barrier epoch $barrierEpoch, which does not match " +
-            s"the current synchronized requestMethod `$requestMethodToSync`"
+            s"`$curReqMethod` during barrier epoch $barrierEpoch, which does not match " +
+            s"the current synchronized requestMethod `$requestMethod`"
           ))
         )
         cleanupBarrierStage(barrierId)
@@ -186,10 +173,11 @@ private[spark] class BarrierCoordinator(
         }
         // Add the requester to array of RPCCallContexts pending for reply.
         requesters += requester
-        allGatherMessages(partitionId) = allGatherMessage
+        messages(request.partitionId) = request.message
         logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received update from Task " +
           s"$taskId, current progress: ${requesters.size}/$numTasks.")
-        if (maybeFinishAllRequesters(requesters, numTasks)) {
+        if (requesters.size == numTasks) {
+          requesters.foreach(_.reply(messages))
           // Finished current barrier() call successfully, clean up ContextBarrierState and
           // increase the barrier epoch.
           logInfo(s"Barrier sync epoch $barrierEpoch from $barrierId received all updates from " +
@@ -201,25 +189,6 @@ private[spark] class BarrierCoordinator(
       }
     }
 
-    // Finish all the blocking barrier sync requests from a stage attempt successfully if we
-    // have received all the sync requests.
-    private def maybeFinishAllRequesters(
-        requesters: ArrayBuffer[RpcCallContext],
-        numTasks: Int): Boolean = {
-      if (requesters.size == numTasks) {
-        requestMethodToSync match {
-          case RequestMethod.BARRIER =>
-            requesters.foreach(_.reply(""))
-          case RequestMethod.ALL_GATHER =>
-            val json: String = compact(render(allGatherMessages))
-            requesters.foreach(_.reply(json))
-        }
-        true
-      } else {
-        false
-      }
-    }
-
     // Cleanup the internal state of a barrier stage attempt.
     def clear(): Unit = synchronized {
       // The global sync fails so the stage is expected to retry another attempt, all sync
@@ -239,11 +208,11 @@ private[spark] class BarrierCoordinator(
   }
 
   override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
-    case request: RequestToSync =>
+    case request @ RequestToSync(numTasks, stageId, stageAttemptId, _, _, _, _, _) =>
       // Get or init the ContextBarrierState correspond to the stage attempt.
-      val barrierId = ContextBarrierId(request.stageId, request.stageAttemptId)
+      val barrierId = ContextBarrierId(stageId, stageAttemptId)
       states.computeIfAbsent(barrierId,
-        (key: ContextBarrierId) => new ContextBarrierState(key, request.numTasks))
+        (key: ContextBarrierId) => new ContextBarrierState(key, numTasks))
       val barrierState = states.get(barrierId)
 
       barrierState.handleRequest(context, request)
@@ -256,61 +225,28 @@ private[spark] class BarrierCoordinator(
 
 private[spark] sealed trait BarrierCoordinatorMessage extends Serializable
 
-private[spark] sealed trait RequestToSync extends BarrierCoordinatorMessage {
-  def numTasks: Int
-  def stageId: Int
-  def stageAttemptId: Int
-  def taskAttemptId: Long
-  def barrierEpoch: Int
-  def partitionId: Int
-  def requestMethod: RequestMethod.Value
-}
-
-/**
- * A global sync request message from BarrierTaskContext, by `barrier()` call. Each request is
- * identified by stageId + stageAttemptId + barrierEpoch.
- *
- * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
- * @param stageId ID of current stage
- * @param stageAttemptId ID of current stage attempt
- * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
- * @param partitionId ID of the current partition the task is assigned to
- * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
- */
-private[spark] case class BarrierRequestToSync(
-  numTasks: Int,
-  stageId: Int,
-  stageAttemptId: Int,
-  taskAttemptId: Long,
-  barrierEpoch: Int,
-  partitionId: Int,
-  requestMethod: RequestMethod.Value
-) extends RequestToSync
-
 /**
- * A global sync request message from BarrierTaskContext, by `allGather()` call. Each request is
+ * A global sync request message from BarrierTaskContext. Each request is
  * identified by stageId + stageAttemptId + barrierEpoch.
  *
  * @param numTasks The number of global sync requests the BarrierCoordinator shall receive
  * @param stageId ID of current stage
  * @param stageAttemptId ID of current stage attempt
  * @param taskAttemptId Unique ID of current task
- * @param barrierEpoch ID of the `barrier()` call, a task may consist multiple `barrier()` calls
+ * @param barrierEpoch ID of a runBarrier() call, a task may consist multiple runBarrier() calls
  * @param partitionId ID of the current partition the task is assigned to
+ * @param message Message sent from the BarrierTaskContext
  * @param requestMethod The BarrierTaskContext method that was called to trigger BarrierCoordinator
- * @param allGatherMessage Message sent from the BarrierTaskContext if requestMethod is ALL_GATHER
  */
-private[spark] case class AllGatherRequestToSync(
+private[spark] case class RequestToSync(
   numTasks: Int,
   stageId: Int,
   stageAttemptId: Int,
   taskAttemptId: Long,
   barrierEpoch: Int,
   partitionId: Int,
-  requestMethod: RequestMethod.Value,
-  allGatherMessage: String
-) extends RequestToSync
+  message: String,
+  requestMethod: RequestMethod.Value) extends BarrierCoordinatorMessage
 
 private[spark] object RequestMethod extends Enumeration {
   val BARRIER, ALL_GATHER = Value
diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 0c2ceb1a02c7b..06f8024847b90 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -17,20 +17,13 @@
 
 package org.apache.spark
 
-import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
-import org.json4s.DefaultFormats
-import org.json4s.JsonAST._
-import org.json4s.JsonDSL._
-import org.json4s.jackson.JsonMethods.parse
-
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
 import org.apache.spark.internal.Logging
@@ -67,31 +60,7 @@ class BarrierTaskContext private[spark] (
   // from different tasks within the same barrier stage attempt to succeed.
   private lazy val numTasks = getTaskInfos().size
 
-  private def getRequestToSync(
-    numTasks: Int,
-    stageId: Int,
-    stageAttemptNumber: Int,
-    taskAttemptId: Long,
-    barrierEpoch: Int,
-    partitionId: Int,
-    requestMethod: RequestMethod.Value,
-    allGatherMessage: String
-  ): RequestToSync = {
-    requestMethod match {
-      case RequestMethod.BARRIER =>
-        BarrierRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch, partitionId, requestMethod)
-      case RequestMethod.ALL_GATHER =>
-        AllGatherRequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
-          barrierEpoch, partitionId, requestMethod, allGatherMessage)
-    }
-  }
-
-  private def runBarrier(
-    requestMethod: RequestMethod.Value,
-    allGatherMessage: String = ""
-  ): String = {
-
+  private def runBarrier(message: String, requestMethod: RequestMethod.Value): Array[String] = {
     logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) has entered " +
       s"the global sync, current barrier epoch is $barrierEpoch.")
     logTrace("Current callSite: " + Utils.getCallSite())
@@ -108,16 +77,16 @@ class BarrierTaskContext private[spark] (
     // Log the update of global sync every 60 seconds.
     timer.schedule(timerTask, 60000, 60000)
 
-    var json: String = ""
-
     try {
-      val abortableRpcFuture = barrierCoordinator.askAbortable[String](
-        message = getRequestToSync(numTasks, stageId, stageAttemptNumber,
-          taskAttemptId, barrierEpoch, partitionId, requestMethod, allGatherMessage),
+      val abortableRpcFuture = barrierCoordinator.askAbortable[Array[String]](
+        message = RequestToSync(numTasks, stageId, stageAttemptNumber, taskAttemptId,
+          barrierEpoch, partitionId, message, requestMethod),
         // Set a fixed timeout for RPC here, so users shall get a SparkException thrown by
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
 
+      // messages which consist of all barrier tasks' messages
+      var messages: Array[String] = null
       // Wait the RPC future to be completed, but every 1 second it will jump out waiting
       // and check whether current spark task is killed. If killed, then throw
       // a `TaskKilledException`, otherwise continue wait RPC until it completes.
@@ -125,7 +94,7 @@ class BarrierTaskContext private[spark] (
         while (!abortableRpcFuture.toFuture.isCompleted) {
           // wait RPC future for at most 1 second
           try {
-            json = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
+            messages = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
           } catch {
             case _: TimeoutException | _: InterruptedException =>
               // If `TimeoutException` thrown, waiting RPC future reach 1 second.
@@ -144,6 +113,7 @@ class BarrierTaskContext private[spark] (
         "global sync successfully, waited for " +
         s"${MILLISECONDS.toSeconds(System.currentTimeMillis() - startTime)} seconds, " +
         s"current barrier epoch is $barrierEpoch.")
+      messages
     } catch {
       case e: SparkException =>
         logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) failed " +
@@ -155,7 +125,6 @@ class BarrierTaskContext private[spark] (
       timerTask.cancel()
       timer.purge()
     }
-    json
   }
 
   /**
@@ -200,10 +169,7 @@ class BarrierTaskContext private[spark] (
    */
   @Experimental
   @Since("2.4.0")
-  def barrier(): Unit = {
-    runBarrier(RequestMethod.BARRIER)
-    ()
-  }
+  def barrier(): Unit = runBarrier("", RequestMethod.BARRIER)
 
   /**
    * :: Experimental ::
@@ -217,12 +183,7 @@ class BarrierTaskContext private[spark] (
    */
   @Experimental
   @Since("3.0.0")
-  def allGather(message: String): Array[String] = {
-    val json = runBarrier(RequestMethod.ALL_GATHER, message)
-    val jsonArray = parse(json)
-    implicit val formats = DefaultFormats
-    jsonArray.extract[Array[String]]
-  }
+  def allGather(message: String): Array[String] = runBarrier(message, RequestMethod.ALL_GATHER)
 
   /**
    * :: Experimental ::
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 06c9446c7534e..39c4abea933a7 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -414,22 +414,15 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
       )
       val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
       try {
-        var result: String = ""
-        requestMethod match {
+        val messages = requestMethod match {
           case BarrierTaskContextMessageProtocol.BARRIER_FUNCTION =>
             context.asInstanceOf[BarrierTaskContext].barrier()
-            result = BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS
+            Array(BarrierTaskContextMessageProtocol.BARRIER_RESULT_SUCCESS)
           case BarrierTaskContextMessageProtocol.ALL_GATHER_FUNCTION =>
-            val messages: Array[String] = context.asInstanceOf[BarrierTaskContext].allGather(
-              message
-            )
-            result = compact(render(JArray(
-              messages.map(
-                (message) => JString(message)
-              ).toList
-            )))
+            context.asInstanceOf[BarrierTaskContext].allGather(message)
         }
-        writeUTF(result, out)
+        out.writeInt(messages.length)
+        messages.foreach(writeUTF(_, out))
       } catch {
         case e: SparkException =>
           writeUTF(e.getMessage, out)
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 0dd8be72dc904..bcf1fe2c2aa11 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -19,7 +19,6 @@ package org.apache.spark.scheduler
 
 import java.io.File
 
-import scala.collection.mutable.ArrayBuffer
 import scala.util.Random
 
 import org.apache.spark._
diff --git a/python/pyspark/taskcontext.py b/python/pyspark/taskcontext.py
index e4fc64b732ba7..8f419a5e8446a 100644
--- a/python/pyspark/taskcontext.py
+++ b/python/pyspark/taskcontext.py
@@ -19,7 +19,7 @@
 import json
 
 from pyspark.java_gateway import local_connect_and_auth
-from pyspark.serializers import write_int, write_with_length, UTF8Deserializer
+from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
 
 
 class TaskContext(object):
@@ -133,7 +133,10 @@ def _load_from_socket(port, auth_secret, function, all_gather_message=None):
     sockfile.flush()
 
     # Collect result.
-    res = UTF8Deserializer().loads(sockfile)
+    len = read_int(sockfile)
+    res = []
+    for i in range(len):
+        res.append(UTF8Deserializer().loads(sockfile))
 
     # Release resources.
     sockfile.close()
@@ -232,13 +235,7 @@ def allGather(self, message=""):
             raise Exception("Not supported to call barrier() before initialize " +
                             "BarrierTaskContext.")
         else:
-            gathered_items = _load_from_socket(
-                self._port,
-                self._secret,
-                ALL_GATHER_FUNCTION,
-                message,
-            )
-            return [e for e in json.loads(gathered_items)]
+            return _load_from_socket(self._port, self._secret, ALL_GATHER_FUNCTION, message)
 
     def getTaskInfos(self):
         """

From 855a881ea695cdc9f0c22fdb0cfca4e16a07520b Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 23 Apr 2020 10:59:35 +0000
Subject: [PATCH 0546/1280] [SPARK-16775][DOC][FOLLOW-UP] Add migration guide
 for removed accumulator v1 APIs

### What changes were proposed in this pull request?

Add migration guide for removed accumulator v1 APIs.

### Why are the changes needed?

Provide better guidance for users' migration.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass Jenkins.

Closes #28309 from Ngone51/SPARK-16775-migration-guide.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6c018b31e28e2ebd23b2b30f57771ec36bf0402c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/core-migration-guide.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index 33406d010bb98..63baef145f01e 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -35,6 +35,8 @@ license: |
 
 - Deprecated method `AccumulableInfo.apply` have been removed because creating `AccumulableInfo` is disallowed.
 
+- Deprecated accumulator v1 APIs have been removed and please use v2 APIs instead.
+
 - Event log file will be written as UTF-8 encoding, and Spark History Server will replay event log files as UTF-8 encoding. Previously Spark wrote the event log file as default charset of driver JVM process, so Spark History Server of Spark 2.x is needed to read the old event log files in case of incompatible encoding.
 
 - A new protocol for fetching shuffle blocks is used. It's recommended that external shuffle services be upgraded when running Spark 3.0 apps. You can still use old external shuffle services by setting the configuration `spark.shuffle.useOldFetchProtocol` to `true`. Otherwise, Spark may run into errors with messages like `IllegalArgumentException: Unexpected message type: <number>`.

From 1b221f35abd1657a3ecd49335118bfd5dcb811ee Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 23 Apr 2020 14:43:27 +0000
Subject: [PATCH 0547/1280] [SPARK-31472][CORE][3.0] Make sure Barrier Task
 always return messages or exception with abortableRpcFuture check

### What changes were proposed in this pull request?

Rewrite the periodically check logic of  `abortableRpcFuture` to make sure that barrier task would always return either desired messages or expected exception.

This PR also simplify a bit around `AbortableRpcFuture`.

### Why are the changes needed?

Currently, the periodically check logic of  `abortableRpcFuture` is done by following:

```scala
...
var messages: Array[String] = null

while (!abortableRpcFuture.toFuture.isCompleted) {
   messages = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
   ...
}
return messages
```
It's possible that `abortableRpcFuture` complete before next invocation on `messages = ...`. In this case, the task may return null messages or execute successfully while it should throw exception(e.g. `SparkException` from `BarrierCoordinator`).

And here's a flaky test which caused by this bug:

```
[info] BarrierTaskContextSuite:
[info] - share messages with allGather() call *** FAILED *** (18 seconds, 705 milliseconds)
[info]   org.apache.spark.SparkException: Job aborted due to stage failure: Could not recover from a failed barrier ResultStage. Most recent failure reason: Stage failed because barrier task ResultTask(0, 2) finished unsuccessfully.
[info] java.lang.NullPointerException
[info] 	at scala.collection.mutable.ArrayOps$ofRef$.length$extension(ArrayOps.scala:204)
[info] 	at scala.collection.mutable.ArrayOps$ofRef.length(ArrayOps.scala:204)
[info] 	at scala.collection.IndexedSeqOptimized.toList(IndexedSeqOptimized.scala:285)
[info] 	at scala.collection.IndexedSeqOptimized.toList$(IndexedSeqOptimized.scala:284)
[info] 	at scala.collection.mutable.ArrayOps$ofRef.toList(ArrayOps.scala:198)
[info] 	at org.apache.spark.scheduler.BarrierTaskContextSuite.$anonfun$new$4(BarrierTaskContextSuite.scala:68)
...
```

The test exception can be reproduced by changing the line `messages = ...` to the following:

```scala
messages = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 10.micros)
Thread.sleep(5000)
```

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Manually test and update some unit tests.

Closes #28312 from Ngone51/cherry-pick-31472.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/BarrierTaskContext.scala | 30 +++++++++----------
 .../org/apache/spark/rpc/RpcEndpointRef.scala | 10 ++-----
 .../apache/spark/rpc/netty/NettyRpcEnv.scala  | 12 ++++----
 .../org/apache/spark/util/ThreadUtils.scala   |  5 ++--
 .../org/apache/spark/rpc/RpcEnvSuite.scala    | 12 ++++----
 5 files changed, 32 insertions(+), 37 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
index 06f8024847b90..4d765481eb836 100644
--- a/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierTaskContext.scala
@@ -20,9 +20,9 @@ package org.apache.spark
 import java.util.{Properties, Timer, TimerTask}
 
 import scala.collection.JavaConverters._
-import scala.concurrent.TimeoutException
 import scala.concurrent.duration._
 import scala.language.postfixOps
+import scala.util.{Failure, Success => ScalaSuccess, Try}
 
 import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.executor.TaskMetrics
@@ -85,28 +85,26 @@ class BarrierTaskContext private[spark] (
         // BarrierCoordinator on timeout, instead of RPCTimeoutException from the RPC framework.
         timeout = new RpcTimeout(365.days, "barrierTimeout"))
 
-      // messages which consist of all barrier tasks' messages
-      var messages: Array[String] = null
       // Wait the RPC future to be completed, but every 1 second it will jump out waiting
       // and check whether current spark task is killed. If killed, then throw
       // a `TaskKilledException`, otherwise continue wait RPC until it completes.
-      try {
-        while (!abortableRpcFuture.toFuture.isCompleted) {
+
+      while (!abortableRpcFuture.future.isCompleted) {
+        try {
           // wait RPC future for at most 1 second
-          try {
-            messages = ThreadUtils.awaitResult(abortableRpcFuture.toFuture, 1.second)
-          } catch {
-            case _: TimeoutException | _: InterruptedException =>
-              // If `TimeoutException` thrown, waiting RPC future reach 1 second.
-              // If `InterruptedException` thrown, it is possible this task is killed.
-              // So in this two cases, we should check whether task is killed and then
-              // throw `TaskKilledException`
-              taskContext.killTaskIfInterrupted()
+          Thread.sleep(1000)
+        } catch {
+          case _: InterruptedException => // task is killed by driver
+        } finally {
+          Try(taskContext.killTaskIfInterrupted()) match {
+            case ScalaSuccess(_) => // task is still running healthily
+            case Failure(e) => abortableRpcFuture.abort(e)
           }
         }
-      } finally {
-        abortableRpcFuture.abort(taskContext.getKillReason().getOrElse("Unknown reason."))
       }
+      // messages which consist of all barrier tasks' messages. The future will return the
+      // desired messages if it is completed successfully. Otherwise, exception could be thrown.
+      val messages = abortableRpcFuture.future.value.get.get
 
       barrierEpoch += 1
       logInfo(s"Task $taskAttemptId from Stage $stageId(Attempt $stageAttemptNumber) finished " +
diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
index 56f3d377f8e2a..a3d27b0d09923 100644
--- a/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/RpcEndpointRef.scala
@@ -114,11 +114,7 @@ private[spark] class RpcAbortException(message: String) extends Exception(messag
  * A wrapper for [[Future]] but add abort method.
  * This is used in long run RPC and provide an approach to abort the RPC.
  */
-private[spark] class AbortableRpcFuture[T: ClassTag](
-    future: Future[T],
-    onAbort: String => Unit) {
-
-  def abort(reason: String): Unit = onAbort(reason)
-
-  def toFuture: Future[T] = future
+private[spark]
+class AbortableRpcFuture[T: ClassTag](val future: Future[T], onAbort: Throwable => Unit) {
+  def abort(t: Throwable): Unit = onAbort(t)
 }
diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
index 265e158d7c5e3..9259ec7699262 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyRpcEnv.scala
@@ -208,6 +208,7 @@ private[netty] class NettyRpcEnv(
       message: RequestMessage, timeout: RpcTimeout): AbortableRpcFuture[T] = {
     val promise = Promise[Any]()
     val remoteAddr = message.receiver.address
+    var rpcMsg: Option[RpcOutboxMessage] = None
 
     def onFailure(e: Throwable): Unit = {
       if (!promise.tryFailure(e)) {
@@ -226,8 +227,9 @@ private[netty] class NettyRpcEnv(
         }
     }
 
-    def onAbort(reason: String): Unit = {
-      onFailure(new RpcAbortException(reason))
+    def onAbort(t: Throwable): Unit = {
+      onFailure(t)
+      rpcMsg.foreach(_.onAbort())
     }
 
     try {
@@ -242,10 +244,10 @@ private[netty] class NettyRpcEnv(
         val rpcMessage = RpcOutboxMessage(message.serialize(this),
           onFailure,
           (client, response) => onSuccess(deserialize[Any](client, response)))
+        rpcMsg = Option(rpcMessage)
         postToOutbox(message.receiver, rpcMessage)
         promise.future.failed.foreach {
           case _: TimeoutException => rpcMessage.onTimeout()
-          case _: RpcAbortException => rpcMessage.onAbort()
           case _ =>
         }(ThreadUtils.sameThread)
       }
@@ -270,7 +272,7 @@ private[netty] class NettyRpcEnv(
   }
 
   private[netty] def ask[T: ClassTag](message: RequestMessage, timeout: RpcTimeout): Future[T] = {
-    askAbortable(message, timeout).toFuture
+    askAbortable(message, timeout).future
   }
 
   private[netty] def serialize(content: Any): ByteBuffer = {
@@ -547,7 +549,7 @@ private[netty] class NettyRpcEndpointRef(
   }
 
   override def ask[T: ClassTag](message: Any, timeout: RpcTimeout): Future[T] = {
-    askAbortable(message, timeout).toFuture
+    askAbortable(message, timeout).future
   }
 
   override def send(message: Any): Unit = {
diff --git a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
index e7872bb9cb6b0..78206c51c1028 100644
--- a/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/ThreadUtils.scala
@@ -29,7 +29,6 @@ import scala.util.control.NonFatal
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 
 import org.apache.spark.SparkException
-import org.apache.spark.rpc.RpcAbortException
 
 private[spark] object ThreadUtils {
 
@@ -299,7 +298,7 @@ private[spark] object ThreadUtils {
       // TimeoutException and RpcAbortException is thrown in the current thread, so not need to warp
       // the exception.
       case NonFatal(t)
-          if !t.isInstanceOf[TimeoutException] && !t.isInstanceOf[RpcAbortException] =>
+          if !t.isInstanceOf[TimeoutException] =>
         throw new SparkException("Exception thrown in awaitResult: ", t)
     }
   }
@@ -316,7 +315,7 @@ private[spark] object ThreadUtils {
       case e: SparkFatalException =>
         throw e.throwable
       case NonFatal(t)
-        if !t.isInstanceOf[TimeoutException] && !t.isInstanceOf[RpcAbortException] =>
+        if !t.isInstanceOf[TimeoutException] =>
         throw new SparkException("Exception thrown in awaitResult: ", t)
     }
   }
diff --git a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
index c10f2c244e133..01c67b3856df4 100644
--- a/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/RpcEnvSuite.scala
@@ -209,7 +209,7 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
     // Use anotherEnv to find out the RpcEndpointRef
     val rpcEndpointRef = anotherEnv.setupEndpointRef(env.address, "ask-abort")
     try {
-      val e = intercept[RpcAbortException] {
+      val e = intercept[SparkException] {
         val timeout = new RpcTimeout(10.seconds, shortProp)
         val abortableRpcFuture = rpcEndpointRef.askAbortable[String](
           "hello", timeout)
@@ -217,15 +217,15 @@ abstract class RpcEnvSuite extends SparkFunSuite with BeforeAndAfterAll {
         new Thread {
           override def run: Unit = {
             Thread.sleep(100)
-            abortableRpcFuture.abort("TestAbort")
+            abortableRpcFuture.abort(new RuntimeException("TestAbort"))
           }
         }.start()
 
-        timeout.awaitResult(abortableRpcFuture.toFuture)
+        timeout.awaitResult(abortableRpcFuture.future)
       }
-      // The SparkException cause should be a RpcAbortException with "TestAbort" message
-      assert(e.isInstanceOf[RpcAbortException])
-      assert(e.getMessage.contains("TestAbort"))
+      // The SparkException cause should be a RuntimeException with "TestAbort" message
+      assert(e.getCause.isInstanceOf[RuntimeException])
+      assert(e.getCause.getMessage.contains("TestAbort"))
     } finally {
       anotherEnv.shutdown()
       anotherEnv.awaitTermination()

From 55cd20901e1e27af63985a0fee1ac900fdcfccb9 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 23 Apr 2020 15:07:44 +0000
Subject: [PATCH 0548/1280] [SPARK-31522][SQL] Hive metastore client
 initialization related configurations should be static

### What changes were proposed in this pull request?
HiveClient instance is cross-session, the following configurations which are defined in HiveUtils and used to create it should be considered static:

1. spark.sql.hive.metastore.version - used to determine the hive version in Spark
2. spark.sql.hive.metastore.jars - hive metastore related jars location which is used by spark to create hive client
3. spark.sql.hive.metastore.sharedPrefixes and spark.sql.hive.metastore.barrierPrefixes -  package names of classes that are shared or separated between SparkContextLoader and hive client class loader

Those are used only once when creating the hive metastore client. They should be static in SQLConf for retrieving them correctly. We should avoid them being changed by users with SET/RESET command.

Speaking of spark.sql.hive.version - the fake of the spark.sql.hive.metastore.version, it is used by jdbc/thrift client for backward compatibility.

### Why are the changes needed?

bugfix, these configurations should not be changed.

### Does this PR introduce any user-facing change?

Yes, the following set of configs are not allowed to change.
```
Seq("spark.sql.hive.metastore.version ",
      "spark.sql.hive.metastore.jars",
      "spark.sql.hive.metastore.sharedPrefixes",
      "spark.sql.hive.metastore.barrierPrefixes")
```
### How was this patch tested?

add unit test

Closes #28302 from yaooqinn/SPARK-31522.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8dc2c0247be0be370d764653d5a68b7aa7948e39)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/hive/HiveUtils.scala   | 11 +++++------
 .../spark/sql/hive/execution/SQLQuerySuite.scala      | 10 ++++++++++
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index d0f79880e85e9..04caf57efdc74 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -61,7 +61,7 @@ private[spark] object HiveUtils extends Logging {
   /** The version of hive used internally by Spark SQL. */
   val builtinHiveVersion: String = if (isHive23) hiveVersion else "1.2.1"
 
-  val HIVE_METASTORE_VERSION = buildConf("spark.sql.hive.metastore.version")
+  val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version")
     .doc("Version of the Hive metastore. Available options are " +
         "<code>0.12.0</code> through <code>2.3.7</code> and " +
         "<code>3.0.0</code> through <code>3.1.2</code>.")
@@ -75,10 +75,9 @@ private[spark] object HiveUtils extends Logging {
   val FAKE_HIVE_VERSION = buildConf("spark.sql.hive.version")
     .doc(s"deprecated, please use ${HIVE_METASTORE_VERSION.key} to get the Hive version in Spark.")
     .version("1.1.1")
-    .stringConf
-    .createWithDefault(builtinHiveVersion)
+    .fallbackConf(HIVE_METASTORE_VERSION)
 
-  val HIVE_METASTORE_JARS = buildConf("spark.sql.hive.metastore.jars")
+  val HIVE_METASTORE_JARS = buildStaticConf("spark.sql.hive.metastore.jars")
     .doc(s"""
       | Location of the jars that should be used to instantiate the HiveMetastoreClient.
       | This property can be one of three options: "
@@ -137,7 +136,7 @@ private[spark] object HiveUtils extends Logging {
     .booleanConf
     .createWithDefault(true)
 
-  val HIVE_METASTORE_SHARED_PREFIXES = buildConf("spark.sql.hive.metastore.sharedPrefixes")
+  val HIVE_METASTORE_SHARED_PREFIXES = buildStaticConf("spark.sql.hive.metastore.sharedPrefixes")
     .doc("A comma separated list of class prefixes that should be loaded using the classloader " +
       "that is shared between Spark SQL and a specific version of Hive. An example of classes " +
       "that should be shared is JDBC drivers that are needed to talk to the metastore. Other " +
@@ -151,7 +150,7 @@ private[spark] object HiveUtils extends Logging {
   private def jdbcPrefixes = Seq(
     "com.mysql.jdbc", "org.postgresql", "com.microsoft.sqlserver", "oracle.jdbc")
 
-  val HIVE_METASTORE_BARRIER_PREFIXES = buildConf("spark.sql.hive.metastore.barrierPrefixes")
+  val HIVE_METASTORE_BARRIER_PREFIXES = buildStaticConf("spark.sql.hive.metastore.barrierPrefixes")
     .doc("A comma separated list of class prefixes that should explicitly be reloaded for each " +
       "version of Hive that Spark SQL is communicating with. For example, Hive UDFs that are " +
       "declared in a prefix that typically would be shared (i.e. <code>org.apache.spark.*</code>).")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 138dcc586a46a..e93f585b5c86a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2493,6 +2493,16 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
       }
     }
   }
+
+  test("SPARK-31522: hive metastore related configurations should be static") {
+    Seq("spark.sql.hive.metastore.version",
+      "spark.sql.hive.metastore.jars",
+      "spark.sql.hive.metastore.sharedPrefixes",
+      "spark.sql.hive.metastore.barrierPrefixes").foreach { key =>
+      val e = intercept[AnalysisException](sql(s"set $key=abc"))
+      assert(e.getMessage.contains("Cannot modify the value of a static config"))
+    }
+  }
 }
 
 class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite

From 4246732bde8911c5c71f56a622c551faa8609ecc Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Fri, 24 Apr 2020 11:19:20 +0900
Subject: [PATCH 0549/1280] [SPARK-31526][SQL][TESTS] Add a new test suite for
 ExpressionInfo

### What changes were proposed in this pull request?

This PR intends to add a new test suite for `ExpressionInfo`. Major changes are as follows;

 - Added a new test suite named `ExpressionInfoSuite`
 - To improve test coverage, added a test for error handling in `ExpressionInfoSuite`
 - Moved the `ExpressionInfo`-related tests from `UDFSuite` to `ExpressionInfoSuite`
 - Moved the related tests from `SQLQuerySuite` to `ExpressionInfoSuite`
 - Added a comment in `ExpressionInfoSuite` (followup of https://github.com/apache/spark/pull/28224)

### Why are the changes needed?

To improve test suites/coverage.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #28308 from maropu/SPARK-31526.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 42f496f6ac82e51b5ce2463a375f78cb263f1e32)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../expressions/ExpressionDescription.java    |   6 +
 .../org/apache/spark/sql/SQLQuerySuite.scala  |  80 ---------
 .../scala/org/apache/spark/sql/UDFSuite.scala |  31 ----
 .../sql/expressions/ExpressionInfoSuite.scala | 156 ++++++++++++++++++
 4 files changed, 162 insertions(+), 111 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
index 089fbe53d5171..579f4b387b3f9 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java
@@ -103,6 +103,12 @@
     String arguments() default "";
     String examples() default "";
     String note() default "";
+    /**
+     * Valid group names are almost the same with one defined as `groupname` in
+     * `sql/functions.scala`. But, `collection_funcs` is split into fine-grained three groups:
+     * `array_funcs`, `map_funcs`, and `json_funcs`. See `ExpressionInfo` for the
+     * detailed group names.
+     */
     String group() default "";
     String since() default "";
     String deprecated() default "";
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 3c514f25b835c..58a40b0df9ea4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -22,8 +22,6 @@ import java.net.{MalformedURLException, URL}
 import java.sql.{Date, Timestamp}
 import java.util.concurrent.atomic.AtomicBoolean
 
-import scala.collection.parallel.immutable.ParVector
-
 import org.apache.spark.{AccumulatorSuite, SparkException}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
 import org.apache.spark.sql.catalyst.expressions.GenericRow
@@ -31,7 +29,6 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
 import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.catalyst.util.StringUtils
-import org.apache.spark.sql.execution.HiveResult.hiveResultString
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
@@ -126,83 +123,6 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
-  test("using _FUNC_ instead of function names in examples") {
-    val exampleRe = "(>.*;)".r
-    val setStmtRe = "(?i)^(>\\s+set\\s+).+".r
-    val ignoreSet = Set(
-      // Examples for CaseWhen show simpler syntax:
-      // `CASE WHEN ... THEN ... WHEN ... THEN ... END`
-      "org.apache.spark.sql.catalyst.expressions.CaseWhen",
-      // _FUNC_ is replaced by `locate` but `locate(... IN ...)` is not supported
-      "org.apache.spark.sql.catalyst.expressions.StringLocate",
-      // _FUNC_ is replaced by `%` which causes a parsing error on `SELECT %(2, 1.8)`
-      "org.apache.spark.sql.catalyst.expressions.Remainder",
-      // Examples demonstrate alternative names, see SPARK-20749
-      "org.apache.spark.sql.catalyst.expressions.Length")
-    spark.sessionState.functionRegistry.listFunction().foreach { funcId =>
-      val info = spark.sessionState.catalog.lookupFunctionInfo(funcId)
-      val className = info.getClassName
-      withClue(s"Expression class '$className'") {
-        val exprExamples = info.getOriginalExamples
-        if (!exprExamples.isEmpty && !ignoreSet.contains(className)) {
-          assert(exampleRe.findAllIn(exprExamples).toIterable
-            .filter(setStmtRe.findFirstIn(_).isEmpty) // Ignore SET commands
-            .forall(_.contains("_FUNC_")))
-        }
-      }
-    }
-  }
-
-  test("check outputs of expression examples") {
-    def unindentAndTrim(s: String): String = {
-      s.replaceAll("\n\\s+", "\n").trim
-    }
-    val beginSqlStmtRe = "  > ".r
-    val endSqlStmtRe = ";\n".r
-    def checkExampleSyntax(example: String): Unit = {
-      val beginStmtNum = beginSqlStmtRe.findAllIn(example).length
-      val endStmtNum = endSqlStmtRe.findAllIn(example).length
-      assert(beginStmtNum === endStmtNum,
-        "The number of ` > ` does not match to the number of `;`")
-    }
-    val exampleRe = """^(.+);\n(?s)(.+)$""".r
-    val ignoreSet = Set(
-      // One of examples shows getting the current timestamp
-      "org.apache.spark.sql.catalyst.expressions.UnixTimestamp",
-      // Random output without a seed
-      "org.apache.spark.sql.catalyst.expressions.Rand",
-      "org.apache.spark.sql.catalyst.expressions.Randn",
-      "org.apache.spark.sql.catalyst.expressions.Shuffle",
-      "org.apache.spark.sql.catalyst.expressions.Uuid",
-      // The example calls methods that return unstable results.
-      "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection")
-
-    val parFuncs = new ParVector(spark.sessionState.functionRegistry.listFunction().toVector)
-    parFuncs.foreach { funcId =>
-      // Examples can change settings. We clone the session to prevent tests clashing.
-      val clonedSpark = spark.cloneSession()
-      // Coalescing partitions can change result order, so disable it.
-      clonedSpark.sessionState.conf.setConf(SQLConf.COALESCE_PARTITIONS_ENABLED, false)
-      val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId)
-      val className = info.getClassName
-      if (!ignoreSet.contains(className)) {
-        withClue(s"Function '${info.getName}', Expression class '$className'") {
-          val example = info.getExamples
-          checkExampleSyntax(example)
-          example.split("  > ").toList.foreach(_ match {
-            case exampleRe(sql, output) =>
-              val df = clonedSpark.sql(sql)
-              val actual = unindentAndTrim(
-                hiveResultString(df.queryExecution.executedPlan).mkString("\n"))
-              val expected = unindentAndTrim(output)
-              assert(actual === expected)
-            case _ =>
-          })
-        }
-      }
-    }
-  }
-
   test("SPARK-6743: no columns from cache") {
     Seq(
       (83, 0, 38),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index 92ea0ce246d4e..e2747d7db9f3f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -20,8 +20,6 @@ package org.apache.spark.sql
 import java.math.BigDecimal
 
 import org.apache.spark.sql.api.java._
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.{QueryExecution, SimpleMode}
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
@@ -534,35 +532,6 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     assert(spark.range(2).select(nonDeterministicJavaUDF()).distinct().count() == 2)
   }
 
-  test("Replace _FUNC_ in UDF ExpressionInfo") {
-    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("upper"))
-    assert(info.getName === "upper")
-    assert(info.getClassName === "org.apache.spark.sql.catalyst.expressions.Upper")
-    assert(info.getUsage === "upper(str) - Returns `str` with all characters changed to uppercase.")
-    assert(info.getExamples.contains("> SELECT upper('SparkSql');"))
-    assert(info.getSince === "1.0.1")
-    assert(info.getNote === "")
-    assert(info.getExtended.contains("> SELECT upper('SparkSql');"))
-  }
-
-  test("group info in ExpressionInfo") {
-    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("sum"))
-    assert(info.getGroup === "agg_funcs")
-
-    Seq("agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs", "window_funcs")
-        .foreach { groupName =>
-      val info = new ExpressionInfo(
-        "testClass", null, "testName", null, "", "", "", groupName, "", "")
-      assert(info.getGroup === groupName)
-    }
-
-    val errMsg = intercept[IllegalArgumentException] {
-      val invalidGroupName = "invalid_group_funcs"
-      new ExpressionInfo("testClass", null, "testName", null, "", "", "", invalidGroupName, "", "")
-    }.getMessage
-    assert(errMsg.contains("'group' is malformed in the expression [testName]."))
-  }
-
   test("SPARK-28521 error message for CAST(parameter types contains DataType)") {
     val e = intercept[AnalysisException] {
       spark.sql("SELECT CAST(1)")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
new file mode 100644
index 0000000000000..9a6fe46c48ec7
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.expressions
+
+import scala.collection.parallel.immutable.ParVector
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.FunctionIdentifier
+import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
+import org.apache.spark.sql.execution.HiveResult.hiveResultString
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
+
+  test("Replace _FUNC_ in ExpressionInfo") {
+    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("upper"))
+    assert(info.getName === "upper")
+    assert(info.getClassName === "org.apache.spark.sql.catalyst.expressions.Upper")
+    assert(info.getUsage === "upper(str) - Returns `str` with all characters changed to uppercase.")
+    assert(info.getExamples.contains("> SELECT upper('SparkSql');"))
+    assert(info.getSince === "1.0.1")
+    assert(info.getNote === "")
+    assert(info.getExtended.contains("> SELECT upper('SparkSql');"))
+  }
+
+  test("group info in ExpressionInfo") {
+    val info = spark.sessionState.catalog.lookupFunctionInfo(FunctionIdentifier("sum"))
+    assert(info.getGroup === "agg_funcs")
+
+    Seq("agg_funcs", "array_funcs", "datetime_funcs", "json_funcs", "map_funcs", "window_funcs")
+        .foreach { groupName =>
+      val info = new ExpressionInfo(
+        "testClass", null, "testName", null, "", "", "", groupName, "", "")
+      assert(info.getGroup === groupName)
+    }
+
+    val errMsg = intercept[IllegalArgumentException] {
+      val invalidGroupName = "invalid_group_funcs"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", invalidGroupName, "", "")
+    }.getMessage
+    assert(errMsg.contains("'group' is malformed in the expression [testName]."))
+  }
+
+  test("error handling in ExpressionInfo") {
+    val errMsg1 = intercept[IllegalArgumentException] {
+      val invalidNote = "  invalid note"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", invalidNote, "", "", "")
+    }.getMessage
+    assert(errMsg1.contains("'note' is malformed in the expression [testName]."))
+
+    val errMsg2 = intercept[IllegalArgumentException] {
+      val invalidSince = "-3.0.0"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", "", invalidSince, "")
+    }.getMessage
+    assert(errMsg2.contains("'since' is malformed in the expression [testName]."))
+
+    val errMsg3 = intercept[IllegalArgumentException] {
+      val invalidDeprecated = "  invalid deprecated"
+      new ExpressionInfo("testClass", null, "testName", null, "", "", "", "", "", invalidDeprecated)
+    }.getMessage
+    assert(errMsg3.contains("'deprecated' is malformed in the expression [testName]."))
+  }
+
+  test("using _FUNC_ instead of function names in examples") {
+    val exampleRe = "(>.*;)".r
+    val setStmtRe = "(?i)^(>\\s+set\\s+).+".r
+    val ignoreSet = Set(
+      // Examples for CaseWhen show simpler syntax:
+      // `CASE WHEN ... THEN ... WHEN ... THEN ... END`
+      "org.apache.spark.sql.catalyst.expressions.CaseWhen",
+      // _FUNC_ is replaced by `locate` but `locate(... IN ...)` is not supported
+      "org.apache.spark.sql.catalyst.expressions.StringLocate",
+      // _FUNC_ is replaced by `%` which causes a parsing error on `SELECT %(2, 1.8)`
+      "org.apache.spark.sql.catalyst.expressions.Remainder",
+      // Examples demonstrate alternative names, see SPARK-20749
+      "org.apache.spark.sql.catalyst.expressions.Length")
+    spark.sessionState.functionRegistry.listFunction().foreach { funcId =>
+      val info = spark.sessionState.catalog.lookupFunctionInfo(funcId)
+      val className = info.getClassName
+      withClue(s"Expression class '$className'") {
+        val exprExamples = info.getOriginalExamples
+        if (!exprExamples.isEmpty && !ignoreSet.contains(className)) {
+          assert(exampleRe.findAllIn(exprExamples).toIterable
+            .filter(setStmtRe.findFirstIn(_).isEmpty) // Ignore SET commands
+            .forall(_.contains("_FUNC_")))
+        }
+      }
+    }
+  }
+
+  test("check outputs of expression examples") {
+    def unindentAndTrim(s: String): String = {
+      s.replaceAll("\n\\s+", "\n").trim
+    }
+    val beginSqlStmtRe = "  > ".r
+    val endSqlStmtRe = ";\n".r
+    def checkExampleSyntax(example: String): Unit = {
+      val beginStmtNum = beginSqlStmtRe.findAllIn(example).length
+      val endStmtNum = endSqlStmtRe.findAllIn(example).length
+      assert(beginStmtNum === endStmtNum,
+        "The number of ` > ` does not match to the number of `;`")
+    }
+    val exampleRe = """^(.+);\n(?s)(.+)$""".r
+    val ignoreSet = Set(
+      // One of examples shows getting the current timestamp
+      "org.apache.spark.sql.catalyst.expressions.UnixTimestamp",
+      // Random output without a seed
+      "org.apache.spark.sql.catalyst.expressions.Rand",
+      "org.apache.spark.sql.catalyst.expressions.Randn",
+      "org.apache.spark.sql.catalyst.expressions.Shuffle",
+      "org.apache.spark.sql.catalyst.expressions.Uuid",
+      // The example calls methods that return unstable results.
+      "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection")
+
+    val parFuncs = new ParVector(spark.sessionState.functionRegistry.listFunction().toVector)
+    parFuncs.foreach { funcId =>
+      // Examples can change settings. We clone the session to prevent tests clashing.
+      val clonedSpark = spark.cloneSession()
+      // Coalescing partitions can change result order, so disable it.
+      clonedSpark.sessionState.conf.setConf(SQLConf.COALESCE_PARTITIONS_ENABLED, false)
+      val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId)
+      val className = info.getClassName
+      if (!ignoreSet.contains(className)) {
+        withClue(s"Function '${info.getName}', Expression class '$className'") {
+          val example = info.getExamples
+          checkExampleSyntax(example)
+          example.split("  > ").toList.foreach {
+            case exampleRe(sql, output) =>
+              val df = clonedSpark.sql(sql)
+              val actual = unindentAndTrim(
+                hiveResultString(df.queryExecution.executedPlan).mkString("\n"))
+              val expected = unindentAndTrim(output)
+              assert(actual === expected)
+            case _ =>
+          }
+        }
+      }
+    }
+  }
+}

From b666d8360e39aba583c1b98a6c93826046fccc1b Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 24 Apr 2020 02:21:53 +0000
Subject: [PATCH 0550/1280] [SPARK-31488][SQL] Support `java.time.LocalDate` in
 Parquet filter pushdown

### What changes were proposed in this pull request?
1. Modified `ParquetFilters.valueCanMakeFilterOn()` to accept filters with `java.time.LocalDate` attributes.
2. Modified `ParquetFilters.dateToDays()` to support both types `java.sql.Date` and `java.time.LocalDate` in conversions to days.
3. Add implicit conversion from `LocalDate` to `Expression` (`Literal`).

### Why are the changes needed?
To support pushed down filters with `java.time.LocalDate` attributes. Before the changes, date filters are not pushed down to Parquet datasource when `spark.sql.datetime.java8API.enabled` is `true`.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added a test to `ParquetFilterSuite`

Closes #28259 from MaxGekk/parquet-filter-java8-date-time.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 26165427c7ec0b0b64a527edae42b05ba9d47a19)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/dsl/package.scala      |  2 +
 .../datasources/parquet/ParquetFilters.scala  | 21 ++--
 .../parquet/ParquetFilterSuite.scala          | 99 ++++++++++---------
 3 files changed, 69 insertions(+), 53 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index b4a8bafe22dfb..cc96d905a86f2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst
 
 import java.sql.{Date, Timestamp}
+import java.time.LocalDate
 
 import scala.language.implicitConversions
 
@@ -146,6 +147,7 @@ package object dsl {
     implicit def doubleToLiteral(d: Double): Literal = Literal(d)
     implicit def stringToLiteral(s: String): Literal = Literal.create(s, StringType)
     implicit def dateToLiteral(d: Date): Literal = Literal(d)
+    implicit def localDateToLiteral(d: LocalDate): Literal = Literal(d)
     implicit def bigDecimalToLiteral(d: BigDecimal): Literal = Literal(d.underlying())
     implicit def bigDecimalToLiteral(d: java.math.BigDecimal): Literal = Literal(d)
     implicit def decimalToLiteral(d: Decimal): Literal = Literal(d)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index f206f59dacdc6..d89186af8c8e5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.lang.{Boolean => JBoolean, Double => JDouble, Float => JFloat, Long => JLong}
 import java.math.{BigDecimal => JBigDecimal}
 import java.sql.{Date, Timestamp}
+import java.time.LocalDate
 import java.util.Locale
 
 import scala.collection.JavaConverters.asScalaBufferConverter
@@ -123,8 +124,9 @@ class ParquetFilters(
   private val ParquetTimestampMicrosType = ParquetSchemaType(TIMESTAMP_MICROS, INT64, 0, null)
   private val ParquetTimestampMillisType = ParquetSchemaType(TIMESTAMP_MILLIS, INT64, 0, null)
 
-  private def dateToDays(date: Date): SQLDate = {
-    DateTimeUtils.fromJavaDate(date)
+  private def dateToDays(date: Any): SQLDate = date match {
+    case d: Date => DateTimeUtils.fromJavaDate(d)
+    case ld: LocalDate => DateTimeUtils.localDateToDays(ld)
   }
 
   private def decimalToInt32(decimal: JBigDecimal): Integer = decimal.unscaledValue().intValue()
@@ -173,7 +175,7 @@ class ParquetFilters(
     case ParquetDateType if pushDownDate =>
       (n: Array[String], v: Any) => FilterApi.eq(
         intColumn(n),
-        Option(v).map(date => dateToDays(date.asInstanceOf[Date]).asInstanceOf[Integer]).orNull)
+        Option(v).map(date => dateToDays(date).asInstanceOf[Integer]).orNull)
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
@@ -224,7 +226,7 @@ class ParquetFilters(
     case ParquetDateType if pushDownDate =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         intColumn(n),
-        Option(v).map(date => dateToDays(date.asInstanceOf[Date]).asInstanceOf[Integer]).orNull)
+        Option(v).map(date => dateToDays(date).asInstanceOf[Integer]).orNull)
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
@@ -269,7 +271,7 @@ class ParquetFilters(
         FilterApi.lt(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
       (n: Array[String], v: Any) =>
-        FilterApi.lt(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+        FilterApi.lt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.lt(
         longColumn(n),
@@ -310,7 +312,7 @@ class ParquetFilters(
         FilterApi.ltEq(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
       (n: Array[String], v: Any) =>
-        FilterApi.ltEq(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+        FilterApi.ltEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.ltEq(
         longColumn(n),
@@ -351,7 +353,7 @@ class ParquetFilters(
         FilterApi.gt(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
       (n: Array[String], v: Any) =>
-        FilterApi.gt(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+        FilterApi.gt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.gt(
         longColumn(n),
@@ -392,7 +394,7 @@ class ParquetFilters(
         FilterApi.gtEq(binaryColumn(n), Binary.fromReusedByteArray(v.asInstanceOf[Array[Byte]]))
     case ParquetDateType if pushDownDate =>
       (n: Array[String], v: Any) =>
-        FilterApi.gtEq(intColumn(n), dateToDays(v.asInstanceOf[Date]).asInstanceOf[Integer])
+        FilterApi.gtEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.gtEq(
         longColumn(n),
@@ -471,7 +473,8 @@ class ParquetFilters(
       case ParquetDoubleType => value.isInstanceOf[JDouble]
       case ParquetStringType => value.isInstanceOf[String]
       case ParquetBinaryType => value.isInstanceOf[Array[Byte]]
-      case ParquetDateType => value.isInstanceOf[Date]
+      case ParquetDateType =>
+        value.isInstanceOf[Date] || value.isInstanceOf[LocalDate]
       case ParquetTimestampMicrosType | ParquetTimestampMillisType =>
         value.isInstanceOf[Timestamp]
       case ParquetSchemaType(DECIMAL, INT32, _, decimalMeta) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index d1161e33b0941..20bfb32e2c0d4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.math.{BigDecimal => JBigDecimal}
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
+import java.time.LocalDate
 
 import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators}
 import org.apache.parquet.filter2.predicate.FilterApi._
@@ -525,52 +526,62 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
       def date: Date = Date.valueOf(s)
     }
 
-    val data = Seq("2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21").map(_.date)
+    val data = Seq("2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21")
     import testImplicits._
-    withNestedDataFrame(data.map(i => Tuple1(i)).toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val dateAttr: Expression = df(colName).expr
-        assert(df(colName).expr.dataType === DateType)
 
-        checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
-          data.map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
-          Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i.date))))
-
-        checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
-          resultFun("2018-03-21".date))
-        checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
-          resultFun("2018-03-21".date))
-
-        checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
-          resultFun("2018-03-21".date))
-        checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
-          resultFun("2018-03-18".date))
-        checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
-          resultFun("2018-03-21".date))
-
-        checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
-          resultFun("2018-03-21".date))
-        checkFilterPredicate(
-          dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
-          classOf[Operators.Or],
-          Seq(Row(resultFun("2018-03-18".date)), Row(resultFun("2018-03-21".date))))
+    Seq(false, true).foreach { java8Api =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+        val df = data.map(i => Tuple1(Date.valueOf(i))).toDF()
+        withNestedDataFrame(df) { case (inputDF, colName, fun) =>
+          def resultFun(dateStr: String): Any = {
+            val parsed = if (java8Api) LocalDate.parse(dateStr) else Date.valueOf(dateStr)
+            fun(parsed)
+          }
+          withParquetDataFrame(inputDF) { implicit df =>
+            val dateAttr: Expression = df(colName).expr
+            assert(df(colName).expr.dataType === DateType)
+
+            checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+            checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
+              data.map(i => Row.apply(resultFun(i))))
+
+            checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
+              Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i))))
+
+            checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
+              resultFun("2018-03-21"))
+            checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
+              resultFun("2018-03-21"))
+
+            checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
+              resultFun("2018-03-21"))
+            checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
+              resultFun("2018-03-18"))
+            checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
+              resultFun("2018-03-21"))
+
+            checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
+              resultFun("2018-03-21"))
+            checkFilterPredicate(
+              dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
+              classOf[Operators.Or],
+              Seq(Row(resultFun("2018-03-18")), Row(resultFun("2018-03-21"))))
+          }
+        }
       }
     }
   }

From 5065a50e16076b8c875d1d24bed5742f828adb1c Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Fri, 24 Apr 2020 12:34:44 +0900
Subject: [PATCH 0551/1280] [SPARK-30804][SS] Measure and log elapsed time for
 "compact" operation in CompactibleFileStreamLog

### What changes were proposed in this pull request?

This patch adds some log messages to log elapsed time for "compact" operation in FileStreamSourceLog and FileStreamSinkLog (added in CompactibleFileStreamLog) to help investigating the mysterious latency spike during the batch run.

### Why are the changes needed?

Tracking latency is a critical aspect of streaming query. While "compact" operation may bring nontrivial latency (it's even synchronous, adding all the latency to the batch run), it's not measured and end users have to guess.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

N/A for UT. Manual test with streaming query using file source & file sink.

> grep "for compact batch" <driver log>

```
...
20/02/20 19:27:36 WARN FileStreamSinkLog: Compacting took 24473 ms (load: 14185 ms, write: 10288 ms) for compact batch 21359
20/02/20 19:27:39 WARN FileStreamSinkLog: Loaded 1068000 entries (397985432 bytes in memory), and wrote 1068000 entries for compact batch 21359
20/02/20 19:29:52 WARN FileStreamSourceLog: Compacting took 3777 ms (load: 1524 ms, write: 2253 ms) for compact batch 21369
20/02/20 19:29:52 WARN FileStreamSourceLog: Loaded 229477 entries (68970112 bytes in memory), and wrote 229477 entries for compact batch 21369
20/02/20 19:30:17 WARN FileStreamSinkLog: Compacting took 24183 ms (load: 12992 ms, write: 11191 ms) for compact batch 21369
20/02/20 19:30:20 WARN FileStreamSinkLog: Loaded 1068500 entries (398171880 bytes in memory), and wrote 1068500 entries for compact batch 21369
...
```

![Screen Shot 2020-02-21 at 12 34 22 PM](https://user-images.githubusercontent.com/1317309/75002142-c6830100-54a6-11ea-8da6-17afb056653b.png)

This messages are explaining why the operation duration peaks per every 10 batches which is compact interval. Latency from addBatch heavily increases in each peak which DOES NOT mean it takes more time to write outputs, but we have no idea if such message is not presented.

NOTE: The output may be a bit different from the code, as it may be changed a bit during review phase.

Closes #27557 from HeartSaVioR/SPARK-30804.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 39bc50dbf8ca835389f63b47baca129e088c5a19)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../streaming/CompactibleFileStreamLog.scala  | 39 ++++++++++++++-----
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
index 905bce4d614e6..10bcfe6649802 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/CompactibleFileStreamLog.scala
@@ -28,6 +28,7 @@ import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
 import org.apache.spark.sql.SparkSession
+import org.apache.spark.util.{SizeEstimator, Utils}
 
 /**
  * An abstract class for compactible metadata logs. It will write one log file for each batch.
@@ -177,16 +178,35 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
    * corresponding `batchId` file. It will delete expired files as well if enabled.
    */
   private def compact(batchId: Long, logs: Array[T]): Boolean = {
-    val validBatches = getValidBatchesBeforeCompactionBatch(batchId, compactInterval)
-    val allLogs = validBatches.flatMap { id =>
-      super.get(id).getOrElse {
-        throw new IllegalStateException(
-          s"${batchIdToPath(id)} doesn't exist when compacting batch $batchId " +
-            s"(compactInterval: $compactInterval)")
-      }
-    } ++ logs
+    val (allLogs, loadElapsedMs) = Utils.timeTakenMs {
+      val validBatches = getValidBatchesBeforeCompactionBatch(batchId, compactInterval)
+      validBatches.flatMap { id =>
+        super.get(id).getOrElse {
+          throw new IllegalStateException(
+            s"${batchIdToPath(id)} doesn't exist when compacting batch $batchId " +
+              s"(compactInterval: $compactInterval)")
+        }
+      } ++ logs
+    }
+    val compactedLogs = compactLogs(allLogs)
+
     // Return false as there is another writer.
-    super.add(batchId, compactLogs(allLogs).toArray)
+    val (writeSucceed, writeElapsedMs) = Utils.timeTakenMs {
+      super.add(batchId, compactedLogs.toArray)
+    }
+
+    val elapsedMs = loadElapsedMs + writeElapsedMs
+    if (elapsedMs >= COMPACT_LATENCY_WARN_THRESHOLD_MS) {
+      logWarning(s"Compacting took $elapsedMs ms (load: $loadElapsedMs ms," +
+        s" write: $writeElapsedMs ms) for compact batch $batchId")
+      logWarning(s"Loaded ${allLogs.size} entries (estimated ${SizeEstimator.estimate(allLogs)} " +
+        s"bytes in memory), and wrote ${compactedLogs.size} entries for compact batch $batchId")
+    } else {
+      logDebug(s"Compacting took $elapsedMs ms (load: $loadElapsedMs ms," +
+        s" write: $writeElapsedMs ms) for compact batch $batchId")
+    }
+
+    writeSucceed
   }
 
   /**
@@ -268,6 +288,7 @@ abstract class CompactibleFileStreamLog[T <: AnyRef : ClassTag](
 
 object CompactibleFileStreamLog {
   val COMPACT_FILE_SUFFIX = ".compact"
+  val COMPACT_LATENCY_WARN_THRESHOLD_MS = 2000
 
   def getBatchIdFromFileName(fileName: String): Long = {
     fileName.stripSuffix(COMPACT_FILE_SUFFIX).toLong

From a0a673cd03fe47e9b22e67c8f69d73ee5dfc32f0 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 22 Apr 2020 00:38:57 +0900
Subject: [PATCH 0552/1280] [SPARK-31510][R][BUILD] Set setwd in R
 documentation build
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Seems like in certain environment, it requires to set `setwd` as below:

```
> library(devtools); devtools::document(pkg="./pkg", roclets=c("rd"))
Loading required package: usethis
Error: Could not find package root, is your working directory inside a package?
```

see also https://stackoverflow.com/questions/52670051/how-to-troubleshoot-error-could-not-find-package-root and https://groups.google.com/forum/#!topic/rdevtools/79jjjdc_wjg

We can make up another story too. For example, if you set a specific directory in your `~/.Rprofile`, then R documentation build will fail as below:

```
echo 'setwd("~")' > ~/.Rprofile
sh R/create-rd.sh
```

```
Using R_SCRIPT_PATH = /usr/local/bin
Loading required package: usethis
Error: Can't find './pkg'.
Execution halted
```

This PR proposes to set the `setwd` explicitly so it does not get affected on the global environment.

To make R dev env more independent.

No, dev only.

Manually tested:

```bash
echo 'setwd("~")' > ~/.Rprofile
sh R/create-rd.sh
```

Before:

```
Using R_SCRIPT_PATH = /usr/local/bin
Loading required package: usethis
Error: Can't find './pkg'.
Execution halted
```

After:

```
Using R_SCRIPT_PATH = /usr/local/bin
Loading required package: usethis
Updating SparkR documentation
Loading SparkR
Creating a new generic function for ‘as.data.frame’ in package ‘SparkR’
Creating a new generic function for ‘colnames’ in package ‘SparkR’
Creating a new generic function for ‘colnames<-’ in package ‘SparkR’
Creating a new generic function for ‘cov’ in package ‘SparkR’
Creating a new generic function for ‘drop’ in package ‘SparkR’
Creating a new generic function for ‘na.omit’ in package ‘SparkR’
Creating a new generic function for ‘filter’ in package ‘SparkR’
Creating a new generic function for ‘intersect’ in package ‘SparkR’
...
```

Closes #28285
---
 R/create-rd.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/R/create-rd.sh b/R/create-rd.sh
index ff622a41a46c0..aaad3b1aafa0a 100755
--- a/R/create-rd.sh
+++ b/R/create-rd.sh
@@ -34,4 +34,4 @@ pushd "$FWDIR" > /dev/null
 . "$FWDIR/find-r.sh"
 
 # Generate Rd files if devtools is installed
-"$R_SCRIPT_PATH/Rscript" -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); devtools::document(pkg="./pkg", roclets=c("rd")) }'
+"$R_SCRIPT_PATH/Rscript" -e ' if("devtools" %in% rownames(installed.packages())) { library(devtools); setwd("'$FWDIR'"); devtools::document(pkg="./pkg", roclets=c("rd")) }'

From cec09b05f822da54bf78a6682fe9c0fba7896ef0 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 24 Apr 2020 08:05:27 +0000
Subject: [PATCH 0553/1280] [SPARK-31502][SQL][DOCS] Document identifier in SQL
 Reference

### What changes were proposed in this pull request?
Document identifier in SQL Reference

### Why are the changes needed?
make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes
<img width="1049" alt="Screen Shot 2020-04-23 at 11 14 10 PM" src="https://user-images.githubusercontent.com/13592258/80180695-2f2a4f00-85b8-11ea-819b-f96872956d05.png">

<img width="1050" alt="Screen Shot 2020-04-23 at 11 32 32 PM" src="https://user-images.githubusercontent.com/13592258/80182062-e6c06080-85ba-11ea-9502-1c38358c97c9.png">

### How was this patch tested?
Manually build and check

Closes #28277 from huaxingao/identifier.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b14b980ab86a5e68e0affa62a875e2db6812ed2e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/_data/menu-sql.yaml   |  2 +
 docs/sql-ref-identifier.md | 80 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 docs/sql-ref-identifier.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 5d8c265bf84fe..26cca61a9d6ac 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -78,6 +78,8 @@
   subitems:
     - text: Data Types
       url: sql-ref-datatypes.html
+    - text: Identifiers
+      url: sql-ref-identifier.html
     - text: Literals
       url: sql-ref-literals.html
     - text: Null Semantics
diff --git a/docs/sql-ref-identifier.md b/docs/sql-ref-identifier.md
new file mode 100644
index 0000000000000..89cde21e6fdb6
--- /dev/null
+++ b/docs/sql-ref-identifier.md
@@ -0,0 +1,80 @@
+---
+layout: global
+title: Identifiers
+displayTitle: Identifiers
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+An identifier is a string used to identify a database object such as a table, view, schema, column, etc. Spark SQL has regular identifiers and delimited identifiers, which are enclosed within backticks. Both regular identifiers and delimited identifiers are case-insensitive.
+
+### Syntax
+
+#### Regular Identifier
+
+{% highlight sql %}
+{ letter | digit | '_' } [ , ... ]
+{% endhighlight %}
+Note: If `spark.sql.ansi.enabled` is set to true, ANSI SQL reserved keywords cannot be used as identifiers. For more details, please refer to [ANSI Compliance](sql-ref-ansi-compliance.html).
+
+#### Delimited Identifier
+
+{% highlight sql %}
+`c [ ... ]`
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>letter</em></code></dt>
+  <dd>
+    Any letter from A-Z or a-z.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>digit</em></code></dt>
+  <dd>
+    Any numeral from 0 to 9.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>c</em></code></dt>
+  <dd>
+    Any character from the character set. Use <code>`</code> to escape special characters (e.g., <code>`</code>).
+  </dd>
+</dl>
+
+### Examples
+
+{% highlight sql %}
+-- This CREATE TABLE fails with ParseException because of the illegal identifier name a.b
+CREATE TABLE test (a.b int);
+org.apache.spark.sql.catalyst.parser.ParseException:
+no viable alternative at input 'CREATE TABLE test (a.'(line 1, pos 20)
+
+-- This CREATE TABLE works
+CREATE TABLE test (`a.b` int);
+
+-- This CREATE TABLE fails with ParseException because special character ` is not escaped
+CREATE TABLE test1 (`a`b` int);
+org.apache.spark.sql.catalyst.parser.ParseException:
+no viable alternative at input 'CREATE TABLE test (`a`b`'(line 1, pos 23)
+
+-- This CREATE TABLE works
+CREATE TABLE test (`a``b` int);
+{% endhighlight %}

From da6f398db2838aca6e0dc18866715a14b9b2aded Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 24 Apr 2020 19:10:18 +0900
Subject: [PATCH 0554/1280] [SPARK-31010][SQL][DOC][FOLLOW-UP] Improve
 deprecated warning message for untyped scala udf

### What changes were proposed in this pull request?

Give more friendly warning message/migration guide of deprecated scala udf to users.

### Why are the changes needed?

User can not distinguish function signature between typed and untyped scala udf. Instead, we shall tell user what to do directly.

### Does this PR introduce any user-facing change?

No, it's newly added in Spark 3.0.

### How was this patch tested?

Pass Jenkins.

Closes #28311 from Ngone51/update_udf_doc.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 463c54419bf663615eb72e24b82e940feb85c68c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md                                 | 2 +-
 .../src/main/scala/org/apache/spark/sql/functions.scala     | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 854c9eae98a1a..39619f60d9aba 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -75,7 +75,7 @@ license: |
 
   - In Spark version 2.4 and below, you can create a map with duplicated keys via built-in functions like `CreateMap`, `StringToMap`, etc. The behavior of map with duplicated keys is undefined, for example, map look up respects the duplicated key appears first, `Dataset.collect` only keeps the duplicated key appears last, `MapKeys` returns duplicated keys, etc. In Spark 3.0, Spark throws `RuntimeException` when duplicated keys are found. You can set `spark.sql.mapKeyDedupPolicy` to `LAST_WIN` to deduplicate map keys with last wins policy. Users may still read map values with duplicated keys from data sources which do not enforce it (for example, Parquet), the behavior is undefined.
 
-  - In Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. In Spark version 2.4 and below, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF returns null if the input values is null. However, in Spark 3.0, the UDF returns the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` returns null in Spark 2.4 and below if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
+  - In Spark 3.0, using `org.apache.spark.sql.functions.udf(AnyRef, DataType)` is not allowed by default. Remove the return type parameter to automatically switch to typed Scala udf is recommended, or set `spark.sql.legacy.allowUntypedScalaUDF` to true to keep using it. In Spark version 2.4 and below, if `org.apache.spark.sql.functions.udf(AnyRef, DataType)` gets a Scala closure with primitive-type argument, the returned UDF returns null if the input values is null. However, in Spark 3.0, the UDF returns the default value of the Java type if the input value is null. For example, `val f = udf((x: Int) => x, IntegerType)`, `f($"x")` returns null in Spark 2.4 and below if column `x` is null, and return 0 in Spark 3.0. This behavior change is introduced because Spark 3.0 is built with Scala 2.12 by default.
 
   - In Spark 3.0, a higher-order function `exists` follows the three-valued boolean logic, that is, if the `predicate` returns any `null`s and no `true` is obtained, then `exists` returns `null` instead of `false`. For example, `exists(array(1, null, 3), x -> x % 2 == 0)` is `null`. The previous behaviorcan be restored by setting `spark.sql.legacy.followThreeValuedLogicInArrayExists` to `false`.
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 782be98c3fa2c..9fd47186cc005 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -4833,8 +4833,8 @@ object functions {
    * @group udf_funcs
    * @since 2.0.0
    */
-  @deprecated("Untyped Scala UDF API is deprecated, please use typed Scala UDF API such as " +
-    "'def udf[RT: TypeTag](f: Function0[RT]): UserDefinedFunction' instead.", "3.0.0")
+  @deprecated("Scala `udf` method with return type parameter is deprecated. " +
+    "Please use Scala `udf` method without return type parameter.", "3.0.0")
   def udf(f: AnyRef, dataType: DataType): UserDefinedFunction = {
     if (!SQLConf.get.getConf(SQLConf.LEGACY_ALLOW_UNTYPED_SCALA_UDF)) {
       val errorMsg = "You're using untyped Scala UDF, which does not have the input type " +
@@ -4842,7 +4842,7 @@ object functions {
         "argument, and the closure will see the default value of the Java type for the null " +
         "argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. " +
         "To get rid of this error, you could:\n" +
-        "1. use typed Scala UDF APIs, e.g. `udf((x: Int) => x)`\n" +
+        "1. use typed Scala UDF APIs(without return type parameter), e.g. `udf((x: Int) => x)`\n" +
         "2. use Java UDF APIs, e.g. `udf(new UDF1[String, Integer] { " +
         "override def call(s: String): Integer = s.length() }, IntegerType)`, " +
         "if input types are all non primitive\n" +

From f499f456d7318dc287b628b1d47916c9e91bedf4 Mon Sep 17 00:00:00 2001
From: Jian Tang <jian_tang@apple.com>
Date: Fri, 24 Apr 2020 22:10:58 +0000
Subject: [PATCH 0555/1280] [SPARK-31364][SQL][TESTS] Benchmark Parquet Nested
 Field Predicate Pushdown

### What changes were proposed in this pull request?

This PR aims to add a benchmark suite for nested predicate pushdown with parquet file:

Performance comparison: Nested predicate pushdown disabled vs enabled,  with the following queries scenarios:

1.  When predicate pushed down, parquet reader are able to filter out all the row groups without loading them.

2. When predicate pushed down, parquet reader only loads one of the row groups.

3. When predicate pushed down, parquet reader can't filter out any row group in order to see if we introduce too much overhead or not when enabling nested predicate push down.

### Why are the changes needed?

No benchmark exists today for nested fields predicate pushdown performance evaluation.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
 Benchmark runs and reporting result.

Closes #28319 from JiJiTang/SPARK-31364.

Authored-by: Jian Tang <jian_tang@apple.com>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit 6a576161ae3a113c2e329595b9a6f56d4210e0cf)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 ...edicatePushDownBenchmark-jdk11-results.txt |  21 ++++
 ...stedPredicatePushDownBenchmark-results.txt |  21 ++++
 ...quetNestedPredicatePushDownBenchmark.scala | 113 ++++++++++++++++++
 3 files changed, 155 insertions(+)
 create mode 100644 sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
 create mode 100644 sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala

diff --git a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
new file mode 100644
index 0000000000000..c33f8a372d82b
--- /dev/null
+++ b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-jdk11-results.txt
@@ -0,0 +1,21 @@
+OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip all row groups:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 34214          35752         NaN          3.1         326.3       1.0X
+With nested predicate Pushdown                       86            102          11       1216.2           0.8     396.8X
+
+OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip some row groups:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 34211          35162         843          3.1         326.3       1.0X
+With nested predicate Pushdown                     3470           3514          36         30.2          33.1       9.9X
+
+OpenJDK 64-Bit Server VM 11.0.2+9 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip no row groups:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 37533          37919         329          2.8         357.9       1.0X
+With nested predicate Pushdown                    37876          39132         536          2.8         361.2       1.0X
+
diff --git a/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
new file mode 100644
index 0000000000000..35dd4f0a5e9cb
--- /dev/null
+++ b/sql/core/benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt
@@ -0,0 +1,21 @@
+OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip all row groups:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 30687          31552         NaN          3.4         292.7       1.0X
+With nested predicate Pushdown                      105            150          61        999.3           1.0     292.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip some row groups:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 30505          31828         NaN          3.4         290.9       1.0X
+With nested predicate Pushdown                     3156           3215          77         33.2          30.1       9.7X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-b09 on Mac OS X 10.14.6
+Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+Can skip no row groups:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+Without nested predicate Pushdown                 34475          35302         NaN          3.0         328.8       1.0X
+With nested predicate Pushdown                    34003          34596         567          3.1         324.3       1.0X
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
new file mode 100644
index 0000000000000..11bc91a4b1551
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.benchmark
+
+import org.apache.spark.SparkConf
+import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Synthetic benchmark for nested fields predicate push down performance for Parquet datasource.
+ * To run this benchmark:
+ * {{{
+ *   1. without sbt:
+ *      bin/spark-submit --class <this class> --jars <spark core test jar> <sql core test jar>
+ *   2. build/sbt "sql/test:runMain <this class>"
+ *   3. generate result:
+ *      SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain <this class>"
+ *      Results will be written to "benchmarks/ParquetNestedPredicatePushDownBenchmark-results.txt".
+ * }}}
+ */
+object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
+
+  private val N = 100 * 1024 * 1024
+  private val NUMBER_OF_ITER = 10
+
+  private val df: DataFrame = spark
+    .range(1, N, 1, 4)
+    .toDF("id")
+    .selectExpr("id", "STRUCT(id x, STRUCT(CAST(id AS STRING) z) y) nested")
+    .sort("id")
+
+  private def addCase(
+      benchmark: Benchmark,
+      inputPath: String,
+      enableNestedPD: Boolean,
+      name: String,
+      withFilter: DataFrame => DataFrame): Unit = {
+    val loadDF = spark.read.parquet(inputPath)
+    benchmark.addCase(name) { _ =>
+      withSQLConf((SQLConf.NESTED_PREDICATE_PUSHDOWN_ENABLED.key, enableNestedPD.toString)) {
+        withFilter(loadDF).noop()
+      }
+    }
+  }
+
+  private def createAndRunBenchmark(name: String, withFilter: DataFrame => DataFrame): Unit = {
+    withTempPath { tempDir =>
+      val outputPath = tempDir.getCanonicalPath
+      df.write.mode(SaveMode.Overwrite).parquet(outputPath)
+      val benchmark = new Benchmark(name, N, NUMBER_OF_ITER, output = output)
+      addCase(
+        benchmark,
+        outputPath,
+        enableNestedPD = false,
+        "Without nested predicate Pushdown",
+        withFilter)
+      addCase(
+        benchmark,
+        outputPath,
+        enableNestedPD = true,
+        "With nested predicate Pushdown",
+        withFilter)
+      benchmark.run()
+    }
+  }
+
+  /**
+   * Benchmark for sorted data with a filter which allows to filter out all the row groups
+   * when nested fields predicate push down enabled
+   */
+  def runLoadNoRowGroupWhenPredicatePushedDown(): Unit = {
+    createAndRunBenchmark("Can skip all row groups", _.filter("nested.x < 0"))
+  }
+
+  /**
+   * Benchmark with a filter which allows to load only some row groups
+   * when nested fields predicate push down enabled
+   */
+  def runLoadSomeRowGroupWhenPredicatePushedDown(): Unit = {
+    createAndRunBenchmark("Can skip some row groups", _.filter("nested.x = 100"))
+  }
+
+  /**
+   * Benchmark with a filter which still requires to
+   * load all the row groups on sorted data to see if we introduce too much
+   * overhead or not if enable nested predicate push down.
+   */
+  def runLoadAllRowGroupsWhenPredicatePushedDown(): Unit = {
+    createAndRunBenchmark("Can skip no row groups", _.filter(s"nested.x >= 0 and nested.x <= $N"))
+  }
+
+  override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+    runLoadNoRowGroupWhenPredicatePushedDown()
+    runLoadSomeRowGroupWhenPredicatePushedDown()
+    runLoadAllRowGroupsWhenPredicatePushedDown()
+  }
+}

From 0dbd69c61492f537bc0326d6ad86b616577f46df Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sat, 25 Apr 2020 08:53:00 +0900
Subject: [PATCH 0556/1280] [SPARK-31532][SQL] Builder should not propagate
 static sql configs to the existing active or default SparkSession

### What changes were proposed in this pull request?

SparkSessionBuilder shoud not propagate static sql configurations to the existing active/default SparkSession
This seems a long-standing bug.

```scala
scala> spark.sql("set spark.sql.warehouse.dir").show
+--------------------+--------------------+
|                 key|               value|
+--------------------+--------------------+
|spark.sql.warehou...|file:/Users/kenty...|
+--------------------+--------------------+

scala> spark.sql("set spark.sql.warehouse.dir=2");
org.apache.spark.sql.AnalysisException: Cannot modify the value of a static config: spark.sql.warehouse.dir;
  at org.apache.spark.sql.RuntimeConfig.requireNonStaticConf(RuntimeConfig.scala:154)
  at org.apache.spark.sql.RuntimeConfig.set(RuntimeConfig.scala:42)
  at org.apache.spark.sql.execution.command.SetCommand.$anonfun$x$7$6(SetCommand.scala:100)
  at org.apache.spark.sql.execution.command.SetCommand.run(SetCommand.scala:156)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
  at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229)
  at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3644)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3642)
  at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229)
  at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
  at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
  at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602)
  ... 47 elided

scala> import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SparkSession

scala> SparkSession.builder.config("spark.sql.warehouse.dir", "xyz").get
getClass   getOrCreate

scala> SparkSession.builder.config("spark.sql.warehouse.dir", "xyz").getOrCreate
20/04/23 23:49:13 WARN SparkSession$Builder: Using an existing SparkSession; some configuration may not take effect.
res7: org.apache.spark.sql.SparkSession = org.apache.spark.sql.SparkSession6403d574

scala> spark.sql("set spark.sql.warehouse.dir").show
+--------------------+-----+
|                 key|value|
+--------------------+-----+
|spark.sql.warehou...|  xyz|
+--------------------+-----+

scala>
OptionsAttachments
```

### Why are the changes needed?
bugfix as shown in the previous section

### Does this PR introduce any user-facing change?

Yes, static SQL configurations with SparkSession.builder.config do not propagate to any existing or new SparkSession instances.

### How was this patch tested?

new ut.

Closes #28316 from yaooqinn/SPARK-31532.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 8424f552293677717da7411ed43e68e73aa7f0d6)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../org/apache/spark/sql/SparkSession.scala   | 28 +++++++----
 .../spark/sql/SparkSessionBuilderSuite.scala  | 49 ++++++++++++++++++-
 2 files changed, 67 insertions(+), 10 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 731aae882a7b8..be597edecba98 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -895,7 +895,7 @@ object SparkSession extends Logging {
      * SparkSession exists, the method creates a new SparkSession and assigns the
      * newly created SparkSession as the global default.
      *
-     * In case an existing SparkSession is returned, the config options specified in
+     * In case an existing SparkSession is returned, the non-static config options specified in
      * this builder will be applied to the existing SparkSession.
      *
      * @since 2.0.0
@@ -905,10 +905,7 @@ object SparkSession extends Logging {
       // Get the session from current thread's active session.
       var session = activeThreadSession.get()
       if ((session ne null) && !session.sparkContext.isStopped) {
-        options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
-        if (options.nonEmpty) {
-          logWarning("Using an existing SparkSession; some configuration may not take effect.")
-        }
+        applyModifiableSettings(session)
         return session
       }
 
@@ -917,10 +914,7 @@ object SparkSession extends Logging {
         // If the current thread does not have an active session, get it from the global session.
         session = defaultSession.get()
         if ((session ne null) && !session.sparkContext.isStopped) {
-          options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
-          if (options.nonEmpty) {
-            logWarning("Using an existing SparkSession; some configuration may not take effect.")
-          }
+          applyModifiableSettings(session)
           return session
         }
 
@@ -959,6 +953,22 @@ object SparkSession extends Logging {
 
       return session
     }
+
+    private def applyModifiableSettings(session: SparkSession): Unit = {
+      val (staticConfs, otherConfs) =
+        options.partition(kv => SQLConf.staticConfKeys.contains(kv._1))
+
+      otherConfs.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
+
+      if (staticConfs.nonEmpty) {
+        logWarning("Using an existing SparkSession; the static sql configurations will not take" +
+          " effect.")
+      }
+      if (otherConfs.nonEmpty) {
+        logWarning("Using an existing SparkSession; some spark core configurations may not take" +
+          " effect.")
+      }
+    }
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index f2386413bea18..7b76d0702d835 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -22,7 +22,7 @@ import org.scalatest.BeforeAndAfterEach
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
+import org.apache.spark.sql.internal.StaticSQLConf._
 
 /**
  * Test cases for the builder pattern of [[SparkSession]].
@@ -168,4 +168,51 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     assert(session.sessionState.conf.getConfString("spark.app.name") === "test-app-SPARK-31234")
     assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
   }
+
+  test("SPARK-31532: should not propagate static sql configs to the existing" +
+    " active/default SparkSession") {
+    val session = SparkSession.builder()
+      .master("local")
+      .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31532")
+      .config("spark.app.name", "test-app-SPARK-31532")
+      .getOrCreate()
+    // do not propagate static sql configs to the existing active session
+    val session1 = SparkSession
+      .builder()
+      .config(GLOBAL_TEMP_DATABASE.key, "globalTempDB-SPARK-31532-1")
+      .getOrCreate()
+    assert(session.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532")
+    assert(session1.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532")
+
+    // do not propagate static sql configs to the existing default session
+    SparkSession.clearActiveSession()
+    val session2 = SparkSession
+      .builder()
+      .config(WAREHOUSE_PATH.key, "SPARK-31532-db")
+      .config(GLOBAL_TEMP_DATABASE.key, value = "globalTempDB-SPARK-31532-2")
+      .getOrCreate()
+
+    assert(!session.conf.get(WAREHOUSE_PATH).contains("SPARK-31532-db"))
+    assert(session.conf.get(WAREHOUSE_PATH) === session2.conf.get(WAREHOUSE_PATH))
+    assert(session2.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532")
+  }
+
+  test("SPARK-31532: propagate static sql configs if no existing SparkSession") {
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("test-app-SPARK-31532-2")
+      .set(GLOBAL_TEMP_DATABASE.key, "globaltempdb-spark-31532")
+      .set(WAREHOUSE_PATH.key, "SPARK-31532-db")
+    SparkContext.getOrCreate(conf)
+
+    // propagate static sql configs if no existing session
+    val session = SparkSession
+      .builder()
+      .config(GLOBAL_TEMP_DATABASE.key, "globalTempDB-SPARK-31532-2")
+      .config(WAREHOUSE_PATH.key, "SPARK-31532-db-2")
+      .getOrCreate()
+    assert(session.conf.get("spark.app.name") === "test-app-SPARK-31532-2")
+    assert(session.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532-2")
+    assert(session.conf.get(WAREHOUSE_PATH) === "SPARK-31532-db-2")
+  }
 }

From 37002fe69a58ba071ada798842d7e77c4cd6e47e Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 25 Apr 2020 09:02:16 +0900
Subject: [PATCH 0557/1280] [SPARK-31491][SQL][DOCS] Re-arrange Data Types page
 to document Floating Point Special Values

### What changes were proposed in this pull request?
Re-arrange Data Types page to document Floating Point Special Values

### Why are the changes needed?
To complete SQL Reference

### Does this PR introduce any user-facing change?
Yes

- add Floating Point Special Values in Data Types page
- move NaN Semantics to Data Types page

<img width="1050" alt="Screen Shot 2020-04-24 at 9 14 57 AM" src="https://user-images.githubusercontent.com/13592258/80233996-3da25600-860c-11ea-8285-538efc16e431.png">

<img width="1050" alt="Screen Shot 2020-04-24 at 9 15 22 AM" src="https://user-images.githubusercontent.com/13592258/80234001-4004b000-860c-11ea-8954-72f63c92d50d.png">

<img width="1049" alt="Screen Shot 2020-04-24 at 9 15 44 AM" src="https://user-images.githubusercontent.com/13592258/80234006-41ce7380-860c-11ea-96bf-15e1aa2102ff.png">

### How was this patch tested?
Manually build and check

Closes #28264 from huaxingao/datatypes.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 054bef94ca7e84ff8e2e27af65e00e183f7be6da)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml      |   2 -
 docs/sql-ref-datatypes.md     | 119 ++++++++++++++++++++++++++++++++++
 docs/sql-ref-nan-semantics.md |  29 ---------
 3 files changed, 119 insertions(+), 31 deletions(-)
 delete mode 100644 docs/sql-ref-nan-semantics.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 26cca61a9d6ac..1097079e4597d 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -84,8 +84,6 @@
       url: sql-ref-literals.html
     - text: Null Semantics
       url: sql-ref-null-semantics.html
-    - text: NaN Semantics
-      url: sql-ref-nan-semantics.html
     - text: ANSI Compliance
       url: sql-ref-ansi-compliance.html
       subitems:
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 150e194ae6cde..0d49f6f882228 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -19,6 +19,8 @@ license: |
   limitations under the License.
 ---
 
+### Supported Data Types
+
 Spark SQL and DataFrames support the following data types:
 
 * Numeric types
@@ -706,3 +708,120 @@ The following table shows the type names as well as aliases used in Spark SQL pa
 </table>
 </div>
 </div>
+
+### Floating Point Special Values
+
+Spark SQL supports several special floating point values in a case-insensitive manner:
+
+ * Inf/+Inf/Infinity/+Infinity: positive infinity
+   * ```FloatType```: equivalent to Scala <code>Float.PositiveInfinity</code>.
+   * ```DoubleType```: equivalent to Scala <code>Double.PositiveInfinity</code>.
+ * -Inf/-Infinity: negative infinity
+   * ```FloatType```: equivalent to Scala <code>Float.NegativeInfinity</code>.
+   * ```DoubleType```: equivalent to Scala <code>Double.NegativeInfinity</code>.
+ * NaN: not a number
+   * ```FloatType```: equivalent to Scala <code>Float.NaN</code>.
+   * ```DoubleType```:  equivalent to Scala <code>Double.NaN</code>.
+
+#### Positive/Negative Infinity Semantics
+
+There is special handling for positive and negative infinity. They have the following semantics:
+
+ * Positive infinity multiplied by any positive value returns positive infinity.
+ * Negative infinity multiplied by any positive value returns negative infinity.
+ * Positive infinity multiplied by any negative value returns negative infinity.
+ * Negative infinity multiplied by any negative value returns positive infinity.
+ * Positive/negative infinity multiplied by 0 returns NaN.
+ * Positive/negative infinity is equal to itself.
+ * In aggregations, all positive infinity values are grouped together. Similarly, all negative infinity values are grouped together.
+ * Positive infinity and negative infinity are treated as normal values in join keys.
+ * Positive infinity sorts lower than NaN and higher than any other values.
+ * Negative infinity sorts lower than any other values.
+
+#### NaN Semantics
+
+There is special handling for not-a-number (NaN) when dealing with `float` or `double` types that
+do not exactly match standard floating point semantics.
+Specifically:
+
+ * NaN = NaN returns true.
+ * In aggregations, all NaN values are grouped together.
+ * NaN is treated as a normal value in join keys.
+ * NaN values go last when in ascending order, larger than any other numeric value.
+
+#### Examples
+
+{% highlight sql %}
+SELECT double('infinity') AS col;
++--------+
+|     col|
++--------+
+|Infinity|
++--------+
+
+SELECT float('-inf') AS col;
++---------+
+|      col|
++---------+
+|-Infinity|
++---------+
+
+SELECT float('NaN') AS col;
++---+
+|col|
++---+
+|NaN|
++---+
+
+SELECT double('infinity') * 0 AS col;
++---+
+|col|
++---+
+|NaN|
++---+
+
+SELECT double('-infinity') * (-1234567) AS col;
++--------+
+|     col|
++--------+
+|Infinity|
++--------+
+
+SELECT double('infinity') < double('NaN') AS col;
++----+
+| col|
++----+
+|true|
++----+
+
+SELECT double('NaN') = double('NaN') AS col;
++----+
+| col|
++----+
+|true|
++----+
+
+SELECT double('inf') = double('infinity') AS col;
++----+
+| col|
++----+
+|true|
++----+
+
+CREATE TABLE test (c1 int, c2 double);
+INSERT INTO test VALUES (1, double('infinity'));
+INSERT INTO test VALUES (2, double('infinity'));
+INSERT INTO test VALUES (3, double('inf'));
+INSERT INTO test VALUES (4, double('-inf'));
+INSERT INTO test VALUES (5, double('NaN'));
+INSERT INTO test VALUES (6, double('NaN'));
+INSERT INTO test VALUES (7, double('-infinity'));
+SELECT COUNT(*), c2 FROM test GROUP BY c2;
++---------+---------+
+| count(1)|       c2|
++---------+---------+
+|        2|      NaN|
+|        2|-Infinity|
+|        3| Infinity|
++---------+---------+
+{% endhighlight %}
\ No newline at end of file
diff --git a/docs/sql-ref-nan-semantics.md b/docs/sql-ref-nan-semantics.md
deleted file mode 100644
index f6a85728263a7..0000000000000
--- a/docs/sql-ref-nan-semantics.md
+++ /dev/null
@@ -1,29 +0,0 @@
----
-layout: global
-title: Nan Semantics
-displayTitle: NaN Semantics
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-There is specially handling for not-a-number (NaN) when dealing with `float` or `double` types that
-does not exactly match standard floating point semantics.
-Specifically:
- 
- - NaN = NaN returns true.
- - In aggregations, all NaN values are grouped together.
- - NaN is treated as a normal value in join keys.
- - NaN values go last when in ascending order, larger than any other numeric value.

From a26d769c1342f70b80f04846ba933358a3f2e311 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 24 Apr 2020 18:04:26 -0700
Subject: [PATCH 0558/1280] [SPARK-31552][SQL] Fix ClassCastException in
 ScalaReflection arrayClassFor

### What changes were proposed in this pull request?

the 2 method `arrayClassFor` and `dataTypeFor` in `ScalaReflection` call each other circularly, the cases in `dataTypeFor` are not fully handled in `arrayClassFor`

For example:
```scala
scala> implicit def newArrayEncoder[T <: Array[_] : TypeTag]: Encoder[T] = ExpressionEncoder()
newArrayEncoder: [T <: Array[_]](implicit evidence$1: reflect.runtime.universe.TypeTag[T])org.apache.spark.sql.Encoder[T]

scala> val decOne = Decimal(1, 38, 18)
decOne: org.apache.spark.sql.types.Decimal = 1E-18

scala> val decTwo = Decimal(2, 38, 18)
decTwo: org.apache.spark.sql.types.Decimal = 2E-18

scala> val decSpark = Array(decOne, decTwo)
decSpark: Array[org.apache.spark.sql.types.Decimal] = Array(1E-18, 2E-18)

scala> Seq(decSpark).toDF()
java.lang.ClassCastException: org.apache.spark.sql.types.DecimalType cannot be cast to org.apache.spark.sql.types.ObjectType
  at org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$arrayClassFor$1(ScalaReflection.scala:131)
  at scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
  at org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:879)
  at org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:878)
  at org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:49)
  at org.apache.spark.sql.catalyst.ScalaReflection$.arrayClassFor(ScalaReflection.scala:120)
  at org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$dataTypeFor$1(ScalaReflection.scala:105)
  at scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
  at org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:879)
  at org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:878)
  at org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:49)
  at org.apache.spark.sql.catalyst.ScalaReflection$.dataTypeFor(ScalaReflection.scala:88)
  at org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$serializerForType$1(ScalaReflection.scala:399)
  at scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
  at org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:879)
  at org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:878)
  at org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:49)
  at org.apache.spark.sql.catalyst.ScalaReflection$.serializerForType(ScalaReflection.scala:393)
  at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:57)
  at newArrayEncoder(<console>:57)
  ... 53 elided

scala>
```

In this PR, we add the missing cases to `arrayClassFor`

### Why are the changes needed?

bugfix as described above

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

add a test for array encoders

Closes #28324 from yaooqinn/SPARK-31552.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit caf3ab84113c29c51049c1c906c91462e233e9d9)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/ScalaReflection.scala  |  3 +
 .../org/apache/spark/sql/DataFrameSuite.scala | 82 ++++++++++++++++++-
 2 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 9c8da32cbc0ae..05de21b9eafec 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -126,6 +126,9 @@ object ScalaReflection extends ScalaReflection {
       case t if isSubtype(t, definitions.ShortTpe) => classOf[Array[Short]]
       case t if isSubtype(t, definitions.ByteTpe) => classOf[Array[Byte]]
       case t if isSubtype(t, definitions.BooleanTpe) => classOf[Array[Boolean]]
+      case t if isSubtype(t, localTypeOf[Array[Byte]]) => classOf[Array[Array[Byte]]]
+      case t if isSubtype(t, localTypeOf[CalendarInterval]) => classOf[Array[CalendarInterval]]
+      case t if isSubtype(t, localTypeOf[Decimal]) => classOf[Array[Decimal]]
       case other =>
         // There is probably a better way to do this, but I couldn't find it...
         val elementType = dataTypeFor(other).asInstanceOf[ObjectType].cls
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index f797290028e6f..f20e684bf7657 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -23,6 +23,7 @@ import java.sql.{Date, Timestamp}
 import java.util.UUID
 import java.util.concurrent.atomic.AtomicLong
 
+import scala.reflect.runtime.universe.TypeTag
 import scala.util.Random
 
 import org.scalatest.Matchers._
@@ -30,10 +31,11 @@ import org.scalatest.Matchers._
 import org.apache.spark.SparkException
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.expressions.Uuid
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, OneRowRelation, Union}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, OneRowRelation}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.{FilterExec, QueryExecution, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
@@ -41,7 +43,7 @@ import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ReusedExc
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
-import org.apache.spark.sql.test.SQLTestData.{DecimalData, NullStrings, TestData2}
+import org.apache.spark.sql.test.SQLTestData.{DecimalData, TestData2}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 import org.apache.spark.util.Utils
@@ -2358,6 +2360,80 @@ class DataFrameSuite extends QueryTest
     val df = Seq((1, new CalendarInterval(1, 2, 3))).toDF("a", "b")
     checkAnswer(df.selectExpr("b"), Row(new CalendarInterval(1, 2, 3)))
   }
+
+  test("SPARK-31552: array encoder with different types") {
+    // primitives
+    val booleans = Array(true, false)
+    checkAnswer(Seq(booleans).toDF(), Row(booleans))
+
+    val bytes = Array(1.toByte, 2.toByte)
+    checkAnswer(Seq(bytes).toDF(), Row(bytes))
+    val shorts = Array(1.toShort, 2.toShort)
+    checkAnswer(Seq(shorts).toDF(), Row(shorts))
+    val ints = Array(1, 2)
+    checkAnswer(Seq(ints).toDF(), Row(ints))
+    val longs = Array(1L, 2L)
+    checkAnswer(Seq(longs).toDF(), Row(longs))
+
+    val floats = Array(1.0F, 2.0F)
+    checkAnswer(Seq(floats).toDF(), Row(floats))
+    val doubles = Array(1.0D, 2.0D)
+    checkAnswer(Seq(doubles).toDF(), Row(doubles))
+
+    val strings = Array("2020-04-24", "2020-04-25")
+    checkAnswer(Seq(strings).toDF(), Row(strings))
+
+    // tuples
+    val decOne = Decimal(1, 38, 18)
+    val decTwo = Decimal(2, 38, 18)
+    val tuple1 = (1, 2.2, "3.33", decOne, Date.valueOf("2012-11-22"))
+    val tuple2 = (2, 3.3, "4.44", decTwo, Date.valueOf("2022-11-22"))
+    checkAnswer(Seq(Array(tuple1, tuple2)).toDF(), Seq(Seq(tuple1, tuple2)).toDF())
+
+    // case classes
+    val gbks = Array(GroupByKey(1, 2), GroupByKey(4, 5))
+    checkAnswer(Seq(gbks).toDF(), Row(Array(Row(1, 2), Row(4, 5))))
+
+    // We can move this implicit def to [[SQLImplicits]] when we eventually make fully
+    // support for array encoder like Seq and Set
+    // For now cases below, decimal/datetime/interval/binary/nested types, etc,
+    // are not supported by array
+    implicit def newArrayEncoder[T <: Array[_] : TypeTag]: Encoder[T] = ExpressionEncoder()
+
+    // decimals
+    val decSpark = Array(decOne, decTwo)
+    val decScala = decSpark.map(_.toBigDecimal)
+    val decJava = decSpark.map(_.toJavaBigDecimal)
+    checkAnswer(Seq(decSpark).toDF(), Row(decJava))
+    checkAnswer(Seq(decScala).toDF(), Row(decJava))
+    checkAnswer(Seq(decJava).toDF(), Row(decJava))
+
+    // datetimes and intervals
+    val dates = strings.map(Date.valueOf)
+    checkAnswer(Seq(dates).toDF(), Row(dates))
+    val localDates = dates.map(d => DateTimeUtils.daysToLocalDate(DateTimeUtils.fromJavaDate(d)))
+    checkAnswer(Seq(localDates).toDF(), Row(dates))
+
+    val timestamps =
+      Array(Timestamp.valueOf("2020-04-24 12:34:56"), Timestamp.valueOf("2020-04-24 11:22:33"))
+    checkAnswer(Seq(timestamps).toDF(), Row(timestamps))
+    val instants =
+      timestamps.map(t => DateTimeUtils.microsToInstant(DateTimeUtils.fromJavaTimestamp(t)))
+    checkAnswer(Seq(instants).toDF(), Row(timestamps))
+
+    val intervals = Array(new CalendarInterval(1, 2, 3), new CalendarInterval(4, 5, 6))
+    checkAnswer(Seq(intervals).toDF(), Row(intervals))
+
+    // binary
+    val bins = Array(Array(1.toByte), Array(2.toByte), Array(3.toByte), Array(4.toByte))
+    checkAnswer(Seq(bins).toDF(), Row(bins))
+
+    // nested
+    val nestedIntArray = Array(Array(1), Array(2))
+    checkAnswer(Seq(nestedIntArray).toDF(), Row(nestedIntArray.map(wrapIntArray)))
+    val nestedDecArray = Array(decSpark)
+    checkAnswer(Seq(nestedDecArray).toDF(), Row(Array(wrapRefArray(decJava))))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)

From 4cc779999bb708426a319314b549da6a8a0951d6 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Fri, 24 Apr 2020 18:59:15 -0700
Subject: [PATCH 0559/1280] [SPARK-31560][SQL][TESTS] Add V1/V2 tests for
 TextSuite and WholeTextFileSuite

### What changes were proposed in this pull request?

 Add V1/V2 tests for TextSuite and WholeTextFileSuite

### Why are the changes needed?

This is missing part since #24207. We should have these tests for test coverage.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Unit tests.

Closes #28335 from gengliangwang/testV2Suite.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 16b961526dcd7062be3e0ce59f9cbcf22a6887ea)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../execution/datasources/text/TextSuite.scala | 18 ++++++++++++++++--
 .../datasources/text/WholeTextFileSuite.scala  | 17 ++++++++++++++++-
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
index 539ff0d0e905c..7e97994476694 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
@@ -24,14 +24,14 @@ import java.nio.file.Files
 import org.apache.hadoop.io.SequenceFile.CompressionType
 import org.apache.hadoop.io.compress.GzipCodec
 
-import org.apache.spark.TestUtils
+import org.apache.spark.{SparkConf, TestUtils}
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{StringType, StructType}
 import org.apache.spark.util.Utils
 
-class TextSuite extends QueryTest with SharedSparkSession {
+abstract class TextSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
 
   test("reading text file") {
@@ -234,3 +234,17 @@ class TextSuite extends QueryTest with SharedSparkSession {
     assert(data.length == 4)
   }
 }
+
+class TextV1Suite extends TextSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "text")
+}
+
+class TextV2Suite extends TextSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
index 5e3b3441aa74f..f4812844cbae3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/WholeTextFileSuite.scala
@@ -19,12 +19,13 @@ package org.apache.spark.sql.execution.datasources.text
 
 import java.io.File
 
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.{QueryTest, Row}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{StringType, StructType}
 
-class WholeTextFileSuite extends QueryTest with SharedSparkSession {
+abstract class WholeTextFileSuite extends QueryTest with SharedSparkSession {
 
   // Hadoop's FileSystem caching does not use the Configuration as part of its cache key, which
   // can cause Filesystem.get(Configuration) to return a cached instance created with a different
@@ -103,3 +104,17 @@ class WholeTextFileSuite extends QueryTest with SharedSparkSession {
     }
   }
 }
+
+class WholeTextFileV1Suite extends WholeTextFileSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "text")
+}
+
+class WholeTextFileV2Suite extends WholeTextFileSuite {
+  override protected def sparkConf: SparkConf =
+    super
+      .sparkConf
+      .set(SQLConf.USE_V1_SOURCE_LIST, "")
+}

From d955f7fd5429be1007a5183ef0e8cc7602a732fe Mon Sep 17 00:00:00 2001
From: Wei Zhang <wezhang@outlook.com>
Date: Fri, 24 Apr 2020 21:52:50 -0700
Subject: [PATCH 0560/1280] [SPARK-31516][DOC] Fix non-existed metric
 hiveClientCalls.count of CodeGenerator in DOC
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR proposes to remove the non-existed `hiveClientCalls.count` metric documentation of `CodeGenerator` of the Spark metrics system in the monitoring guide.

There is a duplicated `hiveClientCalls.count` metric in both `namespace=HiveExternalCatalog` and  `namespace=CodeGenerator` bullet lists, but there is only one defined inside object `HiveCatalogMetrics`.

Closes #28292 from wezhang/monitoringdoc.

Authored-by: Wei Zhang <wezhang@outlook.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 3e83ccc5d826e7a26e5a104ee0f3424199251a80)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/monitoring.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index b72fd027da597..7e41c9df4efa0 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -1056,7 +1056,6 @@ This is the component with the largest amount of instrumented metrics
   - compilationTime (histogram)
   - generatedClassSize (histogram)
   - generatedMethodSize (histogram)
-  - hiveClientCalls.count
   - sourceCodeSize (histogram)
 
 - namespace=DAGScheduler

From b1d9088b1e1f8888ecdcabf65520775f75ab6a0d Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 24 Apr 2020 22:11:35 -0700
Subject: [PATCH 0561/1280] [SPARK-31521][CORE] Correct the fetch size when
 merging blocks into a merged block

### What changes were proposed in this pull request?

Fix the wrong fetch size.

### Why are the changes needed?

The fetch size should be the sum of the size of merged block and the total size of those merging blocks. But we missed the size of merged block.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added a regression test.

Closes #28301 from Ngone51/fix_merged_block_size.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ab8cada1f9536a9506b4ede0f6b6339d5951d5b8)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../storage/ShuffleBlockFetcherIterator.scala | 156 ++++++++++--------
 .../ShuffleBlockFetcherIteratorSuite.scala    |  20 +++
 2 files changed, 106 insertions(+), 70 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
index 404e055f914e8..5efbc0703f729 100644
--- a/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
+++ b/core/src/main/scala/org/apache/spark/storage/ShuffleBlockFetcherIterator.scala
@@ -297,13 +297,15 @@ final class ShuffleBlockFetcherIterator(
       if (address.executorId == blockManager.blockManagerId.executorId) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
-          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)))
+          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch)
+        numBlocksToFetch += mergedBlockInfos.size
         localBlocks ++= mergedBlockInfos.map(info => (info.blockId, info.mapIndex))
         localBlockBytes += mergedBlockInfos.map(_.size).sum
       } else if (hostLocalDirReadingEnabled && address.host == blockManager.blockManagerId.host) {
         checkBlockSizes(blockInfos)
         val mergedBlockInfos = mergeContinuousShuffleBlockIdsIfNeeded(
-          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)))
+          blockInfos.map(info => FetchBlockInfo(info._1, info._2, info._3)), doBatchFetch)
+        numBlocksToFetch += mergedBlockInfos.size
         val blocksForAddress =
           mergedBlockInfos.map(info => (info.blockId, info.size, info.mapIndex))
         hostLocalBlocksByExecutor += address -> blocksForAddress
@@ -340,7 +342,8 @@ final class ShuffleBlockFetcherIterator(
       address: BlockManagerId,
       isLast: Boolean,
       collectedRemoteRequests: ArrayBuffer[FetchRequest]): Seq[FetchBlockInfo] = {
-    val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks)
+    val mergedBlocks = mergeContinuousShuffleBlockIdsIfNeeded(curBlocks, doBatchFetch)
+    numBlocksToFetch += mergedBlocks.size
     var retBlocks = Seq.empty[FetchBlockInfo]
     if (mergedBlocks.length <= maxBlocksInFlightPerAddress) {
       collectedRemoteRequests += createFetchRequest(mergedBlocks, address)
@@ -400,73 +403,6 @@ final class ShuffleBlockFetcherIterator(
     blockInfos.foreach { case (blockId, size, _) => assertPositiveBlockSize(blockId, size) }
   }
 
-  private[this] def mergeContinuousShuffleBlockIdsIfNeeded(
-      blocks: Seq[FetchBlockInfo]): Seq[FetchBlockInfo] = {
-    val result = if (doBatchFetch) {
-      var curBlocks = new ArrayBuffer[FetchBlockInfo]
-      val mergedBlockInfo = new ArrayBuffer[FetchBlockInfo]
-
-      def mergeFetchBlockInfo(toBeMerged: ArrayBuffer[FetchBlockInfo]): FetchBlockInfo = {
-        val startBlockId = toBeMerged.head.blockId.asInstanceOf[ShuffleBlockId]
-
-        // The last merged block may comes from the input, and we can merge more blocks
-        // into it, if the map id is the same.
-        def shouldMergeIntoPreviousBatchBlockId =
-          mergedBlockInfo.last.blockId.asInstanceOf[ShuffleBlockBatchId].mapId == startBlockId.mapId
-
-        val startReduceId = if (mergedBlockInfo.nonEmpty && shouldMergeIntoPreviousBatchBlockId) {
-          // Remove the previous batch block id as we will add a new one to replace it.
-          mergedBlockInfo.remove(mergedBlockInfo.length - 1).blockId
-            .asInstanceOf[ShuffleBlockBatchId].startReduceId
-        } else {
-          startBlockId.reduceId
-        }
-
-        FetchBlockInfo(
-          ShuffleBlockBatchId(
-            startBlockId.shuffleId,
-            startBlockId.mapId,
-            startReduceId,
-            toBeMerged.last.blockId.asInstanceOf[ShuffleBlockId].reduceId + 1),
-          toBeMerged.map(_.size).sum,
-          toBeMerged.head.mapIndex)
-      }
-
-      val iter = blocks.iterator
-      while (iter.hasNext) {
-        val info = iter.next()
-        // It's possible that the input block id is already a batch ID. For example, we merge some
-        // blocks, and then make fetch requests with the merged blocks according to "max blocks per
-        // request". The last fetch request may be too small, and we give up and put the remaining
-        // merged blocks back to the input list.
-        if (info.blockId.isInstanceOf[ShuffleBlockBatchId]) {
-          mergedBlockInfo += info
-        } else {
-          if (curBlocks.isEmpty) {
-            curBlocks += info
-          } else {
-            val curBlockId = info.blockId.asInstanceOf[ShuffleBlockId]
-            val currentMapId = curBlocks.head.blockId.asInstanceOf[ShuffleBlockId].mapId
-            if (curBlockId.mapId != currentMapId) {
-              mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
-              curBlocks.clear()
-            }
-            curBlocks += info
-          }
-        }
-      }
-      if (curBlocks.nonEmpty) {
-        mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
-      }
-      mergedBlockInfo
-    } else {
-      blocks
-    }
-    // update metrics
-    numBlocksToFetch += result.size
-    result
-  }
-
   /**
    * Fetch the local blocks while we are fetching remote blocks. This is ok because
    * `ManagedBuffer`'s memory is allocated lazily when we create the input stream, so all we
@@ -915,6 +851,86 @@ private class ShuffleFetchCompletionListener(var data: ShuffleBlockFetcherIterat
 private[storage]
 object ShuffleBlockFetcherIterator {
 
+  /**
+   * This function is used to merged blocks when doBatchFetch is true. Blocks which have the
+   * same `mapId` can be merged into one block batch. The block batch is specified by a range
+   * of reduceId, which implies the continuous shuffle blocks that we can fetch in a batch.
+   * For example, input blocks like (shuffle_0_0_0, shuffle_0_0_1, shuffle_0_1_0) can be
+   * merged into (shuffle_0_0_0_2, shuffle_0_1_0_1), and input blocks like (shuffle_0_0_0_2,
+   * shuffle_0_0_2, shuffle_0_0_3) can be merged into (shuffle_0_0_0_4).
+   *
+   * @param blocks blocks to be merged if possible. May contains already merged blocks.
+   * @param doBatchFetch whether to merge blocks.
+   * @return the input blocks if doBatchFetch=false, or the merged blocks if doBatchFetch=true.
+   */
+  def mergeContinuousShuffleBlockIdsIfNeeded(
+      blocks: Seq[FetchBlockInfo],
+      doBatchFetch: Boolean): Seq[FetchBlockInfo] = {
+    val result = if (doBatchFetch) {
+      var curBlocks = new ArrayBuffer[FetchBlockInfo]
+      val mergedBlockInfo = new ArrayBuffer[FetchBlockInfo]
+
+      def mergeFetchBlockInfo(toBeMerged: ArrayBuffer[FetchBlockInfo]): FetchBlockInfo = {
+        val startBlockId = toBeMerged.head.blockId.asInstanceOf[ShuffleBlockId]
+
+        // The last merged block may comes from the input, and we can merge more blocks
+        // into it, if the map id is the same.
+        def shouldMergeIntoPreviousBatchBlockId =
+          mergedBlockInfo.last.blockId.asInstanceOf[ShuffleBlockBatchId].mapId == startBlockId.mapId
+
+        val (startReduceId, size) =
+          if (mergedBlockInfo.nonEmpty && shouldMergeIntoPreviousBatchBlockId) {
+            // Remove the previous batch block id as we will add a new one to replace it.
+            val removed = mergedBlockInfo.remove(mergedBlockInfo.length - 1)
+            (removed.blockId.asInstanceOf[ShuffleBlockBatchId].startReduceId,
+              removed.size + toBeMerged.map(_.size).sum)
+          } else {
+            (startBlockId.reduceId, toBeMerged.map(_.size).sum)
+          }
+
+        FetchBlockInfo(
+          ShuffleBlockBatchId(
+            startBlockId.shuffleId,
+            startBlockId.mapId,
+            startReduceId,
+            toBeMerged.last.blockId.asInstanceOf[ShuffleBlockId].reduceId + 1),
+          size,
+          toBeMerged.head.mapIndex)
+      }
+
+      val iter = blocks.iterator
+      while (iter.hasNext) {
+        val info = iter.next()
+        // It's possible that the input block id is already a batch ID. For example, we merge some
+        // blocks, and then make fetch requests with the merged blocks according to "max blocks per
+        // request". The last fetch request may be too small, and we give up and put the remaining
+        // merged blocks back to the input list.
+        if (info.blockId.isInstanceOf[ShuffleBlockBatchId]) {
+          mergedBlockInfo += info
+        } else {
+          if (curBlocks.isEmpty) {
+            curBlocks += info
+          } else {
+            val curBlockId = info.blockId.asInstanceOf[ShuffleBlockId]
+            val currentMapId = curBlocks.head.blockId.asInstanceOf[ShuffleBlockId].mapId
+            if (curBlockId.mapId != currentMapId) {
+              mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
+              curBlocks.clear()
+            }
+            curBlocks += info
+          }
+        }
+      }
+      if (curBlocks.nonEmpty) {
+        mergedBlockInfo += mergeFetchBlockInfo(curBlocks)
+      }
+      mergedBlockInfo
+    } else {
+      blocks
+    }
+    result
+  }
+
   /**
    * The block information to fetch used in FetchRequest.
    * @param blockId block id
diff --git a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
index 773629c2c9a49..43917a5b83bb0 100644
--- a/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/ShuffleBlockFetcherIteratorSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer}
 import org.apache.spark.network.shuffle.{BlockFetchingListener, DownloadFileManager, ExternalBlockStoreClient}
 import org.apache.spark.network.util.LimitedInputStream
 import org.apache.spark.shuffle.FetchFailedException
+import org.apache.spark.storage.ShuffleBlockFetcherIterator.FetchBlockInfo
 import org.apache.spark.util.Utils
 
 
@@ -1071,4 +1072,23 @@ class ShuffleBlockFetcherIteratorSuite extends SparkFunSuite with PrivateMethodT
     val e = intercept[FetchFailedException] { iterator.next() }
     assert(e.getMessage.contains("Received a zero-size buffer"))
   }
+
+  test("SPARK-31521: correct the fetch size when merging blocks into a merged block") {
+    val bId1 = ShuffleBlockBatchId(0, 0, 0, 5)
+    val bId2 = ShuffleBlockId(0, 0, 6)
+    val bId3 = ShuffleBlockId(0, 0, 7)
+    val block1 = FetchBlockInfo(bId1, 40, 0)
+    val block2 = FetchBlockInfo(bId2, 50, 0)
+    val block3 = FetchBlockInfo(bId3, 60, 0)
+    val inputBlocks = Seq(block1, block2, block3)
+
+    val mergedBlocks = ShuffleBlockFetcherIterator.
+      mergeContinuousShuffleBlockIdsIfNeeded(inputBlocks, true)
+    assert(mergedBlocks.size === 1)
+    val mergedBlock = mergedBlocks.head
+    val mergedBlockId = mergedBlock.blockId.asInstanceOf[ShuffleBlockBatchId]
+    assert(mergedBlockId.startReduceId === bId1.startReduceId)
+    assert(mergedBlockId.endReduceId === bId3.reduceId + 1)
+    assert(mergedBlock.size === inputBlocks.map(_.size).sum)
+  }
 }

From dbcf855d8448b7bf8ba9d8c4a08f8bef14be2805 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sat, 25 Apr 2020 09:29:51 -0700
Subject: [PATCH 0562/1280] [SPARK-31563][SQL] Fix failure of InSet.sql for
 collections of Catalyst's internal types

### What changes were proposed in this pull request?
In the PR, I propose to fix the `InSet.sql` method for the cases when input collection contains values of internal Catalyst's types, for instance `UTF8String`. Elements of the input set `hset` are converted to Scala types, and wrapped by `Literal` to properly form SQL view of the input collection.

### Why are the changes needed?
The changes fixed the bug in `InSet.sql` that makes wrong assumption about types of collection elements. See more details in SPARK-31563.

### Does this PR introduce any user-facing change?
Highly likely, not.

### How was this patch tested?
Added a test to `ColumnExpressionSuite`

Closes #28343 from MaxGekk/fix-InSet-sql.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7d8216a6642f40af0d1b623129b1d5f4c86bec68)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/expressions/predicates.scala       | 5 ++++-
 .../org/apache/spark/sql/ColumnExpressionSuite.scala      | 8 +++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index bd190c3e5abc7..ac492cf227301 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
 
 import scala.collection.immutable.TreeSet
 
+import org.apache.spark.sql.catalyst.CatalystTypeConverters.convertToScala
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReference
@@ -519,7 +520,9 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
 
   override def sql: String = {
     val valueSQL = child.sql
-    val listSQL = hset.toSeq.map(Literal(_).sql).mkString(", ")
+    val listSQL = hset.toSeq
+      .map(elem => Literal(convertToScala(elem, child.dataType)).sql)
+      .mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index a9ee25b10dc02..b72d92b9e2a83 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -26,12 +26,13 @@ import org.apache.hadoop.io.{LongWritable, Text}
 import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat}
 import org.scalatest.Matchers._
 
-import org.apache.spark.sql.catalyst.expressions.{In, InSet, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{In, InSet, Literal, NamedExpression}
 import org.apache.spark.sql.execution.ProjectExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 
 class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -869,4 +870,9 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
       df.select(typedLit(("a", 2, 1.0))),
       Row(Row("a", 2, 1.0)) :: Nil)
   }
+
+  test("SPARK-31563: sql of InSet for UTF8String collection") {
+    val inSet = InSet(Literal("a"), Set("a", "b").map(UTF8String.fromString))
+    assert(inSet.sql === "('a' IN ('a', 'b'))")
+  }
 }

From eb54e6c3e8490cb63b52ccea65b6ca4f195fbcf9 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sat, 25 Apr 2020 09:39:44 -0700
Subject: [PATCH 0563/1280] [SPARK-31528][SQL][3.0] Remove millennium, century,
 decade from trunc/date_trunc fucntions

The PR backport https://github.com/apache/spark/pull/28313 from master to 3.0

### What changes were proposed in this pull request?

Similar to https://jira.apache.org/jira/browse/SPARK-31507, millennium, century, and decade are not commonly used in most modern platforms.

For example
Negative:
https://docs.snowflake.com/en/sql-reference/functions-date-time.html#supported-date-and-time-parts
https://prestodb.io/docs/current/functions/datetime.html#date_trunc
https://teradata.github.io/presto/docs/148t/functions/datetime.html#date_trunc
https://www.oracletutorial.com/oracle-date-functions/oracle-trunc/

Positive:
https://docs.aws.amazon.com/redshift/latest/dg/r_Dateparts_for_datetime_functions.html
https://www.postgresql.org/docs/9.1/functions-datetime.html#FUNCTIONS-DATETIME-TRUNC

This PR removes these `fmt`s support for trunc and date_trunc functions.

### Why are the changes needed?

clean uncommon datetime unit for easy maintenance, we can add them back if they are found very useful later.

### Does this PR introduce any user-facing change?
no, targeting 3.0.0, these are newly added in 3.0.0

### How was this patch tested?

remove and modify existing units tests

Closes #28340 from yaooqinn/SPARK-31528-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/datetimeExpressions.scala     | 38 +++++----
 .../sql/catalyst/util/DateTimeUtils.scala     | 26 +-----
 .../expressions/DateExpressionsSuite.scala    | 84 +++++++++----------
 .../catalyst/util/DateTimeUtilsSuite.scala    |  6 --
 .../org/apache/spark/sql/functions.scala      |  4 +-
 .../apache/spark/sql/DateFunctionsSuite.scala | 20 ++---
 6 files changed, 79 insertions(+), 99 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 6ca6ab7a3c1ac..f828bd43e53f9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -1621,7 +1621,15 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
 @ExpressionDescription(
   usage = """
     _FUNC_(date, fmt) - Returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.
-    `fmt` should be one of ["week", "mon", "month", "mm", "quarter", "year", "yyyy", "yy", "decade", "century", "millennium"]
+  """,
+  arguments = """
+     Arguments:
+       * date - date value or valid date string
+       * fmt - the format representing the unit to be truncated to
+           - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `date` falls in
+           - "QUARTER" - truncate to the first date of the quarter that the `date` falls in
+           - "MONTH", "MM", "MON" - truncate to the first date of the month that the `date` falls in
+           - "WEEK" - truncate to the Monday of the week that the `date` falls in
   """,
   examples = """
     Examples:
@@ -1633,12 +1641,6 @@ trait TruncInstant extends BinaryExpression with ImplicitCastInputTypes {
        2009-02-01
       > SELECT _FUNC_('2015-10-27', 'YEAR');
        2015-01-01
-      > SELECT _FUNC_('2015-10-27', 'DECADE');
-       2010-01-01
-      > SELECT _FUNC_('1981-01-19', 'century');
-       1901-01-01
-      > SELECT _FUNC_('1981-01-19', 'millennium');
-       1001-01-01
   """,
   group = "datetime_funcs",
   since = "1.5.0")
@@ -1673,9 +1675,21 @@ case class TruncDate(date: Expression, format: Expression)
 @ExpressionDescription(
   usage = """
     _FUNC_(fmt, ts) - Returns timestamp `ts` truncated to the unit specified by the format model `fmt`.
-    `fmt` should be one of ["MILLENNIUM", "CENTURY", "DECADE", "YEAR", "YYYY", "YY",
-                            "QUARTER", "MON", "MONTH", "MM", "WEEK", "DAY", "DD",
-                            "HOUR", "MINUTE", "SECOND", "MILLISECOND", "MICROSECOND"]
+  """,
+  arguments = """
+     Arguments:
+       * fmt - the format representing the unit to be truncated to
+           - "YEAR", "YYYY", "YY" - truncate to the first date of the year that the `ts` falls in, the time part will be zero out
+           - "QUARTER" - truncate to the first date of the quarter that the `ts` falls in, the time part will be zero out
+           - "MONTH", "MM", "MON" - truncate to the first date of the month that the `ts` falls in, the time part will be zero out
+           - "WEEK" - truncate to the Monday of the week that the `ts` falls in, the time part will be zero out
+           - "DAY", "DD" - zero out the time part
+           - "HOUR" - zero out the minute and second with fraction part
+           - "MINUTE"- zero out the second with fraction part
+           - "SECOND" -  zero out the second fraction part
+           - "MILLISECOND" - zero out the microseconds
+           - "MICROSECOND" - everything remains
+       * ts - datetime value or valid timestamp string
   """,
   examples = """
     Examples:
@@ -1689,10 +1703,6 @@ case class TruncDate(date: Expression, format: Expression)
        2015-03-05 09:00:00
       > SELECT _FUNC_('MILLISECOND', '2015-03-05T09:32:05.123456');
        2015-03-05 09:32:05.123
-      > SELECT _FUNC_('DECADE', '2015-03-05T09:32:05.123456');
-       2010-01-01 00:00:00
-      > SELECT _FUNC_('CENTURY', '2015-03-05T09:32:05.123456');
-       2001-01-01 00:00:00
   """,
   group = "datetime_funcs",
   since = "2.3.0")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 0e9a15be1a6c6..a2fa87a4a2360 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -718,40 +718,25 @@ object DateTimeUtils {
   private[sql] val TRUNC_TO_MINUTE = 3
   private[sql] val TRUNC_TO_HOUR = 4
   private[sql] val TRUNC_TO_DAY = 5
-  // The levels from TRUNC_TO_WEEK to TRUNC_TO_MILLENNIUM are used in truncations
+  // The levels from TRUNC_TO_WEEK to TRUNC_TO_YEAR are used in truncations
   // of DATE and TIMESTAMP values.
   private[sql] val TRUNC_TO_WEEK = 6
   private[sql] val MIN_LEVEL_OF_DATE_TRUNC = TRUNC_TO_WEEK
   private[sql] val TRUNC_TO_MONTH = 7
   private[sql] val TRUNC_TO_QUARTER = 8
   private[sql] val TRUNC_TO_YEAR = 9
-  private[sql] val TRUNC_TO_DECADE = 10
-  private[sql] val TRUNC_TO_CENTURY = 11
-  private[sql] val TRUNC_TO_MILLENNIUM = 12
 
   /**
    * Returns the trunc date from original date and trunc level.
-   * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 6.
+   * Trunc level should be generated using `parseTruncLevel()`, should be between 6 and 9.
    */
   def truncDate(d: SQLDate, level: Int): SQLDate = {
-    def truncToYearLevel(divider: Int, adjust: Int): SQLDate = {
-      val oldYear = getYear(d)
-      var newYear = Math.floorDiv(oldYear, divider)
-      if (adjust > 0 && Math.floorMod(oldYear, divider) == 0) {
-        newYear -= 1
-      }
-      newYear = newYear * divider + adjust
-      localDateToDays(LocalDate.of(newYear, 1, 1))
-    }
     level match {
       case TRUNC_TO_WEEK => getNextDateForDayOfWeek(d - 7, MONDAY)
       case TRUNC_TO_MONTH => d - DateTimeUtils.getDayOfMonth(d) + 1
       case TRUNC_TO_QUARTER =>
         localDateToDays(daysToLocalDate(d).`with`(IsoFields.DAY_OF_QUARTER, 1L))
       case TRUNC_TO_YEAR => d - DateTimeUtils.getDayInYear(d) + 1
-      case TRUNC_TO_DECADE => truncToYearLevel(10, 0)
-      case TRUNC_TO_CENTURY => truncToYearLevel(100, 1)
-      case TRUNC_TO_MILLENNIUM => truncToYearLevel(1000, 1)
       case _ =>
         // caller make sure that this should never be reached
         sys.error(s"Invalid trunc level: $level")
@@ -765,7 +750,7 @@ object DateTimeUtils {
 
   /**
    * Returns the trunc date time from original date time and trunc level.
-   * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 12.
+   * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 9.
    */
   def truncTimestamp(t: SQLTimestamp, level: Int, zoneId: ZoneId): SQLTimestamp = {
     level match {
@@ -789,7 +774,7 @@ object DateTimeUtils {
   }
 
   /**
-   * Returns the truncate level, could be from TRUNC_TO_MICROSECOND to TRUNC_TO_MILLENNIUM,
+   * Returns the truncate level, could be from TRUNC_TO_MICROSECOND to TRUNC_TO_YEAR,
    * or TRUNC_INVALID, TRUNC_INVALID means unsupported truncate level.
    */
   def parseTruncLevel(format: UTF8String): Int = {
@@ -807,9 +792,6 @@ object DateTimeUtils {
         case "MON" | "MONTH" | "MM" => TRUNC_TO_MONTH
         case "QUARTER" => TRUNC_TO_QUARTER
         case "YEAR" | "YYYY" | "YY" => TRUNC_TO_YEAR
-        case "DECADE" => TRUNC_TO_DECADE
-        case "CENTURY" => TRUNC_TO_CENTURY
-        case "MILLENNIUM" => TRUNC_TO_MILLENNIUM
         case _ => TRUNC_INVALID
       }
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index b3547172fe488..1d01f4b36dc86 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -611,105 +611,103 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       NextDay(Literal(Date.valueOf("2015-07-23")), Literal("\"quote")) :: Nil)
   }
 
+  private def testTruncDate(input: Date, fmt: String, expected: Date): Unit = {
+    checkEvaluation(TruncDate(Literal.create(input, DateType), Literal.create(fmt, StringType)),
+      expected)
+    checkEvaluation(
+      TruncDate(Literal.create(input, DateType), NonFoldableLiteral.create(fmt, StringType)),
+      expected)
+  }
+
   test("TruncDate") {
-    def testTrunc(input: Date, fmt: String, expected: Date): Unit = {
-      checkEvaluation(TruncDate(Literal.create(input, DateType), Literal.create(fmt, StringType)),
-        expected)
-      checkEvaluation(
-        TruncDate(Literal.create(input, DateType), NonFoldableLiteral.create(fmt, StringType)),
-        expected)
-    }
     val date = Date.valueOf("2015-07-22")
     Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt =>
-      testTrunc(date, fmt, Date.valueOf("2015-01-01"))
+      testTruncDate(date, fmt, Date.valueOf("2015-01-01"))
     }
     Seq("month", "MONTH", "mon", "MON", "mm", "MM").foreach { fmt =>
-      testTrunc(date, fmt, Date.valueOf("2015-07-01"))
+      testTruncDate(date, fmt, Date.valueOf("2015-07-01"))
     }
-    testTrunc(date, "DD", null)
-    testTrunc(date, "SECOND", null)
-    testTrunc(date, "HOUR", null)
-    testTrunc(date, null, null)
-    testTrunc(null, "MON", null)
-    testTrunc(null, null, null)
+    testTruncDate(date, "DD", null)
+    testTruncDate(date, "SECOND", null)
+    testTruncDate(date, "HOUR", null)
+    testTruncDate(null, "MON", null)
     // Test escaping of format
     GenerateUnsafeProjection.generate(TruncDate(Literal(0, DateType), Literal("\"quote")) :: Nil)
 
-    testTrunc(Date.valueOf("2000-03-08"), "decade", Date.valueOf("2000-01-01"))
-    testTrunc(Date.valueOf("2000-03-08"), "century", Date.valueOf("1901-01-01"))
   }
 
-  test("TruncTimestamp") {
-    def testTrunc(input: Timestamp, fmt: String, expected: Timestamp): Unit = {
-      checkEvaluation(
-        TruncTimestamp(Literal.create(fmt, StringType), Literal.create(input, TimestampType)),
-        expected)
-      checkEvaluation(
-        TruncTimestamp(
-          NonFoldableLiteral.create(fmt, StringType), Literal.create(input, TimestampType)),
-        expected)
-    }
+  private def testTruncTimestamp(input: Timestamp, fmt: String, expected: Timestamp): Unit = {
+    checkEvaluation(
+      TruncTimestamp(Literal.create(fmt, StringType), Literal.create(input, TimestampType)),
+      expected)
+    checkEvaluation(
+      TruncTimestamp(
+        NonFoldableLiteral.create(fmt, StringType), Literal.create(input, TimestampType)),
+      expected)
+  }
 
+  test("TruncTimestamp") {
     withDefaultTimeZone(UTC) {
       val inputDate = Timestamp.valueOf("2015-07-22 05:30:06")
 
       Seq("yyyy", "YYYY", "year", "YEAR", "yy", "YY").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-01-01 00:00:00"))
       }
 
       Seq("month", "MONTH", "mon", "MON", "mm", "MM").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-01 00:00:00"))
       }
 
       Seq("DAY", "day", "DD", "dd").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 00:00:00"))
       }
 
       Seq("HOUR", "hour").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 05:00:00"))
       }
 
       Seq("MINUTE", "minute").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 05:30:00"))
       }
 
       Seq("SECOND", "second").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-22 05:30:06"))
       }
 
       Seq("WEEK", "week").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-20 00:00:00"))
       }
 
       Seq("QUARTER", "quarter").foreach { fmt =>
-        testTrunc(
+        testTruncTimestamp(
           inputDate, fmt,
           Timestamp.valueOf("2015-07-01 00:00:00"))
       }
 
-      testTrunc(inputDate, "INVALID", null)
-      testTrunc(inputDate, null, null)
-      testTrunc(null, "MON", null)
-      testTrunc(null, null, null)
+      testTruncTimestamp(null, "MON", null)
+    }
+  }
 
-      testTrunc(Timestamp.valueOf("2000-03-08 11:12:13"), "decade",
-        Timestamp.valueOf("2000-01-01 00:00:00"))
-      testTrunc(Timestamp.valueOf("2000-03-08 11:12:13"), "century",
-        Timestamp.valueOf("1901-01-01 00:00:00"))
+  test("unsupported fmt fields for trunc/date_trunc results null") {
+    Seq("INVALID", "decade", "century", "millennium", "whatever", null).foreach { field =>
+      testTruncDate(Date.valueOf("2000-03-08"), field, null)
+      testTruncDate(null, field, null)
+      testTruncTimestamp(Timestamp.valueOf("2000-03-08 11:12:13"), field, null)
+      testTruncTimestamp(null, field, null)
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 6a1ee9f72e1a6..3981792797325 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -547,9 +547,6 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", defaultInputTS2.get)
     testTrunc(DateTimeUtils.TRUNC_TO_MICROSECOND, "2015-03-05T09:32:05.359123", defaultInputTS.get)
     testTrunc(DateTimeUtils.TRUNC_TO_MILLISECOND, "2015-03-05T09:32:05.359", defaultInputTS.get)
-    testTrunc(DateTimeUtils.TRUNC_TO_DECADE, "2010-01-01", defaultInputTS.get)
-    testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "2001-01-01", defaultInputTS.get)
-    testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", defaultInputTS.get)
 
     for (zid <- ALL_TIMEZONES) {
       withDefaultTimeZone(zid) {
@@ -580,9 +577,6 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
         testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS.get, zid)
         testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-01-01T00:00:00", inputTS1.get, zid)
         testTrunc(DateTimeUtils.TRUNC_TO_QUARTER, "2015-04-01T00:00:00", inputTS2.get, zid)
-        testTrunc(DateTimeUtils.TRUNC_TO_DECADE, "1990-01-01", inputTS5.get, zid)
-        testTrunc(DateTimeUtils.TRUNC_TO_CENTURY, "1901-01-01", inputTS5.get, zid)
-        testTrunc(DateTimeUtils.TRUNC_TO_MILLENNIUM, "2001-01-01", inputTS.get, zid)
       }
     }
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 9fd47186cc005..7c1457df9ca3c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -3099,6 +3099,7 @@ object functions {
    *             cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @param format: 'year', 'yyyy', 'yy' to truncate by year,
    *               or 'month', 'mon', 'mm' to truncate by month
+   *               Other options are: 'week', 'quarter'
    *
    * @return A date, or null if `date` was a string that could not be cast to a date or `format`
    *         was an invalid value
@@ -3117,7 +3118,8 @@ object functions {
    * @param format: 'year', 'yyyy', 'yy' to truncate by year,
    *                'month', 'mon', 'mm' to truncate by month,
    *                'day', 'dd' to truncate by day,
-   *                Other options are: 'second', 'minute', 'hour', 'week', 'month', 'quarter'
+   *                Other options are:
+   *                'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'
    * @param timestamp A date, timestamp or string. If a string, the data must be in a format that
    *                  can be cast to a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`
    * @return A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 6ed748304686b..5946536925ff3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -474,10 +474,6 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df.selectExpr("trunc(t, 'Month')"),
       Seq(Row(Date.valueOf("2015-07-01")), Row(Date.valueOf("2014-12-01"))))
-
-    checkAnswer(
-      df.selectExpr("trunc(t, 'decade')"),
-      Seq(Row(Date.valueOf("2010-01-01")), Row(Date.valueOf("2010-01-01"))))
   }
 
   test("function date_trunc") {
@@ -529,17 +525,15 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
       df.selectExpr("date_trunc('MILLISECOND', t)"),
       Seq(Row(Timestamp.valueOf("2015-07-22 10:01:40.123")),
         Row(Timestamp.valueOf("2014-12-31 05:29:06.123"))))
+  }
 
+  test("unsupported fmt fields for trunc/date_trunc results null") {
+    Seq("INVALID", "decade", "century", "millennium", "whatever", null).foreach { f =>
     checkAnswer(
-      df.selectExpr("date_trunc('DECADE', t)"),
-      Seq(Row(Timestamp.valueOf("2010-01-01 00:00:00")),
-        Row(Timestamp.valueOf("2010-01-01 00:00:00"))))
-
-    Seq("century", "millennium").foreach { level =>
-      checkAnswer(
-        df.selectExpr(s"date_trunc('$level', t)"),
-        Seq(Row(Timestamp.valueOf("2001-01-01 00:00:00")),
-          Row(Timestamp.valueOf("2001-01-01 00:00:00"))))
+      Seq(Date.valueOf("2014-12-31"))
+        .toDF("dt")
+        .selectExpr(s"date_trunc('$f', dt)", "trunc(dt, '$f')"),
+      Row(null, null))
     }
   }
 

From 6c6283b1e68b174b212f3352b1496ab9e92450ad Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sat, 25 Apr 2020 10:27:05 -0700
Subject: [PATCH 0564/1280] [SPARK-31564][TESTS] Fix flaky
 AllExecutionsPageSuite for checking 1970

### What changes were proposed in this pull request?

Fix flakiness by checking `1970/01/01` instead of `1970`.
The test was added by SPARK-27125 for 3.0.0.

### Why are the changes needed?

the `org.apache.spark.sql.execution.ui.AllExecutionsPageSuite.SPARK-27019:correctly display SQL page when event reordering happens` test is flaky for just checking the `html` content not containing 1970. I will add a ticket to check and fix that.
In the specific failure https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/121799/testReport, it failed because the `html`

```
...
<td sorttable_customkey="1587806019707">
...

```
contained `1970`.

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

passing jenkins

Closes #28344 from yaooqinn/SPARK-31564.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7959808e962584704cba1ed26a4bcece724424d5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
index 298afa880c930..24b8a973ade38 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/AllExecutionsPageSuite.scala
@@ -57,7 +57,7 @@ class AllExecutionsPageSuite extends SharedSparkSession with BeforeAndAfter {
 
     val html = renderSQLPage(request, tab, statusStore).toString().toLowerCase(Locale.ROOT)
     assert(html.contains("failed queries"))
-    assert(!html.contains("1970"))
+    assert(!html.contains("1970/01/01"))
   }
 
   test("sorting should be successful") {

From 27a9c1da6f9ced760c33ff624c2eadc5f6a85dd9 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Sat, 25 Apr 2020 13:43:52 -0700
Subject: [PATCH 0565/1280] [SPARK-31558][UI] Code clean up in spark-sql-viz.js

### What changes were proposed in this pull request?

1. Remove console.log(), which seems unnecessary in the releases.
2. Replace the double equals to triple equals
3. Reuse jquery selector.

### Why are the changes needed?

For better code quality.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Existing tests + manual test.

Closes #28333 from gengliangwang/removeLog.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit f59ebdef5b9c03dab99835527554ba8bebe38548)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../spark/sql/execution/ui/static/spark-sql-viz.js     | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
index bded921a2b83e..724cec10d230e 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.js
@@ -61,7 +61,7 @@ function planVizContainer() { return d3.select("#plan-viz-graph"); }
  * node, it will display the details of this SparkPlan node in the right.
  */
 function setupTooltipForSparkPlanNode(nodeId) {
-  var nodeTooltip = d3.select("#plan-meta-data-" + nodeId).text()
+  var nodeTooltip = d3.select("#plan-meta-data-" + nodeId).text();
   d3.select("svg g .node_" + nodeId)
     .on('mouseover', function(d) {
       var domNode = d3.select(this).node();
@@ -127,10 +127,8 @@ function preprocessGraphLayout(g) {
  */
 function resizeSvg(svg) {
   var allClusters = svg.selectAll("g rect")[0];
-  console.log(allClusters);
   var startX = -PlanVizConstants.svgMarginX +
     toFloat(d3.min(allClusters, function(e) {
-      console.log(e);
       return getAbsolutePosition(d3.select(e)).x;
     }));
   var startY = -PlanVizConstants.svgMarginY +
@@ -183,7 +181,7 @@ function getAbsolutePosition(d3selection) {
     // Climb upwards to find how our parents are translated
     obj = d3.select(obj.node().parentNode);
     // Stop when we've reached the graph container itself
-    if (obj.node() == planVizContainer().node()) {
+    if (obj.node() === planVizContainer().node()) {
       break;
     }
   }
@@ -215,8 +213,8 @@ function postprocessForAdditionalMetrics() {
   checkboxNode.click(function() {
       onClickAdditionalMetricsCheckbox($(this));
   });
-  var isChecked = window.localStorage.getItem("stageId-and-taskId-checked") == "true";
-  $("#stageId-and-taskId-checkbox").prop("checked", isChecked);
+  var isChecked = window.localStorage.getItem("stageId-and-taskId-checked") === "true";
+  checkboxNode.prop("checked", isChecked);
   onClickAdditionalMetricsCheckbox(checkboxNode);
 }
 

From 3d8a62ded8a7058f9a292e402e8e6f688d1883c7 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sun, 26 Apr 2020 11:46:52 -0700
Subject: [PATCH 0566/1280] [SPARK-31562][SQL] Update ExpressionDescription for
 substring, current_date, and current_timestamp

### What changes were proposed in this pull request?

This PR intends to add entries for substring, current_date, and current_timestamp in the SQL built-in function documents. Specifically, the entries are as follows;

 - SELECT current_date;
 - SELECT current_timestamp;
 - SELECT substring('abcd' FROM 1);
 - SELECT substring('abcd' FROM 1 FOR 2);

### Why are the changes needed?

To make the SQL (built-in functions) references complete.

### Does this PR introduce any user-facing change?

<img width="1040" alt="Screen Shot 2020-04-25 at 16 51 07" src="https://user-images.githubusercontent.com/692303/80274851-6ca5ee00-8718-11ea-9a35-9ae82008cb4b.png">

<img width="974" alt="Screen Shot 2020-04-25 at 17 24 24" src="https://user-images.githubusercontent.com/692303/80275032-a88d8300-8719-11ea-92ec-95b80169ae28.png">

<img width="862" alt="Screen Shot 2020-04-25 at 17 27 48" src="https://user-images.githubusercontent.com/692303/80275114-36696e00-871a-11ea-8e39-02e93eabb92f.png">

### How was this patch tested?

Added test examples.

Closes #28342 from maropu/SPARK-31562.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e01125db0d351001dc971e33f4b15403f24d97b4)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/FunctionRegistry.scala  |  2 +-
 .../UnsupportedOperationChecker.scala         |  6 +-
 .../expressions/datetimeExpressions.scala     | 63 +++++++++++++++----
 .../expressions/stringExpressions.scala       | 12 +++-
 .../catalyst/optimizer/finishAnalysis.scala   |  2 +-
 .../sql/expressions/ExpressionInfoSuite.scala |  3 +
 sql/gen-sql-functions-docs.py                 |  8 ++-
 7 files changed, 74 insertions(+), 22 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 6cd492f0765c8..d3b073119e640 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -404,7 +404,7 @@ object FunctionRegistry {
     expression[Month]("month"),
     expression[MonthsBetween]("months_between"),
     expression[NextDay]("next_day"),
-    expression[CurrentTimestamp]("now", true),
+    expression[Now]("now"),
     expression[Quarter]("quarter"),
     expression[Second]("second"),
     expression[ParseToTimestamp]("to_timestamp"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
index 2f8cb26ffaa9c..423f89fefa093 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
@@ -19,13 +19,11 @@ package org.apache.spark.sql.catalyst.analysis
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID, Now}
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 
 /**
@@ -412,7 +410,7 @@ object UnsupportedOperationChecker extends Logging {
 
       subPlan.expressions.foreach { e =>
         if (e.collectLeaves().exists {
-          case (_: CurrentTimestamp | _: CurrentDate) => true
+          case (_: CurrentTimestamp | _: Now | _: CurrentDate) => true
           case _ => false
         }) {
           throwError(s"Continuous processing does not support current time operations.")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index f828bd43e53f9..8f57421044b2b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -29,7 +29,6 @@ import org.apache.commons.text.StringEscapeUtils
 import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter}
@@ -63,7 +62,21 @@ trait TimeZoneAwareExpression extends Expression {
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the current date at the start of query evaluation.",
+  usage = """
+    _FUNC_() - Returns the current date at the start of query evaluation.
+
+    _FUNC_ - Returns the current date at the start of query evaluation.
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       2020-04-25
+      > SELECT _FUNC_;
+       2020-04-25
+  """,
+  note = """
+    The syntax without braces has been supported since 2.0.1.
+  """,
   group = "datetime_funcs",
   since = "1.5.0")
 case class CurrentDate(timeZoneId: Option[String] = None)
@@ -84,6 +97,13 @@ case class CurrentDate(timeZoneId: Option[String] = None)
   override def prettyName: String = "current_date"
 }
 
+abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallback {
+  override def foldable: Boolean = true
+  override def nullable: Boolean = false
+  override def dataType: DataType = TimestampType
+  override def eval(input: InternalRow): Any = currentTimestamp()
+}
+
 /**
  * Returns the current timestamp at the start of query evaluation.
  * All calls of current_timestamp within the same query return the same value.
@@ -91,19 +111,38 @@ case class CurrentDate(timeZoneId: Option[String] = None)
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.",
+  usage = """
+    _FUNC_() - Returns the current timestamp at the start of query evaluation.
+
+    _FUNC_ - Returns the current timestamp at the start of query evaluation.
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       2020-04-25 15:49:11.914
+      > SELECT _FUNC_;
+       2020-04-25 15:49:11.914
+  """,
+  note = """
+    The syntax without braces has been supported since 2.0.1.
+  """,
   group = "datetime_funcs",
   since = "1.5.0")
-case class CurrentTimestamp() extends LeafExpression with CodegenFallback {
-  override def foldable: Boolean = true
-  override def nullable: Boolean = false
-
-  override def dataType: DataType = TimestampType
-
-  override def eval(input: InternalRow): Any = currentTimestamp()
+case class CurrentTimestamp() extends CurrentTimestampLike {
+  override def prettyName: String = "current_timestamp"
+}
 
-  override def prettyName: String =
-    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("current_timestamp")
+@ExpressionDescription(
+  usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       2020-04-25 15:49:11.914
+  """,
+  group = "datetime_funcs",
+  since = "1.6.0")
+case class Now() extends CurrentTimestampLike {
+  override def prettyName: String = "now"
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 3723680ff5db2..82b1e5f0998b0 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -1619,7 +1619,11 @@ case class StringSpace(child: Expression)
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.",
+  usage = """
+    _FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+
+    _FUNC_(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_('Spark SQL', 5);
@@ -1628,6 +1632,12 @@ case class StringSpace(child: Expression)
        SQL
       > SELECT _FUNC_('Spark SQL', 5, 1);
        k
+      > SELECT _FUNC_('Spark SQL' FROM 5);
+       k SQL
+      > SELECT _FUNC_('Spark SQL' FROM -3);
+       SQL
+      > SELECT _FUNC_('Spark SQL' FROM 5 FOR 1);
+       k
   """,
   since = "1.5.0")
 // scalastyle:on line.size.limit
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index c79bf3e20b776..80d85827657fd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -85,7 +85,7 @@ object ComputeCurrentTime extends Rule[LogicalPlan] {
             LocalDate.now(DateTimeUtils.getZoneId(timeZoneId)),
             DateType)
         })
-      case CurrentTimestamp() => currentTime
+      case CurrentTimestamp() | Now() => currentTime
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
index 9a6fe46c48ec7..e18514c6f93f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala
@@ -120,6 +120,9 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession {
     val ignoreSet = Set(
       // One of examples shows getting the current timestamp
       "org.apache.spark.sql.catalyst.expressions.UnixTimestamp",
+      "org.apache.spark.sql.catalyst.expressions.CurrentDate",
+      "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp",
+      "org.apache.spark.sql.catalyst.expressions.Now",
       // Random output without a seed
       "org.apache.spark.sql.catalyst.expressions.Rand",
       "org.apache.spark.sql.catalyst.expressions.Randn",
diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py
index 7f0b2ae582f56..c07734e273051 100644
--- a/sql/gen-sql-functions-docs.py
+++ b/sql/gen-sql-functions-docs.py
@@ -104,9 +104,11 @@ def _make_pretty_usage(infos):
     result.append("  <tbody>")
 
     for info in infos:
-        # Extracts (signature, description) pairs from `info.usage`, e.g.,
-        # the signature is `func(expr)` and the description is `...` in an usage `func(expr) - ...`.
-        usages = iter(re.split(r"(%s\(.*\)) - " % info.name, info.usage.strip())[1:])
+        # Extracts (signature, description) pairs from `info.usage`.
+        # Expected formats are as follows;
+        #  - `_FUNC_(...) - description`, or
+        #  - `_FUNC_ - description`
+        usages = iter(re.split(r"(%s.*) - " % info.name, info.usage.strip())[1:])
         for (sig, description) in zip(usages, usages):
             result.append("    <tr>")
             result.append("      <td>%s</td>" % sig)

From 79444d6d6f1e63f28a27420b3165e1115c35c687 Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Sun, 26 Apr 2020 15:31:32 -0700
Subject: [PATCH 0567/1280] [SPARK-31535][SQL] Fix nested CTE substitution

### What changes were proposed in this pull request?

This PR fixes a CTE substitution issue so as to the following SQL return the correct empty result:
```
WITH t(c) AS (SELECT 1)
SELECT * FROM t
WHERE c IN (
  WITH t(c) AS (SELECT 2)
  SELECT * FROM t
)
```
Before this PR the result was `1`.

### Why are the changes needed?
To fix a correctness issue.

### Does this PR introduce any user-facing change?
Yes, fixes a correctness issue.

### How was this patch tested?
Added new test case.

Closes #28318 from peter-toth/SPARK-31535-fix-nested-cte-substitution.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 4f53bfbbd55401d56dda2fbdbbbae4eea660ded6)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/CTESubstitution.scala   | 61 +++++++------------
 .../resources/sql-tests/inputs/cte-nested.sql |  8 +++
 .../sql-tests/results/cte-legacy.sql.out      | 15 ++++-
 .../sql-tests/results/cte-nested.sql.out      | 16 ++++-
 .../sql-tests/results/cte-nonlegacy.sql.out   | 15 ++++-
 5 files changed, 72 insertions(+), 43 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index c8078e10fd044..d9fdb56739e00 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -31,28 +31,28 @@ object CTESubstitution extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
     LegacyBehaviorPolicy.withName(SQLConf.get.getConf(LEGACY_CTE_PRECEDENCE_POLICY)) match {
       case LegacyBehaviorPolicy.EXCEPTION =>
-        assertNoNameConflictsInCTE(plan, inTraverse = false)
-        traverseAndSubstituteCTE(plan, inTraverse = false)
+        assertNoNameConflictsInCTE(plan)
+        traverseAndSubstituteCTE(plan)
       case LegacyBehaviorPolicy.LEGACY =>
         legacyTraverseAndSubstituteCTE(plan)
       case LegacyBehaviorPolicy.CORRECTED =>
-        traverseAndSubstituteCTE(plan, inTraverse = false)
+        traverseAndSubstituteCTE(plan)
     }
   }
 
   /**
    * Check the plan to be traversed has naming conflicts in nested CTE or not, traverse through
-   * child, innerChildren and subquery for the current plan.
+   * child, innerChildren and subquery expressions for the current plan.
    */
   private def assertNoNameConflictsInCTE(
       plan: LogicalPlan,
-      inTraverse: Boolean,
-      cteNames: Set[String] = Set.empty): Unit = {
-    plan.foreach {
+      outerCTERelationNames: Set[String] = Set.empty,
+      namesInSubqueries: Set[String] = Set.empty): Unit = {
+    plan match {
       case w @ With(child, relations) =>
         val newNames = relations.map {
           case (cteName, _) =>
-            if (cteNames.contains(cteName)) {
+            if (outerCTERelationNames.contains(cteName)) {
               throw new AnalysisException(s"Name $cteName is ambiguous in nested CTE. " +
                 s"Please set ${LEGACY_CTE_PRECEDENCE_POLICY.key} to CORRECTED so that name " +
                 "defined in inner CTE takes precedence. If set it to LEGACY, outer CTE " +
@@ -60,24 +60,15 @@ object CTESubstitution extends Rule[LogicalPlan] {
             } else {
               cteName
             }
-        }.toSet
-        child.transformExpressions {
-          case e: SubqueryExpression =>
-            assertNoNameConflictsInCTE(e.plan, inTraverse = true, cteNames ++ newNames)
-            e
-        }
-        w.innerChildren.foreach { p =>
-          assertNoNameConflictsInCTE(p, inTraverse = true, cteNames ++ newNames)
-        }
-
-      case other if inTraverse =>
-        other.transformExpressions {
-          case e: SubqueryExpression =>
-            assertNoNameConflictsInCTE(e.plan, inTraverse = true, cteNames)
-            e
-        }
+        }.toSet ++ namesInSubqueries
+        assertNoNameConflictsInCTE(child, outerCTERelationNames, newNames)
+        w.innerChildren.foreach(assertNoNameConflictsInCTE(_, newNames, newNames))
 
-      case _ =>
+      case other =>
+        other.subqueries.foreach(
+          assertNoNameConflictsInCTE(_, namesInSubqueries, namesInSubqueries))
+        other.children.foreach(
+          assertNoNameConflictsInCTE(_, outerCTERelationNames, namesInSubqueries))
     }
   }
 
@@ -131,37 +122,27 @@ object CTESubstitution extends Rule[LogicalPlan] {
    *     SELECT * FROM t
    *   )
    * @param plan the plan to be traversed
-   * @param inTraverse whether the current traverse is called from another traverse, only in this
-   *                   case name collision can occur
    * @return the plan where CTE substitution is applied
    */
-  private def traverseAndSubstituteCTE(plan: LogicalPlan, inTraverse: Boolean): LogicalPlan = {
+  private def traverseAndSubstituteCTE(plan: LogicalPlan): LogicalPlan = {
     plan.resolveOperatorsUp {
       case With(child: LogicalPlan, relations) =>
-        // child might contain an inner CTE that has priority so traverse and substitute inner CTEs
-        // in child first
-        val traversedChild: LogicalPlan = child transformExpressions {
-          case e: SubqueryExpression => e.withNewPlan(traverseAndSubstituteCTE(e.plan, true))
-        }
-
         // Substitute CTE definitions from last to first as a CTE definition can reference a
         // previous one
-        relations.foldRight(traversedChild) {
+        relations.foldRight(child) {
           case ((cteName, ctePlan), currentPlan) =>
             // A CTE definition might contain an inner CTE that has priority, so traverse and
             // substitute CTE defined in ctePlan.
             // A CTE definition might not be used at all or might be used multiple times. To avoid
             // computation if it is not used and to avoid multiple recomputation if it is used
             // multiple times we use a lazy construct with call-by-name parameter passing.
-            lazy val substitutedCTEPlan = traverseAndSubstituteCTE(ctePlan, true)
+            lazy val substitutedCTEPlan = traverseAndSubstituteCTE(ctePlan)
             substituteCTE(currentPlan, cteName, substitutedCTEPlan)
         }
 
-      // CTE name collision can occur only when inTraverse is true, it helps to avoid eager CTE
-      // substitution in a subquery expression.
-      case other if inTraverse =>
+      case other =>
         other.transformExpressions {
-          case e: SubqueryExpression => e.withNewPlan(traverseAndSubstituteCTE(e.plan, true))
+          case e: SubqueryExpression => e.withNewPlan(traverseAndSubstituteCTE(e.plan))
         }
     }
   }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
index 5e5e3a51f1f5d..134f88bf3302f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
@@ -103,3 +103,11 @@ SELECT (
     SELECT * FROM t
   )
 );
+
+-- CTE in subquery expression shadows outer 4
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+);
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
index e8020e1a454e6..629daf7410e5a 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 12
+-- Number of queries: 13
 
 
 -- !query
@@ -166,3 +166,16 @@ SELECT (
 struct<scalarsubquery():int>
 -- !query output
 1
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<c:int>
+-- !query output
+1
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
index 64d635adae428..34e9be0ca0003 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 12
+-- Number of queries: 13
 
 
 -- !query
@@ -172,3 +172,17 @@ struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
 Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
index 9422bb642d96f..6eba1ad72523d 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 12
+-- Number of queries: 13
 
 
 -- !query
@@ -166,3 +166,16 @@ SELECT (
 struct<scalarsubquery():int>
 -- !query output
 3
+
+
+-- !query
+WITH t(c) AS (SELECT 1)
+SELECT * FROM t
+WHERE c IN (
+  WITH t(c) AS (SELECT 2)
+  SELECT * FROM t
+)
+-- !query schema
+struct<c:int>
+-- !query output
+

From c2bc58c2ded8ff55ff495b966d8d2bab29549fa0 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 26 Apr 2020 15:49:00 -0700
Subject: [PATCH 0568/1280] [SPARK-31489][SQL] Fix pushing down filters with
 `java.time.LocalDate` values in ORC

### What changes were proposed in this pull request?
Convert `java.time.LocalDate` to `java.sql.Date` in pushed down filters to ORC datasource when Java 8 time API enabled.

Closes #28272

### Why are the changes needed?
The changes fix the exception raised while pushing date filters when `spark.sql.datetime.java8API.enabled` is set to `true`:
```
Wrong value class java.time.LocalDate for DATE.EQUALS leaf
java.lang.IllegalArgumentException: Wrong value class java.time.LocalDate for DATE.EQUALS leaf
	at org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$PredicateLeafImpl.checkLiteralType(SearchArgumentImpl.java:192)
	at org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$PredicateLeafImpl.<init>(SearchArgumentImpl.java:75)
	at org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$BuilderImpl.equals(SearchArgumentImpl.java:352)
	at org.apache.spark.sql.execution.datasources.orc.OrcFilters$.buildLeafSearchArgument(OrcFilters.scala:229)
```

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
Added tests to `OrcFilterSuite`.

Closes #28261 from MaxGekk/orc-date-filter-pushdown.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit bd139bda4a0e01961b333349e2a60ed0258d8d56)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../datasources/orc/OrcFilters.scala          |  5 +++
 .../datasources/orc/OrcFilterSuite.scala      | 43 +++++++++++--------
 .../datasources/orc/OrcFilters.scala          |  5 +++
 .../datasources/orc/OrcFilterSuite.scala      | 43 +++++++++++--------
 4 files changed, 60 insertions(+), 36 deletions(-)

diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index f5abd30854e00..a01d5a44da714 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.time.LocalDate
+
 import org.apache.orc.storage.common.`type`.HiveDecimal
 import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
 import org.apache.orc.storage.ql.io.sarg.SearchArgument.Builder
@@ -24,6 +26,7 @@ import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.orc.storage.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
@@ -162,6 +165,8 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       value.asInstanceOf[Number].doubleValue()
     case _: DecimalType =>
       new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
+    case _: DateType if value.isInstanceOf[LocalDate] =>
+      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
     case _ => value
   }
 
diff --git a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index ee5162bced8ac..a1c325e7bb876 100644
--- a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -299,26 +299,33 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - date") {
-    val dates = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
+    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
       Date.valueOf(day)
     }
-    withOrcDataFrame(dates.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(dates(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(dates(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(dates(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(dates(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val dates = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 675e089153679..445a52cece1c3 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.time.LocalDate
+
 import org.apache.hadoop.hive.common.`type`.HiveDecimal
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
 import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder
@@ -24,6 +26,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
@@ -162,6 +165,8 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       value.asInstanceOf[Number].doubleValue()
     case _: DecimalType =>
       new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
+    case _: DateType if value.isInstanceOf[LocalDate] =>
+      toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
     case _ => value
   }
 
diff --git a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index 1baa69e82bb18..815af05beb002 100644
--- a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -300,26 +300,33 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - date") {
-    val dates = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
+    val input = Seq("2017-08-18", "2017-08-19", "2017-08-20", "2017-08-21").map { day =>
       Date.valueOf(day)
     }
-    withOrcDataFrame(dates.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(dates(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(dates(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(dates(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(dates(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(dates(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val dates = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === dates(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> dates(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < dates(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > dates(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= dates(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= dates(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(dates(0) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(dates(0) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(dates(1) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(dates(2) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(0) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(dates(3) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 

From a602ab6c1a2f5d2e47970c9f64c62d4e9500dce6 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sun, 26 Apr 2020 16:57:23 -0700
Subject: [PATCH 0569/1280] [SPARK-31565][WEBUI] Unify the font color of label
 among all DAG-viz

### What changes were proposed in this pull request?

This PR unifies the font color of label as `#333333` among all DAG-viz.

### Why are the changes needed?

For the consistent appearance among all DAG-viz.
There are three types of DAG-viz in the WebUI.
One is for stages, another one is for RDDs and the last one is for query plans.
But the font color of labels are slightly different among them.

For stages, the color is `#333333` (simply 333) which is specified by `spark-dag-viz.css`.
<img width="355" alt="job-graph" src="https://user-images.githubusercontent.com/4736016/80321397-b517f580-8857-11ea-8c8e-cf68f648ab05.png">
<img width="310" alt="job-graph-color" src="https://user-images.githubusercontent.com/4736016/80321399-ba754000-8857-11ea-8708-83bdef4bc1d1.png">

For RDDs, the color is `#212529` which is specified by `bootstrap.min.js`.
<img width="386" alt="stage-graph" src="https://user-images.githubusercontent.com/4736016/80321438-f0b2bf80-8857-11ea-9c2a-13fa0fd1431c.png">
<img width="313" alt="stage-graph-color" src="https://user-images.githubusercontent.com/4736016/80321444-fa3c2780-8857-11ea-81b2-4f1203d47896.png">

For query plans, the color is `black` which is specified by `spark-sql-viz.css`.
<img width="449" alt="plan-graph" src="https://user-images.githubusercontent.com/4736016/80321490-61f27280-8858-11ea-9c3a-2c98d3d4d03b.png">
<img width="316" alt="plan-graph-color" src="https://user-images.githubusercontent.com/4736016/80321496-6ae34400-8858-11ea-8fe8-0d6e4a821608.png">

After the change, the appearance is like as follows (no change for stages).

For RDDs.
<img width="389" alt="stage-graph-fixed" src="https://user-images.githubusercontent.com/4736016/80321613-6b300f00-8859-11ea-912f-d92474aa9f47.png">

For query plans.
<img width="456" alt="plan-graph-fixed" src="https://user-images.githubusercontent.com/4736016/80321638-9a468080-8859-11ea-974c-33c56a8ffe1a.png">

### Does this PR introduce any user-facing change?

Yes. The unified color is slightly lighter than ever.

### How was this patch tested?

Confirmed that the color code among all DAG-viz are `#333333` using browser's debug console.

Closes #28352 from sarutak/unify-label-color.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 91ec2eacfaa030c7767bf1d192d5db8f56ff6854)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../main/resources/org/apache/spark/ui/static/spark-dag-viz.css | 1 +
 .../org/apache/spark/sql/execution/ui/static/spark-sql-viz.css  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
index 1fbc90b832bc9..e44a724c202f7 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.css
@@ -22,6 +22,7 @@
 #dag-viz-graph .label {
   font-weight: normal;
   text-shadow: none;
+  color: #333;
 }
 
 #dag-viz-graph svg path {
diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index ffc9acdce928f..25198db6fb8f2 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -18,7 +18,7 @@
 #plan-viz-graph .label {
   font-weight: normal;
   text-shadow: none;
-  color: black;
+  color: #333;
 }
 
 #plan-viz-graph svg g.cluster rect {

From 442117812ca6edc6e0ab271da829032b9637e89e Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Sun, 26 Apr 2020 21:04:14 -0700
Subject: [PATCH 0570/1280] [SPARK-31497][ML][PYSPARK] Fix Pyspark
 CrossValidator/TrainValidationSplit with pipeline estimator cannot save and
 load model

### What changes were proposed in this pull request?
Fix Pyspark CrossValidator/TrainValidationSplit with pipeline estimator cannot save and load model.

Most pyspark estimators/transformers inherit `JavaParams`, but some estimators are special (in order to support pure python implemented nested estimators/transformers):
* Pipeline
* OneVsRest
* CrossValidator
* TrainValidationSplit

But note that, currently, in pyspark, estimators listed above, their model reader/writer do NOT support pure python implemented nested estimators/transformers. Because they use java reader/writer wrapper as python side reader/writer.

Pyspark CrossValidator/TrainValidationSplit model reader/writer require all estimators define the `_transfer_param_map_to_java` and `_transfer_param_map_from_java` (used in model read/write).

OneVsRest class already defines the two methods, but Pipeline do not, so it lead to this bug.

In this PR I add `_transfer_param_map_to_java` and `_transfer_param_map_from_java` into Pipeline class.

### Why are the changes needed?
Bug fix.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Unit test.

Manually test in pyspark shell:
1) CrossValidator with Simple Pipeline estimator
```
from pyspark.ml import Pipeline
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.feature import HashingTF, Tokenizer
from pyspark.ml.tuning import CrossValidator, CrossValidatorModel, ParamGridBuilder

training = spark.createDataFrame([
    (0, "a b c d e spark", 1.0),
    (1, "b d", 0.0),
    (2, "spark f g h", 1.0),
    (3, "hadoop mapreduce", 0.0),
    (4, "b spark who", 1.0),
    (5, "g d a y", 0.0),
    (6, "spark fly", 1.0),
    (7, "was mapreduce", 0.0),
], ["id", "text", "label"])

# Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
tokenizer = Tokenizer(inputCol="text", outputCol="words")
hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
lr = LogisticRegression(maxIter=10)
pipeline = Pipeline(stages=[tokenizer, hashingTF, lr])

paramGrid = ParamGridBuilder() \
    .addGrid(hashingTF.numFeatures, [10, 100, 1000]) \
    .addGrid(lr.regParam, [0.1, 0.01]) \
    .build()
crossval = CrossValidator(estimator=pipeline,
                          estimatorParamMaps=paramGrid,
                          evaluator=BinaryClassificationEvaluator(),
                          numFolds=2)  # use 3+ folds in practice

# Run cross-validation, and choose the best set of parameters.
cvModel = crossval.fit(training)

cvModel.save('/tmp/cv_model001')
CrossValidatorModel.load('/tmp/cv_model001')
```

2) CrossValidator with Pipeline estimator which include a OneVsRest estimator stage, and OneVsRest estimator nest a LogisticRegression estimator.

```
from pyspark.ml.linalg import Vectors
from pyspark.ml import Estimator, Model
from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel, OneVsRest
from pyspark.ml.evaluation import BinaryClassificationEvaluator, \
    MulticlassClassificationEvaluator, RegressionEvaluator
from pyspark.ml.linalg import Vectors
from pyspark.ml.param import Param, Params
from pyspark.ml.tuning import CrossValidator, CrossValidatorModel, ParamGridBuilder, \
    TrainValidationSplit, TrainValidationSplitModel
from pyspark.sql.functions import rand
from pyspark.testing.mlutils import SparkSessionTestCase

dataset = spark.createDataFrame(
    [(Vectors.dense([0.0]), 0.0),
     (Vectors.dense([0.4]), 1.0),
     (Vectors.dense([0.5]), 0.0),
     (Vectors.dense([0.6]), 1.0),
     (Vectors.dense([1.0]), 1.0)] * 10,
    ["features", "label"])

ova = OneVsRest(classifier=LogisticRegression())
lr1 = LogisticRegression().setMaxIter(100)
lr2 = LogisticRegression().setMaxIter(150)
grid = ParamGridBuilder().addGrid(ova.classifier, [lr1, lr2]).build()
evaluator = MulticlassClassificationEvaluator()

pipeline = Pipeline(stages=[ova])

cv = CrossValidator(estimator=pipeline, estimatorParamMaps=grid, evaluator=evaluator)
cvModel = cv.fit(dataset)
cvModel.save('/tmp/model002')

cvModel2 = CrossValidatorModel.load('/tmp/model002')
```

TrainValidationSplit testing code are similar so I do not paste them.

Closes #28279 from WeichenXu123/fix_pipeline_tuning.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: Xiangrui Meng <meng@databricks.com>
(cherry picked from commit 4a21c4cc92805b034ade0593eea3c4a9b6122083)
Signed-off-by: Xiangrui Meng <meng@databricks.com>
---
 python/pyspark/ml/pipeline.py          |  53 +++++++++-
 python/pyspark/ml/tests/test_tuning.py | 139 ++++++++++++++++++++++++-
 2 files changed, 189 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py
index 09e0748ffbb3b..53d07ec9660d9 100644
--- a/python/pyspark/ml/pipeline.py
+++ b/python/pyspark/ml/pipeline.py
@@ -25,8 +25,8 @@
 from pyspark.ml.base import Estimator, Model, Transformer
 from pyspark.ml.param import Param, Params
 from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaParams
-from pyspark.ml.common import inherit_doc
+from pyspark.ml.wrapper import JavaParams, JavaWrapper
+from pyspark.ml.common import inherit_doc, _java2py, _py2java
 
 
 @inherit_doc
@@ -174,6 +174,55 @@ def _to_java(self):
 
         return _java_obj
 
+    def _make_java_param_pair(self, param, value):
+        """
+        Makes a Java param pair.
+        """
+        sc = SparkContext._active_spark_context
+        param = self._resolveParam(param)
+        java_param = sc._jvm.org.apache.spark.ml.param.Param(param.parent, param.name, param.doc)
+        if isinstance(value, Params) and hasattr(value, "_to_java"):
+            # Convert JavaEstimator/JavaTransformer object or Estimator/Transformer object which
+            # implements `_to_java` method (such as OneVsRest, Pipeline object) to java object.
+            # used in the case of an estimator having another estimator as a parameter
+            # the reason why this is not in _py2java in common.py is that importing
+            # Estimator and Model in common.py results in a circular import with inherit_doc
+            java_value = value._to_java()
+        else:
+            java_value = _py2java(sc, value)
+        return java_param.w(java_value)
+
+    def _transfer_param_map_to_java(self, pyParamMap):
+        """
+        Transforms a Python ParamMap into a Java ParamMap.
+        """
+        paramMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
+        for param in self.params:
+            if param in pyParamMap:
+                pair = self._make_java_param_pair(param, pyParamMap[param])
+                paramMap.put([pair])
+        return paramMap
+
+    def _transfer_param_map_from_java(self, javaParamMap):
+        """
+        Transforms a Java ParamMap into a Python ParamMap.
+        """
+        sc = SparkContext._active_spark_context
+        paramMap = dict()
+        for pair in javaParamMap.toList():
+            param = pair.param()
+            if self.hasParam(str(param.name())):
+                java_obj = pair.value()
+                if sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage").isInstance(java_obj):
+                    # Note: JavaParams._from_java support both JavaEstimator/JavaTransformer class
+                    # and Estimator/Transformer class which implements `_from_java` static method
+                    # (such as OneVsRest, Pipeline class).
+                    py_obj = JavaParams._from_java(java_obj)
+                else:
+                    py_obj = _java2py(sc, java_obj)
+                paramMap[self.getParam(param.name())] = py_obj
+        return paramMap
+
 
 @inherit_doc
 class PipelineWriter(MLWriter):
diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index 9d8ba37c60da4..6bcc3f93e1b3b 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -18,7 +18,8 @@
 import tempfile
 import unittest
 
-from pyspark.ml import Estimator, Model
+from pyspark.ml.feature import HashingTF, Tokenizer
+from pyspark.ml import Estimator, Pipeline, Model
 from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel, OneVsRest
 from pyspark.ml.evaluation import BinaryClassificationEvaluator, \
     MulticlassClassificationEvaluator, RegressionEvaluator
@@ -310,6 +311,75 @@ def test_save_load_nested_estimator(self):
         loadedModel = CrossValidatorModel.load(cvModelPath)
         self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
 
+    def test_save_load_pipeline_estimator(self):
+        temp_path = tempfile.mkdtemp()
+        training = self.spark.createDataFrame([
+            (0, "a b c d e spark", 1.0),
+            (1, "b d", 0.0),
+            (2, "spark f g h", 1.0),
+            (3, "hadoop mapreduce", 0.0),
+            (4, "b spark who", 1.0),
+            (5, "g d a y", 0.0),
+            (6, "spark fly", 1.0),
+            (7, "was mapreduce", 0.0),
+        ], ["id", "text", "label"])
+
+        # Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
+        tokenizer = Tokenizer(inputCol="text", outputCol="words")
+        hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
+
+        ova = OneVsRest(classifier=LogisticRegression())
+        lr1 = LogisticRegression().setMaxIter(5)
+        lr2 = LogisticRegression().setMaxIter(10)
+
+        pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
+
+        paramGrid = ParamGridBuilder() \
+            .addGrid(hashingTF.numFeatures, [10, 100]) \
+            .addGrid(ova.classifier, [lr1, lr2]) \
+            .build()
+
+        crossval = CrossValidator(estimator=pipeline,
+                                  estimatorParamMaps=paramGrid,
+                                  evaluator=MulticlassClassificationEvaluator(),
+                                  numFolds=2)  # use 3+ folds in practice
+
+        # Run cross-validation, and choose the best set of parameters.
+        cvModel = crossval.fit(training)
+
+        # test save/load of CrossValidatorModel
+        cvModelPath = temp_path + "/cvModel"
+        cvModel.save(cvModelPath)
+        loadedModel = CrossValidatorModel.load(cvModelPath)
+        self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
+        self.assertEqual(len(loadedModel.bestModel.stages), len(cvModel.bestModel.stages))
+        for loadedStage, originalStage in zip(loadedModel.bestModel.stages,
+                                              cvModel.bestModel.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
+        # Test nested pipeline
+        nested_pipeline = Pipeline(stages=[tokenizer, Pipeline(stages=[hashingTF, ova])])
+        crossval2 = CrossValidator(estimator=nested_pipeline,
+                                   estimatorParamMaps=paramGrid,
+                                   evaluator=MulticlassClassificationEvaluator(),
+                                   numFolds=2)  # use 3+ folds in practice
+
+        # Run cross-validation, and choose the best set of parameters.
+        cvModel2 = crossval2.fit(training)
+        # test save/load of CrossValidatorModel
+        cvModelPath2 = temp_path + "/cvModel2"
+        cvModel2.save(cvModelPath2)
+        loadedModel2 = CrossValidatorModel.load(cvModelPath2)
+        self.assertEqual(loadedModel2.bestModel.uid, cvModel2.bestModel.uid)
+        loaded_nested_pipeline_model = loadedModel2.bestModel.stages[1]
+        original_nested_pipeline_model = cvModel2.bestModel.stages[1]
+        self.assertEqual(loaded_nested_pipeline_model.uid, original_nested_pipeline_model.uid)
+        self.assertEqual(len(loaded_nested_pipeline_model.stages),
+                         len(original_nested_pipeline_model.stages))
+        for loadedStage, originalStage in zip(loaded_nested_pipeline_model.stages,
+                                              original_nested_pipeline_model.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
 
 class TrainValidationSplitTests(SparkSessionTestCase):
 
@@ -511,6 +581,73 @@ def test_save_load_nested_estimator(self):
         loadedModel = TrainValidationSplitModel.load(tvsModelPath)
         self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
 
+    def test_save_load_pipeline_estimator(self):
+        temp_path = tempfile.mkdtemp()
+        training = self.spark.createDataFrame([
+            (0, "a b c d e spark", 1.0),
+            (1, "b d", 0.0),
+            (2, "spark f g h", 1.0),
+            (3, "hadoop mapreduce", 0.0),
+            (4, "b spark who", 1.0),
+            (5, "g d a y", 0.0),
+            (6, "spark fly", 1.0),
+            (7, "was mapreduce", 0.0),
+        ], ["id", "text", "label"])
+
+        # Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
+        tokenizer = Tokenizer(inputCol="text", outputCol="words")
+        hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
+
+        ova = OneVsRest(classifier=LogisticRegression())
+        lr1 = LogisticRegression().setMaxIter(5)
+        lr2 = LogisticRegression().setMaxIter(10)
+
+        pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
+
+        paramGrid = ParamGridBuilder() \
+            .addGrid(hashingTF.numFeatures, [10, 100]) \
+            .addGrid(ova.classifier, [lr1, lr2]) \
+            .build()
+
+        tvs = TrainValidationSplit(estimator=pipeline,
+                                   estimatorParamMaps=paramGrid,
+                                   evaluator=MulticlassClassificationEvaluator())
+
+        # Run train validation split, and choose the best set of parameters.
+        tvsModel = tvs.fit(training)
+
+        # test save/load of CrossValidatorModel
+        tvsModelPath = temp_path + "/tvsModel"
+        tvsModel.save(tvsModelPath)
+        loadedModel = TrainValidationSplitModel.load(tvsModelPath)
+        self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
+        self.assertEqual(len(loadedModel.bestModel.stages), len(tvsModel.bestModel.stages))
+        for loadedStage, originalStage in zip(loadedModel.bestModel.stages,
+                                              tvsModel.bestModel.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
+        # Test nested pipeline
+        nested_pipeline = Pipeline(stages=[tokenizer, Pipeline(stages=[hashingTF, ova])])
+        tvs2 = TrainValidationSplit(estimator=nested_pipeline,
+                                    estimatorParamMaps=paramGrid,
+                                    evaluator=MulticlassClassificationEvaluator())
+
+        # Run train validation split, and choose the best set of parameters.
+        tvsModel2 = tvs2.fit(training)
+        # test save/load of CrossValidatorModel
+        tvsModelPath2 = temp_path + "/tvsModel2"
+        tvsModel2.save(tvsModelPath2)
+        loadedModel2 = TrainValidationSplitModel.load(tvsModelPath2)
+        self.assertEqual(loadedModel2.bestModel.uid, tvsModel2.bestModel.uid)
+        loaded_nested_pipeline_model = loadedModel2.bestModel.stages[1]
+        original_nested_pipeline_model = tvsModel2.bestModel.stages[1]
+        self.assertEqual(loaded_nested_pipeline_model.uid, original_nested_pipeline_model.uid)
+        self.assertEqual(len(loaded_nested_pipeline_model.stages),
+                         len(original_nested_pipeline_model.stages))
+        for loadedStage, originalStage in zip(loaded_nested_pipeline_model.stages,
+                                              original_nested_pipeline_model.stages):
+            self.assertEqual(loadedStage.uid, originalStage.uid)
+
     def test_copy(self):
         dataset = self.spark.createDataFrame([
             (10, 10.0),

From a48872a0ac0d7fbacea06f9a85e9a630597ee544 Mon Sep 17 00:00:00 2001
From: Juliusz Sompolski <julek@databricks.com>
Date: Mon, 27 Apr 2020 04:27:55 +0000
Subject: [PATCH 0571/1280] [SPARK-31388][SQL][TESTS]
 org.apache.spark.sql.hive.thriftserver.CliSuite doesn't match results
 correctly

### What changes were proposed in this pull request?

`CliSuite.runCliWithin` was not matching for expected results correctly. It was matching for expected lines anywhere in stdout or stderr.
On the example of `Single command with --database` test:
In
```
    runCliWithin(2.minute)(
      "CREATE DATABASE hive_db_test;"
        -> "",
      "USE hive_test;"
        -> "",
      "CREATE TABLE hive_test(key INT, val STRING);"
        -> "",
      "SHOW TABLES;"
        -> "hive_test"
    )
```
It was looking for lines containing "", "", "" and then "hive_test".
However, the string "hive_test" was contained in "hive_test_db", and hence:
```
2020-04-08 17:53:12,752 INFO  CliSuite - 2020-04-08 17:53:12.752 - stderr> Spark master: local, Application Id: local-1586368384172
2020-04-08 17:53:12,765 INFO  CliSuite - stderr> found expected output line 0: ""
2020-04-08 17:53:12,765 INFO  CliSuite - 2020-04-08 17:53:12.765 - stdout> spark-sql> CREATE DATABASE hive_db_test;
2020-04-08 17:53:12,765 INFO  CliSuite - stdout> found expected output line 1: ""
2020-04-08 17:53:17,688 INFO  CliSuite - 2020-04-08 17:53:17.688 - stderr> chgrp: changing ownership of 'file:///tmp/spark-8811f069-4cba-4c71-a5d6-62dd925fb5ff': chown: changing group of '/tmp/spark-8811f069-4cba-4c71-a5d6-62dd925fb5ff': Operation not permitted
2020-04-08 17:53:12,765 INFO  CliSuite - stderr> found expected output line 2: ""
2020-04-08 17:53:18,069 INFO  CliSuite - 2020-04-08 17:53:18.069 - stderr> Time taken: 5.265 seconds
2020-04-08 17:53:18,087 INFO  CliSuite - 2020-04-08 17:53:18.087 - stdout> spark-sql> USE hive_test;
2020-04-08 17:53:12,765 INFO  CliSuite - stdout> found expected output line 3: "hive_test"
2020-04-08 17:53:21,742 INFO  CliSuite - Found all expected output.
```
Because of that, it could kill the CLI process without really even creating the table. This was not expected. The test could be flaky depending on whether process.destroy() in the finally block managed to kill it before it actually creates the table.

I make the output checking more robust to not match on unexpected output, by making it check the echo of query output on the CLI. Also, wait for the CLI process to finish gracefully (triggered by closing its stdin), instead of killing it forcibly.

### Why are the changes needed?

org.apache.spark.sql.hive.thriftserver.CliSuite was flaky, and didn't test outputs as expected.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Existing tests in CLISuite. Tested several times with no flakiness. Was getting flaky results almost on every run before.
```
[info] CliSuite:
[info] - load warehouse dir from hive-site.xml (12 seconds, 568 milliseconds)
[info] - load warehouse dir from --hiveconf (10 seconds, 648 milliseconds)
[info] - load warehouse dir from --conf spark(.hadoop).hive.* (20 seconds, 653 milliseconds)
[info] - load warehouse dir from spark.sql.warehouse.dir (9 seconds, 763 milliseconds)
[info] - Simple commands (16 seconds, 238 milliseconds)
[info] - Single command with -e (9 seconds, 967 milliseconds)
[info] - Single command with --database (21 seconds, 205 milliseconds)
[info] - Commands using SerDe provided in --jars (15 seconds, 51 milliseconds)
[info] - SPARK-29022: Commands using SerDe provided in --hive.aux.jars.path (14 seconds, 625 milliseconds)
[info] - SPARK-11188 Analysis error reporting (7 seconds, 960 milliseconds)
[info] - SPARK-11624 Spark SQL CLI should set sessionState only once (7 seconds, 424 milliseconds)
[info] - list jars (9 seconds, 520 milliseconds)
[info] - list jar <jarfile> (9 seconds, 277 milliseconds)
[info] - list files (9 seconds, 828 milliseconds)
[info] - list file <filepath> (9 seconds, 646 milliseconds)
[info] - apply hiveconf from cli command (9 seconds, 469 milliseconds)
[info] - Support hive.aux.jars.path (10 seconds, 676 milliseconds)
[info] - SPARK-28840 test --jars command (10 seconds, 921 milliseconds)
[info] - SPARK-28840 test --jars and hive.aux.jars.path command (11 seconds, 49 milliseconds)
[info] - SPARK-29022 Commands using SerDe provided in ADD JAR sql (14 seconds, 210 milliseconds)
[info] - SPARK-26321 Should not split semicolon within quoted string literals (12 seconds, 729 milliseconds)
[info] - Pad Decimal numbers with trailing zeros to the scale of the column (10 seconds, 381 milliseconds)
[info] - SPARK-30049 Should not complain for quotes in commented lines (10 seconds, 935 milliseconds)
[info] - SPARK-30049 Should not complain for quotes in commented with multi-lines (20 seconds, 731 milliseconds)
```

Closes #28156 from juliuszsompolski/SPARK-31388.

Authored-by: Juliusz Sompolski <julek@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 560bd5401f65b7813b0ac4856cf9934d2ed24594)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/hive/thriftserver/CliSuite.scala      | 70 +++++++++++++------
 1 file changed, 48 insertions(+), 22 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index c393054051fc4..abefb46b32140 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -87,9 +87,25 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       useExternalHiveFile: Boolean = false)(
       queriesAndExpectedAnswers: (String, String)*): Unit = {
 
-    val (queries, expectedAnswers) = queriesAndExpectedAnswers.unzip
     // Explicitly adds ENTER for each statement to make sure they are actually entered into the CLI.
-    val queriesString = queries.map(_ + "\n").mkString
+    val queriesString = queriesAndExpectedAnswers.map(_._1 + "\n").mkString
+    // spark-sql echoes the queries on STDOUT, expect first an echo of the query, then the answer.
+    val expectedAnswers = queriesAndExpectedAnswers.flatMap {
+      case (query, answer) =>
+        if (query == "") {
+          // empty query means a command launched with -e
+          Seq(answer)
+        } else {
+          // spark-sql echoes the submitted queries
+          val queryEcho = query.split("\n").toList match {
+            case firstLine :: tail =>
+              s"spark-sql> $firstLine" :: tail.map(l => s"         > $l")
+          }
+          // longer lines sometimes get split in the output,
+          // match the first 60 characters of each query line
+          queryEcho.map(_.take(60)) :+ answer
+        }
+    }
 
     val extraHive = if (useExternalHiveFile) {
       s"--driver-class-path ${System.getProperty("user.dir")}/src/test/noclasspath"
@@ -122,10 +138,13 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
     def captureOutput(source: String)(line: String): Unit = lock.synchronized {
       // This test suite sometimes gets extremely slow out of unknown reason on Jenkins.  Here we
       // add a timestamp to provide more diagnosis information.
-      buffer += s"${new Timestamp(new Date().getTime)} - $source> $line"
+      val newLine = s"${new Timestamp(new Date().getTime)} - $source> $line"
+      log.info(newLine)
+      buffer += newLine
 
       // If we haven't found all expected answers and another expected answer comes up...
       if (next < expectedAnswers.size && line.contains(expectedAnswers(next))) {
+        log.info(s"$source> found expected output line $next: '${expectedAnswers(next)}'")
         next += 1
         // If all expected answers have been found...
         if (next == expectedAnswers.size) {
@@ -153,6 +172,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
 
     try {
       ThreadUtils.awaitResult(foundAllExpectedAnswers.future, timeout)
+      log.info("Found all expected output.")
     } catch { case cause: Throwable =>
       val message =
         s"""
@@ -161,8 +181,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
            |=======================
            |Spark SQL CLI command line: ${command.mkString(" ")}
            |Exception: $cause
-           |Executed query $next "${queries(next)}",
-           |But failed to capture expected output "${expectedAnswers(next)}" within $timeout.
+           |Failed to capture next expected output "${expectedAnswers(next)}" within $timeout.
            |
            |${buffer.mkString("\n")}
            |===========================
@@ -172,7 +191,13 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       logError(message, cause)
       fail(message, cause)
     } finally {
-      process.destroy()
+      if (!process.waitFor(1, MINUTES)) {
+        try {
+          fail("spark-sql did not exit gracefully.")
+        } finally {
+          process.destroy()
+        }
+      }
     }
   }
 
@@ -233,7 +258,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
         -> "",
       "SHOW TABLES;"
         -> "hive_test",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE hive_test;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE hive_test;""".stripMargin
         -> "",
       "CACHE TABLE hive_test;"
         -> "",
@@ -250,18 +276,18 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
 
   test("Single command with --database") {
     runCliWithin(2.minute)(
-      "CREATE DATABASE hive_test_db;"
+      "CREATE DATABASE hive_db_test;"
         -> "",
-      "USE hive_test_db;"
+      "USE hive_db_test;"
         -> "",
-      "CREATE TABLE hive_test(key INT, val STRING);"
+      "CREATE TABLE hive_table_test(key INT, val STRING);"
         -> "",
       "SHOW TABLES;"
-        -> "hive_test"
+        -> "hive_table_test"
     )
 
-    runCliWithin(2.minute, Seq("--database", "hive_test_db", "-e", "SHOW TABLES;"))(
-      "" -> "hive_test"
+    runCliWithin(2.minute, Seq("--database", "hive_db_test", "-e", "SHOW TABLES;"))(
+      "" -> "hive_table_test"
     )
   }
 
@@ -273,12 +299,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
 
     runCliWithin(3.minute, Seq("--jars", s"$jarFile"))(
       """CREATE TABLE t1(key string, val string)
-        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
-      """.stripMargin
+        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';""".stripMargin
         -> "",
       "CREATE TABLE sourceTable (key INT, val STRING) USING hive;"
         -> "",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE sourceTable;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE sourceTable;""".stripMargin
         -> "",
       "INSERT INTO TABLE t1 SELECT key, val FROM sourceTable;"
         -> "",
@@ -299,12 +325,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       3.minute,
       Seq("--conf", s"spark.hadoop.${ConfVars.HIVEAUXJARS}=$hiveContribJar"))(
       """CREATE TABLE addJarWithHiveAux(key string, val string)
-        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
-      """.stripMargin
+        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';""".stripMargin
         -> "",
       "CREATE TABLE sourceTableForWithHiveAux (key INT, val STRING) USING hive;"
         -> "",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE sourceTableForWithHiveAux;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE sourceTableForWithHiveAux;""".stripMargin
         -> "",
       "INSERT INTO TABLE addJarWithHiveAux SELECT key, val FROM sourceTableForWithHiveAux;"
         -> "",
@@ -419,12 +445,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       3.minute)(
       s"ADD JAR ${hiveContribJar};" -> "",
       """CREATE TABLE addJarWithSQL(key string, val string)
-        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';
-      """.stripMargin
+        |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe';""".stripMargin
         -> "",
       "CREATE TABLE sourceTableForWithSQL(key INT, val STRING) USING hive;"
         -> "",
-      s"LOAD DATA LOCAL INPATH '$dataFilePath' OVERWRITE INTO TABLE sourceTableForWithSQL;"
+      s"""LOAD DATA LOCAL INPATH '$dataFilePath'
+         |OVERWRITE INTO TABLE sourceTableForWithSQL;""".stripMargin
         -> "",
       "INSERT INTO TABLE addJarWithSQL SELECT key, val FROM sourceTableForWithSQL;"
         -> "",

From b0155e36c3f267743c65a259e2be16324714de4a Mon Sep 17 00:00:00 2001
From: Bruce Robbins <bersprockets@gmail.com>
Date: Mon, 27 Apr 2020 05:00:36 +0000
Subject: [PATCH 0572/1280] [SPARK-31557][SQL] Legacy time parser should return
 Gregorian days rather than Julian days

This PR modifies LegacyDateFormatter#parse to return proleptic Gregorian days rather than hybrid Julian days.

The legacy time parser currently returns epoch days in the hybrid Julian calendar. However, the callers to the legacy parser (e.g., UnivocityParser, JacksonParser) expect epoch days in the proleptic Gregorian calendar. As a result, pre-Gregorian dates like '1000-01-01' get interpreted as '1000-01-06'.

No

Manual testing and modified existing unit tests.

Closes #28345 from bersprockets/SPARK-31557.

Authored-by: Bruce Robbins <bersprockets@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a911287244a98aa9e6464bcdd97c80e7ad732788)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateFormatter.scala     |  3 +-
 .../spark/sql/util/DateFormatterSuite.scala   | 76 +++++++++++--------
 2 files changed, 44 insertions(+), 35 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index d2e4e8b367a0a..0f79c1a6a751d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -66,8 +66,7 @@ trait LegacyDateFormatter extends DateFormatter {
   def formatDate(d: Date): String
 
   override def parse(s: String): Int = {
-    val milliseconds = parseToDate(s).getTime
-    DateTimeUtils.millisToDays(milliseconds)
+    fromJavaDate(new java.sql.Date(parseToDate(s).getTime))
   }
 
   override def format(days: Int): String = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index a40dbcc2ec664..2df1d490b70bb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
 class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   test("parsing dates") {
@@ -47,45 +48,54 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("roundtrip date -> days -> date") {
-    Seq(
-      "0050-01-01",
-      "0953-02-02",
-      "1423-03-08",
-      "1969-12-31",
-      "1972-08-25",
-      "1975-09-26",
-      "2018-12-12",
-      "2038-01-01",
-      "5010-11-17").foreach { date =>
-      DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-          val formatter = DateFormatter(getZoneId(timeZone))
-          val days = formatter.parse(date)
-          val formatted = formatter.format(days)
-          assert(date === formatted)
+    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
+        Seq(
+          "0050-01-01",
+          "0953-02-02",
+          "1423-03-08",
+          "1582-10-15",
+          "1969-12-31",
+          "1972-08-25",
+          "1975-09-26",
+          "2018-12-12",
+          "2038-01-01",
+          "5010-11-17").foreach { date =>
+          DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+            withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+              val formatter = DateFormatter(getZoneId(timeZone))
+              val days = formatter.parse(date)
+              val formatted = formatter.format(days)
+              assert(date === formatted)
+            }
+          }
         }
       }
     }
   }
 
   test("roundtrip days -> date -> days") {
-    Seq(
-      -701265,
-      -371419,
-      -199722,
-      -1,
-      0,
-      967,
-      2094,
-      17877,
-      24837,
-      1110657).foreach { days =>
-      DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-          val formatter = DateFormatter(getZoneId(timeZone))
-          val date = formatter.format(days)
-          val parsed = formatter.parse(date)
-          assert(days === parsed)
+    LegacyBehaviorPolicy.values.foreach { parserPolicy =>
+      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
+        Seq(
+          -701265,
+          -371419,
+          -199722,
+          -1,
+          0,
+          967,
+          2094,
+          17877,
+          24837,
+          1110657).foreach { days =>
+          DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+            withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+              val formatter = DateFormatter(getZoneId(timeZone))
+              val date = formatter.format(days)
+              val parsed = formatter.parse(date)
+              assert(days === parsed)
+            }
+          }
         }
       }
     }

From fcfac8f672ba6c8e49350f2c5fdd957584255327 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Mon, 27 Apr 2020 05:28:46 +0000
Subject: [PATCH 0573/1280] [SPARK-31527][SQL] date add/subtract interval only
 allow those day precision in ansi mode
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

To follow ANSI，the expressions - `date + interval`, `interval + date` and `date - interval` should only accept intervals which the `microseconds` part is 0.

Better ANSI compliance

No, this PR should target 3.0.0 in which this feature is newly added.

add more unit tests

Closes #28310 from yaooqinn/SPARK-31527.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ebc8fa50d0422f3db47b2c45025c7f2efe6ee39a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  27 +-
 .../expressions/datetimeExpressions.scala     |  63 ++
 .../sql/catalyst/util/DateTimeUtils.scala     |  16 +
 .../expressions/DateExpressionsSuite.scala    |  36 +-
 .../ExpressionSQLBuilderSuite.scala           |   4 +
 .../catalyst/util/DateTimeUtilsSuite.scala    |  10 +-
 .../sql-tests/inputs/ansi/datetime.sql        |   1 +
 .../sql-tests/results/ansi/datetime.sql.out   | 732 ++++++++++++++++++
 .../sql-tests/results/ansi/interval.sql.out   |   7 +-
 .../sql-tests/results/datetime.sql.out        |   4 +-
 .../sql-tests/results/interval.sql.out        |   4 +-
 .../native/dateTimeOperations.sql.out         |   6 +-
 12 files changed, 888 insertions(+), 22 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index e4be634d0d2f2..6de3189c2b669 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -246,7 +246,7 @@ class Analyzer(
       ResolveLambdaVariables(conf) ::
       ResolveTimeZone(conf) ::
       ResolveRandomSeed ::
-      ResolveBinaryArithmetic(conf) ::
+      ResolveBinaryArithmetic ::
       TypeCoercion.typeCoercionRules(conf) ++
       extendedResolutionRules : _*),
     Batch("Post-Hoc Resolution", Once, postHocResolutionRules: _*),
@@ -268,17 +268,21 @@ class Analyzer(
   /**
    * For [[Add]]:
    * 1. if both side are interval, stays the same;
-   * 2. else if one side is interval, turns it to [[TimeAdd]];
-   * 3. else if one side is date, turns it to [[DateAdd]] ;
-   * 4. else stays the same.
+   * 2. else if one side is date and the other is interval,
+   *    turns it to [[DateAddInterval]];
+   * 3. else if one side is interval, turns it to [[TimeAdd]];
+   * 4. else if one side is date, turns it to [[DateAdd]] ;
+   * 5. else stays the same.
    *
    * For [[Subtract]]:
    * 1. if both side are interval, stays the same;
-   * 2. else if the right side is an interval, turns it to [[TimeSub]];
-   * 3. else if one side is timestamp, turns it to [[SubtractTimestamps]];
-   * 4. else if the right side is date, turns it to [[DateDiff]]/[[SubtractDates]];
-   * 5. else if the left side is date, turns it to [[DateSub]];
-   * 6. else turns it to stays the same.
+   * 2. else if the left side is date and the right side is interval,
+   *    turns it to [[DateAddInterval(l, -r)]];
+   * 3. else if the right side is an interval, turns it to [[TimeSub]];
+   * 4. else if one side is timestamp, turns it to [[SubtractTimestamps]];
+   * 5. else if the right side is date, turns it to [[DateDiff]]/[[SubtractDates]];
+   * 6. else if the left side is date, turns it to [[DateSub]];
+   * 7. else turns it to stays the same.
    *
    * For [[Multiply]]:
    * 1. If one side is interval, turns it to [[MultiplyInterval]];
@@ -288,12 +292,14 @@ class Analyzer(
    * 1. If the left side is interval, turns it to [[DivideInterval]];
    * 2. otherwise, stays the same.
    */
-  case class ResolveBinaryArithmetic(conf: SQLConf) extends Rule[LogicalPlan] {
+  object ResolveBinaryArithmetic extends Rule[LogicalPlan] {
     override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       case p: LogicalPlan => p.transformExpressionsUp {
         case a @ Add(l, r) if a.childrenResolved => (l.dataType, r.dataType) match {
           case (CalendarIntervalType, CalendarIntervalType) => a
+          case (DateType, CalendarIntervalType) => DateAddInterval(l, r)
           case (_, CalendarIntervalType) => Cast(TimeAdd(l, r), l.dataType)
+          case (CalendarIntervalType, DateType) => DateAddInterval(r, l)
           case (CalendarIntervalType, _) => Cast(TimeAdd(r, l), r.dataType)
           case (DateType, dt) if dt != StringType => DateAdd(l, r)
           case (dt, DateType) if dt != StringType => DateAdd(r, l)
@@ -301,6 +307,7 @@ class Analyzer(
         }
         case s @ Subtract(l, r) if s.childrenResolved => (l.dataType, r.dataType) match {
           case (CalendarIntervalType, CalendarIntervalType) => s
+          case (DateType, CalendarIntervalType) => DateAddInterval(l, UnaryMinus(r))
           case (_, CalendarIntervalType) => Cast(TimeSub(l, r), l.dataType)
           case (TimestampType, _) => SubtractTimestamps(l, r)
           case (_, TimestampType) => SubtractTimestamps(l, r)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 8f57421044b2b..6e95f648b64f4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -35,6 +35,7 @@ import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, Tim
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.catalyst.util.LegacyDateFormats.SIMPLE_DATE_FORMAT
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
@@ -1195,6 +1196,68 @@ case class TimeAdd(start: Expression, interval: Expression, timeZoneId: Option[S
   }
 }
 
+/**
+ * Adds date and an interval.
+ *
+ * When ansi mode is on, the microseconds part of interval needs to be 0, otherwise a runtime
+ * [[IllegalArgumentException]] will be raised.
+ * When ansi mode is off, if the microseconds part of interval is 0, we perform date + interval
+ * for better performance. if the microseconds part is not 0, then the date will be converted to a
+ * timestamp to add with the whole interval parts.
+ */
+case class DateAddInterval(
+    start: Expression,
+    interval: Expression,
+    timeZoneId: Option[String] = None,
+    ansiEnabled: Boolean = SQLConf.get.ansiEnabled)
+  extends BinaryExpression with ExpectsInputTypes with TimeZoneAwareExpression {
+
+  override def left: Expression = start
+  override def right: Expression = interval
+
+  override def toString: String = s"$left + $right"
+  override def sql: String = s"${left.sql} + ${right.sql}"
+  override def inputTypes: Seq[AbstractDataType] = Seq(DateType, CalendarIntervalType)
+
+  override def dataType: DataType = DateType
+
+  override def nullSafeEval(start: Any, interval: Any): Any = {
+    val itvl = interval.asInstanceOf[CalendarInterval]
+    if (ansiEnabled || itvl.microseconds == 0) {
+      DateTimeUtils.dateAddInterval(start.asInstanceOf[Int], itvl)
+    } else {
+      val startTs = DateTimeUtils.epochDaysToMicros(start.asInstanceOf[Int], zoneId)
+      val resultTs = DateTimeUtils.timestampAddInterval(
+        startTs, itvl.months, itvl.days, itvl.microseconds, zoneId)
+      DateTimeUtils.microsToEpochDays(resultTs, zoneId)
+    }
+  }
+
+  override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+    val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+    nullSafeCodeGen(ctx, ev, (sd, i) => if (ansiEnabled) {
+      s"""${ev.value} = $dtu.dateAddInterval($sd, $i);"""
+    } else {
+      val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
+      val startTs = ctx.freshName("startTs")
+      val resultTs = ctx.freshName("resultTs")
+      s"""
+         |if ($i.microseconds == 0) {
+         |  ${ev.value} = $dtu.dateAddInterval($sd, $i);
+         |} else {
+         |  long $startTs = $dtu.epochDaysToMicros($sd, $zid);
+         |  long $resultTs =
+         |    $dtu.timestampAddInterval($startTs, $i.months, $i.days, $i.microseconds, $zid);
+         |  ${ev.value} = $dtu.microsToEpochDays($resultTs, $zid);
+         |}
+         |""".stripMargin
+    })
+  }
+
+  override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
+    copy(timeZoneId = Option(timeZoneId))
+}
+
 /**
  * This is a common function for databases supporting TIMESTAMP WITHOUT TIMEZONE. This function
  * takes a timestamp which is timezone-agnostic, and interprets it as a timestamp in UTC, and
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index a2fa87a4a2360..fb7875dc24e70 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -618,6 +618,22 @@ object DateTimeUtils {
     instantToMicros(resultTimestamp.toInstant)
   }
 
+  /**
+   * Add the date and the interval's months and days.
+   * Returns a date value, expressed in days since 1.1.1970.
+   *
+   * @throws DateTimeException if the result exceeds the supported date range
+   * @throws IllegalArgumentException if the interval has `microseconds` part
+   */
+  def dateAddInterval(
+     start: SQLDate,
+     interval: CalendarInterval): SQLDate = {
+    require(interval.microseconds == 0,
+      "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date")
+    val ld = LocalDate.ofEpochDay(start).plusMonths(interval.months).plusDays(interval.days)
+    localDateToDays(ld)
+  }
+
   /**
    * Returns number of months between time1 and time2. time1 and time2 are expressed in
    * microseconds since 1.1.1970. If time1 is later than time2, the result is positive.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 1d01f4b36dc86..dd5a532834160 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -27,7 +27,7 @@ import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
 import org.apache.spark.sql.catalyst.util.{DateTimeUtils, IntervalUtils, TimestampFormatter}
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND
+import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
@@ -359,6 +359,40 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkConsistencyBetweenInterpretedAndCodegen(DateAdd, DateType, IntegerType)
   }
 
+  test("date add interval") {
+    val d = Date.valueOf("2016-02-28")
+    Seq("true", "false") foreach { flag =>
+      withSQLConf((SQLConf.ANSI_ENABLED.key, flag)) {
+        checkEvaluation(
+          DateAddInterval(Literal(d), Literal(new CalendarInterval(0, 1, 0))),
+          DateTimeUtils.fromJavaDate(Date.valueOf("2016-02-29")))
+        checkEvaluation(
+          DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 0))),
+          DateTimeUtils.fromJavaDate(Date.valueOf("2016-03-29")))
+        checkEvaluation(DateAddInterval(Literal(d), Literal.create(null, CalendarIntervalType)),
+          null)
+        checkEvaluation(DateAddInterval(Literal.create(null, DateType),
+          Literal(new CalendarInterval(1, 1, 0))),
+          null)
+      }
+    }
+
+    withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) {
+      checkExceptionInExpression[IllegalArgumentException](
+        DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))),
+        "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date")
+    }
+
+    withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) {
+      checkEvaluation(
+        DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25))),
+        DateTimeUtils.fromJavaDate(Date.valueOf("2016-03-29")))
+      checkEvaluation(
+        DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))),
+        DateTimeUtils.fromJavaDate(Date.valueOf("2016-03-30")))
+    }
+  }
+
   test("date_sub") {
     checkEvaluation(
       DateSub(Literal(Date.valueOf("2015-01-01")), Literal(1.toByte)),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
index 492d97ba9d524..ab0ef22bb746f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionSQLBuilderSuite.scala
@@ -176,5 +176,9 @@ class ExpressionSQLBuilderSuite extends SparkFunSuite {
       TimeSub('a, interval),
       "`a` - INTERVAL '1 hours'"
     )
+    checkSQL(
+      DateAddInterval('a, interval),
+      "`a` + INTERVAL '1 hours'"
+    )
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 3981792797325..b547c445f06fe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 
 class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
 
@@ -391,6 +391,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     assert(dateAddMonths(input, -13) === days(1996, 1, 28))
   }
 
+  test("date add interval with day precision") {
+    val input = days(1997, 2, 28, 10, 30)
+    assert(dateAddInterval(input, new CalendarInterval(36, 0, 0)) === days(2000, 2, 28))
+    assert(dateAddInterval(input, new CalendarInterval(36, 47, 0)) === days(2000, 4, 15))
+    assert(dateAddInterval(input, new CalendarInterval(-13, 0, 0)) === days(1996, 1, 28))
+    intercept[IllegalArgumentException](dateAddInterval(input, new CalendarInterval(36, 47, 1)))
+  }
+
   test("timestamp add months") {
     val ts1 = date(1997, 2, 28, 10, 30, 0)
     val ts2 = date(2000, 2, 28, 10, 30, 0, 123000)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql
new file mode 100644
index 0000000000000..58ecf80637883
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/datetime.sql
@@ -0,0 +1 @@
+--IMPORT datetime.sql
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
new file mode 100644
index 0000000000000..81a73a6377715
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -0,0 +1,732 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 85
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'current_date'(line 1, pos 22)
+
+== SQL ==
+select current_date = current_date(), current_timestamp = current_timestamp()
+----------------------^^^
+
+
+-- !query
+select to_date(null), to_date('2016-12-31'), to_date('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_date(NULL):date,to_date('2016-12-31'):date,to_date('2016-12-31', 'yyyy-MM-dd'):date>
+-- !query output
+NULL	2016-12-31	2016-12-31
+
+
+-- !query
+select to_timestamp(null), to_timestamp('2016-12-31 00:12:00'), to_timestamp('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_timestamp(NULL):timestamp,to_timestamp('2016-12-31 00:12:00'):timestamp,to_timestamp('2016-12-31', 'yyyy-MM-dd'):timestamp>
+-- !query output
+NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
+
+
+-- !query
+select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
+-- !query schema
+struct<dayofweek(CAST(2007-02-03 AS DATE)):int,dayofweek(CAST(2009-07-30 AS DATE)):int,dayofweek(CAST(2017-05-27 AS DATE)):int,dayofweek(CAST(NULL AS DATE)):int,dayofweek(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+7	5	7	NULL	6
+
+
+-- !query
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'current_date'(line 4, pos 10)
+
+== SQL ==
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+----------^^^
+
+
+-- !query
+select current_date, current_timestamp from ttf1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Table or view not found: ttf1; line 1 pos 44
+
+
+-- !query
+create temporary view ttf2 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf2(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'current_date'(line 1, pos 22)
+
+== SQL ==
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+----------------------^^^
+
+
+-- !query
+select a, b from ttf2 order by a, current_date
+-- !query schema
+struct<a:int,b:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
+-- !query schema
+struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):int,weekday(CAST(2017-05-27 AS DATE)):int,weekday(CAST(NULL AS DATE)):int,weekday(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+5	3	5	NULL	4
+
+
+-- !query
+select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+no viable alternative at input 'year'(line 1, pos 7)
+
+== SQL ==
+select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
+-------^^^
+
+
+-- !query
+select date '2019-01-01\t'
+-- !query schema
+struct<DATE '2019-01-01':date>
+-- !query output
+2019-01-01
+
+
+-- !query
+select timestamp '2019-01-01\t'
+-- !query schema
+struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
+-- !query output
+2019-01-01 00:00:00
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' + interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-13 11:11:11
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' - interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-09 11:11:11
+
+
+-- !query
+select date'2011-11-11 11:11:11' + interval '2' second
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
+
+
+-- !query
+select date'2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
+
+
+-- !query
+select '2011-11-11' - interval '2' day
+-- !query schema
+struct<CAST(CAST(2011-11-11 AS TIMESTAMP) - INTERVAL '2 days' AS STRING):string>
+-- !query output
+2011-11-09 00:00:00
+
+
+-- !query
+select '2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<CAST(CAST(2011-11-11 11:11:11 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+2011-11-11 11:11:09
+
+
+-- !query
+select '1' - interval '2' second
+-- !query schema
+struct<CAST(CAST(1 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+NULL
+
+
+-- !query
+select 1 - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve '1 - INTERVAL '2 seconds'' due to data type mismatch: argument 1 requires timestamp type, however, '1' is of int type.; line 1 pos 7
+
+
+-- !query
+select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(DATE '2020-01-01' AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+2078 hours 48 minutes 47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
+-- !query output
+-2078 hours -48 minutes -47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - null
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(NULL AS TIMESTAMP)):interval>
+-- !query output
+NULL
+
+
+-- !query
+select null - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(NULL AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date_add('2011-11-11', 1Y)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1S)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1L)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1L)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1L' is of bigint type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1.0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1.0BD)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1E1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', '1')
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_add' function needs to be an integer.;
+
+
+-- !query
+select date_add(date'2011-11-11', 1)
+-- !query schema
+struct<date_add(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_sub(date'2011-11-11', 1)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.;
+
+
+-- !query
+select date_sub(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(null, 1)
+-- !query schema
+struct<date_sub(CAST(NULL AS DATE), 1):date>
+-- !query output
+NULL
+
+
+-- !query
+select date_sub(date'2011-11-11', null)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date'2011-11-11' + 1E1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select null + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' + 7Y
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select 7S + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select date '2001-10-01' - 7
+-- !query schema
+struct<date_sub(DATE '2001-10-01', 7):date>
+-- !query output
+2001-09-24
+
+
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select date '2001-09-28' + null
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' - null
+-- !query schema
+struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select null - date '2019-10-06'
+-- !query schema
+struct<subtractdates(CAST(NULL AS DATE), DATE '2019-10-06'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-10-01' - date '2001-09-28'
+-- !query schema
+struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
+-- !query output
+3 days
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 03:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 08:11:12.12345
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123456
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.123
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+2019-10-06 10:11:00
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06S10:11:12.12345', 'yyyy-MM-dd\'S\'HH:mm:ss.SSSSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12345
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.12342019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+2019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+0019-10-06 10:11:12.1234
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp('2019-10-06S', 'yyyy-MM-dd\'S\''):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: e
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: c
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+-- !query schema
+struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+-- !query output
+2019-10-06 Sunday
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.12
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index d4238c73e39e1..e8201d46924da 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -689,7 +689,7 @@ select
   interval '2-2' year to month + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '2 years 2 months') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date>
+struct<dateval:date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + (- INTERVAL '-2 years -2 months'):date,dateval + INTERVAL '2 years 2 months':date,dateval + INTERVAL '-2 years -2 months':date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + INTERVAL '2 years 2 months':date>
 -- !query output
 2012-01-01	2009-11-01	2014-03-01	2014-03-01	2009-11-01	2009-11-01	2014-03-01
 
@@ -732,9 +732,10 @@ select
   interval '99 11:22:33.123456789' day to second + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date>
+struct<>
 -- !query output
-2012-01-01	2011-09-23	2012-04-09	2012-04-09	2011-09-23	2011-09-23	2012-04-09
+java.lang.IllegalArgumentException
+requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index aec04da4dd54d..2e600850c48b9 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -134,7 +134,7 @@ struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):ti
 -- !query
 select date'2011-11-11 11:11:11' + interval '2' second
 -- !query schema
-struct<CAST(CAST(DATE '2011-11-11' AS TIMESTAMP) + INTERVAL '2 seconds' AS DATE):date>
+struct<DATE '2011-11-11' + INTERVAL '2 seconds':date>
 -- !query output
 2011-11-11
 
@@ -142,7 +142,7 @@ struct<CAST(CAST(DATE '2011-11-11' AS TIMESTAMP) + INTERVAL '2 seconds' AS DATE)
 -- !query
 select date'2011-11-11 11:11:11' - interval '2' second
 -- !query schema
-struct<CAST(CAST(DATE '2011-11-11' AS TIMESTAMP) - INTERVAL '2 seconds' AS DATE):date>
+struct<DATE '2011-11-11' + (- INTERVAL '2 seconds'):date>
 -- !query output
 2011-11-10
 
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 0d758ca1f24ca..84e3fb3c238a4 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -668,7 +668,7 @@ select
   interval '2-2' year to month + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-2 years -2 months' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '2 years 2 months') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '2 years 2 months' AS DATE):date>
+struct<dateval:date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + (- INTERVAL '-2 years -2 months'):date,dateval + INTERVAL '2 years 2 months':date,dateval + INTERVAL '-2 years -2 months':date,dateval + (- INTERVAL '2 years 2 months'):date,dateval + INTERVAL '2 years 2 months':date>
 -- !query output
 2012-01-01	2009-11-01	2014-03-01	2014-03-01	2009-11-01	2009-11-01	2014-03-01
 
@@ -711,7 +711,7 @@ select
   interval '99 11:22:33.123456789' day to second + dateval
 from interval_arithmetic
 -- !query schema
-struct<dateval:date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) - INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds' AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds') AS DATE):date,CAST(CAST(dateval AS TIMESTAMP) + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' AS DATE):date>
+struct<dateval:date,dateval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds'):date,dateval + (- INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds'):date,dateval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date,dateval + INTERVAL '-99 days -11 hours -22 minutes -33.123456 seconds':date,dateval + (- INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds'):date,dateval + INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds':date>
 -- !query output
 2012-01-01	2011-09-23	2012-04-09	2012-04-09	2011-09-23	2011-09-23	2012-04-09
 
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
index d5c27ade8e152..7b1fcddfdac7d 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
@@ -118,7 +118,7 @@ struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days' AS TIME
 -- !query
 select cast('2017-12-11 09:30:00' as date) + interval 2 day
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) + INTERVAL '2 days' AS DATE):date>
+struct<CAST(2017-12-11 09:30:00 AS DATE) + INTERVAL '2 days':date>
 -- !query output
 2017-12-13
 
@@ -231,7 +231,7 @@ struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) + INTERVAL '2 days' AS TIME
 -- !query
 select interval 2 day + cast('2017-12-11 09:30:00' as date)
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) + INTERVAL '2 days' AS DATE):date>
+struct<CAST(2017-12-11 09:30:00 AS DATE) + INTERVAL '2 days':date>
 -- !query output
 2017-12-13
 
@@ -344,6 +344,6 @@ struct<CAST(CAST(2017-12-11 09:30:00.0 AS TIMESTAMP) - INTERVAL '2 days' AS TIME
 -- !query
 select cast('2017-12-11 09:30:00' as date) - interval 2 day
 -- !query schema
-struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) - INTERVAL '2 days' AS DATE):date>
+struct<CAST(2017-12-11 09:30:00 AS DATE) + (- INTERVAL '2 days'):date>
 -- !query output
 2017-12-09

From dfede11d121519a4d2bfcf79639d5bbdbd80c0f0 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 27 Apr 2020 14:49:22 +0900
Subject: [PATCH 0574/1280] [SPARK-29664][PYTHON][SQL][FOLLOW-UP] Add
 deprecation warnings for getItem instead

### What changes were proposed in this pull request?

This PR proposes to use a different approach instead of breaking it per Micheal's rubric added at https://spark.apache.org/versioning-policy.html. It deprecates the behaviour for now. It will be gradually removed in the future releases.

After this change,

```python
import warnings
warnings.simplefilter("always")
from pyspark.sql.functions import *
df = spark.range(2)
map_col = create_map(lit(0), lit(100), lit(1), lit(200))
df.withColumn("mapped", map_col.getItem(col('id'))).show()
```

```
/.../python/pyspark/sql/column.py:311: DeprecationWarning: A column as 'key' in getItem is
deprecated as of Spark 3.0, and will not be supported in the future release. Use `column[key]`
or `column.key` syntax instead.
  DeprecationWarning)
...
```

```python
import warnings
warnings.simplefilter("always")
from pyspark.sql.functions import *
df = spark.range(2)
struct_col = struct(lit(0), lit(100), lit(1), lit(200))
df.withColumn("struct", struct_col.getField(lit("col1"))).show()
```

```
/.../spark/python/pyspark/sql/column.py:336: DeprecationWarning: A column as 'name'
in getField is deprecated as of Spark 3.0, and will not be supported in the future release. Use
`column[name]` or `column.name` syntax instead.
  DeprecationWarning)
```

### Why are the changes needed?

To prevent the radical behaviour change after the amended versioning policy.

### Does this PR introduce any user-facing change?

Yes, it will show the deprecated warning message.

### How was this patch tested?

Manually tested.

Closes #28327 from HyukjinKwon/SPARK-29664.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 5dd581c88ab111377175b673994153072fe9ec77)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/pyspark-migration-guide.md         |  2 --
 python/pyspark/sql/column.py            | 23 +++++++++++++++++------
 python/pyspark/sql/tests/test_column.py | 14 +-------------
 3 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index 92388fffcea52..6f0fbbfb78de8 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -43,8 +43,6 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
    
 - In Spark 3.0, `createDataFrame(..., verifySchema=True)` validates `LongType` as well in PySpark. Previously, `LongType` was not verified and resulted in `None` in case the value overflows. To restore this behavior, `verifySchema` can be set to `False` to disable the validation.
 
-- In Spark 3.0, `Column.getItem` is fixed such that it does not call `Column.apply`. Consequently, if `Column` is used as an argument to `getItem`, the indexing operator should be used. For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
-
 - As of Spark 3.0, `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to `true` for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
 
 ## Upgrading from PySpark 2.3 to 2.4
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 9b728b393452f..ef4944c9121a4 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -17,11 +17,14 @@
 
 import sys
 import json
+import warnings
 
 if sys.version >= '3':
     basestring = str
     long = int
 
+from py4j.java_gateway import is_instance_of
+
 from pyspark import copy_func, since
 from pyspark.context import SparkContext
 from pyspark.rdd import ignore_unicode_prefix
@@ -296,12 +299,14 @@ def getItem(self, key):
         +----+------+
         |   1| value|
         +----+------+
-
-        .. versionchanged:: 3.0
-           If `key` is a `Column` object, the indexing operator should be used instead.
-           For example, `map_col.getItem(col('id'))` should be replaced with `map_col[col('id')]`.
         """
-        return _bin_op("getItem")(self, key)
+        if isinstance(key, Column):
+            warnings.warn(
+                "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
+                "be supported in the future release. Use `column[key]` or `column.key` syntax "
+                "instead.",
+                DeprecationWarning)
+        return self[key]
 
     @since(1.3)
     def getField(self, name):
@@ -323,12 +328,18 @@ def getField(self, name):
         |  1|
         +---+
         """
+        if isinstance(name, Column):
+            warnings.warn(
+                "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
+                "be supported in the future release. Use `column[name]` or `column.name` syntax "
+                "instead.",
+                DeprecationWarning)
         return self[name]
 
     def __getattr__(self, item):
         if item.startswith("__"):
             raise AttributeError(item)
-        return self.getField(item)
+        return self[item]
 
     def __getitem__(self, k):
         if isinstance(k, slice):
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index d9d933110dab5..58bf896a10c2a 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -18,8 +18,6 @@
 
 import sys
 
-from py4j.protocol import Py4JJavaError
-
 from pyspark.sql import Column, Row
 from pyspark.sql.types import *
 from pyspark.sql.utils import AnalysisException
@@ -87,7 +85,7 @@ def test_column_operators(self):
                                 "Cannot apply 'in' operator against a column",
                                 lambda: 1 in cs)
 
-    def test_column_apply(self):
+    def test_column_accessor(self):
         from pyspark.sql.functions import col
 
         self.assertIsInstance(col("foo")[1:3], Column)
@@ -95,16 +93,6 @@ def test_column_apply(self):
         self.assertIsInstance(col("foo")["bar"], Column)
         self.assertRaises(ValueError, lambda: col("foo")[0:10:2])
 
-    def test_column_getitem(self):
-        from pyspark.sql.functions import col, create_map, lit
-
-        map_col = create_map(lit(0), lit(100), lit(1), lit(200))
-        self.assertRaisesRegexp(
-            Py4JJavaError,
-            "Unsupported literal type class org.apache.spark.sql.Column id",
-            lambda: map_col.getItem(col('id'))
-        )
-
     def test_column_select(self):
         df = self.df
         self.assertEqual(self.testData, df.select("*").collect())

From 74605950da6f115bf491d2371c30f58689b26623 Mon Sep 17 00:00:00 2001
From: Michael Chirico <michael.chirico@grabtaxi.com>
Date: Mon, 27 Apr 2020 17:02:18 +0900
Subject: [PATCH 0575/1280] [SPARK-31568][R][DOCS] Add detail about func/key in
 gapply to documentation

Improve documentation for `gapply` in `SparkR`

Spent a long time this weekend trying to figure out just what exactly `key` is in `gapply`'s `func`. I had assumed it would be a _named_ list, but apparently not -- the examples are working because `schema` is applying the name and the names of the output `data.frame` don't matter.

As near as I can tell the description I've added is correct, namely, that `key` is an unnamed list.

No? Not in code. Only documentation.

Not. Documentation only

Closes #28350 from MichaelChirico/r-gapply-key-doc.

Authored-by: Michael Chirico <michael.chirico@grabtaxi.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/DataFrame.R | 112 ++++++++++++++++++++++++++------------------
 1 file changed, 67 insertions(+), 45 deletions(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 14d2076e88eff..2ebd42a263ce0 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1657,9 +1657,7 @@ setMethod("dapplyCollect",
 #'
 #' @param cols grouping columns.
 #' @param func a function to be applied to each group partition specified by grouping
-#'             column of the SparkDataFrame. The function \code{func} takes as argument
-#'             a key - grouping columns and a data frame - a local R data.frame.
-#'             The output of \code{func} is a local R data.frame.
+#'             column of the SparkDataFrame. See Details.
 #' @param schema the schema of the resulting SparkDataFrame after the function is applied.
 #'               The schema must match to output of \code{func}. It has to be defined for each
 #'               output column with preferred output column name and corresponding data type.
@@ -1669,29 +1667,43 @@ setMethod("dapplyCollect",
 #' @aliases gapply,SparkDataFrame-method
 #' @rdname gapply
 #' @name gapply
+#' @details
+#' \code{func} is a function of two arguments. The first, usually named \code{key}
+#' (though this is not enforced) corresponds to the grouping key, will be an
+#' unnamed \code{list} of \code{length(cols)} length-one objects corresponding
+#' to the grouping columns' values for the current group.
+#'
+#' The second, herein \code{x}, will be a local \code{\link{data.frame}} with the
+#' columns of the input not in \code{cols} for the rows corresponding to \code{key}.
+#'
+#' The output of \code{func} must be a \code{data.frame} matching \code{schema} --
+#' in particular this means the names of the output \code{data.frame} are irrelevant
+#'
 #' @seealso \link{gapplyCollect}
 #' @examples
 #'
 #' \dontrun{
-#' Computes the arithmetic mean of the second column by grouping
-#' on the first and third columns. Output the grouping values and the average.
+#' # Computes the arithmetic mean of the second column by grouping
+#' # on the first and third columns. Output the grouping values and the average.
 #'
 #' df <- createDataFrame (
 #' list(list(1L, 1, "1", 0.1), list(1L, 2, "1", 0.2), list(3L, 3, "3", 0.3)),
 #'   c("a", "b", "c", "d"))
 #'
-#' Here our output contains three columns, the key which is a combination of two
-#' columns with data types integer and string and the mean which is a double.
+#' # Here our output contains three columns, the key which is a combination of two
+#' # columns with data types integer and string and the mean which is a double.
 #' schema <- structType(structField("a", "integer"), structField("c", "string"),
 #'   structField("avg", "double"))
 #' result <- gapply(
 #'   df,
 #'   c("a", "c"),
 #'   function(key, x) {
+#'     # key will either be list(1L, '1') (for the group where a=1L,c='1') or
+#'     #   list(3L, '3') (for the group where a=3L,c='3')
 #'     y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE)
 #' }, schema)
 #'
-#' The schema also can be specified in a DDL-formatted string.
+#' # The schema also can be specified in a DDL-formatted string.
 #' schema <- "a INT, c STRING, avg DOUBLE"
 #' result <- gapply(
 #'   df,
@@ -1700,8 +1712,8 @@ setMethod("dapplyCollect",
 #'     y <- data.frame(key, mean(x$b), stringsAsFactors = FALSE)
 #' }, schema)
 #'
-#' We can also group the data and afterwards call gapply on GroupedData.
-#' For Example:
+#' # We can also group the data and afterwards call gapply on GroupedData.
+#' # For example:
 #' gdf <- group_by(df, "a", "c")
 #' result <- gapply(
 #'   gdf,
@@ -1710,15 +1722,15 @@ setMethod("dapplyCollect",
 #' }, schema)
 #' collect(result)
 #'
-#' Result
-#' ------
-#' a c avg
-#' 3 3 3.0
-#' 1 1 1.5
+#' # Result
+#' # ------
+#' # a c avg
+#' # 3 3 3.0
+#' # 1 1 1.5
 #'
-#' Fits linear models on iris dataset by grouping on the 'Species' column and
-#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
-#' and 'Petal_Width' as training features.
+#' # Fits linear models on iris dataset by grouping on the 'Species' column and
+#' # using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
+#' # and 'Petal_Width' as training features.
 #'
 #' df <- createDataFrame (iris)
 #' schema <- structType(structField("(Intercept)", "double"),
@@ -1734,12 +1746,12 @@ setMethod("dapplyCollect",
 #'   }, schema)
 #' collect(df1)
 #'
-#' Result
-#' ---------
-#' Model  (Intercept)  Sepal_Width  Petal_Length  Petal_Width
-#' 1        0.699883    0.3303370    0.9455356    -0.1697527
-#' 2        1.895540    0.3868576    0.9083370    -0.6792238
-#' 3        2.351890    0.6548350    0.2375602     0.2521257
+#' # Result
+#' # ---------
+#' # Model  (Intercept)  Sepal_Width  Petal_Length  Petal_Width
+#' # 1        0.699883    0.3303370    0.9455356    -0.1697527
+#' # 2        1.895540    0.3868576    0.9083370    -0.6792238
+#' # 3        2.351890    0.6548350    0.2375602     0.2521257
 #'
 #'}
 #' @note gapply(SparkDataFrame) since 2.0.0
@@ -1757,20 +1769,30 @@ setMethod("gapply",
 #'
 #' @param cols grouping columns.
 #' @param func a function to be applied to each group partition specified by grouping
-#'             column of the SparkDataFrame. The function \code{func} takes as argument
-#'             a key - grouping columns and a data frame - a local R data.frame.
-#'             The output of \code{func} is a local R data.frame.
+#'             column of the SparkDataFrame. See Details.
 #' @return A data.frame.
 #' @family SparkDataFrame functions
 #' @aliases gapplyCollect,SparkDataFrame-method
 #' @rdname gapplyCollect
 #' @name gapplyCollect
+#' @details
+#' \code{func} is a function of two arguments. The first, usually named \code{key}
+#' (though this is not enforced) corresponds to the grouping key, will be an
+#' unnamed \code{list} of \code{length(cols)} length-one objects corresponding
+#' to the grouping columns' values for the current group.
+#'
+#' The second, herein \code{x}, will be a local \code{\link{data.frame}} with the
+#' columns of the input not in \code{cols} for the rows corresponding to \code{key}.
+#'
+#' The output of \code{func} must be a \code{data.frame} matching \code{schema} --
+#' in particular this means the names of the output \code{data.frame} are irrelevant
+#'
 #' @seealso \link{gapply}
 #' @examples
 #'
 #' \dontrun{
-#' Computes the arithmetic mean of the second column by grouping
-#' on the first and third columns. Output the grouping values and the average.
+#' # Computes the arithmetic mean of the second column by grouping
+#' # on the first and third columns. Output the grouping values and the average.
 #'
 #' df <- createDataFrame (
 #' list(list(1L, 1, "1", 0.1), list(1L, 2, "1", 0.2), list(3L, 3, "3", 0.3)),
@@ -1785,8 +1807,8 @@ setMethod("gapply",
 #'     y
 #'   })
 #'
-#' We can also group the data and afterwards call gapply on GroupedData.
-#' For Example:
+#' # We can also group the data and afterwards call gapply on GroupedData.
+#' # For example:
 #' gdf <- group_by(df, "a", "c")
 #' result <- gapplyCollect(
 #'   gdf,
@@ -1796,15 +1818,15 @@ setMethod("gapply",
 #'     y
 #'   })
 #'
-#' Result
-#' ------
-#' key_a key_c mean_b
-#' 3 3 3.0
-#' 1 1 1.5
+#' # Result
+#' # ------
+#' # key_a key_c mean_b
+#' # 3 3 3.0
+#' # 1 1 1.5
 #'
-#' Fits linear models on iris dataset by grouping on the 'Species' column and
-#' using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
-#' and 'Petal_Width' as training features.
+#' # Fits linear models on iris dataset by grouping on the 'Species' column and
+#' # using 'Sepal_Length' as a target variable, 'Sepal_Width', 'Petal_Length'
+#' # and 'Petal_Width' as training features.
 #'
 #' df <- createDataFrame (iris)
 #' result <- gapplyCollect(
@@ -1816,12 +1838,12 @@ setMethod("gapply",
 #'     data.frame(t(coef(m)))
 #'   })
 #'
-#' Result
-#'---------
-#' Model  X.Intercept.  Sepal_Width  Petal_Length  Petal_Width
-#' 1        0.699883    0.3303370    0.9455356    -0.1697527
-#' 2        1.895540    0.3868576    0.9083370    -0.6792238
-#' 3        2.351890    0.6548350    0.2375602     0.2521257
+#' # Result
+#' # ---------
+#' # Model  X.Intercept.  Sepal_Width  Petal_Length  Petal_Width
+#' # 1        0.699883    0.3303370    0.9455356    -0.1697527
+#' # 2        1.895540    0.3868576    0.9083370    -0.6792238
+#' # 3        2.351890    0.6548350    0.2375602     0.2521257
 #'
 #'}
 #' @note gapplyCollect(SparkDataFrame) since 2.0.0

From db013704c051a5946d659a3371df95e93416252d Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Mon, 27 Apr 2020 17:08:52 +0900
Subject: [PATCH 0576/1280] [SPARK-31550][SQL][DOCS] Set nondeterministic
 configurations with general meanings in sql configuration doc

### What changes were proposed in this pull request?

```scala
spark.sql.session.timeZone

spark.sql.warehouse.dir
```
these 2 configs are nondeterministic and vary with environments

Besides, reflect code in `gen-sql-config-docs.py` via  https://github.com/apache/spark/pull/28274#discussion_r412893096 and `configuration.md` via https://github.com/apache/spark/pull/28274#discussion_r412894905
### Why are the changes needed?

doc fix

### Does this PR introduce any user-facing change?

no
### How was this patch tested?

verify locally
![image](https://user-images.githubusercontent.com/8326978/80179099-5e7da200-8632-11ea-803f-d47a93151869.png)

Closes #28322 from yaooqinn/SPARK-31550.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 5ba467ca1d7860d56611c2f6ff44a3086e14573e)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md                         | 13 +++++----
 .../spark/sql/api/python/PythonSQLUtils.scala | 15 ++++++----
 .../sql/api/python/PythonSQLUtilsSuite.scala  |  2 +-
 sql/create-docs.sh                            |  7 ++---
 sql/gen-sql-config-docs.py                    | 28 +++++++++----------
 5 files changed, 33 insertions(+), 32 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 9e91d08591e45..a97d3e4c33d44 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2613,6 +2613,9 @@ Spark subsystems.
 
 ### Spark SQL
 
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-runtime-sql-config-table.html' %}
+
 #### Runtime SQL Configuration
 
 Runtime SQL configurations are per-session, mutable Spark SQL configurations. They can be set with initial values by the config file
@@ -2620,13 +2623,13 @@ and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` th
 Also, they can be set and queried by SET commands and rest to their initial values by RESET command,
 or by `SparkSession.conf`'s setter and getter methods in runtime.
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'generated-runtime-sql-config-table.html' %}
-        {% include_relative generated-runtime-sql-config-table.html %}
+{% include_relative generated-runtime-sql-config-table.html %}
         {% break %}
     {% endif %}
 {% endfor %}
 
+{% for static_file in site.static_files %}
+    {% if static_file.name == 'generated-static-sql-config-table.html' %}
 
 #### Static SQL Configuration
 
@@ -2634,9 +2637,7 @@ Static SQL configurations are cross-session, immutable Spark SQL configurations.
 and command-line options with `--conf/-c` prefixed, or by setting `SparkConf` that are used to create `SparkSession`.
 External users can query the static sql config values via `SparkSession.conf` or via set command, e.g. `SET spark.sql.extensions;`, but cannot set/unset them.
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'generated-static-sql-config-table.html' %}
-        {% include_relative generated-static-sql-config-table.html %}
+{% include_relative generated-static-sql-config-table.html %}
         {% break %}
     {% endif %}
 {% endfor %}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index 2e5f59edcf1da..89bb4f362de74 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -40,17 +40,20 @@ private[sql] object PythonSQLUtils {
     FunctionRegistry.functionSet.flatMap(f => FunctionRegistry.builtin.lookupFunction(f)).toArray
   }
 
-  def listSQLConfigs(): Array[(String, String, String, String)] = {
+  private def listAllSQLConfigs(): Seq[(String, String, String, String)] = {
     val conf = new SQLConf()
+    // Force to build static SQL configurations
+    StaticSQLConf
+    conf.getAllDefinedConfs
+  }
+
+  def listRuntimeSQLConfigs(): Array[(String, String, String, String)] = {
     // Py4J doesn't seem to translate Seq well, so we convert to an Array.
-    conf.getAllDefinedConfs.filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+    listAllSQLConfigs().filterNot(p => SQLConf.staticConfKeys.contains(p._1)).toArray
   }
 
   def listStaticSQLConfigs(): Array[(String, String, String, String)] = {
-    val conf = new SQLConf()
-    // Force to build static SQL configurations
-    StaticSQLConf
-    conf.getAllDefinedConfs.filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray
+    listAllSQLConfigs().filter(p => SQLConf.staticConfKeys.contains(p._1)).toArray
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
index 0d18d123e328a..524fc8a9b0dea 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/api/python/PythonSQLUtilsSuite.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 class PythonSQLUtilsSuite extends SparkFunSuite {
 
   test("listing sql configurations contains runtime ones only") {
-    val configs = PythonSQLUtils.listSQLConfigs()
+    val configs = PythonSQLUtils.listRuntimeSQLConfigs()
 
     // static sql configurations
     assert(!configs.exists(entry => entry._1 == StaticSQLConf.SPARK_SESSION_EXTENSIONS.key),
diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 336afc4fcb9f4..6614c714e90c7 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -45,11 +45,8 @@ mkdir docs
 echo "Generating SQL API Markdown files."
 "$SPARK_HOME/bin/spark-submit" gen-sql-api-docs.py
 
-echo "Generating runtime SQL runtime configuration table HTML file."
-"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py runtime
-
-echo "Generating static SQL configuration table HTML file."
-"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py static
+echo "Generating SQL configuration table HTML file."
+"$SPARK_HOME/bin/spark-submit" gen-sql-config-docs.py
 
 echo "Generating HTML files for SQL function table and examples."
 "$SPARK_HOME/bin/spark-submit" gen-sql-functions-docs.py
diff --git a/sql/gen-sql-config-docs.py b/sql/gen-sql-config-docs.py
index 848d2f21f1142..f1980bcc0d80d 100644
--- a/sql/gen-sql-config-docs.py
+++ b/sql/gen-sql-config-docs.py
@@ -17,7 +17,7 @@
 
 import os
 import re
-import sys
+
 from collections import namedtuple
 from textwrap import dedent
 
@@ -31,11 +31,11 @@
     "SQLConfEntry", ["name", "default", "description", "version"])
 
 
-def get_public_sql_configs(jvm, group):
+def get_sql_configs(jvm, group):
     if group == "static":
         config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listStaticSQLConfigs()
     else:
-        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listSQLConfigs()
+        config_set = jvm.org.apache.spark.sql.api.python.PythonSQLUtils.listRuntimeSQLConfigs()
     sql_configs = [
         SQLConfEntry(
             name=_sql_config._1(),
@@ -81,7 +81,11 @@ def generate_sql_configs_table_html(sql_configs, path):
             """
         ))
         for config in sorted(sql_configs, key=lambda x: x.name):
-            if config.default == "<undefined>":
+            if config.name == "spark.sql.session.timeZone":
+                default = "(value of local timezone)"
+            elif config.name == "spark.sql.warehouse.dir":
+                default = "(value of <code>$PWD/spark-warehouse</code>)"
+            elif config.default == "<undefined>":
                 default = "(none)"
             elif config.default.startswith("<value of "):
                 referenced_config_name = value_reference_pattern.match(config.default).group(1)
@@ -119,17 +123,13 @@ def generate_sql_configs_table_html(sql_configs, path):
 
 
 if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: ./bin/spark-submit sql/gen-sql-config-docs.py <static|runtime>")
-        sys.exit(-1)
-    else:
-        group = sys.argv[1]
-
     jvm = launch_gateway().jvm
-    sql_configs = get_public_sql_configs(jvm, group)
+    docs_root_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs")
 
-    spark_root_dir = os.path.dirname(os.path.dirname(__file__))
-    sql_configs_table_path = os.path\
-        .join(spark_root_dir, "docs", "generated-" + group + "-sql-config-table.html")
+    sql_configs = get_sql_configs(jvm, "runtime")
+    sql_configs_table_path = os.path.join(docs_root_dir, "generated-runtime-sql-config-table.html")
+    generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)
 
+    sql_configs = get_sql_configs(jvm, "static")
+    sql_configs_table_path = os.path.join(docs_root_dir, "generated-static-sql-config-table.html")
     generate_sql_configs_table_html(sql_configs, path=sql_configs_table_path)

From 60e69834d3cf32531d81d6583e689bd1b4c252fb Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 27 Apr 2020 12:05:53 +0000
Subject: [PATCH 0577/1280] [SPARK-31485][CORE][3.0] Avoid application hang if
 only partial barrier tasks launched

### What changes were proposed in this pull request?

Use `TaskSetManager. abort ` to abort a barrier stage instead of throwing exception within `resourceOffers`.

### Why are the changes needed?

Any non fatal exception thrown within Spark RPC framework can be swallowed:

https://github.com/apache/spark/blob/100fc58da54e026cda87832a10e2d06eaeccdf87/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala#L202-L211

 The method `TaskSchedulerImpl.resourceOffers` is also within the scope of Spark RPC framework. Thus, throw exception inside `resourceOffers` won't fail the application.

 As a result, if a barrier stage fail the require check at `require(addressesWithDescs.size == taskSet.numTasks, ...)`, the barrier stage will fail the check again and again util all tasks from `TaskSetManager` dequeued.   But since the barrier stage isn't really executed, the application will hang.

The issue can be reproduced by the following test:

```scala
initLocalClusterSparkContext(2)
val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
val dep = new OneToOneDependency[Int](rdd0)
val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq("executor_h_0"),Seq("executor_h_0")))
rdd.barrier().mapPartitions { iter =>
  BarrierTaskContext.get().barrier()
  iter
}.collect()
```
### Does this PR introduce any user-facing change?

Yes, application hang previously but fail-fast after this fix.

### How was this patch tested?

Added a regression test.

Closes #28356 from Ngone51/bp-spark-31485.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/scheduler/TaskSchedulerImpl.scala   | 34 +++++++++++++------
 .../scheduler/BarrierTaskContextSuite.scala   | 20 +++++++++--
 2 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index ed304736356c1..97125f6238b22 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -57,6 +57,11 @@ import org.apache.spark.util.{AccumulatorV2, SystemClock, ThreadUtils, Utils}
  *   * Periodic revival of all offers from the CoarseGrainedSchedulerBackend, to accommodate delay
  *      scheduling
  *   * task-result-getter threads
+ *
+ * CAUTION: Any non fatal exception thrown within Spark RPC framework can be swallowed.
+ * Thus, throwing exception in methods like resourceOffers, statusUpdate won't fail
+ * the application, but could lead to undefined behavior. Instead, we shall use method like
+ * TaskSetManger.abort() to abort a stage and then fail the application (SPARK-31485).
  */
 private[spark] class TaskSchedulerImpl(
     val sc: SparkContext,
@@ -356,9 +361,7 @@ private[spark] class TaskSchedulerImpl(
               // addresses are the same as that we allocated in taskSet.resourceOffer() since it's
               // synchronized. We don't remove the exact addresses allocated because the current
               // approach produces the identical result with less time complexity.
-              availableResources(i).getOrElse(rName,
-                throw new SparkException(s"Try to acquire resource $rName that doesn't exist."))
-                .remove(0, rInfo.addresses.size)
+              availableResources(i)(rName).remove(0, rInfo.addresses.size)
             }
             // Only update hosts for a barrier task.
             if (taskSet.isBarrier) {
@@ -516,11 +519,18 @@ private[spark] class TaskSchedulerImpl(
           // Check whether the barrier tasks are partially launched.
           // TODO SPARK-24818 handle the assert failure case (that can happen when some locality
           // requirements are not fulfilled, and we should revert the launched tasks).
-          require(addressesWithDescs.size == taskSet.numTasks,
-            s"Skip current round of resource offers for barrier stage ${taskSet.stageId} " +
-              s"because only ${addressesWithDescs.size} out of a total number of " +
-              s"${taskSet.numTasks} tasks got resource offers. The resource offers may have " +
-              "been blacklisted or cannot fulfill task locality requirements.")
+          if (addressesWithDescs.size != taskSet.numTasks) {
+            val errorMsg =
+              s"Fail resource offers for barrier stage ${taskSet.stageId} because only " +
+                s"${addressesWithDescs.size} out of a total number of ${taskSet.numTasks}" +
+                s" tasks got resource offers. This happens because barrier execution currently " +
+                s"does not work gracefully with delay scheduling. We highly recommend you to " +
+                s"disable delay scheduling by setting spark.locality.wait=0 as a workaround if " +
+                s"you see this error frequently."
+            logWarning(errorMsg)
+            taskSet.abort(errorMsg)
+            throw new SparkException(errorMsg)
+          }
 
           // materialize the barrier coordinator.
           maybeInitBarrierCoordinator()
@@ -582,8 +592,12 @@ private[spark] class TaskSchedulerImpl(
             if (state == TaskState.LOST) {
               // TaskState.LOST is only used by the deprecated Mesos fine-grained scheduling mode,
               // where each executor corresponds to a single task, so mark the executor as failed.
-              val execId = taskIdToExecutorId.getOrElse(tid, throw new IllegalStateException(
-                "taskIdToTaskSetManager.contains(tid) <=> taskIdToExecutorId.contains(tid)"))
+              val execId = taskIdToExecutorId.getOrElse(tid, {
+                val errorMsg =
+                  "taskIdToTaskSetManager.contains(tid) <=> taskIdToExecutorId.contains(tid)"
+                taskSet.abort(errorMsg)
+                throw new SparkException(errorMsg)
+              })
               if (executorIdToRunningTaskIds.contains(execId)) {
                 reason = Some(
                   SlaveLost(s"Task $tid was lost, so marking the executor as lost as well."))
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index bcf1fe2c2aa11..2242d2807652a 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -26,11 +26,11 @@ import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 
 class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
 
-  def initLocalClusterSparkContext(): Unit = {
+  def initLocalClusterSparkContext(numWorker: Int = 4): Unit = {
     val conf = new SparkConf()
       // Init local cluster here so each barrier task runs in a separated process, thus `barrier()`
       // call is actually useful.
-      .setMaster("local-cluster[4, 1, 1024]")
+      .setMaster(s"local-cluster[$numWorker, 1, 1024]")
       .setAppName("test-cluster")
       .set(TEST_NO_STAGE_RETRY, true)
     sc = new SparkContext(conf)
@@ -276,4 +276,20 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
     initLocalClusterSparkContext()
     testBarrierTaskKilled(interruptOnKill = true)
   }
+
+  test("SPARK-31485: barrier stage should fail if only partial tasks are launched") {
+    initLocalClusterSparkContext(2)
+    val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
+    val dep = new OneToOneDependency[Int](rdd0)
+    // set up a barrier stage with 2 tasks and both tasks prefer executor 0 (only 1 core) for
+    // scheduling. So, one of tasks won't be scheduled in one round of resource offer.
+    val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq("executor_h_0"), Seq("executor_h_0")))
+    val errorMsg = intercept[SparkException] {
+      rdd.barrier().mapPartitions { iter =>
+        BarrierTaskContext.get().barrier()
+        iter
+      }.collect()
+    }.getMessage
+    assert(errorMsg.contains("Fail resource offers for barrier stage"))
+  }
 }

From 3b30066e7a2cb27e59e53419226e64f6a90201d6 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 27 Apr 2020 13:39:36 +0000
Subject: [PATCH 0578/1280] [SPARK-31529][SQL][3.0] Remove extra whitespaces in
 formatted explain

### What changes were proposed in this pull request?

Remove all the extra whitespaces in the formatted explain.

### Why are the changes needed?

The number of extra whitespaces of the formatted explain becomes different between master and branch-3.0. This causes a problem that whenever we backport formatted explain related tests from master to branch-3.0, it will fail branch-3.0. Besides, extra whitespaces are always disallowed in Spark. Thus, we should remove them as possible as we can.

### Does this PR introduce any user-facing change?

No, formatted explain is newly added in Spark 3.0.

### How was this patch tested?

Updated sql query tests.

Closes #28364 from Ngone51/bp-spark31529.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/plans/QueryPlan.scala  |  19 +-
 .../sql/execution/DataSourceScanExec.scala    |   8 +-
 .../spark/sql/execution/ExplainUtils.scala    |  10 +-
 .../spark/sql/execution/SparkPlan.scala       |  27 +-
 .../aggregate/BaseAggregateExec.scala         |   4 +-
 .../execution/basicPhysicalOperators.scala    |   8 +-
 .../sql/execution/exchange/Exchange.scala     |   5 +-
 .../joins/CartesianProductExec.scala          |   2 +-
 .../spark/sql/execution/joins/HashJoin.scala  |   4 +-
 .../execution/joins/SortMergeJoinExec.scala   |   4 +-
 .../sql-tests/results/explain-aqe.sql.out     | 372 +++++++++---------
 .../sql-tests/results/explain.sql.out         | 340 ++++++++--------
 .../org/apache/spark/sql/ExplainSuite.scala   |  10 +-
 13 files changed, 401 insertions(+), 412 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 13e5b129765e2..7133fb231d672 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -195,24 +195,27 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
   }
 
   def verboseStringWithOperatorId(): String = {
-    val codegenIdStr =
-      getTagValue(QueryPlan.CODEGEN_ID_TAG).map(id => s"[codegen id : $id]").getOrElse("")
-    val operatorId = getTagValue(QueryPlan.OP_ID_TAG).map(id => s"$id").getOrElse("unknown")
-    val baseStr = s"($operatorId) $nodeName $codegenIdStr"
     val argumentString = argString(SQLConf.get.maxToStringFields)
 
     if (argumentString.nonEmpty) {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |Arguments: $argumentString
-      """.stripMargin
+         |""".stripMargin
     } else {
       s"""
-         |$baseStr
-      """.stripMargin
+         |$formattedNodeName
+         |""".stripMargin
     }
   }
 
+  protected def formattedNodeName: String = {
+    val opId = getTagValue(QueryPlan.OP_ID_TAG).map(id => s"$id").getOrElse("unknown")
+    val codegenId =
+      getTagValue(QueryPlan.CODEGEN_ID_TAG).map(id => s" [codegen id : $id]").getOrElse("")
+    s"($opId) $nodeName$codegenId"
+  }
+
   /**
    * All the top-level subqueries of the current plan node. Nested subqueries are not included.
    */
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index bd0e1d08f60e6..0be76ad6c5bd3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -75,10 +75,10 @@ trait DataSourceScanExec extends LeafExecNode {
     }
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Output", output)}
        |${metadataStr.mkString("\n")}
-     """.stripMargin
+       |""".stripMargin
   }
 
   /**
@@ -377,10 +377,10 @@ case class FileSourceScanExec(
     }
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Output", output)}
        |${metadataStr.mkString("\n")}
-     """.stripMargin
+       |""".stripMargin
   }
 
   lazy val inputRDD: RDD[InternalRow] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index aec1c93072410..b54bd6a579b66 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -233,20 +233,12 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
 
   /**
    * Returns the operator identifier for the supplied plan by retrieving the
-   * `operationId` tag value.`
+   * `operationId` tag value.
    */
   def getOpId(plan: QueryPlan[_]): String = {
     plan.getTagValue(QueryPlan.OP_ID_TAG).map(v => s"$v").getOrElse("unknown")
   }
 
-  /**
-   * Returns the operator identifier for the supplied plan by retrieving the
-   * `codegenId` tag value.`
-   */
-  def getCodegenId(plan: QueryPlan[_]): String = {
-    plan.getTagValue(QueryPlan.CODEGEN_ID_TAG).map(v => s"[codegen id : $v]").getOrElse("")
-  }
-
   def removeTags(plan: QueryPlan[_]): Unit = {
     def remove(p: QueryPlan[_], children: Seq[QueryPlan[_]]): Unit = {
       p.unsetTagValue(QueryPlan.OP_ID_TAG)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index e1a6495530a30..ead8c00031112 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -514,20 +514,19 @@ trait LeafExecNode extends SparkPlan {
   override def producedAttributes: AttributeSet = outputSet
   override def verboseStringWithOperatorId(): String = {
     val argumentString = argString(SQLConf.get.maxToStringFields)
-    val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
     val outputStr = s"${ExplainUtils.generateFieldString("Output", output)}"
 
     if (argumentString.nonEmpty) {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |$outputStr
          |Arguments: $argumentString
-      """.stripMargin
+         |""".stripMargin
     } else {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |$outputStr
-      """.stripMargin
+         |""".stripMargin
     }
   }
 }
@@ -545,20 +544,19 @@ trait UnaryExecNode extends SparkPlan {
   override final def children: Seq[SparkPlan] = child :: Nil
   override def verboseStringWithOperatorId(): String = {
     val argumentString = argString(SQLConf.get.maxToStringFields)
-    val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
     val inputStr = s"${ExplainUtils.generateFieldString("Input", child.output)}"
 
     if (argumentString.nonEmpty) {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |$inputStr
          |Arguments: $argumentString
-      """.stripMargin
+         |""".stripMargin
     } else {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |$inputStr
-      """.stripMargin
+         |""".stripMargin
     }
   }
 }
@@ -570,23 +568,22 @@ trait BinaryExecNode extends SparkPlan {
   override final def children: Seq[SparkPlan] = Seq(left, right)
   override def verboseStringWithOperatorId(): String = {
     val argumentString = argString(SQLConf.get.maxToStringFields)
-    val baseStr = s"(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}"
     val leftOutputStr = s"${ExplainUtils.generateFieldString("Left output", left.output)}"
     val rightOutputStr = s"${ExplainUtils.generateFieldString("Right output", right.output)}"
 
     if (argumentString.nonEmpty) {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |$leftOutputStr
          |$rightOutputStr
          |Arguments: $argumentString
-      """.stripMargin
+         |""".stripMargin
     } else {
       s"""
-         |$baseStr
+         |$formattedNodeName
          |$leftOutputStr
          |$rightOutputStr
-      """.stripMargin
+         |""".stripMargin
     }
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
index 19d7263feb2d1..f506bdddc16b5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
@@ -32,12 +32,12 @@ trait BaseAggregateExec extends UnaryExecNode {
 
   override def verboseStringWithOperatorId(): String = {
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Input", child.output)}
        |${ExplainUtils.generateFieldString("Keys", groupingExpressions)}
        |${ExplainUtils.generateFieldString("Functions", aggregateExpressions)}
        |${ExplainUtils.generateFieldString("Aggregate Attributes", aggregateAttributes)}
        |${ExplainUtils.generateFieldString("Results", resultExpressions)}
-     """.stripMargin
+       |""".stripMargin
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index c3e259d196ba2..036b02953a430 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -86,10 +86,10 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
 
   override def verboseStringWithOperatorId(): String = {
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Output", projectList)}
        |${ExplainUtils.generateFieldString("Input", child.output)}
-     """.stripMargin
+       |""".stripMargin
   }
 }
 
@@ -243,10 +243,10 @@ case class FilterExec(condition: Expression, child: SparkPlan)
 
   override def verboseStringWithOperatorId(): String = {
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Input", child.output)}
        |Condition : ${condition}
-     """.stripMargin
+       |""".stripMargin
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
index dda9a637194fa..c4062879c2727 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/Exchange.scala
@@ -88,12 +88,11 @@ case class ReusedExchangeExec(override val output: Seq[Attribute], child: Exchan
   }
 
   override def verboseStringWithOperatorId(): String = {
-    val cdgen = ExplainUtils.getCodegenId(this)
     val reuse_op_str = ExplainUtils.getOpId(child)
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${cdgen} [Reuses operator id: $reuse_op_str]
+       |$formattedNodeName [Reuses operator id: $reuse_op_str]
        |${ExplainUtils.generateFieldString("Output", output)}
-     """.stripMargin
+       |""".stripMargin
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
index 7e2f487fdcc5d..91016a9060303 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/CartesianProductExec.scala
@@ -72,7 +72,7 @@ case class CartesianProductExec(
     } else "None"
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
      """.stripMargin
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 99cf60273bf08..305741e2984a4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -48,11 +48,11 @@ trait HashJoin {
     } else "None"
 
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Left keys", leftKeys)}
        |${ExplainUtils.generateFieldString("Right keys", rightKeys)}
        |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
-     """.stripMargin
+       |""".stripMargin
   }
 
   override def output: Seq[Attribute] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
index 7a08dd1afd3a6..5b5904f1575fd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala
@@ -62,11 +62,11 @@ case class SortMergeJoinExec(
       s"${condition.get}"
     } else "None"
     s"""
-       |(${ExplainUtils.getOpId(this)}) $nodeName ${ExplainUtils.getCodegenId(this)}
+       |$formattedNodeName
        |${ExplainUtils.generateFieldString("Left keys", leftKeys)}
        |${ExplainUtils.generateFieldString("Right keys", rightKeys)}
        |${ExplainUtils.generateFieldString("Join condition", joinCondStr)}
-     """.stripMargin
+       |""".stripMargin
   }
 
   override def output: Seq[Attribute] = {
diff --git a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
index f00dd81a0c581..36757863ffcb5 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain-aqe.sql.out
@@ -64,48 +64,48 @@ AdaptiveSparkPlan (9)
                      +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) HashAggregate 
+
+(4) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(5) Exchange 
+
+(5) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(6) HashAggregate 
+
+(6) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
-     
-(7) Exchange 
+
+(7) Exchange
 Input [2]: [key#x, max(val)#x]
 Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
-      
-(8) Sort 
+
+(8) Sort
 Input [2]: [key#x, max(val)#x]
 Arguments: [key#x ASC NULLS FIRST], true, 0
-      
-(9) AdaptiveSparkPlan 
+
+(9) AdaptiveSparkPlan
 Output [2]: [key#x, max(val)#x]
 Arguments: isFinalPlan=false
 
@@ -132,48 +132,48 @@ AdaptiveSparkPlan (9)
                      +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) HashAggregate 
+
+(4) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(5) Exchange 
+
+(5) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(6) HashAggregate 
+
+(6) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
-     
-(7) Filter 
+
+(7) Filter
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
 Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
-     
-(8) Project 
+
+(8) Project
 Output [2]: [key#x, max(val)#x]
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
-     
-(9) AdaptiveSparkPlan 
+
+(9) AdaptiveSparkPlan
 Output [2]: [key#x, max(val)#x]
 Arguments: isFinalPlan=false
 
@@ -200,57 +200,57 @@ AdaptiveSparkPlan (11)
                   +- Scan parquet default.explain_temp1 (4)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) Scan parquet default.explain_temp1 
+
+(4) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
-(5) Filter 
+
+(5) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
-(6) Project 
+
+(6) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(7) Union 
-      
-(8) HashAggregate 
+
+(7) Union
+
+(8) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
 Results [2]: [key#x, val#x]
-     
-(9) Exchange 
+
+(9) Exchange
 Input [2]: [key#x, val#x]
 Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
-      
-(10) HashAggregate 
+
+(10) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
 Results [2]: [key#x, val#x]
-     
-(11) AdaptiveSparkPlan 
+
+(11) AdaptiveSparkPlan
 Output [2]: [key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -276,46 +276,46 @@ AdaptiveSparkPlan (9)
             +- Scan parquet default.explain_temp2 (4)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) Scan parquet default.explain_temp2 
+
+(4) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
-(5) Filter 
+
+(5) Filter
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
-(6) Project 
+
+(6) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(7) BroadcastExchange 
+
+(7) BroadcastExchange
 Input [2]: [key#x, val#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
-(8) BroadcastHashJoin 
+
+(8) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-     
-(9) AdaptiveSparkPlan 
+
+(9) AdaptiveSparkPlan
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -339,37 +339,37 @@ AdaptiveSparkPlan (7)
             +- Scan parquet default.explain_temp2 (2)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Scan parquet default.explain_temp2 
+
+(2) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
-(3) Filter 
+
+(3) Filter
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
-(4) Project 
+
+(4) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(5) BroadcastExchange 
+
+(5) BroadcastExchange
 Input [2]: [key#x, val#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
-(6) BroadcastHashJoin 
+
+(6) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-     
-(7) AdaptiveSparkPlan 
+
+(7) AdaptiveSparkPlan
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -395,22 +395,22 @@ AdaptiveSparkPlan (4)
       +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), IsNotNull(val), GreaterThan(val,3)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery subquery#x, [id=#x])) AND (val#x > 3))
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) AdaptiveSparkPlan 
+
+(4) AdaptiveSparkPlan
 Output [2]: [key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -435,17 +435,17 @@ AdaptiveSparkPlan (3)
    +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : ((key#x = Subquery subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery subquery#x, [id=#x]))
-     
-(3) AdaptiveSparkPlan 
+
+(3) AdaptiveSparkPlan
 Output [2]: [key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -463,17 +463,17 @@ AdaptiveSparkPlan (3)
    +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output: []
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<>
-     
-(2) Project 
+
+(2) Project
 Output [1]: [(Subquery subquery#x, [id=#x] + Subquery subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
 Input: []
-     
-(3) AdaptiveSparkPlan 
+
+(3) AdaptiveSparkPlan
 Output [1]: [(scalarsubquery() + scalarsubquery())#x]
 Arguments: isFinalPlan=false
 
@@ -501,46 +501,46 @@ AdaptiveSparkPlan (9)
             +- Scan parquet default.explain_temp1 (4)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) Scan parquet default.explain_temp1 
+
+(4) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
-(5) Filter 
+
+(5) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
-(6) Project 
+
+(6) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(7) BroadcastExchange 
+
+(7) BroadcastExchange
 Input [2]: [key#x, val#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
-(8) BroadcastHashJoin 
+
+(8) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-     
-(9) AdaptiveSparkPlan 
+
+(9) AdaptiveSparkPlan
 Output [4]: [key#x, val#x, key#x, val#x]
 Arguments: isFinalPlan=false
 
@@ -575,82 +575,82 @@ AdaptiveSparkPlan (15)
                      +- Scan parquet default.explain_temp1 (7)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
-(2) Filter 
+
+(2) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
-(3) Project 
+
+(3) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(4) HashAggregate 
+
+(4) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(5) Exchange 
+
+(5) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(6) HashAggregate 
+
+(6) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
-     
-(7) Scan parquet default.explain_temp1 
+
+(7) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
-(8) Filter 
+
+(8) Filter
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
-(9) Project 
+
+(9) Project
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(10) HashAggregate 
+
+(10) HashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(11) Exchange 
+
+(11) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(12) HashAggregate 
+
+(12) HashAggregate
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
-     
-(13) BroadcastExchange 
+
+(13) BroadcastExchange
 Input [2]: [key#x, max(val)#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
-(14) BroadcastHashJoin 
+
+(14) BroadcastHashJoin
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
 Join condition: None
-     
-(15) AdaptiveSparkPlan 
+
+(15) AdaptiveSparkPlan
 Output [4]: [key#x, max(val)#x, key#x, max(val)#x]
 Arguments: isFinalPlan=false
 
@@ -669,16 +669,16 @@ Execute CreateViewCommand (1)
             +- UnresolvedRelation (3)
 
 
-(1) Execute CreateViewCommand 
+(1) Execute CreateViewCommand
 Output: []
-      
-(2) CreateViewCommand 
+
+(2) CreateViewCommand
 Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
-      
-(3) UnresolvedRelation 
+
+(3) UnresolvedRelation
 Arguments: [explain_temp1]
-      
-(4) Project 
+
+(4) Project
 Arguments: ['key, 'val]
 
 
@@ -699,31 +699,31 @@ AdaptiveSparkPlan (5)
          +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
-(2) HashAggregate 
+
+(2) HashAggregate
 Input [2]: [key#x, val#x]
 Keys: []
 Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
 Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
 Results [3]: [count#xL, sum#xL, count#xL]
-     
-(3) Exchange 
+
+(3) Exchange
 Input [3]: [count#xL, sum#xL, count#xL]
 Arguments: SinglePartition, true, [id=#x]
-      
-(4) HashAggregate 
+
+(4) HashAggregate
 Input [3]: [count#xL, sum#xL, count#xL]
 Keys: []
 Functions [3]: [count(val#x), sum(cast(key#x as bigint)), count(key#x)]
 Aggregate Attributes [3]: [count(val#x)#xL, sum(cast(key#x as bigint))#xL, count(key#x)#xL]
 Results [2]: [(count(val#x)#xL + sum(cast(key#x as bigint))#xL) AS TOTAL#xL, count(key#x)#xL AS count(key) FILTER (WHERE (val > 1))#xL]
-     
-(5) AdaptiveSparkPlan 
+
+(5) AdaptiveSparkPlan
 Output [2]: [TOTAL#xL, count(key) FILTER (WHERE (val > 1))#xL]
 Arguments: isFinalPlan=false
 
@@ -744,31 +744,31 @@ AdaptiveSparkPlan (5)
          +- Scan parquet default.explain_temp4 (1)
 
 
-(1) Scan parquet default.explain_temp4 
+(1) Scan parquet default.explain_temp4
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp4]
 ReadSchema: struct<key:int,val:string>
-     
-(2) ObjectHashAggregate 
+
+(2) ObjectHashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_collect_set(val#x, 0, 0)]
 Aggregate Attributes [1]: [buf#x]
 Results [2]: [key#x, buf#x]
-     
-(3) Exchange 
+
+(3) Exchange
 Input [2]: [key#x, buf#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(4) ObjectHashAggregate 
+
+(4) ObjectHashAggregate
 Input [2]: [key#x, buf#x]
 Keys [1]: [key#x]
 Functions [1]: [collect_set(val#x, 0, 0)]
 Aggregate Attributes [1]: [collect_set(val#x, 0, 0)#x]
 Results [2]: [key#x, sort_array(collect_set(val#x, 0, 0)#x, true)[0] AS sort_array(collect_set(val), true)[0]#x]
-     
-(5) AdaptiveSparkPlan 
+
+(5) AdaptiveSparkPlan
 Output [2]: [key#x, sort_array(collect_set(val), true)[0]#x]
 Arguments: isFinalPlan=false
 
@@ -791,39 +791,39 @@ AdaptiveSparkPlan (7)
                +- Scan parquet default.explain_temp4 (1)
 
 
-(1) Scan parquet default.explain_temp4 
+(1) Scan parquet default.explain_temp4
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp4]
 ReadSchema: struct<key:int,val:string>
-     
-(2) Sort 
+
+(2) Sort
 Input [2]: [key#x, val#x]
 Arguments: [key#x ASC NULLS FIRST], false, 0
-      
-(3) SortAggregate 
+
+(3) SortAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_min(val#x)]
 Aggregate Attributes [1]: [min#x]
 Results [2]: [key#x, min#x]
-     
-(4) Exchange 
+
+(4) Exchange
 Input [2]: [key#x, min#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(5) Sort 
+
+(5) Sort
 Input [2]: [key#x, min#x]
 Arguments: [key#x ASC NULLS FIRST], false, 0
-      
-(6) SortAggregate 
+
+(6) SortAggregate
 Input [2]: [key#x, min#x]
 Keys [1]: [key#x]
 Functions [1]: [min(val#x)]
 Aggregate Attributes [1]: [min(val#x)#x]
 Results [2]: [key#x, min(val#x)#x AS min(val)#x]
-     
-(7) AdaptiveSparkPlan 
+
+(7) AdaptiveSparkPlan
 Output [2]: [key#x, min(val)#x]
 Arguments: isFinalPlan=false
 
diff --git a/sql/core/src/test/resources/sql-tests/results/explain.sql.out b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
index 1a18d563588c0..2b07dac0e5d0a 100644
--- a/sql/core/src/test/resources/sql-tests/results/explain.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/explain.sql.out
@@ -64,46 +64,46 @@ struct<plan:string>
                      +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (4) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
+
 (5) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(6) Exchange 
+
+(6) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
+
 (7) HashAggregate [codegen id : 2]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
-     
-(8) Exchange 
+
+(8) Exchange
 Input [2]: [key#x, max(val)#x]
 Arguments: rangepartitioning(key#x ASC NULLS FIRST, 4), true, [id=#x]
-      
+
 (9) Sort [codegen id : 3]
 Input [2]: [key#x, max(val)#x]
 Arguments: [key#x ASC NULLS FIRST], true, 0
@@ -131,46 +131,46 @@ struct<plan:string>
                      +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (4) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
+
 (5) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(6) Exchange 
+
+(6) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
+
 (7) HashAggregate [codegen id : 2]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [3]: [key#x, max(val#x)#x AS max(val)#x, max(val#x)#x AS max(val#x)#x]
-     
+
 (8) Filter [codegen id : 2]
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
 Condition : (isnotnull(max(val#x)#x) AND (max(val#x)#x > 0))
-     
+
 (9) Project [codegen id : 2]
 Output [2]: [key#x, max(val)#x]
 Input [3]: [key#x, max(val)#x, max(val#x)#x]
@@ -199,55 +199,55 @@ struct<plan:string>
                   +- Scan parquet default.explain_temp1 (5)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (4) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(5) Scan parquet default.explain_temp1 
+
+(5) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 2]
 Input [2]: [key#x, val#x]
-      
+
 (7) Filter [codegen id : 2]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 0))
-     
+
 (8) Project [codegen id : 2]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(9) Union 
-      
+
+(9) Union
+
 (10) HashAggregate [codegen id : 3]
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
 Functions: []
 Aggregate Attributes: []
 Results [2]: [key#x, val#x]
-     
-(11) Exchange 
+
+(11) Exchange
 Input [2]: [key#x, val#x]
 Arguments: hashpartitioning(key#x, val#x, 4), true, [id=#x]
-      
+
 (12) HashAggregate [codegen id : 4]
 Input [2]: [key#x, val#x]
 Keys [2]: [key#x, val#x]
@@ -278,46 +278,46 @@ struct<plan:string>
             +- Scan parquet default.explain_temp2 (5)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 2]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 2]
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
+
 (4) Project [codegen id : 2]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(5) Scan parquet default.explain_temp2 
+
+(5) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (7) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
+
 (8) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(9) BroadcastExchange 
+
+(9) BroadcastExchange
 Input [2]: [key#x, val#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
+
 (10) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
@@ -344,37 +344,37 @@ struct<plan:string>
             +- Scan parquet default.explain_temp2 (3)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 2]
 Input [2]: [key#x, val#x]
-      
-(3) Scan parquet default.explain_temp2 
+
+(3) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (4) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (5) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : isnotnull(key#x)
-     
+
 (6) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(7) BroadcastExchange 
+
+(7) BroadcastExchange
 Input [2]: [key#x, val#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
+
 (8) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
@@ -402,24 +402,24 @@ struct<plan:string>
       +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), IsNotNull(val), GreaterThan(val,3)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x > 3))
-     
+
 (4) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
+
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
@@ -432,42 +432,42 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp2 (5)
 
 
-(5) Scan parquet default.explain_temp2 
+(5) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(key), IsNotNull(val), EqualTo(val,2)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (7) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (((isnotnull(key#x) AND isnotnull(val#x)) AND (key#x = Subquery scalar-subquery#x, [id=#x])) AND (val#x = 2))
-     
+
 (8) Project [codegen id : 1]
 Output [1]: [key#x]
 Input [2]: [key#x, val#x]
-     
+
 (9) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_max(key#x)]
 Aggregate Attributes [1]: [max#x]
 Results [1]: [max#x]
-     
-(10) Exchange 
+
+(10) Exchange
 Input [1]: [max#x]
 Arguments: SinglePartition, true, [id=#x]
-      
+
 (11) HashAggregate [codegen id : 2]
 Input [1]: [max#x]
 Keys: []
 Functions [1]: [max(key#x)]
 Aggregate Attributes [1]: [max(key#x)#x]
 Results [1]: [max(key#x)#x AS max(key)#x]
-     
+
 Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (18)
 +- Exchange (17)
@@ -478,35 +478,35 @@ Subquery:2 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp3 (12)
 
 
-(12) Scan parquet default.explain_temp3 
+(12) Scan parquet default.explain_temp3
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (13) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (14) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
-     
+
 (15) Project [codegen id : 1]
 Output [1]: [key#x]
 Input [2]: [key#x, val#x]
-     
+
 (16) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_max(key#x)]
 Aggregate Attributes [1]: [max#x]
 Results [1]: [max#x]
-     
-(17) Exchange 
+
+(17) Exchange
 Input [1]: [max#x]
 Arguments: SinglePartition, true, [id=#x]
-      
+
 (18) HashAggregate [codegen id : 2]
 Input [1]: [max#x]
 Keys: []
@@ -535,19 +535,19 @@ struct<plan:string>
    +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : ((key#x = Subquery scalar-subquery#x, [id=#x]) OR (cast(key#x as double) = Subquery scalar-subquery#x, [id=#x]))
-     
+
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
@@ -560,42 +560,42 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp2 (4)
 
 
-(4) Scan parquet default.explain_temp2 
+(4) Scan parquet default.explain_temp2
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp2]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (5) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (6) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
-     
+
 (7) Project [codegen id : 1]
 Output [1]: [key#x]
 Input [2]: [key#x, val#x]
-     
+
 (8) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_max(key#x)]
 Aggregate Attributes [1]: [max#x]
 Results [1]: [max#x]
-     
-(9) Exchange 
+
+(9) Exchange
 Input [1]: [max#x]
 Arguments: SinglePartition, true, [id=#x]
-      
+
 (10) HashAggregate [codegen id : 2]
 Input [1]: [max#x]
 Keys: []
 Functions [1]: [max(key#x)]
 Aggregate Attributes [1]: [max(key#x)#x]
 Results [1]: [max(key#x)#x AS max(key)#x]
-     
+
 Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
 * HashAggregate (17)
 +- Exchange (16)
@@ -606,35 +606,35 @@ Subquery:2 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
                +- Scan parquet default.explain_temp3 (11)
 
 
-(11) Scan parquet default.explain_temp3 
+(11) Scan parquet default.explain_temp3
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp3]
 PushedFilters: [IsNotNull(val), GreaterThan(val,0)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (12) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (13) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(val#x) AND (val#x > 0))
-     
+
 (14) Project [codegen id : 1]
 Output [1]: [key#x]
 Input [2]: [key#x, val#x]
-     
+
 (15) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_avg(cast(key#x as bigint))]
 Aggregate Attributes [2]: [sum#x, count#xL]
 Results [2]: [sum#x, count#xL]
-     
-(16) Exchange 
+
+(16) Exchange
 Input [2]: [sum#x, count#xL]
 Arguments: SinglePartition, true, [id=#x]
-      
+
 (17) HashAggregate [codegen id : 2]
 Input [2]: [sum#x, count#xL]
 Keys: []
@@ -656,19 +656,19 @@ struct<plan:string>
    +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output: []
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input: []
-      
+
 (3) Project [codegen id : 1]
 Output [1]: [(Subquery scalar-subquery#x, [id=#x] + ReusedSubquery Subquery scalar-subquery#x, [id=#x]) AS (scalarsubquery() + scalarsubquery())#x]
 Input: []
-     
+
 ===== Subqueries =====
 
 Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery#x, [id=#x]
@@ -679,33 +679,33 @@ Subquery:1 Hosting operator id = 3 Hosting Expression = Subquery scalar-subquery
          +- Scan parquet default.explain_temp1 (4)
 
 
-(4) Scan parquet default.explain_temp1 
+(4) Scan parquet default.explain_temp1
 Output [1]: [key#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int>
-     
+
 (5) ColumnarToRow [codegen id : 1]
 Input [1]: [key#x]
-      
+
 (6) HashAggregate [codegen id : 1]
 Input [1]: [key#x]
 Keys: []
 Functions [1]: [partial_avg(cast(key#x as bigint))]
 Aggregate Attributes [2]: [sum#x, count#xL]
 Results [2]: [sum#x, count#xL]
-     
-(7) Exchange 
+
+(7) Exchange
 Input [2]: [sum#x, count#xL]
 Arguments: SinglePartition, true, [id=#x]
-      
+
 (8) HashAggregate [codegen id : 2]
 Input [2]: [sum#x, count#xL]
 Keys: []
 Functions [1]: [avg(cast(key#x as bigint))]
 Aggregate Attributes [1]: [avg(cast(key#x as bigint))#x]
 Results [1]: [avg(cast(key#x as bigint))#x AS avg(key)#x]
-     
+
 Subquery:2 Hosting operator id = 3 Hosting Expression = ReusedSubquery Subquery scalar-subquery#x, [id=#x]
 
 
@@ -733,46 +733,46 @@ struct<plan:string>
             +- Scan parquet default.explain_temp1 (5)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 2]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 2]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
+
 (4) Project [codegen id : 2]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(5) Scan parquet default.explain_temp1 
+
+(5) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (6) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (7) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
+
 (8) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
-(9) BroadcastExchange 
+
+(9) BroadcastExchange
 Input [2]: [key#x, val#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
+
 (10) BroadcastHashJoin [codegen id : 2]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
@@ -805,56 +805,56 @@ struct<plan:string>
       +- ReusedExchange (8)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 PushedFilters: [IsNotNull(key), GreaterThan(key,10)]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Filter [codegen id : 1]
 Input [2]: [key#x, val#x]
 Condition : (isnotnull(key#x) AND (key#x > 10))
-     
+
 (4) Project [codegen id : 1]
 Output [2]: [key#x, val#x]
 Input [2]: [key#x, val#x]
-     
+
 (5) HashAggregate [codegen id : 1]
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_max(val#x)]
 Aggregate Attributes [1]: [max#x]
 Results [2]: [key#x, max#x]
-     
-(6) Exchange 
+
+(6) Exchange
 Input [2]: [key#x, max#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
+
 (7) HashAggregate [codegen id : 4]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
-     
-(8) ReusedExchange  [Reuses operator id: 6]
+
+(8) ReusedExchange [Reuses operator id: 6]
 Output [2]: [key#x, max#x]
-     
+
 (9) HashAggregate [codegen id : 3]
 Input [2]: [key#x, max#x]
 Keys [1]: [key#x]
 Functions [1]: [max(val#x)]
 Aggregate Attributes [1]: [max(val#x)#x]
 Results [2]: [key#x, max(val#x)#x AS max(val)#x]
-     
-(10) BroadcastExchange 
+
+(10) BroadcastExchange
 Input [2]: [key#x, max(val)#x]
 Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint))), [id=#x]
-      
+
 (11) BroadcastHashJoin [codegen id : 4]
 Left keys [1]: [key#x]
 Right keys [1]: [key#x]
@@ -875,16 +875,16 @@ Execute CreateViewCommand (1)
             +- UnresolvedRelation (3)
 
 
-(1) Execute CreateViewCommand 
+(1) Execute CreateViewCommand
 Output: []
-      
-(2) CreateViewCommand 
+
+(2) CreateViewCommand
 Arguments: `default`.`explain_view`, SELECT key, val FROM explain_temp1, false, false, PersistedView
-      
-(3) UnresolvedRelation 
+
+(3) UnresolvedRelation
 Arguments: [explain_temp1]
-      
-(4) Project 
+
+(4) Project
 Arguments: ['key, 'val]
 
 
@@ -905,26 +905,26 @@ struct<plan:string>
          +- Scan parquet default.explain_temp1 (1)
 
 
-(1) Scan parquet default.explain_temp1 
+(1) Scan parquet default.explain_temp1
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp1]
 ReadSchema: struct<key:int,val:int>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
-(3) HashAggregate 
+
+(3) HashAggregate
 Input [2]: [key#x, val#x]
 Keys: []
 Functions [3]: [partial_count(val#x), partial_sum(cast(key#x as bigint)), partial_count(key#x) FILTER (WHERE (val#x > 1))]
 Aggregate Attributes [3]: [count#xL, sum#xL, count#xL]
 Results [3]: [count#xL, sum#xL, count#xL]
-     
-(4) Exchange 
+
+(4) Exchange
 Input [3]: [count#xL, sum#xL, count#xL]
 Arguments: SinglePartition, true, [id=#x]
-      
+
 (5) HashAggregate [codegen id : 2]
 Input [3]: [count#xL, sum#xL, count#xL]
 Keys: []
@@ -949,27 +949,27 @@ ObjectHashAggregate (5)
          +- Scan parquet default.explain_temp4 (1)
 
 
-(1) Scan parquet default.explain_temp4 
+(1) Scan parquet default.explain_temp4
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp4]
 ReadSchema: struct<key:int,val:string>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
-(3) ObjectHashAggregate 
+
+(3) ObjectHashAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_collect_set(val#x, 0, 0)]
 Aggregate Attributes [1]: [buf#x]
 Results [2]: [key#x, buf#x]
-     
-(4) Exchange 
+
+(4) Exchange
 Input [2]: [key#x, buf#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
-(5) ObjectHashAggregate 
+
+(5) ObjectHashAggregate
 Input [2]: [key#x, buf#x]
 Keys [1]: [key#x]
 Functions [1]: [collect_set(val#x, 0, 0)]
@@ -995,35 +995,35 @@ SortAggregate (7)
                +- Scan parquet default.explain_temp4 (1)
 
 
-(1) Scan parquet default.explain_temp4 
+(1) Scan parquet default.explain_temp4
 Output [2]: [key#x, val#x]
 Batched: true
 Location [not included in comparison]/{warehouse_dir}/explain_temp4]
 ReadSchema: struct<key:int,val:string>
-     
+
 (2) ColumnarToRow [codegen id : 1]
 Input [2]: [key#x, val#x]
-      
+
 (3) Sort [codegen id : 1]
 Input [2]: [key#x, val#x]
 Arguments: [key#x ASC NULLS FIRST], false, 0
-      
-(4) SortAggregate 
+
+(4) SortAggregate
 Input [2]: [key#x, val#x]
 Keys [1]: [key#x]
 Functions [1]: [partial_min(val#x)]
 Aggregate Attributes [1]: [min#x]
 Results [2]: [key#x, min#x]
-     
-(5) Exchange 
+
+(5) Exchange
 Input [2]: [key#x, min#x]
 Arguments: hashpartitioning(key#x, 4), true, [id=#x]
-      
+
 (6) Sort [codegen id : 2]
 Input [2]: [key#x, min#x]
 Arguments: [key#x ASC NULLS FIRST], false, 0
-      
-(7) SortAggregate 
+
+(7) SortAggregate
 Input [2]: [key#x, min#x]
 Keys [1]: [key#x]
 Functions [1]: [min(val#x)]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index a1b6d71fb3803..d41d624f1762d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -354,8 +354,6 @@ class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuit
     val testDf = df1.join(df2, "k").groupBy("k").agg(count("v1"), sum("v1"), avg("v2"))
     // trigger the final plan for AQE
     testDf.collect()
-    // whitespace
-    val ws = " "
     //   == Physical Plan ==
     //   AdaptiveSparkPlan (14)
     //   +- * HashAggregate (13)
@@ -375,22 +373,22 @@ class ExplainSuiteAE extends ExplainSuiteHelper with EnableAdaptiveExecutionSuit
       testDf,
       FormattedMode,
       s"""
-         |(6) BroadcastQueryStage$ws
+         |(6) BroadcastQueryStage
          |Output [2]: [k#x, v2#x]
          |Arguments: 0
          |""".stripMargin,
       s"""
-         |(11) ShuffleQueryStage$ws
+         |(11) ShuffleQueryStage
          |Output [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
          |Arguments: 1
          |""".stripMargin,
       s"""
-         |(12) CustomShuffleReader$ws
+         |(12) CustomShuffleReader
          |Input [5]: [k#x, count#xL, sum#xL, sum#x, count#xL]
          |Arguments: coalesced
          |""".stripMargin,
       s"""
-         |(14) AdaptiveSparkPlan$ws
+         |(14) AdaptiveSparkPlan
          |Output [4]: [k#x, count(v1)#xL, sum(v1)#xL, avg(v2)#x]
          |Arguments: isFinalPlan=true
          |""".stripMargin

From 6f10c8a9f44586b0d9e40b4cc0e2700a071186a0 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 27 Apr 2020 09:45:00 -0500
Subject: [PATCH 0579/1280] [SPARK-31569][SQL][DOCS] Add links to subsections
 in SQL Reference main page

### What changes were proposed in this pull request?
Add links to subsections in SQL Reference main page

### Why are the changes needed?
Make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes
before:
<img width="1050" alt="Screen Shot 2020-04-26 at 10 52 42 PM" src="https://user-images.githubusercontent.com/13592258/80338238-a9551080-8810-11ea-8ae8-d6707fde2cac.png">

after:
<img width="1050" alt="Screen Shot 2020-04-26 at 10 51 58 PM" src="https://user-images.githubusercontent.com/13592258/80338241-ac500100-8810-11ea-8518-95c4f8c0a2eb.png">

### How was this patch tested?
Manually build and check.

Closes #28360 from huaxingao/sql-ref.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 7735db2a273edf208ae50e88926c9f7a77e5dbac)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/sql-ref.md | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/docs/sql-ref.md b/docs/sql-ref.md
index 6c57b0d6fdf6f..db51fe1978eec 100644
--- a/docs/sql-ref.md
+++ b/docs/sql-ref.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Reference
-displayTitle: Reference
+title: SQL Reference
+displayTitle: SQL Reference
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -19,7 +19,21 @@ license: |
   limitations under the License.
 ---
 
-Spark SQL is Apache Spark's module for working with structured data.
-This guide is a reference for Structured Query Language (SQL) for Apache 
-Spark. This document describes the SQL constructs supported by Spark in detail
-along with usage examples when applicable.
+Spark SQL is Apache Spark's module for working with structured data. This guide is a reference for Structured Query Language (SQL) and includes syntax, semantics, keywords, and examples for common SQL usage. It contains information for the following topics:
+
+ * [Data Types](sql-ref-datatypes.html)
+ * [Identifiers](sql-ref-identifier.html)
+ * [Literals](sql-ref-literals.html)
+ * [Null Semanitics](sql-ref-null-semantics.html)
+ * [ANSI Compliance](sql-ref-ansi-compliance.html)
+ * [SQL Syntax](sql-ref-syntax.html)
+   * [DDL Statements](sql-ref-syntax-ddl.html)
+   * [DML Statements](sql-ref-syntax-ddl.html)
+   * [Data Retrieval Statements](sql-ref-syntax-qry.html)
+   * [Auxiliary Statements](sql-ref-syntax-aux.html)
+ * [Functions](sql-ref-functions.html)
+   * [Built-in Functions](sql-ref-functions-builtin.html)
+   * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+   * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+   * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
+ * [Datetime Pattern](sql-ref-datetime-pattern.html)

From ce540b96efcb47d1869e30f4b2e4967af01cb21f Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Mon, 27 Apr 2020 13:34:43 -0700
Subject: [PATCH 0580/1280] [SPARK-31565][WEBUI][FOLLOWUP] Add font color
 setting of DAG-viz for query plan

### What changes were proposed in this pull request?

This PR adds a font color setting of DAG-viz for query plan.

### Why are the changes needed?

#28352 aimed to unify the font color of all DAG-viz in WebUI but there is one part left over.

Before this change applied, the appearance of a query plan is like as follows.
<img width="456" alt="plan-graph-fixed" src="https://user-images.githubusercontent.com/4736016/80325600-ca4d4e00-8870-11ea-945c-64971dbb752c.png">
The color of `WholeStageCodegen (1)` and its following text (`duration: total...`) is slightly darker than `SerializeFromObject`.

After this change, those color is unified as `#333333`.
<img width="450" alt="plan-graph-fixed2" src="https://user-images.githubusercontent.com/4736016/80325651-fb2d8300-8870-11ea-8ed8-178c124d224c.png">

### Does this PR introduce any user-facing change?

Slightly yes.

### How was this patch tested?

I confirmed the style of `fill` and `color` is `#333333` by debug console of Chrome.
<img width="321" alt="fill" src="https://user-images.githubusercontent.com/4736016/80325760-6c6d3600-8871-11ea-82e7-e789bf741f2a.png">
<img width="316" alt="color" src="https://user-images.githubusercontent.com/4736016/80325765-70995380-8871-11ea-8976-7020205d585c.png">

Closes #28355 from sarutak/followup-SPARK-31565.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7d4d05c684d15a7ab136773c27af1ec0ec8a9930)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/sql/execution/ui/static/spark-sql-viz.css    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
index 25198db6fb8f2..aea5e2e93c661 100644
--- a/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
+++ b/sql/core/src/main/resources/org/apache/spark/sql/execution/ui/static/spark-sql-viz.css
@@ -38,6 +38,10 @@
   font-weight: bold;
 }
 
+#plan-viz-graph svg text {
+  fill: #333;
+}
+
 #plan-viz-graph svg path {
   stroke: #444;
   stroke-width: 1.5px;

From d2724828dfdca9d3e45590078ab525bd73b3c17e Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Mon, 27 Apr 2020 15:07:52 -0700
Subject: [PATCH 0581/1280] [SPARK-27340][SS] Alias on TimeWindow expression
 cause watermark metadata lost

Credit to LiangchangZ, this PR reuses the UT as well as integrate test in #24457. Thanks Liangchang for your solid work.

### What changes were proposed in this pull request?
Make metadata propagatable between Aliases.

### Why are the changes needed?
In Structured Streaming, we added an Alias for TimeWindow by default.
https://github.com/apache/spark/blob/590b9a0132b68d9523e663997def957b2e46dfb1/sql/core/src/main/scala/org/apache/spark/sql/functions.scala#L3272-L3273
For some cases like stream join with watermark and window, users need to add an alias for convenience(we also added one in StreamingJoinSuite). The current metadata handling logic for `as` will lose the watermark metadata
https://github.com/apache/spark/blob/590b9a0132b68d9523e663997def957b2e46dfb1/sql/core/src/main/scala/org/apache/spark/sql/Column.scala#L1049-L1054
 and finally cause the AnalysisException:
```
Stream-stream outer join between two streaming DataFrame/Datasets is not supported without a watermark in the join keys, or a watermark on the nullable side and an appropriate range condition
```

### Does this PR introduce any user-facing change?
Bugfix for an alias on time window with watermark.

### How was this patch tested?
New UTs added. One for the functionality and one for explaining the common scenario.

Closes #28326 from xuanyuanking/SPARK-27340.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ba7adc494923de8104ab37d412edd78afe540f45)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/sql/Column.scala   | 14 +---------
 .../streaming/EventTimeWatermarkSuite.scala   | 11 ++++++++
 .../sql/streaming/StreamingJoinSuite.scala    | 26 +++++++++++++++++++
 3 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 49c9f830fb27e..50bc7ec5f2afa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -972,9 +972,6 @@ class Column(val expr: Expression) extends Logging {
    *   df.select($"colA".as("colB"))
    * }}}
    *
-   * If the current column has metadata associated with it, this metadata will be propagated
-   * to the new column.  If this not desired, use `as` with explicitly empty metadata.
-   *
    * @group expr_ops
    * @since 1.3.0
    */
@@ -1011,9 +1008,6 @@ class Column(val expr: Expression) extends Logging {
    *   df.select($"colA".as('colB))
    * }}}
    *
-   * If the current column has metadata associated with it, this metadata will be propagated
-   * to the new column.  If this not desired, use `as` with explicitly empty metadata.
-   *
    * @group expr_ops
    * @since 1.3.0
    */
@@ -1040,17 +1034,11 @@ class Column(val expr: Expression) extends Logging {
    *   df.select($"colA".name("colB"))
    * }}}
    *
-   * If the current column has metadata associated with it, this metadata will be propagated
-   * to the new column.  If this not desired, use `as` with explicitly empty metadata.
-   *
    * @group expr_ops
    * @since 2.0.0
    */
   def name(alias: String): Column = withExpr {
-    normalizedExpr() match {
-      case ne: NamedExpression => Alias(expr, alias)(explicitMetadata = Some(ne.metadata))
-      case other => Alias(other, alias)()
-    }
+    Alias(expr, alias)()
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
index f29a6c7f77079..231317705f3a8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
@@ -593,6 +593,17 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     }
   }
 
+  test("SPARK-27340 Alias on TimeWindow expression cause watermark metadata lost") {
+    val inputData = MemoryStream[Int]
+    val aliasWindow = inputData.toDF()
+      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withWatermark("eventTime", "10 seconds")
+      .select(window($"eventTime", "5 seconds") as 'aliasWindow)
+    // Check the eventTime metadata is kept in the top level alias.
+    assert(aliasWindow.logicalPlan.output.exists(
+      _.metadata.contains(EventTimeWatermark.delayKey)))
+  }
+
   test("test no-data flag") {
     val flagKey = SQLConf.STREAMING_NO_DATA_MICRO_BATCHES_ENABLED.key
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
index 3f218c9cb7fd9..33f899b98373f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
@@ -991,4 +991,30 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
       )
     }
   }
+
+  test("SPARK-27340 Windowed left out join with Alias on TimeWindow") {
+    val (leftInput, df1) = setupStream("left", 2)
+    val (rightInput, df2) = setupStream("right", 3)
+    val left = df1.select('key, window('leftTime, "10 second") as 'leftWindow, 'leftValue)
+    val right = df2.select('key, window('rightTime, "10 second") as 'rightWindow, 'rightValue)
+    val joined = left.join(
+      right,
+      left("key") === right("key") && left("leftWindow") === right("rightWindow"),
+      "left_outer")
+      .select(left("key"), $"leftWindow.end".cast("long"), 'leftValue, 'rightValue)
+
+    testStream(joined)(
+      // Test inner part of the join.
+      MultiAddData(leftInput, 1, 2, 3, 4, 5)(rightInput, 3, 4, 5, 6, 7),
+      CheckNewAnswer((3, 10, 6, 9), (4, 10, 8, 12), (5, 10, 10, 15)),
+
+      MultiAddData(leftInput, 21)(rightInput, 22), // watermark = 11, no-data-batch computes nulls
+      CheckNewAnswer(Row(1, 10, 2, null), Row(2, 10, 4, null)),
+      assertNumStateRows(total = 2, updated = 12),
+
+      AddData(leftInput, 22),
+      CheckNewAnswer(Row(22, 30, 44, 66)),
+      assertNumStateRows(total = 3, updated = 1)
+    )
+  }
 }

From fae25030de002099f5ad3e741e4fec1281ea29ab Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 27 Apr 2020 16:47:39 -0700
Subject: [PATCH 0582/1280] [SPARK-31577][SQL] Fix case-sensitivity and forward
 name conflict problems when check name conflicts of CTE relations

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/28318, to make the code more readable, by adding some comments to explain the trick and simplify the code to use a boolean flag instead of 2 string sets.

This PR also fixes various problems:
1. the name check should consider case sensitivity
2. forward name conflicts like `with t as (with t2 as ...), t2 as ...` is not a real conflict and we shouldn't fail.

### Why are the changes needed?

correct the behavior

### Does this PR introduce any user-facing change?

yes, fix the fore-mentioned behaviors.

### How was this patch tested?

new tests

Closes #28371 from cloud-fan/followup.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 2f4f38b6f16cdc4bdb2f78c671707760cf429e4f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/analysis/CTESubstitution.scala   | 48 +++++++++++-------
 .../resources/sql-tests/inputs/cte-nested.sql | 25 ++++++++++
 .../sql-tests/results/cte-legacy.sql.out      | 42 +++++++++++++++-
 .../sql-tests/results/cte-nested.sql.out      | 49 +++++++++++++++++--
 .../sql-tests/results/cte-nonlegacy.sql.out   | 42 +++++++++++++++-
 5 files changed, 182 insertions(+), 24 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
index d9fdb56739e00..41e4c1fc39303 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala
@@ -17,9 +17,11 @@
 
 package org.apache.spark.sql.catalyst.analysis
 
+import scala.collection.mutable
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, With}
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias, With}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.{LEGACY_CTE_PRECEDENCE_POLICY, LegacyBehaviorPolicy}
@@ -41,34 +43,44 @@ object CTESubstitution extends Rule[LogicalPlan] {
   }
 
   /**
-   * Check the plan to be traversed has naming conflicts in nested CTE or not, traverse through
-   * child, innerChildren and subquery expressions for the current plan.
+   * Spark 3.0 changes the CTE relations resolution, and inner relations take precedence. This is
+   * correct but we need to warn users about this behavior change under EXCEPTION mode, when we see
+   * CTE relations with conflicting names.
+   *
+   * Note that, before Spark 3.0 the parser didn't support CTE in the FROM clause. For example,
+   * `WITH ... SELECT * FROM (WITH ... SELECT ...)` was not supported. We should not fail for this
+   * case, as Spark versions before 3.0 can't run it anyway. The parameter `startOfQuery` is used
+   * to indicate where we can define CTE relations before Spark 3.0, and we should only check
+   * name conflicts when `startOfQuery` is true.
    */
   private def assertNoNameConflictsInCTE(
       plan: LogicalPlan,
-      outerCTERelationNames: Set[String] = Set.empty,
-      namesInSubqueries: Set[String] = Set.empty): Unit = {
+      outerCTERelationNames: Seq[String] = Nil,
+      startOfQuery: Boolean = true): Unit = {
+    val resolver = SQLConf.get.resolver
     plan match {
-      case w @ With(child, relations) =>
-        val newNames = relations.map {
-          case (cteName, _) =>
-            if (outerCTERelationNames.contains(cteName)) {
-              throw new AnalysisException(s"Name $cteName is ambiguous in nested CTE. " +
+      case With(child, relations) =>
+        val newNames = mutable.ArrayBuffer.empty[String]
+        newNames ++= outerCTERelationNames
+        relations.foreach {
+          case (name, relation) =>
+            if (startOfQuery && outerCTERelationNames.exists(resolver(_, name))) {
+              throw new AnalysisException(s"Name $name is ambiguous in nested CTE. " +
                 s"Please set ${LEGACY_CTE_PRECEDENCE_POLICY.key} to CORRECTED so that name " +
                 "defined in inner CTE takes precedence. If set it to LEGACY, outer CTE " +
                 "definitions will take precedence. See more details in SPARK-28228.")
-            } else {
-              cteName
             }
-        }.toSet ++ namesInSubqueries
-        assertNoNameConflictsInCTE(child, outerCTERelationNames, newNames)
-        w.innerChildren.foreach(assertNoNameConflictsInCTE(_, newNames, newNames))
+            // CTE relation is defined as `SubqueryAlias`. Here we skip it and check the child
+            // directly, so that `startOfQuery` is set correctly.
+            assertNoNameConflictsInCTE(relation.child, newNames)
+            newNames += name
+        }
+        assertNoNameConflictsInCTE(child, newNames, startOfQuery = false)
 
       case other =>
-        other.subqueries.foreach(
-          assertNoNameConflictsInCTE(_, namesInSubqueries, namesInSubqueries))
+        other.subqueries.foreach(assertNoNameConflictsInCTE(_, outerCTERelationNames))
         other.children.foreach(
-          assertNoNameConflictsInCTE(_, outerCTERelationNames, namesInSubqueries))
+          assertNoNameConflictsInCTE(_, outerCTERelationNames, startOfQuery = false))
     }
   }
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
index 134f88bf3302f..3b64b5daa82db 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cte-nested.sql
@@ -111,3 +111,28 @@ WHERE c IN (
   WITH t(c) AS (SELECT 2)
   SELECT * FROM t
 );
+
+-- forward name conflict is not a real conflict
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t;
+
+-- case insensitive name conflicts: in other CTE relations
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t;
+
+-- case insensitive name conflicts: in subquery expressions
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+);
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
index 629daf7410e5a..4d0e5ea829d3f 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 16
 
 
 -- !query
@@ -179,3 +179,43 @@ WHERE c IN (
 struct<c:int>
 -- !query output
 1
+
+
+-- !query
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+1
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
index 34e9be0ca0003..2f736c7b4978f 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 16
 
 
 -- !query
@@ -64,10 +64,9 @@ WITH
   )
 SELECT * FROM t2
 -- !query schema
-struct<>
+struct<scalarsubquery():int>
 -- !query output
-org.apache.spark.sql.AnalysisException
-Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+2
 
 
 -- !query
@@ -186,3 +185,45 @@ struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
 Name t is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
+
+
+-- !query
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Name aBc is ambiguous in nested CTE. Please set spark.sql.legacy.ctePrecedencePolicy to CORRECTED so that name defined in inner CTE takes precedence. If set it to LEGACY, outer CTE definitions will take precedence. See more details in SPARK-28228.;
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
index 6eba1ad72523d..74394ee3ffc89 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 16
 
 
 -- !query
@@ -179,3 +179,43 @@ WHERE c IN (
 struct<c:int>
 -- !query output
 
+
+
+-- !query
+WITH
+  t AS (
+    WITH t2 AS (SELECT 1)
+    SELECT * FROM t2
+  ),
+  t2 AS (SELECT 2)
+SELECT * FROM t
+-- !query schema
+struct<1:int>
+-- !query output
+1
+
+
+-- !query
+WITH
+  abc AS (SELECT 1),
+  t AS (
+    WITH aBc AS (SELECT 2)
+    SELECT * FROM aBC
+  )
+SELECT * FROM t
+-- !query schema
+struct<2:int>
+-- !query output
+2
+
+
+-- !query
+WITH abc AS (SELECT 1)
+SELECT (
+  WITH aBc AS (SELECT 2)
+  SELECT * FROM aBC
+)
+-- !query schema
+struct<scalarsubquery():int>
+-- !query output
+2

From dae15d68d3feddd08aab4b5b831267d4f11b730a Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 27 Apr 2020 18:56:30 -0700
Subject: [PATCH 0583/1280] [SPARK-31580][BUILD] Upgrade Apache ORC to 1.5.10

### What changes were proposed in this pull request?

This PR aims to upgrade Apache ORC to 1.5.10.

### Why are the changes needed?

Apache ORC 1.5.10 is a maintenance release with the following patches.

- [ORC-621](https://issues.apache.org/jira/browse/ORC-621) Need reader fix for ORC-569
- [ORC-616](https://issues.apache.org/jira/browse/ORC-616) In Patched Base encoding, the value of headerThirdByte goes beyond the range of byte
- [ORC-613](https://issues.apache.org/jira/browse/ORC-613) OrcMapredRecordReader mis-reuse struct object when actual children schema differs
- [ORC-610](https://issues.apache.org/jira/browse/ORC-610) Updated Copyright year in the NOTICE file

The following is release note.
- https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12318320&version=12346912

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the Jenkins with the existing ORC tests and a newly added test case.

- The first commit is already tested in `hive-2.3` profile with both native ORC implementation and Hive 2.3 ORC implementation. (https://github.com/apache/spark/pull/28373#issuecomment-620265114)
- The latest run is about to make the test case disable in `hive-1.2` profile which doesn't use Apache ORC.
  - `hive-1.2`: https://github.com/apache/spark/pull/28373#issuecomment-620325906

Closes #28373 from dongjoon-hyun/SPARK-ORC-1.5.10.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 79eaaaf6daeff6b048a81f4ef60fcc48395a2772)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2         |   6 +++---
 dev/deps/spark-deps-hadoop-2.7-hive-2.3         |   6 +++---
 dev/deps/spark-deps-hadoop-3.2-hive-2.3         |   6 +++---
 pom.xml                                         |   2 +-
 .../TestStringDictionary.testRowIndex.orc       | Bin 0 -> 74580 bytes
 .../datasources/orc/OrcSourceSuite.scala        |   6 ++++++
 .../spark/sql/hive/orc/HiveOrcSourceSuite.scala |   7 +++++++
 7 files changed, 23 insertions(+), 10 deletions(-)
 create mode 100644 sql/core/src/test/resources/test-data/TestStringDictionary.testRowIndex.orc

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 6e8368b36397b..82b0115427735 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -160,9 +160,9 @@ objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.9/nohive/orc-core-1.5.9-nohive.jar
-orc-mapreduce/1.5.9/nohive/orc-mapreduce-1.5.9-nohive.jar
-orc-shims/1.5.9//orc-shims-1.5.9.jar
+orc-core/1.5.10/nohive/orc-core-1.5.10-nohive.jar
+orc-mapreduce/1.5.10/nohive/orc-mapreduce-1.5.10-nohive.jar
+orc-shims/1.5.10//orc-shims-1.5.10.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index beb6c83bfa088..fb2b894df366f 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -175,9 +175,9 @@ objenesis/2.5.1//objenesis-2.5.1.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.9//orc-core-1.5.9.jar
-orc-mapreduce/1.5.9//orc-mapreduce-1.5.9.jar
-orc-shims/1.5.9//orc-shims-1.5.9.jar
+orc-core/1.5.10//orc-core-1.5.10.jar
+orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar
+orc-shims/1.5.10//orc-shims-1.5.10.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 69c7cdf8c0ed3..db74c8960b36b 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -190,9 +190,9 @@ okhttp/2.7.5//okhttp-2.7.5.jar
 okhttp/3.12.6//okhttp-3.12.6.jar
 okio/1.15.0//okio-1.15.0.jar
 opencsv/2.3//opencsv-2.3.jar
-orc-core/1.5.9//orc-core-1.5.9.jar
-orc-mapreduce/1.5.9//orc-mapreduce-1.5.9.jar
-orc-shims/1.5.9//orc-shims-1.5.9.jar
+orc-core/1.5.10//orc-core-1.5.10.jar
+orc-mapreduce/1.5.10//orc-mapreduce-1.5.10.jar
+orc-shims/1.5.10//orc-shims-1.5.10.jar
 oro/2.0.8//oro-2.0.8.jar
 osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
 paranamer/2.8//paranamer-2.8.jar
diff --git a/pom.xml b/pom.xml
index 05479738965d4..f6eca3b65073f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,7 +135,7 @@
     <kafka.version>2.4.1</kafka.version>
     <derby.version>10.12.1.1</derby.version>
     <parquet.version>1.10.1</parquet.version>
-    <orc.version>1.5.9</orc.version>
+    <orc.version>1.5.10</orc.version>
     <orc.classifier></orc.classifier>
     <hive.parquet.group>com.twitter</hive.parquet.group>
     <hive.parquet.version>1.6.0</hive.parquet.version>
diff --git a/sql/core/src/test/resources/test-data/TestStringDictionary.testRowIndex.orc b/sql/core/src/test/resources/test-data/TestStringDictionary.testRowIndex.orc
new file mode 100644
index 0000000000000000000000000000000000000000..cba483d1f070fcfb3a60f23f91ecc677236fb948
GIT binary patch
literal 74580
zcmdqKd0f=x_6MGo+D&s>zvMP(Wpy<nH582{Z{4ixEz?v|LDJL;1aZLy8OSWDTqsQ`
zHL1L|h>lwbqCiT58;~WAv51I@G6EWn2nhV%=bX<r1NZm)`*&Zx8fW>;XP)Od+xxuF
zc^*cMdA`SOx0O$In|kq`*Y0}r+_x**-8QD(ZPWA9?>Y4Jz01n(c(Bi~-o3Yc`}If9
z{d6w#%&R$ft@ydaOV9ii@bq1q7v4VO%?G<&d-=2SV~1-i|0>HU@j4YY<f9aO`s<mt
zOPQ}fpZWOKy!Se8Tk>0vVNd#W_w=hA9`8FYqOWiFHR*#NU$$$MSK!?(UZW4BoSYH+
zONWo{eP?`!6@gt=%$f3V$9MmhKk&^5*WPJ0?Y<d1rTx42KfU_s;|;ymc%3g#?scQ)
z<wO4-Kk0$!$xZEU&uiyX60|$%U-SQ2W=;Lu2>)I`j$8GE+k(=ig(vby{W38y|Jc8?
zUwtflcuRe0OJibeOH*dc)yrj<uUx)(x$1JnykQOA4Z|9$F83~P{omiqTmRRmy!C(m
z%Ul2V>Fguk(|Qc-(lu_y^Vz-2>E-l|17Gd>{fgJK|6cz2>|^xOuZ{mQ2Dbh$W?-xT
zjxRiS>-wy&@v+Aj-kOwMUOLd+vhMoLlodId(|wxzmtP;%A<x}A_S~%{wH^O||K$9w
zk87XEOZ1NYUwu+`>%&^NykPIx|J5huw^r5;$t(4a{U3ee9}#oz*3b1zYVWA%m}lDP
z?)|^;#sd{k<c->x=<Po{=KQT+>OZdiA9$nX+^waxraXV||Jf(!Z!NF&%**s{`Colf
zb}O#dJulX~<$v=@xz8(?=Fc9M=KJNU@KN7wE?RTt@{MV2yu7nV*YX4FSInOLO1Afo
z$KTo*ux3D>drS7B5!nOslDg)2@0ig8pU+-V8-h1}kDG=k1G5)->0cW4@71{S=))BS
z`e!~DU)JxYem1SI{w4L7e*a1Tk~)e}w_ynCmy{p#N&dIX4cFbbDt3P5Qj_iE!xyr~
zbzd5t8uHesrhu@{f0nI2Hw}Pgu3uoq(VvI9TdT7V#wE?@;a!pS&=h(sC<(k?wv|6>
z`6ioyO?|{yZ}TT>>07V&x}(Chv9@d&uupwN<n(hJO$jrfZ1tXP_tBs0@2H4Q&09m?
z3hIhmM-D&I`n?}z6g14aL1i0F0dDSe>Ckw5%Zt8MaJU!!P)<j8kJ<COj>1n@hFfn&
z6s*Iz_s+gncKfWkhvSk4cc_mjn1;awEU>E+`fy)ozS<X8(?<zDp*;X^HkxdUvb8}u
z-hQbAP!E1`6JLFd7R5heU_|pTcv8!^44|ntUHN<o01{T?37S{hz=*s!d;IRs8~tbF
zi4PBzSEykR-)PlvOu+UKjr}_P+N8TXRMGdcr}$%1UR@e-`~D&2=O7+;i03#x!RH~}
zG10Dp%bIR|h%XsrG#GaZ+qoghjqXUx9>Bv2I|Cund&Xgne~yK-APAzYN;{u5j)m}B
zdgqOqG-{u6nxvJ6nP&N{_1CkpB*89A68k;`kKv!rvOMc<_3$fBfgsZ6m*AhYgIQlU
zVLZ0=Mfn)pfOjy#$SiNavm`xMr?4I}Lx*YL-~gGUb-gYedM|v=N^Mz_`q9(y^>}BN
zv#QyuQu%AuG^wKXwTh<3-h4#4G~XK&l<)if1L{UgYEVH(*$W+jkesC*+u<>p8b99T
zwFz~@7yYJ{$p<}X<`0uXn8M0=@v1L<0&^niP3quojeRhA9}UP`(KJ!Qy?D<t{C9ms
zqBBdAW&_4JE0^W5b)+LDL9{xPRMmIH=fhU{(iJ=q*15M1bO!cuo`UCH>V*b{eO;6?
z*-UacOokj8v>(qM|Lo{Ts}!0G%}rW99827Aep>WKy39pf4-EM);>x%>pbHrnn+t1F
z!=FdIBVyRPXqX~{jCHHqnwROh>E~{-eHrGJ^h{TvAZJ?&{ta&AW6cochbngR4}|*C
z&nzcr1VyAT9)B^K?83Ib9FJ~GX{r}5-n_|Pft&=ni>>|P75%%MB`?Qn$@kjtfFz6X
zUc;dRS9x^`EdRN_ql+FD>}*pHKj?o)g*&ONK@T(X)-3@Va<<UrbGiZ@kt@;{tNG<L
zzUf4s(s`D+HHf5h$ejdUkbH?A*&q5o_b_=4I16jxKxmz2R<T<>`HPcGeu2rk58-1t
zc>Cy_(@UYRv<Q#$$Ls)2qoFm+omV%?=g~VVf=gl61`lsr>W<;M$aeBBc!Xe@tR4z%
zS$)|NMpohC09z#auppZUChyzL;NA=Pb6e6A*Vqrw<;(anzFM1!^St8ThfB_4t2skn
zW=I#zZ#$;nF@8fodWw$<X_Bvy>+?PGj6ps39m0p!*|glDFKmlrU*)}UAbGQ;Ph`c}
z2W*oo9BlghJSXka-DLEw9w%>|YGvcmAMM}#B;PkY#uc~vW?EgK<L0fr%a_d~M$`pH
z&U<^_y!g$V7h8v=4H}k~bMmBbKu_QV-)|Gt_&(DgK%fz5q9}jyO;6=n($Oz4M+x`e
zKXl|wj1En9H4#YIw6hoC6nUY1*tn?g2O5Z>xXPer)3n}vQx)OdVDQqgjIWdza%EM;
z$GtUJQ>8(fV;`^3U(hI%U|R_%cCTPplS2Z5zH(H%@I_Pq>oKVzjgt<k8c3?Uo@A2n
z2{aB%O=9GT-{-G~`3@;&Xt<a<E%wN6yV;X@1&|eZk$j{6o-q5|ff4t_PM#K2{`0;%
zHj%m*$n}njv!;YU#jhJA@nBzrt_AV)f2}i!&8k25Xm=v8thlkPa(%yzi~R#}aqZrc
z#g_Vr^pLQw7#M5aO0p`vy3?!c4XApFYH&$=#i1<n54^wlH1uvsXNvdGLAv8H8mgn^
zOd#PM8~dA3o?-DI)>?4@_}$vFZmEu+>mOj}!?<BBJ$h{!c(sC%kiqg24giF?yl0y(
zG22Ras!U^zwWpCWEYDL+lkrIqt!SUf>a-{t&;p~YbbD|^zZ%7J3Q|)YNhhE3orx3y
z^vC-rt;~vPru!Rl)5o8-6#{Zbs2GEEK(Nfa8L)BJuO9^La>X~JcD3y&@BfCOR#50F
z@;jZ$VZAuekK7}9i|O$GVFVrN)uhk)7U8}Jpr&dt-}2rp0*mZ|MDYmtCVzRX6_7}N
z<v2SF2hUzn5&M$4HF5SWyDl9gjAZp%P)bqOm!MMG(VgtP73HM`kNh2;q&eGpwbDHp
z)kSV+3otyEL{H$bAiBYiX<rsa^>9(U&VGTOCPQ$$N|vR?g6^;B$WKCd_lXE@Bp*SV
zf(gg90weS*p6{-k;_nYbbi$A63B<=ZgRFzYJAj{YQcqjIB*v_^D3Uf~d?g5U?A0o0
zF9~T2jND$MS(n5xZCa)!I-W0J4DQ|+*zpOMm|MQJt(fh$<+uVdy7HRSFl{4dw#Fjv
zXI0RmW32SSlNo7?phWpN+*$s%M8q|hOmFJFSfPwl54#gKkn}~>$f%E-RqYd$UP3{Q
zG}7iaz;#-|SK%4`Np}p%%8nI0o&`1v_V6_l0h``f(s^U7i?kZnkL=Lu`<}l71uS(|
zfrcv`IF7f09ecIHj#FnxlVvFwpJ5cgiuvsyyg2N~Rd;C%_*#>jqyXJ$*)99}wgwaa
zZvi>&s3~%WPMxN-#v&OZxYE-_-tLiNqm1R#6eq|)vEN)nVkY>+(mtLA#$K0vK48}q
z)#|9($!jj(mUTOZ0AC68#s<bP435!u$H%z^CNen#symp;9&UkcZJjY^8Sw-G8U%vK
zO~$D3+KU}PyEseO=6*y+CRRHGlhc%wm_7E<eBfgUlGqCm!<od5lbISO64mtsJ+4_L
z##Q?U_>SMNA&-)-Jxj_jU*43os(Qt$>e=(=<&PVIkRM;IF%SsvM;UG0s5RexOeHjK
z+){~&n%SEf4lruT$Hj7qd76H$WwC-h&OSOuV>>ctsdUHUn|<5BBk_^biB;K~NZgCu
zb*6vKl3f~}A%w&b=IrKo2!v>43%)|hSzIy7I26|)(yc3@=gE{Sd{f`N4q96^>u-P|
z5(-Fx+2Z0av#;Vn$galxS9kUeu88FEovw(a;rcK*R2PT1#g~bXvyAqun<#w;47k59
z?9RpYj`3n!Rx%g{{S1{89zM^)RfNwHzW<f^LB1sY{aWq`PGn@@AAvv}$%IT83P-Bu
zSG?S0`>Zoz8HOo5>D6IKz<%_K^_x;c09qD;HRt}QZ9L^%MhKjn$2|9&cPm=t#}NY~
z&X%vpT?<rgi=AR+d$@Eg26%1OKpm-7vXwV&{Els*_53zY*YR``z|h$$hi_eU>Q6hI
zf;NTlhS{ULR>LJkYN;v1e#&hYUoRUrP;h?)hu$7^;g7TSTZhfAjdSsEv7iiKs6dK%
zWQf>2LJP5t>8H|QVKJV`R>#l#N=Lr(w`XK7`lHtt1Fn=CtUDrz&h;^&*SpTMgZ|x4
z*=8F~(<4Zbq#6a7WZ1GSybWB^{!8#Vq{{~K>82#)hD-(%80zk^=(G%qQ)3RoHg|V{
zNd`n5d*Z}Ht>}@&wG0P>fNlV)43Dv&8}NxBmZr~2+kzziyVErtoyO^#s+57<*Xa0<
zX)W7}U>gNy=$zWWgvv9}K7_jpH%>p3&p^YFZtt}qbaUL(zy3a$FES=$*qroG+Yv7E
ztq)uycXQdno8GM~&7RHqG?NO>(Aa+IS633jJs6RD6g-dtvfIM8wFOAJPgAjn4%QmJ
ze{x_eawWX7H6XDBGTK(eB*-e6Qv8-TI<h-4a0r$&UE2U84cC6sWyOAz2vy>;(r-re
zJbJC3T&YGR_+0~P^5>=1<!Jf*a~<+$mt}l~EGH^;Rz+6&t;r!H6m;CVu8QI$f#Zf=
zS2-&&WM5X(X_yyNDLT7QcRnj}T8~Xh*O7Hx3tWk;VOd28fK}(GyW|(#jELO6k6xxd
z{1<(;0HCDd&=i%TV`RjH9dV(>c3KmSA4%G42i^*Jl*4_{el6`J^zYcR96-w1%-EGW
zuKD&*SX9B%o}a8h`a?0;kp5WO_h|)ecpmH#nCKK*Fmw=$+k@eigmQjvA32N>W#oX5
z$o@#A^!=ZJ^bO<3tKdh%tyo{Lt{GVAf)e5)_fiH;lL{Uio*RqFNZQQ9tv(!Q2hR&f
zs*}l<Eg?urgk>Sob1q_;Afwzr9~8U7o-otvnVC@dnOB`<f9c5W{j%z+$Xj+UO@Z&s
z2-$1)u3P+LB%(*t<Pb`<aczJNBBhJ=9fC{U4(5*BEEq2Q<K}LClpUYq{+`{uqo_t;
zNL|!C;z|HWoSr@Q*2cQE85&nX_<%W&stVtyK2?#z)qN9H3d46<>vw0Ly`zf;M5>Ea
z=wSXzcok<3#P{>h=(bLbOXCf5-4PWFT>{T8KkWO`>RpXD^iohs3W1l!FZ6N6H(>o8
z*Fw9p!=ap&ph*%j{l5P$D!m`CoBEe2?0pGE^jz{Qi~S?c#=AB9UbIRcL|D__?rYut
zXxSlG4%+4+0tCZzAw6bW%@Yxe#TyDLBqtQr1NfU=scvqzrQ^GzyM<vc;Y(HnbD2aK
zFXDHshR2TC3`r?$k_{5O+ws@(TSVR(Xc6eC=_f66E5J*QZRYiWS?NwT6?csh8KaZJ
z(^)lJ#;=*;56$vDn=r+{#%oAj@ZXWU>X}>JCAbI|(;E5m(_K)d0q~+4$yrwGba#(G
zkfkO%2Zid~IQx;nKZsvMWKkdg)=g?QkwN?k*Eb4!f_{DM)0O2!U9xDH6aw)~BM{z+
za{vZsgQ(h5-E;_OpOQQfDp*SpRCnzmnyC9yj0spxW)6Jz$;!?fzpp8Fjut#K#`(I_
z7}*JMr|!(0a)LYd%rm7g;MPzI+&Thd@Etf|S>t^8{tw03T7$^vE(mU^ZW%-hVo;OE
z{w`$=tj9PZp`av<4_jJxRAg<Lrm-fhpL;v|6>mUk@zT~w7cMqL<ChOG4@N<Oboc06
zI(H)J_`vo1b|=<-KZO~i2G12-L$dChL!<6En)Nh|<@^eb!edDuaAr1qL=}(?xe?&y
znk_f{|Jk~BGrTo5v*iKS(CUYyxeP#>u22nA_pxRO1tw7X|6*+g9IG^L0kaF7bMe3B
z<n2*H#Qm6L;G$;x_gYtTzJbe!4eTebY40T^nv9&fR$)qHeR+`n9TC=DEUfugWD7JV
z10^#i>c6lj?*En;Zzy?mT-XMC(#Lz2%6v&xwIoVm!@|0`vJ4wO#gw=@qZ1urvqYN_
zCj-Jl;SAXI41w1c_^3mb!qu&*m(V?6iG;<ZiSfdv*GKgHvY!qDDvz-ecR1|LrNxPJ
z6S<gpxfYJ3`sTT;$OnF!9Zha%zf#2s2!nx~jZR&qk5Y1i>@xb!q*S4tdE9I=;diE(
z-4#wm$A&3i$jrQ$s#YzLJwX(+wJ-$DGE)wt+Ui=%SvIK^sUnVq*i@nOm{f|pQ&x$c
zrbh>I_0NKwwA}j##cHl+^YRj8ftLeZ!V(Fxq$2&^%swbd4rE&-*0At&Ie{1u-*{ny
z&9#CaC)R*I$%iT$E1>PhbU@>ZGZv2juJyNP`2~*P>XU#QDZP1zvHC0J9R`++NNjNn
zT7eY!!8SmO$4t^gVXSDPMXDZJmhAE(PDLTaSz@GLx};cb9OIVnJ5)~`i*ZtYRtb_W
zyqGm^kJ+aymL)WAO&PD?gt=7Bsyue?=r!fB;qGHhL)!F2%@#1xn9doBz&r-xKXqhM
zrVu1fctpo2giOXIwuMYAA3rHQB!((Z<D%eEZBZm{iCW2pS{tOZB>2Z;3J0Tui>eAB
zv{ogbx+=>(PpLn4W^S^2z2_IxuV$p{EiH9bbnlF`buI-XPPNBGw*gHoM>9wh+pOi-
z0vg1cD23pr15*bn@lG<Sp(y~%4*DbN=9|6|M1~lm5EwKt6eKL>yBL`Kn^<ysX#mx}
z@1bjf=m}%f!%5MBiOjefs;5z(UTDRgL_MACMV%nd#SYt2s}=?XTB%fJHT;hRbwbnZ
zK^+l@Qsj=vaqv{7o@BaBvf}@bX^jOX;M!v1dnZD~6VB6m7D?Ba{kV7}B_w|AnDSpB
ziMh+H9-g#<;Tj#Ad_0bB!j6QkSO{q3DtU3wPNsk_qbEaiycf;c9t&9aoaS_9Qmctw
z4vQVB81RB?Wy6$`oK|f02>O{<MZ#oc5LeJ35}=wn?>mH*u^M_uP>C5K9ID~*nY-;=
z>`FG@sBP|Dn)v-VThy8S#*U#25=;qS@1o3y=@Isdc*VuHVwAZhpW)Il1nIGMnsDM&
zq?^0;>-3?Nyn?fh_5-L%`a<e@Ko}?(!uSE2ny2=VSI1%$%_Pt7xK<B+T^)8}E&0;4
zOUEk68_d0kDC4mxN8lp!U(jqqh)16T|8;ir$?bh1$|g4nujHkxFAvCX_9s-L6#dAV
z_IX;+>-M3zI#M1}E?I)+GxQp%LobyHLwR7crlv}r5mXZ2WkgC@_YDO3*jGxnHzr#w
zQ2ZN@=I-^*1ks{|Gna0|+&2F$kosN(i%DSl9ba~}wAlAr(J5pguDCn&H~C>>UQiAb
zKQyxkDY=yJX;##t)RO`^aRh>`grcOB#x~sx7fZz8U>dY>Qam-uI$Qtl1cq_k2GBei
z>p}X?qhic#fNeh@pt~|L?(J00MGH5iPr~OJLb$s}n<2bucCoh~WGPuUMRLNy5R@T$
z8D^AtgJQp)yoD}3Ls9T36AIPT3da^XDV}{HXKuopZKvJ^GP{9YT!B+)`A=M6l!^_P
z8I$kZ(9ad_@T^?PsY?gNX7zrc%2j9|8S<poUcxArEB2iUmX+MeU1d6jNl>@x_xDLH
z*GsM;v`+x%A1R>*-4idF5-Vn5FUx4sJ5XZZ>8&y-qB`V)?CZw0Ml2iJC;~PA@i8x$
zlt{THNZNzaeq12^C<0idK=;xaI7Y1{NP1(4%WJ6c9vD$wH^#10t~BxZL24{eSaN6)
z%2uzfUV=VZXH`sm3{6Y<bG@7pd_x3sPk$w)BTj@f8_P^ox7~>&0oW%}1h5ATN$l6i
zAk>Q~4JA7mAvSGj<dUW?%g*^#=Wo4DO?R6(I8=XiGN*QNkDeu+=Np<#hSpq0?+FS#
z;T@h`jbOjyHm$UyHh_579B;4DAv*c7qrZ0LVy)ez?!%9?3XEU~LKwlx8XGddRP9Wu
z>$6I`G{0c0QlH5%Yn`!Ib*{v7)3mnW$#2@VN-c;AJD$haT_<2rO)-pnC*fLc3A4D#
zgwe8ek8r}k?y>3j0Jd+v$JMj5WS=BlNsQQZ99#Zl^pNC-7W`Q@jGNg+44eT5CxTpY
zC7Xj!2zo)|hF%8>uGb+=r2(8Yy#I%ylzJzI!<3qGi9A%LsG!oj&~SUS)|uhLH;<ez
z-7k7t>e!Obe9!{Q)%H5Bq4x$_-4x;51^lq^54sP0LQCT$nPVhySC$&cDFa7~2>pZb
z1YI@#JCN%jh|d%WZC3J1Z7fgxe5oNB$^^cB3mNch=tO;G{>RaXQbY5{0!|LrgBmo?
z;}|%fnpwk$Dx&1UwiI+}{c-BEs5Q8X&o;6LVB4gMMB~1U%qK2H=B`CdEUu^45^gz?
z_-h~VEw7Ow7%Oc>?_3eYDD)PwWYrf`0BAr(JH*FG^_3s(v#Yp2h6Yb6Dy1pB`U=Ir
zliAlPZ&NTBtpIg;nl>m@lc6=}T47+>Eh<$SD1fM^JeXA|&uTO_^_q(D^-@ADdEgbO
zQHcSK`69X+ig<aHs7f?^!ehO%EREd(b*S%<AP!d|U!_!!dxUf)y}!FLcr%?xS$LmO
zGkcKZ(}On+b6~cXpeiOu_<v9riWN{Ufh#vPCH&iC$fDUcbSlKd3+xW=iw+`atg@s;
z*F0;2B?!6^Msr@b)-5adN1~SMk|dB;GdZHi?}Wf<oWDvDL>^1l*CpSy1n0z!oZgN4
z<h7UwS&`I#+E>|c$HfM$o^28L(325z!~9p4V(I>vev?Za-qGN*Mpj4#Qdn13#4A&z
z$jE)Yf*(|eGBAM2`n6?(+2}=|lrLEne1l<6v!S8*=+YTmd=%)!C60$aCoxs;e&Qm?
z`kz*oQ+tQbQeq(VeFOW?1s$GL@&)GkbxtM7TYO_iTzM9=o-ASeYANyPRb!62q)9Re
z4Mzn&H-R%UiOM{y+fPBgT{=$M`ey9dHg(Vu7pmJAWhC@!4e7VdJ%>A}^|lR4022Sh
z4N*tARjV7fe+De0a&_WtBYDMSc$fl(xM>S*Aazy2P``#H;a&X_r3j=8&^8_rjy^Wl
z^dA2^TPXmK{z+4ylrPn|04KnuozOk4aY>2KRU$FCm7FOdwHVj5V!sXtwI<o3jas2t
zs{I_!!oOzuA<@32m%S4Kk!uqxh&f?CTG;?_1tgIxf|(DFC>BxGP%kA@xRAD+JO0Sf
z*o&pLA1n=vEMX(!Dg!2gXwZb+LMhQ#3*Mer6yw_NsyCe|m?XY~#(8L4ZtA{yUftZH
zQxrAjE41FzOGn2lWf!0^weZ7TD7;;#@h)ZG4oWwA14`Vp8qnF4BxRd3F=umLZ1wKL
z6_#FW+bHHV{ve!-3SFf8)fRVox(~4e&%&1#FiVGJbbJ%dN%M`A57ed%%2B;ICq)18
zI~><F$|U$P>3<c<S2!v%b$e(hqrMVAsm8YrX3U<>a`3!VQzd#aj`5s{NK8z)L88_j
z?~9g5Pk{bta~mZN%kXG|7grNdt(&r8>P#B><Lt5-WpR8a<;;jKigf%$+9I)-5p06&
zyOCBC3B5Eso7*z+8M<lJm9-AH8HL6k3{VS{N7JZnZa-RuA(3v(i6*MHw6!u94WU|M
zjJ@9lCn`ZB7VO90l{S+UBJg#G0z75!bS<}_lb$CtTw2->F)e}IxFKdYY?S~ZT#&*G
z5f%wFb=bBKE(mpTt}|_?7q+Dxg!k|YU|cnKIxxNC+#!$(v}+j`#L5;e?tS({!i;VW
zB|8wF*?0Z`y+NfwknF>6YpAh55g|;8AY3;hjV$y&E6gS|Lvq*oRezR6-6i!41JVgW
zOmb~Ci9gQj!);Zv$%sfBN3U1Ox1u!PM6J#f=QeU%HjpA(W8eNJP^KoW!)e?tWPVgA
zJ41kodg@(}L{uF{vjCNsL!Dk$jo5txtC*QE(H6LG7U5C`c!8#ILjnYxsJm8Z4}tH0
z^i3^1C1tZmJ)q^C($~e7#>Ufh?F%DrrockHLi{$lgr_DwtV|x;?wBJDF`~L`aqiRh
za95Zi6L;KJ3RmocPkwACc_!vfGlj8f?M78`m9SUJ5^;B$CehUiXLdSTccF~@{ERdb
zN#z3SptROxCNz){Wz|K1(BUR-L@RJy#n2&t+KEPZQK4Er!muq0U*!#B3LQlY>5gzN
zsk^QV|8bMvNzrM3eEAt%^aQRq(6OXmtyzkZw+0Avak`kWsYeIeKUTRa-W=Ps@0+9l
z>^>rX^(jO>`vK;02vjmlGAcOCGmBZbuH5}WeL$9m+ap?9u2xvKe>bDYjoOTtej_nA
zv^<eTnfKPU(qy2pv$cNPs$!Sal--PYIdEXKDTfQN+HyBxtVr#!(9i)bH@`9iB5CSc
zvXI8u;JGc1G1}aVgrk3BtfX{Pu!p0i(ie$aI{IM)0hzcl!OQ98Zf&Qx%BtB9W2(^K
zm}YVEHL{GXAfhe!F^q6|8WoNQ$yyr76}MHc8I8r6`eg_*vKZQ^vo0LU-$y?_G=MjV
z6~MXD=f!d0<E2MHzip?5x|>#b%xAIrvT3wl#R^SOK^bhPxPl-^vc^~rM#-zGo~fgi
zv_+DEB73xktiA&NrHK#>H`RkE6ihR4czkJLXF-OD8)P2dOxVDUz>xNu#fd7_diJcY
z;RMYp(W!%9HuSPd!_)?DmILF<2!|i5)6HQO3tk*i=RGW~V(aE-v113!%G`pJUt!12
zs#VqdDqqU>u30x+#V9HOSy3DQqDGt0K1Jyg37<PnhbGc*a<z8Rkk>TaWARhlVfTBv
zZsMU<mJNOkQJgp4?EmWXR;?*DWoe_=s-wBO82P_!OrG>+*G98dF94NtrdoX*FoY5<
zwLz8{%#v+2F8#600g^J<M%<x8e4&ctyD>$!J<hqy!4~(haO^NAI4P<AG`wb%sgjS8
z+jQJ_qE&5f#4y1<hDZ1N4E{$cA@MtwUWP3eN(gMXV6KDe+0L6bO((?CD4%xbQyNt+
zLvNIA_;eO$jwRZjQ)yQ7yuVeOZ1Xp0=)U!?&c|7qmD}MX9IjF+h@{|-9eoskgLl3$
zmn%F5{9=v$`5oV@(k`_-Q9@x1XvxOgsX>LD6ySk`z9uwBoMz{@`duwCGBz-v_sVU8
zMqko~K;EC^h&)4ADR~2%UO#%YCw@(dk5&iZWe+Yh#06!+8PHg!ykltkMei<k05=b}
zqT>UueZ2rPe$nuc-x<mX3e6-$D?5=rUqRO5Nh?r$g8GOFJV3+zM`6PW7D@I>%MBZ%
zWP1`7-I!pp(&j(3pV-`TxUN%B!nCfMH5Cw`$pBQ+NungK1vrxHPM4Fi3m+{SyyK)%
zt1vxYXKr5epE?8-m|7`PYnXSvX;o{OR4GEt!N#uF4#K^<us01AjT{|9AyX{uX>oM?
z!o2DY0<7>loS-s#wZ`?I6wisI9u2U~%zT!s=d^dkrHdn24L{s(#F)jjQRXb$iMeH3
zfl0~6SOy<X^|%Z~LsTnhh6ZLR>edWvs|FSGRq)${iVR4Xme?RWMS@u`8G&Bf!<-OM
zX~E<(WEJ&;q-ku6Dkq(IP@6d@uP-m04c$-DYnV}&_ba5PnrzV*ZRO~lB`C{QsxK#b
zw<F%X@`f^_dD$daW+Eyz9P_1pV}^aXo*<GW=5Yc}A=IO{Qs#6inf!Jz`}#owyxe}t
z`8zb#0^M+0TW)|u?5c?y`Q@2pdlPY*gqC?@V4uk|RhE<a8#pJ@sx`4_wD;dpUwfod
zDnghq(gy;DgQrS-<nAWc?k(T&Mj>Jap$lrinXOY|>#aUx-kx^O=c^DDVzAD!Q-t`K
zXd?Uns|UZUXh>x@MnGx#>_3c6INZkm!XsV1YYrCp0(whlhVV^@DN&1Tm$vT6PvBoz
zq%rM%{JX|A-pJUA?inhRawjk|F4#`lPu5$<9=1*=pv{(Pf3OsrJBeN~nUnBfD=_x*
z3?yUCV`ufqpzHu8*s{gmScK3XL}EFn4sk^=c9G`(%qj$^T>J#ZDgLEH>OL*nH!e&Q
zwd>hJt0np2+$xjWAxM+o{e!VMwOaDW%q)i<LYuZu!!X*K_%Ce(5wQi_?Ba1h>#l@^
zAf*D%@!^y29Ng|~i@dn68ytmK9U7c%eX7&Q-B66v{F2*1fI%(D(l#bygyF!UX>S-W
zgacby`*fykJJxv^ee(}Po`yRf;l`30w8m<ECwtM!_i{Fshm3o2P{V1y%pk{zZ@{S}
zzRrL}T3{8#lacO!N<%z4l$rFNM(Pk&k-a=&9(<+q#-5Zg<1QbC{t5fYc|>S)A%iWP
zm&1jaG&*8~V+NAKu=DJBvdh@^U`Y!UhUNl6)_9QhTtu#_p-o$CLiTJah_I|=zlYSx
zQ-&G<xZuc?SCpE!$_3&yi!3DT?#$C0E%F|0<Z~?`rL2-r!M;TfY2Ziuiv>DKqfj3!
zYfiV(VmstFEo?aN0SX*q2dow+uj*ZRvr+;hn7%y@*)|9rLK<&SF_!xP_2Ah^TWpwP
zFw~kx3lq(In3I(5N=}7Mlbwi}wE2{Gy0bHcmp0ttzLNs_^heu>ytav%M(l1H!NTKi
z6ph>ngVkBy#8wqHbUlz+ZbRUX-3Mmf$bH4%kTn*Uskl$xix3wnxv5p@R>|e`Z|Wf-
zYK(mzmXoyI$rCE35x{(COG~P@{ewTuXOheIj3GT_zALzqqd6yiq?<*NAk(-6-EkdX
zyPp#&m9R<Li<h>9@f(tca6_VT*I*y87Ofyhl*nEnjY2*VVdPI(!8I6WJU&m_n>k^^
z(G5&KC2uy6h@2Kx=d{5%4i6Dsm38DtB3T@Lu4W6HqQO#B$Lh+7mZ0>XU7^~IMCpBS
z@0zqfxm2ktXU0coqXc+Us$5m*Pz#w&cYcB<6hbvPoruBBbm|^)T-=BL1LFAAy_r1I
zjCE)iCny<k!?0V~|1O+7Ks<->fTQ^PB^D#mM%V^VPh~Qs#V{IkVCsBoBj;);fD%{5
z+>do#O5BHGKN^uvZ%8PhDxb(wI}pG$OE)L+{4r0#fMN>s@q5N5w;9H>Q)**8Zq@uT
z8?dh?XQz!fdQKD+$(S97xsYweyl_;}Q)UL)u+3>8?-E_O-yGnZv+*@ykJHAMOsO&n
zU$|zB(VcOm9Pz0*OUfIl`HE{S^j6#ktS-XM0-J~IS(XS{M39EGEL*8-F=^8+j&|pm
zbHjc8)PSX?81YdG?q>9F<kyyu7mm|_%YA<s+=}EpR|LrIQc;RJyn^wX0BR?6-SMUo
zfD0NQc^5a~=-iCDKW}pSqm0%CaH@_qA#JB7Q3Hpu`Vk<k<+R&{3ascN0i;78yW@E@
z*hW&jyN_X)%Oi2pE7OL<`RzpoquZqE>~t~N$ByZ9q@&I$u!n;5<xJ49*#^vL8PdYX
zPom3>(=g@n`#r^Vy`obm2PBgqHByGex*F|9bw5Pyql~oWu$m<DcJBK>-h0XS@kyMO
zDO;Yo|Il&FL+Be32=R$yHbG#GQ4w0JKbLd)n1YFGXEWKYlz92Q9sN;@TI9Z*>qyuc
zzyXuJV_zX+4aNB_>m&uu0qUVm3IuJq@4+z)h>oF<`er?d&OUSJ#`J8O9ylV_KD%er
z%JN|i$5U4|XXE*7<^6KJe)#I;OSguto;^3FU|Mlw{kS!=YbtQQN_t|jSJ%3j$yNAp
z;HnL^sQ3=405q&_y0x`yPVS=Q#r~PAR+L}jSM}otttwixFC?4pf;SHo=GI3}lP`ql
z?N~pXZafHg38{;T>4~q)E8G*W=?ls|H`F3akuPiv_9{=UDWdOg4A_q+kCvwn%~XFO
z=Ku9;bu!uljJ2(?*Q2;NthuK2%uMf%`EmP0j(=fVP_S-*d-c}-Z}}Ai?9{@U&Gy6V
zFzbz&RBZiQj|&C|fE+4|#sx)DC-AL-5x9biJMgN9vD^!Qc%y6L_7gH%>|adOXbn6i
z-v`@j@k8`UZ5fek^{{*IM0^FRQjrw|e9AwWQI{3u#m_0d#!nHnO+*n98<Xh!!1VPI
zL^=AwF9S>A4%Fj8cd*MxQRkwO&;wA>*kB{tuTJe=F{%+ii@nl3nN_$^SXXUE>v3TS
zTp5N}eVP`bC@>dAe*f6>Gzv83;&M>KJk{T0=;q<M_}S#`Ik+VlYFJodJ_1A%vSs=d
zn^N!ricJ`)wFw97(zul=b-}2Tme+nSZ&RMBeIxA#U6>1!gjZ#{Kg|$`gC|8p1ggXk
z5tHaED7W%L;9BFV1vVz*3o->oxPy7RbG_WH-DtJEP}sshB|Q4Qvq_p8l900C>=hn%
z4_?{nvX&4c?K-0$M7AYDp<zZ=(NGGlNeTQWwd(RDY3xvl<CaER0Yqk+3ic=QP=i~^
zEOK`9?!-*GJz*%iwkwI;MvJ@^jRTqliCiJQsV<V6AheM(&8AT<D5v`t$TT5U8Wsi<
zR*s{R?x_HGCNcv57eWi;eF>rq+E3EQiMcdtnIB6n&AYU*jJkw)sw6B$aVyW?3S)u5
zeQB!!Wh1#jGt+5-I_8lQAOpueDoJ<76_wIRURl<71OK2K68~Vz*}Tr8E3Bqa2z;Jv
zP!UDl#efjNX&ev#Lf?DV?^2fI$0AE3tK|ZSRk2)}*tjgZL`)V23GFRmbH$z*T0LGT
zd7rgfZV$K0`aoY5ZBCoTtI(Nk1^|O5j5tcSV2J_(V40C#(XdE^VUV~58nJmj0UpT)
zt+iMAq+VQ<x95^Bd&OaZIpyWDMv&!`7a+jJ_i^$9J>5}%dG#Jtm}r!_1vJ4GST04r
zBySchdoCQkN3M*ggx^tTL|BvU4swzYq5s;aknK;2G?hs-e86T>MEDeKJm~(Qj}e7=
zeo;50F*_>@H5Y)3E?1lwpW+*xN_+CjdW}}JzMSNNvY_U;To%HY7lQ3FV?YkhoWs;a
zXD4OTq+zN_%ShAUnW4jg9dxG@Gl_x?A&7h~ji1&Pi7G`%RY5Va1)*eBAn!;MN^7=E
z#vMq`SR48ta2Gr{R58Bs2B|Z+TXO8wk_7?DVp6FMcOr5GziOSGI*T2&J1c^-g9aoa
z&%v@q#}?noFeF+G2{z(d9X_4eqYNbLoUS@099fD~;8K0P8!v`9=hD^A(&u+Y`f7Bb
zA6MjvMtu@=Xb+NMQmB6mJ9We-k|+;3(&ERf0nEbdfXPV+qxlEe*p*#E?~AAa8v~WD
z;q}?Z2T_!Qqo!b{NECtXzpA#ut;0GIXzcF}A%&Akcc+k|*zXyNDdLa{W#;U<$O1y>
zC$Ya9<D^B@y^qnvIebEX9H4lps7z}^RidVohv5+I9LtPh6raTM4UFCwN>?`qK*i`W
zWm#u)L=RoX!$WnB7PoG4?K5A}dAoG{F6bvJqXqEUGt!-mtyQ#t>aJktf?C4;*WRx8
z0US9yI=QCk*@1{?1uMW!4O%sOaxPZIarsbSRlRx=3_6dii+034)fnub*Z9d;^;U-d
zA$!-Z`|02r?eG|U<wmVvH`!jLob@Z$pS@=*w=ZupoIrE6e$k|FSNPHaecWUOdaRaD
z&-2!{a;g2#cT|)u3=SP${!Z7RPeP8rGofJJ$p?Szzd5yl>K}4WtD*hCUmB_qNf0PF
zefz~1o~5PY%Bt1fUtay#N1Hx=;l5rqvAaemzPfm~|Gh*0Doh>G?a|MEz586*nB0Hm
zU61&2MxTm>sblx7p6e6%)_{IPOY0-<c^>Z<{M8%pFH15F&B@wQ`FtKMB<1ObPivbd
zSN_=WDK36zv)_dtri75(i|V6o%M|@ZLdf_ztK#fq2KA#GtY5pa-026O8^yo4b;8l1
zW{-kxxKqGugOg@8;#K6HLm=MV7p8{P{+#vDf!w=6q*~6EZM{yvT6y{oyu=ljva+bd
zSuv~PMwV>57E!S9$WXJ@|NnSom8WsHm2uiHHVxlxo|92qR-M0g!qIxPh>mlomt!#S
zfL!|Ngsti<uSYc9SHRzN(DcVz__oJRKcAO*@m3Lj&7S#dToZo-L$$geJFUO+80UTi
zBkcaM-&OjmZEaID^16w~aNT-ZD%*@?zsXpWkwG-K<9NH!=UT+Ub9S1MtspNh^z}9P
zzL%M=fe6BO@^rTHI;T0#U{7-<j*2r|w{HLw{*`G#_h(HcukZi-v=&VUA^thYb5KCT
z652e>)^0SZ$Td@ZX*j2X0`Q|gn#61?SLjPG^LLoqL*}0NlEaV3aFLcEd><{Xh$x}5
za4kl-H8cRDD>*k|E{S#`uN^-+Oi%7Lx6p5-O(-kdy4~axn7rs*Sz0?5(dis$@Ql|<
zxA>MM=g2jhf<YB0k#?0lQ@b;<ysROvPXqlsCdCZurzIAPl9Ww~S@LZyDMDXUVm`id
z8hsu<fho6UO!vo1)GhgW+lA<gtgoPaB;k^j^0KX0KpvSPoa!Q6-w;s>A-?qquc=Y&
zgL>Jlq%jF30unpEK?=Nit?Q+hv8<evl-Ysl$I3#8Hvdn~mksTqC~y&J<>`(P@}Ikq
z(LeevZVnZu88*4Wj@;|27N+{8R<a6ToC4Wxff(_r<+0PGfDLcI1+CNU(@wJRn_rU#
z+xXp8LE~?TVOp-);x^Vqr@qe0+k9ms4_y(OovhtAYUd?VFE3V;f^Dp+dv;@;qkf$-
z{sz=&LHzI&$6~N}uktn4*q2f#q?3s|i8SioJ*tF7UrVB3p<&~nb+0zxxa>41tf#-n
z*Q_Mrxsd%{Gbv&*Bop=2VV0gIN|IB-HvZ_Y8NMXQl1B$xj*!U~tUr!_Mn*z<N#CK`
zG>g@geTeDGiGSsllB_JJ18K!b%WdWUtixi=+j+(Kv0=d7SWBqXm%3p-dfc%Dd$35L
znzsqVSbaN-V8X{^$os9TKxZVQNEO`|0)au%AY;E#?GTeSMT%b)?ae(#PNVnr=E*RK
zD=bjmsOub`k+y|;;8tT<!sfRp%^liuCVzon;O}H=Molc+=eX9R3+&w2CoYTZF2C$r
zMDW7Y$?O>W@(TY+rV(GieIKbZ=8I6f;XkB#VVFY3b6=1?TKi$1G~vBoa2VyJSj{J!
zXMBh^{4mA6moTpl51FkyhRqMcp!x3NgO1VoEniJ#<ElRXe~M_><hB&T93^P5*Hp}>
zUoXDM9=n`IF+qF<`3;BH8u&@*OU4JkX-fO7yGH@mE9p{Uaw3LaL(UdXG-@lm)QoPe
zG=E3oHWqDA0BJS#@LcC5$t};P(C`(_SDl&V@2Q>7%Swd!uutSgaQvL*I4rS@=O^lO
z&@7>Z7i&9I&wl(Bj9P53q3{&X+0ilz3nG)wh}Pl;GRVgBr=hSho(^7%>P{wGKeRNG
z*CO}4mV$;Y;fRD@7W`QDJ`H0EBm3~8?4BQVl)sNpalf(TEkfYDD2!oM^68?n7MyHn
zuYEgrvE#3kNjaMX@}vH$ys|kUe#DMVj>`qVkJw?3pX_Tt=~n9lbg?oAC)QklBTxOm
z@mONc9K4IloX<Mt|GkMFAGoGpqF$Qy;6m)O>K1?Tfc}3~MZ0^o@A2*FW^>FF559%{
zT??P8C|HQfUiMo<>3>t<Ie;VMm+6YKUVH<T1<QABRQ_-B`Sk;rAqB9CzPaw=t&wEi
z00rYVkX$Kv4&dNv`Rl9evBeZQ&kpKX*}QIE@D}XnT9|rU4|e2hSw$9+Pyc!ZIrWj6
zsFV=z5q*SlKw9R0T4oqg@es2$zyOWdygnTA<=TyPU(B^XPBzntZw|+V`+o2**ZY3@
zl;LB!NFHxH!9T_D2!43ztRl;d__$8}c}}XX7F=2q*XgzLDgK^gYRM0@9B#n0i^Rz;
zOzoAr9q_@8LXa2hl^95K>2(g7a5~k|0550qQKN71fq#c#5jv^ha1H0Obplz+Jy}Rb
zDa1^t@Uwm`w9;+nJA~V4VEh)WcH&ceR@aKZHP2}d!=(;C-1>S%(7od07V%B^`RyJK
z4rS=$#i|tG+$Y?A5+>kwohHC@By^?&y5WCBlIM|Vf-W7JW5)G}^xqg*iAAz04nIq?
zzrwLa!k6H!*r)!Q;!jgzA3>A89&s=gqsgQ&(75nOJ*iFlD30bP35kZ=#y*W;>B-SV
z2R*+|<L7JHU8T*Z@Yh$qpUGhq4dk>u40@73o9U}Y2B>7gV8y+5`r{J0)>GTr2VF!C
zMi^mT3jEDD1_um+7-WPFk~qT>02t%7<JkP#e4Y<r4!JXl$dlTM$It<^by6q&g#y)B
zzdBtV(vM-1LBh6`pRw>s>PTa6Oc<^Cu<#59{YL-oK4e=wLzPpivt%J|$lAUSgcu<_
zFEaxx4v#eVAz!n&4DqO=+0KD)|9qV67G_7}$)@Sp$A2j@=DE@4KtM|#TFDoa5aRR5
zPMeBxBID^|(&sMk@5ME&!j1UxF{0&u-e0V3x;27C9Ix5AHU2Syy1oo}^+#K~KLk}7
z9xY!Nl4hxBl|v=VYJ>`H4v+#IqhU5?XE{F6eg=*l_R;X#J}>AWwgM=a8$b?IaSvdR
zQxHXdC(^bS%SD)iiaJO`XaZ0?p5sQKk@z!!a4EQ3lshY(+~Ta<V%vpV{h>-{b7&b$
z5%IqyQ&kjYErP+qRI~SG#!_4TqWwrSaU2d(f;0rYs2_Iu`O(@|Y8FUAE$;FT6U=QA
zg?tT;BfK@v2Y{HiaA6me2TwJn18%bWHD47}$2bpzU~QzJWHwKLtl&rY%%L~gvo_>>
z$vObkK+(Hp=kH1|gyLfQClgRBS8jj>@Ivz--W|OjvZpj={2X##P-v8A-^p`!ma0#p
z=A(XQzK|`I0)5Bjgjy&p2*#vWoC7(o%?Sml*h`it&>gGc|Jg2j&668zS$G!XxmdOc
z$yLraF_uc!Sq=f_B6-mYkjK36i#k4i%)kDz?E+~nzRI&33gs~EYL#qr>T69Li=11X
zm1N|;?+?L>Xe&y~gJr%li$ExcI|9%=RS4Aa)<9tC7G%-ZI&uCs@;AtmZEp8(C`g1x
zjbSAcRek??j#DM<b2a!EGchi1BQ{=PQ5)e*F`Pd+1sk)>kliZIoIR1fUoWJ$Jjv<o
zd+i)?R=u$k4M-InO7stdM?+opmt^8BX41*vk3^M2#*6ib`PYOde-G5Mr?Q~@WK!GD
zGIk1HsL59W3jN#I_tQ4;OkJNxonQ+=n39w_M<0F{y+n*u1g+F8A8kIq$cz6+|B!-}
z8jr1Q1xR*ntefu9)*V8pwsnWw*SER@g@!Jc33(Kx5{9X%WG=*(a7i$Y#&|TZ!&4kQ
zY_@t{5>`SFR2i3%@j=9OLr&tXyQ0z;$R8tqcj|qPeqk(MnG8dz8}mAAV7pGD#A2Ub
zl<nU3f#cKK=GoP;pO4*2&xZB|A9En~y`kgaZH5b_BKV*4<@RBh*)miM2WGf)s_;Gr
z<UlFYb88&e$!~G!P_rwy)ZKm?DGZb$BC?a_#FFFs;`YhD0e7<1!e$Ub@VIlO-vyta
zX3#|JnP*6EnFW~QBi|*@iy4#kv2$4rT4;;zK{*Pw@4e4zhyygrP9X#PH7<$=xxn}R
zl^2m0x6A#mNruP0BiU$@Idfh+oXpk3y=1cg+=E#kS}3t{GE0wzE5Od|&8ttF1*8<r
z;DF*Hr*hTfW>%DC^rUE_lM+4Kr+C$@WLBf5T>sSC@&u>Jl*d8DEWW1r-3YhAG-wMi
z0FVs7deg@R>RN}xXE|SZJ)-}8yfhw5(Y?DKe{X=)V*5VY8dW%8$&8B`B{f^VtIVP;
zRId~u65#FW$rK>4JF{#nlrta<lV>05@dmIUWj{*PWv_!_>FXa(sE8tq^;6`%;-Mq~
zrhx}Uvsa3=!AY+d<c1bT@KqmNuqMUgMn;U@d;uz__&DKyFbqe|3VmIEv<8t_(n!v>
zQLpSb%G;N`ans-Tik5I(AwCj8*S|`2qk>FT^xbr1Fs~C&LB(?%KavN8jrZ3_M{}^Q
zeAQil2U3D_gP}k#i(pRT4e?_h00Id5!?Iyq><kUjp(q2qWV}o8AM&8jg<Cm<lPsLA
zj%9#!*-y~n`u)%@gn14Wq18|+;5tYY3zvL}0U33Pn62HnZ?B&OG5zQ?)9xu6&!MoY
zSs=(U(_$@Ct)#^e!fhrrsg(S4&F?y3vkzX!FVgJC;CaMmSWzc4J|Rpp*6c?F-l9|t
zHS&S#G@_Udgs<(DbWEEDXFzV%9|9hUKJa`wVYZqckxT>4W(38TGMxEyNt`_^ui+-+
zD<X(lK@sUm2M@E4GWr+d3EiQ|!%)`N9fbSMz2-axkd6%UqXTi-TQ)v<^!Yr0WzuR8
z<-tLwl*g002}p~D-*6{+*8of);)6B?oLTB+8rnkYQK!>0>DMCC2XeUQZ}LqXPd~@%
z69l~p$VK*#JFt&&{5(aQ6h(ZF3SIwN{G+E&;Er_^F<6GNBVlI5Gmdyv01qV(<J>o8
zm0{%<c8QC#Uti8lFJ~H(*Ezf;4Mt#qzDTMkbds+R{32y;24)aw&X5QTY>AtMw=(f<
z-3@^*8!Tu_(i6jD^GUJ<2;cB%YGNX=HK6hSjLka53?HR&H%)W#cP<nzhm}%l5GFqr
zATt~$Az*wkv#1j=K{612<myAi+b4|pjT5H$CI1DM#pTVoNK|gYHs;#audRqc#y>hm
z%baj;&em+t&u01**@(_wj!B&0NNu{#6wIaxj><NkeUjssSoe}JFie~fc|!l6O=z#t
zDmpV#=XGbiGgu*7rKVFC7qF6a9WB%WPqcN1EuGuCgWcTL9o$}S>kbb+<a!4R>cKI&
z<xI?QMFT34v={a=EO07o=8cp))mjXN2q}aFEYJ;;tt0q00oX!lXiDkmvspAoS77_S
zCJ>4V<h{wTD%c2-C`3L9b0Y^sMyLQE`||M!Qe;u2rQ~$^(emN(=gCQ<az*d`$wnJl
zbqs3LaOg~OKeE8cAkNU3R<w7z928JWgOV}^=^>H~%Qxw5QsDTtk#k*x@$>I;W_^li
z8v^H2o@$6nWL*~}=0uT8VQ&9^)0dffCHXH#6zKHoPPSHq&9;B`i?thLkCVD1X@u^>
zq13p~YaZe67_`_25bnfFxcaPr>1LQgjr+FIi;gU6c{+3}#B=*lL#F@NTjXf8{Fk2~
z+qaMC$gX<})8AVBDB8Bs1Q0hNomUhnP?bUZLx|GB)kW9@MM8-2^g_-7?(*L5<~in!
z+HHPXio%3Z&lxD+HM@wcQ>XL2D=cCA@b@B-SbJ^*Hi6r)vu*u#gsC?1W1MFWD->h~
z;gQobR6EXn?qaD2;!oyZy&lnf0?~(hZ3c4eIS&qH|35eFz_K)Wa>YiiF}9Bn%Prc(
zfDJ^_&1k4V0cYDgpD}n&R^2J#aEMLu6SxHEDM=L9cI^H<R7&=tmZTTe?w}S`@P^3D
zaA*y_PL(qvtS}eO*18E5K-GZ|17FdCD6*TeC>Q7$<A+{PG5c#Ox(;^0g|{yT_oE!2
z>WyM?oB|`_w%f??-4Mpw2?kENglNc_GTYI&N*rZvWPm1audap|mjV=CFaf?*Y!_9d
z(7+1xNRr2Nshc%E-hOPZH@SbVUsM)S6{oT8QJR;;bs-I9y-eS^7TlWJP4+v>lUt7(
zf~uztOEV~eHB8P<m0}bX+m8m{1Oq>&VF26X|1k}XB{I?gHLO)D+4>exl_M+{;ddp|
zO+B}P>S<mFy#~)om<<Uu9t&45I#SOiU?U#e;EOgV7Yt*4YX-i7Re<#o!oK)CZ49KU
zDgGJK9}Wk$0Wz2_;ZTpWild_`LvgCH4V$+>tAy*)E7}H(G9=tpd_&#`e-y8}MFpTl
zE(eH#fS8-(x(?mP=6?yvp*uln%uiLewEmzg<xSRYDgLHn&aT{AY~dmCpA4*>{pYz}
z3}y_el4JO^dX;x^vc1?pvv_b^M^)iy!;znaAC0jPja6x=4_clwCT1MI-D{z9nFKPO
zShHRj^bXQLBFwPl&~kJy^bE>-J}(wbp3%e_enl+63L$+U>;Q5XdP4<M2>-q!L1TI>
zK$f{`-U0~?0`!HfGPj%Qngv=5iJXa80$!ypZsv{^xs{YXvGI$dHr+9trJ+-Wl0A*+
zk33)$#szN1fG>M@LBOMa0b{^R9Lc~^0ofQnTn<+k%`fg_7CVHgP^k!;QUbYZW}x~T
zayGj{JrBp1<KoMxkYxF*^2>2tQgVH^@Tm&lN<$3?91DRJRHu4BqOmVJeDU3MfvZ9E
z&WbfZDm9{-gHLGJ`T~(l<*4SfzS_H%SPdj)O*HEa4S^npPH(;07MQtXMjM>W;(58$
z%wzn7$d~7@FN&s)lt@^|I5O)3@};If>IlZ*C|f0^^@ya1$=R&$Zx!U(Pmu<*KWUgM
zfL@qVhP?z|te);tsuaY@_S=#g>`nkhrFNzCXajf6r7okd@_ZA8j0A)zqoxN}UE}*R
zjk7hHY`FmV1(n{H=5R<X5EZad_f5mp5L*E;%wvWq6$-9+2hY<O3~`4k3cXm}kttRc
z_HZWeS;0=8VT>eE0<CZVRR=H9Ow#1-PC@Q~`?7|bq6WmMv!i+w{_w9KM6{B{@~$8p
z6+Ba(Yk!Rmz!2bL5bDlzbq9GMdtxqWxusyA7`?;+rv%}^Ny0R_(&8D%OoT=PQ+|PY
z!5O$CZQ|I<vLh<qvXT*874=jUMPb005_thXLN=Cn;VxjHVh=Pht7qAE4j2;3Ilsv`
zHAJlh)m;1HiA+_JWw#r0_N~~djb+DR!If=%6I*8S%E3fW2)`J1D@WE(Uqv^cDF!x8
z+NVfv?`3N&zRFg>O;D75?LXV&sK%gqA{1IW_4$LFqohEmC9WSP0XWha7?ZFo%^5D-
zj4?&x8NS(_KbOLcR0`YIpyYyRrINN(O6<96I@(7vx>=eZLG~@cg?-T%LODqyW_WDx
zW&A<>?Bc*1C)%5A*QYo|2}0V#1AI6OV;sr}Bh`?oWJvmf?g31NAv9}Fr>;fGMe{T3
zX;iQv8Yq`;I=zT;Dq;%l?@<%nm|Le%0S#Tq;%)f{$F5U!zuPaXXa?8vZSAq|`CUMr
ziTp4YTtx*TVl+Yf37;71g>G-WqgS2KvO#h8&`e)}F<NU7CfENjgB;2880Q8sY;-MY
z-jLWV*T8+1=iZ7*`*8h4?$46Uiv7)Kz_Lu&2>9sFV0SQjV<um=mjQ=F3$A>QWQ=X~
zpCLq>+?pR9IkQ8XsgH#jBW@v~;<evdb(Z2NG%hXnS5Z41WemOxDV;0I*p89}<f?!(
zIAXTlBOBs2#xhJ6`Or;5_g`p@Tn0}?>PMq8gvU;c*i}oX&DeT&ozlwo+7bRSX`7k;
z8tlI0>!%1cx27jXwG$_ziiz=P$`}pBKp$F|`Ss(vXo*$ST-K9Jy73^D5Gk)Qpzk5;
z;Ab*jq)@UO9bv&NZhf)HjS1}LTQbHinIQ#MAWH;%>0<-v#4Jwjm*VwO+*(9PZaoQ2
zc?L%ThwnFU=6I&Th$Z7fq1J2QcV=cTWhUa+ne!?(wa#E{Bc4W^fWOOV{fdQOv<86y
zx8o0O!1C5NB9UsmzW7IEF`MYruCZq0KYr`sC4kq+^Ab)5U5YC8XdD0g9Xug@1H#Kp
zSWh?x=T<F^vp;0Yr|v!<A7VxQ%od;6hRX$mlBoNI8)LRbIkrhV3<^to%<1P*!MW=_
ztzN`L3pJ?VuR#T#)0<K7j#4l>^S;nCz!{rRHIz&0p8G(1*H7zWsr%tWF8Ao{4SHLe
z4#au4?smc7gEDC?IcMrZ)M}LP!t=Xo3bwj_<onxs*ESlMntjS2J{csSG#JcQD!6cS
zX&eqVejjd<8ex>|kEndM8?!^iqH(;wPqW)YJw&~j`v>gb{Ku7mSqunbAXi!ceEonS
zTd^+(pHe9}k|z8H|I?5D@$G>2CeDy9afSrLgH+A^Cbc2?FQT+8$SE>(JSR<D8ZRi_
zYnXs}H`huf0>iZs=xG>nRY{r1`*Q=9)X%Lu?!q4_)1w*3>V{(nrLHS=E%x3454VEG
zOxD=k9(VnqdL?NjOhFi8wsS|pDg*l|Ola^!VNR_hDkEBl6aRJy69FWmb(C<!p5vcx
z9KfXcMQ(lgOAzVDE!d)j$ZNn!0dueho-QV-H#X@`CF4biKa-ODvpL+2!Tl(X_I;4t
zj&u!qXPMw9=gfhJsfva%g4!ek9{ZX;O(1fx*h)+_`5U!Oma^;~raEq<ATKE1ru2SX
zJb>laxVWHqQp@OGN~fsidK@1PHiT|z^sx*N=P<ifRW&5FAMVF$_!E*N-=C)v^6!9#
zGFGg3@GrdwckPlzb@G?mDWm2wYG)xq_^epTn2-^YBV%xV?&F{Ut`j(izw9%h$&@xH
zrHvVu1rklyjd1a}1YK{tA532?!A|N^qGBc&tA2eD%-IKxx7Hp)s;)~b=0Q;8ErdY`
zXbJC!NAG)(5xR!FQ)Rs?{9WQo(+iLBa(Pr(*6#D-=3l4{W|aDo4wq1bptmCHFYmfp
zn`tt+9s7y-6|E*gYPIxBsZcWJ9{7=fGU?G;Tz7DBIxUmI);u<b#y+sWAd!9rD2h=j
zTj_HHcy`2>sI4FAKyM6xnBStJ%EHu72P2uiCvwl9_4t15l<{0%%EjN;@Clq{BnLQ+
zn8c5g4;h2~@K{%KwSE&eo|q3II@OT>e^WHS!;v;kh>2?vv$RBlE?}C>1C54)V7Sdp
zM(go1Pl|gjzT;+HBr4#+ii;4>j%|zHv5Z<y7JCs8C`~FWyZdwLEi(+S9r;yCJ&Z*p
zEX;4rLP|wYDrEkB|4W6;WYZD`=f*q#$J159Oh`0hiJCsvc2jV}AdL-bVD&GTQOTN;
zZ?xsV1JDs$#Xf^GC0ZrhvMg|Cl!8jLl4P{w!LYVsUPjM`J-S|bL+F@RKql;lQs8Ac
zBm9N`-i%N;#m2bwr^+jQqyTg^lVvU@a>E`~fC422)px7_Swvkj#+)T(22(E1Y~cp6
zK%usTh3DsO1;mmA6X%jh#{})q2tcoPnw6w&pK&HR)R2N8^|&4}3rv%1M_F1}?CAsC
z#*5<71W+&L^r#sHNpJf`ITkVrgZAD#+A%b-_@NL)4Y%B&i_VtOOLH;;z@9jh8;$p7
zF`tsCwY7jWsi^fcGsK{Sjx{D^|E!v&ZW`yJvo@Ch5Q-5%WO(-t;d0d0VEM$-`644E
z^?Ldge|PHVHs77MNvWt%)2nphBVIJTCi%Ak#Hde$$)DYabc|9vRy_{w@eke@av#p8
zb0@4Tq!#BszIzSr9m2pfVswem?;YllUnym3rc_Kb#osH^R(%wH{rabd-7^A}X0K`W
z*7X7{x1*^kO|IaRvO(sgh!Z5zMi>b|0pDmYP3(0h;hSeC8Rn`RqIE=^r0YC=ffVtN
z2)|^=<IQ10AG9iUnp*7uE7mw4y|iHeY)32L68|aNmg*`)IQv1#sPSA`6gP}$n+S*o
z9ajY2KI*nSMixjx735&;Sc$oKQU?Q22USnkO7~%<l%3YEZM>l~UKXu(&O%kFvQA*V
z6I`Q7M(VjK{;?FW4Q^BazM=L$eK{|z>|yK6Nm39%vgMmSe$5oSSTJhei{;kjkR3|U
zBGps+mZ-~O$L1S#D*wl*$m*Fa`&@XRK~wr1wM!|(0|5ia&|=Qt>j*LU7bQ*NJ|XIt
zP+B{>WXgUWSv5|aGkk#6TrrD1RDzAD{V`r+<|OQ|-77dos4ISK7!zr~PSqDCEx8q&
zxX*;2v>G1=HKlVI4H?Vd<U-CX2BrA=Ax115$4P|+1s+@9M$&He%u;}&p_z9qib=pM
zBTl{{Ei4wEbQ;cW23GG|%9TE8ot27cw>MQf4BwI(KWQ)6n1ephBA`4>T|UBSc23z7
zMQ!;`TM66$i*sB~6O1`L4*zC<z>U%ys85#*%k3ASoLNeMgd~Mq18d{)%w49Trd=)Z
zkB(FVyS@#rE$itNHM@1apjS7f%9NEDf)0OKR^d<Xj=e|(q1(;P(p}3fKInYm)}G@d
zf@6Jc_nJLUyu*#Hl-IF?K|zf4yilcHWUmiV)35OMw?uQ&_95)|t{vz}89imq-;5Xk
zp%>XZm0-gpJ5g>gGS>aBjGv+KfeeAkWx6e#p~?n&%f9CJKR{7TN_Eh7I4gKH>PCtB
zhB45`@?&gwjN03mDnuaf-I5(#v}52Nwa}+C6`VQPer3zjbd`vTVe4mUSWLRtSsy1(
zma&SF(;)40P72cl6QT`ELJ-=9e@IQVv>~icr)z9|+9(*^ZDOlM?*~|k?`T^$aTLjR
zYMq<^`DRqAsd+9kh|d|vHUa)K*O&LpdF0e1&7GKhRn@4YZHI@rH_(3DX4%O75Esg{
zZf3UA=AH0}Qajd3G5brSScGsK%AJVr+R#od8VOt`yR?QabGn98%E+JpuFR9XwXmC<
z^;oOM@M<byl4=gtDjOFaWDl|Q2UXqrFQp=_kTm3OrbG0dLtMe>f0u?lbVN~7%zKi~
z)ZWl{D&1&Wl=6z!q@wO$nqPYMo~{aIW7t&Gcq80_8Wlj&&_EsNEbd}^<uUE&aNwZA
z>MU4%Oxwc5Uy`j)-Gdn-VH>zonquJ9bDxiz;BuTG{sc5d<evYEUO=|pd$mUGO{cBZ
zN*T}=*9GcY!Mm+dg_P2w-!Je&^h%1Y?CR{#e}+C?Px3}i8E*Oiwb28!v_~6nKKckt
zm)j*16j5^WVd-VeZLx<L;%|6wzjNlN;%vWRyi*sX5%mnH%`onVHQe3Jz*NNjEEn^J
zNZQz;Q7-WhO1p%Sm^Y7p!=QhRTzY;y%hk;&{qk=PXw|b8_2ygXC<jGIH45#YhUIPW
zGVi>rQMoKZ^;Oh6LUWNX;=LUjl!@UCj3&T^1(}TP(Uk2u$5F;YAfm)s_?h@?*i5Uv
zR{+FM;I;8^mZQ~%G4r!fg<$SDltKmf;75IZv~`yZeg7Dv)nmH$JpYCyXA(O3F%p$`
z$JYzAlwB*tbE+1<@+sQ6%n^@$?eu$1wnwjMg-oS9L~g{sD+O(J($@ih!18F?Ge>?9
zeXaNe;g|Tm)7(8=h2E%oYM1d+wjbe-U!s22PKawku<V+H`${DWchucfu_x$dLzv)6
zTJHOnHA3t~4->(Au|bKM3T>KjkF-KcR>dp>MGK|(?4SGRtJMSnEr;H<m*KEr+0;Vt
zPwMVFb_+-#XDcm-jHxr5&RvZy9b5<bX*`XvH5nhl9p|C{*1%#o4sQY(6|X~m{z*|F
z6$dNWY2X-6Tnw!f=rTts_gbL)EDG5gkw&FU6dH}&ez`KnD;8|4F06X*uZpPbtS!}r
zhZTMz<F68>zKI+Cc@6Xl+zYWM4O9Qv#v6tZwt0^rO)8iP4z0umK#S)McNaTtLlOX<
zU`M8`?@i)2>tJQr;iQ5>VF`OlQC#WN>OdQ+(xT6mz2AjtPAb=Ok2&d~Y$$8GR)Pj!
z-K`{Fyu(yy$5C_$(9+XEXpc*^OJpOY&P_<EBJ#{<I?50~d+-tNk^$H3)EYA7Nz8%(
zv5~Y`GK$~Kb8a5|W)OM^5%~eJsSbf!i&l>4_Cq;xM=DEb|I}t&ikUymCve16b}g#o
zqzKXeer(sNi!6xt#b$0t@HUDfDwW-7{6xQ~Xzuhz3d7>s%Euk_>?E2f7Q_?Y21jrp
zuDu<0M}KzWFD@%@WzhI*f4)MPSpWfaWp+b$e75Sh27-S!Y+L853BPg+|IT1vtV=i9
zViLHlAh3wv9+7Q$Asa*V-KjiJs4G)S%{jcOL6}iBJ|t5CvRxEeq!u5zZsZjK5{;>t
zL>-ET{+)9N@usxJGnU-Zy~DvBJ)tL2>=c4x+F)%up<t+bETxvuL2cNK(Sl>-L<`0-
zZ^%whPUnR{c&PU6@##@MQMA9h?PpEx%CvHkx`%mV-S;1Bbs=?oHI6|Z5zp@$w_$^M
ziG6hup$yrAq+pB2MX^8dyRLxc|9)l~$C=zI3gmA)-2vN*oohvcZ&G^yN!z<!2%lL=
zyiuO4&`7}yyTIORjG_63+j+SUrsT=CK}_yPy6|KXBj;{r&646j=LcyM)C5Dx*~r;Q
zI=3CQ47d)?`UnF-+T-&%)*0jyX>YYlS%Ol@53RV4@aQ2)>_GMQ0f4md|7cw>Vp3`W
z*Or?GEu#)%3dO>HZitme+)1^wDAFj+@%&!X@7<+^CFN2neL;FX8ZJ*&HndR}f^<zd
zMjtUW)zj`(6&F}zKbqG1^5J={F1IiJ25qgaD)>i+lJnrqfZ(S(lREVN3jsxal3dT_
zFi$<mrW4W-4TA>Y1S0|#vK<^e+%l4~SiN_`He#{95ZJ@Q(UdnxHl^cg<M}xbZ)u=_
zr!kb0?87^95eb;dz>WJO#0a-$;1tSM=!qJ3u|V`AOPD$ONlS3)00)T?SnOhMq-BPM
zHz!l2(>#tFU>X<GhHF|0FEtH9w*9o?J$bvkbeqOKEY-iQ3{_=*zNvDGj)8wrG;zQ;
zigL1su;3fsPBq{e$$oOaV4P0sPeDp{`-*x$|GHp&iMLWfeEv*sC2jO^?k|s}5HCys
z_pK-m6cE(8FW9HawtKFhm|vI$P=JCN)$nyl_-Hs!UfJ!5p<po(-(vzlM8?ORRJ7d_
z??cV0n)FmYu%>GqydxIA*7eO%x#&ED=D?<DhSA&*gESF$&q$3B#sfuNf)o_%ZEn4f
z=d|92<bIahlL!JhHHl!37qRV+0Jc7vgyA^;LTSqtgvE~;Gf*W0>T=!EdhR2&*Ts+&
zCrO<N8p^?2dI?Zdit$YLJRcZfs3&a=wNY8g|JH>x-q$R<aAlLTefd9Rv$OF{7M`W+
zK0iOwTR!4REmt?)EMOLtcA4vScRIxe@a|u{>)0bwgPE4zI1)*!dx;Eh0IE<JY<lov
zyiF{bu@so`LK|JX8Jp-$D%&Vqf>~3Vo<_KB)n>Ef6vx-duJfQB%AVdoSto*vj(^RB
z9STd@@ez(evzhdzOo9({z-0N@n06hc63AYaT@k@?LWaGTNk&2W5)(dB6!-tn+fu)C
zNLL8M(DsvCXS<y)VB<rpLeZ$Zq|>z(HT~QthHg8g&19*!xv3Nu%N-VoosFixut8D-
zeu+epI}wH?MXYOi<Cm{DI_`VyTiV`Iqiq&|iXy{V;xBzSRLSGh$jHsgjy1zk4|I*&
zx4&rGEB>R?B80uOn}6O8o*N@T1#?phFYP9<p3PA|?`p{><zc8M+1Jy6a5(PxlHtG+
zhI5_TyVg6a^HEhq%0n}PVW91td(iKhhy&PaZfDm{B~!Y;SJD#|mc`8%ZcT2o<)k;{
zmrrg=s9o#`cCV}SkH^1$)djJZ;-nTI{5!O!U`_4f!{tZo3l{p+RRxq^zi=zWXVb>u
zl{s@uCs!S<Pya1vPh#*Z?)jF+g1@d`tqAx%XH9KLeoV}czv$A2g>z!OEIy6(_)uOd
zUs_X}bcOEv-@lVzwBx(@CzKnG%BxxHho61LeN-Oae%n8DJzfSZzEQh)i+T}m_(#6_
z9c^@TOKAW@`FbAb;^ZePkiHr3Be&h_m!0*`qeto&rY7aS<x^a6X@I#pr1oLc0=P|d
z=^}t6Zj6uH4?S=<EvTH~g_M$B<E%woyw_Y_X5CANv`t0}SPixPN8@c|yg6B4!HwY&
zD&_}b08fJe!_gl`ooE237wM1$FHpibeG63_{s&njoZ8yj>k$DEAbt-~1OuZ@+(_Y$
z(HL<SO*{_9!bwDc9rTMkz{6&gj`XFs=+Gp51LP$C9*()gXb1q8m%UoD?bxlB8dFAN
zW>fzw$%Q`IStX69b8yG7@t6bR2Y#V2AEOMj@4~5_v@Ahkd%A%{mZ9IksJ;i}Rwrg0
z4z!adhRFc;&}lI=bO0*+EI1z;Y16Gq^i2?2G$JJO{A+x?9==KaCv*dx4@@_}(-55g
z!MEI<$kU~Jjj7^edMC_n#0(u{L&NDLlT9Hw(;ntjK$nLAqwuQzfr%VW<^|*F4UWf>
zPI4z<O<0M)7l^xAaU2Q0a%wxR!FKw*Xj^vHxVfdx4REC8W!2a{Mq@?psE%n82?=UJ
z1EWWe0!t&ljFgVX2$aO)Hq3#hDRYLbGWFZhItJqg);j9oX&i~mN~q7D*9-C;nT$K`
zUk`GQzMo4vrL7qx<Sf_q@o}M<d+7@#)SxiS3Os|JR?|6Uq$HZQYU18X>jOcPsvBqN
zM|TlTS2PdBC(_khiI5`Jf@g}fjF$wbQdudU@u5_yv*=7hY$I<}HFz%0vV!gc5YXzN
zZHARzvkuiR7T~yB*bDw4MWuVOO(gWi^;YOMYXK?N_u){~wC)MD?u~TcA>eKTHmJfY
zpv5g1Oz(k5BTHrb<TZt<;WHFp@h#N-f%3_;urr_DNV%kGf7+~;&-iz8u{fwY@T!xN
zkeuT+vr|=q&&?wP*6ad-&K(?!zuZP_Mc(U66T=ZxY~?r~r6TJqe1H{Gl${j?rASi9
zi*NY^CMNTk*%#T#(a6jc$AQ;-&9i6{fN<)3aLzdCB55lf4M{pn0%F646JqlX$IQm9
zanWK$m&c*(rLQ%jB-C&%9Zg8hgDhe`k`Jbi+scv#?pFtl<;7{oSf~nAxK#h7c%Cs9
zzu4on)T~Qz2y{VYSKz}n&(NuNQ1LW&Htb6^^W5u2LSkb(m5ZYTX&9_Q6(k(G$69*C
z7|AF_m<y^_Df$cbp%oY9mPZhIsuQhB>qhskF2Zl)Oa>S+j#_hPhYL@P1(fMWmYy|L
z_>EfSPm5RQ$KFX<+Z((x$RbEdj9v$BF%PDzIP}i+>e<NphfTJ*bB4B%XJ<9GY7S{D
zANUrsp+FN}xG$`jLxn^Q=gC6B0Rm_RLlyB76vrEi-S*`;MFB<iTKnPYTvm*%guM-T
zf07K_sIrm(#w2Y7DFP{2vY^~3wiFIR#&G8BhTuFM{lB1r{1-QbBPx@^Xl|N^GDcD{
zPpF9ct1@jm7!s4Et?eu0Mmp-TCyG3U(UP`Abr}Lb#(r7}P^!_zvW-$CVg=?C7J04K
zW2-6OmQ{d=GpJTc3Jvk7oU)2kfLz(L<8US39$T7(zjcP_TT?KfJk`a;$*4lV?AbVZ
zdq95rs2#N7TRDW!`PrklRn)J_2xwKbT373dZ1^u~_`L#l=R1_7*8AN^pZM#i^r+XX
zWW!U|XH53!XX}A7YFza715R7B@i#*0dv`s=<t@EihxTNd^3K!WC)b7{ZOU;Lz1f&I
zQ&Bn2{F?L;Id8Q(doqkN8}$r6jWW6yl_r=KZ&#(mI+yRHW<;j@TFRUH*YoiYoYkdi
zJ;}C~h3w;PEi;HT<a*|dTB+aAQoKwW!x(!jM-CH~4gEWG5o*uH1U_NK+DT&}VnHM+
zEg%V!UrNrP6HK-b!}u&q7&%&}nbthao1t?z&nj$LoH07A{_?x&g$<V<$Sj<5_N!k)
z@RR*kC<y>%?95&qO4q(EdtsRrwYlF5SA>Lo1kH?(L3X@-gm?HD7+Hb&z0hDm#FD-X
z$m;*CwD8DDb3DL?%`?Xnq-t4*b#3`jXewc$%4}Rv6Ba|rQuQ2SfFu5bY=ro<ihKWw
zEnw!I&$mQ^X1fI7)l3l7t)&u;2hydRkcJLBTZjFQg@4eIq{Jo|_yxC9;hayDMB+fy
zhTvWmAfIS$Y*i>T%i;@D%x|vjXyBK*ep7NpihNRrq@R-JE&Gg@xfEi^gPI4t&#^Lj
zD-8i>!>Y+-<We+e7zAAj60_uKu9Y<PKd5Z=mR1}Zng4HZpr#VSH9hq}yU5V}qK{1K
zupkgm05B|raU3?v`T^&udL?8yunHK-#BIn^;Z>Or7iss;<N?yLIUt>+kpmbA%|Y=G
z$+?|#_;hFF*ceCL10)t^uq(LX49*(88|adl&I;KO>`AL>V3yo>Gc*beCzjERrEVF^
zxv5iY8UE4&9aQlk(^-+cVNolrB<F_UubSn|LJQ(JKqEu1FwP-%N>!W7=|CC(`*0dj
z6K~APhvUZq)gAc9h4;EG9B>=ql6Q&8E>pi>J!y62{4`csQSavazA`8)VMpXrcgdl-
zuTa~E>`LIy4Qt1q?9vl9kWuljRhz$iF&+HGM@aEG*33bmn&Rk%-UayU6gsn3n$VE3
zG^ETVAx+6w4q8BJ)YvkWX3gi<*!+_&Org`Fo4AV#xD6{xXNKGE=*#DXvzGwgQ0?3N
zzVW1Mo>$Zl-vMvIxJhir1DVamP<fb2IO*x7wABCyAuS)$=^-JqiA|OTUJK`?Onrto
z4@$c{H+PvQcqpey2ZU){E}g~T-cd>^T9dU<xk!JKzmJ^LJX_A*<C-){()VYB@equr
zVQe5K&VT(`+C|Si6f2!{>_pugrtfu-L)~pl!xXP14Qb3*nLnqa8C~~@a{%L0l^^;E
zvc);S%#$-)?0YQ*a0x98EKqg`mav^v4RWrDkxjNdm&o#5K<7>IzD(ZUzX6A%+b0|#
zg#fAr$b<7^<a~_gYT1PYi6JYQF|TaTz@7F=8=Yd^5KP-^`AAo)7GeRwBl6BKjzoY5
zAk;3d<`>4+&$uC}a)uOAu8J%#f8y%s{9Wi(rvdEO?JZQRQjav|`)t_&u!WUo;d4`z
z{mJFdygWM{Ouk{vr{G{0(fK@bkXIUU+;rAduur&eGrJ<4c(*R~l^zNF4NsKT{IEad
ziMNP~FZhetGwY65cyp;jmvZ2++!LyJl1ggATJKdyXGf#s3x`Hq3X<gXbe<3vj(k>d
zPdol_2mVL}t!aUJf(1FS@`<P_S}L+GUZTK|8+nnOLqah%H^KgGXwxL=R?rH~NUo@|
zSelhCWk2p$J}So%({3zm9LHHWgOW17sA$z6eDT1JTZ^1)SPX5RmPTi_9f<$+`S+4m
zC+h9m*zq~(6C(VgmfhCXHvjmb6E+g$L?;qcy^cDR_12%bk7{KrIiVlr{~NW>J)kv5
z6B42;NK60mCY_!~_(nMvjn)mJxp;V3%dSD`6Ui0AQ7!o5^OpDd`Z0VA9Qzf~RU%7+
zS0hEsm|WW8N17N+8@h`vS1L25b`ZRodG-=paAV6OP~^r#anN~KK=A8VEqVjO;xkt>
z8Hy}Ex35mCyjL&zqGeqiNcd`VzIHXzBGVtulbAbid@`ku&$;0We|3k_SVmc4rzl`A
zNSP?Vxqr*od7H!Pek?2d$8nA|RD|K(JXFU10T%iGGr^6E+_n@pytAZWUG1_hTuJ^g
zi`yvsaB+zalmwhV5FWb&YR-MyyoO_hnl?!lh&?gr255A)D73LG^CRo^6s6}`35~RS
zW#VA<i@en<iZ`=ZJMp4R_kjSbbH7RXN)FSdKS4DMYaV$ej}EruwjZs|1Pi^py#_3}
z3d0*C=tT6p&#LFc;7@R>horTifSvqw&azVnVd7n24*4Kc!}gu#oBvOD-yRoLwf;?-
z+Aj;NQ(=ljmy?~MAT<S)fiA}^sqBzSqJX#3ATEZa442V7MrBHBWGUh)LIncU8b#!y
zl2RrhD2#GbBmo%_Z5U8)zTf9rduH|wo!<Aa_w)Po@gsZoZSA$zbNfD*y|#F3U%*gN
zuEW5uLn~g_7Zw$1YFeUhJD`%O8jNDqNRI3}Yg$7N0%#mO0OW{jW2RIrtV0-m{o)_G
z322y$ed(pMkXKCwlnlB#b}bg-+Y`<#T1PK1fk@~?$xIB67yFZ?@b<qeE~6<Bxr~|O
z1<z;e#o}~~eozNRC98FQ7R%G(eF#r-R%r>sn4HkX(mFp#ZUR&>e89&jEGnls7aIDb
z<uP7?h{mz)pg|NglX_*CK#<n!%eNe1qRp$>@D#wGh7zCs1&vIWcBwlhAkbX9W}u^n
zf;vQGup%PCphxzyCT7OOc<A=A^JEAWJ~J>#Cdqj2C9EikAqhf}X~p!ZeHO{qw3QAf
z{2d&O02$;QPLm83#M+1@;$(;V2B(y?QZB_zL)cF3updiK;Pt=(=`*ZzH;B=M>D4Ki
zPscehy_}+!y%DWz1O!mi!d<dTdBgJ}zbF}DE?uWpT@5ozQT=|Nh9Ao&l;MM+VLK(-
z=+!mZ`pd<G*hylvEPpLt>vdu?nM!G4Hby>Jp!g3~zfrxsZw|h4&qa4;4y6U*#2U;5
zb+`hbk&N93hzSW)Vg~T4#UcT7C<~lS1<+haS&NiQmU`#A_)sqxz@S#v{cB&!<Po=*
zPdvBiuKHWY$3B|X+Ix@x$&6hOyuILOr_mhDk6y|!qKUCp_jt!DX#VtNZxOp5U!a8I
z{zr)$`6NLOghVDXkCZ3jE#)52mIo|q@(qOh!cY(-vSoPD;egjX@>!H`%_Ggo`jyM{
zXb-P3^);wg9>j#rszJ3BKAaAlvJKD1>cw;wCZ-dQvn{J>qhdt;2R%iGuV!Wc)=PL$
z=fQ8Fr}3jyt9~s~5l9m3#&BiL!>Y%c6OwB1;WflufF!5=jNWIwQ%X3oafzPaCeLTL
zJ%Ds_@i|-%h11CRDc~1$uKVVFxB!C`!xM$V{0zf00ogPo>Ac2_I25j?^`vXuj02@_
zV$hJ{W(yea>evZ!H69Ik@jcSVRWOnQj8C-1L>N9?!w}JMf`&8ERtL2=EBkx&D=xIy
zjkYuOM3L0iL<5GQd(dJS2gA@p?zC*N1P3oGHcPgXRq{ZGSgI-@%65r5XJ-EI4yFC!
z`QKMKl{H+U!6ehViUz*MrO<fo9O47j+w|1r0E1uK%H_+CmzQ6jHLW-$Wx|FHuV+u5
ztQfO>+4AM8%6k$roAYzcYvq`0hJ3{G#RNO5OrK;3aQ*$PN(M&6nSjNjipTOyz(zjG
z^*s@AH@@Un0t)`H1>GOEFj8;)_=EYW^%+YQbLFp%sRGC~33$J(k+|WE1^(&VUF}JW
z<aEx#tLLH;VRU?7L;8b6Gu6~GlAknPB95@wx^%DSBioQ@6*LB;Kq8)5Ekvg^5!~rq
z>GoS~e|SjMA|YJF1*uzkZ3`BywJ&_d5{8SRh9O~XcJ_YX_`V-X^ke1}&CGn!njyg>
zxMk3b9lVU0U~cC3|NNpd0Ljg3Rl&IJUk1s8zQv+<0q3BRj&rpr$v|rpA&2~ZUbU!a
znZ_C~mo7oD34_?Pei;RIA|oJEVk-Cvs}ypM<V&9W+3mMH-%r(_ElB76$|QPurIHu)
zari{AySa<k#+qdktp8iyu(xy{VI0*yC*?YbrfKbBo_HTI%g0HeM`V{kl&g=M45hRe
zJb|L%b0^Pu{CfCyUT8jTG`*LHA$Hl>X~ilop(K!87JC8uE>FFWx|^jw!bIfJ^*>7L
zNrO^iU<a|t1$hbmU3?e5L<a%Hw9}azI|4`?Q%CW2(jtGANwlzS<vhKpWAx?sUAX13
z-qz$B@}i04B!A3}2H5=}Mv8$pMtwt%&sw)qPbH;-iu4OQ^!~NDg>VjdXMIr@I*+$;
zA<xHN<Hrew&g$#_K%w|<?nx#-6Z(fOf#U;0_*vyZNsx7TCM!Uu7KAd2{~E`<h%rlr
z<_7HT#Mh`ciL^cVg0#6Kb1v@;_n9QylkCciaXbo%E6rnk+EH-e%{^ZDz7GZ!(s;ON
z?UFE|M0=#@s?grU$Jn_Sk~bWM;*x$|7_&e8>2FX#N!2c}95jR{?MS)4wLRX@OGcpC
z95c{d(SNS@hM-y8U-!l}I%?t{>W+P`kof^`^wy0OE0T$~?s(mR$P~?8arDV9A1!pg
zW<3cPG)@2K3(RC>{$VyGyWkqnJZ-``sOy^~X%uu=Q(`PEGAtRxa|7QAhB63cH5fA(
z29v+@T#VU_L#<%QS=+*eh{^<2{oA*`@AuQESY*>+h{XDC=x&Mi-|qJpazp`7B2Yko
zwT`#m^9&``l|^4-W7&e;kB#ypOFzlhr~vE8&S&D1g8x8vq5U`(DPAh{S~cF*y)Y9}
zO#A~&M+jV(A>QWa&_E4Y^HFv>GR>P#6sh8%NV4>wv3$}DXm_lEzc$yJa<O}n=%yFa
zCd)8f_7$EpF$Rl8fW88TgMv$U-@>M!R(nug*kb}HlpUBWMg=_b4`Hue*&PN}C)J~y
zF+}W3mH+@^6b&{j-We=NX#<*Nlv0CWh^Hh(0H1J366}fz#Zi2yuL0xcwSZ&?l^>!S
z%m`^P2~@id2$h3ZIS3s_D%etRfKUArH$&?eVkTva<|x055bKF@l{ybx_s#k7H+~?o
zx8%&-ADB6lxrhuI$)0x2bXFE3!p6_D^E7fMM3K=P%aeNHNxU3S1v^v9VCPU;j;m1|
zgca`I@27~doZm(YWX>p3`UVXfLc_GPSx#Kb-2@N_Xd0}gjon^<Un(tvvf`rwQT5=L
zZ*q7GD9CrHF5#;e&W4Wk;<kZsrTltJ%V#Mmr!zLJ#2SAUB5FgJ#eld;Cav+OY=7Gv
zpi!D$j%vq(#o|1IIOe5nqK_OA>-yp7c@wCJjy(EDN1rHKq?0YtCqPJaop_Vo@=Wx`
zpG+;7&t&gL|L`RK1{JotC|Zf6`mTiv7ZOB~se*u1lNh}{5V=5oY-I~)T(Fp+sQ@lE
ze*jznEjN|3(S}tvIs>}YRAJMFZ6<6jDFi1UnSM!UgzcNfR*YZ^h$s%_SRW#r4@n-r
z$b6OYP>VshiAL?uNjm##og{V=3^6gOcLbPJNEr+$iB{K<qR5}vQGjVe@eq^`3d#c7
zb%{v_N=h8pZO-La!Ay{5^<c<r2!L;7vhapH2!Zc!X<!IGDjpCV@0aZ($tgLA9ebN^
z*gB>Q<Cx`7nrhIKCl4A}BfqUAwSsf7EFVpUQ8n$dRfU=$<@wO(M$-@^AKBoFvisM?
z4|m6kC+K-3j30}c=gOQ7>`3{r$`P+`pl1-Zp~Y8n_jwWWG-1WHXWxTv5g5k;t$68o
zi`6n2enML{HXQvBiZwdq;5kx*1{ds*adDu=PCbg5HEJ~p!1kR1Cy-8pmqgD)fQP^>
zVm_Vm!;>l)_=4ti1q8fjRICc01^Z<yfiO7?)!L~MF$~?<OlxQKA0r?~IEFRukwx3|
zUpDjW^OC9)9j##WCF9EA8rOWL$_jtr)xomPW~s#p2q%~Z@GMjlzaS&CJHelW^i&rt
zJC-40*#k_5^U8AZif9CX>X84ARY$~8s4~)|LgD=C7XCMLh$Y7!<$#mF@Qfk0HY4Mc
z0!ef|&{u*h5bFD%Hb<r2F+1bKmrQKNmGBoy@Oe_Y-JXFlB}tOmkJwXx0AVUb?wOr`
zxw+^*yj-lZDd0N&&9b{ta+AJ}nr7lm3P-Kdn<&WYLvkFB7HO7Z3=j_K1$q4R`#1Qg
zkVnNM;mGKhuF|O9Rtx{1ou%MKvtT0<K_*Baa~4}WzfCA@qCQA4p6&iZz%c7p_kS(I
zu0jpT*Tef!tcO>^fi1Bf-tA925<svtfRpN&8h?OZLTfP$NvQ~XU3Ukj{k+w{LdS{n
zWBLq8brT}#48*HzVduo9X39p+J^5m3n3WNCtpyX3rzW$Kn#u`m<>he~4ov9Qtzqpf
z+$*UteIwp0W27t!8jN(R9wc5IvV<*8r-jF7T=z$S=|Eb}!%7l@M<a^bzmoDh9%ecg
zJO|OGgsPl0zHj16a!(M2fjltYtG*9WHOw}Z)T5LXi%{`~_j*2U6EBznnwam)a%neX
zBf+JU)&&k#6OcEt8;>!5I2!=cq3c;e=VDS>eJBq@Bp@G*S6G}h3kukJ@#>P~3km|M
z&R_{>wUy`@Mu;UFVHtth2prOjx4_ycOx`@GsaB7g=dFmK@REPp#O5}NIK9zStD>R7
zoicBtze|Oc;H#z0`~CWWyX~|IPh7i(Eka#l*`)T$YvE~2JXLqC@R&R~zqRG$Gzz%@
zt(3+rUv4^tNC+0Zvxsa^9A$rU*NBa#G*7bW{qL@on>YfqKiRi&-0M4yU#%Rr!&EZU
z>5yeSIxQXT7~dS1aM7i_cnx1`b*A{RsRi?yu2o;l;yuQwjly4Vj=+eB5Ky(@jhH_|
zE)Vw%h`E@M%d~Fhid-{t87c(RBIE+(^81ki$R(i%qztr@*nrK@kXul~3dLD?ild3_
zg{$sQv9^yYjTEtA)hk;ZRlDx*0&fc-%K#_f)l6TKZcFzlzvaK#gMghPv|-kq)!!e5
zm0~a=gZbHgEL9Q%8#RpTo1djsPi3~lkPb&2Lbsj)P^3Tm@8C~Zm}9$j?C)1PkxN{u
z2?pfxXJBEJSmB}h=7@Ewgzn+LhUp^eVDi_E-n%pO_K$Ql1h~QM##c^)O~q$;HcEi1
z_5g>#Xp`l8N%*@0`3Y#+F6@$sK#1ouLAoDy%NL*I6M0(R8SZe(i69q+FBtReOGfJ>
zo<{)nmX#^&yIsNlf2Ht`n|M+ta=hcYwIT*m!s;W)lQ5sndh)i-aU~-JHJn}rOo+IK
z$+0Eh*ZGUZA`~+u@z%y=6zHZ`$ZhEbi|3ylh%1Jx0qSItXQBbK>)DqC(w4#McV;*7
zN+L4Y{LJS8zeHuf<a(Nv;jN=0O58|coIx8t6XW7IawVUc)G=P{CC#O{E;E34T~b%h
zm5w2H_Rbn(e45_kWwnX*i7|Iua+#K<kTqGZ+RBg{$LZaTnH*3N8H0xb(jMrKx`FyJ
z*oFcmFnxGu_)CoqTsDCq8ROoAwDF1yLfxV31@<@khO<5VmUZ%~QtNP^!E3(5`vj`w
zrexG8wuZ8=n2Lp9o%fs)*3kSn>yvXaG321Iy5M93#yGLugk?k+)963g&wG~<mOv3?
z<#ZRHwRPupRAja+SGGX0*~WL@Ml1AkK*cP`S-W8$=|ddyIE3A8#tRuciL6ruUSKqO
zQsUdJ(h9|P69kZf`GE^{7p>`HSA40FFoH!iSDWx=wlNSRX2=v(a%{1VDm`#VBSlRt
z?RJIueu4P#DQL{KZz~!aCIiYFL8j0Omnxu`tWnAcgwU*-6gP8Kg)DWG8}*DWAB&*R
ze3{k&g0zkP!sG~vUKpcDhZIJW&XFZeEIuKao5-qE7MbT!hQc?n1}}&3Rg|x}+(#Z+
zdkQRQ9Yi_sJ%rU5uaO`)=|vGyHv^KocSf^n<yDELkF{~iC^v0|b+Cn=UP7SE!ZGkJ
zCTMn?Bebm+PEYLKb4jGUq~sfOBlM?;Sev1Y;r>^V7^6(M%wf^}ia;ocb+uXpKTzM$
zL%Ol#U^^5RPeKGE439io#P4^49$Mcw2YBt-xiVd2UWVTbrsC?+qR7SOw1vUYE21fc
zPmm%)bOn%QiNlyN2@F||m)olq(YC(k>J#xzs7Hq==);s2XOfEyd&Hu|<^_oXK$^pq
zL&YoU@*>~Py!>HLna~Gih6DoXYWc_leMm>&_?U5#852>%ZOTwAn=D}+bK{D|Vhusd
zWRzkflaOlINKH~90xjjPj1jq9V#xHiALyeBJ7|64o~NQnZB2`nNuEgewS@2Z1Q!gj
zd&Qf@Ay$Z-Ho4lf3BUx}RJbi1lvTw9v1yQHAw|+h^;QPM6^`snhi(L0Uu$?~4%-*`
z?G%AfycOoaU?r3ivPL_gv^h_y%}I{&@jZ4ykv7V@xtpH!-ZZ2<?aT<<Y@3%VAL%=N
z+QsmCXS&NzlXhL%I%TV3)jPix_CyvcZ*{5r+r4g;rfiwEWm;GQR<Q1B=`Ty!Hf=?M
zLUDPSwlS;NKjDyKg;JsN=vdiu{Z!*O#V;nrE9}I9*6U{)(~93r*rk}EOjdaqR?>mg
zV()}43KwO7Dz9VZ<?9uEz&l~!zXOUeWxXoTuu^xusxh;8ZUP;^0u&uLTCSHjrW8+0
zNKiP61FhF98#9W16S5V7$^uo4;YRCq|AgW~!^+;df3i8<xU=}Rgl!5}WqAMLo3fo8
z9S^O}!A*UkceQ?4spJ3hACH&ra8qCG{Y3v){b64k=%#MyU911A{uthDU+R(+=vEjW
z*U)?P=34!q9H`&cb3Xd|pIq>NZ{q*k73=k`rRUw$jlF->A8Yh>rCx5gF7*CYf2`BH
zloq<(y4d^I{INmrT%$f8WoYRt*gvhXJw?^g<C!k~2*|{bJRB*_F#T}7UG<qh$m)N6
zP|LW|jLVk)qyLst?3TUL@_&2VMnz-ZR;&N60agvt1L`ZR8l(qA|FGM#L3%)Y$Zo3!
z=>e`e0~)>1`;mThX_#AK!(aBt2K|)McDKUD-ar0<FDQMnKlr0Rz2;AD|KD&b!gHSE
z*!JAM)%bBa>%kjXqpajGvMs?wapLv%+}|X`;#P)tN#-)P8Qg4_gqIHf5u^3AOY#ER
zK?$RJZ<iw$qlvzCBwm*i8N3#)rD=}h#7w?+cfDUjFFm5{^x)>093N$PDZU_TL(Sou
z=F4;{P2tvDWH5ZF06a>v)9O3ul(_xk3-ln|5~ac^-}GUnnEA)PrA=yd;IHWJgilfZ
z$dB4Ll{V82boYIZXfpAt=2yJXus#2S9kdski7{5K0Vt<Ie7H2?T0{E!=dmz|G6_Aa
z$h_Y%YpXjwOZN}zrBhuTF1Xy_fxe}F=^ozy6Lt7&^C)!VF$z8?8^zQljN+ozC_bHm
zQ7p3>Mbz$LfLmzXgP&GO4HtZQhBg)Be#YxHk(co3#EV5yxG`hKU6VnF)TZ!c46%ee
zHLJDhLeZ?zH2b<EC=AA&R8AV^nC0@WAw0RzJZZV^8253k16&fX+eibHMn+Q~_|&$+
z%`TV^RWhFuz1UY}JV=yDY~?FI(yaz;x*4v-V2sn8ea>+B=5X{lc^8v}h@`c@p}NpZ
zd>|va_uZ#>V3-69j7~n`iJo|Sjl|7COxsVNr)G<XIXdrMpme6+2WCq{+m8`^a_?Lx
z&v_b}nVtNs8at3Kg1EDtdLSBM2Ki-7)8U)e9T;8C9cTj)s{?$c>$iLkR<a)cOnW8J
zlO+i^E$tZCWEnT<)6>w$9G6uC0tFnR%`ea0ATB(U+z6({z`6|D9gl6D_iSm8&nqF|
zJ5{|3ycI$W35q#~yuwp@fJn)sEIAS%4VrvsnpEE)Tg=G4-+zC9X>?D4=681oNZx|X
z7~A4_r-|`p4L8DHnJ4iEOB4~)`_2i&N)LT;&cQRj5eeLr4H0gTHFvtl+XJVFow%0{
zAo64Tz}X=UP1i)rwmXWhH>l?G=FSr7P7)Izp7CE+h=+}9F|nxAm^CpuL#rK(B-30(
zf|~YJM53pwJ4x_@k3nLI9rt6XGk;={Q}DbsU5lDRLQl`F4iIXDIZ`v5WaRyE!vK#g
zh2^JLA%90mC}r|@ua?N{lkG$d)aFiu4DiK`L%vWJn<x@g20rwDOVf-KiGF-(=KZ+z
zwy;bR4}PR{oOm#9_j|TrJp-hz#_u}lhP{a-wBEn5jxf?Oiw3DpC=|n@wzqO;Sm#aq
zp0(l`nrzXw&sQa@0g5aos^~epB(;^Q7&8F^r6WD$!K%uK|Nc8v>}30E@y0erEH=Ei
z!P~ei9p%l2K<Q}krCfXuHYju60e!>dQgv`;N??Mbuc@UxEK}8?D^EG^pzO|^l@O;?
zb!<zSR;X!x;rRsr1HPvhozIwcdQp2>#^t?T&6OQzd-TrFcbtuko~S$?CH{X<7w}rb
zvif7|8Au!omEo4GnnUkZEJ{Kv)PND=>}@YQ^w4QOtZl387>Q=-KN{VlV%%}XL)J#A
z-uV%O>6$2d7C-M&nqrE)Q9VPW;ep(*50ddrdM^EMjw)QZPhn(I`48tRwj9g7yuqKH
zKl~f)YWk@i{b65TE_8yBLt$fFN1DY);+7H;zOW@%Sh%Y0Bu?z%LkSREu%bA`+#PlR
zhr+-Na{XWbvtZ>umdiLYc|mOcLQ)1E+AR+Gakq$Zbs=q4u!f{y(lcQQs*QW9qVG*A
zQ$yTYO_rku_D%I>0ST(G%@HOO+xJM4HMe#@bc9*AnpG8xOYAcXyu#y1tmG2tw2IX^
z_uEOsc9`$!UUkeoAaTO5di{Vg*gwK;%ctrx#-WXWbz37%<z?#nT<Crth+QUGJXS6Y
z0))+I3a=}T@4@JPWGw`fDcSMox49=VhLM{)&!fiS*Y-hY3@fdJ=JspAPWKSMo(8k1
zOm91=!@G%&&#C;~$rx9ifN6%dC`Gp%m-;z1-z0IDo5E~c9jjv_@yrjfdq;Xu+y&_%
zfX%5xeJg<-d~;386NX)9pT>r88x*KCMqwwq;&nyuM&yL9PzzY)#Kzn6%A|p~!t*6o
zixv4<q<hgxp_TiN<|;}N?12g@W8$)<ckC0Lb`QVW&)Sn75WOJ+&s#S%+o_q<s2@*2
zz#$rX;m_>?s$d2V_K<R;jbZA?n)j0rc>xW|H%!YdPj71^`)o<eZ4bU@+Xeg32Ldxi
zBuGYPr{Kv7!a+`TOiP;6KsOjHYtvnDqQYe5w}R9pe}t7Hd+xvh$oi3fpGd+G+xL!b
zQXyTEyenRhNl&N0TKj0&t3WHU4E^RCmY+^m=P@KEt(tYzi8oYj7ny$U%r#{1wf4z1
zIHiFN&)822#%R%XRgbV_<o1Q5&c$>z<6N6jRt0%ki3E1Hu14_2lAYRSF*H6CmlMiH
zgr~dOj;H&QM)N+n-bvHU_O!Do686-kNSHvG-f`Wwmn<qNEGEuTK{DD2yfk;<!Jt(d
z*(HGXxIR+1U1$*+2w~6tMuz?SP}pQ(O=tv~_ry>#oEV?;zZM1?=s3fAp+868!(iIT
zlvkO((T(ZxfY3b-8{i3?<VITQ?_=v)cxNl~fu@-_FRhQ@aPF+l+C&mea2m~kA(uAZ
zk$Z7Xmyl1>#(Vo-h^uLrh1ayFJ3b2{O-ygz)N5<i$@4FkRi(QmFuHD*Y1SNcb|XXN
zP%jcJ!mAKd?bC-&uGV`#dES9Y(KkHeMjp&<dVCr7-hD$3(@4oWXP$6yYhScRfO^0*
znl~P$ZuUDeyf2P<51A0J&-*nMOagff{AzGB<_+Kw^OpOykl3cfL{dPHt5~a|_9R{C
zbWCWqV0$_^IEHl2ozSx;O*fsD?k_9mW)L7rH8LkNaDSsr|B&=A?~zEZc}_eFY6H)J
z`nhViWz|yK0RSc>l_%+HB*8x=-ausW4){BZ10XwfBe~I}9G$10X5h@2cY1ynnYPD5
z3A0J|gdy7t$IX^c4%zOVZDBwrmKOE8PD_aUbCBcskcxpuftL(zxV2B)%s9phTR2L{
zBsbq2fa&SabKEr`P;$kieVB%upkR%;nJI%h&YI8*5KKNlwmrOjAU^799y8a}?v*|X
zXC03k_N^jXZ3IA46QrcZ<Tmzx1lhPu9tQI&eoX$!HJ<9S8IP0t%H8o8c(rHF(Rkep
z!mhLGXHPt`VF5WC)4L{=<4vZvC6B-_Ci+FblEPY0Zr&N34<8~FBjX3g*18W*%56Pe
z6}fszW_Vl?s|L1-+~j3A!QeM>KSQN7ww1$Y$#%|ATiUhlosFc!$%2?M&hX2biIyWH
z^Fc7ZASOovB?RLo><B;adKeGw02He*fx?LE?*52X$@I)bG6g}l#O8q)#>MM<>5(!=
zR+srX^Rn<7m^Yzv$zB})HF*l5f=;2J97#bHo?Yz4Rw$T#d00lY>-v!#IFNaieBJsO
zR+j_)44r)eIZUt_WFL+GQ~lU5av+6CU{#Z<BEw*qUF&y{421AL1lnw4t0E>B;D!dt
z!15*iz|M5fKPv3I{FLMJxBdX&Jl)BlnJFY%v_f`fHqZ_J4;wlQ{6;|3uxl8PVR=tO
zd+KbHhvhgc#H3UO%Mt0iN4)w6v6MU!5)f>J0jpg!=u$1d8(!ogd`aQ`F(bvV>2x4u
zAWksOwHs}rGYFfAAeL9`0&ADBd$%-y<`af;odr{xfOmI_BX>R5$dv3mYT6)-h9lRj
zT18J~(CeQF$DnHw)c2?1hjw-Oc!P2nPvGDKjHLzR0Xzc2d=_f%gYZDQ-$J~Nh?^OH
zwLSGqT-+0#Mp(K(AgJ)Z0q$+L&k^J0q+D5=A6qM+zwAUE#3^^&;ieoyPjzw^-*D`N
z(UnMN*M+6C#DSPUu(5@~bFN9_hi;f}n-oar%3#INO@X<zuX2nawFXus1E`E}Ngo+8
z!>lir#U^7WnYja>K7*JI36g{{7#<i>w*ksp117Bq8wy_pHW1<&aEO%AguAo-%@G3T
zhl0;)lMBhl&}apnAc=-cO;snL22DP{2%eG4ALuzRK`<_bHJAfq9}WhnWcQtYLlMMG
z_hp}{D;J)Yh`TVOhGs8ho5pZdgkrLHAtc~R_K-X(qfOP?pGN>-Jx!bNR5+mU0HOj$
zF(&(N9GhWCf!3LoDswsG-WLvc;?jYRVXk2VYcmV@vL$zqgiyhD(EKJIocIKyv1+o$
zvS@5w+{-vP?*NEKy#pFygYLZX#evs80RY^)2IHaz$lM!<%8?VsD#Md=dz)($B{U7S
z%C5&DH)wAP$I;#mhBe{}pC#9Ldcd*doIsM%X4Om!*kVH1-oGPG7dxwu0)}O$g&RRU
zO#6?NSEXnFQbh-KQ5;d^xqZpM>2Q?e$?W(pNMrw6Mg0z_=*&dg`|WXI=2-HWbid4`
zi(h*W!Loy1D8NI!Edrg2xZzOR*O0q<AS&Bp114Yb%6@W27{al{C)sUTjN^EaP}O{P
zY$)^5a<6P?dI=gfu1N06tWMDDTf#Qre`aY)p`y9`yhC$&V1mBIcs#o}s-t**LR`|0
zX{uf~uZc?h+Nvnj_nzsvvAucM&f<2D?9RqskMYF``Zp)Y{<KGHg1k!09*v;8B6>@$
z&KmDR=HSf9>y5p}KZy0nnrru<d$|Fr6{$_n_$d}AJmWL|UvB%az2N6qoUL(ym4HOM
zCY8lcx!m-X-HM=Xi?Ebo$Ogp6!#y3Zp4mTJ-|#woEz@~~i$Ma~M_3P6e5bk)g9vpg
zIDbZogWC?f6%}p8F*$F7zd@d51wBREAs6TS8eRvlg8?@qz|jsCr7MG?{Y$Ld38NE;
z1nF;vKMt1NftawqFaOM&KhW<L(DpH&wXZ<(A;R1Q^fh|;Dfci9!VFt}F{TTx2CvoE
zp?f%g$gB3zr+eGCHh5H*ZJL5Bwl7yU6>YEP@y^XK4b{<VsC+7knh)t2M{_|xc`lre
z-MotK^IU9(PNkj!wxF}E9)*q6L)b_>2N6s@b{=anPn322aR>u<6mxw}>bkfg8Fw3J
z)2|^$?P8!2kYOstkaxY4$#X6S4a4OLZy<{3ga8>m54Ygq2WD??yLBsWtp4uiOF=qx
z-6JgGWc(COCf;K*5{8LOBMvd03KEa7CE}*J&pSXHM4Zvl+)%X#g8DHzdL&?EM8_ky
z_R;apu%2`|@AxFOV-03H(I3$I3cPC4ivH9b5yePX%oy#YX4N<jq^#PljD{08aFaN>
zOt4xhvBhRMe+7kGM&&av)Ffbv_A5+-B1R12Gpdzp>UIsy;eK}_epL_x)6MAmn>BHm
z8;O}E?)F+nS4*=@^WhX(*Z&iZ6F3pg!$_Jgka*lZoLC(M2dht~+s|BaXOhG*#{z^9
z!BH=S@iX-$=J=Gl1YO2+&aH|*G&6@mHo?6#ha@_on3TxrQxJ0j!&NXf=kky-qFF_J
z1M%!G*jLQna%Ng6LS{kSfwT{C2hvcGU~QWVJ@azsA_$A7jV3}8S?arBCzxb2QQO61
z1=WlU+<PfEmbg7p*NJgaG=b)&vo^Aalmz6QhK3DYPu7Rpn<xplAUp5KMVYz!>UT5{
z|22&$%)zXOW)jI%#KR@y@LDqsk0@B1c9|7L9g~m*46=jgh@p_i;YmMwlVo!RMH_J=
zj|vjZ$0~T_HM$^D8A_rd7z`I}?Yq{`lO;B72fIM`>C58WnJ)qPam^VA5|~ssM8@o?
zVc-y=E_{2F=F=P{s?eO7bP;+;?JJ9EU<*eOJ5Ba%Dl*uPZVqFL@VGKbhmv|uCy7oA
zp$UK<CAzbE1vX*@BDzhJF6rU(=4)z_X|_pVv+g4mV$>c$kkoq?E{vA%upsN?GMYoc
zPMQJ4TJTiLW$|3DAIQL@p3xr?ON1dv_DKu?%8wCO>2sNcvXS0=5cw$@$p8+fG5eWK
z0?RYY2FPV(P$N}Y$qSu&fF@B=%u%kpZpDu3K7#A@9^$lMcp*>-5)n%y-brFyww*{t
ze_jvC#yU@u<QO?*7+)k)xk9idUCYLf4U)ukFi~XZki#U79|Psuz1)G@wlp{N6r(U`
z+KQnh+2HQlui0wndpF<Evsc(no_WF6G`+$Qr)hqVf_y>uHMbF`_0!}SH4|&AV^M|I
zx_ajP&Sr|(9*^6MASnfp2AT`DRj``iv#2)OyaG^3ECu`z-61rTd=_Hs*{p-P@mxvs
zYeqG{JHD{JQq@tTSM;emS`N%sc9)cMwyX-(J?<h~=J(iz7D(_`MNb|gLKNlMO9`kc
zWnlot>c)3>9O!yEA+}x?tck5pes=K&PN|s_Q<OoQyosYlR*AE}Z5K%~dB|vc^Aj^g
z%*t3CRKJ6AT#qj?=hj3T3_&W7Ht&Sk?Afm+Y?tzDoMVeheq(2GsjLc#LSVy6i>|jn
zuBxbZYoD4m34Vk!445>Xyblc=8}E2_L5`N}#asMcZSbWys?;$#0!2Y)4}~zYR=rym
zD&gcdgV|$yn5X>gsa;vVYsdl4$5uz+xWK@p5lh5xc6LRKC}4y<RAhEkwZE1zve0a&
zgs>>%nEIZCrin#Dsi!}X2R-D36h|^=q!Fe6=NXRIb23c02<!zS0&ygSo8dTz@aGnY
z_!y6<kz-OKSm5}wln^8jm&7q9XS8G1G$^sI_~<W~RavFMuZyHS#<_^*n`3ZMax+&%
zq;ZVG@r=x^<MdhGjc_HWHALQ5{M~RLCpHoh{l|bsgzO%T)|qm9KkvJ_{Yq@})$*p6
zh-i(WCBLw`bCKF*qP`@m-f2Nu1ft_`z$h)BO3%{Xc1VLh!r8kuh)BXwdFTL~c{tR{
zA@SbiY8|k%e^7)|qBqby8zN|AU$aB?!f{3p-M1*}2k2z#=&?=a9g$JZy)oXp-NM@W
z6rrMc5;TyX7mZWobeu=J7XJ<NR>35r&<f=P?f*j=vJlcTQx8&b;WeI`<<S`J7Egr1
z1x+B5w}uciFdsXA$IjMet_Z2&RG9w;WI@1n2gtnixl#oOR20$-r;tON!EuK<9LHm3
zPz@KUAwhW&w&uW^{YsGq7W~T52G!u-BG(ea``|Y89!Q%egn~bYvA^<6wKgI4FSyi@
z;Z5$SC_11ir2qt{JY82Cqa`A>A|-SE+WN!&|A!(1=q=eRHMq2pGW^|BoslJkp=;$r
z6?7Y_yw9HKpYL9;<N7HY7LP|H0`hM=AWN_y?}_<bfg%#P`e=IpQ1XYtzU^ot_PP09
zd0uJ+owWMA4=8>j(vT`H_z~%EDeOqmH_?juaayEHhCLqb?~iGaRcH~>)_lcDCW>4z
z0j3KTLKP}kkH_oEIpmh@FV$k`iXLUh9Qa5U95U?vf_0FHZUem|{n0)couz*KEP|6F
zhQ=}6Z?<YVPcJx{1B8whbBqNduvT;LHnSngHsCZsp@<~XgADhih<L6PNX{fB;JL3>
z9#WcBLvv2SnC?sSMeP{}Mol5a^(06uu9Fs_+BAUWoAv<Eqp7LG9CBc}r9jJhdS*ri
zs|@T)5{>X5KlkJPLZfo{i8Fz*d-6#~u!0nMP!V_v94q8n#*+*382%eZQG8lRG)j%Y
z2-;!&KpfzqNRsPORL7;$ws3Td<dCd{6r@Hztl)D|_{DWstZ}J2Atdtyb&h8W>jgN{
z@FWp7;jPF(q}(Q^RH^`?G+OMxe$a{vJ@`h|>L)N(5&F(v1uqU{y|shFGeK1+dPveW
zzxSi3+E?WS;)qJH6SIlP-=;CS>mE3Y#2Dx(at<=9vLzJvLt{)?L$qL(=(~c|j8c~G
zgVI5oe(Zx(rGsh#sep0080d}g*y}mUTiwWKBHYPoN(Dzp7mG|LXSCZihcKwHlaxRZ
zVV=m*_lVvw`-%dJQB_h*mWWHfIcTrRlJi@uDG?9KxNHOy!t?~U5k)w()cs^N2b<yX
zLAf#ggkG*GJS)_~Ey^6s0_uY}_9He{rjF=n(Q*+`dDsR;UuWL5Laitbnwvp2LsW;H
z-C{1vX+{;#*{&e24l3x?$_j6cRYHEN1xC_!1_=NoAjrRm46N4Cwxx|QC%;UUKr{B8
zLLrZ_y@^Uu;j=4@tEU-znE}*{yUG{ID~lxL^ZvIM_0Q-6^4Q+*%rWX*=48smu$lPx
z*L_bl0O^$73;s6$tS*TQK*$g72-KcgFt=1MGDim~_>Q#blZ$PWx+?2;1iv8_#_$X?
zU9ZMo-XjA_oQe44bvfe_M>;P%;n$F7Hy70%&<>>B!Ix>A@%?3vDdh{kATHf<<!ynS
zxB!l|Sw&k^@zpT@8{uhh4-AujxD~<k=llXmxvBS>k`;^uqHoP5{c)#A^bq@`YD0)m
zIVB~*2r?#p@sgJ18dX+++6%~rSyu?9juh7kIn%KQ<tX*&rLr6^PmSt%V$1eaPclpF
zi)DsG{QW*Pilaae0xh~+w7>(R00U!Li+Te13Y-P37IAPS?sM8W8_j@E%DE)j+5Zd~
zZt#FY%G^Q_LL@RoimrR_0!|B(j|n}?xiMYw=K{$XlO+gLzgqy(G4!y=PYiyOkDUqD
zQelQv!cmP3J?aZRYIuuU6)raNBCCBBD(z7b`8I3!F@NXM61Iz6L2As9ZIDWq87X+x
ziV80{e@5VFV$d-=qx~tpLDzujq$>Gu@sGwzAai-|RVen2OeFygm;=0r1L=Q)`{cQZ
zZ$;^KKHGmoF4#5e46D>bNaDQA1D`4TyUs<_XO8<A?L5H-WMOD@KUZ&vib|>8((voO
zHFn0FX9rbvi3%V}zxb~b`Arr+#vIN|xrQP!v<wnrt_jPXS;YkA3q?ViAJ;>M*1iYm
z0!6pAh;4?li)_Z)0?s`FSrZCmY0O^+mqksQ5yg5$oaqMYi|)%xPPj^!%G{KC8$>`0
zscqcAkH|uekLyf^bj+oc6XR4TJ3xg${V1}!RA8OTGL>(j`aIOFQE~AZ2hKorRbMJH
z?DVlS=Fr&Sv&jxTFFXXc8J9beTM;n6{@$7h5}?k|<4AF2?BuK)g$QDD>OZKlBLIhl
zHILq%`dNpWn}+9OBPW>M9p57w2IR_GDM+T99A7{QillXfFk*dZ1=k$}9D+GE8q3(1
z_i3|`s6SoQ94;aENuH7o_5Ao;Tpl4uvkKL&za@bA$mJ+8YP!}>*&>mE@$?uk%qXI{
zpafs$_>$Ra@#P8xDGj~r9z-lL7zB~?dW!n&@3pa+a%0E3h6{Hbw|T?Hrg358{J+ir
ze8g`HhgE!bZ}7;2Zf+x2f2NF|lXKSXt5r`a9v*8u^r3eK*}R2I$6s97xOD0KJLg}h
zuqmHDe%`#PH!s~ebpAuPFBm=g>uq0qKl|*`+n@PolH18aHd(lQ{sU)YzL@&yjx}@U
zyQKW|NzqexR*$;-?pLSJbMScKv(J1^9(!!+^j96O41I2#(<Ix69y~lh^|o&l7w6=B
z`unf<-*x!8JC078G~RW>mZeKi-7)`To2{2l{pZ3F8=E`)M%*@X)v6JX46?cFA1=@Q
P&xaj@2hZ?%(f<DcTf9O3

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 806f0d49bcf82..73873684f6aaf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -589,4 +589,10 @@ class OrcSourceSuite extends OrcSuite with SharedSparkSession {
   test("SPARK-11412 read and merge orc schemas in parallel") {
     testMergeSchemasInParallel(OrcUtils.readOrcSchemasInParallel)
   }
+
+  test("SPARK-31580: Read a file written before ORC-569") {
+    // Test ORC file came from ORC-621
+    val df = readResourceOrcFile("test-data/TestStringDictionary.testRowIndex.orc")
+    assert(df.where("str < 'row 001000'").count() === 1000)
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index f3e712d6c0a4a..91fd8a47339fc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -320,4 +320,11 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
       }
     }
   }
+
+  test("SPARK-31580: Read a file written before ORC-569") {
+    assume(HiveUtils.isHive23) // Hive 1.2 doesn't use Apache ORC
+    // Test ORC file came from ORC-621
+    val df = readResourceOrcFile("test-data/TestStringDictionary.testRowIndex.orc")
+    assert(df.where("str < 'row 001000'").count() === 1000)
+  }
 }

From 9b41ae20cf450d26c9f0311d342722a0e6309255 Mon Sep 17 00:00:00 2001
From: Michael Chirico <michael.chirico@grabtaxi.com>
Date: Tue, 28 Apr 2020 10:58:48 +0900
Subject: [PATCH 0584/1280] [SPARK-31568][R] Replaces paste(sep="") to paste0

### What changes were proposed in this pull request?

All instances of `paste(..., sep = "")` in the code are replaced with `paste0` which is more performant

### Why are the changes needed?

Performance & consistency (`paste0` is already used extensively in the R package)

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

None

Closes #28374 from MichaelChirico/r-paste0.

Authored-by: Michael Chirico <michael.chirico@grabtaxi.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a68d98cf4f25cb357353860ca4f9a8d909a52ee1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/DataFrame.R                   | 10 ++++-----
 R/pkg/R/RDD.R                         |  2 +-
 R/pkg/R/mllib_classification.R        |  2 +-
 R/pkg/R/schema.R                      |  7 +++---
 R/pkg/R/sparkR.R                      |  2 +-
 R/pkg/R/utils.R                       |  2 +-
 R/pkg/tests/fulltests/test_sparkSQL.R | 32 +++++++++++++--------------
 7 files changed, 28 insertions(+), 29 deletions(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 2ebd42a263ce0..a7348044dd2be 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -271,7 +271,7 @@ setMethod("show", "SparkDataFrame",
                 paste(l, collapse = ":")
               })
               s <- paste(cols, collapse = ", ")
-              cat(paste(class(object), "[", s, "]\n", sep = ""))
+              cat(paste0(class(object), "[", s, "]\n"))
             }
           })
 
@@ -2755,10 +2755,10 @@ setMethod("merge",
               colY <- joinY[[i]]
 
               if (colX %in% by) {
-                colX <- paste(colX, suffixes[1], sep = "")
+                colX <- paste0(colX, suffixes[1])
               }
               if (colY %in% by) {
-                colY <- paste(colY, suffixes[2], sep = "")
+                colY <- paste0(colY, suffixes[2])
               }
 
               colX <- getColumn(xsel, colX)
@@ -2773,7 +2773,7 @@ setMethod("merge",
 
             # sorts the result by 'by' columns if sort = TRUE
             if (sort && length(by) > 0) {
-              colNameWithSuffix <- paste(by, suffixes[2], sep = "")
+              colNameWithSuffix <- paste0(by, suffixes[2])
               joinRes <- do.call("arrange", c(joinRes, colNameWithSuffix, decreasing = FALSE))
             }
 
@@ -2796,7 +2796,7 @@ genAliasesForIntersectedCols <- function(x, intersectedColNames, suffix) {
   cols <- lapply(allColNames, function(colName) {
     col <- getColumn(x, colName)
     if (colName %in% intersectedColNames) {
-      newJoin <- paste(colName, suffix, sep = "")
+      newJoin <- paste0(colName, suffix)
       if (newJoin %in% allColNames) {
         stop("The following column name: ", newJoin, " occurs more than once in the 'DataFrame'.",
           "Please use different suffixes for the intersected columns.")
diff --git a/R/pkg/R/RDD.R b/R/pkg/R/RDD.R
index 6e89b4bb4d964..7ee725d90d550 100644
--- a/R/pkg/R/RDD.R
+++ b/R/pkg/R/RDD.R
@@ -69,7 +69,7 @@ setMethod("initialize", "RDD", function(.Object, jrdd, serializedMode,
 
 setMethod("showRDD", "RDD",
           function(object) {
-              cat(paste(callJMethod(getJRDD(object), "toString"), "\n", sep = ""))
+              cat(paste0(callJMethod(getJRDD(object), "toString"), "\n"))
           })
 
 setMethod("initialize", "PipelinedRDD", function(.Object, prev, func, jrdd_val) {
diff --git a/R/pkg/R/mllib_classification.R b/R/pkg/R/mllib_classification.R
index 3ad824e1e634a..1af81f8f84b76 100644
--- a/R/pkg/R/mllib_classification.R
+++ b/R/pkg/R/mllib_classification.R
@@ -332,7 +332,7 @@ setMethod("spark.logit", signature(data = "SparkDataFrame", formula = "formula")
               if (!is.null(lowerBoundsOnCoefficients) && (row != nrow(upperBoundsOnCoefficients)
                 || col != ncol(upperBoundsOnCoefficients))) {
                 stop(paste0("dimension of upperBoundsOnCoefficients ",
-                           "is not the same as lowerBoundsOnCoefficients", sep = ""))
+                            "is not the same as lowerBoundsOnCoefficients"))
               }
 
               if (is.null(lowerBoundsOnCoefficients)) {
diff --git a/R/pkg/R/schema.R b/R/pkg/R/schema.R
index 9831fc3cc6d01..8d2d9a12530b9 100644
--- a/R/pkg/R/schema.R
+++ b/R/pkg/R/schema.R
@@ -99,10 +99,9 @@ print.structType <- function(x, ...) {
   cat("StructType\n",
       sapply(x$fields(),
              function(field) {
-               paste("|-", "name = \"", field$name(),
-                     "\", type = \"", field$dataType.toString(),
-                     "\", nullable = ", field$nullable(), "\n",
-                     sep = "")
+               paste0("|-", "name = \"", field$name(),
+                      "\", type = \"", field$dataType.toString(),
+                      "\", nullable = ", field$nullable(), "\n")
              }),
       sep = "")
 }
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index cc8c92b8ab262..2ece83a77ce99 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -244,7 +244,7 @@ sparkR.sparkContext <- function(
     uriSep <- "////"
   }
   localJarPaths <- lapply(jars,
-                          function(j) { utils::URLencode(paste("file:", uriSep, j, sep = "")) })
+                          function(j) { utils::URLencode(paste0("file:", uriSep, j)) })
 
   # Set the start time to identify jobjs
   # Seconds resolution is good enough for this purpose, so use ints
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index a8c1ddb3dd20b..9d7d1a452c237 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -137,7 +137,7 @@ hashCode <- function(key) {
       as.integer(hashC)
     }
   } else {
-    warning(paste("Could not hash object, returning 0", sep = ""))
+    warning("Could not hash object, returning 0")
     as.integer(0)
   }
 }
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 32536de3e9548..0b5527130e2da 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -2507,8 +2507,8 @@ test_that("join(), crossJoin() and merge() on a DataFrame", {
   writeLines(mockLines3, jsonPath3)
   df3 <- read.json(jsonPath3)
   expect_error(merge(df, df3),
-               paste("The following column name: name_y occurs more than once in the 'DataFrame'.",
-                     "Please use different suffixes for the intersected columns.", sep = ""))
+               paste0("The following column name: name_y occurs more than once in the 'DataFrame'.",
+                      "Please use different suffixes for the intersected columns."))
 
   unlink(jsonPath2)
   unlink(jsonPath3)
@@ -2551,20 +2551,20 @@ test_that("toJSON() on DataFrame", {
 
 test_that("showDF()", {
   df <- read.json(jsonPath)
-  expected <- paste("+----+-------+\n",
-                    "| age|   name|\n",
-                    "+----+-------+\n",
-                    "|null|Michael|\n",
-                    "|  30|   Andy|\n",
-                    "|  19| Justin|\n",
-                    "+----+-------+\n", sep = "")
-  expected2 <- paste("+---+----+\n",
-                     "|age|name|\n",
-                     "+---+----+\n",
-                     "|nul| Mic|\n",
-                     "| 30| And|\n",
-                     "| 19| Jus|\n",
-                     "+---+----+\n", sep = "")
+  expected <- paste("+----+-------+",
+                    "| age|   name|",
+                    "+----+-------+",
+                    "|null|Michael|",
+                    "|  30|   Andy|",
+                    "|  19| Justin|",
+                    "+----+-------+\n", sep = "\n")
+  expected2 <- paste("+---+----+",
+                     "|age|name|",
+                     "+---+----+",
+                     "|nul| Mic|",
+                     "| 30| And|",
+                     "| 19| Jus|",
+                     "+---+----+\n", sep = "\n")
   expect_output(showDF(df), expected)
   expect_output(showDF(df, truncate = 3), expected2)
 })

From e86e2669173ad1afa2a955f60a3838ac4af22363 Mon Sep 17 00:00:00 2001
From: Michael Chirico <michael.chirico@grabtaxi.com>
Date: Tue, 28 Apr 2020 11:03:51 +0900
Subject: [PATCH 0585/1280] [SPARK-31578][R] Vectorize schema validation for
 arrow in types.R

### What changes were proposed in this pull request?

Repeated `sapply` avoided in internal `checkSchemaInArrow`

### Why are the changes needed?

Current implementation is doubly inefficient:

 1. Repeatedly doing the same (95%) `sapply` loop
 2. Doing scalar `==` on a vector (`==` should be done over the whole vector for efficiency)

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

By my trusty friend the CI bots

Closes #28372 from MichaelChirico/vectorize-types.

Authored-by: Michael Chirico <michael.chirico@grabtaxi.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 410fa913215665db72f871b556569cba3dc9ee0a)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/types.R | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/R/pkg/R/types.R b/R/pkg/R/types.R
index 55f75508e88ff..5b41f5963e2a3 100644
--- a/R/pkg/R/types.R
+++ b/R/pkg/R/types.R
@@ -94,27 +94,22 @@ checkSchemaInArrow <- function(schema) {
   }
 
   # Both cases below produce a corrupt value for unknown reason. It needs to be investigated.
-  if (any(sapply(schema$fields(), function(x) x$dataType.toString() == "FloatType"))) {
+  field_strings <- sapply(schema$fields(), function(x) x$dataType.toString())
+  if (any(field_strings == "FloatType")) {
     stop("Arrow optimization in R does not support float type yet.")
   }
-  if (any(sapply(schema$fields(), function(x) x$dataType.toString() == "BinaryType"))) {
+  if (any(field_strings == "BinaryType")) {
     stop("Arrow optimization in R does not support binary type yet.")
   }
-  if (any(sapply(schema$fields(),
-                 function(x) startsWith(x$dataType.toString(),
-                 "ArrayType")))) {
+  if (any(startsWith(field_strings, "ArrayType"))) {
     stop("Arrow optimization in R does not support array type yet.")
   }
 
   # Arrow optimization in Spark does not yet support both cases below.
-  if (any(sapply(schema$fields(),
-                 function(x) startsWith(x$dataType.toString(),
-                 "StructType")))) {
+  if (any(startsWith(field_strings, "StructType"))) {
     stop("Arrow optimization in R does not support nested struct type yet.")
   }
-  if (any(sapply(schema$fields(),
-                 function(x) startsWith(x$dataType.toString(),
-                 "MapType")))) {
+  if (any(startsWith(field_strings, "MapType"))) {
     stop("Arrow optimization in R does not support map type yet.")
   }
 }

From d24254ba4aa138ea199dfa3a9439a6634422f280 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Tue, 28 Apr 2020 13:22:43 +0900
Subject: [PATCH 0586/1280] [SPARK-31589][INFRA] Use `r-lib/actions/setup-r` in
 GitHub Action

### What changes were proposed in this pull request?

This PR aims to use `r-lib/actions/setup-r` because it's more stable and maintained by 3rd party.

### Why are the changes needed?

This will recover the current outage. In addition, this will be more robust in the future.
As of now, this is tested via https://github.com/dongjoon-hyun/spark/pull/17 .

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Pass the GitHub Actions, especially `Linter R` and `Generate Documents`.

Closes #28382 from dongjoon-hyun/SPARK-31589.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 2d3e9601b58fbe33aeedb106be7e2a1fafa2e1fd)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/master.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 33ea335e95111..36f9e33f9bbb4 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -103,12 +103,12 @@ jobs:
     - uses: actions/setup-java@v1
       with:
         java-version: '11'
-    - name: install R
+    - uses: r-lib/actions/setup-r@v1
+      with:
+        r-version: '3.6.2'
+    - name: Install lib
       run: |
-        echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' | sudo tee -a /etc/apt/sources.list
-        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
-        sudo apt-get update
-        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
+        sudo apt-get install -y libcurl4-openssl-dev
     - name: install R packages
       run: |
         sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
@@ -139,12 +139,12 @@ jobs:
     - uses: actions/setup-ruby@v1
       with:
         ruby-version: '2.7'
-    - name: Install R
+    - uses: r-lib/actions/setup-r@v1
+      with:
+        r-version: '3.6.2'
+    - name: Install lib and pandoc
       run: |
-        echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' | sudo tee -a /etc/apt/sources.list
-        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
-        sudo apt-get update
-        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev pandoc
+        sudo apt-get install -y libcurl4-openssl-dev pandoc
     - name: Install packages
       run: |
         pip install sphinx mkdocs numpy

From b100ed2d3ee635e4e61af750267171e4a1739ccc Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Tue, 28 Apr 2020 08:11:41 +0000
Subject: [PATCH 0587/1280] [SPARK-31519][SQL] Cast in having aggregate
 expressions returns the wrong result

### What changes were proposed in this pull request?
Add a new logical node AggregateWithHaving, and the parser should create this plan for HAVING. The analyzer resolves it to Filter(..., Aggregate(...)).

### Why are the changes needed?
The SQL parser in Spark creates Filter(..., Aggregate(...)) for the HAVING query, and Spark has a special analyzer rule ResolveAggregateFunctions to resolve the aggregate functions and grouping columns in the Filter operator.

It works for simple cases in a very tricky way as it relies on rule execution order:
1. Rule ResolveReferences hits the Aggregate operator and resolves attributes inside aggregate functions, but the function itself is still unresolved as it's an UnresolvedFunction. This stops resolving the Filter operator as the child Aggrege operator is still unresolved.
2. Rule ResolveFunctions resolves UnresolvedFunction. This makes the Aggrege operator resolved.
3. Rule ResolveAggregateFunctions resolves the Filter operator if its child is a resolved Aggregate. This rule can correctly resolve the grouping columns.

In the example query, I put a CAST, which needs to be resolved by rule ResolveTimeZone, which runs after ResolveAggregateFunctions. This breaks step 3 as the Aggregate operator is unresolved at that time. Then the analyzer starts next round and the Filter operator is resolved by ResolveReferences, which wrongly resolves the grouping columns.

See the demo below:
```
SELECT SUM(a) AS b, '2020-01-01' AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10
```
The query's result is
```
+---+----------+
|  b|      fake|
+---+----------+
|  2|2020-01-01|
+---+----------+
```
But if we add CAST, it will return an empty result.
```
SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10
```

### Does this PR introduce any user-facing change?
Yes, bug fix for cast in having aggregate expressions.

### How was this patch tested?
New UT added.

Closes #28294 from xuanyuanking/SPARK-31519.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6ed2dfbba193d29436dccae4c379dae7b5ba5bdb)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 133 ++++++++++--------
 .../sql/catalyst/analysis/unresolved.scala    |  13 +-
 .../spark/sql/catalyst/dsl/package.scala      |   8 ++
 .../sql/catalyst/parser/AstBuilder.scala      |   7 +-
 .../sql/catalyst/parser/PlanParserSuite.scala |   5 +-
 .../resources/sql-tests/inputs/having.sql     |   3 +
 .../sql-tests/results/having.sql.out          |  10 +-
 .../results/postgreSQL/window_part3.sql.out   |   6 +-
 .../sql/DataFrameWindowFunctionsSuite.scala   |  30 ++--
 9 files changed, 135 insertions(+), 80 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 6de3189c2b669..152a0c4b5f99a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1235,13 +1235,13 @@ class Analyzer(
     /**
      * Resolves the attribute and extract value expressions(s) by traversing the
      * input expression in top down manner. The traversal is done in top-down manner as
-     * we need to skip over unbound lamda function expression. The lamda expressions are
+     * we need to skip over unbound lambda function expression. The lambda expressions are
      * resolved in a different rule [[ResolveLambdaVariables]]
      *
      * Example :
      * SELECT transform(array(1, 2, 3), (x, i) -> x + i)"
      *
-     * In the case above, x and i are resolved as lamda variables in [[ResolveLambdaVariables]]
+     * In the case above, x and i are resolved as lambda variables in [[ResolveLambdaVariables]]
      *
      * Note : In this routine, the unresolved attributes are resolved from the input plan's
      * children attributes.
@@ -1396,6 +1396,9 @@ class Analyzer(
               notMatchedActions = newNotMatchedActions)
         }
 
+      // Skip the having clause here, this will be handled in ResolveAggregateFunctions.
+      case h: AggregateWithHaving => h
+
       case q: LogicalPlan =>
         logTrace(s"Attempting to resolve ${q.simpleString(SQLConf.get.maxToStringFields)}")
         q.mapExpressions(resolveExpressionTopDown(_, q))
@@ -2036,62 +2039,14 @@ class Analyzer(
    */
   object ResolveAggregateFunctions extends Rule[LogicalPlan] {
     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
-      case f @ Filter(cond, agg @ Aggregate(grouping, originalAggExprs, child)) if agg.resolved =>
+      // Resolve aggregate with having clause to Filter(..., Aggregate()). Note, to avoid wrongly
+      // resolve the having condition expression, here we skip resolving it in ResolveReferences
+      // and transform it to Filter after aggregate is resolved. See more details in SPARK-31519.
+      case AggregateWithHaving(cond, agg: Aggregate) if agg.resolved =>
+        resolveHaving(Filter(cond, agg), agg)
 
-        // Try resolving the condition of the filter as though it is in the aggregate clause
-        try {
-          val aggregatedCondition =
-            Aggregate(
-              grouping,
-              Alias(cond, "havingCondition")() :: Nil,
-              child)
-          val resolvedOperator = executeSameContext(aggregatedCondition)
-          def resolvedAggregateFilter =
-            resolvedOperator
-              .asInstanceOf[Aggregate]
-              .aggregateExpressions.head
-
-          // If resolution was successful and we see the filter has an aggregate in it, add it to
-          // the original aggregate operator.
-          if (resolvedOperator.resolved) {
-            // Try to replace all aggregate expressions in the filter by an alias.
-            val aggregateExpressions = ArrayBuffer.empty[NamedExpression]
-            val transformedAggregateFilter = resolvedAggregateFilter.transform {
-              case ae: AggregateExpression =>
-                val alias = Alias(ae, ae.toString)()
-                aggregateExpressions += alias
-                alias.toAttribute
-              // Grouping functions are handled in the rule [[ResolveGroupingAnalytics]].
-              case e: Expression if grouping.exists(_.semanticEquals(e)) &&
-                  !ResolveGroupingAnalytics.hasGroupingFunction(e) &&
-                  !agg.output.exists(_.semanticEquals(e)) =>
-                e match {
-                  case ne: NamedExpression =>
-                    aggregateExpressions += ne
-                    ne.toAttribute
-                  case _ =>
-                    val alias = Alias(e, e.toString)()
-                    aggregateExpressions += alias
-                    alias.toAttribute
-                }
-            }
-
-            // Push the aggregate expressions into the aggregate (if any).
-            if (aggregateExpressions.nonEmpty) {
-              Project(agg.output,
-                Filter(transformedAggregateFilter,
-                  agg.copy(aggregateExpressions = originalAggExprs ++ aggregateExpressions)))
-            } else {
-              f
-            }
-          } else {
-            f
-          }
-        } catch {
-          // Attempting to resolve in the aggregate can result in ambiguity.  When this happens,
-          // just return the original plan.
-          case ae: AnalysisException => f
-        }
+      case f @ Filter(_, agg: Aggregate) if agg.resolved =>
+        resolveHaving(f, agg)
 
       case sort @ Sort(sortOrder, global, aggregate: Aggregate) if aggregate.resolved =>
 
@@ -2162,6 +2117,63 @@ class Analyzer(
     def containsAggregate(condition: Expression): Boolean = {
       condition.find(_.isInstanceOf[AggregateExpression]).isDefined
     }
+
+    def resolveHaving(filter: Filter, agg: Aggregate): LogicalPlan = {
+      // Try resolving the condition of the filter as though it is in the aggregate clause
+      try {
+        val aggregatedCondition =
+          Aggregate(
+            agg.groupingExpressions,
+            Alias(filter.condition, "havingCondition")() :: Nil,
+            agg.child)
+        val resolvedOperator = executeSameContext(aggregatedCondition)
+        def resolvedAggregateFilter =
+          resolvedOperator
+            .asInstanceOf[Aggregate]
+            .aggregateExpressions.head
+
+        // If resolution was successful and we see the filter has an aggregate in it, add it to
+        // the original aggregate operator.
+        if (resolvedOperator.resolved) {
+          // Try to replace all aggregate expressions in the filter by an alias.
+          val aggregateExpressions = ArrayBuffer.empty[NamedExpression]
+          val transformedAggregateFilter = resolvedAggregateFilter.transform {
+            case ae: AggregateExpression =>
+              val alias = Alias(ae, ae.toString)()
+              aggregateExpressions += alias
+              alias.toAttribute
+            // Grouping functions are handled in the rule [[ResolveGroupingAnalytics]].
+            case e: Expression if agg.groupingExpressions.exists(_.semanticEquals(e)) &&
+                !ResolveGroupingAnalytics.hasGroupingFunction(e) &&
+                !agg.output.exists(_.semanticEquals(e)) =>
+              e match {
+                case ne: NamedExpression =>
+                  aggregateExpressions += ne
+                  ne.toAttribute
+                case _ =>
+                  val alias = Alias(e, e.toString)()
+                  aggregateExpressions += alias
+                  alias.toAttribute
+              }
+          }
+
+          // Push the aggregate expressions into the aggregate (if any).
+          if (aggregateExpressions.nonEmpty) {
+            Project(agg.output,
+              Filter(transformedAggregateFilter,
+                agg.copy(aggregateExpressions = agg.aggregateExpressions ++ aggregateExpressions)))
+          } else {
+            filter
+          }
+        } else {
+          filter
+        }
+      } catch {
+        // Attempting to resolve in the aggregate can result in ambiguity.  When this happens,
+        // just return the original plan.
+        case ae: AnalysisException => filter
+      }
+    }
   }
 
   /**
@@ -2586,11 +2598,14 @@ class Analyzer(
     def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
 
       case Filter(condition, _) if hasWindowFunction(condition) =>
-        failAnalysis("It is not allowed to use window functions inside WHERE and HAVING clauses")
+        failAnalysis("It is not allowed to use window functions inside WHERE clause")
+
+      case AggregateWithHaving(condition, _) if hasWindowFunction(condition) =>
+        failAnalysis("It is not allowed to use window functions inside HAVING clause")
 
       // Aggregate with Having clause. This rule works with an unresolved Aggregate because
       // a resolved Aggregate will not have Window Functions.
-      case f @ Filter(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
+      case f @ AggregateWithHaving(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
         if child.resolved &&
            hasWindowFunction(aggregateExprs) &&
            a.expressions.forall(_.resolved) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 6048d98033961..806cdeb95cca4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.errors.TreeNodeException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.parser.ParserUtils
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, UnaryNode}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LeafNode, LogicalPlan, UnaryNode}
 import org.apache.spark.sql.catalyst.trees.TreeNode
 import org.apache.spark.sql.catalyst.util.quoteIdentifier
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
@@ -538,3 +538,14 @@ case class UnresolvedOrdinal(ordinal: Int)
   override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
   override lazy val resolved = false
 }
+
+/**
+ * Represents unresolved aggregate with having clause, it is turned by the analyzer into a Filter.
+ */
+case class AggregateWithHaving(
+    havingCondition: Expression,
+    child: Aggregate)
+  extends UnaryNode {
+  override lazy val resolved: Boolean = false
+  override def output: Seq[Attribute] = child.output
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index cc96d905a86f2..f135f50493ed8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -364,6 +364,14 @@ package object dsl {
         Aggregate(groupingExprs, aliasedExprs, logicalPlan)
       }
 
+      def having(
+          groupingExprs: Expression*)(
+          aggregateExprs: Expression*)(
+          havingCondition: Expression): LogicalPlan = {
+        AggregateWithHaving(havingCondition,
+          groupBy(groupingExprs: _*)(aggregateExprs: _*).asInstanceOf[Aggregate])
+      }
+
       def window(
           windowExpressions: Seq[NamedExpression],
           partitionSpec: Seq[Expression],
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 6c5917053e135..2a28e4bdbb90e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -629,7 +629,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case p: Predicate => p
       case e => Cast(e, BooleanType)
     }
-    Filter(predicate, plan)
+    plan match {
+      case aggregate: Aggregate =>
+        AggregateWithHaving(predicate, aggregate)
+      case _ =>
+        Filter(predicate, plan)
+    }
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 11230e4065419..714f8c53e1d35 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -110,7 +110,7 @@ class PlanParserSuite extends AnalysisTest {
     assertEqual("select a, b from db.c where x < 1", table("db", "c").where('x < 1).select('a, 'b))
     assertEqual(
       "select a, b from db.c having x < 1",
-      table("db", "c").groupBy()('a, 'b).where('x < 1))
+      table("db", "c").having()('a, 'b)('x < 1))
     assertEqual("select distinct a, b from db.c", Distinct(table("db", "c").select('a, 'b)))
     assertEqual("select all a, b from db.c", table("db", "c").select('a, 'b))
     assertEqual("select from tbl", OneRowRelation().select('from.as("tbl")))
@@ -476,8 +476,7 @@ class PlanParserSuite extends AnalysisTest {
     assertEqual(
       "select g from t group by g having a > (select b from s)",
       table("t")
-        .groupBy('g)('g)
-        .where('a > ScalarSubquery(table("s").select('b))))
+        .having('g)('g)('a > ScalarSubquery(table("s").select('b))))
   }
 
   test("table reference") {
diff --git a/sql/core/src/test/resources/sql-tests/inputs/having.sql b/sql/core/src/test/resources/sql-tests/inputs/having.sql
index 868a911e787f6..6868b5902939d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/having.sql
@@ -16,3 +16,6 @@ SELECT MIN(t.v) FROM (SELECT * FROM hav WHERE v > 0) t HAVING(COUNT(1) > 0);
 
 -- SPARK-20329: make sure we handle timezones correctly
 SELECT a + b FROM VALUES (1L, 2), (3L, 4) AS T(a, b) GROUP BY a + b HAVING a + b > 1;
+
+-- SPARK-31519: Cast in having aggregate expressions returns the wrong result
+SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10
diff --git a/sql/core/src/test/resources/sql-tests/results/having.sql.out b/sql/core/src/test/resources/sql-tests/results/having.sql.out
index 5bd185d7b815d..aa8ff73723586 100644
--- a/sql/core/src/test/resources/sql-tests/results/having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/having.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 5
+-- Number of queries: 6
 
 
 -- !query
@@ -47,3 +47,11 @@ struct<(a + CAST(b AS BIGINT)):bigint>
 -- !query output
 3
 7
+
+
+-- !query
+SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10
+-- !query schema
+struct<b:bigint,fake:date>
+-- !query output
+2	2020-01-01
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
index acce688092570..08eba6797b01d 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part3.sql.out
@@ -294,7 +294,7 @@ SELECT * FROM empsalary WHERE row_number() OVER (ORDER BY salary) < 10
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE and HAVING clauses;
+It is not allowed to use window functions inside WHERE clause;
 
 
 -- !query
@@ -341,7 +341,7 @@ SELECT * FROM empsalary WHERE (rank() OVER (ORDER BY random())) > 10
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE and HAVING clauses;
+It is not allowed to use window functions inside WHERE clause;
 
 
 -- !query
@@ -350,7 +350,7 @@ SELECT * FROM empsalary WHERE rank() OVER (ORDER BY random())
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-It is not allowed to use window functions inside WHERE and HAVING clauses;
+It is not allowed to use window functions inside WHERE clause;
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
index d398657ec0b6e..f72ccaa63b42b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -665,40 +665,46 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("SPARK-24575: Window functions inside WHERE and HAVING clauses") {
-    def checkAnalysisError(df: => DataFrame): Unit = {
+    def checkAnalysisError(df: => DataFrame, clause: String): Unit = {
       val thrownException = the[AnalysisException] thrownBy {
         df.queryExecution.analyzed
       }
-      assert(thrownException.message.contains("window functions inside WHERE and HAVING clauses"))
+      assert(thrownException.message.contains(s"window functions inside $clause clause"))
     }
 
-    checkAnalysisError(testData2.select("a").where(rank().over(Window.orderBy($"b")) === 1))
-    checkAnalysisError(testData2.where($"b" === 2 && rank().over(Window.orderBy($"b")) === 1))
+    checkAnalysisError(
+      testData2.select("a").where(rank().over(Window.orderBy($"b")) === 1), "WHERE")
+    checkAnalysisError(
+      testData2.where($"b" === 2 && rank().over(Window.orderBy($"b")) === 1), "WHERE")
     checkAnalysisError(
       testData2.groupBy($"a")
         .agg(avg($"b").as("avgb"))
-        .where($"a" > $"avgb" && rank().over(Window.orderBy($"a")) === 1))
+        .where($"a" > $"avgb" && rank().over(Window.orderBy($"a")) === 1), "WHERE")
     checkAnalysisError(
       testData2.groupBy($"a")
         .agg(max($"b").as("maxb"), sum($"b").as("sumb"))
-        .where(rank().over(Window.orderBy($"a")) === 1))
+        .where(rank().over(Window.orderBy($"a")) === 1), "WHERE")
     checkAnalysisError(
       testData2.groupBy($"a")
         .agg(max($"b").as("maxb"), sum($"b").as("sumb"))
-        .where($"sumb" === 5 && rank().over(Window.orderBy($"a")) === 1))
+        .where($"sumb" === 5 && rank().over(Window.orderBy($"a")) === 1), "WHERE")
 
-    checkAnalysisError(sql("SELECT a FROM testData2 WHERE RANK() OVER(ORDER BY b) = 1"))
-    checkAnalysisError(sql("SELECT * FROM testData2 WHERE b = 2 AND RANK() OVER(ORDER BY b) = 1"))
+    checkAnalysisError(sql("SELECT a FROM testData2 WHERE RANK() OVER(ORDER BY b) = 1"), "WHERE")
+    checkAnalysisError(
+      sql("SELECT * FROM testData2 WHERE b = 2 AND RANK() OVER(ORDER BY b) = 1"), "WHERE")
     checkAnalysisError(
-      sql("SELECT * FROM testData2 GROUP BY a HAVING a > AVG(b) AND RANK() OVER(ORDER BY a) = 1"))
+      sql("SELECT * FROM testData2 GROUP BY a HAVING a > AVG(b) AND RANK() OVER(ORDER BY a) = 1"),
+      "HAVING")
     checkAnalysisError(
-      sql("SELECT a, MAX(b), SUM(b) FROM testData2 GROUP BY a HAVING RANK() OVER(ORDER BY a) = 1"))
+      sql("SELECT a, MAX(b), SUM(b) FROM testData2 GROUP BY a HAVING RANK() OVER(ORDER BY a) = 1"),
+      "HAVING")
     checkAnalysisError(
       sql(
         s"""SELECT a, MAX(b)
            |FROM testData2
            |GROUP BY a
-           |HAVING SUM(b) = 5 AND RANK() OVER(ORDER BY a) = 1""".stripMargin))
+           |HAVING SUM(b) = 5 AND RANK() OVER(ORDER BY a) = 1""".stripMargin),
+      "HAVING")
   }
 
   test("window functions in multiple selects") {

From d918477a9b96ac9fa8ca5116814d93737985e8c9 Mon Sep 17 00:00:00 2001
From: Michael Chirico <michael.chirico@grabtaxi.com>
Date: Tue, 28 Apr 2020 17:24:21 +0900
Subject: [PATCH 0588/1280] [SPARK-31573][R] Apply fixed=TRUE as appropriate to
 regex usage in R

### What changes were proposed in this pull request?

For regex functions in base R (`gsub`, `grep`, `grepl`, `strsplit`, `gregexpr`), supplying the `fixed=TRUE` option will be more performant.

### Why are the changes needed?

This is a minor fix for performance

### Does this PR introduce any user-facing change?

No (although some internal code was applying fixed-as-regex in some cases that could technically have been over-broad and caught unintended patterns)

### How was this patch tested?

Not

Closes #28367 from MichaelChirico/r-regex-fixed.

Authored-by: Michael Chirico <michael.chirico@grabtaxi.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c011502ee3f60d2467e4c89d4e0863174592fff6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/DataFrame.R  |  2 +-
 R/pkg/R/SQLContext.R |  7 ++++---
 R/pkg/R/client.R     | 10 +++++-----
 R/pkg/R/install.R    |  4 ++--
 R/pkg/R/schema.R     |  2 +-
 R/pkg/R/sparkR.R     |  4 ++--
 R/pkg/R/utils.R      | 16 ++++++++++------
 7 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index a7348044dd2be..f67f5fd9d45fb 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -2614,7 +2614,7 @@ setMethod("join",
                     "left", "leftouter", "left_outer",
                     "right", "rightouter", "right_outer",
                     "semi", "left_semi", "leftsemi", "anti", "left_anti", "leftanti")) {
-                  joinType <- gsub("_", "", joinType)
+                  joinType <- gsub("_", "", joinType, fixed = TRUE)
                   sdf <- callJMethod(x@sdf, "join", y@sdf, joinExpr@jc, joinType)
                 } else {
                   stop(paste("joinType must be one of the following types:",
diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R
index c6842912706af..1ef2641742704 100644
--- a/R/pkg/R/SQLContext.R
+++ b/R/pkg/R/SQLContext.R
@@ -110,10 +110,11 @@ sparkR.conf <- function(key, defaultValue) {
     value <- if (missing(defaultValue)) {
       tryCatch(callJMethod(conf, "get", key),
               error = function(e) {
-                if (any(grep("java.util.NoSuchElementException", as.character(e)))) {
+                estr <- as.character(e)
+                if (any(grepl("java.util.NoSuchElementException", estr, fixed = TRUE))) {
                   stop(paste0("Config '", key, "' is not set"))
                 } else {
-                  stop(paste0("Unknown error: ", as.character(e)))
+                  stop(paste0("Unknown error: ", estr))
                 }
               })
     } else {
@@ -205,7 +206,7 @@ getSchema <- function(schema, firstRow = NULL, rdd = NULL) {
     # SPAKR-SQL does not support '.' in column name, so replace it with '_'
     # TODO(davies): remove this once SPARK-2775 is fixed
     names <- lapply(names, function(n) {
-      nn <- gsub("[.]", "_", n)
+      nn <- gsub(".", "_", n, fixed = TRUE)
       if (nn != n) {
         warning(paste("Use", nn, "instead of", n, "as column name"))
       }
diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R
index 2ff68ab7b9d77..872b21443eaad 100644
--- a/R/pkg/R/client.R
+++ b/R/pkg/R/client.R
@@ -65,8 +65,8 @@ checkJavaVersion <- function() {
   javaHome <- Sys.getenv("JAVA_HOME")
   javaReqs <- utils::packageDescription(utils::packageName(), fields = c("SystemRequirements"))
   sparkJavaVersions <- strsplit(javaReqs, "[(,)]")[[1]]
-  minJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[2]], ">= ")[[1]][[2]])
-  maxJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[3]], "< ")[[1]][[2]])
+  minJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[2]], ">= ", fixed = TRUE)[[1]][[2]])
+  maxJavaVersion <- as.numeric(strsplit(sparkJavaVersions[[3]], "< ", fixed = TRUE)[[1]][[2]])
   if (javaHome != "") {
     javaBin <- file.path(javaHome, "bin", javaBin)
   }
@@ -89,13 +89,13 @@ checkJavaVersion <- function() {
     })
   javaVersionFilter <- Filter(
       function(x) {
-        grepl(" version", x)
+        grepl(" version", x, fixed = TRUE)
       }, javaVersionOut)
 
-  javaVersionStr <- strsplit(javaVersionFilter[[1]], "[\"]")[[1L]][2]
+  javaVersionStr <- strsplit(javaVersionFilter[[1]], '"', fixed = TRUE)[[1L]][2]
   # javaVersionStr is of the form 1.8.0_92/9.0.x/11.0.x.
   # We are using 8, 9, 10, 11 for sparkJavaVersion.
-  versions <- strsplit(javaVersionStr, "[.]")[[1L]]
+  versions <- strsplit(javaVersionStr, ".", fixed = TRUE)[[1L]]
   if ("1" == versions[1]) {
     javaVersionNum <- as.integer(versions[2])
   } else {
diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R
index 6d1edf6b6f3cf..8c5355a8324f9 100644
--- a/R/pkg/R/install.R
+++ b/R/pkg/R/install.R
@@ -214,9 +214,9 @@ getPreferredMirror <- function(version, packageName) {
                         file.path("spark", version, packageName),
                         ".tgz&as_json=1")
   textLines <- readLines(jsonUrl, warn = FALSE)
-  rowNum <- grep("\"preferred\"", textLines)
+  rowNum <- grep('"preferred"', textLines, fixed = TRUE)
   linePreferred <- textLines[rowNum]
-  matchInfo <- regexpr("\"[A-Za-z][A-Za-z0-9+-.]*://.+\"", linePreferred)
+  matchInfo <- regexpr('"[A-Za-z][A-Za-z0-9+-.]*://.+"', linePreferred)
   if (matchInfo != -1) {
     startPos <- matchInfo + 1
     endPos <- matchInfo + attr(matchInfo, "match.length") - 2
diff --git a/R/pkg/R/schema.R b/R/pkg/R/schema.R
index 8d2d9a12530b9..89d5c2cd1a5e2 100644
--- a/R/pkg/R/schema.R
+++ b/R/pkg/R/schema.R
@@ -182,7 +182,7 @@ checkType <- function(type) {
                 # strsplit does not return the final empty string, so check if
                 # the final char is ","
                 if (substr(fieldsString, nchar(fieldsString), nchar(fieldsString)) != ",") {
-                  fields <- strsplit(fieldsString, ",")[[1]]
+                  fields <- strsplit(fieldsString, ",", fixed = TRUE)[[1]]
                   for (field in fields) {
                     m <- regexec("^(.+):(.+)$", field)
                     matchedStrings <- regmatches(field, m)
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index 2ece83a77ce99..9ba36ad46740a 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -435,7 +435,7 @@ sparkR.session <- function(
   # Check if version number of SparkSession matches version number of SparkR package
   jvmVersion <- callJMethod(sparkSession, "version")
   # Remove -SNAPSHOT from jvm versions
-  jvmVersionStrip <- gsub("-SNAPSHOT", "", jvmVersion)
+  jvmVersionStrip <- gsub("-SNAPSHOT", "", jvmVersion, fixed = TRUE)
   rPackageVersion <- paste0(packageVersion("SparkR"))
 
   if (jvmVersionStrip != rPackageVersion) {
@@ -606,7 +606,7 @@ getClientModeSparkSubmitOpts <- function(submitOps, sparkEnvirMap) {
     # process only if --option is not already specified
     if (!is.null(opsValue) &&
         nchar(opsValue) > 1 &&
-        !grepl(sparkConfToSubmitOps[[conf]], submitOps)) {
+        !grepl(sparkConfToSubmitOps[[conf]], submitOps, fixed = TRUE)) {
       # put "" around value in case it has spaces
       paste0(sparkConfToSubmitOps[[conf]], " \"", opsValue, "\" ")
     } else {
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index 9d7d1a452c237..c60e4db1496d0 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -818,7 +818,8 @@ captureJVMException <- function(e, method) {
   }
 
   # StreamingQueryException could wrap an IllegalArgumentException, so look for that first
-  if (any(grep("org.apache.spark.sql.streaming.StreamingQueryException: ", stacktrace))) {
+  if (any(grep("org.apache.spark.sql.streaming.StreamingQueryException: ",
+               stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.streaming.StreamingQueryException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
@@ -826,14 +827,14 @@ captureJVMException <- function(e, method) {
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
     stop(paste0(rmsg, "streaming query error - ", first), call. = FALSE)
-  } else if (any(grep("java.lang.IllegalArgumentException: ", stacktrace))) {
+  } else if (any(grep("java.lang.IllegalArgumentException: ", stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "java.lang.IllegalArgumentException: ", fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
     stop(paste0(rmsg, "illegal argument - ", first), call. = FALSE)
-  } else if (any(grep("org.apache.spark.sql.AnalysisException: ", stacktrace))) {
+  } else if (any(grep("org.apache.spark.sql.AnalysisException: ", stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.AnalysisException: ", fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
@@ -841,7 +842,8 @@ captureJVMException <- function(e, method) {
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
     stop(paste0(rmsg, "analysis error - ", first), call. = FALSE)
   } else
-    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ", stacktrace))) {
+    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
+                 stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
@@ -850,7 +852,8 @@ captureJVMException <- function(e, method) {
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
     stop(paste0(rmsg, "no such database - ", first), call. = FALSE)
   } else
-    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ", stacktrace))) {
+    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
+                 stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
@@ -858,7 +861,8 @@ captureJVMException <- function(e, method) {
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
     stop(paste0(rmsg, "no such table - ", first), call. = FALSE)
-  } else if (any(grep("org.apache.spark.sql.catalyst.parser.ParseException: ", stacktrace))) {
+  } else if (any(grep("org.apache.spark.sql.catalyst.parser.ParseException: ",
+                      stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.parser.ParseException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.

From efe700c90235950df9ced1cb5512770b0e10b9c1 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 28 Apr 2020 14:10:50 +0000
Subject: [PATCH 0589/1280] [SPARK-31553][SQL] Revert "[SPARK-29048] Improve
 performance on Column.isInCollection() with a large size collection"

### What changes were proposed in this pull request?
This reverts commit 5631a96367d2576e1e0f95d7ae529468da8f5fa8.

Closes #28328

### Why are the changes needed?
The PR  https://github.com/apache/spark/pull/25754 introduced a bug in `isInCollection`. For example, if the SQL config `spark.sql.optimizer.inSetConversionThreshold`is set to 10 (by default):
```scala
val set = (0 to 20).map(_.toString).toSet
val data = Seq("1").toDF("x")
data.select($"x".isInCollection(set).as("isInCollection")).show()
```
The function must return **'true'** because "1" is in the set of "0" ... "20" but it returns "false":
```
+--------------+
|isInCollection|
+--------------+
|         false|
+--------------+
```

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
```
$ ./build/sbt "test:testOnly *ColumnExpressionSuite"
```

Closes #28388 from MaxGekk/fix-isInCollection-revert.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b7cabc80e6df523f0377b651fdbdc2a669c11550)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/Column.scala   | 10 +----
 .../spark/sql/ColumnExpressionSuite.scala     | 45 +++++++------------
 2 files changed, 18 insertions(+), 37 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 50bc7ec5f2afa..6913d4e676d66 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -31,7 +31,6 @@ import org.apache.spark.sql.catalyst.util.toPrettySQL
 import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
 import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 private[sql] object Column {
@@ -827,14 +826,7 @@ class Column(val expr: Expression) extends Logging {
    * @group expr_ops
    * @since 2.4.0
    */
-  def isInCollection(values: scala.collection.Iterable[_]): Column = withExpr {
-    val hSet = values.toSet[Any]
-    if (hSet.size > SQLConf.get.optimizerInSetConversionThreshold) {
-      InSet(expr, hSet)
-    } else {
-      In(expr, values.toSeq.map(lit(_).expr))
-    }
-  }
+  def isInCollection(values: scala.collection.Iterable[_]): Column = isin(values.toSeq: _*)
 
   /**
    * A boolean expression that is evaluated to true if the value of this expression is contained
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index b72d92b9e2a83..8d3b56242ec5a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.io.{LongWritable, Text}
 import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat => NewTextInputFormat}
 import org.scalatest.Matchers._
 
-import org.apache.spark.sql.catalyst.expressions.{In, InSet, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{InSet, Literal, NamedExpression}
 import org.apache.spark.sql.execution.ProjectExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -455,36 +455,25 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
 
   test("isInCollection: Scala Collection") {
     val df = Seq((1, "x"), (2, "y"), (3, "z")).toDF("a", "b")
+    // Test with different types of collections
+    checkAnswer(df.filter($"a".isInCollection(Seq(3, 1))),
+      df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+    checkAnswer(df.filter($"a".isInCollection(Seq(1, 2).toSet)),
+      df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
+    checkAnswer(df.filter($"a".isInCollection(Seq(3, 2).toArray)),
+      df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
+    checkAnswer(df.filter($"a".isInCollection(Seq(3, 1).toList)),
+      df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
 
-    Seq(1, 2).foreach { conf =>
-      withSQLConf(SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> conf.toString) {
-        if (conf <= 1) {
-          assert($"a".isInCollection(Seq(3, 1)).expr.isInstanceOf[InSet], "Expect expr to be InSet")
-        } else {
-          assert($"a".isInCollection(Seq(3, 1)).expr.isInstanceOf[In], "Expect expr to be In")
-        }
-
-        // Test with different types of collections
-        checkAnswer(df.filter($"a".isInCollection(Seq(3, 1))),
-          df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
-        checkAnswer(df.filter($"a".isInCollection(Seq(1, 2).toSet)),
-          df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
-        checkAnswer(df.filter($"a".isInCollection(Seq(3, 2).toArray)),
-          df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
-        checkAnswer(df.filter($"a".isInCollection(Seq(3, 1).toList)),
-          df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
-
-        val df2 = Seq((1, Seq(1)), (2, Seq(2)), (3, Seq(3))).toDF("a", "b")
+    val df2 = Seq((1, Seq(1)), (2, Seq(2)), (3, Seq(3))).toDF("a", "b")
 
-        val e = intercept[AnalysisException] {
-          df2.filter($"a".isInCollection(Seq($"b")))
-        }
-        Seq("cannot resolve",
-          "due to data type mismatch: Arguments must be same type but were").foreach { s =>
-            assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
-          }
-      }
+    val e = intercept[AnalysisException] {
+      df2.filter($"a".isInCollection(Seq($"b")))
     }
+    Seq("cannot resolve", "due to data type mismatch: Arguments must be same type but were")
+      .foreach { s =>
+        assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
+      }
   }
 
   test("&&") {

From 60165750d82b10b18ca86404f44ecc712d3f50a9 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 28 Apr 2020 15:39:28 +0000
Subject: [PATCH 0590/1280] [SPARK-31527][SQL][TESTS][FOLLOWUP] Add a benchmark
 test for datetime add/subtract interval operations

### What changes were proposed in this pull request?
With https://github.com/apache/spark/pull/28310, the operation of date +/- interval(m, d, 0) has been improved a lot.

According to the benchmark results, about 75% time cost is reduced because of no casting date to timestamp back and forth.

In this PR, we add a benchmark for these operations, and timestamp +/- interval operations as accessories.

### Why are the changes needed?

Performance test coverage, since these operations are missing in the DateTimeBenchmark.

### Does this PR introduce any user-facing change?

No, just test

### How was this patch tested?

regenerated benchmark results

Closes #28369 from yaooqinn/SPARK-31527-F.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 54996be4d29a5c8c8a840577aadc1bcfee8e2ec8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeBenchmark-jdk11-results.txt       | 468 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 468 +++++++++---------
 .../benchmark/DateTimeBenchmark.scala         |  42 ++
 3 files changed, 532 insertions(+), 446 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index a08fe0031166b..1004bcf1aa286 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -1,435 +1,457 @@
+================================================================================================
+datetime +/- interval
+================================================================================================
+
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+date + interval(m)                                  919            933          22          0.0   306237514.3       1.0X
+date + interval(m, d)                               910            916           9          0.0   303338619.0       1.0X
+date + interval(m, d, ms)                          3912           3923          16          0.0  1303942791.7       0.2X
+date - interval(m)                                  883            887           6          0.0   294268789.3       1.0X
+date - interval(m, d)                               898            911          18          0.0   299453403.0       1.0X
+date - interval(m, d, ms)                          3937           3944          11          0.0  1312269472.0       0.2X
+timestamp + interval(m)                            2226           2236          14          0.0   741972014.3       0.4X
+timestamp + interval(m, d)                         2264           2274          13          0.0   754709121.0       0.4X
+timestamp + interval(m, d, ms)                     2202           2223          30          0.0   734001075.0       0.4X
+timestamp - interval(m)                            1992           2005          17          0.0   664152744.7       0.5X
+timestamp - interval(m, d)                         2069           2075           9          0.0   689631159.0       0.4X
+timestamp - interval(m, d, ms)                     2240           2244           6          0.0   746538728.0       0.4X
+
+
 ================================================================================================
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    434            445          17         23.1          43.4       1.0X
-cast to timestamp wholestage on                     388            429          57         25.8          38.8       1.1X
+cast to timestamp wholestage off                    178            180           3         56.1          17.8       1.0X
+cast to timestamp wholestage on                     189            192           4         53.0          18.9       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1363           1374          16          7.3         136.3       1.0X
-year of timestamp wholestage on                    1285           1295           9          7.8         128.5       1.1X
+year of timestamp wholestage off                    760            761           1         13.2          76.0       1.0X
+year of timestamp wholestage on                     731            741          10         13.7          73.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1619           1623           6          6.2         161.9       1.0X
-quarter of timestamp wholestage on                 1590           1606          16          6.3         159.0       1.0X
+quarter of timestamp wholestage off                1005           1013          10          9.9         100.5       1.0X
+quarter of timestamp wholestage on                  981            986           3         10.2          98.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1264           1269           8          7.9         126.4       1.0X
-month of timestamp wholestage on                   1277           1290          13          7.8         127.7       1.0X
+month of timestamp wholestage off                   754            758           6         13.3          75.4       1.0X
+month of timestamp wholestage on                    719            729          11         13.9          71.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1854           1865          16          5.4         185.4       1.0X
-weekofyear of timestamp wholestage on              1837           1855          24          5.4         183.7       1.0X
+weekofyear of timestamp wholestage off             1085           1088           4          9.2         108.5       1.0X
+weekofyear of timestamp wholestage on              1075           1091          13          9.3         107.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1248           1252           6          8.0         124.8       1.0X
-day of timestamp wholestage on                     1283           1288           7          7.8         128.3       1.0X
+day of timestamp wholestage off                     751            770          27         13.3          75.1       1.0X
+day of timestamp wholestage on                      735            741           7         13.6          73.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1288           1294           9          7.8         128.8       1.0X
-dayofyear of timestamp wholestage on               1312           1316           5          7.6         131.2       1.0X
+dayofyear of timestamp wholestage off               765            769           5         13.1          76.5       1.0X
+dayofyear of timestamp wholestage on                762            770           7         13.1          76.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1254           1263          13          8.0         125.4       1.0X
-dayofmonth of timestamp wholestage on              1274           1293          17          7.8         127.4       1.0X
+dayofmonth of timestamp wholestage off              780            782           2         12.8          78.0       1.0X
+dayofmonth of timestamp wholestage on               720            736          12         13.9          72.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1419           1422           5          7.0         141.9       1.0X
-dayofweek of timestamp wholestage on               1416           1425           8          7.1         141.6       1.0X
+dayofweek of timestamp wholestage off               887            899          17         11.3          88.7       1.0X
+dayofweek of timestamp wholestage on                820            847          20         12.2          82.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1386           1386           0          7.2         138.6       1.0X
-weekday of timestamp wholestage on                 1348           1354           5          7.4         134.8       1.0X
+weekday of timestamp wholestage off                 821            825           5         12.2          82.1       1.0X
+weekday of timestamp wholestage on                  802            814           9         12.5          80.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1017           1020           4          9.8         101.7       1.0X
-hour of timestamp wholestage on                     996           1006           9         10.0          99.6       1.0X
+hour of timestamp wholestage off                    611            622          14         16.4          61.1       1.0X
+hour of timestamp wholestage on                     571            577           8         17.5          57.1       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  994            996           2         10.1          99.4       1.0X
-minute of timestamp wholestage on                   974            991          15         10.3          97.4       1.0X
+minute of timestamp wholestage off                  607            615          12         16.5          60.7       1.0X
+minute of timestamp wholestage on                   573            580           6         17.5          57.3       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1018           1022           6          9.8         101.8       1.0X
-second of timestamp wholestage on                   976            987          13         10.2          97.6       1.0X
+second of timestamp wholestage off                  615            616           2         16.3          61.5       1.0X
+second of timestamp wholestage on                   564            575           8         17.7          56.4       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         297            299           2         33.7          29.7       1.0X
-current_date wholestage on                          315            322           7         31.8          31.5       0.9X
+current_date wholestage off                         166            169           4         60.4          16.6       1.0X
+current_date wholestage on                          150            153           3         66.7          15.0       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    293            299           9         34.2          29.3       1.0X
-current_timestamp wholestage on                     307            332          23         32.6          30.7       1.0X
+current_timestamp wholestage off                    179            181           2         55.8          17.9       1.0X
+current_timestamp wholestage on                     162            324         138         61.9          16.2       1.1X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1083           1087           7          9.2         108.3       1.0X
-cast to date wholestage on                         1057           1078          18          9.5         105.7       1.0X
+cast to date wholestage off                         658            661           5         15.2          65.8       1.0X
+cast to date wholestage on                          644            654          10         15.5          64.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1249           1262          18          8.0         124.9       1.0X
-last_day wholestage on                             1288           1297          10          7.8         128.8       1.0X
+last_day wholestage off                             768            772           5         13.0          76.8       1.0X
+last_day wholestage on                              737            750          12         13.6          73.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1128           1133           7          8.9         112.8       1.0X
-next_day wholestage on                             1106           1115           9          9.0         110.6       1.0X
+next_day wholestage off                             691            704          17         14.5          69.1       1.0X
+next_day wholestage on                              664            676          10         15.1          66.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1062           1084          31          9.4         106.2       1.0X
-date_add wholestage on                             1079           1093          24          9.3         107.9       1.0X
+date_add wholestage off                             646            646           0         15.5          64.6       1.0X
+date_add wholestage on                              623            640          13         16.1          62.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1077           1081           5          9.3         107.7       1.0X
-date_sub wholestage on                             1086           1105          16          9.2         108.6       1.0X
+date_sub wholestage off                             638            645           9         15.7          63.8       1.0X
+date_sub wholestage on                              618            629           8         16.2          61.8       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1437           1440           4          7.0         143.7       1.0X
-add_months wholestage on                           1452           1459           7          6.9         145.2       1.0X
+add_months wholestage off                           892            896           5         11.2          89.2       1.0X
+add_months wholestage on                            926            938           7         10.8          92.6       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5638           5653          21          1.8         563.8       1.0X
-format date wholestage on                          5464           5482          17          1.8         546.4       1.0X
+format date wholestage off                         3395           3439          62          2.9         339.5       1.0X
+format date wholestage on                          3418           3438          14          2.9         341.8       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7394           7395           1          1.4         739.4       1.0X
-from_unixtime wholestage on                        7306           7332          27          1.4         730.6       1.0X
+from_unixtime wholestage off                       4565           4592          38          2.2         456.5       1.0X
+from_unixtime wholestage on                        4608           4635          32          2.2         460.8       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1265           1268           4          7.9         126.5       1.0X
-from_utc_timestamp wholestage on                   1288           1300          11          7.8         128.8       1.0X
+from_utc_timestamp wholestage off                   801            807           9         12.5          80.1       1.0X
+from_utc_timestamp wholestage on                    819            830           7         12.2          81.9       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1760           1766           8          5.7         176.0       1.0X
-to_utc_timestamp wholestage on                     1732           1743          13          5.8         173.2       1.0X
+to_utc_timestamp wholestage off                    1108           1114           8          9.0         110.8       1.0X
+to_utc_timestamp wholestage on                     1067           1078          13          9.4         106.7       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        359            360           1         27.8          35.9       1.0X
-cast interval wholestage on                         327            337           9         30.6          32.7       1.1X
+cast interval wholestage off                        211            213           2         47.4          21.1       1.0X
+cast interval wholestage on                         185            188           3         54.1          18.5       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1849           1853           5          5.4         184.9       1.0X
-datediff wholestage on                             1819           1826           6          5.5         181.9       1.0X
+datediff wholestage off                            1120           1120           1          8.9         112.0       1.0X
+datediff wholestage on                             1174           1205          19          8.5         117.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      6061           6066           7          1.6         606.1       1.0X
-months_between wholestage on                       6022           6034          12          1.7         602.2       1.0X
+months_between wholestage off                      3669           3688          26          2.7         366.9       1.0X
+months_between wholestage on                       3687           3819         181          2.7         368.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2249           2298          69          0.4        2249.4       1.0X
-window wholestage on                              47857          47942         130          0.0       47857.1       0.0X
+window wholestage off                              1147           1148           1          0.9        1146.6       1.0X
+window wholestage on                              16997          17207         226          0.1       16996.7       0.1X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2530           2535           7          4.0         253.0       1.0X
-date_trunc YEAR wholestage on                      2490           2506          18          4.0         249.0       1.0X
+date_trunc YEAR wholestage off                     1824           1859          50          5.5         182.4       1.0X
+date_trunc YEAR wholestage on                      1844           1942          71          5.4         184.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2525           2530           6          4.0         252.5       1.0X
-date_trunc YYYY wholestage on                      2484           2492           7          4.0         248.4       1.0X
+date_trunc YYYY wholestage off                     1808           1815          11          5.5         180.8       1.0X
+date_trunc YYYY wholestage on                      1833           1864          49          5.5         183.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2530           2530           0          4.0         253.0       1.0X
-date_trunc YY wholestage on                        2485           2494          11          4.0         248.5       1.0X
+date_trunc YY wholestage off                       1867           1883          23          5.4         186.7       1.0X
+date_trunc YY wholestage on                        1843           1861          15          5.4         184.3       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2507           2511           5          4.0         250.7       1.0X
-date_trunc MON wholestage on                       2501           2510          10          4.0         250.1       1.0X
+date_trunc MON wholestage off                      1845           1858          18          5.4         184.5       1.0X
+date_trunc MON wholestage on                       1830           1893          42          5.5         183.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2517           2521           5          4.0         251.7       1.0X
-date_trunc MONTH wholestage on                     2507           2517          15          4.0         250.7       1.0X
+date_trunc MONTH wholestage off                    1822           1855          47          5.5         182.2       1.0X
+date_trunc MONTH wholestage on                     1832           1863          20          5.5         183.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2502           2504           4          4.0         250.2       1.0X
-date_trunc MM wholestage on                        2504           2509           4          4.0         250.4       1.0X
+date_trunc MM wholestage off                       1843           1848           7          5.4         184.3       1.0X
+date_trunc MM wholestage on                        1886           1905          14          5.3         188.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2343           2344           3          4.3         234.3       1.0X
-date_trunc DAY wholestage on                       2308           2328          22          4.3         230.8       1.0X
+date_trunc DAY wholestage off                      1542           1545           4          6.5         154.2       1.0X
+date_trunc DAY wholestage on                       1610           1616           5          6.2         161.0       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2344           2349           7          4.3         234.4       1.0X
-date_trunc DD wholestage on                        2312           2326          16          4.3         231.2       1.0X
+date_trunc DD wholestage off                       1521           1529          11          6.6         152.1       1.0X
+date_trunc DD wholestage on                        1595           1611          21          6.3         159.5       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2330           2332           2          4.3         233.0       1.0X
-date_trunc HOUR wholestage on                      2269           2273           4          4.4         226.9       1.0X
+date_trunc HOUR wholestage off                     1496           1543          67          6.7         149.6       1.0X
+date_trunc HOUR wholestage on                      1567           1594          18          6.4         156.7       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    396            397           1         25.2          39.6       1.0X
-date_trunc MINUTE wholestage on                     371            379           5         27.0          37.1       1.1X
+date_trunc MINUTE wholestage off                    230            230           1         43.5          23.0       1.0X
+date_trunc MINUTE wholestage on                     288            295           7         34.7          28.8       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    364            376          18         27.5          36.4       1.0X
-date_trunc SECOND wholestage on                     373            376           2         26.8          37.3       1.0X
+date_trunc SECOND wholestage off                    247            249           4         40.5          24.7       1.0X
+date_trunc SECOND wholestage on                     297            314          12         33.6          29.7       0.8X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2433           2437           6          4.1         243.3       1.0X
-date_trunc WEEK wholestage on                      2402           2418          21          4.2         240.2       1.0X
+date_trunc WEEK wholestage off                     1786           1788           3          5.6         178.6       1.0X
+date_trunc WEEK wholestage on                      1786           1832          46          5.6         178.6       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3304           3310           9          3.0         330.4       1.0X
-date_trunc QUARTER wholestage on                   3283           3289           5          3.0         328.3       1.0X
+date_trunc QUARTER wholestage off                  2319           2365          66          4.3         231.9       1.0X
+date_trunc QUARTER wholestage on                   2424           2551         182          4.1         242.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           323            324           1         30.9          32.3       1.0X
-trunc year wholestage on                            344            349          10         29.1          34.4       0.9X
+trunc year wholestage off                           180            189          12         55.5          18.0       1.0X
+trunc year wholestage on                            271            277           5         36.9          27.1       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           333            337           6         30.0          33.3       1.0X
-trunc yyyy wholestage on                            345            351           7         29.0          34.5       1.0X
+trunc yyyy wholestage off                           189            191           4         52.9          18.9       1.0X
+trunc yyyy wholestage on                            276            284           6         36.2          27.6       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             315            317           3         31.7          31.5       1.0X
-trunc yy wholestage on                              340            345           6         29.4          34.0       0.9X
+trunc yy wholestage off                             189            190           1         52.9          18.9       1.0X
+trunc yy wholestage on                              279            294          11         35.9          27.9       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            314            314           1         31.9          31.4       1.0X
-trunc mon wholestage on                             342            343           1         29.3          34.2       0.9X
+trunc mon wholestage off                            185            186           1         54.1          18.5       1.0X
+trunc mon wholestage on                             272            285          13         36.8          27.2       0.7X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          311            316           8         32.2          31.1       1.0X
-trunc month wholestage on                           341            343           1         29.3          34.1       0.9X
+trunc month wholestage off                          190            190           1         52.6          19.0       1.0X
+trunc month wholestage on                           293            300           4         34.1          29.3       0.6X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             309            311           3         32.4          30.9       1.0X
-trunc mm wholestage on                              342            347           4         29.3          34.2       0.9X
+trunc mm wholestage off                             178            182           6         56.3          17.8       1.0X
+trunc mm wholestage on                              306            312           5         32.7          30.6       0.6X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     175            179           6          5.7         175.3       1.0X
-to timestamp str wholestage on                      170            177          13          5.9         169.6       1.0X
+to timestamp str wholestage off                     111            117           8          9.0         110.9       1.0X
+to timestamp str wholestage on                      101            109           6          9.9         100.6       1.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1563           1577          20          0.6        1563.1       1.0X
-to_timestamp wholestage on                         1587           1591           3          0.6        1587.1       1.0X
+to_timestamp wholestage off                         735            746          15          1.4         734.9       1.0X
+to_timestamp wholestage on                          708            725          11          1.4         708.2       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1646           1647           2          0.6        1646.1       1.0X
-to_unix_timestamp wholestage on                    1602           1609           8          0.6        1602.3       1.0X
+to_unix_timestamp wholestage off                    718            727          12          1.4         717.9       1.0X
+to_unix_timestamp wholestage on                     739            755          12          1.4         739.1       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          216            217           1          4.6         216.0       1.0X
-to date str wholestage on                           222            226           3          4.5         221.9       1.0X
+to date str wholestage off                          124            125           1          8.0         124.4       1.0X
+to date str wholestage on                           134            138           3          7.5         133.9       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3800           3806           8          0.3        3800.1       1.0X
-to_date wholestage on                              3796           3808          10          0.3        3795.5       1.0X
+to_date wholestage off                             1510           1544          48          0.7        1510.4       1.0X
+to_date wholestage on                              1544           1557          15          0.6        1544.2       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  427            436          12         11.7          85.3       1.0X
-From java.time.LocalDate                            359            363           6         13.9          71.9       1.2X
-Collect java.sql.Date                              2131           2482         403          2.3         426.1       0.2X
-Collect java.time.LocalDate                        1457           1871         495          3.4         291.4       0.3X
-From java.sql.Timestamp                             369            371           1         13.5          73.8       1.2X
-From java.time.Instant                              325            330           6         15.4          65.0       1.3X
-Collect longs                                      1370           1388          20          3.6         274.1       0.3X
-Collect java.sql.Timestamp                         1701           1721          17          2.9         340.2       0.3X
-Collect java.time.Instant                          1817           1959         159          2.8         363.4       0.2X
+From java.sql.Date                                  269            278           9         18.6          53.7       1.0X
+From java.time.LocalDate                            227            234           7         22.0          45.4       1.2X
+Collect java.sql.Date                              1164           1272         141          4.3         232.8       0.2X
+Collect java.time.LocalDate                        1070           1130          59          4.7         214.1       0.3X
+From java.sql.Timestamp                             246            248           2         20.3          49.2       1.1X
+From java.time.Instant                              214            216           2         23.4          42.8       1.3X
+Collect longs                                       814            831          15          6.1         162.7       0.3X
+Collect java.sql.Timestamp                         1016           1096          78          4.9         203.2       0.3X
+Collect java.time.Instant                          1012           1093          86          4.9         202.4       0.3X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 045daaaad80a1..dba6c909be637 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -1,435 +1,457 @@
+================================================================================================
+datetime +/- interval
+================================================================================================
+
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+date + interval(m)                                 1003           1012          13          0.0   334353721.7       1.0X
+date + interval(m, d)                              1154           1174          29          0.0   384575202.7       0.9X
+date + interval(m, d, ms)                          4338           4366          40          0.0  1446002701.3       0.2X
+date - interval(m)                                  850            858           8          0.0   283424914.7       1.2X
+date - interval(m, d)                              1017           1031          19          0.0   339034354.7       1.0X
+date - interval(m, d, ms)                          4699           4717          25          0.0  1566218686.3       0.2X
+timestamp + interval(m)                            2044           2046           3          0.0   681382301.0       0.5X
+timestamp + interval(m, d)                         2215           2249          48          0.0   738464286.7       0.5X
+timestamp + interval(m, d, ms)                     2053           2063          13          0.0   684393366.0       0.5X
+timestamp - interval(m)                            1668           1677          12          0.0   556138256.7       0.6X
+timestamp - interval(m, d)                         1865           1882          25          0.0   621574795.3       0.5X
+timestamp - interval(m, d, ms)                     2075           2077           3          0.0   691569937.3       0.5X
+
+
 ================================================================================================
 Extract components
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    422            452          43         23.7          42.2       1.0X
-cast to timestamp wholestage on                     366            382          16         27.3          36.6       1.2X
+cast to timestamp wholestage off                    192            192           0         52.2          19.2       1.0X
+cast to timestamp wholestage on                     163            166           3         61.3          16.3       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1298           1324          37          7.7         129.8       1.0X
-year of timestamp wholestage on                    1201           1214          13          8.3         120.1       1.1X
+year of timestamp wholestage off                    743            745           4         13.5          74.3       1.0X
+year of timestamp wholestage on                     708            715           5         14.1          70.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1443           1444           1          6.9         144.3       1.0X
-quarter of timestamp wholestage on                 1401           1415           8          7.1         140.1       1.0X
+quarter of timestamp wholestage off                 848            857          12         11.8          84.8       1.0X
+quarter of timestamp wholestage on                  803            813           6         12.5          80.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1230           1240          13          8.1         123.0       1.0X
-month of timestamp wholestage on                   1252           1262           8          8.0         125.2       1.0X
+month of timestamp wholestage off                   740            745           7         13.5          74.0       1.0X
+month of timestamp wholestage on                    703            710           5         14.2          70.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             2122           2123           2          4.7         212.2       1.0X
-weekofyear of timestamp wholestage on              2398           2413          13          4.2         239.8       0.9X
+weekofyear of timestamp wholestage off             1162           1182          28          8.6         116.2       1.0X
+weekofyear of timestamp wholestage on              1093           1102           9          9.2         109.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1225           1226           2          8.2         122.5       1.0X
-day of timestamp wholestage on                     1249           1259           8          8.0         124.9       1.0X
+day of timestamp wholestage off                     742            748           9         13.5          74.2       1.0X
+day of timestamp wholestage on                      703            713           7         14.2          70.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1274           1279           6          7.8         127.4       1.0X
-dayofyear of timestamp wholestage on               1278           1294          25          7.8         127.8       1.0X
+dayofyear of timestamp wholestage off               791            799          11         12.6          79.1       1.0X
+dayofyear of timestamp wholestage on                732            744           9         13.7          73.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1221           1222           0          8.2         122.1       1.0X
-dayofmonth of timestamp wholestage on              1233           1243           8          8.1         123.3       1.0X
+dayofmonth of timestamp wholestage off              738            752          20         13.6          73.8       1.0X
+dayofmonth of timestamp wholestage on               695            712           9         14.4          69.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1411           1422          16          7.1         141.1       1.0X
-dayofweek of timestamp wholestage on               1393           1398           5          7.2         139.3       1.0X
+dayofweek of timestamp wholestage off               854            856           3         11.7          85.4       1.0X
+dayofweek of timestamp wholestage on                819            839          16         12.2          81.9       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1351           1352           1          7.4         135.1       1.0X
-weekday of timestamp wholestage on                 1335           1342           5          7.5         133.5       1.0X
+weekday of timestamp wholestage off                 816            821           7         12.3          81.6       1.0X
+weekday of timestamp wholestage on                  788            800           8         12.7          78.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    998            999           1         10.0          99.8       1.0X
-hour of timestamp wholestage on                     975            989          15         10.3          97.5       1.0X
+hour of timestamp wholestage off                    595            595           1         16.8          59.5       1.0X
+hour of timestamp wholestage on                     533            541          10         18.8          53.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1005           1012          10         10.0         100.5       1.0X
-minute of timestamp wholestage on                   978           1003          35         10.2          97.8       1.0X
+minute of timestamp wholestage off                  585            588           4         17.1          58.5       1.0X
+minute of timestamp wholestage on                   532            545          11         18.8          53.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  963            965           4         10.4          96.3       1.0X
-second of timestamp wholestage on                   977            988          16         10.2          97.7       1.0X
+second of timestamp wholestage off                  579            589          13         17.3          57.9       1.0X
+second of timestamp wholestage on                   529            537           6         18.9          52.9       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         289            295           8         34.6          28.9       1.0X
-current_date wholestage on                          304            314           9         32.9          30.4       1.0X
+current_date wholestage off                         171            174           4         58.4          17.1       1.0X
+current_date wholestage on                          152            155           3         65.6          15.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    296            300           7         33.8          29.6       1.0X
-current_timestamp wholestage on                     302            311          11         33.1          30.2       1.0X
+current_timestamp wholestage off                    178            181           4         56.2          17.8       1.0X
+current_timestamp wholestage on                     138            149           7         72.6          13.8       1.3X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1071           1078          10          9.3         107.1       1.0X
-cast to date wholestage on                         1018           1027           7          9.8         101.8       1.1X
+cast to date wholestage off                         630            640          14         15.9          63.0       1.0X
+cast to date wholestage on                          591            594           5         16.9          59.1       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1261           1264           5          7.9         126.1       1.0X
-last_day wholestage on                             1259           1271           9          7.9         125.9       1.0X
+last_day wholestage off                             759            778          26         13.2          75.9       1.0X
+last_day wholestage on                              727            736           9         13.8          72.7       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1119           1147          41          8.9         111.9       1.0X
-next_day wholestage on                             1076           1082           8          9.3         107.6       1.0X
+next_day wholestage off                             649            659          15         15.4          64.9       1.0X
+next_day wholestage on                              628            629           1         15.9          62.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1062           1070          12          9.4         106.2       1.0X
-date_add wholestage on                             1058           1060           2          9.5         105.8       1.0X
+date_add wholestage off                             621            622           1         16.1          62.1       1.0X
+date_add wholestage on                              600            606           6         16.7          60.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1048           1080          45          9.5         104.8       1.0X
-date_sub wholestage on                             1060           1066           8          9.4         106.0       1.0X
+date_sub wholestage off                             611            626          21         16.4          61.1       1.0X
+date_sub wholestage on                              588            600           7         17.0          58.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1368           1371           4          7.3         136.8       1.0X
-add_months wholestage on                           1423           1429           7          7.0         142.3       1.0X
+add_months wholestage off                           843            845           2         11.9          84.3       1.0X
+add_months wholestage on                            818            831          11         12.2          81.8       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5865           5898          47          1.7         586.5       1.0X
-format date wholestage on                          5778           5788           8          1.7         577.8       1.0X
+format date wholestage off                         3557           3569          18          2.8         355.7       1.0X
+format date wholestage on                          3564           3588          17          2.8         356.4       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8886           8899          18          1.1         888.6       1.0X
-from_unixtime wholestage on                        8831           8841           7          1.1         883.1       1.0X
+from_unixtime wholestage off                       4875           4887          17          2.1         487.5       1.0X
+from_unixtime wholestage on                        4845           4870          16          2.1         484.5       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1093           1103          14          9.1         109.3       1.0X
-from_utc_timestamp wholestage on                   1136           1147          11          8.8         113.6       1.0X
+from_utc_timestamp wholestage off                   665            671           8         15.0          66.5       1.0X
+from_utc_timestamp wholestage on                    654            672          14         15.3          65.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1640           1646           8          6.1         164.0       1.0X
-to_utc_timestamp wholestage on                     1626           1641          12          6.1         162.6       1.0X
+to_utc_timestamp wholestage off                     982            983           1         10.2          98.2       1.0X
+to_utc_timestamp wholestage on                      877            889           9         11.4          87.7       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        338            348          14         29.6          33.8       1.0X
-cast interval wholestage on                         318            323           4         31.4          31.8       1.1X
+cast interval wholestage off                        200            206           9         50.1          20.0       1.0X
+cast interval wholestage on                         157            163           5         63.6          15.7       1.3X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1826           1829           4          5.5         182.6       1.0X
-datediff wholestage on                             1785           1797          11          5.6         178.5       1.0X
+datediff wholestage off                            1065           1068           4          9.4         106.5       1.0X
+datediff wholestage on                             1028           1047          15          9.7         102.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      4795           4801           9          2.1         479.5       1.0X
-months_between wholestage on                       4752           4766          19          2.1         475.2       1.0X
+months_between wholestage off                      3102           3111          13          3.2         310.2       1.0X
+months_between wholestage on                       2970           3028          46          3.4         297.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2452           2590         195          0.4        2452.1       1.0X
-window wholestage on                              42821          42877          54          0.0       42820.7       0.1X
+window wholestage off                              1142           1154          16          0.9        1142.2       1.0X
+window wholestage on                              14817          15049         257          0.1       14816.5       0.1X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2408           2413           7          4.2         240.8       1.0X
-date_trunc YEAR wholestage on                      2311           2318           6          4.3         231.1       1.0X
+date_trunc YEAR wholestage off                     1516           1518           2          6.6         151.6       1.0X
+date_trunc YEAR wholestage on                      1458           1468           9          6.9         145.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2404           2414          15          4.2         240.4       1.0X
-date_trunc YYYY wholestage on                      2317           2343          17          4.3         231.7       1.0X
+date_trunc YYYY wholestage off                     1535           1535           1          6.5         153.5       1.0X
+date_trunc YYYY wholestage on                      1453           1461           7          6.9         145.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2403           2415          17          4.2         240.3       1.0X
-date_trunc YY wholestage on                        2298           2316          13          4.4         229.8       1.0X
+date_trunc YY wholestage off                       1561           1567           9          6.4         156.1       1.0X
+date_trunc YY wholestage on                        1452           1467          16          6.9         145.2       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2370           2378          10          4.2         237.0       1.0X
-date_trunc MON wholestage on                       2329           2343          13          4.3         232.9       1.0X
+date_trunc MON wholestage off                      1522           1531          13          6.6         152.2       1.0X
+date_trunc MON wholestage on                       1458           1467           7          6.9         145.8       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2370           2376           7          4.2         237.0       1.0X
-date_trunc MONTH wholestage on                     2329           2347          15          4.3         232.9       1.0X
+date_trunc MONTH wholestage off                    1518           1519           0          6.6         151.8       1.0X
+date_trunc MONTH wholestage on                     1452           1465          16          6.9         145.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2363           2364           2          4.2         236.3       1.0X
-date_trunc MM wholestage on                        2323           2342          15          4.3         232.3       1.0X
+date_trunc MM wholestage off                       1531           1532           1          6.5         153.1       1.0X
+date_trunc MM wholestage on                        1453           1463           8          6.9         145.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2222           2226           7          4.5         222.2       1.0X
-date_trunc DAY wholestage on                       2162           2169           6          4.6         216.2       1.0X
+date_trunc DAY wholestage off                      1287           1309          31          7.8         128.7       1.0X
+date_trunc DAY wholestage on                       1310           1337          16          7.6         131.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2228           2232           5          4.5         222.8       1.0X
-date_trunc DD wholestage on                        2164           2171           6          4.6         216.4       1.0X
+date_trunc DD wholestage off                       1322           1328           9          7.6         132.2       1.0X
+date_trunc DD wholestage on                        1282           1324          28          7.8         128.2       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2236           2242           9          4.5         223.6       1.0X
-date_trunc HOUR wholestage on                      2212           2227          16          4.5         221.2       1.0X
+date_trunc HOUR wholestage off                     1379           1393          20          7.3         137.9       1.0X
+date_trunc HOUR wholestage on                      1288           1302          11          7.8         128.8       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    368            371           4         27.2          36.8       1.0X
-date_trunc MINUTE wholestage on                     331            335           4         30.2          33.1       1.1X
+date_trunc MINUTE wholestage off                    243            245           2         41.2          24.3       1.0X
+date_trunc MINUTE wholestage on                     213            219           8         47.0          21.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    383            383           0         26.1          38.3       1.0X
-date_trunc SECOND wholestage on                     328            335           8         30.5          32.8       1.2X
+date_trunc SECOND wholestage off                    238            245          11         42.1          23.8       1.0X
+date_trunc SECOND wholestage on                     201            210           9         49.7          20.1       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2259           2277          26          4.4         225.9       1.0X
-date_trunc WEEK wholestage on                      2201           2227          18          4.5         220.1       1.0X
+date_trunc WEEK wholestage off                     1443           1477          49          6.9         144.3       1.0X
+date_trunc WEEK wholestage on                      1491           1516          17          6.7         149.1       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3168           3183          22          3.2         316.8       1.0X
-date_trunc QUARTER wholestage on                   3130           3139          10          3.2         313.0       1.0X
+date_trunc QUARTER wholestage off                  2017           2039          32          5.0         201.7       1.0X
+date_trunc QUARTER wholestage on                   1966           2005          36          5.1         196.6       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           326            330           5         30.7          32.6       1.0X
-trunc year wholestage on                            295            298           3         33.8          29.5       1.1X
+trunc year wholestage off                           206            206           1         48.6          20.6       1.0X
+trunc year wholestage on                            175            178           2         57.2          17.5       1.2X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           322            323           0         31.0          32.2       1.0X
-trunc yyyy wholestage on                            295            297           2         33.9          29.5       1.1X
+trunc yyyy wholestage off                           188            189           2         53.2          18.8       1.0X
+trunc yyyy wholestage on                            176            180           4         56.9          17.6       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             324            325           2         30.9          32.4       1.0X
-trunc yy wholestage on                              295            300           4         33.9          29.5       1.1X
+trunc yy wholestage off                             191            191           0         52.4          19.1       1.0X
+trunc yy wholestage on                              175            180           4         57.0          17.5       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            323            336          19         31.0          32.3       1.0X
-trunc mon wholestage on                             294            297           3         34.0          29.4       1.1X
+trunc mon wholestage off                            203            205           3         49.3          20.3       1.0X
+trunc mon wholestage on                             183            186           2         54.8          18.3       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          322            322           0         31.1          32.2       1.0X
-trunc month wholestage on                           295            297           4         33.9          29.5       1.1X
+trunc month wholestage off                          199            199           0         50.3          19.9       1.0X
+trunc month wholestage on                           177            179           2         56.4          17.7       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             322            324           2         31.0          32.2       1.0X
-trunc mm wholestage on                              297            306           6         33.7          29.7       1.1X
+trunc mm wholestage off                             198            198           1         50.5          19.8       1.0X
+trunc mm wholestage on                              180            183           3         55.7          18.0       1.1X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     214            215           1          4.7         214.1       1.0X
-to timestamp str wholestage on                      212            217           4          4.7         212.1       1.0X
+to timestamp str wholestage off                     138            139           2          7.2         138.2       1.0X
+to timestamp str wholestage on                      129            138           7          7.8         128.9       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1543           1545           3          0.6        1542.8       1.0X
-to_timestamp wholestage on                         1541           1550          12          0.6        1541.0       1.0X
+to_timestamp wholestage off                         885            889           5          1.1         885.3       1.0X
+to_timestamp wholestage on                          854            866          10          1.2         854.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1590           1592           2          0.6        1590.4       1.0X
-to_unix_timestamp wholestage on                    1527           1531           4          0.7        1527.0       1.0X
+to_unix_timestamp wholestage off                    848            856          12          1.2         848.1       1.0X
+to_unix_timestamp wholestage on                     826            850          18          1.2         826.4       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          264            267           4          3.8         264.0       1.0X
-to date str wholestage on                           261            263           3          3.8         261.2       1.0X
+to date str wholestage off                          167            171           5          6.0         167.2       1.0X
+to date str wholestage on                           165            173           4          6.1         165.0       1.0X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3578           3590          16          0.3        3578.2       1.0X
-to_date wholestage on                              3495           3504           9          0.3        3495.4       1.0X
+to_date wholestage off                             1612           1633          31          0.6        1611.7       1.0X
+to_date wholestage on                              1588           1605          19          0.6        1588.2       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
-Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
+Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  414            424          12         12.1          82.9       1.0X
-From java.time.LocalDate                            350            355           4         14.3          69.9       1.2X
-Collect java.sql.Date                              2064           2709        1101          2.4         412.8       0.2X
-Collect java.time.LocalDate                        1467           1493          31          3.4         293.5       0.3X
-From java.sql.Timestamp                             396            401           4         12.6          79.3       1.0X
-From java.time.Instant                              260            266           7         19.2          52.0       1.6X
-Collect longs                                      1336           1475         178          3.7         267.2       0.3X
-Collect java.sql.Timestamp                         2175           2887        1226          2.3         435.0       0.2X
-Collect java.time.Instant                          1679           1917         228          3.0         335.7       0.2X
+From java.sql.Date                                  245            247           1         20.4          49.0       1.0X
+From java.time.LocalDate                            228            233           4         21.9          45.6       1.1X
+Collect java.sql.Date                              1239           1361         209          4.0         247.9       0.2X
+Collect java.time.LocalDate                        1049           1107          54          4.8         209.8       0.2X
+From java.sql.Timestamp                             247            252           4         20.2          49.5       1.0X
+From java.time.Instant                              156            158           3         32.1          31.2       1.6X
+Collect longs                                       854            910          59          5.9         170.8       0.3X
+Collect java.sql.Timestamp                         1133           1140          12          4.4         226.6       0.2X
+Collect java.time.Instant                          1108           1159          74          4.5         221.7       0.2X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 0620227687468..0034819b58893 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -60,6 +60,48 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
     withDefaultTimeZone(LA) {
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
         val N = 10000000
+        runBenchmark("datetime +/- interval") {
+          val benchmark = new Benchmark("datetime +/- interval", 3, output = output)
+          val ts = "cast(id as timestamp)"
+          val dt = s"cast($ts as date)"
+          benchmark.addCase("date + interval(m)") { _ =>
+            doBenchmark(N, s"$dt + interval 1 month")
+          }
+          benchmark.addCase("date + interval(m, d)") { _ =>
+            doBenchmark(N, s"$dt + interval 1 month 2 day")
+          }
+          benchmark.addCase("date + interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$dt + interval 1 month 2 day 5 hour")
+          }
+          benchmark.addCase("date - interval(m)") { _ =>
+            doBenchmark(N, s"$dt - interval 1 month")
+          }
+          benchmark.addCase("date - interval(m, d)") { _ =>
+            doBenchmark(N, s"$dt - interval 1 month 2 day")
+          }
+          benchmark.addCase("date - interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$dt - interval 1 month 2 day 5 hour")
+          }
+          benchmark.addCase("timestamp + interval(m)") { _ =>
+            doBenchmark(N, s"$ts + interval 1 month")
+          }
+          benchmark.addCase("timestamp + interval(m, d)") { _ =>
+            doBenchmark(N, s"$ts + interval 1 month 2 day")
+          }
+          benchmark.addCase("timestamp + interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$ts + interval 1 month 2 day 5 hour")
+          }
+          benchmark.addCase("timestamp - interval(m)") { _ =>
+            doBenchmark(N, s"$ts - interval 1 month")
+          }
+          benchmark.addCase("timestamp - interval(m, d)") { _ =>
+            doBenchmark(N, s"$ts - interval 1 month 2 day")
+          }
+          benchmark.addCase("timestamp - interval(m, d, ms)") { _ =>
+            doBenchmark(N, s"$ts - interval 1 month 2 day 5 hour")
+          }
+          benchmark.run()
+        }
         runBenchmark("Extract components") {
           run(N, "cast to timestamp", "cast(id as timestamp)")
           run(N, "year")

From b85262dd3a847ccdd4dde8c14f5d0d95f43ce172 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Tue, 28 Apr 2020 08:57:48 -0700
Subject: [PATCH 0591/1280] [SPARK-31534][WEBUI][3.0] Text for tooltip should
 be escaped

### What changes were proposed in this pull request?
This PR backports #28317 which escapes text for tooltip for DAG Viz and Timeline View.

### Why are the changes needed?

This is a bug.
Normally, DAG Viz and Timeline View show tooltip like as follows.

<img width="278" alt="dag-viz-tooltip" src="https://user-images.githubusercontent.com/4736016/80127481-5a6c6880-85cf-11ea-8daf-cfd59aa3ba09.png">
<img width="477" alt="timeline-tooltip" src="https://user-images.githubusercontent.com/4736016/80127500-60624980-85cf-11ea-9b0f-cce301019e3a.png">

They contain a callsite properly.
However, if a callsite contains characters which should be escaped for HTML without escaping , the corresponding tooltips wouldn't show the callsite and its following text properly.
<img width="179" alt="dag-viz-tooltip-before-fixed" src="https://user-images.githubusercontent.com/4736016/80128480-b1267200-85d0-11ea-8035-ad68ae5fbcab.png">
<img width="261" alt="timeline-tooltip-before-fixed" src="https://user-images.githubusercontent.com/4736016/80128492-b5eb2600-85d0-11ea-9556-c48490110244.png">

The reason of this issue is that the source texts of the tooltip texts are not escaped.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

I tested manually.
First, I ran a job `sc.parallelize(1 to 10).collect` in Spark Shell then, visited AllJobsPage and JobPage and confirmed tooltip texts.
<img width="277" alt="timeline-tooltip-fixed-3 0" src="https://user-images.githubusercontent.com/4736016/80332616-41411180-8886-11ea-8d93-28e1c5265115.png">
<img width="191" alt="dag-tooltip-fixed-3 0" src="https://user-images.githubusercontent.com/4736016/80332625-44d49880-8886-11ea-8f2e-de8df1369e62.png">
And I confirmed that the appearance of the label of DAG-viz in StagePage is not changed.
<img width="313" alt="stage-page-dag" src="https://user-images.githubusercontent.com/4736016/80332725-9b41d700-8886-11ea-9adb-40d50ad29f86.png">

I also added a testcase.
With this testcase, an error message related to JavaScript is shown.

`TypeError: Cannot call method "indexOf" of undefined (http://192.168.1.209:59376/static/spark-dag-viz.js#378)`

This is thrown from `interpretLineBreak` in `spark-dag-viz.js`.
HtmlUnit seems not to support `innerHTML` for text content (I tried replacing it with `textContent` and the error message is not shown).
But DOMs which is needed by the testcase added is already rendered before `interpretLineBreak` is called. So the testcase successfully passes.

Closes #28359 from sarutak/fix-tooltip-branch-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/ui/static/spark-dag-viz.js   | 19 ++-------
 .../apache/spark/ui/jobs/AllJobsPage.scala    |  7 ++--
 .../org/apache/spark/ui/jobs/JobPage.scala    |  7 ++--
 .../spark/ui/scope/RDDOperationGraph.scala    |  6 ++-
 .../org/apache/spark/ui/UISeleniumSuite.scala | 42 +++++++++++++++----
 5 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index 25dec9d3788ba..ae02defd9bb9c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -173,8 +173,8 @@ function renderDagViz(forJob) {
   });
 
   metadataContainer().selectAll(".barrier-rdd").each(function() {
-    var rddId = d3.select(this).text().trim()
-    var clusterId = VizConstants.clusterPrefix + rddId
+    var rddId = d3.select(this).text().trim();
+    var clusterId = VizConstants.clusterPrefix + rddId;
     svg.selectAll("g." + clusterId).classed("barrier", true)
   });
 
@@ -282,11 +282,7 @@ function renderDagVizForJob(svgContainer) {
 
 /* Render the dot file as an SVG in the given container. */
 function renderDot(dot, container, forJob) {
-  var escaped_dot = dot
-    .replace(/&lt;/g, "<")
-    .replace(/&gt;/g, ">")
-    .replace(/&quot;/g, "\"");
-  var g = graphlibDot.read(escaped_dot);
+  var g = graphlibDot.read(dot);
   var renderer = new dagreD3.render();
   preprocessGraphLayout(g, forJob);
   renderer(container, g);
@@ -498,18 +494,11 @@ function connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer) {
   edgesContainer.append("path").datum(points).attr("d", line);
 }
 
-/*
- * Replace `/n` with `<br/>`
- */
-function replaceLineBreak(str) {
-    return str.replace("\\n", "<br/>");
-}
-
 /* (Job page only) Helper function to add tooltips for RDDs. */
 function addTooltipsForRDDs(svgContainer) {
   svgContainer.selectAll("g.node").each(function() {
     var node = d3.select(this);
-    var tooltipText = replaceLineBreak(node.attr("name"));
+    var tooltipText = node.attr("name");
     if (tooltipText) {
       node.select("circle")
         .attr("data-toggle", "tooltip")
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
index f53e67ff5cc98..06762f69f2ecb 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -88,7 +88,8 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
       // extra layers of escaping in order to embed this in a Javascript string literal.
       val escapedDesc = Utility.escape(jobDescription)
-      val jsEscapedDesc = StringEscapeUtils.escapeEcmaScript(escapedDesc)
+      val jsEscapedDescForTooltip = StringEscapeUtils.escapeEcmaScript(Utility.escape(escapedDesc))
+      val jsEscapedDescForLabel = StringEscapeUtils.escapeEcmaScript(escapedDesc)
       val jobEventJsonAsStr =
         s"""
            |{
@@ -98,7 +99,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
            |  'end': new Date(${completionTime}),
            |  'content': '<div class="application-timeline-content"' +
            |     'data-html="true" data-placement="top" data-toggle="tooltip"' +
-           |     'data-title="${jsEscapedDesc} (Job ${jobId})<br>' +
+           |     'data-title="${jsEscapedDescForTooltip} (Job ${jobId})<br>' +
            |     'Status: ${status}<br>' +
            |     'Submitted: ${UIUtils.formatDate(new Date(submissionTime))}' +
            |     '${
@@ -108,7 +109,7 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
                        ""
                      }
                   }">' +
-           |    '${jsEscapedDesc} (Job ${jobId})</div>'
+           |    '${jsEscapedDescForLabel} (Job ${jobId})</div>'
            |}
          """.stripMargin
       jobEventJsonAsStr
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index 12f1aa25e8d2a..54582df919a43 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -69,7 +69,8 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
       // extra layers of escaping in order to embed this in a Javascript string literal.
       val escapedName = Utility.escape(name)
-      val jsEscapedName = StringEscapeUtils.escapeEcmaScript(escapedName)
+      val jsEscapedNameForTooltip = StringEscapeUtils.escapeEcmaScript(Utility.escape(escapedName))
+      val jsEscapedNameForLabel = StringEscapeUtils.escapeEcmaScript(escapedName)
       s"""
          |{
          |  'className': 'stage job-timeline-object ${status}',
@@ -78,7 +79,7 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
          |  'end': new Date(${completionTime}),
          |  'content': '<div class="job-timeline-content" data-toggle="tooltip"' +
          |   'data-placement="top" data-html="true"' +
-         |   'data-title="${jsEscapedName} (Stage ${stageId}.${attemptId})<br>' +
+         |   'data-title="${jsEscapedNameForTooltip} (Stage ${stageId}.${attemptId})<br>' +
          |   'Status: ${status.toUpperCase(Locale.ROOT)}<br>' +
          |   'Submitted: ${UIUtils.formatDate(submissionTime)}' +
          |   '${
@@ -88,7 +89,7 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
                    ""
                  }
               }">' +
-         |    '${jsEscapedName} (Stage ${stageId}.${attemptId})</div>',
+         |    '${jsEscapedNameForLabel} (Stage ${stageId}.${attemptId})</div>',
          |}
        """.stripMargin
     }
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
index 9ace324322947..842ee7aaf49bf 100644
--- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
+++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
@@ -21,6 +21,7 @@ import java.util.Objects
 
 import scala.collection.mutable
 import scala.collection.mutable.{ListBuffer, StringBuilder}
+import scala.xml.Utility
 
 import org.apache.commons.text.StringEscapeUtils
 
@@ -245,8 +246,9 @@ private[spark] object RDDOperationGraph extends Logging {
     } else {
       ""
     }
-    val label = s"${node.name} [${node.id}]$isCached$isBarrier\n${node.callsite}"
-    s"""${node.id} [label="${StringEscapeUtils.escapeJava(label)}"]"""
+    val escapedCallsite = Utility.escape(node.callsite)
+    val label = s"${node.name} [${node.id}]$isCached$isBarrier<br>${escapedCallsite}"
+    s"""${node.id} [labelType="html" label="${StringEscapeUtils.escapeJava(label)}"]"""
   }
 
   /** Update the dot representation of the RDDOperationGraph in cluster to subgraph. */
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 757e03b2b50e5..3ec9385116408 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -44,6 +44,7 @@ import org.apache.spark.internal.config.Status._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.status.api.v1.{JacksonMessageWriter, RDDDataDistribution, StageStatus}
+import org.apache.spark.util.CallSite
 
 private[spark] class SparkUICssErrorHandler extends DefaultCssErrorHandler {
 
@@ -687,29 +688,29 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         assert(stage0.contains("digraph G {\n  subgraph clusterstage_0 {\n    " +
           "label=&quot;Stage 0&quot;;\n    subgraph "))
         assert(stage0.contains("{\n      label=&quot;parallelize&quot;;\n      " +
-          "0 [label=&quot;ParallelCollectionRDD [0]"))
+          "0 [labelType=&quot;html&quot; label=&quot;ParallelCollectionRDD [0]"))
         assert(stage0.contains("{\n      label=&quot;map&quot;;\n      " +
-          "1 [label=&quot;MapPartitionsRDD [1]"))
+          "1 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [1]"))
         assert(stage0.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "2 [label=&quot;MapPartitionsRDD [2]"))
+          "2 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [2]"))
 
         val stage1 = Source.fromURL(sc.ui.get.webUrl +
           "/stages/stage/?id=1&attempt=0&expandDagViz=true").mkString
         assert(stage1.contains("digraph G {\n  subgraph clusterstage_1 {\n    " +
           "label=&quot;Stage 1&quot;;\n    subgraph "))
         assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "3 [label=&quot;ShuffledRDD [3]"))
+          "3 [labelType=&quot;html&quot; label=&quot;ShuffledRDD [3]"))
         assert(stage1.contains("{\n      label=&quot;map&quot;;\n      " +
-          "4 [label=&quot;MapPartitionsRDD [4]"))
+          "4 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [4]"))
         assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "5 [label=&quot;MapPartitionsRDD [5]"))
+          "5 [labelType=&quot;html&quot; label=&quot;MapPartitionsRDD [5]"))
 
         val stage2 = Source.fromURL(sc.ui.get.webUrl +
           "/stages/stage/?id=2&attempt=0&expandDagViz=true").mkString
         assert(stage2.contains("digraph G {\n  subgraph clusterstage_2 {\n    " +
           "label=&quot;Stage 2&quot;;\n    subgraph "))
         assert(stage2.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-          "6 [label=&quot;ShuffledRDD [6]"))
+          "6 [labelType=&quot;html&quot; label=&quot;ShuffledRDD [6]"))
       }
     }
   }
@@ -772,6 +773,33 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
     }
   }
 
+  test("SPARK-31534: text for tooltip should be escaped") {
+    withSpark(newSparkContext()) { sc =>
+      sc.setLocalProperty(CallSite.LONG_FORM, "collect at <console>:25")
+      sc.setLocalProperty(CallSite.SHORT_FORM, "collect at <console>:25")
+      sc.parallelize(1 to 10).collect
+
+      val driver = webDriver.asInstanceOf[HtmlUnitDriver]
+      driver.setJavascriptEnabled(true)
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs")
+        val jobDesc =
+          driver.findElement(By.cssSelector("div[class='application-timeline-content']"))
+        jobDesc.getAttribute("data-title") should include  ("collect at &lt;console&gt;:25")
+
+        goToUi(sc, "/jobs/job/?id=0")
+        val stageDesc = driver.findElement(By.cssSelector("div[class='job-timeline-content']"))
+        stageDesc.getAttribute("data-title") should include ("collect at &lt;console&gt;:25")
+
+        // Open DAG Viz.
+        driver.findElement(By.id("job-dag-viz")).click()
+        val nodeDesc = driver.findElement(By.cssSelector("g[class='node_0 node']"))
+        nodeDesc.getAttribute("name") should include ("collect at &lt;console&gt;:25")
+      }
+    }
+  }
+
   def goToUi(sc: SparkContext, path: String): Unit = {
     goToUi(sc.ui.get, path)
   }

From f8ff9c5eff55ba7003a51f9ac91786d16764f4c9 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Tue, 28 Apr 2020 11:17:45 -0500
Subject: [PATCH 0592/1280] [SPARK-29458][SQL][DOCS] Add a paragraph for scalar
 function in sql getting started

### What changes were proposed in this pull request?
Add a paragraph for scalar function in sql getting started

### Why are the changes needed?
To make 3.0 doc complete.

### Does this PR introduce any user-facing change?
before:
<img width="870" alt="Screen Shot 2020-04-21 at 10 11 12 PM" src="https://user-images.githubusercontent.com/13592258/79943182-16d1fd00-841d-11ea-9744-9cdd58d83f81.png">

after:
<img width="865" alt="Screen Shot 2020-04-22 at 11 49 59 PM" src="https://user-images.githubusercontent.com/13592258/80068256-26704500-84f4-11ea-9845-c835927c027e.png">

<img width="1033" alt="Screen Shot 2020-04-23 at 6 22 53 PM" src="https://user-images.githubusercontent.com/13592258/80165100-82d47280-858f-11ea-8c84-1ef702cc1bff.png">

### How was this patch tested?

Closes #28290 from huaxingao/scalar.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit dcc09022f1b8ecedf6b64bf35ce5d83500211351)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/sql-getting-started.md | 13 +++++--------
 docs/sql-ref-functions.md   |  7 +++++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/sql-getting-started.md b/docs/sql-getting-started.md
index dab34afaaabb2..5a6f18263ac6b 100644
--- a/docs/sql-getting-started.md
+++ b/docs/sql-getting-started.md
@@ -347,16 +347,13 @@ For example:
 </div>
 
 ## Scalar Functions
-(to be filled soon)
 
-## Aggregations
+Scalar functions are functions that return a single value per row, as opposed to aggregation functions, which return a value for a group of rows. Spark SQL supports a variety of [Built-in Scalar Functions](sql-ref-functions.html#scalar-functions). It also supports [User Defined Scalar Functions](sql-ref-functions-udf-scalar.html).
 
-The [built-in DataFrames functions](api/scala/org/apache/spark/sql/functions$.html) provide common
-aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, etc.
-While those functions are designed for DataFrames, Spark SQL also has type-safe versions for some of them in
-[Scala](api/scala/org/apache/spark/sql/expressions/scalalang/typed$.html) and
-[Java](api/java/org/apache/spark/sql/expressions/javalang/typed.html) to work with strongly typed Datasets.
-Moreover, users are not limited to the predefined aggregate functions and can create their own. For more details
+## Aggregate Functions
+
+Aggregate functions are functions that return a single value on a group of rows. The [Built-in Aggregation Functions](sql-ref-functions-builtin.html#aggregate-functions) provide common aggregations such as `count()`, `countDistinct()`, `avg()`, `max()`, `min()`, etc.
+Users are not limited to the predefined aggregate functions and can create their own. For more details
 about user defined aggregate functions, please refer to the documentation of
 [User Defined Aggregate Functions](sql-ref-functions-udf-aggregate.html).
 
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index 6368fb705f893..7493b8bbb7cdf 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -27,13 +27,16 @@ Built-in functions are commonly used routines that Spark SQL predefines and a co
 Spark SQL has some categories of frequently-used built-in functions for aggregtion, arrays/maps, date/timestamp, and JSON data.
 This subsection presents the usages and descriptions of these functions.
 
- * [Aggregate Functions](sql-ref-functions-builtin.html#aggregate-functions)
- * [Window Functions](sql-ref-functions-builtin.html#window-functions)
+#### Scalar Functions
  * [Array Functions](sql-ref-functions-builtin.html#array-functions)
  * [Map Functions](sql-ref-functions-builtin.html#map-functions)
  * [Date and Timestamp Functions](sql-ref-functions-builtin.html#date-and-timestamp-functions)
  * [JSON Functions](sql-ref-functions-builtin.html#json-functions)
 
+#### Aggregate-like Functions
+ * [Aggregate Functions](sql-ref-functions-builtin.html#aggregate-functions)
+ * [Window Functions](sql-ref-functions-builtin.html#window-functions)
+
 ### UDFs (User-Defined Functions)
 
 User-Defined Functions (UDFs) are a feature of Spark SQL that allows users to define their own functions when the system's built-in functions are not enough to perform the desired task. To use UDFs in Spark SQL, users must first define the function, then register the function with Spark, and finally call the registered function. The User-Defined Functions can act on a single row or act on multiple rows at once. Spark SQL also supports integration of existing Hive implementations of UDFs, UDAFs and UDTFs.

From da8c7b8ceffa1566ae35280a2d1c3abcbff47542 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Wed, 29 Apr 2020 09:17:23 +0900
Subject: [PATCH 0593/1280] [SPARK-31556][SQL][DOCS] Document LIKE clause in
 SQL Reference

### What changes were proposed in this pull request?
Document LIKE clause in SQL Reference

### Why are the changes needed?
To make SQL Reference complete

### Does this PR introduce any user-facing change?
Yes

<img width="1050" alt="Screen Shot 2020-04-25 at 5 49 57 PM" src="https://user-images.githubusercontent.com/13592258/80294346-5babab80-871d-11ea-8ac9-51bbab0aca88.png">

<img width="1050" alt="Screen Shot 2020-04-25 at 5 50 24 PM" src="https://user-images.githubusercontent.com/13592258/80294347-5ea69c00-871d-11ea-8c51-7a90ee20f7da.png">

<img width="1050" alt="Screen Shot 2020-04-25 at 5 50 42 PM" src="https://user-images.githubusercontent.com/13592258/80294351-61a18c80-871d-11ea-9e75-e3345d2f52f5.png">

### How was this patch tested?
Manually build and check

Closes #28332 from huaxingao/where_clause.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit d34cb59fb311c3d700e4f4f877b61b17cea313ee)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml                  |   2 +
 docs/sql-ref-syntax-aux-show-databases.md |  13 ++-
 docs/sql-ref-syntax-aux-show-functions.md |   8 +-
 docs/sql-ref-syntax-aux-show-table.md     |  14 +--
 docs/sql-ref-syntax-aux-show-tables.md    |  10 +-
 docs/sql-ref-syntax-aux-show-views.md     |  12 +--
 docs/sql-ref-syntax-qry-explain.md        |   2 +-
 docs/sql-ref-syntax-qry-select-like.md    | 120 ++++++++++++++++++++++
 8 files changed, 154 insertions(+), 27 deletions(-)
 create mode 100644 docs/sql-ref-syntax-qry-select-like.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 1097079e4597d..dfe4cfab2a2ab 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -170,6 +170,8 @@
                   url: sql-ref-syntax-qry-select-inline-table.html
                 - text: Common Table Expression
                   url: sql-ref-syntax-qry-select-cte.html
+                - text: LIKE Predicate
+                  url: sql-ref-syntax-qry-select-like.html
                 - text: Window Function
                   url: sql-ref-syntax-qry-window.html
             - text: EXPLAIN
diff --git a/docs/sql-ref-syntax-aux-show-databases.md b/docs/sql-ref-syntax-aux-show-databases.md
index 0ed34527bcbf3..359900928c0c5 100644
--- a/docs/sql-ref-syntax-aux-show-databases.md
+++ b/docs/sql-ref-syntax-aux-show-databases.md
@@ -29,16 +29,21 @@ and mean the same thing.
 ### Syntax
 
 {% highlight sql %}
-SHOW { DATABASES | SCHEMAS } [ LIKE string_pattern ]
+SHOW { DATABASES | SCHEMAS } [ LIKE regex_pattern ]
 {% endhighlight %}
 
 ### Parameters
 
 <dl>
-  <dt><code><em>LIKE string_pattern</em></code></dt>
+  <dt><code><em>regex_pattern</em></code></dt>
   <dd>
-    Specifies a string pattern that is used to match the databases in the system. In 
-    the specified string pattern <code>'*'</code> matches any number of characters.
+    Specifies a regular expression pattern that is used to filter the results of the
+    statement.
+    <ul>
+      <li>Only <code>*</code> and <code>|</code> are allowed as wildcard pattern.</li>
+      <li>Excluding <code>*</code> and <code>|</code>, the remaining pattern follows the regular expression semantics.</li>
+      <li>The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
+    </ul>
   </dd>
 </dl>
 
diff --git a/docs/sql-ref-syntax-aux-show-functions.md b/docs/sql-ref-syntax-aux-show-functions.md
index da33d999f0b38..ed22a3a901c31 100644
--- a/docs/sql-ref-syntax-aux-show-functions.md
+++ b/docs/sql-ref-syntax-aux-show-functions.md
@@ -58,12 +58,12 @@ SHOW [ function_kind ] FUNCTIONS ( [ LIKE ] function_name | regex_pattern )
   </dd>
   <dt><code><em>regex_pattern</em></code></dt>
   <dd>
-    Specifies a regular expression pattern that is used to limit the results of the
+    Specifies a regular expression pattern that is used to filter the results of the
     statement.
     <ul>
-      <li>Only `*` and `|` are allowed as wildcard pattern.</li>
-      <li>Excluding `*` and `|` the remaining pattern follows the regex semantics.</li>
-      <li>The leading and trailing blanks are trimmed in the input pattern before processing.</li> 
+      <li>Only <code>*</code> and <code>|</code> are allowed as wildcard pattern.</li>
+      <li>Excluding <code>*</code> and <code>|</code>, the remaining pattern follows the regular expression semantics.</li>
+      <li>The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
     </ul>
   </dd>
 </dl>
diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index 1aa44d3ab30ea..c688a99df3ae2 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -33,7 +33,7 @@ cannot be used with a partition specification.
 ### Syntax
 
 {% highlight sql %}
-SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE 'identifier_with_wildcards'
+SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE regex_pattern
     [ partition_spec ]
 {% endhighlight %}
 
@@ -44,15 +44,15 @@ SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE 'identifier_with_wildcards'
   <dd>
     Specifies database name. If not provided, will use the current database.
   </dd>
-  <dt><code><em>LIKE string_pattern</em></code></dt>
+  <dt><code><em>regex_pattern</em></code></dt>
   <dd>
     Specifies the regular expression pattern that is used to filter out unwanted tables.
-    <ul> 
-       <li> Except for `*` and `|` character, the pattern works like a regex.</li>
-       <li> `*` alone matches 0 or more characters and `|` is used to separate multiple different regexes,
+    <ul>
+       <li> Except for <code>*</code> and <code>|</code> character, the pattern works like a regular expression.</li>
+       <li> <code>*</code> alone matches 0 or more characters and <code>|</code> is used to separate multiple different regular expressions,
              any of which can match. </li>
-       <li> The leading and trailing blanks are trimmed in the input pattern before processing.</li>
-    </ul> 
+       <li> The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
+    </ul>
   </dd>
   <dt><code><em>partition_spec</em></code></dt>
   <dd>
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index 0b7062ec8eff7..f4b3dff850d44 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -29,7 +29,7 @@ current database.
 ### Syntax
 
 {% highlight sql %}
-SHOW TABLES [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
+SHOW TABLES [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
 {% endhighlight %}
 
 ### Parameters
@@ -39,14 +39,14 @@ SHOW TABLES [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
   <dd>
      Specifies the database name from which tables are listed.
   </dd>
-  <dt><code><em>LIKE regex_pattern</em></code></dt>
+  <dt><code><em>regex_pattern</em></code></dt>
   <dd>
      Specifies the regular expression pattern that is used to filter out unwanted tables. 
      <ul> 
-          <li> Except for `*` and `|` character, the pattern works like a regex.</li>
-          <li> `*` alone matches 0 or more characters and `|` is used to separate multiple different regexes,
+          <li> Except for <code>*</code> and <code>|</code> character, the pattern works like a regular expression.</li>
+          <li> <code>*</code> alone matches 0 or more characters and <code>|</code> is used to separate multiple different regular expressions,
            any of which can match. </li>
-          <li> The leading and trailing blanks are trimmed in the input pattern before processing.</li>
+          <li> The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
      </ul>
     
   </dd>
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
index aec3716c2889f..0d9210b024e3a 100644
--- a/docs/sql-ref-syntax-aux-show-views.md
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -30,7 +30,7 @@ regardless of a given database.
 
 ### Syntax
 {% highlight sql %}
-SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
+SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
 {% endhighlight %}
 
 ### Parameters
@@ -39,14 +39,14 @@ SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE 'regex_pattern' ]
   <dd>
      Specifies the database name from which views are listed.
   </dd>
-  <dt><code><em>LIKE regex_pattern</em></code></dt>
+  <dt><code><em>regex_pattern</em></code></dt>
   <dd>
-     Specifies the regular expression pattern that is used to filter out unwanted views. 
+     Specifies the regular expression pattern that is used to filter out unwanted views.
      <ul> 
-          <li> Except for `*` and `|` character, the pattern works like a regex.</li>
-          <li> `*` alone matches 0 or more characters and `|` is used to separate multiple different regexes,
+          <li>Except for <code>*</code> and <code>|</code> character, the pattern works like a regular expression.</li>
+          <li><code>*</code> alone matches 0 or more characters and <code>|</code> is used to separate multiple different regular expressions,
            any of which can match. </li>
-          <li> The leading and trailing blanks are trimmed in the input pattern before processing.</li>
+          <li>The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
      </ul>
   </dd>
 </dl>
diff --git a/docs/sql-ref-syntax-qry-explain.md b/docs/sql-ref-syntax-qry-explain.md
index 6a7c2ace8223b..7ad297d9c76dd 100644
--- a/docs/sql-ref-syntax-qry-explain.md
+++ b/docs/sql-ref-syntax-qry-explain.md
@@ -27,7 +27,7 @@ By default, this clause provides information about a physical plan only.
 ### Syntax
 
 {% highlight sql %}
-EXPLAIN [EXTENDED | CODEGEN | COST | FORMATTED] statement
+EXPLAIN [ EXTENDED | CODEGEN | COST | FORMATTED ] statement
 {% endhighlight %}
 
 ### Parameters
diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md
new file mode 100644
index 0000000000000..408673c532ddd
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-like.md
@@ -0,0 +1,120 @@
+---
+layout: global
+title: LIKE Predicate
+displayTitle: LIKE Predicate
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+A LIKE predicate is used to search for a specific pattern.
+
+### Syntax
+
+{% highlight sql %}
+[ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | RLIKE regex_pattern }
+{% endhighlight %}
+
+### Parameters
+
+<dl>
+  <dt><code><em>search_pattern</em></code></dt>
+  <dd>
+    Specifies a string pattern to be searched by the <code>LIKE</code> clause. It can contain special pattern-matching characters:
+    <ul>
+      <li><code>%</code></li> matches zero or more characters.
+      <li><code>_</code></li> matches exactly one character.
+    </ul>
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>esc_char</em></code></dt>
+  <dd>
+    Specifies the escape character. The default escape character is <code>\</code>.
+  </dd>
+</dl>
+<dl>
+  <dt><code><em>regex_pattern</em></code></dt>
+  <dd>
+    Specifies a regular expression search pattern to be searched by the <code>RLIKE</code> clause.
+  </dd>
+</dl>
+
+### Examples
+
+{% highlight sql %}
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'John', 30),
+    (200, 'Mary', NULL),
+    (300, 'Mike', 80),
+    (400, 'Dan',  50),
+    (500, 'Evan_w', 16);
+
+SELECT * FROM person WHERE name LIKE 'M%';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name LIKE 'M_ry';
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name NOT LIKE 'M_ry';
++---+------+---+
+| id|  name|age|
++---+------+---+
+|500|Evan_W| 16|
+|300|  Mike| 80|
+|100|  John| 30|
+|400|   Dan| 50|
++---+------+---+
+
+SELECT * FROM person WHERE name RLIKE '[MD]';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|400| Dan|  50|
+|200|Mary|null|
++---+----+----+
+
+SELECT * FROM person WHERE name LIKE '%\_%';
++---+------+---+
+| id|  name|age|
++---+------+---+
+|500|Evan_W| 16|
++---+------+---+
+
+SELECT * FROM person WHERE name LIKE '%$_%' ESCAPE '$';
++---+------+---+
+| id|  name|age|
++---+------+---+
+|500|Evan_W| 16|
++---+------+---+
+{% endhighlight %}
+
+### Related Statements
+
+ * [SELECT](sql-ref-syntax-qry-select.html)
+ * [WHERE Clause](sql-ref-syntax-qry-select-where.html)

From 0058212dc60129357ad90e2c04bcb22094a9dd84 Mon Sep 17 00:00:00 2001
From: Baohe Zhang <baohe.zhang@verizonmedia.com>
Date: Tue, 28 Apr 2020 17:27:13 -0700
Subject: [PATCH 0594/1280] [SPARK-31584][WEBUI] Fix NullPointerException when
 parsing event log with InMemoryStore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
https://github.com/apache/spark/pull/27716 introduced parent index for InMemoryStore. When the method "deleteParentIndex(Object key)" in InMemoryStore.java is called and the key is not contained in "NaturalKeys v",  A java.lang.NullPointerException will be thrown. This patch fixed the issue by updating the if condition.

### Why are the changes needed?
Fixed a minor bug.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Added a unit test for deleteParentIndex.

Closes #28378 from baohe-zhang/SPARK-31584.

Authored-by: Baohe Zhang <baohe.zhang@verizonmedia.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 3808014a2fa763866ad55776ccb397de4d13afcc)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../spark/util/kvstore/InMemoryStore.java     |  2 +-
 .../spark/util/kvstore/CustomType2.java       | 50 +++++++++++++++++++
 .../util/kvstore/InMemoryStoreSuite.java      | 42 ++++++++++++++++
 3 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java

diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index e929c6c45ebd3..42e090bc83ed1 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -293,7 +293,7 @@ public boolean delete(Object key, T value) {
     private void deleteParentIndex(Object key) {
       if (hasNaturalParentIndex) {
         for (NaturalKeys v : parentToChildrenMap.values()) {
-          if (v.remove(asKey(key))) {
+          if (v.remove(asKey(key)) != null) {
             // `v` can be empty after removing the natural key and we can remove it from
             // `parentToChildrenMap`. However, `parentToChildrenMap` is a ConcurrentMap and such
             // checking and deleting can be slow.
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
new file mode 100644
index 0000000000000..3bb66bb3ec700
--- /dev/null
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/CustomType2.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util.kvstore;
+
+public class CustomType2 {
+
+  @KVIndex(parent = "parentId")
+  public String key;
+
+  @KVIndex("id")
+  public String id;
+
+  @KVIndex("parentId")
+  public String parentId;
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof CustomType2) {
+      CustomType2 other = (CustomType2) o;
+      return id.equals(other.id) && parentId.equals(other.parentId);
+    }
+    return false;
+  }
+
+  @Override
+  public int hashCode() {
+    return id.hashCode() ^ parentId.hashCode();
+  }
+
+  @Override
+  public String toString() {
+    return "CustomType2[key=" + key + ",id=" + id + ",parentId=" + parentId;
+  }
+
+}
diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
index da5267669d5f1..35656fb12238a 100644
--- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
+++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/InMemoryStoreSuite.java
@@ -210,4 +210,46 @@ public void testBasicIteration() throws Exception {
     assertFalse(store.view(t1.getClass()).first(t2.id).skip(1).iterator().hasNext());
   }
 
+  @Test
+  public void testDeleteParentIndex() throws Exception {
+    KVStore store = new InMemoryStore();
+
+    CustomType2 t1 = new CustomType2();
+    t1.key = "key1";
+    t1.id = "id1";
+    t1.parentId = "parentId1";
+    store.write(t1);
+
+    CustomType2 t2 = new CustomType2();
+    t2.key = "key2";
+    t2.id = "id2";
+    t2.parentId = "parentId1";
+    store.write(t2);
+
+    CustomType2 t3 = new CustomType2();
+    t3.key = "key3";
+    t3.id = "id1";
+    t3.parentId = "parentId2";
+    store.write(t3);
+
+    CustomType2 t4 = new CustomType2();
+    t4.key = "key4";
+    t4.id = "id2";
+    t4.parentId = "parentId2";
+    store.write(t4);
+
+    assertEquals(4, store.count(CustomType2.class));
+
+    store.delete(t1.getClass(), t1.key);
+    assertEquals(3, store.count(CustomType2.class));
+
+    store.delete(t2.getClass(), t2.key);
+    assertEquals(2, store.count(CustomType2.class));
+
+    store.delete(t3.getClass(), t3.key);
+    assertEquals(1, store.count(CustomType2.class));
+
+    store.delete(t4.getClass(), t4.key);
+    assertEquals(0, store.count(CustomType2.class));
+  }
 }

From cf6a3de217d4fae7d46ab024b0a13ccdc9cde907 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 29 Apr 2020 15:34:45 +0900
Subject: [PATCH 0595/1280] [SPARK-31596][SQL][DOCS] Generate SQL
 Configurations from hive module to configuration doc

### What changes were proposed in this pull request?

This PR adds `-Phive` profile to the pre-build phase to build the hive module to dev classpath.
Then reflect the HiveUtils object to dump all configurations in the class.

### Why are the changes needed?

supply SQL configurations from hive module to doc

### Does this PR introduce any user-facing change?

NO

### How was this patch tested?

passing Jenkins
 add verified locally

![image](https://user-images.githubusercontent.com/8326978/80492333-6fae1200-8996-11ea-99fd-595ee18c67e5.png)

Closes #28394 from yaooqinn/SPARK-31596.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 295d866969e01e70835210880ed27022d837a8f4)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_plugins/copy_api_dirs.rb                     |  4 ++--
 .../spark/sql/api/python/PythonSQLUtils.scala      | 14 +++++++++++++-
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index f95e4e2f97792..8e2a06e4bc9a8 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -157,8 +157,8 @@
     curr_dir = pwd
     cd("..")
 
-    puts "Running 'build/sbt clean package' from " + pwd + "; this may take a few minutes..."
-    system("build/sbt clean package") || raise("SQL doc generation failed")
+    puts "Running 'build/sbt clean package -Phive' from " + pwd + "; this may take a few minutes..."
+    system("build/sbt clean package -Phive") || raise("SQL doc generation failed")
 
     puts "Moving back into docs dir."
     cd("docs")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
index 89bb4f362de74..38254609e8f67 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala
@@ -20,10 +20,14 @@ package org.apache.spark.sql.api.python
 import java.io.InputStream
 import java.nio.channels.Channels
 
+import scala.util.control.NonFatal
+
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.api.python.PythonRDDServer
+import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.catalyst.ScalaReflection
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
@@ -32,7 +36,7 @@ import org.apache.spark.sql.execution.arrow.ArrowConverters
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.DataType
 
-private[sql] object PythonSQLUtils {
+private[sql] object PythonSQLUtils extends Logging {
   def parseDataType(typeText: String): DataType = CatalystSqlParser.parseDataType(typeText)
 
   // This is needed when generating SQL documentation for built-in functions.
@@ -44,6 +48,14 @@ private[sql] object PythonSQLUtils {
     val conf = new SQLConf()
     // Force to build static SQL configurations
     StaticSQLConf
+    // Force to build SQL configurations from Hive module
+    try {
+      val symbol = ScalaReflection.mirror.staticModule("org.apache.spark.sql.hive.HiveUtils")
+      ScalaReflection.mirror.reflectModule(symbol).instance
+    } catch {
+      case NonFatal(e) =>
+        logWarning("Cannot generated sql configurations from hive module", e)
+    }
     conf.getAllDefinedConfs
   }
 

From 7686d7f3ee340537ddcf31596755acd010f9b56f Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 29 Apr 2020 06:44:22 +0000
Subject: [PATCH 0596/1280] [SPARK-31563][SQL][FOLLOWUP] Create literals
 directly from Catalyst's internal value in InSet.sql

### What changes were proposed in this pull request?
In the PR, I propose to simplify the code of `InSet.sql` and create `Literal` instances directly from Catalyst's internal values by using the default `Literal` constructor.

### Why are the changes needed?
This simplifies code and avoids unnecessary conversions to external types.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By existing test `SPARK-31563: sql of InSet for UTF8String collection` in `ColumnExpressionSuite`.

Closes #28399 from MaxGekk/fix-InSet-sql-followup.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 86761861c28e1c854b4f78f1c078591b11b7daf3)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/catalyst/expressions/predicates.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index ac492cf227301..2c4f41f98ac20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -521,7 +521,7 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
   override def sql: String = {
     val valueSQL = child.sql
     val listSQL = hset.toSeq
-      .map(elem => Literal(convertToScala(elem, child.dataType)).sql)
+      .map(elem => Literal(elem, child.dataType).sql)
       .mkString(", ")
     s"($valueSQL IN ($listSQL))"
   }

From 940b1d860310ec7fe5484e73bb8a9d597af7a12a Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Wed, 29 Apr 2020 06:48:46 +0000
Subject: [PATCH 0597/1280] [SPARK-29492][SQL] Reset HiveSession's SessionState
 conf's ClassLoader when sync mode

### What changes were proposed in this pull request?
Run sql in spark thrift server, each session 's thrift server about method will be called in one thread, but when running query statement,  we have two mode:
 1. sync
 2. async
 https://github.com/apache/spark/blob/5a482e72091c8db940408905e8c044f7f5d7814f/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala#L205-L238

In sync mode, we just submit query in current session's corresponding thread and wait Spark to running query and return result,  and the query method will always wait for query return.
In async mode, in SparkExecuteStatementOperation, we will submit query in a backend thread pool, and update operation state,  after submitted to backend thread poll, ExecuteStatement method will return a OperationHandle to client side, and client side will request operation status continuously. after backend thread running sql and return , it will update corresponding  operation status, when client got operation status is final status, it will got error or start fetching result of this operation.

When we use pyhive connect to SparkThriftServer, it will run statement in sync mode.
When we query data of hive table , it will check serde class in HiveTableScanExec#addColumnMetadataToConf

https://github.com/apache/spark/blob/5a482e72091c8db940408905e8c044f7f5d7814f/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveTableScanExec.scala#L123

```
  public Class<? extends Deserializer> getDeserializerClass() {
    try {
      return Class.forName(this.getSerdeClassName(), true, Utilities.getSessionSpecifiedClassLoader());
    } catch (ClassNotFoundException var2) {
      throw new RuntimeException(var2);
    }
  }

 public static ClassLoader getSessionSpecifiedClassLoader() {
    SessionState state = SessionState.get();
    if (state != null && state.getConf() != null) {
      ClassLoader sessionCL = state.getConf().getClassLoader();
      if (sessionCL != null) {
        if (LOG.isTraceEnabled()) {
          LOG.trace("Use session specified class loader");
        }

        return sessionCL;
      } else {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Session specified class loader not found, use thread based class loader");
        }

        return JavaUtils.getClassLoader();
      }
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Hive Conf not found or Session not initiated, use thread based class loader instead");
      }

      return JavaUtils.getClassLoader();
    }
  }
```
Since we run statement in sync mode, it will use HiveSession's SessionState,  and use it's conf's classLoader. then error happened.
```
Current operation state RUNNING_STATE,
java.lang.RuntimeException: java.lang.ClassNotFoundException:
xxx.xxx.xxxJsonSerDe
  at org.apache.hadoop.hive.ql.plan.TableDesc.getDeserializerClass(TableDesc.java:74)
  at org.apache.spark.sql.hive.execution.HiveTableScanExec.addColumnMetadataToConf(HiveTableScanExec.scala:123)
  at org.apache.spark.sql.hive.execution.HiveTableScanExec.hadoopConf$lzycompute(HiveTableScanExec.scala:101)
	  at org.apache.spark.sql.hive.execution.HiveTableScanExec.hadoopConf(HiveTableScanExec.scala:98)
	  at org.apache.spark.sql.hive.execution.HiveTableScanExec.org$apache$spark$sql$hive$execution$HiveTableScanExec$$hadoopReader$lzycompute(HiveTableScanExec.scala:110)
  at org.apache.spark.sql.hive.execution.HiveTableScanExec.org$apache$spark$sql$hive$execution$HiveTableScanExec$$hadoopReader(HiveTableScanExec.scala:105)
	  at org.apache.spark.sql.hive.execution.HiveTableScanExec$$anonfun$11.apply(HiveTableScanExec.scala:192)
	  at org.apache.spark.sql.hive.execution.HiveTableScanExec$$anonfun$11.apply(HiveTableScanExec.scala:192)
```
We should reset it when we start run sql in sync mode.
### Why are the changes needed?
Fix bug

### Does this PR introduce any user-facing change?
NO

### How was this patch tested?
UT

Closes #26141 from AngersZhuuuu/add_jar_in_sync_mode.

Lead-authored-by: angerszhu <angers.zhu@gmail.com>
Co-authored-by: AngersZhuuuu <angers.zhu@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6bc8d84130ae2dcbcd8f5c3361577b19f2eb5d8b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../SparkExecuteStatementOperation.scala      |  5 +++
 .../HiveThriftServer2Suites.scala             | 40 +++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index cf0e5ebf3a2b1..d14d70f7d3d83 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -274,6 +274,11 @@ private[hive] class SparkExecuteStatementOperation(
       val executionHiveClassLoader = sqlContext.sharedState.jarClassLoader
       Thread.currentThread().setContextClassLoader(executionHiveClassLoader)
 
+      // Always set the session state classloader to `executionHiveClassLoader` even for sync mode
+      if (!runInBackground) {
+        parentSession.getSessionState.getConf.setClassLoader(executionHiveClassLoader)
+      }
+
       sqlContext.sparkContext.setJobGroup(statementId, statement)
       result = sqlContext.sql(statement)
       logDebug(result.queryExecution.toString())
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 28f67cbbf7b46..639dc4d13f673 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -771,6 +771,46 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       client.closeSession(sessionHandle)
     }
   }
+
+  test("SPARK-29492: use add jar in sync mode") {
+    withCLIServiceClient { client =>
+      val user = System.getProperty("user.name")
+      val sessionHandle = client.openSession(user, "")
+      withJdbcStatement("smallKV", "addJar") { statement =>
+        val confOverlay = new java.util.HashMap[java.lang.String, java.lang.String]
+        val jarFile = HiveTestJars.getHiveHcatalogCoreJar().getCanonicalPath
+
+        Seq(s"ADD JAR $jarFile",
+          "CREATE TABLE smallKV(key INT, val STRING) USING hive",
+          s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO TABLE smallKV")
+          .foreach(query => client.executeStatement(sessionHandle, query, confOverlay))
+
+        client.executeStatement(sessionHandle,
+          """CREATE TABLE addJar(key string)
+            |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+          """.stripMargin, confOverlay)
+
+        client.executeStatement(sessionHandle,
+          "INSERT INTO TABLE addJar SELECT 'k1' as key FROM smallKV limit 1", confOverlay)
+
+        val operationHandle = client.executeStatement(
+          sessionHandle,
+          "SELECT key FROM addJar",
+          confOverlay)
+
+        // Fetch result first time
+        assertResult(1, "Fetching result first time from next row") {
+
+          val rows_next = client.fetchResults(
+            operationHandle,
+            FetchOrientation.FETCH_NEXT,
+            1000,
+            FetchType.QUERY_OUTPUT)
+          rows_next.numRows()
+        }
+      }
+    }
+  }
 }
 
 class SingleSessionSuite extends HiveThriftJdbcTest {

From d771d79d0374a167630ebc3985035cfb39e3c990 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 29 Apr 2020 06:56:33 +0000
Subject: [PATCH 0598/1280] [SPARK-31597][SQL] extracting day from intervals
 should be interval.days + days in interval.microsecond

### What changes were proposed in this pull request?

With suggestion from cloud-fan https://github.com/apache/spark/pull/28222#issuecomment-620586933

I Checked with both Presto and PostgresSQL, one is implemented intervals with ANSI style year-month/day-time, and the other is mixed and Non-ANSI. They both add the exceeded days in interval time part to the total days of the operation which extracts day from interval values.

```sql

presto> SELECT EXTRACT(DAY FROM (cast('2020-01-15 00:00:00' as timestamp) - cast('2020-01-01 00:00:00' as timestamp)));
_col0
-------
14
(1 row)

Query 20200428_135239_00000_ahn7x, FINISHED, 1 node
Splits: 17 total, 17 done (100.00%)
0:01 [0 rows, 0B] [0 rows/s, 0B/s]

presto> SELECT EXTRACT(DAY FROM (cast('2020-01-15 00:00:00' as timestamp) - cast('2020-01-01 00:00:01' as timestamp)));
_col0
-------
13
(1 row)

Query 20200428_135246_00001_ahn7x, FINISHED, 1 node
Splits: 17 total, 17 done (100.00%)
0:00 [0 rows, 0B] [0 rows/s, 0B/s]

presto>

```

```sql

postgres=# SELECT EXTRACT(DAY FROM (cast('2020-01-15 00:00:00' as timestamp) - cast('2020-01-01 00:00:00' as timestamp)));
date_part
-----------
14
(1 row)

postgres=# SELECT EXTRACT(DAY FROM (cast('2020-01-15 00:00:00' as timestamp) - cast('2020-01-01 00:00:01' as timestamp)));
date_part
-----------
13

```

```
spark-sql> SELECT EXTRACT(DAY FROM (cast('2020-01-15 00:00:00' as timestamp) - cast('2020-01-01 00:00:01' as timestamp)));
0
spark-sql> SELECT EXTRACT(DAY FROM (cast('2020-01-15 00:00:00' as timestamp) - cast('2020-01-01 00:00:00' as timestamp)));
0
```

In ANSI standard, the day is exact 24 hours, so we don't need to worry about the conceptual day for interval extraction. The meaning of the conceptual day only takes effect when we add it to a zoned timestamp value.

### Why are the changes needed?

Both satisfy the ANSI standard and common use cases in modern SQL platforms

### Does this PR introduce any user-facing change?

No, it new in 3.0
### How was this patch tested?

add more uts

Closes #28396 from yaooqinn/SPARK-31597.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ea525fe8c0cc6336a7ba8d98bada3198795f8aed)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/IntervalUtils.scala     |  6 ++--
 .../IntervalExpressionsSuite.scala            |  3 ++
 .../sql-tests/results/extract.sql.out         | 32 +++++++++----------
 3 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 3afa0c7c089aa..833942d7b0104 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.math.BigDecimal
 import java.util.concurrent.TimeUnit
 
 import scala.util.control.NonFatal
@@ -55,11 +54,12 @@ object IntervalUtils {
   }
 
   def getDays(interval: CalendarInterval): Int = {
-    interval.days
+    val daysInMicroseconds = (interval.microseconds / MICROS_PER_DAY).toInt
+    Math.addExact(interval.days, daysInMicroseconds)
   }
 
   def getHours(interval: CalendarInterval): Long = {
-    interval.microseconds / MICROS_PER_HOUR
+    (interval.microseconds % MICROS_PER_DAY) / MICROS_PER_HOUR
   }
 
   def getMinutes(interval: CalendarInterval): Byte = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index a2ee1c6ef2793..8c972a9917043 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -68,6 +68,7 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Years and months must not be taken into account
     checkEvaluation(ExtractIntervalDays("100 year 10 months 5 days"), 5)
     checkEvaluation(ExtractIntervalDays(largeInterval), 31)
+    checkEvaluation(ExtractIntervalDays("25 hours"), 1)
   }
 
   test("hours") {
@@ -81,6 +82,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     // Minutes should be taken into account
     checkEvaluation(ExtractIntervalHours("10 hours 100 minutes"), 11L)
     checkEvaluation(ExtractIntervalHours(largeInterval), 11L)
+    checkEvaluation(ExtractIntervalHours("25 hours"), 1L)
+
   }
 
   test("minutes") {
diff --git a/sql/core/src/test/resources/sql-tests/results/extract.sql.out b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
index aea09e4b90f4e..29cbefdb38541 100644
--- a/sql/core/src/test/resources/sql-tests/results/extract.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/extract.sql.out
@@ -135,7 +135,7 @@ select extract(day from c), extract(day from i) from t
 -- !query schema
 struct<extract('day' FROM t.`c`):int,extract('day' FROM t.`i`):int>
 -- !query output
-6	30
+6	31
 
 
 -- !query
@@ -143,7 +143,7 @@ select extract(d from c), extract(d from i) from t
 -- !query schema
 struct<extract('d' FROM t.`c`):int,extract('d' FROM t.`i`):int>
 -- !query output
-6	30
+6	31
 
 
 -- !query
@@ -151,7 +151,7 @@ select extract(days from c), extract(days from i) from t
 -- !query schema
 struct<extract('days' FROM t.`c`):int,extract('days' FROM t.`i`):int>
 -- !query output
-6	30
+6	31
 
 
 -- !query
@@ -199,7 +199,7 @@ select extract(hour from c), extract(hour from i) from t
 -- !query schema
 struct<extract('hour' FROM t.`c`):int,extract('hour' FROM t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -207,7 +207,7 @@ select extract(h from c), extract(h from i) from t
 -- !query schema
 struct<extract('h' FROM t.`c`):int,extract('h' FROM t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -215,7 +215,7 @@ select extract(hours from c), extract(hours from i) from t
 -- !query schema
 struct<extract('hours' FROM t.`c`):int,extract('hours' FROM t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -223,7 +223,7 @@ select extract(hr from c), extract(hr from i) from t
 -- !query schema
 struct<extract('hr' FROM t.`c`):int,extract('hr' FROM t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -231,7 +231,7 @@ select extract(hrs from c), extract(hrs from i) from t
 -- !query schema
 struct<extract('hrs' FROM t.`c`):int,extract('hrs' FROM t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -457,7 +457,7 @@ select date_part('day', c), date_part('day', i) from t
 -- !query schema
 struct<date_part('day', t.`c`):int,date_part('day', t.`i`):int>
 -- !query output
-6	30
+6	31
 
 
 -- !query
@@ -465,7 +465,7 @@ select date_part('d', c), date_part('d', i) from t
 -- !query schema
 struct<date_part('d', t.`c`):int,date_part('d', t.`i`):int>
 -- !query output
-6	30
+6	31
 
 
 -- !query
@@ -473,7 +473,7 @@ select date_part('days', c), date_part('days', i) from t
 -- !query schema
 struct<date_part('days', t.`c`):int,date_part('days', t.`i`):int>
 -- !query output
-6	30
+6	31
 
 
 -- !query
@@ -521,7 +521,7 @@ select date_part('hour', c), date_part('hour', i) from t
 -- !query schema
 struct<date_part('hour', t.`c`):int,date_part('hour', t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -529,7 +529,7 @@ select date_part('h', c), date_part('h', i) from t
 -- !query schema
 struct<date_part('h', t.`c`):int,date_part('h', t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -537,7 +537,7 @@ select date_part('hours', c), date_part('hours', i) from t
 -- !query schema
 struct<date_part('hours', t.`c`):int,date_part('hours', t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -545,7 +545,7 @@ select date_part('hr', c), date_part('hr', i) from t
 -- !query schema
 struct<date_part('hr', t.`c`):int,date_part('hr', t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query
@@ -553,7 +553,7 @@ select date_part('hrs', c), date_part('hrs', i) from t
 -- !query schema
 struct<date_part('hrs', t.`c`):int,date_part('hrs', t.`i`):bigint>
 -- !query output
-7	40
+7	16
 
 
 -- !query

From dde2dd60414e1a1ccf0dd4fcd228fbed57b37b8f Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Wed, 29 Apr 2020 07:06:45 +0000
Subject: [PATCH 0599/1280] [SPARK-30282][SQL][FOLLOWUP] SHOW TBLPROPERTIES
 should support views

### What changes were proposed in this pull request?

This PR addresses two things:
- `SHOW TBLPROPERTIES` should supports view (a regression introduced by #26921)
- `SHOW TBLPROPERTIES` on a temporary view should return empty result (2.4 behavior instead of throwing `AnalysisException`.

### Why are the changes needed?

It's a bug.

### Does this PR introduce any user-facing change?

Yes, now `SHOW TBLPROPERTIES` works on views:
```
scala> sql("CREATE VIEW view TBLPROPERTIES('p1'='v1', 'p2'='v2') AS SELECT 1 AS c1")
scala> sql("SHOW TBLPROPERTIES view").show(truncate=false)
+---------------------------------+-------------+
|key                              |value        |
+---------------------------------+-------------+
|view.catalogAndNamespace.numParts|2            |
|view.query.out.col.0             |c1           |
|view.query.out.numCols           |1            |
|p2                               |v2           |
|view.catalogAndNamespace.part.0  |spark_catalog|
|p1                               |v1           |
|view.catalogAndNamespace.part.1  |default      |
+---------------------------------+-------------+
```
And for a temporary view:
```
scala> sql("CREATE TEMPORARY VIEW tview TBLPROPERTIES('p1'='v1', 'p2'='v2') AS SELECT 1 AS c1")
scala> sql("SHOW TBLPROPERTIES tview").show(truncate=false)
+---+-----+
|key|value|
+---+-----+
+---+-----+
```

### How was this patch tested?

Added tests.

Closes #28375 from imback82/show_tblproperties_followup.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 36803031e850b08d689df90d15c75e1a1eeb28a8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |   2 +-
 .../sql/catalyst/parser/AstBuilder.scala      |   2 +-
 .../sql/catalyst/parser/DDLParserSuite.scala  |   4 +-
 .../analysis/ResolveSessionCatalog.scala      |   3 +
 .../spark/sql/execution/command/tables.scala  |  23 ++--
 .../sql-tests/inputs/show-tblproperties.sql   |  26 ++++
 .../results/show-tblproperties.sql.out        | 114 ++++++++++++++++++
 .../sql/hive/execution/HiveCommandSuite.scala |  17 +--
 8 files changed, 162 insertions(+), 29 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 39619f60d9aba..e68198afe327b 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -53,7 +53,7 @@ license: |
  
   - In Spark 3.0, you can use `ADD FILE` to add file directories as well. Earlier you could add only single files using this command. To restore the behavior of earlier versions, set `spark.sql.legacy.addSingleFileInAddFile` to `true`.
 
-  - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`. Also, `SHOW TBLPROPERTIES` on a temporary view causes `AnalysisException`. In Spark version 2.4 and below, it returned an empty result.
+  - In Spark 3.0, `SHOW TBLPROPERTIES` throws `AnalysisException` if the table does not exist. In Spark version 2.4 and below, this scenario caused `NoSuchTableException`.
 
   - In Spark 3.0, `SHOW CREATE TABLE` always returns Spark DDL, even when the given table is a Hive SerDe table. For generating Hive DDL, use `SHOW CREATE TABLE AS SERDE` command instead.
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 2a28e4bdbb90e..024d7a451f200 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -3537,7 +3537,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
   override def visitShowTblProperties(
       ctx: ShowTblPropertiesContext): LogicalPlan = withOrigin(ctx) {
     ShowTableProperties(
-      UnresolvedTable(visitMultipartIdentifier(ctx.table)),
+      UnresolvedTableOrView(visitMultipartIdentifier(ctx.table)),
       Option(ctx.key).map(visitTablePropertyKey))
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 1bef63898085c..e05a51e2afab2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -1964,11 +1964,11 @@ class DDLParserSuite extends AnalysisTest {
   test("SHOW TBLPROPERTIES table") {
     comparePlans(
       parsePlan("SHOW TBLPROPERTIES a.b.c"),
-      ShowTableProperties(UnresolvedTable(Seq("a", "b", "c")), None))
+      ShowTableProperties(UnresolvedTableOrView(Seq("a", "b", "c")), None))
 
     comparePlans(
       parsePlan("SHOW TBLPROPERTIES a.b.c('propKey1')"),
-      ShowTableProperties(UnresolvedTable(Seq("a", "b", "c")), Some("propKey1")))
+      ShowTableProperties(UnresolvedTableOrView(Seq("a", "b", "c")), Some("propKey1")))
   }
 
   test("DESCRIBE FUNCTION") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index 58d74e041ecc6..bb31c132a66ff 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -538,6 +538,9 @@ class ResolveSessionCatalog(
     case ShowTableProperties(r: ResolvedTable, propertyKey) if isSessionCatalog(r.catalog) =>
       ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
 
+    case ShowTableProperties(r: ResolvedView, propertyKey) =>
+      ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
+
     case DescribeFunctionStatement(nameParts, extended) =>
       val functionIdent =
         parseSessionCatalogFunctionIdentifier(nameParts, "DESCRIBE FUNCTION")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 2c9ca36c7fcdc..fc8cc11bb1067 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -918,15 +918,20 @@ case class ShowTablePropertiesCommand(table: TableIdentifier, propertyKey: Optio
   }
 
   override def run(sparkSession: SparkSession): Seq[Row] = {
-    val catalogTable = sparkSession.sessionState.catalog.getTableMetadata(table)
-    propertyKey match {
-      case Some(p) =>
-        val propValue = catalogTable
-          .properties
-          .getOrElse(p, s"Table ${catalogTable.qualifiedName} does not have property: $p")
-        Seq(Row(propValue))
-      case None =>
-        catalogTable.properties.map(p => Row(p._1, p._2)).toSeq
+    val catalog = sparkSession.sessionState.catalog
+    if (catalog.isTemporaryTable(table)) {
+      Seq.empty[Row]
+    } else {
+      val catalogTable = catalog.getTableMetadata(table)
+      propertyKey match {
+        case Some(p) =>
+          val propValue = catalogTable
+            .properties
+            .getOrElse(p, s"Table ${catalogTable.qualifiedName} does not have property: $p")
+          Seq(Row(propValue))
+        case None =>
+          catalogTable.properties.map(p => Row(p._1, p._2)).toSeq
+      }
     }
   }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
new file mode 100644
index 0000000000000..2861b2b43a113
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/show-tblproperties.sql
@@ -0,0 +1,26 @@
+-- create a table with properties
+CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
+TBLPROPERTIES('p1'='v1', 'p2'='v2');
+
+SHOW TBLPROPERTIES tbl;
+SHOW TBLPROPERTIES tbl("p1");
+SHOW TBLPROPERTIES tbl("p3");
+
+DROP TABLE tbl;
+
+-- create a view with properties
+CREATE VIEW view TBLPROPERTIES('p1'='v1', 'p2'='v2') AS SELECT 1 AS c1;
+
+SHOW TBLPROPERTIES view;
+SHOW TBLPROPERTIES view("p1");
+SHOW TBLPROPERTIES view("p3");
+
+DROP VIEW view;
+
+-- create a temporary view with properties
+CREATE TEMPORARY VIEW tv TBLPROPERTIES('p1'='v1') AS SELECT 1 AS c1;
+
+-- Properties for a temporary view should be empty
+SHOW TBLPROPERTIES tv;
+
+DROP VIEW tv;
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
new file mode 100644
index 0000000000000..6984b34c365ec
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/show-tblproperties.sql.out
@@ -0,0 +1,114 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+CREATE TABLE tbl (a INT, b STRING, c INT) USING parquet
+TBLPROPERTIES('p1'='v1', 'p2'='v2')
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW TBLPROPERTIES tbl
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+p1	v1
+p2	v2
+
+
+-- !query
+SHOW TBLPROPERTIES tbl("p1")
+-- !query schema
+struct<value:string>
+-- !query output
+v1
+
+
+-- !query
+SHOW TBLPROPERTIES tbl("p3")
+-- !query schema
+struct<value:string>
+-- !query output
+Table default.tbl does not have property: p3
+
+
+-- !query
+DROP TABLE tbl
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE VIEW view TBLPROPERTIES('p1'='v1', 'p2'='v2') AS SELECT 1 AS c1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW TBLPROPERTIES view
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+p1	v1
+p2	v2
+view.catalogAndNamespace.numParts	2
+view.catalogAndNamespace.part.0	spark_catalog
+view.catalogAndNamespace.part.1	default
+view.query.out.col.0	c1
+view.query.out.numCols	1
+
+
+-- !query
+SHOW TBLPROPERTIES view("p1")
+-- !query schema
+struct<value:string>
+-- !query output
+v1
+
+
+-- !query
+SHOW TBLPROPERTIES view("p3")
+-- !query schema
+struct<value:string>
+-- !query output
+Table default.view does not have property: p3
+
+
+-- !query
+DROP VIEW view
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+CREATE TEMPORARY VIEW tv TBLPROPERTIES('p1'='v1') AS SELECT 1 AS c1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SHOW TBLPROPERTIES tv
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+
+
+
+-- !query
+DROP VIEW tv
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index bd85ad4e4944c..dcec8bf5c0cc6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -179,7 +179,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     val message = intercept[AnalysisException] {
       sql("SHOW TBLPROPERTIES badtable")
     }.getMessage
-    assert(message.contains("Table not found: badtable"))
+    assert(message.contains("Table or view not found: badtable"))
 
     // When key is not found, a row containing the error is returned.
     checkAnswer(
@@ -193,21 +193,6 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
     checkAnswer(sql("SHOW TBLPROPERTIES parquet_tab2('`prop2Key`')"), Row("prop2Val"))
   }
 
-  test("show tblproperties for spark temporary table - AnalysisException is thrown") {
-    withTempView("parquet_temp") {
-      sql(
-        """
-         |CREATE TEMPORARY VIEW parquet_temp (c1 INT, c2 STRING)
-         |USING org.apache.spark.sql.parquet.DefaultSource
-        """.stripMargin)
-
-      val message = intercept[AnalysisException] {
-        sql("SHOW TBLPROPERTIES parquet_temp")
-      }.getMessage
-      assert(message.contains("parquet_temp is a temp view not table"))
-    }
-  }
-
   Seq(true, false).foreach { local =>
     val loadQuery = if (local) "LOAD DATA LOCAL" else "LOAD DATA"
     test(loadQuery) {

From b4e63acc14fcdd48af0d0edfe44f482c901f9cf9 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 29 Apr 2020 07:19:34 +0000
Subject: [PATCH 0600/1280] [SPARK-31557][SQL][TESTS][FOLLOWUP] Check rebasing
 in all legacy formatters

### What changes were proposed in this pull request?
- Check all available legacy formats in the tests added by https://github.com/apache/spark/pull/28345
- Check dates rebasing in legacy parsers for only one direction either days -> string or string -> days.

### Why are the changes needed?
Round trip tests can hide issues in dates rebasing. For example, if we remove rebasing from legacy parsers (from `parse()` and `format()`) the tests will pass.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By running `DateFormatterSuite`.

Closes #28398 from MaxGekk/test-rebasing-in-legacy-date-formatter.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 73eac7565d7cd185d12a46637703ebff73649e40)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/util/DateFormatterSuite.scala   | 98 ++++++++++++-------
 1 file changed, 64 insertions(+), 34 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index 2df1d490b70bb..5e2b6a7c7fafe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -50,23 +50,29 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   test("roundtrip date -> days -> date") {
     LegacyBehaviorPolicy.values.foreach { parserPolicy =>
       withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
-        Seq(
-          "0050-01-01",
-          "0953-02-02",
-          "1423-03-08",
-          "1582-10-15",
-          "1969-12-31",
-          "1972-08-25",
-          "1975-09-26",
-          "2018-12-12",
-          "2038-01-01",
-          "5010-11-17").foreach { date =>
-          DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-            withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-              val formatter = DateFormatter(getZoneId(timeZone))
-              val days = formatter.parse(date)
-              val formatted = formatter.format(days)
-              assert(date === formatted)
+        LegacyDateFormats.values.foreach { legacyFormat =>
+          Seq(
+            "0050-01-01",
+            "0953-02-02",
+            "1423-03-08",
+            "1582-10-15",
+            "1969-12-31",
+            "1972-08-25",
+            "1975-09-26",
+            "2018-12-12",
+            "2038-01-01",
+            "5010-11-17").foreach { date =>
+            DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+                val formatter = DateFormatter(
+                  DateFormatter.defaultPattern,
+                  getZoneId(timeZone),
+                  DateFormatter.defaultLocale,
+                  legacyFormat)
+                val days = formatter.parse(date)
+                val formatted = formatter.format(days)
+                assert(date === formatted)
+              }
             }
           }
         }
@@ -77,23 +83,29 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   test("roundtrip days -> date -> days") {
     LegacyBehaviorPolicy.values.foreach { parserPolicy =>
       withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> parserPolicy.toString) {
-        Seq(
-          -701265,
-          -371419,
-          -199722,
-          -1,
-          0,
-          967,
-          2094,
-          17877,
-          24837,
-          1110657).foreach { days =>
-          DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
-            withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
-              val formatter = DateFormatter(getZoneId(timeZone))
-              val date = formatter.format(days)
-              val parsed = formatter.parse(date)
-              assert(days === parsed)
+        LegacyDateFormats.values.foreach { legacyFormat =>
+          Seq(
+            -701265,
+            -371419,
+            -199722,
+            -1,
+            0,
+            967,
+            2094,
+            17877,
+            24837,
+            1110657).foreach { days =>
+            DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+              withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+                val formatter = DateFormatter(
+                  DateFormatter.defaultPattern,
+                  getZoneId(timeZone),
+                  DateFormatter.defaultLocale,
+                  legacyFormat)
+                val date = formatter.format(days)
+                val parsed = formatter.parse(date)
+                assert(days === parsed)
+              }
             }
           }
         }
@@ -146,4 +158,22 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
     assert(formatter2.parse("BC 1234-02-22") === localDateToDays(LocalDate.of(-1233, 2, 22)))
     assert(formatter2.parse("AD 1234-02-22") === localDateToDays(LocalDate.of(1234, 2, 22)))
   }
+
+  test("SPARK-31557: rebasing in legacy formatters/parsers") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
+      LegacyDateFormats.values.foreach { legacyFormat =>
+        DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
+            val formatter = DateFormatter(
+              DateFormatter.defaultPattern,
+              getZoneId(timeZone),
+              DateFormatter.defaultLocale,
+              legacyFormat)
+            assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
+            assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
+          }
+        }
+      }
+    }
+  }
 }

From 06e1c0d788407af6101bf66d28cd44ca943d7a42 Mon Sep 17 00:00:00 2001
From: Michael Chirico <michael.chirico@grabtaxi.com>
Date: Wed, 29 Apr 2020 18:42:32 +0900
Subject: [PATCH 0601/1280] [SPARK-29339][R][FOLLOW-UP] Remove
 requireNamespace1 workaround for arrow

### What changes were proposed in this pull request?

`requireNamespace1` was used to get `SparkR` on CRAN while Suggesting `arrow` while `arrow` was not yet available on CRAN.

### Why are the changes needed?

Now `arrow` is on CRAN, we can properly use `requireNamespace` without triggering CRAN failures.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

AppVeyor will test, and CRAN check in Jenkins build.

Closes #28387 from MichaelChirico/r-require-arrow.

Authored-by: Michael Chirico <michael.chirico@grabtaxi.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 226301a6bc9e675f00e0c66ae31bc2d297e3649f)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/DataFrame.R | 15 +++++----------
 R/pkg/R/types.R     |  3 +--
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index f67f5fd9d45fb..09e831814b893 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1226,12 +1226,7 @@ setMethod("collect",
               # empty data.frame with 0 columns and 0 rows
               data.frame()
             } else if (useArrow) {
-              requireNamespace1 <- requireNamespace
-              if (requireNamespace1("arrow", quietly = TRUE)) {
-                read_arrow <- get("read_arrow", envir = asNamespace("arrow"), inherits = FALSE)
-                # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-                useAsTibble <- exists("as_tibble", envir = asNamespace("arrow"))
-
+              if (requireNamespace("arrow", quietly = TRUE)) {
                 portAuth <- callJMethod(x@sdf, "collectAsArrowToR")
                 port <- portAuth[[1]]
                 authSecret <- portAuth[[2]]
@@ -1239,10 +1234,10 @@ setMethod("collect",
                   port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
                 output <- tryCatch({
                   doServerAuth(conn, authSecret)
-                  arrowTable <- read_arrow(readRaw(conn))
-                  if (useAsTibble) {
-                    as_tibble <- get("as_tibble", envir = asNamespace("arrow"))
-                    as.data.frame(as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
+                  arrowTable <- arrow::read_arrow(readRaw(conn))
+                  # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
+                  if (exists("as_tibble", envir = asNamespace("arrow"))) {
+                    as.data.frame(arrow::as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
                   } else {
                     as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
                   }
diff --git a/R/pkg/R/types.R b/R/pkg/R/types.R
index 5b41f5963e2a3..5d48a9eee2799 100644
--- a/R/pkg/R/types.R
+++ b/R/pkg/R/types.R
@@ -88,8 +88,7 @@ specialtypeshandle <- function(type) {
 checkSchemaInArrow <- function(schema) {
   stopifnot(inherits(schema, "structType"))
 
-  requireNamespace1 <- requireNamespace
-  if (!requireNamespace1("arrow", quietly = TRUE)) {
+  if (!requireNamespace("arrow", quietly = TRUE)) {
     stop("'arrow' package should be installed.")
   }
 

From 9ccf828ddbad1af118320c458c4e1054e7e32bfa Mon Sep 17 00:00:00 2001
From: DB Tsai <d_tsai@apple.com>
Date: Wed, 29 Apr 2020 21:10:40 +0000
Subject: [PATCH 0602/1280] [SPARK-31582][YARN] Being able to not populate
 Hadoop classpath

### What changes were proposed in this pull request?
We are adding a new Spark Yarn configuration, `spark.yarn.populateHadoopClasspath` to not populate Hadoop classpath from `yarn.application.classpath` and `mapreduce.application.classpath`.

### Why are the changes needed?
Spark Yarn client populates extra Hadoop classpath from `yarn.application.classpath` and `mapreduce.application.classpath` when a job is submitted to a Yarn Hadoop cluster.

However, for `with-hadoop` Spark build that embeds Hadoop runtime, it can cause jar conflicts because Spark distribution can contain different version of Hadoop jars.

One case we have is when a user uses an Apache Spark distribution with its-own embedded hadoop, and submits a job to Cloudera or Hortonworks Yarn clusters, because of two different incompatible Hadoop jars in the classpath, it runs into errors.

By not populating the Hadoop classpath from the clusters can address this issue.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
An UT is added, but very hard to add a new integration test since this requires using different incompatible versions of Hadoop.

We also manually tested this PR, and we are able to submit a Spark job using Spark distribution built with Apache Hadoop 2.10 to CDH 5.6 without populating CDH classpath.

Closes #28376 from dbtsai/yarn-classpath.

Authored-by: DB Tsai <d_tsai@apple.com>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit ecfee82fda5f0403024ff64f16bc767b8d1e3e3d)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 docs/running-on-yarn.md                       | 11 +++++++++++
 .../org/apache/spark/deploy/yarn/Client.scala |  5 ++++-
 .../org/apache/spark/deploy/yarn/config.scala |  9 +++++++++
 .../spark/deploy/yarn/ClientSuite.scala       | 19 +++++++++++++++++++
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index eec73e8ff67e7..166fb874dd9c1 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -385,6 +385,17 @@ To use a custom metrics.properties for the application master and executors, upd
   </td>
   <td>1.4.0</td>
 </tr>
+<tr>
+  <td><code>spark.yarn.populateHadoopClasspath</code></td>
+  <td>true</td>
+  <td>
+    Whether to populate Hadoop classpath from <code>yarn.application.classpath</code> and
+    <code>mapreduce.application.classpath</code> Note that if this is set to <code>false</code>, 
+    it requires a <code>with-Hadoop</code> Spark distribution that bundles Hadoop runtime or
+    user has to provide a Hadoop installation separately.
+  </td>
+  <td>2.4.6</td>
+</tr>
 <tr>
   <td><code>spark.yarn.maxAppAttempts</code></td>
   <td><code>yarn.resourcemanager.am.max-attempts</code> in YARN</td>
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 696afaacb0e79..6da6a8daad2b0 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -1353,7 +1353,10 @@ private object Client extends Logging {
       }
     }
 
-    populateHadoopClasspath(conf, env)
+    if (sparkConf.get(POPULATE_HADOOP_CLASSPATH)) {
+      populateHadoopClasspath(conf, env)
+    }
+
     sys.env.get(ENV_DIST_CLASSPATH).foreach { cp =>
       addClasspathEntry(getClusterPath(sparkConf, cp), env)
     }
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
index b3a3570f7107a..3797491bb2e92 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -70,6 +70,15 @@ package object config {
     .booleanConf
     .createWithDefault(false)
 
+  private[spark] val POPULATE_HADOOP_CLASSPATH = ConfigBuilder("spark.yarn.populateHadoopClasspath")
+    .doc("Whether to populate Hadoop classpath from `yarn.application.classpath` and " +
+      "`mapreduce.application.classpath` Note that if this is set to `false`, it requires " +
+      "a `with-Hadoop` Spark distribution that bundles Hadoop runtime or user has to provide " +
+      "a Hadoop installation separately.")
+    .version("2.4.6")
+    .booleanConf
+    .createWithDefault(true)
+
   private[spark] val GATEWAY_ROOT_PATH = ConfigBuilder("spark.yarn.config.gatewayPath")
     .doc("Root of configuration paths that is present on gateway nodes, and will be replaced " +
       "with the corresponding path in cluster machines.")
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
index b42c8b933dda0..680ff9991f294 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
@@ -485,6 +485,25 @@ class ClientSuite extends SparkFunSuite with Matchers {
     }
   }
 
+  test("SPARK-31582 Being able to not populate Hadoop classpath") {
+    Seq(true, false).foreach { populateHadoopClassPath =>
+      withAppConf(Fixtures.mapAppConf) { conf =>
+        val sparkConf = new SparkConf()
+          .set(POPULATE_HADOOP_CLASSPATH, populateHadoopClassPath)
+        val env = new MutableHashMap[String, String]()
+        val args = new ClientArguments(Array("--jar", USER))
+        populateClasspath(args, conf, sparkConf, env)
+        if (populateHadoopClassPath) {
+          classpath(env) should
+            (contain (Fixtures.knownYARNAppCP) and contain (Fixtures.knownMRAppCP))
+        } else {
+          classpath(env) should
+            (not contain (Fixtures.knownYARNAppCP) and not contain (Fixtures.knownMRAppCP))
+        }
+      }
+    }
+  }
+
   private val matching = Seq(
     ("files URI match test1", "file:///file1", "file:///file2"),
     ("files URI match test2", "file:///c:file1", "file://c:file2"),

From 281200f59b120bd8b2c75269c69e2cdd6fe20e0a Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 30 Apr 2020 03:20:10 +0000
Subject: [PATCH 0603/1280] [SPARK-31553][SQL][TESTS][FOLLOWUP] Tests for
 collection elem types of `isInCollection`

### What changes were proposed in this pull request?
- Add tests for different element types of collections that could be passed to `isInCollection`. Added tests for types that can pass the check `In`.`checkInputDataTypes()`.
- Test different switch thresholds in the `isInCollection: Scala Collection` test.

### Why are the changes needed?
To prevent regressions like introduced by https://github.com/apache/spark/pull/25754 and reverted by https://github.com/apache/spark/pull/28388

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
By existing and new tests in `ColumnExpressionSuite`

Closes #28405 from MaxGekk/test-isInCollection.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 91648654da259c63178f3fb3f94e3e62e1ef1e45)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/ColumnExpressionSuite.scala     | 93 +++++++++++++++----
 1 file changed, 75 insertions(+), 18 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index 8d3b56242ec5a..4bf19532edd94 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-import java.sql.Date
+import java.sql.{Date, Timestamp}
 import java.util.Locale
 
 import scala.collection.JavaConverters._
@@ -454,26 +454,83 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
   }
 
   test("isInCollection: Scala Collection") {
-    val df = Seq((1, "x"), (2, "y"), (3, "z")).toDF("a", "b")
-    // Test with different types of collections
-    checkAnswer(df.filter($"a".isInCollection(Seq(3, 1))),
-      df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
-    checkAnswer(df.filter($"a".isInCollection(Seq(1, 2).toSet)),
-      df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
-    checkAnswer(df.filter($"a".isInCollection(Seq(3, 2).toArray)),
-      df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
-    checkAnswer(df.filter($"a".isInCollection(Seq(3, 1).toList)),
-      df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+    Seq(0, 1, 10).foreach { optThreshold =>
+      Seq(0, 1, 10).foreach { switchThreshold =>
+        withSQLConf(
+          SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> optThreshold.toString,
+          SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key -> switchThreshold.toString) {
+          val df = Seq((1, "x"), (2, "y"), (3, "z")).toDF("a", "b")
+          // Test with different types of collections
+          checkAnswer(df.filter($"a".isInCollection(Seq(3, 1))),
+            df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+          checkAnswer(df.filter($"a".isInCollection(Seq(1, 2).toSet)),
+            df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
+          checkAnswer(df.filter($"a".isInCollection(Seq(3, 2).toArray)),
+            df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
+          checkAnswer(df.filter($"a".isInCollection(Seq(3, 1).toList)),
+            df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+
+          val df2 = Seq((1, Seq(1)), (2, Seq(2)), (3, Seq(3))).toDF("a", "b")
 
-    val df2 = Seq((1, Seq(1)), (2, Seq(2)), (3, Seq(3))).toDF("a", "b")
-
-    val e = intercept[AnalysisException] {
-      df2.filter($"a".isInCollection(Seq($"b")))
+          val e = intercept[AnalysisException] {
+            df2.filter($"a".isInCollection(Seq($"b")))
+          }
+          Seq("cannot resolve", "due to data type mismatch: Arguments must be same type but were")
+            .foreach { s =>
+              assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
+            }
+        }
+      }
     }
-    Seq("cannot resolve", "due to data type mismatch: Arguments must be same type but were")
-      .foreach { s =>
-        assert(e.getMessage.toLowerCase(Locale.ROOT).contains(s.toLowerCase(Locale.ROOT)))
+  }
+
+  test("SPARK-31553: isInCollection - collection element types") {
+    val expected = Seq(Row(true), Row(false))
+    Seq(0, 1, 10).foreach { optThreshold =>
+      Seq(0, 1, 10).foreach { switchThreshold =>
+        withSQLConf(
+          SQLConf.OPTIMIZER_INSET_CONVERSION_THRESHOLD.key -> optThreshold.toString,
+          SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key -> switchThreshold.toString) {
+          checkAnswer(Seq(0).toDS.select($"value".isInCollection(Seq(null))), Seq(Row(null)))
+          checkAnswer(
+            Seq(true).toDS.select($"value".isInCollection(Seq(true, false))),
+            Seq(Row(true)))
+          checkAnswer(
+            Seq(0.toByte, 1.toByte).toDS.select($"value".isInCollection(Seq(0.toByte, 2.toByte))),
+            expected)
+          checkAnswer(
+            Seq(0.toShort, 1.toShort).toDS
+              .select($"value".isInCollection(Seq(0.toShort, 2.toShort))),
+            expected)
+          checkAnswer(Seq(0, 1).toDS.select($"value".isInCollection(Seq(0, 2))), expected)
+          checkAnswer(Seq(0L, 1L).toDS.select($"value".isInCollection(Seq(0L, 2L))), expected)
+          checkAnswer(Seq(0.0f, 1.0f).toDS
+            .select($"value".isInCollection(Seq(0.0f, 2.0f))), expected)
+          checkAnswer(Seq(0.0D, 1.0D).toDS
+            .select($"value".isInCollection(Seq(0.0D, 2.0D))), expected)
+          checkAnswer(
+            Seq(BigDecimal(0), BigDecimal(2)).toDS
+              .select($"value".isInCollection(Seq(BigDecimal(0), BigDecimal(1)))),
+            expected)
+          checkAnswer(
+            Seq("abc", "def").toDS.select($"value".isInCollection(Seq("abc", "xyz"))),
+            expected)
+          checkAnswer(
+            Seq(Date.valueOf("2020-04-29"), Date.valueOf("2020-05-01")).toDS
+              .select($"value".isInCollection(
+                Seq(Date.valueOf("2020-04-29"), Date.valueOf("2020-04-30")))),
+            expected)
+          checkAnswer(
+            Seq(new Timestamp(0), new Timestamp(2)).toDS
+              .select($"value".isInCollection(Seq(new Timestamp(0), new Timestamp(1)))),
+            expected)
+          checkAnswer(
+            Seq(Array("a", "b"), Array("c", "d")).toDS
+              .select($"value".isInCollection(Seq(Array("a", "b"), Array("x", "z")))),
+            expected)
+        }
       }
+    }
   }
 
   test("&&") {

From 82b8f7fc9d21f4ac506d8cd613158f0511f5cb1d Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 30 Apr 2020 09:15:12 +0530
Subject: [PATCH 0604/1280] [SPARK-31601][K8S] Fix
 spark.kubernetes.executor.podNamePrefix to work

### What changes were proposed in this pull request?

This PR aims to fix `spark.kubernetes.executor.podNamePrefix` to work.

### Why are the changes needed?

Currently, the configuration is broken like the following.
```
bin/spark-submit \
--master k8s://$K8S_MASTER \
--deploy-mode cluster \
--name spark-pi \
--class org.apache.spark.examples.SparkPi \
-c spark.kubernetes.container.image=spark:pr \
-c spark.kubernetes.driver.pod.name=mypod \
-c spark.kubernetes.executor.podNamePrefix=mypod \
local:///opt/spark/examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar
```

**BEFORE SPARK-31601**
```
pod/mypod                              1/1     Running     0          9s
pod/spark-pi-7469dd71c499fafb-exec-1   1/1     Running     0          4s
pod/spark-pi-7469dd71c499fafb-exec-2   1/1     Running     0          4s
```

**AFTER SPARK-31601**
```
pod/mypod                              1/1     Running     0          8s
pod/mypod-exec-1                       1/1     Running     0          3s
pod/mypod-exec-2                       1/1     Running     0          3s
```

### Does this PR introduce any user-facing change?

Yes. This is a bug fix. The conf will work as described in the documentation.

### How was this patch tested?

Pass the Jenkins and run the above comment manually.

Closes #28401 from dongjoon-hyun/SPARK-31601.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Prashant Sharma <prashsh1@in.ibm.com>
(cherry picked from commit 85dad37f69ebb617c8ac015dbbbda11054170298)
Signed-off-by: Prashant Sharma <prashsh1@in.ibm.com>
---
 .../spark/deploy/k8s/features/BasicDriverFeatureStep.scala       | 1 -
 .../spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index e2d70d7692ef3..1edd3f725b56b 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -158,7 +158,6 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
     val additionalProps = mutable.Map(
       KUBERNETES_DRIVER_POD_NAME.key -> driverPodName,
       "spark.app.id" -> conf.appId,
-      KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> conf.resourceNamePrefix,
       KUBERNETES_DRIVER_SUBMIT_CHECK.key -> "true",
       MEMORY_OVERHEAD_FACTOR.key -> overheadFactor.toString)
     // try upload local, resolvable files to a hadoop compatible file system
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index ce66afd9448a9..89e7ff95b5124 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -124,7 +124,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
     val expectedSparkConf = Map(
       KUBERNETES_DRIVER_POD_NAME.key -> "spark-driver-pod",
       "spark.app.id" -> KubernetesTestConf.APP_ID,
-      KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> kubernetesConf.resourceNamePrefix,
       "spark.kubernetes.submitInDriver" -> "true",
       MEMORY_OVERHEAD_FACTOR.key -> MEMORY_OVERHEAD_FACTOR.defaultValue.get.toString)
     assert(featureStep.getAdditionalPodSystemProperties() === expectedSparkConf)

From 75fc3840b704306fbf64d059898b854e8c46caf9 Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Thu, 30 Apr 2020 03:58:04 +0000
Subject: [PATCH 0605/1280] [SPARK-31372][SQL][TEST] Display expression schema
 for double check

### What changes were proposed in this pull request?
Although SPARK-30184 Implement a helper method for aliasing functions, developers always forget to using this improvement.
We need to add more powerful guarantees so that aliases outputed by built-in functions are correct.
This PR extracts the SQL from the example of expressions, and output the SQL and its schema into one golden file.
By checking the golden file, we can find the expressions whose aliases are not displayed correctly, and then fix them.

### Why are the changes needed?
Ensure that the output alias is correct

### Does this PR introduce any user-facing change?
'No'.

### How was this patch tested?
Jenkins test.

Closes #28194 from beliefer/check-expression-schema.

Lead-authored-by: beliefer <beliefer@163.com>
Co-authored-by: gengjiaan <gengjiaan@360.cn>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1d1bb79bc695dbaa00699e6fc9073233b60ed395)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql-functions/sql-expression-schema.md    | 341 ++++++++++++++++++
 .../spark/sql/ExpressionsSchemaSuite.scala    | 187 ++++++++++
 2 files changed, 528 insertions(+)
 create mode 100644 sql/core/src/test/resources/sql-functions/sql-expression-schema.md
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala

diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
new file mode 100644
index 0000000000000..1e22ae2eefeb2
--- /dev/null
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -0,0 +1,341 @@
+<!-- Automatically generated byExpressionsSchemaSuite -->
+## Summary
+  - Number of queries: 333
+  - Number of expressions that missing example: 34
+  - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,struct,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch
+## Schema of Built-in Functions
+| Class name | Function name or alias | Query example | Output schema |
+| ---------- | ---------------------- | ------------- | ------------- |
+| org.apache.spark.sql.catalyst.expressions.Abs | abs | SELECT abs(-1) | struct<abs(-1):int> |
+| org.apache.spark.sql.catalyst.expressions.Acos | acos | SELECT acos(1) | struct<ACOS(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Acosh | acosh | SELECT acosh(1) | struct<ACOSH(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Add | + | SELECT 1 + 2 | struct<(1 + 2):int> |
+| org.apache.spark.sql.catalyst.expressions.AddMonths | add_months | SELECT add_months('2016-08-31', 1) | struct<add_months(CAST(2016-08-31 AS DATE), 1):date> |
+| org.apache.spark.sql.catalyst.expressions.And | and | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.ArrayAggregate | aggregate | SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x) | struct<aggregate(array(1, 2, 3), 0, lambdafunction((namedlambdavariable() + namedlambdavariable()), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayContains | array_contains | SELECT array_contains(array(1, 2, 3), 2) | struct<array_contains(array(1, 2, 3), 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArrayDistinct | array_distinct | SELECT array_distinct(array(1, 2, 3, null, 3)) | struct<array_distinct(array(1, 2, 3, CAST(NULL AS INT), 3)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayExcept | array_except | SELECT array_except(array(1, 2, 3), array(1, 3, 5)) | struct<array_except(array(1, 2, 3), array(1, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayExists | exists | SELECT exists(array(1, 2, 3), x -> x % 2 == 0) | struct<exists(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 0), namedlambdavariable())):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArrayFilter | filter | SELECT filter(array(1, 2, 3), x -> x % 2 == 1) | struct<filter(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 1), namedlambdavariable())):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayForAll | forall | SELECT forall(array(1, 2, 3), x -> x % 2 == 0) | struct<forall(array(1, 2, 3), lambdafunction(((namedlambdavariable() % 2) = 0), namedlambdavariable())):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArrayIntersect | array_intersect | SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)) | struct<array_intersect(array(1, 2, 3), array(1, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayJoin | array_join | SELECT array_join(array('hello', 'world'), ' ') | struct<array_join(array(hello, world),  ):string> |
+| org.apache.spark.sql.catalyst.expressions.ArrayMax | array_max | SELECT array_max(array(1, 20, null, 3)) | struct<array_max(array(1, 20, CAST(NULL AS INT), 3)):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayMin | array_min | SELECT array_min(array(1, 20, null, 3)) | struct<array_min(array(1, 20, CAST(NULL AS INT), 3)):int> |
+| org.apache.spark.sql.catalyst.expressions.ArrayPosition | array_position | SELECT array_position(array(3, 2, 1), 1) | struct<array_position(array(3, 2, 1), 1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.ArrayRemove | array_remove | SELECT array_remove(array(1, 2, 3, null, 3), 3) | struct<array_remove(array(1, 2, 3, CAST(NULL AS INT), 3), 3):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayRepeat | array_repeat | SELECT array_repeat('123', 2) | struct<array_repeat(123, 2):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.ArraySort | array_sort | SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end) | struct<array_sort(array(5, 6, 1), lambdafunction(CASE WHEN (namedlambdavariable() < namedlambdavariable()) THEN -1 WHEN (namedlambdavariable() > namedlambdavariable()) THEN 1 ELSE 0 END, namedlambdavariable(), namedlambdavariable())):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayTransform | transform | SELECT transform(array(1, 2, 3), x -> x + 1) | struct<transform(array(1, 2, 3), lambdafunction((namedlambdavariable() + 1), namedlambdavariable())):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArrayUnion | array_union | SELECT array_union(array(1, 2, 3), array(1, 3, 5)) | struct<array_union(array(1, 2, 3), array(1, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.ArraysOverlap | arrays_overlap | SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)) | struct<arrays_overlap(array(1, 2, 3), array(3, 4, 5)):boolean> |
+| org.apache.spark.sql.catalyst.expressions.ArraysZip | arrays_zip | SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)) | struct<arrays_zip(array(1, 2, 3), array(2, 3, 4)):array<struct<0:int,1:int>>> |
+| org.apache.spark.sql.catalyst.expressions.Ascii | ascii | SELECT ascii('222') | struct<ascii(222):int> |
+| org.apache.spark.sql.catalyst.expressions.Asin | asin | SELECT asin(0) | struct<ASIN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Asinh | asinh | SELECT asinh(0) | struct<ASINH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.AssertTrue | assert_true | SELECT assert_true(0 < 1) | struct<assert_true((0 < 1)):null> |
+| org.apache.spark.sql.catalyst.expressions.Atan | atan | SELECT atan(0) | struct<ATAN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Atan2 | atan2 | SELECT atan2(0, 0) | struct<ATAN2(CAST(0 AS DOUBLE), CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Atanh | atanh | SELECT atanh(0) | struct<ATANH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.BRound | bround | SELECT bround(2.5, 0) | struct<bround(2.5, 0):decimal(2,0)> |
+| org.apache.spark.sql.catalyst.expressions.Base64 | base64 | SELECT base64('Spark SQL') | struct<base64(CAST(Spark SQL AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Bin | bin | SELECT bin(13) | struct<bin(CAST(13 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.BitLength | bit_length | SELECT bit_length('Spark SQL') | struct<bit_length(Spark SQL):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseAnd | & | SELECT 3 & 5 | struct<(3 & 5):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseCount | bit_count | SELECT bit_count(0) | struct<bit_count(0):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseNot | ~ | SELECT ~ 0 | struct<~0:int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseOr | &#124; | SELECT 3 &#124; 5 | struct<(3 &#124; 5):int> |
+| org.apache.spark.sql.catalyst.expressions.BitwiseXor | ^ | SELECT 3 ^ 5 | struct<(3 ^ 5):int> |
+| org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection | java_method | SELECT java_method('java.util.UUID', 'randomUUID') | struct<java_method(java.util.UUID, randomUUID):string> |
+| org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection | reflect | SELECT reflect('java.util.UUID', 'randomUUID') | struct<reflect(java.util.UUID, randomUUID):string> |
+| org.apache.spark.sql.catalyst.expressions.CaseWhen | when | SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END | struct<CASE WHEN (1 > 0) THEN CAST(1 AS DECIMAL(11,1)) WHEN (2 > 0) THEN CAST(2.0 AS DECIMAL(11,1)) ELSE CAST(1.2 AS DECIMAL(11,1)) END:decimal(11,1)> |
+| org.apache.spark.sql.catalyst.expressions.Cast | string | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | cast | SELECT cast('10' as int) | struct<CAST(10 AS INT):int> |
+| org.apache.spark.sql.catalyst.expressions.Cast | tinyint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | double | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | smallint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | date | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | decimal | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | boolean | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | float | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | binary | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | bigint | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | int | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cast | timestamp | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Cbrt | cbrt | SELECT cbrt(27.0) | struct<CBRT(CAST(27.0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Ceil | ceil | SELECT ceil(-0.1) | struct<CEIL(-0.1):decimal(1,0)> |
+| org.apache.spark.sql.catalyst.expressions.Ceil | ceiling | SELECT ceiling(-0.1) | struct<ceiling(-0.1):decimal(1,0)> |
+| org.apache.spark.sql.catalyst.expressions.Chr | char | SELECT char(65) | struct<char(CAST(65 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Chr | chr | SELECT chr(65) | struct<chr(CAST(65 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Coalesce | coalesce | SELECT coalesce(NULL, 1, NULL) | struct<coalesce(CAST(NULL AS INT), 1, CAST(NULL AS INT)):int> |
+| org.apache.spark.sql.catalyst.expressions.Concat | concat | SELECT concat('Spark', 'SQL') | struct<concat(Spark, SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.ConcatWs | concat_ws | SELECT concat_ws(' ', 'Spark', 'SQL') | struct<concat_ws( , Spark, SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.Conv | conv | SELECT conv('100', 2, 10) | struct<conv(100, 2, 10):string> |
+| org.apache.spark.sql.catalyst.expressions.Cos | cos | SELECT cos(0) | struct<COS(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Cosh | cosh | SELECT cosh(0) | struct<COSH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Cot | cot | SELECT cot(1) | struct<COT(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Crc32 | crc32 | SELECT crc32('Spark') | struct<crc32(CAST(Spark AS BINARY)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.CreateArray | array | SELECT array(1, 2, 3) | struct<array(1, 2, 3):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.CreateMap | map | SELECT map(1.0, '2', 3.0, '4') | struct<map(1.0, 2, 3.0, 4):map<decimal(2,1),string>> |
+| org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | named_struct | SELECT named_struct("a", 1, "b", 2, "c", 3) | struct<named_struct(a, 1, b, 2, c, 3):struct<a:int,b:int,c:int>> |
+| org.apache.spark.sql.catalyst.expressions.CsvToStructs | from_csv | SELECT from_csv('1, 0.8', 'a INT, b DOUBLE') | struct<from_csv(1, 0.8):struct<a:int,b:double>> |
+| org.apache.spark.sql.catalyst.expressions.Cube | cube | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name, age) | struct<name:string,age:int,count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.CumeDist | cume_dist | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.CurrentDatabase | current_database | SELECT current_database() | struct<current_database():string> |
+| org.apache.spark.sql.catalyst.expressions.CurrentDate | current_date | SELECT current_date() | struct<current_date():date> |
+| org.apache.spark.sql.catalyst.expressions.CurrentTimestamp | current_timestamp | SELECT current_timestamp() | struct<current_timestamp():timestamp> |
+| org.apache.spark.sql.catalyst.expressions.DateAdd | date_add | SELECT date_add('2016-07-30', 1) | struct<date_add(CAST(2016-07-30 AS DATE), 1):date> |
+| org.apache.spark.sql.catalyst.expressions.DateDiff | datediff | SELECT datediff('2009-07-31', '2009-07-30') | struct<datediff(CAST(2009-07-31 AS DATE), CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DateFormatClass | date_format | SELECT date_format('2016-04-08', 'y') | struct<date_format(CAST(2016-04-08 AS TIMESTAMP), y):string> |
+| org.apache.spark.sql.catalyst.expressions.DatePart | date_part | SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456') | struct<date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'):int> |
+| org.apache.spark.sql.catalyst.expressions.DateSub | date_sub | SELECT date_sub('2016-07-30', 1) | struct<date_sub(CAST(2016-07-30 AS DATE), 1):date> |
+| org.apache.spark.sql.catalyst.expressions.DayOfMonth | day | SELECT day('2009-07-30') | struct<day(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfMonth | dayofmonth | SELECT dayofmonth('2009-07-30') | struct<dayofmonth(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfWeek | dayofweek | SELECT dayofweek('2009-07-30') | struct<dayofweek(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.DayOfYear | dayofyear | SELECT dayofyear('2016-04-09') | struct<dayofyear(CAST(2016-04-09 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.Decode | decode | SELECT decode(encode('abc', 'utf-8'), 'utf-8') | struct<decode(encode(abc, utf-8), utf-8):string> |
+| org.apache.spark.sql.catalyst.expressions.DenseRank | dense_rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Divide | / | SELECT 3 / 2 | struct<(CAST(3 AS DOUBLE) / CAST(2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.ElementAt | element_at | SELECT element_at(array(1, 2, 3), 2) | struct<element_at(array(1, 2, 3), 2):int> |
+| org.apache.spark.sql.catalyst.expressions.Elt | elt | SELECT elt(1, 'scala', 'java') | struct<elt(1, scala, java):string> |
+| org.apache.spark.sql.catalyst.expressions.Encode | encode | SELECT encode('abc', 'utf-8') | struct<encode(abc, utf-8):binary> |
+| org.apache.spark.sql.catalyst.expressions.EqualNullSafe | <=> | SELECT 2 <=> 2 | struct<(2 <=> 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.EqualTo | = | SELECT 2 = 2 | struct<(2 = 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.EqualTo | == | SELECT 2 == 2 | struct<(2 = 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.EulerNumber | e | SELECT e() | struct<E():double> |
+| org.apache.spark.sql.catalyst.expressions.Exp | exp | SELECT exp(0) | struct<EXP(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Explode | explode | SELECT explode(array(10, 20)) | struct<col:int> |
+| org.apache.spark.sql.catalyst.expressions.Explode | explode_outer | SELECT explode_outer(array(10, 20)) | struct<col:int> |
+| org.apache.spark.sql.catalyst.expressions.Expm1 | expm1 | SELECT expm1(0) | struct<EXPM1(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Extract | extract | SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456') | struct<extract('YEAR' FROM TIMESTAMP '2019-08-12 01:00:00.123456'):int> |
+| org.apache.spark.sql.catalyst.expressions.Factorial | factorial | SELECT factorial(5) | struct<factorial(5):bigint> |
+| org.apache.spark.sql.catalyst.expressions.FindInSet | find_in_set | SELECT find_in_set('ab','abc,b,ab,c,def') | struct<find_in_set(ab, abc,b,ab,c,def):int> |
+| org.apache.spark.sql.catalyst.expressions.Flatten | flatten | SELECT flatten(array(array(1, 2), array(3, 4))) | struct<flatten(array(array(1, 2), array(3, 4))):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.Floor | floor | SELECT floor(-0.1) | struct<FLOOR(-0.1):decimal(1,0)> |
+| org.apache.spark.sql.catalyst.expressions.FormatNumber | format_number | SELECT format_number(12332.123456, 4) | struct<format_number(12332.123456, 4):string> |
+| org.apache.spark.sql.catalyst.expressions.FormatString | printf | SELECT printf("Hello World %d %s", 100, "days") | struct<printf(Hello World %d %s, 100, days):string> |
+| org.apache.spark.sql.catalyst.expressions.FormatString | format_string | SELECT format_string("Hello World %d %s", 100, "days") | struct<format_string(Hello World %d %s, 100, days):string> |
+| org.apache.spark.sql.catalyst.expressions.FromUTCTimestamp | from_utc_timestamp | SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<from_utc_timestamp(CAST(2016-08-31 AS TIMESTAMP), Asia/Seoul):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.FromUnixTime | from_unixtime | SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss') | struct<from_unixtime(CAST(0 AS BIGINT), yyyy-MM-dd HH:mm:ss):string> |
+| org.apache.spark.sql.catalyst.expressions.GetJsonObject | get_json_object | SELECT get_json_object('{"a":"b"}', '$.a') | struct<get_json_object({"a":"b"}, $.a):string> |
+| org.apache.spark.sql.catalyst.expressions.GreaterThan | > | SELECT 2 > 1 | struct<(2 > 1):boolean> |
+| org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual | >= | SELECT 2 >= 1 | struct<(2 >= 1):boolean> |
+| org.apache.spark.sql.catalyst.expressions.Greatest | greatest | SELECT greatest(10, 9, 2, 4, 3) | struct<greatest(10, 9, 2, 4, 3):int> |
+| org.apache.spark.sql.catalyst.expressions.Grouping | grouping | SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name) | struct<name:string,grouping(name):tinyint,sum(age):bigint> |
+| org.apache.spark.sql.catalyst.expressions.GroupingID | grouping_id | SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height) | struct<name:string,grouping_id():bigint,sum(age):bigint,avg(height):double> |
+| org.apache.spark.sql.catalyst.expressions.Hex | hex | SELECT hex(17) | struct<hex(CAST(17 AS BIGINT)):string> |
+| org.apache.spark.sql.catalyst.expressions.Hour | hour | SELECT hour('2009-07-30 12:58:59') | struct<hour(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.Hypot | hypot | SELECT hypot(3, 4) | struct<HYPOT(CAST(3 AS DOUBLE), CAST(4 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.If | if | SELECT if(1 < 2, 'a', 'b') | struct<(IF((1 < 2), a, b)):string> |
+| org.apache.spark.sql.catalyst.expressions.IfNull | ifnull | SELECT ifnull(NULL, array('2')) | struct<ifnull(NULL, array('2')):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.In | in | SELECT 1 in(1, 2, 3) | struct<(1 IN (1, 2, 3)):boolean> |
+| org.apache.spark.sql.catalyst.expressions.InitCap | initcap | SELECT initcap('sPark sql') | struct<initcap(sPark sql):string> |
+| org.apache.spark.sql.catalyst.expressions.Inline | inline | SELECT inline(array(struct(1, 'a'), struct(2, 'b'))) | struct<col1:int,col2:string> |
+| org.apache.spark.sql.catalyst.expressions.Inline | inline_outer | SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))) | struct<col1:int,col2:string> |
+| org.apache.spark.sql.catalyst.expressions.InputFileBlockLength | input_file_block_length | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.InputFileBlockStart | input_file_block_start | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.InputFileName | input_file_name | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.IntegralDivide | div | SELECT 3 div 2 | struct<(3 div 2):bigint> |
+| org.apache.spark.sql.catalyst.expressions.IsNaN | isnan | SELECT isnan(cast('NaN' as double)) | struct<isnan(CAST(NaN AS DOUBLE)):boolean> |
+| org.apache.spark.sql.catalyst.expressions.IsNotNull | isnotnull | SELECT isnotnull(1) | struct<(1 IS NOT NULL):boolean> |
+| org.apache.spark.sql.catalyst.expressions.IsNull | isnull | SELECT isnull(1) | struct<(1 IS NULL):boolean> |
+| org.apache.spark.sql.catalyst.expressions.JsonObjectKeys | json_object_keys | SELECT json_object_keys('{}') | struct<json_object_keys({}):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.JsonToStructs | from_json | SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE') | struct<from_json({"a":1, "b":0.8}):struct<a:int,b:double>> |
+| org.apache.spark.sql.catalyst.expressions.JsonTuple | json_tuple | SELECT json_tuple('{"a":1, "b":2}', 'a', 'b') | struct<c0:string,c1:string> |
+| org.apache.spark.sql.catalyst.expressions.Lag | lag | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.LastDay | last_day | SELECT last_day('2009-01-12') | struct<last_day(CAST(2009-01-12 AS DATE)):date> |
+| org.apache.spark.sql.catalyst.expressions.Lead | lead | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Least | least | SELECT least(10, 9, 2, 4, 3) | struct<least(10, 9, 2, 4, 3):int> |
+| org.apache.spark.sql.catalyst.expressions.Left | left | SELECT left('Spark SQL', 3) | struct<left('Spark SQL', 3):string> |
+| org.apache.spark.sql.catalyst.expressions.Length | character_length | SELECT character_length('Spark SQL ') | struct<character_length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.Length | char_length | SELECT char_length('Spark SQL ') | struct<char_length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.Length | length | SELECT length('Spark SQL ') | struct<length(Spark SQL ):int> |
+| org.apache.spark.sql.catalyst.expressions.LengthOfJsonArray | json_array_length | SELECT json_array_length('[1,2,3,4]') | struct<json_array_length([1,2,3,4]):int> |
+| org.apache.spark.sql.catalyst.expressions.LessThan | < | SELECT 1 < 2 | struct<(1 < 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.LessThanOrEqual | <= | SELECT 2 <= 2 | struct<(2 <= 2):boolean> |
+| org.apache.spark.sql.catalyst.expressions.Levenshtein | levenshtein | SELECT levenshtein('kitten', 'sitting') | struct<levenshtein(kitten, sitting):int> |
+| org.apache.spark.sql.catalyst.expressions.Like | like | SELECT like('Spark', '_park') | struct<Spark LIKE _park:boolean> |
+| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<LOG(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log10 | log10 | SELECT log10(10) | struct<LOG10(CAST(10 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log1p | log1p | SELECT log1p(0) | struct<LOG1P(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log2 | log2 | SELECT log2(2) | struct<LOG2(CAST(2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Logarithm | log | SELECT log(10, 100) | struct<LOG(CAST(10 AS DOUBLE), CAST(100 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lower(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lower | SELECT lower('SparkSql') | struct<lower(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.MakeDate | make_date | SELECT make_date(2013, 7, 15) | struct<make_date(2013, 7, 15):date> |
+| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(8,6))):interval> |
+| org.apache.spark.sql.catalyst.expressions.MakeTimestamp | make_timestamp | SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887) | struct<make_timestamp(2014, 12, 28, 6, 30, CAST(45.887 AS DECIMAL(8,6))):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.MapConcat | map_concat | SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')) | struct<map_concat(map(1, a, 2, b), map(3, c)):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.MapEntries | map_entries | SELECT map_entries(map(1, 'a', 2, 'b')) | struct<map_entries(map(1, a, 2, b)):array<struct<key:int,value:string>>> |
+| org.apache.spark.sql.catalyst.expressions.MapFilter | map_filter | SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v) | struct<map_filter(map(1, 0, 2, 2, 3, -1), lambdafunction((namedlambdavariable() > namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.MapFromArrays | map_from_arrays | SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')) | struct<map_from_arrays(array(1.0, 3.0), array(2, 4)):map<decimal(2,1),string>> |
+| org.apache.spark.sql.catalyst.expressions.MapFromEntries | map_from_entries | SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))) | struct<map_from_entries(array(named_struct(col1, 1, col2, a), named_struct(col1, 2, col2, b))):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.MapKeys | map_keys | SELECT map_keys(map(1, 'a', 2, 'b')) | struct<map_keys(map(1, a, 2, b)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct<map_values(map(1, a, 2, b)):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct<map_zip_with(map(1, a, 2, b), map(1, x, 2, y), lambdafunction(concat(namedlambdavariable(), namedlambdavariable()), namedlambdavariable(), namedlambdavariable(), namedlambdavariable())):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.Md5 | md5 | SELECT md5('Spark') | struct<md5(CAST(Spark AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Minute | minute | SELECT minute('2009-07-30 12:58:59') | struct<minute(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.MonotonicallyIncreasingID | monotonically_increasing_id | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Month | month | SELECT month('2016-07-30') | struct<month(CAST(2016-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.MonthsBetween | months_between | SELECT months_between('1997-02-28 10:30:00', '1996-10-30') | struct<months_between(CAST(1997-02-28 10:30:00 AS TIMESTAMP), CAST(1996-10-30 AS TIMESTAMP), true):double> |
+| org.apache.spark.sql.catalyst.expressions.Multiply | * | SELECT 2 * 3 | struct<(2 * 3):int> |
+| org.apache.spark.sql.catalyst.expressions.Murmur3Hash | hash | SELECT hash('Spark', array(123), 2) | struct<hash(Spark, array(123), 2):int> |
+| org.apache.spark.sql.catalyst.expressions.NTile | ntile | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.NaNvl | nanvl | SELECT nanvl(cast('NaN' as double), 123) | struct<nanvl(CAST(NaN AS DOUBLE), CAST(123 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.NamedStruct | struct | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.NextDay | next_day | SELECT next_day('2015-01-14', 'TU') | struct<next_day(CAST(2015-01-14 AS DATE), TU):date> |
+| org.apache.spark.sql.catalyst.expressions.Not | ! | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Not | not | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Now | now | SELECT now() | struct<now():timestamp> |
+| org.apache.spark.sql.catalyst.expressions.NullIf | nullif | SELECT nullif(2, 2) | struct<nullif(2, 2):int> |
+| org.apache.spark.sql.catalyst.expressions.Nvl | nvl | SELECT nvl(NULL, array('2')) | struct<nvl(NULL, array('2')):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.Nvl2 | nvl2 | SELECT nvl2(NULL, 2, 1) | struct<nvl2(NULL, 2, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length | SELECT octet_length('Spark SQL') | struct<octet_length(Spark SQL):int> |
+| org.apache.spark.sql.catalyst.expressions.Or | or | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT overlay('Spark SQL' PLACING '_' FROM 6) | struct<overlay(Spark SQL, _, 6, -1):string> |
+| org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT to_date('2009-07-30 04:17:52') | struct<to_date('2009-07-30 04:17:52'):date> |
+| org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp | SELECT to_timestamp('2016-12-31 00:12:00') | struct<to_timestamp('2016-12-31 00:12:00'):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.ParseUrl | parse_url | SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST') | struct<parse_url(http://spark.apache.org/path?query=1, HOST):string> |
+| org.apache.spark.sql.catalyst.expressions.PercentRank | percent_rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Pi | pi | SELECT pi() | struct<PI():double> |
+| org.apache.spark.sql.catalyst.expressions.Pmod | pmod | SELECT pmod(10, 3) | struct<pmod(10, 3):int> |
+| org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode_outer | SELECT posexplode_outer(array(10,20)) | struct<pos:int,col:int> |
+| org.apache.spark.sql.catalyst.expressions.PosExplode | posexplode | SELECT posexplode(array(10,20)) | struct<pos:int,col:int> |
+| org.apache.spark.sql.catalyst.expressions.Pow | pow | SELECT pow(2, 3) | struct<pow(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Pow | power | SELECT power(2, 3) | struct<POWER(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Quarter | quarter | SELECT quarter('2016-08-31') | struct<quarter(CAST(2016-08-31 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.RLike | rlike | SELECT '%SystemDrive%\Users\John' rlike '%SystemDrive%\\Users.*' | struct<%SystemDrive%UsersJohn RLIKE %SystemDrive%\Users.*:boolean> |
+| org.apache.spark.sql.catalyst.expressions.Rand | random | SELECT random() | struct<rand():double> |
+| org.apache.spark.sql.catalyst.expressions.Rand | rand | SELECT rand() | struct<rand():double> |
+| org.apache.spark.sql.catalyst.expressions.Randn | randn | SELECT randn() | struct<randn():double> |
+| org.apache.spark.sql.catalyst.expressions.Rank | rank | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.RegExpExtract | regexp_extract | SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1) | struct<regexp_extract(100-200, (\d+)-(\d+), 1):string> |
+| org.apache.spark.sql.catalyst.expressions.RegExpReplace | regexp_replace | SELECT regexp_replace('100-200', '(\\d+)', 'num') | struct<regexp_replace(100-200, (\d+), num):string> |
+| org.apache.spark.sql.catalyst.expressions.Remainder | % | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
+| org.apache.spark.sql.catalyst.expressions.Remainder | mod | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
+| org.apache.spark.sql.catalyst.expressions.Reverse | reverse | SELECT reverse('Spark SQL') | struct<reverse(Spark SQL):string> |
+| org.apache.spark.sql.catalyst.expressions.Right | right | SELECT right('Spark SQL', 3) | struct<right('Spark SQL', 3):string> |
+| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<ROUND(CAST(12.3456 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Rollup | rollup | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY rollup(name, age) | struct<name:string,age:int,count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Round | round | SELECT round(2.5, 0) | struct<round(2.5, 0):decimal(2,0)> |
+| org.apache.spark.sql.catalyst.expressions.RowNumber | row_number | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.SchemaOfCsv | schema_of_csv | SELECT schema_of_csv('1,abc') | struct<schema_of_csv(1,abc):string> |
+| org.apache.spark.sql.catalyst.expressions.SchemaOfJson | schema_of_json | SELECT schema_of_json('[{"col":0}]') | struct<schema_of_json([{"col":0}]):string> |
+| org.apache.spark.sql.catalyst.expressions.Second | second | SELECT second('2009-07-30 12:58:59') | struct<second(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
+| org.apache.spark.sql.catalyst.expressions.Sentences | sentences | SELECT sentences('Hi there! Good morning.') | struct<sentences(Hi there! Good morning., , ):array<array<string>>> |
+| org.apache.spark.sql.catalyst.expressions.Sequence | sequence | SELECT sequence(1, 5) | struct<sequence(1, 5):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.Sha1 | sha1 | SELECT sha1('Spark') | struct<sha1(CAST(Spark AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha1 | sha | SELECT sha('Spark') | struct<sha(CAST(Spark AS BINARY)):string> |
+| org.apache.spark.sql.catalyst.expressions.Sha2 | sha2 | SELECT sha2('Spark', 256) | struct<sha2(CAST(Spark AS BINARY), 256):string> |
+| org.apache.spark.sql.catalyst.expressions.ShiftLeft | shiftleft | SELECT shiftleft(2, 1) | struct<shiftleft(2, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.ShiftRight | shiftright | SELECT shiftright(4, 1) | struct<shiftright(4, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.ShiftRightUnsigned | shiftrightunsigned | SELECT shiftrightunsigned(4, 1) | struct<shiftrightunsigned(4, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.Shuffle | shuffle | SELECT shuffle(array(1, 20, 3, 5)) | struct<shuffle(array(1, 20, 3, 5)):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.Signum | signum | SELECT signum(40) | struct<SIGNUM(CAST(40 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Signum | sign | SELECT sign(40) | struct<sign(CAST(40 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Sin | sin | SELECT sin(0) | struct<SIN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Sinh | sinh | SELECT sinh(0) | struct<SINH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Size | size | SELECT size(array('b', 'd', 'c', 'a')) | struct<size(array(b, d, c, a)):int> |
+| org.apache.spark.sql.catalyst.expressions.Size | cardinality | SELECT cardinality(array('b', 'd', 'c', 'a')) | struct<cardinality(array(b, d, c, a)):int> |
+| org.apache.spark.sql.catalyst.expressions.Slice | slice | SELECT slice(array(1, 2, 3, 4), 2, 2) | struct<slice(array(1, 2, 3, 4), 2, 2):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.SortArray | sort_array | SELECT sort_array(array('b', 'd', null, 'c', 'a'), true) | struct<sort_array(array(b, d, CAST(NULL AS STRING), c, a), true):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.SoundEx | soundex | SELECT soundex('Miller') | struct<soundex(Miller):string> |
+| org.apache.spark.sql.catalyst.expressions.SparkPartitionID | spark_partition_id | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.SparkVersion | version | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Sqrt | sqrt | SELECT sqrt(4) | struct<SQRT(CAST(4 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Stack | stack | SELECT stack(2, 1, 2, 3) | struct<col0:int,col1:int> |
+| org.apache.spark.sql.catalyst.expressions.StringInstr | instr | SELECT instr('SparkSQL', 'SQL') | struct<instr(SparkSQL, SQL):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLPad | lpad | SELECT lpad('hi', 5, '??') | struct<lpad(hi, 5, ??):string> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | locate | SELECT locate('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringRPad | rpad | SELECT rpad('hi', 5, '??') | struct<rpad(hi, 5, ??):string> |
+| org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
+| org.apache.spark.sql.catalyst.expressions.StringReplace | replace | SELECT replace('ABCabc', 'abc', 'DEF') | struct<replace(ABCabc, abc, DEF):string> |
+| org.apache.spark.sql.catalyst.expressions.StringSpace | space | SELECT concat(space(2), '1') | struct<concat(space(2), 1):string> |
+| org.apache.spark.sql.catalyst.expressions.StringSplit | split | SELECT split('oneAtwoBthreeC', '[ABC]') | struct<split(oneAtwoBthreeC, [ABC], -1):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.StringToMap | str_to_map | SELECT str_to_map('a:1,b:2,c:3', ',', ':') | struct<str_to_map(a:1,b:2,c:3, ,, :):map<string,string>> |
+| org.apache.spark.sql.catalyst.expressions.StringTranslate | translate | SELECT translate('AaBbCc', 'abc', '123') | struct<translate(AaBbCc, abc, 123):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrim | trim | SELECT trim('    SparkSQL   ') | struct<trim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrimLeft | ltrim | SELECT ltrim('    SparkSQL   ') | struct<ltrim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StringTrimRight | rtrim | SELECT rtrim('    SparkSQL   ') | struct<rtrim(    SparkSQL   ):string> |
+| org.apache.spark.sql.catalyst.expressions.StructsToCsv | to_csv | SELECT to_csv(named_struct('a', 1, 'b', 2)) | struct<to_csv(named_struct(a, 1, b, 2)):string> |
+| org.apache.spark.sql.catalyst.expressions.StructsToJson | to_json | SELECT to_json(named_struct('a', 1, 'b', 2)) | struct<to_json(named_struct(a, 1, b, 2)):string> |
+| org.apache.spark.sql.catalyst.expressions.Substring | substr | SELECT substr('Spark SQL', 5) | struct<substr(Spark SQL, 5, 2147483647):string> |
+| org.apache.spark.sql.catalyst.expressions.Substring | substring | SELECT substring('Spark SQL', 5) | struct<substring(Spark SQL, 5, 2147483647):string> |
+| org.apache.spark.sql.catalyst.expressions.SubstringIndex | substring_index | SELECT substring_index('www.apache.org', '.', 2) | struct<substring_index(www.apache.org, ., 2):string> |
+| org.apache.spark.sql.catalyst.expressions.Subtract | - | SELECT 2 - 1 | struct<(2 - 1):int> |
+| org.apache.spark.sql.catalyst.expressions.Tan | tan | SELECT tan(0) | struct<TAN(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Tanh | tanh | SELECT tanh(0) | struct<TANH(CAST(0 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.TimeWindow | window | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.ToDegrees | degrees | SELECT degrees(3.141592653589793) | struct<DEGREES(CAST(3.141592653589793 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.ToRadians | radians | SELECT radians(180) | struct<RADIANS(CAST(180 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.ToUTCTimestamp | to_utc_timestamp | SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul') | struct<to_utc_timestamp(CAST(2016-08-31 AS TIMESTAMP), Asia/Seoul):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.ToUnixTimestamp | to_unix_timestamp | SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd') | struct<to_unix_timestamp(2016-04-08, yyyy-MM-dd):bigint> |
+| org.apache.spark.sql.catalyst.expressions.TransformKeys | transform_keys | SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1) | struct<transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.TransformValues | transform_values | SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1) | struct<transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), lambdafunction((namedlambdavariable() + 1), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
+| org.apache.spark.sql.catalyst.expressions.TruncDate | trunc | SELECT trunc('2019-08-04', 'week') | struct<trunc(CAST(2019-08-04 AS DATE), week):date> |
+| org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc | SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR, CAST(2015-03-05T09:32:05.359 AS TIMESTAMP)):timestamp> |
+| org.apache.spark.sql.catalyst.expressions.TypeOf | typeof | SELECT typeof(1) | struct<typeof(1):string> |
+| org.apache.spark.sql.catalyst.expressions.UnBase64 | unbase64 | SELECT unbase64('U3BhcmsgU1FM') | struct<unbase64(U3BhcmsgU1FM):binary> |
+| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<(- 1):int> |
+| org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct<decode(unhex(537061726B2053514C), UTF-8):string> |
+| org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct<unix_timestamp(current_timestamp(), yyyy-MM-dd HH:mm:ss):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Upper | ucase | SELECT ucase('SparkSql') | struct<ucase(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Upper | upper | SELECT upper('SparkSql') | struct<upper(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Uuid | uuid | SELECT uuid() | struct<uuid():string> |
+| org.apache.spark.sql.catalyst.expressions.WeekDay | weekday | SELECT weekday('2009-07-30') | struct<weekday(CAST(2009-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.WeekOfYear | weekofyear | SELECT weekofyear('2008-02-20') | struct<weekofyear(CAST(2008-02-20 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.XxHash64 | xxhash64 | SELECT xxhash64('Spark', array(123), 2) | struct<xxhash64(Spark, array(123), 2):bigint> |
+| org.apache.spark.sql.catalyst.expressions.Year | year | SELECT year('2016-07-30') | struct<year(CAST(2016-07-30 AS DATE)):int> |
+| org.apache.spark.sql.catalyst.expressions.ZipWith | zip_with | SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)) | struct<zip_with(array(1, 2, 3), array(a, b, c), lambdafunction(named_struct(y, namedlambdavariable(), x, namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):array<struct<y:string,x:int>>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | approx_percentile | SELECT approx_percentile(10.0, array(0.5, 0.4, 0.1), 100) | struct<approx_percentile(10.0, array(0.5, 0.4, 0.1), 100):array<decimal(3,1)>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | percentile_approx | SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100) | struct<percentile_approx(10.0, array(0.5, 0.4, 0.1), 100):array<decimal(3,1)>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Average | avg | SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<avg(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Average | mean | SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<mean(col):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BitAndAgg | bit_and | SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_and(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BitOrAgg | bit_or | SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_or(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BitXorAgg | bit_xor | SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_xor(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | every | SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<every(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolAnd | bool_and | SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col) | struct<bool_and(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | bool_or | SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<bool_or(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | some | SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<some(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.BoolOr | any | SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col) | struct<any(col):boolean> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CollectList | collect_list | SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col) | struct<collect_list(col):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CollectSet | collect_set | SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col) | struct<collect_set(col):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Corr | corr | SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2) | struct<corr(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Count | count | SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col) | struct<count(1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CountIf | count_if | SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col) | struct<count_if(((col % 2) = 0)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CountMinSketchAgg | count_min_sketch | N/A | N/A |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CovPopulation | covar_pop | SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_pop(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.CovSample | covar_samp | SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_samp(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first_value | SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first_value(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first | SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus | approx_count_distinct | SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1) | struct<approx_count_distinct(col1):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Kurtosis | kurtosis | SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<kurtosis(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Max | max | SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col) | struct<max(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<maxby(x, y):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Min | min | SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col) | struct<min(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<minby(x, y):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Percentile | percentile | SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col) | struct<percentile(col, CAST(0.3 AS DOUBLE), 1):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Skewness | skewness | SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<skewness(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop | stddev_pop | SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_pop(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev_samp | SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_samp(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | stddev | SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.StddevSamp | std | SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<std(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Sum | sum | SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col) | struct<sum(col):bigint> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VariancePop | var_pop | SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_pop(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | var_samp | SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<var_samp(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.VarianceSamp | variance | SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<variance(CAST(col AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathBoolean | xpath_boolean | SELECT xpath_boolean('<a><b>1</b></a>','a/b') | struct<xpath_boolean(<a><b>1</b></a>, a/b):boolean> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_number | SELECT xpath_number('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_number(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathDouble | xpath_double | SELECT xpath_double('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_double(<a><b>1</b><b>2</b></a>, sum(a/b)):double> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathFloat | xpath_float | SELECT xpath_float('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_float(<a><b>1</b><b>2</b></a>, sum(a/b)):float> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathInt | xpath_int | SELECT xpath_int('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_int(<a><b>1</b><b>2</b></a>, sum(a/b)):int> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') | struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>, a/b/text()):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathLong | xpath_long | SELECT xpath_long('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_long(<a><b>1</b><b>2</b></a>, sum(a/b)):bigint> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathShort | xpath_short | SELECT xpath_short('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_short(<a><b>1</b><b>2</b></a>, sum(a/b)):smallint> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
\ No newline at end of file
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
new file mode 100644
index 0000000000000..dd72473f0ea6c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.io.File
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile}
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.tags.ExtendedSQLTest
+
+// scalastyle:off line.size.limit
+/**
+ * End-to-end test cases for SQL schemas of expression examples.
+ * The golden result file is "spark/sql/core/src/test/resources/sql-functions/sql-expression-schema.md".
+ *
+ * To run the entire test suite:
+ * {{{
+ *   build/sbt "sql/test-only *ExpressionsSchemaSuite"
+ * }}}
+ *
+ * To re-generate golden files for entire suite, run:
+ * {{{
+ *   SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "sql/test-only *ExpressionsSchemaSuite"
+ * }}}
+ *
+ * For example:
+ * {{{
+ *   ...
+ *   @ExpressionDescription(
+ *     usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.",
+ *     examples = """
+ *       Examples:
+ *         > SELECT _FUNC_('123', 2);
+ *          123123
+ *     """,
+ *     since = "1.5.0")
+ *   case class StringRepeat(str: Expression, times: Expression)
+ *   ...
+ * }}}
+ *
+ * The format for golden result files look roughly like:
+ * {{{
+ *   ...
+ *   | org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
+ *   ...
+ * }}}
+ */
+// scalastyle:on line.size.limit
+@ExtendedSQLTest
+class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
+
+  private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
+
+  private val baseResourcePath = {
+    // We use a path based on Spark home for 2 reasons:
+    //   1. Maven can't get correct resource directory when resources in other jars.
+    //   2. We test subclasses in the hive-thriftserver module.
+    val sparkHome = {
+      assert(sys.props.contains("spark.test.home") ||
+        sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
+      sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+    }
+
+    java.nio.file.Paths.get(sparkHome,
+      "sql", "core", "src", "test", "resources", "sql-functions").toFile
+  }
+
+  private val resultFile = new File(baseResourcePath, "sql-expression-schema.md")
+
+  /** A single SQL query's SQL and schema. */
+  protected case class QueryOutput(
+      className: String,
+      funcName: String,
+      sql: String = "N/A",
+      schema: String = "N/A") {
+    override def toString: String = {
+      s"| $className | $funcName | $sql | $schema |"
+    }
+  }
+
+  test("Check schemas for expression examples") {
+    val exampleRe = """^(.+);\n(?s)(.+)$""".r
+    val funInfos = spark.sessionState.functionRegistry.listFunction().map { funcId =>
+      spark.sessionState.catalog.lookupFunctionInfo(funcId)
+    }
+
+    val classFunsMap = funInfos.groupBy(_.getClassName).toSeq.sortBy(_._1)
+    val outputBuffer = new ArrayBuffer[String]
+    val outputs = new ArrayBuffer[QueryOutput]
+    val missingExamples = new ArrayBuffer[String]
+
+    classFunsMap.foreach { kv =>
+      val className = kv._1
+      kv._2.foreach { funInfo =>
+        val example = funInfo.getExamples
+        val funcName = funInfo.getName.replaceAll("\\|", "&#124;")
+        if (example == "") {
+          val queryOutput = QueryOutput(className, funcName)
+          outputBuffer += queryOutput.toString
+          outputs += queryOutput
+          missingExamples += funcName
+        }
+
+        // If expression exists 'Examples' segment, the first element is 'Examples'. Because
+        // this test case is only used to print aliases of expressions for double checking.
+        // Therefore, we only need to output the first SQL and its corresponding schema.
+        // Note: We need to filter out the commands that set the parameters, such as:
+        // SET spark.sql.parser.escapedStringLiterals=true
+        example.split("  > ").tail.filterNot(_.trim.startsWith("SET")).take(1).foreach {
+          case exampleRe(sql, _) =>
+            val df = spark.sql(sql)
+            val escapedSql = sql.replaceAll("\\|", "&#124;")
+            val schema = df.schema.catalogString.replaceAll("\\|", "&#124;")
+            val queryOutput = QueryOutput(className, funcName, escapedSql, schema)
+            outputBuffer += queryOutput.toString
+            outputs += queryOutput
+          case _ =>
+        }
+      }
+    }
+
+    if (regenerateGoldenFiles) {
+      val missingExampleStr = missingExamples.mkString(",")
+      val goldenOutput = {
+        s"<!-- Automatically generated by${getClass.getSimpleName} -->\n" +
+        "## Summary\n" +
+        s"  - Number of queries: ${outputs.size}\n" +
+        s"  - Number of expressions that missing example: ${missingExamples.size}\n" +
+        s"  - Expressions missing examples: $missingExampleStr\n" +
+        "## Schema of Built-in Functions\n" +
+        "| Class name | Function name or alias | Query example | Output schema |\n" +
+        "| ---------- | ---------------------- | ------------- | ------------- |\n" +
+        outputBuffer.mkString("\n")
+      }
+      val parent = resultFile.getParentFile
+      if (!parent.exists()) {
+        assert(parent.mkdirs(), "Could not create directory: " + parent)
+      }
+      stringToFile(resultFile, goldenOutput)
+    }
+
+    val expectedOutputs: Seq[QueryOutput] = {
+      val goldenOutput = fileToString(resultFile)
+      val lines = goldenOutput.split("\n")
+
+      // The header of golden file has one line, plus four lines of the summary and three
+      // lines of the header of schema table.
+      assert(lines.size == outputs.size + 8,
+        s"Expected ${outputs.size + 8} blocks in result file but got ${lines.size}. " +
+          s"Try regenerate the result files.")
+
+      Seq.tabulate(outputs.size) { i =>
+        val segments = lines(i + 8).split('|')
+        QueryOutput(
+          className = segments(1).trim,
+          funcName = segments(2).trim,
+          sql = segments(3).trim,
+          schema = segments(4).trim)
+      }
+    }
+
+    // Compare results.
+    assert(expectedOutputs.size == outputs.size, s"Number of queries not equals")
+
+    outputs.zip(expectedOutputs).foreach { case (output, expected) =>
+      assert(expected.sql == output.sql, "SQL query did not match")
+      assert(expected.schema == output.schema, s"Schema did not match for query ${expected.sql}")
+    }
+  }
+}

From 4864680d18ec0ae02721bbbbe009101488db672f Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Thu, 30 Apr 2020 05:47:42 +0000
Subject: [PATCH 0606/1280] [SPARK-31030][DOCS][FOLLOWUP] Replace HTML Table by
 Markdown Table

### What changes were proposed in this pull request?

This PR is to clean up the markdown file in datetime-pattern page.

- Replace HTML table by MD table

### Why are the changes needed?
Make the doc cleaner and easily editable by MD editors.

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
**Before**
![Screen Shot 2020-04-29 at 7 59 10 PM](https://user-images.githubusercontent.com/11567269/80668093-c9294600-8a55-11ea-9dca-d558203298f8.png)

**After**

![Screen Shot 2020-04-29 at 8 13 38 PM](https://user-images.githubusercontent.com/11567269/80668146-f1b14000-8a55-11ea-8d47-8dc8a0378271.png)

Closes #28415 from gatorsmile/cleanupUDFPage.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f56c6630fbb3230f0eb9549b45105a1a015abd4c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md | 220 +++++--------------------------
 1 file changed, 32 insertions(+), 188 deletions(-)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 80585a4fd484c..e606383c94321 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -28,194 +28,38 @@ There are several common scenarios for datetime usage in Spark:
 
 Spark uses pattern letters in the following table for date and timestamp parsing and formatting:
 
-<table class="table">
-<tr>
-  <th> <b>Symbol</b> </th>
-  <th> <b>Meaning</b> </th>
-  <th> <b>Presentation</b> </th>
-  <th> <b>Examples</b> </th>
-</tr>
-<tr>
-  <td> <b>G</b> </td>
-  <td> era </td>
-  <td> text </td>
-  <td> AD; Anno Domini; A </td>
-</tr>
-<tr>
-  <td> <b>y</b> </td>
-  <td> year </td>
-  <td> year </td>
-  <td> 2020; 20 </td>
-</tr>
-<tr>
-  <td> <b>D</b> </td>
-  <td> day-of-year </td>
-  <td> number </td>
-  <td> 189 </td>
-</tr>
-<tr>
-  <td> <b>M/L</b> </td>
-  <td> month-of-year </td>
-  <td> number/text </td>
-  <td> 7; 07; Jul; July; J </td>
-</tr>
-<tr>
-  <td> <b>d</b> </td>
-  <td> day-of-month </td>
-  <td> number </td>
-  <td> 28 </td>
-</tr>
-<tr>
-  <td> <b>Q/q</b> </td>
-  <td> quarter-of-year </td>
-  <td> number/text </td>
-  <td> 3; 03; Q3; 3rd quarter </td>
-</tr>
-<tr>
-  <td> <b>Y</b> </td>
-  <td> week-based-year </td>
-  <td> year </td>
-  <td> 1996; 96 </td>
-</tr>
-<tr>
-  <td> <b>w</b> </td>
-  <td> week-of-week-based-year </td>
-  <td> number </td>
-  <td> 27 </td>
-</tr>
-<tr>
-  <td> <b>W</b> </td>
-  <td> week-of-month </td>
-  <td> number </td>
-  <td> 4 </td>
-</tr>
-<tr>
-  <td> <b>E</b> </td>
-  <td> day-of-week </td>
-  <td> text </td>
-  <td> Tue; Tuesday; T </td>
-</tr>
-<tr>
-  <td> <b>u</b> </td>
-  <td> localized day-of-week </td>
-  <td> number/text </td>
-  <td> 2; 02; Tue; Tuesday; T </td>
-</tr>
-<tr>
-  <td> <b>F</b> </td>
-  <td> week-of-month </td>
-  <td> number </td>
-  <td> 3 </td>
-</tr>
-<tr>
-  <td> <b>a</b> </td>
-  <td> am-pm-of-day </td>
-  <td> text </td>
-  <td> PM </td>
-</tr>
-<tr>
-  <td> <b>h</b> </td>
-  <td> clock-hour-of-am-pm (1-12) </td>
-  <td> number </td>
-  <td> 12 </td>
-</tr>
-<tr>
-  <td> <b>K</b> </td>
-  <td> hour-of-am-pm (0-11) </td>
-  <td> number </td>
-  <td> 0 </td>
-</tr>
-<tr>
-  <td> <b>k</b> </td>
-  <td> clock-hour-of-day (1-24) </td>
-  <td> number </td>
-  <td> 0 </td>
-</tr>
-<tr>
-  <td> <b>H</b> </td>
-  <td> hour-of-day (0-23)  </td>
-  <td> number </td>
-  <td> 0 </td>
-</tr>
-<tr>
-  <td> <b>m</b> </td>
-  <td> minute-of-hour </td>
-  <td> number </td>
-  <td> 30 </td>
-</tr>
-<tr>
-  <td> <b>s</b> </td>
-  <td> second-of-minute </td>
-  <td> number </td>
-  <td> 55 </td>
-</tr>
-<tr>
-  <td> <b>S</b> </td>
-  <td> fraction-of-second </td>
-  <td> fraction </td>
-  <td> 978 </td>
-</tr>
-<tr>
-  <td> <b>V</b> </td>
-  <td> time-zone ID </td>
-  <td> zone-id </td>
-  <td> America/Los_Angeles; Z; -08:30 </td>
-</tr>
-<tr>
-  <td> <b>z</b> </td>
-  <td> time-zone name </td>
-  <td> zone-name </td>
-  <td> Pacific Standard Time; PST </td>
-</tr>
-<tr>
-  <td> <b>O</b> </td>
-  <td> localized zone-offset </td>
-  <td> offset-O </td>
-  <td> GMT+8; GMT+08:00; UTC-08:00; </td>
-</tr>
-<tr>
-  <td> <b>X</b> </td>
-  <td> zone-offset 'Z' for zero </td>
-  <td> offset-X </td>
-  <td> Z; -08; -0830; -08:30; -083015; -08:30:15; </td>
-</tr>
-<tr>
-  <td> <b>x</b> </td>
-  <td> zone-offset </td>
-  <td> offset-x </td>
-  <td> +0000; -08; -0830; -08:30; -083015; -08:30:15; </td>
-</tr>
-<tr>
-  <td> <b>Z</b> </td>
-  <td> zone-offset </td>
-  <td> offset-Z </td>
-  <td> +0000; -0800; -08:00; </td>
-</tr>
-<tr>
-  <td> <b>'</b> </td>
-  <td> escape for text </td>
-  <td> delimiter </td>
-  <td></td>
-</tr>
-<tr>
-  <td> <b>''</b> </td>
-  <td> single quote </td>
-  <td> literal </td>
-  <td> ' </td>
-</tr>
-<tr>
-  <td> <b>[</b> </td>
-  <td> optional section start </td>
-  <td>  </td>
-  <td>  </td>
-</tr>
-<tr>
-  <td> <b>]</b> </td>
-  <td> optional section end </td>
-  <td>  </td>
-  <td>  </td>
-</tr>
-</table>
+|Symbol|Meaning|Presentation|Examples|
+|--- |--- |--- |--- |
+|**G**|era|text|AD; Anno Domini; A|
+|**y**|year|year|2020; 20|
+|**D**|day-of-year|number|189|
+|**M/L**|month-of-year|number/text|7; 07; Jul; July; J|
+|**d**|day-of-month|number|28|
+|**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
+|**Y**|week-based-year|year|1996; 96|
+|**w**|week-of-week-based-year|number|27|
+|**W**|week-of-month|number|4|
+|**E**|day-of-week|text|Tue; Tuesday; T|
+|**u**|localized day-of-week|number/text|2; 02; Tue; Tuesday; T|
+|**F**|week-of-month|number|3|
+|**a**|am-pm-of-day|text|PM|
+|**h**|clock-hour-of-am-pm (1-12)|number|12|
+|**K**|hour-of-am-pm (0-11)|number|0|
+|**k**|clock-hour-of-day (1-24)|number|0|
+|**H**|hour-of-day (0-23)|number|0|
+|**m**|minute-of-hour|number|30|
+|**s**|second-of-minute|number|55|
+|**S**|fraction-of-second|fraction|978|
+|**V**|time-zone ID|zone-id|America/Los_Angeles; Z; -08:30|
+|**z**|time-zone name|zone-name|Pacific Standard Time; PST|
+|**O**|localized zone-offset|offset-O|GMT+8; GMT+08:00; UTC-08:00;|
+|**X**|zone-offset 'Z' for zero|offset-X|Z; -08; -0830; -08:30; -083015; -08:30:15;|
+|**x**|zone-offset|offset-x|+0000; -08; -0830; -08:30; -083015; -08:30:15;|
+|**Z**|zone-offset|offset-Z|+0000; -0800; -08:00;|
+|**'**|escape for text|delimiter||
+|**''**|single quote|literal|'|
+|**[**|optional section start|||
+|**]**|optional section end|||
 
 The count of pattern letters determines the format.
 

From e9ca66096dc415e5046dba4f3f1738b7a96e39be Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Thu, 30 Apr 2020 06:24:00 +0000
Subject: [PATCH 0607/1280] [SPARK-27340][SS][TESTS][FOLLOW-UP] Rephrase API
 comments and simplify tests

### What changes were proposed in this pull request?

- Rephrase the API doc for `Column.as`
- Simplify the UTs

### Why are the changes needed?
Address comments in https://github.com/apache/spark/pull/28326

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
New UT added.

Closes #28390 from xuanyuanking/SPARK-27340-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 7195a18bf24d9506d2f8d9d4d93ff679b3d21b65)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |  2 ++
 .../scala/org/apache/spark/sql/Column.scala   | 12 +++++++++
 .../streaming/EventTimeWatermarkSuite.scala   | 16 ++++++++++++
 .../sql/streaming/StreamingJoinSuite.scala    | 26 -------------------
 4 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index e68198afe327b..774fb2caea252 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -30,6 +30,8 @@ license: |
 
   - In Spark 2.4 and below, `Dataset.groupByKey` results to a grouped dataset with key attribute is wrongly named as "value", if the key is non-struct type, for example, int, string, array, etc. This is counterintuitive and makes the schema of aggregation queries unexpected. For example, the schema of `ds.groupByKey(...).count()` is `(value, count)`. Since Spark 3.0, we name the grouping attribute to "key". The old behavior is preserved under a newly added configuration `spark.sql.legacy.dataset.nameNonStructGroupingKeyAsValue` with a default value of `false`.
 
+  - In Spark 3.0, the column metadata will always be propagated in the API `Column.name` and `Column.as`. In Spark version 2.4 and earlier, the metadata of `NamedExpression` is set as the `explicitMetadata` for the new column at the time the API is called, it won't change even if the underlying `NamedExpression` changes metadata. To restore the behavior before Spark 2.4, you can use the API `as(alias: String, metadata: Metadata)` with explicit metadata.
+
 ### DDL Statements
 
   - In Spark 3.0, `CREATE TABLE` without a specific provider uses the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and below, it was Hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 6913d4e676d66..2144472937f9b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -964,6 +964,10 @@ class Column(val expr: Expression) extends Logging {
    *   df.select($"colA".as("colB"))
    * }}}
    *
+   * If the current column has metadata associated with it, this metadata will be propagated
+   * to the new column. If this not desired, use the API `as(alias: String, metadata: Metadata)`
+   * with explicit metadata.
+   *
    * @group expr_ops
    * @since 1.3.0
    */
@@ -1000,6 +1004,10 @@ class Column(val expr: Expression) extends Logging {
    *   df.select($"colA".as('colB))
    * }}}
    *
+   * If the current column has metadata associated with it, this metadata will be propagated
+   * to the new column. If this not desired, use the API `as(alias: String, metadata: Metadata)`
+   * with explicit metadata.
+   *
    * @group expr_ops
    * @since 1.3.0
    */
@@ -1026,6 +1034,10 @@ class Column(val expr: Expression) extends Logging {
    *   df.select($"colA".name("colB"))
    * }}}
    *
+   * If the current column has metadata associated with it, this metadata will be propagated
+   * to the new column. If this not desired, use the API `as(alias: String, metadata: Metadata)`
+   * with explicit metadata.
+   *
    * @group expr_ops
    * @since 2.0.0
    */
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
index 231317705f3a8..6486e1aee8649 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
@@ -602,6 +602,22 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
     // Check the eventTime metadata is kept in the top level alias.
     assert(aliasWindow.logicalPlan.output.exists(
       _.metadata.contains(EventTimeWatermark.delayKey)))
+
+    val windowedAggregation = aliasWindow
+      .groupBy('aliasWindow)
+      .agg(count("*") as 'count)
+      .select($"aliasWindow".getField("start").cast("long").as[Long], $"count".as[Long])
+
+    testStream(windowedAggregation)(
+      AddData(inputData, 10, 11, 12, 13, 14, 15),
+      CheckNewAnswer(),
+      AddData(inputData, 25), // Advance watermark to 15 seconds
+      CheckNewAnswer((10, 5)),
+      assertNumStateRows(2),
+      AddData(inputData, 10), // Should not emit anything as data less than watermark
+      CheckNewAnswer(),
+      assertNumStateRows(2)
+    )
   }
 
   test("test no-data flag") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
index 33f899b98373f..3f218c9cb7fd9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
@@ -991,30 +991,4 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
       )
     }
   }
-
-  test("SPARK-27340 Windowed left out join with Alias on TimeWindow") {
-    val (leftInput, df1) = setupStream("left", 2)
-    val (rightInput, df2) = setupStream("right", 3)
-    val left = df1.select('key, window('leftTime, "10 second") as 'leftWindow, 'leftValue)
-    val right = df2.select('key, window('rightTime, "10 second") as 'rightWindow, 'rightValue)
-    val joined = left.join(
-      right,
-      left("key") === right("key") && left("leftWindow") === right("rightWindow"),
-      "left_outer")
-      .select(left("key"), $"leftWindow.end".cast("long"), 'leftValue, 'rightValue)
-
-    testStream(joined)(
-      // Test inner part of the join.
-      MultiAddData(leftInput, 1, 2, 3, 4, 5)(rightInput, 3, 4, 5, 6, 7),
-      CheckNewAnswer((3, 10, 6, 9), (4, 10, 8, 12), (5, 10, 10, 15)),
-
-      MultiAddData(leftInput, 21)(rightInput, 22), // watermark = 11, no-data-batch computes nulls
-      CheckNewAnswer(Row(1, 10, 2, null), Row(2, 10, 4, null)),
-      assertNumStateRows(total = 2, updated = 12),
-
-      AddData(leftInput, 22),
-      CheckNewAnswer(Row(22, 30, 44, 66)),
-      assertNumStateRows(total = 3, updated = 1)
-    )
-  }
 }

From 9d4a864e12df9a710468abd70e8a69a3508fec0f Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 30 Apr 2020 12:45:32 +0000
Subject: [PATCH 0608/1280] [SPARK-31557][SQL] Fix timestamps rebasing in
 legacy parsers

In the PR, I propose to fix two legacy timestamp formatter `LegacySimpleTimestampFormatter` and `LegacyFastTimestampFormatter` to perform micros rebasing in parsing/formatting from/to strings.

Legacy timestamps formatters operate on the hybrid calendar (Julian + Gregorian), so, the input micros should be rebased to have the same date-time fields as in Proleptic Gregorian calendar used by Spark SQL, see SPARK-26651.

Yes

Added tests to `TimestampFormatterSuite`

Closes #28408 from MaxGekk/fix-rebasing-in-legacy-timestamp-formatter.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c09cfb9808d0e399b97781aae0da50332ba4b49b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalyst/util/TimestampFormatter.scala    | 15 +++++----
 .../sql/util/TimestampFormatterSuite.scala    | 33 +++++++++++++++++--
 2 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 0c4fa17cdcd20..47d6b47f32fa1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.sql.Timestamp
 import java.text.{ParseException, ParsePosition, SimpleDateFormat}
 import java.time._
 import java.time.format.{DateTimeFormatter, DateTimeParseException}
@@ -30,6 +31,7 @@ import org.apache.commons.lang3.time.FastDateFormat
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.catalyst.util.LegacyDateFormats.{LegacyDateFormat, LENIENT_SIMPLE_DATE_FORMAT}
+import org.apache.spark.sql.catalyst.util.RebaseDateTime._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.types.Decimal
@@ -154,12 +156,14 @@ class LegacyFastTimestampFormatter(
     }
     val micros = cal.getMicros()
     cal.set(Calendar.MILLISECOND, 0)
-    Math.addExact(fromMillis(cal.getTimeInMillis), micros)
+    val julianMicros = Math.addExact(fromMillis(cal.getTimeInMillis), micros)
+    rebaseJulianToGregorianMicros(julianMicros)
   }
 
   def format(timestamp: SQLTimestamp): String = {
-    cal.setTimeInMillis(Math.floorDiv(timestamp, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
-    cal.setMicros(Math.floorMod(timestamp, MICROS_PER_SECOND))
+    val julianMicros = rebaseGregorianToJulianMicros(timestamp)
+    cal.setTimeInMillis(Math.floorDiv(julianMicros, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
+    cal.setMicros(Math.floorMod(julianMicros, MICROS_PER_SECOND))
     fastDateFormat.format(cal)
   }
 }
@@ -177,12 +181,11 @@ class LegacySimpleTimestampFormatter(
   }
 
   override def parse(s: String): Long = {
-    fromMillis(sdf.parse(s).getTime)
+    fromJavaTimestamp(new Timestamp(sdf.parse(s).getTime))
   }
 
   override def format(us: Long): String = {
-    val timestamp = DateTimeUtils.toJavaTimestamp(us)
-    sdf.format(timestamp)
+    sdf.format(toJavaTimestamp(us))
   }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index b2c39242aad53..5d27a6b8cce1e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -24,9 +24,11 @@ import org.scalatest.Matchers
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, LegacyDateFormats, TimestampFormatter}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CET, PST, UTC}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.instantToMicros
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.unsafe.types.UTF8String
 
 class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
@@ -260,6 +262,33 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       LocalDateTime.of(-1233, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
     assert(formatter2.parse("AD 1234-02-22 02:22:22") === instantToMicros(
       LocalDateTime.of(1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+  }
+
+  test("SPARK-31557: rebasing in legacy formatters/parsers") {
+    withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
+      LegacyDateFormats.values.foreach { legacyFormat =>
+        DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
+            DateTimeTestUtils.withDefaultTimeZone(zoneId) {
+              withClue(s"${zoneId.getId} legacyFormat = $legacyFormat") {
+                val formatter = TimestampFormatter(
+                  TimestampFormatter.defaultPattern,
+                  zoneId,
+                  TimestampFormatter.defaultLocale,
+                  legacyFormat,
+                  needVarLengthSecondFraction = false)
+                assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
+                  .atZone(zoneId)
+                  .toLocalDateTime === LocalDateTime.of(1000, 1, 1, 1, 2, 3))
 
+                assert(formatter.format(instantToMicros(
+                  LocalDateTime.of(1000, 1, 1, 1, 2, 3)
+                    .atZone(zoneId).toInstant)) === "1000-01-01 01:02:03")
+              }
+            }
+          }
+        }
+      }
+    }
   }
 }

From f5e018edc71fd5ddf5ce4f82d02ac777bc3d7280 Mon Sep 17 00:00:00 2001
From: Xiao Li <gatorsmile@gmail.com>
Date: Thu, 30 Apr 2020 09:34:56 -0700
Subject: [PATCH 0609/1280] [SPARK-28806][DOCS][FOLLOW-UP] Remove unneeded HTML
 from the MD file

### What changes were proposed in this pull request?
This PR is to clean up the markdown file in SHOW COLUMNS page.

- remove the unneeded embedded inline HTML markup by using the basic markdown syntax.
- use the ``` sql for highlighting the SQL syntax.

### Why are the changes needed?
Make the doc cleaner and easily editable by MD editors.

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
**Before**

![Screen Shot 2020-04-29 at 5 20 11 PM](https://user-images.githubusercontent.com/11567269/80661963-fa4d4a80-8a44-11ea-9dea-c43cda6de010.png)

**After**

![Screen Shot 2020-04-29 at 6 03 50 PM](https://user-images.githubusercontent.com/11567269/80661940-f15c7900-8a44-11ea-9943-a83e8d8618fb.png)

Closes #28414 from gatorsmile/cleanupShowColumns.

Lead-authored-by: Xiao Li <gatorsmile@gmail.com>
Co-authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit b5ecc41c73018bbc742186d2e752101a99cfe852)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 docs/sql-ref-syntax-aux-show-columns.md | 51 +++++++++++--------------
 1 file changed, 22 insertions(+), 29 deletions(-)

diff --git a/docs/sql-ref-syntax-aux-show-columns.md b/docs/sql-ref-syntax-aux-show-columns.md
index 8f73aac0e3a61..c8c90a9d343e7 100644
--- a/docs/sql-ref-syntax-aux-show-columns.md
+++ b/docs/sql-ref-syntax-aux-show-columns.md
@@ -25,41 +25,34 @@ Return the list of columns in a table. If the table does not exist, an exception
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SHOW COLUMNS table_identifier [ database ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
+* **table_identifier**
+
     Specifies the table name of an existing table. The table may be optionally qualified
-    with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        { IN | FROM } [ database_name . ] table_name
-      </code><br><br>
-    <b>Note:</b>
-    Keywords <code>IN</code> and <code>FROM</code> are interchangeable.
-  </dd>
-  <dt><code><em>database</em></code></dt>
-  <dd>
+    with a database name.
+
+    **Syntax:** `{ IN | FROM } [ database_name . ] table_name`
+
+    **Note:** Keywords `IN` and `FROM` are interchangeable.
+
+* **database**
+
     Specifies an optional database name. The table is resolved from this database when it
-    is specified. Please note that when this parameter is specified then table
-    name should not be qualified with a different database name. <br><br>
-    <b>Syntax:</b>
-      <code>
-        { IN | FROM } database_name
-      </code><br><br>
-    <b>Note:</b>
-    Keywords <code>IN</code> and <code>FROM</code> are interchangeable.
-  </dd>
-</dl>
+    is specified. When this parameter is specified then table
+    name should not be qualified with a different database name. 
+
+    **Syntax:** `{ IN | FROM } database_name`
+
+    **Note:** Keywords `IN` and `FROM` are interchangeable.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create `customer` table in `salesdb` database;
 USE salesdb;
 CREATE TABLE customer(
@@ -96,9 +89,9 @@ SHOW COLUMNS IN customer IN salesdb;
   |     name|
   |cust_addr|
   +---------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
- * [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [SHOW TABLE](sql-ref-syntax-aux-show-table.html)

From f7c1feba123534bf9a64e7c381464c64c4572308 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 1 May 2020 06:30:35 +0900
Subject: [PATCH 0610/1280] [SPARK-31612][SQL][DOCS] SQL Reference clean up

### What changes were proposed in this pull request?
SQL Reference cleanup

### Why are the changes needed?
To complete SQL Reference

### Does this PR introduce _any_ user-facing change?
updated sql-ref-syntax-qry.html

before
<img width="1100" alt="Screen Shot 2020-04-29 at 11 08 25 PM" src="https://user-images.githubusercontent.com/13592258/80677799-70b27280-8a6e-11ea-8e3f-a768f29d0377.png">

after
<img width="1100" alt="Screen Shot 2020-04-29 at 11 05 55 PM" src="https://user-images.githubusercontent.com/13592258/80677803-74de9000-8a6e-11ea-880c-aa05c53254a6.png">

### How was this patch tested?
Manually build and check

Closes #28417 from huaxingao/cleanup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 2410a45703b829391211caaf1a745511f95298ad)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-syntax-aux-describe-database.md   |  2 +-
 docs/sql-ref-syntax-aux-show-tables.md         |  2 +-
 docs/sql-ref-syntax-aux-show-views.md          |  2 +-
 docs/sql-ref-syntax-ddl-alter-database.md      |  2 +-
 docs/sql-ref-syntax-ddl-alter-table.md         |  4 ++--
 docs/sql-ref-syntax-ddl-alter-view.md          |  6 +++---
 docs/sql-ref-syntax-ddl-create-function.md     |  4 ++--
 ...l-ref-syntax-ddl-create-table-datasource.md |  2 +-
 docs/sql-ref-syntax-ddl-create-view.md         |  4 ++--
 docs/sql-ref-syntax-ddl-drop-database.md       |  6 +++---
 docs/sql-ref-syntax-ddl-drop-function.md       | 18 +++++++++++++-----
 docs/sql-ref-syntax-ddl-drop-table.md          |  2 +-
 docs/sql-ref-syntax-ddl-drop-view.md           |  2 +-
 docs/sql-ref-syntax-ddl-truncate-table.md      |  6 +++---
 docs/sql-ref-syntax-dml-insert-into.md         |  4 ++--
 ...ntax-dml-insert-overwrite-directory-hive.md |  2 +-
 ...ef-syntax-dml-insert-overwrite-directory.md |  2 +-
 ...ql-ref-syntax-dml-insert-overwrite-table.md |  2 +-
 docs/sql-ref-syntax-qry-select-usedb.md        |  2 +-
 docs/sql-ref-syntax-qry.md                     | 11 ++++++++++-
 20 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/docs/sql-ref-syntax-aux-describe-database.md b/docs/sql-ref-syntax-aux-describe-database.md
index 2f7b1ce984d3e..590438b3cf719 100644
--- a/docs/sql-ref-syntax-aux-describe-database.md
+++ b/docs/sql-ref-syntax-aux-describe-database.md
@@ -42,7 +42,7 @@ interchangeable.
   </dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- Create employees DATABASE
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index f4b3dff850d44..cd54d450a449b 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -52,7 +52,7 @@ SHOW TABLES [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
   </dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- List all tables in default database
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
index 0d9210b024e3a..b1a8d3ba09014 100644
--- a/docs/sql-ref-syntax-aux-show-views.md
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -51,7 +51,7 @@ SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
   </dd>
 </dl>
 
-### Example
+### Examples
 {% highlight sql %}
 -- Create views in different databases, also create global/local temp views.
 CREATE VIEW sam AS SELECT id, salary FROM employee WHERE name = 'sam';
diff --git a/docs/sql-ref-syntax-ddl-alter-database.md b/docs/sql-ref-syntax-ddl-alter-database.md
index 520aba35567e8..65b85dcae15e7 100644
--- a/docs/sql-ref-syntax-ddl-alter-database.md
+++ b/docs/sql-ref-syntax-ddl-alter-database.md
@@ -31,7 +31,7 @@ for a database and may be used for auditing purposes.
 
 {% highlight sql %}
 ALTER { DATABASE | SCHEMA } database_name
-    SET DBPROPERTIES ( property_name = property_value, ... )
+    SET DBPROPERTIES ( property_name = property_value [ , ... ] )
 {% endhighlight %}
 
 ### Parameters
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index edb081b7f45c0..0a74aa0f843e2 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -66,7 +66,7 @@ ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
 #### Syntax
 
 {% highlight sql %}
-ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , col_spec ... ] )
+ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , ... ] )
 {% endhighlight %}
 
 #### Parameters
@@ -84,7 +84,7 @@ ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , col_spec ... ] )
 
 <dl>
   <dt><code><em>COLUMNS ( col_spec )</em></code></dt>
-  <dd>Specifies the columns to be added to be renamed.</dd>
+  <dd>Specifies the columns to be added.</dd>
 </dl>
 
 ### ALTER OR CHANGE COLUMN
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index 8116c97cc2f41..06c1b6534976f 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -105,7 +105,7 @@ ALTER VIEW view_identifier UNSET TBLPROPERTIES [ IF EXISTS ]  ( property_key [ ,
 </dl>
 
 #### ALTER View AS SELECT
-`ALTER VIEW view_identifier AS SELECT` statement changes the definition of a view, the `SELECT` statement must be valid,
+`ALTER VIEW view_identifier AS SELECT` statement changes the definition of a view. The `SELECT` statement must be valid,
 and the `view_identifier` must exist.
 
 #### Syntax
@@ -127,7 +127,7 @@ Note that `ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPE
   </dd>
   <dt><code><em>select_statement</em></code></dt>
   <dd>
-    Specifies the definition of the view, detail check <a href="sql-ref-syntax-qry-select.html">select_statement</a>
+    Specifies the definition of the view. Check <a href="sql-ref-syntax-qry-select.html">select_statement</a> for details.
   </dd>
 </dl>
 
@@ -136,7 +136,7 @@ Note that `ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPE
 {% highlight sql %}
 -- Rename only changes the view name.
 -- The source and target databases of the view have to be the same.
--- Use qualified or unqualified name for the source and target view  
+-- Use qualified or unqualified name for the source and target view.
 ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;
 
 -- Verify that the new view is created.
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index 2bd26d18c7736..5009069b5f8f1 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -28,8 +28,8 @@ all sessions. The resources specified in the `USING` clause are made available
 to all executors when they are executed for the first time. In addition to the
 SQL interface, spark allows users to create custom user defined scalar and
 aggregate functions using Scala, Python and Java APIs. Please refer to 
-[scalar_functions](sql-getting-started.html#scalar-functions) and 
-[aggregate functions](sql-getting-started#aggregations) for more information.
+[Scalar UDFs](sql-ref-functions-udf-scalar.html) and
+[UDAFs](sql-ref-functions-udf-aggregate.html) for more information.
 
 ### Syntax
 
diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index 715b64c33baed..54827fd63568d 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -28,7 +28,7 @@ The `CREATE TABLE` statement defines a new table using a Data Source.
 {% highlight sql %}
 CREATE TABLE [ IF NOT EXISTS ] table_identifier
     [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
-    [USING data_source]
+    [ USING data_source ]
     [ OPTIONS ( key1=val1, key2=val2, ... ) ]
     [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
     [ CLUSTERED BY ( col_name3, col_name4, ... ) 
diff --git a/docs/sql-ref-syntax-ddl-create-view.md b/docs/sql-ref-syntax-ddl-create-view.md
index 5d9db2ab36b5d..ba8c1df1223a3 100644
--- a/docs/sql-ref-syntax-ddl-create-view.md
+++ b/docs/sql-ref-syntax-ddl-create-view.md
@@ -42,7 +42,7 @@ CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_ident
   <dt><code><em>[ GLOBAL ] TEMPORARY</em></code></dt>
   <dd>TEMPORARY views are session-scoped and will be dropped when session ends 
       because it skips persisting the definition in the underlying metastore, if any.
-      GLOBAL TEMPORARY views are tied to a system preserved temporary database `global_temp`.</dd>
+      GLOBAL TEMPORARY views are tied to a system preserved temporary database <code>global_temp</code>.</dd>
 </dl>
 <dl>
   <dt><code><em>IF NOT EXISTS</em></code></dt>
@@ -64,7 +64,7 @@ CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_ident
     <ul>
       <li><code>[ ( column_name [ COMMENT column_comment ], ... ) ]</code> to specify column-level comments.</li>
       <li><code>[ COMMENT view_comment ]</code> to specify view-level comments.</li>
-      <li><code>[ TBLPROPERTIES ( property_name = property_value, ... ) ]</code> to add metadata key-value pairs.</li>
+      <li><code>[ TBLPROPERTIES ( property_name = property_value [ , ... ] ) ]</code> to add metadata key-value pairs.</li>
     </ul>  
   </dd>
 </dl>
diff --git a/docs/sql-ref-syntax-ddl-drop-database.md b/docs/sql-ref-syntax-ddl-drop-database.md
index 431139101eba4..7467e7a4ad6e7 100644
--- a/docs/sql-ref-syntax-ddl-drop-database.md
+++ b/docs/sql-ref-syntax-ddl-drop-database.md
@@ -27,14 +27,14 @@ exception will be thrown if the database does not exist in the system.
 ### Syntax
 
 {% highlight sql %}
-DROP ( DATABASE | SCHEMA ) [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
+DROP { DATABASE | SCHEMA } [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
 {% endhighlight %}
 
 ### Parameters
 
 <dl>
   <dt><code><em>DATABASE | SCHEMA</em></code></dt>
-  <dd>`DATABASE` and `SCHEMA` mean the same thing, either of them can be used.</dd>
+  <dd><code>DATABASE</code> and <code>SCHEMA</code> mean the same thing, either of them can be used.</dd>
 </dl>
 
 <dl>
@@ -52,7 +52,7 @@ DROP ( DATABASE | SCHEMA ) [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
   <dd>If specified, will drop all the associated tables and functions.</dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- Create `inventory_db` Database
diff --git a/docs/sql-ref-syntax-ddl-drop-function.md b/docs/sql-ref-syntax-ddl-drop-function.md
index f7ad18553e304..b783b0ebd2bc8 100644
--- a/docs/sql-ref-syntax-ddl-drop-function.md
+++ b/docs/sql-ref-syntax-ddl-drop-function.md
@@ -27,19 +27,26 @@ be thrown if the function does not exist.
 ### Syntax
 
 {% highlight sql %}
-DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] [ db_name. ] function_name
+DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] function_name
 {% endhighlight %}
 
 ### Parameters
 
 <dl>
   <dt><code><em>function_name</em></code></dt>
-  <dd>The name of an existing function.</dd>
+  <dd>
+    Specifies the name of an existing function. The function name may be
+    optionally qualified with a database name. <br><br>
+    <b>Syntax:</b>
+      <code>
+        [ database_name. ] function_name
+      </code>
+  </dd>
 </dl>
 
 <dl>
   <dt><code><em>TEMPORARY</em></code></dt>
-  <dd>Should be used to delete the `temporary` function.</dd>
+  <dd>Should be used to delete the <code>TEMPORARY</code> function.</dd>
 </dl>
 
 <dl>
@@ -47,7 +54,7 @@ DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] [ db_name. ] function_name
   <dd>If specified, no exception is thrown when the function does not exist.</dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- Create a permanent function `test_avg`
@@ -62,7 +69,8 @@ SHOW USER FUNCTIONS;
   +----------------+
 
 -- Create Temporary function `test_avg`
-CREATE TEMPORARY FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
+CREATE TEMPORARY FUNCTION test_avg AS
+    'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
 
 -- List user functions
 SHOW USER FUNCTIONS;
diff --git a/docs/sql-ref-syntax-ddl-drop-table.md b/docs/sql-ref-syntax-ddl-drop-table.md
index 32a9cc7bb27db..c943b922ae812 100644
--- a/docs/sql-ref-syntax-ddl-drop-table.md
+++ b/docs/sql-ref-syntax-ddl-drop-table.md
@@ -49,7 +49,7 @@ DROP TABLE [ IF EXISTS ] table_identifier
   </dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- Assumes a table named `employeetable` exists.
diff --git a/docs/sql-ref-syntax-ddl-drop-view.md b/docs/sql-ref-syntax-ddl-drop-view.md
index ae976c125f5f8..ad018b5e6fd5c 100644
--- a/docs/sql-ref-syntax-ddl-drop-view.md
+++ b/docs/sql-ref-syntax-ddl-drop-view.md
@@ -46,7 +46,7 @@ DROP VIEW [ IF EXISTS ] view_identifier
   </dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- Assumes a view named `employeeView` exists.
diff --git a/docs/sql-ref-syntax-ddl-truncate-table.md b/docs/sql-ref-syntax-ddl-truncate-table.md
index 6377e83570207..e2d1834302776 100644
--- a/docs/sql-ref-syntax-ddl-truncate-table.md
+++ b/docs/sql-ref-syntax-ddl-truncate-table.md
@@ -61,7 +61,7 @@ TRUNCATE TABLE table_identifier [ partition_spec ]
 -- Create table Student with partition
 CREATE TABLE Student (name STRING, rollno INT) PARTITIONED BY (age INT);
 
-SELECT * from Student;
+SELECT * FROM Student;
   +----+------+---+
   |name|rollno|age|
   +----+------+---+
@@ -74,7 +74,7 @@ SELECT * from Student;
 TRUNCATE TABLE Student partition(age=10);
 
 -- After truncate execution, records belonging to partition age=10 are removed
-SELECT * from Student;
+SELECT * FROM Student;
   +----+------+---+
   |name|rollno|age|
   +----+------+---+
@@ -84,7 +84,7 @@ SELECT * from Student;
 -- Removes all rows from the table from all partitions
 TRUNCATE TABLE Student;
 
-SELECT * from Student;
+SELECT * FROM Student;
   +----+------+---+
   |name|rollno|age|
   +----+------+---+
diff --git a/docs/sql-ref-syntax-dml-insert-into.md b/docs/sql-ref-syntax-dml-insert-into.md
index ba65334ef8f61..2498726602837 100644
--- a/docs/sql-ref-syntax-dml-insert-into.md
+++ b/docs/sql-ref-syntax-dml-insert-into.md
@@ -27,7 +27,7 @@ The `INSERT INTO` statement inserts new rows into a table. The inserted rows can
 
 {% highlight sql %}
 INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
 {% endhighlight %}
 
 ### Parameters
@@ -50,7 +50,7 @@ INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
     for partitions.<br><br>
     <b>Syntax:</b>
       <code>
-        PARTITION (partition_col_name  = partition_col_val [ , ... ])
+        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
       </code>
   </dd>
 </dl>
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
index 3ab0994cf06e7..3cd2107668fbe 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
@@ -29,7 +29,7 @@ Hive support must be enabled to use this command. The inserted rows can be speci
 {% highlight sql %}
 INSERT OVERWRITE [ LOCAL ] DIRECTORY directory_path
     [ ROW FORMAT row_format ] [ STORED AS file_format ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
 {% endhighlight %}
 
 ### Parameters
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
index 645396620f21a..6ce7f50588e32 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
@@ -28,7 +28,7 @@ The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the d
 {% highlight sql %}
 INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
     USING file_format [ OPTIONS ( key = val [ , ... ] ) ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
 {% endhighlight %}
 
 ### Parameters
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index added8e1976be..f2874b5ac2304 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -27,7 +27,7 @@ The `INSERT OVERWRITE` statement overwrites the existing data in the table using
 
 {% highlight sql %}
 INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
-    { { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] } | query }
+    { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
 {% endhighlight %}
 
 ### Parameters
diff --git a/docs/sql-ref-syntax-qry-select-usedb.md b/docs/sql-ref-syntax-qry-select-usedb.md
index 2a05085218978..bb95a8e4ddf30 100644
--- a/docs/sql-ref-syntax-qry-select-usedb.md
+++ b/docs/sql-ref-syntax-qry-select-usedb.md
@@ -41,7 +41,7 @@ USE database_name
   </dd>
 </dl>
 
-### Example
+### Examples
 
 {% highlight sql %}
 -- Use the 'userdb' which exists.
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 477c347eed800..325c9b69f12f9 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -22,7 +22,8 @@ license: |
 Spark supports <code>SELECT</code> statement that is used to retrieve rows
 from one or more tables according to the specified clauses. The full syntax
 and brief description of supported clauses are explained in
-[SELECT](sql-ref-syntax-qry-select.html) section. Spark also provides the
+[SELECT](sql-ref-syntax-qry-select.html) section. The SQL statements related
+to SELECT are also included in this section. Spark also provides the
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
@@ -34,4 +35,12 @@ ability to generate logical and physical plan for a given query using
  * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
  * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
  * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+ * [JOIN](sql-ref-syntax-qry-select-join.html)
+ * [Join Hints](sql-ref-syntax-qry-select-hints.html)
+ * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+ * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+ * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+ * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+ * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+ * [Window Function](sql-ref-syntax-qry-window.html)
  * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)

From a281b9cc64910f7f708341e8379cd2878461186a Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Fri, 1 May 2020 10:08:16 +0900
Subject: [PATCH 0611/1280] [SPARK-31549][PYSPARK] Add a develop API invoking
 collect on Python RDD with user-specified job group

### What changes were proposed in this pull request?
I add a new API in pyspark RDD class:

def collectWithJobGroup(self, groupId, description, interruptOnCancel=False)

This API do the same thing with `rdd.collect`, but it can specify the job group when do collect.
The purpose of adding this API is, if we use:

```
sc.setJobGroup("group-id...")
rdd.collect()
```
The `setJobGroup` API in pyspark won't work correctly. This related to a bug discussed in
https://issues.apache.org/jira/browse/SPARK-31549

Note:

This PR is a rather temporary workaround for `PYSPARK_PIN_THREAD`, and as a step to migrate to  `PYSPARK_PIN_THREAD` smoothly. It targets Spark 3.0.

- `PYSPARK_PIN_THREAD` is unstable at this moment that affects whole PySpark applications.
- It is impossible to make it runtime configuration as it has to be set before JVM is launched.
- There is a thread leak issue between Python and JVM. We should address but it's not a release blocker for Spark 3.0 since the feature is experimental. I plan to handle this after Spark 3.0 due to stability.

Once `PYSPARK_PIN_THREAD` is enabled by default, we should remove this API out ideally. I will target to deprecate this API in Spark 3.1.

### Why are the changes needed?
Fix bug.

### Does this PR introduce any user-facing change?
A develop API in pyspark: `pyspark.RDD. collectWithJobGroup`

### How was this patch tested?
Unit test.

Closes #28395 from WeichenXu123/collect_with_job_group.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit ee1de66fe4e05754ea3f33b75b83c54772b00112)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/api/python/PythonRDD.scala   | 15 +++++
 python/pyspark/rdd.py                         | 13 ++++
 python/pyspark/tests/test_rdd.py              | 62 +++++++++++++++++++
 3 files changed, 90 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
index 6dc1721f56adf..a577194a48006 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -168,6 +168,21 @@ private[spark] object PythonRDD extends Logging {
     serveIterator(rdd.collect().iterator, s"serve RDD ${rdd.id}")
   }
 
+  /**
+   * A helper function to collect an RDD as an iterator, then serve it via socket.
+   * This method is similar with `PythonRDD.collectAndServe`, but user can specify job group id,
+   * job description, and interruptOnCancel option.
+   */
+  def collectAndServeWithJobGroup[T](
+      rdd: RDD[T],
+      groupId: String,
+      description: String,
+      interruptOnCancel: Boolean): Array[Any] = {
+    val sc = rdd.sparkContext
+    sc.setJobGroup(groupId, description, interruptOnCancel)
+    serveIterator(rdd.collect().iterator, s"serve RDD ${rdd.id}")
+  }
+
   /**
    * A helper function to create a local RDD iterator and serve it via socket. Partitions are
    * are collected as separate jobs, by order of index. Partition data is first requested by a
diff --git a/python/pyspark/rdd.py b/python/pyspark/rdd.py
index 52ab86c0d88ee..b5b72dade12a8 100644
--- a/python/pyspark/rdd.py
+++ b/python/pyspark/rdd.py
@@ -889,6 +889,19 @@ def collect(self):
             sock_info = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
         return list(_load_from_socket(sock_info, self._jrdd_deserializer))
 
+    def collectWithJobGroup(self, groupId, description, interruptOnCancel=False):
+        """
+        .. note:: Experimental
+
+        When collect rdd, use this method to specify job group.
+
+        .. versionadded:: 3.0.0
+        """
+        with SCCallSiteSync(self.context) as css:
+            sock_info = self.ctx._jvm.PythonRDD.collectAndServeWithJobGroup(
+                self._jrdd.rdd(), groupId, description, interruptOnCancel)
+        return list(_load_from_socket(sock_info, self._jrdd_deserializer))
+
     def reduce(self, f):
         """
         Reduces the elements of this RDD using the specified commutative and
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index 15cc48ae2153d..6b11d6807bbb0 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -784,6 +784,68 @@ def fail_last(x):
             self.assertEqual(i, next(it))
 
 
+    def test_multiple_group_jobs(self):
+        import threading
+        group_a = "job_ids_to_cancel"
+        group_b = "job_ids_to_run"
+
+        threads = []
+        thread_ids = range(4)
+        thread_ids_to_cancel = [i for i in thread_ids if i % 2 == 0]
+        thread_ids_to_run = [i for i in thread_ids if i % 2 != 0]
+
+        # A list which records whether job is cancelled.
+        # The index of the array is the thread index which job run in.
+        is_job_cancelled = [False for _ in thread_ids]
+
+        def run_job(job_group, index):
+            """
+            Executes a job with the group ``job_group``. Each job waits for 3 seconds
+            and then exits.
+            """
+            try:
+                self.sc.parallelize([15]).map(lambda x: time.sleep(x)) \
+                    .collectWithJobGroup(job_group, "test rdd collect with setting job group")
+                is_job_cancelled[index] = False
+            except Exception:
+                # Assume that exception means job cancellation.
+                is_job_cancelled[index] = True
+
+        # Test if job succeeded when not cancelled.
+        run_job(group_a, 0)
+        self.assertFalse(is_job_cancelled[0])
+
+        # Run jobs
+        for i in thread_ids_to_cancel:
+            t = threading.Thread(target=run_job, args=(group_a, i))
+            t.start()
+            threads.append(t)
+
+        for i in thread_ids_to_run:
+            t = threading.Thread(target=run_job, args=(group_b, i))
+            t.start()
+            threads.append(t)
+
+        # Wait to make sure all jobs are executed.
+        time.sleep(3)
+        # And then, cancel one job group.
+        self.sc.cancelJobGroup(group_a)
+
+        # Wait until all threads launching jobs are finished.
+        for t in threads:
+            t.join()
+
+        for i in thread_ids_to_cancel:
+            self.assertTrue(
+                is_job_cancelled[i],
+                "Thread {i}: Job in group A was not cancelled.".format(i=i))
+
+        for i in thread_ids_to_run:
+            self.assertFalse(
+                is_job_cancelled[i],
+                "Thread {i}: Job in group B did not succeeded.".format(i=i))
+
+
 if __name__ == "__main__":
     import unittest
     from pyspark.tests.test_rdd import *

From 7be1484272f8d4a17a990e57e6c30db4bef61fb6 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Fri, 1 May 2020 10:32:37 +0900
Subject: [PATCH 0612/1280] [SPARK-28424][TESTS][FOLLOW-UP] Add test cases for
 all interval units

Add test cases covering all interval units:  MICROSECOND MILLISECOND SECOND MINUTE HOUR DAY WEEK MONTH YEAR

For test coverage.

No.

Test only.

Closes #28418 from xuanyuanking/SPARK-28424.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit aec8b694359a5a7fb42adfcbfe2f3cef2d307e28)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/parser/ExpressionParserSuite.scala    |  5 +++++
 .../src/test/resources/sql-tests/inputs/interval.sql   |  1 +
 .../resources/sql-tests/results/ansi/interval.sql.out  | 10 +++++++++-
 .../test/resources/sql-tests/results/interval.sql.out  | 10 +++++++++-
 4 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 522d49e448f1f..c74fab1b27c65 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -456,6 +456,11 @@ class ExpressionParserSuite extends AnalysisTest {
     assertEqual(
       "-interval '3 month 1 hour'",
       UnaryMinus(Literal(IntervalUtils.stringToInterval("interval 3 month 1 hour"))))
+    val intervalStrWithAllUnits = "1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds " +
+      "100 millisecond 200 microseconds"
+    assertEqual(
+      s"interval '$intervalStrWithAllUnits'",
+      Literal(IntervalUtils.stringToInterval(intervalStrWithAllUnits)))
 
     // Binary.
     assertEqual("X'A'", Literal(Array(0x0a).map(_.toByte)))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index fec11b4a9038e..600df2c213519 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -71,6 +71,7 @@ select map(1, interval 1 day, 2, interval 3 week);
 -- typed interval expression
 select interval 'interval 3 year 1 hour';
 select interval '3 year 1 hour';
+SELECT interval '1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds 100 millisecond 200 microseconds';
 
 -- malformed interval literal
 select interval;
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index e8201d46924da..6a2b301c6fb18 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 89
+-- Number of queries: 90
 
 
 -- !query
@@ -471,6 +471,14 @@ struct<INTERVAL '3 years 1 hours':interval>
 3 years 1 hours
 
 
+-- !query
+SELECT interval '1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds 100 millisecond 200 microseconds'
+-- !query schema
+struct<INTERVAL '1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds':interval>
+-- !query output
+1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds
+
+
 -- !query
 select interval
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 84e3fb3c238a4..94562cae951a0 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 89
+-- Number of queries: 90
 
 
 -- !query
@@ -460,6 +460,14 @@ struct<INTERVAL '3 years 1 hours':interval>
 3 years 1 hours
 
 
+-- !query
+SELECT interval '1 year 3 months 2 weeks 2 days 1 hour 3 minutes 2 seconds 100 millisecond 200 microseconds'
+-- !query schema
+struct<INTERVAL '1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds':interval>
+-- !query output
+1 years 3 months 16 days 1 hours 3 minutes 2.1002 seconds
+
+
 -- !query
 select interval
 -- !query schema

From 78df2caec8c94c31e5c9ddc30ed8acb424084181 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Fri, 1 May 2020 11:46:17 +0900
Subject: [PATCH 0613/1280] [SPARK-31619][CORE] Rename config
 "spark.dynamicAllocation.shuffleTimeout" to
 "spark.dynamicAllocation.shuffleTracking.timeout"

### What changes were proposed in this pull request?
The "spark.dynamicAllocation.shuffleTimeout" configuration only takes effect if "spark.dynamicAllocation.shuffleTracking.enabled" is true, so we should re-namespace that configuration so that it's nested under the "shuffleTracking" one.

### How was this patch tested?
Covered by current existing test cases.

Closes #28426 from jiangxb1987/confName.

Authored-by: Xingbo Jiang <xingbo.jiang@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b7cde42b04b21c9bfee6535199cf385855c15853)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/ExecutorAllocationManager.scala |  2 +-
 .../org/apache/spark/internal/config/package.scala   |  6 +++---
 .../spark/scheduler/dynalloc/ExecutorMonitor.scala   |  4 ++--
 .../scheduler/dynalloc/ExecutorMonitorSuite.scala    | 12 ++++++------
 docs/configuration.md                                |  2 +-
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 677386cc7a572..74103dc3e25b8 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -196,7 +196,7 @@ private[spark] class ExecutorAllocationManager(
         s"s${DYN_ALLOCATION_SUSTAINED_SCHEDULER_BACKLOG_TIMEOUT.key} must be > 0!")
     }
     if (!conf.get(config.SHUFFLE_SERVICE_ENABLED)) {
-      if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING)) {
+      if (conf.get(config.DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)) {
         logWarning("Dynamic allocation without a shuffle service is an experimental feature.")
       } else if (!testing) {
         throw new SparkException("Dynamic allocation of executors requires the external " +
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index f70ee2e5c71c7..bfdd06021757a 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -520,14 +520,14 @@ package object config {
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
       .createWithDefault(60)
 
-  private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING =
+  private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED =
     ConfigBuilder("spark.dynamicAllocation.shuffleTracking.enabled")
       .version("3.0.0")
       .booleanConf
       .createWithDefault(false)
 
-  private[spark] val DYN_ALLOCATION_SHUFFLE_TIMEOUT =
-    ConfigBuilder("spark.dynamicAllocation.shuffleTimeout")
+  private[spark] val DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT =
+    ConfigBuilder("spark.dynamicAllocation.shuffleTracking.timeout")
       .version("3.0.0")
       .timeConf(TimeUnit.MILLISECONDS)
       .checkValue(_ >= 0L, "Timeout must be >= 0.")
diff --git a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
index a24f1902faa31..3664d3ff4d8cf 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala
@@ -45,12 +45,12 @@ private[spark] class ExecutorMonitor(
   private val storageTimeoutNs = TimeUnit.SECONDS.toNanos(
     conf.get(DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT))
   private val shuffleTimeoutNs = TimeUnit.MILLISECONDS.toNanos(
-    conf.get(DYN_ALLOCATION_SHUFFLE_TIMEOUT))
+    conf.get(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT))
 
   private val fetchFromShuffleSvcEnabled = conf.get(SHUFFLE_SERVICE_ENABLED) &&
     conf.get(SHUFFLE_SERVICE_FETCH_RDD_ENABLED)
   private val shuffleTrackingEnabled = !conf.get(SHUFFLE_SERVICE_ENABLED) &&
-    conf.get(DYN_ALLOCATION_SHUFFLE_TRACKING)
+    conf.get(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED)
 
   private val executors = new ConcurrentHashMap[String, Tracker]()
   private val execResourceProfileCount = new ConcurrentHashMap[Int, Int]()
diff --git a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
index 615389ae5c2d4..c5f73150f5db5 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitorSuite.scala
@@ -42,7 +42,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
   private val conf = new SparkConf()
     .set(DYN_ALLOCATION_EXECUTOR_IDLE_TIMEOUT.key, "60s")
     .set(DYN_ALLOCATION_CACHED_EXECUTOR_IDLE_TIMEOUT.key, "120s")
-    .set(DYN_ALLOCATION_SHUFFLE_TIMEOUT.key, "240s")
+    .set(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT.key, "240s")
     .set(SHUFFLE_SERVICE_ENABLED, true)
 
   private var monitor: ExecutorMonitor = _
@@ -287,7 +287,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("shuffle block tracking") {
     val bus = mockListenerBus()
-    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING, true).set(SHUFFLE_SERVICE_ENABLED, false)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, false)
     monitor = new ExecutorMonitor(conf, client, bus, clock)
 
     // 3 jobs: 2 and 3 share a shuffle, 1 has a separate shuffle.
@@ -355,7 +355,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("SPARK-28839: Avoids NPE in context cleaner when shuffle service is on") {
     val bus = mockListenerBus()
-    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING, true).set(SHUFFLE_SERVICE_ENABLED, true)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, true)
     monitor = new ExecutorMonitor(conf, client, bus, clock) {
       override def onOtherEvent(event: SparkListenerEvent): Unit = {
         throw new IllegalStateException("No event should be sent.")
@@ -367,7 +367,7 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("shuffle tracking with multiple executors and concurrent jobs") {
     val bus = mockListenerBus()
-    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING, true).set(SHUFFLE_SERVICE_ENABLED, false)
+    conf.set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true).set(SHUFFLE_SERVICE_ENABLED, false)
     monitor = new ExecutorMonitor(conf, client, bus, clock)
 
     monitor.onExecutorAdded(SparkListenerExecutorAdded(clock.getTimeMillis(), "1", execInfo))
@@ -410,8 +410,8 @@ class ExecutorMonitorSuite extends SparkFunSuite {
 
   test("SPARK-28455: avoid overflow in timeout calculation") {
     conf
-      .set(DYN_ALLOCATION_SHUFFLE_TIMEOUT, Long.MaxValue)
-      .set(DYN_ALLOCATION_SHUFFLE_TRACKING, true)
+      .set(DYN_ALLOCATION_SHUFFLE_TRACKING_TIMEOUT, Long.MaxValue)
+      .set(DYN_ALLOCATION_SHUFFLE_TRACKING_ENABLED, true)
       .set(SHUFFLE_SERVICE_ENABLED, false)
     monitor = new ExecutorMonitor(conf, client, null, clock)
 
diff --git a/docs/configuration.md b/docs/configuration.md
index a97d3e4c33d44..2701fdb030ce8 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -2551,7 +2551,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>3.0.0</td>
 </tr>
 <tr>
-  <td><code>spark.dynamicAllocation.shuffleTimeout</code></td>
+  <td><code>spark.dynamicAllocation.shuffleTracking.timeout</code></td>
   <td><code>infinity</code></td>
   <td>
     When shuffle tracking is enabled, controls the timeout for executors that are holding shuffle

From cafd67e952f88580530c4102c85f58278c3472bb Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 1 May 2020 12:12:30 +0900
Subject: [PATCH 0614/1280] [SPARK-31549][PYSPARK][FOLLOW-UP] Remove a newline
 between class methods for linter check

---
 python/pyspark/tests/test_rdd.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index 6b11d6807bbb0..e2d910ccd15c3 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -783,7 +783,6 @@ def fail_last(x):
         for i in range(4):
             self.assertEqual(i, next(it))
 
-
     def test_multiple_group_jobs(self):
         import threading
         group_a = "job_ids_to_cancel"

From 7c6b9708b6fbc81d583081a7b027fe1cce493b6c Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Fri, 1 May 2020 18:37:41 +0900
Subject: [PATCH 0615/1280] [SPARK-31372][SQL][TEST][FOLLOWUP][3.0] Update the
 golden file of ExpressionsSchemaSuite

### What changes were proposed in this pull request?

This PR is a follow-up PR to update the golden file of `ExpressionsSchemaSuite`.

### Why are the changes needed?

To recover tests in branch-3.0.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #28427 from maropu/SPARK-31372-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../resources/sql-functions/sql-expression-schema.md     | 9 ++-------
 .../org/apache/spark/sql/ExpressionsSchemaSuite.scala    | 7 ++++++-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 1e22ae2eefeb2..2091de2ecfd9b 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -1,6 +1,6 @@
 <!-- Automatically generated byExpressionsSchemaSuite -->
 ## Summary
-  - Number of queries: 333
+  - Number of queries: 328
   - Number of expressions that missing example: 34
   - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,struct,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch
 ## Schema of Built-in Functions
@@ -123,7 +123,7 @@
 | org.apache.spark.sql.catalyst.expressions.GreaterThanOrEqual | >= | SELECT 2 >= 1 | struct<(2 >= 1):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Greatest | greatest | SELECT greatest(10, 9, 2, 4, 3) | struct<greatest(10, 9, 2, 4, 3):int> |
 | org.apache.spark.sql.catalyst.expressions.Grouping | grouping | SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name) | struct<name:string,grouping(name):tinyint,sum(age):bigint> |
-| org.apache.spark.sql.catalyst.expressions.GroupingID | grouping_id | SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height) | struct<name:string,grouping_id():bigint,sum(age):bigint,avg(height):double> |
+| org.apache.spark.sql.catalyst.expressions.GroupingID | grouping_id | SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height) | struct<name:string,grouping_id():int,sum(age):bigint,avg(height):double> |
 | org.apache.spark.sql.catalyst.expressions.Hex | hex | SELECT hex(17) | struct<hex(CAST(17 AS BIGINT)):string> |
 | org.apache.spark.sql.catalyst.expressions.Hour | hour | SELECT hour('2009-07-30 12:58:59') | struct<hour(CAST(2009-07-30 12:58:59 AS TIMESTAMP)):int> |
 | org.apache.spark.sql.catalyst.expressions.Hypot | hypot | SELECT hypot(3, 4) | struct<HYPOT(CAST(3 AS DOUBLE), CAST(4 AS DOUBLE)):double> |
@@ -140,7 +140,6 @@
 | org.apache.spark.sql.catalyst.expressions.IsNaN | isnan | SELECT isnan(cast('NaN' as double)) | struct<isnan(CAST(NaN AS DOUBLE)):boolean> |
 | org.apache.spark.sql.catalyst.expressions.IsNotNull | isnotnull | SELECT isnotnull(1) | struct<(1 IS NOT NULL):boolean> |
 | org.apache.spark.sql.catalyst.expressions.IsNull | isnull | SELECT isnull(1) | struct<(1 IS NULL):boolean> |
-| org.apache.spark.sql.catalyst.expressions.JsonObjectKeys | json_object_keys | SELECT json_object_keys('{}') | struct<json_object_keys({}):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.JsonToStructs | from_json | SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE') | struct<from_json({"a":1, "b":0.8}):struct<a:int,b:double>> |
 | org.apache.spark.sql.catalyst.expressions.JsonTuple | json_tuple | SELECT json_tuple('{"a":1, "b":2}', 'a', 'b') | struct<c0:string,c1:string> |
 | org.apache.spark.sql.catalyst.expressions.Lag | lag | N/A | N/A |
@@ -151,7 +150,6 @@
 | org.apache.spark.sql.catalyst.expressions.Length | character_length | SELECT character_length('Spark SQL ') | struct<character_length(Spark SQL ):int> |
 | org.apache.spark.sql.catalyst.expressions.Length | char_length | SELECT char_length('Spark SQL ') | struct<char_length(Spark SQL ):int> |
 | org.apache.spark.sql.catalyst.expressions.Length | length | SELECT length('Spark SQL ') | struct<length(Spark SQL ):int> |
-| org.apache.spark.sql.catalyst.expressions.LengthOfJsonArray | json_array_length | SELECT json_array_length('[1,2,3,4]') | struct<json_array_length([1,2,3,4]):int> |
 | org.apache.spark.sql.catalyst.expressions.LessThan | < | SELECT 1 < 2 | struct<(1 < 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.LessThanOrEqual | <= | SELECT 2 <= 2 | struct<(2 <= 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Levenshtein | levenshtein | SELECT levenshtein('kitten', 'sitting') | struct<levenshtein(kitten, sitting):int> |
@@ -206,9 +204,6 @@
 | org.apache.spark.sql.catalyst.expressions.Pow | power | SELECT power(2, 3) | struct<POWER(CAST(2 AS DOUBLE), CAST(3 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Quarter | quarter | SELECT quarter('2016-08-31') | struct<quarter(CAST(2016-08-31 AS DATE)):int> |
 | org.apache.spark.sql.catalyst.expressions.RLike | rlike | SELECT '%SystemDrive%\Users\John' rlike '%SystemDrive%\\Users.*' | struct<%SystemDrive%UsersJohn RLIKE %SystemDrive%\Users.*:boolean> |
-| org.apache.spark.sql.catalyst.expressions.Rand | random | SELECT random() | struct<rand():double> |
-| org.apache.spark.sql.catalyst.expressions.Rand | rand | SELECT rand() | struct<rand():double> |
-| org.apache.spark.sql.catalyst.expressions.Randn | randn | SELECT randn() | struct<randn():double> |
 | org.apache.spark.sql.catalyst.expressions.Rank | rank | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.RegExpExtract | regexp_extract | SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1) | struct<regexp_extract(100-200, (\d+)-(\d+), 1):string> |
 | org.apache.spark.sql.catalyst.expressions.RegExpReplace | regexp_replace | SELECT regexp_replace('100-200', '(\\d+)', 'num') | struct<regexp_replace(100-200, (\d+), num):string> |
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
index dd72473f0ea6c..024d79fc9de9b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -84,6 +84,11 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
 
   private val resultFile = new File(baseResourcePath, "sql-expression-schema.md")
 
+  private val ignoreSet = Set(
+    // Output name with a random seed
+    "org.apache.spark.sql.catalyst.expressions.Rand",
+    "org.apache.spark.sql.catalyst.expressions.Randn")
+
   /** A single SQL query's SQL and schema. */
   protected case class QueryOutput(
       className: String,
@@ -106,7 +111,7 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
     val outputs = new ArrayBuffer[QueryOutput]
     val missingExamples = new ArrayBuffer[String]
 
-    classFunsMap.foreach { kv =>
+    classFunsMap.filterNot(v => ignoreSet.contains(v._1)).foreach { kv =>
       val className = kv._1
       kv._2.foreach { funInfo =>
         val example = funInfo.getExamples

From 1795a70bb04fad1b8cf76271443a448f8d72fc8a Mon Sep 17 00:00:00 2001
From: Pablo Langa <soypab@gmail.com>
Date: Fri, 1 May 2020 22:09:04 +0900
Subject: [PATCH 0616/1280] [SPARK-31500][SQL] collect_set() of BinaryType
 returns duplicate elements

### What changes were proposed in this pull request?

The collect_set() aggregate function should produce a set of distinct elements. When the column argument's type is BinayType this is not the case.

Example:
```scala
import org.apache.spark.sql.functions._
import org.apache.spark.sql.expressions.Window

case class R(id: String, value: String, bytes: Array[Byte])
def makeR(id: String, value: String) = R(id, value, value.getBytes)
val df = Seq(makeR("a", "dog"), makeR("a", "cat"), makeR("a", "cat"), makeR("b", "fish")).toDF()
// In the example below "bytesSet" erroneously has duplicates but "stringSet" does not (as expected).
df.agg(collect_set('value) as "stringSet", collect_set('bytes) as "byteSet").show(truncate=false)
// The same problem is displayed when using window functions.
val win = Window.partitionBy('id).rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)
val result = df.select(
  collect_set('value).over(win) as "stringSet",
  collect_set('bytes).over(win) as "bytesSet"
)
.select('stringSet, 'bytesSet, size('stringSet) as "stringSetSize", size('bytesSet) as "bytesSetSize")
.show()
```

We use a HashSet buffer to accumulate the results, the problem is that arrays equality in Scala don't behave as expected, arrays ara just plain java arrays and the equality don't compare the content of the arrays
Array(1, 2, 3) == Array(1, 2, 3)  => False
The result is that duplicates are not removed in the hashset

The solution proposed is that in the last stage, when we have all the data in the Hashset buffer, we delete duplicates changing the type of the elements and then transform it to the original type.
This transformation is only applied when we have a BinaryType

### Why are the changes needed?
Fix the bug explained

### Does this PR introduce any user-facing change?
Yes. Now `collect_set()` correctly deduplicates array of byte.

### How was this patch tested?
Unit testing

Closes #28351 from planga82/feature/SPARK-31500_COLLECT_SET_bug.

Authored-by: Pablo Langa <soypab@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4fecc20f6ecdfe642890cf0a368a85558c40a47c)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../expressions/aggregate/collect.scala       | 45 ++++++++++++++++---
 .../spark/sql/DataFrameAggregateSuite.scala   | 16 +++++++
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
index 5848aa3f840c6..0a3d87623be8b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala
@@ -23,6 +23,7 @@ import scala.collection.mutable
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.util.ArrayData
 import org.apache.spark.sql.catalyst.util.GenericArrayData
 import org.apache.spark.sql.types._
 
@@ -46,13 +47,15 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
   // actual order of input rows.
   override lazy val deterministic: Boolean = false
 
+  protected def convertToBufferElement(value: Any): Any
+
   override def update(buffer: T, input: InternalRow): T = {
     val value = child.eval(input)
 
     // Do not allow null values. We follow the semantics of Hive's collect_list/collect_set here.
     // See: org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMkCollectionEvaluator
     if (value != null) {
-      buffer += InternalRow.copyValue(value)
+      buffer += convertToBufferElement(value)
     }
     buffer
   }
@@ -61,12 +64,10 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
     buffer ++= other
   }
 
-  override def eval(buffer: T): Any = {
-    new GenericArrayData(buffer.toArray)
-  }
+  protected val bufferElementType: DataType
 
   private lazy val projection = UnsafeProjection.create(
-    Array[DataType](ArrayType(elementType = child.dataType, containsNull = false)))
+    Array[DataType](ArrayType(elementType = bufferElementType, containsNull = false)))
   private lazy val row = new UnsafeRow(1)
 
   override def serialize(obj: T): Array[Byte] = {
@@ -77,7 +78,7 @@ abstract class Collect[T <: Growable[Any] with Iterable[Any]] extends TypedImper
   override def deserialize(bytes: Array[Byte]): T = {
     val buffer = createAggregationBuffer()
     row.pointTo(bytes, bytes.length)
-    row.getArray(0).foreach(child.dataType, (_, x: Any) => buffer += x)
+    row.getArray(0).foreach(bufferElementType, (_, x: Any) => buffer += x)
     buffer
   }
 }
@@ -105,6 +106,10 @@ case class CollectList(
 
   def this(child: Expression) = this(child, 0, 0)
 
+  override lazy val bufferElementType = child.dataType
+
+  override def convertToBufferElement(value: Any): Any = InternalRow.copyValue(value)
+
   override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
     copy(mutableAggBufferOffset = newMutableAggBufferOffset)
 
@@ -114,6 +119,10 @@ case class CollectList(
   override def createAggregationBuffer(): mutable.ArrayBuffer[Any] = mutable.ArrayBuffer.empty
 
   override def prettyName: String = "collect_list"
+
+  override def eval(buffer: mutable.ArrayBuffer[Any]): Any = {
+    new GenericArrayData(buffer.toArray)
+  }
 }
 
 /**
@@ -139,6 +148,30 @@ case class CollectSet(
 
   def this(child: Expression) = this(child, 0, 0)
 
+  override lazy val bufferElementType = child.dataType match {
+    case BinaryType => ArrayType(ByteType)
+    case other => other
+  }
+
+  override def convertToBufferElement(value: Any): Any = child.dataType match {
+    /*
+     * collect_set() of BinaryType should not return duplicate elements,
+     * Java byte arrays use referential equality and identity hash codes
+     * so we need to use a different catalyst value for arrays
+     */
+    case BinaryType => UnsafeArrayData.fromPrimitiveArray(value.asInstanceOf[Array[Byte]])
+    case _ => InternalRow.copyValue(value)
+  }
+
+  override def eval(buffer: mutable.HashSet[Any]): Any = {
+    val array = child.dataType match {
+      case BinaryType =>
+        buffer.iterator.map(_.asInstanceOf[ArrayData].toByteArray).toArray
+      case _ => buffer.toArray
+    }
+    new GenericArrayData(array)
+  }
+
   override def checkInputDataTypes(): TypeCheckResult = {
     if (!child.dataType.existsRecursively(_.isInstanceOf[MapType])) {
       TypeCheckResult.TypeCheckSuccess
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 288f3dac36621..4edf3a5d39fdd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -530,6 +530,22 @@ class DataFrameAggregateSuite extends QueryTest
     )
   }
 
+  test("SPARK-31500: collect_set() of BinaryType returns duplicate elements") {
+    val bytesTest1 = "test1".getBytes
+    val bytesTest2 = "test2".getBytes
+    val df = Seq(bytesTest1, bytesTest1, bytesTest2).toDF("a")
+    checkAnswer(df.select(size(collect_set($"a"))), Row(2) :: Nil)
+
+    val a = "aa".getBytes
+    val b = "bb".getBytes
+    val c = "cc".getBytes
+    val d = "dd".getBytes
+    val df1 = Seq((a, b), (a, b), (c, d))
+      .toDF("x", "y")
+      .select(struct($"x", $"y").as("a"))
+    checkAnswer(df1.select(size(collect_set($"a"))), Row(2) :: Nil)
+  }
+
   test("collect_set functions cannot have maps") {
     val df = Seq((1, 3, 0), (2, 3, 0), (3, 4, 1))
       .toDF("a", "x", "y")

From 7ddaaed4feec6e227f5178b1d888730751ba6d29 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 1 May 2020 10:11:43 -0700
Subject: [PATCH 0617/1280] [MINOR][SQL][DOCS] Remove two leading spaces from
 sql tables

### What changes were proposed in this pull request?
Remove two leading spaces from sql tables.

### Why are the changes needed?

Follow the format of other references such as https://docs.snowflake.com/en/sql-reference/constructs/join.html, https://docs.oracle.com/cd/B19306_01/server.102/b14200/statements_10002.htm, https://www.postgresql.org/docs/10/sql-select.html.

### Does this PR introduce any user-facing change?

before
```
SELECT * FROM  test;
  +-+
  ...
  +-+
```
after
```
SELECT * FROM  test;
+-+
...
+-+
```

### How was this patch tested?
Manually build and check

Closes #28348 from huaxingao/sql-format.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 75da05038b68839c2b665675c80455826fc426b5)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 docs/sql-ref-ansi-compliance.md               |  40 +-
 docs/sql-ref-functions-udf-hive.md            |  82 +--
 docs/sql-ref-null-semantics.md                | 512 +++++++++---------
 docs/sql-ref-syntax-aux-analyze-table.md      |  88 +--
 docs/sql-ref-syntax-aux-conf-mgmt-set.md      |  10 +-
 docs/sql-ref-syntax-aux-describe-database.md  |  44 +-
 docs/sql-ref-syntax-aux-describe-function.md  |  84 +--
 docs/sql-ref-syntax-aux-describe-query.md     |  60 +-
 docs/sql-ref-syntax-aux-describe-table.md     | 164 +++---
 docs/sql-ref-syntax-aux-show-columns.md       |  42 +-
 docs/sql-ref-syntax-aux-show-create-table.md  |  20 +-
 docs/sql-ref-syntax-aux-show-databases.md     |  40 +-
 docs/sql-ref-syntax-aux-show-functions.md     |  96 ++--
 docs/sql-ref-syntax-aux-show-partitions.md    |  60 +-
 docs/sql-ref-syntax-aux-show-table.md         | 178 +++---
 docs/sql-ref-syntax-aux-show-tables.md        |  64 +--
 docs/sql-ref-syntax-aux-show-tblproperties.md |  48 +-
 docs/sql-ref-syntax-aux-show-views.md         |  68 +--
 docs/sql-ref-syntax-ddl-alter-database.md     |  16 +-
 docs/sql-ref-syntax-ddl-alter-table.md        | 252 ++++-----
 docs/sql-ref-syntax-ddl-alter-view.md         | 112 ++--
 docs/sql-ref-syntax-ddl-create-database.md    |  16 +-
 docs/sql-ref-syntax-ddl-create-function.md    |  46 +-
 docs/sql-ref-syntax-ddl-drop-function.md      |  32 +-
 docs/sql-ref-syntax-ddl-repair-table.md       |  18 +-
 docs/sql-ref-syntax-ddl-truncate-table.md     |  32 +-
 docs/sql-ref-syntax-dml-insert-into.md        | 164 +++---
 ...l-ref-syntax-dml-insert-overwrite-table.md | 124 ++---
 docs/sql-ref-syntax-dml-load.md               |  44 +-
 docs/sql-ref-syntax-qry-aggregation.md        |  22 -
 docs/sql-ref-syntax-qry-explain.md            | 100 ++--
 docs/sql-ref-syntax-qry-sampling.md           |  82 +--
 docs/sql-ref-syntax-qry-select-clusterby.md   |  40 +-
 docs/sql-ref-syntax-qry-select-cte.md         |  60 +-
 docs/sql-ref-syntax-qry-select-distinct.md    |  22 -
 ...sql-ref-syntax-qry-select-distribute-by.md |  40 +-
 docs/sql-ref-syntax-qry-select-groupby.md     | 216 ++++----
 docs/sql-ref-syntax-qry-select-having.md      |  68 +--
 .../sql-ref-syntax-qry-select-inline-table.md |  36 +-
 docs/sql-ref-syntax-qry-select-join.md        | 175 +++---
 docs/sql-ref-syntax-qry-select-limit.md       |  50 +-
 docs/sql-ref-syntax-qry-select-orderby.md     |  90 +--
 docs/sql-ref-syntax-qry-select-setops.md      | 190 +++----
 docs/sql-ref-syntax-qry-select-sortby.md      | 132 ++---
 docs/sql-ref-syntax-qry-select-tvf.md         |  68 +--
 docs/sql-ref-syntax-qry-select-where.md       |  82 +--
 docs/sql-ref-syntax-qry-window.md             | 168 +++---
 47 files changed, 2076 insertions(+), 2121 deletions(-)
 delete mode 100644 docs/sql-ref-syntax-qry-aggregation.md
 delete mode 100644 docs/sql-ref-syntax-qry-select-distinct.md

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 3a0c2c123d81b..6cf16535eecf8 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -73,11 +73,11 @@ SELECT 2147483647 + 1;
 
 -- `spark.sql.ansi.enabled=false`
 SELECT 2147483647 + 1;
-  +----------------+
-  |(2147483647 + 1)|
-  +----------------+
-  |     -2147483648|
-  +----------------+
++----------------+
+|(2147483647 + 1)|
++----------------+
+|     -2147483648|
++----------------+
 {% endhighlight %}
 
 ### Type Conversion
@@ -101,18 +101,18 @@ SELECT CAST(2147483648L AS INT);
 
 -- `spark.sql.ansi.enabled=false` (This is a default behaviour)
 SELECT CAST('a' AS INT);
-  +--------------+
-  |CAST(a AS INT)|
-  +--------------+
-  |          null|
-  +--------------+
++--------------+
+|CAST(a AS INT)|
++--------------+
+|          null|
++--------------+
 
 SELECT CAST(2147483648L AS INT);
-  +-----------------------+
-  |CAST(2147483648 AS INT)|
-  +-----------------------+
-  |            -2147483648|
-  +-----------------------+
++-----------------------+
+|CAST(2147483648 AS INT)|
++-----------------------+
+|            -2147483648|
++-----------------------+
 
 -- Examples of store assignment rules
 CREATE TABLE t (v INT);
@@ -125,11 +125,11 @@ INSERT INTO t VALUES ('1');
 -- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
 INSERT INTO t VALUES ('1');
 SELECT * FROM t;
-  +---+
-  |  v|
-  +---+
-  |  1|
-  +---+
++---+
+|  v|
++---+
+|  1|
++---+
 {% endhighlight %}
 
 ### SQL Functions
diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md
index a87266dec2ea1..97d72c44ef016 100644
--- a/docs/sql-ref-functions-udf-hive.md
+++ b/docs/sql-ref-functions-udf-hive.md
@@ -36,22 +36,22 @@ An example below uses [GenericUDFAbs](https://github.com/apache/hive/blob/master
 CREATE TEMPORARY FUNCTION testUDF AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs';
 
 SELECT * FROM t;
-  +-----+
-  |value|
-  +-----+
-  | -1.0|
-  |  2.0|
-  | -3.0|
-  +-----+
++-----+
+|value|
++-----+
+| -1.0|
+|  2.0|
+| -3.0|
++-----+
 
 SELECT testUDF(value) FROM t;
-  +--------------+
-  |testUDF(value)|
-  +--------------+
-  |           1.0|
-  |           2.0|
-  |           3.0|
-  +--------------+
++--------------+
+|testUDF(value)|
++--------------+
+|           1.0|
+|           2.0|
+|           3.0|
++--------------+
 {% endhighlight %}
 
 
@@ -63,22 +63,22 @@ CREATE TEMPORARY FUNCTION hiveUDTF
     AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode';
 
 SELECT * FROM t;
-  +------+
-  | value|
-  +------+
-  |[1, 2]|
-  |[3, 4]|
-  +------+
++------+
+| value|
++------+
+|[1, 2]|
+|[3, 4]|
++------+
 
 SELECT hiveUDTF(value) FROM t;
-  +---+
-  |col|
-  +---+
-  |  1|
-  |  2|
-  |  3|
-  |  4|
-  +---+
++---+
+|col|
++---+
+|  1|
+|  2|
+|  3|
+|  4|
++---+
 {% endhighlight %}
 
 Hive has two UDAF interfaces: [UDAF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDAF.java) and [GenericUDAFResolver](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver.java).
@@ -90,19 +90,19 @@ CREATE TEMPORARY FUNCTION hiveUDAF
     AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum';
 
 SELECT * FROM t;
-  +---+-----+
-  |key|value|
-  +---+-----+
-  |  a|    1|
-  |  a|    2|
-  |  b|    3|
-  +---+-----+
++---+-----+
+|key|value|
++---+-----+
+|  a|    1|
+|  a|    2|
+|  b|    3|
++---+-----+
 
 SELECT key, hiveUDAF(value) FROM t GROUP BY key;
-  +---+---------------+
-  |key|hiveUDAF(value)|
-  +---+---------------+
-  |  b|              3|
-  |  a|              3|
-  +---+---------------+
++---+---------------+
+|key|hiveUDAF(value)|
++---+---------------+
+|  b|              3|
+|  a|              3|
++---+---------------+
 {% endhighlight %}
\ No newline at end of file
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index dc48a36cadb3c..d1f4f17596bd4 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -119,35 +119,35 @@ one or both operands are `NULL`:
 {% highlight sql %}
 -- Normal comparison operators return `NULL` when one of the operand is `NULL`.
 SELECT 5 > null AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 -- Normal comparison operators return `NULL` when both the operands are `NULL`.
 SELECT null = null AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 -- Null-safe equal operator return `False` when one of the operand is `NULL`
 SELECT 5 <=> null AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |            false|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|            false|
++-----------------+
 
 -- Null-safe equal operator return `True` when one of the operand is `NULL`
 SELECT NULL <=> NULL;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             true|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             true|
++-----------------+
 {% endhighlight %}
 
 ### Logical Operators <a name="logical-operators"></a>
@@ -212,27 +212,27 @@ The following tables illustrate the behavior of logical operators when one or bo
 {% highlight sql %}
 -- Normal comparison operators return `NULL` when one of the operands is `NULL`.
 SELECT (true OR null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             true|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             true|
++-----------------+
 
 -- Normal comparison operators return `NULL` when both the operands are `NULL`.
 SELECT (null OR false) AS expression_output
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 -- Null-safe equal operator returns `False` when one of the operands is `NULL`
 SELECT NOT(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 {% endhighlight %}
 
 ### Expressions <a name="expressions"></a>
@@ -254,25 +254,25 @@ expression are `NULL` and most of the expressions fall in this category.
 
 {% highlight sql %}
 SELECT concat('John', null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 SELECT positive(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 SELECT to_date(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 {% endhighlight %}
 
 #### Expressions That Can Process Null Value Operands <a name="can-process-null"></a>
@@ -298,34 +298,34 @@ returns the first non `NULL` value in its list of operands. However, `coalesce`
 
 {% highlight sql %}
 SELECT isnull(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             true|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             true|
++-----------------+
 
 -- Returns the first occurrence of non `NULL` value.
 SELECT coalesce(null, null, 3, null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |                3|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|                3|
++-----------------+
 
 -- Returns `NULL` as all its operands are `NULL`. 
 SELECT coalesce(null, null, null, null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |             null|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|             null|
++-----------------+
 
 SELECT isnan(null) AS expression_output;
-  +-----------------+
-  |expression_output|
-  +-----------------+
-  |            false|
-  +-----------------+
++-----------------+
+|expression_output|
++-----------------+
+|            false|
++-----------------+
 {% endhighlight %}
 
 #### Builtin Aggregate Expressions <a name="built-in-aggregate"></a>
@@ -349,44 +349,44 @@ the rules of how `NULL` values are handled by aggregate functions.
 {% highlight sql %}
 -- `count(*)` does not skip `NULL` values.
 SELECT count(*) FROM person;
-  +--------+
-  |count(1)|
-  +--------+
-  |       7|
-  +--------+
++--------+
+|count(1)|
++--------+
+|       7|
++--------+
 
 -- `NULL` values in column `age` are skipped from processing.
 SELECT count(age) FROM person;
-  +----------+
-  |count(age)|
-  +----------+
-  |         5|
-  +----------+
++----------+
+|count(age)|
++----------+
+|         5|
++----------+
 
 -- `count(*)` on an empty input set returns 0. This is unlike the other
 -- aggregate functions, such as `max`, which return `NULL`.
 SELECT count(*) FROM person where 1 = 0;
-  +--------+
-  |count(1)|
-  +--------+
-  |       0|
-  +--------+
++--------+
+|count(1)|
++--------+
+|       0|
++--------+
 
 -- `NULL` values are excluded from computation of maximum value.
 SELECT max(age) FROM person;
-  +--------+
-  |max(age)|
-  +--------+
-  |      50|
-  +--------+
++--------+
+|max(age)|
++--------+
+|      50|
++--------+
 
 -- `max` returns `NULL` on an empty input set.
 SELECT max(age) FROM person where 1 = 0;
-  +--------+
-  |max(age)|
-  +--------+
-  |    null|
-  +--------+
++--------+
+|max(age)|
++--------+
+|    null|
++--------+
 {% endhighlight %}
 
 ### Condition Expressions in WHERE, HAVING and JOIN Clauses <a name="condition-expressions"></a>
@@ -401,71 +401,71 @@ For all the three operators, a condition expression is a boolean expression and
 {% highlight sql %}
 -- Persons whose age is unknown (`NULL`) are filtered out from the result set.
 SELECT * FROM person WHERE age > 0;
-  +--------+---+
-  |    name|age|
-  +--------+---+
-  |Michelle| 30|
-  |    Fred| 50|
-  |    Mike| 18|
-  |     Dan| 50|
-  |     Joe| 30|
-  +--------+---+
++--------+---+
+|    name|age|
++--------+---+
+|Michelle| 30|
+|    Fred| 50|
+|    Mike| 18|
+|     Dan| 50|
+|     Joe| 30|
++--------+---+
 
 -- `IS NULL` expression is used in disjunction to select the persons
 -- with unknown (`NULL`) records.
 SELECT * FROM person WHERE age > 0 OR age IS NULL;
-  +--------+----+
-  |    name| age|
-  +--------+----+
-  |  Albert|null|
-  |Michelle|  30|
-  |    Fred|  50|
-  |    Mike|  18|
-  |     Dan|  50|
-  |   Marry|null|
-  |     Joe|  30|
-  +--------+----+
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|Michelle|  30|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
+|   Marry|null|
+|     Joe|  30|
++--------+----+
 
 -- Person with unknown(`NULL`) ages are skipped from processing.
 SELECT * FROM person GROUP BY age HAVING max(age) > 18;
-  +---+--------+                                                                  
-  |age|count(1)|
-  +---+--------+
-  | 50|       2|
-  | 30|       2|
-  +---+--------+
++---+--------+
+|age|count(1)|
++---+--------+
+| 50|       2|
+| 30|       2|
++---+--------+
 
 -- A self join case with a join condition `p1.age = p2.age AND p1.name = p2.name`.
 -- The persons with unknown age (`NULL`) are filtered out by the join operator.
 SELECT * FROM person p1, person p2
     WHERE p1.age = p2.age
     AND p1.name = p2.name;
-  +--------+---+--------+---+
-  |    name|age|    name|age|
-  +--------+---+--------+---+
-  |Michelle| 30|Michelle| 30|
-  |    Fred| 50|    Fred| 50|
-  |    Mike| 18|    Mike| 18|
-  |     Dan| 50|     Dan| 50|
-  |     Joe| 30|     Joe| 30|
-  +--------+---+--------+---+
++--------+---+--------+---+
+|    name|age|    name|age|
++--------+---+--------+---+
+|Michelle| 30|Michelle| 30|
+|    Fred| 50|    Fred| 50|
+|    Mike| 18|    Mike| 18|
+|     Dan| 50|     Dan| 50|
+|     Joe| 30|     Joe| 30|
++--------+---+--------+---+
 
 -- The age column from both legs of join are compared using null-safe equal which
 -- is why the persons with unknown age (`NULL`) are qualified by the join.
 SELECT * FROM person p1, person p2
     WHERE p1.age <=> p2.age
     AND p1.name = p2.name;
-  +--------+----+--------+----+
-  |    name| age|    name| age|
-  +--------+----+--------+----+
-  |  Albert|null|  Albert|null|
-  |Michelle|  30|Michelle|  30|
-  |    Fred|  50|    Fred|  50|
-  |    Mike|  18|    Mike|  18|
-  |     Dan|  50|     Dan|  50|
-  |   Marry|null|   Marry|null|
-  |     Joe|  30|     Joe|  30|
-  +--------+----+--------+----+
++--------+----+--------+----+
+|    name| age|    name| age|
++--------+----+--------+----+
+|  Albert|null|  Albert|null|
+|Michelle|  30|Michelle|  30|
+|    Fred|  50|    Fred|  50|
+|    Mike|  18|    Mike|  18|
+|     Dan|  50|     Dan|  50|
+|   Marry|null|   Marry|null|
+|     Joe|  30|     Joe|  30|
++--------+----+--------+----+
 {% endhighlight %}
 
 ### Aggregate Operator (GROUP BY, DISTINCT) <a name="aggregate-operator"></a>
@@ -480,25 +480,25 @@ standard and with other enterprise database management systems.
 {% highlight sql %}
 -- `NULL` values are put in one bucket in `GROUP BY` processing.
 SELECT age, count(*) FROM person GROUP BY age;
-  +----+--------+                                                                 
-  | age|count(1)|
-  +----+--------+
-  |null|       2|
-  |  50|       2|
-  |  30|       2|
-  |  18|       1|
-  +----+--------+
++----+--------+
+| age|count(1)|
++----+--------+
+|null|       2|
+|  50|       2|
+|  30|       2|
+|  18|       1|
++----+--------+
 
 -- All `NULL` ages are considered one distinct value in `DISTINCT` processing.
 SELECT DISTINCT age FROM person;
-  +----+
-  | age|
-  +----+
-  |null|
-  |  50|
-  |  30|
-  |  18|
-  +----+
++----+
+| age|
++----+
+|null|
+|  50|
+|  30|
+|  18|
++----+
 {% endhighlight %}
 
 ### Sort Operator (ORDER BY Clause) <a name="order-by"></a>
@@ -513,47 +513,47 @@ the `NULL` values are placed at first.
 -- `NULL` values are shown at first and other values
 -- are sorted in ascending way.
 SELECT age, name FROM person ORDER BY age;
-  +----+--------+
-  | age|    name|
-  +----+--------+
-  |null|   Marry|
-  |null|  Albert|
-  |  18|    Mike|
-  |  30|Michelle|
-  |  30|     Joe|
-  |  50|    Fred|
-  |  50|     Dan|
-  +----+--------+
++----+--------+
+| age|    name|
++----+--------+
+|null|   Marry|
+|null|  Albert|
+|  18|    Mike|
+|  30|Michelle|
+|  30|     Joe|
+|  50|    Fred|
+|  50|     Dan|
++----+--------+
 
 -- Column values other than `NULL` are sorted in ascending
 -- way and `NULL` values are shown at the last.
 SELECT age, name FROM person ORDER BY age NULLS LAST;
-  +----+--------+
-  | age|    name|
-  +----+--------+
-  |  18|    Mike|
-  |  30|Michelle|
-  |  30|     Joe|
-  |  50|     Dan|
-  |  50|    Fred|
-  |null|   Marry|
-  |null|  Albert|
-  +----+--------+
++----+--------+
+| age|    name|
++----+--------+
+|  18|    Mike|
+|  30|Michelle|
+|  30|     Joe|
+|  50|     Dan|
+|  50|    Fred|
+|null|   Marry|
+|null|  Albert|
++----+--------+
 
 -- Columns other than `NULL` values are sorted in descending
 -- and `NULL` values are shown at the last.
 SELECT age, name FROM person ORDER BY age DESC NULLS LAST;
-  +----+--------+
-  | age|    name|
-  +----+--------+
-  |  50|    Fred|
-  |  50|     Dan|
-  |  30|Michelle|
-  |  30|     Joe|
-  |  18|    Mike|
-  |null|   Marry|
-  |null|  Albert|
-  +----+--------+
++----+--------+
+| age|    name|
++----+--------+
+|  50|    Fred|
+|  50|     Dan|
+|  30|Michelle|
+|  30|     Joe|
+|  18|    Mike|
+|null|   Marry|
+|null|  Albert|
++----+--------+
 {% endhighlight %}
 
 ### Set Operators (UNION, INTERSECT, EXCEPT) <a name="set-operators"></a>
@@ -573,27 +573,27 @@ CREATE VIEW unknown_age SELECT * FROM person WHERE age IS NULL;
 SELECT name, age FROM person
     INTERSECT
     SELECT name, age from unknown_age;
-  +------+----+                                                                   
-  |  name| age|
-  +------+----+
-  |Albert|null|
-  | Marry|null|
-  +------+----+
++------+----+
+|  name| age|
++------+----+
+|Albert|null|
+| Marry|null|
++------+----+
 
 -- `NULL` values from two legs of the `EXCEPT` are not in output. 
 -- This basically shows that the comparison happens in a null-safe manner.
 SELECT age, name FROM person
     EXCEPT
     SELECT age FROM unknown_age;
-  +---+--------+                                                                  
-  |age|    name|
-  +---+--------+
-  | 30|     Joe|
-  | 50|    Fred|
-  | 30|Michelle|
-  | 18|    Mike|
-  | 50|     Dan|
-  +---+--------+
++---+--------+
+|age|    name|
++---+--------+
+| 30|     Joe|
+| 50|    Fred|
+| 30|Michelle|
+| 18|    Mike|
+| 50|     Dan|
++---+--------+
 
 -- Performs `UNION` operation between two sets of data. 
 -- The comparison between columns of the row ae done in
@@ -601,17 +601,17 @@ SELECT age, name FROM person
 SELECT name, age FROM person
     UNION 
     SELECT name, age FROM unknown_age;
-  +--------+----+                                                                 
-  |    name| age|
-  +--------+----+
-  |  Albert|null|
-  |     Joe|  30|
-  |Michelle|  30|
-  |   Marry|null|
-  |    Fred|  50|
-  |    Mike|  18|
-  |     Dan|  50|
-  +--------+----+
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|     Joe|  30|
+|Michelle|  30|
+|   Marry|null|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
++--------+----+
 {% endhighlight %}
 
 ### EXISTS/NOT EXISTS Subquery <a name="exists-not-exists"></a>
@@ -633,39 +633,39 @@ semijoins / anti-semijoins without special provisions for null awareness.
 -- Even if subquery produces rows with `NULL` values, the `EXISTS` expression
 -- evaluates to `TRUE` as the subquery produces 1 row.
 SELECT * FROM person WHERE EXISTS (SELECT null);
-  +--------+----+                                                                 
-  |    name| age|
-  +--------+----+
-  |  Albert|null|
-  |Michelle|  30|
-  |    Fred|  50|
-  |    Mike|  18|
-  |     Dan|  50|
-  |   Marry|null|
-  |     Joe|  30|
-  +--------+----+
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|Michelle|  30|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
+|   Marry|null|
+|     Joe|  30|
++--------+----+
 
 -- `NOT EXISTS` expression returns `FALSE`. It returns `TRUE` only when
 -- subquery produces no rows. In this case, it returns 1 row.
 SELECT * FROM person WHERE NOT EXISTS (SELECT null);
-  +----+---+
-  |name|age|
-  +----+---+
-  +----+---+
++----+---+
+|name|age|
++----+---+
++----+---+
 
 -- `NOT EXISTS` expression returns `TRUE`.
 SELECT * FROM person WHERE NOT EXISTS (SELECT 1 WHERE 1 = 0);
-  +--------+----+
-  |    name| age|
-  +--------+----+
-  |  Albert|null|
-  |Michelle|  30|
-  |    Fred|  50|
-  |    Mike|  18|
-  |     Dan|  50|
-  |   Marry|null|
-  |     Joe|  30|
-  +--------+----+
++--------+----+
+|    name| age|
++--------+----+
+|  Albert|null|
+|Michelle|  30|
+|    Fred|  50|
+|    Mike|  18|
+|     Dan|  50|
+|   Marry|null|
+|     Joe|  30|
++--------+----+
 {% endhighlight %}
 
 ### IN/NOT IN Subquery <a name="in-not-in"></a>
@@ -696,29 +696,29 @@ and because NOT UNKNOWN is again UNKNOWN.
 -- The subquery has only `NULL` value in its result set. Therefore,
 -- the result of `IN` predicate is UNKNOWN.
 SELECT * FROM person WHERE age IN (SELECT null);
-  +----+---+
-  |name|age|
-  +----+---+
-  +----+---+
++----+---+
+|name|age|
++----+---+
++----+---+
 
 -- The subquery has `NULL` value in the result set as well as a valid 
 -- value `50`. Rows with age = 50 are returned. 
 SELECT * FROM person
     WHERE age IN (SELECT age FROM VALUES (50), (null) sub(age));
-  +----+---+
-  |name|age|
-  +----+---+
-  |Fred| 50|
-  | Dan| 50|
-  +----+---+
++----+---+
+|name|age|
++----+---+
+|Fred| 50|
+| Dan| 50|
++----+---+
 
 -- Since subquery has `NULL` value in the result set, the `NOT IN`
 -- predicate would return UNKNOWN. Hence, no rows are
 -- qualified for this query.
 SELECT * FROM person
     WHERE age NOT IN (SELECT age FROM VALUES (50), (null) sub(age));
-  +----+---+
-  |name|age|
-  +----+---+
-  +----+---+
++----+---+
+|name|age|
++----+---+
++----+---+
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-aux-analyze-table.md b/docs/sql-ref-syntax-aux-analyze-table.md
index 739e692680233..f6a6c5f4bc555 100644
--- a/docs/sql-ref-syntax-aux-analyze-table.md
+++ b/docs/sql-ref-syntax-aux-analyze-table.md
@@ -79,60 +79,60 @@ INSERT INTO students PARTITION (student_id = 222222) VALUES ('John');
 ANALYZE TABLE students COMPUTE STATISTICS NOSCAN;
 
 DESC EXTENDED students;
-  +--------------------+--------------------+-------+
-  |            col_name|           data_type|comment|
-  +--------------------+--------------------+-------+
-  |                name|              string|   null|
-  |          student_id|                 int|   null|
-  |                 ...|                 ...|    ...|
-  |          Statistics|           864 bytes|       |
-  |                 ...|                 ...|    ...|
-  |  Partition Provider|             Catalog|       |
-  +--------------------+--------------------+-------+
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|          Statistics|           864 bytes|       |
+|                 ...|                 ...|    ...|
+|  Partition Provider|             Catalog|       |
++--------------------+--------------------+-------+
 
 ANALYZE TABLE students COMPUTE STATISTICS;
 
 DESC EXTENDED students;
-  +--------------------+--------------------+-------+
-  |            col_name|           data_type|comment|
-  +--------------------+--------------------+-------+
-  |                name|              string|   null|
-  |          student_id|                 int|   null|
-  |                 ...|                 ...|    ...|
-  |          Statistics|   864 bytes, 2 rows|       |
-  |                 ...|                 ...|    ...|
-  |  Partition Provider|             Catalog|       |
-  +--------------------+--------------------+-------+
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|          Statistics|   864 bytes, 2 rows|       |
+|                 ...|                 ...|    ...|
+|  Partition Provider|             Catalog|       |
++--------------------+--------------------+-------+
 
 ANALYZE TABLE students PARTITION (student_id = 111111) COMPUTE STATISTICS;
 
 DESC EXTENDED students PARTITION (student_id = 111111);
-  +--------------------+--------------------+-------+
-  |            col_name|           data_type|comment|
-  +--------------------+--------------------+-------+
-  |                name|              string|   null|
-  |          student_id|                 int|   null|
-  |                 ...|                 ...|    ...|
-  |Partition Statistics|   432 bytes, 1 rows|       |
-  |                 ...|                 ...|    ...|
-  |        OutputFormat|org.apache.hadoop...|       |
-  +--------------------+--------------------+-------+
++--------------------+--------------------+-------+
+|            col_name|           data_type|comment|
++--------------------+--------------------+-------+
+|                name|              string|   null|
+|          student_id|                 int|   null|
+|                 ...|                 ...|    ...|
+|Partition Statistics|   432 bytes, 1 rows|       |
+|                 ...|                 ...|    ...|
+|        OutputFormat|org.apache.hadoop...|       |
++--------------------+--------------------+-------+
 
 ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name;
 
 DESC EXTENDED students name;
-  +--------------+----------+
-  |     info_name|info_value|
-  +--------------+----------+
-  |      col_name|      name|
-  |     data_type|    string|
-  |       comment|      NULL|
-  |           min|      NULL|
-  |           max|      NULL|
-  |     num_nulls|         0|
-  |distinct_count|         2|
-  |   avg_col_len|         4|
-  |   max_col_len|         4|
-  |     histogram|      NULL|
-  +--------------+----------+
++--------------+----------+
+|     info_name|info_value|
++--------------+----------+
+|      col_name|      name|
+|     data_type|    string|
+|       comment|      NULL|
+|           min|      NULL|
+|           max|      NULL|
+|     num_nulls|         0|
+|distinct_count|         2|
+|   avg_col_len|         4|
+|   max_col_len|         4|
+|     histogram|      NULL|
++--------------+----------+
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-set.md b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
index 2ca51307c3aae..330a1a6a399ff 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-set.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
@@ -62,11 +62,11 @@ SET;
 
 -- List the value of specified property key.
 SET spark.sql.variable.substitute;
-  +-----------------------------+-----+
-  |                          key|value|
-  +-----------------------------+-----+
-  |spark.sql.variable.substitute|false|
-  +-----------------------------+-----+
++-----------------------------+-----+
+|                          key|value|
++-----------------------------+-----+
+|spark.sql.variable.substitute|false|
++-----------------------------+-----+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-describe-database.md b/docs/sql-ref-syntax-aux-describe-database.md
index 590438b3cf719..39a40ddac800f 100644
--- a/docs/sql-ref-syntax-aux-describe-database.md
+++ b/docs/sql-ref-syntax-aux-describe-database.md
@@ -52,13 +52,13 @@ CREATE DATABASE employees COMMENT 'For software companies';
 -- Returns Database Name, Description and Root location of the filesystem
 -- for the employees DATABASE.
 DESCRIBE DATABASE employees;
-  +-------------------------+-----------------------------+
-  |database_description_item|   database_description_value|
-  +-------------------------+-----------------------------+
-  |            Database Name|                    employees|
-  |              Description|       For software companies|
-  |                 Location|file:/Users/Temp/employees.db|
-  +-------------------------+-----------------------------+
++-------------------------+-----------------------------+
+|database_description_item|   database_description_value|
++-------------------------+-----------------------------+
+|            Database Name|                    employees|
+|              Description|       For software companies|
+|                 Location|file:/Users/Temp/employees.db|
++-------------------------+-----------------------------+
 
 -- Create employees DATABASE
 CREATE DATABASE employees COMMENT 'For software companies';
@@ -68,27 +68,27 @@ ALTER DATABASE employees SET DBPROPERTIES ('Create-by' = 'Kevin', 'Create-date'
 
 -- Describe employees DATABASE with EXTENDED option to return additional database properties
 DESCRIBE DATABASE EXTENDED employees;
-  +-------------------------+---------------------------------------------+
-  |database_description_item|                   database_description_value|
-  +-------------------------+---------------------------------------------+
-  |            Database Name|                                    employees|
-  |              Description|                       For software companies|
-  |                 Location|                file:/Users/Temp/employees.db|
-  |               Properties|((Create-by,kevin), (Create-date,09/01/2019))|
-  +-------------------------+---------------------------------------------+
++-------------------------+---------------------------------------------+
+|database_description_item|                   database_description_value|
++-------------------------+---------------------------------------------+
+|            Database Name|                                    employees|
+|              Description|                       For software companies|
+|                 Location|                file:/Users/Temp/employees.db|
+|               Properties|((Create-by,kevin), (Create-date,09/01/2019))|
++-------------------------+---------------------------------------------+
 
 -- Create deployment SCHEMA
 CREATE SCHEMA deployment COMMENT 'Deployment environment';
 
 -- Describe deployment, the DATABASE and SCHEMA are interchangeable, their meaning are the same.
 DESC DATABASE deployment;
-  +-------------------------+------------------------------+
-  |database_description_item|database_description_value    |
-  +-------------------------+------------------------------+
-  |            Database Name|                    deployment|
-  |              Description|        Deployment environment|
-  |                 Location|file:/Users/Temp/deployment.db|
-  +-------------------------+------------------------------+
++-------------------------+------------------------------+
+|database_description_item|database_description_value    |
++-------------------------+------------------------------+
+|            Database Name|                    deployment|
+|              Description|        Deployment environment|
+|                 Location|file:/Users/Temp/deployment.db|
++-------------------------+------------------------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-describe-function.md b/docs/sql-ref-syntax-aux-describe-function.md
index a4ff76bddf782..76c9efad2fa7d 100644
--- a/docs/sql-ref-syntax-aux-describe-function.md
+++ b/docs/sql-ref-syntax-aux-describe-function.md
@@ -54,59 +54,59 @@ metadata information is returned along with the extended usage information.
 -- Describe a builtin scalar function.
 -- Returns function name, implementing class and usage
 DESC FUNCTION abs;
-  +-------------------------------------------------------------------+
-  |function_desc                                                      |
-  +-------------------------------------------------------------------+
-  |Function: abs                                                      |
-  |Class: org.apache.spark.sql.catalyst.expressions.Abs               |
-  |Usage: abs(expr) - Returns the absolute value of the numeric value.|
-  +-------------------------------------------------------------------+
++-------------------------------------------------------------------+
+|function_desc                                                      |
++-------------------------------------------------------------------+
+|Function: abs                                                      |
+|Class: org.apache.spark.sql.catalyst.expressions.Abs               |
+|Usage: abs(expr) - Returns the absolute value of the numeric value.|
++-------------------------------------------------------------------+
 
 -- Describe a builtin scalar function.
 -- Returns function name, implementing class and usage and examples.
 DESC FUNCTION EXTENDED abs;
-  +-------------------------------------------------------------------+
-  |function_desc                                                      |
-  +-------------------------------------------------------------------+
-  |Function: abs                                                      |
-  |Class: org.apache.spark.sql.catalyst.expressions.Abs               |
-  |Usage: abs(expr) - Returns the absolute value of the numeric value.|
-  |Extended Usage:                                                    |
-  |    Examples:                                                      |
-  |      > SELECT abs(-1);                                            |
-  |       1                                                           |
-  |                                                                   |
-  +-------------------------------------------------------------------+
++-------------------------------------------------------------------+
+|function_desc                                                      |
++-------------------------------------------------------------------+
+|Function: abs                                                      |
+|Class: org.apache.spark.sql.catalyst.expressions.Abs               |
+|Usage: abs(expr) - Returns the absolute value of the numeric value.|
+|Extended Usage:                                                    |
+|    Examples:                                                      |
+|      > SELECT abs(-1);                                            |
+|       1                                                           |
+|                                                                   |
++-------------------------------------------------------------------+
 
 -- Describe a builtin aggregate function
 DESC FUNCTION max;
-  +--------------------------------------------------------------+
-  |function_desc                                                 |
-  +--------------------------------------------------------------+
-  |Function: max                                                 |
-  |Class: org.apache.spark.sql.catalyst.expressions.aggregate.Max|
-  |Usage: max(expr) - Returns the maximum value of `expr`.       |
-  +--------------------------------------------------------------+
++--------------------------------------------------------------+
+|function_desc                                                 |
++--------------------------------------------------------------+
+|Function: max                                                 |
+|Class: org.apache.spark.sql.catalyst.expressions.aggregate.Max|
+|Usage: max(expr) - Returns the maximum value of `expr`.       |
++--------------------------------------------------------------+
 
 -- Describe a builtin user defined aggregate function
 -- Returns function name, implementing class and usage and examples.
 DESC FUNCTION EXTENDED explode
-  +---------------------------------------------------------------+
-  |function_desc                                                  |
-  +---------------------------------------------------------------+
-  |Function: explode                                              |
-  |Class: org.apache.spark.sql.catalyst.expressions.Explode       | 
-  |Usage: explode(expr) - Separates the elements of array `expr`  |
-  | into multiple rows, or the elements of map `expr` into        |
-  | multiple rows and columns. Unless specified otherwise, uses   |
-  | the default column name `col` for elements of the array or    |
-  | `key` and `value` for the elements of the map.                |
-  |Extended Usage:                                                |
-  |    Examples:                                                  |
-  |      > SELECT explode(array(10, 20));                         |
-  |       10                                                      |
-  |       20                                                      |
-  +---------------------------------------------------------------+
++---------------------------------------------------------------+
+|function_desc                                                  |
++---------------------------------------------------------------+
+|Function: explode                                              |
+|Class: org.apache.spark.sql.catalyst.expressions.Explode       |
+|Usage: explode(expr) - Separates the elements of array `expr`  |
+| into multiple rows, or the elements of map `expr` into        |
+| multiple rows and columns. Unless specified otherwise, uses   |
+| the default column name `col` for elements of the array or    |
+| `key` and `value` for the elements of the map.                |
+|Extended Usage:                                                |
+|    Examples:                                                  |
+|      > SELECT explode(array(10, 20));                         |
+|       10                                                      |
+|       20                                                      |
++---------------------------------------------------------------+
 
 {% endhighlight %}
 
diff --git a/docs/sql-ref-syntax-aux-describe-query.md b/docs/sql-ref-syntax-aux-describe-query.md
index f64416adc556d..07ac39b4431f4 100644
--- a/docs/sql-ref-syntax-aux-describe-query.md
+++ b/docs/sql-ref-syntax-aux-describe-query.md
@@ -59,50 +59,50 @@ CREATE TABLE person (name STRING , age INT COMMENT 'Age column', address STRING)
 
 -- Returns column metadata information for a simple select query
 DESCRIBE QUERY select age, sum(age) FROM person GROUP BY age;
-  +--------+---------+----------+
-  |col_name|data_type|   comment|
-  +--------+---------+----------+
-  |     age|      int|Age column|
-  |sum(age)|   bigint|      null|
-  +--------+---------+----------+
++--------+---------+----------+
+|col_name|data_type|   comment|
++--------+---------+----------+
+|     age|      int|Age column|
+|sum(age)|   bigint|      null|
++--------+---------+----------+
 
 -- Returns column metadata information for common table expression (`CTE`).
 DESCRIBE QUERY WITH all_names_cte
     AS (SELECT name from person) SELECT * FROM all_names_cte;
-  +--------+---------+-------+
-  |col_name|data_type|comment|
-  +--------+---------+-------+
-  |    name|   string|   null|
-  +--------+---------+-------+
++--------+---------+-------+
+|col_name|data_type|comment|
++--------+---------+-------+
+|    name|   string|   null|
++--------+---------+-------+
 
 -- Returns column metadata information for a inline table.
 DESC QUERY VALUES(100, 'John', 10000.20D) AS employee(id, name, salary);
-  +--------+---------+-------+
-  |col_name|data_type|comment|
-  +--------+---------+-------+
-  |      id|      int|   null|
-  |    name|   string|   null|
-  |  salary|   double|   null|
-  +--------+---------+-------+
++--------+---------+-------+
+|col_name|data_type|comment|
++--------+---------+-------+
+|      id|      int|   null|
+|    name|   string|   null|
+|  salary|   double|   null|
++--------+---------+-------+
 
 -- Returns column metadata information for `TABLE` statement.
 DESC QUERY TABLE person;
-  +--------+---------+----------+
-  |col_name|data_type|   comment|
-  +--------+---------+----------+
-  |    name|   string|      null|
-  |     age|      int| Agecolumn|
-  | address|   string|      null|
-  +--------+---------+----------+
++--------+---------+----------+
+|col_name|data_type|   comment|
++--------+---------+----------+
+|    name|   string|      null|
+|     age|      int| Agecolumn|
+| address|   string|      null|
++--------+---------+----------+
 
 -- Returns column metadata information for a `FROM` statement.
 -- `QUERY` clause is optional and can be omitted.
 DESCRIBE FROM person SELECT age;
-  +--------+---------+----------+
-  |col_name|data_type|   comment|
-  +--------+---------+----------+
-  |     age|      int| Agecolumn|
-  +--------+---------+----------+
++--------+---------+----------+
+|col_name|data_type|   comment|
++--------+---------+----------+
+|     age|      int| Agecolumn|
++--------+---------+----------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-describe-table.md b/docs/sql-ref-syntax-aux-describe-table.md
index a8eee97b4dc1e..63bf056d785cc 100644
--- a/docs/sql-ref-syntax-aux-describe-table.md
+++ b/docs/sql-ref-syntax-aux-describe-table.md
@@ -88,101 +88,101 @@ INSERT INTO customer PARTITION (state = 'AR') VALUES (100, 'Mike');
     
 -- Returns basic metadata information for unqualified table `customer`
 DESCRIBE TABLE customer;
-  +-----------------------+---------+----------+
-  |               col_name|data_type|   comment|
-  +-----------------------+---------+----------+
-  |                cust_id|      int|      null|
-  |                   name|   string|Short name|
-  |                  state|   string|      null|
-  |# Partition Information|         |          |
-  |             # col_name|data_type|   comment|
-  |                  state|   string|      null|
-  +-----------------------+---------+----------+
++-----------------------+---------+----------+
+|               col_name|data_type|   comment|
++-----------------------+---------+----------+
+|                cust_id|      int|      null|
+|                   name|   string|Short name|
+|                  state|   string|      null|
+|# Partition Information|         |          |
+|             # col_name|data_type|   comment|
+|                  state|   string|      null|
++-----------------------+---------+----------+
 
 -- Returns basic metadata information for qualified table `customer`
 DESCRIBE TABLE salesdb.customer;
-  +-----------------------+---------+----------+
-  |               col_name|data_type|   comment|
-  +-----------------------+---------+----------+
-  |                cust_id|      int|      null|
-  |                   name|   string|Short name|
-  |                  state|   string|      null|
-  |# Partition Information|         |          |
-  |             # col_name|data_type|   comment|
-  |                  state|   string|      null|
-  +-----------------------+---------+----------+
++-----------------------+---------+----------+
+|               col_name|data_type|   comment|
++-----------------------+---------+----------+
+|                cust_id|      int|      null|
+|                   name|   string|Short name|
+|                  state|   string|      null|
+|# Partition Information|         |          |
+|             # col_name|data_type|   comment|
+|                  state|   string|      null|
++-----------------------+---------+----------+
 
 -- Returns additional metadata such as parent database, owner, access time etc.
 DESCRIBE TABLE EXTENDED customer;
-  +----------------------------+------------------------------+----------+
-  |                    col_name|                     data_type|   comment|
-  +----------------------------+------------------------------+----------+
-  |                     cust_id|                           int|      null|
-  |                        name|                        string|Short name|
-  |                       state|                        string|      null|
-  |     # Partition Information|                              |          |
-  |                  # col_name|                     data_type|   comment|
-  |                       state|                        string|      null|
-  |                            |                              |          |
-  |# Detailed Table Information|                              |          |
-  |                    Database|                       default|          |
-  |                       Table|                      customer|          |
-  |                       Owner|                 <TABLE OWNER>|          |
-  |                Created Time|  Tue Apr 07 22:56:34 JST 2020|          |
-  |                 Last Access|                       UNKNOWN|          |
-  |                  Created By|               <SPARK VERSION>|          |
-  |                        Type|                       MANAGED|          |
-  |                    Provider|                       parquet|          |
-  |                    Location|file:/tmp/salesdb.db/custom...|          |
-  |               Serde Library|org.apache.hadoop.hive.ql.i...|          |
-  |                 InputFormat|org.apache.hadoop.hive.ql.i...|          |
-  |                OutputFormat|org.apache.hadoop.hive.ql.i...|          |
-  |          Partition Provider|                       Catalog|          |
-  +----------------------------+------------------------------+----------+
++----------------------------+------------------------------+----------+
+|                    col_name|                     data_type|   comment|
++----------------------------+------------------------------+----------+
+|                     cust_id|                           int|      null|
+|                        name|                        string|Short name|
+|                       state|                        string|      null|
+|     # Partition Information|                              |          |
+|                  # col_name|                     data_type|   comment|
+|                       state|                        string|      null|
+|                            |                              |          |
+|# Detailed Table Information|                              |          |
+|                    Database|                       default|          |
+|                       Table|                      customer|          |
+|                       Owner|                 <TABLE OWNER>|          |
+|                Created Time|  Tue Apr 07 22:56:34 JST 2020|          |
+|                 Last Access|                       UNKNOWN|          |
+|                  Created By|               <SPARK VERSION>|          |
+|                        Type|                       MANAGED|          |
+|                    Provider|                       parquet|          |
+|                    Location|file:/tmp/salesdb.db/custom...|          |
+|               Serde Library|org.apache.hadoop.hive.ql.i...|          |
+|                 InputFormat|org.apache.hadoop.hive.ql.i...|          |
+|                OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+|          Partition Provider|                       Catalog|          |
++----------------------------+------------------------------+----------+
 
 -- Returns partition metadata such as partitioning column name, column type and comment.
 DESCRIBE TABLE EXTENDED customer PARTITION (state = 'AR');
-  +------------------------------+------------------------------+----------+
-  |                      col_name|                     data_type|   comment|
-  +------------------------------+------------------------------+----------+
-  |                       cust_id|                           int|      null|
-  |                          name|                        string|Short name|
-  |                         state|                        string|      null|
-  |       # Partition Information|                              |          |
-  |                    # col_name|                     data_type|   comment|
-  |                         state|                        string|      null|
-  |                              |                              |          |
-  |# Detailed Partition Inform...|                              |          |
-  |                      Database|                       default|          |
-  |                         Table|                      customer|          |
-  |              Partition Values|                    [state=AR]|          |
-  |                      Location|file:/tmp/salesdb.db/custom...|          |
-  |                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
-  |                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
-  |                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
-  |            Storage Properties|[serialization.format=1, pa...|          |
-  |          Partition Parameters|{transient_lastDdlTime=1586...|          |
-  |                  Created Time|  Tue Apr 07 23:05:43 JST 2020|          |
-  |                   Last Access|                       UNKNOWN|          |
-  |          Partition Statistics|                     659 bytes|          |
-  |                              |                              |          |
-  |         # Storage Information|                              |          |
-  |                      Location|file:/tmp/salesdb.db/custom...|          |
-  |                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
-  |                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
-  |                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
-  +------------------------------+------------------------------+----------+
++------------------------------+------------------------------+----------+
+|                      col_name|                     data_type|   comment|
++------------------------------+------------------------------+----------+
+|                       cust_id|                           int|      null|
+|                          name|                        string|Short name|
+|                         state|                        string|      null|
+|       # Partition Information|                              |          |
+|                    # col_name|                     data_type|   comment|
+|                         state|                        string|      null|
+|                              |                              |          |
+|# Detailed Partition Inform...|                              |          |
+|                      Database|                       default|          |
+|                         Table|                      customer|          |
+|              Partition Values|                    [state=AR]|          |
+|                      Location|file:/tmp/salesdb.db/custom...|          |
+|                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
+|                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
+|                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
+|            Storage Properties|[serialization.format=1, pa...|          |
+|          Partition Parameters|{transient_lastDdlTime=1586...|          |
+|                  Created Time|  Tue Apr 07 23:05:43 JST 2020|          |
+|                   Last Access|                       UNKNOWN|          |
+|          Partition Statistics|                     659 bytes|          |
+|                              |                              |          |
+|         # Storage Information|                              |          |
+|                      Location|file:/tmp/salesdb.db/custom...|          |
+|                 Serde Library|org.apache.hadoop.hive.ql.i...|          |
+|                   InputFormat|org.apache.hadoop.hive.ql.i...|          |
+|                  OutputFormat|org.apache.hadoop.hive.ql.i...|          |
++------------------------------+------------------------------+----------+
 
 -- Returns the metadata for `name` column.
 -- Optional `TABLE` clause is omitted and column is fully qualified.
 DESCRIBE customer salesdb.customer.name;
-  +---------+----------+
-  |info_name|info_value|
-  +---------+----------+
-  | col_name|      name|
-  |data_type|    string|
-  |  comment|Short name|
-  +---------+----------+
++---------+----------+
+|info_name|info_value|
++---------+----------+
+| col_name|      name|
+|data_type|    string|
+|  comment|Short name|
++---------+----------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-columns.md b/docs/sql-ref-syntax-aux-show-columns.md
index c8c90a9d343e7..7229bba23d2bf 100644
--- a/docs/sql-ref-syntax-aux-show-columns.md
+++ b/docs/sql-ref-syntax-aux-show-columns.md
@@ -62,33 +62,33 @@ CREATE TABLE customer(
 
 -- List the columns of `customer` table in current database.
 SHOW COLUMNS IN customer;
-  +---------+
-  | col_name|
-  +---------+
-  |  cust_cd|
-  |     name|
-  |cust_addr|
-  +---------+
++---------+
+| col_name|
++---------+
+|  cust_cd|
+|     name|
+|cust_addr|
++---------+
 
 -- List the columns of `customer` table in `salesdb` database.
 SHOW COLUMNS IN salesdb.customer;
-  +---------+
-  | col_name|
-  +---------+
-  |  cust_cd|
-  |     name|
-  |cust_addr|
-  +---------+
++---------+
+| col_name|
++---------+
+|  cust_cd|
+|     name|
+|cust_addr|
++---------+
 
 -- List the columns of `customer` table in `salesdb` database
 SHOW COLUMNS IN customer IN salesdb;
-  +---------+
-  | col_name|
-  +---------+
-  |  cust_cd|
-  |     name|
-  |cust_addr|
-  +---------+
++---------+
+| col_name|
++---------+
+|  cust_cd|
+|     name|
+|cust_addr|
++---------+
 ```
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-create-table.md b/docs/sql-ref-syntax-aux-show-create-table.md
index 0a37c96bfc5ab..47a5290f1d022 100644
--- a/docs/sql-ref-syntax-aux-show-create-table.md
+++ b/docs/sql-ref-syntax-aux-show-create-table.md
@@ -50,16 +50,16 @@ CREATE TABLE test (c INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     TBLPROPERTIES ('prop1' = 'value1', 'prop2' = 'value2');
 
 SHOW CREATE TABLE test;
-  +----------------------------------------------------+
-  |                                      createtab_stmt|
-  +----------------------------------------------------+
-  |CREATE TABLE `default`.`test` (`c` INT)
-   USING text
-   TBLPROPERTIES (
-     'transient_lastDdlTime' = '1586269021',
-     'prop1' = 'value1',
-     'prop2' = 'value2')
-  +----------------------------------------------------+
++----------------------------------------------------+
+|                                      createtab_stmt|
++----------------------------------------------------+
+|CREATE TABLE `default`.`test` (`c` INT)
+ USING text
+ TBLPROPERTIES (
+   'transient_lastDdlTime' = '1586269021',
+   'prop1' = 'value1',
+   'prop2' = 'value2')
++----------------------------------------------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-databases.md b/docs/sql-ref-syntax-aux-show-databases.md
index 359900928c0c5..c84898aa81459 100644
--- a/docs/sql-ref-syntax-aux-show-databases.md
+++ b/docs/sql-ref-syntax-aux-show-databases.md
@@ -57,32 +57,32 @@ CREATE DATABASE payments_db;
 
 -- Lists all the databases. 
 SHOW DATABASES;
-  +------------+
-  |databaseName|
-  +------------+
-  |     default|
-  | payments_db|
-  |  payroll_db|
-  +------------+
++------------+
+|databaseName|
++------------+
+|     default|
+| payments_db|
+|  payroll_db|
++------------+
   
 -- Lists databases with name starting with string pattern `pay`
 SHOW DATABASES LIKE 'pay*';
-  +------------+
-  |databaseName|
-  +------------+
-  | payments_db|
-  |  payroll_db|
-  +------------+
++------------+
+|databaseName|
++------------+
+| payments_db|
+|  payroll_db|
++------------+
   
 -- Lists all databases. Keywords SCHEMAS and DATABASES are interchangeable. 
 SHOW SCHEMAS;
-  +------------+
-  |databaseName|
-  +------------+
-  |     default|
-  | payments_db|
-  |  payroll_db|
-  +------------+
++------------+
+|databaseName|
++------------+
+|     default|
+| payments_db|
+|  payroll_db|
++------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-functions.md b/docs/sql-ref-syntax-aux-show-functions.md
index ed22a3a901c31..8a6de402c7f20 100644
--- a/docs/sql-ref-syntax-aux-show-functions.md
+++ b/docs/sql-ref-syntax-aux-show-functions.md
@@ -74,70 +74,70 @@ SHOW [ function_kind ] FUNCTIONS ( [ LIKE ] function_name | regex_pattern )
 -- List a system function `trim` by searching both user defined and system
 -- defined functions.
 SHOW FUNCTIONS trim;
-  +--------+
-  |function|
-  +--------+
-  |    trim|
-  +--------+
++--------+
+|function|
++--------+
+|    trim|
++--------+
 
 -- List a system function `concat` by searching system defined functions.
 SHOW SYSTEM FUNCTIONS concat;
-  +--------+
-  |function|
-  +--------+
-  |  concat|
-  +--------+
++--------+
+|function|
++--------+
+|  concat|
++--------+
 
 -- List a qualified function `max` from database `salesdb`. 
 SHOW SYSTEM FUNCTIONS salesdb.max;
-  +--------+
-  |function|
-  +--------+
-  |     max|
-  +--------+
++--------+
+|function|
++--------+
+|     max|
++--------+
 
 -- List all functions starting with `t`
 SHOW FUNCTIONS LIKE 't*';
-  +-----------------+
-  |         function|
-  +-----------------+
-  |              tan|
-  |             tanh|
-  |        timestamp|
-  |          tinyint|
-  |           to_csv|
-  |          to_date|
-  |          to_json|
-  |     to_timestamp|
-  |to_unix_timestamp|
-  | to_utc_timestamp|
-  |        transform|
-  |   transform_keys|
-  | transform_values|
-  |        translate|
-  |             trim|
-  |            trunc|
-  |           typeof|
-  +-----------------+
++-----------------+
+|         function|
++-----------------+
+|              tan|
+|             tanh|
+|        timestamp|
+|          tinyint|
+|           to_csv|
+|          to_date|
+|          to_json|
+|     to_timestamp|
+|to_unix_timestamp|
+| to_utc_timestamp|
+|        transform|
+|   transform_keys|
+| transform_values|
+|        translate|
+|             trim|
+|            trunc|
+|           typeof|
++-----------------+
 
 -- List all functions starting with `yea` or `windo`
 SHOW FUNCTIONS LIKE 'yea*|windo*';
-  +--------+
-  |function|
-  +--------+
-  |  window|
-  |    year|
-  +--------+
++--------+
+|function|
++--------+
+|  window|
+|    year|
++--------+
 
 -- Use normal regex pattern to list function names that has 4 characters
 -- with `t` as the starting character.
 SHOW FUNCTIONS LIKE 't[a-z][a-z][a-z]';
-  +--------+
-  |function|
-  +--------+
-  |    tanh|
-  |    trim|
-  +--------+
++--------+
+|function|
++--------+
+|    tanh|
+|    trim|
++--------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-partitions.md b/docs/sql-ref-syntax-aux-show-partitions.md
index 31b881ea8f141..592833b23eb09 100644
--- a/docs/sql-ref-syntax-aux-show-partitions.md
+++ b/docs/sql-ref-syntax-aux-show-partitions.md
@@ -67,48 +67,48 @@ INSERT INTO customer PARTITION (state = 'AZ', city = 'Peoria') VALUES (300, 'Dan
 
 -- Lists all partitions for table `customer`
 SHOW PARTITIONS customer;
-  +----------------------+
-  |             partition|
-  +----------------------+
-  |  state=AZ/city=Peoria|
-  | state=CA/city=Fremont|
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+|  state=AZ/city=Peoria|
+| state=CA/city=Fremont|
+|state=CA/city=San Jose|
++----------------------+
 
 -- Lists all partitions for the qualified table `customer`
 SHOW PARTITIONS salesdb.customer;
-  +----------------------+
-  |             partition|
-  +----------------------+
-  |  state=AZ/city=Peoria|
-  | state=CA/city=Fremont|
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+|  state=AZ/city=Peoria|
+| state=CA/city=Fremont|
+|state=CA/city=San Jose|
++----------------------+
 
 -- Specify a full partition spec to list specific partition
 SHOW PARTITIONS customer PARTITION (state = 'CA', city = 'Fremont');
-  +---------------------+
-  |            partition|
-  +---------------------+
-  |state=CA/city=Fremont|
-  +---------------------+
++---------------------+
+|            partition|
++---------------------+
+|state=CA/city=Fremont|
++---------------------+
 
 -- Specify a partial partition spec to list the specific partitions
 SHOW PARTITIONS customer PARTITION (state = 'CA');
-  +----------------------+
-  |             partition|
-  +----------------------+
-  | state=CA/city=Fremont|
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+| state=CA/city=Fremont|
+|state=CA/city=San Jose|
++----------------------+
 
 -- Specify a partial spec to list specific partition
 SHOW PARTITIONS customer PARTITION (city =  'San Jose');
-  +----------------------+
-  |             partition|
-  +----------------------+
-  |state=CA/city=San Jose|
-  +----------------------+
++----------------------+
+|             partition|
++----------------------+
+|state=CA/city=San Jose|
++----------------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index c688a99df3ae2..3f588045790b2 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -75,104 +75,104 @@ INSERT INTO employee PARTITION (grade = 2) VALUES ('suj');
 
  -- Show the details of the table
 SHOW TABLE EXTENDED LIKE 'employee';
-  +--------+---------+-----------+--------------------------------------------------------------+
-  |database|tableName|isTemporary|                         information                          |
-  +--------+---------+-----------+--------------------------------------------------------------+
-  |default |employee |false      |Database: default
-                                  Table: employee
-                                  Owner: root
-                                  Created Time: Fri Aug 30 15:10:21 IST 2019
-                                  Last Access: Thu Jan 01 05:30:00 IST 1970
-                                  Created By: Spark 3.0.0-SNAPSHOT
-                                  Type: MANAGED
-                                  Provider: hive
-                                  Table Properties: [transient_lastDdlTime=1567158021]
-                                  Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                  Serde Library: org.apache.hadoop.hive.serde2.lazy   
-                                  .LazySimpleSerDe
-                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                  OutputFormat: org.apache.hadoop.hive.ql.io
-                                  .HiveIgnoreKeyTextOutputFormat
-                                  Storage Properties: [serialization.format=1]
-                                  Partition Provider: Catalog
-                                  Partition Columns: [`grade`]
-                                  Schema: root
-                                   |-- name: string (nullable = true)
-                                   |-- grade: integer (nullable = true)
++--------+---------+-----------+--------------------------------------------------------------+
+|database|tableName|isTemporary|                         information                          |
++--------+---------+-----------+--------------------------------------------------------------+
+|default |employee |false      |Database: default
+                                Table: employee
+                                Owner: root
+                                Created Time: Fri Aug 30 15:10:21 IST 2019
+                                Last Access: Thu Jan 01 05:30:00 IST 1970
+                                Created By: Spark 3.0.0-SNAPSHOT
+                                Type: MANAGED
+                                Provider: hive
+                                Table Properties: [transient_lastDdlTime=1567158021]
+                                Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                .LazySimpleSerDe
+                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                OutputFormat: org.apache.hadoop.hive.ql.io
+                                .HiveIgnoreKeyTextOutputFormat
+                                Storage Properties: [serialization.format=1]
+                                Partition Provider: Catalog
+                                Partition Columns: [`grade`]
+                                Schema: root
+                                 |-- name: string (nullable = true)
+                                 |-- grade: integer (nullable = true)
                                                                                                             
-  +--------+---------+-----------+--------------------------------------------------------------+
++--------+---------+-----------+--------------------------------------------------------------+
 
 -- showing the multiple table details with pattern matching
 SHOW TABLE EXTENDED  LIKE `employe*`;
-  +--------+---------+-----------+--------------------------------------------------------------+
-  |database|tableName|isTemporary|                         information                          |
-  +--------+---------+-----------+--------------------------------------------------------------+
-  |default |employee |false      |Database: default
-                                  Table: employee
-                                  Owner: root
-                                  Created Time: Fri Aug 30 15:10:21 IST 2019
-                                  Last Access: Thu Jan 01 05:30:00 IST 1970
-                                  Created By: Spark 3.0.0-SNAPSHOT
-                                  Type: MANAGED
-                                  Provider: hive
-                                  Table Properties: [transient_lastDdlTime=1567158021]
-                                  Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                  Serde Library: org.apache.hadoop.hive.serde2.lazy
-                                  .LazySimpleSerDe
-                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                  OutputFormat: org.apache.hadoop.hive.ql.io
-                                  .HiveIgnoreKeyTextOutputFormat
-                                  Storage Properties: [serialization.format=1]
-                                  Partition Provider: Catalog
-                                  Partition Columns: [`grade`]
-                                  Schema: root
-                                   |-- name: string (nullable = true)
-                                   |-- grade: integer (nullable = true)
++--------+---------+-----------+--------------------------------------------------------------+
+|database|tableName|isTemporary|                         information                          |
++--------+---------+-----------+--------------------------------------------------------------+
+|default |employee |false      |Database: default
+                                Table: employee
+                                Owner: root
+                                Created Time: Fri Aug 30 15:10:21 IST 2019
+                                Last Access: Thu Jan 01 05:30:00 IST 1970
+                                Created By: Spark 3.0.0-SNAPSHOT
+                                Type: MANAGED
+                                Provider: hive
+                                Table Properties: [transient_lastDdlTime=1567158021]
+                                Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                .LazySimpleSerDe
+                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                OutputFormat: org.apache.hadoop.hive.ql.io
+                                .HiveIgnoreKeyTextOutputFormat
+                                Storage Properties: [serialization.format=1]
+                                Partition Provider: Catalog
+                                Partition Columns: [`grade`]
+                                Schema: root
+                                 |-- name: string (nullable = true)
+                                 |-- grade: integer (nullable = true)
   
-  |default |employee1|false      |Database: default
-                                  Table: employee1
-                                  Owner: root
-                                  Created Time: Fri Aug 30 15:22:33 IST 2019
-                                  Last Access: Thu Jan 01 05:30:00 IST 1970
-                                  Created By: Spark 3.0.0-SNAPSHOT
-                                  Type: MANAGED
-                                  Provider: hive
-                                  Table Properties: [transient_lastDdlTime=1567158753]
-                                  Location: file:/opt/spark1/spark/spark-warehouse/employee1
-                                  Serde Library: org.apache.hadoop.hive.serde2.lazy
-                                  .LazySimpleSerDe
-                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                  OutputFormat: org.apache.hadoop.hive.ql.io
-                                  .HiveIgnoreKeyTextOutputFormat
-                                  Storage Properties: [serialization.format=1]
-                                  Partition Provider: Catalog
-                                  Schema: root
-                                   |-- name: string (nullable = true)
+|default |employee1|false      |Database: default
+                                Table: employee1
+                                Owner: root
+                                Created Time: Fri Aug 30 15:22:33 IST 2019
+                                Last Access: Thu Jan 01 05:30:00 IST 1970
+                                Created By: Spark 3.0.0-SNAPSHOT
+                                Type: MANAGED
+                                Provider: hive
+                                Table Properties: [transient_lastDdlTime=1567158753]
+                                Location: file:/opt/spark1/spark/spark-warehouse/employee1
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                .LazySimpleSerDe
+                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                OutputFormat: org.apache.hadoop.hive.ql.io
+                                .HiveIgnoreKeyTextOutputFormat
+                                Storage Properties: [serialization.format=1]
+                                Partition Provider: Catalog
+                                Schema: root
+                                 |-- name: string (nullable = true)
                                                                                                                
-  +--------+---------+----------+---------------------------------------------------------------+
++--------+---------+----------+---------------------------------------------------------------+
   
 -- show partition file system details
 SHOW TABLE EXTENDED  IN `default` LIKE `employee` PARTITION (`grade=1`);
-  +--------+---------+-----------+--------------------------------------------------------------+
-  |database|tableName|isTemporary|                         information                          | 
-  +--------+---------+-----------+--------------------------------------------------------------+
-  |default |employee |false      |Partition Values: [grade=1]
-                                  Location: file:/opt/spark1/spark/spark-warehouse/employee
-                                  /grade=1
-                                  Serde Library: org.apache.hadoop.hive.serde2.lazy
-                                  .LazySimpleSerDe
-                                  InputFormat: org.apache.hadoop.mapred.TextInputFormat
-                                  OutputFormat: org.apache.hadoop.hive.ql.io
-                                  .HiveIgnoreKeyTextOutputFormat
-                                  Storage Properties: [serialization.format=1]
-                                  Partition Parameters: {rawDataSize=-1, numFiles=1, 
-                                  transient_lastDdlTime=1567158221, totalSize=4, 
-                                  COLUMN_STATS_ACCURATE=false, numRows=-1}
-                                  Created Time: Fri Aug 30 15:13:41 IST 2019
-                                  Last Access: Thu Jan 01 05:30:00 IST 1970
-                                  Partition Statistics: 4 bytes
-                                                                                                                                                                            |
-  +--------+---------+-----------+--------------------------------------------------------------+
++--------+---------+-----------+--------------------------------------------------------------+
+|database|tableName|isTemporary|                         information                          |
++--------+---------+-----------+--------------------------------------------------------------+
+|default |employee |false      |Partition Values: [grade=1]
+                                Location: file:/opt/spark1/spark/spark-warehouse/employee
+                                /grade=1
+                                Serde Library: org.apache.hadoop.hive.serde2.lazy
+                                .LazySimpleSerDe
+                                InputFormat: org.apache.hadoop.mapred.TextInputFormat
+                                OutputFormat: org.apache.hadoop.hive.ql.io
+                                .HiveIgnoreKeyTextOutputFormat
+                                Storage Properties: [serialization.format=1]
+                                Partition Parameters: {rawDataSize=-1, numFiles=1,
+                                transient_lastDdlTime=1567158221, totalSize=4,
+                                COLUMN_STATS_ACCURATE=false, numRows=-1}
+                                Created Time: Fri Aug 30 15:13:41 IST 2019
+                                Last Access: Thu Jan 01 05:30:00 IST 1970
+                                Partition Statistics: 4 bytes
+                                                                                                                                                                          |
++--------+---------+-----------+--------------------------------------------------------------+
 
 -- show partition file system details with regex fails as shown below
 SHOW TABLE EXTENDED  IN `default` LIKE `empl*` PARTITION (`grade=1`);
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index cd54d450a449b..62eb3ddb18b5c 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -57,50 +57,50 @@ SHOW TABLES [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
 {% highlight sql %}
 -- List all tables in default database
 SHOW TABLES;
-  +--------+---------+-----------+
-  |database|tableName|isTemporary|
-  +--------+---------+-----------+
-  | default|      sam|      false|
-  | default|     sam1|      false|
-  | default|      suj|      false|
-  +--------+---------+-----------+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+| default|      sam|      false|
+| default|     sam1|      false|
+| default|      suj|      false|
++--------+---------+-----------+
 
 -- List all tables from userdb database 
 SHOW TABLES FROM userdb;
-  +--------+---------+-----------+
-  |database|tableName|isTemporary|
-  +--------+---------+-----------+
-  |  userdb|    user1|      false|
-  |  userdb|    user2|      false|
-  +--------+---------+-----------+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+|  userdb|    user1|      false|
+|  userdb|    user2|      false|
++--------+---------+-----------+
 
 -- List all tables in userdb database
 SHOW TABLES IN userdb;
-  +--------+---------+-----------+
-  |database|tableName|isTemporary|
-  +--------+---------+-----------+
-  |  userdb|    user1|      false|
-  |  userdb|    user2|      false|
-  +--------+---------+-----------+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+|  userdb|    user1|      false|
+|  userdb|    user2|      false|
++--------+---------+-----------+
 
 -- List all tables from default database matching the pattern `sam*`
 SHOW TABLES FROM default LIKE 'sam*';
-  +--------+---------+-----------+
-  |database|tableName|isTemporary|
-  +--------+---------+-----------+
-  | default|      sam|      false|
-  | default|     sam1|      false|
-  +--------+---------+-----------+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+| default|      sam|      false|
+| default|     sam1|      false|
++--------+---------+-----------+
   
 -- List all tables matching the pattern `sam*|suj`
 SHOW TABLES LIKE 'sam*|suj';
-  +--------+---------+-----------+
-  |database|tableName|isTemporary|
-  +--------+---------+-----------+
-  | default|      sam|      false|
-  | default|     sam1|      false|
-  | default|      suj|      false|
-  +--------+---------+-----------+
++--------+---------+-----------+
+|database|tableName|isTemporary|
++--------+---------+-----------+
+| default|      sam|      false|
+| default|     sam1|      false|
+| default|      suj|      false|
++--------+---------+-----------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-tblproperties.md b/docs/sql-ref-syntax-aux-show-tblproperties.md
index 26e57ef0aba24..662aaad069dd9 100644
--- a/docs/sql-ref-syntax-aux-show-tblproperties.md
+++ b/docs/sql-ref-syntax-aux-show-tblproperties.md
@@ -76,40 +76,40 @@ CREATE TABLE customer(cust_code INT, name VARCHAR(100), cust_addr STRING)
 
 -- show all the user specified properties for table `customer`
 SHOW TBLPROPERTIES customer;
-  +---------------------+----------+
-  |                  key|     value|
-  +---------------------+----------+
-  |      created.by.user|      John|
-  |         created.date|01-01-2001|
-  |transient_lastDdlTime|1567554931|
-  +---------------------+----------+
++---------------------+----------+
+|                  key|     value|
++---------------------+----------+
+|      created.by.user|      John|
+|         created.date|01-01-2001|
+|transient_lastDdlTime|1567554931|
++---------------------+----------+
 
 -- show all the user specified properties for a qualified table `customer`
 -- in database `salesdb`
 SHOW TBLPROPERTIES salesdb.customer;
-  +---------------------+----------+
-  |                  key|     value|
-  +---------------------+----------+
-  |      created.by.user|      John|
-  |         created.date|01-01-2001|
-  |transient_lastDdlTime|1567554931|
-  +---------------------+----------+
++---------------------+----------+
+|                  key|     value|
++---------------------+----------+
+|      created.by.user|      John|
+|         created.date|01-01-2001|
+|transient_lastDdlTime|1567554931|
++---------------------+----------+
 
 -- show value for unquoted property key `created.by.user`
 SHOW TBLPROPERTIES customer (created.by.user);
-  +-----+
-  |value|
-  +-----+
-  | John|
-  +-----+
++-----+
+|value|
++-----+
+| John|
++-----+
 
 -- show value for property `created.date`` specified as string literal
 SHOW TBLPROPERTIES customer ('created.date');
-  +----------+
-  |     value|
-  +----------+
-  |01-01-2001|
-  +----------+
++----------+
+|     value|
++----------+
+|01-01-2001|
++----------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
index b1a8d3ba09014..29ad6caf140f8 100644
--- a/docs/sql-ref-syntax-aux-show-views.md
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -66,52 +66,52 @@ CREATE TEMP VIEW temp2 AS SELECT 1 as col1;
 
 -- List all views in default database
 SHOW VIEWS;
-  +-------------+------------+--------------+--+
-  | namespace   | viewName   | isTemporary  |
-  +-------------+------------+--------------+--+
-  | default     | sam        | false        |
-  | default     | sam1       | false        |
-  | default     | suj        | false        |
-  |             | temp2      | true         |
-  +-------------+------------+--------------+--+
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| default     | sam        | false        |
+| default     | sam1       | false        |
+| default     | suj        | false        |
+|             | temp2      | true         |
++-------------+------------+--------------+
 
 -- List all views from userdb database 
 SHOW VIEWS FROM userdb;
-  +-------------+------------+--------------+--+
-  | namespace   | viewName   | isTemporary  |
-  +-------------+------------+--------------+--+
-  | userdb      | user1      | false        |
-  | userdb      | user2      | false        |
-  |             | temp2      | true         |
-  +-------------+------------+--------------+--+
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| userdb      | user1      | false        |
+| userdb      | user2      | false        |
+|             | temp2      | true         |
++-------------+------------+--------------+
   
 -- List all views in global temp view database 
 SHOW VIEWS IN global_temp;
-  +-------------+------------+--------------+--+
-  | namespace   | viewName   | isTemporary  |
-  +-------------+------------+--------------+--+
-  | global_temp | temp1      | true         |
-  |             | temp2      | true         |
-  +-------------+------------+--------------+--+
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| global_temp | temp1      | true         |
+|             | temp2      | true         |
++-------------+------------+--------------+
 
 -- List all views from default database matching the pattern `sam*`
 SHOW VIEWS FROM default LIKE 'sam*';
-  +-----------+------------+--------------+--+
-  | namespace | viewName   | isTemporary  |
-  +-----------+------------+--------------+--+
-  | default   | sam        | false        |
-  | default   | sam1       | false        |
-  +-----------+------------+--------------+--+
++-----------+------------+--------------+
+| namespace | viewName   | isTemporary  |
++-----------+------------+--------------+
+| default   | sam        | false        |
+| default   | sam1       | false        |
++-----------+------------+--------------+
 
 -- List all views from the current database matching the pattern `sam|suj｜temp*`
 SHOW VIEWS LIKE 'sam|suj|temp*';
-  +-------------+------------+--------------+--+
-  | namespace   | viewName   | isTemporary  |
-  +-------------+------------+--------------+--+
-  | default     | sam        | false        |
-  | default     | suj        | false        |
-  |             | temp2      | true         |
-  +-------------+------------+--------------+--+
++-------------+------------+--------------+
+| namespace   | viewName   | isTemporary  |
++-------------+------------+--------------+
+| default     | sam        | false        |
+| default     | suj        | false        |
+|             | temp2      | true         |
++-------------+------------+--------------+
 
 {% endhighlight %}
 
diff --git a/docs/sql-ref-syntax-ddl-alter-database.md b/docs/sql-ref-syntax-ddl-alter-database.md
index 65b85dcae15e7..2d5860c2ea920 100644
--- a/docs/sql-ref-syntax-ddl-alter-database.md
+++ b/docs/sql-ref-syntax-ddl-alter-database.md
@@ -52,14 +52,14 @@ ALTER DATABASE inventory SET DBPROPERTIES ('Edited-by' = 'John', 'Edit-date' = '
 
 -- Verify that properties are set.
 DESCRIBE DATABASE EXTENDED inventory;
-   +-------------------------+------------------------------------------+
-   |database_description_item|                database_description_value|
-   +-------------------------+------------------------------------------+
-   |            Database Name|                                 inventory|
-   |              Description|                                          |
-   |                 Location|   file:/temp/spark-warehouse/inventory.db|
-   |               Properties|((Edit-date,01/01/2001), (Edited-by,John))|
-   +-------------------------+------------------------------------------+
++-------------------------+------------------------------------------+
+|database_description_item|                database_description_value|
++-------------------------+------------------------------------------+
+|            Database Name|                                 inventory|
+|              Description|                                          |
+|                 Location|   file:/temp/spark-warehouse/inventory.db|
+|               Properties|((Edit-date,01/01/2001), (Edited-by,John))|
++-------------------------+------------------------------------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index 0a74aa0f843e2..f81585fef3aae 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -282,186 +282,186 @@ ALTER TABLE table_identifier [ partition_spec ] SET LOCATION 'new_location'
 {% highlight sql %}
 -- RENAME table 
 DESC student;
-  +-----------------------+---------+-------+
-  |               col_name|data_type|comment|
-  +-----------------------+---------+-------+
-  |                   name|   string|   NULL|
-  |                 rollno|      int|   NULL|
-  |                    age|      int|   NULL|
-  |# Partition Information|         |       |
-  |             # col_name|data_type|comment|
-  |                    age|      int|   NULL|
-  +-----------------------+---------+-------+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
 ALTER TABLE Student RENAME TO StudentInfo;
 
 -- After Renaming the table
 DESC StudentInfo;
-  +-----------------------+---------+-------+
-  |               col_name|data_type|comment|
-  +-----------------------+---------+-------+
-  |                   name|   string|   NULL|
-  |                 rollno|      int|   NULL|
-  |                    age|      int|   NULL|
-  |# Partition Information|         |       |
-  |             # col_name|data_type|comment|
-  |                    age|      int|   NULL|
-  +-----------------------+---------+-------+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
 -- RENAME partition
 
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=10|
-  |   age=11|
-  |   age=12|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=10|
+|   age=11|
+|   age=12|
++---------+
 
 ALTER TABLE default.StudentInfo PARTITION (age='10') RENAME TO PARTITION (age='15');
 
 -- After renaming Partition
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
 
 -- Add new columns to a table
 DESC StudentInfo;
-  +-----------------------+---------+-------+
-  |               col_name|data_type|comment|
-  +-----------------------+---------+-------+
-  |                   name|   string|   NULL|
-  |                 rollno|      int|   NULL|
-  |                    age|      int|   NULL|
-  |# Partition Information|         |       |
-  |             # col_name|data_type|comment|
-  |                    age|      int|   NULL|
-  +-----------------------+---------+-------+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
 ALTER TABLE StudentInfo ADD columns (LastName string, DOB timestamp);
 
 -- After Adding New columns to the table
 DESC StudentInfo;
-  +-----------------------+---------+-------+
-  |               col_name|data_type|comment|
-  +-----------------------+---------+-------+
-  |                   name|   string|   NULL|
-  |                 rollno|      int|   NULL|
-  |               LastName|   string|   NULL|
-  |                    DOB|timestamp|   NULL|
-  |                    age|      int|   NULL|
-  |# Partition Information|         |       |
-  |             # col_name|data_type|comment|
-  |                    age|      int|   NULL|
-  +-----------------------+---------+-------+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|               LastName|   string|   NULL|
+|                    DOB|timestamp|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
 -- Add a new partition to a table 
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
 
 ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18);
 
 -- After adding a new partition to the table
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  |   age=18|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
+|   age=18|
++---------+
 
 -- Drop a partition from the table 
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  |   age=18|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
+|   age=18|
++---------+
 
 ALTER TABLE StudentInfo DROP IF EXISTS PARTITION (age=18);
 
 -- After dropping the partition of the table
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
 
 -- Adding multiple partitions to the table
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
++---------+
 
 ALTER TABLE StudentInfo ADD IF NOT EXISTS PARTITION (age=18) PARTITION (age=20);
 
 -- After adding multiple partitions to the table
 SHOW PARTITIONS StudentInfo;
-  +---------+
-  |partition|
-  +---------+
-  |   age=11|
-  |   age=12|
-  |   age=15|
-  |   age=18|
-  |   age=20|
-  +---------+
++---------+
+|partition|
++---------+
+|   age=11|
+|   age=12|
+|   age=15|
+|   age=18|
+|   age=20|
++---------+
 
 -- ALTER OR CHANGE COLUMNS
 DESC StudentInfo;
-  +-----------------------+---------+-------+
-  |               col_name|data_type|comment|
-  +-----------------------+---------+-------+
-  |                   name|   string|   NULL|
-  |                 rollno|      int|   NULL|
-  |               LastName|   string|   NULL|
-  |                    DOB|timestamp|   NULL|
-  |                    age|      int|   NULL|
-  |# Partition Information|         |       |
-  |             # col_name|data_type|comment|
-  |                    age|      int|   NULL|
-  +-----------------------+---------+-------+
++-----------------------+---------+-------+
+|               col_name|data_type|comment|
++-----------------------+---------+-------+
+|                   name|   string|   NULL|
+|                 rollno|      int|   NULL|
+|               LastName|   string|   NULL|
+|                    DOB|timestamp|   NULL|
+|                    age|      int|   NULL|
+|# Partition Information|         |       |
+|             # col_name|data_type|comment|
+|                    age|      int|   NULL|
++-----------------------+---------+-------+
 
 ALTER TABLE StudentInfo ALTER COLUMN name COMMENT "new comment";
 
 --After ALTER or CHANGE COLUMNS
 DESC StudentInfo;
-  +-----------------------+---------+-----------+
-  |               col_name|data_type|    comment|
-  +-----------------------+---------+-----------+
-  |                   name|   string|new comment|
-  |                 rollno|      int|       NULL|
-  |               LastName|   string|       NULL|
-  |                    DOB|timestamp|       NULL|
-  |                    age|      int|       NULL|
-  |# Partition Information|         |           |
-  |             # col_name|data_type|    comment|
-  |                    age|      int|       NULL|
-  +-----------------------+---------+-----------+
++-----------------------+---------+-----------+
+|               col_name|data_type|    comment|
++-----------------------+---------+-----------+
+|                   name|   string|new comment|
+|                 rollno|      int|       NULL|
+|               LastName|   string|       NULL|
+|                    DOB|timestamp|       NULL|
+|                    age|      int|       NULL|
+|# Partition Information|         |           |
+|             # col_name|data_type|    comment|
+|                    age|      int|       NULL|
++-----------------------+---------+-----------+
 
 -- Change the fileformat
 ALTER TABLE loc_orc SET fileformat orc;
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index 06c1b6534976f..c2887692949ea 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -141,83 +141,83 @@ ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;
 
 -- Verify that the new view is created.
 DESCRIBE TABLE EXTENDED tempdb1.v2;
-  +----------------------------+----------+-------+
-  |                    col_name|data_type |comment|
-  +----------------------------+----------+-------+
-  |                          c1|       int|   null|
-  |                          c2|    string|   null|
-  |                            |          |       |
-  |# Detailed Table Information|          |       |
-  |                    Database|   tempdb1|       |
-  |                       Table|        v2|       |
-  +----------------------------+----------+-------+
++----------------------------+----------+-------+
+|                    col_name|data_type |comment|
++----------------------------+----------+-------+
+|                          c1|       int|   null|
+|                          c2|    string|   null|
+|                            |          |       |
+|# Detailed Table Information|          |       |
+|                    Database|   tempdb1|       |
+|                       Table|        v2|       |
++----------------------------+----------+-------+
 
 -- Before ALTER VIEW SET TBLPROPERTIES
 DESC TABLE EXTENDED tempdb1.v2;
-  +----------------------------+----------+-------+
-  |                    col_name| data_type|comment|
-  +----------------------------+----------+-------+
-  |                          c1|       int|   null|
-  |                          c2|    string|   null|
-  |                            |          |       |
-  |# Detailed Table Information|          |       |
-  |                    Database|   tempdb1|       |
-  |                       Table|        v2|       |
-  |            Table Properties|    [....]|       |
-  +----------------------------+----------+-------+
++----------------------------+----------+-------+
+|                    col_name| data_type|comment|
++----------------------------+----------+-------+
+|                          c1|       int|   null|
+|                          c2|    string|   null|
+|                            |          |       |
+|# Detailed Table Information|          |       |
+|                    Database|   tempdb1|       |
+|                       Table|        v2|       |
+|            Table Properties|    [....]|       |
++----------------------------+----------+-------+
 
 -- Set properties in TBLPROPERTIES
 ALTER VIEW tempdb1.v2 SET TBLPROPERTIES ('created.by.user' = "John", 'created.date' = '01-01-2001' );
 
 -- Use `DESCRIBE TABLE EXTENDED tempdb1.v2` to verify
 DESC TABLE EXTENDED tempdb1.v2;
-  +----------------------------+-----------------------------------------------------+-------+
-  |                    col_name|                                            data_type|comment|
-  +----------------------------+-----------------------------------------------------+-------+
-  |                          c1|                                                  int|   null|
-  |                          c2|                                               string|   null|
-  |                            |                                                     |       |
-  |# Detailed Table Information|                                                     |       |
-  |                    Database|                                              tempdb1|       |
-  |                       Table|                                                   v2|       |
-  |            Table Properties|[created.by.user=John, created.date=01-01-2001, ....]|       |
-  +----------------------------+-----------------------------------------------------+-------+
++----------------------------+-----------------------------------------------------+-------+
+|                    col_name|                                            data_type|comment|
++----------------------------+-----------------------------------------------------+-------+
+|                          c1|                                                  int|   null|
+|                          c2|                                               string|   null|
+|                            |                                                     |       |
+|# Detailed Table Information|                                                     |       |
+|                    Database|                                              tempdb1|       |
+|                       Table|                                                   v2|       |
+|            Table Properties|[created.by.user=John, created.date=01-01-2001, ....]|       |
++----------------------------+-----------------------------------------------------+-------+
 
 -- Remove the key `created.by.user` and `created.date` from `TBLPROPERTIES`
 ALTER VIEW tempdb1.v2 UNSET TBLPROPERTIES ('created.by.user', 'created.date');
 
 --Use `DESC TABLE EXTENDED tempdb1.v2` to verify the changes
 DESC TABLE EXTENDED tempdb1.v2;
-  +----------------------------+----------+-------+
-  |                    col_name| data_type|comment|
-  +----------------------------+----------+-------+
-  |                          c1|       int|   null|
-  |                          c2|    string|   null|
-  |                            |          |       |
-  |# Detailed Table Information|          |       |
-  |                    Database|   tempdb1|       |
-  |                       Table|        v2|       |
-  |            Table Properties|    [....]|       |
-  +----------------------------+----------+-------+
++----------------------------+----------+-------+
+|                    col_name| data_type|comment|
++----------------------------+----------+-------+
+|                          c1|       int|   null|
+|                          c2|    string|   null|
+|                            |          |       |
+|# Detailed Table Information|          |       |
+|                    Database|   tempdb1|       |
+|                       Table|        v2|       |
+|            Table Properties|    [....]|       |
++----------------------------+----------+-------+
 
 -- Change the view definition
 ALTER VIEW tempdb1.v2 AS SELECT * FROM tempdb1.v1;
 
 -- Use `DESC TABLE EXTENDED` to verify
 DESC TABLE EXTENDED tempdb1.v2;
-  +----------------------------+---------------------------+-------+
-  |                    col_name|                  data_type|comment|
-  +----------------------------+---------------------------+-------+
-  |                          c1|                        int|   null|
-  |                          c2|                     string|   null|
-  |                            |                           |       |
-  |# Detailed Table Information|                           |       |
-  |                    Database|                    tempdb1|       |
-  |                       Table|                         v2|       |
-  |                        Type|                       VIEW|       |
-  |                   View Text|   select * from tempdb1.v1|       |
-  |          View Original Text|   select * from tempdb1.v1|       |
-  +----------------------------+---------------------------+-------+
++----------------------------+---------------------------+-------+
+|                    col_name|                  data_type|comment|
++----------------------------+---------------------------+-------+
+|                          c1|                        int|   null|
+|                          c2|                     string|   null|
+|                            |                           |       |
+|# Detailed Table Information|                           |       |
+|                    Database|                    tempdb1|       |
+|                       Table|                         v2|       |
+|                        Type|                       VIEW|       |
+|                   View Text|   select * from tempdb1.v1|       |
+|          View Original Text|   select * from tempdb1.v1|       |
++----------------------------+---------------------------+-------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-ddl-create-database.md b/docs/sql-ref-syntax-ddl-create-database.md
index 6f74acdb60bf7..0ef0dfbdaed2b 100644
--- a/docs/sql-ref-syntax-ddl-create-database.md
+++ b/docs/sql-ref-syntax-ddl-create-database.md
@@ -68,14 +68,14 @@ CREATE DATABASE IF NOT EXISTS customer_db COMMENT 'This is customer database' LO
 
 -- Verify that properties are set.
 DESCRIBE DATABASE EXTENDED customer_db;
-   +-------------------------+--------------------------+
-   |database_description_item|database_description_value|
-   +-------------------------+--------------------------+
-   |            Database Name|               customer_db|
-   |              Description| This is customer database|
-   |                 Location|     hdfs://hacluster/user|
-   |               Properties|   ((ID,001), (Name,John))|
-   +-------------------------+--------------------------+
++-------------------------+--------------------------+
+|database_description_item|database_description_value|
++-------------------------+--------------------------+
+|            Database Name|               customer_db|
+|              Description| This is customer database|
+|                 Location|     hdfs://hacluster/user|
+|               Properties|   ((ID,001), (Name,John))|
++-------------------------+--------------------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index 5009069b5f8f1..e3f21f70f7c18 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -114,20 +114,20 @@ CREATE FUNCTION simple_udf AS 'SimpleUdf'
 
 -- Verify that the function is in the registry.
 SHOW USER FUNCTIONS;
-  +------------------+
-  |          function|
-  +------------------+
-  |default.simple_udf|
-  +------------------+
++------------------+
+|          function|
++------------------+
+|default.simple_udf|
++------------------+
 
 -- Invoke the function. Every selected value should be incremented by 10.
 SELECT simple_udf(c1) AS function_return_value FROM t1;
-  +---------------------+                                                         
-  |function_return_value|
-  +---------------------+
-  |                   11|
-  |                   12|
-  +---------------------+
++---------------------+
+|function_return_value|
++---------------------+
+|                   11|
+|                   12|
++---------------------+
 
 -- Created a temporary function.
 CREATE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf' 
@@ -137,12 +137,12 @@ CREATE TEMPORARY FUNCTION simple_temp_udf AS 'SimpleUdf'
 -- Please note that the temporary function does not have a qualified
 -- database associated with it.
 SHOW USER FUNCTIONS;
-  +------------------+
-  |          function|
-  +------------------+
-  |default.simple_udf|
-  |   simple_temp_udf|
-  +------------------+
++------------------+
+|          function|
++------------------+
+|default.simple_udf|
+|   simple_temp_udf|
++------------------+
 
 -- 1. Modify `SimpleUdf`'s implementation to add supplied integral value by 20.
 --    import org.apache.hadoop.hive.ql.exec.UDF;
@@ -160,12 +160,12 @@ CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
 
 -- Invoke the function. Every selected value should be incremented by 20.
 SELECT simple_udf(c1) AS function_return_value FROM t1;
-  +---------------------+                                                         
-  |function_return_value|
-  +---------------------+
-  |                   21|
-  |                   22|
-  +---------------------+
++---------------------+
+|function_return_value|
++---------------------+
+|                   21|
+|                   22|
++---------------------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-ddl-drop-function.md b/docs/sql-ref-syntax-ddl-drop-function.md
index b783b0ebd2bc8..66a405c24e413 100644
--- a/docs/sql-ref-syntax-ddl-drop-function.md
+++ b/docs/sql-ref-syntax-ddl-drop-function.md
@@ -62,11 +62,11 @@ CREATE FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAv
 
 -- List user functions
 SHOW USER FUNCTIONS;
-  +----------------+
-  |        function|
-  +----------------+
-  |default.test_avg|
-  +----------------+
++----------------+
+|        function|
++----------------+
+|default.test_avg|
++----------------+
 
 -- Create Temporary function `test_avg`
 CREATE TEMPORARY FUNCTION test_avg AS
@@ -74,12 +74,12 @@ CREATE TEMPORARY FUNCTION test_avg AS
 
 -- List user functions
 SHOW USER FUNCTIONS;
-  +----------------+
-  |        function|
-  +----------------+
-  |default.test_avg|
-  |        test_avg|
-  +----------------+
++----------------+
+|        function|
++----------------+
+|default.test_avg|
+|        test_avg|
++----------------+
 
 -- Drop Permanent function
 DROP FUNCTION test_avg;
@@ -92,11 +92,11 @@ DROP FUNCTION test_avg;
 
 -- List the functions after dropping, it should list only temporary function
 SHOW USER FUNCTIONS;
-  +--------+
-  |function|
-  +--------+
-  |test_avg|
-  +--------+
++--------+
+|function|
++--------+
+|test_avg|
++--------+
   
 -- Drop Temporary function
 DROP TEMPORARY FUNCTION IF EXISTS test_avg;
diff --git a/docs/sql-ref-syntax-ddl-repair-table.md b/docs/sql-ref-syntax-ddl-repair-table.md
index 499b2bff54d8d..c48b731512ad3 100644
--- a/docs/sql-ref-syntax-ddl-repair-table.md
+++ b/docs/sql-ref-syntax-ddl-repair-table.md
@@ -57,15 +57,15 @@ MSCK REPAIR TABLE t1;
 
 -- SELECT * FROM t1 returns results
 SELECT * FROM t1;
-  +-------+---+
-  |   name|age|
-  +-------+---+
-  |Michael| 20|
-  +-------+---+
-  | Justin| 19|
-  +-------+---+
-  |   Andy| 30|
-  +-------+---+
++-------+---+
+|   name|age|
++-------+---+
+|Michael| 20|
++-------+---+
+| Justin| 19|
++-------+---+
+|   Andy| 30|
++-------+---+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-ddl-truncate-table.md b/docs/sql-ref-syntax-ddl-truncate-table.md
index e2d1834302776..820f439f97a4b 100644
--- a/docs/sql-ref-syntax-ddl-truncate-table.md
+++ b/docs/sql-ref-syntax-ddl-truncate-table.md
@@ -62,33 +62,33 @@ TRUNCATE TABLE table_identifier [ partition_spec ]
 CREATE TABLE Student (name STRING, rollno INT) PARTITIONED BY (age INT);
 
 SELECT * FROM Student;
-  +----+------+---+
-  |name|rollno|age|
-  +----+------+---+
-  | ABC|     1| 10|
-  | DEF|     2| 10|
-  | XYZ|     3| 12|
-  +----+------+---+
++----+------+---+
+|name|rollno|age|
++----+------+---+
+| ABC|     1| 10|
+| DEF|     2| 10|
+| XYZ|     3| 12|
++----+------+---+
 
 -- Removes all rows from the table in the partition specified
 TRUNCATE TABLE Student partition(age=10);
 
 -- After truncate execution, records belonging to partition age=10 are removed
 SELECT * FROM Student;
-  +----+------+---+
-  |name|rollno|age|
-  +----+------+---+
-  | XYZ|     3| 12|
-  +----+------+---+
++----+------+---+
+|name|rollno|age|
++----+------+---+
+| XYZ|     3| 12|
++----+------+---+
 
 -- Removes all rows from the table from all partitions
 TRUNCATE TABLE Student;
 
 SELECT * FROM Student;
-  +----+------+---+
-  |name|rollno|age|
-  +----+------+---+
-  +----+------+---+
++----+------+---+
+|name|rollno|age|
++----+------+---+
++----+------+---+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-dml-insert-into.md b/docs/sql-ref-syntax-dml-insert-into.md
index 2498726602837..924831f7feedd 100644
--- a/docs/sql-ref-syntax-dml-insert-into.md
+++ b/docs/sql-ref-syntax-dml-insert-into.md
@@ -83,11 +83,11 @@ INSERT INTO students VALUES
     ('Amy Smith', '123 Park Ave, San Jose', 111111);
 
 SELECT * FROM students;
-  +---------+---------------------+----------+
-  |     name|              address|student_id|
-  +---------+---------------------+----------+
-  |Amy Smith|123 Park Ave,San Jose|    111111|
-  +---------+---------------------+----------+
++---------+---------------------+----------+
+|     name|              address|student_id|
++---------+---------------------+----------+
+|Amy Smith|123 Park Ave,San Jose|    111111|
++---------+---------------------+----------+
 {% endhighlight %}
 
 #### Multi-Row Insert Using a VALUES Clause
@@ -98,15 +98,15 @@ INSERT INTO students VALUES
     ('Cathy Johnson', '789 Race Ave, Palo Alto', 333333);
 
 SELECT * FROM students;
-  +-------------+------------------------+----------+
-  |         name|                 address|student_id|
-  +-------------+------------------------+----------+
-  |    Amy Smith|  123 Park Ave, San Jose|    111111|
-  +-------------+------------------------+----------+
-  |    Bob Brown|456 Taylor St, Cupertino|    222222|
-  +-------------+------------------------+----------+
-  |Cathy Johnson| 789 Race Ave, Palo Alto|    333333|
-  +--------------+-----------------------+----------+
++-------------+------------------------+----------+
+|         name|                 address|student_id|
++-------------+------------------------+----------+
+|    Amy Smith|  123 Park Ave, San Jose|    111111|
++-------------+------------------------+----------+
+|    Bob Brown|456 Taylor St, Cupertino|    222222|
++-------------+------------------------+----------+
+|Cathy Johnson| 789 Race Ave, Palo Alto|    333333|
++--------------+-----------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a SELECT Statement
@@ -114,29 +114,29 @@ SELECT * FROM students;
 {% highlight sql %}
 -- Assuming the persons table has already been created and populated.
 SELECT * FROM persons;
-  +-------------+-------------------------+---------+
-  |         name|                  address|      ssn|
-  +-------------+-------------------------+---------+
-  |Dora Williams|134 Forest Ave, Melo Park|123456789|
-  +-------------+-------------------------+---------+
-  |  Eddie Davis|  245 Market St, Milpitas|345678901|
-  +-------------+-------------------------+---------+
++-------------+-------------------------+---------+
+|         name|                  address|      ssn|
++-------------+-------------------------+---------+
+|Dora Williams|134 Forest Ave, Melo Park|123456789|
++-------------+-------------------------+---------+
+|  Eddie Davis|  245 Market St, Milpitas|345678901|
++-------------+-------------------------+---------+
 
 INSERT INTO students PARTITION (student_id = 444444)
     SELECT name, address FROM persons WHERE name = "Dora Williams";
 
 SELECT * FROM students;
-  +-------------+-------------------------+----------+
-  |         name|                  address|student_id|
-  +-------------+-------------------------+----------+
-  |    Amy Smith|   123 Park Ave, San Jose|    111111|
-  +-------------+-------------------------+----------+
-  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
-  +-------------+-------------------------+----------+
-  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-  +-------------+-------------------------+----------+
-  |Dora Williams|134 Forest Ave, Melo Park|    444444|
-  +-------------+-------------------------+----------+
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|   123 Park Ave, San Jose|    111111|
++-------------+-------------------------+----------+
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
++-------------+-------------------------+----------+
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
++-------------+-------------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a TABLE Statement
@@ -144,32 +144,32 @@ SELECT * FROM students;
 {% highlight sql %}
 -- Assuming the visiting_students table has already been created and populated.
 SELECT * FROM visiting_students;
-  +-------------+---------------------+----------+
-  |         name|              address|student_id|
-  +-------------+---------------------+----------+
-  |Fleur Laurent|345 Copper St, London|    777777|
-  +-------------+---------------------+----------+
-  |Gordon Martin| 779 Lake Ave, Oxford|    888888|
-  +-------------+---------------------+----------+
++-------------+---------------------+----------+
+|         name|              address|student_id|
++-------------+---------------------+----------+
+|Fleur Laurent|345 Copper St, London|    777777|
++-------------+---------------------+----------+
+|Gordon Martin| 779 Lake Ave, Oxford|    888888|
++-------------+---------------------+----------+
 
 INSERT INTO students TABLE visiting_students;
 
 SELECT * FROM students;
-  +-------------+-------------------------+----------+
-  |         name|                  address|student_id|
-  +-------------+-------------------------+----------+
-  |    Amy Smith|    123 Park Ave,San Jose|    111111|
-  +-------------+-------------------------+----------+
-  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
-  +-------------+-------------------------+----------+
-  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-  +-------------+-------------------------+----------+
-  |Dora Williams|134 Forest Ave, Melo Park|    444444|
-  +-------------+-------------------------+----------+
-  |Fleur Laurent|    345 Copper St, London|    777777|
-  +-------------+-------------------------+----------+
-  |Gordon Martin|     779 Lake Ave, Oxford|    888888|
-  +-------------+-------------------------+----------+
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|    123 Park Ave,San Jose|    111111|
++-------------+-------------------------+----------+
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
++-------------+-------------------------+----------+
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
++-------------+-------------------------+----------+
+|Fleur Laurent|    345 Copper St, London|    777777|
++-------------+-------------------------+----------+
+|Gordon Martin|     779 Lake Ave, Oxford|    888888|
++-------------+-------------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a FROM Statement
@@ -177,39 +177,39 @@ SELECT * FROM students;
 {% highlight sql %}
 -- Assuming the applicants table has already been created and populated.
 SELECT * FROM applicants;
-  +-----------+--------------------------+----------+---------+
-  |       name|                   address|student_id|qualified|
-  +-----------+--------------------------+----------+---------+
-  |Helen Davis| 469 Mission St, San Diego|    999999|     true|
-  +-----------+--------------------------+----------+---------+
-  |   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
-  +-----------+--------------------------+----------+---------+
-  | Jason Wang|     908 Bird St, Saratoga|    121212|     true|
-  +-----------+--------------------------+----------+---------+
++-----------+--------------------------+----------+---------+
+|       name|                   address|student_id|qualified|
++-----------+--------------------------+----------+---------+
+|Helen Davis| 469 Mission St, San Diego|    999999|     true|
++-----------+--------------------------+----------+---------+
+|   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
++-----------+--------------------------+----------+---------+
+| Jason Wang|     908 Bird St, Saratoga|    121212|     true|
++-----------+--------------------------+----------+---------+
 
 INSERT INTO students
      FROM applicants SELECT name, address, id applicants WHERE qualified = true;
 
 SELECT * FROM students;
-  +-------------+-------------------------+----------+
-  |         name|                  address|student_id|
-  +-------------+-------------------------+----------+
-  |    Amy Smith|   123 Park Ave, San Jose|    111111|
-  +-------------+-------------------------+----------+
-  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
-  +-------------+-------------------------+----------+
-  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-  +-------------+-------------------------+----------+
-  |Dora Williams|134 Forest Ave, Melo Park|    444444|
-  +-------------+-------------------------+----------+
-  |Fleur Laurent|    345 Copper St, London|    777777|
-  +-------------+-------------------------+----------+
-  |Gordon Martin|     779 Lake Ave, Oxford|    888888|
-  +-------------+-------------------------+----------+
-  |  Helen Davis|469 Mission St, San Diego|    999999|
-  +-------------+-------------------------+----------+
-  |   Jason Wang|    908 Bird St, Saratoga|    121212|
-  +-------------+-------------------------+----------+
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|   123 Park Ave, San Jose|    111111|
++-------------+-------------------------+----------+
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
++-------------+-------------------------+----------+
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
++-------------+-------------------------+----------+
+|Fleur Laurent|    345 Copper St, London|    777777|
++-------------+-------------------------+----------+
+|Gordon Martin|     779 Lake Ave, Oxford|    888888|
++-------------+-------------------------+----------+
+|  Helen Davis|469 Mission St, San Diego|    999999|
++-------------+-------------------------+----------+
+|   Jason Wang|    908 Bird St, Saratoga|    121212|
++-------------+-------------------------+----------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index f2874b5ac2304..5c760f00ed0c4 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -78,30 +78,30 @@ INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
 {% highlight sql %}
 -- Assuming the students table has already been created and populated.
 SELECT * FROM students;
-  +-------------+-------------------------+----------+
-  |         name|                  address|student_id|
-  +-------------+-------------------------+----------+
-  |    Amy Smith|   123 Park Ave, San Jose|    111111|
-  |    Bob Brown| 456 Taylor St, Cupertino|    222222|
-  |Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
-  |Dora Williams|134 Forest Ave, Melo Park|    444444|
-  |Fleur Laurent|    345 Copper St, London|    777777|
-  |Gordon Martin|     779 Lake Ave, Oxford|    888888|
-  |  Helen Davis|469 Mission St, San Diego|    999999|
-  |   Jason Wang|    908 Bird St, Saratoga|    121212|
-  +-------------+-------------------------+----------+
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|    Amy Smith|   123 Park Ave, San Jose|    111111|
+|    Bob Brown| 456 Taylor St, Cupertino|    222222|
+|Cathy Johnson|  789 Race Ave, Palo Alto|    333333|
+|Dora Williams|134 Forest Ave, Melo Park|    444444|
+|Fleur Laurent|    345 Copper St, London|    777777|
+|Gordon Martin|     779 Lake Ave, Oxford|    888888|
+|  Helen Davis|469 Mission St, San Diego|    999999|
+|   Jason Wang|    908 Bird St, Saratoga|    121212|
++-------------+-------------------------+----------+
 
 INSERT OVERWRITE students VALUES
     ('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
     ('Brian Reed', '723 Kern Ave, Palo Alto', 222222);
 
 SELECT * FROM students;
-  +----------+-----------------------+----------+
-  |      name|                address|student_id|
-  +----------+-----------------------+----------+
-  |Ashua Hill|456 Erica Ct, Cupertino|    111111|
-  |Brian Reed|723 Kern Ave, Palo Alto|    222222|
-  +----------+-----------------------+----------+
++----------+-----------------------+----------+
+|      name|                address|student_id|
++----------+-----------------------+----------+
+|Ashua Hill|456 Erica Ct, Cupertino|    111111|
+|Brian Reed|723 Kern Ave, Palo Alto|    222222|
++----------+-----------------------+----------+
 
 {% endhighlight %}
 
@@ -110,25 +110,25 @@ SELECT * FROM students;
 {% highlight sql %}
 -- Assuming the persons table has already been created and populated.
 SELECT * FROM persons;
-  +-------------+-------------------------+---------+
-  |         name|                  address|      ssn|
-  +-------------+-------------------------+---------+
-  |Dora Williams|134 Forest Ave, Melo Park|123456789|
-  +-------------+-------------------------+---------+
-  |  Eddie Davis|   245 Market St,Milpitas|345678901|
-  +-------------+-------------------------+---------+
++-------------+-------------------------+---------+
+|         name|                  address|      ssn|
++-------------+-------------------------+---------+
+|Dora Williams|134 Forest Ave, Melo Park|123456789|
++-------------+-------------------------+---------+
+|  Eddie Davis|   245 Market St,Milpitas|345678901|
++-------------+-------------------------+---------+
 
 INSERT OVERWRITE students PARTITION (student_id = 222222)
     SELECT name, address FROM persons WHERE name = "Dora Williams";
 
 SELECT * FROM students;
-  +-------------+-------------------------+----------+
-  |         name|                  address|student_id|
-  +-------------+-------------------------+----------+
-  |   Ashua Hill|  456 Erica Ct, Cupertino|    111111|
-  +-------------+-------------------------+----------+
-  |Dora Williams|134 Forest Ave, Melo Park|    222222|
-  +-------------+-------------------------+----------+
++-------------+-------------------------+----------+
+|         name|                  address|student_id|
++-------------+-------------------------+----------+
+|   Ashua Hill|  456 Erica Ct, Cupertino|    111111|
++-------------+-------------------------+----------+
+|Dora Williams|134 Forest Ave, Melo Park|    222222|
++-------------+-------------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a TABLE Statement
@@ -136,24 +136,24 @@ SELECT * FROM students;
 {% highlight sql %}
 -- Assuming the visiting_students table has already been created and populated.
 SELECT * FROM visiting_students;
-  +-------------+---------------------+----------+
-  |         name|              address|student_id|
-  +-------------+---------------------+----------+
-  |Fleur Laurent|345 Copper St, London|    777777|
-  +-------------+---------------------+----------+
-  |Gordon Martin| 779 Lake Ave, Oxford|    888888|
-  +-------------+---------------------+----------+
++-------------+---------------------+----------+
+|         name|              address|student_id|
++-------------+---------------------+----------+
+|Fleur Laurent|345 Copper St, London|    777777|
++-------------+---------------------+----------+
+|Gordon Martin| 779 Lake Ave, Oxford|    888888|
++-------------+---------------------+----------+
 
 INSERT OVERWRITE students TABLE visiting_students;
 
 SELECT * FROM students;
-  +-------------+---------------------+----------+
-  |         name|              address|student_id|
-  +-------------+---------------------+----------+
-  |Fleur Laurent|345 Copper St, London|    777777|
-  +-------------+---------------------+----------+
-  |Gordon Martin| 779 Lake Ave, Oxford|    888888|
-  +-------------+---------------------+----------+
++-------------+---------------------+----------+
+|         name|              address|student_id|
++-------------+---------------------+----------+
+|Fleur Laurent|345 Copper St, London|    777777|
++-------------+---------------------+----------+
+|Gordon Martin| 779 Lake Ave, Oxford|    888888|
++-------------+---------------------+----------+
 {% endhighlight %}
 
 #### Insert Using a FROM Statement
@@ -161,27 +161,27 @@ SELECT * FROM students;
 {% highlight sql %}
 -- Assuming the applicants table has already been created and populated.
 SELECT * FROM applicants;
-  +-----------+--------------------------+----------+---------+
-  |       name|                   address|student_id|qualified|
-  +-----------+--------------------------+----------+---------+
-  |Helen Davis| 469 Mission St, San Diego|    999999|     true|
-  +-----------+--------------------------+----------+---------+
-  |   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
-  +-----------+--------------------------+----------+---------+
-  | Jason Wang|     908 Bird St, Saratoga|    121212|     true|
-  +-----------+--------------------------+----------+---------+
++-----------+--------------------------+----------+---------+
+|       name|                   address|student_id|qualified|
++-----------+--------------------------+----------+---------+
+|Helen Davis| 469 Mission St, San Diego|    999999|     true|
++-----------+--------------------------+----------+---------+
+|   Ivy King|367 Leigh Ave, Santa Clara|    101010|    false|
++-----------+--------------------------+----------+---------+
+| Jason Wang|     908 Bird St, Saratoga|    121212|     true|
++-----------+--------------------------+----------+---------+
 
 INSERT OVERWRITE students
     FROM applicants SELECT name, address, id applicants WHERE qualified = true;
 
 SELECT * FROM students;
-  +-----------+-------------------------+----------+
-  |       name|                  address|student_id|
-  +-----------+-------------------------+----------+
-  |Helen Davis|469 Mission St, San Diego|    999999|
-  +-----------+-------------------------+----------+
-  | Jason Wang|    908 Bird St, Saratoga|    121212|
-  +-----------+-------------------------+----------+
++-----------+-------------------------+----------+
+|       name|                  address|student_id|
++-----------+-------------------------+----------+
+|Helen Davis|469 Mission St, San Diego|    999999|
++-----------+-------------------------+----------+
+| Jason Wang|    908 Bird St, Saratoga|    121212|
++-----------+-------------------------+----------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-dml-load.md b/docs/sql-ref-syntax-dml-load.md
index 9a9bf230e3101..01ece31bd17fa 100644
--- a/docs/sql-ref-syntax-dml-load.md
+++ b/docs/sql-ref-syntax-dml-load.md
@@ -75,11 +75,11 @@ LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ part
 -- Example without partition specification.
 -- Assuming the students table has already been created and populated.
 SELECT * FROM students;
-  +---------+----------------------+----------+
-  |     name|               address|student_id|
-  +---------+----------------------+----------+
-  |Amy Smith|123 Park Ave, San Jose|    111111|
-  +---------+----------------------+----------+
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Amy Smith|123 Park Ave, San Jose|    111111|
++---------+----------------------+----------+
 
 CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT) USING HIVE;
 
@@ -87,11 +87,11 @@ CREATE TABLE test_load (name VARCHAR(64), address VARCHAR(64), student_id INT) U
 LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;
 
 SELECT * FROM test_load;
-  +---------+----------------------+----------+
-  |     name|               address|student_id|
-  +---------+----------------------+----------+
-  |Amy Smith|123 Park Ave, San Jose|    111111|
-  +---------+----------------------+----------+
++---------+----------------------+----------+
+|     name|               address|student_id|
++---------+----------------------+----------+
+|Amy Smith|123 Park Ave, San Jose|    111111|
++---------+----------------------+----------+
 
 -- Example with partition specification.
 CREATE TABLE test_partition (c1 INT, c2 INT, c3 INT) PARTITIONED BY (c2, c3);
@@ -103,13 +103,13 @@ INSERT INTO test_partition PARTITION (c2 = 5, c3 = 6) VALUES (4);
 INSERT INTO test_partition PARTITION (c2 = 8, c3 = 9) VALUES (7);
 
 SELECT * FROM test_partition;
-  +---+---+---+
-  | c1| c2| c3|
-  +---+---+---+
-  |  1|  2|  3|
-  |  4|  5|  6|
-  |  7|  8|  9|
-  +---+---+---+
++---+---+---+
+| c1| c2| c3|
++---+---+---+
+|  1|  2|  3|
+|  4|  5|  6|
+|  7|  8|  9|
++---+---+---+
 
 CREATE TABLE test_load_partition (c1 INT, c2 INT, c3 INT) USING HIVE PARTITIONED BY (c2, c3);
 
@@ -118,9 +118,9 @@ LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
     OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);
 
 SELECT * FROM test_load_partition;
-  +---+---+---+
-  | c1| c2| c3|
-  +---+---+---+
-  |  1|  2|  3|
-  +---+---+---+
++---+---+---+
+| c1| c2| c3|
++---+---+---+
+|  1|  2|  3|
++---+---+---+
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-qry-aggregation.md b/docs/sql-ref-syntax-qry-aggregation.md
deleted file mode 100644
index fbe6e61c84fd1..0000000000000
--- a/docs/sql-ref-syntax-qry-aggregation.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Aggregation (CUBE/ROLLUP/GROUPING)
-displayTitle: Aggregation (CUBE/ROLLUP/GROUPING) 
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-syntax-qry-explain.md b/docs/sql-ref-syntax-qry-explain.md
index 7ad297d9c76dd..298a2edaea1f2 100644
--- a/docs/sql-ref-syntax-qry-explain.md
+++ b/docs/sql-ref-syntax-qry-explain.md
@@ -68,68 +68,68 @@ EXPLAIN [ EXTENDED | CODEGEN | COST | FORMATTED ] statement
 {% highlight sql %}
 -- Default Output
 EXPLAIN select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
-  +----------------------------------------------------+
-  |                                                plan|
-  +----------------------------------------------------+
-  | == Physical Plan ==
-   *(2) HashAggregate(keys=[k#33], functions=[sum(cast(v#34 as bigint))])
-   +- Exchange hashpartitioning(k#33, 200), true, [id=#59]
-      +- *(1) HashAggregate(keys=[k#33], functions=[partial_sum(cast(v#34 as bigint))])
-         +- *(1) LocalTableScan [k#33, v#34]
-  |
-  +----------------------------------------------------
++----------------------------------------------------+
+|                                                plan|
++----------------------------------------------------+
+| == Physical Plan ==
+ *(2) HashAggregate(keys=[k#33], functions=[sum(cast(v#34 as bigint))])
+ +- Exchange hashpartitioning(k#33, 200), true, [id=#59]
+    +- *(1) HashAggregate(keys=[k#33], functions=[partial_sum(cast(v#34 as bigint))])
+       +- *(1) LocalTableScan [k#33, v#34]
+|
++----------------------------------------------------
 
 -- Using Extended
 EXPLAIN EXTENDED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
-  +----------------------------------------------------+
-  |                                                plan|
-  +----------------------------------------------------+
-  | == Parsed Logical Plan ==
-   'Aggregate ['k], ['k, unresolvedalias('sum('v), None)]
-   +- 'SubqueryAlias `t`
-      +- 'UnresolvedInlineTable [k, v], [List(1, 2), List(1, 3)]
++----------------------------------------------------+
+|                                                plan|
++----------------------------------------------------+
+| == Parsed Logical Plan ==
+ 'Aggregate ['k], ['k, unresolvedalias('sum('v), None)]
+ +- 'SubqueryAlias `t`
+    +- 'UnresolvedInlineTable [k, v], [List(1, 2), List(1, 3)]
    
-   == Analyzed Logical Plan ==
-   k: int, sum(v): bigint
-   Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
-   +- SubqueryAlias `t`
-      +- LocalRelation [k#47, v#48]
+ == Analyzed Logical Plan ==
+ k: int, sum(v): bigint
+ Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
+ +- SubqueryAlias `t`
+    +- LocalRelation [k#47, v#48]
    
-   == Optimized Logical Plan ==
-   Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
-   +- LocalRelation [k#47, v#48]
+ == Optimized Logical Plan ==
+ Aggregate [k#47], [k#47, sum(cast(v#48 as bigint)) AS sum(v)#50L]
+ +- LocalRelation [k#47, v#48]
    
-   == Physical Plan ==
-   *(2) HashAggregate(keys=[k#47], functions=[sum(cast(v#48 as bigint))], output=[k#47, sum(v)#50L])
-  +- Exchange hashpartitioning(k#47, 200), true, [id=#79]
-     +- *(1) HashAggregate(keys=[k#47], functions=[partial_sum(cast(v#48 as bigint))], output=[k#47, sum#52L])
-      +- *(1) LocalTableScan [k#47, v#48]
-  |
-  +----------------------------------------------------+
+ == Physical Plan ==
+ *(2) HashAggregate(keys=[k#47], functions=[sum(cast(v#48 as bigint))], output=[k#47, sum(v)#50L])
++- Exchange hashpartitioning(k#47, 200), true, [id=#79]
+   +- *(1) HashAggregate(keys=[k#47], functions=[partial_sum(cast(v#48 as bigint))], output=[k#47, sum#52L])
+    +- *(1) LocalTableScan [k#47, v#48]
+|
++----------------------------------------------------+
 
 -- Using Formatted
 EXPLAIN FORMATTED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
-  +----------------------------------------------------+
-  |                                                plan|
-  +----------------------------------------------------+
-  | == Physical Plan ==
-   * HashAggregate (4)
-   +- Exchange (3)
-      +- * HashAggregate (2)
-         +- * LocalTableScan (1)
++----------------------------------------------------+
+|                                                plan|
++----------------------------------------------------+
+| == Physical Plan ==
+ * HashAggregate (4)
+ +- Exchange (3)
+    +- * HashAggregate (2)
+       +- * LocalTableScan (1)
    
    
-   (1) LocalTableScan [codegen id : 1]
-   Output: [k#19, v#20]
+ (1) LocalTableScan [codegen id : 1]
+ Output: [k#19, v#20]
         
-   (2) HashAggregate [codegen id : 1]
-   Input: [k#19, v#20]
+ (2) HashAggregate [codegen id : 1]
+ Input: [k#19, v#20]
         
-   (3) Exchange 
-   Input: [k#19, sum#24L]
+ (3) Exchange
+ Input: [k#19, sum#24L]
         
-   (4) HashAggregate [codegen id : 2]
-   Input: [k#19, sum#24L] 
-  |
-  +----------------------------------------------------+
+ (4) HashAggregate [codegen id : 2]
+ Input: [k#19, sum#24L]
+|
++----------------------------------------------------+
 {% endhighlight %}
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-sampling.md
index 3bc45cc48b78f..82f6588e6c504 100644
--- a/docs/sql-ref-syntax-qry-sampling.md
+++ b/docs/sql-ref-syntax-qry-sampling.md
@@ -40,53 +40,53 @@ TABLESAMPLE ((integer_expression | decimal_expression) PERCENT)
 
 {% highlight sql %}
 SELECT * FROM test;
-  +--+----+
-  |id|name|
-  +--+----+
-  | 5|Alex|
-  | 8|Lucy|
-  | 2|Mary|
-  | 4|Fred|
-  | 1|Lisa|
-  | 9|Eric|
-  |10|Adam|
-  | 6|Mark|
-  | 7|Lily|
-  | 3|Evan|
-  +--+----+
++--+----+
+|id|name|
++--+----+
+| 5|Alex|
+| 8|Lucy|
+| 2|Mary|
+| 4|Fred|
+| 1|Lisa|
+| 9|Eric|
+|10|Adam|
+| 6|Mark|
+| 7|Lily|
+| 3|Evan|
++--+----+
 
 SELECT * FROM test TABLESAMPLE (50 PERCENT);
-  +--+----+
-  |id|name|
-  +--+----+
-  | 5|Alex|
-  | 2|Mary|
-  | 4|Fred|
-  | 9|Eric|
-  |10|Adam|
-  | 3|Evan|
-  +--+----+
++--+----+
+|id|name|
++--+----+
+| 5|Alex|
+| 2|Mary|
+| 4|Fred|
+| 9|Eric|
+|10|Adam|
+| 3|Evan|
++--+----+
 
 SELECT * FROM test TABLESAMPLE (5 ROWS);
-  +--+----+
-  |id|name|
-  +--+----+
-  | 5|Alex|
-  | 8|Lucy|
-  | 2|Mary|
-  | 4|Fred|
-  | 1|Lisa|
-  +--+----+
++--+----+
+|id|name|
++--+----+
+| 5|Alex|
+| 8|Lucy|
+| 2|Mary|
+| 4|Fred|
+| 1|Lisa|
++--+----+
 
 SELECT * FROM test TABLESAMPLE (BUCKET 4 OUT OF 10);
-  +--+----+
-  |id|name|
-  +--+----+
-  | 8|Lucy|
-  | 2|Mary|
-  | 9|Eric|
-  | 6|Mark|
-  +--+----+
++--+----+
+|id|name|
++--+----+
+| 8|Lucy|
+| 2|Mary|
+| 9|Eric|
+| 6|Mark|
++--+----+
 {% endhighlight %}
 
 ### Related Statement
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index 687b2b512cd90..ac1e1ccb00ac9 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -64,32 +64,32 @@ SET spark.sql.shuffle.partitions = 2;
 -- of a query when `CLUSTER BY` is not used vs when it's used. The query below produces rows
 -- where age column is not sorted.
 SELECT age, name FROM person;
-  +---+-------+
-  |age|   name|
-  +---+-------+
-  | 16|Shone S|
-  | 25|Zen Hui|
-  | 16| Jack N|
-  | 25| Mike A|
-  | 18| John A|
-  | 18| Anil B|
-  +---+-------+
++---+-------+
+|age|   name|
++---+-------+
+| 16|Shone S|
+| 25|Zen Hui|
+| 16| Jack N|
+| 25| Mike A|
+| 18| John A|
+| 18| Anil B|
++---+-------+
 
 -- Produces rows clustered by age. Persons with same age are clustered together.
 -- In the query below, persons with age 18 and 25 are in first partition and the
 -- persons with age 16 are in the second partition. The rows are sorted based
 -- on age within each partition.
 SELECT age, name FROM person CLUSTER BY age;
-  +---+-------+
-  |age|   name|
-  +---+-------+
-  | 18| John A|
-  | 18| Anil B|
-  | 25|Zen Hui|
-  | 25| Mike A|
-  | 16|Shone S|
-  | 16| Jack N|
-  +---+-------+
++---+-------+
+|age|   name|
++---+-------+
+| 18| John A|
+| 18| Anil B|
+| 25|Zen Hui|
+| 25| Mike A|
+| 16|Shone S|
+| 16| Jack N|
++---+-------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-cte.md b/docs/sql-ref-syntax-qry-select-cte.md
index 2146f8e0cd557..2408c884c64b5 100644
--- a/docs/sql-ref-syntax-qry-select-cte.md
+++ b/docs/sql-ref-syntax-qry-select-cte.md
@@ -55,11 +55,11 @@ expression_name [ ( column_name [ , ... ] ) ] [ AS ] ( [ common_table_expression
 -- CTE with multiple column aliases
 WITH t(x, y) AS (SELECT 1, 2)
 SELECT * FROM t WHERE x = 1 AND y = 2;
-  +---+---+
-  |  x|  y|
-  +---+---+
-  |  1|  2|
-  +---+---+
++---+---+
+|  x|  y|
++---+---+
+|  1|  2|
++---+---+
 
 -- CTE in CTE definition
 WITH t as (
@@ -67,44 +67,44 @@ WITH t as (
     SELECT * FROM t2
 )
 SELECT * FROM t;
-  +---+
-  |  1|
-  +---+
-  |  1|
-  +---+
++---+
+|  1|
++---+
+|  1|
++---+
 
 -- CTE in subquery
 SELECT max(c) FROM (
     WITH t(c) AS (SELECT 1)
     SELECT * FROM t
 );
-  +------+
-  |max(c)|
-  +------+
-  |     1|
-  +------+
++------+
+|max(c)|
++------+
+|     1|
++------+
 
 -- CTE in subquery expression
 SELECT (
     WITH t AS (SELECT 1)
     SELECT * FROM t
 );
-  +----------------+
-  |scalarsubquery()|
-  +----------------+
-  |               1|
-  +----------------+
++----------------+
+|scalarsubquery()|
++----------------+
+|               1|
++----------------+
 
 -- CTE in CREATE VIEW statement
 CREATE VIEW v AS
     WITH t(a, b, c, d) AS (SELECT 1, 2, 3, 4)
     SELECT * FROM t;
 SELECT * FROM v;
-  +---+---+---+---+
-  |  a|  b|  c|  d|
-  +---+---+---+---+
-  |  1|  2|  3|  4|
-  +---+---+---+---+
++---+---+---+---+
+|  a|  b|  c|  d|
++---+---+---+---+
+|  1|  2|  3|  4|
++---+---+---+---+
 
 -- If name conflict is detected in nested CTE, then AnalysisException is thrown by default.
 -- SET spark.sql.legacy.ctePrecedencePolicy = CORRECTED (which is recommended),
@@ -117,11 +117,11 @@ WITH
         SELECT * FROM t
     )
 SELECT * FROM t2;
-  +---+
-  |  2|
-  +---+
-  |  2|
-  +---+
++---+
+|  2|
++---+
+|  2|
++---+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-distinct.md b/docs/sql-ref-syntax-qry-select-distinct.md
deleted file mode 100644
index 2ed7931c202f3..0000000000000
--- a/docs/sql-ref-syntax-qry-select-distinct.md
+++ /dev/null
@@ -1,22 +0,0 @@
----
-layout: global
-title: Distinct operator
-displayTitle: Distinct operator
-license: |
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
- 
-     http://www.apache.org/licenses/LICENSE-2.0
- 
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
----
-
-**This page is under construction**
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index 18d73c7cdff19..9e2db27ae7161 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -61,30 +61,30 @@ SET spark.sql.shuffle.partitions = 2;
 -- behavior of `DISTRIBUTE BY`. The query below produces rows where age columns are not
 -- clustered together.
 SELECT age, name FROM person;
-  +---+-------+
-  |age|   name|
-  +---+-------+
-  | 16|Shone S|
-  | 25|Zen Hui|
-  | 16| Jack N|
-  | 25| Mike A|
-  | 18| John A|
-  | 18| Anil B|
-  +---+-------+
++---+-------+
+|age|   name|
++---+-------+
+| 16|Shone S|
+| 25|Zen Hui|
+| 16| Jack N|
+| 25| Mike A|
+| 18| John A|
+| 18| Anil B|
++---+-------+
 
 -- Produces rows clustered by age. Persons with same age are clustered together.
 -- Unlike `CLUSTER BY` clause, the rows are not sorted within a partition.
 SELECT age, name FROM person DISTRIBUTE BY age;
-  +---+-------+
-  |age|   name|
-  +---+-------+
-  | 25|Zen Hui|
-  | 25| Mike A|
-  | 18| John A|
-  | 18| Anil B|
-  | 16|Shone S|
-  | 16| Jack N|
-  +---+-------+
++---+-------+
+|age|   name|
++---+-------+
+| 25|Zen Hui|
+| 25| Mike A|
+| 18| John A|
+| 18| Anil B|
+| 16|Shone S|
+| 16| Jack N|
++---+-------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index 1676ca9c6d6aa..22fe782f9eaa7 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -112,58 +112,58 @@ INSERT INTO dealer VALUES
 
 -- Sum of quantity per dealership. Group by `id`.
 SELECT id, sum(quantity) FROM dealer GROUP BY id ORDER BY id;
-  +---+-------------+
-  | id|sum(quantity)|
-  +---+-------------+
-  |100|           32|
-  |200|           33|
-  |300|           13|
-  +---+-------------+
++---+-------------+
+| id|sum(quantity)|
++---+-------------+
+|100|           32|
+|200|           33|
+|300|           13|
++---+-------------+
 
 -- Use column position in GROUP by clause.
 SELECT id, sum(quantity) FROM dealer GROUP BY 1 ORDER BY 1;
-  +---+-------------+
-  | id|sum(quantity)|
-  +---+-------------+
-  |100|           32|
-  |200|           33|
-  |300|           13|
-  +---+-------------+
++---+-------------+
+| id|sum(quantity)|
++---+-------------+
+|100|           32|
+|200|           33|
+|300|           13|
++---+-------------+
 
 -- Multiple aggregations.
 -- 1. Sum of quantity per dealership.
 -- 2. Max quantity per dealership.
 SELECT id, sum(quantity) AS sum, max(quantity) AS max FROM dealer GROUP BY id ORDER BY id;
-  +---+---+---+
-  | id|sum|max|
-  +---+---+---+
-  |100| 32| 15|
-  |200| 33| 20|
-  |300| 13|  8|
-  +---+---+---+
++---+---+---+
+| id|sum|max|
++---+---+---+
+|100| 32| 15|
+|200| 33| 20|
+|300| 13|  8|
++---+---+---+
 
 -- Count the number of distinct dealer cities per car_model.
 SELECT car_model, count(DISTINCT city) AS count FROM dealer GROUP BY car_model;
-  +------------+-----+
-  |   car_model|count|
-  +------------+-----+
-  | Honda Civic|    3|
-  |   Honda CRV|    2|
-  |Honda Accord|    3|
-  +------------+-----+
++------------+-----+
+|   car_model|count|
++------------+-----+
+| Honda Civic|    3|
+|   Honda CRV|    2|
+|Honda Accord|    3|
++------------+-----+
 
 -- Sum of only 'Honda Civic' and 'Honda CRV' quantities per dealership.
 SELECT id, sum(quantity) FILTER (
             WHERE car_model IN ('Honda Civic', 'Honda CRV')
         ) AS `sum(quantity)` FROM dealer
     GROUP BY id ORDER BY id;
-  +---+-------------+
-  | id|sum(quantity)|
-  +---+-------------+
-  |100|           17|
-  |200|           23|
-  |300|            5|
-  +---+-------------+
++---+-------------+
+| id|sum(quantity)|
++---+-------------+
+|100|           17|
+|200|           23|
+|300|            5|
++---+-------------+
 
 -- Aggregations using multiple sets of grouping columns in a single statement.
 -- Following performs aggregations based on four sets of grouping columns.
@@ -174,97 +174,97 @@ SELECT id, sum(quantity) FILTER (
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
     ORDER BY city;
-  +--------+------------+---+
-  |    city|   car_model|sum|
-  +--------+------------+---+
-  |    null|        null| 78|
-  |    null| HondaAccord| 33|
-  |    null|    HondaCRV| 10|
-  |    null|  HondaCivic| 35|
-  |  Dublin|        null| 33|
-  |  Dublin| HondaAccord| 10|
-  |  Dublin|    HondaCRV|  3|
-  |  Dublin|  HondaCivic| 20|
-  | Fremont|        null| 32|
-  | Fremont| HondaAccord| 15|
-  | Fremont|    HondaCRV|  7|
-  | Fremont|  HondaCivic| 10|
-  | SanJose|        null| 13|
-  | SanJose| HondaAccord|  8|
-  | SanJose|  HondaCivic|  5|
-  +--------+------------+---+
++--------+------------+---+
+|    city|   car_model|sum|
++--------+------------+---+
+|    null|        null| 78|
+|    null| HondaAccord| 33|
+|    null|    HondaCRV| 10|
+|    null|  HondaCivic| 35|
+|  Dublin|        null| 33|
+|  Dublin| HondaAccord| 10|
+|  Dublin|    HondaCRV|  3|
+|  Dublin|  HondaCivic| 20|
+| Fremont|        null| 32|
+| Fremont| HondaAccord| 15|
+| Fremont|    HondaCRV|  7|
+| Fremont|  HondaCivic| 10|
+| SanJose|        null| 13|
+| SanJose| HondaAccord|  8|
+| SanJose|  HondaCivic|  5|
++--------+------------+---+
 
 -- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS`
 -- specifications are present.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
     ORDER BY city, car_model;
-  +--------+------------+---+
-  |    city|   car_model|sum|
-  +--------+------------+---+
-  |    null|        null| 78|
-  |    null| HondaAccord| 33|
-  |    null|    HondaCRV| 10|
-  |    null|  HondaCivic| 35|
-  |  Dublin|        null| 33|
-  |  Dublin| HondaAccord| 10|
-  |  Dublin|    HondaCRV|  3|
-  |  Dublin|  HondaCivic| 20|
-  | Fremont|        null| 32|
-  | Fremont| HondaAccord| 15|
-  | Fremont|    HondaCRV|  7|
-  | Fremont|  HondaCivic| 10|
-  | SanJose|        null| 13|
-  | SanJose| HondaAccord|  8|
-  | SanJose|  HondaCivic|  5|
-  +--------+------------+---+
++--------+------------+---+
+|    city|   car_model|sum|
++--------+------------+---+
+|    null|        null| 78|
+|    null| HondaAccord| 33|
+|    null|    HondaCRV| 10|
+|    null|  HondaCivic| 35|
+|  Dublin|        null| 33|
+|  Dublin| HondaAccord| 10|
+|  Dublin|    HondaCRV|  3|
+|  Dublin|  HondaCivic| 20|
+| Fremont|        null| 32|
+| Fremont| HondaAccord| 15|
+| Fremont|    HondaCRV|  7|
+| Fremont|  HondaCivic| 10|
+| SanJose|        null| 13|
+| SanJose| HondaAccord|  8|
+| SanJose|  HondaCivic|  5|
++--------+------------+---+
 
 -- Group by processing with `ROLLUP` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY city, car_model WITH ROLLUP
     ORDER BY city, car_model;
-  +--------+------------+---+
-  |    city|   car_model|sum|
-  +--------+------------+---+
-  |    null|        null| 78|
-  |  Dublin|        null| 33|
-  |  Dublin| HondaAccord| 10|
-  |  Dublin|    HondaCRV|  3|
-  |  Dublin|  HondaCivic| 20|
-  | Fremont|        null| 32|
-  | Fremont| HondaAccord| 15|
-  | Fremont|    HondaCRV|  7|
-  | Fremont|  HondaCivic| 10|
-  | SanJose|        null| 13|
-  | SanJose| HondaAccord|  8|
-  | SanJose|  HondaCivic|  5|
-  +--------+------------+---+
++--------+------------+---+
+|    city|   car_model|sum|
++--------+------------+---+
+|    null|        null| 78|
+|  Dublin|        null| 33|
+|  Dublin| HondaAccord| 10|
+|  Dublin|    HondaCRV|  3|
+|  Dublin|  HondaCivic| 20|
+| Fremont|        null| 32|
+| Fremont| HondaAccord| 15|
+| Fremont|    HondaCRV|  7|
+| Fremont|  HondaCivic| 10|
+| SanJose|        null| 13|
+| SanJose| HondaAccord|  8|
+| SanJose|  HondaCivic|  5|
++--------+------------+---+
 
 -- Group by processing with `CUBE` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY city, car_model WITH CUBE
     ORDER BY city, car_model;
-  +--------+------------+---+
-  |    city|   car_model|sum|
-  +--------+------------+---+
-  |    null|        null| 78|
-  |    null| HondaAccord| 33|
-  |    null|    HondaCRV| 10|
-  |    null|  HondaCivic| 35|
-  |  Dublin|        null| 33|
-  |  Dublin| HondaAccord| 10|
-  |  Dublin|    HondaCRV|  3|
-  |  Dublin|  HondaCivic| 20|
-  | Fremont|        null| 32|
-  | Fremont| HondaAccord| 15|
-  | Fremont|    HondaCRV|  7|
-  | Fremont|  HondaCivic| 10|
-  | SanJose|        null| 13|
-  | SanJose| HondaAccord|  8|
-  | SanJose|  HondaCivic|  5|
-  +--------+------------+---+
++--------+------------+---+
+|    city|   car_model|sum|
++--------+------------+---+
+|    null|        null| 78|
+|    null| HondaAccord| 33|
+|    null|    HondaCRV| 10|
+|    null|  HondaCivic| 35|
+|  Dublin|        null| 33|
+|  Dublin| HondaAccord| 10|
+|  Dublin|    HondaCRV|  3|
+|  Dublin|  HondaCivic| 20|
+| Fremont|        null| 32|
+| Fremont| HondaAccord| 15|
+| Fremont|    HondaCRV|  7|
+| Fremont|  HondaCivic| 10|
+| SanJose|        null| 13|
+| SanJose| HondaAccord|  8|
+| SanJose|  HondaCivic|  5|
++--------+------------+---+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index b84ad17955864..c8c4f2c38104c 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -67,56 +67,56 @@ INSERT INTO dealer VALUES
 
 -- `HAVING` clause referring to column in `GROUP BY`.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING city = 'Fremont';
-  +-------+---+
-  |   city|sum|
-  +-------+---+
-  |Fremont| 32|
-  +-------+---+
++-------+---+
+|   city|sum|
++-------+---+
+|Fremont| 32|
++-------+---+
 
 -- `HAVING` clause referring to aggregate function.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum(quantity) > 15;
-  +-------+---+
-  |   city|sum|
-  +-------+---+
-  | Dublin| 33|
-  |Fremont| 32|
-  +-------+---+
++-------+---+
+|   city|sum|
++-------+---+
+| Dublin| 33|
+|Fremont| 32|
++-------+---+
 
 -- `HAVING` clause referring to aggregate function by its alias.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING sum > 15;
-  +-------+---+
-  |   city|sum|
-  +-------+---+
-  | Dublin| 33|
-  |Fremont| 32|
-  +-------+---+
++-------+---+
+|   city|sum|
++-------+---+
+| Dublin| 33|
+|Fremont| 32|
++-------+---+
 
 -- `HAVING` clause referring to a different aggregate function than what is present in
 -- `SELECT` list.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING max(quantity) > 15;
-  +------+---+
-  |  city|sum|
-  +------+---+
-  |Dublin| 33|
-  +------+---+
++------+---+
+|  city|sum|
++------+---+
+|Dublin| 33|
++------+---+
 
 -- `HAVING` clause referring to constant expression.
 SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING 1 > 0 ORDER BY city;
-  +--------+---+
-  |    city|sum|
-  +--------+---+
-  |  Dublin| 33|
-  | Fremont| 32|
-  |San Jose| 13|
-  +--------+---+
++--------+---+
+|    city|sum|
++--------+---+
+|  Dublin| 33|
+| Fremont| 32|
+|San Jose| 13|
++--------+---+
 
 -- `HAVING` clause without a `GROUP BY` clause.
 SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
-  +---+
-  |sum|
-  +---+
-  | 78|
-  +---+
++---+
+|sum|
++---+
+| 78|
++---+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-inline-table.md b/docs/sql-ref-syntax-qry-select-inline-table.md
index 7a532857bc322..9c33cbc679f06 100644
--- a/docs/sql-ref-syntax-qry-select-inline-table.md
+++ b/docs/sql-ref-syntax-qry-select-inline-table.md
@@ -53,30 +53,30 @@ VALUES ( expression [ , ... ] ) [ table_alias ]
 {% highlight sql %}
 -- single row, without a table alias
 SELECT * FROM VALUES ("one", 1);
-  +----+----+
-  |col1|col2|
-  +----+----+
-  | one|   1|
-  +----+----+
++----+----+
+|col1|col2|
++----+----+
+| one|   1|
++----+----+
 
 -- three rows with a table alias
 SELECT * FROM VALUES ("one", 1), ("two", 2), ("three", null) AS data(a, b);
-  +-----+----+
-  |    a|   b|
-  +-----+----+
-  |  one|   1|
-  |  two|   2|
-  |three|null|
-  +-----+----+
++-----+----+
+|    a|   b|
++-----+----+
+|  one|   1|
+|  two|   2|
+|three|null|
++-----+----+
 
 -- complex types with a table alias
 SELECT * FROM VALUES ("one", array(0, 1)), ("two", array(2, 3)) AS data(a, b);
-  +---+------+
-  |  a|     b|
-  +---+------+
-  |one|[0, 1]|
-  |two|[2, 3]|
-  +---+------+
++---+------+
+|  a|     b|
++---+------+
+|one|[0, 1]|
+|two|[2, 3]|
++---+------+
 {% endhighlight %}
 
 ### Related Statement
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index 41b7603a3a25e..0b1bb1eb8fd61 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -139,121 +139,120 @@ An anti join returns values from the left relation that has no match with the ri
 {% highlight sql %}
 -- Use employee and department tables to demonstrate different type of joins.
 SELECT * FROM employee;
-
-  +---+-----+------+
-  | id| name|deptno|
-  +---+-----+------+
-  |105|Chloe|     5|
-  |103| Paul|     3|
-  |101| John|     1|
-  |102| Lisa|     2|
-  |104| Evan|     4|
-  |106|  Amy|     6|
-  +---+-----+------+
++---+-----+------+
+| id| name|deptno|
++---+-----+------+
+|105|Chloe|     5|
+|103| Paul|     3|
+|101| John|     1|
+|102| Lisa|     2|
+|104| Evan|     4|
+|106|  Amy|     6|
++---+-----+------+
 
 SELECT * FROM department;
-  +------+-----------+
-  |deptno|   deptname|
-  +------+-----------+
-  |     3|Engineering|
-  |     2|      Sales|
-  |     1|  Marketing|
-  +------+-----------+
++------+-----------+
+|deptno|   deptname|
++------+-----------+
+|     3|Engineering|
+|     2|      Sales|
+|     1|  Marketing|
++------+-----------+
 
 -- Use employee and department tables to demonstrate inner join.
 SELECT id, name, employee.deptno, deptname
     FROM employee INNER JOIN department ON employee.deptno = department.deptno;
-  +---+-----+------+-----------|
-  | id| name|deptno|   deptname|
-  +---+-----+------+-----------|
-  |103| Paul|     3|Engineering|
-  |101| John|     1|  Marketing|
-  |102| Lisa|     2|      Sales|
-  +---+-----+------+-----------|
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|103| Paul|     3|Engineering|
+|101| John|     1|  Marketing|
+|102| Lisa|     2|      Sales|
++---+-----+------+-----------|
 
 -- Use employee and department tables to demonstrate left join.
 SELECT id, name, employee.deptno, deptname
     FROM employee LEFT JOIN department ON employee.deptno = department.deptno;
-  +---+-----+------+-----------|
-  | id| name|deptno|   deptname|
-  +---+-----+------+-----------|
-  |105|Chloe|     5|       NULL|
-  |103| Paul|     3|Engineering|
-  |101| John|     1|  Marketing|
-  |102| Lisa|     2|      Sales|
-  |104| Evan|     4|       NULL|
-  |106|  Amy|     6|       NULL|
-  +---+-----+------+-----------|
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|105|Chloe|     5|       NULL|
+|103| Paul|     3|Engineering|
+|101| John|     1|  Marketing|
+|102| Lisa|     2|      Sales|
+|104| Evan|     4|       NULL|
+|106|  Amy|     6|       NULL|
++---+-----+------+-----------|
 
 -- Use employee and department tables to demonstrate right join.
 SELECT id, name, employee.deptno, deptname
     FROM employee RIGHT JOIN department ON employee.deptno = department.deptno;
-  +---+-----+------+-----------|
-  | id| name|deptno|   deptname|
-  +---+-----+------+-----------|
-  |103| Paul|     3|Engineering|
-  |101| John|     1|  Marketing|
-  |102| Lisa|     2|      Sales|
-  +---+-----+------+-----------|
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|103| Paul|     3|Engineering|
+|101| John|     1|  Marketing|
+|102| Lisa|     2|      Sales|
++---+-----+------+-----------|
 
 -- Use employee and department tables to demonstrate full join.
 SELECT id, name, employee.deptno, deptname
     FROM employee FULL JOIN department ON employee.deptno = department.deptno;
-  +---+-----+------+-----------|
-  | id| name|deptno|   deptname|
-  +---+-----+------+-----------|
-  |101| John|     1|  Marketing|
-  |106|  Amy|     6|       NULL|
-  |103| Paul|     3|Engineering|
-  |105|Chloe|     5|       NULL|
-  |104| Evan|     4|       NULL|
-  |102| Lisa|     2|      Sales|
-  +---+-----+------+-----------|
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|101| John|     1|  Marketing|
+|106|  Amy|     6|       NULL|
+|103| Paul|     3|Engineering|
+|105|Chloe|     5|       NULL|
+|104| Evan|     4|       NULL|
+|102| Lisa|     2|      Sales|
++---+-----+------+-----------|
 
 -- Use employee and department tables to demonstrate cross join.
 SELECT id, name, employee.deptno, deptname FROM employee CROSS JOIN department;
-  +---+-----+------+-----------|
-  | id| name|deptno|   deptname|
-  +---+-----+------+-----------|
-  |105|Chloe|     5|Engineering|
-  |105|Chloe|     5|  Marketing|
-  |105|Chloe|     5|      Sales|
-  |103| Paul|     3|Engineering|
-  |103| Paul|     3|  Marketing|
-  |103| Paul|     3|      Sales|
-  |101| John|     1|Engineering|
-  |101| John|     1|  Marketing|
-  |101| John|     1|      Sales|
-  |102| Lisa|     2|Engineering|
-  |102| Lisa|     2|  Marketing|
-  |102| Lisa|     2|      Sales|
-  |104| Evan|     4|Engineering|
-  |104| Evan|     4|  Marketing|
-  |104| Evan|     4|      Sales|
-  |106|  Amy|     4|Engineering|
-  |106|  Amy|     4|  Marketing|
-  |106|  Amy|     4|      Sales|
-  +---+-----+------+-----------|
++---+-----+------+-----------|
+| id| name|deptno|   deptname|
++---+-----+------+-----------|
+|105|Chloe|     5|Engineering|
+|105|Chloe|     5|  Marketing|
+|105|Chloe|     5|      Sales|
+|103| Paul|     3|Engineering|
+|103| Paul|     3|  Marketing|
+|103| Paul|     3|      Sales|
+|101| John|     1|Engineering|
+|101| John|     1|  Marketing|
+|101| John|     1|      Sales|
+|102| Lisa|     2|Engineering|
+|102| Lisa|     2|  Marketing|
+|102| Lisa|     2|      Sales|
+|104| Evan|     4|Engineering|
+|104| Evan|     4|  Marketing|
+|104| Evan|     4|      Sales|
+|106|  Amy|     4|Engineering|
+|106|  Amy|     4|  Marketing|
+|106|  Amy|     4|      Sales|
++---+-----+------+-----------|
 
 -- Use employee and department tables to demonstrate semi join.
 SELECT * FROM employee SEMI JOIN department ON employee.deptno = department.deptno;
-  +---+-----+------+
-  | id| name|deptno|
-  +---+-----+------+
-  |103| Paul|     3|
-  |101| John|     1|
-  |102| Lisa|     2|
-  +---+-----+------+
++---+-----+------+
+| id| name|deptno|
++---+-----+------+
+|103| Paul|     3|
+|101| John|     1|
+|102| Lisa|     2|
++---+-----+------+
 
 -- Use employee and department tables to demonstrate anti join.
 SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.deptno;
-  +---+-----+------+
-  | id| name|deptno|
-  +---+-----+------+
-  |105|Chloe|     5|
-  |104| Evan|     4|
-  |106|  Amy|     6|
-  +---+-----+------+
++---+-----+------+
+| id| name|deptno|
++---+-----+------+
+|105|Chloe|     5|
+|104| Evan|     4|
+|106|  Amy|     6|
++---+-----+------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 0ceb705889b47..eaeaed068102f 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -60,37 +60,37 @@ INSERT INTO person VALUES
 
 -- Select the first two rows.
 SELECT name, age FROM person ORDER BY name LIMIT 2;
-  +------+---+
-  |  name|age|
-  +------+---+
-  |Anil B| 18|
-  |Jack N| 16|
-  +------+---+
++------+---+
+|  name|age|
++------+---+
+|Anil B| 18|
+|Jack N| 16|
++------+---+
 
 -- Specifying ALL option on LIMIT returns all the rows.
 SELECT name, age FROM person ORDER BY name LIMIT ALL;
-  +-------+---+
-  |   name|age|
-  +-------+---+
-  | Anil B| 18|
-  | Jack N| 16|
-  | John A| 18|
-  | Mike A| 25|
-  |Shone S| 16|
-  |Zen Hui| 25|
-  +-------+---+
++-------+---+
+|   name|age|
++-------+---+
+| Anil B| 18|
+| Jack N| 16|
+| John A| 18|
+| Mike A| 25|
+|Shone S| 16|
+|Zen Hui| 25|
++-------+---+
 
 -- A function expression as an input to LIMIT.
 SELECT name, age FROM person ORDER BY name LIMIT length('SPARK');
-  +-------+---+
-  |   name|age|
-  +-------+---+
-  | Anil B| 18|
-  | Jack N| 16|
-  | John A| 18|
-  | Mike A| 25|
-  |Shone S| 16|
-  +-------+---+
++-------+---+
+|   name|age|
++-------+---+
+| Anil B| 18|
+| Jack N| 16|
+| John A| 18|
+| Mike A| 25|
+|Shone S| 16|
++-------+---+
 
 -- A non-foldable expression as an input to LIMIT is not allowed.
 SELECT name, age FROM person ORDER BY name LIMIT length(name);
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index cc75367968053..d927177398f7f 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -81,64 +81,64 @@ INSERT INTO person VALUES
 
 -- Sort rows by age. By default rows are sorted in ascending manner with NULL FIRST.
 SELECT name, age FROM person ORDER BY age;
-  +-----+----+
-  | name| age|
-  +-----+----+
-  |Jerry|null|
-  | Mary|null|
-  | John|  30|
-  |  Dan|  50|
-  | Mike|  80|
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+|Jerry|null|
+| Mary|null|
+| John|  30|
+|  Dan|  50|
+| Mike|  80|
++-----+----+
 
 -- Sort rows in ascending manner keeping null values to be last.
 SELECT name, age FROM person ORDER BY age NULLS LAST;
-  +-----+----+
-  | name| age|
-  +-----+----+
-  | John|  30|
-  |  Dan|  50|
-  | Mike|  80|
-  | Mary|null|
-  |Jerry|null|
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+| John|  30|
+|  Dan|  50|
+| Mike|  80|
+| Mary|null|
+|Jerry|null|
++-----+----+
 
 -- Sort rows by age in descending manner, which defaults to NULL LAST.
 SELECT name, age FROM person ORDER BY age DESC;
-  +-----+----+
-  | name| age|
-  +-----+----+
-  | Mike|  80|
-  |  Dan|  50|
-  | John|  30|
-  |Jerry|null|
-  | Mary|null|
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+| Mike|  80|
+|  Dan|  50|
+| John|  30|
+|Jerry|null|
+| Mary|null|
++-----+----+
 
 -- Sort rows in ascending manner keeping null values to be first.
 SELECT name, age FROM person ORDER BY age DESC NULLS FIRST;
-  +-----+----+
-  | name| age|
-  +-----+----+
-  |Jerry|null|
-  | Mary|null|
-  | Mike|  80|
-  |  Dan|  50|
-  | John|  30|
-  +-----+----+
++-----+----+
+| name| age|
++-----+----+
+|Jerry|null|
+| Mary|null|
+| Mike|  80|
+|  Dan|  50|
+| John|  30|
++-----+----+
 
 -- Sort rows based on more than one column with each column having different
 -- sort direction.
 SELECT * FROM person ORDER BY name ASC, age DESC;
-  +---+-----+----+
-  | id| name| age|
-  +---+-----+----+
-  |500|  Dan|  50|
-  |400|Jerry|null|
-  |100| John|  30|
-  |200| Mary|null|
-  |300| Mike|  80|
-  +---+-----+----+
++---+-----+----+
+| id| name| age|
++---+-----+----+
+|500|  Dan|  50|
+|400|Jerry|null|
+|100| John|  30|
+|200| Mary|null|
+|300| Mike|  80|
++---+-----+----+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-setops.md b/docs/sql-ref-syntax-qry-select-setops.md
index 09a207a70c149..98c20941d16bf 100644
--- a/docs/sql-ref-syntax-qry-select-setops.md
+++ b/docs/sql-ref-syntax-qry-select-setops.md
@@ -44,60 +44,60 @@ Note that input relations must have the same number of columns and compatible da
 {% highlight sql %}
 -- Use number1 and number2 tables to demonstrate set operators in this page.
 SELECT * FROM number1;
-  +---+
-  |  c|
-  +---+
-  |  3|
-  |  1|
-  |  2|
-  |  2|
-  |  3|
-  |  4|
-  +---+
++---+
+|  c|
++---+
+|  3|
+|  1|
+|  2|
+|  2|
+|  3|
+|  4|
++---+
   
 SELECT * FROM number2;
-  +---+
-  |  c|
-  +---+
-  |  5|
-  |  1|
-  |  2|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  5|
+|  1|
+|  2|
+|  2|
++---+
 
 SELECT c FROM number1 EXCEPT SELECT c FROM number2;
-  +---+
-  |  c|
-  +---+
-  |  3|
-  |  4|
-  +---+
++---+
+|  c|
++---+
+|  3|
+|  4|
++---+
 
 SELECT c FROM number1 MINUS SELECT c FROM number2;
-  +---+
-  |  c|
-  +---+
-  |  3|
-  |  4|
-  +---+
++---+
+|  c|
++---+
+|  3|
+|  4|
++---+
 
 SELECT c FROM number1 EXCEPT ALL (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  3|
-  |  3|
-  |  4|
-  +---+
++---+
+|  c|
++---+
+|  3|
+|  3|
+|  4|
++---+
 
 SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  3|
-  |  3|
-  |  4|
-  +---+
++---+
+|  c|
++---+
+|  3|
+|  3|
+|  4|
++---+
 {% endhighlight %}
 
 ### INTERSECT
@@ -114,29 +114,29 @@ SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
 
 {% highlight sql %}
 (SELECT c FROM number1) INTERSECT (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  1|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  1|
+|  2|
++---+
 
 (SELECT c FROM number1) INTERSECT DISTINCT (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  1|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  1|
+|  2|
++---+
 
 (SELECT c FROM number1) INTERSECT ALL (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  1|
-  |  2|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  1|
+|  2|
+|  2|
++---+
 {% endhighlight %}
 
 ### UNION
@@ -153,42 +153,42 @@ SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
 
 {% highlight sql %}
 (SELECT c FROM number1) UNION (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  1|
-  |  3|
-  |  5|
-  |  4|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  1|
+|  3|
+|  5|
+|  4|
+|  2|
++---+
 
 (SELECT c FROM number1) UNION DISTINCT (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  1|
-  |  3|
-  |  5|
-  |  4|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  1|
+|  3|
+|  5|
+|  4|
+|  2|
++---+
 
 SELECT c FROM number1 UNION ALL (SELECT c FROM number2);
-  +---+
-  |  c|
-  +---+
-  |  3|
-  |  1|
-  |  2|
-  |  2|
-  |  3|
-  |  4|
-  |  5|
-  |  1|
-  |  2|
-  |  2|
-  +---+
++---+
+|  c|
++---+
+|  3|
+|  1|
+|  2|
+|  2|
+|  3|
+|  4|
+|  5|
+|  1|
+|  2|
+|  2|
++---+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index 315faa5e7d501..1dfa10429709e 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -89,89 +89,89 @@ INSERT INTO person VALUES
 
 -- Sort rows by `name` within each partition in ascending manner
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY name;
-  +--------+----+--------+
-  |    name| age|zip_code|
-  +--------+----+--------+
-  |  Anil K|  27|   94588|
-  |  Dan Li|  18|   94588|
-  |  John V|null|   94588|
-  | Zen Hui|  50|   94588|
-  |Aryan B.|  18|   94511|
-  | David K|  42|   94511|
-  |Lalit B.|null|   94511|
-  +--------+----+--------+
++--------+----+--------+
+|    name| age|zip_code|
++--------+----+--------+
+|  Anil K|  27|   94588|
+|  Dan Li|  18|   94588|
+|  John V|null|   94588|
+| Zen Hui|  50|   94588|
+|Aryan B.|  18|   94511|
+| David K|  42|   94511|
+|Lalit B.|null|   94511|
++--------+----+--------+
 
 -- Sort rows within each partition using column position.
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY 1;
-  +--------+----+--------+
-  |    name| age|zip_code|
-  +--------+----+--------+
-  |  Anil K|  27|   94588|
-  |  Dan Li|  18|   94588|
-  |  John V|null|   94588|
-  | Zen Hui|  50|   94588|
-  |Aryan B.|  18|   94511|
-  | David K|  42|   94511|
-  |Lalit B.|null|   94511|
-  +--------+----+--------+
++--------+----+--------+
+|    name| age|zip_code|
++--------+----+--------+
+|  Anil K|  27|   94588|
+|  Dan Li|  18|   94588|
+|  John V|null|   94588|
+| Zen Hui|  50|   94588|
+|Aryan B.|  18|   94511|
+| David K|  42|   94511|
+|Lalit B.|null|   94511|
++--------+----+--------+
 
 -- Sort rows within partition in ascending manner keeping null values to be last.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age NULLS LAST;
-  +----+--------+--------+
-  | age|    name|zip_code|
-  +----+--------+--------+
-  |  18|  Dan Li|   94588|
-  |  27|  Anil K|   94588|
-  |  50| Zen Hui|   94588|
-  |null|  John V|   94588|
-  |  18|Aryan B.|   94511|
-  |  42| David K|   94511|
-  |null|Lalit B.|   94511|
-  +----+--------+--------+
++----+--------+--------+
+| age|    name|zip_code|
++----+--------+--------+
+|  18|  Dan Li|   94588|
+|  27|  Anil K|   94588|
+|  50| Zen Hui|   94588|
+|null|  John V|   94588|
+|  18|Aryan B.|   94511|
+|  42| David K|   94511|
+|null|Lalit B.|   94511|
++----+--------+--------+
 
 -- Sort rows by age within each partition in descending manner, which defaults to NULL LAST.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC;
-  +----+--------+--------+
-  | age|    name|zip_code|
-  +----+--------+--------+
-  |  50| Zen Hui|   94588|
-  |  27|  Anil K|   94588|
-  |  18|  Dan Li|   94588|
-  |null|  John V|   94588|
-  |  42| David K|   94511|
-  |  18|Aryan B.|   94511|
-  |null|Lalit B.|   94511|
-  +----+--------+--------+
++----+--------+--------+
+| age|    name|zip_code|
++----+--------+--------+
+|  50| Zen Hui|   94588|
+|  27|  Anil K|   94588|
+|  18|  Dan Li|   94588|
+|null|  John V|   94588|
+|  42| David K|   94511|
+|  18|Aryan B.|   94511|
+|null|Lalit B.|   94511|
++----+--------+--------+
 
 -- Sort rows by age within each partition in descending manner keeping null values to be first.
 SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC NULLS FIRST;
-  +----+--------+--------+
-  | age|    name|zip_code|
-  +----+--------+--------+
-  |null|  John V|   94588|
-  |  50| Zen Hui|   94588|
-  |  27|  Anil K|   94588|
-  |  18|  Dan Li|   94588|
-  |null|Lalit B.|   94511|
-  |  42| David K|   94511|
-  |  18|Aryan B.|   94511|
-  +----+--------+--------+
++----+--------+--------+
+| age|    name|zip_code|
++----+--------+--------+
+|null|  John V|   94588|
+|  50| Zen Hui|   94588|
+|  27|  Anil K|   94588|
+|  18|  Dan Li|   94588|
+|null|Lalit B.|   94511|
+|  42| David K|   94511|
+|  18|Aryan B.|   94511|
++----+--------+--------+
 
 -- Sort rows within each partition based on more than one column with each column having
 -- different sort direction.
 SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
     SORT BY name ASC, age DESC;
-  +--------+----+--------+
-  |    name| age|zip_code|
-  +--------+----+--------+
-  |  Anil K|  27|   94588|
-  |  Dan Li|  18|   94588|
-  |  John V|null|   94588|
-  | Zen Hui|  50|   94588|
-  |Aryan B.|  18|   94511|
-  | David K|  42|   94511|
-  |Lalit B.|null|   94511|
-  +--------+----+--------+
++--------+----+--------+
+|    name| age|zip_code|
++--------+----+--------+
+|  Anil K|  27|   94588|
+|  Dan Li|  18|   94588|
+|  John V|null|   94588|
+| Zen Hui|  50|   94588|
+|Aryan B.|  18|   94511|
+| David K|  42|   94511|
+|Lalit B.|null|   94511|
++--------+----+--------+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
index bbfd870d0cbe7..ac8fecaae6413 100644
--- a/docs/sql-ref-syntax-qry-select-tvf.md
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -81,49 +81,49 @@ function_name ( expression [ , ... ] ) [ table_alias ]
 {% highlight sql %}
 -- range call with end
 SELECT * FROM range(6 + cos(3));
-  +---+
-  | id|
-  +---+
-  |  0|
-  |  1|
-  |  2|
-  |  3|
-  |  4|
-  +---+
++---+
+| id|
++---+
+|  0|
+|  1|
+|  2|
+|  3|
+|  4|
++---+
 
 -- range call with start and end
 SELECT * FROM range(5, 10);
-  +---+
-  | id|
-  +---+
-  |  5|
-  |  6|
-  |  7|
-  |  8|
-  |  9|
-  +---+
++---+
+| id|
++---+
+|  5|
+|  6|
+|  7|
+|  8|
+|  9|
++---+
 
 -- range call with numPartitions
 SELECT * FROM range(0, 10, 2, 200);
-  +---+
-  | id|
-  +---+
-  |  0|
-  |  2|
-  |  4|
-  |  6|
-  |  8|
-  +---+
++---+
+| id|
++---+
+|  0|
+|  2|
+|  4|
+|  6|
+|  8|
++---+
 
 -- range call with a table alias
 SELECT * FROM range(5, 8) AS test;
-  +---+
-  | id|
-  +---+
-  |  5|
-  |  6|
-  |  7|
-  +---+
++---+
+| id|
++---+
+|  5|
+|  6|
+|  7|
++---+
 {% endhighlight %}
 
 ### Related Statement
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index 1960367cd42f0..360313fcfff1c 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -53,57 +53,57 @@ INSERT INTO person VALUES
 
 -- Comparison operator in `WHERE` clause.
 SELECT * FROM person WHERE id > 200 ORDER BY id;
-  +---+----+---+
-  | id|name|age|
-  +---+----+---+
-  |300|Mike| 80|
-  |400| Dan| 50|
-  +---+----+---+
++---+----+---+
+| id|name|age|
++---+----+---+
+|300|Mike| 80|
+|400| Dan| 50|
++---+----+---+
 
 -- Comparison and logical operators in `WHERE` clause.
 SELECT * FROM person WHERE id = 200 OR id = 300 ORDER BY id;
-  +---+----+----+
-  | id|name| age|
-  +---+----+----+
-  |200|Mary|null|
-  |300|Mike|  80|
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
 
 -- IS NULL expression in `WHERE` clause.
 SELECT * FROM person WHERE id > 300 OR age IS NULL ORDER BY id;
-  +---+----+----+
-  | id|name| age|
-  +---+----+----+
-  |200|Mary|null|
-  |400| Dan|  50|
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
+|400| Dan|  50|
++---+----+----+
 
 -- Function expression in `WHERE` clause.
 SELECT * FROM person WHERE length(name) > 3 ORDER BY id;
-  +---+----+----+
-  | id|name| age|
-  +---+----+----+
-  |100|John|  30|
-  |200|Mary|null|
-  |300|Mike|  80|
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|100|John|  30|
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
 
 -- `BETWEEN` expression in `WHERE` clause.
 SELECT * FROM person WHERE id BETWEEN 200 AND 300 ORDER BY id;
-  +---+----+----+
-  | id|name| age|
-  +---+----+----+
-  |200|Mary|null|
-  |300|Mike|  80|
-  +---+----+----+
++---+----+----+
+| id|name| age|
++---+----+----+
+|200|Mary|null|
+|300|Mike|  80|
++---+----+----+
 
 -- Scalar Subquery in `WHERE` clause.
 SELECT * FROM person WHERE age > (SELECT avg(age) FROM person);
-  +---+----+---+
-  | id|name|age|
-  +---+----+---+
-  |300|Mike| 80|
-  +---+----+---+
++---+----+---+
+| id|name|age|
++---+----+---+
+|300|Mike| 80|
++---+----+---+
 
 -- Correlated Subquery in `WHERE` clause.
 SELECT * FROM person AS parent
@@ -111,11 +111,11 @@ SELECT * FROM person AS parent
         SELECT 1 FROM person AS child
         WHERE parent.id = child.id AND child.age IS NULL
     );
-  +---+----+----+
-  |id |name|age |
-  +---+----+----+
-  |200|Mary|null|
-  +---+----+----+
++---+----+----+
+|id |name|age |
++---+----+----+
+|200|Mary|null|
++---+----+----+
 {% endhighlight %}
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-window.md b/docs/sql-ref-syntax-qry-window.md
index 4ec1af70f6079..e3762925760e2 100644
--- a/docs/sql-ref-syntax-qry-window.md
+++ b/docs/sql-ref-syntax-qry-window.md
@@ -99,106 +99,106 @@ INSERT INTO employees VALUES ("Paul", "Engineering", 29000, 23);
 INSERT INTO employees VALUES ("Chloe", "Engineering", 23000, 25);
 
 SELECT * FROM employees;
-  +-----+-----------+------+-----+
-  | name|       dept|salary|  age|
-  +-----+-----------+------+-----+
-  |Chloe|Engineering| 23000|   25|
-  | Fred|Engineering| 21000|   28|
-  | Paul|Engineering| 29000|   23|
-  |Helen|  Marketing| 29000|   40|
-  |  Tom|Engineering| 23000|   33|
-  | Jane|  Marketing| 29000|   28|
-  | Jeff|  Marketing| 35000|   38|
-  | Evan|      Sales| 32000|   38|
-  | Lisa|      Sales| 10000|   35|
-  | Alex|      Sales| 30000|   33|
-  +-----+-----------+------+-----+
++-----+-----------+------+-----+
+| name|       dept|salary|  age|
++-----+-----------+------+-----+
+|Chloe|Engineering| 23000|   25|
+| Fred|Engineering| 21000|   28|
+| Paul|Engineering| 29000|   23|
+|Helen|  Marketing| 29000|   40|
+|  Tom|Engineering| 23000|   33|
+| Jane|  Marketing| 29000|   28|
+| Jeff|  Marketing| 35000|   38|
+| Evan|      Sales| 32000|   38|
+| Lisa|      Sales| 10000|   35|
+| Alex|      Sales| 30000|   33|
++-----+-----------+------+-----+
 
 SELECT name, dept, RANK() OVER (PARTITION BY dept ORDER BY salary) AS rank FROM employees;
-  +-----+-----------+------+----+
-  | name|       dept|salary|rank|
-  +-----+-----------+------+----+
-  | Lisa|      Sales| 10000|   1|
-  | Alex|      Sales| 30000|   2|
-  | Evan|      Sales| 32000|   3|
-  | Fred|Engineering| 21000|   1|
-  |  Tom|Engineering| 23000|   2|
-  |Chloe|Engineering| 23000|   2|
-  | Paul|Engineering| 29000|   4|
-  |Helen|  Marketing| 29000|   1|
-  | Jane|  Marketing| 29000|   1|
-  | Jeff|  Marketing| 35000|   3|
-  +-----+-----------+------+----+
++-----+-----------+------+----+
+| name|       dept|salary|rank|
++-----+-----------+------+----+
+| Lisa|      Sales| 10000|   1|
+| Alex|      Sales| 30000|   2|
+| Evan|      Sales| 32000|   3|
+| Fred|Engineering| 21000|   1|
+|  Tom|Engineering| 23000|   2|
+|Chloe|Engineering| 23000|   2|
+| Paul|Engineering| 29000|   4|
+|Helen|  Marketing| 29000|   1|
+| Jane|  Marketing| 29000|   1|
+| Jeff|  Marketing| 35000|   3|
++-----+-----------+------+----+
 
 SELECT name, dept, DENSE_RANK() OVER (PARTITION BY dept ORDER BY salary ROWS BETWEEN
     UNBOUNDED PRECEDING AND CURRENT ROW) AS dense_rank FROM employees;
-  +-----+-----------+------+----------+
-  | name|       dept|salary|dense_rank|
-  +-----+-----------+------+----------+
-  | Lisa|      Sales| 10000|         1|
-  | Alex|      Sales| 30000|         2|
-  | Evan|      Sales| 32000|         3|
-  | Fred|Engineering| 21000|         1|
-  |  Tom|Engineering| 23000|         2|
-  |Chloe|Engineering| 23000|         2|
-  | Paul|Engineering| 29000|         3|
-  |Helen|  Marketing| 29000|         1|
-  | Jane|  Marketing| 29000|         1|
-  | Jeff|  Marketing| 35000|         2|
-  +-----+-----------+------+----------+
++-----+-----------+------+----------+
+| name|       dept|salary|dense_rank|
++-----+-----------+------+----------+
+| Lisa|      Sales| 10000|         1|
+| Alex|      Sales| 30000|         2|
+| Evan|      Sales| 32000|         3|
+| Fred|Engineering| 21000|         1|
+|  Tom|Engineering| 23000|         2|
+|Chloe|Engineering| 23000|         2|
+| Paul|Engineering| 29000|         3|
+|Helen|  Marketing| 29000|         1|
+| Jane|  Marketing| 29000|         1|
+| Jeff|  Marketing| 35000|         2|
++-----+-----------+------+----------+
 
 SELECT name, dept, age, CUME_DIST() OVER (PARTITION BY dept ORDER BY age
     RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cume_dist FROM employees;
-  +-----+-----------+------+------------------+
-  | name|       dept|age   |         cume_dist|
-  +-----+-----------+------+------------------+
-  | Alex|      Sales|    33|0.3333333333333333|
-  | Lisa|      Sales|    35|0.6666666666666666|
-  | Evan|      Sales|    38|               1.0|
-  | Paul|Engineering|    23|              0.25|
-  |Chloe|Engineering|    25|              0.75|
-  | Fred|Engineering|    28|              0.25|
-  |  Tom|Engineering|    33|               1.0|
-  | Jane|  Marketing|    28|0.3333333333333333|
-  | Jeff|  Marketing|    38|0.6666666666666666|
-  |Helen|  Marketing|    40|               1.0|
-  +-----+-----------+------+------------------+
++-----+-----------+------+------------------+
+| name|       dept|age   |         cume_dist|
++-----+-----------+------+------------------+
+| Alex|      Sales|    33|0.3333333333333333|
+| Lisa|      Sales|    35|0.6666666666666666|
+| Evan|      Sales|    38|               1.0|
+| Paul|Engineering|    23|              0.25|
+|Chloe|Engineering|    25|              0.75|
+| Fred|Engineering|    28|              0.25|
+|  Tom|Engineering|    33|               1.0|
+| Jane|  Marketing|    28|0.3333333333333333|
+| Jeff|  Marketing|    38|0.6666666666666666|
+|Helen|  Marketing|    40|               1.0|
++-----+-----------+------+------------------+
 
 SELECT name, dept, salary, MIN(salary) OVER (PARTITION BY dept ORDER BY salary) AS min
     FROM employees;
-  +-----+-----------+------+-----+
-  | name|       dept|salary|  min|
-  +-----+-----------+------+-----+
-  | Lisa|      Sales| 10000|10000|
-  | Alex|      Sales| 30000|10000|
-  | Evan|      Sales| 32000|10000|
-  |Helen|  Marketing| 29000|29000|
-  | Jane|  Marketing| 29000|29000|
-  | Jeff|  Marketing| 35000|29000|
-  | Fred|Engineering| 21000|21000|
-  |  Tom|Engineering| 23000|21000|
-  |Chloe|Engineering| 23000|21000|
-  | Paul|Engineering| 29000|21000|
-  +-----+-----------+------+-----+
++-----+-----------+------+-----+
+| name|       dept|salary|  min|
++-----+-----------+------+-----+
+| Lisa|      Sales| 10000|10000|
+| Alex|      Sales| 30000|10000|
+| Evan|      Sales| 32000|10000|
+|Helen|  Marketing| 29000|29000|
+| Jane|  Marketing| 29000|29000|
+| Jeff|  Marketing| 35000|29000|
+| Fred|Engineering| 21000|21000|
+|  Tom|Engineering| 23000|21000|
+|Chloe|Engineering| 23000|21000|
+| Paul|Engineering| 29000|21000|
++-----+-----------+------+-----+
 
 SELECT name, salary,
     LAG(salary) OVER (PARTITION BY dept ORDER BY salary) AS lag,
     LEAD(salary, 1, 0) OVER (PARTITION BY dept ORDER BY salary) AS lead
     FROM employees;
-  +-----+-----------+------+-----+-----+
-  | name|       dept|salary|  lag| lead|
-  +-----+-----------+------+-----+-----+
-  | Lisa|      Sales| 10000|NULL |30000|
-  | Alex|      Sales| 30000|10000|32000|
-  | Evan|      Sales| 32000|30000|    0|
-  | Fred|Engineering| 21000| NULL|23000|
-  |Chloe|Engineering| 23000|21000|23000|
-  |  Tom|Engineering| 23000|23000|29000|
-  | Paul|Engineering| 29000|23000|    0|
-  |Helen|  Marketing| 29000| NULL|29000|
-  | Jane|  Marketing| 29000|29000|35000|
-  | Jeff|  Marketing| 35000|29000|    0|
-  +-----+-----------+------+-----+-----+
++-----+-----------+------+-----+-----+
+| name|       dept|salary|  lag| lead|
++-----+-----------+------+-----+-----+
+| Lisa|      Sales| 10000|NULL |30000|
+| Alex|      Sales| 30000|10000|32000|
+| Evan|      Sales| 32000|30000|    0|
+| Fred|Engineering| 21000| NULL|23000|
+|Chloe|Engineering| 23000|21000|23000|
+|  Tom|Engineering| 23000|23000|29000|
+| Paul|Engineering| 29000|23000|    0|
+|Helen|  Marketing| 29000| NULL|29000|
+| Jane|  Marketing| 29000|29000|35000|
+| Jeff|  Marketing| 35000|29000|    0|
++-----+-----------+------+-----+-----+
 {% endhighlight %}
 
 ### Related Statements

From 3aa659ce29877f386a24da9d04e66069d04afaa8 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sat, 2 May 2020 17:54:36 +0900
Subject: [PATCH 0618/1280] [MINOR][SQL][TESTS] Disable UI in SQL benchmarks by
 default

### What changes were proposed in this pull request?
Set `spark.ui.enabled` to `false` in `SqlBasedBenchmark.getSparkSession`. This disables UI in all SQL benchmarks by default.

### Why are the changes needed?
UI overhead lowers numbers in the `Relative` column and impacts on `Stdev` in benchmark results.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Checked by running `DateTimeRebaseBenchmark`.

Closes #28432 from MaxGekk/ui-off-in-benchmarks.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 13dddee9a8490ead00ff00bd741db4a170dfd759)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/execution/benchmark/DataSourceReadBenchmark.scala | 2 --
 .../spark/sql/execution/benchmark/FilterPushdownBenchmark.scala | 2 --
 .../spark/sql/execution/benchmark/SqlBasedBenchmark.scala       | 2 ++
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
index d29c5e3f88010..0fc43c7052d06 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DataSourceReadBenchmark.scala
@@ -23,7 +23,6 @@ import scala.util.Random
 
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.internal.config.UI._
 import org.apache.spark.sql.{DataFrame, DataFrameWriter, Row, SparkSession}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.parquet.{SpecificParquetRecordReaderBase, VectorizedParquetRecordReader}
@@ -52,7 +51,6 @@ object DataSourceReadBenchmark extends SqlBasedBenchmark {
       .set("spark.master", "local[1]")
       .setIfMissing("spark.driver.memory", "3g")
       .setIfMissing("spark.executor.memory", "3g")
-      .setIfMissing(UI_ENABLED, false)
 
     val sparkSession = SparkSession.builder.config(conf).getOrCreate()
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
index 444ffa4f99697..b3f65d40ad95b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/FilterPushdownBenchmark.scala
@@ -23,7 +23,6 @@ import scala.util.Random
 
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
-import org.apache.spark.internal.config.UI._
 import org.apache.spark.sql.{DataFrame, SparkSession}
 import org.apache.spark.sql.functions.monotonically_increasing_id
 import org.apache.spark.sql.internal.SQLConf
@@ -49,7 +48,6 @@ object FilterPushdownBenchmark extends SqlBasedBenchmark {
       .set("spark.master", "local[1]")
       .setIfMissing("spark.driver.memory", "3g")
       .setIfMissing("spark.executor.memory", "3g")
-      .setIfMissing(UI_ENABLED, false)
       .setIfMissing("orc.compression", "snappy")
       .setIfMissing("spark.sql.parquet.compression.codec", "snappy")
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
index ee7a03e5e0542..28387dcef125b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/SqlBasedBenchmark.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.execution.benchmark
 
 import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
+import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.{Dataset, SparkSession}
 import org.apache.spark.sql.SaveMode.Overwrite
 import org.apache.spark.sql.catalyst.plans.SQLHelper
@@ -37,6 +38,7 @@ trait SqlBasedBenchmark extends BenchmarkBase with SQLHelper {
       .appName(this.getClass.getCanonicalName)
       .config(SQLConf.SHUFFLE_PARTITIONS.key, 1)
       .config(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key, 1)
+      .config(UI_ENABLED.key, false)
       .getOrCreate()
   }
 

From 9c101e982bb990a4b9297460f42f4f48a9be6f50 Mon Sep 17 00:00:00 2001
From: Michael Chirico <michael.chirico@grabtaxi.com>
Date: Sun, 3 May 2020 12:40:20 +0900
Subject: [PATCH 0619/1280] [SPARK-31571][R] Overhaul stop/message/warning
 calls to be more canonical

### What changes were proposed in this pull request?

Internal usages like `{stop,warning,message}({paste,paste0,sprintf}` and `{stop,warning,message}(some_literal_string_as_variable` have been removed and replaced as appropriate.

### Why are the changes needed?

CRAN policy recommends against using such constructions to build error messages, in particular because it makes the process of creating portable error messages for the package more onerous.

### Does this PR introduce any user-facing change?

There may be some small grammatical changes visible in error messaging.

### How was this patch tested?

Not done

Closes #28365 from MichaelChirico/r-stop-paste.

Authored-by: Michael Chirico <michael.chirico@grabtaxi.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f53d8c63e80172295e2fbc805c0c391bdececcaa)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/DataFrame.R                   | 40 ++++++++---------
 R/pkg/R/RDD.R                         |  2 +-
 R/pkg/R/SQLContext.R                  | 17 ++++----
 R/pkg/R/client.R                      |  7 ++-
 R/pkg/R/context.R                     |  8 ++--
 R/pkg/R/deserialize.R                 |  2 +-
 R/pkg/R/group.R                       |  4 +-
 R/pkg/R/install.R                     | 62 +++++++++++----------------
 R/pkg/R/mllib_classification.R        |  4 +-
 R/pkg/R/mllib_stat.R                  |  3 +-
 R/pkg/R/pairRDD.R                     |  2 +-
 R/pkg/R/schema.R                      |  2 +-
 R/pkg/R/serialize.R                   |  4 +-
 R/pkg/R/sparkR.R                      |  9 ++--
 R/pkg/R/utils.R                       | 53 +++++++++++------------
 R/pkg/tests/fulltests/test_sparkSQL.R |  2 +-
 16 files changed, 103 insertions(+), 118 deletions(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 09e831814b893..15b3ce2935427 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -431,7 +431,7 @@ setMethod("coltypes",
                 if (is.null(type)) {
                   specialtype <- specialtypeshandle(x)
                   if (is.null(specialtype)) {
-                    stop(paste("Unsupported data type: ", x))
+                    stop("Unsupported data type: ", x)
                   }
                   type <- PRIMITIVE_TYPES[[specialtype]]
                 }
@@ -829,8 +829,8 @@ setMethod("repartitionByRange",
                 jcol <- lapply(cols, function(c) { c@jc })
                 sdf <- callJMethod(x@sdf, "repartitionByRange", numToInt(numPartitions), jcol)
               } else {
-                stop(paste("numPartitions and col must be numeric and Column; however, got",
-                           class(numPartitions), "and", class(col)))
+                stop("numPartitions and col must be numeric and Column; however, got ",
+                     class(numPartitions), " and ", class(col))
               }
             } else if (!is.null(col))  {
               # only columns are specified
@@ -839,7 +839,7 @@ setMethod("repartitionByRange",
                 jcol <- lapply(cols, function(c) { c@jc })
                 sdf <- callJMethod(x@sdf, "repartitionByRange", jcol)
               } else {
-                stop(paste("col must be Column; however, got", class(col)))
+                stop("col must be Column; however, got ", class(col))
               }
             } else if (!is.null(numPartitions)) {
               # only numPartitions is specified
@@ -1068,10 +1068,10 @@ setMethod("sample",
           signature(x = "SparkDataFrame"),
           function(x, withReplacement = FALSE, fraction, seed) {
             if (!is.numeric(fraction)) {
-              stop(paste("fraction must be numeric; however, got", class(fraction)))
+              stop("fraction must be numeric; however, got ", class(fraction))
             }
             if (!is.logical(withReplacement)) {
-              stop(paste("withReplacement must be logical; however, got", class(withReplacement)))
+              stop("withReplacement must be logical; however, got ", class(withReplacement))
             }
 
             if (!missing(seed)) {
@@ -1211,11 +1211,10 @@ setMethod("collect",
                 checkSchemaInArrow(schema(x))
                 TRUE
               }, error = function(e) {
-                warning(paste0("The conversion from Spark DataFrame to R DataFrame was attempted ",
-                               "with Arrow optimization because ",
-                               "'spark.sql.execution.arrow.sparkr.enabled' is set to true; ",
-                               "however, failed, attempting non-optimization. Reason: ",
-                               e))
+                warning("The conversion from Spark DataFrame to R DataFrame was attempted ",
+                        "with Arrow optimization because ",
+                        "'spark.sql.execution.arrow.sparkr.enabled' is set to true; ",
+                        "however, failed, attempting non-optimization. Reason: ", e)
                 FALSE
               })
             }
@@ -1508,8 +1507,8 @@ dapplyInternal <- function(x, func, schema) {
     if (inherits(schema, "structType")) {
       checkSchemaInArrow(schema)
     } else if (is.null(schema)) {
-      stop(paste0("Arrow optimization does not support 'dapplyCollect' yet. Please disable ",
-                  "Arrow optimization or use 'collect' and 'dapply' APIs instead."))
+      stop("Arrow optimization does not support 'dapplyCollect' yet. Please disable ",
+           "Arrow optimization or use 'collect' and 'dapply' APIs instead.")
     } else {
       stop("'schema' should be DDL-formatted string or structType.")
     }
@@ -2012,8 +2011,8 @@ setMethod("[", signature(x = "SparkDataFrame"),
               x
             } else {
               if (class(i) != "Column") {
-                stop(paste0("Expressions other than filtering predicates are not supported ",
-                      "in the first parameter of extract operator [ or subset() method."))
+                stop("Expressions other than filtering predicates are not supported ",
+                      "in the first parameter of extract operator [ or subset() method.")
               }
               filter(x, i)
             }
@@ -2604,18 +2603,17 @@ setMethod("join",
               if (is.null(joinType)) {
                 sdf <- callJMethod(x@sdf, "join", y@sdf, joinExpr@jc)
               } else {
-                if (joinType %in% c("inner", "cross",
+                validJoinTypes <- c("inner", "cross",
                     "outer", "full", "fullouter", "full_outer",
                     "left", "leftouter", "left_outer",
                     "right", "rightouter", "right_outer",
-                    "semi", "left_semi", "leftsemi", "anti", "left_anti", "leftanti")) {
+                    "semi", "leftsemi", "left_semi", "anti", "leftanti", "left_anti")
+                if (joinType %in% validJoinTypes) {
                   joinType <- gsub("_", "", joinType, fixed = TRUE)
                   sdf <- callJMethod(x@sdf, "join", y@sdf, joinExpr@jc, joinType)
                 } else {
-                  stop(paste("joinType must be one of the following types:",
-                       "'inner', 'cross', 'outer', 'full', 'fullouter', 'full_outer',",
-                       "'left', 'leftouter', 'left_outer', 'right', 'rightouter', 'right_outer',",
-                       "'semi', 'leftsemi', 'left_semi', 'anti', 'leftanti' or 'left_anti'."))
+                  stop("joinType must be one of the following types: ",
+                       "'", paste(validJoinTypes, collapse = "', '"), "'")
                 }
               }
             }
diff --git a/R/pkg/R/RDD.R b/R/pkg/R/RDD.R
index 7ee725d90d550..7a1d157bb8a36 100644
--- a/R/pkg/R/RDD.R
+++ b/R/pkg/R/RDD.R
@@ -947,7 +947,7 @@ setMethod("takeSample", signature(x = "RDD", withReplacement = "logical",
             MAXINT <- .Machine$integer.max
 
             if (num < 0)
-              stop(paste("Negative number of elements requested"))
+              stop("Negative number of elements requested")
 
             if (initialCount > MAXINT - 1) {
               maxSelected <- MAXINT - 1
diff --git a/R/pkg/R/SQLContext.R b/R/pkg/R/SQLContext.R
index 1ef2641742704..c0ac68332ec41 100644
--- a/R/pkg/R/SQLContext.R
+++ b/R/pkg/R/SQLContext.R
@@ -34,7 +34,7 @@ getInternalType <- function(x) {
          Date = "date",
          POSIXlt = "timestamp",
          POSIXct = "timestamp",
-         stop(paste("Unsupported type for SparkDataFrame:", class(x))))
+         stop("Unsupported type for SparkDataFrame: ", class(x)))
 }
 
 #' return the SparkSession
@@ -112,9 +112,9 @@ sparkR.conf <- function(key, defaultValue) {
               error = function(e) {
                 estr <- as.character(e)
                 if (any(grepl("java.util.NoSuchElementException", estr, fixed = TRUE))) {
-                  stop(paste0("Config '", key, "' is not set"))
+                  stop("Config '", key, "' is not set")
                 } else {
-                  stop(paste0("Unknown error: ", estr))
+                  stop("Unknown error: ", estr)
                 }
               })
     } else {
@@ -208,7 +208,7 @@ getSchema <- function(schema, firstRow = NULL, rdd = NULL) {
     names <- lapply(names, function(n) {
       nn <- gsub(".", "_", n, fixed = TRUE)
       if (nn != n) {
-        warning(paste("Use", nn, "instead of", n, "as column name"))
+        warning("Use ", nn, " instead of ", n, " as column name")
       }
       nn
     })
@@ -290,10 +290,9 @@ createDataFrame <- function(data, schema = NULL, samplingRatio = 1.0,
         TRUE
       },
       error = function(e) {
-        warning(paste0("createDataFrame attempted Arrow optimization because ",
-                       "'spark.sql.execution.arrow.sparkr.enabled' is set to true; however, ",
-                       "failed, attempting non-optimization. Reason: ",
-                       e))
+        warning("createDataFrame attempted Arrow optimization because ",
+                "'spark.sql.execution.arrow.sparkr.enabled' is set to true; however, ",
+                "failed, attempting non-optimization. Reason: ", e)
         FALSE
       })
     }
@@ -326,7 +325,7 @@ createDataFrame <- function(data, schema = NULL, samplingRatio = 1.0,
   } else if (inherits(data, "RDD")) {
     rdd <- data
   } else {
-    stop(paste("unexpected type:", class(data)))
+    stop("unexpected type: ", class(data))
   }
 
   schema <- getSchema(schema, firstRow, rdd)
diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R
index 872b21443eaad..797a5c7da1549 100644
--- a/R/pkg/R/client.R
+++ b/R/pkg/R/client.R
@@ -102,10 +102,9 @@ checkJavaVersion <- function() {
     javaVersionNum <- as.integer(versions[1])
   }
   if (javaVersionNum < minJavaVersion || javaVersionNum >= maxJavaVersion) {
-    stop(paste0("Java version, greater than or equal to ", minJavaVersion,
-                " and less than ", maxJavaVersion,
-                ", is required for this package; found version: ",
-                javaVersionStr))
+    stop("Java version, greater than or equal to ", minJavaVersion,
+         " and less than ", maxJavaVersion, ", is required for this ",
+         "package; found version: ", javaVersionStr)
   }
   return(javaVersionNum)
 }
diff --git a/R/pkg/R/context.R b/R/pkg/R/context.R
index d96a287f818a2..e3c9d9f8793d6 100644
--- a/R/pkg/R/context.R
+++ b/R/pkg/R/context.R
@@ -144,13 +144,13 @@ parallelize <- function(sc, coll, numSlices = 1) {
   if ((!is.list(coll) && !is.vector(coll)) || is.data.frame(coll)) {
   # nolint end
     if (is.data.frame(coll)) {
-      message(paste("context.R: A data frame is parallelized by columns."))
+      message("context.R: A data frame is parallelized by columns.")
     } else {
       if (is.matrix(coll)) {
-        message(paste("context.R: A matrix is parallelized by elements."))
+        message("context.R: A matrix is parallelized by elements.")
       } else {
-        message(paste("context.R: parallelize() currently only supports lists and vectors.",
-                      "Calling as.list() to coerce coll into a list."))
+        message("context.R: parallelize() currently only supports lists and vectors. ",
+                "Calling as.list() to coerce coll into a list.")
       }
     }
     coll <- as.list(coll)
diff --git a/R/pkg/R/deserialize.R b/R/pkg/R/deserialize.R
index ca4a6e342d772..3e7c456bd548d 100644
--- a/R/pkg/R/deserialize.R
+++ b/R/pkg/R/deserialize.R
@@ -57,7 +57,7 @@ readTypedObject <- function(con, type) {
     "s" = readStruct(con),
     "n" = NULL,
     "j" = getJobj(readString(con)),
-    stop(paste("Unsupported type for deserialization", type)))
+    stop("Unsupported type for deserialization ", type))
 }
 
 readStringData <- function(con, len) {
diff --git a/R/pkg/R/group.R b/R/pkg/R/group.R
index 2b7995e1e37f6..99d62240a3b2a 100644
--- a/R/pkg/R/group.R
+++ b/R/pkg/R/group.R
@@ -234,8 +234,8 @@ gapplyInternal <- function(x, func, schema) {
     if (inherits(schema, "structType")) {
       checkSchemaInArrow(schema)
     } else if (is.null(schema)) {
-      stop(paste0("Arrow optimization does not support 'gapplyCollect' yet. Please disable ",
-                  "Arrow optimization or use 'collect' and 'gapply' APIs instead."))
+      stop("Arrow optimization does not support 'gapplyCollect' yet. Please disable ",
+           "Arrow optimization or use 'collect' and 'gapply' APIs instead.")
     } else {
       stop("'schema' should be DDL-formatted string or structType.")
     }
diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R
index 8c5355a8324f9..ea2c0b4c0f42f 100644
--- a/R/pkg/R/install.R
+++ b/R/pkg/R/install.R
@@ -89,8 +89,8 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
   }
 
   if (overwrite) {
-    message(paste0("Overwrite = TRUE: download and overwrite the tar file",
-                   "and Spark package directory if they exist."))
+    message("Overwrite = TRUE: download and overwrite the tar file",
+            "and Spark package directory if they exist.")
   }
 
   releaseUrl <- Sys.getenv("SPARKR_RELEASE_DOWNLOAD_URL")
@@ -103,12 +103,11 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
   # can use dir.exists(packageLocalDir) under R 3.2.0 or later
   if (!is.na(file.info(packageLocalDir)$isdir) && !overwrite) {
     if (releaseUrl != "") {
-      message(paste(packageName, "found, setting SPARK_HOME to", packageLocalDir))
+      message(packageName, " found, setting SPARK_HOME to ", packageLocalDir)
     } else {
-      fmt <- "%s for Hadoop %s found, setting SPARK_HOME to %s"
-      msg <- sprintf(fmt, version, ifelse(hadoopVersion == "without", "Free build", hadoopVersion),
-                     packageLocalDir)
-      message(msg)
+      message(version, " for Hadoop ",
+              if (hadoopVersion == "without") "Free build" else hadoopVersion,
+              " found, setting SPARK_HOME to ", packageLocalDir)
     }
     Sys.setenv(SPARK_HOME = packageLocalDir)
     return(invisible(packageLocalDir))
@@ -127,26 +126,23 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
       success <- downloadUrl(releaseUrl, packageLocalPath)
       if (!success) {
         unlink(packageLocalPath)
-        stop(paste0("Fetch failed from ", releaseUrl))
+        stop("Fetch failed from ", releaseUrl)
       }
     } else {
       robustDownloadTar(mirrorUrl, version, hadoopVersion, packageName, packageLocalPath)
     }
   }
 
-  message(sprintf("Installing to %s", localDir))
+  message("Installing to ", localDir)
   # There are two ways untar can fail - untar could stop() on errors like incomplete block on file
   # or, tar command can return failure code
   success <- tryCatch(untar(tarfile = packageLocalPath, exdir = localDir) == 0,
                      error = function(e) {
-                       message(e)
-                       message()
+                       message(e, "\n")
                        FALSE
                      },
                      warning = function(w) {
-                       # Treat warning as error, add an empty line with message()
-                       message(w)
-                       message()
+                       message(w, "\n")
                        FALSE
                      })
   if (!tarExists || overwrite || !success) {
@@ -160,7 +156,7 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL,
   if (!success) stop("Extract archive failed.")
   message("DONE.")
   Sys.setenv(SPARK_HOME = packageLocalDir)
-  message(paste("SPARK_HOME set to", packageLocalDir))
+  message("SPARK_HOME set to ", packageLocalDir)
   invisible(packageLocalDir)
 }
 
@@ -173,7 +169,7 @@ robustDownloadTar <- function(mirrorUrl, version, hadoopVersion, packageName, pa
     if (success) {
       return()
     } else {
-      message(paste0("Unable to download from mirrorUrl: ", mirrorUrl))
+      message("Unable to download from mirrorUrl: ", mirrorUrl)
     }
   } else {
     message("MirrorUrl not provided.")
@@ -201,11 +197,9 @@ robustDownloadTar <- function(mirrorUrl, version, hadoopVersion, packageName, pa
     # remove any partially downloaded file
     unlink(packageLocalPath)
     message("Unable to download from default mirror site: ", mirrorUrl)
-    msg <- sprintf(paste("Unable to download Spark %s for Hadoop %s.",
-                         "Please check network connection, Hadoop version,",
-                         "or provide other mirror sites."),
-                   version, ifelse(hadoopVersion == "without", "Free build", hadoopVersion))
-    stop(msg)
+    stop("Unable to download Spark ", version,
+         " for Hadoop ", if (hadoopVersion == "without") "Free build" else hadoopVersion,
+         ". Please check network connection, Hadoop version, or provide other mirror sites.")
   }
 }
 
@@ -222,7 +216,7 @@ getPreferredMirror <- function(version, packageName) {
     endPos <- matchInfo + attr(matchInfo, "match.length") - 2
     mirrorPreferred <- base::substr(linePreferred, startPos, endPos)
     mirrorPreferred <- paste0(mirrorPreferred, "spark")
-    message(sprintf("Preferred mirror site found: %s", mirrorPreferred))
+    message("Preferred mirror site found: ", mirrorPreferred)
   } else {
     mirrorPreferred <- NULL
   }
@@ -231,24 +225,20 @@ getPreferredMirror <- function(version, packageName) {
 
 directDownloadTar <- function(mirrorUrl, version, hadoopVersion, packageName, packageLocalPath) {
   packageRemotePath <- paste0(file.path(mirrorUrl, version, packageName), ".tgz")
-  fmt <- "Downloading %s for Hadoop %s from:\n- %s"
-  msg <- sprintf(fmt, version, ifelse(hadoopVersion == "without", "Free build", hadoopVersion),
-                 packageRemotePath)
-  message(msg)
+  message("Downloading ", version, " for Hadoop ",
+          if (hadoopVersion == "without") "Free build" else hadoopVersion,
+          " from:\n- ", packageRemotePath)
   downloadUrl(packageRemotePath, packageLocalPath)
 }
 
 downloadUrl <- function(remotePath, localPath) {
   isFail <- tryCatch(download.file(remotePath, localPath),
                      error = function(e) {
-                       message(e)
-                       message()
+                       message(e, "\n")
                        TRUE
                      },
                      warning = function(w) {
-                       # Treat warning as error, add an empty line with message()
-                       message(w)
-                       message()
+                       message(w, "\n")
                        TRUE
                      })
   !isFail
@@ -279,9 +269,9 @@ sparkCachePath <- function() {
       winAppPath <- Sys.getenv("USERPROFILE", unset = NA)
     }
     if (is.na(winAppPath)) {
-      stop(paste("%LOCALAPPDATA% and %USERPROFILE% not found.",
-                   "Please define the environment variable",
-                   "or restart and enter an installation path in localDir."))
+      stop("%LOCALAPPDATA% and %USERPROFILE% not found. ",
+           "Please define the environment variable ",
+           "or restart and enter an installation path in localDir.")
     } else {
       path <- file.path(winAppPath, "Apache", "Spark", "Cache")
     }
@@ -293,7 +283,7 @@ sparkCachePath <- function() {
         Sys.getenv("XDG_CACHE_HOME", file.path(Sys.getenv("HOME"), ".cache")), "spark")
     }
   } else {
-    stop(sprintf("Unknown OS: %s", .Platform$OS.type))
+    stop("Unknown OS: ", .Platform$OS.type)
   }
   normalizePath(path, mustWork = FALSE)
 }
@@ -322,7 +312,7 @@ installInstruction <- function(mode) {
            "If you need further help, ",
            "contact the administrators of the cluster.")
   } else {
-    stop(paste0("No instruction found for ", mode, " mode."))
+    stop("No instruction found for mode ", mode)
   }
 }
 
diff --git a/R/pkg/R/mllib_classification.R b/R/pkg/R/mllib_classification.R
index 1af81f8f84b76..53c9775963d23 100644
--- a/R/pkg/R/mllib_classification.R
+++ b/R/pkg/R/mllib_classification.R
@@ -331,8 +331,8 @@ setMethod("spark.logit", signature(data = "SparkDataFrame", formula = "formula")
 
               if (!is.null(lowerBoundsOnCoefficients) && (row != nrow(upperBoundsOnCoefficients)
                 || col != ncol(upperBoundsOnCoefficients))) {
-                stop(paste0("dimension of upperBoundsOnCoefficients ",
-                            "is not the same as lowerBoundsOnCoefficients"))
+                stop("dimension of upperBoundsOnCoefficients ",
+                     "is not the same as lowerBoundsOnCoefficients")
               }
 
               if (is.null(lowerBoundsOnCoefficients)) {
diff --git a/R/pkg/R/mllib_stat.R b/R/pkg/R/mllib_stat.R
index f8c3329359961..6db4d5d4831dd 100644
--- a/R/pkg/R/mllib_stat.R
+++ b/R/pkg/R/mllib_stat.R
@@ -69,8 +69,7 @@ setMethod("spark.kstest", signature(data = "SparkDataFrame"),
           function(data, testCol = "test", nullHypothesis = c("norm"), distParams = c(0, 1)) {
             tryCatch(match.arg(nullHypothesis),
                      error = function(e) {
-                       msg <- paste("Distribution", nullHypothesis, "is not supported.")
-                       stop(msg)
+                       stop("Distribution ", nullHypothesis, " is not supported.")
                      })
             if (nullHypothesis == "norm") {
               distParams <- as.numeric(distParams)
diff --git a/R/pkg/R/pairRDD.R b/R/pkg/R/pairRDD.R
index 9c2e57d3067db..b29381bb900fb 100644
--- a/R/pkg/R/pairRDD.R
+++ b/R/pkg/R/pairRDD.R
@@ -906,7 +906,7 @@ setMethod("sampleByKey",
 
             for (elem in fractions) {
               if (elem < 0.0) {
-                stop(paste("Negative fraction value ", fractions[which(fractions == elem)]))
+                stop("Negative fraction value ", fractions[which(fractions == elem)])
               }
             }
 
diff --git a/R/pkg/R/schema.R b/R/pkg/R/schema.R
index 89d5c2cd1a5e2..7044ede0cc58b 100644
--- a/R/pkg/R/schema.R
+++ b/R/pkg/R/schema.R
@@ -199,7 +199,7 @@ checkType <- function(type) {
             })
   }
 
-  stop(paste("Unsupported type for SparkDataframe:", type))
+  stop("Unsupported type for SparkDataframe: ", type)
 }
 
 #' @param type The data type of the field
diff --git a/R/pkg/R/serialize.R b/R/pkg/R/serialize.R
index cb3c1c59d12ed..7760d9be16f0b 100644
--- a/R/pkg/R/serialize.R
+++ b/R/pkg/R/serialize.R
@@ -84,7 +84,7 @@ writeObject <- function(con, object, writeType = TRUE) {
          Date = writeDate(con, object),
          POSIXlt = writeTime(con, object),
          POSIXct = writeTime(con, object),
-         stop(paste("Unsupported type for serialization", type)))
+         stop("Unsupported type for serialization ", type))
 }
 
 writeVoid <- function(con) {
@@ -158,7 +158,7 @@ writeType <- function(con, class) {
                  Date = "D",
                  POSIXlt = "t",
                  POSIXct = "t",
-                 stop(paste("Unsupported type for serialization", class)))
+                 stop("Unsupported type for serialization ", class))
   writeBin(charToRaw(type), con)
 }
 
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index 9ba36ad46740a..e4a11a5f78a71 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -154,8 +154,8 @@ sparkR.sparkContext <- function(
   connectionTimeout <- as.numeric(Sys.getenv("SPARKR_BACKEND_CONNECTION_TIMEOUT", "6000"))
   if (existingPort != "") {
     if (length(packages) != 0) {
-      warning(paste("sparkPackages has no effect when using spark-submit or sparkR shell",
-                    " please use the --packages commandline instead", sep = ","))
+      warning("sparkPackages has no effect when using spark-submit or sparkR shell, ",
+              "please use the --packages commandline instead")
     }
     backendPort <- existingPort
     authSecret <- Sys.getenv("SPARKR_BACKEND_AUTH_SECRET")
@@ -439,8 +439,9 @@ sparkR.session <- function(
   rPackageVersion <- paste0(packageVersion("SparkR"))
 
   if (jvmVersionStrip != rPackageVersion) {
-    warning(paste("Version mismatch between Spark JVM and SparkR package. JVM version was",
-                  jvmVersion, ", while R package version was", rPackageVersion))
+    warning("Version mismatch between Spark JVM and SparkR package. ",
+            "JVM version was ", jvmVersion,
+            ", while R package version was ", rPackageVersion)
   }
 
   sparkSession
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index c60e4db1496d0..65db9c21d9dbb 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -46,9 +46,9 @@ convertJListToRList <- function(jList, flatten, logicalUpperBound = NULL,
                 res <- list(unserialize(keyBytes),
                   unserialize(valBytes))
               } else {
-                stop(paste("utils.R: convertJListToRList only supports",
-                  "RDD[Array[Byte]] and",
-                  "JavaPairRDD[Array[Byte], Array[Byte]] for now"))
+                stop("utils.R: convertJListToRList only supports ",
+                  "RDD[Array[Byte]] and ",
+                  "JavaPairRDD[Array[Byte], Array[Byte]] for now")
               }
             } else {
               if (inherits(obj, "raw")) {
@@ -354,8 +354,8 @@ varargsToStrEnv <- function(...) {
       } else {
         value <- pairs[[name]]
         if (!(is.logical(value) || is.numeric(value) || is.character(value) || is.null(value))) {
-          stop(paste0("Unsupported type for ", name, " : ", class(value),
-               ". Supported types are logical, numeric, character and NULL."), call. = FALSE)
+          stop("Unsupported type for ", name, " : ", toString(class(value)), ". ",
+               "Supported types are logical, numeric, character and NULL.", call. = FALSE)
         }
         if (is.logical(value)) {
           env[[name]] <- tolower(as.character(value))
@@ -369,8 +369,7 @@ varargsToStrEnv <- function(...) {
   }
 
   if (length(ignoredNames) != 0) {
-    warning(paste0("Unnamed arguments ignored: ", paste(ignoredNames, collapse = ", "), "."),
-            call. = FALSE)
+    warning("Unnamed arguments ignored: ", toString(ignoredNames), ".", call. = FALSE)
   }
   env
 }
@@ -449,7 +448,7 @@ storageLevelToString <- function(levelObj) {
 # the user to type (for example) `5` instead of `5L` to avoid a confusing error message.
 numToInt <- function(num) {
   if (as.integer(num) != num) {
-    warning(paste("Coercing", as.list(sys.call())[[2]], "to integer."))
+    warning("Coercing ", as.list(sys.call())[[2L]], " to integer.")
   }
   as.integer(num)
 }
@@ -650,8 +649,8 @@ mergePartitions <- function(rdd, zip) {
         # For zip operation, check if corresponding partitions
         # of both RDDs have the same number of elements.
         if (zip && lengthOfKeys != lengthOfValues) {
-          stop(paste("Can only zip RDDs with same number of elements",
-                     "in each pair of corresponding partitions."))
+          stop("Can only zip RDDs with same number of elements ",
+               "in each pair of corresponding partitions.")
         }
 
         if (lengthOfKeys > 1) {
@@ -804,7 +803,7 @@ handledCallJMethod <- function(obj, method, ...) {
 
 captureJVMException <- function(e, method) {
   rawmsg <- as.character(e)
-  if (any(grep("^Error in .*?: ", rawmsg))) {
+  if (any(grepl("^Error in .*?: ", rawmsg))) {
     # If the exception message starts with "Error in ...", this is possibly
     # "Error in invokeJava(...)". Here, it replaces the characters to
     # `paste("Error in", method, ":")` in order to identify which function
@@ -818,58 +817,58 @@ captureJVMException <- function(e, method) {
   }
 
   # StreamingQueryException could wrap an IllegalArgumentException, so look for that first
-  if (any(grep("org.apache.spark.sql.streaming.StreamingQueryException: ",
-               stacktrace, fixed = TRUE))) {
+  if (any(grepl("org.apache.spark.sql.streaming.StreamingQueryException: ",
+                stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.streaming.StreamingQueryException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "streaming query error - ", first), call. = FALSE)
-  } else if (any(grep("java.lang.IllegalArgumentException: ", stacktrace, fixed = TRUE))) {
+    stop(rmsg, "streaming query error - ", first, call. = FALSE)
+  } else if (any(grepl("java.lang.IllegalArgumentException: ", stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "java.lang.IllegalArgumentException: ", fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "illegal argument - ", first), call. = FALSE)
-  } else if (any(grep("org.apache.spark.sql.AnalysisException: ", stacktrace, fixed = TRUE))) {
+    stop(rmsg, "illegal argument - ", first, call. = FALSE)
+  } else if (any(grepl("org.apache.spark.sql.AnalysisException: ", stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.AnalysisException: ", fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "analysis error - ", first), call. = FALSE)
+    stop(rmsg, "analysis error - ", first, call. = FALSE)
   } else
-    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
-                 stacktrace, fixed = TRUE))) {
+    if (any(grepl("org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
+                  stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "no such database - ", first), call. = FALSE)
+    stop(rmsg, "no such database - ", first, call. = FALSE)
   } else
-    if (any(grep("org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
-                 stacktrace, fixed = TRUE))) {
+    if (any(grepl("org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
+                  stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.analysis.NoSuchTableException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "no such table - ", first), call. = FALSE)
-  } else if (any(grep("org.apache.spark.sql.catalyst.parser.ParseException: ",
-                      stacktrace, fixed = TRUE))) {
+    stop(rmsg, "no such table - ", first, call. = FALSE)
+  } else if (any(grepl("org.apache.spark.sql.catalyst.parser.ParseException: ",
+                       stacktrace, fixed = TRUE))) {
     msg <- strsplit(stacktrace, "org.apache.spark.sql.catalyst.parser.ParseException: ",
                     fixed = TRUE)[[1]]
     # Extract "Error in ..." message.
     rmsg <- msg[1]
     # Extract the first message of JVM exception.
     first <- strsplit(msg[2], "\r?\n\tat")[[1]][1]
-    stop(paste0(rmsg, "parse error - ", first), call. = FALSE)
+    stop(rmsg, "parse error - ", first, call. = FALSE)
   } else {
     stop(stacktrace, call. = FALSE)
   }
diff --git a/R/pkg/tests/fulltests/test_sparkSQL.R b/R/pkg/tests/fulltests/test_sparkSQL.R
index 0b5527130e2da..fb0ed7451cdb9 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL.R
@@ -2472,7 +2472,7 @@ test_that("join(), crossJoin() and merge() on a DataFrame", {
   error_msg <- paste("joinType must be one of the following types:",
                  "'inner', 'cross', 'outer', 'full', 'fullouter', 'full_outer',",
                  "'left', 'leftouter', 'left_outer', 'right', 'rightouter', 'right_outer',",
-                 "'semi', 'leftsemi', 'left_semi', 'anti', 'leftanti' or 'left_anti'.")
+                 "'semi', 'leftsemi', 'left_semi', 'anti', 'leftanti', 'left_anti'")
   expect_error(join(df2, df, df2$name == df$name, "invalid"), error_msg)
 
   merged <- merge(df, df2, by.x = "name", by.y = "name", all.x = TRUE, all.y = TRUE)

From d4008804f987fa3d3405335e2469886a0d61dd67 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 4 May 2020 09:39:50 +0900
Subject: [PATCH 0620/1280] [SPARK-31527][SQL][TESTS][FOLLOWUP] Fix the number
 of rows in `DateTimeBenchmark`

### What changes were proposed in this pull request?
- Changed to the number of rows in benchmark cases from 3 to the actual number `N`.
- Regenerated benchmark results in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

### Why are the changes needed?
The changes are needed to have:
- Correct benchmark results
- Base line for other perf improvements that can be checked in the same environment.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the benchmark and checking its output.

Closes #28440 from MaxGekk/SPARK-31527-DateTimeBenchmark-followup.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 2fb85f6b684843f337b6e73ba57ee9e57a53496d)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../DateTimeBenchmark-jdk11-results.txt       | 474 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 474 +++++++++---------
 .../benchmark/DateTimeBenchmark.scala         |   2 +-
 3 files changed, 475 insertions(+), 475 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 1004bcf1aa286..61b4c762a752e 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -2,456 +2,456 @@
 datetime +/- interval
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                  919            933          22          0.0   306237514.3       1.0X
-date + interval(m, d)                               910            916           9          0.0   303338619.0       1.0X
-date + interval(m, d, ms)                          3912           3923          16          0.0  1303942791.7       0.2X
-date - interval(m)                                  883            887           6          0.0   294268789.3       1.0X
-date - interval(m, d)                               898            911          18          0.0   299453403.0       1.0X
-date - interval(m, d, ms)                          3937           3944          11          0.0  1312269472.0       0.2X
-timestamp + interval(m)                            2226           2236          14          0.0   741972014.3       0.4X
-timestamp + interval(m, d)                         2264           2274          13          0.0   754709121.0       0.4X
-timestamp + interval(m, d, ms)                     2202           2223          30          0.0   734001075.0       0.4X
-timestamp - interval(m)                            1992           2005          17          0.0   664152744.7       0.5X
-timestamp - interval(m, d)                         2069           2075           9          0.0   689631159.0       0.4X
-timestamp - interval(m, d, ms)                     2240           2244           6          0.0   746538728.0       0.4X
+date + interval(m)                                 1485           1567         116          6.7         148.5       1.0X
+date + interval(m, d)                              1504           1510           9          6.6         150.4       1.0X
+date + interval(m, d, ms)                          7000           7013          18          1.4         700.0       0.2X
+date - interval(m)                                 1466           1478          17          6.8         146.6       1.0X
+date - interval(m, d)                              1533           1534           1          6.5         153.3       1.0X
+date - interval(m, d, ms)                          7014           7019           7          1.4         701.4       0.2X
+timestamp + interval(m)                            3062           3064           3          3.3         306.2       0.5X
+timestamp + interval(m, d)                         3133           3136           5          3.2         313.3       0.5X
+timestamp + interval(m, d, ms)                     3401           3402           3          2.9         340.1       0.4X
+timestamp - interval(m)                            3025           3037          17          3.3         302.5       0.5X
+timestamp - interval(m, d)                         3083           3120          51          3.2         308.3       0.5X
+timestamp - interval(m, d, ms)                     3371           3379          11          3.0         337.1       0.4X
 
 
 ================================================================================================
 Extract components
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    178            180           3         56.1          17.8       1.0X
-cast to timestamp wholestage on                     189            192           4         53.0          18.9       0.9X
+cast to timestamp wholestage off                    339            339           1         29.5          33.9       1.0X
+cast to timestamp wholestage on                     330            337           9         30.3          33.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                    760            761           1         13.2          76.0       1.0X
-year of timestamp wholestage on                     731            741          10         13.7          73.1       1.0X
+year of timestamp wholestage off                   1226           1237          15          8.2         122.6       1.0X
+year of timestamp wholestage on                    1230           1242           9          8.1         123.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1005           1013          10          9.9         100.5       1.0X
-quarter of timestamp wholestage on                  981            986           3         10.2          98.1       1.0X
+quarter of timestamp wholestage off                1602           1606           7          6.2         160.2       1.0X
+quarter of timestamp wholestage on                 1511           1514           3          6.6         151.1       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                   754            758           6         13.3          75.4       1.0X
-month of timestamp wholestage on                    719            729          11         13.9          71.9       1.0X
+month of timestamp wholestage off                  1227           1233           8          8.1         122.7       1.0X
+month of timestamp wholestage on                   1226           1242          28          8.2         122.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1085           1088           4          9.2         108.5       1.0X
-weekofyear of timestamp wholestage on              1075           1091          13          9.3         107.5       1.0X
+weekofyear of timestamp wholestage off             1965           1980          20          5.1         196.5       1.0X
+weekofyear of timestamp wholestage on              1816           1833          17          5.5         181.6       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                     751            770          27         13.3          75.1       1.0X
-day of timestamp wholestage on                      735            741           7         13.6          73.5       1.0X
+day of timestamp wholestage off                    1229           1231           3          8.1         122.9       1.0X
+day of timestamp wholestage on                     1222           1230          10          8.2         122.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off               765            769           5         13.1          76.5       1.0X
-dayofyear of timestamp wholestage on                762            770           7         13.1          76.2       1.0X
+dayofyear of timestamp wholestage off              1294           1297           4          7.7         129.4       1.0X
+dayofyear of timestamp wholestage on               1257           1264           6          8.0         125.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off              780            782           2         12.8          78.0       1.0X
-dayofmonth of timestamp wholestage on               720            736          12         13.9          72.0       1.1X
+dayofmonth of timestamp wholestage off             1247           1253           8          8.0         124.7       1.0X
+dayofmonth of timestamp wholestage on              1225           1229           4          8.2         122.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off               887            899          17         11.3          88.7       1.0X
-dayofweek of timestamp wholestage on                820            847          20         12.2          82.0       1.1X
+dayofweek of timestamp wholestage off              1416           1416           0          7.1         141.6       1.0X
+dayofweek of timestamp wholestage on               1376           1382           8          7.3         137.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                 821            825           5         12.2          82.1       1.0X
-weekday of timestamp wholestage on                  802            814           9         12.5          80.2       1.0X
+weekday of timestamp wholestage off                1350           1351           1          7.4         135.0       1.0X
+weekday of timestamp wholestage on                 1308           1318          13          7.6         130.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    611            622          14         16.4          61.1       1.0X
-hour of timestamp wholestage on                     571            577           8         17.5          57.1       1.1X
+hour of timestamp wholestage off                   1004           1007           3         10.0         100.4       1.0X
+hour of timestamp wholestage on                     928            938           7         10.8          92.8       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  607            615          12         16.5          60.7       1.0X
-minute of timestamp wholestage on                   573            580           6         17.5          57.3       1.1X
+minute of timestamp wholestage off                 1009           1020          15          9.9         100.9       1.0X
+minute of timestamp wholestage on                   933            935           2         10.7          93.3       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  615            616           2         16.3          61.5       1.0X
-second of timestamp wholestage on                   564            575           8         17.7          56.4       1.1X
+second of timestamp wholestage off                  995            995           0         10.0          99.5       1.0X
+second of timestamp wholestage on                   932            937           8         10.7          93.2       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         166            169           4         60.4          16.6       1.0X
-current_date wholestage on                          150            153           3         66.7          15.0       1.1X
+current_date wholestage off                         292            316          34         34.2          29.2       1.0X
+current_date wholestage on                          270            276           6         37.0          27.0       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    179            181           2         55.8          17.9       1.0X
-current_timestamp wholestage on                     162            324         138         61.9          16.2       1.1X
+current_timestamp wholestage off                    313            328          20         31.9          31.3       1.0X
+current_timestamp wholestage on                     270            331          95         37.0          27.0       1.2X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         658            661           5         15.2          65.8       1.0X
-cast to date wholestage on                          644            654          10         15.5          64.4       1.0X
+cast to date wholestage off                        1078           1081           3          9.3         107.8       1.0X
+cast to date wholestage on                         1035           1040           7          9.7         103.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                             768            772           5         13.0          76.8       1.0X
-last_day wholestage on                              737            750          12         13.6          73.7       1.0X
+last_day wholestage off                            1265           1266           3          7.9         126.5       1.0X
+last_day wholestage on                             1236           1246          10          8.1         123.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             691            704          17         14.5          69.1       1.0X
-next_day wholestage on                              664            676          10         15.1          66.4       1.0X
+next_day wholestage off                            1118           1118           1          8.9         111.8       1.0X
+next_day wholestage on                             1085           1090           8          9.2         108.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             646            646           0         15.5          64.6       1.0X
-date_add wholestage on                              623            640          13         16.1          62.3       1.0X
+date_add wholestage off                            1052           1054           4          9.5         105.2       1.0X
+date_add wholestage on                             1046           1051           6          9.6         104.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             638            645           9         15.7          63.8       1.0X
-date_sub wholestage on                              618            629           8         16.2          61.8       1.0X
+date_sub wholestage off                            1075           1075           0          9.3         107.5       1.0X
+date_sub wholestage on                             1043           1046           3          9.6         104.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                           892            896           5         11.2          89.2       1.0X
-add_months wholestage on                            926            938           7         10.8          92.6       1.0X
+add_months wholestage off                          1409           1409           0          7.1         140.9       1.0X
+add_months wholestage on                           1448           1453           4          6.9         144.8       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         3395           3439          62          2.9         339.5       1.0X
-format date wholestage on                          3418           3438          14          2.9         341.8       1.0X
+format date wholestage off                         5373           5390          24          1.9         537.3       1.0X
+format date wholestage on                          5337           5346          12          1.9         533.7       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       4565           4592          38          2.2         456.5       1.0X
-from_unixtime wholestage on                        4608           4635          32          2.2         460.8       1.0X
+from_unixtime wholestage off                       7302           7308           9          1.4         730.2       1.0X
+from_unixtime wholestage on                        7298           7319          16          1.4         729.8       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   801            807           9         12.5          80.1       1.0X
-from_utc_timestamp wholestage on                    819            830           7         12.2          81.9       1.0X
+from_utc_timestamp wholestage off                  1322           1355          48          7.6         132.2       1.0X
+from_utc_timestamp wholestage on                   1290           1294           5          7.8         129.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1108           1114           8          9.0         110.8       1.0X
-to_utc_timestamp wholestage on                     1067           1078          13          9.4         106.7       1.0X
+to_utc_timestamp wholestage off                    1692           1705          18          5.9         169.2       1.0X
+to_utc_timestamp wholestage on                     1653           1657           4          6.1         165.3       1.0X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        211            213           2         47.4          21.1       1.0X
-cast interval wholestage on                         185            188           3         54.1          18.5       1.1X
+cast interval wholestage off                        340            356          22         29.4          34.0       1.0X
+cast interval wholestage on                         293            296           2         34.1          29.3       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1120           1120           1          8.9         112.0       1.0X
-datediff wholestage on                             1174           1205          19          8.5         117.4       1.0X
+datediff wholestage off                            1843           1862          28          5.4         184.3       1.0X
+datediff wholestage on                             1766           1780          16          5.7         176.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      3669           3688          26          2.7         366.9       1.0X
-months_between wholestage on                       3687           3819         181          2.7         368.7       1.0X
+months_between wholestage off                      5856           5858           2          1.7         585.6       1.0X
+months_between wholestage on                       5799           5815          14          1.7         579.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1147           1148           1          0.9        1146.6       1.0X
-window wholestage on                              16997          17207         226          0.1       16996.7       0.1X
+window wholestage off                              2017           2147         183          0.5        2017.4       1.0X
+window wholestage on                              47789          47910          91          0.0       47788.6       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1824           1859          50          5.5         182.4       1.0X
-date_trunc YEAR wholestage on                      1844           1942          71          5.4         184.4       1.0X
+date_trunc YEAR wholestage off                     2689           2689           1          3.7         268.9       1.0X
+date_trunc YEAR wholestage on                      2655           2670          17          3.8         265.5       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1808           1815          11          5.5         180.8       1.0X
-date_trunc YYYY wholestage on                      1833           1864          49          5.5         183.3       1.0X
+date_trunc YYYY wholestage off                     2698           2700           3          3.7         269.8       1.0X
+date_trunc YYYY wholestage on                      2654           2660           6          3.8         265.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1867           1883          23          5.4         186.7       1.0X
-date_trunc YY wholestage on                        1843           1861          15          5.4         184.3       1.0X
+date_trunc YY wholestage off                       2692           2697           7          3.7         269.2       1.0X
+date_trunc YY wholestage on                        2653           2662           7          3.8         265.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1845           1858          18          5.4         184.5       1.0X
-date_trunc MON wholestage on                       1830           1893          42          5.5         183.0       1.0X
+date_trunc MON wholestage off                      2752           2756           6          3.6         275.2       1.0X
+date_trunc MON wholestage on                       2666           2675          15          3.8         266.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1822           1855          47          5.5         182.2       1.0X
-date_trunc MONTH wholestage on                     1832           1863          20          5.5         183.2       1.0X
+date_trunc MONTH wholestage off                    2743           2746           4          3.6         274.3       1.0X
+date_trunc MONTH wholestage on                     2667           2673           8          3.7         266.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1843           1848           7          5.4         184.3       1.0X
-date_trunc MM wholestage on                        1886           1905          14          5.3         188.6       1.0X
+date_trunc MM wholestage off                       2741           2741           1          3.6         274.1       1.0X
+date_trunc MM wholestage on                        2670           2678           7          3.7         267.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1542           1545           4          6.5         154.2       1.0X
-date_trunc DAY wholestage on                       1610           1616           5          6.2         161.0       1.0X
+date_trunc DAY wholestage off                      2338           2342           6          4.3         233.8       1.0X
+date_trunc DAY wholestage on                       2269           2277           7          4.4         226.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1521           1529          11          6.6         152.1       1.0X
-date_trunc DD wholestage on                        1595           1611          21          6.3         159.5       1.0X
+date_trunc DD wholestage off                       2324           2325           1          4.3         232.4       1.0X
+date_trunc DD wholestage on                        2270           2273           2          4.4         227.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1496           1543          67          6.7         149.6       1.0X
-date_trunc HOUR wholestage on                      1567           1594          18          6.4         156.7       1.0X
+date_trunc HOUR wholestage off                     2325           2326           1          4.3         232.5       1.0X
+date_trunc HOUR wholestage on                      2284           2295           8          4.4         228.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    230            230           1         43.5          23.0       1.0X
-date_trunc MINUTE wholestage on                     288            295           7         34.7          28.8       0.8X
+date_trunc MINUTE wholestage off                    407            408           0         24.5          40.7       1.0X
+date_trunc MINUTE wholestage on                     382            386           3         26.1          38.2       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    247            249           4         40.5          24.7       1.0X
-date_trunc SECOND wholestage on                     297            314          12         33.6          29.7       0.8X
+date_trunc SECOND wholestage off                    404            404           1         24.8          40.4       1.0X
+date_trunc SECOND wholestage on                     386            390           4         25.9          38.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1786           1788           3          5.6         178.6       1.0X
-date_trunc WEEK wholestage on                      1786           1832          46          5.6         178.6       1.0X
+date_trunc WEEK wholestage off                     2693           2694           2          3.7         269.3       1.0X
+date_trunc WEEK wholestage on                      2619           2629          10          3.8         261.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  2319           2365          66          4.3         231.9       1.0X
-date_trunc QUARTER wholestage on                   2424           2551         182          4.1         242.4       1.0X
+date_trunc QUARTER wholestage off                  3454           3466          17          2.9         345.4       1.0X
+date_trunc QUARTER wholestage on                   3384           3404          24          3.0         338.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           180            189          12         55.5          18.0       1.0X
-trunc year wholestage on                            271            277           5         36.9          27.1       0.7X
+trunc year wholestage off                           339            340           1         29.5          33.9       1.0X
+trunc year wholestage on                            337            347           9         29.7          33.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           189            191           4         52.9          18.9       1.0X
-trunc yyyy wholestage on                            276            284           6         36.2          27.6       0.7X
+trunc yyyy wholestage off                           347            348           2         28.8          34.7       1.0X
+trunc yyyy wholestage on                            334            335           2         29.9          33.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             189            190           1         52.9          18.9       1.0X
-trunc yy wholestage on                              279            294          11         35.9          27.9       0.7X
+trunc yy wholestage off                             339            346          11         29.5          33.9       1.0X
+trunc yy wholestage on                              333            338           5         30.0          33.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            185            186           1         54.1          18.5       1.0X
-trunc mon wholestage on                             272            285          13         36.8          27.2       0.7X
+trunc mon wholestage off                            339            347          11         29.5          33.9       1.0X
+trunc mon wholestage on                             331            336           4         30.2          33.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          190            190           1         52.6          19.0       1.0X
-trunc month wholestage on                           293            300           4         34.1          29.3       0.6X
+trunc month wholestage off                          341            344           3         29.3          34.1       1.0X
+trunc month wholestage on                           332            338           9         30.1          33.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             178            182           6         56.3          17.8       1.0X
-trunc mm wholestage on                              306            312           5         32.7          30.6       0.6X
+trunc mm wholestage off                             337            338           1         29.6          33.7       1.0X
+trunc mm wholestage on                              332            336           5         30.1          33.2       1.0X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     111            117           8          9.0         110.9       1.0X
-to timestamp str wholestage on                      101            109           6          9.9         100.6       1.1X
+to timestamp str wholestage off                     184            187           4          5.4         183.9       1.0X
+to timestamp str wholestage on                      159            162           2          6.3         159.4       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                         735            746          15          1.4         734.9       1.0X
-to_timestamp wholestage on                          708            725          11          1.4         708.2       1.0X
+to_timestamp wholestage off                        1683           1689           8          0.6        1683.3       1.0X
+to_timestamp wholestage on                         1722           1725           4          0.6        1721.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                    718            727          12          1.4         717.9       1.0X
-to_unix_timestamp wholestage on                     739            755          12          1.4         739.1       1.0X
+to_unix_timestamp wholestage off                   1733           1736           4          0.6        1733.1       1.0X
+to_unix_timestamp wholestage on                    1687           1690           4          0.6        1686.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          124            125           1          8.0         124.4       1.0X
-to date str wholestage on                           134            138           3          7.5         133.9       0.9X
+to date str wholestage off                          218            220           4          4.6         217.6       1.0X
+to date str wholestage on                           213            215           2          4.7         212.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             1510           1544          48          0.7        1510.4       1.0X
-to_date wholestage on                              1544           1557          15          0.6        1544.2       1.0X
+to_date wholestage off                             3697           3699           2          0.3        3697.2       1.0X
+to_date wholestage on                              3603           3624          15          0.3        3602.7       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  269            278           9         18.6          53.7       1.0X
-From java.time.LocalDate                            227            234           7         22.0          45.4       1.2X
-Collect java.sql.Date                              1164           1272         141          4.3         232.8       0.2X
-Collect java.time.LocalDate                        1070           1130          59          4.7         214.1       0.3X
-From java.sql.Timestamp                             246            248           2         20.3          49.2       1.1X
-From java.time.Instant                              214            216           2         23.4          42.8       1.3X
-Collect longs                                       814            831          15          6.1         162.7       0.3X
-Collect java.sql.Timestamp                         1016           1096          78          4.9         203.2       0.3X
-Collect java.time.Instant                          1012           1093          86          4.9         202.4       0.3X
+From java.sql.Date                                  432            436           7         11.6          86.4       1.0X
+From java.time.LocalDate                            343            347           6         14.6          68.6       1.3X
+Collect java.sql.Date                              1888           2453         971          2.6         377.6       0.2X
+Collect java.time.LocalDate                        1779           1820          42          2.8         355.7       0.2X
+From java.sql.Timestamp                             375            384           9         13.3          75.0       1.2X
+From java.time.Instant                              317            326           8         15.8          63.5       1.4X
+Collect longs                                      1338           1428         115          3.7         267.6       0.3X
+Collect java.sql.Timestamp                         1716           2014         281          2.9         343.1       0.3X
+Collect java.time.Instant                          1832           1970         122          2.7         366.5       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index dba6c909be637..3ef2f922f95bf 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -2,456 +2,456 @@
 datetime +/- interval
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1003           1012          13          0.0   334353721.7       1.0X
-date + interval(m, d)                              1154           1174          29          0.0   384575202.7       0.9X
-date + interval(m, d, ms)                          4338           4366          40          0.0  1446002701.3       0.2X
-date - interval(m)                                  850            858           8          0.0   283424914.7       1.2X
-date - interval(m, d)                              1017           1031          19          0.0   339034354.7       1.0X
-date - interval(m, d, ms)                          4699           4717          25          0.0  1566218686.3       0.2X
-timestamp + interval(m)                            2044           2046           3          0.0   681382301.0       0.5X
-timestamp + interval(m, d)                         2215           2249          48          0.0   738464286.7       0.5X
-timestamp + interval(m, d, ms)                     2053           2063          13          0.0   684393366.0       0.5X
-timestamp - interval(m)                            1668           1677          12          0.0   556138256.7       0.6X
-timestamp - interval(m, d)                         1865           1882          25          0.0   621574795.3       0.5X
-timestamp - interval(m, d, ms)                     2075           2077           3          0.0   691569937.3       0.5X
+date + interval(m)                                 1613           1622          13          6.2         161.3       1.0X
+date + interval(m, d)                              1729           1752          32          5.8         172.9       0.9X
+date + interval(m, d, ms)                          6421           6424           5          1.6         642.1       0.3X
+date - interval(m)                                 1441           1443           2          6.9         144.1       1.1X
+date - interval(m, d)                              1687           1689           2          5.9         168.7       1.0X
+date - interval(m, d, ms)                          6617           6625          11          1.5         661.7       0.2X
+timestamp + interval(m)                            2713           2733          28          3.7         271.3       0.6X
+timestamp + interval(m, d)                         3027           3032           8          3.3         302.7       0.5X
+timestamp + interval(m, d, ms)                     3501           3509          12          2.9         350.1       0.5X
+timestamp - interval(m)                            2892           2895           4          3.5         289.2       0.6X
+timestamp - interval(m, d)                         3190           3196           9          3.1         319.0       0.5X
+timestamp - interval(m, d, ms)                     3497           3500           5          2.9         349.7       0.5X
 
 
 ================================================================================================
 Extract components
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    192            192           0         52.2          19.2       1.0X
-cast to timestamp wholestage on                     163            166           3         61.3          16.3       1.2X
+cast to timestamp wholestage off                    321            323           2         31.1          32.1       1.0X
+cast to timestamp wholestage on                     294            306          10         34.0          29.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                    743            745           4         13.5          74.3       1.0X
-year of timestamp wholestage on                     708            715           5         14.1          70.8       1.0X
+year of timestamp wholestage off                   1235           1242           9          8.1         123.5       1.0X
+year of timestamp wholestage on                    1208           1216           8          8.3         120.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                 848            857          12         11.8          84.8       1.0X
-quarter of timestamp wholestage on                  803            813           6         12.5          80.3       1.1X
+quarter of timestamp wholestage off                1415           1424          12          7.1         141.5       1.0X
+quarter of timestamp wholestage on                 1338           1341           4          7.5         133.8       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                   740            745           7         13.5          74.0       1.0X
-month of timestamp wholestage on                    703            710           5         14.2          70.3       1.1X
+month of timestamp wholestage off                  1224           1225           1          8.2         122.4       1.0X
+month of timestamp wholestage on                   1193           1202           8          8.4         119.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1162           1182          28          8.6         116.2       1.0X
-weekofyear of timestamp wholestage on              1093           1102           9          9.2         109.3       1.1X
+weekofyear of timestamp wholestage off             1864           1866           3          5.4         186.4       1.0X
+weekofyear of timestamp wholestage on              1827           1840           7          5.5         182.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                     742            748           9         13.5          74.2       1.0X
-day of timestamp wholestage on                      703            713           7         14.2          70.3       1.1X
+day of timestamp wholestage off                    1209           1211           2          8.3         120.9       1.0X
+day of timestamp wholestage on                     1191           1194           6          8.4         119.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off               791            799          11         12.6          79.1       1.0X
-dayofyear of timestamp wholestage on                732            744           9         13.7          73.2       1.1X
+dayofyear of timestamp wholestage off              1270           1271           2          7.9         127.0       1.0X
+dayofyear of timestamp wholestage on               1241           1250          12          8.1         124.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off              738            752          20         13.6          73.8       1.0X
-dayofmonth of timestamp wholestage on               695            712           9         14.4          69.5       1.1X
+dayofmonth of timestamp wholestage off             1236           1250          20          8.1         123.6       1.0X
+dayofmonth of timestamp wholestage on              1193           1195           3          8.4         119.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off               854            856           3         11.7          85.4       1.0X
-dayofweek of timestamp wholestage on                819            839          16         12.2          81.9       1.0X
+dayofweek of timestamp wholestage off              1402           1405           4          7.1         140.2       1.0X
+dayofweek of timestamp wholestage on               1352           1359           7          7.4         135.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                 816            821           7         12.3          81.6       1.0X
-weekday of timestamp wholestage on                  788            800           8         12.7          78.8       1.0X
+weekday of timestamp wholestage off                1346           1347           2          7.4         134.6       1.0X
+weekday of timestamp wholestage on                 1294           1299           7          7.7         129.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    595            595           1         16.8          59.5       1.0X
-hour of timestamp wholestage on                     533            541          10         18.8          53.3       1.1X
+hour of timestamp wholestage off                   1000           1008          11         10.0         100.0       1.0X
+hour of timestamp wholestage on                     936            941           6         10.7          93.6       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  585            588           4         17.1          58.5       1.0X
-minute of timestamp wholestage on                   532            545          11         18.8          53.2       1.1X
+minute of timestamp wholestage off                  969            976          10         10.3          96.9       1.0X
+minute of timestamp wholestage on                   933            936           4         10.7          93.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  579            589          13         17.3          57.9       1.0X
-second of timestamp wholestage on                   529            537           6         18.9          52.9       1.1X
+second of timestamp wholestage off                 1002           1005           3         10.0         100.2       1.0X
+second of timestamp wholestage on                   935            938           2         10.7          93.5       1.1X
 
 
 ================================================================================================
 Current date and time
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         171            174           4         58.4          17.1       1.0X
-current_date wholestage on                          152            155           3         65.6          15.2       1.1X
+current_date wholestage off                         308            316          11         32.5          30.8       1.0X
+current_date wholestage on                          265            275          12         37.8          26.5       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    178            181           4         56.2          17.8       1.0X
-current_timestamp wholestage on                     138            149           7         72.6          13.8       1.3X
+current_timestamp wholestage off                    307            312           7         32.6          30.7       1.0X
+current_timestamp wholestage on                     263            268           5         38.1          26.3       1.2X
 
 
 ================================================================================================
 Date arithmetic
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                         630            640          14         15.9          63.0       1.0X
-cast to date wholestage on                          591            594           5         16.9          59.1       1.1X
+cast to date wholestage off                        1061           1065           5          9.4         106.1       1.0X
+cast to date wholestage on                          985            991          11         10.2          98.5       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                             759            778          26         13.2          75.9       1.0X
-last_day wholestage on                              727            736           9         13.8          72.7       1.0X
+last_day wholestage off                            1261           1262           1          7.9         126.1       1.0X
+last_day wholestage on                             1223           1235          12          8.2         122.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                             649            659          15         15.4          64.9       1.0X
-next_day wholestage on                              628            629           1         15.9          62.8       1.0X
+next_day wholestage off                            1114           1119           7          9.0         111.4       1.0X
+next_day wholestage on                             1034           1038           3          9.7         103.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                             621            622           1         16.1          62.1       1.0X
-date_add wholestage on                              600            606           6         16.7          60.0       1.0X
+date_add wholestage off                            1059           1076          25          9.4         105.9       1.0X
+date_add wholestage on                             1012           1021           9          9.9         101.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                             611            626          21         16.4          61.1       1.0X
-date_sub wholestage on                              588            600           7         17.0          58.8       1.0X
+date_sub wholestage off                            1046           1046           0          9.6         104.6       1.0X
+date_sub wholestage on                             1019           1023           3          9.8         101.9       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                           843            845           2         11.9          84.3       1.0X
-add_months wholestage on                            818            831          11         12.2          81.8       1.0X
+add_months wholestage off                          1392           1393           1          7.2         139.2       1.0X
+add_months wholestage on                           1335           1346          14          7.5         133.5       1.0X
 
 
 ================================================================================================
 Formatting dates
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         3557           3569          18          2.8         355.7       1.0X
-format date wholestage on                          3564           3588          17          2.8         356.4       1.0X
+format date wholestage off                         5959           5994          50          1.7         595.9       1.0X
+format date wholestage on                          5991           6008          28          1.7         599.1       1.0X
 
 
 ================================================================================================
 Formatting timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       4875           4887          17          2.1         487.5       1.0X
-from_unixtime wholestage on                        4845           4870          16          2.1         484.5       1.0X
+from_unixtime wholestage off                       8851           8872          29          1.1         885.1       1.0X
+from_unixtime wholestage on                        8855           8872          10          1.1         885.5       1.0X
 
 
 ================================================================================================
 Convert timestamps
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                   665            671           8         15.0          66.5       1.0X
-from_utc_timestamp wholestage on                    654            672          14         15.3          65.4       1.0X
+from_utc_timestamp wholestage off                  1105           1107           2          9.0         110.5       1.0X
+from_utc_timestamp wholestage on                   1072           1084          11          9.3         107.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                     982            983           1         10.2          98.2       1.0X
-to_utc_timestamp wholestage on                      877            889           9         11.4          87.7       1.1X
+to_utc_timestamp wholestage off                    1531           1534           3          6.5         153.1       1.0X
+to_utc_timestamp wholestage on                     1451           1463          14          6.9         145.1       1.1X
 
 
 ================================================================================================
 Intervals
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        200            206           9         50.1          20.0       1.0X
-cast interval wholestage on                         157            163           5         63.6          15.7       1.3X
+cast interval wholestage off                        360            366           8         27.8          36.0       1.0X
+cast interval wholestage on                         286            292           7         35.0          28.6       1.3X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1065           1068           4          9.4         106.5       1.0X
-datediff wholestage on                             1028           1047          15          9.7         102.8       1.0X
+datediff wholestage off                            1809           1814           8          5.5         180.9       1.0X
+datediff wholestage on                             1742           1751           8          5.7         174.2       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      3102           3111          13          3.2         310.2       1.0X
-months_between wholestage on                       2970           3028          46          3.4         297.0       1.0X
+months_between wholestage off                      5007           5007           1          2.0         500.7       1.0X
+months_between wholestage on                       4957           4980          35          2.0         495.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1142           1154          16          0.9        1142.2       1.0X
-window wholestage on                              14817          15049         257          0.1       14816.5       0.1X
+window wholestage off                              1945           2027         116          0.5        1945.3       1.0X
+window wholestage on                              45637          45648           8          0.0       45637.2       0.0X
 
 
 ================================================================================================
 Truncation
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     1516           1518           2          6.6         151.6       1.0X
-date_trunc YEAR wholestage on                      1458           1468           9          6.9         145.8       1.0X
+date_trunc YEAR wholestage off                     2463           2465           2          4.1         246.3       1.0X
+date_trunc YEAR wholestage on                      2406           2409           3          4.2         240.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     1535           1535           1          6.5         153.5       1.0X
-date_trunc YYYY wholestage on                      1453           1461           7          6.9         145.3       1.1X
+date_trunc YYYY wholestage off                     2462           2463           1          4.1         246.2       1.0X
+date_trunc YYYY wholestage on                      2407           2411           6          4.2         240.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       1561           1567           9          6.4         156.1       1.0X
-date_trunc YY wholestage on                        1452           1467          16          6.9         145.2       1.1X
+date_trunc YY wholestage off                       2462           2466           6          4.1         246.2       1.0X
+date_trunc YY wholestage on                        2401           2406           4          4.2         240.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      1522           1531          13          6.6         152.2       1.0X
-date_trunc MON wholestage on                       1458           1467           7          6.9         145.8       1.0X
+date_trunc MON wholestage off                      2437           2437           0          4.1         243.7       1.0X
+date_trunc MON wholestage on                       2416           2421           6          4.1         241.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    1518           1519           0          6.6         151.8       1.0X
-date_trunc MONTH wholestage on                     1452           1465          16          6.9         145.2       1.0X
+date_trunc MONTH wholestage off                    2430           2437           9          4.1         243.0       1.0X
+date_trunc MONTH wholestage on                     2417           2423           5          4.1         241.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       1531           1532           1          6.5         153.1       1.0X
-date_trunc MM wholestage on                        1453           1463           8          6.9         145.3       1.1X
+date_trunc MM wholestage off                       2429           2431           3          4.1         242.9       1.0X
+date_trunc MM wholestage on                        2417           2421           4          4.1         241.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      1287           1309          31          7.8         128.7       1.0X
-date_trunc DAY wholestage on                       1310           1337          16          7.6         131.0       1.0X
+date_trunc DAY wholestage off                      2074           2075           2          4.8         207.4       1.0X
+date_trunc DAY wholestage on                       2001           2010          16          5.0         200.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       1322           1328           9          7.6         132.2       1.0X
-date_trunc DD wholestage on                        1282           1324          28          7.8         128.2       1.0X
+date_trunc DD wholestage off                       2067           2067           0          4.8         206.7       1.0X
+date_trunc DD wholestage on                        2000           2003           3          5.0         200.0       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     1379           1393          20          7.3         137.9       1.0X
-date_trunc HOUR wholestage on                      1288           1302          11          7.8         128.8       1.1X
+date_trunc HOUR wholestage off                     2074           2084          14          4.8         207.4       1.0X
+date_trunc HOUR wholestage on                      2057           2067          10          4.9         205.7       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    243            245           2         41.2          24.3       1.0X
-date_trunc MINUTE wholestage on                     213            219           8         47.0          21.3       1.1X
+date_trunc MINUTE wholestage off                    362            364           3         27.6          36.2       1.0X
+date_trunc MINUTE wholestage on                     319            333          14         31.3          31.9       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    238            245          11         42.1          23.8       1.0X
-date_trunc SECOND wholestage on                     201            210           9         49.7          20.1       1.2X
+date_trunc SECOND wholestage off                    361            366           7         27.7          36.1       1.0X
+date_trunc SECOND wholestage on                     324            341          23         30.9          32.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     1443           1477          49          6.9         144.3       1.0X
-date_trunc WEEK wholestage on                      1491           1516          17          6.7         149.1       1.0X
+date_trunc WEEK wholestage off                     2385           2393          11          4.2         238.5       1.0X
+date_trunc WEEK wholestage on                      2313           2322           6          4.3         231.3       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  2017           2039          32          5.0         201.7       1.0X
-date_trunc QUARTER wholestage on                   1966           2005          36          5.1         196.6       1.0X
+date_trunc QUARTER wholestage off                  3278           3280           2          3.1         327.8       1.0X
+date_trunc QUARTER wholestage on                   3228           3234           8          3.1         322.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           206            206           1         48.6          20.6       1.0X
-trunc year wholestage on                            175            178           2         57.2          17.5       1.2X
+trunc year wholestage off                           328            331           4         30.5          32.8       1.0X
+trunc year wholestage on                            286            295           9         35.0          28.6       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           188            189           2         53.2          18.8       1.0X
-trunc yyyy wholestage on                            176            180           4         56.9          17.6       1.1X
+trunc yyyy wholestage off                           317            319           3         31.5          31.7       1.0X
+trunc yyyy wholestage on                            283            287           6         35.3          28.3       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             191            191           0         52.4          19.1       1.0X
-trunc yy wholestage on                              175            180           4         57.0          17.5       1.1X
+trunc yy wholestage off                             321            321           0         31.1          32.1       1.0X
+trunc yy wholestage on                              284            293          11         35.2          28.4       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            203            205           3         49.3          20.3       1.0X
-trunc mon wholestage on                             183            186           2         54.8          18.3       1.1X
+trunc mon wholestage off                            318            319           1         31.4          31.8       1.0X
+trunc mon wholestage on                             283            287           4         35.4          28.3       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          199            199           0         50.3          19.9       1.0X
-trunc month wholestage on                           177            179           2         56.4          17.7       1.1X
+trunc month wholestage off                          319            321           3         31.3          31.9       1.0X
+trunc month wholestage on                           286            293           7         35.0          28.6       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             198            198           1         50.5          19.8       1.0X
-trunc mm wholestage on                              180            183           3         55.7          18.0       1.1X
+trunc mm wholestage off                             317            319           2         31.5          31.7       1.0X
+trunc mm wholestage on                              282            285           3         35.4          28.2       1.1X
 
 
 ================================================================================================
 Parsing
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     138            139           2          7.2         138.2       1.0X
-to timestamp str wholestage on                      129            138           7          7.8         128.9       1.1X
+to timestamp str wholestage off                     219            220           0          4.6         219.4       1.0X
+to timestamp str wholestage on                      214            218           6          4.7         214.1       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                         885            889           5          1.1         885.3       1.0X
-to_timestamp wholestage on                          854            866          10          1.2         854.0       1.0X
+to_timestamp wholestage off                        1912           1913           2          0.5        1912.0       1.0X
+to_timestamp wholestage on                         1671           1675           7          0.6        1670.8       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                    848            856          12          1.2         848.1       1.0X
-to_unix_timestamp wholestage on                     826            850          18          1.2         826.4       1.0X
+to_unix_timestamp wholestage off                   1761           1763           3          0.6        1761.1       1.0X
+to_unix_timestamp wholestage on                    1695           1697           2          0.6        1695.4       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          167            171           5          6.0         167.2       1.0X
-to date str wholestage on                           165            173           4          6.1         165.0       1.0X
+to date str wholestage off                          267            272           7          3.7         266.9       1.0X
+to date str wholestage on                           266            267           2          3.8         265.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             1612           1633          31          0.6        1611.7       1.0X
-to_date wholestage on                              1588           1605          19          0.6        1588.2       1.0X
+to_date wholestage off                             3705           3743          55          0.3        3704.6       1.0X
+to_date wholestage on                              3736           3746          11          0.3        3736.4       1.0X
 
 
 ================================================================================================
 Conversion from/to external types
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_251-b08 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  245            247           1         20.4          49.0       1.0X
-From java.time.LocalDate                            228            233           4         21.9          45.6       1.1X
-Collect java.sql.Date                              1239           1361         209          4.0         247.9       0.2X
-Collect java.time.LocalDate                        1049           1107          54          4.8         209.8       0.2X
-From java.sql.Timestamp                             247            252           4         20.2          49.5       1.0X
-From java.time.Instant                              156            158           3         32.1          31.2       1.6X
-Collect longs                                       854            910          59          5.9         170.8       0.3X
-Collect java.sql.Timestamp                         1133           1140          12          4.4         226.6       0.2X
-Collect java.time.Instant                          1108           1159          74          4.5         221.7       0.2X
+From java.sql.Date                                  400            406           6         12.5          80.1       1.0X
+From java.time.LocalDate                            343            349           7         14.6          68.6       1.2X
+Collect java.sql.Date                              1904           2739        1170          2.6         380.9       0.2X
+Collect java.time.LocalDate                        1477           1495          19          3.4         295.3       0.3X
+From java.sql.Timestamp                             376            388          10         13.3          75.2       1.1X
+From java.time.Instant                              237            239           3         21.1          47.4       1.7X
+Collect longs                                      1258           1356         111          4.0         251.7       0.3X
+Collect java.sql.Timestamp                         1878           1937          64          2.7         375.6       0.2X
+Collect java.time.Instant                          1667           1904         238          3.0         333.4       0.2X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index 0034819b58893..f56efa3bba600 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -61,7 +61,7 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
         val N = 10000000
         runBenchmark("datetime +/- interval") {
-          val benchmark = new Benchmark("datetime +/- interval", 3, output = output)
+          val benchmark = new Benchmark("datetime +/- interval", N, output = output)
           val ts = "cast(id as timestamp)"
           val dt = s"cast($ts as date)"
           benchmark.addCase("date + interval(m)") { _ =>

From 7d482150f14bb95ed923a39951e2b474ea604e43 Mon Sep 17 00:00:00 2001
From: Tianshi Zhu <zhutianshirea@gmail.com>
Date: Mon, 4 May 2020 14:50:38 +0900
Subject: [PATCH 0621/1280] [SPARK-31267][SQL] Flaky test:
 WholeStageCodegenSparkSubmitSuite.Generated code on driver should not embed
 platform-specific constant

### What changes were proposed in this pull request?

Allow customized timeouts for `runSparkSubmit`, which will make flaky tests more likely to pass by using a larger timeout value.

I was able to reproduce the test failure on my laptop, which took 1.5 - 2 minutes to finish the test. After increasing the timeout, the test now can pass locally.

### Why are the changes needed?

This allows slow tests to use a larger timeout, so they are more likely to succeed.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

The test was able to pass on my local env after the change.

Closes #28438 from tianshizz/SPARK-31267.

Authored-by: Tianshi Zhu <zhutianshirea@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a222644e1df907d0aba19634a166e146dfb4f551)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala     | 5 +++--
 .../sql/execution/WholeStageCodegenSparkSubmitSuite.scala    | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 9d4736825618e..071508713e405 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path}
 import org.scalatest.{BeforeAndAfterEach, Matchers}
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
+import org.scalatest.time.Span
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark._
@@ -1417,7 +1418,7 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits {
   implicit val defaultSignaler: Signaler = ThreadSignaler
 
   // NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly.
-  def runSparkSubmit(args: Seq[String], root: String = ".."): Unit = {
+  def runSparkSubmit(args: Seq[String], root: String = "..", timeout: Span = 1.minute): Unit = {
     val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
     val sparkSubmitFile = if (Utils.isWindows) {
       new File(s"$root\\bin\\spark-submit.cmd")
@@ -1430,7 +1431,7 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits {
       Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome))
 
     try {
-      val exitCode = failAfter(1.minute) { process.waitFor() }
+      val exitCode = failAfter(timeout) { process.waitFor() }
       if (exitCode != 0) {
         fail(s"Process returned with exit code $exitCode. See the log4j logs for more detail.")
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
index f6814d8ff8a3d..c5a01de911962 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
 
 import org.scalatest.{Assertions, BeforeAndAfterEach, Matchers}
 import org.scalatest.concurrent.TimeLimits
+import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkFunSuite, TestUtils}
 import org.apache.spark.deploy.SparkSubmitSuite
@@ -50,7 +51,7 @@ class WholeStageCodegenSparkSubmitSuite extends SparkFunSuite
       "--conf", "spark.executor.extraJavaOptions=-XX:+UseCompressedOops",
       "--conf", "spark.sql.adaptive.enabled=false",
       unusedJar.toString)
-    SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..")
+    SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..", 3.minutes)
   }
 }
 

From 8584f4de9e37bfd4da2e5cb55aa767873766f0ba Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Mon, 4 May 2020 14:59:33 +0900
Subject: [PATCH 0622/1280] [SPARK-31626][SQL] Port HIVE-10415:
 hive.start.cleanup.scratchdir configuration is not taking effect

### What changes were proposed in this pull request?
This pr port [HIVE-10415](https://issues.apache.org/jira/browse/HIVE-10415): `hive.start.cleanup.scratchdir` configuration is not taking effect.

### Why are the changes needed?

I encountered this issue:
![image](https://user-images.githubusercontent.com/5399861/80869375-aeafd080-8cd2-11ea-8573-93ec4b422be1.png)
I'd like to make `hive.start.cleanup.scratchdir` effective to reduce this issue.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Unit test

Closes #28436 from wangyum/SPARK-31626.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 7ef0b69a92db91d0c09e65eb9dcfb973def71814)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../hive/thriftserver/HiveThriftServer2.scala |  3 ++
 .../HiveThriftServer2Suites.scala             | 33 +++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
index f15193b0dc3cc..f9f2ceeed8a75 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.thriftserver
 import java.util.Locale
 import java.util.concurrent.atomic.AtomicBoolean
 
+import org.apache.hadoop.hive.common.ServerUtils
 import org.apache.hadoop.hive.conf.HiveConf
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
 import org.apache.hive.service.cli.thrift.{ThriftBinaryCLIService, ThriftHttpCLIService}
@@ -101,6 +102,8 @@ object HiveThriftServer2 extends Logging {
       SparkSQLEnv.sqlContext.sessionState.newHadoopConf())
 
     try {
+      // Cleanup the scratch dir before starting
+      ServerUtils.cleanUpScratchDir(executionHive.conf)
       val server = new HiveThriftServer2(SparkSQLEnv.sqlContext)
       server.init(executionHive.conf)
       server.start()
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 639dc4d13f673..0cec63460814c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -924,6 +924,39 @@ class SingleSessionSuite extends HiveThriftJdbcTest {
   }
 }
 
+class HiveThriftCleanUpScratchDirSuite extends HiveThriftJdbcTest{
+  var tempScratchDir: File = _
+
+  override protected def beforeAll(): Unit = {
+    tempScratchDir = Utils.createTempDir()
+    tempScratchDir.setWritable(true, false)
+    assert(tempScratchDir.list().isEmpty)
+    new File(tempScratchDir.getAbsolutePath + File.separator + "SPARK-31626").createNewFile()
+    assert(tempScratchDir.list().nonEmpty)
+    super.beforeAll()
+  }
+
+  override def mode: ServerMode.Value = ServerMode.binary
+
+  override protected def extraConf: Seq[String] =
+    s" --hiveconf ${ConfVars.HIVE_START_CLEANUP_SCRATCHDIR}=true " ::
+       s"--hiveconf ${ConfVars.SCRATCHDIR}=${tempScratchDir.getAbsolutePath}" :: Nil
+
+  test("Cleanup the Hive scratchdir when starting the Hive Server") {
+    assert(!tempScratchDir.exists())
+    withJdbcStatement() { statement =>
+      val rs = statement.executeQuery("SELECT id FROM range(1)")
+      assert(rs.next())
+      assert(rs.getLong(1) === 0L)
+    }
+  }
+
+  override protected def afterAll(): Unit = {
+    Utils.deleteRecursively(tempScratchDir)
+    super.afterAll()
+  }
+}
+
 class HiveThriftHttpServerSuite extends HiveThriftJdbcTest {
   override def mode: ServerMode.Value = ServerMode.http
 

From b3d2eaa58c29dab6f6f33ba20d692e2257c2744b Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 4 May 2020 15:30:10 +0900
Subject: [PATCH 0623/1280] [SPARK-31606][SQL] Reduce the perf regression of
 vectorized parquet reader caused by datetime rebase

### What changes were proposed in this pull request?

Push the rebase logic to the lower level of the parquet vectorized reader, to make the final code more vectorization-friendly.

### Why are the changes needed?

Parquet vectorized reader is carefully implemented, to make it more likely to be vectorized by the JVM. However, the newly added datetime rebase degrade the performance a lot, as it breaks vectorization, even if the datetime values don't need to rebase (this is very likely as dates before 1582 is rare).

### Does this PR introduce any user-facing change?

no

### How was this patch tested?

Run part of the `DateTimeRebaseBenchmark` locally. The results:
before this patch
```
[info] Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
[info] ------------------------------------------------------------------------------------------------------------------------
[info] after 1582, vec on, rebase off                     2677           2838         142         37.4          26.8       1.0X
[info] after 1582, vec on, rebase on                      3828           4331         805         26.1          38.3       0.7X
[info] before 1582, vec on, rebase off                    2903           2926          34         34.4          29.0       0.9X
[info] before 1582, vec on, rebase on                     4163           4197          38         24.0          41.6       0.6X

[info] Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
[info] ------------------------------------------------------------------------------------------------------------------------
[info] after 1900, vec on, rebase off                     3537           3627         104         28.3          35.4       1.0X
[info] after 1900, vec on, rebase on                      6891           7010         105         14.5          68.9       0.5X
[info] before 1900, vec on, rebase off                    3692           3770          72         27.1          36.9       1.0X
[info] before 1900, vec on, rebase on                     7588           7610          30         13.2          75.9       0.5X
```

After this patch
```
[info] Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
[info] ------------------------------------------------------------------------------------------------------------------------
[info] after 1582, vec on, rebase off                     2758           2944         197         36.3          27.6       1.0X
[info] after 1582, vec on, rebase on                      2908           2966          51         34.4          29.1       0.9X
[info] before 1582, vec on, rebase off                    2840           2878          37         35.2          28.4       1.0X
[info] before 1582, vec on, rebase on                     3407           3433          24         29.4          34.1       0.8X

[info] Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
[info] ------------------------------------------------------------------------------------------------------------------------
[info] after 1900, vec on, rebase off                     3861           4003         139         25.9          38.6       1.0X
[info] after 1900, vec on, rebase on                      4194           4283          77         23.8          41.9       0.9X
[info] before 1900, vec on, rebase off                    3849           3937          79         26.0          38.5       1.0X
[info] before 1900, vec on, rebase on                     7512           7546          55         13.3          75.1       0.5X
```

Date type is 30% faster if the values don't need to rebase, 20% faster if need to rebase.
Timestamp type is 60% faster if the values don't need to rebase, no difference if need to rebase.

Closes #28406 from cloud-fan/perf.

Lead-authored-by: Wenchen Fan <wenchen@databricks.com>
Co-authored-by: Maxim Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f72220b8ab256e8e6532205a4ce51d50b69c26e9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/util/RebaseDateTime.scala    |   4 +
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 104 +++++++++---------
 .../DateTimeRebaseBenchmark-results.txt       | 104 +++++++++---------
 .../parquet/VectorizedColumnReader.java       |  22 +---
 .../parquet/VectorizedPlainValuesReader.java  |  55 +++++++++
 .../parquet/VectorizedRleValuesReader.java    |  85 ++++++++++++++
 .../parquet/VectorizedValuesReader.java       |   2 +
 .../benchmark/DateTimeRebaseBenchmark.scala   |  79 ++++++-------
 8 files changed, 295 insertions(+), 160 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 95c20a3c6d9d4..8ad7f719bb751 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -71,6 +71,8 @@ object RebaseDateTime {
     -719164, -682945, -646420, -609895, -536845, -500320, -463795,
     -390745, -354220, -317695, -244645, -208120, -171595, -141427)
 
+  final val lastSwitchJulianDay: Int = julianGregDiffSwitchDay.last
+
   // The first days of Common Era (CE) which is mapped to the '0001-01-01' date in Julian calendar.
   private final val julianCommonEraStartDay = julianGregDiffSwitchDay(0)
 
@@ -416,6 +418,8 @@ object RebaseDateTime {
   // in the interval: [julianGregDiffSwitchMicros(i), julianGregDiffSwitchMicros(i+1))
   private val julianGregRebaseMap = loadRebaseRecords("julian-gregorian-rebase-micros.json")
 
+  final val lastSwitchJulianTs: Long = julianGregRebaseMap.values.map(_.switches.last).max
+
   /**
    * An optimized version of [[rebaseJulianToGregorianMicros(ZoneId, Long)]]. This method leverages
    * the pre-calculated rebasing maps to save calculation. If the rebasing map doesn't contain
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 2a9322a4b462a..03e0d7b8bc575 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -2,93 +2,93 @@
 Rebasing dates/timestamps in Parquet datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  21171          21171           0          4.7         211.7       1.0X
-before 1582, noop                                 11036          11036           0          9.1         110.4       1.9X
-after 1582, rebase off                            34321          34321           0          2.9         343.2       0.6X
-after 1582, rebase on                             33269          33269           0          3.0         332.7       0.6X
-before 1582, rebase off                           22016          22016           0          4.5         220.2       1.0X
-before 1582, rebase on                            23338          23338           0          4.3         233.4       0.9X
+after 1582, noop                                  20073          20073           0          5.0         200.7       1.0X
+before 1582, noop                                 10985          10985           0          9.1         109.9       1.8X
+after 1582, rebase off                            32245          32245           0          3.1         322.4       0.6X
+after 1582, rebase on                             31434          31434           0          3.2         314.3       0.6X
+before 1582, rebase off                           21590          21590           0          4.6         215.9       0.9X
+before 1582, rebase on                            22963          22963           0          4.4         229.6       0.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12791          13089         287          7.8         127.9       1.0X
-after 1582, vec off, rebase on                    13203          13271          81          7.6         132.0       1.0X
-after 1582, vec on, rebase off                     3709           3764          49         27.0          37.1       3.4X
-after 1582, vec on, rebase on                      5082           5114          29         19.7          50.8       2.5X
-before 1582, vec off, rebase off                  13059          13153          87          7.7         130.6       1.0X
-before 1582, vec off, rebase on                   14211          14236          27          7.0         142.1       0.9X
-before 1582, vec on, rebase off                    3687           3749          72         27.1          36.9       3.5X
-before 1582, vec on, rebase on                     5449           5497          56         18.4          54.5       2.3X
+after 1582, vec off, rebase off                   12815          12858          40          7.8         128.1       1.0X
+after 1582, vec off, rebase on                    13030          13167         148          7.7         130.3       1.0X
+after 1582, vec on, rebase off                     3705           3712           6         27.0          37.1       3.5X
+after 1582, vec on, rebase on                      3788           3791           3         26.4          37.9       3.4X
+before 1582, vec off, rebase off                  12873          12943          61          7.8         128.7       1.0X
+before 1582, vec off, rebase on                   14072          14165          80          7.1         140.7       0.9X
+before 1582, vec on, rebase off                    3694           3708          15         27.1          36.9       3.5X
+before 1582, vec on, rebase on                     4403           4484          81         22.7          44.0       2.9X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2831           2831           0         35.3          28.3       1.0X
-before 1582, noop                                  2816           2816           0         35.5          28.2       1.0X
-after 1582, rebase off                            15543          15543           0          6.4         155.4       0.2X
-after 1582, rebase on                             18391          18391           0          5.4         183.9       0.2X
-before 1582, rebase off                           15747          15747           0          6.4         157.5       0.2X
-before 1582, rebase on                            18846          18846           0          5.3         188.5       0.2X
+after 1900, noop                                   3032           3032           0         33.0          30.3       1.0X
+before 1900, noop                                  3043           3043           0         32.9          30.4       1.0X
+after 1900, rebase off                            15634          15634           0          6.4         156.3       0.2X
+after 1900, rebase on                             18233          18233           0          5.5         182.3       0.2X
+before 1900, rebase off                           15820          15820           0          6.3         158.2       0.2X
+before 1900, rebase on                            19921          19921           0          5.0         199.2       0.2X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   16126          16216          78          6.2         161.3       1.0X
-after 1582, vec off, rebase on                    18277          18453         165          5.5         182.8       0.9X
-after 1582, vec on, rebase off                     5030           5067          42         19.9          50.3       3.2X
-after 1582, vec on, rebase on                      8553           8583          43         11.7          85.5       1.9X
-before 1582, vec off, rebase off                  15828          15872          39          6.3         158.3       1.0X
-before 1582, vec off, rebase on                   18899          18959         103          5.3         189.0       0.9X
-before 1582, vec on, rebase off                    4961           5009          43         20.2          49.6       3.3X
-before 1582, vec on, rebase on                     9099           9140          40         11.0          91.0       1.8X
+after 1900, vec off, rebase off                   14987          15008          18          6.7         149.9       1.0X
+after 1900, vec off, rebase on                    17500          17628         210          5.7         175.0       0.9X
+after 1900, vec on, rebase off                     5030           5036           7         19.9          50.3       3.0X
+after 1900, vec on, rebase on                      5066           5109          44         19.7          50.7       3.0X
+before 1900, vec off, rebase off                  15094          15213         121          6.6         150.9       1.0X
+before 1900, vec off, rebase on                   18098          18175         101          5.5         181.0       0.8X
+before 1900, vec on, rebase off                    5008           5012           4         20.0          50.1       3.0X
+before 1900, vec on, rebase on                     8803           8848          55         11.4          88.0       1.7X
 
 
 ================================================================================================
 Rebasing dates/timestamps in ORC datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  21026          21026           0          4.8         210.3       1.0X
-before 1582, noop                                 11040          11040           0          9.1         110.4       1.9X
-after 1582                                        28171          28171           0          3.5         281.7       0.7X
-before 1582                                       18955          18955           0          5.3         189.5       1.1X
+after 1582, noop                                  19593          19593           0          5.1         195.9       1.0X
+before 1582, noop                                 10581          10581           0          9.5         105.8       1.9X
+after 1582                                        27843          27843           0          3.6         278.4       0.7X
+before 1582                                       19435          19435           0          5.1         194.4       1.0X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10876          10931          49          9.2         108.8       1.0X
-after 1582, vec on                                 3900           3913          20         25.6          39.0       2.8X
-before 1582, vec off                              11165          11174          12          9.0         111.6       1.0X
-before 1582, vec on                                4208           4214           7         23.8          42.1       2.6X
+after 1582, vec off                               10395          10507         119          9.6         103.9       1.0X
+after 1582, vec on                                 3921           3945          22         25.5          39.2       2.7X
+before 1582, vec off                              10762          10860         127          9.3         107.6       1.0X
+before 1582, vec on                                4194           4226          41         23.8          41.9       2.5X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2924           2924           0         34.2          29.2       1.0X
-before 1582, noop                                  2820           2820           0         35.5          28.2       1.0X
-after 1582                                        22228          22228           0          4.5         222.3       0.1X
-before 1582                                       22590          22590           0          4.4         225.9       0.1X
+after 1900, noop                                   3003           3003           0         33.3          30.0       1.0X
+before 1900, noop                                  3016           3016           0         33.2          30.2       1.0X
+after 1900                                        21804          21804           0          4.6         218.0       0.1X
+before 1900                                       23920          23920           0          4.2         239.2       0.1X
 
-OpenJDK 64-Bit Server VM 11.0.6+10-post-Ubuntu-1ubuntu118.04.1 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               13591          13658          59          7.4         135.9       1.0X
-after 1582, vec on                                 7399           7488         126         13.5          74.0       1.8X
-before 1582, vec off                              14065          14096          30          7.1         140.7       1.0X
-before 1582, vec on                                7950           8127         249         12.6          79.5       1.7X
+after 1900, vec off                               14112          14128          17          7.1         141.1       1.0X
+after 1900, vec on                                 7347           7459         134         13.6          73.5       1.9X
+before 1900, vec off                              15170          15192          27          6.6         151.7       0.9X
+before 1900, vec on                                8280           8312          52         12.1          82.8       1.7X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index 050950571511d..a32a1ad8af89e 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -2,93 +2,93 @@
 Rebasing dates/timestamps in Parquet datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  24114          24114           0          4.1         241.1       1.0X
-before 1582, noop                                 10250          10250           0          9.8         102.5       2.4X
-after 1582, rebase off                            36672          36672           0          2.7         366.7       0.7X
-after 1582, rebase on                             37123          37123           0          2.7         371.2       0.6X
-before 1582, rebase off                           21925          21925           0          4.6         219.2       1.1X
-before 1582, rebase on                            22341          22341           0          4.5         223.4       1.1X
+after 1582, noop                                  23088          23088           0          4.3         230.9       1.0X
+before 1582, noop                                 10782          10782           0          9.3         107.8       2.1X
+after 1582, rebase off                            34821          34821           0          2.9         348.2       0.7X
+after 1582, rebase on                             35040          35040           0          2.9         350.4       0.7X
+before 1582, rebase off                           22151          22151           0          4.5         221.5       1.0X
+before 1582, rebase on                            24677          24677           0          4.1         246.8       0.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12456          12601         126          8.0         124.6       1.0X
-after 1582, vec off, rebase on                    13299          13336          32          7.5         133.0       0.9X
-after 1582, vec on, rebase off                     3623           3660          40         27.6          36.2       3.4X
-after 1582, vec on, rebase on                      5160           5177          15         19.4          51.6       2.4X
-before 1582, vec off, rebase off                  13177          13264          76          7.6         131.8       0.9X
-before 1582, vec off, rebase on                   14102          14149          46          7.1         141.0       0.9X
-before 1582, vec on, rebase off                    3649           3670          34         27.4          36.5       3.4X
-before 1582, vec on, rebase on                     5652           5667          15         17.7          56.5       2.2X
+after 1582, vec off, rebase off                   13559          13650          79          7.4         135.6       1.0X
+after 1582, vec off, rebase on                    12942          12973          28          7.7         129.4       1.0X
+after 1582, vec on, rebase off                     3657           3689          29         27.3          36.6       3.7X
+after 1582, vec on, rebase on                      3859           3902          53         25.9          38.6       3.5X
+before 1582, vec off, rebase off                  12588          12607          17          7.9         125.9       1.1X
+before 1582, vec off, rebase on                   13396          13420          25          7.5         134.0       1.0X
+before 1582, vec on, rebase off                    3631           3650          19         27.5          36.3       3.7X
+before 1582, vec on, rebase on                     4706           4755          77         21.3          47.1       2.9X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2871           2871           0         34.8          28.7       1.0X
-before 1582, noop                                  2753           2753           0         36.3          27.5       1.0X
-after 1582, rebase off                            15927          15927           0          6.3         159.3       0.2X
-after 1582, rebase on                             19138          19138           0          5.2         191.4       0.1X
-before 1582, rebase off                           16137          16137           0          6.2         161.4       0.2X
-before 1582, rebase on                            19584          19584           0          5.1         195.8       0.1X
+after 1900, noop                                   2681           2681           0         37.3          26.8       1.0X
+before 1900, noop                                  3051           3051           0         32.8          30.5       0.9X
+after 1900, rebase off                            16901          16901           0          5.9         169.0       0.2X
+after 1900, rebase on                             19725          19725           0          5.1         197.3       0.1X
+before 1900, rebase off                           16900          16900           0          5.9         169.0       0.2X
+before 1900, rebase on                            20381          20381           0          4.9         203.8       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   14995          15047          47          6.7         150.0       1.0X
-after 1582, vec off, rebase on                    18111          18146          37          5.5         181.1       0.8X
-after 1582, vec on, rebase off                     4837           4873          44         20.7          48.4       3.1X
-after 1582, vec on, rebase on                      9542           9669         111         10.5          95.4       1.6X
-before 1582, vec off, rebase off                  14993          15090          94          6.7         149.9       1.0X
-before 1582, vec off, rebase on                   18675          18712          64          5.4         186.7       0.8X
-before 1582, vec on, rebase off                    4908           4923          15         20.4          49.1       3.1X
-before 1582, vec on, rebase on                    10128          10148          19          9.9         101.3       1.5X
+after 1900, vec off, rebase off                   15236          15291          62          6.6         152.4       1.0X
+after 1900, vec off, rebase on                    17832          18047         187          5.6         178.3       0.9X
+after 1900, vec on, rebase off                     4875           4901          31         20.5          48.7       3.1X
+after 1900, vec on, rebase on                      5354           5386          37         18.7          53.5       2.8X
+before 1900, vec off, rebase off                  15229          15338         108          6.6         152.3       1.0X
+before 1900, vec off, rebase on                   18626          18668          44          5.4         186.3       0.8X
+before 1900, vec on, rebase off                    4968           4975           6         20.1          49.7       3.1X
+before 1900, vec on, rebase on                     9913           9932          16         10.1          99.1       1.5X
 
 
 ================================================================================================
 Rebasing dates/timestamps in ORC datasource
 ================================================================================================
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23977          23977           0          4.2         239.8       1.0X
-before 1582, noop                                 10094          10094           0          9.9         100.9       2.4X
-after 1582                                        33115          33115           0          3.0         331.2       0.7X
-before 1582                                       19430          19430           0          5.1         194.3       1.2X
+after 1582, noop                                  22942          22942           0          4.4         229.4       1.0X
+before 1582, noop                                 11035          11035           0          9.1         110.4       2.1X
+after 1582                                        31341          31341           0          3.2         313.4       0.7X
+before 1582                                       20376          20376           0          4.9         203.8       1.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10217          10241          21          9.8         102.2       1.0X
-after 1582, vec on                                 3671           3691          31         27.2          36.7       2.8X
-before 1582, vec off                              10800          10874         114          9.3         108.0       0.9X
-before 1582, vec on                                4118           4165          74         24.3          41.2       2.5X
+after 1582, vec off                               10361          10378          29          9.7         103.6       1.0X
+after 1582, vec on                                 3820           3828          11         26.2          38.2       2.7X
+before 1582, vec off                              10709          10720          13          9.3         107.1       1.0X
+before 1582, vec on                                4136           4153          15         24.2          41.4       2.5X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                   2691           2691           0         37.2          26.9       1.0X
-before 1582, noop                                  2743           2743           0         36.5          27.4       1.0X
-after 1582                                        21409          21409           0          4.7         214.1       0.1X
-before 1582                                       22554          22554           0          4.4         225.5       0.1X
+after 1900, noop                                   2888           2888           0         34.6          28.9       1.0X
+before 1900, noop                                  2823           2823           0         35.4          28.2       1.0X
+after 1900                                        19790          19790           0          5.1         197.9       0.1X
+before 1900                                       20774          20774           0          4.8         207.7       0.1X
 
-OpenJDK 64-Bit Server VM 1.8.0_242-8u242-b08-0ubuntu3~18.04-b08 on Linux 4.15.0-1063-aws
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               14752          14855         103          6.8         147.5       1.0X
-after 1582, vec on                                 8146           8185          34         12.3          81.5       1.8X
-before 1582, vec off                              15247          15294          46          6.6         152.5       1.0X
-before 1582, vec on                                8414           8466          52         11.9          84.1       1.8X
+after 1900, vec off                               14649          14687          38          6.8         146.5       1.0X
+after 1900, vec on                                 7850           7937         130         12.7          78.5       1.9X
+before 1900, vec off                              15354          15417         108          6.5         153.5       1.0X
+before 1900, vec on                                8382           8408          22         11.9          83.8       1.7X
 
 
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 7d35e96a78991..9482244deca4e 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -423,15 +423,8 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
     } else if (column.dataType() == DataTypes.DateType ) {
       if (rebaseDateTime) {
-        for (int i = 0; i < num; i++) {
-          if (defColumn.readInteger() == maxDefLevel) {
-            column.putInt(
-              rowId + i,
-              RebaseDateTime.rebaseJulianToGregorianDays(dataColumn.readInteger()));
-          } else {
-            column.putNull(rowId + i);
-          }
-        }
+        defColumn.readIntegersWithRebase(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
       } else {
         defColumn.readIntegers(
            num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
@@ -449,15 +442,8 @@ private void readLongBatch(int rowId, int num, WritableColumnVector column) thro
         num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
     } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
       if (rebaseDateTime) {
-        for (int i = 0; i < num; i++) {
-          if (defColumn.readInteger() == maxDefLevel) {
-            column.putLong(
-              rowId + i,
-              RebaseDateTime.rebaseJulianToGregorianMicros(dataColumn.readLong()));
-          } else {
-            column.putNull(rowId + i);
-          }
-        }
+        defColumn.readLongsWithRebase(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
       } else {
         defColumn.readLongs(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
index c62dc3d86386e..2ed2e11b60c03 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
@@ -22,6 +22,7 @@
 
 import org.apache.parquet.bytes.ByteBufferInputStream;
 import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
 import org.apache.parquet.column.values.ValuesReader;
@@ -81,6 +82,33 @@ public final void readIntegers(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  // A fork of `readIntegers` to rebase the date values. For performance reasons, this method
+  // iterates the values twice: check if we need to rebase first, then go to the optimized branch
+  // if rebase is not needed.
+  @Override
+  public final void readIntegersWithRebase(int total, WritableColumnVector c, int rowId) {
+    int requiredBytes = total * 4;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+    boolean rebase = false;
+    for (int i = 0; i < total; i += 1) {
+      rebase |= buffer.getInt(buffer.position() + i * 4) < RebaseDateTime.lastSwitchJulianDay();
+    }
+    if (rebase) {
+      for (int i = 0; i < total; i += 1) {
+        c.putInt(rowId + i, RebaseDateTime.rebaseJulianToGregorianDays(buffer.getInt()));
+      }
+    } else {
+      if (buffer.hasArray()) {
+        int offset = buffer.arrayOffset() + buffer.position();
+        c.putIntsLittleEndian(rowId, total, buffer.array(), offset);
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putInt(rowId + i, buffer.getInt());
+        }
+      }
+    }
+  }
+
   @Override
   public final void readLongs(int total, WritableColumnVector c, int rowId) {
     int requiredBytes = total * 8;
@@ -96,6 +124,33 @@ public final void readLongs(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  // A fork of `readLongs` to rebase the timestamp values. For performance reasons, this method
+  // iterates the values twice: check if we need to rebase first, then go to the optimized branch
+  // if rebase is not needed.
+  @Override
+  public final void readLongsWithRebase(int total, WritableColumnVector c, int rowId) {
+    int requiredBytes = total * 8;
+    ByteBuffer buffer = getBuffer(requiredBytes);
+    boolean rebase = false;
+    for (int i = 0; i < total; i += 1) {
+      rebase |= buffer.getLong(buffer.position() + i * 8) < RebaseDateTime.lastSwitchJulianTs();
+    }
+    if (rebase) {
+      for (int i = 0; i < total; i += 1) {
+        c.putLong(rowId + i, RebaseDateTime.rebaseJulianToGregorianMicros(buffer.getLong()));
+      }
+    } else {
+      if (buffer.hasArray()) {
+        int offset = buffer.arrayOffset() + buffer.position();
+        c.putLongsLittleEndian(rowId, total, buffer.array(), offset);
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putLong(rowId + i, buffer.getLong());
+        }
+      }
+    }
+  }
+
   @Override
   public final void readFloats(int total, WritableColumnVector c, int rowId) {
     int requiredBytes = total * 4;
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index fe3d31ae8e746..4d72a33fcf774 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -26,6 +26,7 @@
 import org.apache.parquet.io.ParquetDecodingException;
 import org.apache.parquet.io.api.Binary;
 
+import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
 import java.io.IOException;
@@ -203,6 +204,43 @@ public void readIntegers(
     }
   }
 
+  // A fork of `readIntegers`, which rebases the date int value (days) before filling
+  // the Spark column vector.
+  public void readIntegersWithRebase(
+      int total,
+      WritableColumnVector c,
+      int rowId,
+      int level,
+      VectorizedValuesReader data) throws IOException {
+    int left = total;
+    while (left > 0) {
+      if (this.currentCount == 0) this.readNextGroup();
+      int n = Math.min(left, this.currentCount);
+      switch (mode) {
+        case RLE:
+          if (currentValue == level) {
+            data.readIntegersWithRebase(n, c, rowId);
+          } else {
+            c.putNulls(rowId, n);
+          }
+          break;
+        case PACKED:
+          for (int i = 0; i < n; ++i) {
+            if (currentBuffer[currentBufferIdx++] == level) {
+              c.putInt(rowId + i,
+                RebaseDateTime.rebaseJulianToGregorianDays(data.readInteger()));
+            } else {
+              c.putNull(rowId + i);
+            }
+          }
+          break;
+      }
+      rowId += n;
+      left -= n;
+      currentCount -= n;
+    }
+  }
+
   // TODO: can this code duplication be removed without a perf penalty?
   public void readBooleans(
       int total,
@@ -342,6 +380,43 @@ public void readLongs(
     }
   }
 
+  // A fork of `readLongs`, which rebases the timestamp long value (microseconds) before filling
+  // the Spark column vector.
+  public void readLongsWithRebase(
+      int total,
+      WritableColumnVector c,
+      int rowId,
+      int level,
+      VectorizedValuesReader data) throws IOException {
+    int left = total;
+    while (left > 0) {
+      if (this.currentCount == 0) this.readNextGroup();
+      int n = Math.min(left, this.currentCount);
+      switch (mode) {
+        case RLE:
+          if (currentValue == level) {
+            data.readLongsWithRebase(n, c, rowId);
+          } else {
+            c.putNulls(rowId, n);
+          }
+          break;
+        case PACKED:
+          for (int i = 0; i < n; ++i) {
+            if (currentBuffer[currentBufferIdx++] == level) {
+              c.putLong(rowId + i,
+                RebaseDateTime.rebaseJulianToGregorianMicros(data.readLong()));
+            } else {
+              c.putNull(rowId + i);
+            }
+          }
+          break;
+      }
+      rowId += n;
+      left -= n;
+      currentCount -= n;
+    }
+  }
+
   public void readFloats(
       int total,
       WritableColumnVector c,
@@ -508,6 +583,11 @@ public void readIntegers(int total, WritableColumnVector c, int rowId) {
     }
   }
 
+  @Override
+  public void readIntegersWithRebase(int total, WritableColumnVector c, int rowId) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public byte readByte() {
     throw new UnsupportedOperationException("only readInts is valid.");
@@ -523,6 +603,11 @@ public void readLongs(int total, WritableColumnVector c, int rowId) {
     throw new UnsupportedOperationException("only readInts is valid.");
   }
 
+  @Override
+  public void readLongsWithRebase(int total, WritableColumnVector c, int rowId) {
+    throw new UnsupportedOperationException("only readInts is valid.");
+  }
+
   @Override
   public void readBinary(int total, WritableColumnVector c, int rowId) {
     throw new UnsupportedOperationException("only readInts is valid.");
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
index 57d92ae27ece8..809ac44cc8272 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
@@ -40,7 +40,9 @@ public interface VectorizedValuesReader {
   void readBooleans(int total, WritableColumnVector c, int rowId);
   void readBytes(int total, WritableColumnVector c, int rowId);
   void readIntegers(int total, WritableColumnVector c, int rowId);
+  void readIntegersWithRebase(int total, WritableColumnVector c, int rowId);
   void readLongs(int total, WritableColumnVector c, int rowId);
+  void readLongsWithRebase(int total, WritableColumnVector c, int rowId);
   void readFloats(int total, WritableColumnVector c, int rowId);
   void readDoubles(int total, WritableColumnVector c, int rowId);
   void readBinary(int total, WritableColumnVector c, int rowId);
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index 077ac28c149ee..7968836a00d0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -49,15 +49,15 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
       .select($"seconds".cast("timestamp").as("ts"))
   }
 
-  private def genTsAfter1582(cardinality: Int): DataFrame = {
-    val start = LocalDateTime.of(1582, 10, 15, 0, 0, 0)
+  private def genTsAfter1900(cardinality: Int): DataFrame = {
+    val start = LocalDateTime.of(1900, 1, 31, 0, 0, 0)
     val end = LocalDateTime.of(3000, 1, 1, 0, 0, 0)
     genTs(cardinality, start, end)
   }
 
-  private def genTsBefore1582(cardinality: Int): DataFrame = {
+  private def genTsBefore1900(cardinality: Int): DataFrame = {
     val start = LocalDateTime.of(10, 1, 1, 0, 0, 0)
-    val end = LocalDateTime.of(1580, 1, 1, 0, 0, 0)
+    val end = LocalDateTime.of(1900, 1, 1, 0, 0, 0)
     genTs(cardinality, start, end)
   }
 
@@ -71,34 +71,35 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
   }
 
   private def genDateAfter1582(cardinality: Int): DataFrame = {
-    val start = LocalDate.of(1582, 10, 15)
+    val start = LocalDate.of(1582, 10, 31)
     val end = LocalDate.of(3000, 1, 1)
     genDate(cardinality, start, end)
   }
 
   private def genDateBefore1582(cardinality: Int): DataFrame = {
     val start = LocalDate.of(10, 1, 1)
-    val end = LocalDate.of(1580, 1, 1)
+    val end = LocalDate.of(1580, 10, 1)
     genDate(cardinality, start, end)
   }
 
-  private def genDF(cardinality: Int, dateTime: String, after1582: Boolean): DataFrame = {
-    (dateTime, after1582) match {
+  private def genDF(cardinality: Int, dateTime: String, modernDates: Boolean): DataFrame = {
+    (dateTime, modernDates) match {
       case ("date", true) => genDateAfter1582(cardinality)
       case ("date", false) => genDateBefore1582(cardinality)
-      case ("timestamp", true) => genTsAfter1582(cardinality)
-      case ("timestamp", false) => genTsBefore1582(cardinality)
+      case ("timestamp", true) => genTsAfter1900(cardinality)
+      case ("timestamp", false) => genTsBefore1900(cardinality)
       case _ => throw new IllegalArgumentException(
-        s"cardinality = $cardinality dateTime = $dateTime after1582 = $after1582")
+        s"cardinality = $cardinality dateTime = $dateTime modernDates = $modernDates")
     }
   }
 
   private def benchmarkInputs(benchmark: Benchmark, rowsNum: Int, dateTime: String): Unit = {
-    benchmark.addCase("after 1582, noop", 1) { _ =>
-      genDF(rowsNum, dateTime, after1582 = true).noop()
+    val year = if (dateTime == "date") 1582 else 1900
+    benchmark.addCase(s"after $year, noop", 1) { _ =>
+      genDF(rowsNum, dateTime, modernDates = true).noop()
     }
-    benchmark.addCase("before 1582, noop", 1) { _ =>
-      genDF(rowsNum, dateTime, after1582 = false).noop()
+    benchmark.addCase(s"before $year, noop", 1) { _ =>
+      genDF(rowsNum, dateTime, modernDates = false).noop()
     }
   }
 
@@ -107,23 +108,26 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
   }
 
   private def caseName(
-      after1582: Boolean,
+      modernDates: Boolean,
+      dateTime: String,
       rebase: Option[Boolean] = None,
       vec: Option[Boolean] = None): String = {
-    val period = if (after1582) "after" else "before"
+    val period = if (modernDates) "after" else "before"
+    val year = if (dateTime == "date") 1582 else 1900
     val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("")
     val rebaseFlag = rebase.map(flagToStr).map(flag => s", rebase $flag").getOrElse("")
-    s"$period 1582$vecFlag$rebaseFlag"
+    s"$period $year$vecFlag$rebaseFlag"
   }
 
   private def getPath(
       basePath: File,
       dateTime: String,
-      after1582: Boolean,
+      modernDates: Boolean,
       rebase: Option[Boolean] = None): String = {
-    val period = if (after1582) "after" else "before"
+    val period = if (modernDates) "after" else "before"
+    val year = if (dateTime == "date") 1582 else 1900
     val rebaseFlag = rebase.map(flagToStr).map(flag => s"_$flag").getOrElse("")
-    basePath.getAbsolutePath + s"/${dateTime}_${period}_1582$rebaseFlag"
+    basePath.getAbsolutePath + s"/${dateTime}_${period}_$year$rebaseFlag"
   }
 
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
@@ -139,16 +143,16 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 rowsNum,
                 output = output)
               benchmarkInputs(benchmark, rowsNum, dateTime)
-              Seq(true, false).foreach { after1582 =>
+              Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { rebase =>
-                  benchmark.addCase(caseName(after1582, Some(rebase)), 1) { _ =>
+                  benchmark.addCase(caseName(modernDates, dateTime, Some(rebase)), 1) { _ =>
                     withSQLConf(
                       SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
-                      genDF(rowsNum, dateTime, after1582)
+                      genDF(rowsNum, dateTime, modernDates)
                         .write
                         .mode("overwrite")
                         .format("parquet")
-                        .save(getPath(path, dateTime, after1582, Some(rebase)))
+                        .save(getPath(path, dateTime, modernDates, Some(rebase)))
                     }
                   }
                 }
@@ -157,16 +161,15 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
 
               val benchmark2 = new Benchmark(
                 s"Load ${dateTime}s from parquet", rowsNum, output = output)
-              Seq(true, false).foreach { after1582 =>
+              Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { vec =>
                   Seq(false, true).foreach { rebase =>
-                    benchmark2.addCase(caseName(after1582, Some(rebase), Some(vec)), 3) { _ =>
-                      withSQLConf(
-                        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString,
-                        SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                    val name = caseName(modernDates, dateTime, Some(rebase), Some(vec))
+                    benchmark2.addCase(name, 3) { _ =>
+                      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString) {
                         spark.read
                           .format("parquet")
-                          .load(getPath(path, dateTime, after1582, Some(rebase)))
+                          .load(getPath(path, dateTime, modernDates, Some(rebase)))
                           .noop()
                       }
                     }
@@ -183,13 +186,13 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
             Seq("date", "timestamp").foreach { dateTime =>
               val benchmark = new Benchmark(s"Save ${dateTime}s to ORC", rowsNum, output = output)
               benchmarkInputs(benchmark, rowsNum, dateTime)
-              Seq(true, false).foreach { after1582 =>
-                benchmark.addCase(caseName(after1582), 1) { _ =>
-                  genDF(rowsNum, dateTime, after1582)
+              Seq(true, false).foreach { modernDates =>
+                benchmark.addCase(caseName(modernDates, dateTime), 1) { _ =>
+                  genDF(rowsNum, dateTime, modernDates)
                     .write
                     .mode("overwrite")
                     .format("orc")
-                    .save(getPath(path, dateTime, after1582))
+                    .save(getPath(path, dateTime, modernDates))
                 }
               }
               benchmark.run()
@@ -198,14 +201,14 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 s"Load ${dateTime}s from ORC",
                 rowsNum,
                 output = output)
-              Seq(true, false).foreach { after1582 =>
+              Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { vec =>
-                  benchmark2.addCase(caseName(after1582, vec = Some(vec)), 3) { _ =>
+                  benchmark2.addCase(caseName(modernDates, dateTime, vec = Some(vec)), 3) { _ =>
                     withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vec.toString) {
                       spark
                         .read
                         .format("orc")
-                        .load(getPath(path, dateTime, after1582))
+                        .load(getPath(path, dateTime, modernDates))
                         .noop()
                     }
                   }

From 04b369942a0fb850a4df8dd0d9370114177d386a Mon Sep 17 00:00:00 2001
From: Kazuaki Ishizaki <ishizaki@jp.ibm.com>
Date: Mon, 4 May 2020 16:53:50 +0900
Subject: [PATCH 0624/1280] [MINOR][DOCS] Fix typo in documents

Fixed typo in `docs` directory and in `project/MimaExcludes.scala`

Better readability of documents

No

No test needed

Closes #28447 from kiszk/typo_20200504.

Authored-by: Kazuaki Ishizaki <ishizaki@jp.ibm.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 35fcc8d5c58626836cd4d99a472e7350ea3acd0d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/spark-standalone.md                  | 2 +-
 docs/sql-migration-guide.md               | 6 +++---
 docs/sql-ref-functions-udf-hive.md        | 4 ++--
 docs/sql-ref-functions.md                 | 2 +-
 docs/sql-ref-syntax-aux-describe-query.md | 2 +-
 docs/web-ui.md                            | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 2c2ed53b478c3..1e6f8c586d546 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -335,7 +335,7 @@ SPARK_WORKER_OPTS supports the following system properties:
     overlap with `spark.worker.cleanup.enabled`, as this enables cleanup of non-shuffle files in
     local directories of a dead executor, while `spark.worker.cleanup.enabled` enables cleanup of
     all files/subdirectories of a stopped and timeout application.
-    This only affects Standalone mode, support of other cluster manangers can be added in the future.
+    This only affects Standalone mode, support of other cluster managers can be added in the future.
   </td>
   <td>2.4.0</td>
 </tr>
diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 774fb2caea252..d515b57cbea22 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -36,7 +36,7 @@ license: |
 
   - In Spark 3.0, `CREATE TABLE` without a specific provider uses the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and below, it was Hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
 
-  - In Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception is thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and below, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to a integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
+  - In Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception is thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and below, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to an integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
 
   - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
 
@@ -44,7 +44,7 @@ license: |
 
   - Refreshing a cached table would trigger a table uncache operation and then a table cache (lazily) operation. In Spark version 2.4 and below, the cache name and storage level are not preserved before the uncache operation. Therefore, the cache name and storage level could be changed unexpectedly. In Spark 3.0, cache name and storage level are first preserved for cache recreation. It helps to maintain a consistent cache behavior upon table refreshing.
 
-  - In Spark 3.0, the properties listing below become reserved; commands fail if you specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. You need their specific clauses to specify them, for example, `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. You can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, for example: `SET DBPROTERTIES('location'='/tmp')` will have no effect. In Spark version 2.4 and below, these properties are neither reserved nor have side effects, for example, `SET DBPROTERTIES('location'='/tmp')` do not change the location of the database but only create a headless property just like `'a'='b'`.
+  - In Spark 3.0, the properties listing below become reserved; commands fail if you specify reserved properties in places like `CREATE DATABASE ... WITH DBPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. You need their specific clauses to specify them, for example, `CREATE DATABASE test COMMENT 'any comment' LOCATION 'some path'`. You can set `spark.sql.legacy.notReserveProperties` to `true` to ignore the `ParseException`, in this case, these properties will be silently removed, for example: `SET DBPROPERTIES('location'='/tmp')` will have no effect. In Spark version 2.4 and below, these properties are neither reserved nor have side effects, for example, `SET DBPROPERTIES('location'='/tmp')` do not change the location of the database but only create a headless property just like `'a'='b'`.
 
     | Property (case sensitive) | Database Reserved | Table Reserved | Remarks |
     | ------------------------- | ----------------- | -------------- | ------- |
@@ -124,7 +124,7 @@ license: |
 
   - In Spark 3.0, negative scale of decimal is not allowed by default, for example, data type of literal like `1E10BD` is `DecimalType(11, 0)`. In Spark version 2.4 and below, it was `DecimalType(2, -9)`. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.allowNegativeScaleOfDecimal` to `true`.
 
-  - In Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with a integral string representation is coerced to a double value, for example, `+'1'` returns `1.0`. In Spark version 2.4 and below, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, for example, `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, for example, in Spark 2.4, the result of `+'1'` is string `1`.
+  - In Spark 3.0, the unary arithmetic operator plus(`+`) only accepts string, numeric and interval type values as inputs. Besides, `+` with an integral string representation is coerced to a double value, for example, `+'1'` returns `1.0`. In Spark version 2.4 and below, this operator is ignored. There is no type checking for it, thus, all type values with a `+` prefix are valid, for example, `+ array(1, 2)` is valid and results `[1, 2]`. Besides, there is no type coercion for it at all, for example, in Spark 2.4, the result of `+'1'` is string `1`.
 
   - In Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`.
 
diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md
index 97d72c44ef016..7a7129de23836 100644
--- a/docs/sql-ref-functions-udf-hive.md
+++ b/docs/sql-ref-functions-udf-hive.md
@@ -30,7 +30,7 @@ An example below uses [GenericUDFAbs](https://github.com/apache/hive/blob/master
 
 {% highlight sql %}
 -- Register `GenericUDFAbs` and use it in Spark SQL.
--- Note that, if you use your own programmed one, you need to add a JAR containig it
+-- Note that, if you use your own programmed one, you need to add a JAR containing it
 -- into a classpath,
 -- e.g., ADD JAR yourHiveUDF.jar;
 CREATE TEMPORARY FUNCTION testUDF AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs';
@@ -105,4 +105,4 @@ SELECT key, hiveUDAF(value) FROM t GROUP BY key;
 |  b|              3|
 |  a|              3|
 +---+---------------+
-{% endhighlight %}
\ No newline at end of file
+{% endhighlight %}
diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
index 7493b8bbb7cdf..67951a9695f5e 100644
--- a/docs/sql-ref-functions.md
+++ b/docs/sql-ref-functions.md
@@ -24,7 +24,7 @@ Built-in functions are commonly used routines that Spark SQL predefines and a co
 
 ### Built-in Functions
 
-Spark SQL has some categories of frequently-used built-in functions for aggregtion, arrays/maps, date/timestamp, and JSON data.
+Spark SQL has some categories of frequently-used built-in functions for aggregation, arrays/maps, date/timestamp, and JSON data.
 This subsection presents the usages and descriptions of these functions.
 
 #### Scalar Functions
diff --git a/docs/sql-ref-syntax-aux-describe-query.md b/docs/sql-ref-syntax-aux-describe-query.md
index 07ac39b4431f4..41e66dcb2e316 100644
--- a/docs/sql-ref-syntax-aux-describe-query.md
+++ b/docs/sql-ref-syntax-aux-describe-query.md
@@ -75,7 +75,7 @@ DESCRIBE QUERY WITH all_names_cte
 |    name|   string|   null|
 +--------+---------+-------+
 
--- Returns column metadata information for a inline table.
+-- Returns column metadata information for an inline table.
 DESC QUERY VALUES(100, 'John', 10000.20D) AS employee(id, name, salary);
 +--------+---------+-------+
 |col_name|data_type|comment|
diff --git a/docs/web-ui.md b/docs/web-ui.md
index e28a689c8de50..251e0a01c4189 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -99,7 +99,7 @@ This page displays the details of a specific job identified by its job ID.
 The Stages tab displays a summary page that shows the current state of all stages of all jobs in
 the Spark application.
 
-At the beginning of the page is the summary with the count of all stages by status (active, pending, completed, sikipped, and failed)
+At the beginning of the page is the summary with the count of all stages by status (active, pending, completed, skipped, and failed)
 
 <p style="text-align: center;">
   <img src="img/AllStagesPageDetail1.png" title="Stages header" alt="Stages header" width="30%">
@@ -136,7 +136,7 @@ Summary metrics for all task are represented in a table and in a timeline.
 * **[Tasks deserialization time](configuration.html#compression-and-serialization)**
 * **Duration of tasks**.
 * **GC time** is the total JVM garbage collection time.
-* **Result serialization time** is the time spent serializing the task result on a executor before sending it back to the driver.
+* **Result serialization time** is the time spent serializing the task result on an executor before sending it back to the driver.
 * **Getting result time** is the time that the driver spends fetching task results from workers.
 * **Scheduler delay** is the time the task waits to be scheduled for execution.
 * **Peak execution memory** is the maximum memory used by the internal data structures created during shuffles, aggregations and joins.

From 357fdb265e8f1e84115b8fbf6c8b08ca4f67ea53 Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Mon, 4 May 2020 12:22:29 +0000
Subject: [PATCH 0625/1280] [SPARK-31624] Fix SHOW TBLPROPERTIES for V2 tables
 that leverage the session catalog

## What changes were proposed in this pull request?

SHOW TBLPROPERTIES does not get the correct table properties for tables using the Session Catalog. This PR fixes that, by explicitly falling back to the V1 implementation if the table is in fact a V1 table. We also hide the reserved table properties for V2 tables, as users do not have control over setting these table properties. Henceforth, if they cannot be set or controlled by the user, then they shouldn't be displayed as such.

### Why are the changes needed?

Shows the incorrect table properties, i.e. only what exists in the Hive MetaStore for V2 tables that may have table properties outside of the MetaStore.

### Does this PR introduce _any_ user-facing change?

Fixes a bug

### How was this patch tested?

Regression test

Closes #28434 from brkyvz/ddlCommands.

Authored-by: Burak Yavuz <brkyvz@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 02a319d7e157c195d0a2b8c2bb992d980dde7d5c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../analysis/ResolveSessionCatalog.scala       |  3 ++-
 .../v2/ShowTablePropertiesExec.scala           |  8 ++++++--
 .../DataSourceV2SQLSessionCatalogSuite.scala   | 18 +++++++++++++++++-
 .../sql/connector/DataSourceV2SQLSuite.scala   |  2 --
 4 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
index bb31c132a66ff..9a8d2f0a142d4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala
@@ -535,7 +535,8 @@ class ResolveSessionCatalog(
             "SHOW VIEWS, only SessionCatalog supports this command.")
       }
 
-    case ShowTableProperties(r: ResolvedTable, propertyKey) if isSessionCatalog(r.catalog) =>
+    case ShowTableProperties(
+        r @ ResolvedTable(_, _, _: V1Table), propertyKey) if isSessionCatalog(r.catalog) =>
       ShowTablePropertiesCommand(r.identifier.asTableIdentifier, propertyKey)
 
     case ShowTableProperties(r: ResolvedView, propertyKey) =>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
index 0bcd7ea541045..fef63cb8253ca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Attribute, GenericRowWithSchema}
-import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, GenericRowWithSchema}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Table, TableCatalog}
 
 /**
  * Physical plan node for showing table properties.
@@ -30,11 +30,15 @@ case class ShowTablePropertiesExec(
     catalogTable: Table,
     propertyKey: Option[String]) extends V2CommandExec {
 
+  override def producedAttributes: AttributeSet = AttributeSet(output)
+
   override protected def run(): Seq[InternalRow] = {
     import scala.collection.JavaConverters._
     val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
+    // The reservered properties are accessible through DESCRIBE
     val properties = catalogTable.properties.asScala
+      .filter { case (k, v) => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(k) }
     propertyKey match {
       case Some(p) =>
         val propValue = properties
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
index 249b27c28b072..cf00b3b5e4410 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSessionCatalogSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.connector
 
-import org.apache.spark.sql.{DataFrame, SaveMode}
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
 import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
 
 class DataSourceV2SQLSessionCatalogSuite
@@ -63,4 +63,20 @@ class DataSourceV2SQLSessionCatalogSuite
       }
     }
   }
+
+  test("SPARK-31624: SHOW TBLPROPERTIES working with V2 tables and the session catalog") {
+    val t1 = "tbl"
+    withTable(t1) {
+      sql(s"CREATE TABLE $t1 (id bigint, data string) USING $v2Format TBLPROPERTIES " +
+        "(key='v', key2='v2')")
+
+      checkAnswer(sql(s"SHOW TBLPROPERTIES $t1"), Seq(Row("key", "v"), Row("key2", "v2")))
+
+      checkAnswer(sql(s"SHOW TBLPROPERTIES $t1('key')"), Row("key", "v"))
+
+      checkAnswer(
+        sql(s"SHOW TBLPROPERTIES $t1('keyX')"),
+        Row("keyX", s"Table default.$t1 does not have property: keyX"))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 3244684c33965..e947e15a179e8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -2122,8 +2122,6 @@ class DataSourceV2SQLSuite
         .add("value", StringType, nullable = false)
 
       val expected = Seq(
-        Row(TableCatalog.PROP_OWNER, defaultUser),
-        Row("provider", provider),
         Row("status", status),
         Row("user", user))
 

From 8a97e4e422ff5724a1b315cf37ada8bb0ced38d7 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 4 May 2020 08:14:12 -0700
Subject: [PATCH 0626/1280] [SPARK-31633][BUILD] Upgrade SLF4J from 1.7.16 to
 1.7.30

### What changes were proposed in this pull request?

This PR aims to upgrade SLF4J from 1.7.16 to 1.7.30.

### Why are the changes needed?

SLF4J 1.7.23+ is required to enable `slf4j-log4j12` with MDC feature to run under Java 9. Also, this will bring all latest bug fixes.
- http://www.slf4j.org/news.html

> When running under Java 9, log4j version 1.2.x is unable to correctly parse the "java.version" system property. Assuming an inccorect Java version, it proceeded to disable its MDC functionality. The slf4j-log4j12 module shipping in this release fixes the issue by tweaking MDC internals by reflection, allowing log4j to run under Java 9. See also SLF4J-393.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins with the existing tests.

Closes #28446 from dongjoon-hyun/SPARK-31633.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e7995c2ddcfd43c8cd99d2a54009139752b66e69)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 8 ++++----
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 8 ++++----
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 8 ++++----
 pom.xml                                 | 2 +-
 4 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 82b0115427735..296f518f1678d 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -112,7 +112,7 @@ javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.2//jaxb-api-2.2.2.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
-jcl-over-slf4j/1.7.16//jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
 jersey-client/2.30//jersey-client-2.30.jar
 jersey-common/2.30//jersey-common-2.30.jar
@@ -135,7 +135,7 @@ json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.16//jul-to-slf4j-1.7.16.jar
+jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
 kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
 kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
@@ -184,8 +184,8 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.16//slf4j-api-1.7.16.jar
-slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar
+slf4j-api/1.7.30//slf4j-api-1.7.30.jar
+slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
 snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
 snappy/0.2//snappy-0.2.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index fb2b894df366f..753c0dae55806 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -126,7 +126,7 @@ javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
 javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.2//jaxb-api-2.2.2.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
-jcl-over-slf4j/1.7.16//jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
 jersey-client/2.30//jersey-client-2.30.jar
 jersey-common/2.30//jersey-common-2.30.jar
@@ -150,7 +150,7 @@ json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.16//jul-to-slf4j-1.7.16.jar
+jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
 kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
 kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
@@ -198,8 +198,8 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.16//slf4j-api-1.7.16.jar
-slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar
+slf4j-api/1.7.30//slf4j-api-1.7.30.jar
+slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
 snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index db74c8960b36b..4a29f10153d9c 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -126,7 +126,7 @@ javolution/5.5.1//javolution-5.5.1.jar
 jaxb-api/2.2.11//jaxb-api-2.2.11.jar
 jaxb-runtime/2.3.2//jaxb-runtime-2.3.2.jar
 jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
-jcl-over-slf4j/1.7.16//jcl-over-slf4j-1.7.16.jar
+jcl-over-slf4j/1.7.30//jcl-over-slf4j-1.7.30.jar
 jdo-api/3.0.1//jdo-api-3.0.1.jar
 jersey-client/2.30//jersey-client-2.30.jar
 jersey-common/2.30//jersey-common-2.30.jar
@@ -148,7 +148,7 @@ json4s-scalap_2.12/3.6.6//json4s-scalap_2.12-3.6.6.jar
 jsp-api/2.1//jsp-api-2.1.jar
 jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
-jul-to-slf4j/1.7.16//jul-to-slf4j-1.7.16.jar
+jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kerb-admin/1.0.1//kerb-admin-1.0.1.jar
 kerb-client/1.0.1//kerb-client-1.0.1.jar
 kerb-common/1.0.1//kerb-common-1.0.1.jar
@@ -214,8 +214,8 @@ scala-reflect/2.12.10//scala-reflect-2.12.10.jar
 scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
 shapeless_2.12/2.3.3//shapeless_2.12-2.3.3.jar
 shims/0.7.45//shims-0.7.45.jar
-slf4j-api/1.7.16//slf4j-api-1.7.16.jar
-slf4j-log4j12/1.7.16//slf4j-log4j12-1.7.16.jar
+slf4j-api/1.7.30//slf4j-api-1.7.30.jar
+slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
 snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
diff --git a/pom.xml b/pom.xml
index f6eca3b65073f..c79ba5a21d6ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -117,7 +117,7 @@
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <maven.version>3.6.3</maven.version>
     <sbt.project.name>spark</sbt.project.name>
-    <slf4j.version>1.7.16</slf4j.version>
+    <slf4j.version>1.7.30</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
     <hadoop.version>2.7.4</hadoop.version>
     <protobuf.version>2.5.0</protobuf.version>

From c4b292e0368060abb81cb76439ca4a796df88b69 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 4 May 2020 17:27:02 -0700
Subject: [PATCH 0627/1280] [SPARK-31639] Revert SPARK-27528 Use Parquet
 logical type TIMESTAMP_MICROS by default

### What changes were proposed in this pull request?
This reverts commit https://github.com/apache/spark/commit/43a73e387cb843486adcf5b8bbd8b99010ce6e02. It sets `INT96` as the timestamp type while saving timestamps to parquet files.

### Why are the changes needed?
To be compatible with Hive and Presto that don't support the `TIMESTAMP_MICROS` type in current stable releases.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By existing test suites.

Closes #28450 from MaxGekk/parquet-int96.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 372ccba0632a76a7b02cb2c558a3ecd4fae839e5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-migration-guide.md                               | 2 --
 .../scala/org/apache/spark/sql/internal/SQLConf.scala     | 2 +-
 .../parquet/ParquetInteroperabilitySuite.scala            | 8 ++------
 .../org/apache/spark/sql/internal/SQLConfSuite.scala      | 2 +-
 4 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index d515b57cbea22..748d68957c896 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -186,8 +186,6 @@ license: |
 
   - In Spark version 2.4 and below, CSV datasource converts a malformed CSV string to a row with all `null`s in the PERMISSIVE mode. In Spark 3.0, the returned row can contain non-`null` fields if some of CSV column values were parsed and converted to desired types successfully.
 
-  - In Spark 3.0, parquet logical type `TIMESTAMP_MICROS` is used by default while saving `TIMESTAMP` columns. In Spark version 2.4 and below, `TIMESTAMP` columns are saved as `INT96` in parquet files. Note that, some SQL systems such as Hive 1.x and Impala 2.x can only read `INT96` timestamps, you can set `spark.sql.parquet.outputTimestampType` as `INT96` to restore the previous behavior and keep interoperability.
-
   - In Spark 3.0, when Avro files are written with user provided schema, the fields are matched by field names between catalyst schema and Avro schema instead of positions.
 
   - In Spark 3.0, when Avro files are written with user provided non-nullable schema, even the catalyst schema is nullable, Spark is still able to write the files. However, Spark throws runtime NullPointerException if any of the records contains null.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index f6b0bbdc627ae..72946a9fe01b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -620,7 +620,7 @@ object SQLConf {
     .stringConf
     .transform(_.toUpperCase(Locale.ROOT))
     .checkValues(ParquetOutputTimestampType.values.map(_.toString))
-    .createWithDefault(ParquetOutputTimestampType.TIMESTAMP_MICROS.toString)
+    .createWithDefault(ParquetOutputTimestampType.INT96.toString)
 
   val PARQUET_COMPRESSION = buildConf("spark.sql.parquet.compression.codec")
     .doc("Sets the compression codec used when writing Parquet files. If either `compression` or " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
index 649a46f190580..7d75077a9732a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
@@ -119,12 +119,8 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
       ).map { s => java.sql.Timestamp.valueOf(s) }
       import testImplicits._
       // match the column names of the file from impala
-      withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-        SQLConf.ParquetOutputTimestampType.INT96.toString) {
-        val df = spark.createDataset(ts).toDF().repartition(1)
-          .withColumnRenamed("value", "ts")
-        df.write.parquet(tableDir.getAbsolutePath)
-      }
+      val df = spark.createDataset(ts).toDF().repartition(1).withColumnRenamed("value", "ts")
+      df.write.parquet(tableDir.getAbsolutePath)
       FileUtils.copyFile(new File(impalaPath), new File(tableDir, "part-00001.parq"))
 
       Seq(false, true).foreach { int96TimestampConversion =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
index feccf52d0d3bc..f1cd37f91d78a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala
@@ -275,7 +275,7 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
 
     // check default value
     assert(spark.sessionState.conf.parquetOutputTimestampType ==
-      SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS)
+      SQLConf.ParquetOutputTimestampType.INT96)
 
     spark.sessionState.conf.setConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE, "timestamp_micros")
     assert(spark.sessionState.conf.parquetOutputTimestampType ==

From 567e65ea2804ac66b582429bd888c59e245e727d Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 5 May 2020 10:04:16 +0900
Subject: [PATCH 0628/1280] [SPARK-31372][SQL][TEST][FOLLOW-UP] Improve
 ExpressionsSchemaSuite so that easy to track the diff

### What changes were proposed in this pull request?
This PR follows up https://github.com/apache/spark/pull/28194.
As discussed at https://github.com/apache/spark/pull/28194/files#r418418796.
This PR will improve `ExpressionsSchemaSuite` so that easy to track the diff.
Although `ExpressionsSchemaSuite` at line
https://github.com/apache/spark/blob/b7cde42b04b21c9bfee6535199cf385855c15853/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala#L165
just want to compare the total size between expected output size and the newest output size, the scalatest framework will output the extra information contains all the content of expected output and newest output.
This PR will try to avoid this issue.
After this PR, the exception looks like below:
```
[info] - Check schemas for expression examples *** FAILED *** (7 seconds, 336 milliseconds)
[info]   340 did not equal 341 Expected 332 blocks in result file but got 333. Try regenerate the result files. (ExpressionsSchemaSuite.scala:167)
[info]   org.scalatest.exceptions.TestFailedException:
[info]   at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
[info]   at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
[info]   at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
[info]   at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
[info]   at org.apache.spark.sql.ExpressionsSchemaSuite.$anonfun$new$1(ExpressionsSchemaSuite.scala:167)
```

### Why are the changes needed?
Make the exception more concise and clear.

### Does this PR introduce _any_ user-facing change?
'No'.

### How was this patch tested?
Jenkins test.

Closes #28430 from beliefer/improve-expressions-schema-suite.

Authored-by: beliefer <beliefer@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b9494206a50d39973b46f32f2d44cc8099c078d4)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/ExpressionsSchemaSuite.scala    | 46 ++++++++++---------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
index 024d79fc9de9b..d83aa4ff4818b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -141,19 +141,19 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
       }
     }
 
+    val header = Seq(
+      s"<!-- Automatically generated by${getClass.getSimpleName} -->",
+      "## Summary",
+      s"  - Number of queries: ${outputs.size}",
+      s"  - Number of expressions that missing example: ${missingExamples.size}",
+      s"  - Expressions missing examples: ${missingExamples.mkString(",")}",
+      "## Schema of Built-in Functions",
+      "| Class name | Function name or alias | Query example | Output schema |",
+      "| ---------- | ---------------------- | ------------- | ------------- |"
+    )
+
     if (regenerateGoldenFiles) {
-      val missingExampleStr = missingExamples.mkString(",")
-      val goldenOutput = {
-        s"<!-- Automatically generated by${getClass.getSimpleName} -->\n" +
-        "## Summary\n" +
-        s"  - Number of queries: ${outputs.size}\n" +
-        s"  - Number of expressions that missing example: ${missingExamples.size}\n" +
-        s"  - Expressions missing examples: $missingExampleStr\n" +
-        "## Schema of Built-in Functions\n" +
-        "| Class name | Function name or alias | Query example | Output schema |\n" +
-        "| ---------- | ---------------------- | ------------- | ------------- |\n" +
-        outputBuffer.mkString("\n")
-      }
+      val goldenOutput = (header ++ outputBuffer).mkString("\n")
       val parent = resultFile.getParentFile
       if (!parent.exists()) {
         assert(parent.mkdirs(), "Could not create directory: " + parent)
@@ -161,18 +161,19 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
       stringToFile(resultFile, goldenOutput)
     }
 
+    val outputSize = outputs.size
+    val headerSize = header.size
     val expectedOutputs: Seq[QueryOutput] = {
-      val goldenOutput = fileToString(resultFile)
-      val lines = goldenOutput.split("\n")
+      val expectedGoldenOutput = fileToString(resultFile)
+      val lines = expectedGoldenOutput.split("\n")
+      val expectedSize = lines.size
 
-      // The header of golden file has one line, plus four lines of the summary and three
-      // lines of the header of schema table.
-      assert(lines.size == outputs.size + 8,
-        s"Expected ${outputs.size + 8} blocks in result file but got ${lines.size}. " +
-          s"Try regenerate the result files.")
+      assert(expectedSize == outputSize + headerSize,
+        s"Expected $expectedSize blocks in result file but got " +
+          s"${outputSize + headerSize}. Try regenerate the result files.")
 
-      Seq.tabulate(outputs.size) { i =>
-        val segments = lines(i + 8).split('|')
+      Seq.tabulate(outputSize) { i =>
+        val segments = lines(i + headerSize).split('|')
         QueryOutput(
           className = segments(1).trim,
           funcName = segments(2).trim,
@@ -182,7 +183,8 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
     }
 
     // Compare results.
-    assert(expectedOutputs.size == outputs.size, s"Number of queries not equals")
+    assert(expectedOutputs.size == outputSize,
+      "The number of queries not equals the number of expected queries.")
 
     outputs.zip(expectedOutputs).foreach { case (output, expected) =>
       assert(expected.sql == output.sql, "SQL query did not match")

From b8143cbbe94f134039b369c1120ab54aa3824291 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Tue, 5 May 2020 10:07:30 +0900
Subject: [PATCH 0629/1280] [SPARK-27963][FOLLOW-UP][DOCS][CORE] Remove `for
 testing` because CleanerListener is used ExecutorMonitor during dynamic
 allocation

### What changes were proposed in this pull request?

This PR aims to remove `for testing` from `CleanerListener` class description to promote this private class more clearly.

### Why are the changes needed?

After SPARK-27963 (Allow dynamic allocation without a shuffle service), `CleanerListener` is used in `ExecutorMonitor` during dynamic allocation. Specifically, `CleanerListener.shuffleCleaned` is used.
- https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/scheduler/dynalloc/ExecutorMonitor.scala#L385-L392

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

This is a private doc-only change.

Closes #28452 from dongjoon-hyun/SPARK-MINOR.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0907f2e7b505adf4e96a1fa7a80629680c3bf5bf)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/ContextCleaner.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/ContextCleaner.scala b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
index 9506c36bf9c8c..7c3d6d98a33ed 100644
--- a/core/src/main/scala/org/apache/spark/ContextCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/ContextCleaner.scala
@@ -281,7 +281,7 @@ private object ContextCleaner {
 }
 
 /**
- * Listener class used for testing when any item has been cleaned by the Cleaner class.
+ * Listener class used when any item has been cleaned by the Cleaner class.
  */
 private[spark] trait CleanerListener {
   def rddCleaned(rddId: Int): Unit

From d2a7b170b57db511a3b656b90ee497fc2e6b1f29 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 5 May 2020 05:40:15 +0000
Subject: [PATCH 0630/1280] [SPARK-31623][SQL][TESTS] Benchmark rebasing of
 INT96 and TIMESTAMP_MILLIS timestamps in read/write

### What changes were proposed in this pull request?
Add new benchmarks to `DateTimeRebaseBenchmark` for reading/writing timestamps of INT96 and TIMESTAMP_MICROS column types. Here are benchmark results for reading timestamps after 1582 year with default settings (rebasing is off for TIMESTAMP_MICROS/TIMESTAMP_MILLIS,  and rebasing on for INT96):

timestamp type | vectorized off (ns/row) | vectorized on (ns/row)
--|--|--
TIMESTAMP_MICROS| 160.1 | 50.2
INT96 | 215.6 | 117.8
TIMESTAMP_MILLIS | 159.9 | 60.6

### Why are the changes needed?
To compare default timestamp type `TIMESTAMP_MICROS` with other types in the case if an user decides to switch on them.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the benchmarks via:
```
SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/test:runMain org.apache.spark.sql.execution.benchmark.DateTimeRebaseBenchmark"
```
in the environment:
| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_252-8u252 and OpenJDK 64-Bit Server VM 11.0.7+10 |

Closes #28431 from MaxGekk/parquet-timestamps-DateTimeRebaseBenchmark.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 735771e7b4f7c6dfe70a1a6f59b0646dcc6bacd7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 152 ++++++++++++------
 .../DateTimeRebaseBenchmark-results.txt       | 152 ++++++++++++------
 .../benchmark/DateTimeRebaseBenchmark.scala   |  52 +++---
 3 files changed, 234 insertions(+), 122 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 03e0d7b8bc575..8c07bdb245b48 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -4,51 +4,99 @@ Rebasing dates/timestamps in Parquet datasource
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  20073          20073           0          5.0         200.7       1.0X
-before 1582, noop                                 10985          10985           0          9.1         109.9       1.8X
-after 1582, rebase off                            32245          32245           0          3.1         322.4       0.6X
-after 1582, rebase on                             31434          31434           0          3.2         314.3       0.6X
-before 1582, rebase off                           21590          21590           0          4.6         215.9       0.9X
-before 1582, rebase on                            22963          22963           0          4.4         229.6       0.9X
+after 1582, noop                                  19806          19806           0          5.0         198.1       1.0X
+before 1582, noop                                 10689          10689           0          9.4         106.9       1.9X
+after 1582, rebase off                            31053          31053           0          3.2         310.5       0.6X
+after 1582, rebase on                             32316          32316           0          3.1         323.2       0.6X
+before 1582, rebase off                           21765          21765           0          4.6         217.6       0.9X
+before 1582, rebase on                            23011          23011           0          4.3         230.1       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12815          12858          40          7.8         128.1       1.0X
-after 1582, vec off, rebase on                    13030          13167         148          7.7         130.3       1.0X
-after 1582, vec on, rebase off                     3705           3712           6         27.0          37.1       3.5X
-after 1582, vec on, rebase on                      3788           3791           3         26.4          37.9       3.4X
-before 1582, vec off, rebase off                  12873          12943          61          7.8         128.7       1.0X
-before 1582, vec off, rebase on                   14072          14165          80          7.1         140.7       0.9X
-before 1582, vec on, rebase off                    3694           3708          15         27.1          36.9       3.5X
-before 1582, vec on, rebase on                     4403           4484          81         22.7          44.0       2.9X
+after 1582, vec off, rebase off                   12834          12921         140          7.8         128.3       1.0X
+after 1582, vec off, rebase on                    13031          13071          38          7.7         130.3       1.0X
+after 1582, vec on, rebase off                     3642           3679          60         27.5          36.4       3.5X
+after 1582, vec on, rebase on                      3747           3789          49         26.7          37.5       3.4X
+before 1582, vec off, rebase off                  13009          13061          72          7.7         130.1       1.0X
+before 1582, vec off, rebase on                   13928          13945          16          7.2         139.3       0.9X
+before 1582, vec on, rebase off                    3656           3665          13         27.4          36.6       3.5X
+before 1582, vec on, rebase on                     4361           4392          48         22.9          43.6       2.9X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   3032           3032           0         33.0          30.3       1.0X
-before 1900, noop                                  3043           3043           0         32.9          30.4       1.0X
-after 1900, rebase off                            15634          15634           0          6.4         156.3       0.2X
-after 1900, rebase on                             18233          18233           0          5.5         182.3       0.2X
-before 1900, rebase off                           15820          15820           0          6.3         158.2       0.2X
-before 1900, rebase on                            19921          19921           0          5.0         199.2       0.2X
+after 1900, noop                                   2986           2986           0         33.5          29.9       1.0X
+before 1900, noop                                  2995           2995           0         33.4          30.0       1.0X
+after 1900, rebase off                            31493          31493           0          3.2         314.9       0.1X
+after 1900, rebase on                             31052          31052           0          3.2         310.5       0.1X
+before 1900, rebase off                           32102          32102           0          3.1         321.0       0.1X
+before 1900, rebase on                            32094          32094           0          3.1         320.9       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   14987          15008          18          6.7         149.9       1.0X
-after 1900, vec off, rebase on                    17500          17628         210          5.7         175.0       0.9X
-after 1900, vec on, rebase off                     5030           5036           7         19.9          50.3       3.0X
-after 1900, vec on, rebase on                      5066           5109          44         19.7          50.7       3.0X
-before 1900, vec off, rebase off                  15094          15213         121          6.6         150.9       1.0X
-before 1900, vec off, rebase on                   18098          18175         101          5.5         181.0       0.8X
-before 1900, vec on, rebase off                    5008           5012           4         20.0          50.1       3.0X
-before 1900, vec on, rebase on                     8803           8848          55         11.4          88.0       1.7X
+after 1900, vec off, rebase off                   20404          20422          29          4.9         204.0       1.0X
+after 1900, vec off, rebase on                    20198          20325         116          5.0         202.0       1.0X
+after 1900, vec on, rebase off                    12022          12051          39          8.3         120.2       1.7X
+after 1900, vec on, rebase on                     12019          12042          40          8.3         120.2       1.7X
+before 1900, vec off, rebase off                  21220          21263          38          4.7         212.2       1.0X
+before 1900, vec off, rebase on                   21172          21236          91          4.7         211.7       1.0X
+before 1900, vec on, rebase off                   12743          12792          46          7.8         127.4       1.6X
+before 1900, vec on, rebase on                    12665          12761          83          7.9         126.6       1.6X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   3047           3047           0         32.8          30.5       1.0X
+before 1900, noop                                  2973           2973           0         33.6          29.7       1.0X
+after 1900, rebase off                            16269          16269           0          6.1         162.7       0.2X
+after 1900, rebase on                             18375          18375           0          5.4         183.7       0.2X
+before 1900, rebase off                           16875          16875           0          5.9         168.7       0.2X
+before 1900, rebase on                            19574          19574           0          5.1         195.7       0.2X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase off                   15571          15671          95          6.4         155.7       1.0X
+after 1900, vec off, rebase on                    18076          18139          92          5.5         180.8       0.9X
+after 1900, vec on, rebase off                     4853           4888          38         20.6          48.5       3.2X
+after 1900, vec on, rebase on                      4924           4977          45         20.3          49.2       3.2X
+before 1900, vec off, rebase off                  15446          15485          34          6.5         154.5       1.0X
+before 1900, vec off, rebase on                   18743          18871         192          5.3         187.4       0.8X
+before 1900, vec on, rebase off                    4931           4944          20         20.3          49.3       3.2X
+before 1900, vec on, rebase on                     8217           8238          19         12.2          82.2       1.9X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2977           2977           0         33.6          29.8       1.0X
+before 1900, noop                                  2981           2981           0         33.5          29.8       1.0X
+after 1900, rebase off                            15336          15336           0          6.5         153.4       0.2X
+after 1900, rebase on                             17932          17932           0          5.6         179.3       0.2X
+before 1900, rebase off                           15507          15507           0          6.4         155.1       0.2X
+before 1900, rebase on                            19280          19280           0          5.2         192.8       0.2X
+
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase off                   15690          15838         159          6.4         156.9       1.0X
+after 1900, vec off, rebase on                    18019          18185         194          5.5         180.2       0.9X
+after 1900, vec on, rebase off                     6021           6038          16         16.6          60.2       2.6X
+after 1900, vec on, rebase on                      8351           8383          38         12.0          83.5       1.9X
+before 1900, vec off, rebase off                  15863          15909          46          6.3         158.6       1.0X
+before 1900, vec off, rebase on                   18810          18955         179          5.3         188.1       0.8X
+before 1900, vec on, rebase off                    6055           6095          41         16.5          60.5       2.6X
+before 1900, vec on, rebase on                     8955           9007          50         11.2          89.5       1.8X
 
 
 ================================================================================================
@@ -57,38 +105,38 @@ Rebasing dates/timestamps in ORC datasource
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  19593          19593           0          5.1         195.9       1.0X
-before 1582, noop                                 10581          10581           0          9.5         105.8       1.9X
-after 1582                                        27843          27843           0          3.6         278.4       0.7X
-before 1582                                       19435          19435           0          5.1         194.4       1.0X
+after 1582, noop                                  19561          19561           0          5.1         195.6       1.0X
+before 1582, noop                                 10676          10676           0          9.4         106.8       1.8X
+after 1582                                        27414          27414           0          3.6         274.1       0.7X
+before 1582                                       19257          19257           0          5.2         192.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10395          10507         119          9.6         103.9       1.0X
-after 1582, vec on                                 3921           3945          22         25.5          39.2       2.7X
-before 1582, vec off                              10762          10860         127          9.3         107.6       1.0X
-before 1582, vec on                                4194           4226          41         23.8          41.9       2.5X
+after 1582, vec off                               11016          11051          42          9.1         110.2       1.0X
+after 1582, vec on                                 3960           3971          16         25.3          39.6       2.8X
+before 1582, vec off                              11442          11466          23          8.7         114.4       1.0X
+before 1582, vec on                                4269           4288          20         23.4          42.7       2.6X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   3003           3003           0         33.3          30.0       1.0X
-before 1900, noop                                  3016           3016           0         33.2          30.2       1.0X
-after 1900                                        21804          21804           0          4.6         218.0       0.1X
-before 1900                                       23920          23920           0          4.2         239.2       0.1X
+after 1900, noop                                   2980           2980           0         33.6          29.8       1.0X
+before 1900, noop                                  2972           2972           0         33.6          29.7       1.0X
+after 1900                                        20638          20638           0          4.8         206.4       0.1X
+before 1900                                       22420          22420           0          4.5         224.2       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               14112          14128          17          7.1         141.1       1.0X
-after 1900, vec on                                 7347           7459         134         13.6          73.5       1.9X
-before 1900, vec off                              15170          15192          27          6.6         151.7       0.9X
-before 1900, vec on                                8280           8312          52         12.1          82.8       1.7X
+after 1900, vec off                               14368          14449         137          7.0         143.7       1.0X
+after 1900, vec on                                 7233           7324         146         13.8          72.3       2.0X
+before 1900, vec off                              14936          14951          17          6.7         149.4       1.0X
+before 1900, vec on                                7767           7770           3         12.9          77.7       1.8X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index a32a1ad8af89e..3ce5b36c35315 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -4,51 +4,99 @@ Rebasing dates/timestamps in Parquet datasource
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save dates to parquet:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23088          23088           0          4.3         230.9       1.0X
-before 1582, noop                                 10782          10782           0          9.3         107.8       2.1X
-after 1582, rebase off                            34821          34821           0          2.9         348.2       0.7X
-after 1582, rebase on                             35040          35040           0          2.9         350.4       0.7X
-before 1582, rebase off                           22151          22151           0          4.5         221.5       1.0X
-before 1582, rebase on                            24677          24677           0          4.1         246.8       0.9X
+after 1582, noop                                  23101          23101           0          4.3         231.0       1.0X
+before 1582, noop                                 10164          10164           0          9.8         101.6       2.3X
+after 1582, rebase off                            37390          37390           0          2.7         373.9       0.6X
+after 1582, rebase on                             36108          36108           0          2.8         361.1       0.6X
+before 1582, rebase off                           22982          22982           0          4.4         229.8       1.0X
+before 1582, rebase on                            23899          23899           0          4.2         239.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load dates from parquet:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13559          13650          79          7.4         135.6       1.0X
-after 1582, vec off, rebase on                    12942          12973          28          7.7         129.4       1.0X
-after 1582, vec on, rebase off                     3657           3689          29         27.3          36.6       3.7X
-after 1582, vec on, rebase on                      3859           3902          53         25.9          38.6       3.5X
-before 1582, vec off, rebase off                  12588          12607          17          7.9         125.9       1.1X
-before 1582, vec off, rebase on                   13396          13420          25          7.5         134.0       1.0X
-before 1582, vec on, rebase off                    3631           3650          19         27.5          36.3       3.7X
-before 1582, vec on, rebase on                     4706           4755          77         21.3          47.1       2.9X
+after 1582, vec off, rebase off                   12544          12579          41          8.0         125.4       1.0X
+after 1582, vec off, rebase on                    13127          13238          97          7.6         131.3       1.0X
+after 1582, vec on, rebase off                     3705           3715           8         27.0          37.1       3.4X
+after 1582, vec on, rebase on                      3866           3884          17         25.9          38.7       3.2X
+before 1582, vec off, rebase off                  12855          12862          11          7.8         128.6       1.0X
+before 1582, vec off, rebase on                   13723          13758          31          7.3         137.2       0.9X
+before 1582, vec on, rebase off                    3706           3724          25         27.0          37.1       3.4X
+before 1582, vec on, rebase on                     4872           4932          71         20.5          48.7       2.6X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save timestamps to parquet:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2681           2681           0         37.3          26.8       1.0X
-before 1900, noop                                  3051           3051           0         32.8          30.5       0.9X
-after 1900, rebase off                            16901          16901           0          5.9         169.0       0.2X
-after 1900, rebase on                             19725          19725           0          5.1         197.3       0.1X
-before 1900, rebase off                           16900          16900           0          5.9         169.0       0.2X
-before 1900, rebase on                            20381          20381           0          4.9         203.8       0.1X
+after 1900, noop                                   2862           2862           0         34.9          28.6       1.0X
+before 1900, noop                                  2838           2838           0         35.2          28.4       1.0X
+after 1900, rebase off                            33249          33249           0          3.0         332.5       0.1X
+after 1900, rebase on                             33912          33912           0          2.9         339.1       0.1X
+before 1900, rebase off                           36083          36083           0          2.8         360.8       0.1X
+before 1900, rebase on                            35901          35901           0          2.8         359.0       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load timestamps from parquet:             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15236          15291          62          6.6         152.4       1.0X
-after 1900, vec off, rebase on                    17832          18047         187          5.6         178.3       0.9X
-after 1900, vec on, rebase off                     4875           4901          31         20.5          48.7       3.1X
-after 1900, vec on, rebase on                      5354           5386          37         18.7          53.5       2.8X
-before 1900, vec off, rebase off                  15229          15338         108          6.6         152.3       1.0X
-before 1900, vec off, rebase on                   18626          18668          44          5.4         186.3       0.8X
-before 1900, vec on, rebase off                    4968           4975           6         20.1          49.7       3.1X
-before 1900, vec on, rebase on                     9913           9932          16         10.1          99.1       1.5X
+after 1900, vec off, rebase off                   19452          19496          38          5.1         194.5       1.0X
+after 1900, vec off, rebase on                    19505          19579          65          5.1         195.0       1.0X
+after 1900, vec on, rebase off                    11612          11703          86          8.6         116.1       1.7X
+after 1900, vec on, rebase on                     11533          11642          96          8.7         115.3       1.7X
+before 1900, vec off, rebase off                  20312          20368          88          4.9         203.1       1.0X
+before 1900, vec off, rebase on                   20217          20284          63          4.9         202.2       1.0X
+before 1900, vec on, rebase off                   12301          12372          64          8.1         123.0       1.6X
+before 1900, vec on, rebase on                    12253          12354          88          8.2         122.5       1.6X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2924           2924           0         34.2          29.2       1.0X
+before 1900, noop                                  2942           2942           0         34.0          29.4       1.0X
+after 1900, rebase off                            17323          17323           0          5.8         173.2       0.2X
+after 1900, rebase on                             20229          20229           0          4.9         202.3       0.1X
+before 1900, rebase off                           17166          17166           0          5.8         171.7       0.2X
+before 1900, rebase on                            21740          21740           0          4.6         217.4       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase off                   15031          15079          74          6.7         150.3       1.0X
+after 1900, vec off, rebase on                    18414          18429          18          5.4         184.1       0.8X
+after 1900, vec on, rebase off                     5020           5031          10         19.9          50.2       3.0X
+after 1900, vec on, rebase on                      5489           5527          35         18.2          54.9       2.7X
+before 1900, vec off, rebase off                  15062          15096          39          6.6         150.6       1.0X
+before 1900, vec off, rebase on                   19088          19137          64          5.2         190.9       0.8X
+before 1900, vec on, rebase off                    5021           5025           4         19.9          50.2       3.0X
+before 1900, vec on, rebase on                     9880           9913          47         10.1          98.8       1.5X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, noop                                   2951           2951           0         33.9          29.5       1.0X
+before 1900, noop                                  2999           2999           0         33.3          30.0       1.0X
+after 1900, rebase off                            16457          16457           0          6.1         164.6       0.2X
+after 1900, rebase on                             19587          19587           0          5.1         195.9       0.2X
+before 1900, rebase off                           16509          16509           0          6.1         165.1       0.2X
+before 1900, rebase on                            21375          21375           0          4.7         213.8       0.1X
+
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
+Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+------------------------------------------------------------------------------------------------------------------------
+after 1900, vec off, rebase off                   15121          15250         127          6.6         151.2       1.0X
+after 1900, vec off, rebase on                    18436          18501          56          5.4         184.4       0.8X
+after 1900, vec on, rebase off                     6440           6464          21         15.5          64.4       2.3X
+after 1900, vec on, rebase on                      9547           9605          85         10.5          95.5       1.6X
+before 1900, vec off, rebase off                  15432          15467          59          6.5         154.3       1.0X
+before 1900, vec off, rebase on                   19236          19278          45          5.2         192.4       0.8X
+before 1900, vec on, rebase off                    6424           6436          15         15.6          64.2       2.4X
+before 1900, vec on, rebase on                    10297          10373          75          9.7         103.0       1.5X
 
 
 ================================================================================================
@@ -57,38 +105,38 @@ Rebasing dates/timestamps in ORC datasource
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save dates to ORC:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  22942          22942           0          4.4         229.4       1.0X
-before 1582, noop                                 11035          11035           0          9.1         110.4       2.1X
-after 1582                                        31341          31341           0          3.2         313.4       0.7X
-before 1582                                       20376          20376           0          4.9         203.8       1.1X
+after 1582, noop                                  23323          23323           0          4.3         233.2       1.0X
+before 1582, noop                                 10178          10178           0          9.8         101.8       2.3X
+after 1582                                        31568          31568           0          3.2         315.7       0.7X
+before 1582                                       19918          19918           0          5.0         199.2       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load dates from ORC:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10361          10378          29          9.7         103.6       1.0X
-after 1582, vec on                                 3820           3828          11         26.2          38.2       2.7X
-before 1582, vec off                              10709          10720          13          9.3         107.1       1.0X
-before 1582, vec on                                4136           4153          15         24.2          41.4       2.5X
+after 1582, vec off                               10580          10610          39          9.5         105.8       1.0X
+after 1582, vec on                                 3746           3765          19         26.7          37.5       2.8X
+before 1582, vec off                              11052          11143         124          9.0         110.5       1.0X
+before 1582, vec on                                4170           4187          16         24.0          41.7       2.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Save timestamps to ORC:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2888           2888           0         34.6          28.9       1.0X
-before 1900, noop                                  2823           2823           0         35.4          28.2       1.0X
-after 1900                                        19790          19790           0          5.1         197.9       0.1X
-before 1900                                       20774          20774           0          4.8         207.7       0.1X
+after 1900, noop                                   2994           2994           0         33.4          29.9       1.0X
+before 1900, noop                                  2986           2986           0         33.5          29.9       1.0X
+after 1900                                        22517          22517           0          4.4         225.2       0.1X
+before 1900                                       23253          23253           0          4.3         232.5       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
-Load timestamps from ORC:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
+Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               14649          14687          38          6.8         146.5       1.0X
-after 1900, vec on                                 7850           7937         130         12.7          78.5       1.9X
-before 1900, vec off                              15354          15417         108          6.5         153.5       1.0X
-before 1900, vec on                                8382           8408          22         11.9          83.8       1.7X
+after 1900, vec off                               14042          14199         152          7.1         140.4       1.0X
+after 1900, vec on                                 7873           7901          25         12.7          78.7       1.8X
+before 1900, vec off                              14716          14768          48          6.8         147.2       1.0X
+before 1900, vec on                                8491           8503          10         11.8          84.9       1.7X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index 7968836a00d0f..aa47d36fe8c87 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -25,6 +25,12 @@ import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
+
+object DateTime extends Enumeration {
+  type DateTime = Value
+  val DATE, TIMESTAMP, TIMESTAMP_INT96, TIMESTAMP_MICROS, TIMESTAMP_MILLIS = Value
+}
 
 /**
  * Synthetic benchmark for rebasing of date and timestamp in read/write.
@@ -40,6 +46,7 @@ import org.apache.spark.sql.internal.SQLConf
  */
 object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
   import spark.implicits._
+  import DateTime._
 
   private def genTs(cardinality: Int, start: LocalDateTime, end: LocalDateTime): DataFrame = {
     val startSec = start.toEpochSecond(ZoneOffset.UTC)
@@ -82,19 +89,19 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
     genDate(cardinality, start, end)
   }
 
-  private def genDF(cardinality: Int, dateTime: String, modernDates: Boolean): DataFrame = {
-    (dateTime, modernDates) match {
-      case ("date", true) => genDateAfter1582(cardinality)
-      case ("date", false) => genDateBefore1582(cardinality)
-      case ("timestamp", true) => genTsAfter1900(cardinality)
-      case ("timestamp", false) => genTsBefore1900(cardinality)
+  private def genDF(cardinality: Int, dateTime: DateTime, modernDates: Boolean): DataFrame = {
+    dateTime match {
+      case DATE =>
+        if (modernDates) genDateAfter1582(cardinality) else genDateBefore1582(cardinality)
+      case TIMESTAMP | TIMESTAMP_INT96 | TIMESTAMP_MICROS | TIMESTAMP_MILLIS =>
+        if (modernDates) genTsAfter1900(cardinality) else genTsBefore1900(cardinality)
       case _ => throw new IllegalArgumentException(
         s"cardinality = $cardinality dateTime = $dateTime modernDates = $modernDates")
     }
   }
 
-  private def benchmarkInputs(benchmark: Benchmark, rowsNum: Int, dateTime: String): Unit = {
-    val year = if (dateTime == "date") 1582 else 1900
+  private def benchmarkInputs(benchmark: Benchmark, rowsNum: Int, dateTime: DateTime): Unit = {
+    val year = if (dateTime == DATE) 1582 else 1900
     benchmark.addCase(s"after $year, noop", 1) { _ =>
       genDF(rowsNum, dateTime, modernDates = true).noop()
     }
@@ -109,11 +116,11 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
 
   private def caseName(
       modernDates: Boolean,
-      dateTime: String,
+      dateTime: DateTime,
       rebase: Option[Boolean] = None,
       vec: Option[Boolean] = None): String = {
     val period = if (modernDates) "after" else "before"
-    val year = if (dateTime == "date") 1582 else 1900
+    val year = if (dateTime == DATE) 1582 else 1900
     val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("")
     val rebaseFlag = rebase.map(flagToStr).map(flag => s", rebase $flag").getOrElse("")
     s"$period $year$vecFlag$rebaseFlag"
@@ -121,15 +128,21 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
 
   private def getPath(
       basePath: File,
-      dateTime: String,
+      dateTime: DateTime,
       modernDates: Boolean,
       rebase: Option[Boolean] = None): String = {
     val period = if (modernDates) "after" else "before"
-    val year = if (dateTime == "date") 1582 else 1900
+    val year = if (dateTime == DATE) 1582 else 1900
     val rebaseFlag = rebase.map(flagToStr).map(flag => s"_$flag").getOrElse("")
     basePath.getAbsolutePath + s"/${dateTime}_${period}_$year$rebaseFlag"
   }
 
+  private def getOutputType(dateTime: DateTime): String = dateTime match {
+    case TIMESTAMP_INT96 => ParquetOutputTimestampType.INT96.toString
+    case TIMESTAMP_MILLIS => ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString
+    case _ => ParquetOutputTimestampType.TIMESTAMP_MICROS.toString
+  }
+
   override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
     val rowsNum = 100000000
 
@@ -137,9 +150,11 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
         withTempPath { path =>
           runBenchmark("Rebasing dates/timestamps in Parquet datasource") {
-            Seq("date", "timestamp").foreach { dateTime =>
+            Seq(
+              DATE, TIMESTAMP_INT96, TIMESTAMP_MICROS, TIMESTAMP_MILLIS
+            ).foreach { dateTime =>
               val benchmark = new Benchmark(
-                s"Save ${dateTime}s to parquet",
+                s"Save $dateTime to parquet",
                 rowsNum,
                 output = output)
               benchmarkInputs(benchmark, rowsNum, dateTime)
@@ -147,6 +162,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 Seq(false, true).foreach { rebase =>
                   benchmark.addCase(caseName(modernDates, dateTime, Some(rebase)), 1) { _ =>
                     withSQLConf(
+                      SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime),
                       SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
                       genDF(rowsNum, dateTime, modernDates)
                         .write
@@ -160,7 +176,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
               benchmark.run()
 
               val benchmark2 = new Benchmark(
-                s"Load ${dateTime}s from parquet", rowsNum, output = output)
+                s"Load $dateTime from parquet", rowsNum, output = output)
               Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { vec =>
                   Seq(false, true).foreach { rebase =>
@@ -183,8 +199,8 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
 
         withTempPath { path =>
           runBenchmark("Rebasing dates/timestamps in ORC datasource") {
-            Seq("date", "timestamp").foreach { dateTime =>
-              val benchmark = new Benchmark(s"Save ${dateTime}s to ORC", rowsNum, output = output)
+            Seq(DATE, TIMESTAMP).foreach { dateTime =>
+              val benchmark = new Benchmark(s"Save $dateTime to ORC", rowsNum, output = output)
               benchmarkInputs(benchmark, rowsNum, dateTime)
               Seq(true, false).foreach { modernDates =>
                 benchmark.addCase(caseName(modernDates, dateTime), 1) { _ =>
@@ -198,7 +214,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
               benchmark.run()
 
               val benchmark2 = new Benchmark(
-                s"Load ${dateTime}s from ORC",
+                s"Load $dateTime from ORC",
                 rowsNum,
                 output = output)
               Seq(true, false).foreach { modernDates =>

From edc0cb97477bfc37a69ee1fa4e94c52bf84fc5b9 Mon Sep 17 00:00:00 2001
From: turbofei <fwang12@ebay.com>
Date: Tue, 5 May 2020 15:14:33 +0900
Subject: [PATCH 0631/1280] [SPARK-31467][SQL][TEST] Refactor the sql tests to
 prevent TableAlreadyExistsException

### What changes were proposed in this pull request?
If we add UT in hive/SQLQuerySuite or other sql test suites and use table named `test`.
We may meet TableAlreadyExistsException.

```
org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException: Table or view 'test' already exists in database 'default'
```

The reason is that, there is some tests that does not clean up the tables/views.
In this PR, I add `withTempViews` for these tests.

### Why are the changes needed?
To fix the TableAlreadyExistsException issue when adding an UT, which uses table named `test` or others, in some sql test suites, such as hive/SQLQuerySuite.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?

Existed UT.

Closes #28239 from turboFei/SPARK-31467.

Authored-by: turbofei <fwang12@ebay.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 8d1f7d2a4ae08135df56cc32462b29703be46c42)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../apache/spark/sql/CachedTableSuite.scala   |  238 +--
 .../spark/sql/ColumnExpressionSuite.scala     |   40 +-
 .../org/apache/spark/sql/DataFrameSuite.scala |   87 +-
 .../sql/DataFrameWindowFunctionsSuite.scala   |  122 +-
 .../org/apache/spark/sql/JoinSuite.scala      |  270 ++--
 .../apache/spark/sql/JsonFunctionsSuite.scala |   72 +-
 .../org/apache/spark/sql/SQLQuerySuite.scala  |  895 ++++++-----
 .../sql/ScalaReflectionRelationSuite.scala    |   96 +-
 .../org/apache/spark/sql/SubquerySuite.scala  |   40 +-
 .../spark/sql/UserDefinedTypeSuite.scala      |   14 +-
 .../execution/SQLWindowFunctionSuite.scala    |  433 +++---
 .../columnar/InMemoryColumnarQuerySuite.scala |   46 +-
 .../datasources/json/JsonSuite.scala          | 1367 +++++++++--------
 .../execution/joins/BroadcastJoinSuite.scala  |   37 +-
 .../execution/metric/SQLMetricsSuite.scala    |   45 +-
 .../org/apache/spark/sql/jdbc/JDBCSuite.scala |   14 +-
 .../spark/sql/sources/InsertSuite.scala       |   16 +-
 .../spark/sql/sources/SaveLoadSuite.scala     |   36 +-
 .../spark/sql/streaming/StreamSuite.scala     |   16 +-
 .../continuous/ContinuousSuite.scala          |   22 +-
 .../execution/AggregationQuerySuite.scala     |   78 +-
 .../sql/hive/execution/HiveDDLSuite.scala     |   54 +-
 .../sql/hive/execution/HiveQuerySuite.scala   |   72 +-
 .../sql/hive/execution/SQLQuerySuite.scala    |  535 ++++---
 24 files changed, 2445 insertions(+), 2200 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 9baaaa9e07c0e..20f2a7f947b81 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -141,27 +141,31 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("uncaching temp table") {
-    testData.select("key").createOrReplaceTempView("tempTable1")
-    testData.select("key").createOrReplaceTempView("tempTable2")
-    spark.catalog.cacheTable("tempTable1")
+    withTempView("tempTable1", "tempTable2") {
+      testData.select("key").createOrReplaceTempView("tempTable1")
+      testData.select("key").createOrReplaceTempView("tempTable2")
+      spark.catalog.cacheTable("tempTable1")
 
-    assertCached(sql("SELECT COUNT(*) FROM tempTable1"))
-    assertCached(sql("SELECT COUNT(*) FROM tempTable2"))
+      assertCached(sql("SELECT COUNT(*) FROM tempTable1"))
+      assertCached(sql("SELECT COUNT(*) FROM tempTable2"))
 
-    // Is this valid?
-    uncacheTable("tempTable2")
+      // Is this valid?
+      uncacheTable("tempTable2")
 
-    // Should this be cached?
-    assertCached(sql("SELECT COUNT(*) FROM tempTable1"), 0)
+      // Should this be cached?
+      assertCached(sql("SELECT COUNT(*) FROM tempTable1"), 0)
+    }
   }
 
   test("too big for memory") {
-    val data = "*" * 1000
-    sparkContext.parallelize(1 to 200000, 1).map(_ => BigData(data)).toDF()
-      .createOrReplaceTempView("bigData")
-    spark.table("bigData").persist(StorageLevel.MEMORY_AND_DISK)
-    assert(spark.table("bigData").count() === 200000L)
-    spark.table("bigData").unpersist(blocking = true)
+    withTempView("bigData") {
+      val data = "*" * 1000
+      sparkContext.parallelize(1 to 200000, 1).map(_ => BigData(data)).toDF()
+        .createOrReplaceTempView("bigData")
+      spark.table("bigData").persist(StorageLevel.MEMORY_AND_DISK)
+      assert(spark.table("bigData").count() === 200000L)
+      spark.table("bigData").unpersist(blocking = true)
+    }
   }
 
   test("calling .cache() should use in-memory columnar caching") {
@@ -225,12 +229,14 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("SELECT star from cached table") {
-    sql("SELECT * FROM testData").createOrReplaceTempView("selectStar")
-    spark.catalog.cacheTable("selectStar")
-    checkAnswer(
-      sql("SELECT * FROM selectStar WHERE key = 1"),
-      Seq(Row(1, "1")))
-    uncacheTable("selectStar")
+    withTempView("selectStar") {
+      sql("SELECT * FROM testData").createOrReplaceTempView("selectStar")
+      spark.catalog.cacheTable("selectStar")
+      checkAnswer(
+        sql("SELECT * FROM selectStar WHERE key = 1"),
+        Seq(Row(1, "1")))
+      uncacheTable("selectStar")
+    }
   }
 
   test("Self-join cached") {
@@ -375,102 +381,112 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("Drops temporary table") {
-    testData.select("key").createOrReplaceTempView("t1")
-    spark.table("t1")
-    spark.catalog.dropTempView("t1")
-    intercept[AnalysisException](spark.table("t1"))
+    withTempView("t1") {
+      testData.select("key").createOrReplaceTempView("t1")
+      spark.table("t1")
+      spark.catalog.dropTempView("t1")
+      intercept[AnalysisException](spark.table("t1"))
+    }
   }
 
   test("Drops cached temporary table") {
-    testData.select("key").createOrReplaceTempView("t1")
-    testData.select("key").createOrReplaceTempView("t2")
-    spark.catalog.cacheTable("t1")
+    withTempView("t1", "t2") {
+      testData.select("key").createOrReplaceTempView("t1")
+      testData.select("key").createOrReplaceTempView("t2")
+      spark.catalog.cacheTable("t1")
 
-    assert(spark.catalog.isCached("t1"))
-    assert(spark.catalog.isCached("t2"))
+      assert(spark.catalog.isCached("t1"))
+      assert(spark.catalog.isCached("t2"))
 
-    spark.catalog.dropTempView("t1")
-    intercept[AnalysisException](spark.table("t1"))
-    assert(!spark.catalog.isCached("t2"))
+      spark.catalog.dropTempView("t1")
+      intercept[AnalysisException](spark.table("t1"))
+      assert(!spark.catalog.isCached("t2"))
+    }
   }
 
   test("Clear all cache") {
-    sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
-    sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
-    spark.catalog.cacheTable("t1")
-    spark.catalog.cacheTable("t2")
-    spark.catalog.clearCache()
-    assert(spark.sharedState.cacheManager.isEmpty)
-
-    sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
-    sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
-    spark.catalog.cacheTable("t1")
-    spark.catalog.cacheTable("t2")
-    sql("Clear CACHE")
-    assert(spark.sharedState.cacheManager.isEmpty)
+    withTempView("t1", "t2") {
+      sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
+      sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
+      spark.catalog.cacheTable("t1")
+      spark.catalog.cacheTable("t2")
+      spark.catalog.clearCache()
+      assert(spark.sharedState.cacheManager.isEmpty)
+
+      sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
+      sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
+      spark.catalog.cacheTable("t1")
+      spark.catalog.cacheTable("t2")
+      sql("Clear CACHE")
+      assert(spark.sharedState.cacheManager.isEmpty)
+    }
   }
 
   test("Ensure accumulators to be cleared after GC when uncacheTable") {
-    sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
-    sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
+    withTempView("t1", "t2") {
+      sql("SELECT key FROM testData LIMIT 10").createOrReplaceTempView("t1")
+      sql("SELECT key FROM testData LIMIT 5").createOrReplaceTempView("t2")
 
-    spark.catalog.cacheTable("t1")
-    spark.catalog.cacheTable("t2")
+      spark.catalog.cacheTable("t1")
+      spark.catalog.cacheTable("t2")
 
-    sql("SELECT * FROM t1").count()
-    sql("SELECT * FROM t2").count()
-    sql("SELECT * FROM t1").count()
-    sql("SELECT * FROM t2").count()
+      sql("SELECT * FROM t1").count()
+      sql("SELECT * FROM t2").count()
+      sql("SELECT * FROM t1").count()
+      sql("SELECT * FROM t2").count()
+
+      val toBeCleanedAccIds = new HashSet[Long]
+
+      val accId1 = spark.table("t1").queryExecution.withCachedData.collect {
+        case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
+      }.head
+      toBeCleanedAccIds += accId1
+
+      val accId2 = spark.table("t1").queryExecution.withCachedData.collect {
+        case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
+      }.head
+      toBeCleanedAccIds += accId2
+
+      val cleanerListener = new CleanerListener {
+        def rddCleaned(rddId: Int): Unit = {}
+        def shuffleCleaned(shuffleId: Int): Unit = {}
+        def broadcastCleaned(broadcastId: Long): Unit = {}
+        def accumCleaned(accId: Long): Unit = {
+          toBeCleanedAccIds.synchronized { toBeCleanedAccIds -= accId }
+        }
+        def checkpointCleaned(rddId: Long): Unit = {}
+      }
+      spark.sparkContext.cleaner.get.attachListener(cleanerListener)
 
-    val toBeCleanedAccIds = new HashSet[Long]
+      uncacheTable("t1")
+      uncacheTable("t2")
 
-    val accId1 = spark.table("t1").queryExecution.withCachedData.collect {
-      case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
-    }.head
-    toBeCleanedAccIds += accId1
+      System.gc()
 
-    val accId2 = spark.table("t1").queryExecution.withCachedData.collect {
-      case i: InMemoryRelation => i.cacheBuilder.sizeInBytesStats.id
-    }.head
-    toBeCleanedAccIds += accId2
-
-    val cleanerListener = new CleanerListener {
-      def rddCleaned(rddId: Int): Unit = {}
-      def shuffleCleaned(shuffleId: Int): Unit = {}
-      def broadcastCleaned(broadcastId: Long): Unit = {}
-      def accumCleaned(accId: Long): Unit = {
-        toBeCleanedAccIds.synchronized { toBeCleanedAccIds -= accId }
+      eventually(timeout(10.seconds)) {
+        assert(toBeCleanedAccIds.synchronized { toBeCleanedAccIds.isEmpty },
+          "batchStats accumulators should be cleared after GC when uncacheTable")
       }
-      def checkpointCleaned(rddId: Long): Unit = {}
-    }
-    spark.sparkContext.cleaner.get.attachListener(cleanerListener)
-
-    uncacheTable("t1")
-    uncacheTable("t2")
-
-    System.gc()
 
-    eventually(timeout(10.seconds)) {
-      assert(toBeCleanedAccIds.synchronized { toBeCleanedAccIds.isEmpty },
-        "batchStats accumulators should be cleared after GC when uncacheTable")
+      assert(AccumulatorContext.get(accId1).isEmpty)
+      assert(AccumulatorContext.get(accId2).isEmpty)
     }
-
-    assert(AccumulatorContext.get(accId1).isEmpty)
-    assert(AccumulatorContext.get(accId2).isEmpty)
   }
 
   test("SPARK-10327 Cache Table is not working while subquery has alias in its project list") {
-    sparkContext.parallelize((1, 1) :: (2, 2) :: Nil)
-      .toDF("key", "value").selectExpr("key", "value", "key+1").createOrReplaceTempView("abc")
-    spark.catalog.cacheTable("abc")
-
-    val sparkPlan = sql(
-      """select a.key, b.key, c.key from
-        |abc a join abc b on a.key=b.key
-        |join abc c on a.key=c.key""".stripMargin).queryExecution.sparkPlan
-
-    assert(sparkPlan.collect { case e: InMemoryTableScanExec => e }.size === 3)
-    assert(sparkPlan.collect { case e: RDDScanExec => e }.size === 0)
+    withTempView("abc") {
+      sparkContext.parallelize((1, 1) :: (2, 2) :: Nil)
+        .toDF("key", "value").selectExpr("key", "value", "key+1").createOrReplaceTempView("abc")
+      spark.catalog.cacheTable("abc")
+
+      val sparkPlan = sql(
+        """select a.key, b.key, c.key from
+          |abc a join abc b on a.key=b.key
+          |join abc c on a.key=c.key""".stripMargin).queryExecution.sparkPlan
+
+      assert(sparkPlan.collect { case e: InMemoryTableScanExec => e }.size === 3)
+      assert(sparkPlan.collect { case e: RDDScanExec => e }.size === 0)
+    }
   }
 
   /**
@@ -628,26 +644,30 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
   }
 
   test("SPARK-15870 DataFrame can't execute after uncacheTable") {
-    val selectStar = sql("SELECT * FROM testData WHERE key = 1")
-    selectStar.createOrReplaceTempView("selectStar")
+    withTempView("selectStar") {
+      val selectStar = sql("SELECT * FROM testData WHERE key = 1")
+      selectStar.createOrReplaceTempView("selectStar")
 
-    spark.catalog.cacheTable("selectStar")
-    checkAnswer(
-      selectStar,
-      Seq(Row(1, "1")))
+      spark.catalog.cacheTable("selectStar")
+      checkAnswer(
+        selectStar,
+        Seq(Row(1, "1")))
 
-    uncacheTable("selectStar")
-    checkAnswer(
-      selectStar,
-      Seq(Row(1, "1")))
+      uncacheTable("selectStar")
+      checkAnswer(
+        selectStar,
+        Seq(Row(1, "1")))
+    }
   }
 
   test("SPARK-15915 Logical plans should use canonicalized plan when override sameResult") {
-    val localRelation = Seq(1, 2, 3).toDF()
-    localRelation.createOrReplaceTempView("localRelation")
+    withTempView("localRelation") {
+      val localRelation = Seq(1, 2, 3).toDF()
+      localRelation.createOrReplaceTempView("localRelation")
 
-    spark.catalog.cacheTable("localRelation")
-    assert(getNumInMemoryRelations(localRelation) == 1)
+      spark.catalog.cacheTable("localRelation")
+      assert(getNumInMemoryRelations(localRelation) == 1)
+    }
   }
 
   test("SPARK-19093 Caching in side subquery") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index 4bf19532edd94..fa06484a73d95 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -260,26 +260,28 @@ class ColumnExpressionSuite extends QueryTest with SharedSparkSession {
   }
 
   test("nanvl") {
-    val testData = spark.createDataFrame(sparkContext.parallelize(
-      Row(null, 3.0, Double.NaN, Double.PositiveInfinity, 1.0f, 4) :: Nil),
-      StructType(Seq(StructField("a", DoubleType), StructField("b", DoubleType),
-        StructField("c", DoubleType), StructField("d", DoubleType),
-        StructField("e", FloatType), StructField("f", IntegerType))))
+    withTempView("t") {
+      val testData = spark.createDataFrame(sparkContext.parallelize(
+        Row(null, 3.0, Double.NaN, Double.PositiveInfinity, 1.0f, 4) :: Nil),
+        StructType(Seq(StructField("a", DoubleType), StructField("b", DoubleType),
+          StructField("c", DoubleType), StructField("d", DoubleType),
+          StructField("e", FloatType), StructField("f", IntegerType))))
 
-    checkAnswer(
-      testData.select(
-        nanvl($"a", lit(5)), nanvl($"b", lit(10)), nanvl(lit(10), $"b"),
-        nanvl($"c", lit(null).cast(DoubleType)), nanvl($"d", lit(10)),
-        nanvl($"b", $"e"), nanvl($"e", $"f")),
-      Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
-    )
-    testData.createOrReplaceTempView("t")
-    checkAnswer(
-      sql(
-        "select nanvl(a, 5), nanvl(b, 10), nanvl(10, b), nanvl(c, null), nanvl(d, 10), " +
-          " nanvl(b, e), nanvl(e, f) from t"),
-      Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
-    )
+      checkAnswer(
+        testData.select(
+          nanvl($"a", lit(5)), nanvl($"b", lit(10)), nanvl(lit(10), $"b"),
+          nanvl($"c", lit(null).cast(DoubleType)), nanvl($"d", lit(10)),
+          nanvl($"b", $"e"), nanvl($"e", $"f")),
+        Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
+      )
+      testData.createOrReplaceTempView("t")
+      checkAnswer(
+        sql(
+          "select nanvl(a, 5), nanvl(b, 10), nanvl(10, b), nanvl(c, null), nanvl(d, 10), " +
+            " nanvl(b, e), nanvl(e, f) from t"),
+        Row(null, 3.0, 10.0, null, Double.PositiveInfinity, 3.0, 1.0)
+      )
+    }
   }
 
   test("===") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index f20e684bf7657..4e91a7c7bb0f4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1384,47 +1384,48 @@ class DataFrameSuite extends QueryTest
 
   test("SPARK-6941: Better error message for inserting into RDD-based Table") {
     withTempDir { dir =>
+      withTempView("parquet_base", "json_base", "rdd_base", "indirect_ds", "one_row") {
+        val tempParquetFile = new File(dir, "tmp_parquet")
+        val tempJsonFile = new File(dir, "tmp_json")
+
+        val df = Seq(Tuple1(1)).toDF()
+        val insertion = Seq(Tuple1(2)).toDF("col")
+
+        // pass case: parquet table (HadoopFsRelation)
+        df.write.mode(SaveMode.Overwrite).parquet(tempParquetFile.getCanonicalPath)
+        val pdf = spark.read.parquet(tempParquetFile.getCanonicalPath)
+        pdf.createOrReplaceTempView("parquet_base")
+
+        insertion.write.insertInto("parquet_base")
+
+        // pass case: json table (InsertableRelation)
+        df.write.mode(SaveMode.Overwrite).json(tempJsonFile.getCanonicalPath)
+        val jdf = spark.read.json(tempJsonFile.getCanonicalPath)
+        jdf.createOrReplaceTempView("json_base")
+        insertion.write.mode(SaveMode.Overwrite).insertInto("json_base")
+
+        // error cases: insert into an RDD
+        df.createOrReplaceTempView("rdd_base")
+        val e1 = intercept[AnalysisException] {
+          insertion.write.insertInto("rdd_base")
+        }
+        assert(e1.getMessage.contains("Inserting into an RDD-based table is not allowed."))
 
-      val tempParquetFile = new File(dir, "tmp_parquet")
-      val tempJsonFile = new File(dir, "tmp_json")
-
-      val df = Seq(Tuple1(1)).toDF()
-      val insertion = Seq(Tuple1(2)).toDF("col")
-
-      // pass case: parquet table (HadoopFsRelation)
-      df.write.mode(SaveMode.Overwrite).parquet(tempParquetFile.getCanonicalPath)
-      val pdf = spark.read.parquet(tempParquetFile.getCanonicalPath)
-      pdf.createOrReplaceTempView("parquet_base")
-
-      insertion.write.insertInto("parquet_base")
-
-      // pass case: json table (InsertableRelation)
-      df.write.mode(SaveMode.Overwrite).json(tempJsonFile.getCanonicalPath)
-      val jdf = spark.read.json(tempJsonFile.getCanonicalPath)
-      jdf.createOrReplaceTempView("json_base")
-      insertion.write.mode(SaveMode.Overwrite).insertInto("json_base")
-
-      // error cases: insert into an RDD
-      df.createOrReplaceTempView("rdd_base")
-      val e1 = intercept[AnalysisException] {
-        insertion.write.insertInto("rdd_base")
-      }
-      assert(e1.getMessage.contains("Inserting into an RDD-based table is not allowed."))
-
-      // error case: insert into a logical plan that is not a LeafNode
-      val indirectDS = pdf.select("_1").filter($"_1" > 5)
-      indirectDS.createOrReplaceTempView("indirect_ds")
-      val e2 = intercept[AnalysisException] {
-        insertion.write.insertInto("indirect_ds")
-      }
-      assert(e2.getMessage.contains("Inserting into an RDD-based table is not allowed."))
+        // error case: insert into a logical plan that is not a LeafNode
+        val indirectDS = pdf.select("_1").filter($"_1" > 5)
+        indirectDS.createOrReplaceTempView("indirect_ds")
+        val e2 = intercept[AnalysisException] {
+          insertion.write.insertInto("indirect_ds")
+        }
+        assert(e2.getMessage.contains("Inserting into an RDD-based table is not allowed."))
 
-      // error case: insert into an OneRowRelation
-      Dataset.ofRows(spark, OneRowRelation()).createOrReplaceTempView("one_row")
-      val e3 = intercept[AnalysisException] {
-        insertion.write.insertInto("one_row")
+        // error case: insert into an OneRowRelation
+        Dataset.ofRows(spark, OneRowRelation()).createOrReplaceTempView("one_row")
+        val e3 = intercept[AnalysisException] {
+          insertion.write.insertInto("one_row")
+        }
+        assert(e3.getMessage.contains("Inserting into an RDD-based table is not allowed."))
       }
-      assert(e3.getMessage.contains("Inserting into an RDD-based table is not allowed."))
     }
   }
 
@@ -1796,13 +1797,17 @@ class DataFrameSuite extends QueryTest
     val df = Seq("foo", "bar").map(Tuple1.apply).toDF("col")
     // invalid table names
     Seq("11111", "t~", "#$@sum", "table!#").foreach { name =>
-      val m = intercept[AnalysisException](df.createOrReplaceTempView(name)).getMessage
-      assert(m.contains(s"Invalid view name: $name"))
+      withTempView(name) {
+        val m = intercept[AnalysisException](df.createOrReplaceTempView(name)).getMessage
+        assert(m.contains(s"Invalid view name: $name"))
+      }
     }
 
     // valid table names
     Seq("table1", "`11111`", "`t~`", "`#$@sum`", "`table!#`").foreach { name =>
-      df.createOrReplaceTempView(name)
+      withTempView(name) {
+        df.createOrReplaceTempView(name)
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
index f72ccaa63b42b..09ce43e4e2b27 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala
@@ -61,26 +61,28 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("rank functions in unspecific window") {
-    val df = Seq((1, "1"), (2, "2"), (1, "2"), (2, "2")).toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    checkAnswer(
-      df.select(
-        $"key",
-        max("key").over(Window.partitionBy("value").orderBy("key")),
-        min("key").over(Window.partitionBy("value").orderBy("key")),
-        mean("key").over(Window.partitionBy("value").orderBy("key")),
-        count("key").over(Window.partitionBy("value").orderBy("key")),
-        sum("key").over(Window.partitionBy("value").orderBy("key")),
-        ntile(2).over(Window.partitionBy("value").orderBy("key")),
-        row_number().over(Window.partitionBy("value").orderBy("key")),
-        dense_rank().over(Window.partitionBy("value").orderBy("key")),
-        rank().over(Window.partitionBy("value").orderBy("key")),
-        cume_dist().over(Window.partitionBy("value").orderBy("key")),
-        percent_rank().over(Window.partitionBy("value").orderBy("key"))),
-      Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d, 0.0d) ::
-        Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d / 3.0d, 0.0d) ::
-        Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 1, 2, 2, 2, 1.0d, 0.5d) ::
-        Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 2, 3, 2, 2, 1.0d, 0.5d) :: Nil)
+    withTempView("window_table") {
+      val df = Seq((1, "1"), (2, "2"), (1, "2"), (2, "2")).toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      checkAnswer(
+        df.select(
+          $"key",
+          max("key").over(Window.partitionBy("value").orderBy("key")),
+          min("key").over(Window.partitionBy("value").orderBy("key")),
+          mean("key").over(Window.partitionBy("value").orderBy("key")),
+          count("key").over(Window.partitionBy("value").orderBy("key")),
+          sum("key").over(Window.partitionBy("value").orderBy("key")),
+          ntile(2).over(Window.partitionBy("value").orderBy("key")),
+          row_number().over(Window.partitionBy("value").orderBy("key")),
+          dense_rank().over(Window.partitionBy("value").orderBy("key")),
+          rank().over(Window.partitionBy("value").orderBy("key")),
+          cume_dist().over(Window.partitionBy("value").orderBy("key")),
+          percent_rank().over(Window.partitionBy("value").orderBy("key"))),
+        Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d, 0.0d) ::
+          Row(1, 1, 1, 1.0d, 1, 1, 1, 1, 1, 1, 1.0d / 3.0d, 0.0d) ::
+          Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 1, 2, 2, 2, 1.0d, 0.5d) ::
+          Row(2, 2, 1, 5.0d / 3.0d, 3, 5, 2, 3, 2, 2, 1.0d, 0.5d) :: Nil)
+    }
   }
 
   test("window function should fail if order by clause is not specified") {
@@ -348,15 +350,17 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("SPARK-16195 empty over spec") {
-    val df = Seq(("a", 1), ("a", 1), ("a", 2), ("b", 2)).
-      toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    checkAnswer(
-      df.select($"key", $"value", sum($"value").over(), avg($"value").over()),
-      Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
-    checkAnswer(
-      sql("select key, value, sum(value) over(), avg(value) over() from window_table"),
-      Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
+    withTempView("window_table") {
+      val df = Seq(("a", 1), ("a", 1), ("a", 2), ("b", 2)).
+        toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      checkAnswer(
+        df.select($"key", $"value", sum($"value").over(), avg($"value").over()),
+        Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
+      checkAnswer(
+        sql("select key, value, sum(value) over(), avg(value) over() from window_table"),
+        Seq(Row("a", 1, 6, 1.5), Row("a", 1, 6, 1.5), Row("a", 2, 6, 1.5), Row("b", 2, 6, 1.5)))
+    }
   }
 
   test("window function with udaf") {
@@ -548,37 +552,41 @@ class DataFrameWindowFunctionsSuite extends QueryTest
   }
 
   test("aggregation and rows between with unbounded + predicate pushdown") {
-    val df = Seq((1, "1"), (2, "2"), (2, "3"), (1, "3"), (3, "2"), (4, "3")).toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    val selectList = Seq($"key", $"value",
-      last("key").over(
-        Window.partitionBy($"value").orderBy($"key").rowsBetween(0, Long.MaxValue)),
-      last("key").over(
-        Window.partitionBy($"value").orderBy($"key").rowsBetween(Long.MinValue, 0)),
-      last("key").over(Window.partitionBy($"value").orderBy($"key").rowsBetween(-1, 1)))
-
-    checkAnswer(
-      df.select(selectList: _*).where($"value" < "3"),
-      Seq(Row(1, "1", 1, 1, 1), Row(2, "2", 3, 2, 3), Row(3, "2", 3, 3, 3)))
+    withTempView("window_table") {
+      val df = Seq((1, "1"), (2, "2"), (2, "3"), (1, "3"), (3, "2"), (4, "3")).toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      val selectList = Seq($"key", $"value",
+        last("key").over(
+          Window.partitionBy($"value").orderBy($"key").rowsBetween(0, Long.MaxValue)),
+        last("key").over(
+          Window.partitionBy($"value").orderBy($"key").rowsBetween(Long.MinValue, 0)),
+        last("key").over(Window.partitionBy($"value").orderBy($"key").rowsBetween(-1, 1)))
+
+      checkAnswer(
+        df.select(selectList: _*).where($"value" < "3"),
+        Seq(Row(1, "1", 1, 1, 1), Row(2, "2", 3, 2, 3), Row(3, "2", 3, 3, 3)))
+    }
   }
 
   test("aggregation and range between with unbounded + predicate pushdown") {
-    val df = Seq((5, "1"), (5, "2"), (4, "2"), (6, "2"), (3, "1"), (2, "2")).toDF("key", "value")
-    df.createOrReplaceTempView("window_table")
-    val selectList = Seq($"key", $"value",
-      last("value").over(
-        Window.partitionBy($"value").orderBy($"key").rangeBetween(-2, -1)).equalTo("2")
-        .as("last_v"),
-      avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(Long.MinValue, 1))
-        .as("avg_key1"),
-      avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(0, Long.MaxValue))
-        .as("avg_key2"),
-      avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(-1, 1))
-        .as("avg_key3"))
-
-    checkAnswer(
-      df.select(selectList: _*).where($"value" < 2),
-      Seq(Row(3, "1", null, 3.0, 4.0, 3.0), Row(5, "1", false, 4.0, 5.0, 5.0)))
+    withTempView("window_table") {
+      val df = Seq((5, "1"), (5, "2"), (4, "2"), (6, "2"), (3, "1"), (2, "2")).toDF("key", "value")
+      df.createOrReplaceTempView("window_table")
+      val selectList = Seq($"key", $"value",
+        last("value").over(
+          Window.partitionBy($"value").orderBy($"key").rangeBetween(-2, -1)).equalTo("2")
+          .as("last_v"),
+        avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(Long.MinValue, 1))
+          .as("avg_key1"),
+        avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(0, Long.MaxValue))
+          .as("avg_key2"),
+        avg("key").over(Window.partitionBy("value").orderBy("key").rangeBetween(-1, 1))
+          .as("avg_key3"))
+
+      checkAnswer(
+        df.select(selectList: _*).where($"value" < 2),
+        Seq(Row(3, "1", null, 3.0, 4.0, 3.0), Row(5, "1", false, 4.0, 5.0, 5.0)))
+    }
   }
 
   test("Window spill with less than the inMemoryThreshold") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
index f45bd950040ce..fe6775cc7f9b9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
@@ -401,94 +401,96 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
   }
 
   test("full outer join") {
-    upperCaseData.where('N <= 4).createOrReplaceTempView("`left`")
-    upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
+    withTempView("`left`", "`right`") {
+      upperCaseData.where('N <= 4).createOrReplaceTempView("`left`")
+      upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
 
-    val left = UnresolvedRelation(TableIdentifier("left"))
-    val right = UnresolvedRelation(TableIdentifier("right"))
+      val left = UnresolvedRelation(TableIdentifier("left"))
+      val right = UnresolvedRelation(TableIdentifier("right"))
 
-    checkAnswer(
-      left.join(right, $"left.N" === $"right.N", "full"),
-      Row(1, "A", null, null) ::
-        Row(2, "B", null, null) ::
-        Row(3, "C", 3, "C") ::
-        Row(4, "D", 4, "D") ::
-        Row(null, null, 5, "E") ::
-        Row(null, null, 6, "F") :: Nil)
+      checkAnswer(
+        left.join(right, $"left.N" === $"right.N", "full"),
+        Row(1, "A", null, null) ::
+          Row(2, "B", null, null) ::
+          Row(3, "C", 3, "C") ::
+          Row(4, "D", 4, "D") ::
+          Row(null, null, 5, "E") ::
+          Row(null, null, 6, "F") :: Nil)
 
-    checkAnswer(
-      left.join(right, ($"left.N" === $"right.N") && ($"left.N" =!= 3), "full"),
-      Row(1, "A", null, null) ::
-        Row(2, "B", null, null) ::
-        Row(3, "C", null, null) ::
-        Row(null, null, 3, "C") ::
-        Row(4, "D", 4, "D") ::
-        Row(null, null, 5, "E") ::
-        Row(null, null, 6, "F") :: Nil)
+      checkAnswer(
+        left.join(right, ($"left.N" === $"right.N") && ($"left.N" =!= 3), "full"),
+        Row(1, "A", null, null) ::
+          Row(2, "B", null, null) ::
+          Row(3, "C", null, null) ::
+          Row(null, null, 3, "C") ::
+          Row(4, "D", 4, "D") ::
+          Row(null, null, 5, "E") ::
+          Row(null, null, 6, "F") :: Nil)
 
-    checkAnswer(
-      left.join(right, ($"left.N" === $"right.N") && ($"right.N" =!= 3), "full"),
-      Row(1, "A", null, null) ::
-        Row(2, "B", null, null) ::
-        Row(3, "C", null, null) ::
-        Row(null, null, 3, "C") ::
-        Row(4, "D", 4, "D") ::
-        Row(null, null, 5, "E") ::
-        Row(null, null, 6, "F") :: Nil)
-
-    // Make sure we are UnknownPartitioning as the outputPartitioning for the outer join
-    // operator.
-    checkAnswer(
-      sql(
-        """
-        |SELECT l.a, count(*)
-        |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N)
-        |GROUP BY l.a
-      """.
-          stripMargin),
-      Row(null, 10))
+      checkAnswer(
+        left.join(right, ($"left.N" === $"right.N") && ($"right.N" =!= 3), "full"),
+        Row(1, "A", null, null) ::
+          Row(2, "B", null, null) ::
+          Row(3, "C", null, null) ::
+          Row(null, null, 3, "C") ::
+          Row(4, "D", 4, "D") ::
+          Row(null, null, 5, "E") ::
+          Row(null, null, 6, "F") :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT r.N, count(*)
+      // Make sure we are UnknownPartitioning as the outputPartitioning for the outer join
+      // operator.
+      checkAnswer(
+        sql(
+          """
+          |SELECT l.a, count(*)
           |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N)
-          |GROUP BY r.N
-        """.stripMargin),
-      Row
+          |GROUP BY l.a
+        """.
+            stripMargin),
+        Row(null, 10))
+
+      checkAnswer(
+        sql(
+          """
+            |SELECT r.N, count(*)
+            |FROM allNulls l FULL OUTER JOIN upperCaseData r ON (l.a = r.N)
+            |GROUP BY r.N
+          """.stripMargin),
+        Row
         (1, 1) ::
-        Row(2, 1) ::
-        Row(3, 1) ::
-        Row(4, 1) ::
-        Row(5, 1) ::
-        Row(6, 1) ::
-        Row(null, 4) :: Nil)
+          Row(2, 1) ::
+          Row(3, 1) ::
+          Row(4, 1) ::
+          Row(5, 1) ::
+          Row(6, 1) ::
+          Row(null, 4) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT l.N, count(*)
-          |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
-          |GROUP BY l.N
-        """.stripMargin),
-      Row(1
-        , 1) ::
-        Row(2, 1) ::
-        Row(3, 1) ::
-        Row(4, 1) ::
-        Row(5, 1) ::
-        Row(6, 1) ::
-        Row(null, 4) :: Nil)
+      checkAnswer(
+        sql(
+          """
+            |SELECT l.N, count(*)
+            |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
+            |GROUP BY l.N
+          """.stripMargin),
+        Row(1
+          , 1) ::
+          Row(2, 1) ::
+          Row(3, 1) ::
+          Row(4, 1) ::
+          Row(5, 1) ::
+          Row(6, 1) ::
+          Row(null, 4) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-        |SELECT r.a, count(*)
-        |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
-        |GROUP BY r.a
-      """.
-          stripMargin),
-      Row(null, 10))
+      checkAnswer(
+        sql(
+          """
+          |SELECT r.a, count(*)
+          |FROM upperCaseData l FULL OUTER JOIN allNulls r ON (l.N = r.a)
+          |GROUP BY r.a
+        """.
+            stripMargin),
+        Row(null, 10))
+    }
   }
 
   test("broadcasted existence join operator selection") {
@@ -614,63 +616,65 @@ class JoinSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlan
   }
 
   test("cross join detection") {
-    testData.createOrReplaceTempView("A")
-    testData.createOrReplaceTempView("B")
-    testData2.createOrReplaceTempView("C")
-    testData3.createOrReplaceTempView("D")
-    upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
-    val cartesianQueries = Seq(
-      /** The following should error out since there is no explicit cross join */
-      "SELECT * FROM testData inner join testData2",
-      "SELECT * FROM testData left outer join testData2",
-      "SELECT * FROM testData right outer join testData2",
-      "SELECT * FROM testData full outer join testData2",
-      "SELECT * FROM testData, testData2",
-      "SELECT * FROM testData, testData2 where testData.key = 1 and testData2.a = 22",
-      /** The following should fail because after reordering there are cartesian products */
-      "select * from (A join B on (A.key = B.key)) join D on (A.key=D.a) join C",
-      "select * from ((A join B on (A.key = B.key)) join C) join D on (A.key = D.a)",
-      /** Cartesian product involving C, which is not involved in a CROSS join */
-      "select * from ((A join B on (A.key = B.key)) cross join D) join C on (A.key = D.a)");
-
-    def checkCartesianDetection(query: String): Unit = {
-      val e = intercept[Exception] {
-        checkAnswer(sql(query), Nil);
+    withTempView("A", "B", "C", "D") {
+      testData.createOrReplaceTempView("A")
+      testData.createOrReplaceTempView("B")
+      testData2.createOrReplaceTempView("C")
+      testData3.createOrReplaceTempView("D")
+      upperCaseData.where('N >= 3).createOrReplaceTempView("`right`")
+      val cartesianQueries = Seq(
+        /** The following should error out since there is no explicit cross join */
+        "SELECT * FROM testData inner join testData2",
+        "SELECT * FROM testData left outer join testData2",
+        "SELECT * FROM testData right outer join testData2",
+        "SELECT * FROM testData full outer join testData2",
+        "SELECT * FROM testData, testData2",
+        "SELECT * FROM testData, testData2 where testData.key = 1 and testData2.a = 22",
+        /** The following should fail because after reordering there are cartesian products */
+        "select * from (A join B on (A.key = B.key)) join D on (A.key=D.a) join C",
+        "select * from ((A join B on (A.key = B.key)) join C) join D on (A.key = D.a)",
+        /** Cartesian product involving C, which is not involved in a CROSS join */
+        "select * from ((A join B on (A.key = B.key)) cross join D) join C on (A.key = D.a)");
+
+      def checkCartesianDetection(query: String): Unit = {
+        val e = intercept[Exception] {
+          checkAnswer(sql(query), Nil);
+        }
+        assert(e.getMessage.contains("Detected implicit cartesian product"))
       }
-      assert(e.getMessage.contains("Detected implicit cartesian product"))
-    }
 
-    withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
-      cartesianQueries.foreach(checkCartesianDetection)
-    }
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+        cartesianQueries.foreach(checkCartesianDetection)
+      }
 
-    // Check that left_semi, left_anti, existence joins without conditions do not throw
-    // an exception if cross joins are disabled
-    withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
-      checkAnswer(
-        sql("SELECT * FROM testData3 LEFT SEMI JOIN testData2"),
-        Row(1, null) :: Row (2, 2) :: Nil)
-      checkAnswer(
-        sql("SELECT * FROM testData3 LEFT ANTI JOIN testData2"),
-        Nil)
-      checkAnswer(
-        sql(
-          """
-            |SELECT a FROM testData3
-            |WHERE
-            |  EXISTS (SELECT * FROM testData)
-            |OR
-            |  EXISTS (SELECT * FROM testData2)""".stripMargin),
-        Row(1) :: Row(2) :: Nil)
-      checkAnswer(
-        sql(
-          """
-            |SELECT key FROM testData
-            |WHERE
-            |  key IN (SELECT a FROM testData2)
-            |OR
-            |  key IN (SELECT a FROM testData3)""".stripMargin),
-        Row(1) :: Row(2) :: Row(3) :: Nil)
+      // Check that left_semi, left_anti, existence joins without conditions do not throw
+      // an exception if cross joins are disabled
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+        checkAnswer(
+          sql("SELECT * FROM testData3 LEFT SEMI JOIN testData2"),
+          Row(1, null) :: Row (2, 2) :: Nil)
+        checkAnswer(
+          sql("SELECT * FROM testData3 LEFT ANTI JOIN testData2"),
+          Nil)
+        checkAnswer(
+          sql(
+            """
+              |SELECT a FROM testData3
+              |WHERE
+              |  EXISTS (SELECT * FROM testData)
+              |OR
+              |  EXISTS (SELECT * FROM testData2)""".stripMargin),
+          Row(1) :: Row(2) :: Nil)
+        checkAnswer(
+          sql(
+            """
+              |SELECT key FROM testData
+              |WHERE
+              |  key IN (SELECT a FROM testData2)
+              |OR
+              |  key IN (SELECT a FROM testData3)""".stripMargin),
+          Row(1) :: Row(2) :: Row(3) :: Nil)
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index b989b5dadd224..a4ed7ccc5508a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -429,57 +429,69 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
   }
 
   test("from_json - array of arrays") {
-    val jsonDF = Seq("[[1], [2, 3], [4, 5, 6]]").toDF("a")
-    val schema = new ArrayType(ArrayType(IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("[[1], [2, 3], [4, 5, 6]]").toDF("a")
+      val schema = new ArrayType(ArrayType(IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select json[0][0], json[1][1], json[2][2] from jsonTable"),
-      Seq(Row(1, 3, 6)))
+      checkAnswer(
+        sql("select json[0][0], json[1][1], json[2][2] from jsonTable"),
+        Seq(Row(1, 3, 6)))
+    }
   }
 
   test("from_json - array of arrays - malformed row") {
-    val jsonDF = Seq("[[1], [2, 3], 4, 5, 6]]").toDF("a")
-    val schema = new ArrayType(ArrayType(IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("[[1], [2, 3], 4, 5, 6]]").toDF("a")
+      val schema = new ArrayType(ArrayType(IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(sql("select json[0] from jsonTable"), Seq(Row(null)))
+      checkAnswer(sql("select json[0] from jsonTable"), Seq(Row(null)))
+    }
   }
 
   test("from_json - array of structs") {
-    val jsonDF = Seq("""[{"a":1}, {"a":2}, {"a":3}]""").toDF("a")
-    val schema = new ArrayType(new StructType().add("a", IntegerType), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1}, {"a":2}, {"a":3}]""").toDF("a")
+      val schema = new ArrayType(new StructType().add("a", IntegerType), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select json[0], json[1], json[2] from jsonTable"),
-      Seq(Row(Row(1), Row(2), Row(3))))
+      checkAnswer(
+        sql("select json[0], json[1], json[2] from jsonTable"),
+        Seq(Row(Row(1), Row(2), Row(3))))
+    }
   }
 
   test("from_json - array of structs - malformed row") {
-    val jsonDF = Seq("""[{"a":1}, {"a:2}, {"a":3}]""").toDF("a")
-    val schema = new ArrayType(new StructType().add("a", IntegerType), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1}, {"a:2}, {"a":3}]""").toDF("a")
+      val schema = new ArrayType(new StructType().add("a", IntegerType), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(sql("select json[0], json[1]from jsonTable"), Seq(Row(null, null)))
+      checkAnswer(sql("select json[0], json[1]from jsonTable"), Seq(Row(null, null)))
+    }
   }
 
   test("from_json - array of maps") {
-    val jsonDF = Seq("""[{"a":1}, {"b":2}]""").toDF("a")
-    val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1}, {"b":2}]""").toDF("a")
+      val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("""select json[0], json[1] from jsonTable"""),
-      Seq(Row(Map("a" -> 1), Map("b" -> 2))))
+      checkAnswer(
+        sql("""select json[0], json[1] from jsonTable"""),
+        Seq(Row(Map("a" -> 1), Map("b" -> 2))))
+    }
   }
 
   test("from_json - array of maps - malformed row") {
-    val jsonDF = Seq("""[{"a":1} "b":2}]""").toDF("a")
-    val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
-    jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = Seq("""[{"a":1} "b":2}]""").toDF("a")
+      val schema = new ArrayType(MapType(StringType, IntegerType, false), false)
+      jsonDF.select(from_json($"a", schema) as "json").createOrReplaceTempView("jsonTable")
 
-    checkAnswer(sql("""select json[0] from jsonTable"""), Seq(Row(null)))
+      checkAnswer(sql("""select json[0] from jsonTable"""), Seq(Row(null)))
+    }
   }
 
   test("to_json - array of primitive types") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 58a40b0df9ea4..d336f528f5a05 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -50,13 +50,15 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   setupTestData()
 
   test("SPARK-8010: promote numeric to string") {
-    val df = Seq((1, 1)).toDF("key", "value")
-    df.createOrReplaceTempView("src")
-    val queryCaseWhen = sql("select case when true then 1.0 else '1' end from src ")
-    val queryCoalesce = sql("select coalesce(null, 1, '1') from src ")
+    withTempView("src") {
+      val df = Seq((1, 1)).toDF("key", "value")
+      df.createOrReplaceTempView("src")
+      val queryCaseWhen = sql("select case when true then 1.0 else '1' end from src ")
+      val queryCoalesce = sql("select coalesce(null, 1, '1') from src ")
 
-    checkAnswer(queryCaseWhen, Row("1.0") :: Nil)
-    checkAnswer(queryCoalesce, Row("1") :: Nil)
+      checkAnswer(queryCaseWhen, Row("1.0") :: Nil)
+      checkAnswer(queryCoalesce, Row("1") :: Nil)
+    }
   }
 
   test("show functions") {
@@ -124,31 +126,35 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-6743: no columns from cache") {
-    Seq(
-      (83, 0, 38),
-      (26, 0, 79),
-      (43, 81, 24)
-    ).toDF("a", "b", "c").createOrReplaceTempView("cachedData")
+    withTempView("cachedData") {
+      Seq(
+        (83, 0, 38),
+        (26, 0, 79),
+        (43, 81, 24)
+      ).toDF("a", "b", "c").createOrReplaceTempView("cachedData")
 
-    spark.catalog.cacheTable("cachedData")
-    withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
-      checkAnswer(
-        sql("SELECT t1.b FROM cachedData, cachedData t1 GROUP BY t1.b"),
-        Row(0) :: Row(81) :: Nil)
+      spark.catalog.cacheTable("cachedData")
+      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+        checkAnswer(
+          sql("SELECT t1.b FROM cachedData, cachedData t1 GROUP BY t1.b"),
+          Row(0) :: Row(81) :: Nil)
+      }
     }
   }
 
   test("self join with aliases") {
-    Seq(1, 2, 3).map(i => (i, i.toString)).toDF("int", "str").createOrReplaceTempView("df")
+    withTempView("df") {
+      Seq(1, 2, 3).map(i => (i, i.toString)).toDF("int", "str").createOrReplaceTempView("df")
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT x.str, COUNT(*)
-          |FROM df x JOIN df y ON x.str = y.str
-          |GROUP BY x.str
+      checkAnswer(
+        sql(
+          """
+            |SELECT x.str, COUNT(*)
+            |FROM df x JOIN df y ON x.str = y.str
+            |GROUP BY x.str
         """.stripMargin),
-      Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+        Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+    }
   }
 
   test("support table.star") {
@@ -162,6 +168,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("self join with alias in agg") {
+    withTempView("df") {
       Seq(1, 2, 3)
         .map(i => (i, i.toString))
         .toDF("int", "str")
@@ -169,14 +176,15 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
         .agg($"str", count("str").as("strCount"))
         .createOrReplaceTempView("df")
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT x.str, SUM(x.strCount)
-          |FROM df x JOIN df y ON x.str = y.str
-          |GROUP BY x.str
+      checkAnswer(
+        sql(
+          """
+            |SELECT x.str, SUM(x.strCount)
+            |FROM df x JOIN df y ON x.str = y.str
+            |GROUP BY x.str
         """.stripMargin),
-      Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+        Row("1", 1) :: Row("2", 1) :: Row("3", 1) :: Nil)
+    }
   }
 
   test("SPARK-8668 expr function") {
@@ -223,41 +231,47 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("grouping on nested fields") {
-    spark.read
-      .json(Seq("""{"nested": {"attribute": 1}, "value": 2}""").toDS())
-     .createOrReplaceTempView("rows")
+    withTempView("rows") {
+      spark.read
+        .json(Seq("""{"nested": {"attribute": 1}, "value": 2}""").toDS())
+       .createOrReplaceTempView("rows")
 
-    checkAnswer(
-      sql(
-        """
-          |select attribute, sum(cnt)
-          |from (
-          |  select nested.attribute, count(*) as cnt
-          |  from rows
-          |  group by nested.attribute) a
-          |group by attribute
+      checkAnswer(
+        sql(
+          """
+            |select attribute, sum(cnt)
+            |from (
+            |  select nested.attribute, count(*) as cnt
+            |  from rows
+            |  group by nested.attribute) a
+            |group by attribute
         """.stripMargin),
-      Row(1, 1) :: Nil)
+        Row(1, 1) :: Nil)
+    }
   }
 
   test("SPARK-6201 IN type conversion") {
-    spark.read
-      .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}").toDS())
-      .createOrReplaceTempView("d")
+    withTempView("d") {
+      spark.read
+        .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}").toDS())
+        .createOrReplaceTempView("d")
 
-    checkAnswer(
-      sql("select * from d where d.a in (1,2)"),
-      Seq(Row("1"), Row("2")))
+      checkAnswer(
+        sql("select * from d where d.a in (1,2)"),
+        Seq(Row("1"), Row("2")))
+    }
   }
 
   test("SPARK-11226 Skip empty line in json file") {
-    spark.read
-      .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}", "").toDS())
-      .createOrReplaceTempView("d")
+    withTempView("d") {
+      spark.read
+        .json(Seq("{\"a\": \"1\"}}", "{\"a\": \"2\"}}", "{\"a\": \"3\"}}", "").toDS())
+        .createOrReplaceTempView("d")
 
-    checkAnswer(
-      sql("select count(1) from d"),
-      Seq(Row(3)))
+      checkAnswer(
+        sql("select count(1) from d"),
+        Seq(Row(3)))
+    }
   }
 
   test("SPARK-8828 sum should return null if all input values are null") {
@@ -419,40 +433,42 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-3173 Timestamp support in the parser") {
-    (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time").createOrReplaceTempView("timestamps")
+    withTempView("timestamps") {
+      (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time").createOrReplaceTempView("timestamps")
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.0'"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.0'"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00")))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time=CAST('1969-12-31 16:00:00.001' AS TIMESTAMP)"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time=CAST('1969-12-31 16:00:00.001' AS TIMESTAMP)"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.001'"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.001'"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE '1969-12-31 16:00:00.001'=time"),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE '1969-12-31 16:00:00.001'=time"),
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.001")))
 
-    checkAnswer(sql(
-      """SELECT time FROM timestamps WHERE time<'1969-12-31 16:00:00.003'
+      checkAnswer(sql(
+        """SELECT time FROM timestamps WHERE time<'1969-12-31 16:00:00.003'
           AND time>'1969-12-31 16:00:00.001'"""),
-      Row(Timestamp.valueOf("1969-12-31 16:00:00.002")))
+        Row(Timestamp.valueOf("1969-12-31 16:00:00.002")))
 
-    checkAnswer(sql(
-      """
-        |SELECT time FROM timestamps
-        |WHERE time IN ('1969-12-31 16:00:00.001','1969-12-31 16:00:00.002')
+      checkAnswer(sql(
+        """
+          |SELECT time FROM timestamps
+          |WHERE time IN ('1969-12-31 16:00:00.001','1969-12-31 16:00:00.002')
       """.stripMargin),
-      Seq(Row(Timestamp.valueOf("1969-12-31 16:00:00.001")),
-        Row(Timestamp.valueOf("1969-12-31 16:00:00.002"))))
+        Seq(Row(Timestamp.valueOf("1969-12-31 16:00:00.001")),
+          Row(Timestamp.valueOf("1969-12-31 16:00:00.002"))))
 
-    checkAnswer(sql(
-      "SELECT time FROM timestamps WHERE time='123'"),
-      Nil)
+      checkAnswer(sql(
+        "SELECT time FROM timestamps WHERE time='123'"),
+        Nil)
+    }
   }
 
   test("left semi greater than predicate") {
@@ -781,20 +797,22 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-3349 partitioning after limit") {
-    sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n DESC")
-      .limit(2)
-      .createOrReplaceTempView("subset1")
-    sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n ASC")
-      .limit(2)
-      .createOrReplaceTempView("subset2")
-    checkAnswer(
-      sql("SELECT * FROM lowerCaseData INNER JOIN subset1 ON subset1.n = lowerCaseData.n"),
-      Row(3, "c", 3) ::
-      Row(4, "d", 4) :: Nil)
-    checkAnswer(
-      sql("SELECT * FROM lowerCaseData INNER JOIN subset2 ON subset2.n = lowerCaseData.n"),
-      Row(1, "a", 1) ::
-      Row(2, "b", 2) :: Nil)
+    withTempView("subset1", "subset2") {
+      sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n DESC")
+        .limit(2)
+        .createOrReplaceTempView("subset1")
+      sql("SELECT DISTINCT n FROM lowerCaseData ORDER BY n ASC")
+        .limit(2)
+        .createOrReplaceTempView("subset2")
+      checkAnswer(
+        sql("SELECT * FROM lowerCaseData INNER JOIN subset1 ON subset1.n = lowerCaseData.n"),
+        Row(3, "c", 3) ::
+        Row(4, "d", 4) :: Nil)
+      checkAnswer(
+        sql("SELECT * FROM lowerCaseData INNER JOIN subset2 ON subset2.n = lowerCaseData.n"),
+        Row(1, "a", 1) ::
+        Row(2, "b", 2) :: Nil)
+    }
   }
 
   test("mixed-case keywords") {
@@ -1037,84 +1055,87 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("apply schema") {
-    val schema1 = StructType(
-      StructField("f1", IntegerType, false) ::
-      StructField("f2", StringType, false) ::
-      StructField("f3", BooleanType, false) ::
-      StructField("f4", IntegerType, true) :: Nil)
-
-    val rowRDD1 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+    withTempView("applySchema1", "applySchema2", "applySchema3") {
+      val schema1 = StructType(
+        StructField("f1", IntegerType, false) ::
+        StructField("f2", StringType, false) ::
+        StructField("f3", BooleanType, false) ::
+        StructField("f4", IntegerType, true) :: Nil)
+
+      val rowRDD1 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(values(0).toInt, values(1), values(2).toBoolean, v4)
       }
-      Row(values(0).toInt, values(1), values(2).toBoolean, v4)
-    }
-
-    val df1 = spark.createDataFrame(rowRDD1, schema1)
-    df1.createOrReplaceTempView("applySchema1")
-    checkAnswer(
-      sql("SELECT * FROM applySchema1"),
-      Row(1, "A1", true, null) ::
-      Row(2, "B2", false, null) ::
-      Row(3, "C3", true, null) ::
-      Row(4, "D4", true, 2147483644) :: Nil)
 
-    checkAnswer(
-      sql("SELECT f1, f4 FROM applySchema1"),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, null) ::
-      Row(4, 2147483644) :: Nil)
-
-    val schema2 = StructType(
-      StructField("f1", StructType(
-        StructField("f11", IntegerType, false) ::
-        StructField("f12", BooleanType, false) :: Nil), false) ::
-      StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
+      val df1 = spark.createDataFrame(rowRDD1, schema1)
+      df1.createOrReplaceTempView("applySchema1")
+      checkAnswer(
+        sql("SELECT * FROM applySchema1"),
+        Row(1, "A1", true, null) ::
+        Row(2, "B2", false, null) ::
+        Row(3, "C3", true, null) ::
+        Row(4, "D4", true, 2147483644) :: Nil)
 
-    val rowRDD2 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+      checkAnswer(
+        sql("SELECT f1, f4 FROM applySchema1"),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, null) ::
+        Row(4, 2147483644) :: Nil)
+
+      val schema2 = StructType(
+        StructField("f1", StructType(
+          StructField("f11", IntegerType, false) ::
+          StructField("f12", BooleanType, false) :: Nil), false) ::
+        StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
+
+      val rowRDD2 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
       }
-      Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
-    }
-
-    val df2 = spark.createDataFrame(rowRDD2, schema2)
-    df2.createOrReplaceTempView("applySchema2")
-    checkAnswer(
-      sql("SELECT * FROM applySchema2"),
-      Row(Row(1, true), Map("A1" -> null)) ::
-      Row(Row(2, false), Map("B2" -> null)) ::
-      Row(Row(3, true), Map("C3" -> null)) ::
-      Row(Row(4, true), Map("D4" -> 2147483644)) :: Nil)
 
-    checkAnswer(
-      sql("SELECT f1.f11, f2['D4'] FROM applySchema2"),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, null) ::
-      Row(4, 2147483644) :: Nil)
+      val df2 = spark.createDataFrame(rowRDD2, schema2)
+      df2.createOrReplaceTempView("applySchema2")
+      checkAnswer(
+        sql("SELECT * FROM applySchema2"),
+        Row(Row(1, true), Map("A1" -> null)) ::
+        Row(Row(2, false), Map("B2" -> null)) ::
+        Row(Row(3, true), Map("C3" -> null)) ::
+        Row(Row(4, true), Map("D4" -> 2147483644)) :: Nil)
 
-    // The value of a MapType column can be a mutable map.
-    val rowRDD3 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+      checkAnswer(
+        sql("SELECT f1.f11, f2['D4'] FROM applySchema2"),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, null) ::
+        Row(4, 2147483644) :: Nil)
+
+      // The value of a MapType column can be a mutable map.
+      val rowRDD3 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(Row(values(0).toInt, values(2).toBoolean),
+          scala.collection.mutable.Map(values(1) -> v4))
       }
-      Row(Row(values(0).toInt, values(2).toBoolean), scala.collection.mutable.Map(values(1) -> v4))
-    }
 
-    val df3 = spark.createDataFrame(rowRDD3, schema2)
-    df3.createOrReplaceTempView("applySchema3")
+      val df3 = spark.createDataFrame(rowRDD3, schema2)
+      df3.createOrReplaceTempView("applySchema3")
 
-    checkAnswer(
-      sql("SELECT f1.f11, f2['D4'] FROM applySchema3"),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, null) ::
-      Row(4, 2147483644) :: Nil)
+      checkAnswer(
+        sql("SELECT f1.f11, f2['D4'] FROM applySchema3"),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, null) ::
+        Row(4, 2147483644) :: Nil)
+    }
   }
 
   test("SPARK-3423 BETWEEN") {
@@ -1166,28 +1187,30 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("metadata is propagated correctly") {
-    val person: DataFrame = sql("SELECT * FROM person")
-    val schema = person.schema
-    val docKey = "doc"
-    val docValue = "first name"
-    val metadata = new MetadataBuilder()
-      .putString(docKey, docValue)
-      .build()
-    val schemaWithMeta = new StructType(Array(
-      schema("id"), schema("name").copy(metadata = metadata), schema("age")))
-    val personWithMeta = spark.createDataFrame(person.rdd, schemaWithMeta)
-    def validateMetadata(rdd: DataFrame): Unit = {
-      assert(rdd.schema("name").metadata.getString(docKey) == docValue)
-    }
-    personWithMeta.createOrReplaceTempView("personWithMeta")
-    validateMetadata(personWithMeta.select($"name"))
-    validateMetadata(personWithMeta.select($"name"))
-    validateMetadata(personWithMeta.select($"id", $"name"))
-    validateMetadata(sql("SELECT * FROM personWithMeta"))
-    validateMetadata(sql("SELECT id, name FROM personWithMeta"))
-    validateMetadata(sql("SELECT * FROM personWithMeta JOIN salary ON id = personId"))
-    validateMetadata(sql(
-      "SELECT name, salary FROM personWithMeta JOIN salary ON id = personId"))
+    withTempView("personWithMeta") {
+      val person: DataFrame = sql("SELECT * FROM person")
+      val schema = person.schema
+      val docKey = "doc"
+      val docValue = "first name"
+      val metadata = new MetadataBuilder()
+        .putString(docKey, docValue)
+        .build()
+      val schemaWithMeta = new StructType(Array(
+        schema("id"), schema("name").copy(metadata = metadata), schema("age")))
+      val personWithMeta = spark.createDataFrame(person.rdd, schemaWithMeta)
+      def validateMetadata(rdd: DataFrame): Unit = {
+        assert(rdd.schema("name").metadata.getString(docKey) == docValue)
+      }
+      personWithMeta.createOrReplaceTempView("personWithMeta")
+      validateMetadata(personWithMeta.select($"name"))
+      validateMetadata(personWithMeta.select($"name"))
+      validateMetadata(personWithMeta.select($"id", $"name"))
+      validateMetadata(sql("SELECT * FROM personWithMeta"))
+      validateMetadata(sql("SELECT id, name FROM personWithMeta"))
+      validateMetadata(sql("SELECT * FROM personWithMeta JOIN salary ON id = personId"))
+      validateMetadata(sql(
+        "SELECT name, salary FROM personWithMeta JOIN salary ON id = personId"))
+    }
   }
 
   test("SPARK-3371 Renaming a function expression with group by gives error") {
@@ -1229,10 +1252,12 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("SPARK-3483 Special chars in column names") {
-    val data = Seq("""{"key?number1": "value1", "key.number2": "value2"}""").toDS()
-    spark.read.json(data).createOrReplaceTempView("records")
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
-      sql("SELECT `key?number1`, `key.number2` FROM records")
+    withTempView("records") {
+      val data = Seq("""{"key?number1": "value1", "key.number2": "value2"}""").toDS()
+      spark.read.json(data).createOrReplaceTempView("records")
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        sql("SELECT `key?number1`, `key.number2` FROM records")
+      }
     }
   }
 
@@ -1299,138 +1324,152 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("Supporting relational operator '<=>' in Spark SQL") {
-    val nullCheckData1 = TestData(1, "1") :: TestData(2, null) :: Nil
-    val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i)))
-    rdd1.toDF().createOrReplaceTempView("nulldata1")
-    val nullCheckData2 = TestData(1, "1") :: TestData(2, null) :: Nil
-    val rdd2 = sparkContext.parallelize((0 to 1).map(i => nullCheckData2(i)))
-    rdd2.toDF().createOrReplaceTempView("nulldata2")
-    checkAnswer(sql("SELECT nulldata1.key FROM nulldata1 join " +
-      "nulldata2 on nulldata1.value <=> nulldata2.value"),
+    withTempView("nulldata1", "nulldata2") {
+      val nullCheckData1 = TestData(1, "1") :: TestData(2, null) :: Nil
+      val rdd1 = sparkContext.parallelize((0 to 1).map(i => nullCheckData1(i)))
+      rdd1.toDF().createOrReplaceTempView("nulldata1")
+      val nullCheckData2 = TestData(1, "1") :: TestData(2, null) :: Nil
+      val rdd2 = sparkContext.parallelize((0 to 1).map(i => nullCheckData2(i)))
+      rdd2.toDF().createOrReplaceTempView("nulldata2")
+      checkAnswer(sql("SELECT nulldata1.key FROM nulldata1 join " +
+        "nulldata2 on nulldata1.value <=> nulldata2.value"),
         (1 to 2).map(i => Row(i)))
+    }
   }
 
   test("Multi-column COUNT(DISTINCT ...)") {
-    val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
-    val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
-    rdd.toDF().createOrReplaceTempView("distinctData")
-    checkAnswer(sql("SELECT COUNT(DISTINCT key,value) FROM distinctData"), Row(2))
+    withTempView("distinctData") {
+      val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
+      val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
+      rdd.toDF().createOrReplaceTempView("distinctData")
+      checkAnswer(sql("SELECT COUNT(DISTINCT key,value) FROM distinctData"), Row(2))
+    }
   }
 
   test("SPARK-4699 case sensitivity SQL query") {
-    withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
-      val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
-      val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
-      rdd.toDF().createOrReplaceTempView("testTable1")
-      checkAnswer(sql("SELECT VALUE FROM TESTTABLE1 where KEY = 1"), Row("val_1"))
+    withTempView("testTable1") {
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+        val data = TestData(1, "val_1") :: TestData(2, "val_2") :: Nil
+        val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
+        rdd.toDF().createOrReplaceTempView("testTable1")
+        checkAnswer(sql("SELECT VALUE FROM TESTTABLE1 where KEY = 1"), Row("val_1"))
+      }
     }
   }
 
   test("SPARK-6145: ORDER BY test for nested fields") {
-    spark.read
-      .json(Seq("""{"a": {"b": 1, "a": {"a": 1}}, "c": [{"d": 1}]}""").toDS())
-      .createOrReplaceTempView("nestedOrder")
+    withTempView("nestedOrder") {
+      spark.read
+        .json(Seq("""{"a": {"b": 1, "a": {"a": 1}}, "c": [{"d": 1}]}""").toDS())
+        .createOrReplaceTempView("nestedOrder")
 
-    checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.b"), Row(1))
-    checkAnswer(sql("SELECT a.b FROM nestedOrder ORDER BY a.b"), Row(1))
-    checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.a.a"), Row(1))
-    checkAnswer(sql("SELECT a.a.a FROM nestedOrder ORDER BY a.a.a"), Row(1))
-    checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY c[0].d"), Row(1))
-    checkAnswer(sql("SELECT c[0].d FROM nestedOrder ORDER BY c[0].d"), Row(1))
+      checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.b"), Row(1))
+      checkAnswer(sql("SELECT a.b FROM nestedOrder ORDER BY a.b"), Row(1))
+      checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY a.a.a"), Row(1))
+      checkAnswer(sql("SELECT a.a.a FROM nestedOrder ORDER BY a.a.a"), Row(1))
+      checkAnswer(sql("SELECT 1 FROM nestedOrder ORDER BY c[0].d"), Row(1))
+      checkAnswer(sql("SELECT c[0].d FROM nestedOrder ORDER BY c[0].d"), Row(1))
+    }
   }
 
   test("SPARK-6145: special cases") {
-    spark.read
-      .json(Seq("""{"a": {"b": [1]}, "b": [{"a": 1}], "_c0": {"a": 1}}""").toDS())
-      .createOrReplaceTempView("t")
+    withTempView("t") {
+      spark.read
+        .json(Seq("""{"a": {"b": [1]}, "b": [{"a": 1}], "_c0": {"a": 1}}""").toDS())
+        .createOrReplaceTempView("t")
 
-    checkAnswer(sql("SELECT a.b[0] FROM t ORDER BY _c0.a"), Row(1))
-    checkAnswer(sql("SELECT b[0].a FROM t ORDER BY _c0.a"), Row(1))
+      checkAnswer(sql("SELECT a.b[0] FROM t ORDER BY _c0.a"), Row(1))
+      checkAnswer(sql("SELECT b[0].a FROM t ORDER BY _c0.a"), Row(1))
+    }
   }
 
   test("SPARK-6898: complete support for special chars in column names") {
-    spark.read
-      .json(Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS())
-      .createOrReplaceTempView("t")
+    withTempView("t") {
+      spark.read
+        .json(Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS())
+        .createOrReplaceTempView("t")
 
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
-      checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+      }
     }
   }
 
   test("SPARK-6583 order by aggregated function") {
-    Seq("1" -> 3, "1" -> 4, "2" -> 7, "2" -> 8, "3" -> 5, "3" -> 6, "4" -> 1, "4" -> 2)
-      .toDF("a", "b").createOrReplaceTempView("orderByData")
+    withTempView("orderByData") {
+      Seq("1" -> 3, "1" -> 4, "2" -> 7, "2" -> 8, "3" -> 5, "3" -> 6, "4" -> 1, "4" -> 2)
+        .toDF("a", "b").createOrReplaceTempView("orderByData")
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT a
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT a
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b)
         """.stripMargin),
-      Row("4") :: Row("1") :: Row("3") :: Row("2") :: Nil)
+        Row("4") :: Row("1") :: Row("3") :: Row("2") :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT sum(b)
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT sum(b)
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b)
         """.stripMargin),
-      Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
+        Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT sum(b)
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b), max(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT sum(b)
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b), max(b)
         """.stripMargin),
-      Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
+        Row(3) :: Row(7) :: Row(11) :: Row(15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT a, sum(b)
-          |FROM orderByData
-          |GROUP BY a
-          |ORDER BY sum(b)
+      checkAnswer(
+        sql(
+          """
+            |SELECT a, sum(b)
+            |FROM orderByData
+            |GROUP BY a
+            |ORDER BY sum(b)
         """.stripMargin),
-      Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
+        Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
+      checkAnswer(
+        sql(
+          """
             |SELECT a, sum(b)
             |FROM orderByData
             |GROUP BY a
             |ORDER BY sum(b) + 1
           """.stripMargin),
-      Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
+        Row("4", 3) :: Row("1", 7) :: Row("3", 11) :: Row("2", 15) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
+      checkAnswer(
+        sql(
+          """
             |SELECT count(*)
             |FROM orderByData
             |GROUP BY a
             |ORDER BY count(*)
           """.stripMargin),
-      Row(2) :: Row(2) :: Row(2) :: Row(2) :: Nil)
+        Row(2) :: Row(2) :: Row(2) :: Row(2) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
+      checkAnswer(
+        sql(
+          """
             |SELECT a
             |FROM orderByData
             |GROUP BY a
             |ORDER BY a, count(*), sum(b)
           """.stripMargin),
-      Row("1") :: Row("2") :: Row("3") :: Row("4") :: Nil)
+        Row("1") :: Row("2") :: Row("3") :: Row("4") :: Nil)
+    }
   }
 
   test("SPARK-7952: fix the equality check between boolean and numeric types") {
@@ -1742,137 +1781,141 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("Struct Star Expansion") {
-    val structDf = testData2.select("a", "b").as("record")
+    withTempView("structTable", "nestedStructTable", "specialCharacterTable", "nameConflict") {
+      val structDf = testData2.select("a", "b").as("record")
 
-    checkAnswer(
-      structDf.select($"record.a", $"record.b"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
+      checkAnswer(
+        structDf.select($"record.a", $"record.b"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    checkAnswer(
-      structDf.select($"record.*"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
+      checkAnswer(
+        structDf.select($"record.*"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    checkAnswer(
-      structDf.select($"record.*", $"record.*"),
-      Row(1, 1, 1, 1) :: Row(1, 2, 1, 2) :: Row(2, 1, 2, 1) :: Row(2, 2, 2, 2) ::
-        Row(3, 1, 3, 1) :: Row(3, 2, 3, 2) :: Nil)
+      checkAnswer(
+        structDf.select($"record.*", $"record.*"),
+        Row(1, 1, 1, 1) :: Row(1, 2, 1, 2) :: Row(2, 1, 2, 1) :: Row(2, 2, 2, 2) ::
+          Row(3, 1, 3, 1) :: Row(3, 2, 3, 2) :: Nil)
 
-    checkAnswer(
-      sql("select struct(a, b) as r1, struct(b, a) as r2 from testData2").select($"r1.*", $"r2.*"),
-      Row(1, 1, 1, 1) :: Row(1, 2, 2, 1) :: Row(2, 1, 1, 2) :: Row(2, 2, 2, 2) ::
-        Row(3, 1, 1, 3) :: Row(3, 2, 2, 3) :: Nil)
+      checkAnswer(
+        sql("select struct(a, b) as r1, struct(b, a) as r2 from testData2")
+          .select($"r1.*", $"r2.*"),
+        Row(1, 1, 1, 1) :: Row(1, 2, 2, 1) :: Row(2, 1, 1, 2) :: Row(2, 2, 2, 2) ::
+          Row(3, 1, 1, 3) :: Row(3, 2, 2, 3) :: Nil)
 
-    // Try with a temporary view
-    sql("select struct(a, b) as record from testData2").createOrReplaceTempView("structTable")
-    checkAnswer(
-      sql("SELECT record.* FROM structTable"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
+      // Try with a temporary view
+      sql("select struct(a, b) as record from testData2").createOrReplaceTempView("structTable")
+      checkAnswer(
+        sql("SELECT record.* FROM structTable"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    checkAnswer(sql(
-      """
-        | SELECT min(struct(record.*)) FROM
-        |   (select struct(a,b) as record from testData2) tmp
+      checkAnswer(sql(
+        """
+          | SELECT min(struct(record.*)) FROM
+          |   (select struct(a,b) as record from testData2) tmp
       """.stripMargin),
-      Row(Row(1, 1)) :: Nil)
+        Row(Row(1, 1)) :: Nil)
 
-    // Try with an alias on the select list
-    checkAnswer(sql(
-      """
-        | SELECT max(struct(record.*)) as r FROM
-        |   (select struct(a,b) as record from testData2) tmp
+      // Try with an alias on the select list
+      checkAnswer(sql(
+        """
+          | SELECT max(struct(record.*)) as r FROM
+          |   (select struct(a,b) as record from testData2) tmp
       """.stripMargin).select($"r.*"),
-      Row(3, 2) :: Nil)
+        Row(3, 2) :: Nil)
 
-    // With GROUP BY
-    checkAnswer(sql(
-      """
-        | SELECT min(struct(record.*)) FROM
-        |   (select a as a, struct(a,b) as record from testData2) tmp
-        | GROUP BY a
+      // With GROUP BY
+      checkAnswer(sql(
+        """
+          | SELECT min(struct(record.*)) FROM
+          |   (select a as a, struct(a,b) as record from testData2) tmp
+          | GROUP BY a
       """.stripMargin),
-      Row(Row(1, 1)) :: Row(Row(2, 1)) :: Row(Row(3, 1)) :: Nil)
+        Row(Row(1, 1)) :: Row(Row(2, 1)) :: Row(Row(3, 1)) :: Nil)
 
-    // With GROUP BY and alias
-    checkAnswer(sql(
-      """
-        | SELECT max(struct(record.*)) as r FROM
-        |   (select a as a, struct(a,b) as record from testData2) tmp
-        | GROUP BY a
+      // With GROUP BY and alias
+      checkAnswer(sql(
+        """
+          | SELECT max(struct(record.*)) as r FROM
+          |   (select a as a, struct(a,b) as record from testData2) tmp
+          | GROUP BY a
       """.stripMargin).select($"r.*"),
-      Row(1, 2) :: Row(2, 2) :: Row(3, 2) :: Nil)
+        Row(1, 2) :: Row(2, 2) :: Row(3, 2) :: Nil)
 
-    // With GROUP BY and alias and additional fields in the struct
-    checkAnswer(sql(
-      """
-        | SELECT max(struct(a, record.*, b)) as r FROM
-        |   (select a as a, b as b, struct(a,b) as record from testData2) tmp
-        | GROUP BY a
+      // With GROUP BY and alias and additional fields in the struct
+      checkAnswer(sql(
+        """
+          | SELECT max(struct(a, record.*, b)) as r FROM
+          |   (select a as a, b as b, struct(a,b) as record from testData2) tmp
+          | GROUP BY a
       """.stripMargin).select($"r.*"),
-      Row(1, 1, 2, 2) :: Row(2, 2, 2, 2) :: Row(3, 3, 2, 2) :: Nil)
+        Row(1, 1, 2, 2) :: Row(2, 2, 2, 2) :: Row(3, 3, 2, 2) :: Nil)
 
-    // Create a data set that contains nested structs.
-    val nestedStructData = sql(
-      """
-        | SELECT struct(r1, r2) as record FROM
-        |   (SELECT struct(a, b) as r1, struct(b, a) as r2 FROM testData2) tmp
+      // Create a data set that contains nested structs.
+      val nestedStructData = sql(
+        """
+          | SELECT struct(r1, r2) as record FROM
+          |   (SELECT struct(a, b) as r1, struct(b, a) as r2 FROM testData2) tmp
       """.stripMargin)
 
-    checkAnswer(nestedStructData.select($"record.*"),
-      Row(Row(1, 1), Row(1, 1)) :: Row(Row(1, 2), Row(2, 1)) :: Row(Row(2, 1), Row(1, 2)) ::
-        Row(Row(2, 2), Row(2, 2)) :: Row(Row(3, 1), Row(1, 3)) :: Row(Row(3, 2), Row(2, 3)) :: Nil)
-    checkAnswer(nestedStructData.select($"record.r1"),
-      Row(Row(1, 1)) :: Row(Row(1, 2)) :: Row(Row(2, 1)) :: Row(Row(2, 2)) ::
-        Row(Row(3, 1)) :: Row(Row(3, 2)) :: Nil)
-    checkAnswer(
-      nestedStructData.select($"record.r1.*"),
-      Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
-
-    // Try with a temporary view
-    withTempView("nestedStructTable") {
-      nestedStructData.createOrReplaceTempView("nestedStructTable")
+      checkAnswer(nestedStructData.select($"record.*"),
+        Row(Row(1, 1), Row(1, 1)) :: Row(Row(1, 2), Row(2, 1)) :: Row(Row(2, 1), Row(1, 2)) ::
+          Row(Row(2, 2), Row(2, 2)) :: Row(Row(3, 1), Row(1, 3)) :: Row(Row(3, 2), Row(2, 3)) ::
+          Nil)
+      checkAnswer(nestedStructData.select($"record.r1"),
+        Row(Row(1, 1)) :: Row(Row(1, 2)) :: Row(Row(2, 1)) :: Row(Row(2, 2)) ::
+          Row(Row(3, 1)) :: Row(Row(3, 2)) :: Nil)
       checkAnswer(
-        sql("SELECT record.* FROM nestedStructTable"),
-        nestedStructData.select($"record.*"))
-      checkAnswer(
-        sql("SELECT record.r1 FROM nestedStructTable"),
-        nestedStructData.select($"record.r1"))
-      checkAnswer(
-        sql("SELECT record.r1.* FROM nestedStructTable"),
-        nestedStructData.select($"record.r1.*"))
-
-      // Try resolving something not there.
-      assert(intercept[AnalysisException](sql("SELECT abc.* FROM nestedStructTable"))
-        .getMessage.contains("cannot resolve"))
-    }
+        nestedStructData.select($"record.r1.*"),
+        Row(1, 1) :: Row(1, 2) :: Row(2, 1) :: Row(2, 2) :: Row(3, 1) :: Row(3, 2) :: Nil)
 
-    // Create paths with unusual characters
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
-      val specialCharacterPath = sql(
-        """
-        | SELECT struct(`col$.a_`, `a.b.c.`) as `r&&b.c` FROM
-        |   (SELECT struct(a, b) as `col$.a_`, struct(b, a) as `a.b.c.` FROM testData2) tmp
-      """.stripMargin)
-      withTempView("specialCharacterTable") {
-        specialCharacterPath.createOrReplaceTempView("specialCharacterTable")
+      // Try with a temporary view
+      withTempView("nestedStructTable") {
+        nestedStructData.createOrReplaceTempView("nestedStructTable")
         checkAnswer(
-          specialCharacterPath.select($"`r&&b.c`.*"),
+          sql("SELECT record.* FROM nestedStructTable"),
           nestedStructData.select($"record.*"))
         checkAnswer(
-        sql(
-          "SELECT `r&&b.c`.`col$.a_` FROM specialCharacterTable"),
-        nestedStructData.select($"record.r1"))
-        checkAnswer(
-          sql("SELECT `r&&b.c`.`a.b.c.` FROM specialCharacterTable"),
-          nestedStructData.select($"record.r2"))
+          sql("SELECT record.r1 FROM nestedStructTable"),
+          nestedStructData.select($"record.r1"))
         checkAnswer(
-          sql("SELECT `r&&b.c`.`col$.a_`.* FROM specialCharacterTable"),
+          sql("SELECT record.r1.* FROM nestedStructTable"),
           nestedStructData.select($"record.r1.*"))
+
+        // Try resolving something not there.
+        assert(intercept[AnalysisException](sql("SELECT abc.* FROM nestedStructTable"))
+          .getMessage.contains("cannot resolve"))
       }
-    }
 
-    // Try star expanding a scalar. This should fail.
-    assert(intercept[AnalysisException](sql("select a.* from testData2")).getMessage.contains(
-      "Can only star expand struct data types."))
+      // Create paths with unusual characters
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        val specialCharacterPath = sql(
+          """
+          | SELECT struct(`col$.a_`, `a.b.c.`) as `r&&b.c` FROM
+          |   (SELECT struct(a, b) as `col$.a_`, struct(b, a) as `a.b.c.` FROM testData2) tmp
+      """.stripMargin)
+        withTempView("specialCharacterTable") {
+          specialCharacterPath.createOrReplaceTempView("specialCharacterTable")
+          checkAnswer(
+            specialCharacterPath.select($"`r&&b.c`.*"),
+            nestedStructData.select($"record.*"))
+          checkAnswer(
+          sql(
+            "SELECT `r&&b.c`.`col$.a_` FROM specialCharacterTable"),
+          nestedStructData.select($"record.r1"))
+          checkAnswer(
+            sql("SELECT `r&&b.c`.`a.b.c.` FROM specialCharacterTable"),
+            nestedStructData.select($"record.r2"))
+          checkAnswer(
+            sql("SELECT `r&&b.c`.`col$.a_`.* FROM specialCharacterTable"),
+            nestedStructData.select($"record.r1.*"))
+        }
+      }
+
+      // Try star expanding a scalar. This should fail.
+      assert(intercept[AnalysisException](sql("select a.* from testData2")).getMessage.contains(
+        "Can only star expand struct data types."))
+    }
   }
 
   test("Struct Star Expansion - Name conflict") {
@@ -3129,38 +3172,40 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("string date comparison") {
-    spark.range(1).selectExpr("date '2000-01-01' as d").createOrReplaceTempView("t1")
-    val result = Date.valueOf("2000-01-01")
-    checkAnswer(sql("select * from t1 where d < '2000'"), Nil)
-    checkAnswer(sql("select * from t1 where d < '2001'"), Row(result))
-    checkAnswer(sql("select * from t1 where d < '2000-01'"), Nil)
-    checkAnswer(sql("select * from t1 where d < '2000-01-01'"), Nil)
-    checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Nil)
-    checkAnswer(sql("select * from t1 where d <= '2000-1-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d <= '1999-12-30'"), Nil)
-    checkAnswer(sql("select * from t1 where d = '2000-1-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d = '2000-01-01'"), Row(result))
-    checkAnswer(sql("select * from t1 where d = '2000-1-02'"), Nil)
-    checkAnswer(sql("select * from t1 where d > '2000-01-01'"), Nil)
-    checkAnswer(sql("select * from t1 where d > '1999'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-1-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-1-01'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-01-1'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-01-01'"), Row(result))
-    checkAnswer(sql("select * from t1 where d >= '2000-01-02'"), Nil)
-    checkAnswer(sql("select * from t1 where '2000' >= d"), Row(result))
-    checkAnswer(sql("select * from t1 where d > '2000-13'"), Nil)
-
-    withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+    withTempView("t1") {
+      spark.range(1).selectExpr("date '2000-01-01' as d").createOrReplaceTempView("t1")
+      val result = Date.valueOf("2000-01-01")
       checkAnswer(sql("select * from t1 where d < '2000'"), Nil)
       checkAnswer(sql("select * from t1 where d < '2001'"), Row(result))
-      checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Row(result))
-      checkAnswer(sql("select * from t1 where d <= '1999'"), Nil)
+      checkAnswer(sql("select * from t1 where d < '2000-01'"), Nil)
+      checkAnswer(sql("select * from t1 where d < '2000-01-01'"), Nil)
+      checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Nil)
+      checkAnswer(sql("select * from t1 where d <= '2000-1-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d <= '1999-12-30'"), Nil)
+      checkAnswer(sql("select * from t1 where d = '2000-1-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d = '2000-01-01'"), Row(result))
+      checkAnswer(sql("select * from t1 where d = '2000-1-02'"), Nil)
+      checkAnswer(sql("select * from t1 where d > '2000-01-01'"), Nil)
+      checkAnswer(sql("select * from t1 where d > '1999'"), Row(result))
       checkAnswer(sql("select * from t1 where d >= '2000'"), Row(result))
-      checkAnswer(sql("select * from t1 where d > '1999-13'"), Row(result))
-      checkAnswer(sql("select to_date('2000-01-01') > '1'"), Row(true))
+      checkAnswer(sql("select * from t1 where d >= '2000-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-1-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-1-01'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-01-1'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-01-01'"), Row(result))
+      checkAnswer(sql("select * from t1 where d >= '2000-01-02'"), Nil)
+      checkAnswer(sql("select * from t1 where '2000' >= d"), Row(result))
+      checkAnswer(sql("select * from t1 where d > '2000-13'"), Nil)
+
+      withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+        checkAnswer(sql("select * from t1 where d < '2000'"), Nil)
+        checkAnswer(sql("select * from t1 where d < '2001'"), Row(result))
+        checkAnswer(sql("select * from t1 where d < '2000-1-1'"), Row(result))
+        checkAnswer(sql("select * from t1 where d <= '1999'"), Nil)
+        checkAnswer(sql("select * from t1 where d >= '2000'"), Row(result))
+        checkAnswer(sql("select * from t1 where d > '1999-13'"), Row(result))
+        checkAnswer(sql("select to_date('2000-01-01') > '1'"), Row(true))
+      }
     }
   }
 
@@ -3210,28 +3255,30 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   test("SPARK-28156: self-join should not miss cached view") {
     withTable("table1") {
       withView("table1_vw") {
-        val df = Seq.tabulate(5) { x => (x, x + 1, x + 2, x + 3) }.toDF("a", "b", "c", "d")
-        df.write.mode("overwrite").format("orc").saveAsTable("table1")
-        sql("drop view if exists table1_vw")
-        sql("create view table1_vw as select * from table1")
-
-        val cachedView = sql("select a, b, c, d from table1_vw")
-
-        cachedView.createOrReplaceTempView("cachedview")
-        cachedView.persist()
-
-        val queryDf = sql(
-          s"""select leftside.a, leftside.b
-             |from cachedview leftside
-             |join cachedview rightside
-             |on leftside.a = rightside.a
+        withTempView("cachedview") {
+          val df = Seq.tabulate(5) { x => (x, x + 1, x + 2, x + 3) }.toDF("a", "b", "c", "d")
+          df.write.mode("overwrite").format("orc").saveAsTable("table1")
+          sql("drop view if exists table1_vw")
+          sql("create view table1_vw as select * from table1")
+
+          val cachedView = sql("select a, b, c, d from table1_vw")
+
+          cachedView.createOrReplaceTempView("cachedview")
+          cachedView.persist()
+
+          val queryDf = sql(
+            s"""select leftside.a, leftside.b
+               |from cachedview leftside
+               |join cachedview rightside
+               |on leftside.a = rightside.a
            """.stripMargin)
 
-        val inMemoryTableScan = collect(queryDf.queryExecution.executedPlan) {
-          case i: InMemoryTableScanExec => i
+          val inMemoryTableScan = collect(queryDf.queryExecution.executedPlan) {
+            case i: InMemoryTableScanExec => i
+          }
+          assert(inMemoryTableScan.size == 2)
+          checkAnswer(queryDf, Row(0, 1) :: Row(1, 2) :: Row(2, 3) :: Row(3, 4) :: Row(4, 5) :: Nil)
         }
-        assert(inMemoryTableScan.size == 2)
-        checkAnswer(queryDf, Row(0, 1) :: Row(1, 2) :: Row(2, 3) :: Row(3, 4) :: Row(4, 5) :: Nil)
       }
     }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
index d51a4618af519..66c6fbeabbf55 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ScalaReflectionRelationSuite.scala
@@ -83,69 +83,79 @@ class ScalaReflectionRelationSuite extends SparkFunSuite with SharedSparkSession
   private case class InvalidInJava2(`0`: Int)
 
   test("query case class RDD") {
-    val data = ReflectData("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
-      new java.math.BigDecimal(1), Date.valueOf("1970-01-01"), new Timestamp(12345), Seq(1, 2, 3),
-      new java.math.BigInteger("1"), scala.math.BigInt(1))
-    Seq(data).toDF().createOrReplaceTempView("reflectData")
-
-    assert(sql("SELECT * FROM reflectData").collect().head ===
-      Row("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
-        new java.math.BigDecimal(1), Date.valueOf("1970-01-01"),
-        new Timestamp(12345), Seq(1, 2, 3), new java.math.BigDecimal(1),
-        new java.math.BigDecimal(1)))
+    withTempView("reflectData") {
+      val data = ReflectData("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
+        new java.math.BigDecimal(1), Date.valueOf("1970-01-01"), new Timestamp(12345), Seq(1, 2, 3),
+        new java.math.BigInteger("1"), scala.math.BigInt(1))
+      Seq(data).toDF().createOrReplaceTempView("reflectData")
+
+      assert(sql("SELECT * FROM reflectData").collect().head ===
+        Row("a", 1, 1L, 1.toFloat, 1.toDouble, 1.toShort, 1.toByte, true,
+          new java.math.BigDecimal(1), Date.valueOf("1970-01-01"),
+          new Timestamp(12345), Seq(1, 2, 3), new java.math.BigDecimal(1),
+          new java.math.BigDecimal(1)))
+    }
   }
 
   test("query case class RDD with nulls") {
-    val data = NullReflectData(null, null, null, null, null, null, null)
-    Seq(data).toDF().createOrReplaceTempView("reflectNullData")
+    withTempView("reflectNullData") {
+      val data = NullReflectData(null, null, null, null, null, null, null)
+      Seq(data).toDF().createOrReplaceTempView("reflectNullData")
 
-    assert(sql("SELECT * FROM reflectNullData").collect().head ===
-      Row.fromSeq(Seq.fill(7)(null)))
+      assert(sql("SELECT * FROM reflectNullData").collect().head ===
+        Row.fromSeq(Seq.fill(7)(null)))
+    }
   }
 
   test("query case class RDD with Nones") {
-    val data = OptionalReflectData(None, None, None, None, None, None, None)
-    Seq(data).toDF().createOrReplaceTempView("reflectOptionalData")
+    withTempView("reflectOptionalData") {
+      val data = OptionalReflectData(None, None, None, None, None, None, None)
+      Seq(data).toDF().createOrReplaceTempView("reflectOptionalData")
 
-    assert(sql("SELECT * FROM reflectOptionalData").collect().head ===
-      Row.fromSeq(Seq.fill(7)(null)))
+      assert(sql("SELECT * FROM reflectOptionalData").collect().head ===
+        Row.fromSeq(Seq.fill(7)(null)))
+    }
   }
 
   // Equality is broken for Arrays, so we test that separately.
   test("query binary data") {
-    Seq(ReflectBinary(Array[Byte](1))).toDF().createOrReplaceTempView("reflectBinary")
+    withTempView("reflectBinary") {
+      Seq(ReflectBinary(Array[Byte](1))).toDF().createOrReplaceTempView("reflectBinary")
 
-    val result = sql("SELECT data FROM reflectBinary")
-      .collect().head(0).asInstanceOf[Array[Byte]]
-    assert(result.toSeq === Seq[Byte](1))
+      val result = sql("SELECT data FROM reflectBinary")
+        .collect().head(0).asInstanceOf[Array[Byte]]
+      assert(result.toSeq === Seq[Byte](1))
+    }
   }
 
   test("query complex data") {
-    val data = ComplexReflectData(
-      Seq(1, 2, 3),
-      Seq(Some(1), Some(2), None),
-      Map(1 -> 10L, 2 -> 20L),
-      Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None),
-      Data(
-        Seq(10, 20, 30),
-        Seq(Some(10), Some(20), None),
-        Map(10 -> 100L, 20 -> 200L),
-        Map(10 -> Some(100L), 20 -> Some(200L), 30 -> None),
-        Nested(None, "abc")))
-
-    Seq(data).toDF().createOrReplaceTempView("reflectComplexData")
-    assert(sql("SELECT * FROM reflectComplexData").collect().head ===
-      Row(
+    withTempView("reflectComplexData") {
+      val data = ComplexReflectData(
         Seq(1, 2, 3),
-        Seq(1, 2, null),
+        Seq(Some(1), Some(2), None),
         Map(1 -> 10L, 2 -> 20L),
-        Map(1 -> 10L, 2 -> 20L, 3 -> null),
-        Row(
+        Map(1 -> Some(10L), 2 -> Some(20L), 3 -> None),
+        Data(
           Seq(10, 20, 30),
-          Seq(10, 20, null),
+          Seq(Some(10), Some(20), None),
           Map(10 -> 100L, 20 -> 200L),
-          Map(10 -> 100L, 20 -> 200L, 30 -> null),
-          Row(null, "abc"))))
+          Map(10 -> Some(100L), 20 -> Some(200L), 30 -> None),
+          Nested(None, "abc")))
+
+      Seq(data).toDF().createOrReplaceTempView("reflectComplexData")
+      assert(sql("SELECT * FROM reflectComplexData").collect().head ===
+        Row(
+          Seq(1, 2, 3),
+          Seq(1, 2, null),
+          Map(1 -> 10L, 2 -> 20L),
+          Map(1 -> 10L, 2 -> 20L, 3 -> null),
+          Row(
+            Seq(10, 20, 30),
+            Seq(10, 20, null),
+            Map(10 -> 100L, 20 -> 200L),
+            Map(10 -> 100L, 20 -> 200L, 30 -> null),
+            Row(null, "abc"))))
+    }
   }
 
   test("better error message when use java reserved keyword as field name") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index 760b03d5e25f8..141f1279a6ee5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -153,29 +153,31 @@ class SubquerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   }
 
   test("uncorrelated scalar subquery on a DataFrame generated query") {
-    val df = Seq((1, "one"), (2, "two"), (3, "three")).toDF("key", "value")
-    df.createOrReplaceTempView("subqueryData")
+    withTempView("subqueryData") {
+      val df = Seq((1, "one"), (2, "two"), (3, "three")).toDF("key", "value")
+      df.createOrReplaceTempView("subqueryData")
 
-    checkAnswer(
-      sql("select (select key from subqueryData where key > 2 order by key limit 1) + 1"),
-      Array(Row(4))
-    )
+      checkAnswer(
+        sql("select (select key from subqueryData where key > 2 order by key limit 1) + 1"),
+        Array(Row(4))
+      )
 
-    checkAnswer(
-      sql("select -(select max(key) from subqueryData)"),
-      Array(Row(-3))
-    )
+      checkAnswer(
+        sql("select -(select max(key) from subqueryData)"),
+        Array(Row(-3))
+      )
 
-    checkAnswer(
-      sql("select (select value from subqueryData limit 0)"),
-      Array(Row(null))
-    )
+      checkAnswer(
+        sql("select (select value from subqueryData limit 0)"),
+        Array(Row(null))
+      )
 
-    checkAnswer(
-      sql("select (select min(value) from subqueryData" +
-        " where key = (select max(key) from subqueryData) - 1)"),
-      Array(Row("two"))
-    )
+      checkAnswer(
+        sql("select (select min(value) from subqueryData" +
+          " where key = (select max(key) from subqueryData) - 1)"),
+        Array(Row("two"))
+      )
+    }
   }
 
   test("SPARK-15677: Queries against local relations with scalar subquery in Select list") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index 157610f5d8a9e..ed8ab1cb3a603 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -150,12 +150,14 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
   }
 
   test("UDTs and UDFs") {
-    spark.udf.register("testType",
-      (d: TestUDT.MyDenseVector) => d.isInstanceOf[TestUDT.MyDenseVector])
-    pointsRDD.createOrReplaceTempView("points")
-    checkAnswer(
-      sql("SELECT testType(features) from points"),
-      Seq(Row(true), Row(true)))
+    withTempView("points") {
+      spark.udf.register("testType",
+        (d: TestUDT.MyDenseVector) => d.isInstanceOf[TestUDT.MyDenseVector])
+      pointsRDD.createOrReplaceTempView("points")
+      checkAnswer(
+        sql("SELECT testType(features) from points"),
+        Seq(Row(true), Row(true)))
+    }
   }
 
   testStandardAndLegacyModes("UDTs with Parquet") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
index 7aabf4d039f08..67ec1028f1998 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLWindowFunctionSuite.scala
@@ -41,67 +41,70 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product), sum(sum(product)) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 5, 11),
-        ("a", 6, 11),
-        ("b", 7, 15),
-        ("b", 8, 15),
-        ("c", 9, 19),
-        ("c", 10, 19)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product) - 1, sum(sum(product)) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 4, 11),
-        ("a", 5, 11),
-        ("b", 6, 15),
-        ("b", 7, 15),
-        ("c", 8, 19),
-        ("c", 9, 19)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product), sum(product) / sum(sum(product)) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 5, 5d/11),
-        ("a", 6, 6d/11),
-        ("b", 7, 7d/15),
-        ("b", 8, 8d/15),
-        ("c", 10, 10d/19),
-        ("c", 9, 9d/19)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product), sum(product) / sum(sum(product) - 1) over (partition by area)
-          |from windowData group by month, area
-        """.stripMargin),
-      Seq(
-        ("a", 5, 5d/9),
-        ("a", 6, 6d/9),
-        ("b", 7, 7d/13),
-        ("b", 8, 8d/13),
-        ("c", 10, 10d/17),
-        ("c", 9, 9d/17)
-      ).map(i => Row(i._1, i._2, i._3)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product), sum(sum(product)) over (partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 5, 11),
+          ("a", 6, 11),
+          ("b", 7, 15),
+          ("b", 8, 15),
+          ("c", 9, 19),
+          ("c", 10, 19)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product) - 1, sum(sum(product)) over (partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 4, 11),
+          ("a", 5, 11),
+          ("b", 6, 15),
+          ("b", 7, 15),
+          ("c", 8, 19),
+          ("c", 9, 19)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product), sum(product) / sum(sum(product)) over (partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 5, 5d/11),
+          ("a", 6, 6d/11),
+          ("b", 7, 7d/15),
+          ("b", 8, 8d/15),
+          ("c", 10, 10d/19),
+          ("c", 9, 9d/19)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product), sum(product) / sum(sum(product) - 1) over
+            |(partition by area)
+            |from windowData group by month, area
+          """.stripMargin),
+        Seq(
+          ("a", 5, 5d/9),
+          ("a", 6, 6d/9),
+          ("b", 7, 7d/13),
+          ("b", 8, 8d/13),
+          ("c", 10, 10d/17),
+          ("c", 9, 9d/17)
+        ).map(i => Row(i._1, i._2, i._3)))
+    }
   }
 
   test("window function: refer column in inner select block") {
@@ -113,22 +116,24 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
-          |from (select month, area, product, 1 as tmp1 from windowData) tmp
-        """.stripMargin),
-      Seq(
-        ("a", 2),
-        ("a", 3),
-        ("b", 2),
-        ("b", 3),
-        ("c", 2),
-        ("c", 3)
-      ).map(i => Row(i._1, i._2)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
+            |from (select month, area, product, 1 as tmp1 from windowData) tmp
+          """.stripMargin),
+        Seq(
+          ("a", 2),
+          ("a", 3),
+          ("b", 2),
+          ("b", 3),
+          ("c", 2),
+          ("c", 3)
+        ).map(i => Row(i._1, i._2)))
+    }
   }
 
   test("window function: partition and order expressions") {
@@ -140,38 +145,40 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select month, area, product, sum(product + 1) over (partition by 1 order by 2)
-          |from windowData
-        """.stripMargin),
-      Seq(
-        (1, "a", 5, 51),
-        (2, "a", 6, 51),
-        (3, "b", 7, 51),
-        (4, "b", 8, 51),
-        (5, "c", 9, 51),
-        (6, "c", 10, 51)
-      ).map(i => Row(i._1, i._2, i._3, i._4)))
-
-    checkAnswer(
-      sql(
-        """
-          |select month, area, product, sum(product)
-          |over (partition by month % 2 order by 10 - product)
-          |from windowData
-        """.stripMargin),
-      Seq(
-        (1, "a", 5, 21),
-        (2, "a", 6, 24),
-        (3, "b", 7, 16),
-        (4, "b", 8, 18),
-        (5, "c", 9, 9),
-        (6, "c", 10, 10)
-      ).map(i => Row(i._1, i._2, i._3, i._4)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select month, area, product, sum(product + 1) over (partition by 1 order by 2)
+            |from windowData
+          """.stripMargin),
+        Seq(
+          (1, "a", 5, 51),
+          (2, "a", 6, 51),
+          (3, "b", 7, 51),
+          (4, "b", 8, 51),
+          (5, "c", 9, 51),
+          (6, "c", 10, 51)
+        ).map(i => Row(i._1, i._2, i._3, i._4)))
+
+      checkAnswer(
+        sql(
+          """
+            |select month, area, product, sum(product)
+            |over (partition by month % 2 order by 10 - product)
+            |from windowData
+          """.stripMargin),
+        Seq(
+          (1, "a", 5, 21),
+          (2, "a", 6, 24),
+          (3, "b", 7, 16),
+          (4, "b", 8, 18),
+          (5, "c", 9, 9),
+          (6, "c", 10, 10)
+        ).map(i => Row(i._1, i._2, i._3, i._4)))
+    }
   }
 
   test("window function: distinct should not be silently ignored") {
@@ -183,16 +190,18 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    val e = intercept[AnalysisException] {
-      sql(
-        """
-          |select month, area, product, sum(distinct product + 1) over (partition by 1 order by 2)
-          |from windowData
-        """.stripMargin)
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      val e = intercept[AnalysisException] {
+        sql(
+          """
+            |select month, area, product, sum(distinct product + 1) over (partition by 1 order by 2)
+            |from windowData
+          """.stripMargin)
+      }
+      assert(e.getMessage.contains("Distinct window functions are not supported"))
     }
-    assert(e.getMessage.contains("Distinct window functions are not supported"))
   }
 
   test("window function: expressions in arguments of a window functions") {
@@ -204,23 +213,25 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 10)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select month, area, month % 2,
-          |lag(product, 1 + 1, product) over (partition by month % 2 order by area)
-          |from windowData
-        """.stripMargin),
-      Seq(
-        (1, "a", 1, 5),
-        (2, "a", 0, 6),
-        (3, "b", 1, 7),
-        (4, "b", 0, 8),
-        (5, "c", 1, 5),
-        (6, "c", 0, 6)
-      ).map(i => Row(i._1, i._2, i._3, i._4)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select month, area, month % 2,
+            |lag(product, 1 + 1, product) over (partition by month % 2 order by area)
+            |from windowData
+          """.stripMargin),
+        Seq(
+          (1, "a", 1, 5),
+          (2, "a", 0, 6),
+          (3, "b", 1, 7),
+          (4, "b", 0, 8),
+          (5, "c", 1, 5),
+          (6, "c", 0, 6)
+        ).map(i => Row(i._1, i._2, i._3, i._4)))
+    }
   }
 
 
@@ -233,63 +244,65 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 11)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql("select month, product, sum(product + 1) over() from windowData order by area"),
-      Seq(
-        (2, 6, 57),
-        (3, 7, 57),
-        (4, 8, 57),
-        (5, 9, 57),
-        (6, 11, 57),
-        (1, 10, 57)
-      ).map(i => Row(i._1, i._2, i._3)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
-          |from (select month, area, product as p, 1 as tmp1 from windowData) tmp order by p
-        """.stripMargin),
-      Seq(
-        ("a", 2),
-        ("b", 2),
-        ("b", 3),
-        ("c", 2),
-        ("d", 2),
-        ("c", 3)
-      ).map(i => Row(i._1, i._2)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, rank() over (partition by area order by month) as c1
-          |from windowData group by product, area, month order by product, area
-        """.stripMargin),
-      Seq(
-        ("a", 1),
-        ("b", 1),
-        ("b", 2),
-        ("c", 1),
-        ("d", 1),
-        ("c", 2)
-      ).map(i => Row(i._1, i._2)))
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product) / sum(sum(product)) over (partition by area) as c1
-          |from windowData group by area, month order by month, c1
-        """.stripMargin),
-      Seq(
-        ("d", 1.0),
-        ("a", 1.0),
-        ("b", 0.4666666666666667),
-        ("b", 0.5333333333333333),
-        ("c", 0.45),
-        ("c", 0.55)
-      ).map(i => Row(i._1, i._2)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql("select month, product, sum(product + 1) over() from windowData order by area"),
+        Seq(
+          (2, 6, 57),
+          (3, 7, 57),
+          (4, 8, 57),
+          (5, 9, 57),
+          (6, 11, 57),
+          (1, 10, 57)
+        ).map(i => Row(i._1, i._2, i._3)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, rank() over (partition by area order by tmp.month) + tmp.tmp1 as c1
+            |from (select month, area, product as p, 1 as tmp1 from windowData) tmp order by p
+          """.stripMargin),
+        Seq(
+          ("a", 2),
+          ("b", 2),
+          ("b", 3),
+          ("c", 2),
+          ("d", 2),
+          ("c", 3)
+        ).map(i => Row(i._1, i._2)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, rank() over (partition by area order by month) as c1
+            |from windowData group by product, area, month order by product, area
+          """.stripMargin),
+        Seq(
+          ("a", 1),
+          ("b", 1),
+          ("b", 2),
+          ("c", 1),
+          ("d", 1),
+          ("c", 2)
+        ).map(i => Row(i._1, i._2)))
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product) / sum(sum(product)) over (partition by area) as c1
+            |from windowData group by area, month order by month, c1
+          """.stripMargin),
+        Seq(
+          ("d", 1.0),
+          ("a", 1.0),
+          ("b", 0.4666666666666667),
+          ("b", 0.5333333333333333),
+          ("c", 0.45),
+          ("c", 0.55)
+        ).map(i => Row(i._1, i._2)))
+    }
   }
 
   // todo: fix this test case by reimplementing the function ResolveAggregateFunctions
@@ -302,23 +315,25 @@ class SQLWindowFunctionSuite extends QueryTest with SharedSparkSession {
       WindowData(5, "c", 9),
       WindowData(6, "c", 11)
     )
-    sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
-
-    checkAnswer(
-      sql(
-        """
-          |select area, sum(product) over () as c from windowData
-          |where product > 3 group by area, product
-          |having avg(month) > 0 order by avg(month), product
-        """.stripMargin),
-      Seq(
-        ("a", 51),
-        ("b", 51),
-        ("b", 51),
-        ("c", 51),
-        ("c", 51),
-        ("d", 51)
-      ).map(i => Row(i._1, i._2)))
+    withTempView("windowData") {
+      sparkContext.parallelize(data).toDF().createOrReplaceTempView("windowData")
+
+      checkAnswer(
+        sql(
+          """
+            |select area, sum(product) over () as c from windowData
+            |where product > 3 group by area, product
+            |having avg(month) > 0 order by avg(month), product
+          """.stripMargin),
+        Seq(
+          ("a", 51),
+          ("b", 51),
+          ("b", 51),
+          ("c", 51),
+          ("c", 51),
+          ("d", 51)
+        ).map(i => Row(i._1, i._2)))
+    }
   }
 
   test("window function: multiple window expressions in a single expression") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
index 77047f329e105..18f29f7b90ad5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
@@ -126,13 +126,15 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   }
 
   test("default size avoids broadcast") {
-    // TODO: Improve this test when we have better statistics
-    sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
-      .toDF().createOrReplaceTempView("sizeTst")
-    spark.catalog.cacheTable("sizeTst")
-    assert(
-      spark.table("sizeTst").queryExecution.analyzed.stats.sizeInBytes >
-        spark.conf.get(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD))
+    withTempView("sizeTst") {
+      // TODO: Improve this test when we have better statistics
+      sparkContext.parallelize(1 to 10).map(i => TestData(i, i.toString))
+        .toDF().createOrReplaceTempView("sizeTst")
+      spark.catalog.cacheTable("sizeTst")
+      assert(
+        spark.table("sizeTst").queryExecution.analyzed.stats.sizeInBytes >
+          spark.conf.get(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD))
+    }
   }
 
   test("projection") {
@@ -187,18 +189,20 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
   }
 
   test("SPARK-2729 regression: timestamp data type") {
-    val timestamps = (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time")
-    timestamps.createOrReplaceTempView("timestamps")
+    withTempView("timestamps") {
+      val timestamps = (0 to 3).map(i => Tuple1(new Timestamp(i))).toDF("time")
+      timestamps.createOrReplaceTempView("timestamps")
 
-    checkAnswer(
-      sql("SELECT time FROM timestamps"),
-      timestamps.collect().toSeq)
+      checkAnswer(
+        sql("SELECT time FROM timestamps"),
+        timestamps.collect().toSeq)
 
-    spark.catalog.cacheTable("timestamps")
+      spark.catalog.cacheTable("timestamps")
 
-    checkAnswer(
-      sql("SELECT time FROM timestamps"),
-      timestamps.collect().toSeq)
+      checkAnswer(
+        sql("SELECT time FROM timestamps"),
+        timestamps.collect().toSeq)
+    }
   }
 
   test("SPARK-3320 regression: batched column buffer building should work with empty partitions") {
@@ -229,10 +233,12 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
 
     assert(df.schema.head.dataType === DecimalType(15, 10))
 
-    df.cache().createOrReplaceTempView("test_fixed_decimal")
-    checkAnswer(
-      sql("SELECT * FROM test_fixed_decimal"),
-      (1 to 10).map(i => Row(Decimal(i, 15, 10).toJavaBigDecimal)))
+    withTempView("test_fixed_decimal") {
+      df.cache().createOrReplaceTempView("test_fixed_decimal")
+      checkAnswer(
+        sql("SELECT * FROM test_fixed_decimal"),
+        (1 to 10).map(i => Row(Decimal(i, 15, 10).toJavaBigDecimal)))
+    }
   }
 
   test("test different data types") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index be95f336598f4..999eadbf40c4a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -249,495 +249,523 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("Complex field and type inferring with null in sampling") {
-    val jsonDF = spark.read.json(jsonNullStruct)
-    val expectedSchema = StructType(
-      StructField("headers", StructType(
-        StructField("Charset", StringType, true) ::
-          StructField("Host", StringType, true) :: Nil)
-        , true) ::
-        StructField("ip", StringType, true) ::
-        StructField("nullstr", StringType, true):: Nil)
-
-    assert(expectedSchema === jsonDF.schema)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(jsonNullStruct)
+      val expectedSchema = StructType(
+        StructField("headers", StructType(
+          StructField("Charset", StringType, true) ::
+            StructField("Host", StringType, true) :: Nil)
+          , true) ::
+          StructField("ip", StringType, true) ::
+          StructField("nullstr", StringType, true):: Nil)
+
+      assert(expectedSchema === jsonDF.schema)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select nullstr, headers.Host from jsonTable"),
-      Seq(Row("", "1.abc.com"), Row("", null), Row("", null), Row(null, null))
-    )
+      checkAnswer(
+        sql("select nullstr, headers.Host from jsonTable"),
+        Seq(Row("", "1.abc.com"), Row("", null), Row("", null), Row(null, null))
+      )
+    }
   }
 
   test("Primitive field and type inferring") {
-    val jsonDF = spark.read.json(primitiveFieldAndType)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(primitiveFieldAndType)
 
-    val expectedSchema = StructType(
-      StructField("bigInteger", DecimalType(20, 0), true) ::
-      StructField("boolean", BooleanType, true) ::
-      StructField("double", DoubleType, true) ::
-      StructField("integer", LongType, true) ::
-      StructField("long", LongType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("bigInteger", DecimalType(20, 0), true) ::
+        StructField("boolean", BooleanType, true) ::
+        StructField("double", DoubleType, true) ::
+        StructField("integer", LongType, true) ::
+        StructField("long", LongType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-        true,
-        1.7976931348623157,
-        10,
-        21474836470L,
-        null,
-        "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+          true,
+          1.7976931348623157,
+          10,
+          21474836470L,
+          null,
+          "this is a simple string.")
+      )
+    }
   }
 
   test("Complex field and type inferring") {
-    val jsonDF = spark.read.json(complexFieldAndType1)
-
-    val expectedSchema = StructType(
-      StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
-      StructField("arrayOfArray2", ArrayType(ArrayType(DoubleType, true), true), true) ::
-      StructField("arrayOfBigInteger", ArrayType(DecimalType(21, 0), true), true) ::
-      StructField("arrayOfBoolean", ArrayType(BooleanType, true), true) ::
-      StructField("arrayOfDouble", ArrayType(DoubleType, true), true) ::
-      StructField("arrayOfInteger", ArrayType(LongType, true), true) ::
-      StructField("arrayOfLong", ArrayType(LongType, true), true) ::
-      StructField("arrayOfNull", ArrayType(StringType, true), true) ::
-      StructField("arrayOfString", ArrayType(StringType, true), true) ::
-      StructField("arrayOfStruct", ArrayType(
-        StructType(
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType1)
+
+      val expectedSchema = StructType(
+        StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
+        StructField("arrayOfArray2", ArrayType(ArrayType(DoubleType, true), true), true) ::
+        StructField("arrayOfBigInteger", ArrayType(DecimalType(21, 0), true), true) ::
+        StructField("arrayOfBoolean", ArrayType(BooleanType, true), true) ::
+        StructField("arrayOfDouble", ArrayType(DoubleType, true), true) ::
+        StructField("arrayOfInteger", ArrayType(LongType, true), true) ::
+        StructField("arrayOfLong", ArrayType(LongType, true), true) ::
+        StructField("arrayOfNull", ArrayType(StringType, true), true) ::
+        StructField("arrayOfString", ArrayType(StringType, true), true) ::
+        StructField("arrayOfStruct", ArrayType(
+          StructType(
+            StructField("field1", BooleanType, true) ::
+            StructField("field2", StringType, true) ::
+            StructField("field3", StringType, true) :: Nil), true), true) ::
+        StructField("struct", StructType(
           StructField("field1", BooleanType, true) ::
-          StructField("field2", StringType, true) ::
-          StructField("field3", StringType, true) :: Nil), true), true) ::
-      StructField("struct", StructType(
-        StructField("field1", BooleanType, true) ::
-        StructField("field2", DecimalType(20, 0), true) :: Nil), true) ::
-      StructField("structWithArrayFields", StructType(
-        StructField("field1", ArrayType(LongType, true), true) ::
-        StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
+          StructField("field2", DecimalType(20, 0), true) :: Nil), true) ::
+        StructField("structWithArrayFields", StructType(
+          StructField("field1", ArrayType(LongType, true), true) ::
+          StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    // Access elements of a primitive array.
-    checkAnswer(
-      sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
-      Row("str1", "str2", null)
-    )
+      // Access elements of a primitive array.
+      checkAnswer(
+        sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
+        Row("str1", "str2", null)
+      )
 
-    // Access an array of null values.
-    checkAnswer(
-      sql("select arrayOfNull from jsonTable"),
-      Row(Seq(null, null, null, null))
-    )
+      // Access an array of null values.
+      checkAnswer(
+        sql("select arrayOfNull from jsonTable"),
+        Row(Seq(null, null, null, null))
+      )
 
-    // Access elements of a BigInteger array (we use DecimalType internally).
-    checkAnswer(
-      sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from jsonTable"),
-      Row(new java.math.BigDecimal("922337203685477580700"),
-        new java.math.BigDecimal("-922337203685477580800"), null)
-    )
+      // Access elements of a BigInteger array (we use DecimalType internally).
+      checkAnswer(
+        sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from " +
+          "jsonTable"),
+        Row(new java.math.BigDecimal("922337203685477580700"),
+          new java.math.BigDecimal("-922337203685477580800"), null)
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
-      Row(Seq("1", "2", "3"), Seq("str1", "str2"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
+        Row(Seq("1", "2", "3"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
-      Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
+        Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
+      )
 
-    // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
-    checkAnswer(
-      sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
-      Row("str2", 2.1)
-    )
+      // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
+      checkAnswer(
+        sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
+        Row("str2", 2.1)
+      )
 
-    // Access elements of an array of structs.
-    checkAnswer(
-      sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
-        "from jsonTable"),
-      Row(
-        Row(true, "str1", null),
-        Row(false, null, null),
-        Row(null, null, null),
-        null)
-    )
+      // Access elements of an array of structs.
+      checkAnswer(
+        sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
+          "from jsonTable"),
+        Row(
+          Row(true, "str1", null),
+          Row(false, null, null),
+          Row(null, null, null),
+          null)
+      )
 
-    // Access a struct and fields inside of it.
-    checkAnswer(
-      sql("select struct, struct.field1, struct.field2 from jsonTable"),
-      Row(
-        Row(true, new java.math.BigDecimal("92233720368547758070")),
-        true,
-        new java.math.BigDecimal("92233720368547758070")) :: Nil
-    )
+      // Access a struct and fields inside of it.
+      checkAnswer(
+        sql("select struct, struct.field1, struct.field2 from jsonTable"),
+        Row(
+          Row(true, new java.math.BigDecimal("92233720368547758070")),
+          true,
+          new java.math.BigDecimal("92233720368547758070")) :: Nil
+      )
 
-    // Access an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
-      Row(Seq(4, 5, 6), Seq("str1", "str2"))
-    )
+      // Access an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
+        Row(Seq(4, 5, 6), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from jsonTable"),
-      Row(5, null)
-    )
+      // Access elements of an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from " +
+          "jsonTable"),
+        Row(5, null)
+      )
+    }
   }
 
   test("GetField operation on complex data type") {
-    val jsonDF = spark.read.json(complexFieldAndType1)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType1)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
-      Row(true, "str1")
-    )
+      checkAnswer(
+        sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
+        Row(true, "str1")
+      )
 
-    // Getting all values of a specific field from an array of structs.
-    checkAnswer(
-      sql("select arrayOfStruct.field1, arrayOfStruct.field2 from jsonTable"),
-      Row(Seq(true, false, null), Seq("str1", null, null))
-    )
+      // Getting all values of a specific field from an array of structs.
+      checkAnswer(
+        sql("select arrayOfStruct.field1, arrayOfStruct.field2 from jsonTable"),
+        Row(Seq(true, false, null), Seq("str1", null, null))
+      )
+    }
   }
 
   test("Type conflict in primitive field values") {
-    val jsonDF = spark.read.json(primitiveFieldValueTypeConflict)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(primitiveFieldValueTypeConflict)
 
-    val expectedSchema = StructType(
-      StructField("num_bool", StringType, true) ::
-      StructField("num_num_1", LongType, true) ::
-      StructField("num_num_2", DoubleType, true) ::
-      StructField("num_num_3", DoubleType, true) ::
-      StructField("num_str", StringType, true) ::
-      StructField("str_bool", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("num_bool", StringType, true) ::
+        StructField("num_num_1", LongType, true) ::
+        StructField("num_num_2", DoubleType, true) ::
+        StructField("num_num_3", DoubleType, true) ::
+        StructField("num_str", StringType, true) ::
+        StructField("str_bool", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row("true", 11L, null, 1.1, "13.1", "str1") ::
-        Row("12", null, 21474836470.9, null, null, "true") ::
-        Row("false", 21474836470L, 92233720368547758070d, 100, "str1", "false") ::
-        Row(null, 21474836570L, 1.1, 21474836470L, "92233720368547758070", null) :: Nil
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row("true", 11L, null, 1.1, "13.1", "str1") ::
+          Row("12", null, 21474836470.9, null, null, "true") ::
+          Row("false", 21474836470L, 92233720368547758070d, 100, "str1", "false") ::
+          Row(null, 21474836570L, 1.1, 21474836470L, "92233720368547758070", null) :: Nil
+      )
 
-    // Number and Boolean conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_bool - 10 from jsonTable where num_bool > 11"),
-      Row(2)
-    )
+      // Number and Boolean conflict: resolve the type as number in this query.
+      checkAnswer(
+        sql("select num_bool - 10 from jsonTable where num_bool > 11"),
+        Row(2)
+      )
 
-    // Widening to LongType
-    checkAnswer(
-      sql("select num_num_1 - 100 from jsonTable where num_num_1 > 11"),
-      Row(21474836370L) :: Row(21474836470L) :: Nil
-    )
+      // Widening to LongType
+      checkAnswer(
+        sql("select num_num_1 - 100 from jsonTable where num_num_1 > 11"),
+        Row(21474836370L) :: Row(21474836470L) :: Nil
+      )
 
-    checkAnswer(
-      sql("select num_num_1 - 100 from jsonTable where num_num_1 > 10"),
-      Row(-89) :: Row(21474836370L) :: Row(21474836470L) :: Nil
-    )
+      checkAnswer(
+        sql("select num_num_1 - 100 from jsonTable where num_num_1 > 10"),
+        Row(-89) :: Row(21474836370L) :: Row(21474836470L) :: Nil
+      )
 
-    // Widening to DecimalType
-    checkAnswer(
-      sql("select num_num_2 + 1.3 from jsonTable where num_num_2 > 1.1"),
-      Row(21474836472.2) ::
-        Row(92233720368547758071.3) :: Nil
-    )
+      // Widening to DecimalType
+      checkAnswer(
+        sql("select num_num_2 + 1.3 from jsonTable where num_num_2 > 1.1"),
+        Row(21474836472.2) ::
+          Row(92233720368547758071.3) :: Nil
+      )
 
-    // Widening to Double
-    checkAnswer(
-      sql("select num_num_3 + 1.2 from jsonTable where num_num_3 > 1.1"),
-      Row(101.2) :: Row(21474836471.2) :: Nil
-    )
+      // Widening to Double
+      checkAnswer(
+        sql("select num_num_3 + 1.2 from jsonTable where num_num_3 > 1.1"),
+        Row(101.2) :: Row(21474836471.2) :: Nil
+      )
 
-    // Number and String conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_str + 1.2 from jsonTable where num_str > 14d"),
-      Row(92233720368547758071.2)
-    )
+      // Number and String conflict: resolve the type as number in this query.
+      checkAnswer(
+        sql("select num_str + 1.2 from jsonTable where num_str > 14d"),
+        Row(92233720368547758071.2)
+      )
 
-    // Number and String conflict: resolve the type as number in this query.
-    checkAnswer(
-      sql("select num_str + 1.2 from jsonTable where num_str >= 92233720368547758060"),
-      Row(new java.math.BigDecimal("92233720368547758071.2").doubleValue)
-    )
+      // Number and String conflict: resolve the type as number in this query.
+      checkAnswer(
+        sql("select num_str + 1.2 from jsonTable where num_str >= 92233720368547758060"),
+        Row(new java.math.BigDecimal("92233720368547758071.2").doubleValue)
+      )
 
-    // String and Boolean conflict: resolve the type as string.
-    checkAnswer(
-      sql("select * from jsonTable where str_bool = 'str1'"),
-      Row("true", 11L, null, 1.1, "13.1", "str1")
-    )
+      // String and Boolean conflict: resolve the type as string.
+      checkAnswer(
+        sql("select * from jsonTable where str_bool = 'str1'"),
+        Row("true", 11L, null, 1.1, "13.1", "str1")
+      )
+    }
   }
 
   test("Type conflict in complex field values") {
-    val jsonDF = spark.read.json(complexFieldValueTypeConflict)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldValueTypeConflict)
 
-    val expectedSchema = StructType(
-      StructField("array", ArrayType(LongType, true), true) ::
-      StructField("num_struct", StringType, true) ::
-      StructField("str_array", StringType, true) ::
-      StructField("struct", StructType(
-        StructField("field", StringType, true) :: Nil), true) ::
-      StructField("struct_array", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("array", ArrayType(LongType, true), true) ::
+        StructField("num_struct", StringType, true) ::
+        StructField("str_array", StringType, true) ::
+        StructField("struct", StructType(
+          StructField("field", StringType, true) :: Nil), true) ::
+        StructField("struct_array", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(Seq(), "11", "[1,2,3]", Row(null), "[]") ::
-        Row(null, """{"field":false}""", null, null, "{}") ::
-        Row(Seq(4, 5, 6), null, "str", Row(null), "[7,8,9]") ::
-        Row(Seq(7), "{}", """["str1","str2",33]""", Row("str"), """{"field":true}""") :: Nil
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(Seq(), "11", "[1,2,3]", Row(null), "[]") ::
+          Row(null, """{"field":false}""", null, null, "{}") ::
+          Row(Seq(4, 5, 6), null, "str", Row(null), "[7,8,9]") ::
+          Row(Seq(7), "{}", """["str1","str2",33]""", Row("str"), """{"field":true}""") :: Nil
+      )
+    }
   }
 
   test("Type conflict in array elements") {
-    val jsonDF = spark.read.json(arrayElementTypeConflict)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(arrayElementTypeConflict)
 
-    val expectedSchema = StructType(
-      StructField("array1", ArrayType(StringType, true), true) ::
-      StructField("array2", ArrayType(StructType(
-        StructField("field", LongType, true) :: Nil), true), true) ::
-      StructField("array3", ArrayType(StringType, true), true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("array1", ArrayType(StringType, true), true) ::
+        StructField("array2", ArrayType(StructType(
+          StructField("field", LongType, true) :: Nil), true), true) ::
+        StructField("array3", ArrayType(StringType, true), true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(Seq("1", "1.1", "true", null, "[]", "{}", "[2,3,4]",
-        """{"field":"str"}"""), Seq(Row(214748364700L), Row(1)), null) ::
-      Row(null, null, Seq("""{"field":"str"}""", """{"field":1}""")) ::
-      Row(null, null, Seq("1", "2", "3")) :: Nil
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(Seq("1", "1.1", "true", null, "[]", "{}", "[2,3,4]",
+          """{"field":"str"}"""), Seq(Row(214748364700L), Row(1)), null) ::
+        Row(null, null, Seq("""{"field":"str"}""", """{"field":1}""")) ::
+        Row(null, null, Seq("1", "2", "3")) :: Nil
+      )
 
-    // Treat an element as a number.
-    checkAnswer(
-      sql("select array1[0] + 1 from jsonTable where array1 is not null"),
-      Row(2)
-    )
+      // Treat an element as a number.
+      checkAnswer(
+        sql("select array1[0] + 1 from jsonTable where array1 is not null"),
+        Row(2)
+      )
+    }
   }
 
   test("Handling missing fields") {
-    val jsonDF = spark.read.json(missingFields)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(missingFields)
 
-    val expectedSchema = StructType(
-      StructField("a", BooleanType, true) ::
-      StructField("b", LongType, true) ::
-      StructField("c", ArrayType(LongType, true), true) ::
-      StructField("d", StructType(
-        StructField("field", BooleanType, true) :: Nil), true) ::
-      StructField("e", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("a", BooleanType, true) ::
+        StructField("b", LongType, true) ::
+        StructField("c", ArrayType(LongType, true), true) ::
+        StructField("d", StructType(
+          StructField("field", BooleanType, true) :: Nil), true) ::
+        StructField("e", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
+    }
   }
 
   test("Loading a JSON dataset from a text file") {
-    val dir = Utils.createTempDir()
-    dir.delete()
-    val path = dir.getCanonicalPath
-    primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
-    val jsonDF = spark.read.json(path)
+    withTempView("jsonTable") {
+      val dir = Utils.createTempDir()
+      dir.delete()
+      val path = dir.getCanonicalPath
+      primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
+      val jsonDF = spark.read.json(path)
 
-    val expectedSchema = StructType(
-      StructField("bigInteger", DecimalType(20, 0), true) ::
-      StructField("boolean", BooleanType, true) ::
-      StructField("double", DoubleType, true) ::
-      StructField("integer", LongType, true) ::
-      StructField("long", LongType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("bigInteger", DecimalType(20, 0), true) ::
+        StructField("boolean", BooleanType, true) ::
+        StructField("double", DoubleType, true) ::
+        StructField("integer", LongType, true) ::
+        StructField("long", LongType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-      true,
-      1.7976931348623157,
-      10,
-      21474836470L,
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+        true,
+        1.7976931348623157,
+        10,
+        21474836470L,
+        null,
+        "this is a simple string.")
+      )
+    }
   }
 
   test("Loading a JSON dataset primitivesAsString returns schema with primitive types as strings") {
-    val dir = Utils.createTempDir()
-    dir.delete()
-    val path = dir.getCanonicalPath
-    primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
-    val jsonDF = spark.read.option("primitivesAsString", "true").json(path)
-
-    val expectedSchema = StructType(
-      StructField("bigInteger", StringType, true) ::
-      StructField("boolean", StringType, true) ::
-      StructField("double", StringType, true) ::
-      StructField("integer", StringType, true) ::
-      StructField("long", StringType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+    withTempView("jsonTable") {
+      val dir = Utils.createTempDir()
+      dir.delete()
+      val path = dir.getCanonicalPath
+      primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
+      val jsonDF = spark.read.option("primitivesAsString", "true").json(path)
+
+      val expectedSchema = StructType(
+        StructField("bigInteger", StringType, true) ::
+        StructField("boolean", StringType, true) ::
+        StructField("double", StringType, true) ::
+        StructField("integer", StringType, true) ::
+        StructField("long", StringType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row("92233720368547758070",
-      "true",
-      "1.7976931348623157",
-      "10",
-      "21474836470",
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row("92233720368547758070",
+        "true",
+        "1.7976931348623157",
+        "10",
+        "21474836470",
+        null,
+        "this is a simple string.")
+      )
+    }
   }
 
   test("Loading a JSON dataset primitivesAsString returns complex fields as strings") {
-    val jsonDF = spark.read.option("primitivesAsString", "true").json(complexFieldAndType1)
-
-    val expectedSchema = StructType(
-      StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
-      StructField("arrayOfArray2", ArrayType(ArrayType(StringType, true), true), true) ::
-      StructField("arrayOfBigInteger", ArrayType(StringType, true), true) ::
-      StructField("arrayOfBoolean", ArrayType(StringType, true), true) ::
-      StructField("arrayOfDouble", ArrayType(StringType, true), true) ::
-      StructField("arrayOfInteger", ArrayType(StringType, true), true) ::
-      StructField("arrayOfLong", ArrayType(StringType, true), true) ::
-      StructField("arrayOfNull", ArrayType(StringType, true), true) ::
-      StructField("arrayOfString", ArrayType(StringType, true), true) ::
-      StructField("arrayOfStruct", ArrayType(
-        StructType(
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.option("primitivesAsString", "true").json(complexFieldAndType1)
+
+      val expectedSchema = StructType(
+        StructField("arrayOfArray1", ArrayType(ArrayType(StringType, true), true), true) ::
+        StructField("arrayOfArray2", ArrayType(ArrayType(StringType, true), true), true) ::
+        StructField("arrayOfBigInteger", ArrayType(StringType, true), true) ::
+        StructField("arrayOfBoolean", ArrayType(StringType, true), true) ::
+        StructField("arrayOfDouble", ArrayType(StringType, true), true) ::
+        StructField("arrayOfInteger", ArrayType(StringType, true), true) ::
+        StructField("arrayOfLong", ArrayType(StringType, true), true) ::
+        StructField("arrayOfNull", ArrayType(StringType, true), true) ::
+        StructField("arrayOfString", ArrayType(StringType, true), true) ::
+        StructField("arrayOfStruct", ArrayType(
+          StructType(
+            StructField("field1", StringType, true) ::
+            StructField("field2", StringType, true) ::
+            StructField("field3", StringType, true) :: Nil), true), true) ::
+        StructField("struct", StructType(
           StructField("field1", StringType, true) ::
-          StructField("field2", StringType, true) ::
-          StructField("field3", StringType, true) :: Nil), true), true) ::
-      StructField("struct", StructType(
-        StructField("field1", StringType, true) ::
-        StructField("field2", StringType, true) :: Nil), true) ::
-      StructField("structWithArrayFields", StructType(
-        StructField("field1", ArrayType(StringType, true), true) ::
-        StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
+          StructField("field2", StringType, true) :: Nil), true) ::
+        StructField("structWithArrayFields", StructType(
+          StructField("field1", ArrayType(StringType, true), true) ::
+          StructField("field2", ArrayType(StringType, true), true) :: Nil), true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    // Access elements of a primitive array.
-    checkAnswer(
-      sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
-      Row("str1", "str2", null)
-    )
+      // Access elements of a primitive array.
+      checkAnswer(
+        sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from jsonTable"),
+        Row("str1", "str2", null)
+      )
 
-    // Access an array of null values.
-    checkAnswer(
-      sql("select arrayOfNull from jsonTable"),
-      Row(Seq(null, null, null, null))
-    )
+      // Access an array of null values.
+      checkAnswer(
+        sql("select arrayOfNull from jsonTable"),
+        Row(Seq(null, null, null, null))
+      )
 
-    // Access elements of a BigInteger array (we use DecimalType internally).
-    checkAnswer(
-      sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from jsonTable"),
-      Row("922337203685477580700", "-922337203685477580800", null)
-    )
+      // Access elements of a BigInteger array (we use DecimalType internally).
+      checkAnswer(
+        sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] from " +
+          "jsonTable"),
+        Row("922337203685477580700", "-922337203685477580800", null)
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
-      Row(Seq("1", "2", "3"), Seq("str1", "str2"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray1[0], arrayOfArray1[1] from jsonTable"),
+        Row(Seq("1", "2", "3"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
-      Row(Seq("1", "2", "3"), Seq("1.1", "2.1", "3.1"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray2[0], arrayOfArray2[1] from jsonTable"),
+        Row(Seq("1", "2", "3"), Seq("1.1", "2.1", "3.1"))
+      )
 
-    // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
-    checkAnswer(
-      sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
-      Row("str2", "2.1")
-    )
+      // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
+      checkAnswer(
+        sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from jsonTable"),
+        Row("str2", "2.1")
+      )
 
-    // Access elements of an array of structs.
-    checkAnswer(
-      sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
-        "from jsonTable"),
-      Row(
-        Row("true", "str1", null),
-        Row("false", null, null),
-        Row(null, null, null),
-        null)
-    )
+      // Access elements of an array of structs.
+      checkAnswer(
+        sql("select arrayOfStruct[0], arrayOfStruct[1], arrayOfStruct[2], arrayOfStruct[3] " +
+          "from jsonTable"),
+        Row(
+          Row("true", "str1", null),
+          Row("false", null, null),
+          Row(null, null, null),
+          null)
+      )
 
-    // Access a struct and fields inside of it.
-    checkAnswer(
-      sql("select struct, struct.field1, struct.field2 from jsonTable"),
-      Row(
-        Row("true", "92233720368547758070"),
-        "true",
-        "92233720368547758070") :: Nil
-    )
+      // Access a struct and fields inside of it.
+      checkAnswer(
+        sql("select struct, struct.field1, struct.field2 from jsonTable"),
+        Row(
+          Row("true", "92233720368547758070"),
+          "true",
+          "92233720368547758070") :: Nil
+      )
 
-    // Access an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
-      Row(Seq("4", "5", "6"), Seq("str1", "str2"))
-    )
+      // Access an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1, structWithArrayFields.field2 from jsonTable"),
+        Row(Seq("4", "5", "6"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from jsonTable"),
-      Row("5", null)
-    )
+      // Access elements of an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] from " +
+          "jsonTable"),
+        Row("5", null)
+      )
+    }
   }
 
   test("Loading a JSON dataset prefersDecimal returns schema with float types as BigDecimal") {
-    val jsonDF = spark.read.option("prefersDecimal", "true").json(primitiveFieldAndType)
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.option("prefersDecimal", "true").json(primitiveFieldAndType)
 
-    val expectedSchema = StructType(
-      StructField("bigInteger", DecimalType(20, 0), true) ::
-        StructField("boolean", BooleanType, true) ::
-        StructField("double", DecimalType(17, 16), true) ::
-        StructField("integer", LongType, true) ::
-        StructField("long", LongType, true) ::
-        StructField("null", StringType, true) ::
-        StructField("string", StringType, true) :: Nil)
+      val expectedSchema = StructType(
+        StructField("bigInteger", DecimalType(20, 0), true) ::
+          StructField("boolean", BooleanType, true) ::
+          StructField("double", DecimalType(17, 16), true) ::
+          StructField("integer", LongType, true) ::
+          StructField("long", LongType, true) ::
+          StructField("null", StringType, true) ::
+          StructField("string", StringType, true) :: Nil)
 
-    assert(expectedSchema === jsonDF.schema)
+      assert(expectedSchema === jsonDF.schema)
 
-    jsonDF.createOrReplaceTempView("jsonTable")
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select * from jsonTable"),
-      Row(BigDecimal("92233720368547758070"),
-        true,
-        BigDecimal("1.7976931348623157"),
-        10,
-        21474836470L,
-        null,
-        "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable"),
+        Row(BigDecimal("92233720368547758070"),
+          true,
+          BigDecimal("1.7976931348623157"),
+          10,
+          21474836470L,
+          null,
+          "this is a simple string.")
+      )
+    }
   }
 
   test("Find compatible types even if inferred DecimalType is not capable of other IntegralType") {
@@ -834,171 +862,182 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("Applying schemas") {
-    val dir = Utils.createTempDir()
-    dir.delete()
-    val path = dir.getCanonicalPath
-    primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
+    withTempView("jsonTable1", "jsonTable2") {
+      val dir = Utils.createTempDir()
+      dir.delete()
+      val path = dir.getCanonicalPath
+      primitiveFieldAndType.map(record => record.replaceAll("\n", " ")).write.text(path)
 
-    val schema = StructType(
-      StructField("bigInteger", DecimalType.SYSTEM_DEFAULT, true) ::
-      StructField("boolean", BooleanType, true) ::
-      StructField("double", DoubleType, true) ::
-      StructField("integer", IntegerType, true) ::
-      StructField("long", LongType, true) ::
-      StructField("null", StringType, true) ::
-      StructField("string", StringType, true) :: Nil)
+      val schema = StructType(
+        StructField("bigInteger", DecimalType.SYSTEM_DEFAULT, true) ::
+        StructField("boolean", BooleanType, true) ::
+        StructField("double", DoubleType, true) ::
+        StructField("integer", IntegerType, true) ::
+        StructField("long", LongType, true) ::
+        StructField("null", StringType, true) ::
+        StructField("string", StringType, true) :: Nil)
 
-    val jsonDF1 = spark.read.schema(schema).json(path)
+      val jsonDF1 = spark.read.schema(schema).json(path)
 
-    assert(schema === jsonDF1.schema)
+      assert(schema === jsonDF1.schema)
 
-    jsonDF1.createOrReplaceTempView("jsonTable1")
+      jsonDF1.createOrReplaceTempView("jsonTable1")
 
-    checkAnswer(
-      sql("select * from jsonTable1"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-      true,
-      1.7976931348623157,
-      10,
-      21474836470L,
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable1"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+        true,
+        1.7976931348623157,
+        10,
+        21474836470L,
+        null,
+        "this is a simple string.")
+      )
 
-    val jsonDF2 = spark.read.schema(schema).json(primitiveFieldAndType)
+      val jsonDF2 = spark.read.schema(schema).json(primitiveFieldAndType)
 
-    assert(schema === jsonDF2.schema)
+      assert(schema === jsonDF2.schema)
 
-    jsonDF2.createOrReplaceTempView("jsonTable2")
+      jsonDF2.createOrReplaceTempView("jsonTable2")
 
-    checkAnswer(
-      sql("select * from jsonTable2"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-      true,
-      1.7976931348623157,
-      10,
-      21474836470L,
-      null,
-      "this is a simple string.")
-    )
+      checkAnswer(
+        sql("select * from jsonTable2"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+        true,
+        1.7976931348623157,
+        10,
+        21474836470L,
+        null,
+        "this is a simple string.")
+      )
+    }
   }
 
   test("Applying schemas with MapType") {
-    val schemaWithSimpleMap = StructType(
-      StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
-    val jsonWithSimpleMap = spark.read.schema(schemaWithSimpleMap).json(mapType1)
+    withTempView("jsonWithSimpleMap", "jsonWithComplexMap") {
+      val schemaWithSimpleMap = StructType(
+        StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
+      val jsonWithSimpleMap = spark.read.schema(schemaWithSimpleMap).json(mapType1)
 
-    jsonWithSimpleMap.createOrReplaceTempView("jsonWithSimpleMap")
+      jsonWithSimpleMap.createOrReplaceTempView("jsonWithSimpleMap")
 
-    checkAnswer(
-      sql("select `map` from jsonWithSimpleMap"),
-      Row(Map("a" -> 1)) ::
-      Row(Map("b" -> 2)) ::
-      Row(Map("c" -> 3)) ::
-      Row(Map("c" -> 1, "d" -> 4)) ::
-      Row(Map("e" -> null)) :: Nil
-    )
-
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
       checkAnswer(
-        sql("select `map`['c'] from jsonWithSimpleMap"),
-        Row(null) ::
-        Row(null) ::
-        Row(3) ::
-        Row(1) ::
-        Row(null) :: Nil
+        sql("select `map` from jsonWithSimpleMap"),
+        Row(Map("a" -> 1)) ::
+        Row(Map("b" -> 2)) ::
+        Row(Map("c" -> 3)) ::
+        Row(Map("c" -> 1, "d" -> 4)) ::
+        Row(Map("e" -> null)) :: Nil
       )
-    }
 
-    val innerStruct = StructType(
-      StructField("field1", ArrayType(IntegerType, true), true) ::
-      StructField("field2", IntegerType, true) :: Nil)
-    val schemaWithComplexMap = StructType(
-      StructField("map", MapType(StringType, innerStruct, true), false) :: Nil)
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        checkAnswer(
+          sql("select `map`['c'] from jsonWithSimpleMap"),
+          Row(null) ::
+          Row(null) ::
+          Row(3) ::
+          Row(1) ::
+          Row(null) :: Nil
+        )
+      }
 
-    val jsonWithComplexMap = spark.read.schema(schemaWithComplexMap).json(mapType2)
+      val innerStruct = StructType(
+        StructField("field1", ArrayType(IntegerType, true), true) ::
+        StructField("field2", IntegerType, true) :: Nil)
+      val schemaWithComplexMap = StructType(
+        StructField("map", MapType(StringType, innerStruct, true), false) :: Nil)
 
-    jsonWithComplexMap.createOrReplaceTempView("jsonWithComplexMap")
+      val jsonWithComplexMap = spark.read.schema(schemaWithComplexMap).json(mapType2)
 
-    checkAnswer(
-      sql("select `map` from jsonWithComplexMap"),
-      Row(Map("a" -> Row(Seq(1, 2, 3, null), null))) ::
-      Row(Map("b" -> Row(null, 2))) ::
-      Row(Map("c" -> Row(Seq(), 4))) ::
-      Row(Map("c" -> Row(null, 3), "d" -> Row(Seq(null), null))) ::
-      Row(Map("e" -> null)) ::
-      Row(Map("f" -> Row(null, null))) :: Nil
-    )
+      jsonWithComplexMap.createOrReplaceTempView("jsonWithComplexMap")
 
-    withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
       checkAnswer(
-        sql("select `map`['a'].field1, `map`['c'].field2 from jsonWithComplexMap"),
-        Row(Seq(1, 2, 3, null), null) ::
-        Row(null, null) ::
-        Row(null, 4) ::
-        Row(null, 3) ::
-        Row(null, null) ::
-        Row(null, null) :: Nil
+        sql("select `map` from jsonWithComplexMap"),
+        Row(Map("a" -> Row(Seq(1, 2, 3, null), null))) ::
+        Row(Map("b" -> Row(null, 2))) ::
+        Row(Map("c" -> Row(Seq(), 4))) ::
+        Row(Map("c" -> Row(null, 3), "d" -> Row(Seq(null), null))) ::
+        Row(Map("e" -> null)) ::
+        Row(Map("f" -> Row(null, null))) :: Nil
       )
+
+      withSQLConf(SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
+        checkAnswer(
+          sql("select `map`['a'].field1, `map`['c'].field2 from jsonWithComplexMap"),
+          Row(Seq(1, 2, 3, null), null) ::
+          Row(null, null) ::
+          Row(null, 4) ::
+          Row(null, 3) ::
+          Row(null, null) ::
+          Row(null, null) :: Nil
+        )
+      }
     }
   }
 
   test("SPARK-2096 Correctly parse dot notations") {
-    val jsonDF = spark.read.json(complexFieldAndType2)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType2)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
-      Row(true, "str1")
-    )
-    checkAnswer(
-      sql(
-        """
-          |select complexArrayOfStruct[0].field1[1].inner2[0], complexArrayOfStruct[1].field2[0][1]
-          |from jsonTable
-        """.stripMargin),
-      Row("str2", 6)
-    )
+      checkAnswer(
+        sql("select arrayOfStruct[0].field1, arrayOfStruct[0].field2 from jsonTable"),
+        Row(true, "str1")
+      )
+      checkAnswer(
+        sql(
+          """
+            |select complexArrayOfStruct[0].field1[1].inner2[0],
+            |complexArrayOfStruct[1].field2[0][1]
+            |from jsonTable
+          """.stripMargin),
+        Row("str2", 6)
+      )
+    }
   }
 
   test("SPARK-3390 Complex arrays") {
-    val jsonDF = spark.read.json(complexFieldAndType2)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(complexFieldAndType2)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql(
-        """
-          |select arrayOfArray1[0][0][0], arrayOfArray1[1][0][1], arrayOfArray1[1][1][0]
-          |from jsonTable
-        """.stripMargin),
-      Row(5, 7, 8)
-    )
-    checkAnswer(
-      sql(
-        """
-          |select arrayOfArray2[0][0][0].inner1, arrayOfArray2[1][0],
-          |arrayOfArray2[1][1][1].inner2[0], arrayOfArray2[2][0][0].inner3[0][0].inner4
-          |from jsonTable
-        """.stripMargin),
-      Row("str1", Nil, "str4", 2)
-    )
+      checkAnswer(
+        sql(
+          """
+            |select arrayOfArray1[0][0][0], arrayOfArray1[1][0][1], arrayOfArray1[1][1][0]
+            |from jsonTable
+          """.stripMargin),
+        Row(5, 7, 8)
+      )
+      checkAnswer(
+        sql(
+          """
+            |select arrayOfArray2[0][0][0].inner1, arrayOfArray2[1][0],
+            |arrayOfArray2[1][1][1].inner2[0], arrayOfArray2[2][0][0].inner3[0][0].inner4
+            |from jsonTable
+          """.stripMargin),
+        Row("str1", Nil, "str4", 2)
+      )
+    }
   }
 
   test("SPARK-3308 Read top level JSON arrays") {
-    val jsonDF = spark.read.json(jsonArray)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(jsonArray)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    checkAnswer(
-      sql(
-        """
-          |select a, b, c
-          |from jsonTable
-        """.stripMargin),
-      Row("str_a_1", null, null) ::
-        Row("str_a_2", null, null) ::
-        Row(null, "str_b_3", null) ::
-        Row("str_a_4", "str_b_4", "str_c_4") :: Nil
-    )
+      checkAnswer(
+        sql(
+          """
+            |select a, b, c
+            |from jsonTable
+          """.stripMargin),
+        Row("str_a_1", null, null) ::
+          Row("str_a_2", null, null) ::
+          Row(null, "str_b_3", null) ::
+          Row("str_a_4", "str_b_4", "str_c_4") :: Nil
+      )
+    }
   }
 
   test("Corrupt records: FAILFAST mode") {
@@ -1140,158 +1179,162 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("SPARK-4068: nulls in arrays") {
-    val jsonDF = spark.read.json(nullsInArrays)
-    jsonDF.createOrReplaceTempView("jsonTable")
+    withTempView("jsonTable") {
+      val jsonDF = spark.read.json(nullsInArrays)
+      jsonDF.createOrReplaceTempView("jsonTable")
 
-    val schema = StructType(
-      StructField("field1",
-        ArrayType(ArrayType(ArrayType(ArrayType(StringType, true), true), true), true), true) ::
-      StructField("field2",
-        ArrayType(ArrayType(
-          StructType(StructField("Test", LongType, true) :: Nil), true), true), true) ::
-      StructField("field3",
-        ArrayType(ArrayType(
-          StructType(StructField("Test", StringType, true) :: Nil), true), true), true) ::
-      StructField("field4",
-        ArrayType(ArrayType(ArrayType(LongType, true), true), true), true) :: Nil)
+      val schema = StructType(
+        StructField("field1",
+          ArrayType(ArrayType(ArrayType(ArrayType(StringType, true), true), true), true), true) ::
+        StructField("field2",
+          ArrayType(ArrayType(
+            StructType(StructField("Test", LongType, true) :: Nil), true), true), true) ::
+        StructField("field3",
+          ArrayType(ArrayType(
+            StructType(StructField("Test", StringType, true) :: Nil), true), true), true) ::
+        StructField("field4",
+          ArrayType(ArrayType(ArrayType(LongType, true), true), true), true) :: Nil)
 
-    assert(schema === jsonDF.schema)
+      assert(schema === jsonDF.schema)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT field1, field2, field3, field4
-          |FROM jsonTable
-        """.stripMargin),
-      Row(Seq(Seq(null), Seq(Seq(Seq("Test")))), null, null, null) ::
-        Row(null, Seq(null, Seq(Row(1))), null, null) ::
-        Row(null, null, Seq(Seq(null), Seq(Row("2"))), null) ::
-        Row(null, null, null, Seq(Seq(null, Seq(1, 2, 3)))) :: Nil
-    )
+      checkAnswer(
+        sql(
+          """
+            |SELECT field1, field2, field3, field4
+            |FROM jsonTable
+          """.stripMargin),
+        Row(Seq(Seq(null), Seq(Seq(Seq("Test")))), null, null, null) ::
+          Row(null, Seq(null, Seq(Row(1))), null, null) ::
+          Row(null, null, Seq(Seq(null), Seq(Row("2"))), null) ::
+          Row(null, null, null, Seq(Seq(null, Seq(1, 2, 3)))) :: Nil
+      )
+    }
   }
 
   test("SPARK-4228 DataFrame to JSON") {
-    val schema1 = StructType(
-      StructField("f1", IntegerType, false) ::
-      StructField("f2", StringType, false) ::
-      StructField("f3", BooleanType, false) ::
-      StructField("f4", ArrayType(StringType), nullable = true) ::
-      StructField("f5", IntegerType, true) :: Nil)
-
-    val rowRDD1 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v5 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+    withTempView("applySchema1", "applySchema2", "primitiveTable", "complexTable") {
+      val schema1 = StructType(
+        StructField("f1", IntegerType, false) ::
+        StructField("f2", StringType, false) ::
+        StructField("f3", BooleanType, false) ::
+        StructField("f4", ArrayType(StringType), nullable = true) ::
+        StructField("f5", IntegerType, true) :: Nil)
+
+      val rowRDD1 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v5 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(values(0).toInt, values(1), values(2).toBoolean, r.split(",").toList, v5)
       }
-      Row(values(0).toInt, values(1), values(2).toBoolean, r.split(",").toList, v5)
-    }
-
-    val df1 = spark.createDataFrame(rowRDD1, schema1)
-    df1.createOrReplaceTempView("applySchema1")
-    val df2 = df1.toDF
-    val result = df2.toJSON.collect()
-    // scalastyle:off
-    assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"]}")
-    assert(result(3) === "{\"f1\":4,\"f2\":\"D4\",\"f3\":true,\"f4\":[\"4\",\" D4\",\" true\",\" 2147483644\"],\"f5\":2147483644}")
-    // scalastyle:on
 
-    val schema2 = StructType(
-      StructField("f1", StructType(
-        StructField("f11", IntegerType, false) ::
-        StructField("f12", BooleanType, false) :: Nil), false) ::
-      StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
-
-    val rowRDD2 = unparsedStrings.map { r =>
-      val values = r.split(",").map(_.trim)
-      val v4 = try values(3).toInt catch {
-        case _: NumberFormatException => null
+      val df1 = spark.createDataFrame(rowRDD1, schema1)
+      df1.createOrReplaceTempView("applySchema1")
+      val df2 = df1.toDF
+      val result = df2.toJSON.collect()
+      // scalastyle:off
+      assert(result(0) === "{\"f1\":1,\"f2\":\"A1\",\"f3\":true,\"f4\":[\"1\",\" A1\",\" true\",\" null\"]}")
+      assert(result(3) === "{\"f1\":4,\"f2\":\"D4\",\"f3\":true,\"f4\":[\"4\",\" D4\",\" true\",\" 2147483644\"],\"f5\":2147483644}")
+      // scalastyle:on
+
+      val schema2 = StructType(
+        StructField("f1", StructType(
+          StructField("f11", IntegerType, false) ::
+          StructField("f12", BooleanType, false) :: Nil), false) ::
+        StructField("f2", MapType(StringType, IntegerType, true), false) :: Nil)
+
+      val rowRDD2 = unparsedStrings.map { r =>
+        val values = r.split(",").map(_.trim)
+        val v4 = try values(3).toInt catch {
+          case _: NumberFormatException => null
+        }
+        Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
       }
-      Row(Row(values(0).toInt, values(2).toBoolean), Map(values(1) -> v4))
-    }
 
-    val df3 = spark.createDataFrame(rowRDD2, schema2)
-    df3.createOrReplaceTempView("applySchema2")
-    val df4 = df3.toDF
-    val result2 = df4.toJSON.collect()
+      val df3 = spark.createDataFrame(rowRDD2, schema2)
+      df3.createOrReplaceTempView("applySchema2")
+      val df4 = df3.toDF
+      val result2 = df4.toJSON.collect()
 
-    assert(result2(1) === "{\"f1\":{\"f11\":2,\"f12\":false},\"f2\":{\"B2\":null}}")
-    assert(result2(3) === "{\"f1\":{\"f11\":4,\"f12\":true},\"f2\":{\"D4\":2147483644}}")
+      assert(result2(1) === "{\"f1\":{\"f11\":2,\"f12\":false},\"f2\":{\"B2\":null}}")
+      assert(result2(3) === "{\"f1\":{\"f11\":4,\"f12\":true},\"f2\":{\"D4\":2147483644}}")
 
-    val jsonDF = spark.read.json(primitiveFieldAndType)
-    val primTable = spark.read.json(jsonDF.toJSON)
-    primTable.createOrReplaceTempView("primitiveTable")
-    checkAnswer(
-        sql("select * from primitiveTable"),
-      Row(new java.math.BigDecimal("92233720368547758070"),
-        true,
-        1.7976931348623157,
-        10,
-        21474836470L,
-        "this is a simple string.")
+      val jsonDF = spark.read.json(primitiveFieldAndType)
+      val primTable = spark.read.json(jsonDF.toJSON)
+      primTable.createOrReplaceTempView("primitiveTable")
+      checkAnswer(
+          sql("select * from primitiveTable"),
+        Row(new java.math.BigDecimal("92233720368547758070"),
+          true,
+          1.7976931348623157,
+          10,
+          21474836470L,
+          "this is a simple string.")
       )
 
-    val complexJsonDF = spark.read.json(complexFieldAndType1)
-    val compTable = spark.read.json(complexJsonDF.toJSON)
-    compTable.createOrReplaceTempView("complexTable")
-    // Access elements of a primitive array.
-    checkAnswer(
-      sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from complexTable"),
-      Row("str1", "str2", null)
-    )
+      val complexJsonDF = spark.read.json(complexFieldAndType1)
+      val compTable = spark.read.json(complexJsonDF.toJSON)
+      compTable.createOrReplaceTempView("complexTable")
+      // Access elements of a primitive array.
+      checkAnswer(
+        sql("select arrayOfString[0], arrayOfString[1], arrayOfString[2] from complexTable"),
+        Row("str1", "str2", null)
+      )
 
-    // Access an array of null values.
-    checkAnswer(
-      sql("select arrayOfNull from complexTable"),
-      Row(Seq(null, null, null, null))
-    )
+      // Access an array of null values.
+      checkAnswer(
+        sql("select arrayOfNull from complexTable"),
+        Row(Seq(null, null, null, null))
+      )
 
-    // Access elements of a BigInteger array (we use DecimalType internally).
-    checkAnswer(
-      sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] " +
-        " from complexTable"),
-      Row(new java.math.BigDecimal("922337203685477580700"),
-        new java.math.BigDecimal("-922337203685477580800"), null)
-    )
+      // Access elements of a BigInteger array (we use DecimalType internally).
+      checkAnswer(
+        sql("select arrayOfBigInteger[0], arrayOfBigInteger[1], arrayOfBigInteger[2] " +
+          " from complexTable"),
+        Row(new java.math.BigDecimal("922337203685477580700"),
+          new java.math.BigDecimal("-922337203685477580800"), null)
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray1[0], arrayOfArray1[1] from complexTable"),
-      Row(Seq("1", "2", "3"), Seq("str1", "str2"))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray1[0], arrayOfArray1[1] from complexTable"),
+        Row(Seq("1", "2", "3"), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array of arrays.
-    checkAnswer(
-      sql("select arrayOfArray2[0], arrayOfArray2[1] from complexTable"),
-      Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
-    )
+      // Access elements of an array of arrays.
+      checkAnswer(
+        sql("select arrayOfArray2[0], arrayOfArray2[1] from complexTable"),
+        Row(Seq(1.0, 2.0, 3.0), Seq(1.1, 2.1, 3.1))
+      )
 
-    // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
-    checkAnswer(
-      sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from complexTable"),
-      Row("str2", 2.1)
-    )
+      // Access elements of an array inside a filed with the type of ArrayType(ArrayType).
+      checkAnswer(
+        sql("select arrayOfArray1[1][1], arrayOfArray2[1][1] from complexTable"),
+        Row("str2", 2.1)
+      )
 
-    // Access a struct and fields inside of it.
-    checkAnswer(
-      sql("select struct, struct.field1, struct.field2 from complexTable"),
-      Row(
-        Row(true, new java.math.BigDecimal("92233720368547758070")),
-        true,
-        new java.math.BigDecimal("92233720368547758070")) :: Nil
-    )
+      // Access a struct and fields inside of it.
+      checkAnswer(
+        sql("select struct, struct.field1, struct.field2 from complexTable"),
+        Row(
+          Row(true, new java.math.BigDecimal("92233720368547758070")),
+          true,
+          new java.math.BigDecimal("92233720368547758070")) :: Nil
+      )
 
-    // Access an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1, structWithArrayFields.field2 from complexTable"),
-      Row(Seq(4, 5, 6), Seq("str1", "str2"))
-    )
+      // Access an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1, structWithArrayFields.field2 from complexTable"),
+        Row(Seq(4, 5, 6), Seq("str1", "str2"))
+      )
 
-    // Access elements of an array field of a struct.
-    checkAnswer(
-      sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] " +
-        "from complexTable"),
-      Row(5, null)
-    )
+      // Access elements of an array field of a struct.
+      checkAnswer(
+        sql("select structWithArrayFields.field1[1], structWithArrayFields.field2[3] " +
+          "from complexTable"),
+        Row(5, null)
+      )
+    }
   }
 
   test("Dataset toJSON doesn't construct rdd") {
@@ -1372,20 +1415,21 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     }
 
     withTempPath(root => {
-      val d1 = new File(root, "d1=1")
-      // root/dt=1/col1=abc
-      val p1_col1 = makePartition(
-        sparkContext.parallelize(2 to 5).map(i => s"""{"a": 1, "b": "str$i"}"""),
-        d1,
-        "col1",
-        "abc")
-
-      // root/dt=1/col1=abd
-      val p2 = makePartition(
-        sparkContext.parallelize(6 to 10).map(i => s"""{"a": 1, "b": "str$i"}"""),
-        d1,
-        "col1",
-        "abd")
+      withTempView("test_myjson_with_part") {
+        val d1 = new File(root, "d1=1")
+        // root/dt=1/col1=abc
+        val p1_col1 = makePartition(
+          sparkContext.parallelize(2 to 5).map(i => s"""{"a": 1, "b": "str$i"}"""),
+          d1,
+          "col1",
+          "abc")
+
+        // root/dt=1/col1=abd
+        val p2 = makePartition(
+          sparkContext.parallelize(6 to 10).map(i => s"""{"a": 1, "b": "str$i"}"""),
+          d1,
+          "col1",
+          "abd")
 
         spark.read.json(root.getAbsolutePath).createOrReplaceTempView("test_myjson_with_part")
         checkAnswer(sql(
@@ -1394,6 +1438,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
           "SELECT count(a) FROM test_myjson_with_part where d1 = 1 and col1='abd'"), Row(5))
         checkAnswer(sql(
           "SELECT count(a) FROM test_myjson_with_part where d1 = 1"), Row(9))
+      }
     })
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 64ecf5deeff4b..1be9308c06d8c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -207,24 +207,25 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
 
   test("broadcast hint in SQL") {
     import org.apache.spark.sql.catalyst.plans.logical.Join
-
-    spark.range(10).createOrReplaceTempView("t")
-    spark.range(10).createOrReplaceTempView("u")
-
-    for (name <- Seq("BROADCAST", "BROADCASTJOIN", "MAPJOIN")) {
-      val plan1 = sql(s"SELECT /*+ $name(t) */ * FROM t JOIN u ON t.id = u.id").queryExecution
-        .optimizedPlan
-      val plan2 = sql(s"SELECT /*+ $name(u) */ * FROM t JOIN u ON t.id = u.id").queryExecution
-        .optimizedPlan
-      val plan3 = sql(s"SELECT /*+ $name(v) */ * FROM t JOIN u ON t.id = u.id").queryExecution
-        .optimizedPlan
-
-      assert(plan1.asInstanceOf[Join].hint.leftHint.get.strategy.contains(BROADCAST))
-      assert(plan1.asInstanceOf[Join].hint.rightHint.isEmpty)
-      assert(plan2.asInstanceOf[Join].hint.leftHint.isEmpty)
-      assert(plan2.asInstanceOf[Join].hint.rightHint.get.strategy.contains(BROADCAST))
-      assert(plan3.asInstanceOf[Join].hint.leftHint.isEmpty)
-      assert(plan3.asInstanceOf[Join].hint.rightHint.isEmpty)
+    withTempView("t", "u") {
+      spark.range(10).createOrReplaceTempView("t")
+      spark.range(10).createOrReplaceTempView("u")
+
+      for (name <- Seq("BROADCAST", "BROADCASTJOIN", "MAPJOIN")) {
+        val plan1 = sql(s"SELECT /*+ $name(t) */ * FROM t JOIN u ON t.id = u.id").queryExecution
+          .optimizedPlan
+        val plan2 = sql(s"SELECT /*+ $name(u) */ * FROM t JOIN u ON t.id = u.id").queryExecution
+          .optimizedPlan
+        val plan3 = sql(s"SELECT /*+ $name(v) */ * FROM t JOIN u ON t.id = u.id").queryExecution
+          .optimizedPlan
+
+        assert(plan1.asInstanceOf[Join].hint.leftHint.get.strategy.contains(BROADCAST))
+        assert(plan1.asInstanceOf[Join].hint.rightHint.isEmpty)
+        assert(plan2.asInstanceOf[Join].hint.leftHint.isEmpty)
+        assert(plan2.asInstanceOf[Join].hint.rightHint.get.strategy.contains(BROADCAST))
+        assert(plan3.asInstanceOf[Join].hint.leftHint.isEmpty)
+        assert(plan3.asInstanceOf[Join].hint.rightHint.isEmpty)
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
index 1c4a8189349ea..00c0315ae3779 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/metric/SQLMetricsSuite.scala
@@ -463,27 +463,30 @@ class SQLMetricsSuite extends SharedSparkSession with SQLMetricsTestUtils
     // TODO: test file source V2 as well when its statistics is correctly computed.
     withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") {
       withTempDir { tempDir =>
-        val dir = new File(tempDir, "pqS").getCanonicalPath
-
-        spark.range(10).write.parquet(dir)
-        spark.read.parquet(dir).createOrReplaceTempView("pqS")
-
-        // The executed plan looks like:
-        // Exchange RoundRobinPartitioning(2)
-        // +- BroadcastNestedLoopJoin BuildLeft, Cross
-        //   :- BroadcastExchange IdentityBroadcastMode
-        //   :  +- Exchange RoundRobinPartitioning(3)
-        //   :     +- *Range (0, 30, step=1, splits=2)
-        //   +- *FileScan parquet [id#465L] Batched: true, Format: Parquet, Location: ...(ignored)
-        val res3 = InputOutputMetricsHelper.run(
-          spark.range(30).repartition(3).crossJoin(sql("select * from pqS")).repartition(2).toDF()
-        )
-        // The query above is executed in the following stages:
-        //   1. range(30)                   => (30, 0, 30)
-        //   2. sql("select * from pqS")    => (0, 30, 0)
-        //   3. crossJoin(...) of 1. and 2. => (10, 0, 300)
-        //   4. shuffle & return results    => (0, 300, 0)
-        assert(res3 === (30L, 0L, 30L) :: (0L, 30L, 0L) :: (10L, 0L, 300L) :: (0L, 300L, 0L) :: Nil)
+        withTempView("pqS") {
+          val dir = new File(tempDir, "pqS").getCanonicalPath
+
+          spark.range(10).write.parquet(dir)
+          spark.read.parquet(dir).createOrReplaceTempView("pqS")
+
+          // The executed plan looks like:
+          // Exchange RoundRobinPartitioning(2)
+          // +- BroadcastNestedLoopJoin BuildLeft, Cross
+          //   :- BroadcastExchange IdentityBroadcastMode
+          //   :  +- Exchange RoundRobinPartitioning(3)
+          //   :     +- *Range (0, 30, step=1, splits=2)
+          //   +- *FileScan parquet [id#465L] Batched: true, Format: Parquet, Location: ...(ignored)
+          val res3 = InputOutputMetricsHelper.run(
+            spark.range(30).repartition(3).crossJoin(sql("select * from pqS")).repartition(2).toDF()
+          )
+          // The query above is executed in the following stages:
+          //   1. range(30)                   => (30, 0, 30)
+          //   2. sql("select * from pqS")    => (0, 30, 0)
+          //   3. crossJoin(...) of 1. and 2. => (10, 0, 300)
+          //   4. shuffle & return results    => (0, 300, 0)
+          assert(res3 === (30L, 0L, 30L) :: (0L, 30L, 0L) :: (10L, 0L, 300L) :: (0L, 300L, 0L) ::
+            Nil)
+        }
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index fd691f547eb05..7c723f083642d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -635,12 +635,14 @@ class JDBCSuite extends QueryTest
   }
 
   test("test DATE types in cache") {
-    val rows = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()).collect()
-    spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties())
-      .cache().createOrReplaceTempView("mycached_date")
-    val cachedRows = sql("select * from mycached_date").collect()
-    assert(rows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
-    assert(cachedRows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+    withTempView("mycached_date") {
+      val rows = spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties()).collect()
+      spark.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties())
+        .cache().createOrReplaceTempView("mycached_date")
+      val cachedRows = sql("select * from mycached_date").collect()
+      assert(rows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+      assert(cachedRows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+    }
   }
 
   test("test types for null value") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 2d666371da21b..bb762d26e973c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -523,15 +523,17 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
 
   test("new partitions should be added to catalog after writing to catalog table") {
     val table = "partitioned_catalog_table"
+    val tempTable = "partitioned_catalog_temp_table"
     val numParts = 210
     withTable(table) {
-      val df = (1 to numParts).map(i => (i, i)).toDF("part", "col1")
-      val tempTable = "partitioned_catalog_temp_table"
-      df.createOrReplaceTempView(tempTable)
-      sql(s"CREATE TABLE $table (part Int, col1 Int) USING parquet PARTITIONED BY (part)")
-      sql(s"INSERT INTO TABLE $table SELECT * from $tempTable")
-      val partitions = spark.sessionState.catalog.listPartitionNames(TableIdentifier(table))
-      assert(partitions.size == numParts)
+      withTempView(tempTable) {
+        val df = (1 to numParts).map(i => (i, i)).toDF("part", "col1")
+        df.createOrReplaceTempView(tempTable)
+        sql(s"CREATE TABLE $table (part Int, col1 Int) USING parquet PARTITIONED BY (part)")
+        sql(s"INSERT INTO TABLE $table SELECT * from $tempTable")
+        val partitions = spark.sessionState.catalog.listPartitionNames(TableIdentifier(table))
+        assert(partitions.size == numParts)
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
index ce3ec7f97a537..1e0d9bd9990e2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
@@ -103,30 +103,32 @@ class SaveLoadSuite extends DataSourceTest with SharedSparkSession with BeforeAn
   }
 
   test("save and save again") {
-    df.write.json(path.toString)
-
-    val message = intercept[AnalysisException] {
+    withTempView("jsonTable2") {
       df.write.json(path.toString)
-    }.getMessage
 
-    assert(
-      message.contains("already exists"),
-      "We should complain that the path already exists.")
+      val message = intercept[AnalysisException] {
+        df.write.json(path.toString)
+      }.getMessage
 
-    if (path.exists()) Utils.deleteRecursively(path)
+      assert(
+        message.contains("already exists"),
+        "We should complain that the path already exists.")
 
-    df.write.json(path.toString)
-    checkLoad()
+      if (path.exists()) Utils.deleteRecursively(path)
 
-    df.write.mode(SaveMode.Overwrite).json(path.toString)
-    checkLoad()
+      df.write.json(path.toString)
+      checkLoad()
 
-    // verify the append mode
-    df.write.mode(SaveMode.Append).json(path.toString)
-    val df2 = df.union(df)
-    df2.createOrReplaceTempView("jsonTable2")
+      df.write.mode(SaveMode.Overwrite).json(path.toString)
+      checkLoad()
 
-    checkLoad(df2, "jsonTable2")
+      // verify the append mode
+      df.write.mode(SaveMode.Append).json(path.toString)
+      val df2 = df.union(df)
+      df2.createOrReplaceTempView("jsonTable2")
+
+      checkLoad(df2, "jsonTable2")
+    }
   }
 
   test("SPARK-23459: Improve error message when specified unknown column in partition columns") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
index 5bcc4e90a2043..d8d997e1768bd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamSuite.scala
@@ -193,13 +193,15 @@ class StreamSuite extends StreamTest {
   }
 
   test("sql queries") {
-    val inputData = MemoryStream[Int]
-    inputData.toDF().createOrReplaceTempView("stream")
-    val evens = sql("SELECT * FROM stream WHERE value % 2 = 0")
-
-    testStream(evens)(
-      AddData(inputData, 1, 2, 3, 4),
-      CheckAnswer(2, 4))
+    withTempView("stream") {
+      val inputData = MemoryStream[Int]
+      inputData.toDF().createOrReplaceTempView("stream")
+      val evens = sql("SELECT * FROM stream WHERE value % 2 = 0")
+
+      testStream(evens)(
+        AddData(inputData, 1, 2, 3, 4),
+        CheckAnswer(2, 4))
+    }
   }
 
   test("DataFrame reuse") {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
index 8599ceb833ca4..0d17f2e0bc7fb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/continuous/ContinuousSuite.scala
@@ -181,17 +181,19 @@ class ContinuousSuite extends ContinuousSuiteBase {
   }
 
   test("subquery alias") {
-    val input = ContinuousMemoryStream[Int]
-    input.toDF().createOrReplaceTempView("memory")
-    val test = spark.sql("select value from memory where value > 2")
+    withTempView("memory") {
+      val input = ContinuousMemoryStream[Int]
+      input.toDF().createOrReplaceTempView("memory")
+      val test = spark.sql("select value from memory where value > 2")
 
-    testStream(test)(
-      AddData(input, 0, 1),
-      CheckAnswer(),
-      StopStream,
-      AddData(input, 2, 3, 4),
-      StartStream(),
-      CheckAnswer(3, 4))
+      testStream(test)(
+        AddData(input, 0, 1),
+        CheckAnswer(),
+        StopStream,
+        AddData(input, 2, 3, 4),
+        StartStream(),
+        CheckAnswer(3, 4))
+    }
   }
 
   test("repeatedly restart") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
index ce40a65ed6bac..fac981267f4d7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
@@ -204,11 +204,13 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
   }
 
   test("group by function") {
-    Seq((1, 2)).toDF("a", "b").createOrReplaceTempView("data")
+    withTempView("data") {
+      Seq((1, 2)).toDF("a", "b").createOrReplaceTempView("data")
 
-    checkAnswer(
-      sql("SELECT floor(a) AS a, collect_set(b) FROM data GROUP BY floor(a) ORDER BY a"),
-      Row(1, Array(2)) :: Nil)
+      checkAnswer(
+        sql("SELECT floor(a) AS a, collect_set(b) FROM data GROUP BY floor(a) ORDER BY a"),
+        Row(1, Array(2)) :: Nil)
+    }
   }
 
   test("empty table") {
@@ -800,43 +802,45 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
       (5, 8, 17),
       (6, 2, 11)).toDF("a", "b", "c")
 
-    covar_tab.createOrReplaceTempView("covar_tab")
+    withTempView("covar_tab") {
+      covar_tab.createOrReplaceTempView("covar_tab")
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT corr(b, c) FROM covar_tab WHERE a < 1
-        """.stripMargin),
-      Row(null) :: Nil)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT corr(b, c) FROM covar_tab WHERE a < 1
+          """.stripMargin),
+        Row(null) :: Nil)
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT corr(b, c) FROM covar_tab WHERE a < 3
-        """.stripMargin),
-      Row(null) :: Nil)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT corr(b, c) FROM covar_tab WHERE a < 3
+          """.stripMargin),
+        Row(null) :: Nil)
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT corr(b, c) FROM covar_tab WHERE a = 3
-        """.stripMargin),
-      Row(Double.NaN) :: Nil)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT corr(b, c) FROM covar_tab WHERE a = 3
+          """.stripMargin),
+        Row(Double.NaN) :: Nil)
 
-    checkAnswer(
-      spark.sql(
-        """
-          |SELECT a, corr(b, c) FROM covar_tab GROUP BY a ORDER BY a
-        """.stripMargin),
-      Row(1, null) ::
-      Row(2, null) ::
-      Row(3, Double.NaN) ::
-      Row(4, Double.NaN) ::
-      Row(5, Double.NaN) ::
-      Row(6, Double.NaN) :: Nil)
-
-    val corr7 = spark.sql("SELECT corr(b, c) FROM covar_tab").collect()(0).getDouble(0)
-    assert(math.abs(corr7 - 0.6633880657639323) < 1e-12)
+      checkAnswer(
+        spark.sql(
+          """
+            |SELECT a, corr(b, c) FROM covar_tab GROUP BY a ORDER BY a
+          """.stripMargin),
+        Row(1, null) ::
+        Row(2, null) ::
+        Row(3, Double.NaN) ::
+        Row(4, Double.NaN) ::
+        Row(5, Double.NaN) ::
+        Row(6, Double.NaN) :: Nil)
+
+      val corr7 = spark.sql("SELECT corr(b, c) FROM covar_tab").collect()(0).getDouble(0)
+      assert(math.abs(corr7 - 0.6633880657639323) < 1e-12)
+    }
   }
 
   test("covariance: covar_pop and covar_samp") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 08d2506376cd2..61e1fefb5b5df 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -2315,37 +2315,39 @@ class HiveDDLSuite
 
     implicit val _sqlContext = spark.sqlContext
 
-    Seq((1, "one"), (2, "two"), (4, "four")).toDF("number", "word").createOrReplaceTempView("t1")
-    // Make a table and ensure it will be broadcast.
-    sql("""CREATE TABLE smallTable(word string, number int)
-          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-          |STORED AS TEXTFILE
-        """.stripMargin)
+    withTempView("t1") {
+      Seq((1, "one"), (2, "two"), (4, "four")).toDF("number", "word").createOrReplaceTempView("t1")
+      // Make a table and ensure it will be broadcast.
+      sql("""CREATE TABLE smallTable(word string, number int)
+            |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+            |STORED AS TEXTFILE
+          """.stripMargin)
 
-    sql(
-      """INSERT INTO smallTable
-        |SELECT word, number from t1
-      """.stripMargin)
+      sql(
+        """INSERT INTO smallTable
+          |SELECT word, number from t1
+        """.stripMargin)
 
-    val inputData = MemoryStream[Int]
-    val joined = inputData.toDS().toDF()
-      .join(spark.table("smallTable"), $"value" === $"number")
+      val inputData = MemoryStream[Int]
+      val joined = inputData.toDS().toDF()
+        .join(spark.table("smallTable"), $"value" === $"number")
 
-    val sq = joined.writeStream
-      .format("memory")
-      .queryName("t2")
-      .start()
-    try {
-      inputData.addData(1, 2)
+      val sq = joined.writeStream
+        .format("memory")
+        .queryName("t2")
+        .start()
+      try {
+        inputData.addData(1, 2)
 
-      sq.processAllAvailable()
+        sq.processAllAvailable()
 
-      checkAnswer(
-        spark.table("t2"),
-        Seq(Row(1, "one", 1), Row(2, "two", 2))
-      )
-    } finally {
-      sq.stop()
+        checkAnswer(
+          spark.table("t2"),
+          Seq(Row(1, "one", 1), Row(2, "two", 2))
+        )
+      } finally {
+        sq.stop()
+      }
     }
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 5a8365017a5ba..1e89db2bdd01a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -698,15 +698,17 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     "SELECT srcalias.KEY, SRCALIAS.value FROM sRc SrCAlias WHERE SrCAlias.kEy < 15")
 
   test("case sensitivity: created temporary view") {
-    val testData =
-      TestHive.sparkContext.parallelize(
-        TestData(1, "str1") ::
-        TestData(2, "str2") :: Nil)
-    testData.toDF().createOrReplaceTempView("REGisteredTABle")
-
-    assertResult(Array(Row(2, "str2"))) {
-      sql("SELECT tablealias.A, TABLEALIAS.b FROM reGisteredTABle TableAlias " +
-        "WHERE TableAliaS.a > 1").collect()
+    withTempView("REGisteredTABle") {
+      val testData =
+        TestHive.sparkContext.parallelize(
+          TestData(1, "str1") ::
+          TestData(2, "str2") :: Nil)
+      testData.toDF().createOrReplaceTempView("REGisteredTABle")
+
+      assertResult(Array(Row(2, "str2"))) {
+        sql("SELECT tablealias.A, TABLEALIAS.b FROM reGisteredTABle TableAlias " +
+          "WHERE TableAliaS.a > 1").collect()
+      }
     }
   }
 
@@ -725,16 +727,18 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   }
 
   test("SPARK-2180: HAVING support in GROUP BY clauses (positive)") {
-    val fixture = List(("foo", 2), ("bar", 1), ("foo", 4), ("bar", 3))
-      .zipWithIndex.map {case ((value, attr), key) => HavingRow(key, value, attr)}
-    TestHive.sparkContext.parallelize(fixture).toDF().createOrReplaceTempView("having_test")
-    val results =
-      sql("SELECT value, max(attr) AS attr FROM having_test GROUP BY value HAVING attr > 3")
-      .collect()
-      .map(x => (x.getString(0), x.getInt(1)))
+    withTempView("having_test") {
+      val fixture = List(("foo", 2), ("bar", 1), ("foo", 4), ("bar", 3))
+        .zipWithIndex.map {case ((value, attr), key) => HavingRow(key, value, attr)}
+      TestHive.sparkContext.parallelize(fixture).toDF().createOrReplaceTempView("having_test")
+      val results =
+        sql("SELECT value, max(attr) AS attr FROM having_test GROUP BY value HAVING attr > 3")
+        .collect()
+        .map(x => (x.getString(0), x.getInt(1)))
 
-    assert(results === Array(("foo", 4)))
-    TestHive.reset()
+      assert(results === Array(("foo", 4)))
+      TestHive.reset()
+    }
   }
 
   test("SPARK-2180: HAVING with non-boolean clause raises no exceptions") {
@@ -966,22 +970,24 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   }
 
   test("SPARK-3414 regression: should store analyzed logical plan when creating a temporary view") {
-    sparkContext.makeRDD(Seq.empty[LogEntry]).toDF().createOrReplaceTempView("rawLogs")
-    sparkContext.makeRDD(Seq.empty[LogFile]).toDF().createOrReplaceTempView("logFiles")
+    withTempView("rawLogs", "logFiles", "boom") {
+      sparkContext.makeRDD(Seq.empty[LogEntry]).toDF().createOrReplaceTempView("rawLogs")
+      sparkContext.makeRDD(Seq.empty[LogFile]).toDF().createOrReplaceTempView("logFiles")
 
-    sql(
-      """
-      SELECT name, message
-      FROM rawLogs
-      JOIN (
-        SELECT name
-        FROM logFiles
-      ) files
-      ON rawLogs.filename = files.name
-      """).createOrReplaceTempView("boom")
-
-    // This should be successfully analyzed
-    sql("SELECT * FROM boom").queryExecution.analyzed
+      sql(
+        """
+        SELECT name, message
+        FROM rawLogs
+        JOIN (
+          SELECT name
+          FROM logFiles
+        ) files
+        ON rawLogs.filename = files.name
+        """).createOrReplaceTempView("boom")
+
+      // This should be successfully analyzed
+      sql("SELECT * FROM boom").queryExecution.analyzed
+    }
   }
 
   test("SPARK-3810: PreprocessTableInsertion static partitioning support") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index e93f585b5c86a..79c6ade2807d3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -89,24 +89,28 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("script") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    assume(TestUtils.testCommandAvailable("echo | sed"))
-    val scriptFilePath = getTestResourcePath("test_script.sh")
-    val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
-    df.createOrReplaceTempView("script_table")
-    val query1 = sql(
-      s"""
-        |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
-        |REDUCE c1, c2, c3 USING 'bash $scriptFilePath' AS
-        |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
-    checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+    withTempView("script_table") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      assume(TestUtils.testCommandAvailable("echo | sed"))
+      val scriptFilePath = getTestResourcePath("test_script.sh")
+      val df = Seq(("x1", "y1", "z1"), ("x2", "y2", "z2")).toDF("c1", "c2", "c3")
+      df.createOrReplaceTempView("script_table")
+      val query1 = sql(
+        s"""
+          |SELECT col1 FROM (from(SELECT c1, c2, c3 FROM script_table) tempt_table
+          |REDUCE c1, c2, c3 USING 'bash $scriptFilePath' AS
+          |(col1 STRING, col2 STRING)) script_test_table""".stripMargin)
+      checkAnswer(query1, Row("x1_y1") :: Row("x2_y2") :: Nil)
+    }
   }
 
   test("SPARK-6835: udtf in lateral view") {
-    val df = Seq((1, 1)).toDF("c1", "c2")
-    df.createOrReplaceTempView("table1")
-    val query = sql("SELECT c1, v FROM table1 LATERAL VIEW stack(3, 1, c1 + 1, c1 + 2) d AS v")
-    checkAnswer(query, Row(1, 1) :: Row(1, 2) :: Row(1, 3) :: Nil)
+    withTempView("table1") {
+      val df = Seq((1, 1)).toDF("c1", "c2")
+      df.createOrReplaceTempView("table1")
+      val query = sql("SELECT c1, v FROM table1 LATERAL VIEW stack(3, 1, c1 + 1, c1 + 2) d AS v")
+      checkAnswer(query, Row(1, 1) :: Row(1, 2) :: Row(1, 3) :: Nil)
+    }
   }
 
   test("SPARK-13651: generator outputs shouldn't be resolved from its child's output") {
@@ -120,70 +124,72 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("SPARK-6851: Self-joined converted parquet tables") {
-    val orders = Seq(
-      Order(1, "Atlas", "MTB", 234, "2015-01-07", "John D", "Pacifica", "CA", 20151),
-      Order(3, "Swift", "MTB", 285, "2015-01-17", "John S", "Redwood City", "CA", 20151),
-      Order(4, "Atlas", "Hybrid", 303, "2015-01-23", "Jones S", "San Mateo", "CA", 20151),
-      Order(7, "Next", "MTB", 356, "2015-01-04", "Jane D", "Daly City", "CA", 20151),
-      Order(10, "Next", "YFlikr", 187, "2015-01-09", "John D", "Fremont", "CA", 20151),
-      Order(11, "Swift", "YFlikr", 187, "2015-01-23", "John D", "Hayward", "CA", 20151),
-      Order(2, "Next", "Hybrid", 324, "2015-02-03", "Jane D", "Daly City", "CA", 20152),
-      Order(5, "Next", "Street", 187, "2015-02-08", "John D", "Fremont", "CA", 20152),
-      Order(6, "Atlas", "Street", 154, "2015-02-09", "John D", "Pacifica", "CA", 20152),
-      Order(8, "Swift", "Hybrid", 485, "2015-02-19", "John S", "Redwood City", "CA", 20152),
-      Order(9, "Atlas", "Split", 303, "2015-02-28", "Jones S", "San Mateo", "CA", 20152))
-
-    val orderUpdates = Seq(
-      Order(1, "Atlas", "MTB", 434, "2015-01-07", "John D", "Pacifica", "CA", 20151),
-      Order(11, "Swift", "YFlikr", 137, "2015-01-23", "John D", "Hayward", "CA", 20151))
-
-    orders.toDF.createOrReplaceTempView("orders1")
-    orderUpdates.toDF.createOrReplaceTempView("orderupdates1")
-
-    withTable("orders", "orderupdates") {
-      sql(
-        """CREATE TABLE orders(
-          |  id INT,
-          |  make String,
-          |  type String,
-          |  price INT,
-          |  pdate String,
-          |  customer String,
-          |  city String)
-          |PARTITIONED BY (state STRING, month INT)
-          |STORED AS PARQUET
-        """.stripMargin)
+    withTempView("orders1", "orderupdates1") {
+      val orders = Seq(
+        Order(1, "Atlas", "MTB", 234, "2015-01-07", "John D", "Pacifica", "CA", 20151),
+        Order(3, "Swift", "MTB", 285, "2015-01-17", "John S", "Redwood City", "CA", 20151),
+        Order(4, "Atlas", "Hybrid", 303, "2015-01-23", "Jones S", "San Mateo", "CA", 20151),
+        Order(7, "Next", "MTB", 356, "2015-01-04", "Jane D", "Daly City", "CA", 20151),
+        Order(10, "Next", "YFlikr", 187, "2015-01-09", "John D", "Fremont", "CA", 20151),
+        Order(11, "Swift", "YFlikr", 187, "2015-01-23", "John D", "Hayward", "CA", 20151),
+        Order(2, "Next", "Hybrid", 324, "2015-02-03", "Jane D", "Daly City", "CA", 20152),
+        Order(5, "Next", "Street", 187, "2015-02-08", "John D", "Fremont", "CA", 20152),
+        Order(6, "Atlas", "Street", 154, "2015-02-09", "John D", "Pacifica", "CA", 20152),
+        Order(8, "Swift", "Hybrid", 485, "2015-02-19", "John S", "Redwood City", "CA", 20152),
+        Order(9, "Atlas", "Split", 303, "2015-02-28", "Jones S", "San Mateo", "CA", 20152))
+
+      val orderUpdates = Seq(
+        Order(1, "Atlas", "MTB", 434, "2015-01-07", "John D", "Pacifica", "CA", 20151),
+        Order(11, "Swift", "YFlikr", 137, "2015-01-23", "John D", "Hayward", "CA", 20151))
+
+      orders.toDF.createOrReplaceTempView("orders1")
+      orderUpdates.toDF.createOrReplaceTempView("orderupdates1")
+
+      withTable("orders", "orderupdates") {
+        sql(
+          """CREATE TABLE orders(
+            |  id INT,
+            |  make String,
+            |  type String,
+            |  price INT,
+            |  pdate String,
+            |  customer String,
+            |  city String)
+            |PARTITIONED BY (state STRING, month INT)
+            |STORED AS PARQUET
+          """.stripMargin)
 
-      sql(
-        """CREATE TABLE orderupdates(
-          |  id INT,
-          |  make String,
-          |  type String,
-          |  price INT,
-          |  pdate String,
-          |  customer String,
-          |  city String)
-          |PARTITIONED BY (state STRING, month INT)
-          |STORED AS PARQUET
-        """.stripMargin)
+        sql(
+          """CREATE TABLE orderupdates(
+            |  id INT,
+            |  make String,
+            |  type String,
+            |  price INT,
+            |  pdate String,
+            |  customer String,
+            |  city String)
+            |PARTITIONED BY (state STRING, month INT)
+            |STORED AS PARQUET
+          """.stripMargin)
 
-      sql("set hive.exec.dynamic.partition.mode=nonstrict")
-      sql("INSERT INTO TABLE orders PARTITION(state, month) SELECT * FROM orders1")
-      sql("INSERT INTO TABLE orderupdates PARTITION(state, month) SELECT * FROM orderupdates1")
+        sql("set hive.exec.dynamic.partition.mode=nonstrict")
+        sql("INSERT INTO TABLE orders PARTITION(state, month) SELECT * FROM orders1")
+        sql("INSERT INTO TABLE orderupdates PARTITION(state, month) SELECT * FROM orderupdates1")
 
-      checkAnswer(
-        sql(
-          """
-            |select orders.state, orders.month
-            |from orders
-            |join (
-            |  select distinct orders.state,orders.month
-            |  from orders
-            |  join orderupdates
-            |    on orderupdates.id = orders.id) ao
-            |  on ao.state = orders.state and ao.month = orders.month
-          """.stripMargin),
-        (1 to 6).map(_ => Row("CA", 20151)))
+        checkAnswer(
+          sql(
+            """
+              |select orders.state, orders.month
+              |from orders
+              |join (
+              |  select distinct orders.state,orders.month
+              |  from orders
+              |  join orderupdates
+              |    on orderupdates.id = orders.id) ao
+              |  on ao.state = orders.state and ao.month = orders.month
+            """.stripMargin),
+          (1 to 6).map(_ => Row("CA", 20151)))
+      }
     }
   }
 
@@ -336,71 +342,76 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("SPARK-5371: union with null and sum") {
-    val df = Seq((1, 1)).toDF("c1", "c2")
-    df.createOrReplaceTempView("table1")
-
-    val query = sql(
-      """
-        |SELECT
-        |  MIN(c1),
-        |  MIN(c2)
-        |FROM (
-        |  SELECT
-        |    SUM(c1) c1,
-        |    NULL c2
-        |  FROM table1
-        |  UNION ALL
-        |  SELECT
-        |    NULL c1,
-        |    SUM(c2) c2
-        |  FROM table1
-        |) a
-      """.stripMargin)
-    checkAnswer(query, Row(1, 1) :: Nil)
-  }
-
-  test("CTAS with WITH clause") {
+    withTempView("table1") {
+      val df = Seq((1, 1)).toDF("c1", "c2")
+      df.createOrReplaceTempView("table1")
 
-    val df = Seq((1, 1)).toDF("c1", "c2")
-    df.createOrReplaceTempView("table1")
-    withTable("with_table1") {
-      sql(
+      val query = sql(
         """
-          |CREATE TABLE with_table1 AS
-          |WITH T AS (
-          |  SELECT *
+          |SELECT
+          |  MIN(c1),
+          |  MIN(c2)
+          |FROM (
+          |  SELECT
+          |    SUM(c1) c1,
+          |    NULL c2
+          |  FROM table1
+          |  UNION ALL
+          |  SELECT
+          |    NULL c1,
+          |    SUM(c2) c2
           |  FROM table1
-          |)
-          |SELECT *
-          |FROM T
+          |) a
         """.stripMargin)
-      val query = sql("SELECT * FROM with_table1")
       checkAnswer(query, Row(1, 1) :: Nil)
     }
   }
 
+  test("CTAS with WITH clause") {
+    withTempView("table1") {
+      val df = Seq((1, 1)).toDF("c1", "c2")
+      df.createOrReplaceTempView("table1")
+      withTable("with_table1") {
+        sql(
+          """
+            |CREATE TABLE with_table1 AS
+            |WITH T AS (
+            |  SELECT *
+            |  FROM table1
+            |)
+            |SELECT *
+            |FROM T
+          """.stripMargin)
+        val query = sql("SELECT * FROM with_table1")
+        checkAnswer(query, Row(1, 1) :: Nil)
+      }
+    }
+  }
+
   test("explode nested Field") {
-    Seq(NestedArray1(NestedArray2(Seq(1, 2, 3)))).toDF.createOrReplaceTempView("nestedArray")
-    checkAnswer(
-      sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+    withTempView("nestedArray") {
+      Seq(NestedArray1(NestedArray2(Seq(1, 2, 3)))).toDF.createOrReplaceTempView("nestedArray")
+      checkAnswer(
+        sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
 
-    checkAnswer(
-      sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(
+        sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
 
-    checkAnswer(
-      sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(
+        sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
 
-    checkAnswer(
-      sql(
-        """
-          |SELECT `weird``tab`.`weird``col`
-          |FROM nestedArray
-          |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`
-        """.stripMargin),
-      Row(1) :: Row(2) :: Row(3) :: Nil)
+      checkAnswer(
+        sql(
+          """
+            |SELECT `weird``tab`.`weird``col`
+            |FROM nestedArray
+            |LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`
+          """.stripMargin),
+        Row(1) :: Row(2) :: Row(3) :: Nil)
+    }
   }
 
   test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") {
@@ -742,20 +753,22 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("double nested data") {
-    withTable("test_ctas_1234") {
-      sparkContext.parallelize(Nested1(Nested2(Nested3(1))) :: Nil)
-        .toDF().createOrReplaceTempView("nested")
-      checkAnswer(
-        sql("SELECT f1.f2.f3 FROM nested"),
-        Row(1))
+    withTempView("nested") {
+      withTable("test_ctas_1234") {
+        sparkContext.parallelize(Nested1(Nested2(Nested3(1))) :: Nil)
+          .toDF().createOrReplaceTempView("nested")
+        checkAnswer(
+          sql("SELECT f1.f2.f3 FROM nested"),
+          Row(1))
 
-      sql("CREATE TABLE test_ctas_1234 AS SELECT * from nested")
-      checkAnswer(
-        sql("SELECT * FROM test_ctas_1234"),
-        sql("SELECT * FROM nested").collect().toSeq)
+        sql("CREATE TABLE test_ctas_1234 AS SELECT * from nested")
+        checkAnswer(
+          sql("SELECT * FROM test_ctas_1234"),
+          sql("SELECT * FROM nested").collect().toSeq)
 
-      intercept[AnalysisException] {
-        sql("CREATE TABLE test_ctas_1234 AS SELECT * from notexists").collect()
+        intercept[AnalysisException] {
+          sql("CREATE TABLE test_ctas_1234 AS SELECT * from notexists").collect()
+        }
       }
     }
   }
@@ -832,13 +845,15 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("SPARK-4963 DataFrame sample on mutable row return wrong result") {
-    sql("SELECT * FROM src WHERE key % 2 = 0")
-      .sample(withReplacement = false, fraction = 0.3)
-      .createOrReplaceTempView("sampled")
-    (1 to 10).foreach { i =>
-      checkAnswer(
-        sql("SELECT * FROM sampled WHERE key % 2 = 1"),
-        Seq.empty[Row])
+    withTempView("sampled") {
+      sql("SELECT * FROM src WHERE key % 2 = 0")
+        .sample(withReplacement = false, fraction = 0.3)
+        .createOrReplaceTempView("sampled")
+      (1 to 10).foreach { i =>
+        checkAnswer(
+          sql("SELECT * FROM sampled WHERE key % 2 = 1"),
+          Seq.empty[Row])
+      }
     }
   }
 
@@ -899,33 +914,38 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("resolve udtf in projection #1") {
-    val ds = (1 to 5).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
-    read.json(ds).createOrReplaceTempView("data")
-    val df = sql("SELECT explode(a) AS val FROM data")
-    val col = df("val")
+    withTempView("data") {
+      val ds = (1 to 5).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
+      read.json(ds).createOrReplaceTempView("data")
+      sql("SELECT explode(a) AS val FROM data")
+    }
   }
 
   test("resolve udtf in projection #2") {
-    val ds = (1 to 2).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
-    read.json(ds).createOrReplaceTempView("data")
-    checkAnswer(sql("SELECT explode(map(1, 1)) FROM data LIMIT 1"), Row(1, 1) :: Nil)
-    checkAnswer(sql("SELECT explode(map(1, 1)) as (k1, k2) FROM data LIMIT 1"), Row(1, 1) :: Nil)
-    intercept[AnalysisException] {
-      sql("SELECT explode(map(1, 1)) as k1 FROM data LIMIT 1")
-    }
+    withTempView("data") {
+      val ds = (1 to 2).map(i => s"""{"a":[$i, ${i + 1}]}""").toDS()
+      read.json(ds).createOrReplaceTempView("data")
+      checkAnswer(sql("SELECT explode(map(1, 1)) FROM data LIMIT 1"), Row(1, 1) :: Nil)
+      checkAnswer(sql("SELECT explode(map(1, 1)) as (k1, k2) FROM data LIMIT 1"), Row(1, 1) :: Nil)
+      intercept[AnalysisException] {
+        sql("SELECT explode(map(1, 1)) as k1 FROM data LIMIT 1")
+      }
 
-    intercept[AnalysisException] {
-      sql("SELECT explode(map(1, 1)) as (k1, k2, k3) FROM data LIMIT 1")
+      intercept[AnalysisException] {
+        sql("SELECT explode(map(1, 1)) as (k1, k2, k3) FROM data LIMIT 1")
+      }
     }
   }
 
   // TGF with non-TGF in project is allowed in Spark SQL, but not in Hive
   test("TGF with non-TGF in projection") {
-    val ds = Seq("""{"a": "1", "b":"1"}""").toDS()
-    read.json(ds).createOrReplaceTempView("data")
-    checkAnswer(
-      sql("SELECT explode(map(a, b)) as (k1, k2), a, b FROM data"),
-      Row("1", "1", "1", "1") :: Nil)
+    withTempView("data") {
+      val ds = Seq("""{"a": "1", "b":"1"}""").toDS()
+      read.json(ds).createOrReplaceTempView("data")
+      checkAnswer(
+        sql("SELECT explode(map(a, b)) as (k1, k2), a, b FROM data"),
+        Row("1", "1", "1", "1") :: Nil)
+    }
   }
 
   test("logical.Project should not be resolved if it contains aggregates or generators") {
@@ -975,47 +995,58 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("SPARK-5203 union with different decimal precision") {
-    Seq.empty[(java.math.BigDecimal, java.math.BigDecimal)]
-      .toDF("d1", "d2")
-      .select($"d1".cast(DecimalType(10, 5)).as("d"))
-      .createOrReplaceTempView("dn")
+    withTempView("dn") {
+      Seq.empty[(java.math.BigDecimal, java.math.BigDecimal)]
+        .toDF("d1", "d2")
+        .select($"d1".cast(DecimalType(10, 5)).as("d"))
+        .createOrReplaceTempView("dn")
 
-    sql("select d from dn union all select d * 2 from dn")
-      .queryExecution.analyzed
+      sql("select d from dn union all select d * 2 from dn")
+        .queryExecution.analyzed
+    }
   }
 
   test("Star Expansion - script transform") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 100000).map { i => (i, i, i) }
-    data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
-    assert(100000 === sql("SELECT TRANSFORM (*) USING 'cat' FROM script_trans").count())
+    withTempView("script_trans") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 100000).map { i => (i, i, i) }
+      data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
+      assert(100000 === sql("SELECT TRANSFORM (*) USING 'cat' FROM script_trans").count())
+    }
   }
 
   test("test script transform for stdout") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 100000).map { i => (i, i, i) }
-    data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
-    assert(100000 ===
-      sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat' AS (a,b,c) FROM script_trans").count())
+    withTempView("script_trans") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 100000).map { i => (i, i, i) }
+      data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
+      assert(100000 ===
+        sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat' AS (a,b,c) FROM script_trans").count())
+    }
   }
 
   test("test script transform for stderr") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 100000).map { i => (i, i, i) }
-    data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
-    assert(0 ===
-      sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat 1>&2' AS (a,b,c) FROM script_trans").count())
+    withTempView("script_trans") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 100000).map { i => (i, i, i) }
+      data.toDF("d1", "d2", "d3").createOrReplaceTempView("script_trans")
+      assert(0 ===
+        sql("SELECT TRANSFORM (d1, d2, d3) USING 'cat 1>&2' AS (a,b,c) FROM script_trans").count())
+    }
   }
 
   test("test script transform data type") {
-    assume(TestUtils.testCommandAvailable("/bin/bash"))
-    val data = (1 to 5).map { i => (i, i) }
-    data.toDF("key", "value").createOrReplaceTempView("test")
-    checkAnswer(
-      sql("""FROM
-          |(FROM test SELECT TRANSFORM(key, value) USING 'cat' AS (`thing1` int, thing2 string)) t
-          |SELECT thing1 + 1
-        """.stripMargin), (2 to 6).map(i => Row(i)))
+    withTempView("test") {
+      assume(TestUtils.testCommandAvailable("/bin/bash"))
+      val data = (1 to 5).map { i => (i, i) }
+      data.toDF("key", "value").createOrReplaceTempView("test")
+      checkAnswer(
+        sql(
+          """FROM
+            |(FROM test SELECT TRANSFORM(key, value) USING 'cat' AS (`thing1` int, thing2 string)) t
+            |SELECT thing1 + 1
+          """.stripMargin), (2 to 6).map(i => Row(i)))
+    }
   }
 
   test("Sorting columns are not in Generate") {
@@ -1046,27 +1077,31 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("test case key when") {
-    (1 to 5).map(i => (i, i.toString)).toDF("k", "v").createOrReplaceTempView("t")
-    checkAnswer(
-      sql("SELECT CASE k WHEN 2 THEN 22 WHEN 4 THEN 44 ELSE 0 END, v FROM t"),
-      Row(0, "1") :: Row(22, "2") :: Row(0, "3") :: Row(44, "4") :: Row(0, "5") :: Nil)
+    withTempView("t") {
+      (1 to 5).map(i => (i, i.toString)).toDF("k", "v").createOrReplaceTempView("t")
+      checkAnswer(
+        sql("SELECT CASE k WHEN 2 THEN 22 WHEN 4 THEN 44 ELSE 0 END, v FROM t"),
+        Row(0, "1") :: Row(22, "2") :: Row(0, "3") :: Row(44, "4") :: Row(0, "5") :: Nil)
+    }
   }
 
   test("SPARK-7269 Check analysis failed in case in-sensitive") {
-    Seq(1, 2, 3).map { i =>
-      (i.toString, i.toString)
-    }.toDF("key", "value").createOrReplaceTempView("df_analysis")
-    sql("SELECT kEy from df_analysis group by key").collect()
-    sql("SELECT kEy+3 from df_analysis group by key+3").collect()
-    sql("SELECT kEy+3, a.kEy, A.kEy from df_analysis A group by key").collect()
-    sql("SELECT cast(kEy+1 as Int) from df_analysis A group by cast(key+1 as int)").collect()
-    sql("SELECT cast(kEy+1 as Int) from df_analysis A group by key+1").collect()
-    sql("SELECT 2 from df_analysis A group by key+1").collect()
-    intercept[AnalysisException] {
-      sql("SELECT kEy+1 from df_analysis group by key+3")
-    }
-    intercept[AnalysisException] {
-      sql("SELECT cast(key+2 as Int) from df_analysis A group by cast(key+1 as int)")
+    withTempView("df_analysis") {
+      Seq(1, 2, 3).map { i =>
+        (i.toString, i.toString)
+      }.toDF("key", "value").createOrReplaceTempView("df_analysis")
+      sql("SELECT kEy from df_analysis group by key").collect()
+      sql("SELECT kEy+3 from df_analysis group by key+3").collect()
+      sql("SELECT kEy+3, a.kEy, A.kEy from df_analysis A group by key").collect()
+      sql("SELECT cast(kEy+1 as Int) from df_analysis A group by cast(key+1 as int)").collect()
+      sql("SELECT cast(kEy+1 as Int) from df_analysis A group by key+1").collect()
+      sql("SELECT 2 from df_analysis A group by key+1").collect()
+      intercept[AnalysisException] {
+        sql("SELECT kEy+1 from df_analysis group by key+3")
+      }
+      intercept[AnalysisException] {
+        sql("SELECT cast(key+2 as Int) from df_analysis A group by cast(key+1 as int)")
+      }
     }
   }
 
@@ -1179,10 +1214,12 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 
   test("SPARK-9371: fix the support for special chars in column names for hive context") {
-    val ds = Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS()
-    read.json(ds).createOrReplaceTempView("t")
+    withTempView("t") {
+      val ds = Seq("""{"a": {"c.b": 1}, "b.$q": [{"a@!.q": 1}], "q.w": {"w.i&": [1]}}""").toDS()
+      read.json(ds).createOrReplaceTempView("t")
 
-    checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+      checkAnswer(sql("SELECT a.`c.b`, `b.$q`[0].`a@!.q`, `q.w`.`w.i&`[0] FROM t"), Row(1, 1, 1))
+    }
   }
 
   test("specifying database name for a temporary view is not allowed") {
@@ -1237,43 +1274,47 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
 
   ignore("SPARK-10310: " +
     "script transformation using default input/output SerDe and record reader/writer") {
-    spark
-      .range(5)
-      .selectExpr("id AS a", "id AS b")
-      .createOrReplaceTempView("test")
+    withTempView("test") {
+      spark
+        .range(5)
+        .selectExpr("id AS a", "id AS b")
+        .createOrReplaceTempView("test")
 
-    val scriptFilePath = getTestResourcePath("data")
-    checkAnswer(
-      sql(
-        s"""FROM(
-          |  FROM test SELECT TRANSFORM(a, b)
-          |  USING 'python $scriptFilePath/scripts/test_transform.py "\t"'
-          |  AS (c STRING, d STRING)
-          |) t
-          |SELECT c
-        """.stripMargin),
-      (0 until 5).map(i => Row(i + "#")))
+      val scriptFilePath = getTestResourcePath("data")
+      checkAnswer(
+        sql(
+          s"""FROM(
+            |  FROM test SELECT TRANSFORM(a, b)
+            |  USING 'python $scriptFilePath/scripts/test_transform.py "\t"'
+            |  AS (c STRING, d STRING)
+            |) t
+            |SELECT c
+          """.stripMargin),
+        (0 until 5).map(i => Row(i + "#")))
+    }
   }
 
   ignore("SPARK-10310: script transformation using LazySimpleSerDe") {
-    spark
-      .range(5)
-      .selectExpr("id AS a", "id AS b")
-      .createOrReplaceTempView("test")
-
-    val scriptFilePath = getTestResourcePath("data")
-    val df = sql(
-      s"""FROM test
-        |SELECT TRANSFORM(a, b)
-        |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-        |WITH SERDEPROPERTIES('field.delim' = '|')
-        |USING 'python $scriptFilePath/scripts/test_transform.py "|"'
-        |AS (c STRING, d STRING)
-        |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
-        |WITH SERDEPROPERTIES('field.delim' = '|')
-      """.stripMargin)
+    withTempView("test") {
+      spark
+        .range(5)
+        .selectExpr("id AS a", "id AS b")
+        .createOrReplaceTempView("test")
+
+      val scriptFilePath = getTestResourcePath("data")
+      val df = sql(
+        s"""FROM test
+          |SELECT TRANSFORM(a, b)
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+          |WITH SERDEPROPERTIES('field.delim' = '|')
+          |USING 'python $scriptFilePath/scripts/test_transform.py "|"'
+          |AS (c STRING, d STRING)
+          |ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+          |WITH SERDEPROPERTIES('field.delim' = '|')
+        """.stripMargin)
 
-    checkAnswer(df, (0 until 5).map(i => Row(i + "#", i + "#")))
+      checkAnswer(df, (0 until 5).map(i => Row(i + "#", i + "#")))
+    }
   }
 
   test("SPARK-10741: Sort on Aggregate using parquet") {

From ccde0a1ae2d880585cb554cc67f75ef972a78c67 Mon Sep 17 00:00:00 2001
From: Dilip Biswal <dkbiswal@gmail.com>
Date: Tue, 5 May 2020 15:21:14 +0900
Subject: [PATCH 0632/1280] [SPARK-31030][DOCS][FOLLOWUP] Replace HTML Table by
 Markdown Table

### What changes were proposed in this pull request?
This PR is to clean up the markdown file in remaining pages in sql reference. The first one was done by gatorsmile in  [28415](https://github.com/apache/spark/pull/28415)

- Replace HTML table by MD table
- **sql-ref-ansi-compliance.md**
<img width="967" alt="Screen Shot 2020-05-01 at 4 36 35 PM" src="https://user-images.githubusercontent.com/14225158/80848981-1cbca080-8bca-11ea-8a5d-63174b31c800.png">
- **sql-ref-datatypes.md (Scala)**
<img width="967" alt="Screen Shot 2020-05-01 at 4 37 30 PM" src="https://user-images.githubusercontent.com/14225158/80849057-6a390d80-8bca-11ea-8866-ab08bab31432.png">
<img width="967" alt="Screen Shot 2020-05-01 at 4 39 18 PM" src="https://user-images.githubusercontent.com/14225158/80849061-6c9b6780-8bca-11ea-834c-eb93d3ab47ae.png">
- **sql-ref-datatypes.md (Java)**
<img width="967" alt="Screen Shot 2020-05-01 at 4 41 24 PM" src="https://user-images.githubusercontent.com/14225158/80849138-b3895d00-8bca-11ea-9d3b-555acad2086c.png">
<img width="967" alt="Screen Shot 2020-05-01 at 4 41 39 PM" src="https://user-images.githubusercontent.com/14225158/80849140-b6844d80-8bca-11ea-9ca9-1812b6a76c02.png">
- **sql-ref-datatypes.md (Python)**
<img width="967" alt="Screen Shot 2020-05-01 at 4 43 36 PM" src="https://user-images.githubusercontent.com/14225158/80849202-0400ba80-8bcb-11ea-96a5-7caecbf9dbbf.png">
<img width="967" alt="Screen Shot 2020-05-01 at 4 43 54 PM" src="https://user-images.githubusercontent.com/14225158/80849205-06fbab00-8bcb-11ea-8f00-6df52b151684.png">
- **sql-ref-datatypes.md (R)**
<img width="967" alt="Screen Shot 2020-05-01 at 4 45 16 PM" src="https://user-images.githubusercontent.com/14225158/80849288-5fcb4380-8bcb-11ea-8277-8589b5bb31bc.png">
<img width="967" alt="Screen Shot 2020-05-01 at 4 45 36 PM" src="https://user-images.githubusercontent.com/14225158/80849294-62c63400-8bcb-11ea-9438-b4f1193bc757.png">
- **sql-ref-datatypes.md (SQL)**
<img width="967" alt="Screen Shot 2020-05-01 at 4 48 02 PM" src="https://user-images.githubusercontent.com/14225158/80849336-986b1d00-8bcb-11ea-9736-5fb40496b681.png">
- **sql-ref-syntax-qry-select-tvf.md**
<img width="967" alt="Screen Shot 2020-05-01 at 4 49 32 PM" src="https://user-images.githubusercontent.com/14225158/80849399-d10af680-8bcb-11ea-8dc2-e3e750e21a59.png">

### Why are the changes needed?
Make the doc cleaner and easily editable by MD editors

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Manually using jekyll serve

Closes #28433 from dilipbiswal/sql-doc-table-cleanup.

Authored-by: Dilip Biswal <dkbiswal@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 5052d9557d964c07d0b8bd2e2b08ede7c6958118)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-ansi-compliance.md       | 542 ++++++++++----------
 docs/sql-ref-datatypes.md             | 695 ++++----------------------
 docs/sql-ref-datetime-pattern.md      |   8 +-
 docs/sql-ref-null-semantics.md        | 131 ++---
 docs/sql-ref-syntax-qry-select-tvf.md |  33 +-
 5 files changed, 388 insertions(+), 1021 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 6cf16535eecf8..93fb10b24e99f 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -27,35 +27,10 @@ The casting behaviours are defined as store assignment rules in the standard.
 
 When `spark.sql.storeAssignmentPolicy` is set to `ANSI`, Spark SQL complies with the ANSI store assignment rules. This is a separate configuration because its default value is `ANSI`, while the configuration `spark.sql.ansi.enabled` is disabled by default.
 
-<table class="table">
-<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
-<tr>
-  <td><code>spark.sql.ansi.enabled</code></td>
-  <td>false</td>
-  <td>
-    (Experimental) When true, Spark tries to conform to the ANSI SQL specification:
-    1. Spark will throw a runtime exception if an overflow occurs in any operation on integral/decimal field.
-    2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in the SQL parser.
-  </td>
-  <td>3.0.0</td>
-</tr>
-<tr>
-  <td><code>spark.sql.storeAssignmentPolicy</code></td>
-  <td>ANSI</td>
-  <td>
-    (Experimental) When inserting a value into a column with different data type, Spark will perform type coercion.
-    Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy,
-    Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL.
-    It disallows certain unreasonable type conversions such as converting string to int or double to boolean.
-    With legacy policy, Spark allows the type coercion as long as it is a valid Cast, which is very loose.
-    e.g. converting string to int or double to boolean is allowed.
-    It is also the only behavior in Spark 2.x and it is compatible with Hive.
-    With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion,
-    e.g. converting double to int or decimal to double is not allowed.
-  </td>
-  <td>3.0.0</td>
-</tr>
-</table>
+|Property Name|Default|Meaning|Since Version|
+|-------------|-------|-------|-------------|
+|`spark.sql.ansi.enabled`|false|(Experimental) When true, Spark tries to conform to the ANSI SQL specification: <br/> 1. Spark will throw a runtime exception if an overflow occurs in any operation on integral/decimal field. <br/> 2. Spark will forbid using the reserved keywords of ANSI SQL as identifiers in the SQL parser.|3.0.0|
+|`spark.sql.storeAssignmentPolicy`|ANSI|(Experimental) When inserting a value into a column with different data type, Spark will perform type coercion.  Currently, we support 3 policies for the type coercion rules: ANSI, legacy and strict. With ANSI policy, Spark performs the type coercion as per ANSI SQL. In practice, the behavior is mostly the same as PostgreSQL.  It disallows certain unreasonable type conversions such as converting string to int or double to boolean.  With legacy policy, Spark allows the type coercion as long as it is a valid Cast, which is very loose.  e.g. converting string to int or double to boolean is allowed.  It is also the only behavior in Spark 2.x and it is compatible with Hive.  With strict policy, Spark doesn't allow any possible precision loss or data truncation in type coercion, e.g. converting double to int or decimal to double is not allowed.|3.0.0|
 
 The following subsections present behaviour changes in arithmetic operations, type conversions, and SQL parsing when the ANSI mode enabled.
 
@@ -152,259 +127,256 @@ By default `spark.sql.ansi.enabled` is false.
 
 Below is a list of all the keywords in Spark SQL.
 
-<table class="table">
-  <tr><th rowspan="2" style="vertical-align: middle;"><b>Keyword</b></th><th colspan="2"><b>Spark SQL</b></th><th rowspan="2" style="vertical-align: middle;"><b>SQL-2011</b></th></tr>
-  <tr><th><b>ANSI mode</b></th><th><b>default mode</b></th></tr>
-  <tr><td>ADD</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>AFTER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ALL</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ALTER</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ANALYZE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>AND</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ANTI</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ANY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ARCHIVE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ARRAY</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>AS</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ASC</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>AT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>AUTHORIZATION</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>BETWEEN</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>BOTH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>BUCKET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>BUCKETS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>BY</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CACHE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CASCADE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CASE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CAST</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CHANGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CHECK</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CLEAR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CLUSTER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CLUSTERED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CODEGEN</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COLLATE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COLLECTION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COLUMN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COLUMNS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMMENT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMMIT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COMPACT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMPACTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>COMPUTE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CONCATENATE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CONSTRAINT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>COST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>CREATE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CROSS</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>CUBE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_DATE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_TIME</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_TIMESTAMP</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>CURRENT_USER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DATA</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DATABASE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DATABASES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DAY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DBPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DEFINED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DELETE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DELIMITED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DESC</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DESCRIBE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DFS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DIRECTORIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DIRECTORY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DISTINCT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>DISTRIBUTE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DIV</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>DROP</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ELSE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>END</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ESCAPE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ESCAPED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXCEPT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>EXCHANGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXISTS</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>EXPLAIN</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXPORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXTENDED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>EXTERNAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>EXTRACT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FALSE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FETCH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FIELDS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FILTER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FILEFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FIRST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FOLLOWING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FOR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FOREIGN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FORMATTED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>FROM</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FULL</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>FUNCTION</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>FUNCTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>GLOBAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>GRANT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>GROUP</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>GROUPING</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>HAVING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>HOUR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>IF</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>IGNORE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>IMPORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>IN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>INDEX</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INDEXES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INNER</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>INPATH</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INPUTFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>INSERT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>INTERSECT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>INTERVAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>INTO</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>IS</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ITEMS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>JOIN</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>KEYS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LAST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LATERAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LAZY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LEADING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LEFT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>LIKE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LIMIT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LINES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LIST</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOAD</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOCAL</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>LOCATION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOCKS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>LOGICAL</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MACRO</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MAP</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MATCHED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MERGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MINUS</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>MINUTE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>MONTH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>MSCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>NAMESPACE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>NAMESPACES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>NATURAL</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>NO</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>NOT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>NULL</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>NULLS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OF</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ON</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>ONLY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OPTION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OPTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ORDER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OUT</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OUTER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OUTPUTFORMAT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OVER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OVERLAPS</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>OVERLAY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>OVERWRITE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PARTITION</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>PARTITIONED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PARTITIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PERCENT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PIVOT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PLACING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>POSITION</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>PRECEDING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PRIMARY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>PRINCIPALS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>PURGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>QUERY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RANGE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>RECORDREADER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RECORDWRITER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RECOVER</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REDUCE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REFERENCES</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>REFRESH</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RENAME</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REPAIR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REPLACE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RESET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>RESTRICT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>REVOKE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>RIGHT</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>RLIKE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ROLE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ROLES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>ROLLBACK</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ROLLUP</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ROW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>ROWS</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SCHEMA</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SECOND</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SELECT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SEMI</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SEPARATED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SERDE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SERDEPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SESSION_USER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SET</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SETS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SHOW</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SKEWED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SOME</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>SORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SORTED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>START</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>STATISTICS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>STORED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>STRATIFY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>STRUCT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SUBSTR</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>SUBSTRING</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TABLE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TABLES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TABLESAMPLE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TBLPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TEMPORARY</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TERMINATED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>THEN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TO</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TOUCH</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRAILING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TRANSACTION</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRANSACTIONS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRANSFORM</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRIM</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>TRUE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>TRUNCATE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNARCHIVE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNBOUNDED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNCACHE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNION</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNIQUE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNKNOWN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>UNLOCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UNSET</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>UPDATE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>USE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>USER</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>USING</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
-  <tr><td>VALUES</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>VIEW</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>VIEWS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
-  <tr><td>WHEN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>WHERE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>WINDOW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>WITH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-  <tr><td>YEAR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
-</table>
+|Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>Default Mode|SQL-2011|
+|-------|----------------------|-------------------------|--------|
+|ADD|non-reserved|non-reserved|non-reserved|
+|AFTER|non-reserved|non-reserved|non-reserved|
+|ALL|reserved|non-reserved|reserved|
+|ALTER|non-reserved|non-reserved|reserved|
+|ANALYZE|non-reserved|non-reserved|non-reserved|
+|AND|reserved|non-reserved|reserved|
+|ANTI|reserved|strict-non-reserved|non-reserved|
+|ANY|reserved|non-reserved|reserved|
+|ARCHIVE|non-reserved|non-reserved|non-reserved|
+|ARRAY|non-reserved|non-reserved|reserved|
+|AS|reserved|non-reserved|reserved|
+|ASC|non-reserved|non-reserved|non-reserved|
+|AT|non-reserved|non-reserved|reserved|
+|AUTHORIZATION|reserved|non-reserved|reserved|
+|BETWEEN|non-reserved|non-reserved|reserved|
+|BOTH|reserved|non-reserved|reserved|
+|BUCKET|non-reserved|non-reserved|non-reserved|
+|BUCKETS|non-reserved|non-reserved|non-reserved|
+|BY|non-reserved|non-reserved|reserved|
+|CACHE|non-reserved|non-reserved|non-reserved|
+|CASCADE|non-reserved|non-reserved|reserved|
+|CASE|reserved|non-reserved|reserved|
+|CAST|reserved|non-reserved|reserved|
+|CHANGE|non-reserved|non-reserved|non-reserved|
+|CHECK|reserved|non-reserved|reserved|
+|CLEAR|non-reserved|non-reserved|non-reserved|
+|CLUSTER|non-reserved|non-reserved|non-reserved|
+|CLUSTERED|non-reserved|non-reserved|non-reserved|
+|CODEGEN|non-reserved|non-reserved|non-reserved|
+|COLLATE|reserved|non-reserved|reserved|
+|COLLECTION|non-reserved|non-reserved|non-reserved|
+|COLUMN|reserved|non-reserved|reserved|
+|COLUMNS|non-reserved|non-reserved|non-reserved|
+|COMMENT|non-reserved|non-reserved|non-reserved|
+|COMMIT|non-reserved|non-reserved|reserved|
+|COMPACT|non-reserved|non-reserved|non-reserved|
+|COMPACTIONS|non-reserved|non-reserved|non-reserved|
+|COMPUTE|non-reserved|non-reserved|non-reserved|
+|CONCATENATE|non-reserved|non-reserved|non-reserved|
+|CONSTRAINT|reserved|non-reserved|reserved|
+|COST|non-reserved|non-reserved|non-reserved|
+|CREATE|reserved|non-reserved|reserved|
+|CROSS|reserved|strict-non-reserved|reserved|
+|CUBE|non-reserved|non-reserved|reserved|
+|CURRENT|non-reserved|non-reserved|reserved|
+|CURRENT_DATE|reserved|non-reserved|reserved|
+|CURRENT_TIME|reserved|non-reserved|reserved|
+|CURRENT_TIMESTAMP|reserved|non-reserved|reserved|
+|CURRENT_USER|reserved|non-reserved|reserved|
+|DATA|non-reserved|non-reserved|non-reserved|
+|DATABASE|non-reserved|non-reserved|non-reserved|
+|DATABASES|non-reserved|non-reserved|non-reserved|
+|DAY|reserved|non-reserved|reserved|
+|DBPROPERTIES|non-reserved|non-reserved|non-reserved|
+|DEFINED|non-reserved|non-reserved|non-reserved|
+|DELETE|non-reserved|non-reserved|reserved|
+|DELIMITED|non-reserved|non-reserved|non-reserved|
+|DESC|non-reserved|non-reserved|non-reserved|
+|DESCRIBE|non-reserved|non-reserved|reserved|
+|DFS|non-reserved|non-reserved|non-reserved|
+|DIRECTORIES|non-reserved|non-reserved|non-reserved|
+|DIRECTORY|non-reserved|non-reserved|non-reserved|
+|DISTINCT|reserved|non-reserved|reserved|
+|DISTRIBUTE|non-reserved|non-reserved|non-reserved|
+|DIV|non-reserved|non-reserved|non-reserved|
+|DROP|non-reserved|non-reserved|reserved|
+|ELSE|reserved|non-reserved|reserved|
+|END|reserved|non-reserved|reserved|
+|ESCAPE|reserved|non-reserved|reserved|
+|ESCAPED|non-reserved|non-reserved|non-reserved|
+|EXCEPT|reserved|strict-non-reserved|reserved|
+|EXCHANGE|non-reserved|non-reserved|non-reserved|
+|EXISTS|non-reserved|non-reserved|reserved|
+|EXPLAIN|non-reserved|non-reserved|non-reserved|
+|EXPORT|non-reserved|non-reserved|non-reserved|
+|EXTENDED|non-reserved|non-reserved|non-reserved|
+|EXTERNAL|non-reserved|non-reserved|reserved|
+|EXTRACT|non-reserved|non-reserved|reserved|
+|FALSE|reserved|non-reserved|reserved|
+|FETCH|reserved|non-reserved|reserved|
+|FIELDS|non-reserved|non-reserved|non-reserved|
+|FILTER|reserved|non-reserved|reserved|
+|FILEFORMAT|non-reserved|non-reserved|non-reserved|
+|FIRST|non-reserved|non-reserved|non-reserved|
+|FOLLOWING|non-reserved|non-reserved|non-reserved|
+|FOR|reserved|non-reserved|reserved|
+|FOREIGN|reserved|non-reserved|reserved|
+|FORMAT|non-reserved|non-reserved|non-reserved|
+|FORMATTED|non-reserved|non-reserved|non-reserved|
+|FROM|reserved|non-reserved|reserved|
+|FULL|reserved|strict-non-reserved|reserved|
+|FUNCTION|non-reserved|non-reserved|reserved|
+|FUNCTIONS|non-reserved|non-reserved|non-reserved|
+|GLOBAL|non-reserved|non-reserved|reserved|
+|GRANT|reserved|non-reserved|reserved|
+|GROUP|reserved|non-reserved|reserved|
+|GROUPING|non-reserved|non-reserved|reserved|
+|HAVING|reserved|non-reserved|reserved|
+|HOUR|reserved|non-reserved|reserved|
+|IF|non-reserved|non-reserved|reserved|
+|IGNORE|non-reserved|non-reserved|non-reserved|
+|IMPORT|non-reserved|non-reserved|non-reserved|
+|IN|reserved|non-reserved|reserved|
+|INDEX|non-reserved|non-reserved|non-reserved|
+|INDEXES|non-reserved|non-reserved|non-reserved|
+|INNER|reserved|strict-non-reserved|reserved|
+|INPATH|non-reserved|non-reserved|non-reserved|
+|INPUTFORMAT|non-reserved|non-reserved|non-reserved|
+|INSERT|non-reserved|non-reserved|reserved|
+|INTERSECT|reserved|strict-non-reserved|reserved|
+|INTERVAL|non-reserved|non-reserved|reserved|
+|INTO|reserved|non-reserved|reserved|
+|IS|reserved|non-reserved|reserved|
+|ITEMS|non-reserved|non-reserved|non-reserved|
+|JOIN|reserved|strict-non-reserved|reserved|
+|KEYS|non-reserved|non-reserved|non-reserved|
+|LAST|non-reserved|non-reserved|non-reserved|
+|LATERAL|non-reserved|non-reserved|reserved|
+|LAZY|non-reserved|non-reserved|non-reserved|
+|LEADING|reserved|non-reserved|reserved|
+|LEFT|reserved|strict-non-reserved|reserved|
+|LIKE|non-reserved|non-reserved|reserved|
+|LIMIT|non-reserved|non-reserved|non-reserved|
+|LINES|non-reserved|non-reserved|non-reserved|
+|LIST|non-reserved|non-reserved|non-reserved|
+|LOAD|non-reserved|non-reserved|non-reserved|
+|LOCAL|non-reserved|non-reserved|reserved|
+|LOCATION|non-reserved|non-reserved|non-reserved|
+|LOCK|non-reserved|non-reserved|non-reserved|
+|LOCKS|non-reserved|non-reserved|non-reserved|
+|LOGICAL|non-reserved|non-reserved|non-reserved|
+|MACRO|non-reserved|non-reserved|non-reserved|
+|MAP|non-reserved|non-reserved|non-reserved|
+|MATCHED|non-reserved|non-reserved|non-reserved|
+|MERGE|non-reserved|non-reserved|non-reserved|
+|MINUS|reserved|strict-non-reserved|non-reserved|
+|MINUTE|reserved|non-reserved|reserved|
+|MONTH|reserved|non-reserved|reserved|
+|MSCK|non-reserved|non-reserved|non-reserved|
+|NAMESPACE|non-reserved|non-reserved|non-reserved|
+|NAMESPACES|non-reserved|non-reserved|non-reserved|
+|NATURAL|reserved|strict-non-reserved|reserved|
+|NO|non-reserved|non-reserved|reserved|
+|NOT|reserved|non-reserved|reserved|
+|NULL|reserved|non-reserved|reserved|
+|NULLS|non-reserved|non-reserved|non-reserved|
+|OF|non-reserved|non-reserved|reserved|
+|ON|reserved|strict-non-reserved|reserved|
+|ONLY|reserved|non-reserved|reserved|
+|OPTION|non-reserved|non-reserved|non-reserved|
+|OPTIONS|non-reserved|non-reserved|non-reserved|
+|OR|reserved|non-reserved|reserved|
+|ORDER|reserved|non-reserved|reserved|
+|OUT|non-reserved|non-reserved|reserved|
+|OUTER|reserved|non-reserved|reserved|
+|OUTPUTFORMAT|non-reserved|non-reserved|non-reserved|
+|OVER|non-reserved|non-reserved|non-reserved|
+|OVERLAPS|reserved|non-reserved|reserved|
+|OVERLAY|non-reserved|non-reserved|non-reserved|
+|OVERWRITE|non-reserved|non-reserved|non-reserved|
+|PARTITION|non-reserved|non-reserved|reserved|
+|PARTITIONED|non-reserved|non-reserved|non-reserved|
+|PARTITIONS|non-reserved|non-reserved|non-reserved|
+|PERCENT|non-reserved|non-reserved|non-reserved|
+|PIVOT|non-reserved|non-reserved|non-reserved|
+|PLACING|non-reserved|non-reserved|non-reserved|
+|POSITION|non-reserved|non-reserved|reserved|
+|PRECEDING|non-reserved|non-reserved|non-reserved|
+|PRIMARY|reserved|non-reserved|reserved|
+|PRINCIPALS|non-reserved|non-reserved|non-reserved|
+|PROPERTIES|non-reserved|non-reserved|non-reserved|
+|PURGE|non-reserved|non-reserved|non-reserved|
+|QUERY|non-reserved|non-reserved|non-reserved|
+|RECORDREADER|non-reserved|non-reserved|non-reserved|
+|RECORDWRITER|non-reserved|non-reserved|non-reserved|
+|RECOVER|non-reserved|non-reserved|non-reserved|
+|REDUCE|non-reserved|non-reserved|non-reserved|
+|REFERENCES|reserved|non-reserved|reserved|
+|REFRESH|non-reserved|non-reserved|non-reserved|
+|RENAME|non-reserved|non-reserved|non-reserved|
+|REPAIR|non-reserved|non-reserved|non-reserved|
+|REPLACE|non-reserved|non-reserved|non-reserved|
+|RESET|non-reserved|non-reserved|non-reserved|
+|RESTRICT|non-reserved|non-reserved|non-reserved|
+|REVOKE|non-reserved|non-reserved|reserved|
+|RIGHT|reserved|strict-non-reserved|reserved|
+|RLIKE|non-reserved|non-reserved|non-reserved|
+|ROLE|non-reserved|non-reserved|non-reserved|
+|ROLES|non-reserved|non-reserved|non-reserved|
+|ROLLBACK|non-reserved|non-reserved|reserved|
+|ROLLUP|non-reserved|non-reserved|reserved|
+|ROW|non-reserved|non-reserved|reserved|
+|ROWS|non-reserved|non-reserved|reserved|
+|SCHEMA|non-reserved|non-reserved|non-reserved|
+|SECOND|reserved|non-reserved|reserved|
+|SELECT|reserved|non-reserved|reserved|
+|SEMI|reserved|strict-non-reserved|non-reserved|
+|SEPARATED|non-reserved|non-reserved|non-reserved|
+|SERDE|non-reserved|non-reserved|non-reserved|
+|SERDEPROPERTIES|non-reserved|non-reserved|non-reserved|
+|SESSION_USER|reserved|non-reserved|reserved|
+|SET|non-reserved|non-reserved|reserved|
+|SETS|non-reserved|non-reserved|non-reserved|
+|SHOW|non-reserved|non-reserved|non-reserved|
+|SKEWED|non-reserved|non-reserved|non-reserved|
+|SOME|reserved|non-reserved|reserved|
+|SORT|non-reserved|non-reserved|non-reserved|
+|SORTED|non-reserved|non-reserved|non-reserved|
+|START|non-reserved|non-reserved|reserved|
+|STATISTICS|non-reserved|non-reserved|non-reserved|
+|STORED|non-reserved|non-reserved|non-reserved|
+|STRATIFY|non-reserved|non-reserved|non-reserved|
+|STRUCT|non-reserved|non-reserved|non-reserved|
+|SUBSTR|non-reserved|non-reserved|non-reserved|
+|SUBSTRING|non-reserved|non-reserved|non-reserved|
+|TABLE|reserved|non-reserved|reserved|
+|TABLES|non-reserved|non-reserved|non-reserved|
+|TABLESAMPLE|non-reserved|non-reserved|reserved|
+|TBLPROPERTIES|non-reserved|non-reserved|non-reserved|
+|TEMPORARY|non-reserved|non-reserved|non-reserved|
+|TERMINATED|non-reserved|non-reserved|non-reserved|
+|THEN|reserved|non-reserved|reserved|
+|TO|reserved|non-reserved|reserved|
+|TOUCH|non-reserved|non-reserved|non-reserved|
+|TRAILING|reserved|non-reserved|reserved|
+|TRANSACTION|non-reserved|non-reserved|non-reserved|
+|TRANSACTIONS|non-reserved|non-reserved|non-reserved|
+|TRANSFORM|non-reserved|non-reserved|non-reserved|
+|TRIM|non-reserved|non-reserved|non-reserved|
+|TRUE|non-reserved|non-reserved|reserved|
+|TRUNCATE|non-reserved|non-reserved|reserved|
+|UNARCHIVE|non-reserved|non-reserved|non-reserved|
+|UNBOUNDED|non-reserved|non-reserved|non-reserved|
+|UNCACHE|non-reserved|non-reserved|non-reserved|
+|UNION|reserved|strict-non-reserved|reserved|
+|UNIQUE|reserved|non-reserved|reserved|
+|UNKNOWN|reserved|non-reserved|reserved|
+|UNLOCK|non-reserved|non-reserved|non-reserved|
+|UNSET|non-reserved|non-reserved|non-reserved|
+|UPDATE|non-reserved|non-reserved|reserved|
+|USE|non-reserved|non-reserved|non-reserved|
+|USER|reserved|non-reserved|reserved|
+|USING|reserved|strict-non-reserved|reserved|
+|VALUES|non-reserved|non-reserved|reserved|
+|VIEW|non-reserved|non-reserved|non-reserved|
+|VIEWS|non-reserved|non-reserved|non-reserved|
+|WHEN|reserved|non-reserved|reserved|
+|WHERE|reserved|non-reserved|reserved|
+|WINDOW|non-reserved|non-reserved|reserved|
+|WITH|reserved|non-reserved|reserved|
+|YEAR|reserved|non-reserved|reserved|
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 0d49f6f882228..3f6b6b590f843 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -71,130 +71,24 @@ You can access them by doing
 
 {% include_example data_types scala/org/apache/spark/examples/sql/SparkSQLExample.scala %}
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in Scala</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td> Byte </td>
-  <td>
-  ByteType
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td> Short </td>
-  <td>
-  ShortType
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> Int </td>
-  <td>
-  IntegerType
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td> Long </td>
-  <td>
-  LongType
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td> Float </td>
-  <td>
-  FloatType
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> Double </td>
-  <td>
-  DoubleType
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> java.math.BigDecimal </td>
-  <td>
-  DecimalType
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> String </td>
-  <td>
-  StringType
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> Array[Byte] </td>
-  <td>
-  BinaryType
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> Boolean </td>
-  <td>
-  BooleanType
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> java.sql.Timestamp </td>
-  <td>
-  TimestampType
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> java.sql.Date </td>
-  <td>
-  DateType
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> scala.collection.Seq </td>
-  <td>
-  ArrayType(<i>elementType</i>, [<i>containsNull</i>])<br />
-  <b>Note:</b> The default value of <i>containsNull</i> is <i>true</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> scala.collection.Map </td>
-  <td>
-  MapType(<i>keyType</i>, <i>valueType</i>, [<i>valueContainsNull</i>])<br />
-  <b>Note:</b> The default value of <i>valueContainsNull</i> is <i>true</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> org.apache.spark.sql.Row </td>
-  <td>
-  StructType(<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a Seq of StructFields. Also, two fields with the same
-  name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in Scala of the data type of this field
-  (For example, Int for a StructField with the data type IntegerType) </td>
-  <td>
-  StructField(<i>name</i>, <i>dataType</i>, [<i>nullable</i>])<br />
-  <b>Note:</b> The default value of <i>nullable</i> is <i>true</i>.
-  </td>
-</tr>
-</table>
+|Data type|Value type in Scala|API to access or create a data type|
+|---------|-------------------|-----------------------------------|
+|**ByteType**|Byte|ByteType|
+|**ShortType**|Short|ShortType|
+|**IntegerType**|Int|IntegerType|
+|**LongType**|Long|LongType|
+|**FloatType**|Float|FloatType|
+|**DoubleType**|Double|DoubleType|
+|**DecimalType**|java.math.BigDecimal|DecimalType|
+|**StringType**|String|StringType|
+|**BinaryType**|Array[Byte]|BinaryType|
+|**BooleanType**|Boolean|BooleanType|
+|**TimestampType**|java.sql.Timestamp|TimestampType|
+|**DateType**|java.sql.Date|DateType|
+|**ArrayType**|scala.collection.Seq|ArrayType(*elementType*, [*containsNull]*)<br/>**Note:** The default value of *containsNull* is true.|
+|**MapType**|scala.collection.Map|MapType(*keyType*, *valueType*, [*valueContainsNull]*)<br/>**Note:** The default value of *valueContainsNull* is true.|
+|**StructType**|org.apache.spark.sql.Row|StructType(*fields*)<br/>**Note:** *fields* is a Seq of StructFields. Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in Scala of the data type of this field(For example, Int for a StructField with the data type IntegerType)|StructField(*name*, *dataType*, [*nullable*])<br/>**Note:** The default value of *nullable* is true.|
 
 </div>
 
@@ -205,132 +99,24 @@ All data types of Spark SQL are located in the package of
 please use factory methods provided in
 `org.apache.spark.sql.types.DataTypes`.
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in Java</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td> byte or Byte </td>
-  <td>
-  DataTypes.ByteType
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td> short or Short </td>
-  <td>
-  DataTypes.ShortType
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> int or Integer </td>
-  <td>
-  DataTypes.IntegerType
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td> long or Long </td>
-  <td>
-  DataTypes.LongType
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td> float or Float </td>
-  <td>
-  DataTypes.FloatType
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> double or Double </td>
-  <td>
-  DataTypes.DoubleType
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> java.math.BigDecimal </td>
-  <td>
-  DataTypes.createDecimalType()<br />
-  DataTypes.createDecimalType(<i>precision</i>, <i>scale</i>).
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> String </td>
-  <td>
-  DataTypes.StringType
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> byte[] </td>
-  <td>
-  DataTypes.BinaryType
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> boolean or Boolean </td>
-  <td>
-  DataTypes.BooleanType
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> java.sql.Timestamp </td>
-  <td>
-  DataTypes.TimestampType
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> java.sql.Date </td>
-  <td>
-  DataTypes.DateType
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> java.util.List </td>
-  <td>
-  DataTypes.createArrayType(<i>elementType</i>)<br />
-  <b>Note:</b> The value of <i>containsNull</i> will be <i>true</i><br />
-  DataTypes.createArrayType(<i>elementType</i>, <i>containsNull</i>).
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> java.util.Map </td>
-  <td>
-  DataTypes.createMapType(<i>keyType</i>, <i>valueType</i>)<br />
-  <b>Note:</b> The value of <i>valueContainsNull</i> will be <i>true</i>.<br />
-  DataTypes.createMapType(<i>keyType</i>, <i>valueType</i>, <i>valueContainsNull</i>)<br />
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> org.apache.spark.sql.Row </td>
-  <td>
-  DataTypes.createStructType(<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a List or an array of StructFields.
-  Also, two fields with the same name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in Java of the data type of this field
-  (For example, int for a StructField with the data type IntegerType) </td>
-  <td>
-  DataTypes.createStructField(<i>name</i>, <i>dataType</i>, <i>nullable</i>)
-  </td>
-</tr>
-</table>
+|Data type|Value type in Java|API to access or create a data type|
+|---------|------------------|-----------------------------------|
+|**ByteType**|byte or Byte|DataTypes.ByteType|
+|**ShortType**|short or Short|DataTypes.ShortType|
+|**IntegerType**|int or Integer|DataTypes.IntegerType|
+|**LongType**|long or Long|DataTypes.LongType|
+|**FloatType**|float or Float|DataTypes.FloatType|
+|**DoubleType**|double or Double|DataTypes.DoubleType|
+|**DecimalType**|java.math.BigDecimal|DataTypes.createDecimalType()<br/>DataTypes.createDecimalType(*precision*, *scale*).|
+|**StringType**|String|DataTypes.StringType|
+|**BinaryType**|byte[]|DataTypes.BinaryType|
+|**BooleanType**|boolean or Boolean|DataTypes.BooleanType|
+|**TimestampType**|java.sql.Timestamp|DataTypes.TimestampType|
+|**DateType**|java.sql.Date|DataTypes.DateType|
+|**ArrayType**|java.util.List|DataTypes.createArrayType(*elementType*)<br/>**Note:** The value of *containsNull* will be true.<br/>DataTypes.createArrayType(*elementType*, *containsNull*).|
+|**MapType**|java.util.Map|DataTypes.createMapType(*keyType*, *valueType*)<br/>**Note:** The value of *valueContainsNull* will be true.<br/>DataTypes.createMapType(*keyType*, *valueType*, *valueContainsNull*)|
+|**StructType**|org.apache.spark.sql.Row|DataTypes.createStructType(*fields*)<br/>**Note:** *fields* is a List or an array of StructFields.Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in Java of the data type of this field (For example, int for a StructField with the data type IntegerType)|DataTypes.createStructField(*name*, *dataType*, *nullable*)| 
 
 </div>
 
@@ -342,370 +128,73 @@ You can access them by doing
 from pyspark.sql.types import *
 {% endhighlight %}
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in Python</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td>
-  int or long <br />
-  <b>Note:</b> Numbers will be converted to 1-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -128 to 127.
-  </td>
-  <td>
-  ByteType()
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td>
-  int or long <br />
-  <b>Note:</b> Numbers will be converted to 2-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -32768 to 32767.
-  </td>
-  <td>
-  ShortType()
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> int or long </td>
-  <td>
-  IntegerType()
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td>
-  long <br />
-  <b>Note:</b> Numbers will be converted to 8-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of
-  -9223372036854775808 to 9223372036854775807.
-  Otherwise, please convert data to decimal.Decimal and use DecimalType.
-  </td>
-  <td>
-  LongType()
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td>
-  float <br />
-  <b>Note:</b> Numbers will be converted to 4-byte single-precision floating
-  point numbers at runtime.
-  </td>
-  <td>
-  FloatType()
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> float </td>
-  <td>
-  DoubleType()
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> decimal.Decimal </td>
-  <td>
-  DecimalType()
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> string </td>
-  <td>
-  StringType()
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> bytearray </td>
-  <td>
-  BinaryType()
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> bool </td>
-  <td>
-  BooleanType()
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> datetime.datetime </td>
-  <td>
-  TimestampType()
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> datetime.date </td>
-  <td>
-  DateType()
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> list, tuple, or array </td>
-  <td>
-  ArrayType(<i>elementType</i>, [<i>containsNull</i>])<br />
-  <b>Note:</b> The default value of <i>containsNull</i> is <i>True</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> dict </td>
-  <td>
-  MapType(<i>keyType</i>, <i>valueType</i>, [<i>valueContainsNull</i>])<br />
-  <b>Note:</b> The default value of <i>valueContainsNull</i> is <i>True</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> list or tuple </td>
-  <td>
-  StructType(<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a Seq of StructFields. Also, two fields with the same
-  name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in Python of the data type of this field
-  (For example, Int for a StructField with the data type IntegerType) </td>
-  <td>
-  StructField(<i>name</i>, <i>dataType</i>, [<i>nullable</i>])<br />
-  <b>Note:</b> The default value of <i>nullable</i> is <i>True</i>.
-  </td>
-</tr>
-</table>
+|Data type|Value type in Python|API to access or create a data type|
+|---------|--------------------|-----------------------------------|
+|**ByteType**|int or long<br/>**Note:** Numbers will be converted to 1-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -128 to 127.|ByteType()|
+|**ShortType**|int or long<br/>**Note:** Numbers will be converted to 2-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -32768 to 32767.|ShortType()|
+|**IntegerType**|int or long|IntegerType()|
+|**LongType**|long<br/>**Note:** Numbers will be converted to 8-byte signed integer numbers at runtime. Please make sure that numbers are within the range of -9223372036854775808 to 9223372036854775807.Otherwise, please convert data to decimal.Decimal and use DecimalType.|LongType()|
+|**FloatType**|float<br/>**Note:** Numbers will be converted to 4-byte single-precision floating point numbers at runtime.|FloatType()|
+|**DoubleType**|float|DoubleType()|
+|**DecimalType**|decimal.Decimal|DecimalType()|
+|**StringType**|string|StringType()|
+|**BinaryType**|bytearray|BinaryType()|
+|**BooleanType**|bool|BooleanType()|
+|**TimestampType**|datetime.datetime|TimestampType()|
+|**DateType**|datetime.date|DateType()|
+|**ArrayType**|list, tuple, or array|ArrayType(*elementType*, [*containsNull*])<br/>**Note:**The default value of *containsNull* is True.|
+|**MapType**|dict|MapType(*keyType*, *valueType*, [*valueContainsNull]*)<br/>**Note:**The default value of *valueContainsNull* is True.|
+|**StructType**|list or tuple|StructType(*fields*)<br/>**Note:** *fields* is a Seq of StructFields. Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in Python of the data type of this field<br/>(For example, Int for a StructField with the data type IntegerType)|StructField(*name*, *dataType*, [*nullable*])<br/>**Note:** The default value of *nullable* is True.|
 
 </div>
 
 <div data-lang="r"  markdown="1">
 
-<table class="table">
-<tr>
-  <th style="width:20%">Data type</th>
-  <th style="width:40%">Value type in R</th>
-  <th>API to access or create a data type</th></tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td>
-  integer <br />
-  <b>Note:</b> Numbers will be converted to 1-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -128 to 127.
-  </td>
-  <td>
-  "byte"
-  </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td>
-  integer <br />
-  <b>Note:</b> Numbers will be converted to 2-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of -32768 to 32767.
-  </td>
-  <td>
-  "short"
-  </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> integer </td>
-  <td>
-  "integer"
-  </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td>
-  integer <br />
-  <b>Note:</b> Numbers will be converted to 8-byte signed integer numbers at runtime.
-  Please make sure that numbers are within the range of
-  -9223372036854775808 to 9223372036854775807.
-  Otherwise, please convert data to decimal.Decimal and use DecimalType.
-  </td>
-  <td>
-  "long"
-  </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td>
-  numeric <br />
-  <b>Note:</b> Numbers will be converted to 4-byte single-precision floating
-  point numbers at runtime.
-  </td>
-  <td>
-  "float"
-  </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> numeric </td>
-  <td>
-  "double"
-  </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> Not supported </td>
-  <td>
-   Not supported
-  </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> character </td>
-  <td>
-  "string"
-  </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> raw </td>
-  <td>
-  "binary"
-  </td>
-</tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> logical </td>
-  <td>
-  "bool"
-  </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> POSIXct </td>
-  <td>
-  "timestamp"
-  </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> Date </td>
-  <td>
-  "date"
-  </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> vector or list </td>
-  <td>
-  list(type="array", elementType=<i>elementType</i>, containsNull=[<i>containsNull</i>])<br />
-  <b>Note:</b> The default value of <i>containsNull</i> is <i>TRUE</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> environment </td>
-  <td>
-  list(type="map", keyType=<i>keyType</i>, valueType=<i>valueType</i>, valueContainsNull=[<i>valueContainsNull</i>])<br />
-  <b>Note:</b> The default value of <i>valueContainsNull</i> is <i>TRUE</i>.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> named list</td>
-  <td>
-  list(type="struct", fields=<i>fields</i>)<br />
-  <b>Note:</b> <i>fields</i> is a Seq of StructFields. Also, two fields with the same
-  name are not allowed.
-  </td>
-</tr>
-<tr>
-  <td> <b>StructField</b> </td>
-  <td> The value type in R of the data type of this field
-  (For example, integer for a StructField with the data type IntegerType) </td>
-  <td>
-  list(name=<i>name</i>, type=<i>dataType</i>, nullable=[<i>nullable</i>])<br />
-  <b>Note:</b> The default value of <i>nullable</i> is <i>TRUE</i>.
-  </td>
-</tr>
-</table>
+|Data type|Value type in R|API to access or create a data type|
+|---------|---------------|-----------------------------------|
+|**ByteType**|integer <br/>**Note:** Numbers will be converted to 1-byte signed integer numbers at runtime.  Please make sure that numbers are within the range of -128 to 127.|"byte"|
+|**ShortType**|integer <br/>**Note:** Numbers will be converted to 2-byte signed integer numbers at runtime.  Please make sure that numbers are within the range of -32768 to 32767.|"short"|
+|**IntegerType**|integer|"integer"|
+|**LongType**|integer <br/>**Note:** Numbers will be converted to 8-byte signed integer numbers at runtime.  Please make sure that numbers are within the range of -9223372036854775808 to 9223372036854775807.  Otherwise, please convert data to decimal.Decimal and use DecimalType.|"long"|
+|**FloatType**|numeric <br/>**Note:** Numbers will be converted to 4-byte single-precision floating point numbers at runtime.|"float"|
+|**DoubleType**|numeric|"double"|
+|**DecimalType**|Not supported|Not supported|
+|**StringType**|character|"string"|
+|**BinaryType**|raw|"binary"|
+|**BooleanType**|logical|"bool"|
+|**TimestampType**|POSIXct|"timestamp"|
+|**DateType**|Date|"date"|
+|**ArrayType**|vector or list|list(type="array", elementType=*elementType*, containsNull=[*containsNull*])<br/>**Note:** The default value of *containsNull* is TRUE.|
+|**MapType**|environment|list(type="map", keyType=*keyType*, valueType=*valueType*, valueContainsNull=[*valueContainsNull*])<br/> **Note:** The default value of *valueContainsNull* is TRUE.|
+|**StructType**|named list|list(type="struct", fields=*fields*)<br/> **Note:** *fields* is a Seq of StructFields. Also, two fields with the same name are not allowed.|
+|**StructField**|The value type in R of the data type of this field (For example, integer for a StructField with the data type IntegerType)|list(name=*name*, type=*dataType*, nullable=[*nullable*])<br/> **Note:** The default value of *nullable* is TRUE.|
+
 </div>
 
 <div data-lang="SQL"  markdown="1">
 
 The following table shows the type names as well as aliases used in Spark SQL parser for each data type.
 
-<table class="table">
-<tr>
-  <th style="width:40%">Data type</th>
-  <th>SQL name</th></tr>
-<tr>
-  <td> <b>BooleanType</b> </td>
-  <td> BOOLEAN </td>
-</tr>
-<tr>
-  <td> <b>ByteType</b> </td>
-  <td> BYTE, TINYINT </td>
-</tr>
-<tr>
-  <td> <b>ShortType</b> </td>
-  <td> SHORT, SMALLINT </td>
-</tr>
-<tr>
-  <td> <b>IntegerType</b> </td>
-  <td> INT, INTEGER </td>
-</tr>
-<tr>
-  <td> <b>LongType</b> </td>
-  <td> LONG, BIGINT </td>
-</tr>
-<tr>
-  <td> <b>FloatType</b> </td>
-  <td> FLOAT, REAL </td>
-</tr>
-<tr>
-  <td> <b>DoubleType</b> </td>
-  <td> DOUBLE </td>
-</tr>
-<tr>
-  <td> <b>DateType</b> </td>
-  <td> DATE </td>
-</tr>
-<tr>
-  <td> <b>TimestampType</b> </td>
-  <td> TIMESTAMP </td>
-</tr>
-<tr>
-  <td> <b>StringType</b> </td>
-  <td> STRING </td>
-</tr>
-<tr>
-  <td> <b>BinaryType</b> </td>
-  <td> BINARY </td>
-</tr>
-<tr>
-  <td> <b>DecimalType</b> </td>
-  <td> DECIMAL, DEC, NUMERIC </td>
-</tr>
-<tr>
-  <td> <b>CalendarIntervalType</b> </td>
-  <td> INTERVAL </td>
-</tr>
-<tr>
-  <td> <b>ArrayType</b> </td>
-  <td> ARRAY&lt;element_type&gt; </td>
-</tr>
-<tr>
-  <td> <b>StructType</b> </td>
-  <td> STRUCT&lt;field1_name: field1_type, field2_name: field2_type, ...&gt; </td>
-</tr>
-<tr>
-  <td> <b>MapType</b> </td>
-  <td> MAP&lt;key_type, value_type&gt; </td>
-</tr>
-</table>
+|Data type|SQL name|
+|---------|--------|
+|**BooleanType**|BOOLEAN|
+|**ByteType**|BYTE, TINYINT|
+|**ShortType**|SHORT, SMALLINT|
+|**IntegerType**|INT, INTEGER|
+|**LongType**|LONG, BIGINT|
+|**FloatType**|FLOAT, REAL|
+|**DoubleType**|DOUBLE|
+|**DateType**|DATE|
+|**TimestampType**|TIMESTAMP|
+|**StringType**|STRING|
+|**BinaryType**|BINARY|
+|**DecimalType**|DECIMAL, DEC, NUMERIC|
+|**CalendarIntervalType**|INTERVAL|
+|**ArrayType**|ARRAY<element_type>|
+|**StructType**|STRUCT<field1_name: field1_type, field2_name: field2_type, ...>|
+|**MapType**|MAP<key_type, value_type>|
+
 </div>
 </div>
 
diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index e606383c94321..b65bb1319fb9d 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -29,7 +29,7 @@ There are several common scenarios for datetime usage in Spark:
 Spark uses pattern letters in the following table for date and timestamp parsing and formatting:
 
 |Symbol|Meaning|Presentation|Examples|
-|--- |--- |--- |--- |
+|------|-------|------------|--------|
 |**G**|era|text|AD; Anno Domini; A|
 |**y**|year|year|2020; 20|
 |**D**|day-of-year|number|189|
@@ -56,10 +56,10 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 |**X**|zone-offset 'Z' for zero|offset-X|Z; -08; -0830; -08:30; -083015; -08:30:15;|
 |**x**|zone-offset|offset-x|+0000; -08; -0830; -08:30; -083015; -08:30:15;|
 |**Z**|zone-offset|offset-Z|+0000; -0800; -08:00;|
-|**'**|escape for text|delimiter||
+|**'**|escape for text|delimiter| |
 |**''**|single quote|literal|'|
-|**[**|optional section start|||
-|**]**|optional section end|||
+|**[**|optional section start| | |
+|**]**|optional section end| | |
 
 The count of pattern letters determines the format.
 
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index d1f4f17596bd4..56b5cde630eaf 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -49,18 +49,18 @@ other `SQL` constructs.
 </style>
 
 The following illustrates the schema layout and data of a table named `person`. The data contains `NULL` values in
-the `age` column and this table will be used in various examples in the sections below.
+the `age` column and this table will be used in various examples in the sections below.<br/>
 **<u>TABLE: person</u>**
-<table class="tsclass" border="1">
-<tr><th>Id</th><th>Name</th><th>Age</th></tr>
-<tr><td>100</td><td>Joe</td><td>30</td></tr>
-<tr><td>200</td><td>Marry</td><td>NULL</td></tr>
-<tr><td>300</td><td>Mike</td><td>18</td></tr>
-<tr><td>400</td><td>Fred</td><td>50</td></tr>
-<tr><td>500</td><td>Albert</td><td>NULL</td></tr>
-<tr><td>600</td><td>Michelle</td><td>30</td></tr>
-<tr><td>700</td><td>Dan</td><td>50</td></tr>
-</table>
+
+|Id |Name|Age|
+|---|----|---|
+|100|Joe|30|
+|200|Marry|NULL|
+|300|Mike|18|
+|400|Fred|50|
+|500|Albert|NULL|
+|600|Michelle|30|
+|700|Dan|50|
 
 ### Comparison Operators <a name="comp-operators"></a>
 
@@ -71,48 +71,11 @@ equal operator ('<=>'), which returns `False` when one of the operand is `NULL`
 both the operands are `NULL`. The following table illustrates the behaviour of comparison operators when
 one or both operands are `NULL`:
 
-<table class="tsclass" border="1">
-  <tr>
-    <th>Left Operand</th>
-    <th>Right  Operand</th>
-    <th>></th>
-    <th>>=</th>
-    <th>=</th>
-    <th><</th>
-    <th><=</th>
-    <th><=></th>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>Any value</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>False</td>
-  </tr>
-  <tr>
-    <td>Any value</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>False</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>True</td>
-  </tr>
-</table>
+|Left Operand|Right  Operand|>  |>= |=  |<  |<= |<=>|
+|------------|--------------|---|---|---|---|---|---|
+|NULL|Any value|NULL|NULL|NULL|NULL|NULL|False|
+|Any value|NULL|NULL|NULL|NULL|NULL|NULL|False|
+|NULL|NULL|NULL|NULL|NULL|NULL|NULL|True|
 
 ### Examples
 
@@ -157,55 +120,17 @@ as the arguments and return a `Boolean` value.
 
 The following tables illustrate the behavior of logical operators when one or both operands are `NULL`.
 
-<table class="tsclass" border="1">
-  <tr>
-    <th>Left Operand</th>
-    <th>Right Operand</th>
-    <th>OR</th>
-    <th>AND</th>
-  </tr>
-  <tr>
-    <td>True</td>
-    <td>NULL</td>
-    <td>True</td>
-    <td>NULL</td>
-  </tr>
-  <tr>
-    <td>False</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>False</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>True</td>
-    <td>True</td>
-    <td>NULL</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>False</td>
-    <td>NULL</td>
-    <td>NULL</td>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-    <td>NULL</td>
-  </tr>
-</table>
-<br>
-<table class="tsclass" border="1">
-  <tr>
-    <th>operand</th>
-    <th>NOT</th>
-  </tr>
-  <tr>
-    <td>NULL</td>
-    <td>NULL</td>
-  </tr>
-</table>
+|Left Operand|Right Operand|OR |AND|
+|------------|-------------|---|---|
+|True|NULL|True|NULL|
+|False|NULL|NULL|False|
+|NULL|True|True|NULL|
+|NULL|False|NULL|NULL|
+|NULL|NULL|NULL|NULL|
+
+|operand|NOT|
+|-------|---|
+|NULL|NULL|
 
 ### Examples
 
@@ -335,7 +260,7 @@ the rules of how `NULL` values are handled by aggregate functions.
 - `NULL` values are ignored from processing by all the aggregate functions.
   - Only exception to this rule is COUNT(*) function.
 - Some aggregate functions return `NULL` when all input values are `NULL` or the input data set
-  is empty.<br> The list of these functions is:
+  is empty.<br/> The list of these functions is:
   - MAX
   - MIN
   - SUM
diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
index ac8fecaae6413..1d9505a2bc11b 100644
--- a/docs/sql-ref-syntax-qry-select-tvf.md
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -40,7 +40,7 @@ function_name ( expression [ , ... ] ) [ table_alias ]
 <dl>
   <dt><code><em>table_alias</em></code></dt>
   <dd>
-    Specifies a temporary name with an optional column name list. <br><br>
+    Specifies a temporary name with an optional column name list. <br/><br/>
     <b>Syntax:</b>
       <code>
         [ AS ] table_name [ ( column_name [ , ... ] ) ]
@@ -50,31 +50,12 @@ function_name ( expression [ , ... ] ) [ table_alias ]
 
 ### Supported Table-valued Functions
 
-<table class="table">
-  <thead>
-    <tr><th style="width:25%">Function</th><th>Argument Type(s)</th><th>Description</th></tr>
-  </thead>
-    <tr>
-      <td><b> range </b>( <i>end</i> )</td>
-      <td> Long </td>
-      <td>Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from 0 to <code>end</code> (exclusive) with step value 1.</td>
-    </tr>
-    <tr>
-      <td><b> range </b>( <i> start, end</i> )</td>
-      <td> Long, Long </td>
-      <td width="60%">Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from <code>start</code> to <code>end</code> (exclusive) with step value 1.</td>
-    </tr>
-    <tr>
-      <td><b> range </b>( <i> start, end, step</i> )</td>
-      <td> Long, Long, Long </td>
-      <td width="60%">Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from <code>start</code> to <code>end</code> (exclusive) with <code>step</code> value.</td>
-     </tr>
-    <tr>
-      <td><b> range </b>( <i> start, end, step, numPartitions</i> )</td>
-      <td> Long, Long, Long, Int </td>
-      <td width="60%">Creates a table with a single <code>LongType</code> column named <code>id</code>, containing rows in a range from <code>start</code> to <code>end</code> (exclusive) with <code>step</code> value, with partition number <code>numPartitions</code> specified. </td>
-    </tr>
-</table>
+|Function|Argument Type(s)|Description|
+|--------|----------------|-----------|
+|**range** ( *end* )|Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from 0 to *end* (exclusive) with step value 1.|
+|**range** ( *start, end* )|Long, Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with step value 1.|
+|**range** ( *start, end, step* )|Long, Long, Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value.|
+|**range** ( *start, end, step, numPartitions* )|Long, Long, Long, Int|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value, with partition number *numPartitions* specified.|
 
 ### Examples
 

From 8c077b0a3e27e86ce3dbbf491521cd15b8f9fe79 Mon Sep 17 00:00:00 2001
From: Akshat Bordia <akshat.bordia31@gmail.com>
Date: Tue, 5 May 2020 08:58:37 -0500
Subject: [PATCH 0633/1280] [SPARK-31621][CORE] Fixing Spark Master UI Issue
 when application is waiting for workers to launch driver

### What changes were proposed in this pull request?
Fixing an issue where Spark Master UI Fails to load if the application is waiting for workers to launch driver.

**Root Cause:**
This is happening due to the fact that the submitted application is waiting for a worker to be free to run the driver. Due to this resource is set to null in the formatResourcesAddresses method and this is running into null pointer exception.
![image](https://user-images.githubusercontent.com/31816865/80801557-77ee9300-8bca-11ea-92b7-b8df58b68de3.png)

**Fix:**
Added a null check before forming a resource address and display "None" if the driver isn't launched yet.

### Why are the changes needed?

Spark Master UI should load as expected when applications are waiting for workers to run driver.

### Does this PR introduce _any_ user-facing change?
The worker column in Spark Master UI will show "None" if the driver hasn't been launched yet.
![image](https://user-images.githubusercontent.com/31816865/80801671-be43f200-8bca-11ea-86c3-381925f82cc7.png)

### How was this patch tested?
Tested on a local setup. Launched 2 applications and ensured that Spark Master UI loads fine.
![image](https://user-images.githubusercontent.com/31816865/80801883-5b9f2600-8bcb-11ea-8a1a-cc597aabc4c2.png)

Closes #28429 from akshatb1/MasterUIBug.

Authored-by: Akshat Bordia <akshat.bordia31@gmail.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit c71198ab6c8c9ded6a52eb97859b39dc2119b5fd)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../main/scala/org/apache/spark/deploy/master/DriverInfo.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala b/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala
index bf68ba8e15af4..252e7048ba105 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/DriverInfo.scala
@@ -37,7 +37,7 @@ private[deploy] class DriverInfo(
   @transient var worker: Option[WorkerInfo] = None
   // resources(e.f. gpu/fpga) allocated to this driver
   // map from resource name to ResourceInformation
-  private var _resources: Map[String, ResourceInformation] = _
+  private var _resources: Map[String, ResourceInformation] = Map.empty
 
   init()
 

From b29bbed6a6389db3d680e770334125375c940860 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 5 May 2020 14:11:53 +0000
Subject: [PATCH 0634/1280] [SPARK-31630][SQL] Fix perf regression by skipping
 timestamps rebasing after some threshold

### What changes were proposed in this pull request?
Skip timestamps rebasing after a global threshold when there is no difference between Julian and Gregorian calendars. This allows to avoid checking hash maps of switch points, and fixes perf regressions in `toJavaTimestamp()` and `fromJavaTimestamp()`.

### Why are the changes needed?
The changes fix perf regressions of conversions to/from external type `java.sql.Timestamp`.

Before (see the PR's results https://github.com/apache/spark/pull/28440):
```
================================================================================================
Conversion from/to external types
================================================================================================

OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
Intel(R) Xeon(R) CPU E5-2670 v2  2.50GHz
To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Timestamp                             376            388          10         13.3          75.2       1.1X
Collect java.sql.Timestamp                         1878           1937          64          2.7         375.6       0.2X
```

After:
```
================================================================================================
Conversion from/to external types
================================================================================================

OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
Intel(R) Xeon(R) CPU E5-2670 v2  2.50GHz
To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
------------------------------------------------------------------------------------------------------------------------
From java.sql.Timestamp                             249            264          24         20.1          49.8       1.7X
Collect java.sql.Timestamp                         1503           1523          24          3.3         300.5       0.3X
```

Perf improvements in average of:

1. From java.sql.Timestamp is ~ 34%
2. To java.sql.Timestamps is ~16%

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By existing test suites `DateTimeUtilsSuite` and `RebaseDateTimeSuite`.

Closes #28441 from MaxGekk/opt-rebase-common-threshold.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bef5828e12500630d7efc8e0c005182b25ef2b7f)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    |  47 +++-
 .../DateTimeBenchmark-jdk11-results.txt       | 254 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 254 +++++++++---------
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 142 +++++-----
 .../DateTimeRebaseBenchmark-results.txt       | 144 +++++-----
 5 files changed, 431 insertions(+), 410 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 8ad7f719bb751..eb67ff71f6694 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -286,6 +286,17 @@ object RebaseDateTime {
    */
   private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json")
 
+  private def getLastSwitchTs(rebaseMap: AnyRefMap[String, RebaseInfo]): Long = {
+    val latestTs = rebaseMap.values.map(_.switches.last).max
+    require(rebaseMap.values.forall(_.diffs.last == 0),
+      s"Differences between Julian and Gregorian calendar after ${microsToInstant(latestTs)} " +
+      "are expected to be zero for all available time zones.")
+    latestTs
+  }
+  // The switch time point after which all diffs between Gregorian and Julian calendars
+  // across all time zones are zero
+  private final val lastSwitchGregorianTs: Long = getLastSwitchTs(gregJulianRebaseMap)
+
   private final val gregorianStartTs = LocalDateTime.of(gregorianStartDate, LocalTime.MIDNIGHT)
   private final val julianEndTs = LocalDateTime.of(
     julianEndDate,
@@ -344,13 +355,17 @@ object RebaseDateTime {
    * @return The rebased microseconds since the epoch in Julian calendar.
    */
   def rebaseGregorianToJulianMicros(micros: Long): Long = {
-    val timeZone = TimeZone.getDefault
-    val tzId = timeZone.getID
-    val rebaseRecord = gregJulianRebaseMap.getOrNull(tzId)
-    if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
-      rebaseGregorianToJulianMicros(timeZone.toZoneId, micros)
+    if (micros >= lastSwitchGregorianTs) {
+      micros
     } else {
-      rebaseMicros(rebaseRecord, micros)
+      val timeZone = TimeZone.getDefault
+      val tzId = timeZone.getID
+      val rebaseRecord = gregJulianRebaseMap.getOrNull(tzId)
+      if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
+        rebaseGregorianToJulianMicros(timeZone.toZoneId, micros)
+      } else {
+        rebaseMicros(rebaseRecord, micros)
+      }
     }
   }
 
@@ -418,7 +433,9 @@ object RebaseDateTime {
   // in the interval: [julianGregDiffSwitchMicros(i), julianGregDiffSwitchMicros(i+1))
   private val julianGregRebaseMap = loadRebaseRecords("julian-gregorian-rebase-micros.json")
 
-  final val lastSwitchJulianTs: Long = julianGregRebaseMap.values.map(_.switches.last).max
+  // The switch time point after which all diffs between Julian and Gregorian calendars
+  // across all time zones are zero
+  final val lastSwitchJulianTs: Long = getLastSwitchTs(julianGregRebaseMap)
 
   /**
    * An optimized version of [[rebaseJulianToGregorianMicros(ZoneId, Long)]]. This method leverages
@@ -434,13 +451,17 @@ object RebaseDateTime {
    * @return The rebased microseconds since the epoch in Proleptic Gregorian calendar.
    */
   def rebaseJulianToGregorianMicros(micros: Long): Long = {
-    val timeZone = TimeZone.getDefault
-    val tzId = timeZone.getID
-    val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
-    if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
-      rebaseJulianToGregorianMicros(timeZone.toZoneId, micros)
+    if (micros >= lastSwitchJulianTs) {
+      micros
     } else {
-      rebaseMicros(rebaseRecord, micros)
+      val timeZone = TimeZone.getDefault
+      val tzId = timeZone.getID
+      val rebaseRecord = julianGregRebaseMap.getOrNull(tzId)
+      if (rebaseRecord == null || micros < rebaseRecord.switches(0)) {
+        rebaseJulianToGregorianMicros(timeZone.toZoneId, micros)
+      } else {
+        rebaseMicros(rebaseRecord, micros)
+      }
     }
   }
 }
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 61b4c762a752e..61ca342a0d559 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,18 +6,18 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1485           1567         116          6.7         148.5       1.0X
-date + interval(m, d)                              1504           1510           9          6.6         150.4       1.0X
-date + interval(m, d, ms)                          7000           7013          18          1.4         700.0       0.2X
-date - interval(m)                                 1466           1478          17          6.8         146.6       1.0X
-date - interval(m, d)                              1533           1534           1          6.5         153.3       1.0X
-date - interval(m, d, ms)                          7014           7019           7          1.4         701.4       0.2X
-timestamp + interval(m)                            3062           3064           3          3.3         306.2       0.5X
-timestamp + interval(m, d)                         3133           3136           5          3.2         313.3       0.5X
-timestamp + interval(m, d, ms)                     3401           3402           3          2.9         340.1       0.4X
-timestamp - interval(m)                            3025           3037          17          3.3         302.5       0.5X
-timestamp - interval(m, d)                         3083           3120          51          3.2         308.3       0.5X
-timestamp - interval(m, d, ms)                     3371           3379          11          3.0         337.1       0.4X
+date + interval(m)                                 1496           1569         104          6.7         149.6       1.0X
+date + interval(m, d)                              1514           1526          17          6.6         151.4       1.0X
+date + interval(m, d, ms)                          6231           6253          30          1.6         623.1       0.2X
+date - interval(m)                                 1481           1487           9          6.8         148.1       1.0X
+date - interval(m, d)                              1550           1552           2          6.5         155.0       1.0X
+date - interval(m, d, ms)                          6269           6272           4          1.6         626.9       0.2X
+timestamp + interval(m)                            3017           3056          54          3.3         301.7       0.5X
+timestamp + interval(m, d)                         3146           3148           3          3.2         314.6       0.5X
+timestamp + interval(m, d, ms)                     3446           3460          20          2.9         344.6       0.4X
+timestamp - interval(m)                            3045           3059          19          3.3         304.5       0.5X
+timestamp - interval(m, d)                         3147           3164          25          3.2         314.7       0.5X
+timestamp - interval(m, d, ms)                     3425           3442          25          2.9         342.5       0.4X
 
 
 ================================================================================================
@@ -28,92 +28,92 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    339            339           1         29.5          33.9       1.0X
-cast to timestamp wholestage on                     330            337           9         30.3          33.0       1.0X
+cast to timestamp wholestage off                    332            336           5         30.1          33.2       1.0X
+cast to timestamp wholestage on                     333            344          10         30.0          33.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1226           1237          15          8.2         122.6       1.0X
-year of timestamp wholestage on                    1230           1242           9          8.1         123.0       1.0X
+year of timestamp wholestage off                   1246           1257          16          8.0         124.6       1.0X
+year of timestamp wholestage on                    1209           1218          12          8.3         120.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1602           1606           7          6.2         160.2       1.0X
-quarter of timestamp wholestage on                 1511           1514           3          6.6         151.1       1.1X
+quarter of timestamp wholestage off                1608           1616          11          6.2         160.8       1.0X
+quarter of timestamp wholestage on                 1540           1552          10          6.5         154.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1227           1233           8          8.1         122.7       1.0X
-month of timestamp wholestage on                   1226           1242          28          8.2         122.6       1.0X
+month of timestamp wholestage off                  1242           1246           6          8.1         124.2       1.0X
+month of timestamp wholestage on                   1202           1212          11          8.3         120.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1965           1980          20          5.1         196.5       1.0X
-weekofyear of timestamp wholestage on              1816           1833          17          5.5         181.6       1.1X
+weekofyear of timestamp wholestage off             1879           1885           8          5.3         187.9       1.0X
+weekofyear of timestamp wholestage on              1832           1845          10          5.5         183.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1229           1231           3          8.1         122.9       1.0X
-day of timestamp wholestage on                     1222           1230          10          8.2         122.2       1.0X
+day of timestamp wholestage off                    1236           1239           4          8.1         123.6       1.0X
+day of timestamp wholestage on                     1206           1219          17          8.3         120.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1294           1297           4          7.7         129.4       1.0X
-dayofyear of timestamp wholestage on               1257           1264           6          8.0         125.7       1.0X
+dayofyear of timestamp wholestage off              1308           1309           1          7.6         130.8       1.0X
+dayofyear of timestamp wholestage on               1239           1255          15          8.1         123.9       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1247           1253           8          8.0         124.7       1.0X
-dayofmonth of timestamp wholestage on              1225           1229           4          8.2         122.5       1.0X
+dayofmonth of timestamp wholestage off             1259           1263           5          7.9         125.9       1.0X
+dayofmonth of timestamp wholestage on              1201           1205           5          8.3         120.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1416           1416           0          7.1         141.6       1.0X
-dayofweek of timestamp wholestage on               1376           1382           8          7.3         137.6       1.0X
+dayofweek of timestamp wholestage off              1406           1410           6          7.1         140.6       1.0X
+dayofweek of timestamp wholestage on               1387           1402          15          7.2         138.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1350           1351           1          7.4         135.0       1.0X
-weekday of timestamp wholestage on                 1308           1318          13          7.6         130.8       1.0X
+weekday of timestamp wholestage off                1355           1367          18          7.4         135.5       1.0X
+weekday of timestamp wholestage on                 1311           1321          10          7.6         131.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1004           1007           3         10.0         100.4       1.0X
-hour of timestamp wholestage on                     928            938           7         10.8          92.8       1.1X
+hour of timestamp wholestage off                    996            997           2         10.0          99.6       1.0X
+hour of timestamp wholestage on                     930            936           6         10.7          93.0       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1009           1020          15          9.9         100.9       1.0X
-minute of timestamp wholestage on                   933            935           2         10.7          93.3       1.1X
+minute of timestamp wholestage off                 1005           1012          10          9.9         100.5       1.0X
+minute of timestamp wholestage on                   949            952           3         10.5          94.9       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  995            995           0         10.0          99.5       1.0X
-second of timestamp wholestage on                   932            937           8         10.7          93.2       1.1X
+second of timestamp wholestage off                 1013           1014           1          9.9         101.3       1.0X
+second of timestamp wholestage on                   933            934           2         10.7          93.3       1.1X
 
 
 ================================================================================================
@@ -124,15 +124,15 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         292            316          34         34.2          29.2       1.0X
-current_date wholestage on                          270            276           6         37.0          27.0       1.1X
+current_date wholestage off                         291            293           2         34.3          29.1       1.0X
+current_date wholestage on                          280            284           3         35.7          28.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    313            328          20         31.9          31.3       1.0X
-current_timestamp wholestage on                     270            331          95         37.0          27.0       1.2X
+current_timestamp wholestage off                    311            324          18         32.1          31.1       1.0X
+current_timestamp wholestage on                     275            364          85         36.3          27.5       1.1X
 
 
 ================================================================================================
@@ -143,43 +143,43 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1078           1081           3          9.3         107.8       1.0X
-cast to date wholestage on                         1035           1040           7          9.7         103.5       1.0X
+cast to date wholestage off                        1077           1079           3          9.3         107.7       1.0X
+cast to date wholestage on                         1018           1030          14          9.8         101.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1265           1266           3          7.9         126.5       1.0X
-last_day wholestage on                             1236           1246          10          8.1         123.6       1.0X
+last_day wholestage off                            1257           1260           4          8.0         125.7       1.0X
+last_day wholestage on                             1218           1227          14          8.2         121.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1118           1118           1          8.9         111.8       1.0X
-next_day wholestage on                             1085           1090           8          9.2         108.5       1.0X
+next_day wholestage off                            1140           1141           1          8.8         114.0       1.0X
+next_day wholestage on                             1067           1076          11          9.4         106.7       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1052           1054           4          9.5         105.2       1.0X
-date_add wholestage on                             1046           1051           6          9.6         104.6       1.0X
+date_add wholestage off                            1062           1064           3          9.4         106.2       1.0X
+date_add wholestage on                             1046           1055          11          9.6         104.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1075           1075           0          9.3         107.5       1.0X
-date_sub wholestage on                             1043           1046           3          9.6         104.3       1.0X
+date_sub wholestage off                            1082           1083           1          9.2         108.2       1.0X
+date_sub wholestage on                             1047           1056          12          9.6         104.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1409           1409           0          7.1         140.9       1.0X
-add_months wholestage on                           1448           1453           4          6.9         144.8       1.0X
+add_months wholestage off                          1430           1431           1          7.0         143.0       1.0X
+add_months wholestage on                           1441           1446           8          6.9         144.1       1.0X
 
 
 ================================================================================================
@@ -190,8 +190,8 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5373           5390          24          1.9         537.3       1.0X
-format date wholestage on                          5337           5346          12          1.9         533.7       1.0X
+format date wholestage off                         5442           5549         150          1.8         544.2       1.0X
+format date wholestage on                          5529           5655         236          1.8         552.9       1.0X
 
 
 ================================================================================================
@@ -202,8 +202,8 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7302           7308           9          1.4         730.2       1.0X
-from_unixtime wholestage on                        7298           7319          16          1.4         729.8       1.0X
+from_unixtime wholestage off                       7416           7440          34          1.3         741.6       1.0X
+from_unixtime wholestage on                        7372           7391          17          1.4         737.2       1.0X
 
 
 ================================================================================================
@@ -214,15 +214,15 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1322           1355          48          7.6         132.2       1.0X
-from_utc_timestamp wholestage on                   1290           1294           5          7.8         129.0       1.0X
+from_utc_timestamp wholestage off                  1316           1320           6          7.6         131.6       1.0X
+from_utc_timestamp wholestage on                   1268           1272           4          7.9         126.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1692           1705          18          5.9         169.2       1.0X
-to_utc_timestamp wholestage on                     1653           1657           4          6.1         165.3       1.0X
+to_utc_timestamp wholestage off                    1653           1657           6          6.0         165.3       1.0X
+to_utc_timestamp wholestage on                     1594           1599           4          6.3         159.4       1.0X
 
 
 ================================================================================================
@@ -233,29 +233,29 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        340            356          22         29.4          34.0       1.0X
-cast interval wholestage on                         293            296           2         34.1          29.3       1.2X
+cast interval wholestage off                        341            343           3         29.4          34.1       1.0X
+cast interval wholestage on                         279            282           1         35.8          27.9       1.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1843           1862          28          5.4         184.3       1.0X
-datediff wholestage on                             1766           1780          16          5.7         176.6       1.0X
+datediff wholestage off                            1862           1865           4          5.4         186.2       1.0X
+datediff wholestage on                             1769           1783          15          5.7         176.9       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5856           5858           2          1.7         585.6       1.0X
-months_between wholestage on                       5799           5815          14          1.7         579.9       1.0X
+months_between wholestage off                      5594           5599           7          1.8         559.4       1.0X
+months_between wholestage on                       5498           5508          11          1.8         549.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2017           2147         183          0.5        2017.4       1.0X
-window wholestage on                              47789          47910          91          0.0       47788.6       0.0X
+window wholestage off                              2044           2127         117          0.5        2044.3       1.0X
+window wholestage on                              48057          48109          54          0.0       48056.9       0.0X
 
 
 ================================================================================================
@@ -266,134 +266,134 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2689           2689           1          3.7         268.9       1.0X
-date_trunc YEAR wholestage on                      2655           2670          17          3.8         265.5       1.0X
+date_trunc YEAR wholestage off                     2540           2542           3          3.9         254.0       1.0X
+date_trunc YEAR wholestage on                      2486           2507          29          4.0         248.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2698           2700           3          3.7         269.8       1.0X
-date_trunc YYYY wholestage on                      2654           2660           6          3.8         265.4       1.0X
+date_trunc YYYY wholestage off                     2542           2543           3          3.9         254.2       1.0X
+date_trunc YYYY wholestage on                      2491           2498           9          4.0         249.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2692           2697           7          3.7         269.2       1.0X
-date_trunc YY wholestage on                        2653           2662           7          3.8         265.3       1.0X
+date_trunc YY wholestage off                       2545           2569          35          3.9         254.5       1.0X
+date_trunc YY wholestage on                        2487           2493           4          4.0         248.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2752           2756           6          3.6         275.2       1.0X
-date_trunc MON wholestage on                       2666           2675          15          3.8         266.6       1.0X
+date_trunc MON wholestage off                      2590           2590           1          3.9         259.0       1.0X
+date_trunc MON wholestage on                       2506           2520          12          4.0         250.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2743           2746           4          3.6         274.3       1.0X
-date_trunc MONTH wholestage on                     2667           2673           8          3.7         266.7       1.0X
+date_trunc MONTH wholestage off                    2595           2603          11          3.9         259.5       1.0X
+date_trunc MONTH wholestage on                     2505           2516          12          4.0         250.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2741           2741           1          3.6         274.1       1.0X
-date_trunc MM wholestage on                        2670           2678           7          3.7         267.0       1.0X
+date_trunc MM wholestage off                       2605           2612          10          3.8         260.5       1.0X
+date_trunc MM wholestage on                        2501           2515          11          4.0         250.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2338           2342           6          4.3         233.8       1.0X
-date_trunc DAY wholestage on                       2269           2277           7          4.4         226.9       1.0X
+date_trunc DAY wholestage off                      2225           2229           5          4.5         222.5       1.0X
+date_trunc DAY wholestage on                       2184           2196           9          4.6         218.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2324           2325           1          4.3         232.4       1.0X
-date_trunc DD wholestage on                        2270           2273           2          4.4         227.0       1.0X
+date_trunc DD wholestage off                       2232           2236           6          4.5         223.2       1.0X
+date_trunc DD wholestage on                        2183           2190           6          4.6         218.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2325           2326           1          4.3         232.5       1.0X
-date_trunc HOUR wholestage on                      2284           2295           8          4.4         228.4       1.0X
+date_trunc HOUR wholestage off                     2194           2199           7          4.6         219.4       1.0X
+date_trunc HOUR wholestage on                      2160           2166           5          4.6         216.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    407            408           0         24.5          40.7       1.0X
-date_trunc MINUTE wholestage on                     382            386           3         26.1          38.2       1.1X
+date_trunc MINUTE wholestage off                    390            396           9         25.7          39.0       1.0X
+date_trunc MINUTE wholestage on                     331            337           7         30.2          33.1       1.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    404            404           1         24.8          40.4       1.0X
-date_trunc SECOND wholestage on                     386            390           4         25.9          38.6       1.0X
+date_trunc SECOND wholestage off                    375            381           8         26.7          37.5       1.0X
+date_trunc SECOND wholestage on                     332            346          14         30.1          33.2       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2693           2694           2          3.7         269.3       1.0X
-date_trunc WEEK wholestage on                      2619           2629          10          3.8         261.9       1.0X
+date_trunc WEEK wholestage off                     2439           2443           6          4.1         243.9       1.0X
+date_trunc WEEK wholestage on                      2390           2409          32          4.2         239.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3454           3466          17          2.9         345.4       1.0X
-date_trunc QUARTER wholestage on                   3384           3404          24          3.0         338.4       1.0X
+date_trunc QUARTER wholestage off                  3290           3292           4          3.0         329.0       1.0X
+date_trunc QUARTER wholestage on                   3214           3218           3          3.1         321.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           339            340           1         29.5          33.9       1.0X
-trunc year wholestage on                            337            347           9         29.7          33.7       1.0X
+trunc year wholestage off                           308            310           3         32.5          30.8       1.0X
+trunc year wholestage on                            289            293           6         34.7          28.9       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           347            348           2         28.8          34.7       1.0X
-trunc yyyy wholestage on                            334            335           2         29.9          33.4       1.0X
+trunc yyyy wholestage off                           309            311           3         32.4          30.9       1.0X
+trunc yyyy wholestage on                            289            294           7         34.6          28.9       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             339            346          11         29.5          33.9       1.0X
-trunc yy wholestage on                              333            338           5         30.0          33.3       1.0X
+trunc yy wholestage off                             311            311           0         32.2          31.1       1.0X
+trunc yy wholestage on                              288            294           7         34.7          28.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            339            347          11         29.5          33.9       1.0X
-trunc mon wholestage on                             331            336           4         30.2          33.1       1.0X
+trunc mon wholestage off                            313            313           0         32.0          31.3       1.0X
+trunc mon wholestage on                             287            290           2         34.8          28.7       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          341            344           3         29.3          34.1       1.0X
-trunc month wholestage on                           332            338           9         30.1          33.2       1.0X
+trunc month wholestage off                          310            310           0         32.3          31.0       1.0X
+trunc month wholestage on                           287            290           2         34.8          28.7       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             337            338           1         29.6          33.7       1.0X
-trunc mm wholestage on                              332            336           5         30.1          33.2       1.0X
+trunc mm wholestage off                             311            312           1         32.1          31.1       1.0X
+trunc mm wholestage on                              287            296           9         34.8          28.7       1.1X
 
 
 ================================================================================================
@@ -404,36 +404,36 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     184            187           4          5.4         183.9       1.0X
-to timestamp str wholestage on                      159            162           2          6.3         159.4       1.2X
+to timestamp str wholestage off                     169            170           1          5.9         168.9       1.0X
+to timestamp str wholestage on                      161            168          11          6.2         161.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1683           1689           8          0.6        1683.3       1.0X
-to_timestamp wholestage on                         1722           1725           4          0.6        1721.6       1.0X
+to_timestamp wholestage off                        1360           1361           1          0.7        1359.6       1.0X
+to_timestamp wholestage on                         1362           1366           6          0.7        1362.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1733           1736           4          0.6        1733.1       1.0X
-to_unix_timestamp wholestage on                    1687           1690           4          0.6        1686.6       1.0X
+to_unix_timestamp wholestage off                   1343           1346           4          0.7        1342.6       1.0X
+to_unix_timestamp wholestage on                    1356           1359           2          0.7        1356.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          218            220           4          4.6         217.6       1.0X
-to date str wholestage on                           213            215           2          4.7         212.6       1.0X
+to date str wholestage off                          227            230           4          4.4         227.0       1.0X
+to date str wholestage on                           299            302           3          3.3         299.0       0.8X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3697           3699           2          0.3        3697.2       1.0X
-to_date wholestage on                              3603           3624          15          0.3        3602.7       1.0X
+to_date wholestage off                             3413           3440          38          0.3        3413.0       1.0X
+to_date wholestage on                              3392           3402          12          0.3        3392.3       1.0X
 
 
 ================================================================================================
@@ -444,14 +444,14 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  432            436           7         11.6          86.4       1.0X
-From java.time.LocalDate                            343            347           6         14.6          68.6       1.3X
-Collect java.sql.Date                              1888           2453         971          2.6         377.6       0.2X
-Collect java.time.LocalDate                        1779           1820          42          2.8         355.7       0.2X
-From java.sql.Timestamp                             375            384           9         13.3          75.0       1.2X
-From java.time.Instant                              317            326           8         15.8          63.5       1.4X
-Collect longs                                      1338           1428         115          3.7         267.6       0.3X
-Collect java.sql.Timestamp                         1716           2014         281          2.9         343.1       0.3X
-Collect java.time.Instant                          1832           1970         122          2.7         366.5       0.2X
+From java.sql.Date                                  410            415           7         12.2          82.0       1.0X
+From java.time.LocalDate                            332            333           1         15.1          66.4       1.2X
+Collect java.sql.Date                              1891           2542         829          2.6         378.1       0.2X
+Collect java.time.LocalDate                        1630           2138         441          3.1         326.0       0.3X
+From java.sql.Timestamp                             254            259           6         19.7          50.9       1.6X
+From java.time.Instant                              302            306           4         16.6          60.3       1.4X
+Collect longs                                      1134           1265         117          4.4         226.8       0.4X
+Collect java.sql.Timestamp                         1441           1458          16          3.5         288.1       0.3X
+Collect java.time.Instant                          1680           1928         253          3.0         336.0       0.2X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 3ef2f922f95bf..7586295778bd8 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,18 +6,18 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1613           1622          13          6.2         161.3       1.0X
-date + interval(m, d)                              1729           1752          32          5.8         172.9       0.9X
-date + interval(m, d, ms)                          6421           6424           5          1.6         642.1       0.3X
-date - interval(m)                                 1441           1443           2          6.9         144.1       1.1X
-date - interval(m, d)                              1687           1689           2          5.9         168.7       1.0X
-date - interval(m, d, ms)                          6617           6625          11          1.5         661.7       0.2X
-timestamp + interval(m)                            2713           2733          28          3.7         271.3       0.6X
-timestamp + interval(m, d)                         3027           3032           8          3.3         302.7       0.5X
-timestamp + interval(m, d, ms)                     3501           3509          12          2.9         350.1       0.5X
-timestamp - interval(m)                            2892           2895           4          3.5         289.2       0.6X
-timestamp - interval(m, d)                         3190           3196           9          3.1         319.0       0.5X
-timestamp - interval(m, d, ms)                     3497           3500           5          2.9         349.7       0.5X
+date + interval(m)                                 1638           1701          89          6.1         163.8       1.0X
+date + interval(m, d)                              1785           1790           7          5.6         178.5       0.9X
+date + interval(m, d, ms)                          6229           6270          58          1.6         622.9       0.3X
+date - interval(m)                                 1500           1503           4          6.7         150.0       1.1X
+date - interval(m, d)                              1764           1766           3          5.7         176.4       0.9X
+date - interval(m, d, ms)                          6428           6446          25          1.6         642.8       0.3X
+timestamp + interval(m)                            2719           2722           4          3.7         271.9       0.6X
+timestamp + interval(m, d)                         3011           3021          14          3.3         301.1       0.5X
+timestamp + interval(m, d, ms)                     3405           3412           9          2.9         340.5       0.5X
+timestamp - interval(m)                            2759           2764           7          3.6         275.9       0.6X
+timestamp - interval(m, d)                         3094           3112          25          3.2         309.4       0.5X
+timestamp - interval(m, d, ms)                     3388           3392           5          3.0         338.8       0.5X
 
 
 ================================================================================================
@@ -28,92 +28,92 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    321            323           2         31.1          32.1       1.0X
-cast to timestamp wholestage on                     294            306          10         34.0          29.4       1.1X
+cast to timestamp wholestage off                    319            323           6         31.4          31.9       1.0X
+cast to timestamp wholestage on                     304            311           8         32.9          30.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1235           1242           9          8.1         123.5       1.0X
-year of timestamp wholestage on                    1208           1216           8          8.3         120.8       1.0X
+year of timestamp wholestage off                   1234           1239           6          8.1         123.4       1.0X
+year of timestamp wholestage on                    1229           1244          22          8.1         122.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1415           1424          12          7.1         141.5       1.0X
-quarter of timestamp wholestage on                 1338           1341           4          7.5         133.8       1.1X
+quarter of timestamp wholestage off                1440           1445           7          6.9         144.0       1.0X
+quarter of timestamp wholestage on                 1358           1361           3          7.4         135.8       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1224           1225           1          8.2         122.4       1.0X
-month of timestamp wholestage on                   1193           1202           8          8.4         119.3       1.0X
+month of timestamp wholestage off                  1239           1240           1          8.1         123.9       1.0X
+month of timestamp wholestage on                   1221           1239          26          8.2         122.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1864           1866           3          5.4         186.4       1.0X
-weekofyear of timestamp wholestage on              1827           1840           7          5.5         182.7       1.0X
+weekofyear of timestamp wholestage off             1926           1934          11          5.2         192.6       1.0X
+weekofyear of timestamp wholestage on              1901           1911          10          5.3         190.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1209           1211           2          8.3         120.9       1.0X
-day of timestamp wholestage on                     1191           1194           6          8.4         119.1       1.0X
+day of timestamp wholestage off                    1225           1229           6          8.2         122.5       1.0X
+day of timestamp wholestage on                     1217           1225           7          8.2         121.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1270           1271           2          7.9         127.0       1.0X
-dayofyear of timestamp wholestage on               1241           1250          12          8.1         124.1       1.0X
+dayofyear of timestamp wholestage off              1290           1295           7          7.8         129.0       1.0X
+dayofyear of timestamp wholestage on               1262           1270           7          7.9         126.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1236           1250          20          8.1         123.6       1.0X
-dayofmonth of timestamp wholestage on              1193           1195           3          8.4         119.3       1.0X
+dayofmonth of timestamp wholestage off             1239           1239           1          8.1         123.9       1.0X
+dayofmonth of timestamp wholestage on              1215           1222           8          8.2         121.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1402           1405           4          7.1         140.2       1.0X
-dayofweek of timestamp wholestage on               1352           1359           7          7.4         135.2       1.0X
+dayofweek of timestamp wholestage off              1421           1422           2          7.0         142.1       1.0X
+dayofweek of timestamp wholestage on               1379           1388           8          7.3         137.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1346           1347           2          7.4         134.6       1.0X
-weekday of timestamp wholestage on                 1294           1299           7          7.7         129.4       1.0X
+weekday of timestamp wholestage off                1349           1351           2          7.4         134.9       1.0X
+weekday of timestamp wholestage on                 1320           1327           8          7.6         132.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1000           1008          11         10.0         100.0       1.0X
-hour of timestamp wholestage on                     936            941           6         10.7          93.6       1.1X
+hour of timestamp wholestage off                   1024           1024           0          9.8         102.4       1.0X
+hour of timestamp wholestage on                     921            929          11         10.9          92.1       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  969            976          10         10.3          96.9       1.0X
-minute of timestamp wholestage on                   933            936           4         10.7          93.3       1.0X
+minute of timestamp wholestage off                  977            982           6         10.2          97.7       1.0X
+minute of timestamp wholestage on                   927            929           2         10.8          92.7       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1002           1005           3         10.0         100.2       1.0X
-second of timestamp wholestage on                   935            938           2         10.7          93.5       1.1X
+second of timestamp wholestage off                  987            989           3         10.1          98.7       1.0X
+second of timestamp wholestage on                   923            926           5         10.8          92.3       1.1X
 
 
 ================================================================================================
@@ -124,15 +124,15 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         308            316          11         32.5          30.8       1.0X
-current_date wholestage on                          265            275          12         37.8          26.5       1.2X
+current_date wholestage off                         303            311          12         33.0          30.3       1.0X
+current_date wholestage on                          266            271           5         37.5          26.6       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    307            312           7         32.6          30.7       1.0X
-current_timestamp wholestage on                     263            268           5         38.1          26.3       1.2X
+current_timestamp wholestage off                    297            297           1         33.7          29.7       1.0X
+current_timestamp wholestage on                     264            272           7         37.8          26.4       1.1X
 
 
 ================================================================================================
@@ -143,43 +143,43 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1061           1065           5          9.4         106.1       1.0X
-cast to date wholestage on                          985            991          11         10.2          98.5       1.1X
+cast to date wholestage off                        1062           1063           2          9.4         106.2       1.0X
+cast to date wholestage on                         1007           1021          20          9.9         100.7       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1261           1262           1          7.9         126.1       1.0X
-last_day wholestage on                             1223           1235          12          8.2         122.3       1.0X
+last_day wholestage off                            1262           1265           5          7.9         126.2       1.0X
+last_day wholestage on                             1244           1256          14          8.0         124.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1114           1119           7          9.0         111.4       1.0X
-next_day wholestage on                             1034           1038           3          9.7         103.4       1.1X
+next_day wholestage off                            1119           1121           2          8.9         111.9       1.0X
+next_day wholestage on                             1057           1063           6          9.5         105.7       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1059           1076          25          9.4         105.9       1.0X
-date_add wholestage on                             1012           1021           9          9.9         101.2       1.0X
+date_add wholestage off                            1054           1059           7          9.5         105.4       1.0X
+date_add wholestage on                             1037           1069          52          9.6         103.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1046           1046           0          9.6         104.6       1.0X
-date_sub wholestage on                             1019           1023           3          9.8         101.9       1.0X
+date_sub wholestage off                            1054           1056           4          9.5         105.4       1.0X
+date_sub wholestage on                             1036           1040           4          9.7         103.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1392           1393           1          7.2         139.2       1.0X
-add_months wholestage on                           1335           1346          14          7.5         133.5       1.0X
+add_months wholestage off                          1408           1421          19          7.1         140.8       1.0X
+add_months wholestage on                           1434           1440           7          7.0         143.4       1.0X
 
 
 ================================================================================================
@@ -190,8 +190,8 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5959           5994          50          1.7         595.9       1.0X
-format date wholestage on                          5991           6008          28          1.7         599.1       1.0X
+format date wholestage off                         5937           6169         328          1.7         593.7       1.0X
+format date wholestage on                          5836           5878          74          1.7         583.6       1.0X
 
 
 ================================================================================================
@@ -202,8 +202,8 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8851           8872          29          1.1         885.1       1.0X
-from_unixtime wholestage on                        8855           8872          10          1.1         885.5       1.0X
+from_unixtime wholestage off                       8904           8914          14          1.1         890.4       1.0X
+from_unixtime wholestage on                        8918           8936          13          1.1         891.8       1.0X
 
 
 ================================================================================================
@@ -214,15 +214,15 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1105           1107           2          9.0         110.5       1.0X
-from_utc_timestamp wholestage on                   1072           1084          11          9.3         107.2       1.0X
+from_utc_timestamp wholestage off                  1110           1112           3          9.0         111.0       1.0X
+from_utc_timestamp wholestage on                   1115           1119           3          9.0         111.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1531           1534           3          6.5         153.1       1.0X
-to_utc_timestamp wholestage on                     1451           1463          14          6.9         145.1       1.1X
+to_utc_timestamp wholestage off                    1524           1525           1          6.6         152.4       1.0X
+to_utc_timestamp wholestage on                     1450           1458          14          6.9         145.0       1.1X
 
 
 ================================================================================================
@@ -233,29 +233,29 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        360            366           8         27.8          36.0       1.0X
-cast interval wholestage on                         286            292           7         35.0          28.6       1.3X
+cast interval wholestage off                        341            342           1         29.3          34.1       1.0X
+cast interval wholestage on                         285            294           7         35.1          28.5       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1809           1814           8          5.5         180.9       1.0X
-datediff wholestage on                             1742           1751           8          5.7         174.2       1.0X
+datediff wholestage off                            1874           1881          10          5.3         187.4       1.0X
+datediff wholestage on                             1785           1791           3          5.6         178.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5007           5007           1          2.0         500.7       1.0X
-months_between wholestage on                       4957           4980          35          2.0         495.7       1.0X
+months_between wholestage off                      5038           5042           5          2.0         503.8       1.0X
+months_between wholestage on                       4979           4987           8          2.0         497.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1945           2027         116          0.5        1945.3       1.0X
-window wholestage on                              45637          45648           8          0.0       45637.2       0.0X
+window wholestage off                              1716           1841         177          0.6        1716.2       1.0X
+window wholestage on                              46024          46063          27          0.0       46024.1       0.0X
 
 
 ================================================================================================
@@ -266,134 +266,134 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2463           2465           2          4.1         246.3       1.0X
-date_trunc YEAR wholestage on                      2406           2409           3          4.2         240.6       1.0X
+date_trunc YEAR wholestage off                     2428           2429           2          4.1         242.8       1.0X
+date_trunc YEAR wholestage on                      2451           2469          12          4.1         245.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2462           2463           1          4.1         246.2       1.0X
-date_trunc YYYY wholestage on                      2407           2411           6          4.2         240.7       1.0X
+date_trunc YYYY wholestage off                     2423           2426           3          4.1         242.3       1.0X
+date_trunc YYYY wholestage on                      2454           2462           8          4.1         245.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2462           2466           6          4.1         246.2       1.0X
-date_trunc YY wholestage on                        2401           2406           4          4.2         240.1       1.0X
+date_trunc YY wholestage off                       2421           2441          28          4.1         242.1       1.0X
+date_trunc YY wholestage on                        2453           2461           9          4.1         245.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2437           2437           0          4.1         243.7       1.0X
-date_trunc MON wholestage on                       2416           2421           6          4.1         241.6       1.0X
+date_trunc MON wholestage off                      2425           2427           3          4.1         242.5       1.0X
+date_trunc MON wholestage on                       2431           2438           9          4.1         243.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2430           2437           9          4.1         243.0       1.0X
-date_trunc MONTH wholestage on                     2417           2423           5          4.1         241.7       1.0X
+date_trunc MONTH wholestage off                    2427           2433           8          4.1         242.7       1.0X
+date_trunc MONTH wholestage on                     2429           2435           4          4.1         242.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2429           2431           3          4.1         242.9       1.0X
-date_trunc MM wholestage on                        2417           2421           4          4.1         241.7       1.0X
+date_trunc MM wholestage off                       2425           2431           9          4.1         242.5       1.0X
+date_trunc MM wholestage on                        2430           2435           4          4.1         243.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2074           2075           2          4.8         207.4       1.0X
-date_trunc DAY wholestage on                       2001           2010          16          5.0         200.1       1.0X
+date_trunc DAY wholestage off                      2117           2119           4          4.7         211.7       1.0X
+date_trunc DAY wholestage on                       2036           2118         174          4.9         203.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2067           2067           0          4.8         206.7       1.0X
-date_trunc DD wholestage on                        2000           2003           3          5.0         200.0       1.0X
+date_trunc DD wholestage off                       2116           2119           5          4.7         211.6       1.0X
+date_trunc DD wholestage on                        2035           2043          10          4.9         203.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2074           2084          14          4.8         207.4       1.0X
-date_trunc HOUR wholestage on                      2057           2067          10          4.9         205.7       1.0X
+date_trunc HOUR wholestage off                     2013           2014           2          5.0         201.3       1.0X
+date_trunc HOUR wholestage on                      2077           2088          13          4.8         207.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    362            364           3         27.6          36.2       1.0X
-date_trunc MINUTE wholestage on                     319            333          14         31.3          31.9       1.1X
+date_trunc MINUTE wholestage off                    363            368           8         27.6          36.3       1.0X
+date_trunc MINUTE wholestage on                     321            326           7         31.2          32.1       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    361            366           7         27.7          36.1       1.0X
-date_trunc SECOND wholestage on                     324            341          23         30.9          32.4       1.1X
+date_trunc SECOND wholestage off                    365            366           0         27.4          36.5       1.0X
+date_trunc SECOND wholestage on                     319            332          16         31.4          31.9       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2385           2393          11          4.2         238.5       1.0X
-date_trunc WEEK wholestage on                      2313           2322           6          4.3         231.3       1.0X
+date_trunc WEEK wholestage off                     2371           2376           7          4.2         237.1       1.0X
+date_trunc WEEK wholestage on                      2314           2322           8          4.3         231.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3278           3280           2          3.1         327.8       1.0X
-date_trunc QUARTER wholestage on                   3228           3234           8          3.1         322.8       1.0X
+date_trunc QUARTER wholestage off                  3334           3335           1          3.0         333.4       1.0X
+date_trunc QUARTER wholestage on                   3286           3291           7          3.0         328.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           328            331           4         30.5          32.8       1.0X
-trunc year wholestage on                            286            295           9         35.0          28.6       1.1X
+trunc year wholestage off                           303            304           2         33.0          30.3       1.0X
+trunc year wholestage on                            283            291           5         35.3          28.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           317            319           3         31.5          31.7       1.0X
-trunc yyyy wholestage on                            283            287           6         35.3          28.3       1.1X
+trunc yyyy wholestage off                           324            330           8         30.9          32.4       1.0X
+trunc yyyy wholestage on                            283            291           9         35.3          28.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             321            321           0         31.1          32.1       1.0X
-trunc yy wholestage on                              284            293          11         35.2          28.4       1.1X
+trunc yy wholestage off                             304            305           3         32.9          30.4       1.0X
+trunc yy wholestage on                              283            302          28         35.3          28.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            318            319           1         31.4          31.8       1.0X
-trunc mon wholestage on                             283            287           4         35.4          28.3       1.1X
+trunc mon wholestage off                            315            319           6         31.7          31.5       1.0X
+trunc mon wholestage on                             284            287           5         35.3          28.4       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          319            321           3         31.3          31.9       1.0X
-trunc month wholestage on                           286            293           7         35.0          28.6       1.1X
+trunc month wholestage off                          305            314          13         32.8          30.5       1.0X
+trunc month wholestage on                           283            292          14         35.3          28.3       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             317            319           2         31.5          31.7       1.0X
-trunc mm wholestage on                              282            285           3         35.4          28.2       1.1X
+trunc mm wholestage off                             301            301           0         33.2          30.1       1.0X
+trunc mm wholestage on                              285            290           7         35.1          28.5       1.1X
 
 
 ================================================================================================
@@ -404,36 +404,36 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     219            220           0          4.6         219.4       1.0X
-to timestamp str wholestage on                      214            218           6          4.7         214.1       1.0X
+to timestamp str wholestage off                     218            220           3          4.6         218.4       1.0X
+to timestamp str wholestage on                      213            216           6          4.7         212.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1912           1913           2          0.5        1912.0       1.0X
-to_timestamp wholestage on                         1671           1675           7          0.6        1670.8       1.1X
+to_timestamp wholestage off                        1838           1842           5          0.5        1838.1       1.0X
+to_timestamp wholestage on                         1952           1971          11          0.5        1952.2       0.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1761           1763           3          0.6        1761.1       1.0X
-to_unix_timestamp wholestage on                    1695           1697           2          0.6        1695.4       1.0X
+to_unix_timestamp wholestage off                   1987           1988           1          0.5        1986.9       1.0X
+to_unix_timestamp wholestage on                    1944           1948           3          0.5        1944.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          267            272           7          3.7         266.9       1.0X
-to date str wholestage on                           266            267           2          3.8         265.8       1.0X
+to date str wholestage off                          263            264           0          3.8         263.5       1.0X
+to date str wholestage on                           263            265           2          3.8         262.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3705           3743          55          0.3        3704.6       1.0X
-to_date wholestage on                              3736           3746          11          0.3        3736.4       1.0X
+to_date wholestage off                             3560           3567          11          0.3        3559.7       1.0X
+to_date wholestage on                              3525           3534          10          0.3        3524.8       1.0X
 
 
 ================================================================================================
@@ -444,14 +444,14 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  400            406           6         12.5          80.1       1.0X
-From java.time.LocalDate                            343            349           7         14.6          68.6       1.2X
-Collect java.sql.Date                              1904           2739        1170          2.6         380.9       0.2X
-Collect java.time.LocalDate                        1477           1495          19          3.4         295.3       0.3X
-From java.sql.Timestamp                             376            388          10         13.3          75.2       1.1X
-From java.time.Instant                              237            239           3         21.1          47.4       1.7X
-Collect longs                                      1258           1356         111          4.0         251.7       0.3X
-Collect java.sql.Timestamp                         1878           1937          64          2.7         375.6       0.2X
-Collect java.time.Instant                          1667           1904         238          3.0         333.4       0.2X
+From java.sql.Date                                  405            416          16         12.3          81.0       1.0X
+From java.time.LocalDate                            344            352          14         14.5          68.8       1.2X
+Collect java.sql.Date                              1622           2553        1372          3.1         324.4       0.2X
+Collect java.time.LocalDate                        1464           1482          20          3.4         292.8       0.3X
+From java.sql.Timestamp                             248            258          15         20.2          49.6       1.6X
+From java.time.Instant                              237            243           7         21.1          47.4       1.7X
+Collect longs                                      1252           1341         109          4.0         250.5       0.3X
+Collect java.sql.Timestamp                         1515           1516           2          3.3         302.9       0.3X
+Collect java.time.Instant                          1379           1490          96          3.6         275.8       0.3X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 8c07bdb245b48..5d107c1e50fd8 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,97 +6,97 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  19806          19806           0          5.0         198.1       1.0X
-before 1582, noop                                 10689          10689           0          9.4         106.9       1.9X
-after 1582, rebase off                            31053          31053           0          3.2         310.5       0.6X
-after 1582, rebase on                             32316          32316           0          3.1         323.2       0.6X
-before 1582, rebase off                           21765          21765           0          4.6         217.6       0.9X
-before 1582, rebase on                            23011          23011           0          4.3         230.1       0.9X
+after 1582, noop                                  20802          20802           0          4.8         208.0       1.0X
+before 1582, noop                                 10728          10728           0          9.3         107.3       1.9X
+after 1582, rebase off                            32924          32924           0          3.0         329.2       0.6X
+after 1582, rebase on                             32627          32627           0          3.1         326.3       0.6X
+before 1582, rebase off                           21576          21576           0          4.6         215.8       1.0X
+before 1582, rebase on                            23115          23115           0          4.3         231.2       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12834          12921         140          7.8         128.3       1.0X
-after 1582, vec off, rebase on                    13031          13071          38          7.7         130.3       1.0X
-after 1582, vec on, rebase off                     3642           3679          60         27.5          36.4       3.5X
-after 1582, vec on, rebase on                      3747           3789          49         26.7          37.5       3.4X
-before 1582, vec off, rebase off                  13009          13061          72          7.7         130.1       1.0X
-before 1582, vec off, rebase on                   13928          13945          16          7.2         139.3       0.9X
-before 1582, vec on, rebase off                    3656           3665          13         27.4          36.6       3.5X
-before 1582, vec on, rebase on                     4361           4392          48         22.9          43.6       2.9X
+after 1582, vec off, rebase off                   12880          12984         178          7.8         128.8       1.0X
+after 1582, vec off, rebase on                    13118          13255         174          7.6         131.2       1.0X
+after 1582, vec on, rebase off                     3645           3698          76         27.4          36.4       3.5X
+after 1582, vec on, rebase on                      3709           3727          15         27.0          37.1       3.5X
+before 1582, vec off, rebase off                  13014          13051          36          7.7         130.1       1.0X
+before 1582, vec off, rebase on                   14195          14242          48          7.0         142.0       0.9X
+before 1582, vec on, rebase off                    3680           3773          92         27.2          36.8       3.5X
+before 1582, vec on, rebase on                     4310           4381          87         23.2          43.1       3.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2986           2986           0         33.5          29.9       1.0X
+after 1900, noop                                   3026           3026           0         33.1          30.3       1.0X
 before 1900, noop                                  2995           2995           0         33.4          30.0       1.0X
-after 1900, rebase off                            31493          31493           0          3.2         314.9       0.1X
-after 1900, rebase on                             31052          31052           0          3.2         310.5       0.1X
-before 1900, rebase off                           32102          32102           0          3.1         321.0       0.1X
-before 1900, rebase on                            32094          32094           0          3.1         320.9       0.1X
+after 1900, rebase off                            24294          24294           0          4.1         242.9       0.1X
+after 1900, rebase on                             24480          24480           0          4.1         244.8       0.1X
+before 1900, rebase off                           31120          31120           0          3.2         311.2       0.1X
+before 1900, rebase on                            31201          31201           0          3.2         312.0       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   20404          20422          29          4.9         204.0       1.0X
-after 1900, vec off, rebase on                    20198          20325         116          5.0         202.0       1.0X
-after 1900, vec on, rebase off                    12022          12051          39          8.3         120.2       1.7X
-after 1900, vec on, rebase on                     12019          12042          40          8.3         120.2       1.7X
-before 1900, vec off, rebase off                  21220          21263          38          4.7         212.2       1.0X
-before 1900, vec off, rebase on                   21172          21236          91          4.7         211.7       1.0X
-before 1900, vec on, rebase off                   12743          12792          46          7.8         127.4       1.6X
-before 1900, vec on, rebase on                    12665          12761          83          7.9         126.6       1.6X
+after 1900, vec off, rebase off                   18283          18309          39          5.5         182.8       1.0X
+after 1900, vec off, rebase on                    18235          18269          53          5.5         182.4       1.0X
+after 1900, vec on, rebase off                     9563           9589          27         10.5          95.6       1.9X
+after 1900, vec on, rebase on                      9463           9554          81         10.6          94.6       1.9X
+before 1900, vec off, rebase off                  21377          21469         118          4.7         213.8       0.9X
+before 1900, vec off, rebase on                   21265          21422         156          4.7         212.7       0.9X
+before 1900, vec on, rebase off                   12481          12524          46          8.0         124.8       1.5X
+before 1900, vec on, rebase on                    12360          12482         105          8.1         123.6       1.5X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   3047           3047           0         32.8          30.5       1.0X
-before 1900, noop                                  2973           2973           0         33.6          29.7       1.0X
-after 1900, rebase off                            16269          16269           0          6.1         162.7       0.2X
-after 1900, rebase on                             18375          18375           0          5.4         183.7       0.2X
-before 1900, rebase off                           16875          16875           0          5.9         168.7       0.2X
-before 1900, rebase on                            19574          19574           0          5.1         195.7       0.2X
+after 1900, noop                                   2984           2984           0         33.5          29.8       1.0X
+before 1900, noop                                  3003           3003           0         33.3          30.0       1.0X
+after 1900, rebase off                            15814          15814           0          6.3         158.1       0.2X
+after 1900, rebase on                             16250          16250           0          6.2         162.5       0.2X
+before 1900, rebase off                           16026          16026           0          6.2         160.3       0.2X
+before 1900, rebase on                            19735          19735           0          5.1         197.3       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15571          15671          95          6.4         155.7       1.0X
-after 1900, vec off, rebase on                    18076          18139          92          5.5         180.8       0.9X
-after 1900, vec on, rebase off                     4853           4888          38         20.6          48.5       3.2X
-after 1900, vec on, rebase on                      4924           4977          45         20.3          49.2       3.2X
-before 1900, vec off, rebase off                  15446          15485          34          6.5         154.5       1.0X
-before 1900, vec off, rebase on                   18743          18871         192          5.3         187.4       0.8X
-before 1900, vec on, rebase off                    4931           4944          20         20.3          49.3       3.2X
-before 1900, vec on, rebase on                     8217           8238          19         12.2          82.2       1.9X
+after 1900, vec off, rebase off                   15292          15351          57          6.5         152.9       1.0X
+after 1900, vec off, rebase on                    15753          15886         173          6.3         157.5       1.0X
+after 1900, vec on, rebase off                     4879           4923          52         20.5          48.8       3.1X
+after 1900, vec on, rebase on                      5018           5038          18         19.9          50.2       3.0X
+before 1900, vec off, rebase off                  15257          15311          53          6.6         152.6       1.0X
+before 1900, vec off, rebase on                   18459          18537          90          5.4         184.6       0.8X
+before 1900, vec on, rebase off                    4929           4946          15         20.3          49.3       3.1X
+before 1900, vec on, rebase on                     8254           8339          93         12.1          82.5       1.9X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2977           2977           0         33.6          29.8       1.0X
-before 1900, noop                                  2981           2981           0         33.5          29.8       1.0X
-after 1900, rebase off                            15336          15336           0          6.5         153.4       0.2X
-after 1900, rebase on                             17932          17932           0          5.6         179.3       0.2X
-before 1900, rebase off                           15507          15507           0          6.4         155.1       0.2X
-before 1900, rebase on                            19280          19280           0          5.2         192.8       0.2X
+after 1900, noop                                   2987           2987           0         33.5          29.9       1.0X
+before 1900, noop                                  3002           3002           0         33.3          30.0       1.0X
+after 1900, rebase off                            15215          15215           0          6.6         152.1       0.2X
+after 1900, rebase on                             15577          15577           0          6.4         155.8       0.2X
+before 1900, rebase off                           15505          15505           0          6.4         155.1       0.2X
+before 1900, rebase on                            19143          19143           0          5.2         191.4       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15690          15838         159          6.4         156.9       1.0X
-after 1900, vec off, rebase on                    18019          18185         194          5.5         180.2       0.9X
-after 1900, vec on, rebase off                     6021           6038          16         16.6          60.2       2.6X
-after 1900, vec on, rebase on                      8351           8383          38         12.0          83.5       1.9X
-before 1900, vec off, rebase off                  15863          15909          46          6.3         158.6       1.0X
-before 1900, vec off, rebase on                   18810          18955         179          5.3         188.1       0.8X
-before 1900, vec on, rebase off                    6055           6095          41         16.5          60.5       2.6X
-before 1900, vec on, rebase on                     8955           9007          50         11.2          89.5       1.8X
+after 1900, vec off, rebase off                   15330          15436         113          6.5         153.3       1.0X
+after 1900, vec off, rebase on                    15515          15549          30          6.4         155.1       1.0X
+after 1900, vec on, rebase off                     6056           6074          19         16.5          60.6       2.5X
+after 1900, vec on, rebase on                      6376           6390          14         15.7          63.8       2.4X
+before 1900, vec off, rebase off                  15490          15523          36          6.5         154.9       1.0X
+before 1900, vec off, rebase on                   18613          18685         118          5.4         186.1       0.8X
+before 1900, vec on, rebase off                    6065           6109          41         16.5          60.6       2.5X
+before 1900, vec on, rebase on                     9052           9082          32         11.0          90.5       1.7X
 
 
 ================================================================================================
@@ -107,36 +107,36 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  19561          19561           0          5.1         195.6       1.0X
-before 1582, noop                                 10676          10676           0          9.4         106.8       1.8X
-after 1582                                        27414          27414           0          3.6         274.1       0.7X
-before 1582                                       19257          19257           0          5.2         192.6       1.0X
+after 1582, noop                                  20653          20653           0          4.8         206.5       1.0X
+before 1582, noop                                 10707          10707           0          9.3         107.1       1.9X
+after 1582                                        28288          28288           0          3.5         282.9       0.7X
+before 1582                                       19196          19196           0          5.2         192.0       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               11016          11051          42          9.1         110.2       1.0X
-after 1582, vec on                                 3960           3971          16         25.3          39.6       2.8X
-before 1582, vec off                              11442          11466          23          8.7         114.4       1.0X
-before 1582, vec on                                4269           4288          20         23.4          42.7       2.6X
+after 1582, vec off                               10596          10621          37          9.4         106.0       1.0X
+after 1582, vec on                                 3886           3938          61         25.7          38.9       2.7X
+before 1582, vec off                              10955          10984          26          9.1         109.6       1.0X
+before 1582, vec on                                4236           4258          24         23.6          42.4       2.5X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2980           2980           0         33.6          29.8       1.0X
-before 1900, noop                                  2972           2972           0         33.6          29.7       1.0X
-after 1900                                        20638          20638           0          4.8         206.4       0.1X
-before 1900                                       22420          22420           0          4.5         224.2       0.1X
+after 1900, noop                                   2988           2988           0         33.5          29.9       1.0X
+before 1900, noop                                  3007           3007           0         33.3          30.1       1.0X
+after 1900                                        18082          18082           0          5.5         180.8       0.2X
+before 1900                                       22669          22669           0          4.4         226.7       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               14368          14449         137          7.0         143.7       1.0X
-after 1900, vec on                                 7233           7324         146         13.8          72.3       2.0X
-before 1900, vec off                              14936          14951          17          6.7         149.4       1.0X
-before 1900, vec on                                7767           7770           3         12.9          77.7       1.8X
+after 1900, vec off                               12029          12035           9          8.3         120.3       1.0X
+after 1900, vec on                                 5194           5197           3         19.3          51.9       2.3X
+before 1900, vec off                              14853          14875          23          6.7         148.5       0.8X
+before 1900, vec on                                7797           7836          60         12.8          78.0       1.5X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index 3ce5b36c35315..f2af3cc000f04 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,97 +6,97 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23101          23101           0          4.3         231.0       1.0X
-before 1582, noop                                 10164          10164           0          9.8         101.6       2.3X
-after 1582, rebase off                            37390          37390           0          2.7         373.9       0.6X
-after 1582, rebase on                             36108          36108           0          2.8         361.1       0.6X
-before 1582, rebase off                           22982          22982           0          4.4         229.8       1.0X
-before 1582, rebase on                            23899          23899           0          4.2         239.0       1.0X
+after 1582, noop                                  23567          23567           0          4.2         235.7       1.0X
+before 1582, noop                                 10570          10570           0          9.5         105.7       2.2X
+after 1582, rebase off                            35335          35335           0          2.8         353.3       0.7X
+after 1582, rebase on                             35645          35645           0          2.8         356.5       0.7X
+before 1582, rebase off                           21824          21824           0          4.6         218.2       1.1X
+before 1582, rebase on                            22532          22532           0          4.4         225.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12544          12579          41          8.0         125.4       1.0X
-after 1582, vec off, rebase on                    13127          13238          97          7.6         131.3       1.0X
-after 1582, vec on, rebase off                     3705           3715           8         27.0          37.1       3.4X
-after 1582, vec on, rebase on                      3866           3884          17         25.9          38.7       3.2X
-before 1582, vec off, rebase off                  12855          12862          11          7.8         128.6       1.0X
-before 1582, vec off, rebase on                   13723          13758          31          7.3         137.2       0.9X
-before 1582, vec on, rebase off                    3706           3724          25         27.0          37.1       3.4X
-before 1582, vec on, rebase on                     4872           4932          71         20.5          48.7       2.6X
+after 1582, vec off, rebase off                   13194          13266          81          7.6         131.9       1.0X
+after 1582, vec off, rebase on                    13402          13466          89          7.5         134.0       1.0X
+after 1582, vec on, rebase off                     3627           3657          29         27.6          36.3       3.6X
+after 1582, vec on, rebase on                      3818           3839          26         26.2          38.2       3.5X
+before 1582, vec off, rebase off                  13075          13146         115          7.6         130.7       1.0X
+before 1582, vec off, rebase on                   13794          13804          13          7.2         137.9       1.0X
+before 1582, vec on, rebase off                    3655           3675          21         27.4          36.6       3.6X
+before 1582, vec on, rebase on                     4579           4634          72         21.8          45.8       2.9X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2862           2862           0         34.9          28.6       1.0X
-before 1900, noop                                  2838           2838           0         35.2          28.4       1.0X
-after 1900, rebase off                            33249          33249           0          3.0         332.5       0.1X
-after 1900, rebase on                             33912          33912           0          2.9         339.1       0.1X
-before 1900, rebase off                           36083          36083           0          2.8         360.8       0.1X
-before 1900, rebase on                            35901          35901           0          2.8         359.0       0.1X
+after 1900, noop                                   2671           2671           0         37.4          26.7       1.0X
+before 1900, noop                                  2685           2685           0         37.2          26.8       1.0X
+after 1900, rebase off                            23899          23899           0          4.2         239.0       0.1X
+after 1900, rebase on                             24030          24030           0          4.2         240.3       0.1X
+before 1900, rebase off                           30178          30178           0          3.3         301.8       0.1X
+before 1900, rebase on                            30127          30127           0          3.3         301.3       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   19452          19496          38          5.1         194.5       1.0X
-after 1900, vec off, rebase on                    19505          19579          65          5.1         195.0       1.0X
-after 1900, vec on, rebase off                    11612          11703          86          8.6         116.1       1.7X
-after 1900, vec on, rebase on                     11533          11642          96          8.7         115.3       1.7X
-before 1900, vec off, rebase off                  20312          20368          88          4.9         203.1       1.0X
-before 1900, vec off, rebase on                   20217          20284          63          4.9         202.2       1.0X
-before 1900, vec on, rebase off                   12301          12372          64          8.1         123.0       1.6X
-before 1900, vec on, rebase on                    12253          12354          88          8.2         122.5       1.6X
+after 1900, vec off, rebase off                   16613          16685          75          6.0         166.1       1.0X
+after 1900, vec off, rebase on                    16487          16541          47          6.1         164.9       1.0X
+after 1900, vec on, rebase off                     8840           8870          49         11.3          88.4       1.9X
+after 1900, vec on, rebase on                      8795           8813          20         11.4          87.9       1.9X
+before 1900, vec off, rebase off                  20400          20441          62          4.9         204.0       0.8X
+before 1900, vec off, rebase on                   20430          20481          60          4.9         204.3       0.8X
+before 1900, vec on, rebase off                   12211          12290          73          8.2         122.1       1.4X
+before 1900, vec on, rebase on                    12231          12321          95          8.2         122.3       1.4X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2924           2924           0         34.2          29.2       1.0X
-before 1900, noop                                  2942           2942           0         34.0          29.4       1.0X
-after 1900, rebase off                            17323          17323           0          5.8         173.2       0.2X
-after 1900, rebase on                             20229          20229           0          4.9         202.3       0.1X
-before 1900, rebase off                           17166          17166           0          5.8         171.7       0.2X
-before 1900, rebase on                            21740          21740           0          4.6         217.4       0.1X
+after 1900, noop                                   2836           2836           0         35.3          28.4       1.0X
+before 1900, noop                                  2812           2812           0         35.6          28.1       1.0X
+after 1900, rebase off                            15976          15976           0          6.3         159.8       0.2X
+after 1900, rebase on                             16197          16197           0          6.2         162.0       0.2X
+before 1900, rebase off                           16140          16140           0          6.2         161.4       0.2X
+before 1900, rebase on                            20410          20410           0          4.9         204.1       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15031          15079          74          6.7         150.3       1.0X
-after 1900, vec off, rebase on                    18414          18429          18          5.4         184.1       0.8X
-after 1900, vec on, rebase off                     5020           5031          10         19.9          50.2       3.0X
-after 1900, vec on, rebase on                      5489           5527          35         18.2          54.9       2.7X
-before 1900, vec off, rebase off                  15062          15096          39          6.6         150.6       1.0X
-before 1900, vec off, rebase on                   19088          19137          64          5.2         190.9       0.8X
-before 1900, vec on, rebase off                    5021           5025           4         19.9          50.2       3.0X
-before 1900, vec on, rebase on                     9880           9913          47         10.1          98.8       1.5X
+after 1900, vec off, rebase off                   15297          15324          40          6.5         153.0       1.0X
+after 1900, vec off, rebase on                    15771          15832          59          6.3         157.7       1.0X
+after 1900, vec on, rebase off                     4922           4949          32         20.3          49.2       3.1X
+after 1900, vec on, rebase on                      5392           5411          17         18.5          53.9       2.8X
+before 1900, vec off, rebase off                  15227          15385         141          6.6         152.3       1.0X
+before 1900, vec off, rebase on                   19611          19658          41          5.1         196.1       0.8X
+before 1900, vec on, rebase off                    4965           5013          54         20.1          49.6       3.1X
+before 1900, vec on, rebase on                     9847           9873          43         10.2          98.5       1.6X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2951           2951           0         33.9          29.5       1.0X
-before 1900, noop                                  2999           2999           0         33.3          30.0       1.0X
-after 1900, rebase off                            16457          16457           0          6.1         164.6       0.2X
-after 1900, rebase on                             19587          19587           0          5.1         195.9       0.2X
-before 1900, rebase off                           16509          16509           0          6.1         165.1       0.2X
-before 1900, rebase on                            21375          21375           0          4.7         213.8       0.1X
+after 1900, noop                                   2818           2818           0         35.5          28.2       1.0X
+before 1900, noop                                  2805           2805           0         35.6          28.1       1.0X
+after 1900, rebase off                            15182          15182           0          6.6         151.8       0.2X
+after 1900, rebase on                             15614          15614           0          6.4         156.1       0.2X
+before 1900, rebase off                           15404          15404           0          6.5         154.0       0.2X
+before 1900, rebase on                            19747          19747           0          5.1         197.5       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15121          15250         127          6.6         151.2       1.0X
-after 1900, vec off, rebase on                    18436          18501          56          5.4         184.4       0.8X
-after 1900, vec on, rebase off                     6440           6464          21         15.5          64.4       2.3X
-after 1900, vec on, rebase on                      9547           9605          85         10.5          95.5       1.6X
-before 1900, vec off, rebase off                  15432          15467          59          6.5         154.3       1.0X
-before 1900, vec off, rebase on                   19236          19278          45          5.2         192.4       0.8X
-before 1900, vec on, rebase off                    6424           6436          15         15.6          64.2       2.4X
-before 1900, vec on, rebase on                    10297          10373          75          9.7         103.0       1.5X
+after 1900, vec off, rebase off                   15622          15649          24          6.4         156.2       1.0X
+after 1900, vec off, rebase on                    15572          15677         119          6.4         155.7       1.0X
+after 1900, vec on, rebase off                     6345           6358          15         15.8          63.5       2.5X
+after 1900, vec on, rebase on                      6780           6834          92         14.8          67.8       2.3X
+before 1900, vec off, rebase off                  15540          15584          38          6.4         155.4       1.0X
+before 1900, vec off, rebase on                   19590          19653          55          5.1         195.9       0.8X
+before 1900, vec on, rebase off                    6374           6381          10         15.7          63.7       2.5X
+before 1900, vec on, rebase on                    10530          10544          25          9.5         105.3       1.5X
 
 
 ================================================================================================
@@ -107,36 +107,36 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23323          23323           0          4.3         233.2       1.0X
-before 1582, noop                                 10178          10178           0          9.8         101.8       2.3X
-after 1582                                        31568          31568           0          3.2         315.7       0.7X
-before 1582                                       19918          19918           0          5.0         199.2       1.2X
+after 1582, noop                                  23825          23825           0          4.2         238.2       1.0X
+before 1582, noop                                 10501          10501           0          9.5         105.0       2.3X
+after 1582                                        32134          32134           0          3.1         321.3       0.7X
+before 1582                                       19947          19947           0          5.0         199.5       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10580          10610          39          9.5         105.8       1.0X
-after 1582, vec on                                 3746           3765          19         26.7          37.5       2.8X
-before 1582, vec off                              11052          11143         124          9.0         110.5       1.0X
-before 1582, vec on                                4170           4187          16         24.0          41.7       2.5X
+after 1582, vec off                               10034          10056          22         10.0         100.3       1.0X
+after 1582, vec on                                 3664           3698          30         27.3          36.6       2.7X
+before 1582, vec off                              10472          10502          30          9.5         104.7       1.0X
+before 1582, vec on                                4052           4098          42         24.7          40.5       2.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2994           2994           0         33.4          29.9       1.0X
-before 1900, noop                                  2986           2986           0         33.5          29.9       1.0X
-after 1900                                        22517          22517           0          4.4         225.2       0.1X
-before 1900                                       23253          23253           0          4.3         232.5       0.1X
+after 1900, noop                                   2812           2812           0         35.6          28.1       1.0X
+before 1900, noop                                  2801           2801           0         35.7          28.0       1.0X
+after 1900                                        18290          18290           0          5.5         182.9       0.2X
+before 1900                                       22344          22344           0          4.5         223.4       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               14042          14199         152          7.1         140.4       1.0X
-after 1900, vec on                                 7873           7901          25         12.7          78.7       1.8X
-before 1900, vec off                              14716          14768          48          6.8         147.2       1.0X
-before 1900, vec on                                8491           8503          10         11.8          84.9       1.7X
+after 1900, vec off                               11257          11279          32          8.9         112.6       1.0X
+after 1900, vec on                                 5296           5310          15         18.9          53.0       2.1X
+before 1900, vec off                              14700          14758          72          6.8         147.0       0.8X
+before 1900, vec on                                8576           8665         150         11.7          85.8       1.3X
 
 

From 080c51e6b6268002948dd14171233bd35d954529 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 5 May 2020 14:15:31 +0000
Subject: [PATCH 0635/1280] [SPARK-31641][SQL] Fix days conversions by JSON
 legacy parser

### What changes were proposed in this pull request?
Perform days rebasing while converting days from JSON string field. In Spark 2.4 and earlier versions, the days are interpreted as days since the epoch in the hybrid calendar (Julian + Gregorian since 1582-10-15). Since Spark 3.0, the base calendar was switched to Proleptic Gregorian calendar, so, the days should be rebased to represent the same local date.

### Why are the changes needed?
The changes fix a bug and restore compatibility with Spark 2.4 in which:
```scala
scala> spark.read.schema("d date").json(Seq("{'d': '-141704'}").toDS).show
+----------+
|         d|
+----------+
|1582-01-01|
+----------+
```

### Does this PR introduce _any_ user-facing change?
Yes.

Before:
```scala
scala> spark.read.schema("d date").json(Seq("{'d': '-141704'}").toDS).show
+----------+
|         d|
+----------+
|1582-01-11|
+----------+
```

After:
```scala
scala> spark.read.schema("d date").json(Seq("{'d': '-141704'}").toDS).show
+----------+
|         d|
+----------+
|1582-01-01|
+----------+
```

### How was this patch tested?
Add a test to `JsonSuite`.

Closes #28453 from MaxGekk/json-rebase-legacy-days.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bd264299317bba91f2dc1dc27fd51e6bc0609d66)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/json/JacksonParser.scala      |  2 +-
 .../sql/execution/datasources/json/JsonSuite.scala   | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index 8965a81feee19..a52c3450e83df 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -259,7 +259,7 @@ class JacksonParser(
                 // In Spark 1.5.0, we store the data as number of days since epoch in string.
                 // So, we just convert it to Int.
                 try {
-                  parser.getText.toInt
+                  RebaseDateTime.rebaseJulianToGregorianDays(parser.getText.toInt)
                 } catch {
                   case _: NumberFormatException => throw e
                 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 999eadbf40c4a..4982991bdda56 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2653,13 +2653,17 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     }
   }
 
-  test("SPARK-30960: parse date/timestamp string with legacy format") {
-    val ds = Seq("{'t': '2020-1-12 3:23:34.12', 'd': '2020-1-12 T', 'd2': '12345'}").toDS()
-    val json = spark.read.schema("t timestamp, d date, d2 date").json(ds)
+  test("SPARK-30960, SPARK-31641: parse date/timestamp string with legacy format") {
+    val julianDay = -141704 // 1582-01-01 in Julian calendar
+    val ds = Seq(
+      s"{'t': '2020-1-12 3:23:34.12', 'd': '2020-1-12 T', 'd2': '12345', 'd3': '$julianDay'}"
+    ).toDS()
+    val json = spark.read.schema("t timestamp, d date, d2 date, d3 date").json(ds)
     checkAnswer(json, Row(
       Timestamp.valueOf("2020-1-12 3:23:34.12"),
       Date.valueOf("2020-1-12"),
-      Date.valueOf(LocalDate.ofEpochDay(12345))))
+      Date.valueOf(LocalDate.ofEpochDay(12345)),
+      Date.valueOf("1582-01-01")))
   }
 
   test("exception mode for parsing date/timestamp string") {

From 5b2d1f44591ad1de6252ea66dc69b3f37fb6feb9 Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Tue, 5 May 2020 12:17:24 -0700
Subject: [PATCH 0636/1280] [SPARK-31644][BUILD] Make Spark's guava version
 configurable from the command line

### What changes were proposed in this pull request?

This adds the maven property guava.version which can be
used to control the guava version for a build.

It does not change the current version.

### Why are the changes needed?

All future Hadoop releases are going to be built with a later guava version, including Hadoop 3.1.4. This means to run the spark tests with that release you need to update the spark guava version. This patch lets whoever builds spark do this locally.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

ran the hadoop-cloud module tests with the 3.1.4 RC0

```
mvn -T 1  -Phadoop-3.2 -Dhadoop.version=3.1.4 -Psnapshots-and-staging -Phadoop-cloud,yarn,kinesis-asl test --pl hadoop-cloud
```

observed the linkage problem

```
  java.lang.NoSuchMethodError: com.google.common.base.Preconditions.checkArgument(ZLjava/lang/String;Ljava/lang/Object;)V
  at org.apache.hadoop.conf.Configuration.set(Configuration.java:1357)
```

made the version configurable, retested with

```
-Phadoop-3.2 -Dhadoop.version=3.1.4 -Psnapshots-and-staging Dguava.version=27.0-jre
```

all good.

Closes #28455 from steveloughran/SPARK-31644-guava-version.

Authored-by: Steve Loughran <stevel@cloudera.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 86c4e4352565cf271a96b650f533ed89a725322c)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 pom.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index c79ba5a21d6ea..870d0da1f48eb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,6 +178,7 @@
     <!-- org.apache.commons/commons-pool2/-->
     <commons-pool2.version>2.6.2</commons-pool2.version>
     <datanucleus-core.version>4.1.17</datanucleus-core.version>
+    <guava.version>14.0.1</guava.version>
     <janino.version>3.0.16</janino.version>
     <jersey.version>2.30</jersey.version>
     <joda.version>2.10.5</joda.version>
@@ -444,7 +445,7 @@
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>14.0.1</version>
+        <version>${guava.version}</version>
         <scope>provided</scope>
       </dependency>
       <dependency>

From b763c23307a69f018b833add14b7da18fa35df02 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 5 May 2020 12:36:42 -0700
Subject: [PATCH 0637/1280] [SPARK-31643][TEST] Fix flaky
 o.a.s.scheduler.BarrierTaskContextSuite.barrier task killed, interrupt

### What changes were proposed in this pull request?

Make sure the task has nearly reached `context.barrier()` before killing.

### Why are the changes needed?

In case of the task is killed before it reaches `context.barrier()`, the task will not create the expected file.

```
Error Message
org.scalatest.exceptions.TestFailedException: new java.io.File(dir, killedFlagFile).exists() was false Expect barrier task being killed.
Stacktrace
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: new java.io.File(dir, killedFlagFile).exists() was false Expect barrier task being killed.
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
	at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
	at org.apache.spark.scheduler.BarrierTaskContextSuite.$anonfun$testBarrierTaskKilled$1(BarrierTaskContextSuite.scala:266)
	at org.apache.spark.scheduler.BarrierTaskContextSuite.$anonfun$testBarrierTaskKilled$1$adapted(BarrierTaskContextSuite.scala:226)
	at org.apache.spark.SparkFunSuite.withTempDir(SparkFunSuite.scala:163)
	at org.apache.spark.scheduler.BarrierTaskContextSuite.testBarrierTaskKilled(BarrierTaskContextSuite.scala:226)
	at org.apache.spark.scheduler.BarrierTaskContextSuite.$anonfun$new$29(BarrierTaskContextSuite.scala:277)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
```

[Here's](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/122273/testReport/org.apache.spark.scheduler/BarrierTaskContextSuite/barrier_task_killed__interrupt/) the full error messages.

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #28454 from Ngone51/fix_kill_interrupt.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 61a6ca5d3f623c2a8b49277ac62d77bf4dbfa84f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/scheduler/BarrierTaskContextSuite.scala   | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 2242d2807652a..c4e5e7c700652 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -21,10 +21,13 @@ import java.io.File
 
 import scala.util.Random
 
+import org.scalatest.concurrent.Eventually
+import org.scalatest.time.SpanSugar._
+
 import org.apache.spark._
 import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 
-class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
+class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with Eventually {
 
   def initLocalClusterSparkContext(numWorker: Int = 4): Unit = {
     val conf = new SparkConf()
@@ -224,12 +227,14 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
 
   def testBarrierTaskKilled(interruptOnKill: Boolean): Unit = {
     withTempDir { dir =>
+      val runningFlagFile = "barrier.task.running"
       val killedFlagFile = "barrier.task.killed"
       val rdd = sc.makeRDD(Seq(0, 1), 2)
       val rdd2 = rdd.barrier().mapPartitions { it =>
         val context = BarrierTaskContext.get()
         if (context.partitionId() == 0) {
           try {
+            new File(dir, runningFlagFile).createNewFile()
             context.barrier()
           } catch {
             case _: TaskKilledException =>
@@ -248,8 +253,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext {
           if (partitionId == 0) {
             new Thread {
               override def run: Unit = {
-                Thread.sleep(1000)
-                sc.killTaskAttempt(taskStart.taskInfo.taskId, interruptThread = interruptOnKill)
+                eventually(timeout(10.seconds)) {
+                  assert(new File(dir, runningFlagFile).exists())
+                  sc.killTaskAttempt(taskStart.taskInfo.taskId, interruptThread = interruptOnKill)
+                }
               }
             }.start()
           }

From 45ed71212d4a1649cfe39b485b5daefbedb5d591 Mon Sep 17 00:00:00 2001
From: sychen <sychen@ctrip.com>
Date: Wed, 6 May 2020 10:56:19 +0900
Subject: [PATCH 0638/1280] [SPARK-31590][SQL] Metadata-only queries should not
 include subquery in partition filters

### What changes were proposed in this pull request?
Metadata-only queries should not include subquery in partition filters.

### Why are the changes needed?

Apply the `OptimizeMetadataOnlyQuery` rule again, will get the exception `Cannot evaluate expression: scalar-subquery`.

### Does this PR introduce any user-facing change?
Yes. When `spark.sql.optimizer.metadataOnly` is enabled, it succeeds when the queries include subquery in partition filters.

### How was this patch tested?

add UT

Closes #28383 from cxzl25/fix_SPARK-31590.

Authored-by: sychen <sychen@ctrip.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 588966d696373c11e963116a0e08ee33c30f0dfb)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/execution/OptimizeMetadataOnlyQuery.scala  |  2 +-
 .../execution/OptimizeMetadataOnlyQuerySuite.scala | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
index 47284f3448b7b..492d177c7c773 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala
@@ -117,7 +117,7 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
         case a: AttributeReference =>
           a.withName(relation.output.find(_.semanticEquals(a)).get.name)
       }
-    }
+    }.filterNot(SubqueryExpression.hasSubquery)
 
     child transform {
       case plan if plan eq relation =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala
index afb438e0bbc72..68691e2f7fdac 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuerySuite.scala
@@ -103,6 +103,20 @@ class OptimizeMetadataOnlyQuerySuite extends QueryTest with SharedSparkSession {
     "select partcol2, min(partcol1) from srcpart where partcol1 = 0 group by partcol2",
     "select max(c1) from (select partcol1 + 1 as c1 from srcpart where partcol1 = 0) t")
 
+  testMetadataOnly(
+    "SPARK-31590 Metadata-only queries should not include subquery in partition filters",
+    """
+      |SELECT partcol1, MAX(partcol2) AS partcol2
+      |FROM srcpart
+      |WHERE partcol1 = (
+      |  SELECT MAX(partcol1)
+      |  FROM srcpart
+      |)
+      |AND partcol2 = 'even'
+      |GROUP BY partcol1
+      |""".stripMargin
+  )
+
   testNotMetadataOnly(
     "Don't optimize metadata only query for non-partition columns",
     "select col1 from srcpart group by col1",

From fcd566ea15ac1a58efe152570b933f4d7a3d7c4e Mon Sep 17 00:00:00 2001
From: Daoyuan Wang <me@daoyuan.wang>
Date: Wed, 6 May 2020 04:34:43 +0000
Subject: [PATCH 0639/1280] [SPARK-31595][SQL] Spark sql should allow unescaped
 quote mark in quoted string

### What changes were proposed in this pull request?
`def splitSemiColon` cannot handle unescaped quote mark like "'" or '"' correctly. When there are unmatched quotes in a string, `splitSemiColon` will not drop off semicolon as expected.

### Why are the changes needed?
Some regex expression will use quote mark in string. We should process semicolon correctly.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Added Unit test and also manual test.

Closes #28393 from adrian-wang/unescaped.

Authored-by: Daoyuan Wang <me@daoyuan.wang>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 53a9bf8fece7322312cbe93c9224c04f645a0f5e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala  | 9 +++++++--
 .../apache/spark/sql/hive/thriftserver/CliSuite.scala    | 9 +++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 5ed0cb0407484..bffa24c469601 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -507,6 +507,9 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
   }
 
   // Adapted splitSemiColon from Hive 2.3's CliDriver.splitSemiColon.
+  // Note: [SPARK-31595] if there is a `'` in a double quoted string, or a `"` in a single quoted
+  // string, the origin implementation from Hive will not drop the trailing semicolon as expected,
+  // hence we refined this function a little bit.
   private def splitSemiColon(line: String): JList[String] = {
     var insideSingleQuote = false
     var insideDoubleQuote = false
@@ -519,13 +522,15 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
     for (index <- 0 until line.length) {
       if (line.charAt(index) == '\'' && !insideComment) {
         // take a look to see if it is escaped
-        if (!escape) {
+        // See the comment above about SPARK-31595
+        if (!escape && !insideDoubleQuote) {
           // flip the boolean variable
           insideSingleQuote = !insideSingleQuote
         }
       } else if (line.charAt(index) == '\"' && !insideComment) {
         // take a look to see if it is escaped
-        if (!escape) {
+        // See the comment above about SPARK-31595
+        if (!escape && !insideSingleQuote) {
           // flip the boolean variable
           insideDoubleQuote = !insideDoubleQuote
         }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index abefb46b32140..265e7772a691c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -500,4 +500,13 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
         |   ;""".stripMargin -> "testcomment"
     )
   }
+
+  test("SPARK-31595 Should allow unescaped quote mark in quoted string") {
+    runCliWithin(1.minute)(
+      "SELECT '\"legal string a';select 1 + 234;".stripMargin -> "235"
+    )
+    runCliWithin(1.minute)(
+      "SELECT \"legal 'string b\";select 22222 + 1;".stripMargin -> "22223"
+    )
+  }
 }

From 860849d73c994a6f8970dab631e097cb234c7abe Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Wed, 6 May 2020 04:50:06 +0000
Subject: [PATCH 0640/1280] [SPARK-31365][SQL] Enable nested predicate pushdown
 per data sources

### What changes were proposed in this pull request?

This patch proposes to replace `NESTED_PREDICATE_PUSHDOWN_ENABLED` with `NESTED_PREDICATE_PUSHDOWN_V1_SOURCE_LIST` which can configure which v1 data sources are enabled with nested predicate pushdown.

### Why are the changes needed?

We added nested predicate pushdown feature that is configured by `NESTED_PREDICATE_PUSHDOWN_ENABLED`. However, this config is all or nothing config, and applies on all data sources.

In order to not introduce API breaking change after enabling nested predicate pushdown, we'd like to set nested predicate pushdown per data sources. Please also refer to the comments https://github.com/apache/spark/pull/27728#discussion_r410829720.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Added/Modified unit tests.

Closes #28366 from viirya/SPARK-31365.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 4952f1a03cc48d9f1c3d2539ffa19bf051e398bf)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  21 ++--
 .../sql/execution/DataSourceScanExec.scala    |   5 +-
 .../datasources/DataSourceStrategy.scala      |  94 ++++++++++------
 .../datasources/DataSourceUtils.scala         |  16 +++
 .../datasources/FileSourceStrategy.scala      |   7 +-
 .../datasources/v2/DataSourceV2Strategy.scala |   7 +-
 .../datasources/v2/PushDownUtils.scala        |   3 +-
 .../v2/V2ScanRelationPushDown.scala           |   2 +-
 ...quetNestedPredicatePushDownBenchmark.scala |   8 +-
 .../datasources/DataSourceStrategySuite.scala |  20 +++-
 .../parquet/ParquetFilterSuite.scala          | 103 +++++++++++-------
 11 files changed, 186 insertions(+), 100 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 72946a9fe01b5..51404a254a3f0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2063,16 +2063,17 @@ object SQLConf {
       .booleanConf
       .createWithDefault(true)
 
-  val NESTED_PREDICATE_PUSHDOWN_ENABLED =
-    buildConf("spark.sql.optimizer.nestedPredicatePushdown.enabled")
-      .internal()
-      .doc("When true, Spark tries to push down predicates for nested columns and or names " +
-        "containing `dots` to data sources. Currently, Parquet implements both optimizations " +
-        "while ORC only supports predicates for names containing `dots`. The other data sources" +
-        "don't support this feature yet.")
+  val NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST =
+    buildConf("spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources")
+      .internal()
+      .doc("A comma-separated list of data source short names or fully qualified data source " +
+        "implementation class names for which Spark tries to push down predicates for nested " +
+        "columns and/or names containing `dots` to data sources. Currently, Parquet implements " +
+        "both optimizations while ORC only supports predicates for names containing `dots`. The " +
+        "other data sources don't support this feature yet. So the default value is 'parquet,orc'.")
       .version("3.0.0")
-      .booleanConf
-      .createWithDefault(true)
+      .stringConf
+      .createWithDefault("parquet,orc")
 
   val SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED =
     buildConf("spark.sql.optimizer.serializer.nestedSchemaPruning.enabled")
@@ -3077,8 +3078,6 @@ class SQLConf extends Serializable with Logging {
 
   def nestedSchemaPruningEnabled: Boolean = getConf(NESTED_SCHEMA_PRUNING_ENABLED)
 
-  def nestedPredicatePushdownEnabled: Boolean = getConf(NESTED_PREDICATE_PUSHDOWN_ENABLED)
-
   def serializerNestedSchemaPruningEnabled: Boolean =
     getConf(SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED)
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 0be76ad6c5bd3..90a3f9788ca42 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -326,7 +326,10 @@ case class FileSourceScanExec(
   }
 
   @transient
-  private lazy val pushedDownFilters = dataFilters.flatMap(DataSourceStrategy.translateFilter)
+  private lazy val pushedDownFilters = {
+    val supportNestedPredicatePushdown = DataSourceUtils.supportNestedPredicatePushdown(relation)
+    dataFilters.flatMap(DataSourceStrategy.translateFilter(_, supportNestedPredicatePushdown))
+  }
 
   override lazy val metadata: Map[String, String] = {
     def seqToString(seq: Seq[Any]) = seq.mkString("[", ", ", "]")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
index a58038d127818..23454d7d5e7f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategy.scala
@@ -448,60 +448,62 @@ object DataSourceStrategy {
     }
   }
 
-  private def translateLeafNodeFilter(predicate: Expression): Option[Filter] = predicate match {
-    case expressions.EqualTo(PushableColumn(name), Literal(v, t)) =>
+  private def translateLeafNodeFilter(
+      predicate: Expression,
+      pushableColumn: PushableColumnBase): Option[Filter] = predicate match {
+    case expressions.EqualTo(pushableColumn(name), Literal(v, t)) =>
       Some(sources.EqualTo(name, convertToScala(v, t)))
-    case expressions.EqualTo(Literal(v, t), PushableColumn(name)) =>
+    case expressions.EqualTo(Literal(v, t), pushableColumn(name)) =>
       Some(sources.EqualTo(name, convertToScala(v, t)))
 
-    case expressions.EqualNullSafe(PushableColumn(name), Literal(v, t)) =>
+    case expressions.EqualNullSafe(pushableColumn(name), Literal(v, t)) =>
       Some(sources.EqualNullSafe(name, convertToScala(v, t)))
-    case expressions.EqualNullSafe(Literal(v, t), PushableColumn(name)) =>
+    case expressions.EqualNullSafe(Literal(v, t), pushableColumn(name)) =>
       Some(sources.EqualNullSafe(name, convertToScala(v, t)))
 
-    case expressions.GreaterThan(PushableColumn(name), Literal(v, t)) =>
+    case expressions.GreaterThan(pushableColumn(name), Literal(v, t)) =>
       Some(sources.GreaterThan(name, convertToScala(v, t)))
-    case expressions.GreaterThan(Literal(v, t), PushableColumn(name)) =>
+    case expressions.GreaterThan(Literal(v, t), pushableColumn(name)) =>
       Some(sources.LessThan(name, convertToScala(v, t)))
 
-    case expressions.LessThan(PushableColumn(name), Literal(v, t)) =>
+    case expressions.LessThan(pushableColumn(name), Literal(v, t)) =>
       Some(sources.LessThan(name, convertToScala(v, t)))
-    case expressions.LessThan(Literal(v, t), PushableColumn(name)) =>
+    case expressions.LessThan(Literal(v, t), pushableColumn(name)) =>
       Some(sources.GreaterThan(name, convertToScala(v, t)))
 
-    case expressions.GreaterThanOrEqual(PushableColumn(name), Literal(v, t)) =>
+    case expressions.GreaterThanOrEqual(pushableColumn(name), Literal(v, t)) =>
       Some(sources.GreaterThanOrEqual(name, convertToScala(v, t)))
-    case expressions.GreaterThanOrEqual(Literal(v, t), PushableColumn(name)) =>
+    case expressions.GreaterThanOrEqual(Literal(v, t), pushableColumn(name)) =>
       Some(sources.LessThanOrEqual(name, convertToScala(v, t)))
 
-    case expressions.LessThanOrEqual(PushableColumn(name), Literal(v, t)) =>
+    case expressions.LessThanOrEqual(pushableColumn(name), Literal(v, t)) =>
       Some(sources.LessThanOrEqual(name, convertToScala(v, t)))
-    case expressions.LessThanOrEqual(Literal(v, t), PushableColumn(name)) =>
+    case expressions.LessThanOrEqual(Literal(v, t), pushableColumn(name)) =>
       Some(sources.GreaterThanOrEqual(name, convertToScala(v, t)))
 
-    case expressions.InSet(e @ PushableColumn(name), set) =>
+    case expressions.InSet(e @ pushableColumn(name), set) =>
       val toScala = CatalystTypeConverters.createToScalaConverter(e.dataType)
       Some(sources.In(name, set.toArray.map(toScala)))
 
     // Because we only convert In to InSet in Optimizer when there are more than certain
     // items. So it is possible we still get an In expression here that needs to be pushed
     // down.
-    case expressions.In(e @ PushableColumn(name), list) if list.forall(_.isInstanceOf[Literal]) =>
+    case expressions.In(e @ pushableColumn(name), list) if list.forall(_.isInstanceOf[Literal]) =>
       val hSet = list.map(_.eval(EmptyRow))
       val toScala = CatalystTypeConverters.createToScalaConverter(e.dataType)
       Some(sources.In(name, hSet.toArray.map(toScala)))
 
-    case expressions.IsNull(PushableColumn(name)) =>
+    case expressions.IsNull(pushableColumn(name)) =>
       Some(sources.IsNull(name))
-    case expressions.IsNotNull(PushableColumn(name)) =>
+    case expressions.IsNotNull(pushableColumn(name)) =>
       Some(sources.IsNotNull(name))
-    case expressions.StartsWith(PushableColumn(name), Literal(v: UTF8String, StringType)) =>
+    case expressions.StartsWith(pushableColumn(name), Literal(v: UTF8String, StringType)) =>
       Some(sources.StringStartsWith(name, v.toString))
 
-    case expressions.EndsWith(PushableColumn(name), Literal(v: UTF8String, StringType)) =>
+    case expressions.EndsWith(pushableColumn(name), Literal(v: UTF8String, StringType)) =>
       Some(sources.StringEndsWith(name, v.toString))
 
-    case expressions.Contains(PushableColumn(name), Literal(v: UTF8String, StringType)) =>
+    case expressions.Contains(pushableColumn(name), Literal(v: UTF8String, StringType)) =>
       Some(sources.StringContains(name, v.toString))
 
     case expressions.Literal(true, BooleanType) =>
@@ -518,8 +520,9 @@ object DataSourceStrategy {
    *
    * @return a `Some[Filter]` if the input [[Expression]] is convertible, otherwise a `None`.
    */
-  protected[sql] def translateFilter(predicate: Expression): Option[Filter] = {
-    translateFilterWithMapping(predicate, None)
+  protected[sql] def translateFilter(
+      predicate: Expression, supportNestedPredicatePushdown: Boolean): Option[Filter] = {
+    translateFilterWithMapping(predicate, None, supportNestedPredicatePushdown)
   }
 
   /**
@@ -529,11 +532,13 @@ object DataSourceStrategy {
    * @param translatedFilterToExpr An optional map from leaf node filter expressions to its
    *                               translated [[Filter]]. The map is used for rebuilding
    *                               [[Expression]] from [[Filter]].
+   * @param nestedPredicatePushdownEnabled Whether nested predicate pushdown is enabled.
    * @return a `Some[Filter]` if the input [[Expression]] is convertible, otherwise a `None`.
    */
   protected[sql] def translateFilterWithMapping(
       predicate: Expression,
-      translatedFilterToExpr: Option[mutable.HashMap[sources.Filter, Expression]])
+      translatedFilterToExpr: Option[mutable.HashMap[sources.Filter, Expression]],
+      nestedPredicatePushdownEnabled: Boolean)
     : Option[Filter] = {
     predicate match {
       case expressions.And(left, right) =>
@@ -547,21 +552,26 @@ object DataSourceStrategy {
         // Pushing one leg of AND down is only safe to do at the top level.
         // You can see ParquetFilters' createFilter for more details.
         for {
-          leftFilter <- translateFilterWithMapping(left, translatedFilterToExpr)
-          rightFilter <- translateFilterWithMapping(right, translatedFilterToExpr)
+          leftFilter <- translateFilterWithMapping(
+            left, translatedFilterToExpr, nestedPredicatePushdownEnabled)
+          rightFilter <- translateFilterWithMapping(
+            right, translatedFilterToExpr, nestedPredicatePushdownEnabled)
         } yield sources.And(leftFilter, rightFilter)
 
       case expressions.Or(left, right) =>
         for {
-          leftFilter <- translateFilterWithMapping(left, translatedFilterToExpr)
-          rightFilter <- translateFilterWithMapping(right, translatedFilterToExpr)
+          leftFilter <- translateFilterWithMapping(
+            left, translatedFilterToExpr, nestedPredicatePushdownEnabled)
+          rightFilter <- translateFilterWithMapping(
+            right, translatedFilterToExpr, nestedPredicatePushdownEnabled)
         } yield sources.Or(leftFilter, rightFilter)
 
       case expressions.Not(child) =>
-        translateFilterWithMapping(child, translatedFilterToExpr).map(sources.Not)
+        translateFilterWithMapping(child, translatedFilterToExpr, nestedPredicatePushdownEnabled)
+          .map(sources.Not)
 
       case other =>
-        val filter = translateLeafNodeFilter(other)
+        val filter = translateLeafNodeFilter(other, PushableColumn(nestedPredicatePushdownEnabled))
         if (filter.isDefined && translatedFilterToExpr.isDefined) {
           translatedFilterToExpr.get(filter.get) = predicate
         }
@@ -608,8 +618,9 @@ object DataSourceStrategy {
 
     // A map from original Catalyst expressions to corresponding translated data source filters.
     // If a predicate is not in this map, it means it cannot be pushed down.
+    val supportNestedPredicatePushdown = DataSourceUtils.supportNestedPredicatePushdown(relation)
     val translatedMap: Map[Expression, Filter] = predicates.flatMap { p =>
-      translateFilter(p).map(f => p -> f)
+      translateFilter(p, supportNestedPredicatePushdown).map(f => p -> f)
     }.toMap
 
     val pushedFilters: Seq[Filter] = translatedMap.values.toSeq
@@ -650,9 +661,10 @@ object DataSourceStrategy {
 /**
  * Find the column name of an expression that can be pushed down.
  */
-object PushableColumn {
+abstract class PushableColumnBase {
+  val nestedPredicatePushdownEnabled: Boolean
+
   def unapply(e: Expression): Option[String] = {
-    val nestedPredicatePushdownEnabled = SQLConf.get.nestedPredicatePushdownEnabled
     import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.MultipartIdentifierHelper
     def helper(e: Expression): Option[Seq[String]] = e match {
       case a: Attribute =>
@@ -668,3 +680,21 @@ object PushableColumn {
     helper(e).map(_.quoted)
   }
 }
+
+object PushableColumn {
+  def apply(nestedPredicatePushdownEnabled: Boolean): PushableColumnBase = {
+    if (nestedPredicatePushdownEnabled) {
+      PushableColumnAndNestedColumn
+    } else {
+      PushableColumnWithoutNestedColumn
+    }
+  }
+}
+
+object PushableColumnAndNestedColumn extends PushableColumnBase {
+  override val nestedPredicatePushdownEnabled = true
+}
+
+object PushableColumnWithoutNestedColumn extends PushableColumnBase {
+  override val nestedPredicatePushdownEnabled = false
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index b19de6d3deb89..45a9b1a808cf3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.execution.datasources
 
+import java.util.Locale
+
 import org.apache.hadoop.fs.Path
 import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
@@ -24,6 +26,7 @@ import org.json4s.jackson.Serialization
 import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
@@ -68,6 +71,19 @@ object DataSourceUtils {
   private[sql] def isDataFile(fileName: String) =
     !(fileName.startsWith("_") || fileName.startsWith("."))
 
+  /**
+   * Returns if the given relation's V1 datasource provider supports nested predicate pushdown.
+   */
+  private[sql] def supportNestedPredicatePushdown(relation: BaseRelation): Boolean =
+    relation match {
+      case hs: HadoopFsRelation =>
+        val supportedDatasources =
+          Utils.stringToSeq(SQLConf.get.getConf(SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST)
+            .toLowerCase(Locale.ROOT))
+        supportedDatasources.contains(hs.toString)
+      case _ => false
+    }
+
   def needRebaseDateTime(lookupFileMeta: String => String): Option[Boolean] = {
     if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") {
       return Some(false)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index f45495121a980..477937d66ad9b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -178,8 +178,11 @@ object FileSourceStrategy extends Strategy with Logging {
       // Partition keys are not available in the statistics of the files.
       val dataFilters =
         normalizedFiltersWithoutSubqueries.filter(_.references.intersect(partitionSet).isEmpty)
-      logInfo(s"Pushed Filters: " +
-        s"${dataFilters.flatMap(DataSourceStrategy.translateFilter).mkString(",")}")
+      val supportNestedPredicatePushdown =
+        DataSourceUtils.supportNestedPredicatePushdown(fsRelation)
+      val pushedFilters = dataFilters
+        .flatMap(DataSourceStrategy.translateFilter(_, supportNestedPredicatePushdown))
+      logInfo(s"Pushed Filters: ${pushedFilters.mkString(",")}")
 
       // Predicates with both partition keys and attributes need to be evaluated after the scan.
       val afterScanFilters = filterSet -- partitionKeyFilters.filter(_.references.nonEmpty)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index 8f4e2d256c714..cca80c0cb6d57 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -180,8 +180,9 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     case OverwriteByExpression(r: DataSourceV2Relation, deleteExpr, query, writeOptions, _) =>
       // fail if any filter cannot be converted. correctness depends on removing all matching data.
       val filters = splitConjunctivePredicates(deleteExpr).map {
-        filter => DataSourceStrategy.translateFilter(deleteExpr).getOrElse(
-          throw new AnalysisException(s"Cannot translate expression to source filter: $filter"))
+        filter => DataSourceStrategy.translateFilter(deleteExpr,
+          supportNestedPredicatePushdown = true).getOrElse(
+            throw new AnalysisException(s"Cannot translate expression to source filter: $filter"))
       }.toArray
       r.table.asWritable match {
         case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
@@ -205,7 +206,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
           // correctness depends on removing all matching data.
           val filters = DataSourceStrategy.normalizeExprs(condition.toSeq, output)
               .flatMap(splitConjunctivePredicates(_).map {
-                f => DataSourceStrategy.translateFilter(f).getOrElse(
+                f => DataSourceStrategy.translateFilter(f, true).getOrElse(
                   throw new AnalysisException(s"Exec update failed:" +
                       s" cannot translate expression to source filter: $f"))
               }).toArray
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
index 33338b06565c9..1a6f03f54f2e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/PushDownUtils.scala
@@ -48,7 +48,8 @@ object PushDownUtils extends PredicateHelper {
 
         for (filterExpr <- filters) {
           val translated =
-            DataSourceStrategy.translateFilterWithMapping(filterExpr, Some(translatedFilterToExpr))
+            DataSourceStrategy.translateFilterWithMapping(filterExpr, Some(translatedFilterToExpr),
+              nestedPredicatePushdownEnabled = true)
           if (translated.isEmpty) {
             untranslatableExprs += filterExpr
           } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
index 59089fa6b77e9..b168e848f0b6f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanRelationPushDown.scala
@@ -59,7 +59,7 @@ object V2ScanRelationPushDown extends Rule[LogicalPlan] {
 
       val wrappedScan = scan match {
         case v1: V1Scan =>
-          val translated = filters.flatMap(DataSourceStrategy.translateFilter)
+          val translated = filters.flatMap(DataSourceStrategy.translateFilter(_, true))
           V1ScanWrapper(v1, translated, pushedFilters)
         case _ => scan
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
index 11bc91a4b1551..d2bd962b50654 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/ParquetNestedPredicatePushDownBenchmark.scala
@@ -48,12 +48,12 @@ object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
   private def addCase(
       benchmark: Benchmark,
       inputPath: String,
-      enableNestedPD: Boolean,
+      enableNestedPD: String,
       name: String,
       withFilter: DataFrame => DataFrame): Unit = {
     val loadDF = spark.read.parquet(inputPath)
     benchmark.addCase(name) { _ =>
-      withSQLConf((SQLConf.NESTED_PREDICATE_PUSHDOWN_ENABLED.key, enableNestedPD.toString)) {
+      withSQLConf((SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST.key, enableNestedPD)) {
         withFilter(loadDF).noop()
       }
     }
@@ -67,13 +67,13 @@ object ParquetNestedPredicatePushDownBenchmark extends SqlBasedBenchmark {
       addCase(
         benchmark,
         outputPath,
-        enableNestedPD = false,
+        enableNestedPD = "",
         "Without nested predicate Pushdown",
         withFilter)
       addCase(
         benchmark,
         outputPath,
-        enableNestedPD = true,
+        enableNestedPD = "parquet",
         "With nested predicate Pushdown",
         withFilter)
       benchmark.run()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
index a775a97895cfc..b94918eccd46e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/DataSourceStrategySuite.scala
@@ -289,14 +289,26 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
   test("SPARK-31027 test `PushableColumn.unapply` that finds the column name of " +
     "an expression that can be pushed down") {
     attrInts.foreach { case (attrInt, colName) =>
-      assert(PushableColumn.unapply(attrInt) === Some(colName))
+      assert(PushableColumnAndNestedColumn.unapply(attrInt) === Some(colName))
+
+      if (colName.contains(".")) {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrInt) === None)
+      } else {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrInt) === Some(colName))
+      }
     }
     attrStrs.foreach { case (attrStr, colName) =>
-      assert(PushableColumn.unapply(attrStr) === Some(colName))
+      assert(PushableColumnAndNestedColumn.unapply(attrStr) === Some(colName))
+
+      if (colName.contains(".")) {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrStr) === None)
+      } else {
+        assert(PushableColumnWithoutNestedColumn.unapply(attrStr) === Some(colName))
+      }
     }
 
     // `Abs(col)` can not be pushed down, so it returns `None`
-    assert(PushableColumn.unapply(Abs('col.int)) === None)
+    assert(PushableColumnAndNestedColumn.unapply(Abs('col.int)) === None)
   }
 
   /**
@@ -305,7 +317,7 @@ class DataSourceStrategySuite extends PlanTest with SharedSparkSession {
    */
   def testTranslateFilter(catalystFilter: Expression, result: Option[sources.Filter]): Unit = {
     assertResult(result) {
-      DataSourceStrategy.translateFilter(catalystFilter)
+      DataSourceStrategy.translateFilter(catalystFilter, true)
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 20bfb32e2c0d4..5cf21293fd07f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -33,7 +33,8 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.optimizer.InferFiltersFromConstraints
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
-import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation}
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.parseColumnPath
+import org.apache.spark.sql.execution.datasources.{DataSourceStrategy, HadoopFsRelation, LogicalRelation, PushableColumnAndNestedColumn}
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanRelation
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.functions._
@@ -1588,47 +1589,67 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
       expected: Seq[Row]): Unit = {
     val output = predicate.collect { case a: Attribute => a }.distinct
 
-    withSQLConf(
-      SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_TIMESTAMP_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED.key -> "true",
-      SQLConf.PARQUET_FILTER_PUSHDOWN_STRING_STARTSWITH_ENABLED.key -> "true",
-      // Disable adding filters from constraints because it adds, for instance,
-      // is-not-null to pushed filters, which makes it hard to test if the pushed
-      // filter is expected or not (this had to be fixed with SPARK-13495).
-      SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> InferFiltersFromConstraints.ruleName,
-      SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false") {
-      val query = df
-        .select(output.map(e => Column(e)): _*)
-        .where(Column(predicate))
+    Seq(("parquet", true), ("", false)).map { case (pushdownDsList, nestedPredicatePushdown) =>
+      withSQLConf(
+        SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_TIMESTAMP_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_DECIMAL_ENABLED.key -> "true",
+        SQLConf.PARQUET_FILTER_PUSHDOWN_STRING_STARTSWITH_ENABLED.key -> "true",
+        // Disable adding filters from constraints because it adds, for instance,
+        // is-not-null to pushed filters, which makes it hard to test if the pushed
+        // filter is expected or not (this had to be fixed with SPARK-13495).
+        SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> InferFiltersFromConstraints.ruleName,
+        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
+        SQLConf.NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST.key -> pushdownDsList) {
+        val query = df
+          .select(output.map(e => Column(e)): _*)
+          .where(Column(predicate))
+
+        val nestedOrAttributes = predicate.collectFirst {
+          case g: GetStructField => g
+          case a: Attribute => a
+        }
+        assert(nestedOrAttributes.isDefined, "No GetStructField nor Attribute is detected.")
+
+        val parsed = parseColumnPath(
+          PushableColumnAndNestedColumn.unapply(nestedOrAttributes.get).get)
+
+        val containsNestedColumnOrDot = parsed.length > 1 || parsed(0).contains(".")
+
+        var maybeRelation: Option[HadoopFsRelation] = None
+        val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {
+          case PhysicalOperation(_, filters,
+          LogicalRelation(relation: HadoopFsRelation, _, _, _)) =>
+            maybeRelation = Some(relation)
+            filters
+        }.flatten.reduceLeftOption(_ && _)
+        assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query")
+
+        val (_, selectedFilters, _) =
+          DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)
+        // If predicates contains nested column or dot, we push down the predicates only if
+        // "parquet" is in `NESTED_PREDICATE_PUSHDOWN_V1_SOURCE_LIST`.
+        if (nestedPredicatePushdown || !containsNestedColumnOrDot) {
+          assert(selectedFilters.nonEmpty, "No filter is pushed down")
+          val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
+          val parquetFilters = createParquetFilters(schema)
+          // In this test suite, all the simple predicates are convertible here.
+          assert(parquetFilters.convertibleFilters(selectedFilters) === selectedFilters)
+          val pushedParquetFilters = selectedFilters.map { pred =>
+            val maybeFilter = parquetFilters.createFilter(pred)
+            assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $pred")
+            maybeFilter.get
+          }
+          // Doesn't bother checking type parameters here (e.g. `Eq[Integer]`)
+          assert(pushedParquetFilters.exists(_.getClass === filterClass),
+            s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
 
-      var maybeRelation: Option[HadoopFsRelation] = None
-      val maybeAnalyzedPredicate = query.queryExecution.optimizedPlan.collect {
-        case PhysicalOperation(_, filters,
-        LogicalRelation(relation: HadoopFsRelation, _, _, _)) =>
-          maybeRelation = Some(relation)
-          filters
-      }.flatten.reduceLeftOption(_ && _)
-      assert(maybeAnalyzedPredicate.isDefined, "No filter is analyzed from the given query")
-
-      val (_, selectedFilters, _) =
-        DataSourceStrategy.selectFilters(maybeRelation.get, maybeAnalyzedPredicate.toSeq)
-      assert(selectedFilters.nonEmpty, "No filter is pushed down")
-      val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
-      val parquetFilters = createParquetFilters(schema)
-      // In this test suite, all the simple predicates are convertible here.
-      assert(parquetFilters.convertibleFilters(selectedFilters) === selectedFilters)
-      val pushedParquetFilters = selectedFilters.map { pred =>
-        val maybeFilter = parquetFilters.createFilter(pred)
-        assert(maybeFilter.isDefined, s"Couldn't generate filter predicate for $pred")
-        maybeFilter.get
+          checker(stripSparkFilter(query), expected)
+        } else {
+          assert(selectedFilters.isEmpty, "There is filter pushed down")
+        }
       }
-      // Doesn't bother checking type parameters here (e.g. `Eq[Integer]`)
-      assert(pushedParquetFilters.exists(_.getClass === filterClass),
-        s"${pushedParquetFilters.map(_.getClass).toList} did not contain ${filterClass}.")
-
-      checker(stripSparkFilter(query), expected)
     }
   }
 }
@@ -1667,7 +1688,7 @@ class ParquetV2FilterSuite extends ParquetFilterSuite {
         case PhysicalOperation(_, filters,
             DataSourceV2ScanRelation(_, scan: ParquetScan, _)) =>
           assert(filters.nonEmpty, "No filter is analyzed from the given query")
-          val sourceFilters = filters.flatMap(DataSourceStrategy.translateFilter).toArray
+          val sourceFilters = filters.flatMap(DataSourceStrategy.translateFilter(_, true)).toArray
           val pushedFilters = scan.pushedFilters
           assert(pushedFilters.nonEmpty, "No filter is pushed down")
           val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)

From c1b5a4f1877d057973cb0667cdbb7c27550033b8 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 6 May 2020 12:52:53 +0000
Subject: [PATCH 0641/1280] [SPARK-31650][SQL] Fix wrong UI in case of
 AdaptiveSparkPlanExec has unmanaged subqueries

### What changes were proposed in this pull request?

Make the non-subquery `AdaptiveSparkPlanExec` update UI again after execute/executeCollect/executeTake/executeTail if the `AdaptiveSparkPlanExec` has subqueries which do not belong to any query stages.

### Why are the changes needed?

If there're subqueries do not belong to any query stages of the main query, the main query could get final physical plan and update UI before those subqueries finished. As a result, the UI can not reflect the change from the subqueries, e.g. new nodes generated from subqueries.

Before:

<img width="335" alt="before_aqe_ui" src="https://user-images.githubusercontent.com/16397174/81149758-671a9480-8fb1-11ea-84c4-9a4520e2b08e.png">

After:
<img width="546" alt="after_aqe_ui" src="https://user-images.githubusercontent.com/16397174/81149752-63870d80-8fb1-11ea-9852-f41e11afe216.png">

### Does this PR introduce _any_ user-facing change?

No(AQE feature hasn't been released).

### How was this patch tested?

Tested manually.

Closes #28460 from Ngone51/fix_aqe_ui.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b16ea8e1ab58bd24c50d31ce0dfc6c79c87fa3b2)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      | 41 ++++++++++++++-----
 .../adaptive/AdaptiveQueryExecSuite.scala     |  7 +++-
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index f00dce2ca8ae9..cd6936bb22bd6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -138,6 +138,13 @@ case class AdaptiveSparkPlanExec(
     executedPlan.resetMetrics()
   }
 
+  private def getExecutionId: Option[Long] = {
+    // If the `QueryExecution` does not match the current execution ID, it means the execution ID
+    // belongs to another (parent) query, and we should not call update UI in this query.
+    Option(context.session.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY))
+      .map(_.toLong).filter(SQLExecution.getQueryExecution(_) eq context.qe)
+  }
+
   private def getFinalPhysicalPlan(): SparkPlan = lock.synchronized {
     if (isFinalPlan) return currentPhysicalPlan
 
@@ -145,11 +152,7 @@ case class AdaptiveSparkPlanExec(
     // `plan.queryExecution.rdd`, we need to set active session here as new plan nodes can be
     // created in the middle of the execution.
     context.session.withActive {
-      // If the `QueryExecution` does not match the current execution ID, it means the execution ID
-      // belongs to another (parent) query, and we should not call update UI in this query.
-      val executionId =
-        Option(context.session.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY))
-          .map(_.toLong).filter(SQLExecution.getQueryExecution(_) eq context.qe)
+      val executionId = getExecutionId
       var currentLogicalPlan = currentPhysicalPlan.logicalLink.get
       var result = createQueryStages(currentPhysicalPlan)
       val events = new LinkedBlockingQueue[StageMaterializationEvent]()
@@ -230,25 +233,43 @@ case class AdaptiveSparkPlanExec(
       currentPhysicalPlan = applyPhysicalRules(result.newPlan, queryStageOptimizerRules)
       isFinalPlan = true
       executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
-      logOnLevel(s"Final plan: $currentPhysicalPlan")
       currentPhysicalPlan
     }
   }
 
+  // Use a lazy val to avoid this being called more than once.
+  @transient private lazy val finalPlanUpdate: Unit = {
+    // Subqueries that don't belong to any query stage of the main query will execute after the
+    // last UI update in `getFinalPhysicalPlan`, so we need to update UI here again to make sure
+    // the newly generated nodes of those subqueries are updated.
+    if (!isSubquery && currentPhysicalPlan.find(_.subqueries.nonEmpty).isDefined) {
+      getExecutionId.foreach(onUpdatePlan(_, Seq.empty))
+    }
+    logOnLevel(s"Final plan: $currentPhysicalPlan")
+  }
+
   override def executeCollect(): Array[InternalRow] = {
-    getFinalPhysicalPlan().executeCollect()
+    val rdd = getFinalPhysicalPlan().executeCollect()
+    finalPlanUpdate
+    rdd
   }
 
   override def executeTake(n: Int): Array[InternalRow] = {
-    getFinalPhysicalPlan().executeTake(n)
+    val rdd = getFinalPhysicalPlan().executeTake(n)
+    finalPlanUpdate
+    rdd
   }
 
   override def executeTail(n: Int): Array[InternalRow] = {
-    getFinalPhysicalPlan().executeTail(n)
+    val rdd = getFinalPhysicalPlan().executeTail(n)
+    finalPlanUpdate
+    rdd
   }
 
   override def doExecute(): RDD[InternalRow] = {
-    getFinalPhysicalPlan().execute()
+    val rdd = getFinalPhysicalPlan().execute()
+    finalPlanUpdate
+    rdd
   }
 
   protected override def stringArgs: Iterator[Any] = Iterator(s"isFinalPlan=$isFinalPlan")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index c6caffafa3841..f30d1e9ab9a5d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -72,12 +72,15 @@ class AdaptiveQueryExecSuite
     }
     val planAfter = dfAdaptive.queryExecution.executedPlan
     assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
+    val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
 
     spark.sparkContext.listenerBus.waitUntilEmpty()
-    assert(finalPlanCnt == 1)
+    // AQE will post `SparkListenerSQLAdaptiveExecutionUpdate` twice in case of subqueries that
+    // exist out of query stages.
+    val expectedFinalPlanCnt = adaptivePlan.find(_.subqueries.nonEmpty).map(_ => 2).getOrElse(1)
+    assert(finalPlanCnt == expectedFinalPlanCnt)
     spark.sparkContext.removeSparkListener(listener)
 
-    val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan
     val exchanges = adaptivePlan.collect {
       case e: Exchange => e
     }

From f8a20c470bf115b0834970ce02eb2ec103e0f6df Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 7 May 2020 09:00:59 +0900
Subject: [PATCH 0642/1280] [SPARK-31647][SQL] Deprecate
 'spark.sql.optimizer.metadataOnly' configuration

### What changes were proposed in this pull request?

This PR proposes to deprecate 'spark.sql.optimizer.metadataOnly' configuration and remove it in the future release.

### Why are the changes needed?

This optimization can cause a potential correctness issue, see also SPARK-26709.
Also, it seems difficult to extend the optimization. Basically you should whitelist all available functions. It costs some maintenance overhead, see also SPARK-31590.

Looks we should just better let users use `SparkSessionExtensions` instead if they must use, and remove it in Spark side.

### Does this PR introduce _any_ user-facing change?

Yes, setting `spark.sql.optimizer.metadataOnly` will show a deprecation warning:

```scala
scala> spark.conf.unset("spark.sql.optimizer.metadataOnly")
```
```
20/05/06 12:57:23 WARN SQLConf: The SQL config 'spark.sql.optimizer.metadataOnly' has been
 deprecated in Spark v3.0 and may be removed in the future. Avoid to depend on this optimization
 to prevent a potential correctness issue. If you must use, use 'SparkSessionExtensions' instead to
inject it as a custom rule.
```
```scala
scala> spark.conf.set("spark.sql.optimizer.metadataOnly", "true")
```
```
20/05/06 12:57:44 WARN SQLConf: The SQL config 'spark.sql.optimizer.metadataOnly' has been
deprecated in Spark v3.0 and may be removed in the future. Avoid to depend on this optimization
 to prevent a potential correctness issue. If you must use, use 'SparkSessionExtensions' instead to
inject it as a custom rule.
```

### How was this patch tested?

Manually tested.

Closes #28459 from HyukjinKwon/SPARK-31647.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 5c5dd77d6a29b014b3fe4b4015f5c7199650a378)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../scala/org/apache/spark/sql/internal/SQLConf.scala | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 51404a254a3f0..8d673c504d77a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -844,8 +844,10 @@ object SQLConf {
     .doc("When true, enable the metadata-only query optimization that use the table's metadata " +
       "to produce the partition columns instead of table scans. It applies when all the columns " +
       "scanned are partition columns and the query has an aggregate operator that satisfies " +
-      "distinct semantics. By default the optimization is disabled, since it may return " +
-      "incorrect results when the files are empty.")
+      "distinct semantics. By default the optimization is disabled, and deprecated as of Spark " +
+      "3.0 since it may return incorrect results when the files are empty, see also SPARK-26709." +
+      "It will be removed in the future releases. If you must use, use 'SparkSessionExtensions' " +
+      "instead to inject it as a custom rule.")
     .version("2.1.1")
     .booleanConf
     .createWithDefault(false)
@@ -2587,7 +2589,10 @@ object SQLConf {
       DeprecatedConfig(ARROW_FALLBACK_ENABLED.key, "3.0",
         s"Use '${ARROW_PYSPARK_FALLBACK_ENABLED.key}' instead of it."),
       DeprecatedConfig(SHUFFLE_TARGET_POSTSHUFFLE_INPUT_SIZE.key, "3.0",
-        s"Use '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}' instead of it.")
+        s"Use '${ADVISORY_PARTITION_SIZE_IN_BYTES.key}' instead of it."),
+      DeprecatedConfig(OPTIMIZER_METADATA_ONLY.key, "3.0",
+        "Avoid to depend on this optimization to prevent a potential correctness issue. " +
+          "If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.")
     )
 
     Map(configs.map { cfg => cfg.key -> cfg } : _*)

From 43d8d54e8beab25dbdf75ca93943f774b93297ea Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 7 May 2020 09:46:42 +0900
Subject: [PATCH 0643/1280] [SPARK-31361][SQL][FOLLOWUP] Use
 LEGACY_PARQUET_REBASE_DATETIME_IN_READ instead of avro config in
 ParquetIOSuite

### What changes were proposed in this pull request?
Replace the Avro SQL config `LEGACY_AVRO_REBASE_DATETIME_IN_READ ` by `LEGACY_PARQUET_REBASE_DATETIME_IN_READ ` in `ParquetIOSuite`.

### Why are the changes needed?
Avro config is not relevant to the parquet tests.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `ParquetIOSuite` via
```
./build/sbt "test:testOnly *ParquetIOSuite"
```

Closes #28461 from MaxGekk/fix-conf-in-ParquetIOSuite.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3d38bc2605ab01d61127c09e1bf6ed6a6683ed3e)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/execution/datasources/parquet/ParquetIOSuite.scala    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 239db7d4f30d2..7f0a2286690bf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -955,7 +955,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
             // The file metadata indicates if it needs rebase or not, so we can always get the
             // correct result regardless of the "rebaseInRead" config.
             Seq(true, false).foreach { rebase =>
-              withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
                 checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
               }
             }
@@ -984,7 +984,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       // The file metadata indicates if it needs rebase or not, so we can always get the correct
       // result regardless of the "rebaseInRead" config.
       Seq(true, false).foreach { rebase =>
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
           checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
         }
       }

From dc7324e5e39783995b90e64d4737127c10a210cf Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Thu, 7 May 2020 09:57:08 +0900
Subject: [PATCH 0644/1280] [SPARK-31365][SQL][FOLLOWUP] Refine config document
 for nested predicate pushdown

### What changes were proposed in this pull request?

This is a followup to address the https://github.com/apache/spark/pull/28366#discussion_r420611872 by refining the SQL config document.

### Why are the changes needed?

Make developers less confusing.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Only doc change.

Closes #28468 from viirya/SPARK-31365-followup.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 9bf738724a3895551464d8ba0d455bc90868983f)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 8d673c504d77a..6c18280ce4d55 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2070,7 +2070,8 @@ object SQLConf {
       .internal()
       .doc("A comma-separated list of data source short names or fully qualified data source " +
         "implementation class names for which Spark tries to push down predicates for nested " +
-        "columns and/or names containing `dots` to data sources. Currently, Parquet implements " +
+        "columns and/or names containing `dots` to data sources. This configuration is only " +
+        "effective with file-based data source in DSv1. Currently, Parquet implements " +
         "both optimizations while ORC only supports predicates for names containing `dots`. The " +
         "other data sources don't support this feature yet. So the default value is 'parquet,orc'.")
       .version("3.0.0")

From dfb916f6b65b05dd7fd58853d97e06bc7e75d8be Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 7 May 2020 07:52:29 +0000
Subject: [PATCH 0645/1280] [SPARK-31361][SQL][TESTS][FOLLOWUP] Check
 non-vectorized Parquet reader while date/timestamp rebasing

### What changes were proposed in this pull request?
In PR, I propose to modify two tests of `ParquetIOSuite`:
- SPARK-31159: rebasing timestamps in write
- SPARK-31159: rebasing dates in write

to check non-vectorized Parquet reader together with vectorized reader.

### Why are the changes needed?
To improve test coverage and make sure that non-vectorized reader behaves similar to the vectorized reader.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `PaquetIOSuite`:
```
$ ./build/sbt "test:testOnly *ParquetIOSuite"
```

Closes #28466 from MaxGekk/test-novec-rebase-ParquetIOSuite.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 272d229005b7166ab83bbb8f44a4d5e9d89424a1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/parquet/ParquetIOSuite.scala  | 54 +++++++++++--------
 1 file changed, 32 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 7f0a2286690bf..af66aa092c37e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -952,18 +952,24 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
                 .write
                 .parquet(path)
             }
-            // The file metadata indicates if it needs rebase or not, so we can always get the
-            // correct result regardless of the "rebaseInRead" config.
-            Seq(true, false).foreach { rebase =>
-              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
-                checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
-              }
-            }
 
-            // Force to not rebase to prove the written datetime values are rebased and we will get
-            // wrong result if we don't rebase while reading.
-            withSQLConf("spark.test.forceNoRebase" -> "true") {
-              checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(nonRebased)))
+            Seq(false, true).foreach { vectorized =>
+              withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+                // The file metadata indicates if it needs rebase or not, so we can always get the
+                // correct result regardless of the "rebaseInRead" config.
+                Seq(true, false).foreach { rebase =>
+                  withSQLConf(
+                    SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                    checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
+                  }
+                }
+
+                // Force to not rebase to prove the written datetime values are rebased
+                // and we will get wrong result if we don't rebase while reading.
+                withSQLConf("spark.test.forceNoRebase" -> "true") {
+                  checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(nonRebased)))
+                }
+              }
             }
           }
         }
@@ -981,18 +987,22 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           .parquet(path)
       }
 
-      // The file metadata indicates if it needs rebase or not, so we can always get the correct
-      // result regardless of the "rebaseInRead" config.
-      Seq(true, false).foreach { rebase =>
-        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
-          checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
-        }
-      }
+      Seq(false, true).foreach { vectorized =>
+        withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+          // The file metadata indicates if it needs rebase or not, so we can always get the correct
+          // result regardless of the "rebaseInRead" config.
+          Seq(true, false).foreach { rebase =>
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+              checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
+            }
+          }
 
-      // Force to not rebase to prove the written datetime values are rebased and we will get
-      // wrong result if we don't rebase while reading.
-      withSQLConf("spark.test.forceNoRebase" -> "true") {
-        checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-07")))
+          // Force to not rebase to prove the written datetime values are rebased and we will get
+          // wrong result if we don't rebase while reading.
+          withSQLConf("spark.test.forceNoRebase" -> "true") {
+            checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-07")))
+          }
+        }
       }
     }
   }

From fafe0f311cc1c48002b68f26ab9b274ffd565665 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 7 May 2020 14:37:03 +0900
Subject: [PATCH 0646/1280] [SPARK-31631][TESTS] Fix test flakiness caused by
 MiniKdc which throws 'address in use' BindException with retry

### What changes were proposed in this pull request?
The `Kafka*Suite`s are flaky because of the Hadoop MiniKdc issue - https://issues.apache.org/jira/browse/HADOOP-12656
> Looking at MiniKdc implementation, if port is 0, the constructor use ServerSocket to find an unused port, assign the port number to the member variable port and close the ServerSocket object; later, in initKDCServer(), instantiate a TcpTransport object and bind at that port.

> It appears that the port may be used in between, and then throw the exception.

Related test failures are suspected,  such as https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/122225/testReport/org.apache.spark.sql.kafka010/KafkaDelegationTokenSuite/_It_is_not_a_test_it_is_a_sbt_testing_SuiteSelector_/

```scala
[info] org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite *** ABORTED *** (15 seconds, 426 milliseconds)
[info]   java.net.BindException: Address already in use
[info]   at sun.nio.ch.Net.bind0(Native Method)
[info]   at sun.nio.ch.Net.bind(Net.java:433)
[info]   at sun.nio.ch.Net.bind(Net.java:425)
[info]   at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
[info]   at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
[info]   at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:198)
[info]   at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:51)
[info]   at org.apache.mina.core.polling.AbstractPollingIoAcceptor.registerHandles(AbstractPollingIoAcceptor.java:547)
[info]   at org.apache.mina.core.polling.AbstractPollingIoAcceptor.access$400(AbstractPollingIoAcceptor.java:68)
[info]   at org.apache.mina.core.polling.AbstractPollingIoAcceptor$Acceptor.run(AbstractPollingIoAcceptor.java:422)
[info]   at org.apache.mina.util.NamePreservingRunnable.run(NamePreservingRunnable.java:64)
[info]   at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[info]   at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[info]   at java.lang.Thread.run(Thread.java:748)
```
After comparing the error stack trace with similar issues reported  in different projects, such as
https://issues.apache.org/jira/browse/KAFKA-3453
https://issues.apache.org/jira/browse/HBASE-14734

We can be sure that they are caused by the same problem issued in HADOOP-12656.

In the PR, We apply the approach from HBASE first before we finally drop Hadoop 2.7.x

### Why are the changes needed?

fix test flakiness

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?

the test itself passing Jenkins

Closes #28442 from yaooqinn/SPARK-31631.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../HadoopDelegationTokenManagerSuite.scala   | 30 +++++++++++++++++--
 .../spark/sql/kafka010/KafkaTestUtils.scala   | 29 ++++++++++++++++--
 2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
index 275bca3459855..fc28968922c7a 100644
--- a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
@@ -19,10 +19,14 @@ package org.apache.spark.deploy.security
 
 import java.security.PrivilegedExceptionAction
 
+import scala.util.control.NonFatal
+
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION
 import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.{Credentials, UserGroupInformation}
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -88,8 +92,30 @@ class HadoopDelegationTokenManagerSuite extends SparkFunSuite {
       // krb5.conf. MiniKdc sets "java.security.krb5.conf" in start and removes it when stop called.
       val kdcDir = Utils.createTempDir()
       val kdcConf = MiniKdc.createConf()
-      kdc = new MiniKdc(kdcConf, kdcDir)
-      kdc.start()
+      // The port for MiniKdc service gets selected in the constructor, but will be bound
+      // to it later in MiniKdc.start() -> MiniKdc.initKDCServer() -> KdcServer.start().
+      // In meantime, when some other service might capture the port during this progress, and
+      // cause BindException.
+      // This makes our tests which have dedicated JVMs and rely on MiniKDC being flaky
+      //
+      // https://issues.apache.org/jira/browse/HADOOP-12656 get fixed in Hadoop 2.8.0.
+      //
+      // The workaround here is to periodically repeat this process with a timeout , since we are
+      // using Hadoop 2.7.4 as default.
+      // https://issues.apache.org/jira/browse/SPARK-31631
+      eventually(timeout(10.seconds), interval(1.second)) {
+        try {
+          kdc = new MiniKdc(kdcConf, kdcDir)
+          kdc.start()
+        } catch {
+          case NonFatal(e) =>
+            if (kdc != null) {
+              kdc.stop()
+              kdc = null
+            }
+            throw e
+        }
+      }
 
       val krbConf = new Configuration()
       krbConf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index c1ca557061ba1..5db8999a69509 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -27,6 +27,7 @@ import javax.security.auth.login.Configuration
 import scala.collection.JavaConverters._
 import scala.io.Source
 import scala.util.Random
+import scala.util.control.NonFatal
 
 import com.google.common.io.Files
 import kafka.api.Request
@@ -36,7 +37,7 @@ import kafka.zk.KafkaZkClient
 import org.apache.hadoop.minikdc.MiniKdc
 import org.apache.hadoop.security.UserGroupInformation
 import org.apache.kafka.clients.CommonClientConfigs
-import org.apache.kafka.clients.admin.{AdminClient, CreatePartitionsOptions, ListConsumerGroupsResult, NewPartitions, NewTopic}
+import org.apache.kafka.clients.admin._
 import org.apache.kafka.clients.consumer.KafkaConsumer
 import org.apache.kafka.clients.producer._
 import org.apache.kafka.common.TopicPartition
@@ -136,8 +137,30 @@ class KafkaTestUtils(
     val kdcDir = Utils.createTempDir()
     val kdcConf = MiniKdc.createConf()
     kdcConf.setProperty(MiniKdc.DEBUG, "true")
-    kdc = new MiniKdc(kdcConf, kdcDir)
-    kdc.start()
+    // The port for MiniKdc service gets selected in the constructor, but will be bound
+    // to it later in MiniKdc.start() -> MiniKdc.initKDCServer() -> KdcServer.start().
+    // In meantime, when some other service might capture the port during this progress, and
+    // cause BindException.
+    // This makes our tests which have dedicated JVMs and rely on MiniKDC being flaky
+    //
+    // https://issues.apache.org/jira/browse/HADOOP-12656 get fixed in Hadoop 2.8.0.
+    //
+    // The workaround here is to periodically repeat this process with a timeout , since we are
+    // using Hadoop 2.7.4 as default.
+    // https://issues.apache.org/jira/browse/SPARK-31631
+    eventually(timeout(10.seconds), interval(1.second)) {
+      try {
+        kdc = new MiniKdc(kdcConf, kdcDir)
+        kdc.start()
+      } catch {
+        case NonFatal(e) =>
+          if (kdc != null) {
+            kdc.stop()
+            kdc = null
+          }
+          throw e
+      }
+    }
     // TODO https://issues.apache.org/jira/browse/SPARK-30037
     // Need to build spark's own MiniKDC and customize krb5.conf like Kafka
     rewriteKrb5Conf()

From 36b1a79d8aabb1cfdbfbeca69001a6fa39076031 Mon Sep 17 00:00:00 2001
From: tianlzhang <tianlzhang@ebay.com>
Date: Thu, 7 May 2020 15:22:13 -0700
Subject: [PATCH 0647/1280] [SPARK-31646][SHUFFLE] Remove unused
 registeredConnections counter from ShuffleMetrics

### What changes were proposed in this pull request?
Remove unused `registeredConnections` counter from `ExternalBlockHandler#ShuffleMetrics`

This was added by SPARK-25642 at 3.0.0
- https://github.com/apache/spark/commit/8dd29fe36b781d115213b1d6a8446ad04e9239bb

### Why are the changes needed?
It's `registeredConnections` counter created in `TransportContext` that's really counting the numbers and it's misleading for people who want to add new metrics like `registeredConnections`.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Add UTs to ensure all expected metrics are registered for `ExternalShuffleService` and `YarnShuffleService`

Closes #28457 from manuzhang/spark-31611-pre.

Lead-authored-by: tianlzhang <tianlzhang@ebay.com>
Co-authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit dad61ed46521157c42144920b42b91bcba8295d3)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../network/shuffle/ExternalBlockHandler.java |  3 -
 .../ExternalShuffleServiceMetricsSuite.scala  | 63 +++++++++++++++++++
 .../yarn/YarnShuffleServiceMetricsSuite.scala |  2 +-
 .../yarn/YarnShuffleServiceSuite.scala        | 25 ++++++++
 4 files changed, 89 insertions(+), 4 deletions(-)
 create mode 100644 core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala

diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
index 8c05288fb4111..33865a21ea914 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockHandler.java
@@ -229,8 +229,6 @@ public class ShuffleMetrics implements MetricSet {
     private final Meter blockTransferRateBytes = new Meter();
     // Number of active connections to the shuffle service
     private Counter activeConnections = new Counter();
-    // Number of registered connections to the shuffle service
-    private Counter registeredConnections = new Counter();
     // Number of exceptions caught in connections to the shuffle service
     private Counter caughtExceptions = new Counter();
 
@@ -242,7 +240,6 @@ public ShuffleMetrics() {
       allMetrics.put("registeredExecutorsSize",
                      (Gauge<Integer>) () -> blockManager.getRegisteredExecutorsSize());
       allMetrics.put("numActiveConnections", activeConnections);
-      allMetrics.put("numRegisteredConnections", registeredConnections);
       allMetrics.put("numCaughtExceptions", caughtExceptions);
     }
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
new file mode 100644
index 0000000000000..4757c4aa1220e
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config.SHUFFLE_SERVICE_ENABLED
+import org.apache.spark.util.Utils
+
+class ExternalShuffleServiceMetricsSuite extends SparkFunSuite {
+
+  var sparkConf: SparkConf = _
+  var externalShuffleService: ExternalShuffleService = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    sparkConf = new SparkConf()
+    sparkConf.set(SHUFFLE_SERVICE_ENABLED, true)
+    sparkConf.set("spark.local.dir", System.getProperty("java.io.tmpdir"))
+    Utils.loadDefaultSparkProperties(sparkConf, null)
+    val securityManager = new SecurityManager(sparkConf)
+    externalShuffleService = new ExternalShuffleService(sparkConf, securityManager)
+    externalShuffleService.start()
+  }
+
+  override def afterAll(): Unit = {
+    externalShuffleService.stop()
+  }
+
+  test("SPARK-31646: metrics should be registered") {
+    val sourceRef = classOf[ExternalShuffleService].getDeclaredField("shuffleServiceSource")
+    sourceRef.setAccessible(true)
+    val source = sourceRef.get(externalShuffleService).asInstanceOf[ExternalShuffleServiceSource]
+    assert(source.metricRegistry.getMetrics.keySet().asScala ==
+      Set(
+        "blockTransferRateBytes",
+        "numActiveConnections",
+        "numCaughtExceptions",
+        "numRegisteredConnections",
+        "openBlockRequestLatencyMillis",
+        "registeredExecutorsSize",
+        "registerExecutorRequestLatencyMillis",
+        "shuffle-server.usedDirectMemory",
+        "shuffle-server.usedHeapMemory")
+    )
+  }
+}
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
index e260c0b60b87c..632c66d77b707 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceMetricsSuite.scala
@@ -39,7 +39,7 @@ class YarnShuffleServiceMetricsSuite extends SparkFunSuite with Matchers {
     val allMetrics = Set(
       "openBlockRequestLatencyMillis", "registerExecutorRequestLatencyMillis",
       "blockTransferRateBytes", "registeredExecutorsSize", "numActiveConnections",
-      "numRegisteredConnections", "numCaughtExceptions")
+      "numCaughtExceptions")
 
     metrics.getMetrics.keySet().asScala should be (allMetrics)
   }
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
index 381a93580f961..d46cfa85c8d7b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/network/yarn/YarnShuffleServiceSuite.scala
@@ -23,9 +23,13 @@ import java.nio.file.attribute.PosixFilePermission._
 import java.util.EnumSet
 
 import scala.annotation.tailrec
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
 
+import com.codahale.metrics.MetricSet
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.metrics2.impl.MetricsSystemImpl
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem
 import org.apache.hadoop.service.ServiceStateException
 import org.apache.hadoop.yarn.api.records.ApplicationId
 import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -381,4 +385,25 @@ class YarnShuffleServiceSuite extends SparkFunSuite with Matchers with BeforeAnd
     s1.secretsFile should be (null)
   }
 
+  test("SPARK-31646: metrics should be registered into Node Manager's metrics system") {
+    s1 = new YarnShuffleService
+    s1.init(yarnConfig)
+
+    val metricsSource = DefaultMetricsSystem.instance.asInstanceOf[MetricsSystemImpl]
+      .getSource("sparkShuffleService").asInstanceOf[YarnShuffleServiceMetrics]
+    val metricSetRef = classOf[YarnShuffleServiceMetrics].getDeclaredField("metricSet")
+    metricSetRef.setAccessible(true)
+    val metrics = metricSetRef.get(metricsSource).asInstanceOf[MetricSet].getMetrics
+
+    assert(metrics.keySet().asScala == Set(
+      "blockTransferRateBytes",
+      "numActiveConnections",
+      "numCaughtExceptions",
+      "numRegisteredConnections",
+      "openBlockRequestLatencyMillis",
+      "registeredExecutorsSize",
+      "registerExecutorRequestLatencyMillis"
+    ))
+  }
+
 }

From 57b55eacaaa37b2b5aae2620fb29e046ee736b08 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 7 May 2020 17:50:32 -0700
Subject: [PATCH 0648/1280] [SPARK-31646][FOLLOWUP][TESTS] Add clean up code
 and disable irrelevent conf

(cherry picked from commit 24fac1e0c70a783b4d240607639ff20d7dd24191)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/deploy/ExternalShuffleServiceMetricsSuite.scala | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
index 4757c4aa1220e..d681c13337e0d 100644
--- a/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/ExternalShuffleServiceMetricsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.deploy
 import scala.collection.JavaConverters._
 
 import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config.SHUFFLE_SERVICE_ENABLED
+import org.apache.spark.internal.config.{SHUFFLE_SERVICE_DB_ENABLED, SHUFFLE_SERVICE_ENABLED}
 import org.apache.spark.util.Utils
 
 class ExternalShuffleServiceMetricsSuite extends SparkFunSuite {
@@ -32,6 +32,7 @@ class ExternalShuffleServiceMetricsSuite extends SparkFunSuite {
     super.beforeAll()
     sparkConf = new SparkConf()
     sparkConf.set(SHUFFLE_SERVICE_ENABLED, true)
+    sparkConf.set(SHUFFLE_SERVICE_DB_ENABLED, false)
     sparkConf.set("spark.local.dir", System.getProperty("java.io.tmpdir"))
     Utils.loadDefaultSparkProperties(sparkConf, null)
     val securityManager = new SecurityManager(sparkConf)
@@ -40,7 +41,10 @@ class ExternalShuffleServiceMetricsSuite extends SparkFunSuite {
   }
 
   override def afterAll(): Unit = {
-    externalShuffleService.stop()
+    if (externalShuffleService != null) {
+      externalShuffleService.stop()
+    }
+    super.afterAll()
   }
 
   test("SPARK-31646: metrics should be registered") {

From ba4392217b461d20bfd10dbc00714dbb7268d71a Mon Sep 17 00:00:00 2001
From: manuzhang <owenzhang1990@gmail.com>
Date: Fri, 8 May 2020 10:24:13 -0700
Subject: [PATCH 0649/1280] [SPARK-31658][SQL] Fix SQL UI not showing write
 commands of AQE plan

Show write commands on SQL UI of an AQE plan

Currently the leaf node of an AQE plan is always a `AdaptiveSparkPlan` which is not true when it's a child of a write command. Hence, the node of the write command as well as its metrics are not shown on the SQL UI.

![image](https://user-images.githubusercontent.com/1191767/81288918-1893f580-9098-11ea-9771-e3d0820ba806.png)

![image](https://user-images.githubusercontent.com/1191767/81289008-3a8d7800-9098-11ea-93ec-516bbaf25d2d.png)

No

Add UT.

Closes #28474 from manuzhang/aqe-ui.

Lead-authored-by: manuzhang <owenzhang1990@gmail.com>
Co-authored-by: Xiao Li <gatorsmile@gmail.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 77c690a7252b22c9dd8f3cb7ac32f79fd6845cad)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      |  4 +--
 .../adaptive/AdaptiveQueryExecSuite.scala     | 35 +++++++++++++++++--
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index cd6936bb22bd6..90d1db9e4b743 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -526,8 +526,8 @@ case class AdaptiveSparkPlanExec(
     } else {
       context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate(
         executionId,
-        SQLExecution.getQueryExecution(executionId).toString,
-        SparkPlanInfo.fromSparkPlan(this)))
+        context.qe.toString,
+        SparkPlanInfo.fromSparkPlan(context.qe.executedPlan)))
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index f30d1e9ab9a5d..29b97557b9448 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -805,9 +805,11 @@ class AdaptiveQueryExecSuite
   test("SPARK-30953: InsertAdaptiveSparkPlan should apply AQE on child plan of write commands") {
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
-      val plan = sql("CREATE TABLE t1 AS SELECT 1 col").queryExecution.executedPlan
-      assert(plan.isInstanceOf[DataWritingCommandExec])
-      assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec])
+      withTable("t1") {
+        val plan = sql("CREATE TABLE t1 AS SELECT 1 col").queryExecution.executedPlan
+        assert(plan.isInstanceOf[DataWritingCommandExec])
+        assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec])
+      }
     }
   }
 
@@ -847,4 +849,31 @@ class AdaptiveQueryExecSuite
       }
     }
   }
+
+  test("SPARK-31658: SQL UI should show write commands") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
+      SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
+      withTable("t1") {
+        var checkDone = false
+        val listener = new SparkListener {
+          override def onOtherEvent(event: SparkListenerEvent): Unit = {
+            event match {
+              case SparkListenerSQLAdaptiveExecutionUpdate(_, _, planInfo) =>
+                assert(planInfo.nodeName == "Execute CreateDataSourceTableAsSelectCommand")
+                checkDone = true
+              case _ => // ignore other events
+            }
+          }
+        }
+        spark.sparkContext.addSparkListener(listener)
+        try {
+          sql("CREATE TABLE t1 AS SELECT 1 col").collect()
+          spark.sparkContext.listenerBus.waitUntilEmpty()
+          assert(checkDone)
+        } finally {
+          spark.sparkContext.removeSparkListener(listener)
+        }
+      }
+    }
+  }
 }

From eb278bad4cf1b5804584cdb4f6714c205149824e Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 10 May 2020 13:31:26 +0900
Subject: [PATCH 0650/1280] [SPARK-31662][SQL] Fix loading of dates before
 1582-10-15 from dictionary encoded Parquet columns

### What changes were proposed in this pull request?
Modified the `decodeDictionaryIds()` method `VectorizedColumnReader` to handle especially the `DateType` when passed parameter `rebaseDateTime` is true. In that case, decoded days are rebased from the hybrid calendar to Proleptic Gregorian calendar using `RebaseDateTime`.`rebaseJulianToGregorianDays()`.

### Why are the changes needed?
This fixes the bug of loading dates before the cutover day from dictionary encoded column in parquet files. The code below forces dictionary encoding:
```scala
spark.conf.set("spark.sql.legacy.parquet.rebaseDateTimeInWrite.enabled", true)
Seq.tabulate(8)(_ => "1001-01-01").toDF("dateS")
  .select($"dateS".cast("date").as("date")).repartition(1)
  .write
  .option("parquet.enable.dictionary", true)
  .parquet(path)
```
Load the dates back:
```scala
spark.read.parquet(path).show(false)
+----------+
|date      |
+----------+
|1001-01-07|
...
|1001-01-07|
+----------+
```
Expected values **must be 1000-01-01** but not 1001-01-07.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes:
```scala
spark.read.parquet(path).show(false)
+----------+
|date      |
+----------+
|1001-01-01|
...
|1001-01-01|
+----------+
```

### How was this patch tested?
Modified the test `SPARK-31159: rebasing dates in write` in `ParquetIOSuite` to checked reading dictionary encoded dates.

Closes #28479 from MaxGekk/fix-datetime-rebase-parquet-dict-enc.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit ce63bef1dac91f82cda5ebb47b38bd98eaf8164f)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../parquet/VectorizedColumnReader.java       | 37 +++++++++++---
 .../datasources/parquet/ParquetIOSuite.scala  | 49 +++++++++++--------
 2 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 9482244deca4e..03056f51be64f 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -152,6 +152,24 @@ private boolean next() throws IOException {
     return definitionLevelColumn.nextInt() == maxDefLevel;
   }
 
+  private boolean isLazyDecodingSupported(PrimitiveType.PrimitiveTypeName typeName) {
+    boolean isSupported = false;
+    switch (typeName) {
+      case INT32:
+        isSupported = originalType != OriginalType.DATE || !rebaseDateTime;
+        break;
+      case INT64:
+        isSupported = originalType != OriginalType.TIMESTAMP_MILLIS;
+        break;
+      case FLOAT:
+      case DOUBLE:
+      case BINARY:
+        isSupported = true;
+        break;
+    }
+    return isSupported;
+  }
+
   /**
    * Reads `total` values from this columnReader into column.
    */
@@ -181,13 +199,7 @@ void readBatch(int total, WritableColumnVector column) throws IOException {
 
         // TIMESTAMP_MILLIS encoded as INT64 can't be lazily decoded as we need to post process
         // the values to add microseconds precision.
-        if (column.hasDictionary() || (rowId == 0 &&
-            (typeName == PrimitiveType.PrimitiveTypeName.INT32 ||
-            (typeName == PrimitiveType.PrimitiveTypeName.INT64 &&
-              originalType != OriginalType.TIMESTAMP_MILLIS) ||
-            typeName == PrimitiveType.PrimitiveTypeName.FLOAT ||
-            typeName == PrimitiveType.PrimitiveTypeName.DOUBLE ||
-            typeName == PrimitiveType.PrimitiveTypeName.BINARY))) {
+        if (column.hasDictionary() || (rowId == 0 && isLazyDecodingSupported(typeName))) {
           // Column vector supports lazy decoding of dictionary values so just set the dictionary.
           // We can't do this if rowId != 0 AND the column doesn't have a dictionary (i.e. some
           // non-dictionary encoded values have already been added).
@@ -266,7 +278,8 @@ private void decodeDictionaryIds(
     switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) {
       case INT32:
         if (column.dataType() == DataTypes.IntegerType ||
-            DecimalType.is32BitDecimalType(column.dataType())) {
+            DecimalType.is32BitDecimalType(column.dataType()) ||
+            (column.dataType() == DataTypes.DateType && !rebaseDateTime)) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putInt(i, dictionary.decodeToInt(dictionaryIds.getDictId(i)));
@@ -284,6 +297,14 @@ private void decodeDictionaryIds(
               column.putShort(i, (short) dictionary.decodeToInt(dictionaryIds.getDictId(i)));
             }
           }
+        } else if (column.dataType() == DataTypes.DateType) {
+          for (int i = rowId; i < rowId + num; ++i) {
+            if (!column.isNullAt(i)) {
+              int julianDays = dictionary.decodeToInt(dictionaryIds.getDictId(i));
+              int gregorianDays = RebaseDateTime.rebaseJulianToGregorianDays(julianDays);
+              column.putInt(i, gregorianDays);
+            }
+          }
         } else {
           throw constructConvertNotSupportedException(descriptor, column);
         }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index af66aa092c37e..198b68bd86ecc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -978,29 +978,38 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   }
 
   test("SPARK-31159: rebasing dates in write") {
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
-        Seq("1001-01-01").toDF("dateS")
-          .select($"dateS".cast("date").as("date"))
-          .write
-          .parquet(path)
-      }
+    val N = 8
+    Seq(false, true).foreach { dictionaryEncoding =>
+      withTempPath { dir =>
+        val path = dir.getAbsolutePath
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+          Seq.tabulate(N)(_ => "1001-01-01").toDF("dateS")
+            .select($"dateS".cast("date").as("date"))
+            .repartition(1)
+            .write
+            .option("parquet.enable.dictionary", dictionaryEncoding)
+            .parquet(path)
+        }
 
-      Seq(false, true).foreach { vectorized =>
-        withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-          // The file metadata indicates if it needs rebase or not, so we can always get the correct
-          // result regardless of the "rebaseInRead" config.
-          Seq(true, false).foreach { rebase =>
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
-              checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-01")))
+        Seq(false, true).foreach { vectorized =>
+          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+            // The file metadata indicates if it needs rebase or not, so we can always get
+            // the correct result regardless of the "rebaseInRead" config.
+            Seq(true, false).foreach { rebase =>
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                checkAnswer(
+                  spark.read.parquet(path),
+                  Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
+              }
             }
-          }
 
-          // Force to not rebase to prove the written datetime values are rebased and we will get
-          // wrong result if we don't rebase while reading.
-          withSQLConf("spark.test.forceNoRebase" -> "true") {
-            checkAnswer(spark.read.parquet(path), Row(Date.valueOf("1001-01-07")))
+            // Force to not rebase to prove the written datetime values are rebased and we will get
+            // wrong result if we don't rebase while reading.
+            withSQLConf("spark.test.forceNoRebase" -> "true") {
+              checkAnswer(
+                spark.read.parquet(path),
+                Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07"))))
+            }
           }
         }
       }

From 6786500d49efc00a658a917cb6fbf9d87090a64b Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sun, 10 May 2020 12:57:25 -0500
Subject: [PATCH 0651/1280] [SPARK-31636][SQL][DOCS] Remove HTML syntax in SQL
 reference

### What changes were proposed in this pull request?
Remove the unneeded embedded inline HTML markup by using the basic markdown syntax.
Please see #28414

### Why are the changes needed?
Make the doc cleaner and easily editable by MD editors.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manually build and check

Closes #28451 from huaxingao/html_cleanup.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit a75dc80a76ef7666330be2cb1de87e89a4103d95)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                      |  20 +-
 docs/sql-ref-ansi-compliance.md               |  18 +-
 docs/sql-ref-datatypes.md                     |   4 +-
 docs/sql-ref-functions-builtin.md             |   2 +-
 docs/sql-ref-functions-udf-aggregate.md       | 101 +++----
 docs/sql-ref-functions-udf-hive.md            |  12 +-
 docs/sql-ref-functions-udf-scalar.md          |  28 +-
 docs/sql-ref-identifier.md                    |  37 +--
 docs/sql-ref-literals.md                      | 282 ++++++++----------
 docs/sql-ref-null-semantics.md                |  44 +--
 docs/sql-ref-syntax-aux-analyze-table.md      |  64 ++--
 docs/sql-ref-syntax-aux-cache-cache-table.md  |  98 +++---
 docs/sql-ref-syntax-aux-cache-clear-cache.md  |  16 +-
 docs/sql-ref-syntax-aux-cache-refresh.md      |  24 +-
 .../sql-ref-syntax-aux-cache-uncache-table.md |  31 +-
 docs/sql-ref-syntax-aux-conf-mgmt-reset.md    |  10 +-
 docs/sql-ref-syntax-aux-conf-mgmt-set.md      |  31 +-
 docs/sql-ref-syntax-aux-describe-database.md  |  21 +-
 docs/sql-ref-syntax-aux-describe-function.md  |  30 +-
 docs/sql-ref-syntax-aux-describe-query.md     |  44 ++-
 docs/sql-ref-syntax-aux-describe-table.md     |  62 ++--
 docs/sql-ref-syntax-aux-refresh-table.md      |  31 +-
 ...l-ref-syntax-aux-resource-mgmt-add-file.md |  21 +-
 ...ql-ref-syntax-aux-resource-mgmt-add-jar.md |  21 +-
 ...-ref-syntax-aux-resource-mgmt-list-file.md |  14 +-
 ...l-ref-syntax-aux-resource-mgmt-list-jar.md |  14 +-
 docs/sql-ref-syntax-aux-show-columns.md       |   2 +-
 docs/sql-ref-syntax-aux-show-create-table.md  |  27 +-
 docs/sql-ref-syntax-aux-show-databases.md     |  32 +-
 docs/sql-ref-syntax-aux-show-functions.md     |  60 ++--
 docs/sql-ref-syntax-aux-show-partitions.md    |  47 ++-
 docs/sql-ref-syntax-aux-show-table.md         |  60 ++--
 docs/sql-ref-syntax-aux-show-tables.md        |  41 ++-
 docs/sql-ref-syntax-aux-show-tblproperties.md |  51 ++--
 docs/sql-ref-syntax-aux-show-views.md         |  45 ++-
 docs/sql-ref-syntax-aux-show.md               |   4 +-
 docs/sql-ref-syntax-ddl-alter-database.md     |  17 +-
 docs/sql-ref-syntax-ddl-alter-table.md        | 256 +++++++---------
 docs/sql-ref-syntax-ddl-alter-view.md         | 124 ++++----
 docs/sql-ref-syntax-ddl-create-database.md    |  39 +--
 docs/sql-ref-syntax-ddl-create-function.md    |  85 +++---
 ...-ref-syntax-ddl-create-table-datasource.md | 100 +++----
 ...-ref-syntax-ddl-create-table-hiveformat.md |  99 +++---
 docs/sql-ref-syntax-ddl-create-table-like.md  |  73 ++---
 docs/sql-ref-syntax-ddl-create-table.md       |  10 +-
 docs/sql-ref-syntax-ddl-create-view.md        |  82 +++--
 docs/sql-ref-syntax-ddl-drop-database.md      |  42 ++-
 docs/sql-ref-syntax-ddl-drop-function.md      |  55 ++--
 docs/sql-ref-syntax-ddl-drop-table.md         |  45 ++-
 docs/sql-ref-syntax-ddl-drop-view.md          |  49 ++-
 docs/sql-ref-syntax-ddl-repair-table.md       |  25 +-
 docs/sql-ref-syntax-ddl-truncate-table.md     |  43 ++-
 docs/sql-ref-syntax-dml-insert-into.md        |  90 +++---
 ...tax-dml-insert-overwrite-directory-hive.md |  75 ++---
 ...f-syntax-dml-insert-overwrite-directory.md |  74 ++---
 ...l-ref-syntax-dml-insert-overwrite-table.md |  87 +++---
 docs/sql-ref-syntax-dml-insert.md             |   8 +-
 docs/sql-ref-syntax-dml-load.md               |  67 ++---
 docs/sql-ref-syntax-dml.md                    |   4 +-
 docs/sql-ref-syntax-qry-explain.md            |  58 ++--
 docs/sql-ref-syntax-qry-sampling.md           |  20 +-
 docs/sql-ref-syntax-qry-select-clusterby.md   |  33 +-
 docs/sql-ref-syntax-qry-select-cte.md         |  35 +--
 ...sql-ref-syntax-qry-select-distribute-by.md |  33 +-
 docs/sql-ref-syntax-qry-select-groupby.md     | 261 ++++++++--------
 docs/sql-ref-syntax-qry-select-having.md      |  54 ++--
 docs/sql-ref-syntax-qry-select-hints.md       |  56 ++--
 .../sql-ref-syntax-qry-select-inline-table.md |  35 +--
 docs/sql-ref-syntax-qry-select-join.md        | 185 +++++-------
 docs/sql-ref-syntax-qry-select-like.md        |  51 ++--
 docs/sql-ref-syntax-qry-select-limit.md       |  41 ++-
 docs/sql-ref-syntax-qry-select-orderby.md     |  82 +++--
 docs/sql-ref-syntax-qry-select-setops.md      |  26 +-
 docs/sql-ref-syntax-qry-select-sortby.md      |  84 +++---
 docs/sql-ref-syntax-qry-select-tvf.md         |  35 +--
 docs/sql-ref-syntax-qry-select-usedb.md       |  26 +-
 docs/sql-ref-syntax-qry-select-where.md       |  37 ++-
 docs/sql-ref-syntax-qry-select.md             | 209 +++++++------
 docs/sql-ref-syntax-qry-window.md             |  91 +++---
 docs/sql-ref-syntax-qry.md                    |  35 +--
 80 files changed, 1979 insertions(+), 2506 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index dfe4cfab2a2ab..57fc493dad2f2 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -156,22 +156,22 @@
                   url: sql-ref-syntax-qry-select-distribute-by.html
                 - text: LIMIT Clause 
                   url: sql-ref-syntax-qry-select-limit.html
+                - text: Common Table Expression
+                  url: sql-ref-syntax-qry-select-cte.html
+                - text: Inline Table
+                  url: sql-ref-syntax-qry-select-inline-table.html
                 - text: JOIN
                   url: sql-ref-syntax-qry-select-join.html
                 - text: Join Hints
                   url: sql-ref-syntax-qry-select-hints.html
+                - text: LIKE Predicate
+                  url: sql-ref-syntax-qry-select-like.html
                 - text: Set Operators
                   url: sql-ref-syntax-qry-select-setops.html
                 - text: TABLESAMPLE
                   url: sql-ref-syntax-qry-sampling.html
                 - text: Table-valued Function
                   url: sql-ref-syntax-qry-select-tvf.html
-                - text: Inline Table
-                  url: sql-ref-syntax-qry-select-inline-table.html
-                - text: Common Table Expression
-                  url: sql-ref-syntax-qry-select-cte.html
-                - text: LIKE Predicate
-                  url: sql-ref-syntax-qry-select-like.html
                 - text: Window Function
                   url: sql-ref-syntax-qry-window.html
             - text: EXPLAIN
@@ -213,20 +213,20 @@
               subitems:
                 - text: SHOW COLUMNS 
                   url: sql-ref-syntax-aux-show-columns.html
+                - text: SHOW CREATE TABLE
+                  url: sql-ref-syntax-aux-show-create-table.html
                 - text: SHOW DATABASES
                   url: sql-ref-syntax-aux-show-databases.html
                 - text: SHOW FUNCTIONS
                   url: sql-ref-syntax-aux-show-functions.html
+                - text: SHOW PARTITIONS
+                  url: sql-ref-syntax-aux-show-partitions.html
                 - text: SHOW TABLE
                   url: sql-ref-syntax-aux-show-table.html
                 - text: SHOW TABLES
                   url: sql-ref-syntax-aux-show-tables.html
                 - text: SHOW TBLPROPERTIES
                   url: sql-ref-syntax-aux-show-tblproperties.html
-                - text: SHOW PARTITIONS 
-                  url: sql-ref-syntax-aux-show-partitions.html
-                - text: SHOW CREATE TABLE
-                  url: sql-ref-syntax-aux-show-create-table.html
                 - text: SHOW VIEWS
                   url: sql-ref-syntax-aux-show-views.html
             - text: CONFIGURATION MANAGEMENT
diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 93fb10b24e99f..b62834ebe9067 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -41,10 +41,10 @@ This means that in case an operation causes overflows, the result is the same wi
 On the other hand, Spark SQL returns null for decimal overflows.
 When `spark.sql.ansi.enabled` is set to `true` and an overflow occurs in numeric and interval arithmetic operations, it throws an arithmetic exception at runtime.
 
-{% highlight sql %}
+```sql
 -- `spark.sql.ansi.enabled=true`
 SELECT 2147483647 + 1;
-  java.lang.ArithmeticException: integer overflow
+java.lang.ArithmeticException: integer overflow
 
 -- `spark.sql.ansi.enabled=false`
 SELECT 2147483647 + 1;
@@ -53,7 +53,7 @@ SELECT 2147483647 + 1;
 +----------------+
 |     -2147483648|
 +----------------+
-{% endhighlight %}
+```
 
 ### Type Conversion
 
@@ -64,15 +64,15 @@ On the other hand, `INSERT INTO` syntax throws an analysis exception when the AN
 Currently, the ANSI mode affects explicit casting and assignment casting only.
 In future releases, the behaviour of type coercion might change along with the other two type conversion rules.
 
-{% highlight sql %}
+```sql
 -- Examples of explicit casting
 
 -- `spark.sql.ansi.enabled=true`
 SELECT CAST('a' AS INT);
-  java.lang.NumberFormatException: invalid input syntax for type numeric: a
+java.lang.NumberFormatException: invalid input syntax for type numeric: a
 
 SELECT CAST(2147483648L AS INT);
-  java.lang.ArithmeticException: Casting 2147483648 to int causes overflow
+java.lang.ArithmeticException: Casting 2147483648 to int causes overflow
 
 -- `spark.sql.ansi.enabled=false` (This is a default behaviour)
 SELECT CAST('a' AS INT);
@@ -94,8 +94,8 @@ CREATE TABLE t (v INT);
 
 -- `spark.sql.storeAssignmentPolicy=ANSI`
 INSERT INTO t VALUES ('1');
-  org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
-  - Cannot safely cast 'v': StringType to IntegerType;
+org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
+- Cannot safely cast 'v': StringType to IntegerType;
 
 -- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
 INSERT INTO t VALUES ('1');
@@ -105,7 +105,7 @@ SELECT * FROM t;
 +---+
 |  1|
 +---+
-{% endhighlight %}
+```
 
 ### SQL Functions
 
diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index 3f6b6b590f843..f27f1a0ca967f 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -240,7 +240,7 @@ Specifically:
 
 #### Examples
 
-{% highlight sql %}
+```sql
 SELECT double('infinity') AS col;
 +--------+
 |     col|
@@ -313,4 +313,4 @@ SELECT COUNT(*), c2 FROM test GROUP BY c2;
 |        2|-Infinity|
 |        3| Infinity|
 +---------+---------+
-{% endhighlight %}
\ No newline at end of file
+```
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
index 1bca68e5f19df..cabb83e09fde9 100644
--- a/docs/sql-ref-functions-builtin.md
+++ b/docs/sql-ref-functions-builtin.md
@@ -70,7 +70,7 @@ license: |
 ### JSON Functions
 {% include_relative generated-json-funcs-table.html %}
 #### Examples
-{% include_relative generated-agg-funcs-examples.html %}
+{% include_relative generated-json-funcs-examples.html %}
         {% break %}
     {% endif %}
 {% endfor %}
diff --git a/docs/sql-ref-functions-udf-aggregate.md b/docs/sql-ref-functions-udf-aggregate.md
index 3fde94d6bc4bf..da3182149410b 100644
--- a/docs/sql-ref-functions-udf-aggregate.md
+++ b/docs/sql-ref-functions-udf-aggregate.md
@@ -27,46 +27,35 @@ User-Defined Aggregate Functions (UDAFs) are user-programmable routines that act
 
 A base class for user-defined aggregations, which can be used in Dataset operations to take all of the elements of a group and reduce them to a single value.
 
- * IN - The input type for the aggregation.
- * BUF - The type of the intermediate value of the reduction.
- * OUT - The type of the final output result.
+  ***IN*** - The input type for the aggregation.
+
+  ***BUF*** - The type of the intermediate value of the reduction.
+
+  ***OUT*** - The type of the final output result.
+
+* **bufferEncoder: Encoder[BUF]**
 
-<dl>
-  <dt><code><em>bufferEncoder: Encoder[BUF]</em></code></dt>
-  <dd>
     Specifies the Encoder for the intermediate value type.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>finish(reduction: BUF): OUT</em></code></dt>
-  <dd>
+
+* **finish(reduction: BUF): OUT**
+
     Transform the output of the reduction.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>merge(b1: BUF, b2: BUF): BUF</em></code></dt>
-  <dd>
+
+* **merge(b1: BUF, b2: BUF): BUF**
+
     Merge two intermediate values.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>outputEncoder: Encoder[OUT]</em></code></dt>
-  <dd>
+
+* **outputEncoder: Encoder[OUT]**
+
     Specifies the Encoder for the final output value type.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>reduce(b: BUF, a: IN): BUF</em></code></dt>
-  <dd>
-     Aggregate input value <code>a</code> into current intermediate value. For performance, the function may modify <code>b</code> and return it instead of constructing new object for <code>b</code>.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>zero: BUF</em></code></dt>
-  <dd>
+
+* **reduce(b: BUF, a: IN): BUF**
+
+     Aggregate input value `a` into current intermediate value. For performance, the function may modify `b` and return it instead of constructing new object for `b`.
+
+* **zero: BUF**
+
     The initial value of the intermediate result for this aggregation.
-  </dd>
-</dl>
 
 ### Examples
 
@@ -95,16 +84,16 @@ For example, a user-defined average for untyped DataFrames can look like:
   {% include_example untyped_custom_aggregation java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java%}
 </div>
 <div data-lang="SQL"  markdown="1">
-{% highlight sql %}
+```sql
 -- Compile and place UDAF MyAverage in a JAR file called `MyAverage.jar` in /tmp.
 CREATE FUNCTION myAverage AS 'MyAverage' USING JAR '/tmp/MyAverage.jar';
 
 SHOW USER FUNCTIONS;
--- +------------------+
--- |          function|
--- +------------------+
--- | default.myAverage|
--- +------------------+
++------------------+
+|          function|
++------------------+
+| default.myAverage|
++------------------+
 
 CREATE TEMPORARY VIEW employees
 USING org.apache.spark.sql.json
@@ -113,26 +102,26 @@ OPTIONS (
 );
 
 SELECT * FROM employees;
--- +-------+------+
--- |   name|salary|
--- +-------+------+
--- |Michael|  3000|
--- |   Andy|  4500|
--- | Justin|  3500|
--- |  Berta|  4000|
--- +-------+------+
++-------+------+
+|   name|salary|
++-------+------+
+|Michael|  3000|
+|   Andy|  4500|
+| Justin|  3500|
+|  Berta|  4000|
++-------+------+
 
 SELECT myAverage(salary) as average_salary FROM employees;
--- +--------------+
--- |average_salary|
--- +--------------+
--- |        3750.0|
--- +--------------+
-{% endhighlight %}
++--------------+
+|average_salary|
++--------------+
+|        3750.0|
++--------------+
+```
 </div>
 </div>
 
 ### Related Statements
 
- * [Scalar User Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
- * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
+* [Scalar User Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-functions-udf-hive.md b/docs/sql-ref-functions-udf-hive.md
index 7a7129de23836..819c446c411d2 100644
--- a/docs/sql-ref-functions-udf-hive.md
+++ b/docs/sql-ref-functions-udf-hive.md
@@ -28,7 +28,7 @@ Spark SQL supports integration of Hive UDFs, UDAFs and UDTFs. Similar to Spark U
 Hive has two UDF interfaces: [UDF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDF.java) and [GenericUDF](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java).
 An example below uses [GenericUDFAbs](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java) derived from `GenericUDF`.
 
-{% highlight sql %}
+```sql
 -- Register `GenericUDFAbs` and use it in Spark SQL.
 -- Note that, if you use your own programmed one, you need to add a JAR containing it
 -- into a classpath,
@@ -52,12 +52,12 @@ SELECT testUDF(value) FROM t;
 |           2.0|
 |           3.0|
 +--------------+
-{% endhighlight %}
+```
 
 
 An example below uses [GenericUDTFExplode](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFExplode.java) derived from [GenericUDTF](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java).
 
-{% highlight sql %}
+```sql
 -- Register `GenericUDTFExplode` and use it in Spark SQL
 CREATE TEMPORARY FUNCTION hiveUDTF
     AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode';
@@ -79,12 +79,12 @@ SELECT hiveUDTF(value) FROM t;
 |  3|
 |  4|
 +---+
-{% endhighlight %}
+```
 
 Hive has two UDAF interfaces: [UDAF](https://github.com/apache/hive/blob/master/udf/src/java/org/apache/hadoop/hive/ql/exec/UDAF.java) and [GenericUDAFResolver](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver.java).
 An example below uses [GenericUDAFSum](https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java) derived from `GenericUDAFResolver`.
 
-{% highlight sql %}
+```sql
 -- Register `GenericUDAFSum` and use it in Spark SQL
 CREATE TEMPORARY FUNCTION hiveUDAF
     AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum';
@@ -105,4 +105,4 @@ SELECT key, hiveUDAF(value) FROM t GROUP BY key;
 |  b|              3|
 |  a|              3|
 +---+---------------+
-{% endhighlight %}
+```
\ No newline at end of file
diff --git a/docs/sql-ref-functions-udf-scalar.md b/docs/sql-ref-functions-udf-scalar.md
index 2cb25f275cb59..97f5a89d3fb19 100644
--- a/docs/sql-ref-functions-udf-scalar.md
+++ b/docs/sql-ref-functions-udf-scalar.md
@@ -26,24 +26,18 @@ User-Defined Functions (UDFs) are user-programmable routines that act on one row
 ### UserDefinedFunction
 
 To define the properties of a user-defined function, the user can use some of the methods defined in this class.
-<dl>
-  <dt><code><em>asNonNullable(): UserDefinedFunction</em></code></dt>
-  <dd>
+
+* **asNonNullable(): UserDefinedFunction**
+
     Updates UserDefinedFunction to non-nullable.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>asNondeterministic(): UserDefinedFunction</em></code></dt>
-  <dd>
+
+* **asNondeterministic(): UserDefinedFunction**
+
     Updates UserDefinedFunction to nondeterministic.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>withName(name: String): UserDefinedFunction</em></code></dt>
-  <dd>
+
+* **withName(name: String): UserDefinedFunction**
+
     Updates UserDefinedFunction with a given name.
-  </dd>
-</dl>
 
 ### Examples
 
@@ -57,5 +51,5 @@ To define the properties of a user-defined function, the user can use some of th
 </div>
 
 ### Related Statements
- * [User Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
- * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
+* [User Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+* [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
diff --git a/docs/sql-ref-identifier.md b/docs/sql-ref-identifier.md
index 89cde21e6fdb6..f65d491cc2fc4 100644
--- a/docs/sql-ref-identifier.md
+++ b/docs/sql-ref-identifier.md
@@ -27,41 +27,34 @@ An identifier is a string used to identify a database object such as a table, vi
 
 #### Regular Identifier
 
-{% highlight sql %}
+```sql
 { letter | digit | '_' } [ , ... ]
-{% endhighlight %}
-Note: If `spark.sql.ansi.enabled` is set to true, ANSI SQL reserved keywords cannot be used as identifiers. For more details, please refer to [ANSI Compliance](sql-ref-ansi-compliance.html).
+```
+**Note:** If `spark.sql.ansi.enabled` is set to true, ANSI SQL reserved keywords cannot be used as identifiers. For more details, please refer to [ANSI Compliance](sql-ref-ansi-compliance.html).
 
 #### Delimited Identifier
 
-{% highlight sql %}
+```sql
 `c [ ... ]`
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>letter</em></code></dt>
-  <dd>
+* **letter**
+
     Any letter from A-Z or a-z.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>digit</em></code></dt>
-  <dd>
+
+* **digit**
+
     Any numeral from 0 to 9.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>c</em></code></dt>
-  <dd>
+
+* **c**
+
     Any character from the character set. Use <code>`</code> to escape special characters (e.g., <code>`</code>).
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- This CREATE TABLE fails with ParseException because of the illegal identifier name a.b
 CREATE TABLE test (a.b int);
 org.apache.spark.sql.catalyst.parser.ParseException:
@@ -77,4 +70,4 @@ no viable alternative at input 'CREATE TABLE test (`a`b`'(line 1, pos 23)
 
 -- This CREATE TABLE works
 CREATE TABLE test (`a``b` int);
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-literals.md b/docs/sql-ref-literals.md
index 0088f79cb7007..b83f7f0a97c24 100644
--- a/docs/sql-ref-literals.md
+++ b/docs/sql-ref-literals.md
@@ -35,22 +35,19 @@ A string literal is used to specify a character string value.
 
 #### Syntax
 
-{% highlight sql %}
-'c [ ... ]' | "c [ ... ]"
-{% endhighlight %}
+```sql
+'char [ ... ]' | "char [ ... ]"
+```
 
-#### <em>Parameters</em>
+#### Parameters
+
+* **char**
 
-<dl>
-  <dt><code><em>c</em></code></dt>
-  <dd>
-    One character from the character set. Use <code>\</code> to escape special characters (e.g., <code>'</code> or <code>\</code>).
-  </dd>
-</dl>
+    One character from the character set. Use `\` to escape special characters (e.g., `'` or `\`).
 
-#### <em>Examples</em>
+#### Examples
 
-{% highlight sql %}
+```sql
 SELECT 'Hello, World!' AS col;
 +-------------+
 |          col|
@@ -71,7 +68,7 @@ SELECT 'it\'s $10.' AS col;
 +---------+
 |It's $10.|
 +---------+
-{% endhighlight %}
+```
 
 ### Binary Literal
 
@@ -79,29 +76,26 @@ A binary literal is used to specify a byte sequence value.
 
 #### Syntax
 
-{% highlight sql %}
-X { 'c [ ... ]' | "c [ ... ]" }
-{% endhighlight %}
+```sql
+X { 'num [ ... ]' | "num [ ... ]" }
+```
+
+#### Parameters
 
-#### <em>Parameters</em>
+* **num**
 
-<dl>
-  <dt><code><em>c</em></code></dt>
-  <dd>
-    One character from the character set.
-  </dd>
-</dl>
+    Any hexadecimal number from 0 to F.
 
-#### <em>Examples</em>
+#### Examples
 
-{% highlight sql %}
+```sql
 SELECT X'123456' AS col;
 +----------+
 |       col|
 +----------+
 |[12 34 56]|
 +----------+
-{% endhighlight %}
+```
 
 ### Null Literal
 
@@ -109,20 +103,20 @@ A null literal is used to specify a null value.
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 NULL
-{% endhighlight %}
+```
 
 #### Examples
 
-{% highlight sql %}
+```sql
 SELECT NULL AS col;
 +----+
 | col|
 +----+
 |NULL|
 +----+
-{% endhighlight %}
+```
 
 ### Boolean Literal
 
@@ -130,20 +124,20 @@ A boolean literal is used to specify a boolean value.
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 TRUE | FALSE
-{% endhighlight %}
+```
 
 #### Examples
 
-{% highlight sql %}
+```sql
 SELECT TRUE AS col;
 +----+
 | col|
 +----+
 |true|
 +----+
-{% endhighlight %}
+```
 
 ### Numeric Literal
 
@@ -151,48 +145,37 @@ A numeric literal is used to specify a fixed or floating-point number.
 
 #### Integral Literal
 
-#### Syntax
+##### Syntax
 
-{% highlight sql %}
+```sql
 [ + | - ] digit [ ... ] [ L | S | Y ]
-{% endhighlight %}
+```
 
-#### Parameters
+##### Parameters
+
+* **digit**
 
-<dl>
-  <dt><code><em>digit</em></code></dt>
-  <dd>
     Any numeral from 0 to 9.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>L</em></code></dt>
-  <dd>
-    Case insensitive, indicates <code>BIGINT</code>, which is a 8-byte signed integer number.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>S</em></code></dt>
-  <dd>
-    Case insensitive, indicates <code>SMALLINT</code>, which is a 2-byte signed integer number.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>Y</em></code></dt>
-  <dd>
-    Case insensitive, indicates <code>TINYINT</code>, which is a 1-byte signed integer number.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>default (no postfix)</em></code></dt>
-  <dd>
+
+* **L**
+
+    Case insensitive, indicates `BIGINT`, which is an 8-byte signed integer number.
+
+* **S**
+
+    Case insensitive, indicates `SMALLINT`, which is a 2-byte signed integer number.
+
+* **Y**
+
+    Case insensitive, indicates `TINYINT`, which is a 1-byte signed integer number.
+
+* **default (no postfix)**
+
     Indicates a 4-byte signed integer number.
-  </dd>
-</dl>
 
-#### Examples
+##### Examples
 
-{% highlight sql %}
+```sql
 SELECT -2147483648 AS col;
 +-----------+
 |        col|
@@ -220,56 +203,49 @@ SELECT 482S AS col;
 +---+
 |482|
 +---+
-{% endhighlight %}
+```
 
 #### Fractional Literals
 
-#### Syntax
+##### Syntax
 
 decimal literals:
-{% highlight sql %}
+```sql
 decimal_digits { [ BD ] | [ exponent BD ] } | digit [ ... ] [ exponent ] BD
-{% endhighlight %}
+```
 
 double literals:
-{% highlight sql %}
+```sql
 decimal_digits  { D | exponent [ D ] }  | digit [ ... ] { exponent [ D ] | [ exponent ] D }
-{% endhighlight %}
+```
 
 While decimal_digits is defined as
-{% highlight sql %}
+```sql
 [ + | - ] { digit [ ... ] . [ digit [ ... ] ] | . digit [ ... ] }
-{% endhighlight %}
+```
 
 and exponent is defined as
-{% highlight sql %}
+```sql
 E [ + | - ] digit [ ... ]
-{% endhighlight %}
+```
 
-#### Parameters
+##### Parameters
+
+* **digit**
 
-<dl>
-  <dt><code><em>digit</em></code></dt>
-  <dd>
     Any numeral from 0 to 9.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>D</em></code></dt>
-  <dd>
-    Case insensitive, indicates <code>DOUBLE</code>, which is a 8-byte double-precision floating point number.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>BD</em></code></dt>
-  <dd>
-    Case insensitive, indicates <code>DECIMAL</code>, with the total number of digits as precision and the number of digits to right of decimal point as scale.
-  </dd>
-</dl>
 
-#### Examples
+* **D**
+
+    Case insensitive, indicates `DOUBLE`, which is an 8-byte double-precision floating point number.
+
+* **BD**
+
+    Case insensitive, indicates `DECIMAL`, with the total number of digits as precision and the number of digits to right of decimal point as scale.
 
-{% highlight sql %}
+##### Examples
+
+```sql
 SELECT 12.578 AS col;
 +------+
 |   col|
@@ -353,7 +329,7 @@ SELECT -3.E-3D AS col;
 +------+
 |-0.003|
 +------+
-{% endhighlight %}
+```
 
 ### Datetime Literal
 
@@ -361,19 +337,19 @@ A Datetime literal is used to specify a datetime value.
 
 #### Date Literal
 
-#### Syntax
+##### Syntax
 
-{% highlight sql %}
+```sql
 DATE { 'yyyy' |
        'yyyy-[m]m' |
        'yyyy-[m]m-[d]d' |
        'yyyy-[m]m-[d]d[T]' }
-{% endhighlight %}
-Note: defaults to <code>01</code> if month or day is not specified.
+```
+**Note:** defaults to `01` if month or day is not specified.
 
-#### Examples
+##### Examples
 
-{% highlight sql %}
+```sql
 SELECT DATE '1997' AS col;
 +----------+
 |       col|
@@ -394,13 +370,13 @@ SELECT DATE '2011-11-11' AS col;
 +----------+
 |2011-11-11|
 +----------+
-{% endhighlight %}
+```
 
 #### Timestamp Literal
 
-#### Syntax
+##### Syntax
 
-{% highlight sql %}
+```sql
 TIMESTAMP { 'yyyy' |
             'yyyy-[m]m' |
             'yyyy-[m]m-[d]d' |
@@ -409,27 +385,23 @@ TIMESTAMP { 'yyyy' |
             'yyyy-[m]m-[d]d[T][h]h:[m]m[:]' |
             'yyyy-[m]m-[d]d[T][h]h:[m]m:[s]s[.]' |
             'yyyy-[m]m-[d]d[T][h]h:[m]m:[s]s.[ms][ms][ms][us][us][us][zone_id]'}
-{% endhighlight %}
-Note: defaults to <code>00</code> if hour, minute or second is not specified. <br><br>
+```
+**Note:** defaults to `00` if hour, minute or second is not specified.
 `zone_id` should have one of the forms:
-<ul>
-  <li>Z - Zulu time zone UTC+0</li>
-  <li>+|-[h]h:[m]m</li>
-  <li>An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-, and a suffix in the formats:
-    <ul>
-      <li>+|-h[h]</li>
-      <li>+|-hh[:]mm</li>
-      <li>+|-hh:mm:ss</li>
-      <li>+|-hhmmss</li>
-    </ul>
-  </li>
-  <li>Region-based zone IDs in the form <code>area/city</code>, such as <code>Europe/Paris</code></li>
-</ul>
-Note: defaults to the session local timezone (set via <code>spark.sql.session.timeZone</code>) if <code>zone_id</code> is not specified.
+* Z - Zulu time zone UTC+0
+* `+|-[h]h:[m]m`
+* An id with one of the prefixes UTC+, UTC-, GMT+, GMT-, UT+ or UT-, and a suffix in the formats:
+  * `+|-h[h]`
+  * `+|-hh[:]mm`
+  * `+|-hh:mm:ss`
+  * `+|-hhmmss`
+* Region-based zone IDs in the form `area/city`, such as `Europe/Paris`
 
-#### Examples
+**Note:** defaults to the session local timezone (set via `spark.sql.session.timeZone`) if `zone_id` is not specified.
 
-{% highlight sql %}
+##### Examples
+
+```sql
 SELECT TIMESTAMP '1997-01-31 09:26:56.123' AS col;
 +-----------------------+
 |                    col|
@@ -450,50 +422,40 @@ SELECT TIMESTAMP '1997-01' AS col;
 +-------------------+
 |1997-01-01 00:00:00|
 +-------------------+
-{% endhighlight %}
+```
 
 ### Interval Literal
 
 An interval literal is used to specify a fixed period of time.
 
-#### Syntax
-{% highlight sql %}
-{ INTERVAL interval_value interval_unit [ interval_value interval_unit ... ] |
-  INTERVAL 'interval_value interval_unit [ interval_value interval_unit ... ]' |
-  INTERVAL interval_string_value interval_unit TO interval_unit }
-{% endhighlight %}
+```sql
+INTERVAL interval_value interval_unit [ interval_value interval_unit ... ] |
+INTERVAL 'interval_value interval_unit [ interval_value interval_unit ... ]' |
+INTERVAL interval_string_value interval_unit TO interval_unit
+```
 
 #### Parameters
 
-<dl>
-  <dt><code><em>interval_value</em></code></dt>
-  <dd>
-    <b>Syntax:</b>
-      <code>
-        [ + | - ] number_value | '[ + | - ] number_value'
-      </code><br>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>interval_string_value</em></code></dt>
-    <dd>
-      year-month/day-time interval string.
-    </dd>
-</dl>
-<dl>
-  <dt><code><em>interval_unit</em></code></dt>
-  <dd>
-    <b>Syntax:</b><br>
-      <code>
-        YEAR[S] | MONTH[S] | WEEK[S] | DAY[S] | HOUR[S] | MINUTE[S] | SECOND[S] | <br>
-        MILLISECOND[S] | MICROSECOND[S]
-      </code>
-  </dd>
-</dl>
+* **interval_value**
+
+    **Syntax:**
+
+      [ + | - ] number_value | '[ + | - ] number_value'
+
+* **interval_string_value**
+
+     year-month/day-time interval string.
+
+* **interval_unit**
+
+    **Syntax:**
+
+      YEAR[S] | MONTH[S] | WEEK[S] | DAY[S] | HOUR[S] | MINUTE[S] | SECOND[S] |
+      MILLISECOND[S] | MICROSECOND[S]
 
 #### Examples
 
-{% highlight sql %}
+```sql
 SELECT INTERVAL 3 YEAR AS col;
 +-------+
 |    col|
@@ -536,4 +498,4 @@ SELECT INTERVAL '20 15:40:32.99899999' DAY TO SECOND AS col;
 +---------------------------------------------+
 |20 days 15 hours 40 minutes 32.998999 seconds|
 +---------------------------------------------+
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index 56b5cde630eaf..fb5d2a312d0e1 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -79,7 +79,7 @@ one or both operands are `NULL`:
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Normal comparison operators return `NULL` when one of the operand is `NULL`.
 SELECT 5 > null AS expression_output;
 +-----------------+
@@ -111,7 +111,7 @@ SELECT NULL <=> NULL;
 +-----------------+
 |             true|
 +-----------------+
-{% endhighlight %}
+```
 
 ### Logical Operators <a name="logical-operators"></a>
 
@@ -134,7 +134,7 @@ The following tables illustrate the behavior of logical operators when one or bo
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Normal comparison operators return `NULL` when one of the operands is `NULL`.
 SELECT (true OR null) AS expression_output;
 +-----------------+
@@ -158,7 +158,7 @@ SELECT NOT(null) AS expression_output;
 +-----------------+
 |             null|
 +-----------------+
-{% endhighlight %}
+```
 
 ### Expressions <a name="expressions"></a>
 
@@ -177,7 +177,7 @@ expression are `NULL` and most of the expressions fall in this category.
 
 ##### Examples
 
-{% highlight sql %}
+```sql
 SELECT concat('John', null) AS expression_output;
 +-----------------+
 |expression_output|
@@ -198,7 +198,7 @@ SELECT to_date(null) AS expression_output;
 +-----------------+
 |             null|
 +-----------------+
-{% endhighlight %}
+```
 
 #### Expressions That Can Process Null Value Operands <a name="can-process-null"></a>
 
@@ -221,7 +221,7 @@ returns the first non `NULL` value in its list of operands. However, `coalesce`
 
 ##### Examples
 
-{% highlight sql %}
+```sql
 SELECT isnull(null) AS expression_output;
 +-----------------+
 |expression_output|
@@ -251,7 +251,7 @@ SELECT isnan(null) AS expression_output;
 +-----------------+
 |            false|
 +-----------------+
-{% endhighlight %}
+```
 
 #### Builtin Aggregate Expressions <a name="built-in-aggregate"></a>
 
@@ -271,7 +271,7 @@ the rules of how `NULL` values are handled by aggregate functions.
    
 #### Examples
 
-{% highlight sql %}
+```sql
 -- `count(*)` does not skip `NULL` values.
 SELECT count(*) FROM person;
 +--------+
@@ -312,7 +312,7 @@ SELECT max(age) FROM person where 1 = 0;
 +--------+
 |    null|
 +--------+
-{% endhighlight %}
+```
 
 ### Condition Expressions in WHERE, HAVING and JOIN Clauses <a name="condition-expressions"></a>
 
@@ -323,7 +323,7 @@ For all the three operators, a condition expression is a boolean expression and
 
 #### Examples
 
-{% highlight sql %}
+```sql
 -- Persons whose age is unknown (`NULL`) are filtered out from the result set.
 SELECT * FROM person WHERE age > 0;
 +--------+---+
@@ -391,7 +391,7 @@ SELECT * FROM person p1, person p2
 |   Marry|null|   Marry|null|
 |     Joe|  30|     Joe|  30|
 +--------+----+--------+----+
-{% endhighlight %}
+```
 
 ### Aggregate Operator (GROUP BY, DISTINCT) <a name="aggregate-operator"></a>
 
@@ -402,7 +402,7 @@ standard and with other enterprise database management systems.
 
 #### Examples
 
-{% highlight sql %}
+```sql
 -- `NULL` values are put in one bucket in `GROUP BY` processing.
 SELECT age, count(*) FROM person GROUP BY age;
 +----+--------+
@@ -424,7 +424,7 @@ SELECT DISTINCT age FROM person;
 |  30|
 |  18|
 +----+
-{% endhighlight %}
+```
 
 ### Sort Operator (ORDER BY Clause) <a name="order-by"></a>
 
@@ -434,7 +434,7 @@ the `NULL` values are placed at first.
 
 #### Examples
 
-{% highlight sql %}
+```sql
 -- `NULL` values are shown at first and other values
 -- are sorted in ascending way.
 SELECT age, name FROM person ORDER BY age;
@@ -479,7 +479,7 @@ SELECT age, name FROM person ORDER BY age DESC NULLS LAST;
 |null|   Marry|
 |null|  Albert|
 +----+--------+
-{% endhighlight %}
+```
 
 ### Set Operators (UNION, INTERSECT, EXCEPT) <a name="set-operators"></a>
 
@@ -489,7 +489,7 @@ equal unlike the regular `EqualTo`(`=`) operator.
 
 #### Examples
 
-{% highlight sql %}
+```sql
 CREATE VIEW unknown_age SELECT * FROM person WHERE age IS NULL;
 
 -- Only common rows between two legs of `INTERSECT` are in the 
@@ -537,7 +537,7 @@ SELECT name, age FROM person
 |    Mike|  18|
 |     Dan|  50|
 +--------+----+
-{% endhighlight %}
+```
 
 ### EXISTS/NOT EXISTS Subquery <a name="exists-not-exists"></a>
 
@@ -554,7 +554,7 @@ semijoins / anti-semijoins without special provisions for null awareness.
 
 #### Examples
 
-{% highlight sql %}
+```sql
 -- Even if subquery produces rows with `NULL` values, the `EXISTS` expression
 -- evaluates to `TRUE` as the subquery produces 1 row.
 SELECT * FROM person WHERE EXISTS (SELECT null);
@@ -591,7 +591,7 @@ SELECT * FROM person WHERE NOT EXISTS (SELECT 1 WHERE 1 = 0);
 |   Marry|null|
 |     Joe|  30|
 +--------+----+
-{% endhighlight %}
+```
 
 ### IN/NOT IN Subquery <a name="in-not-in"></a>
 
@@ -617,7 +617,7 @@ and because NOT UNKNOWN is again UNKNOWN.
  
 #### Examples
 
-{% highlight sql %}
+```sql
 -- The subquery has only `NULL` value in its result set. Therefore,
 -- the result of `IN` predicate is UNKNOWN.
 SELECT * FROM person WHERE age IN (SELECT null);
@@ -646,4 +646,4 @@ SELECT * FROM person
 |name|age|
 +----+---+
 +----+---+
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-syntax-aux-analyze-table.md b/docs/sql-ref-syntax-aux-analyze-table.md
index f6a6c5f4bc555..8f43d7388d7db 100644
--- a/docs/sql-ref-syntax-aux-analyze-table.md
+++ b/docs/sql-ref-syntax-aux-analyze-table.md
@@ -25,53 +25,39 @@ The `ANALYZE TABLE` statement collects statistics about the table to be used by
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 ANALYZE TABLE table_identifier [ partition_spec ]
     COMPUTE STATISTICS [ NOSCAN | FOR COLUMNS col [ , ... ] | FOR ALL COLUMNS ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. When specified, partition statistics is returned.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>[ NOSCAN | FOR COLUMNS col [ , ... ] | FOR ALL COLUMNS ]</em></code></dt>
-    <dd>
-      <ul>
-        <li> If no analyze option is specified, <code>ANALYZE TABLE</code> collects the table's number of rows and size in bytes. </li>
-        <li> <b>NOSCAN</b>
-          <br> Collect only the table's size in bytes ( which does not require scanning the entire table ). </li>
-        <li> <b>FOR COLUMNS col [ , ... ] <code> | </code> FOR ALL COLUMNS</b>
-          <br> Collect column statistics for each column specified, or alternatively for every column, as well as table statistics.
-        </li>
-      </ul>
-     </dd>
-</dl>
+    for partitions. When specified, partition statistics is returned.
+
+    **Syntax:** `PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )`
+
+* **[ NOSCAN `|` FOR COLUMNS col [ , ... ] `|` FOR ALL COLUMNS ]**
+
+     * If no analyze option is specified, `ANALYZE TABLE` collects the table's number of rows and size in bytes.
+     * **NOSCAN**
+
+       Collects only the table's size in bytes ( which does not require scanning the entire table ).
+     * **FOR COLUMNS col [ , ... ] `|` FOR ALL COLUMNS**
+
+       Collects column statistics for each column specified, or alternatively for every column, as well as table statistics.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE students (name STRING, student_id INT) PARTITIONED BY (student_id);
 INSERT INTO students PARTITION (student_id = 111111) VALUES ('Mark');
 INSERT INTO students PARTITION (student_id = 222222) VALUES ('John');
@@ -135,4 +121,4 @@ DESC EXTENDED students name;
 |   max_col_len|         4|
 |     histogram|      NULL|
 +--------------+----------+
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index 11f682cc10891..193e209d792b3 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -26,71 +26,57 @@ This reduces scanning of the original files in future queries.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CACHE [ LAZY ] TABLE table_identifier
     [ OPTIONS ( 'storageLevel' [ = ] value ) ] [ [ AS ] query ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>LAZY</em></code></dt>
-  <dd>Only cache the table when it is first used, instead of immediately.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the table or view name to be cached. The table or view name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>OPTIONS ( 'storageLevel' [ = ] value )</em></code></dt>
-  <dd>
-  <code>OPTIONS</code> clause with <code>storageLevel</code> key and value pair. A Warning is issued when a key other than <code>storageLevel</code> is used. The valid options for <code>storageLevel</code> are:
-    <ul>
-      <li><code>NONE</code></li>
-      <li><code>DISK_ONLY</code></li>
-      <li><code>DISK_ONLY_2</code></li>
-      <li><code>MEMORY_ONLY</code></li>
-      <li><code>MEMORY_ONLY_2</code></li>
-      <li><code>MEMORY_ONLY_SER</code></li>
-      <li><code>MEMORY_ONLY_SER_2</code></li>
-      <li><code>MEMORY_AND_DISK</code></li>
-      <li><code>MEMORY_AND_DISK_2</code></li>
-      <li><code>MEMORY_AND_DISK_SER</code></li>
-      <li><code>MEMORY_AND_DISK_SER_2</code></li>
-      <li><code>OFF_HEAP</code></li>
-    </ul>
-    An Exception is thrown when an invalid value is set for <code>storageLevel</code>. If <code>storageLevel</code> is not explicitly set using <code>OPTIONS</code> clause, the default <code>storageLevel</code> is set to <code>MEMORY_AND_DISK</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be cached. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+* **LAZY**
+
+    Only cache the table when it is first used, instead of immediately.
+
+* **table_identifier**
+
+    Specifies the table or view name to be cached. The table or view name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **OPTIONS ( 'storageLevel' [ = ] value )**
+
+    `OPTIONS` clause with `storageLevel` key and value pair. A Warning is issued when a key other than `storageLevel` is used. The valid options for `storageLevel` are:
+     * `NONE`
+     * `DISK_ONLY`
+     * `DISK_ONLY_2`
+     * `MEMORY_ONLY`
+     * `MEMORY_ONLY_2`
+     * `MEMORY_ONLY_SER`
+     * `MEMORY_ONLY_SER_2`
+     * `MEMORY_AND_DISK`
+     * `MEMORY_AND_DISK_2`
+     * `MEMORY_AND_DISK_SER`
+     * `MEMORY_AND_DISK_SER_2`
+     * `OFF_HEAP`
+
+    An Exception is thrown when an invalid value is set for `storageLevel`. If `storageLevel` is not explicitly set using `OPTIONS` clause, the default `storageLevel` is set to `MEMORY_AND_DISK`.
+
+* **query**
+
+    A query that produces the rows to be cached. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testData;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
- * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
- * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-clear-cache.md b/docs/sql-ref-syntax-aux-cache-clear-cache.md
index 47889691148b7..ee33e6a98296d 100644
--- a/docs/sql-ref-syntax-aux-cache-clear-cache.md
+++ b/docs/sql-ref-syntax-aux-cache-clear-cache.md
@@ -25,19 +25,19 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CLEAR CACHE
-{% endhighlight %}
+```
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CLEAR CACHE;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
- * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
- * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-refresh.md b/docs/sql-ref-syntax-aux-cache-refresh.md
index 25f7ede1d324e..82bc12da5d1ac 100644
--- a/docs/sql-ref-syntax-aux-cache-refresh.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh.md
@@ -27,32 +27,30 @@ invalidate everything that is cached.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 REFRESH resource_path
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
- <dt><code><em>resource_path</em></code></dt>
- <dd>The path of the resource that is to be refreshed.</dd>
-</dl>
+* **resource_path**
+
+    The path of the resource that is to be refreshed.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- The Path is resolved using the datasource's File Index.
- 
 CREATE TABLE test(ID INT) using parquet;
 INSERT INTO test SELECT 1000;
 CACHE TABLE test;
 INSERT INTO test SELECT 100;
 REFRESH "hdfs://path/to/table";
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
- * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
- * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index 95fd91c3c4807..c5a8fbbe08281 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -26,32 +26,27 @@ underlying entries should already have been brought to cache by previous `CACHE
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 UNCACHE TABLE [ IF EXISTS ] table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the table or view name to be uncached. The table or view name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+* **table_identifier**
+
+    Specifies the table or view name to be uncached. The table or view name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 UNCACHE TABLE t1;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
- * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
- * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
index e7e6dda4e25ee..4caf57a232f89 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-reset.md
@@ -25,17 +25,17 @@ Reset any runtime configurations specific to the current session which were set
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 RESET
-{% endhighlight %}
+```
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Reset any runtime configurations specific to the current session which were set via the SET command to their default values.
 RESET;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
+* [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt-set.md b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
index 330a1a6a399ff..f97b7f2a8efed 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt-set.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt-set.md
@@ -25,32 +25,29 @@ The SET command sets a property, returns the value of an existing property or re
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SET
 SET [ -v ]
 SET property_key[ = property_value ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>-v</em></code></dt>
-  <dd>Outputs the key, value and meaning of existing SQLConf properties.</dd>
-</dl>
+* **-v**
 
-<dl>
-  <dt><code><em>property_key</em></code></dt>
-  <dd>Returns the value of specified property key.</dd>
-</dl>
+    Outputs the key, value and meaning of existing SQLConf properties.
 
-<dl>
-  <dt><code><em>property_key=property_value</em></code></dt>
-  <dd>Sets the value for a given property key. If an old value exists for a given property key, then it gets overridden by the new value.</dd>
-</dl>
+* **property_key**
+
+    Returns the value of specified property key.
+
+* **property_key=property_value**
+
+     Sets the value for a given property key. If an old value exists for a given property key, then it gets overridden by the new value.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Set a property.
 SET spark.sql.variable.substitute=false;
 
@@ -67,8 +64,8 @@ SET spark.sql.variable.substitute;
 +-----------------------------+-----+
 |spark.sql.variable.substitute|false|
 +-----------------------------+-----+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+* [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref-syntax-aux-describe-database.md b/docs/sql-ref-syntax-aux-describe-database.md
index 39a40ddac800f..143fa78b205ca 100644
--- a/docs/sql-ref-syntax-aux-describe-database.md
+++ b/docs/sql-ref-syntax-aux-describe-database.md
@@ -28,23 +28,20 @@ interchangeable.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 { DESC | DESCRIBE } DATABASE [ EXTENDED ] db_name
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>db_name</em></code></dt>
-  <dd>
+* **db_name**
+
     Specifies a name of an existing database or an existing schema in the system. If the name does not exist, an
     exception is thrown.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create employees DATABASE
 CREATE DATABASE employees COMMENT 'For software companies';
 
@@ -89,10 +86,10 @@ DESC DATABASE deployment;
 |              Description|        Deployment environment|
 |                 Location|file:/Users/Temp/deployment.db|
 +-------------------------+------------------------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
- * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
- * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-describe-function.md b/docs/sql-ref-syntax-aux-describe-function.md
index 76c9efad2fa7d..a871fb5bfd406 100644
--- a/docs/sql-ref-syntax-aux-describe-function.md
+++ b/docs/sql-ref-syntax-aux-describe-function.md
@@ -28,29 +28,24 @@ metadata information is returned along with the extended usage information.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 { DESC | DESCRIBE } FUNCTION [ EXTENDED ] function_name
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
+* **function_name**
+
     Specifies a name of an existing function in the system. The function name may be
     optionally qualified with a database name. If `function_name` is qualified with
     a database then the function is resolved from the user specified database, otherwise
-    it is resolved from the current database.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] function_name
-      </code>
-  </dd>
-</dl>
+    it is resolved from the current database.
+
+    **Syntax:** `[ database_name. ] function_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Describe a builtin scalar function.
 -- Returns function name, implementing class and usage
 DESC FUNCTION abs;
@@ -107,11 +102,10 @@ DESC FUNCTION EXTENDED explode
 |       10                                                      |
 |       20                                                      |
 +---------------------------------------------------------------+
-
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
- * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
- * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
diff --git a/docs/sql-ref-syntax-aux-describe-query.md b/docs/sql-ref-syntax-aux-describe-query.md
index 41e66dcb2e316..b2a74cbd06078 100644
--- a/docs/sql-ref-syntax-aux-describe-query.md
+++ b/docs/sql-ref-syntax-aux-describe-query.md
@@ -27,38 +27,36 @@ describe the query output.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 { DESC | DESCRIBE } [ QUERY ] input_statement
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>QUERY</em></code></dt>
-  <dd>This clause is optional and may be omitted.</dd>
-  <dt><code><em>input_statement</em></code></dt>
-  <dd>
+* **QUERY**
+    This clause is optional and may be omitted.
+
+* **input_statement**
+
     Specifies a result set producing statement and may be one of the following: 
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>CTE(Common table expression)</code> statement</li>
-      <li>an <code>INLINE TABLE</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-    Please refer to <a href="sql-ref-syntax-qry-select.html">select-statement</a>
+
+    * a `SELECT` statement
+    * a `CTE(Common table expression)` statement
+    * an `INLINE TABLE` statement
+    * a `TABLE` statement
+    * a `FROM` statement`
+
+    Please refer to [select-statement](sql-ref-syntax-qry-select.html)
     for a detailed syntax of the query parameter.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create table `person`
 CREATE TABLE person (name STRING , age INT COMMENT 'Age column', address STRING);
 
 -- Returns column metadata information for a simple select query
-DESCRIBE QUERY select age, sum(age) FROM person GROUP BY age;
+DESCRIBE QUERY SELECT age, sum(age) FROM person GROUP BY age;
 +--------+---------+----------+
 |col_name|data_type|   comment|
 +--------+---------+----------+
@@ -103,10 +101,10 @@ DESCRIBE FROM person SELECT age;
 +--------+---------+----------+
 |     age|      int| Agecolumn|
 +--------+---------+----------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
- * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
- * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-describe-table.md b/docs/sql-ref-syntax-aux-describe-table.md
index 63bf056d785cc..4b6e1e8c3461e 100644
--- a/docs/sql-ref-syntax-aux-describe-table.md
+++ b/docs/sql-ref-syntax-aux-describe-table.md
@@ -28,53 +28,43 @@ to return the metadata pertaining to a partition or column respectively.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 { DESC | DESCRIBE } [ TABLE ] [ format ] table_identifier [ partition_spec ] [ col_name ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>format</em></code></dt>
-  <dd>
+* **format**
+
     Specifies the optional format of describe output. If `EXTENDED` is specified
     then additional metadata information (such as parent database, owner, and access time)
     is returned. 
-  </dd>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. When specified, additional partition metadata is returned.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>  
-  <dt><code><em>col_name</em></code></dt>
-  <dd>
+    for partitions. When specified, additional partition metadata is returned.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+* **col_name**
+
     An optional parameter that specifies the column name that needs to be described.
     The supplied column name may be optionally qualified. Parameters `partition_spec`
     and `col_name` are  mutually exclusive and can not be specified together. Currently
-    nested columns are not allowed to be specified.<br><br>
+    nested columns are not allowed to be specified.
     
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] [ table_name. ] column_name
-      </code>
-   </dd>
-</dl>
+    **Syntax:** `[ database_name. ] [ table_name. ] column_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Creates a table `customer`. Assumes current database is `salesdb`.
 CREATE TABLE customer(
         cust_id INT,
@@ -183,10 +173,10 @@ DESCRIBE customer salesdb.customer.name;
 |data_type|    string|
 |  comment|Short name|
 +---------+----------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
- * [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
- * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE QUERY](sql-ref-syntax-aux-describe-query.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-refresh-table.md
index 165ca68309f4a..8d4a804f88671 100644
--- a/docs/sql-ref-syntax-aux-refresh-table.md
+++ b/docs/sql-ref-syntax-aux-refresh-table.md
@@ -27,26 +27,21 @@ lazy manner when the cached table or the query associated with it is executed ag
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 REFRESH [TABLE] table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which is either a qualified or unqualified name that designates a table/view. If no database identifier is provided, it refers to a temporary view or a table/view in the current database.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which is either a qualified or unqualified name that designates a table/view. If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- The cached entries of the table will be refreshed  
 -- The table is resolved from the current database as the table name is unqualified.
 REFRESH TABLE tbl1;
@@ -54,11 +49,11 @@ REFRESH TABLE tbl1;
 -- The cached entries of the view will be refreshed or invalidated
 -- The view is resolved from tempDB database, as the view name is qualified.
 REFRESH TABLE tempDB.view1;   
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
- * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
- * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
+* [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
+* [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
+* [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
+* [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
index 0028884308890..9203293d0c981 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-file.md
@@ -25,30 +25,29 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 ADD FILE resource_name
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
- <dt><code><em>resource_name</em></code></dt>
- <dd>The name of the file or directory to be added.</dd>
-</dl>
+* **resource_name**
+
+    The name of the file or directory to be added.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 ADD FILE /tmp/test;
 ADD FILE "/path/to/file/abc.txt";
 ADD FILE '/another/test.txt';
 ADD FILE "/path with space/abc.txt";
 ADD FILE "/path/to/some/directory";
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
- * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
- * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
 
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
index c4020347c1be0..4694bff99daf5 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md
@@ -25,28 +25,27 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 ADD JAR file_name
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
- <dt><code><em>file_name</em></code></dt>
- <dd>The name of the JAR file to be added. It could be either on a local file system or a distributed file system.</dd>
-</dl>
+* **file_name**
+
+    The name of the JAR file to be added. It could be either on a local file system or a distributed file system.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 ADD JAR /tmp/test.jar;
 ADD JAR "/path/to/some.jar";
 ADD JAR '/some/other.jar';
 ADD JAR "/path with space/abc.jar";
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
- * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
- * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
index eec98e1fbffb5..9b9a7df7f612f 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-file.md
@@ -25,13 +25,13 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 LIST FILE
-{% endhighlight %}
+```
 
 ### Examples
 
-{% highlight sql %}
+```sql
 ADD FILE /tmp/test;
 ADD FILE /tmp/test_2;
 LIST FILE;
@@ -42,11 +42,11 @@ file:/private/tmp/test_2
 LIST FILE /tmp/test /some/random/file /another/random/file
 --output
 file:/private/tmp/test
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
- * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
- * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
 
diff --git a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
index dca4252c90ef2..04aa52c2ad8af 100644
--- a/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
+++ b/docs/sql-ref-syntax-aux-resource-mgmt-list-jar.md
@@ -25,13 +25,13 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 LIST JAR
-{% endhighlight %}
+```
 
 ### Examples
 
-{% highlight sql %}
+```sql
 ADD JAR /tmp/test.jar;
 ADD JAR /tmp/test_2.jar;
 LIST JAR;
@@ -42,11 +42,11 @@ spark://192.168.1.112:62859/jars/test_2.jar
 LIST JAR /tmp/test.jar /some/random.jar /another/random.jar;
 -- output
 spark://192.168.1.112:62859/jars/test.jar
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
- * [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
- * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
+* [ADD JAR](sql-ref-syntax-aux-resource-mgmt-add-jar.html)
+* [ADD FILE](sql-ref-syntax-aux-resource-mgmt-add-file.html)
+* [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
 
diff --git a/docs/sql-ref-syntax-aux-show-columns.md b/docs/sql-ref-syntax-aux-show-columns.md
index 7229bba23d2bf..b76db252f1a0f 100644
--- a/docs/sql-ref-syntax-aux-show-columns.md
+++ b/docs/sql-ref-syntax-aux-show-columns.md
@@ -21,7 +21,7 @@ license: |
 
 ### Description
 
-Return the list of columns in a table. If the table does not exist, an exception is thrown.
+Returns the list of columns in a table. If the table does not exist, an exception is thrown.
 
 ### Syntax
 
diff --git a/docs/sql-ref-syntax-aux-show-create-table.md b/docs/sql-ref-syntax-aux-show-create-table.md
index 47a5290f1d022..ae8c10e2d0178 100644
--- a/docs/sql-ref-syntax-aux-show-create-table.md
+++ b/docs/sql-ref-syntax-aux-show-create-table.md
@@ -25,26 +25,21 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SHOW CREATE TABLE table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table or view name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table or view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE test (c INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE
     TBLPROPERTIES ('prop1' = 'value1', 'prop2' = 'value2');
@@ -60,9 +55,9 @@ SHOW CREATE TABLE test;
    'prop1' = 'value1',
    'prop2' = 'value2')
 +----------------------------------------------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
diff --git a/docs/sql-ref-syntax-aux-show-databases.md b/docs/sql-ref-syntax-aux-show-databases.md
index c84898aa81459..44c0fbbef3929 100644
--- a/docs/sql-ref-syntax-aux-show-databases.md
+++ b/docs/sql-ref-syntax-aux-show-databases.md
@@ -21,35 +21,31 @@ license: |
 
 ### Description
 
-Lists the databases that match an optionally supplied string pattern. If no
+Lists the databases that match an optionally supplied regular expression pattern. If no
 pattern is supplied then the command lists all the databases in the system.
 Please note that the usage of `SCHEMAS` and `DATABASES` are interchangeable
 and mean the same thing.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SHOW { DATABASES | SCHEMAS } [ LIKE regex_pattern ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
+* **regex_pattern**
+
     Specifies a regular expression pattern that is used to filter the results of the
     statement.
-    <ul>
-      <li>Only <code>*</code> and <code>|</code> are allowed as wildcard pattern.</li>
-      <li>Excluding <code>*</code> and <code>|</code>, the remaining pattern follows the regular expression semantics.</li>
-      <li>The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
-    </ul>
-  </dd>
-</dl>
+    * Except for `*` and `|` character, the pattern works like a regular expression.
+    * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+    * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create database. Assumes a database named `default` already exists in
 -- the system. 
 CREATE DATABASE payroll_db;
@@ -83,10 +79,10 @@ SHOW SCHEMAS;
 | payments_db|
 |  payroll_db|
 +------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
- * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
- * [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [ALTER DATABASE](sql-ref-syntax-ddl-alter-database.html)
diff --git a/docs/sql-ref-syntax-aux-show-functions.md b/docs/sql-ref-syntax-aux-show-functions.md
index 8a6de402c7f20..b4dd72801202e 100644
--- a/docs/sql-ref-syntax-aux-show-functions.md
+++ b/docs/sql-ref-syntax-aux-show-functions.md
@@ -9,7 +9,7 @@ license: |
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
- 
+
      http://www.apache.org/licenses/LICENSE-2.0
  
   Unless required by applicable law or agreed to in writing, software
@@ -29,48 +29,42 @@ clause is optional and supported only for compatibility with other systems.
 
 ### Syntax
 
-{% highlight sql %}
-SHOW [ function_kind ] FUNCTIONS ( [ LIKE ] function_name | regex_pattern )
-{% endhighlight %}
+```sql
+SHOW [ function_kind ] FUNCTIONS [ [ LIKE ] { function_name | regex_pattern } ]
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>function_kind</em></code></dt>
-  <dd>
+* **function_kind**
+
     Specifies the name space of the function to be searched upon. The valid name spaces are :
-    <ul>
-      <li><b>USER</b> - Looks up the function(s) among the user defined functions.</li>
-      <li><b>SYSTEM</b> - Looks up the function(s) among the system defined functions.</li>
-      <li><b>ALL</b> -  Looks up the function(s) among both user and system defined functions.</li>
-    </ul>
-  </dd>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
+
+    * **USER** - Looks up the function(s) among the user defined functions.
+    * **SYSTEM** - Looks up the function(s) among the system defined functions.
+    * **ALL** -  Looks up the function(s) among both user and system defined functions.
+
+* **function_name**
+
     Specifies a name of an existing function in the system. The function name may be
     optionally qualified with a database name. If `function_name` is qualified with
     a database then the function is resolved from the user specified database, otherwise
-    it is resolved from the current database.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [database_name.]function_name
-      </code>
-  </dd>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
+    it is resolved from the current database.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+* **regex_pattern**
+
     Specifies a regular expression pattern that is used to filter the results of the
     statement.
-    <ul>
-      <li>Only <code>*</code> and <code>|</code> are allowed as wildcard pattern.</li>
-      <li>Excluding <code>*</code> and <code>|</code>, the remaining pattern follows the regular expression semantics.</li>
-      <li>The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
-    </ul>
-  </dd>
-</dl>
+
+    * Except for `*` and `|` character, the pattern works like a regular expression.
+    * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+    * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- List a system function `trim` by searching both user defined and system
 -- defined functions.
 SHOW FUNCTIONS trim;
@@ -138,8 +132,8 @@ SHOW FUNCTIONS LIKE 't[a-z][a-z][a-z]';
 |    tanh|
 |    trim|
 +--------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
diff --git a/docs/sql-ref-syntax-aux-show-partitions.md b/docs/sql-ref-syntax-aux-show-partitions.md
index 592833b23eb09..d93825550413f 100644
--- a/docs/sql-ref-syntax-aux-show-partitions.md
+++ b/docs/sql-ref-syntax-aux-show-partitions.md
@@ -27,37 +27,28 @@ partition spec.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SHOW PARTITIONS table_identifier [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. When specified, the partitions that match the partition spec are returned.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
+    for partitions. When specified, the partitions that match the partition specification are returned.
+
+    **Syntax:** `PARTITION ( partition_col_name = partition_col_val [ , ... ] )`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- create a partitioned table and insert a few rows.
 USE salesdb;
 CREATE TABLE customer(id INT, name STRING) PARTITIONED BY (state STRING, city STRING);
@@ -109,11 +100,11 @@ SHOW PARTITIONS customer PARTITION (city =  'San Jose');
 +----------------------+
 |state=CA/city=San Jose|
 +----------------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [INSERT STATEMENT](sql-ref-syntax-dml-insert.html)
- * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
- * [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [INSERT STATEMENT](sql-ref-syntax-dml-insert.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [SHOW TABLE](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index 3f588045790b2..0ce0a3eefa538 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -32,42 +32,36 @@ cannot be used with a partition specification.
 
 ### Syntax
 
-{% highlight sql %}
-SHOW TABLE EXTENDED [ IN | FROM database_name ] LIKE regex_pattern
+```sql
+SHOW TABLE EXTENDED [ { IN | FROM } database_name ] LIKE regex_pattern
     [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
- <dt><code><em>IN|FROM database_name</em></code></dt>
-  <dd>
+* **{ IN`|`FROM } database_name**
+
     Specifies database name. If not provided, will use the current database.
-  </dd>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
+
+* **regex_pattern**
+
     Specifies the regular expression pattern that is used to filter out unwanted tables.
-    <ul>
-       <li> Except for <code>*</code> and <code>|</code> character, the pattern works like a regular expression.</li>
-       <li> <code>*</code> alone matches 0 or more characters and <code>|</code> is used to separate multiple different regular expressions,
-             any of which can match. </li>
-       <li> The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
-    </ul>
-  </dd>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+
+    * Except for `*` and `|` character, the pattern works like a regular expression.
+    * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+      any of which can match.
+    * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions. Note that a table regex cannot be used with a partition specification.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
+    for partitions. Note that a table regex cannot be used with a partition specification.
+
+    **Syntax:** `PARTITION ( partition_col_name = partition_col_val [ , ... ] )`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Assumes `employee` table created with partitioned by column `grade`
 CREATE TABLE employee(name STRING, grade INT) PARTITIONED BY (grade);
 INSERT INTO employee PARTITION (grade = 1) VALUES ('sam');
@@ -152,7 +146,7 @@ SHOW TABLE EXTENDED  LIKE `employe*`;
 +--------+---------+----------+---------------------------------------------------------------+
   
 -- show partition file system details
-SHOW TABLE EXTENDED  IN `default` LIKE `employee` PARTITION (`grade=1`);
+SHOW TABLE EXTENDED  IN default LIKE `employee` PARTITION (`grade=1`);
 +--------+---------+-----------+--------------------------------------------------------------+
 |database|tableName|isTemporary|                         information                          |
 +--------+---------+-----------+--------------------------------------------------------------+
@@ -175,12 +169,12 @@ SHOW TABLE EXTENDED  IN `default` LIKE `employee` PARTITION (`grade=1`);
 +--------+---------+-----------+--------------------------------------------------------------+
 
 -- show partition file system details with regex fails as shown below
-SHOW TABLE EXTENDED  IN `default` LIKE `empl*` PARTITION (`grade=1`);
-  Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
-   Table or view 'emplo*' not found in database 'default'; (state=,code=0)
-{% endhighlight %}
+SHOW TABLE EXTENDED  IN default LIKE `empl*` PARTITION (`grade=1`);
+Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
+ Table or view 'emplo*' not found in database 'default'; (state=,code=0)
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [DESCRIBE TABLE](sql-ref-syntax-aux-describe-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-tables.md b/docs/sql-ref-syntax-aux-show-tables.md
index 62eb3ddb18b5c..fef9722a444f8 100644
--- a/docs/sql-ref-syntax-aux-show-tables.md
+++ b/docs/sql-ref-syntax-aux-show-tables.md
@@ -28,33 +28,28 @@ current database.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SHOW TABLES [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>{ FROM | IN } database_name</em></code></dt>
-  <dd>
+* **{ FROM `|` IN } database_name**
+
      Specifies the database name from which tables are listed.
-  </dd>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
+
+* **regex_pattern**
+
      Specifies the regular expression pattern that is used to filter out unwanted tables. 
-     <ul> 
-          <li> Except for <code>*</code> and <code>|</code> character, the pattern works like a regular expression.</li>
-          <li> <code>*</code> alone matches 0 or more characters and <code>|</code> is used to separate multiple different regular expressions,
-           any of which can match. </li>
-          <li> The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
-     </ul>
-    
-  </dd>
-</dl>
+
+     * Except for `*` and `|` character, the pattern works like a regular expression.
+     * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+     * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- List all tables in default database
 SHOW TABLES;
 +--------+---------+-----------+
@@ -101,11 +96,11 @@ SHOW TABLES LIKE 'sam*|suj';
 | default|     sam1|      false|
 | default|      suj|      false|
 +--------+---------+-----------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
- * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
- * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show-tblproperties.md b/docs/sql-ref-syntax-aux-show-tblproperties.md
index 662aaad069dd9..5b7ddcbcd9534 100644
--- a/docs/sql-ref-syntax-aux-show-tblproperties.md
+++ b/docs/sql-ref-syntax-aux-show-tblproperties.md
@@ -26,37 +26,30 @@ a property key. If no key is specified then all the properties are returned.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SHOW TBLPROPERTIES table_identifier 
    [ ( unquoted_property_key | property_key_as_string_literal ) ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
+* **table_identifier**
+
     Specifies the table name of an existing table. The table may be optionally qualified
-    with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-  <dt><code><em>unquoted_property_key</em></code></dt>
-  <dd>
+    with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **unquoted_property_key**
+
     Specifies the property key in unquoted form. The key may consists of multiple
-    parts separated by dot.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ key_part1 ] [ .key_part2 ] [ ... ]
-      </code>
-  </dd>   
-  <dt><code><em>property_key_as_string_literal</em></code></dt>
-  <dd>
+    parts separated by dot.
+
+    **Syntax:** `[ key_part1 ] [ .key_part2 ] [ ... ]`
+
+* **property_key_as_string_literal**
+
     Specifies a property key value as a string literal.
-  </dd>
-</dl>
 
 **Note**
 - Property value returned by this statement excludes some properties 
@@ -68,7 +61,7 @@ SHOW TBLPROPERTIES table_identifier
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- create a table `customer` in database `salesdb`
 USE salesdb;
 CREATE TABLE customer(cust_code INT, name VARCHAR(100), cust_addr STRING)
@@ -110,11 +103,11 @@ SHOW TBLPROPERTIES customer ('created.date');
 +----------+
 |01-01-2001|
 +----------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [ALTER TABLE SET TBLPROPERTIES](sql-ref-syntax-ddl-alter-table.html)
- * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
- * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [ALTER TABLE SET TBLPROPERTIES](sql-ref-syntax-ddl-alter-table.html)
+* [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
+* [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
diff --git a/docs/sql-ref-syntax-aux-show-views.md b/docs/sql-ref-syntax-aux-show-views.md
index 29ad6caf140f8..5003c092cabce 100644
--- a/docs/sql-ref-syntax-aux-show-views.md
+++ b/docs/sql-ref-syntax-aux-show-views.md
@@ -29,30 +29,26 @@ list global temporary views. Note that the command also lists local temporary vi
 regardless of a given database.
 
 ### Syntax
-{% highlight sql %}
+```sql
 SHOW VIEWS [ { FROM | IN } database_name ] [ LIKE regex_pattern ]
-{% endhighlight %}
+```
 
 ### Parameters
-<dl>
-  <dt><code><em>{ FROM | IN } database_name</em></code></dt>
-  <dd>
+* **{ FROM `|` IN } database_name**
+
      Specifies the database name from which views are listed.
-  </dd>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
+
+* **regex_pattern**
+
      Specifies the regular expression pattern that is used to filter out unwanted views.
-     <ul> 
-          <li>Except for <code>*</code> and <code>|</code> character, the pattern works like a regular expression.</li>
-          <li><code>*</code> alone matches 0 or more characters and <code>|</code> is used to separate multiple different regular expressions,
-           any of which can match. </li>
-          <li>The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.</li>
-     </ul>
-  </dd>
-</dl>
+
+     * Except for `*` and `|` character, the pattern works like a regular expression.
+     * `*` alone matches 0 or more characters and `|` is used to separate multiple different regular expressions,
+       any of which can match.
+     * The leading and trailing blanks are trimmed in the input pattern before processing. The pattern match is case-insensitive.
 
 ### Examples
-{% highlight sql %}
+```sql
 -- Create views in different databases, also create global/local temp views.
 CREATE VIEW sam AS SELECT id, salary FROM employee WHERE name = 'sam';
 CREATE VIEW sam1 AS SELECT id, salary FROM employee WHERE name = 'sam1';
@@ -61,8 +57,8 @@ USE userdb;
 CREATE VIEW user1 AS SELECT id, salary FROM default.employee WHERE name = 'user1';
 CREATE VIEW user2 AS SELECT id, salary FROM default.employee WHERE name = 'user2';
 USE default;
-CREATE GLOBAL TEMP VIEW temp1 AS SELECT 1 as col1;
-CREATE TEMP VIEW temp2 AS SELECT 1 as col1;
+CREATE GLOBAL TEMP VIEW temp1 AS SELECT 1 AS col1;
+CREATE TEMP VIEW temp2 AS SELECT 1 AS col1;
 
 -- List all views in default database
 SHOW VIEWS;
@@ -112,11 +108,10 @@ SHOW VIEWS LIKE 'sam|suj|temp*';
 | default     | suj        | false        |
 |             | temp2      | true         |
 +-------------+------------+--------------+
-
-{% endhighlight %}
+```
 
 ### Related statements
-- [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
-- [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
-- [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
-- [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+* [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-aux-show.md b/docs/sql-ref-syntax-aux-show.md
index 424fe71370897..9f64ea2d50ae1 100644
--- a/docs/sql-ref-syntax-aux-show.md
+++ b/docs/sql-ref-syntax-aux-show.md
@@ -20,11 +20,11 @@ license: |
 ---
 
  * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
+ * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
  * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
  * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+ * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
  * [SHOW TABLE EXTENDED](sql-ref-syntax-aux-show-table.html)
  * [SHOW TABLES](sql-ref-syntax-aux-show-tables.html)
  * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
- * [SHOW PARTITIONS](sql-ref-syntax-aux-show-partitions.html)
- * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
  * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-database.md b/docs/sql-ref-syntax-ddl-alter-database.md
index 2d5860c2ea920..fbc454e25fb0c 100644
--- a/docs/sql-ref-syntax-ddl-alter-database.md
+++ b/docs/sql-ref-syntax-ddl-alter-database.md
@@ -29,21 +29,20 @@ for a database and may be used for auditing purposes.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 ALTER { DATABASE | SCHEMA } database_name
     SET DBPROPERTIES ( property_name = property_value [ , ... ] )
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>database_name</em></code></dt>
-  <dd>Specifies the name of the database to be altered.</dd>
-</dl>
+* **database_name**
+
+    Specifies the name of the database to be altered.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Creates a database named `inventory`.
 CREATE DATABASE inventory;
 
@@ -60,8 +59,8 @@ DESCRIBE DATABASE EXTENDED inventory;
 |                 Location|   file:/temp/spark-warehouse/inventory.db|
 |               Properties|((Edit-date,01/01/2001), (Edited-by,John))|
 +-------------------------+------------------------------------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-table.md b/docs/sql-ref-syntax-ddl-alter-table.md
index f81585fef3aae..eb0e9a9f9cf73 100644
--- a/docs/sql-ref-syntax-ddl-alter-table.md
+++ b/docs/sql-ref-syntax-ddl-alter-table.md
@@ -29,35 +29,25 @@ license: |
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 ALTER TABLE table_identifier RENAME TO table_identifier
 
 ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
-{% endhighlight %}
+```
 
 #### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    Partition to be renamed. <br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Partition to be renamed.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
 ### ADD COLUMNS
 
@@ -65,66 +55,47 @@ ALTER TABLE table_identifier partition_spec RENAME TO partition_spec
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 ALTER TABLE table_identifier ADD COLUMNS ( col_spec [ , ... ] )
-{% endhighlight %}
+```
 
 #### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>COLUMNS ( col_spec )</em></code></dt>
-  <dd>Specifies the columns to be added.</dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **COLUMNS ( col_spec )**
+
+    Specifies the columns to be added.
 
 ### ALTER OR CHANGE COLUMN
 
-`ALTER TABLE ALTER COLUMN` or `ALTER TABLE CHANGE COLUMN` statement changes column's comment.
+`ALTER TABLE ALTER COLUMN` or `ALTER TABLE CHANGE COLUMN` statement changes column's definition.
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnAction
-{% endhighlight %}
+```
 
 #### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>COLUMN col_spec</em></code></dt>
-  <dd>Specifies the column to be altered or be changed.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>alterColumnAction</em></code></dt>
-   <dd>
-     Change the comment string.<br><br>
-     <b>Syntax:</b>
-        <code>
-            COMMENT STRING
-        </code>     
-   </dd>     
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **COLUMNS ( col_spec )**
+
+    Specifies the column to be altered or be changed.
+
+* **alterColumnAction**
+
+    Change column's definition.
 
 ### ADD AND DROP PARTITION
 
@@ -134,34 +105,24 @@ ALTER TABLE table_identifier { ALTER | CHANGE } [ COLUMN ] col_spec alterColumnA
 
 ##### Syntax
 
-{% highlight sql %}
+```sql
 ALTER TABLE table_identifier ADD [IF NOT EXISTS] 
     ( partition_spec [ partition_spec ... ] )
-{% endhighlight %}
+```
      
 ##### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    Partition to be added. <br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>     
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Partition to be added.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
 #### DROP PARTITION
 
@@ -169,33 +130,23 @@ ALTER TABLE table_identifier ADD [IF NOT EXISTS]
 
 ##### Syntax
 
-{% highlight sql %}
+```sql
 ALTER TABLE table_identifier DROP [ IF EXISTS ] partition_spec [PURGE]
-{% endhighlight %}
+```
      
 ##### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    Partition to be dropped. <br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>     
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Partition to be dropped.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
      
 ### SET AND UNSET
 
@@ -208,30 +159,28 @@ this overrides the old value with the new one.
 
 ##### Syntax
 
-{% highlight sql %}
+```sql
 -- Set Table Properties 
 ALTER TABLE table_identifier SET TBLPROPERTIES ( key1 = val1, key2 = val2, ... )
 
 -- Unset Table Properties
 ALTER TABLE table_identifier UNSET TBLPROPERTIES [ IF EXISTS ] ( key1, key2, ... )
-{% endhighlight %}
+```
 
 #### SET SERDE
 
-`ALTER TABLE SET` command is used for setting the SERDE or SERDE properties in Hive tables. If a particular property was already set,  
-this overrides the old value with the new one.
+`ALTER TABLE SET` command is used for setting the SERDE or SERDE properties in Hive tables. If a particular property was already set, this overrides the old value with the new one.
 
 ##### Syntax
 
-{% highlight sql %}
+```sql
 -- Set SERDE Properties
 ALTER TABLE table_identifier [ partition_spec ]
     SET SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )
 
 ALTER TABLE table_identifier [ partition_spec ] SET SERDE serde_class_name
     [ WITH SERDEPROPERTIES ( key1 = val1, key2 = val2, ... ) ]
-
-{% endhighlight %}
+```
 
 #### SET LOCATION And SET FILE FORMAT
 
@@ -240,46 +189,35 @@ existing tables.
 
 ##### Syntax
 
-{% highlight sql %}
+```sql
 -- Changing File Format
 ALTER TABLE table_identifier [ partition_spec ] SET FILEFORMAT file_format
 
 -- Changing File Location
 ALTER TABLE table_identifier [ partition_spec ] SET LOCATION 'new_location'
-{% endhighlight %}
+```
 
 #### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
-    Specifies the partition on which the property has to be set.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )</em></code></dt>
-  <dd>Specifies the SERDE properties to be set.</dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
+    Specifies the partition on which the property has to be set.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+* **SERDEPROPERTIES ( key1 = val1, key2 = val2, ... )**
+
+    Specifies the SERDE properties to be set.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- RENAME table 
 DESC student;
 +-----------------------+---------+-------+
@@ -477,13 +415,19 @@ ALTER TABLE test_tab SET SERDE 'org.apache.hadoop.hive.serde2.columnar.LazyBinar
 ALTER TABLE dbx.tab1 SET SERDE 'org.apache.hadoop' WITH SERDEPROPERTIES ('k' = 'v', 'kay' = 'vee')
 
 -- SET TABLE PROPERTIES
-ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('winner' = 'loser')
+ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('winner' = 'loser');
+
+-- SET TABLE COMMENT Using SET PROPERTIES
+ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('comment' = 'A table comment.');
+
+-- Alter TABLE COMMENT Using SET PROPERTIES
+ALTER TABLE dbx.tab1 SET TBLPROPERTIES ('comment' = 'This is a new comment.');
 
 -- DROP TABLE PROPERTIES
-ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner')
-{% endhighlight %}
+ALTER TABLE dbx.tab1 UNSET TBLPROPERTIES ('winner');
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
diff --git a/docs/sql-ref-syntax-ddl-alter-view.md b/docs/sql-ref-syntax-ddl-alter-view.md
index c2887692949ea..a34e77decf593 100644
--- a/docs/sql-ref-syntax-ddl-alter-view.md
+++ b/docs/sql-ref-syntax-ddl-alter-view.md
@@ -29,21 +29,16 @@ Renames the existing view. If the new view name already exists in the source dat
 does not support moving the views across databases.
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier RENAME TO view_identifier
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
 
 #### SET View Properties
 Set one or more properties of an existing view. The properties are the key value pairs. If the properties' keys exist, 
@@ -51,89 +46,70 @@ the values are replaced with the new values. If the properties' keys do not exis
 the properties.
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier SET TBLPROPERTIES ( property_key = property_val [ , ... ] )
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-  <dt><code><em>property_key</em></code></dt>
-  <dd>
-    Specifies the property key. The key may consists of multiple parts separated by dot.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ key_part1 ] [ .key_part2 ] [ ... ]
-      </code>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **property_key**
+
+    Specifies the property key. The key may consists of multiple parts separated by dot.
+
+    **Syntax:** `[ key_part1 ] [ .key_part2 ] [ ... ]`
 
 #### UNSET View Properties
 Drop one or more properties of an existing view. If the specified keys do not exist, an exception is thrown. Use 
 `IF EXISTS` to avoid the exception. 
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier UNSET TBLPROPERTIES [ IF EXISTS ]  ( property_key [ , ... ] )
-{% endhighlight %}
+```
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-  <dt><code><em>property_key</em></code></dt>
-  <dd>
-    Specifies the property key. The key may consists of multiple parts separated by dot.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ key_part1 ] [ .key_part2 ] [ ... ]
-      </code>
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **property_key**
+
+    Specifies the property key. The key may consists of multiple parts separated by dot.
+
+    **Syntax:** `[ key_part1 ] [ .key_part2 ] [ ... ]`
 
 #### ALTER View AS SELECT
 `ALTER VIEW view_identifier AS SELECT` statement changes the definition of a view. The `SELECT` statement must be valid,
 and the `view_identifier` must exist.
 
 #### Syntax
-{% highlight sql %}
+```sql
 ALTER VIEW view_identifier AS select_statement
-{% endhighlight %}
+```
 
 Note that `ALTER VIEW` statement does not support `SET SERDE` or `SET SERDEPROPERTIES` properties.
 
 #### Parameters
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-  <dt><code><em>select_statement</em></code></dt>
-  <dd>
-    Specifies the definition of the view. Check <a href="sql-ref-syntax-qry-select.html">select_statement</a> for details.
-  </dd>
-</dl>
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **select_statement**
+
+    Specifies the definition of the view. Check [select_statement](sql-ref-syntax-qry-select.html) for details.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Rename only changes the view name.
 -- The source and target databases of the view have to be the same.
 -- Use qualified or unqualified name for the source and target view.
@@ -218,11 +194,11 @@ DESC TABLE EXTENDED tempdb1.v2;
 |                   View Text|   select * from tempdb1.v1|       |
 |          View Original Text|   select * from tempdb1.v1|       |
 +----------------------------+---------------------------+-------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [describe-table](sql-ref-syntax-aux-describe-table.html)
- * [create-view](sql-ref-syntax-ddl-create-view.html)
- * [drop-view](sql-ref-syntax-ddl-drop-view.html)
- * [show-views](sql-ref-syntax-aux-show-views.html)
+* [describe-table](sql-ref-syntax-aux-describe-table.html)
+* [create-view](sql-ref-syntax-ddl-create-view.html)
+* [drop-view](sql-ref-syntax-ddl-drop-view.html)
+* [show-views](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-create-database.md b/docs/sql-ref-syntax-ddl-create-database.md
index 0ef0dfbdaed2b..9d8bf47844724 100644
--- a/docs/sql-ref-syntax-ddl-create-database.md
+++ b/docs/sql-ref-syntax-ddl-create-database.md
@@ -25,35 +25,38 @@ Creates a database with the specified name. If database with the same name alrea
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CREATE { DATABASE | SCHEMA } [ IF NOT EXISTS ] database_name
     [ COMMENT database_comment ]
     [ LOCATION database_directory ]
     [ WITH DBPROPERTIES ( property_name = property_value [ , ... ] ) ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-    <dt><code><em>database_name</em></code></dt>
-    <dd>Specifies the name of the database to be created.</dd>
+* **database_name**
 
-    <dt><code><em>IF NOT EXISTS</em></code></dt>
-    <dd>Creates a database with the given name if it doesn't exists. If a database with the same name already exists, nothing will happen.</dd>
+    Specifies the name of the database to be created.
 
-    <dt><code><em>database_directory</em></code></dt>
-    <dd>Path of the file system in which the specified database is to be created. If the specified path does not exist in the underlying file system, this command creates a directory with the path. If the location is not specified, the database will be created in the default warehouse directory, whose path is configured by the static configuration spark.sql.warehouse.dir.</dd>
+* **IF NOT EXISTS**
 
-    <dt><code><em>database_comment</em></code></dt>
-    <dd>Specifies the description for the database.</dd>
+    Creates a database with the given name if it does not exist. If a database with the same name already exists, nothing will happen.
 
-    <dt><code><em>WITH DBPROPERTIES ( property_name=property_value [ , ... ] )</em></code></dt>
-    <dd>Specifies the properties for the database in key-value pairs.</dd>
-</dl>
+* **database_directory**
+
+    Path of the file system in which the specified database is to be created. If the specified path does not exist in the underlying file system, this command creates a directory with the path. If the location is not specified, the database will be created in the default warehouse directory, whose path is configured by the static configuration spark.sql.warehouse.dir.
+
+* **database_comment**
+
+    Specifies the description for the database.
+
+* **WITH DBPROPERTIES ( property_name=property_value [ , ... ] )**
+
+    Specifies the properties for the database in key-value pairs.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create database `customer_db`. This throws exception if database with name customer_db
 -- already exists.
 CREATE DATABASE customer_db;
@@ -76,9 +79,9 @@ DESCRIBE DATABASE EXTENDED customer_db;
 |                 Location|     hdfs://hacluster/user|
 |               Properties|   ((ID,001), (Name,John))|
 +-------------------------+--------------------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
- * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index e3f21f70f7c18..aa6c1fad7b56b 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -33,68 +33,59 @@ aggregate functions using Scala, Python and Java APIs. Please refer to
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CREATE [ OR REPLACE ] [ TEMPORARY ] FUNCTION [ IF NOT EXISTS ]
     function_name AS class_name [ resource_locations ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>OR REPLACE</em></code></dt>
-  <dd>
+* **OR REPLACE**
+
     If specified, the resources for the function are reloaded. This is mainly useful
     to pick up any changes made to the implementation of the function. This
-    parameter is mutually exclusive to <code>IF NOT EXISTS</code> and can not
+    parameter is mutually exclusive to `IF NOT EXISTS` and can not
     be specified together.
-  </dd>
-  <dt><code><em>TEMPORARY</em></code></dt>
-  <dd>
-    Indicates the scope of function being created. When <code>TEMPORARY</code> is specified, the
+
+* **TEMPORARY**
+
+    Indicates the scope of function being created. When `TEMPORARY` is specified, the
     created function is valid and visible in the current session. No persistent
     entry is made in the catalog for these kind of functions.
-  </dd>
-  <dt><code><em>IF NOT EXISTS</em></code></dt>
-  <dd>
+
+* **IF NOT EXISTS**
+
     If specified, creates the function only when it does not exist. The creation
     of function succeeds (no error is thrown) if the specified function already
-    exists in the system. This parameter is mutually exclusive to <code> OR REPLACE</code> 
+    exists in the system. This parameter is mutually exclusive to `OR REPLACE`
     and can not be specified together.
-  </dd>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
-    Specifies a name of function to be created. The function name may be
-    optionally qualified with a database name. <br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] function_name
-      </code>
-  </dd>
-  <dt><code><em>class_name</em></code></dt>
-  <dd>
+
+* **function_name**
+
+    Specifies a name of function to be created. The function name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+* **class_name**
+
     Specifies the name of the class that provides the implementation for function to be created.
     The implementing class should extend one of the base classes as follows:
-    <ul>
-      <li>Should extend <code>UDF</code> or <code>UDAF</code> in <code>org.apache.hadoop.hive.ql.exec</code> package.</li>
-      <li>Should extend <code>AbstractGenericUDAFResolver</code>, <code>GenericUDF</code>, or
-          <code>GenericUDTF</code> in <code>org.apache.hadoop.hive.ql.udf.generic</code> package.</li>
-      <li>Should extend <code>UserDefinedAggregateFunction</code> in <code>org.apache.spark.sql.expressions</code> package.</li>
-    </ul>
-  </dd>
-  <dt><code><em>resource_locations</em></code></dt>
-  <dd>
+
+    * Should extend `UDF` or `UDAF` in `org.apache.hadoop.hive.ql.exec` package.
+    * Should extend `AbstractGenericUDAFResolver`, `GenericUDF`, or
+      `GenericUDTF` in `org.apache.hadoop.hive.ql.udf.generic` package.
+    * Should extend `UserDefinedAggregateFunction` in `org.apache.spark.sql.expressions` package.
+
+* **resource_locations**
+
     Specifies the list of resources that contain the implementation of the function
-    along with its dependencies. <br><br>
-    <b>Syntax:</b>
-      <code>
-        USING { { (JAR | FILE ) resource_uri } , ... }
-      </code>
-  </dd>
-</dl>
+    along with its dependencies.
+
+    **Syntax:** `USING { { (JAR | FILE ) resource_uri } , ... }`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- 1. Create a simple UDF `SimpleUdf` that increments the supplied integral value by 10.
 --    import org.apache.hadoop.hive.ql.exec.UDF;
 --    public class SimpleUdf extends UDF {
@@ -166,10 +157,10 @@ SELECT simple_udf(c1) AS function_return_value FROM t1;
 |                   21|
 |                   22|
 +---------------------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
- * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
- * [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
+* [SHOW FUNCTIONS](sql-ref-syntax-aux-show-functions.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [DROP FUNCTION](sql-ref-syntax-ddl-drop-function.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index 54827fd63568d..b592116c2a9e4 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -25,7 +25,7 @@ The `CREATE TABLE` statement defines a new table using a Data Source.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CREATE TABLE [ IF NOT EXISTS ] table_identifier
     [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
     [ USING data_source ]
@@ -38,62 +38,52 @@ CREATE TABLE [ IF NOT EXISTS ] table_identifier
     [ COMMENT table_comment ]
     [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
     [ AS select_statement ]
-{% endhighlight %}
+```
 
 Note that, the clauses between the USING clause and the AS SELECT clause can come in
 as any order. For example, you can write COMMENT table_comment after TBLPROPERTIES.
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>USING data_source</em></code></dt>
-  <dd>Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.</dd>
-</dl> 
-
-<dl>
-  <dt><code><em>PARTITIONED BY</em></code></dt>
-  <dd>Partitions are created on the table, based on the columns specified.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>CLUSTERED BY</em></code></dt>
-  <dd>
-	Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.<br><br>
-   	<b>NOTE:</b>Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.<br>
-	<dt><code><em>SORTED BY</em></code></dt>
-	<dd>Determines the order in which the data is stored in buckets. Default is Ascending order.</dd>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCATION</em></code></dt>
-  <dd>Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>COMMENT</em></code></dt>
-  <dd>A string literal to describe the table.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>A list of key-value pairs that is used to tag the table definition.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>AS select_statement</em></code></dt>
-  <dd>The table is populated using the data from the select statement.</dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **USING data_source**
+
+    Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.
+
+* **PARTITIONED BY**
+
+    Partitions are created on the table, based on the columns specified.
+
+* **CLUSTERED BY**
+
+    Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.
+
+    **NOTE:** Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.
+
+* **SORTED BY**
+
+    Determines the order in which the data is stored in buckets. Default is Ascending order.
+
+* **LOCATION**
+
+    Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.
+
+* **COMMENT**
+
+    A string literal to describe the table.
+
+* **TBLPROPERTIES**
+
+    A list of key-value pairs that is used to tag the table definition.
+
+* **AS select_statement**
+
+    The table is populated using the data from the select statement.
 
 ### Data Source Interaction
 
@@ -110,7 +100,7 @@ input query, to make sure the table gets created contains exactly the same data
 
 ### Examples
 
-{% highlight sql %}
+```sql
 
 --Use data source
 CREATE TABLE student (id INT, name STRING, age INT) USING CSV;
@@ -137,9 +127,9 @@ CREATE TABLE student (id INT, name STRING, age INT)
     USING CSV
     PARTITIONED BY (age)
     CLUSTERED BY (Id) INTO 4 buckets;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
- * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
+* [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
+* [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 06f353ad2f103..38f8856a24e3d 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -25,7 +25,7 @@ The `CREATE TABLE` statement defines a new table using Hive format.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
     [ ( col_name1[:] col_type1 [ COMMENT col_comment1 ], ... ) ]
     [ COMMENT table_comment ]
@@ -36,67 +36,54 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
     [ LOCATION path ]
     [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
     [ AS select_statement ]
-{% endhighlight %}
+```
 
 Note that, the clauses between the columns definition clause and the AS SELECT clause can come in
 as any order. For example, you can write COMMENT table_comment after TBLPROPERTIES.
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>EXTERNAL</em></code></dt>
-  <dd>Table is defined using the path provided as LOCATION, does not use default location for this table.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>PARTITIONED BY</em></code></dt>
-  <dd>Partitions are created on the table, based on the columns specified.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>ROW FORMAT</em></code></dt>
-  <dd>SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>STORED AS</em></code></dt>
-    <dd>File format for table storage, could be TEXTFILE, ORC, PARQUET,etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCATION</em></code></dt>
-  <dd>Path to the directory where table data is stored, Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>COMMENT</em></code></dt>
-  <dd>A string literal to describe the table.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>A list of key-value pairs that is used to tag the table definition.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>AS select_statement</em></code></dt>
-  <dd>The table is populated using the data from the select statement.</dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **EXTERNAL**
+
+    Table is defined using the path provided as LOCATION, does not use default location for this table.
+
+* **PARTITIONED BY**
+
+    Partitions are created on the table, based on the columns specified.
+
+* **ROW FORMAT**
+
+    SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.
+
+* **STORED AS**
+
+    File format for table storage, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **LOCATION**
+
+    Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc.
+
+* **COMMENT**
+
+    A string literal to describe the table.
+
+* **TBLPROPERTIES**
+
+    A list of key-value pairs that is used to tag the table definition.
+
+* **AS select_statement**
+
+    The table is populated using the data from the select statement.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 --Use hive format
 CREATE TABLE student (id INT, name STRING, age INT) STORED AS ORC;
 
@@ -130,9 +117,9 @@ CREATE TABLE student (id INT, name STRING)
 CREATE TABLE student (id INT,name STRING)
     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
- * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
+* [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+* [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
diff --git a/docs/sql-ref-syntax-ddl-create-table-like.md b/docs/sql-ref-syntax-ddl-create-table-like.md
index fe1dc4b1ef258..cfb959ca6b23d 100644
--- a/docs/sql-ref-syntax-ddl-create-table-like.md
+++ b/docs/sql-ref-syntax-ddl-create-table-like.md
@@ -25,57 +25,46 @@ The `CREATE TABLE` statement defines a new table using the definition/metadata o
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CREATE TABLE [IF NOT EXISTS] table_identifier LIKE source_table_identifier
     USING data_source
     [ ROW FORMAT row_format ]
     [ STORED AS file_format ]
     [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
     [ LOCATION path ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>  [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>USING data_source</em></code></dt>
-  <dd>Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.</dd>
-</dl> 
-
-<dl>
-  <dt><code><em>ROW FORMAT</em></code></dt>
-  <dd>SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>STORED AS</em></code></dt>
-  <dd>File format for table storage, could be TEXTFILE, ORC, PARQUET,etc.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>TBLPROPERTIES</em></code></dt>
-  <dd>Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCATION</em></code></dt>
-  <dd>Path to the directory where table data is stored,Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc. Location to create an external table.</dd>
-</dl>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **USING data_source**
+
+    Data Source is the input format used to create the table. Data source can be CSV, TXT, ORC, JDBC, PARQUET, etc.
+
+* **ROW FORMAT**
+
+    SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.
+
+* **STORED AS**
+
+    File format for table storage, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **TBLPROPERTIES**
+
+    Table properties that have to be set are specified, such as `created.by.user`, `owner`, etc.
+
+* **LOCATION**
+
+    Path to the directory where table data is stored, which could be a path on distributed storage like HDFS, etc. Location to create an external table.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create table using an existing table
 CREATE TABLE Student_Dupli like Student;
 
@@ -90,10 +79,10 @@ CREATE TABLE Student_Dupli like Student
     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE
     TBLPROPERTIES ('owner'='xxxx');
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
- * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
+* [CREATE TABLE USING DATASOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+* [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
 
diff --git a/docs/sql-ref-syntax-ddl-create-table.md b/docs/sql-ref-syntax-ddl-create-table.md
index b0388adbc9a38..85dc2020e6585 100644
--- a/docs/sql-ref-syntax-ddl-create-table.md
+++ b/docs/sql-ref-syntax-ddl-create-table.md
@@ -25,11 +25,11 @@ license: |
 
 The CREATE statements:
 
- * [CREATE TABLE USING DATA_SOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
- * [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
- * [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
+* [CREATE TABLE USING DATA_SOURCE](sql-ref-syntax-ddl-create-table-datasource.html)
+* [CREATE TABLE USING HIVE FORMAT](sql-ref-syntax-ddl-create-table-hiveformat.html)
+* [CREATE TABLE LIKE](sql-ref-syntax-ddl-create-table-like.html)
 
 ### Related Statements
 
- * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
- * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+* [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
diff --git a/docs/sql-ref-syntax-ddl-create-view.md b/docs/sql-ref-syntax-ddl-create-view.md
index ba8c1df1223a3..1a9c1f62728e7 100644
--- a/docs/sql-ref-syntax-ddl-create-view.md
+++ b/docs/sql-ref-syntax-ddl-create-view.md
@@ -27,55 +27,47 @@ a virtual table that has no physical data therefore other operations like
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_identifier
     create_view_clauses AS query
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>OR REPLACE</em></code></dt>
-  <dd>If a view of same name already exists, it will be replaced.</dd>
-</dl>
-<dl>
-  <dt><code><em>[ GLOBAL ] TEMPORARY</em></code></dt>
-  <dd>TEMPORARY views are session-scoped and will be dropped when session ends 
-      because it skips persisting the definition in the underlying metastore, if any.
-      GLOBAL TEMPORARY views are tied to a system preserved temporary database <code>global_temp</code>.</dd>
-</dl>
-<dl>
-  <dt><code><em>IF NOT EXISTS</em></code></dt>
-  <dd>Creates a view if it does not exists.</dd>
-</dl>
-<dl>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies a view name, which may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>create_view_clauses</em></code></dt>
-  <dd>These clauses are optional and order insensitive. It can be of following formats.
-    <ul>
-      <li><code>[ ( column_name [ COMMENT column_comment ], ... ) ]</code> to specify column-level comments.</li>
-      <li><code>[ COMMENT view_comment ]</code> to specify view-level comments.</li>
-      <li><code>[ TBLPROPERTIES ( property_name = property_value [ , ... ] ) ]</code> to add metadata key-value pairs.</li>
-    </ul>  
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A <a href="sql-ref-syntax-qry-select.html">SELECT</a> statement that constructs the view from base tables or other views.</dd>
-</dl>
+* **OR REPLACE**
+
+    If a view of same name already exists, it will be replaced.
+
+* **[ GLOBAL ] TEMPORARY**
+
+    TEMPORARY views are session-scoped and will be dropped when session ends
+    because it skips persisting the definition in the underlying metastore, if any.
+    GLOBAL TEMPORARY views are tied to a system preserved temporary database `global_temp`.
+
+* **IF NOT EXISTS**
+
+    Creates a view if it does not exist.
+
+* **view_identifier**
+
+    Specifies a view name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
+
+* **create_view_clauses**
+
+    These clauses are optional and order insensitive. It can be of following formats.
+
+    * `[ ( column_name [ COMMENT column_comment ], ... ) ]` to specify column-level comments.
+    * `[ COMMENT view_comment ]` to specify view-level comments.
+    * `[ TBLPROPERTIES ( property_name = property_value [ , ... ] ) ]` to add metadata key-value pairs.
+
+* **query**
+  A [SELECT](sql-ref-syntax-qry-select.html) statement that constructs the view from base tables or other views.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create or replace view for `experienced_employee` with comments.
 CREATE OR REPLACE VIEW experienced_employee
     (ID COMMENT 'Unique identification number', Name) 
@@ -88,10 +80,10 @@ CREATE GLOBAL TEMPORARY VIEW IF NOT EXISTS subscribed_movies
     AS SELECT mo.member_id, mb.full_name, mo.movie_title
         FROM movies AS mo INNER JOIN members AS mb 
         ON mo.member_id = mb.id;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
- * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
- * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+* [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+* [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
+* [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-database.md b/docs/sql-ref-syntax-ddl-drop-database.md
index 7467e7a4ad6e7..4a3bc0c68b6d4 100644
--- a/docs/sql-ref-syntax-ddl-drop-database.md
+++ b/docs/sql-ref-syntax-ddl-drop-database.md
@@ -26,35 +26,31 @@ exception will be thrown if the database does not exist in the system.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 DROP { DATABASE | SCHEMA } [ IF EXISTS ] dbname [ RESTRICT | CASCADE ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>DATABASE | SCHEMA</em></code></dt>
-  <dd><code>DATABASE</code> and <code>SCHEMA</code> mean the same thing, either of them can be used.</dd>
-</dl>
+* **DATABASE `|` SCHEMA**
 
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>If specified, no exception is thrown when the database does not exist.</dd>
-</dl>
+    `DATABASE` and `SCHEMA` mean the same thing, either of them can be used.
 
-<dl>
-  <dt><code><em>RESTRICT</em></code></dt>
-  <dd>If specified, will restrict dropping a non-empty database and is enabled by default.</dd>
-</dl>
+* **IF EXISTS**
 
-<dl>
-  <dt><code><em>CASCADE</em></code></dt>
-  <dd>If specified, will drop all the associated tables and functions.</dd>
-</dl>
+    If specified, no exception is thrown when the database does not exist.
+
+* **RESTRICT**
+
+    If specified, will restrict dropping a non-empty database and is enabled by default.
+
+* **CASCADE**
+
+    If specified, will drop all the associated tables and functions.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create `inventory_db` Database
 CREATE DATABASE inventory_db COMMENT 'This database is used to maintain Inventory';
 
@@ -63,10 +59,10 @@ DROP DATABASE inventory_db CASCADE;
 
 -- Drop the database using IF EXISTS
 DROP DATABASE IF EXISTS inventory_db CASCADE;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
- * [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
- * [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DESCRIBE DATABASE](sql-ref-syntax-aux-describe-database.html)
+* [SHOW DATABASES](sql-ref-syntax-aux-show-databases.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-function.md b/docs/sql-ref-syntax-ddl-drop-function.md
index 66a405c24e413..bef31d74afcff 100644
--- a/docs/sql-ref-syntax-ddl-drop-function.md
+++ b/docs/sql-ref-syntax-ddl-drop-function.md
@@ -26,39 +26,32 @@ be thrown if the function does not exist.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 DROP [ TEMPORARY ] FUNCTION [ IF EXISTS ] function_name
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>function_name</em></code></dt>
-  <dd>
+* **function_name**
+
     Specifies the name of an existing function. The function name may be
-    optionally qualified with a database name. <br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] function_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>TEMPORARY</em></code></dt>
-  <dd>Should be used to delete the <code>TEMPORARY</code> function.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>If specified, no exception is thrown when the function does not exist.</dd>
-</dl>
+    optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] function_name`
+
+* **TEMPORARY**
+
+    Should be used to delete the `TEMPORARY` function.
+
+* **IF EXISTS**
+
+    If specified, no exception is thrown when the function does not exist.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create a permanent function `test_avg`
-CREATE FUNCTION test_avg as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
+CREATE FUNCTION test_avg AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage';
 
 -- List user functions
 SHOW USER FUNCTIONS;
@@ -86,9 +79,9 @@ DROP FUNCTION test_avg;
 
 -- Try to drop Permanent function which is not present
 DROP FUNCTION test_avg;
-  Error: Error running query:
-  org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException:
-  Function 'default.test_avg' not found in database 'default'; (state=,code=0)
+Error: Error running query:
+org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException:
+Function 'default.test_avg' not found in database 'default'; (state=,code=0)
 
 -- List the functions after dropping, it should list only temporary function
 SHOW USER FUNCTIONS;
@@ -100,10 +93,10 @@ SHOW USER FUNCTIONS;
   
 -- Drop Temporary function
 DROP TEMPORARY FUNCTION IF EXISTS test_avg;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
- * [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
- * [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
+* [CREATE FUNCTION](sql-ref-syntax-ddl-create-function.html)
+* [DESCRIBE FUNCTION](sql-ref-syntax-aux-describe-function.html)
+* [SHOW FUNCTION](sql-ref-syntax-aux-show-functions.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-table.md b/docs/sql-ref-syntax-ddl-drop-table.md
index c943b922ae812..a15a9928f437d 100644
--- a/docs/sql-ref-syntax-ddl-drop-table.md
+++ b/docs/sql-ref-syntax-ddl-drop-table.md
@@ -28,49 +28,44 @@ In case of an external table, only the associated metadata information is remove
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 DROP TABLE [ IF EXISTS ] table_identifier
-{% endhighlight %}
+```
 
 ### Parameter
 
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>
-     If specified, no exception is thrown when the table does not exists.
-  </dd>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the table name to be dropped. The table name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+* **IF EXISTS**
+
+    If specified, no exception is thrown when the table does not exist.
+
+* **table_identifier**
+
+    Specifies the table name to be dropped. The table name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Assumes a table named `employeetable` exists.
 DROP TABLE employeetable;
 
 -- Assumes a table named `employeetable` exists in the `userdb` database
 DROP TABLE userdb.employeetable;
 
--- Assumes a table named `employeetable` does not exists.
+-- Assumes a table named `employeetable` does not exist.
 -- Throws exception
 DROP TABLE employeetable;
-  Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeetable;
-  (state=,code=0)
+Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeetable;
+(state=,code=0)
 
--- Assumes a table named `employeetable` does not exists,Try with IF EXISTS
+-- Assumes a table named `employeetable` does not exist,Try with IF EXISTS
 -- this time it will not throw exception
 DROP TABLE IF EXISTS employeetable;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
- * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
- * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [CREATE TABLE](sql-ref-syntax-ddl-create-table.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-drop-view.md b/docs/sql-ref-syntax-ddl-drop-view.md
index ad018b5e6fd5c..5b680d7f907e0 100644
--- a/docs/sql-ref-syntax-ddl-drop-view.md
+++ b/docs/sql-ref-syntax-ddl-drop-view.md
@@ -25,51 +25,46 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 DROP VIEW [ IF EXISTS ] view_identifier
-{% endhighlight %}
+```
 
 ### Parameter
 
-<dl>
-  <dt><code><em>IF EXISTS</em></code></dt>
-  <dd>
-     If specified, no exception is thrown when the view does not exists.
-  </dd>
-  <dt><code><em>view_identifier</em></code></dt>
-  <dd>
-    Specifies the view name to be dropped. The view name may be optionally qualified with a database name.<br><br>
-    <b> Syntax:</b>
-      <code>
-        [ database_name. ] view_name
-      </code>
-  </dd>
-</dl>
+* **IF EXISTS**
+
+    If specified, no exception is thrown when the view does not exist.
+
+* **view_identifier**
+
+    Specifies the view name to be dropped. The view name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] view_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Assumes a view named `employeeView` exists.
 DROP VIEW employeeView;
 
 -- Assumes a view named `employeeView` exists in the `userdb` database
 DROP VIEW userdb.employeeView;
 
--- Assumes a view named `employeeView` does not exists.
+-- Assumes a view named `employeeView` does not exist.
 -- Throws exception
 DROP VIEW employeeView;
-  Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeeView;
-  (state=,code=0)
+Error: org.apache.spark.sql.AnalysisException: Table or view not found: employeeView;
+(state=,code=0)
 
--- Assumes a view named `employeeView` does not exists,Try with IF EXISTS
+-- Assumes a view named `employeeView` does not exist,Try with IF EXISTS
 -- this time it will not throw exception
 DROP VIEW IF EXISTS employeeView;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
- * [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
- * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
- * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
- * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [CREATE VIEW](sql-ref-syntax-ddl-create-view.html)
+* [ALTER VIEW](sql-ref-syntax-ddl-alter-view.html)
+* [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
diff --git a/docs/sql-ref-syntax-ddl-repair-table.md b/docs/sql-ref-syntax-ddl-repair-table.md
index c48b731512ad3..c2ef0a7b7fbe9 100644
--- a/docs/sql-ref-syntax-ddl-repair-table.md
+++ b/docs/sql-ref-syntax-ddl-repair-table.md
@@ -25,26 +25,21 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 MSCK REPAIR TABLE table_identifier
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies the name of the table to be repaired. The table name may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
+* **table_identifier**
+
+    Specifies the name of the table to be repaired. The table name may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- create a partitioned table from existing data /tmp/namesAndAges.parquet
 CREATE TABLE t1 (name STRING, age INT) USING parquet PARTITIONED BY (age)
     LOCATION "/tmp/namesAndAges.parquet";
@@ -66,8 +61,8 @@ SELECT * FROM t1;
 +-------+---+
 |   Andy| 30|
 +-------+---+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+* [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
diff --git a/docs/sql-ref-syntax-ddl-truncate-table.md b/docs/sql-ref-syntax-ddl-truncate-table.md
index 820f439f97a4b..6139814a3259a 100644
--- a/docs/sql-ref-syntax-ddl-truncate-table.md
+++ b/docs/sql-ref-syntax-ddl-truncate-table.md
@@ -27,37 +27,28 @@ in `partition_spec`. If no `partition_spec` is specified it will remove all part
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 TRUNCATE TABLE table_identifier [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
+    for partitions.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Create table Student with partition
 CREATE TABLE Student (name STRING, rollno INT) PARTITIONED BY (age INT);
 
@@ -89,9 +80,9 @@ SELECT * FROM Student;
 |name|rollno|age|
 +----+------+---+
 +----+------+---+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
- * [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
+* [DROP TABLE](sql-ref-syntax-ddl-drop-table.html)
+* [ALTER TABLE](sql-ref-syntax-ddl-alter-table.html)
diff --git a/docs/sql-ref-syntax-dml-insert-into.md b/docs/sql-ref-syntax-dml-insert-into.md
index 924831f7feedd..ed5da2b2d28df 100644
--- a/docs/sql-ref-syntax-dml-insert-into.md
+++ b/docs/sql-ref-syntax-dml-insert-into.md
@@ -25,57 +25,43 @@ The `INSERT INTO` statement inserts new rows into a table. The inserted rows can
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 INSERT INTO [ TABLE ] table_identifier [ partition_spec ]
     { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name  = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+    for partitions.
+
+    **Syntax:** `PARTITION ( partition_col_name  = partition_col_val [ , ... ] )`
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
 
 #### Single Row Insert Using a VALUES Clause
 
-{% highlight sql %}
+```sql
 CREATE TABLE students (name VARCHAR(64), address VARCHAR(64), student_id INT)
     USING PARQUET PARTITIONED BY (student_id);
 
@@ -88,11 +74,11 @@ SELECT * FROM students;
 +---------+---------------------+----------+
 |Amy Smith|123 Park Ave,San Jose|    111111|
 +---------+---------------------+----------+
-{% endhighlight %}
+```
 
 #### Multi-Row Insert Using a VALUES Clause
 
-{% highlight sql %}
+```sql
 INSERT INTO students VALUES
     ('Bob Brown', '456 Taylor St, Cupertino', 222222),
     ('Cathy Johnson', '789 Race Ave, Palo Alto', 333333);
@@ -107,11 +93,11 @@ SELECT * FROM students;
 +-------------+------------------------+----------+
 |Cathy Johnson| 789 Race Ave, Palo Alto|    333333|
 +--------------+-----------------------+----------+
-{% endhighlight %}
+```
 
 #### Insert Using a SELECT Statement
 
-{% highlight sql %}
+```sql
 -- Assuming the persons table has already been created and populated.
 SELECT * FROM persons;
 +-------------+-------------------------+---------+
@@ -137,11 +123,11 @@ SELECT * FROM students;
 +-------------+-------------------------+----------+
 |Dora Williams|134 Forest Ave, Melo Park|    444444|
 +-------------+-------------------------+----------+
-{% endhighlight %}
+```
 
 #### Insert Using a TABLE Statement
 
-{% highlight sql %}
+```sql
 -- Assuming the visiting_students table has already been created and populated.
 SELECT * FROM visiting_students;
 +-------------+---------------------+----------+
@@ -170,11 +156,11 @@ SELECT * FROM students;
 +-------------+-------------------------+----------+
 |Gordon Martin|     779 Lake Ave, Oxford|    888888|
 +-------------+-------------------------+----------+
-{% endhighlight %}
+```
 
 #### Insert Using a FROM Statement
 
-{% highlight sql %}
+```sql
 -- Assuming the applicants table has already been created and populated.
 SELECT * FROM applicants;
 +-----------+--------------------------+----------+---------+
@@ -210,10 +196,10 @@ SELECT * FROM students;
 +-------------+-------------------------+----------+
 |   Jason Wang|    908 Bird St, Saratoga|    121212|
 +-------------+-------------------------+----------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
- * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
- * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
index 3cd2107668fbe..8ed6a3cd1be09 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory-hive.md
@@ -26,56 +26,41 @@ Hive support must be enabled to use this command. The inserted rows can be speci
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 INSERT OVERWRITE [ LOCAL ] DIRECTORY directory_path
     [ ROW FORMAT row_format ] [ STORED AS file_format ]
     { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>directory_path</em></code></dt>
-  <dd>
-  Specifies the destination directory. The <code>LOCAL</code> keyword is used to specify that the directory is on the local file system.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>row_format</em></code></dt>
-  <dd>
-  Specifies the row format for this insert. Valid options are <code>SERDE</code> clause and <code>DELIMITED</code> clause. <code>SERDE</code> clause can be used to specify a custom <code>SerDe</code> for this insert. Alternatively, <code>DELIMITED</code> clause can be used to specify the native <code>SerDe</code> and state the delimiter, escape character, null character, and so on.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>file_format</em></code></dt>
-  <dd>
-  Specifies the file format for this insert. Valid options are <code>TEXTFILE</code>, <code>SEQUENCEFILE</code>, <code>RCFILE</code>, <code>ORC</code>, <code>PARQUET</code>, and <code>AVRO</code>. You can also specify your own input and output format using <code>INPUTFORMAT</code> and <code>OUTPUTFORMAT</code>. <code>ROW FORMAT SERDE</code> can only be used with <code>TEXTFILE</code>, <code>SEQUENCEFILE</code>, or <code>RCFILE</code>, while <code>ROW FORMAT DELIMITED</code> can only be used with <code>TEXTFILE</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>
-  Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+* **directory_path**
+
+    Specifies the destination directory. The `LOCAL` keyword is used to specify that the directory is on the local file system.
+
+* **row_format**
+
+    Specifies the row format for this insert. Valid options are `SERDE` clause and `DELIMITED` clause. `SERDE` clause can be used to specify a custom `SerDe` for this insert. Alternatively, `DELIMITED` clause can be used to specify the native `SerDe` and state the delimiter, escape character, null character, and so on.
+
+* **file_format**
+
+    Specifies the file format for this insert. Valid options are `TEXTFILE`, `SEQUENCEFILE`, `RCFILE`, `ORC`, `PARQUET`, and `AVRO`. You can also specify your own input and output format using `INPUTFORMAT` and `OUTPUTFORMAT`. `ROW FORMAT SERDE` can only be used with `TEXTFILE`, `SEQUENCEFILE`, or `RCFILE`, while `ROW FORMAT DELIMITED` can only be used with `TEXTFILE`.
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
 
-{% highlight sql %}
+```sql
 INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
     STORED AS orc
     SELECT * FROM test_table;
@@ -83,10 +68,10 @@ INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
 INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     SELECT * FROM test_table;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
- * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
- * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
index 6ce7f50588e32..fd7437d37c909 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-directory.md
@@ -25,54 +25,42 @@ The `INSERT OVERWRITE DIRECTORY` statement overwrites the existing data in the d
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 INSERT OVERWRITE [ LOCAL ] DIRECTORY [ directory_path ]
     USING file_format [ OPTIONS ( key = val [ , ... ] ) ]
     { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>directory_path</em></code></dt>
-  <dd>
-  Specifies the destination directory. It can also be specified in <code>OPTIONS</code> using <code>path</code>. The <code>LOCAL</code> keyword is used to specify that the directory is on the local file system.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>file_format</em></code></dt>
-  <dd>
-  Specifies the file format to use for the insert. Valid options are <code>TEXT</code>, <code>CSV</code>, <code>JSON</code>, <code>JDBC</code>, <code>PARQUET</code>, <code>ORC</code>, <code>HIVE</code>, <code>LIBSVM</code>, or a fully qualified class name of a custom implementation of <code>org.apache.spark.sql.execution.datasources.FileFormat</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>OPTIONS ( key = val [ , ... ] )</em></code></dt>
-  <dd>Specifies one or more options for the writing of the file format.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>
-  Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+* **directory_path**
+
+    Specifies the destination directory. It can also be specified in `OPTIONS` using `path`.
+    The `LOCAL` keyword is used to specify that the directory is on the local file system.
+
+* **file_format**
+
+    Specifies the file format to use for the insert. Valid options are `TEXT`, `CSV`, `JSON`, `JDBC`, `PARQUET`, `ORC`, `HIVE`, `LIBSVM`, or a fully qualified class name of a custom implementation of `org.apache.spark.sql.execution.datasources.FileFormat`.
+
+* **OPTIONS ( key = val [ , ... ] )**
+
+    Specifies one or more options for the writing of the file format.
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
 
-{% highlight sql %}
+```sql
 INSERT OVERWRITE DIRECTORY '/tmp/destination'
     USING parquet
     OPTIONS (col1 1, col2 2, col3 'test')
@@ -82,10 +70,10 @@ INSERT OVERWRITE DIRECTORY
     USING parquet
     OPTIONS ('path' '/tmp/destination', col1 1, col2 2, col3 'test')
     SELECT * FROM test_table;
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
- * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
- * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert-overwrite-table.md b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
index 5c760f00ed0c4..ecfd060dfd5ee 100644
--- a/docs/sql-ref-syntax-dml-insert-overwrite-table.md
+++ b/docs/sql-ref-syntax-dml-insert-overwrite-table.md
@@ -25,57 +25,43 @@ The `INSERT OVERWRITE` statement overwrites the existing data in the table using
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 INSERT OVERWRITE [ TABLE ] table_identifier [ partition_spec [ IF NOT EXISTS ] ]
     { VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ] | query }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>VALUES ( { value | NULL } [ , ... ] ) [ , ( ... ) ]</em></code></dt>
-  <dd>Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted. A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>A query that produces the rows to be inserted. It can be in one of following formats:
-    <ul>
-      <li>a <code>SELECT</code> statement</li>
-      <li>a <code>TABLE</code> statement</li>
-      <li>a <code>FROM</code> statement</li>
-    </ul>
-   </dd>
-</dl>
+    for partitions.
+
+    **Syntax:** `PARTITION ( partition_col_name [ = partition_col_val ] [ , ... ] )`
+
+* **VALUES ( { value `|` NULL } [ , ... ] ) [ , ( ... ) ]**
+
+    Specifies the values to be inserted. Either an explicitly specified value or a NULL can be inserted.
+    A comma must be used to separate each value in the clause. More than one set of values can be specified to insert multiple rows.
+
+* **query**
+
+    A query that produces the rows to be inserted. It can be in one of following formats:
+    * a `SELECT` statement
+    * a `TABLE` statement
+    * a `FROM` statement
 
 ### Examples
 
 #### Insert Using a VALUES Clause
 
-{% highlight sql %}
+```sql
 -- Assuming the students table has already been created and populated.
 SELECT * FROM students;
 +-------------+-------------------------+----------+
@@ -102,12 +88,11 @@ SELECT * FROM students;
 |Ashua Hill|456 Erica Ct, Cupertino|    111111|
 |Brian Reed|723 Kern Ave, Palo Alto|    222222|
 +----------+-----------------------+----------+
-
-{% endhighlight %}
+```
 
 #### Insert Using a SELECT Statement
 
-{% highlight sql %}
+```sql
 -- Assuming the persons table has already been created and populated.
 SELECT * FROM persons;
 +-------------+-------------------------+---------+
@@ -129,11 +114,11 @@ SELECT * FROM students;
 +-------------+-------------------------+----------+
 |Dora Williams|134 Forest Ave, Melo Park|    222222|
 +-------------+-------------------------+----------+
-{% endhighlight %}
+```
 
 #### Insert Using a TABLE Statement
 
-{% highlight sql %}
+```sql
 -- Assuming the visiting_students table has already been created and populated.
 SELECT * FROM visiting_students;
 +-------------+---------------------+----------+
@@ -154,11 +139,11 @@ SELECT * FROM students;
 +-------------+---------------------+----------+
 |Gordon Martin| 779 Lake Ave, Oxford|    888888|
 +-------------+---------------------+----------+
-{% endhighlight %}
+```
 
 #### Insert Using a FROM Statement
 
-{% highlight sql %}
+```sql
 -- Assuming the applicants table has already been created and populated.
 SELECT * FROM applicants;
 +-----------+--------------------------+----------+---------+
@@ -182,10 +167,10 @@ SELECT * FROM students;
 +-----------+-------------------------+----------+
 | Jason Wang|    908 Bird St, Saratoga|    121212|
 +-----------+-------------------------+----------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
- * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
- * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-insert.md b/docs/sql-ref-syntax-dml-insert.md
index 2345add2460c8..62f6dee876450 100644
--- a/docs/sql-ref-syntax-dml-insert.md
+++ b/docs/sql-ref-syntax-dml-insert.md
@@ -21,7 +21,7 @@ license: |
 
 The INSERT statements:
 
- * [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
- * [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
- * [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
- * [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
+* [INSERT INTO statement](sql-ref-syntax-dml-insert-into.html)
+* [INSERT OVERWRITE statement](sql-ref-syntax-dml-insert-overwrite-table.html)
+* [INSERT OVERWRITE DIRECTORY statement](sql-ref-syntax-dml-insert-overwrite-directory.html)
+* [INSERT OVERWRITE DIRECTORY with Hive format statement](sql-ref-syntax-dml-insert-overwrite-directory-hive.html)
diff --git a/docs/sql-ref-syntax-dml-load.md b/docs/sql-ref-syntax-dml-load.md
index 01ece31bd17fa..9381b4267fb24 100644
--- a/docs/sql-ref-syntax-dml-load.md
+++ b/docs/sql-ref-syntax-dml-load.md
@@ -25,53 +25,40 @@ license: |
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 LOAD DATA [ LOCAL ] INPATH path [ OVERWRITE ] INTO TABLE table_identifier [ partition_spec ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>path</em></code></dt>
-  <dd>Path of the file system. It can be either an absolute or a relative path.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>table_identifier</em></code></dt>
-  <dd>
-    Specifies a table name, which may be optionally qualified with a database name.<br><br>
-    <b>Syntax:</b>
-      <code>
-        [ database_name. ] table_name
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>partition_spec</em></code></dt>
-  <dd>
+* **path**
+
+    Path of the file system. It can be either an absolute or a relative path.
+
+* **table_identifier**
+
+    Specifies a table name, which may be optionally qualified with a database name.
+
+    **Syntax:** `[ database_name. ] table_name`
+
+* **partition_spec**
+
     An optional parameter that specifies a comma separated list of key and value pairs
-    for partitions.<br><br>
-    <b>Syntax:</b>
-      <code>
-        PARTITION ( partition_col_name = partition_col_val [ , ... ] )
-      </code>
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>LOCAL</em></code></dt>
-  <dd>If specified, it causes the <code>INPATH</code> to be resolved against the local file system, instead of the default file system, which is typically a distributed storage.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>OVERWRITE</em></code></dt>
-  <dd>By default, new data is appended to the table. If <code>OVERWRITE</code> is used, the table is instead overwritten with new data.</dd>
-</dl>
+    for partitions.
+
+    **Syntax:** `PARTITION ( partition_col_name = partition_col_val [ , ... ] )`
+
+* **LOCAL**
+
+    If specified, it causes the `INPATH` to be resolved against the local file system, instead of the default file system, which is typically a distributed storage.
+
+* **OVERWRITE**
+
+    By default, new data is appended to the table. If `OVERWRITE` is used, the table is instead overwritten with new data.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Example without partition specification.
 -- Assuming the students table has already been created and populated.
 SELECT * FROM students;
@@ -123,4 +110,4 @@ SELECT * FROM test_load_partition;
 +---+---+---+
 |  1|  2|  3|
 +---+---+---+
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-syntax-dml.md b/docs/sql-ref-syntax-dml.md
index 9f75990555f64..fc408e1d38d26 100644
--- a/docs/sql-ref-syntax-dml.md
+++ b/docs/sql-ref-syntax-dml.md
@@ -21,5 +21,5 @@ license: |
 
 Data Manipulation Statements are used to add, change, or delete data. Spark SQL supports the following Data Manipulation Statements:
 
- * [INSERT](sql-ref-syntax-dml-insert.html)
- * [LOAD](sql-ref-syntax-dml-load.html)
+* [INSERT](sql-ref-syntax-dml-insert.html)
+* [LOAD](sql-ref-syntax-dml-load.html)
diff --git a/docs/sql-ref-syntax-qry-explain.md b/docs/sql-ref-syntax-qry-explain.md
index 298a2edaea1f2..7b84264a28cca 100644
--- a/docs/sql-ref-syntax-qry-explain.md
+++ b/docs/sql-ref-syntax-qry-explain.md
@@ -26,46 +26,38 @@ By default, this clause provides information about a physical plan only.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 EXPLAIN [ EXTENDED | CODEGEN | COST | FORMATTED ] statement
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>EXTENDED</em></code></dt>
-  <dd>Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan.
-   Parsed Logical plan is a unresolved plan that extracted from the query.
-   Analyzed logical plans transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects.
-   The optimized logical plan transforms through a set of optimization rules, resulting in the physical plan.
-  </dd>
-</dl> 
-
-<dl>
-  <dt><code><em>CODEGEN</em></code></dt>
-  <dd>Generates code for the statement, if any and a physical plan.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>COST</em></code></dt>
-  <dd>If plan node statistics are available, generates a logical plan and the statistics.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>FORMATTED</em></code></dt>
-  <dd>Generates two sections: a physical plan outline and node details.</dd>
-</dl>
-
-<dl>
-  <dt><code><em>statement</em></code></dt>
-  <dd>
+* **EXTENDED**
+
+    Generates parsed logical plan, analyzed logical plan, optimized logical plan and physical plan.
+    Parsed Logical plan is a unresolved plan that extracted from the query.
+    Analyzed logical plans transforms which translates unresolvedAttribute and unresolvedRelation into fully typed objects.
+    The optimized logical plan transforms through a set of optimization rules, resulting in the physical plan.
+
+* **CODEGEN**
+
+    Generates code for the statement, if any and a physical plan.
+
+* **COST**
+
+    If plan node statistics are available, generates a logical plan and the statistics.
+
+* **FORMATTED**
+
+    Generates two sections: a physical plan outline and node details.
+
+* **statement**
+
     Specifies a SQL statement to be explained.
-  </dd>
- </dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Default Output
 EXPLAIN select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;
 +----------------------------------------------------+
@@ -132,4 +124,4 @@ EXPLAIN FORMATTED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k
  Input: [k#19, sum#24L]
 |
 +----------------------------------------------------+
-{% endhighlight %}
+```
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-sampling.md
index 82f6588e6c504..f9c7b760a239f 100644
--- a/docs/sql-ref-syntax-qry-sampling.md
+++ b/docs/sql-ref-syntax-qry-sampling.md
@@ -26,19 +26,19 @@ The `TABLESAMPLE` statement is used to sample the table. It supports the followi
   * `TABLESAMPLE`(x `PERCENT`): Sample the table down to the given percentage. Note that percentages are defined as a number between 0 and 100.
   * `TABLESAMPLE`(`BUCKET` x `OUT OF` y): Sample the table down to a `x` out of `y` fraction.
 
-Note: `TABLESAMPLE` returns the approximate number of rows or fraction requested.
+**Note:** `TABLESAMPLE` returns the approximate number of rows or fraction requested.
 
 ### Syntax
 
-{% highlight sql %}
-TABLESAMPLE ((integer_expression | decimal_expression) PERCENT)
-    | TABLESAMPLE (integer_expression ROWS)
-    | TABLESAMPLE (BUCKET integer_expression OUT OF integer_expression)
-{% endhighlight %}
+```sql
+TABLESAMPLE ({ integer_expression | decimal_expression } PERCENT)
+    | TABLESAMPLE ( integer_expression ROWS )
+    | TABLESAMPLE ( BUCKET integer_expression OUT OF integer_expression )
+```
 
 ### Examples
 
-{% highlight sql %}
+```sql
 SELECT * FROM test;
 +--+----+
 |id|name|
@@ -87,8 +87,8 @@ SELECT * FROM test TABLESAMPLE (BUCKET 4 OUT OF 10);
 | 9|Eric|
 | 6|Mark|
 +--+----+
-{% endhighlight %}
+```
 
-### Related Statement
+### Related Statements
 
-  * [SELECT](sql-ref-syntax-qry-select.html)
\ No newline at end of file
+* [SELECT](sql-ref-syntax-qry-select.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index ac1e1ccb00ac9..e3bd2ed926ecc 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -21,7 +21,7 @@ license: |
 
 ### Description
 
-The <code>CLUSTER BY</code> clause is used to first repartition the data based
+The `CLUSTER BY` clause is used to first repartition the data based
 on the input expressions and then sort the data within each partition. This is
 semantically equivalent to performing a
 [DISTRIBUTE BY](sql-ref-syntax-qry-select-distribute-by.html) followed by a
@@ -30,22 +30,19 @@ resultant rows are sorted within each partition and does not guarantee a total o
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 CLUSTER BY { expression [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
+* **expression**
+
     Specifies combination of one or more values, operators and SQL functions that results in a value.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
     ('Zen Hui', 25),
@@ -90,15 +87,15 @@ SELECT age, name FROM person CLUSTER BY age;
 | 16|Shone S|
 | 16| Jack N|
 +---+-------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-cte.md b/docs/sql-ref-syntax-qry-select-cte.md
index 2408c884c64b5..351de64a2d026 100644
--- a/docs/sql-ref-syntax-qry-select-cte.md
+++ b/docs/sql-ref-syntax-qry-select-cte.md
@@ -25,33 +25,28 @@ A common table expression (CTE) defines a temporary result set that a user can r
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 WITH common_table_expression [ , ... ]
-{% endhighlight %}
+```
 
 While `common_table_expression` is defined as
-{% highlight sql %}
-expression_name [ ( column_name [ , ... ] ) ] [ AS ] ( [ common_table_expression ] query )
-{% endhighlight %}
+```sql
+expression_name [ ( column_name [ , ... ] ) ] [ AS ] ( query )
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>expression_name</em></code></dt>
-  <dd>
+* **expression_name**
+
     Specifies a name for the common table expression.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>query</em></code></dt>
-  <dd>
-    A <a href="sql-ref-syntax-qry-select.html">SELECT</a> statement.
-  </dd>
-</dl>
+
+* **query**
+
+    A [SELECT statement](sql-ref-syntax-qry-select.html).
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- CTE with multiple column aliases
 WITH t(x, y) AS (SELECT 1, 2)
 SELECT * FROM t WHERE x = 1 AND y = 2;
@@ -62,7 +57,7 @@ SELECT * FROM t WHERE x = 1 AND y = 2;
 +---+---+
 
 -- CTE in CTE definition
-WITH t as (
+WITH t AS (
     WITH t2 AS (SELECT 1)
     SELECT * FROM t2
 )
@@ -122,8 +117,8 @@ SELECT * FROM t2;
 +---+
 |  2|
 +---+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT](sql-ref-syntax-qry-select.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index 9e2db27ae7161..1fdfb91dad286 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -21,28 +21,25 @@ license: |
 
 ### Description
 
-The <code>DISTRIBUTE BY</code> clause is used to repartition the data based
+The `DISTRIBUTE BY` clause is used to repartition the data based
 on the input expressions. Unlike the [CLUSTER BY](sql-ref-syntax-qry-select-clusterby.html)
 clause, this does not sort the data within each partition.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 DISTRIBUTE BY { expression [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
+* **expression**
+
     Specifies combination of one or more values, operators and SQL functions that results in a value.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
     ('Zen Hui', 25),
@@ -85,15 +82,15 @@ SELECT age, name FROM person DISTRIBUTE BY age;
 | 16|Shone S|
 | 16| Jack N|
 +---+-------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index 22fe782f9eaa7..bd9377ef78df6 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -21,84 +21,79 @@ license: |
 
 ### Description
 
-The <code>GROUP BY</code> clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
+The `GROUP BY` clause is used to group the rows based on a set of specified grouping expressions and compute aggregations on
 the group of rows based on one or more specified aggregate functions. Spark also supports advanced aggregations to do multiple
 aggregations for the same input record set via `GROUPING SETS`, `CUBE`, `ROLLUP` clauses.
 When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 GROUP BY group_expression [ , group_expression [ , ... ] ]
     [ { WITH ROLLUP | WITH CUBE | GROUPING SETS (grouping_set [ , ...]) } ]
 
 GROUP BY GROUPING SETS (grouping_set [ , ...])
-{% endhighlight %}
+```
 
 While aggregate functions are defined as
-{% highlight sql %}
+```sql
 aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_expression ) ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>GROUPING SETS</em></code></dt>
-  <dd>
+* **GROUPING SETS**
+
     Groups the rows for each subset of the expressions specified in the grouping sets. For example,
-    <code>GROUP BY GROUPING SETS (warehouse, product)</code> is semantically equivalent
-    to union of results of <code>GROUP BY warehouse</code> and <code>GROUP BY product</code>. This clause
-    is a shorthand for a <code>UNION ALL</code> where each leg of the <code>UNION ALL</code>
-    operator performs aggregation of subset of the columns specified in the <code>GROUPING SETS</code> clause.
-  </dd>
-  <dt><code><em>grouping_set</em></code></dt>
-  <dd>
-    A grouping set is specified by zero or more comma-separated expressions in parentheses.<br><br>
-    <b>Syntax:</b>
-      <code>
-        ([expression [, ...]])
-      </code>
-  </dd>
-  <dt><code><em>grouping_expression</em></code></dt>
-  <dd>
+    `GROUP BY GROUPING SETS (warehouse, product)` is semantically equivalent
+    to union of results of `GROUP BY warehouse` and `GROUP BY product`. This clause
+    is a shorthand for a `UNION ALL` where each leg of the `UNION ALL`
+    operator performs aggregation of subset of the columns specified in the `GROUPING SETS` clause.
+
+* **grouping_set**
+
+    A grouping set is specified by zero or more comma-separated expressions in parentheses.
+
+    **Syntax:** `( [ expression [ , ... ] ] )`
+
+* **grouping_expression**
+
     Specifies the critieria based on which the rows are grouped together. The grouping of rows is performed based on
     result values of the grouping expressions. A grouping expression may be a column alias, a column position
     or an expression.
-  </dd>
-  <dt><code><em>ROLLUP</em></code></dt>
-  <dd>
+
+* **ROLLUP**
+
     Specifies multiple levels of aggregations in a single statement. This clause is used to compute aggregations
-    based on multiple grouping sets. <code>ROLLUP</code> is a shorthand for <code>GROUPING SETS</code>. For example,
-    <code>GROUP BY warehouse, product WITH ROLLUP</code> is equivalent to <code>GROUP BY GROUPING SETS
-    ((warehouse, product), (warehouse), ())</code>.
-    The N elements of a <code>ROLLUP</code> specification results in N+1 <code>GROUPING SETS</code>.
-  </dd>
-  <dt><code><em>CUBE</em></code></dt>
-  <dd>
-    <code>CUBE</code> clause is used to perform aggregations based on combination of grouping columns specified in the
-    <code>GROUP BY</code> clause. <code>CUBE</code> is a shorthand for <code>GROUPING SETS</code>. For example,
-    <code>GROUP BY warehouse, product WITH CUBE</code> is equivalent to <code>GROUP BY GROUPING SETS
-    ((warehouse, product), (warehouse), (product), ())</code>.
-    The N elements of a <code>CUBE</code> specification results in 2^N <code>GROUPING SETS</code>.
-  </dd>
-  <dt><code><em>aggregate_name</em></code></dt>
-  <dd>
+    based on multiple grouping sets. `ROLLUP` is a shorthand for `GROUPING SETS`. For example,
+    `GROUP BY warehouse, product WITH ROLLUP` is equivalent to `GROUP BY GROUPING SETS
+    ((warehouse, product), (warehouse), ())`.
+    The N elements of a `ROLLUP` specification results in N+1 `GROUPING SETS`.
+
+* **CUBE**
+
+    `CUBE` clause is used to perform aggregations based on combination of grouping columns specified in the
+    `GROUP BY` clause. `CUBE` is a shorthand for `GROUPING SETS`. For example,
+    `GROUP BY warehouse, product WITH CUBE` is equivalent to `GROUP BY GROUPING SETS
+    ((warehouse, product), (warehouse), (product), ())`.
+    The N elements of a `CUBE` specification results in 2^N `GROUPING SETS`.
+
+* **aggregate_name**
+
     Specifies an aggregate function name (MIN, MAX, COUNT, SUM, AVG, etc.).
-  </dd>
-  <dt><code><em>DISTINCT</em></code></dt>
-  <dd>
+
+* **DISTINCT**
+
     Removes duplicates in input rows before they are passed to aggregate functions.
-  </dd>
-  <dt><code><em>FILTER</em></code></dt>
-  <dd>
-    Filters the input rows for which the <code>boolean_expression</code> in the <code>WHERE</code> clause evaluates
+
+* **FILTER**
+
+    Filters the input rows for which the `boolean_expression` in the `WHERE` clause evaluates
     to true are passed to the aggregate function; other rows are discarded.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
 INSERT INTO dealer VALUES
     (100, 'Fremont', 'Honda Civic', 10),
@@ -174,106 +169,106 @@ SELECT id, sum(quantity) FILTER (
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
     ORDER BY city;
-+--------+------------+---+
-|    city|   car_model|sum|
-+--------+------------+---+
-|    null|        null| 78|
-|    null| HondaAccord| 33|
-|    null|    HondaCRV| 10|
-|    null|  HondaCivic| 35|
-|  Dublin|        null| 33|
-|  Dublin| HondaAccord| 10|
-|  Dublin|    HondaCRV|  3|
-|  Dublin|  HondaCivic| 20|
-| Fremont|        null| 32|
-| Fremont| HondaAccord| 15|
-| Fremont|    HondaCRV|  7|
-| Fremont|  HondaCivic| 10|
-| SanJose|        null| 13|
-| SanJose| HondaAccord|  8|
-| SanJose|  HondaCivic|  5|
-+--------+------------+---+
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|     null| HondaAccord| 33|
+|     null|    HondaCRV| 10|
+|     null|  HondaCivic| 35|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
 
 -- Alternate syntax for `GROUPING SETS` in which both `GROUP BY` and `GROUPING SETS`
 -- specifications are present.
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY city, car_model GROUPING SETS ((city, car_model), (city), (car_model), ())
     ORDER BY city, car_model;
-+--------+------------+---+
-|    city|   car_model|sum|
-+--------+------------+---+
-|    null|        null| 78|
-|    null| HondaAccord| 33|
-|    null|    HondaCRV| 10|
-|    null|  HondaCivic| 35|
-|  Dublin|        null| 33|
-|  Dublin| HondaAccord| 10|
-|  Dublin|    HondaCRV|  3|
-|  Dublin|  HondaCivic| 20|
-| Fremont|        null| 32|
-| Fremont| HondaAccord| 15|
-| Fremont|    HondaCRV|  7|
-| Fremont|  HondaCivic| 10|
-| SanJose|        null| 13|
-| SanJose| HondaAccord|  8|
-| SanJose|  HondaCivic|  5|
-+--------+------------+---+
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|     null| HondaAccord| 33|
+|     null|    HondaCRV| 10|
+|     null|  HondaCivic| 35|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
 
 -- Group by processing with `ROLLUP` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY city, car_model WITH ROLLUP
     ORDER BY city, car_model;
-+--------+------------+---+
-|    city|   car_model|sum|
-+--------+------------+---+
-|    null|        null| 78|
-|  Dublin|        null| 33|
-|  Dublin| HondaAccord| 10|
-|  Dublin|    HondaCRV|  3|
-|  Dublin|  HondaCivic| 20|
-| Fremont|        null| 32|
-| Fremont| HondaAccord| 15|
-| Fremont|    HondaCRV|  7|
-| Fremont|  HondaCivic| 10|
-| SanJose|        null| 13|
-| SanJose| HondaAccord|  8|
-| SanJose|  HondaCivic|  5|
-+--------+------------+---+
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
 
 -- Group by processing with `CUBE` clause.
 -- Equivalent GROUP BY GROUPING SETS ((city, car_model), (city), (car_model), ())
 SELECT city, car_model, sum(quantity) AS sum FROM dealer
     GROUP BY city, car_model WITH CUBE
     ORDER BY city, car_model;
-+--------+------------+---+
-|    city|   car_model|sum|
-+--------+------------+---+
-|    null|        null| 78|
-|    null| HondaAccord| 33|
-|    null|    HondaCRV| 10|
-|    null|  HondaCivic| 35|
-|  Dublin|        null| 33|
-|  Dublin| HondaAccord| 10|
-|  Dublin|    HondaCRV|  3|
-|  Dublin|  HondaCivic| 20|
-| Fremont|        null| 32|
-| Fremont| HondaAccord| 15|
-| Fremont|    HondaCRV|  7|
-| Fremont|  HondaCivic| 10|
-| SanJose|        null| 13|
-| SanJose| HondaAccord|  8|
-| SanJose|  HondaCivic|  5|
-+--------+------------+---+
-{% endhighlight %}
++---------+------------+---+
+|     city|   car_model|sum|
++---------+------------+---+
+|     null|        null| 78|
+|     null| HondaAccord| 33|
+|     null|    HondaCRV| 10|
+|     null|  HondaCivic| 35|
+|   Dublin|        null| 33|
+|   Dublin| HondaAccord| 10|
+|   Dublin|    HondaCRV|  3|
+|   Dublin|  HondaCivic| 20|
+|  Fremont|        null| 32|
+|  Fremont| HondaAccord| 15|
+|  Fremont|    HondaCRV|  7|
+|  Fremont|  HondaCivic| 10|
+| San Jose|        null| 13|
+| San Jose| HondaAccord|  8|
+| San Jose|  HondaCivic|  5|
++---------+------------+---+
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index c8c4f2c38104c..935782c551e1f 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -21,39 +21,35 @@ license: |
 
 ### Description
 
-The <code>HAVING</code> clause is used to filter the results produced by
-<code>GROUP BY</code> based on the specified condition. It is often used
+The `HAVING` clause is used to filter the results produced by
+`GROUP BY` based on the specified condition. It is often used
 in conjunction with a [GROUP BY](sql-ref-syntax-qry-select-groupby.html)
 clause.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 HAVING boolean_expression
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>boolean_expression</em></code></dt>
-  <dd>
-    Specifies any expression that evaluates to a result type <code>boolean</code>. Two or
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
     more expressions may be combined together using the logical
-    operators ( <code>AND</code>, <code>OR</code> ).<br><br>
-
-    <b>Note</b><br>
-    The expressions specified in the <code>HAVING</code> clause can only refer to:
-     <ol>
-      <li>Constants</li>
-      <li>Expressions that appear in GROUP BY</li>
-      <li>Aggregate functions</li>
-    </ol>
-  </dd>
-</dl>
+    operators ( `AND`, `OR` ).
+
+    **Note**
+
+    The expressions specified in the `HAVING` clause can only refer to:
+    1. Constants
+    2. Expressions that appear in GROUP BY
+    3. Aggregate functions
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE dealer (id INT, city STRING, car_model STRING, quantity INT);
 INSERT INTO dealer VALUES
     (100, 'Fremont', 'Honda Civic', 10),
@@ -117,15 +113,15 @@ SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
 +---+
 | 78|
 +---+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index 16f4f95f90ea1..4bb48b08d5e3b 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -23,39 +23,33 @@ license: |
 
 Join Hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
 
+### Syntax
+
+```sql
+/*+ join_hint [ , ... ] */
+```
+
 ### Join Hints Types
 
-<dl>
-  <dt><code><em>BROADCAST</em></code></dt>
-  <dd>
-    Suggests that Spark use broadcast join. The join side with the hint will be broadcast regardless of <code>autoBroadcastJoinThreshold</code>. If both sides of the join have the broadcast hints, the one with the smaller size (based on stats) will be broadcast. The aliases for <code>BROADCAST</code> are <code>BROADCASTJOIN</code> and <code>MAPJOIN</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>MERGE</em></code></dt>
-  <dd>
-     Suggests that Spark use shuffle sort merge join. The aliases for <code>MERGE</code> are <code>SHUFFLE_MERGE</code> and <code>MERGEJOIN</code>.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>SHUFFLE_HASH</em></code></dt>
-  <dd>
-     Suggests that Spark use shuffle hash join. If both sides have the shuffle hash hints, Spark chooses the smaller side (based on stats) as the build side.
-  </dd>
-</dl>
-
-<dl>
-  <dt><code><em>SHUFFLE_REPLICATE_NL</em></code></dt>
-  <dd>
+* **BROADCAST**
+
+    Suggests that Spark use broadcast join. The join side with the hint will be broadcast regardless of `autoBroadcastJoinThreshold`. If both sides of the join have the broadcast hints, the one with the smaller size (based on stats) will be broadcast. The aliases for `BROADCAST` are `BROADCASTJOIN` and `MAPJOIN`.
+
+* **MERGE**
+
+    Suggests that Spark use shuffle sort merge join. The aliases for `MERGE` are `SHUFFLE_MERGE` and `MERGEJOIN`.
+
+* **SHUFFLE_HASH**
+
+    Suggests that Spark use shuffle hash join. If both sides have the shuffle hash hints, Spark chooses the smaller side (based on stats) as the build side.
+
+* **SHUFFLE_REPLICATE_NL**
+
     Suggests that Spark use shuffle-and-replicate nested loop join.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Join Hints for broadcast join
 SELECT /*+ BROADCAST(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
 SELECT /*+ BROADCASTJOIN (t1) */ * FROM t1 left JOIN t2 ON t1.key = t2.key;
@@ -78,10 +72,10 @@ SELECT /*+ SHUFFLE_REPLICATE_NL(t1) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.ke
 -- Spark will issue Warning in the following example
 -- org.apache.spark.sql.catalyst.analysis.HintErrorLogger: Hint (strategy=merge)
 -- is overridden by another hint and will not take effect.
-SELECT /*+ BROADCAST(t1) */ /*+ MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
-{% endhighlight %}
+SELECT /*+ BROADCAST(t1), MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;
+```
 
 ### Related Statements
 
- * [JOIN](sql-ref-syntax-qry-select-join.html)
- * [SELECT](sql-ref-syntax-qry-select.html)
+* [JOIN](sql-ref-syntax-qry-select-join.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-inline-table.md b/docs/sql-ref-syntax-qry-select-inline-table.md
index 9c33cbc679f06..7f0372ea787e5 100644
--- a/docs/sql-ref-syntax-qry-select-inline-table.md
+++ b/docs/sql-ref-syntax-qry-select-inline-table.md
@@ -25,32 +25,25 @@ An inline table is a temporary table created using a VALUES clause.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 VALUES ( expression [ , ... ] ) [ table_alias ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
+* **expression**
+
     Specifies a combination of one or more values, operators and SQL functions that results in a value.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>table_alias</em></code></dt>
-  <dd>
-    Specifies a temporary name with an optional column name list. <br><br>
-    <b>Syntax:</b>
-      <code>
-        [ AS ] table_name [ ( column_name [ , ... ] ) ]
-      </code>
-  </dd>
-</dl>
+
+* **table_alias**
+
+    Specifies a temporary name with an optional column name list.
+
+    **Syntax:** `[ AS ] table_name [ ( column_name [ , ... ] ) ]`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- single row, without a table alias
 SELECT * FROM VALUES ("one", 1);
 +----+----+
@@ -77,8 +70,8 @@ SELECT * FROM VALUES ("one", array(0, 1)), ("two", array(2, 3)) AS data(a, b);
 |one|[0, 1]|
 |two|[2, 3]|
 +---+------+
-{% endhighlight %}
+```
 
-### Related Statement
+### Related Statements
 
- * [SELECT](sql-ref-syntax-qry-select.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index 0b1bb1eb8fd61..28b21f5e3f0ff 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -25,118 +25,95 @@ A SQL join is used to combine rows from two relations based on join criteria. Th
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 relation { [ join_type ] JOIN relation [ join_criteria ] | NATURAL join_type JOIN relation }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>relation</em></code></dt>
-  <dd>
+* **relation**
+
     Specifies the relation to be joined.
-  </dd>
-  <dt><code><em>join_type</em></code></dt>
-  <dd>
-    Specifies the join type.<br><br>
-    <b>Syntax:</b><br>
-      <code>
-        [ INNER ]
-        | CROSS
-        | LEFT [ OUTER ]
-        | [ LEFT ] SEMI
-        | RIGHT [ OUTER ]
-        | FULL [ OUTER ]
-        | [ LEFT ] ANTI
-      </code>
-  </dd>
-  <dt><code><em>join_criteria</em></code></dt>
-  <dd>
-    Specifies how the rows from one relation will be combined with the rows of another relation.<br><br>
-    <b>Syntax:</b>
-      <code>
-        ON boolean_expression | USING ( column_name [ , column_name ... ] )
-      </code> <br><br>
-      <code>boolean_expression</code><br>
-      Specifies an expression with a return type of boolean.
-  </dd>
-</dl>
+
+* **join_type**
+
+    Specifies the join type.
+
+    **Syntax:**
+
+    `[ INNER ] | CROSS | LEFT [ OUTER ] | [ LEFT ] SEMI | RIGHT [ OUTER ] | FULL [ OUTER ] | [ LEFT ] ANTI`
+
+* **join_criteria**
+
+    Specifies how the rows from one relation will be combined with the rows of another relation.
+
+    **Syntax:** `ON boolean_expression | USING ( column_name [ , ... ] )`
+
+    `boolean_expression`
+
+    Specifies an expression with a return type of boolean.
 
 ### Join Types
 
-#### <b>Inner Join</b>
-
-<dd>
-The inner join is the default join in Spark SQL. It selects rows that have matching values in both relations.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation [ INNER ] JOIN relation [ join_criteria ]
-    </code>
-</dd>
-
-#### <b>Left Join </b>
-
-<dd>
-A left join returns all values from the left relation and the matched values from the right relation, or appends NULL if there is no match. It is also referred to as a left outer join.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation LEFT [ OUTER ] JOIN relation [ join_criteria ]
-    </code>
-</dd>
-
-#### <b>Right Join </b>
-
-<dd>
-A right join returns all values from the right relation and the matched values from the left relation, or appends NULL if there is no match. It is also referred to as a right outer join.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation RIGHT [ OUTER ] JOIN relation [ join_criteria ]
-    </code>
-</dd>
-
-#### <b>Full Join </b>
-
-<dd>
-A full join returns all values from both relations, appending NULL values on the side that does not have a match. It is also referred to as a full outer join.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation FULL [ OUTER ] JOIN relation [ join_criteria ]
-    </code>
-</dd>
-
-#### <b>Cross Join </b>
-
-<dd>
-A cross join returns the Cartesian product of two relations.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation CROSS JOIN relation [ join_criteria ]
-    </code>
-</dd>
-
-#### <b>Semi Join </b>
-
-<dd>
-A semi join returns values from the left side of the relation that has a match with the right. It is also referred to as a left semi join.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation [ LEFT ] SEMI JOIN relation [ join_criteria ]
-    </code>
-</dd>
-
-#### <b>Anti Join </b>
-
-<dd>
-An anti join returns values from the left relation that has no match with the right. It is also referred to as a left anti join.<br><br>
-  <b>Syntax:</b><br>
-    <code>
-    relation [ LEFT ] ANTI JOIN relation [ join_criteria ]
-    </code>
-</dd>
+#### **Inner Join**
+
+The inner join is the default join in Spark SQL. It selects rows that have matching values in both relations.
+
+**Syntax:**
+
+`relation [ INNER ] JOIN relation [ join_criteria ]`
+
+#### **Left Join**
+
+A left join returns all values from the left relation and the matched values from the right relation, or appends NULL if there is no match. It is also referred to as a left outer join.
+
+**Syntax:**
+
+`relation LEFT [ OUTER ] JOIN relation [ join_criteria ]`
+
+#### **Right Join**
+
+A right join returns all values from the right relation and the matched values from the left relation, or appends NULL if there is no match. It is also referred to as a right outer join.
+
+**Syntax:**
+
+`relation RIGHT [ OUTER ] JOIN relation [ join_criteria ]`
+
+#### **Full Join**
+
+A full join returns all values from both relations, appending NULL values on the side that does not have a match. It is also referred to as a full outer join.
+
+**Syntax:**
+
+`relation FULL [ OUTER ] JOIN relation [ join_criteria ]`
+
+#### **Cross Join**
+
+A cross join returns the Cartesian product of two relations.
+
+**Syntax:**
+
+`relation CROSS JOIN relation [ join_criteria ]`
+
+#### **Semi Join**
+
+A semi join returns values from the left side of the relation that has a match with the right. It is also referred to as a left semi join.
+
+**Syntax:**
+
+`relation [ LEFT ] SEMI JOIN relation [ join_criteria ]`
+
+#### **Anti Join**
+
+An anti join returns values from the left relation that has no match with the right. It is also referred to as a left anti join.
+
+**Syntax:**
+
+`relation [ LEFT ] ANTI JOIN relation [ join_criteria ]`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Use employee and department tables to demonstrate different type of joins.
 SELECT * FROM employee;
 +---+-----+------+
@@ -253,9 +230,9 @@ SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.dept
 |104| Evan|     4|
 |106|  Amy|     6|
 +---+-----+------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
-  * [SELECT](sql-ref-syntax-qry-select.html)
-  * [Join Hints](sql-ref-syntax-qry-select-hints.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [Join Hints](sql-ref-syntax-qry-select-hints.html)
diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md
index 408673c532ddd..feb5eb7b3c80d 100644
--- a/docs/sql-ref-syntax-qry-select-like.md
+++ b/docs/sql-ref-syntax-qry-select-like.md
@@ -25,38 +25,30 @@ A LIKE predicate is used to search for a specific pattern.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 [ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | RLIKE regex_pattern }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>search_pattern</em></code></dt>
-  <dd>
-    Specifies a string pattern to be searched by the <code>LIKE</code> clause. It can contain special pattern-matching characters:
-    <ul>
-      <li><code>%</code></li> matches zero or more characters.
-      <li><code>_</code></li> matches exactly one character.
-    </ul>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>esc_char</em></code></dt>
-  <dd>
-    Specifies the escape character. The default escape character is <code>\</code>.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>regex_pattern</em></code></dt>
-  <dd>
-    Specifies a regular expression search pattern to be searched by the <code>RLIKE</code> clause.
-  </dd>
-</dl>
+* **search_pattern**
+
+    Specifies a string pattern to be searched by the `LIKE` clause. It can contain special pattern-matching characters:
+
+    * `%` matches zero or more characters.
+    * `_` matches exactly one character.
+
+* **esc_char**
+
+    Specifies the escape character. The default escape character is `\`.
+
+* **regex_pattern**
+
+    Specifies a regular expression search pattern to be searched by the `RLIKE` clause.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
     (100, 'John', 30),
@@ -90,12 +82,11 @@ SELECT * FROM person WHERE name NOT LIKE 'M_ry';
 |400|   Dan| 50|
 +---+------+---+
 
-SELECT * FROM person WHERE name RLIKE '[MD]';
+SELECT * FROM person WHERE name RLIKE 'M+';
 +---+----+----+
 | id|name| age|
 +---+----+----+
 |300|Mike|  80|
-|400| Dan|  50|
 |200|Mary|null|
 +---+----+----+
 
@@ -112,9 +103,9 @@ SELECT * FROM person WHERE name LIKE '%$_%' ESCAPE '$';
 +---+------+---+
 |500|Evan_W| 16|
 +---+------+---+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index eaeaed068102f..03c4df3cbc442 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -21,34 +21,31 @@ license: |
 
 ### Description
 
-The <code>LIMIT</code> clause is used to constrain the number of rows returned by
+The `LIMIT` clause is used to constrain the number of rows returned by
 the [SELECT](sql-ref-syntax-qry-select.html) statement. In general, this clause
 is used in conjunction with [ORDER BY](sql-ref-syntax-qry-select-orderby.html) to
 ensure that the results are deterministic.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 LIMIT { ALL | integer_expression }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>ALL</em></code></dt>
-  <dd>
+* **ALL**
+
     If specified, the query returns all the rows. In other words, no limit is applied if this
     option is specified.
-  </dd>
-  <dt><code><em>integer_expression</em></code></dt>
-  <dd>
+
+* **integer_expression**
+
     Specifies a foldable expression that returns an integer.
-  </dd>
-</dl>
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (name STRING, age INT);
 INSERT INTO person VALUES
     ('Zen Hui', 25),
@@ -94,16 +91,16 @@ SELECT name, age FROM person ORDER BY name LIMIT length('SPARK');
 
 -- A non-foldable expression as an input to LIMIT is not allowed.
 SELECT name, age FROM person ORDER BY name LIMIT length(name);
-  org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a constant value ...
-{% endhighlight %}
+org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a constant value ...
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index d927177398f7f..85bbe514cdc95 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -21,56 +21,48 @@ license: |
 
 ### Description
 
-The <code>ORDER BY</code> clause is used to return the result rows in a sorted manner
+The `ORDER BY` clause is used to return the result rows in a sorted manner
 in the user specified order. Unlike the [SORT BY](sql-ref-syntax-qry-select-sortby.html)
 clause, this clause guarantees a total order in the output.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 ORDER BY { expression [ sort_direction | nulls_sort_oder ] [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>ORDER BY</em></code></dt>
-  <dd>
-    Specifies a comma-separated list of expressions along with optional parameters <code>sort_direction</code>
-    and <code>nulls_sort_order</code> which are used to sort the rows.
-  </dd>
-  <dt><code><em>sort_direction</em></code></dt>
-  <dd>
+* **ORDER BY**
+
+    Specifies a comma-separated list of expressions along with optional parameters `sort_direction`
+    and `nulls_sort_order` which are used to sort the rows.
+
+* **sort_direction**
+
     Optionally specifies whether to sort the rows in ascending or descending
-    order. The valid values for the sort direction are <code>ASC</code> for ascending
-    and <code>DESC</code> for descending. If sort direction is not explicitly specified, then by default
-    rows are sorted ascending. <br><br>
-    <b>Syntax:</b>
-    <code>
-       [ ASC | DESC ]
-    </code>
-  </dd>
-  <dt><code><em>nulls_sort_order</em></code></dt>
-  <dd>
+    order. The valid values for the sort direction are `ASC` for ascending
+    and `DESC` for descending. If sort direction is not explicitly specified, then by default
+    rows are sorted ascending.
+
+    **Syntax:** [ ASC `|` DESC ]
+
+* **nulls_sort_order**
+
     Optionally specifies whether NULL values are returned before/after non-NULL values. If
-    <code>null_sort_order</code> is not specified, then NULLs sort first if sort order is
-    <code>ASC</code> and NULLS sort last if sort order is <code>DESC</code>.<br><br>
-    <ol>
-      <li> If <code>NULLS FIRST</code> is specified, then NULL values are returned first
-           regardless of the sort order.</li>
-      <li>If <code>NULLS LAST</code> is specified, then NULL values are returned last regardless of
-           the sort order. </li>
-    </ol><br>
-    <b>Syntax:</b>
-    <code>
-       [ NULLS { FIRST | LAST } ]
-    </code>
-  </dd>
-</dl>
+    `null_sort_order` is not specified, then NULLs sort first if sort order is
+    `ASC` and NULLS sort last if sort order is `DESC`.
+
+    1. If `NULLS FIRST` is specified, then NULL values are returned first
+       regardless of the sort order.
+    2. If `NULLS LAST` is specified, then NULL values are returned last regardless of
+       the sort order.
+
+    **Syntax:** `[ NULLS { FIRST | LAST } ]`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
     (100, 'John', 30),
@@ -139,15 +131,15 @@ SELECT * FROM person ORDER BY name ASC, age DESC;
 |200| Mary|null|
 |300| Mike|  80|
 +---+-----+----+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-setops.md b/docs/sql-ref-syntax-qry-select-setops.md
index 98c20941d16bf..8cd12c37fa603 100644
--- a/docs/sql-ref-syntax-qry-select-setops.md
+++ b/docs/sql-ref-syntax-qry-select-setops.md
@@ -35,13 +35,13 @@ Note that input relations must have the same number of columns and compatible da
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 [ ( ] relation [ ) ] EXCEPT | MINUS [ ALL | DISTINCT ] [ ( ] relation [ ) ]
-{% endhighlight %}
+```
 
 #### Examples
 
-{% highlight sql %}
+```sql
 -- Use number1 and number2 tables to demonstrate set operators in this page.
 SELECT * FROM number1;
 +---+
@@ -98,7 +98,7 @@ SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
 |  3|
 |  4|
 +---+
-{% endhighlight %}
+```
 
 ### INTERSECT
 
@@ -106,13 +106,13 @@ SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 [ ( ] relation [ ) ] INTERSECT [ ALL | DISTINCT ] [ ( ] relation [ ) ]
-{% endhighlight %}
+```
 
 #### Examples
 
-{% highlight sql %}
+```sql
 (SELECT c FROM number1) INTERSECT (SELECT c FROM number2);
 +---+
 |  c|
@@ -137,7 +137,7 @@ SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
 |  2|
 |  2|
 +---+
-{% endhighlight %}
+```
 
 ### UNION
 
@@ -145,13 +145,13 @@ SELECT c FROM number1 MINUS ALL (SELECT c FROM number2);
 
 #### Syntax
 
-{% highlight sql %}
+```sql
 [ ( ] relation [ ) ] UNION [ ALL | DISTINCT ] [ ( ] relation [ ) ]
-{% endhighlight %}
+```
 
 ### Examples
 
-{% highlight sql %}
+```sql
 (SELECT c FROM number1) UNION (SELECT c FROM number2);
 +---+
 |  c|
@@ -189,8 +189,8 @@ SELECT c FROM number1 UNION ALL (SELECT c FROM number2);
 |  2|
 |  2|
 +---+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Statement](sql-ref-syntax-qry-select.html)
+* [SELECT Statement](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index 1dfa10429709e..554bdb569d005 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -21,58 +21,50 @@ license: |
 
 ### Description
 
-The <code>SORT BY</code> clause is used to return the result rows sorted
+The `SORT BY` clause is used to return the result rows sorted
 within each partition in the user specified order. When there is more than one partition
-<code>SORT BY</code> may return result that is partially ordered. This is different
+`SORT BY` may return result that is partially ordered. This is different
 than [ORDER BY](sql-ref-syntax-qry-select-orderby.html) clause which guarantees a
 total order of the output.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 SORT BY { expression [ sort_direction | nulls_sort_order ] [ , ... ] }
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>SORT BY</em></code></dt>
-  <dd>
-    Specifies a comma-separated list of expressions along with optional parameters <code>sort_direction</code>
-    and <code>nulls_sort_order</code> which are used to sort the rows within each partition.
-  </dd>
-  <dt><code><em>sort_direction</em></code></dt>
-  <dd>
+* **SORT BY**
+
+    Specifies a comma-separated list of expressions along with optional parameters `sort_direction`
+    and `nulls_sort_order` which are used to sort the rows within each partition.
+
+* **sort_direction**
+
     Optionally specifies whether to sort the rows in ascending or descending
-    order. The valid values for the sort direction are <code>ASC</code> for ascending
-    and <code>DESC</code> for descending. If sort direction is not explicitly specified, then by default
-    rows are sorted ascending. <br><br>
-    <b>Syntax:</b>
-    <code>
-       [ ASC | DESC ]
-    </code>
-  </dd>
-  <dt><code><em>nulls_sort_order</em></code></dt>
-  <dd>
+    order. The valid values for the sort direction are `ASC` for ascending
+    and `DESC` for descending. If sort direction is not explicitly specified, then by default
+    rows are sorted ascending.
+
+    **Syntax:** `[ ASC | DESC ]`
+
+* **nulls_sort_order**
+
     Optionally specifies whether NULL values are returned before/after non-NULL values. If
-    <code>null_sort_order</code> is not specified, then NULLs sort first if sort order is
-    <code>ASC</code> and NULLS sort last if sort order is <code>DESC</code>.<br><br>
-    <ol>
-      <li> If <code>NULLS FIRST</code> is specified, then NULL values are returned first
-           regardless of the sort order.</li>
-      <li>If <code>NULLS LAST</code> is specified, then NULL values are returned last regardless of
-           the sort order. </li>
-    </ol><br>
-    <b>Syntax:</b>
-    <code>
-       [ NULLS { FIRST | LAST } ]
-    </code>
-  </dd>
-</dl>
+    `null_sort_order` is not specified, then NULLs sort first if sort order is
+    `ASC` and NULLS sort last if sort order is `DESC`.
+
+    1. If `NULLS FIRST` is specified, then NULL values are returned first
+       regardless of the sort order.
+    2. If `NULLS LAST` is specified, then NULL values are returned last regardless of
+       the sort order.
+
+    **Syntax:** `[ NULLS { FIRST | LAST } ]`
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (zip_code INT, name STRING, age INT);
 INSERT INTO person VALUES
     (94588, 'Zen Hui', 50),
@@ -172,15 +164,15 @@ SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
 | David K|  42|   94511|
 |Lalit B.|null|   94511|
 +--------+----+--------+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
index 1d9505a2bc11b..cc8d7c34645fb 100644
--- a/docs/sql-ref-syntax-qry-select-tvf.md
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -25,28 +25,21 @@ A table-valued function (TVF) is a function that returns a relation or a set of
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 function_name ( expression [ , ... ] ) [ table_alias ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
+* **expression**
+
     Specifies a combination of one or more values, operators and SQL functions that results in a value.
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>table_alias</em></code></dt>
-  <dd>
-    Specifies a temporary name with an optional column name list. <br/><br/>
-    <b>Syntax:</b>
-      <code>
-        [ AS ] table_name [ ( column_name [ , ... ] ) ]
-      </code>
-  </dd>
-</dl>
+
+* **table_alias**
+
+    Specifies a temporary name with an optional column name list.
+
+    **Syntax:** `[ AS ] table_name [ ( column_name [ , ... ] ) ]`
 
 ### Supported Table-valued Functions
 
@@ -59,7 +52,7 @@ function_name ( expression [ , ... ] ) [ table_alias ]
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- range call with end
 SELECT * FROM range(6 + cos(3));
 +---+
@@ -105,8 +98,8 @@ SELECT * FROM range(5, 8) AS test;
 |  6|
 |  7|
 +---+
-{% endhighlight %}
+```
 
-### Related Statement
+### Related Statements
 
- * [SELECT](sql-ref-syntax-qry-select.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select-usedb.md b/docs/sql-ref-syntax-qry-select-usedb.md
index bb95a8e4ddf30..90076e0306125 100644
--- a/docs/sql-ref-syntax-qry-select-usedb.md
+++ b/docs/sql-ref-syntax-qry-select-usedb.md
@@ -28,32 +28,30 @@ The default database name is 'default'.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 USE database_name
-{% endhighlight %}
+```
 
 ### Parameter
 
-<dl>
-  <dt><code><em>database_name</em></code></dt>
-  <dd>
-     Name of the database will be used. If the database does not exist, an exception will be thrown.
-  </dd>
-</dl>
+* **database_name**
+
+    Name of the database will be used. If the database does not exist, an exception will be thrown.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 -- Use the 'userdb' which exists.
 USE userdb;
 
 -- Use the 'userdb1' which doesn't exist
 USE userdb1;
-  Error: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: Database 'userdb1' not found;(state=,code=0)
-{% endhighlight %}
+Error: org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException: Database 'userdb1' not found;
+(state=,code=0)
+```
 
 ### Related Statements
 
- * [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
- * [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
- * [CREATE TABLE ](sql-ref-syntax-ddl-create-table.html)
+* [CREATE DATABASE](sql-ref-syntax-ddl-create-database.html)
+* [DROP DATABASE](sql-ref-syntax-ddl-drop-database.html)
+* [CREATE TABLE ](sql-ref-syntax-ddl-create-table.html)
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index 360313fcfff1c..ca3f5ec7866c6 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -21,29 +21,26 @@ license: |
 
 ### Description
 
-The <code>WHERE</code> clause is used to limit the results of the <code>FROM</code>
+The `WHERE` clause is used to limit the results of the `FROM`
 clause of a query or a subquery based on the specified condition.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 WHERE boolean_expression
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>boolean_expression</em></code></dt>
-  <dd>
-    Specifies any expression that evaluates to a result type <code>boolean</code>. Two or
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
     more expressions may be combined together using the logical
-    operators ( <code>AND</code>, <code>OR</code> ).
-  </dd>
-</dl>
+    operators ( `AND`, `OR` ).
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE person (id INT, name STRING, age INT);
 INSERT INTO person VALUES
     (100, 'John', 30),
@@ -116,15 +113,15 @@ SELECT * FROM person AS parent
 +---+----+----+
 |200|Mary|null|
 +---+----+----+
-{% endhighlight %}
+```
 
 ### Related Statements
 
- * [SELECT Main](sql-ref-syntax-qry-select.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index bc2cc0269124e..1aeecdb982c4c 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -28,7 +28,7 @@ of a query along with examples.
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 [ WITH with_query [ , ... ] ]
 select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_statement, ... ]
     [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
@@ -37,126 +37,125 @@ select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_stat
     [ DISTRIBUTE BY { expression [, ...] } ]
     [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
     [ LIMIT { ALL | expression } ]
-{% endhighlight %}
+```
 
 While `select_statement` is defined as
-{% highlight sql %}
+```sql
 SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
     FROM { from_item [ , ...] }
     [ WHERE boolean_expression ]
     [ GROUP BY expression [ , ...] ]
     [ HAVING boolean_expression ]
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>with_query</em></code></dt>
-  <dd>
-    Specifies the <a href="sql-ref-syntax-qry-select-cte.html">common table expressions (CTEs)</a> before the main query block.
+* **with_query**
+
+    Specifies the [common table expressions (CTEs)](sql-ref-syntax-qry-select-cte.html) before the main query block.
     These table expressions are allowed to be referenced later in the FROM clause. This is useful to abstract
     out repeated subquery blocks in the FROM clause and improves readability of the query.
-  </dd>
-  <dt><code><em>hints</em></code></dt>
-  <dd>
+
+* **hints**
+
     Hints can be specified to help spark optimizer make better planning decisions. Currently spark supports hints
     that influence selection of join strategies and repartitioning of the data.
-  </dd>
-  <dt><code><em>ALL</em></code></dt>
-  <dd>
+
+* **ALL**
+
     Select all matching rows from the relation and is enabled by default.
-  </dd>
-  <dt><code><em>DISTINCT</em></code></dt>
-  <dd>
+
+* **DISTINCT**
+
     Select all matching rows from the relation after removing duplicates in results.
-  </dd>
-  <dt><code><em>named_expression</em></code></dt>
-  <dd>
-    An expression with an assigned name. In general, it denotes a column expression.<br><br>
-    <b>Syntax:</b>
-      <code>
-        expression [AS] [alias]
-      </code>
-  </dd>
-  <dt><code><em>from_item</em></code></dt>
-  <dd>
-    Specifies a source of input for the query. It can be one of the following:
-    <ol>
-      <li>Table relation</li>
-      <li><a href="sql-ref-syntax-qry-select-join.html">Join relation</a></li>
-      <li><a href="sql-ref-syntax-qry-select-tvf.html">Table-value function</a></li>
-      <li><a href="sql-ref-syntax-qry-select-inline-table.html">Inline table</a></li>
-      <li>Subquery</li>
-    </ol>
-  </dd>
-  <dt><code><em>WHERE</em></code></dt>
-  <dd>
-    Filters the result of the FROM clause based on the supplied predicates.
-  </dd>
-  <dt><code><em>GROUP BY</em></code></dt>
-  <dd>
-    Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
-    (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
-    When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
-  </dd>
-  <dt><code><em>HAVING</em></code></dt>
-  <dd>
-    Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
-    filter rows after the grouping is performed. If HAVING is specified without GROUP BY, it indicates a GROUP BY
-    without grouping expressions (global aggregate).
-  </dd>
-  <dt><code><em>ORDER BY</em></code></dt>
-  <dd>
-    Specifies an ordering of the rows of the complete result set of the query. The output rows are ordered
-    across the partitions. This parameter is mutually exclusive with <code>SORT BY</code>,
-    <code>CLUSTER BY</code> and <code>DISTRIBUTE BY</code> and can not be specified together.
-  </dd>
-  <dt><code><em>SORT BY</em></code></dt>
-  <dd>
-    Specifies an ordering by which the rows are ordered within each partition. This parameter is mutually
-    exclusive with <code>ORDER BY</code> and <code>CLUSTER BY</code> and can not be specified together.
-  </dd>
-  <dt><code><em>CLUSTER BY</em></code></dt>
-  <dd>
-    Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
-    the same effect of using <code>DISTRIBUTE BY</code> and <code>SORT BY</code> together.
-  </dd>
-  <dt><code><em>DISTRIBUTE BY</em></code></dt>
-  <dd>
-    Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually
-    exclusive with <code>ORDER BY</code> and <code>CLUSTER BY</code> and can not be specified together.
-  </dd>
-  <dt><code><em>LIMIT</em></code></dt>
-  <dd>
-    Specifies the maximum number of rows that can be returned by a statement or subquery. This clause
-    is mostly used in the conjunction with <code>ORDER BY</code> to produce a deterministic result.
-  </dd>
-  <dt><code><em>boolean_expression</em></code></dt>
-  <dd>
-    Specifies an expression with a return type of boolean.
-  </dd>
-  <dt><code><em>expression</em></code></dt>
-  <dd>
-    Specifies a combination of one or more values, operators, and SQL functions that evaluates to a value.
-  </dd>
-  <dt><code><em>named_window</em></code></dt>
-  <dd>
-    Specifies aliases for one or more source window specifications. The source window specifications can
-    be referenced in the widow definitions in the query.
-  </dd>
-</dl>
+
+* **named_expression**
+
+    An expression with an assigned name. In general, it denotes a column expression.
+
+    **Syntax:** `expression [AS] [alias]`
+
+ * **from_item**
+
+     Specifies a source of input for the query. It can be one of the following:
+     * Table relation
+     * [Join relation](sql-ref-syntax-qry-select-join.html)
+     * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
+     * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
+     * Subquery
+
+
+ * **WHERE**
+
+     Filters the result of the FROM clause based on the supplied predicates.
+
+ * **GROUP BY**
+
+     Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
+     (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
+     When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
+
+ * **HAVING**
+
+     Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
+     filter rows after the grouping is performed. If HAVING is specified without GROUP BY, it indicates a GROUP BY
+     without grouping expressions (global aggregate).
+
+ * **ORDER BY**
+
+     Specifies an ordering of the rows of the complete result set of the query. The output rows are ordered
+     across the partitions. This parameter is mutually exclusive with `SORT BY`,
+     `CLUSTER BY` and `DISTRIBUTE BY` and can not be specified together.
+
+ * **SORT BY**
+
+     Specifies an ordering by which the rows are ordered within each partition. This parameter is mutually
+     exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
+
+ * **CLUSTER BY**
+
+     Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
+     the same effect of using `DISTRIBUTE BY` and `SORT BY` together.
+
+ * **DISTRIBUTE BY**
+
+     Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually
+     exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
+
+ * **LIMIT**
+
+     Specifies the maximum number of rows that can be returned by a statement or subquery. This clause
+     is mostly used in the conjunction with `ORDER BY` to produce a deterministic result.
+
+ * **boolean_expression**
+
+     Specifies an expression with a return type of boolean.
+
+ * **expression**
+
+     Specifies a combination of one or more values, operators, and SQL functions that evaluates to a value.
+
+ * **named_window**
+
+      Specifies aliases for one or more source window specifications. The source window specifications can
+      be referenced in the widow definitions in the query.
 
 ### Related Statements
 
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
- * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
- * [JOIN](sql-ref-syntax-qry-select-join.html)
- * [SET Operators](sql-ref-syntax-qry-select-setops.html)
- * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+* [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+* [JOIN](sql-ref-syntax-qry-select-join.html)
+* [Join Hints](sql-ref-syntax-qry-select-hints.html)
+* [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+* [Set Operators](sql-ref-syntax-qry-select-setops.html)
+* [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+* [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+* [Window Function](sql-ref-syntax-qry-window.html)
diff --git a/docs/sql-ref-syntax-qry-window.md b/docs/sql-ref-syntax-qry-window.md
index e3762925760e2..a1c2b18b04fce 100644
--- a/docs/sql-ref-syntax-qry-window.md
+++ b/docs/sql-ref-syntax-qry-window.md
@@ -25,67 +25,52 @@ Window functions operate on a group of rows, referred to as a window, and calcul
 
 ### Syntax
 
-{% highlight sql %}
+```sql
 window_function OVER
 ( [  { PARTITION | DISTRIBUTE } BY partition_col_name = partition_col_val ( [ , ... ] ) ]
   { ORDER | SORT } BY expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ]
   [ window_frame ] )
-{% endhighlight %}
+```
 
 ### Parameters
 
-<dl>
-  <dt><code><em>window_function</em></code></dt>
-  <dd>
-    <ul>
-      <li>Ranking Functions</li>
-      <br>
-      <b>Syntax:</b>
-        <code>
-          RANK | DENSE_RANK | PERCENT_RANK | NTILE | ROW_NUMBER
-        </code>
-    </ul>
-    <ul>
-      <li>Analytic Functions</li>
-      <br>
-      <b>Syntax:</b>
-        <code>
-          CUME_DIST | LAG | LEAD
-        </code>
-    </ul>
-    <ul>
-      <li>Aggregate Functions</li>
-      <br>
-      <b>Syntax:</b>
-        <code>
-          MAX | MIN | COUNT | SUM | AVG | ...
-        </code>
-        <br>
-        Please refer to the <a href="api/sql/">Built-in Functions</a> document for a complete list of Spark aggregate functions.
-    </ul>
-  </dd>
-</dl>
-<dl>
-  <dt><code><em>window_frame</em></code></dt>
-  <dd>
-    Specifies which row to start the window on and where to end it.<br>
-    <b>Syntax:</b><br>
-      <code>{ RANGE | ROWS } { frame_start | BETWEEN frame_start AND frame_end }</code><br>
-      If frame_end is omitted it defaults to CURRENT ROW.<br><br>
-      <ul>
-      <code>frame_start</code> and <code>frame_end</code> have the following syntax<br>
-      <b>Syntax:</b><br>
-        <code>
-          UNBOUNDED PRECEDING | offset PRECEDING | CURRENT ROW | offset FOLLOWING | UNBOUNDED FOLLOWING
-        </code><br>
-        <code>offset:</code>specifies the <code>offset</code> from the position of the current row.
-      </ul>
-  </dd>
-</dl>
+* **window_function**
+
+    * Ranking Functions
+
+      **Syntax:** `RANK | DENSE_RANK | PERCENT_RANK | NTILE | ROW_NUMBER`
+
+    * Analytic Functions
+
+      **Syntax:** `CUME_DIST | LAG | LEAD`
+
+    * Aggregate Functions
+
+      **Syntax:** `MAX | MIN | COUNT | SUM | AVG | ...`
+
+      Please refer to the [Built-in Aggregation Functions](sql-ref-functions-builtin.html#aggregate-functions) document for a complete list of Spark aggregate functions.
+
+* **window_frame**
+
+    Specifies which row to start the window on and where to end it.
+
+    **Syntax:**
+
+    `{ RANGE | ROWS } { frame_start | BETWEEN frame_start AND frame_end }`
+
+    * `frame_start` and `frame_end` have the following syntax:
+
+      **Syntax:**
+
+      `UNBOUNDED PRECEDING | offset PRECEDING | CURRENT ROW | offset FOLLOWING | UNBOUNDED FOLLOWING`
+
+      `offset:` specifies the `offset` from the position of the current row.
+
+    **Note:** If `frame_end` is omitted it defaults to `CURRENT ROW`.
 
 ### Examples
 
-{% highlight sql %}
+```sql
 CREATE TABLE employees (name STRING, dept STRING, salary INT, age INT);
 
 INSERT INTO employees VALUES ("Lisa", "Sales", 10000, 35);
@@ -199,8 +184,8 @@ SELECT name, salary,
 | Jane|  Marketing| 29000|29000|35000|
 | Jeff|  Marketing| 35000|29000|    0|
 +-----+-----------+------+-----+-----+
-{% endhighlight %}
+```
 
 ### Related Statements
 
-  * [SELECT](sql-ref-syntax-qry-select.html)
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 325c9b69f12f9..1171fead55e30 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -27,20 +27,21 @@ to SELECT are also included in this section. Spark also provides the
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
- * [JOIN](sql-ref-syntax-qry-select-join.html)
- * [Join Hints](sql-ref-syntax-qry-select-hints.html)
- * [Set Operators](sql-ref-syntax-qry-select-setops.html)
- * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
- * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
- * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
- * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
- * [Window Function](sql-ref-syntax-qry-window.html)
- * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+* [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+* [JOIN](sql-ref-syntax-qry-select-join.html)
+* [Join Hints](sql-ref-syntax-qry-select-hints.html)
+* [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+* [Set Operators](sql-ref-syntax-qry-select-setops.html)
+* [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+* [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+* [Window Function](sql-ref-syntax-qry-window.html)
+* [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)

From 6f7c71947073f147bc35da196139d5ceb6fbdf45 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 10 May 2020 14:22:12 -0500
Subject: [PATCH 0652/1280] [SPARK-31669][SQL][TESTS] Fix RowEncoderSuite
 failures on non-existing dates/timestamps

### What changes were proposed in this pull request?
Shift non-existing dates in Proleptic Gregorian calendar by 1 day. The reason for that is `RowEncoderSuite` generates random dates/timestamps in the hybrid calendar, and some dates/timestamps don't exist in Proleptic Gregorian calendar like 1000-02-29 because 1000 is not leap year in Proleptic Gregorian calendar.

### Why are the changes needed?
This makes RowEncoderSuite much stable.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running RowEncoderSuite and set non-existing date manually:
```scala
val date = new java.sql.Date(1000 - 1900, 1, 29)
Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
```

Closes #28486 from MaxGekk/fix-RowEncoderSuite.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 9f768fa9916dec3cc695e3f28ec77148d81d335f)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../spark/sql/RandomDataGenerator.scala       | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index a7c20c34d78bc..5a4d23debced5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -18,9 +18,10 @@
 package org.apache.spark.sql
 
 import java.math.MathContext
+import java.sql.{Date, Timestamp}
 
 import scala.collection.mutable
-import scala.util.Random
+import scala.util.{Random, Try}
 
 import org.apache.spark.sql.catalyst.CatalystTypeConverters
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
@@ -172,7 +173,15 @@ object RandomDataGenerator {
               // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
               milliseconds = rand.nextLong() % 253402329599999L
             }
-            DateTimeUtils.toJavaDate((milliseconds / MILLIS_PER_DAY).toInt)
+            val date = DateTimeUtils.toJavaDate((milliseconds / MILLIS_PER_DAY).toInt)
+            // The generated `date` is based on the hybrid calendar Julian + Gregorian since
+            // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+            // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
+            // a local date in Proleptic Gregorian calendar to satisfy this requirement.
+            // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+            // As the consequence of that, 29 February of such years might not exist in Proleptic
+            // Gregorian calendar. When this happens, we shift the date by one day.
+            Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
           }
         Some(generator)
       case TimestampType =>
@@ -188,7 +197,15 @@ object RandomDataGenerator {
               milliseconds = rand.nextLong() % 253402329599999L
             }
             // DateTimeUtils.toJavaTimestamp takes microsecond.
-            DateTimeUtils.toJavaTimestamp(milliseconds * 1000)
+            val ts = DateTimeUtils.toJavaTimestamp(milliseconds * 1000)
+            // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
+            // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+            // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
+            // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement.
+            // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+            // As the consequence of that, 29 February of such years might not exist in Proleptic
+            // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
+            Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
           }
         Some(generator)
       case CalendarIntervalType => Some(() => {

From 5c6a4fc8a71fcca9110c8c18ebd44d935514fcc1 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 11 May 2020 04:58:08 +0000
Subject: [PATCH 0653/1280] [SPARK-31672][SQL] Fix loading of timestamps before
 1582-10-15 from dictionary encoded Parquet columns

Modified the `decodeDictionaryIds()` method of `VectorizedColumnReader` to handle especially `TimestampType` when the passed parameter `rebaseDateTime` is true. In that case, decoded milliseconds/microseconds are rebased from the hybrid calendar to Proleptic Gregorian calendar using `RebaseDateTime`.`rebaseJulianToGregorianMicros()`.

This fixes the bug of loading timestamps before the cutover day from dictionary encoded column in parquet files. The code below forces dictionary encoding:
```scala
spark.conf.set("spark.sql.legacy.parquet.rebaseDateTimeInWrite.enabled", true)
scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MICROS")
scala>
Seq.tabulate(8)(_ => "1001-01-01 01:02:03.123").toDF("tsS")
  .select($"tsS".cast("timestamp").as("ts")).repartition(1)
  .write
  .option("parquet.enable.dictionary", true)
  .parquet(path)
```
Load the dates back:
```scala
scala> spark.read.parquet(path).show(false)
+-----------------------+
|ts                     |
+-----------------------+
|1001-01-07 00:32:20.123|
...
|1001-01-07 00:32:20.123|
+-----------------------+
```
Expected values **must be 1001-01-01 01:02:03.123** but not 1001-01-07 00:32:20.123.

Yes. After the changes:
```scala
scala> spark.read.parquet(path).show(false)
+-----------------------+
|ts                     |
+-----------------------+
|1001-01-01 01:02:03.123|
...
|1001-01-01 01:02:03.123|
+-----------------------+
```

Modified the test `SPARK-31159: rebasing timestamps in write` in `ParquetIOSuite` to checked reading dictionary encoded dates.

Closes #28489 from MaxGekk/fix-ts-rebase-parquet-dict-enc.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5d5866be12259c40972f7404f64d830cab87401f)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../parquet/VectorizedColumnReader.java       | 31 +++++++--
 .../datasources/parquet/ParquetIOSuite.scala  | 65 +++++++++++--------
 2 files changed, 64 insertions(+), 32 deletions(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 03056f51be64f..11ce11dd7219c 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -159,7 +159,11 @@ private boolean isLazyDecodingSupported(PrimitiveType.PrimitiveTypeName typeName
         isSupported = originalType != OriginalType.DATE || !rebaseDateTime;
         break;
       case INT64:
-        isSupported = originalType != OriginalType.TIMESTAMP_MILLIS;
+        if (originalType == OriginalType.TIMESTAMP_MICROS) {
+          isSupported = !rebaseDateTime;
+        } else {
+          isSupported = originalType != OriginalType.TIMESTAMP_MILLIS;
+        }
         break;
       case FLOAT:
       case DOUBLE:
@@ -313,17 +317,36 @@ private void decodeDictionaryIds(
       case INT64:
         if (column.dataType() == DataTypes.LongType ||
             DecimalType.is64BitDecimalType(column.dataType()) ||
-            originalType == OriginalType.TIMESTAMP_MICROS) {
+            (originalType == OriginalType.TIMESTAMP_MICROS && !rebaseDateTime)) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putLong(i, dictionary.decodeToLong(dictionaryIds.getDictId(i)));
             }
           }
         } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
+          if (rebaseDateTime) {
+            for (int i = rowId; i < rowId + num; ++i) {
+              if (!column.isNullAt(i)) {
+                long julianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+                long julianMicros = DateTimeUtils.fromMillis(julianMillis);
+                long gregorianMicros = RebaseDateTime.rebaseJulianToGregorianMicros(julianMicros);
+                column.putLong(i, gregorianMicros);
+              }
+            }
+          } else {
+            for (int i = rowId; i < rowId + num; ++i) {
+              if (!column.isNullAt(i)) {
+                long gregorianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+                column.putLong(i, DateTimeUtils.fromMillis(gregorianMillis));
+              }
+            }
+          }
+        } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
-              column.putLong(i,
-                DateTimeUtils.fromMillis(dictionary.decodeToLong(dictionaryIds.getDictId(i))));
+              long julianMicros = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+              long gregorianMicros = RebaseDateTime.rebaseJulianToGregorianMicros(julianMicros);
+              column.putLong(i, gregorianMicros);
             }
           }
         } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 198b68bd86ecc..cf2c7c8b2f516 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -937,37 +937,46 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   }
 
   test("SPARK-31159: rebasing timestamps in write") {
-    Seq(
-      ("TIMESTAMP_MILLIS", "1001-01-01 01:02:03.123", "1001-01-07 01:09:05.123"),
-      ("TIMESTAMP_MICROS", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456"),
-      ("INT96", "1001-01-01 01:02:03.123456", "1001-01-01 01:02:03.123456")
-    ).foreach { case (outType, tsStr, nonRebased) =>
-      withClue(s"output type $outType") {
-        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
-          withTempPath { dir =>
-            val path = dir.getAbsolutePath
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
-              Seq(tsStr).toDF("tsS")
-                .select($"tsS".cast("timestamp").as("ts"))
-                .write
-                .parquet(path)
-            }
+    val N = 8
+    Seq(false, true).foreach { dictionaryEncoding =>
+      Seq(
+        ("TIMESTAMP_MILLIS", "1001-01-01 01:02:03.123", "1001-01-07 01:09:05.123"),
+        ("TIMESTAMP_MICROS", "1001-01-01 01:02:03.123456", "1001-01-07 01:09:05.123456"),
+        ("INT96", "1001-01-01 01:02:03.123456", "1001-01-01 01:02:03.123456")
+      ).foreach { case (outType, tsStr, nonRebased) =>
+        withClue(s"output type $outType") {
+          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
+            withTempPath { dir =>
+              val path = dir.getAbsolutePath
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+                Seq.tabulate(N)(_ => tsStr).toDF("tsS")
+                  .select($"tsS".cast("timestamp").as("ts"))
+                  .repartition(1)
+                  .write
+                  .option("parquet.enable.dictionary", dictionaryEncoding)
+                  .parquet(path)
+              }
 
-            Seq(false, true).foreach { vectorized =>
-              withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-                // The file metadata indicates if it needs rebase or not, so we can always get the
-                // correct result regardless of the "rebaseInRead" config.
-                Seq(true, false).foreach { rebase =>
-                  withSQLConf(
-                    SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
-                    checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(tsStr)))
+              Seq(false, true).foreach { vectorized =>
+                withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
+                  // The file metadata indicates if it needs rebase or not, so we can always get the
+                  // correct result regardless of the "rebaseInRead" config.
+                  Seq(true, false).foreach { rebase =>
+                    withSQLConf(
+                      SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                      checkAnswer(
+                        spark.read.parquet(path),
+                        Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
+                    }
                   }
-                }
 
-                // Force to not rebase to prove the written datetime values are rebased
-                // and we will get wrong result if we don't rebase while reading.
-                withSQLConf("spark.test.forceNoRebase" -> "true") {
-                  checkAnswer(spark.read.parquet(path), Row(Timestamp.valueOf(nonRebased)))
+                  // Force to not rebase to prove the written datetime values are rebased
+                  // and we will get wrong result if we don't rebase while reading.
+                  withSQLConf("spark.test.forceNoRebase" -> "true") {
+                    checkAnswer(
+                      spark.read.parquet(path),
+                      Seq.tabulate(N)(_ => Row(Timestamp.valueOf(nonRebased))))
+                  }
                 }
               }
             }

From e2bf140c68ef38216167e0872b964c3964ca0d9f Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sun, 10 May 2020 22:32:26 -0700
Subject: [PATCH 0654/1280] [SPARK-31674][CORE][DOCS] Make Prometheus metric
 endpoints experimental

### What changes were proposed in this pull request?

This PR aims to new Prometheus-format metric endpoints experimental in Apache Spark 3.0.0.

### Why are the changes needed?

Although the new metrics are disabled by default, we had better make it experimental explicitly in Apache Spark 3.0.0 since the output format is still not fixed. We can finalize it in Apache Spark 3.1.0.

### Does this PR introduce _any_ user-facing change?

Only doc-change is visible to the users.

### How was this patch tested?

Manually check the code since this is a documentation and class annotation change.

Closes #28495 from dongjoon-hyun/SPARK-31674.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b80309bdb4d26556bd3da6a61cac464cdbdd1fe1)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/metrics/sink/PrometheusServlet.scala     | 3 +++
 .../org/apache/spark/status/api/v1/PrometheusResource.scala   | 3 +++
 docs/monitoring.md                                            | 4 ++--
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
index 7c33bce78378d..011c7bc3f0051 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
@@ -24,15 +24,18 @@ import com.codahale.metrics.MetricRegistry
 import org.eclipse.jetty.servlet.ServletContextHandler
 
 import org.apache.spark.{SecurityManager, SparkConf}
+import org.apache.spark.annotation.Experimental
 import org.apache.spark.ui.JettyUtils._
 
 /**
+ * :: Experimental ::
  * This exposes the metrics of the given registry with Prometheus format.
  *
  * The output is consistent with /metrics/json result in terms of item ordering
  * and with the previous result of Spark JMX Sink + Prometheus JMX Converter combination
  * in terms of key string format.
  */
+@Experimental
 private[spark] class PrometheusServlet(
     val property: Properties,
     val registry: MetricRegistry,
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
index f9fb78e65a3d9..2a5f151d99c67 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
@@ -23,15 +23,18 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.glassfish.jersey.server.ServerProperties
 import org.glassfish.jersey.servlet.ServletContainer
 
+import org.apache.spark.annotation.Experimental
 import org.apache.spark.ui.SparkUI
 
 /**
+ * :: Experimental ::
  * This aims to expose Executor metrics like REST API which is documented in
  *
  *    https://spark.apache.org/docs/3.0.0/monitoring.html#executor-metrics
  *
  * Note that this is based on ExecutorSummary which is different from ExecutorSource.
  */
+@Experimental
 @Path("/executors")
 private[v1] class PrometheusResource extends ApiRequestContext {
   @GET
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 7e41c9df4efa0..4da0f8e9d71e9 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -715,7 +715,7 @@ A list of the available metrics, with a short description:
 Executor-level metrics are sent from each executor to the driver as part of the Heartbeat to describe the performance metrics of Executor itself like JVM heap memory, GC information.
 Executor metric values and their measured memory peak values per executor are exposed via the REST API in JSON format and in Prometheus format.
 The JSON end point is exposed at: `/applications/[app-id]/executors`, and the Prometheus endpoint at: `/metrics/executors/prometheus`.
-The Prometheus endpoint is conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
+The Prometheus endpoint is experimental and conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
 In addition, aggregated per-stage peak values of the executor memory metrics are written to the event log if
 `spark.eventLog.logStageExecutorMetrics` is true.  
 Executor memory metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
@@ -963,7 +963,7 @@ Each instance can report to zero or more _sinks_. Sinks are contained in the
 * `CSVSink`: Exports metrics data to CSV files at regular intervals.
 * `JmxSink`: Registers metrics for viewing in a JMX console.
 * `MetricsServlet`: Adds a servlet within the existing Spark UI to serve metrics data as JSON data.
-* `PrometheusServlet`: Adds a servlet within the existing Spark UI to serve metrics data in Prometheus format.
+* `PrometheusServlet`: (Experimental) Adds a servlet within the existing Spark UI to serve metrics data in Prometheus format.
 * `GraphiteSink`: Sends metrics to a Graphite node.
 * `Slf4jSink`: Sends metrics to slf4j as log entries.
 * `StatsdSink`: Sends metrics to a StatsD node.

From 7b567e4f1414e6a3def73f0d3b2f8b058ae58d43 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 11 May 2020 12:59:41 +0000
Subject: [PATCH 0655/1280] [SPARK-31665][SQL][TESTS] Check parquet dictionary
 encoding of random dates/timestamps

### What changes were proposed in this pull request?
Modified `RandomDataGenerator.forType` for DateType and TimestampType to generate special date//timestamp values with 0.5 probability. This will trigger dictionary encoding in Parquet datasource test  HadoopFsRelationTest "test all data types". Currently, dictionary encoding is tested only for numeric types like ShortType.

### Why are the changes needed?
To extend test coverage. Currently, probability of testing of dictionary encoding in the test HadoopFsRelationTest "test all data types" for DateType and TimestampType is close to zero because dates/timestamps are uniformly distributed in wide range, and the chance of generating the same values is pretty low. In this way, parquet datasource cannot apply dictionary encoding for such column types.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `ParquetHadoopFsRelationSuite` and `JsonHadoopFsRelationSuite`.

Closes #28481 from MaxGekk/test-random-parquet-dict-enc.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 32a5398b659695c338cd002d9094bdf19a89a716)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/RandomDataGenerator.scala       | 100 ++++++++++--------
 .../sql/sources/HadoopFsRelationTest.scala    |   2 +-
 2 files changed, 58 insertions(+), 44 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index 5a4d23debced5..cf8d772d61590 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -162,52 +162,66 @@ object RandomDataGenerator {
       })
       case BooleanType => Some(() => rand.nextBoolean())
       case DateType =>
-        val generator =
-          () => {
-            var milliseconds = rand.nextLong() % 253402329599999L
-            // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
-            // for "0001-01-01 00:00:00.000000". We need to find a
-            // number that is greater or equals to this number as a valid timestamp value.
-            while (milliseconds < -62135740800000L) {
-              // 253402329599999L is the number of milliseconds since
-              // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
-              milliseconds = rand.nextLong() % 253402329599999L
-            }
-            val date = DateTimeUtils.toJavaDate((milliseconds / MILLIS_PER_DAY).toInt)
-            // The generated `date` is based on the hybrid calendar Julian + Gregorian since
-            // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
-            // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
-            // a local date in Proleptic Gregorian calendar to satisfy this requirement.
-            // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
-            // As the consequence of that, 29 February of such years might not exist in Proleptic
-            // Gregorian calendar. When this happens, we shift the date by one day.
-            Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
+        def uniformDateRand(rand: Random): java.sql.Date = {
+          var milliseconds = rand.nextLong() % 253402329599999L
+          // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
+          // for "0001-01-01 00:00:00.000000". We need to find a
+          // number that is greater or equals to this number as a valid timestamp value.
+          while (milliseconds < -62135740800000L) {
+            // 253402329599999L is the number of milliseconds since
+            // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
+            milliseconds = rand.nextLong() % 253402329599999L
           }
-        Some(generator)
+          val date = DateTimeUtils.toJavaDate((milliseconds / MILLIS_PER_DAY).toInt)
+          // The generated `date` is based on the hybrid calendar Julian + Gregorian since
+          // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+          // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
+          // a local date in Proleptic Gregorian calendar to satisfy this requirement.
+          // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+          // As the consequence of that, 29 February of such years might not exist in Proleptic
+          // Gregorian calendar. When this happens, we shift the date by one day.
+          Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
+        }
+        randomNumeric[java.sql.Date](
+          rand,
+          uniformDateRand,
+          Seq(
+            "0001-01-01", // the fist day of Common Era
+            "1582-10-15", // the cutover date from Julian to Gregorian calendar
+            "1970-01-01", // the epoch date
+            "9999-12-31"  // the last supported date according to SQL standard
+          ).map(java.sql.Date.valueOf))
       case TimestampType =>
-        val generator =
-          () => {
-            var milliseconds = rand.nextLong() % 253402329599999L
-            // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
-            // for "0001-01-01 00:00:00.000000". We need to find a
-            // number that is greater or equals to this number as a valid timestamp value.
-            while (milliseconds < -62135740800000L) {
-              // 253402329599999L is the number of milliseconds since
-              // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
-              milliseconds = rand.nextLong() % 253402329599999L
-            }
-            // DateTimeUtils.toJavaTimestamp takes microsecond.
-            val ts = DateTimeUtils.toJavaTimestamp(milliseconds * 1000)
-            // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
-            // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
-            // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
-            // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement.
-            // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
-            // As the consequence of that, 29 February of such years might not exist in Proleptic
-            // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
-            Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
+        def uniformTimestampRand(rand: Random): java.sql.Timestamp = {
+          var milliseconds = rand.nextLong() % 253402329599999L
+          // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
+          // for "0001-01-01 00:00:00.000000". We need to find a
+          // number that is greater or equals to this number as a valid timestamp value.
+          while (milliseconds < -62135740800000L) {
+            // 253402329599999L is the number of milliseconds since
+            // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
+            milliseconds = rand.nextLong() % 253402329599999L
           }
-        Some(generator)
+          // DateTimeUtils.toJavaTimestamp takes microsecond.
+          val ts = DateTimeUtils.toJavaTimestamp(milliseconds * 1000)
+          // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
+          // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+          // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
+          // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement.
+          // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+          // As the consequence of that, 29 February of such years might not exist in Proleptic
+          // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
+          Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
+        }
+        randomNumeric[java.sql.Timestamp](
+          rand,
+          uniformTimestampRand,
+          Seq(
+            "0001-01-01 00:00:00", // the fist timestamp of Common Era
+            "1582-10-15 23:59:59", // the cutover date from Julian to Gregorian calendar
+            "1970-01-01 00:00:00", // the epoch timestamp
+            "9999-12-31 23:59:59.999" // the last supported timestamp according to SQL standard
+          ).map(java.sql.Timestamp.valueOf))
       case CalendarIntervalType => Some(() => {
         val months = rand.nextInt(1000)
         val days = rand.nextInt(10000)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index 4ada5077aec7f..9cf171934a0a2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -159,7 +159,7 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
               .add("index", IntegerType, nullable = false)
               .add("col", dataType, nullable = true)
             val rdd =
-              spark.sparkContext.parallelize((1 to 10).map(i => Row(i, dataGenerator())))
+              spark.sparkContext.parallelize((1 to 20).map(i => Row(i, dataGenerator())))
             val df = spark.createDataFrame(rdd, schema).orderBy("index").coalesce(1)
 
             df.write

From 37c352c592a9b8650dc5d1c5413af7a96f01631b Mon Sep 17 00:00:00 2001
From: oleg <oleg@nexla.com>
Date: Mon, 11 May 2020 13:10:39 -0700
Subject: [PATCH 0656/1280] [SPARK-31456][CORE] Fix shutdown hook priority edge
 cases

### What changes were proposed in this pull request?
Fix application order for shutdown hooks for the priorities of Int.MaxValue, Int.MinValue

### Why are the changes needed?
The bug causes out-of-order execution of shutdown hooks if their priorities were Int.MinValue or Int.MaxValue

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added a test covering the change.

Closes #28494 from oleg-smith/SPARK-31456_shutdown_hook_priority.

Authored-by: oleg <oleg@nexla.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit d7c3e9e53e01011f809b6cb145349ee8a9c5e5f0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/util/ShutdownHookManager.scala   | 4 +---
 core/src/test/scala/org/apache/spark/util/UtilsSuite.scala  | 6 +++++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
index 4f1311224bb95..4db268604a3e9 100644
--- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
+++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala
@@ -209,9 +209,7 @@ private [util] class SparkShutdownHookManager {
 private class SparkShutdownHook(private val priority: Int, hook: () => Unit)
   extends Comparable[SparkShutdownHook] {
 
-  override def compareTo(other: SparkShutdownHook): Int = {
-    other.priority - priority
-  }
+  override def compareTo(other: SparkShutdownHook): Int = other.priority.compareTo(priority)
 
   def run(): Unit = hook()
 
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index f5e438b0f1a52..931eb6b5413f7 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -745,10 +745,14 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
     manager.add(3, () => output += 3)
     manager.add(2, () => output += 2)
     manager.add(4, () => output += 4)
+    manager.add(Int.MinValue, () => output += Int.MinValue)
+    manager.add(Int.MinValue, () => output += Int.MinValue)
+    manager.add(Int.MaxValue, () => output += Int.MaxValue)
+    manager.add(Int.MaxValue, () => output += Int.MaxValue)
     manager.remove(hook1)
 
     manager.runAll()
-    assert(output.toList === List(4, 3, 2))
+    assert(output.toList === List(Int.MaxValue, Int.MaxValue, 4, 3, 2, Int.MinValue, Int.MinValue))
   }
 
   test("isInDirectory") {

From 7e226a25efeaf083c95f04ee0d9c3a6e5b6d763d Mon Sep 17 00:00:00 2001
From: fan31415 <fan12356789@gmail.com>
Date: Mon, 11 May 2020 18:23:23 -0500
Subject: [PATCH 0657/1280] [SPARK-31671][ML] Wrong error message in
 VectorAssembler

### What changes were proposed in this pull request?
When input column lengths can not be inferred and handleInvalid = "keep",  VectorAssembler will throw a runtime exception. However the error message with this exception is not consistent. I change the content of this error message to make it work properly.

### Why are the changes needed?
This is a bug. Here is a simple example to reproduce it.

```
// create a df without vector size
val df = Seq(
  (Vectors.dense(1.0), Vectors.dense(2.0))
).toDF("n1", "n2")

// only set vector size hint for n1 column
val hintedDf = new VectorSizeHint()
  .setInputCol("n1")
  .setSize(1)
  .transform(df)

// assemble n1, n2
val output = new VectorAssembler()
  .setInputCols(Array("n1", "n2"))
  .setOutputCol("features")
  .setHandleInvalid("keep")
  .transform(hintedDf)

// because only n1 has vector size, the error message should tell us to set vector size for n2 too
output.show()
```

Expected error message:

```
Can not infer column lengths with handleInvalid = "keep". Consider using VectorSizeHint to add metadata for columns: [n2].
```

Actual error message:

```
Can not infer column lengths with handleInvalid = "keep". Consider using VectorSizeHint to add metadata for columns: [n1, n2].
```

This introduce difficulties when I try to resolve this exception, for I do not know which column required vectorSizeHint. This is especially troublesome when you have a large number of columns to deal with.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Add test in VectorAssemblerSuite.

Closes #28487 from fan31415/SPARK-31671.

Lead-authored-by: fan31415 <fan12356789@gmail.com>
Co-authored-by: yijiefan <fanyije@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 64fb358a994d3fff651a742fa067c194b7455853)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../org/apache/spark/ml/feature/VectorAssembler.scala |  2 +-
 .../spark/ml/feature/VectorAssemblerSuite.scala       | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
index 30700122665de..7bc5e56aaebf2 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
@@ -233,7 +233,7 @@ object VectorAssembler extends DefaultParamsReadable[VectorAssembler] {
         getVectorLengthsFromFirstRow(dataset.na.drop(missingColumns), missingColumns)
       case (true, VectorAssembler.KEEP_INVALID) => throw new RuntimeException(
         s"""Can not infer column lengths with handleInvalid = "keep". Consider using VectorSizeHint
-           |to add metadata for columns: ${columns.mkString("[", ", ", "]")}."""
+           |to add metadata for columns: ${missingColumns.mkString("[", ", ", "]")}."""
           .stripMargin.replaceAll("\n", " "))
       case (_, _) => Map.empty
     }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
index a4d388fd321db..4957f6f1f46a7 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorAssemblerSuite.scala
@@ -261,4 +261,15 @@ class VectorAssemblerSuite
     val output = vectorAssembler.transform(dfWithNullsAndNaNs)
     assert(output.select("a").limit(1).collect().head == Row(Vectors.sparse(0, Seq.empty)))
   }
+
+  test("SPARK-31671: should give explicit error message when can not infer column lengths") {
+    val df = Seq(
+      (Vectors.dense(1.0), Vectors.dense(2.0))
+    ).toDF("n1", "n2")
+    val hintedDf = new VectorSizeHint().setInputCol("n1").setSize(1).transform(df)
+    val assembler = new VectorAssembler()
+      .setInputCols(Array("n1", "n2")).setOutputCol("features")
+    assert(!intercept[RuntimeException](assembler.setHandleInvalid("keep").transform(hintedDf))
+      .getMessage.contains("n1"), "should only show no vector size columns' name")
+  }
 }

From 9469831c3751e898ebe78cb642266b50ea167f22 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Mon, 11 May 2020 17:25:41 -0700
Subject: [PATCH 0658/1280] [SPARK-31559][YARN] Re-obtain tokens at the startup
 of AM for yarn cluster mode if principal and keytab are available

### What changes were proposed in this pull request?

This patch re-obtain tokens at the start of AM for yarn cluster mode, if principal and keytab are available. It basically transfers the credentials from the original user, so this patch puts the new tokens into credentials from the original user via overwriting.

To obtain tokens from providers in user application, this patch leverages the user classloader as context classloader while initializing token manager in the startup of AM.

### Why are the changes needed?

Submitter will obtain delegation tokens for yarn-cluster mode, and add these credentials to the launch context. AM will be launched with these credentials, and AM and driver are able to leverage these tokens.

In Yarn cluster mode, driver is launched in AM, which in turn initializes token manager (while initializing SparkContext) and obtain delegation tokens (+ schedule to renew) if both principal and keytab are available.

That said, even we provide principal and keytab to run application with yarn-cluster mode, AM always starts with initial tokens from launch context until token manager runs and obtains delegation tokens.

So there's a "gap", and if user codes (driver) access to external system with delegation tokens (e.g. HDFS) before initializing SparkContext, it cannot leverage the tokens token manager will obtain. It will make the application fail if AM is killed "after" the initial tokens are expired and relaunched.

This is even a regression: see below codes in branch-2.4:

https://github.com/apache/spark/blob/branch-2.4/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala

https://github.com/apache/spark/blob/branch-2.4/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala

In Spark 2.4.x, AM runs AMCredentialRenewer at initialization, and AMCredentialRenew obtains tokens and merge with credentials being provided with launch context of AM. So it guarantees new tokens in driver run.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Manually tested with specifically crafted application (simple reproducer) - https://github.com/HeartSaVioR/spark-delegation-token-experiment/blob/master/src/main/scala/net/heartsavior/spark/example/LongRunningAppWithHDFSConfig.scala

Before this patch, new AM attempt failed when I killed AM after the expiration of tokens. After this patch the new AM attempt runs fine.

Closes #28336 from HeartSaVioR/SPARK-31559.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Marcelo Vanzin <vanzin@apache.org>
(cherry picked from commit 842b1dcdff0ecab4af9f292c2ff7b2b9ae1ac40a)
Signed-off-by: Marcelo Vanzin <vanzin@apache.org>
---
 .../spark/deploy/yarn/ApplicationMaster.scala     | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 1e8f4084ef9c7..862acd8c03a82 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
 import org.apache.spark._
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.history.HistoryServer
+import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.deploy.yarn.config._
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config._
@@ -860,10 +861,22 @@ object ApplicationMaster extends Logging {
     val ugi = sparkConf.get(PRINCIPAL) match {
       // We only need to log in with the keytab in cluster mode. In client mode, the driver
       // handles the user keytab.
-      case Some(principal) if amArgs.userClass != null =>
+      case Some(principal) if master.isClusterMode =>
         val originalCreds = UserGroupInformation.getCurrentUser().getCredentials()
         SparkHadoopUtil.get.loginUserFromKeytab(principal, sparkConf.get(KEYTAB).orNull)
         val newUGI = UserGroupInformation.getCurrentUser()
+
+        if (master.appAttemptId == null || master.appAttemptId.getAttemptId > 1) {
+          // Re-obtain delegation tokens if this is not a first attempt, as they might be outdated
+          // as of now. Add the fresh tokens on top of the original user's credentials (overwrite).
+          // Set the context class loader so that the token manager has access to jars
+          // distributed by the user.
+          Utils.withContextClassLoader(master.userClassLoader) {
+            val credentialManager = new HadoopDelegationTokenManager(sparkConf, yarnConf, null)
+            credentialManager.obtainDelegationTokens(originalCreds)
+          }
+        }
+
         // Transfer the original user's tokens to the new user, since it may contain needed tokens
         // (such as those user to connect to YARN).
         newUGI.addCredentials(originalCreds)

From cb253b1c5f9b3e6c71adb5d43d8e82514aba00ef Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Tue, 12 May 2020 10:25:04 +0900
Subject: [PATCH 0659/1280] [SPARK-31393][SQL] Show the correct alias in schema
 for expression

### What changes were proposed in this pull request?
Some alias of expression can not display correctly in schema. This PR will fix them.
- `TimeWindow`
- `MaxBy`
- `MinBy`
- `UnaryMinus`
- `BitwiseCount`

This PR also fix a typo issue, please look at https://github.com/apache/spark/blob/b7cde42b04b21c9bfee6535199cf385855c15853/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala#L142

Note:

1. `MaxBy` and `MinBy` extends `MaxMinBy` and the latter add a method `funcName` not needed.  We can reuse `prettyName` to replace `funcName`.
2. Spark SQL exists some function no elegant implementation.For example: `BitwiseCount` override the sql method show below:
`override def sql: String = s"bit_count(${child.sql})"`
I don't think it's elegant enough. Because `Expression` gives the following definitions.
```
  def sql: String = {
    val childrenSQL = children.map(_.sql).mkString(", ")
    s"$prettyName($childrenSQL)"
  }
```
By this definition, `BitwiseCount` should override `prettyName` method.

### Why are the changes needed?
Improve the implement of some expression.

### Does this PR introduce any user-facing change?
 'Yes'. This PR will let user see the correct alias in schema.

### How was this patch tested?
Jenkins test.

Closes #28164 from beliefer/elegant-pretty-name-for-function.

Lead-authored-by: beliefer <beliefer@163.com>
Co-authored-by: gengjiaan <gengjiaan@360.cn>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a89006aba03a623960e5c4c6864ca8c899c81db9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/catalyst/analysis/FunctionRegistry.scala   | 2 +-
 .../spark/sql/catalyst/expressions/TimeWindow.scala      | 1 +
 .../catalyst/expressions/aggregate/MaxByAndMinBy.scala   | 9 +++++----
 .../spark/sql/catalyst/expressions/arithmetic.scala      | 7 ++++++-
 .../sql/catalyst/expressions/bitwiseExpressions.scala    | 4 ++--
 .../resources/sql-functions/sql-expression-schema.md     | 8 ++++----
 .../test/resources/sql-tests/results/operators.sql.out   | 2 +-
 .../org/apache/spark/sql/ExpressionsSchemaSuite.scala    | 2 +-
 8 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index d3b073119e640..c2c0df5f6432c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -253,7 +253,7 @@ object FunctionRegistry {
     expression[Log2]("log2"),
     expression[Log]("ln"),
     expression[Remainder]("mod", true),
-    expression[UnaryMinus]("negative"),
+    expression[UnaryMinus]("negative", true),
     expression[Pi]("pi"),
     expression[Pmod]("pmod"),
     expression[UnaryPositive]("positive"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
index caacb71814f17..82d689477080d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/TimeWindow.scala
@@ -63,6 +63,7 @@ case class TimeWindow(
   override def dataType: DataType = new StructType()
     .add(StructField("start", TimestampType))
     .add(StructField("end", TimestampType))
+  override def prettyName: String = "window"
 
   // This expression is replaced in the analyzer.
   override lazy val resolved = false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
index 2e202240923c3..6d3d3dafe16e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/MaxByAndMinBy.scala
@@ -31,7 +31,6 @@ abstract class MaxMinBy extends DeclarativeAggregate {
   def valueExpr: Expression
   def orderingExpr: Expression
 
-  protected def funcName: String
   // The predicate compares two ordering values.
   protected def predicate(oldExpr: Expression, newExpr: Expression): Expression
   // The arithmetic expression returns greatest/least value of all parameters.
@@ -46,7 +45,7 @@ abstract class MaxMinBy extends DeclarativeAggregate {
   override def dataType: DataType = valueExpr.dataType
 
   override def checkInputDataTypes(): TypeCheckResult =
-    TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $funcName")
+    TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $prettyName")
 
   // The attributes used to keep extremum (max or min) and associated aggregated values.
   private lazy val extremumOrdering =
@@ -101,7 +100,8 @@ abstract class MaxMinBy extends DeclarativeAggregate {
   group = "agg_funcs",
   since = "3.0.0")
 case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
-  override protected def funcName: String = "max_by"
+
+  override def prettyName: String = "max_by"
 
   override protected def predicate(oldExpr: Expression, newExpr: Expression): Expression =
     oldExpr > newExpr
@@ -120,7 +120,8 @@ case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMin
   group = "agg_funcs",
   since = "3.0.0")
 case class MinBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
-  override protected def funcName: String = "min_by"
+
+  override def prettyName: String = "min_by"
 
   override protected def predicate(oldExpr: Expression, newExpr: Expression): Expression =
     oldExpr < newExpr
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 6a64819aabb48..354845d5ccd8b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -86,7 +86,12 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
     case _ => numeric.negate(input)
   }
 
-  override def sql: String = s"(- ${child.sql})"
+  override def sql: String = {
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("-") match {
+      case "-" => s"(- ${child.sql})"
+      case funcName => s"$funcName(${child.sql})"
+    }
+  }
 }
 
 @ExpressionDescription(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
index 72a8f7e99729b..7b819db32e425 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala
@@ -172,6 +172,8 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI
 
   override def toString: String = s"bit_count($child)"
 
+  override def prettyName: String = "bit_count"
+
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = child.dataType match {
     case BooleanType => defineCodeGen(ctx, ev, c => s"if ($c) 1 else 0")
     case _ => defineCodeGen(ctx, ev, c => s"java.lang.Long.bitCount($c)")
@@ -184,6 +186,4 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI
     case IntegerType => java.lang.Long.bitCount(input.asInstanceOf[Int])
     case LongType => java.lang.Long.bitCount(input.asInstanceOf[Long])
   }
-
-  override def sql: String = s"bit_count(${child.sql})"
 }
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 2091de2ecfd9b..c3ae2a73160d3 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -1,4 +1,4 @@
-<!-- Automatically generated byExpressionsSchemaSuite -->
+<!-- Automatically generated by ExpressionsSchemaSuite -->
 ## Summary
   - Number of queries: 328
   - Number of expressions that missing example: 34
@@ -273,7 +273,7 @@
 | org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc | SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR, CAST(2015-03-05T09:32:05.359 AS TIMESTAMP)):timestamp> |
 | org.apache.spark.sql.catalyst.expressions.TypeOf | typeof | SELECT typeof(1) | struct<typeof(1):string> |
 | org.apache.spark.sql.catalyst.expressions.UnBase64 | unbase64 | SELECT unbase64('U3BhcmsgU1FM') | struct<unbase64(U3BhcmsgU1FM):binary> |
-| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<(- 1):int> |
+| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<negative(1):int> |
 | org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct<decode(unhex(537061726B2053514C), UTF-8):string> |
 | org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct<unix_timestamp(current_timestamp(), yyyy-MM-dd HH:mm:ss):bigint> |
@@ -312,9 +312,9 @@
 | org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col, false):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col, false):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Max | max | SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col) | struct<max(col):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<maxby(x, y):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<max_by(x, y):string> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Min | min | SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col) | struct<min(col):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<minby(x, y):string> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<min_by(x, y):string> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Percentile | percentile | SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col) | struct<percentile(col, CAST(0.3 AS DOUBLE), 1):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Skewness | skewness | SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<skewness(CAST(col AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop | stddev_pop | SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_pop(CAST(col AS DOUBLE)):double> |
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index cf857cf9f98ad..a94a123b1b8ab 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -437,7 +437,7 @@ struct<abs(-3.13):decimal(3,2),abs(CAST(-2.19 AS DOUBLE)):double>
 -- !query
 select positive('-1.11'), positive(-1.11), negative('-1.11'), negative(-1.11)
 -- !query schema
-struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),(- CAST(-1.11 AS DOUBLE)):double,(- -1.11):decimal(3,2)>
+struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),negative(CAST(-1.11 AS DOUBLE)):double,negative(-1.11):decimal(3,2)>
 -- !query output
 -1.11	-1.11	1.11	1.11
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
index d83aa4ff4818b..d69ecd7d5660f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -142,7 +142,7 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
     }
 
     val header = Seq(
-      s"<!-- Automatically generated by${getClass.getSimpleName} -->",
+      s"<!-- Automatically generated by ${getClass.getSimpleName} -->",
       "## Summary",
       s"  - Number of queries: ${outputs.size}",
       s"  - Number of expressions that missing example: ${missingExamples.size}",

From 2549e38690fd461c7d01518f1c5df2452efa66b5 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 12 May 2020 13:37:13 +0000
Subject: [PATCH 0660/1280] [SPARK-31678][SQL] Print error stack trace for
 Spark SQL CLI when error occurs

### What changes were proposed in this pull request?

When I was finding the root cause for SPARK-31675, I noticed that it was very difficult for me to see what was actually going on, since it output nothing else but only
```sql
Error in query: java.lang.IllegalArgumentException: Wrong FS: blablah/.hive-staging_blahbla, expected: hdfs://cluster1
```

It is really hard for us to find causes through such a simple error message without a certain amount of experience.

In this PR, I propose to print all of the stack traces when AnalysisException occurs if there are underlying root causes, also we can escape this via `-S` option.

### Why are the changes needed?

In SPARK-11188,

>For analysis exceptions in the sql-shell, we should only print the error message to the screen. The stacktrace will never have useful information since this error is used to signify an error with the query.

But nowadays, some `AnalysisException`s do have useful information for us to debug, e.g. the `AnalysisException` below may contain exceptions from hive or Hadoop side.

https://github.com/apache/spark/blob/a28ed86a387b286745b30cd4d90b3d558205a5a7/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala#L97-L112

```scala
	at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:649)
	at org.apache.hadoop.fs.FileSystem.makeQualified(FileSystem.java:468)
	at org.apache.hadoop.hive.common.FileUtils.isSubDir(FileUtils.java:626)
	at org.apache.hadoop.hive.ql.metadata.Hive.replaceFiles(Hive.java:2850)
	at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1398)
	at org.apache.hadoop.hive.ql.metadata.Hive.loadDynamicPartitions(Hive.java:1593)
```

### Does this PR introduce _any_ user-facing change?

Yes, `bin/spark-sql` will print all the stack trace when an AnalysisException which contains root causes occurs, before this fix, only the message will be printed.

#### before

```scala
Error in query: java.lang.IllegalArgumentException: Wrong FS: hdfs:..., expected: hdfs://hz-cluster10;
```

#### After
```scala
Error in query: java.lang.IllegalArgumentException: Wrong FS: hdfs:..., expected: hdfs://hz-cluster10;
org.apache.spark.sql.AnalysisException: java.lang.IllegalArgumentException: Wrong FS: ..., expected: hdfs://hz-cluster10;
	at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:109)
	at org.apache.spark.sql.hive.HiveExternalCatalog.loadPartition(HiveExternalCatalog.scala:890)
	at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.loadPartition(ExternalCatalogWithListener.scala:179)
	at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:312)
	at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:101)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:108)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:106)
	at org.apache.spark.sql.execution.command.DataWritingCommandExec.executeCollect(commands.scala:120)
	at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229)
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3644)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3642)
	at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229)
	at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
	at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
	at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
	at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602)
	at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:650)
	at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:63)
	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:376)
	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.$anonfun$processLine$1(SparkSQLCLIDriver.scala:486)
	at scala.collection.Iterator.foreach(Iterator.scala:941)
	at scala.collection.Iterator.foreach$(Iterator.scala:941)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
	at scala.collection.IterableLike.foreach(IterableLike.scala:74)
	at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processLine(SparkSQLCLIDriver.scala:480)
	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:282)
	at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
	at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:934)
	at org.apache.spark.deploy.SparkSubmit$$anon$1.run(SparkSubmit.scala:165)
	at org.apache.spark.deploy.SparkSubmit$$anon$1.run(SparkSubmit.scala:163)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746)
	at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:163)
	at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
	at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
	at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1013)
	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1022)
	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.lang.IllegalArgumentException: Wrong FS: ..., expected: hdfs://hz-cluster10
	at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:649)
	at org.apache.hadoop.hdfs.DistributedFileSystem.getPathName(DistributedFileSystem.java:194)
	at org.apache.hadoop.hdfs.DistributedFileSystem.getEZForPath(DistributedFileSystem.java:2093)
	at org.apache.hadoop.hdfs.client.HdfsAdmin.getEncryptionZoneForPath(HdfsAdmin.java:289)
	at org.apache.hadoop.hive.shims.Hadoop23Shims$HdfsEncryptionShim.isPathEncrypted(Hadoop23Shims.java:1221)
	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:2607)
	at org.apache.hadoop.hive.ql.metadata.Hive.replaceFiles(Hive.java:2892)
	at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1398)
	at org.apache.hadoop.hive.ql.metadata.Hive.loadPartition(Hive.java:1324)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.spark.sql.hive.client.Shim_v0_14.loadPartition(HiveShim.scala:927)
	at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$loadPartition$1(HiveClientImpl.scala:870)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:294)
	at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:227)
	at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:226)
	at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:276)
	at org.apache.spark.sql.hive.client.HiveClientImpl.loadPartition(HiveClientImpl.scala:860)
	at org.apache.spark.sql.hive.HiveExternalCatalog.$anonfun$loadPartition$1(HiveExternalCatalog.scala:911)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:99)
	... 52 more
```

You can use `-S` option to restore old behavior if you find the error is too verbose.

### How was this patch tested?

Existing CliSuite -
`SPARK-11188 Analysis error reporting`

Add new test and verify manually

Closes #28499 from yaooqinn/SPARK-31678.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ce714d81894a48e2d06c530674c2190e0483e1b4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../hive/thriftserver/SparkSQLCLIDriver.scala  | 16 ++++++++++++----
 .../spark/sql/hive/thriftserver/CliSuite.scala | 18 ++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index bffa24c469601..50b5eb43bd1ad 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -380,10 +380,18 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
 
           ret = rc.getResponseCode
           if (ret != 0) {
-            // For analysis exception, only the error is printed out to the console.
-            rc.getException() match {
-              case e : AnalysisException =>
-                err.println(s"""Error in query: ${e.getMessage}""")
+            rc.getException match {
+              case e: AnalysisException => e.cause match {
+                case Some(_) if !sessionState.getIsSilent =>
+                  err.println(
+                    s"""Error in query: ${e.getMessage}
+                       |${org.apache.hadoop.util.StringUtils.stringifyException(e)}
+                     """.stripMargin)
+                // For analysis exceptions in silent mode or simple ones that only related to the
+                // query itself, such as `NoSuchDatabaseException`, only the error is printed out
+                // to the console.
+                case _ => err.println(s"""Error in query: ${e.getMessage}""")
+              }
               case _ => err.println(rc.getErrorMessage())
             }
             driver.close()
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index 265e7772a691c..bcdcd072eeeca 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -509,4 +509,22 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       "SELECT \"legal 'string b\";select 22222 + 1;".stripMargin -> "22223"
     )
   }
+
+  test("AnalysisException with root cause will be printStacktrace") {
+    // If it is not in silent mode, will print the stacktrace
+    runCliWithin(
+      1.minute,
+      extraArgs = Seq("--hiveconf", "hive.session.silent=false",
+        "-e", "select date_sub(date'2011-11-11', '1.2');"),
+      errorResponses = Seq("NumberFormatException"))(
+      ("", "Error in query: The second argument of 'date_sub' function needs to be an integer."),
+      ("", "NumberFormatException: invalid input syntax for type numeric: 1.2"))
+    // If it is in silent mode, will print the error message only
+    runCliWithin(
+      1.minute,
+      extraArgs = Seq("--conf", "spark.hive.session.silent=true",
+        "-e", "select date_sub(date'2011-11-11', '1.2');"),
+      errorResponses = Seq("AnalysisException"))(
+      ("", "Error in query: The second argument of 'date_sub' function needs to be an integer."))
+  }
 }

From cbe75bb8879ec408088eaf6944284a893bb63c92 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 12 May 2020 14:05:31 +0000
Subject: [PATCH 0661/1280] [SPARK-31680][SQL][TESTS] Support Java 8 datetime
 types by Random data generator

### What changes were proposed in this pull request?
Generates java.time.Instant/java.time.LocalDate for DateType/TimestampType by `RandomDataGenerator.forType` when the SQL config `spark.sql.datetime.java8API.enabled` is set to `true`.

### Why are the changes needed?
To improve test coverage, and check java.time.Instant/java.time.LocalDate types in round trip tests.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running modified test suites `RowEncoderSuite`, `RandomDataGeneratorSuite` and `HadoopFsRelationTest`.

Closes #28502 from MaxGekk/random-java8-datetime.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a3fafddf390fd180047a0b9ef46f052a9b6813e0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/RandomDataGenerator.scala       | 105 +++++++++++-------
 .../spark/sql/RandomDataGeneratorSuite.scala  |  32 +++---
 .../catalyst/encoders/RowEncoderSuite.scala   |  36 +++---
 .../sql/sources/HadoopFsRelationTest.scala    |  75 +++++++------
 4 files changed, 146 insertions(+), 102 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index cf8d772d61590..6a5bdc4f6fc3d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -19,13 +19,15 @@ package org.apache.spark.sql
 
 import java.math.MathContext
 import java.sql.{Date, Timestamp}
+import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
 
 import scala.collection.mutable
 import scala.util.{Random, Try}
 
 import org.apache.spark.sql.catalyst.CatalystTypeConverters
-import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
+import org.apache.spark.sql.catalyst.util.DateTimeConstants.{MICROS_PER_MILLIS, MILLIS_PER_DAY}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
 /**
@@ -162,7 +164,7 @@ object RandomDataGenerator {
       })
       case BooleanType => Some(() => rand.nextBoolean())
       case DateType =>
-        def uniformDateRand(rand: Random): java.sql.Date = {
+        def uniformDaysRand(rand: Random): Int = {
           var milliseconds = rand.nextLong() % 253402329599999L
           // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
           // for "0001-01-01 00:00:00.000000". We need to find a
@@ -172,27 +174,37 @@ object RandomDataGenerator {
             // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
             milliseconds = rand.nextLong() % 253402329599999L
           }
-          val date = DateTimeUtils.toJavaDate((milliseconds / MILLIS_PER_DAY).toInt)
-          // The generated `date` is based on the hybrid calendar Julian + Gregorian since
-          // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
-          // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
-          // a local date in Proleptic Gregorian calendar to satisfy this requirement.
-          // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
-          // As the consequence of that, 29 February of such years might not exist in Proleptic
-          // Gregorian calendar. When this happens, we shift the date by one day.
-          Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
+          (milliseconds / MILLIS_PER_DAY).toInt
+        }
+        val specialDates = Seq(
+          "0001-01-01", // the fist day of Common Era
+          "1582-10-15", // the cutover date from Julian to Gregorian calendar
+          "1970-01-01", // the epoch date
+          "9999-12-31" // the last supported date according to SQL standard
+        )
+        if (SQLConf.get.getConf(SQLConf.DATETIME_JAVA8API_ENABLED)) {
+          randomNumeric[LocalDate](
+            rand,
+            (rand: Random) => LocalDate.ofEpochDay(uniformDaysRand(rand)),
+            specialDates.map(LocalDate.parse))
+        } else {
+          randomNumeric[java.sql.Date](
+            rand,
+            (rand: Random) => {
+              val date = DateTimeUtils.toJavaDate(uniformDaysRand(rand))
+              // The generated `date` is based on the hybrid calendar Julian + Gregorian since
+              // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+              // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `date` to
+              // a local date in Proleptic Gregorian calendar to satisfy this requirement. Some
+              // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+              // As the consequence of that, 29 February of such years might not exist in Proleptic
+              // Gregorian calendar. When this happens, we shift the date by one day.
+              Try { date.toLocalDate; date }.getOrElse(new Date(date.getTime + MILLIS_PER_DAY))
+            },
+            specialDates.map(java.sql.Date.valueOf))
         }
-        randomNumeric[java.sql.Date](
-          rand,
-          uniformDateRand,
-          Seq(
-            "0001-01-01", // the fist day of Common Era
-            "1582-10-15", // the cutover date from Julian to Gregorian calendar
-            "1970-01-01", // the epoch date
-            "9999-12-31"  // the last supported date according to SQL standard
-          ).map(java.sql.Date.valueOf))
       case TimestampType =>
-        def uniformTimestampRand(rand: Random): java.sql.Timestamp = {
+        def uniformMicorsRand(rand: Random): Long = {
           var milliseconds = rand.nextLong() % 253402329599999L
           // -62135740800000L is the number of milliseconds before January 1, 1970, 00:00:00 GMT
           // for "0001-01-01 00:00:00.000000". We need to find a
@@ -202,26 +214,39 @@ object RandomDataGenerator {
             // January 1, 1970, 00:00:00 GMT for "9999-12-31 23:59:59.999999".
             milliseconds = rand.nextLong() % 253402329599999L
           }
-          // DateTimeUtils.toJavaTimestamp takes microsecond.
-          val ts = DateTimeUtils.toJavaTimestamp(milliseconds * 1000)
-          // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
-          // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
-          // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
-          // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement.
-          // Some years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
-          // As the consequence of that, 29 February of such years might not exist in Proleptic
-          // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
-          Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
+          milliseconds * MICROS_PER_MILLIS
+        }
+        val specialTs = Seq(
+          "0001-01-01 00:00:00", // the fist timestamp of Common Era
+          "1582-10-15 23:59:59", // the cutover date from Julian to Gregorian calendar
+          "1970-01-01 00:00:00", // the epoch timestamp
+          "9999-12-31 23:59:59"  // the last supported timestamp according to SQL standard
+        )
+        if (SQLConf.get.getConf(SQLConf.DATETIME_JAVA8API_ENABLED)) {
+          randomNumeric[Instant](
+            rand,
+            (rand: Random) => DateTimeUtils.microsToInstant(uniformMicorsRand(rand)),
+            specialTs.map { s =>
+              val ldt = LocalDateTime.parse(s.replace(" ", "T"))
+              ldt.atZone(ZoneId.systemDefault()).toInstant
+            })
+        } else {
+          randomNumeric[java.sql.Timestamp](
+            rand,
+            (rand: Random) => {
+              // DateTimeUtils.toJavaTimestamp takes microsecond.
+              val ts = DateTimeUtils.toJavaTimestamp(uniformMicorsRand(rand))
+              // The generated `ts` is based on the hybrid calendar Julian + Gregorian since
+              // 1582-10-15 but it should be valid in Proleptic Gregorian calendar too which is used
+              // by Spark SQL since version 3.0 (see SPARK-26651). We try to convert `ts` to
+              // a local timestamp in Proleptic Gregorian calendar to satisfy this requirement. Some
+              // years are leap years in Julian calendar but not in Proleptic Gregorian calendar.
+              // As the consequence of that, 29 February of such years might not exist in Proleptic
+              // Gregorian calendar. When this happens, we shift the timestamp `ts` by one day.
+              Try { ts.toLocalDateTime; ts }.getOrElse(new Timestamp(ts.getTime + MILLIS_PER_DAY))
+            },
+            specialTs.map(java.sql.Timestamp.valueOf))
         }
-        randomNumeric[java.sql.Timestamp](
-          rand,
-          uniformTimestampRand,
-          Seq(
-            "0001-01-01 00:00:00", // the fist timestamp of Common Era
-            "1582-10-15 23:59:59", // the cutover date from Julian to Gregorian calendar
-            "1970-01-01 00:00:00", // the epoch timestamp
-            "9999-12-31 23:59:59.999" // the last supported timestamp according to SQL standard
-          ).map(java.sql.Timestamp.valueOf))
       case CalendarIntervalType => Some(() => {
         val months = rand.nextInt(1000)
         val days = rand.nextInt(10000)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
index 3e62ca069e9ea..cb335e5f435a3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGeneratorSuite.scala
@@ -24,30 +24,36 @@ import scala.util.Random
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.CatalystTypeConverters
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 /**
  * Tests of [[RandomDataGenerator]].
  */
-class RandomDataGeneratorSuite extends SparkFunSuite {
+class RandomDataGeneratorSuite extends SparkFunSuite with SQLHelper {
 
   /**
    * Tests random data generation for the given type by using it to generate random values then
    * converting those values into their Catalyst equivalents using CatalystTypeConverters.
    */
   def testRandomDataGeneration(dataType: DataType, nullable: Boolean = true): Unit = {
-    val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dataType)
-    val generator = RandomDataGenerator.forType(dataType, nullable, new Random(33)).getOrElse {
-      fail(s"Random data generator was not defined for $dataType")
-    }
-    if (nullable) {
-      assert(Iterator.fill(100)(generator()).contains(null))
-    } else {
-      assert(!Iterator.fill(100)(generator()).contains(null))
-    }
-    for (_ <- 1 to 10) {
-      val generatedValue = generator()
-      toCatalyst(generatedValue)
+    Seq(false, true).foreach { java8Api =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+        val toCatalyst = CatalystTypeConverters.createToCatalystConverter(dataType)
+        val generator = RandomDataGenerator.forType(dataType, nullable, new Random(33)).getOrElse {
+          fail(s"Random data generator was not defined for $dataType")
+        }
+        if (nullable) {
+          assert(Iterator.fill(100)(generator()).contains(null))
+        } else {
+          assert(!Iterator.fill(100)(generator()).contains(null))
+        }
+        for (_ <- 1 to 10) {
+          val generatedValue = generator()
+          toCatalyst(generatedValue)
+        }
+      }
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index c1158e001a780..fd24f058f357c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -377,23 +377,27 @@ class RowEncoderSuite extends CodegenInterpretedPlanTest {
 
   private def encodeDecodeTest(schema: StructType): Unit = {
     test(s"encode/decode: ${schema.simpleString}") {
-      val encoder = RowEncoder(schema).resolveAndBind()
-      val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get
-
-      var input: Row = null
-      try {
-        for (_ <- 1 to 5) {
-          input = inputGenerator.apply().asInstanceOf[Row]
-          val convertedBack = roundTrip(encoder, input)
-          assert(input == convertedBack)
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          val encoder = RowEncoder(schema).resolveAndBind()
+          val inputGenerator = RandomDataGenerator.forType(schema, nullable = false).get
+
+          var input: Row = null
+          try {
+            for (_ <- 1 to 5) {
+              input = inputGenerator.apply().asInstanceOf[Row]
+              val convertedBack = roundTrip(encoder, input)
+              assert(input == convertedBack)
+            }
+          } catch {
+            case e: Exception =>
+              fail(
+                s"""
+                   |schema: ${schema.simpleString}
+                   |input: ${input}
+                 """.stripMargin, e)
+          }
         }
-      } catch {
-        case e: Exception =>
-          fail(
-            s"""
-               |schema: ${schema.simpleString}
-               |input: ${input}
-             """.stripMargin, e)
       }
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index 9cf171934a0a2..42b6862907d8d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -145,40 +145,49 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
 
           val seed = System.nanoTime()
           withClue(s"Random data generated with the seed: ${seed}") {
-            val dataGenerator = RandomDataGenerator.forType(
-              dataType = dataType,
-              nullable = true,
-              new Random(seed)
-            ).getOrElse {
-              fail(s"Failed to create data generator for schema $dataType")
+            val java8ApiConfValues = if (dataType == DateType || dataType == TimestampType) {
+              Seq(false, true)
+            } else {
+              Seq(false)
+            }
+            java8ApiConfValues.foreach { java8Api =>
+              withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+                val dataGenerator = RandomDataGenerator.forType(
+                  dataType = dataType,
+                  nullable = true,
+                  new Random(seed)
+                ).getOrElse {
+                  fail(s"Failed to create data generator for schema $dataType")
+                }
+
+                // Create a DF for the schema with random data. The index field is used to sort the
+                // DataFrame.  This is a workaround for SPARK-10591.
+                val schema = new StructType()
+                  .add("index", IntegerType, nullable = false)
+                  .add("col", dataType, nullable = true)
+                val rdd =
+                  spark.sparkContext.parallelize((1 to 20).map(i => Row(i, dataGenerator())))
+                val df = spark.createDataFrame(rdd, schema).orderBy("index").coalesce(1)
+
+                df.write
+                  .mode("overwrite")
+                  .format(dataSourceName)
+                  .option("dataSchema", df.schema.json)
+                  .options(extraOptions)
+                  .save(path)
+
+                val loadedDF = spark
+                  .read
+                  .format(dataSourceName)
+                  .option("dataSchema", df.schema.json)
+                  .schema(df.schema)
+                  .options(extraOptions)
+                  .load(path)
+                  .orderBy("index")
+
+                checkAnswer(loadedDF, df)
+              }
             }
-
-            // Create a DF for the schema with random data. The index field is used to sort the
-            // DataFrame.  This is a workaround for SPARK-10591.
-            val schema = new StructType()
-              .add("index", IntegerType, nullable = false)
-              .add("col", dataType, nullable = true)
-            val rdd =
-              spark.sparkContext.parallelize((1 to 20).map(i => Row(i, dataGenerator())))
-            val df = spark.createDataFrame(rdd, schema).orderBy("index").coalesce(1)
-
-            df.write
-              .mode("overwrite")
-              .format(dataSourceName)
-              .option("dataSchema", df.schema.json)
-              .options(extraOptions)
-              .save(path)
-
-            val loadedDF = spark
-              .read
-              .format(dataSourceName)
-              .option("dataSchema", df.schema.json)
-              .schema(df.schema)
-              .options(extraOptions)
-              .load(path)
-              .orderBy("index")
-
-            checkAnswer(loadedDF, df)
           }
         }
       }

From b50d53b1079ea32c75f9192f27b2b07cdec69641 Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Tue, 12 May 2020 08:54:28 -0700
Subject: [PATCH 0662/1280] [SPARK-31610][SPARK-31668][ML] Address hashingTF
 saving&loading bug and expose hashFunc property in HashingTF

### What changes were proposed in this pull request?
Expose hashFunc property in HashingTF

Some third-party library such as mleap need to access it.
See background description here:
https://github.com/combust/mleap/pull/665#issuecomment-621258623

### Why are the changes needed?
See https://github.com/combust/mleap/pull/665#issuecomment-621258623

### Does this PR introduce any user-facing change?
No. Only add a package private constructor.

### How was this patch tested?
N/A

Closes #28413 from WeichenXu123/hashing_tf_expose_hashfunc.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: Xiangrui Meng <meng@databricks.com>
(cherry picked from commit e248bc7af6086cde7dd89a51459ae6a221a600c8)
Signed-off-by: Xiangrui Meng <meng@databricks.com>
---
 .../apache/spark/ml/feature/HashingTF.scala   | 40 ++++++++++++++-----
 .../spark/ml/feature/HashingTFSuite.scala     |  4 ++
 2 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
index 80bf85936aace..d2bb013448aae 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala
@@ -42,14 +42,17 @@ import org.apache.spark.util.VersionUtils.majorMinorVersion
  * otherwise the features will not be mapped evenly to the columns.
  */
 @Since("1.2.0")
-class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
+class HashingTF @Since("3.0.0") private[ml] (
+    @Since("1.4.0") override val uid: String,
+    @Since("3.1.0") val hashFuncVersion: Int)
   extends Transformer with HasInputCol with HasOutputCol with HasNumFeatures
     with DefaultParamsWritable {
 
-  private var hashFunc: Any => Int = FeatureHasher.murmur3Hash
-
   @Since("1.2.0")
-  def this() = this(Identifiable.randomUID("hashingTF"))
+  def this() = this(Identifiable.randomUID("hashingTF"), HashingTF.SPARK_3_MURMUR3_HASH)
+
+  @Since("1.4.0")
+  def this(uid: String) = this(uid, hashFuncVersion = HashingTF.SPARK_3_MURMUR3_HASH)
 
   /** @group setParam */
   @Since("1.4.0")
@@ -122,7 +125,12 @@ class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
    */
   @Since("3.0.0")
   def indexOf(term: Any): Int = {
-    Utils.nonNegativeMod(hashFunc(term), $(numFeatures))
+    val hashValue = hashFuncVersion match {
+      case HashingTF.SPARK_2_MURMUR3_HASH => OldHashingTF.murmur3Hash(term)
+      case HashingTF.SPARK_3_MURMUR3_HASH => FeatureHasher.murmur3Hash(term)
+      case _ => throw new IllegalArgumentException("Illegal hash function version setting.")
+    }
+    Utils.nonNegativeMod(hashValue, $(numFeatures))
   }
 
   @Since("1.4.1")
@@ -132,27 +140,41 @@ class HashingTF @Since("1.4.0") (@Since("1.4.0") override val uid: String)
   override def toString: String = {
     s"HashingTF: uid=$uid, binary=${$(binary)}, numFeatures=${$(numFeatures)}"
   }
+
+  @Since("3.0.0")
+  override def save(path: String): Unit = {
+    require(hashFuncVersion == HashingTF.SPARK_3_MURMUR3_HASH,
+      "Cannot save model which is loaded from lower version spark saved model. We can address " +
+      "it by (1) use old spark version to save the model, or (2) use new version spark to " +
+      "re-train the pipeline.")
+    super.save(path)
+  }
 }
 
 @Since("1.6.0")
 object HashingTF extends DefaultParamsReadable[HashingTF] {
 
+  private[ml] val SPARK_2_MURMUR3_HASH = 1
+  private[ml] val SPARK_3_MURMUR3_HASH = 2
+
   private class HashingTFReader extends MLReader[HashingTF] {
 
     private val className = classOf[HashingTF].getName
 
     override def load(path: String): HashingTF = {
       val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
-      val hashingTF = new HashingTF(metadata.uid)
-      metadata.getAndSetParams(hashingTF)
 
       // We support loading old `HashingTF` saved by previous Spark versions.
       // Previous `HashingTF` uses `mllib.feature.HashingTF.murmur3Hash`, but new `HashingTF` uses
       // `ml.Feature.FeatureHasher.murmur3Hash`.
       val (majorVersion, _) = majorMinorVersion(metadata.sparkVersion)
-      if (majorVersion < 3) {
-        hashingTF.hashFunc = OldHashingTF.murmur3Hash
+      val hashFuncVersion = if (majorVersion < 3) {
+        SPARK_2_MURMUR3_HASH
+      } else {
+        SPARK_3_MURMUR3_HASH
       }
+      val hashingTF = new HashingTF(metadata.uid, hashFuncVersion = hashFuncVersion)
+      metadata.getAndSetParams(hashingTF)
       hashingTF
     }
   }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
index 722302e5a165f..8fd192fa56500 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
@@ -100,6 +100,10 @@ class HashingTFSuite extends MLTest with DefaultReadWriteTest {
     val metadata = spark.read.json(s"$hashingTFPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
     assert(sparkVersionStr == "2.4.4")
+
+    intercept[IllegalArgumentException] {
+      loadedHashingTF.save(hashingTFPath)
+    }
   }
 
   test("read/write") {

From 512cb2f0246a0d020f0ba726b4596555b15797c6 Mon Sep 17 00:00:00 2001
From: Ali Smesseim <ali.smesseim@databricks.com>
Date: Tue, 12 May 2020 09:14:34 -0700
Subject: [PATCH 0663/1280] [SPARK-31387] Handle unknown operation/session ID
 in HiveThriftServer2Listener

### What changes were proposed in this pull request?

The update methods in HiveThriftServer2Listener now check if the parameter operation/session ID actually exist in the `sessionList` and `executionList` respectively. This prevents NullPointerExceptions if the operation or session ID is unknown. Instead, a warning is written to the log.

Also, in HiveSessionImpl.close(), we catch any exception thrown by `operationManager.closeOperation`. If for any reason this throws an exception, other operations are not prevented from being closed.

### Why are the changes needed?

The listener's update methods would throw an exception if the operation or session ID is unknown. In Spark 2, where the listener is called directly, this hampers with the caller's control flow. In Spark 3, the exception is caught by the ListenerBus but results in an uninformative NullPointerException.

In HiveSessionImpl.close(), if an exception is thrown when closing an operation, all following operations are not closed.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Unit tests

Closes #28155 from alismess-db/hive-thriftserver-listener-update-safer.

Authored-by: Ali Smesseim <ali.smesseim@databricks.com>
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
(cherry picked from commit 6994c64efd5770a8fd33220cbcaddc1d96fed886)
Signed-off-by: gatorsmile <gatorsmile@gmail.com>
---
 .../ui/HiveThriftServer2Listener.scala        | 120 +++++++++++-------
 .../thriftserver/HiveSessionImplSuite.scala   |  73 +++++++++++
 .../ui/HiveThriftServer2ListenerSuite.scala   |  16 +++
 .../service/cli/session/HiveSessionImpl.java  |   6 +-
 .../service/cli/session/HiveSessionImpl.java  |   6 +-
 5 files changed, 170 insertions(+), 51 deletions(-)
 create mode 100644 sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
index 6d0a506fa94dc..20a8f2c833c21 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
@@ -25,6 +25,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.hive.service.server.HiveServer2
 
 import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Status.LIVE_ENTITY_UPDATE_PERIOD
 import org.apache.spark.scheduler._
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.ExecutionState
@@ -38,7 +39,7 @@ private[thriftserver] class HiveThriftServer2Listener(
     kvstore: ElementTrackingStore,
     sparkConf: SparkConf,
     server: Option[HiveServer2],
-    live: Boolean = true) extends SparkListener {
+    live: Boolean = true) extends SparkListener with Logging {
 
   private val sessionList = new ConcurrentHashMap[String, LiveSessionData]()
   private val executionList = new ConcurrentHashMap[String, LiveExecutionData]()
@@ -131,60 +132,81 @@ private[thriftserver] class HiveThriftServer2Listener(
     updateLiveStore(session)
   }
 
-  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit = {
-    val session = sessionList.get(e.sessionId)
-    session.finishTimestamp = e.finishTime
-    updateStoreWithTriggerEnabled(session)
-    sessionList.remove(e.sessionId)
-  }
+  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit =
+    Option(sessionList.get(e.sessionId)) match {
+      case None => logWarning(s"onSessionClosed called with unknown session id: ${e.sessionId}")
+      case Some(sessionData) =>
+        val session = sessionData
+        session.finishTimestamp = e.finishTime
+        updateStoreWithTriggerEnabled(session)
+        sessionList.remove(e.sessionId)
+    }
 
-  private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit = {
-    val info = getOrCreateExecution(
-      e.id,
-      e.statement,
-      e.sessionId,
-      e.startTime,
-      e.userName)
-
-    info.state = ExecutionState.STARTED
-    executionList.put(e.id, info)
-    sessionList.get(e.sessionId).totalExecution += 1
-    executionList.get(e.id).groupId = e.groupId
-    updateLiveStore(executionList.get(e.id))
-    updateLiveStore(sessionList.get(e.sessionId))
-  }
+  private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit =
+    Option(sessionList.get(e.sessionId)) match {
+      case None => logWarning(s"onOperationStart called with unknown session id: ${e.sessionId}")
+      case Some(sessionData) =>
+        val info = getOrCreateExecution(
+          e.id,
+          e.statement,
+          e.sessionId,
+          e.startTime,
+          e.userName)
+
+        info.state = ExecutionState.STARTED
+        executionList.put(e.id, info)
+        sessionData.totalExecution += 1
+        executionList.get(e.id).groupId = e.groupId
+        updateLiveStore(executionList.get(e.id))
+        updateLiveStore(sessionData)
+    }
 
-  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit = {
-    executionList.get(e.id).executePlan = e.executionPlan
-    executionList.get(e.id).state = ExecutionState.COMPILED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit =
+    Option(executionList.get(e.id)) match {
+      case None => logWarning(s"onOperationParsed called with unknown operation id: ${e.id}")
+      case Some(executionData) =>
+        executionData.executePlan = e.executionPlan
+        executionData.state = ExecutionState.COMPILED
+        updateLiveStore(executionData)
+    }
 
-  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).state = ExecutionState.CANCELED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit =
+    Option(executionList.get(e.id)) match {
+      case None => logWarning(s"onOperationCanceled called with unknown operation id: ${e.id}")
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.CANCELED
+        updateLiveStore(executionData)
+    }
 
-  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).detail = e.errorMsg
-    executionList.get(e.id).state = ExecutionState.FAILED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit =
+    Option(executionList.get(e.id)) match {
+      case None => logWarning(s"onOperationError called with unknown operation id: ${e.id}")
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.detail = e.errorMsg
+        executionData.state = ExecutionState.FAILED
+        updateLiveStore(executionData)
+    }
 
-  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).state = ExecutionState.FINISHED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit =
+    Option(executionList.get(e.id)) match {
+      case None => logWarning(s"onOperationFinished called with unknown operation id: ${e.id}")
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.FINISHED
+        updateLiveStore(executionData)
+    }
 
-  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit = {
-    executionList.get(e.id).closeTimestamp = e.closeTime
-    executionList.get(e.id).state = ExecutionState.CLOSED
-    updateStoreWithTriggerEnabled(executionList.get(e.id))
-    executionList.remove(e.id)
-  }
+  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit =
+    Option(executionList.get(e.id)) match {
+      case None => logWarning(s"onOperationClosed called with unknown operation id: ${e.id}")
+      case Some(executionData) =>
+        executionData.closeTimestamp = e.closeTime
+        executionData.state = ExecutionState.CLOSED
+        updateStoreWithTriggerEnabled(executionData)
+        executionList.remove(e.id)
+    }
 
   // Update both live and history stores. Trigger is enabled by default, hence
   // it will cleanup the entity which exceeds the threshold.
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
new file mode 100644
index 0000000000000..05d540d782e31
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive.thriftserver
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.hive.conf.HiveConf
+import org.apache.hive.service.cli.OperationHandle
+import org.apache.hive.service.cli.operation.{GetCatalogsOperation, OperationManager}
+import org.apache.hive.service.cli.session.{HiveSessionImpl, SessionManager}
+import org.mockito.Mockito.{mock, verify, when}
+import org.mockito.invocation.InvocationOnMock
+
+import org.apache.spark.SparkFunSuite
+
+class HiveSessionImplSuite extends SparkFunSuite {
+  private var session: HiveSessionImpl = _
+  private var operationManager: OperationManager = _
+
+  override def beforeAll() {
+    super.beforeAll()
+
+    session = new HiveSessionImpl(
+      ThriftserverShimUtils.testedProtocolVersions.head,
+      "",
+      "",
+      new HiveConf(),
+      ""
+    )
+    val sessionManager = mock(classOf[SessionManager])
+    session.setSessionManager(sessionManager)
+    operationManager = mock(classOf[OperationManager])
+    session.setOperationManager(operationManager)
+    when(operationManager.newGetCatalogsOperation(session)).thenAnswer(
+      (_: InvocationOnMock) => {
+        val operation = mock(classOf[GetCatalogsOperation])
+        when(operation.getHandle).thenReturn(mock(classOf[OperationHandle]))
+        operation
+      }
+    )
+
+    session.open(Map.empty[String, String].asJava)
+  }
+
+  test("SPARK-31387 - session.close() closes all sessions regardless of thrown exceptions") {
+    val operationHandle1 = session.getCatalogs
+    val operationHandle2 = session.getCatalogs
+
+    when(operationManager.closeOperation(operationHandle1))
+      .thenThrow(classOf[NullPointerException])
+    when(operationManager.closeOperation(operationHandle2))
+      .thenThrow(classOf[NullPointerException])
+
+    session.close()
+
+    verify(operationManager).closeOperation(operationHandle1)
+    verify(operationManager).closeOperation(operationHandle2)
+  }
+}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
index 075032fa5d099..ea2523dfb581e 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
@@ -140,6 +140,22 @@ class HiveThriftServer2ListenerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(listener.noLiveData())
   }
 
+  test("SPARK-31387 - listener update methods should not throw exception with unknown input") {
+    val (statusStore: HiveThriftServer2AppStatusStore,
+    listener: HiveThriftServer2Listener) = createAppStatusStore(true)
+    val unknownSession = "unknown_session"
+    val unknownOperation = "unknown_operation"
+    listener.onOtherEvent(SparkListenerThriftServerSessionClosed(unknownSession, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationStart("id", unknownSession,
+      "stmt", "groupId", 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationParsed(unknownOperation, "query"))
+    listener.onOtherEvent(SparkListenerThriftServerOperationCanceled(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationError(unknownOperation,
+      "msg", "trace", 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationFinish(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationClosed(unknownOperation, 0))
+  }
+
   private def createProperties: Properties = {
     val properties = new Properties()
     properties.setProperty(SparkContext.SPARK_JOB_GROUP_ID, "groupId")
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 745f385e87f78..3e2c3def48d89 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -636,7 +636,11 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        operationManager.closeOperation(opHandle);
+        try {
+          operationManager.closeOperation(opHandle);
+        } catch (Exception e) {
+          LOG.warn("Exception is thrown closing operation " + opHandle, e);
+        }
       }
       opHandleSet.clear();
       // Cleanup session log directory.
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 14e9c4704c977..5cdae00460e34 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -650,7 +650,11 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        operationManager.closeOperation(opHandle);
+        try {
+          operationManager.closeOperation(opHandle);
+        } catch (Exception e) {
+          LOG.warn("Exception is thrown closing operation " + opHandle, e);
+        }
       }
       opHandleSet.clear();
       // Cleanup session log directory.

From e892a016699d996b959b4db01242cff934d62f76 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Tue, 12 May 2020 19:57:48 +0000
Subject: [PATCH 0664/1280] [SPARK-31683][CORE] Make Prometheus output
 consistent with DropWizard 4.1 result

### What changes were proposed in this pull request?

This PR aims to update Prometheus-related output format to be consistent with DropWizard 4.1 result.
- Add `Number` metrics for gauges metrics.
- Add `type` labels.

### Why are the changes needed?

SPARK-29032 added Prometheus support. After that, SPARK-29674 upgraded DropWizard for JDK9+ support and this caused difference in output labels and number of keys for Guage metrics. The current status is different from Apache Spark 2.4.5. Since we cannot change DropWizard, this PR aims to be consistent in Apache Spark 3.0.0 only.

**DropWizard 3.x**
```
metrics_master_aliveWorkers_Value 1.0
```

**DropWizard 4.1**
```
metrics_master_aliveWorkers_Value{type="gauges",} 1.0
metrics_master_aliveWorkers_Number{type="gauges",} 1.0
```

### Does this PR introduce _any_ user-facing change?

Yes, but this is a new feature in 3.0.0.

### How was this patch tested?

Manually check the output like the following.

**JMXExporter Result**
```
$ curl -s http://localhost:8088/ | grep "^metrics_master" | sort
metrics_master_aliveWorkers_Number{type="gauges",} 1.0
metrics_master_aliveWorkers_Value{type="gauges",} 1.0
metrics_master_apps_Number{type="gauges",} 0.0
metrics_master_apps_Value{type="gauges",} 0.0
metrics_master_waitingApps_Number{type="gauges",} 0.0
metrics_master_waitingApps_Value{type="gauges",} 0.0
metrics_master_workers_Number{type="gauges",} 1.0
metrics_master_workers_Value{type="gauges",} 1.0
```

**This PR**
```
$ curl -s http://localhost:8080/metrics/master/prometheus/ | grep master
metrics_master_aliveWorkers_Number{type="gauges"} 1
metrics_master_aliveWorkers_Value{type="gauges"} 1
metrics_master_apps_Number{type="gauges"} 0
metrics_master_apps_Value{type="gauges"} 0
metrics_master_waitingApps_Number{type="gauges"} 0
metrics_master_waitingApps_Value{type="gauges"} 0
metrics_master_workers_Number{type="gauges"} 1
metrics_master_workers_Value{type="gauges"} 1
```

Closes #28510 from dongjoon-hyun/SPARK-31683.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit 07209f3e2deab824f04484fa6b8bab0ec0a635d6)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 .../metrics/sink/PrometheusServlet.scala      | 73 ++++++++++---------
 .../status/api/v1/PrometheusResource.scala    | 52 ++++++-------
 2 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
index 011c7bc3f0051..59b863b89f75a 100644
--- a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala
@@ -56,58 +56,65 @@ private[spark] class PrometheusServlet(
   def getMetricsSnapshot(request: HttpServletRequest): String = {
     import scala.collection.JavaConverters._
 
+    val guagesLabel = """{type="gauges"}"""
+    val countersLabel = """{type="counters"}"""
+    val metersLabel = countersLabel
+    val histogramslabels = """{type="histograms"}"""
+    val timersLabels = """{type="timers"}"""
+
     val sb = new StringBuilder()
     registry.getGauges.asScala.foreach { case (k, v) =>
       if (!v.getValue.isInstanceOf[String]) {
-        sb.append(s"${normalizeKey(k)}Value ${v.getValue}\n")
+        sb.append(s"${normalizeKey(k)}Number$guagesLabel ${v.getValue}\n")
+        sb.append(s"${normalizeKey(k)}Value$guagesLabel ${v.getValue}\n")
       }
     }
     registry.getCounters.asScala.foreach { case (k, v) =>
-      sb.append(s"${normalizeKey(k)}Count ${v.getCount}\n")
+      sb.append(s"${normalizeKey(k)}Count$countersLabel ${v.getCount}\n")
     }
     registry.getHistograms.asScala.foreach { case (k, h) =>
       val snapshot = h.getSnapshot
       val prefix = normalizeKey(k)
-      sb.append(s"${prefix}Count ${h.getCount}\n")
-      sb.append(s"${prefix}Max ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Mean ${snapshot.getMean}\n")
-      sb.append(s"${prefix}Min ${snapshot.getMin}\n")
-      sb.append(s"${prefix}50thPercentile ${snapshot.getMedian}\n")
-      sb.append(s"${prefix}75thPercentile ${snapshot.get75thPercentile}\n")
-      sb.append(s"${prefix}95thPercentile ${snapshot.get95thPercentile}\n")
-      sb.append(s"${prefix}98thPercentile ${snapshot.get98thPercentile}\n")
-      sb.append(s"${prefix}99thPercentile ${snapshot.get99thPercentile}\n")
-      sb.append(s"${prefix}999thPercentile ${snapshot.get999thPercentile}\n")
-      sb.append(s"${prefix}StdDev ${snapshot.getStdDev}\n")
+      sb.append(s"${prefix}Count$histogramslabels ${h.getCount}\n")
+      sb.append(s"${prefix}Max$histogramslabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Mean$histogramslabels ${snapshot.getMean}\n")
+      sb.append(s"${prefix}Min$histogramslabels ${snapshot.getMin}\n")
+      sb.append(s"${prefix}50thPercentile$histogramslabels ${snapshot.getMedian}\n")
+      sb.append(s"${prefix}75thPercentile$histogramslabels ${snapshot.get75thPercentile}\n")
+      sb.append(s"${prefix}95thPercentile$histogramslabels ${snapshot.get95thPercentile}\n")
+      sb.append(s"${prefix}98thPercentile$histogramslabels ${snapshot.get98thPercentile}\n")
+      sb.append(s"${prefix}99thPercentile$histogramslabels ${snapshot.get99thPercentile}\n")
+      sb.append(s"${prefix}999thPercentile$histogramslabels ${snapshot.get999thPercentile}\n")
+      sb.append(s"${prefix}StdDev$histogramslabels ${snapshot.getStdDev}\n")
     }
     registry.getMeters.entrySet.iterator.asScala.foreach { kv =>
       val prefix = normalizeKey(kv.getKey)
       val meter = kv.getValue
-      sb.append(s"${prefix}Count ${meter.getCount}\n")
-      sb.append(s"${prefix}MeanRate ${meter.getMeanRate}\n")
-      sb.append(s"${prefix}OneMinuteRate ${meter.getOneMinuteRate}\n")
-      sb.append(s"${prefix}FiveMinuteRate ${meter.getFiveMinuteRate}\n")
-      sb.append(s"${prefix}FifteenMinuteRate ${meter.getFifteenMinuteRate}\n")
+      sb.append(s"${prefix}Count$metersLabel ${meter.getCount}\n")
+      sb.append(s"${prefix}MeanRate$metersLabel ${meter.getMeanRate}\n")
+      sb.append(s"${prefix}OneMinuteRate$metersLabel ${meter.getOneMinuteRate}\n")
+      sb.append(s"${prefix}FiveMinuteRate$metersLabel ${meter.getFiveMinuteRate}\n")
+      sb.append(s"${prefix}FifteenMinuteRate$metersLabel ${meter.getFifteenMinuteRate}\n")
     }
     registry.getTimers.entrySet.iterator.asScala.foreach { kv =>
       val prefix = normalizeKey(kv.getKey)
       val timer = kv.getValue
       val snapshot = timer.getSnapshot
-      sb.append(s"${prefix}Count ${timer.getCount}\n")
-      sb.append(s"${prefix}Max ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Mean ${snapshot.getMax}\n")
-      sb.append(s"${prefix}Min ${snapshot.getMin}\n")
-      sb.append(s"${prefix}50thPercentile ${snapshot.getMedian}\n")
-      sb.append(s"${prefix}75thPercentile ${snapshot.get75thPercentile}\n")
-      sb.append(s"${prefix}95thPercentile ${snapshot.get95thPercentile}\n")
-      sb.append(s"${prefix}98thPercentile ${snapshot.get98thPercentile}\n")
-      sb.append(s"${prefix}99thPercentile ${snapshot.get99thPercentile}\n")
-      sb.append(s"${prefix}999thPercentile ${snapshot.get999thPercentile}\n")
-      sb.append(s"${prefix}StdDev ${snapshot.getStdDev}\n")
-      sb.append(s"${prefix}FifteenMinuteRate ${timer.getFifteenMinuteRate}\n")
-      sb.append(s"${prefix}FiveMinuteRate ${timer.getFiveMinuteRate}\n")
-      sb.append(s"${prefix}OneMinuteRate ${timer.getOneMinuteRate}\n")
-      sb.append(s"${prefix}MeanRate ${timer.getMeanRate}\n")
+      sb.append(s"${prefix}Count$timersLabels ${timer.getCount}\n")
+      sb.append(s"${prefix}Max$timersLabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Mean$timersLabels ${snapshot.getMax}\n")
+      sb.append(s"${prefix}Min$timersLabels ${snapshot.getMin}\n")
+      sb.append(s"${prefix}50thPercentile$timersLabels ${snapshot.getMedian}\n")
+      sb.append(s"${prefix}75thPercentile$timersLabels ${snapshot.get75thPercentile}\n")
+      sb.append(s"${prefix}95thPercentile$timersLabels ${snapshot.get95thPercentile}\n")
+      sb.append(s"${prefix}98thPercentile$timersLabels ${snapshot.get98thPercentile}\n")
+      sb.append(s"${prefix}99thPercentile$timersLabels ${snapshot.get99thPercentile}\n")
+      sb.append(s"${prefix}999thPercentile$timersLabels ${snapshot.get999thPercentile}\n")
+      sb.append(s"${prefix}StdDev$timersLabels ${snapshot.getStdDev}\n")
+      sb.append(s"${prefix}FifteenMinuteRate$timersLabels ${timer.getFifteenMinuteRate}\n")
+      sb.append(s"${prefix}FiveMinuteRate$timersLabels ${timer.getFiveMinuteRate}\n")
+      sb.append(s"${prefix}OneMinuteRate$timersLabels ${timer.getOneMinuteRate}\n")
+      sb.append(s"${prefix}MeanRate$timersLabels ${timer.getMeanRate}\n")
     }
     sb.toString()
   }
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
index 2a5f151d99c67..4ed3d4525a185 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
@@ -50,27 +50,27 @@ private[v1] class PrometheusResource extends ApiRequestContext {
         "application_name" -> store.applicationInfo.name,
         "executor_id" -> executor.id
       ).map { case (k, v) => s"""$k="$v"""" }.mkString("{", ", ", "}")
-      sb.append(s"${prefix}rddBlocks_Count$labels ${executor.rddBlocks}\n")
-      sb.append(s"${prefix}memoryUsed_Count$labels ${executor.memoryUsed}\n")
-      sb.append(s"${prefix}diskUsed_Count$labels ${executor.diskUsed}\n")
-      sb.append(s"${prefix}totalCores_Count$labels ${executor.totalCores}\n")
-      sb.append(s"${prefix}maxTasks_Count$labels ${executor.maxTasks}\n")
-      sb.append(s"${prefix}activeTasks_Count$labels ${executor.activeTasks}\n")
-      sb.append(s"${prefix}failedTasks_Count$labels ${executor.failedTasks}\n")
-      sb.append(s"${prefix}completedTasks_Count$labels ${executor.completedTasks}\n")
-      sb.append(s"${prefix}totalTasks_Count$labels ${executor.totalTasks}\n")
-      sb.append(s"${prefix}totalDuration_Value$labels ${executor.totalDuration}\n")
-      sb.append(s"${prefix}totalGCTime_Value$labels ${executor.totalGCTime}\n")
-      sb.append(s"${prefix}totalInputBytes_Count$labels ${executor.totalInputBytes}\n")
-      sb.append(s"${prefix}totalShuffleRead_Count$labels ${executor.totalShuffleRead}\n")
-      sb.append(s"${prefix}totalShuffleWrite_Count$labels ${executor.totalShuffleWrite}\n")
-      sb.append(s"${prefix}maxMemory_Count$labels ${executor.maxMemory}\n")
+      sb.append(s"${prefix}rddBlocks$labels ${executor.rddBlocks}\n")
+      sb.append(s"${prefix}memoryUsed_bytes$labels ${executor.memoryUsed}\n")
+      sb.append(s"${prefix}diskUsed_bytes$labels ${executor.diskUsed}\n")
+      sb.append(s"${prefix}totalCores$labels ${executor.totalCores}\n")
+      sb.append(s"${prefix}maxTasks$labels ${executor.maxTasks}\n")
+      sb.append(s"${prefix}activeTasks$labels ${executor.activeTasks}\n")
+      sb.append(s"${prefix}failedTasks_total$labels ${executor.failedTasks}\n")
+      sb.append(s"${prefix}completedTasks_total$labels ${executor.completedTasks}\n")
+      sb.append(s"${prefix}totalTasks_total$labels ${executor.totalTasks}\n")
+      sb.append(s"${prefix}totalDuration_seconds_total$labels ${executor.totalDuration * 0.001}\n")
+      sb.append(s"${prefix}totalGCTime_seconds_total$labels ${executor.totalGCTime * 0.001}\n")
+      sb.append(s"${prefix}totalInputBytes_bytes_total$labels ${executor.totalInputBytes}\n")
+      sb.append(s"${prefix}totalShuffleRead_bytes_total$labels ${executor.totalShuffleRead}\n")
+      sb.append(s"${prefix}totalShuffleWrite_bytes_total$labels ${executor.totalShuffleWrite}\n")
+      sb.append(s"${prefix}maxMemory_bytes$labels ${executor.maxMemory}\n")
       executor.executorLogs.foreach { case (k, v) => }
       executor.memoryMetrics.foreach { m =>
-        sb.append(s"${prefix}usedOnHeapStorageMemory_Count$labels ${m.usedOnHeapStorageMemory}\n")
-        sb.append(s"${prefix}usedOffHeapStorageMemory_Count$labels ${m.usedOffHeapStorageMemory}\n")
-        sb.append(s"${prefix}totalOnHeapStorageMemory_Count$labels ${m.totalOnHeapStorageMemory}\n")
-        sb.append(s"${prefix}totalOffHeapStorageMemory_Count$labels " +
+        sb.append(s"${prefix}usedOnHeapStorageMemory_bytes$labels ${m.usedOnHeapStorageMemory}\n")
+        sb.append(s"${prefix}usedOffHeapStorageMemory_bytes$labels ${m.usedOffHeapStorageMemory}\n")
+        sb.append(s"${prefix}totalOnHeapStorageMemory_bytes$labels ${m.totalOnHeapStorageMemory}\n")
+        sb.append(s"${prefix}totalOffHeapStorageMemory_bytes$labels " +
           s"${m.totalOffHeapStorageMemory}\n")
       }
       executor.peakMemoryMetrics.foreach { m =>
@@ -90,14 +90,16 @@ private[v1] class PrometheusResource extends ApiRequestContext {
           "ProcessTreePythonVMemory",
           "ProcessTreePythonRSSMemory",
           "ProcessTreeOtherVMemory",
-          "ProcessTreeOtherRSSMemory",
-          "MinorGCCount",
-          "MinorGCTime",
-          "MajorGCCount",
-          "MajorGCTime"
+          "ProcessTreeOtherRSSMemory"
         )
         names.foreach { name =>
-          sb.append(s"$prefix${name}_Count$labels ${m.getMetricValue(name)}\n")
+          sb.append(s"$prefix${name}_bytes$labels ${m.getMetricValue(name)}\n")
+        }
+        Seq("MinorGCCount", "MajorGCCount").foreach { name =>
+          sb.append(s"$prefix${name}_total$labels ${m.getMetricValue(name)}\n")
+        }
+        Seq("MinorGCTime", "MajorGCTime").foreach { name =>
+          sb.append(s"$prefix${name}_seconds_total$labels ${m.getMetricValue(name) * 0.001}\n")
         }
       }
     }

From ce52f61f720783e8eeb3313c763493054599091a Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Tue, 12 May 2020 14:24:56 -0700
Subject: [PATCH 0665/1280] [SPARK-31691][INFRA] release-build.sh should ignore
 a fallback output from `build/mvn`

### What changes were proposed in this pull request?

This PR adds `i` option to ignore additional `build/mvn` output which is irrelevant to version string.

### Why are the changes needed?

SPARK-28963 added additional output message, `Falling back to archive.apache.org to download Maven` in build/mvn. This breaks `dev/create-release/release-build.sh` and currently Spark Packaging Jenkins job is hitting this issue consistently and broken.
- https://amplab.cs.berkeley.edu/jenkins/view/Spark%20Packaging/job/spark-master-maven-snapshots/2912/console

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

This happens only when the mirror fails. So, this is verified manually hiject the script. It works like the following.
```
$ echo 'Falling back to archive.apache.org to download Maven' > out
$ build/mvn help:evaluate -Dexpression=project.version >> out
Using `mvn` from path: /Users/dongjoon/PRS/SPARK_RELEASE_2/build/apache-maven-3.6.3/bin/mvn
$ cat out | grep -v INFO | grep -v WARNING | grep -v Download
Falling back to archive.apache.org to download Maven
3.1.0-SNAPSHOT
$ cat out | grep -v INFO | grep -v WARNING | grep -vi Download
3.1.0-SNAPSHOT
```

Closes #28514 from dongjoon-hyun/SPARK_RELEASE_2.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 3772154442e6341ea97a2f41cd672413de918e85)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/create-release/release-build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 022d3af95c05d..655b079ac28bf 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -103,7 +103,7 @@ if [ -z "$SPARK_VERSION" ]; then
   # Run $MVN in a separate command so that 'set -e' does the right thing.
   TMP=$(mktemp)
   $MVN help:evaluate -Dexpression=project.version > $TMP
-  SPARK_VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -v Download)
+  SPARK_VERSION=$(cat $TMP | grep -v INFO | grep -v WARNING | grep -vi Download)
   rm $TMP
 fi
 

From da71e303b904e76cd68a5ed57111828acb7353f6 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 13 May 2020 01:46:40 -0700
Subject: [PATCH 0666/1280] [SPARK-31697][WEBUI] HistoryServer should set
 Content-Type

### What changes were proposed in this pull request?

This PR changes HistoryServer to set Content-Type.

I noticed that we will get html as plain text when we access to wrong URLs which represent non-existence appId on HistoryServer.

```
<html>
      <head>
        <meta http-equiv="Content-type" content="text/html; charset=utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="stylesheet" href="/static/bootstrap.min.css" type="text/css"/><link rel="stylesheet" href="/static/vis-timeline-graph2d.min.css" type="text/css"/><link rel="stylesheet" href="/static/webui.css" type="text/css"/><link rel="stylesheet" href="/static/timeline-view.css" type="text/css"/><script src="/static/sorttable.js"></script><script src="/static/jquery-3.4.1.min.js"></script><script src="/static/vis-timeline-graph2d.min.js"></script><script src="/static/bootstrap.bundle.min.js"></script><script src="/static/initialize-tooltips.js"></script><script src="/static/table.js"></script><script src="/static/timeline-view.js"></script><script src="/static/log-view.js"></script><script src="/static/webui.js"></script><script>setUIRoot('')</script>

        <link rel="shortcut icon" href="/static/spark-logo-77x50px-hd.png"></link>
        <title>Not Found</title>
      </head>
      <body>
        <div class="container-fluid">
          <div class="row">
            <div class="col-12">
              <h3 style="vertical-align: middle; display: inline-block;">
                <a style="text-decoration: none" href="/">
                  <img src="/static/spark-logo-77x50px-hd.png"/>
                  <span class="version" style="margin-right: 15px;">3.1.0-SNAPSHOT</span>
                </a>
                Not Found
              </h3>
            </div>
          </div>
          <div class="row">
            <div class="col-12">
              <div class="row">Application local-1589239 not found.</div>
            </div>
          </div>
        </div>
      </body>
    </html>
```
The reason is Content-Type not set. I confirmed it with `curl -I http://localhost:18080/history/<wrong-appId>`
```
HTTP/1.1 404 Not Found
Date: Wed, 13 May 2020 06:59:29 GMT
Cache-Control: no-cache, no-store, must-revalidate
X-Frame-Options: SAMEORIGIN
X-XSS-Protection: 1; mode=block
X-Content-Type-Options: nosniff
Content-Length: 1778
Server: Jetty(9.4.18.v20190429)
```

### Why are the changes needed?

This is a bug.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

I added a test case for this issue.

Closes #28519 from sarutak/fix-content-type.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7952f44dacd18891e4f78c91146c1cf37dda6a46)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/deploy/history/HistoryServer.scala   |  3 +++
 .../spark/deploy/history/HistoryServerSuite.scala     | 11 +++++++++++
 2 files changed, 14 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index 62cac261ae014..7e0d311311f30 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -69,6 +69,9 @@ class HistoryServer(
 
   private val loaderServlet = new HttpServlet {
     protected override def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = {
+
+      res.setContentType("text/html;charset=utf-8")
+
       // Parse the URI created by getAttemptURI(). It contains an app ID and an optional
       // attempt ID (separated by a slash).
       val parts = Option(req.getPathInfo()).getOrElse("").split("/")
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 206db0feb5716..56cc3daecb3a6 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -693,6 +693,17 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     out.close()
   }
 
+  test("SPARK-31697: HistoryServer should set Content-Type") {
+    val port = server.boundPort
+    val nonExistenceAppId = "local-non-existence"
+    val url = new URL(s"http://localhost:$port/history/$nonExistenceAppId")
+    val conn = url.openConnection().asInstanceOf[HttpURLConnection]
+    conn.setRequestMethod("GET")
+    conn.connect()
+    val expectedContentType = "text/html;charset=utf-8"
+    val actualContentType = conn.getContentType
+    assert(actualContentType === expectedContentType)
+  }
 }
 
 object HistoryServerSuite {

From f71edf16b1b34659bd92900003154c9b4d811d7e Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 13 May 2020 10:03:12 -0700
Subject: [PATCH 0667/1280] [SPARK-31701][R][SQL] Bump up the minimum Arrow
 version as 0.15.1 in SparkR

### What changes were proposed in this pull request?

This PR proposes to set the minimum Arrow version as 0.15.1 to be consistent with PySpark side at.

### Why are the changes needed?

It will reduce the maintenance overhead to match the Arrow versions, and minimize the supported range. SparkR Arrow optimization is experimental yet.

### Does this PR introduce _any_ user-facing change?

No, it's the change in unreleased branches only.

### How was this patch tested?

0.15.x was already tested at SPARK-29378, and we're testing the latest version of SparkR currently in AppVeyor. I already manually tested too.

Closes #28520 from HyukjinKwon/SPARK-31701.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e1315cd65631823123af0d14771b0f699809251b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 R/pkg/DESCRIPTION |  2 +-
 docs/sparkr.md    | 13 ++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 3eff30b576748..21f3eaa5c15f6 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -23,7 +23,7 @@ Suggests:
     testthat,
     e1071,
     survival,
-    arrow
+    arrow (>= 0.15.1)
 Collate:
     'schema.R'
     'generics.R'
diff --git a/docs/sparkr.md b/docs/sparkr.md
index 24fa3b4feac19..9cdc39296f3fe 100644
--- a/docs/sparkr.md
+++ b/docs/sparkr.md
@@ -663,20 +663,15 @@ Apache Arrow is an in-memory columnar data format that is used in Spark to effic
 
 ## Ensure Arrow Installed
 
-Arrow R library is available on CRAN as of [ARROW-3204](https://issues.apache.org/jira/browse/ARROW-3204). It can be installed as below.
+Arrow R library is available on CRAN and it can be installed as below.
 
 ```bash
 Rscript -e 'install.packages("arrow", repos="https://cloud.r-project.org/")'
 ```
+Please refer [the official documentation of Apache Arrow](https://arrow.apache.org/docs/r/) for more detials.
 
-If you need to install old versions, it should be installed directly from Github. You can use `remotes::install_github` as below.
-
-```bash
-Rscript -e 'remotes::install_github("apache/arrow@apache-arrow-0.12.1", subdir = "r")'
-```
-
-`apache-arrow-0.12.1` is a version tag that can be checked in [Arrow at Github](https://github.com/apache/arrow/releases). You must ensure that Arrow R package is installed and available on all cluster nodes.
-The current supported minimum version is 0.12.1; however, this might change between the minor releases since Arrow optimization in SparkR is experimental.
+Note that you must ensure that Arrow R package is installed and available on all cluster nodes.
+The current supported minimum version is 0.15.1; however, this might change between the minor releases since Arrow optimization in SparkR is experimental.
 
 ## Enabling for Conversion to/from R DataFrame, `dapply` and `gapply`
 

From 54ab70b84b16b354a6c4f1217580efc887f74917 Mon Sep 17 00:00:00 2001
From: Xingcan Cui <xccui@apache.org>
Date: Thu, 14 May 2020 12:07:22 +0900
Subject: [PATCH 0668/1280] [SPARK-31632][CORE][WEBUI] Enrich the exception
 message when application information is unavailable

### What changes were proposed in this pull request?

This PR caught the `NoSuchElementException` and enriched the error message for `AppStatusStore.applicationInfo()` when Spark is starting up and the application information is unavailable.

### Why are the changes needed?
During the initialization of `SparkContext`, it first starts the Web UI and then set up the `LiveListenerBus` thread for dispatching the `SparkListenerApplicationStart` event (which will trigger writing the requested `ApplicationInfo` to `InMemoryStore`). If the Web UI is accessed before this info's being written to `InMemoryStore`, the following `NoSuchElementException` will be thrown.
```
 WARN org.eclipse.jetty.server.HttpChannel: /jobs/
 java.util.NoSuchElementException
 at java.util.Collections$EmptyIterator.next(Collections.java:4191)
 at org.apache.spark.util.kvstore.InMemoryStore$InMemoryIterator.next(InMemoryStore.java:467)
 at org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:39)
 at org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:266)
 at org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:89)
 at org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:80)
 at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
 at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
 at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
 at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
 at org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)
 at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
 at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
 at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
 at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
 at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
 at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
 at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
 at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
 at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
 at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:753)
 at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
 at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
 at org.eclipse.jetty.server.Server.handle(Server.java:505)
 at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
 at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
 at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
 at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
 at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
 at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
 at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
 at java.lang.Thread.run(Thread.java:748)
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manually tested.

This can be reproduced:

1. `./bin/spark-shell`
2. at the same time, open `http://localhost:4040/jobs/` in your browser with quickly refreshing.

Closes #28444 from xccui/SPARK-31632.

Authored-by: Xingcan Cui <xccui@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 42951e6786319481220ba4abfad015a8d11749f3)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/status/AppStatusStore.scala      | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 6b89812cc2bf0..42d4071d61c8b 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -22,7 +22,7 @@ import java.util.{List => JList}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.HashMap
 
-import org.apache.spark.{JobExecutionStatus, SparkConf}
+import org.apache.spark.{JobExecutionStatus, SparkConf, SparkException}
 import org.apache.spark.status.api.v1
 import org.apache.spark.ui.scope._
 import org.apache.spark.util.Utils
@@ -36,7 +36,14 @@ private[spark] class AppStatusStore(
     val listener: Option[AppStatusListener] = None) {
 
   def applicationInfo(): v1.ApplicationInfo = {
-    store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+    try {
+      // The ApplicationInfo may not be available when Spark is starting up.
+      store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+    } catch {
+      case _: NoSuchElementException =>
+        throw new SparkException("Failed to get the application information. " +
+          "If you are starting up Spark, please wait a while until it's ready.")
+    }
   }
 
   def environmentInfo(): v1.ApplicationEnvironmentInfo = {

From d639a12ef243e1e8d20bd06d3a97d00e47f05517 Mon Sep 17 00:00:00 2001
From: Karuppayya Rajendran <karuppayya1990@gmail.com>
Date: Wed, 13 May 2020 23:18:38 -0700
Subject: [PATCH 0669/1280] [SPARK-31692][SQL] Pass hadoop confs  specifed via
 Spark confs to URLStreamHandlerfactory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
Pass hadoop confs  specifed via Spark confs to URLStreamHandlerfactory

### Why are the changes needed?

**BEFORE**
```
➜  spark git:(SPARK-31692) ✗ ./bin/spark-shell --conf spark.hadoop.fs.file.impl=org.apache.hadoop.fs.RawLocalFileSystem

scala> spark.sharedState
res0: org.apache.spark.sql.internal.SharedState = org.apache.spark.sql.internal.SharedState5793cd84

scala> new java.net.URL("file:///tmp/1.txt").openConnection.getInputStream
res1: java.io.InputStream = org.apache.hadoop.fs.ChecksumFileSystem$FSDataBoundedInputStream22846025

scala> import org.apache.hadoop.fs._
import org.apache.hadoop.fs._

scala>  FileSystem.get(new Path("file:///tmp/1.txt").toUri, spark.sparkContext.hadoopConfiguration)
res2: org.apache.hadoop.fs.FileSystem = org.apache.hadoop.fs.LocalFileSystem5a930c03
```

**AFTER**
```
➜  spark git:(SPARK-31692) ✗ ./bin/spark-shell --conf spark.hadoop.fs.file.impl=org.apache.hadoop.fs.RawLocalFileSystem

scala> spark.sharedState
res0: org.apache.spark.sql.internal.SharedState = org.apache.spark.sql.internal.SharedState5c24a636

scala> new java.net.URL("file:///tmp/1.txt").openConnection.getInputStream
res1: java.io.InputStream = org.apache.hadoop.fs.FSDataInputStream2ba8f528

scala> import org.apache.hadoop.fs._
import org.apache.hadoop.fs._

scala>  FileSystem.get(new Path("file:///tmp/1.txt").toUri, spark.sparkContext.hadoopConfiguration)
res2: org.apache.hadoop.fs.FileSystem = LocalFS

scala>  FileSystem.get(new Path("file:///tmp/1.txt").toUri, spark.sparkContext.hadoopConfiguration).getClass
res3: Class[_ <: org.apache.hadoop.fs.FileSystem] = class org.apache.hadoop.fs.RawLocalFileSystem
```
The type of FileSystem object created(you can check the last statement in the above snippets) in the above two cases are different, which should not have been the case

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Tested locally.
Added Unit test

Closes #28516 from karuppayya/SPARK-31692.

Authored-by: Karuppayya Rajendran <karuppayya1990@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 72601460ada41761737f39d5dff8e69444fce2ba)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/internal/SharedState.scala      |  6 +-
 .../spark/sql/internal/SharedStateSuite.scala | 55 +++++++++++++++++++
 2 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index 47119ab903da7..ce4385d88f1e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -53,7 +53,7 @@ private[sql] class SharedState(
     initialConfigs: scala.collection.Map[String, String])
   extends Logging {
 
-  SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf)
+  SharedState.setFsUrlStreamHandlerFactory(sparkContext.conf, sparkContext.hadoopConfiguration)
 
   private val (conf, hadoopConf) = {
     // Load hive-site.xml into hadoopConf and determine the warehouse path which will be set into
@@ -174,13 +174,13 @@ private[sql] class SharedState(
 object SharedState extends Logging {
   @volatile private var fsUrlStreamHandlerFactoryInitialized = false
 
-  private def setFsUrlStreamHandlerFactory(conf: SparkConf): Unit = {
+  private def setFsUrlStreamHandlerFactory(conf: SparkConf, hadoopConf: Configuration): Unit = {
     if (!fsUrlStreamHandlerFactoryInitialized &&
         conf.get(DEFAULT_URL_STREAM_HANDLER_FACTORY_ENABLED)) {
       synchronized {
         if (!fsUrlStreamHandlerFactoryInitialized) {
           try {
-            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory())
+            URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(hadoopConf))
             fsUrlStreamHandlerFactoryInitialized = true
           } catch {
             case NonFatal(_) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
new file mode 100644
index 0000000000000..81bf15342423c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal
+
+import java.net.URL
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FsUrlStreamHandlerFactory
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+
+/**
+ * Tests for [[org.apache.spark.sql.internal.SharedState]].
+ */
+class SharedStateSuite extends SharedSparkSession {
+
+  override protected def sparkConf: SparkConf = {
+    super.sparkConf
+      .set("spark.hadoop.fs.defaultFS", "file:///")
+  }
+
+  test("SPARK-31692: Url handler factory should have the hadoop configs from Spark conf") {
+    // Accessing shared state to init the object since it is `lazy val`
+    spark.sharedState
+    val field = classOf[URL].getDeclaredField("factory")
+    field.setAccessible(true)
+    val value = field.get(null)
+    assert(value.isInstanceOf[FsUrlStreamHandlerFactory])
+    val streamFactory = value.asInstanceOf[FsUrlStreamHandlerFactory]
+
+    val confField = classOf[FsUrlStreamHandlerFactory].getDeclaredField("conf")
+    confField.setAccessible(true)
+    val conf = confField.get(streamFactory)
+
+    assert(conf.isInstanceOf[Configuration])
+    assert(conf.asInstanceOf[Configuration].get("fs.defaultFS") == "file:///")
+  }
+}

From f5cf11c4d39190f7f5f8a20c8c634c0dc2d6c212 Mon Sep 17 00:00:00 2001
From: "sunke.03" <sunke.03@bytedance.com>
Date: Thu, 14 May 2020 13:55:24 +0000
Subject: [PATCH 0670/1280] =?UTF-8?q?[SPARK-30973][SQL]=20ScriptTransforma?=
 =?UTF-8?q?tionExec=20should=20wait=20for=20the=20termination=20=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

This PR try to fix a bug in `org.apache.spark.sql.hive.execution.ScriptTransformationExec`. This bug appears in our online cluster.  `ScriptTransformationExec` should throw an exception, when user uses a python script which contains parse error.  But current implementation may miss this case of failure.

### Why are the changes needed?

When user uses a python script which contains a parse error, there will be no output. So  `scriptOutputReader.next(scriptOutputWritable) <= 0` matches, then we use `checkFailureAndPropagate()` to check the `proc`.  But the `proc` may still be alive and `writerThread.exception` is not defined,  `checkFailureAndPropagate` cannot check this case of failure.  In the end, the Spark SQL job runs successfully and returns no result. In fact, the SparK SQL job should fails and shows the exception properly.

For example, the error python script is blow.
``` python
# encoding: utf8
import unknow_module
import sys

for line in sys.stdin:
    print line
```
The bug can be reproduced by running the following code in our cluter.
```
spark.range(100*100).toDF("index").createOrReplaceTempView("test")
spark.sql("select TRANSFORM(index) USING 'python error_python.py' as new_index from test").collect.foreach(println)
```

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Existing UT

Closes #27724 from slamke/transformation.

Authored-by: sunke.03 <sunke.03@bytedance.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ddbce4edee6d4de30e6900bc0f03728a989aef0a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  9 +++++
 .../execution/ScriptTransformationExec.scala  | 12 ++++++-
 .../execution/ScriptTransformationSuite.scala | 36 +++++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 6c18280ce4d55..31038a0e26f7e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2561,6 +2561,15 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val SCRIPT_TRANSFORMATION_EXIT_TIMEOUT =
+    buildConf("spark.sql.scriptTransformation.exitTimeoutInSeconds")
+      .internal()
+      .doc("Timeout for executor to wait for the termination of transformation script when EOF.")
+      .version("3.0.0")
+      .timeConf(TimeUnit.SECONDS)
+      .checkValue(_ > 0, "The timeout value must be positive")
+      .createWithDefault(10L)
+
   /**
    * Holds information about keys that have been deprecated.
    *
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
index 40f7b4e8db7c5..c7183fd7385a6 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/ScriptTransformationExec.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive.execution
 import java.io._
 import java.nio.charset.StandardCharsets
 import java.util.Properties
+import java.util.concurrent.TimeUnit
 import javax.annotation.Nullable
 
 import scala.collection.JavaConverters._
@@ -42,6 +43,7 @@ import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.hive.HiveInspectors
 import org.apache.spark.sql.hive.HiveShim._
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.DataType
 import org.apache.spark.util.{CircularBuffer, RedirectThread, SerializableConfiguration, Utils}
 
@@ -136,6 +138,15 @@ case class ScriptTransformationExec(
             throw writerThread.exception.get
           }
 
+          // There can be a lag between reader read EOF and the process termination.
+          // If the script fails to startup, this kind of error may be missed.
+          // So explicitly waiting for the process termination.
+          val timeout = conf.getConf(SQLConf.SCRIPT_TRANSFORMATION_EXIT_TIMEOUT)
+          val exitRes = proc.waitFor(timeout, TimeUnit.SECONDS)
+          if (!exitRes) {
+            log.warn(s"Transformation script process exits timeout in $timeout seconds")
+          }
+
           if (!proc.isAlive) {
             val exitCode = proc.exitValue()
             if (exitCode != 0) {
@@ -173,7 +184,6 @@ case class ScriptTransformationExec(
                     // Ideally the proc should *not* be alive at this point but
                     // there can be a lag between EOF being written out and the process
                     // being terminated. So explicitly waiting for the process to be done.
-                    proc.waitFor()
                     checkFailureAndPropagate()
                     return false
                 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
index 7153d3f03cd57..b97eb869a9e54 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
@@ -227,6 +227,42 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
         'e.cast("string")).collect())
     }
   }
+
+  test("SPARK-30973: TRANSFORM should wait for the termination of the script (no serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[SparkException] {
+      val plan =
+        new ScriptTransformationExec(
+          input = Seq(rowsDf.col("a").expr),
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = noSerdeIOSchema)
+      SparkPlanTest.executePlan(plan, hiveContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
+
+  test("SPARK-30973: TRANSFORM should wait for the termination of the script (with serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+
+    val rowsDf = Seq("a", "b", "c").map(Tuple1.apply).toDF("a")
+    val e = intercept[SparkException] {
+      val plan =
+        new ScriptTransformationExec(
+          input = Seq(rowsDf.col("a").expr),
+          script = "some_non_existent_command",
+          output = Seq(AttributeReference("a", StringType)()),
+          child = rowsDf.queryExecution.sparkPlan,
+          ioschema = serdeIOSchema)
+      SparkPlanTest.executePlan(plan, hiveContext)
+    }
+    assert(e.getMessage.contains("Subprocess exited with status"))
+    assert(uncaughtExceptionHandler.exception.isEmpty)
+  }
 }
 
 private case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {

From 00e6acc9c6d45c5dd3b3f70c87909743a8073dba Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Thu, 14 May 2020 09:24:40 -0500
Subject: [PATCH 0671/1280] [SPARK-31676][ML] QuantileDiscretizer raise error
 parameter splits given invalid value (splits array includes -0.0 and 0.0)

### What changes were proposed in this pull request?

In QuantileDiscretizer.getDistinctSplits, before invoking distinct, normalize all -0.0 and 0.0 to be 0.0
```
    for (i <- 0 until splits.length) {
      if (splits(i) == -0.0) {
        splits(i) = 0.0
      }
    }
```
### Why are the changes needed?
Fix bug.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Unit test.

#### Manually test:

~~~scala
import scala.util.Random
val rng = new Random(3)

val a1 = Array.tabulate(200)(_=>rng.nextDouble * 2.0 - 1.0) ++ Array.fill(20)(0.0) ++ Array.fill(20)(-0.0)

import spark.implicits._
val df1 = sc.parallelize(a1, 2).toDF("id")

import org.apache.spark.ml.feature.QuantileDiscretizer
val qd = new QuantileDiscretizer().setInputCol("id").setOutputCol("out").setNumBuckets(200).setRelativeError(0.0)

val model = qd.fit(df1) // will raise error in spark master.
~~~

### Explain
scala `0.0 == -0.0` is True but `0.0.hashCode == -0.0.hashCode()` is False. This break the contract between equals() and hashCode() If two objects are equal, then they must have the same hash code.

And array.distinct will rely on elem.hashCode so it leads to this error.

Test code on distinct
```
import scala.util.Random
val rng = new Random(3)

val a1 = Array.tabulate(200)(_=>rng.nextDouble * 2.0 - 1.0) ++ Array.fill(20)(0.0) ++ Array.fill(20)(-0.0)
a1.distinct.sorted.foreach(x => print(x.toString + "\n"))
```

Then you will see output like:
```
...
-0.009292684662246975
-0.0033280686465135823
-0.0
0.0
0.0022219556032221366
0.02217419561977274
...
```

Closes #28498 from WeichenXu123/SPARK-31676.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit b2300fca1e1a22d74c6eeda37942920a6c6299ff)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../spark/ml/feature/QuantileDiscretizer.scala | 12 ++++++++++++
 .../ml/feature/QuantileDiscretizerSuite.scala  | 18 ++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
index 216d99d01f2f7..4eedfc4dc0efa 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
@@ -236,6 +236,18 @@ final class QuantileDiscretizer @Since("1.6.0") (@Since("1.6.0") override val ui
   private def getDistinctSplits(splits: Array[Double]): Array[Double] = {
     splits(0) = Double.NegativeInfinity
     splits(splits.length - 1) = Double.PositiveInfinity
+
+    // 0.0 and -0.0 are distinct values, array.distinct will preserve both of them.
+    // but 0.0 > -0.0 is False which will break the parameter validation checking.
+    // and in scala <= 2.12, there's bug which will cause array.distinct generate
+    // non-deterministic results when array contains both 0.0 and -0.0
+    // So that here we should first normalize all 0.0 and -0.0 to be 0.0
+    // See https://github.com/scala/bug/issues/11995
+    for (i <- 0 until splits.length) {
+      if (splits(i) == -0.0) {
+        splits(i) = 0.0
+      }
+    }
     val distinctSplits = splits.distinct
     if (splits.length != distinctSplits.length) {
       log.warn(s"Some quantiles were identical. Bucketing to ${distinctSplits.length - 1}" +
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
index 6f6ab26cbac43..682b87a0f68d3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/QuantileDiscretizerSuite.scala
@@ -512,4 +512,22 @@ class QuantileDiscretizerSuite extends MLTest with DefaultReadWriteTest {
     assert(observedNumBuckets === numBuckets,
       "Observed number of buckets does not equal expected number of buckets.")
   }
+
+  test("[SPARK-31676] QuantileDiscretizer raise error parameter splits given invalid value") {
+    import scala.util.Random
+    val rng = new Random(3)
+
+    val a1 = Array.tabulate(200)(_ => rng.nextDouble * 2.0 - 1.0) ++
+      Array.fill(20)(0.0) ++ Array.fill(20)(-0.0)
+
+    val df1 = sc.parallelize(a1, 2).toDF("id")
+
+    val qd = new QuantileDiscretizer()
+      .setInputCol("id")
+      .setOutputCol("out")
+      .setNumBuckets(200)
+      .setRelativeError(0.0)
+
+    qd.fit(df1) // assert no exception raised here.
+  }
 }

From 6834f4691b3e2603d410bfe24f0db0b6e3a36446 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 14 May 2020 10:54:35 -0500
Subject: [PATCH 0672/1280] [SPARK-31681][ML][PYSPARK] Python multiclass
 logistic regression evaluate should return LogisticRegressionSummary

### What changes were proposed in this pull request?
Return LogisticRegressionSummary for multiclass logistic regression evaluate in PySpark

### Why are the changes needed?
Currently we have
```
    since("2.0.0")
    def evaluate(self, dataset):
        if not isinstance(dataset, DataFrame):
            raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
        java_blr_summary = self._call_java("evaluate", dataset)
        return BinaryLogisticRegressionSummary(java_blr_summary)
```
we should return LogisticRegressionSummary for multiclass logistic regression

### Does this PR introduce _any_ user-facing change?
Yes
return LogisticRegressionSummary instead of BinaryLogisticRegressionSummary for multiclass logistic regression in Python

### How was this patch tested?
unit test

Closes #28503 from huaxingao/lr_summary.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit e10516ae63cfc58f2d493e4d3f19940d45c8f033)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 python/pyspark/ml/classification.py              | 5 ++++-
 python/pyspark/ml/tests/test_training_summary.py | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 1436b78b11202..424e16a7abf47 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -831,7 +831,10 @@ def evaluate(self, dataset):
         if not isinstance(dataset, DataFrame):
             raise ValueError("dataset must be a DataFrame but got %s." % type(dataset))
         java_blr_summary = self._call_java("evaluate", dataset)
-        return BinaryLogisticRegressionSummary(java_blr_summary)
+        if self.numClasses <= 2:
+            return BinaryLogisticRegressionSummary(java_blr_summary)
+        else:
+            return LogisticRegressionSummary(java_blr_summary)
 
 
 class LogisticRegressionSummary(JavaWrapper):
diff --git a/python/pyspark/ml/tests/test_training_summary.py b/python/pyspark/ml/tests/test_training_summary.py
index 1d19ebf9a34a0..b5054095d190b 100644
--- a/python/pyspark/ml/tests/test_training_summary.py
+++ b/python/pyspark/ml/tests/test_training_summary.py
@@ -21,7 +21,8 @@
 if sys.version > '3':
     basestring = str
 
-from pyspark.ml.classification import LogisticRegression
+from pyspark.ml.classification import BinaryLogisticRegressionSummary, LogisticRegression, \
+    LogisticRegressionSummary
 from pyspark.ml.clustering import BisectingKMeans, GaussianMixture, KMeans
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.regression import GeneralizedLinearRegression, LinearRegression
@@ -149,6 +150,7 @@ def test_binary_logistic_regression_summary(self):
         # test evaluation (with training dataset) produces a summary with same values
         # one check is enough to verify a summary is returned, Scala version runs full test
         sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, BinaryLogisticRegressionSummary))
         self.assertAlmostEqual(sameSummary.areaUnderROC, s.areaUnderROC)
 
     def test_multiclass_logistic_regression_summary(self):
@@ -187,6 +189,8 @@ def test_multiclass_logistic_regression_summary(self):
         # test evaluation (with training dataset) produces a summary with same values
         # one check is enough to verify a summary is returned, Scala version runs full test
         sameSummary = model.evaluate(df)
+        self.assertTrue(isinstance(sameSummary, LogisticRegressionSummary))
+        self.assertFalse(isinstance(sameSummary, BinaryLogisticRegressionSummary))
         self.assertAlmostEqual(sameSummary.accuracy, s.accuracy)
 
     def test_gaussian_mixture_summary(self):

From 541d451cd7062f04e53d93719034877f27452a3e Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 13 May 2020 13:59:42 -0700
Subject: [PATCH 0673/1280] [SPARK-31696][K8S] Support driver service
 annotation in K8S

### What changes were proposed in this pull request?

This PR aims to add `spark.kubernetes.driver.service.annotation` like `spark.kubernetes.driver.service.annotation`.

### Why are the changes needed?

Annotations are used in many ways. One example is that Prometheus monitoring system search metric endpoint via annotation.
- https://github.com/helm/charts/tree/master/stable/prometheus#scraping-pod-metrics-via-annotations

### Does this PR introduce _any_ user-facing change?

Yes. The documentation is added.

### How was this patch tested?

Pass Jenkins with the updated unit tests.

Closes #28518 from dongjoon-hyun/SPARK-31696.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c8f3bd861d96cf3f7b01cd9f864c181a57e1c77a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/running-on-kubernetes.md                       | 13 +++++++++++--
 .../scala/org/apache/spark/deploy/k8s/Config.scala  |  1 +
 .../apache/spark/deploy/k8s/KubernetesConf.scala    |  5 +++++
 .../k8s/features/DriverServiceFeatureStep.scala     |  1 +
 .../spark/deploy/k8s/KubernetesTestConf.scala       |  2 ++
 .../features/DriverServiceFeatureStepSuite.scala    | 13 +++++++++++--
 6 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 4f228a5a101f5..2739149d531d5 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -803,11 +803,20 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>spark.kubernetes.driver.annotation.[AnnotationName]</code></td>
   <td>(none)</td>
   <td>
-    Add the annotation specified by <code>AnnotationName</code> to the driver pod.
+    Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the driver pod.
     For example, <code>spark.kubernetes.driver.annotation.something=true</code>.
   </td>
   <td>2.3.0</td>
 </tr>
+<tr>
+  <td><code>spark.kubernetes.driver.service.annotation.[AnnotationName]</code></td>
+  <td>(none)</td>
+  <td>
+    Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the driver service.
+    For example, <code>spark.kubernetes.driver.service.annotation.something=true</code>.
+  </td>
+  <td>3.1.0</td>
+</tr>
 <tr>
   <td><code>spark.kubernetes.executor.label.[LabelName]</code></td>
   <td>(none)</td>
@@ -823,7 +832,7 @@ See the [configuration page](configuration.html) for information on Spark config
   <td><code>spark.kubernetes.executor.annotation.[AnnotationName]</code></td>
   <td>(none)</td>
   <td>
-    Add the annotation specified by <code>AnnotationName</code> to the executor pods.
+    Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the executor pods.
     For example, <code>spark.kubernetes.executor.annotation.something=true</code>.
   </td>
   <td>2.3.0</td>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 8684a602f2760..22f4c7505c09b 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -411,6 +411,7 @@ private[spark] object Config extends Logging {
 
   val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label."
   val KUBERNETES_DRIVER_ANNOTATION_PREFIX = "spark.kubernetes.driver.annotation."
+  val KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX = "spark.kubernetes.driver.service.annotation."
   val KUBERNETES_DRIVER_SECRETS_PREFIX = "spark.kubernetes.driver.secrets."
   val KUBERNETES_DRIVER_SECRET_KEY_REF_PREFIX = "spark.kubernetes.driver.secretKeyRef."
   val KUBERNETES_DRIVER_VOLUMES_PREFIX = "spark.kubernetes.driver.volumes."
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
index 09943b7974ed9..6dcae3e8b0e39 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
@@ -106,6 +106,11 @@ private[spark] class KubernetesDriverConf(
     KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf, KUBERNETES_DRIVER_ANNOTATION_PREFIX)
   }
 
+  def serviceAnnotations: Map[String, String] = {
+    KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf,
+      KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX)
+  }
+
   override def secretNamesToMountPaths: Map[String, String] = {
     KubernetesUtils.parsePrefixedKeyValuePairs(sparkConf, KUBERNETES_DRIVER_SECRETS_PREFIX)
   }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
index 925bcdf3e637f..1e9c60c871479 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStep.scala
@@ -69,6 +69,7 @@ private[spark] class DriverServiceFeatureStep(
     val driverService = new ServiceBuilder()
       .withNewMetadata()
         .withName(resolvedServiceName)
+        .addToAnnotations(kubernetesConf.serviceAnnotations.asJava)
         .endMetadata()
       .withNewSpec()
         .withClusterIP("None")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
index ee830a91f3272..7763b116f98ab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesTestConf.scala
@@ -48,6 +48,7 @@ object KubernetesTestConf {
       labels: Map[String, String] = Map.empty,
       environment: Map[String, String] = Map.empty,
       annotations: Map[String, String] = Map.empty,
+      serviceAnnotations: Map[String, String] = Map.empty,
       secretEnvNamesToKeyRefs: Map[String, String] = Map.empty,
       secretNamesToMountPaths: Map[String, String] = Map.empty,
       volumes: Seq[KubernetesVolumeSpec] = Seq.empty): KubernetesDriverConf = {
@@ -59,6 +60,7 @@ object KubernetesTestConf {
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_LABEL_PREFIX, labels)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_ENV_PREFIX, environment)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_ANNOTATION_PREFIX, annotations)
+    setPrefixedConfigs(conf, KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX, serviceAnnotations)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_SECRETS_PREFIX, secretNamesToMountPaths)
     setPrefixedConfigs(conf, KUBERNETES_DRIVER_SECRET_KEY_REF_PREFIX, secretEnvNamesToKeyRefs)
     setVolumeSpecs(conf, KUBERNETES_DRIVER_VOLUMES_PREFIX, volumes)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
index 9068289bab581..18afd10395566 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/DriverServiceFeatureStepSuite.scala
@@ -38,6 +38,9 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
   private val DRIVER_LABELS = Map(
     "label1key" -> "label1value",
     "label2key" -> "label2value")
+  private val DRIVER_SERVICE_ANNOTATIONS = Map(
+    "annotation1key" -> "annotation1value",
+    "annotation2key" -> "annotation2value")
 
   test("Headless service has a port for the driver RPC, the block manager and driver ui.") {
     val sparkConf = new SparkConf(false)
@@ -46,7 +49,8 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
       .set(UI_PORT, 4080)
     val kconf = KubernetesTestConf.createDriverConf(
       sparkConf = sparkConf,
-      labels = DRIVER_LABELS)
+      labels = DRIVER_LABELS,
+      serviceAnnotations = DRIVER_SERVICE_ANNOTATIONS)
     val configurationStep = new DriverServiceFeatureStep(kconf)
     assert(configurationStep.configurePod(SparkPod.initialPod()) === SparkPod.initialPod())
     assert(configurationStep.getAdditionalKubernetesResources().size === 1)
@@ -79,7 +83,9 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
   }
 
   test("Ports should resolve to defaults in SparkConf and in the service.") {
-    val kconf = KubernetesTestConf.createDriverConf(labels = DRIVER_LABELS)
+    val kconf = KubernetesTestConf.createDriverConf(
+      labels = DRIVER_LABELS,
+      serviceAnnotations = DRIVER_SERVICE_ANNOTATIONS)
     val configurationStep = new DriverServiceFeatureStep(kconf)
     val resolvedService = configurationStep
       .getAdditionalKubernetesResources()
@@ -164,6 +170,9 @@ class DriverServiceFeatureStepSuite extends SparkFunSuite {
     DRIVER_LABELS.foreach { case (k, v) =>
       assert(service.getSpec.getSelector.get(k) === v)
     }
+    DRIVER_SERVICE_ANNOTATIONS.foreach { case (k, v) =>
+      assert(service.getMetadata.getAnnotations.get(k) === v)
+    }
     assert(service.getSpec.getPorts.size() === 3)
     val driverServicePorts = service.getSpec.getPorts.asScala
     assert(driverServicePorts.head.getName === DRIVER_PORT_NAME)

From ca9cde8e76abfdb73f0202fbf043e4d895d7163b Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 14 May 2020 10:25:22 -0700
Subject: [PATCH 0674/1280] [SPARK-31696][DOCS][FOLLOWUP] Update version in
 documentation

# What changes were proposed in this pull request?

This PR is a follow-up to fix a version of configuration document.

### Why are the changes needed?

The original PR is backported to branch-3.0.

### Does this PR introduce _any_ user-facing change?

Yes.

### How was this patch tested?

Manual.

Closes #28530 from dongjoon-hyun/SPARK-31696-2.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7ce3f76af6b72e88722a89f792e6fded9c586795)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/running-on-kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 2739149d531d5..3abb891ee0d0b 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -815,7 +815,7 @@ See the [configuration page](configuration.html) for information on Spark config
     Add the Kubernetes <a href="https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/">annotation</a> specified by <code>AnnotationName</code> to the driver service.
     For example, <code>spark.kubernetes.driver.service.annotation.something=true</code>.
   </td>
-  <td>3.1.0</td>
+  <td>3.0.0</td>
 </tr>
 <tr>
   <td><code>spark.kubernetes.executor.label.[LabelName]</code></td>

From cf708f970b640722062b3102b8757a554aa0c841 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 14 May 2020 12:06:13 -0700
Subject: [PATCH 0675/1280] Revert "[SPARK-31387] Handle unknown
 operation/session ID in HiveThriftServer2Listener"

This reverts commit 512cb2f0246a0d020f0ba726b4596555b15797c6.
---
 .../ui/HiveThriftServer2Listener.scala        | 120 +++++++-----------
 .../thriftserver/HiveSessionImplSuite.scala   |  73 -----------
 .../ui/HiveThriftServer2ListenerSuite.scala   |  16 ---
 .../service/cli/session/HiveSessionImpl.java  |   6 +-
 .../service/cli/session/HiveSessionImpl.java  |   6 +-
 5 files changed, 51 insertions(+), 170 deletions(-)
 delete mode 100644 sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
index 20a8f2c833c21..6d0a506fa94dc 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
@@ -25,7 +25,6 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.hive.service.server.HiveServer2
 
 import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Status.LIVE_ENTITY_UPDATE_PERIOD
 import org.apache.spark.scheduler._
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.ExecutionState
@@ -39,7 +38,7 @@ private[thriftserver] class HiveThriftServer2Listener(
     kvstore: ElementTrackingStore,
     sparkConf: SparkConf,
     server: Option[HiveServer2],
-    live: Boolean = true) extends SparkListener with Logging {
+    live: Boolean = true) extends SparkListener {
 
   private val sessionList = new ConcurrentHashMap[String, LiveSessionData]()
   private val executionList = new ConcurrentHashMap[String, LiveExecutionData]()
@@ -132,81 +131,60 @@ private[thriftserver] class HiveThriftServer2Listener(
     updateLiveStore(session)
   }
 
-  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit =
-    Option(sessionList.get(e.sessionId)) match {
-      case None => logWarning(s"onSessionClosed called with unknown session id: ${e.sessionId}")
-      case Some(sessionData) =>
-        val session = sessionData
-        session.finishTimestamp = e.finishTime
-        updateStoreWithTriggerEnabled(session)
-        sessionList.remove(e.sessionId)
-    }
+  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit = {
+    val session = sessionList.get(e.sessionId)
+    session.finishTimestamp = e.finishTime
+    updateStoreWithTriggerEnabled(session)
+    sessionList.remove(e.sessionId)
+  }
 
-  private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit =
-    Option(sessionList.get(e.sessionId)) match {
-      case None => logWarning(s"onOperationStart called with unknown session id: ${e.sessionId}")
-      case Some(sessionData) =>
-        val info = getOrCreateExecution(
-          e.id,
-          e.statement,
-          e.sessionId,
-          e.startTime,
-          e.userName)
-
-        info.state = ExecutionState.STARTED
-        executionList.put(e.id, info)
-        sessionData.totalExecution += 1
-        executionList.get(e.id).groupId = e.groupId
-        updateLiveStore(executionList.get(e.id))
-        updateLiveStore(sessionData)
-    }
+  private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit = {
+    val info = getOrCreateExecution(
+      e.id,
+      e.statement,
+      e.sessionId,
+      e.startTime,
+      e.userName)
+
+    info.state = ExecutionState.STARTED
+    executionList.put(e.id, info)
+    sessionList.get(e.sessionId).totalExecution += 1
+    executionList.get(e.id).groupId = e.groupId
+    updateLiveStore(executionList.get(e.id))
+    updateLiveStore(sessionList.get(e.sessionId))
+  }
 
-  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit =
-    Option(executionList.get(e.id)) match {
-      case None => logWarning(s"onOperationParsed called with unknown operation id: ${e.id}")
-      case Some(executionData) =>
-        executionData.executePlan = e.executionPlan
-        executionData.state = ExecutionState.COMPILED
-        updateLiveStore(executionData)
-    }
+  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit = {
+    executionList.get(e.id).executePlan = e.executionPlan
+    executionList.get(e.id).state = ExecutionState.COMPILED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit =
-    Option(executionList.get(e.id)) match {
-      case None => logWarning(s"onOperationCanceled called with unknown operation id: ${e.id}")
-      case Some(executionData) =>
-        executionData.finishTimestamp = e.finishTime
-        executionData.state = ExecutionState.CANCELED
-        updateLiveStore(executionData)
-    }
+  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit = {
+    executionList.get(e.id).finishTimestamp = e.finishTime
+    executionList.get(e.id).state = ExecutionState.CANCELED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit =
-    Option(executionList.get(e.id)) match {
-      case None => logWarning(s"onOperationError called with unknown operation id: ${e.id}")
-      case Some(executionData) =>
-        executionData.finishTimestamp = e.finishTime
-        executionData.detail = e.errorMsg
-        executionData.state = ExecutionState.FAILED
-        updateLiveStore(executionData)
-    }
+  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit = {
+    executionList.get(e.id).finishTimestamp = e.finishTime
+    executionList.get(e.id).detail = e.errorMsg
+    executionList.get(e.id).state = ExecutionState.FAILED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit =
-    Option(executionList.get(e.id)) match {
-      case None => logWarning(s"onOperationFinished called with unknown operation id: ${e.id}")
-      case Some(executionData) =>
-        executionData.finishTimestamp = e.finishTime
-        executionData.state = ExecutionState.FINISHED
-        updateLiveStore(executionData)
-    }
+  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit = {
+    executionList.get(e.id).finishTimestamp = e.finishTime
+    executionList.get(e.id).state = ExecutionState.FINISHED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit =
-    Option(executionList.get(e.id)) match {
-      case None => logWarning(s"onOperationClosed called with unknown operation id: ${e.id}")
-      case Some(executionData) =>
-        executionData.closeTimestamp = e.closeTime
-        executionData.state = ExecutionState.CLOSED
-        updateStoreWithTriggerEnabled(executionData)
-        executionList.remove(e.id)
-    }
+  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit = {
+    executionList.get(e.id).closeTimestamp = e.closeTime
+    executionList.get(e.id).state = ExecutionState.CLOSED
+    updateStoreWithTriggerEnabled(executionList.get(e.id))
+    executionList.remove(e.id)
+  }
 
   // Update both live and history stores. Trigger is enabled by default, hence
   // it will cleanup the entity which exceeds the threshold.
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
deleted file mode 100644
index 05d540d782e31..0000000000000
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive.thriftserver
-
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hive.service.cli.OperationHandle
-import org.apache.hive.service.cli.operation.{GetCatalogsOperation, OperationManager}
-import org.apache.hive.service.cli.session.{HiveSessionImpl, SessionManager}
-import org.mockito.Mockito.{mock, verify, when}
-import org.mockito.invocation.InvocationOnMock
-
-import org.apache.spark.SparkFunSuite
-
-class HiveSessionImplSuite extends SparkFunSuite {
-  private var session: HiveSessionImpl = _
-  private var operationManager: OperationManager = _
-
-  override def beforeAll() {
-    super.beforeAll()
-
-    session = new HiveSessionImpl(
-      ThriftserverShimUtils.testedProtocolVersions.head,
-      "",
-      "",
-      new HiveConf(),
-      ""
-    )
-    val sessionManager = mock(classOf[SessionManager])
-    session.setSessionManager(sessionManager)
-    operationManager = mock(classOf[OperationManager])
-    session.setOperationManager(operationManager)
-    when(operationManager.newGetCatalogsOperation(session)).thenAnswer(
-      (_: InvocationOnMock) => {
-        val operation = mock(classOf[GetCatalogsOperation])
-        when(operation.getHandle).thenReturn(mock(classOf[OperationHandle]))
-        operation
-      }
-    )
-
-    session.open(Map.empty[String, String].asJava)
-  }
-
-  test("SPARK-31387 - session.close() closes all sessions regardless of thrown exceptions") {
-    val operationHandle1 = session.getCatalogs
-    val operationHandle2 = session.getCatalogs
-
-    when(operationManager.closeOperation(operationHandle1))
-      .thenThrow(classOf[NullPointerException])
-    when(operationManager.closeOperation(operationHandle2))
-      .thenThrow(classOf[NullPointerException])
-
-    session.close()
-
-    verify(operationManager).closeOperation(operationHandle1)
-    verify(operationManager).closeOperation(operationHandle2)
-  }
-}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
index ea2523dfb581e..075032fa5d099 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
@@ -140,22 +140,6 @@ class HiveThriftServer2ListenerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(listener.noLiveData())
   }
 
-  test("SPARK-31387 - listener update methods should not throw exception with unknown input") {
-    val (statusStore: HiveThriftServer2AppStatusStore,
-    listener: HiveThriftServer2Listener) = createAppStatusStore(true)
-    val unknownSession = "unknown_session"
-    val unknownOperation = "unknown_operation"
-    listener.onOtherEvent(SparkListenerThriftServerSessionClosed(unknownSession, 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationStart("id", unknownSession,
-      "stmt", "groupId", 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationParsed(unknownOperation, "query"))
-    listener.onOtherEvent(SparkListenerThriftServerOperationCanceled(unknownOperation, 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationError(unknownOperation,
-      "msg", "trace", 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationFinish(unknownOperation, 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationClosed(unknownOperation, 0))
-  }
-
   private def createProperties: Properties = {
     val properties = new Properties()
     properties.setProperty(SparkContext.SPARK_JOB_GROUP_ID, "groupId")
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 3e2c3def48d89..745f385e87f78 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -636,11 +636,7 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        try {
-          operationManager.closeOperation(opHandle);
-        } catch (Exception e) {
-          LOG.warn("Exception is thrown closing operation " + opHandle, e);
-        }
+        operationManager.closeOperation(opHandle);
       }
       opHandleSet.clear();
       // Cleanup session log directory.
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 5cdae00460e34..14e9c4704c977 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -650,11 +650,7 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        try {
-          operationManager.closeOperation(opHandle);
-        } catch (Exception e) {
-          LOG.warn("Exception is thrown closing operation " + opHandle, e);
-        }
+        operationManager.closeOperation(opHandle);
       }
       opHandleSet.clear();
       // Cleanup session log directory.

From 8ba157808bfd962d00e7f4dde2ce81870f8cc7b5 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 14 May 2020 19:28:25 -0700
Subject: [PATCH 0676/1280] [SPARK-31713][INFRA] Make test-dependencies.sh
 detect version string correctly

### What changes were proposed in this pull request?

This PR makes `test-dependencies.sh` detect the version string correctly by ignoring all the other lines.

### Why are the changes needed?

Currently, all SBT jobs are broken like the following.
- https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test%20(Dashboard)/job/spark-branch-3.0-test-sbt-hadoop-3.2-hive-2.3/476/console
```
[error] running /home/jenkins/workspace/spark-branch-3.0-test-sbt-hadoop-3.2-hive-2.3/dev/test-dependencies.sh ; received return code 1
Build step 'Execute shell' marked build as failure
```

The reason is that the script detects the old version like `Falling back to archive.apache.org to download Maven 3.1.0-SNAPSHOT` when `build/mvn` did fallback.

Specifically, in the script, `OLD_VERSION` became `Falling back to archive.apache.org to download Maven 3.1.0-SNAPSHOT` instead of `3.1.0-SNAPSHOT` if build/mvn did fallback. Then, `pom.xml` file is corrupted like the following at the end and the exit code become `1` instead of `0`. It causes Jenkins jobs fails
```
-    <version>3.1.0-SNAPSHOT</version>
+    <version>Falling</version>
```

**NO FALLBACK**
```
$ build/mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.6.0:exec
Using `mvn` from path: /Users/dongjoon/APACHE/spark-merge/build/apache-maven-3.6.3/bin/mvn
3.1.0-SNAPSHOT
```

**FALLBACK**
```
$ build/mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.6.0:exec
Falling back to archive.apache.org to download Maven
Using `mvn` from path: /Users/dongjoon/APACHE/spark-merge/build/apache-maven-3.6.3/bin/mvn
3.1.0-SNAPSHOT
```

**In the script**
```
$ echo $(build/mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:1.6.0:exec)
Using `mvn` from path: /Users/dongjoon/APACHE/spark-merge/build/apache-maven-3.6.3/bin/mvn
Falling back to archive.apache.org to download Maven 3.1.0-SNAPSHOT
```

This PR will prevent irrelevant logs like `Falling back to archive.apache.org to download Maven`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the PR Builder.

Closes #28532 from dongjoon-hyun/SPARK-31713.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit cd5fbcf9a0151f10553f67bcaa22b8122b3cf263)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/test-dependencies.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh
index 936ac00f6b9e7..46f1f55938120 100755
--- a/dev/test-dependencies.sh
+++ b/dev/test-dependencies.sh
@@ -47,7 +47,7 @@ OLD_VERSION=$($MVN -q \
     -Dexec.executable="echo" \
     -Dexec.args='${project.version}' \
     --non-recursive \
-    org.codehaus.mojo:exec-maven-plugin:1.6.0:exec)
+    org.codehaus.mojo:exec-maven-plugin:1.6.0:exec | grep -E '[0-9]+\.[0-9]+\.[0-9]+')
 if [ $? != 0 ]; then
     echo -e "Error while getting version string from Maven:\n$OLD_VERSION"
     exit 1

From 635feaa0ef6298e336a447e1fdcfeca403b741bd Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 15 May 2020 04:24:58 +0000
Subject: [PATCH 0677/1280] [SPARK-31712][SQL][TESTS] Check casting timestamps
 before the epoch to Byte/Short/Int/Long types

### What changes were proposed in this pull request?
Added tests to check casting timestamps before 1970-01-01 00:00:00Z to ByteType, ShortType, IntegerType and LongType in ansi and non-ansi modes.

### Why are the changes needed?
To improve test coverage and prevent errors while modifying the CAST expression code.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the modified test suites:
```
$ ./build/sbt "test:testOnly *CastSuite"
```

Closes #28531 from MaxGekk/test-cast-timestamp-to-byte.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c7ce37dfa713f80c5f0157719f0e3d9bf0d271dd)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/expressions/CastSuite.scala  | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 5c57843980374..334b43e79ffc7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -1299,6 +1299,18 @@ class CastSuite extends CastSuiteBase {
       }
     }
   }
+
+  test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
+    withDefaultTimeZone(UTC) {
+      val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
+      assert(negativeTs.getTime < 0)
+      val expectedSecs = Math.floorDiv(negativeTs.getTime, MILLIS_PER_SECOND)
+      checkEvaluation(cast(negativeTs, ByteType), expectedSecs.toByte)
+      checkEvaluation(cast(negativeTs, ShortType), expectedSecs.toShort)
+      checkEvaluation(cast(negativeTs, IntegerType), expectedSecs.toInt)
+      checkEvaluation(cast(negativeTs, LongType), expectedSecs)
+    }
+  }
 }
 
 /**
@@ -1341,4 +1353,19 @@ class AnsiCastSuite extends CastSuiteBase {
         cast("abc.com", dataType), "invalid input")
     }
   }
+
+  test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
+    withDefaultTimeZone(UTC) {
+      val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
+      assert(negativeTs.getTime < 0)
+      val expectedSecs = Math.floorDiv(negativeTs.getTime, MILLIS_PER_SECOND)
+      checkExceptionInExpression[ArithmeticException](
+        cast(negativeTs, ByteType), "to byte causes overflow")
+      checkExceptionInExpression[ArithmeticException](
+        cast(negativeTs, ShortType), "to short causes overflow")
+      checkExceptionInExpression[ArithmeticException](
+        cast(negativeTs, IntegerType), "to int causes overflow")
+      checkEvaluation(cast(negativeTs, LongType), expectedSecs)
+    }
+  }
 }

From d270be43e38f1b44a99c4035cc1829751c223346 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 15 May 2020 06:36:34 +0000
Subject: [PATCH 0678/1280] [SPARK-31715][SQL][TEST] Fix flaky SparkSQLEnvSuite
 that sometimes varies single derby instance standard
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/122622/testReport/junit/org.apache.spark.sql.hive.thriftserver/SparkSQLEnvSuite/SPARK_29604_external_listeners_should_be_initialized_with_Spark_classloader/history/?start=25

According to the test report history of SparkSQLEnvSuite，this test fails frequently which is caused by single derby instance restriction.

```java
Caused by: sbt.ForkMain$ForkError: org.apache.derby.iapi.error.StandardException: Another instance of Derby may have already booted the database /home/jenkins/workspace/SparkPullRequestBuilder/sql/hive-thriftserver/metastore_db.
	at org.apache.derby.iapi.error.StandardException.newException(Unknown Source)
	at org.apache.derby.impl.store.raw.data.BaseDataFileFactory.privGetJBMSLockOnDB(Unknown Source)
	at org.apache.derby.impl.store.raw.data.BaseDataFileFactory.run(Unknown Source)
	at java.security.AccessController.doPrivileged(Native Method)
	at org.apache.derby.impl.store.raw.data.BaseDataFileFactory.getJBMSLockOnDB(Unknown Source)
	at org.apache.derby.impl.store.raw.data.BaseDataFileFactory.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)
	at org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)
	at org.apache.derby.impl.store.raw.RawStore.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)
	at org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)
	at org.apache.derby.impl.store.access.RAMAccessManager.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.startModule(Unknown Source)
	at org.apache.derby.iapi.services.monitor.Monitor.bootServiceModule(Unknown Source)
	at org.apache.derby.impl.db.BasicDatabase.bootStore(Unknown Source)
	at org.apache.derby.impl.db.BasicDatabase.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.boot(Unknown Source)
	at org.apache.derby.impl.services.monitor.TopService.bootModule(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.bootService(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.startProviderService(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.findProviderAndStartService(Unknown Source)
	at org.apache.derby.impl.services.monitor.BaseMonitor.startPersistentService(Unknown Source)
	at org.apache.derby.iapi.services.monitor.Monitor.startPersistentService(Unknown Source)
	... 138 more
```

This PR adds a separate directory to locate the metastore_db for this test which runs in a dedicated JVM.

Besides, diable the UI for the potential race on `spark.ui.port` which may also let the test case become flaky.

### Why are the changes needed?
test fix

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
SparkSQLEnvSuite itself.

Closes #28537 from yaooqinn/SPARK-31715.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 503faa24d33eb52da79ad99e39c8c011597499ea)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
index ffd1fc48f19fe..f28faea2be868 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnvSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import org.apache.commons.io.FileUtils
 import test.custom.listener.{DummyQueryExecutionListener, DummyStreamingQueryListener}
 
 import org.apache.spark.SparkFunSuite
@@ -25,10 +26,19 @@ import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.hive.HiveUtils.{HIVE_METASTORE_JARS, HIVE_METASTORE_VERSION}
 import org.apache.spark.sql.hive.test.TestHiveContext
 import org.apache.spark.sql.internal.StaticSQLConf.{QUERY_EXECUTION_LISTENERS, STREAMING_QUERY_LISTENERS, WAREHOUSE_PATH}
+import org.apache.spark.util.Utils
 
 class SparkSQLEnvSuite extends SparkFunSuite {
   test("SPARK-29604 external listeners should be initialized with Spark classloader") {
+    val metastorePath = Utils.createTempDir("spark_derby")
+    FileUtils.forceDelete(metastorePath)
+
+    val jdbcUrl = s"jdbc:derby:;databaseName=$metastorePath;create=true"
+
     withSystemProperties(
+      "javax.jdo.option.ConnectionURL" -> jdbcUrl,
+      "derby.system.durability" -> "test",
+      "spark.ui.enabled" -> "false",
       QUERY_EXECUTION_LISTENERS.key -> classOf[DummyQueryExecutionListener].getCanonicalName,
       STREAMING_QUERY_LISTENERS.key -> classOf[DummyStreamingQueryListener].getCanonicalName,
       WAREHOUSE_PATH.key -> TestHiveContext.makeWarehouseDir().toURI.getPath,

From 435f12699a5d66b97053c916ff4030efee648522 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 15 May 2020 00:30:25 -0700
Subject: [PATCH 0679/1280] [SPARK-31716][SQL] Use fallback versions in
 HiveExternalCatalogVersionsSuite

# What changes were proposed in this pull request?

This PR aims to provide a fallback version instead of `Nil` in `HiveExternalCatalogVersionsSuite`. The provided fallback Spark versions recovers Jenkins jobs instead of failing.

### Why are the changes needed?

Currently, `HiveExternalCatalogVersionsSuite` is aborted in all Jenkins jobs except JDK11 Jenkins jobs which don't have old Spark releases supporting JDK11.
```
HiveExternalCatalogVersionsSuite:
org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite *** ABORTED ***
  Exception encountered when invoking run on a nested suite - Fail to get the lates Spark versions to test. (HiveExternalCatalogVersionsSuite.scala:180)
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins

Closes #28536 from dongjoon-hyun/SPARK-HiveExternalCatalogVersionsSuite.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 5d90886523c415768c65ea9cba7db24bc508a23b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 3b5a1247bc09c..8be3d26bfc93a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -242,7 +242,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
         .filter(_ < org.apache.spark.SPARK_VERSION)
     } catch {
       // do not throw exception during object initialization.
-      case NonFatal(_) => Nil
+      case NonFatal(_) => Seq("2.3.4", "2.4.5") // A temporary fallback to use a specific version
     }
   }
 

From bdb57853d8a0b87fde8359c11626af87a97b1bd0 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 15 May 2020 15:36:28 +0000
Subject: [PATCH 0680/1280] [SPARK-31620][SQL] Fix reference binding failure in
 case of an final agg contains subquery

### What changes were proposed in this pull request?

Instead of using `child.output` directly, we should use `inputAggBufferAttributes` from the current agg expression  for `Final` and `PartialMerge` aggregates to bind references for their `mergeExpression`.

### Why are the changes needed?

When planning aggregates, the partial aggregate uses agg fucs' `inputAggBufferAttributes` as its output, see https://github.com/apache/spark/blob/v3.0.0-rc1/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala#L105

For final `HashAggregateExec`, we need to bind the `DeclarativeAggregate.mergeExpressions` with the output of the partial aggregate operator, see https://github.com/apache/spark/blob/v3.0.0-rc1/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala#L348

This is usually fine. However, if we copy the agg func somehow after agg planning, like `PlanSubqueries`, the `DeclarativeAggregate` will be replaced by a new instance with new `inputAggBufferAttributes` and `mergeExpressions`. Then we can't bind the `mergeExpressions` with the output of the partial aggregate operator, as it uses the `inputAggBufferAttributes` of the original `DeclarativeAggregate` before copy.

Note that, `ImperativeAggregate` doesn't have this problem, as we don't need to bind its `mergeExpressions`. It has a different mechanism to access buffer values, via `mutableAggBufferOffset` and `inputAggBufferOffset`.

### Does this PR introduce _any_ user-facing change?

Yes, user hit error previously but run query successfully after this change.

### How was this patch tested?

Added a regression test.

Closes #28496 from Ngone51/spark-31620.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d8b001fa872f735df4344321c33780a892da9b41)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../aggregate/BaseAggregateExec.scala         | 26 ++++++++++++-
 .../aggregate/HashAggregateExec.scala         |  6 +--
 .../aggregate/ObjectHashAggregateExec.scala   |  2 +-
 .../aggregate/SortAggregateExec.scala         |  2 +-
 .../spark/sql/DataFrameAggregateSuite.scala   | 39 +++++++++++++++++++
 5 files changed, 69 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
index f506bdddc16b5..f1e053f7fb2a5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/BaseAggregateExec.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution.aggregate
 
 import org.apache.spark.sql.catalyst.expressions.{Attribute, NamedExpression}
-import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Final, PartialMerge}
 import org.apache.spark.sql.execution.{ExplainUtils, UnaryExecNode}
 
 /**
@@ -40,4 +40,28 @@ trait BaseAggregateExec extends UnaryExecNode {
        |${ExplainUtils.generateFieldString("Results", resultExpressions)}
        |""".stripMargin
   }
+
+  protected def inputAttributes: Seq[Attribute] = {
+    val modes = aggregateExpressions.map(_.mode).distinct
+    if (modes.contains(Final) || modes.contains(PartialMerge)) {
+      // SPARK-31620: when planning aggregates, the partial aggregate uses aggregate function's
+      // `inputAggBufferAttributes` as its output. And Final and PartialMerge aggregate rely on the
+      // output to bind references for `DeclarativeAggregate.mergeExpressions`. But if we copy the
+      // aggregate function somehow after aggregate planning, like `PlanSubqueries`, the
+      // `DeclarativeAggregate` will be replaced by a new instance with new
+      // `inputAggBufferAttributes` and `mergeExpressions`. Then Final and PartialMerge aggregate
+      // can't bind the `mergeExpressions` with the output of the partial aggregate, as they use
+      // the `inputAggBufferAttributes` of the original `DeclarativeAggregate` before copy. Instead,
+      // we shall use `inputAggBufferAttributes` after copy to match the new `mergeExpressions`.
+      val aggAttrs = aggregateExpressions
+        // there're exactly four cases needs `inputAggBufferAttributes` from child according to the
+        // agg planning in `AggUtils`: Partial -> Final, PartialMerge -> Final,
+        // Partial -> PartialMerge, PartialMerge -> PartialMerge.
+        .filter(a => a.mode == Final || a.mode == PartialMerge).map(_.aggregateFunction)
+        .flatMap(_.inputAggBufferAttributes)
+      child.output.dropRight(aggAttrs.length) ++ aggAttrs
+    } else {
+      child.output
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index 87a408174be92..56641bee2111b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -127,7 +127,7 @@ case class HashAggregateExec(
             resultExpressions,
             (expressions, inputSchema) =>
               MutableProjection.create(expressions, inputSchema),
-            child.output,
+            inputAttributes,
             iter,
             testFallbackStartsAt,
             numOutputRows,
@@ -332,7 +332,7 @@ case class HashAggregateExec(
   private def doConsumeWithoutKeys(ctx: CodegenContext, input: Seq[ExprCode]): String = {
     // only have DeclarativeAggregate
     val functions = aggregateExpressions.map(_.aggregateFunction.asInstanceOf[DeclarativeAggregate])
-    val inputAttrs = functions.flatMap(_.aggBufferAttributes) ++ child.output
+    val inputAttrs = functions.flatMap(_.aggBufferAttributes) ++ inputAttributes
     // To individually generate code for each aggregate function, an element in `updateExprs` holds
     // all the expressions for the buffer of an aggregation function.
     val updateExprs = aggregateExpressions.map { e =>
@@ -929,7 +929,7 @@ case class HashAggregateExec(
       }
     }
 
-    val inputAttr = aggregateBufferAttributes ++ child.output
+    val inputAttr = aggregateBufferAttributes ++ inputAttributes
     // Here we set `currentVars(0)` to `currentVars(numBufferSlots)` to null, so that when
     // generating code for buffer columns, we use `INPUT_ROW`(will be the buffer row), while
     // generating input columns, we use `currentVars`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
index 3fb58eb2cc8ba..f1c0719ff8948 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectHashAggregateExec.scala
@@ -123,7 +123,7 @@ case class ObjectHashAggregateExec(
             resultExpressions,
             (expressions, inputSchema) =>
               MutableProjection.create(expressions, inputSchema),
-            child.output,
+            inputAttributes,
             iter,
             fallbackCountThreshold,
             numOutputRows)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
index 77ed469016fa3..c23e19d3b6262 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/SortAggregateExec.scala
@@ -86,7 +86,7 @@ case class SortAggregateExec(
         val outputIter = new SortBasedAggregationIterator(
           partIndex,
           groupingExpressions,
-          child.output,
+          inputAttributes,
           iter,
           aggregateExpressions,
           aggregateAttributes,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 4edf3a5d39fdd..2293d4ae61aff 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -973,4 +973,43 @@ class DataFrameAggregateSuite extends QueryTest
       assert(error.message.contains("function count_if requires boolean type"))
     }
   }
+
+  Seq(true, false).foreach { value =>
+    test(s"SPARK-31620: agg with subquery (whole-stage-codegen = $value)") {
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> value.toString) {
+        withTempView("t1", "t2") {
+          sql("create temporary view t1 as select * from values (1, 2) as t1(a, b)")
+          sql("create temporary view t2 as select * from values (3, 4) as t2(c, d)")
+
+          // test without grouping keys
+          checkAnswer(sql("select sum(if(c > (select a from t1), d, 0)) as csum from t2"),
+            Row(4) :: Nil)
+
+          // test with grouping keys
+          checkAnswer(sql("select c, sum(if(c > (select a from t1), d, 0)) as csum from " +
+            "t2 group by c"), Row(3, 4) :: Nil)
+
+          // test with distinct
+          checkAnswer(sql("select avg(distinct(d)), sum(distinct(if(c > (select a from t1)," +
+            " d, 0))) as csum from t2 group by c"), Row(4, 4) :: Nil)
+
+          // test subquery with agg
+          checkAnswer(sql("select sum(distinct(if(c > (select sum(distinct(a)) from t1)," +
+            " d, 0))) as csum from t2 group by c"), Row(4) :: Nil)
+
+          // test SortAggregateExec
+          var df = sql("select max(if(c > (select a from t1), 'str1', 'str2')) as csum from t2")
+          assert(df.queryExecution.executedPlan
+            .find { case _: SortAggregateExec => true }.isDefined)
+          checkAnswer(df, Row("str1") :: Nil)
+
+          // test ObjectHashAggregateExec
+          df = sql("select collect_list(d), sum(if(c > (select a from t1), d, 0)) as csum from t2")
+          assert(df.queryExecution.executedPlan
+            .find { case _: ObjectHashAggregateExec => true }.isDefined)
+          checkAnswer(df, Row(Array(4), 4) :: Nil)
+        }
+      }
+    }
+  }
 }

From f9486d57b45a2e8c93ec41939d748a252a993598 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sat, 16 May 2020 04:37:18 +0000
Subject: [PATCH 0681/1280] [SPARK-31663][SQL] Grouping sets with having clause
 returns the wrong result

- Resolve the havingcondition with expanding the GROUPING SETS/CUBE/ROLLUP expressions together in `ResolveGroupingAnalytics`:
    - Change the operations resolving directions to top-down.
    - Try resolving the condition of the filter as though it is in the aggregate clause by reusing the function in `ResolveAggregateFunctions`
    - Push the aggregate expressions into the aggregate which contains the expanded operations.
- Use UnresolvedHaving for all having clause.

Correctness bug fix. See the demo and analysis in SPARK-31663.

Yes, correctness bug fix for HAVING with GROUPING SETS.

New UTs added.

Closes #28501 from xuanyuanking/SPARK-31663.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 86bd37f37eb1e534c520dc9a02387debf9fa05a1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 136 ++++++++++++++----
 .../sql/catalyst/analysis/unresolved.scala    |   7 +-
 .../spark/sql/catalyst/dsl/package.scala      |   2 +-
 .../sql/catalyst/parser/AstBuilder.scala      |   7 +-
 .../resources/sql-tests/inputs/having.sql     |   7 +-
 .../sql-tests/results/having.sql.out          |  28 +++-
 6 files changed, 146 insertions(+), 41 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 152a0c4b5f99a..0d71aee157743 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -530,20 +530,13 @@ class Analyzer(
       }.asInstanceOf[NamedExpression]
     }
 
-    /*
-     * Construct [[Aggregate]] operator from Cube/Rollup/GroupingSets.
-     */
-    private def constructAggregate(
+    private def getFinalGroupByExpressions(
         selectedGroupByExprs: Seq[Seq[Expression]],
-        groupByExprs: Seq[Expression],
-        aggregationExprs: Seq[NamedExpression],
-        child: LogicalPlan): LogicalPlan = {
-      val gid = AttributeReference(VirtualColumn.groupingIdName, IntegerType, false)()
-
+        groupByExprs: Seq[Expression]): Seq[Expression] = {
       // In case of ANSI-SQL compliant syntax for GROUPING SETS, groupByExprs is optional and
       // can be null. In such case, we derive the groupByExprs from the user supplied values for
       // grouping sets.
-      val finalGroupByExpressions = if (groupByExprs == Nil) {
+      if (groupByExprs == Nil) {
         selectedGroupByExprs.flatten.foldLeft(Seq.empty[Expression]) { (result, currentExpr) =>
           // Only unique expressions are included in the group by expressions and is determined
           // based on their semantic equality. Example. grouping sets ((a * b), (b * a)) results
@@ -557,6 +550,18 @@ class Analyzer(
       } else {
         groupByExprs
       }
+    }
+
+    /*
+     * Construct [[Aggregate]] operator from Cube/Rollup/GroupingSets.
+     */
+    private def constructAggregate(
+        selectedGroupByExprs: Seq[Seq[Expression]],
+        groupByExprs: Seq[Expression],
+        aggregationExprs: Seq[NamedExpression],
+        child: LogicalPlan): LogicalPlan = {
+      val gid = AttributeReference(VirtualColumn.groupingIdName, IntegerType, false)()
+      val finalGroupByExpressions = getFinalGroupByExpressions(selectedGroupByExprs, groupByExprs)
 
       // Expand works by setting grouping expressions to null as determined by the
       // `selectedGroupByExprs`. To prevent these null values from being used in an aggregate
@@ -588,8 +593,70 @@ class Analyzer(
       }
     }
 
-    // This require transformUp to replace grouping()/grouping_id() in resolved Filter/Sort
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
+    private def tryResolveHavingCondition(h: UnresolvedHaving): LogicalPlan = {
+      val aggForResolving = h.child match {
+        // For CUBE/ROLLUP expressions, to avoid resolving repeatedly, here we delete them from
+        // groupingExpressions for condition resolving.
+        case a @ Aggregate(Seq(c @ Cube(groupByExprs)), _, _) =>
+          a.copy(groupingExpressions = groupByExprs)
+        case a @ Aggregate(Seq(r @ Rollup(groupByExprs)), _, _) =>
+          a.copy(groupingExpressions = groupByExprs)
+        case g: GroupingSets =>
+          Aggregate(
+            getFinalGroupByExpressions(g.selectedGroupByExprs, g.groupByExprs),
+            g.aggregations, g.child)
+      }
+      // Try resolving the condition of the filter as though it is in the aggregate clause
+      val resolvedInfo =
+        ResolveAggregateFunctions.resolveFilterCondInAggregate(h.havingCondition, aggForResolving)
+
+      // Push the aggregate expressions into the aggregate (if any).
+      if (resolvedInfo.nonEmpty) {
+        val (extraAggExprs, resolvedHavingCond) = resolvedInfo.get
+        val newChild = h.child match {
+          case Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, child) =>
+            constructAggregate(
+              cubeExprs(groupByExprs), groupByExprs, aggregateExpressions ++ extraAggExprs, child)
+          case Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, child) =>
+            constructAggregate(
+              rollupExprs(groupByExprs), groupByExprs, aggregateExpressions ++ extraAggExprs, child)
+          case x: GroupingSets =>
+            constructAggregate(
+              x.selectedGroupByExprs, x.groupByExprs, x.aggregations ++ extraAggExprs, x.child)
+        }
+
+        // Since the exprId of extraAggExprs will be changed in the constructed aggregate, and the
+        // aggregateExpressions keeps the input order. So here we build an exprMap to resolve the
+        // condition again.
+        val exprMap = extraAggExprs.zip(
+          newChild.asInstanceOf[Aggregate].aggregateExpressions.takeRight(
+            extraAggExprs.length)).toMap
+        val newCond = resolvedHavingCond.transform {
+          case ne: NamedExpression if exprMap.contains(ne) => exprMap(ne)
+        }
+        Project(newChild.output.dropRight(extraAggExprs.length),
+          Filter(newCond, newChild))
+      } else {
+        h
+      }
+    }
+
+    // This require transformDown to resolve having condition when generating aggregate node for
+    // CUBE/ROLLUP/GROUPING SETS. This also replace grouping()/grouping_id() in resolved
+    // Filter/Sort.
+    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsDown {
+      case h @ UnresolvedHaving(
+          _, agg @ Aggregate(Seq(c @ Cube(groupByExprs)), aggregateExpressions, _))
+          if agg.childrenResolved && (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
+        tryResolveHavingCondition(h)
+      case h @ UnresolvedHaving(
+          _, agg @ Aggregate(Seq(r @ Rollup(groupByExprs)), aggregateExpressions, _))
+          if agg.childrenResolved && (groupByExprs ++ aggregateExpressions).forall(_.resolved) =>
+        tryResolveHavingCondition(h)
+      case h @ UnresolvedHaving(_, g: GroupingSets)
+          if g.childrenResolved && g.expressions.forall(_.resolved) =>
+        tryResolveHavingCondition(h)
+
       case a if !a.childrenResolved => a // be sure all of the children are resolved.
 
       // Ensure group by expressions and aggregate expressions have been resolved.
@@ -1397,7 +1464,7 @@ class Analyzer(
         }
 
       // Skip the having clause here, this will be handled in ResolveAggregateFunctions.
-      case h: AggregateWithHaving => h
+      case h: UnresolvedHaving => h
 
       case q: LogicalPlan =>
         logTrace(s"Attempting to resolve ${q.simpleString(SQLConf.get.maxToStringFields)}")
@@ -2042,7 +2109,7 @@ class Analyzer(
       // Resolve aggregate with having clause to Filter(..., Aggregate()). Note, to avoid wrongly
       // resolve the having condition expression, here we skip resolving it in ResolveReferences
       // and transform it to Filter after aggregate is resolved. See more details in SPARK-31519.
-      case AggregateWithHaving(cond, agg: Aggregate) if agg.resolved =>
+      case UnresolvedHaving(cond, agg: Aggregate) if agg.resolved =>
         resolveHaving(Filter(cond, agg), agg)
 
       case f @ Filter(_, agg: Aggregate) if agg.resolved =>
@@ -2118,13 +2185,13 @@ class Analyzer(
       condition.find(_.isInstanceOf[AggregateExpression]).isDefined
     }
 
-    def resolveHaving(filter: Filter, agg: Aggregate): LogicalPlan = {
-      // Try resolving the condition of the filter as though it is in the aggregate clause
+    def resolveFilterCondInAggregate(
+        filterCond: Expression, agg: Aggregate): Option[(Seq[NamedExpression], Expression)] = {
       try {
         val aggregatedCondition =
           Aggregate(
             agg.groupingExpressions,
-            Alias(filter.condition, "havingCondition")() :: Nil,
+            Alias(filterCond, "havingCondition")() :: Nil,
             agg.child)
         val resolvedOperator = executeSameContext(aggregatedCondition)
         def resolvedAggregateFilter =
@@ -2156,22 +2223,33 @@ class Analyzer(
                   alias.toAttribute
               }
           }
-
-          // Push the aggregate expressions into the aggregate (if any).
           if (aggregateExpressions.nonEmpty) {
-            Project(agg.output,
-              Filter(transformedAggregateFilter,
-                agg.copy(aggregateExpressions = agg.aggregateExpressions ++ aggregateExpressions)))
+            Some(aggregateExpressions, transformedAggregateFilter)
           } else {
-            filter
+            None
           }
         } else {
-          filter
+          None
         }
       } catch {
-        // Attempting to resolve in the aggregate can result in ambiguity.  When this happens,
-        // just return the original plan.
-        case ae: AnalysisException => filter
+        // Attempting to resolve in the aggregate can result in ambiguity. When this happens,
+        // just return None and the caller side will return the original plan.
+        case ae: AnalysisException => None
+      }
+    }
+
+    def resolveHaving(filter: Filter, agg: Aggregate): LogicalPlan = {
+      // Try resolving the condition of the filter as though it is in the aggregate clause
+      val resolvedInfo = resolveFilterCondInAggregate(filter.condition, agg)
+
+      // Push the aggregate expressions into the aggregate (if any).
+      if (resolvedInfo.nonEmpty) {
+        val (aggregateExpressions, resolvedHavingCond) = resolvedInfo.get
+        Project(agg.output,
+          Filter(resolvedHavingCond,
+            agg.copy(aggregateExpressions = agg.aggregateExpressions ++ aggregateExpressions)))
+      } else {
+        filter
       }
     }
   }
@@ -2600,12 +2678,12 @@ class Analyzer(
       case Filter(condition, _) if hasWindowFunction(condition) =>
         failAnalysis("It is not allowed to use window functions inside WHERE clause")
 
-      case AggregateWithHaving(condition, _) if hasWindowFunction(condition) =>
+      case UnresolvedHaving(condition, _) if hasWindowFunction(condition) =>
         failAnalysis("It is not allowed to use window functions inside HAVING clause")
 
       // Aggregate with Having clause. This rule works with an unresolved Aggregate because
       // a resolved Aggregate will not have Window Functions.
-      case f @ AggregateWithHaving(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
+      case f @ UnresolvedHaving(condition, a @ Aggregate(groupingExprs, aggregateExprs, child))
         if child.resolved &&
            hasWindowFunction(aggregateExprs) &&
            a.expressions.forall(_.resolved) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 806cdeb95cca4..b28be042c43f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -540,11 +540,12 @@ case class UnresolvedOrdinal(ordinal: Int)
 }
 
 /**
- * Represents unresolved aggregate with having clause, it is turned by the analyzer into a Filter.
+ * Represents unresolved having clause, the child for it can be Aggregate, GroupingSets, Rollup
+ * and Cube. It is turned by the analyzer into a Filter.
  */
-case class AggregateWithHaving(
+case class UnresolvedHaving(
     havingCondition: Expression,
-    child: Aggregate)
+    child: LogicalPlan)
   extends UnaryNode {
   override lazy val resolved: Boolean = false
   override def output: Seq[Attribute] = child.output
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index f135f50493ed8..fe3fea5e35b1b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -368,7 +368,7 @@ package object dsl {
           groupingExprs: Expression*)(
           aggregateExprs: Expression*)(
           havingCondition: Expression): LogicalPlan = {
-        AggregateWithHaving(havingCondition,
+        UnresolvedHaving(havingCondition,
           groupBy(groupingExprs: _*)(aggregateExprs: _*).asInstanceOf[Aggregate])
       }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 024d7a451f200..886f77e473580 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -629,12 +629,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       case p: Predicate => p
       case e => Cast(e, BooleanType)
     }
-    plan match {
-      case aggregate: Aggregate =>
-        AggregateWithHaving(predicate, aggregate)
-      case _ =>
-        Filter(predicate, plan)
-    }
+    UnresolvedHaving(predicate, plan)
   }
 
   /**
diff --git a/sql/core/src/test/resources/sql-tests/inputs/having.sql b/sql/core/src/test/resources/sql-tests/inputs/having.sql
index 6868b5902939d..3b75be19b5677 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/having.sql
@@ -18,4 +18,9 @@ SELECT MIN(t.v) FROM (SELECT * FROM hav WHERE v > 0) t HAVING(COUNT(1) > 0);
 SELECT a + b FROM VALUES (1L, 2), (3L, 4) AS T(a, b) GROUP BY a + b HAVING a + b > 1;
 
 -- SPARK-31519: Cast in having aggregate expressions returns the wrong result
-SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10
+SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY b HAVING b > 10;
+
+-- SPARK-31663: Grouping sets with having clause returns the wrong result
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY GROUPING SETS ((b), (a, b)) HAVING b > 10;
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY CUBE(a, b) HAVING b > 10;
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b) HAVING b > 10;
diff --git a/sql/core/src/test/resources/sql-tests/results/having.sql.out b/sql/core/src/test/resources/sql-tests/results/having.sql.out
index aa8ff73723586..1b3ac7865159f 100644
--- a/sql/core/src/test/resources/sql-tests/results/having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/having.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 6
+-- Number of queries: 9
 
 
 -- !query
@@ -55,3 +55,29 @@ SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2,
 struct<b:bigint,fake:date>
 -- !query output
 2	2020-01-01
+
+
+-- !query
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY GROUPING SETS ((b), (a, b)) HAVING b > 10
+-- !query schema
+struct<b:bigint>
+-- !query output
+2
+2
+
+
+-- !query
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY CUBE(a, b) HAVING b > 10
+-- !query schema
+struct<b:bigint>
+-- !query output
+2
+2
+
+
+-- !query
+SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b) HAVING b > 10
+-- !query schema
+struct<b:bigint>
+-- !query output
+2

From 847eec404f4b2f93842ec2d51528d1c73830332d Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sat, 16 May 2020 09:11:21 +0000
Subject: [PATCH 0682/1280] [SPARK-31289][TEST][TEST-HIVE1.2] Eliminate
 org.apache.spark.sql.hive.thriftserver.CliSuite flakiness

### What changes were proposed in this pull request?

CliSuite seems to be flaky while using metastoreDir per test.
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120470/testReport/org.apache.spark.sql.hive.thriftserver/CliSuite/

https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120470/testReport/junit/org.apache.spark.sql.hive.thriftserver/CliSuite/history/

According to the error stack trace in the failed test, the test failed to instantiate a hive metastore client because of derby requirements.
```scala
Caused by: ERROR XBM0A:
 The database directory '/home/jenkins/workspace/SparkPullRequestBuilder/target/tmp/spark-9249ce52-0a06-42b6-a3df-e6295e880df0' exists.
However, it does not contain the expected 'service.properties' file.
Perhaps Derby was brought down in the middle of creating this database.
You may want to delete this directory and try creating the database again.
```

The derby requires the metastore dir does not exist, but it does exist probably due to the test case before it failed to clear the metastore dir

In this PR, the metastore is shared across the tests of CliSuite except those explicitly asked a separated metastore env itself

### Why are the changes needed?

CliSuite seems to be flaky while using metastoreDir per test.
 To eliminate test flakiness
### Does this PR introduce any user-facing change?

no
### How was this patch tested?

modified test

Closes #28055 from yaooqinn/clisuite.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1d66085a93a875247f19d710a5b5458ce1842c73)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/hive/thriftserver/CliSuite.scala      | 96 ++++++++++++-------
 1 file changed, 59 insertions(+), 37 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index bcdcd072eeeca..b41a4b7f891db 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -39,7 +39,7 @@ import org.apache.spark.util.{ThreadUtils, Utils}
 /**
  * A test suite for the `spark-sql` CLI tool.
  */
-class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterEach with Logging {
+class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
   val warehousePath = Utils.createTempDir()
   val metastorePath = Utils.createTempDir()
   val scratchDirPath = Utils.createTempDir()
@@ -62,12 +62,6 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
     }
   }
 
-  override def afterEach(): Unit = {
-    // Only running `runCliWithin` in a single test case will share the same temporary
-    // Hive metastore
-    Utils.deleteRecursively(metastorePath)
-  }
-
   /**
    * Run a CLI operation and expect all the queries and expected answers to be returned.
    *
@@ -77,6 +71,12 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
    *                       is taken as an immediate error condition. That is: if a line containing
    *                       with one of these strings is found, fail the test immediately.
    *                       The default value is `Seq("Error:")`
+   * @param maybeWarehouse an option for warehouse path, which will be set via
+   *                       `hive.metastore.warehouse.dir`.
+   * @param useExternalHiveFile whether to load the hive-site.xml from `src/test/noclasspath` or
+   *                            not, disabled by default
+   * @param metastore which path the embedded derby database for metastore locates. Use the the
+   *                  global `metastorePath` by default
    * @param queriesAndExpectedAnswers one or more tuples of query + answer
    */
   def runCliWithin(
@@ -84,7 +84,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       extraArgs: Seq[String] = Seq.empty,
       errorResponses: Seq[String] = Seq("Error:"),
       maybeWarehouse: Option[File] = Some(warehousePath),
-      useExternalHiveFile: Boolean = false)(
+      useExternalHiveFile: Boolean = false,
+      metastore: File = metastorePath)(
       queriesAndExpectedAnswers: (String, String)*): Unit = {
 
     // Explicitly adds ENTER for each statement to make sure they are actually entered into the CLI.
@@ -116,7 +117,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
       maybeWarehouse.map(dir => s"--hiveconf ${ConfVars.METASTOREWAREHOUSE}=$dir").getOrElse("")
     val command = {
       val cliScript = "../../bin/spark-sql".split("/").mkString(File.separator)
-      val jdbcUrl = s"jdbc:derby:;databaseName=$metastorePath;create=true"
+      val jdbcUrl = s"jdbc:derby:;databaseName=$metastore;create=true"
       s"""$cliScript
          |  --master local
          |  --driver-java-options -Dderby.system.durability=test
@@ -202,9 +203,18 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
   }
 
   test("load warehouse dir from hive-site.xml") {
-    runCliWithin(1.minute, maybeWarehouse = None, useExternalHiveFile = true)(
-      "desc database default;" -> "hive_one",
-      "set spark.sql.warehouse.dir;" -> "hive_one")
+    val metastore = Utils.createTempDir()
+    metastore.delete()
+    try {
+      runCliWithin(1.minute,
+        maybeWarehouse = None,
+        useExternalHiveFile = true,
+        metastore = metastore)(
+        "desc database default;" -> "hive_one",
+        "set spark.sql.warehouse.dir;" -> "hive_one")
+    } finally {
+      Utils.deleteRecursively(metastore)
+    }
   }
 
   test("load warehouse dir from --hiveconf") {
@@ -218,35 +228,47 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfterE
 
   test("load warehouse dir from --conf spark(.hadoop).hive.*") {
     // override conf from hive-site.xml
-    runCliWithin(
-      2.minute,
-      extraArgs = Seq("--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"),
-      maybeWarehouse = None,
-      useExternalHiveFile = true)(
-      "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
-      "create database cliTestDb;" -> "",
-      "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
-      "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
-
-    // override conf from --hiveconf too
-    runCliWithin(
-      2.minute,
-      extraArgs = Seq("--conf", s"spark.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"))(
-      "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
-      "create database cliTestDb;" -> "",
-      "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
-      "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+    val metastore = Utils.createTempDir()
+    metastore.delete()
+    try {
+      runCliWithin(2.minute,
+        extraArgs =
+          Seq("--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"),
+        maybeWarehouse = None,
+        useExternalHiveFile = true,
+        metastore = metastore)(
+        "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
+        "create database cliTestDb;" -> "",
+        "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
+        "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+
+      // override conf from --hiveconf too
+      runCliWithin(2.minute,
+        extraArgs = Seq("--conf", s"spark.${ConfVars.METASTOREWAREHOUSE}=$sparkWareHouseDir"),
+        metastore = metastore)(
+        "desc database default;" -> sparkWareHouseDir.getAbsolutePath,
+        "create database cliTestDb;" -> "",
+        "desc database cliTestDb;" -> sparkWareHouseDir.getAbsolutePath,
+        "set spark.sql.warehouse.dir;" -> sparkWareHouseDir.getAbsolutePath)
+    } finally {
+      Utils.deleteRecursively(metastore)
+    }
   }
 
   test("load warehouse dir from spark.sql.warehouse.dir") {
     // spark.sql.warehouse.dir overrides all hive ones
-    runCliWithin(
-      2.minute,
-      extraArgs =
-        Seq("--conf",
-          s"${StaticSQLConf.WAREHOUSE_PATH.key}=${sparkWareHouseDir}1",
-          "--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=${sparkWareHouseDir}2"))(
-      "desc database default;" -> sparkWareHouseDir.getAbsolutePath.concat("1"))
+    val metastore = Utils.createTempDir()
+    metastore.delete()
+    try {
+      runCliWithin(2.minute,
+        extraArgs = Seq(
+            "--conf", s"${StaticSQLConf.WAREHOUSE_PATH.key}=${sparkWareHouseDir}1",
+            "--conf", s"spark.hadoop.${ConfVars.METASTOREWAREHOUSE}=${sparkWareHouseDir}2"),
+        metastore = metastore)(
+        "desc database default;" -> sparkWareHouseDir.getAbsolutePath.concat("1"))
+    } finally {
+      Utils.deleteRecursively(metastore)
+    }
   }
 
   test("Simple commands") {

From dfcba5523ebcd90491ab0e005f4bbce37378ca41 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Sat, 16 May 2020 07:33:58 -0700
Subject: [PATCH 0683/1280] [SPARK-31732][TESTS] Disable some flaky tests
 temporarily

### What changes were proposed in this pull request?

It's quite annoying to be blocked by flaky tests in several PRs. This PR disables them. The tests come from 3 PRs I'm recently watching:
https://github.com/apache/spark/pull/28526
https://github.com/apache/spark/pull/28463
https://github.com/apache/spark/pull/28517

### Why are the changes needed?

To make PR builder more stable

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

N/A

Closes #28547 from cloud-fan/test.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 2012d5847520c8aba54e8e3e6a634976a3c7657d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/deploy/history/HistoryServerSuite.scala  | 3 ++-
 .../apache/spark/scheduler/BarrierTaskContextSuite.scala  | 6 ++++--
 .../spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala   | 3 ++-
 .../apache/spark/sql/kafka010/KafkaRelationSuite.scala    | 3 ++-
 .../spark/streaming/kafka010/DirectKafkaStreamSuite.scala | 8 +++++---
 .../apache/spark/streaming/StreamingContextSuite.scala    | 3 ++-
 6 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 56cc3daecb3a6..c55b29b15051d 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -314,7 +314,8 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
     all (directSiteRelativeLinks) should not startWith (knoxBaseUrl)
   }
 
-  test("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
+  // TODO (SPARK-31723): re-enable it
+  ignore("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") {
     val uiRoot = Option(System.getenv("APPLICATION_WEB_PROXY_BASE")).getOrElse("/testwebproxybase")
     val page = new HistoryPage(server)
     val request = mock[HttpServletRequest]
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index c4e5e7c700652..17bc3391cf946 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -39,7 +39,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     sc = new SparkContext(conf)
   }
 
-  test("global sync by barrier() call") {
+  // TODO (SPARK-31730): re-enable it
+  ignore("global sync by barrier() call") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -131,7 +132,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     assert(times2.max - times2.min <= 1000)
   }
 
-  test("support multiple barrier() call within a single task") {
+  // TODO (SPARK-31730): re-enable it
+  ignore("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index a4601b91af0d6..bdad214a91343 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -349,7 +349,8 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     )
   }
 
-  test("subscribing topic by pattern with topic deletions") {
+  // TODO (SPARK-31731): re-enable it
+  ignore("subscribing topic by pattern with topic deletions") {
     val topicPrefix = newTopic()
     val topic = topicPrefix + "-seems"
     val topic2 = topicPrefix + "-bad"
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
index 32d056140a0d7..e5f3a229622e1 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
@@ -179,7 +179,8 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
       ("3", Seq(("e", "f".getBytes(UTF_8)), ("e", "g".getBytes(UTF_8))))).toDF)
   }
 
-  test("timestamp provided for starting and ending") {
+  // TODO (SPARK-31729): re-enable it
+  ignore("timestamp provided for starting and ending") {
     val (topic, timestamps) = prepareTimestampRelatedUnitTest
 
     // timestamp both presented: starting "first" ending "finalized"
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
index 925327d9d58e6..72cf3e8118228 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
@@ -332,7 +332,8 @@ class DirectKafkaStreamSuite
   }
 
   // Test to verify the offset ranges can be recovered from the checkpoints
-  test("offset recovery") {
+  // TODO (SPARK-31722): re-enable it
+  ignore("offset recovery") {
     val topic = "recovery"
     kafkaTestUtils.createTopic(topic)
     testDir = Utils.createTempDir()
@@ -418,8 +419,9 @@ class DirectKafkaStreamSuite
     ssc.stop()
   }
 
-    // Test to verify the offsets can be recovered from Kafka
-  test("offset recovery from kafka") {
+  // Test to verify the offsets can be recovered from Kafka
+  // TODO (SPARK-31722): re-enable it
+  ignore("offset recovery from kafka") {
     val topic = "recoveryfromkafka"
     kafkaTestUtils.createTopic(topic)
 
diff --git a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
index 1d6637861511f..4eff464dcdafb 100644
--- a/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
+++ b/streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala
@@ -293,7 +293,8 @@ class StreamingContextSuite
     }
   }
 
-  test("stop gracefully") {
+  // TODO (SPARK-31728): re-enable it
+  ignore("stop gracefully") {
     val conf = new SparkConf().setMaster(master).setAppName(appName)
     conf.set("spark.dummyTimeConfig", "3600s")
     val sc = new SparkContext(conf)

From e7902063e7a84ac1f2946220ac53ef417fae0a80 Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Sat, 16 May 2020 13:23:49 -0700
Subject: [PATCH 0684/1280] [SPARK-31655][BUILD][3.0] Upgrade snappy-java to
 1.1.7.5

### What changes were proposed in this pull request?

snappy-java have release v1.1.7.5, upgrade to latest version.

Fixed in v1.1.7.4
- Caching internal buffers for SnappyFramed streams #234
- Fixed the native lib for ppc64le to work with glibc 2.17 (Previously it depended on 2.22)

Fixed in v1.1.7.5
- Fixes java.lang.NoClassDefFoundError: org/xerial/snappy/pool/DefaultPoolFactory in 1.1.7.4

https://github.com/xerial/snappy-java/compare/1.1.7.3...1.1.7.5

v 1.1.7.5 release note:
https://github.com/xerial/snappy-java/commit/edc4ec28bdb15a32b6c41ca9e8b195e635bec3a3
### Why are the changes needed?
Fix bug

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
No need

Closes #28509 from AngersZhuuuu/SPARK-31655-3.0.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +-
 pom.xml                                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 296f518f1678d..b307e573a3511 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -187,7 +187,7 @@ shims/0.7.45//shims-0.7.45.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
+snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
 snappy/0.2//snappy-0.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 753c0dae55806..b6d6e34aa49b0 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -201,7 +201,7 @@ shims/0.7.45//shims-0.7.45.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
+snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 4a29f10153d9c..c8d2c827249b7 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -217,7 +217,7 @@ shims/0.7.45//shims-0.7.45.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.3//snappy-java-1.1.7.3.jar
+snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
diff --git a/pom.xml b/pom.xml
index 870d0da1f48eb..37548de1813de 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,7 +167,7 @@
     <scalafmt.skip>true</scalafmt.skip>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-    <snappy.version>1.1.7.3</snappy.version>
+    <snappy.version>1.1.7.5</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <commons-codec.version>1.10</commons-codec.version>
     <commons-io.version>2.4</commons-io.version>

From 4f5df2c7b46b93687c634fbcd3c18c24aa7988a3 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 17 May 2020 02:26:00 +0000
Subject: [PATCH 0685/1280] [SPARK-31725][CORE][SQL][TESTS] Set
 America/Los_Angeles time zone and Locale.US in tests by default

### What changes were proposed in this pull request?
Set default time zone and locale in the default constructor of `SparkFunSuite`:
- Default time zone to `America/Los_Angeles`
- Default locale to `Locale.US`

### Why are the changes needed?
1. To deduplicate code by moving common time zone and locale settings to one place SparkFunSuite
2. To have the same default time zone and locale in all tests. This should prevent errors like https://github.com/apache/spark/pull/28538

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
by running all affected test suites

Closes #28548 from MaxGekk/timezone-settings-SparkFunSuite.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5539ecfdac6c06285c7449494196ea3b4eb4cf87)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/SparkFunSuite.scala  |  6 ++++++
 .../expressions/SortOrderExpressionsSuite.scala | 11 +----------
 .../scala/org/apache/spark/sql/QueryTest.scala  |  7 +------
 .../apache/spark/sql/SQLQueryTestSuite.scala    | 11 +----------
 .../hive/execution/HiveCompatibilitySuite.scala |  9 ---------
 .../HiveWindowFunctionQuerySuite.scala          | 17 -----------------
 .../spark/sql/hive/HiveInspectorSuite.scala     |  6 ------
 .../sql/hive/execution/HiveQuerySuite.scala     | 11 +----------
 8 files changed, 10 insertions(+), 68 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
index cf4400e080e37..ec641f8294b29 100644
--- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark
 
 // scalastyle:off
 import java.io.File
+import java.util.{Locale, TimeZone}
 
 import org.apache.log4j.spi.LoggingEvent
 
@@ -63,6 +64,11 @@ abstract class SparkFunSuite
   with Logging {
 // scalastyle:on
 
+  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
+  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
+  // Add Locale setting
+  Locale.setDefault(Locale.US)
+
   protected val enableAutoThreadAudit = true
 
   protected override def beforeAll(): Unit = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
index f2696849d7753..4e6976f76ea5f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.Timestamp
-import java.util.TimeZone
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.types._
@@ -35,15 +34,7 @@ class SortOrderExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     val l1 = Literal.create(20132983L, LongType)
     val l2 = Literal.create(-20132983L, LongType)
     val millis = 1524954911000L
-    // Explicitly choose a time zone, since Date objects can create different values depending on
-    // local time zone of the machine on which the test is running
-    val oldDefaultTZ = TimeZone.getDefault
-    val d1 = try {
-      TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-      Literal.create(new java.sql.Date(millis), DateType)
-    } finally {
-      TimeZone.setDefault(oldDefaultTZ)
-    }
+    val d1 = Literal.create(new java.sql.Date(millis), DateType)
     val t1 = Literal.create(new Timestamp(millis), TimestampType)
     val f1 = Literal.create(0.7788229f, FloatType)
     val f2 = Literal.create(-0.7788229f, FloatType)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index 4a21ae9242039..e52d2262a6bf8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql
 
-import java.util.{Locale, TimeZone}
+import java.util.TimeZone
 
 import scala.collection.JavaConverters._
 
@@ -35,11 +35,6 @@ abstract class QueryTest extends PlanTest {
 
   protected def spark: SparkSession
 
-  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-  // Add Locale setting
-  Locale.setDefault(Locale.US)
-
   /**
    * Runs the plan and makes sure the answer contains all of the keywords.
    */
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 4a39d95773e6b..005cac754d975 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql
 
 import java.io.File
-import java.util.{Locale, TimeZone}
+import java.util.Locale
 import java.util.regex.Pattern
 
 import scala.collection.mutable.{ArrayBuffer, HashMap}
@@ -670,23 +670,14 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     session.sql("DROP TABLE IF EXISTS tenk1")
   }
 
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
-
   override def beforeAll(): Unit = {
     super.beforeAll()
     createTestTables(spark)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
     RuleExecutor.resetMetrics()
   }
 
   override def afterAll(): Unit = {
     try {
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       removeTestTables(spark)
 
       // For debugging dump some statistics about how much time was spent in various optimizer rules
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 29825e5116ef9..d941c4bc75b42 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive.execution
 
 import java.io.File
-import java.util.{Locale, TimeZone}
 
 import org.scalatest.BeforeAndAfter
 
@@ -36,8 +35,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   private lazy val hiveQueryDir = TestHive.getHiveFile(
     "ql/src/test/queries/clientpositive".split("/").mkString(File.separator))
 
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
   private val originalColumnBatchSize = TestHive.conf.columnBatchSize
   private val originalInMemoryPartitionPruning = TestHive.conf.inMemoryPartitionPruning
   private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
@@ -51,10 +48,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
     // Set a relatively small column batch size for testing purposes
     TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, 5)
     // Enable in-memory partition pruning for testing purposes
@@ -73,8 +66,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, originalColumnBatchSize)
       TestHive.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
       TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
index ed23f65815917..2c0970c85449f 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive.execution
 
 import java.io.File
-import java.util.{Locale, TimeZone}
 
 import org.scalatest.BeforeAndAfter
 
@@ -33,17 +32,11 @@ import org.apache.spark.util.Utils
  * files, every `createQueryTest` calls should explicitly set `reset` to `false`.
  */
 class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfter {
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
   private val testTempDir = Utils.createTempDir()
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
 
     // Create the table used in windowing.q
     sql("DROP TABLE IF EXISTS part")
@@ -103,8 +96,6 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       TestHive.reset()
     } finally {
       super.afterAll()
@@ -747,17 +738,11 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte
 
 class HiveWindowFunctionQueryFileSuite
   extends HiveCompatibilitySuite with BeforeAndAfter {
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
   private val testTempDir = Utils.createTempDir()
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
 
     // The following settings are used for generating golden files with Hive.
     // We have to use kryo to correctly let Hive serialize plans with window functions.
@@ -772,8 +757,6 @@ class HiveWindowFunctionQueryFileSuite
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       TestHive.reset()
     } finally {
       super.afterAll()
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 5912992694e84..13c48f38e7f78 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.hive
 
 import java.util
-import java.util.{Locale, TimeZone}
 
 import org.apache.hadoop.hive.ql.udf.UDAFPercentile
 import org.apache.hadoop.hive.serde2.io.DoubleWritable
@@ -74,11 +73,6 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
       .get())
   }
 
-  // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-  TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-  // Add Locale setting
-  Locale.setDefault(Locale.US)
-
   val data =
     Literal(true) ::
     Literal(null) ::
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 1e89db2bdd01a..63b985fbe4d32 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.execution
 import java.io.File
 import java.net.URI
 import java.sql.Timestamp
-import java.util.{Locale, TimeZone}
+import java.util.Locale
 
 import scala.util.Try
 
@@ -47,9 +47,6 @@ case class TestData(a: Int, b: String)
  * included in the hive distribution.
  */
 class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAndAfter {
-  private val originalTimeZone = TimeZone.getDefault
-  private val originalLocale = Locale.getDefault
-
   import org.apache.spark.sql.hive.test.TestHive.implicits._
 
   private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
@@ -59,10 +56,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   override def beforeAll(): Unit = {
     super.beforeAll()
     TestHive.setCacheTables(true)
-    // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
-    TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-    // Add Locale setting
-    Locale.setDefault(Locale.US)
     // Ensures that cross joins are enabled so that we can test them
     TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, true)
   }
@@ -70,8 +63,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
   override def afterAll(): Unit = {
     try {
       TestHive.setCacheTables(false)
-      TimeZone.setDefault(originalTimeZone)
-      Locale.setDefault(originalLocale)
       sql("DROP TEMPORARY FUNCTION IF EXISTS udtf_count2")
       TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
     } finally {

From 3b2d3946f762dc96414a01c5c996371625953bd8 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Sun, 17 May 2020 02:27:23 +0000
Subject: [PATCH 0686/1280] [SPARK-31707][SQL] Revert SPARK-30098 Use default
 datasource as provider for CREATE TABLE syntax

### What changes were proposed in this pull request?

This patch effectively reverts SPARK-30098 via below changes:

* Removed the config
* Removed the changes done in parser rule
* Removed the usage of config in tests
  * Removed tests which depend on the config
  * Rolled back some tests to before SPARK-30098 which were affected by SPARK-30098
* Reflect the change into docs (migration doc, create table syntax)

### Why are the changes needed?

SPARK-30098 brought confusion and frustration on using create table DDL query, and we agreed about the bad effect on the change.

Please go through the [discussion thread](http://apache-spark-developers-list.1001551.n3.nabble.com/DISCUSS-Resolve-ambiguous-parser-rule-between-two-quot-create-table-quot-s-td29051i20.html) to see the details.

### Does this PR introduce _any_ user-facing change?

No, compared to Spark 2.4.x. End users tried to experiment with Spark 3.0.0 previews will see the change that the behavior is going back to Spark 2.4.x, but I believe we won't guarantee compatibility in preview releases.

### How was this patch tested?

Existing UTs.

Closes #28517 from HeartSaVioR/revert-SPARK-30098.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d2bec5e265e0aa4fa527c3f43cfe738cdbdc4598)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |   2 -
 ...-ref-syntax-ddl-create-table-datasource.md |   2 +-
 .../spark/sql/catalyst/parser/SqlBase.g4      |  13 +-
 .../sql/catalyst/parser/ParseDriver.scala     |   2 -
 .../apache/spark/sql/internal/SQLConf.scala   |  12 --
 .../sql/catalyst/parser/DDLParserSuite.scala  |  29 ++--
 .../sql/connector/DataSourceV2SQLSuite.scala  |   8 +-
 .../adaptive/AdaptiveQueryExecSuite.scala     |   4 +-
 .../execution/command/DDLParserSuite.scala    | 133 +++++++++---------
 .../execution/HiveCompatibilitySuite.scala    |   4 -
 .../sql/hive/HiveShowCreateTableSuite.scala   |  16 ---
 .../sql/hive/execution/HiveDDLSuite.scala     |  27 ----
 .../sql/hive/execution/HiveSerDeSuite.scala   |   6 +-
 13 files changed, 91 insertions(+), 167 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 748d68957c896..b0725d1889776 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -34,8 +34,6 @@ license: |
 
 ### DDL Statements
 
-  - In Spark 3.0, `CREATE TABLE` without a specific provider uses the value of `spark.sql.sources.default` as its provider. In Spark version 2.4 and below, it was Hive. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.createHiveTableByDefault.enabled` to `true`.
-
   - In Spark 3.0, when inserting a value into a table column with a different data type, the type coercion is performed as per ANSI SQL standard. Certain unreasonable type conversions such as converting `string` to `int` and `double` to `boolean` are disallowed. A runtime exception is thrown if the value is out-of-range for the data type of the column. In Spark version 2.4 and below, type conversions during table insertion are allowed as long as they are valid `Cast`. When inserting an out-of-range value to an integral field, the low-order bits of the value is inserted(the same as Java/Scala numeric type casting). For example, if 257 is inserted to a field of byte type, the result is 1. The behavior is controlled by the option `spark.sql.storeAssignmentPolicy`, with a default value as "ANSI". Setting the option as "Legacy" restores the previous behavior.
 
   - The `ADD JAR` command previously returned a result set with the single value 0. It now returns an empty result set.
diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index b592116c2a9e4..d334447a91011 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -28,7 +28,7 @@ The `CREATE TABLE` statement defines a new table using a Data Source.
 ```sql
 CREATE TABLE [ IF NOT EXISTS ] table_identifier
     [ ( col_name1 col_type1 [ COMMENT col_comment1 ], ... ) ]
-    [ USING data_source ]
+    USING data_source
     [ OPTIONS ( key1=val1, key2=val2, ... ) ]
     [ PARTITIONED BY ( col_name1, col_name2, ... ) ]
     [ CLUSTERED BY ( col_name3, col_name4, ... ) 
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 39f2942df5921..613582fa0a99e 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -29,12 +29,6 @@ grammar SqlBase;
    */
   public boolean legacy_exponent_literal_as_decimal_enabled = false;
 
-  /**
-   * When false, CREATE TABLE syntax without a provider will use
-   * the value of spark.sql.sources.default as its provider.
-   */
-  public boolean legacy_create_hive_table_by_default_enabled = false;
-
   /**
    * Verify whether current token is a valid decimal token (which contains dot).
    * Returns true if the character that follows the token is not a digit or letter or underscore.
@@ -107,12 +101,7 @@ statement
         (RESTRICT | CASCADE)?                                          #dropNamespace
     | SHOW (DATABASES | NAMESPACES) ((FROM | IN) multipartIdentifier)?
         (LIKE? pattern=STRING)?                                        #showNamespaces
-    | {!legacy_create_hive_table_by_default_enabled}?
-        createTableHeader ('(' colTypeList ')')? tableProvider?
-        createTableClauses
-        (AS? query)?                                                   #createTable
-    | {legacy_create_hive_table_by_default_enabled}?
-        createTableHeader ('(' colTypeList ')')? tableProvider
+    | createTableHeader ('(' colTypeList ')')? tableProvider
         createTableClauses
         (AS? query)?                                                   #createTable
     | createTableHeader ('(' columns=colTypeList ')')?
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
index 590193bddafb5..fab282f15f215 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
@@ -98,7 +98,6 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     lexer.addErrorListener(ParseErrorListener)
     lexer.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
     lexer.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
-    lexer.legacy_create_hive_table_by_default_enabled = conf.createHiveTableByDefaultEnabled
     lexer.SQL_standard_keyword_behavior = conf.ansiEnabled
 
     val tokenStream = new CommonTokenStream(lexer)
@@ -108,7 +107,6 @@ abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Log
     parser.addErrorListener(ParseErrorListener)
     parser.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced
     parser.legacy_exponent_literal_as_decimal_enabled = conf.exponentLiteralAsDecimalEnabled
-    parser.legacy_create_hive_table_by_default_enabled = conf.createHiveTableByDefaultEnabled
     parser.SQL_standard_keyword_behavior = conf.ansiEnabled
 
     try {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 31038a0e26f7e..b073f7ea599fd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2228,15 +2228,6 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
-  val LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED =
-    buildConf("spark.sql.legacy.createHiveTableByDefault.enabled")
-      .internal()
-      .doc("When set to true, CREATE TABLE syntax without a provider will use hive " +
-        s"instead of the value of ${DEFAULT_DATA_SOURCE_NAME.key}.")
-      .version("3.0.0")
-      .booleanConf
-      .createWithDefault(false)
-
   val LEGACY_BUCKETED_TABLE_SCAN_OUTPUT_ORDERING =
     buildConf("spark.sql.legacy.bucketedTableScan.outputOrdering")
       .internal()
@@ -3126,9 +3117,6 @@ class SQLConf extends Serializable with Logging {
   def allowNegativeScaleOfDecimalEnabled: Boolean =
     getConf(SQLConf.LEGACY_ALLOW_NEGATIVE_SCALE_OF_DECIMAL_ENABLED)
 
-  def createHiveTableByDefaultEnabled: Boolean =
-    getConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
-
   def truncateTableIgnorePermissionAcl: Boolean =
     getConf(SQLConf.TRUNCATE_TABLE_IGNORE_PERMISSION_ACL)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index e05a51e2afab2..20b98550b2c3e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -2150,21 +2150,20 @@ class DDLParserSuite extends AnalysisTest {
       CommentOnTable(UnresolvedTable(Seq("a", "b", "c")), "xYz"))
   }
 
-  test("create table - without using") {
-    withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "false") {
-      val sql = "CREATE TABLE 1m.2g(a INT)"
-      val expectedTableSpec = TableSpec(
-        Seq("1m", "2g"),
-        Some(new StructType().add("a", IntegerType)),
-        Seq.empty[Transform],
-        None,
-        Map.empty[String, String],
-        None,
-        Map.empty[String, String],
-        None,
-        None)
+  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
+  ignore("create table - without using") {
+    val sql = "CREATE TABLE 1m.2g(a INT)"
+    val expectedTableSpec = TableSpec(
+      Seq("1m", "2g"),
+      Some(new StructType().add("a", IntegerType)),
+      Seq.empty[Transform],
+      None,
+      Map.empty[String, String],
+      None,
+      Map.empty[String, String],
+      None,
+      None)
 
-      testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
-    }
+    testCreateOrReplaceDdl(sql, expectedTableSpec, expectedIfNotExists = false)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index e947e15a179e8..8462ce5a6c44f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -256,8 +256,8 @@ class DataSourceV2SQLSuite
     checkAnswer(spark.internalCreateDataFrame(rdd, table.schema), Seq.empty)
   }
 
-  test("CreateTable: without USING clause") {
-    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key, "false")
+  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
+  ignore("CreateTable: without USING clause") {
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
     val testCatalog = catalog("testcat").asTableCatalog
@@ -681,8 +681,8 @@ class DataSourceV2SQLSuite
     }
   }
 
-  test("CreateTableAsSelect: without USING clause") {
-    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key, "false")
+  // TODO: ignored by SPARK-31707, restore the test after create table syntax unification
+  ignore("CreateTableAsSelect: without USING clause") {
     // unset this config to use the default v2 session catalog.
     spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
     val testCatalog = catalog("testcat").asTableCatalog
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 29b97557b9448..18923b209b533 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -806,7 +806,7 @@ class AdaptiveQueryExecSuite
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.ADAPTIVE_EXECUTION_FORCE_APPLY.key -> "true") {
       withTable("t1") {
-        val plan = sql("CREATE TABLE t1 AS SELECT 1 col").queryExecution.executedPlan
+        val plan = sql("CREATE TABLE t1 USING parquet AS SELECT 1 col").queryExecution.executedPlan
         assert(plan.isInstanceOf[DataWritingCommandExec])
         assert(plan.asInstanceOf[DataWritingCommandExec].child.isInstanceOf[AdaptiveSparkPlanExec])
       }
@@ -867,7 +867,7 @@ class AdaptiveQueryExecSuite
         }
         spark.sparkContext.addSparkListener(listener)
         try {
-          sql("CREATE TABLE t1 AS SELECT 1 col").collect()
+          sql("CREATE TABLE t1 USING parquet AS SELECT 1 col").collect()
           spark.sparkContext.listenerBus.waitUntilEmpty()
           assert(checkDone)
         } finally {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
index 28e5082886b67..c6a533dfae4d0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala
@@ -40,8 +40,7 @@ import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
 
 class DDLParserSuite extends AnalysisTest with SharedSparkSession {
-  private lazy val parser = new SparkSqlParser(new SQLConf().copy(
-    SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED -> false))
+  private lazy val parser = new SparkSqlParser(new SQLConf)
 
   private def assertUnsupported(sql: String, containsThesePhrases: Seq[String] = Seq()): Unit = {
     val e = intercept[ParseException] {
@@ -76,12 +75,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
     }.head
   }
 
-  private def withCreateTableStatement(sql: String)(prediction: CreateTableStatement => Unit)
-    : Unit = {
-    val statement = parser.parsePlan(sql).asInstanceOf[CreateTableStatement]
-    prediction(statement)
-  }
-
   test("alter database - property values must be set") {
     assertUnsupported(
       sql = "ALTER DATABASE my_db SET DBPROPERTIES('key_without_value', 'key_with_value'='x')",
@@ -487,17 +480,21 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
 
   test("Test CTAS #3") {
     val s3 = """CREATE TABLE page_view AS SELECT * FROM src"""
-    val statement = parser.parsePlan(s3).asInstanceOf[CreateTableAsSelectStatement]
-    assert(statement.tableName(0) == "page_view")
-    assert(statement.asSelect == parser.parsePlan("SELECT * FROM src"))
-    assert(statement.partitioning.isEmpty)
-    assert(statement.bucketSpec.isEmpty)
-    assert(statement.properties.isEmpty)
-    assert(statement.provider.isEmpty)
-    assert(statement.options.isEmpty)
-    assert(statement.location.isEmpty)
-    assert(statement.comment.isEmpty)
-    assert(!statement.ifNotExists)
+    val (desc, exists) = extractTableDesc(s3)
+    assert(exists == false)
+    assert(desc.identifier.database == None)
+    assert(desc.identifier.table == "page_view")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.storage.locationUri == None)
+    assert(desc.schema.isEmpty)
+    assert(desc.viewText == None) // TODO will be SQLText
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.properties == Map())
+    assert(desc.storage.inputFormat == Some("org.apache.hadoop.mapred.TextInputFormat"))
+    assert(desc.storage.outputFormat ==
+      Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc.properties == Map())
   }
 
   test("Test CTAS #4") {
@@ -657,60 +654,67 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
 
   test("create table - basic") {
     val query = "CREATE TABLE my_table (id int, name string)"
-    withCreateTableStatement(query) { state =>
-      assert(state.tableName(0) == "my_table")
-      assert(state.tableSchema == new StructType().add("id", "int").add("name", "string"))
-      assert(state.partitioning.isEmpty)
-      assert(state.bucketSpec.isEmpty)
-      assert(state.properties.isEmpty)
-      assert(state.provider.isEmpty)
-      assert(state.options.isEmpty)
-      assert(state.location.isEmpty)
-      assert(state.comment.isEmpty)
-      assert(!state.ifNotExists)
-    }
+    val (desc, allowExisting) = extractTableDesc(query)
+    assert(!allowExisting)
+    assert(desc.identifier.database.isEmpty)
+    assert(desc.identifier.table == "my_table")
+    assert(desc.tableType == CatalogTableType.MANAGED)
+    assert(desc.schema == new StructType().add("id", "int").add("name", "string"))
+    assert(desc.partitionColumnNames.isEmpty)
+    assert(desc.bucketSpec.isEmpty)
+    assert(desc.viewText.isEmpty)
+    assert(desc.viewQueryColumnNames.isEmpty)
+    assert(desc.storage.locationUri.isEmpty)
+    assert(desc.storage.inputFormat ==
+      Some("org.apache.hadoop.mapred.TextInputFormat"))
+    assert(desc.storage.outputFormat ==
+      Some("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"))
+    assert(desc.storage.serde == Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"))
+    assert(desc.storage.properties.isEmpty)
+    assert(desc.properties.isEmpty)
+    assert(desc.comment.isEmpty)
   }
 
   test("create table - with database name") {
     val query = "CREATE TABLE dbx.my_table (id int, name string)"
-    withCreateTableStatement(query) { state =>
-      assert(state.tableName(0) == "dbx")
-      assert(state.tableName(1) == "my_table")
-    }
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.identifier.database == Some("dbx"))
+    assert(desc.identifier.table == "my_table")
   }
 
   test("create table - temporary") {
     val query = "CREATE TEMPORARY TABLE tab1 (id int, name string)"
     val e = intercept[ParseException] { parser.parsePlan(query) }
-    assert(e.message.contains("CREATE TEMPORARY TABLE without a provider is not allowed."))
+    assert(e.message.contains("CREATE TEMPORARY TABLE is not supported yet"))
   }
 
   test("create table - external") {
     val query = "CREATE EXTERNAL TABLE tab1 (id int, name string) LOCATION '/path/to/nowhere'"
-    val e = intercept[ParseException] { parser.parsePlan(query) }
-    assert(e.message.contains("Operation not allowed: CREATE EXTERNAL TABLE ..."))
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.tableType == CatalogTableType.EXTERNAL)
+    assert(desc.storage.locationUri == Some(new URI("/path/to/nowhere")))
   }
 
   test("create table - if not exists") {
     val query = "CREATE TABLE IF NOT EXISTS tab1 (id int, name string)"
-    withCreateTableStatement(query) { state =>
-      assert(state.ifNotExists)
-    }
+    val (_, allowExisting) = extractTableDesc(query)
+    assert(allowExisting)
   }
 
   test("create table - comment") {
     val query = "CREATE TABLE my_table (id int, name string) COMMENT 'its hot as hell below'"
-    withCreateTableStatement(query) { state =>
-      assert(state.comment == Some("its hot as hell below"))
-    }
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.comment == Some("its hot as hell below"))
   }
 
   test("create table - partitioned columns") {
-    val query = "CREATE TABLE my_table (id int, name string) PARTITIONED BY (id)"
-    withCreateTableStatement(query) { state =>
-      val transform = IdentityTransform(FieldReference(Seq("id")))
-      assert(state.partitioning == Seq(transform))
-    }
+    val query = "CREATE TABLE my_table (id int, name string) PARTITIONED BY (month int)"
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.schema == new StructType()
+      .add("id", "int")
+      .add("name", "string")
+      .add("month", "int"))
+    assert(desc.partitionColumnNames == Seq("month"))
   }
 
   test("create table - clustered by") {
@@ -726,22 +730,20 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
        """
 
     val query1 = s"$baseQuery INTO $numBuckets BUCKETS"
-    withCreateTableStatement(query1) { state =>
-      assert(state.bucketSpec.isDefined)
-      val bucketSpec = state.bucketSpec.get
-      assert(bucketSpec.numBuckets == numBuckets)
-      assert(bucketSpec.bucketColumnNames.head.equals(bucketedColumn))
-      assert(bucketSpec.sortColumnNames.isEmpty)
-    }
+    val (desc1, _) = extractTableDesc(query1)
+    assert(desc1.bucketSpec.isDefined)
+    val bucketSpec1 = desc1.bucketSpec.get
+    assert(bucketSpec1.numBuckets == numBuckets)
+    assert(bucketSpec1.bucketColumnNames.head.equals(bucketedColumn))
+    assert(bucketSpec1.sortColumnNames.isEmpty)
 
     val query2 = s"$baseQuery SORTED BY($sortColumn) INTO $numBuckets BUCKETS"
-    withCreateTableStatement(query2) { state =>
-      assert(state.bucketSpec.isDefined)
-      val bucketSpec = state.bucketSpec.get
-      assert(bucketSpec.numBuckets == numBuckets)
-      assert(bucketSpec.bucketColumnNames.head.equals(bucketedColumn))
-      assert(bucketSpec.sortColumnNames.head.equals(sortColumn))
-    }
+    val (desc2, _) = extractTableDesc(query2)
+    assert(desc2.bucketSpec.isDefined)
+    val bucketSpec2 = desc2.bucketSpec.get
+    assert(bucketSpec2.numBuckets == numBuckets)
+    assert(bucketSpec2.bucketColumnNames.head.equals(bucketedColumn))
+    assert(bucketSpec2.sortColumnNames.head.equals(sortColumn))
   }
 
   test("create table(hive) - skewed by") {
@@ -811,9 +813,8 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession {
 
   test("create table - properties") {
     val query = "CREATE TABLE my_table (id int, name string) TBLPROPERTIES ('k1'='v1', 'k2'='v2')"
-    withCreateTableStatement(query) { state =>
-      assert(state.properties == Map("k1" -> "v1", "k2" -> "v2"))
-    }
+    val (desc, _) = extractTableDesc(query)
+    assert(desc.properties == Map("k1" -> "v1", "k2" -> "v2"))
   }
 
   test("create table(hive) - everything!") {
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index d941c4bc75b42..db1f6fbd97d90 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -39,7 +39,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   private val originalInMemoryPartitionPruning = TestHive.conf.inMemoryPartitionPruning
   private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
   private val originalSessionLocalTimeZone = TestHive.conf.sessionLocalTimeZone
-  private val originalCreateHiveTable = TestHive.conf.createHiveTableByDefaultEnabled
 
   def testCases: Seq[(String, File)] = {
     hiveQueryDir.listFiles.map(f => f.getName.stripSuffix(".q") -> f)
@@ -59,7 +58,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
     // Fix session local timezone to America/Los_Angeles for those timezone sensitive tests
     // (timestamp_*)
     TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, "America/Los_Angeles")
-    TestHive.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED, true)
     RuleExecutor.resetMetrics()
   }
 
@@ -70,8 +68,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
       TestHive.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
       TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
       TestHive.setConf(SQLConf.SESSION_LOCAL_TIMEZONE, originalSessionLocalTimeZone)
-      TestHive.setConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED,
-        originalCreateHiveTable)
 
       // For debugging dump some statistics about how much time was spent in various optimizer rules
       logWarning(RuleExecutor.dumpTimeSpent())
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
index 1e31e8b1bf234..cfcf70c0e79f0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala
@@ -25,22 +25,6 @@ import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
 
 class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton {
 
-  private var origCreateHiveTableConfig = false
-
-  protected override def beforeAll(): Unit = {
-    super.beforeAll()
-    origCreateHiveTableConfig =
-      spark.conf.get(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED)
-    spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key, true)
-  }
-
-  protected override def afterAll(): Unit = {
-    spark.conf.set(
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key,
-      origCreateHiveTableConfig)
-    super.afterAll()
-  }
-
   test("view") {
     Seq(true, false).foreach { serde =>
       withView("v1") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 61e1fefb5b5df..e8548fd62ddc1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -2706,33 +2706,6 @@ class HiveDDLSuite
     }
   }
 
-  test("SPARK-30098: create table without provider should " +
-    "use default data source under non-legacy mode") {
-    val catalog = spark.sessionState.catalog
-    withSQLConf(
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "false") {
-      withTable("s") {
-        val defaultProvider = conf.defaultDataSourceName
-        sql("CREATE TABLE s(a INT, b INT)")
-        val table = catalog.getTableMetadata(TableIdentifier("s"))
-        assert(table.provider === Some(defaultProvider))
-      }
-    }
-  }
-
-  test("SPARK-30098: create table without provider should " +
-    "use hive under legacy mode") {
-    val catalog = spark.sessionState.catalog
-    withSQLConf(
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
-      withTable("s") {
-        sql("CREATE TABLE s(a INT, b INT)")
-        val table = catalog.getTableMetadata(TableIdentifier("s"))
-        assert(table.provider === Some("hive"))
-      }
-    }
-  }
-
   test("SPARK-30785: create table like a partitioned table") {
     val catalog = spark.sessionState.catalog
     withTable("sc_part", "ta_part") {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
index d2d350221aca0..24b1e3405379c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeSuite.scala
@@ -87,8 +87,7 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
     SQLConf.withExistingConf(TestHive.conf)(super.withSQLConf(pairs: _*)(f))
 
   test("Test the default fileformat for Hive-serde tables") {
-    withSQLConf("hive.default.fileformat" -> "orc",
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
+    withSQLConf("hive.default.fileformat" -> "orc") {
       val (desc, exists) = extractTableDesc(
         "CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
       assert(exists)
@@ -97,8 +96,7 @@ class HiveSerDeSuite extends HiveComparisonTest with PlanTest with BeforeAndAfte
       assert(desc.storage.serde == Some("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
     }
 
-    withSQLConf("hive.default.fileformat" -> "parquet",
-      SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
+    withSQLConf("hive.default.fileformat" -> "parquet") {
       val (desc, exists) = extractTableDesc("CREATE TABLE IF NOT EXISTS fileformat_test (id int)")
       assert(exists)
       val input = desc.storage.inputFormat

From afe2247cffda4dc46c41e3db9d1dc9853beadd28 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Sun, 17 May 2020 02:32:39 +0000
Subject: [PATCH 0687/1280] [SPARK-31405][SQL][3.0] Fail by default when
 reading/writing legacy datetime values from/to Parquet/Avro files

### What changes were proposed in this pull request?

When reading/writing datetime values that before the rebase switch day, from/to Avro/Parquet files, fail by default and ask users to set a config to explicitly do rebase or not.

### Why are the changes needed?

Rebase or not rebase have different behaviors and we should let users decide it explicitly. In most cases, users won't hit this exception as it only affects ancient datetime values.

### Does this PR introduce _any_ user-facing change?

Yes, now users will see an error when reading/writing dates before 1582-10-15 or timestamps before 1900-01-01 from/to Parquet/Avro files, with an error message to ask setting a config.

### How was this patch tested?

updated tests

Closes #28526 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/SparkException.scala     |   2 +-
 .../spark/sql/avro/AvroDeserializer.scala     |  37 +++----
 .../spark/sql/avro/AvroFileFormat.scala       |  10 +-
 .../spark/sql/avro/AvroOutputWriter.scala     |  13 ++-
 .../spark/sql/avro/AvroSerializer.scala       |  30 ++---
 .../v2/avro/AvroPartitionReaderFactory.scala  |   9 +-
 .../AvroCatalystDataConversionSuite.scala     |   2 +-
 .../org/apache/spark/sql/avro/AvroSuite.scala |  91 +++++++++++-----
 .../sql/catalyst/json/JacksonParser.scala     |   1 +
 .../sql/catalyst/util/RebaseDateTime.scala    |   4 +-
 .../apache/spark/sql/internal/SQLConf.scala   | 100 ++++++++---------
 .../parquet/VectorizedColumnReader.java       |  91 +++++++++++-----
 .../VectorizedParquetRecordReader.java        |  12 +-
 .../parquet/VectorizedPlainValuesReader.java  |  29 +++--
 .../parquet/VectorizedRleValuesReader.java    |  31 +++---
 .../parquet/VectorizedValuesReader.java       |   4 +-
 .../datasources/DataSourceUtils.scala         | 103 +++++++++++++++++-
 .../execution/datasources/FileScanRDD.scala   |   6 +-
 .../parquet/ParquetFileFormat.scala           |  11 +-
 .../parquet/ParquetReadSupport.scala          |   7 +-
 .../parquet/ParquetRecordMaterializer.scala   |   9 +-
 .../parquet/ParquetRowConverter.scala         |  68 +++++-------
 .../parquet/ParquetWriteSupport.scala         |  46 ++++----
 .../ParquetPartitionReaderFactory.scala       |  23 ++--
 .../benchmark/DateTimeRebaseBenchmark.scala   |   4 +-
 .../datasources/parquet/ParquetIOSuite.scala  |  89 +++++++++------
 .../sql/sources/HadoopFsRelationTest.scala    |   6 +-
 27 files changed, 519 insertions(+), 319 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala b/core/src/main/scala/org/apache/spark/SparkException.scala
index 81c087e314be1..41382133bd84c 100644
--- a/core/src/main/scala/org/apache/spark/SparkException.scala
+++ b/core/src/main/scala/org/apache/spark/SparkException.scala
@@ -48,5 +48,5 @@ private[spark] case class ExecutorDeadException(message: String)
  * Exception thrown when Spark returns different result after upgrading to a new version.
  */
 private[spark] class SparkUpgradeException(version: String, message: String, cause: Throwable)
-  extends SparkException("You may get a different result due to the upgrading of Spark" +
+  extends RuntimeException("You may get a different result due to the upgrading of Spark" +
     s" $version: $message", cause)
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 27206edb287f9..4fc804067d54f 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -34,22 +34,33 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
-import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 /**
  * A deserializer to deserialize data in avro format to data in catalyst format.
  */
-class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType, rebaseDateTime: Boolean) {
+class AvroDeserializer(
+    rootAvroType: Schema,
+    rootCatalystType: DataType,
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value) {
 
   def this(rootAvroType: Schema, rootCatalystType: DataType) {
     this(rootAvroType, rootCatalystType,
-      SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ))
+      LegacyBehaviorPolicy.withName(
+        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ)))
   }
 
   private lazy val decimalConversions = new DecimalConversion()
 
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInRead(
+    datetimeRebaseMode, "Avro")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
+    datetimeRebaseMode, "Avro")
+
   private val converter: Any => Any = rootCatalystType match {
     // A shortcut for empty schema.
     case st: StructType if st.isEmpty =>
@@ -96,13 +107,8 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType, rebaseD
       case (INT, IntegerType) => (updater, ordinal, value) =>
         updater.setInt(ordinal, value.asInstanceOf[Int])
 
-      case (INT, DateType) if rebaseDateTime => (updater, ordinal, value) =>
-        val days = value.asInstanceOf[Int]
-        val rebasedDays = rebaseJulianToGregorianDays(days)
-        updater.setInt(ordinal, rebasedDays)
-
       case (INT, DateType) => (updater, ordinal, value) =>
-        updater.setInt(ordinal, value.asInstanceOf[Int])
+        updater.setInt(ordinal, dateRebaseFunc(value.asInstanceOf[Int]))
 
       case (LONG, LongType) => (updater, ordinal, value) =>
         updater.setLong(ordinal, value.asInstanceOf[Long])
@@ -110,22 +116,13 @@ class AvroDeserializer(rootAvroType: Schema, rootCatalystType: DataType, rebaseD
       case (LONG, TimestampType) => avroType.getLogicalType match {
         // For backward compatibility, if the Avro type is Long and it is not logical type
         // (the `null` case), the value is processed as timestamp type with millisecond precision.
-        case null | _: TimestampMillis if rebaseDateTime => (updater, ordinal, value) =>
-          val millis = value.asInstanceOf[Long]
-          val micros = DateTimeUtils.fromMillis(millis)
-          val rebasedMicros = rebaseJulianToGregorianMicros(micros)
-          updater.setLong(ordinal, rebasedMicros)
         case null | _: TimestampMillis => (updater, ordinal, value) =>
           val millis = value.asInstanceOf[Long]
           val micros = DateTimeUtils.fromMillis(millis)
-          updater.setLong(ordinal, micros)
-        case _: TimestampMicros if rebaseDateTime => (updater, ordinal, value) =>
-          val micros = value.asInstanceOf[Long]
-          val rebasedMicros = rebaseJulianToGregorianMicros(micros)
-          updater.setLong(ordinal, rebasedMicros)
+          updater.setLong(ordinal, timestampRebaseFunc(micros))
         case _: TimestampMicros => (updater, ordinal, value) =>
           val micros = value.asInstanceOf[Long]
-          updater.setLong(ordinal, micros)
+          updater.setLong(ordinal, timestampRebaseFunc(micros))
         case other => throw new IncompatibleSchemaException(
           s"Cannot convert Avro logical type ${other} to Catalyst Timestamp type.")
       }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
index e69c95b797c73..59d54bc433f8b 100755
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala
@@ -124,12 +124,12 @@ private[sql] class AvroFileFormat extends FileFormat
         reader.sync(file.start)
         val stop = file.start + file.length
 
-        val rebaseDateTime = DataSourceUtils.needRebaseDateTime(
-          reader.asInstanceOf[DataFileReader[_]].getMetaString).getOrElse {
-          SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ)
-        }
+        val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+          reader.asInstanceOf[DataFileReader[_]].getMetaString,
+          SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
+
         val deserializer = new AvroDeserializer(
-          userProvidedSchema.getOrElse(reader.getSchema), requiredSchema, rebaseDateTime)
+          userProvidedSchema.getOrElse(reader.getSchema), requiredSchema, datetimeRebaseMode)
 
         new Iterator[InternalRow] {
           private[this] var completed = false
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
index 82a568049990e..ac9608c867937 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOutputWriter.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriter
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 // NOTE: This class is instantiated and used on executor side only, no need to be serializable.
@@ -43,12 +44,12 @@ private[avro] class AvroOutputWriter(
     avroSchema: Schema) extends OutputWriter {
 
   // Whether to rebase datetimes from Gregorian to Julian calendar in write
-  private val rebaseDateTime: Boolean =
-    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE)
+  private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
+    SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE))
 
   // The input rows will never be null.
   private lazy val serializer =
-    new AvroSerializer(schema, avroSchema, nullable = false, rebaseDateTime)
+    new AvroSerializer(schema, avroSchema, nullable = false, datetimeRebaseMode)
 
   /**
    * Overrides the couple of methods responsible for generating the output streams / files so
@@ -56,7 +57,11 @@ private[avro] class AvroOutputWriter(
    */
   private val recordWriter: RecordWriter[AvroKey[GenericRecord], NullWritable] = {
     val fileMeta = Map(SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT) ++ {
-      if (rebaseDateTime) Some(SPARK_LEGACY_DATETIME -> "") else None
+      if (datetimeRebaseMode == LegacyBehaviorPolicy.LEGACY) {
+        Some(SPARK_LEGACY_DATETIME -> "")
+      } else {
+        None
+      }
     }
 
     new SparkAvroKeyOutputFormat(fileMeta.asJava) {
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index dc232168fd241..d6cfbc5ffe758 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -35,8 +35,9 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 /**
@@ -46,17 +47,24 @@ class AvroSerializer(
     rootCatalystType: DataType,
     rootAvroType: Schema,
     nullable: Boolean,
-    rebaseDateTime: Boolean) extends Logging {
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value) extends Logging {
 
   def this(rootCatalystType: DataType, rootAvroType: Schema, nullable: Boolean) {
     this(rootCatalystType, rootAvroType, nullable,
-      SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE))
+      LegacyBehaviorPolicy.withName(SQLConf.get.getConf(
+        SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE)))
   }
 
   def serialize(catalystData: Any): Any = {
     converter.apply(catalystData)
   }
 
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInWrite(
+    datetimeRebaseMode, "Avro")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInWrite(
+    datetimeRebaseMode, "Avro")
+
   private val converter: Any => Any = {
     val actualAvroType = resolveNullableType(rootAvroType, nullable)
     val baseConverter = rootCatalystType match {
@@ -146,24 +154,16 @@ class AvroSerializer(
       case (BinaryType, BYTES) =>
         (getter, ordinal) => ByteBuffer.wrap(getter.getBinary(ordinal))
 
-      case (DateType, INT) if rebaseDateTime =>
-        (getter, ordinal) => rebaseGregorianToJulianDays(getter.getInt(ordinal))
-
       case (DateType, INT) =>
-        (getter, ordinal) => getter.getInt(ordinal)
+        (getter, ordinal) => dateRebaseFunc(getter.getInt(ordinal))
 
       case (TimestampType, LONG) => avroType.getLogicalType match {
           // For backward compatibility, if the Avro type is Long and it is not logical type
           // (the `null` case), output the timestamp value as with millisecond precision.
-          case null | _: TimestampMillis if rebaseDateTime => (getter, ordinal) =>
-            val micros = getter.getLong(ordinal)
-            val rebasedMicros = rebaseGregorianToJulianMicros(micros)
-            DateTimeUtils.toMillis(rebasedMicros)
           case null | _: TimestampMillis => (getter, ordinal) =>
-            DateTimeUtils.toMillis(getter.getLong(ordinal))
-          case _: TimestampMicros if rebaseDateTime => (getter, ordinal) =>
-            rebaseGregorianToJulianMicros(getter.getLong(ordinal))
-          case _: TimestampMicros => (getter, ordinal) => getter.getLong(ordinal)
+            DateTimeUtils.toMillis(timestampRebaseFunc(getter.getLong(ordinal)))
+          case _: TimestampMicros => (getter, ordinal) =>
+            timestampRebaseFunc(getter.getLong(ordinal))
           case other => throw new IncompatibleSchemaException(
             s"Cannot convert Catalyst Timestamp type to Avro logical type ${other}")
         }
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
index 712aec6acbd56..15918f46a83bb 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroPartitionReaderFactory.scala
@@ -88,12 +88,11 @@ case class AvroPartitionReaderFactory(
       reader.sync(partitionedFile.start)
       val stop = partitionedFile.start + partitionedFile.length
 
-      val rebaseDateTime = DataSourceUtils.needRebaseDateTime(
-        reader.asInstanceOf[DataFileReader[_]].getMetaString).getOrElse {
-        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ)
-      }
+      val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+        reader.asInstanceOf[DataFileReader[_]].getMetaString,
+        SQLConf.get.getConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ))
       val deserializer = new AvroDeserializer(
-        userProvidedSchema.getOrElse(reader.getSchema), readDataSchema, rebaseDateTime)
+        userProvidedSchema.getOrElse(reader.getSchema), readDataSchema, datetimeRebaseMode)
 
       val fileReader = new PartitionReader[InternalRow] {
         private[this] var completed = false
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
index 64d790bc4acd4..c8a1f670bda9e 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroCatalystDataConversionSuite.scala
@@ -288,7 +288,7 @@ class AvroCatalystDataConversionSuite extends SparkFunSuite
       """.stripMargin
     val avroSchema = new Schema.Parser().parse(jsonFormatSchema)
     val dataType = SchemaConverters.toSqlType(avroSchema).dataType
-    val deserializer = new AvroDeserializer(avroSchema, dataType, rebaseDateTime = false)
+    val deserializer = new AvroDeserializer(avroSchema, dataType)
 
     def checkDeserialization(data: GenericData.Record, expected: Any): Unit = {
       assert(checkResult(
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 3e754f02911dc..a5c1fb15add5c 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -33,7 +33,7 @@ import org.apache.avro.generic.{GenericData, GenericDatumReader, GenericDatumWri
 import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed}
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException}
+import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf, SparkException, SparkUpgradeException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
@@ -43,6 +43,7 @@ import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.v2.avro.AvroScan
@@ -1538,13 +1539,28 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         val path3_0_rebase = paths(1).getCanonicalPath
         if (dt == "date") {
           val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
-          df.write.format("avro").save(path3_0)
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+
+          // By default we should fail to write ancient datetime values.
+          var e = intercept[SparkException](df.write.format("avro").save(path3_0))
+          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          // By default we should fail to read ancient datetime values.
+          e = intercept[SparkException](spark.read.format("avro").load(path2_4).collect())
+          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+            df.write.format("avro").mode("overwrite").save(path3_0)
+          }
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
             df.write.format("avro").save(path3_0_rebase)
           }
-          checkAnswer(
-            spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
-            1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+
+          // For Avro files written by Spark 3.0, we know the writer info and don't need the config
+          // to guide the rebase behavior.
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+            checkAnswer(
+              spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+              1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+          }
         } else {
           val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
           val avroSchema =
@@ -1556,24 +1572,39 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
               |    {"name": "ts", "type": {"type": "long", "logicalType": "$dt"}}
               |  ]
               |}""".stripMargin
-          df.write.format("avro").option("avroSchema", avroSchema).save(path3_0)
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+
+          // By default we should fail to write ancient datetime values.
+          var e = intercept[SparkException] {
+            df.write.format("avro").option("avroSchema", avroSchema).save(path3_0)
+          }
+          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          // By default we should fail to read ancient datetime values.
+          e = intercept[SparkException](spark.read.format("avro").load(path2_4).collect())
+          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+            df.write.format("avro").option("avroSchema", avroSchema).mode("overwrite").save(path3_0)
+          }
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
             df.write.format("avro").option("avroSchema", avroSchema).save(path3_0_rebase)
           }
-          checkAnswer(
-            spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
-            1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+
+          // For Avro files written by Spark 3.0, we know the writer info and don't need the config
+          // to guide the rebase behavior.
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+            checkAnswer(
+              spark.read.format("avro").load(path2_4, path3_0, path3_0_rebase),
+              1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+          }
         }
       }
     }
 
-    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> "true") {
-      checkReadMixedFiles("before_1582_date_v2_4.avro", "date", "1001-01-01")
-      checkReadMixedFiles(
-        "before_1582_ts_micros_v2_4.avro", "timestamp-micros", "1001-01-01 01:02:03.123456")
-      checkReadMixedFiles(
-        "before_1582_ts_millis_v2_4.avro", "timestamp-millis", "1001-01-01 01:02:03.124")
-    }
+    checkReadMixedFiles("before_1582_date_v2_4.avro", "date", "1001-01-01")
+    checkReadMixedFiles(
+      "before_1582_ts_micros_v2_4.avro", "timestamp-micros", "1001-01-01 01:02:03.123456")
+    checkReadMixedFiles(
+      "before_1582_ts_millis_v2_4.avro", "timestamp-millis", "1001-01-01 01:02:03.124")
   }
 
   test("SPARK-31183: rebasing microseconds timestamps in write") {
@@ -1581,7 +1612,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     val nonRebased = "1001-01-07 01:09:05.123456"
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
         Seq(tsStr).toDF("tsS")
           .select($"tsS".cast("timestamp").as("ts"))
           .write.format("avro")
@@ -1589,9 +1620,9 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       }
 
       // The file metadata indicates if it needs rebase or not, so we can always get the correct
-      // result regardless of the "rebaseInRead" config.
-      Seq(true, false).foreach { rebase =>
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+      // result regardless of the "rebase mode" config.
+      Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
           checkAnswer(spark.read.format("avro").load(path), Row(Timestamp.valueOf(tsStr)))
         }
       }
@@ -1622,7 +1653,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         |}""".stripMargin
       withTempPath { dir =>
         val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
           Seq(tsStr).toDF("tsS")
             .select($"tsS".cast("timestamp").as("ts"))
             .write
@@ -1632,9 +1663,9 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
         }
 
         // The file metadata indicates if it needs rebase or not, so we can always get the correct
-        // result regardless of the "rebaseInRead" config.
-        Seq(true, false).foreach { rebase =>
-          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+        // result regardless of the "rebase mode" config.
+        Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+          withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
             checkAnswer(
               spark.read.schema("ts timestamp").format("avro").load(path),
               Row(Timestamp.valueOf(rebased)))
@@ -1655,7 +1686,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
   test("SPARK-31183: rebasing dates in write") {
     withTempPath { dir =>
       val path = dir.getAbsolutePath
-      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_WRITE.key -> "true") {
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
         Seq("1001-01-01").toDF("dateS")
           .select($"dateS".cast("date").as("date"))
           .write.format("avro")
@@ -1663,9 +1694,9 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       }
 
       // The file metadata indicates if it needs rebase or not, so we can always get the correct
-      // result regardless of the "rebaseInRead" config.
-      Seq(true, false).foreach { rebase =>
-        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+      // result regardless of the "rebase mode" config.
+      Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+        withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
           checkAnswer(spark.read.format("avro").load(path), Row(Date.valueOf("1001-01-01")))
         }
       }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index a52c3450e83df..ef987931e928a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -456,6 +456,7 @@ class JacksonParser(
         }
       }
     } catch {
+      case e: SparkUpgradeException => throw e
       case e @ (_: RuntimeException | _: JsonProcessingException | _: MalformedInputException) =>
         // JSON parser currently doesn't support partial results for corrupted records.
         // For such records, all fields other than the field configured by
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index eb67ff71f6694..e29fa4b37844d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -146,6 +146,8 @@ object RebaseDateTime {
     -354226, -317702, -244653, -208129, -171605, -141436, -141435, -141434,
     -141433, -141432, -141431, -141430, -141429, -141428, -141427)
 
+  final val lastSwitchGregorianDay: Int = gregJulianDiffSwitchDay.last
+
   // The first days of Common Era (CE) which is mapped to the '0001-01-01' date
   // in Proleptic Gregorian calendar.
   private final val gregorianCommonEraStartDay = gregJulianDiffSwitchDay(0)
@@ -295,7 +297,7 @@ object RebaseDateTime {
   }
   // The switch time point after which all diffs between Gregorian and Julian calendars
   // across all time zones are zero
-  private final val lastSwitchGregorianTs: Long = getLastSwitchTs(gregJulianRebaseMap)
+  final val lastSwitchGregorianTs: Long = getLastSwitchTs(gregJulianRebaseMap)
 
   private final val gregorianStartTs = LocalDateTime.of(gregorianStartDate, LocalTime.MIDNIGHT)
   private final val julianEndTs = LocalDateTime.of(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b073f7ea599fd..d0b55ceb7b20f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2500,57 +2500,63 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
-  val LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE =
-    buildConf("spark.sql.legacy.parquet.rebaseDateTimeInWrite.enabled")
-      .internal()
-      .doc("When true, rebase dates/timestamps from Proleptic Gregorian calendar " +
-        "to the hybrid calendar (Julian + Gregorian) in write. " +
-        "The rebasing is performed by converting micros/millis/days to " +
-        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
-        "timestamp in the target calendar, and getting the number of micros/millis/days " +
-        "since the epoch 1970-01-01 00:00:00Z.")
+  val LEGACY_PARQUET_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.legacy.parquet.datetimeRebaseModeInWrite")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar " +
+        "to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. " +
+        "When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars.")
       .version("3.0.0")
-      .booleanConf
-      .createWithDefault(false)
-
-  val LEGACY_PARQUET_REBASE_DATETIME_IN_READ =
-    buildConf("spark.sql.legacy.parquet.rebaseDateTimeInRead.enabled")
-      .internal()
-      .doc("When true, rebase dates/timestamps " +
-        "from the hybrid calendar to Proleptic Gregorian calendar in read. " +
-        "The rebasing is performed by converting micros/millis/days to " +
-        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
-        "timestamp in the target calendar, and getting the number of micros/millis/days " +
-        "since the epoch 1970-01-01 00:00:00Z.")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val LEGACY_PARQUET_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.legacy.parquet.datetimeRebaseModeInRead")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + " +
+        "Gregorian) calendar to Proleptic Gregorian calendar when reading Parquet files. " +
+        "When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the reading if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
+        "only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.")
       .version("3.0.0")
-      .booleanConf
-      .createWithDefault(false)
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
-  val LEGACY_AVRO_REBASE_DATETIME_IN_WRITE =
-    buildConf("spark.sql.legacy.avro.rebaseDateTimeInWrite.enabled")
+  val LEGACY_AVRO_REBASE_MODE_IN_WRITE =
+    buildConf("spark.sql.legacy.avro.datetimeRebaseModeInWrite")
       .internal()
-      .doc("When true, rebase dates/timestamps from Proleptic Gregorian calendar " +
-        "to the hybrid calendar (Julian + Gregorian) in write. " +
-        "The rebasing is performed by converting micros/millis/days to " +
-        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
-        "timestamp in the target calendar, and getting the number of micros/millis/days " +
-        "since the epoch 1970-01-01 00:00:00Z.")
+      .doc("When LEGACY, Spark will rebase dates/timestamps from Proleptic Gregorian calendar " +
+        "to the legacy hybrid (Julian + Gregorian) calendar when writing Avro files. " +
+        "When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars.")
       .version("3.0.0")
-      .booleanConf
-      .createWithDefault(false)
-
-  val LEGACY_AVRO_REBASE_DATETIME_IN_READ =
-    buildConf("spark.sql.legacy.avro.rebaseDateTimeInRead.enabled")
-      .internal()
-      .doc("When true, rebase dates/timestamps " +
-        "from the hybrid calendar to Proleptic Gregorian calendar in read. " +
-        "The rebasing is performed by converting micros/millis/days to " +
-        "a local date/timestamp in the source calendar, interpreting the resulted date/" +
-        "timestamp in the target calendar, and getting the number of micros/millis/days " +
-        "since the epoch 1970-01-01 00:00:00Z.")
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
+
+  val LEGACY_AVRO_REBASE_MODE_IN_READ =
+    buildConf("spark.sql.legacy.avro.datetimeRebaseModeInRead")
+      .internal()
+      .doc("When LEGACY, Spark will rebase dates/timestamps from the legacy hybrid (Julian + " +
+        "Gregorian) calendar to Proleptic Gregorian calendar when reading Avro files. " +
+        "When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. " +
+        "When EXCEPTION, which is the default, Spark will fail the reading if it sees " +
+        "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
+        "only effective if the writer info (like Spark, Hive) of the Avro files is unknown.")
       .version("3.0.0")
-      .booleanConf
-      .createWithDefault(false)
+      .stringConf
+      .transform(_.toUpperCase(Locale.ROOT))
+      .checkValues(LegacyBehaviorPolicy.values.map(_.toString))
+      .createWithDefault(LegacyBehaviorPolicy.EXCEPTION.toString)
 
   val SCRIPT_TRANSFORMATION_EXIT_TIMEOUT =
     buildConf("spark.sql.scriptTransformation.exitTimeoutInSeconds")
@@ -3136,10 +3142,6 @@ class SQLConf extends Serializable with Logging {
 
   def csvFilterPushDown: Boolean = getConf(CSV_FILTER_PUSHDOWN_ENABLED)
 
-  def parquetRebaseDateTimeInReadEnabled: Boolean = {
-    getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ)
-  }
-
   /** ********************** SQLConf functionality methods ************ */
 
   /** Set Spark SQL configuration properties. */
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
index 11ce11dd7219c..f264281c02f45 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedColumnReader.java
@@ -36,6 +36,7 @@
 
 import org.apache.spark.sql.catalyst.util.DateTimeUtils;
 import org.apache.spark.sql.catalyst.util.RebaseDateTime;
+import org.apache.spark.sql.execution.datasources.DataSourceUtils;
 import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 import org.apache.spark.sql.types.DataTypes;
@@ -102,14 +103,14 @@ public class VectorizedColumnReader {
   // The timezone conversion to apply to int96 timestamps. Null if no conversion.
   private final ZoneId convertTz;
   private static final ZoneId UTC = ZoneOffset.UTC;
-  private final boolean rebaseDateTime;
+  private final String datetimeRebaseMode;
 
   public VectorizedColumnReader(
       ColumnDescriptor descriptor,
       OriginalType originalType,
       PageReader pageReader,
       ZoneId convertTz,
-      boolean rebaseDateTime) throws IOException {
+      String datetimeRebaseMode) throws IOException {
     this.descriptor = descriptor;
     this.pageReader = pageReader;
     this.convertTz = convertTz;
@@ -132,7 +133,9 @@ public VectorizedColumnReader(
     if (totalValueCount == 0) {
       throw new IOException("totalValueCount == 0");
     }
-    this.rebaseDateTime = rebaseDateTime;
+    assert "LEGACY".equals(datetimeRebaseMode) || "EXCEPTION".equals(datetimeRebaseMode) ||
+      "CORRECTED".equals(datetimeRebaseMode);
+    this.datetimeRebaseMode = datetimeRebaseMode;
   }
 
   /**
@@ -156,11 +159,11 @@ private boolean isLazyDecodingSupported(PrimitiveType.PrimitiveTypeName typeName
     boolean isSupported = false;
     switch (typeName) {
       case INT32:
-        isSupported = originalType != OriginalType.DATE || !rebaseDateTime;
+        isSupported = originalType != OriginalType.DATE || "CORRECTED".equals(datetimeRebaseMode);
         break;
       case INT64:
         if (originalType == OriginalType.TIMESTAMP_MICROS) {
-          isSupported = !rebaseDateTime;
+          isSupported = "CORRECTED".equals(datetimeRebaseMode);
         } else {
           isSupported = originalType != OriginalType.TIMESTAMP_MILLIS;
         }
@@ -174,6 +177,30 @@ private boolean isLazyDecodingSupported(PrimitiveType.PrimitiveTypeName typeName
     return isSupported;
   }
 
+  static int rebaseDays(int julianDays, final boolean failIfRebase) {
+    if (failIfRebase) {
+      if (julianDays < RebaseDateTime.lastSwitchJulianDay()) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        return julianDays;
+      }
+    } else {
+      return RebaseDateTime.rebaseJulianToGregorianDays(julianDays);
+    }
+  }
+
+  static long rebaseMicros(long julianMicros, final boolean failIfRebase) {
+    if (failIfRebase) {
+      if (julianMicros < RebaseDateTime.lastSwitchJulianTs()) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        return julianMicros;
+      }
+    } else {
+      return RebaseDateTime.rebaseJulianToGregorianMicros(julianMicros);
+    }
+  }
+
   /**
    * Reads `total` values from this columnReader into column.
    */
@@ -283,7 +310,7 @@ private void decodeDictionaryIds(
       case INT32:
         if (column.dataType() == DataTypes.IntegerType ||
             DecimalType.is32BitDecimalType(column.dataType()) ||
-            (column.dataType() == DataTypes.DateType && !rebaseDateTime)) {
+            (column.dataType() == DataTypes.DateType && "CORRECTED".equals(datetimeRebaseMode))) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putInt(i, dictionary.decodeToInt(dictionaryIds.getDictId(i)));
@@ -302,11 +329,11 @@ private void decodeDictionaryIds(
             }
           }
         } else if (column.dataType() == DataTypes.DateType) {
+          final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               int julianDays = dictionary.decodeToInt(dictionaryIds.getDictId(i));
-              int gregorianDays = RebaseDateTime.rebaseJulianToGregorianDays(julianDays);
-              column.putInt(i, gregorianDays);
+              column.putInt(i, rebaseDays(julianDays, failIfRebase));
             }
           }
         } else {
@@ -317,36 +344,37 @@ private void decodeDictionaryIds(
       case INT64:
         if (column.dataType() == DataTypes.LongType ||
             DecimalType.is64BitDecimalType(column.dataType()) ||
-            (originalType == OriginalType.TIMESTAMP_MICROS && !rebaseDateTime)) {
+            (originalType == OriginalType.TIMESTAMP_MICROS &&
+              "CORRECTED".equals(datetimeRebaseMode))) {
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               column.putLong(i, dictionary.decodeToLong(dictionaryIds.getDictId(i)));
             }
           }
         } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
-          if (rebaseDateTime) {
+          if ("CORRECTED".equals(datetimeRebaseMode)) {
             for (int i = rowId; i < rowId + num; ++i) {
               if (!column.isNullAt(i)) {
-                long julianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
-                long julianMicros = DateTimeUtils.fromMillis(julianMillis);
-                long gregorianMicros = RebaseDateTime.rebaseJulianToGregorianMicros(julianMicros);
-                column.putLong(i, gregorianMicros);
+                long gregorianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+                column.putLong(i, DateTimeUtils.fromMillis(gregorianMillis));
               }
             }
           } else {
+            final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
             for (int i = rowId; i < rowId + num; ++i) {
               if (!column.isNullAt(i)) {
-                long gregorianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
-                column.putLong(i, DateTimeUtils.fromMillis(gregorianMillis));
+                long julianMillis = dictionary.decodeToLong(dictionaryIds.getDictId(i));
+                long julianMicros = DateTimeUtils.fromMillis(julianMillis);
+                column.putLong(i, rebaseMicros(julianMicros, failIfRebase));
               }
             }
           }
         } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
+          final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
           for (int i = rowId; i < rowId + num; ++i) {
             if (!column.isNullAt(i)) {
               long julianMicros = dictionary.decodeToLong(dictionaryIds.getDictId(i));
-              long gregorianMicros = RebaseDateTime.rebaseJulianToGregorianMicros(julianMicros);
-              column.putLong(i, gregorianMicros);
+              column.putLong(i, rebaseMicros(julianMicros, failIfRebase));
             }
           }
         } else {
@@ -466,12 +494,13 @@ private void readIntBatch(int rowId, int num, WritableColumnVector column) throw
       defColumn.readShorts(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
     } else if (column.dataType() == DataTypes.DateType ) {
-      if (rebaseDateTime) {
-        defColumn.readIntegersWithRebase(
+      if ("CORRECTED".equals(datetimeRebaseMode)) {
+        defColumn.readIntegers(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
       } else {
-        defColumn.readIntegers(
-           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+        boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+        defColumn.readIntegersWithRebase(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn, failIfRebase);
       }
     } else {
       throw constructConvertNotSupportedException(descriptor, column);
@@ -485,27 +514,29 @@ private void readLongBatch(int rowId, int num, WritableColumnVector column) thro
       defColumn.readLongs(
         num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
     } else if (originalType == OriginalType.TIMESTAMP_MICROS) {
-      if (rebaseDateTime) {
-        defColumn.readLongsWithRebase(
-          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
-      } else {
+      if ("CORRECTED".equals(datetimeRebaseMode)) {
         defColumn.readLongs(
           num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn);
+      } else {
+        boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
+        defColumn.readLongsWithRebase(
+          num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn, failIfRebase);
       }
     } else if (originalType == OriginalType.TIMESTAMP_MILLIS) {
-      if (rebaseDateTime) {
+      if ("CORRECTED".equals(datetimeRebaseMode)) {
         for (int i = 0; i < num; i++) {
           if (defColumn.readInteger() == maxDefLevel) {
-            long micros = DateTimeUtils.fromMillis(dataColumn.readLong());
-            column.putLong(rowId + i, RebaseDateTime.rebaseJulianToGregorianMicros(micros));
+            column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong()));
           } else {
             column.putNull(rowId + i);
           }
         }
       } else {
+        final boolean failIfRebase = "EXCEPTION".equals(datetimeRebaseMode);
         for (int i = 0; i < num; i++) {
           if (defColumn.readInteger() == maxDefLevel) {
-            column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong()));
+            long julianMicros = DateTimeUtils.fromMillis(dataColumn.readLong());
+            column.putLong(rowId + i, rebaseMicros(julianMicros, failIfRebase));
           } else {
             column.putNull(rowId + i);
           }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
index c9590b97ce9cd..b40cc154d76fe 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedParquetRecordReader.java
@@ -89,9 +89,9 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
   private final ZoneId convertTz;
 
   /**
-   * true if need to rebase date/timestamp from Julian to Proleptic Gregorian calendar.
+   * The mode of rebasing date/timestamp from Julian to Proleptic Gregorian calendar.
    */
-  private final boolean rebaseDateTime;
+  private final String datetimeRebaseMode;
 
   /**
    * columnBatch object that is used for batch decoding. This is created on first use and triggers
@@ -122,16 +122,16 @@ public class VectorizedParquetRecordReader extends SpecificParquetRecordReaderBa
   private final MemoryMode MEMORY_MODE;
 
   public VectorizedParquetRecordReader(
-    ZoneId convertTz, boolean rebaseDateTime, boolean useOffHeap, int capacity) {
+    ZoneId convertTz, String datetimeRebaseMode, boolean useOffHeap, int capacity) {
     this.convertTz = convertTz;
-    this.rebaseDateTime = rebaseDateTime;
+    this.datetimeRebaseMode = datetimeRebaseMode;
     MEMORY_MODE = useOffHeap ? MemoryMode.OFF_HEAP : MemoryMode.ON_HEAP;
     this.capacity = capacity;
   }
 
   // For test only.
   public VectorizedParquetRecordReader(boolean useOffHeap, int capacity) {
-    this(null, false, useOffHeap, capacity);
+    this(null, "CORRECTED", useOffHeap, capacity);
   }
 
   /**
@@ -321,7 +321,7 @@ private void checkEndOfRowGroup() throws IOException {
     for (int i = 0; i < columns.size(); ++i) {
       if (missingColumns[i]) continue;
       columnReaders[i] = new VectorizedColumnReader(columns.get(i), types.get(i).getOriginalType(),
-        pages.getPageReader(columns.get(i)), convertTz, rebaseDateTime);
+        pages.getPageReader(columns.get(i)), convertTz, datetimeRebaseMode);
     }
     totalCountLoadedSoFar += pages.getRowCount();
   }
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
index 2ed2e11b60c03..eddbf39178e9a 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
@@ -21,13 +21,14 @@
 import java.nio.ByteOrder;
 
 import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.io.ParquetDecodingException;
+
 import org.apache.spark.sql.catalyst.util.RebaseDateTime;
+import org.apache.spark.sql.execution.datasources.DataSourceUtils;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
-import org.apache.parquet.column.values.ValuesReader;
-import org.apache.parquet.io.api.Binary;
-
 /**
  * An implementation of the Parquet PLAIN decoder that supports the vectorized interface.
  */
@@ -86,7 +87,8 @@ public final void readIntegers(int total, WritableColumnVector c, int rowId) {
   // iterates the values twice: check if we need to rebase first, then go to the optimized branch
   // if rebase is not needed.
   @Override
-  public final void readIntegersWithRebase(int total, WritableColumnVector c, int rowId) {
+  public final void readIntegersWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
     int requiredBytes = total * 4;
     ByteBuffer buffer = getBuffer(requiredBytes);
     boolean rebase = false;
@@ -94,8 +96,12 @@ public final void readIntegersWithRebase(int total, WritableColumnVector c, int
       rebase |= buffer.getInt(buffer.position() + i * 4) < RebaseDateTime.lastSwitchJulianDay();
     }
     if (rebase) {
-      for (int i = 0; i < total; i += 1) {
-        c.putInt(rowId + i, RebaseDateTime.rebaseJulianToGregorianDays(buffer.getInt()));
+      if (failIfRebase) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putInt(rowId + i, RebaseDateTime.rebaseJulianToGregorianDays(buffer.getInt()));
+        }
       }
     } else {
       if (buffer.hasArray()) {
@@ -128,7 +134,8 @@ public final void readLongs(int total, WritableColumnVector c, int rowId) {
   // iterates the values twice: check if we need to rebase first, then go to the optimized branch
   // if rebase is not needed.
   @Override
-  public final void readLongsWithRebase(int total, WritableColumnVector c, int rowId) {
+  public final void readLongsWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
     int requiredBytes = total * 8;
     ByteBuffer buffer = getBuffer(requiredBytes);
     boolean rebase = false;
@@ -136,8 +143,12 @@ public final void readLongsWithRebase(int total, WritableColumnVector c, int row
       rebase |= buffer.getLong(buffer.position() + i * 8) < RebaseDateTime.lastSwitchJulianTs();
     }
     if (rebase) {
-      for (int i = 0; i < total; i += 1) {
-        c.putLong(rowId + i, RebaseDateTime.rebaseJulianToGregorianMicros(buffer.getLong()));
+      if (failIfRebase) {
+        throw DataSourceUtils.newRebaseExceptionInRead("Parquet");
+      } else {
+        for (int i = 0; i < total; i += 1) {
+          c.putLong(rowId + i, RebaseDateTime.rebaseJulianToGregorianMicros(buffer.getLong()));
+        }
       }
     } else {
       if (buffer.hasArray()) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index 4d72a33fcf774..24347a4e3a0c5 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.execution.datasources.parquet;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
 import org.apache.parquet.Preconditions;
 import org.apache.parquet.bytes.ByteBufferInputStream;
 import org.apache.parquet.bytes.BytesUtils;
@@ -26,12 +29,8 @@
 import org.apache.parquet.io.ParquetDecodingException;
 import org.apache.parquet.io.api.Binary;
 
-import org.apache.spark.sql.catalyst.util.RebaseDateTime;
 import org.apache.spark.sql.execution.vectorized.WritableColumnVector;
 
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
 /**
  * A values reader for Parquet's run-length encoded data. This is based off of the version in
  * parquet-mr with these changes:
@@ -211,7 +210,8 @@ public void readIntegersWithRebase(
       WritableColumnVector c,
       int rowId,
       int level,
-      VectorizedValuesReader data) throws IOException {
+      VectorizedValuesReader data,
+      final boolean failIfRebase) throws IOException {
     int left = total;
     while (left > 0) {
       if (this.currentCount == 0) this.readNextGroup();
@@ -219,7 +219,7 @@ public void readIntegersWithRebase(
       switch (mode) {
         case RLE:
           if (currentValue == level) {
-            data.readIntegersWithRebase(n, c, rowId);
+            data.readIntegersWithRebase(n, c, rowId, failIfRebase);
           } else {
             c.putNulls(rowId, n);
           }
@@ -227,8 +227,8 @@ public void readIntegersWithRebase(
         case PACKED:
           for (int i = 0; i < n; ++i) {
             if (currentBuffer[currentBufferIdx++] == level) {
-              c.putInt(rowId + i,
-                RebaseDateTime.rebaseJulianToGregorianDays(data.readInteger()));
+              int julianDays = data.readInteger();
+              c.putInt(rowId + i, VectorizedColumnReader.rebaseDays(julianDays, failIfRebase));
             } else {
               c.putNull(rowId + i);
             }
@@ -387,7 +387,8 @@ public void readLongsWithRebase(
       WritableColumnVector c,
       int rowId,
       int level,
-      VectorizedValuesReader data) throws IOException {
+      VectorizedValuesReader data,
+      final boolean failIfRebase) throws IOException {
     int left = total;
     while (left > 0) {
       if (this.currentCount == 0) this.readNextGroup();
@@ -395,7 +396,7 @@ public void readLongsWithRebase(
       switch (mode) {
         case RLE:
           if (currentValue == level) {
-            data.readLongsWithRebase(n, c, rowId);
+            data.readLongsWithRebase(n, c, rowId, failIfRebase);
           } else {
             c.putNulls(rowId, n);
           }
@@ -403,8 +404,8 @@ public void readLongsWithRebase(
         case PACKED:
           for (int i = 0; i < n; ++i) {
             if (currentBuffer[currentBufferIdx++] == level) {
-              c.putLong(rowId + i,
-                RebaseDateTime.rebaseJulianToGregorianMicros(data.readLong()));
+              long julianMicros = data.readLong();
+              c.putLong(rowId + i, VectorizedColumnReader.rebaseMicros(julianMicros, failIfRebase));
             } else {
               c.putNull(rowId + i);
             }
@@ -584,7 +585,8 @@ public void readIntegers(int total, WritableColumnVector c, int rowId) {
   }
 
   @Override
-  public void readIntegersWithRebase(int total, WritableColumnVector c, int rowId) {
+  public void readIntegersWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
     throw new UnsupportedOperationException("only readInts is valid.");
   }
 
@@ -604,7 +606,8 @@ public void readLongs(int total, WritableColumnVector c, int rowId) {
   }
 
   @Override
-  public void readLongsWithRebase(int total, WritableColumnVector c, int rowId) {
+  public void readLongsWithRebase(
+      int total, WritableColumnVector c, int rowId, boolean failIfRebase) {
     throw new UnsupportedOperationException("only readInts is valid.");
   }
 
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
index 809ac44cc8272..35db8f235ed60 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedValuesReader.java
@@ -40,9 +40,9 @@ public interface VectorizedValuesReader {
   void readBooleans(int total, WritableColumnVector c, int rowId);
   void readBytes(int total, WritableColumnVector c, int rowId);
   void readIntegers(int total, WritableColumnVector c, int rowId);
-  void readIntegersWithRebase(int total, WritableColumnVector c, int rowId);
+  void readIntegersWithRebase(int total, WritableColumnVector c, int rowId, boolean failIfRebase);
   void readLongs(int total, WritableColumnVector c, int rowId);
-  void readLongsWithRebase(int total, WritableColumnVector c, int rowId);
+  void readLongsWithRebase(int total, WritableColumnVector c, int rowId, boolean failIfRebase);
   void readFloats(int total, WritableColumnVector c, int rowId);
   void readDoubles(int total, WritableColumnVector c, int rowId);
   void readBinary(int total, WritableColumnVector c, int rowId);
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
index 45a9b1a808cf3..abb74d8d09ec6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala
@@ -23,9 +23,12 @@ import org.apache.hadoop.fs.Path
 import org.json4s.NoTypeHints
 import org.json4s.jackson.Serialization
 
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.util.RebaseDateTime
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
@@ -84,17 +87,107 @@ object DataSourceUtils {
       case _ => false
     }
 
-  def needRebaseDateTime(lookupFileMeta: String => String): Option[Boolean] = {
+  def datetimeRebaseMode(
+      lookupFileMeta: String => String,
+      modeByConfig: String): LegacyBehaviorPolicy.Value = {
     if (Utils.isTesting && SQLConf.get.getConfString("spark.test.forceNoRebase", "") == "true") {
-      return Some(false)
+      return LegacyBehaviorPolicy.CORRECTED
     }
-    // If there is no version, we return None and let the caller side to decide.
+    // If there is no version, we return the mode specified by the config.
     Option(lookupFileMeta(SPARK_VERSION_METADATA_KEY)).map { version =>
       // Files written by Spark 2.4 and earlier follow the legacy hybrid calendar and we need to
       // rebase the datetime values.
       // Files written by Spark 3.0 and latter may also need the rebase if they were written with
-      // the "rebaseInWrite" config enabled.
-      version < "3.0.0" || lookupFileMeta(SPARK_LEGACY_DATETIME) != null
+      // the "LEGACY" rebase mode.
+      if (version < "3.0.0" || lookupFileMeta(SPARK_LEGACY_DATETIME) != null) {
+        LegacyBehaviorPolicy.LEGACY
+      } else {
+        LegacyBehaviorPolicy.CORRECTED
+      }
+    }.getOrElse(LegacyBehaviorPolicy.withName(modeByConfig))
+  }
+
+  def newRebaseExceptionInRead(format: String): SparkUpgradeException = {
+    val config = if (format == "Parquet") {
+      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key
+    } else if (format == "Avro") {
+      SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key
+    } else {
+      throw new IllegalStateException("unrecognized format " + format)
     }
+    new SparkUpgradeException("3.0", "reading dates before 1582-10-15 or timestamps before " +
+      s"1900-01-01T00:00:00Z from $format files can be ambiguous, as the files may be written by " +
+      "Spark 2.x or legacy versions of Hive, which uses a legacy hybrid calendar that is " +
+      "different from Spark 3.0+'s Proleptic Gregorian calendar. See more details in " +
+      s"SPARK-31404. You can set $config to 'LEGACY' to rebase the datetime values w.r.t. " +
+      s"the calendar difference during reading. Or set $config to 'CORRECTED' to read the " +
+      "datetime values as it is.", null)
+  }
+
+  def newRebaseExceptionInWrite(format: String): SparkUpgradeException = {
+    val config = if (format == "Parquet") {
+      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key
+    } else if (format == "Avro") {
+      SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key
+    } else {
+      throw new IllegalStateException("unrecognized format " + format)
+    }
+    new SparkUpgradeException("3.0", "writing dates before 1582-10-15 or timestamps before " +
+      s"1900-01-01T00:00:00Z into $format files can be dangerous, as the files may be read by " +
+      "Spark 2.x or legacy versions of Hive later, which uses a legacy hybrid calendar that is " +
+      "different from Spark 3.0+'s Proleptic Gregorian calendar. See more details in " +
+      s"SPARK-31404. You can set $config to 'LEGACY' to rebase the datetime values w.r.t. " +
+      "the calendar difference during writing, to get maximum interoperability. Or set " +
+      s"$config to 'CORRECTED' to write the datetime values as it is, if you are 100% sure that " +
+      "the written files will only be read by Spark 3.0+ or other systems that use Proleptic " +
+      "Gregorian calendar.", null)
+  }
+
+  def creteDateRebaseFuncInRead(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Int => Int = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
+      if (days < RebaseDateTime.lastSwitchJulianDay) {
+        throw DataSourceUtils.newRebaseExceptionInRead(format)
+      }
+      days
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianDays
+    case LegacyBehaviorPolicy.CORRECTED => identity[Int]
+  }
+
+  def creteDateRebaseFuncInWrite(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Int => Int = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => days: Int =>
+      if (days < RebaseDateTime.lastSwitchGregorianDay) {
+        throw DataSourceUtils.newRebaseExceptionInWrite(format)
+      }
+      days
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianDays
+    case LegacyBehaviorPolicy.CORRECTED => identity[Int]
+  }
+
+  def creteTimestampRebaseFuncInRead(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Long => Long = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
+      if (micros < RebaseDateTime.lastSwitchJulianTs) {
+        throw DataSourceUtils.newRebaseExceptionInRead(format)
+      }
+      micros
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseJulianToGregorianMicros
+    case LegacyBehaviorPolicy.CORRECTED => identity[Long]
+  }
+
+  def creteTimestampRebaseFuncInWrite(
+      rebaseMode: LegacyBehaviorPolicy.Value,
+      format: String): Long => Long = rebaseMode match {
+    case LegacyBehaviorPolicy.EXCEPTION => micros: Long =>
+      if (micros < RebaseDateTime.lastSwitchGregorianTs) {
+        throw DataSourceUtils.newRebaseExceptionInWrite(format)
+      }
+      micros
+    case LegacyBehaviorPolicy.LEGACY => RebaseDateTime.rebaseGregorianToJulianMicros
+    case LegacyBehaviorPolicy.CORRECTED => identity[Long]
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
index 542c996a5342d..fc59336d6107c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileScanRDD.scala
@@ -21,7 +21,7 @@ import java.io.{FileNotFoundException, IOException}
 
 import org.apache.parquet.io.ParquetDecodingException
 
-import org.apache.spark.{Partition => RDDPartition, TaskContext}
+import org.apache.spark.{Partition => RDDPartition, SparkUpgradeException, TaskContext}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.rdd.{InputFileBlockHolder, RDD}
 import org.apache.spark.sql.SparkSession
@@ -178,7 +178,9 @@ class FileScanRDD(
                 s"Expected: ${e.getLogicalType}, Found: ${e.getPhysicalType}"
               throw new QueryExecutionException(message, e)
             case e: ParquetDecodingException =>
-              if (e.getMessage.contains("Can not read value at")) {
+              if (e.getCause.isInstanceOf[SparkUpgradeException]) {
+                throw e.getCause
+              } else if (e.getMessage.contains("Can not read value at")) {
                 val message = "Encounter error while reading parquet files. " +
                   "One possible cause: Parquet column cannot be converted in the " +
                   "corresponding files. Details: "
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index c6d9ddf370e22..71874104fcf4f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -300,10 +300,9 @@ class ParquetFileFormat
           None
         }
 
-      val rebaseDateTime = DataSourceUtils.needRebaseDateTime(
-        footerFileMetaData.getKeyValueMetaData.get).getOrElse {
-        SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ)
-      }
+      val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+        footerFileMetaData.getKeyValueMetaData.get,
+        SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
 
       val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
       val hadoopAttemptContext =
@@ -318,7 +317,7 @@ class ParquetFileFormat
       if (enableVectorizedReader) {
         val vectorizedReader = new VectorizedParquetRecordReader(
           convertTz.orNull,
-          rebaseDateTime,
+          datetimeRebaseMode.toString,
           enableOffHeapColumnVector && taskContext.isDefined,
           capacity)
         val iter = new RecordReaderIterator(vectorizedReader)
@@ -337,7 +336,7 @@ class ParquetFileFormat
         logDebug(s"Falling back to parquet-mr")
         // ParquetRecordReader returns InternalRow
         val readSupport = new ParquetReadSupport(
-          convertTz, enableVectorizedReader = false, rebaseDateTime)
+          convertTz, enableVectorizedReader = false, datetimeRebaseMode)
         val reader = if (pushed.isDefined && enableRecordFilter) {
           val parquetFilter = FilterCompat.get(pushed.get, null)
           new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
index 28165e0bbecde..a30d1c26b3b2d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetReadSupport.scala
@@ -32,6 +32,7 @@ import org.apache.parquet.schema.Type.Repetition
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 /**
@@ -53,7 +54,7 @@ import org.apache.spark.sql.types._
 class ParquetReadSupport(
     val convertTz: Option[ZoneId],
     enableVectorizedReader: Boolean,
-    rebaseDateTime: Boolean)
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value)
   extends ReadSupport[InternalRow] with Logging {
   private var catalystRequestedSchema: StructType = _
 
@@ -61,7 +62,7 @@ class ParquetReadSupport(
     // We need a zero-arg constructor for SpecificParquetRecordReaderBase.  But that is only
     // used in the vectorized reader, where we get the convertTz/rebaseDateTime value directly,
     // and the values here are ignored.
-    this(None, enableVectorizedReader = true, rebaseDateTime = false)
+    this(None, enableVectorizedReader = true, datetimeRebaseMode = LegacyBehaviorPolicy.CORRECTED)
   }
 
   /**
@@ -130,7 +131,7 @@ class ParquetReadSupport(
       ParquetReadSupport.expandUDT(catalystRequestedSchema),
       new ParquetToSparkSchemaConverter(conf),
       convertTz,
-      rebaseDateTime)
+      datetimeRebaseMode)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
index ec037130aa7e9..bb528d548b6ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRecordMaterializer.scala
@@ -23,6 +23,7 @@ import org.apache.parquet.io.api.{GroupConverter, RecordMaterializer}
 import org.apache.parquet.schema.MessageType
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -32,19 +33,19 @@ import org.apache.spark.sql.types.StructType
  * @param catalystSchema Catalyst schema of the rows to be constructed
  * @param schemaConverter A Parquet-Catalyst schema converter that helps initializing row converters
  * @param convertTz the optional time zone to convert to int96 data
- * @param rebaseDateTime true if need to rebase date/timestamp from Julian to Proleptic Gregorian
- *                       calendar
+ * @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
+ *                           calendar
  */
 private[parquet] class ParquetRecordMaterializer(
     parquetSchema: MessageType,
     catalystSchema: StructType,
     schemaConverter: ParquetToSparkSchemaConverter,
     convertTz: Option[ZoneId],
-    rebaseDateTime: Boolean)
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value)
   extends RecordMaterializer[InternalRow] {
 
   private val rootConverter = new ParquetRowConverter(
-    schemaConverter, parquetSchema, catalystSchema, convertTz, rebaseDateTime, NoopUpdater)
+    schemaConverter, parquetSchema, catalystSchema, convertTz, datetimeRebaseMode, NoopUpdater)
 
   override def getCurrentRecord: InternalRow = rootConverter.currentRecord
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
index 08fbca2995c04..9d37f17a24fb7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetRowConverter.scala
@@ -35,8 +35,9 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, CaseInsensitiveMap, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.SQLTimestamp
-import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -121,8 +122,8 @@ private[parquet] class ParquetPrimitiveConverter(val updater: ParentContainerUpd
  * @param catalystType Spark SQL schema that corresponds to the Parquet record type. User-defined
  *        types should have been expanded.
  * @param convertTz the optional time zone to convert to int96 data
- * @param rebaseDateTime true if need to rebase date/timestamp from Julian to Proleptic Gregorian
- *                       calendar
+ * @param datetimeRebaseMode the mode of rebasing date/timestamp from Julian to Proleptic Gregorian
+ *                           calendar
  * @param updater An updater which propagates converted field values to the parent container
  */
 private[parquet] class ParquetRowConverter(
@@ -130,7 +131,7 @@ private[parquet] class ParquetRowConverter(
     parquetType: GroupType,
     catalystType: StructType,
     convertTz: Option[ZoneId],
-    rebaseDateTime: Boolean,
+    datetimeRebaseMode: LegacyBehaviorPolicy.Value,
     updater: ParentContainerUpdater)
   extends ParquetGroupConverter(updater) with Logging {
 
@@ -181,6 +182,12 @@ private[parquet] class ParquetRowConverter(
    */
   def currentRecord: InternalRow = currentRow
 
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInRead(
+    datetimeRebaseMode, "Parquet")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInRead(
+    datetimeRebaseMode, "Parquet")
+
   // Converters for each field.
   private[this] val fieldConverters: Array[Converter with HasParentContainerUpdater] = {
     // (SPARK-31116) Use case insensitive map if spark.sql.caseSensitive is false
@@ -275,35 +282,17 @@ private[parquet] class ParquetRowConverter(
         new ParquetStringConverter(updater)
 
       case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MICROS =>
-        if (rebaseDateTime) {
-          new ParquetPrimitiveConverter(updater) {
-            override def addLong(value: Long): Unit = {
-              val rebased = rebaseJulianToGregorianMicros(value)
-              updater.setLong(rebased)
-            }
-          }
-        } else {
-          new ParquetPrimitiveConverter(updater) {
-            override def addLong(value: Long): Unit = {
-              updater.setLong(value)
-            }
+        new ParquetPrimitiveConverter(updater) {
+          override def addLong(value: Long): Unit = {
+            updater.setLong(timestampRebaseFunc(value))
           }
         }
 
       case TimestampType if parquetType.getOriginalType == OriginalType.TIMESTAMP_MILLIS =>
-        if (rebaseDateTime) {
-          new ParquetPrimitiveConverter(updater) {
-            override def addLong(value: Long): Unit = {
-              val micros = DateTimeUtils.fromMillis(value)
-              val rebased = rebaseJulianToGregorianMicros(micros)
-              updater.setLong(rebased)
-            }
-          }
-        } else {
-          new ParquetPrimitiveConverter(updater) {
-            override def addLong(value: Long): Unit = {
-              updater.setLong(DateTimeUtils.fromMillis(value))
-            }
+        new ParquetPrimitiveConverter(updater) {
+          override def addLong(value: Long): Unit = {
+            val micros = DateTimeUtils.fromMillis(value)
+            updater.setLong(timestampRebaseFunc(micros))
           }
         }
 
@@ -328,17 +317,9 @@ private[parquet] class ParquetRowConverter(
         }
 
       case DateType =>
-        if (rebaseDateTime) {
-          new ParquetPrimitiveConverter(updater) {
-            override def addInt(value: Int): Unit = {
-              updater.set(rebaseJulianToGregorianDays(value))
-            }
-          }
-        } else {
-          new ParquetPrimitiveConverter(updater) {
-            override def addInt(value: Int): Unit = {
-              updater.set(value)
-            }
+        new ParquetPrimitiveConverter(updater) {
+          override def addInt(value: Int): Unit = {
+            updater.set(dateRebaseFunc(value))
           }
         }
 
@@ -386,7 +367,12 @@ private[parquet] class ParquetRowConverter(
           }
         }
         new ParquetRowConverter(
-          schemaConverter, parquetType.asGroupType(), t, convertTz, rebaseDateTime, wrappedUpdater)
+          schemaConverter,
+          parquetType.asGroupType(),
+          t,
+          convertTz,
+          datetimeRebaseMode,
+          wrappedUpdater)
 
       case t =>
         throw new RuntimeException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
index e367b9cc774f3..4e535c0c5ea99 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala
@@ -35,8 +35,9 @@ import org.apache.spark.sql.{SPARK_LEGACY_DATETIME, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.SpecializedGetters
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.catalyst.util.RebaseDateTime._
+import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.types._
 
 /**
@@ -78,9 +79,14 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
   private val decimalBuffer =
     new Array[Byte](Decimal.minBytesForPrecision(DecimalType.MAX_PRECISION))
 
-  // Whether to rebase datetimes from Gregorian to Julian calendar in write
-  private val rebaseDateTime: Boolean =
-    SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE)
+  private val datetimeRebaseMode = LegacyBehaviorPolicy.withName(
+    SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE))
+
+  private val dateRebaseFunc = DataSourceUtils.creteDateRebaseFuncInWrite(
+    datetimeRebaseMode, "Parquet")
+
+  private val timestampRebaseFunc = DataSourceUtils.creteTimestampRebaseFuncInWrite(
+    datetimeRebaseMode, "Parquet")
 
   override def init(configuration: Configuration): WriteContext = {
     val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA)
@@ -103,7 +109,13 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
     val metadata = Map(
       SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT,
       ParquetReadSupport.SPARK_METADATA_KEY -> schemaString
-    ) ++ (if (rebaseDateTime) Some(SPARK_LEGACY_DATETIME -> "") else None)
+    ) ++ {
+      if (datetimeRebaseMode == LegacyBehaviorPolicy.LEGACY) {
+        Some(SPARK_LEGACY_DATETIME -> "")
+      } else {
+        None
+      }
+    }
 
     logInfo(
       s"""Initialized Parquet WriteSupport with Catalyst schema:
@@ -152,12 +164,11 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
         (row: SpecializedGetters, ordinal: Int) =>
           recordConsumer.addInteger(row.getShort(ordinal))
 
-      case DateType if rebaseDateTime =>
+      case DateType =>
         (row: SpecializedGetters, ordinal: Int) =>
-          val rebasedDays = rebaseGregorianToJulianDays(row.getInt(ordinal))
-          recordConsumer.addInteger(rebasedDays)
+          recordConsumer.addInteger(dateRebaseFunc(row.getInt(ordinal)))
 
-      case IntegerType | DateType =>
+      case IntegerType =>
         (row: SpecializedGetters, ordinal: Int) =>
           recordConsumer.addInteger(row.getInt(ordinal))
 
@@ -187,24 +198,15 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging {
               buf.order(ByteOrder.LITTLE_ENDIAN).putLong(timeOfDayNanos).putInt(julianDay)
               recordConsumer.addBinary(Binary.fromReusedByteArray(timestampBuffer))
 
-          case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS if rebaseDateTime =>
-            (row: SpecializedGetters, ordinal: Int) =>
-              val rebasedMicros = rebaseGregorianToJulianMicros(row.getLong(ordinal))
-              recordConsumer.addLong(rebasedMicros)
-
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS =>
             (row: SpecializedGetters, ordinal: Int) =>
-              recordConsumer.addLong(row.getLong(ordinal))
-
-          case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS if rebaseDateTime =>
-            (row: SpecializedGetters, ordinal: Int) =>
-              val rebasedMicros = rebaseGregorianToJulianMicros(row.getLong(ordinal))
-              val millis = DateTimeUtils.toMillis(rebasedMicros)
-              recordConsumer.addLong(millis)
+              val micros = row.getLong(ordinal)
+              recordConsumer.addLong(timestampRebaseFunc(micros))
 
           case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS =>
             (row: SpecializedGetters, ordinal: Int) =>
-              val millis = DateTimeUtils.toMillis(row.getLong(ordinal))
+              val micros = row.getLong(ordinal)
+              val millis = DateTimeUtils.toMillis(timestampRebaseFunc(micros))
               recordConsumer.addLong(millis)
         }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
index 1925fa1796d48..3b482b0c8ab62 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetPartitionReaderFactory.scala
@@ -37,6 +37,7 @@ import org.apache.spark.sql.execution.datasources.{DataSourceUtils, PartitionedF
 import org.apache.spark.sql.execution.datasources.parquet._
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.{AtomicType, StructType}
 import org.apache.spark.sql.vectorized.ColumnarBatch
@@ -116,8 +117,9 @@ case class ParquetPartitionReaderFactory(
   private def buildReaderBase[T](
       file: PartitionedFile,
       buildReaderFunc: (
-        ParquetInputSplit, InternalRow, TaskAttemptContextImpl, Option[FilterPredicate],
-          Option[ZoneId], Boolean) => RecordReader[Void, T]): RecordReader[Void, T] = {
+        ParquetInputSplit, InternalRow, TaskAttemptContextImpl,
+          Option[FilterPredicate], Option[ZoneId],
+          LegacyBehaviorPolicy.Value) => RecordReader[Void, T]): RecordReader[Void, T] = {
     val conf = broadcastedConf.value.value
 
     val filePath = new Path(new URI(file.filePath))
@@ -169,12 +171,11 @@ case class ParquetPartitionReaderFactory(
     if (pushed.isDefined) {
       ParquetInputFormat.setFilterPredicate(hadoopAttemptContext.getConfiguration, pushed.get)
     }
-    val rebaseDatetime = DataSourceUtils.needRebaseDateTime(
-      footerFileMetaData.getKeyValueMetaData.get).getOrElse {
-      SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ)
-    }
+    val datetimeRebaseMode = DataSourceUtils.datetimeRebaseMode(
+      footerFileMetaData.getKeyValueMetaData.get,
+      SQLConf.get.getConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ))
     val reader = buildReaderFunc(
-      split, file.partitionValues, hadoopAttemptContext, pushed, convertTz, rebaseDatetime)
+      split, file.partitionValues, hadoopAttemptContext, pushed, convertTz, datetimeRebaseMode)
     reader.initialize(split, hadoopAttemptContext)
     reader
   }
@@ -189,12 +190,12 @@ case class ParquetPartitionReaderFactory(
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
       convertTz: Option[ZoneId],
-      needDateTimeRebase: Boolean): RecordReader[Void, InternalRow] = {
+      datetimeRebaseMode: LegacyBehaviorPolicy.Value): RecordReader[Void, InternalRow] = {
     logDebug(s"Falling back to parquet-mr")
     val taskContext = Option(TaskContext.get())
     // ParquetRecordReader returns InternalRow
     val readSupport = new ParquetReadSupport(
-      convertTz, enableVectorizedReader = false, needDateTimeRebase)
+      convertTz, enableVectorizedReader = false, datetimeRebaseMode)
     val reader = if (pushed.isDefined && enableRecordFilter) {
       val parquetFilter = FilterCompat.get(pushed.get, null)
       new ParquetRecordReader[InternalRow](readSupport, parquetFilter)
@@ -220,11 +221,11 @@ case class ParquetPartitionReaderFactory(
       hadoopAttemptContext: TaskAttemptContextImpl,
       pushed: Option[FilterPredicate],
       convertTz: Option[ZoneId],
-      rebaseDatetime: Boolean): VectorizedParquetRecordReader = {
+      datetimeRebaseMode: LegacyBehaviorPolicy.Value): VectorizedParquetRecordReader = {
     val taskContext = Option(TaskContext.get())
     val vectorizedReader = new VectorizedParquetRecordReader(
       convertTz.orNull,
-      rebaseDatetime,
+      datetimeRebaseMode.toString,
       enableOffHeapColumnVector && taskContext.isDefined,
       capacity)
     val iter = new RecordReaderIterator(vectorizedReader)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index aa47d36fe8c87..d6167f98b5a51 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
 
 object DateTime extends Enumeration {
@@ -161,9 +162,10 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
               Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { rebase =>
                   benchmark.addCase(caseName(modernDates, dateTime, Some(rebase)), 1) { _ =>
+                    val mode = if (rebase) LEGACY else CORRECTED
                     withSQLConf(
                       SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime),
-                      SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> rebase.toString) {
+                      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) {
                       genDF(rowsNum, dateTime, modernDates)
                         .write
                         .mode("overwrite")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index cf2c7c8b2f516..87b4db3fe087a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -41,7 +41,7 @@ import org.apache.parquet.hadoop.util.HadoopInputFile
 import org.apache.parquet.io.api.RecordConsumer
 import org.apache.parquet.schema.{MessageType, MessageTypeParser}
 
-import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
+import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, SparkUpgradeException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeRow}
@@ -49,6 +49,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -892,41 +893,67 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         val path3_0_rebase = paths(1).getCanonicalPath
         if (dt == "date") {
           val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
-          df.write.parquet(path3_0)
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+
+          // By default we should fail to write ancient datetime values.
+          var e = intercept[SparkException](df.write.parquet(path3_0))
+          assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+          // By default we should fail to read ancient datetime values.
+          e = intercept[SparkException](spark.read.parquet(path2_4).collect())
+          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+            df.write.mode("overwrite").parquet(path3_0)
+          }
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
             df.write.parquet(path3_0_rebase)
           }
-          checkAnswer(
-            spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
-            1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+
+          // For Parquet files written by Spark 3.0, we know the writer info and don't need the
+          // config to guide the rebase behavior.
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+            checkAnswer(
+              spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
+              1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
+          }
         } else {
           val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
           withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> dt) {
-            df.write.parquet(path3_0)
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+            // By default we should fail to write ancient datetime values.
+            var e = intercept[SparkException](df.write.parquet(path3_0))
+            assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
+            // By default we should fail to read ancient datetime values.
+            e = intercept[SparkException](spark.read.parquet(path2_4).collect())
+            assert(e.getCause.isInstanceOf[SparkUpgradeException])
+
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
+              df.write.mode("overwrite").parquet(path3_0)
+            }
+            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
               df.write.parquet(path3_0_rebase)
             }
           }
-          checkAnswer(
-            spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
-            1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+          // For Parquet files written by Spark 3.0, we know the writer info and don't need the
+          // config to guide the rebase behavior.
+          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+            checkAnswer(
+              spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
+              1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
+          }
         }
       }
     }
 
     Seq(false, true).foreach { vectorized =>
       withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> "true") {
-          checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
-          checkReadMixedFiles(
-            "before_1582_timestamp_micros_v2_4.snappy.parquet",
-            "TIMESTAMP_MICROS",
-            "1001-01-01 01:02:03.123456")
-          checkReadMixedFiles(
-            "before_1582_timestamp_millis_v2_4.snappy.parquet",
-            "TIMESTAMP_MILLIS",
-            "1001-01-01 01:02:03.123")
-        }
+        checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
+        checkReadMixedFiles(
+          "before_1582_timestamp_micros_v2_4.snappy.parquet",
+          "TIMESTAMP_MICROS",
+          "1001-01-01 01:02:03.123456")
+        checkReadMixedFiles(
+          "before_1582_timestamp_millis_v2_4.snappy.parquet",
+          "TIMESTAMP_MILLIS",
+          "1001-01-01 01:02:03.123")
 
         // INT96 is a legacy timestamp format and we always rebase the seconds for it.
         checkAnswer(readResourceParquetFile(
@@ -948,7 +975,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> outType) {
             withTempPath { dir =>
               val path = dir.getAbsolutePath
-              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
                 Seq.tabulate(N)(_ => tsStr).toDF("tsS")
                   .select($"tsS".cast("timestamp").as("ts"))
                   .repartition(1)
@@ -960,10 +987,10 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
               Seq(false, true).foreach { vectorized =>
                 withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
                   // The file metadata indicates if it needs rebase or not, so we can always get the
-                  // correct result regardless of the "rebaseInRead" config.
-                  Seq(true, false).foreach { rebase =>
+                  // correct result regardless of the "rebase mode" config.
+                  Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
                     withSQLConf(
-                      SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+                      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> mode.toString) {
                       checkAnswer(
                         spark.read.parquet(path),
                         Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
@@ -991,7 +1018,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
     Seq(false, true).foreach { dictionaryEncoding =>
       withTempPath { dir =>
         val path = dir.getAbsolutePath
-        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_WRITE.key -> "true") {
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
           Seq.tabulate(N)(_ => "1001-01-01").toDF("dateS")
             .select($"dateS".cast("date").as("date"))
             .repartition(1)
@@ -1002,10 +1029,10 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
         Seq(false, true).foreach { vectorized =>
           withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-            // The file metadata indicates if it needs rebase or not, so we can always get
-            // the correct result regardless of the "rebaseInRead" config.
-            Seq(true, false).foreach { rebase =>
-              withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_DATETIME_IN_READ.key -> rebase.toString) {
+            // The file metadata indicates if it needs rebase or not, so we can always get the
+            // correct result regardless of the "rebase mode" config.
+            Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+              withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
                 checkAnswer(
                   spark.read.parquet(path),
                   Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index 42b6862907d8d..cbea74103343e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.execution.DataSourceScanExec
 import org.apache.spark.sql.execution.datasources._
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
 
@@ -151,7 +152,10 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
               Seq(false)
             }
             java8ApiConfValues.foreach { java8Api =>
-              withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+              withSQLConf(
+                SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString,
+                SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString,
+                SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
                 val dataGenerator = RandomDataGenerator.forType(
                   dataType = dataType,
                   nullable = true,

From ebea296d37fdd5b9296e7a0381dc7e85736ac45b Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sun, 17 May 2020 21:31:06 -0700
Subject: [PATCH 0688/1280] [SPARK-31744][R][INFRA] Remove Hive dependency in
 AppVeyor build temporarily

### What changes were proposed in this pull request?

This PR targets to remove Hive profile in SparkR build at AppVeyor in order to:
- Speed up the build. Currently, SparkR build is [reaching the time limit](https://ci.appveyor.com/project/ApacheSoftwareFoundation/spark/builds/32853533).
- Clean up the build profile.

### Why are the changes needed?

We're hitting a time limit issue again and this PR could reduce the build time.

Seems like we're [already skipping Hive related tests in SparkR](https://ci.appveyor.com/project/ApacheSoftwareFoundation/spark/builds/32853533) for some reasons, see below:

```
test_sparkSQL.R:307: skip: create DataFrame from RDD
Reason: Hive is not build with SparkSQL, skipped
test_sparkSQL.R:1341: skip: test HiveContext
Reason: Hive is not build with SparkSQL, skipped
test_sparkSQL.R:2813: skip: read/write ORC files
Reason: Hive is not build with SparkSQL, skipped
test_sparkSQL.R:2834: skip: read/write ORC files - compression option
Reason: Hive is not build with SparkSQL, skipped
test_sparkSQL.R:3727: skip: enableHiveSupport on SparkSession
Reason: Hive is not build with SparkSQL, skipped
```

Although we build with Hive profile. So, the Hive profile is useless here.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

AppVeyor will test it out.

Closes #28564 from HyukjinKwon/SPARK-31744.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit f352cef077456a7ee3fa44ca1e55b1545c9633c5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 appveyor.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index fc0b7d53ddabc..a4da5f9040ded 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -48,7 +48,8 @@ install:
 build_script:
   # '-Djna.nosys=true' is required to avoid kernel32.dll load failure.
   # See SPARK-28759.
-  - cmd: mvn -DskipTests -Psparkr -Phive -Djna.nosys=true package
+  # Ideally we should check the tests related to Hive in SparkR as well (SPARK-31745).
+  - cmd: mvn -DskipTests -Psparkr -Djna.nosys=true package
 
 environment:
   NOT_CRAN: true

From e56f9ab9422e622b72522dac9b5a104e77c4261c Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sun, 17 May 2020 21:33:42 -0700
Subject: [PATCH 0689/1280] [SPARK-31742][TESTS] Increase the eventually time
 limit for Mino kdc in tests to fix flakiness

### What changes were proposed in this pull request?

This PR is kind of a follow up of SPARK-31631. In some cases, it only attempts once for ~35 seconds. Seems 10 seconds are not enough to try multiple times - took a quick look and seems difficult to manipulate the socket configuration as well.

It simply proposes to increase the time limit for now. It affects master and branch-3.0.

```
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedDueToTimeoutException: The code passed to eventually never returned normally. Attempted 1 times over 34.294744142999996 seconds. Last failure message: Address already in use.
	at org.scalatest.concurrent.Eventually.tryTryAgain$1(Eventually.scala:432)
	at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:439)
	at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:391)
	at org.scalatest.concurrent.Eventually$.eventually(Eventually.scala:479)
	at org.scalatest.concurrent.Eventually.eventually(Eventually.scala:308)
	at org.scalatest.concurrent.Eventually.eventually$(Eventually.scala:307)
	at org.scalatest.concurrent.Eventually$.eventually(Eventually.scala:479)
	at org.apache.spark.deploy.security.HadoopDelegationTokenManagerSuite.$anonfun$new$4(HadoopDelegationTokenManagerSuite.scala:106)
	at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
	at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
	at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
	at org.scalatest.Transformer.apply(Transformer.scala:22)
	at org.scalatest.Transformer.apply(Transformer.scala:20)
	at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
	at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:157)
	at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
	at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
	at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
	at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
	at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:59)
	at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
	at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
	at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:59)
	at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
	at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
	at scala.collection.immutable.List.foreach(List.scala:392)
	at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
	at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
	at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
	at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
	at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
	at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
	at org.scalatest.Suite.run(Suite.scala:1124)
	at org.scalatest.Suite.run$(Suite.scala:1106)
	at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
	at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
	at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
	at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
	at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:59)
	at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
	at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:59)
	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
	at sbt.ForkMain$Run$2.call(ForkMain.java:296)
	at sbt.ForkMain$Run$2.call(ForkMain.java:286)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: sbt.ForkMain$ForkError: java.net.BindException: Address already in use
	at sun.nio.ch.Net.bind0(Native Method)
	at sun.nio.ch.Net.bind(Net.java:433)
	at sun.nio.ch.Net.bind(Net.java:425)
	at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
	at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
	at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:198)
	at org.apache.mina.transport.socket.nio.NioSocketAcceptor.open(NioSocketAcceptor.java:51)
	at org.apache.mina.core.polling.AbstractPollingIoAcceptor.registerHandles(AbstractPollingIoAcceptor.java:547)
	at org.apache.mina.core.polling.AbstractPollingIoAcceptor.access$400(AbstractPollingIoAcceptor.java:68)
	at org.apache.mina.core.polling.AbstractPollingIoAcceptor$Acceptor.run(AbstractPollingIoAcceptor.java:422)
	at org.apache.mina.util.NamePreservingRunnable.run(NamePreservingRunnable.java:64)
	... 3 more
```

### Why are the changes needed?

To fix flakiness in the tests.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Jenkins will test it out.

Closes #28562 from HyukjinKwon/SPARK-31742.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c6d13099624d0a7a73bfad29aa1fa70444ba9411)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../deploy/security/HadoopDelegationTokenManagerSuite.scala     | 2 +-
 .../scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
index fc28968922c7a..d9d559509f4fb 100644
--- a/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManagerSuite.scala
@@ -103,7 +103,7 @@ class HadoopDelegationTokenManagerSuite extends SparkFunSuite {
       // The workaround here is to periodically repeat this process with a timeout , since we are
       // using Hadoop 2.7.4 as default.
       // https://issues.apache.org/jira/browse/SPARK-31631
-      eventually(timeout(10.seconds), interval(1.second)) {
+      eventually(timeout(60.seconds), interval(1.second)) {
         try {
           kdc = new MiniKdc(kdcConf, kdcDir)
           kdc.start()
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index 5db8999a69509..a0a113c7a9449 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -148,7 +148,7 @@ class KafkaTestUtils(
     // The workaround here is to periodically repeat this process with a timeout , since we are
     // using Hadoop 2.7.4 as default.
     // https://issues.apache.org/jira/browse/SPARK-31631
-    eventually(timeout(10.seconds), interval(1.second)) {
+    eventually(timeout(60.seconds), interval(1.second)) {
       try {
         kdc = new MiniKdc(kdcConf, kdcDir)
         kdc.start()

From a111597c66fb893ea93b9d10aa2316e4f7fc0ead Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sun, 17 May 2020 21:35:42 -0700
Subject: [PATCH 0690/1280] [SPARK-31743][CORE] Add spark_info metric into
 PrometheusResource

### What changes were proposed in this pull request?

This PR aims to add `spark_info` metric into `PrometheusResource`.

### Why are the changes needed?

This exposes Apache Spark version and revision like the following.

![Screen Shot 2020-05-17 at 6 02 20 PM](https://user-images.githubusercontent.com/9700541/82165091-990ce000-9868-11ea-82d5-8ea344eef646.png)

![Screen Shot 2020-05-17 at 6 06 32 PM](https://user-images.githubusercontent.com/9700541/82165247-2cdeac00-9869-11ea-83aa-e8083fa12a92.png)

### Does this PR introduce _any_ user-facing change?

Yes, but it's exposed as an additional metric.

### How was this patch tested?

Manual.
```
$ bin/spark-shell --driver-memory 4G -c spark.ui.prometheus.enabled=true

$ curl -s http://localhost:4041/metrics/executors/prometheus/ | head -n1
spark_info{version="3.1.0", revision="097d5098cca987e5f7bbb8394783c01517ebed0f"} 1.0
```

Closes #28563 from dongjoon-hyun/SPARK-31743.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 64795f9e0c85a999bf808432d0d533843bea0a31)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/status/api/v1/PrometheusResource.scala     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
index 4ed3d4525a185..9658e5e627724 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala
@@ -23,6 +23,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
 import org.glassfish.jersey.server.ServerProperties
 import org.glassfish.jersey.servlet.ServletContainer
 
+import org.apache.spark.{SPARK_REVISION, SPARK_VERSION_SHORT}
 import org.apache.spark.annotation.Experimental
 import org.apache.spark.ui.SparkUI
 
@@ -42,6 +43,7 @@ private[v1] class PrometheusResource extends ApiRequestContext {
   @Produces(Array(MediaType.TEXT_PLAIN))
   def executors(): String = {
     val sb = new StringBuilder
+    sb.append(s"""spark_info{version="$SPARK_VERSION_SHORT", revision="$SPARK_REVISION"} 1.0\n""")
     val store = uiRoot.asInstanceOf[SparkUI].store
     store.executorList(true).foreach { executor =>
       val prefix = "metrics_executor_"

From 5955933cd037cc2cc0b0b0aaab19457ff050b4fe Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 18 May 2020 05:00:50 +0000
Subject: [PATCH 0691/1280] [SPARK-31727][SQL] Fix error message of casting
 timestamp to int in ANSI non-codegen mode

### What changes were proposed in this pull request?
Change timestamp casting to int in ANSI and non-codegen mode, and make the error message consistent to the error messages in the codegen mode. In particular, casting to int is implemented in the same way as casting to short and byte.

### Why are the changes needed?
1. The error message in the non-codegen mode is diversed from the error message in the codegen mode.
2. The error message contains intermediate results that could confuse.

### Does this PR introduce _any_ user-facing change?
Yes. Before the changes, the error message of casting timestamp to int contains intermediate result but after the changes it contains the input values which causes arithmetic overflow.

### How was this patch tested?
By running the modified test suite `AnsiCastSuite`.

Closes #28549 from MaxGekk/fix-error-msg-cast-timestamp.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit fc5b90243ca1cd460988735dd4170b533d73e8e5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/catalyst/expressions/Cast.scala   |  9 ++++++++-
 .../spark/sql/catalyst/expressions/CastSuite.scala     | 10 ++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index fa615d71a61a0..9bc86d4fb4e42 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -511,7 +511,14 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     case DateType =>
       buildCast[Int](_, d => null)
     case TimestampType if ansiEnabled =>
-      buildCast[Long](_, t => LongExactNumeric.toInt(timestampToLong(t)))
+      buildCast[Long](_, t => {
+        val longValue = timestampToLong(t)
+        if (longValue == longValue.toInt) {
+          longValue.toInt
+        } else {
+          throw new ArithmeticException(s"Casting $t to int causes overflow")
+        }
+      })
     case TimestampType =>
       buildCast[Long](_, t => timestampToLong(t).toInt)
     case x: NumericType if ansiEnabled =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index 334b43e79ffc7..fd0ba67ac28b6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -1355,16 +1355,14 @@ class AnsiCastSuite extends CastSuiteBase {
   }
 
   test("cast a timestamp before the epoch 1970-01-01 00:00:00Z") {
+    def errMsg(t: String): String = s"Casting -2198208303900000 to $t causes overflow"
     withDefaultTimeZone(UTC) {
       val negativeTs = Timestamp.valueOf("1900-05-05 18:34:56.1")
       assert(negativeTs.getTime < 0)
       val expectedSecs = Math.floorDiv(negativeTs.getTime, MILLIS_PER_SECOND)
-      checkExceptionInExpression[ArithmeticException](
-        cast(negativeTs, ByteType), "to byte causes overflow")
-      checkExceptionInExpression[ArithmeticException](
-        cast(negativeTs, ShortType), "to short causes overflow")
-      checkExceptionInExpression[ArithmeticException](
-        cast(negativeTs, IntegerType), "to int causes overflow")
+      checkExceptionInExpression[ArithmeticException](cast(negativeTs, ByteType), errMsg("byte"))
+      checkExceptionInExpression[ArithmeticException](cast(negativeTs, ShortType), errMsg("short"))
+      checkExceptionInExpression[ArithmeticException](cast(negativeTs, IntegerType), errMsg("int"))
       checkEvaluation(cast(negativeTs, LongType), expectedSecs)
     }
   }

From 3855b390c065b847c85a1414370f495b2560a182 Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Sun, 17 May 2020 22:11:21 -0700
Subject: [PATCH 0692/1280] [SPARK-31740][K8S][TESTS] Use github URL instead of
 a broken link

This PR aims to use GitHub URL instead of a broken link in `BasicTestsSuite.scala`.

Currently, K8s integration test is broken:

https://amplab.cs.berkeley.edu/jenkins/view/Spark%20K8s%20Builds/job/spark-master-test-k8s/534/console

```
- Run SparkRemoteFileTest using a remote data file *** FAILED ***
  The code passed to eventually never returned normally. Attempted 130 times over 2.00109555135 minutes. Last failure message: false was not true. (KubernetesSuite.scala:370)
```

No.

Pass the K8s integration test.

Closes #28561 from williamhyun/williamhyun-patch-1.

Authored-by: williamhyun <62487364+williamhyun@users.noreply.github.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 5bb1a09b5f3a0f91409c7245847ab428c3c58322)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/deploy/k8s/integrationtest/BasicTestsSuite.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 6b340f2558cca..76221e46d1cf4 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -111,6 +111,6 @@ private[spark] object BasicTestsSuite {
   val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
      s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt"
   val REMOTE_PAGE_RANK_DATA_FILE =
-    "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt"
+    "https://raw.githubusercontent.com/apache/spark/master/data/mllib/pagerank_data.txt"
   val REMOTE_PAGE_RANK_FILE_NAME = "pagerank_data.txt"
 }

From 88630a3d626f2e14f6a11fbba391ad239472f0eb Mon Sep 17 00:00:00 2001
From: Kris Mok <kris.mok@databricks.com>
Date: Mon, 18 May 2020 05:32:57 +0000
Subject: [PATCH 0693/1280] [SPARK-31399][CORE][TEST-HADOOP3.2][TEST-JAVA11]
 Support indylambda Scala closure in ClosureCleaner

### What changes were proposed in this pull request?

This PR proposes to enhance Spark's `ClosureCleaner` to support "indylambda" style of Scala closures to the same level as the existing implementation for the old (inner class) style ones. The goal is to reach feature parity with the support of the old style Scala closures, with as close to bug-for-bug compatibility as possible.

Specifically, this PR addresses one lacking support for indylambda closures vs the inner class closures:
- When a closure is declared in a Scala REPL and captures the enclosing REPL line object, such closure should be cleanable (unreferenced fields on the enclosing REPL line object should be cleaned)

This PR maintains the same limitations in the new indylambda closure support as the old inner class closures, in particular the following two:
- Cleaning is only available for one level of REPL line object. If a closure captures state from a REPL line object further out from the immediate enclosing one, it won't be subject to cleaning. See example below.
- "Sibling" closures are not handled yet. A "sibling" closure is defined here as a closure that is directly or indirectly referenced by the starting closure, but isn't lexically enclosing. e.g.
  ```scala
  {
    val siblingClosure = (x: Int) => x + this.fieldA   // captures `this`, references `fieldA` on `this`.
    val startingClosure = (y: Int) => y + this.fieldB + siblingClosure(y)  // captures `this` and `siblingClosure`, references `fieldB` on `this`.
  }
  ```

The changes are intended to be minimal, with further code cleanups planned in separate PRs.

Jargons:
- old, inner class style Scala closures, aka `delambdafy:inline`: default in Scala 2.11 and before
- new, "indylambda" style Scala closures, aka `delambdafy:method`: default in Scala 2.12 and later

### Why are the changes needed?

There had been previous effortsto extend Spark's `ClosureCleaner` to support "indylambda" Scala closures, which is necessary for proper Scala 2.12 support. Most notably the work done for [SPARK-14540](https://issues.apache.org/jira/browse/SPARK-14540).

But the previous efforts had missed one import scenario: a Scala closure declared in a Scala REPL, and it captures the enclosing `this` -- a REPL line object. e.g. in a Spark Shell:
```scala
:pa
class NotSerializableClass(val x: Int)
val ns = new NotSerializableClass(42)
val topLevelValue = "someValue"
val func = (j: Int) => {
  (1 to j).flatMap { x =>
    (1 to x).map { y => y + topLevelValue }
  }
}
<Ctrl+D>
sc.parallelize(0 to 2).map(func).collect
```
In this example, `func` refers to a Scala closure that captures the enclosing `this` because it needs to access `topLevelValue`, which is in turn implemented as a field on the enclosing REPL line object.

The existing `ClosureCleaner` in Spark supports cleaning this case in Scala 2.11-, and this PR brings feature parity to Scala 2.12+.

Note that the existing cleaning logic only supported one level of REPL line object nesting. This PR does not go beyond that. When a closure references state declared a few commands earlier, the cleaning will fail in both Scala 2.11 and Scala 2.12. e.g.
```scala
scala> :pa
// Entering paste mode (ctrl-D to finish)

class NotSerializableClass1(val x: Int)
case class Foo(id: String)
val ns = new NotSerializableClass1(42)
val topLevelValue = "someValue"

// Exiting paste mode, now interpreting.

defined class NotSerializableClass1
defined class Foo
ns: NotSerializableClass1 = NotSerializableClass1615b1baf
topLevelValue: String = someValue

scala> :pa
// Entering paste mode (ctrl-D to finish)

val closure2 = (j: Int) => {
  (1 to j).flatMap { x =>
    (1 to x).map { y => y + topLevelValue } // 2 levels
  }
}

// Exiting paste mode, now interpreting.

closure2: Int => scala.collection.immutable.IndexedSeq[String] = <function1>

scala> sc.parallelize(0 to 2).map(closure2).collect
org.apache.spark.SparkException: Task not serializable
...
```
in the Scala 2.11 / Spark 2.4.x case:
```
Caused by: java.io.NotSerializableException: NotSerializableClass1
Serialization stack:
	- object not serializable (class: NotSerializableClass1, value: NotSerializableClass1615b1baf)
	- field (class: $iw, name: ns, type: class NotSerializableClass1)
	- object (class $iw, $iw64df3f4b)
	- field (class: $iw, name: $iw, type: class $iw)
	- object (class $iw, $iw66e6e5e9)
	- field (class: $line14.$read, name: $iw, type: class $iw)
	- object (class $line14.$read, $line14.$readc310aa3)
	- field (class: $iw, name: $line14$read, type: class $line14.$read)
	- object (class $iw, $iw79224636)
	- field (class: $iw, name: $outer, type: class $iw)
	- object (class $iw, $iw636d4cdc)
	- field (class: $anonfun$1, name: $outer, type: class $iw)
	- object (class $anonfun$1, <function1>)
```
in the Scala 2.12 / Spark master case after this PR:
```
Caused by: java.io.NotSerializableException: NotSerializableClass1
Serialization stack:
	- object not serializable (class: NotSerializableClass1, value: NotSerializableClass16f3b4c9a)
	- field (class: $iw, name: ns, type: class NotSerializableClass1)
	- object (class $iw, $iw2945a3c1)
	- field (class: $iw, name: $iw, type: class $iw)
	- object (class $iw, $iw152705d0)
	- field (class: $line14.$read, name: $iw, type: class $iw)
	- object (class $line14.$read, $line14.$read7cf311eb)
	- field (class: $iw, name: $line14$read, type: class $line14.$read)
	- object (class $iw, $iwd980dac)
	- field (class: $iw, name: $outer, type: class $iw)
	- object (class $iw, $iw557d9532)
	- element of array (index: 0)
	- array (class [Ljava.lang.Object;, size 1)
	- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
	- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class $iw, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic $anonfun$closure2$1$adapted:(L$iw;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;, instantiatedMethodType=(Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;, numCaptured=1])
	- writeReplace data (class: java.lang.invoke.SerializedLambda)
	- object (class $Lambda$2103/815179920, $Lambda$2103/815179920569b57c4)
```

For more background of the new and old ways Scala lowers closures to Java bytecode, please see [A note on how NSC (New Scala Compiler) lowers lambdas](https://gist.github.com/rednaxelafx/e9ecd09bbd1c448dbddad4f4edf25d48#file-notes-md).

For more background on how Spark's `ClosureCleaner` works and what's needed to make it support "indylambda" Scala closures, please refer to [A Note on Apache Spark's ClosureCleaner](https://gist.github.com/rednaxelafx/e9ecd09bbd1c448dbddad4f4edf25d48#file-spark_closurecleaner_notes-md).

#### tl;dr

The `ClosureCleaner` works like a mark-sweep algorithm on fields:
- Finding (a chain of) outer objects referenced by the starting closure;
- Scanning the starting closure and its inner closures and marking the fields on the outer objects accessed;
- Cloning the outer objects, nulling out fields that are not accessed by any closure of concern.

##### Outer Objects

For the old, inner class style Scala closures, the "outer objects" is defined as the lexically enclosing closures of the starting closure, plus an optional enclosing REPL line object if these closures are defined in a Scala REPL. All of them are on a singly-linked `$outer` chain.

For the new, "indylambda" style Scala closures, the capturing implementation changed, so closures no longer refer to their enclosing closures via an `$outer` chain. However, a closure can still capture its enclosing REPL line object, much like the old style closures. The name of the field that captures this reference would be `arg$1` (instead of `$outer`).

So what's missing in the `ClosureCleaner` for the "indylambda" support is find and potentially clone+clean the captured enclosing `this` REPL line object. That's what this PR implements.

##### Inner Closures

The old, inner class style of Scala closures are compiled into separate inner classes, one per lambda body. So in order to discover the implementation (bytecode) of the inner closures, one has to jump over multiple classes. The name of such a class would contain the marker substring `$anonfun$`.

The new, "indylambda" style Scala closures are compiled into **static methods** in the class where the lambdas were declared. So for lexically nested closures, their lambda bodies would all be compiled into static methods **in the same class**. This makes it much easier to discover the implementation (bytecode) of the nested lambda bodies. The name of such a static method would contain the marker substring `$anonfun$`.

Discovery of inner closures involves scanning bytecode for certain patterns that represent the creation of a closure object for the inner closure.
- For inner class style: the closure object creation site is like `new <InnerClassForTheClosure>(captured args)`
- For "indylambda" style: the closure object creation site would be compiled into an `invokedynamic` instruction, with its "bootstrap method" pointing to the same one used by Java 8 for its serializable lambdas, and with the bootstrap method arguments pointing to the implementation method.

### Does this PR introduce _any_ user-facing change?

Yes. Before this PR, Spark 2.4 / 3.0 / master on Scala 2.12 would not support Scala closures declared in a Scala REPL that captures anything from the REPL line objects. After this PR, such scenario is supported.

### How was this patch tested?

Added new unit test case to `org.apache.spark.repl.SingletonReplSuite`. The new test case fails without the fix in this PR, and pases with the fix.

Closes #28463 from rednaxelafx/closure-cleaner-indylambda.

Authored-by: Kris Mok <kris.mok@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit dc01b7556f74e4a9873ceb1f78bc7df4e2ab4a8a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/util/ClosureCleaner.scala    | 420 ++++++++++++++++--
 .../spark/repl/SingletonReplSuite.scala       |  61 +++
 2 files changed, 434 insertions(+), 47 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
index d2ad14f2a1a96..6ffd6605f75b8 100644
--- a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
@@ -18,12 +18,15 @@
 package org.apache.spark.util
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
-import java.lang.invoke.SerializedLambda
+import java.lang.invoke.{MethodHandleInfo, SerializedLambda}
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.{Map, Set, Stack}
 
-import org.apache.xbean.asm7.{ClassReader, ClassVisitor, MethodVisitor, Type}
+import org.apache.commons.lang3.ClassUtils
+import org.apache.xbean.asm7.{ClassReader, ClassVisitor, Handle, MethodVisitor, Type}
 import org.apache.xbean.asm7.Opcodes._
+import org.apache.xbean.asm7.tree.{ClassNode, MethodNode}
 
 import org.apache.spark.{SparkEnv, SparkException}
 import org.apache.spark.internal.Logging
@@ -159,39 +162,6 @@ private[spark] object ClosureCleaner extends Logging {
     clean(closure, checkSerializable, cleanTransitively, Map.empty)
   }
 
-  /**
-   * Try to get a serialized Lambda from the closure.
-   *
-   * @param closure the closure to check.
-   */
-  private def getSerializedLambda(closure: AnyRef): Option[SerializedLambda] = {
-    val isClosureCandidate =
-      closure.getClass.isSynthetic &&
-        closure
-          .getClass
-          .getInterfaces.exists(_.getName == "scala.Serializable")
-
-    if (isClosureCandidate) {
-      try {
-        Option(inspect(closure))
-      } catch {
-        case e: Exception =>
-          // no need to check if debug is enabled here the Spark
-          // logging api covers this.
-          logDebug("Closure is not a serialized lambda.", e)
-          None
-      }
-    } else {
-      None
-    }
-  }
-
-  private def inspect(closure: AnyRef): SerializedLambda = {
-    val writeReplace = closure.getClass.getDeclaredMethod("writeReplace")
-    writeReplace.setAccessible(true)
-    writeReplace.invoke(closure).asInstanceOf[java.lang.invoke.SerializedLambda]
-  }
-
   /**
    * Helper method to clean the given closure in place.
    *
@@ -239,12 +209,12 @@ private[spark] object ClosureCleaner extends Logging {
       cleanTransitively: Boolean,
       accessedFields: Map[Class[_], Set[String]]): Unit = {
 
-    // most likely to be the case with 2.12, 2.13
+    // indylambda check. Most likely to be the case with 2.12, 2.13
     // so we check first
     // non LMF-closures should be less frequent from now on
-    val lambdaFunc = getSerializedLambda(func)
+    val maybeIndylambdaProxy = IndylambdaScalaClosures.getSerializationProxy(func)
 
-    if (!isClosure(func.getClass) && lambdaFunc.isEmpty) {
+    if (!isClosure(func.getClass) && maybeIndylambdaProxy.isEmpty) {
       logDebug(s"Expected a closure; got ${func.getClass.getName}")
       return
     }
@@ -256,7 +226,7 @@ private[spark] object ClosureCleaner extends Logging {
       return
     }
 
-    if (lambdaFunc.isEmpty) {
+    if (maybeIndylambdaProxy.isEmpty) {
       logDebug(s"+++ Cleaning closure $func (${func.getClass.getName}) +++")
 
       // A list of classes that represents closures enclosed in the given one
@@ -300,7 +270,7 @@ private[spark] object ClosureCleaner extends Logging {
         }
       }
 
-      logDebug(s" + fields accessed by starting closure: " + accessedFields.size)
+      logDebug(s" + fields accessed by starting closure: ${accessedFields.size} classes")
       accessedFields.foreach { f => logDebug("     " + f) }
 
       // List of outer (class, object) pairs, ordered from outermost to innermost
@@ -372,14 +342,64 @@ private[spark] object ClosureCleaner extends Logging {
 
       logDebug(s" +++ closure $func (${func.getClass.getName}) is now cleaned +++")
     } else {
-      logDebug(s"Cleaning lambda: ${lambdaFunc.get.getImplMethodName}")
+      val lambdaProxy = maybeIndylambdaProxy.get
+      val implMethodName = lambdaProxy.getImplMethodName
+
+      logDebug(s"Cleaning indylambda closure: $implMethodName")
+
+      // capturing class is the class that declared this lambda
+      val capturingClassName = lambdaProxy.getCapturingClass.replace('/', '.')
+      val classLoader = func.getClass.getClassLoader // this is the safest option
+      // scalastyle:off classforname
+      val capturingClass = Class.forName(capturingClassName, false, classLoader)
+      // scalastyle:on classforname
 
-      val captClass = Utils.classForName(lambdaFunc.get.getCapturingClass.replace('/', '.'),
-        initialize = false, noSparkClassLoader = true)
       // Fail fast if we detect return statements in closures
-      getClassReader(captClass)
-        .accept(new ReturnStatementFinder(Some(lambdaFunc.get.getImplMethodName)), 0)
-      logDebug(s" +++ Lambda closure (${lambdaFunc.get.getImplMethodName}) is now cleaned +++")
+      val capturingClassReader = getClassReader(capturingClass)
+      capturingClassReader.accept(new ReturnStatementFinder(Option(implMethodName)), 0)
+
+      val isClosureDeclaredInScalaRepl = capturingClassName.startsWith("$line") &&
+        capturingClassName.endsWith("$iw")
+      val outerThisOpt = if (lambdaProxy.getCapturedArgCount > 0) {
+        Option(lambdaProxy.getCapturedArg(0))
+      } else {
+        None
+      }
+
+      // only need to clean when there is an enclosing "this" captured by the closure, and it
+      // should be something cleanable, i.e. a Scala REPL line object
+      val needsCleaning = isClosureDeclaredInScalaRepl &&
+        outerThisOpt.isDefined && outerThisOpt.get.getClass.getName == capturingClassName
+
+      if (needsCleaning) {
+        // indylambda closures do not reference enclosing closures via an `$outer` chain, so no
+        // transitive cleaning on the `$outer` chain is needed.
+        // Thus clean() shouldn't be recursively called with a non-empty accessedFields.
+        assert(accessedFields.isEmpty)
+
+        initAccessedFields(accessedFields, Seq(capturingClass))
+        IndylambdaScalaClosures.findAccessedFields(
+          lambdaProxy, classLoader, accessedFields, cleanTransitively)
+
+        logDebug(s" + fields accessed by starting closure: ${accessedFields.size} classes")
+        accessedFields.foreach { f => logDebug("     " + f) }
+
+        if (accessedFields(capturingClass).size < capturingClass.getDeclaredFields.length) {
+          // clone and clean the enclosing `this` only when there are fields to null out
+
+          val outerThis = outerThisOpt.get
+
+          logDebug(s" + cloning instance of REPL class $capturingClassName")
+          val clonedOuterThis = cloneAndSetFields(
+            parent = null, outerThis, capturingClass, accessedFields)
+
+          val outerField = func.getClass.getDeclaredField("arg$1")
+          outerField.setAccessible(true)
+          outerField.set(func, clonedOuterThis)
+        }
+      }
+
+      logDebug(s" +++ indylambda closure ($implMethodName) is now cleaned +++")
     }
 
     if (checkSerializable) {
@@ -414,6 +434,312 @@ private[spark] object ClosureCleaner extends Logging {
   }
 }
 
+private[spark] object IndylambdaScalaClosures extends Logging {
+  // internal name of java.lang.invoke.LambdaMetafactory
+  val LambdaMetafactoryClassName = "java/lang/invoke/LambdaMetafactory"
+  // the method that Scala indylambda use for bootstrap method
+  val LambdaMetafactoryMethodName = "altMetafactory"
+  val LambdaMetafactoryMethodDesc = "(Ljava/lang/invoke/MethodHandles$Lookup;" +
+    "Ljava/lang/String;Ljava/lang/invoke/MethodType;[Ljava/lang/Object;)" +
+    "Ljava/lang/invoke/CallSite;"
+
+  /**
+   * Check if the given reference is a indylambda style Scala closure.
+   * If so (e.g. for Scala 2.12+ closures), return a non-empty serialization proxy
+   * (SerializedLambda) of the closure;
+   * otherwise (e.g. for Scala 2.11 closures) return None.
+   *
+   * @param maybeClosure the closure to check.
+   */
+  def getSerializationProxy(maybeClosure: AnyRef): Option[SerializedLambda] = {
+    def isClosureCandidate(cls: Class[_]): Boolean = {
+      // TODO: maybe lift this restriction to support other functional interfaces in the future
+      val implementedInterfaces = ClassUtils.getAllInterfaces(cls).asScala
+      implementedInterfaces.exists(_.getName.startsWith("scala.Function"))
+    }
+
+    maybeClosure.getClass match {
+      // shortcut the fast check:
+      // 1. indylambda closure classes are generated by Java's LambdaMetafactory, and they're
+      //    always synthetic.
+      // 2. We only care about Serializable closures, so let's check that as well
+      case c if !c.isSynthetic || !maybeClosure.isInstanceOf[Serializable] => None
+
+      case c if isClosureCandidate(c) =>
+        try {
+          Option(inspect(maybeClosure)).filter(isIndylambdaScalaClosure)
+        } catch {
+          case e: Exception =>
+            logDebug("The given reference is not an indylambda Scala closure.", e)
+            None
+        }
+
+      case _ => None
+    }
+  }
+
+  def isIndylambdaScalaClosure(lambdaProxy: SerializedLambda): Boolean = {
+    lambdaProxy.getImplMethodKind == MethodHandleInfo.REF_invokeStatic &&
+      lambdaProxy.getImplMethodName.contains("$anonfun$")
+  }
+
+  def inspect(closure: AnyRef): SerializedLambda = {
+    val writeReplace = closure.getClass.getDeclaredMethod("writeReplace")
+    writeReplace.setAccessible(true)
+    writeReplace.invoke(closure).asInstanceOf[SerializedLambda]
+  }
+
+  /**
+   * Check if the handle represents the LambdaMetafactory that indylambda Scala closures
+   * use for creating the lambda class and getting a closure instance.
+   */
+  def isLambdaMetafactory(bsmHandle: Handle): Boolean = {
+    bsmHandle.getOwner == LambdaMetafactoryClassName &&
+      bsmHandle.getName == LambdaMetafactoryMethodName &&
+      bsmHandle.getDesc == LambdaMetafactoryMethodDesc
+  }
+
+  /**
+   * Check if the handle represents a target method that is:
+   * - a STATIC method that implements a Scala lambda body in the indylambda style
+   * - captures the enclosing `this`, i.e. the first argument is a reference to the same type as
+   *   the owning class.
+   * Returns true if both criteria above are met.
+   */
+  def isLambdaBodyCapturingOuter(handle: Handle, ownerInternalName: String): Boolean = {
+    handle.getTag == H_INVOKESTATIC &&
+      handle.getName.contains("$anonfun$") &&
+      handle.getOwner == ownerInternalName &&
+      handle.getDesc.startsWith(s"(L$ownerInternalName;")
+  }
+
+  /**
+   * Check if the callee of a call site is a inner class constructor.
+   * - A constructor has to be invoked via INVOKESPECIAL
+   * - A constructor's internal name is "&lt;init&gt;" and the return type is "V" (void)
+   * - An inner class' first argument in the signature has to be a reference to the
+   *   enclosing "this", aka `$outer` in Scala.
+   */
+  def isInnerClassCtorCapturingOuter(
+      op: Int, owner: String, name: String, desc: String, callerInternalName: String): Boolean = {
+    op == INVOKESPECIAL && name == "<init>" && desc.startsWith(s"(L$callerInternalName;")
+  }
+
+  /**
+   * Scans an indylambda Scala closure, along with its lexically nested closures, and populate
+   * the accessed fields info on which fields on the outer object are accessed.
+   *
+   * This is equivalent to getInnerClosureClasses() + InnerClosureFinder + FieldAccessFinder fused
+   * into one for processing indylambda closures. The traversal order along the call graph is the
+   * same for all three combined, so they can be fused together easily while maintaining the same
+   * ordering as the existing implementation.
+   *
+   * Precondition: this function expects the `accessedFields` to be populated with all known
+   *               outer classes and their super classes to be in the map as keys, e.g.
+   *               initializing via ClosureCleaner.initAccessedFields.
+   */
+  // scalastyle:off line.size.limit
+  // Example: run the following code snippet in a Spark Shell w/ Scala 2.12+:
+  //   val topLevelValue = "someValue"; val closure = (j: Int) => {
+  //     class InnerFoo {
+  //       val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
+  //     }
+  //     val innerFoo = new InnerFoo
+  //     (1 to j).flatMap(innerFoo.innerClosure)
+  //   }
+  //   sc.parallelize(0 to 2).map(closure).collect
+  //
+  // produces the following trace-level logs:
+  // (slightly simplified:
+  //   - omitting the "ignoring ..." lines;
+  //   - "$iw" is actually "$line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw";
+  //   - "invokedynamic" lines are simplified to just show the name+desc, omitting the bsm info)
+  //   Cleaning indylambda closure: $anonfun$closure$1$adapted
+  //     scanning $iw.$anonfun$closure$1$adapted(L$iw;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
+  //       found intra class call to $iw.$anonfun$closure$1(L$iw;I)Lscala/collection/immutable/IndexedSeq;
+  //     scanning $iw.$anonfun$closure$1(L$iw;I)Lscala/collection/immutable/IndexedSeq;
+  //       found inner class $iw$InnerFoo$1
+  //         found method innerClosure()Lscala/Function1;
+  //         found method $anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //         found method $anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //         found method <init>(L$iw;)V
+  //         found method $anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
+  //         found method $anonfun$innerClosure$1$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
+  //         found method $deserializeLambda$(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;
+  //       found call to outer $iw$InnerFoo$1.innerClosure()Lscala/Function1;
+  //     scanning $iw$InnerFoo$1.innerClosure()Lscala/Function1;
+  //     scanning $iw$InnerFoo$1.$deserializeLambda$(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;
+  //       invokedynamic: lambdaDeserialize(Ljava/lang/invoke/SerializedLambda;)Ljava/lang/Object;, bsm...)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq;
+  //       found intra class call to $iw$InnerFoo$1.$anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //       invokedynamic: apply(L$iw$InnerFoo$1;)Lscala/Function1;, bsm...)
+  //       found inner closure $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String; (6)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
+  //       found intra class call to $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //       found call to outer $iw.topLevelValue()Ljava/lang/String;
+  //     scanning $iw.topLevelValue()Ljava/lang/String;
+  //       found field access topLevelValue on $iw
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String;
+  //       found intra class call to $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //     scanning $iw$InnerFoo$1.<init>(L$iw;)V
+  //       invokedynamic: apply(L$iw$InnerFoo$1;)Lscala/Function1;, bsm...)
+  //       found inner closure $iw$InnerFoo$1.$anonfun$innerClosure$1$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Lscala/collection/immutable/IndexedSeq; (6)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$1(L$iw$InnerFoo$1;I)Lscala/collection/immutable/IndexedSeq;
+  //       invokedynamic: apply(L$iw$InnerFoo$1;)Lscala/Function1;, bsm...)
+  //       found inner closure $iw$InnerFoo$1.$anonfun$innerClosure$2$adapted(L$iw$InnerFoo$1;Ljava/lang/Object;)Ljava/lang/String; (6)
+  //     scanning $iw$InnerFoo$1.$anonfun$innerClosure$2(L$iw$InnerFoo$1;I)Ljava/lang/String;
+  //       found call to outer $iw.topLevelValue()Ljava/lang/String;
+  //     scanning $iw$InnerFoo$1.innerClosure()Lscala/Function1;
+  //    + fields accessed by starting closure: 2 classes
+  //        (class java.lang.Object,Set())
+  //        (class $iw,Set(topLevelValue))
+  //    + cloning instance of REPL class $iw
+  //    +++ indylambda closure ($anonfun$closure$1$adapted) is now cleaned +++
+  //
+  // scalastyle:on line.size.limit
+  def findAccessedFields(
+      lambdaProxy: SerializedLambda,
+      lambdaClassLoader: ClassLoader,
+      accessedFields: Map[Class[_], Set[String]],
+      findTransitively: Boolean): Unit = {
+
+    // We may need to visit the same class multiple times for different methods on it, and we'll
+    // need to lookup by name. So we use ASM's Tree API and cache the ClassNode/MethodNode.
+    val classInfoByInternalName = Map.empty[String, (Class[_], ClassNode)]
+    val methodNodeById = Map.empty[MethodIdentifier[_], MethodNode]
+    def getOrUpdateClassInfo(classInternalName: String): (Class[_], ClassNode) = {
+      val classInfo = classInfoByInternalName.getOrElseUpdate(classInternalName, {
+        val classExternalName = classInternalName.replace('/', '.')
+        // scalastyle:off classforname
+        val clazz = Class.forName(classExternalName, false, lambdaClassLoader)
+        // scalastyle:on classforname
+        val classNode = new ClassNode()
+        val classReader = ClosureCleaner.getClassReader(clazz)
+        classReader.accept(classNode, 0)
+
+        for (m <- classNode.methods.asScala) {
+          methodNodeById(MethodIdentifier(clazz, m.name, m.desc)) = m
+        }
+
+        (clazz, classNode)
+      })
+      classInfo
+    }
+
+    val implClassInternalName = lambdaProxy.getImplClass
+    val (implClass, _) = getOrUpdateClassInfo(implClassInternalName)
+
+    val implMethodId = MethodIdentifier(
+      implClass, lambdaProxy.getImplMethodName, lambdaProxy.getImplMethodSignature)
+
+    // The set internal names of classes that we would consider following the calls into.
+    // Candidates are: known outer class which happens to be the starting closure's impl class,
+    // and all inner classes discovered below.
+    // Note that code in an inner class can make calls to methods in any of its enclosing classes,
+    // e.g.
+    //   starting closure (in class T)
+    //     inner class A
+    //        inner class B
+    //          inner closure
+    // we need to track calls from "inner closure" to outer classes relative to it (class T, A, B)
+    // to better find and track field accesses.
+    val trackedClassInternalNames = Set[String](implClassInternalName)
+
+    // Depth-first search for inner closures and track the fields that were accessed in them.
+    // Start from the lambda body's implementation method, follow method invocations
+    val visited = Set.empty[MethodIdentifier[_]]
+    val stack = Stack[MethodIdentifier[_]](implMethodId)
+    def pushIfNotVisited(methodId: MethodIdentifier[_]): Unit = {
+      if (!visited.contains(methodId)) {
+        stack.push(methodId)
+      }
+    }
+
+    while (!stack.isEmpty) {
+      val currentId = stack.pop
+      visited += currentId
+
+      val currentClass = currentId.cls
+      val currentMethodNode = methodNodeById(currentId)
+      logTrace(s"  scanning ${currentId.cls.getName}.${currentId.name}${currentId.desc}")
+      currentMethodNode.accept(new MethodVisitor(ASM7) {
+        val currentClassName = currentClass.getName
+        val currentClassInternalName = currentClassName.replace('.', '/')
+
+        // Find and update the accessedFields info. Only fields on known outer classes are tracked.
+        // This is the FieldAccessFinder equivalent.
+        override def visitFieldInsn(op: Int, owner: String, name: String, desc: String): Unit = {
+          if (op == GETFIELD || op == PUTFIELD) {
+            val ownerExternalName = owner.replace('/', '.')
+            for (cl <- accessedFields.keys if cl.getName == ownerExternalName) {
+              logTrace(s"    found field access $name on $ownerExternalName")
+              accessedFields(cl) += name
+            }
+          }
+        }
+
+        override def visitMethodInsn(
+            op: Int, owner: String, name: String, desc: String, itf: Boolean): Unit = {
+          val ownerExternalName = owner.replace('/', '.')
+          if (owner == currentClassInternalName) {
+            logTrace(s"    found intra class call to $ownerExternalName.$name$desc")
+            // could be invoking a helper method or a field accessor method, just follow it.
+            pushIfNotVisited(MethodIdentifier(currentClass, name, desc))
+          } else if (isInnerClassCtorCapturingOuter(
+              op, owner, name, desc, currentClassInternalName)) {
+            // Discover inner classes.
+            // This this the InnerClassFinder equivalent for inner classes, which still use the
+            // `$outer` chain. So this is NOT controlled by the `findTransitively` flag.
+            logDebug(s"    found inner class $ownerExternalName")
+            val innerClassInfo = getOrUpdateClassInfo(owner)
+            val innerClass = innerClassInfo._1
+            val innerClassNode = innerClassInfo._2
+            trackedClassInternalNames += owner
+            // We need to visit all methods on the inner class so that we don't missing anything.
+            for (m <- innerClassNode.methods.asScala) {
+              logTrace(s"      found method ${m.name}${m.desc}")
+              pushIfNotVisited(MethodIdentifier(innerClass, m.name, m.desc))
+            }
+          } else if (findTransitively && trackedClassInternalNames.contains(owner)) {
+            logTrace(s"    found call to outer $ownerExternalName.$name$desc")
+            val (calleeClass, _) = getOrUpdateClassInfo(owner) // make sure MethodNodes are cached
+            pushIfNotVisited(MethodIdentifier(calleeClass, name, desc))
+          } else {
+            // keep the same behavior as the original ClosureCleaner
+            logTrace(s"    ignoring call to $ownerExternalName.$name$desc")
+          }
+        }
+
+        // Find the lexically nested closures
+        // This is the InnerClosureFinder equivalent for indylambda nested closures
+        override def visitInvokeDynamicInsn(
+            name: String, desc: String, bsmHandle: Handle, bsmArgs: Object*): Unit = {
+          logTrace(s"    invokedynamic: $name$desc, bsmHandle=$bsmHandle, bsmArgs=$bsmArgs")
+
+          // fast check: we only care about Scala lambda creation
+          // TODO: maybe lift this restriction and support other functional interfaces
+          if (!name.startsWith("apply")) return
+          if (!Type.getReturnType(desc).getDescriptor.startsWith("Lscala/Function")) return
+
+          if (isLambdaMetafactory(bsmHandle)) {
+            // OK we're in the right bootstrap method for serializable Java 8 style lambda creation
+            val targetHandle = bsmArgs(1).asInstanceOf[Handle]
+            if (isLambdaBodyCapturingOuter(targetHandle, currentClassInternalName)) {
+              // this is a lexically nested closure that also captures the enclosing `this`
+              logDebug(s"    found inner closure $targetHandle")
+              val calleeMethodId =
+                MethodIdentifier(currentClass, targetHandle.getName, targetHandle.getDesc)
+              pushIfNotVisited(calleeMethodId)
+            }
+          }
+        }
+      })
+    }
+  }
+}
+
 private[spark] class ReturnStatementInClosureException
   extends SparkException("Return statements aren't allowed in Spark closures")
 
@@ -422,7 +748,7 @@ private class ReturnStatementFinder(targetMethodName: Option[String] = None)
   override def visitMethod(access: Int, name: String, desc: String,
       sig: String, exceptions: Array[String]): MethodVisitor = {
 
-    // $anonfun$ covers Java 8 lambdas
+    // $anonfun$ covers indylambda closures
     if (name.contains("apply") || name.contains("$anonfun$")) {
       // A method with suffix "$adapted" will be generated in cases like
       // { _:Int => return; Seq()} but not { _:Int => return; true}
diff --git a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
index 4795306692f7a..e11a54bc88070 100644
--- a/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/SingletonReplSuite.scala
@@ -380,6 +380,67 @@ class SingletonReplSuite extends SparkFunSuite {
     assertDoesNotContain("Exception", output)
   }
 
+  test("SPARK-31399: should clone+clean line object w/ non-serializable state in ClosureCleaner") {
+    // Test ClosureCleaner when a closure captures the enclosing `this` REPL line object, and that
+    // object contains an unused non-serializable field.
+    // Specifically, the closure in this test case contains a directly nested closure, and the
+    // capture is triggered by the inner closure.
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    // Can't use :paste mode because PipedOutputStream/PipedInputStream doesn't work well with the
+    // EOT control character (i.e. Ctrl+D).
+    // Just write things on a single line to emulate :paste mode.
+
+    // NOTE: in order for this test case to trigger the intended scenario, the following three
+    //       variables need to be in the same "input", which will make the REPL pack them into the
+    //       same REPL line object:
+    //         - ns: a non-serializable state, not accessed by the closure;
+    //         - topLevelValue: a serializable state, accessed by the closure;
+    //         - closure: the starting closure, captures the enclosing REPL line object.
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  (1 to j).flatMap { x =>
+        |    (1 to x).map { y => y + topLevelValue }
+        |  }
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+      "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Exception", output)
+  }
+
+  test("SPARK-31399: ClosureCleaner should discover indirectly nested closure in inner class") {
+    // Similar to the previous test case, but with indirect closure nesting instead.
+    // There's still nested closures involved, but the inner closure is indirectly nested in the
+    // outer closure, with a level of inner class in between them.
+    // This changes how the inner closure references/captures the outer closure/enclosing `this`
+    // REPL line object, and covers a different code path in inner closure discovery.
+
+    // `ns` should be nulled out, but `topLevelValue` should stay intact.
+
+    val output = runInterpreter(
+      """
+        |class NotSerializableClass(val x: Int)
+        |val ns = new NotSerializableClass(42); val topLevelValue = "someValue"; val closure =
+        |(j: Int) => {
+        |  class InnerFoo {
+        |    val innerClosure = (x: Int) => (1 to x).map { y => y + topLevelValue }
+        |  }
+        |  val innerFoo = new InnerFoo
+        |  (1 to j).flatMap(innerFoo.innerClosure)
+        |}
+        |val r = sc.parallelize(0 to 2).map(closure).collect
+      """.stripMargin)
+    assertContains("r: Array[scala.collection.immutable.IndexedSeq[String]] = " +
+       "Array(Vector(), Vector(1someValue), Vector(1someValue, 1someValue, 2someValue))", output)
+    assertDoesNotContain("Array(Vector(), Vector(1null), Vector(1null, 1null, 2null)", output)
+    assertDoesNotContain("Exception", output)
+  }
+
   test("newProductSeqEncoder with REPL defined class") {
     val output = runInterpreter(
       """

From cbd8568ad7588cf14e5519c5aabf88d0b7fb0e33 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 18 May 2020 14:35:02 +0900
Subject: [PATCH 0694/1280] [SPARK-31746][YARN][TESTS] Show the actual error
 message in LocalityPlacementStrategySuite

### What changes were proposed in this pull request?

This PR proposes to show the actual traceback when "handle large number of containers and tasks (SPARK-18750)" test fails in `LocalityPlacementStrategySuite`.

**It does not fully resolve the JIRA SPARK-31746 yet**. I tried to reproduce in my local by controlling the factors in the tests but I couldn't. I double checked the changes in SPARK-18750 are still valid.

### Why are the changes needed?

This test is flaky for an unknown reason (see https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/122768/testReport/org.apache.spark.deploy.yarn/LocalityPlacementStrategySuite/handle_large_number_of_containers_and_tasks__SPARK_18750_/):

```
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: java.lang.StackOverflowError did not equal null
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
	at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
```

After this PR, it will help to investigate the root cause:

**Before**:

```
[info] - handle large number of containers and tasks (SPARK-18750) *** FAILED *** (824 milliseconds)
[info]   java.lang.StackOverflowError did not equal null (LocalityPlacementStrategySuite.scala:49)
[info]   org.scalatest.exceptions.TestFailedException:
[info]   at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
[info]   at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
[info]   at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
[info]   at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
[info]   at org.apache.spark.deploy.yarn.LocalityPlacementStrategySuite.$anonfun$new$1(LocalityPlacementStrategySuite.scala:49)
[info]   at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
[info]   at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
[info]   at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
[info]   at org.scalatest.Transformer.apply(Transformer.scala:22)
[info]   at org.scalatest.Transformer.apply(Transformer.scala:20)
[info]   at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
[info]   at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:157)
[info]   at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
[info]   at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
[info]   at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
[info]   at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
[info]   at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
...
```

**After**:

```
[info] - handle large number of containers and tasks (SPARK-18750) *** FAILED *** (825 milliseconds)
[info]   StackOverflowError should not be thrown; however, got:
[info]
[info]    java.lang.StackOverflowError
[info]   	at scala.collection.TraversableLike.$anonfun$filterImpl$1(TraversableLike.scala:256)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
...

```

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Manually tested by reverting https://github.com/apache/spark/commit/76db394f2baedc2c7b7a52c05314a64ec9068263 locally.

Closes #28566 from HyukjinKwon/SPARK-31746.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3bf7bf99e96fab754679a4f3c893995263161341)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../deploy/yarn/LocalityPlacementStrategySuite.scala   | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
index b7f25656e49ac..3c77872f27861 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.deploy.yarn
 
-import scala.collection.JavaConverters._
+import java.io.{PrintWriter, StringWriter}
+
 import scala.collection.mutable.{HashMap, HashSet, Set}
 
 import org.apache.hadoop.yarn.api.records._
@@ -45,7 +46,11 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
     thread.start()
     thread.join()
 
-    assert(error === null)
+    if (error != null) {
+      val errors = new StringWriter()
+      error.printStackTrace(new PrintWriter(errors))
+      fail(s"StackOverflowError should not be thrown; however, got:\n\n$errors")
+    }
   }
 
   private def runTest(): Unit = {
@@ -56,7 +61,6 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
     // goal is to create enough requests for localized containers (so there should be many
     // tasks on several hosts that have no allocated containers).
 
-    val resource = Resource.newInstance(8 * 1024, 4)
     val strategy = new LocalityPreferredContainerPlacementStrategy(new SparkConf(),
       yarnConf, resource, new MockResolver())
 

From 88e00c367bccf51a343abae2ef734fa028c46f7b Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 18 May 2020 14:55:31 +0900
Subject: [PATCH 0695/1280] Revert "[SPARK-31746][YARN][TESTS] Show the actual
 error message in LocalityPlacementStrategySuite"

This reverts commit cbd8568ad7588cf14e5519c5aabf88d0b7fb0e33.
---
 .../deploy/yarn/LocalityPlacementStrategySuite.scala   | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
index 3c77872f27861..b7f25656e49ac 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -17,8 +17,7 @@
 
 package org.apache.spark.deploy.yarn
 
-import java.io.{PrintWriter, StringWriter}
-
+import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, HashSet, Set}
 
 import org.apache.hadoop.yarn.api.records._
@@ -46,11 +45,7 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
     thread.start()
     thread.join()
 
-    if (error != null) {
-      val errors = new StringWriter()
-      error.printStackTrace(new PrintWriter(errors))
-      fail(s"StackOverflowError should not be thrown; however, got:\n\n$errors")
-    }
+    assert(error === null)
   }
 
   private def runTest(): Unit = {
@@ -61,6 +56,7 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
     // goal is to create enough requests for localized containers (so there should be many
     // tasks on several hosts that have no allocated containers).
 
+    val resource = Resource.newInstance(8 * 1024, 4)
     val strategy = new LocalityPreferredContainerPlacementStrategy(new SparkConf(),
       yarnConf, resource, new MockResolver())
 

From 2cdf4eb7c20393e47709d02454356981df05040a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 18 May 2020 14:35:02 +0900
Subject: [PATCH 0696/1280] [SPARK-31746][YARN][TESTS] Show the actual error
 message in LocalityPlacementStrategySuite

This PR proposes to show the actual traceback when "handle large number of containers and tasks (SPARK-18750)" test fails in `LocalityPlacementStrategySuite`.

**It does not fully resolve the JIRA SPARK-31746 yet**. I tried to reproduce in my local by controlling the factors in the tests but I couldn't. I double checked the changes in SPARK-18750 are still valid.

This test is flaky for an unknown reason (see https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/122768/testReport/org.apache.spark.deploy.yarn/LocalityPlacementStrategySuite/handle_large_number_of_containers_and_tasks__SPARK_18750_/):

```
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: java.lang.StackOverflowError did not equal null
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
	at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
```

After this PR, it will help to investigate the root cause:

**Before**:

```
[info] - handle large number of containers and tasks (SPARK-18750) *** FAILED *** (824 milliseconds)
[info]   java.lang.StackOverflowError did not equal null (LocalityPlacementStrategySuite.scala:49)
[info]   org.scalatest.exceptions.TestFailedException:
[info]   at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
[info]   at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
[info]   at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
[info]   at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:503)
[info]   at org.apache.spark.deploy.yarn.LocalityPlacementStrategySuite.$anonfun$new$1(LocalityPlacementStrategySuite.scala:49)
[info]   at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
[info]   at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
[info]   at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
[info]   at org.scalatest.Transformer.apply(Transformer.scala:22)
[info]   at org.scalatest.Transformer.apply(Transformer.scala:20)
[info]   at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
[info]   at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:157)
[info]   at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
[info]   at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
[info]   at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
[info]   at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
[info]   at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
...
```

**After**:

```
[info] - handle large number of containers and tasks (SPARK-18750) *** FAILED *** (825 milliseconds)
[info]   StackOverflowError should not be thrown; however, got:
[info]
[info]    java.lang.StackOverflowError
[info]   	at scala.collection.TraversableLike.$anonfun$filterImpl$1(TraversableLike.scala:256)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
[info]   	at scala.collection.TraversableLike$WithFilter.$anonfun$foreach$1(TraversableLike.scala:877)
[info]   	at scala.collection.MapLike$MappedValues.$anonfun$foreach$3(MapLike.scala:256)
...

```

No, dev-only.

Manually tested by reverting https://github.com/apache/spark/commit/76db394f2baedc2c7b7a52c05314a64ec9068263 locally.

Closes #28566 from HyukjinKwon/SPARK-31746.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3bf7bf99e96fab754679a4f3c893995263161341)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../deploy/yarn/LocalityPlacementStrategySuite.scala     | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
index b7f25656e49ac..bb14e09ad09c2 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.deploy.yarn
 
-import scala.collection.JavaConverters._
+import java.io.{PrintWriter, StringWriter}
+
 import scala.collection.mutable.{HashMap, HashSet, Set}
 
 import org.apache.hadoop.yarn.api.records._
@@ -45,7 +46,11 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
     thread.start()
     thread.join()
 
-    assert(error === null)
+    if (error != null) {
+      val errors = new StringWriter()
+      error.printStackTrace(new PrintWriter(errors))
+      fail(s"StackOverflowError should not be thrown; however, got:\n\n$errors")
+    }
   }
 
   private def runTest(): Unit = {

From 740da34892030410b2db05725284820bab719e22 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 18 May 2020 12:07:01 +0000
Subject: [PATCH 0697/1280] [SPARK-31738][SQL][DOCS] Describe 'L' and 'M' month
 pattern letters

### What changes were proposed in this pull request?
1. Describe standard 'M' and stand-alone 'L' text forms
2. Add examples for all supported number of month letters

<img width="1047" alt="Screenshot 2020-05-18 at 08 57 31" src="https://user-images.githubusercontent.com/1580697/82178856-b16f1000-98e5-11ea-87c0-456ef94dcd43.png">

### Why are the changes needed?
To improve docs and show how to use month patterns.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By building docs and checking by eyes.

Closes #28558 from MaxGekk/describe-L-M-date-pattern.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b3686a762281ce9bf595bf790f8e4198d3d186b4)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md | 51 ++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index b65bb1319fb9d..df19b9ce4c082 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -76,6 +76,57 @@ The count of pattern letters determines the format.
 
 - Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
 
+- Month: If the number of pattern letters is 3 or more, the month is interpreted as text; otherwise, it is interpreted as a number. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. The difference between the 'standard' and 'stand-alone' forms is trickier to describe as there is no difference in English. However, in other languages there is a difference in the word used when the text is used alone, as opposed to in a complete date. For example, the word used for a month when used alone in a date picker is different to the word used for month in association with a day and year in a date. In Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters (more than 5 letters is invalid):
+  - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "M");
+    1
+    spark-sql> select date_format(date '1970-12-01', "L");
+    12
+    ```
+  - `'MM'` or `'LL'`: Month number in a year starting from 1. Zero padding is added for month 1-9.
+      ```sql
+      spark-sql> select date_format(date '1970-1-01', "LL");
+      01
+      spark-sql> select date_format(date '1970-09-01', "MM");
+      09
+      ```
+  - `'MMM'`: Short textual representation in the standard form. The month pattern should be a part of a date pattern not just a stand-alone month except locales where there is no difference between stand and stand-alone forms like in English.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "d MMM");
+    1 Jan
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'dd MMM', 'locale', 'RU'));
+    01 янв.
+    ```
+  - `'LLL'`: Short textual representation in the stand-alone form. It should be used to format/parse only months without any other date fields.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "LLL");
+    Jan
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'LLL', 'locale', 'RU'));
+    янв.
+    ```
+  - `'MMMM'`: full textual month representation in the standard form. It is used for parsing/formatting months as a part of dates/timestamps.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "MMMM yyyy");
+    January 1970
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'd MMMM', 'locale', 'RU'));
+    1 января
+    ```
+  - `'LLLL'`: full textual month representation in the stand-alone form. The pattern can be used to format/parse only months.
+    ```sql
+    spark-sql> select date_format(date '1970-01-01', "LLLL");
+    January
+    spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'LLLL', 'locale', 'RU'));
+    январь
+    ```
+  - `'LLLLL'` or `'MMMMM'`: Narrow textual representation of standard or stand-alone forms. Typically it is a single letter.
+    ```sql
+    spark-sql> select date_format(date '1970-07-01', "LLLLL");
+    J
+    spark-sql> select date_format(date '1970-01-01', "MMMMM");
+    J
+    ```
+
 - Zone ID(V): This outputs the display the time-zone ID. Pattern letter count must be 2.
 
 - Zone names(z): This outputs the display textual name of the time-zone ID. If the count of letters is one, two or three, then the short name is output. If the count of letters is four, then the full name is output. Five or more letters will fail.

From 29853eca69bceefd227cbe8421a09c116b7b753a Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@apache.org>
Date: Mon, 18 May 2020 13:21:37 +0000
Subject: [PATCH 0698/1280] Preparing Spark release v3.0.0-rc2

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 21f3eaa5c15f6..3bad429ca4086 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.1
+Version: 3.0.0
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 8bef9d8bc56cb..0a52a00c4a694 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index fc1441d80b7bf..fa4fcb1fbfa80 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index de2a6fb18d69b..14a1b7db9e33d 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index 6c0c016fdf94f..e75a843f5e6bb 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index b8df19152a606..004af0a1ab203 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 8119709a6aeff..a35156a4bbfff 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 96ac525880213..dedc7dfc82056 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index f385f07f951ce..ebb05257d85f9 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 0c9befc921af6..ad743a7712812 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index db2f6721e0cc4..a5c3106c4e261 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,8 +14,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.1-SNAPSHOT
-SPARK_VERSION_SHORT: 3.0.1
+SPARK_VERSION: 3.0.0
+SPARK_VERSION_SHORT: 3.0.0
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index 1b59c629a8c73..ec3490545132d 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 87183e72f5de3..70cada7ce95e4 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index b57e3c0e25ae8..32679c759404e 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 9426db5e11354..ae22afc3faeae 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index d77364d0210fe..7b10f0e78bc28 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index 70a3406701315..b2ecc0f81d7ac 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index a338b6511298f..19e7352e3945c 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 599ae172671e5..bb37b213c697b 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 9fa18d9f82833..cfe7b7373b967 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index acfc2c5a36b1a..d2f828421e64f 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index f0666536bb706..ebfaccffcdded 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 48f559e4822f3..8a62a48236aab 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 0759ce5c114b3..c2f25df2f147e 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index ed34a370a8274..bb4b640c68fe9 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index b8ae344e63e65..c41bbe6ea85f8 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 37548de1813de..a15b7a0210a1b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.1-SNAPSHOT</version>
+  <version>3.0.0</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 906298ff9964a..8011a5f7a3423 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.1.dev0"
+__version__ = "3.0.0"
diff --git a/repl/pom.xml b/repl/pom.xml
index b826b87f53efc..83c8d34a3a132 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 9a48bf37b2cd5..d49f0c80be1c1 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 52412570a2788..261bacce5391d 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index e0c997f869aba..c61f8dff8cf55 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 95299d2066ea3..f18f2ce58588a 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 6667c7bd089dc..14a0f665ea84e 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index fbfcdf88fa26e..3388aca287aa3 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index f60820dd45f34..6dec472392574 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index e8c1598b3669c..267bd6039e3fa 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 1e99b6c21522b..d6f4f303ad423 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 5430a3e600365..4a58881077524 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From f6053b94f874c62856baa7bfa35df14c78bebc9f Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@apache.org>
Date: Mon, 18 May 2020 13:21:43 +0000
Subject: [PATCH 0699/1280] Preparing development version 3.0.1-SNAPSHOT

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 3bad429ca4086..21f3eaa5c15f6 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.0
+Version: 3.0.1
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0a52a00c4a694..8bef9d8bc56cb 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index fa4fcb1fbfa80..fc1441d80b7bf 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 14a1b7db9e33d..de2a6fb18d69b 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index e75a843f5e6bb..6c0c016fdf94f 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 004af0a1ab203..b8df19152a606 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index a35156a4bbfff..8119709a6aeff 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index dedc7dfc82056..96ac525880213 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index ebb05257d85f9..f385f07f951ce 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index ad743a7712812..0c9befc921af6 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index a5c3106c4e261..db2f6721e0cc4 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,8 +14,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.0
-SPARK_VERSION_SHORT: 3.0.0
+SPARK_VERSION: 3.0.1-SNAPSHOT
+SPARK_VERSION_SHORT: 3.0.1
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index ec3490545132d..1b59c629a8c73 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 70cada7ce95e4..87183e72f5de3 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 32679c759404e..b57e3c0e25ae8 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index ae22afc3faeae..9426db5e11354 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 7b10f0e78bc28..d77364d0210fe 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index b2ecc0f81d7ac..70a3406701315 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 19e7352e3945c..a338b6511298f 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index bb37b213c697b..599ae172671e5 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index cfe7b7373b967..9fa18d9f82833 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index d2f828421e64f..acfc2c5a36b1a 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index ebfaccffcdded..f0666536bb706 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 8a62a48236aab..48f559e4822f3 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index c2f25df2f147e..0759ce5c114b3 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index bb4b640c68fe9..ed34a370a8274 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index c41bbe6ea85f8..b8ae344e63e65 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index a15b7a0210a1b..37548de1813de 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.0</version>
+  <version>3.0.1-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 8011a5f7a3423..906298ff9964a 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.0"
+__version__ = "3.0.1.dev0"
diff --git a/repl/pom.xml b/repl/pom.xml
index 83c8d34a3a132..b826b87f53efc 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index d49f0c80be1c1..9a48bf37b2cd5 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 261bacce5391d..52412570a2788 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index c61f8dff8cf55..e0c997f869aba 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index f18f2ce58588a..95299d2066ea3 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 14a0f665ea84e..6667c7bd089dc 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 3388aca287aa3..fbfcdf88fa26e 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 6dec472392574..f60820dd45f34 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 267bd6039e3fa..e8c1598b3669c 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d6f4f303ad423..1e99b6c21522b 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 4a58881077524..5430a3e600365 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 847d6d4c6d70fc1b78b0d877a358bd80df686de3 Mon Sep 17 00:00:00 2001
From: Javier Fuentes <j.fuentes.m@icloud.com>
Date: Tue, 19 May 2020 08:57:30 +0900
Subject: [PATCH 0700/1280] [SPARK-31102][SQL][3.0] Spark-sql fails to parse
 when contains comment

This PR introduces a change to false for the insideComment flag on a newline. Fixing the issue introduced by SPARK-30049.

Backport to 3.0 from #27920

Previously on SPARK-30049 a comment containing an unclosed quote produced the following issue:
```
spark-sql> SELECT 1 -- someone's comment here
         > ;
Error in query:
extraneous input ';' expecting <EOF>(line 2, pos 0)

== SQL ==
SELECT 1 -- someone's comment here
;
^^^
```

This was caused because there was no flag for comment sections inside the splitSemiColon method to ignore quotes. SPARK-30049 added that flag and fixed the issue, but introduced the follwoing problem:
```
spark-sql> select
         >   1,
         >   -- two
         >   2;
Error in query:
mismatched input '<EOF>' expecting {'(', 'ADD', 'AFTER', 'ALL', 'ALTER', ...}(line 3, pos 2)
== SQL ==
select
  1,
--^^^
```
This issue is generated by a missing turn-off for the insideComment flag with a newline.

No

- For previous tests using line-continuity(`\`) it was added a line-continuity rule in the SqlBase.g4 file to add the functionality to the SQL context.
- A new test for inline comments was added.

Closes #27920 from javierivanov/SPARK-31102.

Authored-by: Javier Fuentes <j.fuentes.micloud.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>

### What changes were proposed in this pull request?

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #28565 from javierivanov/SPARK-3.0-31102.

Authored-by: Javier Fuentes <j.fuentes.m@icloud.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/catalyst/parser/SqlBase.g4      |  2 +-
 .../sql/catalyst/parser/PlanParserSuite.scala | 10 ++++++++++
 .../hive/thriftserver/SparkSQLCLIDriver.scala | 12 +++++------
 .../sql/hive/thriftserver/CliSuite.scala      | 20 ++++++++++---------
 4 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 613582fa0a99e..2adaa9f46ceba 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -1784,7 +1784,7 @@ fragment LETTER
     ;
 
 SIMPLE_COMMENT
-    : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN)
+    : '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN)
     ;
 
 BRACKETED_EMPTY_COMMENT
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
index 714f8c53e1d35..515ba2a076fa8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala
@@ -55,6 +55,16 @@ class PlanParserSuite extends AnalysisTest {
     With(plan, ctes)
   }
 
+  test("single comment case one") {
+    val plan = table("a").select(star())
+    assertEqual("-- single comment\nSELECT * FROM a", plan)
+  }
+
+  test("single comment case two") {
+    val plan = table("a").select(star())
+    assertEqual("-- single comment\\\nwith line continuity\nSELECT * FROM a", plan)
+  }
+
   test("case insensitive") {
     val plan = table("a").select(star())
     assertEqual("sELEct * FroM a", plan)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 50b5eb43bd1ad..c7848afd822d5 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -524,7 +524,6 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
     var insideComment = false
     var escape = false
     var beginIndex = 0
-    var endIndex = line.length
     val ret = new JArrayList[String]
 
     for (index <- 0 until line.length) {
@@ -552,8 +551,6 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
         } else if (hasNext && line.charAt(index + 1) == '-') {
           // ignore quotes and ;
           insideComment = true
-          // ignore eol
-          endIndex = index
         }
       } else if (line.charAt(index) == ';') {
         if (insideSingleQuote || insideDoubleQuote || insideComment) {
@@ -563,8 +560,11 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
           ret.add(line.substring(beginIndex, index))
           beginIndex = index + 1
         }
-      } else {
-        // nothing to do
+      } else if (line.charAt(index) == '\n') {
+        // with a new line the inline comment should end.
+        if (!escape) {
+          insideComment = false
+        }
       }
       // set the escape
       if (escape) {
@@ -573,7 +573,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
         escape = true
       }
     }
-    ret.add(line.substring(beginIndex, endIndex))
+    ret.add(line.substring(beginIndex))
     ret
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index b41a4b7f891db..ea1a371151c36 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -508,18 +508,20 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
     )
   }
 
-  test("SPARK-30049 Should not complain for quotes in commented with multi-lines") {
+  test("SPARK-31102 spark-sql fails to parse when contains comment") {
     runCliWithin(1.minute)(
-      """SELECT concat('test', 'comment') -- someone's comment here \\
-        | comment continues here with single ' quote \\
-        | extra ' \\
-        |;""".stripMargin -> "testcomment"
+      """SELECT concat('test', 'comment'),
+        |    -- someone's comment here
+        | 2;""".stripMargin -> "testcomment"
     )
+  }
+
+  test("SPARK-30049 Should not complain for quotes in commented with multi-lines") {
     runCliWithin(1.minute)(
-      """SELECT concat('test', 'comment') -- someone's comment here \\
-        |   comment continues here with single ' quote \\
-        |   extra ' \\
-        |   ;""".stripMargin -> "testcomment"
+      """SELECT concat('test', 'comment') -- someone's comment here \
+        | comment continues here with single ' quote \
+        | extra ' \
+        |;""".stripMargin -> "testcomment"
     )
   }
 

From 18925c34ffc403c1422dd8ac8ae731c213521387 Mon Sep 17 00:00:00 2001
From: David Toneian <david@toneian.com>
Date: Mon, 18 May 2020 20:25:02 +0900
Subject: [PATCH 0701/1280] [SPARK-31739][PYSPARK][DOCS][MINOR] Fix docstring
 syntax issues and misplaced space characters

This commit is published into the public domain.

Some syntax issues in docstrings have been fixed.

In some places, the documentation did not render as intended, e.g. parameter documentations were not formatted as such.

Slight improvements in documentation.

Manual testing and `dev/lint-python` run. No new Sphinx warnings arise due to this change.

Closes #28559 from DavidToneian/SPARK-31739.

Authored-by: David Toneian <david@toneian.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/ml/clustering.py  |  2 +-
 python/pyspark/ml/regression.py  |  2 +-
 python/pyspark/ml/util.py        |  1 +
 python/pyspark/mllib/util.py     |  4 ++--
 python/pyspark/sql/dataframe.py  |  6 +++---
 python/pyspark/sql/readwriter.py | 14 +++++++-------
 python/pyspark/sql/streaming.py  | 26 +++++++++++++-------------
 7 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 7465cef7d64d4..e1c2732af31b9 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -795,7 +795,7 @@ def computeCost(self, dataset):
         Computes the sum of squared distances between the input points
         and their corresponding cluster centers.
 
-        ..note:: Deprecated in 3.0.0. It will be removed in future versions. Use
+        .. note:: Deprecated in 3.0.0. It will be removed in future versions. Use
            ClusteringEvaluator instead. You can also get the cost on the training dataset in the
            summary.
         """
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index a4c97827d55de..f367bb8faa825 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -53,7 +53,7 @@ class JavaRegressor(JavaPredictor, _JavaPredictorParams):
 class JavaRegressionModel(JavaPredictionModel, _JavaPredictorParams):
     """
     Java Model produced by a ``_JavaRegressor``.
-    To be mixed in with class:`pyspark.ml.JavaModel`
+    To be mixed in with :class:`pyspark.ml.JavaModel`
 
     .. versionadded:: 3.0.0
     """
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 35ad5518e1c1f..aac2b38d3f57d 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -563,6 +563,7 @@ def loadParamsInstance(path, sc):
 class HasTrainingSummary(object):
     """
     Base class for models that provides Training summary.
+
     .. versionadded:: 3.0.0
     """
 
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index 1a0ce42dc4e4f..f0f9cda4672b1 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -372,7 +372,7 @@ def save(self, sc, path):
          * human-readable (JSON) model metadata to path/metadata/
          * Parquet formatted data to path/data/
 
-        The model may be loaded using py:meth:`Loader.load`.
+        The model may be loaded using :py:meth:`Loader.load`.
 
         :param sc: Spark context used to save model data.
         :param path: Path specifying the directory in which to save
@@ -412,7 +412,7 @@ class Loader(object):
     def load(cls, sc, path):
         """
         Load a model from the given path. The model should have been
-        saved using py:meth:`Saveable.save`.
+        saved using :py:meth:`Saveable.save`.
 
         :param sc: Spark context used for loading model files.
         :param path: Path specifying the directory to which the model
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 2a366dcb9fcbb..6f4fdd34a5f44 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -2138,7 +2138,7 @@ def drop(self, *cols):
 
     @ignore_unicode_prefix
     def toDF(self, *cols):
-        """Returns a new class:`DataFrame` that with new specified column names
+        """Returns a new :class:`DataFrame` that with new specified column names
 
         :param cols: list of new column names (string)
 
@@ -2150,9 +2150,9 @@ def toDF(self, *cols):
 
     @since(3.0)
     def transform(self, func):
-        """Returns a new class:`DataFrame`. Concise syntax for chaining custom transformations.
+        """Returns a new :class:`DataFrame`. Concise syntax for chaining custom transformations.
 
-        :param func: a function that takes and returns a class:`DataFrame`.
+        :param func: a function that takes and returns a :class:`DataFrame`.
 
         >>> from pyspark.sql.functions import col
         >>> df = spark.createDataFrame([(1, 1.0), (2, 2.0)], ["int", "float"])
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 6ad6377288ec5..336345e383729 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -223,15 +223,15 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param mode: allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
                   schema does not have the field, it drops corrupt records during parsing. \
                   When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
                   field in an output schema.
-                *  ``DROPMALFORMED`` : ignores the whole corrupted records.
-                *  ``FAILFAST`` : throws an exception when it meets corrupted records.
+                *  ``DROPMALFORMED``: ignores the whole corrupted records.
+                *  ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -470,7 +470,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                      be controlled by ``spark.sql.csv.parser.columnPruning.enabled``
                      (enabled by default).
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
@@ -479,8 +479,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                   When it meets a record having fewer tokens than the length of the schema, \
                   sets ``null`` to extra fields. When the record has more tokens than the \
                   length of the schema, it drops extra tokens.
-                * ``DROPMALFORMED`` : ignores the whole corrupted records.
-                * ``FAILFAST`` : throws an exception when it meets corrupted records.
+                * ``DROPMALFORMED``: ignores the whole corrupted records.
+                * ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -830,7 +830,7 @@ def save(self, path=None, format=None, mode=None, partitionBy=None, **options):
     def insertInto(self, tableName, overwrite=None):
         """Inserts the content of the :class:`DataFrame` to the specified table.
 
-        It requires that the schema of the class:`DataFrame` is the same as the
+        It requires that the schema of the :class:`DataFrame` is the same as the
         schema of the table.
 
         Optionally overwriting any existing data.
diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py
index 05cf331d897a2..2450a4c93c460 100644
--- a/python/pyspark/sql/streaming.py
+++ b/python/pyspark/sql/streaming.py
@@ -461,15 +461,15 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None,
         :param mode: allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
                   schema does not have the field, it drops corrupt records during parsing. \
                   When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \
                   field in an output schema.
-                *  ``DROPMALFORMED`` : ignores the whole corrupted records.
-                *  ``FAILFAST`` : throws an exception when it meets corrupted records.
+                *  ``DROPMALFORMED``: ignores the whole corrupted records.
+                *  ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -707,7 +707,7 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
         :param mode: allows a mode for dealing with corrupt records during parsing. If None is
                      set, it uses the default value, ``PERMISSIVE``.
 
-                * ``PERMISSIVE`` : when it meets a corrupted record, puts the malformed string \
+                * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \
                   into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \
                   fields to ``null``. To keep corrupt records, an user can set a string type \
                   field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \
@@ -716,8 +716,8 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                   When it meets a record having fewer tokens than the length of the schema, \
                   sets ``null`` to extra fields. When the record has more tokens than the \
                   length of the schema, it drops extra tokens.
-                * ``DROPMALFORMED`` : ignores the whole corrupted records.
-                * ``FAILFAST`` : throws an exception when it meets corrupted records.
+                * ``DROPMALFORMED``: ignores the whole corrupted records.
+                * ``FAILFAST``: throws an exception when it meets corrupted records.
 
         :param columnNameOfCorruptRecord: allows renaming the new field having malformed string
                                           created by ``PERMISSIVE`` mode. This overrides
@@ -795,11 +795,11 @@ def outputMode(self, outputMode):
 
         Options include:
 
-        * `append`:Only the new rows in the streaming DataFrame/Dataset will be written to
+        * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to
            the sink
-        * `complete`:All the rows in the streaming DataFrame/Dataset will be written to the sink
+        * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the sink
            every time these is some updates
-        * `update`:only the rows that were updated in the streaming DataFrame/Dataset will be
+        * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
            written to the sink every time there are some updates. If the query doesn't contain
            aggregations, it will be equivalent to `append` mode.
 
@@ -1170,11 +1170,11 @@ def start(self, path=None, format=None, outputMode=None, partitionBy=None, query
         :param outputMode: specifies how data of a streaming DataFrame/Dataset is written to a
                            streaming sink.
 
-            * `append`:Only the new rows in the streaming DataFrame/Dataset will be written to the
+            * `append`: Only the new rows in the streaming DataFrame/Dataset will be written to the
               sink
-            * `complete`:All the rows in the streaming DataFrame/Dataset will be written to the sink
-               every time these is some updates
-            * `update`:only the rows that were updated in the streaming DataFrame/Dataset will be
+            * `complete`: All the rows in the streaming DataFrame/Dataset will be written to the
+              sink every time these is some updates
+            * `update`: only the rows that were updated in the streaming DataFrame/Dataset will be
               written to the sink every time there are some updates. If the query doesn't contain
               aggregations, it will be equivalent to `append` mode.
         :param partitionBy: names of partitioning columns

From d668d679e93dc4f84bdce15f9f6bdb19552496f1 Mon Sep 17 00:00:00 2001
From: Eren Avsarogullari <eren.avsarogullari@gmail.com>
Date: Mon, 18 May 2020 23:21:32 -0700
Subject: [PATCH 0702/1280] [SPARK-31440][SQL] Improve SQL Rest API

### What changes were proposed in this pull request?
SQL Rest API exposes query execution metrics as Public API. This PR aims to apply following improvements on SQL Rest API by aligning Spark-UI.

**Proposed Improvements:**
1- Support Physical Operations and group metrics per physical operation by aligning Spark UI.
2- Support `wholeStageCodegenId` for Physical Operations
3- `nodeId` can be useful for grouping metrics and sorting physical operations (according to execution order) to differentiate same operators (if used multiple times during the same query execution) and their metrics.
4- Filter `empty` metrics by aligning with Spark UI - SQL Tab. Currently, Spark UI does not show empty metrics.
5- Remove line breakers(`\n`) from `metricValue`.
6- `planDescription` can be `optional` Http parameter to avoid network cost where there is specially complex jobs creating big-plans.
7- `metrics` attribute needs to be exposed at the bottom order as `nodes`. Specially, this can be useful for the user where `nodes` array size is high.
8- `edges` attribute is being exposed to show relationship between `nodes`.
9- Reverse order on `metricDetails` aims to match with Spark UI by supporting Physical Operators' execution order.

### Why are the changes needed?
Proposed improvements provides more useful (e.g: physical operations and metrics correlation, grouping) and clear (e.g: filtering blank metrics, removing line breakers) result for the end-user.

### Does this PR introduce any user-facing change?
Yes. Please find both current and improved versions of the results as attached for following SQL Rest Endpoint:
```
curl -X GET http://localhost:4040/api/v1/applications/$appId/sql/$executionId?details=true
```
**Current version:**
https://issues.apache.org/jira/secure/attachment/12999821/current_version.json

**Improved version:**
https://issues.apache.org/jira/secure/attachment/13000621/improved_version.json

### Backward Compatibility
SQL Rest API will be started to expose with `Spark 3.0` and `3.0.0-preview2` (released on 12/23/19) does not cover this API so if PR can catch 3.0 release, this will not have any backward compatibility issue.

### How was this patch tested?
1. New Unit tests are added.
2. Also, patch has been tested manually through both **Spark Core** and **History Server** Rest APIs.

Closes #28208 from erenavsarogullari/SPARK-31440.

Authored-by: Eren Avsarogullari <eren.avsarogullari@gmail.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit ab4cf49a1ca5a2578ecd0441ad6db8e88593e648)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../sql/execution/ui/SparkPlanGraph.scala     |   6 +-
 .../spark/status/api/v1/sql/SqlResource.scala |  90 ++++++--
 .../apache/spark/status/api/v1/sql/api.scala  |  15 +-
 .../status/api/v1/sql/SqlResourceSuite.scala  | 205 ++++++++++++++++++
 4 files changed, 293 insertions(+), 23 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index 274a5a414ffa2..a798fe02700e3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -153,7 +153,7 @@ object SparkPlanGraph {
  * @param name the name of this SparkPlan node
  * @param metrics metrics that this SparkPlan node will track
  */
-private[ui] class SparkPlanGraphNode(
+class SparkPlanGraphNode(
     val id: Long,
     val name: String,
     val desc: String,
@@ -193,7 +193,7 @@ private[ui] class SparkPlanGraphNode(
 /**
  * Represent a tree of SparkPlan for WholeStageCodegen.
  */
-private[ui] class SparkPlanGraphCluster(
+class SparkPlanGraphCluster(
     id: Long,
     name: String,
     desc: String,
@@ -229,7 +229,7 @@ private[ui] class SparkPlanGraphCluster(
  * Represent an edge in the SparkPlan tree. `fromId` is the child node id, and `toId` is the parent
  * node id.
  */
-private[ui] case class SparkPlanGraphEdge(fromId: Long, toId: Long) {
+case class SparkPlanGraphEdge(fromId: Long, toId: Long) {
 
   def makeDotEdge: String = s"""  $fromId->$toId;\n"""
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
index 346e07f2bef15..c7599f864dd97 100644
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
@@ -21,21 +21,29 @@ import java.util.Date
 import javax.ws.rs._
 import javax.ws.rs.core.MediaType
 
+import scala.util.{Failure, Success, Try}
+
 import org.apache.spark.JobExecutionStatus
-import org.apache.spark.sql.execution.ui.{SQLAppStatusStore, SQLExecutionUIData, SQLPlanMetric}
+import org.apache.spark.sql.execution.ui.{SparkPlanGraph, SparkPlanGraphCluster, SparkPlanGraphNode, SQLAppStatusStore, SQLExecutionUIData}
 import org.apache.spark.status.api.v1.{BaseAppResource, NotFoundException}
 
 @Produces(Array(MediaType.APPLICATION_JSON))
 private[v1] class SqlResource extends BaseAppResource {
 
+  val WHOLE_STAGE_CODEGEN = "WholeStageCodegen"
+
   @GET
   def sqlList(
-      @DefaultValue("false") @QueryParam("details") details: Boolean,
+      @DefaultValue("true") @QueryParam("details") details: Boolean,
+      @DefaultValue("true") @QueryParam("planDescription") planDescription: Boolean,
       @DefaultValue("0") @QueryParam("offset") offset: Int,
       @DefaultValue("20") @QueryParam("length") length: Int): Seq[ExecutionData] = {
     withUI { ui =>
       val sqlStore = new SQLAppStatusStore(ui.store.store)
-      sqlStore.executionsList(offset, length).map(prepareExecutionData(_, details))
+      sqlStore.executionsList(offset, length).map { exec =>
+        val graph = sqlStore.planGraph(exec.executionId)
+        prepareExecutionData(exec, graph, details, planDescription)
+      }
     }
   }
 
@@ -43,24 +51,25 @@ private[v1] class SqlResource extends BaseAppResource {
   @Path("{executionId:\\d+}")
   def sql(
       @PathParam("executionId") execId: Long,
-      @DefaultValue("false") @QueryParam("details") details: Boolean): ExecutionData = {
+      @DefaultValue("true") @QueryParam("details") details: Boolean,
+      @DefaultValue("true") @QueryParam("planDescription")
+      planDescription: Boolean): ExecutionData = {
     withUI { ui =>
       val sqlStore = new SQLAppStatusStore(ui.store.store)
+      val graph = sqlStore.planGraph(execId)
       sqlStore
         .execution(execId)
-        .map(prepareExecutionData(_, details))
-        .getOrElse(throw new NotFoundException("unknown id: " + execId))
+        .map(prepareExecutionData(_, graph, details, planDescription))
+        .getOrElse(throw new NotFoundException("unknown query execution id: " + execId))
     }
   }
 
-  private def printableMetrics(
-      metrics: Seq[SQLPlanMetric],
-      metricValues: Map[Long, String]): Seq[Metrics] = {
-    metrics.map(metric =>
-      Metrics(metric.name, metricValues.get(metric.accumulatorId).getOrElse("")))
-  }
+  private def prepareExecutionData(
+    exec: SQLExecutionUIData,
+    graph: SparkPlanGraph,
+    details: Boolean,
+    planDescription: Boolean): ExecutionData = {
 
-  private def prepareExecutionData(exec: SQLExecutionUIData, details: Boolean): ExecutionData = {
     var running = Seq[Int]()
     var completed = Seq[Int]()
     var failed = Seq[Int]()
@@ -84,18 +93,65 @@ private[v1] class SqlResource extends BaseAppResource {
     }
 
     val duration = exec.completionTime.getOrElse(new Date()).getTime - exec.submissionTime
-    val planDetails = if (details) exec.physicalPlanDescription else ""
-    val metrics = if (details) printableMetrics(exec.metrics, exec.metricValues) else Seq.empty
+    val planDetails = if (planDescription) exec.physicalPlanDescription else ""
+    val nodes = if (details) printableMetrics(graph.allNodes, exec.metricValues) else Seq.empty
+    val edges = if (details) graph.edges else Seq.empty
+
     new ExecutionData(
       exec.executionId,
       status,
       exec.description,
       planDetails,
-      metrics,
       new Date(exec.submissionTime),
       duration,
       running,
       completed,
-      failed)
+      failed,
+      nodes,
+      edges)
   }
+
+  private def printableMetrics(allNodes: Seq[SparkPlanGraphNode],
+    metricValues: Map[Long, String]): Seq[Node] = {
+
+    def getMetric(metricValues: Map[Long, String], accumulatorId: Long,
+      metricName: String): Option[Metric] = {
+
+      metricValues.get(accumulatorId).map( mv => {
+        val metricValue = if (mv.startsWith("\n")) mv.substring(1, mv.length) else mv
+        Metric(metricName, metricValue)
+      })
+    }
+
+    val nodeIdAndWSCGIdMap = getNodeIdAndWSCGIdMap(allNodes)
+    val nodes = allNodes.map { node =>
+      val wholeStageCodegenId = nodeIdAndWSCGIdMap.get(node.id).flatten
+      val metrics =
+        node.metrics.flatMap(m => getMetric(metricValues, m.accumulatorId, m.name.trim))
+      Node(nodeId = node.id, nodeName = node.name.trim, wholeStageCodegenId, metrics)
+    }
+
+    nodes.sortBy(_.nodeId).reverse
+  }
+
+  private def getNodeIdAndWSCGIdMap(allNodes: Seq[SparkPlanGraphNode]): Map[Long, Option[Long]] = {
+    val wscgNodes = allNodes.filter(_.name.trim.startsWith(WHOLE_STAGE_CODEGEN))
+    val nodeIdAndWSCGIdMap: Map[Long, Option[Long]] = wscgNodes.flatMap {
+      _ match {
+        case x: SparkPlanGraphCluster => x.nodes.map(_.id -> getWholeStageCodegenId(x.name.trim))
+        case _ => Seq.empty
+      }
+    }.toMap
+
+    nodeIdAndWSCGIdMap
+  }
+
+  private def getWholeStageCodegenId(wscgNodeName: String): Option[Long] = {
+    Try(wscgNodeName.substring(
+      s"$WHOLE_STAGE_CODEGEN (".length, wscgNodeName.length - 1).toLong) match {
+      case Success(wscgId) => Some(wscgId)
+      case Failure(t) => None
+    }
+  }
+
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
index 7ace66ffb06e1..0ddf66718bce7 100644
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
+++ b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
@@ -18,16 +18,25 @@ package org.apache.spark.status.api.v1.sql
 
 import java.util.Date
 
+import org.apache.spark.sql.execution.ui.SparkPlanGraphEdge
+
 class ExecutionData private[spark] (
     val id: Long,
     val status: String,
     val description: String,
     val planDescription: String,
-    val metrics: Seq[Metrics],
     val submissionTime: Date,
     val duration: Long,
     val runningJobIds: Seq[Int],
     val successJobIds: Seq[Int],
-    val failedJobIds: Seq[Int])
+    val failedJobIds: Seq[Int],
+    val nodes: Seq[Node],
+    val edges: Seq[SparkPlanGraphEdge])
+
+case class Node private[spark](
+    nodeId: Long,
+    nodeName: String,
+    wholeStageCodegenId: Option[Long] = None,
+    metrics: Seq[Metric])
 
-case class Metrics private[spark] (metricName: String, metricValue: String)
+case class Metric private[spark] (name: String, value: String)
diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
new file mode 100644
index 0000000000000..43cca246cc47c
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.status.api.v1.sql
+
+import java.util.Date
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.scalatest.PrivateMethodTester
+
+import org.apache.spark.{JobExecutionStatus, SparkFunSuite}
+import org.apache.spark.sql.execution.ui.{SparkPlanGraph, SparkPlanGraphCluster, SparkPlanGraphEdge, SparkPlanGraphNode, SQLExecutionUIData, SQLPlanMetric}
+
+object SqlResourceSuite {
+
+  val SCAN_TEXT = "Scan text"
+  val FILTER = "Filter"
+  val WHOLE_STAGE_CODEGEN_1 = "WholeStageCodegen (1)"
+  val DURATION = "duration"
+  val NUMBER_OF_OUTPUT_ROWS = "number of output rows"
+  val METADATA_TIME = "metadata time"
+  val NUMBER_OF_FILES_READ = "number of files read"
+  val SIZE_OF_FILES_READ = "size of files read"
+  val PLAN_DESCRIPTION = "== Physical Plan ==\nCollectLimit (3)\n+- * Filter (2)\n +- Scan text..."
+  val DESCRIPTION = "csv at MyDataFrames.scala:57"
+
+  val nodeIdAndWSCGIdMap: Map[Long, Option[Long]] = Map(1L -> Some(1L))
+
+  val filterNode = new SparkPlanGraphNode(1, FILTER, "",
+    metrics = Seq(SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 1, "")))
+  val nodes: Seq[SparkPlanGraphNode] = Seq(
+    new SparkPlanGraphCluster(0, WHOLE_STAGE_CODEGEN_1, "",
+      nodes = ArrayBuffer(filterNode),
+      metrics = Seq(SQLPlanMetric(DURATION, 0, ""))),
+    new SparkPlanGraphNode(2, SCAN_TEXT, "",
+      metrics = Seq(
+      SQLPlanMetric(METADATA_TIME, 2, ""),
+      SQLPlanMetric(NUMBER_OF_FILES_READ, 3, ""),
+      SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 4, ""),
+      SQLPlanMetric(SIZE_OF_FILES_READ, 5, ""))))
+
+  val nodesWhenCodegenIsOff: Seq[SparkPlanGraphNode] =
+    SparkPlanGraph(nodes, edges).allNodes.filterNot(_.name == WHOLE_STAGE_CODEGEN_1)
+
+  val edges: Seq[SparkPlanGraphEdge] =
+    Seq(SparkPlanGraphEdge(3, 2))
+
+  val metrics: Seq[SQLPlanMetric] = {
+    Seq(SQLPlanMetric(DURATION, 0, ""),
+      SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 1, ""),
+      SQLPlanMetric(METADATA_TIME, 2, ""),
+      SQLPlanMetric(NUMBER_OF_FILES_READ, 3, ""),
+      SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 4, ""),
+      SQLPlanMetric(SIZE_OF_FILES_READ, 5, ""))
+  }
+
+  val sqlExecutionUIData: SQLExecutionUIData = {
+    def getMetricValues() = {
+      Map[Long, String](
+        0L -> "0 ms",
+        1L -> "1",
+        2L -> "2 ms",
+        3L -> "1",
+        4L -> "1",
+        5L -> "330.0 B"
+      )
+    }
+
+    new SQLExecutionUIData(
+      executionId = 0,
+      description = DESCRIPTION,
+      details = "",
+      physicalPlanDescription = PLAN_DESCRIPTION,
+      metrics = metrics,
+      submissionTime = 1586768888233L,
+      completionTime = Some(new Date(1586768888999L)),
+      jobs = Map[Int, JobExecutionStatus](
+        0 -> JobExecutionStatus.SUCCEEDED,
+        1 -> JobExecutionStatus.SUCCEEDED),
+      stages = Set[Int](),
+      metricValues = getMetricValues()
+    )
+  }
+
+  private def getNodes(): Seq[Node] = {
+    val node = Node(0, WHOLE_STAGE_CODEGEN_1,
+      wholeStageCodegenId = None, metrics = Seq(Metric(DURATION, "0 ms")))
+    val node2 = Node(1, FILTER,
+      wholeStageCodegenId = Some(1), metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
+    val node3 = Node(2, SCAN_TEXT, wholeStageCodegenId = None,
+      metrics = Seq(Metric(METADATA_TIME, "2 ms"),
+        Metric(NUMBER_OF_FILES_READ, "1"),
+        Metric(NUMBER_OF_OUTPUT_ROWS, "1"),
+        Metric(SIZE_OF_FILES_READ, "330.0 B")))
+
+    // reverse order because of supporting execution order by aligning with Spark-UI
+    Seq(node3, node2, node)
+  }
+
+  private def getExpectedNodesWhenWholeStageCodegenIsOff(): Seq[Node] = {
+    val node = Node(1, FILTER, metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
+    val node2 = Node(2, SCAN_TEXT,
+      metrics = Seq(Metric(METADATA_TIME, "2 ms"),
+        Metric(NUMBER_OF_FILES_READ, "1"),
+        Metric(NUMBER_OF_OUTPUT_ROWS, "1"),
+        Metric(SIZE_OF_FILES_READ, "330.0 B")))
+
+    // reverse order because of supporting execution order by aligning with Spark-UI
+    Seq(node2, node)
+  }
+
+  private def verifyExpectedExecutionData(executionData: ExecutionData,
+    nodes: Seq[Node],
+    edges: Seq[SparkPlanGraphEdge],
+    planDescription: String): Unit = {
+
+    assert(executionData.id == 0)
+    assert(executionData.status == "COMPLETED")
+    assert(executionData.description == DESCRIPTION)
+    assert(executionData.planDescription == planDescription)
+    assert(executionData.submissionTime == new Date(1586768888233L))
+    assert(executionData.duration == 766L)
+    assert(executionData.successJobIds == Seq[Int](0, 1))
+    assert(executionData.runningJobIds == Seq[Int]())
+    assert(executionData.failedJobIds == Seq.empty)
+    assert(executionData.nodes == nodes)
+    assert(executionData.edges == edges)
+  }
+
+}
+
+/**
+ * Sql Resource Public API Unit Tests.
+ */
+class SqlResourceSuite extends SparkFunSuite with PrivateMethodTester {
+
+  import SqlResourceSuite._
+
+  val sqlResource = new SqlResource()
+  val prepareExecutionData = PrivateMethod[ExecutionData]('prepareExecutionData)
+
+  test("Prepare ExecutionData when details = false and planDescription = false") {
+    val executionData =
+      sqlResource invokePrivate prepareExecutionData(
+        sqlExecutionUIData, SparkPlanGraph(Seq.empty, Seq.empty), false, false)
+    verifyExpectedExecutionData(executionData, edges = Seq.empty,
+      nodes = Seq.empty, planDescription = "")
+  }
+
+  test("Prepare ExecutionData when details = true and planDescription = false") {
+    val executionData =
+      sqlResource invokePrivate prepareExecutionData(
+        sqlExecutionUIData, SparkPlanGraph(nodes, edges), true, false)
+    verifyExpectedExecutionData(
+      executionData,
+      nodes = getNodes(),
+      edges,
+      planDescription = "")
+  }
+
+  test("Prepare ExecutionData when details = true and planDescription = true") {
+    val executionData =
+      sqlResource invokePrivate prepareExecutionData(
+        sqlExecutionUIData, SparkPlanGraph(nodes, edges), true, true)
+    verifyExpectedExecutionData(
+      executionData,
+      nodes = getNodes(),
+      edges = edges,
+      planDescription = PLAN_DESCRIPTION)
+  }
+
+  test("Prepare ExecutionData when details = true and planDescription = false and WSCG = off") {
+    val executionData =
+      sqlResource invokePrivate prepareExecutionData(
+        sqlExecutionUIData, SparkPlanGraph(nodesWhenCodegenIsOff, edges), true, false)
+    verifyExpectedExecutionData(
+      executionData,
+      nodes = getExpectedNodesWhenWholeStageCodegenIsOff(),
+      edges = edges,
+      planDescription = "")
+  }
+
+  test("Parse wholeStageCodegenId from nodeName") {
+    val getWholeStageCodegenId = PrivateMethod[Option[Long]]('getWholeStageCodegenId)
+    val wholeStageCodegenId =
+      sqlResource invokePrivate getWholeStageCodegenId(WHOLE_STAGE_CODEGEN_1)
+    assert(wholeStageCodegenId == Some(1))
+  }
+
+}

From 9f34cf56e551c8bb678221cb8671432d4cb02ac0 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 18 May 2020 23:54:41 -0700
Subject: [PATCH 0703/1280] [SPARK-31651][CORE] Improve handling the case where
 different barrier sync types in a single sync

### What changes were proposed in this pull request?

This PR improves handling the case where different barrier sync types in a single sync:

- use `clear` instead of `cleanupBarrierStage `

- make sure all requesters are failed because of "different barrier sync types"

### Why are the changes needed?

Currently, we use `cleanupBarrierStage` to clean up a barrier stage when we detecting the case of "different barrier sync types". But this leads to a problem that we could create new a `ContextBarrierState` for the same stage again if there're on-way requests from tasks. As a result, those task will fail because of killing instead of "different barrier sync types".

Besides, we don't handle the current request which is being handling properly as it will fail due to epoch mismatch instead of "different barrier sync types".

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Updated a existed test.

Closes #28462 from Ngone51/impr_barrier_req.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit 653ca19b1f26155c2b7656e2ebbe227b18383308)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../org/apache/spark/BarrierCoordinator.scala | 30 +++++++++----------
 .../scheduler/BarrierTaskContextSuite.scala   |  5 +---
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
index 5663055129d19..04faf7f87cf2b 100644
--- a/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
+++ b/core/src/main/scala/org/apache/spark/BarrierCoordinator.scala
@@ -21,7 +21,7 @@ import java.util.{Timer, TimerTask}
 import java.util.concurrent.ConcurrentHashMap
 import java.util.function.Consumer
 
-import scala.collection.mutable.ArrayBuffer
+import scala.collection.mutable.{ArrayBuffer, HashSet}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint}
@@ -106,9 +106,11 @@ private[spark] class BarrierCoordinator(
     // The messages will be replied to all tasks once sync finished.
     private val messages = Array.ofDim[String](numTasks)
 
-    // The request method which is called inside this barrier sync. All tasks should make sure
-    // that they're calling the same method within the same barrier sync phase.
-    private var requestMethod: RequestMethod.Value = _
+    // Request methods collected from tasks inside this barrier sync. All tasks should make sure
+    // that they're calling the same method within the same barrier sync phase. In other words,
+    // the size of requestMethods should always be 1 for a legitimate barrier sync. Otherwise,
+    // the barrier sync would fail if the size of requestMethods becomes greater than 1.
+    private val requestMethods = new HashSet[RequestMethod.Value]
 
     // A timer task that ensures we may timeout for a barrier() call.
     private var timerTask: TimerTask = null
@@ -141,17 +143,14 @@ private[spark] class BarrierCoordinator(
       val taskId = request.taskAttemptId
       val epoch = request.barrierEpoch
       val curReqMethod = request.requestMethod
-
-      if (requesters.isEmpty) {
-        requestMethod = curReqMethod
-      } else if (requestMethod != curReqMethod) {
-        requesters.foreach(
-          _.sendFailure(new SparkException(s"$barrierId tried to use requestMethod " +
-            s"`$curReqMethod` during barrier epoch $barrierEpoch, which does not match " +
-            s"the current synchronized requestMethod `$requestMethod`"
-          ))
-        )
-        cleanupBarrierStage(barrierId)
+      requestMethods.add(curReqMethod)
+      if (requestMethods.size > 1) {
+        val error = new SparkException(s"Different barrier sync types found for the " +
+          s"sync $barrierId: ${requestMethods.mkString(", ")}. Please use the " +
+          s"same barrier sync type within a single sync.")
+        (requesters :+ requester).foreach(_.sendFailure(error))
+        clear()
+        return
       }
 
       // Require the number of tasks is correctly set from the BarrierTaskContext.
@@ -184,6 +183,7 @@ private[spark] class BarrierCoordinator(
             s"tasks, finished successfully.")
           barrierEpoch += 1
           requesters.clear()
+          requestMethods.clear()
           cancelTimerTask()
         }
       }
diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 17bc3391cf946..b5614b263ca94 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -96,10 +96,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     val error = intercept[SparkException] {
       rdd2.collect()
     }.getMessage
-    assert(
-      error.contains("does not match the current synchronized requestMethod") ||
-      error.contains("not properly killed")
-    )
+    assert(error.contains("Different barrier sync types found"))
   }
 
   test("successively sync with allGather and barrier") {

From 70ff959c513aad212c2852ab6d8a8176b3e44a6d Mon Sep 17 00:00:00 2001
From: Ali Afroozeh <ali.afroozeh@databricks.com>
Date: Tue, 19 May 2020 11:10:49 +0200
Subject: [PATCH 0704/1280] [SPARK-31721][SQL] Assert optimized is initialized
 before tracking the planning time

### What changes were proposed in this pull request?
The QueryPlanningTracker in QueryExeuction reports the planning time that also includes the optimization time. This happens because the optimizedPlan in QueryExecution is lazy and only will initialize when first called. When df.queryExecution.executedPlan is called, the the tracker starts recording the planning time, and then calls the optimized plan. This causes the planning time to start before optimization and also include the planning time.
This PR fixes this behavior by introducing a method assertOptimized, similar to assertAnalyzed that explicitly initializes the optimized plan. This method is called before measuring the time for sparkPlan and executedPlan. We call it before sparkPlan because that also counts as planning time.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Unit tests

Closes #28543 from dbaliafroozeh/AddAssertOptimized.

Authored-by: Ali Afroozeh <ali.afroozeh@databricks.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit b9cc31cd958fec4b473e757694c5f811316c858e)
Signed-off-by: herman <herman@databricks.com>
---
 .../spark/sql/execution/QueryExecution.scala  | 27 ++++++++++++++-----
 .../QueryPlanningTrackerEndToEndSuite.scala   |  9 +++++++
 .../spark/sql/execution/SparkPlanTest.scala   |  3 ++-
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 99bc45fa9e9e8..1df812d1aa809 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -82,17 +82,30 @@ class QueryExecution(
     sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
   }
 
-  lazy val sparkPlan: SparkPlan = executePhase(QueryPlanningTracker.PLANNING) {
-    // Clone the logical plan here, in case the planner rules change the states of the logical plan.
-    QueryExecution.createSparkPlan(sparkSession, planner, optimizedPlan.clone())
+  private def assertOptimized(): Unit = optimizedPlan
+
+  lazy val sparkPlan: SparkPlan = {
+    // We need to materialize the optimizedPlan here because sparkPlan is also tracked under
+    // the planning phase
+    assertOptimized()
+    executePhase(QueryPlanningTracker.PLANNING) {
+      // Clone the logical plan here, in case the planner rules change the states of the logical
+      // plan.
+      QueryExecution.createSparkPlan(sparkSession, planner, optimizedPlan.clone())
+    }
   }
 
   // executedPlan should not be used to initialize any SparkPlan. It should be
   // only used for execution.
-  lazy val executedPlan: SparkPlan = executePhase(QueryPlanningTracker.PLANNING) {
-    // clone the plan to avoid sharing the plan instance between different stages like analyzing,
-    // optimizing and planning.
-    QueryExecution.prepareForExecution(preparations, sparkPlan.clone())
+  lazy val executedPlan: SparkPlan = {
+    // We need to materialize the optimizedPlan here, before tracking the planning phase, to ensure
+    // that the optimization time is not counted as part of the planning phase.
+    assertOptimized()
+    executePhase(QueryPlanningTracker.PLANNING) {
+      // clone the plan to avoid sharing the plan instance between different stages like analyzing,
+      // optimizing and planning.
+      QueryExecution.prepareForExecution(preparations, sparkPlan.clone())
+    }
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala
index 987338cf6cbbf..5ff459513e848 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/QueryPlanningTrackerEndToEndSuite.scala
@@ -58,4 +58,13 @@ class QueryPlanningTrackerEndToEndSuite extends StreamTest {
       StopStream)
   }
 
+  test("The start times should be in order: parsing <= analysis <= optimization <= planning") {
+    val df = spark.sql("select count(*) from range(1)")
+    df.queryExecution.executedPlan
+    val phases = df.queryExecution.tracker.phases
+    assert(phases("parsing").startTimeMs <= phases("analysis").startTimeMs)
+    assert(phases("analysis").startTimeMs <= phases("optimization").startTimeMs)
+    assert(phases("optimization").startTimeMs <= phases("planning").startTimeMs)
+  }
+
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
index b29e822add8bc..7ddf9d87a6aca 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala
@@ -23,6 +23,7 @@ import scala.util.control.NonFatal
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.{DataFrame, Row, SparkSession, SQLContext}
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.test.SQLTestUtils
 
 /**
@@ -237,7 +238,7 @@ object SparkPlanTest {
    * @param spark SqlContext used for execution of the plan
    */
   def executePlan(outputPlan: SparkPlan, spark: SQLContext): Seq[Row] = {
-    val execution = new QueryExecution(spark.sparkSession, null) {
+    val execution = new QueryExecution(spark.sparkSession, LocalRelation(Nil)) {
       override lazy val sparkPlan: SparkPlan = outputPlan transform {
         case plan: SparkPlan =>
           val inputMap = plan.children.flatMap(_.output).map(a => (a.name, a)).toMap

From d0a7c6fdc1b90aa372c004b66258865ab37c6c3d Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 20 May 2020 11:00:58 +0900
Subject: [PATCH 0705/1280] [SPARK-31750][SQL] Eliminate UpCast if child's
 dataType is DecimalType

### What changes were proposed in this pull request?

Eliminate the `UpCast` if it's child data type is already decimal type.

### Why are the changes needed?

While deserializing internal `Decimal` value to external `BigDecimal`(Java/Scala) value, Spark should also respect `Decimal`'s precision and scale, otherwise it will cause precision lost and look weird in some cases, e.g.:

```
sql("select cast(11111111111111111111111111111111111111 as decimal(38, 0)) as d")
  .write.mode("overwrite")
  .parquet(f.getAbsolutePath)

// can fail
spark.read.parquet(f.getAbsolutePath).as[BigDecimal]
```
```
[info]   org.apache.spark.sql.AnalysisException: Cannot up cast `d` from decimal(38,0) to decimal(38,18).
[info] The type path of the target object is:
[info] - root class: "scala.math.BigDecimal"
[info] You can either add an explicit cast to the input data or choose a higher precision type of the field in the target object;
[info]   at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUpCast$.org$apache$spark$sql$catalyst$analysis$Analyzer$ResolveUpCast$$fail(Analyzer.scala:3060)
[info]   at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUpCast$$anonfun$apply$33$$anonfun$applyOrElse$174.applyOrElse(Analyzer.scala:3087)
[info]   at org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveUpCast$$anonfun$apply$33$$anonfun$applyOrElse$174.applyOrElse(Analyzer.scala:3071)
[info]   at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$1(TreeNode.scala:309)
[info]   at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:72)
[info]   at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:309)
[info]   at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDown$3(TreeNode.scala:314)
```

### Does this PR introduce _any_ user-facing change?

Yes, for cases(cause precision lost) mentioned above will fail before this change but run successfully after this change.

### How was this patch tested?

Added tests.

Closes #28572 from Ngone51/fix_encoder.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../catalyst/DeserializerBuildHelper.scala    |  4 ++++
 .../sql/catalyst/analysis/Analyzer.scala      | 24 +++++++++++++++----
 .../spark/sql/catalyst/expressions/Cast.scala | 10 +++++++-
 .../encoders/EncoderResolutionSuite.scala     | 11 +++++++--
 .../org/apache/spark/sql/DataFrameSuite.scala | 11 +++++++++
 5 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
index e55c25c4b0c54..701e4e3483c0f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/DeserializerBuildHelper.scala
@@ -161,6 +161,10 @@ object DeserializerBuildHelper {
     case _: StructType => expr
     case _: ArrayType => expr
     case _: MapType => expr
+    case _: DecimalType =>
+      // For Scala/Java `BigDecimal`, we accept decimal types of any valid precision/scale.
+      // Here we use the `DecimalType` object to indicate it.
+      UpCast(expr, DecimalType, walkedTypePath.getPaths)
     case _ => UpCast(expr, expected, walkedTypePath.getPaths)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 0d71aee157743..654cf42a07c60 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -3142,15 +3142,29 @@ class Analyzer(
       case p => p transformExpressions {
         case u @ UpCast(child, _, _) if !child.resolved => u
 
-        case UpCast(child, dt: AtomicType, _)
+        case UpCast(_, target, _) if target != DecimalType && !target.isInstanceOf[DataType] =>
+          throw new AnalysisException(
+            s"UpCast only support DecimalType as AbstractDataType yet, but got: $target")
+
+        case UpCast(child, target, walkedTypePath) if target == DecimalType
+          && child.dataType.isInstanceOf[DecimalType] =>
+          assert(walkedTypePath.nonEmpty,
+            "object DecimalType should only be used inside ExpressionEncoder")
+
+          // SPARK-31750: if we want to upcast to the general decimal type, and the `child` is
+          // already decimal type, we can remove the `Upcast` and accept any precision/scale.
+          // This can happen for cases like `spark.read.parquet("/tmp/file").as[BigDecimal]`.
+          child
+
+        case UpCast(child, target: AtomicType, _)
             if SQLConf.get.getConf(SQLConf.LEGACY_LOOSE_UPCAST) &&
               child.dataType == StringType =>
-          Cast(child, dt.asNullable)
+          Cast(child, target.asNullable)
 
-        case UpCast(child, dataType, walkedTypePath) if !Cast.canUpCast(child.dataType, dataType) =>
-          fail(child, dataType, walkedTypePath)
+        case u @ UpCast(child, _, walkedTypePath) if !Cast.canUpCast(child.dataType, u.dataType) =>
+          fail(child, u.dataType, walkedTypePath)
 
-        case UpCast(child, dataType, _) => Cast(child, dataType.asNullable)
+        case u @ UpCast(child, _, _) => Cast(child, u.dataType.asNullable)
       }
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 9bc86d4fb4e42..ef70915a5c969 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -1742,8 +1742,16 @@ case class AnsiCast(child: Expression, dataType: DataType, timeZoneId: Option[St
 /**
  * Cast the child expression to the target data type, but will throw error if the cast might
  * truncate, e.g. long -> int, timestamp -> data.
+ *
+ * Note: `target` is `AbstractDataType`, so that we can put `object DecimalType`, which means
+ * we accept `DecimalType` with any valid precision/scale.
  */
-case class UpCast(child: Expression, dataType: DataType, walkedTypePath: Seq[String] = Nil)
+case class UpCast(child: Expression, target: AbstractDataType, walkedTypePath: Seq[String] = Nil)
   extends UnaryExpression with Unevaluable {
   override lazy val resolved = false
+
+  def dataType: DataType = target match {
+    case DecimalType => DecimalType.SYSTEM_DEFAULT
+    case _ => target.asInstanceOf[DataType]
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
index 48f4ef5051fb3..577814b9c6696 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/EncoderResolutionSuite.scala
@@ -22,9 +22,9 @@ import scala.reflect.runtime.universe.TypeTag
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.PlanTest
-import org.apache.spark.sql.catalyst.util.GenericArrayData
+import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -247,6 +247,13 @@ class EncoderResolutionSuite extends PlanTest {
        """.stripMargin.trim + " of the field in the target object")
   }
 
+  test("SPARK-31750: eliminate UpCast if child's dataType is DecimalType") {
+    val encoder = ExpressionEncoder[Seq[BigDecimal]]
+    val attr = Seq(AttributeReference("a", ArrayType(DecimalType(38, 0)))())
+    // Before SPARK-31750, it will fail because Decimal(38, 0) can not be casted to Decimal(38, 18)
+    testFromRow(encoder, attr, InternalRow(ArrayData.toArrayData(Array(Decimal(1.0)))))
+  }
+
   // test for leaf types
   castSuccess[Int, Long]
   castSuccess[java.sql.Date, java.sql.Timestamp]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 4e91a7c7bb0f4..954a4bd9331ed 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2439,6 +2439,17 @@ class DataFrameSuite extends QueryTest
     val nestedDecArray = Array(decSpark)
     checkAnswer(Seq(nestedDecArray).toDF(), Row(Array(wrapRefArray(decJava))))
   }
+
+  test("SPARK-31750: eliminate UpCast if child's dataType is DecimalType") {
+    withTempPath { f =>
+      sql("select cast(1 as decimal(38, 0)) as d")
+        .write.mode("overwrite")
+        .parquet(f.getAbsolutePath)
+
+      val df = spark.read.parquet(f.getAbsolutePath).as[BigDecimal]
+      assert(df.schema === new StructType().add(StructField("d", DecimalType(38, 0))))
+    }
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)

From a1391cf56b3d3c98960e126fb1cf7811a967d74c Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 20 May 2020 03:45:13 +0000
Subject: [PATCH 0706/1280] [SPARK-31706][SQL] add back the support of
 streaming update mode

### What changes were proposed in this pull request?

This PR adds a private `WriteBuilder` mixin trait: `SupportsStreamingUpdate`, so that the builtin v2 streaming sinks can still support the update mode.

Note: it's private because we don't have a proper design yet. I didn't take the proposal in https://github.com/apache/spark/pull/23702#discussion_r258593059 because we may want something more general, like updating by an expression `key1 = key2 + 10`.

### Why are the changes needed?

In Spark 2.4, all builtin v2 streaming sinks support all streaming output modes, and v2 sinks are enabled by default, see https://issues.apache.org/jira/browse/SPARK-22911

It's too risky for 3.0 to go back to v1 sinks, so I propose to add a private trait to fix builtin v2 sinks, to keep backward compatibility.

### Does this PR introduce _any_ user-facing change?

Yes, now all the builtin v2 streaming sinks support all streaming output modes, which is the same as 2.4

### How was this patch tested?

existing tests.

Closes #28523 from cloud-fan/update.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 34414acfa36d1a91cd6a64761625c8b0bd90c0a7)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/kafka010/KafkaSourceProvider.scala    |  9 ++++---
 .../connector/SupportsStreamingUpdate.scala   | 26 +++++++++++++++++++
 .../datasources/noop/NoopDataSource.scala     |  6 +++--
 .../execution/streaming/StreamExecution.scala | 12 +++------
 .../sql/execution/streaming/console.scala     |  7 ++---
 .../sources/ForeachWriterTable.scala          |  8 +++---
 .../execution/streaming/sources/memory.scala  |  6 ++++-
 7 files changed, 54 insertions(+), 20 deletions(-)
 create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala

diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
index a5e5d01152db8..ede58bd26ce34 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSourceProvider.scala
@@ -33,10 +33,10 @@ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability, TableProvider}
 import org.apache.spark.sql.connector.read.{Batch, Scan, ScanBuilder}
 import org.apache.spark.sql.connector.read.streaming.{ContinuousStream, MicroBatchStream}
-import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, WriteBuilder}
+import org.apache.spark.sql.connector.write.{BatchWrite, LogicalWriteInfo, SupportsTruncate, WriteBuilder}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 import org.apache.spark.sql.execution.streaming.{Sink, Source}
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdate}
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.sql.types.StructType
@@ -394,7 +394,7 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
       () => new KafkaScan(options)
 
     override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-      new WriteBuilder {
+      new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
         private val options = info.options
         private val inputSchema: StructType = info.schema()
         private val topic = Option(options.get(TOPIC_OPTION_KEY)).map(_.trim)
@@ -410,6 +410,9 @@ private[kafka010] class KafkaSourceProvider extends DataSourceRegister
           assert(inputSchema != null)
           new KafkaStreamingWrite(topic, producerParams, inputSchema)
         }
+
+        override def truncate(): WriteBuilder = this
+        override def update(): WriteBuilder = this
       }
     }
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala
new file mode 100644
index 0000000000000..32be74a345c5a
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/connector/SupportsStreamingUpdate.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.internal.connector
+
+import org.apache.spark.sql.connector.write.WriteBuilder
+
+// An internal `WriteBuilder` mixin to support UPDATE streaming output mode.
+// TODO: design an official API for streaming output mode UPDATE.
+trait SupportsStreamingUpdate extends WriteBuilder {
+  def update(): WriteBuilder
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
index 851cc51466a91..8a6c4dce75f30 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/noop/NoopDataSource.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
 import org.apache.spark.sql.connector.write.{BatchWrite, DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdate}
 import org.apache.spark.sql.sources.DataSourceRegister
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -52,8 +52,10 @@ private[noop] object NoopTable extends Table with SupportsWrite {
   }
 }
 
-private[noop] object NoopWriteBuilder extends WriteBuilder with SupportsTruncate {
+private[noop] object NoopWriteBuilder extends WriteBuilder
+  with SupportsTruncate with SupportsStreamingUpdate {
   override def truncate(): WriteBuilder = this
+  override def update(): WriteBuilder = this
   override def buildForBatch(): BatchWrite = NoopBatchWrite
   override def buildForStreaming(): StreamingWrite = NoopStreamingWrite
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
index 9b1951a834d9a..18fe38caa5e65 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamExecution.scala
@@ -44,6 +44,7 @@ import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.command.StreamingExplainCommand
 import org.apache.spark.sql.execution.datasources.v2.StreamWriterCommitProgress
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
 import org.apache.spark.sql.streaming._
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 import org.apache.spark.util.{Clock, UninterruptibleThread, Utils}
@@ -629,14 +630,9 @@ abstract class StreamExecution(
         writeBuilder.asInstanceOf[SupportsTruncate].truncate().buildForStreaming()
 
       case Update =>
-        // Although no v2 sinks really support Update mode now, but during tests we do want them
-        // to pretend to support Update mode, and treat Update mode same as Append mode.
-        if (Utils.isTesting) {
-          writeBuilder.buildForStreaming()
-        } else {
-          throw new IllegalArgumentException(
-            "Data source v2 streaming sinks does not support Update mode.")
-        }
+        require(writeBuilder.isInstanceOf[SupportsStreamingUpdate],
+          table.name + " does not support Update mode.")
+        writeBuilder.asInstanceOf[SupportsStreamingUpdate].update().buildForStreaming()
     }
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
index e471e6c601d16..1e64021c8105e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/console.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapabi
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, SupportsTruncate, WriteBuilder}
 import org.apache.spark.sql.connector.write.streaming.StreamingWrite
 import org.apache.spark.sql.execution.streaming.sources.ConsoleWrite
-import org.apache.spark.sql.internal.connector.SimpleTableProvider
+import org.apache.spark.sql.internal.connector.{SimpleTableProvider, SupportsStreamingUpdate}
 import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister}
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -73,11 +73,12 @@ object ConsoleTable extends Table with SupportsWrite {
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
       private val inputSchema: StructType = info.schema()
 
-      // Do nothing for truncate. Console sink is special that it just prints all the records.
+      // Do nothing for truncate/update. Console sink is special and it just prints all the records.
       override def truncate(): WriteBuilder = this
+      override def update(): WriteBuilder = this
 
       override def buildForStreaming(): StreamingWrite = {
         assert(inputSchema != null)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
index ba54c85d07303..57a73c740310e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/ForeachWriterTable.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapabi
 import org.apache.spark.sql.connector.write.{DataWriter, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
 import org.apache.spark.sql.execution.python.PythonForeachWriter
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -54,12 +55,13 @@ case class ForeachWriterTable[T](
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
       private var inputSchema: StructType = info.schema()
 
-      // Do nothing for truncate. Foreach sink is special that it just forwards all the records to
-      // ForeachWriter.
+      // Do nothing for truncate/update. Foreach sink is special and it just forwards all the
+      // records to ForeachWriter.
       override def truncate(): WriteBuilder = this
+      override def update(): WriteBuilder = this
 
       override def buildForStreaming(): StreamingWrite = {
         new StreamingWrite {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
index deab42bea36ad..03ebbb9f1b376 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/sources/memory.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapabi
 import org.apache.spark.sql.connector.write.{DataWriter, DataWriterFactory, LogicalWriteInfo, PhysicalWriteInfo, SupportsTruncate, WriteBuilder, WriterCommitMessage}
 import org.apache.spark.sql.connector.write.streaming.{StreamingDataWriterFactory, StreamingWrite}
 import org.apache.spark.sql.execution.streaming.Sink
+import org.apache.spark.sql.internal.connector.SupportsStreamingUpdate
 import org.apache.spark.sql.types.StructType
 
 /**
@@ -53,7 +54,7 @@ class MemorySink extends Table with SupportsWrite with Logging {
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
-    new WriteBuilder with SupportsTruncate {
+    new WriteBuilder with SupportsTruncate with SupportsStreamingUpdate {
       private var needTruncate: Boolean = false
       private val inputSchema: StructType = info.schema()
 
@@ -62,6 +63,9 @@ class MemorySink extends Table with SupportsWrite with Logging {
         this
       }
 
+      // The in-memory sink treats update as append.
+      override def update(): WriteBuilder = this
+
       override def buildForStreaming(): StreamingWrite = {
         new MemoryStreamingWrite(MemorySink.this, inputSchema, needTruncate)
       }

From 57797d9116ad5cb5764f84fc190eb40409280aaf Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 20 May 2020 13:37:49 +0900
Subject: [PATCH 0707/1280] [SPARK-31748][PYTHON][3.0] Document resource module
 in PySpark doc and rename/move classes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

This PR partially backports https://github.com/apache/spark/pull/28569.

1.. Rename

```
pyspark
└── resourceinformation.py
    └── class ResourceInformation
```

to

```
pyspark
└── resource.py
    └── class ResourceInformation
```

So, the `ResourceInformation` is consistently imported via `pyspark.resource.ResourceInformation`.

2.. Document the new `pyspark.resource` module

3.. Minor docstring fix e.g.:

```diff
-     param name the name of the resource
-     param addresses an array of strings describing the addresses of the resource
+     :param name: the name of the resource
+     :param addresses: an array of strings describing the addresses of the resource
+
+     .. versionadded:: 3.0.0
```

### Why are the changes needed?

To document APIs, and move Python modules to fewer and simpler modules.

### Does this PR introduce _any_ user-facing change?

No, the changes are in unreleased branches.

### How was this patch tested?

Manually tested via:

```bash
cd python
./run-tests --python-executables=python3 --modules=pyspark-core
```

Closes #28586 from HyukjinKwon/SPARK-31748.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/docs/index.rst                                 |  1 +
 python/docs/pyspark.resource.rst                      | 11 +++++++++++
 python/docs/pyspark.rst                               |  1 +
 python/docs/pyspark.sql.rst                           |  4 ++--
 python/pyspark/__init__.py                            |  2 +-
 python/pyspark/context.py                             |  2 +-
 .../pyspark/{resourceinformation.py => resource.py}   |  6 ++++--
 python/pyspark/worker.py                              |  3 ++-
 8 files changed, 23 insertions(+), 7 deletions(-)
 create mode 100644 python/docs/pyspark.resource.rst
 rename python/pyspark/{resourceinformation.py => resource.py} (89%)

diff --git a/python/docs/index.rst b/python/docs/index.rst
index 0e7b62361802a..6e059264e6bbb 100644
--- a/python/docs/index.rst
+++ b/python/docs/index.rst
@@ -16,6 +16,7 @@ Contents:
    pyspark.streaming
    pyspark.ml
    pyspark.mllib
+   pyspark.resource
 
 
 Core classes:
diff --git a/python/docs/pyspark.resource.rst b/python/docs/pyspark.resource.rst
new file mode 100644
index 0000000000000..7f3a79b9e5b52
--- /dev/null
+++ b/python/docs/pyspark.resource.rst
@@ -0,0 +1,11 @@
+pyspark.resource module
+=======================
+
+Module Contents
+---------------
+
+.. automodule:: pyspark.resource
+    :members:
+    :undoc-members:
+    :inherited-members:
+
diff --git a/python/docs/pyspark.rst b/python/docs/pyspark.rst
index 0df12c49ad033..402d6ce9eb016 100644
--- a/python/docs/pyspark.rst
+++ b/python/docs/pyspark.rst
@@ -11,6 +11,7 @@ Subpackages
     pyspark.streaming
     pyspark.ml
     pyspark.mllib
+    pyspark.resource
 
 Contents
 --------
diff --git a/python/docs/pyspark.sql.rst b/python/docs/pyspark.sql.rst
index b69562e845920..406ada701941a 100644
--- a/python/docs/pyspark.sql.rst
+++ b/python/docs/pyspark.sql.rst
@@ -1,8 +1,8 @@
 pyspark.sql module
 ==================
 
-Module Context
---------------
+Module Contents
+---------------
 
 .. automodule:: pyspark.sql
     :members:
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index 70c0b27a6aa33..b485c3fd0f648 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -54,7 +54,7 @@
 from pyspark.storagelevel import StorageLevel
 from pyspark.accumulators import Accumulator, AccumulatorParam
 from pyspark.broadcast import Broadcast
-from pyspark.resourceinformation import ResourceInformation
+from pyspark.resource import ResourceInformation
 from pyspark.serializers import MarshalSerializer, PickleSerializer
 from pyspark.status import *
 from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 6cc343e3e495c..d5f1506e4a555 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -35,7 +35,7 @@
 from pyspark.serializers import PickleSerializer, BatchedSerializer, UTF8Deserializer, \
     PairDeserializer, AutoBatchedSerializer, NoOpSerializer, ChunkedStream
 from pyspark.storagelevel import StorageLevel
-from pyspark.resourceinformation import ResourceInformation
+from pyspark.resource import ResourceInformation
 from pyspark.rdd import RDD, _load_from_socket, ignore_unicode_prefix
 from pyspark.traceback_utils import CallSite, first_spark_call
 from pyspark.status import StatusTracker
diff --git a/python/pyspark/resourceinformation.py b/python/pyspark/resource.py
similarity index 89%
rename from python/pyspark/resourceinformation.py
rename to python/pyspark/resource.py
index aaed21374b6ee..b0e41cced85b5 100644
--- a/python/pyspark/resourceinformation.py
+++ b/python/pyspark/resource.py
@@ -26,8 +26,10 @@ class ResourceInformation(object):
 
     One example is GPUs, where the addresses would be the indices of the GPUs
 
-    @param name the name of the resource
-    @param addresses an array of strings describing the addresses of the resource
+    :param name: the name of the resource
+    :param addresses: an array of strings describing the addresses of the resource
+
+    .. versionadded:: 3.0.0
     """
 
     def __init__(self, name, addresses):
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index a9a32ed150c0b..5f4a8a2d2db1f 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -19,6 +19,7 @@
 Worker that receives input from Piped RDD.
 """
 from __future__ import print_function
+from __future__ import absolute_import
 import os
 import sys
 import time
@@ -35,7 +36,7 @@
 from pyspark.java_gateway import local_connect_and_auth
 from pyspark.taskcontext import BarrierTaskContext, TaskContext
 from pyspark.files import SparkFiles
-from pyspark.resourceinformation import ResourceInformation
+from pyspark.resource import ResourceInformation
 from pyspark.rdd import PythonEvalType
 from pyspark.serializers import write_with_length, write_int, read_long, read_bool, \
     write_long, read_int, SpecialLengths, UTF8Deserializer, PickleSerializer, \

From e3996f4a8217244e276650b491ecbfd4611cfce8 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 19 May 2020 22:36:36 -0700
Subject: [PATCH 0708/1280] [SPARK-31767][PYTHON][CORE] Remove
 ResourceInformation in pyspark module's namespace

This PR proposes to only allow the import of `ResourceInformation` as below:

```
pyspark.resource.ResourceInformation
```

instead of

```
pyspark.ResourceInformation
pyspark.resource.ResourceInformation
```

because `pyspark.resource` is a separate module, and it is documented so.
The constructor of `ResourceInformation` isn't supposed to directly call anyway.

To keep the code structure coherent.

No, it will be in the unreleased branches.

Manually tested via importing:

Before:

```python
>>> import pyspark
>>> pyspark.ResourceInformation
<class 'pyspark.resource.information.ResourceInformation'>
>>> pyspark.resource.ResourceInformation
<class 'pyspark.resource.information.ResourceInformation'>
```

After:

```python
>>> import pyspark
>>> pyspark.ResourceInformation
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: module 'pyspark' has no attribute 'ResourceInformation'
>>> pyspark.resource.ResourceInformation
<class 'pyspark.resource.information.ResourceInformation'>
```

Also tested via

```bash
cd python
./run-tests --python-executables=python3 --modules=pyspark-core,pyspark-resource
```

Jenkins will test and existing tests should cover.

Closes #28589 from HyukjinKwon/SPARK-31767.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit dc3a606fbd37ede404a61551ff792fa94fe1780f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 python/pyspark/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index b485c3fd0f648..ee153af18c88c 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -54,7 +54,6 @@
 from pyspark.storagelevel import StorageLevel
 from pyspark.accumulators import Accumulator, AccumulatorParam
 from pyspark.broadcast import Broadcast
-from pyspark.resource import ResourceInformation
 from pyspark.serializers import MarshalSerializer, PickleSerializer
 from pyspark.status import *
 from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
@@ -119,5 +118,5 @@ def wrapper(self, *args, **kwargs):
     "SparkConf", "SparkContext", "SparkFiles", "RDD", "StorageLevel", "Broadcast",
     "Accumulator", "AccumulatorParam", "MarshalSerializer", "PickleSerializer",
     "StatusTracker", "SparkJobInfo", "SparkStageInfo", "Profiler", "BasicProfiler", "TaskContext",
-    "RDDBarrier", "BarrierTaskContext", "BarrierTaskInfo", "ResourceInformation",
+    "RDDBarrier", "BarrierTaskContext", "BarrierTaskInfo",
 ]

From 0a20d322cc4d53bcc47cc3feb8b6cf3985f856dc Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 20 May 2020 14:55:23 +0900
Subject: [PATCH 0709/1280] [SPARK-31766][K8S][TESTS] Add Spark version prefix
 to K8s UUID test image tag

### What changes were proposed in this pull request?

This PR aims to add Spark version prefix during generating test image tag for K8s integration testing.

### Why are the changes needed?

This helps to distinguish the images by version.

**BEFORE**
```
$ docker images | grep kubespark
kubespark/spark-py  F7188CBD-AE08-4705-9C8A-D0DD3DC8B86F  ...
kubespark/spark     F7188CBD-AE08-4705-9C8A-D0DD3DC8B86F  ...
```

**AFTER**
```
$ docker images | grep kubespark
kubespark/spark-py  3.1.0-SNAPSHOT_F7188CBD-AE08-4705-9C8A-D0DD3DC8B86F ...
kubespark/spark     3.1.0-SNAPSHOT_F7188CBD-AE08-4705-9C8A-D0DD3DC8B86F ...
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the K8s integration test.

```
...
Successfully tagged kubespark/spark:3.1.0-SNAPSHOT_688b46c8-c119-404d-aadb-d05a14262db7
...
Successfully tagged kubespark/spark-py:3.1.0-SNAPSHOT_688b46c8-c119-404d-aadb-d05a14262db7
...
Successfully tagged kubespark/spark-r:3.1.0-SNAPSHOT_688b46c8-c119-404d-aadb-d05a14262db7
```

Closes #28587 from dongjoon-hyun/SPARK-31766.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b7947e0285f61ac2d8cc3f93c75771a64ad8e1cd)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scripts/setup-integration-test-env.sh                  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index ab906604fce06..2e72ff8e55bfa 100755
--- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -25,6 +25,7 @@ IMAGE_REPO="docker.io/kubespark"
 IMAGE_TAG="N/A"
 JAVA_IMAGE_TAG="8-jre-slim"
 SPARK_TGZ="N/A"
+MVN="$TEST_ROOT_DIR/build/mvn"
 
 # Parse arguments
 while (( "$#" )); do
@@ -84,7 +85,11 @@ fi
 # If there is a specific Spark image skip building and extraction/copy
 if [[ $IMAGE_TAG == "N/A" ]];
 then
-  IMAGE_TAG=$(uuidgen);
+  VERSION=$("$MVN" help:evaluate -Dexpression=project.version \
+    | grep -v "INFO"\
+    | grep -v "WARNING"\
+    | tail -n 1)
+  IMAGE_TAG=${VERSION}_$(uuidgen)
   cd $SPARK_INPUT_DIR
 
   # OpenJDK base-image tag (e.g. 8-jre-slim, 11-jre-slim)

From 4e953394725fc1adcd620f25c6db4096ce3dfe98 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Wed, 20 May 2020 12:20:45 +0000
Subject: [PATCH 0710/1280] Revert [SPARK-27142][SPARK-31440] SQL rest API in
 branch 3.0

### What changes were proposed in this pull request?

Revert https://github.com/apache/spark/pull/28208 and https://github.com/apache/spark/pull/24076 in branch 3.0
### Why are the changes needed?

Unfortunately, the PR  https://github.com/apache/spark/pull/28208 is merged after Spark 3.0 RC 2 cut. Although the improvement is great, we can't break the policy to add new improvement commits into branch 3.0 now.

Also, if we are going to adopt the improvement in a future release, we should not release 3.0 with https://github.com/apache/spark/pull/24076, since the API result will be changed.

After discuss with cloud-fan and gatorsmile offline, we think the best choice is to revert both commits and follow community release policy.

### Does this PR introduce _any_ user-facing change?

Yes, let's hold the SQL rest API until next release.

### How was this patch tested?

Jenkins unit tests.

Closes #28588 from gengliangwang/revertSQLRestAPI.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/ui/SQLAppStatusStore.scala  |   4 -
 .../sql/execution/ui/SparkPlanGraph.scala     |   6 +-
 .../api/v1/sql/ApiSqlRootResource.scala       |  29 ---
 .../spark/status/api/v1/sql/SqlResource.scala | 157 --------------
 .../apache/spark/status/api/v1/sql/api.scala  |  42 ----
 .../status/api/v1/sql/SqlResourceSuite.scala  | 205 ------------------
 6 files changed, 3 insertions(+), 440 deletions(-)
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
 delete mode 100644 sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala
index a90f37a80d525..c6e7f3978469d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SQLAppStatusStore.scala
@@ -42,10 +42,6 @@ class SQLAppStatusStore(
     store.view(classOf[SQLExecutionUIData]).asScala.toSeq
   }
 
-  def executionsList(offset: Int, length: Int): Seq[SQLExecutionUIData] = {
-    store.view(classOf[SQLExecutionUIData]).skip(offset).max(length).asScala.toSeq
-  }
-
   def execution(executionId: Long): Option[SQLExecutionUIData] = {
     try {
       Some(store.read(classOf[SQLExecutionUIData], executionId))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
index a798fe02700e3..274a5a414ffa2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/SparkPlanGraph.scala
@@ -153,7 +153,7 @@ object SparkPlanGraph {
  * @param name the name of this SparkPlan node
  * @param metrics metrics that this SparkPlan node will track
  */
-class SparkPlanGraphNode(
+private[ui] class SparkPlanGraphNode(
     val id: Long,
     val name: String,
     val desc: String,
@@ -193,7 +193,7 @@ class SparkPlanGraphNode(
 /**
  * Represent a tree of SparkPlan for WholeStageCodegen.
  */
-class SparkPlanGraphCluster(
+private[ui] class SparkPlanGraphCluster(
     id: Long,
     name: String,
     desc: String,
@@ -229,7 +229,7 @@ class SparkPlanGraphCluster(
  * Represent an edge in the SparkPlan tree. `fromId` is the child node id, and `toId` is the parent
  * node id.
  */
-case class SparkPlanGraphEdge(fromId: Long, toId: Long) {
+private[ui] case class SparkPlanGraphEdge(fromId: Long, toId: Long) {
 
   def makeDotEdge: String = s"""  $fromId->$toId;\n"""
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
deleted file mode 100644
index 5fc7123c9097b..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/ApiSqlRootResource.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.status.api.v1.sql
-
-import javax.ws.rs.Path
-
-import org.apache.spark.status.api.v1.ApiRequestContext
-
-@Path("/v1")
-private[v1] class ApiSqlRootResource extends ApiRequestContext {
-
-  @Path("applications/{appId}/sql")
-  def sqlList(): Class[SqlResource] = classOf[SqlResource]
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
deleted file mode 100644
index c7599f864dd97..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/SqlResource.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.status.api.v1.sql
-
-import java.util.Date
-import javax.ws.rs._
-import javax.ws.rs.core.MediaType
-
-import scala.util.{Failure, Success, Try}
-
-import org.apache.spark.JobExecutionStatus
-import org.apache.spark.sql.execution.ui.{SparkPlanGraph, SparkPlanGraphCluster, SparkPlanGraphNode, SQLAppStatusStore, SQLExecutionUIData}
-import org.apache.spark.status.api.v1.{BaseAppResource, NotFoundException}
-
-@Produces(Array(MediaType.APPLICATION_JSON))
-private[v1] class SqlResource extends BaseAppResource {
-
-  val WHOLE_STAGE_CODEGEN = "WholeStageCodegen"
-
-  @GET
-  def sqlList(
-      @DefaultValue("true") @QueryParam("details") details: Boolean,
-      @DefaultValue("true") @QueryParam("planDescription") planDescription: Boolean,
-      @DefaultValue("0") @QueryParam("offset") offset: Int,
-      @DefaultValue("20") @QueryParam("length") length: Int): Seq[ExecutionData] = {
-    withUI { ui =>
-      val sqlStore = new SQLAppStatusStore(ui.store.store)
-      sqlStore.executionsList(offset, length).map { exec =>
-        val graph = sqlStore.planGraph(exec.executionId)
-        prepareExecutionData(exec, graph, details, planDescription)
-      }
-    }
-  }
-
-  @GET
-  @Path("{executionId:\\d+}")
-  def sql(
-      @PathParam("executionId") execId: Long,
-      @DefaultValue("true") @QueryParam("details") details: Boolean,
-      @DefaultValue("true") @QueryParam("planDescription")
-      planDescription: Boolean): ExecutionData = {
-    withUI { ui =>
-      val sqlStore = new SQLAppStatusStore(ui.store.store)
-      val graph = sqlStore.planGraph(execId)
-      sqlStore
-        .execution(execId)
-        .map(prepareExecutionData(_, graph, details, planDescription))
-        .getOrElse(throw new NotFoundException("unknown query execution id: " + execId))
-    }
-  }
-
-  private def prepareExecutionData(
-    exec: SQLExecutionUIData,
-    graph: SparkPlanGraph,
-    details: Boolean,
-    planDescription: Boolean): ExecutionData = {
-
-    var running = Seq[Int]()
-    var completed = Seq[Int]()
-    var failed = Seq[Int]()
-
-    exec.jobs.foreach {
-      case (id, JobExecutionStatus.RUNNING) =>
-        running = running :+ id
-      case (id, JobExecutionStatus.SUCCEEDED) =>
-        completed = completed :+ id
-      case (id, JobExecutionStatus.FAILED) =>
-        failed = failed :+ id
-      case _ =>
-    }
-
-    val status = if (exec.jobs.size == completed.size) {
-      "COMPLETED"
-    } else if (failed.nonEmpty) {
-      "FAILED"
-    } else {
-      "RUNNING"
-    }
-
-    val duration = exec.completionTime.getOrElse(new Date()).getTime - exec.submissionTime
-    val planDetails = if (planDescription) exec.physicalPlanDescription else ""
-    val nodes = if (details) printableMetrics(graph.allNodes, exec.metricValues) else Seq.empty
-    val edges = if (details) graph.edges else Seq.empty
-
-    new ExecutionData(
-      exec.executionId,
-      status,
-      exec.description,
-      planDetails,
-      new Date(exec.submissionTime),
-      duration,
-      running,
-      completed,
-      failed,
-      nodes,
-      edges)
-  }
-
-  private def printableMetrics(allNodes: Seq[SparkPlanGraphNode],
-    metricValues: Map[Long, String]): Seq[Node] = {
-
-    def getMetric(metricValues: Map[Long, String], accumulatorId: Long,
-      metricName: String): Option[Metric] = {
-
-      metricValues.get(accumulatorId).map( mv => {
-        val metricValue = if (mv.startsWith("\n")) mv.substring(1, mv.length) else mv
-        Metric(metricName, metricValue)
-      })
-    }
-
-    val nodeIdAndWSCGIdMap = getNodeIdAndWSCGIdMap(allNodes)
-    val nodes = allNodes.map { node =>
-      val wholeStageCodegenId = nodeIdAndWSCGIdMap.get(node.id).flatten
-      val metrics =
-        node.metrics.flatMap(m => getMetric(metricValues, m.accumulatorId, m.name.trim))
-      Node(nodeId = node.id, nodeName = node.name.trim, wholeStageCodegenId, metrics)
-    }
-
-    nodes.sortBy(_.nodeId).reverse
-  }
-
-  private def getNodeIdAndWSCGIdMap(allNodes: Seq[SparkPlanGraphNode]): Map[Long, Option[Long]] = {
-    val wscgNodes = allNodes.filter(_.name.trim.startsWith(WHOLE_STAGE_CODEGEN))
-    val nodeIdAndWSCGIdMap: Map[Long, Option[Long]] = wscgNodes.flatMap {
-      _ match {
-        case x: SparkPlanGraphCluster => x.nodes.map(_.id -> getWholeStageCodegenId(x.name.trim))
-        case _ => Seq.empty
-      }
-    }.toMap
-
-    nodeIdAndWSCGIdMap
-  }
-
-  private def getWholeStageCodegenId(wscgNodeName: String): Option[Long] = {
-    Try(wscgNodeName.substring(
-      s"$WHOLE_STAGE_CODEGEN (".length, wscgNodeName.length - 1).toLong) match {
-      case Success(wscgId) => Some(wscgId)
-      case Failure(t) => None
-    }
-  }
-
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala b/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
deleted file mode 100644
index 0ddf66718bce7..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/status/api/v1/sql/api.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.status.api.v1.sql
-
-import java.util.Date
-
-import org.apache.spark.sql.execution.ui.SparkPlanGraphEdge
-
-class ExecutionData private[spark] (
-    val id: Long,
-    val status: String,
-    val description: String,
-    val planDescription: String,
-    val submissionTime: Date,
-    val duration: Long,
-    val runningJobIds: Seq[Int],
-    val successJobIds: Seq[Int],
-    val failedJobIds: Seq[Int],
-    val nodes: Seq[Node],
-    val edges: Seq[SparkPlanGraphEdge])
-
-case class Node private[spark](
-    nodeId: Long,
-    nodeName: String,
-    wholeStageCodegenId: Option[Long] = None,
-    metrics: Seq[Metric])
-
-case class Metric private[spark] (name: String, value: String)
diff --git a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
deleted file mode 100644
index 43cca246cc47c..0000000000000
--- a/sql/core/src/test/scala/org/apache/spark/status/api/v1/sql/SqlResourceSuite.scala
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.status.api.v1.sql
-
-import java.util.Date
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.scalatest.PrivateMethodTester
-
-import org.apache.spark.{JobExecutionStatus, SparkFunSuite}
-import org.apache.spark.sql.execution.ui.{SparkPlanGraph, SparkPlanGraphCluster, SparkPlanGraphEdge, SparkPlanGraphNode, SQLExecutionUIData, SQLPlanMetric}
-
-object SqlResourceSuite {
-
-  val SCAN_TEXT = "Scan text"
-  val FILTER = "Filter"
-  val WHOLE_STAGE_CODEGEN_1 = "WholeStageCodegen (1)"
-  val DURATION = "duration"
-  val NUMBER_OF_OUTPUT_ROWS = "number of output rows"
-  val METADATA_TIME = "metadata time"
-  val NUMBER_OF_FILES_READ = "number of files read"
-  val SIZE_OF_FILES_READ = "size of files read"
-  val PLAN_DESCRIPTION = "== Physical Plan ==\nCollectLimit (3)\n+- * Filter (2)\n +- Scan text..."
-  val DESCRIPTION = "csv at MyDataFrames.scala:57"
-
-  val nodeIdAndWSCGIdMap: Map[Long, Option[Long]] = Map(1L -> Some(1L))
-
-  val filterNode = new SparkPlanGraphNode(1, FILTER, "",
-    metrics = Seq(SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 1, "")))
-  val nodes: Seq[SparkPlanGraphNode] = Seq(
-    new SparkPlanGraphCluster(0, WHOLE_STAGE_CODEGEN_1, "",
-      nodes = ArrayBuffer(filterNode),
-      metrics = Seq(SQLPlanMetric(DURATION, 0, ""))),
-    new SparkPlanGraphNode(2, SCAN_TEXT, "",
-      metrics = Seq(
-      SQLPlanMetric(METADATA_TIME, 2, ""),
-      SQLPlanMetric(NUMBER_OF_FILES_READ, 3, ""),
-      SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 4, ""),
-      SQLPlanMetric(SIZE_OF_FILES_READ, 5, ""))))
-
-  val nodesWhenCodegenIsOff: Seq[SparkPlanGraphNode] =
-    SparkPlanGraph(nodes, edges).allNodes.filterNot(_.name == WHOLE_STAGE_CODEGEN_1)
-
-  val edges: Seq[SparkPlanGraphEdge] =
-    Seq(SparkPlanGraphEdge(3, 2))
-
-  val metrics: Seq[SQLPlanMetric] = {
-    Seq(SQLPlanMetric(DURATION, 0, ""),
-      SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 1, ""),
-      SQLPlanMetric(METADATA_TIME, 2, ""),
-      SQLPlanMetric(NUMBER_OF_FILES_READ, 3, ""),
-      SQLPlanMetric(NUMBER_OF_OUTPUT_ROWS, 4, ""),
-      SQLPlanMetric(SIZE_OF_FILES_READ, 5, ""))
-  }
-
-  val sqlExecutionUIData: SQLExecutionUIData = {
-    def getMetricValues() = {
-      Map[Long, String](
-        0L -> "0 ms",
-        1L -> "1",
-        2L -> "2 ms",
-        3L -> "1",
-        4L -> "1",
-        5L -> "330.0 B"
-      )
-    }
-
-    new SQLExecutionUIData(
-      executionId = 0,
-      description = DESCRIPTION,
-      details = "",
-      physicalPlanDescription = PLAN_DESCRIPTION,
-      metrics = metrics,
-      submissionTime = 1586768888233L,
-      completionTime = Some(new Date(1586768888999L)),
-      jobs = Map[Int, JobExecutionStatus](
-        0 -> JobExecutionStatus.SUCCEEDED,
-        1 -> JobExecutionStatus.SUCCEEDED),
-      stages = Set[Int](),
-      metricValues = getMetricValues()
-    )
-  }
-
-  private def getNodes(): Seq[Node] = {
-    val node = Node(0, WHOLE_STAGE_CODEGEN_1,
-      wholeStageCodegenId = None, metrics = Seq(Metric(DURATION, "0 ms")))
-    val node2 = Node(1, FILTER,
-      wholeStageCodegenId = Some(1), metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
-    val node3 = Node(2, SCAN_TEXT, wholeStageCodegenId = None,
-      metrics = Seq(Metric(METADATA_TIME, "2 ms"),
-        Metric(NUMBER_OF_FILES_READ, "1"),
-        Metric(NUMBER_OF_OUTPUT_ROWS, "1"),
-        Metric(SIZE_OF_FILES_READ, "330.0 B")))
-
-    // reverse order because of supporting execution order by aligning with Spark-UI
-    Seq(node3, node2, node)
-  }
-
-  private def getExpectedNodesWhenWholeStageCodegenIsOff(): Seq[Node] = {
-    val node = Node(1, FILTER, metrics = Seq(Metric(NUMBER_OF_OUTPUT_ROWS, "1")))
-    val node2 = Node(2, SCAN_TEXT,
-      metrics = Seq(Metric(METADATA_TIME, "2 ms"),
-        Metric(NUMBER_OF_FILES_READ, "1"),
-        Metric(NUMBER_OF_OUTPUT_ROWS, "1"),
-        Metric(SIZE_OF_FILES_READ, "330.0 B")))
-
-    // reverse order because of supporting execution order by aligning with Spark-UI
-    Seq(node2, node)
-  }
-
-  private def verifyExpectedExecutionData(executionData: ExecutionData,
-    nodes: Seq[Node],
-    edges: Seq[SparkPlanGraphEdge],
-    planDescription: String): Unit = {
-
-    assert(executionData.id == 0)
-    assert(executionData.status == "COMPLETED")
-    assert(executionData.description == DESCRIPTION)
-    assert(executionData.planDescription == planDescription)
-    assert(executionData.submissionTime == new Date(1586768888233L))
-    assert(executionData.duration == 766L)
-    assert(executionData.successJobIds == Seq[Int](0, 1))
-    assert(executionData.runningJobIds == Seq[Int]())
-    assert(executionData.failedJobIds == Seq.empty)
-    assert(executionData.nodes == nodes)
-    assert(executionData.edges == edges)
-  }
-
-}
-
-/**
- * Sql Resource Public API Unit Tests.
- */
-class SqlResourceSuite extends SparkFunSuite with PrivateMethodTester {
-
-  import SqlResourceSuite._
-
-  val sqlResource = new SqlResource()
-  val prepareExecutionData = PrivateMethod[ExecutionData]('prepareExecutionData)
-
-  test("Prepare ExecutionData when details = false and planDescription = false") {
-    val executionData =
-      sqlResource invokePrivate prepareExecutionData(
-        sqlExecutionUIData, SparkPlanGraph(Seq.empty, Seq.empty), false, false)
-    verifyExpectedExecutionData(executionData, edges = Seq.empty,
-      nodes = Seq.empty, planDescription = "")
-  }
-
-  test("Prepare ExecutionData when details = true and planDescription = false") {
-    val executionData =
-      sqlResource invokePrivate prepareExecutionData(
-        sqlExecutionUIData, SparkPlanGraph(nodes, edges), true, false)
-    verifyExpectedExecutionData(
-      executionData,
-      nodes = getNodes(),
-      edges,
-      planDescription = "")
-  }
-
-  test("Prepare ExecutionData when details = true and planDescription = true") {
-    val executionData =
-      sqlResource invokePrivate prepareExecutionData(
-        sqlExecutionUIData, SparkPlanGraph(nodes, edges), true, true)
-    verifyExpectedExecutionData(
-      executionData,
-      nodes = getNodes(),
-      edges = edges,
-      planDescription = PLAN_DESCRIPTION)
-  }
-
-  test("Prepare ExecutionData when details = true and planDescription = false and WSCG = off") {
-    val executionData =
-      sqlResource invokePrivate prepareExecutionData(
-        sqlExecutionUIData, SparkPlanGraph(nodesWhenCodegenIsOff, edges), true, false)
-    verifyExpectedExecutionData(
-      executionData,
-      nodes = getExpectedNodesWhenWholeStageCodegenIsOff(),
-      edges = edges,
-      planDescription = "")
-  }
-
-  test("Parse wholeStageCodegenId from nodeName") {
-    val getWholeStageCodegenId = PrivateMethod[Option[Long]]('getWholeStageCodegenId)
-    val wholeStageCodegenId =
-      sqlResource invokePrivate getWholeStageCodegenId(WHOLE_STAGE_CODEGEN_1)
-    assert(wholeStageCodegenId == Some(1))
-  }
-
-}

From 5198b6853b2e3bc69fc013c653aa163c79168366 Mon Sep 17 00:00:00 2001
From: Ali Smesseim <ali.smesseim@databricks.com>
Date: Wed, 20 May 2020 10:30:17 -0700
Subject: [PATCH 0711/1280] [SPARK-31387][SQL] Handle unknown operation/session
 ID in HiveThriftServer2Listener

### What changes were proposed in this pull request?

This is a recreation of #28155, which was reverted due to causing test failures.

The update methods in HiveThriftServer2Listener now check if the parameter operation/session ID actually exist in the `sessionList` and `executionList` respectively. This prevents NullPointerExceptions if the operation or session ID is unknown. Instead, a warning is written to the log.

To improve robustness, we also make the following changes in HiveSessionImpl.close():

- Catch any exception thrown by `operationManager.closeOperation`. If for any reason this throws an exception, other operations are not prevented from being closed.
- Handle not being able to access the scratch directory. When closing, all `.pipeout` files are removed from the scratch directory, which would have resulted in an NPE if the directory does not exist.

### Why are the changes needed?

The listener's update methods would throw an exception if the operation or session ID is unknown. In Spark 2, where the listener is called directly, this changes the caller's control flow. In Spark 3, the exception is caught by the ListenerBus but results in an uninformative NullPointerException.

In HiveSessionImpl.close(), if an exception is thrown when closing an operation, all following operations are not closed.

### Does this PR introduce any user-facing change?

No

### How was this patch tested?

Unit tests

Closes #28544 from alismess-db/hive-thriftserver-listener-update-safer-2.

Authored-by: Ali Smesseim <ali.smesseim@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit d40ecfa3f7b0102063075cafd54451f082ce38e1)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../ui/HiveThriftServer2Listener.scala        | 106 +++++++++++-------
 .../thriftserver/HiveSessionImplSuite.scala   |  73 ++++++++++++
 .../ui/HiveThriftServer2ListenerSuite.scala   |  17 +++
 .../service/cli/session/HiveSessionImpl.java  |  20 +++-
 .../service/cli/session/HiveSessionImpl.java  |  20 +++-
 5 files changed, 183 insertions(+), 53 deletions(-)
 create mode 100644 sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
index 6d0a506fa94dc..6b7e5ee611417 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
@@ -25,6 +25,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.hive.service.server.HiveServer2
 
 import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Status.LIVE_ENTITY_UPDATE_PERIOD
 import org.apache.spark.scheduler._
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.ExecutionState
@@ -38,7 +39,7 @@ private[thriftserver] class HiveThriftServer2Listener(
     kvstore: ElementTrackingStore,
     sparkConf: SparkConf,
     server: Option[HiveServer2],
-    live: Boolean = true) extends SparkListener {
+    live: Boolean = true) extends SparkListener with Logging {
 
   private val sessionList = new ConcurrentHashMap[String, LiveSessionData]()
   private val executionList = new ConcurrentHashMap[String, LiveExecutionData]()
@@ -131,60 +132,83 @@ private[thriftserver] class HiveThriftServer2Listener(
     updateLiveStore(session)
   }
 
-  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit = {
-    val session = sessionList.get(e.sessionId)
-    session.finishTimestamp = e.finishTime
-    updateStoreWithTriggerEnabled(session)
-    sessionList.remove(e.sessionId)
-  }
+  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit =
+    Option(sessionList.get(e.sessionId)) match {
+      case Some(sessionData) =>
+        sessionData.finishTimestamp = e.finishTime
+        updateStoreWithTriggerEnabled(sessionData)
+        sessionList.remove(e.sessionId)
+      case None => logWarning(s"onSessionClosed called with unknown session id: ${e.sessionId}")
+    }
 
   private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit = {
-    val info = getOrCreateExecution(
+    val executionData = getOrCreateExecution(
       e.id,
       e.statement,
       e.sessionId,
       e.startTime,
       e.userName)
 
-    info.state = ExecutionState.STARTED
-    executionList.put(e.id, info)
-    sessionList.get(e.sessionId).totalExecution += 1
-    executionList.get(e.id).groupId = e.groupId
-    updateLiveStore(executionList.get(e.id))
-    updateLiveStore(sessionList.get(e.sessionId))
+    executionData.state = ExecutionState.STARTED
+    executionList.put(e.id, executionData)
+    executionData.groupId = e.groupId
+    updateLiveStore(executionData)
+
+    Option(sessionList.get(e.sessionId)) match {
+      case Some(sessionData) =>
+        sessionData.totalExecution += 1
+        updateLiveStore(sessionData)
+      case None => logWarning(s"onOperationStart called with unknown session id: ${e.sessionId}." +
+        s"Regardless, the operation has been registered.")
+    }
   }
 
-  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit = {
-    executionList.get(e.id).executePlan = e.executionPlan
-    executionList.get(e.id).state = ExecutionState.COMPILED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.executePlan = e.executionPlan
+        executionData.state = ExecutionState.COMPILED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationParsed called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).state = ExecutionState.CANCELED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.CANCELED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationCanceled called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).detail = e.errorMsg
-    executionList.get(e.id).state = ExecutionState.FAILED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.detail = e.errorMsg
+        executionData.state = ExecutionState.FAILED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationError called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit = {
-    executionList.get(e.id).finishTimestamp = e.finishTime
-    executionList.get(e.id).state = ExecutionState.FINISHED
-    updateLiveStore(executionList.get(e.id))
-  }
+  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.finishTimestamp = e.finishTime
+        executionData.state = ExecutionState.FINISHED
+        updateLiveStore(executionData)
+      case None => logWarning(s"onOperationFinished called with unknown operation id: ${e.id}")
+    }
 
-  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit = {
-    executionList.get(e.id).closeTimestamp = e.closeTime
-    executionList.get(e.id).state = ExecutionState.CLOSED
-    updateStoreWithTriggerEnabled(executionList.get(e.id))
-    executionList.remove(e.id)
-  }
+  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit =
+    Option(executionList.get(e.id)) match {
+      case Some(executionData) =>
+        executionData.closeTimestamp = e.closeTime
+        executionData.state = ExecutionState.CLOSED
+        updateStoreWithTriggerEnabled(executionData)
+        executionList.remove(e.id)
+      case None => logWarning(s"onOperationClosed called with unknown operation id: ${e.id}")
+    }
 
   // Update both live and history stores. Trigger is enabled by default, hence
   // it will cleanup the entity which exceeds the threshold.
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
new file mode 100644
index 0000000000000..05d540d782e31
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive.thriftserver
+
+import scala.collection.JavaConverters._
+
+import org.apache.hadoop.hive.conf.HiveConf
+import org.apache.hive.service.cli.OperationHandle
+import org.apache.hive.service.cli.operation.{GetCatalogsOperation, OperationManager}
+import org.apache.hive.service.cli.session.{HiveSessionImpl, SessionManager}
+import org.mockito.Mockito.{mock, verify, when}
+import org.mockito.invocation.InvocationOnMock
+
+import org.apache.spark.SparkFunSuite
+
+class HiveSessionImplSuite extends SparkFunSuite {
+  private var session: HiveSessionImpl = _
+  private var operationManager: OperationManager = _
+
+  override def beforeAll() {
+    super.beforeAll()
+
+    session = new HiveSessionImpl(
+      ThriftserverShimUtils.testedProtocolVersions.head,
+      "",
+      "",
+      new HiveConf(),
+      ""
+    )
+    val sessionManager = mock(classOf[SessionManager])
+    session.setSessionManager(sessionManager)
+    operationManager = mock(classOf[OperationManager])
+    session.setOperationManager(operationManager)
+    when(operationManager.newGetCatalogsOperation(session)).thenAnswer(
+      (_: InvocationOnMock) => {
+        val operation = mock(classOf[GetCatalogsOperation])
+        when(operation.getHandle).thenReturn(mock(classOf[OperationHandle]))
+        operation
+      }
+    )
+
+    session.open(Map.empty[String, String].asJava)
+  }
+
+  test("SPARK-31387 - session.close() closes all sessions regardless of thrown exceptions") {
+    val operationHandle1 = session.getCatalogs
+    val operationHandle2 = session.getCatalogs
+
+    when(operationManager.closeOperation(operationHandle1))
+      .thenThrow(classOf[NullPointerException])
+    when(operationManager.closeOperation(operationHandle2))
+      .thenThrow(classOf[NullPointerException])
+
+    session.close()
+
+    verify(operationManager).closeOperation(operationHandle1)
+    verify(operationManager).closeOperation(operationHandle2)
+  }
+}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
index 075032fa5d099..9a9f574153a0a 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
@@ -140,6 +140,23 @@ class HiveThriftServer2ListenerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(listener.noLiveData())
   }
 
+  test("SPARK-31387 - listener update methods should not throw exception with unknown input") {
+    val (statusStore: HiveThriftServer2AppStatusStore, listener: HiveThriftServer2Listener) =
+      createAppStatusStore(true)
+
+    val unknownSession = "unknown_session"
+    val unknownOperation = "unknown_operation"
+    listener.onOtherEvent(SparkListenerThriftServerSessionClosed(unknownSession, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationStart("id", unknownSession,
+      "stmt", "groupId", 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationParsed(unknownOperation, "query"))
+    listener.onOtherEvent(SparkListenerThriftServerOperationCanceled(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationError(unknownOperation,
+      "msg", "trace", 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationFinish(unknownOperation, 0))
+    listener.onOtherEvent(SparkListenerThriftServerOperationClosed(unknownOperation, 0))
+  }
+
   private def createProperties: Properties = {
     val properties = new Properties()
     properties.setProperty(SparkContext.SPARK_JOB_GROUP_ID, "groupId")
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 745f385e87f78..e3fb54d9f47e9 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -636,7 +636,11 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        operationManager.closeOperation(opHandle);
+        try {
+          operationManager.closeOperation(opHandle);
+        } catch (Exception e) {
+          LOG.warn("Exception is thrown closing operation " + opHandle, e);
+        }
       }
       opHandleSet.clear();
       // Cleanup session log directory.
@@ -674,11 +678,15 @@ private void cleanupPipeoutFile() {
     File[] fileAry = new File(lScratchDir).listFiles(
             (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
 
-    for (File file : fileAry) {
-      try {
-        FileUtils.forceDelete(file);
-      } catch (Exception e) {
-        LOG.error("Failed to cleanup pipeout file: " + file, e);
+    if (fileAry == null) {
+      LOG.error("Unable to access pipeout files in " + lScratchDir);
+    } else {
+      for (File file : fileAry) {
+        try {
+          FileUtils.forceDelete(file);
+        } catch (Exception e) {
+          LOG.error("Failed to cleanup pipeout file: " + file, e);
+        }
       }
     }
   }
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 14e9c4704c977..1b3e8fe6bfb9d 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -650,7 +650,11 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        operationManager.closeOperation(opHandle);
+        try {
+          operationManager.closeOperation(opHandle);
+        } catch (Exception e) {
+          LOG.warn("Exception is thrown closing operation " + opHandle, e);
+        }
       }
       opHandleSet.clear();
       // Cleanup session log directory.
@@ -688,11 +692,15 @@ private void cleanupPipeoutFile() {
     File[] fileAry = new File(lScratchDir).listFiles(
             (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
 
-    for (File file : fileAry) {
-      try {
-        FileUtils.forceDelete(file);
-      } catch (Exception e) {
-        LOG.error("Failed to cleanup pipeout file: " + file, e);
+    if (fileAry == null) {
+      LOG.error("Unable to access pipeout files in " + lScratchDir);
+    } else {
+      for (File file : fileAry) {
+        try {
+          FileUtils.forceDelete(file);
+        } catch (Exception e) {
+          LOG.error("Failed to cleanup pipeout file: " + file, e);
+        }
       }
     }
   }

From 6b31e1af6f5d4866075e8f86a2833733c1acca16 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 21 May 2020 03:11:20 +0900
Subject: [PATCH 0712/1280] [SPARK-30689][CORE][FOLLOW-UP] Add @since
 annotation for ResourceDiscoveryScriptPlugin/ResourceInformation

### What changes were proposed in this pull request?

Added `since 3.0.0` for `ResourceDiscoveryScriptPlugin` and `ResourceInformation`.

### Why are the changes needed?

It's required for exposed APIs(https://github.com/apache/spark/pull/27689#discussion_r426488851).

### Does this PR introduce _any_ user-facing change?

Yes, they can easily know when does Spark introduces the API.

### How was this patch tested?

Pass Jenkins.

Closes #28591 from Ngone51/followup-30689.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 26bc6907222ddb2d800a2c1245232887422e49d9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/resource/ResourceDiscoveryScriptPlugin.scala   | 2 ++
 .../scala/org/apache/spark/resource/ResourceInformation.scala   | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 7027d1e3511b5..11a9bb86d3034 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -32,6 +32,8 @@ import org.apache.spark.util.Utils.executeAndGetOutput
  * and gets the json output back and contructs ResourceInformation objects from that.
  * If the user specifies custom plugins, this is the last one to be executed and
  * throws if the resource isn't discovered.
+ *
+ * @since 3.0.0
  */
 @DeveloperApi
 class ResourceDiscoveryScriptPlugin extends ResourceDiscoveryPlugin with Logging {
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala b/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
index d5ac41b995559..be056e15b6d03 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceInformation.scala
@@ -33,6 +33,8 @@ import org.apache.spark.annotation.Evolving
  *
  * @param name the name of the resource
  * @param addresses an array of strings describing the addresses of the resource
+ *
+ * @since 3.0.0
  */
 @Evolving
 class ResourceInformation(

From 77797db60bc7207bc10fa01ebd5b55e61a870cb5 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 20 May 2020 18:33:38 -0700
Subject: [PATCH 0713/1280] [SPARK-31780][K8S][TESTS] Add R test tag to exclude
 R K8s image building and test

### What changes were proposed in this pull request?

This PR aims to skip R image building and one R test during integration tests by using `--exclude-tags r`.

### Why are the changes needed?

We have only one R integration test case, `Run SparkR on simple dataframe.R example`, for submission test coverage. Since this is rarely changed, we can skip this and save the efforts required for building the whole R image and running the single test.
```
KubernetesSuite:
...
- Run SparkR on simple dataframe.R example
Run completed in 10 minutes, 20 seconds.
Total number of tests run: 20
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the K8S integration test and do the following manually. (Note that R test is skipped)
```
$ resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh --deploy-mode docker-for-desktop --exclude-tags r --spark-tgz $PWD/spark-*.tgz
...
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- All pods have the same service account by default
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
- Start pod creation from template
- PVs with local storage
- Launcher client dependencies
- Test basic decommissioning
Run completed in 10 minutes, 23 seconds.
Total number of tests run: 19
Suites: completed 2, aborted 0
Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```

Closes #28594 from dongjoon-hyun/SPARK-31780.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a06768ec4d5059d1037086fe5495e5d23cde514b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 resource-managers/kubernetes/integration-tests/pom.xml |  3 +++
 .../scripts/setup-integration-test-env.sh              | 10 +++++++++-
 .../deploy/k8s/integrationtest/KubernetesSuite.scala   |  1 +
 .../spark/deploy/k8s/integrationtest/RTestsSuite.scala |  4 ++--
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 52412570a2788..3222ec7db689c 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -121,6 +121,9 @@
 
                 <argument>--spark-tgz</argument>
                 <argument>${spark.kubernetes.test.sparkTgz}</argument>
+
+                <argument>--test-exclude-tags</argument>
+                <argument>"${test.exclude.tags}"</argument>
               </arguments>
             </configuration>
           </execution>
diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
index 2e72ff8e55bfa..beda56cf37c94 100755
--- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
+++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh
@@ -26,6 +26,7 @@ IMAGE_TAG="N/A"
 JAVA_IMAGE_TAG="8-jre-slim"
 SPARK_TGZ="N/A"
 MVN="$TEST_ROOT_DIR/build/mvn"
+EXCLUDE_TAGS=""
 
 # Parse arguments
 while (( "$#" )); do
@@ -58,6 +59,10 @@ while (( "$#" )); do
       SPARK_TGZ="$2"
       shift
       ;;
+    --test-exclude-tags)
+      EXCLUDE_TAGS="$2"
+      shift
+      ;;
     *)
       break
       ;;
@@ -99,7 +104,10 @@ then
   LANGUAGE_BINDING_BUILD_ARGS="-p $DOCKER_FILE_BASE_PATH/bindings/python/Dockerfile"
 
   # Build SparkR image
-  LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R $DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
+  tags=(${EXCLUDE_TAGS//,/ })
+  if [[ ! ${tags[@]} =~ "r" ]]; then
+    LANGUAGE_BINDING_BUILD_ARGS="$LANGUAGE_BINDING_BUILD_ARGS -R $DOCKER_FILE_BASE_PATH/bindings/R/Dockerfile"
+  fi
 
   # Unset SPARK_HOME to let the docker-image-tool script detect SPARK_HOME. Otherwise, it cannot
   # indicate the unpacked directory as its home. See SPARK-28550.
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 0d4fcccc35cf9..00996df416439 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -419,6 +419,7 @@ class KubernetesSuite extends SparkFunSuite
 
 private[spark] object KubernetesSuite {
   val k8sTestTag = Tag("k8s")
+  val rTestTag = Tag("r")
   val MinikubeTag = Tag("minikube")
   val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi"
   val SPARK_DFS_READ_WRITE_TEST = "org.apache.spark.examples.DFSReadWriteTest"
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
index e81562a923228..b7c8886a15ae7 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/RTestsSuite.scala
@@ -19,9 +19,9 @@ package org.apache.spark.deploy.k8s.integrationtest
 private[spark] trait RTestsSuite { k8sSuite: KubernetesSuite =>
 
   import RTestsSuite._
-  import KubernetesSuite.k8sTestTag
+  import KubernetesSuite.{k8sTestTag, rTestTag}
 
-  test("Run SparkR on simple dataframe.R example", k8sTestTag) {
+  test("Run SparkR on simple dataframe.R example", k8sTestTag, rTestTag) {
     sparkAppConf.set("spark.kubernetes.container.image", rImage)
     runSparkApplicationAndVerifyCompletion(
       appResource = SPARK_R_DATAFRAME_TEST,

From b4df7b57be33baee8fea98eb2ff85fe77b9aeef0 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 21 May 2020 04:01:19 +0000
Subject: [PATCH 0714/1280] [SPARK-31762][SQL] Fix perf regression of
 date/timestamp formatting in toHiveString

### What changes were proposed in this pull request?
1. Add new methods that accept date-time Java types to the DateFormatter and TimestampFormatter traits. The methods format input date-time instances to strings:
    - TimestampFormatter:
      - `def format(ts: Timestamp): String`
      - `def format(instant: Instant): String`
    - DateFormatter:
      - `def format(date: Date): String`
      - `def format(localDate: LocalDate): String`
2. Re-use the added methods from `HiveResult.toHiveString`
3. Borrow the code for formatting of `java.sql.Timestamp` from Spark 2.4 `DateTimeUtils.timestampToString` to `FractionTimestampFormatter` because legacy formatters don't support variable length patterns for seconds fractions.

### Why are the changes needed?
To avoid unnecessary overhead of converting Java date-time types to micros/days before formatting. Also formatters have to convert input micros/days back to Java types to pass instances to standard library API.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By existing tests for toHiveString and new tests in `TimestampFormatterSuite`.

Closes #28582 from MaxGekk/opt-format-old-types.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5d673319af81bb826e5f532b5ff25f2d4b2da122)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateFormatter.scala     | 25 ++++--
 .../catalyst/util/TimestampFormatter.scala    | 52 +++++++++++-
 .../spark/sql/util/DateFormatterSuite.scala   | 16 ++--
 .../sql/util/TimestampFormatterSuite.scala    | 84 ++++++++++++-------
 .../spark/sql/execution/HiveResult.scala      | 11 +--
 .../sql-tests/results/postgreSQL/date.sql.out |  2 +-
 6 files changed, 138 insertions(+), 52 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 0f79c1a6a751d..7d94955096196 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -29,7 +29,10 @@ import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 sealed trait DateFormatter extends Serializable {
   def parse(s: String): Int // returns days since epoch
+
   def format(days: Int): String
+  def format(date: Date): String
+  def format(localDate: LocalDate): String
 }
 
 class Iso8601DateFormatter(
@@ -56,22 +59,32 @@ class Iso8601DateFormatter(
     }
   }
 
+  override def format(localDate: LocalDate): String = {
+    localDate.format(formatter)
+  }
+
   override def format(days: Int): String = {
-    LocalDate.ofEpochDay(days).format(formatter)
+    format(LocalDate.ofEpochDay(days))
+  }
+
+  override def format(date: Date): String = {
+    legacyFormatter.format(date)
   }
 }
 
 trait LegacyDateFormatter extends DateFormatter {
   def parseToDate(s: String): Date
-  def formatDate(d: Date): String
 
   override def parse(s: String): Int = {
     fromJavaDate(new java.sql.Date(parseToDate(s).getTime))
   }
 
   override def format(days: Int): String = {
-    val date = DateTimeUtils.toJavaDate(days)
-    formatDate(date)
+    format(DateTimeUtils.toJavaDate(days))
+  }
+
+  override def format(localDate: LocalDate): String = {
+    format(localDateToDays(localDate))
   }
 }
 
@@ -79,14 +92,14 @@ class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDat
   @transient
   private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
   override def parseToDate(s: String): Date = fdf.parse(s)
-  override def formatDate(d: Date): String = fdf.format(d)
+  override def format(d: Date): String = fdf.format(d)
 }
 
 class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
   @transient
   private lazy val sdf = new SimpleDateFormat(pattern, locale)
   override def parseToDate(s: String): Date = sdf.parse(s)
-  override def formatDate(d: Date): String = sdf.format(d)
+  override def format(d: Date): String = sdf.format(d)
 }
 
 object DateFormatter {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 47d6b47f32fa1..d7d63687ff364 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -50,7 +50,10 @@ sealed trait TimestampFormatter extends Serializable {
   @throws(classOf[DateTimeParseException])
   @throws(classOf[DateTimeException])
   def parse(s: String): Long
+
   def format(us: Long): String
+  def format(ts: Timestamp): String
+  def format(instant: Instant): String
 }
 
 class Iso8601TimestampFormatter(
@@ -84,9 +87,17 @@ class Iso8601TimestampFormatter(
     }
   }
 
+  override def format(instant: Instant): String = {
+    formatter.withZone(zoneId).format(instant)
+  }
+
   override def format(us: Long): String = {
     val instant = DateTimeUtils.microsToInstant(us)
-    formatter.withZone(zoneId).format(instant)
+    format(instant)
+  }
+
+  override def format(ts: Timestamp): String = {
+    legacyFormatter.format(ts)
   }
 }
 
@@ -100,10 +111,27 @@ class Iso8601TimestampFormatter(
  */
 class FractionTimestampFormatter(zoneId: ZoneId)
   extends Iso8601TimestampFormatter(
-    "", zoneId, TimestampFormatter.defaultLocale, needVarLengthSecondFraction = false) {
+    TimestampFormatter.defaultPattern,
+    zoneId,
+    TimestampFormatter.defaultLocale,
+    needVarLengthSecondFraction = false) {
 
   @transient
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
+
+  // The new formatter will omit the trailing 0 in the timestamp string, but the legacy formatter
+  // can't. Here we borrow the code from Spark 2.4 DateTimeUtils.timestampToString to omit the
+  // trailing 0 for the legacy formatter as well.
+  override def format(ts: Timestamp): String = {
+    val timestampString = ts.toString
+    val formatted = legacyFormatter.format(ts)
+
+    if (timestampString.length > 19 && timestampString.substring(19) != ".0") {
+      formatted + timestampString.substring(19)
+    } else {
+      formatted
+    }
+  }
 }
 
 /**
@@ -149,7 +177,7 @@ class LegacyFastTimestampFormatter(
     fastDateFormat.getTimeZone,
     fastDateFormat.getPattern.count(_ == 'S'))
 
-  def parse(s: String): SQLTimestamp = {
+  override def parse(s: String): SQLTimestamp = {
     cal.clear() // Clear the calendar because it can be re-used many times
     if (!fastDateFormat.parse(s, new ParsePosition(0), cal)) {
       throw new IllegalArgumentException(s"'$s' is an invalid timestamp")
@@ -160,12 +188,20 @@ class LegacyFastTimestampFormatter(
     rebaseJulianToGregorianMicros(julianMicros)
   }
 
-  def format(timestamp: SQLTimestamp): String = {
+  override def format(timestamp: SQLTimestamp): String = {
     val julianMicros = rebaseGregorianToJulianMicros(timestamp)
     cal.setTimeInMillis(Math.floorDiv(julianMicros, MICROS_PER_SECOND) * MILLIS_PER_SECOND)
     cal.setMicros(Math.floorMod(julianMicros, MICROS_PER_SECOND))
     fastDateFormat.format(cal)
   }
+
+  override def format(ts: Timestamp): String = {
+    format(fromJavaTimestamp(ts))
+  }
+
+  override def format(instant: Instant): String = {
+    format(instantToMicros(instant))
+  }
 }
 
 class LegacySimpleTimestampFormatter(
@@ -187,6 +223,14 @@ class LegacySimpleTimestampFormatter(
   override def format(us: Long): String = {
     sdf.format(toJavaTimestamp(us))
   }
+
+  override def format(ts: Timestamp): String = {
+    sdf.format(ts)
+  }
+
+  override def format(instant: Instant): String = {
+    format(instantToMicros(instant))
+  }
 }
 
 object LegacyDateFormats extends Enumeration {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index 5e2b6a7c7fafe..3954b9b8355c8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -22,7 +22,7 @@ import java.time.{DateTimeException, LocalDate, ZoneOffset}
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util._
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
@@ -41,8 +41,11 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
     DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
         val formatter = DateFormatter(getZoneId(timeZone))
-        val date = formatter.format(17867)
-        assert(date === "2018-12-02")
+        val (days, expected) = (17867, "2018-12-02")
+        val date = formatter.format(days)
+        assert(date === expected)
+        assert(formatter.format(daysToLocalDate(days)) === expected)
+        assert(formatter.format(toJavaDate(days)) === expected)
       }
     }
   }
@@ -70,8 +73,9 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
                   DateFormatter.defaultLocale,
                   legacyFormat)
                 val days = formatter.parse(date)
-                val formatted = formatter.format(days)
-                assert(date === formatted)
+                assert(date === formatter.format(days))
+                assert(date === formatter.format(daysToLocalDate(days)))
+                assert(date === formatter.format(toJavaDate(days)))
               }
             }
           }
@@ -170,7 +174,9 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
               DateFormatter.defaultLocale,
               legacyFormat)
             assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
+            assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01")
             assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
+            assert(formatter.format(java.sql.Date.valueOf("1000-01-01")) === "1000-01-01")
           }
         }
       }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 5d27a6b8cce1e..b467e24b5301a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -57,20 +57,29 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   test("format timestamps using time zones") {
     val microsSinceEpoch = 1543745472001234L
     val expectedTimestamp = Map(
-      "UTC" -> "2018-12-02T10:11:12.001234",
-      PST.getId -> "2018-12-02T02:11:12.001234",
-      CET.getId -> "2018-12-02T11:11:12.001234",
-      "Africa/Dakar" -> "2018-12-02T10:11:12.001234",
-      "America/Los_Angeles" -> "2018-12-02T02:11:12.001234",
-      "Antarctica/Vostok" -> "2018-12-02T16:11:12.001234",
-      "Asia/Hong_Kong" -> "2018-12-02T18:11:12.001234",
-      "Europe/Amsterdam" -> "2018-12-02T11:11:12.001234")
+      "UTC" -> "2018-12-02 10:11:12.001234",
+      PST.getId -> "2018-12-02 02:11:12.001234",
+      CET.getId -> "2018-12-02 11:11:12.001234",
+      "Africa/Dakar" -> "2018-12-02 10:11:12.001234",
+      "America/Los_Angeles" -> "2018-12-02 02:11:12.001234",
+      "Antarctica/Vostok" -> "2018-12-02 16:11:12.001234",
+      "Asia/Hong_Kong" -> "2018-12-02 18:11:12.001234",
+      "Europe/Amsterdam" -> "2018-12-02 11:11:12.001234")
     DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
-      val formatter = TimestampFormatter(
-        "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId))
-      val timestamp = formatter.format(microsSinceEpoch)
-      assert(timestamp === expectedTimestamp(zoneId))
+      Seq(
+        TimestampFormatter(
+          "yyyy-MM-dd HH:mm:ss.SSSSSS",
+          getZoneId(zoneId),
+          // Test only FAST_DATE_FORMAT because other legacy formats don't support formatting
+          // in microsecond precision.
+          LegacyDateFormats.FAST_DATE_FORMAT,
+          needVarLengthSecondFraction = false),
+        TimestampFormatter.getFractionFormatter(getZoneId(zoneId))).foreach { formatter =>
+        val timestamp = formatter.format(microsSinceEpoch)
+        assert(timestamp === expectedTimestamp(zoneId))
+        assert(formatter.format(microsToInstant(microsSinceEpoch)) === expectedTimestamp(zoneId))
+        assert(formatter.format(toJavaTimestamp(microsSinceEpoch)) === expectedTimestamp(zoneId))
+      }
     }
   }
 
@@ -125,20 +134,30 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("format fraction of second") {
-    val formatter = TimestampFormatter.getFractionFormatter(ZoneOffset.UTC)
-    assert(formatter.format(0) === "1970-01-01 00:00:00")
-    assert(formatter.format(1) === "1970-01-01 00:00:00.000001")
-    assert(formatter.format(1000) === "1970-01-01 00:00:00.001")
-    assert(formatter.format(900000) === "1970-01-01 00:00:00.9")
-    assert(formatter.format(1000000) === "1970-01-01 00:00:01")
+    val formatter = TimestampFormatter.getFractionFormatter(UTC)
+    Seq(
+      0 -> "1970-01-01 00:00:00",
+      1 -> "1970-01-01 00:00:00.000001",
+      1000 -> "1970-01-01 00:00:00.001",
+      900000 -> "1970-01-01 00:00:00.9",
+      1000000 -> "1970-01-01 00:00:01").foreach { case (micros, tsStr) =>
+      assert(formatter.format(micros) === tsStr)
+      assert(formatter.format(microsToInstant(micros)) === tsStr)
+      DateTimeTestUtils.withDefaultTimeZone(UTC) {
+        assert(formatter.format(toJavaTimestamp(micros)) === tsStr)
+      }
+    }
   }
 
   test("formatting negative years with default pattern") {
-    val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0)
-      .atZone(ZoneOffset.UTC)
-      .toInstant
+    val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0).atZone(UTC).toInstant
     val micros = DateTimeUtils.instantToMicros(instant)
-    assert(TimestampFormatter(ZoneOffset.UTC).format(micros) === "-0099-01-01 00:00:00")
+    assert(TimestampFormatter(UTC).format(micros) === "-0099-01-01 00:00:00")
+    assert(TimestampFormatter(UTC).format(instant) === "-0099-01-01 00:00:00")
+    DateTimeTestUtils.withDefaultTimeZone(UTC) { // toJavaTimestamp depends on the default time zone
+      assert(TimestampFormatter("yyyy-MM-dd HH:mm:SS G", UTC).format(toJavaTimestamp(micros))
+        === "0100-01-01 00:00:00 BC")
+    }
   }
 
   test("special timestamp values") {
@@ -266,24 +285,31 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
 
   test("SPARK-31557: rebasing in legacy formatters/parsers") {
     withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
-      LegacyDateFormats.values.foreach { legacyFormat =>
-        DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
-          withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
-            DateTimeTestUtils.withDefaultTimeZone(zoneId) {
-              withClue(s"${zoneId.getId} legacyFormat = $legacyFormat") {
-                val formatter = TimestampFormatter(
+      DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+        withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
+          DateTimeTestUtils.withDefaultTimeZone(zoneId) {
+            withClue(s"zoneId = ${zoneId.getId}") {
+              val formatters = LegacyDateFormats.values.map { legacyFormat =>
+                TimestampFormatter(
                   TimestampFormatter.defaultPattern,
                   zoneId,
                   TimestampFormatter.defaultLocale,
                   legacyFormat,
                   needVarLengthSecondFraction = false)
+              }.toSeq :+  TimestampFormatter.getFractionFormatter(zoneId)
+              formatters.foreach { formatter =>
                 assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
                   .atZone(zoneId)
                   .toLocalDateTime === LocalDateTime.of(1000, 1, 1, 1, 2, 3))
 
+                assert(formatter.format(
+                  LocalDateTime.of(1000, 1, 1, 1, 2, 3).atZone(zoneId).toInstant) ===
+                  "1000-01-01 01:02:03")
                 assert(formatter.format(instantToMicros(
                   LocalDateTime.of(1000, 1, 1, 1, 2, 3)
                     .atZone(zoneId).toInstant)) === "1000-01-01 01:02:03")
+                assert(formatter.format(java.sql.Timestamp.valueOf("1000-01-01 01:02:03")) ===
+                  "1000-01-01 01:02:03")
               }
             }
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index 1a84db1970449..73484a212c169 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -80,13 +80,10 @@ object HiveResult {
   def toHiveString(a: (Any, DataType), nested: Boolean = false): String = a match {
     case (null, _) => if (nested) "null" else "NULL"
     case (b, BooleanType) => b.toString
-    case (d: Date, DateType) => dateFormatter.format(DateTimeUtils.fromJavaDate(d))
-    case (ld: LocalDate, DateType) =>
-      dateFormatter.format(DateTimeUtils.localDateToDays(ld))
-    case (t: Timestamp, TimestampType) =>
-      timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t))
-    case (i: Instant, TimestampType) =>
-      timestampFormatter.format(DateTimeUtils.instantToMicros(i))
+    case (d: Date, DateType) => dateFormatter.format(d)
+    case (ld: LocalDate, DateType) => dateFormatter.format(ld)
+    case (t: Timestamp, TimestampType) => timestampFormatter.format(t)
+    case (i: Instant, TimestampType) => timestampFormatter.format(i)
     case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
     case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
     case (n, _: NumericType) => n.toString
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
index 1d862ba8a41a8..151fa1e28d725 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out
@@ -584,7 +584,7 @@ select make_date(-44, 3, 15)
 -- !query schema
 struct<make_date(-44, 3, 15):date>
 -- !query output
--0044-03-15
+0045-03-15
 
 
 -- !query

From 0f2afd3455e164dcd273f6c69774d08f7121af8d Mon Sep 17 00:00:00 2001
From: Vinoo Ganesh <vinoo.ganesh@gmail.com>
Date: Thu, 21 May 2020 16:06:28 +0000
Subject: [PATCH 0715/1280] [SPARK-31354] SparkContext only register one
 SparkSession ApplicationEnd listener

## What changes were proposed in this pull request?

This change was made as a result of the conversation on https://issues.apache.org/jira/browse/SPARK-31354 and is intended to continue work from that ticket here.

This change fixes a memory leak where SparkSession listeners are never cleared off of the SparkContext listener bus.

Before running this PR, the following code:
```
SparkSession.builder().master("local").getOrCreate()
SparkSession.clearActiveSession()
SparkSession.clearDefaultSession()

SparkSession.builder().master("local").getOrCreate()
SparkSession.clearActiveSession()
SparkSession.clearDefaultSession()
```
would result in a SparkContext with the following listeners on the listener bus:
```
[org.apache.spark.status.AppStatusListener5f610071,
org.apache.spark.HeartbeatReceiverd400c17,
org.apache.spark.sql.SparkSession$$anon$125849aeb, <-First instance
org.apache.spark.sql.SparkSession$$anon$1fadb9a0] <- Second instance
```
After this PR, the execution of the same code above results in SparkContext with the following listeners on the listener bus:
```
[org.apache.spark.status.AppStatusListener5f610071,
org.apache.spark.HeartbeatReceiverd400c17,
org.apache.spark.sql.SparkSession$$anon$125849aeb] <-One instance
```
## How was this patch tested?

* Unit test included as a part of the PR

Closes #28128 from vinooganesh/vinooganesh/SPARK-27958.

Lead-authored-by: Vinoo Ganesh <vinoo.ganesh@gmail.com>
Co-authored-by: Vinoo Ganesh <vganesh@palantir.com>
Co-authored-by: Vinoo Ganesh <vinoo@safegraph.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit dae79888dc6476892877d3b3b233381cdbf7fa74)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/SparkSession.scala   | 27 +++++++++++--------
 .../spark/sql/SparkSessionBuilderSuite.scala  | 25 +++++++++++++++++
 2 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index be597edecba98..60a60377d8a3f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
 
 import java.io.Closeable
 import java.util.concurrent.TimeUnit._
-import java.util.concurrent.atomic.AtomicReference
+import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference}
 
 import scala.collection.JavaConverters._
 import scala.reflect.runtime.universe.TypeTag
@@ -49,7 +49,6 @@ import org.apache.spark.sql.types.{DataType, StructType}
 import org.apache.spark.sql.util.ExecutionListenerManager
 import org.apache.spark.util.{CallSite, Utils}
 
-
 /**
  * The entry point to programming Spark with the Dataset and DataFrame API.
  *
@@ -940,15 +939,7 @@ object SparkSession extends Logging {
         options.foreach { case (k, v) => session.initialSessionOptions.put(k, v) }
         setDefaultSession(session)
         setActiveSession(session)
-
-        // Register a successfully instantiated context to the singleton. This should be at the
-        // end of the class definition so that the singleton is updated only if there is no
-        // exception in the construction of the instance.
-        sparkContext.addSparkListener(new SparkListener {
-          override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = {
-            defaultSession.set(null)
-          }
-        })
+        registerContextListener(sparkContext)
       }
 
       return session
@@ -1064,6 +1055,20 @@ object SparkSession extends Logging {
   // Private methods from now on
   ////////////////////////////////////////////////////////////////////////////////////////
 
+  private val listenerRegistered: AtomicBoolean = new AtomicBoolean(false)
+
+  /** Register the AppEnd listener onto the Context  */
+  private def registerContextListener(sparkContext: SparkContext): Unit = {
+    if (!listenerRegistered.get()) {
+      sparkContext.addSparkListener(new SparkListener {
+        override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd): Unit = {
+          defaultSession.set(null)
+        }
+      })
+      listenerRegistered.set(true)
+    }
+  }
+
   /** The active SparkSession for the current thread. */
   private val activeThreadSession = new InheritableThreadLocal[SparkSession]
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 7b76d0702d835..0a522fdbdeed8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -169,6 +169,31 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     assert(session.sessionState.conf.getConf(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31234")
   }
 
+  test("SPARK-31354: SparkContext only register one SparkSession ApplicationEnd listener") {
+    val conf = new SparkConf()
+      .setMaster("local")
+      .setAppName("test-app-SPARK-31354-1")
+    val context = new SparkContext(conf)
+    SparkSession
+      .builder()
+      .sparkContext(context)
+      .master("local")
+      .getOrCreate()
+    val postFirstCreation = context.listenerBus.listeners.size()
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+
+    SparkSession
+      .builder()
+      .sparkContext(context)
+      .master("local")
+      .getOrCreate()
+    val postSecondCreation = context.listenerBus.listeners.size()
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    assert(postFirstCreation == postSecondCreation)
+  }
+
   test("SPARK-31532: should not propagate static sql configs to the existing" +
     " active/default SparkSession") {
     val session = SparkSession.builder()

From cc1e568ea9cff11879a7c3eceadfe455443af3af Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 21 May 2020 14:20:50 -0700
Subject: [PATCH 0716/1280] Revert "[SPARK-31387][SQL] Handle unknown
 operation/session ID in HiveThriftServer2Listener"

This reverts commit 5198b6853b2e3bc69fc013c653aa163c79168366.
---
 .../ui/HiveThriftServer2Listener.scala        | 106 +++++++-----------
 .../thriftserver/HiveSessionImplSuite.scala   |  73 ------------
 .../ui/HiveThriftServer2ListenerSuite.scala   |  17 ---
 .../service/cli/session/HiveSessionImpl.java  |  20 +---
 .../service/cli/session/HiveSessionImpl.java  |  20 +---
 5 files changed, 53 insertions(+), 183 deletions(-)
 delete mode 100644 sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
index 6b7e5ee611417..6d0a506fa94dc 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2Listener.scala
@@ -25,7 +25,6 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.hive.service.server.HiveServer2
 
 import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.Status.LIVE_ENTITY_UPDATE_PERIOD
 import org.apache.spark.scheduler._
 import org.apache.spark.sql.hive.thriftserver.HiveThriftServer2.ExecutionState
@@ -39,7 +38,7 @@ private[thriftserver] class HiveThriftServer2Listener(
     kvstore: ElementTrackingStore,
     sparkConf: SparkConf,
     server: Option[HiveServer2],
-    live: Boolean = true) extends SparkListener with Logging {
+    live: Boolean = true) extends SparkListener {
 
   private val sessionList = new ConcurrentHashMap[String, LiveSessionData]()
   private val executionList = new ConcurrentHashMap[String, LiveExecutionData]()
@@ -132,83 +131,60 @@ private[thriftserver] class HiveThriftServer2Listener(
     updateLiveStore(session)
   }
 
-  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit =
-    Option(sessionList.get(e.sessionId)) match {
-      case Some(sessionData) =>
-        sessionData.finishTimestamp = e.finishTime
-        updateStoreWithTriggerEnabled(sessionData)
-        sessionList.remove(e.sessionId)
-      case None => logWarning(s"onSessionClosed called with unknown session id: ${e.sessionId}")
-    }
+  private def onSessionClosed(e: SparkListenerThriftServerSessionClosed): Unit = {
+    val session = sessionList.get(e.sessionId)
+    session.finishTimestamp = e.finishTime
+    updateStoreWithTriggerEnabled(session)
+    sessionList.remove(e.sessionId)
+  }
 
   private def onOperationStart(e: SparkListenerThriftServerOperationStart): Unit = {
-    val executionData = getOrCreateExecution(
+    val info = getOrCreateExecution(
       e.id,
       e.statement,
       e.sessionId,
       e.startTime,
       e.userName)
 
-    executionData.state = ExecutionState.STARTED
-    executionList.put(e.id, executionData)
-    executionData.groupId = e.groupId
-    updateLiveStore(executionData)
-
-    Option(sessionList.get(e.sessionId)) match {
-      case Some(sessionData) =>
-        sessionData.totalExecution += 1
-        updateLiveStore(sessionData)
-      case None => logWarning(s"onOperationStart called with unknown session id: ${e.sessionId}." +
-        s"Regardless, the operation has been registered.")
-    }
+    info.state = ExecutionState.STARTED
+    executionList.put(e.id, info)
+    sessionList.get(e.sessionId).totalExecution += 1
+    executionList.get(e.id).groupId = e.groupId
+    updateLiveStore(executionList.get(e.id))
+    updateLiveStore(sessionList.get(e.sessionId))
   }
 
-  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit =
-    Option(executionList.get(e.id)) match {
-      case Some(executionData) =>
-        executionData.executePlan = e.executionPlan
-        executionData.state = ExecutionState.COMPILED
-        updateLiveStore(executionData)
-      case None => logWarning(s"onOperationParsed called with unknown operation id: ${e.id}")
-    }
+  private def onOperationParsed(e: SparkListenerThriftServerOperationParsed): Unit = {
+    executionList.get(e.id).executePlan = e.executionPlan
+    executionList.get(e.id).state = ExecutionState.COMPILED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit =
-    Option(executionList.get(e.id)) match {
-      case Some(executionData) =>
-        executionData.finishTimestamp = e.finishTime
-        executionData.state = ExecutionState.CANCELED
-        updateLiveStore(executionData)
-      case None => logWarning(s"onOperationCanceled called with unknown operation id: ${e.id}")
-    }
+  private def onOperationCanceled(e: SparkListenerThriftServerOperationCanceled): Unit = {
+    executionList.get(e.id).finishTimestamp = e.finishTime
+    executionList.get(e.id).state = ExecutionState.CANCELED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit =
-    Option(executionList.get(e.id)) match {
-      case Some(executionData) =>
-        executionData.finishTimestamp = e.finishTime
-        executionData.detail = e.errorMsg
-        executionData.state = ExecutionState.FAILED
-        updateLiveStore(executionData)
-      case None => logWarning(s"onOperationError called with unknown operation id: ${e.id}")
-    }
+  private def onOperationError(e: SparkListenerThriftServerOperationError): Unit = {
+    executionList.get(e.id).finishTimestamp = e.finishTime
+    executionList.get(e.id).detail = e.errorMsg
+    executionList.get(e.id).state = ExecutionState.FAILED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit =
-    Option(executionList.get(e.id)) match {
-      case Some(executionData) =>
-        executionData.finishTimestamp = e.finishTime
-        executionData.state = ExecutionState.FINISHED
-        updateLiveStore(executionData)
-      case None => logWarning(s"onOperationFinished called with unknown operation id: ${e.id}")
-    }
+  private def onOperationFinished(e: SparkListenerThriftServerOperationFinish): Unit = {
+    executionList.get(e.id).finishTimestamp = e.finishTime
+    executionList.get(e.id).state = ExecutionState.FINISHED
+    updateLiveStore(executionList.get(e.id))
+  }
 
-  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit =
-    Option(executionList.get(e.id)) match {
-      case Some(executionData) =>
-        executionData.closeTimestamp = e.closeTime
-        executionData.state = ExecutionState.CLOSED
-        updateStoreWithTriggerEnabled(executionData)
-        executionList.remove(e.id)
-      case None => logWarning(s"onOperationClosed called with unknown operation id: ${e.id}")
-    }
+  private def onOperationClosed(e: SparkListenerThriftServerOperationClosed): Unit = {
+    executionList.get(e.id).closeTimestamp = e.closeTime
+    executionList.get(e.id).state = ExecutionState.CLOSED
+    updateStoreWithTriggerEnabled(executionList.get(e.id))
+    executionList.remove(e.id)
+  }
 
   // Update both live and history stores. Trigger is enabled by default, hence
   // it will cleanup the entity which exceeds the threshold.
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
deleted file mode 100644
index 05d540d782e31..0000000000000
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveSessionImplSuite.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.hive.thriftserver
-
-import scala.collection.JavaConverters._
-
-import org.apache.hadoop.hive.conf.HiveConf
-import org.apache.hive.service.cli.OperationHandle
-import org.apache.hive.service.cli.operation.{GetCatalogsOperation, OperationManager}
-import org.apache.hive.service.cli.session.{HiveSessionImpl, SessionManager}
-import org.mockito.Mockito.{mock, verify, when}
-import org.mockito.invocation.InvocationOnMock
-
-import org.apache.spark.SparkFunSuite
-
-class HiveSessionImplSuite extends SparkFunSuite {
-  private var session: HiveSessionImpl = _
-  private var operationManager: OperationManager = _
-
-  override def beforeAll() {
-    super.beforeAll()
-
-    session = new HiveSessionImpl(
-      ThriftserverShimUtils.testedProtocolVersions.head,
-      "",
-      "",
-      new HiveConf(),
-      ""
-    )
-    val sessionManager = mock(classOf[SessionManager])
-    session.setSessionManager(sessionManager)
-    operationManager = mock(classOf[OperationManager])
-    session.setOperationManager(operationManager)
-    when(operationManager.newGetCatalogsOperation(session)).thenAnswer(
-      (_: InvocationOnMock) => {
-        val operation = mock(classOf[GetCatalogsOperation])
-        when(operation.getHandle).thenReturn(mock(classOf[OperationHandle]))
-        operation
-      }
-    )
-
-    session.open(Map.empty[String, String].asJava)
-  }
-
-  test("SPARK-31387 - session.close() closes all sessions regardless of thrown exceptions") {
-    val operationHandle1 = session.getCatalogs
-    val operationHandle2 = session.getCatalogs
-
-    when(operationManager.closeOperation(operationHandle1))
-      .thenThrow(classOf[NullPointerException])
-    when(operationManager.closeOperation(operationHandle2))
-      .thenThrow(classOf[NullPointerException])
-
-    session.close()
-
-    verify(operationManager).closeOperation(operationHandle1)
-    verify(operationManager).closeOperation(operationHandle2)
-  }
-}
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
index 9a9f574153a0a..075032fa5d099 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ui/HiveThriftServer2ListenerSuite.scala
@@ -140,23 +140,6 @@ class HiveThriftServer2ListenerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(listener.noLiveData())
   }
 
-  test("SPARK-31387 - listener update methods should not throw exception with unknown input") {
-    val (statusStore: HiveThriftServer2AppStatusStore, listener: HiveThriftServer2Listener) =
-      createAppStatusStore(true)
-
-    val unknownSession = "unknown_session"
-    val unknownOperation = "unknown_operation"
-    listener.onOtherEvent(SparkListenerThriftServerSessionClosed(unknownSession, 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationStart("id", unknownSession,
-      "stmt", "groupId", 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationParsed(unknownOperation, "query"))
-    listener.onOtherEvent(SparkListenerThriftServerOperationCanceled(unknownOperation, 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationError(unknownOperation,
-      "msg", "trace", 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationFinish(unknownOperation, 0))
-    listener.onOtherEvent(SparkListenerThriftServerOperationClosed(unknownOperation, 0))
-  }
-
   private def createProperties: Properties = {
     val properties = new Properties()
     properties.setProperty(SparkContext.SPARK_JOB_GROUP_ID, "groupId")
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index e3fb54d9f47e9..745f385e87f78 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -636,11 +636,7 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        try {
-          operationManager.closeOperation(opHandle);
-        } catch (Exception e) {
-          LOG.warn("Exception is thrown closing operation " + opHandle, e);
-        }
+        operationManager.closeOperation(opHandle);
       }
       opHandleSet.clear();
       // Cleanup session log directory.
@@ -678,15 +674,11 @@ private void cleanupPipeoutFile() {
     File[] fileAry = new File(lScratchDir).listFiles(
             (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
 
-    if (fileAry == null) {
-      LOG.error("Unable to access pipeout files in " + lScratchDir);
-    } else {
-      for (File file : fileAry) {
-        try {
-          FileUtils.forceDelete(file);
-        } catch (Exception e) {
-          LOG.error("Failed to cleanup pipeout file: " + file, e);
-        }
+    for (File file : fileAry) {
+      try {
+        FileUtils.forceDelete(file);
+      } catch (Exception e) {
+        LOG.error("Failed to cleanup pipeout file: " + file, e);
       }
     }
   }
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 1b3e8fe6bfb9d..14e9c4704c977 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -650,11 +650,7 @@ public void close() throws HiveSQLException {
       acquire(true);
       // Iterate through the opHandles and close their operations
       for (OperationHandle opHandle : opHandleSet) {
-        try {
-          operationManager.closeOperation(opHandle);
-        } catch (Exception e) {
-          LOG.warn("Exception is thrown closing operation " + opHandle, e);
-        }
+        operationManager.closeOperation(opHandle);
       }
       opHandleSet.clear();
       // Cleanup session log directory.
@@ -692,15 +688,11 @@ private void cleanupPipeoutFile() {
     File[] fileAry = new File(lScratchDir).listFiles(
             (dir, name) -> name.startsWith(sessionID) && name.endsWith(".pipeout"));
 
-    if (fileAry == null) {
-      LOG.error("Unable to access pipeout files in " + lScratchDir);
-    } else {
-      for (File file : fileAry) {
-        try {
-          FileUtils.forceDelete(file);
-        } catch (Exception e) {
-          LOG.error("Failed to cleanup pipeout file: " + file, e);
-        }
+    for (File file : fileAry) {
+      try {
+        FileUtils.forceDelete(file);
+      } catch (Exception e) {
+        LOG.error("Failed to cleanup pipeout file: " + file, e);
       }
     }
   }

From 212ca86b8c2e7671e4980ea6c2b869286a4dac06 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 22 May 2020 09:53:35 +0900
Subject: [PATCH 0717/1280] [SPARK-31785][SQL][TESTS] Add a helper function to
 test all parquet readers

### What changes were proposed in this pull request?
Add `withAllParquetReaders` to `ParquetTest`. The function allow to run a block of code for all available Parquet readers.

### Why are the changes needed?
1. It simplifies tests
2. Allow to test all parquet readers that could be available in projects based on Apache Spark.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running affected test suites.

Closes #28598 from MaxGekk/add-withAllParquetReaders.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 60118a242639df060a9fdcaa4f14cd072ea3d056)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../parquet/ParquetFilterSuite.scala          |  39 +++--
 .../datasources/parquet/ParquetIOSuite.scala  | 144 ++++++++----------
 .../ParquetInteroperabilitySuite.scala        |   8 +-
 .../parquet/ParquetQuerySuite.scala           |  30 ++--
 .../datasources/parquet/ParquetTest.scala     |   7 +
 5 files changed, 106 insertions(+), 122 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 5cf21293fd07f..7b33ceff32e01 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -781,10 +781,9 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("Filter applied on merged Parquet schema with new column should work") {
     import testImplicits._
-    Seq("true", "false").foreach { vectorized =>
+    withAllParquetReaders {
       withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
-        SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true",
-        SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
+        SQLConf.PARQUET_SCHEMA_MERGING_ENABLED.key -> "true") {
         withTempPath { dir =>
           val path1 = s"${dir.getCanonicalPath}/table1"
           (1 to 3).map(i => (i, i.toString)).toDF("a", "b").write.parquet(path1)
@@ -1219,24 +1218,22 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("SPARK-17213: Broken Parquet filter push-down for string columns") {
-    Seq(true, false).foreach { vectorizedEnabled =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorizedEnabled.toString) {
-        withTempPath { dir =>
-          import testImplicits._
+    withAllParquetReaders {
+      withTempPath { dir =>
+        import testImplicits._
 
-          val path = dir.getCanonicalPath
-          // scalastyle:off nonascii
-          Seq("a", "é").toDF("name").write.parquet(path)
-          // scalastyle:on nonascii
+        val path = dir.getCanonicalPath
+        // scalastyle:off nonascii
+        Seq("a", "é").toDF("name").write.parquet(path)
+        // scalastyle:on nonascii
 
-          assert(spark.read.parquet(path).where("name > 'a'").count() == 1)
-          assert(spark.read.parquet(path).where("name >= 'a'").count() == 2)
+        assert(spark.read.parquet(path).where("name > 'a'").count() == 1)
+        assert(spark.read.parquet(path).where("name >= 'a'").count() == 2)
 
-          // scalastyle:off nonascii
-          assert(spark.read.parquet(path).where("name < 'é'").count() == 1)
-          assert(spark.read.parquet(path).where("name <= 'é'").count() == 2)
-          // scalastyle:on nonascii
-        }
+        // scalastyle:off nonascii
+        assert(spark.read.parquet(path).where("name < 'é'").count() == 1)
+        assert(spark.read.parquet(path).where("name <= 'é'").count() == 2)
+        // scalastyle:on nonascii
       }
     }
   }
@@ -1244,8 +1241,8 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   test("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") {
     import testImplicits._
 
-    Seq(true, false).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString,
+    withAllParquetReaders {
+      withSQLConf(
           SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> true.toString,
           SQLConf.SUPPORT_QUOTED_REGEX_COLUMN_NAME.key -> "false") {
         withTempPath { path =>
@@ -1255,7 +1252,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         }
       }
 
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString,
+      withSQLConf(
           // Makes sure disabling 'spark.sql.parquet.recordFilter' still enables
           // row group level filtering.
           SQLConf.PARQUET_RECORD_FILTER_ENABLED.key -> "false",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 87b4db3fe087a..f075d04165697 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -647,47 +647,39 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
   }
 
   test("read dictionary encoded decimals written as INT32") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-i32.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(5, 2) as 'i32_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-i32.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(5, 2) as 'i32_dec))
     }
   }
 
   test("read dictionary encoded decimals written as INT64") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-i64.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'i64_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-i64.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'i64_dec))
     }
   }
 
   test("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // Decimal column in this file is encoded using plain dictionary
-          readResourceParquetFile("test-data/dec-in-fixed-len.parquet"),
-          spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'fixed_len_dec))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // Decimal column in this file is encoded using plain dictionary
+        readResourceParquetFile("test-data/dec-in-fixed-len.parquet"),
+        spark.range(1 << 4).select('id % 10 cast DecimalType(10, 2) as 'fixed_len_dec))
     }
   }
 
   test("read dictionary and plain encoded timestamp_millis written as INT64") {
-    ("true" :: "false" :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-        checkAnswer(
-          // timestamp column in this file is encoded using combination of plain
-          // and dictionary encodings.
-          readResourceParquetFile("test-data/timemillis-in-i64.parquet"),
-          (1 to 3).map(i => Row(new java.sql.Timestamp(10))))
-      }
+    withAllParquetReaders {
+      checkAnswer(
+        // timestamp column in this file is encoded using combination of plain
+        // and dictionary encodings.
+        readResourceParquetFile("test-data/timemillis-in-i64.parquet"),
+        (1 to 3).map(i => Row(new java.sql.Timestamp(10))))
     }
   }
 
@@ -943,23 +935,21 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       }
     }
 
-    Seq(false, true).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
-        checkReadMixedFiles(
-          "before_1582_timestamp_micros_v2_4.snappy.parquet",
-          "TIMESTAMP_MICROS",
-          "1001-01-01 01:02:03.123456")
-        checkReadMixedFiles(
-          "before_1582_timestamp_millis_v2_4.snappy.parquet",
-          "TIMESTAMP_MILLIS",
-          "1001-01-01 01:02:03.123")
-
-        // INT96 is a legacy timestamp format and we always rebase the seconds for it.
-        checkAnswer(readResourceParquetFile(
-          "test-data/before_1582_timestamp_int96_v2_4.snappy.parquet"),
-          Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
-      }
+    withAllParquetReaders {
+      checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
+      checkReadMixedFiles(
+        "before_1582_timestamp_micros_v2_4.snappy.parquet",
+        "TIMESTAMP_MICROS",
+        "1001-01-01 01:02:03.123456")
+      checkReadMixedFiles(
+        "before_1582_timestamp_millis_v2_4.snappy.parquet",
+        "TIMESTAMP_MILLIS",
+        "1001-01-01 01:02:03.123")
+
+      // INT96 is a legacy timestamp format and we always rebase the seconds for it.
+      checkAnswer(readResourceParquetFile(
+        "test-data/before_1582_timestamp_int96_v2_4.snappy.parquet"),
+        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
     }
   }
 
@@ -984,27 +974,25 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
                   .parquet(path)
               }
 
-              Seq(false, true).foreach { vectorized =>
-                withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-                  // The file metadata indicates if it needs rebase or not, so we can always get the
-                  // correct result regardless of the "rebase mode" config.
-                  Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-                    withSQLConf(
-                      SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> mode.toString) {
-                      checkAnswer(
-                        spark.read.parquet(path),
-                        Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
-                    }
-                  }
-
-                  // Force to not rebase to prove the written datetime values are rebased
-                  // and we will get wrong result if we don't rebase while reading.
-                  withSQLConf("spark.test.forceNoRebase" -> "true") {
+              withAllParquetReaders {
+                // The file metadata indicates if it needs rebase or not, so we can always get the
+                // correct result regardless of the "rebase mode" config.
+                Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+                  withSQLConf(
+                    SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> mode.toString) {
                     checkAnswer(
                       spark.read.parquet(path),
-                      Seq.tabulate(N)(_ => Row(Timestamp.valueOf(nonRebased))))
+                      Seq.tabulate(N)(_ => Row(Timestamp.valueOf(tsStr))))
                   }
                 }
+
+                // Force to not rebase to prove the written datetime values are rebased
+                // and we will get wrong result if we don't rebase while reading.
+                withSQLConf("spark.test.forceNoRebase" -> "true") {
+                  checkAnswer(
+                    spark.read.parquet(path),
+                    Seq.tabulate(N)(_ => Row(Timestamp.valueOf(nonRebased))))
+                }
               }
             }
           }
@@ -1027,26 +1015,24 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
             .parquet(path)
         }
 
-        Seq(false, true).foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-            // The file metadata indicates if it needs rebase or not, so we can always get the
-            // correct result regardless of the "rebase mode" config.
-            Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
-              withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
-                checkAnswer(
-                  spark.read.parquet(path),
-                  Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
-              }
-            }
-
-            // Force to not rebase to prove the written datetime values are rebased and we will get
-            // wrong result if we don't rebase while reading.
-            withSQLConf("spark.test.forceNoRebase" -> "true") {
+        withAllParquetReaders {
+          // The file metadata indicates if it needs rebase or not, so we can always get the
+          // correct result regardless of the "rebase mode" config.
+          Seq(LEGACY, CORRECTED, EXCEPTION).foreach { mode =>
+            withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_READ.key -> mode.toString) {
               checkAnswer(
                 spark.read.parquet(path),
-                Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07"))))
+                Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-01"))))
             }
           }
+
+          // Force to not rebase to prove the written datetime values are rebased and we will get
+          // wrong result if we don't rebase while reading.
+          withSQLConf("spark.test.forceNoRebase" -> "true") {
+            checkAnswer(
+              spark.read.parquet(path),
+              Seq.tabulate(N)(_ => Row(Date.valueOf("1001-01-07"))))
+          }
         }
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
index 7d75077a9732a..a14f6416199a1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetInteroperabilitySuite.scala
@@ -124,12 +124,11 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
       FileUtils.copyFile(new File(impalaPath), new File(tableDir, "part-00001.parq"))
 
       Seq(false, true).foreach { int96TimestampConversion =>
-        Seq(false, true).foreach { vectorized =>
+        withAllParquetReaders {
           withSQLConf(
               (SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key,
                 SQLConf.ParquetOutputTimestampType.INT96.toString),
-              (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key, int96TimestampConversion.toString()),
-              (SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key, vectorized.toString())
+              (SQLConf.PARQUET_INT96_TIMESTAMP_CONVERSION.key, int96TimestampConversion.toString())
           ) {
             val readBack = spark.read.parquet(tableDir.getAbsolutePath).collect()
             assert(readBack.size === 6)
@@ -149,7 +148,8 @@ class ParquetInteroperabilitySuite extends ParquetCompatibilityTest with SharedS
             val fullExpectations = (ts ++ impalaExpectations).map(_.toString).sorted.toArray
             val actual = readBack.map(_.getTimestamp(0).toString).sorted
             withClue(
-              s"int96TimestampConversion = $int96TimestampConversion; vectorized = $vectorized") {
+              s"int96TimestampConversion = $int96TimestampConversion; " +
+              s"vectorized = ${SQLConf.get.parquetVectorizedReaderEnabled}") {
               assert(fullExpectations === actual)
 
               // Now test that the behavior is still correct even with a filter which could get
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
index 917aaba2669ce..05d305a9b52ba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala
@@ -168,11 +168,9 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
       withTempPath { file =>
         val df = spark.createDataFrame(sparkContext.parallelize(data), schema)
         df.write.parquet(file.getCanonicalPath)
-        ("true" :: "false" :: Nil).foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-            val df2 = spark.read.parquet(file.getCanonicalPath)
-            checkAnswer(df2, df.collect().toSeq)
-          }
+        withAllParquetReaders {
+          val df2 = spark.read.parquet(file.getCanonicalPath)
+          checkAnswer(df2, df.collect().toSeq)
         }
       }
     }
@@ -791,15 +789,13 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
   }
 
   test("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") {
-    (true :: false :: Nil).foreach { vectorized =>
-      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
-        withTempPath { path =>
-          // Repeated values for dictionary encoding.
-          Seq(Some("A"), Some("A"), None).toDF.repartition(1)
-            .write.parquet(path.getAbsolutePath)
-          val df = spark.read.parquet(path.getAbsolutePath)
-          checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), df)
-        }
+    withAllParquetReaders {
+      withTempPath { path =>
+        // Repeated values for dictionary encoding.
+        Seq(Some("A"), Some("A"), None).toDF.repartition(1)
+          .write.parquet(path.getAbsolutePath)
+        val df = spark.read.parquet(path.getAbsolutePath)
+        checkAnswer(stripSparkFilter(df.where("NOT (value <=> 'A')")), df)
       }
     }
   }
@@ -821,10 +817,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS
         withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> toTsType) {
           write(df2.write.mode(SaveMode.Append))
         }
-        Seq("true", "false").foreach { vectorized =>
-          withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vectorized) {
-            checkAnswer(readback, df1.unionAll(df2))
-          }
+        withAllParquetReaders {
+          checkAnswer(readback, df1.unionAll(df2))
         }
       }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index c833d5f1ab1f6..f5726973d0abf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -162,4 +162,11 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   protected def getResourceParquetFilePath(name: String): String = {
     Thread.currentThread().getContextClassLoader.getResource(name).toString
   }
+
+  def withAllParquetReaders(code: => Unit): Unit = {
+    // test the row-based reader
+    withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false")(code)
+    // test the vectorized reader
+    withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true")(code)
+  }
 }

From ec80e4b5f80876378765544e0c4c6a4444f1a704 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 21 May 2020 23:34:11 -0700
Subject: [PATCH 0718/1280] [SPARK-31784][CORE][TEST] Fix test
 BarrierTaskContextSuite."share messages with allGather() call"

### What changes were proposed in this pull request?

Change from `messages.toList.iterator` to `Iterator.single(messages.toList)`.

### Why are the changes needed?

In this test, the expected result of `rdd2.collect().head` should actually be `List("0", "1", "2", "3")` but is `"0"` now.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Updated test.

Thanks WeichenXu123 reported this problem.

Closes #28596 from Ngone51/fix_allgather_test.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit 83d0967dcc6b205a3fd2003e051f49733f63cb30)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../spark/scheduler/BarrierTaskContextSuite.scala      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index b5614b263ca94..6191e41b4118f 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -69,12 +69,12 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
       // Pass partitionId message in
       val message: String = context.partitionId().toString
       val messages: Array[String] = context.allGather(message)
-      messages.toList.iterator
+      Iterator.single(messages.toList)
     }
-    // Take a sorted list of all the partitionId messages
-    val messages = rdd2.collect().head
-    // All the task partitionIds are shared
-    for((x, i) <- messages.view.zipWithIndex) assert(x.toString == i.toString)
+    val messages = rdd2.collect()
+    // All the task partitionIds are shared across all tasks
+    assert(messages.length === 4)
+    assert(messages.forall(_ == List("0", "1", "2", "3")))
   }
 
   test("throw exception if we attempt to synchronize with different blocking calls") {

From c7591ee8940dcfea819989e06fee8539e262b050 Mon Sep 17 00:00:00 2001
From: GuoPhilipse <guofei_ok@126.com>
Date: Fri, 22 May 2020 22:01:38 +0900
Subject: [PATCH 0719/1280] [SPARK-31790][DOCS] cast(long as timestamp) show
 different result between Hive and Spark

### What changes were proposed in this pull request?
add docs for sql migration-guide

### Why are the changes needed?
let user know more about the cast scenarios in which Hive and Spark generate different results

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
no need to test

Closes #28605 from GuoPhilipse/spark-docs.

Lead-authored-by: GuoPhilipse <guofei_ok@126.com>
Co-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 892b600ce3a579ea4acabf8ff378c19830a7d89c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index b0725d1889776..b7b01d07b9b26 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -955,3 +955,4 @@ Below are the scenarios in which Hive and Spark generate different results:
 * `SQRT(n)` If n < 0, Hive returns null, Spark SQL returns NaN.
 * `ACOS(n)` If n < -1 or n > 1, Hive returns null, Spark SQL returns NaN.
 * `ASIN(n)` If n < -1 or n > 1, Hive returns null, Spark SQL returns NaN.
+* `CAST(n AS TIMESTAMP)` If n is integral numbers, Hive treats n as milliseconds, Spark SQL treats n as seconds.

From 23019aa429d8f0db52b1ed5e9e6dc00ea7b94740 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 23 May 2020 08:43:16 +0900
Subject: [PATCH 0720/1280] [SPARK-31612][SQL][DOCS][FOLLOW-UP] Fix a few
 issues in SQL ref

### What changes were proposed in this pull request?
Fix a few issues in SQL Reference

### Why are the changes needed?
To make SQL Reference look better

### Does this PR introduce _any_ user-facing change?
Yes.
before:
<img width="189" alt="Screen Shot 2020-05-21 at 11 41 34 PM" src="https://user-images.githubusercontent.com/13592258/82639052-d0f38a80-9bbc-11ea-81a4-22def4ca5cc0.png">

after:

<img width="195" alt="Screen Shot 2020-05-21 at 11 41 17 PM" src="https://user-images.githubusercontent.com/13592258/82639063-d5b83e80-9bbc-11ea-84d1-8361e6bee949.png">

before:
<img width="763" alt="Screen Shot 2020-05-21 at 11 45 22 PM" src="https://user-images.githubusercontent.com/13592258/82639252-3e9fb680-9bbd-11ea-863c-e6a6c2f83a06.png">

after:

<img width="724" alt="Screen Shot 2020-05-21 at 11 45 02 PM" src="https://user-images.githubusercontent.com/13592258/82639265-42cbd400-9bbd-11ea-8df2-fc5c255b84d3.png">

before:
<img width="437" alt="Screen Shot 2020-05-21 at 11 41 57 PM" src="https://user-images.githubusercontent.com/13592258/82639072-db158900-9bbc-11ea-9963-731881cda4fd.png">

after

<img width="347" alt="Screen Shot 2020-05-21 at 11 42 26 PM" src="https://user-images.githubusercontent.com/13592258/82639082-dfda3d00-9bbc-11ea-9bd2-f922cc91f175.png">

### How was this patch tested?
Manually build and check

Closes #28608 from huaxingao/doc_fix.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit ad9532a09c70bf6acc8b79b4fdbfcd6afadcbc91)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml             | 42 ++++++++++++++--------------
 docs/sql-ref-syntax-aux-conf-mgmt.md |  2 +-
 docs/sql-ref-syntax-qry.md           | 35 ++++++++++++-----------
 docs/sql-ref-syntax.md               | 28 +++++++++++++------
 docs/sql-ref.md                      | 16 +++++------
 5 files changed, 67 insertions(+), 56 deletions(-)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 57fc493dad2f2..289a9d3a1e9da 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -76,14 +76,6 @@
 - text: SQL Reference
   url: sql-ref.html
   subitems:
-    - text: Data Types
-      url: sql-ref-datatypes.html
-    - text: Identifiers
-      url: sql-ref-identifier.html
-    - text: Literals
-      url: sql-ref-literals.html
-    - text: Null Semantics
-      url: sql-ref-null-semantics.html
     - text: ANSI Compliance
       url: sql-ref-ansi-compliance.html
       subitems:
@@ -93,6 +85,27 @@
           url: sql-ref-ansi-compliance.html#type-conversion
         - text: SQL Keywords
           url: sql-ref-ansi-compliance.html#sql-keywords
+    - text: Data Types
+      url: sql-ref-datatypes.html
+    - text: Datetime Pattern
+      url: sql-ref-datetime-pattern.html
+    - text: Functions
+      url: sql-ref-functions.html
+      subitems:
+      - text: Built-in Functions
+        url: sql-ref-functions-builtin.html
+      - text: Scalar UDFs (User-Defined Functions)
+        url: sql-ref-functions-udf-scalar.html
+      - text: UDAFs (User-Defined Aggregate Functions)
+        url: sql-ref-functions-udf-aggregate.html
+      - text: Integration with Hive UDFs/UDAFs/UDTFs
+        url: sql-ref-functions-udf-hive.html
+    - text: Identifiers
+      url: sql-ref-identifier.html
+    - text: Literals
+      url: sql-ref-literals.html
+    - text: Null Semantics
+      url: sql-ref-null-semantics.html
     - text: SQL Syntax
       url: sql-ref-syntax.html
       subitems:
@@ -247,16 +260,3 @@
                   url: sql-ref-syntax-aux-resource-mgmt-list-file.html
                 - text: LIST JAR
                   url: sql-ref-syntax-aux-resource-mgmt-list-jar.html
-    - text: Functions
-      url: sql-ref-functions.html
-      subitems:
-      - text: Built-in Functions
-        url: sql-ref-functions-builtin.html
-      - text: Scalar UDFs (User-Defined Functions)
-        url: sql-ref-functions-udf-scalar.html
-      - text: UDAFs (User-Defined Aggregate Functions)
-        url: sql-ref-functions-udf-aggregate.html
-      - text: Integration with Hive UDFs/UDAFs/UDTFs
-        url: sql-ref-functions-udf-hive.html
-    - text: Datetime Pattern
-      url: sql-ref-datetime-pattern.html
diff --git a/docs/sql-ref-syntax-aux-conf-mgmt.md b/docs/sql-ref-syntax-aux-conf-mgmt.md
index f5e48ef2fee30..1900fb7f1cb9a 100644
--- a/docs/sql-ref-syntax-aux-conf-mgmt.md
+++ b/docs/sql-ref-syntax-aux-conf-mgmt.md
@@ -20,4 +20,4 @@ license: |
 ---
 
  * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
- * [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 1171fead55e30..8accdfe30764a 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -27,21 +27,22 @@ to SELECT are also included in this section. Spark also provides the
 ability to generate logical and physical plan for a given query using
 [EXPLAIN](sql-ref-syntax-qry-explain.html) statement.
 
-* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
-* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
-* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
-* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-* [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
-* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
-* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
-* [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
-* [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
-* [JOIN](sql-ref-syntax-qry-select-join.html)
-* [Join Hints](sql-ref-syntax-qry-select-hints.html)
-* [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
-* [Set Operators](sql-ref-syntax-qry-select-setops.html)
-* [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
-* [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
-* [Window Function](sql-ref-syntax-qry-window.html)
+* [SELECT Statement](sql-ref-syntax-qry-select.html)
+  * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+  * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+  * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+  * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+  * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+  * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+  * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+  * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+  * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+  * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+  * [JOIN](sql-ref-syntax-qry-select-join.html)
+  * [Join Hints](sql-ref-syntax-qry-select-hints.html)
+  * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+  * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+  * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+  * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+  * [Window Function](sql-ref-syntax-qry-window.html)
 * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 94bd476ffb7b1..98e30652245f6 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -48,15 +48,25 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
 
 ### Data Retrieval Statements
 
- * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
- * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+ * [SELECT Statement](sql-ref-syntax-qry-select.html)
+   * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+   * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
+   * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+   * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+   * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+   * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+   * [JOIN](sql-ref-syntax-qry-select-join.html)
+   * [Join Hints](sql-ref-syntax-qry-select-hints.html)
+   * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
+   * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+   * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+   * [Set Operators](sql-ref-syntax-qry-select-setops.html)
+   * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+   * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+   * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+   * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+   * [Window Function](sql-ref-syntax-qry-window.html)
  * [EXPLAIN](sql-ref-syntax-qry-explain.html)
- * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
- * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
- * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
- * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
- * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
- * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
 
 ### Auxiliary Statements
 
@@ -73,6 +83,7 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
  * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
  * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
  * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
  * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)
  * [SHOW CREATE TABLE](sql-ref-syntax-aux-show-create-table.html)
@@ -84,4 +95,3 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [SHOW TBLPROPERTIES](sql-ref-syntax-aux-show-tblproperties.html)
  * [SHOW VIEWS](sql-ref-syntax-aux-show-views.html)
  * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
- * [UNSET](sql-ref-syntax-aux-conf-mgmt-reset.html)
diff --git a/docs/sql-ref.md b/docs/sql-ref.md
index db51fe1978eec..f88026b7abf02 100644
--- a/docs/sql-ref.md
+++ b/docs/sql-ref.md
@@ -21,19 +21,19 @@ license: |
 
 Spark SQL is Apache Spark's module for working with structured data. This guide is a reference for Structured Query Language (SQL) and includes syntax, semantics, keywords, and examples for common SQL usage. It contains information for the following topics:
 
+ * [ANSI Compliance](sql-ref-ansi-compliance.html)
  * [Data Types](sql-ref-datatypes.html)
+ * [Datetime Pattern](sql-ref-datetime-pattern.html)
+ * [Functions](sql-ref-functions.html)
+   * [Built-in Functions](sql-ref-functions-builtin.html)
+   * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
+   * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
+   * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
  * [Identifiers](sql-ref-identifier.html)
  * [Literals](sql-ref-literals.html)
  * [Null Semanitics](sql-ref-null-semantics.html)
- * [ANSI Compliance](sql-ref-ansi-compliance.html)
  * [SQL Syntax](sql-ref-syntax.html)
    * [DDL Statements](sql-ref-syntax-ddl.html)
-   * [DML Statements](sql-ref-syntax-ddl.html)
+   * [DML Statements](sql-ref-syntax-dml.html)
    * [Data Retrieval Statements](sql-ref-syntax-qry.html)
    * [Auxiliary Statements](sql-ref-syntax-aux.html)
- * [Functions](sql-ref-functions.html)
-   * [Built-in Functions](sql-ref-functions-builtin.html)
-   * [Scalar User-Defined Functions (UDFs)](sql-ref-functions-udf-scalar.html)
-   * [User-Defined Aggregate Functions (UDAFs)](sql-ref-functions-udf-aggregate.html)
-   * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
- * [Datetime Pattern](sql-ref-datetime-pattern.html)

From 218334523dacd116a03f2340ad89e33abe93e452 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sat, 23 May 2020 08:48:29 +0900
Subject: [PATCH 0721/1280] [SPARK-29854][SQL][TESTS] Add tests to check
 lpad/rpad throw an exception for invalid length input

### What changes were proposed in this pull request?

This PR intends to add trivial tests to check https://github.com/apache/spark/pull/27024 has already been fixed in the master.

Closes #27024

### Why are the changes needed?

For test coverage.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #28604 from maropu/SPARK-29854.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 7ca73f03fbc6e213c30e725bf480709ed036a376)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../inputs/ansi/string-functions.sql          |   1 +
 .../sql-tests/inputs/string-functions.sql     |   6 +-
 .../results/ansi/string-functions.sql.out     | 296 ++++++++++++++++++
 .../results/string-functions.sql.out          |  18 +-
 4 files changed, 319 insertions(+), 2 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out

diff --git a/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
new file mode 100644
index 0000000000000..dd28e9b97fb20
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/ansi/string-functions.sql
@@ -0,0 +1 @@
+--IMPORT string-functions.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
index 8e33471e8b129..f5ed2036dc8ac 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/string-functions.sql
@@ -48,4 +48,8 @@ SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz');
 SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD');
 SELECT trim(TRAILING 'xyz' FROM 'testxxzx');
 SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx');
-SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy');
\ No newline at end of file
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy');
+
+-- Check lpad/rpad with invalid length parameter
+SELECT lpad('hi', 'invalid_length');
+SELECT rpad('hi', 'invalid_length');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
new file mode 100644
index 0000000000000..b507713a73d1f
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -0,0 +1,296 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 36
+
+
+-- !query
+select concat_ws()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+requirement failed: concat_ws requires at least one argument.; line 1 pos 7
+
+
+-- !query
+select format_string()
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+requirement failed: format_string() should take at least 1 argument; line 1 pos 7
+
+
+-- !query
+select 'a' || 'b' || 'c'
+-- !query schema
+struct<concat(concat(a, b), c):string>
+-- !query output
+abc
+
+
+-- !query
+select replace('abc', 'b', '123')
+-- !query schema
+struct<replace(abc, b, 123):string>
+-- !query output
+a123c
+
+
+-- !query
+select replace('abc', 'b')
+-- !query schema
+struct<replace(abc, b, ):string>
+-- !query output
+ac
+
+
+-- !query
+select length(uuid()), (uuid() <> uuid())
+-- !query schema
+struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
+-- !query output
+36	true
+
+
+-- !query
+select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
+-- !query schema
+struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+-- !query output
+4	NULL	NULL
+
+
+-- !query
+select left("abcd", 2), left("abcd", 5), left("abcd", '2'), left("abcd", null)
+-- !query schema
+struct<left(abcd, 2):string,left(abcd, 5):string,left(abcd, 2):string,left(abcd, NULL):string>
+-- !query output
+ab	abcd	ab	NULL
+
+
+-- !query
+select left(null, -2), left("abcd", -2), left("abcd", 0), left("abcd", 'a')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: a
+
+
+-- !query
+select right("abcd", 2), right("abcd", 5), right("abcd", '2'), right("abcd", null)
+-- !query schema
+struct<right(abcd, 2):string,right(abcd, 5):string,right(abcd, 2):string,right(abcd, NULL):string>
+-- !query output
+cd	abcd	cd	NULL
+
+
+-- !query
+select right(null, -2), right("abcd", -2), right("abcd", 0), right("abcd", 'a')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: a
+
+
+-- !query
+SELECT split('aa1cc2ee3', '[1-9]+')
+-- !query schema
+struct<split(aa1cc2ee3, [1-9]+, -1):array<string>>
+-- !query output
+["aa","cc","ee",""]
+
+
+-- !query
+SELECT split('aa1cc2ee3', '[1-9]+', 2)
+-- !query schema
+struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
+-- !query output
+["aa","cc2ee3"]
+
+
+-- !query
+SELECT substr('Spark SQL', 5)
+-- !query schema
+struct<substr(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substr('Spark SQL', -3)
+-- !query schema
+struct<substr(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substr('Spark SQL', 5, 1)
+-- !query schema
+struct<substr(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substr('Spark SQL' from 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substr('Spark SQL' from -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substr('Spark SQL' from 5 for 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substring('Spark SQL', 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substring('Spark SQL', -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substring('Spark SQL', 5, 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT substring('Spark SQL' from 5)
+-- !query schema
+struct<substring(Spark SQL, 5, 2147483647):string>
+-- !query output
+k SQL
+
+
+-- !query
+SELECT substring('Spark SQL' from -3)
+-- !query schema
+struct<substring(Spark SQL, -3, 2147483647):string>
+-- !query output
+SQL
+
+
+-- !query
+SELECT substring('Spark SQL' from 5 for 1)
+-- !query schema
+struct<substring(Spark SQL, 5, 1):string>
+-- !query output
+k
+
+
+-- !query
+SELECT trim(" xyz "), ltrim(" xyz "), rtrim(" xyz ")
+-- !query schema
+struct<trim( xyz ):string,ltrim( xyz ):string,rtrim( xyz ):string>
+-- !query output
+xyz	xyz 	 xyz
+
+
+-- !query
+SELECT trim(BOTH 'xyz' FROM 'yxTomxx'), trim('xyz' FROM 'yxTomxx')
+-- !query schema
+struct<TRIM(BOTH xyz FROM yxTomxx):string,TRIM(BOTH xyz FROM yxTomxx):string>
+-- !query output
+Tom	Tom
+
+
+-- !query
+SELECT trim(BOTH 'x' FROM 'xxxbarxxx'), trim('x' FROM 'xxxbarxxx')
+-- !query schema
+struct<TRIM(BOTH x FROM xxxbarxxx):string,TRIM(BOTH x FROM xxxbarxxx):string>
+-- !query output
+bar	bar
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytest')
+-- !query schema
+struct<TRIM(LEADING xyz FROM zzzytest):string>
+-- !query output
+test
+
+
+-- !query
+SELECT trim(LEADING 'xyz' FROM 'zzzytestxyz')
+-- !query schema
+struct<TRIM(LEADING xyz FROM zzzytestxyz):string>
+-- !query output
+testxyz
+
+
+-- !query
+SELECT trim(LEADING 'xy' FROM 'xyxXxyLAST WORD')
+-- !query schema
+struct<TRIM(LEADING xy FROM xyxXxyLAST WORD):string>
+-- !query output
+XxyLAST WORD
+
+
+-- !query
+SELECT trim(TRAILING 'xyz' FROM 'testxxzx')
+-- !query schema
+struct<TRIM(TRAILING xyz FROM testxxzx):string>
+-- !query output
+test
+
+
+-- !query
+SELECT trim(TRAILING 'xyz' FROM 'xyztestxxzx')
+-- !query schema
+struct<TRIM(TRAILING xyz FROM xyztestxxzx):string>
+-- !query output
+xyztest
+
+
+-- !query
+SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
+-- !query schema
+struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
+-- !query output
+TURNERyxX
+
+
+-- !query
+SELECT lpad('hi', 'invalid_length')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: invalid_length
+
+
+-- !query
+SELECT rpad('hi', 'invalid_length')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: invalid_length
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 43c18f5417110..cb5501ddb67af 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 34
+-- Number of queries: 36
 
 
 -- !query
@@ -274,3 +274,19 @@ SELECT trim(TRAILING 'xy' FROM 'TURNERyxXxy')
 struct<TRIM(TRAILING xy FROM TURNERyxXxy):string>
 -- !query output
 TURNERyxX
+
+
+-- !query
+SELECT lpad('hi', 'invalid_length')
+-- !query schema
+struct<lpad(hi, CAST(invalid_length AS INT),  ):string>
+-- !query output
+NULL
+
+
+-- !query
+SELECT rpad('hi', 'invalid_length')
+-- !query schema
+struct<rpad(hi, CAST(invalid_length AS INT),  ):string>
+-- !query output
+NULL

From d8b788e60508e5e6475d10c383035a3e0e1ac39e Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 23 May 2020 14:57:30 -0700
Subject: [PATCH 0722/1280] [SPARK-29854][SQL][TESTS][FOLLOWUP] Regenerate
 string-functions.sql.out

---
 .../resources/sql-tests/results/ansi/string-functions.sql.out | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
index b507713a73d1f..3329150299ec6 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -63,7 +63,7 @@ struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):
 -- !query
 select left("abcd", 2), left("abcd", 5), left("abcd", '2'), left("abcd", null)
 -- !query schema
-struct<left(abcd, 2):string,left(abcd, 5):string,left(abcd, 2):string,left(abcd, NULL):string>
+struct<left('abcd', 2):string,left('abcd', 5):string,left('abcd', '2'):string,left('abcd', NULL):string>
 -- !query output
 ab	abcd	ab	NULL
 
@@ -80,7 +80,7 @@ invalid input syntax for type numeric: a
 -- !query
 select right("abcd", 2), right("abcd", 5), right("abcd", '2'), right("abcd", null)
 -- !query schema
-struct<right(abcd, 2):string,right(abcd, 5):string,right(abcd, 2):string,right(abcd, NULL):string>
+struct<right('abcd', 2):string,right('abcd', 5):string,right('abcd', '2'):string,right('abcd', NULL):string>
 -- !query output
 cd	abcd	cd	NULL
 

From 5f966f91571b4d332dc56914bee3751b7fb19a71 Mon Sep 17 00:00:00 2001
From: Onur Satici <onursatici@gmail.com>
Date: Wed, 5 Feb 2020 01:17:30 -0800
Subject: [PATCH 0723/1280] [SPARK-30715][K8S] Bump fabric8 to 4.7.1

### What changes were proposed in this pull request?
Bump fabric8 kubernetes-client to 4.7.1

### Why are the changes needed?
New fabric8 version brings support for Kubernetes 1.17 clusters.
Full release notes:
- https://github.com/fabric8io/kubernetes-client/releases/tag/v4.7.0
- https://github.com/fabric8io/kubernetes-client/releases/tag/v4.7.1

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing unit and integration tests cover creation of K8S objects. Adjusted them to work with the new fabric8 version

Closes #27443 from onursatici/os/bump-fabric8.

Authored-by: Onur Satici <onursatici@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 86fdb818bf5dfde7744bf2b358876af361ec9a68)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2       |  6 ++---
 dev/deps/spark-deps-hadoop-2.7-hive-2.3       |  6 ++---
 dev/deps/spark-deps-hadoop-3.2-hive-2.3       |  6 ++---
 resource-managers/kubernetes/core/pom.xml     |  2 +-
 .../spark/deploy/k8s/KubernetesUtils.scala    |  6 ++---
 .../k8s/features/BasicDriverFeatureStep.scala | 10 +++------
 .../features/BasicExecutorFeatureStep.scala   | 12 +++-------
 .../features/MountVolumesFeatureStep.scala    |  2 +-
 .../spark/deploy/k8s/PodBuilderSuite.scala    |  5 ++---
 .../BasicDriverFeatureStepSuite.scala         | 22 ++++++++++---------
 .../BasicExecutorFeatureStepSuite.scala       | 15 ++++++++-----
 .../MountVolumesFeatureStepSuite.scala        |  5 +++--
 .../k8s/integrationtest/DepsTestsSuite.scala  |  8 ++-----
 .../k8s/integrationtest/PVTestsSuite.scala    |  5 ++---
 14 files changed, 49 insertions(+), 61 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index b307e573a3511..b375629c37433 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -137,9 +137,9 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
-kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
-kubernetes-model/4.6.4//kubernetes-model-4.6.4.jar
+kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
+kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
+kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index b6d6e34aa49b0..093924f58cb20 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -152,9 +152,9 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
-kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
-kubernetes-model/4.6.4//kubernetes-model-4.6.4.jar
+kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
+kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
+kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index c8d2c827249b7..2db8d3e74d0e6 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -164,9 +164,9 @@ kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar
 kerby-util/1.0.1//kerby-util-1.0.1.jar
 kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.6.4//kubernetes-client-4.6.4.jar
-kubernetes-model-common/4.6.4//kubernetes-model-common-4.6.4.jar
-kubernetes-model/4.6.4//kubernetes-model-4.6.4.jar
+kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
+kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
+kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 9a48bf37b2cd5..9d35c8139c903 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -29,7 +29,7 @@
   <name>Spark Project Kubernetes</name>
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
-    <kubernetes.client.version>4.6.4</kubernetes.client.version>
+    <kubernetes.client.version>4.7.1</kubernetes.client.version>
   </properties>
 
   <dependencies>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
index e234b1780a7d0..c49f4a15de974 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala
@@ -23,7 +23,7 @@ import java.util.UUID
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, PodBuilder, Quantity, QuantityBuilder}
+import io.fabric8.kubernetes.api.model.{Container, ContainerBuilder, ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, PodBuilder, Quantity}
 import io.fabric8.kubernetes.client.KubernetesClient
 import org.apache.commons.codec.binary.Hex
 import org.apache.hadoop.fs.{FileSystem, Path}
@@ -234,9 +234,7 @@ private[spark] object KubernetesUtils extends Logging {
         throw new SparkException(s"Resource: ${request.id.resourceName} was requested, " +
           "but vendor was not specified.")
       }
-      val quantity = new QuantityBuilder(false)
-        .withAmount(request.amount.toString)
-        .build()
+      val quantity = new Quantity(request.amount.toString)
       (KubernetesConf.buildKubernetesResourceName(vendorDomain, request.id.resourceName), quantity)
     }.toMap
   }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index 1edd3f725b56b..63f18129328b0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -80,14 +80,10 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
           .build()
       }
 
-    val driverCpuQuantity = new QuantityBuilder(false)
-      .withAmount(driverCoresRequest)
-      .build()
-    val driverMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${driverMemoryWithOverheadMiB}Mi")
-      .build()
+    val driverCpuQuantity = new Quantity(driverCoresRequest)
+    val driverMemoryQuantity = new Quantity(s"${driverMemoryWithOverheadMiB}Mi")
     val maybeCpuLimitQuantity = driverLimitCores.map { limitCores =>
-      ("cpu", new QuantityBuilder(false).withAmount(limitCores).build())
+      ("cpu", new Quantity(limitCores))
     }
 
     val driverResourceQuantities =
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index d88bd5858bc94..6a26df2997fd2 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -88,12 +88,8 @@ private[spark] class BasicExecutorFeatureStep(
       // Replace dangerous characters in the remaining string with a safe alternative.
       .replaceAll("[^\\w-]+", "_")
 
-    val executorMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${executorMemoryTotal}Mi")
-      .build()
-    val executorCpuQuantity = new QuantityBuilder(false)
-      .withAmount(executorCoresRequest)
-      .build()
+    val executorMemoryQuantity = new Quantity(s"${executorMemoryTotal}Mi")
+    val executorCpuQuantity = new Quantity(executorCoresRequest)
 
     val executorResourceQuantities =
       KubernetesUtils.buildResourcesQuantities(SPARK_EXECUTOR_PREFIX,
@@ -183,9 +179,7 @@ private[spark] class BasicExecutorFeatureStep(
       .addToArgs("executor")
       .build()
     val containerWithLimitCores = executorLimitCores.map { limitCores =>
-      val executorCpuLimitQuantity = new QuantityBuilder(false)
-        .withAmount(limitCores)
-        .build()
+      val executorCpuLimitQuantity = new Quantity(limitCores)
       new ContainerBuilder(executorContainer)
         .editResources()
           .addToLimits("cpu", executorCpuLimitQuantity)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
index 8548e7057cdf0..4599df99b3c61 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
@@ -65,7 +65,7 @@ private[spark] class MountVolumesFeatureStep(conf: KubernetesConf)
           new VolumeBuilder()
             .withEmptyDir(
               new EmptyDirVolumeSource(medium.getOrElse(""),
-              new Quantity(sizeLimit.orNull)))
+                sizeLimit.map(new Quantity(_)).orNull))
       }
 
       val volume = volumeBuilder.withName(spec.volumeName).build()
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
index 707c823d69cf0..26bd317de8ec6 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/PodBuilderSuite.scala
@@ -101,8 +101,7 @@ abstract class PodBuilderSuite extends SparkFunSuite {
     assert(container.getArgs.contains("arg"))
     assert(container.getCommand.equals(List("command").asJava))
     assert(container.getEnv.asScala.exists(_.getName == "env-key"))
-    assert(container.getResources.getLimits.get("gpu") ===
-      new QuantityBuilder().withAmount("1").build())
+    assert(container.getResources.getLimits.get("gpu") === new Quantity("1"))
     assert(container.getSecurityContext.getRunAsNonRoot)
     assert(container.getStdin)
     assert(container.getTerminationMessagePath === "termination-message-path")
@@ -156,7 +155,7 @@ abstract class PodBuilderSuite extends SparkFunSuite {
           .withImagePullPolicy("Always")
           .withName("executor-container")
           .withNewResources()
-            .withLimits(Map("gpu" -> new QuantityBuilder().withAmount("1").build()).asJava)
+            .withLimits(Map("gpu" -> new Quantity("1")).asJava)
             .endResources()
           .withNewSecurityContext()
             .withRunAsNonRoot(true)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index 89e7ff95b5124..c8c934b436ffa 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
 
 import scala.collection.JavaConverters._
 
-import io.fabric8.kubernetes.api.model.{ContainerPort, ContainerPortBuilder, LocalObjectReferenceBuilder}
+import io.fabric8.kubernetes.api.model.{ContainerPort, ContainerPortBuilder, LocalObjectReferenceBuilder, Quantity}
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s.{KubernetesTestConf, SparkPod}
@@ -105,13 +105,13 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
 
     val resourceRequirements = configuredPod.container.getResources
     val requests = resourceRequirements.getRequests.asScala
-    assert(requests("cpu").getAmount === "2")
-    assert(requests("memory").getAmount === "456Mi")
+    assert(amountAndFormat(requests("cpu")) === "2")
+    assert(amountAndFormat(requests("memory")) === "456Mi")
     val limits = resourceRequirements.getLimits.asScala
-    assert(limits("memory").getAmount === "456Mi")
-    assert(limits("cpu").getAmount === "4")
+    assert(amountAndFormat(limits("memory")) === "456Mi")
+    assert(amountAndFormat(limits("cpu")) === "4")
     resources.foreach { case (k8sName, testRInfo) =>
-      assert(limits(k8sName).getAmount === testRInfo.count)
+      assert(amountAndFormat(limits(k8sName)) === testRInfo.count)
     }
 
     val driverPodMetadata = configuredPod.pod.getMetadata
@@ -140,7 +140,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
       .configurePod(basePod)
       .container.getResources
       .getRequests.asScala
-    assert(requests1("cpu").getAmount === "1")
+    assert(amountAndFormat(requests1("cpu")) === "1")
 
     // if spark.driver.cores is set it should be used
     sparkConf.set(DRIVER_CORES, 10)
@@ -148,7 +148,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
       .configurePod(basePod)
       .container.getResources
       .getRequests.asScala
-    assert(requests2("cpu").getAmount === "10")
+    assert(amountAndFormat(requests2("cpu")) === "10")
 
     // spark.kubernetes.driver.request.cores should be preferred over spark.driver.cores
     Seq("0.1", "100m").foreach { value =>
@@ -157,7 +157,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
         .configurePod(basePod)
         .container.getResources
         .getRequests.asScala
-      assert(requests3("cpu").getAmount === value)
+      assert(amountAndFormat(requests3("cpu")) === value)
     }
   }
 
@@ -203,7 +203,7 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
         mainAppResource = resource)
       val step = new BasicDriverFeatureStep(conf)
       val pod = step.configurePod(SparkPod.initialPod())
-      val mem = pod.container.getResources.getRequests.get("memory").getAmount()
+      val mem = amountAndFormat(pod.container.getResources.getRequests.get("memory"))
       val expected = (driverMem + driverMem * expectedFactor).toInt
       assert(mem === s"${expected}Mi")
 
@@ -218,4 +218,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
       .withContainerPort(portNumber)
       .withProtocol("TCP")
       .build()
+
+  private def amountAndFormat(quantity: Quantity): String = quantity.getAmount + quantity.getFormat
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index f375b1fe6a5cd..da50372d04c73 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -128,10 +128,11 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val executor = step.configurePod(SparkPod.initialPod())
 
     assert(executor.container.getResources.getLimits.size() === 3)
-    assert(executor.container.getResources
-      .getLimits.get("memory").getAmount === "1408Mi")
+    assert(amountAndFormat(executor.container.getResources
+      .getLimits.get("memory")) === "1408Mi")
     gpuResources.foreach { case (k8sName, testRInfo) =>
-      assert(executor.container.getResources.getLimits.get(k8sName).getAmount === testRInfo.count)
+      assert(amountAndFormat(
+        executor.container.getResources.getLimits.get(k8sName)) === testRInfo.count)
     }
   }
 
@@ -151,8 +152,8 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     assert(executor.container.getImage === EXECUTOR_IMAGE)
     assert(executor.container.getVolumeMounts.isEmpty)
     assert(executor.container.getResources.getLimits.size() === 1)
-    assert(executor.container.getResources
-      .getLimits.get("memory").getAmount === "1408Mi")
+    assert(amountAndFormat(executor.container.getResources
+      .getLimits.get("memory")) === "1408Mi")
 
     // The pod has no node selector, volumes.
     assert(executor.pod.getSpec.getNodeSelector.isEmpty)
@@ -201,7 +202,7 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf))
     val executor = step.configurePod(SparkPod.initialPod())
     // This is checking that basic executor + executorMemory = 1408 + 42 = 1450
-    assert(executor.container.getResources.getRequests.get("memory").getAmount === "1450Mi")
+    assert(amountAndFormat(executor.container.getResources.getRequests.get("memory")) === "1450Mi")
   }
 
   test("auth secret propagation") {
@@ -273,4 +274,6 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     val expectedEnvs = defaultEnvs ++ additionalEnvVars ++ extraJavaOptsEnvs
     assert(containerEnvs === expectedEnvs)
   }
+
+  private def amountAndFormat(quantity: Quantity): String = quantity.getAmount + quantity.getFormat
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
index 8c430eeb3fa71..3888062785324 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
@@ -79,7 +79,8 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val emptyDir = configuredPod.pod.getSpec.getVolumes.get(0).getEmptyDir
     assert(emptyDir.getMedium === "Memory")
-    assert(emptyDir.getSizeLimit.getAmount === "6G")
+    assert(emptyDir.getSizeLimit.getAmount ===  "6")
+    assert(emptyDir.getSizeLimit.getFormat === "G")
     assert(configuredPod.container.getVolumeMounts.size() === 1)
     assert(configuredPod.container.getVolumeMounts.get(0).getMountPath === "/tmp")
     assert(configuredPod.container.getVolumeMounts.get(0).getName === "testVolume")
@@ -101,7 +102,7 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val emptyDir = configuredPod.pod.getSpec.getVolumes.get(0).getEmptyDir
     assert(emptyDir.getMedium === "")
-    assert(emptyDir.getSizeLimit.getAmount === null)
+    assert(emptyDir.getSizeLimit === null)
     assert(configuredPod.container.getVolumeMounts.size() === 1)
     assert(configuredPod.container.getVolumeMounts.get(0).getMountPath === "/tmp")
     assert(configuredPod.container.getVolumeMounts.get(0).getName === "testVolume")
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index c35aa5cc7a0b1..2d90c06e36390 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -53,12 +53,8 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     ).toArray
 
     val resources = Map(
-      "cpu" -> new QuantityBuilder()
-        .withAmount("1")
-        .build(),
-      "memory" -> new QuantityBuilder()
-        .withAmount("512M")
-        .build()
+      "cpu" -> new Quantity("1"),
+      "memory" -> new Quantity("512M")
     ).asJava
 
     new ContainerBuilder()
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index a7cb84eb4b371..86f8cddcef7da 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -45,7 +45,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
         .withName("test-local-pv")
       .endMetadata()
       .withNewSpec()
-        .withCapacity(Map("storage" -> new QuantityBuilder().withAmount("1Gi").build()).asJava)
+        .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava)
         .withAccessModes("ReadWriteOnce")
         .withPersistentVolumeReclaimPolicy("Retain")
         .withStorageClassName("test-local-storage")
@@ -72,8 +72,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
         .withAccessModes("ReadWriteOnce")
         .withStorageClassName("test-local-storage")
         .withResources(new ResourceRequirementsBuilder()
-        .withRequests(Map("storage" -> new QuantityBuilder()
-          .withAmount("1Gi").build()).asJava).build())
+          .withRequests(Map("storage" -> new Quantity("1Gi")).asJava).build())
       .endSpec()
 
     kubernetesTestComponents

From 1e79c0ddb23ff159f6023b9840d0dc0bdee289c3 Mon Sep 17 00:00:00 2001
From: Prashant Sharma <prashsh1@in.ibm.com>
Date: Sat, 21 Mar 2020 18:26:53 -0700
Subject: [PATCH 0724/1280] [SPARK-30715][K8S][TESTS][FOLLOWUP] Update k8s
 client version in IT as well

### What changes were proposed in this pull request?
This is a follow up for SPARK-30715 . Kubernetes client version in sync in integration-tests and kubernetes/core

### Why are the changes needed?
More than once, the kubernetes client version has gone out of sync between integration tests and kubernetes/core. So brought them up in sync again and added a comment to save us from future need of this additional followup.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually.

Closes #27948 from ScrapCodes/follow-up-spark-30715.

Authored-by: Prashant Sharma <prashsh1@in.ibm.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 3799d2b9d842f4b9f4e78bf701f5e123f0061bad)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 resource-managers/kubernetes/core/pom.xml                 | 1 +
 resource-managers/kubernetes/integration-tests/pom.xml    | 2 +-
 .../deploy/k8s/integrationtest/KubernetesSuite.scala      | 8 ++++----
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 9d35c8139c903..36d9543f61540 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -29,6 +29,7 @@
   <name>Spark Project Kubernetes</name>
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
+    <!-- Note: Please update the kubernetes client version in kubernetes/integration-tests/pom.xml -->
     <kubernetes.client.version>4.7.1</kubernetes.client.version>
   </properties>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 3222ec7db689c..92ddeaef2273c 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -29,7 +29,7 @@
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
     <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
-    <kubernetes-client.version>4.6.4</kubernetes-client.version>
+    <kubernetes-client.version>4.7.1</kubernetes-client.version>
     <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
     <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 00996df416439..dbb84e3e77dbd 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -59,15 +59,15 @@ class KubernetesSuite extends SparkFunSuite
   protected var appLocator: String = _
 
   // Default memory limit is 1024M + 384M (minimum overhead constant)
-  private val baseMemory = s"${1024 + 384}Mi"
+  private val baseMemory = s"${1024 + 384}"
   protected val memOverheadConstant = 0.8
-  private val standardNonJVMMemory = s"${(1024 + 0.4*1024).toInt}Mi"
+  private val standardNonJVMMemory = s"${(1024 + 0.4*1024).toInt}"
   protected val additionalMemory = 200
   // 209715200 is 200Mi
   protected val additionalMemoryInBytes = 209715200
-  private val extraDriverTotalMemory = s"${(1024 + memOverheadConstant*1024).toInt}Mi"
+  private val extraDriverTotalMemory = s"${(1024 + memOverheadConstant*1024).toInt}"
   private val extraExecTotalMemory =
-    s"${(1024 + memOverheadConstant*1024 + additionalMemory).toInt}Mi"
+    s"${(1024 + memOverheadConstant*1024 + additionalMemory).toInt}"
 
   /**
    * Build the image ref for the given image name, taking the repo and tag from the

From f05a26a814c66f61dc9f742ae58ebb0d8787fa60 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 23 May 2020 11:07:45 -0700
Subject: [PATCH 0725/1280] [SPARK-31786][K8S][BUILD] Upgrade kubernetes-client
 to 4.9.2

### What changes were proposed in this pull request?

This PR aims to upgrade `kubernetes-client` library to bring the JDK8 related fixes. Please note that JDK11 works fine without any problem.
- https://github.com/fabric8io/kubernetes-client/releases/tag/v4.9.2
  - JDK8 always uses http/1.1 protocol (Prevent OkHttp from wrongly enabling http/2)

### Why are the changes needed?

OkHttp "wrongly" detects the Platform as Jdk9Platform on JDK 8u251.
- https://github.com/fabric8io/kubernetes-client/issues/2212
- https://stackoverflow.com/questions/61565751/why-am-i-not-able-to-run-sparkpi-example-on-a-kubernetes-k8s-cluster

Although there is a workaround `export HTTP2_DISABLE=true` and `Downgrade JDK or K8s`, we had better avoid this problematic situation.

### Does this PR introduce _any_ user-facing change?

No. This will recover the failures on JDK 8u252.

### How was this patch tested?

- [x] Pass the Jenkins UT (https://github.com/apache/spark/pull/28601#issuecomment-632474270)
- [x] Pass the Jenkins K8S IT with the K8s 1.13 (https://github.com/apache/spark/pull/28601#issuecomment-632438452)
- [x] Manual testing with K8s 1.17.3. (Below)

**v1.17.6 result (on Minikube)**
```
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- All pods have the same service account by default
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
- Start pod creation from template
- PVs with local storage
- Launcher client dependencies
- Test basic decommissioning
Run completed in 8 minutes, 27 seconds.
Total number of tests run: 19
Suites: completed 2, aborted 0
Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```

Closes #28601 from dongjoon-hyun/SPARK-K8S-CLIENT.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 64ffc6649623e3cb568315f57c9e06be3b547c00)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2                | 7 ++++---
 dev/deps/spark-deps-hadoop-2.7-hive-2.3                | 7 ++++---
 dev/deps/spark-deps-hadoop-3.2-hive-2.3                | 7 ++++---
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 5 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index b375629c37433..82d5a06f906d8 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -93,6 +93,7 @@ jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
@@ -137,9 +138,9 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
-kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
-kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
+kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar
+kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar
+kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 093924f58cb20..17c787e42fdba 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -106,6 +106,7 @@ jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
 jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations/2.10.0//jackson-module-jaxb-annotations-2.10.0.jar
@@ -152,9 +153,9 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/1.7.30//jul-to-slf4j-1.7.30.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
-kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
-kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
+kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar
+kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar
+kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 2db8d3e74d0e6..3c3ce2dcdd6d4 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -105,6 +105,7 @@ jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.10.0//jackson-core-2.10.0.jar
 jackson-databind/2.10.0//jackson-databind-2.10.0.jar
 jackson-dataformat-yaml/2.10.0//jackson-dataformat-yaml-2.10.0.jar
+jackson-datatype-jsr310/2.10.3//jackson-datatype-jsr310-2.10.3.jar
 jackson-jaxrs-base/2.9.5//jackson-jaxrs-base-2.9.5.jar
 jackson-jaxrs-json-provider/2.9.5//jackson-jaxrs-json-provider-2.9.5.jar
 jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
@@ -164,9 +165,9 @@ kerby-pkix/1.0.1//kerby-pkix-1.0.1.jar
 kerby-util/1.0.1//kerby-util-1.0.1.jar
 kerby-xdr/1.0.1//kerby-xdr-1.0.1.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/4.7.1//kubernetes-client-4.7.1.jar
-kubernetes-model-common/4.7.1//kubernetes-model-common-4.7.1.jar
-kubernetes-model/4.7.1//kubernetes-model-4.7.1.jar
+kubernetes-client/4.9.2//kubernetes-client-4.9.2.jar
+kubernetes-model-common/4.9.2//kubernetes-model-common-4.9.2.jar
+kubernetes-model/4.9.2//kubernetes-model-4.9.2.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.12.0//libthrift-0.12.0.jar
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 36d9543f61540..168f7f5be8b0f 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -30,7 +30,7 @@
   <properties>
     <sbt.project.name>kubernetes</sbt.project.name>
     <!-- Note: Please update the kubernetes client version in kubernetes/integration-tests/pom.xml -->
-    <kubernetes.client.version>4.7.1</kubernetes.client.version>
+    <kubernetes.client.version>4.9.2</kubernetes.client.version>
   </properties>
 
   <dependencies>
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 92ddeaef2273c..719ebe4456eac 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -29,7 +29,7 @@
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
     <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
-    <kubernetes-client.version>4.7.1</kubernetes-client.version>
+    <kubernetes-client.version>4.9.2</kubernetes-client.version>
     <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
     <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>

From 576c22442c48d26780ca4e335a981c3215e09103 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Sun, 24 May 2020 14:27:16 +0900
Subject: [PATCH 0726/1280] [SPARK-31755][SQL][3.0] allow missing year/hour
 when parsing date/timestamp string

### What changes were proposed in this pull request?

This PR allows missing year and hour fields when parsing date/timestamp string, with 1970 as the default year and 0 as the default hour.

### Why are the changes needed?

To keep backward compatibility with Spark 2.4.

### Does this PR introduce _any_ user-facing change?

Yes.

Spark 2.4:
```
scala> sql("select to_timestamp('16', 'dd')").show
+------------------------+
|to_timestamp('16', 'dd')|
+------------------------+
|     1970-01-16 00:00:00|
+------------------------+

scala> sql("select to_date('16', 'dd')").show
+-------------------+
|to_date('16', 'dd')|
+-------------------+
|         1970-01-16|
+-------------------+

scala> sql("select to_timestamp('2019 40', 'yyyy mm')").show
+----------------------------------+
|to_timestamp('2019 40', 'yyyy mm')|
+----------------------------------+
|               2019-01-01 00:40:00|
+----------------------------------+

scala> sql("select to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss')").show
+----------------------------------------------+
|to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss')|
+----------------------------------------------+
|                           2019-01-01 10:10:10|
+----------------------------------------------+
```

in branch 3.0
```
scala> sql("select to_timestamp('16', 'dd')").show
+--------------------+
|to_timestamp(16, dd)|
+--------------------+
|                null|
+--------------------+

scala> sql("select to_date('16', 'dd')").show
+---------------+
|to_date(16, dd)|
+---------------+
|           null|
+---------------+

scala> sql("select to_timestamp('2019 40', 'yyyy mm')").show
+------------------------------+
|to_timestamp(2019 40, yyyy mm)|
+------------------------------+
|           2019-01-01 00:00:00|
+------------------------------+

scala> sql("select to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss')").show
+------------------------------------------+
|to_timestamp(2019 10:10:10, yyyy hh:mm:ss)|
+------------------------------------------+
|                       2019-01-01 00:00:00|
+------------------------------------------+
```

After this PR, the behavior becomes the same as 2.4.

### How was this patch tested?

new tests

Closes #28612 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/util/DateFormatter.scala     |   2 +-
 .../util/DateTimeFormatterHelper.scala        |  77 ++++++----
 .../catalyst/csv/UnivocityParserSuite.scala   |  14 +-
 .../sql/catalyst/util/DateTimeTestUtils.scala |   8 +-
 .../catalyst/util/DateTimeUtilsSuite.scala    |   4 +-
 .../spark/sql/util/DateFormatterSuite.scala   |  46 ++++--
 .../sql/util/TimestampFormatterSuite.scala    | 139 ++++++++++++++----
 .../resources/sql-tests/inputs/datetime.sql   |  12 +-
 .../sql-tests/inputs/json-functions.sql       |  15 ++
 .../sql-tests/results/ansi/datetime.sql.out   |  50 ++++++-
 .../sql-tests/results/datetime.sql.out        |  50 ++++++-
 .../sql-tests/results/json-functions.sql.out  |  45 +++++-
 12 files changed, 370 insertions(+), 92 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 7d94955096196..8261f57916fa2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -53,7 +53,7 @@ class Iso8601DateFormatter(
     val specialDate = convertSpecialDate(s.trim, zoneId)
     specialDate.getOrElse {
       try {
-        val localDate = LocalDate.parse(s, formatter)
+        val localDate = toLocalDate(formatter.parse(s))
         localDateToDays(localDate)
       } catch checkDiffResult(s, legacyFormatter.parse)
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 05ec23f7ad479..35f95dbffca6e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.util
 
 import java.time._
 import java.time.chrono.IsoChronology
-import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, DateTimeParseException, ResolverStyle}
+import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle}
 import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
 import java.util.Locale
 
@@ -31,17 +31,52 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
 
 trait DateTimeFormatterHelper {
+  private def getOrDefault(accessor: TemporalAccessor, field: ChronoField, default: Int): Int = {
+    if (accessor.isSupported(field)) {
+      accessor.get(field)
+    } else {
+      default
+    }
+  }
+
+  protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
+    val localDate = accessor.query(TemporalQueries.localDate())
+    // If all the date fields are specified, return the local date directly.
+    if (localDate != null) return localDate
+
+    // Users may want to parse only a few datetime fields from a string and extract these fields
+    // later, and we should provide default values for missing fields.
+    // To be compatible with Spark 2.4, we pick 1970 as the default value of year.
+    val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
+    val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+    val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+    LocalDate.of(year, month, day)
+  }
+
+  private def toLocalTime(accessor: TemporalAccessor): LocalTime = {
+    val localTime = accessor.query(TemporalQueries.localTime())
+    // If all the time fields are specified, return the local time directly.
+    if (localTime != null) return localTime
+
+    val hour = if (accessor.isSupported(ChronoField.HOUR_OF_DAY)) {
+      accessor.get(ChronoField.HOUR_OF_DAY)
+    } else if (accessor.isSupported(ChronoField.HOUR_OF_AMPM)) {
+      // When we reach here, it means am/pm is not specified. Here we assume it's am.
+      accessor.get(ChronoField.HOUR_OF_AMPM)
+    } else {
+      0
+    }
+    val minute = getOrDefault(accessor, ChronoField.MINUTE_OF_HOUR, 0)
+    val second = getOrDefault(accessor, ChronoField.SECOND_OF_MINUTE, 0)
+    val nanoSecond = getOrDefault(accessor, ChronoField.NANO_OF_SECOND, 0)
+    LocalTime.of(hour, minute, second, nanoSecond)
+  }
+
   // Converts the parsed temporal object to ZonedDateTime. It sets time components to zeros
   // if they does not exist in the parsed object.
-  protected def toZonedDateTime(
-      temporalAccessor: TemporalAccessor,
-      zoneId: ZoneId): ZonedDateTime = {
-    // Parsed input might not have time related part. In that case, time component is set to zeros.
-    val parsedLocalTime = temporalAccessor.query(TemporalQueries.localTime)
-    val localTime = if (parsedLocalTime == null) LocalTime.MIDNIGHT else parsedLocalTime
-    // Parsed input must have date component. At least, year must present in temporalAccessor.
-    val localDate = temporalAccessor.query(TemporalQueries.localDate)
-
+  protected def toZonedDateTime(accessor: TemporalAccessor, zoneId: ZoneId): ZonedDateTime = {
+    val localDate = toLocalDate(accessor)
+    val localTime = toLocalTime(accessor)
     ZonedDateTime.of(localDate, localTime, zoneId)
   }
 
@@ -72,19 +107,15 @@ trait DateTimeFormatterHelper {
   // DateTimeParseException will address by the caller side.
   protected def checkDiffResult[T](
       s: String, legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
-    case e: DateTimeParseException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION =>
-      val res = try {
-        Some(legacyParseFunc(s))
+    case e: DateTimeException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION =>
+      try {
+        legacyParseFunc(s)
       } catch {
-        case _: Throwable => None
-      }
-      if (res.nonEmpty) {
-        throw new SparkUpgradeException("3.0", s"Fail to parse '$s' in the new parser. You can " +
-          s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
-          s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
-      } else {
-        throw e
+        case _: Throwable => throw e
       }
+      throw new SparkUpgradeException("3.0", s"Fail to parse '$s' in the new parser. You can " +
+        s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
+        s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
   }
 }
 
@@ -101,10 +132,6 @@ private object DateTimeFormatterHelper {
 
   def toFormatter(builder: DateTimeFormatterBuilder, locale: Locale): DateTimeFormatter = {
     builder
-      .parseDefaulting(ChronoField.MONTH_OF_YEAR, 1)
-      .parseDefaulting(ChronoField.DAY_OF_MONTH, 1)
-      .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
-      .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
       .toFormatter(locale)
       .withChronology(IsoChronology.INSTANCE)
       .withResolverStyle(ResolverStyle.STRICT)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
index 4853b4f162be0..474bb53c24564 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala
@@ -325,7 +325,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
       assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45") ==
         date(2020, 1, 12, 12, 3, 45, 0))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       // The legacy format allows arbitrary length of second fraction.
       assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45.1") ==
         date(2020, 1, 12, 12, 3, 45, 100000))
@@ -333,22 +333,22 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
         date(2020, 1, 12, 12, 3, 45, 123400))
       // The legacy format allow date string to end with T or space, with arbitrary string
       assert(parser.makeConverter("t", DateType).apply("2020-1-12T") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12Txyz") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12 ") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12 xyz") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       // The legacy format ignores the "GMT" from the string
       assert(parser.makeConverter("t", TimestampType).apply("2020-1-12 12:3:45GMT") ==
         date(2020, 1, 12, 12, 3, 45, 0))
       assert(parser.makeConverter("t", TimestampType).apply("GMT2020-1-12 12:3:45") ==
         date(2020, 1, 12, 12, 3, 45, 0))
       assert(parser.makeConverter("t", DateType).apply("2020-1-12GMT") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
       assert(parser.makeConverter("t", DateType).apply("GMT2020-1-12") ==
-        days(2020, 1, 12, 0, 0, 0))
+        days(2020, 1, 12))
     }
 
     val options = new CSVOptions(Map.empty[String, String], false, "UTC")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
index bf9e8f71ba1c9..66aef1b4b6cb0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala
@@ -88,12 +88,8 @@ object DateTimeTestUtils {
   def days(
       year: Int,
       month: Byte = 1,
-      day: Byte = 1,
-      hour: Byte = 0,
-      minute: Byte = 0,
-      sec: Byte = 0): Int = {
-    val micros = date(year, month, day, hour, minute, sec)
-    TimeUnit.MICROSECONDS.toDays(micros).toInt
+      day: Byte = 1): Int = {
+    LocalDate.of(year, month, day).toEpochDay.toInt
   }
 
   // Returns microseconds since epoch for current date and give time
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index b547c445f06fe..d526ae18ab65e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -386,13 +386,13 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
   }
 
   test("date add months") {
-    val input = days(1997, 2, 28, 10, 30)
+    val input = days(1997, 2, 28)
     assert(dateAddMonths(input, 36) === days(2000, 2, 28))
     assert(dateAddMonths(input, -13) === days(1996, 1, 28))
   }
 
   test("date add interval with day precision") {
-    val input = days(1997, 2, 28, 10, 30)
+    val input = days(1997, 2, 28)
     assert(dateAddInterval(input, new CalendarInterval(36, 0, 0)) === days(2000, 2, 28))
     assert(dateAddInterval(input, new CalendarInterval(36, 47, 0)) === days(2000, 4, 15))
     assert(dateAddInterval(input, new CalendarInterval(-13, 0, 0)) === days(1996, 1, 28))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index 3954b9b8355c8..7d503cc091172 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -17,18 +17,19 @@
 
 package org.apache.spark.sql.util
 
-import java.time.{DateTimeException, LocalDate, ZoneOffset}
+import java.time.{DateTimeException, LocalDate}
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util._
+import org.apache.spark.sql.catalyst.util.{DateFormatter, LegacyDateFormats}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
 class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   test("parsing dates") {
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+    outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
         val formatter = DateFormatter(getZoneId(timeZone))
         val daysSinceEpoch = formatter.parse("2018-12-02")
@@ -38,7 +39,7 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("format dates") {
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+    outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
         val formatter = DateFormatter(getZoneId(timeZone))
         val (days, expected) = (17867, "2018-12-02")
@@ -65,7 +66,7 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
             "2018-12-12",
             "2038-01-01",
             "5010-11-17").foreach { date =>
-            DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+            outstandingTimezonesIds.foreach { timeZone =>
               withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
                 val formatter = DateFormatter(
                   DateFormatter.defaultPattern,
@@ -99,7 +100,7 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
             17877,
             24837,
             1110657).foreach { days =>
-            DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+            outstandingTimezonesIds.foreach { timeZone =>
               withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
                 val formatter = DateFormatter(
                   DateFormatter.defaultPattern,
@@ -118,14 +119,14 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("parsing date without explicit day") {
-    val formatter = DateFormatter("yyyy MMM", ZoneOffset.UTC)
+    val formatter = DateFormatter("yyyy MMM", UTC)
     val daysSinceEpoch = formatter.parse("2018 Dec")
-    assert(daysSinceEpoch === LocalDate.of(2018, 12, 1).toEpochDay)
+    assert(daysSinceEpoch === days(2018, 12, 1))
   }
 
   test("formatting negative years with default pattern") {
-    val epochDays = LocalDate.of(-99, 1, 1).toEpochDay.toInt
-    assert(DateFormatter(ZoneOffset.UTC).format(epochDays) === "-0099-01-01")
+    val epochDays = days(-99, 1, 1)
+    assert(DateFormatter(UTC).format(epochDays) === "-0099-01-01")
   }
 
   test("special date values") {
@@ -142,8 +143,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("SPARK-30958: parse date with negative year") {
-    val formatter1 = DateFormatter("yyyy-MM-dd", ZoneOffset.UTC)
-    assert(formatter1.parse("-1234-02-22") === localDateToDays(LocalDate.of(-1234, 2, 22)))
+    val formatter1 = DateFormatter("yyyy-MM-dd", UTC)
+    assert(formatter1.parse("-1234-02-22") === days(-1234, 2, 22))
 
     def assertParsingError(f: => Unit): Unit = {
       intercept[Exception](f) match {
@@ -155,18 +156,18 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
     }
 
     // "yyyy" with "G" can't parse negative year or year 0000.
-    val formatter2 = DateFormatter("G yyyy-MM-dd", ZoneOffset.UTC)
+    val formatter2 = DateFormatter("G yyyy-MM-dd", UTC)
     assertParsingError(formatter2.parse("BC -1234-02-22"))
     assertParsingError(formatter2.parse("AD 0000-02-22"))
 
-    assert(formatter2.parse("BC 1234-02-22") === localDateToDays(LocalDate.of(-1233, 2, 22)))
-    assert(formatter2.parse("AD 1234-02-22") === localDateToDays(LocalDate.of(1234, 2, 22)))
+    assert(formatter2.parse("BC 1234-02-22") === days(-1233, 2, 22))
+    assert(formatter2.parse("AD 1234-02-22") === days(1234, 2, 22))
   }
 
   test("SPARK-31557: rebasing in legacy formatters/parsers") {
     withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
       LegacyDateFormats.values.foreach { legacyFormat =>
-        DateTimeTestUtils.outstandingTimezonesIds.foreach { timeZone =>
+        outstandingTimezonesIds.foreach { timeZone =>
           withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
             val formatter = DateFormatter(
               DateFormatter.defaultPattern,
@@ -182,4 +183,17 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
       }
     }
   }
+
+  test("missing date fields") {
+    val formatter = DateFormatter("HH", UTC)
+    val daysSinceEpoch = formatter.parse("20")
+    assert(daysSinceEpoch === days(1970, 1, 1))
+  }
+
+  test("missing year field with invalid date") {
+    val formatter = DateFormatter("MM-dd", UTC)
+    // The date parser in 2.4 accepts 1970-02-29 and turn it into 1970-03-01, so we should get a
+    // SparkUpgradeException here.
+    intercept[SparkUpgradeException](formatter.parse("02-29"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index b467e24b5301a..dccb3defe3728 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -17,15 +17,15 @@
 
 package org.apache.spark.sql.util
 
-import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime, ZoneOffset}
+import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime}
 import java.util.concurrent.TimeUnit
 
 import org.scalatest.Matchers
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils, LegacyDateFormats, TimestampFormatter}
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CET, PST, UTC}
+import org.apache.spark.sql.catalyst.util.{LegacyDateFormats, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
@@ -44,10 +44,10 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       "Antarctica/Vostok" -> 1543723872001234L,
       "Asia/Hong_Kong" -> 1543716672001234L,
       "Europe/Amsterdam" -> 1543741872001234L)
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
+    outstandingTimezonesIds.foreach { zoneId =>
       val formatter = TimestampFormatter(
         "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
-        DateTimeUtils.getZoneId(zoneId),
+        getZoneId(zoneId),
         needVarLengthSecondFraction = true)
       val microsSinceEpoch = formatter.parse(localDate)
       assert(microsSinceEpoch === expectedMicros(zoneId))
@@ -65,7 +65,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       "Antarctica/Vostok" -> "2018-12-02 16:11:12.001234",
       "Asia/Hong_Kong" -> "2018-12-02 18:11:12.001234",
       "Europe/Amsterdam" -> "2018-12-02 11:11:12.001234")
-    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
+    outstandingTimezonesIds.foreach { zoneId =>
       Seq(
         TimestampFormatter(
           "yyyy-MM-dd HH:mm:ss.SSSSSS",
@@ -95,7 +95,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         1543749753123456L,
         2177456523456789L,
         11858049903010203L).foreach { micros =>
-        DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+        outstandingZoneIds.foreach { zoneId =>
           val timestamp = TimestampFormatter(pattern, zoneId).format(micros)
           val parsed = TimestampFormatter(
             pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
@@ -116,7 +116,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       "2018-12-02T11:22:33.123456",
       "2039-01-01T01:02:03.456789",
       "2345-10-07T22:45:03.010203").foreach { timestamp =>
-      DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+      outstandingZoneIds.foreach { zoneId =>
         val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
         val micros = TimestampFormatter(
           pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
@@ -127,10 +127,9 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("case insensitive parsing of am and pm") {
-    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", ZoneOffset.UTC)
+    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", UTC)
     val micros = formatter.parse("2009 Mar 20 11:30:01 am")
-    assert(micros === TimeUnit.SECONDS.toMicros(
-      LocalDateTime.of(2009, 3, 20, 11, 30, 1).toEpochSecond(ZoneOffset.UTC)))
+    assert(micros === date(2009, 3, 20, 11, 30, 1))
   }
 
   test("format fraction of second") {
@@ -143,7 +142,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       1000000 -> "1970-01-01 00:00:01").foreach { case (micros, tsStr) =>
       assert(formatter.format(micros) === tsStr)
       assert(formatter.format(microsToInstant(micros)) === tsStr)
-      DateTimeTestUtils.withDefaultTimeZone(UTC) {
+      withDefaultTimeZone(UTC) {
         assert(formatter.format(toJavaTimestamp(micros)) === tsStr)
       }
     }
@@ -151,10 +150,10 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
 
   test("formatting negative years with default pattern") {
     val instant = LocalDateTime.of(-99, 1, 1, 0, 0, 0).atZone(UTC).toInstant
-    val micros = DateTimeUtils.instantToMicros(instant)
+    val micros = instantToMicros(instant)
     assert(TimestampFormatter(UTC).format(micros) === "-0099-01-01 00:00:00")
     assert(TimestampFormatter(UTC).format(instant) === "-0099-01-01 00:00:00")
-    DateTimeTestUtils.withDefaultTimeZone(UTC) { // toJavaTimestamp depends on the default time zone
+    withDefaultTimeZone(UTC) { // toJavaTimestamp depends on the default time zone
       assert(TimestampFormatter("yyyy-MM-dd HH:mm:SS G", UTC).format(toJavaTimestamp(micros))
         === "0100-01-01 00:00:00 BC")
     }
@@ -181,11 +180,10 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("parsing timestamp strings with various seconds fractions") {
-    DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+    outstandingZoneIds.foreach { zoneId =>
       def check(pattern: String, input: String, reference: String): Unit = {
         val formatter = TimestampFormatter(pattern, zoneId, needVarLengthSecondFraction = true)
-        val expected = DateTimeUtils.stringToTimestamp(
-          UTF8String.fromString(reference), zoneId).get
+        val expected = stringToTimestamp(UTF8String.fromString(reference), zoneId).get
         val actual = formatter.parse(input)
         assert(actual === expected)
       }
@@ -219,11 +217,10 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("formatting timestamp strings up to microsecond precision") {
-    DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+    outstandingZoneIds.foreach { zoneId =>
       def check(pattern: String, input: String, expected: String): Unit = {
         val formatter = TimestampFormatter(pattern, zoneId)
-        val timestamp = DateTimeUtils.stringToTimestamp(
-          UTF8String.fromString(input), zoneId).get
+        val timestamp = stringToTimestamp(UTF8String.fromString(input), zoneId).get
         val actual = formatter.format(timestamp)
         assert(actual === expected)
       }
@@ -259,9 +256,8 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("SPARK-30958: parse timestamp with negative year") {
-    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", ZoneOffset.UTC, true)
-    assert(formatter1.parse("-1234-02-22 02:22:22") === instantToMicros(
-      LocalDateTime.of(-1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, true)
+    assert(formatter1.parse("-1234-02-22 02:22:22") === date(-1234, 2, 22, 2, 22, 22))
 
     def assertParsingError(f: => Unit): Unit = {
       intercept[Exception](f) match {
@@ -277,17 +273,15 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
     assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
 
-    assert(formatter2.parse("BC 1234-02-22 02:22:22") === instantToMicros(
-      LocalDateTime.of(-1233, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
-    assert(formatter2.parse("AD 1234-02-22 02:22:22") === instantToMicros(
-      LocalDateTime.of(1234, 2, 22, 2, 22, 22).toInstant(ZoneOffset.UTC)))
+    assert(formatter2.parse("BC 1234-02-22 02:22:22") === date(-1233, 2, 22, 2, 22, 22))
+    assert(formatter2.parse("AD 1234-02-22 02:22:22") === date(1234, 2, 22, 2, 22, 22))
   }
 
   test("SPARK-31557: rebasing in legacy formatters/parsers") {
     withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> LegacyBehaviorPolicy.LEGACY.toString) {
-      DateTimeTestUtils.outstandingZoneIds.foreach { zoneId =>
+      outstandingZoneIds.foreach { zoneId =>
         withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId.getId) {
-          DateTimeTestUtils.withDefaultTimeZone(zoneId) {
+          withDefaultTimeZone(zoneId) {
             withClue(s"zoneId = ${zoneId.getId}") {
               val formatters = LegacyDateFormats.values.map { legacyFormat =>
                 TimestampFormatter(
@@ -296,7 +290,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
                   TimestampFormatter.defaultLocale,
                   legacyFormat,
                   needVarLengthSecondFraction = false)
-              }.toSeq :+  TimestampFormatter.getFractionFormatter(zoneId)
+              }.toSeq :+ TimestampFormatter.getFractionFormatter(zoneId)
               formatters.foreach { formatter =>
                 assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
                   .atZone(zoneId)
@@ -317,4 +311,89 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       }
     }
   }
+
+  test("parsing hour with various patterns") {
+    def createFormatter(pattern: String): TimestampFormatter = {
+      // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid value range.
+      TimestampFormatter(pattern, UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, false)
+    }
+
+    withClue("HH") {
+      val formatter = createFormatter("yyyy-MM-dd HH")
+
+      val micros1 = formatter.parse("2009-12-12 00")
+      assert(micros1 === date(2009, 12, 12))
+
+      val micros2 = formatter.parse("2009-12-12 15")
+      assert(micros2 === date(2009, 12, 12, 15))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 24"))
+    }
+
+    withClue("kk") {
+      val formatter = createFormatter("yyyy-MM-dd kk")
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00"))
+
+      val micros1 = formatter.parse("2009-12-12 15")
+      assert(micros1 === date(2009, 12, 12, 15))
+
+      val micros2 = formatter.parse("2009-12-12 24")
+      assert(micros2 === date(2009, 12, 12))
+    }
+
+    withClue("KK") {
+      val formatter = createFormatter("yyyy-MM-dd KK a")
+
+      val micros1 = formatter.parse("2009-12-12 00 am")
+      assert(micros1 === date(2009, 12, 12))
+
+      // For `KK`, "12:00:00 am" is the same as "00:00:00 pm".
+      val micros2 = formatter.parse("2009-12-12 12 am")
+      assert(micros2 === date(2009, 12, 12, 12))
+
+      val micros3 = formatter.parse("2009-12-12 00 pm")
+      assert(micros3 === date(2009, 12, 12, 12))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 12 pm"))
+    }
+
+    withClue("hh") {
+      val formatter = createFormatter("yyyy-MM-dd hh a")
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00 am"))
+
+      val micros1 = formatter.parse("2009-12-12 12 am")
+      assert(micros1 === date(2009, 12, 12))
+
+      intercept[DateTimeException](formatter.parse("2009-12-12 00 pm"))
+
+      val micros2 = formatter.parse("2009-12-12 12 pm")
+      assert(micros2 === date(2009, 12, 12, 12))
+    }
+  }
+
+  test("missing date fields") {
+    val formatter = TimestampFormatter("HH:mm:ss", UTC)
+    val micros = formatter.parse("11:30:01")
+    assert(micros === date(1970, 1, 1, 11, 30, 1))
+  }
+
+  test("missing year field with invalid date") {
+    // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid date.
+    val formatter = TimestampFormatter("MM-dd", UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, false)
+    withDefaultTimeZone(UTC)(intercept[DateTimeException](formatter.parse("02-29")))
+  }
+
+  test("missing am/pm field") {
+    val formatter = TimestampFormatter("yyyy hh:mm:ss", UTC)
+    val micros = formatter.parse("2009 11:30:01")
+    assert(micros === date(2009, 1, 1, 11, 30, 1))
+  }
+
+  test("missing time fields") {
+    val formatter = TimestampFormatter("yyyy HH", UTC)
+    val micros = formatter.parse("2009 11")
+    assert(micros === date(2009, 1, 1, 11))
+  }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index fd3325085df96..9be857ef767df 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -86,7 +86,7 @@ select date_sub('2011-11-11', str) from v;
 select null - date '2019-10-06';
 select date '2001-10-01' - date '2001-09-28';
 
--- variable-length tests
+-- variable-length second fraction tests
 select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
@@ -95,7 +95,7 @@ select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zz
 select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
--- exceeded max variable length
+-- second fraction exceeded max variable length
 select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
 -- special cases
 select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]');
@@ -122,3 +122,11 @@ select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); --
 select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail
 select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss"); -- head
 select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss"); -- head but as single quote
+
+-- missing fields
+select to_timestamp("16", "dd");
+select to_timestamp("02-29", "MM-dd");
+select to_date("16", "dd");
+select to_date("02-29", "MM-dd");
+select to_timestamp("2019 40", "yyyy mm");
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss");
diff --git a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
index 6c14eee2e4e61..5bd78f5f6af3a 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/json-functions.sql
@@ -48,6 +48,21 @@ select from_json('[null, {"a":2}]', 'array<struct<a:int>>');
 select from_json('[{"a": 1}, {"b":2}]', 'array<map<string,int>>');
 select from_json('[{"a": 1}, 2]', 'array<map<string,int>>');
 
+-- from_json - datetime type
+select from_json('{"d": "2012-12-15", "t": "2012-12-15 15:15:15"}', 'd date, t timestamp');
+select from_json(
+  '{"d": "12/15 2012", "t": "12/15 2012 15:15:15"}',
+  'd date, t timestamp',
+  map('dateFormat', 'MM/dd yyyy', 'timestampFormat', 'MM/dd yyyy HH:mm:ss'));
+select from_json(
+  '{"d": "02-29"}',
+  'd date',
+  map('dateFormat', 'MM-dd'));
+select from_json(
+  '{"t": "02-29"}',
+  't timestamp',
+  map('timestampFormat', 'MM-dd'));
+
 -- to_json - array type
 select to_json(array('1', '2', '3'));
 select to_json(array(array(1, 2, 3), array(4)));
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index 81a73a6377715..89596c737b3e9 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 85
+-- Number of queries: 91
 
 
 -- !query
@@ -730,3 +730,51 @@ select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
 struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
 -- !query output
 2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp('16', 'dd'):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp('02-29', 'MM-dd'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date('16', 'dd'):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date('02-29', 'MM-dd'):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp('2019 40', 'yyyy mm'):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
+-- !query output
+2019-01-01 10:10:10
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 2e600850c48b9..b26eb2fdd018d 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 85
+-- Number of queries: 91
 
 
 -- !query
@@ -702,3 +702,51 @@ select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
 struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
 -- !query output
 2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp('16', 'dd'):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp('02-29', 'MM-dd'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date('16', 'dd'):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date('02-29', 'MM-dd'):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp('2019 40', 'yyyy mm'):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
+-- !query output
+2019-01-01 10:10:10
diff --git a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
index 21a3531caf732..665c79c4753bc 100644
--- a/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/json-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 42
+-- Number of queries: 46
 
 
 -- !query
@@ -288,6 +288,49 @@ struct<from_json([{"a": 1}, 2]):array<map<string,int>>>
 NULL
 
 
+-- !query
+select from_json('{"d": "2012-12-15", "t": "2012-12-15 15:15:15"}', 'd date, t timestamp')
+-- !query schema
+struct<from_json({"d": "2012-12-15", "t": "2012-12-15 15:15:15"}):struct<d:date,t:timestamp>>
+-- !query output
+{"d":2012-12-15,"t":2012-12-15 15:15:15}
+
+
+-- !query
+select from_json(
+  '{"d": "12/15 2012", "t": "12/15 2012 15:15:15"}',
+  'd date, t timestamp',
+  map('dateFormat', 'MM/dd yyyy', 'timestampFormat', 'MM/dd yyyy HH:mm:ss'))
+-- !query schema
+struct<from_json({"d": "12/15 2012", "t": "12/15 2012 15:15:15"}):struct<d:date,t:timestamp>>
+-- !query output
+{"d":2012-12-15,"t":2012-12-15 15:15:15}
+
+
+-- !query
+select from_json(
+  '{"d": "02-29"}',
+  'd date',
+  map('dateFormat', 'MM-dd'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '02-29' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select from_json(
+  '{"t": "02-29"}',
+  't timestamp',
+  map('timestampFormat', 'MM-dd'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '02-29' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
 -- !query
 select to_json(array('1', '2', '3'))
 -- !query schema

From 72c466e0c37e4cc639040161699b6c0bffde70d5 Mon Sep 17 00:00:00 2001
From: sandeep katta <sandeep.katta2007@gmail.com>
Date: Sun, 24 May 2020 21:39:16 +0900
Subject: [PATCH 0727/1280] [SPARK-31761][SQL][3.0] cast integer to Long to
 avoid IntegerOverflow for IntegralDivide operator

### What changes were proposed in this pull request?
`IntegralDivide` operator returns Long DataType, so integer overflow case should be handled.
If the operands are of type Int it will be casted to Long

### Why are the changes needed?
As `IntegralDivide` returns Long datatype, integer overflow should not happen

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT and also tested in the local cluster

After fix

![image](https://user-images.githubusercontent.com/35216143/82603361-25eccc00-9bd0-11ea-9ca7-001c539e628b.png)

SQL Test

After fix
![image](https://user-images.githubusercontent.com/35216143/82637689-f0250300-9c22-11ea-85c3-886ab2c23471.png)

Before Fix
![image](https://user-images.githubusercontent.com/35216143/82637984-878a5600-9c23-11ea-9e47-5ce2fb923c01.png)

Closes #28628 from sandeep-katta/branch3Backport.

Authored-by: sandeep katta <sandeep.katta2007@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/analysis/TypeCoercion.scala  | 18 ++++++++++++++
 .../sql/catalyst/expressions/arithmetic.scala |  2 +-
 .../catalyst/analysis/TypeCoercionSuite.scala | 24 +++++++++++++++++++
 .../ArithmeticExpressionSuite.scala           |  7 +-----
 .../sql-functions/sql-expression-schema.md    |  2 +-
 .../sql-tests/results/operators.sql.out       |  8 +++----
 .../org/apache/spark/sql/SQLQuerySuite.scala  |  8 +++++++
 7 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index c6e3f56766a8a..a6f8e12849ccb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -61,6 +61,7 @@ object TypeCoercion {
       IfCoercion ::
       StackCoercion ::
       Division ::
+      IntegralDivision ::
       ImplicitTypeCasts ::
       DateTimeOperations ::
       WindowFrameCoercion ::
@@ -684,6 +685,23 @@ object TypeCoercion {
     }
   }
 
+  /**
+   * The DIV operator always returns long-type value.
+   * This rule cast the integral inputs to long type, to avoid overflow during calculation.
+   */
+  object IntegralDivision extends TypeCoercionRule {
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+      case e if !e.childrenResolved => e
+      case d @ IntegralDivide(left, right) =>
+        IntegralDivide(mayCastToLong(left), mayCastToLong(right))
+    }
+
+    private def mayCastToLong(expr: Expression): Expression = expr.dataType match {
+      case _: ByteType | _: ShortType | _: IntegerType => Cast(expr, LongType)
+      case _ => expr
+    }
+  }
+
   /**
    * Coerces the type of different branches of a CASE WHEN statement to a common type.
    */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 354845d5ccd8b..7c521838447d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -412,7 +412,7 @@ case class IntegralDivide(
     left: Expression,
     right: Expression) extends DivModLike {
 
-  override def inputType: AbstractDataType = TypeCollection(IntegralType, DecimalType)
+  override def inputType: AbstractDataType = TypeCollection(LongType, DecimalType)
 
   override def dataType: DataType = LongType
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index e37555f1c0ec3..1ea1ddb8bbd08 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -1559,6 +1559,30 @@ class TypeCoercionSuite extends AnalysisTest {
           Literal.create(null, DecimalType.SYSTEM_DEFAULT)))
     }
   }
+
+  test("SPARK-31761: byte, short and int should be cast to long for IntegralDivide's datatype") {
+    val rules = Seq(FunctionArgumentConversion, Division, ImplicitTypeCasts)
+    // Casts Byte to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toByte, 1.toByte),
+      IntegralDivide(Cast(2.toByte, LongType), Cast(1.toByte, LongType)))
+    // Casts Short to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1.toShort),
+      IntegralDivide(Cast(2.toShort, LongType), Cast(1.toShort, LongType)))
+    // Casts Integer to Long
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2, 1),
+      IntegralDivide(Cast(2, LongType), Cast(1, LongType)))
+    // should not be any change for Long data types
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2L, 1L), IntegralDivide(2L, 1L))
+    // one of the operand is byte
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2L, 1.toByte),
+      IntegralDivide(2L, Cast(1.toByte, LongType)))
+    // one of the operand is short
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2.toShort, 1L),
+      IntegralDivide(Cast(2.toShort, LongType), 1L))
+    // one of the operand is int
+    ruleTest(TypeCoercion.IntegralDivision, IntegralDivide(2, 1L),
+      IntegralDivide(Cast(2, LongType), 1L))
+  }
 }
 
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
index 675f85f9e82ea..f05598aeb5353 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
@@ -173,13 +173,8 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
     }
   }
 
-  test("/ (Divide) for integral type") {
-    checkEvaluation(IntegralDivide(Literal(1.toByte), Literal(2.toByte)), 0L)
-    checkEvaluation(IntegralDivide(Literal(1.toShort), Literal(2.toShort)), 0L)
-    checkEvaluation(IntegralDivide(Literal(1), Literal(2)), 0L)
+  test("/ (Divide) for Long type") {
     checkEvaluation(IntegralDivide(Literal(1.toLong), Literal(2.toLong)), 0L)
-    checkEvaluation(IntegralDivide(positiveShortLit, negativeShortLit), 0L)
-    checkEvaluation(IntegralDivide(positiveIntLit, negativeIntLit), 0L)
     checkEvaluation(IntegralDivide(positiveLongLit, negativeLongLit), 0L)
   }
 
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index c3ae2a73160d3..9e24a54058ec5 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -136,7 +136,7 @@
 | org.apache.spark.sql.catalyst.expressions.InputFileBlockLength | input_file_block_length | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.InputFileBlockStart | input_file_block_start | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.InputFileName | input_file_name | N/A | N/A |
-| org.apache.spark.sql.catalyst.expressions.IntegralDivide | div | SELECT 3 div 2 | struct<(3 div 2):bigint> |
+| org.apache.spark.sql.catalyst.expressions.IntegralDivide | div | SELECT 3 div 2 | struct<(CAST(3 AS BIGINT) div CAST(2 AS BIGINT)):bigint> |
 | org.apache.spark.sql.catalyst.expressions.IsNaN | isnan | SELECT isnan(cast('NaN' as double)) | struct<isnan(CAST(NaN AS DOUBLE)):boolean> |
 | org.apache.spark.sql.catalyst.expressions.IsNotNull | isnotnull | SELECT isnotnull(1) | struct<(1 IS NOT NULL):boolean> |
 | org.apache.spark.sql.catalyst.expressions.IsNull | isnull | SELECT isnull(1) | struct<(1 IS NULL):boolean> |
diff --git a/sql/core/src/test/resources/sql-tests/results/operators.sql.out b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
index a94a123b1b8ab..9accc57d0bf60 100644
--- a/sql/core/src/test/resources/sql-tests/results/operators.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/operators.sql.out
@@ -157,7 +157,7 @@ NULL
 -- !query
 select 5 div 2
 -- !query schema
-struct<(5 div 2):bigint>
+struct<(CAST(5 AS BIGINT) div CAST(2 AS BIGINT)):bigint>
 -- !query output
 2
 
@@ -165,7 +165,7 @@ struct<(5 div 2):bigint>
 -- !query
 select 5 div 0
 -- !query schema
-struct<(5 div 0):bigint>
+struct<(CAST(5 AS BIGINT) div CAST(0 AS BIGINT)):bigint>
 -- !query output
 NULL
 
@@ -173,7 +173,7 @@ NULL
 -- !query
 select 5 div null
 -- !query schema
-struct<(5 div CAST(NULL AS INT)):bigint>
+struct<(CAST(5 AS BIGINT) div CAST(NULL AS BIGINT)):bigint>
 -- !query output
 NULL
 
@@ -181,7 +181,7 @@ NULL
 -- !query
 select null div 5
 -- !query schema
-struct<(CAST(NULL AS INT) div 5):bigint>
+struct<(CAST(NULL AS BIGINT) div CAST(5 AS BIGINT)):bigint>
 -- !query output
 NULL
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index d336f528f5a05..a23e5831f5887 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3441,6 +3441,14 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       assert(SQLConf.get.getConf(SQLConf.CODEGEN_FALLBACK) === true)
     }
   }
+
+  test("SPARK-31761: test byte, short, integer overflow for (Divide) integral type") {
+    checkAnswer(sql("Select -2147483648 DIV -1"), Seq(Row(Integer.MIN_VALUE.toLong * -1)))
+    checkAnswer(sql("select CAST(-128 as Byte) DIV CAST (-1 as Byte)"),
+      Seq(Row(Byte.MinValue.toLong * -1)))
+    checkAnswer(sql("select CAST(-32768 as short) DIV CAST (-1 as short)"),
+      Seq(Row(Short.MinValue.toLong * -1)))
+  }
 }
 
 case class Foo(bar: Option[String])

From 40e5de54e6109762d3ee85197da4fa1a8b20e13b Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Mon, 25 May 2020 10:25:43 +0900
Subject: [PATCH 0728/1280] [SPARK-31807][INFRA] Use python 3 style in
 release-build.sh

### What changes were proposed in this pull request?

This PR aims to use the style that is compatible with both python 2 and 3.

### Why are the changes needed?

This will help python 3 migration.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manual.

Closes #28632 from williamhyun/use_python3_style.

Authored-by: William Hyun <williamhyun3@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 753636e86b1989a9b43ec691de5136a70d11f323)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/create-release/release-build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 655b079ac28bf..e3bcb72ab5c6f 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -380,7 +380,7 @@ if [[ "$1" == "publish-snapshot" ]]; then
   echo "</server></servers></settings>" >> $tmp_settings
 
   # Generate random point for Zinc
-  export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
+  export ZINC_PORT=$(python -S -c "import random; print(random.randrange(3030,4030))")
 
   $MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $SCALA_2_12_PROFILES $PUBLISH_PROFILES deploy
 
@@ -412,7 +412,7 @@ if [[ "$1" == "publish-release" ]]; then
   tmp_repo=$(mktemp -d spark-repo-XXXXX)
 
   # Generate random point for Zinc
-  export ZINC_PORT=$(python -S -c "import random; print random.randrange(3030,4030)")
+  export ZINC_PORT=$(python -S -c "import random; print(random.randrange(3030,4030))")
 
   # TODO: revisit for Scala 2.13 support
 

From be18718380fc501fe2a780debd089a1df91c1699 Mon Sep 17 00:00:00 2001
From: schintap <schintap@verizonmedia.com>
Date: Mon, 25 May 2020 10:29:08 +0900
Subject: [PATCH 0729/1280] [SPARK-31788][CORE][PYTHON] Fix UnionRDD of
 PairRDDs

### What changes were proposed in this pull request?
UnionRDD of PairRDDs causing a bug. The fix is to check for instance type before proceeding

### Why are the changes needed?
Changes are needed to avoid users running into issues with union rdd operation with any other type other than JavaRDD.

### Does this PR introduce _any_ user-facing change?
Yes

Before:
SparkSession available as 'spark'.
>>> rdd1 = sc.parallelize([1,2,3,4,5])
>>> rdd2 = sc.parallelize([6,7,8,9,10])
>>> pairRDD1 = rdd1.zip(rdd2)
>>> unionRDD1 = sc.union([pairRDD1, pairRDD1])
Traceback (most recent call last): File "<stdin>", line 1, in <module> File "/home/gs/spark/latest/python/pyspark/context.py", line 870,
in union jrdds[i] = rdds[i]._jrdd
File "/home/gs/spark/latest/python/lib/py4j-0.10.9-src.zip/py4j/java_collections.py", line 238, in setitem File "/home/gs/spark/latest/python/lib/py4j-0.10.9-src.zip/py4j/java_collections.py", line 221,
in __set_item File "/home/gs/spark/latest/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py", line 332, in get_return_value py4j.protocol.Py4JError: An error occurred while calling None.None. Trace: py4j.Py4JException: Cannot convert org.apache.spark.api.java.JavaPairRDD to org.apache.spark.api.java.JavaRDD at py4j.commands.ArrayCommand.convertArgument(ArrayCommand.java:166) at py4j.commands.ArrayCommand.setArray(ArrayCommand.java:144) at py4j.commands.ArrayCommand.execute(ArrayCommand.java:97) at py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748)

After:
>>> rdd2 = sc.parallelize([6,7,8,9,10])
>>> pairRDD1 = rdd1.zip(rdd2)
>>> unionRDD1 = sc.union([pairRDD1, pairRDD1])
>>> unionRDD1.collect()
[(1, 6), (2, 7), (3, 8), (4, 9), (5, 10), (1, 6), (2, 7), (3, 8), (4, 9), (5, 10)]

### How was this patch tested?
Tested with the reproduced piece of code above manually

Closes #28603 from redsanket/SPARK-31788.

Authored-by: schintap <schintap@verizonmedia.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a61911c50c391e61038cf01611629d2186d17a76)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/context.py        | 12 ++++++++++--
 python/pyspark/tests/test_rdd.py |  9 +++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index d5f1506e4a555..3199aa77565fd 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -25,6 +25,7 @@
 from tempfile import NamedTemporaryFile
 
 from py4j.protocol import Py4JError
+from py4j.java_gateway import is_instance_of
 
 from pyspark import accumulators
 from pyspark.accumulators import Accumulator
@@ -864,10 +865,17 @@ def union(self, rdds):
         first_jrdd_deserializer = rdds[0]._jrdd_deserializer
         if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds):
             rdds = [x._reserialize() for x in rdds]
+        gw = SparkContext._gateway
         cls = SparkContext._jvm.org.apache.spark.api.java.JavaRDD
-        jrdds = SparkContext._gateway.new_array(cls, len(rdds))
+        is_jrdd = is_instance_of(gw, rdds[0]._jrdd, cls)
+        jrdds = gw.new_array(cls, len(rdds))
         for i in range(0, len(rdds)):
-            jrdds[i] = rdds[i]._jrdd
+            if is_jrdd:
+                jrdds[i] = rdds[i]._jrdd
+            else:
+                # zip could return JavaPairRDD hence we ensure `_jrdd`
+                # to be `JavaRDD` by wrapping it in a `map`
+                jrdds[i] = rdds[i].map(lambda x: x)._jrdd
         return RDD(self._jsc.union(jrdds), self, rdds[0]._jrdd_deserializer)
 
     def broadcast(self, value):
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index e2d910ccd15c3..0f1ee5b706611 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -166,6 +166,15 @@ def test_zip_chaining(self):
             set([(x, (x, x)) for x in 'abc'])
         )
 
+    def test_union_pair_rdd(self):
+        # Regression test for SPARK-31788
+        rdd = self.sc.parallelize([1, 2])
+        pair_rdd = rdd.zip(rdd)
+        self.assertEqual(
+            self.sc.union([pair_rdd, pair_rdd]).collect(),
+            [((1, 1), (2, 2)), ((1, 1), (2, 2))]
+        )
+
     def test_deleting_input_files(self):
         # Regression test for SPARK-1025
         tempFile = tempfile.NamedTemporaryFile(delete=False)

From 102089081e246bab306def2b63385bca35b87fe9 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 25 May 2020 12:50:38 +0900
Subject: [PATCH 0730/1280] [SPARK-31802][SQL] Format Java date-time types in
 `Row.jsonValue` directly

### What changes were proposed in this pull request?
Use `format()` methods for Java date-time types in `Row.jsonValue`. The PR https://github.com/apache/spark/pull/28582 added the methods to avoid conversions to days and microseconds.

### Why are the changes needed?
To avoid unnecessary overhead of converting Java date-time types to micros/days before formatting. Also formatters have to convert input micros/days back to Java types to pass instances to standard library API.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By existing tests in `RowJsonSuite`.

Closes #28620 from MaxGekk/toJson-format-Java-datetime-types.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 7f36310500a7ab6e80cc1b1e6c764ae7b9657758)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/Row.scala    | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index 4487a2d7f4358..5b17f1d65f1bf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -571,14 +571,10 @@ trait Row extends Serializable {
       case (s: String, _) => JString(s)
       case (b: Array[Byte], BinaryType) =>
         JString(Base64.getEncoder.encodeToString(b))
-      case (d: LocalDate, _) =>
-        JString(dateFormatter.format(DateTimeUtils.localDateToDays(d)))
-      case (d: Date, _) =>
-        JString(dateFormatter.format(DateTimeUtils.fromJavaDate(d)))
-      case (i: Instant, _) =>
-        JString(timestampFormatter.format(DateTimeUtils.instantToMicros(i)))
-      case (t: Timestamp, _) =>
-        JString(timestampFormatter.format(DateTimeUtils.fromJavaTimestamp(t)))
+      case (d: LocalDate, _) => JString(dateFormatter.format(d))
+      case (d: Date, _) => JString(dateFormatter.format(d))
+      case (i: Instant, _) => JString(timestampFormatter.format(i))
+      case (t: Timestamp, _) => JString(timestampFormatter.format(t))
       case (i: CalendarInterval, _) => JString(i.toString)
       case (a: Array[_], ArrayType(elementType, _)) =>
         iteratorToJsonArray(a.iterator, elementType)

From 942c4fcfb1e5516a6ee1e3dff94f889ecdd4a015 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 25 May 2020 15:00:11 +0000
Subject: [PATCH 0731/1280] [SPARK-31755][SQL][FOLLOWUP] Update date-time, CSV
 and JSON benchmark results

### What changes were proposed in this pull request?
Re-generate results of:
- DateTimeBenchmark
- CSVBenchmark
- JsonBenchmark

in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

### Why are the changes needed?
1. The PR https://github.com/apache/spark/pull/28576 changed date-time parser. The `DateTimeBenchmark` should confirm that the PR didn't slow down date/timestamp parsing.
2. CSV/JSON datasources are affected by the above PR too. This PR updates the benchmark results in the same environment as other benchmarks to have a base line for future optimizations.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running benchmarks via the script:
```python
#!/usr/bin/env python3

import os
from sparktestsupport.shellutils import run_cmd

benchmarks = [
    ['sql/test', 'org.apache.spark.sql.execution.benchmark.DateTimeBenchmark'],
    ['sql/test', 'org.apache.spark.sql.execution.datasources.csv.CSVBenchmark'],
    ['sql/test', 'org.apache.spark.sql.execution.datasources.json.JsonBenchmark']
]

print('Set SPARK_GENERATE_BENCHMARK_FILES=1')
os.environ['SPARK_GENERATE_BENCHMARK_FILES'] = '1'

for b in benchmarks:
    print("Run benchmark: %s" % b[1])
    run_cmd(['build/sbt', '%s:runMain %s' % (b[0], b[1])])
```

Closes #28613 from MaxGekk/missing-hour-year-benchmarks.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 92685c014816e82c339f047f51c5363e32bc7338)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../benchmarks/CSVBenchmark-jdk11-results.txt |  88 +++---
 sql/core/benchmarks/CSVBenchmark-results.txt  |  88 +++---
 .../DateTimeBenchmark-jdk11-results.txt       | 254 +++++++++---------
 .../benchmarks/DateTimeBenchmark-results.txt  | 254 +++++++++---------
 .../JsonBenchmark-jdk11-results.txt           | 130 ++++-----
 sql/core/benchmarks/JsonBenchmark-results.txt | 130 ++++-----
 6 files changed, 472 insertions(+), 472 deletions(-)

diff --git a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
index 147a77ff098d0..0e82b632793d2 100644
--- a/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-jdk11-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 24907          29374         NaN          0.0      498130.5       1.0X
+One quoted string                                 46568          46683         198          0.0      931358.6       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                               62811          63690        1416          0.0       62811.4       1.0X
-Select 100 columns                                23839          24064         230          0.0       23839.5       2.6X
-Select one column                                 19936          20641         827          0.1       19936.4       3.2X
-count()                                            4174           4380         206          0.2        4174.4      15.0X
-Select 100 columns, one bad input field           41015          42380        1688          0.0       41015.4       1.5X
-Select 100 columns, corrupt record field          46281          46338          93          0.0       46280.5       1.4X
+Select 1000 columns                              129836         130796        1404          0.0      129836.0       1.0X
+Select 100 columns                                40444          40679         261          0.0       40443.5       3.2X
+Select one column                                 33429          33475          73          0.0       33428.6       3.9X
+count()                                            7967           8047          73          0.1        7966.7      16.3X
+Select 100 columns, one bad input field           90639          90832         266          0.0       90638.6       1.4X
+Select 100 columns, corrupt record field         109023         109084          74          0.0      109023.3       1.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                       10810          10997         163          0.9        1081.0       1.0X
-Select 1 column + count()                          7608           7641          47          1.3         760.8       1.4X
-count()                                            2415           2462          77          4.1         241.5       4.5X
+Select 10 columns + count()                       20685          20707          35          0.5        2068.5       1.0X
+Select 1 column + count()                         13096          13149          49          0.8        1309.6       1.6X
+count()                                            3994           4001           7          2.5         399.4       5.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      874            914          37         11.4          87.4       1.0X
-to_csv(timestamp)                                  7051           7223         250          1.4         705.1       0.1X
-write timestamps to files                          6712           6741          31          1.5         671.2       0.1X
-Create a dataset of dates                           909            945          35         11.0          90.9       1.0X
-to_csv(date)                                       4222           4231           8          2.4         422.2       0.2X
-write dates to files                               3799           3813          14          2.6         379.9       0.2X
+Create a dataset of timestamps                     2169           2203          32          4.6         216.9       1.0X
+to_csv(timestamp)                                 14401          14591         168          0.7        1440.1       0.2X
+write timestamps to files                         13209          13276          59          0.8        1320.9       0.2X
+Create a dataset of dates                          2231           2248          17          4.5         223.1       1.0X
+to_csv(date)                                      10406          10473          68          1.0        1040.6       0.2X
+write dates to files                               7970           7976           9          1.3         797.0       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1342           1364          35          7.5         134.2       1.0X
-read timestamps from files                        20300          20473         247          0.5        2030.0       0.1X
-infer timestamps from files                       40705          40744          54          0.2        4070.5       0.0X
-read date text from files                          1146           1151           6          8.7         114.6       1.2X
-read date from files                              12278          12408         117          0.8        1227.8       0.1X
-infer date from files                             12734          12872         220          0.8        1273.4       0.1X
-timestamp strings                                  1467           1482          15          6.8         146.7       0.9X
-parse timestamps from Dataset[String]             21708          22234         477          0.5        2170.8       0.1X
-infer timestamps from Dataset[String]             42357          43253         922          0.2        4235.7       0.0X
-date strings                                       1512           1532          18          6.6         151.2       0.9X
-parse dates from Dataset[String]                  13436          13470          33          0.7        1343.6       0.1X
-from_csv(timestamp)                               20390          20486          95          0.5        2039.0       0.1X
-from_csv(date)                                    12592          12693         139          0.8        1259.2       0.1X
+read timestamp text from files                     2387           2391           6          4.2         238.7       1.0X
+read timestamps from files                        53503          53593         124          0.2        5350.3       0.0X
+infer timestamps from files                      107988         108668         647          0.1       10798.8       0.0X
+read date text from files                          2121           2133          12          4.7         212.1       1.1X
+read date from files                              29983          30039          48          0.3        2998.3       0.1X
+infer date from files                             30196          30436         218          0.3        3019.6       0.1X
+timestamp strings                                  3098           3109          10          3.2         309.8       0.8X
+parse timestamps from Dataset[String]             63331          63426          84          0.2        6333.1       0.0X
+infer timestamps from Dataset[String]            124003         124463         490          0.1       12400.3       0.0X
+date strings                                       3423           3429          11          2.9         342.3       0.7X
+parse dates from Dataset[String]                  34235          34314          76          0.3        3423.5       0.1X
+from_csv(timestamp)                               60829          61600         668          0.2        6082.9       0.0X
+from_csv(date)                                    33047          33173         139          0.3        3304.7       0.1X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       12535          12606          67          0.0      125348.8       1.0X
-pushdown disabled                                 12611          12672          91          0.0      126112.9       1.0X
-w/ filters                                         1093           1099          11          0.1       10928.3      11.5X
+w/o filters                                       28752          28765          16          0.0      287516.5       1.0X
+pushdown disabled                                 28856          28880          22          0.0      288556.3       1.0X
+w/ filters                                         1714           1731          15          0.1       17137.3      16.8X
 
 
diff --git a/sql/core/benchmarks/CSVBenchmark-results.txt b/sql/core/benchmarks/CSVBenchmark-results.txt
index 498ca4caa0e45..a3af46c037bf9 100644
--- a/sql/core/benchmarks/CSVBenchmark-results.txt
+++ b/sql/core/benchmarks/CSVBenchmark-results.txt
@@ -2,66 +2,66 @@
 Benchmark to measure CSV read/write performance
 ================================================================================================
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Parsing quoted values:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-One quoted string                                 24073          24109          33          0.0      481463.5       1.0X
+One quoted string                                 45457          45731         344          0.0      909136.8       1.0X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Wide rows with 1000 columns:              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 1000 columns                               58415          59611        2071          0.0       58414.8       1.0X
-Select 100 columns                                22568          23020         594          0.0       22568.0       2.6X
-Select one column                                 18995          19058          99          0.1       18995.0       3.1X
-count()                                            5301           5332          30          0.2        5300.9      11.0X
-Select 100 columns, one bad input field           39736          40153         361          0.0       39736.1       1.5X
-Select 100 columns, corrupt record field          47195          47826         590          0.0       47195.2       1.2X
+Select 1000 columns                              129646         130527        1412          0.0      129646.3       1.0X
+Select 100 columns                                42444          42551         119          0.0       42444.0       3.1X
+Select one column                                 35415          35428          20          0.0       35414.6       3.7X
+count()                                           11114          11128          16          0.1       11113.6      11.7X
+Select 100 columns, one bad input field           93353          93670         275          0.0       93352.6       1.4X
+Select 100 columns, corrupt record field         113569         113952         373          0.0      113568.8       1.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Count a dataset with 10 columns:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns + count()                        9884           9904          25          1.0         988.4       1.0X
-Select 1 column + count()                          6794           6835          46          1.5         679.4       1.5X
-count()                                            2060           2065           5          4.9         206.0       4.8X
+Select 10 columns + count()                       18498          18589          87          0.5        1849.8       1.0X
+Select 1 column + count()                         11078          11095          27          0.9        1107.8       1.7X
+count()                                            3928           3950          22          2.5         392.8       4.7X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      717            732          18         14.0          71.7       1.0X
-to_csv(timestamp)                                  6994           7100         121          1.4         699.4       0.1X
-write timestamps to files                          6417           6435          27          1.6         641.7       0.1X
-Create a dataset of dates                           827            855          24         12.1          82.7       0.9X
-to_csv(date)                                       4408           4438          32          2.3         440.8       0.2X
-write dates to files                               3738           3758          28          2.7         373.8       0.2X
+Create a dataset of timestamps                     1933           1940          11          5.2         193.3       1.0X
+to_csv(timestamp)                                 18078          18243         255          0.6        1807.8       0.1X
+write timestamps to files                         12668          12786         134          0.8        1266.8       0.2X
+Create a dataset of dates                          2196           2201           5          4.6         219.6       0.9X
+to_csv(date)                                       9583           9597          21          1.0         958.3       0.2X
+write dates to files                               7091           7110          20          1.4         709.1       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1121           1176          52          8.9         112.1       1.0X
-read timestamps from files                        21298          21366         105          0.5        2129.8       0.1X
-infer timestamps from files                       41008          41051          39          0.2        4100.8       0.0X
-read date text from files                           962            967           5         10.4          96.2       1.2X
-read date from files                              11749          11772          22          0.9        1174.9       0.1X
-infer date from files                             12426          12459          29          0.8        1242.6       0.1X
-timestamp strings                                  1508           1519           9          6.6         150.8       0.7X
-parse timestamps from Dataset[String]             21674          21997         455          0.5        2167.4       0.1X
-infer timestamps from Dataset[String]             42141          42230         105          0.2        4214.1       0.0X
-date strings                                       1694           1701           8          5.9         169.4       0.7X
-parse dates from Dataset[String]                  12929          12951          25          0.8        1292.9       0.1X
-from_csv(timestamp)                               20603          20786         166          0.5        2060.3       0.1X
-from_csv(date)                                    12325          12338          12          0.8        1232.5       0.1X
+read timestamp text from files                     2166           2177          10          4.6         216.6       1.0X
+read timestamps from files                        53212          53402         281          0.2        5321.2       0.0X
+infer timestamps from files                      109788         110372         570          0.1       10978.8       0.0X
+read date text from files                          1921           1929           8          5.2         192.1       1.1X
+read date from files                              25470          25499          25          0.4        2547.0       0.1X
+infer date from files                             27201          27342         134          0.4        2720.1       0.1X
+timestamp strings                                  3638           3653          19          2.7         363.8       0.6X
+parse timestamps from Dataset[String]             61894          62532         555          0.2        6189.4       0.0X
+infer timestamps from Dataset[String]            125171         125430         236          0.1       12517.1       0.0X
+date strings                                       3736           3749          14          2.7         373.6       0.6X
+parse dates from Dataset[String]                  30787          30829          43          0.3        3078.7       0.1X
+from_csv(timestamp)                               60842          61035         209          0.2        6084.2       0.0X
+from_csv(date)                                    30123          30196          95          0.3        3012.3       0.1X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Filters pushdown:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-w/o filters                                       12455          12474          22          0.0      124553.8       1.0X
-pushdown disabled                                 12462          12486          29          0.0      124624.9       1.0X
-w/ filters                                         1073           1092          18          0.1       10727.6      11.6X
+w/o filters                                       28985          29042          80          0.0      289852.9       1.0X
+pushdown disabled                                 29080          29146          58          0.0      290799.4       1.0X
+w/ filters                                         2072           2084          17          0.0       20722.3      14.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index 61ca342a0d559..f4ed8ce4afaea 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -6,18 +6,18 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1496           1569         104          6.7         149.6       1.0X
-date + interval(m, d)                              1514           1526          17          6.6         151.4       1.0X
-date + interval(m, d, ms)                          6231           6253          30          1.6         623.1       0.2X
-date - interval(m)                                 1481           1487           9          6.8         148.1       1.0X
-date - interval(m, d)                              1550           1552           2          6.5         155.0       1.0X
-date - interval(m, d, ms)                          6269           6272           4          1.6         626.9       0.2X
-timestamp + interval(m)                            3017           3056          54          3.3         301.7       0.5X
-timestamp + interval(m, d)                         3146           3148           3          3.2         314.6       0.5X
-timestamp + interval(m, d, ms)                     3446           3460          20          2.9         344.6       0.4X
-timestamp - interval(m)                            3045           3059          19          3.3         304.5       0.5X
-timestamp - interval(m, d)                         3147           3164          25          3.2         314.7       0.5X
-timestamp - interval(m, d, ms)                     3425           3442          25          2.9         342.5       0.4X
+date + interval(m)                                 1660           1745         120          6.0         166.0       1.0X
+date + interval(m, d)                              1672           1685          19          6.0         167.2       1.0X
+date + interval(m, d, ms)                          6462           6481          27          1.5         646.2       0.3X
+date - interval(m)                                 1456           1480          35          6.9         145.6       1.1X
+date - interval(m, d)                              1501           1509          11          6.7         150.1       1.1X
+date - interval(m, d, ms)                          6457           6466          12          1.5         645.7       0.3X
+timestamp + interval(m)                            2941           2944           4          3.4         294.1       0.6X
+timestamp + interval(m, d)                         3008           3012           6          3.3         300.8       0.6X
+timestamp + interval(m, d, ms)                     3329           3333           6          3.0         332.9       0.5X
+timestamp - interval(m)                            2964           2982          26          3.4         296.4       0.6X
+timestamp - interval(m, d)                         3030           3039          13          3.3         303.0       0.5X
+timestamp - interval(m, d, ms)                     3312           3313           1          3.0         331.2       0.5X
 
 
 ================================================================================================
@@ -28,92 +28,92 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    332            336           5         30.1          33.2       1.0X
-cast to timestamp wholestage on                     333            344          10         30.0          33.3       1.0X
+cast to timestamp wholestage off                    333            334           0         30.0          33.3       1.0X
+cast to timestamp wholestage on                     349            368          12         28.6          34.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1246           1257          16          8.0         124.6       1.0X
-year of timestamp wholestage on                    1209           1218          12          8.3         120.9       1.0X
+year of timestamp wholestage off                   1229           1229           1          8.1         122.9       1.0X
+year of timestamp wholestage on                    1218           1223           5          8.2         121.8       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1608           1616          11          6.2         160.8       1.0X
-quarter of timestamp wholestage on                 1540           1552          10          6.5         154.0       1.0X
+quarter of timestamp wholestage off                1593           1594           2          6.3         159.3       1.0X
+quarter of timestamp wholestage on                 1515           1529          14          6.6         151.5       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1242           1246           6          8.1         124.2       1.0X
-month of timestamp wholestage on                   1202           1212          11          8.3         120.2       1.0X
+month of timestamp wholestage off                  1222           1246          34          8.2         122.2       1.0X
+month of timestamp wholestage on                   1207           1232          31          8.3         120.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1879           1885           8          5.3         187.9       1.0X
-weekofyear of timestamp wholestage on              1832           1845          10          5.5         183.2       1.0X
+weekofyear of timestamp wholestage off             2453           2455           2          4.1         245.3       1.0X
+weekofyear of timestamp wholestage on              2357           2380          22          4.2         235.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1236           1239           4          8.1         123.6       1.0X
-day of timestamp wholestage on                     1206           1219          17          8.3         120.6       1.0X
+day of timestamp wholestage off                    1216           1219           5          8.2         121.6       1.0X
+day of timestamp wholestage on                     1205           1221          25          8.3         120.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1308           1309           1          7.6         130.8       1.0X
-dayofyear of timestamp wholestage on               1239           1255          15          8.1         123.9       1.1X
+dayofyear of timestamp wholestage off              1268           1274           9          7.9         126.8       1.0X
+dayofyear of timestamp wholestage on               1253           1268          10          8.0         125.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1259           1263           5          7.9         125.9       1.0X
-dayofmonth of timestamp wholestage on              1201           1205           5          8.3         120.1       1.0X
+dayofmonth of timestamp wholestage off             1223           1224           1          8.2         122.3       1.0X
+dayofmonth of timestamp wholestage on              1231           1246          14          8.1         123.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1406           1410           6          7.1         140.6       1.0X
-dayofweek of timestamp wholestage on               1387           1402          15          7.2         138.7       1.0X
+dayofweek of timestamp wholestage off              1398           1406          12          7.2         139.8       1.0X
+dayofweek of timestamp wholestage on               1387           1399          15          7.2         138.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1355           1367          18          7.4         135.5       1.0X
-weekday of timestamp wholestage on                 1311           1321          10          7.6         131.1       1.0X
+weekday of timestamp wholestage off                1327           1333           9          7.5         132.7       1.0X
+weekday of timestamp wholestage on                 1329           1333           4          7.5         132.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                    996            997           2         10.0          99.6       1.0X
-hour of timestamp wholestage on                     930            936           6         10.7          93.0       1.1X
+hour of timestamp wholestage off                   1005           1016          15          9.9         100.5       1.0X
+hour of timestamp wholestage on                     934            940           4         10.7          93.4       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                 1005           1012          10          9.9         100.5       1.0X
-minute of timestamp wholestage on                   949            952           3         10.5          94.9       1.1X
+minute of timestamp wholestage off                 1003           1009           8         10.0         100.3       1.0X
+minute of timestamp wholestage on                   934            938           7         10.7          93.4       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                 1013           1014           1          9.9         101.3       1.0X
-second of timestamp wholestage on                   933            934           2         10.7          93.3       1.1X
+second of timestamp wholestage off                  997            998           2         10.0          99.7       1.0X
+second of timestamp wholestage on                   925            935           8         10.8          92.5       1.1X
 
 
 ================================================================================================
@@ -124,15 +124,15 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         291            293           2         34.3          29.1       1.0X
-current_date wholestage on                          280            284           3         35.7          28.0       1.0X
+current_date wholestage off                         297            297           0         33.7          29.7       1.0X
+current_date wholestage on                          280            282           2         35.7          28.0       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    311            324          18         32.1          31.1       1.0X
-current_timestamp wholestage on                     275            364          85         36.3          27.5       1.1X
+current_timestamp wholestage off                    307            337          43         32.6          30.7       1.0X
+current_timestamp wholestage on                     260            284          29         38.4          26.0       1.2X
 
 
 ================================================================================================
@@ -143,43 +143,43 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1077           1079           3          9.3         107.7       1.0X
-cast to date wholestage on                         1018           1030          14          9.8         101.8       1.1X
+cast to date wholestage off                        1066           1073          10          9.4         106.6       1.0X
+cast to date wholestage on                          997           1003           6         10.0          99.7       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1257           1260           4          8.0         125.7       1.0X
-last_day wholestage on                             1218           1227          14          8.2         121.8       1.0X
+last_day wholestage off                            1238           1242           6          8.1         123.8       1.0X
+last_day wholestage on                             1259           1272          12          7.9         125.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1140           1141           1          8.8         114.0       1.0X
-next_day wholestage on                             1067           1076          11          9.4         106.7       1.1X
+next_day wholestage off                            1116           1138          32          9.0         111.6       1.0X
+next_day wholestage on                             1052           1063          11          9.5         105.2       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1062           1064           3          9.4         106.2       1.0X
-date_add wholestage on                             1046           1055          11          9.6         104.6       1.0X
+date_add wholestage off                            1048           1049           1          9.5         104.8       1.0X
+date_add wholestage on                             1035           1039           3          9.7         103.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1082           1083           1          9.2         108.2       1.0X
-date_sub wholestage on                             1047           1056          12          9.6         104.7       1.0X
+date_sub wholestage off                            1119           1127          11          8.9         111.9       1.0X
+date_sub wholestage on                             1028           1039           7          9.7         102.8       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1430           1431           1          7.0         143.0       1.0X
-add_months wholestage on                           1441           1446           8          6.9         144.1       1.0X
+add_months wholestage off                          1421           1421           0          7.0         142.1       1.0X
+add_months wholestage on                           1423           1434          11          7.0         142.3       1.0X
 
 
 ================================================================================================
@@ -190,8 +190,8 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5442           5549         150          1.8         544.2       1.0X
-format date wholestage on                          5529           5655         236          1.8         552.9       1.0X
+format date wholestage off                         5293           5296           5          1.9         529.3       1.0X
+format date wholestage on                          5143           5157          19          1.9         514.3       1.0X
 
 
 ================================================================================================
@@ -202,8 +202,8 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       7416           7440          34          1.3         741.6       1.0X
-from_unixtime wholestage on                        7372           7391          17          1.4         737.2       1.0X
+from_unixtime wholestage off                       7136           7136           1          1.4         713.6       1.0X
+from_unixtime wholestage on                        7049           7068          29          1.4         704.9       1.0X
 
 
 ================================================================================================
@@ -214,15 +214,15 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1316           1320           6          7.6         131.6       1.0X
-from_utc_timestamp wholestage on                   1268           1272           4          7.9         126.8       1.0X
+from_utc_timestamp wholestage off                  1325           1329           6          7.5         132.5       1.0X
+from_utc_timestamp wholestage on                   1269           1273           4          7.9         126.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1653           1657           6          6.0         165.3       1.0X
-to_utc_timestamp wholestage on                     1594           1599           4          6.3         159.4       1.0X
+to_utc_timestamp wholestage off                    1684           1691          10          5.9         168.4       1.0X
+to_utc_timestamp wholestage on                     1641           1648           9          6.1         164.1       1.0X
 
 
 ================================================================================================
@@ -233,29 +233,29 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        341            343           3         29.4          34.1       1.0X
-cast interval wholestage on                         279            282           1         35.8          27.9       1.2X
+cast interval wholestage off                        343            346           4         29.1          34.3       1.0X
+cast interval wholestage on                         281            282           1         35.6          28.1       1.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1862           1865           4          5.4         186.2       1.0X
-datediff wholestage on                             1769           1783          15          5.7         176.9       1.1X
+datediff wholestage off                            1831           1840          13          5.5         183.1       1.0X
+datediff wholestage on                             1759           1769          15          5.7         175.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5594           5599           7          1.8         559.4       1.0X
-months_between wholestage on                       5498           5508          11          1.8         549.8       1.0X
+months_between wholestage off                      5729           5747          25          1.7         572.9       1.0X
+months_between wholestage on                       5710           5720           9          1.8         571.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              2044           2127         117          0.5        2044.3       1.0X
-window wholestage on                              48057          48109          54          0.0       48056.9       0.0X
+window wholestage off                              2183           2189           9          0.5        2182.6       1.0X
+window wholestage on                              46835          46944          88          0.0       46834.8       0.0X
 
 
 ================================================================================================
@@ -266,134 +266,134 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2540           2542           3          3.9         254.0       1.0X
-date_trunc YEAR wholestage on                      2486           2507          29          4.0         248.6       1.0X
+date_trunc YEAR wholestage off                     2668           2672           5          3.7         266.8       1.0X
+date_trunc YEAR wholestage on                      2719           2731           9          3.7         271.9       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2542           2543           3          3.9         254.2       1.0X
-date_trunc YYYY wholestage on                      2491           2498           9          4.0         249.1       1.0X
+date_trunc YYYY wholestage off                     2672           2677           8          3.7         267.2       1.0X
+date_trunc YYYY wholestage on                      2710           2726          12          3.7         271.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2545           2569          35          3.9         254.5       1.0X
-date_trunc YY wholestage on                        2487           2493           4          4.0         248.7       1.0X
+date_trunc YY wholestage off                       2670           2673           4          3.7         267.0       1.0X
+date_trunc YY wholestage on                        2711           2720           7          3.7         271.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2590           2590           1          3.9         259.0       1.0X
-date_trunc MON wholestage on                       2506           2520          12          4.0         250.6       1.0X
+date_trunc MON wholestage off                      2674           2674           0          3.7         267.4       1.0X
+date_trunc MON wholestage on                       2667           2677          10          3.7         266.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2595           2603          11          3.9         259.5       1.0X
-date_trunc MONTH wholestage on                     2505           2516          12          4.0         250.5       1.0X
+date_trunc MONTH wholestage off                    2675           2686          16          3.7         267.5       1.0X
+date_trunc MONTH wholestage on                     2667           2674           6          3.7         266.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2605           2612          10          3.8         260.5       1.0X
-date_trunc MM wholestage on                        2501           2515          11          4.0         250.1       1.0X
+date_trunc MM wholestage off                       2673           2674           1          3.7         267.3       1.0X
+date_trunc MM wholestage on                        2664           2669           4          3.8         266.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2225           2229           5          4.5         222.5       1.0X
-date_trunc DAY wholestage on                       2184           2196           9          4.6         218.4       1.0X
+date_trunc DAY wholestage off                      2281           2288          10          4.4         228.1       1.0X
+date_trunc DAY wholestage on                       2302           2312           8          4.3         230.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2232           2236           6          4.5         223.2       1.0X
-date_trunc DD wholestage on                        2183           2190           6          4.6         218.3       1.0X
+date_trunc DD wholestage off                       2281           2283           3          4.4         228.1       1.0X
+date_trunc DD wholestage on                        2291           2302          11          4.4         229.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2194           2199           7          4.6         219.4       1.0X
-date_trunc HOUR wholestage on                      2160           2166           5          4.6         216.0       1.0X
+date_trunc HOUR wholestage off                     2331           2332           1          4.3         233.1       1.0X
+date_trunc HOUR wholestage on                      2290           2304          11          4.4         229.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    390            396           9         25.7          39.0       1.0X
-date_trunc MINUTE wholestage on                     331            337           7         30.2          33.1       1.2X
+date_trunc MINUTE wholestage off                    379            385           9         26.4          37.9       1.0X
+date_trunc MINUTE wholestage on                     371            376           5         27.0          37.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    375            381           8         26.7          37.5       1.0X
-date_trunc SECOND wholestage on                     332            346          14         30.1          33.2       1.1X
+date_trunc SECOND wholestage off                    375            376           1         26.7          37.5       1.0X
+date_trunc SECOND wholestage on                     370            376           8         27.0          37.0       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2439           2443           6          4.1         243.9       1.0X
-date_trunc WEEK wholestage on                      2390           2409          32          4.2         239.0       1.0X
+date_trunc WEEK wholestage off                     2597           2604          10          3.9         259.7       1.0X
+date_trunc WEEK wholestage on                      2591           2605          13          3.9         259.1       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3290           3292           4          3.0         329.0       1.0X
-date_trunc QUARTER wholestage on                   3214           3218           3          3.1         321.4       1.0X
+date_trunc QUARTER wholestage off                  3501           3511          14          2.9         350.1       1.0X
+date_trunc QUARTER wholestage on                   3477           3489           9          2.9         347.7       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           308            310           3         32.5          30.8       1.0X
-trunc year wholestage on                            289            293           6         34.7          28.9       1.1X
+trunc year wholestage off                           332            334           3         30.1          33.2       1.0X
+trunc year wholestage on                            332            346          17         30.1          33.2       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           309            311           3         32.4          30.9       1.0X
-trunc yyyy wholestage on                            289            294           7         34.6          28.9       1.1X
+trunc yyyy wholestage off                           331            331           0         30.2          33.1       1.0X
+trunc yyyy wholestage on                            336            339           4         29.8          33.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             311            311           0         32.2          31.1       1.0X
-trunc yy wholestage on                              288            294           7         34.7          28.8       1.1X
+trunc yy wholestage off                             330            342          17         30.3          33.0       1.0X
+trunc yy wholestage on                              333            337           3         30.0          33.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            313            313           0         32.0          31.3       1.0X
-trunc mon wholestage on                             287            290           2         34.8          28.7       1.1X
+trunc mon wholestage off                            334            335           1         30.0          33.4       1.0X
+trunc mon wholestage on                             333            347           9         30.0          33.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          310            310           0         32.3          31.0       1.0X
-trunc month wholestage on                           287            290           2         34.8          28.7       1.1X
+trunc month wholestage off                          332            333           1         30.1          33.2       1.0X
+trunc month wholestage on                           333            340           7         30.0          33.3       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             311            312           1         32.1          31.1       1.0X
-trunc mm wholestage on                              287            296           9         34.8          28.7       1.1X
+trunc mm wholestage off                             328            336          11         30.5          32.8       1.0X
+trunc mm wholestage on                              333            343          11         30.0          33.3       1.0X
 
 
 ================================================================================================
@@ -404,36 +404,36 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     169            170           1          5.9         168.9       1.0X
-to timestamp str wholestage on                      161            168          11          6.2         161.0       1.0X
+to timestamp str wholestage off                     170            171           1          5.9         170.1       1.0X
+to timestamp str wholestage on                      172            174           2          5.8         171.6       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1360           1361           1          0.7        1359.6       1.0X
-to_timestamp wholestage on                         1362           1366           6          0.7        1362.0       1.0X
+to_timestamp wholestage off                        1437           1439           3          0.7        1437.0       1.0X
+to_timestamp wholestage on                         1288           1292           5          0.8        1288.1       1.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1343           1346           4          0.7        1342.6       1.0X
-to_unix_timestamp wholestage on                    1356           1359           2          0.7        1356.2       1.0X
+to_unix_timestamp wholestage off                   1352           1353           2          0.7        1352.0       1.0X
+to_unix_timestamp wholestage on                    1314           1319           5          0.8        1314.4       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          227            230           4          4.4         227.0       1.0X
-to date str wholestage on                           299            302           3          3.3         299.0       0.8X
+to date str wholestage off                          211            215           6          4.7         210.7       1.0X
+to date str wholestage on                           217            217           1          4.6         216.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3413           3440          38          0.3        3413.0       1.0X
-to_date wholestage on                              3392           3402          12          0.3        3392.3       1.0X
+to_date wholestage off                             3281           3295          20          0.3        3280.9       1.0X
+to_date wholestage on                              3223           3239          17          0.3        3222.8       1.0X
 
 
 ================================================================================================
@@ -444,14 +444,14 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  410            415           7         12.2          82.0       1.0X
-From java.time.LocalDate                            332            333           1         15.1          66.4       1.2X
-Collect java.sql.Date                              1891           2542         829          2.6         378.1       0.2X
-Collect java.time.LocalDate                        1630           2138         441          3.1         326.0       0.3X
-From java.sql.Timestamp                             254            259           6         19.7          50.9       1.6X
-From java.time.Instant                              302            306           4         16.6          60.3       1.4X
-Collect longs                                      1134           1265         117          4.4         226.8       0.4X
-Collect java.sql.Timestamp                         1441           1458          16          3.5         288.1       0.3X
-Collect java.time.Instant                          1680           1928         253          3.0         336.0       0.2X
+From java.sql.Date                                  446            447           1         11.2          89.1       1.0X
+From java.time.LocalDate                            354            356           1         14.1          70.8       1.3X
+Collect java.sql.Date                              2722           3091         495          1.8         544.4       0.2X
+Collect java.time.LocalDate                        1786           1836          60          2.8         357.2       0.2X
+From java.sql.Timestamp                             275            287          19         18.2          55.0       1.6X
+From java.time.Instant                              325            328           3         15.4          65.0       1.4X
+Collect longs                                      1300           1321          25          3.8         260.0       0.3X
+Collect java.sql.Timestamp                         1450           1557         102          3.4         290.0       0.3X
+Collect java.time.Instant                          1499           1599          87          3.3         299.9       0.3X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 7586295778bd8..7a9aa4badfeb7 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -6,18 +6,18 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datetime +/- interval:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date + interval(m)                                 1638           1701          89          6.1         163.8       1.0X
-date + interval(m, d)                              1785           1790           7          5.6         178.5       0.9X
-date + interval(m, d, ms)                          6229           6270          58          1.6         622.9       0.3X
-date - interval(m)                                 1500           1503           4          6.7         150.0       1.1X
-date - interval(m, d)                              1764           1766           3          5.7         176.4       0.9X
-date - interval(m, d, ms)                          6428           6446          25          1.6         642.8       0.3X
-timestamp + interval(m)                            2719           2722           4          3.7         271.9       0.6X
-timestamp + interval(m, d)                         3011           3021          14          3.3         301.1       0.5X
-timestamp + interval(m, d, ms)                     3405           3412           9          2.9         340.5       0.5X
-timestamp - interval(m)                            2759           2764           7          3.6         275.9       0.6X
-timestamp - interval(m, d)                         3094           3112          25          3.2         309.4       0.5X
-timestamp - interval(m, d, ms)                     3388           3392           5          3.0         338.8       0.5X
+date + interval(m)                                 1555           1634         113          6.4         155.5       1.0X
+date + interval(m, d)                              1774           1797          33          5.6         177.4       0.9X
+date + interval(m, d, ms)                          6293           6335          59          1.6         629.3       0.2X
+date - interval(m)                                 1461           1468          10          6.8         146.1       1.1X
+date - interval(m, d)                              1741           1741           0          5.7         174.1       0.9X
+date - interval(m, d, ms)                          6503           6518          21          1.5         650.3       0.2X
+timestamp + interval(m)                            2384           2385           1          4.2         238.4       0.7X
+timestamp + interval(m, d)                         2683           2684           2          3.7         268.3       0.6X
+timestamp + interval(m, d, ms)                     2987           3001          19          3.3         298.7       0.5X
+timestamp - interval(m)                            2391           2395           5          4.2         239.1       0.7X
+timestamp - interval(m, d)                         2674           2684          14          3.7         267.4       0.6X
+timestamp - interval(m, d, ms)                     3005           3007           3          3.3         300.5       0.5X
 
 
 ================================================================================================
@@ -28,92 +28,92 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to timestamp wholestage off                    319            323           6         31.4          31.9       1.0X
-cast to timestamp wholestage on                     304            311           8         32.9          30.4       1.0X
+cast to timestamp wholestage off                    313            320          10         31.9          31.3       1.0X
+cast to timestamp wholestage on                     325            341          18         30.8          32.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 year of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-year of timestamp wholestage off                   1234           1239           6          8.1         123.4       1.0X
-year of timestamp wholestage on                    1229           1244          22          8.1         122.9       1.0X
+year of timestamp wholestage off                   1216           1216           1          8.2         121.6       1.0X
+year of timestamp wholestage on                    1226           1243          13          8.2         122.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 quarter of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-quarter of timestamp wholestage off                1440           1445           7          6.9         144.0       1.0X
-quarter of timestamp wholestage on                 1358           1361           3          7.4         135.8       1.1X
+quarter of timestamp wholestage off                1417           1421           5          7.1         141.7       1.0X
+quarter of timestamp wholestage on                 1358           1365           8          7.4         135.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 month of timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-month of timestamp wholestage off                  1239           1240           1          8.1         123.9       1.0X
-month of timestamp wholestage on                   1221           1239          26          8.2         122.1       1.0X
+month of timestamp wholestage off                  1219           1220           1          8.2         121.9       1.0X
+month of timestamp wholestage on                   1222           1227           7          8.2         122.2       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekofyear of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekofyear of timestamp wholestage off             1926           1934          11          5.2         192.6       1.0X
-weekofyear of timestamp wholestage on              1901           1911          10          5.3         190.1       1.0X
+weekofyear of timestamp wholestage off             1950           1950           0          5.1         195.0       1.0X
+weekofyear of timestamp wholestage on              1890           1899           8          5.3         189.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 day of timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-day of timestamp wholestage off                    1225           1229           6          8.2         122.5       1.0X
-day of timestamp wholestage on                     1217           1225           7          8.2         121.7       1.0X
+day of timestamp wholestage off                    1212           1213           2          8.3         121.2       1.0X
+day of timestamp wholestage on                     1216           1227          13          8.2         121.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofyear of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofyear of timestamp wholestage off              1290           1295           7          7.8         129.0       1.0X
-dayofyear of timestamp wholestage on               1262           1270           7          7.9         126.2       1.0X
+dayofyear of timestamp wholestage off              1282           1284           3          7.8         128.2       1.0X
+dayofyear of timestamp wholestage on               1269           1274           5          7.9         126.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofmonth of timestamp:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofmonth of timestamp wholestage off             1239           1239           1          8.1         123.9       1.0X
-dayofmonth of timestamp wholestage on              1215           1222           8          8.2         121.5       1.0X
+dayofmonth of timestamp wholestage off             1214           1219           7          8.2         121.4       1.0X
+dayofmonth of timestamp wholestage on              1216           1224           6          8.2         121.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 dayofweek of timestamp:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-dayofweek of timestamp wholestage off              1421           1422           2          7.0         142.1       1.0X
-dayofweek of timestamp wholestage on               1379           1388           8          7.3         137.9       1.0X
+dayofweek of timestamp wholestage off              1403           1430          39          7.1         140.3       1.0X
+dayofweek of timestamp wholestage on               1378           1386           8          7.3         137.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 weekday of timestamp:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-weekday of timestamp wholestage off                1349           1351           2          7.4         134.9       1.0X
-weekday of timestamp wholestage on                 1320           1327           8          7.6         132.0       1.0X
+weekday of timestamp wholestage off                1344           1353          13          7.4         134.4       1.0X
+weekday of timestamp wholestage on                 1316           1322           5          7.6         131.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 hour of timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-hour of timestamp wholestage off                   1024           1024           0          9.8         102.4       1.0X
-hour of timestamp wholestage on                     921            929          11         10.9          92.1       1.1X
+hour of timestamp wholestage off                    992           1000          10         10.1          99.2       1.0X
+hour of timestamp wholestage on                     960            962           3         10.4          96.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 minute of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-minute of timestamp wholestage off                  977            982           6         10.2          97.7       1.0X
-minute of timestamp wholestage on                   927            929           2         10.8          92.7       1.1X
+minute of timestamp wholestage off                  989           1000          16         10.1          98.9       1.0X
+minute of timestamp wholestage on                   965            974          13         10.4          96.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 second of timestamp:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-second of timestamp wholestage off                  987            989           3         10.1          98.7       1.0X
-second of timestamp wholestage on                   923            926           5         10.8          92.3       1.1X
+second of timestamp wholestage off                  974            977           5         10.3          97.4       1.0X
+second of timestamp wholestage on                   959            966           8         10.4          95.9       1.0X
 
 
 ================================================================================================
@@ -124,15 +124,15 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_date wholestage off                         303            311          12         33.0          30.3       1.0X
-current_date wholestage on                          266            271           5         37.5          26.6       1.1X
+current_date wholestage off                         281            282           2         35.6          28.1       1.0X
+current_date wholestage on                          294            300           5         34.0          29.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 current_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-current_timestamp wholestage off                    297            297           1         33.7          29.7       1.0X
-current_timestamp wholestage on                     264            272           7         37.8          26.4       1.1X
+current_timestamp wholestage off                    282            296          19         35.4          28.2       1.0X
+current_timestamp wholestage on                     304            331          31         32.9          30.4       0.9X
 
 
 ================================================================================================
@@ -143,43 +143,43 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast to date:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast to date wholestage off                        1062           1063           2          9.4         106.2       1.0X
-cast to date wholestage on                         1007           1021          20          9.9         100.7       1.1X
+cast to date wholestage off                        1060           1061           1          9.4         106.0       1.0X
+cast to date wholestage on                         1021           1026          10          9.8         102.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 last_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-last_day wholestage off                            1262           1265           5          7.9         126.2       1.0X
-last_day wholestage on                             1244           1256          14          8.0         124.4       1.0X
+last_day wholestage off                            1278           1280           3          7.8         127.8       1.0X
+last_day wholestage on                             1560           1566           6          6.4         156.0       0.8X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 next_day:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-next_day wholestage off                            1119           1121           2          8.9         111.9       1.0X
-next_day wholestage on                             1057           1063           6          9.5         105.7       1.1X
+next_day wholestage off                            1091           1093           3          9.2         109.1       1.0X
+next_day wholestage on                             1070           1076           9          9.3         107.0       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_add:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_add wholestage off                            1054           1059           7          9.5         105.4       1.0X
-date_add wholestage on                             1037           1069          52          9.6         103.7       1.0X
+date_add wholestage off                            1041           1047           8          9.6         104.1       1.0X
+date_add wholestage on                             1044           1050           4          9.6         104.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_sub:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_sub wholestage off                            1054           1056           4          9.5         105.4       1.0X
-date_sub wholestage on                             1036           1040           4          9.7         103.6       1.0X
+date_sub wholestage off                            1038           1040           3          9.6         103.8       1.0X
+date_sub wholestage on                             1057           1061           4          9.5         105.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 add_months:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-add_months wholestage off                          1408           1421          19          7.1         140.8       1.0X
-add_months wholestage on                           1434           1440           7          7.0         143.4       1.0X
+add_months wholestage off                          1401           1401           1          7.1         140.1       1.0X
+add_months wholestage on                           1438           1442           4          7.0         143.8       1.0X
 
 
 ================================================================================================
@@ -190,8 +190,8 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 format date:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-format date wholestage off                         5937           6169         328          1.7         593.7       1.0X
-format date wholestage on                          5836           5878          74          1.7         583.6       1.0X
+format date wholestage off                         5482           5803         454          1.8         548.2       1.0X
+format date wholestage on                          5502           5518           9          1.8         550.2       1.0X
 
 
 ================================================================================================
@@ -202,8 +202,8 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_unixtime:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_unixtime wholestage off                       8904           8914          14          1.1         890.4       1.0X
-from_unixtime wholestage on                        8918           8936          13          1.1         891.8       1.0X
+from_unixtime wholestage off                       8538           8553          22          1.2         853.8       1.0X
+from_unixtime wholestage on                        8545           8552           6          1.2         854.5       1.0X
 
 
 ================================================================================================
@@ -214,15 +214,15 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 from_utc_timestamp:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-from_utc_timestamp wholestage off                  1110           1112           3          9.0         111.0       1.0X
-from_utc_timestamp wholestage on                   1115           1119           3          9.0         111.5       1.0X
+from_utc_timestamp wholestage off                  1094           1099           8          9.1         109.4       1.0X
+from_utc_timestamp wholestage on                   1109           1114           5          9.0         110.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_utc_timestamp:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_utc_timestamp wholestage off                    1524           1525           1          6.6         152.4       1.0X
-to_utc_timestamp wholestage on                     1450           1458          14          6.9         145.0       1.1X
+to_utc_timestamp wholestage off                    1466           1469           4          6.8         146.6       1.0X
+to_utc_timestamp wholestage on                     1401           1408           7          7.1         140.1       1.0X
 
 
 ================================================================================================
@@ -233,29 +233,29 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 cast interval:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-cast interval wholestage off                        341            342           1         29.3          34.1       1.0X
-cast interval wholestage on                         285            294           7         35.1          28.5       1.2X
+cast interval wholestage off                        332            332           0         30.1          33.2       1.0X
+cast interval wholestage on                         315            324          10         31.7          31.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 datediff:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-datediff wholestage off                            1874           1881          10          5.3         187.4       1.0X
-datediff wholestage on                             1785           1791           3          5.6         178.5       1.0X
+datediff wholestage off                            1796           1802           8          5.6         179.6       1.0X
+datediff wholestage on                             1758           1764          10          5.7         175.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 months_between:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-months_between wholestage off                      5038           5042           5          2.0         503.8       1.0X
-months_between wholestage on                       4979           4987           8          2.0         497.9       1.0X
+months_between wholestage off                      4833           4836           4          2.1         483.3       1.0X
+months_between wholestage on                       4777           4780           2          2.1         477.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 window:                                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-window wholestage off                              1716           1841         177          0.6        1716.2       1.0X
-window wholestage on                              46024          46063          27          0.0       46024.1       0.0X
+window wholestage off                              1812           1908         136          0.6        1811.7       1.0X
+window wholestage on                              46279          46376          74          0.0       46278.8       0.0X
 
 
 ================================================================================================
@@ -266,134 +266,134 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YEAR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YEAR wholestage off                     2428           2429           2          4.1         242.8       1.0X
-date_trunc YEAR wholestage on                      2451           2469          12          4.1         245.1       1.0X
+date_trunc YEAR wholestage off                     2367           2368           1          4.2         236.7       1.0X
+date_trunc YEAR wholestage on                      2321           2334          22          4.3         232.1       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YYYY:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YYYY wholestage off                     2423           2426           3          4.1         242.3       1.0X
-date_trunc YYYY wholestage on                      2454           2462           8          4.1         245.4       1.0X
+date_trunc YYYY wholestage off                     2330           2334           5          4.3         233.0       1.0X
+date_trunc YYYY wholestage on                      2326           2332           5          4.3         232.6       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc YY:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc YY wholestage off                       2421           2441          28          4.1         242.1       1.0X
-date_trunc YY wholestage on                        2453           2461           9          4.1         245.3       1.0X
+date_trunc YY wholestage off                       2334           2335           1          4.3         233.4       1.0X
+date_trunc YY wholestage on                        2315           2324           6          4.3         231.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MON:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MON wholestage off                      2425           2427           3          4.1         242.5       1.0X
-date_trunc MON wholestage on                       2431           2438           9          4.1         243.1       1.0X
+date_trunc MON wholestage off                      2327           2330           4          4.3         232.7       1.0X
+date_trunc MON wholestage on                       2279           2289          12          4.4         227.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MONTH:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MONTH wholestage off                    2427           2433           8          4.1         242.7       1.0X
-date_trunc MONTH wholestage on                     2429           2435           4          4.1         242.9       1.0X
+date_trunc MONTH wholestage off                    2330           2332           2          4.3         233.0       1.0X
+date_trunc MONTH wholestage on                     2277           2284           6          4.4         227.7       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MM:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MM wholestage off                       2425           2431           9          4.1         242.5       1.0X
-date_trunc MM wholestage on                        2430           2435           4          4.1         243.0       1.0X
+date_trunc MM wholestage off                       2328           2329           2          4.3         232.8       1.0X
+date_trunc MM wholestage on                        2279           2284           4          4.4         227.9       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DAY:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DAY wholestage off                      2117           2119           4          4.7         211.7       1.0X
-date_trunc DAY wholestage on                       2036           2118         174          4.9         203.6       1.0X
+date_trunc DAY wholestage off                      1974           1984          14          5.1         197.4       1.0X
+date_trunc DAY wholestage on                       1914           1922           7          5.2         191.4       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc DD:                            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc DD wholestage off                       2116           2119           5          4.7         211.6       1.0X
-date_trunc DD wholestage on                        2035           2043          10          4.9         203.5       1.0X
+date_trunc DD wholestage off                       1967           1976          12          5.1         196.7       1.0X
+date_trunc DD wholestage on                        1913           1917           4          5.2         191.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc HOUR:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc HOUR wholestage off                     2013           2014           2          5.0         201.3       1.0X
-date_trunc HOUR wholestage on                      2077           2088          13          4.8         207.7       1.0X
+date_trunc HOUR wholestage off                     1970           1970           0          5.1         197.0       1.0X
+date_trunc HOUR wholestage on                      1945           1946           2          5.1         194.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc MINUTE:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc MINUTE wholestage off                    363            368           8         27.6          36.3       1.0X
-date_trunc MINUTE wholestage on                     321            326           7         31.2          32.1       1.1X
+date_trunc MINUTE wholestage off                    361            361           1         27.7          36.1       1.0X
+date_trunc MINUTE wholestage on                     331            336           4         30.2          33.1       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc SECOND:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc SECOND wholestage off                    365            366           0         27.4          36.5       1.0X
-date_trunc SECOND wholestage on                     319            332          16         31.4          31.9       1.1X
+date_trunc SECOND wholestage off                    360            361           1         27.8          36.0       1.0X
+date_trunc SECOND wholestage on                     335            348          15         29.8          33.5       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc WEEK:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc WEEK wholestage off                     2371           2376           7          4.2         237.1       1.0X
-date_trunc WEEK wholestage on                      2314           2322           8          4.3         231.4       1.0X
+date_trunc WEEK wholestage off                     2232           2236           6          4.5         223.2       1.0X
+date_trunc WEEK wholestage on                      2225           2232           6          4.5         222.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 date_trunc QUARTER:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-date_trunc QUARTER wholestage off                  3334           3335           1          3.0         333.4       1.0X
-date_trunc QUARTER wholestage on                   3286           3291           7          3.0         328.6       1.0X
+date_trunc QUARTER wholestage off                  3083           3086           4          3.2         308.3       1.0X
+date_trunc QUARTER wholestage on                   3073           3086          16          3.3         307.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc year:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc year wholestage off                           303            304           2         33.0          30.3       1.0X
-trunc year wholestage on                            283            291           5         35.3          28.3       1.1X
+trunc year wholestage off                           321            321           0         31.1          32.1       1.0X
+trunc year wholestage on                            299            303           5         33.5          29.9       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yyyy:                               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yyyy wholestage off                           324            330           8         30.9          32.4       1.0X
-trunc yyyy wholestage on                            283            291           9         35.3          28.3       1.1X
+trunc yyyy wholestage off                           323            327           5         30.9          32.3       1.0X
+trunc yyyy wholestage on                            299            302           3         33.4          29.9       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc yy:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc yy wholestage off                             304            305           3         32.9          30.4       1.0X
-trunc yy wholestage on                              283            302          28         35.3          28.3       1.1X
+trunc yy wholestage off                             315            315           1         31.8          31.5       1.0X
+trunc yy wholestage on                              299            304           4         33.4          29.9       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mon:                                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mon wholestage off                            315            319           6         31.7          31.5       1.0X
-trunc mon wholestage on                             284            287           5         35.3          28.4       1.1X
+trunc mon wholestage off                            320            321           1         31.2          32.0       1.0X
+trunc mon wholestage on                             299            307          10         33.4          29.9       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc month:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc month wholestage off                          305            314          13         32.8          30.5       1.0X
-trunc month wholestage on                           283            292          14         35.3          28.3       1.1X
+trunc month wholestage off                          316            317           1         31.6          31.6       1.0X
+trunc month wholestage on                           299            302           5         33.5          29.9       1.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 trunc mm:                                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-trunc mm wholestage off                             301            301           0         33.2          30.1       1.0X
-trunc mm wholestage on                              285            290           7         35.1          28.5       1.1X
+trunc mm wholestage off                             313            313           1         32.0          31.3       1.0X
+trunc mm wholestage on                              298            302           4         33.5          29.8       1.0X
 
 
 ================================================================================================
@@ -404,36 +404,36 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to timestamp str:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to timestamp str wholestage off                     218            220           3          4.6         218.4       1.0X
-to timestamp str wholestage on                      213            216           6          4.7         212.5       1.0X
+to timestamp str wholestage off                     217            217           0          4.6         217.3       1.0X
+to timestamp str wholestage on                      209            212           2          4.8         209.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_timestamp:                             Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_timestamp wholestage off                        1838           1842           5          0.5        1838.1       1.0X
-to_timestamp wholestage on                         1952           1971          11          0.5        1952.2       0.9X
+to_timestamp wholestage off                        1676           1677           2          0.6        1675.6       1.0X
+to_timestamp wholestage on                         1599           1606           8          0.6        1599.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_unix_timestamp:                        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_unix_timestamp wholestage off                   1987           1988           1          0.5        1986.9       1.0X
-to_unix_timestamp wholestage on                    1944           1948           3          0.5        1944.2       1.0X
+to_unix_timestamp wholestage off                   1582           1589           9          0.6        1582.1       1.0X
+to_unix_timestamp wholestage on                    1634           1637           3          0.6        1633.8       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to date str:                              Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to date str wholestage off                          263            264           0          3.8         263.5       1.0X
-to date str wholestage on                           263            265           2          3.8         262.6       1.0X
+to date str wholestage off                          275            282           9          3.6         275.0       1.0X
+to date str wholestage on                           264            265           2          3.8         263.5       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 to_date:                                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-to_date wholestage off                             3560           3567          11          0.3        3559.7       1.0X
-to_date wholestage on                              3525           3534          10          0.3        3524.8       1.0X
+to_date wholestage off                             3170           3188          25          0.3        3170.1       1.0X
+to_date wholestage on                              3134           3143          10          0.3        3134.3       1.0X
 
 
 ================================================================================================
@@ -444,14 +444,14 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 To/from Java's date-time:                 Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-From java.sql.Date                                  405            416          16         12.3          81.0       1.0X
-From java.time.LocalDate                            344            352          14         14.5          68.8       1.2X
-Collect java.sql.Date                              1622           2553        1372          3.1         324.4       0.2X
-Collect java.time.LocalDate                        1464           1482          20          3.4         292.8       0.3X
-From java.sql.Timestamp                             248            258          15         20.2          49.6       1.6X
-From java.time.Instant                              237            243           7         21.1          47.4       1.7X
-Collect longs                                      1252           1341         109          4.0         250.5       0.3X
-Collect java.sql.Timestamp                         1515           1516           2          3.3         302.9       0.3X
-Collect java.time.Instant                          1379           1490          96          3.6         275.8       0.3X
+From java.sql.Date                                  407            413           7         12.3          81.5       1.0X
+From java.time.LocalDate                            340            344           5         14.7          68.1       1.2X
+Collect java.sql.Date                              1700           2658        1422          2.9         340.0       0.2X
+Collect java.time.LocalDate                        1473           1494          30          3.4         294.6       0.3X
+From java.sql.Timestamp                             252            266          13         19.8          50.5       1.6X
+From java.time.Instant                              236            243           7         21.1          47.3       1.7X
+Collect longs                                      1280           1337          79          3.9         256.1       0.3X
+Collect java.sql.Timestamp                         1485           1501          15          3.4         297.0       0.3X
+Collect java.time.Instant                          1441           1465          37          3.5         288.1       0.3X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
index 03bc334471e56..d0cd591da4c94 100644
--- a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
@@ -3,110 +3,110 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       46010          46118         113          2.2         460.1       1.0X
-UTF-8 is set                                      54407          55427        1718          1.8         544.1       0.8X
+No encoding                                       68879          68993         116          1.5         688.8       1.0X
+UTF-8 is set                                     115270         115602         455          0.9        1152.7       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       26614          28220        1461          3.8         266.1       1.0X
-UTF-8 is set                                      42765          43400         550          2.3         427.6       0.6X
+No encoding                                       47452          47538         113          2.1         474.5       1.0X
+UTF-8 is set                                      77330          77354          30          1.3         773.3       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       35696          35821         113          0.3        3569.6       1.0X
-UTF-8 is set                                      55441          56176        1037          0.2        5544.1       0.6X
+No encoding                                       60470          60900         534          0.2        6047.0       1.0X
+UTF-8 is set                                     104733         104931         189          0.1       10473.3       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       61514          62968         NaN          0.0      123027.2       1.0X
-UTF-8 is set                                      72096          72933        1162          0.0      144192.7       0.9X
+No encoding                                      130302         131072         976          0.0      260604.6       1.0X
+UTF-8 is set                                     150860         151284         377          0.0      301720.1       0.9X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                  9859           9913          79          1.0         985.9       1.0X
-Select 1 column                                   10981          11003          36          0.9        1098.1       0.9X
+Select 10 columns                                 18619          18684          99          0.5        1861.9       1.0X
+Select 1 column                                   24227          24270          38          0.4        2422.7       0.8X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      3555           3579          27          2.8         355.5       1.0X
-Short column with UTF-8                            5204           5227          35          1.9         520.4       0.7X
-Wide column without encoding                      60458          60637         164          0.2        6045.8       0.1X
-Wide column with UTF-8                            77544          78111         551          0.1        7754.4       0.0X
+Short column without encoding                      7947           7971          21          1.3         794.7       1.0X
+Short column with UTF-8                           12700          12753          58          0.8        1270.0       0.6X
+Wide column without encoding                      92632          92955         463          0.1        9263.2       0.1X
+Wide column with UTF-8                           147013         147170         188          0.1       14701.3       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           342            346           3         29.2          34.2       1.0X
-from_json                                          7123           7318         179          1.4         712.3       0.0X
-json_tuple                                         9843           9957         132          1.0         984.3       0.0X
-get_json_object                                    7827           8046         194          1.3         782.7       0.0X
+Text read                                           713            734          19         14.0          71.3       1.0X
+from_json                                         22019          22429         456          0.5        2201.9       0.0X
+json_tuple                                        27987          28047          74          0.4        2798.7       0.0X
+get_json_object                                   21468          21870         350          0.5        2146.8       0.0X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          1856           1884          32         26.9          37.1       1.0X
-schema inferring                                  16734          16900         153          3.0         334.7       0.1X
-parsing                                           14884          15203         470          3.4         297.7       0.1X
+Text read                                          2887           2910          24         17.3          57.7       1.0X
+schema inferring                                  31793          31843          43          1.6         635.9       0.1X
+parsing                                           36791          37104         294          1.4         735.8       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5932           6148         228          8.4         118.6       1.0X
-Schema inferring                                  20836          21938        1086          2.4         416.7       0.3X
-Parsing without charset                           18134          18661         457          2.8         362.7       0.3X
-Parsing with UTF-8                                27734          28069         378          1.8         554.7       0.2X
+Text read                                         10570          10611          45          4.7         211.4       1.0X
+Schema inferring                                  48729          48763          41          1.0         974.6       0.2X
+Parsing without charset                           35490          35648         141          1.4         709.8       0.3X
+Parsing with UTF-8                                63853          63994         163          0.8        1277.1       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      889            914          28         11.2          88.9       1.0X
-to_json(timestamp)                                 7920           8172         353          1.3         792.0       0.1X
-write timestamps to files                          6726           6822         129          1.5         672.6       0.1X
-Create a dataset of dates                           953            963          12         10.5          95.3       0.9X
-to_json(date)                                      5370           5705         320          1.9         537.0       0.2X
-write dates to files                               4109           4166          52          2.4         410.9       0.2X
+Create a dataset of timestamps                     2187           2190           5          4.6         218.7       1.0X
+to_json(timestamp)                                16262          16503         323          0.6        1626.2       0.1X
+write timestamps to files                         11679          11692          12          0.9        1167.9       0.2X
+Create a dataset of dates                          2297           2310          12          4.4         229.7       1.0X
+to_json(date)                                     10904          10956          46          0.9        1090.4       0.2X
+write dates to files                               6610           6645          35          1.5         661.0       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 11.0.5+10-LTS on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1614           1675          55          6.2         161.4       1.0X
-read timestamps from files                        16640          16858         209          0.6        1664.0       0.1X
-infer timestamps from files                       33239          33388         227          0.3        3323.9       0.0X
-read date text from files                          1310           1340          44          7.6         131.0       1.2X
-read date from files                               9470           9513          41          1.1         947.0       0.2X
-timestamp strings                                  1303           1342          47          7.7         130.3       1.2X
-parse timestamps from Dataset[String]             17650          18073         380          0.6        1765.0       0.1X
-infer timestamps from Dataset[String]             32623          34065        1330          0.3        3262.3       0.0X
-date strings                                       1864           1871           7          5.4         186.4       0.9X
-parse dates from Dataset[String]                  10914          11316         482          0.9        1091.4       0.1X
-from_json(timestamp)                              21102          21990         929          0.5        2110.2       0.1X
-from_json(date)                                   15275          15961         598          0.7        1527.5       0.1X
+read timestamp text from files                     2524           2530           9          4.0         252.4       1.0X
+read timestamps from files                        41002          41052          59          0.2        4100.2       0.1X
+infer timestamps from files                       84621          84939         526          0.1        8462.1       0.0X
+read date text from files                          2292           2302           9          4.4         229.2       1.1X
+read date from files                              16954          16976          21          0.6        1695.4       0.1X
+timestamp strings                                  3067           3077          13          3.3         306.7       0.8X
+parse timestamps from Dataset[String]             48690          48971         243          0.2        4869.0       0.1X
+infer timestamps from Dataset[String]             97463          97786         338          0.1        9746.3       0.0X
+date strings                                       3952           3956           3          2.5         395.2       0.6X
+parse dates from Dataset[String]                  24210          24241          30          0.4        2421.0       0.1X
+from_json(timestamp)                              71710          72242         629          0.1        7171.0       0.0X
+from_json(date)                                   42465          42481          13          0.2        4246.5       0.1X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt
index 0f188c4cdea56..46d2410fb47c3 100644
--- a/sql/core/benchmarks/JsonBenchmark-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-results.txt
@@ -3,110 +3,110 @@ Benchmark for performance of JSON parsing
 ================================================================================================
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       38998          41002         NaN          2.6         390.0       1.0X
-UTF-8 is set                                      61231          63282        1854          1.6         612.3       0.6X
+No encoding                                       63981          64044          56          1.6         639.8       1.0X
+UTF-8 is set                                     112672         113350         962          0.9        1126.7       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       28272          28338          70          3.5         282.7       1.0X
-UTF-8 is set                                      58681          62243        1517          1.7         586.8       0.5X
+No encoding                                       51256          51449         180          2.0         512.6       1.0X
+UTF-8 is set                                      83694          83859         148          1.2         836.9       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       44026          51829        1329          0.2        4402.6       1.0X
-UTF-8 is set                                      65839          68596         500          0.2        6583.9       0.7X
+No encoding                                       58440          59097         569          0.2        5844.0       1.0X
+UTF-8 is set                                     102746         102883         198          0.1       10274.6       0.6X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       72144          74820         NaN          0.0      144287.6       1.0X
-UTF-8 is set                                      69571          77888         NaN          0.0      139142.3       1.0X
+No encoding                                      128982         129304         356          0.0      257965.0       1.0X
+UTF-8 is set                                     147247         147415         231          0.0      294494.1       0.9X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                  9502           9604         106          1.1         950.2       1.0X
-Select 1 column                                   11861          11948         109          0.8        1186.1       0.8X
+Select 10 columns                                 18837          19048         331          0.5        1883.7       1.0X
+Select 1 column                                   24707          24723          14          0.4        2470.7       0.8X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      3830           3846          15          2.6         383.0       1.0X
-Short column with UTF-8                            5538           5543           7          1.8         553.8       0.7X
-Wide column without encoding                      66899          69158         NaN          0.1        6689.9       0.1X
-Wide column with UTF-8                            90052          93235         NaN          0.1        9005.2       0.0X
+Short column without encoding                      8218           8234          17          1.2         821.8       1.0X
+Short column with UTF-8                           12374          12438         107          0.8        1237.4       0.7X
+Wide column without encoding                     136918         137298         345          0.1       13691.8       0.1X
+Wide column with UTF-8                           176961         177142         257          0.1       17696.1       0.0X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           659            674          13         15.2          65.9       1.0X
-from_json                                          7676           7943         405          1.3         767.6       0.1X
-json_tuple                                         9881          10172         273          1.0         988.1       0.1X
-get_json_object                                    7949           8055         119          1.3         794.9       0.1X
+Text read                                          1268           1278          12          7.9         126.8       1.0X
+from_json                                         23348          23479         176          0.4        2334.8       0.1X
+json_tuple                                        29606          30221        1024          0.3        2960.6       0.0X
+get_json_object                                   21898          22148         226          0.5        2189.8       0.1X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          3314           3326          17         15.1          66.3       1.0X
-schema inferring                                  16549          17037         484          3.0         331.0       0.2X
-parsing                                           15138          15283         172          3.3         302.8       0.2X
+Text read                                          5887           5944          49          8.5         117.7       1.0X
+schema inferring                                  46696          47054         312          1.1         933.9       0.1X
+parsing                                           32336          32450         129          1.5         646.7       0.2X
 
 Preparing data for benchmarking ...
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5136           5446         268          9.7         102.7       1.0X
-Schema inferring                                  19864          20568        1191          2.5         397.3       0.3X
-Parsing without charset                           17535          17888         329          2.9         350.7       0.3X
-Parsing with UTF-8                                25609          25758         218          2.0         512.2       0.2X
+Text read                                          9756           9769          11          5.1         195.1       1.0X
+Schema inferring                                  51318          51433         108          1.0        1026.4       0.2X
+Parsing without charset                           43609          43743         118          1.1         872.2       0.2X
+Parsing with UTF-8                                60775          60844         106          0.8        1215.5       0.2X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                      784            790           7         12.8          78.4       1.0X
-to_json(timestamp)                                 8005           8055          50          1.2         800.5       0.1X
-write timestamps to files                          6515           6559          45          1.5         651.5       0.1X
-Create a dataset of dates                           854            881          24         11.7          85.4       0.9X
-to_json(date)                                      5187           5194           7          1.9         518.7       0.2X
-write dates to files                               3663           3684          22          2.7         366.3       0.2X
+Create a dataset of timestamps                     1998           2015          17          5.0         199.8       1.0X
+to_json(timestamp)                                18156          18317         263          0.6        1815.6       0.1X
+write timestamps to files                         12912          12917           5          0.8        1291.2       0.2X
+Create a dataset of dates                          2209           2270          53          4.5         220.9       0.9X
+to_json(date)                                      9433           9489          90          1.1         943.3       0.2X
+write dates to files                               6915           6923           8          1.4         691.5       0.3X
 
-Java HotSpot(TM) 64-Bit Server VM 1.8.0_231-b11 on Mac OS X 10.15.4
-Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz
+OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
+Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     1297           1316          26          7.7         129.7       1.0X
-read timestamps from files                        16915          17723         963          0.6        1691.5       0.1X
-infer timestamps from files                       33967          34304         360          0.3        3396.7       0.0X
-read date text from files                          1095           1100           7          9.1         109.5       1.2X
-read date from files                               8376           8513         209          1.2         837.6       0.2X
-timestamp strings                                  1807           1816           8          5.5         180.7       0.7X
-parse timestamps from Dataset[String]             18189          18242          74          0.5        1818.9       0.1X
-infer timestamps from Dataset[String]             37906          38547         571          0.3        3790.6       0.0X
-date strings                                       2191           2194           4          4.6         219.1       0.6X
-parse dates from Dataset[String]                  11593          11625          33          0.9        1159.3       0.1X
-from_json(timestamp)                              22589          22650         101          0.4        2258.9       0.1X
-from_json(date)                                   16479          16619         159          0.6        1647.9       0.1X
+read timestamp text from files                     2395           2412          17          4.2         239.5       1.0X
+read timestamps from files                        47269          47334          89          0.2        4726.9       0.1X
+infer timestamps from files                       91806          91851          67          0.1        9180.6       0.0X
+read date text from files                          2118           2133          13          4.7         211.8       1.1X
+read date from files                              17267          17340         115          0.6        1726.7       0.1X
+timestamp strings                                  3906           3935          26          2.6         390.6       0.6X
+parse timestamps from Dataset[String]             52244          52534         279          0.2        5224.4       0.0X
+infer timestamps from Dataset[String]            100488         100714         198          0.1       10048.8       0.0X
+date strings                                       4572           4584          12          2.2         457.2       0.5X
+parse dates from Dataset[String]                  26749          26768          17          0.4        2674.9       0.1X
+from_json(timestamp)                              71414          71867         556          0.1        7141.4       0.0X
+from_json(date)                                   45322          45549         250          0.2        4532.2       0.1X
 
 

From b8da5be6cc3e00118b439ffb94fecb029c09ebf5 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 25 May 2020 18:36:02 -0700
Subject: [PATCH 0732/1280] [SPARK-31818][SQL] Fix pushing down filters with
 `java.time.Instant` values in ORC

### What changes were proposed in this pull request?
Convert `java.time.Instant` to `java.sql.Timestamp` in pushed down filters to ORC datasource when Java 8 time API enabled.

### Why are the changes needed?
The changes fix the exception raised while pushing date filters when `spark.sql.datetime.java8API.enabled` is set to `true`:
```
java.lang.IllegalArgumentException: Wrong value class java.time.Instant for TIMESTAMP.EQUALS leaf
 at org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$PredicateLeafImpl.checkLiteralType(SearchArgumentImpl.java:192)
 at org.apache.hadoop.hive.ql.io.sarg.SearchArgumentImpl$PredicateLeafImpl.<init>(SearchArgumentImpl.java:75)
```

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
Added tests to `OrcFilterSuite`.

Closes #28636 from MaxGekk/orc-timestamp-filter-pushdown.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6c80ebbccb7f354f645dd63a73114332d26f901f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/dsl/package.scala      |  3 +-
 .../datasources/orc/OrcFilters.scala          |  6 +-
 .../datasources/orc/OrcFilterSuite.scala      | 56 +++++++++++--------
 .../datasources/orc/OrcFilters.scala          |  6 +-
 .../datasources/orc/OrcFilterSuite.scala      | 54 +++++++++++-------
 5 files changed, 77 insertions(+), 48 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index fe3fea5e35b1b..26f5bee72092c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst
 
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{Instant, LocalDate}
 
 import scala.language.implicitConversions
 
@@ -152,6 +152,7 @@ package object dsl {
     implicit def bigDecimalToLiteral(d: java.math.BigDecimal): Literal = Literal(d)
     implicit def decimalToLiteral(d: Decimal): Literal = Literal(d)
     implicit def timestampToLiteral(t: Timestamp): Literal = Literal(t)
+    implicit def instantToLiteral(i: Instant): Literal = Literal(i)
     implicit def binaryToLiteral(a: Array[Byte]): Literal = Literal(a)
 
     implicit def symbolToUnresolvedAttribute(s: Symbol): analysis.UnresolvedAttribute =
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index a01d5a44da714..b68563956c82c 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.time.LocalDate
+import java.time.{Instant, LocalDate}
 
 import org.apache.orc.storage.common.`type`.HiveDecimal
 import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
@@ -26,7 +26,7 @@ import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.orc.storage.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
@@ -167,6 +167,8 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
     case _: DateType if value.isInstanceOf[LocalDate] =>
       toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
+    case _: TimestampType if value.isInstanceOf[Instant] =>
+      toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
     case _ => value
   }
 
diff --git a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index a1c325e7bb876..88b4b243b543a 100644
--- a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -245,29 +245,41 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+    val input = Seq(
+      "1000-01-01 01:02:03",
+      "1582-10-01 00:11:22",
+      "1900-01-01 23:59:59",
+      "2020-05-25 10:11:12").map(Timestamp.valueOf)
 
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
-
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) <=> $"_1",
-        PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val timestamps = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
+
+            checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(
+              Literal(timestamps(2)) < $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) >= $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 445a52cece1c3..4b642080d25ad 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
-import java.time.LocalDate
+import java.time.{Instant, LocalDate}
 
 import org.apache.hadoop.hive.common.`type`.HiveDecimal
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
@@ -26,7 +26,7 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.util.DateTimeUtils.{localDateToDays, toJavaDate}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
 import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
@@ -167,6 +167,8 @@ private[sql] object OrcFilters extends OrcFiltersBase {
       new HiveDecimalWritable(HiveDecimal.create(value.asInstanceOf[java.math.BigDecimal]))
     case _: DateType if value.isInstanceOf[LocalDate] =>
       toJavaDate(localDateToDays(value.asInstanceOf[LocalDate]))
+    case _: TimestampType if value.isInstanceOf[Instant] =>
+      toJavaTimestamp(instantToMicros(value.asInstanceOf[Instant]))
     case _ => value
   }
 
diff --git a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index 815af05beb002..2263179515a5f 100644
--- a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -246,29 +246,41 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
   }
 
   test("filter pushdown - timestamp") {
-    val timeString = "2015-08-20 14:57:00"
-    val timestamps = (1 to 4).map { i =>
-      val milliseconds = Timestamp.valueOf(timeString).getTime + i * 3600
-      new Timestamp(milliseconds)
-    }
-    withOrcDataFrame(timestamps.map(Tuple1(_))) { implicit df =>
-      checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
-
-      checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+    val input = Seq(
+      "1000-01-01 01:02:03",
+      "1582-10-01 00:11:22",
+      "1900-01-01 23:59:59",
+      "2020-05-25 10:11:12").map(Timestamp.valueOf)
 
-      checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
+    withOrcFile(input.map(Tuple1(_))) { path =>
+      Seq(false, true).foreach { java8Api =>
+        withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+          readFile(path) { implicit df =>
+            val timestamps = input.map(Literal(_))
+            checkFilterPredicate($"_1".isNull, PredicateLeaf.Operator.IS_NULL)
 
-      checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
-      checkFilterPredicate(
-        Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
-      checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
-      checkFilterPredicate(Literal(timestamps(2)) < $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(0)) >= $"_1", PredicateLeaf.Operator.LESS_THAN_EQUALS)
-      checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" === timestamps(0), PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate($"_1" <=> timestamps(0), PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+
+            checkFilterPredicate($"_1" < timestamps(1), PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate($"_1" > timestamps(2), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" <= timestamps(0), PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate($"_1" >= timestamps(3), PredicateLeaf.Operator.LESS_THAN)
+
+            checkFilterPredicate(Literal(timestamps(0)) === $"_1", PredicateLeaf.Operator.EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) <=> $"_1", PredicateLeaf.Operator.NULL_SAFE_EQUALS)
+            checkFilterPredicate(Literal(timestamps(1)) > $"_1", PredicateLeaf.Operator.LESS_THAN)
+            checkFilterPredicate(
+              Literal(timestamps(2)) < $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(
+              Literal(timestamps(0)) >= $"_1",
+              PredicateLeaf.Operator.LESS_THAN_EQUALS)
+            checkFilterPredicate(Literal(timestamps(3)) <= $"_1", PredicateLeaf.Operator.LESS_THAN)
+          }
+        }
+      }
     }
   }
 

From 1ebc1dfb47ed1f00681dd9f63b660b2195a93fe3 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 25 May 2020 20:36:00 -0700
Subject: [PATCH 0733/1280] [SPARK-31808][SQL] Makes struct function's output
 name and class name pretty

### What changes were proposed in this pull request?

This PR proposes to set the alias, and class name in its `ExpressionInfo` for `struct`.
- Class name in `ExpressionInfo`
  - from: `org.apache.spark.sql.catalyst.expressions.NamedStruct`
  - to:`org.apache.spark.sql.catalyst.expressions.CreateNamedStruct`
- Alias name: `named_struct(col1, v, ...)` -> `struct(v, ...)`

This PR takes over https://github.com/apache/spark/pull/28631

### Why are the changes needed?

To show the correct output name and class names to users.

### Does this PR introduce _any_ user-facing change?

Yes.

**Before:**

```scala
scala> sql("DESC FUNCTION struct").show(false)
+------------------------------------------------------------------------------------+
|function_desc                                                                       |
+------------------------------------------------------------------------------------+
|Function: struct                                                                    |
|Class: org.apache.spark.sql.catalyst.expressions.NamedStruct                        |
|Usage: struct(col1, col2, col3, ...) - Creates a struct with the given field values.|
+------------------------------------------------------------------------------------+
```

```scala
scala> sql("SELECT struct(1, 2)").show(false)
+------------------------------+
|named_struct(col1, 1, col2, 2)|
+------------------------------+
|[1, 2]                        |
+------------------------------+
```

**After:**

```scala
scala> sql("DESC FUNCTION struct").show(false)
+------------------------------------------------------------------------------------+
|function_desc                                                                       |
+------------------------------------------------------------------------------------+
|Function: struct                                                                    |
|Class: org.apache.spark.sql.catalyst.expressions.CreateNamedStruct                  |
|Usage: struct(col1, col2, col3, ...) - Creates a struct with the given field values.|
+------------------------------------------------------------------------------------+
```

```scala
scala> sql("SELECT struct(1, 2)").show(false)
+------------+
|struct(1, 2)|
+------------+
|[1, 2]      |
+------------+
```

### How was this patch tested?

Manually tested, and Jenkins tests.

Closes #28633 from HyukjinKwon/SPARK-31808.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit df2a1fe131476aac128d63df9b06ec4bca0c2c07)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/complexTypeCreator.scala      | 34 ++++++++++++++++---
 .../sql/catalyst/parser/AstBuilder.scala      |  2 +-
 .../org/apache/spark/sql/functions.scala      |  2 +-
 .../sql-functions/sql-expression-schema.md    |  6 ++--
 .../sql-tests/results/group-by-filter.sql.out |  2 +-
 .../sql-tests/results/group-by.sql.out        |  2 +-
 .../sql-tests/results/struct.sql.out          |  2 +-
 .../typeCoercion/native/mapZipWith.sql.out    |  4 +--
 .../results/udf/udf-group-by.sql.out          |  2 +-
 9 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 858c91a4d8e86..5212ef3930bc9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion}
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.{FUNC_ALIAS, FunctionBuilder}
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.util._
@@ -311,7 +311,12 @@ case object NamePlaceholder extends LeafExpression with Unevaluable {
 /**
  * Returns a Row containing the evaluation of all children expressions.
  */
-object CreateStruct extends FunctionBuilder {
+object CreateStruct {
+  /**
+   * Returns a named struct with generated names or using the names when available.
+   * It should not be used for `struct` expressions or functions explicitly called
+   * by users.
+   */
   def apply(children: Seq[Expression]): CreateNamedStruct = {
     CreateNamedStruct(children.zipWithIndex.flatMap {
       case (e: NamedExpression, _) if e.resolved => Seq(Literal(e.name), e)
@@ -320,12 +325,23 @@ object CreateStruct extends FunctionBuilder {
     })
   }
 
+  /**
+   * Returns a named struct with a pretty SQL name. It will show the pretty SQL string
+   * in its output column name as if `struct(...)` was called. Should be
+   * used for `struct` expressions or functions explicitly called by users.
+   */
+  def create(children: Seq[Expression]): CreateNamedStruct = {
+    val expr = CreateStruct(children)
+    expr.setTagValue(FUNC_ALIAS, "struct")
+    expr
+  }
+
   /**
    * Entry to use in the function registry.
    */
   val registryEntry: (String, (ExpressionInfo, FunctionBuilder)) = {
     val info: ExpressionInfo = new ExpressionInfo(
-      "org.apache.spark.sql.catalyst.expressions.NamedStruct",
+      classOf[CreateNamedStruct].getCanonicalName,
       null,
       "struct",
       "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values.",
@@ -335,7 +351,7 @@ object CreateStruct extends FunctionBuilder {
       "",
       "",
       "")
-    ("struct", (info, this))
+    ("struct", (info, this.create))
   }
 }
 
@@ -433,7 +449,15 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression {
        """.stripMargin, isNull = FalseLiteral)
   }
 
-  override def prettyName: String = "named_struct"
+  // There is an alias set at `CreateStruct.create`. If there is an alias,
+  // this is the struct function explicitly called by a user and we should
+  // respect it in the SQL string as `struct(...)`.
+  override def prettyName: String = getTagValue(FUNC_ALIAS).getOrElse("named_struct")
+
+  override def sql: String = getTagValue(FUNC_ALIAS).map { alias =>
+    val childrenSQL = children.indices.filter(_ % 2 == 1).map(children(_).sql).mkString(", ")
+    s"$alias($childrenSQL)"
+  }.getOrElse(super.sql)
 }
 
 /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 886f77e473580..f6877d2ed62f3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1519,7 +1519,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    * Create a [[CreateStruct]] expression.
    */
   override def visitStruct(ctx: StructContext): Expression = withOrigin(ctx) {
-    CreateStruct(ctx.argument.asScala.map(expression))
+    CreateStruct.create(ctx.argument.asScala.map(expression))
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 7c1457df9ca3c..57842ed06e0b3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -1283,7 +1283,7 @@ object functions {
    * @since 1.4.0
    */
   @scala.annotation.varargs
-  def struct(cols: Column*): Column = withExpr { CreateStruct(cols.map(_.expr)) }
+  def struct(cols: Column*): Column = withExpr { CreateStruct.create(cols.map(_.expr)) }
 
   /**
    * Creates a new struct column that composes multiple input columns.
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 9e24a54058ec5..ed972ed1eacff 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -2,7 +2,7 @@
 ## Summary
   - Number of queries: 328
   - Number of expressions that missing example: 34
-  - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,struct,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch
+  - Expressions missing examples: and,string,tinyint,double,smallint,date,decimal,boolean,float,binary,bigint,int,timestamp,struct,cume_dist,dense_rank,input_file_block_length,input_file_block_start,input_file_name,lag,lead,monotonically_increasing_id,ntile,!,not,or,percent_rank,rank,row_number,spark_partition_id,version,window,positive,count_min_sketch
 ## Schema of Built-in Functions
 | Class name | Function name or alias | Query example | Output schema |
 | ---------- | ---------------------- | ------------- | ------------- |
@@ -79,6 +79,7 @@
 | org.apache.spark.sql.catalyst.expressions.CreateArray | array | SELECT array(1, 2, 3) | struct<array(1, 2, 3):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.CreateMap | map | SELECT map(1.0, '2', 3.0, '4') | struct<map(1.0, 2, 3.0, 4):map<decimal(2,1),string>> |
 | org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | named_struct | SELECT named_struct("a", 1, "b", 2, "c", 3) | struct<named_struct(a, 1, b, 2, c, 3):struct<a:int,b:int,c:int>> |
+| org.apache.spark.sql.catalyst.expressions.CreateNamedStruct | struct | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.CsvToStructs | from_csv | SELECT from_csv('1, 0.8', 'a INT, b DOUBLE') | struct<from_csv(1, 0.8):struct<a:int,b:double>> |
 | org.apache.spark.sql.catalyst.expressions.Cube | cube | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name, age) | struct<name:string,age:int,count(1):bigint> |
 | org.apache.spark.sql.catalyst.expressions.CumeDist | cume_dist | N/A | N/A |
@@ -168,7 +169,7 @@
 | org.apache.spark.sql.catalyst.expressions.MapEntries | map_entries | SELECT map_entries(map(1, 'a', 2, 'b')) | struct<map_entries(map(1, a, 2, b)):array<struct<key:int,value:string>>> |
 | org.apache.spark.sql.catalyst.expressions.MapFilter | map_filter | SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v) | struct<map_filter(map(1, 0, 2, 2, 3, -1), lambdafunction((namedlambdavariable() > namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):map<int,int>> |
 | org.apache.spark.sql.catalyst.expressions.MapFromArrays | map_from_arrays | SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')) | struct<map_from_arrays(array(1.0, 3.0), array(2, 4)):map<decimal(2,1),string>> |
-| org.apache.spark.sql.catalyst.expressions.MapFromEntries | map_from_entries | SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))) | struct<map_from_entries(array(named_struct(col1, 1, col2, a), named_struct(col1, 2, col2, b))):map<int,string>> |
+| org.apache.spark.sql.catalyst.expressions.MapFromEntries | map_from_entries | SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))) | struct<map_from_entries(array(struct(1, a), struct(2, b))):map<int,string>> |
 | org.apache.spark.sql.catalyst.expressions.MapKeys | map_keys | SELECT map_keys(map(1, 'a', 2, 'b')) | struct<map_keys(map(1, a, 2, b)):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.MapValues | map_values | SELECT map_values(map(1, 'a', 2, 'b')) | struct<map_values(map(1, a, 2, b)):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.MapZipWith | map_zip_with | SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)) | struct<map_zip_with(map(1, a, 2, b), map(1, x, 2, y), lambdafunction(concat(namedlambdavariable(), namedlambdavariable()), namedlambdavariable(), namedlambdavariable(), namedlambdavariable())):map<int,string>> |
@@ -181,7 +182,6 @@
 | org.apache.spark.sql.catalyst.expressions.Murmur3Hash | hash | SELECT hash('Spark', array(123), 2) | struct<hash(Spark, array(123), 2):int> |
 | org.apache.spark.sql.catalyst.expressions.NTile | ntile | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.NaNvl | nanvl | SELECT nanvl(cast('NaN' as double), 123) | struct<nanvl(CAST(NaN AS DOUBLE), CAST(123 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.NamedStruct | struct | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.NextDay | next_day | SELECT next_day('2015-01-14', 'TU') | struct<next_day(CAST(2015-01-14 AS DATE), TU):date> |
 | org.apache.spark.sql.catalyst.expressions.Not | ! | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.Not | not | N/A | N/A |
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
index a4c7c2cf90cd7..9ff7f711ea296 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by-filter.sql.out
@@ -272,7 +272,7 @@ struct<foo:string,approx_count_distinct(a) FILTER (WHERE (b >= 0)):bigint>
 -- !query
 SELECT 'foo', MAX(STRUCT(a)) FILTER (WHERE b >= 1) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
-struct<foo:string,max(named_struct(a, a)) FILTER (WHERE (b >= 1)):struct<a:int>>
+struct<foo:string,max(struct(a)) FILTER (WHERE (b >= 1)):struct<a:int>>
 -- !query output
 
 
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
index 7bfdd0ad53a95..50eb2a9f22f69 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
@@ -87,7 +87,7 @@ struct<foo:string,approx_count_distinct(a):bigint>
 -- !query
 SELECT 'foo', MAX(STRUCT(a)) FROM testData WHERE a = 0 GROUP BY 1
 -- !query schema
-struct<foo:string,max(named_struct(a, a)):struct<a:int>>
+struct<foo:string,max(struct(a)):struct<a:int>>
 -- !query output
 
 
diff --git a/sql/core/src/test/resources/sql-tests/results/struct.sql.out b/sql/core/src/test/resources/sql-tests/results/struct.sql.out
index f294c5213d319..3b610edc47169 100644
--- a/sql/core/src/test/resources/sql-tests/results/struct.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/struct.sql.out
@@ -83,7 +83,7 @@ struct<ID:int,NST:string>
 -- !query
 SELECT ID, STRUCT(ST.C as STC, ST.D as STD).STD FROM tbl_x
 -- !query schema
-struct<ID:int,named_struct(STC, ST.C AS `C` AS `STC`, STD, ST.D AS `D` AS `STD`).STD:string>
+struct<ID:int,struct(ST.C AS `C` AS `STC`, ST.D AS `D` AS `STD`).STD:string>
 -- !query output
 1	delta
 2	eta
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
index ed7ab5a342c12..d046ff249379f 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapZipWith.sql.out
@@ -85,7 +85,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_zip_with(various_maps.`decimal_map1`, various_maps.`decimal_map2`, lambdafunction(named_struct(NamePlaceholder(), k, NamePlaceholder(), v1, NamePlaceholder(), v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,0), decimal(36,35)].; line 1 pos 7
+cannot resolve 'map_zip_with(various_maps.`decimal_map1`, various_maps.`decimal_map2`, lambdafunction(struct(k, v1, v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,0), decimal(36,35)].; line 1 pos 7
 
 
 -- !query
@@ -113,7 +113,7 @@ FROM various_maps
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-cannot resolve 'map_zip_with(various_maps.`decimal_map2`, various_maps.`int_map`, lambdafunction(named_struct(NamePlaceholder(), k, NamePlaceholder(), v1, NamePlaceholder(), v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,35), int].; line 1 pos 7
+cannot resolve 'map_zip_with(various_maps.`decimal_map2`, various_maps.`int_map`, lambdafunction(struct(k, v1, v2), k, v1, v2))' due to argument data type mismatch: The input to function map_zip_with should have been two maps with compatible key types, but the key types are [decimal(36,35), int].; line 1 pos 7
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
index 6403406413db9..da5256f5c0453 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-group-by.sql.out
@@ -87,7 +87,7 @@ struct<foo:string,CAST(udf(cast(approx_count_distinct(cast(udf(cast(a as string)
 -- !query
 SELECT 'foo', MAX(STRUCT(udf(a))) FROM testData WHERE a = 0 GROUP BY udf(1)
 -- !query schema
-struct<foo:string,max(named_struct(col1, CAST(udf(cast(a as string)) AS INT))):struct<col1:int>>
+struct<foo:string,max(struct(CAST(udf(cast(a as string)) AS INT))):struct<col1:int>>
 -- !query output
 
 

From 363899267d6f8afd5314f694a71fb034a7a2d409 Mon Sep 17 00:00:00 2001
From: Prakhar Jain <prakharjain09@gmail.com>
Date: Tue, 26 May 2020 14:13:02 +0900
Subject: [PATCH 0734/1280] [SPARK-31810][TEST] Fix AlterTableRecoverPartitions
 test using incorrect api to modify RDD_PARALLEL_LISTING_THRESHOLD

### What changes were proposed in this pull request?
Use the correct API in AlterTableRecoverPartition tests to modify the `RDD_PARALLEL_LISTING_THRESHOLD` conf.

### Why are the changes needed?
The existing AlterTableRecoverPartitions test modify the RDD_PARALLEL_LISTING_THRESHOLD as a SQLConf using the withSQLConf API. But since, this is not a SQLConf, it is not overridden and so the test doesn't end up testing the required behaviour.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
This is UT Fix. UTs are still passing after the fix.

Closes #28634 from prakharjain09/SPARK-31810-fix-recover-partitions.

Authored-by: Prakhar Jain <prakharjain09@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 452594f5a43bc5d98fb42bf0927638d139f6d17c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/command/DDLSuite.scala     | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 10ad8acc68937..e4709e469dca3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -1203,14 +1203,24 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
   }
 
   test("alter table: recover partitions (sequential)") {
-    withSQLConf(RDD_PARALLEL_LISTING_THRESHOLD.key -> "10") {
+    val oldRddParallelListingThreshold = spark.sparkContext.conf.get(
+      RDD_PARALLEL_LISTING_THRESHOLD)
+    try {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD.key, "10")
       testRecoverPartitions()
+    } finally {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD, oldRddParallelListingThreshold)
     }
   }
 
   test("alter table: recover partition (parallel)") {
-    withSQLConf(RDD_PARALLEL_LISTING_THRESHOLD.key -> "0") {
+    val oldRddParallelListingThreshold = spark.sparkContext.conf.get(
+      RDD_PARALLEL_LISTING_THRESHOLD)
+    try {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD.key, "0")
       testRecoverPartitions()
+    } finally {
+      spark.sparkContext.conf.set(RDD_PARALLEL_LISTING_THRESHOLD, oldRddParallelListingThreshold)
     }
   }
 

From b35bd5712f38485344d8f6dbf8319dbd49793374 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 26 May 2020 05:15:51 +0000
Subject: [PATCH 0735/1280] [SPARK-31806][SQL][TESTS] Check reading
 date/timestamp from legacy parquet: dictionary encoding, w/o Spark version

### What changes were proposed in this pull request?
1. Add the following parquet files to the resource folder `sql/core/src/test/resources/test-data`:
   - Files saved by Spark 2.4.5 (https://github.com/apache/spark/commit/cee4ecbb16917fa85f02c635925e2687400aa56b) without meta info `org.apache.spark.version`
      - `before_1582_date_v2_4_5.snappy.parquet` with 2 date columns of the type **INT32 L:DATE** - `PLAIN` (8 date values of `1001-01-01`) and `PLAIN_DICTIONARY` (`1001-01-01`..`1001-01-08`).
      - `before_1582_timestamp_micros_v2_4_5.snappy.parquet` with 2 timestamp columns of the type **INT64 L:TIMESTAMP(MICROS,true)** - `PLAIN` (8 date values of `1001-01-01 01:02:03.123456`) and `PLAIN_DICTIONARY` (`1001-01-01 01:02:03.123456`..`1001-01-08 01:02:03.123456`).
      - `before_1582_timestamp_millis_v2_4_5.snappy.parquet` with 2 timestamp columns of the type **INT64 L:TIMESTAMP(MILLIS,true)** - `PLAIN` (8 date values of `1001-01-01 01:02:03.123`) and `PLAIN_DICTIONARY` (`1001-01-01 01:02:03.123`..`1001-01-08 01:02:03.123`).
      - `before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet` with 2 timestamp columns of the type **INT96** - `PLAIN` (8 date values of `1001-01-01 01:02:03.123456`) and `PLAIN` (`1001-01-01 01:02:03.123456`..`1001-01-08 01:02:03.123456`).
      - `before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet` with 2 timestamp columns of the type **INT96** - `PLAIN_DICTIONARY` (8 date values of `1001-01-01 01:02:03.123456`) and `PLAIN_DICTIONARY` (`1001-01-01 01:02:03.123456`..`1001-01-08 01:02:03.123456`).
    - Files saved by Spark 2.4.6-rc3 (https://github.com/apache/spark/commit/570848da7c48ba0cb827ada997e51677ff672a39) with the meta info `org.apache.spark.version = 2.4.6`:
      - `before_1582_date_v2_4_6.snappy.parquet` replaces `before_1582_date_v2_4.snappy.parquet`. And it is similar to `before_1582_date_v2_4_5.snappy.parquet` except Spark version in parquet meta info.
      - `before_1582_timestamp_micros_v2_4_6.snappy.parquet` replaces `before_1582_timestamp_micros_v2_4.snappy.parquet`. And it is similar to `before_1582_timestamp_micros_v2_4_5.snappy.parquet` except meta info.
      - `before_1582_timestamp_millis_v2_4_6.snappy.parquet` replaces `before_1582_timestamp_millis_v2_4.snappy.parquet`. And it is similar to `before_1582_timestamp_millis_v2_4_5.snappy.parquet` except meta info.
      - `before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet` is similar to `before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet` except meta info.
      - `before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet` replaces `before_1582_timestamp_int96_v2_4.snappy.parquet`. And it is similar to `before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet` except meta info.
2. Add new test "generate test files for checking compatibility with Spark 2.4" to `ParquetIOSuite` (marked as ignored). The parquet files above were generated by this test.
3. Modified "SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps" in `ParquetIOSuite` to use new parquet files.

### Why are the changes needed?
To improve test coverage.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `ParquetIOSuite`.

Closes #28630 from MaxGekk/parquet-files-update.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 7e4f5bbd8a40011f6f99f023b05f8c15a4a5453d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../before_1582_date_v2_4.snappy.parquet      | Bin 398 -> 0 bytes
 .../before_1582_date_v2_4_5.snappy.parquet    | Bin 0 -> 660 bytes
 .../before_1582_date_v2_4_6.snappy.parquet    | Bin 0 -> 694 bytes
 ...timestamp_int96_dict_v2_4_5.snappy.parquet | Bin 0 -> 737 bytes
 ...timestamp_int96_dict_v2_4_6.snappy.parquet | Bin 0 -> 771 bytes
 ...imestamp_int96_plain_v2_4_5.snappy.parquet | Bin 0 -> 693 bytes
 ...imestamp_int96_plain_v2_4_6.snappy.parquet | Bin 0 -> 727 bytes
 ...e_1582_timestamp_int96_v2_4.snappy.parquet | Bin 494 -> 0 bytes
 ..._1582_timestamp_micros_v2_4.snappy.parquet | Bin 436 -> 0 bytes
 ...582_timestamp_micros_v2_4_5.snappy.parquet | Bin 0 -> 767 bytes
 ...582_timestamp_micros_v2_4_6.snappy.parquet | Bin 0 -> 801 bytes
 ..._1582_timestamp_millis_v2_4.snappy.parquet | Bin 436 -> 0 bytes
 ...582_timestamp_millis_v2_4_5.snappy.parquet | Bin 0 -> 761 bytes
 ...582_timestamp_millis_v2_4_6.snappy.parquet | Bin 0 -> 795 bytes
 .../datasources/parquet/ParquetIOSuite.scala  | 188 ++++++++++++------
 15 files changed, 130 insertions(+), 58 deletions(-)
 delete mode 100644 sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet
 delete mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet
 delete mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet
 delete mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet
 create mode 100644 sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet

diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4.snappy.parquet
deleted file mode 100644
index 7d5cc12eefe04f18f8a2ba99db22326dc0399a80..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 398
zcmaKp!Arw16vneSMv)y&X&{FZ+}NOo*0F9aUc9NN;YkpYbWL^Gc3aX;Wc2F);J@lc
z-`oUGULNo9eeeCgklE#JKoB`0jz=&w9bz%3{r>U!wXZfB{Fx9!$EO(TQEO)*n0i-T
zvy%Gd(afJ!AqhqV_psAWWu<yuWMj%G>st)0<Bo;sULn}Vc4_Orb@303G6)3sfCpxb
zD07BO{Uqc{0!;jHg<0nU0<3FQ3B7z3g{maLrE|CC)!Rz(n43!1^NM>sTc|uWJig!Z
zQWiUD7ZZeo@i3Kjp35W$9Tt_xe^2sw+cXaia&8X5vqHNYrOjelx`7CM5x9e-UgW7O
tr%{?svnU#!C!xy1a5PH&&<`Uu_UBPDi6(xck~A1n#^8C^Ere$96W`?#bIbq$

diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..edd61c9b9fec8239636030cbda2789e223fc6320
GIT binary patch
literal 660
zcmb7CJxjw-6uk+tWayB=7ZS)&9<(ULkXTHc(#1{0p}303OY^G1^eagfDP8@nTO9;v
z2S@*vf5Am>nrd|sgq$1ByYHObb4hD^-ysr_NpX*eL;{eCmyhos1c*XbO)L^A5ZTbA
z>@=?`to>jBAR8i9MF0h5d!&`gU?IEzC0jBSRA#v>bG|}cBW@74h&#j-kqL;XH+PE3
zfW@arN&AF%8O}+>vZzTx*iOsEoU9BY7WR;-if-s;MK(~lB^M+;zehw>QKy1B6Do4^
zk371GnI$3uX2@PR=#!9pK|$B@&&<u91@%S1Cb}(yW!qUY&<(V~0ntW%#R>30Tkprm
z78@|kV$EY&$uP1o-*CiJEAF3LU7oUlrA&Je#x%VcaN3|r8jsp3HEAd0QIOEa!I<_~
zH)~~Uz{cVt3OTagD2kXLVIPIuq+g!2F`i7!Z|!_*{}UfhK-*o6yU~qHXFN{Aey`+M
zj%_)mHGdRFL5bD9pi}R7UUk!VdB=6D)xdUb*W+7u+w*H)&Gxw;IP1D;0N?{}0zeD@
F$4_X9%2WUV

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_date_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..01f4887f5e994c77fc587378e7f3a19868745681
GIT binary patch
literal 694
zcmb7CJ&W5w5FH8G2+oCytdKy3SvX^ZvPgC;TgFYg<8WcF4I!+obs{2vNpiwr+~(8i
z?uvudDXzSqcg@|b<-`erKtOMor<wOOZ_xPpg-<l1liG-=L<7)jA8zg+2oQrjaq-Yd
zmFSM8$+zWcLi0BU0J@{%X((WzY(&OV1_#-<KV(OSg2q)Tlg=LzpAerBUl3mrJA@Pv
z)gGSJ@+HsSzRD+W#Ej#dL_Jk4ElwuIX{{ucWy+H|GSjdf+imC$3cFfWx63}FnufVk
z%qyj#5C4(Jb}_R?RKN`K!a=_)doPe)<!AP0KNRy;!6LdHgURh28Q2cm;DBgj-r)pb
zyOy=cUNgSrn1$_Qv3DZNlf~SswAnKjfYth|BVVNqH?UfZf@O>MY$ggG^MYH~%7zw~
zOF;)TFS7Ndpe~&zB8_u8c(I{#K9eQ6C@dTrQ7Dkj*J;Y56zeDyF8%3A2b=BI{iWUS
z_5Z{#w_u$g#(g@=>W6%FpZOm1>&<ALq;Z}1!+6@ChGC~21!5WmolfipUJ#0|HwmL&
W*z+P0#eU0n9RPgb1ot?`KlTyfmeeNz

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..c7e8d3926f63ab372d898a40667cb5dc33437ede
GIT binary patch
literal 737
zcmbVKPm9w)6n{x$37A9FoiQURmH|r_Vn{5TZAvfRME0<F712pE)o9XYlc|3wz54+?
z3Vsz&9{d7cJbLs4hzIed*>x>~f{-`K<oD+H=P}2(?}fx54snl&P7DCcgO_9RrU!8O
z({&&~Y_jL!VvsH|UB~(z-Q7_b{sjd9)75dcHE7u0Yvk!s!wc|eNAi*dS9%ugVG%5f
zb+u<r9dvW1&01%aTswCbAvIhawhEoj>kr?F&Zc*)x)jy3hxP1<ECh;??hV~B(rl*M
z&kJQPvIS6^xy&1wF6puZvum)nhv?L%pEdeL8<^AGw7a{1WM>{GHi+KLvv~mwzG~-M
zt%<GJTb+K?5S(J3KTpwb9clw}o&9C?*cYY1SpGJlU!UYYMm<?c<yXrWej$}eg%Zv?
zZOzrIQt}C}RkfNa?(umh^R(uZ`)j@sMFTYxLEyAeWreJjC`$abSmn7$a%{rW(&J~u
ze6rqbyk7*{IsOO3gAF)$PmSL!s{V6X)!A~<5B)Ik!~QT?WqH~c<2ap<=W#r`l|*tL
kMWay~L_rkGgJ2dXhw)*M$RrK-ndbuF13xr?W4zoy0e4TQ`2YX_

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_dict_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..939e2b8088eb0f15d870f7881ac421e26187036a
GIT binary patch
literal 771
zcmbVKL5tHs6n;swC14IwcgBgNSOS(V#39-4W}DKBH<3LoUPW}$Of_rLW|P!aO7H#v
zkAi>2lL!BR7mptO0piJ*rt4Z36oh$`nfcy)-}~O=<o+X{XoL~_n5aYpFkE;&6z@6!
zH@@5f0>mV{E>0S06WwOUpXm0ELjNx)0O+=gv#CJM_E90H$2Bj&vn|Q%2Hflzu!})3
zD8{XhF=klJg|^jHM#=5V$^xW@YyC!{)q4BsCsD2O1Eb7E@$zXoc_A}_Vx)anWm=p}
zs%E*9ry`jFH7(n6d%8{9)}hwcttN-4)TCb(`c3KS<88OwyMK7MTy(4vweDyA0qA{K
zF14B>)?)8f`dLA6jDD`(qCZ;H#B;{x#d9ulfwuVCA%7){GMUfV&OSfj0kFL*8Rc2Z
zu^DFzS@B|i&NEqwSX6?2P?k>hCYR30DXU^JsT|jtCNhmnXY_dK%tTg0b<YskBvwf#
z%SvQ99?ceMDxwsV@V0cF3u0%qT&>*S1nW-!2g8#Uu!oz%?`1`I<FxB@-{XF_A1#tJ
x?uua;PlwYm9PCGdoCd*Q5PN|ag!0gvgwauW<V7-y{XNUI0dPPU-{Ay*|6ee6vCIGf

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..88a94ac482052a66482aad13059be4897b10b850
GIT binary patch
literal 693
zcmbVKO>5gg5FH8E_@IOOv0DjbY8Gm2P=!=Cie-E$<PdTQCf8ER%38IelC4PVw8i*e
z<R0kn=^?k|m}8GQ<`gpiift$@6n2)Gowx7JTXyjN#3Mj{Dde3^bj+%xW*HiM@AC6H
zz_Y7Yuv<}R{iO>4@qvoFu544>YjCW=Qs-==<0J5o_@^2?uWRrEgJ4jMmvyag;52Jm
z6LX@P{X?zHdGYmAIs76rPAcf&wQ5wNWGI*ElRV|g2&isarroSrq-yTfplU8LM5Q|Y
zt<p=SS?ljQ-L3xN+qBRf4upo9*UCm@slHITQ|M0xn|-vs1*bO^s^fvt9v7b-o^v$f
zEIc{oY3iW4(}|EyG5+RcLh^`9Zd@o+CV%9D^;jv3$xt$z#feCxlJ!1J*@$QJ*8B~y
zu>#5@6Q$%?jz^<On({D35Z)FxTNAV1bT+de2^KH+9}FL7U>vOmf0GrBGf|Yuc+~J5
z&vm>;E1V>0)ZpD9io0<Tbl!%(h<(4)iCo|H1F`20gK$6CcS8|IUfZ-S09<gZ01WWM
FJ^<Y|rBVO@

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_plain_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..68bfa33aac13f7d8af14dca353195c8f421bdea2
GIT binary patch
literal 727
zcmbV~&x_MQ6vtmexAc%hT%B<SQY-_OU5G=n-OV<o7r}#gSQf7$GHE6nOxkQRU6s=R
z!n@$#<H4IAJ^Ckj65qDnA4O3R^1_>W-+bTu%;fap(~to9WRXLkxR|v`$72q>9jf<T
zfZZ?mV0%Sj?=MvV$R64FcCDKhKOH!6VBP2Zx=)0{Kk9EeaJ%ck9Snj&G46JqF~eyt
zZ8_$|=HtgsQ>ps(i)Qjlmx{EI;J(dTX+AOQbf+p*J_YJ>kNf?ON80=d+8w^u5SzO6
zyG=h^{myusbbIv=Ztfu+4unAcuhw;wb@x@59~S*+VR4MuH{|qNi@Lyl#%A?PsVapa
za>XRib=~B%Dcd=ahcW`T|C^#YD<tBd&9#yB?2Rn7QK>SDeXth7ysNYriN@6Pi4nfY
za$Tg27(H8vsVXl@7pH`UjnL$!Zj34`+?vjdLL~)?@N4ahOJgxwESLV3!Rl=P!{PZ7
z*ptoR56Zf?A?<}S3}o2bOXhiz_S7&=vtbs;gM%c}SriQhX%GcbtdD|8oE*o;L86m1
R+~>XrfDi2A1)Sn9{R=(^uO$Eg

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_int96_v2_4.snappy.parquet
deleted file mode 100644
index 13254bd93a5e6c54a13beb4314757d76b98117e5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 494
zcmaixPfNov7{=dbrS!0ah$#(3ECt5~wX}|P>+s@D#KZ6^BI%lHvE6Eu`iHS^;dk@m
z(GOxeCb)}U65i+c=6#+dv-9f^>)6Kh3L~robUk>Tu=hiN{f{HSfcNl$*CK3SliJ<i
zbhomg{|5yClOnZtH0WSNyo;`=Sp9e><`2AN7y&kpk&UDP3lU0a&!pHOBSSYyt72p!
z>r=D7w4QmkYrI?i6N7l_Dtc=Nn+>3yihxjiu&nQWRxuUG-(>T}Q{k(SPYsuTy?pUY
zE?LSXvu{?ee5<%Sb%m^(xpX}@%XyIscdF2Wl`WD&$=f!Wmt06zR!UklMZuCnwWzb)
zbAP6~)75%?y9WDm>v>Yv&NHt?zFat=9|nHt^phqpQio0AG@E2`JUmSzo<-4cm<CZ0
b#e5XZ<76C<gM=q(I3Sd&H?K5+nfmE(+~RaD

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4.snappy.parquet
deleted file mode 100644
index 7d2b46e9bea416774f0b5ebc664d9383530ddf24..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 436
zcmZ{hPfNov7{=emF?tYBrZkX432baoL+i9#hZk?+X?POGkaSJ8*zQl0Dl+yd1P@+4
z`5F9hn#sm4dI=BD`#$+SA*0Ki5Cf*z^D#1_iEZlI@b&R|J9+)yD+fA%xBzf;hRk_j
z?IIWv|LRaJMD=*5rVp|d7y)jykc+fx9fN}iCA4i*+#u&o8_34T$U)AV<-A*M`(h8>
ztAFr>n#y1R0Gl`hLf!7Xy60jc44ePf;A&ZL(`Q*r&8zt{FQpc#(83+Bn0{GEHe^ay
z^;9#TWw|U;#fD}y6Xj;4m9e*LIxnTtqFfkjRu_dx3bSIe(r3F{*>JU9PuAdGZ<!}$
z<vqzt<@1>r@-X0`*GcNUNIfx#(`=B%aql#VWEMreUK&I}6wC8q8Ylg@A0#qK!!DuJ
MJaKFRjLd)i023L1p8x;=

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..62e6048354dc1026f86af85a5c8b29eb492dfaf8
GIT binary patch
literal 767
zcmbV~zl+pB6vyA>;*#QufHxt*6zjk(w-7_(a=FcV2sUzv#c`Ez9x~a?YB1T&C0Rwo
zf`urEomg2|SY2gf=Z{LTwiB%HudvXUY}N%=8zIb_`M#IWy!TD^uN=5krYhaIP9-V>
zC?>prb8_S8#ZL;AY~do)7FA4LS!9+w4YvNGLWPlVRSBqwc~0mPQh{IrI(+)@6tDOL
zw4x&7>RtnCx`K1%mzXH)<-zGA4BxwJFELyq=T9)a`gZvvQTjjL-u#MmXn0gIcK6ov
zlxKHu<-<E7;W#LjHzi#TqhYbRF+ZG1V;+t1q^X8!SRKWrvazkSWMf`OB~8`d5$yx%
zD7(wfm%D$MVOVIOOeMhas{qjXOy+E88T$O%8vabu9!r?4=qp~-^c58h6R}WZwNGoB
zih|yqWVdaea-7w!$F}q1*v9#fr=qa4$$dK!1rK?__1C0i#e-C^0n3YQIxLvQMv;ia
zoDB|3Hs(o{s%*kUQ&>ca$P1pND2=Ca%!3$<@NHqS1(*%W+06P4sIC1E!qE)$Yfa-9
zlB|7CWO+0hw_V$HY`47?Ortn#^S&RB`Xk@(T?{-i^1NOzbUerN#kMo_gB^dz2}BUO
RUBfZ~@PV`V8TRql{{_RV;WPjM

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_micros_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..d7fdaa3e67212c1991ba75219827b490c127f43e
GIT binary patch
literal 801
zcmbW0zl+pB6vyA>az~0QQrv_DQ>=koZXphvTQ|2^55Yzbu{f?0&O;`fSq&z;xg@LH
zg#`;y5IeE5u&}zy#?Bv=U~MN@-(O*&FWIaMt~NrLH}icjpLy>avVZNsp)ytJ#!V_w
z89*`N{hPB}M=yR-C}kTLnXXaA)RlQ=vD0AdFDjH630IYXikRnwJ|QIt=Agr;r{{RZ
zAD|@_5!d%>P~8<=D858QSuYOGA7S|3T6u}#3b}ZO;nlaRABocY@%Hvtq(jZ4lCitD
zo+UiJe<vH<6EVj@sk|xaau5#k&5hY%HHmmQ!jq;NreQY~lgdU{UQ>)&9hEdydq=bn
zq^ayKI$!MmVTNI&fijf<$1ejw^E0W~P_k#&R`6$%_E^GXNni4!qA#gnn23c6t9@G0
zR4{B^AE$R2PdLuXs<GJvk!Im|q_1wX%ghD6wN#NEM+_Z0o`{^K<A*F3IS+Ww_1C1Z
z@{>eZeJjh;$so6EYZ!_s$gKWhVU2iPrb;6*Q5WW6EV7)(2}+|$6mdVoB79re)*Nj0
zi|N$<4XDij55mzD^c!{Km*TWhPug&p(`HU%%b$c%(BM5U81{yq*V*=6F?8KdCuqBE
e*Aw0L!1H&!owhIhz-bw_34jlD@k8z7&+!XBE9?dU

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4.snappy.parquet
deleted file mode 100644
index e9825455c20158d1d0cbb8add20ca5833d5be9cc..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 436
zcmZ{hK}*9h7>2*bF?tXWLP`TUl)%OYHMGuk>+s@DJc-vKlCG(iwOiMu${70_9{hd&
zkD1BFE_w+s@AtiV-jK(;rw{|C*z++mqK$3p+G-B#;Cr__DF=FgxBzf*gUoqm9U~YK
z|NdN6qMpB~*_$i{Mu1x#<RYy)m*5~m3GLbxx5%(<1KIl+Imr36oG+_u-=3f+^$(s<
zQyB~ZU>`?7sM}xGuUu5Zu=#Hdu2uy%eO8Usyk36rQfiS3E!^pb>2)R9gehG&GtGRK
z<+4Z>o0!o;l>3oZ#y+g+yp&3dvNG19DGHGkX2oQs&yKgU$!5EK-h%sZU|yBA_bzLd
zFBe|O!+?igFKO~3^~5+%vvC&3gX<)cSriQhX%GcbEQi4?PDb%4NMw?ReL|^u;?x3o
HH2?JjJC=Zu

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_5.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..a7cef9e60f13425560f516f4ffb32f0bb4b1831a
GIT binary patch
literal 761
zcmbV~F^kkd6vrnqyG(IKP$y&{#WHfs9mJ5hTz0cuBZ!D79JV4d+01IVWH*<YJvr9Z
z!sZHXv=9sNOITiEA^0I|#n#sDd`Wg)aJ3P_KQsUL@_X}M=HU9hfEdIim+ugr7zklU
z9-<Tf+xa;`mF<EUq)o76<5gt6ve3?7TvTCnP*X!DU|wqEX~jV+&X2A4RfRt|8!ixT
z-e@?h((+69SqtMUcVB=$m#3g#-@O9ue|p`*gE!ybR$#eA5uNVe+L8q?4<E|WW07-c
zkr>;$Z6xVPZC`9Qts>*;1a_859qM+mLkzly+Xij&h;EtIN6k9Zx_Ezm==JI!W1udK
z&LBF1&Z`$f-OpOXMr~?0Wo!7UZvE6CSkX7eP}4VDL><7Q8f%?huuL%Q-D&yA;{}Ia
zy}E5r9%UZ%e>4-yE2mGqTqvG!#qAH;oT-ySupyJGoQ)K7**FzhBH8f%oK1LMg{m<@
zV40}WTu8<90<6g_%XpkY5_~OOwgR)^e6etU1M1QK2jRg2*|(PmznYhwV^PX<I_U&n
z;Cn%5C!VEQ(&2+B84t!$)VmgkVjPCOUgC#-7>PZ96vh3h@5dref?eu52*HCc!EZQ#
GOaBW#&F0kr

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet b/sql/core/src/test/resources/test-data/before_1582_timestamp_millis_v2_4_6.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..4c213f4540a735c041a85fb12bc90810a3f83eb1
GIT binary patch
literal 795
zcmbW0L5tHs6vtm;y5z8jAWldihZ5Pc3vo!cHf_pY1X)BB7Ox^QX{H)BX|tJBS!(s*
zWe<9B56d3JFX3?y9t1yxSMloA+xjMLEm*w>;h&lJUVd-hKg_|c2LaKDL9X2+D$xLR
zM;yYj|LyXUfDC&m8rdMaW9iGxYGuOqU!ou}DypG?ftWW6c_BHl<owutFFX9fS&JfJ
z|Mr}-Y%RXDUN&_7`u*3aufz%J*Ymfi-B0ftdhhi6nFNakim3M9olQ}&^6;@3JmopV
z5s9{?T3V70s;#T@O{2(II>Mc4*pBVBbcbkmN88Ztc^y$r!~Ce27fMUtTOE3}`iBj+
zht_FC1)N;I09v1wnhlaYZ(GApRP(2TNl9O`p{B1z!FCV}HP*biVj5t3mNhP)P*yOU
zmDXd^XS@{Ycw{wp=}j7fwX;?sjxvf49Zh&e%kguX^NJ;`V%7&`>Q=`Ecl)lW%E_Q|
zJ$IP$ED>)1;nW?mT&ARvm{=56Y0gE(@&ZGnNtUrV!y^2ed+rkK_NTL%_Zv`~{~v@$
zGqCP14t^sqn~O=C0S$Z_G`HhPnk7xvi;`h)7)9;fIOM}HY_}6X^uvgE{6Q3VqplzG
TI0<%a&jG*(I{2dw@LK!=c(3uN

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index f075d04165697..79c32976f02ec 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
+import java.nio.file.{Files, Paths, StandardCopyOption}
 import java.sql.{Date, Timestamp}
 import java.time._
 import java.util.Locale
@@ -45,7 +46,7 @@ import org.apache.spark.{SPARK_VERSION_SHORT, SparkException, SparkUpgradeExcept
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection}
 import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, UnsafeRow}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.catalyst.util.{DateTimeTestUtils, DateTimeUtils}
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
@@ -875,81 +876,152 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
     }
   }
 
+  // It generates input files for the test below:
+  // "SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps"
+  ignore("SPARK-31806: generate test files for checking compatibility with Spark 2.4") {
+    val resourceDir = "sql/core/src/test/resources/test-data"
+    val version = "2_4_5"
+    val N = 8
+    def save(
+        in: Seq[(String, String)],
+        t: String,
+        dstFile: String,
+        options: Map[String, String] = Map.empty): Unit = {
+      withTempDir { dir =>
+        in.toDF("dict", "plain")
+          .select($"dict".cast(t), $"plain".cast(t))
+          .repartition(1)
+          .write
+          .mode("overwrite")
+          .options(options)
+          .parquet(dir.getCanonicalPath)
+        Files.copy(
+          dir.listFiles().filter(_.getName.endsWith(".snappy.parquet")).head.toPath,
+          Paths.get(resourceDir, dstFile),
+          StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    DateTimeTestUtils.withDefaultTimeZone(DateTimeTestUtils.LA) {
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> DateTimeTestUtils.LA.getId) {
+        save(
+          (1 to N).map(i => ("1001-01-01", s"1001-01-0$i")),
+          "date",
+          s"before_1582_date_v$version.snappy.parquet")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MILLIS") {
+          save(
+            (1 to N).map(i => ("1001-01-01 01:02:03.123", s"1001-01-0$i 01:02:03.123")),
+            "timestamp",
+            s"before_1582_timestamp_millis_v$version.snappy.parquet")
+        }
+        val usTs = (1 to N).map(i => ("1001-01-01 01:02:03.123456", s"1001-01-0$i 01:02:03.123456"))
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "TIMESTAMP_MICROS") {
+          save(usTs, "timestamp", s"before_1582_timestamp_micros_v$version.snappy.parquet")
+        }
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> "INT96") {
+          // Comparing to other logical types, Parquet-MR chooses dictionary encoding for the
+          // INT96 logical type because it consumes less memory for small column cardinality.
+          // Huge parquet files doesn't make sense to place to the resource folder. That's why
+          // we explicitly set `parquet.enable.dictionary` and generate two files w/ and w/o
+          // dictionary encoding.
+          save(
+            usTs,
+            "timestamp",
+            s"before_1582_timestamp_int96_plain_v$version.snappy.parquet",
+            Map("parquet.enable.dictionary" -> "false"))
+          save(
+            usTs,
+            "timestamp",
+            s"before_1582_timestamp_int96_dict_v$version.snappy.parquet",
+            Map("parquet.enable.dictionary" -> "true"))
+        }
+      }
+    }
+  }
+
   test("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") {
+    val N = 8
     // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
-    def checkReadMixedFiles(fileName: String, dt: String, dataStr: String): Unit = {
+    def checkReadMixedFiles[T](
+        fileName: String,
+        catalystType: String,
+        rowFunc: Int => (String, String),
+        toJavaType: String => T,
+        checkDefaultLegacyRead: String => Unit,
+        tsOutputType: String = "TIMESTAMP_MICROS"): Unit = {
       withTempPaths(2) { paths =>
         paths.foreach(_.delete())
         val path2_4 = getResourceParquetFilePath("test-data/" + fileName)
         val path3_0 = paths(0).getCanonicalPath
         val path3_0_rebase = paths(1).getCanonicalPath
-        if (dt == "date") {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
-
+        val df = Seq.tabulate(N)(rowFunc).toDF("dict", "plain")
+          .select($"dict".cast(catalystType), $"plain".cast(catalystType))
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> tsOutputType) {
+          checkDefaultLegacyRead(path2_4)
           // By default we should fail to write ancient datetime values.
-          var e = intercept[SparkException](df.write.parquet(path3_0))
+          val e = intercept[SparkException](df.write.parquet(path3_0))
           assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-          // By default we should fail to read ancient datetime values.
-          e = intercept[SparkException](spark.read.parquet(path2_4).collect())
-          assert(e.getCause.isInstanceOf[SparkUpgradeException])
-
           withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
             df.write.mode("overwrite").parquet(path3_0)
           }
           withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
             df.write.parquet(path3_0_rebase)
           }
-
-          // For Parquet files written by Spark 3.0, we know the writer info and don't need the
-          // config to guide the rebase behavior.
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
-            checkAnswer(
-              spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
-              1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
-          }
-        } else {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
-          withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> dt) {
-            // By default we should fail to write ancient datetime values.
-            var e = intercept[SparkException](df.write.parquet(path3_0))
-            assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-            // By default we should fail to read ancient datetime values.
-            e = intercept[SparkException](spark.read.parquet(path2_4).collect())
-            assert(e.getCause.isInstanceOf[SparkUpgradeException])
-
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
-              df.write.mode("overwrite").parquet(path3_0)
-            }
-            withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
-              df.write.parquet(path3_0_rebase)
-            }
-          }
-          // For Parquet files written by Spark 3.0, we know the writer info and don't need the
-          // config to guide the rebase behavior.
-          withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
-            checkAnswer(
-              spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
-              1.to(3).map(_ => Row(java.sql.Timestamp.valueOf(dataStr))))
-          }
+        }
+        // For Parquet files written by Spark 3.0, we know the writer info and don't need the
+        // config to guide the rebase behavior.
+        withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_READ.key -> LEGACY.toString) {
+          checkAnswer(
+            spark.read.format("parquet").load(path2_4, path3_0, path3_0_rebase),
+            (0 until N).flatMap { i =>
+              val (dictS, plainS) = rowFunc(i)
+              Seq.tabulate(3) { _ =>
+                Row(toJavaType(dictS), toJavaType(plainS))
+              }
+            })
         }
       }
     }
-
-    withAllParquetReaders {
-      checkReadMixedFiles("before_1582_date_v2_4.snappy.parquet", "date", "1001-01-01")
-      checkReadMixedFiles(
-        "before_1582_timestamp_micros_v2_4.snappy.parquet",
-        "TIMESTAMP_MICROS",
-        "1001-01-01 01:02:03.123456")
-      checkReadMixedFiles(
-        "before_1582_timestamp_millis_v2_4.snappy.parquet",
-        "TIMESTAMP_MILLIS",
-        "1001-01-01 01:02:03.123")
-
-      // INT96 is a legacy timestamp format and we always rebase the seconds for it.
-      checkAnswer(readResourceParquetFile(
-        "test-data/before_1582_timestamp_int96_v2_4.snappy.parquet"),
-        Row(java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456")))
+    def failInRead(path: String): Unit = {
+      val e = intercept[SparkException](spark.read.parquet(path).collect())
+      assert(e.getCause.isInstanceOf[SparkUpgradeException])
+    }
+    def successInRead(path: String): Unit = spark.read.parquet(path).collect()
+    Seq(
+      // By default we should fail to read ancient datetime values when parquet files don't
+      // contain Spark version.
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _).foreach { case (version, checkDefaultRead) =>
+      withAllParquetReaders {
+        checkReadMixedFiles(
+          s"before_1582_date_v$version.snappy.parquet",
+          "date",
+          (i: Int) => ("1001-01-01", s"1001-01-0${i + 1}"),
+          java.sql.Date.valueOf,
+          checkDefaultRead)
+        checkReadMixedFiles(
+          s"before_1582_timestamp_micros_v$version.snappy.parquet",
+          "timestamp",
+          (i: Int) => ("1001-01-01 01:02:03.123456", s"1001-01-0${i + 1} 01:02:03.123456"),
+          java.sql.Timestamp.valueOf,
+          checkDefaultRead)
+        checkReadMixedFiles(
+          s"before_1582_timestamp_millis_v$version.snappy.parquet",
+          "timestamp",
+          (i: Int) => ("1001-01-01 01:02:03.123", s"1001-01-0${i + 1} 01:02:03.123"),
+          java.sql.Timestamp.valueOf,
+          checkDefaultRead,
+          tsOutputType = "TIMESTAMP_MILLIS")
+        // INT96 is a legacy timestamp format and we always rebase the seconds for it.
+        Seq("plain", "dict").foreach { enc =>
+          checkAnswer(readResourceParquetFile(
+            s"test-data/before_1582_timestamp_int96_${enc}_v$version.snappy.parquet"),
+            Seq.tabulate(N) { i =>
+              Row(
+                java.sql.Timestamp.valueOf("1001-01-01 01:02:03.123456"),
+                java.sql.Timestamp.valueOf(s"1001-01-0${i + 1} 01:02:03.123456"))
+            })
+        }
+      }
     }
   }
 

From cefdd09601d0a11cbfeec800dc00aa5c99b313e9 Mon Sep 17 00:00:00 2001
From: Xingcan Cui <xccui@apache.org>
Date: Tue, 26 May 2020 14:27:42 +0900
Subject: [PATCH 0736/1280] [SPARK-31792][SS][DOCS] Introduce the structured
 streaming UI in the Web UI doc

### What changes were proposed in this pull request?
This PR adds the structured streaming UI introduction to the Web UI doc.

![image](https://user-images.githubusercontent.com/1452518/82642209-92b99380-9bdb-11ea-9a0d-cbb26040b0ef.png)

### Why are the changes needed?
The structured streaming web UI introduced before was missing from the Web UI documentation.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
N.A.

Closes #28609 from xccui/ss-ui-doc.

Authored-by: Xingcan Cui <xccui@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8ba2b47737f83bd27ebef75dea9e73b3649c2ec1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../img/webui-structured-streaming-detail.png | Bin 0 -> 179135 bytes
 docs/web-ui.md                                |  28 ++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 docs/img/webui-structured-streaming-detail.png

diff --git a/docs/img/webui-structured-streaming-detail.png b/docs/img/webui-structured-streaming-detail.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4850523c5c2fe165eb6c1290ccc0dfe0c107b71
GIT binary patch
literal 179135
zcmeFZXE<Ez8$K!pNd&1xPY5E2L^oP&(U~Mh^cFR`FosbRb|QL2FM|+9pJ<~Ki8eZe
z!C;6o+8BK>${BlS|E}}pd^=yxx&G_Al-ISES?{}^_bK;t-z)3|P>ucy>lG?0DtZm|
zXS!5WG%zYETJ}pcl)vbYY1mPIob%LG`-`fykA0akxM;7WqeMkj7DIRRnwm1c?5=L)
zNkzrPd-iosL-+m$71e33#xo^7AM>^2K{s}_j3vM6Q%llY9OTIIT_AVnzWX`o@n3Jw
zU7o1AcrNi>yY9vXYQ;|&#=qI_C~qZbmpr@XNp~Sz<cW9<k5#ucd;IKcgKl@a(T#x|
z<qCP-o7N7}?yKc$g1)7XhdH4F%I@^3QMg4BvFwMUf}(;Ih3dQoMA}PZg<V~dk&##J
z+7R5ogRW6-^xq>(?ILA<e~tmMVBmkhQ!TM5@&9kI6Pb1Me*<6Dfy@6J$nz{+{BQ6s
z<2v~Ke}kv}mjLJf8)Vu4f97NUE?sjTp8vi48cNueny;D_6E!zWf|+gey$f+8)yn0Y
z#e#md$a7BOL*V>2ua9e?755>TT`N7wR~a2SLv4JVH50i~NPi~2CL0WV;9BtzE<P5o
z&Qrl=cY%s_>emM`7t+2Hf-5d%XmXL>s7ZV{qod{W!B)!npQZfKG=a@RyDB54Fg9=3
z!_$Mv-y)MOdZf@BPh?K^@*^!^No}%e<x`I{l?&uerCFMzB^xCK8G9rVb+BPyMIoNo
zFhtkTv9QBqKyf82Hm%$^Eh&Nsi=R{L&2Jj>k`u7!&$B5-@-Z@+%~c7|xz*#bnr?}_
zHIs(15gA7gtf$#FJmvpR;hjNi1~m*x?CBb}`CI_<t>(inZktPfdX@4SM8s57w`JU0
ziU+P9*YacxGCMfzN17J9JeGK!B4-qrGI(|cYg`-u;<f;5jHFG0uSy>mEWvNFD(Rku
zg1PH}fC1;_|BNITeVT<{PxL?Nww{ju5shL4J9f^=N~B~+TaOYtCz!~;HO%^IF+ab7
z=+f*$3<e@}jA~wpzZlfKK$%O*mE$4#li$iLo2Hq4C@%2Cbbj~GT)MD9R}h&Kyu!zY
z&5S`k^)yNA>olKQY}+3D?035#m)as&e)QA%kI7nE?8+<+fA+)FUzK*)qWLmvD@sC+
zi9`{dJT_rGFX1~IEG&r9=DHj&$f(WMo9sYH+((hlAx68*G4H5)kQq(7_(3r6x$UMI
z4oT(!uSjyyLk;V>;XR#=E$Ex&-<u40KGpTit%HnUW$z2`v2dn+Sx{S)|6?iS(q<)i
z)OZgN+me$b8d$}RzO4fiRqz$<SmLGFP#Lq*@tWEXR9+6qBjzP5+09$ouDo<^qs;v{
zFP~7@Ip3Pk9;1`vb0-8yUcNhj{onukls|wTAfL>!31+2PUJ5O*Oer4-tO26Kdm=ev
z&jkaeUjQQ4*f<kV6LoUa4kO3yeTj1U_6YC2#gQt@{xJi#img1nal_u>lY^`#fvEVG
z{!s)SqYRd~S*2iL<GufOK08goRY0y6YtubugDtPGZb8igvWurCmjGYCiK_0=M~=+D
z9R-Kf(F`Zc>bC22*x3DY4h<+XW7{?jy=!KSVy2p$jt_N-Vpf75{m*BFDis6+g$3T{
z^~^(ndHyGdzB|a$3!+kkrbU!ZIldF!YoAtb+Q^N^De#5#2GFB6!s0wmbYCS(d6vE%
z@_EQe9JX0s`8z#8R-t6(M{L~euo+#olx%Ut@NC2RD6Y|;ZTlvmAo}jkrN4WoppHm)
zaiq<we!|~IX#(o0%<_m4<V&__ehC@-7{WKtsy*w;`spUKHDkiX1vb-&3boKQVkr7?
zUx^Q36SO8dzd4nv-Q+z~Ovpy1R|})hL;K+OijqLVs%rKNN*TK)qIrvKHR<ex;e&~e
z+{qYdaFzV%0TAO)?rVs|?*0f~6~CGBGf^lV7|5qNPEb=$6oD#u*FB8sWrFUogB>;9
zXwh+Q@p@Ts$sCgZe)oM|geLJ9c{l$!Yy|e8xg&)9)}xfKD8@N#FQX%xp!Knxk*)mJ
zsAu)mz^zISH(Hk$xqPHR_1Z%v>_L`!LWEM9=cGgZT0l0&j^YJkQXc+s?ug_j8dzfc
zWV9^*ZF9ICV+c@W)b{Y1sNu}=x2w$gTv1->dATiv(Ap%XyU=p(IDl6c-g6qfqil4f
zP?9D)*8s`vZX6d&>0minC|O)eYd<!UStx~%Bu1m!65`)!X9feYf*=n$LB<g4^3TRh
z?Tpp)A;?vh=NGJBLH7ePLx8f?Hj@?27;~=J6tZ0!J7k4fj(lmz@`Y-d<xho+utPmO
z8Ua;ae+5L245I1>@w(mIf*a^Fx7D9s=$b;mHGU$;yL4-kHqKjixFz}Qk|Db_w}FD|
zlNPY`Uv`e8<4L!`zU6#kIj&g`c&esED4Uf0xPHpxO9}ShP4lD3DrgiuBrf+*tZ7iv
z-0wEnIMxpn0M!QTuDhbFR)*~b1cbjsyt|qM^oMp|2R^2_762l$>O(r%@ihn%ESIZ8
z$Nc<+VKML`EOC2r_KnJV?(V_qwRs+UJn_e}n8)O{FGdOOrd#UNa$V9vDeke)?KCU5
zm=It<@wO3Wvf5R*!djt3XyUwkv}GotC55hhr>7y)U?mlp`f*{cC)F9x(l{EE1+tik
zJf;IkcvSQAHMVA<`=k;k&*eOBTTfzes)644FM2mP9<qYJsJ)U|SoZNm|F~@5RiWoc
zSpx4KHleT}eKqG?a1$eE?3@eeN!8E~a`gn}o96OU26&ny@I79GdF238&C1@T_|WPF
zJY=gO<ID7p@S|bR0fA1p_t6$^mic9m(A0fA!d`4(-x~L!`erHnh)3-^rcM6M0g9>Z
z&F=6rY<CtYXtm)qm6l|#`R?n;lT1n5*P>LI-}GQVA`CbLCpqQKOkaiv^jv*IFdHZ^
zOe=e?1fNkz_ZO^<iu)K-g@I9?*+X`A^!%jA;la+2Z$LWMOx+hZ|8G-d17DFVjas04
zca<~NQ*3#4Pdr0D3HCTQGH_qz6{mRh;5<X90BfbAm+rd%lUdX$#i1du)5Q~<_2hG$
z<v>L#4D;`KtVNAX;zFO__yqx*<%YW%H;9#QbULHsz08$8j*g*5brpO?)lQ-ZKvg_A
zner%hQ-x@;BAC2+XU!M0Rn>M2?`3>J{Yw0CAJsxcjbcacGHQ9=T;{pfzT)!!OUH=V
ziOZBnO$B!Zro+l<_CBau30}D!P8+_Zfi;kG^J50Dc*FV|7Bg<GCN~qg-p(^Fx^zbJ
z&%E@fm*F9G)}BQS9W!iUnKkwsz0v*$;<kZ}%qT=Y=_A|UL{QqGlY$2|EV$utf*ji|
zxFwxJ;7==Ot(3sqnRp>Fa*GTATbhE<h4|8#R~Sj<xKb_K33_~Bt@jirk25Z!y81W9
zt@<7|&FMAvW{JUBwo5r00Ee958xEZz3{G@yLo+C3AE73`4?_GzFDNVRS@>IMqm<)s
zl2mV#q-`JiNdai+U>{&h`$g&9?g0ZHE@?YU*bfwkk>yyfX(%E*Bf=z{At3>I(UW&#
zQEoTne`38@NRXNslx}$z`8s$@yEjqc#=XI?hE+z%zgLO=bC)m5X)^MGnZcS|Uk7_T
zX@c-4fz+@=3w%W|@B&fKX7^>$x<rO^r?S+iSjpc@ghDxS6LA-PEL@wtfxF18JzWo9
z)i)`AzvHx=k5M=DRqn{mQ}y(`CL~K?z5o1Wz(KW)fWQd9vNyDc4XjD8%hkx;U3Q2c
z8~^-)g>3_f%0!s|$qf+S<u3tLkj*JNT=H`oA<QdVV_4J#E(>IA@7q~l+I`yyxY{bo
zbHi$xxys7v@FG|bv$fC~C|*T<OSex;jR)EVRQq1(QY2t|_~caN;3f~79>q@jKYf2;
z+~LW+kx+ipm!<^GH7kT2en1i=8n=|-^7-2ZN^tZmXflg>*!*S412?L*En`QHP8UBN
zJJnF^9xRoTethw!N3Ne2fX=&#>3JV?-Uc5|T6oj8gYOrPKS)~NN)}m~XB_x||J~@S
z0HDrw<5Ko;{R(>bC;#{HS~D6Ds3e<mPG=-|8m)eoc4%c2U$P!3`1RmZCBv|lKx16x
z<{RZir)^QQT1u#0|E>^`*PsM1a#}pTXz}i@x_w+u_SGQ%3VV52MhFm8o;sxAhZvIC
zNRZ7eCHSt(c_%h8fTTe62mR2XV4&Yi{4vH~{n0eaNV0#$qlvKX)_G7T8{X?3=+M%$
zzUExo*)taFpCc*HL#m_yR^#g?HatxYYsl$M)qvk?;HKM+LKf^z5MB=mEKIwxB}^zy
z09;=WEnA7avRw|eD*Tg6&D^f2R^ktZ%=9$|l`GS(f6|`T*#qu9S=u=fCSKwa^!i)%
z>+rNjT>yX3jBw%>4QvST925+6c+Ye;lN}%z<#!Gg6i|1plo~lGb1!Jzi(-NZ!4A59
z!9>m<sYmV3Hzv}u(B{L?Qw9F{?&36&JW8~e$BZ=wdAp)~wvOYk2g&;(1-gx#qEm(y
zhDym<pwo$~RiR4o?<b2Wc<DU@B3bY6+KHsya1s$XOSM^%)whu1`w~-FBzE|xwnF3u
zP@KCwDx<FO7=u4cupk3!hq!aNOMB3s3Z1_AYN4UN)$cU0E9+uI6iZyW^c6i~g%*??
zozJf3@e_Zti&0_(c-1LfCRM<Xor+C08b}`-WYQ#tOBnwtSn{v(MPs!?T4dVK(@$O0
zJ)E-3Z@dbiHh1pqk(Ie%6T)v>M+l{Mq$B|X1WPH<YbK*@r<rGMVu$qqItS^=J;kMO
zkZPN^Lb-9}E;`?mIKTj&xrA_SJg_?T02BQdC$dJXhUhePQPC$lch1@O+rnc#oP&XU
zyZ4rcDKXn6b6H_2UZKRWOkf&3B}`mW<fyVnhOOBT6p#Ukx;h0HNt?xzd4-SKW;;=T
z19<3Selo7tb|`V|#*AsxT#2C7J;d4N%QFB9xIqOB(I{Q40_B#5d-amP!v0ND*<0Z6
z%Y55$0{`Cf%*L|yz;dlz;CyNAkdh-jpgivFX)y3aLblzepvYV~h<~L3db;Cu0Qh_v
zAaJ_li()waJN|IgYd`~OGYcl4`~=rMWZc#++2yXuP?OJO94c`1m@3Kk@&z$X?>T4I
zxE-3{d5$jw!sJRb9I(%&Z+J*Oh_DF_da?LHHJNwq==&>6KxKS6z3!2@0IT|VgrqV-
zc>p!&Q~4+&;j&D^^>>b$A3e@ioOF_w67gf<JN9#jx09&+TRt8tFWoEsHGhnQ-I8Ju
z4)EFU>4-(s?w_&&`k&=^V0i%?zJAxhTRP*p%l?f|AD5t#LMs`t@xV3Y5tAH3nH8*=
zHc=}70=;Ev*Gf)V4*{~|#0QJ}+{^DUBDM7S9>&{*+}v6at1ts~RRz^QWb9>nB7k`S
z#qCu0sU6-1f3f$H$()SIAgcOd2ekqdM(bPXMj|Cb+3TBL8m3OD0feItkXnKfjc>&C
z2K}Ga(F1CcZWghz$bgFlE&+`ccT`lG8|krwvp1Uvc-WnEjOLCG8Z?>uQ}r@@QJ{ro
z1Cj4<-W2ur6O-THvC+vx113dE=1n6{gN1S-cOcB*Er^Gc?`pV^_nm{X3I&mf_4P*B
z3BT>S-*!{ZW^cSAmD&L6+cP!qi3hbUma_e~oMe>_d}W(LhP4JXAW~>GSqIjC3!Xqb
z#~oK?NQQHl<8F95>z=dQP_2r^hO-t^&@49a8DF!8_Dv+&G*KjX(gJ1aqPW^HSe*qL
zxFKYrxo-cLot^4Ugl!e|f%xjwj0Q`lNo=SC8SMpoXs;U_RU!cw3Nrep-BQEvqWbNH
zLZlq&HzYuxNwPfnVT;6(tp}HqBOSmisl(r^SBk?ro@P2F!O!7+yZx%D_Y^ROq?c$1
zxX5c!+)0~ZeeDmV@}tP+c`&d<T!_cb;&ViAl|^ai!A&(?wa}$hVP@ie%(%}ZizXsg
zKg5Kq>nB0NsL?ELnS_T$w?CphW>{Pa)lW}{WJ}|@aS+r8&KH;SYgTe|CQ^~{tb9I*
zmFZx9zMw=<mdTKmpyX9?UvdOR^N?(E-1t(A%>nR*$ojCY>+i^`he)vkcvjruy$2EO
zPWAIT7p#|gzO9=*XJg5I|Haq6sYM1;<S{s`=H|$DZ>%4{CE(jzA-;YeBh4Bs6_*n!
zcP=v9f7f+mNkb;6pb9e5MgrQE&WX80g&*|R&{mvWhZrNpcJrejnWl<5u@h9%t_vi_
zsF!AeYI;9kjnP5<o+`1Ezf6#{AqS551vSWE6{^gnpgm?7$s&(J7umSV-WqX{h{NjE
zN#!`?)SGFMMPizR#rpW_Yl-01@)?s=r)R#fHpV{=Onpy!Nlz$OxNhA9n=4<-5j8~v
zJO6yejQ09(j*WiX<=~+DJ@NY&nyhbtMFm*EUv^?SsVVs#{OO!4ul%JK_ZwwJ7%h@v
zhZiwo@|J1Z0$yEG!nU{-|8JQ}@STD+=Pl~O2(2P=EqcaK=#EttZLdFSlt|ee+-+Oe
zDIkTWvzWhjw0$vtXJXafrbUdqEKQE7z=_fB7>%>dvl$TBY@*?-I3L?sD)A>DV;9f8
z3|JOdV;llZW=?(CjE4OVxB{SNCD||NF~c@c4Eya<{_n5TBZUtE&o2W~hB?7kG^NJ8
zr-Ad7+%~NrhA)*X?50)2k}Mw3XHZHkqjR^+NPI<NxV*134Wmb-9Fe74O9n6l3T!|v
z8qybHU{gg(2DvH0i&yiMDx!C@Up13)c6X?^zt&OmNxqUa0Vjl4Y=@PX@Efs*{B##x
zLxw4EPzt!Ne~i`Q-&OuGi|NP$DRcJ3*q*LnlfZS^DH7vAlT*}^T<W(VCHO#`L<%Z3
zPblT1n*8cziTSVIUAg+K4KL``s(ob7CUJ}S#&sg#5ou9X4Bn3}z$@%q6}`x)&s#n2
z(b<t)5$GiDNU=EmXfRA#zG?Ix{#wRj>Z+r;CO=ejKwvltfIH0sO;n-9U(9Fp4U8ox
zd8Xue`fI=w(h^37&r1a6cOOJbH2mJJ$GLpWd7-d(cj+5?J^(!%D45*gx-zi`Gm=Sz
zUgeLf8U;VR0cK_IeJS$qyxuA2$0+ezSKo^vVr-Q#ZoXt2vbYvtcL+;?Z@kQzh|XQW
zP)krx-$$WWeDiBB;nw~epV+xzpO@<(l_mj~04%a;g5~ZK(u>%R-z`xaa^S|oLZPYW
z#kOCv$jNDU5@PMt@ZTdZLNn^ohwMsl^{-iRyEbM_2#e2iz{WdrtixrRW}I>r*fmN7
znF!!4^%M6N=mZ%z9(0Sby#HYB@s^jzqUD5ZSWr}9WleUBc%b#l$R!cPti+##0pLiK
zjG@w#N7NJNJj5TRro;w{8g&CqBTv#xUwZ5I0ku)4E;^hObqS#~_lr;xgq~FIk-aRx
z(}~Ex&mVMpiLBJ=q&byKGgpyHjvT|gRvSu_;!TMHTJPl`W|$q$#u6u~D83U_3NCZG
zu{~k0`FYeE`HCL0Zz%<cSo$>}`zIj3TL~Jt{2<5e=$tfe$DO>eX*K#O@wLn;v`gFu
zk^ZsRd@u4ZUnhOeWZ{1=prR7tQsQqouN%ySh(Qf$Wt<n3-4MTzXPUh3kI1LQC!#>+
zoYMGzCwGNp-DGI)Ell6KD7>XDQOyOQ&H(nhHVnOoQ|hq{4;dc1jNNbloh`zs#Lxa>
zRg{$Usy3-=G|pEqi7}e*0#oWoo1+N49Z&oy#H^hbi@^QJy{-E054imlbq)OUT-BpE
zJ|A&8hziq+)7H$O=QuQPxj?SLB&jw`Y_siTYR6TFlbCAu_5P^s`f<3>w0Yc!tW&vZ
z+Lwx78BV0bWT6vFh5w-Q2lNzVF3WkKAVOSyAUl1_3=Jw?_!A?ka-R)crqxSQg6D8`
zC0$I{EZmmhXn0}&H|w8C#NRrb19b>6L~VYOf)(8S{-CVb32?syQR;sD=bWt8OMuxK
zM`u*yM1wXkvTO6En$2CNZ+8-7tDkorXl<AK*n=RK^x3I0vd$1^*(W01Y5l#Df3AL4
z&ix<oM@9Aj3&{V!wf%o(HgVSVfl4&A&%1hje>H4vIb+)At1ulln_<weHkz<z=CpF3
z<;bgyP4^J==AUz2wgCXYvPWX@&=4nCN};-yC^s#ExZp?7(CKUBoUC}A`o(W?K^}8y
z<tXePOT!E1)GJgzMx(a6*i&z(T-OMqSwC57r9i03WO4h+LO{UO5!=~gI)o#X;18~S
zwP>HDBn(6y)KI%;#VhD_a2KNT06P`lG|!k3i<$@*GQitHxta+?O6f(x%RP=dKNp-T
ziyrk+K+)-VSTR-Kx+nAnqH}-g35z(XkAkCm1VYRpJ$&SQfs9GjCj{CczfsLN@7|9Z
zytRq63`+1xByrTn*Gf<IqYG3e$ex*~AYoF8Z4wiiBeKW7clSd-n%Z>3^`3Z3u81C(
z7xgi6Hr|cfjaq~B`#cu2iWa-iOq31^@|Lt&KK!7s@G%B;u<8Y!sgGR;rUdMxYmI%U
zl_{m;v}@2AI@&!I|DnpWnUCwDmmp^IW;kbcl7HTn^(a*J+KK2yL*B>9dRK()=PigG
zQ6a&v$|sG*Y&CUKi_B)TqPp!yn7J+^as8+<Nbl&Nq|7`v7SrGPGY{GVX@}>=-~H>v
zujeAq7qUxUfb*&5hb;7oFH{MxI$$1im8L&gF`4;no!J4De491aGf7a??8)+wq4@9l
z-PZ>*`Ue-D+m=-;!QkblXMGZH9Ljhi3*BDaz~UdKU=eiL%cao@kapy7f8UBoV8EKk
zYM`HtC+^i(a6cJZwgtzFbGmfudTF-fQa<M<88+8l0VFINbBc=yE`fhI2W7kx37t1}
z>2`K-xI2?<k9|X)6WfFo?4PJ`(vmcdv6v^x7{j=TS+T!9Pp2l(N~-TFzu;P1MTz@=
zZFE_p)WJ2rd9@|S<Ut)H^*M@{k{6kKEjuyMq&`rv^tQgso`I3s*fzVsdcc9wxk@K&
zd)X-_<UYZRq2^bU`wH0*Tyz-{bQdRBS49JM=w(L2rK!VdVBu5N7hyv?oM1=i;f3FR
z9X5H^?<)N$`HH+z>~ffx=n1s?{X!Bm_zE#a)4l(q?XOS1lm(n|p9sl{<G)r`!57z(
zbZ8iQLzLk10!|Xy%cTdTM0fNxcTQj60LV++m8C=~rxXT}(><k$qQ3AU4_UIg>oEm$
z6b%%<QBV^7R@hH@Kp+X&GJr8}zN^*W)Be7_E1}1-1~Sy|oE&vQg@j{z^F~Wq^hD(_
zhKd3#r(!b-Q)fEY^C+<UI)4Vubn|UCnSA^*wlA+ItAG7t&)9W&N&+<U?9!Da71hUI
zUknt?yp6z%h4Swre*pJzujn~>&#%jb0LRZI-!;pZ#rPIkPq`yr^S`zfK5(9()VEE?
z6Y+1lBc`oeba?38$jFMYERb7&o*((DVQN+!bWsQpUcO*U$vH)W$<B|-#0!)Kc^BeD
zK}z0tMC23@znbYA?L}vyGYm76HfwJ~_rHx`NA~aS>k=+<M|TA3!1j3$W>``n@L9hy
zUXs3~d6~d?*G+IZAaSQo&%=w#L)h;bWl0E!@!F%K3!A_q*m$i)6!oP?rDLSu96N62
zb-Y+(egdP?+A1){r|ekDX1Dz28y|0l)I%U##_PhhvaI*rDBR3#qrmHp6E$;?-!Z84
zlnl~lwruu8;SEuJ9}Y#5&-$k%E8DM=R9)Y%O2X>|?|>irO#VJtYrXmOitZ@x6^F|R
zlMhOEXt@q}p@z42ZS__z-TFyz-D@1SmUpff6N8%PK)*hoPc(G~QcK@QLm6CD$0b5w
ziD)L+s&x3befh-`7+ZeBH-=MP?j^lW3aCN-v*6W%5;tYF+#e8322dpPLBiC#zvjqw
z+R-Vs+g^`OwIrZ#0L4PyTv(hmdTPn|oS4(gj(0p03MaNzOA3EYyw;^O=GKfa{Ia)c
zsCy&};MLwB%2`~Snp>yzezVQ^e-xX1Um?@fllAoRXUUsvNlfYTV>T>uJT^n!1xo23
zc&;B_YdOe(nL9z)kqHsK7W%dKNAY*0Jyai!$~gr#jD1;Pu`i*K-+R_r+y2p@_}7F+
zm<V|`m9DSudWf3Zf|a|=#1>nT6T~d~H!S6${^EKPd1{6(r~<TM=w8eX3l?@EV|*W5
z=T%)@>6XF!ItPgf;&qPiNQ?4`;qI5#%Mc&Pz#e8`xNsR?JQrD)4~0Ef*AcJgn$2S{
zh<(4kK&_&**Wn3ir77YgaxvdCV)BY%vBe#A1WI~pPL!L{6H|l&A`ijBugaqxj}<x7
zaYH;6?{8;dS;PVgjHDjAG^V{PJrF&ZMlJ5WG|^3<b=40kr-##Mv7Z(rWn4|zw@l|?
zUPhO;ZUss@Cxwtl(M*+9ES_vbomT+6?i{k!{7un7xY&A|-GJ-3gOwL7QLubJplf%x
zswL;Pk&bEH{)9*#%vBJ_t-D@Lr~~C~HpAC1<zS{uyOg)<EGCW{9-(`E6>pKT0;HS*
zU=AsrVvFPLg!pEICzyxo7s~u_EU~w~8P2pF9Y5my?*B-Oc}-zE^3uuxlDeOF7PrL3
zTE0^0ut}(k^)k(@B(QB+_6_4_U#8UK0$-=1c`0y_j6GM`>Z&8qTk1Z(ea-8V?%IS4
zDk&)&`Ff@`bjucAWokuu5AvRRsZOjMoz^XBzQ+&SO&L<jnUljz%0O0ROw6XN^8zmG
z5_kXNDgnz^>0>!hgw=rsOD<`YaOz9moWj?S;SSISsX?J%`e)WXKfXGW8O1QMg*AI0
z_B@$8>3!};qm876W-7mO?@6(%rExhOix;AH3DWD|suWw~9NyOO7a5p~?mx&(z#`y_
zZ_z2)Wv_yM3z(m#_G|L5;ZkSVfu^6^7Au}@5z~c}7=rKsGdd?(AI_TL!+L@l)}!KK
zM?eK-qfk*vZ|n&#-4ec~aL)pOlPpJ)n(Qx}i)cye=3y1e?_Pn`^)v>F9{rV&?%J_#
zT`I}j@m#wTOhNA4Fg70IhePtBY-z++XD*`6gSwbek%rLuee!LihDN^bx~F@T)P#kN
zuE}&^)y6$WHe0DkMjmt9s93x6?Qyo0^e8e<WnaZ-<6ESGW4XUtj4Ur4jLo@SYu_Zh
z?%&L`X0KrU_?R$52mU_g_<8+!;Lzh>KZ4QrzK=!7jX2+?7V<2sY?85)XX~~Us|!9+
zi?{4!OT<h!;iY%x?nZY$<Z0U|pd1uWO^z%@$m1C-HrPx&p7+}-tjx<Oa=|bwhl%4P
zPou_Sr$^5?XnoDZo6*SiNZ!SSYA*s3czS)FfBzJDn5>mO*MB@xSx*yR>Q~;#Cx*PK
zCi@$Km``0*U@ERP-#+=vY;_sv4)isfBl6}QqgcSkJI96c$ZLN3v<^p&e<{J|FNGVj
zegUyId_qPt74Eq*->()MUaj>%Xd0J%K2RWn5p^>A>{A~PbyZW07n$-cgf%v6V-X;@
zg>%<mj^mC1J}&=Rhz(!q=3i5?jWa)T6AVpeEKcp!`(EXx1ApmHtu6x=9iRJiz!s~D
zsvduG-_w<F7ED@nCbJz0YeDo*-L(`3tQ}~b-MJju8jKNL^R1NszyUjTA6qUYR|au4
zIAP{*r{ll5#fSRgRuHJ(?yuh~`UCAW^($8FCr+$68aJDXn*v>`^7zH*FIOg_?bO~h
zn76^r9Xq#jNB!)XO?Ee1iWRJvR-u3pju>O~1qM^|h>=%_%JL0&Y-L6W9d$ID=9GfZ
z)VzRp7+LJ<-4s$E^};GmU&FaOE^&n2E9+l7aME6Vjfyvw-XjN9F8ku7LK4dc^X!C^
zH{?&%DPXPt>Z|E`&f#W@i%`i^S5YVBl_mfUtgf~rebBgZBtrECvHx$v^33w(l%iG7
z=L7A%X0e_Xz;Q*B2vt*A7f{92<rP&d>+8h>*K$$5Upa%@x<!O~?ryf0D)wP6x@7we
zQugX>P~M|0cBd7)Ew|Fkj*|Ul2WyM-Q*sv`nMLKbxhPzi9(N89=_IqWzL_j}2CeL`
zhG6aPN<Npho+$Um+dkRHPG<x?H<KWtvS=>;T6O4_lEy4<&v`=g@pc*BF%}~pBQBVQ
zGWjrOHqq^9bKEG7RepUk_eqKdl6n?bJ3Aw*RcW?rTpiTk=_MSWlHvIy(5-v{vnO}^
z`14Hfy!Ss~HSP4tMHr`wh3=R4pN0t+rb%l_WMrQV;nR?afon&?l-l>>!I|!_vRArF
znPZblI9D7exT`q^Cc-z->@NVXkP8|R7&|oqxio)CBI17m%WY`K8~ck~a@bkzO5Xc)
zm~}Dx*^c7h(EYp*7D^fu!B$;RlqPu>xsZpKzx=F<?jh;)y7|yIg`G@QZp%SR3$!^j
zAh&YOlYT(Va9ZHtl;d<iZ|y$eZ%0=oLk(&DvUBQ;J}LpsQ@nyrN`{vK`G()^-1EKU
zr06;0V(n;c<l0nk)Ns6Ug=?<u&fLjpaCG!yzr_epnPeEW!OlP=X)r_PN$q(zKb5;Y
z_3z5@9MN%DLe|qm{i`2x^~mDaZujTt07QZC7yfG;uUK5ryH#L7^X??Gq)rs3;GmGJ
zE4{b8eza?FMnTNB(i46={Vnz*ZH2=*ejQ|_#Eh*hp+eW!-STd}-=>R;NAUGB&mK+W
z`YwcHyNbVQZ(#J-p5!lNLD^pR0qO2RwZLm@$r@Fjl4y_BHsf1&)sNh{GoO9mdgD~c
z4cjU{@GWnQp+5J8lqwi-T4w2F!e8tsQeBPOD&{=iR{}X7%6sqVk@3Hk_C|L})FnBC
zK@zo?z0gwLL#0U9Ogv_f#ddbEISW)aXXl1vUH%3mmc;_|K<0|pSV0o|MCn&Qb`#Q6
z^X+vU6MbM>ED&JD=lLxzj!~zHi?n}XnbM0E!KyWqsrzl3FZH3N8qL;-Fbzt%zwZYs
zPrk!4JR+ZX>4%Dj8mCU@qzhR&rO(+U{%Ue-3SPtlTS?YS<g1(lLLm%&9mi?5G4KWU
zw9>Ni6Ka=t6#<)g!{;W&Icy<y8x#gmtEZu|J|gcpx!1s8TQPLP8Yz#XF;)y#M|vxi
zG{+JSn4!`L&=!p?v_qhqzuIONl3ntBA?>PTs_(=!a*X&4rfTMWBFmh1qwYu5r_t7(
zqp=^`Le0Jsoxcp5o<5<(q<10KjKM%2k97}~Ji(W#SK-D((eF%LEOxR9$1t;LQV(<S
ze4Vsr^W6^?;;=py@&dX#@ja0)me}oB7Ld#^gx(aeNDV$(O{(Emf^Y9W!`pP8Ow|8E
zpc9rSKXIYA+omrC9i&-@e=R;zS@o7#xB(I8&oeLY5eX!Sj1s(#W+0=?3s>`^TVqCM
z>&8epNM7?3vW7*(w~#5kNB_}znaP;Ig6GBbgNL?(kS)ynL?8QA!>&XAm;Aq#g-54m
z{HNZ9n=r0HLs83mVU49Q(>(OOGFO$NIZMj4){QyCB&eJl2XBq=dMg@#gb{0{g9Zvx
zZopm!)FzgRGOV}E_d3*cgcIG+M(NX)^me^B>W^nW9E|3-yVqqNywTHhhQ-eunwvYE
zSOIDu85vtXSsNDTNZPE!u-aZ1gj<rSK?V!ss#fs{C4P-&9sYzcguFPS)+N|hZNt<|
zOypv)6C9rwfJlv%&wv=H*v94PcvF+Mbc?r-4PT6wcvca>$zM8pZ-Q68pAPSF-)e@S
z_D<i9l)g_~0uwfBDQjw{Nk>yh>R7I9W>kfjZ!Bs3Qz3TRKiD@F^Fb(`M>pA0!;BRR
z?_ADk?0GT?n5BVj`Rc`}UY!V#=BY~ii4<gwotgxKy2RZAU&?QrCmA*Z^L%In3`ogR
zOT(JuO82`fn_;8Bp1eO9j<sD3Ak_kPYca@x49F8-&gg2MwoGd*_V=5{s`x!rr24iz
ziGS7cD}G=j83jR=qu(-J2IR_t$R~;4KCS<1eWD}pQuHe?unlvaf`t%@l<Qk6%(&fD
zN>%i@fL)WM!RCs;v3wUC&$1zg21-MJnVCll7O#Ob#l3&ycglQ+=(k9xk420o;JXbf
z8F7^gCn9;vh=s|}gYzkMk1M69=9vky0cOEV$9v^YN8j&tS3y&n0Qc;1kWsfVuanB4
zvY2GxYFS^iw2<y2K-YsFW&QGlie~pwhx2&-`PZQ{25cs3lhM@Zub(9X)G9xI+Kw2n
z6xoKZ6?dnn*ge)nIiR@9fJ4t{kJ7Ozndv7F?jgCnO|TA@U;J>brC}zFJy12&^3113
zPK!+5E262SRB|}&{of0Bf<z!CRvdjN<YwzDt6#+AGc@YfpVo{8>c5t3Q#0Aub*4`Z
zH0^#i!*9IGZ)~Z^K6O*`qOWe8fUNiZa9~8d<oO@M*jB_yLL>ZFyhqer^C#eN;|8n!
zNZ3=WhlTm;yFMJA|G3G9MZ#5Q+%%-d4spPw!!w_u<+pVzzcbW#;O;B5J>DWF8fTc<
z5i}L~EY!zj+H*~@s_{}S$-@dD?d_6>WOz>UMD5)RL{y!ODkS8Vjjm45ADnw%zX-JZ
z@%XusPYsl?;-)b;2TAFlLEY;tx~xY{>Z%k2{FJ998CR<MUq!w3zEy9+e}x9diBXKq
z+ZD~^^5M^CLrHeiM}Gi?E{nI9I1K&lhb6l(>Q#lUq!8;`RYjm6!A}Dt@=R>^mb7G4
zr@VZp#j~4t`yOOSF1o?Qx<ro;9yOXpD%^By&c=kxbs(!tpn(Ik=E{EZUkIwP_<^P5
z&FgxlOBrWgZ_#ZU>)^DQLW<k9qi}&6*oK!iRUS2Y>qbjqX15e?@~*n?SVypf-0e*J
zGY}ul@WSaCl)Byv+`AQjMy7rjV&ewKpe+y<10q@{0-O`g?nob;n8o)m7JSMXA{H<b
z7tY|&Nrp5ikgq4bG_P3d=oeHiF{Le?JCGZ$-9KVet8m+Ee(?$*1>*_6bz2~53CC|M
znx3h^8$bl_6P!D<Q$F&q4fx`!XTVo1I8ZOwUGQ<QFy+Yw_jfC1tE4wyuUPdU(|^YA
z4{`Y+2!OV>^PEkq&bn)+JqI69Q_W;w#{q$9nVEhk{df0H%o`%7<oH(%gcw;+iSJ9x
z<{&>N;+OPv(CWaV?fXwyJC@Z~Fjd35!;?URG#a2@PiLXecrHH(;((DUKtrZU6{Tf}
zbz3vi)8dfP0&98mj=nP&)#FHqt?tKMc2MxXC*6dgzY1!yoamF<^eUxMvp;tER^Jp2
zvQVDBYjK}+5|ob`UxaL0g0~HhUWenl2SpZhNE2$7&(Im;)*Wgltr)P!s_C3c!wjpe
zNjZDzoPG84`Tl745z-Cxe4R(q2Q?8Tcy<SYHxosckX7;G-SdVkuk3q+Nwvjw=yOd9
zLiQk4q1u)ZP}mz)aUJ6wbNp6x-*~-Q&g{j*3sLy3{k;Uk1IbleDbkp$)oDv}HPOJj
zpP}jc@e`R${+wpT6Z@)94oR>*p<}USLN-wh7Cm(mXpHq$T5KTL#!pIU<OCowwM)DH
z{a;}d+x~<`M-gnz9*K=)pHpSn3yz3O(5ykCO{MD|zD!&sdXzJOACeMxR>xzU?mbVh
z65h>dGA988SQez7L(lpH(&~GGI`EH1ol{~`zN5@MXD%zT`ogd6g`(|~IkIMuFkV5M
z|EZ(B5?2UtP=vvxIXx<{j9?_#3Yk9)zV2f2H#TT9HBY_vSu<<jc2W1=6rx0*aSg7+
zKn=P2n#Cpx9NvXqj_)aSaxgEJ2BpxWHj4yP&_^|(pS)^6T2&dNY-8jiyiN{ED>TYH
zpWvDKsyS;A^)M<21nt@`1IW|D=v>(Trx+iWdMNN{XrfmcF7Tn)osfCPk1akB+<$Bf
z7Pa@-o)L1zA)Hu-s0W=u9!&?Uiw{(sELyuI`~cNYg_>FLlV6>!F*Q&IBg~iq2{3;g
z=(Kr)WXKOR?HJx`#c-x#wjk}iYMQ<&dlJt%I;}>hBDyRM`wYZJ(<8imJK#PZY6q=b
zdN!wJKXk3YAN}NEez_9z34FLp(B=<a=kN#Kf9qv53zR*zvl35=YaVvU50C#4ij~A4
z)Z_GaYUCHipv`vhrRm7yspZdrV~&6tOStjXp^Xot>Mt2dH7@Ap+H-e_jjQ2(GN)b|
zmEMrLN!RtVTkEx6tS#~QT2+;P3nLy_kNm>Xlx9$qmK$?GrUvf${R8&2X{4{JRo?fY
zqAy;hot|&N9AA`H5|@r!aSud5_EteArg>{FQ{M?`_EQB*$zkOUY$GHaj&=Xul7DPs
zaObQ8m=^mfV|VFi?9a01e~8`;A&nGJjCD0(gE<=oUCFU1K5V|I>O^<YYzdWpEN9VF
z-n4Ppl$_U`=*^V}dSc6Q7ke%i9`|b<Tb&s1MxQ+_(qfmfYx9`9#~#c4S}fsJboAt3
z3szeLn@RNotT^otna@kZ<`+tM79)<98JqyMMEem55m-jP#O|j@m6KmDg<N^todP5z
zn^s1sDH?^9e2uGP3YmrI@gh9O94!BMmlE3bRo4JbKY2@pcXBFOQ#%6UtG~XiQ-=K<
z8djC}+m^k5TZS}g&C#jsQCQ9w<8oK_DA&h~uZTrnGp4YN;f=*q@6Z@966_ID5m1+?
z5mQ94Wvhx>tfF+nejmeoEqYg%qDN{qD}iz02}T1nXQV%*POr0^OUd$1@gsa1mfBXf
zetNCa)p^v~skm}|E?Wui^LVx@$G~R%<l&)_mCN^HhDLLQwTa)*bnU3Wu)TfyvB{#v
zh1PP?Ghcf<`!G={r_bLe=(i@W{tZX8jTR<RlQPGq&7?{ts8i{aS?%akkQk0KNfnIs
zL?kl2<o4(?aX6w?9X^(N&n}>3qfxBfa+Etl(3puZiL70nQEhP=&{2pZIYGkLd&h7v
z27ztj8}7$TY2}#PUjAu>U1B)>)_u%}?dkf`H_yyEW}=6(yq&+rp}$(}b8cTc4Qx{b
zL<J4b(o%C|I$hm9O7QjfRIDSy6>+)M5l#JLWND(OvoAIk83Tbhwju!rGcuJdm76+m
z$FBO@A4NHqlk7Cp+`5`l_&r4`t8eeQsr^n(c3{OKiQwL}A%wkRsfw=njk04|+{(y|
zw6*-noil{`>GGP^fWX>JZ6U{27<Ean$TKWp$?K8(ae3UNf&See4fwzV&Hm@B-QVjG
zJD4rvT#Gh1!v#%|H>uo!YFm>^U=L9e^lQJT)&sK{vRV=CQ_;<1Nt)X#(tEawR&aSo
zNcy)2dXEly&HAURlj|v==0J11+s?WJJ!rL}fd?i()#M1<x@AE%IZ^ujB3|wmE=b>O
zTLUbYn>Xh5iMFsopyLw4g9f$+klGV-rSMLE0n;9|b&nMbGGP_eXQ2)q5$J*?t=?^m
zq*@ZC{Z(=M^LXtXP;unzoZ|Ijhv&*+(MbxX-Ma^GrtY!b(ge!b28A^}VM|*qkgp}>
z()gBl^|)dkWaF&xpO@C6tDH{i@W7(cM4Hfak~7CD0kG)wd_oqJifCKs@2&{27G_=W
z0c+}usmeOQ&4!!WgWg^D)r;fB&6;woQUZk2rsXIM*%LSEG|_fKkU2m`43&+Etmibx
z9{Os+OMM|)?Xs0{p0ewln}>^9Wd&oCcb&Rop4ACNd@BtGs>GN`XVOC-OEHy@q~#uZ
za*X<W^xwnalu2$M{7#o{{jf{bf<gAQwuP>d$+4;FlERYQqQYKRsMLP;t&at8E#G+|
z%ONK#)35enz3gd+C+F1(iGFb#%H9I}CFE0zx+<OunIIqSQi{f42rf;LMEsO6{pe5u
zE!UskUp9>#K~iUo-}q@i>B}vWcTz>IdPOdDVoxb#33Xjf64w?CR3hIL4=!Vu$n1xS
zijh1#=53vhD_bR=V5@0|w4XljA7Oh@edPou=q~IZ<d<`O7IoclP|N~;-H&2uR16ts
z?3T4*Yc;{X^Zj-E)q%_5qvM7Y8nzbTcyJ?G+k8Zz;09ULg0GQ-Yf+v==d2&Nmh{@&
zf|9i3)C6n3Z6{%DjQY=HM+}VJw4N2S%&l^;=YvgZ3%?g$vdBHThpxJ9)Cfwdy^MB<
z+ZT0Ki=)4Y?ORUY-7Pd_=SYkt@n=g&-LQC=6RpT0teZ4~eU(x_vhumkK!zo-fK$#T
zrU?I-cP`1e`I(UMqp14ZVkILQEQrdkg~`~NLs<*Q-HHop?ekU=od(}~JxaLoRdN9q
z9sxru>Kh+?1WYZQ)jxLR*=#~ys8U$vE?pHRH>Hb859ZhQoyZ<s!$}S}Uy0XSh!2T+
zqgios1s#F49K7^LC2hgCQKHd+p~?P>sEnjiqIGcf^swCNer-pb;{KgDsJz$x7ugpy
zc727dZPV4uqRT(ZCCkSR4rbPfh;1)8ldY8PrZupS)Lon&+-3w+0Va%zD$B{*LF>_$
z<zB_rtH}uYCmrsqr2)dw=8yhZS(nL;6gxIJVPn4awo$q5q;n{^7bZAs%;73KT-Jyx
z<N=18MYMnH$u-A3Yh1HCRg(X{dKn3Cl_+nKp=70dm!b$JTzNDGSlN(@6^((W(=B~x
zvOZhf_4K7eq2H9&=52Y`S%1E+E|U(ELng3hWr3t3#|6399@t7C!D2(sX!D|%e0D4w
z0VHR)xco-Ya#U}gldko|jt@;~el1_S<Qy)KN#2|93m596?$@V)fu}E$wtv+Tc|7{$
zk{nH>11Fjn9pe>o+h)G?OJwbxiF|EyuI^p-v{9tO$<bdmjWilLNIB__uD{cVJT({F
zXyQXwE3e7ng<lo+;u9MBzjo0i?W6#Ds*z|SZGXF(y{&3Uf=V#3xrUoL=n1=jf5Ef<
zj9n;b57qp7Pd4`W<QfdWIeVFfRI+TNW=8F02dLTWekJB%REs&%l{gB_#Xt6T(6&oe
zNeq<tYh?0eY-?8bzf<~Z1k(F!&|LXog6YioX}eCSVdr}-p!Lmf{K{A!ZE>Eg=r>C@
zd#W%syX})~0b&gnz#c!>4xoNuHtF|S>UXN}m)-2)s0;=7ky2{XhHGW?!AIY+ZGDHR
z0GE?~&rqzfuX8^==skhpW|y}5V+q3Sls5F8LmnlPkT>~DB&H>4D583<S#z@KroTX7
z$0I6sf2bVb;oOM6nxDZ!cEput@1gjJ#m#T&&E`q8ef>RbY1KC6za&%2S31v#0-0a7
z`PZN+PM#f}^5uJTXqN#Ef<nSb?PVD!%*N%?j)E|#kxp&Q7b^~%vf~a}W^4R!$kjRu
zgL$=)(gi(J+6|88yaw^#Yzaq-GL)hl3zu6jO>@Zw@J^%PXuH~lT<n|U;+oxOL9c?8
zqNNaCpD6@^@vv|q+hXRS;jz!W#|`zRx*7c>4`HsUFA+*pimvNkIt2Vkd1I*)d)4}F
z`O!^MX=gA(f%f1ECd{LYXVSY|*r8r+P2Moh&jmlAGN6$nTJDkx(H1VIwPmdj{m-NO
z!<^b==l)V+gL4%2I^9^TddN7bU?x15rQ<e&d(-0MCL4dNo)Hbg;_f_elslO&7E;s%
zyLvaC+B6P&URn9=>o4mal@`4kOAV1+DO-hm?OAzEZIg%vLI%Oxo~s%Y#{=n{g6c+2
zzr0MC7^+?Rw!IZ8iRg#z@z1o|gX;YBo$FY2_zdfKEYLMAgJs4cZ_l2-gbjT&N7U*B
z_zdHx$wXWfvbPP+Le<qY>c(!qgu9H`f4IpUT!#<ZyRy*OaX#uUabi?zoAX;SQX#I9
zluFQu$Dk6P$z1Hu)xv}zhfe}7Zzb(j3;a_M?*!!S8c*ZLc9357JU)qcC^;1Vm~6XL
z;eJPbhydhQ-NE}XR3V9#83k3s7y@V0cdyZts4lA$)s=(F0L&8{-4`n?HQ~*RQ2mgf
zpQ6VDb$;3xKYo{s_fSck+J20YgT@F+kp`!5wQ?9Myz2%BcgOCPAga&aQU55AX*DfE
zfJ;AmDl(i>r73}cUTh%_^4FrFlslpZBnL6yz8GpIEx`$!_T!fKp}yF2^-k@Dt$#ws
ztzaPeLFy3x+q_}N9jbS~ei=pZ*V>^@ppUd>#b2dPz+e3m?9>iRZ+vv4PHEusGWa3U
z+GRE8jLao1cL_j9Qmc9sz=n(0ru^X_U8iVvOYFR{&eCNi{%@CwwPVH!|9lQxFfagl
zeDJI$x^*YrNL|6S##=lU<G&v)v`I}zIV1GWzv?6(uj}9yF|!(T=4#hRqwL?_oSV80
zH3YwZLpf*W-5&`h;?e&RUjH9X+5eOFzBon+4lmzdXWN=sm5f6DW?U@!+Ye)cg%!z5
zAgn~h0}bOoXfe<4555L1DvSDC>9teP#@XocOJ|09c4Vpf-2wY5)3mRYyG&E;ae}tK
ze_?u!g=N5lQ;r+XmuFL_dwD26ve~n>AyCzBU8;xn-wP?<>#E^O{2Z^uPTD-T1L>6D
zjiaCa60BbzL?2PgD}SC;&XI+XoEC{Y><J%qw*Mm`Qaw$gP^-TwMg60%wz>wMux6_e
zy%A6NVR%M^#P|p1qiS1b0O_S&*4sD!`>dx@dqPX#^{KMO(`Klb&ZNeqE=7Qfw5igM
z-5iV)6?I0k$Z>bz(!S&pZ=!$57ocP+V%(4O`MLj*XoD*ugEPy5<!=<Xtw2m7u-Mo1
z-y0N!;EzV;Uvpn$qA-XVS13?J?aAQx$Vg!4B!|@F(a*G=)`LwS)oe@WcEEY2(<P|A
z^+X{HsZ9Fqmr_b!KgwN=_FM0K;WK%r4cw0M?4igga}gF6QX=;!GZUmjNd1_cm$szR
z)xf@a)|MShyZd91XLgc3bmfoYeAlO~Z5yg9?rC8}#1YsZv0@%&$a}J7SxF{L&uzWz
z^k}@WUBX7v91RN6yd;06@+tZhsG!g3UuK3M&t5-qcD(bLsLuis5_^_*TxOdoOqnTt
zcRq<Fuy^5suOEO9y$XECe;sVR-*6*(JuJY^{ja9U=`U?%50dBd-jm$gF5Ia<A!Li+
zhc8NM&-7e0cdo0vL6=`u@jg)?g;SQ>9&twPH(Q)mWGO8Yzvh3U<f2UzZT`9TFtvCT
zzu<4^+L8Wrz|*MiT0`QE_{$g6C~PQ+o==wt8q5+xXX`nN4SxEkn@yIBQ2Ka{m&_Tr
zid)JloWf!ne6_p4l?PE5vB)ksMLlaSjehvcGL0f_n>h^)_O+7q`k+*GuhxgZ{@Z^q
zi1^W6K@ms6hYY>XE2#;3^tWE6Jo?pbQH5%5#>~oKI$7^LW3sziU28B<S@c>or{i-T
zwVbM`36MD8;gInK4U(5@OXPrqGvIwbMYH)}Vl)~&bXRLV7TBWDCvicx@LqRPPg5M|
znkzEKS(CnNt+e~wVw-EdSxgr%%N<SY?y}l|!7Q}r!PfDN+;j<2XMyDNQmwu=Ctus>
z$V$5pcB$`x7QU7-vf%$ay~KT9f8xP|f1eN))r%{Btl)53&*{&x;|~r$d+Wg4EAZuw
zMoEk(YW2tQ#S=f2g_d?cHb9pXomc&2oo4gd$iC8DD?o8VI?Z=5D}Zb+_O8NHtDUgw
z*4uS~zh=;|c4)Fmf;In=$s>xw^Ll1ESz%g|aXL?^fx}-F{#6X2ZmsjEu7x+}Ek69}
z8%ul>^sDfvVI>OV(|x5m{<>zKsmX#}_fM1?NLgRRFIa1_0q3kz`rrLNy-avJAh2Pj
z!{=33y|McS_H~%@-UU#tfhR^!lg}QkOWA{>$iTt5w!u;Rz?X3yi@S@%o{NokS@%7S
z75NwF=Uc}H<y{r7dQPI;dYaRE>?gD0q#jtC%bK|m{LUoY7TT3bAW@0xYOnj!wA<l=
zSF+kJ7!Eo?;#vN?hX+MxfED;zOex!~CX%ZT6XFo*_fTs+Rr6QJ{Qen%o}$Qcq6r_O
zbkX#n+*k?NXm0~E-DK-_4P};w3RfvFhhXE4-%6REP>s4>d&mDoZ-nE0)gv@uBuk`o
z!Fm)7Ak{W9Q0mW@Mw#&%W-%*3%|iuNheRVLN2x1UD6ECJrK?_7ep>oT<^PdZP*G_*
zSOf!my@Gy{ey@s1RAUF;Y3@u=NEmN)azD!g94M;5u;pD}ikKlzfQ~IbEDv8#VHsw7
z^YFch^8^osq{Ox&DYWZMCnH7_xYWlqGz&D-;`}+2@{S2snB^TkS&M2V!T%QEg+BQf
zMX_AaEyF(+Kl$j2`7;E)_HfQ_3PnI4@}X?>K|$jcK--?gefUGCt^E(sGifeqRn(~W
z_N&b$`vHylx4V&-yWZRt8Cx6*XSqE*bZM{XT*LdaRJpz@S;yDXt4nY0o(PXW5s?V-
zw{DSmj1aHVX=8re@^jCtz#l*hTVs6*NEJ-2Lj(v5{qLR>$P{+w8_R;*Pc@xMI$gKC
zLzMlvz~ND?ni{=NwI~XXDAaeF0gAU=4}^bG4+;VHX7%7DSfpA~AoR7~$A&dTitzX)
zmB?%R96l~~`sgQj;-?rF+dgv$jGP><yyCe1f3f$TaZRo5o2Uh_0xF`Ljv~Ex2u-$9
zq$(gt3(|Yp^b$ZpMWsswDFFl}AWgcV1QC@UN`L?X0#ZT?Ed&T5FblW7`+evCVdlfk
zIlnV!))!@wA6e@u_j8x)x^Cu1JRQtmM|mycL_@D5i8!@8pM!`psN_8d50EjlzJl|P
zU-}J#KQ>oeuFgL?@zRqCTLy5<SsQ&O(f73Qs^uES^-o>P0gNP<mxlTu6oi8tokjO$
zKhJXbqF>*g?5=^Q^0Pg4nw9Vq;{eIb%uw@qLkm!1HUdh_83hjeFdn`nc|Jc}U5r2L
zGBZp6PK=ggW$pFix@dO^kO&V3K6|oPQGV5Q_7mMSondG2d%muq;h)ka94wn@kxqSg
z9N-bx6zVR95J?x)5#Hc`p~v6XHmmd36!8QY)ndUZ={pG>xkz~kfCrR&fAQSrif!MW
zCIA%(1Vdr|pAbD${r6bP2j|rN4iG?qo~Q|;XJ^44CL26<8AAXsFiKC<sfYD2rynsv
zY{Mp7r8c(<v!ssKCff`*8LxXD!2vxL&vT^^rQIu=GAkz0Oih9O`4=Cy$eq+O=p-+D
zA&MZt00J1xP<Hwc>1n^G`6hxSx$uN~DS;-G8YiMGCGJ1-?OAstlfIyWkDX4cbpB1s
zin%rF6d81XLh2XS1vrT+c^3SqZo6{^u!}-t3|fK|EhvjMK*6u-V7zg|SEVfb%;sgT
zrhjdw&iGvParCZ}&LkvUTdd4KV|eXRXa!%2chj|#X{NJD+1GwQPs}BC_(@yqD?$P0
zW%?b0*1B#blN15?D}-5T`v4(BZwVR+h(&xA_?H+F1K{GH`ckjQxO6#KYj*HGAYCFW
z&UJl9s2(|(bRLM0Q$x**f1jYo8ZZC9tvoc^#RvkN6$A#Cg!v-FyVKnz-dnHd1}-bl
zjND0A#QD|0=bsvEE2u5}$Y2L!YbvHx*;4?`oTrsPGG1Ht2QCBXem0LoFa6q{-}Zwc
zTa^x`2Y(wVb7k@Ts&k*~PEiR;zrBd(*7N~gLRNBY4gKh->>B=6zj=KVpC6O6L-1W9
z=tk?BRr0rZ{CrOqK<1Rfhi)%?6l}0%3GvoR-JHdDywuD7<5fq)JnjzQ9;%1yps?YL
zf3Gj=LUs;~GTpcKnUx#wudCN1X(fPtMyrBUbEyh<27lK|?8xnynd(0$WxfNwWp9iu
zp}}l_y~uL9j(%zE#W^WI=e3g}0J#o`eWl3RD)ObVRh1bdTGeX{<Hp95xwq0V9(b?&
zM}<_)XFVHHdjZ=<`kX6o=#9SiihpQD2aT=OD9`2A_h=QOsmlYDqEK}@ztrVzf}o=6
zn;{1IS=%y-AAp(;St|;|<nKruC(9irs)6efX8H`M=|NgYev2xwnRl(MP1gn;rw3d{
zSQLH3ty!I_P(~o#$kUsOyU)&Ny-8XG+?Qdz<Ng!6dkKKIwI8--*tb|DVtqQH6dzh6
zqzhOAgd8vaORj-tM7C%90I5)~y0Hm1++bO$P(Wcxt-`ohLYy_O2s8Ow^@f-5B{sF>
zoCij(-`*S3S_$!6d8ub6wBhgG<pI5w6J-F%25mY$s3oXp6830Wg2S~Fn$>vEk$#dl
zJ1PVU2Td6am!uBBJ_h$i%O7Gx?~5h4@gLqzYCC}+dg>KrkQm*&5OO}jpyP*UY3%ET
z$v@6-6Qf53av!JGE;|_C<$o3(o@Fs2V;R&pC)*$&#`np2z7<jiA}!bCt7C~@JT9Y?
zPPI7a5gRU9E0wmI@z2_+M65K<<-Wx8B;gNg0@*enoRqR<WA?yZdsF4tES6%+fewe)
zwEi^r>k&SY;DJE?_PP#33Kg-V4!^!LIAfU&az<15qy1Q4#R}7o{q^@e90SMP0JN2c
zx(-kiFszxw6e;<{<7+m~MJz3^RxZ>Ye+`Ids82pT5{R)u4Pm1t!~uz=(uY_;{=ftL
zk_Kz|=ALwG{-*j!$I@DLgp~?YBmRt4xSBZp?3^WYgV@tV&raX{*r@!TS9?Hs1F%BS
z-tC`r12^kVdDeRObPezj6AN-7WeGIcoaCpqJC|g*;X4=r*FMpT-1wpn*BV+Mt3Gu>
z-cd{-5u#2*??f}VZt4>`r&1STC7r*ae_2HNz9-{?tZ8*NLkHp;Lkm(BdMi|Q6W9Ef
z6e1nvrk!v&7juu25x7sVl=9N#W=Mz-nDJi58;wm<s?-n0PUgss1R_@_V<;E=mq^)X
z1LVK_GEk3j)2BhQrn7eI<m29_@JBvgV^E0bBuiE&+T0BUoq>Hw1{`N@oFcs5C>(&h
zG??`>%$^}j$?0Q^%|j0XxvjDo^utkC)EJTd_3P%7wScP|wPV14(t_ozb(&ZL^_D&=
zv^PIjl}4ggjVt9b;EqVxNVrsi&nT@BX;AgBy|U+dZNZ;%k6$?4X8qVPM8@fpcO(~}
zp@{^q_U``N1Z1Qxs3#PY=BwR=Ax<`bpBVMAXr9wAHFnBzRcGe7zhiLMd(BO8Ol};*
zka?_9{BzmdyOz}v$(J>j;q(WWKf&|*UuZ-?e=P*=E<j=k5YkB72wD-}2IxHqde?U}
zFxDM>kN;n-zNU53<zWE}j~Fn-K9mvB+ns_c$*NnPKazF9EN|Y`5?^eFp^WGS08uIz
ztP3pZ3?xfG;YnYOiJO^)yBK{=@&-b7bl7#zXfs1z&D;aOW2D*Z84BdvD)M@He-BD{
zkkFj6k^f)%z9B|we$lsV`Gfj%<2_90K5<frs5aF2PJ%&G;9G>|-Js?4Z;2IE_flfF
zBj}fAekL#ni(N{ypbsk$2cxh<4oyYEkZkOoqA+#pK!33GPEWJ+7MMW@*TiX^-$<HV
zkv5ay7hb3ad{ZiJNS3(vX{fj#Z<<N!QM_u6_AIzBpC6DW$K+Yh<S&I8w>~M>I3Kd?
zyBhW(@v+#iA$to5L5vE%S)trxedYJM=;JN9y`bVMY2-Qpz(&0U5yoF_JXAV-&MK|L
zH=H~Mgy$)?5i5JbffHGeD=GxkP)I2yXgJ3=z()CBvxvs;Qix7H?_I*vk{=%0#<-L~
zUe58u(&_q`9{RlxvY=>+R6E4auM7EG*DD16;oq5_^96H*zlve#^8=M`V3>?$jJ$rv
zxJA{C&-QtGx@8i9B8P(!hHW`irONaG?`AA@i7wdRTF|-YZHuJ2FN)AoV&ItSi>xK8
z6<g&1G1SL+Z?^`O2B$mfs)n>}pMv?pM*RrHW2nDMTC`4w)`0FAP{3*KI!u_~cyZ0t
z!5^v96~PS!WUiu50cg5M^Nt&Pk}S$9*ZiDdeAa$<u$p0s0k0_8QfS&hA@5&fMkW~c
zb?6y7<8%#6QK)%sevS>a>$?>C+TdXZxA7d=wm^`Pyc2*iFKxOtDF)^8z@3LGr-|b*
z;bxW52B^zcKoq0>i9yefz{Hl@M{Vnb-;EepG`UyL<$VgtQ;gngi3`jQ8@@kQ*7b>w
z<1CHIOK@6V+purKQv4kthFSUAm~JXkwl3Qtc)kFHHxpQJz|atBU$GzQr<G-$N$}I0
zmwOcyHQdX6xJF0OrqSXQcLQ(IR9kpV^d)N*cU<&z@+=)Od2QR)<Sm*HuaxOWShc!m
zR~ig~t*Z(qo+%?OUo%FzOKf&ozIgNuu@(sgYD#QBsf95C60Akie*AxpgFCNDa}mG*
z585SvYkxqKQ6t26IYY55f^U?QpqhB1utM;ZWu)3<WBst)pKH^NWjm=nH=)FAw_1IU
zx?d;g-LFU*Sl+#3^G(29j!39Kc4n)52ujSKU0mtbg=5cZ*|7ukHz6tOGe!zyGjUeu
z53pl(=9xF%=VKEZU1&pA#MjXv&lFg_Wu*^jH^{=dU@^<<wPFAMyWq>P!_N;&MR${+
zqpqGxo;F;m#!SZd6v0FH#BQl8M*PtTzOit*J>2>PZDNyOoe=sy#|x(T+&4%WrxPdJ
zW}_{=@WK7=uMF**uJf>7{tNWO?-a~q>MK*yxu^M`wygA6Ghc@x!=D|WfXaUB)~>28
z;*uVzpSEn)TmHZS8)l`V&&Dk+K<_j;y)N>R8TkBOcvF2n)C~dXr`Y*4O>Sv^+n&$h
z#TI=mgJfqd>8BGWIaB<!0^Z&K7B`sA*elBZp5@##X3>~NvdfN9ltF=N0Y{aJ;ru70
zY%Y|AI`koPd*rIaVtPo2<8X{_=A`)GVz#ei3Hn!NEjJ7}3UtCIBw&gC6crlSAi#pp
z{K#|ZnZqs&Aj&iuSi@mo1948Q^3D!MS&Jh2bg-Yy7@e%qg8`jETMy74E=!*UJRqlk
zzkeSfSgJ4^gv$lL_4|A$V5Qoigz<91%I<uTIkxjm;TyjS%y2<1b-rFqbJ89@Y94#S
z%<Dq=x~%K#d&)JZ+F$Wl=Jf`&<5QT^2S0yXr#1IT*wPNJWuZk!Rhz(GZJ<HCqkz!J
z_mfPX`Av`_^B_HIop;+rRl#$a^G|8k$cwDXBtb=-RnX#_tQ_+cPwQo&ibyjV`V6J9
zZ~fn7yUe)QHiK1!%wYANC>o(t0JYp#M93anT9Rz~;@F+HzU`TbfgEeNKm2-tpt%0i
z560LUOy=!|5x&u~05|6ziYq=DpY7`q|MAz7>}v`K>XxT&j3@lyj5Yu}Wt2EMJ62Xn
z7g4Eyx`W7|R%_3-tG$LmQo|hiVHw7r!stfUyi;fg8x{?W&va6hsh!O}6ZdKHt@pb>
zGiL)jh67j5-;=P{F7QldM)I{3Q9n)wf{TOX@;~R&gV;T}zD?72D%6_g9h+_!ci&up
z8X^gIv97+xa+oIzY7N{rT-uZOB+D*_HBkLR7D5gu8A~(Jl7}3ViIT>Z<mL4ANfFuS
zclR>_thorM8%9VTv=1Tfz?ol0vqp>Gjt{w}>!hdvtRl0O16hI=kl$dy<TrsYdtyK|
zucao%A%+Ca#}S7XfT--xE3{%w9bUT_*DYh-PXXAcW7cUM>Oo(h0Q-@84w4*mjz79F
zzT;K7{8mgx7JaR^L4^~@IT23J?7G}2nw@i3dVS-q$lR`DbMXUsjboYp(({t2u`=D%
zs5S!wTGQPO|D>!vG1+g?()R%9Bjj$c+JtrYs)eQXbInJJ$7?wQ&I)O?G(*F#T^qwZ
zfh;z-z&aaon3CwWSkRFDexq6WlB}L<iXMly<xKl)bA~fW$PMO(NPSAbJ7AB!<?n8o
zRevn8Tb_JhZJEl_F+7%|80e<j^SrMjbcI^BS~NB0x?cN$@@toHCjvSNqX7@~K%p9`
zg!Y%tPC_!2NlqG=|6ax8uTdmj`<9A<Vb%(DiMlG8m3Lc)qC=`0BS2_Km(zTAhfX_>
zFtNFKT}?fGvY7o`r>~A_!}DVS%>H9<=rQ6;pOPXC2pHa5E%sWD%GXQZN4*56FmO)1
z9M0T)7i-T7WdZCO33ngi%<xC)p}!tq^P(+13vj9=<lUi1U3OT07Rf4Gg&M%R8s{qd
z%iQttX0!qm>5TJTk5@$rha;UhcB548aZ=sSYw*B)X(%A*EhE=xgKZUDO&w2+9o{7&
z)5wf9sO-j=N!ZHNT6B4CH>;kP%G!4J*^_WUmB}fN@ArC#0i#d@`hyTwA)_$ch*?_k
z7|%)b)AFBy+A1p5B^S5W_T*<|u|aOpUCMu+$FN*dV-eN(iGxxw^T~5eSej$_rX7vJ
zC-C#YXOsG~V+J|QD>%`dFJ#GlntE%RW}m)}vXmoTBTE0hOkeB9mz<V$n2Q%%wiA1F
zKP<>Dnwr#h<UUT<k>GuFX~zVaRUC$yuS73@i}J4ZVxVi=Z)tQQsHP3|n)%YO0moM`
z!T&CX$@sNKTkk!C(DCKAC~DRnbGA;JJmV9WOmXF-q4%AJRd(VgZ1PY?iA*|@48qGS
z*v8EgW6Rn&=qr$s_#}<V|EyQ3#9ieZGo;{jY`<nHcezv>$pfd{TNgDWbFP-OGh^~G
zzjl)WtG<I2@-;*pY0XXfg^Y3j+9gj307mJG@{9NOQq&OHEI#3%E|Y~;5tU@JT;=O#
zr*q{?jptIOkE;n?bT30AdNO3Pvp0!7viW*!9mW9xlpL6DNl7VTRWok%eNy7(sc!~U
ziQsL8y)oD@B}yjjN2~8Ppf<W%(qRPTDmgd-AMDT*9mU*}m_D{Vc$=b~=~Vw(b;866
zPD-hdYYx(wcN11uENi!Oyly3~0pFlVg=B|ZMu<I4pqivks`1R2;afM)?t~9{;1b;j
zcq98d{6FVpuiF$$Oh`SZ^W&mFg*Pqg@J;HS8*8*;SGq(889TxK`1{05#kG{4-UY-{
zb*xp(vs8h@eBTXUA6QhqDhD{WzFR-*MH%*i=*0`?%38nDA{3m7tV5Pfb!xlUOBpim
z>6u}=n~k?98BceP^mVLVhuU&cS**jA#KH8>fz&^PC!_QGoip%Pp(KBiKTLDk>)T_N
zj}ZR^6Cahb<q*1^1=2!<S9#j}r{>1%S7R(O-oQ*m=p~)!%dkJZK5}riwI@)*!`YBA
zW{#P$^C|+)jwOAZ%(B+==K7WT_xA<@ceh{pi;t}v^hciExjpyDDmmRPgsHQ$eQTUQ
zE1*NP8Rrk^acuP4+n#dybeMnDp|VyC>&kOcMos#v!)>Kd!ZxXAN})hn`l7$l-<rLJ
zL>pp&n;qB`9qJ1R3TzCpuneOD2oEEi|C~pj>Ac}7nsJ>hsSfX+?Pz{I>e^H#CcOre
zAxGRgN}(L_t+SjTN90_`yzi?xk~Ia?5efsfu>?RO1CS(0lG}O<Y>q#j`UIsFIjDxi
z4qkTE=2xrBI3m4)TD;S&_^w{#3J9w4()8d6f$=`1SrcQ|<2TVAAb}mF*pcsj=dgU&
zND1jVO8M?Vv(#)?r@84lIp~*1TH}1jM%Tx8JkdC2ffzBOZBun_ijF7zRaT(8<#K`N
z$!Cr?3cGZ1{A#udrDRCOV)MKt?V4a$HH_TJ@b?C|c@Gc>`(0f*Hg#X)|C5s4a1lc{
z-_;`-ZVCMk6`~ZFc3G+jL*1(<2Ss3U-JRt8R@)|)EWv}2V;cUE<=famPmxb+KgBkF
zy?gMtEkN(U<O;6lc!mQjAaEky?%-)%8`LvT5z4MRO1+U70DV-q_SQWU=|48c0J#&j
zs?>7j)!xp`y>!)}4_*MjdM4T&KjfGVtfA>IJ_o)io)P>2=^RjyENeTGkZ9slnkS4O
zP2;Shb$M5GEk^(*d1BW|sL_;X4M;u!)LNQ=$+6MGu3119;qFfH&R2k`RlPCZh|O;d
zxZT?fq_c3P9kn5XJ=?GlmPq{b^H0d(x9AeVN#C^Q^?8;Y5+@VkVoH-ifX?{%lPr+b
z<yGgYb{50~Y!VC#s-woIXq7Uhxn;YvxQZW|o9lPkw?O=j7DmeWg{4C5wE5ODBEv2V
zM!z|#FzOP~mRQr-@l|V#C#2{WrxX@?r26MMb(NWgOGU6X`X2aHLYAKAso%ofQ}^o(
z)#0YB3X;M*tn)mSdB;;QR^PR~iicdkO<?;EQvb)4z=uG-OIoQC*0@t-9Dt9o=F`&e
z<0UoLxBJ-(uN10zft)GK{y^~c+PJ=V`X1szDZo0GHUD7HvHL!>?em>175Fv910>zk
zDC)>QCw=Gi(@~K@eXZ@ERXJ&(HWq!GEVF9R@jP|n1;G9+?{EtTe>ZfwIrBzul0bUa
zeoyNtUlpxAEg4Oyj527j+?*5lRJ;%1>ymI-+4Ke(zK+M1K$f^!S_)j$?k{qG4&48I
zDmJ$}kuHQyS#ZWL6t-OHBly9OrzM0}ckwTv7@DPAV58eWV3Se+F?%&w72kH0RN#wM
zdlO$0MP7K<S5fNB>JXn$gb@~9FnM?WmBn!YQeA6@-!@789yQY>%XbyK)Fm3?QTwuE
zs9?nr(99)%eahWDMn|{v*!U`h!2K@)+of%lg?Ec40bLNM=Em7vV}-Mh0t#OHjh-<C
za_F8*+V@UomOt3f)-5B911PC+naOO?0P1-c%Ywfz(YrI>tsT>nve_b@k6HnuHE>D$
zw-)6V1iM^b|4l%2e*Wom4wyg07TuUvzt}EEDiO`K*Mzd`)8vCoI^H1<0`GHXq}BIV
zB$V`KUbH)+7plcA`D^wqPJc;%h_#m@>*Mb|m~LmLuS|~&DhHG<sVy&I*cNh=>Lo^$
zN^w_6Q|b%uB>L4R#Tl!aBcb6@(?EI0RsTwnv-9~Wf3}$#&UKxUc5@9jQ4KzH2o?{C
zHqhwg7Hlk!0x{9~)q7y;73$Uxf)7p0F1{rHT#jTF@ZkRWj&DjE^*4VTuJU@7b07pP
zT4|@)7qxGsqDL7~{39e?oge*zz(O6!niLW})(2>S<$JdmJ#~u&-Om|x+y`PQAG2Ou
z>Wj6;mw57IoIw*%P_B_DubNc&qSha7ziE_H0Ho-p#kU?(h=nutO($^}*NL`VbfC%y
zt?Y0ObUf)oU$?Q=D(abWwy3w;g^nyU3s@Kc%8d?_WqG4p6)J0Qqag$2?~=-odxv;;
ze!;ecrC~B!;@j@i$5|)6uMOR>N)@9C8F1y%p9uey`<N|~`{qd?`QYAS!1h3b3w9Y4
zxKJe06saj@nqQ<%aJgaASESAIRdyE4`>EW(%u&7ECXH0Si}39DAs#g|*9hON6QpqY
zS4OhEBL5a(e`GcRO#7=fV020av!kF3jR{S72w$z(b0o5Xw+qwOoxxUL&<?S4D@Uor
zN9qOYQN82OWPqqOo<j5$W0#Z>aSl)-*5m5++>vEv>5*+uVcRW8Gb|`Zzs*MhRB#LK
zX4jMtS?+b9h1K^|CG#%;(v0(bC>hM1eo~av>Wx|qkzulzz2e>zmc^|W!dGT#9?i)(
zT8k_x3@O~nVy-v@R^t7=ENsj|_TI<OG)obPSB1iS3VkH)w<iMLb$Fy*5_q|)Xv5vg
z&)Ys64>f=YtLAm;p|@i*f^=#DEtD3;I5{mLN4o9qzuKB5^6Sse<yZaLwyn(wVirDW
zc9a1q=VEFql_Fk!KJ2bj;_RTQc15kOF<_9#yHBvK$*m?Qvo`a<6Z>wENq(XFD4hp{
zG)%<GF=G6$f+G|=c?#w)nvOO5vY5RRwmRt1bPDEHDN}Hq8!xL5FW&sfa=C*6$ivOp
z<zR~Dgp6(5?6%fS@op=-<u=K4CE!j5*oYVZd*O`h%Ww3`mr>Fn_ntlJqtsYst<ghj
zdR(q?T8P)uW&gcvBZpILQo&;--u?}I=(U{lW$JbyHKrSYu<U+=vy(o2`dp<CZm<Q&
zwB(ZbJ%KNmHD_l7l3*@98ACt~AcOQ#Q>b)vf~;z>=4b>LfrNFI9FZOL*ZJzRTYZsk
zQICNFPD|8q!z#wV(^|`}J8<%5vU!RSZ&E*M@NGg{w=F(3?i6`5QR0Z$Kc&!9KQ?Wp
zPpx{KajJ@EkrV)kMnrpN2GrbD1qu6mFk<QE*<g5WsCtU4xW|M(BfD%=W6l}uxB2i7
z%k@9`_x+NwIhIg-2~q#`Xu+Gx_K@;tdTjCvp&D1d_!W1JhpA(8J^S!5BAf?7H>T-Z
zGxi-};p5GaBk3MRJ7+4nY(x);w^}s(7B@XXK^7SMdS*X(Sd^KYV%=3B8vmquHqn6w
z)?I&Nqj%4!*uGYAz;*q3*0Rh%2aUl1NGjjYpUC==qqRj*i(;_+aF|;?DG?yEFLP+P
zC*0Tuz{Wrmh~Bai20K``xI&$Lc^R*!ovZ`tL!g~udzuq;40V;MrCG~yqaQ%A9`$Sp
zHJR3~POtIVtM~$l<)%f3+!#yLKb!$%K`d#$Lqn2Kyclk5xL8-3j5xtK|EXh!+{=aG
z7^xM`9TLxGODT$D9E@1jel?29)RMj~{7EjJ-(8BBZElPwU1N#BzZCJhp*8!4kcnsV
z2>KDFH=I{A{j$RNOMFc{$`!~??i3ak%So7YFnp5H8hx^yf^c-C39xm&{=vAk-h!Je
z1TY#(Sh`x;#Fwe+?KIY~W9}Axf@ePjZxVz;z4&_fLZQzb9sK{C_wo+><26e=lmEaD
zakALZtK$pdD-y8H>X^Jv){bGIOM<uAtjT&19ttOTAM}K}>zJmTGYePa8qT?nmj1Of
zPW1plWSN^YXD)v|p*sP@-tXGw@3k$V<oi$TZ8w?r-B4M$Otru@$}8}ltF@m<cvP{*
zyyy~`jvClbKAXI4;0TiwR#@$SIqtr#S3q<5%cIexNL)em&Wc?gQ>`G+f{8-!85WMO
z@k(RqFe%V=$x1;G>W-p(pdOupUVAl5hwRmamu`#?1COZd;#}v-h<Bt<*y^VOys<!q
z5zLF+#&8}OWZ6Pc`V|itxu^~&Fmn*(Z?xfZ5;fH)$~xX9d^x=-WgYy^_%UdJPk6xr
z$Clz3edM~xb{C80uY+Uf0(6lj-CFrNU+zg5OGSWK&85~+jmiR{7EUx)Bc@g#+WmR8
zrwf~;eI<9jtI7iZc!2>Q>V4!5*4HBFEky{cgALDj{LCNv+Jq33bzAKD-%Ywd?cyH6
z;QQO_T^y{wa`{V<6iQd+nU(7R(VW8D0YdD_<_Nn2!SGv}5g}b{g`S-*#`O(m)crZz
zZJeIHRMo0@P*~G=f1_w=vTu=b%I~=tN7BOR3?Kr!FN&T+3GdDs%d5k6S3!zSZ$pXT
z;%F(+1?YCYEl4<jC)Qs^NzZxl8*?OMBYla8hvp#ze9xWDahg%!HqT=V@=n1Z_Y;+%
z-Vo+)zsxv>9}kkmc}IIkm)FjTjS60zKurWo4BUVlXi$iH&dzWF_F^ScFEpei0s+B%
zg}O@R+Ml2y<ZJ4|!#z4n9Zm&7$16pQw8qkpTrU>KLB$If<15kbew`cvcAeZIU_#5~
zpi*(^FH4l%EFsn`iGZP%<zNfitsxs2>Z2+z9+<0&c^3qKK;$}~q3VeWxew_;vZ3Ez
zY2^SNWq|17KL3ulp!|Grw8&4qk6ggz*Tpe@=V)_m=uev6^4&-{G&tv2E!Y!kxd5i8
zX^RLpi?(4#>`vZM^y+;pD!}P+m98_&q`cGNuDCzV)YQ`t%5k(Gl<MO9DT3-je}_y{
z7e)Y}Q3l7$c@j`0TOquYUJGLK-wfEx+_B(*lEH?{Bk_|vG|rn66zye5fYP;m+0bW~
zjO_#?;YEb0tn-)~ckbXQ5r37hAi_|^H?y0hK+p5tgVg6CA+n2;jQ21VO$U>*Fuk@<
z2DHu^%R;;Zd#Q`hC!w^~-iuVrS6L(-!k{=!o3B!=G3Y}kWGz!le{kj~oflqLKpz)k
znnrAiuxJ5tL)bgDdM}5Q1DzU5a*aDP4EjUjYW0;cn((`Wt~=kexWV?it0AbavXIw3
zC9*@^eldQsB=LY7*<A@`T~(fzs!`}Y+a^L5pvfJ&{2-IP&1>?cC-$<yNj4MGU^vh!
zqm`PYo9rvQbJHnFIA4F!Y6pvBgzKC?`Lh3VBuGaW6PST6VkXi&?RNR(kRIs9rd0c@
zh1BQcb%5}?Bc?;5oT9YYO?i>9K2rDxywwLl>v$SpPnXrr+i5rt3iw0zW2o%N&O_dj
zAZ79Nj;tz!wD&wT1=NPs&z!w}U<5JidVaJ5tDqF0F5zNgAkKl7QBSLqZKT~KH$`>O
zHf${eRG`O18o^fJ28&w9?>8n?OSci}VBOm4Yhl~q9%J18Twl!ucW<<JduCPOMM<Wc
zdWh}Bs{v;lAps&FIP}p&BTLoL6#PhSxJUD;%{29r_#VB21TU?8XsBZEpK}>_&UP2$
zv$$e8fhF^E;`L@k+NWOvCH}I}!ov>83*Hu}?cW;0kG548F1wVNW!0o=I&5YR-_YI=
z{DErk4!ykV>Y>?u!yAeB7uXgD>OUhAiwn%=ay&Y-qZTr-H{s>j+!U2<?EMfIu&Wd-
zr~k`l?;LojgbSU@_l0)BMoHx&m6Do%y(^AS8)he*BFcmdnd|X;8OLB3)`Tutjg^Fb
zn<U+PsdxN}MVf1=;ngJ;9@zM!my7M&uhqFNGKE4o3&vlm!zKCx)l(t)p__4=E=zd{
z)spzvmoA)D`et0v7pn=HTac)O?Fr_0z8nBaU#O-(huX^8DRWP!+i|y)4HeS`S<Lwl
zDOm_>5{q~qRW4PwrX`lmDF=r32}ZK4jF<8)=?KTisGU~enVD|(PM4C#h#Ru(T^2D5
zV|1WHY|_9sCyR@C5Z&6fKzjgJ0@ifvWE>Ro%b3Nq9F>-)5(1Mlb`nizvk57&rJ0uv
zy^%E-9+(BktL>%-tn9v%?g3(QtG$`D>oIdH<C<*=LxokL+TWKq3{WoCdG=E1ziz5`
ziUTo9S=pIExw&)$&t0<4cWow1zJrGrh9RgW?g+qLm)-S~QSh?+pfNh}>RWwS0=;<?
zJG9E#(pp{{!?|P?D9Y$mtr{{cJSKVk`-6|)pa}SRi)A)i@=HP|;B52Ej}7~~8K@~<
z2kN`-*1APJp51}2`AMAj=S+N19W{}&8h15WJ-RPyhguD}qiy4}8#&WmyI(DnJi4Df
zUP-=9tDO@e@z#9Xc>u%b_kJ#)jWz+s?7xIKf5J6?oztxM31)rV&SG6kqu+OkP`(sC
zdhJab|H3<b79K3C4wrA5E1<XnUAVk^!Z6o849e3^my9lx#ujr$@6?*xV$%bEWqm$R
ze*|LnM~e$pYdLP)+JMM{BtN&~0D!tSZT%b-;+Ys_V4K_O2cb96iv$O3jxW85=TPHy
zVL~f^|CrSTPMK2@t_%*Bv5G^AUYLH(c^A+>YX6J=$N6mN#>>?({nQU;dASefx12`R
z;n|y6qv|4?yTo(T5n!N&RP0cF$r3W)9qJ>VJ!f#P5$RR;Q-R#@+3|{**UFCgj~soW
z25J8EY4svjvTdH{G}EOH=a>-+C0H{@K>_UC81#%nXp~v49=5aT;$XUqXH`qTuoOpD
z+mQ|?kU%V-D?SN(he=kbvUZy|y$Q1K+&L1`O;|#YuCrPt@ydS+2{P53j74c-q%6vM
zy1w;uqFlpG4fOS%x-XYZo+;~jUx=g!61|UcLdHFKNU&$>Jl_|<HBH#9c2#GS{!3j9
zrLofn(DY{b^W1k^u4d=zyam!I>mz5<)!~0SPNiM(AD~1oMuEG4PAm;}o!37Z;-hLE
zhleZd@U_$spK{tckeTI~W!+%LC)lv4tUDpV!L@duBH!kUjoQZVu0M(69|q|4kqYfW
z{^BkaX@CUwtfq%V)0$6+u3hrI>kRli{=sDd>6r>lLvOP%N9_cGOQ8@jAD4qA*a8~0
zI#^(TquXDAHmjr80F)HcCqlNYV^MF-{a;)w=D@qiYe(Jb)Zye<+D;3Os(5<opYC#(
z<bdG2wl@fF9@|Ge3g3_LTDa~}N!+#Ebq9T?NhR0LHF_2WUn7(akjCo1Cdd(G@BynF
zbQ)9yi_frPSGhu)^Cdj7b9X$$g9360lFQnY1VCka?c=bmaAzGW_oioXzIn$QPmn0Q
zyRHV}3661`Z4xx!n>0J?st#}eJG-T=F!mNtbU~Nr8ao`JJJeWdlfZ1M^W}z#OH!(1
zAA)R0<F<VarVqwJ7YFOc>YELou>MS-iUNtLJ4LsSs0*};*`MaS5{=w^ry&l4!7EQz
za#K1j=^}~%04C7RWDssay1BgaB3Hz8g(Mp2r4$qmA_W#(1yE)}#XbEwlV1xjn9PFp
z_j>^Fo2h9u+jSvE=TH?iI7>fOT_^l`Sy}|LZuys)VmJ9eA(i?s_~fzwAN6wl7c}>O
zR;IcNaIX(`cO&_ewQpQ{ar^pz{4<vJWOLCS$Vb&{upO-pb_3<*lgA!4eU{c#1R7u7
zJzISQYW+O@wMrG>j-6AshaPpPdVll(e)#{)(GY)64|o&rcM{t_FMGF7Ei@_z+kzus
ztXfCj0-xJtDp+2gzn}bl!dy|7y-_aM2<ThT)p!K^gW3r0odcJ2buUg&DTW<k2Sq~~
z-N8W3`_vW57rubHdhX>=uUo4IHBqvP1G5v`y-&)U91e2P#^w-A2z4UEBl9R^c~rS<
z7}-*<H-?|)ey`HtjrnfT+k;B{WLMs-$`@<c%?1z*KVRp&X8-`8p<e(@6|WXZHqW|A
z51|Q~2z3J=9WR~b=K<YHZTJ_pA7h^FGn4@HPoQh?$#No(1VD&*^yc>83tOCb!Q@ZZ
zpV{iETp^9VBD2*UdMBXPntS>AoxfLk-;UfYpjkdD{`8c~-ua(&Mg@<fZjO(dZtCS_
z0sj+tns&)w2m75EIz!2@IuD--zC~wfll*P1#|k1Y=Xg|};~eyinHzkcp$xg$#9Z~$
zkk9V-yKvPLezs|RVfY$2ggdEwM<gvS_j%{prVvD(y<Fg2cjDNQ#CU$2Lm#K}?i+T-
zTI*1UpMv#L(CZ7AGg)kPR@2jN{jAcNX!?Ve*QryZy95A%?@jCdJuy?S0z%U}!suP6
zrj!GFlQsn9Y9V+GI&P{|QGp7Nd=($IVO6pU3745|$qL!#k~Ti@^lm>Y`~sa>$p#m0
zWX3qW>o(>)s!1VuZnZSW0DWL<N$CXHbk{C;NN=Zz7NE(V_<J}V89<^CKyQ4$S)Vm4
z36KtS_)D)j=^KtM_%QYB59NQ6w!Vt-4_WpV@#!LG0H)lnopZLU`!Z2fE}{q1`$~g*
z0Ke~%ub>~$+puwRyTsdgmz<>z2lVBW_Towx-$2$ily_eRt|X6+TB*a+M_>4akFDdJ
zw(hU&r?~p)rH-#@ypS^Ahm_Q)WhS6HqhYz{6ZpW41MMr%M(<_&(j3b@{5`jKCv(wB
z=SOT#^9{83S3imiYkW3dJVJOjJO1U_ip#OWiba|TRC^SQ?4WU<t3-P-_3!~}a{ByE
z%L#Ib$!p`D@VJgI1soGKygPtSBRh@*sLds{f@xsB?-SJF00nMh?7%9(n03#i@u=NV
z@n#F86OS#xpt;=Rt0xWhOYsq|B^}?OYb+a3V%ST1&=CBn_?sBiT@~gd#(34T<(<5x
z&G}1?K*Rtj(@m!MdPS=%K2!4&zMF9D*w)40d#V#?5{P_bUC*q~QBjq;`xGac-bhs5
zZ%4rdL(pVB;?=~X?vu!a;S%N3%}@<iVY3Wuj&)oFX~eki4v`q&(JQ}U!<tPQ*UL1!
zp;jUO_Ns&6LC+EAvHg9cI(%2P7lVaocy*oyA$mvDSrLub6ZN~{8?7&`{E@dBT{TY;
zAd>wM{hS%`Nx3#JkB&{poi@)!oG}>CpDymrt7GlW&Ky0<-atQk1{?{gyY5W*n}0<J
zPN%`K%=a)3aMHH@b@FbsI%hXyAE((wvWc8GZRf>fk5qqe^FM*w3cn*z>ER4+RB!_`
zzM?$U2YP6UQdsyGpu5-8GP<w+$c{2dGtCmuu&msT9S+00&Bxj#@83WwYq{V4*<fcO
zEf#pCn+$V6>sSC08C1(S03iGOwck3>06^ARB!-=ezLdF%f((ZzaQP66?0TnNamLSP
zf52)bj1`1iHh~i5S&U;Pw!R}fjN6nCP}doqSh|mw2EYcz$Gs=dLV1oI3%~u_C;<J}
ze&mqCZR3<*e+wcwgN^zGw;g4(a{zUd$89zcfi@Mi$N8+n_NJ@Td`Hh!&~zF^a@Xf4
zPp3kBzA)voC<~&4V9#Gr$Urt^Q)Mm$7oOm)S}c#iXRL+k9JQxJ6@GZk!#}r+y*g{>
z_$6IVNpxYhkyHCHJNZ>z<J{?EH@PmR2=D_YOK1A^X%KHg6>$`kYn-tEgw@iw96<Xg
zGSsQEy?fl{PB0Dzp5WM-_dv%nAHv%sNh>|NOe~G-`pc5BQ{Ww91q<T|j_eBi5Z2;4
zTA1CkNRb#4E)dj-G$x|_=_~s00F5_|LjGgMzPzRX7tw1Z;lf!Y)Du_U<xoSJGMdgc
zZrQ)_ZQch6bdDOdY72z-b&v{|4#h4|_TK>1o$_L!;Ry~R9{fGqI&oO@tc=|9=40u-
zg-_`Xl<6-W^X~Od17D~vo~ENfM?=Y=uXmAaf=l4{B!savA7g_dllyI;m0rhd)Bo5d
z44!?0SvX}+)qs@H(_+jFAm)zVy~eoS()LzQV%66Pd}Zha`3tM%t)ruCUD&(N6=0C&
zq|Nofdt`NkR`oix{gj+x`8^-ly^)hT_f%OcAw{jywFO5FuFXHyNeG}bHpP<ZVjObG
zYY?2Vi3inO{@ZrlRdx7@|D20;KoSDfj36$5uZ8kF9|QzbxxFLoZ2n|&l@Lhd%Ig8X
z{*ee^N)ePe_}eL#P~mZ*BPJQP&VBwTqbcQLu$v{e@S<{815c6%>#Gg%rGx(4FystC
zyF=8H9wc^b;a3auo6dmd#E;B#%y07?16p_o2DT@#jsu1@SdRoSUe_y+NMdQWc^BRR
zOqqVspP9vocci(|G-u#BxOIVj2h^L7I%s@N9@fsCw)YLpyROUTRIh|V)r;t5NiH+f
zcNOW@%~mu-pnpkappLjYP$I_Nw|6J}HtXiyI{Dt&qk_@A!TLC-Wwan}=W_8zxV{*Y
zi9>B1C*N{lm*W?u3!uT7f1W2#en0TP{LTpVqmIq=ljlwCTDXrL>-+LwJkyAa8>F{%
zV&xSHLdT8?27mv1-9MH)@{|>_f`@lPgmlvY@$w1L(TaQTuUb}GD55j+5&}$)A8Y^m
zU;Y}>5JEp(D-#TG#-+cX0jA@>`n(k2$i$n??qk~=KC%F_FwF$)!T=4<)i=Nd-W1A~
z?0k{S^vEce{s^$az@JSze(W=NiY5Q2$3Q!F)TwgH=Z87hv12Q=|M$cH8%6^_zx-#I
z^XsRv#3!5A7XG^s=;!YiKl4I?PD%eIJ_3GuO6)D<0s72M!M<m}qaI!Du?JuOc4=L1
z{e9=ytVX2WzXeQ3cQ^CYD*kQ8k8XwZ=REl7x}Na-C1miZalugJr=9+Ly#gP)3EtvQ
z{kJF!xWz5L$JF(2YYMoPbd~yI?snh*^2cIG;kPlF%}*5aCE|L{sqC&cLqxc&t)XiN
z9$?^Wfh%k^F)~tLb<g2`1j8Yzsw84FS?|zEc;5`&#MZKNm^k}gg}C-=(fRN;%fG!Y
zLkd!5klHAeW5vMf7D?42{+)>!2~+DN*dWvk=*=7Vl1&ar_t9hMTy$^c#vH$tUyV?+
zeEDSn=Oupf@2E4X^_>l4fMIjSnQdvFx}1!w!1I7<4>w97zzN7@b0DzNP6^&)LXgYV
zGUOSNYAR$4FFnVgyQ)-puS4M8%=*4s{eFdBFAf@i@*Yj(EB@wln+Ik9n5YcZ9$zju
z+g6=SIYkOR+}A=-7fbYP++oXY7^io84gBhY?-cX`C<l3v5`0tA0CPlAFA3+Z^UVtF
z@By;%)nj&dk<mBIwjiX;0x!L+&O-fkQSFpx&;bsogJo7akI%pc8PTJvSRD2eadf1-
z>&;X~fC*$6ZD2U-q|voUrw+gT&$I)tsm2%(34OmfHSIgeFb9ND!L%L|(3gQu20hxO
z*z!4SxoS$fAi<}BPkxo^AP0JLu!XBJHp^imK?alr>b^L4B+ibNfAA%8HNVEzupkN?
zeF@L@k+T{Xs4hChh+C=1JyRsLEz@_VJU_R)<|$6sa+`6Jbg+Y!QX;-Ei43<yQmuW;
zY(zWwZSf!J_%5Nni*DVo^VAAXf~dozBgzh~=m8E=t<aF+jH|24#b@Nj9p4!XAFnjw
z%@@P8r9^X5?vsf6W&w0)(~2SuiM(Cz-M+F<n9@;VXhB@LG&ti*y;#^Zd+5)TwHML4
z544d%>;BswD(A+xtiad2J3V%JZyXfARa1F!2ofWjnH-XDBTIWax9(SYCaE42e}H7}
z8m!MWPo}a-V=O59yBXb(y&8|2@SQW9g`{xmB@&uzb{=PSuC9FC_ZbKE(=~ZKY+t{(
z)A?{~(EoC^+Mo&*KNVX@EH<R_&f0D2#xfuHuG^b8dLMYsVq?fpi>Lu5Uh+)PZ%l`4
zy<?q<1T)X0J+M?Xo_!`;<5deYwm;+{25}zcKCGN(UDm!*o`(lo+s~n~QcR6$=lk|A
zD=XQco$C)bVIl9cviE14eJQwCy;*7<b&<t*HBsF|C-7O0FUwSXX<_v4CVwR>Wo~2|
zgvmL;^0z3`k%vJOSNuUVc-9CrdSom$mK^lVhkkMjiCp|{Rn56yliGXyP0bIui_T8s
z!Ct}shFXuiAj3Xl-u6U7h>;%+6kYcgS6VE+y0gyaO9|W2lz>pL?!hJ&8DB(cl<s5-
z{?!|>r@BT5dzJ6vtV%V_-dk-`I}~8EnMh*EY_7gor{>v#$w9T(B*F+z-dz!57jFJz
z=*qb%54RKNubFY(9lse*4alPN?JmDjJbx|}TV7b})#b4pTb%g5(#IGn*DTX0NqI5u
zMHdl~s&+7@Ts<0ouv#k;ZU=U!%oWmu!a2mQu+AAKdPXeQkSx>t_pOn9(y~<95sa@f
zS8Y_J*?|R-1>hVbm^s=G?1m054JfHZi`@lrt%QXNZhr(sK({d!=&F{qgIp2hK^`l$
zQi4o-?IVe?%J$TEEiSX!X9s5`f6W~Mm+Alt7E338Xu=O9<c?<AyjrW_-^AMJckwqL
zG%1sNO||mYtPXaU7pGKg@TH6G&FsomoL@g@F1Et}&5X9qB(V2dVN|PsnOY?O&{P9v
z3wH`v7X<<QV}@45((^05iKnq|WP_Vp9!gVl4=LafkA3U)36gKZqyB3Om^`_IReSOR
zlM+$eH=I9)fZdJXwyoZ~;dQu|QfN4n?`}lJ@9y(9+w^T)MIK}Gq>!%4=k+^bK5W89
zxA3KFUsTV*67C<0K?<ns8Jx9a{mMAJXXci<Pn>&NPasO5#yuw@?Tz&uoIBD&Tp9mk
zJM&F#uHt6tDE}o7d%01@f9%ZWSt&-R*E$+Pk<HiC>YG385ub_BF^Y2?66#(caD?(3
zuxqQ4a_GL06GD=Qm4B$lkqbz>g29H(e|&>iN~`nt;B3S@zDEW{R<J-8895c2+NxVB
zN7A6&iF>)bX7ap9a?WfU*Gyv2HR3tU?$w$s%IRLBJz}Hbt6@^ao~hSDIW!`mK{m=&
zdhgG|W;ZP8ISjDqQ+GpH4?a7M9qL^*@PiIa2Yr5b3YOjJ9q{!gmI!uOmS;`U5zMJP
zm#suHABV|KeFS^k^ewJuE%$Ctz%jG=?y;2{&J>=e;GIod1vBrdzBU?T@BI{V-Ewcs
zj<{&fl#-|GqO9Jz8@wD_exf37iIxEYx2{C5p(h^|En@pXRZ#Xt@(iXw(Su1%fVw(t
zge`pQLu30f=Yfx^duUS@4o|>k)p7yN#h{5lc%*X<qUmB(@Oj}JEfJ~*4K7hFIr<UU
zrLT!<wIsv}?>{GZ2<_;ps&s72jMaEAZlP7T^VUK0?dCi;)sZ*U-H1CFJ57cfeLvgu
z9eYO0d0psQTERRf4BfoV$jS)kcs_@ti7QY|QJX!z`lQ*%v&%g@{0*^id@I4|Y-AC$
za!G`{D%U~%(nq~$z&<fFE{xtFTpc)ISdA7?Ss0vVCzLNTc0T_@OXdr&s`cn}<D{BK
z)r}jVH;i2fO|<gZJ+=b~*E7HP)&6QfmktJl^oQ!5K~+ZD5p^h^sot88Ywtow)Bjvk
zix7T2wLdX;EJF{Qb)}IqgIm@UP>hgRU%PjgU)4@TWdXWFX{2V(EK9o{N=Bc#9kSN%
zQk_P%eL@fFIhcuf_GauKpP*g1UHJ|9*_2kx#qdWuhTDGqC#^bLZH%{y-;D_5JXyFJ
zvU=5)__UokwQs-Mm9wRY3}>7T>}*IBlAoR5OWn`9qOeB{vIU1TmcVvwQD|tg+9)KQ
zd)Vf%R&)V4GCv;bIS<c$*li$mSDAk&j;q@$9m7UxscGg<sO`x(4nC$<++Q(UKlRAE
zi~q`VMF@t1hesxrs||4_>UTLG{x}F-Ny;ilB$TG}=ppT4=$IKREHZHk^!c3p+|CPG
z-+aOVGB)F3%h0zeOAAE}oI8AOzL^TxPRMmMs$RK%IMLvg+s`G&{u<x?G0#}60G8vL
zsYdpmwE0wU@Nf!WN>AR4?R{4-A&fRg)uSuIgaE9|AqBD}-^R7&MhSB`WN_q5xr(ig
z(Pe;$Mn3OhsEYi@Uz}xzZce>l5f9ApyAn1KWZ8exdRu%^1a~#>N$vi-$nRH8e95@O
z6lJpYeW9LH<d@^SON^easQi224vo}O1IkC6AF9Mub0{*UaKRS}0M$Mt7q@gnh%3IQ
zBkts|`fna|PS|FfF%H%X9+I^XMaFcs`nuuTWpxO?o-Iu5ruB7+zZ6iUJ2IsFs>!%R
z%Gc>nWsz1zoz70-?DS{H-wgG0t0S3?e_L70b8U&*d_H!)if>~u&ZN~3DVmdB1D$Zi
zGVCGd?cis+lUFYLy^TZ0-`%9Dd3O++B^^tS_ZWUKDxH9qHxDE~X>kWMRgjck@#4cg
z!dfy}$=_2Q<FZNOciFoo?6N=eY&=VTCnmyxa$PF&!g%DrTe!|(3CgN*_`^k+P{B){
z%ljufgjrd0NU`+`W?+niE4ndh<AO66$`bqCQDyMy<NN^UeuEv```st{vjtDh3~bw?
zq%jNcQI3GW`Y}?kZOjB>QF(>vF>EzRopM{At@3cj;V8PFRn_1J8AykNt=VOl<NICG
zzS0P}c3Z^cA(gQwVZ>+Sz$YA<=rE62Wo%^t)KR3xs%aU*acK-|;rsB&bE}5bgW0%|
z*b|bj#-D<OV$*tSHITB8S47wXb!q``OXDd|Lk3BPwFOh;X>bmt5l&xPw724)@3}0I
z*?gQFWk7E3btV~BQ&rBwFP`7dl=~sqlNbptcbE3nCtj{5Qxu=7{Sz-VMoDpYuiP@v
z79Z^KZ0;BvHX!!d>cdAUNb!5m%x@1hCqU1zzo&!6sd;NKCaw-LIh7W!`2AH)Xw7n7
zN;_@_mjVLY5u4HGcfG-KN)G{9D;TQcR-040GSyrD3#!dM($DxHsdBu{wrR2lwHN6U
zZl=(VXDyw%td{d!4HrXJt##iS<I-{pn3s>H<>ee2@`wfKgGbT9Ld_o5yET#%FwFC=
z3YFe_@^YQ5w82JLFj|fhZA4hsf=}`|#h30<FV4LWr|jeKRb|!`9FMePWZLnsIR}fH
z!3?i0DOUsj3BHod?hdET!|NP1Bd__127lCvxHBXOLi$Do<TJK9VZEgzeh~e9l*4yj
zd(<a4jn{qPE!n$;`Ela6ohEo~4p!1CcW+G?ZrWQ;M#UIfJB*}X1?^vA+NUV3g&LB#
zHwI~@A^4(ZR6{s!pXY-992^@VtX5De36vr}8cJc7&SNkmxP+3XfyoYM$MXJ*qPyAN
za*xsk&(>R|jFuk=m<vPL$Cf?e3%*D#_Hn<Od=cLpen?=93<4()_i=e^#(^}taiym;
zwBlCyt>FUg`D$<Mkc^qz3!ih`FU~_U@EKM(!Hz9Yh<}Kw7ynWdfooUclbVNzz*!Kl
z=E+4yoz^L%q)Xu!`nL%!?;COWSX!7r)J%qCGZ(w9ksGlO$8~;CY|Dw<uV{+LSxKoc
z8O)UL#rzZDg4NB{;h%>~1%mU?3Xn~AnBlHFf(%4Q?vDgFE^GsWrYzr8M5}Tpq2&QD
zW6M)#zpwE7V$baFAD|be7xNd*$t%Jzx*5RPGkS4l#Us;*n_?E+Gi4+l*^l%WApeNG
zqdKTT$;dA4?1_{C`uDg@&df(=oypce1QB*_D{jAyiW$N^<0*|YA>U5jQa;=$FzHdA
z-PlY>yCV}*m(J>NuzijE8c10yNOTk5dRK?ZaNHZKPdlXx0D7AI?nM$f6AFR~wuSO@
z4i$sN4i_@Ucj4?D2{EM5FhF!?$0`zvJFExdm~g>euL67#j%N!eT;Rm{gQCE?I7`D-
zS@5-yE8e0*CH7m_`iy7#nau@QOKXCE<>Plp8}rHIl;Hq|lV0XLk$SKk7-Ov;Mf{4M
z&8;78AMPo!G^)xnk#r$7H*!p(Dyc(y{)W7J+<6qf;6v}!*=5{B>XNsIk2r3JoTttY
zL{LUUTsIdaBOUj+WD+Ae;<u_OXVfSaDmK=KB|_4evEHEa|L*jBPYWANJPb8fB@2gt
ze8Cy4S`QMfs_~xdh|Hj8j||;Bii<f^1{Z77)KV8mGb=BKtx|ylVZvF84U3N(-*syx
zcI5&%gR?|=&uwN-aSNF$>RYkhO4D;l=;DHa-+1Doo2e>gB4u7YI`33R2_jKxEq}3B
zt@HcUoLTH}>I)S{eG63csLR16Kp^y?&3?$OE3FL08e~-XBt<Y-42_8#a-Hakz9tN`
zAQyIZo~=3Dv|d4ohQF+ml0>TjT?_{m`Y@Ci5bFlf$SsI?1;D`sO))x77?6?v+G{gv
zRIE`opp(|XIb<yT&3s)2#W8KCRZ8Rdal@HtA(&o-Q&x{E#kr^;`C<?4zzXa5`j{+a
zJp9n)H48;$f3r=h$_K0T&BdbuXbP1ge!#63g7H7QPcDK`iq1TwT{AqSa&@zPAnuUA
zsPmI@45RFiLY*GfXg`FU-z~30_vKTxYTkbbm*EmzHs^YS><{S_pZ;?Q)0Ee$b-oe<
zu~SX})YkPS*91QAgA@DG>HRiVHf8NS+vyPu@49nS4t&F}?al4g*o5=dk<69@{|fB2
zJDjtNx)3-UD<96mChFr;UY8!Y>g!(cw7hw@JC5db*+i7H&e|=rI%A-05FK9DoVGN?
zLY_sxUq3*b^QfX(M7JOi9)(&j9gVf#sd29b?UaWUwrS`SOjH<zD<;f;8P_wot$xsY
zQ5dBAy<CrN%QcUaYs>H3+uQ%daKZ1NqAx;3Vo{;GfZtgDAqTmW$fqU(2u3XNlcN(X
z>Y61E6<7&})m!<e^)6hOo;Y3mQmGJwl}AS<(0oFyW?FOmep&*+b9Xq#{iyll3}x<q
zPhtw<Smgf%(lRE_-3Yoxdyb2n)Z=Hr>>ownmH}jYSi+=|uI5y3jxYnqKmPGiwiLMj
zmu0rZ8tzhyANi>?Dqvb)_=PLVh?g6U=1c$Q7fyw~23=Lew8r|WWGP_MZV@+yUaNre
z|1))0W>(~v{QuA&I@SL^azBU|d2Ekxf5~dricuUz^7&z9!_d=JY_h!&_m5!)95(eW
zHLC_;HMzi^=3`sv;z2|^f)kvzJpC1cWY0)5D_ZGi`RCYwSl)l_^s$Ayxx-$f#KIPt
zD?+CH*aG~2@NSCA|Bb!(j%qUd{)L^fAS$SXovJhu5fuapO+h*Y1qJCUgdz|S0tlgq
zD2}3Z=>d@vkrE9dKp>$gB7_<`p(+qU2tis%Lf}5ke1GHo9A~Y&?z;DV*L&BSKX9!G
z&vVY%XYc*lpS{mMv)Z?im@LJyrJA*_u#~W0wseSs$})E#KGg(y*#C0%5#wmwCZ(DS
z=6&~FFDB;a)%USUp(KY_dt1FW|BBQv_Dl-45>^l@n$+X4h8j(^oJUtLQ<=A|H)0M?
z9TeI|aQWzj_WX_7118`upNO*`-gfhENN(o-(Y;UEyJ1{(^8xr4whr<}DPri!Q&0H|
zRFxCya<V2HD;pfs#ZxQkJJM$TI%;M3_l7vb7z(7qbU-n)|B`5|Wg`)|C+nm%7tP*a
z&uzHd|MFl&sCVqSaAnTZHjW4Mm9{{bqrHf5rw1yc_qLG8HiW*?FI&d6zbg$Yn#&KO
zPD^nh%NyU)!xT`zUh?udYtwbxx?K<Xw3M<O#+e1at=H?@g@Nb<;%(<Jv1#7-_rXpK
zY}@i+?{|?@gH}|hbH(?~CiiT7-9SU_w|DTGKhE=w7)ayqO1n1~*y-%tC6<uXwZXEy
z_9}WIC~!$n5;`nLp4Rp>)x<5<Jb8*OJMs`L@sBBn@*v`CLY-gy_DTVvRP?#~E{+=(
zvbOfps&*97Ow5P&J?~N~I8AG<yd0En|I&SskMF$O(aJEFRl!A{iY~3Yq-(B_FC#YH
zad&ot;yixcQu}vAAj}bl8Twxl&Ubqk)PO3E(`6)_puO&IV_HIj&Q)b5kO{70Zv~IN
zJ4v*@q?ED}wzC8&j5o)&anRm<z9cNnd)=XaL;klSosE8_&%I=HnGW<;JjiU%W$Ad6
zEn8MQ{$j~L7iz2S4P|MXriIbfIXXT5J4H6%nJq4RGAbE4`%6vDSsHsHn{tOy`kz<(
zi;!#0%iTXwDP`5fTL0(nnX@oRk@<uR@M5ti=y3hPo@8yJeL`_L#64<=x`FeQwZHK*
z35Pe0K5`JM5<YnKiVsMTvjOiMM{(qd2do17+a|dc{J(eY{|&->ezijU;)3zkyd2o<
zN*X^0<}nupw$@Nu?Vw-qV=kiyE<D4S583xwO2*zyw6SRB3YPYT?LHE6h?0aC+$L{s
z<FvIW?b-7a+b>I-$^N*>YAS+c*S=JU)-08U9~7R<g*Bh#GVNjgctB}*oSTK{`$Bs1
zwPkm!)*BNEMRiA+hgpF$7~%126AC=3fEautU^-g|=Uj+q*>%x_B=>&cSmQ1a!5Q!i
zSE9Q-(g<_8En-Rh3Ep{YE?e3D<-NQ5Hp(7ap+8%V4O-*w@xzVW)#YmXr83&ZfwrYk
z9&3JUN)g{D;aoM*LpjPezX;pN6F>RqyEiXJ1;eurvRW=rl~9}p9{6|Oir^FLPgIV5
z<r$qHTFQDSE>@+E`f#SW)igSEMp?ZCuX!|Y@Xkj50Ne+K(!K@_dH=MginaeI`m&DP
z2z<+$yoW+2pc~&Ke4F;~;(m$VBV#0Od5k{o6R|4|cQmygH5*l0^QpXmOzmHZhsX4H
zKOJtv@8wqrP?s(wgQ4Xuij%^r*^OPCimY%RJknFc<5Hmqf(f}D@z`Honn)lVW@1$h
z&CF{a_J)P+d}!GegiG$r79{wW?dft80opJAXc1UztElqHPc0W%s|$ML$@4K9nMnhK
z<qHzbz0P6rn!$eB&LwH>4(m6aHCk8K!h&V9TEWUO%x_E5K8p@;+A9)wEo(h%V_#_C
z5HU`KuzX<%+!N~8dv{;sTvz`2jfVsNKD3=XI2$^uQ?n?#LqjdlV(?PDs@7#B9OKvE
zKbQQtSO6+g3ep>Pn6KIxZNH_TTH*P-%^7Th4eH+hAO@8(`}=D(<{fk^YZ@H39*P6H
zbOHK!I>=Ko3)Dm1II!l~>BB<BWHFl5lMX#M8rjR8h>q{>rd{X^%7v624jlIK$K>&C
ztLn?%6BaWZcN6PLUvi8&uc<ey{7>^EG-sJ}>Dr_KYj!hLF*?k@RcqWmEW{kU?twL0
zPFM>ax2_18OpvVPR-q=Rv?RHUHp+Rg>+%bZ7R-Dju#KJb6<1*<&>>Ef$Mp=6cJTZ1
zWZ_h7sUph=Mpk(_dwMpBp4<1e-%o;F#t8LQE%fVt!C{=Z{44od!A8i-elY7bB7E>D
zRPUs#aAz19NAlcfMAt#)@;!GWEWo0r7z*&Pmwq#~_Lbs?Z0GQa<y%MjZdK-tfrg{7
zt@jdkNWjAI^JS?JT6O!E9j(Cz#5WE(K|>sg0#T-aybWWBHcG$H$#!n9n_Gl*`BIfu
zA?+XdwlVs;V}nNhaQx7!lz^pI8lu8Ww{F!;`{v+3<yxx#cimQ5TQsp+qpa|k7LQ&D
z&oL#i2CCz~Y$lg}LCRxWcE=UVziw|2Uhb}bo!LxJjR8na<q0M<n2H=FzHqOjSstFO
zioSeCQ$0zw!5#@dNIyBNYuy-4ze(4tCDL7cqvguKRy-HRU`t8#>sZ8XYx}?o${Xst
z9fyNG-W9v@6sv00^#;f(bIW(+;KuEF=xV*yZlq4(muFO~hNC^{`elLLp!iR{GB#ne
z@@zm?(Zo{;uj?WwKL5Fi-cO<oh}FsdjlwU+K__XTfi;?6hp)FDt<r#TSeUoR^9cdY
z7tGNg5x|qa4s3}VM&hM(w%P<aY>Y;P^siM>AsekE->%^4%Dd^Ta%PUGE-c<(eRlmB
zw$A`$-6q9neOIuMRT{Q;K}b7G8K#Q2cVng+wQHAm!f}1K7H(hac<Ka{k7#^{2c(FN
z)#B?9<tIcp>@1`i?PyTeZ>Irz5dEf$egoPpuo%BajyEpf{`$mb_E2>TbVxz|KyFr#
zld5a!!$Vj{{O6NGM1PW`s-ddikwdptUk!^ztE22yaqL!guX1CLqwX`^bx4E**47h=
z%!ih=U;dhukY6nvK1Uhize6r~BV*clH=aCBHEIgV(nFUHP1&IWkEuQ+Mn}#pS2v#t
zB$T|a+d*a()$O0$!6h}fh3YaiiN%gV#W>GCj}F-ZDJ0%r$?ftEIU_l74k3Lh=#XcP
zM_^T7?WBEU=sQ1fj0+j@+Rwx-whKGDQi`iNzt&1}cUpXDeC%DP*zGi@`QDio9;vxy
zVme6p0~8t9YOd)qa=|Tj{)S%1A636i+bX`pvs@lcRI~K4&n(5luCJ;EGuH#dIK`}h
zAs{Ig{z`D6m{(sNpO=%BKj|EN^|}+y`ABTq`U^&Dn@2~}4i7ZQeF+aHufk?lu?P=B
z^w;X4Ezhe&%eVF;W+;$SL?ZM-OEaf1uxc-v<))fbCd6<3;PPT&jYT`h$j9ceXR~<#
zMr085g(ieuerH}OU5O~N9#wP;UCKmry1!1T$S1cJ&Y4yU*SUr7;Sg;4JVVf&z*NOi
z9pdp9bBzInBshaIzI)24%p@q-Aw5S?6FYRl&wNFxBCaSV=yZtP=%{R(!*H0q=afSV
zT#*(^0u?jJKE1gR>nq3yF*c^pc3|Q?2Hbs!pL)xx1N&aR7>?-`C=abzRw@+kTO?i6
zP@?KfsQ&tfd<i+Z9zvq!Kxn-#CT=h|$0|7~nd=d!bScJDlVTJ!I-?ZiIYdkFf9|9K
zQqOwhB+@K0KkN>;2cXhA(fGmN>I8yr<+rA@o;+_W4y!pD9r-r6Q;2wAQ6l3G>V&e1
z<`m=bYq4%dPMFl1J=Ar~wQd6{tu)<T@U?-yvnz4dsWrZr7c%g`QXcUJTiUeamaV_F
zwD1eF9Z;((#0!heEw9M;oK^jnt34ZUjf}mBu@B=`ufMiYRq)%D<Amr4ai`>ig^^?@
z?t*&MTqFHKYeGQznZdSBwadc8C}`nwN4aPDa>vwVRm}v<(Ch^n(lzQjd!)6Ho>AU6
zG}+6WOh4NbOxOc}uzlr|S>-}N+{t{><M!yGNFM>knUOd;VUD%?wp{={D3sSXf-pNn
zvkji5#0LhTN*NV+5`1vZAz>s-Cnkpi)yz`fLMpHWDic*6+N^*MRPrPY`IxSAWzG7A
zxUT7O9yTxW?WGIu<U&j?GkqigZ(B9t=Z(5H7&yqO?-LnppW<7u4D6&8HqOE{5YgF`
zTt-#9_=<uX#fcngKS%lS=#{|cc8}-v{?3)?C$CpaDsQmGS<^s<HLWbSt0rYR16pR9
zjC}p7s}X&qZDA>AX#M3}>8bU7k9a}c8UJ^XdrH%}Xb@(j9yU9`zodp(vp|9w82**&
zB?YYc$$_J9hQI_2I8S02$?ds6!jG<=z3u@;CIjZzh=`)smYauL%~kqWty>Hp6Hi0u
z$H|g-k&}gWr{Kei18k&>+@&{JrGD3F3G3wErlPOp=&L4fh%xAsw<z&`>B5L!5~m58
zP`-6!2YPAbULL+=Rz-^TNSz?>jZVgRDBc>kcT5KjKbiCE4f^7!Ivj!^C($KV%66Av
z+_U^bJlD}H#gtAHJwGJQb(GkpL?tN3<}wpU$ONv{T#(BOft1_#PG5BzOV#PGIT#ow
zHfEny1SgIN>TGRJve&bTNnj>JP`=38B=yFkAos<>#%@RH%E`jUo{&m9G94xwB<gGz
zkgSfi56c2G6Q?BUAsqXWF$ZMzDhw2f8vfkuyI_wY->r1+sr>N#XCgVgzLpbQ<t?{*
zKJ*VyU!C%0<i)%Zd=O-Ot)q#saRq9j@eK9tfa61O0mH>N0*qCV8aS6ct5}YR3sjBv
zaIv#^;+|Q1`B=GQbKhm8hu$@y599g?+tJdx45w2hs|-o6EiF@hhu_c+eDofbx6XvR
z<oibt2IorBr!fdb5MmMLAr)Z~H0<G$HPuI$+nqMxUfd)?+;K;7=h#>OBLQzqS8OqR
z7T+<Shn|U|ziI7CEn)R|UXVm($%(3GNsv<!MGAqSsWNlx1baafym1$VAqh85S9HJ%
zkxE`dh&@upyPM$TkwQZB9paoG?cO1q5<;7&gV@JnBx0s|ikw-VKXGCC9+_7Z?!cZY
z6){DHC#5Ep4#${S2Ly+!DUUhPVn=wep1s{Nu8;u^X46vAO*6kb*(oR6WCpIJJJCu-
zr0I*;63BM!QmK-t)$%1&wj9%m+spKTCs`1JMhE>nO@g@S^|=ZTxC#Xt24XPQ;8h#>
zg&iBSgy3a06QpB5yir}##-a&pK2U?iz6fBr$jNe<g^@p#*=0@WXHtvaLgyrqqecqV
z>;O(`o35vaUAzu7Ig~O=AFlz_sXKn{T(EM}HiU-N!O;<qs|Ds}glLs=foA&e#?a(X
zbSL6WEmpXSJ^Fe9W?V^`^2fPl1`AIRx(iDC{U6*7#dP&GiWHi=6b&a!m75dsB32Z4
zqIr3%;6-mHqM9GKm%gtz8pK6anQ}AeV)X9R6d{9bj&HkZ<uZ{^t&-)dx<}P-Vvt`)
z-G1SwK_DnvPFa({Z;@=XvCu1S(zND0Y`UT`KPf4jGUdPIA~{^s>yW-Q*e?d$r20lm
zy+oud^U2fzfnu+nY-6D`nzoeI&z%mV)8@Dt6PY&X({|YC)j?5ZO)S0?LZ{`fbUGg9
zRdXAd_Gcdz<%FU$@7jBg4CPL~3G@R7-L-Iy@E{?N;z1TL33}l-as3eay@SNfTLWtO
zJ$o&C{frEEe}R4JJ#J&K0aZ6BbYH=_*@NDb?*>+JO2x1FiDEVKJKL#TihY2I+p=kS
z83(M&Q?3>~3ny^Sn;x|yaL(*y`f6Zb_z-K8$`}lEF=s9$o1Ri^8n|vDq{MD{LjiA=
zx5b<M-4!ixs2s`E%C)%QCCeKur7F>qyhrXJE39~fKe!6v1eK1Xe<Zb0PGh&)X=XJ*
zk-}O!pp;pPq8~ehzOzS-R_Q8T$t{wi+maT0!LsDr`Q%DOLsN7pk&Y>&%CGk^>f7$y
zK%p)v!(mAiWmln394(Ltf(t5cO5EK>mO%&^@-?D$oA`UU2!HUGs^S<GMm0kjv`6I}
z1DdN}B{|@&X&}hePAR4dJRw1DO6`q<q6%$*3uoAAQVzmpjbQ(`GZYW$qB+aaa;Aa2
zK}@nmBi*lif>>y;9n!wTb_i~}_%@5${f&+$L$>dkJ=Zl;BInEZ`hc6SZBXmtj!chv
zg<}4S<?B>J*Pw!O3=Uzlp0-}kYew}m%9(T_OPE+u=4k!x#3|Foo~Nx<aQWQ#9JvK$
zT_RCLoNis=!OBIH4_jKR`%>T2*JoF-t(}QCvO-`S@N)Z<e4&UdVOAraSuv8ng2$Z4
zG6I^ztkZi8EY?XQMkA&r7+evnJgd{?*~EK(1t4?+N)8RBYX4UH#9D(fB5ijQf=xGu
z0~Wq!wMGT50huw^cNFt5<Y0N2U^S&^e5tXf&y90x!N#Fxu-7D&f8`}s<GOUys|5OV
zC%+Q?)qCMHP2lt8A0!J)8-^VY(O;oCU5@Phumx6>5W*>m{C+};ezgt6BgaFwLIxW7
zCrU@itYmniDLKOCY(_Q0)n79x8c4`ww%N8K>6qCALNJsIvaMDTn`3K^RvTt6*vd{O
z<s>;++g*naR5W5C(5*)T`_I$5<BI!q%89CJ+O)Xg0PCX_h}Uj?{Yg=9)5^YfX?lpD
zM`tkgeZtZOIa8A~Q8{?}(yEevo&oWu1C!pE#~_P=(e^VGPmN)Yaf}xcObc%f$b=h|
zcY1b}B-l(rhwlxvg{otV%@Ko&qUuTHZ(07R0X6r))rvrH^x3X<|6OmVIV(q$CM)Ha
zD-d&g%jHb5F9R#G+V4g^6KO0CICiW3L$97X1q^wM0Z*?G&@e`w8pjMf+)XRYdf!)>
zjzbQyksKeQ34>A1D&^?+n&QJ|HAt|x<mTeh!1Z3qgRQ`vNgL%TPV>j!z3CXwRnBt|
z{3gKd0g7gJ{+Bb~`Q@hZ(J<MUL{WoD<*-D})Z_(X%6UfnXw#RTF?^s4uKGE3H8b>r
zK((@*5sUqeED-^QHm1<$4Qa7hgkDppWJ%3g;l;qvrzMS8#KW|uQa42HTYc$Ni0bwW
z)If+|aODuc5g<O)K<DMOp=jOK7(RXp^IjF;YkBT-AdX;G51l8CrNm_Fp%AG_^x3ZJ
zLD1K!<ED*RBHliYH1it0+_~fY(AOzBUkJ*^0za2OCliqSP~OzP#){%r-s$c<(AJK@
zWpmJtWx_0|*gJjJQiMpm2pP)<4Pg!^vk7F4T+d`DqAe{qCOugZ${OhQOr;WrI3F8F
zVuFgh`a}A2{iIA)X=qPq9EUU_>gLzV6(q^Q<z343=0=g#L)Ct`P^j1m#wf2%-w?cO
zM(QSYs;-w8i1$3TMWEcSkeQG?%$XVYz?p|~94F!FZvFyHI}SJ}Y(<K`jf%+(izk#c
zVPl%1C`hI*Ev^`)<Ytv`@*aNVXY!zWd}U}P#gv?B4$#rPsFDd3?nG(Cr=q@yCt_QI
zz){ew&soY&-v$8(S2lJZy0KzflPt()qo76sUo!?Hz=o<pLr*?q_?62O`ukL7{8I*^
zi@}(PCV$(3+;GGOdo@sj8gv4*khdwM#a!Vqorl{E+v}2E`1cNU9!euM>&Ij>2HGKh
z7*+)$V&$gJB0UMo04<*{wvV^b#W|1z-xPfILl5zrnOGs_p-<3LACjy(zv4IsvhL*Y
zp;ncF#ZGo%&UMTgcM=ZaQiJ7pM>Aj2m%7_}6;~>7FW}K`egP<|%FMTA5Y-RMG`HaK
zLTgO$u2oD<`k%B}S?}OFzi0JaU(x`RXwFOsDw{}1_6x;2+9gaOU8`;##8ZZGUpYl`
z?e@)|J(YtSy@@DSZW;60NSM_uKIMjJLNGIb<~mPl#WL8uybhcU7o~DtB4pEom(Mj7
zZ#|RVS5%wE8NfIWV8sfBTfX#)T3W1o;Js-H&}3!NrZIE*<(gT5P6nbQnba+RD`1A*
z(LH5X{P~4xq1kf2BnwZqVY9Ldedr5%KJ+ezb<Usg9kgrp`XOj@Uk%3^dd9U@rXM=a
zTu2ybL7THe6fcNpg)A2bb9!9PD;!+o_~<~PsV{J2K$rRGLneq{_hSY0R7-|xW2NZ&
z<(!j-s?;I=jG{zjLQ9V)rHJG>@ama~Rro$5;#JksXZ}Zk)lGZm|IT^^<mRKQKFnXM
zMv2oq-<}pkeNZZ#8wd*X6I9RSyGSjYnF=Di&kCx%?(4|3R2HtCqjZ@equt<{N?|L(
zr6t*XJ6xZX46j4{pU?P}O?oWm$e5-MvNMuByJk;bv>b_sFmZl3jlh2QlsyM<jZJsq
z7xr3*5<s{<kM-c?RxH`-mku@b+-1SU=>sR1?rpu~MlT?rIfQRaVCJKLsaNy78J|9C
zPwdX~BTn_?SJT5odqY3TiNfin>tIO+$B8vFdb5i0I*cq=-ZB*8G}(@ofnGXgT9uaD
z*cEcHSeXa70dlkET~c7|Zv_)z<Bq0n9%}_aWLgucJe|9~cAdb5>1yzg?AG-4(}e~v
zKfyjoSngsfPbN@fay;e3PIr}NHI=KcC=agThF)Dpiaqh4>*vqKO!$8}kcS9siatko
z?QRn{L4cky9}3<-MLOEUR~`W-9xoICMTNF;Ayt_X1gvEpkFs|`BVB1UDOAH$I@2cL
zg5TR3HxHg%f2gdSyj|f3;oE&iscsq<ysW`EQMjifb$RY1U96{ipxt?UKW1&#KW*uv
z6`8?P?(1M42VOpbz|Gx@r`YB?TA|F9!?%j^8BX6@UIjKPc=;0JX|f-8mP>ahFZ-DU
ziQwObAH$E@295SRx(+iUho2;cp!W>>cLsza`h%<J5yKAh^tfJpsP9IS8_=D1K#8c1
ztY(@mRLrelJ!#7trR{vs`F4?ZOJz@lH#xY9ON2@jb+e9o4#aM<0l9yVM?V*SI1D3!
zl|$}#gR|A@ek~X1NXF4!Wfka{8SIDBlVsjLcj*#)Bl3QvaOE5(h&y+^v~fnSw%{Z@
z8V|D`jNT|pl-BQjp*ES<*sJC$jpA;AFYlW4Us|!wV(X|<E@%efSI{N>LE}5Hm=W^A
zXJ1WsD<%zB)##6U>}kpo-Z>{-zFx^5xW-6|;ftJhS9;e;d?Q*e*X`c{iO?O!MnTEh
zjqf3ljbXmlMtd&4lRHy+$Wxv^Dp{U6xY#04O+tG|(EVF4!$aW0b(9@bFKx1d`+E4A
zWd#q+O)@^;*7*tNG|@A@8ze)2qw>7&b%eg^c23VPo~lQN<$3Y`b9tSPbjZHrFX$M?
zg*Wl1rMcw*DVaEDIhJh$<$23DI8h+(|Hx2Yc!ir!c@42~O4LyviqpZ`4?cJxq8Ss~
z=K@#9L<}w9EtIVJ)AUT{LM$+9s_!rJ+8mk#8R6}BD_?Y?Wn!&X%AgzTlEi$V3HC}<
zW<ci(i7&svsY7b7oUG00g#kB#3(G+vbng0L=iCf9K4eJg2Jx%d&=dLHKd1?JV}O%0
zs@ta8&eYtfo13h(X>Zt)DobfTtSMIN>o{>DVxaWs>32uR#xADp-aB<dg>qt*-XJ!9
zEoJ@Ouj9v$Jr=1y9+X~h_HGZaV_AAV3^iRec{4L#eAyaI^mZxAJl80TDLYy$zU^)f
zaEbUbM1Ak_yPG#on<Xw?4X$~>GkhxPX3?E**S>KIkh>lnp4N8Ov!2e~o?Ch-<g7|U
zu>8;$a&G<UtbVO2E4NOobSNrU;j6rj+5w`uM4#4dplv>_2=f^}0;_pPZjOxfzfOdN
z@DJ7FxzEY_?hSQ(ecj(jJv1qeS0AQ3YH#z**&*RJ3+lgB&sWw~uYn5D@T3U8wa47o
z_f4wpoKwb{oS441^-4%FJCL;DLQnS(9j=-EU6yP)b|72eg>Fvg7e4Q6rb`0Z0$b!e
zp0q?`I(RM94|$heoOVn(&s(}L1t-1@Gb!`Y{rJjT0wvJ(oK5*lyjPU4)jg|UV&;%a
zYM2I3PAFT%?4w@%NkOdfT{w>1h&9;OoFtf0k>k;ZqYpnYCf~qcNiRt*$tv;6yeb_f
zcoux>I!x0hQ1{FLw~ub)QdwiHJ$>K;P+jmm=yldk#FqQv8czQEK&>T({!`qdr-!jF
z$6Zxrhbi<GA3ZJGDVI9F`k{RjCv?LQIwbcpP193~SN%DzUHfj6IF)=x9_d%!OwQR1
zE|mry9AN5}TV-B#IWi1Q5PC`_<g5KsYA34N{FI|2DGqxE6ej8Gs&|?`0&l6lBzj(T
zC*lEh;t*CQddh_%kSh+P4Xave!CQ}UKtr7(UA0gnyjoh7CfEn|JExkme`Ff&r8jHr
zvxHRDqP^Ww?;q9^>ui=YBvo1L@hT<YfQ{M%HKPaqFZY9V__l13$jO|R){@?0|7vfl
z@FQ^T68bUS;CWlhLFl`j)+e=Xks?Yv)06gfjzsB2vHh!NG*sMh>Ywl)HX=%^s_wVG
zpmUr{e7wh_{YyRLbB5ZQg$G=1EZ!W<a$vgcIq~zU<mp`VOc1j2>2d2E5Ptkc`{b<;
zg5)-ZTd5KO2jcOG1A*IKf$FK%%zGhwrJ5qWaW6{F3#JTCk=Zzu=S{I=dpoBs6Mj1Q
zEzU0l7goJ%5cl81<HezEHfpex{Jzl4C3VCE_h~Rh4>m9*2p`b&46L_RWni8wRquHO
zz6R9bG`*2#KkeZCLoIl+1s+&^1#T93-tvyAN1D>!mt?61`;2pGE2^By$w(CL%xi6Y
z^$Blkp4sW4l#6Rl)=`tyLcRfx_&aqAaIZ{ntKGeHyfmVNxAru;0pj4=;S`nHbzrEU
zb9^AluU*Ji9r}Rw6>@x-n*did*^zDD4V>RPKl3#71nk$b?d;fr%hTI#9di3+5P709
z@&f3THW{;Bc}2v^Hr_*NfEh+&Hl%Iio~BZ4$)}&IRqgzj2{e3u($w+paui!5lK*1Z
zv8m~4HR0t)g)tp6cY(8EN0+Z??g#xc|7*zSSuxnJXA3J12yJ;D`P2UdMkn&qmjHD)
z3t~T$@Mb{{&rjzC9931@iFgfMj~D-4y`PVJTUtasyxzOg`e+<`_;szzyB-L+rHf)<
zWAPRu)26d{ka)o@6y_=-6l1q}_Br<PSKZB@-?egEqp+JikJ<UlSJykJm*(vEhEc3_
zW}^7qmVg`J6nvLqX=a^}79JANDSCzdY}+EPCBC~ny2UsrXY(ZU1GS4#;NYj#_f4=c
zV0A{<fj}`d3n?RU!`3_89MWFS2Y6weR&VBQb=y8(orm+NVv#zr-A3a}?nY?eJNYku
z|FT()v@lk&f6Er9?7y_C$p!xYWO%e8soS+s<IZ$<+?4G2VdmqHyQ$0Y>Qj^Rl@ZHZ
z6~?>kf;#M=Aw=UN`x}4_xn-j6o;Yx7Q0u!Dh>X)ldKS<QGljCd>ePjGZX`K38hiVt
z4PSa2EKRK&Ke^fSUu<u*;4fP)9s0hV&OJX0noJFYU#=PJ`!K2>uUT!V^$qyVErH%c
zz=1;a)xXT_^V{7W)OV8MtS?ZTZ!g~#RhLRX31PZ`5_@0Ff2)j~O_AXmPbbnwu13tF
zftfxyEu{P_@bmkgcRZp38%*kHxZyVJWyA@UT?;cg)i50uOzq)Q2}ufZv#6)O%vWfI
zXdx3DJ@4JRxj9etKSwb^q)A>>NnFt?;HZ@`qg$^CfLc+_nW?1qN8mD(_ci0_hVa{f
z<=XPO{vT^(@CZCpYwPdzZRt+XP}gQ7)#MjHZk9a>yAZ!=<>HlLOXfOr8s`LT5&4OJ
zl}!GYN1a#IfIqn;@{gIN%<Ch+(5jrZKC1=LhwFAv3m5T>J3_2NJfxogi`<WX$ep9Y
z3Xf=*p1uMac$sImDp`Y(P<2Rin{l<70daMWJ_%B;{2>a~KNjJ*I1IG@>Udt|!^A@Y
zAER#!)*RNav`&6m#^*NbWkmD00~&8f1w)D*r$BxOG{OvOrrUYY=#iuqwxQyNtxj4*
zDMk0+r2G*$_)-_YZ4vZVhP_~)B5Nq~#)y`qt<d-?CnnuYHhq+rB5!G#i-2Vz_xDe3
z3d!n6{cz^_mrbhrFS~Hy^3O-p!v737b+P4h#E(Mq{})M=M*LjMQiTs}wjzw>X?y`_
z!S27=w^u(`@&BV!ID3>sTo@kR*vi6>hFK-8=qNdG^47d0%jwPeZA>D|JJXw>>5$Z3
zRkQO7fW&^Xnkf&qTCsikGS)AJE*Nnu#Ty8<1)BHS!K}%->QOt5j@0loo37%YtJ(&=
zwPdFE4CRO0ps^WMQm`Pg7a@rGOQBz$Kl}9nVDEpj9xmABF4si!na`4fAZ{zF!P!_l
zj7Q-KfOdYA!i3edlV0r=Iox&fjh>p45IW22Gf-=^m1RW}V11EQs47S6$<a6ZctVnU
zY2zc{-^XJ<{^X5rzh#PiMuojUt;y+RPBe^#r(QJHou0r@YM4_4TB+~t$van)C##10
zK=L-u@jbgUGWLVjkJdaH_$d1HoX}kJ-K(+9gD3k6)Z+DMe!L|*4r9B0b$Abv(HnVz
z7w7)+8;AePZ|wc&b`_XsEfDd2Yj913$Mh1sX2Xv)E}xk;dUW_c%5$5F7Sd~@Z!|@Q
zUrYMqyxyJv=3w-(hYdR!-4=O%VcKcCUrrO(mh)lY77q)g%t5<P!D24}Ek&X9&!UK@
z4MJ$2TNFY6+~iWXe#<uZva1O?m4yvGEf0!8OcL~xkAfg4U|M^^B>v`rF6?^l^8_q<
z#^AkbSflapu-?}c-rixt)Ftc76&YrB*x_I*U{CVA*=3RgPoG>@x%PtzQ@oJf)oW^s
z4ET_M0xc|N5l+jip}Op24{t2wdxYF}w!mN2wt>}T%Zm&p)~rtL9(Hr!d<vL&LacfZ
zmx4UDpL3jA*9js1UOoXALNr$u-JD*kiuO2TIl+++v9S-XYsq0gU=staxcx`>Uubsc
z+_AI>W8=Fh@Nt_37tIQp^uU>ZtMSgT(aD9N&o4B8V|mEWOwKDgnWDVQZ!ki6+Yf_;
ziBqkK?|_|^wB!SSIYo6@{`NNU17&XS>k~Y*%K^Ymy_(fOs<(;TH0E@nd>D7hg}kn<
z9qOr_dvh8yU;dKNsg1?2cJcTe>$$%V6tZl?>ZvC#l#Mvb8_s738gqRJOw|awjalfe
zUMYq<hvH3udM9^eZbYwn1oY=|XM^^XFs9sVo@*gVTirgEKemg}bkHEH7kWd0eXvLT
zF<6z<VKg~>H$96}=0!cs#$imv{Jc%|G+*WoXr4Spa`WF>uz<+rdGaH+a06*0DK&GC
z&O_nX9@;JL4#_sU7-s!$fioKSI83od;qJ6k21Omg)_&g+k-osq!pqf+Xd>7R@>8Z;
z%NTf>xd*70zLll$#2Pmd_&UU@pX;f}5|I9#y_b)LQN4f&@!D}_YYNcXa5>;?>8kj-
z$ZJ~2_B6Amlz<xeQ~WbU&8W~3``@+R(5D+yJPX`^xpkb?dHY9uoz~VtW!6Y)2FmZ!
zC){u6wXI(nuDNhr*CV&R&mQ_2WqfS00h;3lYtND!C|y?Hg|OGht&P4yZY<ca))JH_
zH#i&bYgFaa+ZaxGtbA{12Yhxf+zk<QN^>JFgplk+&1#)c6pyh+(=<F3k2v#&NqjnM
zIK9*{)K0bc=@5Jj*0BnDqbY1i(_P&jR&rc*SnsRxG%V*}o60|wC|?1_?j*NOXGNdp
zW+7vib=#I6gCpvidQB0;#pU`VVOmIv!?WR>X7<b_Y!U^|%s=MAa<B=DO&(#FYsUlj
zb+LL8gpy%m>raoxd_1Ddi)g8KW8QwICCynSC;})gm?>f!0$?8YeWf2YIDaV<483{p
z*nL+ce#Dfdy$csc%^P8??gRNHFW=Bh8iL=bWWJdJP1hv%>>iERTxZHRF;U!`1dg8b
zVBRBe_Q;CJ`Jkt8mE+qtmq2&=I-J>WJbm~vSkj7{%P~K&iFxdqofic_JxY?)0gr#+
zrLO0Xz~96^;)91oKZ{z>^Cr)0A?4?qnD7|LHr2lM5o2K~h*b^TCUn}C&1e{L-%Yoe
z+}pZ@;!b-~NbsuPCU_BdP_4uH+~mHkXGH##!$<Ya7RF~GfBAbiT`i<$>|x4CQu~Em
z)=WY0Ap72E$g9v*EdFMe;K*8dFrCLMZIo5)vA$%3r>54miMA4*Nsw<la#1u&7zZ6(
z%)D7~+Slz80!j&RAfpc5S)>GehQz3E2+M4|7B}^bU9Q^7>PcTj#|DfY^4498JM!UV
zcGtK3?j<A6odvtJ!SedX4ERMl{@SjrsuKZODZC|M^Jd&oB1fA~`v>>e)$#~jK#!S%
z6b3}RS>Jz^yNK@>gdEuGpGn|;T@MtPI+}Ry?M$Jm)3|1a#!6Z8=<_o=%*;wfyK41~
z<?ard#WF5YokdW1d#Yd=e`ZYR8=&wxkHA*REvf@?dG6zm$_U~l#5S67OMR!PEOEwB
z;r5ju?NY1$^iurVyxu6a)pw9^@JvqfML+eBt5m`5exOlun|suDL(>fcBmVx=#XX;Q
zBD74yR2YwJ9b$AH)kDN4y)xIMLuRYOs?=88IWRivw_YY@KWO~*e@xG9FIG~d&UW5j
zj^0VX7H&q8NE+{qlVy_Oc=8%4em2-GzaR&Q5tamt-!X;%;q*#<O_5>i#U9*QNMN>L
zZMB&E`eb!c=#c0|tE?K!mwy=lQSP9!ebShE2$)$Wxy^866}vVu>2*asI&&j3nCVg#
zzcy>(KP>@hd({SohHtD&{=~7n$XcvP9+R(;Y^Pxk6T~%b*ClM*2JFdQ^WU<#W0Ha2
zwsVBWR|2U#W}&`)m0kzD37h8Wc?WehA~JDmmznjDiZC8Vikn)Vj)Ft7WdxbZxs|WN
z%`cADV}()&igM;zMr@rhjo6pVSWKU#M9SW<%tHNAN9SW(e{}4bkHG>5_97Pewr%k?
z_`?WlOZ)kV(Hky38+RO*|JL_w0ZQ}O&Wxfzu<qfUK*}?-k`jGYs)nifPms}r!@x6s
z4r<r+{{_5i{`PZ_3fP!`Kq2Z(BzQ~yuAjrQKk0}cXZT;F(2th;FZ$pIrT8xr?f*`s
zy-CVR)I=~XbFG1hy&jMzVIXl>dvR<Zd3WbwW}QW{r-Esgw5L@;Qr^YGt(K?za#`2f
z9_PIyZnk>*NAx+-WmO2Pxt|PG1o|1;SQEn&75AO$fp9*zB+hK|3KYCW9Q%&;u@Fz~
zl@?odME>=qO3GHZB2RsIh|a@6?iYu6c5lbyyw^VuI(#m=8oUbcs>fEuwRc+B8ICV~
zaaVOVOxui&<>!>G&j%Ng!Z8oMNA9QiVdR$0E^-IsXERdV#Q)`ak>n~n{u&q$33OH<
z$oP_?Wiq`*?2UM<Wj*WAZ}Pc)j7=ORzgowxt)IB1J{`(Jd8P9H>%-0^e}d=92rD%7
zROW<Vw@NeEyW(MFS5raRiQu23Le|VOQC+h;%qCS+cN^*ae3$J}kT?spF0P4niP3nL
znS;i9g@ViRuN>xS<6aU?qV|&6KVO*0i_hmM*KZnEdL{589+sb<>vhWd;MQRI7QN7x
zDYRmj#JFm$T;E7J7GJF>0Gh-pcZ*ed00I=-{bRhhFY7|yZ=vI0;XEfTWXm<<CuB6P
z;#4um69}9iZ=ekFj)cU?<8jB4HI?$Mh8OW9SbpWtUHpF--<<-Wmm~cO0(Xr6^wFQG
z6ITa!=S>HUme#myAz3#J?9+G=chloBiZ((=dVxXxlNtH2r!vN@1P;o(c%&L8%7g(2
z!)qa5e;|sEI-6QhKFAOEF`C0uzswBDJaPCe>%)ASdN#gppBsZun47o|3{VvxShCD`
z`zG%nJ<op#J@3xxoHP%9+pakvLRh`2Lif3^oe~Fb=nH6lboJ9Do4+?={yn-#LrFmQ
zPxj#-M9{>%w*lZ$=1A_U3oXWGdN#{*OJl|7bVaD@2<Sl`?c4-At5c617L1J0i+QQ}
z(b&9_E-%b)dbX24iAOgH{E2%K<lR97RSy%PMf$YRKlEy<UjA!eI!WbR>EmckR~Rz+
zN1lVm7%4G&l{~vU3x)1I=Ziq!wz&OY9^f^La*8CBp|1QREc;Vt5-9ONj75UaIrgvi
zCzJAULrGhuz0C}=yT&6Vd8hoYC$NSqUnf4qVKhLBzI#CI3YjuA3jthd=chC1IIKLA
zXa{sI{t3<ak8t=eviJWBq{N^3iytHU|NE!V#2DiqEJ%v#Or8g%?xh4_cBozKj#INn
z8F|aGe?`@Zp2k8pj269z60k+riDns5Q}Rjfq_vaG`rm#GrvI~AELiWUVfm3fK|VJ^
zS5mK!m?YmLL%#uy^9p&Oc=}^ACR4TyU_nP?r?i%WAJa^z87TurVDnS#p~Y~;UZq?-
z;Y3|^=r?(T5clbNvwwMsiV#L(XALa*5Vqk(a5Q$tY!~vd1Noe}S#X6!gEyhs8ejwZ
z#bj@4o;B|VoW{ISs9FnXFeykTBh=Ht|I6_k;D56a@Ny!c-j}SQO5KoOUsvKhkUK6n
zq5~BqPlE>HB=JM`c`_X5cs2W%3zzmCRJpkq=dez*ajyUUUm(YA{)xHec_XH*2~`w%
zfI8H9>Fw#(T%Icb-0bA$svadxru!&Ke#y@*{nE@>W^|}wAOTz|sQ;3Yd*-yjzrdRo
zZ1=JevmVP*aV{L;YGs1h)1~>>>l<a=@$Zd8R;rTw3-uRErz!NuCm$Kw561SL-}Wz0
z`gj|riLAfV9FlH4eW2RURqCUfBGVVAe7{od;NQZ{ilWL$3HjDXU}fv^jDO1uL>@VB
z))Wc;AYv-W3;glQw?7>B>YREeJEcQr@}7;*t4>$NV?YMe&N$t|fEEwrcDBTSr`Y%0
z23Q3_xU<%C-j{b7BI&&6v_)dHknu&KB(BUZ#FV9Epqmuzt*0TWv~v5eo6i9rH^?e?
zdx<$?08rjyyx)n<vw!m25f@7UZ&Lc~dom<N-3lSEKhCx@z{#3Zz?Fi>hC}X1eH8XG
zH2IL32-hm}dK{y|^d5xgB(DGAPbh{}n%!psT)AaS!iReYw`@td@txHFppz4G7p>xL
zQni`)aGb;t#|xetO(AAKlgj4br?*WmoIleT@J{4W%uC4f`_!{1fL!912TkJ@Kws3B
zQ{VFtLLz4j3a-Zfcb_sFr^n;vcJn}6`G}@N<50UBH9Pec*dJ1GJyy%jz@MPl)eWRr
z{MaunzWiPYWR8FlN<H}swCp@v__t)u#d!L5{H)z&LuHFxzZ$vX$xDX5+huIM3@IO5
z4goZJC&K=FgmH4d;Ity->w2W(VMXs<6q|njEPNJU>8{1_qN4IVKrzGZzj&3LXpK&F
z+k;bk1|(s&B-H)5D1pLFIb~^yQP;f9zx$V`*$E((*p)kCI(%=Jn5gY$4q=M>E?Vh{
zD(6|8(Oqmim}hGj=RX*d<1?dixM4rAN>9grT&2zAA22%=Axj{gXr=Q#`vc^Fe>!>K
z%gxWk_h0?9Ng~ky@vpVtY#zG#3H+XO`|}H+E9GC?W<n^WQl38fHW76Yr<~Snv1f+F
zp6+NFTW!S__5LbTxkxZea3)n3L{D@E+5O59`p&o3jqAx3@HC8^-`#*#oTFGxO%vlk
z*tyAVGs{rrA{N5jh$r2FOp~)EY{(3xp0qQ@<}J~nQM3`ww9tNws#r{l6fMkEQAr#o
zhaWg)GN7D5QcvIp%$TJW60xc06IFCi9c*>)am<~0hGAx<9wn)hM`PK+muGC8H(}KF
zSAS*NROER`yD+X-<8R%*Va+6?-&-UbmZw|=5P4i``|2%&X5elX0I650dTBpoB?bxz
zHvu^&n>TN|VS$1q+c#GpNBwE3gQA@%%RWo@^P~!Cbni+zE4w#_o*_3?Al;D1HQ#Pm
zgjz497V6Xe3#+1w$$kYR9+M|W0BNQ&Z|4LI{U$75=9G6g8nPo=b0zNvUoQ5DaD-pX
zy}JXAr=EhN^L)b^gc7LeQZ1y5&+0qL<!7novC6Q)xJY$fYiM(%h&a{2$k`f*ZKDEL
z1o_^!fF8fty!z8t;xBQ=E~4cHiE;x`StjuABwRHAsm<(I;@!kRP22c8*<vrt)|;=m
z$eq&NM&ofdoR}WA4hWiT%iN>+M9KLhq^UE%8{_SRSE*13Ho?N`D;XovJ<&09Lr<eM
z?=pnbWvIHTmKwIy7xAa?)-MfLz3xsg<x1XAH7qc@aQw!4M*)i+E%Yhk9azD7s?~%V
zR>nhmn|5u+_h7-u@8E4f5V~c{;iuns)ff8dyT?z&T13;DJ>@54tc3uo*P8%jOa<g`
zX3Tz$-(d-joA`slzzE7cZO~SEUQyIBN0XFkc}!Df*8@XDt5=fwT%MsAX}m`@q8?^6
zTSwpPwc(T5!8dTfh{uC$))y-+2V>8cY@19g(&+Dtq8XH`4!S)8vWTGAs=ge@lJyJZ
z<SON6vnu>B{`wd}h5F>j6^$xqMrOuY$5olOJlX@`#BV%&GOY)5F%^z&!55Mtl|6Io
z2l@yH)%D1cS8SH&DX&k5xQ4cQPeBkKM(P<gtAo+0z(~(tZcI1px^z;_;^?V!utDYI
zEBv|3si}&$^ryR}Wns;wr)f8p+(BOAYHUNjO+7!e^}9LOQD6ogyXbzQt|_x9=|iAn
z@{OZAA4X=B2_6(bMN({dG2FKte;Rx?alD8a|D-c-b%v3i1!Q5Rnrv5;@4K(=TbN#`
z|05p>3p@rI*gv6c9be|WY17BffA>GOMLET|#9iBEM%!>kT?#5}q^DI#nZ#qW8%mJ}
zC5?fDb;-e8+?ylU67!z0yRtFR1G)2t<}3UGd4Z)vRSuWOt;Jq&f~->Zh|9^^4%%E^
z^|eZi*$W9MmV&@*A}U{Ri=xj#ZUam4$%hn|Z~qWlB9%UO6m+kV2k3wknfSpmb728G
z+N8!ZJ_T!TACdL#=OQ-^TP@_f7FS+u7XeQ8{?ZO4ResYwI`BH&GtZLIoFl$?D`{?W
zffU8ZZP;nPGZm$uci_xdUF|dm1~A)qph3ksDWKs|BSjK=wq3HoR{k+@Pwso{aXLrN
z>YOrg_V5reu2e989{yFMK;C?XIQ2o(HXR_I&)XZBVVf{>M;rNpnUj!cxSM=__G;`O
zTV%&~rHh(3UKuS2kT<v7nTAe2kE2wSiqkptcb@vbl~M<V4=P&5v#XkNo&_22yAD@o
z8Q&CK+w|&92ef{xmxN=zkH=aJp3t-IbvrI0V~ci~-$B#v1u~YkeigEX!^G~Fx!*zu
z8z}sf*O(QCrX^M{gUW=26Sftyz%x|e%h_E`*|Q_P*S-X~eZ>k*T|D>a4*uKleEogI
z(#v0$dKbFKL}nHbGW4;nN2^zS{GRIK+DX7mshQx$VYu)C5~EJ>i#j6cacnN7RC%4y
zqqNym9W5MEeJVWj^P-{lplQp;YLIg`>qXLY|7PO$M_|1e<5-KNPaKwmFi?=SrDD%_
zbU^tWEp+Ne-tztGs7lv+xf%NxU<32c_Zp3-`v`g4pGWrVRcRs*YPIO+W8g|JaHj(`
zX+FXE4}7_Ezdr_gS)YvxVtxt^k?uSBN+PxKazUF4474k&=D>QJtPwgv$y;{&KVCR_
z)Bli>*~y+f^*i5QnPzr09a5gMgtQw|RsKWKD#MjqHG+-g$??Iw>Z+uIZ{ewDZ($%!
zw~ZdovwoA$R+gv*Y}7Ps78;OfHrY0k!h}3qJq)4~?bHpZL2SvC(tVrPbr3K8<$Qg>
z==!8zR7u8aj(zs_sMCcpt+@wyhQ-{x13N<!?j%#X-npC^pXvskH;R(TlN@Nh2)~~T
zX?pK5(oHEM`p*tNM5r1p0*UlJgCY2nj{)PC;P|L(T~)Rzr-eSbJS5=cqtXamGaY|m
zw~iINV@>T34CQ_FJH%;wT4`)oIT-utjv<^JeenF_=XZ4OmfH*q+LuH7@~nFwK92U8
zNnxF66jjX4&F=it=2rL7zS)%G;M)9&%jb6Hiq7xh-ZGdPk?neq4NWZi0N>afdrM=@
z%Ih=U55$y6lFTZ54VQ08U4|oNUI-q`r!Gph0J{@tgQ?PY%FGm$KZCt7pp--O+?UgM
zyER~N*fHj%wa&93we#L5S_l{%(`;8RWX?dv)hxSf4E?&#I<dJp*YDZjOvKXd#UHIu
zhJXMFf_>a&_VEwokoV@u>zLL*(9w&=2%X4}V)_UnefYVHA<^#P-F>2Yk)S7{7XS>k
zB8LLf=@#?3Ns>>hNYz$ztq!LxA(QLtsb^w<)pdT%=YzJ^NmEi}`#1~|Z^+OfVXa>q
z%<dPVv2vWP$?q1A!=BkYS`%?C=YQN*fC_AW7GD1aCN^t%fKTk<zfAOB?YF<n<<H;w
z-+;b0$w=UH$^N@){rS!RKcv55)c>beDb02I%4X#Cp8)KyfBg6Jt>4c5qbBt~()oX3
zaDVdr|5KR$e|5b|;<;td4(vMU7V0dOQP<|KBV%h5j}TD!QztU2@XCf}z*W8Xm`KNg
zqJz1?$S!VyOQA^UhP-|Vacwr$pjSHh%3tR5VWHC<9zOsYrSIN**Nc(*=k?ZOYe14W
zQt&*znN;>rQRA$pHRTvk2rxI=BT>;C>!|>_{s>$FlxJ>MzyeP&<M9JZ0an$*G5jsS
zv>sT^=gP_Upbt=X4QtuIw@sMRdooo=WqO2Z&nymXZ~ikO;NMqs)-rLSHGdY|Aa!4g
z!~D><{w$C_SOUamqc1-IjoZ7}p`r`}_xGnJNmSe+qj1GJ#uJ<44L1hM0WAHxgsEjH
z6P(t!3{Vd(@MB&t%R&?qzD~zs4m}*f&1hs*impz3mxl>sMp^QnNAEY5)Z?bG;xD|_
z;m5lYs>VaFZ34>n-pWWYhg39;EfE0aeaAU<H_oR<;MA?;p710q)31gClRDbrW~dXD
z`Ztc~(2$|oDCoR`3DTve^L2@c^b9-q@JZES$0mSF{VPWSOHZ6Z(^*o#gVRWT_Q^d!
zmTc6`kQ;QMIq?t{GB#<t$l+QUy}fL5tlWk~5G|px-*)2~Zu3dqNgg$`ygQ10JpWGe
zF}RCRBC3d^S(GT{zV9Ek5dC;31;{2Q*ZZXx>YFgP4^v84w(b=UsgQt#)Xah&aeRR~
zw71GO<Er$EUumFtZ_P6_gI6nZVKpQ8C2k4p{gV!=+4qpMMT}BAJN-5PdX<9~yZu-Y
zb~Mc2cFO9y!<-Sk^3<fe^T@k{4sQCYD~PfAoTTmu64m#R(Ql`r`YvNtruQ!u$Iz1B
z<mi!t%eoOYKJ7>SVHL8i>1{Mwzyol%nx8zP_C|G8Z0*EM#&Us+)|rMH%*|m1o+$KE
zHf!KBQ)p_BIo!Xf?eHV=9Zl9+X*fN@-vYQ9thN#qYq3ux>f?ulHq#k{9NS3RjSG#C
zKcbmvS%KY&JaWACjbDCgy>@@rmgI=*^KUtzNv|nAQQY{=u}$1<l(a0rnFlVQ?IMgy
zBi>w6vrK3uC^VMz;zaN^yZg=s593VtJ_1|bva1@s@quw?Z)^Ke9CM6Mv(H4?x05@D
zgJ#K8+-<l9Gho&)=g)??<>#fa{aQb4iw?B*+LmZe?qKx3sO5^-C-*ka@zm+Rb(jwD
zM~eLRSO*!yw4PN^p5;^vf*SNXcSL!r=(E)_g_`F!<8wAicSyv~$?;sQS!XJ0Fg~jK
z@|M3m+2=iI9%n{Mgsu9edlARr{uw(|rcO6SvL0%9AG(2*?3p`isk6zFe8_OSiR#Vm
z*&P}UVPI8UOWOl$BnvMnc#gGP69jp}ybc}1YuI5``U-t5e+>!omE=uY_IStVHdAM@
z>N|-ev;l>)8}=9>_@}q(0_VRi><Nk>rH}h2TXOh&7c`!zt72{WYL=dm?>H9kM_3)%
zc(`w-`Ud^b*<B6Kx|dIX#FP#KX10y|^@hxoM#(z%tx2Hux=NNpQ#+Y3guj+FNcSAr
zSir=lV9snZnvMnyHs+tRcE?yD&)rveJTxYUyc4K1%J_I~yQQ05OhE4bQ0A$c{h_0~
z9GI7;VF!C}=wT!>u3W4gicmhcM=4p<c%49r8SeYQDDhSKHAnVcNzNa5Ee*e*zqStM
zw;@;LTzKA<c7^6I_tx!zA!+s123wN6doYW}`>wNE(0$nzV^wfm)w%a;8@Sv#JK3?P
zX1A<0rDKKG6{^_I3?xezKO&xNO=V?VyPMa5=<m4^UEhb~=+xsX`~<oVFX<AN@17V}
zb54r-jy=yuw1ivbnZ-2=R{2b4-ta@q8jaT(eX~&Yrw723auq&uKiWo~IewM8=v@Fm
zn*_6p$(cXLz0(~gqbdNR(y9ojs%q}F_wNCgnJ42Z5p&yWG&ynp!c6!VfUV8qrV5Fy
zh>rNBD%f&r-b@BZ&jlUI3)S@BMY?7)6O=@H(M#Q#Z~9vPQILg=%dx1<M(c(u)0wx-
z!5aBA`>B(8W6BVcHgaoaq4<s+(aaMF)nod39-^a<8xcXL=j<<xy(NqS;dGBxpW}el
zX@hKxM3wZ41b%&C;l|Crk7H+_@|x3tIMz;joGCx7F&<#;MTMsvl4vmD;G$2@#F#RB
z?5uXxm`d-+&&OtSTJ8qtQpSO{Kc2IXNC<f0)|^P3_|6X$>TXKu8C3G(W*2b<@7Kk}
zc?Z~f&LY)$<KFc+*JAPD%rCUg?K@&xX~O-|{Cj!so}qkaD4cc-Y%P6tP^4{HHPm#k
zy&3<4sIJSkWqyyAMG1sKK=X5<V-+orAtDUb{__s6<J%N3xG-LNN$)+zu=jt(SWd5X
z4b>5u+!=s#so<~fRcpW9n?o`d%Ij7)msPvznLf(MwDY@hCIZ;V`Svi6c>hS4PdNNo
z-V?BPZ)c{7rkwa4q0XJj>%^CKtxV2IoS{}xZSalW3%HgX;j*|j9I76to7IW&yZBNV
zocc~|vMTlPm(gMcV=gY*L+_Yxo8(gF&`ZOL^*Q>MAmh=DLnCy2wxw3nz4}_0Yj3(V
zf%fXKW=_b+daJ?P)IFIkw;sOv*bY$2#LOeNwUINbhmW4)`;K{2FHhC}57yp19_sat
zA6A_<g;pgYAxnme$Tr$+36&)4q(T#8U$P8ssAOM8w#iNjS;jsSl4Kc%Fvb|!24hAH
zGZ<r@&veeI&i8x1&+qj-uixLNGw#oQU-x}q_jSD&dFZyFeUVceuKSl^4v2inIlQ)X
z(NStCV&VDo&d;8hZ7GTuw{PtXBPs>sis#e3&blc2ndkTBth|<Jfi`rv0<e)|A|?F6
z{AKA5!cwZNn#<3;*|f7<ZFRT#%O{<R#9Qq(Je?EWHB~{*8{Cd)s=w5&Js@bRD>4xG
zZng6<awEeQ3_sWYH2@POAlmvVGhOpWVnP@Vf5-f;O$Y0N|HL+W4eLdZ`5l|BfUr9J
z;eq5*ky<?$WyqlK)xCn&R=k*;71>o=tHjB%3wWEi2KZ{ls9}np;@11zpeodm!&W!G
zu=O|zV_zm&2dK{P?C;&XB~qid^gZfmM97<XavIB+3D3oy&jW;_voLC{8T}tf5-eYt
zvRD1LSqvIztIGIQ(pNhL{@;$%OKRV*F<zz(pAO$UXe_P%x(yuHd1>|hKBe#exL|47
z>mE|t>z5gGs@XV==wf|1yZb{4{a(EJ+Of925;CBveq&FiMX!(KpqCnotEnC$<da*c
z=2N}V5tkQg{M@lxqkVqnI#G3^O$$0BwP~w@AzWwJm7cyhxCCH6H}3@|gPCQht<Fp_
zSO>eG9`3KN6fV6@i^|G4TmL5OTw}k;j(ugA@Z6Bi>(5q?dYX)`oIOPJc4>W4jVKMk
zqo9y}>skAap2`8wtBYO^>YA_3$3>Rgo+jMf)M0tyP@56{W`E4nZ{^k*rZ}>q#SQlD
z5w4#4b<3^*WvQ=w%eB<`A_RaEQBNRvky$Pn<9Gi1_;2_A>zA9#(sZ<L38?^T>K&BS
zNAWU%n!2~HjaQ<%!42(`dr$F?&Ni$}*Rf0aXKEU{H<5GGXbo?YgfB{4dz3&1J(HH&
zPy0ic0YOcksx!GIDgdOq)O2#7(AShXefkO@r~ZB}VjOfU3TcYXWww{GBc`@2dp=Sg
zV{WOIn`63)F7du6rZ9cyqOL`cpaO0WEEYR4U+mb_V25Moyk)5cG0v<7&M>7NKicm5
zm~GPtA3#+Fq7*OHNJrOy39XkTlowO-Yb0v@Sy1K2t~Y^Armj;F+GDtcZ;Rj`3d67Q
zeLkQ*!2u)9kO!9odh1`_TCRgixbz&RwSEFRgK#4|WN-}Pd*08d<*Rc-TOAm$<*^55
zhU@0+JDe2hC&tr_;Zf4?=cn3@uE)(u*gwwd?P@(UeQ|3PEl0Y&KC`sH`Q-P*4qB#|
zomHVQpbGOaJOO?T{^>nF-heMH^xar=W3CDR@H^k|s2uZ@1(^vSG4mxhZ9j#i&MrF9
zK{-ThDF}uiZ7pWW1xHfr1E{Sr7R_t6H%I34i)<nBUM6{CMg`LW-fxL)_rZQ|<*mdR
z_=JXveZ<D}UUdIrM{7p!2s_JETVhhwij9n#s@U0y1Y8ID0#uiopRG%S+j-`v*#5pE
za!dsU@83@p3rMnn<(?^_Q#s#1$gNA6DR#f7rFv&=ZE_g)`6LT&`L?h)LS8&Ho)_8p
zM1wD2RZqk|Q2Z3r=8`|0VQ9fQM~pelUx)fWV(NNo*7`sVOX}E~pY(i?$x8azho^1^
zlmbU`1z1`8X{I3%K`^z;wJrOA9kjW%&fJl?e&~SJXLZFVI^g3hK%141M7QA^Ii-_f
zXz+#-5bGTlb(|GMF!WL*N#y$4P-VgocNET<tsia)XSU4glJaJ^C4zml4|;$6tb=3i
z3n@OScKIYJ%qUoSiK5P12CnyxAdzG{dsTQn&7jSU04p(>P^3T#pH7CUvvPF$l|ep4
zqu|?f7wH0_hJF}qlpr7=TRepePN--|IV*w=HdpCT7ZnrbXUroDlcW6ymfyDiVhz1I
zXOUr=@2SXZvUrNWg5a<94JhN>$SK@lI(uaH-dnp+cSKIjKs;al$7Udze|s9L)_`T)
z_WFKZ#@ycCM)65^>{m;v8awg_sm-O>MoNfbeg6S11iuu-9~5-ufR;H>Oq$eis4mv+
z*P@Ahe)YXDP=gwnc^I3~_~dIXG3JKd+uplo{q(e4<#zAStV&;w%JZUcFa<u3GH;#E
zYe2QdwCS+eu?U|i1-*D5QrmQ_sCv$U153_K+C6_V&K9h`eneA2F+O`eDDv@`8630s
zeBQK8{{=jn2o$ld?ZBi=`_C}9u~%Saj<(;h?#j5<?P8@0&sXe}?@KYw<Q1B@<+{_y
zY=6k6pThxLP^PA#b(%HAH&m*-o&-nxE2U!JLFAJxFU;QknvdL>huwPB<~H2yeu5ie
zcv~j+df}GcaF@mwh)qHS3j`f{|IHRXN6!j1_9zGOX>B-<@N4Q7TRd$WrMrqJPp`Xl
z7Zne43dPSs-QZZ^-;VZETOFW4V{a{Aw!(bz!!h*{S?lA5-qj!$Bw6S%mO&`J2GZN=
ztv+Xc^~=}XS%wG`9LHuIzeWw{JUc_F-R`)>KFs(HYxT=ybxm8iOS0O7D+hustoSip
zt@N^?(c@+o)fke7l8kzEdktg$J9&!R39GO~`%zWiQ!<Py>1o8_S2uz5;lL=+CaKx{
zRtoPP#f)qmJCzo6X8^94cj3lVHo_ct`|flFVCslkG)!IjXibAou?->$f-p?`JJnJF
z+ewDA6HBYS-_1QpT$ywo?G~(Pu`aWwdU`ktAWV(QuZfZ)-QH*2XoQsn+|KoDwO$2@
zOHK`Sv#TTx1v;J<K>9t$dsX;{UG;Ba!m#_LZr+o!y<P5#zHM>eDQaA1KHWGYZWB~7
zndf+5-P$J!`olx(@SyQ719-KVf)els(R8d*$a$^sF3Q+M=FEDw)Y5A&%O|plHI&)_
z_a~_}ze;K+2#mXCuIfX~%>YaA7GqI2j@iiri8+g{0wABB-!EyK$|`x`l==^3D)k!z
z4c{EccZ8HO>7iMY;<2YC5|BLvUDCZr({N0OJ$cf&xV+>&>bCC2r*p*fjZ*c+L+G?U
z>LIxSHkCQ<py<KT`%HSCdBn}`<Casak`W_Q$~l=xtQ&Dg9VeLzJYyy8WY=CoM%<#@
z;SLHgG4wj^+5Y-VpZI#Vf<7_Epb&FwgI@)^BtW%(_dfg9YSJFy1S(WtJ>#dmViqH!
zULfI@oz+JbnryyT&UwTHMA5g!;Ag&BvvZf&#x*!V6#X7pzozEw_<bs2IwLN5*qa!4
zV;RlReQ>X>YE-vB4yC-YtjO;8%(LjIJej9zLkW`|J@D#$M3zb2CEa_2{rk;-B_scL
ztCVhd;jh?@eM>X9ZHkZeEqP2iy~=?_efzas+&Wm5LqXN&8e5w}%GZxIw)}8VSvmFO
z;h%fx$B6pn_aCt6e;e*ypM0~wEu+{z5v3LSi+aqhER7`Rj2@-<d%rI--%IgVTdZiu
zp<3L2%`f*)S(jb6zH3vg(6@B|wvA#KI?-Z~aWL{{yOCS)dgzq+A{-b>bB(Q0A!VYg
z|CAv?wCPtq!u=Vyj%)5Q-t{#%``<qIu0OnL=>R3%IrFpG^P{*{WFe<eT4a8J;(z*m
z5#j=*O&%7oQ}1V0E4FDqy<G6^HHG~FTP>Tf0evynLVNqEp}%oO>_+RWkDF1m!BEl2
z09<0ieTx^K4t^8K%G&D60>{b#CiA9;^u^_1xGaa-L;CVW8lz;&!-F<Z*Ujcu9)bTX
z|8qRihrvz;r}GWQd<+Mf1e`4-!BR?n<V~S>`(=yBy8#*dIv`dFecVF-^~@)zUvQ&C
zDji(T>o!Cs05?)7mRx!#q11CEVAq+#4IlhgF*MCY&bvj!mCA^yj(t22U1KA6j~THR
zvJ;gRxyTcTtPO%c7x8nFu{y4I1PV0tS{y8s1rXT>MN^wMz3H^vr@0yEGr!>0QNXSR
z3IrFl*H7r`#|v*#&*=Z*wp-a5mpSekTy`fHcn3v@Dt&4?MgZ2~R~&t+B=JSa;nvzO
z>wC$ksG2rr#)(+q+Nfs9rLM<XfeL`YA6H|ZEWNhk$>%~?fg&jPM>9`s=jr*;AODST
zD+EDzLG=##7t~q7=$;q`i0|<XPUTT=gBHBAR5`9_u>VhrjwWJJ7{GIl9IXmzZOVhZ
zD&TntJ<n~+{?3V|=&LwwmW}6jm<$#zOy#~I7+)-1<xP<f+$!pKg<K(YN1X&(&?EP_
zWE!BAnOfPxp|qhJ$FZF0%E{e4AC$$pE%D#D#k#(M-}O}l+pZ-YZ~M|DuIlA~k4E-#
z{qEr>$D8(|bn!|smvMW3xsJ~Oz4yiTG?(&5H9eqRJW>b!JT7=_dm7IzbpY!559X~J
z>}vpUSJ*&e)7Xxa4Lc{9I9k)bzX3{QpSk(o7JwQN<yclM!qeFnJiLO0hqQ{-$N90Z
z@;uC2OzNg$iNJaLtL_ln<0@*v3aTYoR#Jc)ZiBHKE9USki#@Np3K%X6&+=uw<XUH>
ziO9?@D3}@x{|mH)zXIikb*6ZnRg$FF#)s3{EiYYyX<P^TU%@=@-3}BE-i;fQz1>Zu
zd6dRU@769r>=sBWV3jB!JhpS2GruvPJ+??wzlnDNln}4<=J8tPU-+e7zKp;5yG=hq
zOr0sAmtE{)f&OhS7yACsU-CPjSNMMbFOjJb&e9W?bGT5GRVAKVt3SGdzp<Q~X4^h_
z7&6enMh5QG`*TvH;#TzshE17||DdoX;xkE#t+)$H=h9}sTq`&d0<bcw+B63fYm717
zvGbqQfR#HFj5;zsOxVtI0?zGy|4!DnYuC@IF1-$z*Sljt!cNDp)!NPm2`Pe{+hSgs
ziUfB><E+oX-N{~yI0L=p{;UsqrLVE?#XCY|N?vf#>Vt0#WdjROh<*OzZ#76czNeky
zl>dVz@kGHym?t%Ct4UD*as72_R?{#?j8puySCnVuw3b6T`#~LW{{BJ<>|{D){w%I%
zcI5&E^lMiTlTYA&4wLWq{6(f1SybcJ(0}2ZA|HcnY^!UIQnOi_5wBU+c0d$-rw6aX
zAm?!Fki1Pix#6NpgF%`APUg2w@4(espfC2%X-^AEa5^H=Xjs&lf~<5No}LhH|NXa3
z&DEyh>5Qw7>cPMQhu9XJuSjooo?A6{LyVySFs_~dSA20XR3n<~+JxXKK@kjQ9m;eB
zFlkA)+&1Jtn8ASZfm`~hcpR|MWg;to@FW2`2mkg@oV&Y+n`DEkwRk=(0{_oHcJrT~
za5p?so0CWK30}u)aLQP8%sOQ&5rAj(04fmw@)_Y<j8Ib#mKWbXg7Q3^{?9X~9pIcw
z`mJHm@xX#P25tNlx9`MskbT~C(3XE6;`!8iGJnX?)C3|T9kbGP`S}35&DKMJ=Y+_A
z##bP}_I(2_83T6(UKifx((4F9DUv?Urf#MOY<j0aa7YK63<3!)Gx1NJgqP>KJKla{
zAG2SZy8p#N>eI544DS3p=U!a@3elc2<`DE8DHtF(HH;FXo;*Jc93c){f8H7T{9-DZ
zA9An7{YhK*-jqQr|8Y63Tod_Bq<pIbTFVe;{j6fx=k!-^^LpZ6gc<?84=+D%>^j}L
z{sXuL^?GXjtmoke!57;$y9xS0aNk=KR77}2RJl0A-!vd3!vvB`7aMM!Lwn}o*yn8n
z<_0bslmQU`bG!hrqOLIb$h#w{mDdq4Rmd-pY-p%;>$bF_P=jG%GDX!lAmr%z4XIqr
zfi>tqLqSNh`gp?k9e(pmHcH|~ddbE6vP+^Lh}Q>2ndck^JC+39I+m&qKxH>STutPn
zAoHg49`#6uGhH(Dn>GOA%i1qUwFXRch9XZA_`iPpFo4Z`lG-eyB;EnQ9npU(JZSx=
z!o#+-T%_&BxoV6M3Nw+lA<;@ExtD&MaTv^gQz-@<EFZp}{g3M_wIR&#+D=ULR#(2-
zd$#-3%>a!J>@@0t^1(M@m>9|62MJHRpsV<e>yk7~_BWp9qn97oDmG7}-}G>nAONly
z*oY7RBZ7r`xJS>9Ye{cgCrhl<MNpC_9!0@^GBt?EX@IG*s3sh%Ir;IaaW#$a^k)U(
z9t!u2ugs?h6aAcXU(RXItzl`N>=a>*`rB6n<9VaQ_!7&T?KfZJ;R&8TnMK9Wfh*&L
z#&79?n}-a5t6sLR`;o+)Z2E>OR4BRxa0v#PR^K>I*JO1!T;b|Wh;Yh(r*$k;@g;<w
zzFbURpHx1AGfYkk3BW4}YqFPEChb{ggDenTo{D4noUJ^Uw{j0c)sDF>{{SNib$=@=
z+_G7=9l+<T3n{qw?I_~C7J%hXIF-?}_Gb1KR}$koAwIjo%%vp$>xr}>*5o}6r?cac
z$)z05`fA?f)g#(OS1*o1t!o_7QOIB`PtS!YAsEMqUb}(kcZA1+#_9vJ$JLMb*+TUL
z_G~)O-)_$oG*QfL(MpRx79?;?WXVhL(5@-^luSZljNFG4G<5tpguV2HHWpWPK*;S?
zr+VYQq+?HwdwTu}&{;VQ`<k;aXhi{o-diisW3e+-KfrZrD(}UR5pRXMk!%><oF<aA
zJxknc;mNRJ!S`^^$%T9Y;hp%6XcYJO5fvd~d>%h63|O$11Wm11ngkPINum%;HG$}h
z(<Slk$ax5zzeb;B_B1u)+Wcoo)-dZ)Pv!p}h+3eJxt!<bf+2jIqXX{Z-;9*58}%$H
z41+eZdGRk|N~8t-C@>hlfW=s!tgf~ZJSkv(vWdp=9;|ZCINzkmS3HYFSj2laDJnFp
zXoc200)1lM#itvHz!9WuGjz3a%|?hXqi0e1pZGer+TsH_ZU8X;m`bB;Hq)zSKiF0C
zPhpO8yg6=uE>zBMvK&+6oA|t(ADz_;d)XLyth=!u6m<XLY2yvOphi5YQX4WU)#Y50
zOD$v;Ha2&$ThNYBiMd%BEd|muXb!7d%kJUG?V2gl9)jNPz^KqJs6U|Sgn0DxnQ}J?
zz>V`>tNN2m_8ex0GD=(j$%WjlaRca@9htaqa|56XzyUxr!F&5;7Ek`4G#JFf@%I1H
z`k3-salWZ+9P?p4Qc#n49s?%l$13EJo9@Mv*B$bhHM%0(L)*S>NrD1rNYs{R!R5>D
zmF@<oV&;m22MO5i3c2B|;b~5c*x&Oub15#KhljIhaPi#rpXd4K6UV$xJu>omtKZZ6
zI`({C(^f+u$H+gX#Ie45>70$FSWc9T+CRuNj)#}+5W(FF^g=fqhJEDFbI)+d;DU>A
zWr?faG5pnYmit7_u5B9uDl@055X(J`A<}G;-iVTL_DuaJ+(5Lc-iur6RaqATXD%pS
zsB=BK3#q_($_x90l+}Gl*O!#H0=yx9N28UEojNZK>WlCfsy66O38fsEZ?g>A6YyM<
zun>8zbn{5C=a}*{_b2&R^^OjFZv%b6+gtZBJi>=i_=kM!UzA?763tJF3>n(2s+a1q
z>cbC)<TRj(f`i*W=7pmnjCzS>(SHf7*{YXat)apP{QQpb$&KoqhbRb4ppU`)6qn-m
z?UCin(Df5-OJAW1#wltXEGcd+dW)))XmImecp2S;MZ!(=*z)(?*d#!go_+R7PuZ3n
z^h-I9O&$<E#z244{s-v^0S4EFOYTeTLW(l9#_EMd{$8SA%kpCc>2`(91L3d5L+bIh
zD)r-tbxPB#RhRrY!J|0_ywBr7k0bpVJ~&op$eFZUYG#wDFW%BC_{YYAKOjYeV5FC2
zb94EuM79>ei}dx%3g_m^)?x)#u3d5vKeh%t<GFlgUzDpSsp{OMUxaNpvw@MI@}R6u
zau<?+-cVv;IO1Qz+{fB$YFsctVf3<JfSMNU5a3lY7^(r@X9PQ&{pbZ+g&jP6yTGRS
zK@1U87kTYrC1$T88Xsfor^r&3^$ECVihu{pjyr}gG(EN#=7}nGod$Bb@bX;tOM#)b
z?g;hdwo>?{eb_q#0#S9o%{_f9!I6S?BJ~(-8GTu$7ED7sbI>-u1}Drn{M(#Ob+y)C
zPDat?N~Y;Gh@PIz*PvZB8SVB+V&R4tXnv4!?W3WRjY}ndRQe>}y&>41U@&u!?*UoS
zQ~1ZFTMm6sAtFNI=!M8<9@XdGfiWsXRj(S*ItA>3Zw{y|d1s_&7tv(<g<7Xe8@JaE
zM(Gz*OIg^eW%m3r_Ucd&j_4@8{~09!o}M+Ru=;UB{U5n1q{qsHuzMUOV=-k4uut>;
zc|$m~=!6+tUh%~2Lh9|lyu+I25!<-ODz)$SH+$S@6`DUFu>#)F^diS+UvxSQ`B6kl
zS^mQ{mq(hNpe~#2Wt5Oqyl@D(U+2Ib#YUN|q0nwvcJr8~#IvEswGt*Y^=uE{{jnkV
zc;*G%8DW3q>1_8Wt8GIj#9PfR-Qu($)y{1-nnTRqoVM=iLB|M6&g67{6@9ff8~d)U
zdK-=PttTjR*6|W^qHUF>((!MD4_*xpKFVQLR4QaAO>YrbSyZFE`eraRk-dKy7V}*b
zD<}2|^aX!AXt=D1zVD0kx6@dmA(~9+8DA@#`0|IX#wmASXKvOP5qD3R3dr63-GE9!
z>8Ch~edawe_DObm@T}T9Y$>(?hLf;aC`Agf4$YGNQ`zn7MX0i;haBzoc26@pc-Ixe
ziFy?RZG@=EMt1EUQgP>@XSI*wU~$8~MC^pl)3Y?jp$R&C_Yk`&=!PP_>@@7(zN9~R
zL3geVntpYsqa8NB|3fS2i*Ygt?6AM4{GclWd7}>O>T<QyLvQch_c}^8{mMQX<FdK-
z`#D(4D<XQ9+a?8fNH>QbMC`wkx)qM~PAg?ZT^*nn!ES2XX%h@KI9T@uAtYt<jc!AV
zUBbfo6NoX1D)H{x7qSG`sB?H93wDAHI>=ZTPEvSWizQR_=s0FCICxN<a%WJN@8g(6
z>JZ#Fc9yh>&>5gz%6bMnz*cJ?-O}%JFf4Fe8qoSRgYtC#%F#d1PDOgHa_YZoeeXNp
z?ib37>dMkFS6zHHp1*%0JqFpF{w`QF>)k?$&e_Um(56H?iK9cA7Rhy+{p~AtEcNY>
z<jluHRu3XD>De$JL6`52n;t{Sf>$eyMzuVVt~c+Ux^yIFU!BAIM>huTr9ea`RUhL%
zm!qsuyIvDmsa<jIS&$E1aD-U$CFq#Ko$KkfCeuz>t9(rU>jt9nYA{Rm@_no74VMN<
z`mFVxtlF_lHQDL43R#-{Cm3^?Z0ma(*pXJ3sPLP8`Lj+q_JEe)tBVk|9sly3`rt6J
zifqyN>@x?52MVsp|0v;Cd!_(HQCq}F@(P;^UKhe}xTrD@SHETtKxsS;RMJge!Ei_`
zw&Is|n3Au`c!jqyR$yHxTW@n@HcK*3r#TKJ9xKp)m6xP(De11_oil|G)CzAUGyRS=
zzy-!0W%G%YyJ>It5i6&jVmvVjFYfVlJWL%EY^az<$J)nZy@$UymNrBv+&L&bazA`d
zZwyCnpD`c8I;@pt&OM?!_B0aDsNVIg{91yB;bu|2L1!nZPZpP=7G51vO)9Y~7qL;D
z2})?DMwPB&u5Owy3^+6GF?*+Jbs5;IhT1z&+WkJZcF|fV8)<YuaM^(dqN+3I9%Rf9
z7XCK0>p4JDOV~0J`0L$2{@qT3_l4YP3kyIdk9|XR*Nk!PaD3<USBhU_oE771=;rs|
z;%YEJz2Va8oG&1$R{{k`hbp*oneTkaKY?lyMYVM$)gejrgF?~Q<x!iz8f@MsSMWN%
zSuiCuV)%>z!@Fj)d*z2$H*SjNyiTsvcSS0`JEMK+J*RN-^g#b!@|Gixe9wkpFoMq0
z`(vNW6pvR{&4w!YU5J!qc+Q~Yn>JoH=T2JI=>@b&4Q6I7>!re_F<QW&-Z3qtaevF{
zHGTgfzP$^xnE4}3E(C4M)OLML-K>vS({6&V!)E{006(Tyt>R0%P++I^YA0hXZdQ7B
zy_k|xArP^^lt1oP>LrG2k)F4$rM&ji8XA$Mx^`3zVJnDa=@s8~`Pg;zOPu4QA9I)X
zA9HukG!cAI9_3BD*l?9ve@98Gnfvx4zXhtaPxT!^tL$6s50|e33?*Iwy>3tFKmfdH
zTi!effUL|AH$Nq;E?Zfp**Y37ST#$ZIVE3tBEhWca0ti^ue1>MUdIMxX`5Z>)-EVa
zgiVxElE^Rpg*ETKL$cPh_lr?aDURZ7n>H%2EP<<f0G1VUvo^@DaXZnKL&(FVectx8
z&P%()w62oS8M$0S$$<?uma<<~>C6<uiV0W9287_TSx^1|xF4hE3|mAvGF9rtSMvKu
z2v|+}e(6FV?H&sX_)^j?#n9TegG~0}7SH!r^!SeV5}1a;ssg?ck%=;@%U;!6rGSNf
z+AJ$wkl(`ZiGu3lXMbbg`XMSfG8=405@WsUgRewvXY<|g5rJKC>>n$HqjInX@DkKj
zJiDFEg0iYKgH6wIk+((0o7}*^9)F!9tu_TuA)mvG9ru%??CB1dZhDnU82VS&1KgAj
zNk2eE@zD`Ej}lR@9Xv}QK$FE~D%;uplGSkL{E1afhFZnV@lu^)&CN@i&f>~*^y-kt
z@SVGzN9>w5^p|++AM%d=1hW2ASwGPkcMPi;w_tt#5#e;<KpMg((#5gzL~;d*63Ac<
z<?2}R?LM~j%DtL(@5TljND5!z5X@2g&T37|H<vk0q&eK$s?W(Af92i)O4wJdoE_zo
z;y%4PXiq^^6kDW(?pYPZyWPI|ZIDkS6egj8v?{EeJC}~-NLC^Fm`j7OSQOd?gUoaN
z%I@hbo--R>A6W0eQAbM&)AElZSok)&|7@Mbw-@mGCdOWS@>f7uI+z^Ot0)5p3^<&7
z*f;fVLto#2l2$htvY({tEOyEn`IUNOxtVkD-eBc@Pv-8*>DB2mj&GH9n)dO<GD0sL
zRAb|nT%-{y;g#8a&pc<t8L?lz7Du-_+0%R;f@QpQzsp@gcU?!Iwr2xs5L0=j%B-zA
zI_e!RN6gT1iz9CHT?Y7#-7+B<c@;^6L(p5{T9%iS|6oboj#qj^--na#5JbnK3~;Xo
z6s#V!SzZ3>SCO;7ig@(Fp~~-?&*4WStj3LwALb{3XZphM8$xRj*Mb>Zt%C=M-^Je<
z`Lphw{0jEHF^>;QoSoX(X4CKfSe;c{gB*Y%Z=`n6*PHR{4g*eQZINL+nqJbAtz8D}
zAz3b@a=18_<=U?{O?S<ycTsH@a)evgh+5`jXIQvKw4Gf#m07_@u_A&$NB+@+wHrj!
zF73EJs{%H}xUinqv4%?E7X5l+{VMWz&D1e`XO?ggDt~woN?^*)oPI!a97_ldQJSaT
zV&#^@b*B9Xm<xp<if6*2XbyEuU-w5OU;1Yxf3Q|jd~jT1ak&sL!e?ediLjq3Bh`Pj
z)k)4hR{CH4gjnD;vp#UN*8;od9qw#!E?Tf*cFgZZg~d=~cggwG<0j{vapkYyv|Z+n
zbl#vcy;Ma|0$qGjmSJfrem##eX8E!$*7xEUivi|Q=pRX^<t59oG36KwgXX)HX8;ol
zBHOU?75ad>QHsmn+B9UFg*;$McyN}{zf119g1MN9Tq%KhlOTgEd(bdQ++skP0;ih~
zyB(PeG?;bzGAJE8r9OE60gim3zaKp}IMlt@B9z+n1<hPVx8T>+V19lKZ8+5j*&Pg6
zI!KGwRk3sf{Z`B1m#zBf$60>8utKbcGo*5edQjl{0>cS9#00w{^H>gjr-)2tY(Vx9
z_(*~MW_dXi<eSB%DR2hC2<drqSHk`WgMPDQ5S9cu#Z95Qsbm$n(V&;j;_3Tki=3K>
zMfWswqy)va8g4>F-->M!FMIee7v0P0Y>5sEP}TW*SBa8nEYQJDrSaJ+m;lI)tBEzI
z(HEe^F0!+}N9*Um=-`IDY4KK9G}~T{T7;Ok3Bs_YhwZ~(&W}!=aA}Zrb-x;6mFlsh
z)MdWiqY_?Ml1C@BER;6am5+{>w16sKBl2#&p}#EEO0d!GLr0q(@!C-iEjG=|*&jz;
z7$ckyCaamgHX#6xt>d6X@;9_GT+8K=4bvM@J3J5nREjLS`uHxZjz;cRke;ru*i5?$
ze!}p4O8|%46j{l4wxNsXsVXlc6}m}1OD2S(OGxwZMFW=Ao-}Nc$u+Upr5krL*?rUu
z36|H#SPeO2NC3{0QtEB1frLTXuV?|Q53YT%4^~AV_QkxXXKP_<L9^hABBbpM$mj;i
z9(h5#a?Gt?e2#TrJ8QMtC=;m#*MwR*yl1&uwgCtoU+e<Y3<)9Kfibiy=FF@INyUX2
z`f44+7niOYRfysHLhK)96{dlWIj5ZdB`y1WkW0wB!i#x{$VPkLJ(fEFh{u%Y5yJpB
z{OlNjn%}qjXYWcjiU9PM3~z@w432J2u5;e4{Pwd$_*-m*#o_YKYu05UX0WCgww439
zrAx=<`Nu-HLTAV3sp-r@Zv;E7oSIU3WG+<BQnN$OYlhNC%>ubL^nteVA77K&4_;cr
zM}I<PvpWUjDw-iFtR$-w6|8(z`DnEViEwpO<eA#bHs&nLq5f|)VTNbo^}x!Z+a<KU
zeghNL4SF}W*|B!|uG<a45>drf#Y8w{;gV&P-B5o*-mY|C?aI;@-|r8QaY0)`PqPaL
z+l*#>${WH|;CxNCVGQBb2RQJZVAXYipnrFOPFOeSZ?R`e!3+oxMUbQRoIw<Y+_b)a
z=F|3PrRN2h@7+1+pYZS_6uL7)t<a>S4P6O^M^G<}&E>Io&aCfqtXz^oVwHWK-oUA}
z4oZi@hf7!47FjJXeKc@D-mk36Lw@qB1a&d2w)D`x>=Wj(pIB<V8bAZDK1K5|(1N^F
zx@y48lkrN1B-GQg-2av%Xt;L6@Xn4<OPNCz`7?s}QeIm}5a17@wy^Knm=|%j=Dwju
zt*z@d7jn0XvACj%9rztj?&d3L>H8HvR)?UnnO8_waCxkcC+ThZh%7o{Vas8cnBar8
zaA$EVNEqm>Q+GG$X3LJcrpH&T%FD4-7fku%>^4F3iU=!fpGwcbS3cSmkJ2^fN+$4Z
zX;*PAsCK#dBbur^>PVX4iR^AtcOpt)&^~exszw)I8PB-j6MSf*Oog{LYKAQH99|EF
zzpRaNdITXIPs^n{mCdEWR)eaw2~1@QyO-V8i7a=OqMFd=rYN8NgX|&ZI53nc4Rd{p
z&4Gt&B)6fmgYFGDVMl{<7dcra=dJ`oQ0&GkmfN0Mb`L|`7h^K0J64hr8w8FWSn6eC
zCtOUKb9F@R@(2Q?W>&>ub!f0(RN&+FzB&9IAsa^#sWUTR_pehYvs?rKYEcT~pNXXJ
z;{2OG0R4hGT(l2*u!hO>@_@c?7VKX1F!Gz8gSus55Z6bZx2bEWMU_A9lPQ;R++i|U
zUIpFZZMn0|@lGwM;EMWX%@YQ(H+ag58~8RXloYEL<Rg}D?y;)snWXKlc^Avuh|SrT
zGFJK`$Q?COM7V}<stl(&p^n_EEK?w%qfig>LGO}W!(Ut8>Rnl4E;;fAEO<Q0aA)Y%
zip25ut_Cuc3g~#ihb9SDX=NFsZcP3<>2jP|Ca8!Wgn?()p-my>%%FI<#B&y!EPIeZ
zEhgY;8gaQuRaWvSOFw=6G6WGMSjJGa2_^cVrqR<bPYPjU=ueUcEYrG@IdJfr61m6`
zs8vy>(s%B1UA@^y+R0KgU=`-QCpgDr>k|4fgrK{<SlvWBmVqUYmmrPxYEf0<t-v!*
zT0gTtw6>;0TKxFFGL{pL@!4N~T?m&u=O|?zRfr{->HN#Byr9PIoUSd{1i|(A0rsnA
zq-svC9rm_IOcnfiV0(2~&#<sR9u1Vt-@NlqzQ&I{PF=s!v~x#Vyzr?hPT7=w>MZ<<
zbDQbq(&m#O+Zp?~SqmzP&#H)z_n5n2H3*FFztf~dSah$|gk=flQ0MITpQqAWnl`X%
z{WcwK6Lc91`CzBhrc47aIXy@eNc4UWvj=3%AHeYv9?W#Uu^W$rIU)%}(hxPXty0I)
ziU-BA=nKj$ZeCA!31$r$=*%n6liis1L`+2v%+;5WTaTr!5>%XpkHf(loD&u--MR<P
z_!T|SLM<hA(eIEwN&gNh%8t;=c8;nM&AJp|<P$J87WaA%$!MC#V(eL4S!(Z{AxVO0
zNves;SuYlxwKk90j7mc8r&S^Eq?tgn(29Pm#6lj)2yijAZ6PpsbXLW)X$O;@%%~Pa
zu;q`v%VH0`EM51@rYe2`meR2WO=8R?Q=^vH9rU2d3_?FT^Z_!h+|?2a&te6x2KW$(
ziQc65Z=e0kZ9V=SZ&T2|Kik96B>k1aioNxzIY244C?tyLrGP-VtD&DP9SJOye|x7-
zbK$A+7eS};gJi)jSWjlSu=>58^Ra>n3;W`R9-COU$@SUqkNcF-tZOw@ZWL~WdX_`E
zD<+n6;+5*Sf!xFD9$iou{x&2qTi-GRBw%Ij4J1aD1NRWDA=2HjM=m9ZJeju?S8^q2
z8tw+kDte3NXLCx97Q%_B2-EV8G84ktFT~<m^1T7{ELxXH^SitM?pW#LsB(5IxUz^P
zGsIM~Efk$N?f8Df#cLTuK#8|6FfO*w6@NPrmmf~Q^#rv{_PU9gaZ!Y0dxrH`Yc;M`
zG&HzS;4AzrS<GxOD7l$9*J6<?e8R#<BXU7~p_FxLtv;ofeC)Z6!&>$;yI|4~Y>W}*
zxIHc|J7m;C{O+JrB`Z20^Ks`}!UqMt6pd4>Z~iUWZ=#}Y;~N3_U?KababqJOzHr_>
zh0LfcGf6BpP_=Xr@=qwkwJOcZNY8SRcUn-E)L4Y)7cq6{$>vb`rewjs(U39CA#y**
z>5+a)TDsXEomO>+Lj)7O@Pn~0ZlezFSFE#dl^d`13ZMiII&aXPn>%K3T#hhpA;DKV
z0K0{hu)XKC0|}jrfb$K;qC5{(jvy*r2b6k<SCk8m%y>YURRrB>^4ZE5A_CT}1mKjF
zzSt9i4ds&c^!Dn|szwty3sq8Btky{jcX>7<7}Fw)&fR?2KxfT;faeqxE?;Xq_{k#d
zDjrTEMt-45k3CHrb7T6dzV3ow%gcfLB(M+9X!@|4Su)HOYLM@4f;u)pG>=`qZXBrH
zejE+*CeC6l2=3O<Ih(<NliO*IHVD@Vi(G23b8ukd`soOE?LL7OuVfXL>S=}bD%Nlg
z8&=9nQX%>dF-P7U{g>xeQ&P;OKbzkY@85kv6-<l%z;>Lw6t0n%g#eINb)uWW2_d=8
zLx6<T<1NPD0(THR?mCps=B)h^s%Z#ZrUO#%B1%067eb9M?Ko*!XXs=ZuNF0$0h;-^
zqbyO-Zx*q&l1F8G4H$}@7Hpy`sJ=Ux_z^Vurf}D-r1G=3QZt=Xi%f4!-RWPczB!UJ
z#B}LB>T0RKo2+obYS{bhiq|?_?&u7C#>W#lzZODWR}Wi$6tTxtQclZveNM)LJ`HYs
zR~r@4Bv?3xq?gSyi!{HP4a4tWTZEStw-T|l<#r7tl6an;pu`?qt5l-NBC`~QOv8d5
zHucem`m$(oHR(R<h2El{wj8f9A>5kRY!o}bJ}k?!9bd6h;`bGKr-?LU)i{^2TO_5H
zq+E~*jNwlg0$I$f{ii7dx(LV0Fa^3it5f-iS0;W7&)Ge{Fd%l9r|xb{8NU&eTk!Q*
zsPWAB*}?Ef`<hy6FrJ|3)zz#=A6kFNzImKX_AQc`0HkH;0y_=vLW;3Nqj^>%rOsDP
z!kGs;wrpwQT~5>r2V>V<PWH3a=1NBweJiSU(95SfFATEsbn3|M`n({9{r;ucDWA`*
z0Mvtn!F(3wBib~SXo8@uw|3uBx5|sFJ5v34n*hVP@)3!5XEkOhl|z{&we4jzW+H3N
zfFKWBA$KsuRqplnJ3v}w>_#o{LD#k8E>%8)i&zXXi9mY79y;=Yy0=rpt3;QWc<AAD
z-59MYkx-lt*n^5P>OC(AvN6VO1X6^YZ0{GkMFs2Jny+*#;Hl%NWx@mPHy|CS3e2js
z_b{hCoWEyN>rk}<0OqSm8ioQmkOsdhEe<U}=k|->lf~KPGnH4ox}7(W1n-ws=}<L8
zZdkQfe0#_yy6+gBk3MVk=C$@q&}0o^nsZ_<MQ|qe{6o7%U$2zO3_8>uf;deX{dPW9
zv)R9Pj?iZKiP`Ti2~VU_k}mGrC-A`9r`CEAd|O-Crxu^Nrh|!mkO7`!O^2Ee;%qAk
z9Q={VqsIG6+79=TYs8x@Hf^e<&eEa6c~Na`@VQ0yXZniP@mgffIRxtvZh(E38Atbr
z$*o&dBIgR>oq!n+XUTh#t&v5!!|AkLBr}ihR_}J+^bh5%*`&m-RxLv{+kD(;wXunu
zBvQ3<K(#PGiG0}!0h+vj6>HO36{2o6<bzeH#J-aLLt_VX$>}{#aB&uU;9URwsmBJS
zal8Qi4>6a4*E)EEKax-<oOj@9P90Xu9kKhbKt4W;m!$bz_tVQv+iG>9zyB7=gdHrX
zI?|O`!JcJQQsr5fY<<!~*>E@JN*T?G7Vw68wDGhr3B{&Z=sDZFv%J`;k$b%GSIC;v
zK3)B{Im38Z>{7J{XL9g^TW@~>D^h`_dErwVP+}<K<9R6gt8P<v$MU?iJ=WSM{I$zB
zO6dztRK8l_O|ARlU7%6??K+e@p8TNmrCaxC61<bnpB4|=ICwi9Ww#lY8f7;ZFg(%(
zjuLc&>ANT12JF-O$cLAdlwJZ!gnHYH16_)-EVa~LpTh`(-T)Sweq{h11gqhPI+xrb
zl(^mAGs`&K*MIqxmlw$t*0izMpoDOPq*|*$CSnFL@S0T3a}@~{QI}uTYF$rff8eX8
z)})?(SoZ>W6QVZoL_huYF!8x;I(2tPChA$XuB=xYBD+69(sjNk2guqnZ#j`|pnRx!
z2YUqoZNgz>KS6M5v!d31M`op6Ls+c|fw@$e5g|P45OMjC-XPdjyXnBmqf!PqpSov_
zf!1qEZ$KtVt`JU1&;Z|^*=_N3hbNy3(LH?2*cy%{Ln#+ng^CAVIn+9)crE7sXL}B^
zB26*2R*}MTdQxQvPiCiS(mUnOODDqs-=yET8X`UW)T?z)^QFX#Xn9`U+%8F@e34s-
zIOXx7wAq@VyS*@WfZW|UWB-Q_gN?Jdy(li&IutGOoM;ld{~hcmJ+=B{4ysI$N+cg9
zE5$Xl3u}sd(CcME=uh@fcGsc@6)1GO3)+N*F7`#%h^CIeu%zwvd<T{qJgR4^exrA~
zL1?6pmg0hI92El7aCLc>qS{-n?^5LgA(yOAXErrLFaArMBm;30e&15^I_hSs%I&et
zoySjn!q#*ix4o$8G*1+auQ+jU2l0TPLk*GtfSH?aX(4l1Wa(t#)J&%n!J-<upP;+a
zzH>}4GohA0&bm`(l+Ya(lQP(u<=O?FJ=%}u(~TX#K1LZZ3>kikrNuq=c<<``ARCfl
zZF{&a$!?ZpqN*>#5UPAsx2_QTZ(9`A)e1UN!@(5Zes7P>?a$4UwO$?M#Vm^vTXaFF
zn$5Rhjy5J%PkY?j*EyNtP6)Pb1}AjI-3QWHTl^`!Gj2rixwOcYD&z)x#T_cGR~5$G
zD#vuTG`PD=`HD(l%7c5)PG)9DxR}yN(FVnD!OUzXQFAuP>#FqafxQaO!h+W!lFYeM
zSkpjBSKiWhzKp#c03_)*DZlneDDVIfDfpf$G~p!_QZI=bU|yrUSK<v-s$spiORe^O
z^|_()5lG#!Q|QkdPprLeKoavto6Uq9$BE{|OnP?X9#v-}VXZR?Or_V(+8%i1CSMI(
zhSU2Dl7-%l)ka1KP3AChjykc^8!m0sRMb{woyBLSC^zK$L<F)fXyx+o4G7CBq<D5q
z-@YvTUqLC<^aHt6P)JSL-UL&;_y~K8tEZNS{V8&JoGWrU<%}qO-<Eo8zFQliCilTU
z9ByC~?!Kz7SHBn?khm_%o8|KC^fYVYR?PtW2zd2Q?+F}8-DIY0N-$hn<O49hOb$Y;
zw1F6j0Jv*}7F&kOtN3jA;P4ZD*5vY3uR--oYMxz>PN+7`@hbw;tiTjC^0P%Uwwv2y
zv=#l^?^VkF=N+W$HWb4EcyoLIm=H+&p)BcJWR{H{h}MM=qP3LFP5zh5Mh$dl+RzbQ
zf_)TkiSq`laBKDZ=OF!(yf;zzR0_(9iLg8)h=fCwNXH(|+5r2g>-TShY0@he<t(<}
zERwMVcGOi%Ytctso%ZR?tU|RZ_)`|(!WABZP{j3KpZzRM_<G7crj?k<EjN=b^Ak7i
z2LBPZ4G>H!g2ZBLZzI$>Q#qDs5sj%eMljpR7xfM~qZxbzaBZ1de>r_oEEe18agrA{
z`QPSJ51>pyv-iw@qUgB-(8-^=45~x2--;$ry`sczE0z~l8St@nTxoKhKVSEyX4X~^
z&JT-QlOMb_qWMvKlX~e^w!#?BjjYqO1&%y@@I6zh$H(l#w$(*b+P3pxavacS`R(d3
z(TYZTZOTK~Lm_uL7+u8O7nL#()T48>wbKstJO@i6sm+;xsLL6cX1!Z|mSOsz=ttiY
z@WDyZ!mY%KzRLEeyN#TtvYTN9<}GEHTP=(f8`Q31gix9DqrR}K3Wpx1^>52wb{r|y
z6Z{rfeJ-xPFn`OHs@26Z7>V^Z=<ydF@!ylV#|SVaNbS(XVo(0I3J%0Kr~B<&j_#r0
zJN#e><zn1yem>VmCrdWz)S~#G_ER-mz<_znlK}M>Z{&^WyD2?C{Wwzkx(uc#DrF8w
zcGgQL3Ly43P3ZHwP^XSUz~<!Bry4}qBaw@ZU&kXydVJEE?i-8L|FNkae2%nU-Lzsb
z`hEgBb$xBl;hOW^eYpbg&j&y4Oj~Hj)W2@HqO}$wxzhAjvgfcm^jva)^2<&>;cZk0
zgvQ3R;R~0TTiJL7_N(9KO)lHMjlu7o@MEz19GJd5S+&o&D9g)S$;JhFuhNEd=eh1F
zEx{{yES5JjsRmzbx8iQG2366ZU<Tj~LO01BIVHfg7@|?q5e;vBy|eG34-W*WTMQAP
zH9uSnLXE>f+b@sY`9C^&<;4hQ%Zf&}2>Jj6O^ICYVt?48=!5BMi?Q+cu@Ra-CG2&#
z5KS8D-+D%v3y~!I0gi|#>VcdQ>VIB3kL$Fu(7U$fuuw)1G~nF&Xh&JdDHCnZMHPhW
z7XiQ(WKU|#N&lIg1>;S5J-dzbMiJvpUVtgkL0$edl)bk*SEvz(Ch>A0ahFc=KD<=y
z@_D*!*05}kaj|k5Q~^*-0jI-~FG#;MnS*9!SvrVYpH=t08>g}AC`fb9rgY{kIc%L%
zJt6cHQfp-Y)6M5u+Lm$Nv_{5Sl)dL{hp+#%2CNl<(a6fk=qYkemQ?^Sx}N|)p_hSv
z^?yi*;cR8Rk$H^a_pk18RlV^_rmFzi0VT^&wN|-)sE?;zlqqV=0PCr*LItO4_SSUr
z6EQ+xx1^?tD%EW0_5<LCKW3$s(p7NaUf|MoFpPcE)=QLC?-cARdeH<jHEx|ugX#r=
znIQ7}vLWWwsSA0|08SFH#JQ7naVq;a#3xVc*81Kg!R2&TN*bEyzTmDi8$L|^#BwJ`
zb`?+x*4Fd@&y$VMA9O94>8;cBNHOFlB<6;DW0)d12FaP-y5L9(%uwafNr47vP8kM)
z?J+)oc`a?O)nqu*Cf$WIH{qzg99j20P&sN?ds;DQHFs@h2fb?PtLuchYiAo|tyOj0
zX>!GDSbM$gjk6j(XgwTor9q&-+Jw<OmH>PV`|J%)cYE|Pkk>VMR5Rc;<7;0re`h5;
zm_r>_pE|W9dL_kpre}wDtfl;snWCoPa>|Og-#3#+Dr2%JFoygT3)s!6;+9jU!Z#rL
zYyG)q4F(~`+r@Fi!s@F9M}6a5bh@FJED!;%b%x%1)Hjq-$ecxjZOq(Cia;&L1F6V!
z_RvodcrL)BS?T!LC7Z7<;eUDg`{Gz=V1KBr0W~F9Dwv5SS_GsA-AvFnVu?uiJ~*U+
zas4Bau;>Q^Ex(PFuU}glU-1|3GR3bJXdErT@QIl1BM8=aO+$R_8*CV2W)tSd%W*=-
zv^b@O$=(3j+rGFdoHK1JuYaioBTu4sitXrR1cOK@2)`{Q>3JS1=n!bEw#~8DalgV=
zuCdCuzvS1W{b+-=PY!fx+!SbR`BJ(%0J!~S>^(skZ3gS|Y~aEDq?#yKAo^KNZ)77G
zn?VpnarL>F&<iF_<TVLLOG_4D_oM8u8xveDO~UdPI8T*xCZS179b#=*BO)^#lDg$8
z(8LJnCP-_so1+4<QiA59CMU?8Jsqq?y2&`XA#mU9+MNsQy`6NHVq2s1<R?Q6<@1fP
z;&I2OE#DVm;pL2XZnX?j*EC&~JR#jCFy5oszUUUf@t>X;rm&kn$X*nvt{Wx?2doAt
zcGinT9A(W^mg0A0E(}FcyoA_T<Y`<hO~bL;>xjZ$?$;Lts1xP*L-nO=ORvl60~#P2
zG&VcxPto{BlbHvAyD6UG>N2~ypC!sstc=^h!*zlEi@0MO1~|H`h<zr#>N+M`9!Q<A
zU#VzSMlU1s7MceIQd<D4B4_XH029)qBNUzY_%C@8bE)f{bW$R+ZpC1()XjvmGM6*-
z42Oi6L1sIc2VB=t8lZ*e@@oOCI(n(lx(i7rBN!6%j`YgH22NSfvJg?1lyLw&T-Jh@
z-^){Lij~e)*#Liz^d%2d_!)^Ov9%E<o2uKppyrQlKO8^@u|x|@4lT4WRy>BHoW>0&
z@ZMtc;49y`>mI#Z{ZGygx2|)$MUG0^k!}61?aN?j`CMg=cdY`j6g*~qTl54(L$DCs
z!+ZZFZ+~9;7U~C)W&bKbxFD}-2=F8U7KeY4RE!|NFnuMLyM$zW4wMJ#3EpOEl_IQr
z7!{9rvb4`WS7`G&@>UUP%<2g-$+XzI$Z_#kl5q8TyAVcOBxA&bSASe3sk;Evl?AwO
zIw|;^?HXX@wVJd2OK9uPG}~xx7XHb2fE>H1r{4D~GDw2D5KObD0X{>+_Vc0T)(u@U
z5(u2Gu3hpEdm7-%5lECw>c47qb;|T_Tfs#`D-rlmyd{3k_)PHM1fdTKVjrAw0cY^%
zNk7;m8+lIXM+p%hPH6%-=_IZOp0ZR%Pya-SwjcNGs+{PDBw4&ezn1b5cpWXQE=8m`
zexEOl(Jb11LE29avx+xd2yE2E2G1F-^gca@EMh0Dxi{vp(F|$XtSoB-!l)R6MRPp8
zEAvl_9q@XEYoOKoGAY<3=;q2J&jL&vV8m0uBoqj-8ETup&oycRcs>p(!%eM%tI7{I
z@*Dy3|GxsV$K51PUgD`Sn|bUSd&3uBj?W3N)gIl!)7h(J3Z~9BG<ZeQ{rXjsN4n2l
zCqLWu(G<nSjyrP#33B#Uns7ojvUe`{t*_9z3EQCWCa&s_u9NcL0!}zuN)}5Y#_!uG
z8!iBo3AhYzD5EySk(PMjht2};<IulZ7<tFV)Zz1!Ok=TrLpmGx^a4aH0fJdE@6<{|
zgM4;%0p?d{W!BOb6pa;KnTuj3bJM-5BvsC(M6l+Ww7Kv|=k>y)!3ZB{RdLX9z|wqf
zdxFngripN8{p-+(U6+W0{;g-p6`|NWdN*KZ#(5m*Ed>no>!Y<mnpGYhK-vdCpw5{n
z;+ba4>uOnVY}ZWM7T#iwF6S~|@KzGBa?C37o;^VR!5#ndiZ&ZX;P<rAJ_yKl|Gd3w
z7Bb@2Xs{K##)+9xsuKsRHEmNp7Z>UYt$ymTDU<kSuccH)4{JAkeX?%t?#iRU`-(Bl
z#l03Dk6&x|eWWyWOznAG|9Y~rbj4J9ZKa!VC&FuH#wxEk<*Ln5q(c4pT4YZ;r$$Nr
z3%n?Afi9*-$2)9%4>2Q_$Yo5_my0^^2TO7}TL2FYFt7cK%QZa{W}J~Hc4NXs>2Y2*
zOA5YPNiwYCT&rBtl>(;M3^VprX@2slaa-rJoChq2<r?o5IAxC$HVnIj?e@(1G9+kX
zPs!{2{DvGXuibA?!{B|fdHHa|FTn&ig@k^NwMp<d+aj#8@G-WF`D4!1%Qr6o`cn2^
zaRyKTe-Y0*E>86>H5V1GF9s~M)Cb7IZETn0=DhX9;OQRrh&-+z)9hBo)^Nona`3t^
zkRqxnz7=CesjjXyl&*~{VnB)m-*YucQTE>sbl*F=)U4?Gg$Y?@OnbhX;FQcW^V&Eq
z0Sisv{fr@FVxNEH1`t2fps&KA33dahWdv{#NZ-O4j!h?PvL9FkEsJ5g`C`$G2!zxF
z#Xut9274&5$i#}iU_)15@(p5Slo_GbpV17Kq`N(J>A;>E)#Jlot{pn!02qq>J<Y!i
zak0R4UH0N%Y$gBynou0%_q9JQqi+w3Pssr!*_Uem=+9LA9>T?P0@7eO&BM&UR01<s
z!vCUPM~3~oSBFr8yWSw+_j7qyytda56Ng2OzdanJR!;@j%K`$3DPIABKEz2e{X4J=
ze=;mj8UE1ci-`R9Uu|`CtUd@|Nk49|^i}4>7ZG(pJ+GOz1`xe)7UAjAe|tB$sr`qa
z@LquekN@HMvw*klZTkjL=4`*Y0M-b5(iOeb6FX95?Ydhd<)<eYaz@iyJzU8&EK%&D
z4nV4*dHlbbH=KOAPo1iNK{3JiEi8xzsP;mpD1Wl0McCI99XPM^_4Dw2*<XBPMGyav
zj}+oMwmBLV+-<M@6U&6u(RNk5DBkO{cn9!tsNgAyUYk1cKg=k&wtJ$<EH$EVopt()
z^Ld%_TD?gA9B+xW$J+hTX+IDMHQl`A!y^B~-vO{WlvZ*bgePTowSv(A`D=MIslLTj
z?8nm~;@_7=sOeyaDV2CfDFd=kJ>l+B9L?nFlrlyx;D_BHm(b-{_P<Srfj>x_=890t
zHvN|YG2pq#71BX4{>fC{2loBVYFL&Tb}00JjZ7xPv`>m|>~nY8a(Tdr?BOSb82de8
z{9n@$b6z^;Nu{#9oPNo$_W6Q4G;;Z3*X1kRtcw>jj!3*qHdHbaff?H5k?yrV7)$-r
z%tP$I{G#tZHRj3)E1O5-Mmwjca^fPDi}fHw$9EXvs{ripX1-LwTnaVux!pm8o3$ve
z{&Y_Gts(V4oI*~%JfvR0)d|^hZKoI|=s8A^D~5G;<nE6gq55ElhIu^KgvWsCcWA@^
z_|SiMWBYeAT(0@h|8QyhzjzA$m#f=PEBXJApRmS142ii992nb?$1klonED8GJh|Yk
zco8zR)yf9dp$Shfx6iRm9JN-4{E$3VJCN9@z`e@;%LzNy#PJ=DJ!(_a4aoe!prEbS
zaQ_c`Zygn7+y4DxfC>TzA#KquB`qjjQX(x%4IM)>pdzRsAl)F+Eig39C<rJaDMNP+
z0}MF>46(1_4!ym7o_DYPTYIhd-D}VPGjm;M9OrQ!@%`M%)rniVA)uXqs>Vk$8T&6j
z`a&xCv0VB#<+=Io5Pzj5!@Jb6!M|UnU#H$NKO<5n8JU)sQZRI559|vmS&3%{VY2kk
z!K;DwwN|POR5~|$&6o}R-(jTFIH27{_j?%>Q6E(o`A_s18CMDfybBXFJ-1+}SpU`p
z-ddnV2A@2GIz<u%h(O@x;4q8-4Bl=rqD9AKgF7hn7Q%?MEI=%hgXP?A2h*tlRJ%GS
zZe8Us{){;f7v+DLqK>y8t&hPol6!34VQs^(UC}YjbH7g|fA!7&eqd;(`D)X&&G_eF
zYAj&l@wRHiC<s{&rD<MUo4Ak5poc!bw(u-dnyunS?|+A>-}N1cR^AB|cZ-fZmvT#i
zpTVJhX*wUeT9pNXQy#)uVjgeF!M2uyk`L$ctkNyzzJ&Q(>Dp~GmACl{@p3Bvf_4Y-
zrE4QYB?Ydv033CX@Xi;>$F+CCeppEyz=;Oxy=JqHBOXdB|JnF;yv4E2)#hji2Ygb&
z;^i9vO%~x_ZHO7h+)Zk~7jjTcAJ1O=EyZOgu>Kg`d4uMB1ZKRuM}xf!0a6xChP&`0
zW6mr52bLUls8s&f`Hw;?WSNuzUZ#)drN|2!q>e8Hp?cQBsQAB8IR9XwDUty?B@}*0
zo`%XBT^%>wpt@*v@^RWHvk~Kc$Oq1Rzvr@ZI)Bz>{QYUA$f!`(@Gh=_QxVw`jGv1N
zdPNY={w=>_@4#?zCqUlA8|198{4ndl=UH@t8@`pI5)fysJ?;G!wzkV*%OHSYr?aG-
zKc%nlsNw@*PV@&@GoOZBe}x7AX+k!{A{A_f*r_vkCj%er=eOI7I-O*Vp%7z`dKmf0
zL_sKFBIQ5v+a(7e8R|oGfK=DNm%{Ie<9MOvTmR=9*T{)afczFWFk(}#a_kd!ZL8^C
zPPu=y`50^wn)pjUv}-?%o1d!w8wzQ9424_`Wd2Vc7qU>d*0f$lgtpf==NK<Udp2gN
z3}SAn>;{1CvR2W5H^waP68{0U)Bk^hhP(hQRc^Zwx<8t>)p=o+PqH(2oRos6d}AuY
z5-ya=JOFeCvUBkyrnvo|8IgFQo&Rv=PGKD40IJ|K=%`(8`{{eGZbRy%&%^#(7X^GD
zfe+*V2P|0=Jzl#ZZdEU_S6ZFD_b*hW`+ssG@z6p`k+&QZO2!Xae$VSO2YA$|xYa;F
zuBQX&2M?5R+v)-Q$rs^z1Kv*(_Le<(OZ6h=t$MmfNHovJ)I+b8Se*S2zx1kU(N8_F
zIT|ON(9Q0N>Q;T_^*ebTCw*7aV{YRg>N@|y0P&k=MgroBlBvR%fWp}0dgK?TYhykV
z+x&l6+B<)$#Mi|O7ex**=K*&2r@Syt;|ujH5lqL6E0;OGk4p+)ejFsh_%q;H8Gi^I
zdz-E#r~g!&hi*Fdae#~?fk?<d3R<dj@CQASvk@Vnu|fHx`Ipb9Ow9Cc<91ynST7qe
zwIYJGxX-Gu#K)r(gCyCpFKi}=nOs+=pjdhKgI#>Xe^-pVe=c6C=yxowpb&Yc@8+(P
ziSGF0yR{7z*Q8DJ-|)xyVAOA@VWJX7ky`0jvk>mH2??11ylxEn$#CdKIN5mb2exF#
z27g#sO<LbSb#i{97k|E}=Y<c35~z~3>H;!w4}QpZ{gx#O^wJ+Us&Rw&WirGYpw+Ho
z-p4imnv>f4e=;zvCjrdZ2L>r0>7VSc+{uQ|6Mq&_fxRdc@PiWYFTmJeMtCy&kH*0>
z>XVs&N#WcHrqw~BO-V9ght^k}%EF`^l#7hs%mZc_AMJwT<mdV!JjWQZ%m2cNl^@H)
zJaYLL!VMok`Q6I{kB9&K&#41nME_)Dm~w(01&fNV^k;4Nq$hk@PqNo9BGhtOfMl_B
zW_oG=ajC|L&Ie!m6O#D6DcnGdJh;JvD%b)v+Ew1s6Njg97@W<#?Xf35chnoJIRNOV
zbrdgFXMw%BE;0g|D^X$6fPi>WkdCSI0ynAj10`1%wz{fQ>^gXHuPh%H9J)UXVAleo
z|7@;rbT|?b`K^lIDM4sy0f9`PO8B+y-RJ#P$Flz<KNPB5OaQmQy64@^PmOrD$l^&6
zPmBY9`msJp+WM!+KTltGkMW?>o9vq`$9MXw`<i*k4dKk1`>%<q|FG2C^k2WB<`&nF
zxhPXEhGn()V}|!WxU}WA6Z_9JIUPV31@t0@R~~pU7y?>Y4MT03c)272bdx-~npw=m
z=Bx5|`u6km{>v%_w2JopVnis$nf|hH-AgI-Dhr3fqAYzDR)OkWKzF6e5~YCSHOEEh
z4N{*s(9vj8N%`KW*bb;=YX{ZlvIwnCf*Ua%J~maY_@ks}!NOEH*079PIti1@?Z&{M
zV-h1N!2X?W(+>zG$jWfo9>l!<W}x%t>;QR3^p~Exiq);Q6iFICGW(8K?U5O3d7!CT
zZ&xQDVBO5FA2%t|GmX_;X7KMH7Z=v<4ZT(3Tir{?tm|FD5wkV{<Ybgo^4+7gvTS7Y
z_tUjEw;4p=oH0wWgmxq)^&Z)fXX*-N^e=E|HC#+N3xsU|k&R0~j8XH5e!uL!t3~vP
zm=TjN+R)oTU}DM_5o^wB^5HgsmLx46z5exUF;`3@NW|b$$V-<i&id(N2o<e(5`@QA
zATX@vJ}y;s-zZEcgf8Fmr6LrKsi2=JtfJLAPg}PBkbDrx8eG#bSlk*iKG!Cs5BX3+
zjfk@JfcBk@L#TkSdXy-%8XeeH@_IN%XSw^-`Q&|sxb~9&I-=B$Xi~`=6~`Va4WMH&
zLr$xe$f*c&ZcW~`;c0gPG;rCAA)MY}m87pLm!`Uop*+b`9qnR6vEH}+*DDzCB#cMT
z@nz!A?t2#A(s{fHb>w+XuZR*50DXMhou~0&G1^Lhpww;vJTC9K6%&RJ-x~ddYYMSk
z-4=8juz4PZ2S9AL@D$v-aGF%qR)j-dCleG*S66~!)=z%K)}$ZDOrD{#{(VA38t~*)
zBWso&*9O+E&GyU%w&N2>iyMMKFXR_T=uKOo?H<sb{F$Wn5PvtLB-VnZxW#2#xol~S
zl<nCl5ohJwo##}Jf0X0GySwM6$^V{Bg?u{)xR|ghPQ_#|Gps6aTGn{LpEp+Ap~Y^O
z@zKZ|JgA0NJzrZW6J%m)TQyK?!}y&!1^|w=Hv;SG&0OXUwT9DUMry)i0^sQ|^4T77
zQb4m`Vs35>7Ui{(ptBgA)}S~40jF`ST=z3u$vIWhL0RrpDj*DX^UbX?xl<#`CF3y)
z3iF@1#@gqysOk1XE@of2Wg4N(ZqyPQBsc*A+~(%dEdvF9w_13db*9Bonu=D932_|F
zaa0a#SO&1bQYyKD=em_buk0fe&Z<1+tk8QR>RhmQ(RF}!CtAbuy&1?>pcPozV*o(L
zVEWbLCGr#3ih3$x8Lo{s$u7PiXu)d%?sB$L0HhbqM&8c5sBwsKSQp>i_J~rT+>XDR
z3+Uk`d=TEzKgh&IvA?+aLIq<tC4039Di;PV?2}d0sy6SMBprBJ2=K0+qMY;|)Wzj-
zD$N`~L%JB+GFF{qQjN+vwrxbkSk67zOzH1icrKe0b$6(tDC6^XZ;QQXhx22yUmtH_
zdG?O5qny>$0|X<fq+!KGgk6$==gQpDrBHBM7VxLNk||+zSTVNDi2%@NAF*xXYf4>K
zj{A@Tdlq0^%<*DFCnX~mVuGM)@vptyI&wGZxM^0R<4De?gfS*uE#eBrZo+~32S6(G
zXAD&YgQ!lvGHTtE+O=m|QS`<vvTUkzIxL~P1foK6g|sx>-Cv#aMSO%R2x~{{roZ7g
zFt?+6VqtIWB8fDxW2m-QLwEPoi#6{k-3kxTN^xBTQ%L!<@2BVi7Q%k=0r9{7u)TDd
zD>n0fzKYN1X_+*M^eAqJ4m|C!*A-|0Ml`lMIA9zSAv>`qmCU=ByxlXkXHE&2eY6Gu
zl(DC$HknOi9TlGf4paB$(AX+lR$EE~Kf0Y_ofkYDSLYqsyS=5tXLYR4_>(x#Y_fSE
z_bK<x1kaY@hI$kI2TVyy>}0hvy^|a7aV*U|T{&i5Mva|$ZsxI6O3Er_NzRxx#rDW2
zL-l!G`f)3R6@=m*k@IH=2(CbJ-kQXC<JWpN=fLAZq(Z_-JF15D%dAUSdS4SWI~Abz
zn7-?E5AQ?$OzBI$?3z&PQgJP`IT};Rz-dlQ1Mn~$s%k~*9ma}PDhdfn80Px<H{}3s
zwxZJAjOLWS=dT;0)4svGqeF23$?{e3g;P^n*8Y;jVZ<?sg~j-@@fr}ED<6OBg95KA
zaj8hz%3)371yERZZCVjQ)xHI_cwlwD3qR)qlv<^NRlnO8l-bM2ja-!vZvyywb*+q?
z01#-~(|qzCAYJ5!&Z1CkAB}S9&XmphWFc2l!4_kvgrcbUeg@ez4+vEL!wQMAvVhy{
zNOH1X9eWPSn*8^el8(J2y6`h0=U>abJ*K8Q?a?Dn=9Ne*lez&Sb$}1GY-r*GKPsG5
z;#T>CKbQo6Slrfja3EmJ2#jtAG2N_rMX;c~_0}8=8dsn-(0-WM8j`j4(8AsjbB4!k
z;qD?uLkl$4Ey3c~Wd&>j{O5lTO~vcY0-ygM+x1`3vj3Zpl3)+M>FD3Mua)bx`%Q|h
z1Sapuf#V+k`Tr&i&WIUjOUX`gEWZ24dum&tLnA3{l3Yd)kN~k1z=a~g{!2soFDH&j
zv?gvz`^@JfSbua4IK>=aSAYVjP9Sm9M4I4kfi#ezWr}AiawSa(v%sQlE*aT=0bssB
z?X(dQup5CP{iJ)Wd+9`UtT?IST-?n)OW7_2@&j8lCi`_^L^7q)jNeIk`pf0dJ@z;t
z`7@ZKWB1&u%wO7fr6e9vykc;5d)ru4g1`K;8VU!gXDw>Zk5<V+8E_HyR>PSb0x9%B
z6Nf<jq;{NyJ!=hQg6K-r;zxt(@j8gAVE|3a;H)<)ALl#oH<{5icIejaw*ZG)Ofu9F
zqY_8L1RJ-P164IvgK%>A<c+i;Kqy;LDStmwtG*ch{O^kO(|X5>wIAcf)bgzPU8UZ(
z00kZhJ)jcexgiimSFp@RZY_%#^naiEocl28!MkN}I5UaXPmy_ht^D@Ng`~+xl6)l%
zJOwCw5WFvXZ<{Te9T0x~*q%0z#g)hE1SASNRAoY>jS$<qd2S(1w5p$Z;914}Fnf|I
zGoV2|8)7VN4Mr3h*(*LNg%?S^YUWcI(c%q63F}29u9DPtS*@I56x(xBYTNT0ayyG%
z7+9&*ajb0g7<MKcaJxC>XT)>5x6E-D=Z2@f8E-xzPw&h6ndI>8*-%fbTAn-n#a5co
zmL2?Pxw@k~7d9G)anWr<bkv)1!hn|5W${0F&{=&8T(LJFj=uf&(Iai`sR0v$S5qN8
zbJr$M+<$KTnU}=>lHUca#g0OJsLNSP!l*Rx8=jjfY|vsm<*mDtn-#3%c&|0)HGp~p
zPWmV7Pv_Yr+<B1eGq&QE!D*NY{fL2?ou0i2s3USO4i6^pcO#rYN2%*vO)dCp3BG(~
z#-<1bT$Y>|nh-^}Er16_I8cEzt^izNfY9Y`LV{gca~%T<P>+!pqcG<(=X@bsy#P9S
ztD$~8x`DS7N}&xe_KN?=uYn4E>S9A@&}4o3o5vwu;eL$#d4vx9EN!p*(lyP8;62Zh
zW+~TGZc&45LL%Vc@b{!tBQ9^Zyd(gfe;{Fv+0F{BmR$PM^7v}jdjZo5W}o?~fAF7{
zpACu6U)3YOKl-jEl_5taOjM=J4EByZS?)VXslum4SCa!s&GA{$4geI0?dNGvhU)Ce
zhL>H(9lzRis?yzd?GI9?8%qk5FVTpz|F*p}8gs{V!bs`4g(RKb-9+yXl!rz&#U<s^
z;9)p5cWY{?P1g(6dI{jN@sZ0LP*z#<WPiw1Xf%9LaKV*j*Xu8mGQdAbrMeU~U;!j<
ze7W*GZHaeXr}$BVM2^{S>FVcnxJnjrDYx?{If_v0z(zuw%!xFj4X03yfZS?h?b55*
z7g1-=&?`(>TyA>gx+^mQN3Xyb{XA0IFJ3Wz=Em@S6(z^sFU;H)K7;je860k5C^AbP
zxM@&jtErZ8KeFWvgdJ+f8*k9~SP<P}AW3Wgz~)O>!gh6eUy=Ue+{0~jWAK0|^k5Il
zv;q*tQs3TZdyrY2L0s3q!jxr+%ZzcN$r2Sv6HqP!5YGcWV#KCttJLRE(*tU%rcPqA
z-v;hAdUbG7|D@PRlyItCIqQ0ii2Gc(WWf@EVrUGAn=S(2u4oLGlGQ659^0I8Dg`D`
zJk7}CM&1(1gL0;j$~JvNEV6PU>sERc42CKw!_i^ibgrrRdwxu)R;7#GhA5pg&0u7M
z^+q2KrG1?;JDOrRW(O>$X8xYBMRVgX#ulV(Sdz-4gw{!S0-f27Tq;EU;VT<Nj$QVN
zH1YS^<tVg$>=gg`Mcy{Hv#U7D9SUhgL&tlueB6q`Nf<W<{eGsj)q=V*FgG)wPzC9j
z)q|NElH?-qqwLxXAQ}Gdt3_mQXqUpSo=?l+P423#HyJE(M_oZQ!D(n%Zw$gkPq7XU
z8z`syo6mnTq$Oi53>gq>TwsSW8b4YatRH^{nO<{xH}8`A<Djs7E6GP<Fg0ZapTTZT
zGS6K-G6PfTqCd7sX!It4bMQgw>cVI3Mi1xLVFBEm%p6EnZC8xRuKUKj^;L^AdnWL6
zDf4=>J%XtT#K6tuM6H!y&Un@Jnvas90EO%t%DO8%E%qXRnDq;lqq(-?c^rhonEBd+
zM=%Z)t?_(M*s!ACn4Xz&Mwj)lSaKU#j_SkvAdGynC+p#uSW|xq>Wh0O|ALV$8ZS45
zLXVTZYu)MuO^pHF-^|hUtml2H0>ut)v;cp#ST5`vop_KGr6WKik?`rX=IgW`YiJyx
zv=yZO-Gk@WW7;41%VUCD?}hzUKFnHap`|~s*i#kJ9n|Y@${eHC+r_r@)F}}QeiwyJ
zUtJ<gCa<$P%ru=5M?M`LzfgNWPi0d^A?#clU<&E8<Ao4!XUkGVh1+kZCQ(%q+`hob
zAdhwB;L3g@_JP%KJB3i1aJj*E?|FOy){luvA~LkurCQxW-)k`y=`J_$y*@lgFO09G
zuH-gEXuxd}mpgvOGo5_|cW^`c{cbNTB=N)Ku-1-!zeW#s`0tDUH;C7kiL|%qeOrJE
zWIm?p^^NkuR%8eWWwkdyT=&|lIy;rmX>=$iw1N`ErIRIr9`&e23&c=~C0sTvnP9m6
zT7+jWDT>~kbYHT}|K+VDFYZc5^tj8TyNuh$4D-`J#K$_`MW5oe=o&;Cm)}lb0lz<0
zyDWe#%;detf2xG_z#~$YS#M*e%lydiE=R+u-C%$VY`plv6&M(QQgQ8Z`Xu9L0!Zz<
zIiJf%8Rk9R@>i2K=28W>BcGj$%Yt1;S9xi8uk1xGe}4YDptKm*lf{&R>%8f2T1$&Y
zjgOG(^J80k?!Pnn-g9*d6-F35G$Gg5c+&qKB%+3O-S=wfs#ukI^Qg*DgB<{avgN_t
zdM29K@YzUD-ado#a7RhOpd``0@R?=Nwa~e2yF!#=qmw~#H#>4Oj;p|re+~O(pMisW
zQ8F>omVX5ou6}o>$rv;)m_~lkK*!npP2!s8#!q|Xe#^6=Z|@mDoTqD`cs=j-?T#`9
zX=s;N29H>K!OeoS_;WO8oH3L8;Bi0wGj3yR$Y-v7>exq=!1dZPp>f#v$l|Md!jtlG
zwXqz;ortcoW|&B1)l*47b;+c00l=+#PdC{`*!yOJZZ^Mu0`Aq%`PtoZe4;^GS?2l3
z>Q{?Cc=1?)@cGj(l_uhfR~L4MX4W=o>O?gDMaI&4`QOP{1L)TL%kz$R02YI1Xj5Ch
z#`%58B@=l*-*M>0Di9gg{E6t>NLcoVE++S{aHXI9jQ=GZ%MFn7-f#GekChh(=*1Jn
za}NPB1N2F13H{xmK`}CRyS>@TG4HCKhu#(J;aWi$)9=_iy+VJ~R$ZfDi7l8Z=WzS=
zP55fTzGx+OwSrZ_l5*3>>{)9@tv5syA3XtLd+6^)?$}m<j@CmDR=~ud2cw0EJ+*Kq
zeIl$cwF0o=>IZM)eeAaVrQRcM-9Gw2+nQI1by!13)ChfwKQY~d+g=sZSv59V`q>tg
z!33G0H@gMQm8YKR?yKaRynizG8npFjuw&%g?dFBsMyZVZ$tD5&{hFRmdIPO{@^~DU
z*wAaT&zDTn4zvdR?|a;Qd8&k4Vy8dpylEE71URUZW#${i>ad9FLD>N>8m<lfA?)HS
zhn-Zk4M7|`v9Yq2@kCUixUbO=cEsd;`z|AY1Ik80f7aR(QnImd+oWFFuWG>9L;QIb
z;JU~&@Vi88Ny$yws*-7UV9olN)Bx0AW1)tlF>n$G3si(Q8tEF89*hs137%SLvb?Lg
zNI^%M$(rTfl=Y$6=0eYYz*O(LPmhn5fpxoCi)4lZNeVbx4~&Gj4Nt1QBb$J8CRs9s
zlarsp?PxquQ<SvZ|33S(6)5ax@2{Z_ebn22yWat|{X!(O=3z`3k!oqchm4{inik{O
zrN`RBTbHO&1zG@`bndkDNJ!5o?R2A<7`KM=@+uE(V-|x_PQ_7tks>Lf=Gite-7*r3
z_C;3g$QW-K&Dq>dM90s;XpOPAk$1>|O~aBf)0Ek>+B5g+xFA3NF*04-+gY>A9UYeK
z!Lu@#F+<)svmvXrgkE5~ZSox}ehmx&dL*M)Mo2+iEEQ{Y3B^`BWBItv5zDUn#<Ush
z%&5y!=Ej#co2>hlF_fY_L;o+3to%c7)Uw9a^e=QiaO@w-O^>3tSGlY4wA*5wPv|@(
z6p<0(hw-X~8$wY9SZQ0u`?N+AoAQ67YmQGO1EAvsh_kFvN`81v=1Gib(=v7d^5T|6
zt>p)mg*xp$GAZBi)mRnmb&%Eu!0;Xh>)=uafaZCJ#srxr-h*xk#A4HEY)($3lU^d1
z1YXGf(zFc>j@EZrkI63_o#>Ll0hx?P*~x%<MAf4Tpz42pT+`DiyU!7NNr6Busjyp#
zw<NRqORudsr1dp2?)K*Eno;eyn~=mMtj}@EBW?{)Ml&fXA@FC$u0*3%^tN30m)C8;
zB}qxE2f1V&%-^0LUgUA_SFR>?*XR)BO;CX|zc`&DUdt3`j!aMB+eUeZB@efJ!jYa^
z#vTGip(I70Ro-Aa^|(~*S0d<{$NSj9E>~CNE<XwkkDKr`XS|=?Vurdj40Uj^s;Wv0
zqZiBL#Wq<GX>lYjoJV9);B#cibA+vFP(Vmj-IlLe%LC%U>+n|lpN#mY<JKGFG?SbF
z^bX-l2{MT|F`bYem0AP4kWyn(TY8;a(spb-S#rwq$dCWp3x?guLE$}l5>+(Lo=1Ga
z2xRuezWL4RvlDfJFI2ouYarsxU}80Ft%J&p!PSc9@2d+X2kP1nwt}f@rIc>m2JW1T
z3j#(}wPAwi7D_yLl^zmH&rshUePVRvk2z$kj6i+cgkd)tMrwRlTU_~{F0{dJjVE=?
zw#0{=E*C#XG<g0){At&zPewPpj83CZqOVS6@HM=-<N1mx{p`7k@$3&2T+-a1T@o9i
zdSI4~>Zt4+Ufekr)dH)j{H@{LJ?0f*7rw|V?7+86#1zE1bSVqiBE@dPnI_*QHD50X
z=3=Bgl1|MJ@5oyT*d|vfJdK(#xCQC9v6D{4*f*>Gu=@A_%e$}%4MsI7&N2Wp!bL7j
zr&R&x<5kEwAa7ujdnEg+W8nP!l1LHSqc?#HeDe-&WviT~V*r-ID3*sD8+h4!ZX8j5
zk#YFex9W@DqcIQ0>}<Q+#IDaCuoIhwVX`1IzNS1V<YM2qHvW~7;Doz>?*^a-9RInB
zZ@&AVv(=3-vfG<a?Nf~HP<D?EOM;5qB^oeS-uxXbxVbU=$S^U48MGRI?i|^KLEIb=
z`5!c3Q`wd`#DsUGoMLDXD9Y|Hkvn6vx3g!XM<ylY(JHyxwg=p?NqY7H`!$T7Gr0mh
zFdx(l08|x~huEj6*`dC7yCvix7z6%Fly~?rZOrQ7diC>C817^RsyB36r?zRS3_F6e
zTnl(8NndcN4M_?MWQoO&#VGIW7(BTvhONx9kMnBaZqNqmoNZOq(tJvJiaPC)4n-;u
zO+?A1!ikUg3>`t-<M7ah6C+}Eg9AgKD@{m^<Gj3A_^%|qNCOg3o@Rcyw|9|TBbq8g
z`X<4$>7ToiA8u?<!rI!3t0TJB)wMZ3ZEdLdSSkm5ATsGY^hTBw&RwK4|6WDbNkLht
zC;2c3sB5X`>{RAE7*1SnzRLmp<D(bzdDUMkzwuPM36~b>-9=cWn)sZTXPIKf$CBbF
zq-O7|a_MHRFFAMy*PXgd0I<x-O^b-I-?L^c?_Rrnfm_f*MChzoM4iZUugJ#teUkcp
zeJkIGKMh|A7tW+DRPet*DLR^;Q+xwItQ?yFV$9HrdPFQ~aG)R^bxWf}lITX%8XhJD
zl#|{A-2b~Zu9CN_y-T?dW}>e*jL<6TW~G3wtThb0^Xrq0UF%G-*^JdfS+*zhnh;<~
zt<(|(*{QsxmHt15PgtW_uF}oDw5a^GeFJSR&*s+f6TZI}-&;lRMJT75WVOMNP}LLn
zMD}n2DH|2;YIvoWaBQ9ibTknv9qfQEUVLUG@X#jm1PZ@?-+Yvyq#ha=YOegoQn_uj
zSliKO8~*8sQ+Yivy-yP?nbT|2=9UDMocJGnQGHe8SiG<`+}t;pXI*2VUsqBUmRU~0
zWt-0YRYf4ZPm47kUvP3I{J*U-U^uky=A5_@W49={cnFQSf6y-Z)}8zITi23f;uJd;
zwK)qS;8PDj#O7>}GaZWUQxi<)WInkE2%ly25D9(!VFF&CCr;rgkr2H6ST(oba}%gx
zp>gtujt1NLUKj`J!kho7*P-=bMh-2@Oui!`nAuV&!p=?T`cWN{5UgqU@;|NznM!?&
zg{tCze0Pi><j1ayq!lBr4eNaxp(<tE_d4Jn!HoOXSI4g4*BTI!IcO#U0UE%L&|L`4
z0JvI6xxa10B_;9{#<rHnO3DbdVy|w3cU8fs85@^{(u4~hE_$@Im`tO~`91>GT{S^{
zO?n*2E`+U$kY_Ozb(a)~Zax}492tje-aq}6=2tKau!>m4_j0d!@Fw@I-WLP*hs`i@
zO{ax@Qypu*V&F7Ex&Zc~$(a!!%|C<HwU~)WxPJj)B{}2BQAhV=E)rN0;8TJ>L9#A=
zB~F<=Ow@Uf<KSKSdRah;0WH{s+(wiuFfzc3%{2KY9vt~LI3-Z(*EzD}_CfRuqBC{$
zqa?Ak`>l1o{YaIRKWlETR88#GRpE$(rT5}YlYpv$z88P>9v$ztcjT|l0f;779jEtj
zU8GLzK$dF>XNsX)9#?cG_}F+DPR^!0xZO}z>mlWhgtD%nfFc}2P&=l<9#nF#y`XTJ
zsB63{NI$E&w$RZntFD5_O|p4HR&7{Ppzi|MYnb2>z2Dm8qO42=@W0tn04ol+r77}f
zckQ2Lk=KU+BxmfIB+|UtjLB@B;<k;VVtTP>QOf62`jP2iJ(SkKJ7p&SBA(8oGt?(p
zQ9B2fg86gS<ADJ32$7ZKK9-x@xIj!x@TSRh)Q^Jd;!*B4Y~M#<TB&3mz(7fPcPtX!
zo3REAkN02p6sRL72aL^(xkeGz!%>nffc4-=WMb#bC<y@`yYgp8-XCSBU-p5{B0e5R
z>BCO_i|ai{htc>qq5oyZZb<{?@=ZIUQtCX(tFF=N2@k_2R)8?|D=%&y56N=&TB`Fg
zJE7Ucl650_OBnbmRs3xH3_vpX1ETO>W)`C+5|8FGa{9^%?@4y1_M*g)_M{oWG`{?0
z8WWg+$KKcp842$=+9*BIhvrE+ql%w&ZJpCk+2wyB2_3(Ez~Bc(1>U*s<<NsQeoVi2
zF#f%c3s!zwiTL;0f0x6Adr9tZoSW-tJYRZ;(Y;@5yhq)eL&{KM|5LyXfeQaxqWiyN
zUjc(Gy(r`W?cQJ-IQpvkwJVHmH)8;2Nz7+JD?If0QL57>eC5FKHX43ZSJfHqitz$Q
z@zI{@b+EE65IOVzQ8%+000hO5ip+-hd)D&Kk`gRG@n=o{NdD$IQKGwayTGtgQy<hz
z=v6OHF#aTgD)nzh(^%4H>DEYaG7In!rD~_R9`%pn<!hyuQe}DBAC{dMz%KyR69lRx
zzx}c}$LGrHH|~t~rBXt;-o6ySg!YJg-*?B)UFbk3%$t-|98PrK8JGjU2H+L`fq~nV
zSp;uyK|LKb=WGp`5`Bou!z*&Q+Fh3f-klXsRLAmrm>UlZCD>g^bcfs;gKzs4%Mks3
zg7w-N<Q|7?ca0B<U@cX=?RsowPXv6v``*f7l-w~V)V)l`dN@~{VQN;230Um-G|tnX
zP<Qwtv>7^CpIWJ^5H{GT+za|Ty<G7w$cyW(-agoK6-LFhGCLmUKE_%%AtiT*m_Ym2
zMfMfBrHH9Y)PhqHiev?r_3Me1`J1mYEk0!Y%mg2f34q#eN&o8HJbdp|Bh&Q#10lpm
zS<>*~YwFFY!Pa5zrJ>d(A-AWRO0+fR*oGZzR&fqxg}YhK0qg`H!2l0t<fTR};GP`)
z*s-(DfR&0_Yxl4K8H-_Sduw##f{A2-3KzztA*~c?*udafMmYCMi~+?-V@DNqe~0Dv
zuvXuKNui3Zt6{AnO3SqWWA63NlxHL=lsvrF)wGpw9|v3q)`WQeO_lx9^^5EuEs)mC
zb!kW>eG#-rUCdi42Wg!RNN53b3g~NLrlK7Fxh6$NO`}}an(H=Pj<!9*CT!&UOuekL
zGd)GTeE|?M%RTQT%{83~F+uJAbnwzSvw`(uLC#NGwzg%|KtI@ZaIKib`dI2EUcZB6
z!8l+N;_yp3BqZG3rG$T;NXNMrKD;~nk(gjo`qu^dp$YNcDYxbcOY%@mHjyZ2-Mo9`
z<r<$G%v@q)Q9_bqCPmz5EF&d`-`cDtAIRv;Iu)drG45UvGiLCSWX0;3>&yA+^@2J!
zq^Q2MuVyI8WYhUB9j(-RhnC8Www@r!KJKXe$p}OS&>*4v^2`11FB$UA^tG2>ynlen
z%*tu21H2lXo1@Kz{^;U!neU$#aI754XhxWbpU=#%idv<&dvKRkk~<!q+EB_wr73u)
zY$NPsX}Ft4R-8MB;G>7`f>CwhwWB@iN;ua<|1_r*HiD-|ziuCi4xt39r84Iu#njW<
zsrxpj!o_h!rhUka%KqoPD;1sPrK%9JM(ySAnf4nm#$woBC^L%3SefhA>XW8G*X%XK
z7jB3cl_AW_z?rYqHR^cOAMgn+mfpFjL0T;zB|iMMQX(VH?qTzGE5IN8b<2nX{sIQu
zX7x#JunoY|taEuDbg<<O+kO?0PuJETXcWj{P_kn891o1WnFeipp{|nh=-c$NcNZ;p
zEY7I=n;7Ld{zOtg0?v8!?5)0@YoPG&t&3m?+2pN_u-ftyk-`k0qVc1x$|`9Ig&bzm
z5fH2~{at~lNS5=`1sfCw&R0BmpiqmgEo&HxPD~YcERB)u1?ABAfi&vKv?;`faO9cB
zB&mH3Eys;=qmFm8Wb9)H`#KKPzn$FC{eUp<o%Ts4jp-eb=8W#W$nKpM<j~anVTws=
z-))c&KB}pzKf1dSl%kK5o}^pRSdmH{*m$#c%JLbb01JeOr3WZ=I{M*$5`{cZ4;J)z
zjO@iPXl8a}c+{m!gsO|jZr-9*n$JwY+5-wPZig>TYvZ>siwUhN_m<ZHio;3Ft8CXz
zOK76NYAh-xt>y@6<giX9K_a+KK!o55X6t#aI!2TazU;Tw5$g2Cf<@eg-7o)(Ag1PA
z%A~l5Iq1Pmbap(ocSrmd&AT3!?*s&XKaj&Tv*~tRv054+k7`|VSoBCY+X)1imQy@f
zHi<|O_SJRxmZVz3I1_p2=pE&ww9n&dA_T9>Psk43;==_e=l2Aqc!fT`$H;fCHhc+|
zpLS-fYo-$um)uLS`MmqJ`*%`+Q24{AmmxYWc}hKvmDM9FJ=|1{r`p|-2rG(k^4?Kq
z_Jk#lD(zi!E8+e=S<_ogF7$;fNZp+x!oDepcP7$JjfJyl(lKNkUKedrSS8nORucCF
zp_Wu~qnXc7KVv<Tf(Drm=<J1`fVtq7GBtDB!iM|0NchMBVKu$6f4;Cea6wtg_Ev~|
zpK6_{tz>|XKSynzd<zZM7$_#hjyb6VYN}!|Ykl3JD;o*}<y2xd8LyVD)4PiI+=xpG
zZExv=0y`u!G<aMa=ta(VXLkkenqyClvC0Dg-o8n8fj0I6xEprPOanx$dRdlrSXYGL
z>BLWqV8kbn-gjNSI=fF(q&B;+LTE@_sP7UzjFC`#yXbH*->P^4I&UIB2R0qp66O5l
zS}R>yi3m4PCWCLpR{&~Y1T9Bs62mVYv^0?qzU-$&_pR29+8R(=H1Uj7&wul!^|&V4
z#m@4qAlfQAsu}jiOLp;I-4YSO+7Ft$BAM{oIhZekmV0Or$hfwZ>A0QN;d&$HmBSqK
z$lJmV?Yq?r(ilc`&gc!7K^zT-IcaYbZWb*obG1dz01X|_ampcX`V&ja9=66y-o|->
ziBKQQ>}!h~&`x5yL<!d$$}3x@4J+;2!cb2TT#&rhBr>Hdm9d-jWLfaLIQGR6?M);S
zw`EFN!{@(|+_GL4G^X%O$&;g%jkWzS<*N^e@}tCK{2>Th0C=D9pN_IE<TJ1%w@L;5
z@i~h3Vtv*=cU(~V+u<b($bV;CpyG=96bNWr`oXzF^TR&4#Ks)m;&=`Cy2FJZ(qBt*
z8%3BFXgoJ&lH8uSx^DmajE%mXwa%=VxH3~lC&T(|6DsIyn=A2`_Ocv!2W`a}qrmt6
zA}s4(Cw$Xe-(WUnGK}R;5$uHieXlb2>o8a}hrb=}k!d61lX~OMp_L5%6fW7EVltuR
zxKQ)0mro7|SjUJPv_ZJ}nd~M-zeUhRJtX3Lb3G$Dnt>)a{FEg_`sI~;I)aamKe=vC
zVr5w+jZ+R}mrP!^P@Ue^ah|D`QoHC?{MEb$VaY*zl2rS2Xn5vzig;5wOodsg|0ji}
zH(_o^7`|H+<zfHcf4lrD>Dt3?)`A~Ob2e(*PI~k-X3_~F7g=ye5<%?GhcA2#%bbqZ
z+yrS6g6lL;erI;xF9LJoXL*GtSYP2^>*&jVd{(zo)5U{Zi8@@+3{P-okFF9)bJ-wC
z|Hy;PJC00IRcC3_(zN9}bFL#8>dxa5Zi1il2GPu!^ShA$&ORJJ-#{J;`qUa;I4@c;
zcH2DgF=*NP#)#nAHclxQc9nf^bx<E4jz#F?>-z%mf~E4_MeAyP!DTkq<}U!LZd)kn
z-y1%GNsm%2bPqA<kQ@ErLdA)1x|B5`MqOWB{JqjD?3IGw+$`N3Mv3aX*U;YS<RI<$
z<y>k-9vVZtO>e#ftJ!=GqO-Z4gwcnchS&0vU2dD@&-b&8hagsLL&99NFzb4viS8F5
zke5T1W}piTJ`=&yzkP&@VxWu4A-NgmB!2oir(FZ`l#WjN45DVMQAS%}tJ+XZX+UOZ
zaIf}>r7g2lZ$R%xFhja<1y<__xMI3#NP80%{N*`?mokgd;VY+n*~d|SJIkRES6k>7
zy3wMl4`w1KYe2fS>?dvaZXWYp?$oqWk3F47{Bkdx6NW8f``uzlZw`PPcsjp_zeik;
z_5;ydtYtI{LewP#jj8C0RckL;YBW$VDMh~wQuC_hR-e#@Bv;DZIJ@k-jx~?hK1rrR
z^37c`GLb5<wKQ#bj_TYg*00>J7RLN=UTg**)6A#pzZ@c6Ov5v?(F+#llGqO`w+=!{
z7kt^q>(E2Fp*G?xAi9|o-c;hf+9MWwCr0$FAgiMYz9Pz92>=u~5_jNBVJ592g(=#T
z+HZOZC!-4n6w!>`e%W^tZYvdQB%_&ra%ifGkRUdF=-ccrQR{nQfHGEOAYYxI9C3Oq
z)ltP+(T+WvFInxK#G!VjdP!8%MwM5@iz7&5aWL}=O?1v`CHdyK>T`#3<5r0biqZG(
zJtzGldYm&J#pz~#S%ooz><8aBj7W#pl8#Kg;kZCU=wTcL3GEwviYpu0jYirEt$k^i
znlGF39)6_>D6(c(IKSA(yd)o%-wg;CT(x4?fD3^fCZv}jUITz!9)Ka3qvb>WI27hv
z%P0JpC3YA*avwF5IIO>Ya>XG+Qe*u~>MaMYK)xzPDdhW7<4USWeI~ZJ)SID=E+noq
znNzQL?uJy>c6gRIk7^Rdv~h!}*|jtx{e)M<q+*^#KW9Li8Ad?9t&;3{YsjWXTyx%|
zU3{|D5Q1j)k{#f3Sfuh_H5;+O0hFGGrAPe?gb!vV={}RPK}m6kH97p>V_A)8ERK>F
zHaDX;PrcZIJ!UGgr|_rahVWn;=-i|eKOb-*g>fw_tY;1$5*`4HUXXUhg*4vnCj)j@
zqsWT<DXTLBF}GCBnXwEu3<X1q8HrC6$BQ9*#?}(VnV!cp?;8CEN;bzHuT^8ntL_&D
zB5tnwNaRoQm@Ezmx5@6YeGXY1CZg<}Nh`fBBR2r!BXjriI+Y-E;#q?<r+H^2t03D_
z*+D4I&%}an35FS#uL)mtt?80n#^pc?aS~pQF`AgJfI}+MVSj+J-Si6X0cvMr7P-?W
zoI8;y=wX_m<1fPb`=<{%bJXe&k(3}{^kAE&nzbwS8k8+;r{dbA3Lg7lHX6uBs<~Xo
za&d{dw7CL`@b0$d54)b*UYr(o_UmKB6CJp6m7FHT1W*R0U5|^y4G&)A)z9g0#fso+
z*e>Y|p4`v$+cm}aEll?|zG$afS=oD2b{3LwC|}hoHBjjcAv;rw3$o83o(I#du+c-t
zULGXZb<Kp4wG-=&@>x<`dxP_1CR}ts>fAFF4}*PydF+iw(IRc7RVvjUgLN;ip6%6K
zX~)}HAjZgpc&e}PcR=r!G}LpA3BG+qLjLsSft!~}<%L{AMvv327EM<SB{;P1l@pd0
zPEW8^=Y-aA=-+rf`u6%tLsiN5+kT|0RG>4IeG0~JDgqyl@h~-hx?T0En9x%Ms>b<t
z1fe)le!Zl~WkG`@K0JW5Wy4o$6w79Nz*C+!i16wE6n1c)lDocc#17Y002l(4S(5>z
zNd6P4AD9V>p3r8$=9Rd!IeV@IOe5CCc0@LKpA|M*k3+K^3Fmvl%_iXQOB^*Z6%{y`
zF^qRvY$#~|KhE3S(H<sC{>*HI4gq|5TaOD@uOC=n?fJ}wzN|4-NyVqg{=_Y$w|Z;}
zs|MfwXo$nu5m5tq23HFU;qA(Y5^X29ECwO{5PN_%ixkM-M6ML}XvP{+q(xuH#nKsA
zIag&UMi?$sgwx5DP6*lg&-h7gz6hI{4q8}UaH0TS>s*~i9r|vBpMX^G7ox<yeWg*X
zd2~t6n8r++CQiR&$+k%Iz+MOAR%(`o+oNLeii;beM*3KX^Md^`&j4XzVPsdo;IT+O
zv|Z@Yzz+|ogPfbu8kil;)ZW%C_};(*Vmitq$Jp{k3@~pS^kV4h0AXCB&3&iuy}{!c
zy+!?u_~j=GLS>(_2&ZJ)J%+lO(fgoC`Or6y{pp#-pLcOfSQS0#wJKC3m?5H%dW_J=
z25>o~OLQ%`#UFeob!+*A5vlF;8j||Fn&V!_qF=8;%syT9a&9%vq1y__jNMFN1~Hro
z?5{-A$bFb8OS^$}1oa#sR$O`esB21|!3G{^9f>Hf;_`_IpGH~Y7;s*_!o>s*RKGKV
z!J^(JH#)s{xjwST0D!%?oTtuQ<ds0s#FDY<ek?u-)N9ieURMS)irJ7A^sgCyi}8u$
zj2?7j)OlCOWjN^?^3g!GyuCyFm5NJ(?C$M=%J*_DN{SrbBW@iW)it;v2Sn~*qfNv@
z1sZD+9H&HRBNTk#tkWwMUK<9QlIyinog;l@cg?$BZij1q8cd7R|8D=ai7F=Kbe8)>
zo4}IIl_rXD`!(yk*}&RmZNLaqlmQCxbzRM-o9`h35cD)oVfa<L*<Nb>C}fyO-u+<o
z&h7Txz4y?rP>c^|SE1pc&7tI=ZW&YKu;Zq?jCzlo$i>Wgy$4*%gG}H|Fjyz*5c_di
zst0-n=WHlO`)SKczsIaLgxM8xsXRe_?1-9w2r#6lp4f(xB1cDJG0NQD#VC7ZO5#wV
ziJ=XxF!w-KMGf~@ky{?G%_E(N`ER8OdDTu{GTIpuS>f&6jm@XH4fnsg?v>oEd?c9r
zF7>l<i6-^9o%3zS2;WQ7gs9AEYg+LMHh)q^=@o6p3^o(X1`&6Edc-S$An1KFFb^Yz
zlgcmbil_T%-q5xF^vR{sZS@)0f)~kJlf?Wjh!P`6<}sBxd<-$RlKA<{8|YfoYCs$f
zyTGDkZ&%KEoRxonmuzo62nS0st$ISgRZ}s9Rf|i5`swqGP={`8b91nNtV}C$ZkU&=
zhK#WRBLuK}_7d@h>GswC7$M)V3A-1jFxx7ow#x$CRX`%GFnkQiqeYBcQb-(U(%jfm
zyp@!JP*?5l@tyncAd<`7ocd<udpk|myr+IcrbZ*JWQI>G0;@Es61OYpvcKGRaJZ-y
zl3weuca(``cIbx_d62AC@+BcGntBScEu=CVM5i+5SEZGjr+09<Wl~Q-pJa^oMOi-?
zS3&274+PO{7u%^|n^aNdCYze<W$0Sm`w^y*uIZb>v#Yk~;(!Cg^?qh~BU0k|m9lNf
z*T`*=o1h7Vf8RV4B~A(@e7>%}9U~VYC4xC@uYU>J65XzGlP(=ad>V@@er`+-;ErIv
z-=zidA$Oo(+P&r<!$Pl)wNN{imK9dLeqV~!1uGA50&7`qy=xl`48OG~OAfzX3IsTT
zz;&O5pp!u${i+C9nWNh{R%+qh<JAxFYWIK;vyhiFZ1pwGumYaq<{CoO^&8@L&pdSk
z9q1x6DU-_wr{~^121iN|PPPXb#91KtZI7HMS|!p9ao#zU5Bg4qDSc>HBTgAT%&<PX
zf3tf4xB1P7W)&vB=vLXhwqx~q>}BhLNmc`9_3+jk33LH2HFE{gwmPzxp!gVFl>u5|
zi*pWgcTcDVhQVy_GOnt0@ks~_mcb}wl_qkSFd(ItQ)kwn*^H}gUv1w!W~KaczC|{v
z3+zlA$5wcBuE_o-;6`TJ*Q22e!jcCEZw;GW)on-tFFeBI5=-rL;Cr3gcdD*+Y7+*e
zLEqb51O@`>zPL%r9jWo$@B>fT7^Z?}kCD0O;3JQCL45N3t?;Uxt#*#-VFa57A0L?)
z(4+M3@u^S11~Gi7Mk~Z0+|~QR?wS~~0g8&Lu-CWi@$0tgfi2V~J9CtwEeRViLUdN^
z<1m)t$mh3xOeG$EX@aa4V!t0sDvKZ3q$2tb3ai^0EE_h<a7CnkP*MSduaMO18!%X{
zBT^#c{A8IdJ<cy|8vW(NG$f^tJk&KUp80pH;uFS{2TOQ2(6>L<U@fbmGQoE&Ct!`j
zM+lT8hFn#qkX?}%dE%ax_o+vDqXCGG=g0v2V}qjmC8?j?CoWXY<`s)6`}^qwXDT1r
z8$QtNb-{*6Y+99Go3S9?Z>qOi^^4Ut>R%}@(KNW`V-)OrX!q{%$h)p*DbgS*!}E;I
zR`1xvZ8+Jlc(<LdZt`ofcgpCBYW@}{Y+xC^Y%9q&*yU}!;efC>@4d2eD5$y?gM~0`
zd3`u2@-r#lW3DR}K23`SM80hhxLfV6ry{OHG{+GL(pmX!EC=IK^(cb5fSi#Px9f@%
ztMgZ_g{M9E-4MtCnO&X8-Vl2)PblvSoi01-3p1%x@%Jn@^i>?K2DWoM*+fN(AFqXF
z!CfyAqCX(364@TY0ybDW?Mg*Wo}%->x%Sm{LKeeE+-0hUck1oZzfkd17z^M3YPejd
zEWFhd3nqOhh~baprW!_AcP~b9Ig|>beJn#lm72S6i=%Vp_vb9)k`^G&6AJbLvK$`$
z_x6R_iEp=X+YhZSxC}n`9DnB=If5o)*B`N1P$HggHLmX}4-?+u9!3Bjt#z8NzJK_}
zey+^W+i#W5^_I@78$6pLZJ2YQjE3o8A|=c=!*`e?LWKJxY^g797}7Fkg4;zLoDTTi
zaZ4on*1hOvR&!!M2Xh`*MdpFL-f^)1jcoF;Weak(QF}AX^cX;QV5z?<l)4w+%|7P`
zNtNid2<gkP@e1B|cvfTGu3b%kQ)+La%zp6t?#=4Jo$GSKhFtWzb<4Jb%RaGI=0S%2
z@Zi`S#g(w<UoLCaZ01y4zd@?N%y{<m2=F^zC++JBmF(f%TRD)!(F5KU*IP?nPqg|F
zJfT!8<@3(Cgv{9O(P9aII-&zoCj>Lvg|~gx!|U+S!uf9Pb#Amf+1@^LMGh<prlyo0
zhPvgC+N_jFa})8bUxp?3%TrG80*abZl?nL9rrm#Bxrq2>?yoiDWUIl$Q5d&|5}uUV
zXR21|+B2P(kLA#G&4Fla?73>;Su;<^<Z~&<8F#x}!Q7e+ikuK=eU2$D%f0T_H_s2;
zFl@t_!HIEZ3hWf@Wh1R`-iJ=>tv11|ycFcs-sh(E6Qyu;SeORX-(V+du#6);`O=I8
z23jf0&>Fq&(ThrCoum<>Th5_GuZUS~J4UgMTLG5wRpQM6zxuZxY2UG2D95|w_gno^
zS^3+PfDQ+WsQ^-T;0Z=VRo}hm=3Xqv=5YIW<3xjF-ATEOS;fOgs0SgUE6z=)U0<k2
z;nRV<Kdxi}gWc@AiR!G5!i~Ihoq)c{?eG$$IY>r{L@>kJi)68h%e9+#bhFn@)%(zF
zSeIfoq)e`|9v#}dqC7lqHq2&{VK!cV9oNoQRi(&T7AEEx@z$nW*f`L+Bnl>6c{_mu
zJbTVlUae}VSX*mN+<G>U)*>xtpsaf2EwrmP2gbQK>I`_pUFm9w8|08Q*4+StI#WvI
zlRk5Me*@0z$aPR>kcW~qefKsLcW=O*yj;>N$=I*gYK$m-{avpyI`aU@?yCHdG%YeV
zp?_mFuJ2ee-24ZGf~L?%;KtcoHxhb_hi>BI%w}CS8@$k_1^42qt+g2CHD95cD8Wa$
zh*+C}G9pEi6h2&Uam_Piq>hoCw0I*YD6J~v3X7zyW?W9Y8prqu&X<YXOrh{|fI!u&
zk|XRhQY4Gq-ItkI_u-*Q7LZQ2?&)f0_j?WlR+31mlwxjJ_@cef1lJiQxnDCP0SpZ$
z<S<XxG}*2&uua1g7NfV$ZS_D6P94>#N0X?rNegm^tO_idU&e24AJPr-)4bDE@1#LU
zJ%KLam@Fmq<{ocV2wW6w=)&INU#Tc>K4>=q3ix*~Rfk5d7VM3)tfc8tPS%cgdGEW%
zZ0lFusGi0SeKbHT{93p0RYrl6J36zhjI-5JOR)r*`Ao|~4Rcw&!HOX$T^H!oI>^)Q
zGU@_#7C9!bwi_^ef0L{-@vi4oOM&%AOtpvt$^Ce8cOCV8TGvXa1aM-VvohznEt_Qy
zj?o}9@VrJsUqrL1ZATEa=N|AtEu?#etf8JNbflu(C)REr_r3p=THU38k?}@ZD__X`
zNdf0b5_cXiQ57{u@33h2d-ZXUlH95M?+u$hDzxgx;V09El?^M=_9{Mdr*d9g<#ej=
ziw@?oE#*OdDgz9crM?gd+o5{*EJE6vfUn)|&U*xH8If_j#i(66uDoC&Jycw)$z#=;
zo6}Z@#xejhQBjledKs#8rrH_fsyka=ETg~A4;m3g0~t#gj})L|K1uz<8Q%{Zw^hk7
z2y0CGqjmy_I*-!*6m?!D`A3<>3k&0om$Or@gro$P35MXx8?Wp%ck_#7qA0otr?I-J
z@40xf-XfK3UC1x<cYOtWbk9^5XS&gB>3d);IPLEt>ieF^;|zAYf&7u_Pf;R#34~WV
z;!5xJtT1maDWiav;wbL%5!otZg$ZP>O?eL24A^}0npEN!qFnsgQyvu7<<#{9#$YS%
zSmkx-*B>)exG1?j*05Gdj(Atv>{O?|S2CU&i@Jt<#e$=C5i8rh$fPzqa<;)t3gx~P
zyV|<9+joS!^nUl(MRi<7F&{|PYM%~NjSuLW1hjAi_j_{%vfXcCo<07e3`4%o3YUlO
zx5i!cUbPyN47C`r8XF^G_>^^oIG_chNB>IwpMI1&7$I7jTey;6A`=Bw?`sTWb4DC+
z+?7xb6Holo+Q%_`fgKI}2&4EB^aVWXScRkPry6Q}p~~iGNy21dCiOH6jX8^?p!<D|
z3@KTmeG#4H9)c0}VlG|Oa?+^vUALRmJV4PSAQAS*09`KKOp$&IU@YE2Sa&wu)<||`
zmnRQMj%M^e8NDpS#P#@2%zOH+{#gyPd9OrhmT_~(i8Rr3-7ircN$D?%f8o-M*LIsq
zJzcdv`6~9XWR7l;pDJX`_Wdfa5UTrosrb>1O6)N8Ao-@biBrd89Z~ggIGgxX06*{I
z-Q%hkuO}svnQ9t&bElXP?7R#zj}a-r8&G?I-;$qjNN$*q2aY^W@si2{sg>-KpGLAV
z-k81tye5swa=UO~I;+>aS1kI<KaW~{cdm%~n}VE~!(T7Z!X1sf6uDEa?lsiIlMJXM
zS7*HTrJu`2P?w|P77U(X_H6pV!R<;nej1}0HzyTXq^Jv00?cuwApn|u!WZ(_1Dh)>
zSoe&csbz!}X`6^WzlTTy9+*`;QL#jA{syk5O~B<V)_(m&Yv`W^Lta#3L)rz~;cpVu
z4n~F%sk?7{58F-S&>m86!*dFVoMbS$S*g(hy!v}`oMIKJ%@ry07yWO}pJ_ftu&Ef*
z4h!vAcTU;(kfF4dcspJibAWYwks6dvD=}wwsvkt+$NG9k{_i3o`KhS8yUo($k!b(T
zf(>6uH2c&ofZ!mx`JE{UA(Qw(eByQG3@5%06D8p!MmII9=udA+`S-t(%z860r&+I$
zSGl#_ZDLVZ5T5WQQ*k@$@M&OY!Vyk5jh-bS_+uINolOp%?}^2U-P@JT0Tz?h9Np2*
zYp;$i8hWvnc4L8NxkW*LK<{4~kSvSW1AcY_KLbv{uA9gCC3zfxU-Uu!h*$%Ee)vZe
z>3u_$hnUN|A2zwY#%mlC1LDk@ZUCr;CZ^ZBGJt~D{wGia!Ook%zm&}aF1vPbP;hY9
zUc+xIg?hHe3Oww(v2Qx@HYL2XV`(zhcW$}H<ZZ>Ni0}#6HUI6_uE4(3!;ZLFx+=^Q
zglZdc!zk|k0?JgrKEX?C&iH}s+Q#~sjSRHE@!_alNTt=`PWwg$`bftQ>Cn|hH3z~;
zq#4JiC4IG75bsKr#2^zdM<1ZL0pC0x?xnV@ve)ikf6WxPu`o;HGvS&>QX*cl-enm?
zft>0o$=o(pFL~*@9}&;=tMZ^4Xc4R;Y;v1i?b~o##VoZj_;sK<r*!UiC}Y{mF1nt3
zfwg4wDtK?DWNv|zH~FO;{HVTvLI_JJ<ppOpJr__JjcN7g^gk+bpJ!?p`L>Fy-9ON0
ztK!}65)0q$n{*Sv`u<<+y>(QSYxwtzg@J&9D6JwXrSwn&(j}lGIiQ3z$j}ZbrGkKT
zcMKs6!q6}XZUm%<7&@c|28PZd-e)kkd+*;`XPtMg^PaN~|G1X*)OFwYRo~CGGGUvb
z0-Q5yUg&*?nbpra2bPe|e%6AmQw84O#>*}I3H!rJV!WN8@Zy@5&E06af4xsL%vwv4
zD(ghaOa{`HzT@(#6RR*{esE|+Q77nxG|Q8CG-QsXUi>uugrAK0Lc7k2(bfy;ewSKR
z&xbiqJzPE}GcUWjA0iDip^^{9PciR0C?Q=0MoDV<iUewAn^dv3Q+RKIO)Kqy!Y9l!
zw`GCJAjCy)B`!$2IjelxF?nu+R=zU%h~A1n!-I3@C>Us|h8EvEnG{<&LGNq=EoYzd
zeYy`oH5N95S`t2`xB$JaFGPiG05$<(>oW&Q!$uF{pW|6-$Y7qkVfFi1hP{vRL7bHD
zMfaqXJZ!uof;aJUBzxPjbmkN4n6;t$t^;V*wxd~Mdy^nY?6le^&vDSbnejo_*AQfa
zS^H{Ijg@M~=KKC$aP#Ovpeva>2${s0D7L@pagbZMtd#KR?y|a4c3K)w7qAfC9y3QP
zyM*5aNX|ezwvcDuhieJ$gD8V+nc$j{$dT-r*12L}ByHDYeVw>L)UZy}PAV51=uCRT
zt7-oadjY;fD}xB^FKp9*r+CfcJoO!DN>`LgYvQvuJGeQ7-e&6}NNjbgk3Z)q&T?tZ
zcPjps$UvLHt38u#Y2H?(jniNa$|51bV{vu7AhR^XqOPoH=3t9|J2r;J!fo%pj5a>z
zbk#~hZ4Y=Pv^FLJs!yDdc&^A#;zB)LP5--4s+YiX+)hiih}(l9=B;IGHKEPbegi;5
zQ|;=**h1{I63}q^{&*KWugO3({mYv_K&mjd>dY{jvUDF#;I8Yt5YV!XsiKChb0FX5
z9c#7TA<GBl=&Unk@Y;SS(1pbV>jGMloSU@O#_*YRSi6HkC9u@Q_ov9)4T@}cw?upK
zcB{5itEXWs&v(=;3Rh`(yNJxNk=r|VoG@2l_=}h7M*0Ibh4lzh^SP+ymnmIuhN?p@
zP$uA-ViRrPy^%%v3!Q%Vy=NB4Q?I|$U0-VUbZA@!Yls5YbBCT*@aVU8IX;#Y&F9@M
zEGucYi4xP~Q47Awz>oywc?n=BPH|Ffws$%QR!}0+R_I9ksu{NTfO>blzdrYUAJ$Nf
zx34r0991(a2yf;s=6Q|ZURj++%9P_Ty0cfWT8buYr?ZHZQcl75w}g4sx)TkyvU%su
zUIFOp{z4(uJ^tbBmW7@balAk*7a8PV@vXUP?($A1MCGdER&0<*j$)<fz5D6k8qm{4
zPX5UmHCL5_NxYl6?7>l^T(ut}lG6}$)PJB0e30VW0-wS;i#rR5@j`ayQ-=ai3cNv;
zu186@NG6I`azBa`<{xX2*q!U&t=VjeO5ZQSR6MLuso5GI1r4UBMW<GFZx<&gMjW(z
z-a06Sv5Jz!A~{BprX^24nlFi42@l3$!(Bfb?uIljy1>{XG-PxZ<Byj<^HXIg&Y&d7
z0+hzuX?<XQxeE9+%myJ(!z5ujl6;d2IQjr|OYKkcwgUkAE@|ib($;@3VIOESTNU&4
zuugx3S!)We>5;Il6TNKmHfLwyWZ{b0%%<309eJ7|i@+K7Jh4337m}pLtz@Po!OcCR
z0=k{We)%jCv{fP6`Su7B*s1q}r!Lp<achXaz*ZtDc|`Z+Cyn#7?`!u*U^JA3k0s-$
z-BjWI>hJbaJF+<}H`jYZAdy}(dQY1SfWTRM8t9-e68aNUIei1NQ}}!$4h)K$se*Dn
zlJzuRX&Nj>s^6Pg-RsjXeU!Y`q;|0`S;<HBrJ6?x#z(I`G4S)w<gm&1W7(zf=Ls46
z=k@j&Q-Dz+5{`@R20Lv-m)|#RLqBY^Y_oEV6>c$xG`t)tr+>bjk1$p{DeMcpQIXaq
z688Z|0w8_F^Xh`mQv6|7!z6RaHsF{y0Gb5ah<~1%(=)NE2RoMi+xsLddRO{HS=tZQ
zQ?06!L}h29B(%jVojSXcTqWluI%gng&TIE}W*C}F!CT4@ohmbNM)3Cj{)_W>fbQ%@
z_|)RTbbk^4Q`bg6&+G)Wf2&PvYBlM@ro_$my(X=x!P1c+)sxYNGa@(#&gS-3NU^2^
z&fQeqX=r9^20cmRxp%@$+w*Gs?sC+WWii8-c83JnbB(VgE_ko=UD3&^&o8>q0kNoZ
zjeNNNMhfO{<?|A8I)PH~FlnJ}I+1@vERK`xtOa4AQvT!<1H-kbQ(Q3N!gHrSK9~r;
zTY>?pGG*7$ONbe?b-b}8Ko<?a{N~62ijO6^>j6}*#z=@+3F0bdGM{g4zh^R+&++dK
zT&`BkWp38l!~k3qnk6}3K@es90bmBc%#^$n*jmAoKb!N_k$c${w8Q)ME2o#+ymHeu
zr-3bJpm+>P(>A+uTeX4ElZMOv@|h9HQvYc}fTQd3QsTm=FP=G-pV0gq><Z9aPah4J
zyetiwUR>PiIiP`x9oTDojTbcm;`_<wu305IDx3I=-&W4wne`tT-WV~xQe|00q!-=`
z{W3SQjmhXrL=cGYmN%{Yk)@mqmq_Eiw!In{;`%w<>9SFe^?OSz4$q*+cJ3+63yCr2
zr~W*HT5SOJLr%x5v$=OL>8j)zl{!(+?7&-0`#Souy5wBPReP85*5-i7u!_Van+vWa
zI15rcaqo{^_4`nWRqsDei0?9ln16UmKf3Re|2W*giv|w8=D%$nKvY729uwS9z#k|7
zX(r!fVDApSN+KfHzZ(Q0bokd#B5v&Q|N2L`k(9OxLcP~YuQ~@k!X1%Y_&k^3Ls1a}
zY71SJY%p&y81Sq3CzcP(kz%?Ahs8*_>9+Czcy!^aL4TwRkdZ(Fl{xnWo^xkl-+Ha1
z<_?Q_yTzaVAUGhmkrBDS^`nZH(L6|SaXgmc@(we7E%)`xs*(xHvp-f8M*5ixjMH*a
zkJhBSlJ10DU{ZWE8{3F}*fSpR_jc;e39RlbQ-4sns@PU2*!)X=95>yfV$lWVni6LB
zW77G?e_HN$8)6bgG8T_DyKGNEd9OA7;S?O>RiL+)=XYt_|5i<^*%fqd@$vc(D^e0F
zze&0$g{&}RzR@2RnaI9qiXOtDtFytFI28ZO?r}?C+Swmn&~N|U<japc3{>r4FRN8M
zPp3HvQRf_L!rCQW{HwpK!hoS0HV|G@tymqRM+!*ibN^we@-Q)TFgfxqH{Ya9FH-_8
z4l7*kv<z_tD92w4oxrw0kFr8i5mVC>m@)>@eR`VXzb1dw&4xg<*lRB@1MR&*t$$Jc
z<EQ~<ar|g8{^wb9xRZ&eC#OOCvrdBRfc%%+%?1}npaFP9nV#&Duj-}#t*ceQuK_eG
zJXL~iKB|G0p=(wE;XdnM&0$5v!&7>Wr1tXCWnC21ikYmAA4snd@tFMd^M7E~g-6U4
znSeqW)Kx7=we>GiRh*)94Fb*#&dB~L=UPKRfs`qQ1t1C_O}x3VbI<#~Z2UPy&Vqy~
zd)R|jttEZ;wrY$sP<BLn;cbLq&CNn|qvH?7%HZrY!T)U}gd&yFEFgHiL^VDs^j!Z_
zYG`-qjvVWH%X<8;Jw0kyv7kz=l@HKI{zyuGgLI4kbVa{!n%;x~^A;#QYlWL1tQ&Fa
ze49-ooCJy1FI86O<;~yniiXGtePMwj?Tz~$zl2%q^<x{~g<1b$+>*eYbs)D}N-)1^
z)nCo1Kw8k@ztmTd1lZB}oqstY2}zVcy}SQ^i}wEMntfX?|JzO(d5>4NdCkv?X`!cq
zP5n>(F?ei!dA%o}_PGTv$bBeceJUsXa_V+-j~`;Rxw_!)zwev!;?=D>TlJNN-Ju&a
z`m;khN;1b%6g8cKTa~keOq??=hy$XVcWsACz;>;QU!iS#^`ofX=@Lk3Zw4|8yn7Qh
zL{(OB|35B5M$nXiD{kJb1jdgvh^S;A?`q~=i22*rZ}slM4;@YIAu9hjN%4Q8@@wgl
zI;2Pl`X91kjbEbSXNJs8X{6Z(r+x;64a%&q|9!uvn!Jb8BS%7N`Xcx36+=d>Nz-#s
zsujEl7EZATBdz)6!*F-anhwG5O5rulFRSa{s`3>D%K!fGjaLD+CigHwz|*42yDOCl
z+QATXEqKc~GXRzUN5sT!{juWgo&TWQ+fnGZXCq>t4KrT^^?)UAUwr&;=K$LyLNPeX
zR0@`%%47DAY{U2SlIWN$dw+*{$DTQ3%Fi(4Z+rg_ynPeGJOaU~Kngfqr+>;;8i|-8
zIaMnFt^Zbv2B8pn&Gh4*I_7>zAbFjxItEd8ji5=^JoM^yuO<^(VgQM6V2w&{y0rqq
z$5;53-zOfA1>~4SWv>r0?(pCKz?GFIfFM3$RM0Y+8_xGj=}c%al^SZ&H-Fj}_Dm%U
z*E%LXLk&pBv(qKOKjj#w2)fOW7&`)ZfBHA@zF{XO&2OmMPPI~cw%Z$@V-5pQ`LGs|
z;jm)MTBu-r*OW<o(4t)~{g75fpq#0cBq0Y*l=jIUK=njx{VGfnG+~O_HzJ}oCf@+4
zeCOElwH+)0l@CT2utwYFdLk3c6T1eWypjn(RVw2~#+TT6s6}S>gueOQT88S^*SVs=
z#y~_Q`|8gJUJ;5Mz9hDwufVQ^MU|r?+VU#yQz~kFTXvjwE2=l+4YY?v7^(@C;C0re
z>^y`CexwAU#b<YBTDPAxu{w#g=y%0~f6Y(D18Wd)$+yIZbbf<hU3!z>$PJuwwJO<q
z5mUPQ!Vq9y$rVs`m-K9YZ^nDc2=V`n0TwFMg?-y)?17Y=;f+Hi{;L0PN&`Y&k?8HW
zc>up%@lcpb^rCaT><qvQy!o?o1u*Cje_Vgv-1p!4ze>-iN>d|Gcem8eK`V~?W)?>P
zr0ObXV^h0b3oQX^$Lz9IU^clEswz8Zs`jDg+=eja7Y*>jms;2~?zr5dW4bnX5@*r<
zd_}B5q6sB9l6FDKEQ4#xceUaEfxDK!)iuBxz@7F=Q@f9@3)yHRBOtr>*U7=QO)CMo
zs)*VKc5~HW<Aa`xa0_c;o-mYHh__&@1~AhgKj!v3VcDGU){41XsFex`d15pbN=&#f
za{94CtcWRayna0Xt0wfe8OXos@MXzEO#sltp(;%0k#g3ddu>)##N0EL`QFw>Nie>Y
zX}vz&^&(v}PLYY@em8YR6`<PgPOYn7FIiJ<wK6;G&4YO6U8B(*HY}&>f21Axxi|Tl
zVyDT8U>CtYan2o%?(}5~mZ8sWxhmSo&Za|HzQf{hZ@;|~q^dh)-Nb>zVSH=i4yf7D
z{Jm_0;rK08M*JhD&e=C>xV*S0xAmPKV%F{k;hfX8c~>J>{8Zghg$?qh1BH1F@;hlv
zym`aCMfIZF5yTHn<Gc=K%FJ(mnTZ~mG5(<({bx@V;LyqpBFnL9l;&l75aK!tgaz3*
zid*-;+_mLpm+c1X9N~@2ae;Pj3mei@yEFsQUd^Esm8g!oruB__G>~Mv8tgn!xY1U>
z+*v)?xhyxq!eXjeaeehmCDtr0oqbdnXeY9@z^s)qHUqeP>wcg6+tRR&X7^_l<aKTf
zPh$jeiU$Mqie~+9HNaU-D;jIkR|%?29)({^KdiJg$M<yw<XOCyjWd&4=hwTqkyF!^
zKxne;RX>~f-rR1AQUPdc%e$bgJ15UU|0QLt@F<>JD;;sp%69o1F8?{pS+g(Pi*cu^
z#QnX129_ZEUJ*_<v~J;^Jb53`)^v!=@1)DtqA6V7c$uiDHZcAab`%ES@>!kp3AlV+
zr>oA9IVcxLLD2_eFSk1narx0VelDbUP3~L%``3$^o#}y#0J!`xXB`b~Rs|Es_T819
z_{b?!!KzE>YAP`~wBKEk)-fmT-Vy#%UOhK$SS9N0G<OZcjKCa!+d<f8uTs$Fo(;bO
zqKnqNj?P#OiGkF<_=(GZGh~ga${WMn1SCPg27czZ7iF5i4jI?VJ*~>{MV3EW&`BqP
zM6%-rctN)Qbu2=~{>g!f7*oK-vnFNpg_9l-cI33~v;eRISYx{Od%+B%KBAy%y9WaF
zF{iJo0pJC%=h=+fx`(!tfArTOWDGD9GBntMQH}3&`NTpdJOQlT;sXGa5V;^JbNb?Z
zH_#;Isek`1Yx4lK+VMt0MrgQ!Rr9AbsQGACqCPwslE;<j?xquc);IDJ$HKasT_yX1
zoLnK06wj$1sg=^6%N<@q@Br^ZT6jT0S530C<6R|=8}e0^Y^s)qGn=?Uv_}^Ym6&ca
zyPu-zv7r;=oi8Cn(Bc1nlar#Z-I1}xqUb@C(iIcq0OFDdNM|{E+BlISA6T;zx4_zE
zM&)b9ADD|33*Kf#B1SQQT5VX5z{_8+YhYqWaJW}qFUCd~#yFZ=+bC@s$o00$z`G1c
zfO?2BOb(^rIq^o(ZYTuu(by}s=wVo~Gx2U2bC0n!EUh$^f$hOVCzFovy!_uk(9&J9
zk*9vJ?3+D<D?BEUb_H8CKz%9+bm`{#bwloVNXf+7G6%DRo_HmHj%%v%DXwk*d~%#q
z^OVA=sCl@rlH+VTXJJQS8_6|=@zA{!wouED8^C^F7PMi)D5|0_qhp(q@Ej*k@6+&L
zXCju=oJ%Y1N$8F&l3{%<<{k#(b^{NeW3?A6w$}=Jw##!UL)Sa;%Z6`>h=o@Qrh&(~
zY{jo^S@w7|6k2-JTB>P9Wpy}460VAdfKJ05?<v+0Yt~wV5-v%FCbk=F;Z4!Xzz1t|
z+THi-UVkOZU6PLPQ*OyMlTF}uOz%@OU_J+;li-omd7xuE1r{MJaJx&tT#nC;L9{s>
zN;92yECgUaKjY--{v}h#c4Z6@|DSjp+2{Ug(A@VNZz$VBtcpw&ldVZYP&<WP{wUj{
zE#T=_G4lh_+>2AxC&F*63bFP?OT#9{wr|!xG0+REM!eKzBOLr6A0#JYzURcKqAvF9
z=E-NH!fk4Q@bWnbJK$6cu$}0W;41Am=TrOT;8=6rP_+~@IZ*M)t&3c&B^<(Y@?tK?
zzcewVrwV`vZhH=4q0P8oxo6jiy)Cm#l0Y^Yv@bvFw!;=(blBjN`)d%(ZDwtw<`eEH
z)!q7hv1$XtG6R75pRUe+T}j0<*Q*kM`BHWD-+=i*y&`EDJ1*5ske^E@6|q%*a3I;R
zVQD1?-dyK?vk9BNZdf%fV?5c&BQ<&x8#VaM{vGD1TYjpW=yW~y<^jO*J@ll;*nd0S
z-UzGcJC-x$^@MLc5qa21ar+CudDXyaZkJ)#Vz%Z7oR2W9u*4YdXDg~o?9|kGxP;5f
z^Vz$Q{fH8)Q9>j6eV?*(Tqnp%Qpd9HTQ+`vs8u4Us{uZ1gFR91+%`jBUpSej3R&<y
zQ3SO`WE7_x#fsb3?E<d}s{GC&4;Ym5B&Wc-F`r7`#`}sGnI$kYH(0EfhMx_)+~~_t
zM=Q@#6{(>QnvM;>9LNr-D0}}@zgeR)2BD%p=pq8`6a90yd-7Nc)>87!z+dQ47r?~s
zhk+jENt-v)0%ZLDx+<SQujpLtu}_PhLk+pTqn@L!Mn}pP50zHoBoE+Ta>X3N15q1+
zKxt-buJHY+NLlx+JzywR1(K={a}@^tWRvLSH+u{3uOn)v_n$EdJ?tNf@Fsy3tX_lo
z_uk4gNidBez(RgoK>z!^DZs1=KmXrEsegMmqW}9o=l|hvDUc76`aKBy+b?l<y*;nY
z9{)(PcJ2GCSosBb`ak@czD2S?4-4VvMb)oO{M+yR%d`lv=<|&08K3oqPunIm4$CU!
zl)8`S@rtOhMJS?%Zk38T6ljz5!kHI<ydTv+cyki83T92DF&iceIs_$tT<;@k*v0+>
zyk9rG*ch$M2C#LsUUr;*aTy~`xB;IZO$-{38gFXRpga~EhlKYMX|0qTZdKlK-RiC2
z&}tsvhysF->dwf$Y?srCjkVQ=!(cuvfXN4{F8F5<3)44oy_@*S6v)G<3?1Ro-e(zr
z!*a4V_ub>jjRk^=WUx9^<4n@D^<EYy|7zlAUF+44Et5hXR}&(3`b*}-lgb&kFi&ed
zX%M5^i~8kDL9KNq(<Ll<h9d1!nGMcfd#MB`aggvRKtsb?vCRcg#;?PP@cpk=mvked
z%YRGuoG3j^9V>UuHo*~Dx%D;Q82P>@>HxTu4$33#5H5wW)BuwAGu*Y~4~O5oYYLQ9
zprC1-typzdO`sf0k}J`=xG6`coiCWaZUJB~ysU=@-pM|q?KQmaw8P1-i7(;wImyWe
ziQTY<!Y<k99-bBZCCg46e!?RWrc8Y$zOj4|a0gV}QevrNxzbXjXS3Nxx<JC&!1~Kd
zpz4yUh^e72$?1&=i)=1Bd}K{eI11nw2nx0Xh!nP8ZietAZ8TkV(Y59tjA8Np{H7g|
z>8!S05LY}s-=6P`x6dN$tJ(>gHaOe*Wk1VhZbBt{X^7Anv8?wEac@}&C<8uQ);e*1
z1L)=W9H?>JqjyTDAG%EbG|2nOj6eMh5@D9PH9HpQI4kNO>?#zAQyJdRu@b!<CdIqu
zk+hL|bXiIoCwuwjEmKR~;8CmNgIC^{;<_$N!KnIrmnto^frRoP)rP^DjfF9XVW)41
zd#q+uTT0AyE+GQiTyfK-_pr#!h~BYo{d7FYb)f$gE39OBGWlj+g=SoqV8$vEFUcb=
z_70b4l5HU3iF{Vgq;E#?fPyk%)U)?-lCXZJNDhy<Vk>FF*R-eDTzz5X9nnPkrhOnj
zcG^S94$*HKyBBjs6!MNflcSHVWcdxWP6mM1KU*lVAA%OXY?$)y$i#bGgTiP$*s@2g
z4k==zeB%I~Zj0IhH&(n4_8I_RVNTp2ERuaU#I>HWvV5fDhiB$;9dBo%CMSMGHvmad
zjQa+&b@L~!^=QFEU_F4%Pxw66O7RQ}7rE3-1+BR#cc}$ck~puRR0FFnyw+-1&F!OF
zRFDTsWomDH7sgtkaMrtZ_9XPcpyJ>e6GrRVaarDiJeh?Wy-1CT{JY!syF036G@#+$
z^ByMIc~42D<pTJXp=sCioOd*y_Um`<S6yH4d1{d7>yy^X&e_jfl(@E$?he1081&$P
z>pk|hqIHeXqMSRN`;~-z2g9O(<9(w#R{lYjypzpY9MRkH>J||_Vnu<y_5~gl;2Xf8
zQ06$~*mIwu5(NJXRE?%-q{6S&79&0V@jWDuO@H&11jAjh#u{?&CT{Ub(3Fidtn%am
zxrsFSM*}ORgOt&W!k{?`D<EO;VwCI?!^H=9PZ?WZn6M(-mP*W8$Lu{E=JiXGhLXed
zrG^2+=!u3cohy0OP6qvn01g*eR0-B%zxASA0Rg)LGw~}0<>-Ou%|lKJPHE^Fz>Qvd
zhkRu&sWF7GwKP)qe2#?Ung@!*KX&&DctK4W#ZjwIMW_`X4r6HciZ%y6`>IQipQ;;k
z-KpDt--YcTXj&h$mHixiettBbB#m4R2m@U{$Lxp1yn<T_#TRha&S=3MA9@IbYRBGa
zkCNd3>g=r7%nmyLE4ZV*ebO`nX{Zo$mCb1^t&Pm0jcl!@Dge7;NbR`{I6E(R$GK}X
z;kqGuV~Mh}DP1qD5jB~c&~SM#0!@DQi|Oi~fJAt}rD!t~-Lx*z0d<-ttQg54HB`2%
zWL(cl$fV9o2XFP)8)uvxWB3K{^yuu--nxzS;<0$C#bvte6N8=&!h6G1U4k%QXL4|-
zX3q3h?MFHFtS*Vi9AXfEaZ7Kx?EV^Ig*0;R{pg=W`8^|_ympS7TTQQLum85q7}xND
zg+B71e4tRXq`Z}C&5h8@;m_D}Qd2!eV8v&?V>9JM+$gs)^Y2kgD8k-X&IRWBu~sB%
zn)91<zZ9m#9bDR_C_t~wk~}tD+!GgmeswUn$1pCZ&|`FOedw)Kt}8=3T+p-^V8x5k
z3{odPR6%!@Jd>FNIs(Kn4BO{hqb}U-TP#NR-k`qxH7U*RuF`{n;2>kqqH7OkDn#A#
z7{utZ_!kCs9tp9-x#%j>=GZQGm9($51`2Anm4x1FxW`fr?H|#L>ID!imv7FXjpy*n
z0Hk#KWEpyKP2c0H!kQq9@LB~xPT!~dVr8mtHJT4QT~*9U?s2Iman$&i=iibNehgmZ
zos$k$U=p8xLpx-turN<l&J1w>F4DCDN#+;LUxbLy#ynm>yBjLb4^iBWsQg4rqSoW6
zDGp_|7BexldpiPNiVxWwtYR@+*a3*6UT^yQdc85-!ENiNq0+E^7cj$q854GTM~M0i
zUsUV5?~|90H?p`^btbRea(s}A%mSk6OKgDl^pcH(TyGS_ShWDxW<Hn+)phs)JKfSb
z&o(WUT};a8abT_n8s&w5AdU@;MA#IYxGTwBAntnRtE!MxW88_47gb!S8)aN|&MHhr
zKfR59SJ6Qq|LU_G9)&a#XV@vraGYf(4x9MpNZfv<LEds3nPp7h0%QeC69SAXPQE0Z
zDDlF?dh-&T{lc1tB9&SAHEIfwzUUspD9BgSQ)MxR<>X-JiKWYPs30(n@xzftu$a)z
zzx*yQlEZ^FLU(d;!DE@8x}S=a^iGk5`HvLgg*{eSx1N8_Z@qZp)@3yk2C6k3J+Z;d
zO!^la6V?Ap47DC8EzYttM)!MA2aeHn3$mcu=3f`CYOeQp1qv+If5yku90B@(F>9W#
zvZRz}66(565x2()d=+Tzl<r3YOtyT@ju~>$)F|t(RBcO2$Sc*)=G<=2=h*0gXBtbJ
z2cuXnmBBPEFnmvGR%uDxzb>_!K6?KCIh>PRLsgO|cX3nu!~Xhe4wlc>5<&;tUwCr^
zzRBSkv}rIWQLp##mc*)i9!uL!YB~_z4@LMPwFE22Tg^v%ACr9wf%Xgvm;@e8>_(#3
zTANb3fr5WZ(wo(fT<g4#^FQ<)Y`^ZE`ntPuKhj)sto;2Z?(1k86wt4TF1|bRby>I7
zj>ZD6!sYnO)s2y6Je?3Ug{f<I^YR8&yDV}fr~!RehGcgy>#CKT%DX@Vn<xuEPdyH4
z|ALg2gVb&!EOWLP&*-xkR;}MF%wM`yv{))86QwDXIAUF##_^1=sJduZIOy`C+K83U
zpp9I?+6TO3=Xu7kbTr$U-dn=Tw~nOD24{Nc2Jj3zl;kaAMe1WfGUi6wAOO|wS28uW
zYE+cM3h@slMlrPejcar~PO;)UsJEM)WC2j^+OuEOKNsf(_^N`w+yt$ItyjOwBnORD
zFgmviB+}C9&reA7@myO^Ue-$c-TY)PfMibA^-KXtrj9Nhi1CDb=eAE6**n7by5;ab
z3bIp%f!ux2?_$iX{9&zwt-36$#%U3#mBeehZ(}ACBJV3+GDk+FjS1y0I!gfxpFy`9
zmGF}E;@IxUPej@G^@D5<`sQPJfd`ib36D$nvfUaDl+EMwc(&}g^jTeQt`;<)iCO=^
zm|*pe57=$)UWt#dX*U{+<X>r+Uu_oj+NQKXxjcZsLI9Az*zk3T10aY7F^Z>o+>S;*
zidxUhhm;DWtCJWEX1J}z`1`7M&M=o9xYHCu;f#5M0z)E^!#%85&q%zQ1{sfMi`zol
z=GbJUSpbhQa-T>ls@&Luvm)plYrU=^a$#d`y<JBGt~WGd|BEt4IELq+rD!fOVID4-
z$i4E!zI@p6g`2B+r`o#({D6?AmxkdBDJkE7Z`WE>)P!+6{vf2Q5eVsDfQgq4MYAw~
zkS;u|DK>-W>sj1<X*D|6DD7NL>h4yJJi4ol8#KM4{Qi!3Zqfuxut2Ki@Q@u`&he?L
z{li6u@yBv`$Kkd6*)h{h4s)eG?~<K$Hz)kx4+~xdOs1hRTay;q6Izj{#u1)@L@9{Y
zKOKEOmBS~%&}jdaKjA3s6-uA~$U{5N^@x+xWb1dya6sAk!Le(r&(S%B&)pBKBsc+7
zx*EBcNpFURKs%L{QU)-M*?mTWRcJdI02@qMRVz*@mQ~}!XeKFsEfPXJVE8Fw75oCO
zr0;28eJ^a|?OEgYvk7Za*qfc)=$k;*suP?y7*NEbH$TeF>{Gx9zA)%C`t;>2w5FAO
ziZZ9LY9Ml9NoyZ#dgIR0NZvg0qH#_CV5z1Q>?6b$n~wk#t{*DSd+b-(B222!LrpKr
z*{uPD$9A&<YZW^43oWB()f66`7N59Ci@Wtc7^ESHWe9q7y0@m-cqh51&qI^|3jZ~{
z0r*x#cj#(HBmMSzHl(z3nz1OA3h|ckFJuYbA4L1C<a_q(s=gJy);uU)p1yWlclYsD
z7v4ic7z6OT*LSt{f%_zI%enrATMn6&U!oF~ztT_X_8;_eE#5Zz+y76p%YjTZ(F@B&
z{{SoO{9|3Si-T_b5dG=d;s(QCnR?)_5b6CJLHpN|Abj_+SlHL=|3HZU^wtieaP<Go
zv?g4s>|eRyKWQ(7FZPlDf5dGcCbshbXU264nfp1RtRo)_Mm%<DwgIdyCjn<|dWOi5
znu(4PMJPrKWF4XWdH0R;@}R5A7DPO6)6_g}9^|hS6+l-5gIn$sedJRF$-WgFoSCs!
zXqN<}xrly?7r$`Rw<Uz+kK0j6xY$7+KH4Q<Z`pyNsRS}L>y$aLSUi4nd<Y`1sO8J0
z3<;MglXh@~iAd2hzsanQEFFa&f{5*?sQJ|Oj=$LclR<r0IsN2@U$*6<xa_&cT_do0
zl*p8nkd6F}<avlGJ_}4ka-*fB@N_okw%YMbASQ}d`L}EFyqB=3W}u{~DOoofJvc&C
z1oT@0<7R)UR1f8He)a_RHb9tmvAZ$)d?Ib>fLi<rktqSN{*No~mR5W69yF^lOi)rR
zED+Gtp9YCu^h^I6o)`q0U1+GeRl2ebK#l7jfAxHH09z^%AECVBNI^klxbdel6~Nn`
zenELcqEcrb;Q~rkU?7m41vFD<Iycfhw~J$r5)p_3K+=Nb*QX_*oefF6UDb+af-KzD
zY9#nIWfIHnCE_#D&PRz_=m`~ocfP;skpw97s)`&_f8Cho=y0>6g(ztRWPenDfLHUn
zTaH*lK5XRJfBiEcd`3)BWvl_XWxEl$HS}rbU8^e@=-2;womOI*lG)<eA<gOfS$S5G
zMRI!DnC$a{AOHIw_^G;yWqKYs+rBzz2Z<#bY%0ujx=!%DFO-q3#$6;bEIE4-6qp07
z*x#_afZf1mlYKo5Y3CiI*Y3Lt+(TuiMTS1Tnz4zU2<>ZVkdp+6unhn5HFsfYa*2Sg
zmII{^15bqj*9{%jjSVxcFnDu1Y_tpC<<+sZ3RuFQG}%L>G>q{cqOY#;{9A9*koo;7
zgD5t7j#;+Vd8j!WO3ioS%s)8FHH1L~-TbWC3c#C~W%w=vVl`wVq|dnXUU+_k>`=Dq
ze%A`Ip?f5rTk1#VP8GT43Jw$|iO{f?ynj3xqE<GPTniSUXsl=><KfV4aZrPqc{9=(
zIRkdw+PzA2AAS%%O;{mqs{osh_XH8oO+ZY$KO7AKGCcb~W(RByy^5db%b+D@5|)M$
zT3t~bzkoG!6F|?CumG=iLt&-H>6&|JY8q5I(0m=C<5Kwzs~*=m_!gJ@jaL9Fozv(v
zc@SGj<2gs-&zaljG<g{y6fcDZiGQf4BuyFuwnh%;$<RvMS=LNX+d9=vK3D1+vwWmU
zs4~V3BBCOhU@B_{iCJPmH7ECdi~=Bq6^63X^n`}jg^0UKvGB69(;xlSENfG?iqpCB
zv`4_*RO&B3rS&{%UM9P~5I3GYKu`@;tL_R{<f-08y|7%O(0<st!1|)Y{26kHOI$-i
zO?{eD50#&G>@4HWtEhse5!khG0rx?=OP65d4-&V3qRFp)pRuqt@QSwTVHiqETENDf
zCyWt7$m)<BQ3asH^d9y$z~GE$*PU(Cj|;)v3o~IKbBm`wD!Xl2@Vh+M2VG7aIDQw%
z>!^@=9ljO{X$&8g(_b1P9gUnT2RM|$)}oo#sNm+&C^I!*7+x#hPtK_ZAB;Vqz*X*e
z(jDP{O~)i7e}<^Y_i*F-aZ4`!uYiJ>Xtmzny0q{qxnrbc31s}~><==AC8AkTmaimL
z*GsA$)O0{zbKjeL$CWiJjN}P5_LHSxE2XcRqrG?7P0CZQH?vpJ)t>{Y`1#4`@N`;E
zPMRD$!Z}t6>+j+@bLE6c>};@3lTjvVEMgX!doYKolw8|}7;JVw2Vm)PhnwCzg{{Lb
z)+($t&>GSEZe}OJZcS~Z!`zwajIC4af(Otb_R*w%2(PYdQM)iO-M^J8zSsO#Ouz=f
z|FWskLGM4+K2e%>RSLH3Aje_etoyoD#>0J&R>bY@Cssp>`B|ESuTDGJ4Aw7<brU<4
zyz4jszV<hbaN~{L!~Rm_`n7S#AW91}M(B#KsvJt2{Y0>(!b;zTi@h9<Uc$#VTlpgm
zDqYTqT;ePOgfZSUa-Hk(in(lQ4+*m;#GX$F_;ci9oG9*M_&lho%Vok2_jS!E6MJ~^
z>c+&VPuKh^1r=w6?|IEe-s=jR5|!K*G>AoZ5h=w@N1~7OhmvmXA9o|ZevKesxIoM|
zlja57&q-8%#11^LyTONlWB*86SgJ`Z{^Z;Hh3)*9eLceKKQa3ZVI+#U1NU|2jC^V|
z8uYL*EeeHcyqK^ULu0gZ<ypXi!0U>)lGqphW^!Gm)Yah#2~&INF41e3?0RgFc-#5h
zJ7jJxPw|@yz+8eciQA_o2*m?0L1^P4-#GjI{Fe_g>1Pi6mOGSZ&FW%-B0*;y^C^oe
zs*#GTI;JuL;-_q)JF-_4S9r<Y@}&rOb*my?lyqX;;G!H!>;_0>03}OZF8M(8Y)5L1
zYK@g)dJ0k6J;P0zVTMACc=k*45KF!(sLdGr?D~q;Rt|dP*gF+qO&^H=eNCs5e?Z&k
zyl(<rf73~l1RqP#)Ex|;0l<SNCt<?zJ9s#=Req!*&(RY=R9G1rtMv%_pVy35#T9>E
ziMLQc2D^?A7;Y~orh1-kl(kX2!D~1eH8_hFJoe3(2>D@(7-?lY-m;8WYC!Lt&*rE?
zj_U@{G)?d6q@NK6B$Q{A%2hQtP3=be9U}<iH0<y@bP5&hnu|{S(DIy)!ZS8<k{gbH
zVaj1w90JyNn%zXi%qiqG?wOQJz7%gSn7@7M_;ggIXV9m^6TLia<H=ag4PO;EYhL#=
z3fA#%6!Nf!x40RzP$1#66|7vR2AtYeV=@dYPjz*2^Zx3sG9W{)MP@i@+S)eS=q!j$
z@HqiuzA<EuReX;$uj-Q@?b`84bzIcY?st`!6L)?M<@jK7X)x&Vsf5ogSdfs;WH+8~
z*S<VOL*r_O)h+M=onPf!qoiY9?Lk>xUDJWyL2pqrXeeEo#lRwyuZoQXmRQX@-+{O#
zH}`BpX37yFp2Z(_i`=~Css?8PqQ8f1zA((7Lp!}m{-p;xb8CV3LC^J%#UHkSldiY$
zNA?cQj}eA@xZeD8xW^@(hqVi=ULEG5z`TGGqLBMl5V@GL&B3mb+3Y5iae@ao`R1}7
z+T0^hgyNap+7Yb<Hiu#6%66FV&T+Ln0#dMfVxTbm<A_qW#2`v_LJ74Ehx{WZ(#gNn
zJ{5`7w%1&qx3?JgW93h*q91-IBv-r-z5mc@s;`Ao_*=K3-6*O8f^#H{8H(^7TL$`6
z9D2upO$5n4{M<(>M6IuI>UOkK1LFzebj}u!hxpwAabgk9Ah<ZZO;B0AJ1D#`5DRK$
zsXPd@M8x5=oEVxNT3HSCh8T{V1!ZKl<)&#a?5+jhkpMX8N#q0de|ruy428Z~_AaXa
z{Pl*#(hT(;%@uQb&B!^RQLbxFl@QS8DQSOVOgQ)^Nb<Q8^2D{E|D8pDi|+5kMDC}U
zTp9{GoE0M5yq_bE9k{CVx)iwA`?$t;k}_^ISN>9Kk9tK~Vl`2ry4sZZReiJC{saEe
zK-LAr3;sbB4&E#a+@}*?H~_nagc>4%l<0-)kMlPAi}+*~Fc~Sp&Y@vQmaB^vv<-4g
zm3J|gyK8QwdF;lXXJQBoZg$(hiVTOjNCUFdhXK6pQhUFkuu~;3B&{TLC+~;if{!9R
zOVZR|y2@d~hk;f?XX6WpQ>=Wm=cPw16wOnB1`BoBMuEJPFfsj5gDWGYOHBsv#nqzI
z28~GKO3a@|ydpS#+!`IF^;&zEYN|`ZDDU6$oZlB{Q<t9*_hoqZid3RVHj%mW-N;5e
zHR^)s5g%28W^gs=`#R$Zgf<fTHuC0UbsH^pq*-X9`!=RF7`f|M^iNi*x_)AG)<t}b
ziK4u>e#?y=K<Qy9q3<j6<HaqIeH~6ii`AsUuY^g{aY<VCI^z0VNw4}JtFeJv_?Lz{
zQ{MI+JC#7p4>aD=eIGCN5>QzIR+`Sp@)f*RT<~b8;b2D5GtfrP3B#IxeLgGWdd*+l
zz?mZH%{e_dwew6LX5?ST+9oD8A*63AT%k5MV~&EvfOO6m)shAx+G+hr)wow_Kfb>T
z`C<d~kBa<CvTxrCObMe&d@7&PDkhWQrzSX>@Z2Q@YvQR6=KJeI{dVdJ755K#(Rx>r
z)Gu@@Q(d#|5JYT#^8-J>)j9*`_l=DeMiiv7`T~y<C-=wl;oOq*4`L$PjWlWm&(d^U
zlz%*O9%KR>AJcTd1sBnuJDMoZ7x&j??dR6WI)8#0sm?l5+1NWods%m416euWVsEeN
zMX|9N7GmRPmM4slu~=8AUf~vbCJlGwwb|g+UW3vJe5^j}d^dp0(?G5m<3!%Yu-%yG
zw^oR+5V7E1%+`FVefPVm{EFIFpHKsBaYwJ4LxL!ktjkiZL+d3g4P0gE0{#7TopeuL
zS`YKzilTaR`%K0DL1&dsiOR})oK_n9Uo$Dq9VVB`>4S8~JaazsH*-mE%HXL!sBE^n
zrbGWqH`uf<LVvhfc#!b{T7?w&v|C_!0ijmokE8`+FSbg;ZuqOtKHb3Qr@9dUec1Sz
zWH{^$kH3#ZS|sdTtfPKdZbg1relF)J%c4R1p0xbwS{})c0>9H`^G~e0&RGlr6l}Vf
zj3lGVc~{z=#}WZW1q&Nc+IsOxzqPQf`Loh^d(|Va?XUmKC$>%rtELPrc@K`Ql$lW2
zy)L%s7U)gGnvrm!nX;ST&|e9(Fj7{3j0IXV<b}H-<>xB!rIW$?>3O}VjFhJaIAn_T
zh4ByhNnFhyepWoNE|$F*q+&&0*z@6rCc0aKIh!3D83SLBAj#*=%uo{7KLXL?Kh4ZK
zMRVlEU6Qw-43tc-Xq_J_voZqfY8XBt2~kK1C&e)H*RitM6%v9>N3mO=jpC%cWiyof
zt9GB{@Z~6hwy_4%Z0mQ)2Ls|a+V_Rp*5hsn-a(5S#g;a21SlBc3)a>P(xjm@>XRN$
znS0gVHK!5^fteAjLBGYQ%DNb1czGy`eBlX8QlZazjc$QT1C3|*YE;pYkP42{d&0PL
zQfMYlI}Im_mo~>fn`tbTJ}`C_AVPhV+l*51X{e}~z5SKZo!GbhA!oE&(i2L6k=6mT
z?uJ)x2!^>O%$pybSj3u?*Xn5`<q#YwBi4)qRsYJGpeL0_Wd6k(W2IoJumT6UM}3_n
z6Zat1pP8(U{W!l`Dn0Qmm%w%9m7iEhf9%v-4CA53%{<od*Ar|ZKH=1zJZ$#)5`887
z70%P%X-yxs;>gfrSS#*9DU0bR8s1<^qp!|hgmFzYFUUPPcx!nsYw(>}%}sW;N%R8S
z-o$~!2K7KDUOw^TdZAUat%U6W^9Q{aU@#D%HHfo1mPz{iLd^sgYEl4&r1i3Oe7#|%
zflfTq0>?ePz`e9uW#dH8nI05CX=;5-;N}qfg?lXl2fTfr#qttow9bS2k3tjMjh=TW
z)7)kDm5MTvPD4OP^^ujM7G;L8N^4rI5F4(f0@}p-DJA`mDQ((IM9qihD{@Pz@HbG8
zJu8c%QtP6Z8P*eLlZMtik)x#bFVwFSx^WOq0)_*>C5VE}&m>rVD3rf)>rGejtixq}
zxP4*0EVLT26l=CO)TuM^coa~DBlyBjea!Sls^a8h)sL%fK%*zIan3O|m~mpd5*IlB
zGi&(~y@yu}(cn0bheMioE#z~8i?LaYXawrsz0z7@j1!_OYe%3o$zE>aX_tqhca{1H
zy|DSNq0GUIE^7uD-ftnynnrHRSXv6&eqZO&89mCBi-2}RrQ}GL-|r@<gqfr~7eK)@
zQL|59jJUr20JUCF|F~2rvDjmR{!;SF+h-a$_k>HTt)Su3uW3>UeF2w3-xUR5$DjzV
z5YwKosFx$H2xgcP4{;7p<>I9eecJWzGwJ2|wRBB6)nkL;zVqe+K8m2?F`ln!T<iQ@
z2ZLypLCsOR*5VJ$ACfBZzBp;^MR$!YzAt-!iI>u}4507CaeF3B*RtYS8NOizjcItS
zf8VYUKB}$(pA<s<?b9wWP>qq`bDakfJfPP$-fbeR6x4pCq=+dtj{#RX?eK)m)Lguq
z`HL*c_`S1FE0vdwCCCLHMf+5ZTZyg}AYNN~cdIX4te|&FSP_NMu+N&wl(MxU<<iI+
zm}{Y8^Cjhrx~L6G6Y1_PY+iG(M+0dyH-1OOLIS-frE36!FIa{d1Q=l~O_JL8?0r>B
zQ=-)vmmqX3{RhgS9;y%th6fT;jQ=oXFEwamXPY12+$A>dyW{faDI7gC*8SYKpHA6p
z^+x5IwX5d~r0*FFT%`RrlQMxDjs#{lg@b0tb+LtdoaE&;^^p&(L{?5m(eRD-Kc3=&
zxuU)Xi#=s<ytc*WFb85Fd5SybC5nZuVyblB)A8d$Q>6Rz6~XMR-J8QWX`PPVj!epV
zrGuq$7g<MAC3IJ{fU@Y&6`3XDfaQr(my7z=Rd04w`mUzC&X)wcb*m)a=|%0`#yWOc
z7t}T(F7;U7qw^HAdTQUl(8TN2nJ`nxxPv;%#emtUtPL|LTDUzwDm1+Lb*;!a%Ap7D
zYg1~ItDmHPJVyM(%P=T6YGc(L2pcxU2ngbVE#TY!aLj>#(w$q&#MMNfFfP*_m;;jQ
znEez3bRgkA#IPyBIr$Sje;bYI9SL+WeO0qN95zMb2(Vk|^hXpq&zR<yW;k1z^18)!
zLL)LBmU$%T%Jk;-75ZDtd5o`kP4jY2_IcxsWOK`W$J8l@LuHA#{u;K~L@$~Tc`dS*
z%d}T5CUK@(9N`WW*BUv0-sVwQ_nPbe#Eyh7I=p-vSEC~O7G>ReuEoz`0Fn?^!g<pL
zwe2|UC<1X0rMG2>GfTRXA;~y9@1D>3W{_NOV>J`rY~nS~fn$|s!29x<GfFLNV34j<
zPBq1_UZn}#$xp=kQG0Q>PFIkdLL&pNP@o3!j!9~B7nvnTFP3_C3ACam0=jA(Y{vLh
zlJ-)81Q(RSoRIn+Ay%gO5dtp)A+W9r+exrnE?k^X6Gf{COFZ{y7HO-hxHU26+=p=u
zD7HiSG%e8GqzsG&R2<a=`tL6&bJy@C-_Zg_PL-w+>GyhW*C4HJOuDZ7Q@#O)E|e8N
zk|1+B9z>b<$|S&DMo!`BORhZmD1qA_Sp?+2NSA8P^h63!Grgfy;-8dH!@m+Y3psz^
zCON8vYs2;ACClLsrDNpQIzGjWLWQxE=Nld%N+iQ@VarowXQYN67kg~1@yol>bFO?2
zNZJ6##xJw>)s^PkhKi>6kmjTOv97b?$?d%Pou{0{92pXbab^<qvK2}7&Br53t|qZ;
zId3i+d9WIK--LQXXKb)MwVJ)|W&qvX^M&0d#-+7IZ+*E+vW<gJRW+ZkG|UV=^VZm#
zZ+|wHeh727E+mZ>`0l-*IKaQtHzx+WDG`Kz{&F-L)#t|Ul9kSlU!Iu`in)3SH-6zj
zfE%Bw($OVrNq+@@=-q(Ma2U^4xr68kAMU=Ib!cg~Jd2l7cWd^sYPZ#>G7yGoC+5?S
z8I_tFsgsz=1aRpPp&uhiKMa`$L_|@Aj1iN&N*^C4=A6W?g!#LcG<sf=sk-i8W#cWI
z_E&kmG!*{TR}4zJuqPu;R>`FRhvPjgnsvaHqs+br704aE9T(31Infd!%Qzc!0i>qW
zE0@ldViY!<c`e$;a-=;-_kEBcbYb2JQ#kKXvDhFKP@rHXpUatDmN?^J6Hp0SEWpBp
zi<-YwCS3+V>tXCUU_V_xwNQZ8!o5e^H`9(s_P^86?L>asP|`k<sW#MJxUbJvXiM(4
z$g#PqwYE+hxu4gw-^PU|HJ-=k?yZ*LuR{a<POq)lfjx(j_O=0psYn1#`?ojkt9?2F
zfFi3)6QIb<1Ss-5&1&n#AwZ~)#C^-<Q4MKXC<>l^LpIg6KDH5Ht?+yf82OsPAxIdD
zR+<)luvvPrSafXQy;L}5iz7X!v1N+DNa&d{9Z`U_tStknmN55`)o7-n%wT(FeF4h(
zwe|&)*okfpEV6gIW`8q~vv-;*Vv=&us_SSf-eYi2D!^W5*25%;R~0r3wYigyy=ays
zt=sEy1?DU5j9+v0M@`jlrnuas=BuB@=E7f6+Bwn<w-<<;iMpWh3x#m6LAA0ZX?;Nv
zF|0%WVn71->|Wa^+{K-Nw?%iXO4n~6i9u3QKbqUmmsz#<J>jI^B}Khi#M4r8)@}br
z1uc!};GQ5iUeF}Eec^#Uc;b3(7^&?_Rf&LTRf~AcD;x|MGPlr4c~F^)FIh<Xye(c1
z9Vvjy#v`5F<}Z+Uf8LJW@i`p4z&!ZET>3|D8A-l<m3q5ttLaV5J1at>GBhpwk=^=l
zDaw=M=8G06GE2r|Qt$-Rtqf7eHTA>j+Y2Ig5PY!X=5i<3H!Ix>u6wqb+Bf;c<!Xrq
zgo#daXTWRA53+HT8pHVFtC{sMgp>9TZ3V5q4W+ZYwdCkXFIryhjp<A$-t3LY^=kom
zYI8|#1~Zb^18e-xyvvRJq2`i4P=*DFu6)llW#(`LdF4UhIj;R7Z;V+jZgj`B!?Am&
z5MPL-mr1mJRB;xM#2_sL4nFz3NPCQyj2q^jXFUteBmNMKMr$3T5^-xGCfzyU+d*Qm
z$*ZK5YLgkwg>Y!6Iwkyg5(~{*L!1q`b4dX$gC9b?7(z0eCuu^f^!Is^rX<^wK0`^@
zh1VRr7vy(DrV6p3{C75bYwI7di7YPcg#XYKtTtRDtAJ+m&<9ypR3SF6jYI{<Zg@J4
zc4Z#aDA(jBHM>M-s^B*z7kfB>nK9p8!k_u#1!-}y$dN&sLOnKxDfht{SdtRtP{c&3
z6RP)_&!`mu7V+AR>OGk)4uBPCs-L?b$)DFlFN1bq&&#BcS0SWsO^z#vm|PkT0zS%m
ztZK=*3}3D&N-wL<%tm;3apNRqR8ef3;G3_>HUavkI?FwrtmQR&L9f_1_&Xhwm~77T
zE{Ju`I?-?PqaG*H#U!^PWnH~|_yj&8j9^^N;_$txD%GN9#{hqw_|oQ5?4xk;wtVT`
zqL6wn-OZS03Sl|Ps6|}m07Bj^;wxa5WD{xELd)=A9VHT1s}Nmqu+iq5>qDt-iDOMc
z7KOo`nfN3m7tTy64)=umqj9IWT2hyX>HNgd(Hgv)>81Sz_sn4KI?}?TL<tqtey$3b
z1*+nRUvu07D!g<8&hE_1^uanIV@(NyizOAoO&9M^ogZyL%$m7%tXA$}g`{}!+NiyT
zg@yp!<qGo8rD{o<5*N)IE)>1}r}^iS6PVK!9!-HdGLc~Ss}0j5kZF5damY?|!RHYP
zLx={&@tD6Ssc-Fwy2k2eIRvBuHf|ophO$UeC*jq3(IXYkdDWuUH2V6aN#EBJ$~1&a
zPW|mct9yoMPGq&)AZMgyT)kHqrwvQsar*#Un=xaiYFUN$uqY@OjmyB`@}Bx8V<mW2
z97~CJ97UepEDABx9qLp^&ND36RU?D6T0JO-BRAJkdy))rmC=iXLum1RgW3JZo5P}u
z6Io?RCl{W4O`^)c13ss9m-X8k`U-C5YMY%$3#E?fZ-}{fJg^H}&r4<j^JmA$&dvx2
z<Oe{Xhj<Ly93;=Qr#5$uX@f61OSD-;X^%W;rV*{jH&j79pz*kqQQI`{@oD><#n_z1
z!2B?rr)xRB#{_Q?<vyQ^O>5<1Q6sXN1(m-HLpts3r&qJA-U+5!z#bOSmjw+6ewg|0
zoV05Crm7au1im4wd#J)$e@7YjOmI)x_6Sg)AgG}-l1^JvJ9{%7S3cM6!n+*~1#BIV
zntRa@ROq3*E#8T_9fkfja&Li)YU;c$K3S@B-`+O3@Cqbi=&^wg4c~wi81VLIwI?ZY
z#mQN|&uBVKPN;H06N>v_3%BdbT_jQa60>~4HtL-v>b^H|X4i6)=WyV#vjQs}>%E(;
zQI1`Eb@{>8X=8q(J06cV#H!}Sh0nV0+_7FhCyDOlw7|NNojJNzWa#ZJse)&Rs_JKY
zBtq~D(4F$$_OgA6c5x$ob$UCPy3$&Pp7%AbS-=N%u5FlkP<Fk)hTR4nN-5fp?j3Gl
z?U*Tq$A~+t@3@FP$k&&!c4DCMBh_#O15@Pe9Kkf34$(9G4x`70H;Z=m59e_>ru^v)
zk8EBe@cIs?Mek|yauS9lwvZ--a~gjRNG_5Jcw8Y?(%*TZ1QSu}0&CYqbO>?|C9`Ck
zf4Hr-+)C-kEe$rS)T>mpSmqAYy+P|(W_T<VakRh&t}nMoyDg8_E&UV)Mdt@2O?Rkh
zFX3H#GmivvH6R7GdXmcVnUrY9>Y=Y?I5;Tp)G!zHtxXutL4X1kxgm~yVkNN%;9t#Y
z<gq(5RM&XtF<H$WXz9W9NI}(|#2Y6pmNOU4)pw}-8_b~0YmNvQS2tx1eHzZpbt8de
z#|8f!@O8(Dl@tc2=qHTxN`sCg+S3MwS}a9#|LG%`W2Aw}-_Tm+&{y#Lb*{J@Jv;#H
z%VOyXJQqS31O72x4@Tg|EY4RblAAoaRR-M)x}hJ@l&|~s*@JI~*w>UW%7<Tlm0B2d
z%X(6aDRGECt54<4lve`$FE=_?7h-nio~MyL9IDImJ5h4VQmy?d$%mco<?=h(*lA5k
z1GF4IrkOHMe?xeEuj6YWsk1EIUCIo+jmGO_{8m=4QW8tM+5KbJ2GM3G%DFaqlU2az
z_t2&w(8OZn(i(sJnMBAaRisGHdW4~8N3x6@&+I878dN5~K1|)ibfE(CYyqQFI*T;L
z>j)n2TJJS?t7$Vsb6cm!d5F&gih^TD2f<IfYFxGY&9+{5nC}DWUzs24&6El91r#{^
z9dTpk_~va_-w&4A46GnPmVti~=L-`D2ux-0SSOI^11s(NlC)?SVCY1M(jdpK4sBQK
z-jv%&Cr=uOUxih2MDH4J7;G%qL#Lb05uDae#ocQbh>4~UyxZZ~Pz;|=frn7yi>ES-
z^6{UJTOZi(CDq;Gq(9y__LaTVo)TmBY5&~S5iWfBhjFWr%)O->;_y!b!<p}s8%fpp
zT?%U>@w5Y(dlZ2aK*OOYkdMFqBkCJjyS9ihSxE39X&&stnh=Y;1g50(!o~+3u#6i3
zuoD@g2sE3&&<x#qc#xHnSpFFmi+{3!x70X2omYimQER#3Sk-kFh6cCOyM8ONzE1B@
zFgLRAW|Vnh^whqKQe0LaLn~V)Zz=b}e1dl(%H?C&gV%{S^tm~o;xF7ITdla^t7hK%
zs+}{m{K#8m(m=4eP%h)R?0JhzDZtE}2rh{c$la)X-KvA0_F`3ZdsrJRxV=cCQo>Z;
z;<)PUsf20x5u$s?zbDFT<HY}ZyTk^zCCC{oHbI3S3ORawL&JfVNL<YJ+v*;}ox>{6
zs`0lf&$@6)Z$>`svyw|y{6;!d?2M7cPyX9WMGZh@^MqG-5@2?%A)ITWzkm{uVIa(%
zdH<<NB+u^K2u+Pci$6*j<(_1o@}cqRTl*EJvOD5BLg>^O%Jf@qI+JSui@o=bYHC~G
zh1uJ-6&1Ijs0i5TO+Y}pfJkovM7oNAbb$!c6-6vGsZv6d5IRbgZlxDVC_*SIC4^oA
z7(%&o1yn%Y`+VbfzkA2H<M78hTMa8~&iR(-E%SLP|I8y!7s-kkziCpycxb-mq_g(7
zBz(*O^xyL{s?kf^DbD@<Ef%s{2*RA~=(SWVgGo@zDMCGmR{9J^hbRuLmUyf*kAkMW
z3$0>RYfX8g*rv51@RO_tMr6jh^#z``n<KRe)Mr!QeNZ4*&#Kwnv*Q1dz<3|KLUkYc
z`Y)gUsi^i(e?Aogb7N_gKbRG++tn-Vq@ZU3Eq8g|G|)TJ6_Gh9j)8s{BA7NnBejNh
zp~_BXa_B+Oo<V1N&l+kWqNu*RNpF&0P<40}aU#PE9VuNv`53erZGKKX6}eq%-7l|8
z??L@vfXq&6O^Vm$rO4eg6w}AxQ5DcOuFLX+ekbrdF8f3pjJHt4|3$8KyfMHM8yTx7
zmOMX{&lFUzbXDAVrN0~}(w>m9WO}BzP<T57-2$M(gL)5c1E!HxAtR?kuYhzU@2;5T
z>F`CqmoD?j{<}S_w_NqhGiR2U6Gw+s{En`6pg~-z_f%)vDJWRI$tl0nDDD8GV$j&$
z<-bffq|kAc;Q85Wkv(R*PM;?x@X?#<X27xci}hJtathu3%y|34<PPHqT<lIG6b^8T
zdd*Yw&px8>3eLV(?kos*)z{l$hJID`i4S6!CR3vX?bt}T{gf#WkX6R5Ot%k^5LEqr
zy-^+6=xS-y%-=$^*xc!^ICrV#;b<;m^NtF^8MWw-Zo@UK3H2IEJ1T;Bo*L`QqS|s_
z7GTa!$zA=4@$ZayLI@_7I7~J0R%xqR+~V|>BDazI?tR&kEVEbM-hFsab!jXzyP%)8
zf|qKh#!m7HRqs+lh5@PAeag=BkP%}|s}S0vH}HunVB<EzWOspC;C{KUY5*&kyvI>)
zf64tn(IKKzcO&3iKL{<`f1FH|>_6=KuJ*t^)y`PY&uz#*3t!wYeq2#{^EiCE`PBz}
zNk!1T`iCDZ{MN{~A)))udN?ft7oofdKc#}xpDuOmos{*ekU^D?EvTqD*>cs=-VFWF
zkt4QS%#00w1T^=!|LPG@0Uy{}yn2N^NB!Un1Lf9v<xF0)DB}mJeaSOo^TNcL?B1Xf
z)ju7a=;M+OQYuL-f2x{Z%J$NY^=fRKi4j|Ae>{Csba6m>i12XnOIu<8<IjstwT>%z
z{hT?^*yc-z6GCRQ3;Q{2CkZGnqU<fXHWSR7{KtXB{%*SIuqH9OE>9`JH?*IriIa?W
z3IZROn^vYA&5X**TL9mv<l?cH?X?V%2e}+y&3P=|SsWr6M@UNQ1L5cQS~Mkg&U>*}
z_yrD${AIL*KQL=VZ_>zp(MhR%&b-+qj?4-!IT{(6i}aNfXLEEd<9NMBJHDPE5)%s6
zoxkWlu+61YIWX=jH@~v$;I@v_pQXw_;Hink>4?Vg19s*kG?a)oZ(RB0A@dHr17LbJ
z?)vII(*<~dS1zFfj=|G->e?Is%w(-TOv5a0?1tT{7D0&sbl$Cu$?vg@E18Q+6;YAe
zoNzs&RHhw4za@5z>Xc^87-g>uR+)DhSH4(s8om(e4s@zkIj^yBU~%?c%B@+sFPJU(
z9=-74{2`^R0*5j-1fas+Ei5(;nv=C_DF>$jx0fd4YFAugo&I<JY?HmcdJzMbMC+bm
z|AJARQU29B_5yn1y}CW?sbfFXsA)Q>pm6<<drO%rzWPUYe;uHrg=k*GsU!ujyal3e
zoPqE?=XI$!FxeOV=4o|Rw6WTrm^{Ef+Ej~S`&@(5kvL5yAHq;e!0X5E@y%g~E<h|{
z;yf?No$bKl5-{^CgZCGQ?#d9~-FI6VK~0ZI7R-2P3ob`QX7Q<IN8|*=ZJn4kZ$hC8
zB&}+En@?U%&|I)ucJrEvx)*5>o{M`FcZAm@Npslb^seQ_Cuc)d4c|U})S@Gp<d_X6
zaHyn)%~IOQMu}FLmG1_(RL>;oW++)X>XSzUDZ4Gc7|qgp!-L7c&kgdy9Qi|(bE}Lv
z7wq_pv)A#RMQDRG@hfc(a^s-!M&io95Q}FF6EOU7PspUYqbr^Dai0p~s;gE;+NVD^
zSx)2T>{p0wE;pVWsZ5E@SnMErsI4eG&!DQ$Ar!+_E`=NrJ)O+6bzzIkvZ!QOpg=tF
z;~sNwr!iFXVrq`}$*QMhw*97>!j)$0K9-7u(b!p{%IT#~TROh3cn&TSJ+F>y%G^^u
zGL8^uzT|5Mchz`x^YemXQ=P8aM_Li#lRci;kbK&h$<au?0QDjFvM;YqqFS0Y4(;?>
zmOolOml+`D7q}=quBa)uLtCgnAu_c7-o>eXL$`FI!dB75TLid8eAN9(md#w;5j5x$
z<ia_6{M;lM8yTEf4Fy<#Y#=#H2#HI!P71M!3$a_NSCI3>P-asS-cJ^vSC}a|sR=~p
z6_3}@y(7lK^-_u+$3GPI%QX(W1A6co68F8+_B9Fh+awRG4G{=$)DTu@5F{Xb{aCLh
zVVBB!T04)rBPZZF?HAB<1fP8@rX>r~C?3$g>yfS2_(a~RY)83V?RfNDepQMJ7$;Ds
zy@b|}P;KiN06d?w(zmIFyV87(qoU14hsac*pJD(%_<e)?f{%vdpVM2f6G7WyK%-@?
zt>r!~GP*sqfp-(A2F6|m^@^pUE8>9jnL6``0o-SLJEhjcx&(rG`LVbaOm?l1rEvsL
zr_0QJRgZ+q+n*A7;KOl+6P+3-cOg8Dyp|V*<93GA#?En%ya2OWZn;z}-f{Ct!}Sv-
zm%JB=?X&}QJTciay?xw=MVK;WzASl6sBtdV+>*?R;bL(lpoxNnF`h%I7GF<JO&$;z
zFjN^A6Ql;S*OQ;!PeYG0D%0Te-yTD`Vta}JHJwNIH)^_TLP)y%=Y1V;t~W^U3r<PQ
z0tP!+I30UVMj}>Y_Xe)Up?*~APK1{6QxorP^Ji>~OF!@SAOhBSJ?kEMPCt7!>ULU7
zcZG|~9m;BauRq{JF6-h@(T;K~rs7V<!;JzV)4r;WeX+#w#p<H7=%xrT!N~)~r&Uwu
zD1%(KvQ7PH;(0k-;Qoi5%abdC_YSMTZMeRMCSeU9zKJ$;Ea}ZUGy8bvQPDg^g@?^=
z0263d^NGvycJCSJ3NgDYf=~ex?0Fg9SdD#~-y44zPso-Gvl9iZN$q1Dsfa+&g{w<r
zL&b^};-QRO?&Iz~ejo5u*^ySfslD6lyapG`Q7f7%<IB0iM1w&AL?yL|$F#kglb#nx
zoO4_1N}g9mc*P1Gu3S<-oPN(k@0&~SqeS&m&kgm*UY(50uDhSA=2hAooPBoW$q|vI
zWy%-Nxm{csS0>YOfi8neu$2yCey-=#+lrggg19rT_f+{C<|X&f6)Uvr<Yo;)DZ5XP
zrvN8kcGk5wQ4lHLr*aUCSiNXU_<Ng_scs2l*V#u|I4wPuBHNIih~aEh3@MweZBjpJ
z8zD;7>zLP9MB&?J36FL)2P}V+NWx<}Y4;P73(5R(Hq1vC-q%KO_u%q!C(V=-G@aZo
zmOG@cz%N!@B1*q;?~IL*JZhGNvGa<WX4sl-t~W%GQj+KuvTI2+P3D|{DDMeA7Bypd
zK+4~^UH@;9hBI+1@?-*|*r;(=eb>}>Wf*AJ+%{9xeqq?;%@Mbi+sz6Z`>u^Mo`%&(
zw8xVixj>)jyP=tWaQ)#sk+(F9q>3rf^SG94IHAGgJ0wTx?oj79Da`KRVu>H3Ixa!1
zvYYk)b64~pUtrMuvnATS;xJj9aMwY~=`ltYKXv&o&pjy5&iT?J85ZJsC(Cix7by@k
zyeZeLh_bk@y<9?M_+b$+K*L$L{OR9fvt><>K3F6ZiE>{a9xZJSeY734`-&~LD65*j
ziF&_Bfq@GA|Ar3FUEb_D3D1^5x2p@2X;SNL9FV+3?B6DvC~zU4Nh2Z-DAw^1K<Sc{
zq>r7dAr<!1t>F2u1Oek<6ttkW*W{1)@ArLq+nDG+z~7uoB=GinwZ~lGAt>Bf*+cA%
zkYs6kio3PIZyZ#y&`32vE4VfCJQ&4c-ali!T&!hR#>H5iPTOa@G9f<CM6ArD^5z=y
z>RC+C{8Z@cP`Iq^vMh|i2@}5v+4*3{jC}EL`XoH{FP1Me&A8%Cj6$g1&Re{imgtG_
zpK#Zfu<N1RQy)tT`Q)*;fEe*&KJtS_fy=zymos0Fc4cfecHqb<C7ugt=$Cq#jJdp7
z+{7pmfT1rNgmZvke24K3jykjp;hl0l?lZ>`t=9fV6aLq&R!Rb2(e7(~qdaJ~c8lSj
z-Q%JqS!kJ#OS^52D^BjzctNn$!9=9*;!6JndWhGvTYgJQEaGKUui53FKq_h$%bCIc
zK8$)H+Lg9rUfZ<qNo3H?BPDC&$NCm&oK|TZCtG~cso(<`RbSLv=lHK4T#A<o5P?yZ
ze{3tfPxt;L)VsN}{HRXbwWhM|Eui`L<^l|5mxTU2a^#bNfl-rziN#oXWc+xx0@KsN
zX>{z%zU*%Vk0S=;HtfGPc#3bWt166FFMQRC&`0a3SV&4Ra^w{rasRs<@WP?Yq+i`X
z_k*GLrno>4nV~lX%z2`%H`ccxGbChR|4S>lw$^r34+icNB%;bF$My9h8q2cawynsV
zePYW%_ZmfFN-?`oIv3{~<#<-xNY6!}KnE>F^_$h<3g3!1T<}_JjuNxMArYm42yP=5
z{MV&|Gid|YmsdxEMLU%t*XSKTKWX$@rTvwTuO{D#%eNln(lI4o)Y*%8rNDGwb+y}9
zl{_Krr*8s*^ShWw*+nL{Cux7RFQ>yiszG&`>x`+8VC`QW8#syAD$sAv%pE83L?ATB
zj`RHL5$FC(_rBqJ(l05km*<YF^?&kD)&eOEtK|OammDK?+Mq|oIF!u)E6V3wNx#|>
zB0CfQgYNR_+rl=}B0Y)dLf2<pBnveUDRS$y`<sF7zY?o#Dwh*-^P=dC$Q#L8$KcR}
zo8t}ZCfrEKk<mEw2$?9J&+_}!XT8DSLGPVyJj*S#H2N1`j8*d(7<~Z3e4BHX8~a-}
z37DTMdR=Zg;~Pr(rqp@;dk}k(npA^<zURJ=rHQ2*FD+pnxGelHZ3=*fC5e>`g(x;a
zif?+8MnxP7Xi4AGaf@_LEsJIBMxJKl2RxW3!?X45led|!Uy(&cCd}18AZ6^I*=>8<
zj!jRtT26i~hI5PM?|eX@xt%JdzS@KSD}nc}@Q{D;0Y51~FQ66Xv~oF7KAV2Ka4UAM
zQQQKLAF+jQgmgzYJCywXHp&JW_|`uR>f%m1IfLC{$)5H`yz`!5dhU%9z&Br>0{G@T
zO90_FTK`2=<=C0GJpHN90{_7no1KJ<1UJG?PyCEoXw|Nu5|5><za!Fa!<V4BG&>o|
zvx+h|qP1@sHvbPb^&>`fsewDKMwhl+V3CyyD)M?I&qOk~JKmHJq0=C4Ind_#eX_~7
zW@XI;%$NIOXEYZgpC{Yvs1lNQdLOa7U&#HyJN}Wi`@XKz37P|8{pJh=1D|khxad`D
z`^)MOO;6715QuLM2CQj+7Cv;#ug?8x*a!S)j<Cx;QC;80f*40NWsyq-3$h{uA}Za{
zuRYA$wOs)p@$0b>=TrDJ<uYt5VIU1$1bIonZI#n~ZM@LB^B8^bED7_G-nJF64d+Z%
zD_4WO`!`xY&s`<g2^uovMz@B4&=HfnvmR}jb@eUV%}*@?(Frp>t;k_^i;J=t20@&N
zM=C}LhM&B-=Y}GzcbHM2g}?ohUWpxZ%0Feff9b(aXC`2JEN}FJ6tk!R^`C-U*}2+G
zJ<>U#;p`)+Q{*IcWX9^Ck>nqPM*b<$e9*hf*iNZQ*B9`QekQC=Dp}pHq3Jy6PVw4w
zh;J*z92Svn*JZDOPl+gy?*MSg8Hdu=!VHZfY&&PeSErKX+?Nws$uB-r$YAi-ofB2l
zmvfJ|=0p~5pe`nV31B8Z07heiyP&J7#I-E@_k8ecaz6N%nzXSK|H7|e#rI@z5{LM4
z%3lc;lndT~r`9#2{&(KVL-L)6ZBU+i{r3Jym-jobGNd&}OajmX=m1s#ecA_rP%IUf
z^a$IG7>(B`n_pHL>KbP$x*U(HFtCv)m32`nZ9tgo9*Mzb_!yURC~;X-G?Tft&+JP(
zV><t_K6G`oNuSj>k411tj@aN4Q5_lJ+UG*oV>0uqvz6iS<AwJXV2s-xN+sFfyf<{k
zauQhr?pWIPJA%Ql<V%(yfAXbUfNjUj>TS&Jy3lP*S$ahjgL_S&ukLq8_N)17-<U#2
zLE_?*lCz<Rtso8GAI;L5PMUY)0Vn54CQgy=5CQRSqfD+HX2D{eLBl@($(R!<a(k50
z{9_wDBZ>y!-F7aGeJn{-#tn_u9Oxd6!Y&mH%~E=37BdHlxU0xXBE^h>qk^?!Qz`3y
zv%|Y{I8>I<WjR*1RikTeM#55w>tzLSL*c0#$DU|Sqqk?&QT#?lDAq0@t~r;J{Kp5&
z0jatbty%<X{qQbRz?$S@u{QTy?$<}M`}F0c;5~@#-Eo5Z!Mv2wB`AP3p6s>~o8?W=
ztX(QNdk$RehY0GUm7^~zV1rwL*$Dn2H@h!OqiQ**8txsUW*6uY<E6g3y^C@EsFUE*
zGhhPBxFR~YXxpFQG0@XGzhd(gJHwN}h25ZgKia1EW#OT+%;TTf(t~_+UR(-B&x!`a
zwUc|sEDs&&2EOPWM0|5!oUQ#H6jV8`4u)O;GY7s6I3d@we&$%;eb8URvbviRmnZtY
zj2rfgA<o0EJ;-b?;9@6ut<xe$b^ZVso__Ef1ixJpNDWnlxmc*hK2;dLg*vR7T@foj
zJp^3NV{c-?0&XO7L^u=BlC<E<Gqk~Y<P30I*ept<CoEajc=%Mn%UW^?>oR=vf=>mS
z0Dd)0Pgqh;9yh{`JbJ=3;nbSi{R!@ezQKo)Cn`geZ#Irp2BS~9KXs5gPN(Osk-|WQ
z(69O_3MJ-dN4&WMzW_RlSJyWW0FXCYbr%jaU7@wwy5+55GAuV>4--N}#6!m|4Qr2R
z;rp^%kwvqgq(5DQXhSaKA8ja6U%i8tMHi;|)qqA0h{gr)&Ak24>|5}SKaU)M$OkAW
z4$%jsgMTjF$fN?E^griR4dPuOoGUI2N4jRx7+u%XFM@bn<jl5LWjmQZ(};K^I`Mdq
z1xrKpFnP|VZJbF0=h}=A)f)sOxv1Ic4nC!Uw7l^FU9&0En&xu&#bKDySuf?;3TI^X
zK`<_nc!0i8t1*ztyDxk|U!jVTWqNWpoU>x6QU9=eM(`j(e$F^kDYY$p3qM;oweYQH
zY+*jhqfu;IB2AJMJ2KK4PI6@{&3}3rdAvL1u8jqEcoIO+VCl*fKm6fK&~%(7dv|?m
z-{<{3GRvfUrKlCm(6)8~j2e0C0w{i!BZZpDnSE*5%!E3K3Yd%Cz`#yvqNUzx@Y0-7
z)`fX6yXr;jo^b3F|1eVv>pko!8pDf|51j9gVG^69RgPpky|EcB?F=2DLmYh%5a;Rn
z9vcLG;rS{hdR=3|Dn8l($-POr1`(B(@ZAR)`p)D8YVSwH%1`tZ3o~O(&3#nW`iDH_
zq6GNu*zAVDU=TMycsdu>)h|g>j}Ox+&aYn|Vr-#qTEKQ4sv}<x{!!?y95`2CceDS>
zVtZZ{m<2HuKMa6v2<$-QJn#(>iZJYGwlYER+12w<m-WM=6&-(}+HEC6Usojfv%TaL
zF6Y?86=Ps$>@E)WZ+dNz7Grk_J-yc%&!YGVFjk}FnZkJ~Er+YgNgRCHr6rz%ifPkj
zrm5RjlJBbqiso3$v9*S)$cIV7Eh|b{<fe{gv<HN{^kosU#}eS-4Xm?1QJg%ol3U4_
z$3uP)>F~L@rRzONQDUib8F12>rr}cUANRjpZe^82%X0}zt#8+{ebP^!VwZ;uc=sjY
z=q_%K=PM4fij=SC4jLZ<nZSZ7=QlT!7pKc3gFZiFtxmNWJY-&m2?jir*g(Q<8=a^l
z(@8$6jNMOM^N-WnF%H$tCea5tvEOhlBWJVW^{o}AI~>eY+5L5#-)e4Sc*?j$W(QID
zp;VEjzS%=OFe?Y#i^!nkY;X}56jPWFxa>M#&tu$Rp=;~(R^J{(R*rBR`8>>S+X&M!
zLr2lK$cUT891WM3r<L5uLm2*oZ0>gH*Dq?%Jxsvd5oU5Q6k&{1pxyl!V&R&Ps!O;j
z;EuiDh}{)Xj)vL*Zm(z7wiwh?qxH;>`b0>u*1GFrg3>P9w{_3gR$DE<6a_4e(429b
zX3p8y>i6romB!^zU#!YaZ9%l+7c4wxw>0}&rurF%QU}U;?VCOW&pBBgArhQ#3dzxW
z_H5<0DOF8&WU!WTL$M-;T6j6au;Z(dtYVmH2ZL7V>DQ`b`TaBPvhN?{WOTRdL%01E
z@b3%6#8xS$XLl(A7V2UY%BM#Cy6$VVQTC3)bhIb@6e)U7YUc<&L@Cv|M~xD^*6C0H
z_-AS0lzWOsmZe$7`_&mrn}aQH)N7XI_s*Ndx?eaTGGLw^COovD`O`}_hRE5ZTD)>B
z<0`wSaiQS)W&Hs+=l<6(iujdo6_*mO%6Rd<p^-jl1xw#rcga$L<PXTbAZGiUG@3l_
z28q&jIfWds5~H2ry5vDa6F08`QhbrId?5VQfIc1!N{LO`T1Bt0Lz$UX#7aQd&HPoM
z6--4CF5=3u@C0URs<WMb*Z$21v=hpcnB9shhLv$qX<nF*ArpGi<=I&DTFR@)ZR65P
zBAQL7B=SRd|05TbF3~N5D6@ypC+Oadnq|G0S`n(9Cn&i_O7{iRaCaC_wiRx-W&|F6
z<hrro8%&FK((J{~8eBlX)GY&~WL8Jepo4%2bE;iGNkc7er;jR1E@vezV1K(No$~-|
z6_w*peDJv#nIbp-g#+W$gLm&#TpV-eEE%LNorB+exXaKS555(yf(n_`+kqrlojy>S
zjL|)Q3QrvMsQ^S05U{YWny`~M^s1h@s#F}<@UGO=tc3`Zd$iPV`ey=WY72FeHgfjj
zaz1%Fy3RThSHIEfgxVZ9!yacoTCM;se(BhtJ$68G?txT<-mB{m%Y;f17ysft8Hvv7
z(1PnuUu?sFRsI7oS$O+^)t?S*%<?Br8)a#nQ{W4{3-cF-i|mc9*e^EtUZ6vX?q!%a
z)W@=pojk~1c$I)u1%Hr}flxy&(($E|9OgFyb3z`g>D<H}<v}QpEnp(jxH=_ij?nXe
z_P$yd3^nuMl*D{`Zc~5>2%qZZoZo0D=3}XROSv`QReZpo8y^Y`%|gX_3sXW)dHqld
zli&jd2G3c>-WiW}Vn)cR<<KvucL#HG^Ay{{MhWgm<~UCo#OTSHSv*O~VJ01}VBpT<
z6n;`7L=cY7p%X2<JmG|fSmbFT0$J$JeI4%`K<hi(uQ51Ba2=<*=5W|#2=1j)i_FO=
zv$CZnqpJ0@Af7%2BC?hxE->*?KK{%gq%s{b>D`XdiOw--_4WyH7>*>Kfaoz26Pky6
zTwcG6tyRMruVsnP%RZp|4EuDMV^@G;{(Wofo2G8LVE)Ue0;3=|ocp*2r;^0*`F5jA
zr?K-F^!1dEB;#k8>j6>!hpMf@?Ura|k>p+2`=e3K1A-5l`<~Cybx(dYDbonaVF)z%
zdc6J){EZb`E7BxO!KFDx8uhsDQCmJ|el)OK`Tm6%-v;W<%EZhJC7N-_2dgtE6R$q#
zDJ0`1YsunE*W`D5w~*Vqa835)9CK<yW3wAHMbx8ign_WaeoG4+CwUe}gK-v_U{49b
zumfD`rljk?mLHX8bDe;{1GnQ+(pDo`jq~Vun}jtO3ctLuOO#O>b)6og*JcsEbJIWt
zD6as1{^!W*{|<rv;z|dp4=3L!T1MSf@yYWgqpgY1Rr=45pNjgQ<W8CG#$TOx8XOg`
z(~HY93CuHb8^wNn;$c+VncshSkDP?t#k+UoV_{SmD$xUioL5|n<E0+QA35oL?T+=X
zp@UL?YteRG|68TNyXCp?$=&|QPmu-N6RZGNs9Tt}uD|qWwoeUq$3=TH7Tr(81_h}N
zXhw$V^5piQI5=nqsr&AscTyzY<^@(}?MM2G1g|P9g5ub&Lo(m~j^fyVKl=aQ4)kj&
zvE17y{nzpD|C2yE{(!0+#FQGZ4ngF3t3L^+;|{Tes#5IR=6C$t-%;$_HS^G3`oACj
z|GWb*sVpJut5@F~X>S-WEw#?YP$Mc>lFh8~1ESqkrQJEm4*d9ZL2{GO*lbx=Ym=?p
zQW1|zw_BH1rtM=P^zLs5a7gs2njf_y3&n#!D}#0ghpRf$(q=dn*ea!S=^I;otiJ|3
zs)C)a<FaaDwqB6Nv}I@N&q(V@AoO#)&z!h|i;QfX?kw77CDESKHk{MYp!~7ER&`-*
z`T>YBg*H8%+Wbh{@X8#Tu1*laX=k<Y0*ilhPaOMHDZJ^@OeXwEcfmmJn5nU4r&DoK
zotqP`v74|0<;|g=S1(_?UyV^0`QseT>HAtidGIuP_VE4a6Bbt+1-H9CD`{%F*`{Dy
z)6V2E+QH=JA!+#CD40<F5HgsQYXzQgbzxrIDMR>Kd09*Hgmb*BbvE2CTZk9)e9+&f
zM2YzAH&4&4li363Y2*#jXp7{IWv?yOH+ymsR+NJLVeX=7nKg&4GRke2oJYUC>+LCT
z;=8j6@IP;}Uf!L040Tat_EwXV?U~Fc8CXx{v1Wa|!|I|o?~%hWQnPm1$$6<}8&;8a
z&2Lb=y0u~Hnrw)<(MAMerKc~;rar%2GWWWBJJgd4{ahAXX93x(Q$tmw+pe#Oi<ew(
z!0ELW7ns!<$Bv7cnVE`ue3t@0x_Xw%ok(hm^M9DgZxaqf;{$ArqVn<lV^xI_EKv*S
z3@$_Li+MDZXdwT*`nv9gzWpV|Io2PK`7Q$#_ul+Cw*P+AkBXX_TI#!1oLsj#epG3f
zVc=hGPfu19P5K;LYIAPK0M;4`ig@{!d_5!fT|XRNk_b=)3W`uFfui{HQ1*^u-;QBD
zE}jG1)`>bj@PiZ-wZ2gX>WNUwhvLDOo0;JekJiiPUSN?aC<bIyxLGxzCsMq-Q+%;b
zNp{`&YV82_0lZZLGi@mOi4<SX8Kw^)6EFX8wP{;`^iZ55s)tZ2lAlOnAD`7a4bxbs
zK&^Lxv!;l5PpR3)1U-@B_F46IOP5sEbw*<N9JCm9iI#rv{m>IB&K*s$$}iQT`5{^I
zH{+qjurgXYNIis}NO5fMf4Ap{{{C;;9*5l|{eoM}Y?em~BDqE0^^6n<8W*P4*hcc(
zC)-lCJB^mLNYuxcm3KaOWy#L0VM{l*?r1DY6|cr}=|>^TBg6&7%^N1V6;v4iUBY&o
zo0~FoS)Rw|6<YY~!5A7ocJ*1cxn^1TN7|<H=jI?JQmSHY8rzF|x^glN^$Q2xqn2)9
z3g$7svmb3cm0^O2OzfN5;{1yd8J>bQS-O1-Mho$y@OdAyz$yQez#Sj8`Dg%JgqK^u
z^WdGX*U~i43Xn+1B^xV=7_o9&s}g+9z39h+OtmRvc`a$#bCS2(+lI&d^hm<sQ${s6
z&6*3gE2W+LcTLmi0@UaN(NB#E1}GEZ#aOPSWQGn!T7rG{<xU0Ku6+L)ncBpjF_KwS
zn_FI4ku^r{Q3`iS`mCx&%gb2q>c9S7zvPNY)84JjmIYtaItEKw5uKi>MHk+slDMUC
zfnW!mG>74HQ_7k`<oJ@u(~EkR`Dkg&`N)2(8pgLms@0^VgDojJPHV8{-wg}jIaj)p
z&i7m;*k#Qy)$U%`s279!cyF1}OZ-F|R8L_<k7Z~5`4yqKz5Gv^l(O9N3c1(<2J_wa
z|GOj*a?PV>@?|PeNs96MfvPtjCDz%YDEB@IS2aI4c5xb)jvt$g6?QEty=RL*(@MM;
z)<Lf?+HU0R@cNxUsWS<QDaxNanJe?JMx^==QG`CW)%x>5zo@FA79PG>_ymnk#Tnbd
zAH`m&c-z8-a9w^?m=q8ct}3%=lnM_9b|zlzpCl=CEV)6Mx$w2<qMO27C6qAs>`?N8
z{Nrc2bxxv0elEmdYy_365-N(TqM0}{Su7{7gevlH%v{-im^0!&WX5Xm{)3^m2w4u$
zbifwBj{cO2e%K3Hz}qCB@4s-Hw9fFOIXsJf>ZgXorJUp9Mi$E^9h{eBMhlDIe|S&O
zi5$26Xt2_|C67C=ti1O|@zSMCgn0jaVnMWEcjJ*ck~&K>76j)MtdKWaa~pT-u)P&_
z>;2EW(MEq${g*F~35X(S1TcbFqFb38wuFFH%CCB-GHJ`*sfCSpOK{#&YGNB%SYi6u
zyt9W`jvb(P6fN&=bv4%y?^UL~21Rd>L-1DkX2>OSUpVxxxAxUjL)h-I&d#E_>6+rq
zS+A28Uf3wA!ie^RV{M+r84crMNt{v77luFEvD%<2aCu)Xi;rDd3Fl%iT(B~?E{o<?
zIy-pqcEXxaYoBKO=H=LZH+rWkeu7Xq*aZHLIcD>W7df_>xTrSy^5!!dxs|hJ1Sz&M
zKFh0b%m$+iJZ-w8X1etkS_g$}J~bCOmtDzEIwqoDh-?ifTc84xLBkk9YVVp&V1N4E
zYlg{e^mcuR^+cEeSoc|IDdRIw>v^8vzMUf}bji@lFaxUv(_#vg)GVwV9+BIHVsOhA
zu(1_sh>gj?cQ9F{_PW%bvl9gl8wll(?b8wt{k3~Pp#0q}*Czbho&GrPE!um3<K4G~
zeD}%@tiJ*q(*qSH)3e7+*&b-UPWlE+(vH^t3Q)Lfm<Fvr?$y`CU%yo~t@ie{6Z7Nw
z?_<^w`l^Tje{gTkirX&NtnBBcNZFgVd(HS9GXAAdqvZIC(wRNNDB?v@c+ahxkIU{F
z1L{A+<QN9vcjDDeZvUgOofQZAdM=@oVhx*44NT9?PelI&mQg#vDX_L2T7%sbruWxH
zdJvl4^nugm2Fs%>!{tB0+#9aIDBYI!|6@%v`yOvLNf2QEM1at)&yg)_Mx>JVSIYSc
zF9lrQE`jZ9ApU1%A-uFI%;x+!8eu6Y(7|QSoNRfq;$sfN8RL8OD&~mAL&x(o(Rh=E
zj@)(~x{Hz_5@rs=Pe%pq3au@i7J3SlRho2X+?-(;IXe=?W%#C@lHF|A7(@#M^$Uw0
z*A+Nrnr2#A6^IF8xGWiLJLfzXo-^1P7eprQc71t;l9qPwNCy^w*LAdG&^<FmfIp%?
zTi<c{=_qier%W*&#atN~`c^k<hRs9Xq?q}SxZ&^S0N?)aJ6Dyvbgl4cl-p<9;!^{e
zW+-@?@y@Sv%?+~AmXb~&^fmlo>V}LqH11Z3I5pa-=kbQwau0TNB4SQ3Q^Se}#MAnB
znY54n>M&C_**%9pEsD<M=7ey|ci48q-CX)y7v|N+J1dFDO`De?H-wG0=qw`$MTi#p
zK3)TK@M%X&u!X8rAPQ3d2kGtKdOu9Ho$k)I58+&=3TsO~D6U8-k4VF|3S@*Ggq1X%
z>h+8&82rp5dsE?Y7M>-{g>U^Eha!(Hj5>4gz<-%HRt2|a0bn?>SwH%3&Bexzgr2Lj
zOQv*%E20itm4_{h`&~h5*Q(+Uy6q5-p6T9inM=Y^M7PO5dgd|~yA?^T6?V;GzQSvc
zPHGnfuxTuJtJlIr5Jlw&0sVf`YM}YYW+}aY!@1qa%l36L3ybvvnv3qMEK7+a(k;Ej
zEsE^X^t~KE3B?+zcs1c8pB2_A4TY;}7JRjwZU^`-;u_b-i&gZexDbt*`HYeDiYzJY
zE;Q%Yv>FzU#v|-j9D+Qj@1nlN%bRTmo`vh?xJBhb<DM7uu);V|p>UheQvz0oR=QUC
zODi}tYxW}lp7RZ6aU-OQn4N{6N5WOby{e-4$NZLiZ;6~Q_K;cX!;jIPYdutj#`B2r
zQ73UyV?l2USo^|k8~>b~o3nS)F6QrX?Pfa^CpzZ}JAssinK%{ex3olntQ_HZl|9PT
zUyn2u;#km3veeQOX%LtEYM9F1jZQVIvx6@W1JoNa=0slkpkRRe410nOJL1&sG~NJY
zh${p5vfHG`u8izUix}guy0CxnxG}GsFlNDarxE)69(POMj%GR8l@xv;>2CAH?2Nww
zWbu<{LXYon4G{~b_=dsXXI_08BS7d8HfPXMfo`c8C+$ID16w;snm!H`Vzw<=3)vJS
z1V-l*RK3wV({=$n*~QaBQvaQC?!T53s@iQQqSEmt9q--u6Hc_9%j2quB!@m)2Y{-)
zG)|dMfNZ$doqmyiAtQr5`zc<XP1uYaA`rN6>#>G;bj89@w^4S*p(7Wjqg53ip7D5j
zxw8yiHelp2J1G=efa!G<VNK~A?ag`L@0?)yxM0?{nDdHVF3C;fP9afKT|`P7O5l-v
zzzze@c<un}g6mF-c-Egp3%z86+!-Bf_<Ycj%SyaG4_S0LT5%Uj@2%iGdWX};7j?K~
z8z;|H(nrV;8W-wKe+2nLTNKsEiwc_t-x8mudUtYS)v=u9o=;Q}rRg@CaD8j3AO(l=
zuw#p@OzCm-!H7Q~u?I4Lqe6W`1`@QCKf#~WwKu<yf{SyNUWA#A2r(I^*Qne&+;*uW
z53y_qSW`@*16wlN=0846jcUcUkG3HQ$M23#=#5-0)4vL^CZ^HIV@dvVi^)WOa)XN6
zO-4~+Q%tT>8?0g`^iEH~pIUo1oC0_uiWK%=p|bzzJyU_RRM%cFd+OW3pW@Rtn^+jg
zW5uTdXZdPLx{N9e`phoyr*qnytr?UQ+k9xid)`j_6}t3JgUSx`l(Q0AAj-MTH)A)k
zd{%xXE~2gfW%5>v)2jc$Yt*N_P-LsSwBtYjG^7h@kACIoH|k5R-;s34<Nrsy0lfzF
z;LGmoQom3}Zll_S9u~L=f$4MFF3vNL=T$c6_%@yzE5Bt11i@~n<p|AJly8XsezDP!
z8YW?6-8`__^8eF|{0Ara|L%ifx{9c4yNQ8E%)Pf#v`YDFL-zM)ecq%ertF9oLAF26
z-CP5-TL0OGlab-CRYp6Sv5chpymFm(YuYciyzkw+zmMhBjWwvN{S(RqxmqooO8YOT
zL&fIGhT1j71JP$yql`R8{4f8}WhX&;gOOU$?_G>?Z(>h{LR7Zh`D2QGXFdOnIXEo-
z3<?ha>6x6QWue&RvSSCtA121iSsZ0iY4v31y|P<&GSWZnn8(#6UYDMA&jmHbw;B_N
z$#(HqHVI|PA3*J!aIIrr-8Aw%@o~hkR4w1@)D2ILha5QSi#j&z-Jw5EADxZWAj|)+
ztVfz)-&y-Zis2qq+TjYJ;E|uSMmYTYl2SX>n3(+%{%SNZrbOTET__f_TlLE{_dA`K
zD)-&~oHFnCTRp+tnr@Gx_0}4oTGC}!k&(q~9krAUdwTl*33byTG;N$XPm?}Ve))c+
z<{?XL*dZB;`_Fp#e=jwC4;MaUoyRB5w{k$sBBj2&$4hgE{Vp&pk7R60_M7|#PS@?i
zZ*7u=m<D4fu(yPA*Nv|NUcf%ZM(678KMn=BHnEx(87;g*7ag1G*?OBVezQLgEO8>$
zdN+8cnrVh~JD6QfGn;+Pg>?aAPL8={V>{iKK24bq7u>L^>QNedJK8alz9&6S0l1!<
z?sGq-k`Xu?%1X-F1jC1Dg^6^XP}TPdbD9qo*jT+^pSb>X<FZnRMwlpWC#r9TVpRc|
z4Q)2#UqB|=N_V6)imRys{y54vx}YphBBQD&S-?`k<&24AV7Bf^QL=zpF|x6~)@dQ0
z!L2~pc*NCFVYyfU7g*Et4|(U<Y_3B*Exz_Z7|XJaG(-;gLQ;W@|4ZvI$ccb6{g<;7
zjvZqu8Time>5O4AEMSNnrcrHJ_9Q31Sl+94w>e=<cM8$ku(QQ%2WjarTPKhFmDeQ&
z7wybjjudqC^2{n`@josQxCGlKjXx8cLe&RE#MQVg5zf9hKTc6uQR?p$KKX~})Uz!G
z9K846sX6#D+#-BF_8On~J82TWQy3+PVe@lBZ?gdiu&e)`ig->K_fVhF>lU{~L+n|}
zB+gC<NIALjtCU1659X*avLzOeX0?eTHTqnOkDUxv1=4tWqckSbY>-CzMZZ3x?!gwT
zj9=RZYIP*0j<S0i_nt7}Ij7$yUyGp3guh;mjppaC=+OqBNV$@k7u4y&Oq|0IKHI5-
z@_^H>)V0v)fBXV{9d4dkntwu|^m;C)h^xm>pjVOh=+&yI8avV1^0FkwbcyY`g18c<
z7PI-LKLksB+De_&ywQ*zRc_RyP;W@3zw6P%ID0Y4I&W8hGP?N*0rA}{DVbYvq!bhv
zzJiIc)m`Bc>2`&uY0?;aiW=T{kOvpZy<QkBX{=&bIA7n49bkehq1g1@1&AX7nFy<F
z(W1*=Dm`q_XndC*=}AWd{D|7WN#_^yqfJdY@kS=VAYzz0<|XG_dA9GQc=VGK@I=FK
zv1Zm&1N3e@H{Fh|*d*n_hl%ZUh>Q(cblcv2E0;?GY5i!2-@e?B4C&FyhO~7I1q)+A
z#-ClU7#qIY$B`jkth9vqo|L@4+_+F=l5K3I%h5cXBVdLqkyY{NFzQueL?`QuG;~~8
zX^$I!a!?M6ou<v9sEUU%CR|lO-}Bi_`{$d1Af@j}=UeX5`mExTi0Wyh6spJ#3Zfkw
z>iy++r=v>S=tA*e=Z{Kk|5hkzEZkg#G@RJ(vY1T$?%qOB>UyiIH2@6MUUDK6SOVY$
zg1#QEn#tu%X}|Okjfa1Z*}+P+xpwQomr3dSI38+hsEziC0KwJf{5?E}<oOmqXOizU
z7u5T>6C@u*tv*i3Ahsegw-P0;fsh67G5ZKoa!+<`7uc>xGAdg^cIew1u+<TXV+hIJ
z7j@OMU%;B%l`4ve)<Lz?o{Uwzhc6pwY%a`(9Ogv0_8O&$GuwJQJ2oeG_H<E}(S4$K
ztSepO(ig+bHO_Z)Pdvp=jb*iAd%gv3Kfj89U%AX;%0rN4TXVGgB~67T(-$SaxZ>f5
zs~mU!;=8#;o47hUqbHDFv&|)CMqa$vN(!8^%B*v2t(JP_J9`TT-;NFB%a-KFaa-lO
zF|j>n&M0^NN~eg$PY83BM{&!41O<qE+sZbmm{+D^b*heJI}>q05jAz*qZ*)pG->!i
z2rJ}DERN38f05g~QMv>3f43{4>dwY0QS?39TpowxmkZGMRGPRi%@l>_$69rwyZls`
zd^9t!M7!a)xDjeop!SdqXNKNy)r31c=4(@&mN$ldbbdW0J<s)E`#>!dZRw^D)iQrn
z5-O=@k87?vJ>^6i;7^eDUGwbxQ(yFg9!ZQ_o&Xup@8(>$v5CDR@THN6Vda#;pNDid
z7spQxbkkcIg7%JnCua*`e(iL96H|=SLWO-x6L7dGYV=~|8!8dbN*VoA=A9ks*^F#4
zuA^mbc}4g$ZVt7{<`Ef4PaNBpvax$p(m(HHS#63*h8DqIhZ;(8zE3DHe8_vsv(~}$
zBGDtXY8DnuoX_IkrjVJf6+Tb;_1=QkgY}1m#NcygSwvsMNU2)k9=@krcrUUqyiEUv
zG}WfVVRdpEx;+zpx48fC<JhcUP)xaQPeWBvN(@ygoTR;1Hor4lT@j&z)EVLZ?j^i9
zbYL-Kn)Co@?QcXCU}DJQMk@AQ5?Ok$G<(dy@1C!oF@Ucc(%Qdz<<J2G*a!zNH-dXr
z4HDtMJ2oij@@E;ANk*RyR|OD)74e0o?v!FmZO+8L`y!Q!vW(6Ge}k+v8DHeYC|P<i
zRU$D+PMmr8*dp3Fj`HFYfZ3qu*E%$Ap+Hj8K$&c);|J6eAdjGoWRFnwB2Hz0tjZ>z
z%XvJ!_*&%oo!Yu{^l#LwE~O6jWikJ`h8|(zj^OClHl;oXk-)xS0K)=&1MO1G4hlsU
z*1aG>0QHr9-xO_bgv5*Grygb|O!DCMAlLHS6?m4-`Ykp0AdaQZh8>DfjmU|K(=FiF
zl8sZcj1HCXoDE*;q*Q~XCsgOjEG<LiPS_K_zNfoe`=_als4nXMWWd(Hgy+q%Qjht8
zUdmuSW5@&_uG%Sl)^M{lX%Ws+v1o?R@YgxrPaASnun9rm)nUx<?JV7YD!&|NgbdNk
z(mK_5mRUwX1D64dTq#NYMav6t{<Jtnu=3y<!<+V5No_g{bYtx^lEh)!yWkJC(+P@8
zlvj{)y&o0ZA4J>71l`5R(wiU?151wRd;{u8%|#%2KQ=kEO5hOip;=PekxgYUkDRLP
z|Dq1PLEw`kxQ$g!P(oOJfl$?jKRwhfJ&<Cy9T5XEVXh2FP}EfeU&$H5nsD{RmD05X
zt~{`2n$P2DAk&;{ZJzQ-A_Bym8X^d8uv*+5SMkt8A&+I~sgQh(Cj}aRsucpO&)o>&
zoox&8EendW5vs;L#Yd7*#=Y|YC%$iGC>l3b6dgzw4zSctC!`p2!pt#9)Sk)%WWH)O
zpj;w3j7fw)irXMvtO(Tz`=cQ<=gJrq{c7GP1o-}kTg#X0o&sxvLR;S4uRHW#!&Y^M
zoLME?KQtR;|I9iX)em@Zr`mx03ea|Y<;E?uuB^Pww7wz?il!15EMtCZzyem#Fopy8
zbV`C0PzdM^SI_OZXXVt(^v!GOJKOfZ;pw&0uH!2tFUAZEb4O$9h|b+Jwlp4HU|Xod
z7<w%`v&)~86(LLsWmk4n^wO$LLUH17X9I2mC52uxCuT<WnGqEeB%xI@sr6Oz{*D`K
z;H!H1*bxLv9(N>Iq39Na4>sM;q@`=52Gop!(LHyR^jLDS?{e6k#6UL&kOW3+@_4;v
zSRQYMj6-bc@`KF9W-F;(Om)2M!D5X-e!r$p6LkqiNd+M7fQQ#OaC_iOnzmVWsb~A{
zH*v5}=}XWWG)bx~`l%8Q?BM$%fKSWMF@~<%h8n(y`_nkHQF_K~fmG$P6K=rFl!FaN
zuS}R3ax==-yi78pMylr=eRiUz&j6Ao_PDrJD@B$gv54t&V<`05aW~!H?Z7@IJmEzh
zrPLRhbBBh%7yvhK98aZ~nGM;}Q=}}T5RK4*S&&~C;N?D0HX+LwK*}PE6!2L%Aj6AV
z#6`~U#@DZ}D0BzN8dTc%)_MO%o_sGUvfSiN4XxTXSm?Zb`jX3}KQ?<i2hD6J+kvGk
zdS#VwX&wQf3ieT9&ajO68Ef50Dlk<h2JEiAG75ZNjx!y{d|=@oP2S|(ID*?HMk!Qv
zN^@W)chyb;4>eru@l-Y1es%dU-MOW@LcQ1Ql*6}J!-$JoeFi_(X%9#WC&bFJ)s?5u
zhCSj&w9K#$yqEw_4H+mqoF>+Y<c!3+t#He)XpS5v8h}@Fx*t4n$OlHSUx_P>rMk#E
z?2!U(Sie2DhRA!hu%@QOCZuu4_3e1aChqDfMFkLRG$S=rrq5mgkYL~rfKWN3ZsC1K
z2SF74yI1mho%QXNzMC@$D}>LRfd@rA^0d3pzV;$~0ab?N?6hjcdJ!Z**(&kbT1|&i
z7=SGxE{JPN2XCgnOhByIJqO@C%(OT6=zLCmfnh3Vm3+c4gpz&P+fY0ZHvM|O!rh%4
zcupiIWAe;7@K@n|S#7|jKZ@g=Xiy)p=N@<>96)W@yHvQsEFaf#y>?bkqxt)5abk&%
zY90deUb8im?fhE0jYzStAU_8XWKW}d<<#PC?dGbX&bIcqc`i}`I*LGsz?x@Vv<XEt
zyGU8{SJY*v8^$JfDA|<2bOWA9|3Z;N$Gn#$NlRV_k~py-wU#<Wg>i`o8yMIg0xVxn
zsgO)qRS`gYZsB9QnF39;2+&#QTuXz@7o_x}t%bay5JE8UqA?W)ienQ3YsMa~JsTSb
z;KiX-UU`shN6l%1)J{KGt{=d+yL>#_NvBgzq)zjuv2Sb=WnY%Q1i&T17cZvX^enV$
zL>E*F9eF%4%MF1mobtmiq0_90U=xB)>{kRN&husfE&_}ZDi)y8OadEQ8=MOC47l!t
zE0;Mgf!twZeze+AbeK+T=t93x)kL%^uyWAmO`nwn?gc^#z^EL9aE<9vaKs;-%c$<M
z2eUIiOIz3G3t`$XJoj3GWAu6oEWendOKGer8zSfEN$9`BpodDc)w$(zPRw&+4=Vy(
zI^>}<!eLsYH6a5*G+Fbsc2%~zk;7@6I2c2U4UFAYR0i0d+PmQ6X^DYPKuV76t8a1p
zyeS{?IHdJNi+D);z>k;}f|3Wkj%T##urIuYGo;`&YWrrBfG^4^4gBp_0|SlVO;HEO
z)vgMXxt>7&r@t+loHrr|ND0p;Tp71(pVF4NpzX7<GvdARUyy1qpGNF}_WqVQ8|1|%
zo~hSP_i)yc#34nh;;psHAfej1cR<JvSUqy7>zCilet4v3KHrNWNlYEe`vNQ(=xA}Y
zcOv0Pl?Ts%wCSswtifvPVi!KJ6l)6Y;-LfLZ9eS4-hq>cFnm$AC$GHA;ykc7(5?rf
z**fM|`jD<cWZ`5msm`_Q-g}W%H^A))kPmfJ7sjO#o$PJEmO|NM5kqK~J~oP7?wys`
zohF~=?w7Xi#tV$TvIxG=QB{64DN#}#@ak{g97a}|nqac^%PH*zNQ~9x=(EJCV(>NM
ztt9nSP9$c+eS(QJN~yL(0hj}Z6>$JBp!dv1D{$%5Lc%2*cs!y1N#qU)Jg>rnBULL-
z_kyd6LaUt2J_pW?Z^gIkH&u@@$^z#`UI<9T?W}tHqidgFj|<OiXwMXE1&t%jC~H@z
z8a>eoB-N1luR2GP5r*W>5>2<DLj8g>qBbDyTcx(Hf#lHzLuD|;{1xzM2bS)@z&V?W
zGP+)2lJ{D|yA%PBzfK<A<99PDt=ZZNU$U(QB4X0}*z?;442(@+%>^%dCqagq7hnKl
zs*xz9CN)m69g7!(q-z}fq-##{Q`Pc$9u$XQ(dw_0HO53VH@-jt*j-ktoTKw6Kw5@6
z!1ssL;$N6jD!k?wHgSxID)7uSwSvm3hHYJ{jBR~mb%G(GQdC0LpTE4hA})GMKS&%K
zlYMhnTx5}{eo29C12NKeCt+Qe@^q>BNWpW^R|wT}Rdzj#637C3SPOW=rWh{*n5<i%
zsj|oh6*h{>(Q`c&lxW7q63G`Ok}f=CTXjo3-9TjN^GINw35rHBoGLWoTFq{41{|z;
z0@O5s>;cc_GU~<<S>e*B?_i(_1J(i**ws8H(!?Dcxu9nNkfIKd?r^>)yAnQhl;ri%
z)BOQ(v<E!CZxJ*0@S`>R0i3#vwfRAS{KEzg4M+i3d8+T0=bNc@E&bvJ8N{LRZjEi~
zu(p`q=Oy^Wy>>Sv+OxO1lZFNpRq`1{a~)n5bsFIhs?D}`!xi3SWwv6QbgVv_c~y{m
zz1@hBve+YCm7sz8(aQ5gU~_X2D}(Rx$Cj4raAS-Tq>slv96WRtA>q!(NiNLw%|Tf!
z5Vxh)A~hL8KDf9~axKWDK_PY(a%~HmBLNoB3P-aM3zA&9nSc`L8Z|)+2MD!ROC%NY
zijZNAbWsnVnu#-T>og)cmq@|Bn-R9av3l=LW+d1zv1*5|nNMCNlloF=8(mDb!f5A!
zZ;5GBCS1XHELlZO`O+_&e^iUBdkn_=8MxQ-;mtKosR{5}+}m1n!6+w_`QELNsRO>x
z*9nO-av82JjJr=(=4^9@$WT=g6ZM2^#7`ulIHF$A^x1J2zvHHdBc{LJjvaXEo5pD(
z&oH5)$TGHvJ(LrNf<PPi@8Qpx{>PStB53;;6rqwA5bP<<GIKH_2S`e+fS(dLnOX-w
zV8HF0AS<!STnj%eDc}{lLLEX`5XOzaQTF#G3S{BfFB^~~TtzGH<qbu7!XjB@@LI7{
zsk(g7Qo`tr3n}$=?gG6`<;-S5hBO}1V%3%SdY?V8FShCsEZiR8=^<0$ap2D8696>B
zc$M)nCPCJ1^`vr2V-p~}sZAiUhH{c1;qxO`s{o(}&TgaXLLOdgz(h-X-CYvuArCry
zR}_8m6C)Iy+@jw5R^INdP_Dy7iL&oUeAm^9W9E+K`qDN|D@QM5jwsll$?gt`hqE{N
zws1xc%ap*vBndcP3q8X53=l4Jh9pA=L6-(9lj(FEQkKh~X5o+3E&$Qgtwf4R#M63o
z$!Noq0VF<PDB_bg^n!c!NRVPiO^Ng3TCx>^(lt2{ey@@Thmq#K6nSlV%uZr|5|G=7
z;QA99*rd=ZUTDp1t{Iv?FQ^Z{G>-0~^!04688^W943EdI8soO!qhI||I-sG&b#Q2G
zdVetz6AJR4i#My<8kuyr^rjWd>51$)c=-;YApd&Hd%(R+sSF~kNQtDh0Y$J*C?DSr
zg_VzDA7}%lWQsl$%E{<11CVJe*YPlJJ1@!ZFCiM>U#xK}vvj#qAY(OP--w&~qim>;
zAuW*~_#zOrxZNdN-vUE`_i~3oWwaUi_F%8elP~HjXp`cKzhb4XF1GZ<gF{RmhLR^V
ziokj?;M^e-1JK-~o_P>P&Hw5sKX<vjxRU>HXL^HOr%`c66@sp=VDf8JepY@)e&_Ov
zBfnr+*9E&ov)raxk97P!?Bq=)RBVMwL!{G%6+1Ed6I_V<3so{8HOB}8vPrJX-35y{
zApUnc<TjG<MLOlwNfpF;*`j4ukWjNLw3O;BThbDdj`0IW529EX_j65rpZ`t+QVHby
zpVn&-zubH$7_?_qs*UYY1P7htcXLIfPdRA$!fO&k8fT;=aehTY-neus1HfFckLFSU
z=8{n!6;s`zU<e#-!U-onR+tnrOn8Q*I=wJ;M)&HAJP{>d$-L^&DClbJrwFbR;HbPb
z)8q=_X&uvahzcMA9i}DS_B#~7Kkd=9D1+fo9q=!Hs#WW72ZVC$3tLQ60Ft8`BUe6l
zAi-a`p124^xfw_~aFAY5FWui?t^wx|HW1U)x5YJlXP|o6z-)crViFwP6A#>~XPOQ8
zt;OA@M|-0a#AZ7S#}?Z_M2^$=cFGV~ua!osWk@m?TF@L=as|3yFV;xp1eg_owBN0k
z?-jUgWDxiKVU30UEsfs&6K9=tN=#ed)Zx8JU!zs;4xmqym$Kld6Jd?_*YO5vv?Eni
zp+8Hq79q(ZW%{EtInj7eE(Ihs2JJmzA{KVQrV2~8761wGyw4W^f~R16BYIKDRrtkq
z(uI->ae25_W6wp7xtiI5x31&%Y)~=+w^&i$dL*qwC!0PPIEd1K<~R~a<c02Fbe(p-
z8*1814$By-SOaIMRjJk??!dB!aiOsCyzBb0j4FzUN)bKXd!m(UV;?W*MCs9X8ui+p
z?e@%QGxDnBu0U`PnU`2##y|;;>gIk^4#f1r<%ICSM4)It32*9<0!E!2C|_A5oLmu;
zH|>}QpDO~fa8j9+Pp~P>7{qJMeK;kMK{>WGJ)=er3L(@D6Y*HUs(oB>9BV!XU=b8x
zK@}+lHzOvHbf5-}29R=w^9pOpGCos}U4Cu#8YH&>YGke&GG`<?1VJIXy&njEJ`>hm
z1#}(#IN-}-{pE9Vx2KR{e>_QrT#Gck(Zw}0S7<$@j1}#5E&=yKs@fZ33kHNR+YW<@
zWRvn*RevOYd8#mOH(Ii2LA$6N+GMc+H;`f~T=}FxDs=WbdI}tpsW2|2e#HrT@!7E2
zya`JpD+tyaWy#(TctYwJ;OZ;wp^!E*fyPh-@Ys7DO6n!{DCN+ogDkZcffojqy#V7d
zPsUb!wxl>uJEJTB*E_PgJMC<Y_5gsv*3r0vL3$lds%R_Smz3Dlqz)pmc6CKrJ9Ac3
zw(V?)FY7wL6<lsMCkN*YnJpCh_6s3`_ggI7c%A2lbgL`-em4cKuAbikDp56snV{Pc
z$kuGR&rRC~_g=?DxZ;@r$m6V8;+j1W5>U*_1_>;-z&@h^^uwJdV=mh8L9c~bufLN5
zGtTrcKq-(kM~zj(>jNFz0mNa&W?)#2@ew*r`c_uFP^tim2mQ$jl?6pp;+$5qi9<?t
zORSDApLszn6}pO>ryAfP29heCMrd9995HUkq0&+BvrYeHO1}Fu=cY3H5_j89Gh9Rl
z(V$MRz0KOC+%~^vXY|vU7lTfqtDA&vQ!bYk;Jv~>qoQ8{;op8DbB17rg&Y2@>!_jK
zB`&rHjcqm-`M@|`o6VKU;fG2kVA08Ii+=bQk`ozHEL4K_n!WaQD+<_M`<OG@P}j<f
zxl(o!O^{uA6nj<+mk54gZaXsI1u&Wqc+a~T4cK?hoR##NiVpm!V_S9zO3p*B2OJkR
zOAC&N&n<uy%q@Y%=!%5ezC-?fMKtVu`8aJ|zKv&yM{^;Hk@cq=GDedE%gMcx;qK|F
zIJMRci3+pA`%4E~mb3D?+d$(*a&T2#s=mqW>`5QZz_RuzK_T&0kJ0{6PzBUgT58nX
z?uykrGkg{HkkV;vkLBc+!P8JVmcAtn${)0X{6TRJ(aphwN#IIisOf&$)y}E3B`wG*
zww^VUn5rE(CjPadH*6#@5u{R91=}U8iuda7=2#t<YvJTNRLT$<0*b3eE}uvBciX?5
z(F#{qM7`YD2Z*_4miDd24NM?+S<aE99TkT?^-n{p6nVa;FNSgTi_=yMc2F57Njw`$
zuV{zy*R59UOf#&k4fmk$T2-(VDy;urWgwaV@~!NyZ=h5RXq5uyA5<>Y{Y{crOIG<R
zzyo#8SVH*{ZQ)!{tqSXvGvKjkN(ZS!Y1w3O@=DA|&8~Gp2f3yJOrbK_j~nqmOcc+a
zK8vfe@0Ae%T6BoxfJav=RIXTii1t7|K$O~^jP_+~i#ib5IRhMVsH*+CD3@Gotk+`X
z3#)tqU+B-R^Cd`TNOI3gW*G&MgdaUU5AA%+Oy5f=!=QZAFAOTYLRs!SoOUw~y>i8#
zS$<_!^K+rb9_{9A^OWgo&N||1>3D5@Pg2FRYl9UwDjHOL1IF(-s68vO%=V!b)yk$r
zyP7oKoq9A^TKk)mfeJESZr@U6Yn~M>Qnz3xTWm^HYQYM~PLsf+>07Id1{J>$@a1VS
z|M2A~yK?kyG=IC7+BlB&>V8>BuNnjU%6%=5^0%&*OrU}VI#vf{w^xP-?6%=wbI=Yf
zJWCk@*+jBTf(lhyh8BDAE1-DqW`n`m)d{QpulBw>tjV)|+qPP2(PE2=ihv6d5zrzl
ztU6Ja0~{D3wSbf@5I_h?s#YnYg0ch*R9UhNJFFz)LO?+F2q7XP1PBoVfg~jP?k9l7
z+Tr{2{T<(T9H0E*p@t;Sb3gZejq^OO>jIB3-RtijVY25o9Kc`uc667|+;7p}eIrC?
zDndIA;^WdGJ>fx*R)}5_qc?YyEK#<{bOFrhJ}(0ha|^R3hEpcKC&490EKR=S{MG{i
z-vp7Ky|c&Rcel*FI!I#IBfffWjDsGnXoqYP*7tIb+FoEz0f<@cRV#zzCWwTtVTNU1
zyB-)_eKtA)GV#CNPtX<t0)r0rR;~<X5R5lS6IgAnCWWUFccr~^;GoCKGr498?M{fc
z1@4OuJfPx(Xt2kuYq`L&Pb7Xv&Pz4G=h_QX5h9sgv4KvixP8B@`kNEq!M=KMIn&>#
zA5Lc)t54^xoBeA4H>cJr{do^VY^W#jBa!pD#<}N+K&FT*8NrpvR}6sKs7C9BzkNF|
z!hz^%$CMfEiveDIT&y>XS#R)cF3nP55FrD3D!_h0w66HJ>cp@mSaov~pd>t13?`}e
zbGL&yGJQ-Jf$UeI5tk();C5z)<_?v2xPbIA5Ii$wZ2SDlSMT;Zc&1@FBSMG=!a%Ut
zv()nYPgws(Jw6jh5cS7|K&i|QHDD;jhi8snuJR&|H>n(|^swGCcNr;w^t_5uxSU>9
zF`#609`^p4TNbFdDML>hc&D-|z#47c{2gQ({$!B=xEyux=XZ7(NO5PVD)HLiQ3tM0
z-UP}PkM-FB7IwZ{2G$~-vq%s63h_~dR`}`Oi(>yc!y1F_0w4g$S9Ej&WnL%mRM=nc
zDd?AuMR}73%mDYeLSP6iedxFE8QQcCnPF%#uR>1^xQfLqq!)wTIq<IiYsQL(wTCri
zE0abmj{=_?Ww-MN0Hxm9M^;fo;tPW^<hM)t+k5BLiYdw2rod}vZvjdnfNz9vao}#p
zL3Ch`&doRVsWKPG%ayRk;LriBk~$E7fWX!C?#(_TL{Rmd4tP`5I*bUMbPj`9oxW8o
z1YOX{e22%5><e(}d#1>zW@YRG0j*Wvhb)4%;dlo92;`rSfi<rhI|GBZSMD(15pf=N
z06P?i2#0(%K9wiaYLajgYc_X<cTRwCGivw7YLq490e2*A19(25k09>>%*mZut`KZF
z2m@koImZ2f4S=#XG?O8+?m~MU{M8W>X+)t#5N^(pt`m|V{>0xP4zDklB$2S1d~mt=
zsMCR0Tw!qK*RB8C*V3#*iWdV+oxmy2eQBjtT{6hjwpIh|2T(sKaM%?5K&S@fN|4Dl
z2Q)~iu(BE7VI<^)L+8?=-2!aQx9f4O5}fJK_YTBw$w;0=9cJSJtP{6->~o*x$G8`X
zfOrxO6jxjV+|E}b)ipqO?r(N}xxs2E^{ozKK-Bq~idIXIcU#}!>wsb4Aj^0bM(aA?
zgkfGzy_f_rjJInv;Ase;8*Ok%Jpl^?_Vf(b8d1R?NIMaKix73}Ke^VU3SUJ&8`VCz
zJF;RCs$L-lc-AVfPLPQL7kFgcJ_!JufZWS7yZ3AQKr8#M%<+rkBHr;SRlXR8Eu!ye
z0SO*<YS_+r9OR`NdDWrVed0VJZ6Nq0xKHb@IA~j4TfZ-qeHR3KLKc}*cWK=FP)+p=
z83=8{S(^4}y&VqZIn}<x9y-Xlg5vcBc2<9H5)@zaWUw8==|cq*Pi4>2hz_D%%;Y_>
zJomBtHt=aawF2FpT<e_t%MOTMI>x_fP#xq10!$E-bhISy;#FjHD+p(N{d+fRdELdt
ze|gqEg~>Y78)O&;g!0cPA|Y}VAkRCloTikmt2n0k7q^$rhNo?7{qdNgQHWTh(l=_j
zwFD_nQ~CF|(F1|?ojAOehhCn(Rbk^FfpOI%Q^(TFjWn{ok*lb9^m+Fq#<@}CV^Wwt
zC$+X?qPOn5hpUdt{iKW|5hnnWn&`2Tzy8&p8rsqZY;)8p2(>AX?|YmNQWSutRtFhD
zsj{@Zzr9dceW+uPy8a3%2bnvV0OX2#@eDvCf^fP*9s<?PfYR-%o+2e$;8Zi9hjB!V
z;sGh!rRqlDj-kgp`3Nt;kkq|nx-ZleB(fmp0BoVd>OBy`U#6W`7VrEMq!NH4G|Wn|
zkff>y4MKK;i&)_!4itv&S^?nH3IMx*gt-wH6KS>`=X;~vkRp#bw>W?sg<{x;V5XS!
zjPgq!`yv+ru@+DxB&lr%RMfKH4fVIKFxZg@_!Rxk8`*8O+bV1TR9*3_v0C^RnoX{{
zhYge{&u!SPq29`&FH#HRvnyo)?hD!DA|*TAnlB<}DTJHW<3-TXY6r(&{5}9)S8S;+
zUJmpX^nxtBgmN|E#_Tz}-h=Jg4FQk<8uTIpF9@wDfFu6;f&SF5L@AuEHnL;!(-I0L
zFpM(dSPxRSw@QjM8isy#A)(MnbODiz?+jG3IkjCcdbp<o<DGT_M$NvAL)HfmJy|4H
z!BzRLFXN6qU5DNr6dWUkKb<YAW+h183SdTz-M;ps=h8kg-(F?+C~;fNoLd4c3Ggm#
zosopnfG*)wvic$*6a!}E2%rR`%)pkdgh*~EPF=kRNPUh&-~(WatLTmp0pDc|r1_;^
zPMk>42iP~OV_EvvP@kULX*Ys^utS{mj7C`)wHFLUpB-)_kDo#PuHo6$@`aI|0=Jj$
zhG8z{*B@%AxUNB@q$OYNoAe(+U)n!_He%P`Qb?3#xG|3I8&kb&M=r=`qxl1O;X@N!
zT0Qhe6@o!z44`@y|G>m`p_R|?A;d}>X?cgrZNNO*>os#i?*XC!cqjk0lYqYlu<W)N
zXeX`}TkL0HKpq9zbt3L=`WUx3n_E=2G((Kli0=s~zB05E^pgrs#2!s}(S5wzx>ga2
zGSID84j=Q+R59(R{p={LxK3Mg3n)}(_vc<~7?Is^5&ilLw6th9i(kB&GIgY6cZ|Pd
zD@nvO<wVk^BGB1}hqo-jG%DXxr>;E#lWUMLbYSE6<<<+Pd*%4$HA=3+7kfB42+Tb;
z|A1gEFQqX{r%=R`DaA7?)R`%!M@LpzplRm@?FItAeT3ZvO4BU;4KwwNI<oQP+%=mZ
zCuFvNr62%Mv17O2Y%x76Pp2CnS8EgbGZ0@wG>vO&hww-QBw}D|K)?*Jmp$={mH=Xl
z>Jf^C1bw6UL4ZcEb@r3l4cfzbG=G>l)$no)uUP0P-arsvKhf0E!-)e-8MIp<7@U=O
z&ouQguwPYPYJw49Eewh{sIlVxN5Cr(-m~NQQG5_!8cK5Q`$wgeW}%B<Jd)RA${DqP
zL?)N5N|C}93@MWPI`4j-x`|AbJ`Gb0c(n^Y-Gh93azONDJB^G?LSDToZJLUmKuhDK
zlco1APPQ}z{>GKA)EX_tz@8uROcQxdTnu~w?$X}Aen$5c{44aOhE-!?w)UUa;K8#6
z$@e^TSEPd&+fq9rKez=%VwkNf1hm(O`*xJa^c6u803ewFXGRoKA@m6oD%41N@gpHD
zam)}f=Eoud6>og^NEBKfO8h4vWbyS@9A@lFjcE6&FOlPC_`}I?&f~6VbK&?>xFNf~
zRvQ}-%6&DQ)wE41SslkSjq8=&s}9W5G;>m6>QZR%1<$QGi(eWR|MY7^8ehE^<B+%-
zdene$wK=Dv2z;>j^?Sg;<prLEIF4DV)y>0EFT}?a5O5&(+*~rwUJ5i>R()@uHwH80
z943Dn9tI`aI!4uA*RGNZ7ax<CzGHo(6qv;UkiP+utYFew*>rs-B!ij!HaQ$)hzF6Y
zD6*N&X{ZMIhgiz24t83_Y%m;t-z3$KB;b;EGX$Js$L6e>uAB^VJ;FJ#>j6jJcR)J~
zcqq<21^rQfep?+<LG&=l&|$(uDkhM|y@Bq#ix}=Yo+W7OB?KFeK2;iy6vC&Nr*lJT
z)!3IS{kn2akj8%W!$~w!@&)H9S}^jyi6i_USt~E#X!l?))%Xc!-QCPxitU#|4@OVq
zc|TrBo>Jx?D8W{`a4sb9BhcN!aAWMDgG4PCM`jNiUvt8J+IliaX*;O;7!7xW2WE>_
zp_}??1E(so^fG@LTA?%<B^uS>6eFg7;Ik$vNctI8n-mFMnWDf`<UbF()mLeU!atRD
zBUD^}Om5xz`v^;3Zu*;}_cl7z91^}Dk$r`4Jg%c(J?8ky!#re-Z`9TIOeV@oau~IR
zO7u4M4tji>g+ieI<LX4WxYwM3o(dcsYE;%?MYt1ld!|<ovn<;1$6tu8dZ7uiW(rtw
z_cAl1fI3@*+eyKTUA}^!s(ri6nY+5&+Z%?*iNk3OT^9_?!kIUq?*ZSJ2z=q#)Uw0l
z<S;<V0Vk{RC16Xc>^9up76T&I%_N^LVYE!$zgUO*3Ii)ZrUSPSDVo5(wE~eq{Y}3J
z6@t>WWeYvMl_A7dzc6o`9aV<}s84VRU~KhmLiRc|Nw9hsu{LquCK*But8B$!%1H@o
zMvrQ&`R!dj9fMJe+`haT`P4)I`~duj4UuMuVZ7NZ(qm2-p6~TcGaMh{AU(dv{=WBq
zogp%ergHcjx5vL5x;APzd~N@vfe}=P!AxlpCP-uh7yLLaVEaG;_bICHrRSZKpiz-;
zmtjM+fD!3_SJR7uJlis;0GldB4STGGZ$_W%-+xM9PE-VJivsx!JE$su7rM=`w877P
z=~^<^<Hc`wTA8N}UJzT)zM-5G$BwDUfJ#GNjnP3Fj!s)Nigok1ZlmuE3SHMLpWp20
zI&u`$#22_S8#2vPt79)2^g)Grj2;C<UW>Q6%L}9_E=I`CdHhaSxM?%O!hj+t3LhZO
zpclbr|E@eA_(^L*%iyi%y3OsHE~i0_Pz25shzlhgld)W?iWb2~^8m<HHb00bkf$aB
z!h{1n(b^=1GBC*-XbeMxA8Z5K4E7d4Q$r7C^LJIZV)_tYWVD`Ct`5S0h@@V6@k|y6
z$!kEY6+$z|WQ)=uTH>nwo+c;=^OQO!PqB`&8*f*kGh6(y+K>_hgX%frk5Lh;Q5tqu
zLzxBSOd;F<Dmi9W{h-gk{ZjkJ0f?0YaCYBK>!=(xpiCGu<dtTGe#x+;ttCS&Ys{ec
zB_RDdcWWk@4*$(0^>MhMqRV+iiKc5M->LHDH!D<Fgvq6RxBC)?9<oG#Ax`NV%$~tL
z34{!%9Co6I!tx`mxM>SQzK+#~CB2hhN66BdjR^l1650Fkrg29B_mIIE0V00Q@qFZ}
z<1jh+Nx6Y8U!{d4EhE`;rH_7ZaQgifLoV^AX{tmArHe0en=nbmPmlSR@}4~)HuA5u
zCKT4)MD;bLx^fRS_AM6RpWb9D9ETAbyKYFyXKGuKe`aHEKk(}0HsBGM{+GJzi=&$b
zQ5~OE1U5Piz&wGY4sf+**Q=H$e*Jq=rY`x6U*o>T@K7^3)kF><i+k?qaFbh>$kcs)
zP2{K*f{A?t$xT)lTyf(}ni6jfy}0ML)ZK1f&uJJA#f9$*7%U~LC%7z+>|3b-I9->_
zh`z^HK}e7FWP;)XzxZ#U7_~}TUPYVa&ejH=T24E8o3uQ5-uIM_fPl{5zTdMGlE(wK
zHooIj;ap)rz1?pR!eKZ9h*Z6dz-wbu5|aLAv1g&M7m_mqF>9S?S;|rfmsmXn)M2^H
z*8u6&ePO>gd9#tEGT1x0uBp7v-y;G?4&&KdfWA`};A}~Ek{iQz+r5GBSGW_{g1H^F
z%;>b<bj|PTWiM@nRl~b(z6n<1d8khUKN-jxb#&WUxq2JQu*o()YBg(`kuX7h#{9*e
zD@8m1$dI2PWuJWBBvt4pWJ~F{$y+rtg_Q~IDr^Dm$AIVYBL%B3;cH~bjl@N?P+oOR
zp}d)g^!X*1K_ohX;ZMDmrFx|i$wq@BtrMXNglJaTnQGCn<49JkUYDGHa${(G&Y{T`
zCly@+CQSZ~pK!{Wf12iU9$zG@gXo%krO4mqSSsOeH1*Kz2jvI>+;q~m&W>g}`VhOx
z;-^+MrF{)C`$|tNrBSns0lgGpuVVAEf_O2lb${1Pk3>*zX68=NZp=qf`{Js#&`G}y
z=9ZpfMeJYd7^hnc(O*DV5}^_o!g*d(AU=q~?Sj?>o^zPcqZ$!a4TTyzy6-{|YWif<
z(Wu2z&|?YT0-fV)rqDSCJRq&(WCZQbVUXFQ118D@)PkqbC!Hq`Ex`-BwXqe#asugF
zalGj27#Uh$3im1HcY-ZKL_`V)rRjYjTH~hyIvC__{${J9`=-rEP9ozt(FQRkM>D=#
zwNXQNWH4~mz8RHO)MWi1+>n>SG<)xTNDg5zm4&A?nYzruAsxyfK%CtV_WC5o_|juA
zIW<3I%=R~T_9VOeO=9?+gnc#R;WnihMke}omxtt1=T}<(F{Y{OlMjR@bY`oMqZ9jW
zcX<vJF!*wrT1F?QlfUrzIT-oFVaCp0%Fi34Ey&qF8Xbt9WIvXT9+pfP=Hrq-lMBG<
zCtHeOQg;f5f@-tY^L+-U<fb1S3zGRJA!C>KB+>0Fo&6HYv~ITQ*{RSc2SGQ>V!#ZZ
z&PJ{Hi;CS{0TdDntB~TeUisph78A)ds^cvWBXb2J4)Mr0&+>q|QQC{}y8!2uplypu
ze{}>lIjC8D8Zb@`Rs9<ak>0>!qZMzI0t&DTZeqPz9~{Mfhwj~<h`gn6KzO-&rmUt1
zKqbMDoHWrsrKkmnPUaYlnKKCv7|h+o`oZd%j9Z+P3m$36^np+il;Z9IoKbI42Kx-i
z^ws)ua1P2lp~oS10|3R;Sqex7$j<r=4|<Gm?xyg#c9=%Saghg%NJ~Kpp5!=OY=0Kh
z1xq~wORJo=fscs3td=lzRp4?FxrY+CW$0;j#>9SA+zrf-!*AI8NZ4twg^>r)3k`%X
zS?)wpm+xA_NOtS%!;DQ;&lTi(=ya(z39K*?3W8h`P;Xwzo=H6gOG6(QdeW<RRP^YK
z5LdqT(RuZw!fx`46<<snzg`$X+Yck0^=sIthoii%ftqWVulLQ2nJ%<sQA|C2JnH=3
z)TBOMa@MWZ&-8xHNWvGczLHtu{!W$!MpI`it)zc20lmO3ihS+?Da(zU0MZ6(Xt)W`
z+tomd)MsJ9YFTqYRI@fQatM*9WQ*QEIC?@&0T@kX-TlrRtU<c;gm2?`9zZx^HwT%*
zTvxh+Ex<O0)MemW7-2HVa;CCOL6%bwNxaj&`tb-|TCvIxzcu*-p+RwkeL8*3z8Nqd
za*;=1<xUTQE_JUJEJiR8d}zY_37KX%nOSxQXCt^SgojU{hmx=xM9&FFcplM{jT}Z~
zZxk{}^2%dEtMnZ!*hJ(R7XP|qDFHupu6l5Na$^iD$<*?5lhlorz=1f6Jzp1>f$)Vu
zxGRc6qse!DI!l7wWmlbsO%My2^x}@mo&y5^J*KIkf%!x2RPZ+PP8a;zN%K^RM)>l6
zK!0d}l|LUqF5Nl#sQk|*+Jn>^tmbiFRNXjHhBv>vRjVyQ1V0u8+JTZj-zFJh(JXuw
zr^wEd!Rj3l_4i>_fiPk1MgZt5vQKCqzqpfodWvwqiGK_TKb9Ae8h8`_sF8lV8A(<r
z`!*nRSB-Xw`Te(~-I@xvC8&M?PpkOs;nwdUW&c>)<R(jl)-B*A5VzEBt0utMI&_G>
zI5t~TaIx_hAcPH0N9Ovux5<n|YPcD^IRd3#lT~qknDf;uMeW9XQuA0NCr5~il8b$g
zC8GlkZ6=?4$jNmM4)bLAUuQ}QN>sZ#nL^vc;4S%XvWS?RRFp$G$T`QZ$Jw^<<pb{V
zQAdsA`L{t+EnLBMQiFXPn|PF^;@Ckco8I3ske@Kb8&>BSoS?al-%!@I$U+9t>xL!N
zRofh~4_KOR8KPB!pZo3_vShL_iSSq=dp`=ncb5c@Oz<Z7QtQBlAXi=Vbqvy_`*7FG
zhBQG*t>{ci+2gPV#-W7lL!EHCfRocMD?xV{i?cy1Qj7!mKo`4MxN*lFJ$0>2tsQ4T
zKPHN4wBo$F_Ir+PtnHAz%;=43+D6cE_OR%D)p3{rP!^oLP_!;&;J0GmC;~q6x?O)S
zu-yTJXDtClYgh!pv|u1{-3m09cHe>e2SAb}9$tY<6T;IYrU@srjIKgvNe1A<(^f$C
zKmyKYl@SmsL~5Ks@Y1uWfv>|Jl#L}di75xsV8<<SmaZ!TeNxf|Vl+jpmN^nt2in|4
z=0@la%92bZ<3g*MCuXYBQa=NX*>@)xP5rp&+71bp`7;Ilq`~=0Oqj*ApU%#k*lWCy
zTkt$3eFVb#+-?SP$}fO7{*WIOWx>+tb03E?u`jaeD7$h)-+|Fn3U`=+<h!P}$&HI)
zUe`J(!f!{5_8Re{*V2${rkMA?C_u2!fQZFypzEsp0mja$rKM*AUad#*4mBRzg8RN4
zyJ;ccZ@8sr!eW<YX<M!ygQYqFj)ixj=bOG%!-m%b!?{!op&6uX>$}tlbwtC}B7dYp
zE8ck}Egx#hD;APR_Gulrcdk=`6dfQGzLWuC@*v2UuT2C(O|kT_F7tb*fLNd)5fH4D
zxgI$Hzy;ibPwiIFu!RI#b8sLl<uJXZSw}wc@J+zV3i4d)7AjX8O@nNnKCRTE5H2!5
z0%Su#omtgG`PsT^Ere?e;UKAJupX#(IFh|}p{O3<vvXk937F#az_UzrrU<USTPo1G
zoFwW&UX~BqJcgwjrkY%okk5Z=WT8OomILV(;nJ7QCsqOjMf850!uL>Vt>-g)XhlsY
zt}V%&to0lAHs)xeU+k!xycCHrD%^&8WqD|=<#TFi13(CFYSj6XnLfW=pb<?XBP?Y@
z(8YI?E%{S63EVdq5(d#yhG#<1LvmYju9SXk_nRJRIRvbLJF1!?V%l9zl`wQ}RG*-J
z2XY7LY2%+jmh4t+2z0-Vg>QZmaZwA%jGb$Mj>XA=Jydv(4X_-ZNzc@JO{<v|*WNO*
z{sbT%_J9Zqq_$D$6XCL;+*9iA2KK809gAvwpygoBiFn<lUOz(9tYGOlsD+-Gg?>_1
zKnX_LwjRg(Q27z0$^ZgE@7AbZLV83KHw#F(!56!XkuL{-DMH=nSSp(`kSO-)+a&&q
zVu1-*aQU*5pULPRa9v_48Yqc|G+Dv9$iX+Yg>Ksdy9`IYjl2(kH3J~T<~)raUpRa-
zcsjwAGUhRT&me4C@UX6}0)iIytcg_UDzZmR^%^t3Abx&dFclg&#?}%sNpx5m(qyXJ
zFH=}4<(I(zg8Xwr>G&0*o%WSzi^bTfOmufS(uh*WAH{b%;~V>v=#&{8CRYmc`{ZrN
zb2HZSt2$)SWTi+igh}coMFKMtJ=~}fJ<L;>j8wlgJ+&0>st`TmBKVBo=5cy(`ENm9
zmrPT&0PcI8m)g+Uy$+8hK2ElbQ=n#ZgBrXO^J+Dm4*HofBEmY=MuYQ$YO5y&FN~Se
z2v-*SX3ri*#3JFwdT}coX|2)ZeH2)~arlU#QLD1YN?Qw=@jTk#$~`iT2#1o=Gu{HM
z)XS2W9NtVb35&&wSas)j#E0qA-Jw9@14(f}2w>Q|wQtAz$z&P%f;tQxFd0C9=XFiU
zh-7oHSWuj%P6RMyjpU3=XW!ibQU5)cL1U=Fj7n>u_V^X@N`RrUOF#Pk<_zH7A#EWf
zj2U|#IM{zx&<IZ6K^)D5To%w(h=^Sbfyw}$2Axd>#Ej_SR;~E2fwndo)Uv6_)~qwn
z3=-{kfol3@&)pL^%>Ty?3cu&SQ3#gT^W}CgjyqrEM4*|=EZr#J9A)nClr979+?zT5
zwgc1!tlRK@rbqkhm_k1i%h$i6^M;3?W^o^Fu6>1c3=kHATL6|k1vIaTsCaTF3Z4QY
zO>oQUWk5RuO8GR79n2h8$J^Wj^zXe%bO1Oc@d5tUfc^y$qbOTIySxm@!P9me3--7=
zYo?;id@!I{l&<J7ioAQAF`>vmp^jC73<$_1XZNspcE~HE@09VI9qJ=gyUav}Q8sWr
zNR#ga9}?|v5Eg2NzfLdwnZUzHI8T~PPTn2N2qNDFo&Mq*p?-?L3gCse11agaoH?ao
zmSEU$3+}m9weh#$q(Q2lEb9mmL7;)=8RWvz!+wRN8RF^&z$rOp(2YA(PGhljudrL#
zG<?fkAYutxl|c!uGS0U1)RM-2Ruy=TcYtc;su@r`$J0#g&evw_IFL5*JsqW8D>3K?
zi536E+=^}vKz$RH<BGF?l$0rm@))Ojdz3n9gdZH;X?H!+2Wq6af9!fn@JyME2_y1|
zw6vB6Eq}sgsO8R~c#lWU>^QAAKU3y4&8R+=VS3z%f_?x5iR&aHEbP<!5`P0*7gB{i
zE^5#bhO_|`g)0D9*(FxIlsLV|D}l8D*A4v=l;i_tKr7HR4S1iZqhL9FvDV$Qvw#j6
zC+PMDrEa-oEOVF=0J$1>6{s{_`c{O3-Bbel?zk&!m}#hdVYH6pTfK77a&kH?;Bkws
zY-ZR|j$f8fj=SZ-Ba!+-GwPauXs5Al4XDnw>I-rCUdlG4aaoK6{;AUz(|)Rl$ON9C
zdxSL>5T6cdIA{Hy_}b=n*X)J*M;gDQxtcb<2UyvgX;GmYfTx`MCFoE?j~P0E>33oR
zQRz7lAO_Ma5b)2v0DaT!@w#>bbZtw%EM|EB>9hWPaghCZPsp;tQdME!{9Bl{GV{N8
z{>$%aMfRqgGQ9d%Up4Qp=D+-&x+U1GAaFT!@AH3ac0bHtnz@(%AdsQ)Vo+nlIRo*5
zA2Pd)Q1?n`h!Gg9@_qBzk-dL&y?=2f^Q{AzECgC;&TaC)y57Hd((mk8z;XX!5THZw
z0Ut7pKFkhYgQ)&b8{t3P2>Rz$E%toc3jZUUAYQvqR^i>F`hiyAy|tTn;y>91;1&PI
zK>)Groh#4(Y4zrt|973lm27N<yMZksI3|Ue6!J};)Ajr(y7#y><2wXzdtGYiI?S}h
zERgUobui6q1TFf}(@o=dG|7HSn&|ODJXE#!oy_n|r1Akq8gik1H{TU|KVQK2bK`da
zDlDb%Wp)GO-2!Hm9{O;oxPp1?Y=4v+vjcRwbS&XYX}h6}&?iIRX~<tdFqq}iKr8l-
za$t;}{syAQ%4U*2MLF_Hl!b?;9!6mDkPr$Dkm2WOs8ARe)hH^|*J-4ZIJJXq1qA_o
zf0w^()&Eg`u%TjNQCzxu8!ztQ&4?~TkCTjiYCfZoRunHZh$t=e#2m2BhbniN99}|C
zO`pX`_lET95oe<18r<}qH6OVMtHJ2K#)<tuewGi7XG&GeZy-xGmV_<ewbUi~a?^-L
zB_Sat%YVxBh)u&i7v4dK_$411cP!bM^iDnO220me?-1gpJ2hPQ#RcsG?_ZE{Gvk^*
zYEi$x)?=EgccO$)Qkq(+3>u`Hc+|93WPql82HxYo{)?#%PE{)r=E4#?7y7NtMU=s!
zOGy-@dzRq&@J9v}&$ojSu<qL^kH8eMl9t>SXTGSerp?MrmKYpcW7tY~Hm}3~;{<k+
z+hC-LQrdszkZr%AetwhRsh0Z%tuc4<u7i$)`N&uliSTOs`(6F1DNt$V+j_>=7j0%D
z@wssGFT%HZh9*P*PxVWyuOuR!^GZ92%q~zX`8gmYzM3sT`jEJ8F6th84kSVl-#8oL
z*{>osFrLpe{=emYL}X$)w3{+C)A=8!*z{iG|Bs8Quk5JO#-+H9HY3YD2`?z1rL2Y<
zE0D$<yHPBM&_;F4j*a@rFX;w-UuZ}ZlO)FDMU5koBdU{s<j%WxHJ^p%N0B9u!Sl2H
zK&xF8lX);I0j@!tyt4)O-Ml9NVUB;5NzH$L@4A%7IuDQ?vjYJtW9tesecWoI=tIZj
zT#YbZ&tfT1shkI#-R<73@ix`{I(4S(8PwRJT4j%$#`9gNQO7A<DA(27r1!-~rU2lS
znxyI#at~0QD+#d;8yukq&0Nh=A0^PBnP1vca*~jiblXH8=w_}rw}vjaFyPp_baake
zP3DAKY8jN;t4`=0T0P+AQoqFZz&Fw#>7<}ug#SO}h5h9&o@>M?>C(9k^(i6v;Dq2W
zR_{|v@WCm;%<9deKJ#tWpYVoHDZ$^T1fMV?F^K#LGy1=_5tf`0-Jk}Pxl3!9Qx3|=
zoniUF<A7Gkp70_@$C7Z_MUP|atZ;Tl<Z8xi`{x0@9~b986xvWog`lY4Nq1)^vV)Sp
zUsXButm_CEumw>3XJ&r5m`E@X(}3gZO=_9=@az&-%Pu%JS4RIaF>F$3oarRV+KNAx
zS+hy8Cal3}=w_NlcM~-64H6LMV%`o_NXKcO-0y}*hNSYGMVI{Cvbt*TYr5u<5rdJZ
z9ap-!WG8+7Ys1HdpS&W{kCqdQ;8=~~>(mrYFyqZn+tuCc(c5{zgIPYUTD4NjA)sBx
zj$*2<$h+*X<^g{c_PnGAXfFCLyK~t;@%)VVSt9{5rx+Ji;Ri?tJ}lPbe6>yc4nIES
zw7!6EHpWb4Jf{6Vm_r?2ezft>R<`{Ig2oVS@te3S0fcbB3mn_80yA=PwQi$lGr7l0
zUM<sWF~d;O2wm`+S~uP|BrfuHdFFpR9qmqT%rZ`e?W>297D#ZT5#p=DWs6lgYl$|t
zy9vnH+@~kP$s=uG-fT&@<YxEI*3HxuZx^QV*`SYn>Q{D@Az?zw0VhTi8?O!9SC>sY
z0|vYqU|+$H8m}UmumAX*cPp;nwZ|PSrjsYqx7}R{n*y{F?6GnuhE-mlHVi9gJNXqu
ztH$lN$<Ti<-vuQ2bV}pT_d^aUs<r)1J~RWJ$(OB6UH_+)scPe&O7_nzQOCVYRX_0C
zAvl&&Dy!8hnp}`2>4s-%-*GoOJ`{0yZ2r{ZU5t;TgC4K4g@G9!Rw%R1g)FFd9kEy@
zK>MLyf0REo2t9>~W0l%;?I83}Y_bSMCpbpKChXgfMt*iAheN}S3$?8Y1wxNaOep!|
zjCI_jb{vW>XeZu^*e87)VKBWknK?dr2Y@9qo1l4hr?l;cs7UKawkXuCK6*<>-N>bn
z-JPDN8BJ#){?!fZ^=ysGka+rfudKQBEYjDYlGx$@wzW&rx!CNE>6E4V9h?W+nNj+a
zU{q&TVP*l;*@a4X2E*;Ve)!nup>r2$a=z3-av}}oP-{gz<APs${DAUYMtc9fj1(Db
zFn?T~?@FlM)w`I=U+us-++#Kneh*do#IenE@ZWou<)foIm#ofDx8|`3^Iv`!2meHt
z&eTPJB1`{6vh)w}!6(*S2`~k3M|ONLOg5t-0o>6i0fhmWIyJv-+dmrj8E^4+_2!wm
zPsqr>BO`ymdVlXE%3M!vRQ0^^Ge!e#W`yGKoF5N5HdL5dSie)EnQw0Ht6-#Q;8??=
z9t340Och8j^<+`$yOO1UaZ&#EN9T*TR=w{1J3{pDTRI=r0>hdfRh1M#!<w{|uQ|6A
zk)jQ^=zSz4|GM{`k>w^|r$#c7yQ0CQKTy^7ZzIb;DwTh252T;c(k<`vo@ZkV!_{*C
zVvPBmp74JJ1wE;p<2|#+K8*Jywsz<LkM&ZX4aztu-{ehhZ=k%X$?W8-HLmxC5!&>g
zx8Gq!9gd?KNwnqfKf1QYWpdL8Cb@rb^6w_NPbzRKR5+E|DDDc_&WrnkJUZ^oLs`_l
zmuhZuDKCVLtaD=gSIE!bLjTk0=Ab$7MCixnw-)nS(vGB5H#9hnaJw!(28zg7RdPKy
z0}jL46jJ2;!T=vi*&f-V5@~U?{vWf(KM5RX?eBXTW21v#0g6I|hU9@n^xOZ2QbJkd
zydDu`oY4^mG86cn1#0<3675`x;AEENZfNA<%R1l26pHm@jKbi@Zj)Xl1Jfwx!DZI^
z{dtz2PhP4Uw+6Xu|3~h{-sJ~D&kW_8DN-_KrGSv{%)rYFuQd3A-6*l>SRAAjo>sGR
zt1s*_k5(LZ^TnzUcE)l^^u3G-NI(05f)mKDzw2H{v+bhm%qhduw=@%GXRU%M0=huM
ztRZfqZ-=GCO8H49#Bfux5O6|&;87j(lWv$sPD9W{+dN^f9@eLF&@sr~@t2QGbM-rO
zYK#Yuo06I%R_aPyYG%~o+q3i-dq8W9f|jQx=88|P#-=oUTrx?Rq;rEglcyT{Q&LMp
zaWjns(rErwKX|}H&c500b-U=t)^`QC|GZFJL*MVq`Tvu8bOr_#Q%~>O$xjN>50DbS
z1ySdt6`u&>4<d~Ja_!~^ho91-Kih3_TEwvpYWz-Idhmfh;M2JV^*x_ZuRk+A{|^}W
z18-%us#jA|=$TxiUK-<LA@dC&SJF7xjx2PS1aSE&n_OOd()M)g9-b}l_m34Y9&jD*
zIE8+++4)aWS3V>6+DGz42kap-Q5w(+RCAfN+2YO_*M}Gv|HIFnv8wOZ^qd|x)UcD5
zZUnI%C^Gy{*}}g|q&BH((Zhebe<qGW3u^x_C1;y9LlmF5MMV~}^2{HT(LWeen`OG*
z^&Pj{a{6EVYW`v;MRe$9l*OKDs7hJ?{#rapN*aa^O{bs!kLs0x9>>{7nDAWpa^}Ur
zc@q|X0`p92)hJByl{>jV9!W9f9g#U<B5(E0?yuJWx(xc`@QZie@_IuZD9gUAuijF$
zY~cd&&;8B*HH){mhN;i%`hU6;%%C7`iOs3@NR{NZgQ4R5i%$J9RAjkeV_k@*+QQFf
z-kW~V*8B4pn!S>5i{Dm#w;Bw654l%N5AFKF!>^!=lLDnK<>*cS`aV9vHK&ENJzcfq
zj{)6vS?vN*JZMli^L-2M7`}h8f#RtG#X2xJA-Oq9jKLHNCVQ`G5*j6_j$>sfdv-6I
zdHe6Ny_!l7Pwyek>mu)&I~(209Q;GaI2rseEA-W`Vf3eKWG>ELR`Vvj?q~gL#^z1B
zyRhuvzIpb&3l=P!Hwy0axBq*0-bKFs&wu|_#ct}%RBsKc!XvYt$2{6PCP$K_26_%@
zx%oeEFA!E>oN8kSbM~}@wjQ#_Sq^VuHMepcE6Y03(1)d~l#=4sed@XQS*%*AcXdnu
zkd~%@cI`N;U|VmyFXD7=I|Hc~K2fX9YzbRIqd1BCrO{(z<;P(0RW9GmT<y9dmiOlp
zWrRHU>v<58q^5I&<2cdB`xTn2AZhX1fKuEdye3R075@rlT3xw%e8!$DId97(E*ruf
zapl|~Zoyse&*cK}0;>{>v4EC1U%i5$oYIc2LXO>IM2J`UAM@!VF4(|b5c%e2Egdp@
zstbLy9EMbu#xYL6Y3h){pcp+zZEf=z?N?N@GT|BQwnZ5P^<{l_il{<e#E|XU8pS#S
zG_0+bCJl-m_|GrX;@u2j)CtFaSO0=$OqV7+KB1tvIb7{QuHgyP^*E-$)a3Wd)p4l6
zth`W{P;FPvbVsmEEGSYK?kHa5V%TcSI?CUzUzx7y>isIY9m-;J<-p{(h~dV;+?twv
zQEi{5#LP+^?qum+P-6)Wj)fXD*3sSM;4mt+@HmU@!_T?9&lG<<4d;TSe;4m^s;Ari
zvtx0zoy7&>8jhnShujuLTS%~_3cO8mt)oS{7rvpmXu)0hl`hyEry*RccV)l4ZGP7H
z`dg)OEeAqWO_YifX8Dtd3vzIB{#k+qnVZ)-X#2!WJz+}S^?L&jAiaImt`#;evI)3s
zj`q>EX>ee1i$^Xug%g-Wr1kdI3Dq1eSGD4Gfkpn~6FR|E%?yHug!*YzUqjPIbo4bp
zsd)O~(E@Bm3%#SdE?@lsE2a9%&9`?O)=AgBkklK?q{`VpJ6q=y5)0Zxv?O=gxoGkG
zMjW{>?>W$m72zoFbDPi9oszlrLd>x@K@}o;H=9WME*h0=P$Gi&r7JjxcA3Z^+fh07
zHo~@tDtTB+R4RpAaHZgT%)w!fT_dwKu@o0!an49wMg8LvWo@`jc*yONtVO36xYa6!
zy0w8hE&UJN?syoDKuy6g)jwmaSB8f)2_;L!U`R#+(0o)kWZDNl&?3mYA%-mqC$;(E
z?aYdB54$uUl*l<#(2+#()K+ci-8Ugyjl1PMayk4|>dV%(vxh)0)krC92L9;miNjI4
zSV^KSNKjYezoU4cu!L$+*d36_Zo3&06)gkPolyBg&lg>uGALOnJewQXxQ1ErYm`N4
z-4aUZPDkMzl{}OpcjfUKNC5EbSc^kk%$@AwiYQp}s$}WTLtAhQpmNlz8fL{UtmjU<
zXqtu0+Q;>hFG`t`O-WDE+8<RDKX<5Jc?)~p`);HVeY@qd`~Hj*-}$I*Np&4P%bVQM
z>4w<383D)JbUGV(?|w$_@IZW3sDK@iJy;ISxTE+ZoIAQ%RzB^R@0F8MvSMPXA%m-3
zNX-KNcB*7U+6Am!^6$tQ3-(KyjzIT9BIsoy_ZldS2JSrEktAyK?vLpaK6NT=NCVw+
z+UZ}8Bu00w=zHThE$6bC(292yNEGT(i-UR<`bo`6$cy}MW@)aNgXFv3XX`C8jRFvo
zM&x=wPbIbOhl;kW*=FZ%5F6M>#IJ%~*(h^+bwY;Ug|_oM1bWgugc@}tar=nPUK(RM
zBxSu!b7j`#4bbfe;XU(cera46r2E+}(CdDZ5T9Z(6?NnpxU4t}P*;zcu%OdFyG6@7
zWnpr!oQ5oERK1syU?p<((ZMBr4JO@GSL%P6EWLce?B)401S{*udn%|mxds$|qB&gX
z#ebBp<uWuDr-x2@TAPhEZz8r0+e}e&3cQ&idFq->4i4U;Z-i!bRY^Mz7M^pXPtnd-
zjh`lN__-t2qbmol;;lvH);ASy4#$rXU<kf~NkF+yJ*g-%N#@xksU$&Yn!1t9Xd)dT
zP8=A^x?iZBkF1)C{bJ_U>^DX0epc=jX;D7xc4}ksbx|<cD6#+)g^*o_5Ej@=>8WX)
zq3?B6Jy{CGt^%Bk<ncr9q6B?kud0+fZDv#nuG=@y8%{)#t<4z=7ySTgy^nxC*cM%=
zJk$>F{N;>>(k;VjoW5!f@1gc{MaH7n*&&sWW#SmEHFdRZX8eoUzV78vRl}?fgy@%(
z{2bgH*+q+ej{g(amAdkHOBPh}(4e3x$2o=6Vf=LH^eR8$<)gRsH0``)WL)oN729J|
z>8>NDOcAI}?!@cP+}RIbdc{dETY}LtiaKJN>gNI1;P%(|z93R)Qc8A=e6ouy-Gw<E
z+pL~tJ#?xX%mbsgjt)QS_JB=rdb&9z0>NQqw09Kh9Dryo_t;OR=3Y+Pf|?pDFFXv}
z_1T<dygunlFWygAVMtG;Q02#%@ud_vPV#>3zWQX{xPzn>O{8pRee5IZZ;g@}#q^12
zD(Yz+G;J-@66T|J@5={YP7+FGu2xFLeuJtqt84bF-=>z1T66^M^E^CR231gTWm-?U
zSe4nz>$p{8kt+Int3bva&q^lm(>x3-e!D-{dBsaF>%=!Oh&UgYO`>0Xa@}jRLx~<H
zCgA*NSG+7Td`X%hSvMnACG$O#PCJ7Ri|yqs^Hc&;HgMa#-5Gp9c&zqvFWf*g&c>gJ
zwhq5MzTrM`BxiE3VfVfJsGBXIwVN#?3ERjRVSD33^6!$%BN)XzHRLf{mR}U0V-{UX
zl1aQkW+1mQvUnDxb4vniX5&AL*O(WJzZx-S<nsdw2QtvQ+@N^IBk9VcFlu36(o4{J
zY_Rq`l@<n@GR6}HFUgFEv*<LZV2Z#v`?XIp^5W;fT>P>H80N)9k8hgKx~UEzroSqT
z(?T~aI)pD<Jw`DGLN5=m@&I=_D|kQu=%JUTZEMHk%&iluZS^0lGEvu3Ai46|8$uqf
z2g3p?SvzamAWdIIeLFrJ^Kz$R@0^R(xxv>|TKBvZIT)#xmqAdaK$Dig7w&1YLu8CA
zyz=c=Y3N*Vol>^ok3jb5mm?i5PfyV^I#eePdF@_Y@^DyS5k6I33pF0Z=(DHyqe4nL
zi*HT4tV2r@yR>moPnEUnDc%#LGn+i_`t80DnXOCXC;=_rbLzA&P>^ygrf)X<%CW1v
zY*@H-E~LHoL~uu1$0R<vks2+NsdXgpSxCHG^-)+dp%Ra;#QxSHNU;=6Kh3JW^Q2sL
zoW~FuXA#^C1WN}tmTb4PWWGoX0vj*d<C@o-*O&g7SgZV5MWm{6TW)_M+?W2GITC<K
zPJm{t?c3AFTz=3v;Sk0_Um0}^|FF(OUg(S+!)M>xod%kSJjt32un%~^xy*US-NYU+
zh$JLfg(|#iep-u;c43A*6JiaTOBK8YRR{4bly50pztY2K!sFq<-1+>a*RXE!l3NG{
zGL;_+3tGNQDLfhJO#VJ}6S;L8s6?~K;Kx}2{cHxyr&KTnLbckH-mktvq13ft4h_NH
zUv=6_eGNJ^FSrL(G7FPr+#2&<BWlOTU0En?#=h33Yc&R>7H;>;f(ltJJ*sU>aT|kt
z%0yJqWM8w=c7)G6!~&sD4?V6he)`Qo;N%AOf#RmKN&=Z((i+}{WgWQ_k&80?Fw=QB
zir<b_u8i5iT|S;euPCM<r0H@-xGIT2GrjKCb}~J-@SIxlISVhoJPmc4|9axp*aN?t
zRLn34OiXR#OeaR9vJE@)nHNIFfRfbB7gNkGY5`%L4~CqMUfUt_CWxR?Jr@XvUEmui
zZGQ;Nu`zvNgBvcsncq~;+u^rbHwAgr`*RetLaFa)F4|kNu&m^4H9JoWr*9E?#E;;*
z9!I&4FzZ+82u&Cwt(2aqHtt_`16bp?7WwmrkV3m=Kg4T28a!`5@vJ3IGPQ7XYKKg!
zX`!(H)rsQJ;)k6OqruztET%;Bc#!BY0|(lrxx=_8_DIyVcIXCbpS6Z5+6O{mek?te
zWq?UXUnoJk(ccgg92gPVj(Gfw_HirTY0E|}RGLHS>z_tB&ivOJ#ZM7qMZNxmOI=8r
zB1OxoKkjuy|3cn#A`*6d4`xrvo@Wip;(Wvz+rp|@yd;KPWIQ&tk-dWOM7wffgvH?n
z5f%%UExNXT#=3()|2@I^^sAp%;D5XVUg^owZ?O?EK=M+s>`6shq{Z8h1b;bf17hiU
zKR*Jc`@DZH_+S0^D-kVT4#GRuEg4KWwP*P9ngu^>p8fjgNwYvm%xsvwE`0XF?SavR
xett27&G_c9gG%E6Snxdv-=Lol_>jGn&qjL!qmB_sY0x_k?z7nYVE2y~{|`uFp-un*

literal 0
HcmV?d00001

diff --git a/docs/web-ui.md b/docs/web-ui.md
index 251e0a01c4189..6653b00fa77c1 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -406,6 +406,34 @@ Here is the list of SQL metrics:
 
 </table>
 
+## Structured Streaming Tab
+When running Structured Streaming jobs in micro-batch mode, a Structured Streaming tab will be 
+available on the Web UI. The overview page displays some brief statistics for running and completed
+queries. Also, you can check the latest exception of a failed query. For detailed statistics, please
+click a "run id" in the tables.
+
+<p style="text-align: center;">
+  <img src="img/webui-structured-streaming-detail.png" title="Structured Streaming Query Statistics" alt="Structured Streaming Query Statistics">
+</p>
+
+The statistics page displays some useful metrics for insight into the status of your streaming 
+queries. Currently, it contains the following metrics.
+
+* **Input Rate.** The aggregate (across all sources) rate of data arriving.
+* **Process Rate.** The aggregate (across all sources) rate at which Spark is processing data.
+* **Input Rows.** The aggregate (across all sources) number of records processed in a trigger.
+* **Batch Duration.** The process duration of each batch. 
+* **Operation Duration.** The amount of time taken to perform various operations in milliseconds.
+The tracked operations are listed as follows.
+    * addBatch: Adds result data of the current batch to the sink.
+    * getBatch: Gets a new batch of data to process.
+    * latestOffset: Gets the latest offsets for sources. 
+    * queryPlanning: Generates the execution plan.
+    * walCommit: Writes the offsets to the metadata log.
+    
+As an early-release version, the statistics page is still under development and will be improved in
+future releases.
+
 ## Streaming Tab
 The web UI includes a Streaming tab if the application uses Spark streaming. This tab displays
 scheduling delay and processing time for each micro-batch in the data stream, which can be useful

From fcdffbd5c626de371754fae6a32e2cfdccd9153c Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 26 May 2020 11:30:14 +0000
Subject: [PATCH 0737/1280] [SPARK-31771][SQL][3.0] Disable Narrow TextStyle
 for datetime pattern 'G/M/L/E/u/Q/q'

THIS PR BACKPORTS https://github.com/apache/spark/commit/695cb617d42507eded9c7e50bc7cd5333bbe6f83 TO BRANCH-3.0

### What changes were proposed in this pull request?

Five continuous pattern characters with 'G/M/L/E/u/Q/q' means Narrow-Text Style while we turn to use `java.time.DateTimeFormatterBuilder` since 3.0.0, which output the leading single letter of the value, e.g. `December` would be `D`. In Spark 2.4 they mean Full-Text Style.

In this PR, we explicitly disable Narrow-Text Style for these pattern characters.

### Why are the changes needed?

Without this change, there will be a silent data change.

### Does this PR introduce _any_ user-facing change?

Yes, queries with datetime operations using datetime patterns, e.g. `G/M/L/E/u` will fail if the pattern length is 5 and other patterns, e,g. 'k', 'm' also can accept a certain number of letters.

1. datetime patterns that are not supported by the new parser but the legacy will get SparkUpgradeException, e.g. "GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "aa", "aaa". 2 options are given to end-users, one is to use legacy mode, and the other is to follow the new online doc for correct datetime patterns

2, datetime patterns that are not supported by both the new parser and the legacy, e.g.  "QQQQQ", "qqqqq",  will get IllegalArgumentException which is captured by Spark internally and results NULL to end-users.

### How was this patch tested?

add unit tests

Closes #28637 from yaooqinn/SPARK-31771-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              |  50 +-
 .../sql/catalyst/csv/UnivocityParser.scala    |   4 +-
 .../expressions/datetimeExpressions.scala     |   4 +
 .../sql/catalyst/json/JacksonParser.scala     |   4 +-
 .../sql/catalyst/util/DateFormatter.scala     |  16 +-
 .../util/DateTimeFormatterHelper.scala        |  34 +-
 .../catalyst/util/TimestampFormatter.scala    |  15 +-
 .../expressions/DateExpressionsSuite.scala    |   2 +-
 .../util/DateTimeFormatterHelperSuite.scala   |  11 +
 .../sql/util/TimestampFormatterSuite.scala    |  11 +
 .../sql-tests/inputs/datetime-legacy.sql      |   2 +
 .../resources/sql-tests/inputs/datetime.sql   |  20 +
 .../sql-tests/results/ansi/datetime.sql.out   | 163 +++-
 .../sql-tests/results/datetime-legacy.sql.out | 898 ++++++++++++++++++
 .../sql-tests/results/datetime.sql.out        | 163 +++-
 .../native/stringCastAndExpressions.sql.out   |  15 +-
 .../apache/spark/sql/DateFunctionsSuite.scala |  20 +-
 17 files changed, 1381 insertions(+), 51 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index df19b9ce4c082..0e00e7bdbbd6b 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -30,25 +30,25 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 
 |Symbol|Meaning|Presentation|Examples|
 |------|-------|------------|--------|
-|**G**|era|text|AD; Anno Domini; A|
+|**G**|era|text|AD; Anno Domini|
 |**y**|year|year|2020; 20|
-|**D**|day-of-year|number|189|
-|**M/L**|month-of-year|number/text|7; 07; Jul; July; J|
-|**d**|day-of-month|number|28|
+|**D**|day-of-year|number(3)|189|
+|**M/L**|month-of-year|month|7; 07; Jul; July|
+|**d**|day-of-month|number(3)|28|
 |**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
 |**Y**|week-based-year|year|1996; 96|
-|**w**|week-of-week-based-year|number|27|
-|**W**|week-of-month|number|4|
-|**E**|day-of-week|text|Tue; Tuesday; T|
-|**u**|localized day-of-week|number/text|2; 02; Tue; Tuesday; T|
-|**F**|week-of-month|number|3|
-|**a**|am-pm-of-day|text|PM|
-|**h**|clock-hour-of-am-pm (1-12)|number|12|
-|**K**|hour-of-am-pm (0-11)|number|0|
-|**k**|clock-hour-of-day (1-24)|number|0|
-|**H**|hour-of-day (0-23)|number|0|
-|**m**|minute-of-hour|number|30|
-|**s**|second-of-minute|number|55|
+|**w**|week-of-week-based-year|number(2)|27|
+|**W**|week-of-month|number(1)|4|
+|**E**|day-of-week|text|Tue; Tuesday|
+|**u**|localized day-of-week|number/text|2; 02; Tue; Tuesday|
+|**F**|week-of-month|number(1)|3|
+|**a**|am-pm-of-day|am-pm|PM|
+|**h**|clock-hour-of-am-pm (1-12)|number(2)|12|
+|**K**|hour-of-am-pm (0-11)|number(2)|0|
+|**k**|clock-hour-of-day (1-24)|number(2)|0|
+|**H**|hour-of-day (0-23)|number(2)|0|
+|**m**|minute-of-hour|number(2)|30|
+|**s**|second-of-minute|number(2)|55|
 |**S**|fraction-of-second|fraction|978|
 |**V**|time-zone ID|zone-id|America/Los_Angeles; Z; -08:30|
 |**z**|time-zone name|zone-name|Pacific Standard Time; PST|
@@ -63,9 +63,9 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 
 The count of pattern letters determines the format.
 
-- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short form. Exactly 4 pattern letters will use the full form. Exactly 5 pattern letters will use the narrow form. Six or more letters will fail.
+- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short form. Exactly 4 pattern letters will use the full form. Exactly 5 pattern letters will use the narrow form. 5 or more letters will fail.
 
-- Number: If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary. The following pattern letters have constraints on the count of letters. Only one letter 'F' can be specified. Up to two letters of 'd', 'H', 'h', 'K', 'k', 'm', and 's' can be specified. Up to three letters of 'D' can be specified.
+- Number(n): The n here represents the maximum count of letters this type of datetime pattern can be used. If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary.
 
 - Number/Text: If the count of pattern letters is 3 or greater, use the Text rules above. Otherwise use the Number rules above.
 
@@ -76,8 +76,7 @@ The count of pattern letters determines the format.
 
 - Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
 
-- Month: If the number of pattern letters is 3 or more, the month is interpreted as text; otherwise, it is interpreted as a number. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. The difference between the 'standard' and 'stand-alone' forms is trickier to describe as there is no difference in English. However, in other languages there is a difference in the word used when the text is used alone, as opposed to in a complete date. For example, the word used for a month when used alone in a date picker is different to the word used for month in association with a day and year in a date. In Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters (more than 5 letters is invalid):
-  - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
+- Month: If the number of pattern letters is 3 or more, the month is interpreted as text; otherwise, it is interpreted as a number. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. The difference between the 'standard' and 'stand-alone' forms is trickier to describe as there is no difference in English. However, in other languages there is a difference in the word used when the text is used alone, as opposed to in a complete date. For example, the word used for a month when used alone in a date picker is different to the word used for month in association with a day and year in a date. In Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters (more than 4 letters is invalid):  - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
     ```sql
     spark-sql> select date_format(date '1970-01-01', "M");
     1
@@ -119,13 +118,8 @@ The count of pattern letters determines the format.
     spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'LLLL', 'locale', 'RU'));
     январь
     ```
-  - `'LLLLL'` or `'MMMMM'`: Narrow textual representation of standard or stand-alone forms. Typically it is a single letter.
-    ```sql
-    spark-sql> select date_format(date '1970-07-01', "LLLLL");
-    J
-    spark-sql> select date_format(date '1970-01-01', "MMMMM");
-    J
-    ```
+
+- am-pm: This outputs the am-pm-of-day. Pattern letter count must be 1.
 
 - Zone ID(V): This outputs the display the time-zone ID. Pattern letter count must be 2.
 
@@ -147,5 +141,3 @@ More details for the text style:
 - Short Form: Short text, typically an abbreviation. For example, day-of-week Monday might output "Mon".
 
 - Full Form: Full text, typically the full description. For example, day-of-week Monday might output "Monday".
-
-- Narrow Form: Narrow text, typically a single letter. For example, day-of-week Monday might output "M".
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index 8e87a82769471..f2bb7db895ca2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -85,13 +85,13 @@ class UnivocityParser(
   // We preallocate it avoid unnecessary allocations.
   private val noRows = None
 
-  private val timestampFormatter = TimestampFormatter(
+  private lazy val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
     needVarLengthSecondFraction = true)
-  private val dateFormatter = DateFormatter(
+  private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 6e95f648b64f4..a0cf4462e5bb6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -801,6 +801,7 @@ abstract class ToTimestamp
         legacyFormat = SIMPLE_DATE_FORMAT,
         needVarLengthSecondFraction = true)
     } catch {
+      case e: SparkUpgradeException => throw e
       case NonFatal(_) => null
     }
 
@@ -982,6 +983,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
         legacyFormat = SIMPLE_DATE_FORMAT,
         needVarLengthSecondFraction = false)
     } catch {
+      case e: SparkUpgradeException => throw e
       case NonFatal(_) => null
     }
 
@@ -997,6 +999,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
           try {
             UTF8String.fromString(formatter.format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
+            case e: SparkUpgradeException => throw e
             case NonFatal(_) => null
           }
         }
@@ -1014,6 +1017,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
                 needVarLengthSecondFraction = false)
                 .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
+            case e: SparkUpgradeException => throw e
             case NonFatal(_) => null
           }
         }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index ef987931e928a..c4f6121723491 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -56,13 +56,13 @@ class JacksonParser(
 
   private val factory = options.buildJsonFactory()
 
-  private val timestampFormatter = TimestampFormatter(
+  private lazy val timestampFormatter = TimestampFormatter(
     options.timestampFormat,
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
     needVarLengthSecondFraction = true)
-  private val dateFormatter = DateFormatter(
+  private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 8261f57916fa2..c4cedf229d0ca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -33,6 +33,8 @@ sealed trait DateFormatter extends Serializable {
   def format(days: Int): String
   def format(date: Date): String
   def format(localDate: LocalDate): String
+
+  def validatePatternString(): Unit
 }
 
 class Iso8601DateFormatter(
@@ -70,6 +72,12 @@ class Iso8601DateFormatter(
   override def format(date: Date): String = {
     legacyFormatter.format(date)
   }
+
+  override def validatePatternString(): Unit = {
+    try {
+      formatter
+    } catch checkLegacyFormatter(pattern, legacyFormatter.validatePatternString)
+  }
 }
 
 trait LegacyDateFormatter extends DateFormatter {
@@ -93,6 +101,8 @@ class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDat
   private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
   override def parseToDate(s: String): Date = fdf.parse(s)
   override def format(d: Date): String = fdf.format(d)
+
+  override def validatePatternString(): Unit = fdf
 }
 
 class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
@@ -100,6 +110,8 @@ class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyD
   private lazy val sdf = new SimpleDateFormat(pattern, locale)
   override def parseToDate(s: String): Date = sdf.parse(s)
   override def format(d: Date): String = sdf.format(d)
+
+  override def validatePatternString(): Unit = sdf
 }
 
 object DateFormatter {
@@ -118,7 +130,9 @@ object DateFormatter {
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat)
+      val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat)
+      df.validatePatternString()
+      df
     }
   }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 35f95dbffca6e..8289568c449ad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -117,6 +117,34 @@ trait DateTimeFormatterHelper {
         s"set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore the behavior " +
         s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
   }
+
+  /**
+   * When the new DateTimeFormatter failed to initialize because of invalid datetime pattern, it
+   * will throw IllegalArgumentException. If the pattern can be recognized by the legacy formatter
+   * it will raise SparkUpgradeException to tell users to restore the previous behavior via LEGACY
+   * policy or follow our guide to correct their pattern. Otherwise, the original
+   * IllegalArgumentException will be thrown.
+   *
+   * @param pattern the date time pattern
+   * @param tryLegacyFormatter a func to capture exception, identically which forces a legacy
+   *                           datetime formatter to be initialized
+   */
+
+  protected def checkLegacyFormatter(
+      pattern: String,
+      tryLegacyFormatter: => Unit): PartialFunction[Throwable, DateTimeFormatter] = {
+    case e: IllegalArgumentException =>
+      try {
+        tryLegacyFormatter
+      } catch {
+        case _: Throwable => throw e
+      }
+      throw new SparkUpgradeException("3.0", s"Fail to recognize '$pattern' pattern in the" +
+        s" DateTimeFormatter. 1) You can set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY" +
+        s" to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern" +
+        s" with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html",
+        e)
+  }
 }
 
 private object DateTimeFormatterHelper {
@@ -190,7 +218,8 @@ private object DateTimeFormatterHelper {
   }
 
   final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
-
+  final val unsupportedNarrowTextStyle =
+    Set("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "QQQQQ", "qqqqq", "uuuuu")
   /**
    * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
    * parsing/formatting datetime values. The pattern string is incompatible with the one defined
@@ -211,6 +240,9 @@ private object DateTimeFormatterHelper {
           for (c <- patternPart if unsupportedLetters.contains(c)) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
           }
+          for (style <- unsupportedNarrowTextStyle if patternPart.contains(style)) {
+            throw new IllegalArgumentException(s"Too many pattern letters: ${style.head}")
+          }
           // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
           // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
           // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index d7d63687ff364..32b4dcda5dea9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -54,6 +54,7 @@ sealed trait TimestampFormatter extends Serializable {
   def format(us: Long): String
   def format(ts: Timestamp): String
   def format(instant: Instant): String
+  def validatePatternString(): Unit
 }
 
 class Iso8601TimestampFormatter(
@@ -99,6 +100,12 @@ class Iso8601TimestampFormatter(
   override def format(ts: Timestamp): String = {
     legacyFormatter.format(ts)
   }
+
+  override def validatePatternString(): Unit = {
+    try {
+      formatter
+    } catch checkLegacyFormatter(pattern, legacyFormatter.validatePatternString)
+  }
 }
 
 /**
@@ -202,6 +209,8 @@ class LegacyFastTimestampFormatter(
   override def format(instant: Instant): String = {
     format(instantToMicros(instant))
   }
+
+  override def validatePatternString(): Unit = fastDateFormat
 }
 
 class LegacySimpleTimestampFormatter(
@@ -231,6 +240,8 @@ class LegacySimpleTimestampFormatter(
   override def format(instant: Instant): String = {
     format(instantToMicros(instant))
   }
+
+  override def validatePatternString(): Unit = sdf
 }
 
 object LegacyDateFormats extends Enumeration {
@@ -255,8 +266,10 @@ object TimestampFormatter {
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      new Iso8601TimestampFormatter(
+      val tf = new Iso8601TimestampFormatter(
         pattern, zoneId, locale, legacyFormat, needVarLengthSecondFraction)
+      tf.validatePatternString()
+      tf
     }
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index dd5a532834160..5b5c85f96c06f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -267,7 +267,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
 
         // Test escaping of format
         GenerateUnsafeProjection.generate(
-          DateFormatClass(Literal(ts), Literal("\"quote"), JST_OPT) :: Nil)
+          DateFormatClass(Literal(ts), Literal("\""), JST_OPT) :: Nil)
 
         // SPARK-28072 The codegen path should work
         checkEvaluation(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index 817e503584324..34a1ad2e40e37 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -40,6 +40,17 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
       val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
       assert(e.getMessage === s"Illegal pattern character: $l")
     }
+
+    unsupportedNarrowTextStyle.foreach { style =>
+      val e1 = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"yyyy-MM-dd $style")
+      }
+      assert(e1.getMessage === s"Too many pattern letters: ${style.head}")
+      val e2 = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"yyyy-MM-dd $style${style.head}")
+      }
+      assert(e2.getMessage === s"Too many pattern letters: ${style.head}")
+    }
     assert(convertIncompatiblePattern("yyyy-MM-dd uuuu") === "uuuu-MM-dd eeee")
     assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
     assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index dccb3defe3728..4324d3cff63d7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -396,4 +396,15 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     val micros = formatter.parse("2009 11")
     assert(micros === date(2009, 1, 1, 11))
   }
+
+  test("explicitly forbidden datetime patterns") {
+    // not support by the legacy one too
+    Seq("QQQQQ", "qqqqq", "A", "c", "e", "n", "N", "p").foreach { pattern =>
+      intercept[IllegalArgumentException](TimestampFormatter(pattern, UTC).format(0))
+    }
+    // supported by the legacy one, then we will suggest users with SparkUpgradeException
+    Seq("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "aa", "aaa").foreach { pattern =>
+      intercept[SparkUpgradeException](TimestampFormatter(pattern, UTC).format(0))
+    }
+  }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
new file mode 100644
index 0000000000000..e573f8a6b4be9
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime.sql
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 9be857ef767df..99b7cf98f63bc 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -130,3 +130,23 @@ select to_date("16", "dd");
 select to_date("02-29", "MM-dd");
 select to_timestamp("2019 40", "yyyy mm");
 select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss");
+
+-- Unsupported narrow text style
+select date_format(date '2020-05-23', 'GGGGG');
+select date_format(date '2020-05-23', 'MMMMM');
+select date_format(date '2020-05-23', 'LLLLL');
+select date_format(timestamp '2020-05-23', 'EEEEE');
+select date_format(timestamp '2020-05-23', 'uuuuu');
+select date_format('2020-05-23', 'QQQQQ');
+select date_format('2020-05-23', 'qqqqq');
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG');
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE');
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE');
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE');
+select from_unixtime(12345, 'MMMMM');
+select from_unixtime(54321, 'QQQQQ');
+select from_unixtime(23456, 'aaaaa');
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index 89596c737b3e9..ff30bb04ee5c9 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 91
+-- Number of queries: 109
 
 
 -- !query
@@ -778,3 +778,164 @@ select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
 struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
 -- !query output
 2019-01-01 10:10:10
+
+
+-- !query
+select date_format(date '2020-05-23', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'uuuuu')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'uuuuu' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2020-05-23', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2020-05-23', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyy-MM-dd GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(12345, 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(54321, 'QQQQQ')
+-- !query schema
+struct<from_unixtime(CAST(54321 AS BIGINT), QQQQQ):string>
+-- !query output
+NULL
+
+
+-- !query
+select from_unixtime(23456, 'aaaaa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aaaaa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
new file mode 100644
index 0000000000000..30743f882cff4
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -0,0 +1,898 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 109
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp()
+-- !query schema
+struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean>
+-- !query output
+true	true
+
+
+-- !query
+select to_date(null), to_date('2016-12-31'), to_date('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_date(NULL):date,to_date('2016-12-31'):date,to_date('2016-12-31', 'yyyy-MM-dd'):date>
+-- !query output
+NULL	2016-12-31	2016-12-31
+
+
+-- !query
+select to_timestamp(null), to_timestamp('2016-12-31 00:12:00'), to_timestamp('2016-12-31', 'yyyy-MM-dd')
+-- !query schema
+struct<to_timestamp(NULL):timestamp,to_timestamp('2016-12-31 00:12:00'):timestamp,to_timestamp('2016-12-31', 'yyyy-MM-dd'):timestamp>
+-- !query output
+NULL	2016-12-31 00:12:00	2016-12-31 00:00:00
+
+
+-- !query
+select dayofweek('2007-02-03'), dayofweek('2009-07-30'), dayofweek('2017-05-27'), dayofweek(null), dayofweek('1582-10-15 13:10:15')
+-- !query schema
+struct<dayofweek(CAST(2007-02-03 AS DATE)):int,dayofweek(CAST(2009-07-30 AS DATE)):int,dayofweek(CAST(2017-05-27 AS DATE)):int,dayofweek(CAST(NULL AS DATE)):int,dayofweek(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+7	5	7	NULL	6
+
+
+-- !query
+create temporary view ttf1 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf1(current_date, current_timestamp)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date, current_timestamp from ttf1
+-- !query schema
+struct<current_date:int,current_timestamp:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+create temporary view ttf2 as select * from values
+  (1, 2),
+  (2, 3)
+  as ttf2(a, b)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select current_date = current_date(), current_timestamp = current_timestamp(), a, b from ttf2
+-- !query schema
+struct<(current_date() = current_date()):boolean,(current_timestamp() = current_timestamp()):boolean,a:int,b:int>
+-- !query output
+true	true	1	2
+true	true	2	3
+
+
+-- !query
+select a, b from ttf2 order by a, current_date
+-- !query schema
+struct<a:int,b:int>
+-- !query output
+1	2
+2	3
+
+
+-- !query
+select weekday('2007-02-03'), weekday('2009-07-30'), weekday('2017-05-27'), weekday(null), weekday('1582-10-15 13:10:15')
+-- !query schema
+struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):int,weekday(CAST(2017-05-27 AS DATE)):int,weekday(CAST(NULL AS DATE)):int,weekday(CAST(1582-10-15 13:10:15 AS DATE)):int>
+-- !query output
+5	3	5	NULL	4
+
+
+-- !query
+select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
+-- !query schema
+struct<year(CAST(1500-01-01 AS DATE)):int,month(CAST(1500-01-01 AS DATE)):int,dayofyear(CAST(1500-01-01 AS DATE)):int>
+-- !query output
+1500	1	1
+
+
+-- !query
+select date '2019-01-01\t'
+-- !query schema
+struct<DATE '2019-01-01':date>
+-- !query output
+2019-01-01
+
+
+-- !query
+select timestamp '2019-01-01\t'
+-- !query schema
+struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
+-- !query output
+2019-01-01 00:00:00
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' + interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' + INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-13 11:11:11
+
+
+-- !query
+select timestamp'2011-11-11 11:11:11' - interval '2' day
+-- !query schema
+struct<CAST(TIMESTAMP '2011-11-11 11:11:11' - INTERVAL '2 days' AS TIMESTAMP):timestamp>
+-- !query output
+2011-11-09 11:11:11
+
+
+-- !query
+select date'2011-11-11 11:11:11' + interval '2' second
+-- !query schema
+struct<DATE '2011-11-11' + INTERVAL '2 seconds':date>
+-- !query output
+2011-11-11
+
+
+-- !query
+select date'2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<DATE '2011-11-11' + (- INTERVAL '2 seconds'):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select '2011-11-11' - interval '2' day
+-- !query schema
+struct<CAST(CAST(2011-11-11 AS TIMESTAMP) - INTERVAL '2 days' AS STRING):string>
+-- !query output
+2011-11-09 00:00:00
+
+
+-- !query
+select '2011-11-11 11:11:11' - interval '2' second
+-- !query schema
+struct<CAST(CAST(2011-11-11 11:11:11 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+2011-11-11 11:11:09
+
+
+-- !query
+select '1' - interval '2' second
+-- !query schema
+struct<CAST(CAST(1 AS TIMESTAMP) - INTERVAL '2 seconds' AS STRING):string>
+-- !query output
+NULL
+
+
+-- !query
+select 1 - interval '2' second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve '1 - INTERVAL '2 seconds'' due to data type mismatch: argument 1 requires timestamp type, however, '1' is of int type.; line 1 pos 7
+
+
+-- !query
+select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(DATE '2020-01-01' AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+2078 hours 48 minutes 47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
+-- !query output
+-2078 hours -48 minutes -47.654322 seconds
+
+
+-- !query
+select timestamp'2019-10-06 10:11:12.345678' - null
+-- !query schema
+struct<subtracttimestamps(TIMESTAMP '2019-10-06 10:11:12.345678', CAST(NULL AS TIMESTAMP)):interval>
+-- !query output
+NULL
+
+
+-- !query
+select null - timestamp'2019-10-06 10:11:12.345678'
+-- !query schema
+struct<subtracttimestamps(CAST(NULL AS TIMESTAMP), TIMESTAMP '2019-10-06 10:11:12.345678'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date_add('2011-11-11', 1Y)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1S)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', 1L)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1L)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1L' is of bigint type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1.0)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 1.0BD)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '1.0BD' is of decimal(2,1) type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', 1E1)
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date_add('2011-11-11', '1')
+-- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_add' function needs to be an integer.;
+
+
+-- !query
+select date_add(date'2011-11-11', 1)
+-- !query schema
+struct<date_add(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_add(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_sub(date'2011-11-11', 1)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+The second argument of 'date_sub' function needs to be an integer.;
+
+
+-- !query
+select date_sub(timestamp'2011-11-11', 1)
+-- !query schema
+struct<date_sub(CAST(TIMESTAMP '2011-11-11 00:00:00' AS DATE), 1):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(null, 1)
+-- !query schema
+struct<date_sub(CAST(NULL AS DATE), 1):date>
+-- !query output
+NULL
+
+
+-- !query
+select date_sub(date'2011-11-11', null)
+-- !query schema
+struct<date_sub(DATE '2011-11-11', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date'2011-11-11' + 1E1
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', 10.0D)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, '10.0D' is of double type.; line 1 pos 7
+
+
+-- !query
+select date'2011-11-11' + '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(DATE '2011-11-11', CAST('1' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('1' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select null + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' + 7Y
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select 7S + date '2001-09-28'
+-- !query schema
+struct<date_add(DATE '2001-09-28', 7):date>
+-- !query output
+2001-10-05
+
+
+-- !query
+select date '2001-10-01' - 7
+-- !query schema
+struct<date_sub(DATE '2001-10-01', 7):date>
+-- !query output
+2001-09-24
+
+
+-- !query
+select date '2001-10-01' - '7'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(DATE '2001-10-01', CAST('7' AS DOUBLE))' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'CAST('7' AS DOUBLE)' is of double type.; line 1 pos 7
+
+
+-- !query
+select date '2001-09-28' + null
+-- !query schema
+struct<date_add(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-09-28' - null
+-- !query schema
+struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
+-- !query output
+NULL
+
+
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select null - date '2019-10-06'
+-- !query schema
+struct<subtractdates(CAST(NULL AS DATE), DATE '2019-10-06'):interval>
+-- !query output
+NULL
+
+
+-- !query
+select date '2001-10-01' - date '2001-09-28'
+-- !query schema
+struct<subtractdates(DATE '2001-10-01', DATE '2001-09-28'):interval>
+-- !query output
+3 days
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.0', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123UTC', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12345CST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123456PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234567PST', 'yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('123456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]')
+-- !query schema
+struct<to_timestamp('223456 2019-10-06 10:11:12.123456PST', 'SSSSSS yyyy-MM-dd HH:mm:ss.SSSSSS[zzz]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.1234', 'yyyy-MM-dd HH:mm:ss.[SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.123', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12', 'yyyy-MM-dd HH:mm:ss[.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11:12.12', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]')
+-- !query schema
+struct<to_timestamp('2019-10-06 10:11', 'yyyy-MM-dd HH:mm[:ss.SSSSSS]'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019-10-06S10:11:12.12345", "yyyy-MM-dd'S'HH:mm:ss.SSSSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06S10:11:12.12345', 'yyyy-MM-dd\'S\'HH:mm:ss.SSSSSS'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.12342019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.12342019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyyyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyyyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1232019-10-06S10:11", "ss.SSSSyy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1232019-10-06S10:11', 'ss.SSSSyy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm")
+-- !query schema
+struct<to_timestamp('12.1234019-10-06S10:11', 'ss.SSSSy-MM-dd\'S\'HH:mm'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'")
+-- !query schema
+struct<to_timestamp('2019-10-06S', 'yyyy-MM-dd\'S\''):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd")
+-- !query schema
+struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
+-- !query output
+2019-10-06 00:00:00
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'e'
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'c'
+
+
+-- !query
+select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
+-- !query schema
+struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
+-- !query output
+2019-10-06 0007
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'12', 'yyyy-MM-dd\'T\'HH:mm:ss\'\'SSSS'):timestamp>
+-- !query output
+2019-10-06 10:11:12.012
+
+
+-- !query
+select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''")
+-- !query schema
+struct<to_timestamp('2019-10-06T10:11:12\'', 'yyyy-MM-dd\'T\'HH:mm:ss\'\''):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('\'2019-10-06T10:11:12', '\'\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("P2019-10-06T10:11:12", "'P'yyyy-MM-dd'T'HH:mm:ss")
+-- !query schema
+struct<to_timestamp('P2019-10-06T10:11:12', '\'P\'yyyy-MM-dd\'T\'HH:mm:ss'):timestamp>
+-- !query output
+2019-10-06 10:11:12
+
+
+-- !query
+select to_timestamp("16", "dd")
+-- !query schema
+struct<to_timestamp('16', 'dd'):timestamp>
+-- !query output
+1970-01-16 00:00:00
+
+
+-- !query
+select to_timestamp("02-29", "MM-dd")
+-- !query schema
+struct<to_timestamp('02-29', 'MM-dd'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_date("16", "dd")
+-- !query schema
+struct<to_date('16', 'dd'):date>
+-- !query output
+1970-01-16
+
+
+-- !query
+select to_date("02-29", "MM-dd")
+-- !query schema
+struct<to_date('02-29', 'MM-dd'):date>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp("2019 40", "yyyy mm")
+-- !query schema
+struct<to_timestamp('2019 40', 'yyyy mm'):timestamp>
+-- !query output
+2019-01-01 00:40:00
+
+
+-- !query
+select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
+-- !query schema
+struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
+-- !query output
+2019-01-01 10:10:10
+
+
+-- !query
+select date_format(date '2020-05-23', 'GGGGG')
+-- !query schema
+struct<date_format(CAST(DATE '2020-05-23' AS TIMESTAMP), GGGGG):string>
+-- !query output
+AD
+
+
+-- !query
+select date_format(date '2020-05-23', 'MMMMM')
+-- !query schema
+struct<date_format(CAST(DATE '2020-05-23' AS TIMESTAMP), MMMMM):string>
+-- !query output
+May
+
+
+-- !query
+select date_format(date '2020-05-23', 'LLLLL')
+-- !query schema
+struct<date_format(CAST(DATE '2020-05-23' AS TIMESTAMP), LLLLL):string>
+-- !query output
+May
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'EEEEE')
+-- !query schema
+struct<date_format(TIMESTAMP '2020-05-23 00:00:00', EEEEE):string>
+-- !query output
+Saturday
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'uuuuu')
+-- !query schema
+struct<date_format(TIMESTAMP '2020-05-23 00:00:00', uuuuu):string>
+-- !query output
+00006
+
+
+-- !query
+select date_format('2020-05-23', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'Q'
+
+
+-- !query
+select date_format('2020-05-23', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'q'
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE'):timestamp>
+-- !query output
+2020-05-22 00:00:00
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE'):timestamp>
+-- !query output
+2020-05-22 00:00:00
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<unix_timestamp(22 05 2020 Friday, dd MM yyyy EEEEE):bigint>
+-- !query output
+1590130800
+
+
+-- !query
+select from_unixtime(12345, 'MMMMM')
+-- !query schema
+struct<from_unixtime(CAST(12345 AS BIGINT), MMMMM):string>
+-- !query output
+December
+
+
+-- !query
+select from_unixtime(54321, 'QQQQQ')
+-- !query schema
+struct<from_unixtime(CAST(54321 AS BIGINT), QQQQQ):string>
+-- !query output
+NULL
+
+
+-- !query
+select from_unixtime(23456, 'aaaaa')
+-- !query schema
+struct<from_unixtime(CAST(23456 AS BIGINT), aaaaa):string>
+-- !query output
+PM
+
+
+-- !query
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_json({"time":"26/October/2015"}):struct<time:timestamp>>
+-- !query output
+{"time":2015-10-26 00:00:00}
+
+
+-- !query
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_json({"date":"26/October/2015"}):struct<date:date>>
+-- !query output
+{"date":2015-10-26}
+
+
+-- !query
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_csv(26/October/2015):struct<time:timestamp>>
+-- !query output
+{"time":2015-10-26 00:00:00}
+
+
+-- !query
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<from_csv(26/October/2015):struct<date:date>>
+-- !query output
+{"date":2015-10-26}
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index b26eb2fdd018d..dc466d12a7e3f 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 91
+-- Number of queries: 109
 
 
 -- !query
@@ -750,3 +750,164 @@ select to_timestamp("2019 10:10:10", "yyyy hh:mm:ss")
 struct<to_timestamp('2019 10:10:10', 'yyyy hh:mm:ss'):timestamp>
 -- !query output
 2019-01-01 10:10:10
+
+
+-- !query
+select date_format(date '2020-05-23', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(date '2020-05-23', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2020-05-23', 'uuuuu')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'uuuuu' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2020-05-23', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2020-05-23', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select to_timestamp('2019-10-06 A', 'yyyy-MM-dd GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyy-MM-dd GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select to_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select unix_timestamp('22 05 2020 Friday', 'dd MM yyyy EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd MM yyyy EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(12345, 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(54321, 'QQQQQ')
+-- !query schema
+struct<from_unixtime(CAST(54321 AS BIGINT), QQQQQ):string>
+-- !query output
+NULL
+
+
+-- !query
+select from_unixtime(23456, 'aaaaa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aaaaa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
index 5c56eff85b264..b4c052585ef7c 100644
--- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/stringCastAndExpressions.sql.out
@@ -136,9 +136,10 @@ NULL
 -- !query
 select to_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<to_timestamp('2018-01-01', t.`a`):timestamp>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
@@ -152,9 +153,10 @@ NULL
 -- !query
 select to_unix_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<to_unix_timestamp(2018-01-01, a):bigint>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
@@ -168,9 +170,10 @@ NULL
 -- !query
 select unix_timestamp('2018-01-01', a) from t
 -- !query schema
-struct<unix_timestamp(2018-01-01, a):bigint>
+struct<>
 -- !query output
-NULL
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
index 5946536925ff3..247f8153dfde5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala
@@ -23,7 +23,7 @@ import java.time.{Instant, LocalDateTime}
 import java.util.{Locale, TimeZone}
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkException, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{CEST, LA}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.functions._
@@ -450,9 +450,9 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
     checkAnswer(
       df.select(to_date(col("s"), "yyyy-hh-MM")),
       Seq(Row(null), Row(null), Row(null)))
-    checkAnswer(
-      df.select(to_date(col("s"), "yyyy-dd-aa")),
-      Seq(Row(null), Row(null), Row(null)))
+    val e = intercept[SparkUpgradeException](df.select(to_date(col("s"), "yyyy-dd-aa")).collect())
+    assert(e.getCause.isInstanceOf[IllegalArgumentException])
+    assert(e.getMessage.contains("You may get a different result due to the upgrading of Spark"))
 
     // february
     val x1 = "2016-02-29"
@@ -618,8 +618,16 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
           Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null)))
 
         // invalid format
-        checkAnswer(df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')"), Seq(
-          Row(null), Row(null), Row(null), Row(null)))
+        val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')")
+        if (legacyParserPolicy == "legacy") {
+          checkAnswer(invalid,
+            Seq(Row(null), Row(null), Row(null), Row(null)))
+        } else {
+          val e = intercept[SparkUpgradeException](invalid.collect())
+          assert(e.getCause.isInstanceOf[IllegalArgumentException])
+          assert(
+            e.getMessage.contains("You may get a different result due to the upgrading of Spark"))
+        }
 
         // february
         val y1 = "2016-02-29"

From b165b81d705a3b38502288ce76d906af0a293424 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 26 May 2020 12:13:28 +0000
Subject: [PATCH 0738/1280] [SPARK-31820][SQL][TESTS] Fix flaky
 JavaBeanDeserializationSuite

### What changes were proposed in this pull request?
Modified formatting of expected timestamp strings in the test `JavaBeanDeserializationSuite`.`testSpark22000` to correctly format timestamps with **zero** seconds fraction. Current implementation outputs `.0` but must be empty string. From SPARK-31820 failure:
- should be `2020-05-25 12:39:17`
- but incorrect expected string is `2020-05-25 12:39:17.0`

### Why are the changes needed?
To make `JavaBeanDeserializationSuite` stable, and avoid test failures like https://github.com/apache/spark/pull/28630#issuecomment-633695723

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
I changed https://github.com/apache/spark/blob/7dff3b125de23a4d6ce834217ee08973b259414c/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java#L207 to
```java
new java.sql.Timestamp((System.currentTimeMillis() / 1000) * 1000),
```
to force zero seconds fraction.

Closes #28639 from MaxGekk/fix-JavaBeanDeserializationSuite.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 87d34e6b969e9ceba207b996506156bd36f3dd64)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/JavaBeanDeserializationSuite.java   | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
index 5603cb988b8e7..af0a22b036030 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaBeanDeserializationSuite.java
@@ -18,6 +18,8 @@
 package test.org.apache.spark.sql;
 
 import java.io.Serializable;
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.*;
@@ -210,6 +212,17 @@ private static Row createRecordSpark22000Row(Long index) {
     return new GenericRow(values);
   }
 
+  private static String timestampToString(Timestamp ts) {
+    String timestampString = String.valueOf(ts);
+    String formatted = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(ts);
+
+    if (timestampString.length() > 19 && !timestampString.substring(19).equals(".0")) {
+      return formatted + timestampString.substring(19);
+    } else {
+      return formatted;
+    }
+  }
+
   private static RecordSpark22000 createRecordSpark22000(Row recordRow) {
     RecordSpark22000 record = new RecordSpark22000();
     record.setShortField(String.valueOf(recordRow.getShort(0)));
@@ -219,7 +232,7 @@ private static RecordSpark22000 createRecordSpark22000(Row recordRow) {
     record.setDoubleField(String.valueOf(recordRow.getDouble(4)));
     record.setStringField(recordRow.getString(5));
     record.setBooleanField(String.valueOf(recordRow.getBoolean(6)));
-    record.setTimestampField(String.valueOf(recordRow.getTimestamp(7)));
+    record.setTimestampField(timestampToString(recordRow.getTimestamp(7)));
     // This would figure out that null value will not become "null".
     record.setNullIntField(null);
     return record;

From c03a87599c78da38fcc7d95d422bf0bf639c4171 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 27 May 2020 10:16:08 +0900
Subject: [PATCH 0739/1280] Revert "[SPARK-31788][CORE][PYTHON] Fix UnionRDD of
 PairRDDs"

This reverts commit be18718380fc501fe2a780debd089a1df91c1699.
---
 python/pyspark/context.py        | 12 ++----------
 python/pyspark/tests/test_rdd.py |  9 ---------
 2 files changed, 2 insertions(+), 19 deletions(-)

diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 3199aa77565fd..d5f1506e4a555 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -25,7 +25,6 @@
 from tempfile import NamedTemporaryFile
 
 from py4j.protocol import Py4JError
-from py4j.java_gateway import is_instance_of
 
 from pyspark import accumulators
 from pyspark.accumulators import Accumulator
@@ -865,17 +864,10 @@ def union(self, rdds):
         first_jrdd_deserializer = rdds[0]._jrdd_deserializer
         if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds):
             rdds = [x._reserialize() for x in rdds]
-        gw = SparkContext._gateway
         cls = SparkContext._jvm.org.apache.spark.api.java.JavaRDD
-        is_jrdd = is_instance_of(gw, rdds[0]._jrdd, cls)
-        jrdds = gw.new_array(cls, len(rdds))
+        jrdds = SparkContext._gateway.new_array(cls, len(rdds))
         for i in range(0, len(rdds)):
-            if is_jrdd:
-                jrdds[i] = rdds[i]._jrdd
-            else:
-                # zip could return JavaPairRDD hence we ensure `_jrdd`
-                # to be `JavaRDD` by wrapping it in a `map`
-                jrdds[i] = rdds[i].map(lambda x: x)._jrdd
+            jrdds[i] = rdds[i]._jrdd
         return RDD(self._jsc.union(jrdds), self, rdds[0]._jrdd_deserializer)
 
     def broadcast(self, value):
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index 0f1ee5b706611..e2d910ccd15c3 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -166,15 +166,6 @@ def test_zip_chaining(self):
             set([(x, (x, x)) for x in 'abc'])
         )
 
-    def test_union_pair_rdd(self):
-        # Regression test for SPARK-31788
-        rdd = self.sc.parallelize([1, 2])
-        pair_rdd = rdd.zip(rdd)
-        self.assertEqual(
-            self.sc.union([pair_rdd, pair_rdd]).collect(),
-            [((1, 1), (2, 2)), ((1, 1), (2, 2))]
-        )
-
     def test_deleting_input_files(self):
         # Regression test for SPARK-1025
         tempFile = tempfile.NamedTemporaryFile(delete=False)

From ec9bd06e5de517b5a91691249173e845996b07e9 Mon Sep 17 00:00:00 2001
From: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Date: Tue, 26 May 2020 18:21:22 -0700
Subject: [PATCH 0740/1280] [SPARK-31821][BUILD] Remove mssql-jdbc dependencies
 from Hadoop 3.2 profile

### What changes were proposed in this pull request?
There is an unnecessary dependency for `mssql-jdbc`. In this PR I've removed it.

### Why are the changes needed?
Unnecessary dependency.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?

Pass the Jenkins with the following configuration.
- [x] Pass the dependency test.
- [x] SBT with Hadoop-3.2 (https://github.com/apache/spark/pull/28640#issuecomment-634192512)
- [ ] Maven with Hadoop-3.2

Closes #28640 from gaborgsomogyi/SPARK-31821.

Authored-by: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 8f1d77488c07a50b90a113f7faacd3d96f25646b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 1 -
 pom.xml                                 | 4 ++++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 3c3ce2dcdd6d4..b5a10b5dba378 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -183,7 +183,6 @@ metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar
 metrics-json/4.1.1//metrics-json-4.1.1.jar
 metrics-jvm/4.1.1//metrics-jvm-4.1.1.jar
 minlog/1.3.0//minlog-1.3.0.jar
-mssql-jdbc/6.2.1.jre7//mssql-jdbc-6.2.1.jre7.jar
 netty-all/4.1.47.Final//netty-all-4.1.47.Final.jar
 nimbus-jose-jwt/4.41.1//nimbus-jose-jwt-4.41.1.jar
 objenesis/2.5.1//objenesis-2.5.1.jar
diff --git a/pom.xml b/pom.xml
index 37548de1813de..0231f8fd07913 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1345,6 +1345,10 @@
             <groupId>com.zaxxer</groupId>
             <artifactId>HikariCP-java7</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.microsoft.sqlserver</groupId>
+            <artifactId>mssql-jdbc</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>

From 59f9a88e61b680b34f7b4155acb4548a2bd8adbe Mon Sep 17 00:00:00 2001
From: beliefer <beliefer@163.com>
Date: Wed, 27 May 2020 15:05:06 +0900
Subject: [PATCH 0741/1280] [SPARK-31393][SQL][FOLLOW-UP] Show the correct
 alias in schema for expression

Some alias of expression can not display correctly in schema. This PR will fix them.
- `ln`
- `rint`
- `lcase`
- `position`

Improve the implement of some expression.

 'Yes'. This PR will let user see the correct alias in schema.

Jenkins test.

Closes #28551 from beliefer/show-correct-alias-in-schema.

Lead-authored-by: beliefer <beliefer@163.com>
Co-authored-by: gengjiaan <gengjiaan@360.cn>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8f2b6f3a0b9d4297cb1f62e682958239fd6f9dbd)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/analysis/FunctionRegistry.scala   |  4 ++--
 .../sql/catalyst/expressions/mathExpressions.scala |  5 ++++-
 .../catalyst/expressions/stringExpressions.scala   |  6 +++++-
 .../sql-functions/sql-expression-schema.md         | 10 +++++-----
 .../results/ansi/string-functions.sql.out          |  2 +-
 .../sql-tests/results/postgreSQL/numeric.sql.out   | 14 +++++++-------
 .../sql-tests/results/string-functions.sql.out     |  2 +-
 7 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index c2c0df5f6432c..8608558a4cfe9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -339,7 +339,7 @@ object FunctionRegistry {
     expression[GetJsonObject]("get_json_object"),
     expression[InitCap]("initcap"),
     expression[StringInstr]("instr"),
-    expression[Lower]("lcase"),
+    expression[Lower]("lcase", true),
     expression[Length]("length"),
     expression[Levenshtein]("levenshtein"),
     expression[Like]("like"),
@@ -350,7 +350,7 @@ object FunctionRegistry {
     expression[StringTrimLeft]("ltrim"),
     expression[JsonTuple]("json_tuple"),
     expression[ParseUrl]("parse_url"),
-    expression[StringLocate]("position"),
+    expression[StringLocate]("position", true),
     expression[FormatString]("printf", true),
     expression[RegExpExtract]("regexp_extract"),
     expression[RegExpReplace]("regexp_replace"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 66e6334e3a450..8c6fbc0fc8e44 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -491,7 +491,9 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
       > SELECT _FUNC_(1);
        0.0
   """)
-case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG")
+case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG") {
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("ln")
+}
 
 @ExpressionDescription(
   usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.",
@@ -546,6 +548,7 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p,
 // scalastyle:on line.size.limit
 case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") {
   override def funcName: String = "rint"
+  override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("rint")
 }
 
 @ExpressionDescription(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 82b1e5f0998b0..2cd92d7826c64 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -365,6 +365,9 @@ case class Lower(child: Expression) extends UnaryExpression with String2StringEx
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     defineCodeGen(ctx, ev, c => s"($c).toLowerCase()")
   }
+
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("lower")
 }
 
 /** A base trait for functions that compare two strings, returning a boolean. */
@@ -1182,7 +1185,8 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
      """)
   }
 
-  override def prettyName: String = "locate"
+  override def prettyName: String =
+    getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("locate")
 }
 
 /**
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index ed972ed1eacff..8040d0dd637a1 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -155,12 +155,12 @@
 | org.apache.spark.sql.catalyst.expressions.LessThanOrEqual | <= | SELECT 2 <= 2 | struct<(2 <= 2):boolean> |
 | org.apache.spark.sql.catalyst.expressions.Levenshtein | levenshtein | SELECT levenshtein('kitten', 'sitting') | struct<levenshtein(kitten, sitting):int> |
 | org.apache.spark.sql.catalyst.expressions.Like | like | SELECT like('Spark', '_park') | struct<Spark LIKE _park:boolean> |
-| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<LOG(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<ln(CAST(1 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Log10 | log10 | SELECT log10(10) | struct<LOG10(CAST(10 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Log1p | log1p | SELECT log1p(0) | struct<LOG1P(CAST(0 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Log2 | log2 | SELECT log2(2) | struct<LOG2(CAST(2 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Logarithm | log | SELECT log(10, 100) | struct<LOG(CAST(10 AS DOUBLE), CAST(100 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lower(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lcase(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.Lower | lower | SELECT lower('SparkSql') | struct<lower(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.MakeDate | make_date | SELECT make_date(2013, 7, 15) | struct<make_date(2013, 7, 15):date> |
 | org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(8,6))):interval> |
@@ -211,7 +211,7 @@
 | org.apache.spark.sql.catalyst.expressions.Remainder | mod | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
 | org.apache.spark.sql.catalyst.expressions.Reverse | reverse | SELECT reverse('Spark SQL') | struct<reverse(Spark SQL):string> |
 | org.apache.spark.sql.catalyst.expressions.Right | right | SELECT right('Spark SQL', 3) | struct<right('Spark SQL', 3):string> |
-| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<ROUND(CAST(12.3456 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<rint(CAST(12.3456 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.Rollup | rollup | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY rollup(name, age) | struct<name:string,age:int,count(1):bigint> |
 | org.apache.spark.sql.catalyst.expressions.Round | round | SELECT round(2.5, 0) | struct<round(2.5, 0):decimal(2,0)> |
 | org.apache.spark.sql.catalyst.expressions.RowNumber | row_number | N/A | N/A |
@@ -242,7 +242,7 @@
 | org.apache.spark.sql.catalyst.expressions.Stack | stack | SELECT stack(2, 1, 2, 3) | struct<col0:int,col1:int> |
 | org.apache.spark.sql.catalyst.expressions.StringInstr | instr | SELECT instr('SparkSQL', 'SQL') | struct<instr(SparkSQL, SQL):int> |
 | org.apache.spark.sql.catalyst.expressions.StringLPad | lpad | SELECT lpad('hi', 5, '??') | struct<lpad(hi, 5, ??):string> |
-| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<position(bar, foobarbar, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.StringLocate | locate | SELECT locate('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
 | org.apache.spark.sql.catalyst.expressions.StringRPad | rpad | SELECT rpad('hi', 5, '??') | struct<rpad(hi, 5, ??):string> |
 | org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
@@ -333,4 +333,4 @@
 | org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') | struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>, a/b/text()):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathLong | xpath_long | SELECT xpath_long('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_long(<a><b>1</b><b>2</b></a>, sum(a/b)):bigint> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathShort | xpath_short | SELECT xpath_short('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_short(<a><b>1</b><b>2</b></a>, sum(a/b)):smallint> |
-| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
\ No newline at end of file
+| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
index 3329150299ec6..dfdf183233b42 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -55,7 +55,7 @@ struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
 -- !query
 select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
 -- !query schema
-struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+struct<locate(bar, foobarbar, 1):int,position(CAST(NULL AS STRING), foobarbar, 1):int,position(aaads, CAST(NULL AS STRING), 1):int>
 -- !query output
 4	NULL	NULL
 
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
index e59b9d5b63a40..7b7aeb4ec7934 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
@@ -4654,7 +4654,7 @@ struct<id:bigint>
 -- !query
 select ln(1.2345678e-28)
 -- !query schema
-struct<LOG(1.2345678E-28):double>
+struct<ln(1.2345678E-28):double>
 -- !query output
 -64.26166165451762
 
@@ -4662,7 +4662,7 @@ struct<LOG(1.2345678E-28):double>
 -- !query
 select ln(0.0456789)
 -- !query schema
-struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
+struct<ln(CAST(0.0456789 AS DOUBLE)):double>
 -- !query output
 -3.0861187944847437
 
@@ -4670,7 +4670,7 @@ struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
 -- !query
 select ln(0.99949452)
 -- !query schema
-struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
+struct<ln(CAST(0.99949452 AS DOUBLE)):double>
 -- !query output
 -5.056077980832118E-4
 
@@ -4678,7 +4678,7 @@ struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
 -- !query
 select ln(1.00049687395)
 -- !query schema
-struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
+struct<ln(CAST(1.00049687395 AS DOUBLE)):double>
 -- !query output
 4.967505490136803E-4
 
@@ -4686,7 +4686,7 @@ struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
 -- !query
 select ln(1234.567890123456789)
 -- !query schema
-struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
+struct<ln(CAST(1234.567890123456789 AS DOUBLE)):double>
 -- !query output
 7.11847630129779
 
@@ -4694,7 +4694,7 @@ struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
 -- !query
 select ln(5.80397490724e5)
 -- !query schema
-struct<LOG(580397.490724):double>
+struct<ln(580397.490724):double>
 -- !query output
 13.271468476626518
 
@@ -4702,7 +4702,7 @@ struct<LOG(580397.490724):double>
 -- !query
 select ln(9.342536355e34)
 -- !query schema
-struct<LOG(9.342536355E34):double>
+struct<ln(9.342536355E34):double>
 -- !query output
 80.52247093552418
 
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index cb5501ddb67af..c9aad35bec98a 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -55,7 +55,7 @@ struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
 -- !query
 select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
 -- !query schema
-struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+struct<locate(bar, foobarbar, 1):int,position(CAST(NULL AS STRING), foobarbar, 1):int,position(aaads, CAST(NULL AS STRING), 1):int>
 -- !query output
 4	NULL	NULL
 

From 4611d2124cf756471675eefa2760d23ae5818b94 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 27 May 2020 17:26:07 +0000
Subject: [PATCH 0742/1280] [SPARK-31835][SQL][TESTS] Add zoneId to codegen
 related tests in DateExpressionsSuite

### What changes were proposed in this pull request?

This PR modifies some codegen related tests to test escape characters for datetime functions which are time zone aware. If the timezone is absent, the formatter could result in `null` caused by `java.util.NoSuchElementException: None.get` and bypassing the real intention of those test cases.

### Why are the changes needed?

fix tests

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

passing the modified test cases.

Closes #28653 from yaooqinn/SPARK-31835.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 311fe6a880f371c20ca5156ca6eb7dec5a15eff6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/expressions/DateExpressionsSuite.scala     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 5b5c85f96c06f..e00d65fa8678d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -793,7 +793,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       }
     }
     // Test escaping of format
-    GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\"quote")) :: Nil)
+    GenerateUnsafeProjection.generate(FromUnixTime(Literal(0L), Literal("\"quote"), UTC_OPT) :: Nil)
   }
 
   test("unix_timestamp") {
@@ -863,7 +863,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
     // Test escaping of format
     GenerateUnsafeProjection.generate(
-      UnixTimestamp(Literal("2015-07-24"), Literal("\"quote")) :: Nil)
+      UnixTimestamp(Literal("2015-07-24"), Literal("\"quote"), UTC_OPT) :: Nil)
   }
 
   test("to_unix_timestamp") {
@@ -941,7 +941,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
     // Test escaping of format
     GenerateUnsafeProjection.generate(
-      ToUnixTimestamp(Literal("2015-07-24"), Literal("\"quote")) :: Nil)
+      ToUnixTimestamp(Literal("2015-07-24"), Literal("\"quote"), UTC_OPT) :: Nil)
   }
 
   test("datediff") {

From 0383d1efe7a7ada8a202fd411bf32b3ed80c9ce4 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 27 May 2020 18:53:19 +0000
Subject: [PATCH 0743/1280] [SPARK-31827][SQL] fail datetime parsing/formatting
 if detect the Java 8 bug of stand-alone form

If `LLL`/`qqq` is used in the datetime pattern string, and the current JDK in use has a bug for the stand-alone form (see https://bugs.openjdk.java.net/browse/JDK-8114833), throw an exception with a clear error message.

to keep backward compatibility with Spark 2.4

Yes

Spark 2.4
```
scala> sql("select date_format('1990-1-1', 'LLL')").show
+---------------------------------------------+
|date_format(CAST(1990-1-1 AS TIMESTAMP), LLL)|
+---------------------------------------------+
|                                          Jan|
+---------------------------------------------+
```

Spark 3.0 with Java 11
```
scala> sql("select date_format('1990-1-1', 'LLL')").show
+---------------------------------------------+
|date_format(CAST(1990-1-1 AS TIMESTAMP), LLL)|
+---------------------------------------------+
|                                          Jan|
+---------------------------------------------+
```

Spark 3.0 with Java 8
```
// before this PR
+---------------------------------------------+
|date_format(CAST(1990-1-1 AS TIMESTAMP), LLL)|
+---------------------------------------------+
|                                            1|
+---------------------------------------------+
// after this PR
scala> sql("select date_format('1990-1-1', 'LLL')").show
org.apache.spark.SparkUpgradeException
```

manual test with java 8 and 11

Closes #28646 from cloud-fan/format.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md               |  7 ++++---
 .../util/DateTimeFormatterHelper.scala         | 18 +++++++++++++++++-
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 0e00e7bdbbd6b..48e85b450e6b2 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -76,7 +76,8 @@ The count of pattern letters determines the format.
 
 - Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
 
-- Month: If the number of pattern letters is 3 or more, the month is interpreted as text; otherwise, it is interpreted as a number. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. The difference between the 'standard' and 'stand-alone' forms is trickier to describe as there is no difference in English. However, in other languages there is a difference in the word used when the text is used alone, as opposed to in a complete date. For example, the word used for a month when used alone in a date picker is different to the word used for month in association with a day and year in a date. In Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters (more than 4 letters is invalid):  - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
+- Month: It follows the rule of Number/Text. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. These two forms are different only in some certain languages. For example, in Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters:
+  - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
     ```sql
     spark-sql> select date_format(date '1970-01-01', "M");
     1
@@ -106,8 +107,8 @@ The count of pattern letters determines the format.
     ```
   - `'MMMM'`: full textual month representation in the standard form. It is used for parsing/formatting months as a part of dates/timestamps.
     ```sql
-    spark-sql> select date_format(date '1970-01-01', "MMMM yyyy");
-    January 1970
+    spark-sql> select date_format(date '1970-01-01', "d MMMM");
+    1 January
     spark-sql> select to_csv(named_struct('date', date '1970-01-01'), map('dateFormat', 'd MMMM', 'locale', 'RU'));
     1 января
     ```
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 8289568c449ad..353c074caa75e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -217,9 +217,19 @@ private object DateTimeFormatterHelper {
     toFormatter(builder, TimestampFormatter.defaultLocale)
   }
 
+  private final val bugInStandAloneForm = {
+    // Java 8 has a bug for stand-alone form. See https://bugs.openjdk.java.net/browse/JDK-8114833
+    // Note: we only check the US locale so that it's a static check. It can produce false-negative
+    // as some locales are not affected by the bug. Since `L`/`q` is rarely used, we choose to not
+    // complicate the check here.
+    // TODO: remove it when we drop Java 8 support.
+    val formatter = DateTimeFormatter.ofPattern("LLL qqq", Locale.US)
+    formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
+  }
   final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
   final val unsupportedNarrowTextStyle =
-    Set("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "QQQQQ", "qqqqq", "uuuuu")
+    Seq("G", "M", "L", "E", "u", "Q", "q").map(_ * 5).toSet
+
   /**
    * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
    * parsing/formatting datetime values. The pattern string is incompatible with the one defined
@@ -243,6 +253,12 @@ private object DateTimeFormatterHelper {
           for (style <- unsupportedNarrowTextStyle if patternPart.contains(style)) {
             throw new IllegalArgumentException(s"Too many pattern letters: ${style.head}")
           }
+          if (bugInStandAloneForm && (patternPart.contains("LLL") || patternPart.contains("qqq"))) {
+            throw new IllegalArgumentException("Java 8 has a bug to support stand-alone " +
+              "form (3 or more 'L' or 'q' in the pattern string). Please use 'M' or 'Q' instead, " +
+              "or upgrade your Java version. For more details, please read " +
+              "https://bugs.openjdk.java.net/browse/JDK-8114833")
+          }
           // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
           // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
           // string. If parsable, return the result; otherwise, fall back to 'u', and then use the

From cb817bb1cf6b074e075c02880001ec96f2f39de7 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Wed, 27 May 2020 16:37:02 -0700
Subject: [PATCH 0744/1280] [SPARK-31730][CORE][TEST] Fix flaky tests in
 BarrierTaskContextSuite

### What changes were proposed in this pull request?

To wait until all the executors have started before submitting any job. This could avoid the flakiness caused by waiting for executors coming up.

### How was this patch tested?

Existing tests.

Closes #28584 from jiangxb1987/barrierTest.

Authored-by: Xingbo Jiang <xingbo.jiang@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
(cherry picked from commit efe7fd2b6bea4a945ed7f3f486ab279c505378b4)
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../scheduler/BarrierTaskContextSuite.scala   | 26 +++++++------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 6191e41b4118f..54899bfcf34fa 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -25,6 +25,7 @@ import org.scalatest.concurrent.Eventually
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark._
+import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 
 class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with Eventually {
@@ -37,10 +38,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
       .setAppName("test-cluster")
       .set(TEST_NO_STAGE_RETRY, true)
     sc = new SparkContext(conf)
+    TestUtils.waitUntilExecutorsUp(sc, numWorker, 60000)
   }
 
-  // TODO (SPARK-31730): re-enable it
-  ignore("global sync by barrier() call") {
+  test("global sync by barrier() call") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -57,10 +58,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("share messages with allGather() call") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -78,10 +76,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("throw exception if we attempt to synchronize with different blocking calls") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -100,10 +95,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("successively sync with allGather and barrier") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -129,8 +121,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     assert(times2.max - times2.min <= 1000)
   }
 
-  // TODO (SPARK-31730): re-enable it
-  ignore("support multiple barrier() call within a single task") {
+  test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -285,6 +276,9 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
 
   test("SPARK-31485: barrier stage should fail if only partial tasks are launched") {
     initLocalClusterSparkContext(2)
+    // It's required to reset the delay timer when a task is scheduled, otherwise all the tasks
+    // could get scheduled at ANY level.
+    sc.conf.set(config.LEGACY_LOCALITY_WAIT_RESET, true)
     val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
     val dep = new OneToOneDependency[Int](rdd0)
     // set up a barrier stage with 2 tasks and both tasks prefer executor 0 (only 1 core) for

From e7b88e82ec5cee0a738f96127b106358cc97cb4f Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Wed, 27 May 2020 17:21:10 -0700
Subject: [PATCH 0745/1280] Revert "[SPARK-31730][CORE][TEST] Fix flaky tests
 in BarrierTaskContextSuite"

This reverts commit cb817bb1cf6b074e075c02880001ec96f2f39de7.
---
 .../scheduler/BarrierTaskContextSuite.scala   | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 54899bfcf34fa..6191e41b4118f 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -25,7 +25,6 @@ import org.scalatest.concurrent.Eventually
 import org.scalatest.time.SpanSugar._
 
 import org.apache.spark._
-import org.apache.spark.internal.config
 import org.apache.spark.internal.config.Tests.TEST_NO_STAGE_RETRY
 
 class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with Eventually {
@@ -38,10 +37,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
       .setAppName("test-cluster")
       .set(TEST_NO_STAGE_RETRY, true)
     sc = new SparkContext(conf)
-    TestUtils.waitUntilExecutorsUp(sc, numWorker, 60000)
   }
 
-  test("global sync by barrier() call") {
+  // TODO (SPARK-31730): re-enable it
+  ignore("global sync by barrier() call") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -58,7 +57,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("share messages with allGather() call") {
-    initLocalClusterSparkContext()
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -76,7 +78,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("throw exception if we attempt to synchronize with different blocking calls") {
-    initLocalClusterSparkContext()
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -95,7 +100,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("successively sync with allGather and barrier") {
-    initLocalClusterSparkContext()
+    val conf = new SparkConf()
+      .setMaster("local-cluster[4, 1, 1024]")
+      .setAppName("test-cluster")
+    sc = new SparkContext(conf)
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -121,7 +129,8 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     assert(times2.max - times2.min <= 1000)
   }
 
-  test("support multiple barrier() call within a single task") {
+  // TODO (SPARK-31730): re-enable it
+  ignore("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -276,9 +285,6 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
 
   test("SPARK-31485: barrier stage should fail if only partial tasks are launched") {
     initLocalClusterSparkContext(2)
-    // It's required to reset the delay timer when a task is scheduled, otherwise all the tasks
-    // could get scheduled at ANY level.
-    sc.conf.set(config.LEGACY_LOCALITY_WAIT_RESET, true)
     val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
     val dep = new OneToOneDependency[Int](rdd0)
     // set up a barrier stage with 2 tasks and both tasks prefer executor 0 (only 1 core) for

From 11bce50debe2e47a5456f565c04960bd17d0e778 Mon Sep 17 00:00:00 2001
From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Date: Thu, 28 May 2020 09:57:11 +0900
Subject: [PATCH 0746/1280] [SPARK-31839][TESTS] Delete duplicate code in
 castsuit

### What changes were proposed in this pull request?
Delete duplicate code castsuit

### Why are the changes needed?
keep spark code clean

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
no need

Closes #28655 from GuoPhilipse/delete-duplicate-code-castsuit.

Lead-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Co-authored-by: GuoPhilipse <guofei_ok@126.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit dfbc5edf20040e8163ee3beef61f2743a948c508)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/catalyst/expressions/CastSuite.scala    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index fd0ba67ac28b6..f5637200583ef 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -240,7 +240,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
     checkCast(1.5, "1.5")
 
     checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
-    checkEvaluation(cast(cast(1.toDouble, TimestampType), DoubleType), 1.toDouble)
   }
 
   test("cast from string") {

From 9580266c8ee910dbf9a37dd6ff8baa9a94bca38e Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Thu, 28 May 2020 14:25:49 -0700
Subject: [PATCH 0747/1280] [BUILD][INFRA] bump the timeout to match the
 jenkins PRB

### What changes were proposed in this pull request?

bump the timeout to match what's set in jenkins

### Why are the changes needed?

tests be timing out!

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

via jenkins

Closes #28666 from shaneknapp/increase-jenkins-timeout.

Authored-by: shane knapp <incomplete@gmail.com>
Signed-off-by: shane knapp <incomplete@gmail.com>
(cherry picked from commit 9e68affd13a3875b92f0700b8ab7c9d902f1a08c)
Signed-off-by: shane knapp <incomplete@gmail.com>
---
 dev/run-tests-jenkins.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/run-tests-jenkins.py b/dev/run-tests-jenkins.py
index 72e32d4e16e14..13be9592d771f 100755
--- a/dev/run-tests-jenkins.py
+++ b/dev/run-tests-jenkins.py
@@ -198,7 +198,7 @@ def main():
     # format: http://linux.die.net/man/1/timeout
     # must be less than the timeout configured on Jenkins. Usually Jenkins's timeout is higher
     # then this. Please consult with the build manager or a committer when it should be increased.
-    tests_timeout = "400m"
+    tests_timeout = "500m"
 
     # Array to capture all test names to run on the pull request. These tests are represented
     # by their file equivalents in the dev/tests/ directory.

From 3ff4db97fb966e35c0b7450a7210cebf5d331be6 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Thu, 28 May 2020 16:29:40 -0700
Subject: [PATCH 0748/1280] [SPARK-31730][CORE][TEST][3.0] Fix flaky tests in
 BarrierTaskContextSuite

### What changes were proposed in this pull request?

To wait until all the executors have started before submitting any job. This could avoid the flakiness caused by waiting for executors coming up.

### How was this patch tested?

Existing tests.

Closes #28658 from jiangxb1987/barrierTest.

Authored-by: Xingbo Jiang <xingbo.jiang@databricks.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../scheduler/BarrierTaskContextSuite.scala   | 22 +++++--------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 6191e41b4118f..764b4b7587718 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -37,10 +37,10 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
       .setAppName("test-cluster")
       .set(TEST_NO_STAGE_RETRY, true)
     sc = new SparkContext(conf)
+    TestUtils.waitUntilExecutorsUp(sc, numWorker, 60000)
   }
 
-  // TODO (SPARK-31730): re-enable it
-  ignore("global sync by barrier() call") {
+  test("global sync by barrier() call") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
@@ -57,10 +57,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("share messages with allGather() call") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -78,10 +75,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("throw exception if we attempt to synchronize with different blocking calls") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -100,10 +94,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
   }
 
   test("successively sync with allGather and barrier") {
-    val conf = new SparkConf()
-      .setMaster("local-cluster[4, 1, 1024]")
-      .setAppName("test-cluster")
-    sc = new SparkContext(conf)
+    initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>
       val context = BarrierTaskContext.get()
@@ -129,8 +120,7 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
     assert(times2.max - times2.min <= 1000)
   }
 
-  // TODO (SPARK-31730): re-enable it
-  ignore("support multiple barrier() call within a single task") {
+  test("support multiple barrier() call within a single task") {
     initLocalClusterSparkContext()
     val rdd = sc.makeRDD(1 to 10, 4)
     val rdd2 = rdd.barrier().mapPartitions { it =>

From d2fb6e408538e89ac5cc3b3c71925dbaca387bbc Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 29 May 2020 04:13:23 +0000
Subject: [PATCH 0749/1280] [SPARK-31833][SQL][TEST-HIVE1.2] Set
 HiveThriftServer2 with actual port while configured 0

### What changes were proposed in this pull request?

When I was developing some stuff based on the `DeveloperAPI ` `org.apache.spark.sql.hive.thriftserver.HiveThriftServer2#startWithContext`, I need to use thrift port randomly to avoid race on ports. But the `org.apache.hive.service.cli.thrift.ThriftCLIService#getPortNumber` do not respond to me with the actual bound port but always 0.

And the server log is not right too, after starting the server, it's hard to form to the right JDBC connection string.

```
INFO ThriftCLIService: Starting ThriftBinaryCLIService on port 0 with 5...500 worker threads
```

Indeed, the `53742` is the right port
```shell
lsof -nP -p `cat ./pid/spark-kentyao-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid` | grep LISTEN
java    18990 kentyao  288u    IPv6 0x22858e3e60d6a0a7      0t0                 TCP 10.242.189.214:53723 (LISTEN)
java    18990 kentyao  290u    IPv6 0x22858e3e60d68827      0t0                 TCP *:4040 (LISTEN)
java    18990 kentyao  366u    IPv6 0x22858e3e60d66987      0t0                 TCP 10.242.189.214:53724 (LISTEN)
java    18990 kentyao  438u    IPv6 0x22858e3e60d65d47      0t0                 TCP *:53742 (LISTEN)
```

In the PR, when the port is configured 0, the `portNum` will be set to the real used port during the start process.

Also use 0 in thrift related tests to avoid potential flakiness.

### Why are the changes needed?

1 fix API bug
2 reduce test flakiness

### Does this PR introduce _any_ user-facing change?

yes, `org.apache.hive.service.cli.thrift.ThriftCLIService#getPortNumber` will always give you the actual port when it is configured 0.

### How was this patch tested?

modified unit tests

Closes #28651 from yaooqinn/SPARK-31833.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit fe1da296da152528fc159c23e3026a0c19343780)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../thriftserver/SharedThriftServer.scala     | 33 ++++++++++---------
 .../cli/thrift/ThriftBinaryCLIService.java    |  4 +++
 .../cli/thrift/ThriftHttpCLIService.java      |  3 ++
 .../cli/thrift/ThriftBinaryCLIService.java    |  4 +++
 .../cli/thrift/ThriftHttpCLIService.java      |  3 ++
 5 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index ce610098156f3..e002bc0117c8b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -19,29 +19,25 @@ package org.apache.spark.sql.hive.thriftserver
 
 import java.sql.{DriverManager, Statement}
 
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
-import scala.util.{Random, Try}
+import scala.util.Try
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+import org.apache.hive.service.cli.thrift.ThriftCLIService
 
-import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSparkSession
 
 trait SharedThriftServer extends SharedSparkSession {
 
   private var hiveServer2: HiveThriftServer2 = _
+  private var serverPort: Int = 0
 
   override def beforeAll(): Unit = {
     super.beforeAll()
-    // Chooses a random port between 10000 and 19999
-    var listeningPort = 10000 + Random.nextInt(10000)
-
     // Retries up to 3 times with different port numbers if the server fails to start
-    (1 to 3).foldLeft(Try(startThriftServer(listeningPort, 0))) { case (started, attempt) =>
-      started.orElse {
-        listeningPort += 1
-        Try(startThriftServer(listeningPort, attempt))
-      }
+    (1 to 3).foldLeft(Try(startThriftServer(0))) { case (started, attempt) =>
+      started.orElse(Try(startThriftServer(attempt)))
     }.recover {
       case cause: Throwable =>
         throw cause
@@ -59,8 +55,7 @@ trait SharedThriftServer extends SharedSparkSession {
 
   protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
     val user = System.getProperty("user.name")
-
-    val serverPort = hiveServer2.getHiveConf.get(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname)
+    require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
       fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
     val statements = connections.map(_.createStatement())
@@ -73,11 +68,19 @@ trait SharedThriftServer extends SharedSparkSession {
     }
   }
 
-  private def startThriftServer(port: Int, attempt: Int): Unit = {
-    logInfo(s"Trying to start HiveThriftServer2: port=$port, attempt=$attempt")
+  private def startThriftServer(attempt: Int): Unit = {
+    logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, port.toString)
+    // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any free port to use.
+    // It's much more robust than set a random port generated by ourselves ahead
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
     hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+    hiveServer2.getServices.asScala.foreach {
+      case t: ThriftCLIService if t.getPortNumber != 0 =>
+        serverPort = t.getPortNumber
+        logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
+      case _ =>
+    }
 
     // Wait for thrift server to be ready to serve the query, via executing simple query
     // till the query succeeds. See SPARK-30345 for more details.
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index 21b8bf7de75ce..e1ee503b81209 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -76,6 +76,10 @@ public void run() {
             keyStorePassword, sslVersionBlacklist);
       }
 
+      // In case HIVE_SERVER2_THRIFT_PORT or hive.server2.thrift.port is configured with 0 which
+      // represents any free port, we should set it to the actual one
+      portNum = serverSocket.getServerSocket().getLocalPort();
+
       // Server args
       int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
       int requestTimeout = (int) hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 504e63dbc5e5e..1099a00b67eb7 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -143,6 +143,9 @@ public void run() {
       // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc.
       // Finally, start the server
       httpServer.start();
+      // In case HIVE_SERVER2_THRIFT_HTTP_PORT or hive.server2.thrift.http.port is configured with
+      // 0 which represents any free port, we should set it to the actual one
+      portNum = connector.getLocalPort();
       String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
           + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index fc19c65daaf54..a7de9c0f3d0d2 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -77,6 +77,10 @@ public void run() {
             keyStorePassword, sslVersionBlacklist);
       }
 
+      // In case HIVE_SERVER2_THRIFT_PORT or hive.server2.thrift.port is configured with 0 which
+      // represents any free port, we should set it to the actual one
+      portNum = serverSocket.getServerSocket().getLocalPort();
+
       // Server args
       int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
       int requestTimeout = (int) hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 08626e7eb146d..73d5f84476af0 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -144,6 +144,9 @@ public void run() {
       // TODO: check defaults: maxTimeout, keepalive, maxBodySize, bodyRecieveDuration, etc.
       // Finally, start the server
       httpServer.start();
+      // In case HIVE_SERVER2_THRIFT_HTTP_PORT or hive.server2.thrift.http.port is configured with
+      // 0 which represents any free port, we should set it to the actual one
+      portNum = connector.getLocalPort();
       String msg = "Started " + ThriftHttpCLIService.class.getSimpleName() + " in " + schemeName
           + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";

From 52c2988a65d6a2d65f45e222423c59c9fb169b87 Mon Sep 17 00:00:00 2001
From: Maryann Xue <maryann.xue@gmail.com>
Date: Fri, 29 May 2020 04:23:38 +0000
Subject: [PATCH 0750/1280] [SPARK-31862][SQL] Remove exception wrapping in AQE

### What changes were proposed in this pull request?

This PR removes the excessive exception wrapping in AQE so that error messages are less verbose and mostly consistent with non-aqe execution. Exceptions from stage materialization are now only wrapped with `SparkException` if there are multiple stage failures. Also, stage cancelling errors will not be included as part the exception thrown, but rather just be error logged.

### Why are the changes needed?

This will make the AQE error reporting more readable and debuggable.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Updated existing tests.

Closes #28668 from maryannxue/spark-31862.

Authored-by: Maryann Xue <maryann.xue@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 45864faaf2c9837a2ca48c456d3c2300736aa1ba)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      | 38 ++++++++-----------
 .../execution/adaptive/QueryStageExec.scala   |  3 +-
 .../apache/spark/sql/MetadataCacheSuite.scala |  9 ++---
 .../adaptive/AdaptiveQueryExecSuite.scala     |  2 +-
 .../adaptive/AdaptiveTestUtils.scala          | 22 -----------
 .../datasources/json/JsonSuite.scala          |  3 +-
 .../datasources/orc/OrcQuerySuite.scala       |  9 ++---
 .../execution/joins/BroadcastJoinSuite.scala  |  4 +-
 .../spark/sql/sources/BucketedReadSuite.scala |  3 +-
 .../sql/hive/HiveMetadataCacheSuite.scala     |  5 +--
 10 files changed, 33 insertions(+), 65 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 90d1db9e4b743..f6a3333b90cb0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -156,7 +156,7 @@ case class AdaptiveSparkPlanExec(
       var currentLogicalPlan = currentPhysicalPlan.logicalLink.get
       var result = createQueryStages(currentPhysicalPlan)
       val events = new LinkedBlockingQueue[StageMaterializationEvent]()
-      val errors = new mutable.ArrayBuffer[SparkException]()
+      val errors = new mutable.ArrayBuffer[Throwable]()
       var stagesToReplace = Seq.empty[QueryStageExec]
       while (!result.allChildStagesMaterialized) {
         currentPhysicalPlan = result.newPlan
@@ -176,9 +176,7 @@ case class AdaptiveSparkPlanExec(
               }(AdaptiveSparkPlanExec.executionContext)
             } catch {
               case e: Throwable =>
-                val ex = new SparkException(
-                  s"Early failed query stage found: ${stage.treeString}", e)
-                cleanUpAndThrowException(Seq(ex), Some(stage.id))
+                cleanUpAndThrowException(Seq(e), Some(stage.id))
             }
           }
         }
@@ -193,8 +191,7 @@ case class AdaptiveSparkPlanExec(
           case StageSuccess(stage, res) =>
             stage.resultOption = Some(res)
           case StageFailure(stage, ex) =>
-            errors.append(
-              new SparkException(s"Failed to materialize query stage: ${stage.treeString}.", ex))
+            errors.append(ex)
         }
 
         // In case of errors, we cancel all running stages and throw exception.
@@ -536,31 +533,28 @@ case class AdaptiveSparkPlanExec(
    * materialization errors and stage cancellation errors.
    */
   private def cleanUpAndThrowException(
-       errors: Seq[SparkException],
+       errors: Seq[Throwable],
        earlyFailedStage: Option[Int]): Unit = {
-    val runningStages = currentPhysicalPlan.collect {
+    currentPhysicalPlan.foreach {
       // earlyFailedStage is the stage which failed before calling doMaterialize,
       // so we should avoid calling cancel on it to re-trigger the failure again.
-      case s: QueryStageExec if !earlyFailedStage.contains(s.id) => s
-    }
-    val cancelErrors = new mutable.ArrayBuffer[SparkException]()
-    try {
-      runningStages.foreach { s =>
+      case s: QueryStageExec if !earlyFailedStage.contains(s.id) =>
         try {
           s.cancel()
         } catch {
           case NonFatal(t) =>
-            cancelErrors.append(
-              new SparkException(s"Failed to cancel query stage: ${s.treeString}", t))
+            logError(s"Exception in cancelling query stage: ${s.treeString}", t)
         }
-      }
-    } finally {
-      val ex = new SparkException(
-        "Adaptive execution failed due to stage materialization failures.", errors.head)
-      errors.tail.foreach(ex.addSuppressed)
-      cancelErrors.foreach(ex.addSuppressed)
-      throw ex
+      case _ =>
+    }
+    val e = if (errors.size == 1) {
+      errors.head
+    } else {
+      val se = new SparkException("Multiple failures in stage materialization.", errors.head)
+      errors.tail.foreach(se.addSuppressed)
+      se
     }
+    throw e
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
index f414f854b92ae..9a9a8b1c477bc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
@@ -25,6 +25,7 @@ import org.apache.spark.{FutureAction, MapOutputStatistics, SparkException}
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.errors.attachTree
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
@@ -147,7 +148,7 @@ case class ShuffleQueryStageExec(
       throw new IllegalStateException("wrong plan for shuffle stage:\n " + plan.treeString)
   }
 
-  override def doMaterialize(): Future[Any] = {
+  override def doMaterialize(): Future[Any] = attachTree(this, "execute") {
     shuffle.mapOutputStatisticsFuture
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
index a9f443be69cb2..956bd7861d99d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MetadataCacheSuite.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql
 import java.io.File
 
 import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -56,8 +55,8 @@ abstract class MetadataCacheSuite extends QueryTest with SharedSparkSession {
       val e = intercept[SparkException] {
         df.count()
       }
-      assertExceptionMessage(e, "FileNotFoundException")
-      assertExceptionMessage(e, "recreating the Dataset/DataFrame involved")
+      assert(e.getMessage.contains("FileNotFoundException"))
+      assert(e.getMessage.contains("recreating the Dataset/DataFrame involved"))
     }
   }
 }
@@ -85,8 +84,8 @@ class MetadataCacheV1Suite extends MetadataCacheSuite {
       val e = intercept[SparkException] {
         sql("select count(*) from view_refresh").first()
       }
-      assertExceptionMessage(e, "FileNotFoundException")
-      assertExceptionMessage(e, "REFRESH")
+      assert(e.getMessage.contains("FileNotFoundException"))
+      assert(e.getMessage.contains("REFRESH"))
 
       // Refresh and we should be able to read it again.
       spark.catalog.refreshTable("view_refresh")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 18923b209b533..37e8e13a961f2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -706,7 +706,7 @@ class AdaptiveQueryExecSuite
         val error = intercept[Exception] {
           agged.count()
         }
-        assert(error.getCause().toString contains "Early failed query stage found")
+        assert(error.getCause().toString contains "Invalid bucket file")
         assert(error.getSuppressed.size === 0)
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
index ddaeb57d31547..48f85ae76cd8c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveTestUtils.scala
@@ -69,25 +69,3 @@ trait DisableAdaptiveExecutionSuite extends SQLTestUtils {
     }
   }
 }
-
-object AdaptiveTestUtils {
-  def assertExceptionMessage(e: Exception, expected: String): Unit = {
-    val stringWriter = new StringWriter()
-    e.printStackTrace(new PrintWriter(stringWriter))
-    val errorMsg = stringWriter.toString
-    assert(errorMsg.contains(expected))
-  }
-
-  def assertExceptionCause(t: Throwable, causeClass: Class[_]): Unit = {
-    var c = t.getCause
-    var foundCause = false
-    while (c != null && !foundCause) {
-      if (causeClass.isAssignableFrom(c.getClass)) {
-        foundCause = true
-      } else {
-        c = c.getCause
-      }
-    }
-    assert(foundCause, s"Can not find cause: $causeClass")
-  }
-}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 4982991bdda56..19ec5862e6583 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -35,7 +35,6 @@ import org.apache.spark.sql.{functions => F, _}
 import org.apache.spark.sql.catalyst.json._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.ExternalRDD
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.DataSource
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -2239,7 +2238,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         .count()
     }
 
-    assertExceptionMessage(exception, "Malformed records are detected in record parsing")
+    assert(exception.getMessage.contains("Malformed records are detected in record parsing"))
   }
 
   def checkEncoding(expectedEncoding: String, pathToJsonFiles: String,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
index 60f278b8e5bb0..9caf0c836f711 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala
@@ -34,7 +34,6 @@ import org.apache.orc.mapreduce.OrcInputFormat
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation, RecordReaderIterator}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -599,19 +598,19 @@ abstract class OrcQueryTest extends OrcTest {
       val e1 = intercept[SparkException] {
         testIgnoreCorruptFiles()
       }
-      assertExceptionMessage(e1, "Malformed ORC file")
+      assert(e1.getMessage.contains("Malformed ORC file"))
       val e2 = intercept[SparkException] {
         testIgnoreCorruptFilesWithoutSchemaInfer()
       }
-      assertExceptionMessage(e2, "Malformed ORC file")
+      assert(e2.getMessage.contains("Malformed ORC file"))
       val e3 = intercept[SparkException] {
         testAllCorruptFiles()
       }
-      assertExceptionMessage(e3, "Could not read footer for file")
+      assert(e3.getMessage.contains("Could not read footer for file"))
       val e4 = intercept[SparkException] {
         testAllCorruptFilesWithoutSchemaInfer()
       }
-      assertExceptionMessage(e4, "Malformed ORC file")
+      assert(e4.getMessage.contains("Malformed ORC file"))
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 1be9308c06d8c..335ef257920c4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.{Dataset, QueryTest, Row, SparkSession}
 import org.apache.spark.sql.catalyst.expressions.{BitwiseAnd, BitwiseOr, Cast, Literal, ShiftLeft}
 import org.apache.spark.sql.catalyst.plans.logical.BROADCAST
 import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec}
-import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, AdaptiveTestUtils, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.exchange.EnsureRequirements
 import org.apache.spark.sql.functions._
@@ -411,7 +411,7 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
       val e = intercept[Exception] {
         testDf.collect()
       }
-      AdaptiveTestUtils.assertExceptionMessage(e, s"Could not execute broadcast in $timeout secs.")
+      assert(e.getMessage.contains(s"Could not execute broadcast in $timeout secs."))
     }
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index 707d9c2245570..14ba008e89961 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -29,7 +29,6 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
 import org.apache.spark.sql.execution.{DataSourceScanExec, FileSourceScanExec, SortExec, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.execution.datasources.BucketingUtils
 import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
 import org.apache.spark.sql.execution.joins.SortMergeJoinExec
@@ -759,7 +758,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
         agged.count()
       }
 
-      assertExceptionMessage(error, "Invalid bucket file")
+      assert(error.getCause().toString contains "Invalid bucket file")
     }
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
index 743cdbd6457d7..db8ebcd45f3eb 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetadataCacheSuite.scala
@@ -21,7 +21,6 @@ import org.apache.hadoop.fs.Path
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.adaptive.AdaptiveTestUtils.assertExceptionMessage
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
@@ -100,7 +99,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assertExceptionMessage(e, "FileNotFoundException")
+            assert(e.getMessage.contains("FileNotFoundException"))
 
             // Test refreshing the cache.
             spark.catalog.refreshTable("test")
@@ -115,7 +114,7 @@ class HiveMetadataCacheSuite extends QueryTest with SQLTestUtils with TestHiveSi
             val e2 = intercept[SparkException] {
               sql("select * from test").count()
             }
-            assertExceptionMessage(e2, "FileNotFoundException")
+            assert(e.getMessage.contains("FileNotFoundException"))
             spark.catalog.refreshByPath(dir.getAbsolutePath)
             assert(sql("select * from test").count() == 3)
           }

From d7e6e08f34a8a63027cb1651f8d225454f494fc7 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 29 May 2020 05:18:37 +0000
Subject: [PATCH 0751/1280] [SPARK-31855][SQL][TESTS] Check reading
 date/timestamp from Avro files w/ and w/o Spark version

### What changes were proposed in this pull request?
1. Add the following parquet files to the resource folder `external/avro/src/test/resources`:
   - Files saved by Spark 2.4.5 (https://github.com/apache/spark/commit/cee4ecbb16917fa85f02c635925e2687400aa56b) without meta info `org.apache.spark.version`
      - `before_1582_date_v2_4_5.avro` with a date column: `avro.schema	{"type":"record","name":"topLevelRecord","fields":[{"name":"dt","type":[{"type":"int","logicalType":"date"},"null"]}]}`
      - `before_1582_timestamp_millis_v2_4_5.avro` with a timestamp column: `avro.schema	{"type":"record","name":"test","namespace":"logical","fields":[{"name":"dt","type":["null",{"type":"long","logicalType":"timestamp-millis"}],"default":null}]}`
      - `before_1582_timestamp_micros_v2_4_5.avro` with a timestamp column: `avro.schema	{"type":"record","name":"topLevelRecord","fields":[{"name":"dt","type":[{"type":"long","logicalType":"timestamp-micros"},"null"]}]}`
    - Files saved by Spark 2.4.6-rc3 (https://github.com/apache/spark/commit/570848da7c48ba0cb827ada997e51677ff672a39) with the meta info `org.apache.spark.version	2.4.6`:
      - `before_1582_date_v2_4_6.avro` is similar to `before_1582_date_v2_4_5.avro` except Spark version in parquet meta info.
      - `before_1582_timestamp_micros_v2_4_6.avro` is similar to `before_1582_timestamp_micros_v2_4_5.avro` except meta info.
      - `before_1582_timestamp_millis_v2_4_6.avro` is similar to `before_1582_timestamp_millis_v2_4_5.avro` except meta info.
2. Removed a few avro files becaused they are replaced by Avro files generated by Spark 2.4.5 above.
3. Add new test "generate test files for checking compatibility with Spark 2.4" to `AvroSuite` (marked as ignored). The parquet files above were generated by this test.
4. Modified "SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps" in `AvroSuite` to use new parquet files.

### Why are the changes needed?
To improve test coverage.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By `AvroV1Suite` and `AvroV2Suite`.

Closes #28664 from MaxGekk/avro-update-resource-files.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 37a1fb8d089190929b77421fbb449bf0ed1ab25e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../test/resources/before_1582_date_v2_4.avro | Bin 202 -> 0 bytes
 .../resources/before_1582_date_v2_4_5.avro    | Bin 0 -> 200 bytes
 .../resources/before_1582_date_v2_4_6.avro    | Bin 0 -> 231 bytes
 .../before_1582_timestamp_micros_v2_4_5.avro  | Bin 0 -> 218 bytes
 .../before_1582_timestamp_micros_v2_4_6.avro  | Bin 0 -> 249 bytes
 ... before_1582_timestamp_millis_v2_4_5.avro} | Bin 244 -> 244 bytes
 .../before_1582_timestamp_millis_v2_4_6.avro  | Bin 0 -> 275 bytes
 .../resources/before_1582_ts_micros_v2_4.avro | Bin 218 -> 0 bytes
 .../org/apache/spark/sql/avro/AvroSuite.scala | 117 +++++++++++++++---
 9 files changed, 98 insertions(+), 19 deletions(-)
 delete mode 100644 external/avro/src/test/resources/before_1582_date_v2_4.avro
 create mode 100644 external/avro/src/test/resources/before_1582_date_v2_4_5.avro
 create mode 100644 external/avro/src/test/resources/before_1582_date_v2_4_6.avro
 create mode 100644 external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro
 create mode 100644 external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro
 rename external/avro/src/test/resources/{before_1582_ts_millis_v2_4.avro => before_1582_timestamp_millis_v2_4_5.avro} (52%)
 create mode 100644 external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro
 delete mode 100644 external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro

diff --git a/external/avro/src/test/resources/before_1582_date_v2_4.avro b/external/avro/src/test/resources/before_1582_date_v2_4.avro
deleted file mode 100644
index 96aa7cbf176a537ca3e845aab126eba8ed6aba7c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 202
zcmeZI%3@>@ODrqO*DFrWNX<?Bz*wzRQdy9yWTjM;nw(#hqNJmgmzWFUm*f}tq?V=T
z1i{49GE;L>ij}OQt6@q~5=&Bn8XyJ$g`p;8=9K_NbMn(OlM{17Af|(r)&dPL&B;-U
zt&Ob}0lOkOKP5Gpr#LUMprDdrzHQd^1CI?3%Fp`waqSAJTqY3~9)`~|wP*dj)rKwy
E0Gm2Ya{vGU

diff --git a/external/avro/src/test/resources/before_1582_date_v2_4_5.avro b/external/avro/src/test/resources/before_1582_date_v2_4_5.avro
new file mode 100644
index 0000000000000000000000000000000000000000..5c15601f7ee4b853c3d970b42180a4abba94fb76
GIT binary patch
literal 200
zcmeZI%3@>@ODrqO*DFrWNX<=r!&t3UQdy9yWTjM;nw(#hqNJmgmzWFUm*f}tq?V=T
z1i{49GE;L>ij}OQt6@q~N`T5C+JS;lb29TlqB;5LnaPPcArP}u5=&B*YJo<V=Hw{F
z*2dO~fZdRspOTu)Q=FGrP*BMrCd>XZ^F3RB#)8%I$-Ls4Od>2i44-Fe&-!_*4P6cZ
DU{gdb

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_date_v2_4_6.avro b/external/avro/src/test/resources/before_1582_date_v2_4_6.avro
new file mode 100644
index 0000000000000000000000000000000000000000..212ea1d5efa5ce65005e3a9bac918c39af9b5011
GIT binary patch
literal 231
zcmeZI%3@>_ODrqO*DFrWNX<=r!&t3UQdy9yWTjM;nw(#hqNJmgmzWFUm*f}tq?V=T
z1i{49GE;L>ij}OQt6@q~N`T5C+JS;lb29TlqB;5LnaPPcArP}u5=&B*YJo<V=Hw{F
z*2dNv<QJvuB^D$C9idlTkXV$hSC(2-oSC1;Wu#}KXC?x6Vsd^;YBEo8USdH(B}2&H
i4Q8{~<(F1%G!ItQy#9$vgoTIU^GxkoKX0|6%K-qj6I3_=

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_5.avro
new file mode 100644
index 0000000000000000000000000000000000000000..c3445e3999bc1c1d65f9a8753e3a374fa585db78
GIT binary patch
literal 218
zcmeZI%3@>@ODrqO*DFrWNX<=bW2#mvsVqoUvQjEaP0lY$QPNS$OUwoFOY#eRQp-|v
zf?(olnW;G`#Y$Gu)i5O~B|zm6?La}OIXU@x=|JI}{PfJ^#GDX_;U$^5sl_FUxdpnp
znaM@@#Y(k6drEV1lwxaRYem2gO3qJ7P39@iODrg;WSIY)N#B2thSjXs+g|^8So()a
Xj#Gi*|KvwcU#+~DBQ|YPAi4qoE74R^

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro b/external/avro/src/test/resources/before_1582_timestamp_micros_v2_4_6.avro
new file mode 100644
index 0000000000000000000000000000000000000000..96008d2378b1fee0ef8c9ffcf60cab37ec6f2b57
GIT binary patch
literal 249
zcmeZI%3@>_ODrqO*DFrWNX<=bW2#mvsVqoUvQjEaP0lY$QPNS$OUwoFOY#eRQp-|v
zf?(olnW;G`#Y$Gu)i5O~B|zm6?La}OIXU@x=|JI}{PfJ^#GDX_;U$^5sl_FUxdpnp
znaM@@#Y(k6drEV1lwxaRYYp;?()AJx5`iw#D=tVZ%GN7OEh^5;&*L)EGtn~>0lPOj
zKP5Gpr#LUMprDe$R^TZ+&oWi{o7wuRieL7BWRl}lVE8}z(bHEeFXo6%n-qwy002i$
BSrGsL

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro
similarity index 52%
rename from external/avro/src/test/resources/before_1582_ts_millis_v2_4.avro
rename to external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_5.avro
index dbaec814eb9549543358c68f94429140f60318ba..be12a0782073c10f03f7987746be08eee53a3165 100644
GIT binary patch
delta 61
zcmeyu_=RyoG*e2+#MpF$)`j9<mQ7NL&a#mSXD>D0z$C*V!!+~KpZ6a=^M0Mn{tsOq
E03HMy761SM

delta 61
zcmeyu_=RyoG*e0O#MpF$?VH8i?}yp>=|{R5$$vR7#w5cb!!-TUpZ6a=^DbRfvlv|-
E034+nG5`Po

diff --git a/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro b/external/avro/src/test/resources/before_1582_timestamp_millis_v2_4_6.avro
new file mode 100644
index 0000000000000000000000000000000000000000..262f5dd6e77a408637f688d16ba5b81056e0a135
GIT binary patch
literal 275
zcmeZI%3@>_ODrqO*DFrWNX<>$!&I$QQdy9yWTjM;nw(#hqNJmgmzWFUm!uY#KzYRl
ziOC?Loc#36<is4HXj*1!PD-(oRdh8>c?w7|#HeVcywV(?D!9=(`FZI;S*Z39h<zoQ
zxj-8da|?8HGjnn>i<N3)b(B(4(-KQ_N|dZXCe+5(8srzH>m>rM&q&n+x}qpsuPn8w
zI5R(w%Sg{e&rAgDmE`=C)MTFGyu^ZnN``w^m=l*ypC7#1@<7sZ<}6kw84ekynV0^&
O|L~dj>s<DK=<)zyRb%J?

literal 0
HcmV?d00001

diff --git a/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro b/external/avro/src/test/resources/before_1582_ts_micros_v2_4.avro
deleted file mode 100644
index efe5e71a58813ecd1bb2c8105341103518c984d3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 218
zcmeZI%3@>@ODrqO*DFrWNX<=bW2#mvsVqoUvQjEaP0lY$QPNS$OUwoFOY#eRQp-|v
zf?(olnW;G`#Y$Gu)o>-nK;;naKtZTEIr(|%K;fMH^vvYMoDhiNC7HRY#U+Wk1-iMJ
z$wm3aO0__HN^^3QVrye-MZgY9&QD2A<|)ogEGVdC==4kz>^@w+=(}Ubgt!%Rzca~k
WDlq(?{OIYcl^1iwrcDY&R{#JKF;cAn

diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index a5c1fb15add5c..e2ae489446d85 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.avro
 
 import java.io._
 import java.net.URL
-import java.nio.file.{Files, Paths}
+import java.nio.file.{Files, Paths, StandardCopyOption}
 import java.sql.{Date, Timestamp}
 import java.util.{Locale, UUID}
 
@@ -38,7 +38,7 @@ import org.apache.spark.sql._
 import org.apache.spark.sql.TestingUDT.IntervalData
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
 import org.apache.spark.sql.catalyst.plans.logical.Filter
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, UTC}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA, UTC}
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.datasources.{DataSource, FilePartition}
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
@@ -1529,23 +1529,82 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  // It generates input files for the test below:
+  // "SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps"
+  ignore("SPARK-31855: generate test files for checking compatibility with Spark 2.4") {
+    val resourceDir = "external/avro/src/test/resources"
+    val version = "2_4_6"
+    def save(
+      in: Seq[String],
+      t: String,
+      dstFile: String,
+      options: Map[String, String] = Map.empty): Unit = {
+      withTempDir { dir =>
+        in.toDF("dt")
+          .select($"dt".cast(t))
+          .repartition(1)
+          .write
+          .mode("overwrite")
+          .options(options)
+          .format("avro")
+          .save(dir.getCanonicalPath)
+        Files.copy(
+          dir.listFiles().filter(_.getName.endsWith(".avro")).head.toPath,
+          Paths.get(resourceDir, dstFile),
+          StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    withDefaultTimeZone(LA) {
+      withSQLConf(
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> LA.getId) {
+        save(
+          Seq("1001-01-01"),
+          "date",
+          s"before_1582_date_v$version.avro")
+        save(
+          Seq("1001-01-01 01:02:03.123"),
+          "timestamp",
+          s"before_1582_timestamp_millis_v$version.avro",
+          // scalastyle:off line.size.limit
+          Map("avroSchema" ->
+            s"""
+               |  {
+               |    "namespace": "logical",
+               |    "type": "record",
+               |    "name": "test",
+               |    "fields": [
+               |      {"name": "dt", "type": ["null", {"type": "long","logicalType": "timestamp-millis"}], "default": null}
+               |    ]
+               |  }
+               |""".stripMargin))
+        // scalastyle:on line.size.limit
+        save(
+          Seq("1001-01-01 01:02:03.123456"),
+          "timestamp",
+          s"before_1582_timestamp_micros_v$version.avro")
+      }
+    }
+  }
+
   test("SPARK-31183: compatibility with Spark 2.4 in reading dates/timestamps") {
     // test reading the existing 2.4 files and new 3.0 files (with rebase on/off) together.
-    def checkReadMixedFiles(fileName: String, dt: String, dataStr: String): Unit = {
+    def checkReadMixedFiles(
+        fileName: String,
+        dt: String,
+        dataStr: String,
+        checkDefaultLegacyRead: String => Unit): Unit = {
       withTempPaths(2) { paths =>
         paths.foreach(_.delete())
         val path2_4 = getResourceAvroFilePath(fileName)
         val path3_0 = paths(0).getCanonicalPath
         val path3_0_rebase = paths(1).getCanonicalPath
         if (dt == "date") {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("date"))
+          val df = Seq(dataStr).toDF("str").select($"str".cast("date").as("dt"))
 
           // By default we should fail to write ancient datetime values.
-          var e = intercept[SparkException](df.write.format("avro").save(path3_0))
+          val e = intercept[SparkException](df.write.format("avro").save(path3_0))
           assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-          // By default we should fail to read ancient datetime values.
-          e = intercept[SparkException](spark.read.format("avro").load(path2_4).collect())
-          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+          checkDefaultLegacyRead(path2_4)
 
           withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
             df.write.format("avro").mode("overwrite").save(path3_0)
@@ -1562,25 +1621,23 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
               1.to(3).map(_ => Row(java.sql.Date.valueOf(dataStr))))
           }
         } else {
-          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("ts"))
+          val df = Seq(dataStr).toDF("str").select($"str".cast("timestamp").as("dt"))
           val avroSchema =
             s"""
               |{
               |  "type" : "record",
               |  "name" : "test_schema",
               |  "fields" : [
-              |    {"name": "ts", "type": {"type": "long", "logicalType": "$dt"}}
+              |    {"name": "dt", "type": {"type": "long", "logicalType": "$dt"}}
               |  ]
               |}""".stripMargin
 
           // By default we should fail to write ancient datetime values.
-          var e = intercept[SparkException] {
+          val e = intercept[SparkException] {
             df.write.format("avro").option("avroSchema", avroSchema).save(path3_0)
           }
           assert(e.getCause.getCause.getCause.isInstanceOf[SparkUpgradeException])
-          // By default we should fail to read ancient datetime values.
-          e = intercept[SparkException](spark.read.format("avro").load(path2_4).collect())
-          assert(e.getCause.isInstanceOf[SparkUpgradeException])
+          checkDefaultLegacyRead(path2_4)
 
           withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> CORRECTED.toString) {
             df.write.format("avro").option("avroSchema", avroSchema).mode("overwrite").save(path3_0)
@@ -1600,11 +1657,33 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       }
     }
 
-    checkReadMixedFiles("before_1582_date_v2_4.avro", "date", "1001-01-01")
-    checkReadMixedFiles(
-      "before_1582_ts_micros_v2_4.avro", "timestamp-micros", "1001-01-01 01:02:03.123456")
-    checkReadMixedFiles(
-      "before_1582_ts_millis_v2_4.avro", "timestamp-millis", "1001-01-01 01:02:03.124")
+    def failInRead(path: String): Unit = {
+      val e = intercept[SparkException](spark.read.format("avro").load(path).collect())
+      assert(e.getCause.isInstanceOf[SparkUpgradeException])
+    }
+    def successInRead(path: String): Unit = spark.read.format("avro").load(path).collect()
+    Seq(
+      // By default we should fail to read ancient datetime values when parquet files don't
+      // contain Spark version.
+      "2_4_5" -> failInRead _,
+      "2_4_6" -> successInRead _
+    ).foreach { case (version, checkDefaultRead) =>
+      checkReadMixedFiles(
+        s"before_1582_date_v$version.avro",
+        "date",
+        "1001-01-01",
+        checkDefaultRead)
+      checkReadMixedFiles(
+        s"before_1582_timestamp_micros_v$version.avro",
+        "timestamp-micros",
+        "1001-01-01 01:02:03.123456",
+        checkDefaultRead)
+      checkReadMixedFiles(
+        s"before_1582_timestamp_millis_v$version.avro",
+        "timestamp-millis",
+        "1001-01-01 01:02:03.123",
+        checkDefaultRead)
+    }
   }
 
   test("SPARK-31183: rebasing microseconds timestamps in write") {

From 0507e6e8d460b083876bd874115837651c19420e Mon Sep 17 00:00:00 2001
From: Juliusz Sompolski <julek@databricks.com>
Date: Sat, 30 May 2020 06:14:32 +0000
Subject: [PATCH 0752/1280] [SPARK-31859][SPARK-31861][SPARK-31863] Fix
 Thriftserver session timezone issues

### What changes were proposed in this pull request?

Timestamp literals in Spark are interpreted as timestamps in local timezone spark.sql.session.timeZone.

If JDBC client is e.g. in TimeZone UTC-7, and sets spark.sql.session.timeZone to PST, and sends a query "SELECT timestamp '2020-05-20 12:00:00'", and the JVM timezone of the Spark cluster is e.g. UTC+2, then what currently happens is:
* The timestamp literal in the query is interpreted as 12:00:00 UTC-7, i.e. 19:00:00 UTC.
* When it's returned from the query, it is collected as a java.sql.Timestamp object with Dataset.collect(), and put into a Thriftserver RowSet.
* Before sending it over the wire, the Timestamp is converted to String. This happens in explicitly in ColumnValue for RowBasedSet, and implicitly in ColumnBuffer for ColumnBasedSet (all non-primitive types are converted toString() there). The conversion toString uses JVM timezone, which results in a "21:00:00" (UTC+2) string representation.
* The client JDBC application parses gets a "21:00:00" Timestamp back (in it's JVM timezone; if the JDBC application cares about the correct UTC internal value, it should set spark.sql.session.timeZone to be consistent with its JVM timezone)

The problem is caused by the conversion happening in Thriftserver RowSet with the generic toString() function, instead of using HiveResults.toHiveString() that takes care of correct, timezone respecting conversions. This PR fixes it by converting the Timestamp values to String earlier, in SparkExecuteStatementOperation, using that function. This fixes SPARK-31861.

Thriftserver also did not work spark.sql.datetime.java8API.enabled, because the conversions in RowSet expected an Timestamp object instead of Instant object. Using HiveResults.toHiveString() also fixes that. For this reason, we also convert Date values in SparkExecuteStatementOperation as well - so that HiveResults.toHiveString() handles LocalDate as well. This fixes SPARK-31859.

Thriftserver also did not correctly set the active SparkSession. Because of that, configuration obtained using SQLConf.get was not the correct session configuration. This affected getting the correct spark.sql.session.timeZone. It is fixed by extending the use of SparkExecuteStatementOperation.withSchedulerPool to also set the correct active SparkSession. When the correct session is set, we also no longer need to maintain the pool mapping in a sessionToActivePool map. The scheduler pool can be just correctly retrieved from the session config. "withSchedulerPool" is renamed to "withLocalProperties" and moved into a mixin helper trait, because it should be applied with every operation. This fixes SPARK-31863.

I used the opportunity to move some repetitive code from the operations to the mixin helper trait.

Closes #28671 from juliuszsompolski/SPARK-31861.

Authored-by: Juliusz Sompolski <julek@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit af35691de449aca2e8db8d6dd7092255a919a04b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../SparkExecuteStatementOperation.scala      | 59 ++++--------
 .../SparkGetCatalogsOperation.scala           | 14 +--
 .../SparkGetColumnsOperation.scala            | 13 +--
 .../SparkGetFunctionsOperation.scala          | 14 +--
 .../SparkGetSchemasOperation.scala            | 14 +--
 .../SparkGetTableTypesOperation.scala         | 13 +--
 .../SparkGetTablesOperation.scala             | 13 +--
 .../SparkGetTypeInfoOperation.scala           | 13 +--
 .../SparkMetadataOperationUtils.scala         | 34 -------
 .../hive/thriftserver/SparkOperation.scala    | 96 +++++++++++++++++++
 .../thriftserver/SparkSQLSessionManager.scala |  1 -
 .../server/SparkSQLOperationManager.scala     |  5 +-
 .../HiveThriftServer2Suites.scala             | 55 +++++++++++
 .../apache/hive/service/cli/ColumnValue.java  | 20 +---
 .../hive/service/cli/operation/Operation.java |  5 +-
 .../apache/hive/service/cli/ColumnValue.java  | 20 +---
 .../hive/service/cli/operation/Operation.java |  5 +-
 17 files changed, 209 insertions(+), 185 deletions(-)
 delete mode 100644 sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
 create mode 100644 sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index d14d70f7d3d83..b193c73563ae0 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -44,12 +44,13 @@ import org.apache.spark.unsafe.types.CalendarInterval
 import org.apache.spark.util.{Utils => SparkUtils}
 
 private[hive] class SparkExecuteStatementOperation(
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     statement: String,
     confOverlay: JMap[String, String],
     runInBackground: Boolean = true)
-    (sqlContext: SQLContext, sessionToActivePool: JMap[SessionHandle, String])
   extends ExecuteStatementOperation(parentSession, statement, confOverlay, runInBackground)
+  with SparkOperation
   with Logging {
 
   private var result: DataFrame = _
@@ -62,7 +63,6 @@ private[hive] class SparkExecuteStatementOperation(
   private var previousFetchStartOffset: Long = 0
   private var iter: Iterator[SparkRow] = _
   private var dataTypes: Array[DataType] = _
-  private var statementId: String = _
 
   private lazy val resultSchema: TableSchema = {
     if (result == null || result.schema.isEmpty) {
@@ -73,13 +73,6 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  override def close(): Unit = {
-    // RDDs will be cleaned automatically upon garbage collection.
-    logInfo(s"Close statement with $statementId")
-    cleanup(OperationState.CLOSED)
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
-
   def addNonNullColumnValue(from: SparkRow, to: ArrayBuffer[Any], ordinal: Int): Unit = {
     dataTypes(ordinal) match {
       case StringType =>
@@ -100,12 +93,15 @@ private[hive] class SparkExecuteStatementOperation(
         to += from.getByte(ordinal)
       case ShortType =>
         to += from.getShort(ordinal)
-      case DateType =>
-        to += from.getAs[Date](ordinal)
-      case TimestampType =>
-        to += from.getAs[Timestamp](ordinal)
       case BinaryType =>
         to += from.getAs[Array[Byte]](ordinal)
+      // SPARK-31859, SPARK-31861: Date and Timestamp need to be turned to String here to:
+      // - respect spark.sql.session.timeZone
+      // - work with spark.sql.datetime.java8API.enabled
+      // These types have always been sent over the wire as string, converted later.
+      case _: DateType | _: TimestampType =>
+        val hiveString = HiveResult.toHiveString((from.get(ordinal), dataTypes(ordinal)))
+        to += hiveString
       case CalendarIntervalType =>
         to += HiveResult.toHiveString((from.getAs[CalendarInterval](ordinal), CalendarIntervalType))
       case _: ArrayType | _: StructType | _: MapType | _: UserDefinedType[_] =>
@@ -114,7 +110,7 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = withSchedulerPool {
+  def getNextRowSet(order: FetchOrientation, maxRowsL: Long): RowSet = withLocalProperties {
     log.info(s"Received getNextRowSet request order=${order} and maxRowsL=${maxRowsL} " +
       s"with ${statementId}")
     validateDefaultFetchOrientation(order)
@@ -193,7 +189,6 @@ private[hive] class SparkExecuteStatementOperation(
 
   override def runInternal(): Unit = {
     setState(OperationState.PENDING)
-    statementId = UUID.randomUUID().toString
     logInfo(s"Submitting query '$statement' with $statementId")
     HiveThriftServer2.eventManager.onStatementStart(
       statementId,
@@ -217,7 +212,9 @@ private[hive] class SparkExecuteStatementOperation(
             override def run(): Unit = {
               registerCurrentOperationLog()
               try {
-                execute()
+                withLocalProperties {
+                  execute()
+                }
               } catch {
                 case e: HiveSQLException =>
                   setOperationException(e)
@@ -259,7 +256,7 @@ private[hive] class SparkExecuteStatementOperation(
     }
   }
 
-  private def execute(): Unit = withSchedulerPool {
+  private def execute(): Unit = {
     try {
       synchronized {
         if (getStatus.getState.isTerminal) {
@@ -282,13 +279,6 @@ private[hive] class SparkExecuteStatementOperation(
       sqlContext.sparkContext.setJobGroup(statementId, statement)
       result = sqlContext.sql(statement)
       logDebug(result.queryExecution.toString())
-      result.queryExecution.logical match {
-        case SetCommand(Some((SQLConf.THRIFTSERVER_POOL.key, Some(value)))) =>
-          sessionToActivePool.put(parentSession.getSessionHandle, value)
-          logInfo(s"Setting ${SparkContext.SPARK_SCHEDULER_POOL}=$value for future statements " +
-            "in this session.")
-        case _ =>
-      }
       HiveThriftServer2.eventManager.onStatementParsed(statementId,
         result.queryExecution.toString())
       iter = {
@@ -346,38 +336,25 @@ private[hive] class SparkExecuteStatementOperation(
     synchronized {
       if (!getStatus.getState.isTerminal) {
         logInfo(s"Cancel query with $statementId")
-        cleanup(OperationState.CANCELED)
+        cleanup()
+        setState(OperationState.CANCELED)
         HiveThriftServer2.eventManager.onStatementCanceled(statementId)
       }
     }
   }
 
-  private def cleanup(state: OperationState): Unit = {
-    setState(state)
+  override protected def cleanup(): Unit = {
     if (runInBackground) {
       val backgroundHandle = getBackgroundHandle()
       if (backgroundHandle != null) {
         backgroundHandle.cancel(true)
       }
     }
+    // RDDs will be cleaned automatically upon garbage collection.
     if (statementId != null) {
       sqlContext.sparkContext.cancelJobGroup(statementId)
     }
   }
-
-  private def withSchedulerPool[T](body: => T): T = {
-    val pool = sessionToActivePool.get(parentSession.getSessionHandle)
-    if (pool != null) {
-      sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool)
-    }
-    try {
-      body
-    } finally {
-      if (pool != null) {
-        sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, null)
-      }
-    }
-  }
 }
 
 object SparkExecuteStatementOperation {
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
index 2945cfd200e46..55070e035b944 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetCatalogsOperation.scala
@@ -36,19 +36,13 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetCatalogsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetCatalogsOperation(parentSession) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetCatalogsOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     val logMsg = "Listing catalogs"
     logInfo(s"$logMsg with $statementId")
     setState(OperationState.RUNNING)
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
index ff7cbfeae13be..ca8ad5e6ad134 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetColumnsOperation.scala
@@ -48,26 +48,19 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param columnName column name
  */
 private[hive] class SparkGetColumnsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     tableName: String,
     columnName: String)
   extends GetColumnsOperation(parentSession, catalogName, schemaName, tableName, columnName)
-    with Logging {
+  with SparkOperation
+  with Logging {
 
   val catalog: SessionCatalog = sqlContext.sessionState.catalog
 
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
-
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName, tablePattern : $tableName"
     val logMsg = s"Listing columns '$cmdStr, columnName : $columnName'"
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
index d9c12b6ca9e64..f5e647bfd4f38 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetFunctionsOperation.scala
@@ -43,22 +43,16 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param functionName function name pattern
  */
 private[hive] class SparkGetFunctionsOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     functionName: String)
-  extends GetFunctionsOperation(parentSession, catalogName, schemaName, functionName) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetFunctionsOperation(parentSession, catalogName, schemaName, functionName)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val logMsg = s"Listing functions '$cmdStr, functionName : $functionName'"
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
index db19880d1b99f..74220986fcd34 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetSchemasOperation.scala
@@ -40,21 +40,15 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param schemaName database name, null or a concrete database name
  */
 private[hive] class SparkGetSchemasOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String)
-  extends GetSchemasOperation(parentSession, catalogName, schemaName) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetSchemasOperation(parentSession, catalogName, schemaName)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val logMsg = s"Listing databases '$cmdStr'"
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
index b4093e58d3c07..1cf9c3a731af5 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTableTypesOperation.scala
@@ -37,16 +37,11 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetTableTypesOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetTableTypesOperation(parentSession) with SparkMetadataOperationUtils with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetTableTypesOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
     statementId = UUID.randomUUID().toString
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
index 45c6d980aac47..a1d21e2d60c63 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTablesOperation.scala
@@ -46,24 +46,17 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param tableTypes list of allowed table types, e.g. "TABLE", "VIEW"
  */
 private[hive] class SparkGetTablesOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession,
     catalogName: String,
     schemaName: String,
     tableName: String,
     tableTypes: JList[String])
   extends GetTablesOperation(parentSession, catalogName, schemaName, tableName, tableTypes)
-    with SparkMetadataOperationUtils with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
-    statementId = UUID.randomUUID().toString
     // Do not change cmdStr. It's used for Hive auditing and authorization.
     val cmdStr = s"catalog : $catalogName, schemaPattern : $schemaName"
     val tableTypesStr = if (tableTypes == null) "null" else tableTypes.asScala.mkString(",")
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
index dd5668a93f82d..e38139d60df60 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkGetTypeInfoOperation.scala
@@ -36,16 +36,11 @@ import org.apache.spark.util.{Utils => SparkUtils}
  * @param parentSession a HiveSession from SessionManager
  */
 private[hive] class SparkGetTypeInfoOperation(
-    sqlContext: SQLContext,
+    val sqlContext: SQLContext,
     parentSession: HiveSession)
-  extends GetTypeInfoOperation(parentSession) with Logging {
-
-  private var statementId: String = _
-
-  override def close(): Unit = {
-    super.close()
-    HiveThriftServer2.eventManager.onOperationClosed(statementId)
-  }
+  extends GetTypeInfoOperation(parentSession)
+  with SparkOperation
+  with Logging {
 
   override def runInternal(): Unit = {
     statementId = UUID.randomUUID().toString
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
deleted file mode 100644
index f4c4b04bada2a..0000000000000
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkMetadataOperationUtils.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive.thriftserver
-
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType
-import org.apache.spark.sql.catalyst.catalog.CatalogTableType.{EXTERNAL, MANAGED, VIEW}
-
-/**
- * Utils for metadata operations.
- */
-private[hive] trait SparkMetadataOperationUtils {
-
-  def tableTypeString(tableType: CatalogTableType): String = tableType match {
-    case EXTERNAL | MANAGED => "TABLE"
-    case VIEW => "VIEW"
-    case t =>
-      throw new IllegalArgumentException(s"Unknown table type is found: $t")
-  }
-}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
new file mode 100644
index 0000000000000..3da568cfa256e
--- /dev/null
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.thriftserver
+
+import org.apache.hive.service.cli.OperationState
+import org.apache.hive.service.cli.operation.Operation
+
+import org.apache.spark.SparkContext
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.{SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType
+import org.apache.spark.sql.catalyst.catalog.CatalogTableType.{EXTERNAL, MANAGED, VIEW}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.util.Utils
+
+/**
+ * Utils for Spark operations.
+ */
+private[hive] trait SparkOperation extends Operation with Logging {
+
+  protected def sqlContext: SQLContext
+
+  protected var statementId = getHandle().getHandleIdentifier().getPublicId().toString()
+
+  protected def cleanup(): Unit = Unit // noop by default
+
+  abstract override def run(): Unit = {
+    withLocalProperties {
+      super.run()
+    }
+  }
+
+  abstract override def close(): Unit = {
+    cleanup()
+    super.close()
+    logInfo(s"Close statement with $statementId")
+    HiveThriftServer2.eventManager.onOperationClosed(statementId)
+  }
+
+  // Set thread local properties for the execution of the operation.
+  // This method should be applied during the execution of the operation, by all the child threads.
+  // The original spark context local properties will be restored after the operation.
+  //
+  // It is used to:
+  // - set appropriate SparkSession
+  // - set scheduler pool for the operation
+  def withLocalProperties[T](f: => T): T = {
+    val originalProps = Utils.cloneProperties(sqlContext.sparkContext.getLocalProperties)
+    val originalSession = SparkSession.getActiveSession
+
+    try {
+      // Set active SparkSession
+      SparkSession.setActiveSession(sqlContext.sparkSession)
+
+      // Set scheduler pool
+      sqlContext.sparkSession.conf.getOption(SQLConf.THRIFTSERVER_POOL.key) match {
+        case Some(pool) =>
+          sqlContext.sparkContext.setLocalProperty(SparkContext.SPARK_SCHEDULER_POOL, pool)
+        case None =>
+      }
+
+      // run the body
+      f
+    } finally {
+      // reset local properties, will also reset SPARK_SCHEDULER_POOL
+      sqlContext.sparkContext.setLocalProperties(originalProps)
+
+      originalSession match {
+        case Some(session) => SparkSession.setActiveSession(session)
+        case None => SparkSession.clearActiveSession()
+      }
+    }
+  }
+
+  def tableTypeString(tableType: CatalogTableType): String = tableType match {
+    case EXTERNAL | MANAGED => "TABLE"
+    case VIEW => "VIEW"
+    case t =>
+      throw new IllegalArgumentException(s"Unknown table type is found: $t")
+  }
+}
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
index b3171897141c2..e10e7ed1a2769 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
@@ -78,7 +78,6 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
     val ctx = sparkSqlOperationManager.sessionToContexts.getOrDefault(sessionHandle, sqlContext)
     ctx.sparkSession.sessionState.catalog.getTempViewNames().foreach(ctx.uncacheTable)
     super.closeSession(sessionHandle)
-    sparkSqlOperationManager.sessionToActivePool.remove(sessionHandle)
     sparkSqlOperationManager.sessionToContexts.remove(sessionHandle)
   }
 
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
index 3396560f43502..bc9c13eb0d4f8 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/server/SparkSQLOperationManager.scala
@@ -38,7 +38,6 @@ private[thriftserver] class SparkSQLOperationManager()
   val handleToOperation = ReflectionUtils
     .getSuperField[JMap[OperationHandle, Operation]](this, "handleToOperation")
 
-  val sessionToActivePool = new ConcurrentHashMap[SessionHandle, String]()
   val sessionToContexts = new ConcurrentHashMap[SessionHandle, SQLContext]()
 
   override def newExecuteStatementOperation(
@@ -51,8 +50,8 @@ private[thriftserver] class SparkSQLOperationManager()
       s" initialized or had already closed.")
     val conf = sqlContext.sessionState.conf
     val runInBackground = async && conf.getConf(HiveUtils.HIVE_THRIFT_SERVER_ASYNC)
-    val operation = new SparkExecuteStatementOperation(parentSession, statement, confOverlay,
-      runInBackground)(sqlContext, sessionToActivePool)
+    val operation = new SparkExecuteStatementOperation(
+      sqlContext, parentSession, statement, confOverlay, runInBackground)
     handleToOperation.put(operation.getHandle, operation)
     logDebug(s"Created Operation for $statement with session=$parentSession, " +
       s"runInBackground=$runInBackground")
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 0cec63460814c..21256ad02c134 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -811,6 +811,61 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
       }
     }
   }
+
+  test("SPARK-31859 Thriftserver works with spark.sql.datetime.java8API.enabled=true") {
+    withJdbcStatement() { statement =>
+      withJdbcStatement() { st =>
+        st.execute("set spark.sql.datetime.java8API.enabled=true")
+        val rs = st.executeQuery("select date '2020-05-28', timestamp '2020-05-28 00:00:00'")
+        rs.next()
+        assert(rs.getDate(1).toString() == "2020-05-28")
+        assert(rs.getTimestamp(2).toString() == "2020-05-28 00:00:00.0")
+      }
+    }
+  }
+
+  test("SPARK-31861 Thriftserver respects spark.sql.session.timeZone") {
+    withJdbcStatement() { statement =>
+      withJdbcStatement() { st =>
+        st.execute("set spark.sql.session.timeZone=+03:15") // different than Thriftserver's JVM tz
+      val rs = st.executeQuery("select timestamp '2020-05-28 10:00:00'")
+        rs.next()
+        // The timestamp as string is the same as the literal
+        assert(rs.getString(1) == "2020-05-28 10:00:00.0")
+        // Parsing it to java.sql.Timestamp in the client will always result in a timestamp
+        // in client default JVM timezone. The string value of the Timestamp will match the literal,
+        // but if the JDBC application cares about the internal timezone and UTC offset of the
+        // Timestamp object, it should set spark.sql.session.timeZone to match its client JVM tz.
+        assert(rs.getTimestamp(1).toString() == "2020-05-28 10:00:00.0")
+      }
+    }
+  }
+
+  test("SPARK-31863 Session conf should persist between Thriftserver worker threads") {
+    val iter = 20
+    withJdbcStatement() { statement =>
+      // date 'now' is resolved during parsing, and relies on SQLConf.get to
+      // obtain the current set timezone. We exploit this to run this test.
+      // If the timezones are set correctly to 25 hours apart across threads,
+      // the dates should reflect this.
+
+      // iterate a few times for the odd chance the same thread is selected
+      for (_ <- 0 until iter) {
+        statement.execute("SET spark.sql.session.timeZone=GMT-12")
+        val firstResult = statement.executeQuery("SELECT date 'now'")
+        firstResult.next()
+        val beyondDateLineWest = firstResult.getDate(1)
+
+        statement.execute("SET spark.sql.session.timeZone=GMT+13")
+        val secondResult = statement.executeQuery("SELECT date 'now'")
+        secondResult.next()
+        val dateLineEast = secondResult.getDate(1)
+        assert(
+          dateLineEast after beyondDateLineWest,
+          "SQLConf changes should persist across execution threads")
+      }
+    }
+  }
 }
 
 class SingleSessionSuite extends HiveThriftJdbcTest {
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index a770bea9c2aa6..462b93a0f09fe 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -123,22 +123,6 @@ private static TColumnValue stringValue(HiveVarchar value) {
     return TColumnValue.stringVal(tStringValue);
   }
 
-  private static TColumnValue dateValue(Date value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return new TColumnValue(TColumnValue.stringVal(tStringValue));
-  }
-
-  private static TColumnValue timestampValue(Timestamp value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
   private static TColumnValue stringValue(HiveIntervalYearMonth value) {
     TStringValue tStrValue = new TStringValue();
     if (value != null) {
@@ -178,9 +162,9 @@ public static TColumnValue toTColumnValue(Type type, Object value) {
     case VARCHAR_TYPE:
       return stringValue((HiveVarchar)value);
     case DATE_TYPE:
-      return dateValue((Date)value);
     case TIMESTAMP_TYPE:
-      return timestampValue((Timestamp)value);
+      // SPARK-31859, SPARK-31861: converted to string already in SparkExecuteStatementOperation
+      return stringValue((String)value);
     case INTERVAL_YEAR_MONTH_TYPE:
       return stringValue((HiveIntervalYearMonth) value);
     case INTERVAL_DAY_TIME_TYPE:
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
index 51bb28748d9e2..4b331423948fa 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/operation/Operation.java
@@ -280,7 +280,10 @@ public void cancel() throws HiveSQLException {
     throw new UnsupportedOperationException("SQLOperation.cancel()");
   }
 
-  public abstract void close() throws HiveSQLException;
+  public void close() throws HiveSQLException {
+    setState(OperationState.CLOSED);
+    cleanupOperationLog();
+  }
 
   public abstract TableSchema getResultSetSchema() throws HiveSQLException;
 
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
index 53f0465a056d8..85adf55df15e0 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/ColumnValue.java
@@ -124,22 +124,6 @@ private static TColumnValue stringValue(HiveVarchar value) {
     return TColumnValue.stringVal(tStringValue);
   }
 
-  private static TColumnValue dateValue(Date value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return new TColumnValue(TColumnValue.stringVal(tStringValue));
-  }
-
-  private static TColumnValue timestampValue(Timestamp value) {
-    TStringValue tStringValue = new TStringValue();
-    if (value != null) {
-      tStringValue.setValue(value.toString());
-    }
-    return TColumnValue.stringVal(tStringValue);
-  }
-
   private static TColumnValue stringValue(HiveIntervalYearMonth value) {
     TStringValue tStrValue = new TStringValue();
     if (value != null) {
@@ -181,9 +165,9 @@ public static TColumnValue toTColumnValue(TypeDescriptor typeDescriptor, Object
     case VARCHAR_TYPE:
       return stringValue((HiveVarchar)value);
     case DATE_TYPE:
-      return dateValue((Date)value);
     case TIMESTAMP_TYPE:
-      return timestampValue((Timestamp)value);
+      // SPARK-31859, SPARK-31861: converted to string already in SparkExecuteStatementOperation
+      return stringValue((String)value);
     case INTERVAL_YEAR_MONTH_TYPE:
       return stringValue((HiveIntervalYearMonth) value);
     case INTERVAL_DAY_TIME_TYPE:
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
index f26c715add987..558c68f85c16b 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/operation/Operation.java
@@ -298,7 +298,10 @@ public void cancel() throws HiveSQLException {
     throw new UnsupportedOperationException("SQLOperation.cancel()");
   }
 
-  public abstract void close() throws HiveSQLException;
+  public void close() throws HiveSQLException {
+    setState(OperationState.CLOSED);
+    cleanupOperationLog();
+  }
 
   public abstract TableSchema getResultSetSchema() throws HiveSQLException;
 

From 51a68d9eac389a82f3137e5d6e60bfb130d172d1 Mon Sep 17 00:00:00 2001
From: Maryann Xue <maryann.xue@gmail.com>
Date: Sat, 30 May 2020 07:47:29 +0000
Subject: [PATCH 0753/1280] [SPARK-31864][SQL] Adjust AQE skew join trigger
 condition

This PR makes a minor change in deciding whether a partition is skewed by comparing the partition size to the median size of coalesced partitions instead of median size of raw partitions before coalescing.

This change is line with target size criteria of splitting skew join partitions and can also cope with situations of extra empty partitions caused by over-partitioning. This PR has also improved skew join tests in AQE tests.

No.

Updated UTs.

Closes #28669 from maryannxue/spark-31864.

Authored-by: Maryann Xue <maryann.xue@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b9737c3c228f465d332e41f1ea0cece2a5f7667e)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   |  2 +-
 .../adaptive/OptimizeSkewedJoin.scala         | 31 ++++-----
 .../adaptive/AdaptiveQueryExecSuite.scala     | 65 ++++++++-----------
 3 files changed, 42 insertions(+), 56 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index d0b55ceb7b20f..b36c01d0cdaf1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -494,7 +494,7 @@ object SQLConf {
       .version("3.0.0")
       .intConf
       .checkValue(_ > 0, "The skew factor must be positive.")
-      .createWithDefault(10)
+      .createWithDefault(5)
 
   val SKEW_JOIN_SKEWED_PARTITION_THRESHOLD =
     buildConf("spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index d94999a7387a4..91ae0b9ed2ae2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
+import org.apache.spark.{MapOutputTrackerMaster, SparkEnv}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
@@ -70,9 +70,9 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
       size > conf.getConf(SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD)
   }
 
-  private def medianSize(stats: MapOutputStatistics): Long = {
-    val numPartitions = stats.bytesByPartitionId.length
-    val bytes = stats.bytesByPartitionId.sorted
+  private def medianSize(sizes: Seq[Long]): Long = {
+    val numPartitions = sizes.length
+    val bytes = sizes.sorted
     numPartitions match {
       case _ if (numPartitions % 2 == 0) =>
         math.max((bytes(numPartitions / 2) + bytes(numPartitions / 2 - 1)) / 2, 1)
@@ -160,16 +160,16 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
         if supportedJoinTypes.contains(joinType) =>
       assert(left.partitionsWithSizes.length == right.partitionsWithSizes.length)
       val numPartitions = left.partitionsWithSizes.length
-      // We use the median size of the original shuffle partitions to detect skewed partitions.
-      val leftMedSize = medianSize(left.mapStats)
-      val rightMedSize = medianSize(right.mapStats)
+      // Use the median size of the actual (coalesced) partition sizes to detect skewed partitions.
+      val leftMedSize = medianSize(left.partitionsWithSizes.map(_._2))
+      val rightMedSize = medianSize(right.partitionsWithSizes.map(_._2))
       logDebug(
         s"""
           |Optimizing skewed join.
           |Left side partitions size info:
-          |${getSizeInfo(leftMedSize, left.mapStats.bytesByPartitionId)}
+          |${getSizeInfo(leftMedSize, left.partitionsWithSizes.map(_._2))}
           |Right side partitions size info:
-          |${getSizeInfo(rightMedSize, right.mapStats.bytesByPartitionId)}
+          |${getSizeInfo(rightMedSize, right.partitionsWithSizes.map(_._2))}
         """.stripMargin)
       val canSplitLeft = canSplitLeftSide(joinType)
       val canSplitRight = canSplitRightSide(joinType)
@@ -287,17 +287,15 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 private object ShuffleStage {
   def unapply(plan: SparkPlan): Option[ShuffleStageInfo] = plan match {
     case s: ShuffleQueryStageExec if s.mapStats.isDefined =>
-      val mapStats = s.mapStats.get
-      val sizes = mapStats.bytesByPartitionId
+      val sizes = s.mapStats.get.bytesByPartitionId
       val partitions = sizes.zipWithIndex.map {
         case (size, i) => CoalescedPartitionSpec(i, i + 1) -> size
       }
-      Some(ShuffleStageInfo(s, mapStats, partitions))
+      Some(ShuffleStageInfo(s, partitions))
 
     case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs, _)
-      if s.mapStats.isDefined =>
-      val mapStats = s.mapStats.get
-      val sizes = mapStats.bytesByPartitionId
+      if s.mapStats.isDefined && partitionSpecs.nonEmpty =>
+      val sizes = s.mapStats.get.bytesByPartitionId
       val partitions = partitionSpecs.map {
         case spec @ CoalescedPartitionSpec(start, end) =>
           var sum = 0L
@@ -310,7 +308,7 @@ private object ShuffleStage {
         case other => throw new IllegalArgumentException(
           s"Expect CoalescedPartitionSpec but got $other")
       }
-      Some(ShuffleStageInfo(s, mapStats, partitions))
+      Some(ShuffleStageInfo(s, partitions))
 
     case _ => None
   }
@@ -318,7 +316,6 @@ private object ShuffleStage {
 
 private case class ShuffleStageInfo(
     shuffleStage: ShuffleQueryStageExec,
-    mapStats: MapOutputStatistics,
     partitionsWithSizes: Seq[(CoalescedPartitionSpec, Long)])
 
 private class SkewDesc {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 37e8e13a961f2..fed7cdcb9eb6a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -25,7 +25,7 @@ import org.apache.log4j.Level
 import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent, SparkListenerJobStart}
 import org.apache.spark.sql.{QueryTest, Row, SparkSession, Strategy}
 import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
-import org.apache.spark.sql.execution.{ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
+import org.apache.spark.sql.execution.{PartialReducerPartitionSpec, ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.command.DataWritingCommandExec
 import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
@@ -616,76 +616,65 @@ class AdaptiveQueryExecSuite
     }
   }
 
-  // TODO: we need a way to customize data distribution after shuffle, to improve test coverage
-  //       of this case.
   test("SPARK-29544: adaptive skew join with different join types") {
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "2000",
-      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "2000") {
+      SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "100",
+      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "800",
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "800") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
-          .selectExpr("id % 2 as key1", "id as value1")
+          .select(
+            when('id < 250, 249)
+              .when('id >= 750, 1000)
+              .otherwise('id).as("key1"),
+            'id as "value1")
           .createOrReplaceTempView("skewData1")
         spark
           .range(0, 1000, 1, 10)
-          .selectExpr("id % 1 as key2", "id as value2")
+          .select(
+            when('id < 250, 249)
+              .otherwise('id).as("key2"),
+            'id as "value2")
           .createOrReplaceTempView("skewData2")
 
-        def checkSkewJoin(joins: Seq[SortMergeJoinExec], expectedNumPartitions: Int): Unit = {
+        def checkSkewJoin(
+            joins: Seq[SortMergeJoinExec],
+            leftSkewNum: Int,
+            rightSkewNum: Int): Unit = {
           assert(joins.size == 1 && joins.head.isSkewJoin)
           assert(joins.head.left.collect {
             case r: CustomShuffleReaderExec => r
-          }.head.partitionSpecs.length == expectedNumPartitions)
+          }.head.partitionSpecs.collect {
+            case p: PartialReducerPartitionSpec => p.reducerIndex
+          }.distinct.length == leftSkewNum)
           assert(joins.head.right.collect {
             case r: CustomShuffleReaderExec => r
-          }.head.partitionSpecs.length == expectedNumPartitions)
+          }.head.partitionSpecs.collect {
+            case p: PartialReducerPartitionSpec => p.reducerIndex
+          }.distinct.length == rightSkewNum)
         }
 
         // skewed inner join optimization
         val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
           "SELECT * FROM skewData1 join skewData2 ON key1 = key2")
-        // left stats: [3496, 0, 0, 0, 4014]
-        // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, left side is divided
-        //              into 2 splits and right side is divided into 4 splits, so
-        //              2 x 4 sub-partitions.
-        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, and divide into 2 splits, so
-        //              2 sub-partitions.
-        // So total (8 + 1 + 3) partitions.
         val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        checkSkewJoin(innerSmj, 8 + 1 + 2)
+        checkSkewJoin(innerSmj, 2, 1)
 
         // skewed left outer join optimization
         val (_, leftAdaptivePlan) = runAdaptiveAndVerifyResult(
           "SELECT * FROM skewData1 left outer join skewData2 ON key1 = key2")
-        // left stats: [3496, 0, 0, 0, 4014]
-        // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, but left join can't split right side,
-        //              so only left side is divided into 2 splits, and thus 2 sub-partitions.
-        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, and divide into 2 splits, so
-        //              2 sub-partitions.
-        // So total (2 + 1 + 2) partitions.
         val leftSmj = findTopLevelSortMergeJoin(leftAdaptivePlan)
-        checkSkewJoin(leftSmj, 2 + 1 + 2)
+        checkSkewJoin(leftSmj, 2, 0)
 
         // skewed right outer join optimization
         val (_, rightAdaptivePlan) = runAdaptiveAndVerifyResult(
           "SELECT * FROM skewData1 right outer join skewData2 ON key1 = key2")
-        // left stats: [3496, 0, 0, 0, 4014]
-        // right stats:[6292, 0, 0, 0, 0]
-        // Partition 0: both left and right sides are skewed, but right join can't split left side,
-        //              so only right side is divided into 4 splits, and thus 4 sub-partitions.
-        // Partition 1, 2, 3: not skewed, and coalesced into 1 partition.
-        // Partition 4: only left side is skewed, but right join can't split left side, so just
-        //              1 partition.
-        // So total (4 + 1 + 1) partitions.
         val rightSmj = findTopLevelSortMergeJoin(rightAdaptivePlan)
-        checkSkewJoin(rightSmj, 4 + 1 + 1)
+        checkSkewJoin(rightSmj, 0, 1)
       }
     }
   }

From 5fa46eb3d50281943a446e6d10fc7c6621c011cd Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 30 May 2020 14:51:45 -0500
Subject: [PATCH 0754/1280] [SPARK-31866][SQL][DOCS] Add
 COALESCE/REPARTITION/REPARTITION_BY_RANGE Hints to SQL Reference

Add Coalesce/Repartition/Repartition_By_Range Hints to SQL Reference

To make SQL reference complete

<img width="1100" alt="Screen Shot 2020-05-29 at 6 46 38 PM" src="https://user-images.githubusercontent.com/13592258/83316782-d6fcf300-a1dc-11ea-87f6-e357b9c739fd.png">

<img width="1099" alt="Screen Shot 2020-05-29 at 6 43 30 PM" src="https://user-images.githubusercontent.com/13592258/83316784-d8c6b680-a1dc-11ea-95ea-10a1f75dcef9.png">

Only the the above pages are changed. The following two pages are the same as before.

<img width="1100" alt="Screen Shot 2020-05-28 at 10 05 27 PM" src="https://user-images.githubusercontent.com/13592258/83223474-bfb3fc00-a12f-11ea-807a-824a618afa0b.png">

<img width="1099" alt="Screen Shot 2020-05-28 at 10 05 08 PM" src="https://user-images.githubusercontent.com/13592258/83223478-c2165600-a12f-11ea-806e-a1e57dc35ef4.png">

Manually build and check

Closes #28672 from huaxingao/coalesce_hint.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 1b780f364bfbb46944fe805a024bb6c32f5d2dde)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/_data/menu-sql.yaml                      |  8 +-
 docs/sql-performance-tuning.md                |  4 +
 docs/sql-ref-syntax-qry-select-hints.md       | 83 +++++++++++++++++--
 docs/sql-ref-syntax-qry-select-join.md        |  2 +-
 ... => sql-ref-syntax-qry-select-sampling.md} |  0
 ...md => sql-ref-syntax-qry-select-window.md} |  0
 docs/sql-ref-syntax-qry-select.md             |  6 +-
 docs/sql-ref-syntax-qry.md                    |  6 +-
 docs/sql-ref-syntax.md                        |  6 +-
 9 files changed, 95 insertions(+), 20 deletions(-)
 rename docs/{sql-ref-syntax-qry-sampling.md => sql-ref-syntax-qry-select-sampling.md} (100%)
 rename docs/{sql-ref-syntax-qry-window.md => sql-ref-syntax-qry-select-window.md} (100%)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 289a9d3a1e9da..219e6809a96f0 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -171,22 +171,22 @@
                   url: sql-ref-syntax-qry-select-limit.html
                 - text: Common Table Expression
                   url: sql-ref-syntax-qry-select-cte.html
+                - text: Hints
+                  url: sql-ref-syntax-qry-select-hints.html
                 - text: Inline Table
                   url: sql-ref-syntax-qry-select-inline-table.html
                 - text: JOIN
                   url: sql-ref-syntax-qry-select-join.html
-                - text: Join Hints
-                  url: sql-ref-syntax-qry-select-hints.html
                 - text: LIKE Predicate
                   url: sql-ref-syntax-qry-select-like.html
                 - text: Set Operators
                   url: sql-ref-syntax-qry-select-setops.html
                 - text: TABLESAMPLE
-                  url: sql-ref-syntax-qry-sampling.html
+                  url: sql-ref-syntax-qry-select-sampling.html
                 - text: Table-valued Function
                   url: sql-ref-syntax-qry-select-tvf.html
                 - text: Window Function
-                  url: sql-ref-syntax-qry-window.html
+                  url: sql-ref-syntax-qry-select-window.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 5b784a56d476c..5e6f049a51e95 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -179,6 +179,8 @@ SELECT /*+ BROADCAST(r) */ * FROM records r JOIN src s ON r.key = s.key
 </div>
 </div>
 
+For more details please refer to the documentation of [Join Hints](sql-ref-syntax-qry-select-hints.html#join-hints).
+
 ## Coalesce Hints for SQL Queries
 
 Coalesce hints allows the Spark SQL users to control the number of output files just like the
@@ -194,6 +196,8 @@ The "REPARTITION_BY_RANGE" hint must have column names and a partition number is
     SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t
     SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t
 
+For more details please refer to the documentation of [Partitioning Hints](sql-ref-syntax-qry-select-hints.html#partitioning-hints).
+
 ## Adaptive Query Execution
 Adaptive Query Execution (AQE) is an optimization technique in Spark SQL that makes use of the runtime statistics to choose the most efficient query execution plan. AQE is disabled by default. Spark SQL can use the umbrella configuration of `spark.sql.adaptive.enabled` to control whether turn it on/off. As of Spark 3.0, there are three major features in AQE, including coalescing post-shuffle partitions, converting sort-merge join to broadcast join, and skew join optimization.
 
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index 4bb48b08d5e3b..247ce48e79445 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -1,7 +1,7 @@
 ---
 layout: global
-title: Join Hints
-displayTitle: Join Hints
+title: Hints
+displayTitle: Hints
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
@@ -21,15 +21,86 @@ license: |
 
 ### Description
 
-Join Hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
+Hints give users a way to suggest how Spark SQL to use specific approaches to generate its execution plan.
 
 ### Syntax
 
 ```sql
-/*+ join_hint [ , ... ] */
+/*+ hint [ , ... ] */
 ```
 
-### Join Hints Types
+### Partitioning Hints
+
+Partitioning hints allow users to suggest a partitioning stragety that Spark should follow. `COALESCE`, `REPARTITION`,
+and `REPARTITION_BY_RANGE` hints are supported and are equivalent to `coalesce`, `repartition`, and
+`repartitionByRange` [Dataset APIs](api/scala/org/apache/spark/sql/Dataset.html), respectively. These hints give users
+a way to tune performance and control the number of output files in Spark SQL. When multiple partitioning hints are
+specified, multiple nodes are inserted into the logical plan, but the leftmost hint is picked by the optimizer.
+
+#### Partitioning Hints Types
+
+* **COALESCE**
+
+  The `COALESCE` hint can be used to reduce the number of partitions to the specified number of partitions. It takes a partition number as a parameter.
+
+* **REPARTITION**
+
+  The `REPARTITION` hint can be used to repartition to the specified number of partitions using the specified partitioning expressions. It takes a partition number, column names, or both as parameters.
+
+* **REPARTITION_BY_RANGE**
+
+  The `REPARTITION_BY_RANGE` hint can be used to repartition to the specified number of partitions using the specified partitioning expressions. It takes column names and an optional partition number as parameters.
+
+#### Examples
+
+```sql
+SELECT /*+ COALESCE(3) */ * FROM t;
+
+SELECT /*+ REPARTITION(3) */ * FROM t;
+
+SELECT /*+ REPARTITION(c) */ * FROM t;
+
+SELECT /*+ REPARTITION(3, c) */ * FROM t;
+
+SELECT /*+ REPARTITION_BY_RANGE(c) */ * FROM t;
+
+SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t;
+
+-- multiple partitioning hints
+EXPLAIN EXTENDED SELECT /*+ REPARTITION(100), COALESCE(500), REPARTITION_BY_RANGE(3, c) */ * FROM t;
+== Parsed Logical Plan ==
+'UnresolvedHint REPARTITION, [100]
++- 'UnresolvedHint COALESCE, [500]
+   +- 'UnresolvedHint REPARTITION_BY_RANGE, [3, 'c]
+      +- 'Project [*]
+         +- 'UnresolvedRelation [t]
+
+== Analyzed Logical Plan ==
+name: string, c: int
+Repartition 100, true
++- Repartition 500, false
+   +- RepartitionByExpression [c#30 ASC NULLS FIRST], 3
+      +- Project [name#29, c#30]
+         +- SubqueryAlias spark_catalog.default.t
+            +- Relation[name#29,c#30] parquet
+
+== Optimized Logical Plan ==
+Repartition 100, true
++- Relation[name#29,c#30] parquet
+
+== Physical Plan ==
+Exchange RoundRobinPartitioning(100), false, [id=#121]
++- *(1) ColumnarToRow
+   +- FileScan parquet default.t[name#29,c#30] Batched: true, DataFilters: [], Format: Parquet,
+      Location: CatalogFileIndex[file:/spark/spark-warehouse/t], PartitionFilters: [],
+      PushedFilters: [], ReadSchema: struct<name:string>
+```
+
+### Join Hints
+
+Join hints allow users to suggest the join strategy that Spark should use. Prior to Spark 3.0, only the `BROADCAST` Join Hint was supported. `MERGE`, `SHUFFLE_HASH` and `SHUFFLE_REPLICATE_NL` Joint Hints support was added in 3.0. When different join strategy hints are specified on both sides of a join, Spark prioritizes hints in the following order: `BROADCAST` over `MERGE` over `SHUFFLE_HASH` over `SHUFFLE_REPLICATE_NL`. When both sides are specified with the `BROADCAST` hint or the `SHUFFLE_HASH` hint, Spark will pick the build side based on the join type and the sizes of the relations. Since a given strategy may not support all join types, Spark is not guaranteed to use the join strategy suggested by the hint.
+
+#### Join Hints Types
 
 * **BROADCAST**
 
@@ -47,7 +118,7 @@ Join Hints allow users to suggest the join strategy that Spark should use. Prior
 
     Suggests that Spark use shuffle-and-replicate nested loop join.
 
-### Examples
+#### Examples
 
 ```sql
 -- Join Hints for broadcast join
diff --git a/docs/sql-ref-syntax-qry-select-join.md b/docs/sql-ref-syntax-qry-select-join.md
index 28b21f5e3f0ff..09b0efd7b5751 100644
--- a/docs/sql-ref-syntax-qry-select-join.md
+++ b/docs/sql-ref-syntax-qry-select-join.md
@@ -235,4 +235,4 @@ SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.dept
 ### Related Statements
 
 * [SELECT](sql-ref-syntax-qry-select.html)
-* [Join Hints](sql-ref-syntax-qry-select-hints.html)
+* [Hints](sql-ref-syntax-qry-select-hints.html)
diff --git a/docs/sql-ref-syntax-qry-sampling.md b/docs/sql-ref-syntax-qry-select-sampling.md
similarity index 100%
rename from docs/sql-ref-syntax-qry-sampling.md
rename to docs/sql-ref-syntax-qry-select-sampling.md
diff --git a/docs/sql-ref-syntax-qry-window.md b/docs/sql-ref-syntax-qry-select-window.md
similarity index 100%
rename from docs/sql-ref-syntax-qry-window.md
rename to docs/sql-ref-syntax-qry-select-window.md
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 1aeecdb982c4c..987e6479ab20a 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -151,11 +151,11 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
 * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+* [Hints](sql-ref-syntax-qry-select-hints.html)
 * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
 * [JOIN](sql-ref-syntax-qry-select-join.html)
-* [Join Hints](sql-ref-syntax-qry-select-hints.html)
 * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
 * [Set Operators](sql-ref-syntax-qry-select-setops.html)
-* [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+* [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
 * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
-* [Window Function](sql-ref-syntax-qry-window.html)
+* [Window Function](sql-ref-syntax-qry-select-window.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 8accdfe30764a..167c394d0fe49 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -37,12 +37,12 @@ ability to generate logical and physical plan for a given query using
   * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
   * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
   * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
+  * [Hints](sql-ref-syntax-qry-select-hints.html)
   * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
   * [JOIN](sql-ref-syntax-qry-select-join.html)
-  * [Join Hints](sql-ref-syntax-qry-select-hints.html)
   * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
   * [Set Operators](sql-ref-syntax-qry-select-setops.html)
-  * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+  * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
   * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
-  * [Window Function](sql-ref-syntax-qry-window.html)
+  * [Window Function](sql-ref-syntax-qry-select-window.html)
 * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 98e30652245f6..d78a01fd655a2 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -54,18 +54,18 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
    * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
    * [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
    * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+   * [Hints](sql-ref-syntax-qry-select-hints.html)
    * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
    * [JOIN](sql-ref-syntax-qry-select-join.html)
-   * [Join Hints](sql-ref-syntax-qry-select-hints.html)
    * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
    * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
    * [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
    * [Set Operators](sql-ref-syntax-qry-select-setops.html)
    * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
-   * [TABLESAMPLE](sql-ref-syntax-qry-sampling.html)
+   * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
    * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
    * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
-   * [Window Function](sql-ref-syntax-qry-window.html)
+   * [Window Function](sql-ref-syntax-qry-select-window.html)
  * [EXPLAIN](sql-ref-syntax-qry-explain.html)
 
 ### Auxiliary Statements

From dc0d12ac3ebef7c11b89f890757ebbbd5adeecfc Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sun, 31 May 2020 12:34:39 +0000
Subject: [PATCH 0755/1280] [SPARK-31867][SQL] Disable year type datetime
 patterns which are longer than 10

As mentioned in https://github.com/apache/spark/pull/28673 and suggested via cloud-fan at https://github.com/apache/spark/pull/28673#discussion_r432817075

In this PR, we disable datetime pattern in the form of `y..y` and `Y..Y` whose lengths are greater than 10 to avoid sort of JDK bug as described below

he new datetime formatter introduces silent data change like,

```sql
spark-sql> select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
NULL
spark-sql> set spark.sql.legacy.timeParserPolicy=legacy;
spark.sql.legacy.timeParserPolicy	legacy
spark-sql> select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
00000001970-01-01
spark-sql>
```

For patterns that support `SignStyle.EXCEEDS_PAD`, e.g. `y..y`(len >=4), when using the `NumberPrinterParser` to format it

```java
switch (signStyle) {
  case EXCEEDS_PAD:
    if (minWidth < 19 && value >= EXCEED_POINTS[minWidth]) {
      buf.append(decimalStyle.getPositiveSign());
    }
    break;

           ....
```
the `minWidth` == `len(y..y)`
the `EXCEED_POINTS` is

```java
/**
         * Array of 10 to the power of n.
         */
        static final long[] EXCEED_POINTS = new long[] {
            0L,
            10L,
            100L,
            1000L,
            10000L,
            100000L,
            1000000L,
            10000000L,
            100000000L,
            1000000000L,
            10000000000L,
        };
```

So when the `len(y..y)` is greater than 10, ` ArrayIndexOutOfBoundsException` will be raised.

 And at the caller side, for `from_unixtime`, the exception will be suppressed and silent data change occurs. for `date_format`, the `ArrayIndexOutOfBoundsException` will continue.

fix silent data change

Yes, SparkUpgradeException will take place of `null` result when the pattern contains 10 or more continuous  'y' or 'Y'

new tests

Closes #28684 from yaooqinn/SPARK-31867-2.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 547c5bf55265772780098ee5e29baa6f095c246b)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              |  2 +-
 .../util/DateTimeFormatterHelper.scala        | 14 ++++++++--
 .../util/DateTimeFormatterHelperSuite.scala   |  3 +-
 .../sql/util/TimestampFormatterSuite.scala    |  5 ++--
 .../resources/sql-tests/inputs/datetime.sql   |  4 +++
 .../sql-tests/results/ansi/datetime.sql.out   | 28 ++++++++++++++++++-
 .../sql-tests/results/datetime-legacy.sql.out | 26 ++++++++++++++++-
 .../sql-tests/results/datetime.sql.out        | 28 ++++++++++++++++++-
 8 files changed, 99 insertions(+), 11 deletions(-)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 48e85b450e6b2..865b9470ed191 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -74,7 +74,7 @@ The count of pattern letters determines the format.
   For formatting, the fraction length would be padded to the number of contiguous 'S' with zeros.
   Spark supports datetime of micro-of-second precision, which has up to 6 significant digits, but can parse nano-of-second with exceeded part truncated.
 
-- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present.
+- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present. 11 or more letters will fail.
 
 - Month: It follows the rule of Number/Text. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. These two forms are different only in some certain languages. For example, in Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters:
   - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 353c074caa75e..5b9d8396530bc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -227,8 +227,16 @@ private object DateTimeFormatterHelper {
     formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
   }
   final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
-  final val unsupportedNarrowTextStyle =
-    Seq("G", "M", "L", "E", "u", "Q", "q").map(_ * 5).toSet
+  final val unsupportedPatternLengths = {
+    // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
+    // 2.4
+    Seq("G", "M", "L", "E", "u", "Q", "q").map(_ * 5) ++
+      // SPARK-31867: Disable year pattern longer than 10 which will cause Java time library throw
+      // unchecked `ArrayIndexOutOfBoundsException` by the `NumberPrinterParser` for formatting. It
+      // makes the call side difficult to handle exceptions and easily leads to silent data change
+      // because of the exceptions being suppressed.
+      Seq("y", "Y").map(_ * 11)
+  }.toSet
 
   /**
    * In Spark 3.0, we switch to the Proleptic Gregorian calendar and use DateTimeFormatter for
@@ -250,7 +258,7 @@ private object DateTimeFormatterHelper {
           for (c <- patternPart if unsupportedLetters.contains(c)) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
           }
-          for (style <- unsupportedNarrowTextStyle if patternPart.contains(style)) {
+          for (style <- unsupportedPatternLengths if patternPart.contains(style)) {
             throw new IllegalArgumentException(s"Too many pattern letters: ${style.head}")
           }
           if (bugInStandAloneForm && (patternPart.contains("LLL") || patternPart.contains("qqq"))) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index 34a1ad2e40e37..f0cc4d149c165 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -40,8 +40,7 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
       val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
       assert(e.getMessage === s"Illegal pattern character: $l")
     }
-
-    unsupportedNarrowTextStyle.foreach { style =>
+    unsupportedPatternLengths.foreach { style =>
       val e1 = intercept[IllegalArgumentException] {
         convertIncompatiblePattern(s"yyyy-MM-dd $style")
       }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 4324d3cff63d7..a72dfb97cb85c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -403,8 +403,9 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       intercept[IllegalArgumentException](TimestampFormatter(pattern, UTC).format(0))
     }
     // supported by the legacy one, then we will suggest users with SparkUpgradeException
-    Seq("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "aa", "aaa").foreach { pattern =>
-      intercept[SparkUpgradeException](TimestampFormatter(pattern, UTC).format(0))
+    Seq("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "aa", "aaa", "y" * 11, "y" * 11)
+      .foreach { pattern =>
+        intercept[SparkUpgradeException](TimestampFormatter(pattern, UTC).format(0))
     }
   }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 99b7cf98f63bc..4eefa0f675202 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -150,3 +150,7 @@ select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampF
 select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
 select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
 select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
+
+select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
+select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss');
+select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index ff30bb04ee5c9..43fe0a652324e 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 109
+-- Number of queries: 112
 
 
 -- !query
@@ -939,3 +939,29 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(1, 'yyyyyyyyyyy-MM-dd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss')
+-- !query schema
+struct<date_format(TIMESTAMP '2018-11-17 13:33:33', yyyyyyyyyy-MM-dd HH:mm:ss):string>
+-- !query output
+0000002018-11-17 13:33:33
+
+
+-- !query
+select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
index 30743f882cff4..71b106485896b 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 109
+-- Number of queries: 112
 
 
 -- !query
@@ -896,3 +896,27 @@ select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy
 struct<from_csv(26/October/2015):struct<date:date>>
 -- !query output
 {"date":2015-10-26}
+
+
+-- !query
+select from_unixtime(1, 'yyyyyyyyyyy-MM-dd')
+-- !query schema
+struct<from_unixtime(CAST(1 AS BIGINT), yyyyyyyyyyy-MM-dd):string>
+-- !query output
+00000001969-12-31
+
+
+-- !query
+select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss')
+-- !query schema
+struct<date_format(TIMESTAMP '2018-11-17 13:33:33', yyyyyyyyyy-MM-dd HH:mm:ss):string>
+-- !query output
+0000002018-11-17 13:33:33
+
+
+-- !query
+select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
+-- !query schema
+struct<date_format(CAST(DATE '2018-11-17' AS TIMESTAMP), yyyyyyyyyyy-MM-dd):string>
+-- !query output
+00000002018-11-17
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index dc466d12a7e3f..9b1c8472fa75e 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 109
+-- Number of queries: 112
 
 
 -- !query
@@ -911,3 +911,29 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select from_unixtime(1, 'yyyyyyyyyyy-MM-dd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss')
+-- !query schema
+struct<date_format(TIMESTAMP '2018-11-17 13:33:33', yyyyyyyyyy-MM-dd HH:mm:ss):string>
+-- !query output
+0000002018-11-17 13:33:33
+
+
+-- !query
+select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html

From a53394fda600279923c7ac7e61c1d763993e680b Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 31 May 2020 13:05:00 +0000
Subject: [PATCH 0756/1280] [SPARK-31874][SQL] Use `FastDateFormat` as the
 legacy fractional formatter

### What changes were proposed in this pull request?
1. Replace `SimpleDateFormat` by `FastDateFormat` as the legacy formatter of `FractionTimestampFormatter`.
2. Optimise `LegacyFastTimestampFormatter` for `java.sql.Timestamp` w/o fractional part.

### Why are the changes needed?
1. By default `HiveResult`.`hiveResultString` retrieves timestamps values as instances of `java.sql.Timestamp`, and uses the legacy parser `SimpleDateFormat` to convert the timestamps to strings. After the fix https://github.com/apache/spark/pull/28024, the fractional formatter and its companion - legacy formatter `SimpleDateFormat` are created per every value. By switching from `LegacySimpleTimestampFormatter` to `LegacyFastTimestampFormatter`, we can utilize the internal cache of `FastDateFormat`, and avoid parsing the default pattern `yyyy-MM-dd HH:mm:ss`.
2. The second change in the method `def format(ts: Timestamp): String` of `LegacyFastTimestampFormatter` is needed to optimize the formatter for patterns without the fractional part and avoid conversions to microseconds.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By existing tests in `TimestampFormatter`.

Closes #28678 from MaxGekk/fastdateformat-as-legacy-frac-formatter.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 47dc332258bec20c460f666de50d9a8c5c0fbc0a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/util/TimestampFormatter.scala       | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 32b4dcda5dea9..6d1c5353b0151 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -121,6 +121,7 @@ class FractionTimestampFormatter(zoneId: ZoneId)
     TimestampFormatter.defaultPattern,
     zoneId,
     TimestampFormatter.defaultLocale,
+    LegacyDateFormats.FAST_DATE_FORMAT,
     needVarLengthSecondFraction = false) {
 
   @transient
@@ -203,7 +204,11 @@ class LegacyFastTimestampFormatter(
   }
 
   override def format(ts: Timestamp): String = {
-    format(fromJavaTimestamp(ts))
+    if (ts.getNanos == 0) {
+      fastDateFormat.format(ts)
+    } else {
+      format(fromJavaTimestamp(ts))
+    }
   }
 
   override def format(instant: Instant): String = {

From bd7f5da3dfa0ce3edda0c9864cd0f89db744277f Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 1 Jun 2020 09:43:03 +0900
Subject: [PATCH 0757/1280] [SPARK-31788][CORE][DSTREAM][PYTHON] Recover the
 support of union for different types of RDD and DStreams

### What changes were proposed in this pull request?

This PR manually specifies the class for the input array being used in `(SparkContext|StreamingContext).union`. It fixes a regression introduced from SPARK-25737.

```python
rdd1 = sc.parallelize([1,2,3,4,5])
rdd2 = sc.parallelize([6,7,8,9,10])
pairRDD1 = rdd1.zip(rdd2)
sc.union([pairRDD1, pairRDD1]).collect()
```

in the current master and `branch-3.0`:

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/context.py", line 870, in union
    jrdds[i] = rdds[i]._jrdd
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_collections.py", line 238, in __setitem__
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_collections.py", line 221, in __set_item
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py", line 332, in get_return_value
py4j.protocol.Py4JError: An error occurred while calling None.None. Trace:
py4j.Py4JException: Cannot convert org.apache.spark.api.java.JavaPairRDD to org.apache.spark.api.java.JavaRDD
	at py4j.commands.ArrayCommand.convertArgument(ArrayCommand.java:166)
	at py4j.commands.ArrayCommand.setArray(ArrayCommand.java:144)
	at py4j.commands.ArrayCommand.execute(ArrayCommand.java:97)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:748)
```

which works in Spark 2.4.5:

```
[(1, 6), (2, 7), (3, 8), (4, 9), (5, 10), (1, 6), (2, 7), (3, 8), (4, 9), (5, 10)]
```

It assumed the class of the input array is the same `JavaRDD` or `JavaDStream`; however, that can be different such as `JavaPairRDD`.

This fix is based on redsanket's initial approach, and will be co-authored.

### Why are the changes needed?

To fix a regression from Spark 2.4.5.

### Does this PR introduce _any_ user-facing change?

No, it's only in unreleased branches. This is to fix a regression.

### How was this patch tested?

Manually tested, and a unittest was added.

Closes #28648 from HyukjinKwon/SPARK-31788.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 29c51d682b3735123f78cf9cb8610522a9bb86fd)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/context.py           | 18 ++++++++++++++++--
 python/pyspark/streaming/context.py | 15 ++++++++++++---
 python/pyspark/tests/test_rdd.py    | 11 +++++++++++
 3 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index d5f1506e4a555..81b6caab7f450 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -25,6 +25,7 @@
 from tempfile import NamedTemporaryFile
 
 from py4j.protocol import Py4JError
+from py4j.java_gateway import is_instance_of
 
 from pyspark import accumulators
 from pyspark.accumulators import Accumulator
@@ -864,8 +865,21 @@ def union(self, rdds):
         first_jrdd_deserializer = rdds[0]._jrdd_deserializer
         if any(x._jrdd_deserializer != first_jrdd_deserializer for x in rdds):
             rdds = [x._reserialize() for x in rdds]
-        cls = SparkContext._jvm.org.apache.spark.api.java.JavaRDD
-        jrdds = SparkContext._gateway.new_array(cls, len(rdds))
+        gw = SparkContext._gateway
+        jvm = SparkContext._jvm
+        jrdd_cls = jvm.org.apache.spark.api.java.JavaRDD
+        jpair_rdd_cls = jvm.org.apache.spark.api.java.JavaPairRDD
+        jdouble_rdd_cls = jvm.org.apache.spark.api.java.JavaDoubleRDD
+        if is_instance_of(gw, rdds[0]._jrdd, jrdd_cls):
+            cls = jrdd_cls
+        elif is_instance_of(gw, rdds[0]._jrdd, jpair_rdd_cls):
+            cls = jpair_rdd_cls
+        elif is_instance_of(gw, rdds[0]._jrdd, jdouble_rdd_cls):
+            cls = jdouble_rdd_cls
+        else:
+            cls_name = rdds[0]._jrdd.getClass().getCanonicalName()
+            raise TypeError("Unsupported Java RDD class %s" % cls_name)
+        jrdds = gw.new_array(cls, len(rdds))
         for i in range(0, len(rdds)):
             jrdds[i] = rdds[i]._jrdd
         return RDD(self._jsc.union(jrdds), self, rdds[0]._jrdd_deserializer)
diff --git a/python/pyspark/streaming/context.py b/python/pyspark/streaming/context.py
index 769121c19ff4d..6199611940dc9 100644
--- a/python/pyspark/streaming/context.py
+++ b/python/pyspark/streaming/context.py
@@ -17,7 +17,7 @@
 
 from __future__ import print_function
 
-from py4j.java_gateway import java_import
+from py4j.java_gateway import java_import, is_instance_of
 
 from pyspark import RDD, SparkConf
 from pyspark.serializers import NoOpSerializer, UTF8Deserializer, CloudPickleSerializer
@@ -341,8 +341,17 @@ def union(self, *dstreams):
             raise ValueError("All DStreams should have same serializer")
         if len(set(s._slideDuration for s in dstreams)) > 1:
             raise ValueError("All DStreams should have same slide duration")
-        cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaDStream
-        jdstreams = SparkContext._gateway.new_array(cls, len(dstreams))
+        jdstream_cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaDStream
+        jpair_dstream_cls = SparkContext._jvm.org.apache.spark.streaming.api.java.JavaPairDStream
+        gw = SparkContext._gateway
+        if is_instance_of(gw, dstreams[0]._jdstream, jdstream_cls):
+            cls = jdstream_cls
+        elif is_instance_of(gw, dstreams[0]._jdstream, jpair_dstream_cls):
+            cls = jpair_dstream_cls
+        else:
+            cls_name = dstreams[0]._jdstream.getClass().getCanonicalName()
+            raise TypeError("Unsupported Java DStream class %s" % cls_name)
+        jdstreams = gw.new_array(cls, len(dstreams))
         for i in range(0, len(dstreams)):
             jdstreams[i] = dstreams[i]._jdstream
         return DStream(self._jssc.union(jdstreams), self, dstreams[0]._jrdd_deserializer)
diff --git a/python/pyspark/tests/test_rdd.py b/python/pyspark/tests/test_rdd.py
index e2d910ccd15c3..cf582201893aa 100644
--- a/python/pyspark/tests/test_rdd.py
+++ b/python/pyspark/tests/test_rdd.py
@@ -166,6 +166,17 @@ def test_zip_chaining(self):
             set([(x, (x, x)) for x in 'abc'])
         )
 
+    def test_union_pair_rdd(self):
+        # SPARK-31788: test if pair RDDs can be combined by union.
+        rdd = self.sc.parallelize([1, 2])
+        pair_rdd = rdd.zip(rdd)
+        unionRDD = self.sc.union([pair_rdd, pair_rdd])
+        self.assertEqual(
+            set(unionRDD.collect()),
+            set([(1, 1), (2, 2), (1, 1), (2, 2)])
+        )
+        self.assertEqual(unionRDD.count(), 4)
+
     def test_deleting_input_files(self):
         # Regression test for SPARK-1025
         tempFile = tempfile.NamedTemporaryFile(delete=False)

From a5a8ec2ca8e75207b4e6c7b76ab6214be4a4237e Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 1 Jun 2020 09:45:21 +0900
Subject: [PATCH 0758/1280] [SPARK-31849][PYTHON][SQL] Make PySpark SQL
 exceptions more Pythonic

### What changes were proposed in this pull request?

This PR proposes to make PySpark exception more Pythonic by hiding JVM stacktrace by default. It can be enabled by turning on `spark.sql.pyspark.jvmStacktrace.enabled` configuration.

```
Traceback (most recent call last):
  ...
pyspark.sql.utils.PythonException:
  An exception was thrown from Python worker in the executor. The below is the Python worker stacktrace.
Traceback (most recent call last):
  ...
```

If this `spark.sql.pyspark.jvmStacktrace.enabled` is enabled, it appends:

```
JVM stacktrace:
org.apache.spark.Exception: ...
  ...
```

For example, the codes below:

```python
from pyspark.sql.functions import udf
udf
def divide_by_zero(v):
    raise v / 0

spark.range(1).select(divide_by_zero("id")).show()
```

will show an error messages that looks like Python exception thrown from the local.

<details>
<summary>Python exception message when <code>spark.sql.pyspark.jvmStacktrace.enabled</code> is off (default)</summary>

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/dataframe.py", line 427, in show
    print(self._jdf.showString(n, 20, vertical))
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/.../spark/python/pyspark/sql/utils.py", line 131, in deco
    raise_from(converted)
  File "<string>", line 3, in raise_from
pyspark.sql.utils.PythonException:
  An exception was thrown from Python worker in the executor. The below is the Python worker stacktrace.
Traceback (most recent call last):
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 605, in main
    process()
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 597, in process
    serializer.dump_stream(out_iter, outfile)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
    self.serializer.dump_stream(self._batched(iterator), stream)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
    for obj in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 212, in _batched
    for item in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in mapper
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
    return lambda *a: f(*a)
  File "/.../spark/python/lib/pyspark.zip/pyspark/util.py", line 107, in wrapper
    return f(*args, **kwargs)
  File "<stdin>", line 3, in divide_by_zero
ZeroDivisionError: division by zero
```

</details>

<details>
<summary>Python exception message when <code>spark.sql.pyspark.jvmStacktrace.enabled</code> is on</summary>

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/dataframe.py", line 427, in show
    print(self._jdf.showString(n, 20, vertical))
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/.../spark/python/pyspark/sql/utils.py", line 137, in deco
    raise_from(converted)
  File "<string>", line 3, in raise_from
pyspark.sql.utils.PythonException:
  An exception was thrown from Python worker in the executor. The below is the Python worker stacktrace.
Traceback (most recent call last):
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 605, in main
    process()
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 597, in process
    serializer.dump_stream(out_iter, outfile)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
    self.serializer.dump_stream(self._batched(iterator), stream)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
    for obj in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 212, in _batched
    for item in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in mapper
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
    return lambda *a: f(*a)
  File "/.../spark/python/lib/pyspark.zip/pyspark/util.py", line 107, in wrapper
    return f(*args, **kwargs)
  File "<stdin>", line 3, in divide_by_zero
ZeroDivisionError: division by zero

JVM stacktrace:
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 4 times, most recent failure: Lost task 0.3 in stage 1.0 (TID 4, 192.168.35.193, executor 0): org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 605, in main
    process()
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 597, in process
    serializer.dump_stream(out_iter, outfile)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
    self.serializer.dump_stream(self._batched(iterator), stream)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
    for obj in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 212, in _batched
    for item in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in mapper
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
    return lambda *a: f(*a)
  File "/.../spark/python/lib/pyspark.zip/pyspark/util.py", line 107, in wrapper
    return f(*args, **kwargs)
  File "<stdin>", line 3, in divide_by_zero
ZeroDivisionError: division by zero

	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:516)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:81)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:64)
	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:469)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:753)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:340)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:127)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:469)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:472)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2117)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2066)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2065)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2065)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1021)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1021)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1021)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2297)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2246)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2235)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:823)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2108)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2129)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2148)
	at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:467)
	at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:420)
	at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:47)
	at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3653)
	at org.apache.spark.sql.Dataset.$anonfun$head$1(Dataset.scala:2695)
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3644)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3642)
	at org.apache.spark.sql.Dataset.head(Dataset.scala:2695)
	at org.apache.spark.sql.Dataset.take(Dataset.scala:2902)
	at org.apache.spark.sql.Dataset.getRows(Dataset.scala:300)
	at org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 605, in main
    process()
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 597, in process
    serializer.dump_stream(out_iter, outfile)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
    self.serializer.dump_stream(self._batched(iterator), stream)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
    for obj in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 212, in _batched
    for item in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in mapper
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
    return lambda *a: f(*a)
  File "/.../spark/python/lib/pyspark.zip/pyspark/util.py", line 107, in wrapper
    return f(*args, **kwargs)
  File "<stdin>", line 3, in divide_by_zero
ZeroDivisionError: division by zero

	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:516)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:81)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:64)
	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:469)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:753)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:340)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:127)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:469)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:472)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
```

</details>

<details>
<summary>Python exception message without this change</summary>

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/dataframe.py", line 427, in show
    print(self._jdf.showString(n, 20, vertical))
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/.../spark/python/pyspark/sql/utils.py", line 98, in deco
    return f(*a, **kw)
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py", line 328, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling o160.showString.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 10 in stage 5.0 failed 4 times, most recent failure: Lost task 10.3 in stage 5.0 (TID 37, 192.168.35.193, executor 3): org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 605, in main
    process()
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 597, in process
    serializer.dump_stream(out_iter, outfile)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
    self.serializer.dump_stream(self._batched(iterator), stream)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
    for obj in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 212, in _batched
    for item in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in mapper
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
    return lambda *a: f(*a)
  File "/.../spark/python/lib/pyspark.zip/pyspark/util.py", line 107, in wrapper
    return f(*args, **kwargs)
  File "<stdin>", line 3, in divide_by_zero
ZeroDivisionError: division by zero

	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:516)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:81)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:64)
	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:469)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:753)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:340)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:127)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:469)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:472)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2117)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2066)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(DAGScheduler.scala:2065)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2065)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(DAGScheduler.scala:1021)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(DAGScheduler.scala:1021)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1021)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2297)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2246)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2235)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:823)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2108)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2129)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2148)
	at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:467)
	at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:420)
	at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:47)
	at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3653)
	at org.apache.spark.sql.Dataset.$anonfun$head$1(Dataset.scala:2695)
	at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3644)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103)
	at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163)
	at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
	at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
	at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3642)
	at org.apache.spark.sql.Dataset.head(Dataset.scala:2695)
	at org.apache.spark.sql.Dataset.take(Dataset.scala:2902)
	at org.apache.spark.sql.Dataset.getRows(Dataset.scala:300)
	at org.apache.spark.sql.Dataset.showString(Dataset.scala:337)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 605, in main
    process()
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 597, in process
    serializer.dump_stream(out_iter, outfile)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
    self.serializer.dump_stream(self._batched(iterator), stream)
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
    for obj in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/serializers.py", line 212, in _batched
    for item in iterator:
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in mapper
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
  File "/.../spark/python/lib/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
    return lambda *a: f(*a)
  File "/.../spark/python/lib/pyspark.zip/pyspark/util.py", line 107, in wrapper
    return f(*args, **kwargs)
  File "<stdin>", line 3, in divide_by_zero
ZeroDivisionError: division by zero

	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:516)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:81)
	at org.apache.spark.sql.execution.python.PythonUDFRunner$$anon$2.read(PythonUDFRunner.scala:64)
	at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:469)
	at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
	at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:489)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:458)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:753)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:340)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:127)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:469)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:472)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
```

</details>

<br/>

Another example with Python 3.7:

```python
sql("a")
```

<details>
<summary>Python exception message when <code>spark.sql.pyspark.jvmStacktrace.enabled</code> is off (default)</summary>

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/session.py", line 646, in sql
    return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/.../spark/python/pyspark/sql/utils.py", line 131, in deco
    raise_from(converted)
  File "<string>", line 3, in raise_from
pyspark.sql.utils.ParseException:
mismatched input 'a' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
a
^^^
```

</details>

<details>
<summary>Python exception message when <code>spark.sql.pyspark.jvmStacktrace.enabled</code> is on</summary>

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/session.py", line 646, in sql
    return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/.../spark/python/pyspark/sql/utils.py", line 131, in deco
    raise_from(converted)
  File "<string>", line 3, in raise_from
pyspark.sql.utils.ParseException:
mismatched input 'a' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
a
^^^

JVM stacktrace:
org.apache.spark.sql.catalyst.parser.ParseException:
mismatched input 'a' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
a
^^^

	at org.apache.spark.sql.catalyst.parser.ParseException.withCommand(ParseDriver.scala:266)
	at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parse(ParseDriver.scala:133)
	at org.apache.spark.sql.execution.SparkSqlParser.parse(SparkSqlParser.scala:49)
	at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parsePlan(ParseDriver.scala:81)
	at org.apache.spark.sql.SparkSession.$anonfun$sql$2(SparkSession.scala:604)
	at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
	at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:604)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
	at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:601)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:748)
```

</details>

<details>
<summary>Python exception message without this change</summary>

```
Traceback (most recent call last):
  File "/.../spark/python/pyspark/sql/utils.py", line 98, in deco
    return f(*a, **kw)
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py", line 328, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling o26.sql.
: org.apache.spark.sql.catalyst.parser.ParseException:
mismatched input 'a' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
a
^^^

	at org.apache.spark.sql.catalyst.parser.ParseException.withCommand(ParseDriver.scala:266)
	at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parse(ParseDriver.scala:133)
	at org.apache.spark.sql.execution.SparkSqlParser.parse(SparkSqlParser.scala:49)
	at org.apache.spark.sql.catalyst.parser.AbstractSqlParser.parsePlan(ParseDriver.scala:81)
	at org.apache.spark.sql.SparkSession.$anonfun$sql$2(SparkSession.scala:604)
	at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
	at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:604)
	at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
	at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:601)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:748)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/session.py", line 646, in sql
    return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)
  File "/.../spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/.../spark/python/pyspark/sql/utils.py", line 102, in deco
    raise converted
pyspark.sql.utils.ParseException:
mismatched input 'a' expecting {'(', 'ADD', 'ALTER', 'ANALYZE', 'CACHE', 'CLEAR', 'COMMENT', 'COMMIT', 'CREATE', 'DELETE', 'DESC', 'DESCRIBE', 'DFS', 'DROP', 'EXPLAIN', 'EXPORT', 'FROM', 'GRANT', 'IMPORT', 'INSERT', 'LIST', 'LOAD', 'LOCK', 'MAP', 'MERGE', 'MSCK', 'REDUCE', 'REFRESH', 'REPLACE', 'RESET', 'REVOKE', 'ROLLBACK', 'SELECT', 'SET', 'SHOW', 'START', 'TABLE', 'TRUNCATE', 'UNCACHE', 'UNLOCK', 'UPDATE', 'USE', 'VALUES', 'WITH'}(line 1, pos 0)

== SQL ==
a
^^^
```

</details>

### Why are the changes needed?

Currently, PySpark exceptions are very unfriendly to Python users with causing a bunch of JVM stacktrace. See "Python exception message without this change" above.

### Does this PR introduce _any_ user-facing change?

Yes, it will change the exception message. See the examples above.

### How was this patch tested?

Manually tested by

```bash
./bin/pyspark --conf spark.sql.pyspark.jvmStacktrace.enabled=true
```

and running the examples above.

Closes #28661 from HyukjinKwon/python-debug.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e69466056fb2c121b7bbb6ad082f09deb1c41063)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/tests/test_pandas_udf.py   | 14 +++--
 python/pyspark/sql/utils.py                   | 53 +++++++++++++++----
 .../apache/spark/sql/internal/SQLConf.scala   | 11 ++++
 3 files changed, 61 insertions(+), 17 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_udf.py b/python/pyspark/sql/tests/test_pandas_udf.py
index 4218f5cfc401f..7fa65f0e792b9 100644
--- a/python/pyspark/sql/tests/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/test_pandas_udf.py
@@ -19,14 +19,12 @@
 
 from pyspark.sql.functions import udf, pandas_udf, PandasUDFType
 from pyspark.sql.types import *
-from pyspark.sql.utils import ParseException
+from pyspark.sql.utils import ParseException, PythonException
 from pyspark.rdd import PythonEvalType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
 
-from py4j.protocol import Py4JJavaError
-
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,
@@ -157,14 +155,14 @@ def foofoo(x, y):
 
         # plain udf (test for SPARK-23754)
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.withColumn('v', udf(foo)('id')).collect
         )
 
         # pandas scalar udf
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.withColumn(
                 'v', pandas_udf(foo, 'double', PandasUDFType.SCALAR)('id')
@@ -173,7 +171,7 @@ def foofoo(x, y):
 
         # pandas grouped map
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.groupBy('id').apply(
                 pandas_udf(foo, df.schema, PandasUDFType.GROUPED_MAP)
@@ -181,7 +179,7 @@ def foofoo(x, y):
         )
 
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.groupBy('id').apply(
                 pandas_udf(foofoo, df.schema, PandasUDFType.GROUPED_MAP)
@@ -190,7 +188,7 @@ def foofoo(x, y):
 
         # pandas grouped agg
         self.assertRaisesRegexp(
-            Py4JJavaError,
+            PythonException,
             exc_message,
             df.groupBy('id').agg(
                 pandas_udf(foo, 'double', PandasUDFType.GROUPED_AGG)('id')
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 147ac3325efd9..27adc2372ec0f 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -18,8 +18,19 @@
 import py4j
 import sys
 
+from pyspark import SparkContext
+
 if sys.version_info.major >= 3:
     unicode = str
+    # Disable exception chaining (PEP 3134) in captured exceptions
+    # in order to hide JVM stacktace.
+    exec("""
+def raise_from(e):
+    raise e from None
+""")
+else:
+    def raise_from(e):
+        raise e
 
 
 class CapturedException(Exception):
@@ -29,7 +40,11 @@ def __init__(self, desc, stackTrace, cause=None):
         self.cause = convert_exception(cause) if cause is not None else None
 
     def __str__(self):
+        sql_conf = SparkContext._jvm.org.apache.spark.sql.internal.SQLConf.get()
+        debug_enabled = sql_conf.pysparkJVMStacktraceEnabled()
         desc = self.desc
+        if debug_enabled:
+            desc = desc + "\nJVM stacktrace:\n%s" % self.stackTrace
         # encode unicode instance for python2 for human readable description
         if sys.version_info.major < 3 and isinstance(desc, unicode):
             return str(desc.encode('utf-8'))
@@ -67,6 +82,12 @@ class QueryExecutionException(CapturedException):
     """
 
 
+class PythonException(CapturedException):
+    """
+    Exceptions thrown from Python workers.
+    """
+
+
 class UnknownException(CapturedException):
     """
     None of the above exceptions.
@@ -75,21 +96,33 @@ class UnknownException(CapturedException):
 
 def convert_exception(e):
     s = e.toString()
-    stackTrace = '\n\t at '.join(map(lambda x: x.toString(), e.getStackTrace()))
     c = e.getCause()
+
+    jvm = SparkContext._jvm
+    jwriter = jvm.java.io.StringWriter()
+    e.printStackTrace(jvm.java.io.PrintWriter(jwriter))
+    stacktrace = jwriter.toString()
     if s.startswith('org.apache.spark.sql.AnalysisException: '):
-        return AnalysisException(s.split(': ', 1)[1], stackTrace, c)
+        return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.analysis'):
-        return AnalysisException(s.split(': ', 1)[1], stackTrace, c)
+        return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.parser.ParseException: '):
-        return ParseException(s.split(': ', 1)[1], stackTrace, c)
+        return ParseException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.streaming.StreamingQueryException: '):
-        return StreamingQueryException(s.split(': ', 1)[1], stackTrace, c)
+        return StreamingQueryException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.execution.QueryExecutionException: '):
-        return QueryExecutionException(s.split(': ', 1)[1], stackTrace, c)
+        return QueryExecutionException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('java.lang.IllegalArgumentException: '):
-        return IllegalArgumentException(s.split(': ', 1)[1], stackTrace, c)
-    return UnknownException(s, stackTrace, c)
+        return IllegalArgumentException(s.split(': ', 1)[1], stacktrace, c)
+    if c is not None and (
+            c.toString().startswith('org.apache.spark.api.python.PythonException: ')
+            # To make sure this only catches Python UDFs.
+            and any(map(lambda v: "org.apache.spark.sql.execution.python" in v.toString(),
+                        c.getStackTrace()))):
+        msg = ("\n  An exception was thrown from Python worker in the executor. "
+               "The below is the Python worker stacktrace.\n%s" % c.getMessage())
+        return PythonException(msg, stacktrace)
+    return UnknownException(s, stacktrace, c)
 
 
 def capture_sql_exception(f):
@@ -99,7 +132,9 @@ def deco(*a, **kw):
         except py4j.protocol.Py4JJavaError as e:
             converted = convert_exception(e.java_exception)
             if not isinstance(converted, UnknownException):
-                raise converted
+                # Hide where the exception came from that shows a non-Pythonic
+                # JVM exception message.
+                raise_from(converted)
             else:
                 raise
     return deco
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b36c01d0cdaf1..b8b0f3250fad8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1783,6 +1783,15 @@ object SQLConf {
       .version("3.0.0")
       .fallbackConf(ARROW_EXECUTION_ENABLED)
 
+  val PYSPARK_JVM_STACKTRACE_ENABLED =
+    buildConf("spark.sql.pyspark.jvmStacktrace.enabled")
+      .doc("When true, it shows the JVM stacktrace in the user-facing PySpark exception " +
+        "together with Python stacktrace. By default, it is disabled and hides JVM stacktrace " +
+        "and shows a Python-friendly exception only.")
+      .version("3.0.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val ARROW_SPARKR_EXECUTION_ENABLED =
     buildConf("spark.sql.execution.arrow.sparkr.enabled")
       .doc("When true, make use of Apache Arrow for columnar data transfers in SparkR. " +
@@ -3042,6 +3051,8 @@ class SQLConf extends Serializable with Logging {
 
   def arrowPySparkEnabled: Boolean = getConf(ARROW_PYSPARK_EXECUTION_ENABLED)
 
+  def pysparkJVMStacktraceEnabled: Boolean = getConf(PYSPARK_JVM_STACKTRACE_ENABLED)
+
   def arrowSparkREnabled: Boolean = getConf(ARROW_SPARKR_EXECUTION_ENABLED)
 
   def arrowPySparkFallbackEnabled: Boolean = getConf(ARROW_PYSPARK_FALLBACK_ENABLED)

From efa0269080cb7f6e2591caedcdac554beaf2661b Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Mon, 1 Jun 2020 04:50:00 +0000
Subject: [PATCH 0759/1280] [SPARK-31854][SQL] Invoke in MapElementsExec should
 not propagate null

This PR intends to fix a bug of `Dataset.map` below when the whole-stage codegen enabled;
```
scala> val ds = Seq(1.asInstanceOf[Integer], null.asInstanceOf[Integer]).toDS()

scala> sql("SET spark.sql.codegen.wholeStage=true")

scala> ds.map(v=>(v,v)).explain
== Physical Plan ==
*(1) SerializeFromObject [assertnotnull(input[0, scala.Tuple2, true])._1.intValue AS _1#69, assertnotnull(input[0, scala.Tuple2, true])._2.intValue AS _2#70]
+- *(1) MapElements <function1>, obj#68: scala.Tuple2
   +- *(1) DeserializeToObject staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, value#1, true, false), obj#67: java.lang.Integer
      +- LocalTableScan [value#1]

// `AssertNotNull` in `SerializeFromObject` will fail;
scala> ds.map(v => (v, v)).show()
java.lang.NullPointerException: Null value appeared in non-nullable fails:
top level Product input object
If the schema is inferred from a Scala tuple/case class, or a Java bean, please try to use scala.Option[_] or other nullable types (e.g. java.lang.Integer instead of int/scala.Int).

// When the whole-stage codegen disabled, the query works well;
scala> sql("SET spark.sql.codegen.wholeStage=false")
scala> ds.map(v=>(v,v)).show()
+----+----+
|  _1|  _2|
+----+----+
|   1|   1|
|null|null|
+----+----+
```
A root cause is that `Invoke` used in `MapElementsExec` propagates input null, and then [AssertNotNull](https://github.com/apache/spark/blob/1b780f364bfbb46944fe805a024bb6c32f5d2dde/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoder.scala#L253-L255) in `SerializeFromObject` fails because a top-level row becomes null. So, `MapElementsExec` should not return `null` but `(null, null)`.

NOTE: the generated code of the query above in the current master;
```
/* 033 */   private void mapelements_doConsume_0(java.lang.Integer mapelements_expr_0_0, boolean mapelements_exprIsNull_0_0) throws java.io.IOException {
/* 034 */     boolean mapelements_isNull_1 = true;
/* 035 */     scala.Tuple2 mapelements_value_1 = null;
/* 036 */     if (!false) {
/* 037 */       mapelements_resultIsNull_0 = false;
/* 038 */
/* 039 */       if (!mapelements_resultIsNull_0) {
/* 040 */         mapelements_resultIsNull_0 = mapelements_exprIsNull_0_0;
/* 041 */         mapelements_mutableStateArray_0[0] = mapelements_expr_0_0;
/* 042 */       }
/* 043 */
/* 044 */       mapelements_isNull_1 = mapelements_resultIsNull_0;
/* 045 */       if (!mapelements_isNull_1) {
/* 046 */         Object mapelements_funcResult_0 = null;
/* 047 */         mapelements_funcResult_0 = ((scala.Function1) references[1] /* literal */).apply(mapelements_mutableStateArray_0[0]);
/* 048 */
/* 049 */         if (mapelements_funcResult_0 != null) {
/* 050 */           mapelements_value_1 = (scala.Tuple2) mapelements_funcResult_0;
/* 051 */         } else {
/* 052 */           mapelements_isNull_1 = true;
/* 053 */         }
/* 054 */
/* 055 */       }
/* 056 */     }
/* 057 */
/* 058 */     serializefromobject_doConsume_0(mapelements_value_1, mapelements_isNull_1);
/* 059 */
/* 060 */   }
```

The generated code w/ this fix;
```
/* 032 */   private void mapelements_doConsume_0(java.lang.Integer mapelements_expr_0_0, boolean mapelements_exprIsNull_0_0) throws java.io.IOException {
/* 033 */     boolean mapelements_isNull_1 = true;
/* 034 */     scala.Tuple2 mapelements_value_1 = null;
/* 035 */     if (!false) {
/* 036 */       mapelements_mutableStateArray_0[0] = mapelements_expr_0_0;
/* 037 */
/* 038 */       mapelements_isNull_1 = false;
/* 039 */       if (!mapelements_isNull_1) {
/* 040 */         Object mapelements_funcResult_0 = null;
/* 041 */         mapelements_funcResult_0 = ((scala.Function1) references[1] /* literal */).apply(mapelements_mutableStateArray_0[0]);
/* 042 */
/* 043 */         if (mapelements_funcResult_0 != null) {
/* 044 */           mapelements_value_1 = (scala.Tuple2) mapelements_funcResult_0;
/* 045 */           mapelements_isNull_1 = false;
/* 046 */         } else {
/* 047 */           mapelements_isNull_1 = true;
/* 048 */         }
/* 049 */
/* 050 */       }
/* 051 */     }
/* 052 */
/* 053 */     serializefromobject_doConsume_0(mapelements_value_1, mapelements_isNull_1);
/* 054 */
/* 055 */   }
```

Bugfix.

No.

Added tests.

Closes #28681 from maropu/SPARK-31854.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b806fc458265578fddf544363b60fb5e122439b5)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/sql/execution/objects.scala |  4 ++--
 .../test/scala/org/apache/spark/sql/DatasetSuite.scala | 10 ++++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
index d05113431df41..4b2d4195ee906 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
@@ -276,12 +276,12 @@ case class MapElementsExec(
   }
 
   override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
-    val (funcClass, methodName) = func match {
+    val (funcClass, funcName) = func match {
       case m: MapFunction[_, _] => classOf[MapFunction[_, _]] -> "call"
       case _ => FunctionUtils.getFunctionOneName(outputObjectType, child.output(0).dataType)
     }
     val funcObj = Literal.create(func, ObjectType(funcClass))
-    val callFunc = Invoke(funcObj, methodName, outputObjectType, child.output)
+    val callFunc = Invoke(funcObj, funcName, outputObjectType, child.output, propagateNull = false)
 
     val result = BindReferences.bindReference(callFunc, child.output).genCode(ctx)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 4e4558b6f3e2a..a1e813227c9c0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -1901,6 +1901,16 @@ class DatasetSuite extends QueryTest
 
     assert(active eq SparkSession.getActiveSession.get)
   }
+
+  test("SPARK-31854: Invoke in MapElementsExec should not propagate null") {
+    Seq("true", "false").foreach { wholeStage =>
+      withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> wholeStage) {
+        val ds = Seq(1.asInstanceOf[Integer], null.asInstanceOf[Integer]).toDS()
+        val expectedAnswer = Seq[(Integer, Integer)]((1, 1), (null, null))
+        checkDataset(ds.map(v => (v, v)), expectedAnswer: _*)
+      }
+    }
+  }
 }
 
 object AssertExecutionId {

From 805884ad10aca3a532314284b0f2c007d4ca1045 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 1 Jun 2020 15:13:44 +0000
Subject: [PATCH 0760/1280] [SPARK-31885][SQL] Fix filter push down for old
 millis timestamps to Parquet

### What changes were proposed in this pull request?
Fixed conversions of `java.sql.Timestamp` to milliseconds in `ParquetFilter` by using existing functions from `DateTimeUtils` `fromJavaTimestamp()` and `microsToMillis()`.

### Why are the changes needed?
The changes fix the bug:
```scala
scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MILLIS")
scala> spark.conf.set("spark.sql.legacy.parquet.datetimeRebaseModeInWrite", "CORRECTED")
scala> Seq(java.sql.Timestamp.valueOf("1000-06-14 08:28:53.123")).toDF("ts").write.mode("overwrite").parquet("/Users/maximgekk/tmp/ts_millis_old_filter")
scala> spark.read.parquet("/Users/maximgekk/tmp/ts_millis_old_filter").filter($"ts" === "1000-06-14 08:28:53.123").show(false)
+---+
|ts |
+---+
+---+
```

### Does this PR introduce _any_ user-facing change?
Yes, after the changes (for the example above):
```scala
scala> spark.read.parquet("/Users/maximgekk/tmp/ts_millis_old_filter").filter($"ts" === "1000-06-14 08:28:53.123").show(false)
+-----------------------+
|ts                     |
+-----------------------+
|1000-06-14 08:28:53.123|
+-----------------------+
```

### How was this patch tested?
Modified tests in `ParquetFilterSuite` to check old timestamps.

Closes #28693 from MaxGekk/parquet-ts-millis-filter.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9c0dc28a6c33361a412672d63f3b974b75944965)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/parquet/ParquetFilters.scala  | 27 +++++++++----------
 .../parquet/ParquetFilterSuite.scala          | 16 ++++++-----
 .../datasources/parquet/ParquetTest.scala     |  4 ++-
 3 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index d89186af8c8e5..7900693a84820 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -148,6 +148,13 @@ class ParquetFilters(
     Binary.fromConstantByteArray(fixedLengthBytes, 0, numBytes)
   }
 
+  private def timestampToMillis(v: Any): JLong = {
+    val timestamp = v.asInstanceOf[Timestamp]
+    val micros = DateTimeUtils.fromJavaTimestamp(timestamp)
+    val millis = DateTimeUtils.microsToMillis(micros)
+    millis.asInstanceOf[JLong]
+  }
+
   private val makeEq:
     PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
@@ -184,7 +191,7 @@ class ParquetFilters(
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.eq(
@@ -235,7 +242,7 @@ class ParquetFilters(
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.notEq(
@@ -277,9 +284,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.lt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -318,9 +323,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -359,9 +362,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -400,9 +401,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 7b33ceff32e01..c4cf5116c2030 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -589,19 +589,21 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("filter pushdown - timestamp") {
     // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
-    val millisData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123"))
+    val millisData = Seq(
+      Timestamp.valueOf("1000-06-14 08:28:53.123"),
+      Timestamp.valueOf("1582-06-15 08:28:53.001"),
+      Timestamp.valueOf("1900-06-16 08:28:53.0"),
+      Timestamp.valueOf("2018-06-17 08:28:53.999"))
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
       testTimestampPushdown(millisData)
     }
 
     // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
-    val microsData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123456"),
+    val microsData = Seq(
+      Timestamp.valueOf("1000-06-14 08:28:53.123456"),
+      Timestamp.valueOf("1582-06-15 08:28:53.123456"),
+      Timestamp.valueOf("1900-06-16 08:28:53.123456"),
       Timestamp.valueOf("2018-06-17 08:28:53.123456"))
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index f5726973d0abf..105f025adc0ad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -69,7 +69,9 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   protected def withParquetDataFrame(df: DataFrame, testVectorized: Boolean = true)
       (f: DataFrame => Unit): Unit = {
     withTempPath { file =>
-      df.write.format(dataSourceName).save(file.getCanonicalPath)
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
+        df.write.format(dataSourceName).save(file.getCanonicalPath)
+      }
       readFile(file.getCanonicalPath, testVectorized)(f)
     }
   }

From 76a041804d9ba856c221e944b9898c8c62854474 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Mon, 1 Jun 2020 10:20:17 -0700
Subject: [PATCH 0761/1280] Revert "[SPARK-31885][SQL] Fix filter push down for
 old millis timestamps to Parquet"

This reverts commit 805884ad10aca3a532314284b0f2c007d4ca1045.
---
 .../datasources/parquet/ParquetFilters.scala  | 27 ++++++++++---------
 .../parquet/ParquetFilterSuite.scala          | 16 +++++------
 .../datasources/parquet/ParquetTest.scala     |  4 +--
 3 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index 7900693a84820..d89186af8c8e5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -148,13 +148,6 @@ class ParquetFilters(
     Binary.fromConstantByteArray(fixedLengthBytes, 0, numBytes)
   }
 
-  private def timestampToMillis(v: Any): JLong = {
-    val timestamp = v.asInstanceOf[Timestamp]
-    val micros = DateTimeUtils.fromJavaTimestamp(timestamp)
-    val millis = DateTimeUtils.microsToMillis(micros)
-    millis.asInstanceOf[JLong]
-  }
-
   private val makeEq:
     PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
@@ -191,7 +184,7 @@ class ParquetFilters(
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(timestampToMillis).orNull)
+        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.eq(
@@ -242,7 +235,7 @@ class ParquetFilters(
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(timestampToMillis).orNull)
+        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.notEq(
@@ -284,7 +277,9 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMillis(v))
+      (n: Array[String], v: Any) => FilterApi.lt(
+        longColumn(n),
+        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -323,7 +318,9 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMillis(v))
+      (n: Array[String], v: Any) => FilterApi.ltEq(
+        longColumn(n),
+        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -362,7 +359,9 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMillis(v))
+      (n: Array[String], v: Any) => FilterApi.gt(
+        longColumn(n),
+        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -401,7 +400,9 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMillis(v))
+      (n: Array[String], v: Any) => FilterApi.gtEq(
+        longColumn(n),
+        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index c4cf5116c2030..7b33ceff32e01 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -589,21 +589,19 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("filter pushdown - timestamp") {
     // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
-    val millisData = Seq(
-      Timestamp.valueOf("1000-06-14 08:28:53.123"),
-      Timestamp.valueOf("1582-06-15 08:28:53.001"),
-      Timestamp.valueOf("1900-06-16 08:28:53.0"),
-      Timestamp.valueOf("2018-06-17 08:28:53.999"))
+    val millisData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123"),
+      Timestamp.valueOf("2018-06-15 08:28:53.123"),
+      Timestamp.valueOf("2018-06-16 08:28:53.123"),
+      Timestamp.valueOf("2018-06-17 08:28:53.123"))
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
       testTimestampPushdown(millisData)
     }
 
     // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
-    val microsData = Seq(
-      Timestamp.valueOf("1000-06-14 08:28:53.123456"),
-      Timestamp.valueOf("1582-06-15 08:28:53.123456"),
-      Timestamp.valueOf("1900-06-16 08:28:53.123456"),
+    val microsData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123456"),
+      Timestamp.valueOf("2018-06-15 08:28:53.123456"),
+      Timestamp.valueOf("2018-06-16 08:28:53.123456"),
       Timestamp.valueOf("2018-06-17 08:28:53.123456"))
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index 105f025adc0ad..f5726973d0abf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -69,9 +69,7 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   protected def withParquetDataFrame(df: DataFrame, testVectorized: Boolean = true)
       (f: DataFrame => Unit): Unit = {
     withTempPath { file =>
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
-        df.write.format(dataSourceName).save(file.getCanonicalPath)
-      }
+      df.write.format(dataSourceName).save(file.getCanonicalPath)
       readFile(file.getCanonicalPath, testVectorized)(f)
     }
   }

From 95f9f834591eea181ad782306e6689473e8429d1 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 1 Jun 2020 11:19:42 -0700
Subject: [PATCH 0762/1280] [SPARK-31881][K8S][TESTS] Support Hadoop 3.2 K8s
 integration tests

### What changes were proposed in this pull request?

This PR aims to support Hadoop 3.2 K8s integration tests.

### Why are the changes needed?

Currently, K8s integration suite assumes Hadoop 2.7 and has hard-coded parts.

### Does this PR introduce _any_ user-facing change?

No. This is a dev-only change.

### How was this patch tested?

Pass the Jenkins K8s IT (with Hadoop 2.7) and do the manual testing for Hadoop 3.2 as described in `README.md`.

```
./dev/dev-run-integration-tests.sh --hadoop-profile hadoop-3.2
```

I verified this manually like the following.
```
$ resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh \
--spark-tgz .../spark-3.1.0-SNAPSHOT-bin-3.2.0.tgz \
--exclude-tags r \
--hadoop-profile hadoop-3.2
...
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- All pods have the same service account by default
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
- Start pod creation from template
- PVs with local storage
- Launcher client dependencies
- Test basic decommissioning
Run completed in 8 minutes, 49 seconds.
Total number of tests run: 19
Suites: completed 2, aborted 0
Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```

Closes #28689 from dongjoon-hyun/SPARK-31881.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 17586f9ed25cbb1dd7dbb221aecb8749623e2175)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../kubernetes/integration-tests/README.md    |  4 +++
 .../dev/dev-run-integration-tests.sh          |  7 ++++-
 .../kubernetes/integration-tests/pom.xml      | 30 +++++++++++++++----
 .../k8s/integrationtest/DepsTestsSuite.scala  |  8 +++--
 .../deploy/k8s/integrationtest/Utils.scala    |  7 +++++
 5 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md
index 18b91916208d6..6409c227ec287 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -17,6 +17,10 @@ To run tests with Java 11 instead of Java 8, use `--java-image-tag` to specify t
 
     ./dev/dev-run-integration-tests.sh --java-image-tag 11-jre-slim
 
+To run tests with Hadoop 3.2 instead of Hadoop 2.7, use `--hadoop-profile`.
+
+    ./dev/dev-run-integration-tests.sh --hadoop-profile hadoop-3.2
+
 The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should
 run with a minimum of 4 CPUs and 6G of memory:
 
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 607bb243458a6..75e1ca8d577ca 100755
--- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -35,6 +35,7 @@ CONTEXT=
 INCLUDE_TAGS="k8s"
 EXCLUDE_TAGS=
 JAVA_VERSION="8"
+HADOOP_PROFILE="hadoop-2.7"
 MVN="$TEST_ROOT_DIR/build/mvn"
 
 SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version 2>/dev/null\
@@ -109,6 +110,10 @@ while (( "$#" )); do
       JAVA_VERSION="$2"
       shift
       ;;
+    --hadoop-profile)
+      HADOOP_PROFILE="$2"
+      shift
+      ;;
     *)
       break
       ;;
@@ -166,4 +171,4 @@ properties+=(
   -Dspark.kubernetes.test.rImage=$R_IMAGE_NAME
 )
 
-$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}
+$TEST_ROOT_DIR/build/mvn integration-test -f $TEST_ROOT_DIR/pom.xml -pl resource-managers/kubernetes/integration-tests -am -Pscala-$SCALA_VERSION -P$HADOOP_PROFILE -Pkubernetes -Pkubernetes-integration-tests ${properties[@]}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 719ebe4456eac..bda936c1df41a 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -77,12 +77,6 @@
       <artifactId>spark-tags_${scala.binary.version}</artifactId>
       <type>test-jar</type>
     </dependency>
-    <dependency>
-      <groupId>com.amazonaws</groupId>
-      <artifactId>aws-java-sdk</artifactId>
-      <version>1.7.4</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>
@@ -189,4 +183,28 @@
 
   </build>
 
+  <profiles>
+    <profile>
+      <id>hadoop-2.7</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk</artifactId>
+          <version>1.7.4</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>hadoop-3.2</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.amazonaws</groupId>
+          <artifactId>aws-java-sdk-bundle</artifactId>
+          <version>1.11.375</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
 </project>
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index 2d90c06e36390..e712b95cdbcea 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -148,6 +148,11 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
   }
 
   test("Launcher client dependencies", k8sTestTag, MinikubeTag) {
+    val packages = if (Utils.isHadoop3) {
+      "org.apache.hadoop:hadoop-aws:3.2.0"
+    } else {
+      "com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.6"
+    }
     val fileName = Utils.createTempFile(FILE_CONTENTS, HOST_PATH)
     try {
       setupMinioStorage()
@@ -164,8 +169,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
         .set("spark.kubernetes.file.upload.path", s"s3a://$BUCKET")
         .set("spark.files", s"$HOST_PATH/$fileName")
         .set("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem")
-        .set("spark.jars.packages", "com.amazonaws:aws-java-sdk:" +
-          "1.7.4,org.apache.hadoop:hadoop-aws:2.7.6")
+        .set("spark.jars.packages", packages)
         .set("spark.driver.extraJavaOptions", "-Divy.cache.dir=/tmp -Divy.home=/tmp")
       createS3Bucket(ACCESS_KEY, SECRET_KEY, minioUrlStr)
       runSparkRemoteCheckAndVerifyCompletion(appResource = examplesJar,
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
index 9f85805b9d315..0000a94725763 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
@@ -21,13 +21,16 @@ import java.nio.file.{Files, Path}
 import java.util.concurrent.CountDownLatch
 
 import scala.collection.JavaConverters._
+import scala.util.Try
 
 import io.fabric8.kubernetes.client.dsl.ExecListener
 import okhttp3.Response
 import org.apache.commons.io.output.ByteArrayOutputStream
+import org.apache.hadoop.util.VersionInfo
 
 import org.apache.spark.{SPARK_VERSION, SparkException}
 import org.apache.spark.internal.Logging
+import org.apache.spark.util.{Utils => SparkUtils}
 
 object Utils extends Logging {
 
@@ -131,4 +134,8 @@ object Utils extends Logging {
         s"under spark home test dir ${sparkHomeDir.toAbsolutePath}!")
     }
   }
+
+  def isHadoop3(): Boolean = {
+    VersionInfo.getVersion.startsWith("3")
+  }
 }

From 8a7fb262f65131707646bcccd81072bab9b1300c Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Sun, 15 Mar 2020 20:20:23 -0500
Subject: [PATCH 0763/1280] [SPARK-31068][SQL] Avoid IllegalArgumentException
 in broadcast exchange

### What changes were proposed in this pull request?
Fix the IllegalArgumentException in broadcast exchange when numRows over 341 million but less than 512 million.

Since the maximum number of keys that `BytesToBytesMap` supports is 1 << 29, and only 70% of the slots can be used before growing in `HashedRelation`, So here the limitation should not be greater equal than 341 million (1 << 29 / 1.5(357913941)) instead of 512 million.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Manually test.

Closes #27828 from LantaoJin/SPARK-31068.

Lead-authored-by: LantaoJin <jinlantao@gmail.com>
Co-authored-by: Alan Jin <jinlantao@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 08bdc9c9b2e44e066e44d35733e0f3b450d24052)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/unsafe/map/BytesToBytesMap.java       |  3 +--
 .../exchange/BroadcastExchangeExec.scala        | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index b280f7cc35951..64c240cea8d7a 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -96,8 +96,7 @@ public final class BytesToBytesMap extends MemoryConsumer {
    * since that's the largest power-of-2 that's less than Integer.MAX_VALUE. We need two long array
    * entries per key, giving us a maximum capacity of (1 &lt;&lt; 29).
    */
-  @VisibleForTesting
-  static final int MAX_CAPACITY = (1 << 29);
+  public static final int MAX_CAPACITY = (1 << 29);
 
   // This choice of page table size and page size means that we can address up to 500 gigabytes
   // of memory.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
index ba5261c9bb5f8..d35bbe9b8adc0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
@@ -34,7 +34,8 @@ import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
 import org.apache.spark.sql.execution.joins.HashedRelation
 import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
-import org.apache.spark.util.{SparkFatalException, ThreadUtils, Utils}
+import org.apache.spark.unsafe.map.BytesToBytesMap
+import org.apache.spark.util.{SparkFatalException, ThreadUtils}
 
 /**
  * A [[BroadcastExchangeExec]] collects, transforms and finally broadcasts the result of
@@ -43,6 +44,7 @@ import org.apache.spark.util.{SparkFatalException, ThreadUtils, Utils}
 case class BroadcastExchangeExec(
     mode: BroadcastMode,
     child: SparkPlan) extends Exchange {
+  import BroadcastExchangeExec._
 
   private[sql] val runId: UUID = UUID.randomUUID
 
@@ -82,9 +84,9 @@ case class BroadcastExchangeExec(
             val beforeCollect = System.nanoTime()
             // Use executeCollect/executeCollectIterator to avoid conversion to Scala types
             val (numRows, input) = child.executeCollectIterator()
-            if (numRows >= 512000000) {
+            if (numRows >= MAX_BROADCAST_TABLE_ROWS) {
               throw new SparkException(
-                s"Cannot broadcast the table with 512 million or more rows: $numRows rows")
+                s"Cannot broadcast the table over $MAX_BROADCAST_TABLE_ROWS rows: $numRows rows")
             }
 
             val beforeBuild = System.nanoTime()
@@ -104,7 +106,7 @@ case class BroadcastExchangeExec(
             }
 
             longMetric("dataSize") += dataSize
-            if (dataSize >= (8L << 30)) {
+            if (dataSize >= MAX_BROADCAST_TABLE_BYTES) {
               throw new SparkException(
                 s"Cannot broadcast the table that is larger than 8GB: ${dataSize >> 30} GB")
             }
@@ -173,6 +175,13 @@ case class BroadcastExchangeExec(
 }
 
 object BroadcastExchangeExec {
+  // Since the maximum number of keys that BytesToBytesMap supports is 1 << 29,
+  // and only 70% of the slots can be used before growing in HashedRelation,
+  // here the limitation should not be over 341 million.
+  val MAX_BROADCAST_TABLE_ROWS = (BytesToBytesMap.MAX_CAPACITY / 1.5).toLong
+
+  val MAX_BROADCAST_TABLE_BYTES = 8L << 30
+
   private[execution] val executionContext = ExecutionContext.fromExecutorService(
       ThreadUtils.newDaemonCachedThreadPool("broadcast-exchange",
         SQLConf.get.getConf(StaticSQLConf.BROADCAST_EXCHANGE_MAX_THREAD_THRESHOLD)))

From b6c8366cc58b7d5e35ec2a31532ae7ee22275454 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Mon, 1 Jun 2020 14:33:35 -0700
Subject: [PATCH 0764/1280] [SPARK-31764][CORE][3.0] JsonProtocol doesn't write
 RDDInfo#isBarrier

### What changes were proposed in this pull request?

This PR backports the change of #28583 (SPARK-31764) to branch-3.0, which changes JsonProtocol to write RDDInfos#isBarrier.

### Why are the changes needed?

JsonProtocol reads RDDInfos#isBarrier but not write it so it's a bug.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

I added a testcase.

Closes #28660 from sarutak/SPARK-31764-branch-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Xingbo Jiang <xingbo.jiang@databricks.com>
---
 .../org/apache/spark/util/JsonProtocol.scala  |  1 +
 .../scheduler/EventLoggingListenerSuite.scala | 44 +++++++++++++++++++
 .../apache/spark/util/JsonProtocolSuite.scala | 11 +++++
 3 files changed, 56 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index aee986230d4b5..f445fd4c6f7a6 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -475,6 +475,7 @@ private[spark] object JsonProtocol {
     ("Callsite" -> rddInfo.callSite) ~
     ("Parent IDs" -> parentIds) ~
     ("Storage Level" -> storageLevel) ~
+    ("Barrier" -> rddInfo.isBarrier) ~
     ("Number of Partitions" -> rddInfo.numPartitions) ~
     ("Number of Cached Partitions" -> rddInfo.numCachedPartitions) ~
     ("Memory Size" -> rddInfo.memSize) ~
diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index 286924001e920..046564d65b767 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.deploy.history.{EventLogFileReader, SingleEventLogFileWr
 import org.apache.spark.deploy.history.EventLogTestHelper._
 import org.apache.spark.executor.{ExecutorMetrics, TaskMetrics}
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.{EVENT_LOG_DIR, EVENT_LOG_ENABLED}
 import org.apache.spark.io._
 import org.apache.spark.metrics.{ExecutorMetricType, MetricsSystem}
 import org.apache.spark.scheduler.cluster.ExecutorInfo
@@ -99,6 +100,49 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
     testStageExecutorMetricsEventLogging()
   }
 
+  test("SPARK-31764: isBarrier should be logged in event log") {
+    val conf = new SparkConf()
+    conf.set(EVENT_LOG_ENABLED, true)
+    conf.set(EVENT_LOG_DIR, testDirPath.toString)
+    val sc = new SparkContext("local", "test-SPARK-31764", conf)
+    val appId = sc.applicationId
+
+    sc.parallelize(1 to 10)
+      .barrier()
+      .mapPartitions(_.map(elem => (elem, elem)))
+      .filter(elem => elem._1 % 2 == 0)
+      .reduceByKey(_ + _)
+      .collect
+    sc.stop()
+
+    val eventLogStream = EventLogFileReader.openEventLog(new Path(testDirPath, appId), fileSystem)
+    val events = readLines(eventLogStream).map(line => JsonProtocol.sparkEventFromJson(parse(line)))
+    val jobStartEvents = events
+      .filter(event => event.isInstanceOf[SparkListenerJobStart])
+      .map(_.asInstanceOf[SparkListenerJobStart])
+
+    assert(jobStartEvents.size === 1)
+    val stageInfos = jobStartEvents.head.stageInfos
+    assert(stageInfos.size === 2)
+
+    val stage0 = stageInfos(0)
+    val rddInfosInStage0 = stage0.rddInfos
+    assert(rddInfosInStage0.size === 3)
+    val sortedRddInfosInStage0 = rddInfosInStage0.sortBy(_.scope.get.name)
+    assert(sortedRddInfosInStage0(0).scope.get.name === "filter")
+    assert(sortedRddInfosInStage0(0).isBarrier === true)
+    assert(sortedRddInfosInStage0(1).scope.get.name === "mapPartitions")
+    assert(sortedRddInfosInStage0(1).isBarrier === true)
+    assert(sortedRddInfosInStage0(2).scope.get.name === "parallelize")
+    assert(sortedRddInfosInStage0(2).isBarrier === false)
+
+    val stage1 = stageInfos(1)
+    val rddInfosInStage1 = stage1.rddInfos
+    assert(rddInfosInStage1.size === 1)
+    assert(rddInfosInStage1(0).scope.get.name === "reduceByKey")
+    assert(rddInfosInStage1(0).isBarrier === false) // reduceByKey
+  }
+
   /* ----------------- *
    * Actual test logic *
    * ----------------- */
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index f2b9f97d6cf7d..d1f09d809ea50 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -1063,6 +1063,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |          "Deserialized": true,
       |          "Replication": 1
       |        },
+      |        "Barrier" : false,
       |        "Number of Partitions": 201,
       |        "Number of Cached Partitions": 301,
       |        "Memory Size": 401,
@@ -1585,6 +1586,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 200,
       |          "Number of Cached Partitions": 300,
       |          "Memory Size": 400,
@@ -1629,6 +1631,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 400,
       |          "Number of Cached Partitions": 600,
       |          "Memory Size": 800,
@@ -1645,6 +1648,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 401,
       |          "Number of Cached Partitions": 601,
       |          "Memory Size": 801,
@@ -1689,6 +1693,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 600,
       |          "Number of Cached Partitions": 900,
       |          "Memory Size": 1200,
@@ -1705,6 +1710,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 601,
       |          "Number of Cached Partitions": 901,
       |          "Memory Size": 1201,
@@ -1721,6 +1727,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 602,
       |          "Number of Cached Partitions": 902,
       |          "Memory Size": 1202,
@@ -1765,6 +1772,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 800,
       |          "Number of Cached Partitions": 1200,
       |          "Memory Size": 1600,
@@ -1781,6 +1789,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 801,
       |          "Number of Cached Partitions": 1201,
       |          "Memory Size": 1601,
@@ -1797,6 +1806,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 802,
       |          "Number of Cached Partitions": 1202,
       |          "Memory Size": 1602,
@@ -1813,6 +1823,7 @@ private[spark] object JsonProtocolSuite extends Assertions {
       |            "Deserialized": true,
       |            "Replication": 1
       |          },
+      |          "Barrier" : false,
       |          "Number of Partitions": 803,
       |          "Number of Cached Partitions": 1203,
       |          "Memory Size": 1603,

From 387f57c574beb726db4d305f3c1da7d8255ad103 Mon Sep 17 00:00:00 2001
From: Holden Karau <hkarau@apple.com>
Date: Mon, 1 Jun 2020 15:49:17 -0700
Subject: [PATCH 0765/1280] [SPARK-31889][BUILD] Docker release script does not
 allocate enough memory to reliably publish

### What changes were proposed in this pull request?
Allow overriding the zinc options in the docker release and set a higher so the publish step can succeed consistently.

### Why are the changes needed?

The publish step experiences memory pressure.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?
Running test locally with fake user to see if publish step (besides svn part) succeeds

Closes #28698 from holdenk/SPARK-31889-docker-release-script-does-not-allocate-enough-memory-to-reliably-publish.

Authored-by: Holden Karau <hkarau@apple.com>
Signed-off-by: Holden Karau <hkarau@apple.com>
(cherry picked from commit ab9e5a2fe9a0e9e93301096a027d7409fc3c9b64)
Signed-off-by: Holden Karau <hkarau@apple.com>
---
 dev/create-release/do-release-docker.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/create-release/do-release-docker.sh b/dev/create-release/do-release-docker.sh
index 2f794c0e0a174..8f53f4a4e13ad 100755
--- a/dev/create-release/do-release-docker.sh
+++ b/dev/create-release/do-release-docker.sh
@@ -128,6 +128,7 @@ ASF_PASSWORD=$ASF_PASSWORD
 GPG_PASSPHRASE=$GPG_PASSPHRASE
 RELEASE_STEP=$RELEASE_STEP
 USER=$USER
+ZINC_OPTS=${RELEASE_ZINC_OPTS:-"-Xmx4g -XX:ReservedCodeCacheSize=2g"}
 EOF
 
 JAVA_VOL=

From e8045fb99cc463889aebcda1c8c1daa2042f4319 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 1 Jun 2020 16:31:39 -0700
Subject: [PATCH 0766/1280] [SPARK-28344][SQL][FOLLOW-UP] Check the ambiguous
 self-join only if there is a join in the plan

### What changes were proposed in this pull request?

This PR proposes to check `DetectAmbiguousSelfJoin` only if there is `Join` in the plan. Currently, the checking is too strict even to non-join queries.

For example, the codes below don't have join at all but it fails as the ambiguous self-join:

```scala
import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions.sum
val df = Seq(1, 1, 2, 2).toDF("A")
val w = Window.partitionBy(df("A"))
df.select(df("A").alias("X"), sum(df("A")).over(w)).explain(true)
```

It is because `ExtractWindowExpressions` can create a `AttributeReference` with the same metadata but a different expression ID, see:

https://github.com/apache/spark/blob/0fd98abd859049dc3b200492487041eeeaa8f737/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala#L2679
https://github.com/apache/spark/blob/71c73d58f6e88d2558ed2e696897767d93bac60f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala#L63
https://github.com/apache/spark/blob/5945d46c11a86fd85f9e65f24c2e88f368eee01f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/namedExpressions.scala#L180

Before:

```
'Project [A#19 AS X#21, sum(A#19) windowspecdefinition(A#19, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS sum(A) OVER (PARTITION BY A unspecifiedframe$())#23L]
+- Relation[A#19] parquet
```

After:

```
Project [X#21, sum(A) OVER (PARTITION BY A unspecifiedframe$())#23L]
+- Project [X#21, A#19, sum(A) OVER (PARTITION BY A unspecifiedframe$())#23L, sum(A) OVER (PARTITION BY A unspecifiedframe$())#23L]
   +- Window [sum(A#19) windowspecdefinition(A#19, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS sum(A) OVER (PARTITION BY A unspecifiedframe$())#23L], [A#19]
      +- Project [A#19 AS X#21, A#19]
         +- Relation[A#19] parquet
```

`X#21` holds the same metadata of DataFrame ID and column position with `A#19` but it has a different expression ID which ends up with the checking fails.

### Why are the changes needed?

To loose the checking and make users not surprised.

### Does this PR introduce _any_ user-facing change?

It's the changes in unreleased branches only.

### How was this patch tested?

Manually tested and unittest was added.

Closes #28695 from HyukjinKwon/SPARK-28344-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ea45fc51921e64302b9220b264156bb4f757fe01)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../execution/analysis/DetectAmbiguousSelfJoin.scala |  6 ++++++
 .../apache/spark/sql/DataFrameSelfJoinSuite.scala    | 12 ++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
index 614d6c2846bfa..136f7c47f5341 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala
@@ -76,6 +76,8 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
     // We always remove the special metadata from `AttributeReference` at the end of this rule, so
     // Dataset column reference only exists in the root node via Dataset transformations like
     // `Dataset#select`.
+    if (plan.find(_.isInstanceOf[Join]).isEmpty) return stripColumnReferenceMetadataInPlan(plan)
+
     val colRefAttrs = plan.expressions.flatMap(_.collect {
       case a: AttributeReference if isColumnReference(a) => a
     })
@@ -153,6 +155,10 @@ class DetectAmbiguousSelfJoin(conf: SQLConf) extends Rule[LogicalPlan] {
       }
     }
 
+    stripColumnReferenceMetadataInPlan(plan)
+  }
+
+  private def stripColumnReferenceMetadataInPlan(plan: LogicalPlan): LogicalPlan = {
     plan.transformExpressions {
       case a: AttributeReference if isColumnReference(a) =>
         // Remove the special metadata from this `AttributeReference`, as the detection is done.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
index 250ec7dc0ba5a..fb58c9851224b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.expressions.Window
 import org.apache.spark.sql.functions.{count, sum}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -202,4 +203,15 @@ class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession {
       assertAmbiguousSelfJoin(df1.join(df4).join(df2).select(df2("id")))
     }
   }
+
+  test("SPARK-28344: don't fail as ambiguous self join when there is no join") {
+    withSQLConf(
+      SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED.key -> "true") {
+      val df = Seq(1, 1, 2, 2).toDF("a")
+      val w = Window.partitionBy(df("a"))
+      checkAnswer(
+        df.select(df("a").alias("x"), sum(df("a")).over(w)),
+        Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
+    }
+  }
 }

From 6dec082185a714d4ed3818a2d2d4d011fcd372db Mon Sep 17 00:00:00 2001
From: manuzhang <owenzhang1990@gmail.com>
Date: Tue, 2 Jun 2020 02:00:58 +0000
Subject: [PATCH 0767/1280] [SPARK-31870][SQL][TESTS] Fix "Do not optimize skew
 join if additional shuffle" test having no skew join

### What changes were proposed in this pull request?
Fix configurations and ensure there is skew join in the test "Do not optimize skew join if additional shuffle".

### Why are the changes needed?
The existing "Do not optimize skew join if additional shuffle" test has no skew join at all.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Fixed existing test.

Closes #28679 from manuzhang/spark-31870.

Authored-by: manuzhang <owenzhang1990@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 283814a426fb67289ca7923b9d13c5a897f9a98a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../adaptive/AdaptiveQueryExecSuite.scala     | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index fed7cdcb9eb6a..ac0267ac470d5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -597,21 +597,29 @@ class AdaptiveQueryExecSuite
     withSQLConf(
       SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
       SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
-      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "700") {
+      SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100",
+      SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100") {
       withTempView("skewData1", "skewData2") {
         spark
           .range(0, 1000, 1, 10)
-          .selectExpr("id % 2 as key1", "id as value1")
+          .selectExpr("id % 3 as key1", "id as value1")
           .createOrReplaceTempView("skewData1")
         spark
           .range(0, 1000, 1, 10)
           .selectExpr("id % 1 as key2", "id as value2")
           .createOrReplaceTempView("skewData2")
-        val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(
-          "SELECT key1 FROM skewData1 join skewData2 ON key1 = key2 group by key1")
+
+        def checkSkewJoin(query: String, optimizeSkewJoin: Boolean): Unit = {
+          val (_, innerAdaptivePlan) = runAdaptiveAndVerifyResult(query)
+          val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
+          assert(innerSmj.size == 1 && innerSmj.head.isSkewJoin == optimizeSkewJoin)
+        }
+
+        checkSkewJoin(
+          "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2", true)
         // Additional shuffle introduced, so disable the "OptimizeSkewedJoin" optimization
-        val innerSmj = findTopLevelSortMergeJoin(innerAdaptivePlan)
-        assert(innerSmj.size == 1 && !innerSmj.head.isSkewJoin)
+        checkSkewJoin(
+          "SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 GROUP BY key1", false)
       }
     }
   }

From e3a63a89c2de7c9bb3af20728ebe0081b0df1ffa Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 2 Jun 2020 02:29:20 +0000
Subject: [PATCH 0768/1280] [SPARK-31885][SQL][3.0] Fix filter push down for
 old millis timestamps to Parquet

### What changes were proposed in this pull request?
Fixed conversions of `java.sql.Timestamp` to milliseconds in `ParquetFilter` by using existing functions from `DateTimeUtils` `fromJavaTimestamp()` and `microsToMillis()`.

### Why are the changes needed?
The changes fix the bug:
```scala
scala> spark.conf.set("spark.sql.parquet.outputTimestampType", "TIMESTAMP_MILLIS")
scala> spark.conf.set("spark.sql.legacy.parquet.datetimeRebaseModeInWrite", "CORRECTED")
scala> Seq(java.sql.Timestamp.valueOf("1000-06-14 08:28:53.123")).toDF("ts").write.mode("overwrite").parquet("/Users/maximgekk/tmp/ts_millis_old_filter")
scala> spark.read.parquet("/Users/maximgekk/tmp/ts_millis_old_filter").filter($"ts" === "1000-06-14 08:28:53.123").show(false)
+---+
|ts |
+---+
+---+
```

### Does this PR introduce _any_ user-facing change?
Yes, after the changes (for the example above):
```scala
scala> spark.read.parquet("/Users/maximgekk/tmp/ts_millis_old_filter").filter($"ts" === "1000-06-14 08:28:53.123").show(false)
+-----------------------+
|ts                     |
+-----------------------+
|1000-06-14 08:28:53.123|
+-----------------------+
```

### How was this patch tested?
Modified tests in `ParquetFilterSuite` to check old timestamps.

Closes #28699 from MaxGekk/parquet-ts-millis-filter-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/parquet/ParquetFilters.scala  | 27 +++++++++----------
 .../parquet/ParquetFilterSuite.scala          | 16 ++++++-----
 .../datasources/parquet/ParquetTest.scala     |  4 ++-
 3 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index d89186af8c8e5..d0977ba519209 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -148,6 +148,13 @@ class ParquetFilters(
     Binary.fromConstantByteArray(fixedLengthBytes, 0, numBytes)
   }
 
+  private def timestampToMillis(v: Any): JLong = {
+    val timestamp = v.asInstanceOf[Timestamp]
+    val micros = DateTimeUtils.fromJavaTimestamp(timestamp)
+    val millis = DateTimeUtils.toMillis(micros)
+    millis.asInstanceOf[JLong]
+  }
+
   private val makeEq:
     PartialFunction[ParquetSchemaType, (Array[String], Any) => FilterPredicate] = {
     case ParquetBooleanType =>
@@ -184,7 +191,7 @@ class ParquetFilters(
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.eq(
@@ -235,7 +242,7 @@ class ParquetFilters(
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(_.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMillis).orNull)
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) => FilterApi.notEq(
@@ -277,9 +284,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.lt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -318,9 +323,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -359,9 +362,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gt(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
@@ -400,9 +401,7 @@ class ParquetFilters(
         longColumn(n),
         DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
     case ParquetTimestampMillisType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        v.asInstanceOf[Timestamp].getTime.asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMillis(v))
 
     case ParquetSchemaType(DECIMAL, INT32, _, _) if pushDownDecimal =>
       (n: Array[String], v: Any) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 7b33ceff32e01..c4cf5116c2030 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -589,19 +589,21 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("filter pushdown - timestamp") {
     // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
-    val millisData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123"))
+    val millisData = Seq(
+      Timestamp.valueOf("1000-06-14 08:28:53.123"),
+      Timestamp.valueOf("1582-06-15 08:28:53.001"),
+      Timestamp.valueOf("1900-06-16 08:28:53.0"),
+      Timestamp.valueOf("2018-06-17 08:28:53.999"))
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
       testTimestampPushdown(millisData)
     }
 
     // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
-    val microsData = Seq(Timestamp.valueOf("2018-06-14 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-15 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-16 08:28:53.123456"),
+    val microsData = Seq(
+      Timestamp.valueOf("1000-06-14 08:28:53.123456"),
+      Timestamp.valueOf("1582-06-15 08:28:53.123456"),
+      Timestamp.valueOf("1900-06-16 08:28:53.123456"),
       Timestamp.valueOf("2018-06-17 08:28:53.123456"))
     withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
       ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index f5726973d0abf..105f025adc0ad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -69,7 +69,9 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
   protected def withParquetDataFrame(df: DataFrame, testVectorized: Boolean = true)
       (f: DataFrame => Unit): Unit = {
     withTempPath { file =>
-      df.write.format(dataSourceName).save(file.getCanonicalPath)
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
+        df.write.format(dataSourceName).save(file.getCanonicalPath)
+      }
       readFile(file.getCanonicalPath, testVectorized)(f)
     }
   }

From e2fded3e1ffda71d5e08f492d02cc3d20ff1f53c Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 1 Jun 2020 22:03:47 -0700
Subject: [PATCH 0769/1280] [SPARK-29137][ML][PYTHON][TESTS] Increase the
 timeout for StreamingLinearRegressionWithTests.test_train_prediction

### What changes were proposed in this pull request?

It increases the timeout for `StreamingLinearRegressionWithTests.test_train_prediction`

```
Traceback (most recent call last):
  File "/home/jenkins/workspace/SparkPullRequestBuilder3/python/pyspark/mllib/tests/test_streaming_algorithms.py", line 503, in test_train_prediction
    self._eventually(condition)
  File "/home/jenkins/workspace/SparkPullRequestBuilder3/python/pyspark/mllib/tests/test_streaming_algorithms.py", line 69, in _eventually
    lastValue = condition()
  File "/home/jenkins/workspace/SparkPullRequestBuilder3/python/pyspark/mllib/tests/test_streaming_algorithms.py", line 498, in condition
    self.assertGreater(errors[1] - errors[-1], 2)
AssertionError: 1.672640157855923 not greater than 2
```

This could likely happen when the PySpark tests run in parallel and it become slow.

### Why are the changes needed?

To make the tests less flaky. Seems it's being reported multiple times:

https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/123144/consoleFull
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/123146/testReport/
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/123141/testReport/
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/123142/testReport/

### Does this PR introduce _any_ user-facing change?

No, test-only.

### How was this patch tested?

Jenkins will test it out.

Closes #28701 from HyukjinKwon/SPARK-29137.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 64cb6f7066134a0b9e441291992d2da73de5d918)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 python/pyspark/mllib/tests/test_streaming_algorithms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 2077809a043f1..f57de83bae64d 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -463,7 +463,7 @@ def condition():
                 return True
             return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
 
-        eventually(condition)
+        eventually(condition, timeout=180.0)
 
 
 if __name__ == "__main__":

From b1782af007de94aacf4a9ec2bc93930575a1adea Mon Sep 17 00:00:00 2001
From: lipzhu <lipzhu@ebay.com>
Date: Tue, 2 Jun 2020 21:07:10 +0900
Subject: [PATCH 0770/1280] [SPARK-31834][SQL] Improve error message for
 incompatible data types

### What changes were proposed in this pull request?
We should use dataType.catalogString to unified the data type mismatch message.
Before:
```sql
spark-sql> create table SPARK_31834(a int) using parquet;
spark-sql> insert into SPARK_31834 select '1';
Error in query: Cannot write incompatible data to table '`default`.`spark_31834`':
- Cannot safely cast 'a': StringType to IntegerType;
```

After:
```sql
spark-sql> create table SPARK_31834(a int) using parquet;
spark-sql> insert into SPARK_31834 select '1';
Error in query: Cannot write incompatible data to table '`default`.`spark_31834`':
- Cannot safely cast 'a': string to int;
```

### How was this patch tested?
UT.

Closes #28654 from lipzhu/SPARK-31834.

Authored-by: lipzhu <lipzhu@ebay.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit d79a8a88b15645a29fabb245b6db3b2179d0f3c0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-ref-ansi-compliance.md                |  2 +-
 .../org/apache/spark/sql/types/DataType.scala  |  7 ++++---
 .../analysis/DataSourceV2AnalysisSuite.scala   | 10 +++++-----
 .../DataTypeWriteCompatibilitySuite.scala      | 18 +++++++++---------
 .../inputs/postgreSQL/window_part1.sql         |  2 +-
 .../inputs/postgreSQL/window_part3.sql         |  2 +-
 .../apache/spark/sql/sources/InsertSuite.scala | 16 ++++++++--------
 .../sql/test/DataFrameReaderWriterSuite.scala  | 10 +++++-----
 .../spark/sql/hive/client/VersionsSuite.scala  |  2 +-
 9 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index b62834ebe9067..eab194c71ec79 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -95,7 +95,7 @@ CREATE TABLE t (v INT);
 -- `spark.sql.storeAssignmentPolicy=ANSI`
 INSERT INTO t VALUES ('1');
 org.apache.spark.sql.AnalysisException: Cannot write incompatible data to table '`default`.`t`':
-- Cannot safely cast 'v': StringType to IntegerType;
+- Cannot safely cast 'v': string to int;
 
 -- `spark.sql.storeAssignmentPolicy=LEGACY` (This is a legacy behaviour until Spark 2.x)
 INSERT INTO t VALUES ('1');
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 7449a28e069d2..fe8d7efc9dc12 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -457,7 +457,7 @@ object DataType {
 
       case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == STRICT =>
         if (!Cast.canUpCast(w, r)) {
-          addError(s"Cannot safely cast '$context': $w to $r")
+          addError(s"Cannot safely cast '$context': ${w.catalogString} to ${r.catalogString}")
           false
         } else {
           true
@@ -467,7 +467,7 @@ object DataType {
 
       case (w: AtomicType, r: AtomicType) if storeAssignmentPolicy == ANSI =>
         if (!Cast.canANSIStoreAssign(w, r)) {
-          addError(s"Cannot safely cast '$context': $w to $r")
+          addError(s"Cannot safely cast '$context': ${w.catalogString} to ${r.catalogString}")
           false
         } else {
           true
@@ -477,7 +477,8 @@ object DataType {
         true
 
       case (w, r) =>
-        addError(s"Cannot write '$context': $w is incompatible with $r")
+        addError(s"Cannot write '$context': " +
+          s"${w.catalogString} is incompatible with ${r.catalogString}")
         false
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
index c01dea96fe2de..e466d558db1ef 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
@@ -21,7 +21,7 @@ import java.net.URI
 import java.util.Locale
 
 import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
-import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, AttributeReference, Cast, Expression, LessThanOrEqual, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AnsiCast, AttributeReference, Cast, LessThanOrEqual, Literal}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
@@ -143,7 +143,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write", "'table-name'",
-      "Cannot safely cast", "'x'", "'y'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "'y'", "double to float"))
   }
 
   test("byName: multiple field errors are reported") {
@@ -160,7 +160,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write incompatible data to table", "'table-name'",
-      "Cannot safely cast", "'x'", "DoubleType to FloatType",
+      "Cannot safely cast", "'x'", "double to float",
       "Cannot write nullable values to non-null column", "'x'",
       "Cannot find data for output column", "'y'"))
   }
@@ -176,7 +176,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write", "'table-name'",
-      "Cannot safely cast", "'x'", "'y'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "'y'", "double to float"))
   }
 
   test("byPosition: multiple field errors are reported") {
@@ -194,7 +194,7 @@ abstract class DataSourceV2StrictAnalysisSuite extends DataSourceV2AnalysisBaseS
     assertAnalysisError(parsedPlan, Seq(
       "Cannot write incompatible data to table", "'table-name'",
       "Cannot write nullable values to non-null column", "'x'",
-      "Cannot safely cast", "'x'", "DoubleType to FloatType"))
+      "Cannot safely cast", "'x'", "double to float"))
   }
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
index c47332f5d9fcb..1a262d646ca10 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
@@ -80,7 +80,7 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
   test("Check NullType is incompatible with all other types") {
     allNonNullTypes.foreach { t =>
       assertSingleError(NullType, t, "nulls", s"Should not allow writing None to type $t") { err =>
-        assert(err.contains(s"incompatible with $t"))
+        assert(err.contains(s"incompatible with ${t.catalogString}"))
       }
     }
   }
@@ -145,12 +145,12 @@ class ANSIDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBase
   test("Conversions between timestamp and long are not allowed") {
     assertSingleError(LongType, TimestampType, "longToTimestamp",
       "Should not allow long to timestamp") { err =>
-      assert(err.contains("Cannot safely cast 'longToTimestamp': LongType to TimestampType"))
+      assert(err.contains("Cannot safely cast 'longToTimestamp': bigint to timestamp"))
     }
 
     assertSingleError(TimestampType, LongType, "timestampToLong",
       "Should not allow timestamp to long") { err =>
-      assert(err.contains("Cannot safely cast 'timestampToLong': TimestampType to LongType"))
+      assert(err.contains("Cannot safely cast 'timestampToLong': timestamp to bigint"))
     }
   }
 
@@ -209,8 +209,8 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
             s"Should not allow writing $w to $r because cast is not safe") { err =>
             assert(err.contains("'t'"), "Should include the field name context")
             assert(err.contains("Cannot safely cast"), "Should identify unsafe cast")
-            assert(err.contains(s"$w"), "Should include write type")
-            assert(err.contains(s"$r"), "Should include read type")
+            assert(err.contains(s"${w.catalogString}"), "Should include write type")
+            assert(err.contains(s"${r.catalogString}"), "Should include read type")
           }
         }
       }
@@ -413,7 +413,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
     assertNumErrors(writeType, readType, "top", "Should catch 14 errors", 14) { errs =>
       assert(errs(0).contains("'top.a.element'"), "Should identify bad type")
       assert(errs(0).contains("Cannot safely cast"))
-      assert(errs(0).contains("StringType to DoubleType"))
+      assert(errs(0).contains("string to double"))
 
       assert(errs(1).contains("'top.a'"), "Should identify bad type")
       assert(errs(1).contains("Cannot write nullable elements to array of non-nulls"))
@@ -430,11 +430,11 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
 
       assert(errs(5).contains("'top.m.key'"), "Should identify bad type")
       assert(errs(5).contains("Cannot safely cast"))
-      assert(errs(5).contains("StringType to LongType"))
+      assert(errs(5).contains("string to bigint"))
 
       assert(errs(6).contains("'top.m.value'"), "Should identify bad type")
       assert(errs(6).contains("Cannot safely cast"))
-      assert(errs(6).contains("BooleanType to FloatType"))
+      assert(errs(6).contains("boolean to float"))
 
       assert(errs(7).contains("'top.m'"), "Should identify bad type")
       assert(errs(7).contains("Cannot write nullable values to map of non-nulls"))
@@ -452,7 +452,7 @@ abstract class DataTypeWriteCompatibilityBaseSuite extends SparkFunSuite {
 
       assert(errs(11).contains("'top.x'"), "Should identify bad type")
       assert(errs(11).contains("Cannot safely cast"))
-      assert(errs(11).contains("StringType to IntegerType"))
+      assert(errs(11).contains("string to int"))
 
       assert(errs(12).contains("'top'"), "Should identify bad type")
       assert(errs(12).contains("expected 'x', found 'y'"), "Should detect name mismatch")
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
index 087d7a5befd19..6e95aca7aff62 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part1.sql
@@ -146,7 +146,7 @@ SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 WHERE FALSE)s
 -- mixture of agg/wfunc in the same window
 -- SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC);
 
--- Cannot safely cast 'enroll_date': StringType to DateType;
+-- Cannot safely cast 'enroll_date': string to date;
 -- SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER (ORDER BY empno), MAX(depadj) OVER () FROM(
 -- SELECT *,
 --   CASE WHEN enroll_date < '2008-01-01' THEN 2008 - extract(year FROM enroll_date) END * 500 AS bonus,
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
index cd3b74b3aa03f..f4b8454da0d82 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/window_part3.sql
@@ -42,7 +42,7 @@ create table datetimes (
     f_timestamp timestamp
 ) using parquet;
 
--- Spark cannot safely cast StringType to TimestampType
+-- Spark cannot safely cast string to timestamp
 -- [SPARK-29636] Spark can't parse '11:00 BST' or '2000-10-19 10:23:54+01' signatures to timestamp
 insert into datetimes values
 (1, timestamp '11:00', cast ('11:00 BST' as timestamp), cast ('1 year' as timestamp), cast ('2000-10-19 10:23:54+01' as timestamp), timestamp '2000-10-19 10:23:54'),
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index bb762d26e973c..e56ecd1da07b2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -622,12 +622,12 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[AnalysisException] {
           sql("insert into t select 1L, 2")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': LongType to IntegerType"))
+        assert(msg.contains("Cannot safely cast 'i': bigint to int"))
 
         msg = intercept[AnalysisException] {
           sql("insert into t select 1, 2.0")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'd': DecimalType(2,1) to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'd': decimal(2,1) to double"))
 
         msg = intercept[AnalysisException] {
           sql("insert into t select 1, 2.0D, 3")
@@ -659,18 +659,18 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
         var msg = intercept[AnalysisException] {
           sql("insert into t values('a', 'b')")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': StringType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': StringType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': string to int") &&
+          msg.contains("Cannot safely cast 'd': string to double"))
         msg = intercept[AnalysisException] {
           sql("insert into t values(now(), now())")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': TimestampType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': TimestampType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': timestamp to int") &&
+          msg.contains("Cannot safely cast 'd': timestamp to double"))
         msg = intercept[AnalysisException] {
           sql("insert into t values(true, false)")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': BooleanType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': BooleanType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': boolean to int") &&
+          msg.contains("Cannot safely cast 'd': boolean to double"))
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index 9747840ce4032..fe0a8439acc2d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -333,7 +333,7 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         var msg = intercept[AnalysisException] {
           Seq((1L, 2.0)).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': LongType to IntegerType"))
+        assert(msg.contains("Cannot safely cast 'i': bigint to int"))
 
         // Insert into table successfully.
         Seq((1, 2.0)).toDF("i", "d").write.mode("append").saveAsTable("t")
@@ -354,14 +354,14 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
         var msg = intercept[AnalysisException] {
           Seq(("a", "b")).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': StringType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': StringType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': string to int") &&
+          msg.contains("Cannot safely cast 'd': string to double"))
 
         msg = intercept[AnalysisException] {
           Seq((true, false)).toDF("i", "d").write.mode("append").saveAsTable("t")
         }.getMessage
-        assert(msg.contains("Cannot safely cast 'i': BooleanType to IntegerType") &&
-          msg.contains("Cannot safely cast 'd': BooleanType to DoubleType"))
+        assert(msg.contains("Cannot safely cast 'i': boolean to int") &&
+          msg.contains("Cannot safely cast 'd': boolean to double"))
       }
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index d1dd13623650d..8642a5ff16812 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -982,7 +982,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
            """.stripMargin
           )
 
-          val errorMsg = "Cannot safely cast 'f0': DecimalType(2,1) to BinaryType"
+          val errorMsg = "Cannot safely cast 'f0': decimal(2,1) to binary"
 
           if (isPartitioned) {
             val insertStmt = s"INSERT OVERWRITE TABLE $tableName partition (ds='a') SELECT 1.3"

From b85b954c91aa65dd6046e8c1282712fbb13d61e9 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 2 Jun 2020 11:53:58 +0000
Subject: [PATCH 0771/1280] [SPARK-31888][SQL] Support `java.time.Instant` in
 Parquet filter pushdown

1. Modified `ParquetFilters.valueCanMakeFilterOn()` to accept filters with `java.time.Instant` attributes.
2. Added `ParquetFilters.timestampToMicros()` to support both types `java.sql.Timestamp` and `java.time.Instant` in conversions to microseconds.
3. Re-used `timestampToMicros` in constructing of Parquet filters.

To support pushed down filters with `java.time.Instant` attributes. Before the changes, date filters are not pushed down to Parquet datasource when `spark.sql.datetime.java8API.enabled` is `true`.

No

Modified tests to `ParquetFilterSuite` to check the case when Java 8 API is enabled.

Closes #28696 from MaxGekk/support-instant-parquet-filters.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/parquet/ParquetFilters.scala  |  34 +++--
 .../parquet/ParquetFilterSuite.scala          | 119 ++++++++++--------
 2 files changed, 83 insertions(+), 70 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
index d0977ba519209..13dee484bd98a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilters.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.lang.{Boolean => JBoolean, Double => JDouble, Float => JFloat, Long => JLong}
 import java.math.{BigDecimal => JBigDecimal}
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{Instant, LocalDate}
 import java.util.Locale
 
 import scala.collection.JavaConverters.asScalaBufferConverter
@@ -129,6 +129,11 @@ class ParquetFilters(
     case ld: LocalDate => DateTimeUtils.localDateToDays(ld)
   }
 
+  private def timestampToMicros(v: Any): JLong = v match {
+    case i: Instant => DateTimeUtils.instantToMicros(i)
+    case t: Timestamp => DateTimeUtils.fromJavaTimestamp(t)
+  }
+
   private def decimalToInt32(decimal: JBigDecimal): Integer = decimal.unscaledValue().intValue()
 
   private def decimalToInt64(decimal: JBigDecimal): JLong = decimal.unscaledValue().longValue()
@@ -149,8 +154,7 @@ class ParquetFilters(
   }
 
   private def timestampToMillis(v: Any): JLong = {
-    val timestamp = v.asInstanceOf[Timestamp]
-    val micros = DateTimeUtils.fromJavaTimestamp(timestamp)
+    val micros = timestampToMicros(v)
     val millis = DateTimeUtils.toMillis(micros)
     millis.asInstanceOf[JLong]
   }
@@ -186,8 +190,7 @@ class ParquetFilters(
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
-        Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
-          .asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMicros).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.eq(
         longColumn(n),
@@ -237,8 +240,7 @@ class ParquetFilters(
     case ParquetTimestampMicrosType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
-        Option(v).map(t => DateTimeUtils.fromJavaTimestamp(t.asInstanceOf[Timestamp])
-          .asInstanceOf[JLong]).orNull)
+        Option(v).map(timestampToMicros).orNull)
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.notEq(
         longColumn(n),
@@ -280,9 +282,7 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.lt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.lt(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.lt(longColumn(n), timestampToMillis(v))
 
@@ -319,9 +319,7 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.ltEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.ltEq(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.ltEq(longColumn(n), timestampToMillis(v))
 
@@ -358,9 +356,7 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.gt(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gt(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.gt(longColumn(n), timestampToMillis(v))
 
@@ -397,9 +393,7 @@ class ParquetFilters(
       (n: Array[String], v: Any) =>
         FilterApi.gtEq(intColumn(n), dateToDays(v).asInstanceOf[Integer])
     case ParquetTimestampMicrosType if pushDownTimestamp =>
-      (n: Array[String], v: Any) => FilterApi.gtEq(
-        longColumn(n),
-        DateTimeUtils.fromJavaTimestamp(v.asInstanceOf[Timestamp]).asInstanceOf[JLong])
+      (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMicros(v))
     case ParquetTimestampMillisType if pushDownTimestamp =>
       (n: Array[String], v: Any) => FilterApi.gtEq(longColumn(n), timestampToMillis(v))
 
@@ -475,7 +469,7 @@ class ParquetFilters(
       case ParquetDateType =>
         value.isInstanceOf[Date] || value.isInstanceOf[LocalDate]
       case ParquetTimestampMicrosType | ParquetTimestampMillisType =>
-        value.isInstanceOf[Timestamp]
+        value.isInstanceOf[Timestamp] || value.isInstanceOf[Instant]
       case ParquetSchemaType(DECIMAL, INT32, _, decimalMeta) =>
         isDecimalMatched(value, decimalMeta)
       case ParquetSchemaType(DECIMAL, INT64, _, decimalMeta) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index c4cf5116c2030..d20a07f420e87 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.datasources.parquet
 import java.math.{BigDecimal => JBigDecimal}
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.time.LocalDate
+import java.time.{LocalDate, LocalDateTime, ZoneId}
 
 import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators}
 import org.apache.parquet.filter2.predicate.FilterApi._
@@ -143,7 +143,10 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
   }
 
-  private def testTimestampPushdown(data: Seq[Timestamp]): Unit = {
+  private def testTimestampPushdown(data: Seq[String], java8Api: Boolean): Unit = {
+    implicit class StringToTs(s: String) {
+      def ts: Timestamp = Timestamp.valueOf(s)
+    }
     assert(data.size === 4)
     val ts1 = data.head
     val ts2 = data(1)
@@ -151,7 +154,18 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     val ts4 = data(3)
 
     import testImplicits._
-    withNestedDataFrame(data.map(i => Tuple1(i)).toDF()) { case (inputDF, colName, resultFun) =>
+    val df = data.map(i => Tuple1(Timestamp.valueOf(i))).toDF()
+    withNestedDataFrame(df) { case (inputDF, colName, fun) =>
+      def resultFun(tsStr: String): Any = {
+        val parsed = if (java8Api) {
+          LocalDateTime.parse(tsStr.replace(" ", "T"))
+            .atZone(ZoneId.systemDefault())
+            .toInstant
+        } else {
+          Timestamp.valueOf(tsStr)
+        }
+        fun(parsed)
+      }
       withParquetDataFrame(inputDF) { implicit df =>
         val tsAttr = df(colName).expr
         assert(df(colName).expr.dataType === TimestampType)
@@ -160,26 +174,26 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
           data.map(i => Row.apply(resultFun(i))))
 
-        checkFilterPredicate(tsAttr === ts1, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr <=> ts1, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr =!= ts1, classOf[NotEq[_]],
+        checkFilterPredicate(tsAttr === ts1.ts, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr <=> ts1.ts, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr =!= ts1.ts, classOf[NotEq[_]],
           Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
 
-        checkFilterPredicate(tsAttr < ts2, classOf[Lt[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr > ts1, classOf[Gt[_]],
+        checkFilterPredicate(tsAttr < ts2.ts, classOf[Lt[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr > ts1.ts, classOf[Gt[_]],
           Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
-        checkFilterPredicate(tsAttr <= ts1, classOf[LtEq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr >= ts4, classOf[GtEq[_]], resultFun(ts4))
-
-        checkFilterPredicate(Literal(ts1) === tsAttr, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts1) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts2) > tsAttr, classOf[Lt[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts3) < tsAttr, classOf[Gt[_]], resultFun(ts4))
-        checkFilterPredicate(Literal(ts1) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts4) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
-
-        checkFilterPredicate(!(tsAttr < ts4), classOf[GtEq[_]], resultFun(ts4))
-        checkFilterPredicate(tsAttr < ts2 || tsAttr > ts3, classOf[Operators.Or],
+        checkFilterPredicate(tsAttr <= ts1.ts, classOf[LtEq[_]], resultFun(ts1))
+        checkFilterPredicate(tsAttr >= ts4.ts, classOf[GtEq[_]], resultFun(ts4))
+
+        checkFilterPredicate(Literal(ts1.ts) === tsAttr, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts1.ts) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts2.ts) > tsAttr, classOf[Lt[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts3.ts) < tsAttr, classOf[Gt[_]], resultFun(ts4))
+        checkFilterPredicate(Literal(ts1.ts) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
+        checkFilterPredicate(Literal(ts4.ts) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
+
+        checkFilterPredicate(!(tsAttr < ts4.ts), classOf[GtEq[_]], resultFun(ts4))
+        checkFilterPredicate(tsAttr < ts2.ts || tsAttr > ts3.ts, classOf[Operators.Or],
           Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
       }
     }
@@ -588,36 +602,41 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("filter pushdown - timestamp") {
-    // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
-    val millisData = Seq(
-      Timestamp.valueOf("1000-06-14 08:28:53.123"),
-      Timestamp.valueOf("1582-06-15 08:28:53.001"),
-      Timestamp.valueOf("1900-06-16 08:28:53.0"),
-      Timestamp.valueOf("2018-06-17 08:28:53.999"))
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
-      testTimestampPushdown(millisData)
-    }
-
-    // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
-    val microsData = Seq(
-      Timestamp.valueOf("1000-06-14 08:28:53.123456"),
-      Timestamp.valueOf("1582-06-15 08:28:53.123456"),
-      Timestamp.valueOf("1900-06-16 08:28:53.123456"),
-      Timestamp.valueOf("2018-06-17 08:28:53.123456"))
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
-      testTimestampPushdown(microsData)
-    }
-
-    // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
-    withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
-      ParquetOutputTimestampType.INT96.toString) {
-      import testImplicits._
-      withParquetDataFrame(millisData.map(i => Tuple1(i)).toDF()) { implicit df =>
-        val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
-        assertResult(None) {
-          createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+    Seq(true, false).foreach { java8Api =>
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+        // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
+        val millisData = Seq(
+          "1000-06-14 08:28:53.123",
+          "1582-06-15 08:28:53.001",
+          "1900-06-16 08:28:53.0",
+          "2018-06-17 08:28:53.999")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MILLIS.toString) {
+          testTimestampPushdown(millisData, java8Api)
+        }
+
+        // spark.sql.parquet.outputTimestampType = TIMESTAMP_MICROS
+        val microsData = Seq(
+          "1000-06-14 08:28:53.123456",
+          "1582-06-15 08:28:53.123456",
+          "1900-06-16 08:28:53.123456",
+          "2018-06-17 08:28:53.123456")
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.TIMESTAMP_MICROS.toString) {
+          testTimestampPushdown(microsData, java8Api)
+        }
+
+        // spark.sql.parquet.outputTimestampType = INT96 doesn't support pushdown
+        withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
+          ParquetOutputTimestampType.INT96.toString) {
+          import testImplicits._
+          withParquetDataFrame(
+            millisData.map(i => Tuple1(Timestamp.valueOf(i))).toDF()) { implicit df =>
+            val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
+            assertResult(None) {
+              createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+            }
+          }
         }
       }
     }

From b814cbc8121c8d02a81efbae3e3cf1b6a6219262 Mon Sep 17 00:00:00 2001
From: Holden Karau <hkarau@apple.com>
Date: Tue, 2 Jun 2020 11:04:07 -0700
Subject: [PATCH 0772/1280] [SPARK-31860][BUILD] only push release tags on
 succes

### What changes were proposed in this pull request?

Only push the release tag after the build has finished.

### Why are the changes needed?

If the build fails we don't need a release tag.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Running locally with a fake user in https://github.com/apache/spark/pull/28667

Closes #28700 from holdenk/SPARK-31860-build-master-only-push-tags-on-success.

Authored-by: Holden Karau <hkarau@apple.com>
Signed-off-by: Holden Karau <hkarau@apple.com>
(cherry picked from commit 69ba9b662e2ace592380e3cc5de041031dec2254)
Signed-off-by: Holden Karau <hkarau@apple.com>
---
 dev/create-release/do-release.sh    | 14 +++++++++++---
 dev/create-release/release-build.sh |  7 +++++--
 dev/create-release/release-tag.sh   | 16 +++-------------
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/dev/create-release/do-release.sh b/dev/create-release/do-release.sh
index 4f18a55a3bceb..64fba8a56affe 100755
--- a/dev/create-release/do-release.sh
+++ b/dev/create-release/do-release.sh
@@ -17,6 +17,8 @@
 # limitations under the License.
 #
 
+set -e
+
 SELF=$(cd $(dirname $0) && pwd)
 . "$SELF/release-util.sh"
 
@@ -52,9 +54,6 @@ function should_build {
 if should_build "tag" && [ $SKIP_TAG = 0 ]; then
   run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
     "$SELF/release-tag.sh"
-  echo "It may take some time for the tag to be synchronized to github."
-  echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available."
-  read
 else
   echo "Skipping tag creation for $RELEASE_TAG."
 fi
@@ -79,3 +78,12 @@ if should_build "publish"; then
 else
   echo "Skipping publish step."
 fi
+
+if should_build "tag" && [ $SKIP_TAG = 0 ]; then
+  git push origin $RELEASE_TAG
+  if [[ $RELEASE_TAG != *"preview"* ]]; then
+    git push origin HEAD:$GIT_BRANCH
+  else
+    echo "It's preview release. We only push $RELEASE_TAG to remote."
+  fi
+fi
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index e3bcb72ab5c6f..66c51845cc1d0 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -92,9 +92,12 @@ BASE_DIR=$(pwd)
 init_java
 init_maven_sbt
 
-rm -rf spark
-git clone "$ASF_REPO"
+# Only clone repo fresh if not present, otherwise use checkout from the tag step
+if [ ! -d spark ]; then
+  git clone "$ASF_REPO"
+fi
 cd spark
+git fetch
 git checkout $GIT_REF
 git_hash=`git rev-parse --short HEAD`
 echo "Checked out Spark git hash $git_hash"
diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh
index 39856a9955955..e37aa27fc0aac 100755
--- a/dev/create-release/release-tag.sh
+++ b/dev/create-release/release-tag.sh
@@ -25,6 +25,7 @@ function exit_with_usage {
   cat << EOF
 usage: $NAME
 Tags a Spark release on a particular branch.
+You must push the tags after.
 
 Inputs are specified with the following environment variables:
 ASF_USERNAME - Apache Username
@@ -105,19 +106,8 @@ sed -i".tmp7" 's/SPARK_VERSION_SHORT:.*$/SPARK_VERSION_SHORT: '"$R_NEXT_VERSION"
 
 git commit -a -m "Preparing development version $NEXT_VERSION"
 
-if ! is_dry_run; then
-  # Push changes
-  git push origin $RELEASE_TAG
-  if [[ $RELEASE_VERSION != *"preview"* ]]; then
-    git push origin HEAD:$GIT_BRANCH
-  else
-    echo "It's preview release. We only push $RELEASE_TAG to remote."
-  fi
-
-  cd ..
-  rm -rf spark
-else
-  cd ..
+cd ..
+if is_dry_run; then
   mv spark spark.tag
   echo "Clone with version changes and tag available as spark.tag in the output directory."
 fi

From 1292bf22bf0baa60c902a8c6985bc508cbdbef1b Mon Sep 17 00:00:00 2001
From: Holden Karau <hkarau@apple.com>
Date: Tue, 2 Jun 2020 11:11:23 -0700
Subject: [PATCH 0773/1280] [SPARK-31778][K8S][BUILD] Support cross-building
 docker images

### What changes were proposed in this pull request?

Add cross build support to our docker image script using the new dockerx extension.

### Why are the changes needed?

We have a CI for Spark on ARM, we should support building images for ARM and AMD64.

### Does this PR introduce _any_ user-facing change?

Yes, a new flag is added to the docker image build script to cross-build

### How was this patch tested?
Manually ran build script & pushed to https://hub.docker.com/repository/registry-1.docker.io/holdenk/spark/tags?page=1 verified amd64 & arm64 listed.

Closes #28615 from holdenk/cross-build.

Lead-authored-by: Holden Karau <hkarau@apple.com>
Co-authored-by: Holden Karau <holden@pigscanfly.ca>
Signed-off-by: Holden Karau <hkarau@apple.com>
(cherry picked from commit 25702281dc0c7cc333978f51a15ebf9fd02cc684)
Signed-off-by: Holden Karau <hkarau@apple.com>
---
 bin/docker-image-tool.sh | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 57b86254ab424..8a01b80c4164b 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -19,6 +19,8 @@
 # This script builds and pushes docker images when run from a release of Spark
 # with Kubernetes support.
 
+set -x
+
 function error {
   echo "$@" 1>&2
   exit 1
@@ -172,6 +174,7 @@ function build {
   local BASEDOCKERFILE=${BASEDOCKERFILE:-"kubernetes/dockerfiles/spark/Dockerfile"}
   local PYDOCKERFILE=${PYDOCKERFILE:-false}
   local RDOCKERFILE=${RDOCKERFILE:-false}
+  local ARCHS=${ARCHS:-"--platform linux/amd64,linux/arm64"}
 
   (cd $(img_ctx_dir base) && docker build $NOCACHEARG "${BUILD_ARGS[@]}" \
     -t $(image_ref spark) \
@@ -179,6 +182,11 @@ function build {
   if [ $? -ne 0 ]; then
     error "Failed to build Spark JVM Docker image, please refer to Docker build output for details."
   fi
+  if [ "${CROSS_BUILD}" != "false" ]; then
+  (cd $(img_ctx_dir base) && docker buildx build $ARCHS $NOCACHEARG "${BUILD_ARGS[@]}" \
+    -t $(image_ref spark) \
+    -f "$BASEDOCKERFILE" .)
+  fi
 
   if [ "${PYDOCKERFILE}" != "false" ]; then
     (cd $(img_ctx_dir pyspark) && docker build $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
@@ -187,6 +195,11 @@ function build {
       if [ $? -ne 0 ]; then
         error "Failed to build PySpark Docker image, please refer to Docker build output for details."
       fi
+      if [ "${CROSS_BUILD}" != "false" ]; then
+        (cd $(img_ctx_dir pyspark) && docker buildx build $ARCHS $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
+          -t $(image_ref spark-py) \
+          -f "$PYDOCKERFILE" .)
+      fi
   fi
 
   if [ "${RDOCKERFILE}" != "false" ]; then
@@ -196,6 +209,11 @@ function build {
     if [ $? -ne 0 ]; then
       error "Failed to build SparkR Docker image, please refer to Docker build output for details."
     fi
+    if [ "${CROSS_BUILD}" != "false" ]; then
+      (cd $(img_ctx_dir sparkr) && docker buildx build $ARCHS $NOCACHEARG "${BINDING_BUILD_ARGS[@]}" \
+        -t $(image_ref spark-r) \
+        -f "$RDOCKERFILE" .)
+    fi
   fi
 }
 
@@ -227,6 +245,8 @@ Options:
   -n                    Build docker image with --no-cache
   -u uid                UID to use in the USER directive to set the user the main Spark process runs as inside the
                         resulting container
+  -X                    Use docker buildx to cross build. Automatically pushes.
+                        See https://docs.docker.com/buildx/working-with-buildx/ for steps to setup buildx.
   -b arg                Build arg to build or push the image. For multiple build args, this option needs to
                         be used separately for each build arg.
 
@@ -252,6 +272,12 @@ Examples:
   - Build and push JDK11-based image with tag "v3.0.0" to docker.io/myrepo
     $0 -r docker.io/myrepo -t v3.0.0 -b java_image_tag=11-jre-slim build
     $0 -r docker.io/myrepo -t v3.0.0 push
+
+  - Build and push JDK11-based image for multiple archs to docker.io/myrepo
+    $0 -r docker.io/myrepo -t v3.0.0 -X -b java_image_tag=11-jre-slim build
+    # Note: buildx, which does cross building, needs to do the push during build
+    # So there is no seperate push step with -X
+
 EOF
 }
 
@@ -268,7 +294,8 @@ RDOCKERFILE=
 NOCACHEARG=
 BUILD_PARAMS=
 SPARK_UID=
-while getopts f:p:R:mr:t:nb:u: option
+CROSS_BUILD="false"
+while getopts f:p:R:mr:t:Xnb:u: option
 do
  case "${option}"
  in
@@ -279,6 +306,7 @@ do
  t) TAG=${OPTARG};;
  n) NOCACHEARG="--no-cache";;
  b) BUILD_PARAMS=${BUILD_PARAMS}" --build-arg "${OPTARG};;
+ X) CROSS_BUILD=1;;
  m)
    if ! which minikube 1>/dev/null; then
      error "Cannot find minikube."

From 32cb37ccbf8f539f9e8d07f137fce1c4e4db5fb0 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Tue, 2 Jun 2020 16:41:19 -0700
Subject: [PATCH 0774/1280] [SPARK-31756][WEBUI][3.0] Add real headless browser
 support for UI test
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

This PR backports the change of #28627 .
Some fixes for UI like #28690 should be merged to `branch-3.0` including testcases so #28627 is need to be backported.

This PR mainly adds two things.

1. Real headless browser support for UI test
2. A test suite using headless Chrome as one instance of  those browsers.

Also, for environment where Chrome and Chrome driver is not installed, `ChromeUITest` tag is added to filter out the test suite.
By default, test suites with `ChromeUITest` is disabled.

### Why are the changes needed?

In `branch-3.0`, there are two problems for UI test.
1. Lots of tests especially JavaScript related ones are done manually.
Appearance is better to be confirmed by our eyes but logic should be tested by test cases ideally.

2. Compared to the real web browsers, HtmlUnit doesn't seem to support JavaScript enough.
I added a JavaScript related test before for SPARK-31534 using HtmlUnit which is simple library based headless browser for test.
The test I added works somehow but some JavaScript related error is shown in unit-tests.log.

```
======= EXCEPTION START ========
Exception class=[net.sourceforge.htmlunit.corejs.javascript.JavaScriptException]
com.gargoylesoftware.htmlunit.ScriptException: Error: TOOLTIP: Option "sanitizeFn" provided type "window" but expected type "(null|function)". (http://192.168.1.209:60724/static/jquery-3.4.1.min.js#2)
        at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$HtmlUnitContextAction.run(JavaScriptEngine.java:904)
        at net.sourceforge.htmlunit.corejs.javascript.Context.call(Context.java:628)
        at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.call(ContextFactory.java:515)
        at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.callFunction(JavaScriptEngine.java:835)
        at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine.callFunction(JavaScriptEngine.java:807)
        at com.gargoylesoftware.htmlunit.InteractivePage.executeJavaScriptFunctionIfPossible(InteractivePage.java:216)
        at com.gargoylesoftware.htmlunit.javascript.background.JavaScriptFunctionJob.runJavaScript(JavaScriptFunctionJob.java:52)
        at com.gargoylesoftware.htmlunit.javascript.background.JavaScriptExecutionJob.run(JavaScriptExecutionJob.java:102)
        at com.gargoylesoftware.htmlunit.javascript.background.JavaScriptJobManagerImpl.runSingleJob(JavaScriptJobManagerImpl.java:426)
        at com.gargoylesoftware.htmlunit.javascript.background.DefaultJavaScriptExecutor.run(DefaultJavaScriptExecutor.java:157)
        at java.lang.Thread.run(Thread.java:748)
Caused by: net.sourceforge.htmlunit.corejs.javascript.JavaScriptException: Error: TOOLTIP: Option "sanitizeFn" provided type "window" but expected type "(null|function)". (http://192.168.1.209:60724/static/jquery-3.4.1.min.js#2)
        at net.sourceforge.htmlunit.corejs.javascript.Interpreter.interpretLoop(Interpreter.java:1009)
        at net.sourceforge.htmlunit.corejs.javascript.Interpreter.interpret(Interpreter.java:800)
        at net.sourceforge.htmlunit.corejs.javascript.InterpretedFunction.call(InterpretedFunction.java:105)
        at net.sourceforge.htmlunit.corejs.javascript.ContextFactory.doTopCall(ContextFactory.java:413)
        at com.gargoylesoftware.htmlunit.javascript.HtmlUnitContextFactory.doTopCall(HtmlUnitContextFactory.java:252)
        at net.sourceforge.htmlunit.corejs.javascript.ScriptRuntime.doTopCall(ScriptRuntime.java:3264)
        at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$4.doRun(JavaScriptEngine.java:828)
        at com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine$HtmlUnitContextAction.run(JavaScriptEngine.java:889)
        ... 10 more
JavaScriptException value = Error: TOOLTIP: Option "sanitizeFn" provided type "window" but expected type "(null|function)".
== CALLING JAVASCRIPT ==
  function () {
      throw e;
  }
======= EXCEPTION END ========
```
I tried to upgrade HtmlUnit to 2.40.0 but what is worse, the test become not working even though it works on real browsers like Chrome, Safari and Firefox without error.
```
[info] UISeleniumSuite:
[info] - SPARK-31534: text for tooltip should be escaped *** FAILED *** (17 seconds, 745 milliseconds)
[info]   The code passed to eventually never returned normally. Attempted 2 times over 12.910785232 seconds. Last failure message: com.gargoylesoftware.htmlunit.ScriptException: ReferenceError: Assignment to undefined "regeneratorRuntime" in strict mode (http://192.168.1.209:62132/static/vis-timeline-graph2d.min.js#52(Function)#1)
```
To resolve those problems, it's better to support headless browser for UI test.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

I tested with following patterns. Both Chrome and Chrome driver should be installed to test.

1. sbt / with default excluded tags (ChromeUISeleniumSuite is expected to be skipped and SQLQueryTestSuite is expected to succeed)
`build/sbt -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly org.apache.spark.ui.ChromeUISeleniumSuite org.apache.spark.sql.SQLQueryTestSuite"`

2. sbt / overwrite default excluded tags as empty string (Both suites are expected to succeed)
`build/sbt -Dtest.default.exclude.tags= -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly org.apache.spark.ui.ChromeUISeleniumSuite org.apache.spark.sql.SQLQueryTestSuite"`

3. sbt / set `test.exclude.tags` to `org.apache.spark.tags.ExtendedSQLTest` (Both suites are expected to be skipped)
`build/sbt -Dtest.exclude.tags=org.apache.spark.tags.ExtendedSQLTest -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly org.apache.spark.ui.ChromeUISeleniumSuite org.apache.spark.sql.SQLQueryTestSuite"`

4. Maven / with default excluded tags (ChromeUISeleniumSuite is expected to be skipped and SQLQueryTestSuite is expected to succeed)
`build/mvn -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver -Dtest=none -DwildcardSuites=org.apache.spark.ui.ChromeUISeleniumSuite,org.apache.spark.sql.SQLQueryTestSuite test`

5. Maven / overwrite default excluded tags as empty string (Both suites are expected to succeed)
`build/mvn -Dtest.default.exclude.tags= -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver -Dtest=none -DwildcardSuites=org.apache.spark.ui.ChromeUISeleniumSuite,org.apache.spark.sql.SQLQueryTestSuite test`

6. Maven / set `test.exclude.tags` to `org.apache.spark.tags.ExtendedSQLTest` (Both suites are expected to be skipped)
`build/mvn -Dtest.exclude.tags=org.apache.spark.tags.ExtendedSQLTest  -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver -Dtest=none -DwildcardSuites=org.apache.spark.ui.ChromeUISeleniumSuite,org.apache.spark.sql.SQLQueryTestSuite test`

Closes #28702 from sarutak/real-headless-browser-support-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../org/apache/spark/tags/ChromeUITest.java   |  27 +++++
 .../spark/ui/ChromeUISeleniumSuite.scala      |  49 ++++++++
 .../spark/ui/RealBrowserUISeleniumSuite.scala | 109 ++++++++++++++++++
 .../org/apache/spark/ui/UISeleniumSuite.scala |  27 -----
 pom.xml                                       |   7 +-
 project/SparkBuild.scala                      |   7 ++
 6 files changed, 198 insertions(+), 28 deletions(-)
 create mode 100644 common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
 create mode 100644 core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
 create mode 100644 core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala

diff --git a/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
new file mode 100644
index 0000000000000..e3fed3d656d20
--- /dev/null
+++ b/common/tags/src/test/java/org/apache/spark/tags/ChromeUITest.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import java.lang.annotation.*;
+
+import org.scalatest.TagAnnotation;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface ChromeUITest { }
diff --git a/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
new file mode 100644
index 0000000000000..9ba705c4abd75
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/ChromeUISeleniumSuite.scala
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions}
+
+import org.apache.spark.tags.ChromeUITest
+
+/**
+ * Selenium tests for the Spark Web UI with Chrome.
+ */
+@ChromeUITest
+class ChromeUISeleniumSuite extends RealBrowserUISeleniumSuite("webdriver.chrome.driver") {
+
+  override var webDriver: WebDriver = _
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    val chromeOptions = new ChromeOptions
+    chromeOptions.addArguments("--headless", "--disable-gpu")
+    webDriver = new ChromeDriver(chromeOptions)
+  }
+
+  override def afterAll(): Unit = {
+    try {
+      if (webDriver != null) {
+        webDriver.quit()
+      }
+    } finally {
+      super.afterAll()
+    }
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
new file mode 100644
index 0000000000000..84f888267857e
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ui
+
+import org.openqa.selenium.{By, WebDriver}
+import org.scalatest._
+import org.scalatest.concurrent.Eventually._
+import org.scalatest.time.SpanSugar._
+import org.scalatestplus.selenium.WebBrowser
+
+import org.apache.spark._
+import org.apache.spark.LocalSparkContext.withSpark
+import org.apache.spark.internal.config.MEMORY_OFFHEAP_SIZE
+import org.apache.spark.internal.config.UI.{UI_ENABLED, UI_KILL_ENABLED, UI_PORT}
+import org.apache.spark.util.CallSite
+
+/**
+ * Selenium tests for the Spark Web UI with real web browsers.
+ */
+abstract class RealBrowserUISeleniumSuite(val driverProp: String)
+  extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll {
+
+  implicit var webDriver: WebDriver
+  private val driverPropPrefix = "spark.test."
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    assume(
+      sys.props(driverPropPrefix + driverProp) !== null,
+      "System property " + driverPropPrefix + driverProp +
+        " should be set to the corresponding driver path.")
+    sys.props(driverProp) = sys.props(driverPropPrefix + driverProp)
+  }
+
+  override def afterAll(): Unit = {
+    sys.props.remove(driverProp)
+    super.afterAll()
+  }
+
+  test("SPARK-31534: text for tooltip should be escaped") {
+    withSpark(newSparkContext()) { sc =>
+      sc.setLocalProperty(CallSite.LONG_FORM, "collect at <console>:25")
+      sc.setLocalProperty(CallSite.SHORT_FORM, "collect at <console>:25")
+      sc.parallelize(1 to 10).collect
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs")
+
+        val jobDesc =
+          webDriver.findElement(By.cssSelector("div[class='application-timeline-content']"))
+        jobDesc.getAttribute("data-title") should include  ("collect at &lt;console&gt;:25")
+
+        goToUi(sc, "/jobs/job/?id=0")
+        webDriver.get(sc.ui.get.webUrl.stripSuffix("/") + "/jobs/job/?id=0")
+        val stageDesc = webDriver.findElement(By.cssSelector("div[class='job-timeline-content']"))
+        stageDesc.getAttribute("data-title") should include ("collect at &lt;console&gt;:25")
+
+        // Open DAG Viz.
+        webDriver.findElement(By.id("job-dag-viz")).click()
+        val nodeDesc = webDriver.findElement(By.cssSelector("g[class='node_0 node']"))
+        nodeDesc.getAttribute("name") should include ("collect at &lt;console&gt;:25")
+      }
+    }
+  }
+
+  /**
+   * Create a test SparkContext with the SparkUI enabled.
+   * It is safe to `get` the SparkUI directly from the SparkContext returned here.
+   */
+  private def newSparkContext(
+      killEnabled: Boolean = true,
+      master: String = "local",
+      additionalConfs: Map[String, String] = Map.empty): SparkContext = {
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName("test")
+      .set(UI_ENABLED, true)
+      .set(UI_PORT, 0)
+      .set(UI_KILL_ENABLED, killEnabled)
+      .set(MEMORY_OFFHEAP_SIZE.key, "64m")
+    additionalConfs.foreach { case (k, v) => conf.set(k, v) }
+    val sc = new SparkContext(conf)
+    assert(sc.ui.isDefined)
+    sc
+  }
+
+  def goToUi(sc: SparkContext, path: String): Unit = {
+    goToUi(sc.ui.get, path)
+  }
+
+  def goToUi(ui: SparkUI, path: String): Unit = {
+    go to (ui.webUrl.stripSuffix("/") + path)
+  }
+}
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 3ec9385116408..909056eab8c5a 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -773,33 +773,6 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
     }
   }
 
-  test("SPARK-31534: text for tooltip should be escaped") {
-    withSpark(newSparkContext()) { sc =>
-      sc.setLocalProperty(CallSite.LONG_FORM, "collect at <console>:25")
-      sc.setLocalProperty(CallSite.SHORT_FORM, "collect at <console>:25")
-      sc.parallelize(1 to 10).collect
-
-      val driver = webDriver.asInstanceOf[HtmlUnitDriver]
-      driver.setJavascriptEnabled(true)
-
-      eventually(timeout(10.seconds), interval(50.milliseconds)) {
-        goToUi(sc, "/jobs")
-        val jobDesc =
-          driver.findElement(By.cssSelector("div[class='application-timeline-content']"))
-        jobDesc.getAttribute("data-title") should include  ("collect at &lt;console&gt;:25")
-
-        goToUi(sc, "/jobs/job/?id=0")
-        val stageDesc = driver.findElement(By.cssSelector("div[class='job-timeline-content']"))
-        stageDesc.getAttribute("data-title") should include ("collect at &lt;console&gt;:25")
-
-        // Open DAG Viz.
-        driver.findElement(By.id("job-dag-viz")).click()
-        val nodeDesc = driver.findElement(By.cssSelector("g[class='node_0 node']"))
-        nodeDesc.getAttribute("name") should include ("collect at &lt;console&gt;:25")
-      }
-    }
-  }
-
   def goToUi(sc: SparkContext, path: String): Unit = {
     goToUi(sc.ui.get, path)
   }
diff --git a/pom.xml b/pom.xml
index 0231f8fd07913..d8d559550a5be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -204,6 +204,9 @@
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
 
     <test.java.home>${java.home}</test.java.home>
+
+    <!-- Some UI tests require Chrome and Chrome driver installed so those tests are disabled by default. -->
+    <test.default.exclude.tags>org.apache.spark.tags.ChromeUITest</test.default.exclude.tags>
     <test.exclude.tags></test.exclude.tags>
     <test.include.tags></test.include.tags>
 
@@ -243,6 +246,7 @@
       things breaking.
     -->
     <spark.test.home>${session.executionRootDirectory}</spark.test.home>
+    <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
 
     <CodeCacheSize>1g</CodeCacheSize>
   </properties>
@@ -2504,10 +2508,11 @@
               <spark.ui.enabled>false</spark.ui.enabled>
               <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
               <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
+              <spark.test.webdriver.chrome.driver>${spark.test.webdriver.chrome.driver}</spark.test.webdriver.chrome.driver>
               <!-- Needed by sql/hive tests. -->
               <test.src.tables>__not_used__</test.src.tables>
             </systemProperties>
-            <tagsToExclude>${test.exclude.tags}</tagsToExclude>
+            <tagsToExclude>${test.exclude.tags},${test.default.exclude.tags}</tagsToExclude>
             <tagsToInclude>${test.include.tags}</tagsToInclude>
           </configuration>
           <executions>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 7bd92b3f15b56..c9521ea268579 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -966,6 +966,9 @@ object TestSettings {
       "2.12"
     }
      */
+
+  private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest")
+
   lazy val settings = Seq (
     // Fork new JVMs for tests and set Java options for those
     fork := true,
@@ -1003,6 +1006,10 @@ object TestSettings {
       sys.props.get("test.exclude.tags").map { tags =>
         tags.split(",").flatMap { tag => Seq("-l", tag) }.toSeq
       }.getOrElse(Nil): _*),
+    testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest,
+      sys.props.get("test.default.exclude.tags").map(tags => tags.split(",").toSeq)
+        .map(tags => tags.filter(!_.trim.isEmpty)).getOrElse(defaultExcludedTags)
+        .flatMap(tag => Seq("-l", tag)): _*),
     testOptions in Test += Tests.Argument(TestFrameworks.JUnit,
       sys.props.get("test.exclude.tags").map { tags =>
         Seq("--exclude-categories=" + tags)

From a28f6c6db6d071a49601b9185f24f1872c625251 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Tue, 2 Jun 2020 16:45:16 -0700
Subject: [PATCH 0775/1280] [SPARK-31882][WEBUI] DAG-viz is not rendered
 correctly with pagination

### What changes were proposed in this pull request?

This PR fix an issue related to DAG-viz.
Because DAG-viz for a job fetches link urls for each stage from the stage table, rendering can fail with pagination.
You can reproduce this issue with the following operation.
```
sc.parallelize(1 to 10).map(value => (value ,value)).repartition(1).repartition(1).repartition(1).reduceByKey(_ + _).collect
```
And then, visit the corresponding job page.
There are 5 stages so show <5 stages in the paged table.
<img width="1440" alt="dag-rendering-issue1" src="https://user-images.githubusercontent.com/4736016/83376286-c29f3d00-a40c-11ea-891b-eb8f42afbb27.png">
<img width="1439" alt="dag-rendering-issue2" src="https://user-images.githubusercontent.com/4736016/83376288-c3d06a00-a40c-11ea-8bb2-38542e5010c1.png">

### Why are the changes needed?

This is a bug.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Newly added test case with following command.
`build/sbt -Dtest.default.exclude.tags= -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly org.apache.spark.ui.ChromeUISeleniumSuite -- -z SPARK-31882"`

Closes #28690 from sarutak/fix-dag-rendering-issue.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 271eb26c026490f7307ae888200bfc81a645592f)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../apache/spark/ui/static/spark-dag-viz.js   |  7 ++----
 .../org/apache/spark/ui/static/webui.js       |  7 +++++-
 .../scala/org/apache/spark/ui/UIUtils.scala   |  1 +
 .../spark/ui/RealBrowserUISeleniumSuite.scala | 22 +++++++++++++++++++
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index ae02defd9bb9c..fd4a48d2db330 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -216,7 +216,7 @@ function renderDagVizForJob(svgContainer) {
     var dot = metadata.select(".dot-file").text();
     var stageId = metadata.attr("stage-id");
     var containerId = VizConstants.graphPrefix + stageId;
-    var isSkipped = metadata.attr("skipped") == "true";
+    var isSkipped = metadata.attr("skipped") === "true";
     var container;
     if (isSkipped) {
       container = svgContainer
@@ -225,11 +225,8 @@ function renderDagVizForJob(svgContainer) {
         .attr("skipped", "true");
     } else {
       // Link each graph to the corresponding stage page (TODO: handle stage attempts)
-      // Use the link from the stage table so it also works for the history server
       var attemptId = 0;
-      var stageLink = d3.select("#stage-" + stageId + "-" + attemptId)
-        .select("a.name-link")
-        .attr("href");
+      var stageLink = uiRoot + appBasePath + "/stages/stage/?id=" + stageId + "&attempt=" + attemptId;
       container = svgContainer
         .append("a")
         .attr("xlink:href", stageLink)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index 0ba461f02317f..4f8409ca2b7c2 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -16,11 +16,16 @@
  */
 
 var uiRoot = "";
+var appBasePath = "";
 
 function setUIRoot(val) {
     uiRoot = val;
 }
 
+function setAppBasePath(path) {
+    appBasePath = path;
+}
+
 function collapseTablePageLoad(name, table){
   if (window.localStorage.getItem(name) == "true") {
     // Set it to false so that the click function can revert it
@@ -33,7 +38,7 @@ function collapseTable(thisName, table){
     var status = window.localStorage.getItem(thisName) == "true";
     status = !status;
 
-    var thisClass = '.' + thisName
+    var thisClass = '.' + thisName;
 
     // Expand the list of additional metrics.
     var tableDiv = $(thisClass).parent().find('.' + table);
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index aaaa21acfec17..87ab5436fa4b5 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -289,6 +289,7 @@ private[spark] object UIUtils extends Logging {
     <html>
       <head>
         {commonHeaderNodes(request)}
+        <script>setAppBasePath('{activeTab.basePath}')</script>
         {if (showVisualization) vizHeaderNodes(request) else Seq.empty}
         {if (useDataTables) dataTablesHeaderNodes(request) else Seq.empty}
         <link rel="shortcut icon"
diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
index 84f888267857e..06b6483717a65 100644
--- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -78,6 +78,28 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String)
     }
   }
 
+  test("SPARK-31882: Link URL for Stage DAGs should not depend on paged table.") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 100).map(v => (v, v)).repartition(10).reduceByKey(_ + _).collect
+
+      eventually(timeout(10.seconds), interval(50.microseconds)) {
+        val pathWithPagedTable =
+          "/jobs/job/?id=0&completedStage.page=2&completedStage.sort=Stage+Id&" +
+            "completedStage.desc=true&completedStage.pageSize=1#completed"
+        goToUi(sc, pathWithPagedTable)
+
+        // Open DAG Viz.
+        webDriver.findElement(By.id("job-dag-viz")).click()
+        val stages = webDriver.findElements(By.cssSelector("svg[class='job'] > a"))
+        stages.size() should be (3)
+
+        stages.get(0).getAttribute("href") should include ("/stages/stage/?id=0&attempt=0")
+        stages.get(1).getAttribute("href") should include ("/stages/stage/?id=1&attempt=0")
+        stages.get(2).getAttribute("href") should include ("/stages/stage/?id=2&attempt=0")
+      }
+    }
+  }
+
   /**
    * Create a test SparkContext with the SparkUI enabled.
    * It is safe to `get` the SparkUI directly from the SparkContext returned here.

From 8cea968596baec6c71fe2c2bd4f5469b33b583e9 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 3 Jun 2020 12:07:05 +0900
Subject: [PATCH 0776/1280] [SPARK-31895][PYTHON][SQL] Support
 DataFrame.explain(extended: str) case to be consistent with Scala side

### What changes were proposed in this pull request?

Scala:

```scala
scala> spark.range(10).explain("cost")
```
```
== Optimized Logical Plan ==
Range (0, 10, step=1, splits=Some(12)), Statistics(sizeInBytes=80.0 B)

== Physical Plan ==
*(1) Range (0, 10, step=1, splits=12)
```

PySpark:

```python
>>> spark.range(10).explain("cost")
```
```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/.../spark/python/pyspark/sql/dataframe.py", line 333, in explain
    raise TypeError(err_msg)
TypeError: extended (optional) should be provided as bool, got <class 'str'>
```

In addition, it is consistent with other codes too, for example, `DataFrame.sample` also can support `DataFrame.sample(1.0)` and `DataFrame.sample(False)`.

### Why are the changes needed?

To provide the consistent API support across APIs.

### Does this PR introduce _any_ user-facing change?

Nope, it's only changes in unreleased branches.
If this lands to master only, yes, users will be able to set `mode` as `df.explain("...")` in Spark 3.1.

After this PR:

```python
>>> spark.range(10).explain("cost")
```
```
== Optimized Logical Plan ==
Range (0, 10, step=1, splits=Some(12)), Statistics(sizeInBytes=80.0 B)

== Physical Plan ==
*(1) Range (0, 10, step=1, splits=12)
```

### How was this patch tested?

Unittest was added and manually tested as well to make sure:

```python
spark.range(10).explain(True)
spark.range(10).explain(False)
spark.range(10).explain("cost")
spark.range(10).explain(extended="cost")
spark.range(10).explain(mode="cost")
spark.range(10).explain()
spark.range(10).explain(True, "cost")
spark.range(10).explain(1.0)
```

Closes #28711 from HyukjinKwon/SPARK-31895.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e1d52011401c1989f26b230eb8c82adc63e147e7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/dataframe.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 6f4fdd34a5f44..8ba2ffab4a331 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -276,6 +276,8 @@ def explain(self, extended=None, mode=None):
         """Prints the (logical and physical) plans to the console for debugging purpose.
 
         :param extended: boolean, default ``False``. If ``False``, prints only the physical plan.
+            When this is a string without specifying the ``mode``, it works as the mode is
+            specified.
         :param mode: specifies the expected output format of plans.
 
             * ``simple``: Print only a physical plan.
@@ -306,12 +308,17 @@ def explain(self, extended=None, mode=None):
         Output [2]: [age#0, name#1]
         ...
 
+        >>> df.explain("cost")
+        == Optimized Logical Plan ==
+        ...Statistics...
+        ...
+
         .. versionchanged:: 3.0.0
            Added optional argument `mode` to specify the expected output format of plans.
         """
 
         if extended is not None and mode is not None:
-            raise Exception("extended and mode can not be specified simultaneously")
+            raise Exception("extended and mode should not be set together.")
 
         # For the no argument case: df.explain()
         is_no_argument = extended is None and mode is None
@@ -319,18 +326,22 @@ def explain(self, extended=None, mode=None):
         # For the cases below:
         #   explain(True)
         #   explain(extended=False)
-        is_extended_case = extended is not None and isinstance(extended, bool)
+        is_extended_case = isinstance(extended, bool) and mode is None
 
-        # For the mode specified: df.explain(mode="formatted")
-        is_mode_case = mode is not None and isinstance(mode, basestring)
+        # For the case when extended is mode:
+        #   df.explain("formatted")
+        is_extended_as_mode = isinstance(extended, basestring) and mode is None
 
-        if not is_no_argument and not (is_extended_case or is_mode_case):
-            if extended is not None:
-                err_msg = "extended (optional) should be provided as bool" \
-                    ", got {0}".format(type(extended))
-            else:  # For mode case
-                err_msg = "mode (optional) should be provided as str, got {0}".format(type(mode))
-            raise TypeError(err_msg)
+        # For the mode specified:
+        #   df.explain(mode="formatted")
+        is_mode_case = extended is None and isinstance(mode, basestring)
+
+        if not (is_no_argument or is_extended_case or is_extended_as_mode or is_mode_case):
+            argtypes = [
+                str(type(arg)) for arg in [extended, mode] if arg is not None]
+            raise TypeError(
+                "extended (optional) and mode (optional) should be a string "
+                "and bool; however, got [%s]." % ", ".join(argtypes))
 
         # Sets an explain mode depending on a given argument
         if is_no_argument:
@@ -339,6 +350,8 @@ def explain(self, extended=None, mode=None):
             explain_mode = "extended" if extended else "simple"
         elif is_mode_case:
             explain_mode = mode
+        elif is_extended_as_mode:
+            explain_mode = extended
 
         print(self._sc._jvm.PythonSQLUtils.explainString(self._jdf.queryExecution(), explain_mode))
 

From 1b7ae62bf443f20c94daa6ea5cabbd18f96b7919 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 3 Jun 2020 06:07:53 +0000
Subject: [PATCH 0777/1280] [SPARK-31879][SQL] Using GB as default Locale for
 datetime formatters

This PR switches the default Locale from the `US` to `GB` to change the behavior of the first day of the week from Sunday-started to Monday-started as same as v2.4

```sql
spark-sql> select to_timestamp('2020-1-1', 'YYYY-w-u');
2019-12-29 00:00:00
spark-sql> set spark.sql.legacy.timeParserPolicy=legacy;
spark.sql.legacy.timeParserPolicy	legacy
spark-sql> select to_timestamp('2020-1-1', 'YYYY-w-u');
2019-12-30 00:00:00
```

These week-based fields need Locale to express their semantics, the first day of the week varies from country to country.

From the Java doc of WeekFields
```java
    /**
     * Gets the first day-of-week.
     * <p>
     * The first day-of-week varies by culture.
     * For example, the US uses Sunday, while France and the ISO-8601 standard use Monday.
     * This method returns the first day using the standard {code DayOfWeek} enum.
     *
     * return the first day-of-week, not null
     */
    public DayOfWeek getFirstDayOfWeek() {
        return firstDayOfWeek;
    }
```

But for the SimpleDateFormat, the day-of-week is not localized

```
u	Day number of week (1 = Monday, ..., 7 = Sunday)	Number	1
```

Currently, the default locale we use is the US, so the result moved a day backward.

For other countries, please refer to [First Day of the Week in Different Countries](http://chartsbin.com/view/41671)

With this change, it restores the first day of week calculating for functions when using the default locale.

Yes, but the behavior change is used to restore the old one of v2.4

add unit tests

Closes #28692 from yaooqinn/SPARK-31879.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit c59f51bcc207725b8cbc4201df9367f874f5915c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateFormatter.scala      |  8 +++++++-
 .../sql/catalyst/util/TimestampFormatter.scala |  8 +++++++-
 .../resources/sql-tests/inputs/datetime.sql    |  4 ++++
 .../sql-tests/results/ansi/datetime.sql.out    | 18 +++++++++++++++++-
 .../sql-tests/results/datetime-legacy.sql.out  | 18 +++++++++++++++++-
 .../sql-tests/results/datetime.sql.out         | 18 +++++++++++++++++-
 6 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index c4cedf229d0ca..5711ea0d09a72 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -117,7 +117,13 @@ class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyD
 object DateFormatter {
   import LegacyDateFormats._
 
-  val defaultLocale: Locale = Locale.US
+  /**
+   * Before Spark 3.0, the first day-of-week is always Monday. Since Spark 3.0, it depends on the
+   * locale.
+   * We pick GB as the default locale instead of US, to be compatible with Spark 2.x, as US locale
+   * uses Sunday as the first day-of-week. See SPARK-31879.
+   */
+  val defaultLocale: Locale = new Locale("en", "GB")
 
   val defaultPattern: String = "yyyy-MM-dd"
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 6d1c5353b0151..252e7034228be 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -257,7 +257,13 @@ object LegacyDateFormats extends Enumeration {
 object TimestampFormatter {
   import LegacyDateFormats._
 
-  val defaultLocale: Locale = Locale.US
+  /**
+   * Before Spark 3.0, the first day-of-week is always Monday. Since Spark 3.0, it depends on the
+   * locale.
+   * We pick GB as the default locale instead of US, to be compatible with Spark 2.x, as US locale
+   * uses Sunday as the first day-of-week. See SPARK-31879.
+   */
+  val defaultLocale: Locale = new Locale("en", "GB")
 
   def defaultPattern(): String = s"${DateFormatter.defaultPattern} HH:mm:ss"
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 4eefa0f675202..e955c788a04d7 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -154,3 +154,7 @@ select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy
 select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
 select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss');
 select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd');
+
+-- SPARK-31879: the first day of week
+select date_format('2020-01-01', 'YYYY-MM-dd uu');
+select date_format('2020-01-01', 'YYYY-MM-dd uuuu');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index 43fe0a652324e..c3d10b0eaf0f0 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 112
+-- Number of queries: 114
 
 
 -- !query
@@ -965,3 +965,19 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2020-01-01', 'YYYY-MM-dd uu')
+-- !query schema
+struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uu):string>
+-- !query output
+2020-01-01 03
+
+
+-- !query
+select date_format('2020-01-01', 'YYYY-MM-dd uuuu')
+-- !query schema
+struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uuuu):string>
+-- !query output
+2020-01-01 Wednesday
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
index 71b106485896b..ac50c5b880c7b 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 112
+-- Number of queries: 114
 
 
 -- !query
@@ -920,3 +920,19 @@ select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
 struct<date_format(CAST(DATE '2018-11-17' AS TIMESTAMP), yyyyyyyyyyy-MM-dd):string>
 -- !query output
 00000002018-11-17
+
+
+-- !query
+select date_format('2020-01-01', 'YYYY-MM-dd uu')
+-- !query schema
+struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uu):string>
+-- !query output
+2020-01-01 03
+
+
+-- !query
+select date_format('2020-01-01', 'YYYY-MM-dd uuuu')
+-- !query schema
+struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uuuu):string>
+-- !query output
+2020-01-01 0003
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 9b1c8472fa75e..fd037db5c67ae 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 112
+-- Number of queries: 114
 
 
 -- !query
@@ -937,3 +937,19 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2020-01-01', 'YYYY-MM-dd uu')
+-- !query schema
+struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uu):string>
+-- !query output
+2020-01-01 03
+
+
+-- !query
+select date_format('2020-01-01', 'YYYY-MM-dd uuuu')
+-- !query schema
+struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uuuu):string>
+-- !query output
+2020-01-01 Wednesday

From 5a271c9df67419beae791aa1c28e695fc77e591d Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 3 Jun 2020 06:49:58 +0000
Subject: [PATCH 0778/1280] [SPARK-31892][SQL] Disable week-based date filed
 for parsing

This PR disables week-based date filed for parsing

closes #28674

1. It's an un-fixable behavior change to fill the gap between SimpleDateFormat and DateTimeFormater and backward-compatibility for different JDKs.A lot of effort has been made to prove it at https://github.com/apache/spark/pull/28674

2. The existing behavior itself in 2.4 is confusing, e.g.

```sql
spark-sql> select to_timestamp('1', 'w');
1969-12-28 00:00:00
spark-sql> select to_timestamp('1', 'u');
1970-01-05 00:00:00
```
  The 'u' here seems not to go to the Monday of the first week in week-based form or the first day of the year in non-week-based form but go to the Monday of the second week in week-based form.

And, e.g.
```sql
spark-sql> select to_timestamp('2020 2020', 'YYYY yyyy');
2020-01-01 00:00:00
spark-sql> select to_timestamp('2020 2020', 'yyyy YYYY');
2019-12-29 00:00:00
spark-sql> select to_timestamp('2020 2020 1', 'YYYY yyyy w');
NULL
spark-sql> select to_timestamp('2020 2020 1', 'yyyy YYYY w');
2019-12-29 00:00:00
```

  I think we don't need to introduce all the weird behavior from Java

3. The current test coverage for week-based date fields is almost 0%, which indicates that we've never imagined using it.

4. Avoiding JDK bugs

https://issues.apache.org/jira/browse/SPARK-31880

Yes, the 'Y/W/w/u/F/E' pattern cannot be used datetime parsing functions.

more tests added

Closes #28706 from yaooqinn/SPARK-31892.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit afe95bd9ad7a07c49deecf05f0a1000bb8f80caa)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              |  2 ++
 .../sql/catalyst/catalog/interface.scala      |  2 +-
 .../sql/catalyst/csv/CSVInferSchema.scala     |  2 +-
 .../sql/catalyst/csv/UnivocityGenerator.scala |  5 ++--
 .../sql/catalyst/csv/UnivocityParser.scala    |  5 ++--
 .../expressions/datetimeExpressions.scala     | 12 ++++----
 .../sql/catalyst/json/JacksonGenerator.scala  |  5 ++--
 .../sql/catalyst/json/JacksonParser.scala     |  5 ++--
 .../sql/catalyst/json/JsonInferSchema.scala   |  2 +-
 .../sql/catalyst/util/DateFormatter.scala     | 13 +++++----
 .../util/DateTimeFormatterHelper.scala        | 17 +++++++----
 .../catalyst/util/TimestampFormatter.scala    | 16 +++++-----
 .../expressions/DateExpressionsSuite.scala    | 29 +++++++++++++++++++
 .../util/DateTimeFormatterHelperSuite.scala   |  7 +++++
 .../spark/sql/util/DateFormatterSuite.scala   |  9 ++++--
 .../sql/util/TimestampFormatterSuite.scala    | 12 ++++----
 .../datasources/PartitioningUtils.scala       |  2 +-
 .../ParquetPartitionDiscoverySuite.scala      |  2 +-
 18 files changed, 101 insertions(+), 46 deletions(-)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 865b9470ed191..5859ad82525f2 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -136,6 +136,8 @@ The count of pattern letters determines the format.
   During formatting, all valid data will be output even it is in the optional section.
   During parsing, the whole section may be missing from the parsed string.
   An optional section is started by `[` and ended using `]` (or at the end of the pattern).
+  
+- Symbols of 'Y', 'W', 'w', 'E', 'u', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
 
 More details for the text style:
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index d02776b5d86f8..4e63ee7428d72 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -525,7 +525,7 @@ object CatalogColumnStat extends Logging {
     TimestampFormatter(
       format = "yyyy-MM-dd HH:mm:ss.SSSSSS",
       zoneId = ZoneOffset.UTC,
-      needVarLengthSecondFraction = isParsing)
+      isParsing = isParsing)
   }
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index a7c243537acb7..f0df18da8eed6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -35,7 +35,7 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable {
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
+    isParsing = true)
 
   private val decimalParser = if (options.locale == Locale.US) {
     // Special handling the default locale for backward compatibility
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
index 4990da2bf3797..a3ee129cd6d64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala
@@ -47,12 +47,13 @@ class UnivocityGenerator(
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = false)
+    isParsing = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
 
   private def makeConverter(dataType: DataType): ValueConverter = dataType match {
     case DateType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
index f2bb7db895ca2..3898eca79478e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala
@@ -90,12 +90,13 @@ class UnivocityParser(
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
+    isParsing = true)
   private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   private val csvFilters = new CSVFilters(filters, requiredSchema)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index a0cf4462e5bb6..62f38b315b109 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -643,7 +643,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
           format.toString,
           zoneId,
           legacyFormat = SIMPLE_DATE_FORMAT,
-          needVarLengthSecondFraction = false)
+          isParsing = false)
       }
     } else None
   }
@@ -654,7 +654,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
         format.toString,
         zoneId,
         legacyFormat = SIMPLE_DATE_FORMAT,
-        needVarLengthSecondFraction = false)
+        isParsing = false)
     } else {
       formatter.get
     }
@@ -799,7 +799,7 @@ abstract class ToTimestamp
         constFormat.toString,
         zoneId,
         legacyFormat = SIMPLE_DATE_FORMAT,
-        needVarLengthSecondFraction = true)
+        isParsing = true)
     } catch {
       case e: SparkUpgradeException => throw e
       case NonFatal(_) => null
@@ -838,7 +838,7 @@ abstract class ToTimestamp
                 formatString,
                 zoneId,
                 legacyFormat = SIMPLE_DATE_FORMAT,
-                needVarLengthSecondFraction = true)
+                isParsing = true)
                 .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor
             } catch {
               case e: SparkUpgradeException => throw e
@@ -981,7 +981,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
         constFormat.toString,
         zoneId,
         legacyFormat = SIMPLE_DATE_FORMAT,
-        needVarLengthSecondFraction = false)
+        isParsing = false)
     } catch {
       case e: SparkUpgradeException => throw e
       case NonFatal(_) => null
@@ -1014,7 +1014,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
                 f.toString,
                 zoneId,
                 legacyFormat = SIMPLE_DATE_FORMAT,
-                needVarLengthSecondFraction = false)
+                isParsing = false)
                 .format(time.asInstanceOf[Long] * MICROS_PER_SECOND))
           } catch {
             case e: SparkUpgradeException => throw e
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
index f3938feef0a35..fb0ca323af1ff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala
@@ -83,12 +83,13 @@ private[sql] class JacksonGenerator(
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = false)
+    isParsing = false)
   private val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = false)
 
   private def makeWriter(dataType: DataType): ValueWriter = dataType match {
     case NullType =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index c4f6121723491..e038f777c7a41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -61,12 +61,13 @@ class JacksonParser(
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
+    isParsing = true)
   private lazy val dateFormatter = DateFormatter(
     options.dateFormat,
     options.zoneId,
     options.locale,
-    legacyFormat = FAST_DATE_FORMAT)
+    legacyFormat = FAST_DATE_FORMAT,
+    isParsing = true)
 
   /**
    * Create a converter which converts the JSON documents held by the `JsonParser`
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
index 56b12784fd214..de396a4c63458 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala
@@ -43,7 +43,7 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable {
     options.zoneId,
     options.locale,
     legacyFormat = FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = true)
+    isParsing = true)
 
   /**
    * Infer the type of a collection of json records in three stages:
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index 5711ea0d09a72..e178164e9f38c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -41,7 +41,8 @@ class Iso8601DateFormatter(
     pattern: String,
     zoneId: ZoneId,
     locale: Locale,
-    legacyFormat: LegacyDateFormats.LegacyDateFormat)
+    legacyFormat: LegacyDateFormats.LegacyDateFormat,
+    isParsing: Boolean)
   extends DateFormatter with DateTimeFormatterHelper {
 
   @transient
@@ -131,12 +132,13 @@ object DateFormatter {
       format: Option[String],
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
-      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): DateFormatter = {
+      legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
+      isParsing: Boolean = true): DateFormatter = {
     val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
-      val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat)
+      val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat, isParsing)
       df.validatePatternString()
       df
     }
@@ -159,8 +161,9 @@ object DateFormatter {
       format: String,
       zoneId: ZoneId,
       locale: Locale,
-      legacyFormat: LegacyDateFormat): DateFormatter = {
-    getFormatter(Some(format), zoneId, locale, legacyFormat)
+      legacyFormat: LegacyDateFormat,
+      isParsing: Boolean): DateFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
   }
 
   def apply(format: String, zoneId: ZoneId): DateFormatter = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 5b9d8396530bc..ffa7cd438b29b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -89,9 +89,9 @@ trait DateTimeFormatterHelper {
   protected def getOrCreateFormatter(
       pattern: String,
       locale: Locale,
-      needVarLengthSecondFraction: Boolean = false): DateTimeFormatter = {
-    val newPattern = convertIncompatiblePattern(pattern)
-    val useVarLen = needVarLengthSecondFraction && newPattern.contains('S')
+      isParsing: Boolean = false): DateTimeFormatter = {
+    val newPattern = convertIncompatiblePattern(pattern, isParsing)
+    val useVarLen = isParsing && newPattern.contains('S')
     val key = (newPattern, locale, useVarLen)
     var formatter = cache.getIfPresent(key)
     if (formatter == null) {
@@ -227,6 +227,12 @@ private object DateTimeFormatterHelper {
     formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
   }
   final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
+  // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values
+  // to datetime, especially when they are mixed with other non-week-based ones
+  // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
+  // be directly resolved then the `q` do check for whether the month is valid, but if the date
+  // fields is incomplete, e.g. `yM`, the checking will be bypassed.
+  final val unsupportedLettersForParsing = Set('Y', 'W', 'w', 'E', 'u', 'F', 'q', 'Q')
   final val unsupportedPatternLengths = {
     // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
     // 2.4
@@ -246,7 +252,7 @@ private object DateTimeFormatterHelper {
    * @param pattern The input pattern.
    * @return The pattern for new parser
    */
-  def convertIncompatiblePattern(pattern: String): String = {
+  def convertIncompatiblePattern(pattern: String, isParsing: Boolean = false): String = {
     val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
       case (patternPart, index) =>
         // Text can be quoted using single quotes, we only check the non-quote parts.
@@ -255,7 +261,8 @@ private object DateTimeFormatterHelper {
     (pattern + " ").split("'").zipWithIndex.map {
       case (patternPart, index) =>
         if (index % 2 == 0) {
-          for (c <- patternPart if unsupportedLetters.contains(c)) {
+          for (c <- patternPart if unsupportedLetters.contains(c) ||
+            (isParsing && unsupportedLettersForParsing.contains(c))) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
           }
           for (style <- unsupportedPatternLengths if patternPart.contains(style)) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 252e7034228be..0873b85091eeb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -272,13 +272,13 @@ object TimestampFormatter {
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
       legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
-      needVarLengthSecondFraction: Boolean = false): TimestampFormatter = {
+      isParsing: Boolean = false): TimestampFormatter = {
     val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
     } else {
       val tf = new Iso8601TimestampFormatter(
-        pattern, zoneId, locale, legacyFormat, needVarLengthSecondFraction)
+        pattern, zoneId, locale, legacyFormat, isParsing)
       tf.validatePatternString()
       tf
     }
@@ -304,23 +304,23 @@ object TimestampFormatter {
       zoneId: ZoneId,
       locale: Locale,
       legacyFormat: LegacyDateFormat,
-      needVarLengthSecondFraction: Boolean): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, locale, legacyFormat, needVarLengthSecondFraction)
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
   }
 
   def apply(
       format: String,
       zoneId: ZoneId,
       legacyFormat: LegacyDateFormat,
-      needVarLengthSecondFraction: Boolean): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, needVarLengthSecondFraction)
+      isParsing: Boolean): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, isParsing)
   }
 
   def apply(
       format: String,
       zoneId: ZoneId,
-      needVarLengthSecondFraction: Boolean = false): TimestampFormatter = {
-    getFormatter(Some(format), zoneId, needVarLengthSecondFraction = needVarLengthSecondFraction)
+      isParsing: Boolean = false): TimestampFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index e00d65fa8678d..f3b9d6e12f352 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -1147,4 +1147,33 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
           Literal("yyyy-MM-dd'T'HH:mm:ss.SSSz")), "Fail to parse")
     }
   }
+
+  test("Disable week-based date fields and quarter fields for parsing") {
+
+    def checkSparkUpgrade(c: Char): Unit = {
+      checkExceptionInExpression[SparkUpgradeException](
+        new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, "3.0")
+      checkExceptionInExpression[SparkUpgradeException](
+        new ParseToDate(Literal("1"), Literal(c.toString)).child, "3.0")
+      checkExceptionInExpression[SparkUpgradeException](
+        ToUnixTimestamp(Literal("1"), Literal(c.toString)), "3.0")
+      checkExceptionInExpression[SparkUpgradeException](
+        UnixTimestamp(Literal("1"), Literal(c.toString)), "3.0")
+    }
+
+    def checkNullify(c: Char): Unit = {
+      checkEvaluation(new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, null)
+      checkEvaluation(new ParseToDate(Literal("1"), Literal(c.toString)).child, null)
+      checkEvaluation(ToUnixTimestamp(Literal("1"), Literal(c.toString)), null)
+      checkEvaluation(UnixTimestamp(Literal("1"), Literal(c.toString)), null)
+    }
+
+    Seq('Y', 'W', 'w', 'E', 'u', 'F').foreach { l =>
+      checkSparkUpgrade(l)
+    }
+
+    Seq('q', 'Q').foreach { l =>
+      checkNullify(l)
+    }
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index f0cc4d149c165..c68bdacb13af7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -40,6 +40,13 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
       val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
       assert(e.getMessage === s"Illegal pattern character: $l")
     }
+    unsupportedLettersForParsing.foreach { l =>
+      val e = intercept[IllegalArgumentException] {
+        convertIncompatiblePattern(s"$l", isParsing = true)
+      }
+      assert(e.getMessage === s"Illegal pattern character: $l")
+      assert(convertIncompatiblePattern(s"$l").nonEmpty)
+    }
     unsupportedPatternLengths.foreach { style =>
       val e1 = intercept[IllegalArgumentException] {
         convertIncompatiblePattern(s"yyyy-MM-dd $style")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
index 7d503cc091172..22a1396d5efdc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
@@ -72,7 +72,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
                   DateFormatter.defaultPattern,
                   getZoneId(timeZone),
                   DateFormatter.defaultLocale,
-                  legacyFormat)
+                  legacyFormat,
+                  isParsing = false)
                 val days = formatter.parse(date)
                 assert(date === formatter.format(days))
                 assert(date === formatter.format(daysToLocalDate(days)))
@@ -106,7 +107,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
                   DateFormatter.defaultPattern,
                   getZoneId(timeZone),
                   DateFormatter.defaultLocale,
-                  legacyFormat)
+                  legacyFormat,
+                  isParsing = false)
                 val date = formatter.format(days)
                 val parsed = formatter.parse(date)
                 assert(days === parsed)
@@ -173,7 +175,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper {
               DateFormatter.defaultPattern,
               getZoneId(timeZone),
               DateFormatter.defaultLocale,
-              legacyFormat)
+              legacyFormat,
+              isParsing = false)
             assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1))
             assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01")
             assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index a72dfb97cb85c..9870d5e22c25f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -48,7 +48,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       val formatter = TimestampFormatter(
         "yyyy-MM-dd'T'HH:mm:ss.SSSSSS",
         getZoneId(zoneId),
-        needVarLengthSecondFraction = true)
+        isParsing = true)
       val microsSinceEpoch = formatter.parse(localDate)
       assert(microsSinceEpoch === expectedMicros(zoneId))
     }
@@ -73,7 +73,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
           // Test only FAST_DATE_FORMAT because other legacy formats don't support formatting
           // in microsecond precision.
           LegacyDateFormats.FAST_DATE_FORMAT,
-          needVarLengthSecondFraction = false),
+          isParsing = false),
         TimestampFormatter.getFractionFormatter(getZoneId(zoneId))).foreach { formatter =>
         val timestamp = formatter.format(microsSinceEpoch)
         assert(timestamp === expectedTimestamp(zoneId))
@@ -98,7 +98,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         outstandingZoneIds.foreach { zoneId =>
           val timestamp = TimestampFormatter(pattern, zoneId).format(micros)
           val parsed = TimestampFormatter(
-            pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
+            pattern, zoneId, isParsing = true).parse(timestamp)
           assert(micros === parsed)
         }
       }
@@ -119,7 +119,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       outstandingZoneIds.foreach { zoneId =>
         val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
         val micros = TimestampFormatter(
-          pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp)
+          pattern, zoneId, isParsing = true).parse(timestamp)
         val formatted = TimestampFormatter(pattern, zoneId).format(micros)
         assert(timestamp === formatted)
       }
@@ -182,7 +182,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   test("parsing timestamp strings with various seconds fractions") {
     outstandingZoneIds.foreach { zoneId =>
       def check(pattern: String, input: String, reference: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId, needVarLengthSecondFraction = true)
+        val formatter = TimestampFormatter(pattern, zoneId, isParsing = true)
         val expected = stringToTimestamp(UTF8String.fromString(reference), zoneId).get
         val actual = formatter.parse(input)
         assert(actual === expected)
@@ -289,7 +289,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
                   zoneId,
                   TimestampFormatter.defaultLocale,
                   legacyFormat,
-                  needVarLengthSecondFraction = false)
+                  isParsing = false)
               }.toSeq :+ TimestampFormatter.getFractionFormatter(zoneId)
               formatters.foreach { formatter =>
                 assert(microsToInstant(formatter.parse("1000-01-01 01:02:03"))
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index 292ac6db04baf..f7e225b0cdc96 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -134,7 +134,7 @@ object PartitioningUtils {
     val timestampFormatter = TimestampFormatter(
       timestampPartitionPattern,
       zoneId,
-      needVarLengthSecondFraction = true)
+      isParsing = true)
     // First, we need to parse every partition's path and see if we can find partition values.
     val (partitionValues, optDiscoveredBasePaths) = paths.map { path =>
       parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
index d3301ced2ba19..32a9558e91f10 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala
@@ -60,7 +60,7 @@ abstract class ParquetPartitionDiscoverySuite
   val timeZoneId = ZoneId.systemDefault()
   val df = DateFormatter(timeZoneId)
   val tf = TimestampFormatter(
-    timestampPartitionPattern, timeZoneId, needVarLengthSecondFraction = true)
+    timestampPartitionPattern, timeZoneId, isParsing = true)
 
   protected override def beforeAll(): Unit = {
     super.beforeAll()

From f6a77c31fc47ea4e3c40576ed587b4a75ecaeda1 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 3 Jun 2020 01:15:36 -0700
Subject: [PATCH 0779/1280] [SPARK-31886][WEBUI] Fix the wrong coloring of
 nodes in DAG-viz

### What changes were proposed in this pull request?

This PR fixes a wrong coloring issue in the DAG-viz.
In the Job Page and Stage Page, nodes which are associated with "barrier mode" in the DAG-viz will be colored pale green.
But, with some type of jobs, nodes which are not associated with the mode will also colored.
You can reproduce with the following operation.
```
sc.parallelize(1 to 10).barrier.mapPartitions(identity).repartition(1).collect()
```
<img width="376" alt="wrong-coloring" src="https://user-images.githubusercontent.com/4736016/83403670-1711df00-a444-11ea-9457-c683f75bc566.png">

In the screen shot above, `repartition` in `Stage 1` is not associated with barrier mode so the corresponding node should not be colored pale green.

The cause of this issue is the logic which chooses HTML elements to be colored is wrong.
The logic chooses such elements based on whether each element is associated with a style class (`clusterId` in the code).
But when an operation crosses over shuffle (like `repartition` above), a `clusterId` can be duplicated and non-barrier mode node is also associated with the same `clusterId`.

### Why are the changes needed?

This is a bug.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Newly added test case with the following command.
```
build/sbt -Dtest.default.exclude.tags= -Dspark.test.webdriver.chrome.driver=/path/to/chromedriver "testOnly org.apache.spark.ui.ChromeUISeleniumSuite -- -z SPARK-31886"
```

Closes #28694 from sarutak/fix-wrong-barrier-color.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 8ed93c9355bc2af6fe456d88aa693c8db69d0bbf)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../apache/spark/ui/static/spark-dag-viz.js   |  8 ++++--
 .../spark/ui/RealBrowserUISeleniumSuite.scala | 28 +++++++++++++++++++
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
index fd4a48d2db330..474c453643365 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js
@@ -173,9 +173,11 @@ function renderDagViz(forJob) {
   });
 
   metadataContainer().selectAll(".barrier-rdd").each(function() {
-    var rddId = d3.select(this).text().trim();
-    var clusterId = VizConstants.clusterPrefix + rddId;
-    svg.selectAll("g." + clusterId).classed("barrier", true)
+    var opId = d3.select(this).text().trim();
+    var opClusterId = VizConstants.clusterPrefix + opId;
+    var stageId = $(this).parents(".stage-metadata").attr("stage-id");
+    var stageClusterId = VizConstants.graphPrefix + stageId;
+    svg.selectAll("g[id=" + stageClusterId + "] g." + opClusterId).classed("barrier", true)
   });
 
   resizeSvg(svg);
diff --git a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
index 06b6483717a65..4b018f69b1660 100644
--- a/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/RealBrowserUISeleniumSuite.scala
@@ -100,6 +100,34 @@ abstract class RealBrowserUISeleniumSuite(val driverProp: String)
     }
   }
 
+  test("SPARK-31886: Color barrier execution mode RDD correctly") {
+    withSpark(newSparkContext()) { sc =>
+      sc.parallelize(1 to 10).barrier.mapPartitions(identity).repartition(1).collect()
+
+      eventually(timeout(10.seconds), interval(50.milliseconds)) {
+        goToUi(sc, "/jobs/job/?id=0")
+        webDriver.findElement(By.id("job-dag-viz")).click()
+
+        val stage0 = webDriver.findElement(By.cssSelector("g[id='graph_0']"))
+        val stage1 = webDriver.findElement(By.cssSelector("g[id='graph_1']"))
+        val barrieredOps = webDriver.findElements(By.className("barrier-rdd")).iterator()
+
+        while (barrieredOps.hasNext) {
+          val barrieredOpId = barrieredOps.next().getAttribute("innerHTML")
+          val foundInStage0 =
+            stage0.findElements(
+              By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+          assert(foundInStage0.size === 1)
+
+          val foundInStage1 =
+            stage1.findElements(
+              By.cssSelector("g.barrier.cluster.cluster_" + barrieredOpId))
+          assert(foundInStage1.size === 0)
+        }
+      }
+    }
+  }
+
   /**
    * Create a test SparkContext with the SparkUI enabled.
    * It is safe to `get` the SparkUI directly from the SparkContext returned here.

From 4e9eafd75864e82ec9db2743bf8f7f06faeba528 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 3 Jun 2020 02:17:25 -0700
Subject: [PATCH 0780/1280] [SPARK-31881][K8S][TESTS][FOLLOWUP] Activate
 hadoop-2.7 by default in K8S IT

### What changes were proposed in this pull request?

This PR aims to activate `hadoop-2.7` profile by default in Kubernetes IT module.

### Why are the changes needed?

While SPARK-31881 added Hadoop 3.2 support, one default test dependency was moved to `hadoop-2.7` profile. It works when we give one of `hadoop-2.7` and `hadoop-3.2`, but it fails when we don't give any profile.

**BEFORE**
```
$ mvn test-compile -pl resource-managers/kubernetes/integration-tests -Pkubernetes-integration-tests
...
[ERROR] [Error] /APACHE/spark-merge/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala:23:
object amazonaws is not a member of package com
```

**AFTER**
```
$ mvn test-compile -pl resource-managers/kubernetes/integration-tests -Pkubernetes-integration-tests
..
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
```

The default activated profile will be override when we give `hadoop-3.2`.
```
$ mvn help:active-profiles -Pkubernetes-integration-tests
...
Active Profiles for Project 'org.apache.spark:spark-kubernetes-integration-tests_2.12:jar:3.1.0-SNAPSHOT':

The following profiles are active:

 - hadoop-2.7 (source: org.apache.spark:spark-kubernetes-integration-tests_2.12:3.1.0-SNAPSHOT)
 - kubernetes-integration-tests (source: org.apache.spark:spark-parent_2.12:3.1.0-SNAPSHOT)
 - test-java-home (source: org.apache.spark:spark-parent_2.12:3.1.0-SNAPSHOT)
```
```
$ mvn help:active-profiles -Pkubernetes-integration-tests -Phadoop-3.2
...
Active Profiles for Project 'org.apache.spark:spark-kubernetes-integration-tests_2.12:jar:3.1.0-SNAPSHOT':

The following profiles are active:

 - hadoop-3.2 (source: org.apache.spark:spark-kubernetes-integration-tests_2.12:3.1.0-SNAPSHOT)
 - hadoop-3.2 (source: org.apache.spark:spark-parent_2.12:3.1.0-SNAPSHOT)
 - kubernetes-integration-tests (source: org.apache.spark:spark-parent_2.12:3.1.0-SNAPSHOT)
 - test-java-home (source: org.apache.spark:spark-parent_2.12:3.1.0-SNAPSHOT)
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins UT and IT.

Currently, all Jenkins build and tests (UT & IT) passes without this patch. This should be tested manually with the above command.

`hadoop-3.2` K8s IT also passed like the following.
```
KubernetesSuite:
- Run SparkPi with no resources
- Run SparkPi with a very long application name.
- Use SparkLauncher.NO_RESOURCE
- Run SparkPi with a master URL without a scheme.
- Run SparkPi with an argument.
- Run SparkPi with custom labels, annotations, and environment variables.
- All pods have the same service account by default
- Run extraJVMOptions check on driver
- Run SparkRemoteFileTest using a remote data file
- Run SparkPi with env and mount secrets.
- Run PySpark on simple pi.py example
- Run PySpark with Python2 to test a pyfiles example
- Run PySpark with Python3 to test a pyfiles example
- Run PySpark with memory customization
- Run in client mode.
- Start pod creation from template
- PVs with local storage
- Launcher client dependencies
- Test basic decommissioning
Run completed in 8 minutes, 33 seconds.
Total number of tests run: 19
Suites: completed 2, aborted 0
Tests: succeeded 19, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```

Closes #28716 from dongjoon-hyun/SPARK-31881-2.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e5b9b862e6011f37dfc0f646d6c3ae8e545e2cd6)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 resource-managers/kubernetes/integration-tests/pom.xml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index bda936c1df41a..6c258a7ce9042 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -186,6 +186,9 @@
   <profiles>
     <profile>
       <id>hadoop-2.7</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
       <dependencies>
         <dependency>
           <groupId>com.amazonaws</groupId>

From 85a489103465d6dc36d9565e71e38e73fb8e097f Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 3 Jun 2020 12:00:20 +0000
Subject: [PATCH 0781/1280] [SPARK-31878][SQL] Create date formatter only once
 in `HiveResult`

### What changes were proposed in this pull request?
1. Replace `def dateFormatter` to `val dateFormatter`.
2. Modify the `date formatting in hive result` test in `HiveResultSuite` to check modified code on various time zones.

### Why are the changes needed?
To avoid creation of `DateFormatter` per every incoming date in `HiveResult.toHiveString`. This should eliminate unnecessary creation of `SimpleDateFormat` instances and compilation of the default pattern `yyyy-MM-dd`. The changes can speed up processing of legacy date values of the `java.sql.Date` type which is collected by default.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Modified a test in `HiveResultSuite`.

Closes #28687 from MaxGekk/HiveResult-val-dateFormatter.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 125a89ce0880ab4c53dcb1b879f1d65ff0f589df)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/HiveResult.scala      | 24 +++++++++++++++----
 .../spark/sql/execution/HiveResultSuite.scala | 22 ++++++++++-------
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index 73484a212c169..c6b16fb20ed59 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -19,10 +19,10 @@ package org.apache.spark.sql.execution
 
 import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
-import java.time.{Instant, LocalDate}
+import java.time.{Instant, LocalDate, ZoneOffset}
 
 import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, LegacyDateFormats, TimestampFormatter}
 import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand, ShowViewsCommand}
 import org.apache.spark.sql.execution.datasources.v2.{DescribeTableExec, ShowTablesExec}
 import org.apache.spark.sql.internal.SQLConf
@@ -72,9 +72,23 @@ object HiveResult {
     }
   }
 
-  private def zoneId = DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone)
-  private def dateFormatter = DateFormatter(zoneId)
-  private def timestampFormatter = TimestampFormatter.getFractionFormatter(zoneId)
+  // We can create the date formatter only once because it does not depend on Spark's
+  // session time zone controlled by the SQL config `spark.sql.session.timeZone`.
+  // The `zoneId` parameter is used only in parsing of special date values like `now`,
+  // `yesterday` and etc. but not in date formatting. While formatting of:
+  // - `java.time.LocalDate`, zone id is not used by `DateTimeFormatter` at all.
+  // - `java.sql.Date`, the date formatter delegates formatting to the legacy formatter
+  //   which uses the default system time zone `TimeZone.getDefault`. This works correctly
+  //   due to `DateTimeUtils.toJavaDate` which is based on the system time zone too.
+  private val dateFormatter = DateFormatter(
+    format = DateFormatter.defaultPattern,
+    // We can set any time zone id. UTC was taken for simplicity.
+    zoneId = ZoneOffset.UTC,
+    locale = DateFormatter.defaultLocale,
+    // Use `FastDateFormat` as the legacy formatter because it is thread-safe.
+    legacyFormat = LegacyDateFormats.FAST_DATE_FORMAT)
+  private def timestampFormatter = TimestampFormatter.getFractionFormatter(
+    DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
 
   /** Formats a datum (based on the given data type) and returns the string representation. */
   def toHiveString(a: (Any, DataType), nested: Boolean = false): String = a match {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
index 5e81c74420fd0..a0b212d2cf6fd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/HiveResultSuite.scala
@@ -17,21 +17,27 @@
 
 package org.apache.spark.sql.execution
 
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.connector.InMemoryTableCatalog
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePoint, ExamplePointUDT, SharedSparkSession}
 
 class HiveResultSuite extends SharedSparkSession {
   import testImplicits._
 
   test("date formatting in hive result") {
-    val dates = Seq("2018-12-28", "1582-10-03", "1582-10-04", "1582-10-15")
-    val df = dates.toDF("a").selectExpr("cast(a as date) as b")
-    val executedPlan1 = df.queryExecution.executedPlan
-    val result = HiveResult.hiveResultString(executedPlan1)
-    assert(result == dates)
-    val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
-    val result2 = HiveResult.hiveResultString(executedPlan2)
-    assert(result2 == dates.map(x => s"[$x]"))
+    DateTimeTestUtils.outstandingTimezonesIds.foreach { zoneId =>
+      withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> zoneId) {
+        val dates = Seq("2018-12-28", "1582-10-03", "1582-10-04", "1582-10-15")
+        val df = dates.toDF("a").selectExpr("cast(a as date) as b")
+        val executedPlan1 = df.queryExecution.executedPlan
+        val result = HiveResult.hiveResultString(executedPlan1)
+        assert(result == dates)
+        val executedPlan2 = df.selectExpr("array(b)").queryExecution.executedPlan
+        val result2 = HiveResult.hiveResultString(executedPlan2)
+        assert(result2 == dates.map(x => s"[$x]"))
+      }
+    }
   }
 
   test("timestamp formatting in hive result") {

From 7cdd0848ba3c8fa58d97e428d64f5a61d2391b98 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 3 Jun 2020 20:11:41 +0800
Subject: [PATCH 0782/1280] fix compilation

---
 .../main/scala/org/apache/spark/sql/execution/HiveResult.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
index c6b16fb20ed59..9f99bf5011569 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala
@@ -86,7 +86,8 @@ object HiveResult {
     zoneId = ZoneOffset.UTC,
     locale = DateFormatter.defaultLocale,
     // Use `FastDateFormat` as the legacy formatter because it is thread-safe.
-    legacyFormat = LegacyDateFormats.FAST_DATE_FORMAT)
+    legacyFormat = LegacyDateFormats.FAST_DATE_FORMAT,
+    isParsing = false)
   private def timestampFormatter = TimestampFormatter.getFractionFormatter(
     DateTimeUtils.getZoneId(SQLConf.get.sessionLocalTimeZone))
 

From 47edae9d848d745d0aef99cfd893d055c8fc6457 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 3 Jun 2020 13:30:22 +0000
Subject: [PATCH 0783/1280] [SPARK-31896][SQL] Handle am-pm timestamp parsing
 when hour is missing

### What changes were proposed in this pull request?

This PR set the hour to 12/0 when the AMPM_OF_DAY field exists

### Why are the changes needed?

When the hour is absent but the am-pm is present, the time is incorrect for pm

### Does this PR introduce _any_ user-facing change?
yes, the change is user-facing but to change back to 2.4 to keep backward compatibility

e.g.
```sql
spark-sql> select to_timestamp('33:33 PM', 'mm:ss a');
1970-01-01 12:33:33
spark-sql> select to_timestamp('33:33 AM', 'mm:ss a');
1970-01-01 00:33:33

```

otherwise, the results are all `1970-01-01 00:33:33`

### How was this patch tested?

add unit tests

Closes #28713 from yaooqinn/SPARK-31896.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit afcc14c6d27f9e0bd113e0d86b64dc6fa4eed551)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../util/DateTimeFormatterHelper.scala        |  8 +++++++
 .../expressions/DateExpressionsSuite.scala    | 10 ++++++++
 .../sql/util/TimestampFormatterSuite.scala    | 24 ++++++++++++++++---
 3 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index ffa7cd438b29b..eeb56aa9821ca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -62,7 +62,15 @@ trait DateTimeFormatterHelper {
       accessor.get(ChronoField.HOUR_OF_DAY)
     } else if (accessor.isSupported(ChronoField.HOUR_OF_AMPM)) {
       // When we reach here, it means am/pm is not specified. Here we assume it's am.
+      // All of CLOCK_HOUR_OF_AMPM(h)/HOUR_OF_DAY(H)/CLOCK_HOUR_OF_DAY(k)/HOUR_OF_AMPM(K) will
+      // be resolved to HOUR_OF_AMPM here, we do not need to handle them separately
       accessor.get(ChronoField.HOUR_OF_AMPM)
+    } else if (accessor.isSupported(ChronoField.AMPM_OF_DAY) &&
+      accessor.get(ChronoField.AMPM_OF_DAY) == 1) {
+      // When reach here, the `hour` part is missing, and PM is specified.
+      // None of CLOCK_HOUR_OF_AMPM(h)/HOUR_OF_DAY(H)/CLOCK_HOUR_OF_DAY(k)/HOUR_OF_AMPM(K) is
+      // specified
+      12
     } else {
       0
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index f3b9d6e12f352..dcb475924fb42 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -1176,4 +1176,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       checkNullify(l)
     }
   }
+
+
+  test("SPARK-31896: Handle am-pm timestamp parsing when hour is missing") {
+    checkEvaluation(
+      new ParseToTimestamp(Literal("PM"), Literal("a")).child,
+      Timestamp.valueOf("1970-01-01 12:00:00.0"))
+    checkEvaluation(
+      new ParseToTimestamp(Literal("11:11 PM"), Literal("mm:ss a")).child,
+      Timestamp.valueOf("1970-01-01 12:11:11.0"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
index 9870d5e22c25f..efd97a4195566 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
@@ -386,9 +386,11 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("missing am/pm field") {
-    val formatter = TimestampFormatter("yyyy hh:mm:ss", UTC)
-    val micros = formatter.parse("2009 11:30:01")
-    assert(micros === date(2009, 1, 1, 11, 30, 1))
+    Seq("HH", "hh", "KK", "kk").foreach { hour =>
+      val formatter = TimestampFormatter(s"yyyy $hour:mm:ss", UTC)
+      val micros = formatter.parse("2009 11:30:01")
+      assert(micros === date(2009, 1, 1, 11, 30, 1))
+    }
   }
 
   test("missing time fields") {
@@ -397,6 +399,22 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     assert(micros === date(2009, 1, 1, 11))
   }
 
+  test("missing hour field") {
+    val f1 = TimestampFormatter("mm:ss a", UTC)
+    val t1 = f1.parse("30:01 PM")
+    assert(t1 === date(1970, 1, 1, 12, 30, 1))
+    val t2 = f1.parse("30:01 AM")
+    assert(t2 === date(1970, 1, 1, 0, 30, 1))
+    val f2 = TimestampFormatter("mm:ss", UTC)
+    val t3 = f2.parse("30:01")
+    assert(t3 === date(1970, 1, 1, 0, 30, 1))
+    val f3 = TimestampFormatter("a", UTC)
+    val t4 = f3.parse("PM")
+    assert(t4 === date(1970, 1, 1, 12))
+    val t5 = f3.parse("AM")
+    assert(t5 === date(1970))
+  }
+
   test("explicitly forbidden datetime patterns") {
     // not support by the legacy one too
     Seq("QQQQQ", "qqqqq", "A", "c", "e", "n", "N", "p").foreach { pattern =>

From 08198fa10aa45368b90aad2e2eb9eb216461032d Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 4 Jun 2020 02:01:49 +0800
Subject: [PATCH 0784/1280] Revert "[SPARK-31879][SQL] Using GB as default
 Locale for datetime formatters"

This reverts commit 1b7ae62bf443f20c94daa6ea5cabbd18f96b7919.
---
 .../sql/catalyst/util/DateFormatter.scala      |  8 +-------
 .../sql/catalyst/util/TimestampFormatter.scala |  8 +-------
 .../resources/sql-tests/inputs/datetime.sql    |  4 ----
 .../sql-tests/results/ansi/datetime.sql.out    | 18 +-----------------
 .../sql-tests/results/datetime-legacy.sql.out  | 18 +-----------------
 .../sql-tests/results/datetime.sql.out         | 18 +-----------------
 6 files changed, 5 insertions(+), 69 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index e178164e9f38c..b3347ebe2ec78 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -118,13 +118,7 @@ class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyD
 object DateFormatter {
   import LegacyDateFormats._
 
-  /**
-   * Before Spark 3.0, the first day-of-week is always Monday. Since Spark 3.0, it depends on the
-   * locale.
-   * We pick GB as the default locale instead of US, to be compatible with Spark 2.x, as US locale
-   * uses Sunday as the first day-of-week. See SPARK-31879.
-   */
-  val defaultLocale: Locale = new Locale("en", "GB")
+  val defaultLocale: Locale = Locale.US
 
   val defaultPattern: String = "yyyy-MM-dd"
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 0873b85091eeb..e8866d7fd74cc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -257,13 +257,7 @@ object LegacyDateFormats extends Enumeration {
 object TimestampFormatter {
   import LegacyDateFormats._
 
-  /**
-   * Before Spark 3.0, the first day-of-week is always Monday. Since Spark 3.0, it depends on the
-   * locale.
-   * We pick GB as the default locale instead of US, to be compatible with Spark 2.x, as US locale
-   * uses Sunday as the first day-of-week. See SPARK-31879.
-   */
-  val defaultLocale: Locale = new Locale("en", "GB")
+  val defaultLocale: Locale = Locale.US
 
   def defaultPattern(): String = s"${DateFormatter.defaultPattern} HH:mm:ss"
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index e955c788a04d7..4eefa0f675202 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -154,7 +154,3 @@ select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy
 select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
 select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss');
 select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd');
-
--- SPARK-31879: the first day of week
-select date_format('2020-01-01', 'YYYY-MM-dd uu');
-select date_format('2020-01-01', 'YYYY-MM-dd uuuu');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index c3d10b0eaf0f0..43fe0a652324e 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 114
+-- Number of queries: 112
 
 
 -- !query
@@ -965,19 +965,3 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
-
--- !query
-select date_format('2020-01-01', 'YYYY-MM-dd uu')
--- !query schema
-struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uu):string>
--- !query output
-2020-01-01 03
-
-
--- !query
-select date_format('2020-01-01', 'YYYY-MM-dd uuuu')
--- !query schema
-struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uuuu):string>
--- !query output
-2020-01-01 Wednesday
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
index ac50c5b880c7b..71b106485896b 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 114
+-- Number of queries: 112
 
 
 -- !query
@@ -920,19 +920,3 @@ select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
 struct<date_format(CAST(DATE '2018-11-17' AS TIMESTAMP), yyyyyyyyyyy-MM-dd):string>
 -- !query output
 00000002018-11-17
-
-
--- !query
-select date_format('2020-01-01', 'YYYY-MM-dd uu')
--- !query schema
-struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uu):string>
--- !query output
-2020-01-01 03
-
-
--- !query
-select date_format('2020-01-01', 'YYYY-MM-dd uuuu')
--- !query schema
-struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uuuu):string>
--- !query output
-2020-01-01 0003
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index fd037db5c67ae..9b1c8472fa75e 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 114
+-- Number of queries: 112
 
 
 -- !query
@@ -937,19 +937,3 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
-
--- !query
-select date_format('2020-01-01', 'YYYY-MM-dd uu')
--- !query schema
-struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uu):string>
--- !query output
-2020-01-01 03
-
-
--- !query
-select date_format('2020-01-01', 'YYYY-MM-dd uuuu')
--- !query schema
-struct<date_format(CAST(2020-01-01 AS TIMESTAMP), YYYY-MM-dd uuuu):string>
--- !query output
-2020-01-01 Wednesday

From 22d5d0368b9084884071af69106d67e50e6cc07f Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 3 Jun 2020 19:08:36 +0000
Subject: [PATCH 0785/1280] [SPARK-29947][SQL][FOLLOWUP] ResolveRelations
 should return relations with fresh attribute IDs

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/26589, which caches the table relations to speed up the table lookup. However, it brings some side effects: the rule `ResolveRelations` may return exactly the same relations, while before it always returns relations with fresh attribute IDs.

This PR is to eliminate this side effect.

### Why are the changes needed?

There is no bug report yet, but this side effect may impact things like self-join. It's better to restore the 2.4 behavior and always return refresh relations.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

N/A

Closes #28717 from cloud-fan/fix.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit dc0709fa0ca75751d2de4ef95ad077f3e805a6ac)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/catalyst/analysis/Analyzer.scala    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 654cf42a07c60..6fb103e46a77b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1007,7 +1007,7 @@ class Analyzer(
     private def lookupRelation(identifier: Seq[String]): Option[LogicalPlan] = {
       expandRelationName(identifier) match {
         case SessionCatalogAndIdentifier(catalog, ident) =>
-          def loaded = CatalogV2Util.loadTable(catalog, ident).map {
+          lazy val loaded = CatalogV2Util.loadTable(catalog, ident).map {
             case v1Table: V1Table =>
               v1SessionCatalog.getRelation(v1Table.v1Table)
             case table =>
@@ -1016,7 +1016,12 @@ class Analyzer(
                 DataSourceV2Relation.create(table, Some(catalog), Some(ident)))
           }
           val key = catalog.name +: ident.namespace :+ ident.name
-          Option(AnalysisContext.get.relationCache.getOrElseUpdate(key, loaded.orNull))
+          AnalysisContext.get.relationCache.get(key).map(_.transform {
+            case multi: MultiInstanceRelation => multi.newInstance()
+          }).orElse {
+            loaded.foreach(AnalysisContext.get.relationCache.update(key, _))
+            loaded
+          }
         case _ => None
       }
     }

From 3bb08245a9c5e9f9da6f48e65e7dc0f80ad96b4a Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Wed, 3 Jun 2020 18:06:13 -0500
Subject: [PATCH 0786/1280] [SPARK-31853][DOCS] Mention removal of params
 mixins setter in migration guide

### What changes were proposed in this pull request?
The Pyspark Migration Guide needs to mention a breaking change of the Pyspark ML API.

### Why are the changes needed?
In SPARK-29093, all setters have been removed from `Params` mixins in `pyspark.ml.param.shared`. Those setters had been part of the public pyspark ML API, hence this is a breaking change.

### Does this PR introduce _any_ user-facing change?
Only documentation.

### How was this patch tested?
Visually.

Closes #28663 from EnricoMi/branch-pyspark-migration-guide-setters.

Authored-by: Enrico Minack <github@enrico.minack.dev>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 4bbe3c2bb49030256d7e4f6941dd5629ee6d5b66)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/pyspark-migration-guide.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/pyspark-migration-guide.md b/docs/pyspark-migration-guide.md
index 6f0fbbfb78de8..2c9ea410f217e 100644
--- a/docs/pyspark-migration-guide.md
+++ b/docs/pyspark-migration-guide.md
@@ -45,6 +45,8 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 
 - As of Spark 3.0, `Row` field names are no longer sorted alphabetically when constructing with named arguments for Python versions 3.6 and above, and the order of fields will match that as entered. To enable sorted fields by default, as in Spark 2.4, set the environment variable `PYSPARK_ROW_FIELD_SORTING_ENABLED` to `true` for both executors and driver - this environment variable must be consistent on all executors and driver; otherwise, it may cause failures or incorrect answers. For Python versions less than 3.6, the field names will be sorted alphabetically as the only option.
 
+- In Spark 3.0, `pyspark.ml.param.shared.Has*` mixins do not provide any `set*(self, value)` setter methods anymore, use the respective `self.set(self.*, value)` instead. See [SPARK-29093](https://issues.apache.org/jira/browse/SPARK-29093) for details.
+
 ## Upgrading from PySpark 2.3 to 2.4
 
   - In PySpark, when Arrow optimization is enabled, previously `toPandas` just failed when Arrow optimization is unable to be used whereas `createDataFrame` from Pandas DataFrame allowed the fallback to non-optimization. Now, both `toPandas` and `createDataFrame` from Pandas DataFrame allow the fallback by default, which can be switched off by `spark.sql.execution.arrow.fallback.enabled`.

From bbd93739cc47d0bcf11dd59fdd5fd8b3a4c434f8 Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Fri, 5 Jun 2020 12:53:58 +0900
Subject: [PATCH 0787/1280] [SPARK-31903][SQL][PYSPARK][R] Fix toPandas with
 Arrow enabled to show metrics in Query UI

### What changes were proposed in this pull request?

In `Dataset.collectAsArrowToR` and `Dataset.collectAsArrowToPython`, since the code block for `serveToStream` is run in the separate thread, `withAction` finishes as soon as it starts the thread. As a result, it doesn't collect the metrics of the actual action and Query UI shows the plan graph without metrics.

We should call `serveToStream` first, then `withAction` in it.

### Why are the changes needed?

When calling toPandas, usually Query UI shows each plan node's metric and corresponding Stage ID and Task ID:

```py
>>> df = spark.createDataFrame([(1, 10, 'abc'), (2, 20, 'def')], schema=['x', 'y', 'z'])
>>> df.toPandas()
   x   y    z
0  1  10  abc
1  2  20  def
```

![Screen Shot 2020-06-03 at 4 47 07 PM](https://user-images.githubusercontent.com/506656/83815735-bec22380-a675-11ea-8ecc-bf2954731f35.png)

but if Arrow execution is enabled, it shows only plan nodes and the duration is not correct:

```py
>>> spark.conf.set('spark.sql.execution.arrow.pyspark.enabled', True)
>>> df.toPandas()
   x   y    z
0  1  10  abc
1  2  20  def
```

![Screen Shot 2020-06-03 at 4 47 27 PM](https://user-images.githubusercontent.com/506656/83815804-de594c00-a675-11ea-933a-d0ffc0f534dd.png)

### Does this PR introduce _any_ user-facing change?

Yes, the Query UI will show the plan with the correct metrics.

### How was this patch tested?

I checked it manually in my local.

![Screen Shot 2020-06-04 at 3 19 41 PM](https://user-images.githubusercontent.com/506656/83816265-d77f0900-a676-11ea-84b8-2a8d80428bc6.png)

Closes #28730 from ueshin/issues/SPARK-31903/to_pandas_with_arrow_query_ui.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 632b5bce23c94d25712b43be83252b34ebfd3e72)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/Dataset.scala     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index c17535a43a14b..9d4312b21fd0a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -3481,8 +3481,8 @@ class Dataset[T] private[sql](
   private[sql] def collectAsArrowToR(): Array[Any] = {
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
 
-    withAction("collectAsArrowToR", queryExecution) { plan =>
-      RRDD.serveToStream("serve-Arrow") { outputStream =>
+    RRDD.serveToStream("serve-Arrow") { outputStream =>
+      withAction("collectAsArrowToR", queryExecution) { plan =>
         val buffer = new ByteArrayOutputStream()
         val out = new DataOutputStream(outputStream)
         val batchWriter = new ArrowBatchStreamWriter(schema, buffer, timeZoneId)
@@ -3535,8 +3535,8 @@ class Dataset[T] private[sql](
   private[sql] def collectAsArrowToPython: Array[Any] = {
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
 
-    withAction("collectAsArrowToPython", queryExecution) { plan =>
-      PythonRDD.serveToStream("serve-Arrow") { outputStream =>
+    PythonRDD.serveToStream("serve-Arrow") { outputStream =>
+      withAction("collectAsArrowToPython", queryExecution) { plan =>
         val out = new DataOutputStream(outputStream)
         val batchWriter = new ArrowBatchStreamWriter(schema, out, timeZoneId)
 

From 706dbecb356d241d9dd3e2f1ff060502a3cdcb23 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 5 Jun 2020 13:31:35 +0900
Subject: [PATCH 0788/1280] [MINOR][PYTHON] Add one more newline between JVM
 and Python tracebacks

### What changes were proposed in this pull request?

This PR proposes to add one more newline to clearly separate JVM and Python tracebacks:

Before:

```
Traceback (most recent call last):
  ...
pyspark.sql.utils.AnalysisException: Reference 'column' is ambiguous, could be: column, column.;
JVM stacktrace:
org.apache.spark.sql.AnalysisException: Reference 'column' is ambiguous, could be: column, column.;
  ...
```

After:

```
Traceback (most recent call last):
  ...
pyspark.sql.utils.AnalysisException: Reference 'column' is ambiguous, could be: column, column.;

JVM stacktrace:
org.apache.spark.sql.AnalysisException: Reference 'column' is ambiguous, could be: column, column.;
  ...
```

This is kind of a followup of https://github.com/apache/spark/commit/e69466056fb2c121b7bbb6ad082f09deb1c41063 (SPARK-31849).

### Why are the changes needed?

To make it easier to read.

### Does this PR introduce _any_ user-facing change?

It's in the unreleased branches.

### How was this patch tested?

Manually tested.

Closes #28732 from HyukjinKwon/python-minor.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 53ce58da3419223ce2666ba4d77aa4f2db82bdb5)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 27adc2372ec0f..3fd704721a92c 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -44,7 +44,7 @@ def __str__(self):
         debug_enabled = sql_conf.pysparkJVMStacktraceEnabled()
         desc = self.desc
         if debug_enabled:
-            desc = desc + "\nJVM stacktrace:\n%s" % self.stackTrace
+            desc = desc + "\n\nJVM stacktrace:\n%s" % self.stackTrace
         # encode unicode instance for python2 for human readable description
         if sys.version_info.major < 3 and isinstance(desc, unicode):
             return str(desc.encode('utf-8'))

From 4ee13b1214d007538aa0a1e3382c51b917594f43 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 5 Jun 2020 08:14:01 +0000
Subject: [PATCH 0789/1280] [SPARK-31879][SQL][TEST-JAVA11] Make week-based
 pattern invalid for formatting too

After all these attempts https://github.com/apache/spark/pull/28692 and https://github.com/apache/spark/pull/28719 an https://github.com/apache/spark/pull/28727.
they all have limitations as mentioned in their discussions.

Maybe the only way is to forbid them all

These week-based fields need Locale to express their semantics, the first day of the week varies from country to country.

From the Java doc of WeekFields
```java
    /**
     * Gets the first day-of-week.
     * <p>
     * The first day-of-week varies by culture.
     * For example, the US uses Sunday, while France and the ISO-8601 standard use Monday.
     * This method returns the first day using the standard {code DayOfWeek} enum.
     *
     * return the first day-of-week, not null
     */
    public DayOfWeek getFirstDayOfWeek() {
        return firstDayOfWeek;
    }
```

But for the SimpleDateFormat, the day-of-week is not localized

```
u	Day number of week (1 = Monday, ..., 7 = Sunday)	Number	1
```

Currently, the default locale we use is the US, so the result moved a day or a year or a week backward.

e.g.

For the date `2019-12-29(Sunday)`, in the Sunday Start system(e.g. en-US), it belongs to 2020 of week-based-year, in the Monday Start system(en-GB), it goes to 2019. the week-of-week-based-year(w) will be affected too

```sql
spark-sql> SELECT to_csv(named_struct('time', to_timestamp('2019-12-29', 'yyyy-MM-dd')), map('timestampFormat', 'YYYY', 'locale', 'en-US'));
2020
spark-sql> SELECT to_csv(named_struct('time', to_timestamp('2019-12-29', 'yyyy-MM-dd')), map('timestampFormat', 'YYYY', 'locale', 'en-GB'));
2019

spark-sql> SELECT to_csv(named_struct('time', to_timestamp('2019-12-29', 'yyyy-MM-dd')), map('timestampFormat', 'YYYY-ww-uu', 'locale', 'en-US'));
2020-01-01
spark-sql> SELECT to_csv(named_struct('time', to_timestamp('2019-12-29', 'yyyy-MM-dd')), map('timestampFormat', 'YYYY-ww-uu', 'locale', 'en-GB'));
2019-52-07

spark-sql> SELECT to_csv(named_struct('time', to_timestamp('2020-01-05', 'yyyy-MM-dd')), map('timestampFormat', 'YYYY-ww-uu', 'locale', 'en-US'));
2020-02-01
spark-sql> SELECT to_csv(named_struct('time', to_timestamp('2020-01-05', 'yyyy-MM-dd')), map('timestampFormat', 'YYYY-ww-uu', 'locale', 'en-GB'));
2020-01-07
```

For other countries, please refer to [First Day of the Week in Different Countries](http://chartsbin.com/view/41671)

With this change, user can not use 'YwuW',  but 'e' for 'u' instead. This can at least turn this not to be a silent data change.

add unit tests

Closes #28728 from yaooqinn/SPARK-31879-NEW2.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9d5b5d0a5849ac329bbae26d9884d8843d8a8571)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              |  14 +-
 .../sql/catalyst/util/DateFormatter.scala     |  10 +-
 .../util/DateTimeFormatterHelper.scala        |  34 +-
 .../catalyst/util/TimestampFormatter.scala    |  12 +-
 .../expressions/DateExpressionsSuite.scala    |   2 +-
 .../util/DateFormatterSuite.scala             |  13 +-
 .../util/DateTimeFormatterHelperSuite.scala   |  14 +-
 .../util/DatetimeFormatterSuite.scala         |  54 +++
 .../util/TimestampFormatterSuite.scala        |  61 +--
 .../inputs/datetime-formatting-invalid.sql    |  53 +++
 .../inputs/datetime-formatting-legacy.sql     |   2 +
 .../sql-tests/inputs/datetime-formatting.sql  |  68 +++
 .../resources/sql-tests/inputs/datetime.sql   |   4 -
 .../sql-tests/results/ansi/datetime.sql.out   |  28 +-
 .../datetime-formatting-invalid.sql.out       | 335 ++++++++++++++
 .../datetime-formatting-legacy.sql.out        | 401 ++++++++++++++++
 .../results/datetime-formatting.sql.out       | 431 ++++++++++++++++++
 .../sql-tests/results/datetime-legacy.sql.out |  28 +-
 .../sql-tests/results/datetime.sql.out        |  28 +-
 19 files changed, 1421 insertions(+), 171 deletions(-)
 rename sql/catalyst/src/test/scala/org/apache/spark/sql/{ => catalyst}/util/DateFormatterSuite.scala (96%)
 create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
 rename sql/catalyst/src/test/scala/org/apache/spark/sql/{ => catalyst}/util/TimestampFormatterSuite.scala (90%)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 5859ad82525f2..3c0bc754f940b 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -36,11 +36,7 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 |**M/L**|month-of-year|month|7; 07; Jul; July|
 |**d**|day-of-month|number(3)|28|
 |**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
-|**Y**|week-based-year|year|1996; 96|
-|**w**|week-of-week-based-year|number(2)|27|
-|**W**|week-of-month|number(1)|4|
 |**E**|day-of-week|text|Tue; Tuesday|
-|**u**|localized day-of-week|number/text|2; 02; Tue; Tuesday|
 |**F**|week-of-month|number(1)|3|
 |**a**|am-pm-of-day|am-pm|PM|
 |**h**|clock-hour-of-am-pm (1-12)|number(2)|12|
@@ -63,7 +59,7 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 
 The count of pattern letters determines the format.
 
-- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short form. Exactly 4 pattern letters will use the full form. Exactly 5 pattern letters will use the narrow form. 5 or more letters will fail.
+- Text: The text style is determined based on the number of pattern letters used. Less than 4 pattern letters will use the short text form, typically an abbreviation, e.g. day-of-week Monday might output "Mon". Exactly 4 pattern letters will use the full text form, typically the full description, e.g, day-of-week Monday might output "Monday". 5 or more letters will fail.
 
 - Number(n): The n here represents the maximum count of letters this type of datetime pattern can be used. If the count of letters is one, then the value is output using the minimum number of digits and without padding. Otherwise, the count of digits is used as the width of the output field, with the value zero-padded as necessary.
 
@@ -137,10 +133,4 @@ The count of pattern letters determines the format.
   During parsing, the whole section may be missing from the parsed string.
   An optional section is started by `[` and ended using `]` (or at the end of the pattern).
   
-- Symbols of 'Y', 'W', 'w', 'E', 'u', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
-
-More details for the text style:
-
-- Short Form: Short text, typically an abbreviation. For example, day-of-week Monday might output "Mon".
-
-- Full Form: Full text, typically the full description. For example, day-of-week Monday might output "Monday".
+- Symbols of 'E', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index b3347ebe2ec78..fbc9f56dd6045 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -46,7 +46,7 @@ class Iso8601DateFormatter(
   extends DateFormatter with DateTimeFormatterHelper {
 
   @transient
-  private lazy val formatter = getOrCreateFormatter(pattern, locale)
+  private lazy val formatter = getOrCreateFormatter(pattern, locale, isParsing)
 
   @transient
   private lazy val legacyFormatter = DateFormatter.getLegacyFormatter(
@@ -127,7 +127,7 @@ object DateFormatter {
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
       legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
-      isParsing: Boolean = true): DateFormatter = {
+      isParsing: Boolean): DateFormatter = {
     val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
@@ -160,11 +160,11 @@ object DateFormatter {
     getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing)
   }
 
-  def apply(format: String, zoneId: ZoneId): DateFormatter = {
-    getFormatter(Some(format), zoneId)
+  def apply(format: String, zoneId: ZoneId, isParsing: Boolean = false): DateFormatter = {
+    getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): DateFormatter = {
-    getFormatter(None, zoneId)
+    getFormatter(None, zoneId, isParsing = false)
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index eeb56aa9821ca..8e5c8651c8c36 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -97,7 +97,7 @@ trait DateTimeFormatterHelper {
   protected def getOrCreateFormatter(
       pattern: String,
       locale: Locale,
-      isParsing: Boolean = false): DateTimeFormatter = {
+      isParsing: Boolean): DateTimeFormatter = {
     val newPattern = convertIncompatiblePattern(pattern, isParsing)
     val useVarLen = isParsing && newPattern.contains('S')
     val key = (newPattern, locale, useVarLen)
@@ -234,22 +234,27 @@ private object DateTimeFormatterHelper {
     val formatter = DateTimeFormatter.ofPattern("LLL qqq", Locale.US)
     formatter.format(LocalDate.of(2000, 1, 1)) == "1 1"
   }
-  final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
   // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values
-  // to datetime, especially when they are mixed with other non-week-based ones
+  // to datetime, especially when they are mixed with other non-week-based ones;
+  // SPARK-31879: It's also difficult for us to restore the behavior of week-based date fields
+  // formatting, in DateTimeFormatter the first day of week for week-based date fields become
+  // localized, for the default Locale.US, it uses Sunday as the first day of week, while in Spark
+  // 2.4, the SimpleDateFormat uses Monday as the first day of week.
+  final val weekBasedLetters = Set('Y', 'W', 'w', 'u', 'e', 'c')
+  final val unsupportedLetters = Set('A', 'n', 'N', 'p')
   // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
   // be directly resolved then the `q` do check for whether the month is valid, but if the date
   // fields is incomplete, e.g. `yM`, the checking will be bypassed.
-  final val unsupportedLettersForParsing = Set('Y', 'W', 'w', 'E', 'u', 'F', 'q', 'Q')
+  final val unsupportedLettersForParsing = Set('E', 'F', 'q', 'Q')
   final val unsupportedPatternLengths = {
     // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
     // 2.4
-    Seq("G", "M", "L", "E", "u", "Q", "q").map(_ * 5) ++
+    Seq("G", "M", "L", "E", "Q", "q").map(_ * 5) ++
       // SPARK-31867: Disable year pattern longer than 10 which will cause Java time library throw
       // unchecked `ArrayIndexOutOfBoundsException` by the `NumberPrinterParser` for formatting. It
       // makes the call side difficult to handle exceptions and easily leads to silent data change
       // because of the exceptions being suppressed.
-      Seq("y", "Y").map(_ * 11)
+      Seq("y").map(_ * 11)
   }.toSet
 
   /**
@@ -260,7 +265,7 @@ private object DateTimeFormatterHelper {
    * @param pattern The input pattern.
    * @return The pattern for new parser
    */
-  def convertIncompatiblePattern(pattern: String, isParsing: Boolean = false): String = {
+  def convertIncompatiblePattern(pattern: String, isParsing: Boolean): String = {
     val eraDesignatorContained = pattern.split("'").zipWithIndex.exists {
       case (patternPart, index) =>
         // Text can be quoted using single quotes, we only check the non-quote parts.
@@ -269,6 +274,10 @@ private object DateTimeFormatterHelper {
     (pattern + " ").split("'").zipWithIndex.map {
       case (patternPart, index) =>
         if (index % 2 == 0) {
+          for (c <- patternPart if weekBasedLetters.contains(c)) {
+            throw new IllegalArgumentException(s"All week-based patterns are unsupported since" +
+              s" Spark 3.0, detected: $c, Please use the SQL function EXTRACT instead")
+          }
           for (c <- patternPart if unsupportedLetters.contains(c) ||
             (isParsing && unsupportedLettersForParsing.contains(c))) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
@@ -282,20 +291,13 @@ private object DateTimeFormatterHelper {
               "or upgrade your Java version. For more details, please read " +
               "https://bugs.openjdk.java.net/browse/JDK-8114833")
           }
-          // The meaning of 'u' was day number of week in SimpleDateFormat, it was changed to year
-          // in DateTimeFormatter. Substitute 'u' to 'e' and use DateTimeFormatter to parse the
-          // string. If parsable, return the result; otherwise, fall back to 'u', and then use the
-          // legacy SimpleDateFormat parser to parse. When it is successfully parsed, throw an
-          // exception and ask users to change the pattern strings or turn on the legacy mode;
-          // otherwise, return NULL as what Spark 2.4 does.
-          val res = patternPart.replace("u", "e")
           // In DateTimeFormatter, 'u' supports negative years. We substitute 'y' to 'u' here for
           // keeping the support in Spark 3.0. If parse failed in Spark 3.0, fall back to 'y'.
           // We only do this substitution when there is no era designator found in the pattern.
           if (!eraDesignatorContained) {
-            res.replace("y", "u")
+            patternPart.replace("y", "u")
           } else {
-            res
+            patternPart
           }
         } else {
           patternPart
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index e8866d7fd74cc..6bcbb09b22b4b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -62,11 +62,11 @@ class Iso8601TimestampFormatter(
     zoneId: ZoneId,
     locale: Locale,
     legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
-    needVarLengthSecondFraction: Boolean)
+    isParsing: Boolean)
   extends TimestampFormatter with DateTimeFormatterHelper {
   @transient
   protected lazy val formatter: DateTimeFormatter =
-    getOrCreateFormatter(pattern, locale, needVarLengthSecondFraction)
+    getOrCreateFormatter(pattern, locale, isParsing)
 
   @transient
   protected lazy val legacyFormatter = TimestampFormatter.getLegacyFormatter(
@@ -122,7 +122,7 @@ class FractionTimestampFormatter(zoneId: ZoneId)
     zoneId,
     TimestampFormatter.defaultLocale,
     LegacyDateFormats.FAST_DATE_FORMAT,
-    needVarLengthSecondFraction = false) {
+    isParsing = false) {
 
   @transient
   override protected lazy val formatter = DateTimeFormatterHelper.fractionFormatter
@@ -266,7 +266,7 @@ object TimestampFormatter {
       zoneId: ZoneId,
       locale: Locale = defaultLocale,
       legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT,
-      isParsing: Boolean = false): TimestampFormatter = {
+      isParsing: Boolean): TimestampFormatter = {
     val pattern = format.getOrElse(defaultPattern)
     if (SQLConf.get.legacyTimeParserPolicy == LEGACY) {
       getLegacyFormatter(pattern, zoneId, locale, legacyFormat)
@@ -313,12 +313,12 @@ object TimestampFormatter {
   def apply(
       format: String,
       zoneId: ZoneId,
-      isParsing: Boolean = false): TimestampFormatter = {
+      isParsing: Boolean): TimestampFormatter = {
     getFormatter(Some(format), zoneId, isParsing = isParsing)
   }
 
   def apply(zoneId: ZoneId): TimestampFormatter = {
-    getFormatter(None, zoneId)
+    getFormatter(None, zoneId, isParsing = false)
   }
 
   def getFractionFormatter(zoneId: ZoneId): TimestampFormatter = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index dcb475924fb42..df417544dd15c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -41,7 +41,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
   private val JST_OPT = Option(JST.getId)
 
   def toMillis(timestamp: String): Long = {
-    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC)
+    val tf = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
     DateTimeUtils.toMillis(tf.parse(timestamp))
   }
   val date = "2015-04-08 13:10:15"
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
similarity index 96%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
index 22a1396d5efdc..4892deae92b3d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
@@ -15,19 +15,22 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.util
+package org.apache.spark.sql.catalyst.util
 
 import java.time.{DateTimeException, LocalDate}
 
-import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{DateFormatter, LegacyDateFormats}
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 
-class DateFormatterSuite extends SparkFunSuite with SQLHelper {
+class DateFormatterSuite extends DatetimeFormatterSuite {
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    DateFormatter(pattern, UTC, isParsing)
+  }
+
   test("parsing dates") {
     outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
index c68bdacb13af7..0b15e49af0210 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala
@@ -22,27 +22,32 @@ import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._
 
 class DateTimeFormatterHelperSuite extends SparkFunSuite {
 
+  private def convertIncompatiblePattern(pattern: String): String = {
+    DateTimeFormatterHelper.convertIncompatiblePattern(pattern, isParsing = false)
+  }
+
   test("check incompatible pattern") {
-    assert(convertIncompatiblePattern("MM-DD-u") === "MM-DD-e")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz")
       === "uuuu-MM-dd'T'HH:mm:ss.SSSz")
     assert(convertIncompatiblePattern("yyyy-MM'y contains in quoted text'HH:mm:ss")
       === "uuuu-MM'y contains in quoted text'HH:mm:ss")
-    assert(convertIncompatiblePattern("yyyy-MM-dd-u'T'HH:mm:ss.SSSz")
-      === "uuuu-MM-dd-e'T'HH:mm:ss.SSSz")
     assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'HH:mm:ss")
       === "uuuu-MM'u contains in quoted text'HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM'u contains in quoted text'''''HH:mm:ss")
       === "uuuu-MM'u contains in quoted text'''''HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'HH:mm:ss.SSSz G")
       === "yyyy-MM-dd'T'HH:mm:ss.SSSz G")
+    weekBasedLetters.foreach { l =>
+      val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
+      assert(e.getMessage.contains("week-based"))
+    }
     unsupportedLetters.foreach { l =>
       val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G"))
       assert(e.getMessage === s"Illegal pattern character: $l")
     }
     unsupportedLettersForParsing.foreach { l =>
       val e = intercept[IllegalArgumentException] {
-        convertIncompatiblePattern(s"$l", isParsing = true)
+        DateTimeFormatterHelper.convertIncompatiblePattern(s"$l", isParsing = true)
       }
       assert(e.getMessage === s"Illegal pattern character: $l")
       assert(convertIncompatiblePattern(s"$l").nonEmpty)
@@ -57,7 +62,6 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite {
       }
       assert(e2.getMessage === s"Too many pattern letters: ${style.head}")
     }
-    assert(convertIncompatiblePattern("yyyy-MM-dd uuuu") === "uuuu-MM-dd eeee")
     assert(convertIncompatiblePattern("yyyy-MM-dd EEEE") === "uuuu-MM-dd EEEE")
     assert(convertIncompatiblePattern("yyyy-MM-dd'e'HH:mm:ss") === "uuuu-MM-dd'e'HH:mm:ss")
     assert(convertIncompatiblePattern("yyyy-MM-dd'T'") === "uuuu-MM-dd'T'")
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
new file mode 100644
index 0000000000000..31ff50fda1ad6
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import org.scalatest.Matchers
+
+import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+
+trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
+  import DateTimeFormatterHelper._
+  def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
+
+  test("explicitly forbidden datetime patterns") {
+
+    Seq(true, false).foreach { isParsing =>
+      // not support by the legacy one too
+      val unsupportedBoth = Seq("QQQQQ", "qqqqq", "eeeee", "A", "c", "n", "N", "p", "e")
+      unsupportedBoth.foreach { pattern =>
+        intercept[IllegalArgumentException](checkFormatterCreation(pattern, isParsing))
+      }
+      // supported by the legacy one, then we will suggest users with SparkUpgradeException
+      ((weekBasedLetters ++ unsupportedLetters).map(_.toString)
+        ++ unsupportedPatternLengths -- unsupportedBoth).foreach {
+        pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, isParsing))
+      }
+    }
+
+    // not support by the legacy one too
+    val unsupportedBoth = Seq("q", "Q")
+    unsupportedBoth.foreach { pattern =>
+      intercept[IllegalArgumentException](checkFormatterCreation(pattern, true))
+    }
+    // supported by the legacy one, then we will suggest users with SparkUpgradeException
+    (unsupportedLettersForParsing.map(_.toString) -- unsupportedBoth).foreach {
+      pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
similarity index 90%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index efd97a4195566..311097f31a15d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -15,23 +15,23 @@
  * limitations under the License.
  */
 
-package org.apache.spark.sql.util
+package org.apache.spark.sql.catalyst.util
 
 import java.time.{DateTimeException, Instant, LocalDateTime, LocalTime}
 import java.util.concurrent.TimeUnit
 
-import org.scalatest.Matchers
-
-import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.{LegacyDateFormats, TimestampFormatter}
+import org.apache.spark.SparkUpgradeException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy
 import org.apache.spark.unsafe.types.UTF8String
 
-class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
+class TimestampFormatterSuite extends DatetimeFormatterSuite {
+
+  override def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit = {
+    TimestampFormatter(pattern, UTC, isParsing)
+  }
 
   test("parsing timestamps using time zones") {
     val localDate = "2018-12-02T10:11:12.001234"
@@ -96,7 +96,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         2177456523456789L,
         11858049903010203L).foreach { micros =>
         outstandingZoneIds.foreach { zoneId =>
-          val timestamp = TimestampFormatter(pattern, zoneId).format(micros)
+          val timestamp = TimestampFormatter(pattern, zoneId, isParsing = false).format(micros)
           val parsed = TimestampFormatter(
             pattern, zoneId, isParsing = true).parse(timestamp)
           assert(micros === parsed)
@@ -120,14 +120,14 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS"
         val micros = TimestampFormatter(
           pattern, zoneId, isParsing = true).parse(timestamp)
-        val formatted = TimestampFormatter(pattern, zoneId).format(micros)
+        val formatted = TimestampFormatter(pattern, zoneId, isParsing = false).format(micros)
         assert(timestamp === formatted)
       }
     }
   }
 
   test("case insensitive parsing of am and pm") {
-    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", UTC)
+    val formatter = TimestampFormatter("yyyy MMM dd hh:mm:ss a", UTC, isParsing = false)
     val micros = formatter.parse("2009 Mar 20 11:30:01 am")
     assert(micros === date(2009, 3, 20, 11, 30, 1))
   }
@@ -154,8 +154,8 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     assert(TimestampFormatter(UTC).format(micros) === "-0099-01-01 00:00:00")
     assert(TimestampFormatter(UTC).format(instant) === "-0099-01-01 00:00:00")
     withDefaultTimeZone(UTC) { // toJavaTimestamp depends on the default time zone
-      assert(TimestampFormatter("yyyy-MM-dd HH:mm:SS G", UTC).format(toJavaTimestamp(micros))
-        === "0100-01-01 00:00:00 BC")
+      assert(TimestampFormatter("yyyy-MM-dd HH:mm:SS G", UTC, isParsing = false)
+        .format(toJavaTimestamp(micros)) === "0100-01-01 00:00:00 BC")
     }
   }
 
@@ -206,7 +206,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
         "2019-10-14T09:39:07.1", "2019-10-14T09:39:07.1")
 
       try {
-        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId, true)
+        TimestampFormatter("yyyy/MM/dd HH_mm_ss.SSSSSS", zoneId, isParsing = true)
           .parse("2019/11/14 20#25#30.123456")
         fail("Expected to throw an exception for the invalid input")
       } catch {
@@ -219,7 +219,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   test("formatting timestamp strings up to microsecond precision") {
     outstandingZoneIds.foreach { zoneId =>
       def check(pattern: String, input: String, expected: String): Unit = {
-        val formatter = TimestampFormatter(pattern, zoneId)
+        val formatter = TimestampFormatter(pattern, zoneId, isParsing = false)
         val timestamp = stringToTimestamp(UTF8String.fromString(input), zoneId).get
         val actual = formatter.format(timestamp)
         assert(actual === expected)
@@ -256,7 +256,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("SPARK-30958: parse timestamp with negative year") {
-    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, true)
+    val formatter1 = TimestampFormatter("yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
     assert(formatter1.parse("-1234-02-22 02:22:22") === date(-1234, 2, 22, 2, 22, 22))
 
     def assertParsingError(f: => Unit): Unit = {
@@ -269,7 +269,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
     }
 
     // "yyyy" with "G" can't parse negative year or year 0000.
-    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", UTC, true)
+    val formatter2 = TimestampFormatter("G yyyy-MM-dd HH:mm:ss", UTC, isParsing = true)
     assertParsingError(formatter2.parse("BC -1234-02-22 02:22:22"))
     assertParsingError(formatter2.parse("AC 0000-02-22 02:22:22"))
 
@@ -315,7 +315,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   test("parsing hour with various patterns") {
     def createFormatter(pattern: String): TimestampFormatter = {
       // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid value range.
-      TimestampFormatter(pattern, UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, false)
+      TimestampFormatter(pattern, UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, isParsing = true)
     }
 
     withClue("HH") {
@@ -374,56 +374,45 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
   }
 
   test("missing date fields") {
-    val formatter = TimestampFormatter("HH:mm:ss", UTC)
+    val formatter = TimestampFormatter("HH:mm:ss", UTC, isParsing = true)
     val micros = formatter.parse("11:30:01")
     assert(micros === date(1970, 1, 1, 11, 30, 1))
   }
 
   test("missing year field with invalid date") {
     // Use `SIMPLE_DATE_FORMAT`, so that the legacy parser also fails with invalid date.
-    val formatter = TimestampFormatter("MM-dd", UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, false)
+    val formatter =
+      TimestampFormatter("MM-dd", UTC, LegacyDateFormats.SIMPLE_DATE_FORMAT, isParsing = true)
     withDefaultTimeZone(UTC)(intercept[DateTimeException](formatter.parse("02-29")))
   }
 
   test("missing am/pm field") {
     Seq("HH", "hh", "KK", "kk").foreach { hour =>
-      val formatter = TimestampFormatter(s"yyyy $hour:mm:ss", UTC)
+      val formatter = TimestampFormatter(s"yyyy $hour:mm:ss", UTC, isParsing = true)
       val micros = formatter.parse("2009 11:30:01")
       assert(micros === date(2009, 1, 1, 11, 30, 1))
     }
   }
 
   test("missing time fields") {
-    val formatter = TimestampFormatter("yyyy HH", UTC)
+    val formatter = TimestampFormatter("yyyy HH", UTC, isParsing = true)
     val micros = formatter.parse("2009 11")
     assert(micros === date(2009, 1, 1, 11))
   }
 
   test("missing hour field") {
-    val f1 = TimestampFormatter("mm:ss a", UTC)
+    val f1 = TimestampFormatter("mm:ss a", UTC, isParsing = true)
     val t1 = f1.parse("30:01 PM")
     assert(t1 === date(1970, 1, 1, 12, 30, 1))
     val t2 = f1.parse("30:01 AM")
     assert(t2 === date(1970, 1, 1, 0, 30, 1))
-    val f2 = TimestampFormatter("mm:ss", UTC)
+    val f2 = TimestampFormatter("mm:ss", UTC, isParsing = true)
     val t3 = f2.parse("30:01")
     assert(t3 === date(1970, 1, 1, 0, 30, 1))
-    val f3 = TimestampFormatter("a", UTC)
+    val f3 = TimestampFormatter("a", UTC, isParsing = true)
     val t4 = f3.parse("PM")
     assert(t4 === date(1970, 1, 1, 12))
     val t5 = f3.parse("AM")
     assert(t5 === date(1970))
   }
-
-  test("explicitly forbidden datetime patterns") {
-    // not support by the legacy one too
-    Seq("QQQQQ", "qqqqq", "A", "c", "e", "n", "N", "p").foreach { pattern =>
-      intercept[IllegalArgumentException](TimestampFormatter(pattern, UTC).format(0))
-    }
-    // supported by the legacy one, then we will suggest users with SparkUpgradeException
-    Seq("GGGGG", "MMMMM", "LLLLL", "EEEEE", "uuuuu", "aa", "aaa", "y" * 11, "y" * 11)
-      .foreach { pattern =>
-        intercept[SparkUpgradeException](TimestampFormatter(pattern, UTC).format(0))
-    }
-  }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
new file mode 100644
index 0000000000000..9072aa107f252
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
@@ -0,0 +1,53 @@
+--- TESTS FOR DATETIME FORMATTING FUNCTIONS WITH INVALID PATTERNS ---
+
+-- separating this from datetime-formatting.sql, because the text form
+-- for patterns with 5 letters in SimpleDateFormat varies from different JDKs
+select date_format('2018-11-17 13:33:33.333', 'GGGGG');
+-- pattern letter count can not be greater than 10
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyyyyyy');
+-- q/L in JDK 8 will fail when the count is more than 2
+select date_format('2018-11-17 13:33:33.333', 'qqqqq');
+select date_format('2018-11-17 13:33:33.333', 'QQQQQ');
+select date_format('2018-11-17 13:33:33.333', 'MMMMM');
+select date_format('2018-11-17 13:33:33.333', 'LLLLL');
+
+select date_format('2018-11-17 13:33:33.333', 'EEEEE');
+select date_format('2018-11-17 13:33:33.333', 'FF');
+select date_format('2018-11-17 13:33:33.333', 'ddd');
+-- DD is invalid if the day-of-year exceeds 100, but it becomes valid in Java 11
+-- select date_format('2018-11-17 13:33:33.333', 'DD');
+select date_format('2018-11-17 13:33:33.333', 'DDDD');
+select date_format('2018-11-17 13:33:33.333', 'HHH');
+select date_format('2018-11-17 13:33:33.333', 'hhh');
+select date_format('2018-11-17 13:33:33.333', 'kkk');
+select date_format('2018-11-17 13:33:33.333', 'KKK');
+select date_format('2018-11-17 13:33:33.333', 'mmm');
+select date_format('2018-11-17 13:33:33.333', 'sss');
+select date_format('2018-11-17 13:33:33.333', 'SSSSSSSSSS');
+select date_format('2018-11-17 13:33:33.333', 'aa');
+select date_format('2018-11-17 13:33:33.333', 'V');
+select date_format('2018-11-17 13:33:33.333', 'zzzzz');
+select date_format('2018-11-17 13:33:33.333', 'XXXXXX');
+select date_format('2018-11-17 13:33:33.333', 'ZZZZZZ');
+select date_format('2018-11-17 13:33:33.333', 'OO');
+select date_format('2018-11-17 13:33:33.333', 'xxxxxx');
+
+select date_format('2018-11-17 13:33:33.333', 'A');
+select date_format('2018-11-17 13:33:33.333', 'n');
+select date_format('2018-11-17 13:33:33.333', 'N');
+select date_format('2018-11-17 13:33:33.333', 'p');
+
+-- disabled week-based patterns
+select date_format('2018-11-17 13:33:33.333', 'Y');
+select date_format('2018-11-17 13:33:33.333', 'w');
+select date_format('2018-11-17 13:33:33.333', 'W');
+select date_format('2018-11-17 13:33:33.333', 'u');
+select date_format('2018-11-17 13:33:33.333', 'e');
+select date_format('2018-11-17 13:33:33.333', 'c');
+
+-- others
+select date_format('2018-11-17 13:33:33.333', 'B');
+select date_format('2018-11-17 13:33:33.333', 'C');
+select date_format('2018-11-17 13:33:33.333', 'I');
+
+
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
new file mode 100644
index 0000000000000..19cab61a7ee56
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-formatting.sql
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
new file mode 100644
index 0000000000000..3b23a7785f6cd
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
@@ -0,0 +1,68 @@
+--- TESTS FOR DATETIME FORMATTING FUNCTIONS ---
+
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col);
+
+select col, date_format(col, 'G GG GGG GGGG') from v;
+
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy') from v;
+
+select col, date_format(col, 'q qq') from v;
+
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v;
+
+select col, date_format(col, 'M MM MMM MMMM') from v;
+
+select col, date_format(col, 'L LL') from v;
+
+select col, date_format(col, 'E EE EEE EEEE') from v;
+
+select col, date_format(col, 'F') from v;
+
+select col, date_format(col, 'd dd') from v;
+
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles';
+select col, date_format(col, 'D DDD') from v;
+
+select col, date_format(col, 'H HH') from v;
+
+select col, date_format(col, 'h hh') from v;
+
+select col, date_format(col, 'k kk') from v;
+
+select col, date_format(col, 'K KK') from v;
+
+select col, date_format(col, 'm mm') from v;
+
+select col, date_format(col, 's ss') from v;
+
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v;
+
+select col, date_format(col, 'a') from v;
+
+select col, date_format(col, 'VV') from v;
+
+select col, date_format(col, 'z zz zzz zzzz') from v;
+
+select col, date_format(col, 'X XX XXX') from v;
+select col, date_format(col, 'XXXX XXXXX') from v;
+
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v;
+
+select col, date_format(col, 'O OOOO') from v;
+
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v;
+
+-- optional pattern, but the results won't be optional for formatting
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v;
+
+-- literals
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v;
+select col, date_format(col, "''") from v;
+select col, date_format(col, '') from v;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index 4eefa0f675202..e4a2e5d8c8f3f 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -114,10 +114,6 @@ select to_timestamp("12.1234019-10-06S10:11", "ss.SSSSy-MM-dd'S'HH:mm");
 select to_timestamp("2019-10-06S", "yyyy-MM-dd'S'");
 select to_timestamp("S2019-10-06", "'S'yyyy-MM-dd");
 
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee');
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc');
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu');
-
 select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS"); -- middle
 select to_timestamp("2019-10-06T10:11:12'", "yyyy-MM-dd'T'HH:mm:ss''"); -- tail
 select to_timestamp("'2019-10-06T10:11:12", "''yyyy-MM-dd'T'HH:mm:ss"); -- head
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index 43fe0a652324e..1cb1aaa6584db 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 112
+-- Number of queries: 109
 
 
 -- !query
@@ -674,32 +674,6 @@ struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
 2019-10-06 00:00:00
 
 
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
--- !query schema
-struct<>
--- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: e
-
-
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
--- !query schema
-struct<>
--- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: c
-
-
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
--- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
--- !query output
-2019-10-06 Sunday
-
-
 -- !query
 select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
new file mode 100644
index 0000000000000..248157efacde0
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
@@ -0,0 +1,335 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 37
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'GGGGG')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'GGGGG' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyyyyyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'qqqqq')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: q
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'QQQQQ')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: Q
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'MMMMM')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'MMMMM' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'LLLLL')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'LLLLL' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'EEEEE')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'EEEEE' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'FF')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'FF' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'ddd')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'ddd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'DDDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'DDDD' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'HHH')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'HHH' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'hhh')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'hhh' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'kkk')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'kkk' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'KKK')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'KKK' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'mmm')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'mmm' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'sss')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'sss' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'SSSSSSSSSS')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'SSSSSSSSSS' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'aa')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'aa' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'V')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Pattern letter count must be 2: V
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'zzzzz')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'zzzzz' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'XXXXXX')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: X
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'ZZZZZZ')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'ZZZZZZ' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'OO')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Pattern letter count must be 1 or 4: O
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'xxxxxx')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Too many pattern letters: x
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'A')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: A
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'n')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: n
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'N')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: N
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'p')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character: p
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'Y')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'Y' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'w')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'w' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'W')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'W' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'u')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'u' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'e')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+All week-based patterns are unsupported since Spark 3.0, detected: e, Please use the SQL function EXTRACT instead
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'c')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+All week-based patterns are unsupported since Spark 3.0, detected: c, Please use the SQL function EXTRACT instead
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'B')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: B
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'C')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: C
+
+
+-- !query
+select date_format('2018-11-17 13:33:33.333', 'I')
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Unknown pattern letter: I
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
new file mode 100644
index 0000000000000..b7bc448a952a7
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
@@ -0,0 +1,401 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select col, date_format(col, 'G GG GGG GGGG') from v
+-- !query schema
+struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
+-- !query output
+1582-05-31 19:40:35.123	AD AD AD AD
+1969-12-31 15:00:00	AD AD AD AD
+1970-12-31 04:59:59.999	AD AD AD AD
+1996-03-31 07:03:33.123	AD AD AD AD
+2018-11-17 05:33:33.123	AD AD AD AD
+2019-12-31 09:33:33.123	AD AD AD AD
+2100-01-01 01:33:33.123	AD AD AD AD
+
+
+-- !query
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy') from v
+-- !query schema
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy):string>
+-- !query output
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582 0001582 00001582 000001582 0000001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969 0001969 00001969 000001969 0000001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970 0001970 00001970 000001970 0000001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996 0001996 00001996 000001996 0000001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018 0002018 00002018 000002018 0000002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019 0002019 00002019 000002019 0000002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100 0002100 00002100 000002100 0000002100
+
+
+-- !query
+select col, date_format(col, 'q qq') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'q'
+
+
+-- !query
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'Q'
+
+
+-- !query
+select col, date_format(col, 'M MM MMM MMMM') from v
+-- !query schema
+struct<col:timestamp,date_format(col, M MM MMM MMMM):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05 May May
+1969-12-31 15:00:00	12 12 Dec December
+1970-12-31 04:59:59.999	12 12 Dec December
+1996-03-31 07:03:33.123	3 03 Mar March
+2018-11-17 05:33:33.123	11 11 Nov November
+2019-12-31 09:33:33.123	12 12 Dec December
+2100-01-01 01:33:33.123	1 01 Jan January
+
+
+-- !query
+select col, date_format(col, 'L LL') from v
+-- !query schema
+struct<col:timestamp,date_format(col, L LL):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05
+1969-12-31 15:00:00	12 12
+1970-12-31 04:59:59.999	12 12
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	11 11
+2019-12-31 09:33:33.123	12 12
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'E EE EEE EEEE') from v
+-- !query schema
+struct<col:timestamp,date_format(col, E EE EEE EEEE):string>
+-- !query output
+1582-05-31 19:40:35.123	Thu Thu Thu Thursday
+1969-12-31 15:00:00	Wed Wed Wed Wednesday
+1970-12-31 04:59:59.999	Thu Thu Thu Thursday
+1996-03-31 07:03:33.123	Sun Sun Sun Sunday
+2018-11-17 05:33:33.123	Sat Sat Sat Saturday
+2019-12-31 09:33:33.123	Tue Tue Tue Tuesday
+2100-01-01 01:33:33.123	Fri Fri Fri Friday
+
+
+-- !query
+select col, date_format(col, 'F') from v
+-- !query schema
+struct<col:timestamp,date_format(col, F):string>
+-- !query output
+1582-05-31 19:40:35.123	5
+1969-12-31 15:00:00	5
+1970-12-31 04:59:59.999	5
+1996-03-31 07:03:33.123	5
+2018-11-17 05:33:33.123	3
+2019-12-31 09:33:33.123	5
+2100-01-01 01:33:33.123	1
+
+
+-- !query
+select col, date_format(col, 'd dd') from v
+-- !query schema
+struct<col:timestamp,date_format(col, d dd):string>
+-- !query output
+1582-05-31 19:40:35.123	31 31
+1969-12-31 15:00:00	31 31
+1970-12-31 04:59:59.999	31 31
+1996-03-31 07:03:33.123	31 31
+2018-11-17 05:33:33.123	17 17
+2019-12-31 09:33:33.123	31 31
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles'
+-- !query schema
+struct<col:timestamp,date_format(col, DD):string>
+-- !query output
+2100-01-01 01:33:33.123	01
+
+
+-- !query
+select col, date_format(col, 'D DDD') from v
+-- !query schema
+struct<col:timestamp,date_format(col, D DDD):string>
+-- !query output
+1582-05-31 19:40:35.123	151 151
+1969-12-31 15:00:00	365 365
+1970-12-31 04:59:59.999	365 365
+1996-03-31 07:03:33.123	91 091
+2018-11-17 05:33:33.123	321 321
+2019-12-31 09:33:33.123	365 365
+2100-01-01 01:33:33.123	1 001
+
+
+-- !query
+select col, date_format(col, 'H HH') from v
+-- !query schema
+struct<col:timestamp,date_format(col, H HH):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'h hh') from v
+-- !query schema
+struct<col:timestamp,date_format(col, h hh):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'k kk') from v
+-- !query schema
+struct<col:timestamp,date_format(col, k kk):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'K KK') from v
+-- !query schema
+struct<col:timestamp,date_format(col, K KK):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'm mm') from v
+-- !query schema
+struct<col:timestamp,date_format(col, m mm):string>
+-- !query output
+1582-05-31 19:40:35.123	40 40
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 's ss') from v
+-- !query schema
+struct<col:timestamp,date_format(col, s ss):string>
+-- !query output
+1582-05-31 19:40:35.123	35 35
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	33 33
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v
+-- !query schema
+struct<col:timestamp,date_format(col, S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS):string>
+-- !query output
+1582-05-31 19:40:35.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+1969-12-31 15:00:00	0 00 000 0000 00000 000000 0000000 00000000 000000000
+1970-12-31 04:59:59.999	999 999 999 0999 00999 000999 0000999 00000999 000000999
+1996-03-31 07:03:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2018-11-17 05:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2019-12-31 09:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+2100-01-01 01:33:33.123	123 123 123 0123 00123 000123 0000123 00000123 000000123
+
+
+-- !query
+select col, date_format(col, 'a') from v
+-- !query schema
+struct<col:timestamp,date_format(col, a):string>
+-- !query output
+1582-05-31 19:40:35.123	PM
+1969-12-31 15:00:00	PM
+1970-12-31 04:59:59.999	AM
+1996-03-31 07:03:33.123	AM
+2018-11-17 05:33:33.123	AM
+2019-12-31 09:33:33.123	AM
+2100-01-01 01:33:33.123	AM
+
+
+-- !query
+select col, date_format(col, 'VV') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'V'
+
+
+-- !query
+select col, date_format(col, 'z zz zzz zzzz') from v
+-- !query schema
+struct<col:timestamp,date_format(col, z zz zzz zzzz):string>
+-- !query output
+1582-05-31 19:40:35.123	PST PST PST Pacific Standard Time
+1969-12-31 15:00:00	PST PST PST Pacific Standard Time
+1970-12-31 04:59:59.999	PST PST PST Pacific Standard Time
+1996-03-31 07:03:33.123	PST PST PST Pacific Standard Time
+2018-11-17 05:33:33.123	PST PST PST Pacific Standard Time
+2019-12-31 09:33:33.123	PST PST PST Pacific Standard Time
+2100-01-01 01:33:33.123	PST PST PST Pacific Standard Time
+
+
+-- !query
+select col, date_format(col, 'X XX XXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, X XX XXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-08 -0800 -08:00
+1969-12-31 15:00:00	-08 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'XXXX XXXXX') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+invalid ISO 8601 format: length=4
+
+
+-- !query
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Z ZZ ZZZ ZZZZ ZZZZZ):string>
+-- !query output
+1582-05-31 19:40:35.123	-0800 -0800 -0800 -0800 -0800
+1969-12-31 15:00:00	-0800 -0800 -0800 -0800 -0800
+1970-12-31 04:59:59.999	-0800 -0800 -0800 -0800 -0800
+1996-03-31 07:03:33.123	-0800 -0800 -0800 -0800 -0800
+2018-11-17 05:33:33.123	-0800 -0800 -0800 -0800 -0800
+2019-12-31 09:33:33.123	-0800 -0800 -0800 -0800 -0800
+2100-01-01 01:33:33.123	-0800 -0800 -0800 -0800 -0800
+
+
+-- !query
+select col, date_format(col, 'O OOOO') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'O'
+
+
+-- !query
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+Illegal pattern character 'x'
+
+
+-- !query
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v
+-- !query schema
+struct<col:timestamp,date_format(col, [yyyy-MM-dd HH:mm:ss]):string>
+-- !query output
+1582-05-31 19:40:35.123	[1582-05-31 19:40:35]
+1969-12-31 15:00:00	[1969-12-31 15:00:00]
+1970-12-31 04:59:59.999	[1970-12-31 04:59:59]
+1996-03-31 07:03:33.123	[1996-03-31 07:03:33]
+2018-11-17 05:33:33.123	[2018-11-17 05:33:33]
+2019-12-31 09:33:33.123	[2019-12-31 09:33:33]
+2100-01-01 01:33:33.123	[2100-01-01 01:33:33]
+
+
+-- !query
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v
+-- !query schema
+struct<col:timestamp,date_format(col, 姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'):string>
+-- !query output
+1582-05-31 19:40:35.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1969-12-31 15:00:00	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1970-12-31 04:59:59.999	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1996-03-31 07:03:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2018-11-17 05:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2019-12-31 09:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2100-01-01 01:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+
+
+-- !query
+select col, date_format(col, "''") from v
+-- !query schema
+struct<col:timestamp,date_format(col, ''):string>
+-- !query output
+1582-05-31 19:40:35.123	'
+1969-12-31 15:00:00	'
+1970-12-31 04:59:59.999	'
+1996-03-31 07:03:33.123	'
+2018-11-17 05:33:33.123	'
+2019-12-31 09:33:33.123	'
+2100-01-01 01:33:33.123	'
+
+
+-- !query
+select col, date_format(col, '') from v
+-- !query schema
+struct<col:timestamp,date_format(col, ):string>
+-- !query output
+1582-05-31 19:40:35.123	
+1969-12-31 15:00:00	
+1970-12-31 04:59:59.999	
+1996-03-31 07:03:33.123	
+2018-11-17 05:33:33.123	
+2019-12-31 09:33:33.123	
+2100-01-01 01:33:33.123
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
new file mode 100644
index 0000000000000..f724658d354df
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
@@ -0,0 +1,431 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 31
+
+
+-- !query
+create temporary view v as select col from values
+ (timestamp '1582-06-01 11:33:33.123UTC+080000'),
+ (timestamp '1970-01-01 00:00:00.000Europe/Paris'),
+ (timestamp '1970-12-31 23:59:59.999Asia/Srednekolymsk'),
+ (timestamp '1996-04-01 00:33:33.123Australia/Darwin'),
+ (timestamp '2018-11-17 13:33:33.123Z'),
+ (timestamp '2020-01-01 01:33:33.123Asia/Shanghai'),
+ (timestamp '2100-01-01 01:33:33.123America/Los_Angeles') t(col)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select col, date_format(col, 'G GG GGG GGGG') from v
+-- !query schema
+struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
+-- !query output
+1582-05-31 19:40:35.123	AD AD AD Anno Domini
+1969-12-31 15:00:00	AD AD AD Anno Domini
+1970-12-31 04:59:59.999	AD AD AD Anno Domini
+1996-03-31 07:03:33.123	AD AD AD Anno Domini
+2018-11-17 05:33:33.123	AD AD AD Anno Domini
+2019-12-31 09:33:33.123	AD AD AD Anno Domini
+2100-01-01 01:33:33.123	AD AD AD Anno Domini
+
+
+-- !query
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy') from v
+-- !query schema
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy):string>
+-- !query output
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582 0001582 00001582 000001582 0000001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969 0001969 00001969 000001969 0000001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970 0001970 00001970 000001970 0000001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996 0001996 00001996 000001996 0000001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018 0002018 00002018 000002018 0000002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019 0002019 00002019 000002019 0000002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100 0002100 00002100 000002100 0000002100
+
+
+-- !query
+select col, date_format(col, 'q qq') from v
+-- !query schema
+struct<col:timestamp,date_format(col, q qq):string>
+-- !query output
+1582-05-31 19:40:35.123	2 02
+1969-12-31 15:00:00	4 04
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	1 01
+2018-11-17 05:33:33.123	4 04
+2019-12-31 09:33:33.123	4 04
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'Q QQ QQQ QQQQ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Q QQ QQQ QQQQ):string>
+-- !query output
+1582-05-31 19:40:35.123	2 02 Q2 2nd quarter
+1969-12-31 15:00:00	4 04 Q4 4th quarter
+1970-12-31 04:59:59.999	4 04 Q4 4th quarter
+1996-03-31 07:03:33.123	1 01 Q1 1st quarter
+2018-11-17 05:33:33.123	4 04 Q4 4th quarter
+2019-12-31 09:33:33.123	4 04 Q4 4th quarter
+2100-01-01 01:33:33.123	1 01 Q1 1st quarter
+
+
+-- !query
+select col, date_format(col, 'M MM MMM MMMM') from v
+-- !query schema
+struct<col:timestamp,date_format(col, M MM MMM MMMM):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05 May May
+1969-12-31 15:00:00	12 12 Dec December
+1970-12-31 04:59:59.999	12 12 Dec December
+1996-03-31 07:03:33.123	3 03 Mar March
+2018-11-17 05:33:33.123	11 11 Nov November
+2019-12-31 09:33:33.123	12 12 Dec December
+2100-01-01 01:33:33.123	1 01 Jan January
+
+
+-- !query
+select col, date_format(col, 'L LL') from v
+-- !query schema
+struct<col:timestamp,date_format(col, L LL):string>
+-- !query output
+1582-05-31 19:40:35.123	5 05
+1969-12-31 15:00:00	12 12
+1970-12-31 04:59:59.999	12 12
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	11 11
+2019-12-31 09:33:33.123	12 12
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'E EE EEE EEEE') from v
+-- !query schema
+struct<col:timestamp,date_format(col, E EE EEE EEEE):string>
+-- !query output
+1582-05-31 19:40:35.123	Mon Mon Mon Monday
+1969-12-31 15:00:00	Wed Wed Wed Wednesday
+1970-12-31 04:59:59.999	Thu Thu Thu Thursday
+1996-03-31 07:03:33.123	Sun Sun Sun Sunday
+2018-11-17 05:33:33.123	Sat Sat Sat Saturday
+2019-12-31 09:33:33.123	Tue Tue Tue Tuesday
+2100-01-01 01:33:33.123	Fri Fri Fri Friday
+
+
+-- !query
+select col, date_format(col, 'F') from v
+-- !query schema
+struct<col:timestamp,date_format(col, F):string>
+-- !query output
+1582-05-31 19:40:35.123	3
+1969-12-31 15:00:00	3
+1970-12-31 04:59:59.999	3
+1996-03-31 07:03:33.123	3
+2018-11-17 05:33:33.123	3
+2019-12-31 09:33:33.123	3
+2100-01-01 01:33:33.123	1
+
+
+-- !query
+select col, date_format(col, 'd dd') from v
+-- !query schema
+struct<col:timestamp,date_format(col, d dd):string>
+-- !query output
+1582-05-31 19:40:35.123	31 31
+1969-12-31 15:00:00	31 31
+1970-12-31 04:59:59.999	31 31
+1996-03-31 07:03:33.123	31 31
+2018-11-17 05:33:33.123	17 17
+2019-12-31 09:33:33.123	31 31
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'DD') from v where col = timestamp '2100-01-01 01:33:33.123America/Los_Angeles'
+-- !query schema
+struct<col:timestamp,date_format(col, DD):string>
+-- !query output
+2100-01-01 01:33:33.123	01
+
+
+-- !query
+select col, date_format(col, 'D DDD') from v
+-- !query schema
+struct<col:timestamp,date_format(col, D DDD):string>
+-- !query output
+1582-05-31 19:40:35.123	151 151
+1969-12-31 15:00:00	365 365
+1970-12-31 04:59:59.999	365 365
+1996-03-31 07:03:33.123	91 091
+2018-11-17 05:33:33.123	321 321
+2019-12-31 09:33:33.123	365 365
+2100-01-01 01:33:33.123	1 001
+
+
+-- !query
+select col, date_format(col, 'H HH') from v
+-- !query schema
+struct<col:timestamp,date_format(col, H HH):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'h hh') from v
+-- !query schema
+struct<col:timestamp,date_format(col, h hh):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'k kk') from v
+-- !query schema
+struct<col:timestamp,date_format(col, k kk):string>
+-- !query output
+1582-05-31 19:40:35.123	19 19
+1969-12-31 15:00:00	15 15
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'K KK') from v
+-- !query schema
+struct<col:timestamp,date_format(col, K KK):string>
+-- !query output
+1582-05-31 19:40:35.123	7 07
+1969-12-31 15:00:00	3 03
+1970-12-31 04:59:59.999	4 04
+1996-03-31 07:03:33.123	7 07
+2018-11-17 05:33:33.123	5 05
+2019-12-31 09:33:33.123	9 09
+2100-01-01 01:33:33.123	1 01
+
+
+-- !query
+select col, date_format(col, 'm mm') from v
+-- !query schema
+struct<col:timestamp,date_format(col, m mm):string>
+-- !query output
+1582-05-31 19:40:35.123	40 40
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	3 03
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 's ss') from v
+-- !query schema
+struct<col:timestamp,date_format(col, s ss):string>
+-- !query output
+1582-05-31 19:40:35.123	35 35
+1969-12-31 15:00:00	0 00
+1970-12-31 04:59:59.999	59 59
+1996-03-31 07:03:33.123	33 33
+2018-11-17 05:33:33.123	33 33
+2019-12-31 09:33:33.123	33 33
+2100-01-01 01:33:33.123	33 33
+
+
+-- !query
+select col, date_format(col, 'S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS') from v
+-- !query schema
+struct<col:timestamp,date_format(col, S SS SSS SSSS SSSSS SSSSSS SSSSSSS SSSSSSSS SSSSSSSSS):string>
+-- !query output
+1582-05-31 19:40:35.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+1969-12-31 15:00:00	0 00 000 0000 00000 000000 0000000 00000000 000000000
+1970-12-31 04:59:59.999	9 99 999 9990 99900 999000 9990000 99900000 999000000
+1996-03-31 07:03:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2018-11-17 05:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2019-12-31 09:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+2100-01-01 01:33:33.123	1 12 123 1230 12300 123000 1230000 12300000 123000000
+
+
+-- !query
+select col, date_format(col, 'a') from v
+-- !query schema
+struct<col:timestamp,date_format(col, a):string>
+-- !query output
+1582-05-31 19:40:35.123	PM
+1969-12-31 15:00:00	PM
+1970-12-31 04:59:59.999	AM
+1996-03-31 07:03:33.123	AM
+2018-11-17 05:33:33.123	AM
+2019-12-31 09:33:33.123	AM
+2100-01-01 01:33:33.123	AM
+
+
+-- !query
+select col, date_format(col, 'VV') from v
+-- !query schema
+struct<col:timestamp,date_format(col, VV):string>
+-- !query output
+1582-05-31 19:40:35.123	America/Los_Angeles
+1969-12-31 15:00:00	America/Los_Angeles
+1970-12-31 04:59:59.999	America/Los_Angeles
+1996-03-31 07:03:33.123	America/Los_Angeles
+2018-11-17 05:33:33.123	America/Los_Angeles
+2019-12-31 09:33:33.123	America/Los_Angeles
+2100-01-01 01:33:33.123	America/Los_Angeles
+
+
+-- !query
+select col, date_format(col, 'z zz zzz zzzz') from v
+-- !query schema
+struct<col:timestamp,date_format(col, z zz zzz zzzz):string>
+-- !query output
+1582-05-31 19:40:35.123	PST PST PST Pacific Standard Time
+1969-12-31 15:00:00	PST PST PST Pacific Standard Time
+1970-12-31 04:59:59.999	PST PST PST Pacific Standard Time
+1996-03-31 07:03:33.123	PST PST PST Pacific Standard Time
+2018-11-17 05:33:33.123	PST PST PST Pacific Standard Time
+2019-12-31 09:33:33.123	PST PST PST Pacific Standard Time
+2100-01-01 01:33:33.123	PST PST PST Pacific Standard Time
+
+
+-- !query
+select col, date_format(col, 'X XX XXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, X XX XXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -07:52
+1969-12-31 15:00:00	-08 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'XXXX XXXXX') from v
+-- !query schema
+struct<col:timestamp,date_format(col, XXXX XXXXX):string>
+-- !query output
+1582-05-31 19:40:35.123	-075258 -07:52:58
+1969-12-31 15:00:00	-0800 -08:00
+1970-12-31 04:59:59.999	-0800 -08:00
+1996-03-31 07:03:33.123	-0800 -08:00
+2018-11-17 05:33:33.123	-0800 -08:00
+2019-12-31 09:33:33.123	-0800 -08:00
+2100-01-01 01:33:33.123	-0800 -08:00
+
+
+-- !query
+select col, date_format(col, 'Z ZZ ZZZ ZZZZ ZZZZZ') from v
+-- !query schema
+struct<col:timestamp,date_format(col, Z ZZ ZZZ ZZZZ ZZZZZ):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -0752 GMT-07:52:58 -07:52:58
+1969-12-31 15:00:00	-0800 -0800 -0800 GMT-08:00 -08:00
+1970-12-31 04:59:59.999	-0800 -0800 -0800 GMT-08:00 -08:00
+1996-03-31 07:03:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2018-11-17 05:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2019-12-31 09:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+2100-01-01 01:33:33.123	-0800 -0800 -0800 GMT-08:00 -08:00
+
+
+-- !query
+select col, date_format(col, 'O OOOO') from v
+-- !query schema
+struct<col:timestamp,date_format(col, O OOOO):string>
+-- !query output
+1582-05-31 19:40:35.123	GMT-7:52:58 GMT-07:52:58
+1969-12-31 15:00:00	GMT-8 GMT-08:00
+1970-12-31 04:59:59.999	GMT-8 GMT-08:00
+1996-03-31 07:03:33.123	GMT-8 GMT-08:00
+2018-11-17 05:33:33.123	GMT-8 GMT-08:00
+2019-12-31 09:33:33.123	GMT-8 GMT-08:00
+2100-01-01 01:33:33.123	GMT-8 GMT-08:00
+
+
+-- !query
+select col, date_format(col, 'x xx xxx xxxx xxxx xxxxx') from v
+-- !query schema
+struct<col:timestamp,date_format(col, x xx xxx xxxx xxxx xxxxx):string>
+-- !query output
+1582-05-31 19:40:35.123	-0752 -0752 -07:52 -075258 -075258 -07:52:58
+1969-12-31 15:00:00	-08 -0800 -08:00 -0800 -0800 -08:00
+1970-12-31 04:59:59.999	-08 -0800 -08:00 -0800 -0800 -08:00
+1996-03-31 07:03:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2018-11-17 05:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2019-12-31 09:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+2100-01-01 01:33:33.123	-08 -0800 -08:00 -0800 -0800 -08:00
+
+
+-- !query
+select col, date_format(col, '[yyyy-MM-dd HH:mm:ss]') from v
+-- !query schema
+struct<col:timestamp,date_format(col, [yyyy-MM-dd HH:mm:ss]):string>
+-- !query output
+1582-05-31 19:40:35.123	1582-05-31 19:40:35
+1969-12-31 15:00:00	1969-12-31 15:00:00
+1970-12-31 04:59:59.999	1970-12-31 04:59:59
+1996-03-31 07:03:33.123	1996-03-31 07:03:33
+2018-11-17 05:33:33.123	2018-11-17 05:33:33
+2019-12-31 09:33:33.123	2019-12-31 09:33:33
+2100-01-01 01:33:33.123	2100-01-01 01:33:33
+
+
+-- !query
+select col, date_format(col, "姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'") from v
+-- !query schema
+struct<col:timestamp,date_format(col, 姚123'GyYqQMLwWuEFDdhHmsSaVzZxXOV'):string>
+-- !query output
+1582-05-31 19:40:35.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1969-12-31 15:00:00	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1970-12-31 04:59:59.999	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+1996-03-31 07:03:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2018-11-17 05:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2019-12-31 09:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+2100-01-01 01:33:33.123	姚123GyYqQMLwWuEFDdhHmsSaVzZxXOV
+
+
+-- !query
+select col, date_format(col, "''") from v
+-- !query schema
+struct<col:timestamp,date_format(col, ''):string>
+-- !query output
+1582-05-31 19:40:35.123	'
+1969-12-31 15:00:00	'
+1970-12-31 04:59:59.999	'
+1996-03-31 07:03:33.123	'
+2018-11-17 05:33:33.123	'
+2019-12-31 09:33:33.123	'
+2100-01-01 01:33:33.123	'
+
+
+-- !query
+select col, date_format(col, '') from v
+-- !query schema
+struct<col:timestamp,date_format(col, ):string>
+-- !query output
+1582-05-31 19:40:35.123	
+1969-12-31 15:00:00	
+1970-12-31 04:59:59.999	
+1996-03-31 07:03:33.123	
+2018-11-17 05:33:33.123	
+2019-12-31 09:33:33.123	
+2100-01-01 01:33:33.123
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
index 71b106485896b..b244f497aeacc 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 112
+-- Number of queries: 109
 
 
 -- !query
@@ -646,32 +646,6 @@ struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
 2019-10-06 00:00:00
 
 
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
--- !query schema
-struct<>
--- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character 'e'
-
-
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
--- !query schema
-struct<>
--- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character 'c'
-
-
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
--- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
--- !query output
-2019-10-06 0007
-
-
 -- !query
 select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 9b1c8472fa75e..3255761545fdd 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 112
+-- Number of queries: 109
 
 
 -- !query
@@ -646,32 +646,6 @@ struct<to_timestamp('S2019-10-06', '\'S\'yyyy-MM-dd'):timestamp>
 2019-10-06 00:00:00
 
 
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuee')
--- !query schema
-struct<>
--- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: e
-
-
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uucc')
--- !query schema
-struct<>
--- !query output
-java.lang.IllegalArgumentException
-Illegal pattern character: c
-
-
--- !query
-select date_format(timestamp '2019-10-06', 'yyyy-MM-dd uuuu')
--- !query schema
-struct<date_format(TIMESTAMP '2019-10-06 00:00:00', yyyy-MM-dd uuuu):string>
--- !query output
-2019-10-06 Sunday
-
-
 -- !query
 select to_timestamp("2019-10-06T10:11:12'12", "yyyy-MM-dd'T'HH:mm:ss''SSSS")
 -- !query schema

From 8910dae0966ec7bbd9886245310fa07b81426e74 Mon Sep 17 00:00:00 2001
From: Juliusz Sompolski <julek@databricks.com>
Date: Fri, 5 Jun 2020 10:01:00 +0000
Subject: [PATCH 0790/1280] [SPARK-31859][SPARK-31861][FOLLOWUP] Fix typo in
 tests

### What changes were proposed in this pull request?

It appears I have unintentionally used nested JDBC statements in the two tests I added.

### Why are the changes needed?

Cleanup a typo. Please merge to master/branch-3.0

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit tests.

Closes #28735 from juliuszsompolski/SPARK-31859-fixup.

Authored-by: Juliusz Sompolski <julek@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ea010138e95cc9c0f62603d431b906bd5104f4e3)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../HiveThriftServer2Suites.scala             | 36 +++++++++----------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 21256ad02c134..4e6d4e104021f 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -813,31 +813,27 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
   }
 
   test("SPARK-31859 Thriftserver works with spark.sql.datetime.java8API.enabled=true") {
-    withJdbcStatement() { statement =>
-      withJdbcStatement() { st =>
-        st.execute("set spark.sql.datetime.java8API.enabled=true")
-        val rs = st.executeQuery("select date '2020-05-28', timestamp '2020-05-28 00:00:00'")
-        rs.next()
-        assert(rs.getDate(1).toString() == "2020-05-28")
-        assert(rs.getTimestamp(2).toString() == "2020-05-28 00:00:00.0")
-      }
+    withJdbcStatement() { st =>
+      st.execute("set spark.sql.datetime.java8API.enabled=true")
+      val rs = st.executeQuery("select date '2020-05-28', timestamp '2020-05-28 00:00:00'")
+      rs.next()
+      assert(rs.getDate(1).toString() == "2020-05-28")
+      assert(rs.getTimestamp(2).toString() == "2020-05-28 00:00:00.0")
     }
   }
 
   test("SPARK-31861 Thriftserver respects spark.sql.session.timeZone") {
-    withJdbcStatement() { statement =>
-      withJdbcStatement() { st =>
-        st.execute("set spark.sql.session.timeZone=+03:15") // different than Thriftserver's JVM tz
+    withJdbcStatement() { st =>
+      st.execute("set spark.sql.session.timeZone=+03:15") // different than Thriftserver's JVM tz
       val rs = st.executeQuery("select timestamp '2020-05-28 10:00:00'")
-        rs.next()
-        // The timestamp as string is the same as the literal
-        assert(rs.getString(1) == "2020-05-28 10:00:00.0")
-        // Parsing it to java.sql.Timestamp in the client will always result in a timestamp
-        // in client default JVM timezone. The string value of the Timestamp will match the literal,
-        // but if the JDBC application cares about the internal timezone and UTC offset of the
-        // Timestamp object, it should set spark.sql.session.timeZone to match its client JVM tz.
-        assert(rs.getTimestamp(1).toString() == "2020-05-28 10:00:00.0")
-      }
+      rs.next()
+      // The timestamp as string is the same as the literal
+      assert(rs.getString(1) == "2020-05-28 10:00:00.0")
+      // Parsing it to java.sql.Timestamp in the client will always result in a timestamp
+      // in client default JVM timezone. The string value of the Timestamp will match the literal,
+      // but if the JDBC application cares about the internal timezone and UTC offset of the
+      // Timestamp object, it should set spark.sql.session.timeZone to match its client JVM tz.
+      assert(rs.getTimestamp(1).toString() == "2020-05-28 10:00:00.0")
     }
   }
 

From 1fc09e3cfbd08c385a8c5cffa3e85a089852f8bc Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 5 Jun 2020 16:44:16 +0000
Subject: [PATCH 0791/1280] [SPARK-31867][SQL][FOLLOWUP] Check result
 differences for datetime formatting

### What changes were proposed in this pull request?

In this PR, we throw `SparkUpgradeException` when getting `DateTimeException` for datetime formatting in the `EXCEPTION` legacy Time Parser Policy.

### Why are the changes needed?
`DateTimeException` is also declared by `java.time.format.DateTimeFormatter#format`, but in Spark, it can barely occur. We have suspected one that due to a JDK bug so far. see https://bugs.openjdk.java.net/browse/JDK-8079628.

For `from_unixtime` function, we will suppress the DateTimeException caused by `DD` and result `NULL`. It is a silent date change that should be avoided in Java 8.

### Does this PR introduce _any_ user-facing change?

Yes,  when running on Java8 and using `from_unixtime` function with pattern `DD` to format datetimes, if dayofyear>=100, `SparkUpgradeException` will alert users instead of silently resulting null. For `date_format`, `SparkUpgradeException` take the palace of  `DateTimeException`.

### How was this patch tested?

add unit tests.

Closes #28736 from yaooqinn/SPARK-31867-F.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit fc6af9d900ec6f6a1cbe8f987857a69e6ef600d1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateFormatter.scala     |  7 +++--
 .../util/DateTimeFormatterHelper.scala        | 30 ++++++++++++++++---
 .../catalyst/util/TimestampFormatter.scala    |  7 +++--
 .../util/TimestampFormatterSuite.scala        | 15 ++++++++++
 4 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index fbc9f56dd6045..ec8db4653d137 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -58,12 +58,15 @@ class Iso8601DateFormatter(
       try {
         val localDate = toLocalDate(formatter.parse(s))
         localDateToDays(localDate)
-      } catch checkDiffResult(s, legacyFormatter.parse)
+      } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
 
   override def format(localDate: LocalDate): String = {
-    localDate.format(formatter)
+    try {
+      localDate.format(formatter)
+    } catch checkFormattedDiff(toJavaDate(localDateToDays(localDate)),
+      (d: Date) => format(d))
   }
 
   override def format(days: Int): String = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 8e5c8651c8c36..992a2b12a462f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -21,7 +21,7 @@ import java.time._
 import java.time.chrono.IsoChronology
 import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle}
 import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
-import java.util.Locale
+import java.util.{Date, Locale}
 
 import com.google.common.cache.CacheBuilder
 
@@ -109,13 +109,17 @@ trait DateTimeFormatterHelper {
     formatter
   }
 
+  private def needConvertToSparkUpgradeException(e: Throwable): Boolean = e match {
+    case _: DateTimeException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION => true
+    case _ => false
+  }
   // When legacy time parser policy set to EXCEPTION, check whether we will get different results
   // between legacy parser and new parser. If new parser fails but legacy parser works, throw a
   // SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
   // DateTimeParseException will address by the caller side.
-  protected def checkDiffResult[T](
+  protected def checkParsedDiff[T](
       s: String, legacyParseFunc: String => T): PartialFunction[Throwable, T] = {
-    case e: DateTimeException if SQLConf.get.legacyTimeParserPolicy == EXCEPTION =>
+    case e if needConvertToSparkUpgradeException(e) =>
       try {
         legacyParseFunc(s)
       } catch {
@@ -126,6 +130,25 @@ trait DateTimeFormatterHelper {
         s"before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.", e)
   }
 
+  // When legacy time parser policy set to EXCEPTION, check whether we will get different results
+  // between legacy formatter and new formatter. If new formatter fails but legacy formatter works,
+  // throw a SparkUpgradeException. On the contrary, if the legacy policy set to CORRECTED,
+  // DateTimeParseException will address by the caller side.
+  protected def checkFormattedDiff[T <: Date](
+      d: T,
+      legacyFormatFunc: T => String): PartialFunction[Throwable, String] = {
+    case e if needConvertToSparkUpgradeException(e) =>
+      val resultCandidate = try {
+        legacyFormatFunc(d)
+      } catch {
+        case _: Throwable => throw e
+      }
+      throw new SparkUpgradeException("3.0", s"Fail to format it to '$resultCandidate' in the new" +
+        s" formatter. You can set ${SQLConf.LEGACY_TIME_PARSER_POLICY.key} to LEGACY to restore" +
+        " the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid" +
+        " datetime string.", e)
+  }
+
   /**
    * When the new DateTimeFormatter failed to initialize because of invalid datetime pattern, it
    * will throw IllegalArgumentException. If the pattern can be recognized by the legacy formatter
@@ -137,7 +160,6 @@ trait DateTimeFormatterHelper {
    * @param tryLegacyFormatter a func to capture exception, identically which forces a legacy
    *                           datetime formatter to be initialized
    */
-
   protected def checkLegacyFormatter(
       pattern: String,
       tryLegacyFormatter: => Unit): PartialFunction[Throwable, DateTimeFormatter] = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 6bcbb09b22b4b..87d8a3e682b8e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -84,12 +84,15 @@ class Iso8601TimestampFormatter(
         val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
 
         Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond)
-      } catch checkDiffResult(s, legacyFormatter.parse)
+      } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
 
   override def format(instant: Instant): String = {
-    formatter.withZone(zoneId).format(instant)
+    try {
+      formatter.withZone(zoneId).format(instant)
+    } catch checkFormattedDiff(toJavaTimestamp(instantToMicros(instant)),
+      (t: Timestamp) => format(t))
   }
 
   override def format(us: Long): String = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index 311097f31a15d..88f6b0d79f85e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -415,4 +415,19 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
     val t5 = f3.parse("AM")
     assert(t5 === date(1970))
   }
+
+  test("check result differences for datetime formatting") {
+    val formatter = TimestampFormatter("DD", UTC, isParsing = false)
+    assert(formatter.format(date(1970, 1, 3)) == "03")
+    assert(formatter.format(date(1970, 4, 9)) == "99")
+
+    if (System.getProperty("java.version").split("\\D+")(0).toInt < 9) {
+      // https://bugs.openjdk.java.net/browse/JDK-8079628
+      intercept[SparkUpgradeException] {
+        formatter.format(date(1970, 4, 10))
+      }
+    } else {
+      assert(formatter.format(date(1970, 4, 10)) == "100")
+    }
+  }
 }

From fa608b949b854d716904f4e43a4a10c71742b3c6 Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Sat, 6 Jun 2020 07:35:25 +0900
Subject: [PATCH 0792/1280] [SPARK-31904][SQL] Fix case sensitive problem of
 char and varchar partition columns

### What changes were proposed in this pull request?
```sql
CREATE TABLE t1(a STRING, B VARCHAR(10), C CHAR(10)) STORED AS parquet;
CREATE TABLE t2 USING parquet PARTITIONED BY (b, c) AS SELECT * FROM t1;
SELECT * FROM t2 WHERE b = 'A';
```
Above SQL throws MetaException

> Caused by: java.lang.reflect.InvocationTargetException
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.spark.sql.hive.client.Shim_v0_13.getPartitionsByFilter(HiveShim.scala:810)
	... 114 more
Caused by: MetaException(message:Filtering is supported only on partition keys of type string, or integral types)
	at org.apache.hadoop.hive.metastore.parser.ExpressionTree$FilterBuilder.setError(ExpressionTree.java:184)
	at org.apache.hadoop.hive.metastore.parser.ExpressionTree$LeafNode.getJdoFilterPushdownParam(ExpressionTree.java:439)
	at org.apache.hadoop.hive.metastore.parser.ExpressionTree$LeafNode.generateJDOFilterOverPartitions(ExpressionTree.java:356)
	at org.apache.hadoop.hive.metastore.parser.ExpressionTree$LeafNode.generateJDOFilter(ExpressionTree.java:278)
	at org.apache.hadoop.hive.metastore.parser.ExpressionTree.generateJDOFilterFragment(ExpressionTree.java:583)
	at org.apache.hadoop.hive.metastore.ObjectStore.makeQueryFilterString(ObjectStore.java:3315)
	at org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsViaOrmFilter(ObjectStore.java:2768)
	at org.apache.hadoop.hive.metastore.ObjectStore.access$500(ObjectStore.java:182)
	at org.apache.hadoop.hive.metastore.ObjectStore$7.getJdoResult(ObjectStore.java:3248)
	at org.apache.hadoop.hive.metastore.ObjectStore$7.getJdoResult(ObjectStore.java:3232)
	at org.apache.hadoop.hive.metastore.ObjectStore$GetHelper.run(ObjectStore.java:2974)
	at org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsByFilterInternal(ObjectStore.java:3250)
	at org.apache.hadoop.hive.metastore.ObjectStore.getPartitionsByFilter(ObjectStore.java:2906)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.hadoop.hive.metastore.RawStoreProxy.invoke(RawStoreProxy.java:101)
	at com.sun.proxy.$Proxy25.getPartitionsByFilter(Unknown Source)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.get_partitions_by_filter(HiveMetaStore.java:5093)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107)
	at com.sun.proxy.$Proxy26.get_partitions_by_filter(Unknown Source)
	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.listPartitionsByFilter(HiveMetaStoreClient.java:1232)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:173)
	at com.sun.proxy.$Proxy27.listPartitionsByFilter(Unknown Source)
	at org.apache.hadoop.hive.ql.metadata.Hive.getPartitionsByFilter(Hive.java:2679)
	... 119 more

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Add a unit test.

Closes #28724 from LantaoJin/SPARK-31904.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 5079831106ba22f81a26b4a8104a253422fa1b6a)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../org/apache/spark/sql/hive/client/HiveShim.scala    |  3 ++-
 .../apache/spark/sql/hive/execution/HiveDDLSuite.scala | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 2b806609426a1..8df43b785759e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -708,7 +708,8 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
         .map(col => col.getName).toSet
 
       def unapply(attr: Attribute): Option[String] = {
-        if (varcharKeys.contains(attr.name)) {
+        val resolver = SQLConf.get.resolver
+        if (varcharKeys.exists(c => resolver(c, attr.name))) {
           None
         } else if (attr.dataType.isInstanceOf[IntegralType] || attr.dataType == StringType) {
           Some(attr.name)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index e8548fd62ddc1..e8cf4ad5d9f28 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -2720,4 +2720,14 @@ class HiveDDLSuite
       checkAnswer(sql("SHOW PARTITIONS ta_part"), Row("ts=10") :: Nil)
     }
   }
+
+  test("SPARK-31904: Fix case sensitive problem of char and varchar partition columns") {
+    withTable("t1", "t2") {
+      sql("CREATE TABLE t1(a STRING, B VARCHAR(10), C CHAR(10)) STORED AS parquet")
+      sql("CREATE TABLE t2 USING parquet PARTITIONED BY (b, c) AS SELECT * FROM t1")
+      // make sure there is no exception
+      assert(sql("SELECT * FROM t2 WHERE b = 'A'").collect().isEmpty)
+      assert(sql("SELECT * FROM t2 WHERE c = 'A'").collect().isEmpty)
+    }
+  }
 }

From 3fdfce3120f307147244e5eaf46d61419a723d50 Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@apache.org>
Date: Sat, 6 Jun 2020 02:57:35 +0000
Subject: [PATCH 0793/1280] Preparing Spark release v3.0.0-rc3

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 21f3eaa5c15f6..3bad429ca4086 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.1
+Version: 3.0.0
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 8bef9d8bc56cb..0a52a00c4a694 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index fc1441d80b7bf..fa4fcb1fbfa80 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index de2a6fb18d69b..14a1b7db9e33d 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index 6c0c016fdf94f..e75a843f5e6bb 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index b8df19152a606..004af0a1ab203 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 8119709a6aeff..a35156a4bbfff 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 96ac525880213..dedc7dfc82056 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index f385f07f951ce..ebb05257d85f9 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 0c9befc921af6..ad743a7712812 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index db2f6721e0cc4..a5c3106c4e261 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,8 +14,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.1-SNAPSHOT
-SPARK_VERSION_SHORT: 3.0.1
+SPARK_VERSION: 3.0.0
+SPARK_VERSION_SHORT: 3.0.0
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index 1b59c629a8c73..ec3490545132d 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 87183e72f5de3..70cada7ce95e4 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index b57e3c0e25ae8..32679c759404e 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 9426db5e11354..ae22afc3faeae 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index d77364d0210fe..7b10f0e78bc28 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index 70a3406701315..b2ecc0f81d7ac 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index a338b6511298f..19e7352e3945c 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 599ae172671e5..bb37b213c697b 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 9fa18d9f82833..cfe7b7373b967 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index acfc2c5a36b1a..d2f828421e64f 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index f0666536bb706..ebfaccffcdded 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 48f559e4822f3..8a62a48236aab 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 0759ce5c114b3..c2f25df2f147e 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index ed34a370a8274..bb4b640c68fe9 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index b8ae344e63e65..c41bbe6ea85f8 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index d8d559550a5be..2d8a52cfe728c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.1-SNAPSHOT</version>
+  <version>3.0.0</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 906298ff9964a..8011a5f7a3423 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.1.dev0"
+__version__ = "3.0.0"
diff --git a/repl/pom.xml b/repl/pom.xml
index b826b87f53efc..83c8d34a3a132 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 168f7f5be8b0f..f72bacf827dba 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 6c258a7ce9042..9ede26ae51ad9 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index e0c997f869aba..c61f8dff8cf55 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 95299d2066ea3..f18f2ce58588a 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 6667c7bd089dc..14a0f665ea84e 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index fbfcdf88fa26e..3388aca287aa3 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index f60820dd45f34..6dec472392574 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index e8c1598b3669c..267bd6039e3fa 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 1e99b6c21522b..d6f4f303ad423 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 5430a3e600365..4a58881077524 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.0</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 3ea461d61e635835c07bacb5a0c403ae2a3099a0 Mon Sep 17 00:00:00 2001
From: Reynold Xin <rxin@apache.org>
Date: Sat, 6 Jun 2020 02:57:41 +0000
Subject: [PATCH 0794/1280] Preparing development version 3.0.1-SNAPSHOT

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 3bad429ca4086..21f3eaa5c15f6 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.0
+Version: 3.0.1
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0a52a00c4a694..8bef9d8bc56cb 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index fa4fcb1fbfa80..fc1441d80b7bf 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 14a1b7db9e33d..de2a6fb18d69b 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index e75a843f5e6bb..6c0c016fdf94f 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 004af0a1ab203..b8df19152a606 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index a35156a4bbfff..8119709a6aeff 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index dedc7dfc82056..96ac525880213 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index ebb05257d85f9..f385f07f951ce 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index ad743a7712812..0c9befc921af6 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index a5c3106c4e261..db2f6721e0cc4 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -14,8 +14,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.0
-SPARK_VERSION_SHORT: 3.0.0
+SPARK_VERSION: 3.0.1-SNAPSHOT
+SPARK_VERSION_SHORT: 3.0.1
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index ec3490545132d..1b59c629a8c73 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 70cada7ce95e4..87183e72f5de3 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 32679c759404e..b57e3c0e25ae8 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index ae22afc3faeae..9426db5e11354 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 7b10f0e78bc28..d77364d0210fe 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index b2ecc0f81d7ac..70a3406701315 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 19e7352e3945c..a338b6511298f 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index bb37b213c697b..599ae172671e5 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index cfe7b7373b967..9fa18d9f82833 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index d2f828421e64f..acfc2c5a36b1a 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index ebfaccffcdded..f0666536bb706 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 8a62a48236aab..48f559e4822f3 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index c2f25df2f147e..0759ce5c114b3 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index bb4b640c68fe9..ed34a370a8274 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index c41bbe6ea85f8..b8ae344e63e65 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 2d8a52cfe728c..d8d559550a5be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.0</version>
+  <version>3.0.1-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 8011a5f7a3423..906298ff9964a 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.0"
+__version__ = "3.0.1.dev0"
diff --git a/repl/pom.xml b/repl/pom.xml
index 83c8d34a3a132..b826b87f53efc 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index f72bacf827dba..168f7f5be8b0f 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 9ede26ae51ad9..6c258a7ce9042 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index c61f8dff8cf55..e0c997f869aba 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index f18f2ce58588a..95299d2066ea3 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 14a0f665ea84e..6667c7bd089dc 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 3388aca287aa3..fbfcdf88fa26e 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 6dec472392574..f60820dd45f34 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 267bd6039e3fa..e8c1598b3669c 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d6f4f303ad423..1e99b6c21522b 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 4a58881077524..5430a3e600365 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.1-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From c5683fce6f43dd561677830d74af196bc6c97134 Mon Sep 17 00:00:00 2001
From: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Date: Sat, 6 Jun 2020 16:49:48 +0900
Subject: [PATCH 0795/1280] [MINOR][SS][DOCS] fileNameOnly parameter
 description re-unite

### What changes were proposed in this pull request?
`fileNameOnly` parameter is split to 2 pieces in [this](https://github.com/apache/spark/commit/dbb8143501ab87865d6e202c17297b9a73a0b1c3) commit. This PR re-unites it.

### Why are the changes needed?
Parameter description split in doc.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
```
cd docs/
SKIP_API=1 jekyll build
```
Manual webpage check.

Closes #28739 from gaborgsomogyi/datasettxtfix.

Authored-by: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 04f66bfd4eb863253ac9c30594055b8d5997c321)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/structured-streaming-programming-guide.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 1776d23607f7f..69d744d09e1e5 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -540,12 +540,13 @@ Here are the details of all the sources in Spark.
         <br/>
         <code>fileNameOnly</code>: whether to check new files based on only the filename instead of on the full path (default: false). With this set to `true`, the following files would be considered as the same file, because their filenames, "dataset.txt", are the same:
         <br/>
-        <code>maxFileAge</code>: Maximum age of a file that can be found in this directory, before it is ignored. For the first batch all files will be considered valid. If <code>latestFirst</code> is set to `true` and <code>maxFilesPerTrigger</code> is set, then this parameter will be ignored, because old files that are valid, and should be processed, may be ignored. The max age is specified with respect to the timestamp of the latest file, and not the timestamp of the current system.(default: 1 week)
-        <br/>
         "file:///dataset.txt"<br/>
         "s3://a/dataset.txt"<br/>
         "s3n://a/b/dataset.txt"<br/>
-        "s3a://a/b/c/dataset.txt"<br/>
+        "s3a://a/b/c/dataset.txt"
+        <br/>
+        <code>maxFileAge</code>: Maximum age of a file that can be found in this directory, before it is ignored. For the first batch all files will be considered valid. If <code>latestFirst</code> is set to `true` and <code>maxFilesPerTrigger</code> is set, then this parameter will be ignored, because old files that are valid, and should be processed, may be ignored. The max age is specified with respect to the timestamp of the latest file, and not the timestamp of the current system.(default: 1 week)
+        <br/>
         <code>cleanSource</code>: option to clean up completed files after processing.<br/>
         Available options are "archive", "delete", "off". If the option is not provided, the default value is "off".<br/>
         When "archive" is provided, additional option <code>sourceArchiveDir</code> must be provided as well. The value of "sourceArchiveDir" must not match with source pattern in depth (the number of directories from the root directory), where the depth is minimum of depth on both paths. This will ensure archived files are never included as new source files.<br/>

From d571cbfa46048518bdda5ca0c858c769149dac7d Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 8 Jun 2020 15:18:42 +0900
Subject: [PATCH 0796/1280] [SPARK-31849][PYTHON][SQL][FOLLOW-UP] Deduplicate
 and reuse Utils.exceptionString in Python exception handling

### What changes were proposed in this pull request?

This PR proposes to use existing util `org.apache.spark.util.Utils.exceptionString` for the same codes at:

```python
    jwriter = jvm.java.io.StringWriter()
    e.printStackTrace(jvm.java.io.PrintWriter(jwriter))
    stacktrace = jwriter.toString()
```

### Why are the changes needed?

To deduplicate codes. Plus, less communication between JVM and Py4j.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually tested.

Closes #28749 from HyukjinKwon/SPARK-31849-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/utils.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 3fd704721a92c..1dbea12ab53ef 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -97,11 +97,8 @@ class UnknownException(CapturedException):
 def convert_exception(e):
     s = e.toString()
     c = e.getCause()
+    stacktrace = SparkContext._jvm.org.apache.spark.util.Utils.exceptionString(e)
 
-    jvm = SparkContext._jvm
-    jwriter = jvm.java.io.StringWriter()
-    e.printStackTrace(jvm.java.io.PrintWriter(jwriter))
-    stacktrace = jwriter.toString()
     if s.startswith('org.apache.spark.sql.AnalysisException: '):
         return AnalysisException(s.split(': ', 1)[1], stacktrace, c)
     if s.startswith('org.apache.spark.sql.catalyst.analysis'):

From b00ac30dfb621962e5b39c52a3bb09440936a0ff Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Mon, 8 Jun 2020 12:06:17 -0700
Subject: [PATCH 0797/1280] [SPARK-31923][CORE] Ignore internal accumulators
 that use unrecognized types rather than crashing

### What changes were proposed in this pull request?

Ignore internal accumulators that use unrecognized types rather than crashing so that an event log containing such accumulators can still be converted to JSON and logged.

### Why are the changes needed?

A user may use internal accumulators by adding the `internal.metrics.` prefix to the accumulator name to hide sensitive information from UI (Accumulators except internal ones will be shown in Spark UI).

However, `org.apache.spark.util.JsonProtocol.accumValueToJson` assumes an internal accumulator has only 3 possible types: `int`, `long`, and `java.util.List[(BlockId, BlockStatus)]`. When an internal accumulator uses an unexpected type, it will crash.

An event log that contains such accumulator will be dropped because it cannot be converted to JSON, and it will cause weird UI issue when rendering in Spark History Server. For example, if `SparkListenerTaskEnd` is dropped because of this issue, the user will see the task is still running even if it was finished.

It's better to make `accumValueToJson` more robust because it's up to the user to pick up the accumulator name.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

The new unit tests.

Closes #28744 from zsxwing/fix-internal-accum.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
(cherry picked from commit b333ed0c4a5733a9c36ad79de1d4c13c6cf3c5d4)
Signed-off-by: Shixiong Zhu <zsxwing@gmail.com>
---
 .../org/apache/spark/util/JsonProtocol.scala  | 20 ++++++--
 .../apache/spark/util/JsonProtocolSuite.scala | 48 +++++++++++++++++++
 2 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index f445fd4c6f7a6..d53ca0f5ec6aa 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -351,12 +351,22 @@ private[spark] object JsonProtocol {
         case v: Long => JInt(v)
         // We only have 3 kind of internal accumulator types, so if it's not int or long, it must be
         // the blocks accumulator, whose type is `java.util.List[(BlockId, BlockStatus)]`
-        case v =>
-          JArray(v.asInstanceOf[java.util.List[(BlockId, BlockStatus)]].asScala.toList.map {
-            case (id, status) =>
-              ("Block ID" -> id.toString) ~
-              ("Status" -> blockStatusToJson(status))
+        case v: java.util.List[_] =>
+          JArray(v.asScala.toList.flatMap {
+            case (id: BlockId, status: BlockStatus) =>
+              Some(
+                ("Block ID" -> id.toString) ~
+                ("Status" -> blockStatusToJson(status))
+              )
+            case _ =>
+              // Ignore unsupported types. A user may put `METRICS_PREFIX` in the name. We should
+              // not crash.
+              None
           })
+        case _ =>
+          // Ignore unsupported types. A user may put `METRICS_PREFIX` in the name. We should not
+          // crash.
+          JNothing
       }
     } else {
       // For all external accumulators, just use strings
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index d1f09d809ea50..5f1c75319faf8 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -482,6 +482,54 @@ class JsonProtocolSuite extends SparkFunSuite {
     testAccumValue(Some("anything"), 123, JString("123"))
   }
 
+  /** Create an AccumulableInfo and verify we can serialize and deserialize it. */
+  private def testAccumulableInfo(
+      name: String,
+      value: Option[Any],
+      expectedValue: Option[Any]): Unit = {
+    val isInternal = name.startsWith(InternalAccumulator.METRICS_PREFIX)
+    val accum = AccumulableInfo(
+      123L,
+      Some(name),
+      update = value,
+      value = value,
+      internal = isInternal,
+      countFailedValues = false)
+    val json = JsonProtocol.accumulableInfoToJson(accum)
+    val newAccum = JsonProtocol.accumulableInfoFromJson(json)
+    assert(newAccum == accum.copy(update = expectedValue, value = expectedValue))
+  }
+
+  test("SPARK-31923: unexpected value type of internal accumulator") {
+    // Because a user may use `METRICS_PREFIX` in an accumulator name, we should test unexpected
+    // types to make sure we don't crash.
+    import InternalAccumulator.METRICS_PREFIX
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooString",
+      value = Some("foo"),
+      expectedValue = None)
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooList",
+      value = Some(java.util.Arrays.asList("string")),
+      expectedValue = Some(java.util.Collections.emptyList())
+    )
+    val blocks = Seq(
+      (TestBlockId("block1"), BlockStatus(StorageLevel.MEMORY_ONLY, 1L, 2L)),
+      (TestBlockId("block2"), BlockStatus(StorageLevel.DISK_ONLY, 3L, 4L)))
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooList",
+      value = Some(java.util.Arrays.asList(
+        "string",
+        blocks(0),
+        blocks(1))),
+      expectedValue = Some(blocks.asJava)
+    )
+    testAccumulableInfo(
+      METRICS_PREFIX + "fooSet",
+      value = Some(Set("foo")),
+      expectedValue = None)
+  }
+
   test("SPARK-30936: forwards compatibility - ignore unknown fields") {
     val expected = TestListenerEvent("foo", 123)
     val unknownFieldsJson =

From bd0f5f2c57ae954d29b0940c1cff77cd2fff8ed7 Mon Sep 17 00:00:00 2001
From: Holden Karau <hkarau@apple.com>
Date: Mon, 8 Jun 2020 16:03:13 -0700
Subject: [PATCH 0798/1280] [SPARK-31934][BUILD] Remove set -x from docker
 image tool

### What changes were proposed in this pull request?

Remove `set -x` from the docker image tool.

### Why are the changes needed?

The image tool prints out information that may confusing.

### Does this PR introduce _any_ user-facing change?

Less information is displayed by the docker image tool.

### How was this patch tested?

Ran docker image tool locally.

Closes #28759 from holdenk/SPARK-31934-remove-extranious-info-from-the-docker-image-tool.

Authored-by: Holden Karau <hkarau@apple.com>
Signed-off-by: Holden Karau <hkarau@apple.com>
(cherry picked from commit 06959ebc399e4fa6a90c30e4f0c897cad1f6a496)
Signed-off-by: Holden Karau <hkarau@apple.com>
---
 bin/docker-image-tool.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 8a01b80c4164b..6d74f8328aea2 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -19,8 +19,6 @@
 # This script builds and pushes docker images when run from a release of Spark
 # with Kubernetes support.
 
-set -x
-
 function error {
   echo "$@" 1>&2
   exit 1

From 614659b270938c95bba7d3d956c0a1ac3ec5f469 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 9 Jun 2020 10:24:34 +0900
Subject: [PATCH 0799/1280] [SPARK-31849][PYTHON][SQL][FOLLOW-UP] More correct
 error message in Python UDF exception message

### What changes were proposed in this pull request?

This PR proposes to fix wordings in the Python UDF exception error message from:

From:

> An exception was thrown from Python worker in the executor. The below is the Python worker stacktrace.

To:

> An exception was thrown from the Python worker. Please see the stack trace below.

It removes "executor" because Python worker is technically a separate process, and remove the duplicated wording "Python worker" .

### Why are the changes needed?

To give users better exception messages.

### Does this PR introduce _any_ user-facing change?

No, it's in unreleased branches only. If RC3 passes, yes, it will change the exception message.

### How was this patch tested?

Manually tested.

Closes #28762 from HyukjinKwon/SPARK-31849-followup-2.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e28914095aa1fa7a4680b5e4fcf69e3ef64b3dbc)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index 1dbea12ab53ef..1d5bc49d252e2 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -116,8 +116,8 @@ def convert_exception(e):
             # To make sure this only catches Python UDFs.
             and any(map(lambda v: "org.apache.spark.sql.execution.python" in v.toString(),
                         c.getStackTrace()))):
-        msg = ("\n  An exception was thrown from Python worker in the executor. "
-               "The below is the Python worker stacktrace.\n%s" % c.getMessage())
+        msg = ("\n  An exception was thrown from the Python worker. "
+               "Please see the stack trace below.\n%s" % c.getMessage())
         return PythonException(msg, stacktrace)
     return UnknownException(s, stacktrace, c)
 

From f0a27b5488608bf4a4c100c4f26ffa3f7cd6b452 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 9 Jun 2020 04:52:48 +0000
Subject: [PATCH 0800/1280] [SPARK-28199][SPARK-28199][SS][FOLLOWUP] Mention
 the change of into the SS migration guide

### What changes were proposed in this pull request?

SPARK-28199 (#24996) made the trigger related public API to be exposed only from static methods of Trigger class. This is backward incompatible change, so some users may experience compilation error after upgrading to Spark 3.0.0.

While we plan to mention the change into release note, it's good to mention the change to the migration guide doc as well, since the purpose of the doc is to collect the major changes/incompatibilities between versions and end users would refer the doc.

### Why are the changes needed?

SPARK-28199 is technically backward incompatible change and we should kindly guide the change.

### Does this PR introduce _any_ user-facing change?

Doc change.

### How was this patch tested?

N/A, as it's just a doc change.

Closes #28763 from HeartSaVioR/SPARK-28199-FOLLOWUP-doc.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 8305b77796ad45090e9d430e2be59e467fc173d6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/ss-migration-guide.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/ss-migration-guide.md b/docs/ss-migration-guide.md
index 963ef07af7ace..002058b69bf30 100644
--- a/docs/ss-migration-guide.md
+++ b/docs/ss-migration-guide.md
@@ -31,3 +31,5 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide.
 - In Spark 3.0, Structured Streaming forces the source schema into nullable when file-based datasources such as text, json, csv, parquet and orc are used via `spark.readStream(...)`. Previously, it respected the nullability in source schema; however, it caused issues tricky to debug with NPE. To restore the previous behavior, set `spark.sql.streaming.fileSource.schema.forceNullable` to `false`.
 
 - Spark 3.0 fixes the correctness issue on Stream-stream outer join, which changes the schema of state. (See [SPARK-26154](https://issues.apache.org/jira/browse/SPARK-26154) for more details). If you start your query from checkpoint constructed from Spark 2.x which uses stream-stream outer join, Spark 3.0 fails the query. To recalculate outputs, discard the checkpoint and replay previous inputs.
+
+- In Spark 3.0, the deprecated class `org.apache.spark.sql.streaming.ProcessingTime` has been removed. Use `org.apache.spark.sql.streaming.Trigger.ProcessingTime` instead. Likewise, `org.apache.spark.sql.execution.streaming.continuous.ContinuousTrigger` has been removed in favor of `Trigger.Continuous`, and `org.apache.spark.sql.execution.streaming.OneTimeTrigger` has been hidden in favor of `Trigger.Once`.
\ No newline at end of file

From 2a9280ca4a6610bec0453ced7ed12174f8f43e5e Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 9 Jun 2020 04:59:41 +0000
Subject: [PATCH 0801/1280] [SPARK-31932][SQL][TESTS] Add date/timestamp
 benchmarks for `HiveResult.hiveResultString()`

### What changes were proposed in this pull request?
Add benchmarks for `HiveResult.hiveResultString()/toHiveString()` to measure throughput of `toHiveString` for the date/timestamp types:
- java.sql.Date/Timestamp
- java.time.Instant
- java.time.LocalDate

Benchmark results were generated in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_242 and OpenJDK 64-Bit Server VM 11.0.6+10 |

### Why are the changes needed?
To detect perf regressions of `toHiveString` in the future.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `DateTimeBenchmark` and check dataset content.

Closes #28757 from MaxGekk/benchmark-toHiveString.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit ddd8d5f5a0b6db17babc201ba4b73f7df91df1a3)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeBenchmark-jdk11-results.txt       |  4 ++
 .../benchmarks/DateTimeBenchmark-results.txt  |  4 ++
 .../benchmark/DateTimeBenchmark.scala         | 46 +++++++++++++++----
 3 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
index f4ed8ce4afaea..70d888227141d 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-jdk11-results.txt
@@ -453,5 +453,9 @@ From java.time.Instant                              325            328
 Collect longs                                      1300           1321          25          3.8         260.0       0.3X
 Collect java.sql.Timestamp                         1450           1557         102          3.4         290.0       0.3X
 Collect java.time.Instant                          1499           1599          87          3.3         299.9       0.3X
+java.sql.Date to Hive string                      17536          18367        1059          0.3        3507.2       0.0X
+java.time.LocalDate to Hive string                12089          12897         725          0.4        2417.8       0.0X
+java.sql.Timestamp to Hive string                 48014          48625         752          0.1        9602.9       0.0X
+java.time.Instant to Hive string                  37346          37445          93          0.1        7469.1       0.0X
 
 
diff --git a/sql/core/benchmarks/DateTimeBenchmark-results.txt b/sql/core/benchmarks/DateTimeBenchmark-results.txt
index 7a9aa4badfeb7..0795f11a57f28 100644
--- a/sql/core/benchmarks/DateTimeBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeBenchmark-results.txt
@@ -453,5 +453,9 @@ From java.time.Instant                              236            243
 Collect longs                                      1280           1337          79          3.9         256.1       0.3X
 Collect java.sql.Timestamp                         1485           1501          15          3.4         297.0       0.3X
 Collect java.time.Instant                          1441           1465          37          3.5         288.1       0.3X
+java.sql.Date to Hive string                      18745          20895        1364          0.3        3749.0       0.0X
+java.time.LocalDate to Hive string                15296          15450         143          0.3        3059.2       0.0X
+java.sql.Timestamp to Hive string                 46421          47210         946          0.1        9284.2       0.0X
+java.time.Instant to Hive string                  34747          35187         382          0.1        6949.4       0.0X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
index f56efa3bba600..c7b8737b7a753 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeBenchmark.scala
@@ -21,8 +21,10 @@ import java.sql.{Date, Timestamp}
 import java.time.{Instant, LocalDate}
 
 import org.apache.spark.benchmark.Benchmark
+import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
+import org.apache.spark.sql.execution.HiveResult
 import org.apache.spark.sql.internal.SQLConf
 
 /**
@@ -182,14 +184,19 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
           benchmark.addCase("From java.time.LocalDate", numIters) { _ =>
             spark.range(rowsNum).map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY)).noop()
           }
+          def dates = {
+            spark.range(0, rowsNum, 1, 1).map(millis => new Date(millis))
+          }
           benchmark.addCase("Collect java.sql.Date", numIters) { _ =>
-            spark.range(0, rowsNum, 1, 1).map(millis => new Date(millis)).collect()
+            dates.collect()
+          }
+          def localDates = {
+            spark.range(0, rowsNum, 1, 1)
+              .map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY))
           }
           benchmark.addCase("Collect java.time.LocalDate", numIters) { _ =>
             withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
-              spark.range(0, rowsNum, 1, 1)
-                .map(millis => LocalDate.ofEpochDay(millis / MILLIS_PER_DAY))
-                .collect()
+              localDates.collect()
             }
           }
           benchmark.addCase("From java.sql.Timestamp", numIters) { _ =>
@@ -202,14 +209,37 @@ object DateTimeBenchmark extends SqlBasedBenchmark {
             spark.range(0, rowsNum, 1, 1)
               .collect()
           }
+          def timestamps = {
+            spark.range(0, rowsNum, 1, 1).map(millis => new Timestamp(millis))
+          }
           benchmark.addCase("Collect java.sql.Timestamp", numIters) { _ =>
-            spark.range(0, rowsNum, 1, 1).map(millis => new Timestamp(millis)).collect()
+            timestamps.collect()
+          }
+          def instants = {
+            spark.range(0, rowsNum, 1, 1).map(millis => Instant.ofEpochMilli(millis))
           }
           benchmark.addCase("Collect java.time.Instant", numIters) { _ =>
             withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
-              spark.range(0, rowsNum, 1, 1)
-                .map(millis => Instant.ofEpochMilli(millis))
-                .collect()
+              instants.collect()
+            }
+          }
+          def toHiveString(df: Dataset[_]): Unit = {
+            HiveResult.hiveResultString(df.queryExecution.executedPlan)
+          }
+          benchmark.addCase("java.sql.Date to Hive string", numIters) { _ =>
+            toHiveString(dates)
+          }
+          benchmark.addCase("java.time.LocalDate to Hive string", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              toHiveString(localDates)
+            }
+          }
+          benchmark.addCase("java.sql.Timestamp to Hive string", numIters) { _ =>
+            toHiveString(timestamps)
+          }
+          benchmark.addCase("java.time.Instant to Hive string", numIters) { _ =>
+            withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") {
+              toHiveString(instants)
             }
           }
           benchmark.run()

From bdddf31d125652bd8975b3c1a9ce13f124c28698 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 9 Jun 2020 15:20:13 +0000
Subject: [PATCH 0802/1280] [SPARK-31940][SQL][DOCS] Document the default JVM
 time zone in to/fromJavaDate and legacy date formatters

### What changes were proposed in this pull request?
Update comments for `DateTimeUtils`.`toJavaDate` and `fromJavaDate`, and for the legacy date formatters `LegacySimpleDateFormatter` and `LegacyFastDateFormatter` regarding to the default JVM time zone. The comments say that the default JVM time zone is used intentionally for backward compatibility with Spark 2.4 and earlier versions.

Closes #28709

### Why are the changes needed?
To document current behaviour of related methods in `DateTimeUtils` and the legacy date formatters. For example, correctness of `HiveResult.hiveResultString` and `toHiveString` is directly related to the same time zone used by `toJavaDate` and `LegacyFastDateFormatter`.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the Scala style checker `./dev/scalastyle`

Closes #28767 from MaxGekk/doc-legacy-formatters.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit de91915a24a5cb5cd725b0e0a7fa63a73c2fc277)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/DateFormatter.scala     | 27 ++++++++++++
 .../sql/catalyst/util/DateTimeUtils.scala     | 42 ++++++++++---------
 2 files changed, 49 insertions(+), 20 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
index ec8db4653d137..07fa09d64400f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala
@@ -100,6 +100,17 @@ trait LegacyDateFormatter extends DateFormatter {
   }
 }
 
+/**
+ * The legacy formatter is based on Apache Commons FastDateFormat. The formatter uses the default
+ * JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
+ *
+ * Note: Using of the default JVM time zone makes the formatter compatible with the legacy
+ *       `DateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
+ *       JVM time zone too.
+ *
+ * @param pattern `java.text.SimpleDateFormat` compatible pattern.
+ * @param locale The locale overrides the system locale and is used in parsing/formatting.
+ */
 class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
   @transient
   private lazy val fdf = FastDateFormat.getInstance(pattern, locale)
@@ -109,6 +120,22 @@ class LegacyFastDateFormatter(pattern: String, locale: Locale) extends LegacyDat
   override def validatePatternString(): Unit = fdf
 }
 
+// scalastyle:off line.size.limit
+/**
+ * The legacy formatter is based on `java.text.SimpleDateFormat`. The formatter uses the default
+ * JVM time zone intentionally for compatibility with Spark 2.4 and earlier versions.
+ *
+ * Note: Using of the default JVM time zone makes the formatter compatible with the legacy
+ *       `DateTimeUtils` methods `toJavaDate` and `fromJavaDate` that are based on the default
+ *       JVM time zone too.
+ *
+ * @param pattern The pattern describing the date and time format.
+ *                See <a href="https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html">
+ *                Date and Time Patterns</a>
+ * @param locale  The locale whose date format symbols should be used. It overrides the system
+ *                locale in parsing/formatting.
+ */
+// scalastyle:on line.size.limit
 class LegacySimpleDateFormatter(pattern: String, locale: Locale) extends LegacyDateFormatter {
   @transient
   private lazy val sdf = new SimpleDateFormat(pattern, locale)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index fb7875dc24e70..3f8417f2f3651 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -90,19 +90,20 @@ object DateTimeUtils {
   }
 
   /**
-   * Converts an instance of `java.sql.Date` to a number of days since the epoch
-   * 1970-01-01 via extracting date fields `year`, `month`, `days` from the input,
-   * creating a local date in Proleptic Gregorian calendar from the fields, and
-   * getting the number of days from the resulted local date.
+   * Converts a local date at the default JVM time zone to the number of days since 1970-01-01
+   * in the hybrid calendar (Julian + Gregorian) by discarding the time part. The resulted days are
+   * rebased from the hybrid to Proleptic Gregorian calendar. The days rebasing is performed via
+   * UTC time zone for simplicity because the difference between two calendars is the same in
+   * any given time zone and UTC time zone.
    *
-   * This approach was taken to have the same local date as the triple of `year`,
-   * `month`, `day` in the original hybrid calendar used by `java.sql.Date` and
-   * Proleptic Gregorian calendar used by Spark since version 3.0.0, see SPARK-26651.
+   * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
+   *       with Spark 2.4 and earlier versions. The goal of the shift is to get a local date derived
+   *       from the number of days that has the same date fields (year, month, day) as the original
+   *       `date` at the default JVM time zone.
    *
-   * @param date It represents a specific instant in time based on
-   *             the hybrid calendar which combines Julian and
-   *             Gregorian calendars.
-   * @return The number of days since epoch from java.sql.Date.
+   * @param date It represents a specific instant in time based on the hybrid calendar which
+   *             combines Julian and Gregorian calendars.
+   * @return The number of days since the epoch in Proleptic Gregorian calendar.
    */
   def fromJavaDate(date: Date): SQLDate = {
     val millisUtc = date.getTime
@@ -112,17 +113,18 @@ object DateTimeUtils {
   }
 
   /**
-   * The opposite to `fromJavaDate` method which converts a number of days to an
-   * instance of `java.sql.Date`. It builds a local date in Proleptic Gregorian
-   * calendar, extracts date fields `year`, `month`, `day`, and creates a local
-   * date in the hybrid calendar (Julian + Gregorian calendars) from the fields.
+   * Converts days since the epoch 1970-01-01 in Proleptic Gregorian calendar to a local date
+   * at the default JVM time zone in the hybrid calendar (Julian + Gregorian). It rebases the given
+   * days from Proleptic Gregorian to the hybrid calendar at UTC time zone for simplicity because
+   * the difference between two calendars doesn't depend on any time zone. The result is shifted
+   * by the time zone offset in wall clock to have the same date fields (year, month, day)
+   * at the default JVM time zone as the input `daysSinceEpoch` in Proleptic Gregorian calendar.
    *
-   * The purpose of the conversion is to have the same local date as the triple
-   * of `year`, `month`, `day` in the original Proleptic Gregorian calendar and
-   * in the target calender.
+   * Note: The date is shifted by the offset of the default JVM time zone for backward compatibility
+   *       with Spark 2.4 and earlier versions.
    *
-   * @param daysSinceEpoch The number of days since 1970-01-01.
-   * @return A `java.sql.Date` from number of days since epoch.
+   * @param daysSinceEpoch The number of days since 1970-01-01 in Proleptic Gregorian calendar.
+   * @return A local date in the hybrid calendar as `java.sql.Date` from number of days since epoch.
    */
   def toJavaDate(daysSinceEpoch: SQLDate): Date = {
     val rebasedDays = rebaseGregorianToJulianDays(daysSinceEpoch)

From 4879822de785c7507695d81f57f87b60ccf0531e Mon Sep 17 00:00:00 2001
From: turbofei <fwang12@ebay.com>
Date: Tue, 9 Jun 2020 15:57:18 +0000
Subject: [PATCH 0803/1280] [SPARK-29295][SQL][FOLLOWUP] Dynamic partition map
 parsed from partition path should be case insensitive

### What changes were proposed in this pull request?

This is a follow up of https://github.com/apache/spark/pull/25979.
When we inserting overwrite  an external hive partitioned table with upper case dynamic partition key, exception thrown.

like:
```
org.apache.spark.SparkException: Dynamic partition key P1 is not among written partition paths.
```
The root cause is that Hive metastore is not case preserving and keeps partition columns with lower cased names, see details in:

https://github.com/apache/spark/blob/ddd8d5f5a0b6db17babc201ba4b73f7df91df1a3/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala#L895-L901
https://github.com/apache/spark/blob/e28914095aa1fa7a4680b5e4fcf69e3ef64b3dbc/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala#L228-L234

In this PR, we convert the dynamic partition map to a case insensitive map.
### Why are the changes needed?

To fix the issue when inserting overwrite into external hive partitioned table with upper case dynamic partition key.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
UT.

Closes #28765 from turboFei/SPARK-29295-follow-up.

Authored-by: turbofei <fwang12@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 717ec5e9e3275e5bfdd7743fc62f90eff4843655)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/hive/execution/InsertIntoHiveTable.scala    |  6 +++++-
 .../spark/sql/hive/execution/SQLQuerySuite.scala    | 13 +++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 19f439598142e..a0bdbec4694bf 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.{AnalysisException, Row, SparkSession}
 import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType, ExternalCatalog, ExternalCatalogUtils}
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.command.CommandUtils
 import org.apache.spark.sql.hive.HiveShim.{ShimFileSinkDesc => FileSinkDesc}
@@ -223,9 +224,12 @@ case class InsertIntoHiveTable(
                 ExternalCatalogUtils.unescapePathName(splitPart(1))
             }.toMap
 
+            val caseInsensitiveDpMap = CaseInsensitiveMap(dpMap)
+
             val updatedPartitionSpec = partition.map {
               case (key, Some(value)) => key -> value
-              case (key, None) if dpMap.contains(key) => key -> dpMap(key)
+              case (key, None) if caseInsensitiveDpMap.contains(key) =>
+                key -> caseInsensitiveDpMap(key)
               case (key, _) =>
                 throw new SparkException(s"Dynamic partition key $key is not among " +
                   "written partition paths.")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 79c6ade2807d3..d12eae0e410b1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2544,6 +2544,19 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
       assert(e.getMessage.contains("Cannot modify the value of a static config"))
     }
   }
+
+  test("SPARK-29295: dynamic partition map parsed from partition path should be case insensitive") {
+    withTable("t") {
+      withSQLConf("hive.exec.dynamic.partition" -> "true",
+        "hive.exec.dynamic.partition.mode" -> "nonstrict") {
+        withTempDir { loc =>
+          sql(s"CREATE TABLE t(c1 INT) PARTITIONED BY(P1 STRING) LOCATION '${loc.getAbsolutePath}'")
+          sql("INSERT OVERWRITE TABLE t PARTITION(P1) VALUES(1, 'caseSensitive')")
+          checkAnswer(sql("select * from t"), Row(1, "caseSensitive"))
+        }
+      }
+    }
+  }
 }
 
 class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite

From b9a1cd8d4bb1241e27fd78c04a892c10f1a62147 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Tue, 9 Jun 2020 16:07:22 +0000
Subject: [PATCH 0804/1280] [SPARK-31220][SQL] repartition obeys
 initialPartitionNum when adaptiveExecutionEnabled

### What changes were proposed in this pull request?
This PR makes `repartition`/`DISTRIBUTE BY` obeys [initialPartitionNum](https://github.com/apache/spark/blob/af4248b2d661d04fec89b37857a47713246d9465/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L446-L455) when adaptive execution enabled.

### Why are the changes needed?
To make `DISTRIBUTE BY`/`GROUP BY` partitioned by same partition number.
How to reproduce:
```scala
spark.sql("CREATE TABLE spark_31220(id int)")
spark.sql("set spark.sql.adaptive.enabled=true")
spark.sql("set spark.sql.adaptive.coalescePartitions.initialPartitionNum=1000")
```

Before this PR:
```
scala> spark.sql("SELECT id from spark_31220 GROUP BY id").explain
== Physical Plan ==
AdaptiveSparkPlan(isFinalPlan=false)
+- HashAggregate(keys=[id#5], functions=[])
   +- Exchange hashpartitioning(id#5, 1000), true, [id=#171]
      +- HashAggregate(keys=[id#5], functions=[])
         +- FileScan parquet default.spark_31220[id#5]

scala> spark.sql("SELECT id from spark_31220 DISTRIBUTE BY id").explain
== Physical Plan ==
AdaptiveSparkPlan(isFinalPlan=false)
+- Exchange hashpartitioning(id#5, 200), false, [id=#179]
   +- FileScan parquet default.spark_31220[id#5]
```
After this PR:
```
scala> spark.sql("SELECT id from spark_31220 GROUP BY id").explain
== Physical Plan ==
AdaptiveSparkPlan(isFinalPlan=false)
+- HashAggregate(keys=[id#5], functions=[])
   +- Exchange hashpartitioning(id#5, 1000), true, [id=#171]
      +- HashAggregate(keys=[id#5], functions=[])
         +- FileScan parquet default.spark_31220[id#5]

scala> spark.sql("SELECT id from spark_31220 DISTRIBUTE BY id").explain
== Physical Plan ==
AdaptiveSparkPlan(isFinalPlan=false)
+- Exchange hashpartitioning(id#5, 1000), false, [id=#179]
   +- FileScan parquet default.spark_31220[id#5]
```

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Unit test.

Closes #27986 from wangyum/SPARK-31220.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1d1eacde9d1b6fb75a20e4b909d221e70ad737db)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/internal/SQLConf.scala  | 13 +++++++++----
 .../execution/exchange/EnsureRequirements.scala  | 10 ++--------
 .../adaptive/AdaptiveQueryExecSuite.scala        | 16 ++++++++++++++++
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index b8b0f3250fad8..fd57095031648 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2765,7 +2765,15 @@ class SQLConf extends Serializable with Logging {
 
   def cacheVectorizedReaderEnabled: Boolean = getConf(CACHE_VECTORIZED_READER_ENABLED)
 
-  def numShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS)
+  def defaultNumShufflePartitions: Int = getConf(SHUFFLE_PARTITIONS)
+
+  def numShufflePartitions: Int = {
+    if (adaptiveExecutionEnabled && coalesceShufflePartitionsEnabled) {
+      getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(defaultNumShufflePartitions)
+    } else {
+      defaultNumShufflePartitions
+    }
+  }
 
   def adaptiveExecutionEnabled: Boolean = getConf(ADAPTIVE_EXECUTION_ENABLED)
 
@@ -2778,9 +2786,6 @@ class SQLConf extends Serializable with Logging {
 
   def coalesceShufflePartitionsEnabled: Boolean = getConf(COALESCE_PARTITIONS_ENABLED)
 
-  def initialShufflePartitionNum: Int =
-    getConf(COALESCE_PARTITIONS_INITIAL_PARTITION_NUM).getOrElse(numShufflePartitions)
-
   def minBatchesToRetain: Int = getConf(MIN_BATCHES_TO_RETAIN)
 
   def maxBatchesToRetainInMemory: Int = getConf(MAX_BATCHES_TO_RETAIN_IN_MEMORY)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
index 28ef793ed62db..3242ac21ab324 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
@@ -35,12 +35,6 @@ import org.apache.spark.sql.internal.SQLConf
  * the input partition ordering requirements are met.
  */
 case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
-  private def defaultNumPreShufflePartitions: Int =
-    if (conf.adaptiveExecutionEnabled && conf.coalesceShufflePartitionsEnabled) {
-      conf.initialShufflePartitionNum
-    } else {
-      conf.numShufflePartitions
-    }
 
   private def ensureDistributionAndOrdering(operator: SparkPlan): SparkPlan = {
     val requiredChildDistributions: Seq[Distribution] = operator.requiredChildDistribution
@@ -57,7 +51,7 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         BroadcastExchangeExec(mode, child)
       case (child, distribution) =>
         val numPartitions = distribution.requiredNumPartitions
-          .getOrElse(defaultNumPreShufflePartitions)
+          .getOrElse(conf.numShufflePartitions)
         ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child)
     }
 
@@ -95,7 +89,7 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         // expected number of shuffle partitions. However, if it's smaller than
         // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
         // expected number of shuffle partitions.
-        math.max(nonShuffleChildrenNumPartitions.max, conf.numShufflePartitions)
+        math.max(nonShuffleChildrenNumPartitions.max, conf.defaultNumShufflePartitions)
       } else {
         childrenNumPartitions.max
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index ac0267ac470d5..6477ddc11ea72 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -873,4 +873,20 @@ class AdaptiveQueryExecSuite
       }
     }
   }
+
+  test("SPARK-31220 repartition obeys initialPartitionNum when adaptiveExecutionEnabled") {
+    Seq(true, false).foreach { enableAQE =>
+      withSQLConf(
+        SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> enableAQE.toString,
+        SQLConf.SHUFFLE_PARTITIONS.key -> "6",
+        SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "7") {
+        val partitionsNum = spark.range(10).repartition($"id").rdd.collectPartitions().length
+        if (enableAQE) {
+          assert(partitionsNum === 7)
+        } else {
+          assert(partitionsNum === 6)
+        }
+      }
+    }
+  }
 }

From c3e5cd25766e0ee0eaf4513c0639229315fbb1e6 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 9 Jun 2020 09:20:54 -0700
Subject: [PATCH 0805/1280] [SPARK-31921][CORE] Fix the wrong warning: "App
 app-xxx requires more resource than any of Workers could have"

### What changes were proposed in this pull request?

This PR adds the check to see whether the allocated executors for the waiting application is empty before recognizing it as a possible hang application.

### Why are the changes needed?

It's a bugfix. The warning means there are not enough resources for the application to launch at least one executor. But we can still successfully run a job under this warning, which means it does have launched executor.

### Does this PR introduce _any_ user-facing change?

Yes. Before this PR, when using local cluster mode to start spark-shell, e.g. `./bin/spark-shell --master "local-cluster[2, 1, 1024]"`, the user would always see the warning:

```
20/06/06 22:21:02 WARN Utils: Your hostname, C02ZQ051LVDR resolves to a loopback address: 127.0.0.1; using 192.168.1.6 instead (on interface en0)
20/06/06 22:21:02 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address
20/06/06 22:21:02 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark classes ahead of assembly.
NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark classes ahead of assembly.
Spark context Web UI available at http://192.168.1.6:4040
Spark context available as 'sc' (master = local-cluster[2, 1, 1024], app id = app-20200606222107-0000).
Spark session available as 'spark'.
20/06/06 22:21:07 WARN Master: App app-20200606222107-0000 requires more resource than any of Workers could have.
20/06/06 22:21:07 WARN Master: App app-20200606222107-0000 requires more resource than any of Workers could have.
```

After this PR, the warning has gone.

### How was this patch tested?

Tested manually.

Closes #28742 from Ngone51/fix_warning.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 38873d5196108897a29888dcd77a0c0d835772e6)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../main/scala/org/apache/spark/deploy/master/Master.scala    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 8d3795cae707a..3818a872b9f88 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -704,7 +704,9 @@ private[deploy] class Master(
         val usableWorkers = workers.toArray.filter(_.state == WorkerState.ALIVE)
           .filter(canLaunchExecutor(_, app.desc))
           .sortBy(_.coresFree).reverse
-        if (waitingApps.length == 1 && usableWorkers.isEmpty) {
+        val appMayHang = waitingApps.length == 1 &&
+          waitingApps.head.executors.isEmpty && usableWorkers.isEmpty
+        if (appMayHang) {
           logWarning(s"App ${app.id} requires more resource than any of Workers could have.")
         }
         val assignedCores = scheduleExecutorsOnWorkers(app, usableWorkers, spreadOutApps)

From 4b625bd91e60af4fff77b7fc8dcde0261a488260 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 9 Jun 2020 16:49:40 +0000
Subject: [PATCH 0806/1280] [SPARK-31926][SQL][TEST-HIVE1.2] Fix concurrency
 issue for ThriftCLIService to getPortNumber

### What changes were proposed in this pull request?

When` org.apache.spark.sql.hive.thriftserver.HiveThriftServer2#startWithContext` called,
it starts `ThriftCLIService` in the background with a new Thread, at the same time we call `ThriftCLIService.getPortNumber,` we might not get the bound port if it's configured with 0.

This PR moves the  TServer/HttpServer initialization code out of that new Thread.

### Why are the changes needed?

Fix concurrency issue, improve test robustness.

### Does this PR introduce _any_ user-facing change?

NO
### How was this patch tested?

add new tests

Closes #28751 from yaooqinn/SPARK-31926.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 02f32cfae4082696a6c013fb6beb150040be3228)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 project/SparkBuild.scala                      |  3 +-
 .../thriftserver/SharedThriftServer.scala     | 46 +++++++++++++------
 .../ThriftServerQueryTestSuite.scala          |  3 ++
 .../ThriftServerWithSparkContextSuite.scala   | 11 ++++-
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 ++++-
 .../service/cli/thrift/ThriftCLIService.java  |  3 ++
 .../cli/thrift/ThriftHttpCLIService.java      | 21 ++++++---
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 ++++-
 .../service/cli/thrift/ThriftCLIService.java  |  3 ++
 .../cli/thrift/ThriftHttpCLIService.java      | 21 ++++++---
 10 files changed, 104 insertions(+), 29 deletions(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index c9521ea268579..fdd39dd536aca 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -479,7 +479,8 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
-    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextInHttpSuite",
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextInBinarySuite",
     "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index e002bc0117c8b..c9e41db52cd50 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -33,6 +33,8 @@ trait SharedThriftServer extends SharedSparkSession {
   private var hiveServer2: HiveThriftServer2 = _
   private var serverPort: Int = 0
 
+  def mode: ServerMode.Value
+
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Retries up to 3 times with different port numbers if the server fails to start
@@ -53,11 +55,17 @@ trait SharedThriftServer extends SharedSparkSession {
     }
   }
 
+  protected def jdbcUri: String = if (mode == ServerMode.http) {
+    s"jdbc:hive2://localhost:$serverPort/default;transportMode=http;httpPath=cliservice"
+  } else {
+    s"jdbc:hive2://localhost:$serverPort"
+  }
+
   protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
     val user = System.getProperty("user.name")
     require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
-      fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
+      fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
     val statements = connections.map(_.createStatement())
 
     try {
@@ -71,21 +79,33 @@ trait SharedThriftServer extends SharedSparkSession {
   private def startThriftServer(attempt: Int): Unit = {
     logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any free port to use.
+    // Set the HIVE_SERVER2_THRIFT_PORT and HIVE_SERVER2_THRIFT_HTTP_PORT to 0, so it could
+    // randomly pick any free port to use.
     // It's much more robust than set a random port generated by ourselves ahead
     sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
-    hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
-    hiveServer2.getServices.asScala.foreach {
-      case t: ThriftCLIService if t.getPortNumber != 0 =>
-        serverPort = t.getPortNumber
-        logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
-      case _ =>
-    }
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, "0")
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, mode.toString)
+
+    try {
+      hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+      hiveServer2.getServices.asScala.foreach {
+        case t: ThriftCLIService =>
+          serverPort = t.getPortNumber
+          logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
+        case _ =>
+      }
 
-    // Wait for thrift server to be ready to serve the query, via executing simple query
-    // till the query succeeds. See SPARK-30345 for more details.
-    eventually(timeout(30.seconds), interval(1.seconds)) {
-      withJdbcStatement { _.execute("SELECT 1") }
+      // Wait for thrift server to be ready to serve the query, via executing simple query
+      // till the query succeeds. See SPARK-30345 for more details.
+      eventually(timeout(30.seconds), interval(1.seconds)) {
+        withJdbcStatement { _.execute("SELECT 1") }
+      }
+    } catch {
+      case e: Exception =>
+        logError("Error start hive server with Context ", e)
+        if (hiveServer2 != null) {
+          hiveServer2.stop()
+        }
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 15cc3109da3f7..553f10a275bce 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -54,6 +54,9 @@ import org.apache.spark.sql.types._
  */
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
+
+  override def mode: ServerMode.Value = ServerMode.binary
+
   override protected def testFile(fileName: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
     // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index 3e1fce78ae71c..d6420dee41adb 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-class ThriftServerWithSparkContextSuite extends SharedThriftServer {
+trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
 
   test("SPARK-29911: Uncache cached tables when session closed") {
     val cacheManager = spark.sharedState.cacheManager
@@ -42,3 +42,12 @@ class ThriftServerWithSparkContextSuite extends SharedThriftServer {
     }
   }
 }
+
+
+class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite {
+  override def mode: ServerMode.Value = ServerMode.binary
+}
+
+class ThriftServerWithSparkContextInHttpSuite extends ThriftServerWithSparkContextSuite {
+  override def mode: ServerMode.Value = ServerMode.http
+}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index e1ee503b81209..00bdf7e19126e 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -45,7 +46,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -100,6 +101,14 @@ public void run() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + serverSocket.getServerSocket().getLocalPort() + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
       server.serve();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index 8fce9d9383438..783e5795aca76 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -175,6 +175,7 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
+      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -633,6 +634,8 @@ public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
     return resp;
   }
 
+  protected abstract void initializeServer();
+
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 1099a00b67eb7..bd64c777c1d76 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.cli.thrift.TCLIService.Iface;
@@ -53,13 +54,8 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -150,6 +146,19 @@ public void run() {
           + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
+  @Override
+  public void run() {
+    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index a7de9c0f3d0d2..ce79e3c8228a6 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -46,7 +47,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -101,6 +102,14 @@ public void run() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
       server.serve();
     } catch (Throwable t) {
       LOG.error(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index d41c3b493bb47..e46799a1c427d 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -176,6 +176,7 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
+      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -670,6 +671,8 @@ public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
     return resp;
   }
 
+  protected abstract void initializeServer();
+
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 73d5f84476af0..ab9ed5b1f371e 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.rpc.thrift.TCLIService;
@@ -54,13 +55,8 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -151,6 +147,19 @@ public void run() {
           + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
+  @Override
+  public void run() {
+    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.error(

From 89b1d4614ef1a3d15ff0f1e745c770ebd8f5cddb Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Wed, 10 Jun 2020 16:29:43 +0900
Subject: [PATCH 0807/1280] [SPARK-26905][SQL] Add `TYPE` in the ANSI
 non-reserved list

### What changes were proposed in this pull request?

This PR intends to add `TYPE` in the ANSI non-reserved list because it is not reserved in the standard. See SPARK-26905 for a full set of the reserved/non-reserved keywords of `SQL:2016`.

Note: The current master behaviour is as follows;
```
scala> sql("SET spark.sql.ansi.enabled=false")
scala> sql("create table t1 (type int)")
res4: org.apache.spark.sql.DataFrame = []

scala> sql("SET spark.sql.ansi.enabled=true")
scala> sql("create table t2 (type int)")
org.apache.spark.sql.catalyst.parser.ParseException:
no viable alternative at input 'type'(line 1, pos 17)

== SQL ==
create table t2 (type int)
-----------------^^^
```

### Why are the changes needed?

To follow the ANSI/SQL standard.

### Does this PR introduce _any_ user-facing change?

Makes users use `TYPE` as identifiers.

### How was this patch tested?

Update the keyword lists in `TableIdentifierParserSuite`.

Closes #28773 from maropu/SPARK-26905.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>

(cherry picked from commit e14029b18df10db5094f8abf8b9874dbc9186b4e)

Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4  | 1 +
 .../spark/sql/catalyst/parser/TableIdentifierParserSuite.scala   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 2adaa9f46ceba..208a503f41980 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -1153,6 +1153,7 @@ ansiNonReserved
     | TRIM
     | TRUE
     | TRUNCATE
+    | TYPE
     | UNARCHIVE
     | UNBOUNDED
     | UNCACHE
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index d5b0885555462..bd617bf7e3df6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -513,6 +513,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "transform",
     "true",
     "truncate",
+    "type",
     "unarchive",
     "unbounded",
     "uncache",

From 9ba9d857d7ec2619e94bdeb41323c0f938df6347 Mon Sep 17 00:00:00 2001
From: SaurabhChawla <saurabhc@qubole.com>
Date: Wed, 10 Jun 2020 16:51:19 +0900
Subject: [PATCH 0808/1280] [SPARK-31941][CORE] Replace SparkException to
 NoSuchElementException for applicationInfo in AppStatusStore

### What changes were proposed in this pull request?
After SPARK-31632 SparkException is thrown from def applicationInfo
`def applicationInfo(): v1.ApplicationInfo = {
    try {
      // The ApplicationInfo may not be available when Spark is starting up.
      store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
    } catch {
      case _: NoSuchElementException =>
        throw new SparkException("Failed to get the application information. " +
          "If you are starting up Spark, please wait a while until it's ready.")
    }
  }`

Where as the caller for this method def getSparkUser in Spark UI is not handling SparkException in the catch

`def getSparkUser: String = {
    try {
      Option(store.applicationInfo().attempts.head.sparkUser)
        .orElse(store.environmentInfo().systemProperties.toMap.get("user.name"))
        .getOrElse("<unknown>")
    } catch {
      case _: NoSuchElementException => "<unknown>"
    }
  }`

So On using this method (getSparkUser )we can get the application erred out.

As the part of this PR we will replace SparkException to NoSuchElementException for applicationInfo in AppStatusStore

### Why are the changes needed?
On invoking the method getSparkUser, we can get the SparkException on calling store.applicationInfo(). And this is not handled in the catch block and getSparkUser will error out in this scenario

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Done the manual testing using the spark-shell and spark-submit

Closes #28768 from SaurabhChawla100/SPARK-31941.

Authored-by: SaurabhChawla <saurabhc@qubole.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
(cherry picked from commit 82ff29be7afa2ff6350310ab9bdf6b474398fdc1)
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 .../src/main/scala/org/apache/spark/status/AppStatusStore.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 42d4071d61c8b..d7f5ac8d3dbc9 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -41,7 +41,7 @@ private[spark] class AppStatusStore(
       store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
     } catch {
       case _: NoSuchElementException =>
-        throw new SparkException("Failed to get the application information. " +
+        throw new NoSuchElementException("Failed to get the application information. " +
           "If you are starting up Spark, please wait a while until it's ready.")
     }
   }

From 62fbff8ad127f3a6dd2360f3c02a20f4391cdad4 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 10 Jun 2020 13:11:24 -0700
Subject: [PATCH 0809/1280] [SPARK-31956][SQL] Do not fail if there is no
 ambiguous self join

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/28695 , to fix the problem completely.

The root cause is that, `df("col").as("name")` is not a column reference anymore, and should not have the special column metadata. However, this was broken in https://github.com/apache/spark/commit/ba7adc494923de8104ab37d412edd78afe540f45#diff-ac415c903887e49486ba542a65eec980L1050-L1053

This PR fixes the regression, by strip the special column metadata in `Column.name`, which is the behavior before https://github.com/apache/spark/pull/28326 .

### Why are the changes needed?

Fix a regression. We shouldn't fail if there is no ambiguous self-join.

### Does this PR introduce _any_ user-facing change?

Yes, the query in the test can run now.

### How was this patch tested?

updated test

Closes #28783 from cloud-fan/self-join.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c40051932290db3a63f80324900a116019b1e589)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 sql/core/src/main/scala/org/apache/spark/sql/Column.scala  | 2 +-
 .../org/apache/spark/sql/DataFrameSelfJoinSuite.scala      | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 2144472937f9b..e6f7b1d723af6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -1042,7 +1042,7 @@ class Column(val expr: Expression) extends Logging {
    * @since 2.0.0
    */
   def name(alias: String): Column = withExpr {
-    Alias(expr, alias)()
+    Alias(normalizedExpr(), alias)()
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
index fb58c9851224b..3b3b54f75da57 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSelfJoinSuite.scala
@@ -204,7 +204,7 @@ class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession {
     }
   }
 
-  test("SPARK-28344: don't fail as ambiguous self join when there is no join") {
+  test("SPARK-28344: don't fail if there is no ambiguous self join") {
     withSQLConf(
       SQLConf.FAIL_AMBIGUOUS_SELF_JOIN_ENABLED.key -> "true") {
       val df = Seq(1, 1, 2, 2).toDF("a")
@@ -212,6 +212,11 @@ class DataFrameSelfJoinSuite extends QueryTest with SharedSparkSession {
       checkAnswer(
         df.select(df("a").alias("x"), sum(df("a")).over(w)),
         Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
+
+      val joined = df.join(spark.range(1)).select($"a")
+      checkAnswer(
+        joined.select(joined("a").alias("x"), sum(joined("a")).over(w)),
+        Seq((1, 2), (1, 2), (2, 4), (2, 4)).map(Row.fromTuple))
     }
   }
 }

From b9807ac153ee9461d55936c608ea6aa8651ecd13 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 10 Jun 2020 17:39:34 -0700
Subject: [PATCH 0810/1280] Revert "[SPARK-31926][SQL][TEST-HIVE1.2] Fix
 concurrency issue for ThriftCLIService to getPortNumber"

This reverts commit 4b625bd91e60af4fff77b7fc8dcde0261a488260.
---
 project/SparkBuild.scala                      |  3 +-
 .../thriftserver/SharedThriftServer.scala     | 46 ++++++-------------
 .../ThriftServerQueryTestSuite.scala          |  3 --
 .../ThriftServerWithSparkContextSuite.scala   | 11 +----
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 +----
 .../service/cli/thrift/ThriftCLIService.java  |  3 --
 .../cli/thrift/ThriftHttpCLIService.java      | 21 +++------
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 +----
 .../service/cli/thrift/ThriftCLIService.java  |  3 --
 .../cli/thrift/ThriftHttpCLIService.java      | 21 +++------
 10 files changed, 29 insertions(+), 104 deletions(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index fdd39dd536aca..c9521ea268579 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -479,8 +479,7 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
-    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextInHttpSuite",
-    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextInBinarySuite",
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
     "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index c9e41db52cd50..e002bc0117c8b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -33,8 +33,6 @@ trait SharedThriftServer extends SharedSparkSession {
   private var hiveServer2: HiveThriftServer2 = _
   private var serverPort: Int = 0
 
-  def mode: ServerMode.Value
-
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Retries up to 3 times with different port numbers if the server fails to start
@@ -55,17 +53,11 @@ trait SharedThriftServer extends SharedSparkSession {
     }
   }
 
-  protected def jdbcUri: String = if (mode == ServerMode.http) {
-    s"jdbc:hive2://localhost:$serverPort/default;transportMode=http;httpPath=cliservice"
-  } else {
-    s"jdbc:hive2://localhost:$serverPort"
-  }
-
   protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
     val user = System.getProperty("user.name")
     require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
-      fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
+      fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
     val statements = connections.map(_.createStatement())
 
     try {
@@ -79,33 +71,21 @@ trait SharedThriftServer extends SharedSparkSession {
   private def startThriftServer(attempt: Int): Unit = {
     logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    // Set the HIVE_SERVER2_THRIFT_PORT and HIVE_SERVER2_THRIFT_HTTP_PORT to 0, so it could
-    // randomly pick any free port to use.
+    // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any free port to use.
     // It's much more robust than set a random port generated by ourselves ahead
     sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, "0")
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, mode.toString)
-
-    try {
-      hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
-      hiveServer2.getServices.asScala.foreach {
-        case t: ThriftCLIService =>
-          serverPort = t.getPortNumber
-          logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
-        case _ =>
-      }
+    hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+    hiveServer2.getServices.asScala.foreach {
+      case t: ThriftCLIService if t.getPortNumber != 0 =>
+        serverPort = t.getPortNumber
+        logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
+      case _ =>
+    }
 
-      // Wait for thrift server to be ready to serve the query, via executing simple query
-      // till the query succeeds. See SPARK-30345 for more details.
-      eventually(timeout(30.seconds), interval(1.seconds)) {
-        withJdbcStatement { _.execute("SELECT 1") }
-      }
-    } catch {
-      case e: Exception =>
-        logError("Error start hive server with Context ", e)
-        if (hiveServer2 != null) {
-          hiveServer2.stop()
-        }
+    // Wait for thrift server to be ready to serve the query, via executing simple query
+    // till the query succeeds. See SPARK-30345 for more details.
+    eventually(timeout(30.seconds), interval(1.seconds)) {
+      withJdbcStatement { _.execute("SELECT 1") }
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 553f10a275bce..15cc3109da3f7 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -54,9 +54,6 @@ import org.apache.spark.sql.types._
  */
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
-
-  override def mode: ServerMode.Value = ServerMode.binary
-
   override protected def testFile(fileName: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
     // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index d6420dee41adb..3e1fce78ae71c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
+class ThriftServerWithSparkContextSuite extends SharedThriftServer {
 
   test("SPARK-29911: Uncache cached tables when session closed") {
     val cacheManager = spark.sharedState.cacheManager
@@ -42,12 +42,3 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
     }
   }
 }
-
-
-class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite {
-  override def mode: ServerMode.Value = ServerMode.binary
-}
-
-class ThriftServerWithSparkContextInHttpSuite extends ThriftServerWithSparkContextSuite {
-  override def mode: ServerMode.Value = ServerMode.http
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index 00bdf7e19126e..e1ee503b81209 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -46,7 +45,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -101,14 +100,6 @@ protected void initializeServer() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + serverSocket.getServerSocket().getLocalPort() + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  @Override
-  public void run() {
-    try {
       server.serve();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index 783e5795aca76..8fce9d9383438 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -175,7 +175,6 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
-      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -634,8 +633,6 @@ public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
     return resp;
   }
 
-  protected abstract void initializeServer();
-
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index bd64c777c1d76..1099a00b67eb7 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.cli.thrift.TCLIService.Iface;
@@ -54,8 +53,13 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -146,19 +150,6 @@ protected void initializeServer() {
           + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
-  @Override
-  public void run() {
-    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index ce79e3c8228a6..a7de9c0f3d0d2 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -47,7 +46,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -102,14 +101,6 @@ protected void initializeServer() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  @Override
-  public void run() {
-    try {
       server.serve();
     } catch (Throwable t) {
       LOG.error(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index e46799a1c427d..d41c3b493bb47 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -176,7 +176,6 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
-      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -671,8 +670,6 @@ public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
     return resp;
   }
 
-  protected abstract void initializeServer();
-
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index ab9ed5b1f371e..73d5f84476af0 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.rpc.thrift.TCLIService;
@@ -55,8 +54,13 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -147,19 +151,6 @@ protected void initializeServer() {
           + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
-  @Override
-  public void run() {
-    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.error(

From 4638402d4775acf0d7ff190ef5eabb6f2dc85af5 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 11 Jun 2020 03:29:12 +0000
Subject: [PATCH 0811/1280] [SPARK-31939][SQL][TEST-JAVA11] Fix Parsing day of
 year when year field pattern is missing

### What changes were proposed in this pull request?

If a datetime pattern contains no year field, the day of year field should not be ignored if exists

e.g.

```
spark-sql> select to_timestamp('31', 'DD');
1970-01-01 00:00:00
spark-sql> select to_timestamp('31 30', 'DD dd');
1970-01-30 00:00:00

spark.sql.legacy.timeParserPolicy legacy
spark-sql> select to_timestamp('31', 'DD');
1970-01-31 00:00:00
spark-sql> select to_timestamp('31 30', 'DD dd');
NULL
```

This PR only fixes some corner cases that use 'D' pattern to parse datetimes and there is w/o 'y'.

### Why are the changes needed?

fix some corner cases

### Does this PR introduce _any_ user-facing change?

yes, the day of year field will not be ignored

### How was this patch tested?

add unit tests.

Closes #28766 from yaooqinn/SPARK-31939.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 22dda6e18e91c6db6fa8ff9fafaafe09a79db4ea)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../util/DateTimeFormatterHelper.scala        |  26 ++++-
 .../catalyst/util/DateFormatterSuite.scala    |   2 +
 .../util/DatetimeFormatterSuite.scala         |  78 +++++++++++++
 .../util/TimestampFormatterSuite.scala        |   2 +
 .../inputs/datetime-parsing-invalid.sql       |  20 ++++
 .../inputs/datetime-parsing-legacy.sql        |   2 +
 .../sql-tests/inputs/datetime-parsing.sql     |  16 +++
 .../results/datetime-parsing-invalid.sql.out  | 110 ++++++++++++++++++
 .../results/datetime-parsing-legacy.sql.out   | 106 +++++++++++++++++
 .../results/datetime-parsing.sql.out          | 106 +++++++++++++++++
 10 files changed, 465 insertions(+), 3 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 992a2b12a462f..5de06af6af12a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -39,6 +39,18 @@ trait DateTimeFormatterHelper {
     }
   }
 
+  private def verifyLocalDate(
+      accessor: TemporalAccessor, field: ChronoField, candidate: LocalDate): Unit = {
+    if (accessor.isSupported(field)) {
+      val actual = accessor.get(field)
+      val expected = candidate.get(field)
+      if (actual != expected) {
+        throw new DateTimeException(s"Conflict found: Field $field $actual differs from" +
+          s" $field $expected derived from $candidate")
+      }
+    }
+  }
+
   protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
     val localDate = accessor.query(TemporalQueries.localDate())
     // If all the date fields are specified, return the local date directly.
@@ -48,9 +60,17 @@ trait DateTimeFormatterHelper {
     // later, and we should provide default values for missing fields.
     // To be compatible with Spark 2.4, we pick 1970 as the default value of year.
     val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
-    val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
-    val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
-    LocalDate.of(year, month, day)
+    if (accessor.isSupported(ChronoField.DAY_OF_YEAR)) {
+      val dayOfYear = accessor.get(ChronoField.DAY_OF_YEAR)
+      val date = LocalDate.ofYearDay(year, dayOfYear)
+      verifyLocalDate(accessor, ChronoField.MONTH_OF_YEAR, date)
+      verifyLocalDate(accessor, ChronoField.DAY_OF_MONTH, date)
+      date
+    } else {
+      val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+      val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+      LocalDate.of(year, month, day)
+    }
   }
 
   private def toLocalTime(accessor: TemporalAccessor): LocalTime = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
index 4892deae92b3d..0a29d94dd984d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
@@ -31,6 +31,8 @@ class DateFormatterSuite extends DatetimeFormatterSuite {
     DateFormatter(pattern, UTC, isParsing)
   }
 
+  override protected def useDateFormatter: Boolean = true
+
   test("parsing dates") {
     outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
index 31ff50fda1ad6..b78facd963338 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
@@ -17,15 +17,61 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.time.DateTimeException
+
 import org.scalatest.Matchers
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{date, UTC}
 
 trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
   import DateTimeFormatterHelper._
+  import LegacyDateFormats._
   def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
 
+  private def dateFormatter(
+      pattern: String, ldf: LegacyDateFormat = FAST_DATE_FORMAT): DateFormatter = {
+    DateFormatter(pattern, UTC, DateFormatter.defaultLocale, ldf, isParsing = true)
+  }
+
+  private def timestampFormatter(
+      pattern: String, ldf: LegacyDateFormat = SIMPLE_DATE_FORMAT): TimestampFormatter = {
+    TimestampFormatter(pattern, UTC, legacyFormat = ldf, isParsing = true)
+  }
+
+  protected def useDateFormatter: Boolean
+
+  private def assertEqual(pattern: String, datetimeStr: String, expected: Long): Unit = {
+    if (useDateFormatter) {
+      assert(dateFormatter(pattern).parse(datetimeStr) ===
+        DateTimeUtils.microsToEpochDays(expected, UTC))
+    } else {
+      assert(timestampFormatter(pattern).parse(datetimeStr) === expected)
+    }
+  }
+
+  private def assertError(pattern: String, datetimeStr: String, expectedMsg: String): Unit = {
+    if (useDateFormatter) {
+      LegacyDateFormats.values.foreach { ldf =>
+        // The legacy DateFormatter is always lenient by default
+        val e = intercept[SparkUpgradeException](dateFormatter(pattern, ldf).parse(datetimeStr))
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    } else {
+      // In strict mode, the legacy TimestampFormatter fails too
+      val e = intercept[DateTimeException](timestampFormatter(pattern).parse(datetimeStr))
+      assert(e.getMessage.contains(expectedMsg))
+      // In lenient mode, the legacy TimestampFormatter does not fail
+      Seq(FAST_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT).foreach { ldf =>
+        val e = intercept[SparkUpgradeException] {
+          timestampFormatter(pattern, ldf).parse(datetimeStr)
+        }
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    }
+  }
+
   test("explicitly forbidden datetime patterns") {
 
     Seq(true, false).foreach { isParsing =>
@@ -51,4 +97,36 @@ trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
     }
   }
+
+  test("SPARK-31939: Fix Parsing day of year when year field pattern is missing") {
+    // resolved to queryable LocaleDate or fail directly
+    assertEqual("yyyy-dd-DD", "2020-29-60", date(2020, 2, 29))
+    assertError("yyyy-dd-DD", "2020-02-60",
+      "Field DayOfMonth 29 differs from DayOfMonth 2 derived from 2020-02-29")
+    assertEqual("yyyy-MM-DD", "2020-02-60", date(2020, 2, 29))
+    assertError("yyyy-MM-DD", "2020-03-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 2020-02-29")
+    assertEqual("yyyy-MM-dd-DD", "2020-02-29-60", date(2020, 2, 29))
+    assertError("yyyy-MM-dd-DD", "2020-03-01-60",
+      "Field DayOfYear 61 differs from DayOfYear 60 derived from 2020-03-01")
+    assertEqual("yyyy-DDD", "2020-366", date(2020, 12, 31))
+    assertError("yyyy-DDD", "2019-366",
+      "Invalid date 'DayOfYear 366' as '2019' is not a leap year")
+
+    // unresolved and need to check manually(SPARK-31939 fixed)
+    assertEqual("DDD", "365", date(1970, 12, 31))
+    assertError("DDD", "366",
+      "Invalid date 'DayOfYear 366' as '1970' is not a leap year")
+    assertEqual("MM-DD", "03-60", date(1970, 3))
+    assertError("MM-DD", "02-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertEqual("MM-dd-DD", "02-28-59", date(1970, 2, 28))
+    assertError("MM-dd-DD", "02-28-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertError("MM-dd-DD", "02-28-58",
+      "Field DayOfMonth 28 differs from DayOfMonth 27 derived from 1970-02-27")
+    assertEqual("dd-DD", "28-59", date(1970, 2, 28))
+    assertError("dd-DD", "27-59",
+      "Field DayOfMonth 27 differs from DayOfMonth 28 derived from 1970-02-28")
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index 88f6b0d79f85e..e7d935001a76f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -33,6 +33,8 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
     TimestampFormatter(pattern, UTC, isParsing)
   }
 
+  override protected def useDateFormatter: Boolean = false
+
   test("parsing timestamps using time zones") {
     val localDate = "2018-12-02T10:11:12.001234"
     val expectedMicros = Map(
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
new file mode 100644
index 0000000000000..0b313e5a0b9f4
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
@@ -0,0 +1,20 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS WITH INVALID VALUES ---
+
+-- parsing invalid values with pattern 'D'
+select to_timestamp('366', 'D');
+select to_timestamp('9', 'DD');
+-- in java 8 this case is invalid, but valid in java 11, disabled for jenkins
+-- select to_timestamp('100', 'DD');
+select to_timestamp('366', 'DD');
+select to_timestamp('9', 'DDD');
+select to_timestamp('99', 'DDD');
+select to_timestamp('30-365', 'dd-DDD');
+select to_timestamp('11-365', 'MM-DDD');
+select to_timestamp('2019-366', 'yyyy-DDD');
+select to_timestamp('12-30-365', 'MM-dd-DDD');
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD');
+-- add a special case to test csv, because the legacy formatter it uses is lenient then Spark should
+-- throw SparkUpgradeException
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
new file mode 100644
index 0000000000000..ee1afe502ab79
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-parsing.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
new file mode 100644
index 0000000000000..74866d9c6ffa1
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
@@ -0,0 +1,16 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS ---
+
+-- parsing with pattern 'D'
+select to_timestamp('9', 'D');
+select to_timestamp('300', 'D');
+select to_timestamp('09', 'DD');
+select to_timestamp('99', 'DD');
+select to_timestamp('009', 'DDD');
+select to_timestamp('365', 'DDD');
+select to_timestamp('31-365', 'dd-DDD');
+select to_timestamp('12-365', 'MM-DDD');
+select to_timestamp('2020-365', 'yyyy-DDD');
+select to_timestamp('12-31-365', 'MM-dd-DDD');
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD');
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
new file mode 100644
index 0000000000000..dcee0d4c270a3
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
@@ -0,0 +1,110 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+select to_timestamp('366', 'D')
+-- !query schema
+struct<to_timestamp(366, D):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('366', 'DD')
+-- !query schema
+struct<to_timestamp(366, DD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('99', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '99' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('30-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp(30-365, dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('11-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp(11-365, MM-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-366', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp(2019-366, yyyy-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('12-30-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(12-30-365, MM-dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-01-365, yyyy-dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp(2020-10-350, yyyy-MM-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-11-31-366, yyyy-MM-dd-DDD):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-366' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
new file mode 100644
index 0000000000000..742e0839d919a
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
@@ -0,0 +1,106 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp(9, D):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp(300, D):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp(09, DD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp(99, DD):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp(009, DDD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp(365, DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp(31-365, dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp(12-365, MM-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
+-- !query output
+2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
new file mode 100644
index 0000000000000..742e0839d919a
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
@@ -0,0 +1,106 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp(9, D):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp(300, D):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp(09, DD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp(99, DD):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp(009, DDD):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp(365, DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp(31-365, dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp(12-365, MM-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
+-- !query output
+2020-12-31 00:00:00

From 15d2922b1efd8c365059d9e223d1be753d5d16ee Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 10 Jun 2020 15:54:07 -0700
Subject: [PATCH 0812/1280] [SPARK-31915][SQL][PYTHON] Resolve the grouping
 column properly per the case sensitivity in grouped and cogrouped pandas UDFs

### What changes were proposed in this pull request?

This is another approach to fix the issue. See the previous try https://github.com/apache/spark/pull/28745. It was too invasive so I took more conservative approach.

This PR proposes to resolve grouping attributes separately first so it can be properly referred when `FlatMapGroupsInPandas` and `FlatMapCoGroupsInPandas` are resolved without ambiguity.

Previously,

```python
from pyspark.sql.functions import *
df = spark.createDataFrame([[1, 1]], ["column", "Score"])
pandas_udf("column integer, Score float", PandasUDFType.GROUPED_MAP)
def my_pandas_udf(pdf):
    return pdf.assign(Score=0.5)

df.groupby('COLUMN').apply(my_pandas_udf).show()
```

was failed as below:

```
pyspark.sql.utils.AnalysisException: "Reference 'COLUMN' is ambiguous, could be: COLUMN, COLUMN.;"
```
because the unresolved `COLUMN` in `FlatMapGroupsInPandas` doesn't know which reference to take from the child projection.

After this fix, it resolves the child projection first with grouping keys and pass, to `FlatMapGroupsInPandas`, the attribute as a grouping key from the child projection that is positionally selected.

### Why are the changes needed?

To resolve grouping keys correctly.

### Does this PR introduce _any_ user-facing change?

Yes,

```python
from pyspark.sql.functions import *
df = spark.createDataFrame([[1, 1]], ["column", "Score"])
pandas_udf("column integer, Score float", PandasUDFType.GROUPED_MAP)
def my_pandas_udf(pdf):
    return pdf.assign(Score=0.5)

df.groupby('COLUMN').apply(my_pandas_udf).show()
```

```python
df1 = spark.createDataFrame([(1, 1)], ("column", "value"))
df2 = spark.createDataFrame([(1, 1)], ("column", "value"))

df1.groupby("COLUMN").cogroup(
    df2.groupby("COLUMN")
).applyInPandas(lambda r, l: r + l, df1.schema).show()
```

Before:

```
pyspark.sql.utils.AnalysisException: Reference 'COLUMN' is ambiguous, could be: COLUMN, COLUMN.;
```

```
pyspark.sql.utils.AnalysisException: cannot resolve '`COLUMN`' given input columns: [COLUMN, COLUMN, value, value];;
'FlatMapCoGroupsInPandas ['COLUMN], ['COLUMN], <lambda>(column#9L, value#10L, column#13L, value#14L), [column#22L, value#23L]
:- Project [COLUMN#9L, column#9L, value#10L]
:  +- LogicalRDD [column#9L, value#10L], false
+- Project [COLUMN#13L, column#13L, value#14L]
   +- LogicalRDD [column#13L, value#14L], false
```

After:

```
+------+-----+
|column|Score|
+------+-----+
|     1|  0.5|
+------+-----+
```

```
+------+-----+
|column|value|
+------+-----+
|     2|    2|
+------+-----+
```

### How was this patch tested?

Unittests were added and manually tested.

Closes #28777 from HyukjinKwon/SPARK-31915-another.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Bryan Cutler <cutlerb@gmail.com>
---
 .../sql/tests/test_pandas_cogrouped_map.py     | 18 +++++++++++++++++-
 .../sql/tests/test_pandas_grouped_map.py       | 10 ++++++++++
 .../spark/sql/RelationalGroupedDataset.scala   | 17 ++++++++++-------
 3 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_cogrouped_map.py b/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
index 3ed9d2ac62fd3..c1cb30c3caa91 100644
--- a/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_cogrouped_map.py
@@ -19,7 +19,7 @@
 import sys
 
 from pyspark.sql.functions import array, explode, col, lit, udf, sum, pandas_udf, PandasUDFType
-from pyspark.sql.types import DoubleType, StructType, StructField
+from pyspark.sql.types import DoubleType, StructType, StructField, Row
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
 from pyspark.testing.utils import QuietTest
@@ -193,6 +193,22 @@ def test_wrong_args(self):
             left.groupby('id').cogroup(right.groupby('id')) \
                 .applyInPandas(lambda: 1, StructType([StructField("d", DoubleType())]))
 
+    def test_case_insensitive_grouping_column(self):
+        # SPARK-31915: case-insensitive grouping column should work.
+        df1 = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df1.groupby("ColUmn").cogroup(
+            df1.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
+        self.assertEquals(row.asDict(), Row(column=2, value=2).asDict())
+
+        df2 = self.spark.createDataFrame([(1, 1)], ("column", "value"))
+
+        row = df1.groupby("ColUmn").cogroup(
+            df2.groupby("COLUMN")
+        ).applyInPandas(lambda r, l: r + l, "column long, value long").first()
+        self.assertEquals(row.asDict(), Row(column=2, value=2).asDict())
+
     @staticmethod
     def _test_with_key(left, right, isLeft):
 
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index ff53a0c6f2cf2..76119432662ba 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -587,6 +587,16 @@ def f(key, pdf):
         # Check that all group and window_range values from udf matched expected
         self.assertTrue(all([r[0] for r in result]))
 
+    def test_case_insensitive_grouping_column(self):
+        # SPARK-31915: case-insensitive grouping column should work.
+        def my_pandas_udf(pdf):
+            return pdf.assign(score=0.5)
+
+        df = self.spark.createDataFrame([[1, 1]], ["column", "score"])
+        row = df.groupby('COLUMN').applyInPandas(
+            my_pandas_udf, schema="column integer, score float").first()
+        self.assertEquals(row.asDict(), Row(column=1, score=0.5).asDict())
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_grouped_map import *
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
index b1ba7d4538732..c37d8eaa294bf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala
@@ -546,9 +546,10 @@ class RelationalGroupedDataset protected[sql](
       case ne: NamedExpression => ne
       case other => Alias(other, other.toString)()
     }
-    val groupingAttributes = groupingNamedExpressions.map(_.toAttribute)
     val child = df.logicalPlan
-    val project = Project(groupingNamedExpressions ++ child.output, child)
+    val project = df.sparkSession.sessionState.executePlan(
+      Project(groupingNamedExpressions ++ child.output, child)).analyzed
+    val groupingAttributes = project.output.take(groupingNamedExpressions.length)
     val output = expr.dataType.asInstanceOf[StructType].toAttributes
     val plan = FlatMapGroupsInPandas(groupingAttributes, expr, output, project)
 
@@ -583,14 +584,16 @@ class RelationalGroupedDataset protected[sql](
       case other => Alias(other, other.toString)()
     }
 
-    val leftAttributes = leftGroupingNamedExpressions.map(_.toAttribute)
-    val rightAttributes = rightGroupingNamedExpressions.map(_.toAttribute)
-
     val leftChild = df.logicalPlan
     val rightChild = r.df.logicalPlan
 
-    val left = Project(leftGroupingNamedExpressions ++ leftChild.output, leftChild)
-    val right = Project(rightGroupingNamedExpressions ++ rightChild.output, rightChild)
+    val left = df.sparkSession.sessionState.executePlan(
+      Project(leftGroupingNamedExpressions ++ leftChild.output, leftChild)).analyzed
+    val right = r.df.sparkSession.sessionState.executePlan(
+      Project(rightGroupingNamedExpressions ++ rightChild.output, rightChild)).analyzed
+
+    val leftAttributes = left.output.take(leftGroupingNamedExpressions.length)
+    val rightAttributes = right.output.take(rightGroupingNamedExpressions.length)
 
     val output = expr.dataType.asInstanceOf[StructType].toAttributes
     val plan = FlatMapCoGroupsInPandas(leftAttributes, rightAttributes, expr, output, left, right)

From 8ad9b83edc239eae6b468d619419af5c0f41b4d0 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 10 Jun 2020 21:15:40 -0700
Subject: [PATCH 0813/1280] [SPARK-31965][TESTS][PYTHON] Move doctests related
 to Java function registration to test conditionally

### What changes were proposed in this pull request?

This PR proposes to move the doctests in `registerJavaUDAF` and `registerJavaFunction` to the proper unittests that run conditionally when the test classes are present.

Both tests are dependent on the test classes in JVM side, `test.org.apache.spark.sql.JavaStringLength` and `test.org.apache.spark.sql.MyDoubleAvg`. So if you run the tests against the plain `sbt package`, it fails as below:

```
**********************************************************************
File "/.../spark/python/pyspark/sql/udf.py", line 366, in pyspark.sql.udf.UDFRegistration.registerJavaFunction
Failed example:
    spark.udf.registerJavaFunction(
        "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
Exception raised:
    Traceback (most recent call last):
   ...
test.org.apache.spark.sql.JavaStringLength, please make sure it is on the classpath;
...
   6 of   7 in pyspark.sql.udf.UDFRegistration.registerJavaFunction
   2 of   4 in pyspark.sql.udf.UDFRegistration.registerJavaUDAF
***Test Failed*** 8 failures.
```

### Why are the changes needed?

In order to support to run the tests against the plain SBT build. See also https://spark.apache.org/developer-tools.html

### Does this PR introduce _any_ user-facing change?

No, it's test-only.

### How was this patch tested?

Manually tested as below:

```bash
./build/sbt -DskipTests -Phive-thriftserver clean package
cd python
./run-tests --python-executable=python3 --testname="pyspark.sql.udf UserDefinedFunction"
./run-tests --python-executable=python3 --testname="pyspark.sql.tests.test_udf UDFTests"
```

```bash
./build/sbt -DskipTests -Phive-thriftserver clean test:package
cd python
./run-tests --python-executable=python3 --testname="pyspark.sql.udf UserDefinedFunction"
./run-tests --python-executable=python3 --testname="pyspark.sql.tests.test_udf UDFTests"
```

Closes #28795 from HyukjinKwon/SPARK-31965.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 56264fb5d3ad1a488be5e08feb2e0304d1c2ed6a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 python/pyspark/sql/tests/test_udf.py | 28 ++++++++++++++++++++++++++++
 python/pyspark/sql/udf.py            | 14 +++++++++-----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index 061d3f5e1f7ac..ea7ec9f1e7c0c 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -21,6 +21,8 @@
 import tempfile
 import unittest
 
+import py4j
+
 from pyspark import SparkContext
 from pyspark.sql import SparkSession, Column, Row
 from pyspark.sql.functions import UserDefinedFunction, udf
@@ -357,6 +359,32 @@ def test_udf_registration_returns_udf(self):
             df.select(add_four("id").alias("plus_four")).collect()
         )
 
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)
+    def test_register_java_function(self):
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
+        [value] = self.spark.sql("SELECT javaStringLength('test')").first()
+        self.assertEqual(value, 4)
+
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
+        [value] = self.spark.sql("SELECT javaStringLength2('test')").first()
+        self.assertEqual(value, 4)
+
+        self.spark.udf.registerJavaFunction(
+            "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
+        [value] = self.spark.sql("SELECT javaStringLength3('test')").first()
+        self.assertEqual(value, 4)
+
+    @unittest.skipIf(not test_compiled, test_not_compiled_message)
+    def test_register_java_udaf(self):
+        self.spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg")
+        df = self.spark.createDataFrame([(1, "a"), (2, "b"), (3, "a")], ["id", "name"])
+        df.createOrReplaceTempView("df")
+        row = self.spark.sql(
+            "SELECT name, javaUDAF(id) as avg from df group by name order by name desc").first()
+        self.assertEqual(row.asDict(), Row(name='b', avg=102.0).asDict())
+
     def test_non_existed_udf(self):
         spark = self.spark
         self.assertRaisesRegexp(AnalysisException, "Can not load class non_existed_udf",
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 10546ecacc57f..da68583b04e1c 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -365,17 +365,20 @@ def registerJavaFunction(self, name, javaClassName, returnType=None):
         >>> from pyspark.sql.types import IntegerType
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
-        >>> spark.sql("SELECT javaStringLength('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength(test)=4)]
 
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
-        >>> spark.sql("SELECT javaStringLength2('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength2('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength2(test)=4)]
 
         >>> spark.udf.registerJavaFunction(
         ...     "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
-        >>> spark.sql("SELECT javaStringLength3('test')").collect()
+        ... # doctest: +SKIP
+        >>> spark.sql("SELECT javaStringLength3('test')").collect()  # doctest: +SKIP
         [Row(javaStringLength3(test)=4)]
         """
 
@@ -395,10 +398,11 @@ def registerJavaUDAF(self, name, javaClassName):
         :param javaClassName: fully qualified name of java class
 
         >>> spark.udf.registerJavaUDAF("javaUDAF", "test.org.apache.spark.sql.MyDoubleAvg")
+        ... # doctest: +SKIP
         >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "a")],["id", "name"])
         >>> df.createOrReplaceTempView("df")
-        >>> spark.sql("SELECT name, javaUDAF(id) as avg from df group by name order by name desc") \
-                .collect()
+        >>> q = "SELECT name, javaUDAF(id) as avg from df group by name order by name desc"
+        >>> spark.sql(q).collect()  # doctest: +SKIP
         [Row(name=u'b', avg=102.0), Row(name=u'a', avg=102.0)]
         """
 

From 5412009d157f77ee4c90de12079502046f9c8682 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Wed, 10 Jun 2020 21:32:16 -0700
Subject: [PATCH 0814/1280] [SPARK-28199][SS][FOLLOWUP] Remove package private
 in class/object in sql.execution package

### What changes were proposed in this pull request?

This PR proposes to remove package private in classes/objects in sql.execution package, as per SPARK-16964.

### Why are the changes needed?

This is per post-hoc review comment, see https://github.com/apache/spark/pull/24996#discussion_r437126445

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

N/A

Closes #28790 from HeartSaVioR/SPARK-28199-FOLLOWUP-apply-SPARK-16964.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 4afe2b1bc9ef190c0117e28da447871b90100622)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/streaming/Triggers.scala       | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
index d40208f02e7b1..28171f4ca645e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/Triggers.scala
@@ -50,17 +50,17 @@ private object Triggers {
  * A [[Trigger]] that processes only one batch of data in a streaming query then terminates
  * the query.
  */
-private[sql] case object OneTimeTrigger extends Trigger
+case object OneTimeTrigger extends Trigger
 
 /**
  * A [[Trigger]] that runs a query periodically based on the processing time. If `interval` is 0,
  * the query will run as fast as possible.
  */
-private[sql] case class ProcessingTimeTrigger(intervalMs: Long) extends Trigger {
+case class ProcessingTimeTrigger(intervalMs: Long) extends Trigger {
   Triggers.validate(intervalMs)
 }
 
-private[sql] object ProcessingTimeTrigger {
+object ProcessingTimeTrigger {
   import Triggers._
 
   def apply(interval: String): ProcessingTimeTrigger = {
@@ -84,11 +84,11 @@ private[sql] object ProcessingTimeTrigger {
  * A [[Trigger]] that continuously processes streaming data, asynchronously checkpointing at
  * the specified interval.
  */
-private[sql] case class ContinuousTrigger(intervalMs: Long) extends Trigger {
+case class ContinuousTrigger(intervalMs: Long) extends Trigger {
   Triggers.validate(intervalMs)
 }
 
-private[sql] object ContinuousTrigger {
+object ContinuousTrigger {
   import Triggers._
 
   def apply(interval: String): ContinuousTrigger = {

From d1a3fad8eda581a4b7c4350ded073a0c904c424b Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 10 Jun 2020 21:56:35 -0700
Subject: [PATCH 0815/1280] [SPARK-31966][ML][TESTS][PYTHON] Increase the
 timeout for
 StreamingLogisticRegressionWithSGDTests.test_training_and_prediction

### What changes were proposed in this pull request?

This is similar with https://github.com/apache/spark/commit/64cb6f7066134a0b9e441291992d2da73de5d918

The test `StreamingLogisticRegressionWithSGDTests.test_training_and_prediction` seems also flaky. This PR just increases the timeout to 3 mins too. The cause is very likely the time elapsed.

See https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/123787/testReport/pyspark.mllib.tests.test_streaming_algorithms/StreamingLogisticRegressionWithSGDTests/test_training_and_prediction/

```
Traceback (most recent call last):
  File "/home/jenkins/workspace/SparkPullRequestBuilder2/python/pyspark/mllib/tests/test_streaming_algorithms.py", line 330, in test_training_and_prediction
    eventually(condition, timeout=60.0)
  File "/home/jenkins/workspace/SparkPullRequestBuilder2/python/pyspark/testing/utils.py", line 90, in eventually
    % (timeout, lastValue))
AssertionError: Test failed due to timeout after 60 sec, with last condition returning: Latest errors: 0.67, 0.71, 0.78, 0.7, 0.75, 0.74, 0.73, 0.69, 0.62, 0.71, 0.69, 0.75, 0.72, 0.77, 0.71, 0.74, 0.76, 0.78, 0.7, 0.78, 0.8, 0.74, 0.77, 0.75, 0.76, 0.76, 0.75
```

### Why are the changes needed?

To make PR builds more stable.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Jenkins will test them out.

Closes #28798 from HyukjinKwon/SPARK-31966.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 56d4f27cf66cda81940a0a9e7959c03fad2844fc)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 python/pyspark/mllib/tests/test_streaming_algorithms.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index f57de83bae64d..2f35e07cdf71c 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -327,7 +327,7 @@ def condition():
                 return True
             return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
 
-        eventually(condition, timeout=60.0)
+        eventually(condition, timeout=180.0)
 
 
 class StreamingLinearRegressionWithTests(MLLibStreamingTestCase):

From 593b42323255f44b268ca5148f1fa817f3c01de7 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 11 Jun 2020 06:39:14 +0000
Subject: [PATCH 0816/1280] [SPARK-31958][SQL] normalize special floating
 numbers in subquery

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/23388 .

https://github.com/apache/spark/pull/23388 has an issue: it doesn't handle subquery expressions and assumes they will be turned into joins. However, this is not true for non-correlated subquery expressions.

This PR fixes this issue. It now doesn't skip `Subquery`, and subquery expressions will be handled by `OptimizeSubqueries`, which runs the optimizer with the subquery.

Note that, correlated subquery expressions will be handled twice: once in `OptimizeSubqueries`, once later when it becomes join. This is OK as `NormalizeFloatingNumbers` is idempotent now.

### Why are the changes needed?

fix a bug

### Does this PR introduce _any_ user-facing change?

yes, see the newly added test.

### How was this patch tested?

new test

Closes #28785 from cloud-fan/normalize.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6fb9c80da129d0b43f9ff5b8be6ce8bad992a4ed)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../optimizer/NormalizeFloatingNumbers.scala   |  4 ----
 .../org/apache/spark/sql/SQLQuerySuite.scala   | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index 5f94af5ffe636..43738204c6704 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -56,10 +56,6 @@ import org.apache.spark.sql.types._
 object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
 
   def apply(plan: LogicalPlan): LogicalPlan = plan match {
-    // A subquery will be rewritten into join later, and will go through this rule
-    // eventually. Here we skip subquery, as we only need to run this rule once.
-    case _: Subquery => plan
-
     case _ => plan transform {
       case w: Window if w.partitionSpec.exists(p => needNormalize(p)) =>
         // Although the `windowExpressions` may refer to `partitionSpec` expressions, we don't need
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index a23e5831f5887..093f2dbd1e426 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3449,6 +3449,24 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     checkAnswer(sql("select CAST(-32768 as short) DIV CAST (-1 as short)"),
       Seq(Row(Short.MinValue.toLong * -1)))
   }
+
+  test("normalize special floating numbers in subquery") {
+    withTempView("v1", "v2", "v3") {
+      Seq(-0.0).toDF("d").createTempView("v1")
+      Seq(0.0).toDF("d").createTempView("v2")
+      spark.range(2).createTempView("v3")
+
+      // non-correlated subquery
+      checkAnswer(sql("SELECT (SELECT v1.d FROM v1 JOIN v2 ON v1.d = v2.d)"), Row(-0.0))
+      // correlated subquery
+      checkAnswer(
+        sql(
+          """
+            |SELECT id FROM v3 WHERE EXISTS
+            |  (SELECT v1.d FROM v1 JOIN v2 ON v1.d = v2.d WHERE id > 0)
+            |""".stripMargin), Row(1))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From 250c6fdecf169b2a6fa162af5034c3b67ea3f9bc Mon Sep 17 00:00:00 2001
From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Date: Thu, 11 Jun 2020 22:03:40 +0900
Subject: [PATCH 0817/1280] [SPARK-31954][SQL] Delete duplicate testcase in
 HiveQuerySuite

### What changes were proposed in this pull request?
remove duplicate test cases

### Why are the changes needed?
improve test quality

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
No  test

Closes #28782 from GuoPhilipse/31954-delete-duplicate-testcase.

Lead-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Co-authored-by: GuoPhilipse <guofei_ok@126.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 912d45df7c6535336f72c971c90fecd11cfe87e9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 ...ast #3-0-732ed232ac592c5e7f7c913a88874fd2} |  0
 ...ast #4-0-6d2da5cfada03605834e38bc4075bc79} |  0
 ...cast #4-0-732ed232ac592c5e7f7c913a88874fd2 |  1 -
 ...cast #8-0-6d2da5cfada03605834e38bc4075bc79 |  1 -
 .../sql/hive/execution/HiveQuerySuite.scala   | 26 +++++++------------
 5 files changed, 10 insertions(+), 18 deletions(-)
 rename sql/hive/src/test/resources/golden/{timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2 => timestamp cast #3-0-732ed232ac592c5e7f7c913a88874fd2} (100%)
 rename sql/hive/src/test/resources/golden/{timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79 => timestamp cast #4-0-6d2da5cfada03605834e38bc4075bc79} (100%)
 delete mode 100644 sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2
 delete mode 100644 sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79

diff --git a/sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2 b/sql/hive/src/test/resources/golden/timestamp cast #3-0-732ed232ac592c5e7f7c913a88874fd2
similarity index 100%
rename from sql/hive/src/test/resources/golden/timestamp cast #2-0-732ed232ac592c5e7f7c913a88874fd2
rename to sql/hive/src/test/resources/golden/timestamp cast #3-0-732ed232ac592c5e7f7c913a88874fd2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79 b/sql/hive/src/test/resources/golden/timestamp cast #4-0-6d2da5cfada03605834e38bc4075bc79
similarity index 100%
rename from sql/hive/src/test/resources/golden/timestamp cast #6-0-6d2da5cfada03605834e38bc4075bc79
rename to sql/hive/src/test/resources/golden/timestamp cast #4-0-6d2da5cfada03605834e38bc4075bc79
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2 b/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2
deleted file mode 100644
index 5625e59da8873..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #4-0-732ed232ac592c5e7f7c913a88874fd2	
+++ /dev/null
@@ -1 +0,0 @@
-1.2
diff --git a/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79 b/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79
deleted file mode 100644
index 1d94c8a014fb4..0000000000000
--- a/sql/hive/src/test/resources/golden/timestamp cast #8-0-6d2da5cfada03605834e38bc4075bc79	
+++ /dev/null
@@ -1 +0,0 @@
--1.2
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 63b985fbe4d32..b10a8cb8bf2bf 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -556,33 +556,27 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
     assert(1 == res.getDouble(0))
   }
 
-  createQueryTest("timestamp cast #2",
-    "SELECT CAST(CAST(1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
-  test("timestamp cast #3") {
-    val res = sql("SELECT CAST(CAST(1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
-    assert(1200 == res.getInt(0))
+  test("timestamp cast #2") {
+    val res = sql("SELECT CAST(CAST(-1 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1").collect().head
+    assert(-1 == res.get(0))
   }
 
-  createQueryTest("timestamp cast #4",
+  createQueryTest("timestamp cast #3",
     "SELECT CAST(CAST(1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
 
+  createQueryTest("timestamp cast #4",
+    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
+
   test("timestamp cast #5") {
-    val res = sql("SELECT CAST(CAST(-1 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1").collect().head
-    assert(-1 == res.get(0))
+    val res = sql("SELECT CAST(CAST(1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
+    assert(1200 == res.getInt(0))
   }
 
-  createQueryTest("timestamp cast #6",
-    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
-  test("timestamp cast #7") {
+  test("timestamp cast #6") {
     val res = sql("SELECT CAST(CAST(-1200 AS TIMESTAMP) AS INT) FROM src LIMIT 1").collect().head
     assert(-1200 == res.getInt(0))
   }
 
-  createQueryTest("timestamp cast #8",
-    "SELECT CAST(CAST(-1.2 AS TIMESTAMP) AS DOUBLE) FROM src LIMIT 1")
-
   createQueryTest("select null from table",
     "SELECT null FROM src LIMIT 1")
 

From dad163f0f4a70b84f560de4f1f0bb89354e39217 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 11 Jun 2020 22:59:43 +0900
Subject: [PATCH 0818/1280] Revert "[SPARK-31939][SQL][TEST-JAVA11] Fix Parsing
 day of year when year field pattern is missing"

This reverts commit 4638402d4775acf0d7ff190ef5eabb6f2dc85af5.
---
 .../util/DateTimeFormatterHelper.scala        |  26 +----
 .../catalyst/util/DateFormatterSuite.scala    |   2 -
 .../util/DatetimeFormatterSuite.scala         |  78 -------------
 .../util/TimestampFormatterSuite.scala        |   2 -
 .../inputs/datetime-parsing-invalid.sql       |  20 ----
 .../inputs/datetime-parsing-legacy.sql        |   2 -
 .../sql-tests/inputs/datetime-parsing.sql     |  16 ---
 .../results/datetime-parsing-invalid.sql.out  | 110 ------------------
 .../results/datetime-parsing-legacy.sql.out   | 106 -----------------
 .../results/datetime-parsing.sql.out          | 106 -----------------
 10 files changed, 3 insertions(+), 465 deletions(-)
 delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
 delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
 delete mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
 delete mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
 delete mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
 delete mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 5de06af6af12a..992a2b12a462f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -39,18 +39,6 @@ trait DateTimeFormatterHelper {
     }
   }
 
-  private def verifyLocalDate(
-      accessor: TemporalAccessor, field: ChronoField, candidate: LocalDate): Unit = {
-    if (accessor.isSupported(field)) {
-      val actual = accessor.get(field)
-      val expected = candidate.get(field)
-      if (actual != expected) {
-        throw new DateTimeException(s"Conflict found: Field $field $actual differs from" +
-          s" $field $expected derived from $candidate")
-      }
-    }
-  }
-
   protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
     val localDate = accessor.query(TemporalQueries.localDate())
     // If all the date fields are specified, return the local date directly.
@@ -60,17 +48,9 @@ trait DateTimeFormatterHelper {
     // later, and we should provide default values for missing fields.
     // To be compatible with Spark 2.4, we pick 1970 as the default value of year.
     val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
-    if (accessor.isSupported(ChronoField.DAY_OF_YEAR)) {
-      val dayOfYear = accessor.get(ChronoField.DAY_OF_YEAR)
-      val date = LocalDate.ofYearDay(year, dayOfYear)
-      verifyLocalDate(accessor, ChronoField.MONTH_OF_YEAR, date)
-      verifyLocalDate(accessor, ChronoField.DAY_OF_MONTH, date)
-      date
-    } else {
-      val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
-      val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
-      LocalDate.of(year, month, day)
-    }
+    val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+    val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+    LocalDate.of(year, month, day)
   }
 
   private def toLocalTime(accessor: TemporalAccessor): LocalTime = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
index 0a29d94dd984d..4892deae92b3d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
@@ -31,8 +31,6 @@ class DateFormatterSuite extends DatetimeFormatterSuite {
     DateFormatter(pattern, UTC, isParsing)
   }
 
-  override protected def useDateFormatter: Boolean = true
-
   test("parsing dates") {
     outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
index b78facd963338..31ff50fda1ad6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
@@ -17,61 +17,15 @@
 
 package org.apache.spark.sql.catalyst.util
 
-import java.time.DateTimeException
-
 import org.scalatest.Matchers
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{date, UTC}
 
 trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
   import DateTimeFormatterHelper._
-  import LegacyDateFormats._
   def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
 
-  private def dateFormatter(
-      pattern: String, ldf: LegacyDateFormat = FAST_DATE_FORMAT): DateFormatter = {
-    DateFormatter(pattern, UTC, DateFormatter.defaultLocale, ldf, isParsing = true)
-  }
-
-  private def timestampFormatter(
-      pattern: String, ldf: LegacyDateFormat = SIMPLE_DATE_FORMAT): TimestampFormatter = {
-    TimestampFormatter(pattern, UTC, legacyFormat = ldf, isParsing = true)
-  }
-
-  protected def useDateFormatter: Boolean
-
-  private def assertEqual(pattern: String, datetimeStr: String, expected: Long): Unit = {
-    if (useDateFormatter) {
-      assert(dateFormatter(pattern).parse(datetimeStr) ===
-        DateTimeUtils.microsToEpochDays(expected, UTC))
-    } else {
-      assert(timestampFormatter(pattern).parse(datetimeStr) === expected)
-    }
-  }
-
-  private def assertError(pattern: String, datetimeStr: String, expectedMsg: String): Unit = {
-    if (useDateFormatter) {
-      LegacyDateFormats.values.foreach { ldf =>
-        // The legacy DateFormatter is always lenient by default
-        val e = intercept[SparkUpgradeException](dateFormatter(pattern, ldf).parse(datetimeStr))
-        assert(e.getCause.getMessage.contains(expectedMsg))
-      }
-    } else {
-      // In strict mode, the legacy TimestampFormatter fails too
-      val e = intercept[DateTimeException](timestampFormatter(pattern).parse(datetimeStr))
-      assert(e.getMessage.contains(expectedMsg))
-      // In lenient mode, the legacy TimestampFormatter does not fail
-      Seq(FAST_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT).foreach { ldf =>
-        val e = intercept[SparkUpgradeException] {
-          timestampFormatter(pattern, ldf).parse(datetimeStr)
-        }
-        assert(e.getCause.getMessage.contains(expectedMsg))
-      }
-    }
-  }
-
   test("explicitly forbidden datetime patterns") {
 
     Seq(true, false).foreach { isParsing =>
@@ -97,36 +51,4 @@ trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
     }
   }
-
-  test("SPARK-31939: Fix Parsing day of year when year field pattern is missing") {
-    // resolved to queryable LocaleDate or fail directly
-    assertEqual("yyyy-dd-DD", "2020-29-60", date(2020, 2, 29))
-    assertError("yyyy-dd-DD", "2020-02-60",
-      "Field DayOfMonth 29 differs from DayOfMonth 2 derived from 2020-02-29")
-    assertEqual("yyyy-MM-DD", "2020-02-60", date(2020, 2, 29))
-    assertError("yyyy-MM-DD", "2020-03-60",
-      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 2020-02-29")
-    assertEqual("yyyy-MM-dd-DD", "2020-02-29-60", date(2020, 2, 29))
-    assertError("yyyy-MM-dd-DD", "2020-03-01-60",
-      "Field DayOfYear 61 differs from DayOfYear 60 derived from 2020-03-01")
-    assertEqual("yyyy-DDD", "2020-366", date(2020, 12, 31))
-    assertError("yyyy-DDD", "2019-366",
-      "Invalid date 'DayOfYear 366' as '2019' is not a leap year")
-
-    // unresolved and need to check manually(SPARK-31939 fixed)
-    assertEqual("DDD", "365", date(1970, 12, 31))
-    assertError("DDD", "366",
-      "Invalid date 'DayOfYear 366' as '1970' is not a leap year")
-    assertEqual("MM-DD", "03-60", date(1970, 3))
-    assertError("MM-DD", "02-60",
-      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
-    assertEqual("MM-dd-DD", "02-28-59", date(1970, 2, 28))
-    assertError("MM-dd-DD", "02-28-60",
-      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
-    assertError("MM-dd-DD", "02-28-58",
-      "Field DayOfMonth 28 differs from DayOfMonth 27 derived from 1970-02-27")
-    assertEqual("dd-DD", "28-59", date(1970, 2, 28))
-    assertError("dd-DD", "27-59",
-      "Field DayOfMonth 27 differs from DayOfMonth 28 derived from 1970-02-28")
-  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index e7d935001a76f..88f6b0d79f85e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -33,8 +33,6 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
     TimestampFormatter(pattern, UTC, isParsing)
   }
 
-  override protected def useDateFormatter: Boolean = false
-
   test("parsing timestamps using time zones") {
     val localDate = "2018-12-02T10:11:12.001234"
     val expectedMicros = Map(
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
deleted file mode 100644
index 0b313e5a0b9f4..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
+++ /dev/null
@@ -1,20 +0,0 @@
---- TESTS FOR DATETIME PARSING FUNCTIONS WITH INVALID VALUES ---
-
--- parsing invalid values with pattern 'D'
-select to_timestamp('366', 'D');
-select to_timestamp('9', 'DD');
--- in java 8 this case is invalid, but valid in java 11, disabled for jenkins
--- select to_timestamp('100', 'DD');
-select to_timestamp('366', 'DD');
-select to_timestamp('9', 'DDD');
-select to_timestamp('99', 'DDD');
-select to_timestamp('30-365', 'dd-DDD');
-select to_timestamp('11-365', 'MM-DDD');
-select to_timestamp('2019-366', 'yyyy-DDD');
-select to_timestamp('12-30-365', 'MM-dd-DDD');
-select to_timestamp('2020-01-365', 'yyyy-dd-DDD');
-select to_timestamp('2020-10-350', 'yyyy-MM-DDD');
-select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD');
--- add a special case to test csv, because the legacy formatter it uses is lenient then Spark should
--- throw SparkUpgradeException
-select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
deleted file mode 100644
index ee1afe502ab79..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
+++ /dev/null
@@ -1,2 +0,0 @@
---SET spark.sql.legacy.timeParserPolicy=LEGACY
---IMPORT datetime-parsing.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
deleted file mode 100644
index 74866d9c6ffa1..0000000000000
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
+++ /dev/null
@@ -1,16 +0,0 @@
---- TESTS FOR DATETIME PARSING FUNCTIONS ---
-
--- parsing with pattern 'D'
-select to_timestamp('9', 'D');
-select to_timestamp('300', 'D');
-select to_timestamp('09', 'DD');
-select to_timestamp('99', 'DD');
-select to_timestamp('009', 'DDD');
-select to_timestamp('365', 'DDD');
-select to_timestamp('31-365', 'dd-DDD');
-select to_timestamp('12-365', 'MM-DDD');
-select to_timestamp('2020-365', 'yyyy-DDD');
-select to_timestamp('12-31-365', 'MM-dd-DDD');
-select to_timestamp('2020-30-365', 'yyyy-dd-DDD');
-select to_timestamp('2020-12-350', 'yyyy-MM-DDD');
-select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD');
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
deleted file mode 100644
index dcee0d4c270a3..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
+++ /dev/null
@@ -1,110 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
-
-
--- !query
-select to_timestamp('366', 'D')
--- !query schema
-struct<to_timestamp(366, D):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('9', 'DD')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
-
-
--- !query
-select to_timestamp('366', 'DD')
--- !query schema
-struct<to_timestamp(366, DD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('9', 'DDD')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
-
-
--- !query
-select to_timestamp('99', 'DDD')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to parse '99' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
-
-
--- !query
-select to_timestamp('30-365', 'dd-DDD')
--- !query schema
-struct<to_timestamp(30-365, dd-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('11-365', 'MM-DDD')
--- !query schema
-struct<to_timestamp(11-365, MM-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('2019-366', 'yyyy-DDD')
--- !query schema
-struct<to_timestamp(2019-366, yyyy-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('12-30-365', 'MM-dd-DDD')
--- !query schema
-struct<to_timestamp(12-30-365, MM-dd-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('2020-01-365', 'yyyy-dd-DDD')
--- !query schema
-struct<to_timestamp(2020-01-365, yyyy-dd-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('2020-10-350', 'yyyy-MM-DDD')
--- !query schema
-struct<to_timestamp(2020-10-350, yyyy-MM-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD')
--- !query schema
-struct<to_timestamp(2020-11-31-366, yyyy-MM-dd-DDD):timestamp>
--- !query output
-NULL
-
-
--- !query
-select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-366' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
deleted file mode 100644
index 742e0839d919a..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
+++ /dev/null
@@ -1,106 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
-
-
--- !query
-select to_timestamp('9', 'D')
--- !query schema
-struct<to_timestamp(9, D):timestamp>
--- !query output
-1970-01-09 00:00:00
-
-
--- !query
-select to_timestamp('300', 'D')
--- !query schema
-struct<to_timestamp(300, D):timestamp>
--- !query output
-1970-10-27 00:00:00
-
-
--- !query
-select to_timestamp('09', 'DD')
--- !query schema
-struct<to_timestamp(09, DD):timestamp>
--- !query output
-1970-01-09 00:00:00
-
-
--- !query
-select to_timestamp('99', 'DD')
--- !query schema
-struct<to_timestamp(99, DD):timestamp>
--- !query output
-1970-04-09 00:00:00
-
-
--- !query
-select to_timestamp('009', 'DDD')
--- !query schema
-struct<to_timestamp(009, DDD):timestamp>
--- !query output
-1970-01-09 00:00:00
-
-
--- !query
-select to_timestamp('365', 'DDD')
--- !query schema
-struct<to_timestamp(365, DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('31-365', 'dd-DDD')
--- !query schema
-struct<to_timestamp(31-365, dd-DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('12-365', 'MM-DDD')
--- !query schema
-struct<to_timestamp(12-365, MM-DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('2020-365', 'yyyy-DDD')
--- !query schema
-struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
--- !query output
-2020-12-30 00:00:00
-
-
--- !query
-select to_timestamp('12-31-365', 'MM-dd-DDD')
--- !query schema
-struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
--- !query schema
-struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
--- !query output
-2020-12-30 00:00:00
-
-
--- !query
-select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
--- !query schema
-struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
--- !query output
-2020-12-15 00:00:00
-
-
--- !query
-select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
--- !query schema
-struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
--- !query output
-2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
deleted file mode 100644
index 742e0839d919a..0000000000000
--- a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
+++ /dev/null
@@ -1,106 +0,0 @@
--- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
-
-
--- !query
-select to_timestamp('9', 'D')
--- !query schema
-struct<to_timestamp(9, D):timestamp>
--- !query output
-1970-01-09 00:00:00
-
-
--- !query
-select to_timestamp('300', 'D')
--- !query schema
-struct<to_timestamp(300, D):timestamp>
--- !query output
-1970-10-27 00:00:00
-
-
--- !query
-select to_timestamp('09', 'DD')
--- !query schema
-struct<to_timestamp(09, DD):timestamp>
--- !query output
-1970-01-09 00:00:00
-
-
--- !query
-select to_timestamp('99', 'DD')
--- !query schema
-struct<to_timestamp(99, DD):timestamp>
--- !query output
-1970-04-09 00:00:00
-
-
--- !query
-select to_timestamp('009', 'DDD')
--- !query schema
-struct<to_timestamp(009, DDD):timestamp>
--- !query output
-1970-01-09 00:00:00
-
-
--- !query
-select to_timestamp('365', 'DDD')
--- !query schema
-struct<to_timestamp(365, DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('31-365', 'dd-DDD')
--- !query schema
-struct<to_timestamp(31-365, dd-DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('12-365', 'MM-DDD')
--- !query schema
-struct<to_timestamp(12-365, MM-DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('2020-365', 'yyyy-DDD')
--- !query schema
-struct<to_timestamp(2020-365, yyyy-DDD):timestamp>
--- !query output
-2020-12-30 00:00:00
-
-
--- !query
-select to_timestamp('12-31-365', 'MM-dd-DDD')
--- !query schema
-struct<to_timestamp(12-31-365, MM-dd-DDD):timestamp>
--- !query output
-1970-12-31 00:00:00
-
-
--- !query
-select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
--- !query schema
-struct<to_timestamp(2020-30-365, yyyy-dd-DDD):timestamp>
--- !query output
-2020-12-30 00:00:00
-
-
--- !query
-select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
--- !query schema
-struct<to_timestamp(2020-12-350, yyyy-MM-DDD):timestamp>
--- !query output
-2020-12-15 00:00:00
-
-
--- !query
-select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
--- !query schema
-struct<to_timestamp(2020-12-31-366, yyyy-MM-dd-DDD):timestamp>
--- !query output
-2020-12-31 00:00:00

From fe298c34e11823cb1371db1bff425ce8874fface Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Thu, 11 Jun 2020 14:18:19 -0700
Subject: [PATCH 0819/1280] [SPARK-31935][SQL][3.0] Hadoop file system config
 should be effective in data source options

### What changes were proposed in this pull request?

Mkae Hadoop file system config effective in data source options.

From `org.apache.hadoop.fs.FileSystem.java`:
```
  public static FileSystem get(URI uri, Configuration conf) throws IOException {
    String scheme = uri.getScheme();
    String authority = uri.getAuthority();

    if (scheme == null && authority == null) {     // use default FS
      return get(conf);
    }

    if (scheme != null && authority == null) {     // no authority
      URI defaultUri = getDefaultUri(conf);
      if (scheme.equals(defaultUri.getScheme())    // if scheme matches default
          && defaultUri.getAuthority() != null) {  // & default has authority
        return get(defaultUri, conf);              // return default
      }
    }

    String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
    if (conf.getBoolean(disableCacheName, false)) {
      return createFileSystem(uri, conf);
    }

    return CACHE.get(uri, conf);
  }
```
Before changes, the file system configurations in data source options are not propagated in `DataSource.scala`.
After changes, we can specify authority and URI schema related configurations for scanning file systems.

This problem only exists in data source V1. In V2, we already use `sparkSession.sessionState.newHadoopConfWithOptions(options)` in `FileTable`.
### Why are the changes needed?

Allow users to specify authority and URI schema related Hadoop configurations for file source reading.

### Does this PR introduce _any_ user-facing change?

Yes, the file system related Hadoop configuration in data source option will be effective on reading.

### How was this patch tested?

Unit test

Closes #28776 from gengliangwang/SPARK-31935-3.0.

Lead-authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Co-authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../execution/datasources/DataSource.scala    | 13 ++++++------
 .../spark/sql/FileBasedDataSourceSuite.scala  | 20 +++++++++++++++++++
 .../sql/streaming/FileStreamSourceSuite.scala | 12 +++++++++++
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
index 3615afcf86c7a..588a9b4eb4d09 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
@@ -109,6 +109,9 @@ case class DataSource(
 
   private def providingInstance() = providingClass.getConstructor().newInstance()
 
+  private def newHadoopConfiguration(): Configuration =
+    sparkSession.sessionState.newHadoopConfWithOptions(options)
+
   lazy val sourceInfo: SourceInfo = sourceSchema()
   private val caseInsensitiveOptions = CaseInsensitiveMap(options)
   private val equality = sparkSession.sessionState.conf.resolver
@@ -230,7 +233,7 @@ case class DataSource(
         // once the streaming job starts and some upstream source starts dropping data.
         val hdfsPath = new Path(path)
         if (!SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
-          val fs = hdfsPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+          val fs = hdfsPath.getFileSystem(newHadoopConfiguration())
           if (!fs.exists(hdfsPath)) {
             throw new AnalysisException(s"Path does not exist: $path")
           }
@@ -357,7 +360,7 @@ case class DataSource(
       case (format: FileFormat, _)
           if FileStreamSink.hasMetadata(
             caseInsensitiveOptions.get("path").toSeq ++ paths,
-            sparkSession.sessionState.newHadoopConf(),
+            newHadoopConfiguration(),
             sparkSession.sessionState.conf) =>
         val basePath = new Path((caseInsensitiveOptions.get("path").toSeq ++ paths).head)
         val fileCatalog = new MetadataLogFileIndex(sparkSession, basePath,
@@ -449,7 +452,7 @@ case class DataSource(
     val allPaths = paths ++ caseInsensitiveOptions.get("path")
     val outputPath = if (allPaths.length == 1) {
       val path = new Path(allPaths.head)
-      val fs = path.getFileSystem(sparkSession.sessionState.newHadoopConf())
+      val fs = path.getFileSystem(newHadoopConfiguration())
       path.makeQualified(fs.getUri, fs.getWorkingDirectory)
     } else {
       throw new IllegalArgumentException("Expected exactly one path to be specified, but " +
@@ -569,9 +572,7 @@ case class DataSource(
       checkEmptyGlobPath: Boolean,
       checkFilesExist: Boolean): Seq[Path] = {
     val allPaths = caseInsensitiveOptions.get("path") ++ paths
-    val hadoopConf = sparkSession.sessionState.newHadoopConf()
-
-    DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, hadoopConf,
+    DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, newHadoopConfiguration(),
       checkEmptyGlobPath, checkFilesExist)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index cb410b4f0d7dc..d8157d3c779b9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -843,6 +843,26 @@ class FileBasedDataSourceSuite extends QueryTest
     }
   }
 
+  test("SPARK-31935: Hadoop file system config should be effective in data source options") {
+    Seq("parquet", "").foreach { format =>
+      withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> format) {
+        withTempDir { dir =>
+          val path = dir.getCanonicalPath
+          val defaultFs = "nonexistFS://nonexistFS"
+          val expectMessage = "No FileSystem for scheme nonexistFS"
+          val message1 = intercept[java.io.IOException] {
+            spark.range(10).write.option("fs.defaultFS", defaultFs).parquet(path)
+          }.getMessage
+          assert(message1.filterNot(Set(':', '"').contains) == expectMessage)
+          val message2 = intercept[java.io.IOException] {
+            spark.read.option("fs.defaultFS", defaultFs).parquet(path)
+          }.getMessage
+          assert(message2.filterNot(Set(':', '"').contains) == expectMessage)
+        }
+      }
+    }
+  }
+
   test("SPARK-31116: Select nested schema with case insensitive mode") {
     // This test case failed at only Parquet. ORC is added for test coverage parity.
     Seq("orc", "parquet").foreach { format =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
index fa320333143ec..7b16aebc531fb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSourceSuite.scala
@@ -532,6 +532,18 @@ class FileStreamSourceSuite extends FileStreamSourceTest {
     }
   }
 
+  test("SPARK-31935: Hadoop file system config should be effective in data source options") {
+    withTempDir { dir =>
+      val path = dir.getCanonicalPath
+      val defaultFs = "nonexistFS://nonexistFS"
+      val expectMessage = "No FileSystem for scheme nonexistFS"
+      val message = intercept[java.io.IOException] {
+        spark.readStream.option("fs.defaultFS", defaultFs).text(path)
+      }.getMessage
+      assert(message.filterNot(Set(':', '"').contains) == expectMessage)
+    }
+  }
+
   test("read from textfile") {
     withTempDirs { case (src, tmp) =>
       val textStream = spark.readStream.textFile(src.getCanonicalPath)

From f61b31a5a484c7e90920ec36c456594ce92cdf73 Mon Sep 17 00:00:00 2001
From: Dilip Biswal <dkbiswal@gmail.com>
Date: Fri, 12 Jun 2020 09:19:29 +0900
Subject: [PATCH 0820/1280] [SPARK-31916][SQL] StringConcat can lead to
 StringIndexOutOfBoundsException

### What changes were proposed in this pull request?
A minor fix to fix the append method of StringConcat to cap the length at MAX_ROUNDED_ARRAY_LENGTH to make sure it does not overflow and cause StringIndexOutOfBoundsException

Thanks to **Jeffrey Stokes** for reporting the issue and explaining the underlying problem in detail in the JIRA.

### Why are the changes needed?
This fixes StringIndexOutOfBoundsException on an overflow.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Added a test in StringsUtilSuite.

Closes #28750 from dilipbiswal/SPARK-31916.

Authored-by: Dilip Biswal <dkbiswal@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit b87a342c7dd51046fcbe323db640c825646fb8d4)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/catalyst/util/StringUtils.scala |  6 +++-
 .../sql/catalyst/util/StringUtilsSuite.scala  | 32 ++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
index b42ae4e45366e..2a416d6c97584 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/StringUtils.scala
@@ -123,7 +123,11 @@ object StringUtils extends Logging {
           val stringToAppend = if (available >= sLen) s else s.substring(0, available)
           strings.append(stringToAppend)
         }
-        length += sLen
+
+        // Keeps the total length of appended strings. Note that we need to cap the length at
+        // `ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH`; otherwise, we will overflow
+        // length causing StringIndexOutOfBoundsException in the substring call above.
+        length = Math.min(length.toLong + sLen, ByteArrayMethods.MAX_ROUNDED_ARRAY_LENGTH).toInt
       }
     }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
index 67bc4bc81cb92..c68e89fc8a5ca 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/StringUtilsSuite.scala
@@ -18,9 +18,11 @@
 package org.apache.spark.sql.catalyst.util
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.util.StringUtils._
+import org.apache.spark.sql.internal.SQLConf
 
-class StringUtilsSuite extends SparkFunSuite {
+class StringUtilsSuite extends SparkFunSuite with SQLHelper {
 
   test("escapeLikeRegex") {
     val expectedEscapedStrOne = "(?s)\\Qa\\E\\Qb\\E\\Qd\\E\\Qe\\E\\Qf\\E"
@@ -98,4 +100,32 @@ class StringUtilsSuite extends SparkFunSuite {
     assert(checkLimit("1234567"))
     assert(checkLimit("1234567890"))
   }
+
+  test("SPARK-31916: StringConcat doesn't overflow on many inputs") {
+    val concat = new StringConcat(maxLength = 100)
+    val stringToAppend = "Test internal index of StringConcat does not overflow with many " +
+      "append calls"
+    0.to((Integer.MAX_VALUE / stringToAppend.length) + 1).foreach { _ =>
+      concat.append(stringToAppend)
+    }
+    assert(concat.toString.length === 100)
+  }
+
+  test("SPARK-31916: verify that PlanStringConcat's output shows the actual length of the plan") {
+    withSQLConf(SQLConf.MAX_PLAN_STRING_LENGTH.key -> "0") {
+      val concat = new PlanStringConcat()
+      0.to(3).foreach { i =>
+        concat.append(s"plan fragment $i")
+      }
+      assert(concat.toString === "Truncated plan of 60 characters")
+    }
+
+    withSQLConf(SQLConf.MAX_PLAN_STRING_LENGTH.key -> "60") {
+      val concat = new PlanStringConcat()
+      0.to(2).foreach { i =>
+        concat.append(s"plan fragment $i")
+      }
+      assert(concat.toString === "plan fragment 0plan fragment 1... 15 more characters")
+    }
+  }
 }

From 265bf04d7afa12ea6312721003f834e9538c9101 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 12 Jun 2020 11:52:05 +0900
Subject: [PATCH 0821/1280] [SPARK-31939][SQL][TEST-JAVA11][3.0] Fix Parsing
 day of year when year field pattern is missing

### What changes were proposed in this pull request?

This PR brings back 4638402d4775acf0d7ff190ef5eabb6f2dc85af5 reverted by dad163f0f4a70b84f560de4f1f0bb89354e39217 and fix test failure for branch-3.0

diffs made:
  re-genarate sql test results to match 3.0's schema, the RuntimeReplaceable expression has new pretty schema in master branch

### Why are the changes needed?

bring back reverted PR and fix tests

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

passing jenkins

Closes #28800 from yaooqinn/SPARK-31939-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../util/DateTimeFormatterHelper.scala        |  26 ++++-
 .../catalyst/util/DateFormatterSuite.scala    |   2 +
 .../util/DatetimeFormatterSuite.scala         |  78 +++++++++++++
 .../util/TimestampFormatterSuite.scala        |   2 +
 .../inputs/datetime-parsing-invalid.sql       |  20 ++++
 .../inputs/datetime-parsing-legacy.sql        |   2 +
 .../sql-tests/inputs/datetime-parsing.sql     |  16 +++
 .../results/datetime-parsing-invalid.sql.out  | 110 ++++++++++++++++++
 .../results/datetime-parsing-legacy.sql.out   | 106 +++++++++++++++++
 .../results/datetime-parsing.sql.out          | 106 +++++++++++++++++
 10 files changed, 465 insertions(+), 3 deletions(-)
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
 create mode 100644 sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 992a2b12a462f..5de06af6af12a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -39,6 +39,18 @@ trait DateTimeFormatterHelper {
     }
   }
 
+  private def verifyLocalDate(
+      accessor: TemporalAccessor, field: ChronoField, candidate: LocalDate): Unit = {
+    if (accessor.isSupported(field)) {
+      val actual = accessor.get(field)
+      val expected = candidate.get(field)
+      if (actual != expected) {
+        throw new DateTimeException(s"Conflict found: Field $field $actual differs from" +
+          s" $field $expected derived from $candidate")
+      }
+    }
+  }
+
   protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
     val localDate = accessor.query(TemporalQueries.localDate())
     // If all the date fields are specified, return the local date directly.
@@ -48,9 +60,17 @@ trait DateTimeFormatterHelper {
     // later, and we should provide default values for missing fields.
     // To be compatible with Spark 2.4, we pick 1970 as the default value of year.
     val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
-    val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
-    val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
-    LocalDate.of(year, month, day)
+    if (accessor.isSupported(ChronoField.DAY_OF_YEAR)) {
+      val dayOfYear = accessor.get(ChronoField.DAY_OF_YEAR)
+      val date = LocalDate.ofYearDay(year, dayOfYear)
+      verifyLocalDate(accessor, ChronoField.MONTH_OF_YEAR, date)
+      verifyLocalDate(accessor, ChronoField.DAY_OF_MONTH, date)
+      date
+    } else {
+      val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
+      val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
+      LocalDate.of(year, month, day)
+    }
   }
 
   private def toLocalTime(accessor: TemporalAccessor): LocalTime = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
index 4892deae92b3d..0a29d94dd984d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateFormatterSuite.scala
@@ -31,6 +31,8 @@ class DateFormatterSuite extends DatetimeFormatterSuite {
     DateFormatter(pattern, UTC, isParsing)
   }
 
+  override protected def useDateFormatter: Boolean = true
+
   test("parsing dates") {
     outstandingTimezonesIds.foreach { timeZone =>
       withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone) {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
index 31ff50fda1ad6..b78facd963338 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DatetimeFormatterSuite.scala
@@ -17,15 +17,61 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.time.DateTimeException
+
 import org.scalatest.Matchers
 
 import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{date, UTC}
 
 trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers {
   import DateTimeFormatterHelper._
+  import LegacyDateFormats._
   def checkFormatterCreation(pattern: String, isParsing: Boolean): Unit
 
+  private def dateFormatter(
+      pattern: String, ldf: LegacyDateFormat = FAST_DATE_FORMAT): DateFormatter = {
+    DateFormatter(pattern, UTC, DateFormatter.defaultLocale, ldf, isParsing = true)
+  }
+
+  private def timestampFormatter(
+      pattern: String, ldf: LegacyDateFormat = SIMPLE_DATE_FORMAT): TimestampFormatter = {
+    TimestampFormatter(pattern, UTC, legacyFormat = ldf, isParsing = true)
+  }
+
+  protected def useDateFormatter: Boolean
+
+  private def assertEqual(pattern: String, datetimeStr: String, expected: Long): Unit = {
+    if (useDateFormatter) {
+      assert(dateFormatter(pattern).parse(datetimeStr) ===
+        DateTimeUtils.microsToEpochDays(expected, UTC))
+    } else {
+      assert(timestampFormatter(pattern).parse(datetimeStr) === expected)
+    }
+  }
+
+  private def assertError(pattern: String, datetimeStr: String, expectedMsg: String): Unit = {
+    if (useDateFormatter) {
+      LegacyDateFormats.values.foreach { ldf =>
+        // The legacy DateFormatter is always lenient by default
+        val e = intercept[SparkUpgradeException](dateFormatter(pattern, ldf).parse(datetimeStr))
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    } else {
+      // In strict mode, the legacy TimestampFormatter fails too
+      val e = intercept[DateTimeException](timestampFormatter(pattern).parse(datetimeStr))
+      assert(e.getMessage.contains(expectedMsg))
+      // In lenient mode, the legacy TimestampFormatter does not fail
+      Seq(FAST_DATE_FORMAT, LENIENT_SIMPLE_DATE_FORMAT).foreach { ldf =>
+        val e = intercept[SparkUpgradeException] {
+          timestampFormatter(pattern, ldf).parse(datetimeStr)
+        }
+        assert(e.getCause.getMessage.contains(expectedMsg))
+      }
+    }
+  }
+
   test("explicitly forbidden datetime patterns") {
 
     Seq(true, false).foreach { isParsing =>
@@ -51,4 +97,36 @@ trait DatetimeFormatterSuite extends SparkFunSuite with SQLHelper with Matchers
       pattern => intercept[SparkUpgradeException](checkFormatterCreation(pattern, true))
     }
   }
+
+  test("SPARK-31939: Fix Parsing day of year when year field pattern is missing") {
+    // resolved to queryable LocaleDate or fail directly
+    assertEqual("yyyy-dd-DD", "2020-29-60", date(2020, 2, 29))
+    assertError("yyyy-dd-DD", "2020-02-60",
+      "Field DayOfMonth 29 differs from DayOfMonth 2 derived from 2020-02-29")
+    assertEqual("yyyy-MM-DD", "2020-02-60", date(2020, 2, 29))
+    assertError("yyyy-MM-DD", "2020-03-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 2020-02-29")
+    assertEqual("yyyy-MM-dd-DD", "2020-02-29-60", date(2020, 2, 29))
+    assertError("yyyy-MM-dd-DD", "2020-03-01-60",
+      "Field DayOfYear 61 differs from DayOfYear 60 derived from 2020-03-01")
+    assertEqual("yyyy-DDD", "2020-366", date(2020, 12, 31))
+    assertError("yyyy-DDD", "2019-366",
+      "Invalid date 'DayOfYear 366' as '2019' is not a leap year")
+
+    // unresolved and need to check manually(SPARK-31939 fixed)
+    assertEqual("DDD", "365", date(1970, 12, 31))
+    assertError("DDD", "366",
+      "Invalid date 'DayOfYear 366' as '1970' is not a leap year")
+    assertEqual("MM-DD", "03-60", date(1970, 3))
+    assertError("MM-DD", "02-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertEqual("MM-dd-DD", "02-28-59", date(1970, 2, 28))
+    assertError("MM-dd-DD", "02-28-60",
+      "Field MonthOfYear 2 differs from MonthOfYear 3 derived from 1970-03-01")
+    assertError("MM-dd-DD", "02-28-58",
+      "Field DayOfMonth 28 differs from DayOfMonth 27 derived from 1970-02-27")
+    assertEqual("dd-DD", "28-59", date(1970, 2, 28))
+    assertError("dd-DD", "27-59",
+      "Field DayOfMonth 27 differs from DayOfMonth 28 derived from 1970-02-28")
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index 88f6b0d79f85e..e7d935001a76f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -33,6 +33,8 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
     TimestampFormatter(pattern, UTC, isParsing)
   }
 
+  override protected def useDateFormatter: Boolean = false
+
   test("parsing timestamps using time zones") {
     val localDate = "2018-12-02T10:11:12.001234"
     val expectedMicros = Map(
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
new file mode 100644
index 0000000000000..0b313e5a0b9f4
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
@@ -0,0 +1,20 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS WITH INVALID VALUES ---
+
+-- parsing invalid values with pattern 'D'
+select to_timestamp('366', 'D');
+select to_timestamp('9', 'DD');
+-- in java 8 this case is invalid, but valid in java 11, disabled for jenkins
+-- select to_timestamp('100', 'DD');
+select to_timestamp('366', 'DD');
+select to_timestamp('9', 'DDD');
+select to_timestamp('99', 'DDD');
+select to_timestamp('30-365', 'dd-DDD');
+select to_timestamp('11-365', 'MM-DDD');
+select to_timestamp('2019-366', 'yyyy-DDD');
+select to_timestamp('12-30-365', 'MM-dd-DDD');
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD');
+-- add a special case to test csv, because the legacy formatter it uses is lenient then Spark should
+-- throw SparkUpgradeException
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
new file mode 100644
index 0000000000000..ee1afe502ab79
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-legacy.sql
@@ -0,0 +1,2 @@
+--SET spark.sql.legacy.timeParserPolicy=LEGACY
+--IMPORT datetime-parsing.sql
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
new file mode 100644
index 0000000000000..74866d9c6ffa1
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
@@ -0,0 +1,16 @@
+--- TESTS FOR DATETIME PARSING FUNCTIONS ---
+
+-- parsing with pattern 'D'
+select to_timestamp('9', 'D');
+select to_timestamp('300', 'D');
+select to_timestamp('09', 'DD');
+select to_timestamp('99', 'DD');
+select to_timestamp('009', 'DDD');
+select to_timestamp('365', 'DDD');
+select to_timestamp('31-365', 'dd-DDD');
+select to_timestamp('12-365', 'MM-DDD');
+select to_timestamp('2020-365', 'yyyy-DDD');
+select to_timestamp('12-31-365', 'MM-dd-DDD');
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD');
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD');
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD');
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
new file mode 100644
index 0000000000000..22f21c7b2073a
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
@@ -0,0 +1,110 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+select to_timestamp('366', 'D')
+-- !query schema
+struct<to_timestamp('366', 'D'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('366', 'DD')
+-- !query schema
+struct<to_timestamp('366', 'DD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('9', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '9' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('99', 'DDD')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '99' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('30-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp('30-365', 'dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('11-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp('11-365', 'MM-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2019-366', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp('2019-366', 'yyyy-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('12-30-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('12-30-365', 'MM-dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-01-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-01-365', 'yyyy-dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-10-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp('2020-10-350', 'yyyy-MM-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-11-31-366', 'yyyy-MM-dd-DDD'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select from_csv('2018-366', 'date Date', map('dateFormat', 'yyyy-DDD'))
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '2018-366' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
new file mode 100644
index 0000000000000..58443f1de38ca
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
@@ -0,0 +1,106 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp('9', 'D'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp('300', 'D'):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp('09', 'DD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp('99', 'DD'):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp('009', 'DDD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp('365', 'DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp('31-365', 'dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp('12-365', 'MM-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp('2020-365', 'yyyy-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('12-31-365', 'MM-dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-30-365', 'yyyy-dd-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-350', 'yyyy-MM-DDD'):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD'):timestamp>
+-- !query output
+2020-12-31 00:00:00
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
new file mode 100644
index 0000000000000..58443f1de38ca
--- /dev/null
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
@@ -0,0 +1,106 @@
+-- Automatically generated by SQLQueryTestSuite
+-- Number of queries: 13
+
+
+-- !query
+select to_timestamp('9', 'D')
+-- !query schema
+struct<to_timestamp('9', 'D'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('300', 'D')
+-- !query schema
+struct<to_timestamp('300', 'D'):timestamp>
+-- !query output
+1970-10-27 00:00:00
+
+
+-- !query
+select to_timestamp('09', 'DD')
+-- !query schema
+struct<to_timestamp('09', 'DD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'DD')
+-- !query schema
+struct<to_timestamp('99', 'DD'):timestamp>
+-- !query output
+1970-04-09 00:00:00
+
+
+-- !query
+select to_timestamp('009', 'DDD')
+-- !query schema
+struct<to_timestamp('009', 'DDD'):timestamp>
+-- !query output
+1970-01-09 00:00:00
+
+
+-- !query
+select to_timestamp('365', 'DDD')
+-- !query schema
+struct<to_timestamp('365', 'DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('31-365', 'dd-DDD')
+-- !query schema
+struct<to_timestamp('31-365', 'dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('12-365', 'MM-DDD')
+-- !query schema
+struct<to_timestamp('12-365', 'MM-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-365', 'yyyy-DDD')
+-- !query schema
+struct<to_timestamp('2020-365', 'yyyy-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('12-31-365', 'MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('12-31-365', 'MM-dd-DDD'):timestamp>
+-- !query output
+1970-12-31 00:00:00
+
+
+-- !query
+select to_timestamp('2020-30-365', 'yyyy-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-30-365', 'yyyy-dd-DDD'):timestamp>
+-- !query output
+2020-12-30 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-350', 'yyyy-MM-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-350', 'yyyy-MM-DDD'):timestamp>
+-- !query output
+2020-12-15 00:00:00
+
+
+-- !query
+select to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD')
+-- !query schema
+struct<to_timestamp('2020-12-31-366', 'yyyy-MM-dd-DDD'):timestamp>
+-- !query output
+2020-12-31 00:00:00

From 69ede588c9aa56e8d648c2837a537af138b479c7 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 12 Jun 2020 17:51:28 +0800
Subject: [PATCH 0822/1280] Revert "[SPARK-31860][BUILD] only push release tags
 on succes"

This reverts commit b814cbc8121c8d02a81efbae3e3cf1b6a6219262.
---
 dev/create-release/do-release.sh    | 14 +++-----------
 dev/create-release/release-build.sh |  7 ++-----
 dev/create-release/release-tag.sh   | 16 +++++++++++++---
 3 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/dev/create-release/do-release.sh b/dev/create-release/do-release.sh
index 64fba8a56affe..4f18a55a3bceb 100755
--- a/dev/create-release/do-release.sh
+++ b/dev/create-release/do-release.sh
@@ -17,8 +17,6 @@
 # limitations under the License.
 #
 
-set -e
-
 SELF=$(cd $(dirname $0) && pwd)
 . "$SELF/release-util.sh"
 
@@ -54,6 +52,9 @@ function should_build {
 if should_build "tag" && [ $SKIP_TAG = 0 ]; then
   run_silent "Creating release tag $RELEASE_TAG..." "tag.log" \
     "$SELF/release-tag.sh"
+  echo "It may take some time for the tag to be synchronized to github."
+  echo "Press enter when you've verified that the new tag ($RELEASE_TAG) is available."
+  read
 else
   echo "Skipping tag creation for $RELEASE_TAG."
 fi
@@ -78,12 +79,3 @@ if should_build "publish"; then
 else
   echo "Skipping publish step."
 fi
-
-if should_build "tag" && [ $SKIP_TAG = 0 ]; then
-  git push origin $RELEASE_TAG
-  if [[ $RELEASE_TAG != *"preview"* ]]; then
-    git push origin HEAD:$GIT_BRANCH
-  else
-    echo "It's preview release. We only push $RELEASE_TAG to remote."
-  fi
-fi
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 66c51845cc1d0..e3bcb72ab5c6f 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -92,12 +92,9 @@ BASE_DIR=$(pwd)
 init_java
 init_maven_sbt
 
-# Only clone repo fresh if not present, otherwise use checkout from the tag step
-if [ ! -d spark ]; then
-  git clone "$ASF_REPO"
-fi
+rm -rf spark
+git clone "$ASF_REPO"
 cd spark
-git fetch
 git checkout $GIT_REF
 git_hash=`git rev-parse --short HEAD`
 echo "Checked out Spark git hash $git_hash"
diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh
index e37aa27fc0aac..39856a9955955 100755
--- a/dev/create-release/release-tag.sh
+++ b/dev/create-release/release-tag.sh
@@ -25,7 +25,6 @@ function exit_with_usage {
   cat << EOF
 usage: $NAME
 Tags a Spark release on a particular branch.
-You must push the tags after.
 
 Inputs are specified with the following environment variables:
 ASF_USERNAME - Apache Username
@@ -106,8 +105,19 @@ sed -i".tmp7" 's/SPARK_VERSION_SHORT:.*$/SPARK_VERSION_SHORT: '"$R_NEXT_VERSION"
 
 git commit -a -m "Preparing development version $NEXT_VERSION"
 
-cd ..
-if is_dry_run; then
+if ! is_dry_run; then
+  # Push changes
+  git push origin $RELEASE_TAG
+  if [[ $RELEASE_VERSION != *"preview"* ]]; then
+    git push origin HEAD:$GIT_BRANCH
+  else
+    echo "It's preview release. We only push $RELEASE_TAG to remote."
+  fi
+
+  cd ..
+  rm -rf spark
+else
+  cd ..
   mv spark spark.tag
   echo "Clone with version changes and tag available as spark.tag in the output directory."
 fi

From 758c5b1b0528d8038662720ffd871b58f3e13609 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 12 Jun 2020 11:06:52 -0700
Subject: [PATCH 0823/1280] [SPARK-31979] Release script should not fail when
 remove non-existing files

### What changes were proposed in this pull request?

When removing non-existing files in the release script, do not fail.

### Why are the changes needed?

This is to make the release script more robust, as we don't care if the files exist before we remove them.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

tested when cutting 3.0.0 RC

Closes #28815 from cloud-fan/release.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 28f131fc8a4bbabb2d0bca3c46418b78c8ba3e1d)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/create-release/release-build.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index e3bcb72ab5c6f..eb972589a995e 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -164,7 +164,7 @@ fi
 DEST_DIR_NAME="$SPARK_PACKAGE_VERSION"
 
 git clean -d -f -x
-rm .gitignore
+rm -f .gitignore
 cd ..
 
 if [[ "$1" == "package" ]]; then
@@ -174,9 +174,9 @@ if [[ "$1" == "package" ]]; then
 
   # For source release in v2.4+, exclude copy of binary license/notice
   if [[ $SPARK_VERSION > "2.4" ]]; then
-    rm spark-$SPARK_VERSION/LICENSE-binary
-    rm spark-$SPARK_VERSION/NOTICE-binary
-    rm -r spark-$SPARK_VERSION/licenses-binary
+    rm -f spark-$SPARK_VERSION/LICENSE-binary
+    rm -f spark-$SPARK_VERSION/NOTICE-binary
+    rm -rf spark-$SPARK_VERSION/licenses-binary
   fi
 
   tar cvzf spark-$SPARK_VERSION.tgz --exclude spark-$SPARK_VERSION/.git spark-$SPARK_VERSION

From 2612bb80336212a6fb03417db40c27077935fd83 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Fri, 12 Jun 2020 17:27:45 -0700
Subject: [PATCH 0824/1280] [SPARK-31967][UI] Downgrade to vis.js 4.21.0 to fix
 Jobs UI loading time regression

### What changes were proposed in this pull request?

After #28192, the job list page becomes very slow.
For example, after the following operation, the UI loading can take >40 sec.
```
(1 to 1000).foreach(_ => sc.parallelize(1 to 10).collect)
```

This is caused by a  [performance issue of `vis-timeline`](https://github.com/visjs/vis-timeline/issues/379). The serious issue affects both branch-3.0 and branch-2.4

I tried a different version 4.21.0 from https://cdnjs.com/libraries/vis
The infinite drawing issue seems also fixed if the zoom is disabled as default.

### Why are the changes needed?

Fix the serious perf issue in web UI by falling back vis-timeline-graph2d to an ealier version.

### Does this PR introduce _any_ user-facing change?

Yes, fix the UI perf regression

### How was this patch tested?

Manual test

Closes #28811 from gengliangwang/downgrade-3.0.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../ui/static/vis-timeline-graph2d.min.css    |  3 +-
 .../static/vis-timeline-graph2d.min.css.map   |  1 -
 .../ui/static/vis-timeline-graph2d.min.js     | 70 ++++++-------------
 .../ui/static/vis-timeline-graph2d.min.js.map |  1 -
 dev/.rat-excludes                             |  2 -
 licenses-binary/LICENSE-vis-timeline.txt      | 29 ++++----
 licenses/LICENSE-vis-timeline.txt             | 29 ++++----
 7 files changed, 49 insertions(+), 86 deletions(-)
 delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
 delete mode 100644 core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map

diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
index d22a9ac801891..eeacbd802d305 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css
@@ -1,2 +1 @@
-.vis [class*=span]{min-height:0;width:auto}.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper:after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:transparent;width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2,#385380);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(180deg,#3876c2 0,#385380);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#3876c2",endColorstr="#385380",GradientType=0);box-shadow:0 0 1px 0 #111927;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:none}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(180deg,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#9d9d9d",endColorstr="#c8c8c8",GradientType=0)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:transparent;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0) rgba(136,183,213,0) rgba(136,183,213,0) rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0) rgba(194,225,245,0) rgba(194,225,245,0) #f2faff;border-width:12px;margin-top:-12px}div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}.vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content,.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\26F6"}.vis-rolling-mode-btn:hover{opacity:1}.vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-custom-time>.vis-custom-time-marker{background-color:inherit;color:#fff;font-size:12px;white-space:nowrap;padding:3px 5px;top:0;cursor:auto;z-index:inherit}.vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}.vis-loading-screen{width:100%;height:100%;position:absolute;top:0;left:0}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px hsla(0,0%,60.4%,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1{background:#f5f5f5}.vis-label.vis-nested-group.vis-group-level-0{background-color:#fff}.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-left:0}.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-right:0}.vis-label.vis-nested-group.vis-group-level-1{background-color:rgba(0,0,0,.05)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-left:15px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-right:15px}.vis-label.vis-nested-group.vis-group-level-2{background-color:rgba(0,0,0,.1)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-left:30px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-right:30px}.vis-label.vis-nested-group.vis-group-level-3{background-color:rgba(0,0,0,.15)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-left:45px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-right:45px}.vis-label.vis-nested-group.vis-group-level-4{background-color:rgba(0,0,0,.2)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-left:60px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-right:60px}.vis-label.vis-nested-group.vis-group-level-5{background-color:rgba(0,0,0,.25)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-left:75px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-right:75px}.vis-label.vis-nested-group.vis-group-level-6{background-color:rgba(0,0,0,.3)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-left:90px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-right:90px}.vis-label.vis-nested-group.vis-group-level-7{background-color:rgba(0,0,0,.35)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-left:105px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-right:105px}.vis-label.vis-nested-group.vis-group-level-8{background-color:rgba(0,0,0,.4)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-left:120px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-right:120px}.vis-label.vis-nested-group.vis-group-level-9{background-color:rgba(0,0,0,.45)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-left:135px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-right:135px}.vis-label.vis-nested-group{background-color:rgba(0,0,0,.5)}.vis-ltr .vis-label.vis-nested-group .vis-inner{padding-left:150px}.vis-rtl .vis-label.vis-nested-group .vis-inner{padding-right:150px}.vis-group-level-unknown-but-gte1{border:1px solid red}.vis-label.vis-nesting-group:before{display:inline-block;width:15px}.vis-label.vis-nesting-group.expanded:before{content:"\25BC"}.vis-label.vis-nesting-group.collapsed:before{content:"\25B6"}.vis-rtl .vis-label.vis-nesting-group.collapsed:before{content:"\25C0"}.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-left:15px}.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-right:15px}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-labelset{overflow:hidden}.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-group-is-dragging{background:rgba(0,0,0,.1)}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:none}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-left,.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0}.vis-item.vis-range .vis-drag-right{right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-item.vis-cluster{vertical-align:center;text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-cluster-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item.vis-cluster-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}
-/*# sourceMappingURL=vis-timeline-graph2d.min.css.map */
\ No newline at end of file
+.vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}.vis [class*=span]{min-height:0;width:auto}div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper::after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:rgba(0,0,0,0);width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2 0,#385380 100%);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(to bottom,#3876c2 0,#385380 100%);box-shadow:#111927 0 0 1px 0;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:0}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(to bottom,#9d9d9d 0,#c8c8c8 99%)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(to bottom,#dedede 0,#c8c8c8 99%);border:1px solid #999;box-shadow:#aaa 0 0 3px 0;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:0 0;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0);border-left-color:rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0);border-left-color:#f2faff;border-width:12px;margin-top:-12px}div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\26F6"}.vis-rolling-mode-btn:hover{opacity:1}.vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px rgba(154,154,154,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:0 0}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0;right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-nested-group{background:#f5f5f5}.vis-label.vis-nesting-group.expanded:before{content:"\25BC"}.vis-label.vis-nesting-group.collapsed-rtl:before{content:"\25C0"}.vis-label.vis-nesting-group.collapsed:before{content:"\25B6"}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-labelset{position:relative;overflow:hidden;box-sizing:border-box}.vis-labelset .vis-label{position:relative;left:0;top:0;width:100%;color:#4d4d4d;box-sizing:border-box}.vis-labelset .vis-label{border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}.vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}.vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}.vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}.vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
deleted file mode 100644
index 3c165a792d0f3..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.css.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"sources":["bootstrap.css","activator.css","configuration.css","tooltip.css","panel.css","currenttime.css","customtime.css","timeline.css","pathStyles.css","dataaxis.css","itemset.css","labelset.css","timeaxis.css","item.css"],"names":[],"mappings":"AAEA,mBACE,YAAa,CACb,UACF,CCLA,cACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WAAY,CAGZ,UACF,CAEA,YACE,2BACF,CCbA,sBACI,iBAAiB,CACjB,aAAa,CACb,UAAU,CACV,cACJ,CAEA,8BACI,aAAa,CACb,WACJ,CAEA,oCACE,UAAW,CACX,UAAW,CACX,aACF,CAEA,kDACI,aAAa,CACb,WAAW,CACX,qBAAyB,CACzB,wBAAwB,CACxB,iBAAiB,CACjB,eAAe,CACf,SAAS,CACT,gBACJ,CAEA,wCACI,aAAa,CACb,WAAW,CACX,WAAW,CACX,qBAAsB,CACtB,gBAAgB,CAChB,wBAAyB,CACzB,wBAAwB,CACxB,iBAAiB,CACjB,eAAe,CACf,SAAS,CACT,gBAAgB,CAChB,cAAe,CACf,kBACJ,CAEA,8CACI,wBAAyB,CACzB,wBAAwB,CACxB,UACJ,CAEA,sCACI,aAAa,CACb,UAAU,CACV,WAAW,CACX,WAAW,CACX,qBAAsB,CACtB,gBACJ,CAGA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CACA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CACA,oDACI,SAAS,CACT,wBAAyB,CACzB,gBAAgB,CAChB,iBACJ,CAEA,wCACI,cAAc,CACd,eACJ,CAEA,uCACI,WAAW,CACX,WAAW,CACX,gBACJ,CAEA,qDACI,WACJ,CACA,qDACI,WACJ,CAEA,4CACI,OAAO,CACP,UAAU,CACV,WAAW,CACX,qBAAwB,CACxB,iBAAiB,CACjB,SAAW,CACX,QAAU,CACV,cACJ,CAEA,4CACI,SACJ,CAGA,8CACI,iBAAiB,CACjB,QAAQ,CACR,UAAU,CAEV,WAAW,CACX,QAAQ,CACR,mBACJ,CAEA,yCAEI,uBAAwB,CAGxB,mBAAuB,CACvB,4BAA8B,CAG9B,WAAY,CACZ,WACJ,CACA,wEACI,WAAY,CACZ,UAAW,CACX,kBAAmB,CACnB,0DAA+D,CAC/D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GAAmH,CAEnH,qBAAyB,CACzB,yBAAmC,CACnC,iBACJ,CACA,+DACI,uBAAwB,CACxB,wBAAyB,CACzB,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,kBAAmB,CACnB,2DAAgE,CAChE,uGAA6G,CAC7G,uDAAkE,CAClE,yDAA6D,CAC7D,0DAA8D,CAC9D,oDAAgE,CAChE,+GAAmH,CACnH,4BAAmC,CACnC,eACJ,CACA,+CACI,YACJ,CACA,8EACI,kBAAmB,CACnB,0DAA8D,CAC9D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GACJ,CAEA,2DACI,WAAY,CACZ,WAAY,CACZ,kBAAmB,CACnB,0DAA+D,CAC/D,sGAA4G,CAC5G,2DAAiE,CACjE,wDAA4D,CAC5D,yDAA6D,CAC7D,wDAA+D,CAC/D,+GAAmH,CAEnH,qBAAyB,CACzB,yBAAmC,CACnC,iBACJ,CACA,2DACI,WAAY,CACZ,WAAY,CACZ,UAAW,CAEX,iBAAkB,CAClB,kBACJ,CAGA,wDACI,sBAAwB,CACxB,mBACJ,CAEA,oDACI,WAAY,CACZ,UAAW,CAGX,sBAAuB,CAGvB,wBAAyB,CACzB,kBAAmB,CAGnB,iBACJ,CACA,yDACI,eAAgB,CAChB,kBACJ,CACA,yDACI,eAAgB,CAChB,kBACJ,CACA,oDACI,WAAY,CACZ,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,kBACJ,CACA,+DACI,eACJ,CACA,+DACI,eACJ,CAEA,yBACI,iBAAkB,CAClB,6BAAkC,CAClC,wBAAyB,CACzB,gBAAgB,CAChB,WAAW,CACX,WAAW,CACX,iBAAiB,CACjB,UAAc,CACd,cAAc,CACd,iBAAiB,CACjB,0CAA4C,CAC5C,uCAAyC,CACzC,kCACJ,CACA,+DACI,SAAU,CACV,OAAQ,CACR,wBAAyB,CACzB,WAAY,CACZ,QAAS,CACT,OAAQ,CACR,iBAAkB,CAClB,mBACJ,CAEA,+BAEI,2FAAyC,CACzC,gBAAiB,CACjB,eACJ,CACA,gCAEI,gFAA0B,CAC1B,iBAAkB,CAClB,gBACJ,CC/RA,gBACE,iBAAkB,CAClB,iBAAkB,CAClB,WAAY,CACZ,kBAAmB,CAEnB,mBAAoB,CACpB,cAAc,CACd,UAAa,CACb,wBAAyB,CAEzB,sBAAuB,CACvB,yBAA0B,CAC1B,iBAAkB,CAClB,wBAAyB,CAEzB,sCAA2C,CAC3C,mBAAoB,CAEpB,SACF,CCpBA,WACE,iBAAkB,CAElB,SAAU,CACV,QAAS,CAET,qBACF,CAEA,wGAKE,kBACF,CAEA,+DAGE,sBAAuB,CACvB,yBAA0B,CAC1B,eACF,CAEA,iFACE,WAAY,CACZ,iBAAkB,CAClB,iBACF,CAEA,wCACE,aACF,CAMA,8FACE,aACF,CAEA,sDACE,aACF,CAEA,+DAGE,uBAAwB,CACxB,wBACF,CAEA,gBACE,eACF,CAEA,wBACE,iBACF,CAEA,uBACE,iBAAkB,CAClB,UAAW,CACX,UAAW,CACX,kCAIF,CAEA,+BACE,QAAS,CACT,MACF,CAEA,kCACE,WAAY,CACZ,MACF,CChFA,kBACE,wBAAyB,CACzB,SAAU,CACV,SAAU,CACV,mBACF,CAEA,sBACE,WAAY,CACZ,UAAW,CACX,iBAAkB,CAClB,OAAQ,CACR,UAAW,CACX,iBAAkB,CAClB,cAAe,CACf,cAAe,CACf,UAAY,CACZ,UAAY,CACZ,eAAiB,CACjB,iBAAkB,CAClB,kBACF,CACA,6BACE,eACF,CAEA,4BACE,SACF,CC5BA,iBACE,wBAAyB,CACzB,SAAU,CACV,WAAY,CACZ,SACF,CAEA,yCACE,wBAAyB,CACzB,UAAY,CACZ,cAAe,CACf,kBAAmB,CACnB,eAAgB,CAChB,KAAQ,CACR,WAAe,CACf,eACF,CCfA,cACE,iBAAkB,CAClB,wBAAyB,CACzB,eAAgB,CAChB,SAAU,CACV,QAAS,CACT,qBACF,CAEA,oBACE,UAAW,CACX,WAAY,CACZ,iBAAkB,CAClB,KAAM,CACN,MACF,CChBA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,YAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,kBACI,SAAY,CACZ,cAAc,CACd,gBAAgB,CAChB,WACJ,CAEA,kBACI,YAAa,CACb,cAAc,CACd,gBAAgB,CAChB,cACJ,CAEA,wBACI,eAAgB,CAChB,WACJ,CAGA,uBACI,eAAgB,CAChB,gBACJ,CAEA,yBACI,gBAAgB,CAChB,cACJ,CAGA,qCACI,gBAAgB,CAChB,eAAgB,CAChB,SAAa,CACb,cACJ,CAGA,2BACI,gBAAgB,CAChB,cAAc,CACd,SAAa,CACb,cACJ,CAEA,6BACI,eAAgB,CAChB,WACJ,CCxGA,kEACE,iBAAkB,CAClB,UAAW,CACX,QAAS,CACT,uBACF,CAEA,6DACE,oBACF,CAEA,6DACE,oBACF,CAGA,qCACE,UAAW,CACX,iBAAkB,CAClB,aAAc,CACd,kBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,QAAS,CACT,iBAAkB,CAClB,UACF,CAGA,qCACE,iBAAkB,CAClB,UAAW,CACX,aAAc,CACd,kBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,QAAS,CACT,iBAAkB,CAClB,UACF,CAEA,qCACE,iBAAkB,CAClB,aAAc,CACd,kBAAmB,CACnB,WAAY,CACZ,iBACF,CAEA,iDACE,SAAU,CACV,QAAS,CACT,iBAAkB,CAClB,UACF,CAEA,8CACE,QAAS,CACT,iCAAkC,CAClC,8BAA+B,CAC/B,6BAA8B,CAC9B,4BAA6B,CAC7B,4BAA6B,CAC7B,gCAAiC,CACjC,6BAA8B,CAC9B,4BAA6B,CAC7B,2BAA4B,CAC5B,wBACF,CAEA,+CACE,QAAS,CACT,qCAAsC,CACtC,kCAAmC,CACnC,iCAAkC,CAClC,gCAAiC,CACjC,6BAA8B,CAC9B,+BAAgC,CAChC,4BAA6B,CAC7B,2BAA4B,CAC5B,0BAA2B,CAC3B,uBACF,CAEA,YACE,sCAA2C,CAC3C,WAAY,CACZ,wBAAyB,CACzB,4CACF,CAEA,iBAEE,kBAAmB,CACnB,oBACF,CCrGA,aACE,iBAAkB,CAClB,SAAU,CACV,QAAS,CAET,qBACF,CAEA,0DAEE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,gBACF,CAEA,UACE,iBAAkB,CAClB,UAAW,CACX,QAAS,CACT,MAAO,CACP,SACF,CAEA,2BACE,iBAAkB,CAClB,qBAAsB,CACtB,+BACF,CAEA,sCACE,kBACF,CAEA,mBACE,cACF,CAEA,6DACE,kBACF,CACA,8CACE,qBACF,CACA,kEACE,cACF,CACA,kEACE,eACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,+BACF,CACA,kEACE,iBACF,CACA,kEACE,kBACF,CACA,8CACE,gCACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CACA,8CACE,+BACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CACA,8CACE,gCACF,CACA,kEACE,kBACF,CACA,kEACE,mBACF,CAGA,4BACE,+BACF,CACA,gDACE,kBACF,CACA,gDACE,mBACF,CAEA,kCACE,oBACF,CAGA,oCAEE,oBAAqB,CACrB,UACF,CACA,6CACE,eACF,CACA,8CACE,eACF,CACA,uDACE,eACF,CAEA,oEACE,iBACF,CACA,oEACE,kBACF,CAEA,aACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WAAY,CACZ,UACF,CCjLA,cAGE,eAGF,CAEA,uCAPE,iBAAkB,CAIlB,qBAWF,CARA,yBAEE,MAAO,CACP,KAAM,CACN,UAAW,CACX,aAAc,CAMd,+BAHF,CAMA,mCACE,cACF,CAEA,uBACE,yBACF,CAEA,oCACE,kBACF,CAEA,oCACE,oBAAqB,CACrB,WACF,CAEA,+CACE,SACF,CC1CA,eACE,iBAAkB,CAClB,eACF,CAEA,8BACE,KAAM,CACN,MAAO,CACP,UACF,CAEA,8BACE,iBAAkB,CAClB,KAAM,CACN,MAAO,CACP,UAAW,CACX,WACF,CAEA,yBACE,iBAAkB,CAClB,aAAc,CACd,WAAY,CACZ,eAAgB,CAChB,qBAAsB,CAEtB,kBACF,CAEA,qCACE,iBAAkB,CAClB,cAAe,CACf,eAAgB,CAChB,aAAc,CACd,cAAe,CACf,iBACF,CAEA,sCACE,iBAAkB,CAClB,qBACF,CAEA,0CACE,iBAAkB,CAClB,sBACF,CAEA,mCACE,oBACF,CAEA,mCACE,oBACF,CCrDA,UACE,iBAAkB,CAClB,aAAc,CACd,oBAAqB,CACrB,gBAAiB,CACjB,wBAAyB,CACzB,oBAAqB,CACrB,SAEF,CAEA,uBACE,oBAAqB,CACrB,wBAAyB,CAGzB,SACF,CAEA,2BACE,WACF,CAEA,iCACE,wBACF,CAEA,kBACE,iBAAkB,CAClB,kBAAmB,CACnB,iBACF,CAEA,oBACE,eACF,CAEA,kBACE,iBAAkB,CAClB,SAAU,CACV,gBAAiB,CACjB,kBAAmB,CACnB,iBACF,CAEA,oBACE,kBAAmB,CACnB,iBAAkB,CAClB,qBACF,CAEA,yBACE,WAAY,CACZ,qCAA0C,CAC1C,qBAAsB,CACtB,SAAU,CACV,QACF,CAEA,6BACE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,SAAU,CACV,QAAS,CACT,eACF,CAEA,wBACE,kBACF,CAEA,sCACE,iBAAkB,CAClB,oBACF,CAEA,2CACE,iBAAkB,CAClB,oBACF,CAEA,mBACE,SAAU,CACV,iBAAkB,CAClB,OAAQ,CACR,qBAAsB,CACtB,uBACF,CAEA,4BACE,kBAAmB,CACnB,qBAAsB,CACtB,WACF,CAEA,oCACE,iBAAkB,CAClB,kBAAmB,CACnB,UAAY,CACZ,WAAY,CACZ,iBAAkB,CAClB,kBAAmB,CACnB,WAAY,CACZ,iBAAkB,CAClB,cAAgB,CAChB,iBAAmB,CACnB,mBAAqB,CACrB,sBACF,CAEA,gDACE,iBAAkB,CAClB,KAAQ,CACR,UAAW,CACX,WAAY,CACZ,qBAAsB,CACtB,aAAgB,CAChB,cAAe,CAEf,wCAA0C,CAC1C,qCAAuC,CACvC,oCAAsC,CACtC,mCAAqC,CACrC,gCACF,CAEA,sBACE,WACF,CAEA,0BACE,UACF,CAEA,4DACE,eAAgB,CAChB,SAAU,CACV,4BAA8B,CAC9B,cAAe,CACf,eAAiB,CAEjB,mCAAqC,CACrC,gCAAkC,CAClC,+BAAiC,CACjC,8BAAgC,CAChC,2BACF,CAEA,4DACE,cACF,CAEA,wEACE,UACF,CAEA,2BACE,iBAAkB,CAClB,UAAW,CACX,WAAY,CACZ,KAAM,CACN,MAAS,CACT,WACF,CAEA,mCAOE,SAAU,CAEV,eACF,CAEA,uEAXE,iBAAkB,CAClB,UAAW,CACX,aAAc,CACd,aAAc,CACd,WAAY,CACZ,KAgBF,CAVA,oCAOE,UAAW,CAEX,eACF,CAEA,iGAEE,WACF,CAEA,sBACE,qBAAsB,CACtB,iBAAkB,CAClB,kBAAmB,CACnB,iBACF,CAEA,2BACE,SAAU,CACV,iBAAkB,CAClB,OAAQ,CACR,qBAAsB,CACtB,uBACF,CAEA,0BACE,iBAAkB,CAClB,SAAU,CACV,gBAAiB,CACjB,kBAAmB,CACnB,iBACF","file":"vis-timeline-graph2d.min.css","sourcesContent":["/* override some bootstrap styles screwing up the timelines css */\n\n.vis [class*=\"span\"] {\n  min-height: 0;\n  width: auto;\n}\n",".vis .overlay {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n\n  /* Must be displayed above for example selected Timeline items */\n  z-index: 10;\n}\n\n.vis-active {\n  box-shadow: 0 0 10px #86d5f8;\n}\n","div.vis-configuration {\n    position:relative;\n    display:block;\n    float:left;\n    font-size:12px;\n}\n\ndiv.vis-configuration-wrapper {\n    display:block;\n    width:700px;\n}\n\ndiv.vis-configuration-wrapper::after {\n  clear: both;\n  content: \"\";\n  display: block;\n}\n\ndiv.vis-configuration.vis-config-option-container{\n    display:block;\n    width:495px;\n    background-color: #ffffff;\n    border:2px solid #f7f8fa;\n    border-radius:4px;\n    margin-top:20px;\n    left:10px;\n    padding-left:5px;\n}\n\ndiv.vis-configuration.vis-config-button{\n    display:block;\n    width:495px;\n    height:25px;\n    vertical-align: middle;\n    line-height:25px;\n    background-color: #f7f8fa;\n    border:2px solid #ceced0;\n    border-radius:4px;\n    margin-top:20px;\n    left:10px;\n    padding-left:5px;\n    cursor: pointer;\n    margin-bottom:30px;\n}\n\ndiv.vis-configuration.vis-config-button.hover{\n    background-color: #4588e6;\n    border:2px solid #214373;\n    color:#ffffff;\n}\n\ndiv.vis-configuration.vis-config-item{\n    display:block;\n    float:left;\n    width:495px;\n    height:25px;\n    vertical-align: middle;\n    line-height:25px;\n}\n\n\ndiv.vis-configuration.vis-config-item.vis-config-s2{\n    left:10px;\n    background-color: #f7f8fa;\n    padding-left:5px;\n    border-radius:3px;\n}\ndiv.vis-configuration.vis-config-item.vis-config-s3{\n    left:20px;\n    background-color: #e4e9f0;\n    padding-left:5px;\n    border-radius:3px;\n}\ndiv.vis-configuration.vis-config-item.vis-config-s4{\n    left:30px;\n    background-color: #cfd8e6;\n    padding-left:5px;\n    border-radius:3px;\n}\n\ndiv.vis-configuration.vis-config-header{\n    font-size:18px;\n    font-weight: bold;\n}\n\ndiv.vis-configuration.vis-config-label{\n    width:120px;\n    height:25px;\n    line-height: 25px;\n}\n\ndiv.vis-configuration.vis-config-label.vis-config-s3{\n    width:110px;\n}\ndiv.vis-configuration.vis-config-label.vis-config-s4{\n    width:100px;\n}\n\ndiv.vis-configuration.vis-config-colorBlock{\n    top:1px;\n    width:30px;\n    height:19px;\n    border:1px solid #444444;\n    border-radius:2px;\n    padding:0px;\n    margin:0px;\n    cursor:pointer;\n}\n\ninput.vis-configuration.vis-config-checkbox {\n    left:-5px;\n}\n\n\ninput.vis-configuration.vis-config-rangeinput{\n    position:relative;\n    top:-5px;\n    width:60px;\n    /*height:13px;*/\n    padding:1px;\n    margin:0;\n    pointer-events:none;\n}\n\ninput.vis-configuration.vis-config-range{\n    /*removes default webkit styles*/\n    -webkit-appearance: none;\n\n    /*fix for FF unable to apply focus style bug */\n    border: 0px solid white;\n    background-color:rgba(0,0,0,0);\n\n    /*required for proper track sizing in FF*/\n    width: 300px;\n    height:20px;\n}\ninput.vis-configuration.vis-config-range::-webkit-slider-runnable-track {\n    width: 300px;\n    height: 5px;\n    background: #dedede; /* Old browsers */\n    background: -moz-linear-gradient(top,  #dedede 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#dedede), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top, #dedede 0%, #c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #dedede 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n\n    border: 1px solid #999999;\n    box-shadow: #aaaaaa 0px 0px 3px 0px;\n    border-radius: 3px;\n}\ninput.vis-configuration.vis-config-range::-webkit-slider-thumb {\n    -webkit-appearance: none;\n    border: 1px solid #14334b;\n    height: 17px;\n    width: 17px;\n    border-radius: 50%;\n    background: #3876c2; /* Old browsers */\n    background: -moz-linear-gradient(top,  #3876c2 0%, #385380 100%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#3876c2), color-stop(100%,#385380)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #3876c2 0%,#385380 100%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top,  #3876c2 0%,#385380 100%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #3876c2 0%,#385380 100%); /* IE10+ */\n    background: linear-gradient(to bottom,  #3876c2 0%,#385380 100%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#3876c2', endColorstr='#385380',GradientType=0 ); /* IE6-9 */\n    box-shadow: #111927 0px 0px 1px 0px;\n    margin-top: -7px;\n}\ninput.vis-configuration.vis-config-range:focus {\n    outline: none;\n}\ninput.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track {\n    background: #9d9d9d; /* Old browsers */\n    background: -moz-linear-gradient(top, #9d9d9d 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#9d9d9d), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #9d9d9d 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #9d9d9d 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#9d9d9d', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n}\n\ninput.vis-configuration.vis-config-range::-moz-range-track {\n    width: 300px;\n    height: 10px;\n    background: #dedede; /* Old browsers */\n    background: -moz-linear-gradient(top,  #dedede 0%, #c8c8c8 99%); /* FF3.6+ */\n    background: -webkit-gradient(linear, left top, left bottom, color-stop(0%,#dedede), color-stop(99%,#c8c8c8)); /* Chrome,Safari4+ */\n    background: -webkit-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* Chrome10+,Safari5.1+ */\n    background: -o-linear-gradient(top, #dedede 0%, #c8c8c8 99%); /* Opera 11.10+ */\n    background: -ms-linear-gradient(top,  #dedede 0%,#c8c8c8 99%); /* IE10+ */\n    background: linear-gradient(to bottom,  #dedede 0%,#c8c8c8 99%); /* W3C */\n    filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#dedede', endColorstr='#c8c8c8',GradientType=0 ); /* IE6-9 */\n\n    border: 1px solid #999999;\n    box-shadow: #aaaaaa 0px 0px 3px 0px;\n    border-radius: 3px;\n}\ninput.vis-configuration.vis-config-range::-moz-range-thumb {\n    border: none;\n    height: 16px;\n    width: 16px;\n\n    border-radius: 50%;\n    background:  #385380;\n}\n\n/*hide the outline behind the border*/\ninput.vis-configuration.vis-config-range:-moz-focusring{\n    outline: 1px solid white;\n    outline-offset: -1px;\n}\n\ninput.vis-configuration.vis-config-range::-ms-track {\n    width: 300px;\n    height: 5px;\n\n    /*remove bg colour from the track, we'll use ms-fill-lower and ms-fill-upper instead */\n    background: transparent;\n\n    /*leave room for the larger thumb to overflow with a transparent border */\n    border-color: transparent;\n    border-width: 6px 0;\n\n    /*remove default tick marks*/\n    color: transparent;\n}\ninput.vis-configuration.vis-config-range::-ms-fill-lower {\n    background: #777;\n    border-radius: 10px;\n}\ninput.vis-configuration.vis-config-range::-ms-fill-upper {\n    background: #ddd;\n    border-radius: 10px;\n}\ninput.vis-configuration.vis-config-range::-ms-thumb {\n    border: none;\n    height: 16px;\n    width: 16px;\n    border-radius: 50%;\n    background:  #385380;\n}\ninput.vis-configuration.vis-config-range:focus::-ms-fill-lower {\n    background: #888;\n}\ninput.vis-configuration.vis-config-range:focus::-ms-fill-upper {\n    background: #ccc;\n}\n\n.vis-configuration-popup {\n    position: absolute;\n    background: rgba(57, 76, 89, 0.85);\n    border: 2px solid #f2faff;\n    line-height:30px;\n    height:30px;\n    width:150px;\n    text-align:center;\n    color: #ffffff;\n    font-size:14px;\n    border-radius:4px;\n    -webkit-transition: opacity 0.3s ease-in-out;\n    -moz-transition: opacity 0.3s ease-in-out;\n    transition: opacity 0.3s ease-in-out;\n}\n.vis-configuration-popup:after, .vis-configuration-popup:before {\n    left: 100%;\n    top: 50%;\n    border: solid transparent;\n    content: \" \";\n    height: 0;\n    width: 0;\n    position: absolute;\n    pointer-events: none;\n}\n\n.vis-configuration-popup:after {\n    border-color: rgba(136, 183, 213, 0);\n    border-left-color: rgba(57, 76, 89, 0.85);\n    border-width: 8px;\n    margin-top: -8px;\n}\n.vis-configuration-popup:before {\n    border-color: rgba(194, 225, 245, 0);\n    border-left-color: #f2faff;\n    border-width: 12px;\n    margin-top: -12px;\n}","div.vis-tooltip {\n  position: absolute;\n  visibility: hidden;\n  padding: 5px;\n  white-space: nowrap;\n\n  font-family: verdana;\n  font-size:14px;\n  color:#000000;\n  background-color: #f5f4ed;\n\n  -moz-border-radius: 3px;\n  -webkit-border-radius: 3px;\n  border-radius: 3px;\n  border: 1px solid #808074;\n\n  box-shadow: 3px 3px 10px rgba(0, 0, 0, 0.2);\n  pointer-events: none;\n\n  z-index: 5;\n}\n",".vis-panel {\n  position: absolute;\n\n  padding: 0;\n  margin: 0;\n\n  box-sizing: border-box;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-left,\n.vis-panel.vis-right,\n.vis-panel.vis-top,\n.vis-panel.vis-bottom {\n  border: 1px #bfbfbf;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-left,\n.vis-panel.vis-right {\n  border-top-style: solid;\n  border-bottom-style: solid;\n  overflow: hidden;\n}\n\n.vis-left.vis-panel.vis-vertical-scroll, .vis-right.vis-panel.vis-vertical-scroll {\n  height: 100%;\n  overflow-x: hidden;\n  overflow-y: scroll;\n} \n\n.vis-left.vis-panel.vis-vertical-scroll {\n  direction: rtl;\n}\n\n.vis-left.vis-panel.vis-vertical-scroll .vis-content {\n  direction: ltr;\n}\n\n.vis-right.vis-panel.vis-vertical-scroll {\n  direction: ltr;\n}\n\n.vis-right.vis-panel.vis-vertical-scroll .vis-content {\n  direction: rtl;\n}\n\n.vis-panel.vis-center,\n.vis-panel.vis-top,\n.vis-panel.vis-bottom {\n  border-left-style: solid;\n  border-right-style: solid;\n}\n\n.vis-background {\n  overflow: hidden;\n}\n\n.vis-panel > .vis-content {\n  position: relative;\n}\n\n.vis-panel .vis-shadow {\n  position: absolute;\n  width: 100%;\n  height: 1px;\n  box-shadow: 0 0 10px rgba(0,0,0,0.8);\n  /* TODO: find a nice way to ensure vis-shadows are drawn on top of items\n  z-index: 1;\n  */\n}\n\n.vis-panel .vis-shadow.vis-top {\n  top: -1px;\n  left: 0;\n}\n\n.vis-panel .vis-shadow.vis-bottom {\n  bottom: -1px;\n  left: 0;\n}",".vis-current-time {\n  background-color: #FF7F6E;\n  width: 2px;\n  z-index: 1;\n  pointer-events: none;\n}\n\n.vis-rolling-mode-btn {\n  height: 40px;\n  width: 40px;\n  position: absolute;\n  top: 7px;\n  right: 20px;\n  border-radius: 50%;\n  font-size: 28px;\n  cursor: pointer;\n  opacity: 0.8;\n  color: white;\n  font-weight: bold;\n  text-align: center;\n  background: #3876c2;\n}\n.vis-rolling-mode-btn:before {\n  content: \"\\26F6\";\n}\n\n.vis-rolling-mode-btn:hover {\n  opacity: 1;\n}",".vis-custom-time {\n  background-color: #6E94FF;\n  width: 2px;\n  cursor: move;\n  z-index: 1;\n}\n\n.vis-custom-time > .vis-custom-time-marker {\n  background-color: inherit;\n  color: white;\n  font-size: 12px;\n  white-space: nowrap;\n  padding: 3px 5px;\n  top: 0px;\n  cursor: initial;\n  z-index: inherit;\n}","\n.vis-timeline {\n  position: relative;\n  border: 1px solid #bfbfbf;\n  overflow: hidden;\n  padding: 0;\n  margin: 0;\n  box-sizing: border-box;\n}\n\n.vis-loading-screen {\n  width: 100%;\n  height: 100%;\n  position: absolute;\n  top: 0;\n  left: 0;\n}",".vis-graph-group0 {\n    fill:#4f81bd;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #4f81bd;\n}\n\n.vis-graph-group1 {\n    fill:#f79646;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #f79646;\n}\n\n.vis-graph-group2 {\n    fill: #8c51cf;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #8c51cf;\n}\n\n.vis-graph-group3 {\n    fill: #75c841;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #75c841;\n}\n\n.vis-graph-group4 {\n    fill: #ff0100;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #ff0100;\n}\n\n.vis-graph-group5 {\n    fill: #37d8e6;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #37d8e6;\n}\n\n.vis-graph-group6 {\n    fill: #042662;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #042662;\n}\n\n.vis-graph-group7 {\n    fill:#00ff26;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #00ff26;\n}\n\n.vis-graph-group8 {\n    fill:#ff00ff;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #ff00ff;\n}\n\n.vis-graph-group9 {\n    fill: #8f3938;\n    fill-opacity:0;\n    stroke-width:2px;\n    stroke: #8f3938;\n}\n\n.vis-timeline .vis-fill {\n    fill-opacity:0.1;\n    stroke: none;\n}\n\n\n.vis-timeline .vis-bar {\n    fill-opacity:0.5;\n    stroke-width:1px;\n}\n\n.vis-timeline .vis-point {\n    stroke-width:2px;\n    fill-opacity:1.0;\n}\n\n\n.vis-timeline .vis-legend-background {\n    stroke-width:1px;\n    fill-opacity:0.9;\n    fill: #ffffff;\n    stroke: #c2c2c2;\n}\n\n\n.vis-timeline .vis-outline {\n    stroke-width:1px;\n    fill-opacity:1;\n    fill: #ffffff;\n    stroke: #e5e5e5;\n}\n\n.vis-timeline .vis-icon-fill {\n    fill-opacity:0.3;\n    stroke: none;\n}\n","\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal {\n  position: absolute;\n  width: 100%;\n  height: 0;\n  border-bottom: 1px solid;\n}\n\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor {\n  border-color: #e5e5e5;\n}\n\n.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major {\n  border-color: #bfbfbf;\n}\n\n\n.vis-data-axis .vis-y-axis.vis-major {\n  width: 100%;\n  position: absolute;\n  color: #4d4d4d;\n  white-space: nowrap;\n}\n\n.vis-data-axis .vis-y-axis.vis-major.vis-measure {\n  padding: 0;\n  margin: 0;\n  border: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n\n.vis-data-axis .vis-y-axis.vis-minor {\n  position: absolute;\n  width: 100%;\n  color: #bebebe;\n  white-space: nowrap;\n}\n\n.vis-data-axis .vis-y-axis.vis-minor.vis-measure {\n  padding: 0;\n  margin: 0;\n  border: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n.vis-data-axis .vis-y-axis.vis-title {\n  position: absolute;\n  color: #4d4d4d;\n  white-space: nowrap;\n  bottom: 20px;\n  text-align: center;\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-measure {\n  padding: 0;\n  margin: 0;\n  visibility: hidden;\n  width: auto;\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-left {\n  bottom: 0;\n  -webkit-transform-origin: left top;\n  -moz-transform-origin: left top;\n  -ms-transform-origin: left top;\n  -o-transform-origin: left top;\n  transform-origin: left bottom;\n  -webkit-transform: rotate(-90deg);\n  -moz-transform: rotate(-90deg);\n  -ms-transform: rotate(-90deg);\n  -o-transform: rotate(-90deg);\n  transform: rotate(-90deg);\n}\n\n.vis-data-axis .vis-y-axis.vis-title.vis-right {\n  bottom: 0;\n  -webkit-transform-origin: right bottom;\n  -moz-transform-origin: right bottom;\n  -ms-transform-origin: right bottom;\n  -o-transform-origin: right bottom;\n  transform-origin: right bottom;\n  -webkit-transform: rotate(90deg);\n  -moz-transform: rotate(90deg);\n  -ms-transform: rotate(90deg);\n  -o-transform: rotate(90deg);\n  transform: rotate(90deg);\n}\n\n.vis-legend {\n  background-color: rgba(247, 252, 255, 0.65);\n  padding: 5px;\n  border: 1px solid #b3b3b3;\n  box-shadow: 2px 2px 10px rgba(154, 154, 154, 0.55);\n}\n\n.vis-legend-text {\n  /*font-size: 10px;*/\n  white-space: nowrap;\n  display: inline-block\n}","\n.vis-itemset {\n  position: relative;\n  padding: 0;\n  margin: 0;\n\n  box-sizing: border-box;\n}\n\n.vis-itemset .vis-background,\n.vis-itemset .vis-foreground {\n  position: absolute;\n  width: 100%;\n  height: 100%;\n  overflow: visible;\n}\n\n.vis-axis {\n  position: absolute;\n  width: 100%;\n  height: 0;\n  left: 0;\n  z-index: 1;\n}\n\n.vis-foreground .vis-group {\n  position: relative;\n  box-sizing: border-box;\n  border-bottom: 1px solid #bfbfbf;\n}\n\n.vis-foreground .vis-group:last-child {\n  border-bottom: none;\n}\n\n.vis-nesting-group {\n  cursor: pointer;\n}\n\n.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1 {\n  background: #f5f5f5;\n}\n.vis-label.vis-nested-group.vis-group-level-0 {\n  background-color: #ffffff;\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner {\n  padding-left: 0;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner {\n  padding-right: 0;\n}\n.vis-label.vis-nested-group.vis-group-level-1 {\n  background-color: rgba(0, 0, 0, 0.05);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner {\n  padding-left: 15px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner {\n  padding-right: 15px;\n}\n.vis-label.vis-nested-group.vis-group-level-2 {\n  background-color: rgba(0, 0, 0, 0.1);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner {\n  padding-left: 30px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner {\n  padding-right: 30px;\n}\n.vis-label.vis-nested-group.vis-group-level-3 {\n  background-color: rgba(0, 0, 0, 0.15);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner {\n  padding-left: 45px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner {\n  padding-right: 45px;\n}\n.vis-label.vis-nested-group.vis-group-level-4 {\n  background-color: rgba(0, 0, 0, 0.2);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner {\n  padding-left: 60px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner {\n  padding-right: 60px;\n}\n.vis-label.vis-nested-group.vis-group-level-5 {\n  background-color: rgba(0, 0, 0, 0.25);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner {\n  padding-left: 75px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner {\n  padding-right: 75px;\n}\n.vis-label.vis-nested-group.vis-group-level-6 {\n  background-color: rgba(0, 0, 0, 0.3);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner {\n  padding-left: 90px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner {\n  padding-right: 90px;\n}\n.vis-label.vis-nested-group.vis-group-level-7 {\n  background-color: rgba(0, 0, 0, 0.35);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner {\n  padding-left: 105px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner {\n  padding-right: 105px;\n}\n.vis-label.vis-nested-group.vis-group-level-8 {\n  background-color: rgba(0, 0, 0, 0.4);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner {\n  padding-left: 120px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner {\n  padding-right: 120px;\n}\n.vis-label.vis-nested-group.vis-group-level-9 {\n  background-color: rgba(0, 0, 0, 0.45);\n}\n.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner {\n  padding-left: 135px;\n}\n.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner {\n  padding-right: 135px;\n}\n/* default takes over beginning with level-10 (thats why we add .vis-nested-group\n  to the selectors above, to have higher specifity than these rules for the defaults) */\n.vis-label.vis-nested-group {\n  background-color: rgba(0, 0, 0, 0.5);\n}\n.vis-ltr .vis-label.vis-nested-group .vis-inner {\n  padding-left: 150px;\n}\n.vis-rtl .vis-label.vis-nested-group .vis-inner {\n  padding-right: 150px;\n}\n\n.vis-group-level-unknown-but-gte1 {\n  border: 1px solid red;\n}\n\n/* expanded/collapsed indicators */\n.vis-label.vis-nesting-group:before,\n.vis-label.vis-nesting-group:before {\n  display: inline-block;\n  width: 15px;\n}\n.vis-label.vis-nesting-group.expanded:before {\n  content: \"\\25BC\";\n}\n.vis-label.vis-nesting-group.collapsed:before {\n  content: \"\\25B6\";\n}\n.vis-rtl .vis-label.vis-nesting-group.collapsed:before {\n  content: \"\\25C0\";\n}\n/* compensate missing expanded/collapsed indicator, but only at levels > 0 */\n.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0) {\n  padding-left: 15px;\n}\n.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0) {\n  padding-right: 15px;\n}\n\n.vis-overlay {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n  z-index: 10;\n}","\n.vis-labelset {\n  position: relative;\n\n  overflow: hidden;\n\n  box-sizing: border-box;\n}\n\n.vis-labelset .vis-label {\n  position: relative;\n  left: 0;\n  top: 0;\n  width: 100%;\n  color: #4d4d4d;\n\n  box-sizing: border-box;\n}\n\n.vis-labelset .vis-label {\n  border-bottom: 1px solid #bfbfbf;\n}\n\n.vis-labelset .vis-label.draggable {\n  cursor: pointer;\n}\n\n.vis-group-is-dragging {\n  background: rgba(0, 0, 0, .1);\n}\n\n.vis-labelset .vis-label:last-child {\n  border-bottom: none;\n}\n\n.vis-labelset .vis-label .vis-inner {\n  display: inline-block;\n  padding: 5px;\n}\n\n.vis-labelset .vis-label .vis-inner.vis-hidden {\n  padding: 0;\n}\n",".vis-time-axis {\n  position: relative;\n  overflow: hidden;\n}\n\n.vis-time-axis.vis-foreground {\n  top: 0;\n  left: 0;\n  width: 100%;\n}\n\n.vis-time-axis.vis-background {\n  position: absolute;\n  top: 0;\n  left: 0;\n  width: 100%;\n  height: 100%;\n}\n\n.vis-time-axis .vis-text {\n  position: absolute;\n  color: #4d4d4d;\n  padding: 3px;\n  overflow: hidden;\n  box-sizing: border-box;\n\n  white-space: nowrap;\n}\n\n.vis-time-axis .vis-text.vis-measure {\n  position: absolute;\n  padding-left: 0;\n  padding-right: 0;\n  margin-left: 0;\n  margin-right: 0;\n  visibility: hidden;\n}\n\n.vis-time-axis .vis-grid.vis-vertical {\n  position: absolute;\n  border-left: 1px solid;\n}\n\n.vis-time-axis .vis-grid.vis-vertical-rtl {\n  position: absolute;\n  border-right: 1px solid;\n}\n\n.vis-time-axis .vis-grid.vis-minor {\n  border-color: #e5e5e5;\n}\n\n.vis-time-axis .vis-grid.vis-major {\n  border-color: #bfbfbf;\n}\n","\n.vis-item {\n  position: absolute;\n  color: #1A1A1A;\n  border-color: #97B0F8;\n  border-width: 1px;\n  background-color: #D5DDF6;\n  display: inline-block;\n  z-index: 1;\n  /*overflow: hidden;*/\n}\n\n.vis-item.vis-selected {\n  border-color: #FFC200;\n  background-color: #FFF785;\n\n  /* z-index must be higher than the z-index of custom time bar and current time bar */\n  z-index: 2;\n}\n\n.vis-editable.vis-selected {\n  cursor: move;\n}\n\n.vis-item.vis-point.vis-selected {\n  background-color: #FFF785;\n}\n\n.vis-item.vis-box {\n  text-align: center;\n  border-style: solid;\n  border-radius: 2px;\n}\n\n.vis-item.vis-point {\n  background: none;\n}\n\n.vis-item.vis-dot {\n  position: absolute;\n  padding: 0;\n  border-width: 4px;\n  border-style: solid;\n  border-radius: 4px;\n}\n\n.vis-item.vis-range {\n  border-style: solid;\n  border-radius: 2px;\n  box-sizing: border-box;\n}\n\n.vis-item.vis-background {\n  border: none;\n  background-color: rgba(213, 221, 246, 0.4);\n  box-sizing: border-box;\n  padding: 0;\n  margin: 0;\n}\n\n.vis-item .vis-item-overflow {\n  position: relative;\n  width: 100%;\n  height: 100%;\n  padding: 0;\n  margin: 0;\n  overflow: hidden;\n}\n\n.vis-item-visible-frame {\n  white-space: nowrap;\n}\n\n.vis-item.vis-range .vis-item-content {\n  position: relative;\n  display: inline-block;\n}\n\n.vis-item.vis-background .vis-item-content {\n  position: absolute;\n  display: inline-block;\n}\n\n.vis-item.vis-line {\n  padding: 0;\n  position: absolute;\n  width: 0;\n  border-left-width: 1px;\n  border-left-style: solid;\n}\n\n.vis-item .vis-item-content {\n  white-space: nowrap;\n  box-sizing: border-box;\n  padding: 5px;\n}\n\n.vis-item .vis-onUpdateTime-tooltip {\n  position: absolute;\n  background: #4f81bd;\n  color: white;\n  width: 200px;\n  text-align: center;\n  white-space: nowrap;\n  padding: 5px;\n  border-radius: 1px;\n  transition: 0.4s;\n  -o-transition: 0.4s;\n  -moz-transition: 0.4s;\n  -webkit-transition: 0.4s;\n}\n\n.vis-item .vis-delete, .vis-item .vis-delete-rtl {\n  position: absolute;\n  top: 0px;\n  width: 24px;\n  height: 24px;\n  box-sizing: border-box;\n  padding: 0px 5px;\n  cursor: pointer;\n\n  -webkit-transition: background 0.2s linear;\n  -moz-transition: background 0.2s linear;\n  -ms-transition: background 0.2s linear;\n  -o-transition: background 0.2s linear;\n  transition: background 0.2s linear;\n}\n\n.vis-item .vis-delete {\n  right: -24px;\n}\n\n.vis-item .vis-delete-rtl {\n  left: -24px;\n}\n\n.vis-item .vis-delete:after, .vis-item .vis-delete-rtl:after {\n  content: \"\\00D7\"; /* MULTIPLICATION SIGN */\n  color: red;\n  font-family: arial, sans-serif;\n  font-size: 22px;\n  font-weight: bold;\n\n  -webkit-transition: color 0.2s linear;\n  -moz-transition: color 0.2s linear;\n  -ms-transition: color 0.2s linear;\n  -o-transition: color 0.2s linear;\n  transition: color 0.2s linear;\n}\n\n.vis-item .vis-delete:hover, .vis-item .vis-delete-rtl:hover {\n  background: red;\n}\n\n.vis-item .vis-delete:hover:after, .vis-item .vis-delete-rtl:hover:after {\n  color: white;\n}\n\n.vis-item .vis-drag-center {\n  position: absolute;\n  width: 100%;\n  height: 100%;\n  top: 0;\n  left: 0px;\n  cursor: move;\n}\n\n.vis-item.vis-range .vis-drag-left {\n  position: absolute;\n  width: 24px;\n  max-width: 20%;\n  min-width: 2px;\n  height: 100%;\n  top: 0;\n  left: -4px;\n\n  cursor: w-resize;\n}\n\n.vis-item.vis-range .vis-drag-right {\n  position: absolute;\n  width: 24px;\n  max-width: 20%;\n  min-width: 2px;\n  height: 100%;\n  top: 0;\n  right: -4px;\n\n  cursor: e-resize;\n}\n\n.vis-range.vis-item.vis-readonly .vis-drag-left,\n.vis-range.vis-item.vis-readonly .vis-drag-right {\n  cursor: auto;\n}\n\n.vis-item.vis-cluster {\n  vertical-align: center;\n  text-align: center;\n  border-style: solid;\n  border-radius: 2px;\n}\n\n.vis-item.vis-cluster-line {\n  padding: 0;\n  position: absolute;\n  width: 0;\n  border-left-width: 1px;\n  border-left-style: solid;\n}\n\n.vis-item.vis-cluster-dot {\n  position: absolute;\n  padding: 0;\n  border-width: 4px;\n  border-style: solid;\n  border-radius: 4px;\n}"]}
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
index df730409ee491..5cfa0da3d4f38 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js
@@ -1,60 +1,30 @@
 /**
- * vis-timeline and vis-graph2d
- * https://visjs.github.io/vis-timeline/
+ * vis.js
+ * https://github.com/almende/vis
  *
- * Create a fully customizable, interactive timeline with items and ranges.
+ * A dynamic, browser-based visualization library.
  *
- * @version 7.3.4
- * @date    2020-03-18T17:03:58.105Z
- *
- * @copyright (c) 2011-2017 Almende B.V, http://almende.com
- * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+ * @version 4.21.0
+ * @date    2017-10-12
  *
  * @license
- * vis.js is dual licensed under both
+ * Copyright (C) 2011-2017 Almende B.V, http://almende.com
+ *
+ * Vis.js is dual licensed under both
  *
- *   1. The Apache 2.0 License
- *      http://www.apache.org/licenses/LICENSE-2.0
+ * * The Apache 2.0 License
+ *   http://www.apache.org/licenses/LICENSE-2.0
  *
- *   and
+ * and
  *
- *   2. The MIT License
- *      http://opensource.org/licenses/MIT
+ * * The MIT License
+ *   http://opensource.org/licenses/MIT
  *
- * vis.js may be distributed under either license.
+ * Vis.js may be distributed under either license.
  */
-!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports):"function"==typeof define&&define.amd?define(["exports"],e):e((t=t||self).vis=t.vis||{})}(this,(function(t){"use strict";var e="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:{};function i(){throw new Error("Dynamic requires are not currently supported by @rollup/plugin-commonjs")}function n(t,e){return t(e={exports:{}},e.exports),e.exports}function o(t){return t&&t.default||t}var s=n((function(t,e){t.exports=function(){var e,n;function o(){return e.apply(null,arguments)}function s(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function r(t){return null!=t&&"[object Object]"===Object.prototype.toString.call(t)}function a(t){return void 0===t}function l(t){return"number"==typeof t||"[object Number]"===Object.prototype.toString.call(t)}function h(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function d(t,e){var i,n=[];for(i=0;i<t.length;++i)n.push(e(t[i],i));return n}function u(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function c(t,e){for(var i in e)u(e,i)&&(t[i]=e[i]);return u(e,"toString")&&(t.toString=e.toString),u(e,"valueOf")&&(t.valueOf=e.valueOf),t}function p(t,e,i,n){return De(t,e,i,n,!0).utc()}function m(t){return null==t._pf&&(t._pf={empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}),t._pf}function f(t){if(null==t._isValid){var e=m(t),i=n.call(e.parsedDateParts,(function(t){return null!=t})),o=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.weekdayMismatch&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i);if(t._strict&&(o=o&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour),null!=Object.isFrozen&&Object.isFrozen(t))return o;t._isValid=o}return t._isValid}function g(t){var e=p(NaN);return null!=t?c(m(e),t):m(e).userInvalidated=!0,e}n=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,n=0;n<i;n++)if(n in e&&t.call(this,e[n],n,e))return!0;return!1};var v=o.momentProperties=[];function b(t,e){var i,n,o;if(a(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),a(e._i)||(t._i=e._i),a(e._f)||(t._f=e._f),a(e._l)||(t._l=e._l),a(e._strict)||(t._strict=e._strict),a(e._tzm)||(t._tzm=e._tzm),a(e._isUTC)||(t._isUTC=e._isUTC),a(e._offset)||(t._offset=e._offset),a(e._pf)||(t._pf=m(e)),a(e._locale)||(t._locale=e._locale),v.length>0)for(i=0;i<v.length;i++)a(o=e[n=v[i]])||(t[n]=o);return t}var y=!1;function x(t){b(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===y&&(y=!0,o.updateOffset(this),y=!1)}function w(t){return t instanceof x||null!=t&&null!=t._isAMomentObject}function _(t){return t<0?Math.ceil(t)||0:Math.floor(t)}function k(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=_(e)),i}function D(t,e,i){var n,o=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(n=0;n<o;n++)(i&&t[n]!==e[n]||!i&&k(t[n])!==k(e[n]))&&r++;return r+s}function I(t){!1===o.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function C(t,e){var i=!0;return c((function(){if(null!=o.deprecationHandler&&o.deprecationHandler(null,t),i){for(var n,s=[],r=0;r<arguments.length;r++){if(n="","object"==typeof arguments[r]){for(var a in n+="\n["+r+"] ",arguments[0])n+=a+": "+arguments[0][a]+", ";n=n.slice(0,-2)}else n=arguments[r];s.push(n)}I(t+"\nArguments: "+Array.prototype.slice.call(s).join("")+"\n"+(new Error).stack),i=!1}return e.apply(this,arguments)}),e)}var L,G={};function M(t,e){null!=o.deprecationHandler&&o.deprecationHandler(t,e),G[t]||(I(e),G[t]=!0)}function S(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function Q(t,e){var i,n=c({},t);for(i in e)u(e,i)&&(r(t[i])&&r(e[i])?(n[i]={},c(n[i],t[i]),c(n[i],e[i])):null!=e[i]?n[i]=e[i]:delete n[i]);for(i in t)u(t,i)&&!u(e,i)&&r(t[i])&&(n[i]=c({},n[i]));return n}function N(t){null!=t&&this.set(t)}o.suppressDeprecationWarnings=!1,o.deprecationHandler=null,L=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)u(t,e)&&i.push(e);return i};var F={};function Z(t,e){var i=t.toLowerCase();F[i]=F[i+"s"]=F[e]=t}function B(t){return"string"==typeof t?F[t]||F[t.toLowerCase()]:void 0}function O(t){var e,i,n={};for(i in t)u(t,i)&&(e=B(i))&&(n[e]=t[i]);return n}var W={};function A(t,e){W[t]=e}function X(t,e,i){var n=""+Math.abs(t),o=e-n.length;return(t>=0?i?"+":"":"-")+Math.pow(10,Math.max(0,o)).toString().substr(1)+n}var T=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Y=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,E={},R={};function U(t,e,i,n){var o=n;"string"==typeof n&&(o=function(){return this[n]()}),t&&(R[t]=o),e&&(R[e[0]]=function(){return X(o.apply(this,arguments),e[1],e[2])}),i&&(R[i]=function(){return this.localeData().ordinal(o.apply(this,arguments),t)})}function z(t,e){return t.isValid()?(e=V(e,t.localeData()),E[e]=E[e]||function(t){var e,i,n,o=t.match(T);for(e=0,i=o.length;e<i;e++)R[o[e]]?o[e]=R[o[e]]:o[e]=(n=o[e]).match(/\[[\s\S]/)?n.replace(/^\[|\]$/g,""):n.replace(/\\/g,"");return function(e){var n,s="";for(n=0;n<i;n++)s+=S(o[n])?o[n].call(e,t):o[n];return s}}(e),E[e](t)):t.localeData().invalidDate()}function V(t,e){var i=5;function n(t){return e.longDateFormat(t)||t}for(Y.lastIndex=0;i>=0&&Y.test(t);)t=t.replace(Y,n),Y.lastIndex=0,i-=1;return t}var j=/\d/,H=/\d\d/,P=/\d{3}/,J=/\d{4}/,K=/[+-]?\d{6}/,q=/\d\d?/,$=/\d\d\d\d?/,tt=/\d\d\d\d\d\d?/,et=/\d{1,3}/,it=/\d{1,4}/,nt=/[+-]?\d{1,6}/,ot=/\d+/,st=/[+-]?\d+/,rt=/Z|[+-]\d\d:?\d\d/gi,at=/Z|[+-]\d\d(?::?\d\d)?/gi,lt=/[0-9]{0,256}['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFF07\uFF10-\uFFEF]{1,256}|[\u0600-\u06FF\/]{1,256}(\s*?[\u0600-\u06FF]{1,256}){1,2}/i,ht={};function dt(t,e,i){ht[t]=S(e)?e:function(t,n){return t&&i?i:e}}function ut(t,e){return u(ht,t)?ht[t](e._strict,e._locale):new RegExp(ct(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,(function(t,e,i,n,o){return e||i||n||o}))))}function ct(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}var pt={};function mt(t,e){var i,n=e;for("string"==typeof t&&(t=[t]),l(e)&&(n=function(t,i){i[e]=k(t)}),i=0;i<t.length;i++)pt[t[i]]=n}function ft(t,e){mt(t,(function(t,i,n,o){n._w=n._w||{},e(t,n._w,n,o)}))}function gt(t,e,i){null!=e&&u(pt,t)&&pt[t](e,i._a,i,t)}function vt(t){return bt(t)?366:365}function bt(t){return t%4==0&&t%100!=0||t%400==0}U("Y",0,0,(function(){var t=this.year();return t<=9999?""+t:"+"+t})),U(0,["YY",2],0,(function(){return this.year()%100})),U(0,["YYYY",4],0,"year"),U(0,["YYYYY",5],0,"year"),U(0,["YYYYYY",6,!0],0,"year"),Z("year","y"),A("year",1),dt("Y",st),dt("YY",q,H),dt("YYYY",it,J),dt("YYYYY",nt,K),dt("YYYYYY",nt,K),mt(["YYYYY","YYYYYY"],0),mt("YYYY",(function(t,e){e[0]=2===t.length?o.parseTwoDigitYear(t):k(t)})),mt("YY",(function(t,e){e[0]=o.parseTwoDigitYear(t)})),mt("Y",(function(t,e){e[0]=parseInt(t,10)})),o.parseTwoDigitYear=function(t){return k(t)+(k(t)>68?1900:2e3)};var yt,xt=wt("FullYear",!0);function wt(t,e){return function(i){return null!=i?(kt(this,t,i),o.updateOffset(this,e),this):_t(this,t)}}function _t(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function kt(t,e,i){t.isValid()&&!isNaN(i)&&("FullYear"===e&&bt(t.year())&&1===t.month()&&29===t.date()?t._d["set"+(t._isUTC?"UTC":"")+e](i,t.month(),Dt(i,t.month())):t._d["set"+(t._isUTC?"UTC":"")+e](i))}function Dt(t,e){if(isNaN(t)||isNaN(e))return NaN;var i,n=(e%(i=12)+i)%i;return t+=(e-n)/12,1===n?bt(t)?29:28:31-n%7%2}yt=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},U("M",["MM",2],"Mo",(function(){return this.month()+1})),U("MMM",0,0,(function(t){return this.localeData().monthsShort(this,t)})),U("MMMM",0,0,(function(t){return this.localeData().months(this,t)})),Z("month","M"),A("month",8),dt("M",q),dt("MM",q,H),dt("MMM",(function(t,e){return e.monthsShortRegex(t)})),dt("MMMM",(function(t,e){return e.monthsRegex(t)})),mt(["M","MM"],(function(t,e){e[1]=k(t)-1})),mt(["MMM","MMMM"],(function(t,e,i,n){var o=i._locale.monthsParse(t,n,i._strict);null!=o?e[1]=o:m(i).invalidMonth=t}));var It=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,Ct="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),Lt="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_");function Gt(t,e,i){var n,o,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],n=0;n<12;++n)s=p([2e3,n]),this._shortMonthsParse[n]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[n]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?-1!==(o=yt.call(this._shortMonthsParse,r))?o:null:-1!==(o=yt.call(this._longMonthsParse,r))?o:null:"MMM"===e?-1!==(o=yt.call(this._shortMonthsParse,r))||-1!==(o=yt.call(this._longMonthsParse,r))?o:null:-1!==(o=yt.call(this._longMonthsParse,r))||-1!==(o=yt.call(this._shortMonthsParse,r))?o:null}function Mt(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=k(e);else if(!l(e=t.localeData().monthsParse(e)))return t;return i=Math.min(t.date(),Dt(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function St(t){return null!=t?(Mt(this,t),o.updateOffset(this,!0),this):_t(this,"Month")}var Qt=lt,Nt=lt;function Ft(){function t(t,e){return e.length-t.length}var e,i,n=[],o=[],s=[];for(e=0;e<12;e++)i=p([2e3,e]),n.push(this.monthsShort(i,"")),o.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(n.sort(t),o.sort(t),s.sort(t),e=0;e<12;e++)n[e]=ct(n[e]),o[e]=ct(o[e]);for(e=0;e<24;e++)s[e]=ct(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+o.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+n.join("|")+")","i")}function Zt(t,e,i,n,o,s,r){var a;return t<100&&t>=0?(a=new Date(t+400,e,i,n,o,s,r),isFinite(a.getFullYear())&&a.setFullYear(t)):a=new Date(t,e,i,n,o,s,r),a}function Bt(t){var e;if(t<100&&t>=0){var i=Array.prototype.slice.call(arguments);i[0]=t+400,e=new Date(Date.UTC.apply(null,i)),isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t)}else e=new Date(Date.UTC.apply(null,arguments));return e}function Ot(t,e,i){var n=7+e-i;return-(7+Bt(t,0,n).getUTCDay()-e)%7+n-1}function Wt(t,e,i,n,o){var s,r,a=1+7*(e-1)+(7+i-n)%7+Ot(t,n,o);return a<=0?r=vt(s=t-1)+a:a>vt(t)?(s=t+1,r=a-vt(t)):(s=t,r=a),{year:s,dayOfYear:r}}function At(t,e,i){var n,o,s=Ot(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return r<1?n=r+Xt(o=t.year()-1,e,i):r>Xt(t.year(),e,i)?(n=r-Xt(t.year(),e,i),o=t.year()+1):(o=t.year(),n=r),{week:n,year:o}}function Xt(t,e,i){var n=Ot(t,e,i),o=Ot(t+1,e,i);return(vt(t)-n+o)/7}function Tt(t,e){return t.slice(e,7).concat(t.slice(0,e))}U("w",["ww",2],"wo","week"),U("W",["WW",2],"Wo","isoWeek"),Z("week","w"),Z("isoWeek","W"),A("week",5),A("isoWeek",5),dt("w",q),dt("ww",q,H),dt("W",q),dt("WW",q,H),ft(["w","ww","W","WW"],(function(t,e,i,n){e[n.substr(0,1)]=k(t)})),U("d",0,"do","day"),U("dd",0,0,(function(t){return this.localeData().weekdaysMin(this,t)})),U("ddd",0,0,(function(t){return this.localeData().weekdaysShort(this,t)})),U("dddd",0,0,(function(t){return this.localeData().weekdays(this,t)})),U("e",0,0,"weekday"),U("E",0,0,"isoWeekday"),Z("day","d"),Z("weekday","e"),Z("isoWeekday","E"),A("day",11),A("weekday",11),A("isoWeekday",11),dt("d",q),dt("e",q),dt("E",q),dt("dd",(function(t,e){return e.weekdaysMinRegex(t)})),dt("ddd",(function(t,e){return e.weekdaysShortRegex(t)})),dt("dddd",(function(t,e){return e.weekdaysRegex(t)})),ft(["dd","ddd","dddd"],(function(t,e,i,n){var o=i._locale.weekdaysParse(t,n,i._strict);null!=o?e.d=o:m(i).invalidWeekday=t})),ft(["d","e","E"],(function(t,e,i,n){e[n]=k(t)}));var Yt="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Et="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Rt="Su_Mo_Tu_We_Th_Fr_Sa".split("_");function Ut(t,e,i){var n,o,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],n=0;n<7;++n)s=p([2e3,1]).day(n),this._minWeekdaysParse[n]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[n]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[n]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?-1!==(o=yt.call(this._weekdaysParse,r))?o:null:"ddd"===e?-1!==(o=yt.call(this._shortWeekdaysParse,r))?o:null:-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:"dddd"===e?-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._shortWeekdaysParse,r))||-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:"ddd"===e?-1!==(o=yt.call(this._shortWeekdaysParse,r))||-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._minWeekdaysParse,r))?o:null:-1!==(o=yt.call(this._minWeekdaysParse,r))||-1!==(o=yt.call(this._weekdaysParse,r))||-1!==(o=yt.call(this._shortWeekdaysParse,r))?o:null}var zt=lt,Vt=lt,jt=lt;function Ht(){function t(t,e){return e.length-t.length}var e,i,n,o,s,r=[],a=[],l=[],h=[];for(e=0;e<7;e++)i=p([2e3,1]).day(e),n=this.weekdaysMin(i,""),o=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(n),a.push(o),l.push(s),h.push(n),h.push(o),h.push(s);for(r.sort(t),a.sort(t),l.sort(t),h.sort(t),e=0;e<7;e++)a[e]=ct(a[e]),l[e]=ct(l[e]),h[e]=ct(h[e]);this._weekdaysRegex=new RegExp("^("+h.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+l.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function Pt(){return this.hours()%12||12}function Jt(t,e){U(t,0,0,(function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)}))}function Kt(t,e){return e._meridiemParse}U("H",["HH",2],0,"hour"),U("h",["hh",2],0,Pt),U("k",["kk",2],0,(function(){return this.hours()||24})),U("hmm",0,0,(function(){return""+Pt.apply(this)+X(this.minutes(),2)})),U("hmmss",0,0,(function(){return""+Pt.apply(this)+X(this.minutes(),2)+X(this.seconds(),2)})),U("Hmm",0,0,(function(){return""+this.hours()+X(this.minutes(),2)})),U("Hmmss",0,0,(function(){return""+this.hours()+X(this.minutes(),2)+X(this.seconds(),2)})),Jt("a",!0),Jt("A",!1),Z("hour","h"),A("hour",13),dt("a",Kt),dt("A",Kt),dt("H",q),dt("h",q),dt("k",q),dt("HH",q,H),dt("hh",q,H),dt("kk",q,H),dt("hmm",$),dt("hmmss",tt),dt("Hmm",$),dt("Hmmss",tt),mt(["H","HH"],3),mt(["k","kk"],(function(t,e,i){var n=k(t);e[3]=24===n?0:n})),mt(["a","A"],(function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t})),mt(["h","hh"],(function(t,e,i){e[3]=k(t),m(i).bigHour=!0})),mt("hmm",(function(t,e,i){var n=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n)),m(i).bigHour=!0})),mt("hmmss",(function(t,e,i){var n=t.length-4,o=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n,2)),e[5]=k(t.substr(o)),m(i).bigHour=!0})),mt("Hmm",(function(t,e,i){var n=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n))})),mt("Hmmss",(function(t,e,i){var n=t.length-4,o=t.length-2;e[3]=k(t.substr(0,n)),e[4]=k(t.substr(n,2)),e[5]=k(t.substr(o))}));var qt,$t=wt("Hours",!0),te={calendar:{sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},longDateFormat:{LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:/\d{1,2}/,relativeTime:{future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},months:Ct,monthsShort:Lt,week:{dow:0,doy:6},weekdays:Yt,weekdaysMin:Rt,weekdaysShort:Et,meridiemParse:/[ap]\.?m?\.?/i},ee={},ie={};function ne(t){return t?t.toLowerCase().replace("_","-"):t}function oe(e){var n=null;if(!ee[e]&&t&&t.exports)try{n=qt._abbr,i(),se(n)}catch(t){}return ee[e]}function se(t,e){var i;return t&&((i=a(e)?ae(t):re(t,e))?qt=i:"undefined"!=typeof console&&console.warn&&console.warn("Locale "+t+" not found. Did you forget to load it?")),qt._abbr}function re(t,e){if(null!==e){var i,n=te;if(e.abbr=t,null!=ee[t])M("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),n=ee[t]._config;else if(null!=e.parentLocale)if(null!=ee[e.parentLocale])n=ee[e.parentLocale]._config;else{if(null==(i=oe(e.parentLocale)))return ie[e.parentLocale]||(ie[e.parentLocale]=[]),ie[e.parentLocale].push({name:t,config:e}),null;n=i._config}return ee[t]=new N(Q(n,e)),ie[t]&&ie[t].forEach((function(t){re(t.name,t.config)})),se(t),ee[t]}return delete ee[t],null}function ae(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return qt;if(!s(t)){if(e=oe(t))return e;t=[t]}return function(t){for(var e,i,n,o,s=0;s<t.length;){for(e=(o=ne(t[s]).split("-")).length,i=(i=ne(t[s+1]))?i.split("-"):null;e>0;){if(n=oe(o.slice(0,e).join("-")))return n;if(i&&i.length>=e&&D(o,i,!0)>=e-1)break;e--}s++}return qt}(t)}function le(t){var e,i=t._a;return i&&-2===m(t).overflow&&(e=i[1]<0||i[1]>11?1:i[2]<1||i[2]>Dt(i[0],i[1])?2:i[3]<0||i[3]>24||24===i[3]&&(0!==i[4]||0!==i[5]||0!==i[6])?3:i[4]<0||i[4]>59?4:i[5]<0||i[5]>59?5:i[6]<0||i[6]>999?6:-1,m(t)._overflowDayOfYear&&(e<0||e>2)&&(e=2),m(t)._overflowWeeks&&-1===e&&(e=7),m(t)._overflowWeekday&&-1===e&&(e=8),m(t).overflow=e),t}function he(t,e,i){return null!=t?t:null!=e?e:i}function de(t){var e,i,n,s,r,a=[];if(!t._d){for(n=function(t){var e=new Date(o.now());return t._useUTC?[e.getUTCFullYear(),e.getUTCMonth(),e.getUTCDate()]:[e.getFullYear(),e.getMonth(),e.getDate()]}(t),t._w&&null==t._a[2]&&null==t._a[1]&&function(t){var e,i,n,o,s,r,a,l;if(null!=(e=t._w).GG||null!=e.W||null!=e.E)s=1,r=4,i=he(e.GG,t._a[0],At(Ie(),1,4).year),n=he(e.W,1),((o=he(e.E,1))<1||o>7)&&(l=!0);else{s=t._locale._week.dow,r=t._locale._week.doy;var h=At(Ie(),s,r);i=he(e.gg,t._a[0],h.year),n=he(e.w,h.week),null!=e.d?((o=e.d)<0||o>6)&&(l=!0):null!=e.e?(o=e.e+s,(e.e<0||e.e>6)&&(l=!0)):o=s}n<1||n>Xt(i,s,r)?m(t)._overflowWeeks=!0:null!=l?m(t)._overflowWeekday=!0:(a=Wt(i,n,o,s,r),t._a[0]=a.year,t._dayOfYear=a.dayOfYear)}(t),null!=t._dayOfYear&&(r=he(t._a[0],n[0]),(t._dayOfYear>vt(r)||0===t._dayOfYear)&&(m(t)._overflowDayOfYear=!0),i=Bt(r,0,t._dayOfYear),t._a[1]=i.getUTCMonth(),t._a[2]=i.getUTCDate()),e=0;e<3&&null==t._a[e];++e)t._a[e]=a[e]=n[e];for(;e<7;e++)t._a[e]=a[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[3]&&0===t._a[4]&&0===t._a[5]&&0===t._a[6]&&(t._nextDay=!0,t._a[3]=0),t._d=(t._useUTC?Bt:Zt).apply(null,a),s=t._useUTC?t._d.getUTCDay():t._d.getDay(),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[3]=24),t._w&&void 0!==t._w.d&&t._w.d!==s&&(m(t).weekdayMismatch=!0)}}var ue=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,ce=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,pe=/Z|[+-]\d\d(?::?\d\d)?/,me=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],fe=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],ge=/^\/?Date\((\-?\d+)/i;function ve(t){var e,i,n,o,s,r,a=t._i,l=ue.exec(a)||ce.exec(a);if(l){for(m(t).iso=!0,e=0,i=me.length;e<i;e++)if(me[e][1].exec(l[1])){o=me[e][0],n=!1!==me[e][2];break}if(null==o)return void(t._isValid=!1);if(l[3]){for(e=0,i=fe.length;e<i;e++)if(fe[e][1].exec(l[3])){s=(l[2]||" ")+fe[e][0];break}if(null==s)return void(t._isValid=!1)}if(!n&&null!=s)return void(t._isValid=!1);if(l[4]){if(!pe.exec(l[4]))return void(t._isValid=!1);r="Z"}t._f=o+(s||"")+(r||""),_e(t)}else t._isValid=!1}var be=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/;function ye(t){var e=parseInt(t,10);return e<=49?2e3+e:e<=999?1900+e:e}var xe={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};function we(t){var e,i,n,o,s,r,a,l=be.exec(t._i.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").replace(/^\s\s*/,"").replace(/\s\s*$/,""));if(l){var h=(e=l[4],i=l[3],n=l[2],o=l[5],s=l[6],r=l[7],a=[ye(e),Lt.indexOf(i),parseInt(n,10),parseInt(o,10),parseInt(s,10)],r&&a.push(parseInt(r,10)),a);if(!function(t,e,i){return!t||Et.indexOf(t)===new Date(e[0],e[1],e[2]).getDay()||(m(i).weekdayMismatch=!0,i._isValid=!1,!1)}(l[1],h,t))return;t._a=h,t._tzm=function(t,e,i){if(t)return xe[t];if(e)return 0;var n=parseInt(i,10),o=n%100;return(n-o)/100*60+o}(l[8],l[9],l[10]),t._d=Bt.apply(null,t._a),t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),m(t).rfc2822=!0}else t._isValid=!1}function _e(t){if(t._f!==o.ISO_8601)if(t._f!==o.RFC_2822){t._a=[],m(t).empty=!0;var e,i,n,s,r,a=""+t._i,l=a.length,h=0;for(n=V(t._f,t._locale).match(T)||[],e=0;e<n.length;e++)s=n[e],(i=(a.match(ut(s,t))||[])[0])&&((r=a.substr(0,a.indexOf(i))).length>0&&m(t).unusedInput.push(r),a=a.slice(a.indexOf(i)+i.length),h+=i.length),R[s]?(i?m(t).empty=!1:m(t).unusedTokens.push(s),gt(s,i,t)):t._strict&&!i&&m(t).unusedTokens.push(s);m(t).charsLeftOver=l-h,a.length>0&&m(t).unusedInput.push(a),t._a[3]<=12&&!0===m(t).bigHour&&t._a[3]>0&&(m(t).bigHour=void 0),m(t).parsedDateParts=t._a.slice(0),m(t).meridiem=t._meridiem,t._a[3]=function(t,e,i){var n;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?((n=t.isPM(i))&&e<12&&(e+=12),n||12!==e||(e=0),e):e}(t._locale,t._a[3],t._meridiem),de(t),le(t)}else we(t);else ve(t)}function ke(t){var e=t._i,i=t._f;return t._locale=t._locale||ae(t._l),null===e||void 0===i&&""===e?g({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),w(e)?new x(le(e)):(h(e)?t._d=e:s(i)?function(t){var e,i,n,o,s;if(0===t._f.length)return m(t).invalidFormat=!0,void(t._d=new Date(NaN));for(o=0;o<t._f.length;o++)s=0,e=b({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[o],_e(e),f(e)&&(s+=m(e).charsLeftOver,s+=10*m(e).unusedTokens.length,m(e).score=s,(null==n||s<n)&&(n=s,i=e));c(t,i||e)}(t):i?_e(t):function(t){var e=t._i;a(e)?t._d=new Date(o.now()):h(e)?t._d=new Date(e.valueOf()):"string"==typeof e?function(t){var e=ge.exec(t._i);null===e?(ve(t),!1===t._isValid&&(delete t._isValid,we(t),!1===t._isValid&&(delete t._isValid,o.createFromInputFallback(t)))):t._d=new Date(+e[1])}(t):s(e)?(t._a=d(e.slice(0),(function(t){return parseInt(t,10)})),de(t)):r(e)?function(t){if(!t._d){var e=O(t._i);t._a=d([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],(function(t){return t&&parseInt(t,10)})),de(t)}}(t):l(e)?t._d=new Date(e):o.createFromInputFallback(t)}(t),f(t)||(t._d=null),t))}function De(t,e,i,n,o){var a,l={};return!0!==i&&!1!==i||(n=i,i=void 0),(r(t)&&function(t){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(t).length;var e;for(e in t)if(t.hasOwnProperty(e))return!1;return!0}(t)||s(t)&&0===t.length)&&(t=void 0),l._isAMomentObject=!0,l._useUTC=l._isUTC=o,l._l=i,l._i=t,l._f=e,l._strict=n,(a=new x(le(ke(l))))._nextDay&&(a.add(1,"d"),a._nextDay=void 0),a}function Ie(t,e,i,n){return De(t,e,i,n,!1)}o.createFromInputFallback=C("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",(function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))})),o.ISO_8601=function(){},o.RFC_2822=function(){};var Ce=C("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",(function(){var t=Ie.apply(null,arguments);return this.isValid()&&t.isValid()?t<this?this:t:g()})),Le=C("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",(function(){var t=Ie.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:g()}));function Ge(t,e){var i,n;if(1===e.length&&s(e[0])&&(e=e[0]),!e.length)return Ie();for(i=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](i)||(i=e[n]);return i}var Me=["year","quarter","month","week","day","hour","minute","second","millisecond"];function Se(t){var e=O(t),i=e.year||0,n=e.quarter||0,o=e.month||0,s=e.week||e.isoWeek||0,r=e.day||0,a=e.hour||0,l=e.minute||0,h=e.second||0,d=e.millisecond||0;this._isValid=function(t){for(var e in t)if(-1===yt.call(Me,e)||null!=t[e]&&isNaN(t[e]))return!1;for(var i=!1,n=0;n<Me.length;++n)if(t[Me[n]]){if(i)return!1;parseFloat(t[Me[n]])!==k(t[Me[n]])&&(i=!0)}return!0}(e),this._milliseconds=+d+1e3*h+6e4*l+1e3*a*60*60,this._days=+r+7*s,this._months=+o+3*n+12*i,this._data={},this._locale=ae(),this._bubble()}function Qe(t){return t instanceof Se}function Ne(t){return t<0?-1*Math.round(-1*t):Math.round(t)}function Fe(t,e){U(t,0,0,(function(){var t=this.utcOffset(),i="+";return t<0&&(t=-t,i="-"),i+X(~~(t/60),2)+e+X(~~t%60,2)}))}Fe("Z",":"),Fe("ZZ",""),dt("Z",at),dt("ZZ",at),mt(["Z","ZZ"],(function(t,e,i){i._useUTC=!0,i._tzm=Be(at,t)}));var Ze=/([\+\-]|\d\d)/gi;function Be(t,e){var i=(e||"").match(t);if(null===i)return null;var n=((i[i.length-1]||[])+"").match(Ze)||["-",0,0],o=60*n[1]+k(n[2]);return 0===o?0:"+"===n[0]?o:-o}function Oe(t,e){var i,n;return e._isUTC?(i=e.clone(),n=(w(t)||h(t)?t.valueOf():Ie(t).valueOf())-i.valueOf(),i._d.setTime(i._d.valueOf()+n),o.updateOffset(i,!1),i):Ie(t).local()}function We(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function Ae(){return!!this.isValid()&&this._isUTC&&0===this._offset}o.updateOffset=function(){};var Xe=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,Te=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;function Ye(t,e){var i,n,o,s,r,a,h=t,d=null;return Qe(t)?h={ms:t._milliseconds,d:t._days,M:t._months}:l(t)?(h={},e?h[e]=t:h.milliseconds=t):(d=Xe.exec(t))?(i="-"===d[1]?-1:1,h={y:0,d:k(d[2])*i,h:k(d[3])*i,m:k(d[4])*i,s:k(d[5])*i,ms:k(Ne(1e3*d[6]))*i}):(d=Te.exec(t))?(i="-"===d[1]?-1:1,h={y:Ee(d[2],i),M:Ee(d[3],i),w:Ee(d[4],i),d:Ee(d[5],i),h:Ee(d[6],i),m:Ee(d[7],i),s:Ee(d[8],i)}):null==h?h={}:"object"==typeof h&&("from"in h||"to"in h)&&(s=Ie(h.from),r=Ie(h.to),o=s.isValid()&&r.isValid()?(r=Oe(r,s),s.isBefore(r)?a=Re(s,r):((a=Re(r,s)).milliseconds=-a.milliseconds,a.months=-a.months),a):{milliseconds:0,months:0},(h={}).ms=o.milliseconds,h.M=o.months),n=new Se(h),Qe(t)&&u(t,"_locale")&&(n._locale=t._locale),n}function Ee(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function Re(t,e){var i={};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function Ue(t,e){return function(i,n){var o;return null===n||isNaN(+n)||(M(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),o=i,i=n,n=o),ze(this,Ye(i="string"==typeof i?+i:i,n),t),this}}function ze(t,e,i,n){var s=e._milliseconds,r=Ne(e._days),a=Ne(e._months);t.isValid()&&(n=null==n||n,a&&Mt(t,_t(t,"Month")+a*i),r&&kt(t,"Date",_t(t,"Date")+r*i),s&&t._d.setTime(t._d.valueOf()+s*i),n&&o.updateOffset(t,r||a))}Ye.fn=Se.prototype,Ye.invalid=function(){return Ye(NaN)};var Ve=Ue(1,"add"),je=Ue(-1,"subtract");function He(t,e){var i=12*(e.year()-t.year())+(e.month()-t.month()),n=t.clone().add(i,"months");return-(i+(e-n<0?(e-n)/(n-t.clone().add(i-1,"months")):(e-n)/(t.clone().add(i+1,"months")-n)))||0}function Pe(t){var e;return void 0===t?this._locale._abbr:(null!=(e=ae(t))&&(this._locale=e),this)}o.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",o.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var Je=C("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",(function(t){return void 0===t?this.localeData():this.locale(t)}));function Ke(){return this._locale}function qe(t,e){return(t%e+e)%e}function $e(t,e,i){return t<100&&t>=0?new Date(t+400,e,i)-126227808e5:new Date(t,e,i).valueOf()}function ti(t,e,i){return t<100&&t>=0?Date.UTC(t+400,e,i)-126227808e5:Date.UTC(t,e,i)}function ei(t,e){U(0,[t,t.length],0,e)}function ii(t,e,i,n,o){var s;return null==t?At(this,n,o).year:(e>(s=Xt(t,n,o))&&(e=s),ni.call(this,t,e,i,n,o))}function ni(t,e,i,n,o){var s=Wt(t,e,i,n,o),r=Bt(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}U(0,["gg",2],0,(function(){return this.weekYear()%100})),U(0,["GG",2],0,(function(){return this.isoWeekYear()%100})),ei("gggg","weekYear"),ei("ggggg","weekYear"),ei("GGGG","isoWeekYear"),ei("GGGGG","isoWeekYear"),Z("weekYear","gg"),Z("isoWeekYear","GG"),A("weekYear",1),A("isoWeekYear",1),dt("G",st),dt("g",st),dt("GG",q,H),dt("gg",q,H),dt("GGGG",it,J),dt("gggg",it,J),dt("GGGGG",nt,K),dt("ggggg",nt,K),ft(["gggg","ggggg","GGGG","GGGGG"],(function(t,e,i,n){e[n.substr(0,2)]=k(t)})),ft(["gg","GG"],(function(t,e,i,n){e[n]=o.parseTwoDigitYear(t)})),U("Q",0,"Qo","quarter"),Z("quarter","Q"),A("quarter",7),dt("Q",j),mt("Q",(function(t,e){e[1]=3*(k(t)-1)})),U("D",["DD",2],"Do","date"),Z("date","D"),A("date",9),dt("D",q),dt("DD",q,H),dt("Do",(function(t,e){return t?e._dayOfMonthOrdinalParse||e._ordinalParse:e._dayOfMonthOrdinalParseLenient})),mt(["D","DD"],2),mt("Do",(function(t,e){e[2]=k(t.match(q)[0])}));var oi=wt("Date",!0);U("DDD",["DDDD",3],"DDDo","dayOfYear"),Z("dayOfYear","DDD"),A("dayOfYear",4),dt("DDD",et),dt("DDDD",P),mt(["DDD","DDDD"],(function(t,e,i){i._dayOfYear=k(t)})),U("m",["mm",2],0,"minute"),Z("minute","m"),A("minute",14),dt("m",q),dt("mm",q,H),mt(["m","mm"],4);var si=wt("Minutes",!1);U("s",["ss",2],0,"second"),Z("second","s"),A("second",15),dt("s",q),dt("ss",q,H),mt(["s","ss"],5);var ri,ai=wt("Seconds",!1);for(U("S",0,0,(function(){return~~(this.millisecond()/100)})),U(0,["SS",2],0,(function(){return~~(this.millisecond()/10)})),U(0,["SSS",3],0,"millisecond"),U(0,["SSSS",4],0,(function(){return 10*this.millisecond()})),U(0,["SSSSS",5],0,(function(){return 100*this.millisecond()})),U(0,["SSSSSS",6],0,(function(){return 1e3*this.millisecond()})),U(0,["SSSSSSS",7],0,(function(){return 1e4*this.millisecond()})),U(0,["SSSSSSSS",8],0,(function(){return 1e5*this.millisecond()})),U(0,["SSSSSSSSS",9],0,(function(){return 1e6*this.millisecond()})),Z("millisecond","ms"),A("millisecond",16),dt("S",et,j),dt("SS",et,H),dt("SSS",et,P),ri="SSSS";ri.length<=9;ri+="S")dt(ri,ot);function li(t,e){e[6]=k(1e3*("0."+t))}for(ri="S";ri.length<=9;ri+="S")mt(ri,li);var hi=wt("Milliseconds",!1);U("z",0,0,"zoneAbbr"),U("zz",0,0,"zoneName");var di=x.prototype;function ui(t){return t}di.add=Ve,di.calendar=function(t,e){var i=t||Ie(),n=Oe(i,this).startOf("day"),s=o.calendarFormat(this,n)||"sameElse",r=e&&(S(e[s])?e[s].call(this,i):e[s]);return this.format(r||this.localeData().calendar(s,this,Ie(i)))},di.clone=function(){return new x(this)},di.diff=function(t,e,i){var n,o,s;if(!this.isValid())return NaN;if(!(n=Oe(t,this)).isValid())return NaN;switch(o=6e4*(n.utcOffset()-this.utcOffset()),e=B(e)){case"year":s=He(this,n)/12;break;case"month":s=He(this,n);break;case"quarter":s=He(this,n)/3;break;case"second":s=(this-n)/1e3;break;case"minute":s=(this-n)/6e4;break;case"hour":s=(this-n)/36e5;break;case"day":s=(this-n-o)/864e5;break;case"week":s=(this-n-o)/6048e5;break;default:s=this-n}return i?s:_(s)},di.endOf=function(t){var e;if(void 0===(t=B(t))||"millisecond"===t||!this.isValid())return this;var i=this._isUTC?ti:$e;switch(t){case"year":e=i(this.year()+1,0,1)-1;break;case"quarter":e=i(this.year(),this.month()-this.month()%3+3,1)-1;break;case"month":e=i(this.year(),this.month()+1,1)-1;break;case"week":e=i(this.year(),this.month(),this.date()-this.weekday()+7)-1;break;case"isoWeek":e=i(this.year(),this.month(),this.date()-(this.isoWeekday()-1)+7)-1;break;case"day":case"date":e=i(this.year(),this.month(),this.date()+1)-1;break;case"hour":e=this._d.valueOf(),e+=36e5-qe(e+(this._isUTC?0:6e4*this.utcOffset()),36e5)-1;break;case"minute":e=this._d.valueOf(),e+=6e4-qe(e,6e4)-1;break;case"second":e=this._d.valueOf(),e+=1e3-qe(e,1e3)-1}return this._d.setTime(e),o.updateOffset(this,!0),this},di.format=function(t){t||(t=this.isUtc()?o.defaultFormatUtc:o.defaultFormat);var e=z(this,t);return this.localeData().postformat(e)},di.from=function(t,e){return this.isValid()&&(w(t)&&t.isValid()||Ie(t).isValid())?Ye({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()},di.fromNow=function(t){return this.from(Ie(),t)},di.to=function(t,e){return this.isValid()&&(w(t)&&t.isValid()||Ie(t).isValid())?Ye({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()},di.toNow=function(t){return this.to(Ie(),t)},di.get=function(t){return S(this[t=B(t)])?this[t]():this},di.invalidAt=function(){return m(this).overflow},di.isAfter=function(t,e){var i=w(t)?t:Ie(t);return!(!this.isValid()||!i.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf())},di.isBefore=function(t,e){var i=w(t)?t:Ie(t);return!(!this.isValid()||!i.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf())},di.isBetween=function(t,e,i,n){var o=w(t)?t:Ie(t),s=w(e)?e:Ie(e);return!!(this.isValid()&&o.isValid()&&s.isValid())&&("("===(n=n||"()")[0]?this.isAfter(o,i):!this.isBefore(o,i))&&(")"===n[1]?this.isBefore(s,i):!this.isAfter(s,i))},di.isSame=function(t,e){var i,n=w(t)?t:Ie(t);return!(!this.isValid()||!n.isValid())&&("millisecond"===(e=B(e)||"millisecond")?this.valueOf()===n.valueOf():(i=n.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf()))},di.isSameOrAfter=function(t,e){return this.isSame(t,e)||this.isAfter(t,e)},di.isSameOrBefore=function(t,e){return this.isSame(t,e)||this.isBefore(t,e)},di.isValid=function(){return f(this)},di.lang=Je,di.locale=Pe,di.localeData=Ke,di.max=Le,di.min=Ce,di.parsingFlags=function(){return c({},m(this))},di.set=function(t,e){if("object"==typeof t)for(var i=function(t){var e=[];for(var i in t)e.push({unit:i,priority:W[i]});return e.sort((function(t,e){return t.priority-e.priority})),e}(t=O(t)),n=0;n<i.length;n++)this[i[n].unit](t[i[n].unit]);else if(S(this[t=B(t)]))return this[t](e);return this},di.startOf=function(t){var e;if(void 0===(t=B(t))||"millisecond"===t||!this.isValid())return this;var i=this._isUTC?ti:$e;switch(t){case"year":e=i(this.year(),0,1);break;case"quarter":e=i(this.year(),this.month()-this.month()%3,1);break;case"month":e=i(this.year(),this.month(),1);break;case"week":e=i(this.year(),this.month(),this.date()-this.weekday());break;case"isoWeek":e=i(this.year(),this.month(),this.date()-(this.isoWeekday()-1));break;case"day":case"date":e=i(this.year(),this.month(),this.date());break;case"hour":e=this._d.valueOf(),e-=qe(e+(this._isUTC?0:6e4*this.utcOffset()),36e5);break;case"minute":e=this._d.valueOf(),e-=qe(e,6e4);break;case"second":e=this._d.valueOf(),e-=qe(e,1e3)}return this._d.setTime(e),o.updateOffset(this,!0),this},di.subtract=je,di.toArray=function(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]},di.toObject=function(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}},di.toDate=function(){return new Date(this.valueOf())},di.toISOString=function(t){if(!this.isValid())return null;var e=!0!==t,i=e?this.clone().utc():this;return i.year()<0||i.year()>9999?z(i,e?"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYYYY-MM-DD[T]HH:mm:ss.SSSZ"):S(Date.prototype.toISOString)?e?this.toDate().toISOString():new Date(this.valueOf()+60*this.utcOffset()*1e3).toISOString().replace("Z",z(i,"Z")):z(i,e?"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]":"YYYY-MM-DD[T]HH:mm:ss.SSSZ")},di.inspect=function(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var t="moment",e="";this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",e="Z");var i="["+t+'("]',n=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",o=e+'[")]';return this.format(i+n+"-MM-DD[T]HH:mm:ss.SSS"+o)},di.toJSON=function(){return this.isValid()?this.toISOString():null},di.toString=function(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")},di.unix=function(){return Math.floor(this.valueOf()/1e3)},di.valueOf=function(){return this._d.valueOf()-6e4*(this._offset||0)},di.creationData=function(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}},di.year=xt,di.isLeapYear=function(){return bt(this.year())},di.weekYear=function(t){return ii.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)},di.isoWeekYear=function(t){return ii.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)},di.quarter=di.quarters=function(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)},di.month=St,di.daysInMonth=function(){return Dt(this.year(),this.month())},di.week=di.weeks=function(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")},di.isoWeek=di.isoWeeks=function(t){var e=At(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")},di.weeksInYear=function(){var t=this.localeData()._week;return Xt(this.year(),t.dow,t.doy)},di.isoWeeksInYear=function(){return Xt(this.year(),1,4)},di.date=oi,di.day=di.days=function(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=function(t,e){return"string"!=typeof t?t:isNaN(t)?"number"==typeof(t=e.weekdaysParse(t))?t:null:parseInt(t,10)}(t,this.localeData()),this.add(t-e,"d")):e},di.weekday=function(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")},di.isoWeekday=function(t){if(!this.isValid())return null!=t?this:NaN;if(null!=t){var e=function(t,e){return"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t}(t,this.localeData());return this.day(this.day()%7?e:e-7)}return this.day()||7},di.dayOfYear=function(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")},di.hour=di.hours=$t,di.minute=di.minutes=si,di.second=di.seconds=ai,di.millisecond=di.milliseconds=hi,di.utcOffset=function(t,e,i){var n,s=this._offset||0;if(!this.isValid())return null!=t?this:NaN;if(null!=t){if("string"==typeof t){if(null===(t=Be(at,t)))return this}else Math.abs(t)<16&&!i&&(t*=60);return!this._isUTC&&e&&(n=We(this)),this._offset=t,this._isUTC=!0,null!=n&&this.add(n,"m"),s!==t&&(!e||this._changeInProgress?ze(this,Ye(t-s,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,o.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?s:We(this)},di.utc=function(t){return this.utcOffset(0,t)},di.local=function(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(We(this),"m")),this},di.parseZone=function(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var t=Be(rt,this._i);null!=t?this.utcOffset(t):this.utcOffset(0,!0)}return this},di.hasAlignedHourOffset=function(t){return!!this.isValid()&&(t=t?Ie(t).utcOffset():0,(this.utcOffset()-t)%60==0)},di.isDST=function(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()},di.isLocal=function(){return!!this.isValid()&&!this._isUTC},di.isUtcOffset=function(){return!!this.isValid()&&this._isUTC},di.isUtc=Ae,di.isUTC=Ae,di.zoneAbbr=function(){return this._isUTC?"UTC":""},di.zoneName=function(){return this._isUTC?"Coordinated Universal Time":""},di.dates=C("dates accessor is deprecated. Use date instead.",oi),di.months=C("months accessor is deprecated. Use month instead",St),di.years=C("years accessor is deprecated. Use year instead",xt),di.zone=C("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",(function(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()})),di.isDSTShifted=C("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",(function(){if(!a(this._isDSTShifted))return this._isDSTShifted;var t={};if(b(t,this),(t=ke(t))._a){var e=t._isUTC?p(t._a):Ie(t._a);this._isDSTShifted=this.isValid()&&D(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}));var ci=N.prototype;function pi(t,e,i,n){var o=ae(),s=p().set(n,e);return o[i](s,t)}function mi(t,e,i){if(l(t)&&(e=t,t=void 0),t=t||"",null!=e)return pi(t,e,i,"month");var n,o=[];for(n=0;n<12;n++)o[n]=pi(t,n,i,"month");return o}function fi(t,e,i,n){"boolean"==typeof t?(l(e)&&(i=e,e=void 0),e=e||""):(i=e=t,t=!1,l(e)&&(i=e,e=void 0),e=e||"");var o,s=ae(),r=t?s._week.dow:0;if(null!=i)return pi(e,(i+r)%7,n,"day");var a=[];for(o=0;o<7;o++)a[o]=pi(e,(o+r)%7,n,"day");return a}ci.calendar=function(t,e,i){var n=this._calendar[t]||this._calendar.sameElse;return S(n)?n.call(e,i):n},ci.longDateFormat=function(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,(function(t){return t.slice(1)})),this._longDateFormat[t])},ci.invalidDate=function(){return this._invalidDate},ci.ordinal=function(t){return this._ordinal.replace("%d",t)},ci.preparse=ui,ci.postformat=ui,ci.relativeTime=function(t,e,i,n){var o=this._relativeTime[i];return S(o)?o(t,e,i,n):o.replace(/%d/i,t)},ci.pastFuture=function(t,e){var i=this._relativeTime[t>0?"future":"past"];return S(i)?i(e):i.replace(/%s/i,e)},ci.set=function(t){var e,i;for(i in t)S(e=t[i])?this[i]=e:this["_"+i]=e;this._config=t,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)},ci.months=function(t,e){return t?s(this._months)?this._months[t.month()]:this._months[(this._months.isFormat||It).test(e)?"format":"standalone"][t.month()]:s(this._months)?this._months:this._months.standalone},ci.monthsShort=function(t,e){return t?s(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[It.test(e)?"format":"standalone"][t.month()]:s(this._monthsShort)?this._monthsShort:this._monthsShort.standalone},ci.monthsParse=function(t,e,i){var n,o,s;if(this._monthsParseExact)return Gt.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),n=0;n<12;n++){if(o=p([2e3,n]),i&&!this._longMonthsParse[n]&&(this._longMonthsParse[n]=new RegExp("^"+this.months(o,"").replace(".","")+"$","i"),this._shortMonthsParse[n]=new RegExp("^"+this.monthsShort(o,"").replace(".","")+"$","i")),i||this._monthsParse[n]||(s="^"+this.months(o,"")+"|^"+this.monthsShort(o,""),this._monthsParse[n]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[n].test(t))return n;if(i&&"MMM"===e&&this._shortMonthsParse[n].test(t))return n;if(!i&&this._monthsParse[n].test(t))return n}},ci.monthsRegex=function(t){return this._monthsParseExact?(u(this,"_monthsRegex")||Ft.call(this),t?this._monthsStrictRegex:this._monthsRegex):(u(this,"_monthsRegex")||(this._monthsRegex=Nt),this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex)},ci.monthsShortRegex=function(t){return this._monthsParseExact?(u(this,"_monthsRegex")||Ft.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):(u(this,"_monthsShortRegex")||(this._monthsShortRegex=Qt),this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex)},ci.week=function(t){return At(t,this._week.dow,this._week.doy).week},ci.firstDayOfYear=function(){return this._week.doy},ci.firstDayOfWeek=function(){return this._week.dow},ci.weekdays=function(t,e){var i=s(this._weekdays)?this._weekdays:this._weekdays[t&&!0!==t&&this._weekdays.isFormat.test(e)?"format":"standalone"];return!0===t?Tt(i,this._week.dow):t?i[t.day()]:i},ci.weekdaysMin=function(t){return!0===t?Tt(this._weekdaysMin,this._week.dow):t?this._weekdaysMin[t.day()]:this._weekdaysMin},ci.weekdaysShort=function(t){return!0===t?Tt(this._weekdaysShort,this._week.dow):t?this._weekdaysShort[t.day()]:this._weekdaysShort},ci.weekdaysParse=function(t,e,i){var n,o,s;if(this._weekdaysParseExact)return Ut.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),n=0;n<7;n++){if(o=p([2e3,1]).day(n),i&&!this._fullWeekdaysParse[n]&&(this._fullWeekdaysParse[n]=new RegExp("^"+this.weekdays(o,"").replace(".","\\.?")+"$","i"),this._shortWeekdaysParse[n]=new RegExp("^"+this.weekdaysShort(o,"").replace(".","\\.?")+"$","i"),this._minWeekdaysParse[n]=new RegExp("^"+this.weekdaysMin(o,"").replace(".","\\.?")+"$","i")),this._weekdaysParse[n]||(s="^"+this.weekdays(o,"")+"|^"+this.weekdaysShort(o,"")+"|^"+this.weekdaysMin(o,""),this._weekdaysParse[n]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[n].test(t))return n;if(i&&"ddd"===e&&this._shortWeekdaysParse[n].test(t))return n;if(i&&"dd"===e&&this._minWeekdaysParse[n].test(t))return n;if(!i&&this._weekdaysParse[n].test(t))return n}},ci.weekdaysRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):(u(this,"_weekdaysRegex")||(this._weekdaysRegex=zt),this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex)},ci.weekdaysShortRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(u(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=Vt),this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)},ci.weekdaysMinRegex=function(t){return this._weekdaysParseExact?(u(this,"_weekdaysRegex")||Ht.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(u(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=jt),this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)},ci.isPM=function(t){return"p"===(t+"").toLowerCase().charAt(0)},ci.meridiem=function(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"},se("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10;return t+(1===k(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th")}}),o.lang=C("moment.lang is deprecated. Use moment.locale instead.",se),o.langData=C("moment.langData is deprecated. Use moment.localeData instead.",ae);var gi=Math.abs;function vi(t,e,i,n){var o=Ye(e,i);return t._milliseconds+=n*o._milliseconds,t._days+=n*o._days,t._months+=n*o._months,t._bubble()}function bi(t){return t<0?Math.floor(t):Math.ceil(t)}function yi(t){return 4800*t/146097}function xi(t){return 146097*t/4800}function wi(t){return function(){return this.as(t)}}var _i=wi("ms"),ki=wi("s"),Di=wi("m"),Ii=wi("h"),Ci=wi("d"),Li=wi("w"),Gi=wi("M"),Mi=wi("Q"),Si=wi("y");function Qi(t){return function(){return this.isValid()?this._data[t]:NaN}}var Ni=Qi("milliseconds"),Fi=Qi("seconds"),Zi=Qi("minutes"),Bi=Qi("hours"),Oi=Qi("days"),Wi=Qi("months"),Ai=Qi("years"),Xi=Math.round,Ti={ss:44,s:45,m:45,h:22,d:26,M:11};function Yi(t,e,i,n,o){return o.relativeTime(e||1,!!i,t,n)}function Ei(t,e,i){var n=Ye(t).abs(),o=Xi(n.as("s")),s=Xi(n.as("m")),r=Xi(n.as("h")),a=Xi(n.as("d")),l=Xi(n.as("M")),h=Xi(n.as("y")),d=o<=Ti.ss&&["s",o]||o<Ti.s&&["ss",o]||s<=1&&["m"]||s<Ti.m&&["mm",s]||r<=1&&["h"]||r<Ti.h&&["hh",r]||a<=1&&["d"]||a<Ti.d&&["dd",a]||l<=1&&["M"]||l<Ti.M&&["MM",l]||h<=1&&["y"]||["yy",h];return d[2]=e,d[3]=+t>0,d[4]=i,Yi.apply(null,d)}var Ri=Math.abs;function Ui(t){return(t>0)-(t<0)||+t}function zi(){if(!this.isValid())return this.localeData().invalidDate();var t,e,i=Ri(this._milliseconds)/1e3,n=Ri(this._days),o=Ri(this._months);t=_(i/60),e=_(t/60),i%=60,t%=60;var s=_(o/12),r=o%=12,a=n,l=e,h=t,d=i?i.toFixed(3).replace(/\.?0+$/,""):"",u=this.asSeconds();if(!u)return"P0D";var c=u<0?"-":"",p=Ui(this._months)!==Ui(u)?"-":"",m=Ui(this._days)!==Ui(u)?"-":"",f=Ui(this._milliseconds)!==Ui(u)?"-":"";return c+"P"+(s?p+s+"Y":"")+(r?p+r+"M":"")+(a?m+a+"D":"")+(l||h||d?"T":"")+(l?f+l+"H":"")+(h?f+h+"M":"")+(d?f+d+"S":"")}var Vi=Se.prototype;return Vi.isValid=function(){return this._isValid},Vi.abs=function(){var t=this._data;return this._milliseconds=gi(this._milliseconds),this._days=gi(this._days),this._months=gi(this._months),t.milliseconds=gi(t.milliseconds),t.seconds=gi(t.seconds),t.minutes=gi(t.minutes),t.hours=gi(t.hours),t.months=gi(t.months),t.years=gi(t.years),this},Vi.add=function(t,e){return vi(this,t,e,1)},Vi.subtract=function(t,e){return vi(this,t,e,-1)},Vi.as=function(t){if(!this.isValid())return NaN;var e,i,n=this._milliseconds;if("month"===(t=B(t))||"quarter"===t||"year"===t)switch(e=this._days+n/864e5,i=this._months+yi(e),t){case"month":return i;case"quarter":return i/3;case"year":return i/12}else switch(e=this._days+Math.round(xi(this._months)),t){case"week":return e/7+n/6048e5;case"day":return e+n/864e5;case"hour":return 24*e+n/36e5;case"minute":return 1440*e+n/6e4;case"second":return 86400*e+n/1e3;case"millisecond":return Math.floor(864e5*e)+n;default:throw new Error("Unknown unit "+t)}},Vi.asMilliseconds=_i,Vi.asSeconds=ki,Vi.asMinutes=Di,Vi.asHours=Ii,Vi.asDays=Ci,Vi.asWeeks=Li,Vi.asMonths=Gi,Vi.asQuarters=Mi,Vi.asYears=Si,Vi.valueOf=function(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*k(this._months/12):NaN},Vi._bubble=function(){var t,e,i,n,o,s=this._milliseconds,r=this._days,a=this._months,l=this._data;return s>=0&&r>=0&&a>=0||s<=0&&r<=0&&a<=0||(s+=864e5*bi(xi(a)+r),r=0,a=0),l.milliseconds=s%1e3,t=_(s/1e3),l.seconds=t%60,e=_(t/60),l.minutes=e%60,i=_(e/60),l.hours=i%24,r+=_(i/24),o=_(yi(r)),a+=o,r-=bi(xi(o)),n=_(a/12),a%=12,l.days=r,l.months=a,l.years=n,this},Vi.clone=function(){return Ye(this)},Vi.get=function(t){return t=B(t),this.isValid()?this[t+"s"]():NaN},Vi.milliseconds=Ni,Vi.seconds=Fi,Vi.minutes=Zi,Vi.hours=Bi,Vi.days=Oi,Vi.weeks=function(){return _(this.days()/7)},Vi.months=Wi,Vi.years=Ai,Vi.humanize=function(t){if(!this.isValid())return this.localeData().invalidDate();var e=this.localeData(),i=Ei(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)},Vi.toISOString=zi,Vi.toString=zi,Vi.toJSON=zi,Vi.locale=Pe,Vi.localeData=Ke,Vi.toIsoString=C("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",zi),Vi.lang=Je,U("X",0,0,"unix"),U("x",0,0,"valueOf"),dt("x",st),dt("X",/[+-]?\d+(\.\d{1,3})?/),mt("X",(function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))})),mt("x",(function(t,e,i){i._d=new Date(k(t))})),o.version="2.24.0",e=Ie,o.fn=di,o.min=function(){var t=[].slice.call(arguments,0);return Ge("isBefore",t)},o.max=function(){var t=[].slice.call(arguments,0);return Ge("isAfter",t)},o.now=function(){return Date.now?Date.now():+new Date},o.utc=p,o.unix=function(t){return Ie(1e3*t)},o.months=function(t,e){return mi(t,e,"months")},o.isDate=h,o.locale=se,o.invalid=g,o.duration=Ye,o.isMoment=w,o.weekdays=function(t,e,i){return fi(t,e,i,"weekdays")},o.parseZone=function(){return Ie.apply(null,arguments).parseZone()},o.localeData=ae,o.isDuration=Qe,o.monthsShort=function(t,e){return mi(t,e,"monthsShort")},o.weekdaysMin=function(t,e,i){return fi(t,e,i,"weekdaysMin")},o.defineLocale=re,o.updateLocale=function(t,e){if(null!=e){var i,n,o=te;null!=(n=oe(t))&&(o=n._config),(i=new N(e=Q(o,e))).parentLocale=ee[t],ee[t]=i,se(t)}else null!=ee[t]&&(null!=ee[t].parentLocale?ee[t]=ee[t].parentLocale:null!=ee[t]&&delete ee[t]);return ee[t]},o.locales=function(){return L(ee)},o.weekdaysShort=function(t,e,i){return fi(t,e,i,"weekdaysShort")},o.normalizeUnits=B,o.relativeTimeRounding=function(t){return void 0===t?Xi:"function"==typeof t&&(Xi=t,!0)},o.relativeTimeThreshold=function(t,e){return void 0!==Ti[t]&&(void 0===e?Ti[t]:(Ti[t]=e,"s"===t&&(Ti.ss=e-1),!0))},o.calendarFormat=function(t,e){var i=t.diff(e,"days",!0);return i<-6?"sameElse":i<-1?"lastWeek":i<0?"lastDay":i<1?"sameDay":i<2?"nextDay":i<7?"nextWeek":"sameElse"},o.prototype=di,o.HTML5_FMT={DATETIME_LOCAL:"YYYY-MM-DDTHH:mm",DATETIME_LOCAL_SECONDS:"YYYY-MM-DDTHH:mm:ss",DATETIME_LOCAL_MS:"YYYY-MM-DDTHH:mm:ss.SSS",DATE:"YYYY-MM-DD",TIME:"HH:mm",TIME_SECONDS:"HH:mm:ss",TIME_MS:"HH:mm:ss.SSS",WEEK:"GGGG-[W]WW",MONTH:"YYYY-MM"},o}()})),r=(n((function(t,e){(function(t){function e(t,e,i,n){var o={m:["eine Minute","einer Minute"],h:["eine Stunde","einer Stunde"],d:["ein Tag","einem Tag"],dd:[t+" Tage",t+" Tagen"],M:["ein Monat","einem Monat"],MM:[t+" Monate",t+" Monaten"],y:["ein Jahr","einem Jahr"],yy:[t+" Jahre",t+" Jahren"]};return e?o[i][0]:o[i][1]}t.defineLocale("de",{months:"Januar_Februar_März_April_Mai_Juni_Juli_August_September_Oktober_November_Dezember".split("_"),monthsShort:"Jan._Feb._März_Apr._Mai_Juni_Juli_Aug._Sep._Okt._Nov._Dez.".split("_"),monthsParseExact:!0,weekdays:"Sonntag_Montag_Dienstag_Mittwoch_Donnerstag_Freitag_Samstag".split("_"),weekdaysShort:"So._Mo._Di._Mi._Do._Fr._Sa.".split("_"),weekdaysMin:"So_Mo_Di_Mi_Do_Fr_Sa".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D. MMMM YYYY",LLL:"D. MMMM YYYY HH:mm",LLLL:"dddd, D. MMMM YYYY HH:mm"},calendar:{sameDay:"[heute um] LT [Uhr]",sameElse:"L",nextDay:"[morgen um] LT [Uhr]",nextWeek:"dddd [um] LT [Uhr]",lastDay:"[gestern um] LT [Uhr]",lastWeek:"[letzten] dddd [um] LT [Uhr]"},relativeTime:{future:"in %s",past:"vor %s",s:"ein paar Sekunden",ss:"%d Sekunden",m:e,mm:"%d Minuten",h:e,hh:"%d Stunden",d:e,dd:e,M:e,MM:e,y:e,yy:e},dayOfMonthOrdinalParse:/\d{1,2}\./,ordinal:"%d.",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){var e="ene._feb._mar._abr._may._jun._jul._ago._sep._oct._nov._dic.".split("_"),i="ene_feb_mar_abr_may_jun_jul_ago_sep_oct_nov_dic".split("_"),n=[/^ene/i,/^feb/i,/^mar/i,/^abr/i,/^may/i,/^jun/i,/^jul/i,/^ago/i,/^sep/i,/^oct/i,/^nov/i,/^dic/i],o=/^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre|ene\.?|feb\.?|mar\.?|abr\.?|may\.?|jun\.?|jul\.?|ago\.?|sep\.?|oct\.?|nov\.?|dic\.?)/i;t.defineLocale("es",{months:"enero_febrero_marzo_abril_mayo_junio_julio_agosto_septiembre_octubre_noviembre_diciembre".split("_"),monthsShort:function(t,n){return t?/-MMM-/.test(n)?i[t.month()]:e[t.month()]:e},monthsRegex:o,monthsShortRegex:o,monthsStrictRegex:/^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)/i,monthsShortStrictRegex:/^(ene\.?|feb\.?|mar\.?|abr\.?|may\.?|jun\.?|jul\.?|ago\.?|sep\.?|oct\.?|nov\.?|dic\.?)/i,monthsParse:n,longMonthsParse:n,shortMonthsParse:n,weekdays:"domingo_lunes_martes_miércoles_jueves_viernes_sábado".split("_"),weekdaysShort:"dom._lun._mar._mié._jue._vie._sáb.".split("_"),weekdaysMin:"do_lu_ma_mi_ju_vi_sá".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"H:mm",LTS:"H:mm:ss",L:"DD/MM/YYYY",LL:"D [de] MMMM [de] YYYY",LLL:"D [de] MMMM [de] YYYY H:mm",LLLL:"dddd, D [de] MMMM [de] YYYY H:mm"},calendar:{sameDay:function(){return"[hoy a la"+(1!==this.hours()?"s":"")+"] LT"},nextDay:function(){return"[mañana a la"+(1!==this.hours()?"s":"")+"] LT"},nextWeek:function(){return"dddd [a la"+(1!==this.hours()?"s":"")+"] LT"},lastDay:function(){return"[ayer a la"+(1!==this.hours()?"s":"")+"] LT"},lastWeek:function(){return"[el] dddd [pasado a la"+(1!==this.hours()?"s":"")+"] LT"},sameElse:"L"},relativeTime:{future:"en %s",past:"hace %s",s:"unos segundos",ss:"%d segundos",m:"un minuto",mm:"%d minutos",h:"una hora",hh:"%d horas",d:"un día",dd:"%d días",M:"un mes",MM:"%d meses",y:"un año",yy:"%d años"},dayOfMonthOrdinalParse:/\d{1,2}º/,ordinal:"%dº",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("fr",{months:"janvier_février_mars_avril_mai_juin_juillet_août_septembre_octobre_novembre_décembre".split("_"),monthsShort:"janv._févr._mars_avr._mai_juin_juil._août_sept._oct._nov._déc.".split("_"),monthsParseExact:!0,weekdays:"dimanche_lundi_mardi_mercredi_jeudi_vendredi_samedi".split("_"),weekdaysShort:"dim._lun._mar._mer._jeu._ven._sam.".split("_"),weekdaysMin:"di_lu_ma_me_je_ve_sa".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD/MM/YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[Aujourd’hui à] LT",nextDay:"[Demain à] LT",nextWeek:"dddd [à] LT",lastDay:"[Hier à] LT",lastWeek:"dddd [dernier à] LT",sameElse:"L"},relativeTime:{future:"dans %s",past:"il y a %s",s:"quelques secondes",ss:"%d secondes",m:"une minute",mm:"%d minutes",h:"une heure",hh:"%d heures",d:"un jour",dd:"%d jours",M:"un mois",MM:"%d mois",y:"un an",yy:"%d ans"},dayOfMonthOrdinalParse:/\d{1,2}(er|)/,ordinal:function(t,e){switch(e){case"D":return t+(1===t?"er":"");default:case"M":case"Q":case"DDD":case"d":return t+(1===t?"er":"e");case"w":case"W":return t+(1===t?"re":"e")}},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("it",{months:"gennaio_febbraio_marzo_aprile_maggio_giugno_luglio_agosto_settembre_ottobre_novembre_dicembre".split("_"),monthsShort:"gen_feb_mar_apr_mag_giu_lug_ago_set_ott_nov_dic".split("_"),weekdays:"domenica_lunedì_martedì_mercoledì_giovedì_venerdì_sabato".split("_"),weekdaysShort:"dom_lun_mar_mer_gio_ven_sab".split("_"),weekdaysMin:"do_lu_ma_me_gi_ve_sa".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD/MM/YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[Oggi alle] LT",nextDay:"[Domani alle] LT",nextWeek:"dddd [alle] LT",lastDay:"[Ieri alle] LT",lastWeek:function(){switch(this.day()){case 0:return"[la scorsa] dddd [alle] LT";default:return"[lo scorso] dddd [alle] LT"}},sameElse:"L"},relativeTime:{future:function(t){return(/^[0-9].+$/.test(t)?"tra":"in")+" "+t},past:"%s fa",s:"alcuni secondi",ss:"%d secondi",m:"un minuto",mm:"%d minuti",h:"un'ora",hh:"%d ore",d:"un giorno",dd:"%d giorni",M:"un mese",MM:"%d mesi",y:"un anno",yy:"%d anni"},dayOfMonthOrdinalParse:/\d{1,2}º/,ordinal:"%dº",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){t.defineLocale("ja",{months:"一月_二月_三月_四月_五月_六月_七月_八月_九月_十月_十一月_十二月".split("_"),monthsShort:"1月_2月_3月_4月_5月_6月_7月_8月_9月_10月_11月_12月".split("_"),weekdays:"日曜日_月曜日_火曜日_水曜日_木曜日_金曜日_土曜日".split("_"),weekdaysShort:"日_月_火_水_木_金_土".split("_"),weekdaysMin:"日_月_火_水_木_金_土".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"YYYY/MM/DD",LL:"YYYY年M月D日",LLL:"YYYY年M月D日 HH:mm",LLLL:"YYYY年M月D日 dddd HH:mm",l:"YYYY/MM/DD",ll:"YYYY年M月D日",lll:"YYYY年M月D日 HH:mm",llll:"YYYY年M月D日(ddd) HH:mm"},meridiemParse:/午前|午後/i,isPM:function(t){return"午後"===t},meridiem:function(t,e,i){return t<12?"午前":"午後"},calendar:{sameDay:"[今日] LT",nextDay:"[明日] LT",nextWeek:function(t){return t.week()<this.week()?"[来週]dddd LT":"dddd LT"},lastDay:"[昨日] LT",lastWeek:function(t){return this.week()<t.week()?"[先週]dddd LT":"dddd LT"},sameElse:"L"},dayOfMonthOrdinalParse:/\d{1,2}日/,ordinal:function(t,e){switch(e){case"d":case"D":case"DDD":return t+"日";default:return t}},relativeTime:{future:"%s後",past:"%s前",s:"数秒",ss:"%d秒",m:"1分",mm:"%d分",h:"1時間",hh:"%d時間",d:"1日",dd:"%d日",M:"1ヶ月",MM:"%dヶ月",y:"1年",yy:"%d年"}})})(s)})),n((function(t,e){(function(t){var e="jan._feb._mrt._apr._mei_jun._jul._aug._sep._okt._nov._dec.".split("_"),i="jan_feb_mrt_apr_mei_jun_jul_aug_sep_okt_nov_dec".split("_"),n=[/^jan/i,/^feb/i,/^maart|mrt.?$/i,/^apr/i,/^mei$/i,/^jun[i.]?$/i,/^jul[i.]?$/i,/^aug/i,/^sep/i,/^okt/i,/^nov/i,/^dec/i],o=/^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december|jan\.?|feb\.?|mrt\.?|apr\.?|ju[nl]\.?|aug\.?|sep\.?|okt\.?|nov\.?|dec\.?)/i;t.defineLocale("nl",{months:"januari_februari_maart_april_mei_juni_juli_augustus_september_oktober_november_december".split("_"),monthsShort:function(t,n){return t?/-MMM-/.test(n)?i[t.month()]:e[t.month()]:e},monthsRegex:o,monthsShortRegex:o,monthsStrictRegex:/^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december)/i,monthsShortStrictRegex:/^(jan\.?|feb\.?|mrt\.?|apr\.?|mei|ju[nl]\.?|aug\.?|sep\.?|okt\.?|nov\.?|dec\.?)/i,monthsParse:n,longMonthsParse:n,shortMonthsParse:n,weekdays:"zondag_maandag_dinsdag_woensdag_donderdag_vrijdag_zaterdag".split("_"),weekdaysShort:"zo._ma._di._wo._do._vr._za.".split("_"),weekdaysMin:"zo_ma_di_wo_do_vr_za".split("_"),weekdaysParseExact:!0,longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD-MM-YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd D MMMM YYYY HH:mm"},calendar:{sameDay:"[vandaag om] LT",nextDay:"[morgen om] LT",nextWeek:"dddd [om] LT",lastDay:"[gisteren om] LT",lastWeek:"[afgelopen] dddd [om] LT",sameElse:"L"},relativeTime:{future:"over %s",past:"%s geleden",s:"een paar seconden",ss:"%d seconden",m:"één minuut",mm:"%d minuten",h:"één uur",hh:"%d uur",d:"één dag",dd:"%d dagen",M:"één maand",MM:"%d maanden",y:"één jaar",yy:"%d jaar"},dayOfMonthOrdinalParse:/\d{1,2}(ste|de)/,ordinal:function(t){return t+(1===t||8===t||t>=20?"ste":"de")},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){var e="styczeń_luty_marzec_kwiecień_maj_czerwiec_lipiec_sierpień_wrzesień_październik_listopad_grudzień".split("_"),i="stycznia_lutego_marca_kwietnia_maja_czerwca_lipca_sierpnia_września_października_listopada_grudnia".split("_");function n(t){return t%10<5&&t%10>1&&~~(t/10)%10!=1}function o(t,e,i){var o=t+" ";switch(i){case"ss":return o+(n(t)?"sekundy":"sekund");case"m":return e?"minuta":"minutę";case"mm":return o+(n(t)?"minuty":"minut");case"h":return e?"godzina":"godzinę";case"hh":return o+(n(t)?"godziny":"godzin");case"MM":return o+(n(t)?"miesiące":"miesięcy");case"yy":return o+(n(t)?"lata":"lat")}}t.defineLocale("pl",{months:function(t,n){return t?""===n?"("+i[t.month()]+"|"+e[t.month()]+")":/D MMMM/.test(n)?i[t.month()]:e[t.month()]:e},monthsShort:"sty_lut_mar_kwi_maj_cze_lip_sie_wrz_paź_lis_gru".split("_"),weekdays:"niedziela_poniedziałek_wtorek_środa_czwartek_piątek_sobota".split("_"),weekdaysShort:"ndz_pon_wt_śr_czw_pt_sob".split("_"),weekdaysMin:"Nd_Pn_Wt_Śr_Cz_Pt_So".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY",LLL:"D MMMM YYYY HH:mm",LLLL:"dddd, D MMMM YYYY HH:mm"},calendar:{sameDay:"[Dziś o] LT",nextDay:"[Jutro o] LT",nextWeek:function(){switch(this.day()){case 0:return"[W niedzielę o] LT";case 2:return"[We wtorek o] LT";case 3:return"[W środę o] LT";case 6:return"[W sobotę o] LT";default:return"[W] dddd [o] LT"}},lastDay:"[Wczoraj o] LT",lastWeek:function(){switch(this.day()){case 0:return"[W zeszłą niedzielę o] LT";case 3:return"[W zeszłą środę o] LT";case 6:return"[W zeszłą sobotę o] LT";default:return"[W zeszły] dddd [o] LT"}},sameElse:"L"},relativeTime:{future:"za %s",past:"%s temu",s:"kilka sekund",ss:o,m:o,mm:o,h:o,hh:o,d:"1 dzień",dd:"%d dni",M:"miesiąc",MM:o,y:"rok",yy:o},dayOfMonthOrdinalParse:/\d{1,2}\./,ordinal:"%d.",week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){function e(t,e,i){var n,o;return"m"===i?e?"минута":"минуту":t+" "+(n=+t,o={ss:e?"секунда_секунды_секунд":"секунду_секунды_секунд",mm:e?"минута_минуты_минут":"минуту_минуты_минут",hh:"час_часа_часов",dd:"день_дня_дней",MM:"месяц_месяца_месяцев",yy:"год_года_лет"}[i].split("_"),n%10==1&&n%100!=11?o[0]:n%10>=2&&n%10<=4&&(n%100<10||n%100>=20)?o[1]:o[2])}var i=[/^янв/i,/^фев/i,/^мар/i,/^апр/i,/^ма[йя]/i,/^июн/i,/^июл/i,/^авг/i,/^сен/i,/^окт/i,/^ноя/i,/^дек/i];t.defineLocale("ru",{months:{format:"января_февраля_марта_апреля_мая_июня_июля_августа_сентября_октября_ноября_декабря".split("_"),standalone:"январь_февраль_март_апрель_май_июнь_июль_август_сентябрь_октябрь_ноябрь_декабрь".split("_")},monthsShort:{format:"янв._февр._мар._апр._мая_июня_июля_авг._сент._окт._нояб._дек.".split("_"),standalone:"янв._февр._март_апр._май_июнь_июль_авг._сент._окт._нояб._дек.".split("_")},weekdays:{standalone:"воскресенье_понедельник_вторник_среда_четверг_пятница_суббота".split("_"),format:"воскресенье_понедельник_вторник_среду_четверг_пятницу_субботу".split("_"),isFormat:/\[ ?[Вв] ?(?:прошлую|следующую|эту)? ?\] ?dddd/},weekdaysShort:"вс_пн_вт_ср_чт_пт_сб".split("_"),weekdaysMin:"вс_пн_вт_ср_чт_пт_сб".split("_"),monthsParse:i,longMonthsParse:i,shortMonthsParse:i,monthsRegex:/^(январ[ья]|янв\.?|феврал[ья]|февр?\.?|марта?|мар\.?|апрел[ья]|апр\.?|ма[йя]|июн[ья]|июн\.?|июл[ья]|июл\.?|августа?|авг\.?|сентябр[ья]|сент?\.?|октябр[ья]|окт\.?|ноябр[ья]|нояб?\.?|декабр[ья]|дек\.?)/i,monthsShortRegex:/^(январ[ья]|янв\.?|феврал[ья]|февр?\.?|марта?|мар\.?|апрел[ья]|апр\.?|ма[йя]|июн[ья]|июн\.?|июл[ья]|июл\.?|августа?|авг\.?|сентябр[ья]|сент?\.?|октябр[ья]|окт\.?|ноябр[ья]|нояб?\.?|декабр[ья]|дек\.?)/i,monthsStrictRegex:/^(январ[яь]|феврал[яь]|марта?|апрел[яь]|ма[яй]|июн[яь]|июл[яь]|августа?|сентябр[яь]|октябр[яь]|ноябр[яь]|декабр[яь])/i,monthsShortStrictRegex:/^(янв\.|февр?\.|мар[т.]|апр\.|ма[яй]|июн[ья.]|июл[ья.]|авг\.|сент?\.|окт\.|нояб?\.|дек\.)/i,longDateFormat:{LT:"H:mm",LTS:"H:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY г.",LLL:"D MMMM YYYY г., H:mm",LLLL:"dddd, D MMMM YYYY г., H:mm"},calendar:{sameDay:"[Сегодня, в] LT",nextDay:"[Завтра, в] LT",lastDay:"[Вчера, в] LT",nextWeek:function(t){if(t.week()===this.week())return 2===this.day()?"[Во] dddd, [в] LT":"[В] dddd, [в] LT";switch(this.day()){case 0:return"[В следующее] dddd, [в] LT";case 1:case 2:case 4:return"[В следующий] dddd, [в] LT";case 3:case 5:case 6:return"[В следующую] dddd, [в] LT"}},lastWeek:function(t){if(t.week()===this.week())return 2===this.day()?"[Во] dddd, [в] LT":"[В] dddd, [в] LT";switch(this.day()){case 0:return"[В прошлое] dddd, [в] LT";case 1:case 2:case 4:return"[В прошлый] dddd, [в] LT";case 3:case 5:case 6:return"[В прошлую] dddd, [в] LT"}},sameElse:"L"},relativeTime:{future:"через %s",past:"%s назад",s:"несколько секунд",ss:e,m:e,mm:e,h:"час",hh:e,d:"день",dd:e,M:"месяц",MM:e,y:"год",yy:e},meridiemParse:/ночи|утра|дня|вечера/i,isPM:function(t){return/^(дня|вечера)$/.test(t)},meridiem:function(t,e,i){return t<4?"ночи":t<12?"утра":t<17?"дня":"вечера"},dayOfMonthOrdinalParse:/\d{1,2}-(й|го|я)/,ordinal:function(t,e){switch(e){case"M":case"d":case"DDD":return t+"-й";case"D":return t+"-го";case"w":case"W":return t+"-я";default:return t}},week:{dow:1,doy:4}})})(s)})),n((function(t,e){(function(t){function e(t,e,i){var n,o;return"m"===i?e?"хвилина":"хвилину":"h"===i?e?"година":"годину":t+" "+(n=+t,o={ss:e?"секунда_секунди_секунд":"секунду_секунди_секунд",mm:e?"хвилина_хвилини_хвилин":"хвилину_хвилини_хвилин",hh:e?"година_години_годин":"годину_години_годин",dd:"день_дні_днів",MM:"місяць_місяці_місяців",yy:"рік_роки_років"}[i].split("_"),n%10==1&&n%100!=11?o[0]:n%10>=2&&n%10<=4&&(n%100<10||n%100>=20)?o[1]:o[2])}function i(t){return function(){return t+"о"+(11===this.hours()?"б":"")+"] LT"}}t.defineLocale("uk",{months:{format:"січня_лютого_березня_квітня_травня_червня_липня_серпня_вересня_жовтня_листопада_грудня".split("_"),standalone:"січень_лютий_березень_квітень_травень_червень_липень_серпень_вересень_жовтень_листопад_грудень".split("_")},monthsShort:"січ_лют_бер_квіт_трав_черв_лип_серп_вер_жовт_лист_груд".split("_"),weekdays:function(t,e){var i={nominative:"неділя_понеділок_вівторок_середа_четвер_п’ятниця_субота".split("_"),accusative:"неділю_понеділок_вівторок_середу_четвер_п’ятницю_суботу".split("_"),genitive:"неділі_понеділка_вівторка_середи_четверга_п’ятниці_суботи".split("_")};return!0===t?i.nominative.slice(1,7).concat(i.nominative.slice(0,1)):t?i[/(\[[ВвУу]\]) ?dddd/.test(e)?"accusative":/\[?(?:минулої|наступної)? ?\] ?dddd/.test(e)?"genitive":"nominative"][t.day()]:i.nominative},weekdaysShort:"нд_пн_вт_ср_чт_пт_сб".split("_"),weekdaysMin:"нд_пн_вт_ср_чт_пт_сб".split("_"),longDateFormat:{LT:"HH:mm",LTS:"HH:mm:ss",L:"DD.MM.YYYY",LL:"D MMMM YYYY р.",LLL:"D MMMM YYYY р., HH:mm",LLLL:"dddd, D MMMM YYYY р., HH:mm"},calendar:{sameDay:i("[Сьогодні "),nextDay:i("[Завтра "),lastDay:i("[Вчора "),nextWeek:i("[У] dddd ["),lastWeek:function(){switch(this.day()){case 0:case 3:case 5:case 6:return i("[Минулої] dddd [").call(this);case 1:case 2:case 4:return i("[Минулого] dddd [").call(this)}},sameElse:"L"},relativeTime:{future:"за %s",past:"%s тому",s:"декілька секунд",ss:e,m:e,mm:e,h:"годину",hh:e,d:"день",dd:e,M:"місяць",MM:e,y:"рік",yy:e},meridiemParse:/ночі|ранку|дня|вечора/,isPM:function(t){return/^(дня|вечора)$/.test(t)},meridiem:function(t,e,i){return t<4?"ночі":t<12?"ранку":t<17?"дня":"вечора"},dayOfMonthOrdinalParse:/\d{1,2}-(й|го)/,ordinal:function(t,e){switch(e){case"M":case"d":case"DDD":case"w":case"W":return t+"-й";case"D":return t+"-го";default:return t}},week:{dow:1,doy:7}})})(s)})),function(t){return t&&t.Math==Math&&t}),a=r("object"==typeof globalThis&&globalThis)||r("object"==typeof window&&window)||r("object"==typeof self&&self)||r("object"==typeof e&&e)||Function("return this")(),l=function(t){try{return!!t()}catch(t){return!0}},h=!l((function(){return 7!=Object.defineProperty({},1,{get:function(){return 7}})[1]})),d={}.propertyIsEnumerable,u=Object.getOwnPropertyDescriptor,c={f:u&&!d.call({1:2},1)?function(t){var e=u(this,t);return!!e&&e.enumerable}:d},p=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}},m={}.toString,f=function(t){return m.call(t).slice(8,-1)},g="".split,v=l((function(){return!Object("z").propertyIsEnumerable(0)}))?function(t){return"String"==f(t)?g.call(t,""):Object(t)}:Object,b=function(t){if(null==t)throw TypeError("Can't call method on "+t);return t},y=function(t){return v(b(t))},x=function(t){return"object"==typeof t?null!==t:"function"==typeof t},w=function(t,e){if(!x(t))return t;var i,n;if(e&&"function"==typeof(i=t.toString)&&!x(n=i.call(t)))return n;if("function"==typeof(i=t.valueOf)&&!x(n=i.call(t)))return n;if(!e&&"function"==typeof(i=t.toString)&&!x(n=i.call(t)))return n;throw TypeError("Can't convert object to primitive value")},_={}.hasOwnProperty,k=function(t,e){return _.call(t,e)},D=a.document,I=x(D)&&x(D.createElement),C=function(t){return I?D.createElement(t):{}},L=!h&&!l((function(){return 7!=Object.defineProperty(C("div"),"a",{get:function(){return 7}}).a})),G=Object.getOwnPropertyDescriptor,M={f:h?G:function(t,e){if(t=y(t),e=w(e,!0),L)try{return G(t,e)}catch(t){}if(k(t,e))return p(!c.f.call(t,e),t[e])}},S=/#|\.prototype\./,Q=function(t,e){var i=F[N(t)];return i==B||i!=Z&&("function"==typeof e?l(e):!!e)},N=Q.normalize=function(t){return String(t).replace(S,".").toLowerCase()},F=Q.data={},Z=Q.NATIVE="N",B=Q.POLYFILL="P",O=Q,W={},A=function(t){if("function"!=typeof t)throw TypeError(String(t)+" is not a function");return t},X=function(t,e,i){if(A(t),void 0===e)return t;switch(i){case 0:return function(){return t.call(e)};case 1:return function(i){return t.call(e,i)};case 2:return function(i,n){return t.call(e,i,n)};case 3:return function(i,n,o){return t.call(e,i,n,o)}}return function(){return t.apply(e,arguments)}},T=function(t){if(!x(t))throw TypeError(String(t)+" is not an object");return t},Y=Object.defineProperty,E={f:h?Y:function(t,e,i){if(T(t),e=w(e,!0),T(i),L)try{return Y(t,e,i)}catch(t){}if("get"in i||"set"in i)throw TypeError("Accessors not supported");return"value"in i&&(t[e]=i.value),t}},R=h?function(t,e,i){return E.f(t,e,p(1,i))}:function(t,e,i){return t[e]=i,t},U=M.f,z=function(t){var e=function(e,i,n){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(e);case 2:return new t(e,i)}return new t(e,i,n)}return t.apply(this,arguments)};return e.prototype=t.prototype,e},V=function(t,e){var i,n,o,s,r,l,h,d,u=t.target,c=t.global,p=t.stat,m=t.proto,f=c?a:p?a[u]:(a[u]||{}).prototype,g=c?W:W[u]||(W[u]={}),v=g.prototype;for(o in e)i=!O(c?o:u+(p?".":"#")+o,t.forced)&&f&&k(f,o),r=g[o],i&&(l=t.noTargetGet?(d=U(f,o))&&d.value:f[o]),s=i&&l?l:e[o],i&&typeof r==typeof s||(h=t.bind&&i?X(s,a):t.wrap&&i?z(s):m&&"function"==typeof s?X(Function.call,s):s,(t.sham||s&&s.sham||r&&r.sham)&&R(h,"sham",!0),g[o]=h,m&&(k(W,n=u+"Prototype")||R(W,n,{}),W[n][o]=s,t.real&&v&&!v[o]&&R(v,o,s)))};V({target:"Object",stat:!0,forced:!h,sham:!h},{defineProperty:E.f});var j=n((function(t){var e=W.Object,i=t.exports=function(t,i,n){return e.defineProperty(t,i,n)};e.defineProperty.sham&&(i.sham=!0)})),H=j,P=Math.ceil,J=Math.floor,K=function(t){return isNaN(t=+t)?0:(t>0?J:P)(t)},q=Math.min,$=function(t){return t>0?q(K(t),9007199254740991):0},tt=Math.max,et=Math.min,it=function(t,e){var i=K(t);return i<0?tt(i+e,0):et(i,e)},nt=function(t){return function(e,i,n){var o,s=y(e),r=$(s.length),a=it(n,r);if(t&&i!=i){for(;r>a;)if((o=s[a++])!=o)return!0}else for(;r>a;a++)if((t||a in s)&&s[a]===i)return t||a||0;return!t&&-1}},ot={includes:nt(!0),indexOf:nt(!1)},st={},rt=ot.indexOf,at=function(t,e){var i,n=y(t),o=0,s=[];for(i in n)!k(st,i)&&k(n,i)&&s.push(i);for(;e.length>o;)k(n,i=e[o++])&&(~rt(s,i)||s.push(i));return s},lt=["constructor","hasOwnProperty","isPrototypeOf","propertyIsEnumerable","toLocaleString","toString","valueOf"],ht=Object.keys||function(t){return at(t,lt)},dt=h?Object.defineProperties:function(t,e){T(t);for(var i,n=ht(e),o=n.length,s=0;o>s;)E.f(t,i=n[s++],e[i]);return t};V({target:"Object",stat:!0,forced:!h,sham:!h},{defineProperties:dt});var ut=n((function(t){var e=W.Object,i=t.exports=function(t,i){return e.defineProperties(t,i)};e.defineProperties.sham&&(i.sham=!0)})),ct=function(t){return"function"==typeof t?t:void 0},pt=function(t,e){return arguments.length<2?ct(W[t])||ct(a[t]):W[t]&&W[t][e]||a[t]&&a[t][e]},mt=lt.concat("length","prototype"),ft={f:Object.getOwnPropertyNames||function(t){return at(t,mt)}},gt={f:Object.getOwnPropertySymbols},vt=pt("Reflect","ownKeys")||function(t){var e=ft.f(T(t)),i=gt.f;return i?e.concat(i(t)):e},bt=function(t,e,i){var n=w(e);n in t?E.f(t,n,p(0,i)):t[n]=i};V({target:"Object",stat:!0,sham:!h},{getOwnPropertyDescriptors:function(t){for(var e,i,n=y(t),o=M.f,s=vt(n),r={},a=0;s.length>a;)void 0!==(i=o(n,e=s[a++]))&&bt(r,e,i);return r}});var yt=W.Object.getOwnPropertyDescriptors,xt=M.f,wt=l((function(){xt(1)}));V({target:"Object",stat:!0,forced:!h||wt,sham:!h},{getOwnPropertyDescriptor:function(t,e){return xt(y(t),e)}});var _t,kt=n((function(t){var e=W.Object,i=t.exports=function(t,i){return e.getOwnPropertyDescriptor(t,i)};e.getOwnPropertyDescriptor.sham&&(i.sham=!0)})),Dt=!!Object.getOwnPropertySymbols&&!l((function(){return!String(Symbol())})),It=Dt&&!Symbol.sham&&"symbol"==typeof Symbol.iterator,Ct=Array.isArray||function(t){return"Array"==f(t)},Lt=function(t){return Object(b(t))},Gt=pt("document","documentElement"),Mt=a["__core-js_shared__"]||function(t,e){try{R(a,t,e)}catch(i){a[t]=e}return e}("__core-js_shared__",{}),St=n((function(t){(t.exports=function(t,e){return Mt[t]||(Mt[t]=void 0!==e?e:{})})("versions",[]).push({version:"3.6.4",mode:"pure",copyright:"© 2020 Denis Pushkarev (zloirock.ru)"})})),Qt=0,Nt=Math.random(),Ft=function(t){return"Symbol("+String(void 0===t?"":t)+")_"+(++Qt+Nt).toString(36)},Zt=St("keys"),Bt=function(t){return Zt[t]||(Zt[t]=Ft(t))},Ot=Bt("IE_PROTO"),Wt=function(){},At=function(t){return"<script>"+t+"<\/script>"},Xt=function(){try{_t=document.domain&&new ActiveXObject("htmlfile")}catch(t){}var t,e;Xt=_t?function(t){t.write(At("")),t.close();var e=t.parentWindow.Object;return t=null,e}(_t):((e=C("iframe")).style.display="none",Gt.appendChild(e),e.src=String("javascript:"),(t=e.contentWindow.document).open(),t.write(At("document.F=Object")),t.close(),t.F);for(var i=lt.length;i--;)delete Xt.prototype[lt[i]];return Xt()};st[Ot]=!0;var Tt=Object.create||function(t,e){var i;return null!==t?(Wt.prototype=T(t),i=new Wt,Wt.prototype=null,i[Ot]=t):i=Xt(),void 0===e?i:dt(i,e)},Yt=ft.f,Et={}.toString,Rt="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],Ut={f:function(t){return Rt&&"[object Window]"==Et.call(t)?function(t){try{return Yt(t)}catch(t){return Rt.slice()}}(t):Yt(y(t))}},zt=function(t,e,i,n){n&&n.enumerable?t[e]=i:R(t,e,i)},Vt=St("wks"),jt=a.Symbol,Ht=It?jt:jt&&jt.withoutSetter||Ft,Pt=function(t){return k(Vt,t)||(Dt&&k(jt,t)?Vt[t]=jt[t]:Vt[t]=Ht("Symbol."+t)),Vt[t]},Jt={f:Pt},Kt=E.f,qt=function(t){var e=W.Symbol||(W.Symbol={});k(e,t)||Kt(e,t,{value:Jt.f(t)})},$t={};$t[Pt("toStringTag")]="z";var te="[object z]"===String($t),ee=Pt("toStringTag"),ie="Arguments"==f(function(){return arguments}()),ne=te?f:function(t){var e,i,n;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(i=function(t,e){try{return t[e]}catch(t){}}(e=Object(t),ee))?i:ie?f(e):"Object"==(n=f(e))&&"function"==typeof e.callee?"Arguments":n},oe=te?{}.toString:function(){return"[object "+ne(this)+"]"},se=E.f,re=Pt("toStringTag"),ae=function(t,e,i,n){if(t){var o=i?t:t.prototype;k(o,re)||se(o,re,{configurable:!0,value:e}),n&&!te&&R(o,"toString",oe)}},le=Function.toString;"function"!=typeof Mt.inspectSource&&(Mt.inspectSource=function(t){return le.call(t)});var he,de,ue,ce=Mt.inspectSource,pe=a.WeakMap,me="function"==typeof pe&&/native code/.test(ce(pe)),fe=a.WeakMap;if(me){var ge=new fe,ve=ge.get,be=ge.has,ye=ge.set;he=function(t,e){return ye.call(ge,t,e),e},de=function(t){return ve.call(ge,t)||{}},ue=function(t){return be.call(ge,t)}}else{var xe=Bt("state");st[xe]=!0,he=function(t,e){return R(t,xe,e),e},de=function(t){return k(t,xe)?t[xe]:{}},ue=function(t){return k(t,xe)}}var we={set:he,get:de,has:ue,enforce:function(t){return ue(t)?de(t):he(t,{})},getterFor:function(t){return function(e){var i;if(!x(e)||(i=de(e)).type!==t)throw TypeError("Incompatible receiver, "+t+" required");return i}}},_e=Pt("species"),ke=function(t,e){var i;return Ct(t)&&("function"!=typeof(i=t.constructor)||i!==Array&&!Ct(i.prototype)?x(i)&&null===(i=i[_e])&&(i=void 0):i=void 0),new(void 0===i?Array:i)(0===e?0:e)},De=[].push,Ie=function(t){var e=1==t,i=2==t,n=3==t,o=4==t,s=6==t,r=5==t||s;return function(a,l,h,d){for(var u,c,p=Lt(a),m=v(p),f=X(l,h,3),g=$(m.length),b=0,y=d||ke,x=e?y(a,g):i?y(a,0):void 0;g>b;b++)if((r||b in m)&&(c=f(u=m[b],b,p),t))if(e)x[b]=c;else if(c)switch(t){case 3:return!0;case 5:return u;case 6:return b;case 2:De.call(x,u)}else if(o)return!1;return s?-1:n||o?o:x}},Ce={forEach:Ie(0),map:Ie(1),filter:Ie(2),some:Ie(3),every:Ie(4),find:Ie(5),findIndex:Ie(6)},Le=Ce.forEach,Ge=Bt("hidden"),Me=Pt("toPrimitive"),Se=we.set,Qe=we.getterFor("Symbol"),Ne=Object.prototype,Fe=a.Symbol,Ze=pt("JSON","stringify"),Be=M.f,Oe=E.f,We=Ut.f,Ae=c.f,Xe=St("symbols"),Te=St("op-symbols"),Ye=St("string-to-symbol-registry"),Ee=St("symbol-to-string-registry"),Re=St("wks"),Ue=a.QObject,ze=!Ue||!Ue.prototype||!Ue.prototype.findChild,Ve=h&&l((function(){return 7!=Tt(Oe({},"a",{get:function(){return Oe(this,"a",{value:7}).a}})).a}))?function(t,e,i){var n=Be(Ne,e);n&&delete Ne[e],Oe(t,e,i),n&&t!==Ne&&Oe(Ne,e,n)}:Oe,je=function(t,e){var i=Xe[t]=Tt(Fe.prototype);return Se(i,{type:"Symbol",tag:t,description:e}),h||(i.description=e),i},He=It?function(t){return"symbol"==typeof t}:function(t){return Object(t)instanceof Fe},Pe=function(t,e,i){t===Ne&&Pe(Te,e,i),T(t);var n=w(e,!0);return T(i),k(Xe,n)?(i.enumerable?(k(t,Ge)&&t[Ge][n]&&(t[Ge][n]=!1),i=Tt(i,{enumerable:p(0,!1)})):(k(t,Ge)||Oe(t,Ge,p(1,{})),t[Ge][n]=!0),Ve(t,n,i)):Oe(t,n,i)},Je=function(t,e){T(t);var i=y(e),n=ht(i).concat(ti(i));return Le(n,(function(e){h&&!Ke.call(i,e)||Pe(t,e,i[e])})),t},Ke=function(t){var e=w(t,!0),i=Ae.call(this,e);return!(this===Ne&&k(Xe,e)&&!k(Te,e))&&(!(i||!k(this,e)||!k(Xe,e)||k(this,Ge)&&this[Ge][e])||i)},qe=function(t,e){var i=y(t),n=w(e,!0);if(i!==Ne||!k(Xe,n)||k(Te,n)){var o=Be(i,n);return!o||!k(Xe,n)||k(i,Ge)&&i[Ge][n]||(o.enumerable=!0),o}},$e=function(t){var e=We(y(t)),i=[];return Le(e,(function(t){k(Xe,t)||k(st,t)||i.push(t)})),i},ti=function(t){var e=t===Ne,i=We(e?Te:y(t)),n=[];return Le(i,(function(t){!k(Xe,t)||e&&!k(Ne,t)||n.push(Xe[t])})),n};if(Dt||(zt((Fe=function(){if(this instanceof Fe)throw TypeError("Symbol is not a constructor");var t=arguments.length&&void 0!==arguments[0]?String(arguments[0]):void 0,e=Ft(t),i=function(t){this===Ne&&i.call(Te,t),k(this,Ge)&&k(this[Ge],e)&&(this[Ge][e]=!1),Ve(this,e,p(1,t))};return h&&ze&&Ve(Ne,e,{configurable:!0,set:i}),je(e,t)}).prototype,"toString",(function(){return Qe(this).tag})),zt(Fe,"withoutSetter",(function(t){return je(Ft(t),t)})),c.f=Ke,E.f=Pe,M.f=qe,ft.f=Ut.f=$e,gt.f=ti,Jt.f=function(t){return je(Pt(t),t)},h&&Oe(Fe.prototype,"description",{configurable:!0,get:function(){return Qe(this).description}})),V({global:!0,wrap:!0,forced:!Dt,sham:!Dt},{Symbol:Fe}),Le(ht(Re),(function(t){qt(t)})),V({target:"Symbol",stat:!0,forced:!Dt},{for:function(t){var e=String(t);if(k(Ye,e))return Ye[e];var i=Fe(e);return Ye[e]=i,Ee[i]=e,i},keyFor:function(t){if(!He(t))throw TypeError(t+" is not a symbol");if(k(Ee,t))return Ee[t]},useSetter:function(){ze=!0},useSimple:function(){ze=!1}}),V({target:"Object",stat:!0,forced:!Dt,sham:!h},{create:function(t,e){return void 0===e?Tt(t):Je(Tt(t),e)},defineProperty:Pe,defineProperties:Je,getOwnPropertyDescriptor:qe}),V({target:"Object",stat:!0,forced:!Dt},{getOwnPropertyNames:$e,getOwnPropertySymbols:ti}),V({target:"Object",stat:!0,forced:l((function(){gt.f(1)}))},{getOwnPropertySymbols:function(t){return gt.f(Lt(t))}}),Ze){var ei=!Dt||l((function(){var t=Fe();return"[null]"!=Ze([t])||"{}"!=Ze({a:t})||"{}"!=Ze(Object(t))}));V({target:"JSON",stat:!0,forced:ei},{stringify:function(t,e,i){for(var n,o=[t],s=1;arguments.length>s;)o.push(arguments[s++]);if(n=e,(x(e)||void 0!==t)&&!He(t))return Ct(e)||(e=function(t,e){if("function"==typeof n&&(e=n.call(this,t,e)),!He(e))return e}),o[1]=e,Ze.apply(null,o)}})}Fe.prototype[Me]||R(Fe.prototype,Me,Fe.prototype.valueOf),ae(Fe,"Symbol"),st[Ge]=!0;var ii,ni,oi,si=W.Object.getOwnPropertySymbols,ri={},ai=!l((function(){function t(){}return t.prototype.constructor=null,Object.getPrototypeOf(new t)!==t.prototype})),li=Bt("IE_PROTO"),hi=Object.prototype,di=ai?Object.getPrototypeOf:function(t){return t=Lt(t),k(t,li)?t[li]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?hi:null},ui=(Pt("iterator"),!1);[].keys&&("next"in(oi=[].keys())?(ni=di(di(oi)))!==Object.prototype&&(ii=ni):ui=!0),null==ii&&(ii={});var ci={IteratorPrototype:ii,BUGGY_SAFARI_ITERATORS:ui},pi=ci.IteratorPrototype,mi=function(){return this},fi=Object.setPrototypeOf||("__proto__"in{}?function(){var t,e=!1,i={};try{(t=Object.getOwnPropertyDescriptor(Object.prototype,"__proto__").set).call(i,[]),e=i instanceof Array}catch(t){}return function(i,n){return T(i),function(t){if(!x(t)&&null!==t)throw TypeError("Can't set "+String(t)+" as a prototype")}(n),e?t.call(i,n):i.__proto__=n,i}}():void 0),gi=ci.IteratorPrototype,vi=ci.BUGGY_SAFARI_ITERATORS,bi=Pt("iterator"),yi=function(){return this},xi=function(t,e,i,n,o,s,r){!function(t,e,i){var n=e+" Iterator";t.prototype=Tt(pi,{next:p(1,i)}),ae(t,n,!1,!0),ri[n]=mi}(i,e,n);var a,l,h,d=function(t){if(t===o&&g)return g;if(!vi&&t in m)return m[t];switch(t){case"keys":case"values":case"entries":return function(){return new i(this,t)}}return function(){return new i(this)}},u=e+" Iterator",c=!1,m=t.prototype,f=m[bi]||m["@@iterator"]||o&&m[o],g=!vi&&f||d(o),v="Array"==e&&m.entries||f;if(v&&(a=di(v.call(new t)),gi!==Object.prototype&&a.next&&(ae(a,u,!0,!0),ri[u]=yi)),"values"==o&&f&&"values"!==f.name&&(c=!0,g=function(){return f.call(this)}),r&&m[bi]!==g&&R(m,bi,g),ri[e]=g,o)if(l={values:d("values"),keys:s?g:d("keys"),entries:d("entries")},r)for(h in l)!vi&&!c&&h in m||zt(m,h,l[h]);else V({target:e,proto:!0,forced:vi||c},l);return l},wi=we.set,_i=we.getterFor("Array Iterator");xi(Array,"Array",(function(t,e){wi(this,{type:"Array Iterator",target:y(t),index:0,kind:e})}),(function(){var t=_i(this),e=t.target,i=t.kind,n=t.index++;return!e||n>=e.length?(t.target=void 0,{value:void 0,done:!0}):"keys"==i?{value:n,done:!1}:"values"==i?{value:e[n],done:!1}:{value:[n,e[n]],done:!1}}),"values");ri.Arguments=ri.Array;var ki=Pt("toStringTag");for(var Di in{CSSRuleList:0,CSSStyleDeclaration:0,CSSValueList:0,ClientRectList:0,DOMRectList:0,DOMStringList:0,DOMTokenList:1,DataTransferItemList:0,FileList:0,HTMLAllCollection:0,HTMLCollection:0,HTMLFormElement:0,HTMLSelectElement:0,MediaList:0,MimeTypeArray:0,NamedNodeMap:0,NodeList:1,PaintRequestList:0,Plugin:0,PluginArray:0,SVGLengthList:0,SVGNumberList:0,SVGPathSegList:0,SVGPointList:0,SVGStringList:0,SVGTransformList:0,SourceBufferList:0,StyleSheetList:0,TextTrackCueList:0,TextTrackList:0,TouchList:0}){var Ii=a[Di],Ci=Ii&&Ii.prototype;Ci&&ne(Ci)!==ki&&R(Ci,ki,Di),ri[Di]=ri.Array}var Li=function(t){return function(e,i){var n,o,s=String(b(e)),r=K(i),a=s.length;return r<0||r>=a?t?"":void 0:(n=s.charCodeAt(r))<55296||n>56319||r+1===a||(o=s.charCodeAt(r+1))<56320||o>57343?t?s.charAt(r):n:t?s.slice(r,r+2):o-56320+(n-55296<<10)+65536}},Gi={codeAt:Li(!1),charAt:Li(!0)}.charAt,Mi=we.set,Si=we.getterFor("String Iterator");xi(String,"String",(function(t){Mi(this,{type:"String Iterator",string:String(t),index:0})}),(function(){var t,e=Si(this),i=e.string,n=e.index;return n>=i.length?{value:void 0,done:!0}:(t=Gi(i,n),e.index+=t.length,{value:t,done:!1})}));var Qi=Pt("iterator"),Ni=function(t){if(null!=t)return t[Qi]||t["@@iterator"]||ri[ne(t)]},Fi=function(t){var e=Ni(t);if("function"!=typeof e)throw TypeError(String(t)+" is not iterable");return T(e.call(t))};V({target:"Object",stat:!0,sham:!h},{create:Tt});var Zi=W.Object,Bi=function(t,e){return Zi.create(t,e)},Oi=Bi,Wi=j;var Ai=function(t,e,i){return e in t?Wi(t,e,{value:i,enumerable:!0,configurable:!0,writable:!0}):t[e]=i,t},Xi=l((function(){ht(1)}));V({target:"Object",stat:!0,forced:Xi},{keys:function(t){return ht(Lt(t))}});var Ti,Yi=W.Object.keys,Ei="\t\n\v\f\r                　\u2028\u2029\ufeff",Ri="["+Ei+"]",Ui=RegExp("^"+Ri+Ri+"*"),zi=RegExp(Ri+Ri+"*$"),Vi=function(t){return function(e){var i=String(b(e));return 1&t&&(i=i.replace(Ui,"")),2&t&&(i=i.replace(zi,"")),i}},ji={start:Vi(1),end:Vi(2),trim:Vi(3)},Hi=ji.trim;V({target:"String",proto:!0,forced:(Ti="trim",l((function(){return!!Ei[Ti]()||"​…᠎"!="​…᠎"[Ti]()||Ei[Ti].name!==Ti})))},{trim:function(){return Hi(this)}});var Pi=function(t){return W[t+"Prototype"]},Ji=Pi("String").trim,Ki=String.prototype,qi=function(t){var e=t.trim;return"string"==typeof t||t===Ki||t instanceof String&&e===Ki.trim?Ji:e},$i=function(t,e){var i=[][t];return!!i&&l((function(){i.call(null,e||function(){throw 1},1)}))},tn=Object.defineProperty,en={},nn=function(t){throw t},on=function(t,e){if(k(en,t))return en[t];e||(e={});var i=[][t],n=!!k(e,"ACCESSORS")&&e.ACCESSORS,o=k(e,0)?e[0]:nn,s=k(e,1)?e[1]:void 0;return en[t]=!!i&&!l((function(){if(n&&!h)return!0;var t={length:-1};n?tn(t,1,{enumerable:!0,get:nn}):t[1]=1,i.call(t,o,s)}))},sn=Ce.forEach,rn=$i("forEach"),an=on("forEach"),ln=rn&&an?[].forEach:function(t){return sn(this,t,arguments.length>1?arguments[1]:void 0)};V({target:"Array",proto:!0,forced:[].forEach!=ln},{forEach:ln});var hn,dn,un=Pi("Array").forEach,cn=Array.prototype,pn={DOMTokenList:!0,NodeList:!0},mn=function(t){var e=t.forEach;return t===cn||t instanceof Array&&e===cn.forEach||pn.hasOwnProperty(ne(t))?un:e},fn=pt("navigator","userAgent")||"",gn=a.process,vn=gn&&gn.versions,bn=vn&&vn.v8;bn?dn=(hn=bn.split("."))[0]+hn[1]:fn&&(!(hn=fn.match(/Edge\/(\d+)/))||hn[1]>=74)&&(hn=fn.match(/Chrome\/(\d+)/))&&(dn=hn[1]);var yn=dn&&+dn,xn=Pt("species"),wn=function(t){return yn>=51||!l((function(){var e=[];return(e.constructor={})[xn]=function(){return{foo:1}},1!==e[t](Boolean).foo}))},_n=Ce.map,kn=wn("map"),Dn=on("map");V({target:"Array",proto:!0,forced:!kn||!Dn},{map:function(t){return _n(this,t,arguments.length>1?arguments[1]:void 0)}});var In=Pi("Array").map,Cn=Array.prototype,Ln=function(t){var e=t.map;return t===Cn||t instanceof Array&&e===Cn.map?In:e},Gn=ji.trim,Mn=a.parseInt,Sn=/^[+-]?0[Xx]/,Qn=8!==Mn(Ei+"08")||22!==Mn(Ei+"0x16")?function(t,e){var i=Gn(String(t));return Mn(i,e>>>0||(Sn.test(i)?16:10))}:Mn;V({global:!0,forced:parseInt!=Qn},{parseInt:Qn});var Nn=W.parseInt,Fn=c.f,Zn=function(t){return function(e){for(var i,n=y(e),o=ht(n),s=o.length,r=0,a=[];s>r;)i=o[r++],h&&!Fn.call(n,i)||a.push(t?[i,n[i]]:n[i]);return a}},Bn={entries:Zn(!0),values:Zn(!1)}.values;V({target:"Object",stat:!0},{values:function(t){return Bn(t)}});var On=W.Object.values,Wn=Ce.filter,An=wn("filter"),Xn=on("filter");V({target:"Array",proto:!0,forced:!An||!Xn},{filter:function(t){return Wn(this,t,arguments.length>1?arguments[1]:void 0)}});var Tn=Pi("Array").filter,Yn=Array.prototype,En=function(t){var e=t.filter;return t===Yn||t instanceof Array&&e===Yn.filter?Tn:e},Rn=Pt("isConcatSpreadable"),Un=yn>=51||!l((function(){var t=[];return t[Rn]=!1,t.concat()[0]!==t})),zn=wn("concat"),Vn=function(t){if(!x(t))return!1;var e=t[Rn];return void 0!==e?!!e:Ct(t)};V({target:"Array",proto:!0,forced:!Un||!zn},{concat:function(t){var e,i,n,o,s,r=Lt(this),a=ke(r,0),l=0;for(e=-1,n=arguments.length;e<n;e++)if(s=-1===e?r:arguments[e],Vn(s)){if(l+(o=$(s.length))>9007199254740991)throw TypeError("Maximum allowed index exceeded");for(i=0;i<o;i++,l++)i in s&&bt(a,l,s[i])}else{if(l>=9007199254740991)throw TypeError("Maximum allowed index exceeded");bt(a,l++,s)}return a.length=l,a}});var jn=Pi("Array").concat,Hn=Array.prototype,Pn=function(t){var e=t.concat;return t===Hn||t instanceof Array&&e===Hn.concat?jn:e};V({target:"Array",stat:!0},{isArray:Ct});var Jn=W.Array.isArray,Kn=Jn;var qn=function(t){if(Kn(t)){for(var e=0,i=new Array(t.length);e<t.length;e++)i[e]=t[e];return i}},$n=function(t,e,i,n){try{return n?e(T(i)[0],i[1]):e(i)}catch(e){var o=t.return;throw void 0!==o&&T(o.call(t)),e}},to=Pt("iterator"),eo=Array.prototype,io=function(t){return void 0!==t&&(ri.Array===t||eo[to]===t)},no=Pt("iterator"),oo=!1;try{var so=0,ro={next:function(){return{done:!!so++}},return:function(){oo=!0}};ro[no]=function(){return this},Array.from(ro,(function(){throw 2}))}catch(t){}var ao=!function(t,e){if(!e&&!oo)return!1;var i=!1;try{var n={};n[no]=function(){return{next:function(){return{done:i=!0}}}},t(n)}catch(t){}return i}((function(t){Array.from(t)}));V({target:"Array",stat:!0,forced:ao},{from:function(t){var e,i,n,o,s,r,a=Lt(t),l="function"==typeof this?this:Array,h=arguments.length,d=h>1?arguments[1]:void 0,u=void 0!==d,c=Ni(a),p=0;if(u&&(d=X(d,h>2?arguments[2]:void 0,2)),null==c||l==Array&&io(c))for(i=new l(e=$(a.length));e>p;p++)r=u?d(a[p],p):a[p],bt(i,p,r);else for(s=(o=c.call(a)).next,i=new l;!(n=s.call(o)).done;p++)r=u?$n(o,d,[n.value,p],!0):n.value,bt(i,p,r);return i.length=p,i}});var lo=W.Array.from,ho=Pt("iterator"),uo=function(t){var e=Object(t);return void 0!==e[ho]||"@@iterator"in e||ri.hasOwnProperty(ne(e))};var co=function(t){if(uo(Object(t))||"[object Arguments]"===Object.prototype.toString.call(t))return lo(t)};var po=function(){throw new TypeError("Invalid attempt to spread non-iterable instance")};var mo=function(t){return qn(t)||co(t)||po()},fo=wn("slice"),go=on("slice",{ACCESSORS:!0,0:0,1:2}),vo=Pt("species"),bo=[].slice,yo=Math.max;V({target:"Array",proto:!0,forced:!fo||!go},{slice:function(t,e){var i,n,o,s=y(this),r=$(s.length),a=it(t,r),l=it(void 0===e?r:e,r);if(Ct(s)&&("function"!=typeof(i=s.constructor)||i!==Array&&!Ct(i.prototype)?x(i)&&null===(i=i[vo])&&(i=void 0):i=void 0,i===Array||void 0===i))return bo.call(s,a,l);for(n=new(void 0===i?Array:i)(yo(l-a,0)),o=0;a<l;a++,o++)a in s&&bt(n,o,s[a]);return n.length=o,n}});var xo=Pi("Array").slice,wo=Array.prototype,_o=function(t){var e=t.slice;return t===wo||t instanceof Array&&e===wo.slice?xo:e},ko=l((function(){di(1)}));V({target:"Object",stat:!0,forced:ko,sham:!ai},{getPrototypeOf:function(t){return di(Lt(t))}});var Do=W.Object.getPrototypeOf,Io=Do,Co=ot.indexOf,Lo=[].indexOf,Go=!!Lo&&1/[1].indexOf(1,-0)<0,Mo=$i("indexOf"),So=on("indexOf",{ACCESSORS:!0,1:0});V({target:"Array",proto:!0,forced:Go||!Mo||!So},{indexOf:function(t){return Go?Lo.apply(this,arguments)||0:Co(this,t,arguments.length>1?arguments[1]:void 0)}});var Qo=Pi("Array").indexOf,No=Array.prototype,Fo=function(t){var e=t.indexOf;return t===No||t instanceof Array&&e===No.indexOf?Qo:e},Zo=Jn,Bo=Object.assign,Oo=Object.defineProperty,Wo=!Bo||l((function(){if(h&&1!==Bo({b:1},Bo(Oo({},"a",{enumerable:!0,get:function(){Oo(this,"b",{value:3,enumerable:!1})}}),{b:2})).b)return!0;var t={},e={},i=Symbol();return t[i]=7,"abcdefghijklmnopqrst".split("").forEach((function(t){e[t]=t})),7!=Bo({},t)[i]||"abcdefghijklmnopqrst"!=ht(Bo({},e)).join("")}))?function(t,e){for(var i=Lt(t),n=arguments.length,o=1,s=gt.f,r=c.f;n>o;)for(var a,l=v(arguments[o++]),d=s?ht(l).concat(s(l)):ht(l),u=d.length,p=0;u>p;)a=d[p++],h&&!r.call(l,a)||(i[a]=l[a]);return i}:Bo;V({target:"Object",stat:!0,forced:Object.assign!==Wo},{assign:Wo});var Ao=W.Object.assign;qt("iterator");var Xo=Jt.f("iterator"),To=Xo,Yo=Object.freeze({__proto__:null});qt("asyncIterator");var Eo=Object.freeze({__proto__:null});qt("hasInstance"),qt("isConcatSpreadable"),qt("match"),qt("matchAll"),qt("replace"),qt("search"),qt("species"),qt("split"),qt("toPrimitive"),qt("toStringTag"),qt("unscopables"),ae(Math,"Math",!0),ae(a.JSON,"JSON",!0),o(Yo),o(Eo);var Ro=W.Symbol;qt("asyncDispose"),qt("dispose"),qt("observable"),qt("patternMatch"),qt("replaceAll");var Uo=Ro,zo=n((function(t){function e(i){return t.exports=e="function"==typeof Uo&&"symbol"==typeof To?function(t){return typeof t}:function(t){return t&&"function"==typeof Uo&&t.constructor===Uo&&t!==Uo.prototype?"symbol":typeof t},e(i)}t.exports=e}));var Vo=function(t){if(Kn(t))return t};var jo=function(t,e){if(uo(Object(t))||"[object Arguments]"===Object.prototype.toString.call(t)){var i=[],n=!0,o=!1,s=void 0;try{for(var r,a=Fi(t);!(n=(r=a.next()).done)&&(i.push(r.value),!e||i.length!==e);n=!0);}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return i}};var Ho=function(){throw new TypeError("Invalid attempt to destructure non-iterable instance")};var Po=function(t,e){return Vo(t)||jo(t,e)||Ho()};V({target:"Date",stat:!0},{now:function(){return(new Date).getTime()}});var Jo=W.Date.now;function Ko(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}function qo(t){var e=function(){for(var t=$o(),e=t(" "),i=t(" "),n=t(" "),o=0;o<arguments.length;o++)(e-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(e+=1),(i-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(i+=1),(n-=t(o<0||arguments.length<=o?void 0:arguments[o]))<0&&(n+=1);return[e,i,n]}(t),i=Po(e,3),n=i[0],o=i[1],s=i[2],r=1,a=function(){var t=2091639*n+2.3283064365386963e-10*r;return n=o,o=s,s=t-(r=0|t)};return a.uint32=function(){return 4294967296*a()},a.fract53=function(){return a()+11102230246251565e-32*(2097152*a()|0)},a.algorithm="Alea",a.seed=t,a.version="0.9",a}function $o(){var t=4022871197;return function(e){for(var i=e.toString(),n=0;n<i.length;n++){var o=.02519603282416938*(t+=i.charCodeAt(n));o-=t=o>>>0,t=(o*=t)>>>0,t+=4294967296*(o-=t)}return 2.3283064365386963e-10*(t>>>0)}}var ts=/^\/?Date\((-?\d+)/i,es=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i,is=/^#?([a-f\d])([a-f\d])([a-f\d])$/i,ns=/^rgb\( *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *\)$/i,os=/^rgba\( *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *(1?\d{1,2}|2[0-4]\d|25[0-5]) *, *([01]|0?\.\d+) *\)$/i;function ss(t){return t instanceof Number||"number"==typeof t}function rs(t){return t instanceof String||"string"==typeof t}function as(t){return"object"===zo(t)&&null!==t}function ls(t,e,i,n){var o=!1;!0===n&&(o=null===e[i]&&void 0!==t[i]),o?delete t[i]:t[i]=e[i]}var hs=Ao;function ds(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];for(var o in e)if(Object.prototype.hasOwnProperty.call(e,o)||!0===i)if("object"===zo(e[o])&&null!==e[o]&&Io(e[o])===Object.prototype)void 0===t[o]?t[o]=ds({},e[o],i):"object"===zo(t[o])&&null!==t[o]&&Io(t[o])===Object.prototype?ds(t[o],e[o],i):ls(t,e,o,n);else if(Zo(e[o])){var s;t[o]=_o(s=e[o]).call(s)}else ls(t,e,o,n);return t}function us(t){var e=zo(t);return"object"===e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Zo(t)?"Array":t instanceof Date?"Date":"Object":"number"===e?"Number":"boolean"===e?"Boolean":"string"===e?"String":void 0===e?"undefined":e}var cs=On;var ps={asBoolean:function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},asNumber:function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},asString:function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},asSize:function(t,e){return"function"==typeof t&&(t=t()),rs(t)?t:ss(t)?t+"px":e||null},asElement:function(t,e){return"function"==typeof t&&(t=t()),t||e||null}};function ms(t){var e;switch(t.length){case 3:case 4:return(e=is.exec(t))?{r:Nn(e[1]+e[1],16),g:Nn(e[2]+e[2],16),b:Nn(e[3]+e[3],16)}:null;case 6:case 7:return(e=es.exec(t))?{r:Nn(e[1],16),g:Nn(e[2],16),b:Nn(e[3],16)}:null;default:return null}}function fs(t,e,i){var n;return"#"+_o(n=((1<<24)+(t<<16)+(e<<8)+i).toString(16)).call(n,1)}function gs(t,e,i){t/=255,e/=255,i/=255;var n=Math.min(t,Math.min(e,i)),o=Math.max(t,Math.max(e,i));return n===o?{h:0,s:0,v:n}:{h:60*((t===n?3:i===n?1:5)-(t===n?e-i:i===n?t-e:i-t)/(o-n))/360,s:(o-n)/o,v:o}}var vs=function(t){var e,i={};return mn(e=t.split(";")).call(e,(function(t){if(""!=qi(t).call(t)){var e,n,o=t.split(":"),s=qi(e=o[0]).call(e),r=qi(n=o[1]).call(n);i[s]=r}})),i},bs=function(t){var e;return Ln(e=Yi(t)).call(e,(function(e){return e+": "+t[e]})).join("; ")};function ys(t,e,i){var n,o,s,r=Math.floor(6*t),a=6*t-r,l=i*(1-e),h=i*(1-a*e),d=i*(1-(1-a)*e);switch(r%6){case 0:n=i,o=d,s=l;break;case 1:n=h,o=i,s=l;break;case 2:n=l,o=i,s=d;break;case 3:n=l,o=h,s=i;break;case 4:n=d,o=l,s=i;break;case 5:n=i,o=l,s=h}return{r:Math.floor(255*n),g:Math.floor(255*o),b:Math.floor(255*s)}}function xs(t,e,i){var n=ys(t,e,i);return fs(n.r,n.g,n.b)}function ws(t){var e=ms(t);if(!e)throw new TypeError("'".concat(t,"' is not a valid color."));return gs(e.r,e.g,e.b)}function _s(t){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t)}function ks(t){return ns.test(t)}function Ds(t){if(null===t||"object"!==zo(t))return null;if(t instanceof Element)return t;var e=Oi(t);for(var i in t)Object.prototype.hasOwnProperty.call(t,i)&&"object"==zo(t[i])&&(e[i]=Ds(t[i]));return e}var Is=Object.freeze({__proto__:null,Alea:
-/**
-	 * vis-util
-	 * https://github.com/visjs/vis-util
-	 *
-	 * utilitie collection for visjs
-	 *
-	 * @version 4.0.0
-	 * @date    2020-02-29T15:35:13.178Z
-	 *
-	 * @copyright (c) 2011-2017 Almende B.V, http://almende.com
-	 * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
-	 *
-	 * @license
-	 * vis.js is dual licensed under both
-	 *
-	 *   1. The Apache 2.0 License
-	 *      http://www.apache.org/licenses/LICENSE-2.0
-	 *
-	 *   and
-	 *
-	 *   2. The MIT License
-	 *      http://opensource.org/licenses/MIT
-	 *
-	 * vis.js may be distributed under either license.
-	 */
-function(){for(var t=arguments.length,e=new Array(t),i=0;i<t;i++)e[i]=arguments[i];return qo(e.length?e:[Jo()])},HSVToHex:xs,HSVToRGB:ys,RGBToHSV:gs,RGBToHex:fs,addClassName:function(t,e){var i=t.className.split(" "),n=e.split(" ");i=Pn(i).call(i,En(n).call(n,(function(t){return Fo(i).call(i,t)<0}))),t.className=i.join(" ")},addCssText:function(t,e){var i=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=Ko(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=Ko(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},vs(t.style.cssText),{},vs(e));t.style.cssText=bs(i)},addEventListener:function(t,e,i,n){var o;t.addEventListener?(void 0===n&&(n=!1),"mousewheel"===e&&Fo(o=navigator.userAgent).call(o,"Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,n)):t.attachEvent("on"+e,i)},binarySearchCustom:function(t,e,i,n){for(var o=0,s=0,r=t.length-1;s<=r&&o<1e4;){var a=Math.floor((s+r)/2),l=t[a],h=e(void 0===n?l[i]:l[i][n]);if(0==h)return a;-1==h?s=a+1:r=a-1,o++}return-1},binarySearchValue:function(t,e,i,n,o){var s,r,a,l,h=0,d=0,u=t.length-1;for(o=null!=o?o:function(t,e){return t==e?0:t<e?-1:1};d<=u&&h<1e4;){if(l=Math.floor(.5*(u+d)),s=t[Math.max(0,l-1)][i],r=t[l][i],a=t[Math.min(t.length-1,l+1)][i],0==o(r,e))return l;if(o(s,e)<0&&o(r,e)>0)return"before"==n?Math.max(0,l-1):l;if(o(r,e)<0&&o(a,e)>0)return"before"==n?l:Math.min(t.length-1,l+1);o(r,e)<0?d=l+1:u=l-1,h++}return-1},bridgeObject:Ds,copyAndExtendArray:function(t,e){var i;return Pn(i=[]).call(i,mo(t),[e])},copyArray:function(t){return _o(t).call(t)},deepExtend:ds,easingFunctions:{linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return t<.5?2*t*t:(4-2*t)*t-1},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return t<.5?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return t<.5?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return t<.5?16*t*t*t*t*t:1+16*--t*t*t*t*t}},equalArray:function(t,e){if(t.length!==e.length)return!1;for(var i=0,n=t.length;i<n;i++)if(t[i]!=e[i])return!1;return!0},extend:hs,fillIfDefined:function t(e,i){var n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];for(var o in e)if(void 0!==i[o])if(null===i[o]||"object"!==zo(i[o]))ls(e,i,o,n);else{var s=e[o],r=i[o];as(s)&&as(r)&&t(s,r,n)}},forEach:function(t,e){if(Zo(t))for(var i=t.length,n=0;n<i;n++)e(t[n],n,t);else for(var o in t)Object.prototype.hasOwnProperty.call(t,o)&&e(t[o],o,t)},getAbsoluteLeft:function(t){return t.getBoundingClientRect().left},getAbsoluteRight:function(t){return t.getBoundingClientRect().right},getAbsoluteTop:function(t){return t.getBoundingClientRect().top},getScrollBarWidth:function(){var t=document.createElement("p");t.style.width="100%",t.style.height="200px";var e=document.createElement("div");e.style.position="absolute",e.style.top="0px",e.style.left="0px",e.style.visibility="hidden",e.style.width="200px",e.style.height="150px",e.style.overflow="hidden",e.appendChild(t),document.body.appendChild(e);var i=t.offsetWidth;e.style.overflow="scroll";var n=t.offsetWidth;return i==n&&(n=e.clientWidth),document.body.removeChild(e),i-n},getTarget:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:window.event,e=null;return t&&(t.target?e=t.target:t.srcElement&&(e=t.srcElement)),e instanceof Element&&(null==e.nodeType||3!=e.nodeType||(e=e.parentNode)instanceof Element)?e:null},getType:us,hasParent:function(t,e){for(var i=t;i;){if(i===e)return!0;if(!i.parentNode)return!1;i=i.parentNode}return!1},hexToHSV:ws,hexToRGB:ms,insertSort:function(t,e){for(var i=0;i<t.length;i++){var n=t[i],o=void 0;for(o=i;o>0&&e(n,t[o-1])<0;o--)t[o]=t[o-1];t[o]=n}return t},isDate:function(t){if(t instanceof Date)return!0;if(rs(t)){if(ts.exec(t))return!0;if(!isNaN(Date.parse(t)))return!0}return!1},isNumber:ss,isObject:as,isString:rs,isValidHex:_s,isValidRGB:ks,isValidRGBA:function(t){return os.test(t)},mergeOptions:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},o=function(t){return null!=t},s=function(t){return null!==t&&"object"===zo(t)},r=function(t){for(var e in t)if(Object.prototype.hasOwnProperty.call(t,e))return!1;return!0};if(!s(t))throw new Error("Parameter mergeTarget must be an object");if(!s(e))throw new Error("Parameter options must be an object");if(!o(i))throw new Error("Parameter option must have a value");if(!s(n))throw new Error("Parameter globalOptions must be an object");var a=function(t,e,i){s(t[i])||(t[i]={});var n=e[i],o=t[i];for(var r in n)Object.prototype.hasOwnProperty.call(n,r)&&(o[r]=n[r])},l=e[i],h=s(n)&&!r(n),d=h?n[i]:void 0,u=d?d.enabled:void 0;if(void 0!==l){if("boolean"==typeof l)return s(t[i])||(t[i]={}),void(t[i].enabled=l);if(null===l&&!s(t[i])){if(!o(d))return;t[i]=Oi(d)}if(s(l)){var c=!0;void 0!==l.enabled?c=l.enabled:void 0!==u&&(c=d.enabled),a(t,e,i),t[i].enabled=c}}},option:ps,overrideOpacity:function(t,e){if(-1!==Fo(t).call(t,"rgba"))return t;if(-1!==Fo(t).call(t,"rgb")){var i=t.substr(Fo(t).call(t,"(")+1).replace(")","").split(",");return"rgba("+i[0]+","+i[1]+","+i[2]+","+e+")"}var n=ms(t);return null==n?t:"rgba("+n.r+","+n.g+","+n.b+","+e+")"},parseColor:function(t,e){if(rs(t)){var i=t;if(ks(i)){var n,o=Ln(n=i.substr(4).substr(0,i.length-5).split(",")).call(n,(function(t){return Nn(t)}));i=fs(o[0],o[1],o[2])}if(!0===_s(i)){var s=ws(i),r={h:s.h,s:.8*s.s,v:Math.min(1,1.02*s.v)},a={h:s.h,s:Math.min(1,1.25*s.s),v:.8*s.v},l=xs(a.h,a.s,a.v),h=xs(r.h,r.s,r.v);return{background:i,border:l,highlight:{background:h,border:l},hover:{background:h,border:l}}}return{background:i,border:i,highlight:{background:i,border:i},hover:{background:i,border:i}}}return e?{background:t.background||e.background,border:t.border||e.border,highlight:rs(t.highlight)?{border:t.highlight,background:t.highlight}:{background:t.highlight&&t.highlight.background||e.highlight.background,border:t.highlight&&t.highlight.border||e.highlight.border},hover:rs(t.hover)?{border:t.hover,background:t.hover}:{border:t.hover&&t.hover.border||e.hover.border,background:t.hover&&t.hover.background||e.hover.background}}:{background:t.background||void 0,border:t.border||void 0,highlight:rs(t.highlight)?{border:t.highlight,background:t.highlight}:{background:t.highlight&&t.highlight.background||void 0,border:t.highlight&&t.highlight.border||void 0},hover:rs(t.hover)?{border:t.hover,background:t.hover}:{border:t.hover&&t.hover.border||void 0,background:t.hover&&t.hover.background||void 0}}},preventDefault:function(t){t||(t=window.event),t&&(t.preventDefault?t.preventDefault():t.returnValue=!1)},recursiveDOMDelete:function t(e){if(e)for(;!0===e.hasChildNodes();){var i=e.firstChild;i&&(t(i),e.removeChild(i))}},removeClassName:function(t,e){var i=t.className.split(" "),n=e.split(" ");i=En(i).call(i,(function(t){return Fo(n).call(n,t)<0})),t.className=i.join(" ")},removeCssText:function(t,e){var i=vs(t.style.cssText),n=vs(e);for(var o in n)Object.prototype.hasOwnProperty.call(n,o)&&delete i[o];t.style.cssText=bs(i)},removeEventListener:function(t,e,i,n){var o;t.removeEventListener?(void 0===n&&(n=!1),"mousewheel"===e&&Fo(o=navigator.userAgent).call(o,"Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,n)):t.detachEvent("on"+e,i)},selectiveBridgeObject:function(t,e){if(null!==e&&"object"===zo(e)){for(var i=Oi(e),n=0;n<t.length;n++)Object.prototype.hasOwnProperty.call(e,t[n])&&"object"==zo(e[t[n]])&&(i[t[n]]=Ds(e[t[n]]));return i}return null},selectiveDeepExtend:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Zo(i))throw new TypeError("Arrays are not supported by deepExtend");for(var o=0;o<t.length;o++){var s=t[o];if(Object.prototype.hasOwnProperty.call(i,s))if(i[s]&&i[s].constructor===Object)void 0===e[s]&&(e[s]={}),e[s].constructor===Object?ds(e[s],i[s],!1,n):ls(e,i,s,n);else{if(Zo(i[s]))throw new TypeError("Arrays are not supported by deepExtend");ls(e,i,s,n)}}return e},selectiveExtend:function(t,e){if(!Zo(t))throw new Error("Array with property names expected as first argument");for(var i=arguments.length,n=new Array(i>2?i-2:0),o=2;o<i;o++)n[o-2]=arguments[o];for(var s=0,r=n;s<r.length;s++)for(var a=r[s],l=0;l<t.length;l++){var h=t[l];a&&Object.prototype.hasOwnProperty.call(a,h)&&(e[h]=a[h])}return e},selectiveNotDeepExtend:function(t,e,i){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Zo(i))throw new TypeError("Arrays are not supported by deepExtend");for(var o in i)if(Object.prototype.hasOwnProperty.call(i,o)&&-1===Fo(t).call(t,o))if(i[o]&&i[o].constructor===Object)void 0===e[o]&&(e[o]={}),e[o].constructor===Object?ds(e[o],i[o]):ls(e,i,o,n);else if(Zo(i[o])){e[o]=[];for(var s=0;s<i[o].length;s++)e[o].push(i[o][s])}else ls(e,i,o,n);return e},throttle:function(t){var e=!1;return function(){e||(e=!0,requestAnimationFrame((function(){e=!1,t()})))}},toArray:cs,topMost:function(t,e){var i;Zo(e)||(e=[e]);var n=!0,o=!1,s=void 0;try{for(var r,a=Fi(t);!(n=(r=a.next()).done);n=!0){var l=r.value;if(l){i=l[e[0]];for(var h=1;h<e.length;h++)i&&(i=i[e[h]]);if(void 0!==i)break}}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return i},updateProperty:function(t,e,i){return t[e]!==i&&(t[e]=i,!0)}});function Cs(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])}function Ls(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}}function Gs(t){Cs(t),Ls(t),Cs(t)}function Ms(t,e,i){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(n)):(n=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(n)),e[t].used.push(n),n}function Ss(t,e,i,n){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElement(t),void 0!==n?i.insertBefore(o,n):i.appendChild(o)):(o=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==n?i.insertBefore(o,n):i.appendChild(o)),e[t].used.push(o),o}function Qs(t,e,i,n,o,s){var r;if("circle"==i.style?((r=Ms("circle",n,o)).setAttributeNS(null,"cx",t),r.setAttributeNS(null,"cy",e),r.setAttributeNS(null,"r",.5*i.size)):((r=Ms("rect",n,o)).setAttributeNS(null,"x",t-.5*i.size),r.setAttributeNS(null,"y",e-.5*i.size),r.setAttributeNS(null,"width",i.size),r.setAttributeNS(null,"height",i.size)),void 0!==i.styles&&r.setAttributeNS(null,"style",i.styles),r.setAttributeNS(null,"class",i.className+" vis-point"),s){var a=Ms("text",n,o);s.xOffset&&(t+=s.xOffset),s.yOffset&&(e+=s.yOffset),s.content&&(a.textContent=s.content),s.className&&a.setAttributeNS(null,"class",s.className+" vis-label"),a.setAttributeNS(null,"x",t),a.setAttributeNS(null,"y",e)}return r}function Ns(t,e,i,n,o,s,r,a){if(0!=n){n<0&&(e-=n*=-1);var l=Ms("rect",s,r);l.setAttributeNS(null,"x",t-.5*i),l.setAttributeNS(null,"y",e),l.setAttributeNS(null,"width",i),l.setAttributeNS(null,"height",n),l.setAttributeNS(null,"class",o),a&&l.setAttributeNS(null,"style",a)}}function Fs(){try{return navigator?navigator.languages&&navigator.languages.length?navigator.languages:navigator.userLanguage||navigator.language||navigator.browserLanguage||"en":"en"}catch(t){return"en"}}var Zs=Object.freeze({__proto__:null,prepareElements:Cs,cleanupElements:Ls,resetElements:Gs,getSVGElement:Ms,getDOMElement:Ss,drawPoint:Qs,drawBar:Ns,getNavigatorLanguage:Fs});var Bs=function(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t},Os=Pi("Array").entries,Ws=Array.prototype,As={DOMTokenList:!0,NodeList:!0},Xs=function(t){var e=t.entries;return t===Ws||t instanceof Array&&e===Ws.entries||As.hasOwnProperty(ne(t))?Os:e},Ts=pt("JSON","stringify"),Ys=/[\uD800-\uDFFF]/g,Es=/^[\uD800-\uDBFF]$/,Rs=/^[\uDC00-\uDFFF]$/,Us=function(t,e,i){var n=i.charAt(e-1),o=i.charAt(e+1);return Es.test(t)&&!Rs.test(o)||Rs.test(t)&&!Es.test(n)?"\\u"+t.charCodeAt(0).toString(16):t},zs=l((function(){return'"\\udf06\\ud834"'!==Ts("\udf06\ud834")||'"\\udead"'!==Ts("\udead")}));Ts&&V({target:"JSON",stat:!0,forced:zs},{stringify:function(t,e,i){var n=Ts.apply(null,arguments);return"string"==typeof n?n.replace(Ys,Us):n}}),W.JSON||(W.JSON={stringify:JSON.stringify});var Vs=function(t,e,i){return W.JSON.stringify.apply(null,arguments)},js=Pi("Array").values,Hs=Array.prototype,Ps={DOMTokenList:!0,NodeList:!0},Js=function(t){var e=t.values;return t===Hs||t instanceof Array&&e===Hs.values||Ps.hasOwnProperty(ne(t))?js:e},Ks=Pi("Array").keys,qs=Array.prototype,$s={DOMTokenList:!0,NodeList:!0},tr=function(t){var e=t.keys;return t===qs||t instanceof Array&&e===qs.keys||$s.hasOwnProperty(ne(t))?Ks:e},er=Ce.some,ir=$i("some"),nr=on("some");V({target:"Array",proto:!0,forced:!ir||!nr},{some:function(t){return er(this,t,arguments.length>1?arguments[1]:void 0)}});var or=Pi("Array").some,sr=Array.prototype,rr=function(t){var e=t.some;return t===sr||t instanceof Array&&e===sr.some?or:e};var ar=function(t,e){return!e||"object"!==zo(e)&&"function"!=typeof e?Bs(t):e},lr=Do;V({target:"Object",stat:!0},{setPrototypeOf:fi});var hr=W.Object.setPrototypeOf,dr=n((function(t){function e(i){return t.exports=e=hr?lr:function(t){return t.__proto__||lr(t)},e(i)}t.exports=e})),ur=Bi,cr=n((function(t){function e(i,n){return t.exports=e=hr||function(t,e){return t.__proto__=e,t},e(i,n)}t.exports=e}));var pr=function(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=ur(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),e&&cr(t,e)},mr=[],fr=mr.sort,gr=l((function(){mr.sort(void 0)})),vr=l((function(){mr.sort(null)})),br=$i("sort");V({target:"Array",proto:!0,forced:gr||!vr||!br},{sort:function(t){return void 0===t?fr.call(Lt(this)):fr.call(Lt(this),A(t))}});var yr=Pi("Array").sort,xr=Array.prototype,wr=function(t){var e=t.sort;return t===xr||t instanceof Array&&e===xr.sort?yr:e},_r=!l((function(){return Object.isExtensible(Object.preventExtensions({}))})),kr=n((function(t){var e=E.f,i=Ft("meta"),n=0,o=Object.isExtensible||function(){return!0},s=function(t){e(t,i,{value:{objectID:"O"+ ++n,weakData:{}}})},r=t.exports={REQUIRED:!1,fastKey:function(t,e){if(!x(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!k(t,i)){if(!o(t))return"F";if(!e)return"E";s(t)}return t[i].objectID},getWeakData:function(t,e){if(!k(t,i)){if(!o(t))return!0;if(!e)return!1;s(t)}return t[i].weakData},onFreeze:function(t){return _r&&r.REQUIRED&&o(t)&&!k(t,i)&&s(t),t}};st[i]=!0})),Dr=(kr.REQUIRED,kr.fastKey,kr.getWeakData,kr.onFreeze,n((function(t){var e=function(t,e){this.stopped=t,this.result=e};(t.exports=function(t,i,n,o,s){var r,a,l,h,d,u,c,p=X(i,n,o?2:1);if(s)r=t;else{if("function"!=typeof(a=Ni(t)))throw TypeError("Target is not iterable");if(io(a)){for(l=0,h=$(t.length);h>l;l++)if((d=o?p(T(c=t[l])[0],c[1]):p(t[l]))&&d instanceof e)return d;return new e(!1)}r=a.call(t)}for(u=r.next;!(c=u.call(r)).done;)if("object"==typeof(d=$n(r,p,c.value,o))&&d&&d instanceof e)return d;return new e(!1)}).stop=function(t){return new e(!0,t)}}))),Ir=function(t,e,i){if(!(t instanceof e))throw TypeError("Incorrect "+(i?i+" ":"")+"invocation");return t},Cr=E.f,Lr=Ce.forEach,Gr=we.set,Mr=we.getterFor,Sr=function(t,e,i){var n,o=-1!==t.indexOf("Map"),s=-1!==t.indexOf("Weak"),r=o?"set":"add",d=a[t],u=d&&d.prototype,c={};if(h&&"function"==typeof d&&(s||u.forEach&&!l((function(){(new d).entries().next()})))){n=e((function(e,i){Gr(Ir(e,n,t),{type:t,collection:new d}),null!=i&&Dr(i,e[r],e,o)}));var p=Mr(t);Lr(["add","clear","delete","forEach","get","has","set","keys","values","entries"],(function(t){var e="add"==t||"set"==t;t in u&&(!s||"clear"!=t)&&R(n.prototype,t,(function(i,n){var o=p(this).collection;if(!e&&s&&!x(i))return"get"==t&&void 0;var r=o[t](0===i?0:i,n);return e?this:r}))})),s||Cr(n.prototype,"size",{configurable:!0,get:function(){return p(this).collection.size}})}else n=i.getConstructor(e,t,o,r),kr.REQUIRED=!0;return ae(n,t,!1,!0),c[t]=n,V({global:!0,forced:!0},c),s||i.setStrong(n,t,o),n},Qr=function(t,e,i){for(var n in e)i&&i.unsafe&&t[n]?t[n]=e[n]:zt(t,n,e[n],i);return t},Nr=Pt("species"),Fr=E.f,Zr=kr.fastKey,Br=we.set,Or=we.getterFor,Wr={getConstructor:function(t,e,i,n){var o=t((function(t,s){Ir(t,o,e),Br(t,{type:e,index:Tt(null),first:void 0,last:void 0,size:0}),h||(t.size=0),null!=s&&Dr(s,t[n],t,i)})),s=Or(e),r=function(t,e,i){var n,o,r=s(t),l=a(t,e);return l?l.value=i:(r.last=l={index:o=Zr(e,!0),key:e,value:i,previous:n=r.last,next:void 0,removed:!1},r.first||(r.first=l),n&&(n.next=l),h?r.size++:t.size++,"F"!==o&&(r.index[o]=l)),t},a=function(t,e){var i,n=s(t),o=Zr(e);if("F"!==o)return n.index[o];for(i=n.first;i;i=i.next)if(i.key==e)return i};return Qr(o.prototype,{clear:function(){for(var t=s(this),e=t.index,i=t.first;i;)i.removed=!0,i.previous&&(i.previous=i.previous.next=void 0),delete e[i.index],i=i.next;t.first=t.last=void 0,h?t.size=0:this.size=0},delete:function(t){var e=s(this),i=a(this,t);if(i){var n=i.next,o=i.previous;delete e.index[i.index],i.removed=!0,o&&(o.next=n),n&&(n.previous=o),e.first==i&&(e.first=n),e.last==i&&(e.last=o),h?e.size--:this.size--}return!!i},forEach:function(t){for(var e,i=s(this),n=X(t,arguments.length>1?arguments[1]:void 0,3);e=e?e.next:i.first;)for(n(e.value,e.key,this);e&&e.removed;)e=e.previous},has:function(t){return!!a(this,t)}}),Qr(o.prototype,i?{get:function(t){var e=a(this,t);return e&&e.value},set:function(t,e){return r(this,0===t?0:t,e)}}:{add:function(t){return r(this,t=0===t?0:t,t)}}),h&&Fr(o.prototype,"size",{get:function(){return s(this).size}}),o},setStrong:function(t,e,i){var n=e+" Iterator",o=Or(e),s=Or(n);xi(t,e,(function(t,e){Br(this,{type:n,target:t,state:o(t),kind:e,last:void 0})}),(function(){for(var t=s(this),e=t.kind,i=t.last;i&&i.removed;)i=i.previous;return t.target&&(t.last=i=i?i.next:t.state.first)?"keys"==e?{value:i.key,done:!1}:"values"==e?{value:i.value,done:!1}:{value:[i.key,i.value],done:!1}:(t.target=void 0,{value:void 0,done:!0})}),i?"entries":"values",!i,!0),function(t){var e=pt(t),i=E.f;h&&e&&!e[Nr]&&i(e,Nr,{configurable:!0,get:function(){return this}})}(e)}},Ar=(Sr("Set",(function(t){return function(){return t(this,arguments.length?arguments[0]:void 0)}}),Wr),W.Set),Xr=(Sr("Map",(function(t){return function(){return t(this,arguments.length?arguments[0]:void 0)}}),Wr),W.Map),Tr=n((function(t){var e=function(t){var e=Object.prototype,i=e.hasOwnProperty,n="function"==typeof Symbol?Symbol:{},o=n.iterator||"@@iterator",s=n.asyncIterator||"@@asyncIterator",r=n.toStringTag||"@@toStringTag";function a(t,e,i,n){var o=e&&e.prototype instanceof d?e:d,s=Object.create(o.prototype),r=new _(n||[]);return s._invoke=function(t,e,i){var n="suspendedStart";return function(o,s){if("executing"===n)throw new Error("Generator is already running");if("completed"===n){if("throw"===o)throw s;return D()}for(i.method=o,i.arg=s;;){var r=i.delegate;if(r){var a=y(r,i);if(a){if(a===h)continue;return a}}if("next"===i.method)i.sent=i._sent=i.arg;else if("throw"===i.method){if("suspendedStart"===n)throw n="completed",i.arg;i.dispatchException(i.arg)}else"return"===i.method&&i.abrupt("return",i.arg);n="executing";var d=l(t,e,i);if("normal"===d.type){if(n=i.done?"completed":"suspendedYield",d.arg===h)continue;return{value:d.arg,done:i.done}}"throw"===d.type&&(n="completed",i.method="throw",i.arg=d.arg)}}}(t,i,r),s}function l(t,e,i){try{return{type:"normal",arg:t.call(e,i)}}catch(t){return{type:"throw",arg:t}}}t.wrap=a;var h={};function d(){}function u(){}function c(){}var p={};p[o]=function(){return this};var m=Object.getPrototypeOf,f=m&&m(m(k([])));f&&f!==e&&i.call(f,o)&&(p=f);var g=c.prototype=d.prototype=Object.create(p);function v(t){["next","throw","return"].forEach((function(e){t[e]=function(t){return this._invoke(e,t)}}))}function b(t,e){var n;this._invoke=function(o,s){function r(){return new e((function(n,r){!function n(o,s,r,a){var h=l(t[o],t,s);if("throw"!==h.type){var d=h.arg,u=d.value;return u&&"object"==typeof u&&i.call(u,"__await")?e.resolve(u.__await).then((function(t){n("next",t,r,a)}),(function(t){n("throw",t,r,a)})):e.resolve(u).then((function(t){d.value=t,r(d)}),(function(t){return n("throw",t,r,a)}))}a(h.arg)}(o,s,n,r)}))}return n=n?n.then(r,r):r()}}function y(t,e){var i=t.iterator[e.method];if(void 0===i){if(e.delegate=null,"throw"===e.method){if(t.iterator.return&&(e.method="return",e.arg=void 0,y(t,e),"throw"===e.method))return h;e.method="throw",e.arg=new TypeError("The iterator does not provide a 'throw' method")}return h}var n=l(i,t.iterator,e.arg);if("throw"===n.type)return e.method="throw",e.arg=n.arg,e.delegate=null,h;var o=n.arg;return o?o.done?(e[t.resultName]=o.value,e.next=t.nextLoc,"return"!==e.method&&(e.method="next",e.arg=void 0),e.delegate=null,h):o:(e.method="throw",e.arg=new TypeError("iterator result is not an object"),e.delegate=null,h)}function x(t){var e={tryLoc:t[0]};1 in t&&(e.catchLoc=t[1]),2 in t&&(e.finallyLoc=t[2],e.afterLoc=t[3]),this.tryEntries.push(e)}function w(t){var e=t.completion||{};e.type="normal",delete e.arg,t.completion=e}function _(t){this.tryEntries=[{tryLoc:"root"}],t.forEach(x,this),this.reset(!0)}function k(t){if(t){var e=t[o];if(e)return e.call(t);if("function"==typeof t.next)return t;if(!isNaN(t.length)){var n=-1,s=function e(){for(;++n<t.length;)if(i.call(t,n))return e.value=t[n],e.done=!1,e;return e.value=void 0,e.done=!0,e};return s.next=s}}return{next:D}}function D(){return{value:void 0,done:!0}}return u.prototype=g.constructor=c,c.constructor=u,c[r]=u.displayName="GeneratorFunction",t.isGeneratorFunction=function(t){var e="function"==typeof t&&t.constructor;return!!e&&(e===u||"GeneratorFunction"===(e.displayName||e.name))},t.mark=function(t){return Object.setPrototypeOf?Object.setPrototypeOf(t,c):(t.__proto__=c,r in t||(t[r]="GeneratorFunction")),t.prototype=Object.create(g),t},t.awrap=function(t){return{__await:t}},v(b.prototype),b.prototype[s]=function(){return this},t.AsyncIterator=b,t.async=function(e,i,n,o,s){void 0===s&&(s=Promise);var r=new b(a(e,i,n,o),s);return t.isGeneratorFunction(i)?r:r.next().then((function(t){return t.done?t.value:r.next()}))},v(g),g[r]="Generator",g[o]=function(){return this},g.toString=function(){return"[object Generator]"},t.keys=function(t){var e=[];for(var i in t)e.push(i);return e.reverse(),function i(){for(;e.length;){var n=e.pop();if(n in t)return i.value=n,i.done=!1,i}return i.done=!0,i}},t.values=k,_.prototype={constructor:_,reset:function(t){if(this.prev=0,this.next=0,this.sent=this._sent=void 0,this.done=!1,this.delegate=null,this.method="next",this.arg=void 0,this.tryEntries.forEach(w),!t)for(var e in this)"t"===e.charAt(0)&&i.call(this,e)&&!isNaN(+e.slice(1))&&(this[e]=void 0)},stop:function(){this.done=!0;var t=this.tryEntries[0].completion;if("throw"===t.type)throw t.arg;return this.rval},dispatchException:function(t){if(this.done)throw t;var e=this;function n(i,n){return r.type="throw",r.arg=t,e.next=i,n&&(e.method="next",e.arg=void 0),!!n}for(var o=this.tryEntries.length-1;o>=0;--o){var s=this.tryEntries[o],r=s.completion;if("root"===s.tryLoc)return n("end");if(s.tryLoc<=this.prev){var a=i.call(s,"catchLoc"),l=i.call(s,"finallyLoc");if(a&&l){if(this.prev<s.catchLoc)return n(s.catchLoc,!0);if(this.prev<s.finallyLoc)return n(s.finallyLoc)}else if(a){if(this.prev<s.catchLoc)return n(s.catchLoc,!0)}else{if(!l)throw new Error("try statement without catch or finally");if(this.prev<s.finallyLoc)return n(s.finallyLoc)}}}},abrupt:function(t,e){for(var n=this.tryEntries.length-1;n>=0;--n){var o=this.tryEntries[n];if(o.tryLoc<=this.prev&&i.call(o,"finallyLoc")&&this.prev<o.finallyLoc){var s=o;break}}s&&("break"===t||"continue"===t)&&s.tryLoc<=e&&e<=s.finallyLoc&&(s=null);var r=s?s.completion:{};return r.type=t,r.arg=e,s?(this.method="next",this.next=s.finallyLoc,h):this.complete(r)},complete:function(t,e){if("throw"===t.type)throw t.arg;return"break"===t.type||"continue"===t.type?this.next=t.arg:"return"===t.type?(this.rval=this.arg=t.arg,this.method="return",this.next="end"):"normal"===t.type&&e&&(this.next=e),h},finish:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e];if(i.finallyLoc===t)return this.complete(i.completion,i.afterLoc),w(i),h}},catch:function(t){for(var e=this.tryEntries.length-1;e>=0;--e){var i=this.tryEntries[e];if(i.tryLoc===t){var n=i.completion;if("throw"===n.type){var o=n.arg;w(i)}return o}}throw new Error("illegal catch attempt")},delegateYield:function(t,e,i){return this.delegate={iterator:k(t),resultName:e,nextLoc:i},"next"===this.method&&(this.arg=void 0),h}},t}(t.exports);try{regeneratorRuntime=e}catch(t){Function("r","regeneratorRuntime = r")(e)}})),Yr=Xo,Er=wn("splice"),Rr=on("splice",{ACCESSORS:!0,0:0,1:2}),Ur=Math.max,zr=Math.min;V({target:"Array",proto:!0,forced:!Er||!Rr},{splice:function(t,e){var i,n,o,s,r,a,l=Lt(this),h=$(l.length),d=it(t,h),u=arguments.length;if(0===u?i=n=0:1===u?(i=0,n=h-d):(i=u-2,n=zr(Ur(K(e),0),h-d)),h+i-n>9007199254740991)throw TypeError("Maximum allowed length exceeded");for(o=ke(l,n),s=0;s<n;s++)(r=d+s)in l&&bt(o,s,l[r]);if(o.length=n,i<n){for(s=d;s<h-n;s++)a=s+i,(r=s+n)in l?l[a]=l[r]:delete l[a];for(s=h;s>h-n+i;s--)delete l[s-1]}else if(i>n)for(s=h-n;s>d;s--)a=s+i-1,(r=s+n-1)in l?l[a]=l[r]:delete l[a];for(s=0;s<i;s++)l[s+d]=arguments[s+2];return l.length=h-n+i,o}});var Vr=Pi("Array").splice,jr=Array.prototype,Hr=function(t){var e=t.splice;return t===jr||t instanceof Array&&e===jr.splice?Vr:e},Pr=[].slice,Jr=/MSIE .\./.test(fn),Kr=function(t){return function(e,i){var n=arguments.length>2,o=n?Pr.call(arguments,2):void 0;return t(n?function(){("function"==typeof e?e:Function(e)).apply(this,o)}:e,i)}};V({global:!0,bind:!0,forced:Jr},{setTimeout:Kr(a.setTimeout),setInterval:Kr(a.setInterval)});var qr=W.setTimeout,$r=function(t,e,i,n,o,s,r,a){for(var l,h=o,d=0,u=!!r&&X(r,a,3);d<n;){if(d in i){if(l=u?u(i[d],d,e):i[d],s>0&&Ct(l))h=$r(t,e,l,$(l.length),h,s-1)-1;else{if(h>=9007199254740991)throw TypeError("Exceed the acceptable array length");t[h]=l}h++}d++}return h},ta=$r;V({target:"Array",proto:!0},{flatMap:function(t){var e,i=Lt(this),n=$(i.length);return A(t),(e=ke(i,0)).length=ta(e,i,i,n,0,1,t,arguments.length>1?arguments[1]:void 0),e}});var ea=Pi("Array").flatMap,ia=Array.prototype,na=function(t){var e=t.flatMap;return t===ia||t instanceof Array&&e===ia.flatMap?ea:e},oa=function(t){return function(e,i,n,o){A(i);var s=Lt(e),r=v(s),a=$(s.length),l=t?a-1:0,h=t?-1:1;if(n<2)for(;;){if(l in r){o=r[l],l+=h;break}if(l+=h,t?l<0:a<=l)throw TypeError("Reduce of empty array with no initial value")}for(;t?l>=0:a>l;l+=h)l in r&&(o=i(o,r[l],l,s));return o}},sa={left:oa(!1),right:oa(!0)}.left,ra=$i("reduce"),aa=on("reduce",{1:0});V({target:"Array",proto:!0,forced:!ra||!aa},{reduce:function(t){return sa(this,t,arguments.length,arguments.length>1?arguments[1]:void 0)}});var la=Pi("Array").reduce,ha=Array.prototype,da=function(t){var e=t.reduce;return t===ha||t instanceof Array&&e===ha.reduce?la:e},ua=[].slice,ca={},pa=function(t,e,i){if(!(e in ca)){for(var n=[],o=0;o<e;o++)n[o]="a["+o+"]";ca[e]=Function("C,a","return new C("+n.join(",")+")")}return ca[e](t,i)},ma=Function.bind||function(t){var e=A(this),i=ua.call(arguments,1),n=function(){var o=i.concat(ua.call(arguments));return this instanceof n?pa(e,o.length,o):e.apply(t,o)};return x(e.prototype)&&(n.prototype=e.prototype),n};V({target:"Function",proto:!0},{bind:ma});var fa=Pi("Function").bind,ga=Function.prototype,va=function(t){var e=t.bind;return t===ga||t instanceof Function&&e===ga.bind?fa:e};var ba=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")};function ya(t,e){for(var i=0;i<e.length;i++){var n=e[i];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Wi(t,n.key,n)}}var xa=function(t,e,i){return e&&ya(t.prototype,e),i&&ya(t,i),t};function wa(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}function _a(t){return new Da(t)}var ka=function(){function t(e,i,n){var o,s,r;ba(this,t),this._source=e,this._transformers=i,this._target=n,this._listeners={add:va(o=this._add).call(o,this),remove:va(s=this._remove).call(s,this),update:va(r=this._update).call(r,this)}}return xa(t,[{key:"all",value:function(){return this._target.update(this._transformItems(this._source.get())),this}},{key:"start",value:function(){return this._source.on("add",this._listeners.add),this._source.on("remove",this._listeners.remove),this._source.on("update",this._listeners.update),this}},{key:"stop",value:function(){return this._source.off("add",this._listeners.add),this._source.off("remove",this._listeners.remove),this._source.off("update",this._listeners.update),this}},{key:"_transformItems",value:function(t){var e;return da(e=this._transformers).call(e,(function(t,e){return e(t)}),t)}},{key:"_add",value:function(t,e){null!=e&&this._target.add(this._transformItems(this._source.get(e.items)))}},{key:"_update",value:function(t,e){null!=e&&this._target.update(this._transformItems(this._source.get(e.items)))}},{key:"_remove",value:function(t,e){null!=e&&this._target.remove(this._transformItems(e.oldData))}}]),t}(),Da=function(){function t(e){ba(this,t),this._source=e,this._transformers=[]}return xa(t,[{key:"filter",value:function(t){return this._transformers.push((function(e){return En(e).call(e,t)})),this}},{key:"map",value:function(t){return this._transformers.push((function(e){return Ln(e).call(e,t)})),this}},{key:"flatMap",value:function(t){return this._transformers.push((function(e){return na(e).call(e,t)})),this}},{key:"to",value:function(t){return new ka(this._source,this._transformers,t)}}]),t}();for(var Ia=function(t,e){return t(e={exports:{}},e.exports),e.exports}((function(t){var e,i,n="undefined"!=typeof crypto&&crypto.getRandomValues&&va(e=crypto.getRandomValues).call(e,crypto)||"undefined"!=typeof msCrypto&&"function"==typeof window.msCrypto.getRandomValues&&va(i=msCrypto.getRandomValues).call(i,msCrypto);if(n){var o=new Uint8Array(16);t.exports=function(){return n(o),o}}else{var s=new Array(16);t.exports=function(){for(var t,e=0;e<16;e++)0==(3&e)&&(t=4294967296*Math.random()),s[e]=t>>>((3&e)<<3)&255;return s}}})),Ca=[],La=0;La<256;++La)Ca[La]=(La+256).toString(16).substr(1);var Ga=function(t,e){var i=e||0,n=Ca;return[n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]]].join("")};var Ma=function(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"===t?new Array(16):null,t=null);var o=(t=t||{}).random||(t.rng||Ia)();if(o[6]=15&o[6]|64,o[8]=63&o[8]|128,e)for(var s=0;s<16;++s)e[n+s]=o[s];return e||Ga(o)},Sa=/^\/?Date\((-?\d+)/i;function Qa(t){return s.isMoment(t)}function Na(t,e){var i;if(void 0!==t){if(null===t)return null;if(!e)return t;if("string"!=typeof e&&!(e instanceof String))throw new Error("Type must be a string");switch(e){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return rs(t)&&!isNaN(Date.parse(t))?s(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(ss(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(Qa(t))return new Date(t.valueOf());if(rs(t))return(i=Sa.exec(t))?new Date(Number(i[1])):s(new Date(t)).toDate();throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"Moment":if(ss(t))return s(t);if(t instanceof Date)return s(t.valueOf());if(Qa(t))return s(t);if(rs(t))return i=Sa.exec(t),s(i?Number(i[1]):t);throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"ISODate":if(ss(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(Qa(t))return t.toDate().toISOString();if(rs(t))return(i=Sa.exec(t))?new Date(Number(i[1])).toISOString():s(t).format();throw new Error("Cannot convert object of type "+us(t)+" to type ISODate");case"ASPDate":if(ss(t))return"/Date("+t+")/";if(t instanceof Date||Qa(t))return"/Date("+t.valueOf()+")/";if(rs(t))return"/Date("+((i=Sa.exec(t))?new Date(Number(i[1])).valueOf():new Date(t).valueOf())+")/";throw new Error("Cannot convert object of type "+us(t)+" to type ASPDate");default:throw new Error("Unknown type ".concat(e))}}}function Fa(t){return"string"==typeof t||"number"==typeof t}var Za=function(){function t(e){ba(this,t),this._queue=[],this._timeout=null,this._extended=null,this.delay=null,this.max=1/0,this.setOptions(e)}return xa(t,[{key:"setOptions",value:function(t){t&&void 0!==t.delay&&(this.delay=t.delay),t&&void 0!==t.max&&(this.max=t.max),this._flushIfNeeded()}},{key:"destroy",value:function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var n=e[i];n.original?t[n.name]=n.original:delete t[n.name]}this._extended=null}}},{key:"replace",value:function(t,e){var i=this,n=t[e];if(!n)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=arguments.length,e=new Array(t),o=0;o<t;o++)e[o]=arguments[o];i.queue({args:e,fn:n,context:this})}}},{key:"queue",value:function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()}},{key:"_flushIfNeeded",value:function(){var t=this;this._queue.length>this.max&&this.flush(),null!=this._timeout&&(clearTimeout(this._timeout),this._timeout=null),this.queue.length>0&&"number"==typeof this.delay&&(this._timeout=qr((function(){t.flush()}),this.delay))}},{key:"flush",value:function(){var t,e;mn(t=Hr(e=this._queue).call(e,0)).call(t,(function(t){t.fn.apply(t.context||t.fn,t.args||[])}))}}],[{key:"extend",value:function(e,i){var n=new t(i);if(void 0!==e.flush)throw new Error("Target object already has a property flush");e.flush=function(){n.flush()};var o=[{name:"flush",original:void 0}];if(i&&i.replace)for(var s=0;s<i.replace.length;s++){var r=i.replace[s];o.push({name:r,original:e[r]}),n.replace(e,r)}return n._extended={object:e,methods:o},n}}]),t}(),Ba=function(){function t(){ba(this,t),this._subscribers={"*":[],add:[],remove:[],update:[]},this.subscribe=t.prototype.on,this.unsubscribe=t.prototype.off}return xa(t,[{key:"_trigger",value:function(t,e,i){var n,o;if("*"===t)throw new Error("Cannot trigger event *");mn(n=Pn(o=[]).call(o,mo(this._subscribers[t]),mo(this._subscribers["*"]))).call(n,(function(n){n(t,e,null!=i?i:null)}))}},{key:"on",value:function(t,e){"function"==typeof e&&this._subscribers[t].push(e)}},{key:"off",value:function(t,e){var i;this._subscribers[t]=En(i=this._subscribers[t]).call(i,(function(t){return t!==e}))}}]),t}(),Oa=function(){function t(e){ba(this,t),this._pairs=e}return xa(t,[{key:Yr,value:Tr.mark((function t(){var e,i,n,o,s,r,a,l;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[0],l=r[1],t.next=9,[a,l];case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"entries",value:Tr.mark((function t(){var e,i,n,o,s,r,a,l;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[0],l=r[1],t.next=9,[a,l];case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"keys",value:Tr.mark((function t(){var e,i,n,o,s,r,a;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,1),a=r[0],t.next=9,a;case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"values",value:Tr.mark((function t(){var e,i,n,o,s,r,a;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:e=!0,i=!1,n=void 0,t.prev=3,o=Fi(this._pairs);case 5:if(e=(s=o.next()).done){t.next=12;break}return r=Po(s.value,2),a=r[1],t.next=9,a;case 9:e=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),i=!0,n=t.t0;case 18:t.prev=18,t.prev=19,e||null==o.return||o.return();case 21:if(t.prev=21,!i){t.next=24;break}throw n;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,this,[[3,14,18,26],[19,,21,25]])}))},{key:"toIdArray",value:function(){var t;return Ln(t=mo(this._pairs)).call(t,(function(t){return t[0]}))}},{key:"toItemArray",value:function(){var t;return Ln(t=mo(this._pairs)).call(t,(function(t){return t[1]}))}},{key:"toEntryArray",value:function(){return mo(this._pairs)}},{key:"toObjectMap",value:function(){var t=Oi(null),e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this._pairs);!(e=(o=s.next()).done);e=!0){var r=Po(o.value,2),a=r[0],l=r[1];t[a]=l}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}return t}},{key:"toMap",value:function(){return new Xr(this._pairs)}},{key:"toIdSet",value:function(){return new Ar(this.toIdArray())}},{key:"toItemSet",value:function(){return new Ar(this.toItemArray())}},{key:"cache",value:function(){return new t(mo(this._pairs))}},{key:"distinct",value:function(t){var e=new Ar,i=!0,n=!1,o=void 0;try{for(var s,r=Fi(this._pairs);!(i=(s=r.next()).done);i=!0){var a=Po(s.value,2),l=a[0],h=a[1];e.add(t(h,l))}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}return e}},{key:"filter",value:function(e){var i=this._pairs;return new t(Ai({},Yr,Tr.mark((function t(){var n,o,s,r,a,l,h,d;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:n=!0,o=!1,s=void 0,t.prev=3,r=Fi(i);case 5:if(n=(a=r.next()).done){t.next=13;break}if(l=Po(a.value,2),h=l[0],d=l[1],!e(d,h)){t.next=10;break}return t.next=10,[h,d];case 10:n=!0,t.next=5;break;case 13:t.next=19;break;case 15:t.prev=15,t.t0=t.catch(3),o=!0,s=t.t0;case 19:t.prev=19,t.prev=20,n||null==r.return||r.return();case 22:if(t.prev=22,!o){t.next=25;break}throw s;case 25:return t.finish(22);case 26:return t.finish(19);case 27:case"end":return t.stop()}}),t,null,[[3,15,19,27],[20,,22,26]])}))))}},{key:"forEach",value:function(t){var e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this._pairs);!(e=(o=s.next()).done);e=!0){var r=Po(o.value,2),a=r[0];t(r[1],a)}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}}},{key:"map",value:function(e){var i=this._pairs;return new t(Ai({},Yr,Tr.mark((function t(){var n,o,s,r,a,l,h,d;return Tr.wrap((function(t){for(;;)switch(t.prev=t.next){case 0:n=!0,o=!1,s=void 0,t.prev=3,r=Fi(i);case 5:if(n=(a=r.next()).done){t.next=12;break}return l=Po(a.value,2),h=l[0],d=l[1],t.next=9,[h,e(d,h)];case 9:n=!0,t.next=5;break;case 12:t.next=18;break;case 14:t.prev=14,t.t0=t.catch(3),o=!0,s=t.t0;case 18:t.prev=18,t.prev=19,n||null==r.return||r.return();case 21:if(t.prev=21,!o){t.next=24;break}throw s;case 24:return t.finish(21);case 25:return t.finish(18);case 26:case"end":return t.stop()}}),t,null,[[3,14,18,26],[19,,21,25]])}))))}},{key:"max",value:function(t){var e=Fi(this._pairs),i=e.next();if(i.done)return null;for(var n=i.value[1],o=t(i.value[1],i.value[0]);!(i=e.next()).done;){var s=Po(i.value,2),r=s[0],a=s[1],l=t(a,r);l>o&&(o=l,n=a)}return n}},{key:"min",value:function(t){var e=Fi(this._pairs),i=e.next();if(i.done)return null;for(var n=i.value[1],o=t(i.value[1],i.value[0]);!(i=e.next()).done;){var s=Po(i.value,2),r=s[0],a=s[1],l=t(a,r);l<o&&(o=l,n=a)}return n}},{key:"reduce",value:function(t,e){var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(this._pairs);!(i=(s=r.next()).done);i=!0){var a=Po(s.value,2),l=a[0];e=t(e,a[1],l)}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}return e}},{key:"sort",value:function(e){var i=this;return new t(Ai({},Yr,(function(){var t;return Fi(wr(t=mo(i._pairs)).call(t,(function(t,i){var n=Po(t,2),o=n[0],s=n[1],r=Po(i,2),a=r[0],l=r[1];return e(s,l,o,a)})))})))}}]),t}(),Wa=function(){console.warn("Type coercion has been deprecated. Please, use data pipes instead. See https://visjs.github.io/vis-data/data/datapipe.html#TypeCoercion for more details with working migration example.")},Aa=function(t){function e(t,i){var n;if(ba(this,e),n=ar(this,dr(e).call(this)),t&&!Zo(t)&&(i=t,t=[]),n._options=i||{},n._data=new Xr,n.length=0,n._idProp=n._options.fieldId||"id",n._type={},n._options.type){Wa();for(var o=Yi(n._options.type),s=0,r=o.length;s<r;s++){var a=o[s],l=n._options.type[a];n._type[a]="Date"==l||"ISODate"==l||"ASPDate"==l?"Date":l}}return t&&t.length&&n.add(t),n.setOptions(i),n}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&void 0!==t.queue&&(!1===t.queue?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=Za.extend(this,{replace:["add","update","remove"]})),t.queue&&"object"===zo(t.queue)&&this._queue.setOptions(t.queue)))}},{key:"add",value:function(t,e){var i,n=this,o=[];if(Zo(t)){var s=Ln(t).call(t,(function(t){return t[n._idProp]}));if(rr(s).call(s,(function(t){return n._data.has(t)})))throw new Error("A duplicate id was found in the parameter array.");for(var r=0,a=t.length;r<a;r++)i=this._addItem(t[r]),o.push(i)}else{if(!t||"object"!==zo(t))throw new Error("Unknown dataType");i=this._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o}},{key:"update",value:function(t,e){var i=this,n=[],o=[],s=[],r=[],a=this._idProp,l=function(t){var e=t[a];if(null!=e&&i._data.has(e)){var l=t,h=Ao({},i._data.get(e)),d=i._updateItem(l);o.push(d),r.push(l),s.push(h)}else{var u=i._addItem(t);n.push(u)}};if(Zo(t))for(var h=0,d=t.length;h<d;h++)t[h]&&"object"===zo(t[h])?l(t[h]):console.warn("Ignoring input item, which is not an object at index "+h);else{if(!t||"object"!==zo(t))throw new Error("Unknown dataType");l(t)}if(n.length&&this._trigger("add",{items:n},e),o.length){var u={items:o,oldData:s,data:r};this._trigger("update",u,e)}return Pn(n).call(n,o)}},{key:"updateOnly",value:function(t,e){var i,n=this;Zo(t)||(t=[t]);var o=Ln(i=Ln(t).call(t,(function(t){var e=n._data.get(t[n._idProp]);if(null==e)throw new Error("Updating non-existent items is not allowed.");return{oldData:e,update:t}}))).call(i,(function(t){var e=t.oldData,i=t.update,o=e[n._idProp],s=ds(ds({},e),i);return n._data.set(o,s),{id:o,oldData:e,updatedData:s}}));if(o.length){var s={items:Ln(o).call(o,(function(t){return t.id})),oldData:Ln(o).call(o,(function(t){return t.oldData})),data:Ln(o).call(o,(function(t){return t.updatedData}))};return this._trigger("update",s,e),s.items}return[]}},{key:"get",value:function(t,e){var i=void 0,n=void 0,o=void 0;Fa(t)?(i=t,o=e):Zo(t)?(n=t,o=e):o=t;var s=o&&"Object"===o.returnType?"Object":"Array",r=o&&o.type||this._options.type,a=o&&En(o),l=[],h=null,d=null,u=null;if(null!=i)(h=this._getItem(i,r))&&a&&!a(h)&&(h=null);else if(null!=n)for(var c=0,p=n.length;c<p;c++)null==(h=this._getItem(n[c],r))||a&&!a(h)||l.push(h);else for(var m,f=0,g=(d=mo(tr(m=this._data).call(m))).length;f<g;f++)u=d[f],null==(h=this._getItem(u,r))||a&&!a(h)||l.push(h);if(o&&o.order&&null==i&&this._sort(l,o.order),o&&o.fields){var v=o.fields;if(null!=i&&null!=h)h=this._filterFields(h,v);else for(var b=0,y=l.length;b<y;b++)l[b]=this._filterFields(l[b],v)}if("Object"==s){for(var x={},w=0,_=l.length;w<_;w++){var k=l[w];x[k[this._idProp]]=k}return x}return null!=i?h:l}},{key:"getIds",value:function(t){var e,i,n=this._data,o=t&&En(t),s=t&&t.order,r=t&&t.type||this._options.type,a=mo(tr(n).call(n)),l=[];if(o)if(s){i=[];for(var h=0,d=a.length;h<d;h++){var u=a[h];o(e=this._getItem(u,r))&&i.push(e)}this._sort(i,s);for(var c=0,p=i.length;c<p;c++)l.push(i[c][this._idProp])}else for(var m=0,f=a.length;m<f;m++){var g=a[m];o(e=this._getItem(g,r))&&l.push(e[this._idProp])}else if(s){i=[];for(var v=0,b=a.length;v<b;v++){var y=a[v];i.push(n.get(y))}this._sort(i,s);for(var x=0,w=i.length;x<w;x++)l.push(i[x][this._idProp])}else for(var _=0,k=a.length;_<k;_++){var D=a[_];e=n.get(D),l.push(e[this._idProp])}return l}},{key:"getDataSet",value:function(){return this}},{key:"forEach",value:function(t,e){var i=e&&En(e),n=e&&e.type||this._options.type,o=this._data,s=mo(tr(o).call(o));if(e&&e.order)for(var r=this.get(e),a=0,l=r.length;a<l;a++){var h=r[a];t(h,h[this._idProp])}else for(var d=0,u=s.length;d<u;d++){var c=s[d],p=this._getItem(c,n);i&&!i(p)||t(p,c)}}},{key:"map",value:function(t,e){for(var i=e&&En(e),n=e&&e.type||this._options.type,o=[],s=this._data,r=mo(tr(s).call(s)),a=0,l=r.length;a<l;a++){var h=r[a],d=this._getItem(h,n);i&&!i(d)||o.push(t(d,h))}return e&&e.order&&this._sort(o,e.order),o}},{key:"_filterFields",value:function(t,e){var i;return t?da(i=Zo(e)?e:Yi(e)).call(i,(function(e,i){return e[i]=t[i],e}),{}):t}},{key:"_sort",value:function(t,e){if("string"==typeof e){var i=e;wr(t).call(t,(function(t,e){var n=t[i],o=e[i];return n>o?1:n<o?-1:0}))}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");wr(t).call(t,e)}}},{key:"remove",value:function(t,e){for(var i=[],n=[],o=Zo(t)?t:[t],s=0,r=o.length;s<r;s++){var a=this._remove(o[s]);if(a){var l=a[this._idProp];null!=l&&(i.push(l),n.push(a))}}return i.length&&this._trigger("remove",{items:i,oldData:n},e),i}},{key:"_remove",value:function(t){var e;if(Fa(t)?e=t:t&&"object"===zo(t)&&(e=t[this._idProp]),null!=e&&this._data.has(e)){var i=this._data.get(e)||null;return this._data.delete(e),--this.length,i}return null}},{key:"clear",value:function(t){for(var e,i=mo(tr(e=this._data).call(e)),n=[],o=0,s=i.length;o<s;o++)n.push(this._data.get(i[o]));return this._data.clear(),this.length=0,this._trigger("remove",{items:i,oldData:n},t),i}},{key:"max",value:function(t){var e=null,i=null,n=!0,o=!1,s=void 0;try{for(var r,a=Fi(Js(l=this._data).call(l));!(n=(r=a.next()).done);n=!0){var l,h=r.value,d=h[t];"number"==typeof d&&(null==i||d>i)&&(e=h,i=d)}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return e||null}},{key:"min",value:function(t){var e=null,i=null,n=!0,o=!1,s=void 0;try{for(var r,a=Fi(Js(l=this._data).call(l));!(n=(r=a.next()).done);n=!0){var l,h=r.value,d=h[t];"number"==typeof d&&(null==i||d<i)&&(e=h,i=d)}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}return e||null}},{key:"distinct",value:function(t){for(var e=this._data,i=mo(tr(e).call(e)),n=[],o=this._options.type&&this._options.type[t]||null,s=0,r=0,a=i.length;r<a;r++){for(var l=i[r],h=e.get(l)[t],d=!1,u=0;u<s;u++)if(n[u]==h){d=!0;break}d||void 0===h||(n[s]=h,s++)}if(o)for(var c=0,p=n.length;c<p;c++)n[c]=Na(n[c],o);return n}},{key:"_addItem",value:function(t){var e=t[this._idProp];if(null!=e){if(this._data.has(e))throw new Error("Cannot add item: item with id "+e+" already exists")}else e=Ma(),t[this._idProp]=e;for(var i={},n=Yi(t),o=0,s=n.length;o<s;o++){var r=n[o],a=this._type[r];i[r]=Na(t[r],a)}return this._data.set(e,i),++this.length,e}},{key:"_getItem",value:function(t,e){var i,n=this._data.get(t);if(!n)return null;var o=Yi(n);if(e){Wa(),i={};for(var s=0,r=o.length;s<r;s++){var a=o[s],l=n[a];i[a]=Na(l,e[a])}}else i=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=wa(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=wa(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},n);return null==i[this._idProp]&&(i[this._idProp]=n.id),i}},{key:"_updateItem",value:function(t){var e=t[this._idProp];if(null==e)throw new Error("Cannot update item: item has no id (item: "+Vs(t)+")");var i=this._data.get(e);if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var n=Yi(t),o=0,s=n.length;o<s;o++){var r=n[o],a=this._type[r];i[r]=Na(t[r],a)}return e}},{key:"stream",value:function(t){if(t){var e=this._data;return new Oa(Ai({},Yr,Tr.mark((function i(){var n,o,s,r,a,l,h;return Tr.wrap((function(i){for(;;)switch(i.prev=i.next){case 0:n=!0,o=!1,s=void 0,i.prev=3,r=Fi(t);case 5:if(n=(a=r.next()).done){i.next=14;break}if(l=a.value,null==(h=e.get(l))){i.next=11;break}return i.next=11,[l,h];case 11:n=!0,i.next=5;break;case 14:i.next=20;break;case 16:i.prev=16,i.t0=i.catch(3),o=!0,s=i.t0;case 20:i.prev=20,i.prev=21,n||null==r.return||r.return();case 23:if(i.prev=23,!o){i.next=26;break}throw s;case 26:return i.finish(23);case 27:return i.finish(20);case 28:case"end":return i.stop()}}),i,null,[[3,16,20,28],[21,,23,27]])}))))}var i;return new Oa(Ai({},Yr,va(i=Xs(this._data)).call(i,this._data)))}}]),e}(Ba),Xa=function(t){function e(t,i){var n,o;return ba(this,e),(o=ar(this,dr(e).call(this))).length=0,o._ids=new Ar,o._options=i||{},o._listener=va(n=o._onEvent).call(n,Bs(o)),o.setData(t),o}return pr(e,t),xa(e,[{key:"setData",value:function(t){if(this._data){this._data.off&&this._data.off("*",this._listener);var e=this._data.getIds({filter:En(this._options)}),i=this._data.get(e);this._ids.clear(),this.length=0,this._trigger("remove",{items:e,oldData:i})}if(null!=t){this._data=t;for(var n=this._data.getIds({filter:En(this._options)}),o=0,s=n.length;o<s;o++){var r=n[o];this._ids.add(r)}this.length=n.length,this._trigger("add",{items:n})}else this._data=new Aa;this._data.on&&this._data.on("*",this._listener)}},{key:"refresh",value:function(){for(var t=this._data.getIds({filter:En(this._options)}),e=mo(this._ids),i={},n=[],o=[],s=[],r=0,a=t.length;r<a;r++){var l=t[r];i[l]=!0,this._ids.has(l)||(n.push(l),this._ids.add(l))}for(var h=0,d=e.length;h<d;h++){var u=e[h],c=this._data.get(u);null==c?console.error("If you see this, report it please."):i[u]||(o.push(u),s.push(c),this._ids.delete(u))}this.length+=n.length-o.length,n.length&&this._trigger("add",{items:n}),o.length&&this._trigger("remove",{items:o,oldData:s})}},{key:"get",value:function(t,e){if(null==this._data)return null;var i,n=null;Fa(t)||Zo(t)?(n=t,i=e):i=t;var o=Ao({},this._options,i),s=En(this._options),r=i&&En(i);return s&&r&&(o.filter=function(t){return s(t)&&r(t)}),null==n?this._data.get(o):this._data.get(n,o)}},{key:"getIds",value:function(t){if(this._data.length){var e,i=En(this._options),n=null!=t?En(t):null;return e=n?i?function(t){return i(t)&&n(t)}:n:i,this._data.getIds({filter:e,order:t&&t.order})}return[]}},{key:"forEach",value:function(t,e){if(this._data){var i,n,o=En(this._options),s=e&&En(e);n=s?o?function(t){return o(t)&&s(t)}:s:o,mn(i=this._data).call(i,t,{filter:n,order:e&&e.order})}}},{key:"map",value:function(t,e){if(this._data){var i,n,o=En(this._options),s=e&&En(e);return n=s?o?function(t){return o(t)&&s(t)}:s:o,Ln(i=this._data).call(i,t,{filter:n,order:e&&e.order})}return[]}},{key:"getDataSet",value:function(){return this._data.getDataSet()}},{key:"stream",value:function(t){var e;return this._data.stream(t||Ai({},Yr,va(e=tr(this._ids)).call(e,this._ids)))}},{key:"dispose",value:function(){var t;(null===(t=this._data)||void 0===t?void 0:t.off)&&this._data.off("*",this._listener);var e="This data view has already been disposed of.";H(this,"_data",{get:function(){throw new Error(e)},set:function(){throw new Error(e)},configurable:!1})}},{key:"_onEvent",value:function(t,e,i){if(e&&e.items&&this._data){var n=e.items,o=[],s=[],r=[],a=[],l=[],h=[];switch(t){case"add":for(var d=0,u=n.length;d<u;d++){var c=n[d];this.get(c)&&(this._ids.add(c),o.push(c))}break;case"update":for(var p=0,m=n.length;p<m;p++){var f=n[p];this.get(f)?this._ids.has(f)?(s.push(f),l.push(e.data[p]),a.push(e.oldData[p])):(this._ids.add(f),o.push(f)):this._ids.has(f)&&(this._ids.delete(f),r.push(f),h.push(e.oldData[p]))}break;case"remove":for(var g=0,v=n.length;g<v;g++){var b=n[g];this._ids.has(b)&&(this._ids.delete(b),r.push(b),h.push(e.oldData[g]))}}this.length+=o.length-r.length,o.length&&this._trigger("add",{items:o},i),s.length&&this._trigger("update",{items:s,oldData:a,data:l},i),r.length&&this._trigger("remove",{items:r,oldData:h},i)}}}]),e}(Ba),Ta="undefined"!=typeof window&&window.moment||s;function Ya(t,e){var i=Yi(t);if(si){var n=si(t);e&&(n=En(n).call(n,(function(e){return kt(t,e).enumerable}))),i.push.apply(i,n)}return i}var Ea=/^\/?Date\((-?\d+)/i;function Ra(t,e){var i;if(void 0!==t){if(null===t)return null;if(!e)return t;if("string"!=typeof e&&!(e instanceof String))throw new Error("Type must be a string");switch(e){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return rs(t)&&!isNaN(Date.parse(t))?s(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(ss(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(s.isMoment(t))return new Date(t.valueOf());if(rs(t))return(i=Ea.exec(t))?new Date(Number(i[1])):s(new Date(t)).toDate();throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"Moment":if(ss(t))return s(t);if(t instanceof Date)return s(t.valueOf());if(s.isMoment(t))return s(t);if(rs(t))return i=Ea.exec(t),s(i?Number(i[1]):t);throw new Error("Cannot convert object of type "+us(t)+" to type Date");case"ISODate":if(ss(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(s.isMoment(t))return t.toDate().toISOString();if(rs(t))return(i=Ea.exec(t))?new Date(Number(i[1])).toISOString():s(t).format();throw new Error("Cannot convert object of type "+us(t)+" to type ISODate");case"ASPDate":if(ss(t))return"/Date("+t+")/";if(t instanceof Date||s.isMoment(t))return"/Date("+t.valueOf()+")/";if(rs(t))return"/Date("+((i=Ea.exec(t))?new Date(Number(i[1])).valueOf():new Date(t).valueOf())+")/";throw new Error("Cannot convert object of type "+us(t)+" to type ASPDate");default:throw new Error("Unknown type ".concat(e))}}}function Ua(t){var e,i,n,o,s,r,a=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{start:"Date",end:"Date"},l=t._idProp,h=new Aa({fieldId:l}),d=Ln(e=_a(t)).call(e,(function(t){var e;return da(e=Yi(t)).call(e,(function(e,i){return e[i]=Ra(t[i],a[i]),e}),{})})).to(h);return d.all().start(),{add:function(){var e;return(e=t.getDataSet()).add.apply(e,arguments)},remove:function(){var e;return(e=t.getDataSet()).remove.apply(e,arguments)},update:function(){var e;return(e=t.getDataSet()).update.apply(e,arguments)},updateOnly:function(){var e;return(e=t.getDataSet()).updateOnly.apply(e,arguments)},forEach:va(i=mn(h)).call(i,h),get:va(n=h.get).call(n,h),getIds:va(o=h.getIds).call(o,h),off:va(s=h.off).call(s,h),on:va(r=h.on).call(r,h),get length(){return h.length},idProp:l,type:a,rawDS:t,coercedDS:h,dispose:function(){return d.stop()}}}var za=function(t){for(var e=1;e<arguments.length;e++){var i,n=null!=arguments[e]?arguments[e]:{};if(e%2)mn(i=Ya(Object(n),!0)).call(i,(function(e){Ai(t,e,n[e])}));else if(yt)ut(t,yt(n));else{var o;mn(o=Ya(Object(n))).call(o,(function(e){H(t,e,kt(n,e))}))}}return t}({},Is,{convert:Ra}),Va=ji.trim,ja=a.parseFloat,Ha=1/ja(Ei+"-0")!=-1/0?function(t){var e=Va(String(t)),i=ja(e);return 0===i&&"-"==e.charAt(0)?-0:i}:ja;V({global:!0,forced:parseFloat!=Ha},{parseFloat:Ha});var Pa=W.parseFloat,Ja=function(){function t(e,i){ba(this,t),this.options=null,this.props=null}return xa(t,[{key:"setOptions",value:function(t){t&&za.extend(this.options,t)}},{key:"redraw",value:function(){return!1}},{key:"destroy",value:function(){}},{key:"_isResized",value:function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t}}]),t}();V({target:"String",proto:!0},{repeat:"".repeat||function(t){var e=String(b(this)),i="",n=K(t);if(n<0||n==1/0)throw RangeError("Wrong number of repetitions");for(;n>0;(n>>>=1)&&(e+=e))1&n&&(i+=e);return i}});var Ka=Pi("String").repeat,qa=String.prototype,$a=function(t){var e=t.repeat;return"string"==typeof t||t===qa||t instanceof String&&e===qa.repeat?Ka:e};function tl(t,e,i){if(i&&!Zo(i))return tl(t,e,[i]);if(e.hiddenDates=[],i&&1==Zo(i)){for(var n,o=0;o<i.length;o++)if(void 0===$a(i[o])){var s={};s.start=t(i[o].start).toDate().valueOf(),s.end=t(i[o].end).toDate().valueOf(),e.hiddenDates.push(s)}wr(n=e.hiddenDates).call(n,(function(t,e){return t.start-e.start}))}}function el(t,e,i){if(i&&!Zo(i))return el(t,e,[i]);if(i&&void 0!==e.domProps.centerContainer.width){tl(t,e,i);for(var n=t(e.range.start),o=t(e.range.end),s=(e.range.end-e.range.start)/e.domProps.centerContainer.width,r=0;r<i.length;r++)if(void 0!==$a(i[r])){var a=t(i[r].start),l=t(i[r].end);if("Invalid Date"==a._d)throw new Error("Supplied start date is not valid: ".concat(i[r].start));if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: ".concat(i[r].end));if(l-a>=4*s){var h=0,d=o.clone();switch($a(i[r])){case"daily":a.day()!=l.day()&&(h=1),a.dayOfYear(n.dayOfYear()),a.year(n.year()),a.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-h,"days"),d.add(1,"weeks");break;case"weekly":var u=l.diff(a,"days"),c=a.day();a.date(n.date()),a.month(n.month()),a.year(n.year()),l=a.clone(),a.day(c),l.day(c),l.add(u,"days"),a.subtract(1,"weeks"),l.subtract(1,"weeks"),d.add(1,"weeks");break;case"monthly":a.month()!=l.month()&&(h=1),a.month(n.month()),a.year(n.year()),a.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(h,"months"),d.add(1,"months");break;case"yearly":a.year()!=l.year()&&(h=1),a.year(n.year()),a.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(h,"years"),d.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",$a(i[r]))}for(;a<d;)switch(e.hiddenDates.push({start:a.valueOf(),end:l.valueOf()}),$a(i[r])){case"daily":a.add(1,"days"),l.add(1,"days");break;case"weekly":a.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":a.add(1,"months"),l.add(1,"months");break;case"yearly":a.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",$a(i[r]))}e.hiddenDates.push({start:a.valueOf(),end:l.valueOf()})}}il(e);var p=cl(e.range.start,e.hiddenDates),m=cl(e.range.end,e.hiddenDates),f=e.range.start,g=e.range.end;1==p.hidden&&(f=1==e.range.startToFront?p.startDate-1:p.endDate+1),1==m.hidden&&(g=1==e.range.endToFront?m.startDate-1:m.endDate+1),1!=p.hidden&&1!=m.hidden||e.range._applyRange(f,g)}}function il(t){for(var e,i=t.hiddenDates,n=[],o=0;o<i.length;o++)for(var s=0;s<i.length;s++)o!=s&&1!=i[s].remove&&1!=i[o].remove&&(i[s].start>=i[o].start&&i[s].end<=i[o].end?i[s].remove=!0:i[s].start>=i[o].start&&i[s].start<=i[o].end?(i[o].end=i[s].end,i[s].remove=!0):i[s].end>=i[o].start&&i[s].end<=i[o].end&&(i[o].start=i[s].start,i[s].remove=!0));for(o=0;o<i.length;o++)!0!==i[o].remove&&n.push(i[o]);t.hiddenDates=n,wr(e=t.hiddenDates).call(e,(function(t,e){return t.start-e.start}))}function nl(t,e,i){for(var n=!1,o=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(o>=r&&o<a){n=!0;break}}if(1==n&&o<e._end.valueOf()&&o!=i){var l=t(i),h=t(a);l.year()!=h.year()?e.switchedYear=!0:l.month()!=h.month()?e.switchedMonth=!0:l.dayOfYear()!=h.dayOfYear()&&(e.switchedDay=!0),e.current=h}}function ol(t,e,i){var n;if(0==t.body.hiddenDates.length)return n=t.range.conversion(i),(e.valueOf()-n.offset)*n.scale;var o=cl(e,t.body.hiddenDates);1==o.hidden&&(e=o.startDate);var s=rl(t.body.hiddenDates,t.range.start,t.range.end);if(e<t.range.start){n=t.range.conversion(i,s);var r=al(t.body.hiddenDates,e,n.offset);return e=t.options.moment(e).toDate().valueOf(),e+=r,-(n.offset-e.valueOf())*n.scale}if(e>t.range.end){var a={start:t.range.start,end:e};return e=ll(t.options.moment,t.body.hiddenDates,a,e),n=t.range.conversion(i,s),(e.valueOf()-n.offset)*n.scale}return e=ll(t.options.moment,t.body.hiddenDates,t.range,e),n=t.range.conversion(i,s),(e.valueOf()-n.offset)*n.scale}function sl(t,e,i){if(0==t.body.hiddenDates.length){var n=t.range.conversion(i);return new Date(e/n.scale+n.offset)}var o=rl(t.body.hiddenDates,t.range.start,t.range.end),s=(t.range.end-t.range.start-o)*e/i,r=dl(t.body.hiddenDates,t.range,s);return new Date(r+s+t.range.start)}function rl(t,e,i){for(var n=0,o=0;o<t.length;o++){var s=t[o].start,r=t[o].end;s>=e&&r<i&&(n+=r-s)}return n}function al(t,e,i){for(var n=0,o=0;o<t.length;o++){var s=t[o].start,r=t[o].end;s>=e&&r<=i&&(n+=r-s)}return n}function ll(t,e,i,n){return n=t(n).toDate().valueOf(),n-=hl(t,e,i,n)}function hl(t,e,i,n){var o=0;n=t(n).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&n>=a&&(o+=a-r)}return o}function dl(t,e,i){for(var n=0,o=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,l=t[r].end;if(a>=e.start&&l<e.end){if(o+=a-s,s=l,o>=i)break;n+=l-a}}return n}function ul(t,e,i,n){var o=cl(e,t);return 1==o.hidden?i<0?1==n?o.startDate-(o.endDate-e)-1:o.startDate-1:1==n?o.endDate+(e-o.startDate)+1:o.endDate+1:e}function cl(t,e){for(var i=0;i<e.length;i++){var n=e[i].start,o=e[i].end;if(t>=n&&t<o)return{hidden:!0,startDate:n,endDate:o}}return{hidden:!1,startDate:n,endDate:o}}var pl=Object.freeze({__proto__:null,convertHiddenOptions:tl,updateHiddenDates:el,removeDuplicates:il,printDates:function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},stepOverHiddenDates:nl,toScreen:ol,toTime:sl,getHiddenDurationBetween:rl,getHiddenDurationBeforeStart:al,correctTimeForHidden:ll,getHiddenDurationBefore:hl,getAccumulatedHiddenDuration:dl,snapAwayFromHidden:ul,getIsHidden:cl}),ml=function(t){function e(t,i){var n,o,s,r,a,l,h,d;ba(this,e),d=ar(this,dr(e).call(this));var u=Ta().hours(0).minutes(0).seconds(0).milliseconds(0),c=u.clone().add(-3,"days").valueOf(),p=u.clone().add(3,"days").valueOf();return d.millisecondsPerPixelCache=void 0,void 0===i?(d.start=c,d.end=p):(d.start=i.start||c,d.end=i.end||p),d.rolling=!1,d.body=t,d.deltaDifference=0,d.scaleOffset=0,d.startToFront=!1,d.endToFront=!0,d.defaultOptions={rtl:!1,start:null,end:null,moment:Ta,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10,rollingMode:{follow:!1,offset:.5}},d.options=za.extend({},d.defaultOptions),d.props={touch:{}},d.animationTimer=null,d.body.emitter.on("panstart",va(n=d._onDragStart).call(n,Bs(d))),d.body.emitter.on("panmove",va(o=d._onDrag).call(o,Bs(d))),d.body.emitter.on("panend",va(s=d._onDragEnd).call(s,Bs(d))),d.body.emitter.on("mousewheel",va(r=d._onMouseWheel).call(r,Bs(d))),d.body.emitter.on("touch",va(a=d._onTouch).call(a,Bs(d))),d.body.emitter.on("pinch",va(l=d._onPinch).call(l,Bs(d))),d.body.dom.rollingModeBtn.addEventListener("click",va(h=d.startRolling).call(h,Bs(d))),d.setOptions(i),d}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){if(t){za.selectiveExtend(["animation","direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","zoomFriction","rtl","showCurrentTime","rollingMode","horizontalScroll"],this.options,t),t.rollingMode&&t.rollingMode.follow&&this.startRolling(),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}}},{key:"startRolling",value:function(){var t=this;!function e(){t.stopRolling(),t.rolling=!0;var i=t.end-t.start,n=za.convert(new Date,"Date").valueOf(),o=t.options.rollingMode&&t.options.rollingMode.offset||.5,s=n-i*o,r=n+i*(1-o);t.setRange(s,r,{animation:!1}),(i=1/t.conversion(t.body.domProps.center.width).scale/10)<30&&(i=30),i>1e3&&(i=1e3),t.body.dom.rollingModeBtn.style.visibility="hidden",t.currentTimeTimer=qr(e,i)}()}},{key:"stopRolling",value:function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),this.rolling=!1,this.body.dom.rollingModeBtn.style.visibility="visible")}},{key:"setRange",value:function(t,e,i,n,o){i||(i={}),!0!==i.byUser&&(i.byUser=!1);var s=this,r=null!=t?za.convert(t,"Date").valueOf():null,a=null!=e?za.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),this.millisecondsPerPixelCache=void 0,i.animation){var l,h=this.start,d=this.end,u="object"===zo(i.animation)&&"duration"in i.animation?i.animation.duration:500,c="object"===zo(i.animation)&&"easingFunction"in i.animation?i.animation.easingFunction:"easeInOutQuad",p=za.easingFunctions[c];if(!p)throw new Error(Pn(l="Unknown easing function ".concat(Vs(c),". Choose from: ")).call(l,Yi(za.easingFunctions).join(", ")));var m=Jo(),f=!1;return function t(){if(!s.props.touch.dragging){var e=Jo()-m,l=p(e/u),c=e>u,v=c||null===r?r:h+(r-h)*l,b=c||null===a?a:d+(a-d)*l;g=s._applyRange(v,b),el(s.options.moment,s.body,s.options.hiddenDates),f=f||g;var y={start:new Date(s.start),end:new Date(s.end),byUser:i.byUser,event:i.event};if(o&&o(l,g,c),g&&s.body.emitter.emit("rangechange",y),c){if(f&&(s.body.emitter.emit("rangechanged",y),n))return n()}else s.animationTimer=qr(t,20)}}()}var g=this._applyRange(r,a);if(el(this.options.moment,this.body,this.options.hiddenDates),g){var v={start:new Date(this.start),end:new Date(this.end),byUser:i.byUser,event:i.event};if(this.body.emitter.emit("rangechange",v),clearTimeout(s.timeoutID),s.timeoutID=qr((function(){s.body.emitter.emit("rangechanged",v)}),200),n)return n()}}},{key:"getMillisecondsPerPixel",value:function(){return void 0===this.millisecondsPerPixelCache&&(this.millisecondsPerPixelCache=(this.end-this.start)/this.body.dom.center.clientWidth),this.millisecondsPerPixelCache}},{key:"_cancelAnimation",value:function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)}},{key:"_applyRange",value:function(t,e){var i,n=null!=t?za.convert(t,"Date").valueOf():this.start,o=null!=e?za.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?za.convert(this.options.max,"Date").valueOf():null,r=null!=this.options.min?za.convert(this.options.min,"Date").valueOf():null;if(isNaN(n)||null===n)throw new Error('Invalid start "'.concat(t,'"'));if(isNaN(o)||null===o)throw new Error('Invalid end "'.concat(e,'"'));if(o<n&&(o=n),null!==r&&n<r&&(n+=i=r-n,o+=i,null!=s&&o>s&&(o=s)),null!==s&&o>s&&(n-=i=o-s,o-=i,null!=r&&n<r&&(n=r)),null!==this.options.zoomMin){var a=Pa(this.options.zoomMin);if(a<0&&(a=0),o-n<a){this.end-this.start===a&&n>=this.start-.5&&o<=this.end?(n=this.start,o=this.end):(n-=(i=a-(o-n))/2,o+=i/2)}}if(null!==this.options.zoomMax){var l=Pa(this.options.zoomMax);l<0&&(l=0),o-n>l&&(this.end-this.start===l&&n<this.start&&o>this.end?(n=this.start,o=this.end):(n+=(i=o-n-l)/2,o-=i/2))}var h=this.start!=n||this.end!=o;return n>=this.start&&n<=this.end||o>=this.start&&o<=this.end||this.start>=n&&this.start<=o||this.end>=n&&this.end<=o||this.body.emitter.emit("checkRangedItems"),this.start=n,this.end=o,h}},{key:"getRange",value:function(){return{start:this.start,end:this.end}}},{key:"conversion",value:function(t,i){return e.conversion(this.start,this.end,t,i)}},{key:"_onDragStart",value:function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.stopRolling(),this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))}},{key:"_onDrag",value:function(t){if(t&&this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;fl(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var n=this.props.touch.end-this.props.touch.start;n-=rl(this.body.hiddenDates,this.start,this.end);var o,s="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;o=this.options.rtl?i/s*n:-i/s*n;var r=this.props.touch.start+o,a=this.props.touch.end+o,l=ul(this.body.hiddenDates,r,this.previousDelta-i,!0),h=ul(this.body.hiddenDates,a,this.previousDelta-i,!0);if(l!=r||h!=a)return this.deltaDifference+=i,this.props.touch.start=l,this.props.touch.end=h,void this._onDrag(t);this.previousDelta=i,this._applyRange(r,a);var d=new Date(this.start),u=new Date(this.end);this.body.emitter.emit("rangechange",{start:d,end:u,byUser:!0,event:t}),this.body.emitter.emit("panmove")}}},{key:"_onDragEnd",value:function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0,event:t}))}},{key:"_onMouseWheel",value:function(t){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail?e=-t.detail/3:t.deltaY&&(e=-t.deltaY/3),!(this.options.zoomKey&&!t[this.options.zoomKey]&&this.options.zoomable||!this.options.zoomable&&this.options.moveable)&&this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&e){var i,n,o=this.options.zoomFriction||5;if(i=e<0?1-e/o:1/(1+e/o),this.rolling){var s=this.options.rollingMode&&this.options.rollingMode.offset||.5;n=this.start+(this.end-this.start)*s}else{var r=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center);n=this._pointerToDate(r)}this.zoom(i,n,e,t),t.preventDefault()}}},{key:"_onTouch",value:function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.props.touch.centerDate=null,this.scaleOffset=0,this.deltaDifference=0,za.preventDefault(t)}},{key:"_onPinch",value:function(t){if(this.options.zoomable&&this.options.moveable){za.preventDefault(t),this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center),this.props.touch.centerDate=this._pointerToDate(this.props.touch.center)),this.stopRolling();var e=1/(t.scale+this.scaleOffset),i=this.props.touch.centerDate,n=rl(this.body.hiddenDates,this.start,this.end),o=hl(this.options.moment,this.body.hiddenDates,this,i),s=n-o,r=i-o+(this.props.touch.start-(i-o))*e,a=i+s+(this.props.touch.end-(i+s))*e;this.startToFront=1-e<=0,this.endToFront=e-1<=0;var l=ul(this.body.hiddenDates,r,1-e,!0),h=ul(this.body.hiddenDates,a,e-1,!0);l==r&&h==a||(this.props.touch.start=l,this.props.touch.end=h,this.scaleOffset=1-t.scale,r=l,a=h);var d={animation:!1,byUser:!0,event:t};this.setRange(r,a,d),this.startToFront=!1,this.endToFront=!0}}},{key:"_isInsideRange",value:function(t){var e=t.center?t.center.x:t.clientX,i=this.body.dom.centerContainer.getBoundingClientRect(),n=this.options.rtl?e-i.left:i.right-e,o=this.body.util.toTime(n);return o>=this.start&&o<=this.end}},{key:"_pointerToDate",value:function(t){var e,i=this.options.direction;if(fl(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var n=this.body.domProps.center.height;return e=this.conversion(n),t.y/e.scale+e.offset}},{key:"getPointer",value:function(t,e){var i=e.getBoundingClientRect();return this.options.rtl?{x:i.right-t.x,y:t.y-i.top}:{x:t.x-i.left,y:t.y-i.top}}},{key:"zoom",value:function(t,e,i,n){null==e&&(e=(this.start+this.end)/2);var o=rl(this.body.hiddenDates,this.start,this.end),s=hl(this.options.moment,this.body.hiddenDates,this,e),r=o-s,a=e-s+(this.start-(e-s))*t,l=e+r+(this.end-(e+r))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var h=ul(this.body.hiddenDates,a,i,!0),d=ul(this.body.hiddenDates,l,-i,!0);h==a&&d==l||(a=h,l=d);var u={animation:!1,byUser:!0,event:n};this.setRange(a,l,u),this.startToFront=!1,this.endToFront=!0}},{key:"move",value:function(t){var e=this.end-this.start,i=this.start+e*t,n=this.end+e*t;this.start=i,this.end=n}},{key:"moveTo",value:function(t){var e=(this.start+this.end)/2-t,i=this.start-e,n=this.end-e;this.setRange(i,n,{animation:!1,byUser:!0,event:null})}}],[{key:"conversion",value:function(t,e,i,n){return void 0===n&&(n=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-n)}:{offset:0,scale:1}}}]),e}(Ja);function fl(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'.concat(t,'". Choose "horizontal" or "vertical".'))}var gl=W.setInterval,vl=n((function(t){function e(t){if(t)return function(t){for(var i in e.prototype)t[i]=e.prototype[i];return t}(t)}t.exports=e,e.prototype.on=e.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks["$"+t]=this._callbacks["$"+t]||[]).push(e),this},e.prototype.once=function(t,e){function i(){this.off(t,i),e.apply(this,arguments)}return i.fn=e,this.on(t,i),this},e.prototype.off=e.prototype.removeListener=e.prototype.removeAllListeners=e.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i,n=this._callbacks["$"+t];if(!n)return this;if(1==arguments.length)return delete this._callbacks["$"+t],this;for(var o=0;o<n.length;o++)if((i=n[o])===e||i.fn===e){n.splice(o,1);break}return 0===n.length&&delete this._callbacks["$"+t],this},e.prototype.emit=function(t){this._callbacks=this._callbacks||{};for(var e=new Array(arguments.length-1),i=this._callbacks["$"+t],n=1;n<arguments.length;n++)e[n-1]=arguments[n];if(i){n=0;for(var o=(i=i.slice(0)).length;n<o;++n)i[n].apply(this,e)}return this},e.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks["$"+t]||[]},e.prototype.hasListeners=function(t){return!!this.listeners(t).length}})),bl=n((function(t,e){var i;t.exports=(i=null,function t(e,n){var o=n||{preventDefault:!1};if(e.Manager){var s=e,r=function(e,i){var n=Object.create(o);return i&&s.assign(n,i),t(new s(e,n),n)};return s.assign(r,s),r.Manager=function(e,i){var n=Object.create(o);return i&&s.assign(n,i),t(new s.Manager(e,n),n)},r}var a=Object.create(e),l=e.element;function h(t){return t.match(/[^ ]+/g)}function d(t){if("hammer.input"!==t.type){if(t.srcEvent._handled||(t.srcEvent._handled={}),t.srcEvent._handled[t.type])return;t.srcEvent._handled[t.type]=!0}var e=!1;t.stopPropagation=function(){e=!0};var n=t.srcEvent.stopPropagation.bind(t.srcEvent);"function"==typeof n&&(t.srcEvent.stopPropagation=function(){n(),t.stopPropagation()}),t.firstTarget=i;for(var o=i;o&&!e;){var s=o.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[t.type])for(var l=0;l<r.length&&!e;l++)r[l](t);o=o.parentNode}}return l.hammer||(l.hammer=[]),l.hammer.push(a),e.on("hammer.input",(function(t){!0!==o.preventDefault&&o.preventDefault!==t.pointerType||t.preventDefault(),t.isFirst&&(i=t.target)})),a._handlers={},a.on=function(t,i){return h(t).forEach((function(t){var n=a._handlers[t];n||(a._handlers[t]=n=[],e.on(t,d)),n.push(i)})),a},a.off=function(t,i){return h(t).forEach((function(t){var n=a._handlers[t];n&&((n=i?n.filter((function(t){return t!==i})):[]).length>0?a._handlers[t]=n:(e.off(t,d),delete a._handlers[t]))})),a},a.emit=function(t,n){i=n.target,e.emit(t,n)},a.destroy=function(){var t=e.element.hammer,i=t.indexOf(a);-1!==i&&t.splice(i,1),t.length||delete e.element.hammer,a._handlers={},e.destroy()},a})}));
-/*! Hammer.JS - v2.0.17-rc - 2019-12-16
-	 * http://naver.github.io/egjs
-	 *
-	 * Forked By Naver egjs
-	 * Copyright (c) hammerjs
-	 * Licensed under the MIT license */
-function yl(){return(yl=Object.assign||function(t){for(var e=1;e<arguments.length;e++){var i=arguments[e];for(var n in i)Object.prototype.hasOwnProperty.call(i,n)&&(t[n]=i[n])}return t}).apply(this,arguments)}function xl(t,e){t.prototype=Object.create(e.prototype),t.prototype.constructor=t,t.__proto__=e}function wl(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}var _l,kl="function"!=typeof Object.assign?function(t){if(null==t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var n=arguments[i];if(null!=n)for(var o in n)n.hasOwnProperty(o)&&(e[o]=n[o])}return e}:Object.assign,Dl=["","webkit","Moz","MS","ms","o"],Il="undefined"==typeof document?{style:{}}:document.createElement("div"),Cl=Math.round,Ll=Math.abs,Gl=Date.now;function Ml(t,e){for(var i,n,o=e[0].toUpperCase()+e.slice(1),s=0;s<Dl.length;){if((n=(i=Dl[s])?i+o:e)in t)return n;s++}}_l="undefined"==typeof window?{}:window;var Sl=Ml(Il.style,"touchAction"),Ql=void 0!==Sl;var Nl=function(){if(!Ql)return!1;var t={},e=_l.CSS&&_l.CSS.supports;return["auto","manipulation","pan-y","pan-x","pan-x pan-y","none"].forEach((function(i){return t[i]=!e||_l.CSS.supports("touch-action",i)})),t}(),Fl="ontouchstart"in _l,Zl=void 0!==Ml(_l,"PointerEvent"),Bl=Fl&&/mobile|tablet|ip(ad|hone|od)|android/i.test(navigator.userAgent),Ol=["x","y"],Wl=["clientX","clientY"];function Al(t,e,i){var n;if(t)if(t.forEach)t.forEach(e,i);else if(void 0!==t.length)for(n=0;n<t.length;)e.call(i,t[n],n,t),n++;else for(n in t)t.hasOwnProperty(n)&&e.call(i,t[n],n,t)}function Xl(t,e){return"function"==typeof t?t.apply(e&&e[0]||void 0,e):t}function Tl(t,e){return t.indexOf(e)>-1}var Yl=function(){function t(t,e){this.manager=t,this.set(e)}var e=t.prototype;return e.set=function(t){"compute"===t&&(t=this.compute()),Ql&&this.manager.element.style&&Nl[t]&&(this.manager.element.style[Sl]=t),this.actions=t.toLowerCase().trim()},e.update=function(){this.set(this.manager.options.touchAction)},e.compute=function(){var t=[];return Al(this.manager.recognizers,(function(e){Xl(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))})),function(t){if(Tl(t,"none"))return"none";var e=Tl(t,"pan-x"),i=Tl(t,"pan-y");return e&&i?"none":e||i?e?"pan-x":"pan-y":Tl(t,"manipulation")?"manipulation":"auto"}(t.join(" "))},e.preventDefaults=function(t){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)e.preventDefault();else{var n=this.actions,o=Tl(n,"none")&&!Nl.none,s=Tl(n,"pan-y")&&!Nl["pan-y"],r=Tl(n,"pan-x")&&!Nl["pan-x"];if(o){var a=1===t.pointers.length,l=t.distance<2,h=t.deltaTime<250;if(a&&l&&h)return}if(!r||!s)return o||s&&6&i||r&&24&i?this.preventSrc(e):void 0}},e.preventSrc=function(t){this.manager.session.prevented=!0,t.preventDefault()},t}();function El(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}function Rl(t){var e=t.length;if(1===e)return{x:Cl(t[0].clientX),y:Cl(t[0].clientY)};for(var i=0,n=0,o=0;o<e;)i+=t[o].clientX,n+=t[o].clientY,o++;return{x:Cl(i/e),y:Cl(n/e)}}function Ul(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:Cl(t.pointers[i].clientX),clientY:Cl(t.pointers[i].clientY)},i++;return{timeStamp:Gl(),pointers:e,center:Rl(e),deltaX:t.deltaX,deltaY:t.deltaY}}function zl(t,e,i){i||(i=Ol);var n=e[i[0]]-t[i[0]],o=e[i[1]]-t[i[1]];return Math.sqrt(n*n+o*o)}function Vl(t,e,i){i||(i=Ol);var n=e[i[0]]-t[i[0]],o=e[i[1]]-t[i[1]];return 180*Math.atan2(o,n)/Math.PI}function jl(t,e){return t===e?1:Ll(t)>=Ll(e)?t<0?2:4:e<0?8:16}function Hl(t,e,i){return{x:e/t||0,y:i/t||0}}function Pl(t,e){var i=t.session,n=e.pointers,o=n.length;i.firstInput||(i.firstInput=Ul(e)),o>1&&!i.firstMultiple?i.firstMultiple=Ul(e):1===o&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,l=e.center=Rl(n);e.timeStamp=Gl(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=Vl(a,l),e.distance=zl(a,l),function(t,e){var i=e.center,n=t.offsetDelta||{},o=t.prevDelta||{},s=t.prevInput||{};1!==e.eventType&&4!==s.eventType||(o=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},n=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=o.x+(i.x-n.x),e.deltaY=o.y+(i.y-n.y)}(i,e),e.offsetDirection=jl(e.deltaX,e.deltaY);var h,d,u=Hl(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=u.x,e.overallVelocityY=u.y,e.overallVelocity=Ll(u.x)>Ll(u.y)?u.x:u.y,e.scale=r?(h=r.pointers,zl((d=n)[0],d[1],Wl)/zl(h[0],h[1],Wl)):1,e.rotation=r?function(t,e){return Vl(e[1],e[0],Wl)+Vl(t[1],t[0],Wl)}(r.pointers,n):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,function(t,e){var i,n,o,s,r=t.lastInterval||e,a=e.timeStamp-r.timeStamp;if(8!==e.eventType&&(a>25||void 0===r.velocity)){var l=e.deltaX-r.deltaX,h=e.deltaY-r.deltaY,d=Hl(a,l,h);n=d.x,o=d.y,i=Ll(d.x)>Ll(d.y)?d.x:d.y,s=jl(l,h),t.lastInterval=e}else i=r.velocity,n=r.velocityX,o=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=n,e.velocityY=o,e.direction=s}(i,e);var c,p=t.element,m=e.srcEvent;El(c=m.composedPath?m.composedPath()[0]:m.path?m.path[0]:m.target,p)&&(p=c),e.target=p}function Jl(t,e,i){var n=i.pointers.length,o=i.changedPointers.length,s=1&e&&n-o==0,r=12&e&&n-o==0;i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,Pl(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function Kl(t){return t.trim().split(/\s+/g)}function ql(t,e,i){Al(Kl(e),(function(e){t.addEventListener(e,i,!1)}))}function $l(t,e,i){Al(Kl(e),(function(e){t.removeEventListener(e,i,!1)}))}function th(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||window}var eh=function(){function t(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){Xl(t.options.enable,[t])&&i.handler(e)},this.init()}var e=t.prototype;return e.handler=function(){},e.init=function(){this.evEl&&ql(this.element,this.evEl,this.domHandler),this.evTarget&&ql(this.target,this.evTarget,this.domHandler),this.evWin&&ql(th(this.element),this.evWin,this.domHandler)},e.destroy=function(){this.evEl&&$l(this.element,this.evEl,this.domHandler),this.evTarget&&$l(this.target,this.evTarget,this.domHandler),this.evWin&&$l(th(this.element),this.evWin,this.domHandler)},t}();function ih(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var n=0;n<t.length;){if(i&&t[n][i]==e||!i&&t[n]===e)return n;n++}return-1}var nh={pointerdown:1,pointermove:2,pointerup:4,pointercancel:8,pointerout:8},oh={2:"touch",3:"pen",4:"mouse",5:"kinect"},sh="pointerdown",rh="pointermove pointerup pointercancel";_l.MSPointerEvent&&!_l.PointerEvent&&(sh="MSPointerDown",rh="MSPointerMove MSPointerUp MSPointerCancel");var ah=function(t){function e(){var i,n=e.prototype;return n.evEl=sh,n.evWin=rh,(i=t.apply(this,arguments)||this).store=i.manager.session.pointerEvents=[],i}return xl(e,t),e.prototype.handler=function(t){var e=this.store,i=!1,n=t.type.toLowerCase().replace("ms",""),o=nh[n],s=oh[t.pointerType]||t.pointerType,r="touch"===s,a=ih(e,t.pointerId,"pointerId");1&o&&(0===t.button||r)?a<0&&(e.push(t),a=e.length-1):12&o&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,o,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))},e}(eh);function lh(t){return Array.prototype.slice.call(t,0)}function hh(t,e,i){for(var n=[],o=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];ih(o,r)<0&&n.push(t[s]),o[s]=r,s++}return i&&(n=e?n.sort((function(t,i){return t[e]>i[e]})):n.sort()),n}var dh={touchstart:1,touchmove:2,touchend:4,touchcancel:8},uh=function(t){function e(){var i;return e.prototype.evTarget="touchstart touchmove touchend touchcancel",(i=t.apply(this,arguments)||this).targetIds={},i}return xl(e,t),e.prototype.handler=function(t){var e=dh[t.type],i=ch.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})},e}(eh);function ch(t,e){var i,n,o=lh(t.touches),s=this.targetIds;if(3&e&&1===o.length)return s[o[0].identifier]=!0,[o,o];var r=lh(t.changedTouches),a=[],l=this.target;if(n=o.filter((function(t){return El(t.target,l)})),1===e)for(i=0;i<n.length;)s[n[i].identifier]=!0,i++;for(i=0;i<r.length;)s[r[i].identifier]&&a.push(r[i]),12&e&&delete s[r[i].identifier],i++;return a.length?[hh(n.concat(a),"identifier",!0),a]:void 0}var ph={mousedown:1,mousemove:2,mouseup:4},mh=function(t){function e(){var i,n=e.prototype;return n.evEl="mousedown",n.evWin="mousemove mouseup",(i=t.apply(this,arguments)||this).pressed=!1,i}return xl(e,t),e.prototype.handler=function(t){var e=ph[t.type];1&e&&0===t.button&&(this.pressed=!0),2&e&&1!==t.which&&(e=4),this.pressed&&(4&e&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:"mouse",srcEvent:t}))},e}(eh);function fh(t){var e=t.changedPointers[0];if(e.identifier===this.primaryTouch){var i={x:e.clientX,y:e.clientY},n=this.lastTouches;this.lastTouches.push(i);setTimeout((function(){var t=n.indexOf(i);t>-1&&n.splice(t,1)}),2500)}}function gh(t,e){1&t?(this.primaryTouch=e.changedPointers[0].identifier,fh.call(this,e)):12&t&&fh.call(this,e)}function vh(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,n=0;n<this.lastTouches.length;n++){var o=this.lastTouches[n],s=Math.abs(e-o.x),r=Math.abs(i-o.y);if(s<=25&&r<=25)return!0}return!1}var bh=function(){return function(t){function e(e,i){var n;return(n=t.call(this,e,i)||this).handler=function(t,e,i){var o="touch"===i.pointerType,s="mouse"===i.pointerType;if(!(s&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(o)gh.call(wl(wl(n)),e,i);else if(s&&vh.call(wl(wl(n)),i))return;n.callback(t,e,i)}},n.touch=new uh(n.manager,n.handler),n.mouse=new mh(n.manager,n.handler),n.primaryTouch=null,n.lastTouches=[],n}return xl(e,t),e.prototype.destroy=function(){this.touch.destroy(),this.mouse.destroy()},e}(eh)}();function yh(t,e,i){return!!Array.isArray(t)&&(Al(t,i[e],i),!0)}var xh=1;function wh(t,e){var i=e.manager;return i?i.get(t):t}function _h(t){return 16&t?"cancel":8&t?"end":4&t?"move":2&t?"start":""}var kh=function(){function t(t){void 0===t&&(t={}),this.options=yl({enable:!0},t),this.id=xh++,this.manager=null,this.state=1,this.simultaneous={},this.requireFail=[]}var e=t.prototype;return e.set=function(t){return kl(this.options,t),this.manager&&this.manager.touchAction.update(),this},e.recognizeWith=function(t){if(yh(t,"recognizeWith",this))return this;var e=this.simultaneous;return e[(t=wh(t,this)).id]||(e[t.id]=t,t.recognizeWith(this)),this},e.dropRecognizeWith=function(t){return yh(t,"dropRecognizeWith",this)||(t=wh(t,this),delete this.simultaneous[t.id]),this},e.requireFailure=function(t){if(yh(t,"requireFailure",this))return this;var e=this.requireFail;return-1===ih(e,t=wh(t,this))&&(e.push(t),t.requireFailure(this)),this},e.dropRequireFailure=function(t){if(yh(t,"dropRequireFailure",this))return this;t=wh(t,this);var e=ih(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},e.hasRequireFailures=function(){return this.requireFail.length>0},e.canRecognizeWith=function(t){return!!this.simultaneous[t.id]},e.emit=function(t){var e=this,i=this.state;function n(i){e.manager.emit(i,t)}i<8&&n(e.options.event+_h(i)),n(e.options.event),t.additionalEvent&&n(t.additionalEvent),i>=8&&n(e.options.event+_h(i))},e.tryEmit=function(t){if(this.canEmit())return this.emit(t);this.state=32},e.canEmit=function(){for(var t=0;t<this.requireFail.length;){if(!(33&this.requireFail[t].state))return!1;t++}return!0},e.recognize=function(t){var e=kl({},t);if(!Xl(this.options.enable,[this,e]))return this.reset(),void(this.state=32);56&this.state&&(this.state=1),this.state=this.process(e),30&this.state&&this.tryEmit(e)},e.process=function(t){},e.getTouchAction=function(){},e.reset=function(){},t}(),Dh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},e))||this).pTime=!1,i.pCenter=!1,i._timer=null,i._input=null,i.count=0,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["manipulation"]},i.process=function(t){var e=this,i=this.options,n=t.pointers.length===i.pointers,o=t.distance<i.threshold,s=t.deltaTime<i.time;if(this.reset(),1&t.eventType&&0===this.count)return this.failTimeout();if(o&&s&&n){if(4!==t.eventType)return this.failTimeout();var r=!this.pTime||t.timeStamp-this.pTime<i.interval,a=!this.pCenter||zl(this.pCenter,t.center)<i.posThreshold;if(this.pTime=t.timeStamp,this.pCenter=t.center,a&&r?this.count+=1:this.count=1,this._input=t,0===this.count%i.taps)return this.hasRequireFailures()?(this._timer=setTimeout((function(){e.state=8,e.tryEmit()}),i.interval),2):8}return 32},i.failTimeout=function(){var t=this;return this._timer=setTimeout((function(){t.state=32}),this.options.interval),32},i.reset=function(){clearTimeout(this._timer)},i.emit=function(){8===this.state&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))},e}(kh),Ih=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({pointers:1},e))||this}xl(e,t);var i=e.prototype;return i.attrTest=function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},i.process=function(t){var e=this.state,i=t.eventType,n=6&e,o=this.attrTest(t);return n&&(8&i||!o)?16|e:n||o?4&i?8|e:2&e?4|e:2:32},e}(kh);function Ch(t){return 16===t?"down":8===t?"up":2===t?"left":4===t?"right":""}var Lh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"pan",threshold:10,pointers:1,direction:30},e))||this).pX=null,i.pY=null,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){var t=this.options.direction,e=[];return 6&t&&e.push("pan-y"),24&t&&e.push("pan-x"),e},i.directionTest=function(t){var e=this.options,i=!0,n=t.distance,o=t.direction,s=t.deltaX,r=t.deltaY;return o&e.direction||(6&e.direction?(o=0===s?1:s<0?2:4,i=s!==this.pX,n=Math.abs(t.deltaX)):(o=0===r?1:r<0?8:16,i=r!==this.pY,n=Math.abs(t.deltaY))),t.direction=o,i&&n>e.threshold&&o&e.direction},i.attrTest=function(t){return Ih.prototype.attrTest.call(this,t)&&(2&this.state||!(2&this.state)&&this.directionTest(t))},i.emit=function(e){this.pX=e.deltaX,this.pY=e.deltaY;var i=Ch(e.direction);i&&(e.additionalEvent=this.options.event+i),t.prototype.emit.call(this,e)},e}(Ih),Gh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"swipe",threshold:10,velocity:.3,direction:30,pointers:1},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return Lh.prototype.getTouchAction.call(this)},i.attrTest=function(e){var i,n=this.options.direction;return 30&n?i=e.overallVelocity:6&n?i=e.overallVelocityX:24&n&&(i=e.overallVelocityY),t.prototype.attrTest.call(this,e)&&n&e.offsetDirection&&e.distance>this.options.threshold&&e.maxPointers===this.options.pointers&&Ll(i)>this.options.velocity&&4&e.eventType},i.emit=function(t){var e=Ch(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)},e}(Ih),Mh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"pinch",threshold:0,pointers:2},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["none"]},i.attrTest=function(e){return t.prototype.attrTest.call(this,e)&&(Math.abs(e.scale-1)>this.options.threshold||2&this.state)},i.emit=function(e){if(1!==e.scale){var i=e.scale<1?"in":"out";e.additionalEvent=this.options.event+i}t.prototype.emit.call(this,e)},e}(Ih),Sh=function(t){function e(e){return void 0===e&&(e={}),t.call(this,yl({event:"rotate",threshold:0,pointers:2},e))||this}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["none"]},i.attrTest=function(e){return t.prototype.attrTest.call(this,e)&&(Math.abs(e.rotation)>this.options.threshold||2&this.state)},e}(Ih),Qh=function(t){function e(e){var i;return void 0===e&&(e={}),(i=t.call(this,yl({event:"press",pointers:1,time:251,threshold:9},e))||this)._timer=null,i._input=null,i}xl(e,t);var i=e.prototype;return i.getTouchAction=function(){return["auto"]},i.process=function(t){var e=this,i=this.options,n=t.pointers.length===i.pointers,o=t.distance<i.threshold,s=t.deltaTime>i.time;if(this._input=t,!o||!n||12&t.eventType&&!s)this.reset();else if(1&t.eventType)this.reset(),this._timer=setTimeout((function(){e.state=8,e.tryEmit()}),i.time);else if(4&t.eventType)return 8;return 32},i.reset=function(){clearTimeout(this._timer)},i.emit=function(t){8===this.state&&(t&&4&t.eventType?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=Gl(),this.manager.emit(this.options.event,this._input)))},e}(kh),Nh={domEvents:!1,touchAction:"compute",enable:!0,inputTarget:null,inputClass:null,cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}},Fh=[[Sh,{enable:!1}],[Mh,{enable:!1},["rotate"]],[Gh,{direction:6}],[Lh,{direction:6},["swipe"]],[Dh],[Dh,{event:"doubletap",taps:2},["tap"]],[Qh]];function Zh(t,e){var i,n=t.element;n.style&&(Al(t.options.cssProps,(function(o,s){i=Ml(n.style,s),e?(t.oldCssProps[i]=n.style[i],n.style[i]=o):n.style[i]=t.oldCssProps[i]||""})),e||(t.oldCssProps={}))}var Bh=function(){function t(t,e){var i,n=this;this.options=kl({},Nh,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.oldCssProps={},this.element=t,this.input=new((i=this).options.inputClass||(Zl?ah:Bl?uh:Fl?bh:mh))(i,Jl),this.touchAction=new Yl(this,this.options.touchAction),Zh(this,!0),Al(this.options.recognizers,(function(t){var e=n.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])}),this)}var e=t.prototype;return e.set=function(t){return kl(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},e.stop=function(t){this.session.stopped=t?2:1},e.recognize=function(t){var e=this.session;if(!e.stopped){var i;this.touchAction.preventDefaults(t);var n=this.recognizers,o=e.curRecognizer;(!o||o&&8&o.state)&&(e.curRecognizer=null,o=null);for(var s=0;s<n.length;)i=n[s],2===e.stopped||o&&i!==o&&!i.canRecognizeWith(o)?i.reset():i.recognize(t),!o&&14&i.state&&(e.curRecognizer=i,o=i),s++}},e.get=function(t){if(t instanceof kh)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event===t)return e[i];return null},e.add=function(t){if(yh(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},e.remove=function(t){if(yh(t,"remove",this))return this;var e=this.get(t);if(t){var i=this.recognizers,n=ih(i,e);-1!==n&&(i.splice(n,1),this.touchAction.update())}return this},e.on=function(t,e){if(void 0===t||void 0===e)return this;var i=this.handlers;return Al(Kl(t),(function(t){i[t]=i[t]||[],i[t].push(e)})),this},e.off=function(t,e){if(void 0===t)return this;var i=this.handlers;return Al(Kl(t),(function(t){e?i[t]&&i[t].splice(ih(i[t],e),1):delete i[t]})),this},e.emit=function(t,e){this.options.domEvents&&function(t,e){var i=document.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var n=0;n<i.length;)i[n](e),n++}},e.destroy=function(){this.element&&Zh(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null},t}(),Oh={touchstart:1,touchmove:2,touchend:4,touchcancel:8},Wh=function(t){function e(){var i,n=e.prototype;return n.evTarget="touchstart",n.evWin="touchstart touchmove touchend touchcancel",(i=t.apply(this,arguments)||this).started=!1,i}return xl(e,t),e.prototype.handler=function(t){var e=Oh[t.type];if(1===e&&(this.started=!0),this.started){var i=Ah.call(this,t,e);12&e&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}},e}(eh);function Ah(t,e){var i=lh(t.touches),n=lh(t.changedTouches);return 12&e&&(i=hh(i.concat(n),"identifier",!0)),[i,n]}function Xh(t,e,i){var n="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",o=window.console&&(window.console.warn||window.console.log);return o&&o.call(window.console,n,i),t.apply(this,arguments)}}var Th=Xh((function(t,e,i){for(var n=Object.keys(e),o=0;o<n.length;)(!i||i&&void 0===t[n[o]])&&(t[n[o]]=e[n[o]]),o++;return t}),"extend","Use `assign`."),Yh=Xh((function(t,e){return Th(t,e,!0)}),"merge","Use `assign`.");function Eh(t,e,i){var n,o=e.prototype;(n=t.prototype=Object.create(o)).constructor=t,n._super=o,i&&kl(n,i)}function Rh(t,e){return function(){return t.apply(e,arguments)}}var Uh,zh=function(){var t=function(t,e){return void 0===e&&(e={}),new Bh(t,yl({recognizers:Fh.concat()},e))};return t.VERSION="2.0.17-rc",t.DIRECTION_ALL=30,t.DIRECTION_DOWN=16,t.DIRECTION_LEFT=2,t.DIRECTION_RIGHT=4,t.DIRECTION_UP=8,t.DIRECTION_HORIZONTAL=6,t.DIRECTION_VERTICAL=24,t.DIRECTION_NONE=1,t.DIRECTION_DOWN=16,t.INPUT_START=1,t.INPUT_MOVE=2,t.INPUT_END=4,t.INPUT_CANCEL=8,t.STATE_POSSIBLE=1,t.STATE_BEGAN=2,t.STATE_CHANGED=4,t.STATE_ENDED=8,t.STATE_RECOGNIZED=8,t.STATE_CANCELLED=16,t.STATE_FAILED=32,t.Manager=Bh,t.Input=eh,t.TouchAction=Yl,t.TouchInput=uh,t.MouseInput=mh,t.PointerEventInput=ah,t.TouchMouseInput=bh,t.SingleTouchInput=Wh,t.Recognizer=kh,t.AttrRecognizer=Ih,t.Tap=Dh,t.Pan=Lh,t.Swipe=Gh,t.Pinch=Mh,t.Rotate=Sh,t.Press=Qh,t.on=ql,t.off=$l,t.each=Al,t.merge=Yh,t.extend=Th,t.bindFn=Rh,t.assign=kl,t.inherit=Eh,t.bindFn=Rh,t.prefixed=Ml,t.toArray=lh,t.inArray=ih,t.uniqueArray=hh,t.splitStr=Kl,t.boolOrFn=Xl,t.hasParent=El,t.addEventListeners=ql,t.removeEventListeners=$l,t.defaults=kl({},Nh,{preset:Fh}),t}();"undefined"!=typeof window?Uh=bl(window.Hammer||zh,{preventDefault:"mouse"}):Uh=function(){return{on:t=function(){},off:t,destroy:t,emit:t,get:function(e){return{set:t}}};var t};var Vh=Uh;function jh(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)}var Hh=function(){function t(e,i,n,o,s){ba(this,t),this.moment=s&&s.moment||Ta,this.options=s||{},this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(e,i,n),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Zo(o)?this.hiddenDates=o:this.hiddenDates=null!=o?[o]:[],this.format=t.FORMAT}return xa(t,[{key:"setMoment",value:function(t){this.moment=t,this.current=this.moment(this.current.valueOf()),this._start=this.moment(this._start.valueOf()),this._end=this.moment(this._end.valueOf())}},{key:"setFormat",value:function(e){var i=za.deepExtend({},t.FORMAT);this.format=za.deepExtend(i,e)}},{key:"setRange",value:function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=null!=t?this.moment(t.valueOf()):Jo(),this._end=null!=e?this.moment(e.valueOf()):Jo(),this.autoScale&&this.setMinimumStep(i)}},{key:"start",value:function(){this.current=this._start.clone(),this.roundToMinor()}},{key:"roundToMinor",value:function(){switch("week"==this.scale&&this.current.weekday(0),this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"week":case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step){var t=this.current.clone();switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"week":this.current.subtract(this.current.week()%this.step,"week");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}t.isSame(this.current)||(this.current=this.moment(ul(this.hiddenDates,this.current.valueOf(),-1,!0)))}}},{key:"hasNext",value:function(){return this.current.valueOf()<=this._end.valueOf()}},{key:"next",value:function(){var t=this.current.valueOf();switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.month()<6?this.current.subtract(this.current.hours()%this.step,"hour"):this.current.hours()%this.step!=0&&this.current.add(this.step-this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"week":if(0!==this.current.weekday())this.current.weekday(0),this.current.add(this.step,"week");else if(!1===this.options.showMajorLabels)this.current.add(this.step,"week");else{var e=this.current.clone();e.add(1,"week"),e.isSame(this.current,"month")?this.current.add(this.step,"week"):(this.current.add(this.step,"week"),this.current.date(1))}break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()>0&&this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()>0&&this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()>0&&this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()>0&&this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"week":this.current.week()<this.step&&this.current.week(1);break;case"month":this.current.month()<this.step&&this.current.month(0)}this.current.valueOf()==t&&(this.current=this._end.clone()),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,nl(this.moment,this,t)}},{key:"getCurrent",value:function(){return this.current.clone()}},{key:"setScale",value:function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)}},{key:"setAutoScale",value:function(t){this.autoScale=t}},{key:"setMinimumStep",value:function(t){if(null!=t){31104e9>t&&(this.scale="year",this.step=1e3),15552e9>t&&(this.scale="year",this.step=500),31104e8>t&&(this.scale="year",this.step=100),15552e8>t&&(this.scale="year",this.step=50),31104e7>t&&(this.scale="year",this.step=10),15552e7>t&&(this.scale="year",this.step=5),31104e6>t&&(this.scale="year",this.step=1),7776e6>t&&(this.scale="month",this.step=3),2592e6>t&&(this.scale="month",this.step=1),6048e5>t&&this.options.showWeekScale&&(this.scale="week",this.step=1),1728e5>t&&(this.scale="day",this.step=2),864e5>t&&(this.scale="day",this.step=1),432e5>t&&(this.scale="weekday",this.step=1),144e5>t&&(this.scale="hour",this.step=4),36e5>t&&(this.scale="hour",this.step=1),9e5>t&&(this.scale="minute",this.step=15),6e5>t&&(this.scale="minute",this.step=10),3e5>t&&(this.scale="minute",this.step=5),6e4>t&&(this.scale="minute",this.step=1),15e3>t&&(this.scale="second",this.step=15),1e4>t&&(this.scale="second",this.step=10),5e3>t&&(this.scale="second",this.step=5),1e3>t&&(this.scale="second",this.step=1),200>t&&(this.scale="millisecond",this.step=200),100>t&&(this.scale="millisecond",this.step=100),50>t&&(this.scale="millisecond",this.step=50),10>t&&(this.scale="millisecond",this.step=10),5>t&&(this.scale="millisecond",this.step=5),1>t&&(this.scale="millisecond",this.step=1)}}},{key:"isMajor",value:function(){if(1==this.switchedYear)switch(this.scale){case"year":case"month":case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.scale){case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":case"week":return 1==t.date();case"month":return 0==t.month();case"year":default:return!1}}},{key:"getLabelMinor",value:function(t){if(null==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.minorLabels)return this.format.minorLabels(t,this.scale,this.step);var e=this.format.minorLabels[this.scale];switch(this.scale){case"week":if(1===t.date()&&0!==t.weekday())return"";default:return e&&e.length>0?this.moment(t).format(e):""}}},{key:"getLabelMajor",value:function(t){if(null==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.majorLabels)return this.format.majorLabels(t,this.scale,this.step);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""}},{key:"getClassName",value:function(){var t,e=this.moment,i=this.moment(this.current),n=i.locale?i.locale("en"):i.lang("en"),o=this.step,s=[];function r(t){return t/o%2==0?" vis-even":" vis-odd"}function a(t){return t.isSame(Jo(),"day")?" vis-today":t.isSame(e().add(1,"day"),"day")?" vis-tomorrow":t.isSame(e().add(-1,"day"),"day")?" vis-yesterday":""}function l(t){return t.isSame(Jo(),"week")?" vis-current-week":""}function h(t){return t.isSame(Jo(),"month")?" vis-current-month":""}switch(this.scale){case"millisecond":s.push(a(n)),s.push(r(n.milliseconds()));break;case"second":s.push(a(n)),s.push(r(n.seconds()));break;case"minute":s.push(a(n)),s.push(r(n.minutes()));break;case"hour":s.push(Pn(t="vis-h".concat(n.hours())).call(t,4==this.step?"-h"+(n.hours()+4):"")),s.push(a(n)),s.push(r(n.hours()));break;case"weekday":s.push("vis-".concat(n.format("dddd").toLowerCase())),s.push(a(n)),s.push(l(n)),s.push(r(n.date()));break;case"day":s.push("vis-day".concat(n.date())),s.push("vis-".concat(n.format("MMMM").toLowerCase())),s.push(a(n)),s.push(h(n)),s.push(this.step<=2?a(n):""),s.push(this.step<=2?"vis-".concat(n.format("dddd").toLowerCase()):""),s.push(r(n.date()-1));break;case"week":s.push("vis-week".concat(n.format("w"))),s.push(l(n)),s.push(r(n.week()));break;case"month":s.push("vis-".concat(n.format("MMMM").toLowerCase())),s.push(h(n)),s.push(r(n.month()));break;case"year":s.push("vis-year".concat(n.year())),s.push(function(t){return t.isSame(Jo(),"year")?" vis-current-year":""}(n)),s.push(r(n.year()))}return En(s).call(s,String).join(" ")}}],[{key:"snap",value:function(t,e,i){var n=Ta(t);if("year"==e){var o=n.year()+Math.round(n.month()/12);n.year(Math.round(o/i)*i),n.month(0),n.date(0),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("month"==e)n.date()>15?(n.date(1),n.add(1,"month")):n.date(1),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0);else if("week"==e)n.weekday()>2?(n.weekday(0),n.add(1,"week")):n.weekday(0),n.hours(0),n.minutes(0),n.seconds(0),n.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:n.hours(24*Math.round(n.hours()/24));break;default:n.hours(12*Math.round(n.hours()/12))}n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:n.hours(12*Math.round(n.hours()/12));break;default:n.hours(6*Math.round(n.hours()/6))}n.minutes(0),n.seconds(0),n.milliseconds(0)}else if("hour"==e){switch(i){case 4:n.minutes(60*Math.round(n.minutes()/60));break;default:n.minutes(30*Math.round(n.minutes()/30))}n.seconds(0),n.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:n.minutes(5*Math.round(n.minutes()/5)),n.seconds(0);break;case 5:n.seconds(60*Math.round(n.seconds()/60));break;default:n.seconds(30*Math.round(n.seconds()/30))}n.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:n.seconds(5*Math.round(n.seconds()/5)),n.milliseconds(0);break;case 5:n.milliseconds(1e3*Math.round(n.milliseconds()/1e3));break;default:n.milliseconds(500*Math.round(n.milliseconds()/500))}else if("millisecond"==e){var s=i>5?i/2:1;n.milliseconds(Math.round(n.milliseconds()/s)*s)}return n}}]),t}();function Ph(t,e){void 0===e&&(e={});var i=e.insertAt;if(t&&"undefined"!=typeof document){var n=document.head||document.getElementsByTagName("head")[0],o=document.createElement("style");o.type="text/css","top"===i&&n.firstChild?n.insertBefore(o,n.firstChild):n.appendChild(o),o.styleSheet?o.styleSheet.cssText=t:o.appendChild(document.createTextNode(t))}}Hh.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}};Ph(".vis-time-axis{position:relative;overflow:hidden}.vis-time-axis.vis-foreground{top:0;left:0;width:100%}.vis-time-axis.vis-background{position:absolute;top:0;left:0;width:100%;height:100%}.vis-time-axis .vis-text{position:absolute;color:#4d4d4d;padding:3px;overflow:hidden;box-sizing:border-box;white-space:nowrap}.vis-time-axis .vis-text.vis-measure{position:absolute;padding-left:0;padding-right:0;margin-left:0;margin-right:0;visibility:hidden}.vis-time-axis .vis-grid.vis-vertical{position:absolute;border-left:1px solid}.vis-time-axis .vis-grid.vis-vertical-rtl{position:absolute;border-right:1px solid}.vis-time-axis .vis-grid.vis-minor{border-color:#e5e5e5}.vis-time-axis .vis-grid.vis-major{border-color:#bfbfbf}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRpbWVheGlzLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxlQUNFLGlCQUFrQixDQUNsQixlQUNGLENBRUEsOEJBQ0UsS0FBTSxDQUNOLE1BQU8sQ0FDUCxVQUNGLENBRUEsOEJBQ0UsaUJBQWtCLENBQ2xCLEtBQU0sQ0FDTixNQUFPLENBQ1AsVUFBVyxDQUNYLFdBQ0YsQ0FFQSx5QkFDRSxpQkFBa0IsQ0FDbEIsYUFBYyxDQUNkLFdBQVksQ0FDWixlQUFnQixDQUNoQixxQkFBc0IsQ0FFdEIsa0JBQ0YsQ0FFQSxxQ0FDRSxpQkFBa0IsQ0FDbEIsY0FBZSxDQUNmLGVBQWdCLENBQ2hCLGFBQWMsQ0FDZCxjQUFlLENBQ2YsaUJBQ0YsQ0FFQSxzQ0FDRSxpQkFBa0IsQ0FDbEIscUJBQ0YsQ0FFQSwwQ0FDRSxpQkFBa0IsQ0FDbEIsc0JBQ0YsQ0FFQSxtQ0FDRSxvQkFDRixDQUVBLG1DQUNFLG9CQUNGIiwiZmlsZSI6InRpbWVheGlzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi52aXMtdGltZS1heGlzIHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBvdmVyZmxvdzogaGlkZGVuO1xufVxuXG4udmlzLXRpbWUtYXhpcy52aXMtZm9yZWdyb3VuZCB7XG4gIHRvcDogMDtcbiAgbGVmdDogMDtcbiAgd2lkdGg6IDEwMCU7XG59XG5cbi52aXMtdGltZS1heGlzLnZpcy1iYWNrZ3JvdW5kIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtdGV4dCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICM0ZDRkNGQ7XG4gIHBhZGRpbmc6IDNweDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcblxuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLXRleHQudmlzLW1lYXN1cmUge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHBhZGRpbmctbGVmdDogMDtcbiAgcGFkZGluZy1yaWdodDogMDtcbiAgbWFyZ2luLWxlZnQ6IDA7XG4gIG1hcmdpbi1yaWdodDogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLWdyaWQudmlzLXZlcnRpY2FsIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICBib3JkZXItbGVmdDogMXB4IHNvbGlkO1xufVxuXG4udmlzLXRpbWUtYXhpcyAudmlzLWdyaWQudmlzLXZlcnRpY2FsLXJ0bCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgYm9yZGVyLXJpZ2h0OiAxcHggc29saWQ7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtZ3JpZC52aXMtbWlub3Ige1xuICBib3JkZXItY29sb3I6ICNlNWU1ZTU7XG59XG5cbi52aXMtdGltZS1heGlzIC52aXMtZ3JpZC52aXMtbWFqb3Ige1xuICBib3JkZXItY29sb3I6ICNiZmJmYmY7XG59XG4iXX0= */");var Jh=function(t){function e(t,i){var n;return ba(this,e),(n=ar(this,dr(e).call(this))).dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},n.props={range:{start:0,end:0,minimumStep:0},lineTop:0},n.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,maxMinorChars:7,format:Hh.FORMAT,moment:Ta,timeAxis:null},n.options=za.extend({},n.defaultOptions),n.body=t,n._create(),n.setOptions(i),n}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&(za.selectiveExtend(["showMinorLabels","showMajorLabels","showWeekScale","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),za.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===zo(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof Ta.locale?Ta.locale(t.locale):Ta.lang(t.locale)))}},{key:"_create",value:function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"}},{key:"destroy",value:function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null}},{key:"redraw",value:function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,n="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,o=e.parentNode!==n;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,l=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height="".concat(this.props.height,"px"),this._repaintLabels(),a?n.insertBefore(e,a):n.appendChild(e),l?this.body.dom.backgroundVertical.insertBefore(i,l):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||o}},{key:"_repaintLabels",value:function(){var t=this.options.orientation.axis,e=za.convert(this.body.range.start,"Number"),i=za.convert(this.body.range.end,"Number"),n=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),o=n-hl(this.options.moment,this.body.hiddenDates,this.body.range,n);o-=this.body.util.toTime(0).valueOf();var s=new Hh(new Date(e),new Date(i),o,this.body.hiddenDates,this.options);s.setMoment(this.options.moment),this.options.format&&s.setFormat(this.options.format),this.options.timeAxis&&s.setScale(this.options.timeAxis),this.step=s;var r,a,l,h,d,u,c=this.dom;c.redundant.lines=c.lines,c.redundant.majorTexts=c.majorTexts,c.redundant.minorTexts=c.minorTexts,c.lines=[],c.majorTexts=[],c.minorTexts=[];var p,m,f,g=0,v=void 0,b=0;for(s.start(),a=s.getCurrent(),h=this.body.util.toScreen(a);s.hasNext()&&b<1e3;){switch(b++,d=s.isMajor(),f=s.getClassName(),r=a,l=h,s.next(),a=s.getCurrent(),p=g,g=(h=this.body.util.toScreen(a))-l,s.scale){case"week":u=!0;break;default:u=g>=.4*p}if(this.options.showMinorLabels&&u){var y=this._repaintMinorText(l,s.getLabelMinor(r),t,f);y.style.width="".concat(g,"px")}d&&this.options.showMajorLabels?(l>0&&(null==v&&(v=l),y=this._repaintMajorText(l,s.getLabelMajor(r),t,f)),m=this._repaintMajorLine(l,g,t,f)):u?m=this._repaintMinorLine(l,g,t,f):m&&(m.style.width="".concat(Nn(m.style.width)+g,"px"))}if(1e3!==b||Kh||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to ".concat(1e3," lines.")),Kh=!0),this.options.showMajorLabels){var x=this.body.util.toTime(0),w=s.getLabelMajor(x),_=w.length*(this.props.majorCharWidth||10)+10;(null==v||_<v)&&this._repaintMajorText(0,w,t,f)}mn(za).call(za,this.dom.redundant,(function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}}))}},{key:"_repaintMinorText",value:function(t,e,i,n){var o=this.dom.redundant.minorTexts.shift();if(!o){var s=document.createTextNode("");(o=document.createElement("div")).appendChild(s),this.dom.foreground.appendChild(o)}this.dom.minorTexts.push(o),o.innerHTML=e;var r="top"==i?this.props.majorLabelHeight:0;return this._setXY(o,t,r),o.className="vis-text vis-minor ".concat(n),o}},{key:"_repaintMajorText",value:function(t,e,i,n){var o=this.dom.redundant.majorTexts.shift();if(!o){var s=document.createElement("div");(o=document.createElement("div")).appendChild(s),this.dom.foreground.appendChild(o)}o.childNodes[0].innerHTML=e,o.className="vis-text vis-major ".concat(n);var r="top"==i?0:this.props.minorLabelHeight;return this._setXY(o,t,r),this.dom.majorTexts.push(o),o}},{key:"_setXY",value:function(t,e,i){var n,o=this.options.rtl?-1*e:e;t.style.transform=Pn(n="translate(".concat(o,"px, ")).call(n,i,"px)")}},{key:"_repaintMinorLine",value:function(t,e,i,n){var o,s=this.dom.redundant.lines.shift();s||(s=document.createElement("div"),this.dom.background.appendChild(s)),this.dom.lines.push(s);var r=this.props;s.style.width="".concat(e,"px"),s.style.height="".concat(r.minorLineHeight,"px");var a="top"==i?r.majorLabelHeight:this.body.domProps.top.height,l=t-r.minorLineWidth/2;return this._setXY(s,l,a),s.className=Pn(o="vis-grid ".concat(this.options.rtl?"vis-vertical-rtl":"vis-vertical"," vis-minor ")).call(o,n),s}},{key:"_repaintMajorLine",value:function(t,e,i,n){var o,s=this.dom.redundant.lines.shift();s||(s=document.createElement("div"),this.dom.background.appendChild(s)),this.dom.lines.push(s);var r=this.props;s.style.width="".concat(e,"px"),s.style.height="".concat(r.majorLineHeight,"px");var a="top"==i?0:this.body.domProps.top.height,l=t-r.majorLineWidth/2;return this._setXY(s,l,a),s.className=Pn(o="vis-grid ".concat(this.options.rtl?"vis-vertical-rtl":"vis-vertical"," vis-major ")).call(o,n),s}},{key:"_calculateCharSize",value:function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth}}]),e}(Ja),Kh=!1,qh=n((function(t,e){t.exports=function(t){var e,i=t&&t.preventDefault||!1,n=t&&t.container||window,o={},s={keydown:{},keyup:{}},r={};for(e=97;e<=122;e++)r[String.fromCharCode(e)]={code:e-97+65,shift:!1};for(e=65;e<=90;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;e<=9;e++)r[""+e]={code:48+e,shift:!1};for(e=1;e<=12;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;e<=9;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r.delete={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){h(t,"keydown")},l=function(t){h(t,"keyup")},h=function(t,e){if(void 0!==s[e][t.keyCode]){for(var n=s[e][t.keyCode],o=0;o<n.length;o++)(void 0===n[o].shift||1==n[o].shift&&1==t.shiftKey||0==n[o].shift&&0==t.shiftKey)&&n[o].fn(t);1==i&&t.preventDefault()}};return o.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},o.bindAll=function(t,e){for(var i in void 0===e&&(e="keydown"),r)r.hasOwnProperty(i)&&o.bind(i,t,e)},o.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},o.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var n=[],o=s[i][r[t].code];if(void 0!==o)for(var a=0;a<o.length;a++)o[a].fn==e&&o[a].shift==r[t].shift||n.push(s[i][r[t].code][a]);s[i][r[t].code]=n}else s[i][r[t].code]=[]},o.reset=function(){s={keydown:{},keyup:{}}},o.destroy=function(){s={keydown:{},keyup:{}},n.removeEventListener("keydown",a,!0),n.removeEventListener("keyup",l,!0)},n.addEventListener("keydown",a,!0),n.addEventListener("keyup",l,!0),o}}));function $h(t){var e,i;this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=Vh(this.dom.overlay),this.hammer.on("tap",va(e=this._onTapOverlay).call(e,this));var n=this,o=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];mn(o).call(o,(function(t){n.hammer.on(t,(function(t){t.stopPropagation()}))})),document&&document.body&&(this.onClick=function(e){(function(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1})(e.target,t)||n.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=qh(),this.escListener=va(i=this.deactivate).call(i,this)}Ph(".vis .overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}.vis-active{box-shadow:0 0 10px #86d5f8}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImFjdGl2YXRvci5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FDRSxpQkFBa0IsQ0FDbEIsS0FBTSxDQUNOLE1BQU8sQ0FDUCxVQUFXLENBQ1gsV0FBWSxDQUdaLFVBQ0YsQ0FFQSxZQUNFLDJCQUNGIiwiZmlsZSI6ImFjdGl2YXRvci5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyIudmlzIC5vdmVybGF5IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG5cbiAgLyogTXVzdCBiZSBkaXNwbGF5ZWQgYWJvdmUgZm9yIGV4YW1wbGUgc2VsZWN0ZWQgVGltZWxpbmUgaXRlbXMgKi9cbiAgei1pbmRleDogMTA7XG59XG5cbi52aXMtYWN0aXZlIHtcbiAgYm94LXNoYWRvdzogMCAwIDEwcHggIzg2ZDVmODtcbn1cbiJdfQ== */"),vl($h.prototype),$h.current=null,$h.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=null,this.hammer.destroy(),this.hammer=null},$h.prototype.activate=function(){var t;$h.current&&$h.current.deactivate(),$h.current=this,this.active=!0,this.dom.overlay.style.display="none",za.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),va(t=this.keycharm).call(t,"esc",this.escListener)},$h.prototype.deactivate=function(){$h.current===this&&($h.current=null),this.active=!1,this.dom.overlay.style.display="",za.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},$h.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()};var td={current:"current",time:"time",deleteSelected:"Delete selected"},ed={current:"attuale",time:"tempo",deleteSelected:"Cancella la selezione"},id={current:"huidige",time:"tijd",deleteSelected:"Selectie verwijderen"},nd={current:"Aktuelle",time:"Zeit",deleteSelected:"Lösche Auswahl"},od={current:"actuel",time:"heure",deleteSelected:"Effacer la selection"},sd={current:"corriente",time:"hora",deleteSelected:"Eliminar selección"},rd={current:"поточний",time:"час",deleteSelected:"Видалити обране"},ad={current:"текущее",time:"время",deleteSelected:"Удалить выбранное"},ld={current:"aktualny",time:"czas",deleteSelected:"Usuń wybrane"},hd={current:"現在",time:"時刻",deleteSelected:"選択されたものを削除"},dd={en:td,en_EN:td,en_US:td,it:ed,it_IT:ed,it_CH:ed,nl:id,nl_NL:id,nl_BE:id,de:nd,de_DE:nd,fr:od,fr_FR:od,fr_CA:od,fr_BE:od,es:sd,es_ES:sd,uk:rd,uk_UA:rd,ru:ad,ru_RU:ad,pl:ld,pl_PL:ld,ja:hd,ja_JP:hd};Ph(".vis-custom-time{background-color:#6e94ff;width:2px;cursor:move;z-index:1}.vis-custom-time>.vis-custom-time-marker{background-color:inherit;color:#fff;font-size:12px;white-space:nowrap;padding:3px 5px;top:0;cursor:auto;z-index:inherit}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImN1c3RvbXRpbWUuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGlCQUNFLHdCQUF5QixDQUN6QixTQUFVLENBQ1YsV0FBWSxDQUNaLFNBQ0YsQ0FFQSx5Q0FDRSx3QkFBeUIsQ0FDekIsVUFBWSxDQUNaLGNBQWUsQ0FDZixrQkFBbUIsQ0FDbkIsZUFBZ0IsQ0FDaEIsS0FBUSxDQUNSLFdBQWUsQ0FDZixlQUNGIiwiZmlsZSI6ImN1c3RvbXRpbWUuY3NzIiwic291cmNlc0NvbnRlbnQiOlsiLnZpcy1jdXN0b20tdGltZSB7XG4gIGJhY2tncm91bmQtY29sb3I6ICM2RTk0RkY7XG4gIHdpZHRoOiAycHg7XG4gIGN1cnNvcjogbW92ZTtcbiAgei1pbmRleDogMTtcbn1cblxuLnZpcy1jdXN0b20tdGltZSA+IC52aXMtY3VzdG9tLXRpbWUtbWFya2VyIHtcbiAgYmFja2dyb3VuZC1jb2xvcjogaW5oZXJpdDtcbiAgY29sb3I6IHdoaXRlO1xuICBmb250LXNpemU6IDEycHg7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIHBhZGRpbmc6IDNweCA1cHg7XG4gIHRvcDogMHB4O1xuICBjdXJzb3I6IGluaXRpYWw7XG4gIHotaW5kZXg6IGluaGVyaXQ7XG59Il19 */");var ud=function(t){function e(t,i){var n,o;ba(this,e),(o=ar(this,dr(e).call(this))).body=t,o.defaultOptions={moment:Ta,locales:dd,locale:"en",id:void 0,title:void 0},o.options=za.extend({},o.defaultOptions),o.setOptions(i),o.options.locales=za.extend({},dd,o.options.locales);var s=o.defaultOptions.locales[o.defaultOptions.locale];return mn(n=Yi(o.options.locales)).call(n,(function(t){o.options.locales[t]=za.extend({},s,o.options.locales[t])})),i&&null!=i.time?o.customTime=i.time:o.customTime=new Date,o.eventParams={},o._create(),o}return pr(e,t),xa(e,[{key:"setOptions",value:function(t){t&&za.selectiveExtend(["moment","locale","locales","id","title","rtl","snap"],this.options,t)}},{key:"_create",value:function(){var t,e,i,n=document.createElement("div");n["custom-time"]=this,n.className="vis-custom-time ".concat(this.options.id||""),n.style.position="absolute",n.style.top="0px",n.style.height="100%",this.bar=n;var o=document.createElement("div");function s(t){this.body.range._onMouseWheel(t)}o.style.position="relative",o.style.top="0px",this.options.rtl?o.style.right="-10px":o.style.left="-10px",o.style.height="100%",o.style.width="20px",o.addEventListener?(o.addEventListener("mousewheel",va(s).call(s,this),!1),o.addEventListener("DOMMouseScroll",va(s).call(s,this),!1)):o.attachEvent("onmousewheel",va(s).call(s,this)),n.appendChild(o),this.hammer=new Vh(o),this.hammer.on("panstart",va(t=this._onDragStart).call(t,this)),this.hammer.on("panmove",va(e=this._onDrag).call(e,this)),this.hammer.on("panend",va(i=this._onDragEnd).call(i,this)),this.hammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_ALL})}},{key:"destroy",value:function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null}},{key:"redraw",value:function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),i=this.options.locales.en);var n,o=this.options.title;void 0===o?o=(o=Pn(n="".concat(i.time,": ")).call(n,this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"))).charAt(0).toUpperCase()+o.substring(1):"function"==typeof o&&(o=o.call(this,this.customTime));return this.options.rtl?this.bar.style.right="".concat(e,"px"):this.bar.style.left="".concat(e,"px"),this.bar.title=o,!1}},{key:"hide",value:function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)}},{key:"setCustomTime",value:function(t){this.customTime=za.convert(t,"Date"),this.redraw()}},{key:"getCustomTime",value:function(){return new Date(this.customTime.valueOf())}},{key:"setCustomMarker",value:function(t,e){var i,n,o=document.createElement("div");(o.className="vis-custom-time-marker",o.innerHTML=t,o.style.position="absolute",e)&&(o.setAttribute("contenteditable","true"),o.addEventListener("pointerdown",(function(){o.focus()})),o.addEventListener("input",va(i=this._onMarkerChange).call(i,this)),o.title=t,o.addEventListener("blur",va(n=function(t){this.title!=t.target.innerHTML&&(this._onMarkerChanged(t),this.title=t.target.innerHTML)}).call(n,this)));this.bar.appendChild(o)}},{key:"setCustomTitle",value:function(t){this.options.title=t}},{key:"_onDragStart",value:function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()}},{key:"_onDrag",value:function(t){if(this.eventParams.dragging){var e=this.options.rtl?-1*t.deltaX:t.deltaX,i=this.body.util.toScreen(this.eventParams.customTime)+e,n=this.body.util.toTime(i),o=this.body.util.getScale(),s=this.body.util.getStep(),r=this.options.snap,a=r?r(n,o,s):n;this.setCustomTime(a),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation()}}},{key:"_onDragEnd",value:function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation())}},{key:"_onMarkerChange",value:function(t){this.body.emitter.emit("markerchange",{id:this.options.id,title:t.target.innerHTML,event:t}),t.stopPropagation()}},{key:"_onMarkerChanged",value:function(t){this.body.emitter.emit("markerchanged",{id:this.options.id,title:t.target.innerHTML,event:t}),t.stopPropagation()}}],[{key:"customTimeFromTarget",value:function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null}}]),e}(Ja);Ph("\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IiIsImZpbGUiOiJhbmltYXRpb24uY3NzIn0= */");Ph('.vis-current-time{background-color:#ff7f6e;width:2px;z-index:1;pointer-events:none}.vis-rolling-mode-btn{height:40px;width:40px;position:absolute;top:7px;right:20px;border-radius:50%;font-size:28px;cursor:pointer;opacity:.8;color:#fff;font-weight:700;text-align:center;background:#3876c2}.vis-rolling-mode-btn:before{content:"\\26F6"}.vis-rolling-mode-btn:hover{opacity:1}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImN1cnJlbnR0aW1lLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxrQkFDRSx3QkFBeUIsQ0FDekIsU0FBVSxDQUNWLFNBQVUsQ0FDVixtQkFDRixDQUVBLHNCQUNFLFdBQVksQ0FDWixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGNBQWUsQ0FDZixjQUFlLENBQ2YsVUFBWSxDQUNaLFVBQVksQ0FDWixlQUFpQixDQUNqQixpQkFBa0IsQ0FDbEIsa0JBQ0YsQ0FDQSw2QkFDRSxlQUNGLENBRUEsNEJBQ0UsU0FDRiIsImZpbGUiOiJjdXJyZW50dGltZS5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyIudmlzLWN1cnJlbnQtdGltZSB7XG4gIGJhY2tncm91bmQtY29sb3I6ICNGRjdGNkU7XG4gIHdpZHRoOiAycHg7XG4gIHotaW5kZXg6IDE7XG4gIHBvaW50ZXItZXZlbnRzOiBub25lO1xufVxuXG4udmlzLXJvbGxpbmctbW9kZS1idG4ge1xuICBoZWlnaHQ6IDQwcHg7XG4gIHdpZHRoOiA0MHB4O1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHRvcDogN3B4O1xuICByaWdodDogMjBweDtcbiAgYm9yZGVyLXJhZGl1czogNTAlO1xuICBmb250LXNpemU6IDI4cHg7XG4gIGN1cnNvcjogcG9pbnRlcjtcbiAgb3BhY2l0eTogMC44O1xuICBjb2xvcjogd2hpdGU7XG4gIGZvbnQtd2VpZ2h0OiBib2xkO1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG4gIGJhY2tncm91bmQ6ICMzODc2YzI7XG59XG4udmlzLXJvbGxpbmctbW9kZS1idG46YmVmb3JlIHtcbiAgY29udGVudDogXCJcXDI2RjZcIjtcbn1cblxuLnZpcy1yb2xsaW5nLW1vZGUtYnRuOmhvdmVyIHtcbiAgb3BhY2l0eTogMTtcbn0iXX0= */');Ph(".vis-panel{position:absolute;padding:0;margin:0;box-sizing:border-box}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right,.vis-panel.vis-top{border:1px #bfbfbf}.vis-panel.vis-center,.vis-panel.vis-left,.vis-panel.vis-right{border-top-style:solid;border-bottom-style:solid;overflow:hidden}.vis-left.vis-panel.vis-vertical-scroll,.vis-right.vis-panel.vis-vertical-scroll{height:100%;overflow-x:hidden;overflow-y:scroll}.vis-left.vis-panel.vis-vertical-scroll{direction:rtl}.vis-left.vis-panel.vis-vertical-scroll .vis-content,.vis-right.vis-panel.vis-vertical-scroll{direction:ltr}.vis-right.vis-panel.vis-vertical-scroll .vis-content{direction:rtl}.vis-panel.vis-bottom,.vis-panel.vis-center,.vis-panel.vis-top{border-left-style:solid;border-right-style:solid}.vis-background{overflow:hidden}.vis-panel>.vis-content{position:relative}.vis-panel .vis-shadow{position:absolute;width:100%;height:1px;box-shadow:0 0 10px rgba(0,0,0,.8)}.vis-panel .vis-shadow.vis-top{top:-1px;left:0}.vis-panel .vis-shadow.vis-bottom{bottom:-1px;left:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhbmVsLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxXQUNFLGlCQUFrQixDQUVsQixTQUFVLENBQ1YsUUFBUyxDQUVULHFCQUNGLENBRUEsd0dBS0Usa0JBQ0YsQ0FFQSwrREFHRSxzQkFBdUIsQ0FDdkIseUJBQTBCLENBQzFCLGVBQ0YsQ0FFQSxpRkFDRSxXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGlCQUNGLENBRUEsd0NBQ0UsYUFDRixDQU1BLDhGQUNFLGFBQ0YsQ0FFQSxzREFDRSxhQUNGLENBRUEsK0RBR0UsdUJBQXdCLENBQ3hCLHdCQUNGLENBRUEsZ0JBQ0UsZUFDRixDQUVBLHdCQUNFLGlCQUNGLENBRUEsdUJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxVQUFXLENBQ1gsa0NBSUYsQ0FFQSwrQkFDRSxRQUFTLENBQ1QsTUFDRixDQUVBLGtDQUNFLFdBQVksQ0FDWixNQUNGIiwiZmlsZSI6InBhbmVsLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi52aXMtcGFuZWwge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG5cbiAgcGFkZGluZzogMDtcbiAgbWFyZ2luOiAwO1xuXG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtcGFuZWwudmlzLWNlbnRlcixcbi52aXMtcGFuZWwudmlzLWxlZnQsXG4udmlzLXBhbmVsLnZpcy1yaWdodCxcbi52aXMtcGFuZWwudmlzLXRvcCxcbi52aXMtcGFuZWwudmlzLWJvdHRvbSB7XG4gIGJvcmRlcjogMXB4ICNiZmJmYmY7XG59XG5cbi52aXMtcGFuZWwudmlzLWNlbnRlcixcbi52aXMtcGFuZWwudmlzLWxlZnQsXG4udmlzLXBhbmVsLnZpcy1yaWdodCB7XG4gIGJvcmRlci10b3Atc3R5bGU6IHNvbGlkO1xuICBib3JkZXItYm90dG9tLXN0eWxlOiBzb2xpZDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbn1cblxuLnZpcy1sZWZ0LnZpcy1wYW5lbC52aXMtdmVydGljYWwtc2Nyb2xsLCAudmlzLXJpZ2h0LnZpcy1wYW5lbC52aXMtdmVydGljYWwtc2Nyb2xsIHtcbiAgaGVpZ2h0OiAxMDAlO1xuICBvdmVyZmxvdy14OiBoaWRkZW47XG4gIG92ZXJmbG93LXk6IHNjcm9sbDtcbn0gXG5cbi52aXMtbGVmdC52aXMtcGFuZWwudmlzLXZlcnRpY2FsLXNjcm9sbCB7XG4gIGRpcmVjdGlvbjogcnRsO1xufVxuXG4udmlzLWxlZnQudmlzLXBhbmVsLnZpcy12ZXJ0aWNhbC1zY3JvbGwgLnZpcy1jb250ZW50IHtcbiAgZGlyZWN0aW9uOiBsdHI7XG59XG5cbi52aXMtcmlnaHQudmlzLXBhbmVsLnZpcy12ZXJ0aWNhbC1zY3JvbGwge1xuICBkaXJlY3Rpb246IGx0cjtcbn1cblxuLnZpcy1yaWdodC52aXMtcGFuZWwudmlzLXZlcnRpY2FsLXNjcm9sbCAudmlzLWNvbnRlbnQge1xuICBkaXJlY3Rpb246IHJ0bDtcbn1cblxuLnZpcy1wYW5lbC52aXMtY2VudGVyLFxuLnZpcy1wYW5lbC52aXMtdG9wLFxuLnZpcy1wYW5lbC52aXMtYm90dG9tIHtcbiAgYm9yZGVyLWxlZnQtc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmlnaHQtc3R5bGU6IHNvbGlkO1xufVxuXG4udmlzLWJhY2tncm91bmQge1xuICBvdmVyZmxvdzogaGlkZGVuO1xufVxuXG4udmlzLXBhbmVsID4gLnZpcy1jb250ZW50IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xufVxuXG4udmlzLXBhbmVsIC52aXMtc2hhZG93IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB3aWR0aDogMTAwJTtcbiAgaGVpZ2h0OiAxcHg7XG4gIGJveC1zaGFkb3c6IDAgMCAxMHB4IHJnYmEoMCwwLDAsMC44KTtcbiAgLyogVE9ETzogZmluZCBhIG5pY2Ugd2F5IHRvIGVuc3VyZSB2aXMtc2hhZG93cyBhcmUgZHJhd24gb24gdG9wIG9mIGl0ZW1zXG4gIHotaW5kZXg6IDE7XG4gICovXG59XG5cbi52aXMtcGFuZWwgLnZpcy1zaGFkb3cudmlzLXRvcCB7XG4gIHRvcDogLTFweDtcbiAgbGVmdDogMDtcbn1cblxuLnZpcy1wYW5lbCAudmlzLXNoYWRvdy52aXMtYm90dG9tIHtcbiAgYm90dG9tOiAtMXB4O1xuICBsZWZ0OiAwO1xufSJdfQ== */");Ph(".vis-graph-group0{fill:#4f81bd;fill-opacity:0;stroke-width:2px;stroke:#4f81bd}.vis-graph-group1{fill:#f79646;fill-opacity:0;stroke-width:2px;stroke:#f79646}.vis-graph-group2{fill:#8c51cf;fill-opacity:0;stroke-width:2px;stroke:#8c51cf}.vis-graph-group3{fill:#75c841;fill-opacity:0;stroke-width:2px;stroke:#75c841}.vis-graph-group4{fill:#ff0100;fill-opacity:0;stroke-width:2px;stroke:#ff0100}.vis-graph-group5{fill:#37d8e6;fill-opacity:0;stroke-width:2px;stroke:#37d8e6}.vis-graph-group6{fill:#042662;fill-opacity:0;stroke-width:2px;stroke:#042662}.vis-graph-group7{fill:#00ff26;fill-opacity:0;stroke-width:2px;stroke:#00ff26}.vis-graph-group8{fill:#f0f;fill-opacity:0;stroke-width:2px;stroke:#f0f}.vis-graph-group9{fill:#8f3938;fill-opacity:0;stroke-width:2px;stroke:#8f3938}.vis-timeline .vis-fill{fill-opacity:.1;stroke:none}.vis-timeline .vis-bar{fill-opacity:.5;stroke-width:1px}.vis-timeline .vis-point{stroke-width:2px;fill-opacity:1}.vis-timeline .vis-legend-background{stroke-width:1px;fill-opacity:.9;fill:#fff;stroke:#c2c2c2}.vis-timeline .vis-outline{stroke-width:1px;fill-opacity:1;fill:#fff;stroke:#e5e5e5}.vis-timeline .vis-icon-fill{fill-opacity:.3;stroke:none}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInBhdGhTdHlsZXMuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGtCQUNJLFlBQVksQ0FDWixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFZLENBQ1osY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksWUFBYSxDQUNiLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsY0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFhLENBQ2IsY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksWUFBYSxDQUNiLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsY0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSxrQkFDSSxZQUFZLENBQ1osY0FBYyxDQUNkLGdCQUFnQixDQUNoQixjQUNKLENBRUEsa0JBQ0ksU0FBWSxDQUNaLGNBQWMsQ0FDZCxnQkFBZ0IsQ0FDaEIsV0FDSixDQUVBLGtCQUNJLFlBQWEsQ0FDYixjQUFjLENBQ2QsZ0JBQWdCLENBQ2hCLGNBQ0osQ0FFQSx3QkFDSSxlQUFnQixDQUNoQixXQUNKLENBR0EsdUJBQ0ksZUFBZ0IsQ0FDaEIsZ0JBQ0osQ0FFQSx5QkFDSSxnQkFBZ0IsQ0FDaEIsY0FDSixDQUdBLHFDQUNJLGdCQUFnQixDQUNoQixlQUFnQixDQUNoQixTQUFhLENBQ2IsY0FDSixDQUdBLDJCQUNJLGdCQUFnQixDQUNoQixjQUFjLENBQ2QsU0FBYSxDQUNiLGNBQ0osQ0FFQSw2QkFDSSxlQUFnQixDQUNoQixXQUNKIiwiZmlsZSI6InBhdGhTdHlsZXMuY3NzIiwic291cmNlc0NvbnRlbnQiOlsiLnZpcy1ncmFwaC1ncm91cDAge1xuICAgIGZpbGw6IzRmODFiZDtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogIzRmODFiZDtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDEge1xuICAgIGZpbGw6I2Y3OTY0NjtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogI2Y3OTY0Njtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDIge1xuICAgIGZpbGw6ICM4YzUxY2Y7XG4gICAgZmlsbC1vcGFjaXR5OjA7XG4gICAgc3Ryb2tlLXdpZHRoOjJweDtcbiAgICBzdHJva2U6ICM4YzUxY2Y7XG59XG5cbi52aXMtZ3JhcGgtZ3JvdXAzIHtcbiAgICBmaWxsOiAjNzVjODQxO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjNzVjODQxO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwNCB7XG4gICAgZmlsbDogI2ZmMDEwMDtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogI2ZmMDEwMDtcbn1cblxuLnZpcy1ncmFwaC1ncm91cDUge1xuICAgIGZpbGw6ICMzN2Q4ZTY7XG4gICAgZmlsbC1vcGFjaXR5OjA7XG4gICAgc3Ryb2tlLXdpZHRoOjJweDtcbiAgICBzdHJva2U6ICMzN2Q4ZTY7XG59XG5cbi52aXMtZ3JhcGgtZ3JvdXA2IHtcbiAgICBmaWxsOiAjMDQyNjYyO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjMDQyNjYyO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwNyB7XG4gICAgZmlsbDojMDBmZjI2O1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjMDBmZjI2O1xufVxuXG4udmlzLWdyYXBoLWdyb3VwOCB7XG4gICAgZmlsbDojZmYwMGZmO1xuICAgIGZpbGwtb3BhY2l0eTowO1xuICAgIHN0cm9rZS13aWR0aDoycHg7XG4gICAgc3Ryb2tlOiAjZmYwMGZmO1xufVxuXG4udmlzLWdyYXBoLWdyb3VwOSB7XG4gICAgZmlsbDogIzhmMzkzODtcbiAgICBmaWxsLW9wYWNpdHk6MDtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIHN0cm9rZTogIzhmMzkzODtcbn1cblxuLnZpcy10aW1lbGluZSAudmlzLWZpbGwge1xuICAgIGZpbGwtb3BhY2l0eTowLjE7XG4gICAgc3Ryb2tlOiBub25lO1xufVxuXG5cbi52aXMtdGltZWxpbmUgLnZpcy1iYXIge1xuICAgIGZpbGwtb3BhY2l0eTowLjU7XG4gICAgc3Ryb2tlLXdpZHRoOjFweDtcbn1cblxuLnZpcy10aW1lbGluZSAudmlzLXBvaW50IHtcbiAgICBzdHJva2Utd2lkdGg6MnB4O1xuICAgIGZpbGwtb3BhY2l0eToxLjA7XG59XG5cblxuLnZpcy10aW1lbGluZSAudmlzLWxlZ2VuZC1iYWNrZ3JvdW5kIHtcbiAgICBzdHJva2Utd2lkdGg6MXB4O1xuICAgIGZpbGwtb3BhY2l0eTowLjk7XG4gICAgZmlsbDogI2ZmZmZmZjtcbiAgICBzdHJva2U6ICNjMmMyYzI7XG59XG5cblxuLnZpcy10aW1lbGluZSAudmlzLW91dGxpbmUge1xuICAgIHN0cm9rZS13aWR0aDoxcHg7XG4gICAgZmlsbC1vcGFjaXR5OjE7XG4gICAgZmlsbDogI2ZmZmZmZjtcbiAgICBzdHJva2U6ICNlNWU1ZTU7XG59XG5cbi52aXMtdGltZWxpbmUgLnZpcy1pY29uLWZpbGwge1xuICAgIGZpbGwtb3BhY2l0eTowLjM7XG4gICAgc3Ryb2tlOiBub25lO1xufVxuIl19 */");Ph(".vis-timeline{position:relative;border:1px solid #bfbfbf;overflow:hidden;padding:0;margin:0;box-sizing:border-box}.vis-loading-screen{width:100%;height:100%;position:absolute;top:0;left:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRpbWVsaW5lLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxjQUNFLGlCQUFrQixDQUNsQix3QkFBeUIsQ0FDekIsZUFBZ0IsQ0FDaEIsU0FBVSxDQUNWLFFBQVMsQ0FDVCxxQkFDRixDQUVBLG9CQUNFLFVBQVcsQ0FDWCxXQUFZLENBQ1osaUJBQWtCLENBQ2xCLEtBQU0sQ0FDTixNQUNGIiwiZmlsZSI6InRpbWVsaW5lLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy10aW1lbGluZSB7XG4gIHBvc2l0aW9uOiByZWxhdGl2ZTtcbiAgYm9yZGVyOiAxcHggc29saWQgI2JmYmZiZjtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbiAgcGFkZGluZzogMDtcbiAgbWFyZ2luOiAwO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xufVxuXG4udmlzLWxvYWRpbmctc2NyZWVuIHtcbiAgd2lkdGg6IDEwMCU7XG4gIGhlaWdodDogMTAwJTtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG59Il19 */");Ph(".vis [class*=span]{min-height:0;width:auto}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImJvb3RzdHJhcC5jc3MiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBRUEsbUJBQ0UsWUFBYSxDQUNiLFVBQ0YiLCJmaWxlIjoiYm9vdHN0cmFwLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIi8qIG92ZXJyaWRlIHNvbWUgYm9vdHN0cmFwIHN0eWxlcyBzY3Jld2luZyB1cCB0aGUgdGltZWxpbmVzIGNzcyAqL1xuXG4udmlzIFtjbGFzcyo9XCJzcGFuXCJdIHtcbiAgbWluLWhlaWdodDogMDtcbiAgd2lkdGg6IGF1dG87XG59XG4iXX0= */");var cd=function(){function t(){ba(this,t)}return xa(t,[{key:"_create",value:function(t){var e,i,n,o=this;this.dom={},this.dom.container=t,this.dom.container.style.position="relative",this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.rollingModeBtn=document.createElement("div"),this.dom.loadingScreen=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.rollingModeBtn.className="vis-rolling-mode-btn",this.dom.loadingScreen.className="vis-loading-screen",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.rollingModeBtn),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.on("rangechange",(function(){!0===o.initialDrawDone&&o._redraw()})),this.on("rangechanged",(function(){o.initialRangeChangeDone||(o.initialRangeChangeDone=!0)})),this.on("touch",va(e=this._onTouch).call(e,this)),this.on("panmove",va(i=this._onDrag).call(i,this));var s=this;this._origRedraw=va(n=this._redraw).call(n,this),this._redraw=za.throttle(this._origRedraw),this.on("_change",(function(t){s.itemSet&&s.itemSet.initialItemSetDrawn&&t&&1==t.queue?s._redraw():s._origRedraw()})),this.hammer=new Vh(this.dom.root);var r=this.hammer.get("pinch").set({enable:!0});r&&function(t){t.getTouchAction=function(){return["pan-y"]}}(r),this.hammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_ALL}),this.listeners={};var a,l,h=["tap","doubletap","press","pinch","pan","panstart","panmove","panend"];function d(t){this.isActive()&&this.emit("mousewheel",t);var e=0,i=0;if("detail"in t&&(i=-1*t.detail),"wheelDelta"in t&&(i=t.wheelDelta),"wheelDeltaY"in t&&(i=t.wheelDeltaY),"wheelDeltaX"in t&&(e=-1*t.wheelDeltaX),"axis"in t&&t.axis===t.HORIZONTAL_AXIS&&(e=-1*i,i=0),"deltaY"in t&&(i=-1*t.deltaY),"deltaX"in t&&(e=t.deltaX),t.deltaMode&&(1===t.deltaMode?(e*=40,i*=40):(e*=40,i*=800)),this.options.preferZoom){if(!this.options.zoomKey||t[this.options.zoomKey])return}else if(this.options.zoomKey&&t[this.options.zoomKey])return;if(this.options.verticalScroll||this.options.horizontalScroll)if(this.options.verticalScroll&&Math.abs(i)>=Math.abs(e)){var n=this.props.scrollTop,o=n+i;if(this.isActive())this._setScrollTop(o)!==n&&(this._redraw(),this.emit("scroll",t),t.preventDefault())}else if(this.options.horizontalScroll){var s=(Math.abs(e)>=Math.abs(i)?e:i)/120*(this.range.end-this.range.start)/20,r=this.range.start+s,a=this.range.end+s,l={animation:!1,byUser:!0,event:t};this.range.setRange(r,a,l),t.preventDefault()}}mn(h).call(h,(function(t){var e=function(e){s.isActive()&&s.emit(t,e)};s.hammer.on(t,e),s.listeners[t]=e})),jh(this.hammer,(function(t){s.emit("touch",t)})),a=this.hammer,(l=function(t){s.emit("release",t)}).inputHandler=function(t){t.isFinal&&l(t)},a.on("hammer.input",l.inputHandler);var u="onwheel"in document.createElement("div")?"wheel":void 0!==document.onmousewheel?"mousewheel":this.dom.centerContainer.addEventListener?"DOMMouseScroll":"onmousewheel";function c(t){if(s.options.verticalScroll)if(s._isProgramaticallyScrolled)s._isProgramaticallyScrolled=!1;else if(t.preventDefault(),s.isActive()){var e=-t.target.scrollTop;s._setScrollTop(e),s._redraw(),s.emit("scrollSide",t)}}this.dom.top.addEventListener,this.dom.bottom.addEventListener,this.dom.centerContainer.addEventListener(u,va(d).call(d,this),!1),this.dom.top.addEventListener(u,va(d).call(d,this),!1),this.dom.bottom.addEventListener(u,va(d).call(d,this),!1),this.dom.left.parentNode.addEventListener("scroll",va(c).call(c,this)),this.dom.right.parentNode.addEventListener("scroll",va(c).call(c,this));var p=!1;function m(t){var e;if(t.preventDefault&&(s.emit("dragover",s.getEventProperties(t)),t.preventDefault()),Fo(e=t.target.className).call(e,"timeline")>-1&&!p)return t.dataTransfer.dropEffect="move",p=!0,!1}function f(t){t.preventDefault&&t.preventDefault(),t.stopPropagation&&t.stopPropagation();try{var e=JSON.parse(t.dataTransfer.getData("text"));if(!e||!e.content)return}catch(t){return!1}return p=!1,t.center={x:t.clientX,y:t.clientY},"item"!==e.target?s.itemSet._onAddItem(t):s.itemSet._onDropObjectOnItem(t),s.emit("drop",s.getEventProperties(t)),!1}if(this.dom.center.addEventListener("dragover",va(m).call(m,this),!1),this.dom.center.addEventListener("drop",va(f).call(f,this),!1),this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,this.initialRangeChangeDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root),t.appendChild(this.dom.loadingScreen)}},{key:"setOptions",value:function(t){var e;if(t){if(za.selectiveExtend(["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","preferZoom","rtl","zoomKey","horizontalScroll","verticalScroll","longSelectPressTime"],this.options,t),this.dom.rollingModeBtn.style.visibility="hidden",this.options.rtl&&(this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"),this.options.verticalScroll&&(this.options.rtl?this.dom.rightContainer.className="vis-panel vis-right vis-vertical-scroll":this.dom.leftContainer.className="vis-panel vis-left vis-vertical-scroll"),"object"!==zo(this.options.orientation)&&(this.options.orientation={item:void 0,axis:void 0}),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===zo(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var i=this.timeAxis2=new Jh(this.body);i.setOptions=function(t){var e=t?za.extend({},t):{};e.orientation="top",Jh.prototype.setOptions.call(i,e)},this.components.push(i)}}else if(this.timeAxis2){var n,o,s=Fo(n=this.components).call(n,this.timeAxis2);if(-1!==s)Hr(o=this.components).call(o,s,1);this.timeAxis2.destroy(),this.timeAxis2=null}"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&tl(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new $h(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),this._initAutoResize()}if(mn(e=this.components).call(e,(function(e){return e.setOptions(t)})),"configure"in t){var r;this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var a=za.deepExtend({},this.options);mn(r=this.components).call(r,(function(t){za.deepExtend(a,t.options)})),this.configurator.setModuleOptions({global:a})}this._redraw()}},{key:"isActive",value:function(){return!this.activator||this.activator.active}},{key:"destroy",value:function(){var t;for(var e in this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator),this.listeners)this.listeners.hasOwnProperty(e)&&delete this.listeners[e];this.listeners=null,this.hammer&&this.hammer.destroy(),this.hammer=null,mn(t=this.components).call(t,(function(t){return t.destroy()})),this.body=null}},{key:"setCustomTime",value:function(t,e){var i,n=En(i=this.customTimes).call(i,(function(t){return e===t.options.id}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));n.length>0&&n[0].setCustomTime(t)}},{key:"getCustomTime",value:function(t){var e,i=En(e=this.customTimes).call(e,(function(e){return e.options.id===t}));if(0===i.length)throw new Error("No custom time bar found with id ".concat(Vs(t)));return i[0].getCustomTime()}},{key:"setCustomTimeMarker",value:function(t,e,i){var n,o=En(n=this.customTimes).call(n,(function(t){return t.options.id===e}));if(0===o.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));o.length>0&&o[0].setCustomMarker(t,i)}},{key:"setCustomTimeTitle",value:function(t,e){var i,n=En(i=this.customTimes).call(i,(function(t){return t.options.id===e}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(e)));if(n.length>0)return n[0].setCustomTitle(t)}},{key:"getEventProperties",value:function(t){return{event:t}}},{key:"addCustomTime",value:function(t,e){var i,n=void 0!==t?za.convert(t,"Date"):new Date;if(rr(i=this.customTimes).call(i,(function(t){return t.options.id===e})))throw new Error("A custom time with id ".concat(Vs(e)," already exists"));var o=new ud(this.body,za.extend({},this.options,{time:n,id:e,snap:this.itemSet.options.snap}));return this.customTimes.push(o),this.components.push(o),this._redraw(),e}},{key:"removeCustomTime",value:function(t){var e,i=this,n=En(e=this.customTimes).call(e,(function(e){return e.options.id===t}));if(0===n.length)throw new Error("No custom time bar found with id ".concat(Vs(t)));mn(n).call(n,(function(t){var e,n,o,s;Hr(e=i.customTimes).call(e,Fo(n=i.customTimes).call(n,t),1),Hr(o=i.components).call(o,Fo(s=i.components).call(s,t),1),t.destroy()}))}},{key:"getVisibleItems",value:function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]}},{key:"getVisibleGroups",value:function(){return this.itemSet&&this.itemSet.getVisibleGroups()||[]}},{key:"fit",value:function(t,e){var i=this.getDataRange();if(null!==i.min||null!==i.max){var n=i.max-i.min,o=new Date(i.min.valueOf()-.01*n),s=new Date(i.max.valueOf()+.01*n),r=!t||void 0===t.animation||t.animation;this.range.setRange(o,s,{animation:r},e)}}},{key:"getDataRange",value:function(){throw new Error("Cannot invoke abstract method getDataRange")}},{key:"setWindow",value:function(t,e,i,n){var o,s;"function"==typeof arguments[2]&&(n=arguments[2],i={}),1==arguments.length?(o=void 0===(s=arguments[0]).animation||s.animation,this.range.setRange(s.start,s.end,{animation:o})):2==arguments.length&&"function"==typeof arguments[1]?(s=arguments[0],n=arguments[1],o=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:o},n)):(o=!i||void 0===i.animation||i.animation,this.range.setRange(t,e,{animation:o},n))}},{key:"moveTo",value:function(t,e,i){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.range.end-this.range.start,o=za.convert(t,"Date").valueOf(),s=o-n/2,r=o+n/2,a=!e||void 0===e.animation||e.animation;this.range.setRange(s,r,{animation:a},i)}},{key:"getWindow",value:function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}}},{key:"zoomIn",value:function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.getWindow(),o=n.start.valueOf(),s=n.end.valueOf(),r=s-o,a=r/(1+t),l=(r-a)/2,h=o+l,d=s-l;this.setWindow(h,d,e,i)}}},{key:"zoomOut",value:function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var n=this.getWindow(),o=n.start.valueOf(),s=n.end.valueOf(),r=s-o,a=o-r*t/2,l=s+r*t/2;this.setWindow(a,l,e,i)}}},{key:"redraw",value:function(){this._redraw()}},{key:"_redraw",value:function(){var t;this.redrawCount++;var e=this.dom;if(e&&e.container&&0!=e.root.offsetWidth){var i=!1,n=this.options,o=this.props;el(this.options.moment,this.body,this.options.hiddenDates),"top"==n.orientation?(za.addClassName(e.root,"vis-top"),za.removeClassName(e.root,"vis-bottom")):(za.removeClassName(e.root,"vis-top"),za.addClassName(e.root,"vis-bottom")),n.rtl?(za.addClassName(e.root,"vis-rtl"),za.removeClassName(e.root,"vis-ltr")):(za.addClassName(e.root,"vis-ltr"),za.removeClassName(e.root,"vis-rtl")),e.root.style.maxHeight=za.option.asSize(n.maxHeight,""),e.root.style.minHeight=za.option.asSize(n.minHeight,""),e.root.style.width=za.option.asSize(n.width,"");var s=e.root.offsetWidth;o.border.left=1,o.border.right=1,o.border.top=1,o.border.bottom=1,o.center.height=e.center.offsetHeight,o.left.height=e.left.offsetHeight,o.right.height=e.right.offsetHeight,o.top.height=e.top.clientHeight||-o.border.top,o.bottom.height=e.bottom.clientHeight||-o.border.bottom;var r=Math.max(o.left.height,o.center.height,o.right.height),a=o.top.height+r+o.bottom.height+o.border.top+o.border.bottom;e.root.style.height=za.option.asSize(n.height,"".concat(a,"px")),o.root.height=e.root.offsetHeight,o.background.height=o.root.height;var l=o.root.height-o.top.height-o.bottom.height;o.centerContainer.height=l,o.leftContainer.height=l,o.rightContainer.height=o.leftContainer.height,o.root.width=s,o.background.width=o.root.width,this.initialDrawDone||(o.scrollbarWidth=za.getScrollBarWidth());var h=e.leftContainer.clientWidth,d=e.rightContainer.clientWidth;n.verticalScroll?n.rtl?(o.left.width=h||-o.border.left,o.right.width=d+o.scrollbarWidth||-o.border.right):(o.left.width=h+o.scrollbarWidth||-o.border.left,o.right.width=d||-o.border.right):(o.left.width=h||-o.border.left,o.right.width=d||-o.border.right),this._setDOM();var u=this._updateScrollTop();"top"!=n.orientation.item&&(u+=Math.max(o.centerContainer.height-o.center.height-o.border.top-o.border.bottom,0)),e.center.style.transform="translateY(".concat(u,"px)");var c=0==o.scrollTop?"hidden":"",p=o.scrollTop==o.scrollTopMin?"hidden":"";e.shadowTop.style.visibility=c,e.shadowBottom.style.visibility=p,e.shadowTopLeft.style.visibility=c,e.shadowBottomLeft.style.visibility=p,e.shadowTopRight.style.visibility=c,e.shadowBottomRight.style.visibility=p,n.verticalScroll&&(e.rightContainer.className="vis-panel vis-right vis-vertical-scroll",e.leftContainer.className="vis-panel vis-left vis-vertical-scroll",e.shadowTopRight.style.visibility="hidden",e.shadowBottomRight.style.visibility="hidden",e.shadowTopLeft.style.visibility="hidden",e.shadowBottomLeft.style.visibility="hidden",e.left.style.top="0px",e.right.style.top="0px"),(!n.verticalScroll||o.center.height<o.centerContainer.height)&&(e.left.style.top="".concat(u,"px"),e.right.style.top="".concat(u,"px"),e.rightContainer.className=e.rightContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),e.leftContainer.className=e.leftContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),o.left.width=h||-o.border.left,o.right.width=d||-o.border.right,this._setDOM());var m=o.center.height>o.centerContainer.height;this.hammer.get("pan").set({direction:m?Vh.DIRECTION_ALL:Vh.DIRECTION_HORIZONTAL}),this.hammer.get("press").set({time:this.options.longSelectPressTime}),mn(t=this.components).call(t,(function(t){i=t.redraw()||i}));if(i){if(this.redrawCount<5)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.body.emitter.emit("changed")}}},{key:"_setDOM",value:function(){var t=this.props,e=this.dom;t.leftContainer.width=t.left.width,t.rightContainer.width=t.right.width;var i=t.root.width-t.left.width-t.right.width;t.center.width=i,t.centerContainer.width=i,t.top.width=i,t.bottom.width=i,e.background.style.height="".concat(t.background.height,"px"),e.backgroundVertical.style.height="".concat(t.background.height,"px"),e.backgroundHorizontal.style.height="".concat(t.centerContainer.height,"px"),e.centerContainer.style.height="".concat(t.centerContainer.height,"px"),e.leftContainer.style.height="".concat(t.leftContainer.height,"px"),e.rightContainer.style.height="".concat(t.rightContainer.height,"px"),e.background.style.width="".concat(t.background.width,"px"),e.backgroundVertical.style.width="".concat(t.centerContainer.width,"px"),e.backgroundHorizontal.style.width="".concat(t.background.width,"px"),e.centerContainer.style.width="".concat(t.center.width,"px"),e.top.style.width="".concat(t.top.width,"px"),e.bottom.style.width="".concat(t.bottom.width,"px"),e.background.style.left="0",e.background.style.top="0",e.backgroundVertical.style.left="".concat(t.left.width+t.border.left,"px"),e.backgroundVertical.style.top="0",e.backgroundHorizontal.style.left="0",e.backgroundHorizontal.style.top="".concat(t.top.height,"px"),e.centerContainer.style.left="".concat(t.left.width,"px"),e.centerContainer.style.top="".concat(t.top.height,"px"),e.leftContainer.style.left="0",e.leftContainer.style.top="".concat(t.top.height,"px"),e.rightContainer.style.left="".concat(t.left.width+t.center.width,"px"),e.rightContainer.style.top="".concat(t.top.height,"px"),e.top.style.left="".concat(t.left.width,"px"),e.top.style.top="0",e.bottom.style.left="".concat(t.left.width,"px"),e.bottom.style.top="".concat(t.top.height+t.centerContainer.height,"px"),e.center.style.left="0",e.left.style.left="0",e.right.style.left="0"}},{key:"setCurrentTime",value:function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)}},{key:"getCurrentTime",value:function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()}},{key:"_toTime",value:function(t){return sl(this,t,this.props.center.width)}},{key:"_toGlobalTime",value:function(t){return sl(this,t,this.props.root.width)}},{key:"_toScreen",value:function(t){return ol(this,t,this.props.center.width)}},{key:"_toGlobalScreen",value:function(t){return ol(this,t,this.props.root.width)}},{key:"_initAutoResize",value:function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()}},{key:"_startAutoResize",value:function(){var t=this;this._stopAutoResize(),this._onResize=function(){if(1==t.options.autoResize){if(t.dom.root){var e=t.dom.root.offsetHeight,i=t.dom.root.offsetWidth;i==t.props.lastWidth&&e==t.props.lastHeight||(t.props.lastWidth=i,t.props.lastHeight=e,t.props.scrollbarWidth=za.getScrollBarWidth(),t.body.emitter.emit("_change"))}}else t._stopAutoResize()},za.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=gl(this._onResize,1e3)}},{key:"_stopAutoResize",value:function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(za.removeEventListener(window,"resize",this._onResize),this._onResize=null)}},{key:"_onTouch",value:function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop}},{key:"_onPinch",value:function(t){this.touch.allowDragging=!1}},{key:"_onDrag",value:function(t){if(t&&this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),n=this._setScrollTop(this.touch.initialScrollTop+e);this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),n!=i&&this.emit("verticalDrag")}}},{key:"_setScrollTop",value:function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop}},{key:"_updateScrollTop",value:function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),this._isProgramaticallyScrolled=!0,this.props.scrollTop}},{key:"_getScrollTop",value:function(){return this.props.scrollTop}},{key:"_createConfigurator",value:function(){throw new Error("Cannot invoke abstract method _createConfigurator")}}]),t}();vl(cd.prototype);var pd=function(t){function e(t,i){var n,o;ba(this,e),(o=ar(this,dr(e).call(this))).body=t,o.defaultOptions={rtl:!1,showCurrentTime:!0,alignCurrentTime:void 0,moment:Ta,locales:dd,locale:"en"},o.options=za.extend({},o.defaultOptions),o.setOptions(i),o.options.locales=za.extend({},dd,o.options.locales);var s=o.defaultOptions.locales[o.defaultOptions.locale];return mn(n=Yi(o.options.locales)).call(n,(function(t){o.options.locales[t]=za.extend({},s,o.options.locales[t])})),o.offset=0,o._create(),o}return pr(e,t),xa(e,[{key:"_create",value:function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t}},{key:"destroy",value:function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null}},{key:"setOptions",value:function(t){t&&za.selectiveExtend(["rtl","showCurrentTime","alignCurrentTime","moment","locale","locales"],this.options,t)}},{key:"redraw",value:function(){if(this.options.showCurrentTime){var t,e,i=this.body.dom.backgroundVertical;this.bar.parentNode!=i&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),i.appendChild(this.bar),this.start());var n=this.options.moment(Jo()+this.offset);this.options.alignCurrentTime&&(n=n.startOf(this.options.alignCurrentTime));var o=this.body.util.toScreen(n),s=this.options.locales[this.options.locale];s||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),s=this.options.locales.en);var r=Pn(t=Pn(e="".concat(s.current," ")).call(e,s.time,": ")).call(t,n.format("dddd, MMMM Do YYYY, H:mm:ss"));r=r.charAt(0).toUpperCase()+r.substring(1),this.options.rtl?this.bar.style.transform="translateX(".concat(-1*o,"px)"):this.bar.style.transform="translateX(".concat(o,"px)"),this.bar.title=r}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1}},{key:"start",value:function(){var t=this;!function e(){t.stop();var i=1/t.body.range.conversion(t.body.domProps.center.width).scale/10;i<30&&(i=30),i>1e3&&(i=1e3),t.redraw(),t.body.emitter.emit("currentTimeTick"),t.currentTimeTimer=qr(e,i)}()}},{key:"stop",value:function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)}},{key:"setCurrentTime",value:function(t){var e=za.convert(t,"Date").valueOf(),i=Jo();this.offset=e-i,this.redraw()}},{key:"getCurrentTime",value:function(){return new Date(Jo()+this.offset)}}]),e}(Ja),md=Ce.find,fd=!0,gd=on("find");"find"in[]&&Array(1).find((function(){fd=!1})),V({target:"Array",proto:!0,forced:fd||!gd},{find:function(t){return md(this,t,arguments.length>1?arguments[1]:void 0)}});var vd=Pi("Array").find,bd=Array.prototype,yd=function(t){var e=t.find;return t===bd||t instanceof Array&&e===bd.find?vd:e},xd="undefined"!=typeof crypto&&crypto.getRandomValues&&crypto.getRandomValues.bind(crypto)||"undefined"!=typeof msCrypto&&"function"==typeof msCrypto.getRandomValues&&msCrypto.getRandomValues.bind(msCrypto),wd=new Uint8Array(16);function _d(){if(!xd)throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");return xd(wd)}for(var kd=[],Dd=0;Dd<256;++Dd)kd[Dd]=(Dd+256).toString(16).substr(1);function Id(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"===t?new Array(16):null,t=null);var o=(t=t||{}).random||(t.rng||_d)();if(o[6]=15&o[6]|64,o[8]=63&o[8]|128,e)for(var s=0;s<16;++s)e[n+s]=o[s];return e||function(t,e){var i=e||0,n=kd;return[n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],"-",n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]],n[t[i++]]].join("")}(o)}var Cd=ot.includes,Ld=on("indexOf",{ACCESSORS:!0,1:0});V({target:"Array",proto:!0,forced:!Ld},{includes:function(t){return Cd(this,t,arguments.length>1?arguments[1]:void 0)}});var Gd=Pi("Array").includes,Md=Pt("match"),Sd=function(t){if(function(t){var e;return x(t)&&(void 0!==(e=t[Md])?!!e:"RegExp"==f(t))}(t))throw TypeError("The method doesn't accept regular expressions");return t},Qd=Pt("match");V({target:"String",proto:!0,forced:!function(t){var e=/./;try{"/./"[t](e)}catch(i){try{return e[Qd]=!1,"/./"[t](e)}catch(t){}}return!1}("includes")},{includes:function(t){return!!~String(b(this)).indexOf(Sd(t),arguments.length>1?arguments[1]:void 0)}});var Nd=Pi("String").includes,Fd=Array.prototype,Zd=String.prototype,Bd=function(t){var e=t.includes;return t===Fd||t instanceof Array&&e===Fd.includes?Gd:"string"==typeof t||t===Zd||t instanceof String&&e===Zd.includes?Nd:e};function Od(t){wr(t).call(t,(function(t,e){return t.data.start-e.data.start}))}function Wd(t){wr(t).call(t,(function(t,e){return("end"in t.data?t.data.end:t.data.start)-("end"in e.data?e.data.end:e.data.start)}))}function Ad(t,e,i,n){if(i)for(var o=0;o<t.length;o++)t[o].top=null;for(o=0;o<t.length;o++){var s=t[o];if(s.stack&&null===s.top){s.top=e.axis;var r=!1;do{for(var a=null,l=0,h=t.length;l<h;l++){var d=t[l];if(r=n()||!1)return!0;if(null!==d.top&&d!==s&&d.stack&&Rd(s,d,e.item,d.options.rtl)){a=d;break}}null!=a&&(s.top=a.top+a.height+e.item.vertical)}while(a)}}return r}function Xd(t,e,i){for(var n=0;n<t.length;n++)t[n].top=null;var o=i.height;for(n=0;n<t.length;n++){var s=t[n];if(s.stack&&null===s.top){s.top=s.baseTop;do{for(var r=null,a=0,l=t.length;a<l;a++){var h=t[a];if(null!==h.top&&h!==s&&Rd(s,h,e.item,h.options.rtl)){r=h;break}}null!=r&&(s.top=r.top+r.height+e.item.vertical),s.top+s.height>o&&(o=s.top+s.height)}while(r)}}i.height=o-i.top+.5*e.item.vertical}function Td(t,e,i,n){for(var o=0;o<t.length;o++)if(null==t[o].data.subgroup)t[o].top=e.item.vertical;else if(void 0!==t[o].data.subgroup&&n){var s=0;for(var r in i)i.hasOwnProperty(r)&&1==i[r].visible&&i[r].index<i[t[o].data.subgroup].index&&(s+=i[r].height,i[t[o].data.subgroup].top=s);t[o].top=s+.5*e.item.vertical}n||Yd(t,e,i)}function Yd(t,e,i){for(var n in i)if(i.hasOwnProperty(n)){i[n].top=0;do{var o=null;for(var s in i)if(null!==i[s].top&&s!==n&&i[n].index>i[s].index&&Ud(i[n],i[s])){o=i[s];break}null!=o&&(i[n].top=o.top+o.height)}while(o)}for(var r=0;r<t.length;r++)void 0!==t[r].data.subgroup&&(t[r].top=i[t[r].data.subgroup].top+.5*e.item.vertical)}function Ed(t,e,i){var n=!1,o=[];for(var s in i)i[s].hasOwnProperty("index")?o[i[s].index]=s:o.push(s);for(var r=0;r<o.length;r++)if(s=o[r],i.hasOwnProperty(s)){for(var a in n=n||i[s].stack,i[s].top=0,i)i[a].visible&&i[s].index>i[a].index&&(i[s].top+=i[a].height);for(var l=t[s],h=0;h<l.length;h++)void 0!==l[h].data.subgroup&&(l[h].top=i[l[h].data.subgroup].top+.5*e.item.vertical,i[s].stack&&(l[h].baseTop=l[h].top));n&&i[s].stack&&Xd(t[s],e,i[s])}}function Rd(t,e,i,n){return n?t.right-i.horizontal+.001<e.right+e.width&&t.right+t.width+i.horizontal-.001>e.right&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top:t.left-i.horizontal+.001<e.left+e.width&&t.left+t.width+i.horizontal-.001>e.left&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top}function Ud(t,e){var i=t.start<e.end&&t.end>e.start,n=t.top<e.top+e.height&&t.top+t.height>e.top;return i&&n}var zd=Object.freeze({__proto__:null,orderByStart:Od,orderByEnd:Wd,stack:Ad,substack:Xd,nostack:Td,stackSubgroups:Yd,stackSubgroupsWithInnerStack:Ed,collision:Rd,collisionByTimes:Ud}),Vd="__background__",jd=function(){function t(e,i,n){var o=this;if(ba(this,t),this.groupId=e,this.subgroups={},this.subgroupStack={},this.subgroupStackAll=!1,this.subgroupVisibility={},this.doInnerStack=!1,this.shouldBailStackItems=!1,this.subgroupIndex=0,this.subgroupOrderer=i&&i.subgroupOrder,this.itemSet=n,this.isVisible=null,this.stackDirty=!0,this._disposeCallbacks=[],i&&i.nestedGroups&&(this.nestedGroups=i.nestedGroups,0==i.showNested?this.showNested=!1:this.showNested=!0),i&&i.subgroupStack)if("boolean"==typeof i.subgroupStack)this.doInnerStack=i.subgroupStack,this.subgroupStackAll=i.subgroupStack;else for(var s in i.subgroupStack)this.subgroupStack[s]=i.subgroupStack[s],this.doInnerStack=this.doInnerStack||i.subgroupStack[s];i&&i.heightMode?this.heightMode=i.heightMode:this.heightMode=n.options.groupHeightMode,this.nestedInGroup=null,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.itemsInRange=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var r=function(){o.checkRangedItems=!0};this.itemSet.body.emitter.on("checkRangedItems",r),this._disposeCallbacks.push((function(){o.itemSet.body.emitter.off("checkRangedItems",r)})),this._create(),this.setData(i)}return xa(t,[{key:"_create",value:function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["vis-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.style.position="absolute",this.dom.marker.innerHTML="",this.dom.background.appendChild(this.dom.marker)}},{key:"setData",value:function(t){if(!this.itemSet.groupTouchParams.isDragging){var e,i,n;if(t&&t.subgroupVisibility)for(var o in t.subgroupVisibility)this.subgroupVisibility[o]=t.subgroupVisibility[o];if(this.itemSet.options&&this.itemSet.options.groupTemplate)e=(i=va(n=this.itemSet.options.groupTemplate).call(n,this))(t,this.dom.inner);else e=t&&t.content;if(e instanceof Element){for(;this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else e instanceof Object&&e.isReactComponent||(e instanceof Object?i(t,this.dom.inner):this.dom.inner.innerHTML=null!=e?e:this.groupId||"");this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?za.removeClassName(this.dom.inner,"vis-hidden"):za.addClassName(this.dom.inner,"vis-hidden"),t&&t.nestedGroups?(this.nestedGroups&&this.nestedGroups==t.nestedGroups||(this.nestedGroups=t.nestedGroups),void 0===t.showNested&&void 0!==this.showNested||(0==t.showNested?this.showNested=!1:this.showNested=!0),za.addClassName(this.dom.label,"vis-nesting-group"),this.showNested?(za.removeClassName(this.dom.label,"collapsed"),za.addClassName(this.dom.label,"expanded")):(za.removeClassName(this.dom.label,"expanded"),za.addClassName(this.dom.label,"collapsed"))):this.nestedGroups&&(this.nestedGroups=null,za.removeClassName(this.dom.label,"collapsed"),za.removeClassName(this.dom.label,"expanded"),za.removeClassName(this.dom.label,"vis-nesting-group")),t&&(t.treeLevel||t.nestedInGroup)?(za.addClassName(this.dom.label,"vis-nested-group"),t.treeLevel?za.addClassName(this.dom.label,"vis-group-level-"+t.treeLevel):za.addClassName(this.dom.label,"vis-group-level-unknown-but-gte1")):za.addClassName(this.dom.label,"vis-group-level-0");var s=t&&t.className||null;s!=this.className&&(this.className&&(za.removeClassName(this.dom.label,this.className),za.removeClassName(this.dom.foreground,this.className),za.removeClassName(this.dom.background,this.className),za.removeClassName(this.dom.axis,this.className)),za.addClassName(this.dom.label,s),za.addClassName(this.dom.foreground,s),za.addClassName(this.dom.background,s),za.addClassName(this.dom.axis,s),this.className=s),this.style&&(za.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(za.addCssText(this.dom.label,t.style),this.style=t.style)}}},{key:"getLabelWidth",value:function(){return this.props.label.width}},{key:"_didMarkerHeightChange",value:function(){var t=this.dom.marker.clientHeight;if(t!=this.lastMarkerHeight){this.lastMarkerHeight=t;var e={},i=0;if(mn(za).call(za,this.items,(function(t,n){if(t.dirty=!0,t.displayed){e[n]=t.redraw(!0),i=e[n].length}})),i>0)for(var n=function(t){mn(za).call(za,e,(function(e){e[t]()}))},o=0;o<i;o++)n(o);return!0}return!1}},{key:"_calculateGroupSizeAndPosition",value:function(){var t=this.dom.foreground,e=t.offsetTop,i=t.offsetLeft,n=t.offsetWidth;this.top=e,this.right=i,this.width=n}},{key:"_shouldBailItemsRedraw",value:function(){var t=this,e=this.itemSet.options.onTimeout,i={relativeBailingTime:this.itemSet.itemsSettingTime,bailTimeMs:e&&e.timeoutMs,userBailFunction:e&&e.callback,shouldBailStackItems:this.shouldBailStackItems},n=null;if(!this.itemSet.initialDrawDone){if(i.shouldBailStackItems)return!0;Math.abs(Jo()-new Date(i.relativeBailingTime))>i.bailTimeMs&&(i.userBailFunction&&null==this.itemSet.userContinueNotBail?i.userBailFunction((function(e){t.itemSet.userContinueNotBail=e,n=!e})):n=0==t.itemSet.userContinueNotBail)}return n}},{key:"_redrawItems",value:function(t,e,i,n){var o=this;if(t||this.stackDirty||this.isVisible&&!e){var s,r,a,l,h,d,u={byEnd:En(s=this.orderedItems.byEnd).call(s,(function(t){return!t.isCluster})),byStart:En(r=this.orderedItems.byStart).call(r,(function(t){return!t.isCluster}))},c={byEnd:mo(new Ar(En(a=Ln(l=this.orderedItems.byEnd).call(l,(function(t){return t.cluster}))).call(a,(function(t){return!!t})))),byStart:mo(new Ar(En(h=Ln(d=this.orderedItems.byStart).call(d,(function(t){return t.cluster}))).call(h,(function(t){return!!t}))))},p=function(){var t,e,i,s=o._updateItemsInRange(u,En(t=o.visibleItems).call(t,(function(t){return!t.isCluster})),n),r=o._updateClustersInRange(c,En(e=o.visibleItems).call(e,(function(t){return t.isCluster})),n);return Pn(i=[]).call(i,mo(s),mo(r))},m=function(t){var e={},i=function(i){var n,s=En(n=o.visibleItems).call(n,(function(t){return t.data.subgroup===i}));e[i]=t?wr(s).call(s,(function(e,i){return t(e.data,i.data)})):s};for(var n in o.subgroups)i(n);return e};if("function"==typeof this.itemSet.options.order){var f=this;if(this.doInnerStack&&this.itemSet.options.stackSubgroups){Ed(m(this.itemSet.options.order),i,this.subgroups),this.visibleItems=p(),this._updateSubGroupHeights(i)}else{var g,v,b,y;this.visibleItems=p(),this._updateSubGroupHeights(i);var x=wr(g=En(v=_o(b=this.visibleItems).call(b)).call(v,(function(t){return t.isCluster||!t.isCluster&&!t.cluster}))).call(g,(function(t,e){return f.itemSet.options.order(t.data,e.data)}));this.shouldBailStackItems=Ad(x,i,!0,va(y=this._shouldBailItemsRedraw).call(y,this))}}else{var w;if(this.visibleItems=p(),this._updateSubGroupHeights(i),this.itemSet.options.stack)if(this.doInnerStack&&this.itemSet.options.stackSubgroups)Ed(m(),i,this.subgroups);else this.shouldBailStackItems=Ad(this.visibleItems,i,!0,va(w=this._shouldBailItemsRedraw).call(w,this));else Td(this.visibleItems,i,this.subgroups,this.itemSet.options.stackSubgroups)}for(var _=0;_<this.visibleItems.length;_++)this.visibleItems[_].repositionX(),void 0!==this.subgroupVisibility[this.visibleItems[_].data.subgroup]&&(this.subgroupVisibility[this.visibleItems[_].data.subgroup]||this.visibleItems[_].hide());this.itemSet.options.cluster&&mn(za).call(za,this.items,(function(t){t.cluster&&t.displayed&&t.hide()})),this.shouldBailStackItems&&this.itemSet.body.emitter.emit("destroyTimeline"),this.stackDirty=!1}}},{key:"_didResize",value:function(t,e){t=za.updateProperty(this,"height",e)||t;var i=this.dom.inner.clientWidth,n=this.dom.inner.clientHeight;return t=za.updateProperty(this.props.label,"width",i)||t,t=za.updateProperty(this.props.label,"height",n)||t}},{key:"_applyGroupHeight",value:function(t){this.dom.background.style.height="".concat(t,"px"),this.dom.foreground.style.height="".concat(t,"px"),this.dom.label.style.height="".concat(t,"px")}},{key:"_updateItemsVerticalPosition",value:function(t){for(var e=0,i=this.visibleItems.length;e<i;e++){var n=this.visibleItems[e];n.repositionY(t),this.isVisible||this.groupId==Vd||n.displayed&&n.hide()}}},{key:"redraw",value:function(t,e,i,n){var o,s,r,a,l,h,d,u=this,c=!1,p=this.isVisible,m=[function(){i=u._didMarkerHeightChange.call(u)||i},va(o=this._updateSubGroupHeights).call(o,this,e),va(s=this._calculateGroupSizeAndPosition).call(s,this),function(){var i;u.isVisible=va(i=u._isGroupVisible).call(i,u)(t,e)},function(){var n;va(n=u._redrawItems).call(n,u)(i,p,e,t)},va(r=this._updateSubgroupsSizes).call(r,this),function(){var t;h=va(t=u._calculateHeight).call(t,u)(e)},va(a=this._calculateGroupSizeAndPosition).call(a,this),function(){var t;c=va(t=u._didResize).call(t,u)(c,h)},function(){var t;va(t=u._applyGroupHeight).call(t,u)(h)},function(){var t;va(t=u._updateItemsVerticalPosition).call(t,u)(e)},va(l=function(){return!u.isVisible&&u.height&&(c=!1),c}).call(l,this)];return n?m:(mn(m).call(m,(function(t){d=t()})),d)}},{key:"_updateSubGroupHeights",value:function(t){var e=this;if(Yi(this.subgroups).length>0){var i=this;this._resetSubgroups(),mn(za).call(za,this.visibleItems,(function(n){void 0!==n.data.subgroup&&(i.subgroups[n.data.subgroup].height=Math.max(i.subgroups[n.data.subgroup].height,n.height+t.item.vertical),i.subgroups[n.data.subgroup].visible=void 0===e.subgroupVisibility[n.data.subgroup]||Boolean(e.subgroupVisibility[n.data.subgroup]))}))}}},{key:"_isGroupVisible",value:function(t,e){return this.top<=t.body.domProps.centerContainer.height-t.body.domProps.scrollTop+e.axis&&this.top+this.height+e.axis>=-t.body.domProps.scrollTop}},{key:"_calculateHeight",value:function(t){var e,i;if((i="fixed"===this.heightMode?za.toArray(this.items):this.visibleItems).length>0){var n=i[0].top,o=i[0].top+i[0].height;if(mn(za).call(za,i,(function(t){n=Math.min(n,t.top),o=Math.max(o,t.top+t.height)})),n>t.axis){var s=n-t.axis;o-=s,mn(za).call(za,i,(function(t){t.top-=s}))}e=Math.ceil(o+t.item.vertical/2),"fitItems"!==this.heightMode&&(e=Math.max(e,this.props.label.height))}else e=this.props.label.height;return e}},{key:"show",value:function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)}},{key:"hide",value:function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var n=this.dom.axis;n.parentNode&&n.parentNode.removeChild(n)}},{key:"add",value:function(t){var e;if(this.items[t.id]=t,t.setParent(this),this.stackDirty=!0,void 0!==t.data.subgroup&&(this._addToSubgroup(t),this.orderSubgroups()),!Bd(e=this.visibleItems).call(e,t)){var i=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,i)}}},{key:"_addToSubgroup",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:t.data.subgroup;null!=e&&void 0===this.subgroups[e]&&(this.subgroups[e]={height:0,top:0,start:t.data.start,end:t.data.end||t.data.start,visible:!1,index:this.subgroupIndex,items:[],stack:this.subgroupStackAll||this.subgroupStack[e]||!1},this.subgroupIndex++),new Date(t.data.start)<new Date(this.subgroups[e].start)&&(this.subgroups[e].start=t.data.start);var i=t.data.end||t.data.start;new Date(i)>new Date(this.subgroups[e].end)&&(this.subgroups[e].end=i),this.subgroups[e].items.push(t)}},{key:"_updateSubgroupsSizes",value:function(){var t=this;if(t.subgroups){var e=function(e){var i,n=t.subgroups[e].items[0].data.end||t.subgroups[e].items[0].data.start,o=t.subgroups[e].items[0].data.start,s=n-1;mn(i=t.subgroups[e].items).call(i,(function(t){new Date(t.data.start)<new Date(o)&&(o=t.data.start);var e=t.data.end||t.data.start;new Date(e)>new Date(s)&&(s=e)})),t.subgroups[e].start=o,t.subgroups[e].end=new Date(s-1)};for(var i in t.subgroups)e(i)}}},{key:"orderSubgroups",value:function(){if(void 0!==this.subgroupOrderer){var t=[];if("string"==typeof this.subgroupOrderer){for(var e in this.subgroups)t.push({subgroup:e,sortField:this.subgroups[e].items[0].data[this.subgroupOrderer]});wr(t).call(t,(function(t,e){return t.sortField-e.sortField}))}else if("function"==typeof this.subgroupOrderer){for(var i in this.subgroups)t.push(this.subgroups[i].items[0].data);wr(t).call(t,this.subgroupOrderer)}if(t.length>0)for(var n=0;n<t.length;n++)this.subgroups[t[n].subgroup].index=n}}},{key:"_resetSubgroups",value:function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1,this.subgroups[t].height=0)}},{key:"remove",value:function(t){var e,i;delete this.items[t.id],t.setParent(null),this.stackDirty=!0;var n=Fo(e=this.visibleItems).call(e,t);-1!=n&&Hr(i=this.visibleItems).call(i,n,1),void 0!==t.data.subgroup&&(this._removeFromSubgroup(t),this.orderSubgroups())}},{key:"_removeFromSubgroup",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:t.data.subgroup;if(null!=e){var i=this.subgroups[e];if(i){var n,o,s=Fo(n=i.items).call(n,t);if(s>=0)Hr(o=i.items).call(o,s,1),i.items.length?this._updateSubgroupsSizes():delete this.subgroups[e]}}}},{key:"removeFromDataSet",value:function(t){this.itemSet.removeItem(t.id)}},{key:"order",value:function(){for(var t=za.toArray(this.items),e=[],i=[],n=0;n<t.length;n++)void 0!==t[n].data.end&&i.push(t[n]),e.push(t[n]);this.orderedItems={byStart:e,byEnd:i},Od(this.orderedItems.byStart),Wd(this.orderedItems.byEnd)}},{key:"_updateItemsInRange",value:function(t,e,i){var n=[],o={};if(!this.isVisible&&this.groupId!=Vd){for(var s=0;s<e.length;s++){var r=e[s];r.displayed&&r.hide()}return n}var a=(i.end-i.start)/4,l=i.start-a,h=i.end+a;if(e.length>0)for(var d=0;d<e.length;d++)this._checkIfVisibleWithReference(e[d],n,o,i);var u=za.binarySearchCustom(t.byStart,(function(t){return t<l?-1:t<=h?0:1}),"data","start");if(this._traceVisible(u,t.byStart,n,o,(function(t){return t.data.start<l||t.data.start>h})),1==this.checkRangedItems){this.checkRangedItems=!1;for(var c=0;c<t.byEnd.length;c++)this._checkIfVisibleWithReference(t.byEnd[c],n,o,i)}else{var p=za.binarySearchCustom(t.byEnd,(function(t){return t<l?-1:0}),"data","end");this._traceVisible(p,t.byEnd,n,o,(function(t){return t.data.end<l||t.data.start>h}))}for(var m={},f=0,g=0;g<n.length;g++){var v=n[g];if(!v.displayed){m[g]=v.redraw(!0),f=m[g].length}}if(f>0)for(var b=function(t){mn(za).call(za,m,(function(e){e[t]()}))},y=0;y<f;y++)b(y);for(var x=0;x<n.length;x++)n[x].repositionX();return n}},{key:"_traceVisible",value:function(t,e,i,n,o){if(-1!=t){for(var s=t;s>=0;s--){var r=e[s];if(o(r))break;r.isCluster&&!r.hasItems()||r.cluster||void 0===n[r.id]&&(n[r.id]=!0,i.push(r))}for(var a=t+1;a<e.length;a++){var l=e[a];if(o(l))break;l.isCluster&&!l.hasItems()||l.cluster||void 0===n[l.id]&&(n[l.id]=!0,i.push(l))}}}},{key:"_checkIfVisible",value:function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()}},{key:"_checkIfVisibleWithReference",value:function(t,e,i,n){t.isVisible(n)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()}},{key:"_updateClustersInRange",value:function(t,e,i){var n=[],o={};if(e.length>0)for(var s=0;s<e.length;s++)this._checkIfVisibleWithReference(e[s],n,o,i);for(var r=0;r<t.byStart.length;r++)this._checkIfVisibleWithReference(t.byStart[r],n,o,i);for(var a=0;a<t.byEnd.length;a++)this._checkIfVisibleWithReference(t.byEnd[a],n,o,i);for(var l={},h=0,d=0;d<n.length;d++){var u=n[d];if(!u.displayed){l[d]=u.redraw(!0),h=l[d].length}}if(h>0)for(var c=0;c<h;c++)mn(za).call(za,l,(function(t){t[c]()}));for(var p=0;p<n.length;p++)n[p].repositionX();return n}},{key:"changeSubgroup",value:function(t,e,i){this._removeFromSubgroup(t,e),this._addToSubgroup(t,i),this.orderSubgroups()}},{key:"dispose",value:function(){var t;for(this.hide();t=this._disposeCallbacks.pop();)t()}}]),t}(),Hd=function(t){function e(t,i,n){var o;return ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).width=0,o.height=0,o.top=0,o.left=0,o}return pr(e,t),xa(e,[{key:"redraw",value:function(t,e,i){this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var n=0,o=this.visibleItems.length;n<o;n++){this.visibleItems[n].repositionY(e)}return!1}},{key:"show",value:function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)}}]),e}(jd);Ph('.vis-item{position:absolute;color:#1a1a1a;border-color:#97b0f8;border-width:1px;background-color:#d5ddf6;display:inline-block;z-index:1}.vis-item.vis-selected{border-color:#ffc200;background-color:#fff785;z-index:2}.vis-editable.vis-selected{cursor:move}.vis-item.vis-point.vis-selected{background-color:#fff785}.vis-item.vis-box{text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-point{background:none}.vis-item.vis-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}.vis-item.vis-range{border-style:solid;border-radius:2px;box-sizing:border-box}.vis-item.vis-background{border:none;background-color:rgba(213,221,246,.4);box-sizing:border-box;padding:0;margin:0}.vis-item .vis-item-overflow{position:relative;width:100%;height:100%;padding:0;margin:0;overflow:hidden}.vis-item-visible-frame{white-space:nowrap}.vis-item.vis-range .vis-item-content{position:relative;display:inline-block}.vis-item.vis-background .vis-item-content{position:absolute;display:inline-block}.vis-item.vis-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item .vis-item-content{white-space:nowrap;box-sizing:border-box;padding:5px}.vis-item .vis-onUpdateTime-tooltip{position:absolute;background:#4f81bd;color:#fff;width:200px;text-align:center;white-space:nowrap;padding:5px;border-radius:1px;transition:.4s;-o-transition:.4s;-moz-transition:.4s;-webkit-transition:.4s}.vis-item .vis-delete,.vis-item .vis-delete-rtl{position:absolute;top:0;width:24px;height:24px;box-sizing:border-box;padding:0 5px;cursor:pointer;-webkit-transition:background .2s linear;-moz-transition:background .2s linear;-ms-transition:background .2s linear;-o-transition:background .2s linear;transition:background .2s linear}.vis-item .vis-delete{right:-24px}.vis-item .vis-delete-rtl{left:-24px}.vis-item .vis-delete-rtl:after,.vis-item .vis-delete:after{content:"\\00D7";color:red;font-family:arial,sans-serif;font-size:22px;font-weight:700;-webkit-transition:color .2s linear;-moz-transition:color .2s linear;-ms-transition:color .2s linear;-o-transition:color .2s linear;transition:color .2s linear}.vis-item .vis-delete-rtl:hover,.vis-item .vis-delete:hover{background:red}.vis-item .vis-delete-rtl:hover:after,.vis-item .vis-delete:hover:after{color:#fff}.vis-item .vis-drag-center{position:absolute;width:100%;height:100%;top:0;left:0;cursor:move}.vis-item.vis-range .vis-drag-left{left:-4px;cursor:w-resize}.vis-item.vis-range .vis-drag-left,.vis-item.vis-range .vis-drag-right{position:absolute;width:24px;max-width:20%;min-width:2px;height:100%;top:0}.vis-item.vis-range .vis-drag-right{right:-4px;cursor:e-resize}.vis-range.vis-item.vis-readonly .vis-drag-left,.vis-range.vis-item.vis-readonly .vis-drag-right{cursor:auto}.vis-item.vis-cluster{vertical-align:center;text-align:center;border-style:solid;border-radius:2px}.vis-item.vis-cluster-line{padding:0;position:absolute;width:0;border-left-width:1px;border-left-style:solid}.vis-item.vis-cluster-dot{position:absolute;padding:0;border-width:4px;border-style:solid;border-radius:4px}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIml0ZW0uY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLFVBQ0UsaUJBQWtCLENBQ2xCLGFBQWMsQ0FDZCxvQkFBcUIsQ0FDckIsZ0JBQWlCLENBQ2pCLHdCQUF5QixDQUN6QixvQkFBcUIsQ0FDckIsU0FFRixDQUVBLHVCQUNFLG9CQUFxQixDQUNyQix3QkFBeUIsQ0FHekIsU0FDRixDQUVBLDJCQUNFLFdBQ0YsQ0FFQSxpQ0FDRSx3QkFDRixDQUVBLGtCQUNFLGlCQUFrQixDQUNsQixrQkFBbUIsQ0FDbkIsaUJBQ0YsQ0FFQSxvQkFDRSxlQUNGLENBRUEsa0JBQ0UsaUJBQWtCLENBQ2xCLFNBQVUsQ0FDVixnQkFBaUIsQ0FDakIsa0JBQW1CLENBQ25CLGlCQUNGLENBRUEsb0JBQ0Usa0JBQW1CLENBQ25CLGlCQUFrQixDQUNsQixxQkFDRixDQUVBLHlCQUNFLFdBQVksQ0FDWixxQ0FBMEMsQ0FDMUMscUJBQXNCLENBQ3RCLFNBQVUsQ0FDVixRQUNGLENBRUEsNkJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxXQUFZLENBQ1osU0FBVSxDQUNWLFFBQVMsQ0FDVCxlQUNGLENBRUEsd0JBQ0Usa0JBQ0YsQ0FFQSxzQ0FDRSxpQkFBa0IsQ0FDbEIsb0JBQ0YsQ0FFQSwyQ0FDRSxpQkFBa0IsQ0FDbEIsb0JBQ0YsQ0FFQSxtQkFDRSxTQUFVLENBQ1YsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixxQkFBc0IsQ0FDdEIsdUJBQ0YsQ0FFQSw0QkFDRSxrQkFBbUIsQ0FDbkIscUJBQXNCLENBQ3RCLFdBQ0YsQ0FFQSxvQ0FDRSxpQkFBa0IsQ0FDbEIsa0JBQW1CLENBQ25CLFVBQVksQ0FDWixXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGtCQUFtQixDQUNuQixXQUFZLENBQ1osaUJBQWtCLENBQ2xCLGNBQWdCLENBQ2hCLGlCQUFtQixDQUNuQixtQkFBcUIsQ0FDckIsc0JBQ0YsQ0FFQSxnREFDRSxpQkFBa0IsQ0FDbEIsS0FBUSxDQUNSLFVBQVcsQ0FDWCxXQUFZLENBQ1oscUJBQXNCLENBQ3RCLGFBQWdCLENBQ2hCLGNBQWUsQ0FFZix3Q0FBMEMsQ0FDMUMscUNBQXVDLENBQ3ZDLG9DQUFzQyxDQUN0QyxtQ0FBcUMsQ0FDckMsZ0NBQ0YsQ0FFQSxzQkFDRSxXQUNGLENBRUEsMEJBQ0UsVUFDRixDQUVBLDREQUNFLGVBQWdCLENBQ2hCLFNBQVUsQ0FDViw0QkFBOEIsQ0FDOUIsY0FBZSxDQUNmLGVBQWlCLENBRWpCLG1DQUFxQyxDQUNyQyxnQ0FBa0MsQ0FDbEMsK0JBQWlDLENBQ2pDLDhCQUFnQyxDQUNoQywyQkFDRixDQUVBLDREQUNFLGNBQ0YsQ0FFQSx3RUFDRSxVQUNGLENBRUEsMkJBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxXQUFZLENBQ1osS0FBTSxDQUNOLE1BQVMsQ0FDVCxXQUNGLENBRUEsbUNBT0UsU0FBVSxDQUVWLGVBQ0YsQ0FFQSx1RUFYRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLGFBQWMsQ0FDZCxhQUFjLENBQ2QsV0FBWSxDQUNaLEtBZ0JGLENBVkEsb0NBT0UsVUFBVyxDQUVYLGVBQ0YsQ0FFQSxpR0FFRSxXQUNGLENBRUEsc0JBQ0UscUJBQXNCLENBQ3RCLGlCQUFrQixDQUNsQixrQkFBbUIsQ0FDbkIsaUJBQ0YsQ0FFQSwyQkFDRSxTQUFVLENBQ1YsaUJBQWtCLENBQ2xCLE9BQVEsQ0FDUixxQkFBc0IsQ0FDdEIsdUJBQ0YsQ0FFQSwwQkFDRSxpQkFBa0IsQ0FDbEIsU0FBVSxDQUNWLGdCQUFpQixDQUNqQixrQkFBbUIsQ0FDbkIsaUJBQ0YiLCJmaWxlIjoiaXRlbS5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyJcbi52aXMtaXRlbSB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICMxQTFBMUE7XG4gIGJvcmRlci1jb2xvcjogIzk3QjBGODtcbiAgYm9yZGVyLXdpZHRoOiAxcHg7XG4gIGJhY2tncm91bmQtY29sb3I6ICNENURERjY7XG4gIGRpc3BsYXk6IGlubGluZS1ibG9jaztcbiAgei1pbmRleDogMTtcbiAgLypvdmVyZmxvdzogaGlkZGVuOyovXG59XG5cbi52aXMtaXRlbS52aXMtc2VsZWN0ZWQge1xuICBib3JkZXItY29sb3I6ICNGRkMyMDA7XG4gIGJhY2tncm91bmQtY29sb3I6ICNGRkY3ODU7XG5cbiAgLyogei1pbmRleCBtdXN0IGJlIGhpZ2hlciB0aGFuIHRoZSB6LWluZGV4IG9mIGN1c3RvbSB0aW1lIGJhciBhbmQgY3VycmVudCB0aW1lIGJhciAqL1xuICB6LWluZGV4OiAyO1xufVxuXG4udmlzLWVkaXRhYmxlLnZpcy1zZWxlY3RlZCB7XG4gIGN1cnNvcjogbW92ZTtcbn1cblxuLnZpcy1pdGVtLnZpcy1wb2ludC52aXMtc2VsZWN0ZWQge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiAjRkZGNzg1O1xufVxuXG4udmlzLWl0ZW0udmlzLWJveCB7XG4gIHRleHQtYWxpZ246IGNlbnRlcjtcbiAgYm9yZGVyLXN0eWxlOiBzb2xpZDtcbiAgYm9yZGVyLXJhZGl1czogMnB4O1xufVxuXG4udmlzLWl0ZW0udmlzLXBvaW50IHtcbiAgYmFja2dyb3VuZDogbm9uZTtcbn1cblxuLnZpcy1pdGVtLnZpcy1kb3Qge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHBhZGRpbmc6IDA7XG4gIGJvcmRlci13aWR0aDogNHB4O1xuICBib3JkZXItc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmFkaXVzOiA0cHg7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2Uge1xuICBib3JkZXItc3R5bGU6IHNvbGlkO1xuICBib3JkZXItcmFkaXVzOiAycHg7XG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtaXRlbS52aXMtYmFja2dyb3VuZCB7XG4gIGJvcmRlcjogbm9uZTtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgyMTMsIDIyMSwgMjQ2LCAwLjQpO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG59XG5cbi52aXMtaXRlbSAudmlzLWl0ZW0tb3ZlcmZsb3cge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgb3ZlcmZsb3c6IGhpZGRlbjtcbn1cblxuLnZpcy1pdGVtLXZpc2libGUtZnJhbWUge1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLWl0ZW0udmlzLXJhbmdlIC52aXMtaXRlbS1jb250ZW50IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG59XG5cbi52aXMtaXRlbS52aXMtYmFja2dyb3VuZCAudmlzLWl0ZW0tY29udGVudCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgZGlzcGxheTogaW5saW5lLWJsb2NrO1xufVxuXG4udmlzLWl0ZW0udmlzLWxpbmUge1xuICBwYWRkaW5nOiAwO1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAwO1xuICBib3JkZXItbGVmdC13aWR0aDogMXB4O1xuICBib3JkZXItbGVmdC1zdHlsZTogc29saWQ7XG59XG5cbi52aXMtaXRlbSAudmlzLWl0ZW0tY29udGVudCB7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG4gIHBhZGRpbmc6IDVweDtcbn1cblxuLnZpcy1pdGVtIC52aXMtb25VcGRhdGVUaW1lLXRvb2x0aXAge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIGJhY2tncm91bmQ6ICM0ZjgxYmQ7XG4gIGNvbG9yOiB3aGl0ZTtcbiAgd2lkdGg6IDIwMHB4O1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG4gIHBhZGRpbmc6IDVweDtcbiAgYm9yZGVyLXJhZGl1czogMXB4O1xuICB0cmFuc2l0aW9uOiAwLjRzO1xuICAtby10cmFuc2l0aW9uOiAwLjRzO1xuICAtbW96LXRyYW5zaXRpb246IDAuNHM7XG4gIC13ZWJraXQtdHJhbnNpdGlvbjogMC40cztcbn1cblxuLnZpcy1pdGVtIC52aXMtZGVsZXRlLCAudmlzLWl0ZW0gLnZpcy1kZWxldGUtcnRsIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDBweDtcbiAgd2lkdGg6IDI0cHg7XG4gIGhlaWdodDogMjRweDtcbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcbiAgcGFkZGluZzogMHB4IDVweDtcbiAgY3Vyc29yOiBwb2ludGVyO1xuXG4gIC13ZWJraXQtdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgLW1vei10cmFuc2l0aW9uOiBiYWNrZ3JvdW5kIDAuMnMgbGluZWFyO1xuICAtbXMtdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgLW8tdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbiAgdHJhbnNpdGlvbjogYmFja2dyb3VuZCAwLjJzIGxpbmVhcjtcbn1cblxuLnZpcy1pdGVtIC52aXMtZGVsZXRlIHtcbiAgcmlnaHQ6IC0yNHB4O1xufVxuXG4udmlzLWl0ZW0gLnZpcy1kZWxldGUtcnRsIHtcbiAgbGVmdDogLTI0cHg7XG59XG5cbi52aXMtaXRlbSAudmlzLWRlbGV0ZTphZnRlciwgLnZpcy1pdGVtIC52aXMtZGVsZXRlLXJ0bDphZnRlciB7XG4gIGNvbnRlbnQ6IFwiXFwwMEQ3XCI7IC8qIE1VTFRJUExJQ0FUSU9OIFNJR04gKi9cbiAgY29sb3I6IHJlZDtcbiAgZm9udC1mYW1pbHk6IGFyaWFsLCBzYW5zLXNlcmlmO1xuICBmb250LXNpemU6IDIycHg7XG4gIGZvbnQtd2VpZ2h0OiBib2xkO1xuXG4gIC13ZWJraXQtdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIC1tb3otdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIC1tcy10cmFuc2l0aW9uOiBjb2xvciAwLjJzIGxpbmVhcjtcbiAgLW8tdHJhbnNpdGlvbjogY29sb3IgMC4ycyBsaW5lYXI7XG4gIHRyYW5zaXRpb246IGNvbG9yIDAuMnMgbGluZWFyO1xufVxuXG4udmlzLWl0ZW0gLnZpcy1kZWxldGU6aG92ZXIsIC52aXMtaXRlbSAudmlzLWRlbGV0ZS1ydGw6aG92ZXIge1xuICBiYWNrZ3JvdW5kOiByZWQ7XG59XG5cbi52aXMtaXRlbSAudmlzLWRlbGV0ZTpob3ZlcjphZnRlciwgLnZpcy1pdGVtIC52aXMtZGVsZXRlLXJ0bDpob3ZlcjphZnRlciB7XG4gIGNvbG9yOiB3aGl0ZTtcbn1cblxuLnZpcy1pdGVtIC52aXMtZHJhZy1jZW50ZXIge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHRvcDogMDtcbiAgbGVmdDogMHB4O1xuICBjdXJzb3I6IG1vdmU7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2UgLnZpcy1kcmFnLWxlZnQge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAyNHB4O1xuICBtYXgtd2lkdGg6IDIwJTtcbiAgbWluLXdpZHRoOiAycHg7XG4gIGhlaWdodDogMTAwJTtcbiAgdG9wOiAwO1xuICBsZWZ0OiAtNHB4O1xuXG4gIGN1cnNvcjogdy1yZXNpemU7XG59XG5cbi52aXMtaXRlbS52aXMtcmFuZ2UgLnZpcy1kcmFnLXJpZ2h0IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB3aWR0aDogMjRweDtcbiAgbWF4LXdpZHRoOiAyMCU7XG4gIG1pbi13aWR0aDogMnB4O1xuICBoZWlnaHQ6IDEwMCU7XG4gIHRvcDogMDtcbiAgcmlnaHQ6IC00cHg7XG5cbiAgY3Vyc29yOiBlLXJlc2l6ZTtcbn1cblxuLnZpcy1yYW5nZS52aXMtaXRlbS52aXMtcmVhZG9ubHkgLnZpcy1kcmFnLWxlZnQsXG4udmlzLXJhbmdlLnZpcy1pdGVtLnZpcy1yZWFkb25seSAudmlzLWRyYWctcmlnaHQge1xuICBjdXJzb3I6IGF1dG87XG59XG5cbi52aXMtaXRlbS52aXMtY2x1c3RlciB7XG4gIHZlcnRpY2FsLWFsaWduOiBjZW50ZXI7XG4gIHRleHQtYWxpZ246IGNlbnRlcjtcbiAgYm9yZGVyLXN0eWxlOiBzb2xpZDtcbiAgYm9yZGVyLXJhZGl1czogMnB4O1xufVxuXG4udmlzLWl0ZW0udmlzLWNsdXN0ZXItbGluZSB7XG4gIHBhZGRpbmc6IDA7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgd2lkdGg6IDA7XG4gIGJvcmRlci1sZWZ0LXdpZHRoOiAxcHg7XG4gIGJvcmRlci1sZWZ0LXN0eWxlOiBzb2xpZDtcbn1cblxuLnZpcy1pdGVtLnZpcy1jbHVzdGVyLWRvdCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgcGFkZGluZzogMDtcbiAgYm9yZGVyLXdpZHRoOiA0cHg7XG4gIGJvcmRlci1zdHlsZTogc29saWQ7XG4gIGJvcmRlci1yYWRpdXM6IDRweDtcbn0iXX0= */');var Pd=function(){function t(e,i,n){var o,s=this;ba(this,t),this.id=null,this.parent=null,this.data=e,this.dom=null,this.conversion=i||{},this.defaultOptions={locales:dd,locale:"en"},this.options=za.extend({},this.defaultOptions,n),this.options.locales=za.extend({},dd,this.options.locales);var r=this.defaultOptions.locales[this.defaultOptions.locale];mn(o=Yi(this.options.locales)).call(o,(function(t){s.options.locales[t]=za.extend({},r,s.options.locales[t])})),this.selected=!1,this.displayed=!1,this.groupShowing=!0,this.selectable=n&&n.selectable||!1,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.setSelectability(e),this.editable=null,this._updateEditStatus()}return xa(t,[{key:"select",value:function(){this.selectable&&(this.selected=!0,this.dirty=!0,this.displayed&&this.redraw())}},{key:"unselect",value:function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()}},{key:"setData",value:function(t){null!=t.group&&this.data.group!=t.group&&null!=this.parent&&this.parent.itemSet._moveToGroup(this,t.group),this.setSelectability(t),this.parent&&(this.parent.stackDirty=!0),null!=t.subgroup&&this.data.subgroup!=t.subgroup&&null!=this.parent&&this.parent.changeSubgroup(this,this.data.subgroup,t.subgroup),this.data=t,this._updateEditStatus(),this.dirty=!0,this.displayed&&this.redraw()}},{key:"setSelectability",value:function(t){t&&(this.selectable=void 0===t.selectable||Boolean(t.selectable))}},{key:"setParent",value:function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t}},{key:"isVisible",value:function(t){return!1}},{key:"show",value:function(){return!1}},{key:"hide",value:function(){return!1}},{key:"redraw",value:function(){}},{key:"repositionX",value:function(){}},{key:"repositionY",value:function(){}},{key:"_repaintDragCenter",value:function(){if(this.selected&&this.editable.updateTime&&!this.dom.dragCenter){var t,e,i=this,n=document.createElement("div");n.className="vis-drag-center",n.dragCenterItem=this,this.hammerDragCenter=new Vh(n),this.hammerDragCenter.on("tap",(function(t){i.parent.itemSet.body.emitter.emit("click",{event:t,item:i.id})})),this.hammerDragCenter.on("doubletap",(function(t){t.stopPropagation(),i.parent.itemSet._onUpdateItem(i),i.parent.itemSet.body.emitter.emit("doubleClick",{event:t,item:i.id})})),this.hammerDragCenter.on("panstart",(function(t){t.stopPropagation(),i.parent.itemSet._onDragStart(t)})),this.hammerDragCenter.on("panmove",va(t=i.parent.itemSet._onDrag).call(t,i.parent.itemSet)),this.hammerDragCenter.on("panend",va(e=i.parent.itemSet._onDragEnd).call(e,i.parent.itemSet)),this.dom.box?this.dom.dragLeft?this.dom.box.insertBefore(n,this.dom.dragLeft):this.dom.box.appendChild(n):this.dom.point&&this.dom.point.appendChild(n),this.dom.dragCenter=n}else!this.selected&&this.dom.dragCenter&&(this.dom.dragCenter.parentNode&&this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter),this.dom.dragCenter=null,this.hammerDragCenter&&(this.hammerDragCenter.destroy(),this.hammerDragCenter=null))}},{key:"_repaintDeleteButton",value:function(t){var e=(this.options.editable.overrideItems||null==this.editable)&&this.options.editable.remove||!this.options.editable.overrideItems&&null!=this.editable&&this.editable.remove;if(this.selected&&e&&!this.dom.deleteButton){var i=this,n=document.createElement("div");this.options.rtl?n.className="vis-delete-rtl":n.className="vis-delete";var o=this.options.locales[this.options.locale];o||(this.warned||(console.warn("WARNING: options.locales['".concat(this.options.locale,"'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization")),this.warned=!0),o=this.options.locales.en),n.title=o.deleteSelected,this.hammerDeleteButton=new Vh(n).on("tap",(function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)})),t.appendChild(n),this.dom.deleteButton=n}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null,this.hammerDeleteButton&&(this.hammerDeleteButton.destroy(),this.hammerDeleteButton=null))}},{key:"_repaintOnItemUpdateTimeTooltip",value:function(t){if(this.options.tooltipOnItemUpdateTime){var e=(this.options.editable.updateTime||!0===this.data.editable)&&!1!==this.data.editable;if(this.selected&&e&&!this.dom.onItemUpdateTimeTooltip){var i=document.createElement("div");i.className="vis-onUpdateTime-tooltip",t.appendChild(i),this.dom.onItemUpdateTimeTooltip=i}else!this.selected&&this.dom.onItemUpdateTimeTooltip&&(this.dom.onItemUpdateTimeTooltip.parentNode&&this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip),this.dom.onItemUpdateTimeTooltip=null);if(this.dom.onItemUpdateTimeTooltip){this.dom.onItemUpdateTimeTooltip.style.visibility=this.parent.itemSet.touchParams.itemIsDragging?"visible":"hidden",this.dom.onItemUpdateTimeTooltip.style.transform="translateX(-50%)",this.dom.onItemUpdateTimeTooltip.style.left="50%";var n,o,s=this.parent.itemSet.body.domProps.scrollTop;if(("top"==this.options.orientation.item?this.top:this.parent.height-this.top-this.height)+this.parent.top-50<-s?(this.dom.onItemUpdateTimeTooltip.style.bottom="",this.dom.onItemUpdateTimeTooltip.style.top="".concat(this.height+2,"px")):(this.dom.onItemUpdateTimeTooltip.style.top="",this.dom.onItemUpdateTimeTooltip.style.bottom="".concat(this.height+2,"px")),this.options.tooltipOnItemUpdateTime&&this.options.tooltipOnItemUpdateTime.template)n=va(o=this.options.tooltipOnItemUpdateTime.template).call(o,this)(this.data);else n="start: ".concat(Ta(this.data.start).format("MM/DD/YYYY hh:mm")),this.data.end&&(n+="<br> end: ".concat(Ta(this.data.end).format("MM/DD/YYYY hh:mm")));this.dom.onItemUpdateTimeTooltip.innerHTML=n}}}},{key:"_getItemData",value:function(){return this.parent.itemSet.itemsData.get(this.id)}},{key:"_updateContents",value:function(t){var e,i,n,o,s,r,a=this._getItemData(),l=(this.dom.box||this.dom.point).getElementsByClassName("vis-item-visible-frame")[0];this.options.visibleFrameTemplate?n=(o=va(s=this.options.visibleFrameTemplate).call(s,this))(a,l):n="";if(l)if(n instanceof Object&&!(n instanceof Element))o(a,l);else if(this._contentToString(this.itemVisibleFrameContent)!==this._contentToString(n)){if(n instanceof Element)l.innerHTML="",l.appendChild(n);else if(null!=n)l.innerHTML=n;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '.concat(this.id));this.itemVisibleFrameContent=n}this.options.template?e=(i=va(r=this.options.template).call(r,this))(a,t,this.data):e=this.data.content;if(e instanceof Object&&!(e instanceof Element))i(a,t);else if(this._contentToString(this.content)!==this._contentToString(e)){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(null!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '.concat(this.id));this.content=e}}},{key:"_updateDataAttributes",value:function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Zo(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=Yi(this.data)}var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(e);!(i=(s=r.next()).done);i=!0){var a=s.value,l=this.data[a];null!=l?t.setAttribute("data-".concat(a),l):t.removeAttribute("data-".concat(a))}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}}}},{key:"_updateStyle",value:function(t){this.style&&(za.removeCssText(t,this.style),this.style=null),this.data.style&&(za.addCssText(t,this.data.style),this.style=this.data.style)}},{key:"_contentToString",value:function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t}},{key:"_updateEditStatus",value:function(){this.options&&("boolean"==typeof this.options.editable?this.editable={updateTime:this.options.editable,updateGroup:this.options.editable,remove:this.options.editable}:"object"===zo(this.options.editable)&&(this.editable={},za.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.options.editable))),this.options&&this.options.editable&&!0===this.options.editable.overrideItems||this.data&&("boolean"==typeof this.data.editable?this.editable={updateTime:this.data.editable,updateGroup:this.data.editable,remove:this.data.editable}:"object"===zo(this.data.editable)&&(this.editable={},za.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.data.editable)))}},{key:"getWidthLeft",value:function(){return 0}},{key:"getWidthRight",value:function(){return 0}},{key:"getTitle",value:function(){var t;return this.options.tooltip&&this.options.tooltip.template?va(t=this.options.tooltip.template).call(t,this)(this._getItemData(),this.data):this.data.title}}]),t}();Pd.prototype.stack=!0;var Jd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={dot:{width:0,height:0},line:{width:0,height:0}},t&&null==t.start)throw new Error('Property "start" missing in item '.concat(t));return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){if(this.cluster)return!1;var e,i=this.data.align||this.options.align,n=this.width*t.getMillisecondsPerPixel();return e="right"==i?this.data.start.getTime()>t.start&&this.data.start.getTime()-n<t.end:"left"==i?this.data.start.getTime()+n>t.start&&this.data.start.getTime()<t.end:this.data.start.getTime()+n/2>t.start&&this.data.start.getTime()-n/2<t.end,e}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-line",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-dot",this.dom.box["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}if(!this.dom.line.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className="vis-item vis-box".concat(e),this.dom.line.className="vis-item vis-line".concat(e),this.dom.dot.className="vis-item vis-dot".concat(e)}}},{key:"_getDomComponentsSizes",value:function(){return{previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},dot:{height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},line:{width:this.dom.line.offsetWidth},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.props.dot.height=t.dot.height,this.props.dot.width=t.dot.width,this.props.line.width=t.line.width,this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.box)}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){a.dirty&&(s=a._getDomComponentsSizes())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){if(this.displayed){var t=this.dom;t.box.remove?t.box.remove():t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.remove?t.line.remove():t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.remove?t.dot.remove():t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}}},{key:"repositionXY",value:function(){var t=this.options.rtl,e=function(t,e,i){var n,o=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e||void 0!==i){var s=o?-1*e:e;t.style.transform=void 0!==i?void 0!==e?Pn(n="translate(".concat(s,"px, ")).call(n,i,"px)"):"translateY(".concat(i,"px)"):"translateX(".concat(s,"px)")}};e(this.dom.box,this.boxX,this.boxY,t),e(this.dom.dot,this.dotX,this.dotY,t),e(this.dom.line,this.lineX,this.lineY,t)}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start),e=void 0===this.data.align?this.options.align:this.data.align,i=this.props.line.width,n=this.props.dot.width;"right"==e?(this.boxX=t-this.width,this.lineX=t-i,this.dotX=t-i/2-n/2):"left"==e?(this.boxX=t,this.lineX=t,this.dotX=t+i/2-n/2):(this.boxX=t-this.width/2,this.lineX=this.options.rtl?t-i:t-i/2,this.dotX=t-n/2),this.options.rtl?this.right=this.boxX:this.left=this.boxX,this.repositionXY()}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.line.style;if("top"==t){var i=this.parent.top+this.top+1;this.boxY=this.top||0,e.height="".concat(i,"px"),e.bottom="",e.top="0"}else{var n=this.parent.itemSet.props.height-this.parent.top-this.parent.height+this.top;this.boxY=this.parent.height-this.top-(this.height||0),e.height="".concat(n,"px"),e.top="",e.bottom="0"}this.dotY=-this.props.dot.height/2,this.repositionXY()}},{key:"getWidthLeft",value:function(){return this.width/2}},{key:"getWidthRight",value:function(){return this.width/2}}]),e}(Pd),Kd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},t&&null==t.start)throw new Error('Property "start" missing in item '.concat(t));return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){if(this.cluster)return!1;var e=this.width*t.getMillisecondsPerPixel();return this.data.start.getTime()+e>t.start&&this.data.start<t.end}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.point=document.createElement("div"),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.point.appendChild(this.dom.content),this.dom.dot=document.createElement("div"),this.dom.point.appendChild(this.dom.dot),this.dom.point["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.point.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.point)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.point.className="vis-item vis-point".concat(e),this.dom.dot.className="vis-item vis-dot".concat(e)}}},{key:"_getDomComponentsSizes",value:function(){return{dot:{width:this.dom.dot.offsetWidth,height:this.dom.dot.offsetHeight},content:{width:this.dom.content.offsetWidth,height:this.dom.content.offsetHeight},point:{width:this.dom.point.offsetWidth,height:this.dom.point.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.dot.width=t.dot.width,this.props.dot.height=t.dot.height,this.props.content.height=t.content.height,this.options.rtl?this.dom.content.style.marginRight="".concat(2*this.props.dot.width,"px"):this.dom.content.style.marginLeft="".concat(2*this.props.dot.width,"px"),this.width=t.point.width,this.height=t.point.height,this.dom.dot.style.top="".concat((this.height-this.props.dot.height)/2,"px");var e=this.props.dot.width,i=this.options.rtl?e/2*-1:e/2;this.dom.dot.style.transform="translateX(".concat(i,"px"),this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.point),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.point)}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){a.dirty&&(s=a._getDomComponentsSizes())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"repositionXY",value:function(){var t=this.options.rtl,e=function(t,e,i){var n,o=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(void 0!==e||void 0!==i){var s=o?-1*e:e;t.style.transform=void 0!==i?void 0!==e?Pn(n="translate(".concat(s,"px, ")).call(n,i,"px)"):"translateY(".concat(i,"px)"):"translateX(".concat(s,"px)")}};e(this.dom.point,this.pointX,this.pointY,t)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start);this.pointX=t,this.options.rtl?this.right=t-this.props.dot.width:this.left=t-this.props.dot.width,this.repositionXY()}},{key:"repositionY",value:function(){var t=this.options.orientation.item;this.pointY="top"==t?this.top:this.parent.height-this.top-this.height,this.repositionXY()}},{key:"getWidthLeft",value:function(){return this.props.dot.width}},{key:"getWidthRight",value:function(){return this.props.dot.width}}]),e}(Pd),qd=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={content:{width:0}},o.overflow=!1,t){if(null==t.start)throw new Error('Property "start" missing in item '.concat(t.id));if(null==t.end)throw new Error('Property "end" missing in item '.concat(t.id))}return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){return!this.cluster&&(this.data.start<t.end&&this.data.end>t.start)}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.visibleFrame=document.createElement("div"),this.dom.visibleFrame.className="vis-item-visible-frame",this.dom.box.appendChild(this.dom.visibleFrame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dom.box["vis-item"]=this,this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className=this.baseClassName+e,this.dom.content.style.maxWidth="none"}}},{key:"_getDomComponentsSizes",value:function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.frame).overflow,this.whiteSpace="nowrap"!==window.getComputedStyle(this.dom.content).whiteSpace,{content:{width:this.dom.content.offsetWidth},box:{height:this.dom.box.offsetHeight}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.content.width=t.content.width,this.height=t.box.height,this.dom.content.style.maxWidth="",this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDeleteButton(this.dom.box),this._repaintDragCenter(),this._repaintDragLeft(),this._repaintDragRight()}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){var t;a.dirty&&(s=va(t=a._getDomComponentsSizes).call(t,a)())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"show",value:function(t){if(!this.displayed)return this.redraw(t)}},{key:"hide",value:function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}}},{key:"repositionX",value:function(t){var e,i,n=this.parent.width,o=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end),r=void 0===this.data.align?this.options.align:this.data.align;!1===this.data.limitSize||void 0!==t&&!0!==t||(o<-n&&(o=-n),s>2*n&&(s=2*n));var a=Math.max(Math.round(1e3*(s-o))/1e3,1);switch(this.overflow?(this.options.rtl?this.right=o:this.left=o,this.width=a+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=o:this.left=o,this.width=a,i=Math.min(s-o,this.props.content.width)),this.options.rtl?this.dom.box.style.transform="translateX(".concat(-1*this.right,"px)"):this.dom.box.style.transform="translateX(".concat(this.left,"px)"),this.dom.box.style.width="".concat(a,"px"),this.whiteSpace&&(this.height=this.dom.box.offsetHeight),r){case"left":this.dom.content.style.transform="translateX(0)";break;case"right":if(this.options.rtl){var l=-1*Math.max(a-i,0);this.dom.content.style.transform="translateX(".concat(l,"px)")}else this.dom.content.style.transform="translateX(".concat(Math.max(a-i,0),"px)");break;case"center":if(this.options.rtl){var h=-1*Math.max((a-i)/2,0);this.dom.content.style.transform="translateX(".concat(h,"px)")}else this.dom.content.style.transform="translateX(".concat(Math.max((a-i)/2,0),"px)");break;default:if(e=this.overflow?s>0?Math.max(-o,0):-i:o<0?-o:0,this.options.rtl){var d=-1*e;this.dom.content.style.transform="translateX(".concat(d,"px)")}else this.dom.content.style.transform="translateX(".concat(e,"px)")}}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.box;e.style.top="".concat("top"==t?this.top:this.parent.height-this.top-this.height,"px")}},{key:"_repaintDragLeft",value:function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragLeft||(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)}},{key:"_repaintDragRight",value:function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragRight||(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)}}]),e}(Pd);qd.prototype.baseClassName="vis-item vis-range";var $d=function(t){function e(t,i,n){var o;if(ba(this,e),(o=ar(this,dr(e).call(this,t,i,n))).props={content:{width:0}},o.overflow=!1,t){if(null==t.start)throw new Error('Property "start" missing in item '.concat(t.id));if(null==t.end)throw new Error('Property "end" missing in item '.concat(t.id))}return o}return pr(e,t),xa(e,[{key:"isVisible",value:function(t){return this.data.start<t.end&&this.data.end>t.start}},{key:"_createDomElement",value:function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dirty=!0)}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.background;if(!t)throw new Error("Cannot redraw item: parent has no background container element");t.appendChild(this.dom.box)}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var t=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");this.dom.box.className=this.baseClassName+t}}},{key:"_getDomComponentsSizes",value:function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.content).overflow,{content:{width:this.dom.content.offsetWidth}}}},{key:"_updateDomComponentsSizes",value:function(t){this.props.content.width=t.content.width,this.height=0,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a=this,l=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),function(){var t;a.dirty&&(s=va(t=a._getDomComponentsSizes).call(t,a)())},function(){var t;a.dirty&&va(t=a._updateDomComponentsSizes).call(t,a)(s)},va(o=this._repaintDomAdditionals).call(o,this)];return t?l:(mn(l).call(l,(function(t){r=t()})),r)}},{key:"repositionY",value:function(t){var e,i=this.options.orientation.item;if(void 0!==this.data.subgroup){var n=this.data.subgroup;this.dom.box.style.height="".concat(this.parent.subgroups[n].height,"px"),this.dom.box.style.top="".concat("top"==i?this.parent.top+this.parent.subgroups[n].top:this.parent.top+this.parent.height-this.parent.subgroups[n].top-this.parent.subgroups[n].height,"px"),this.dom.box.style.bottom=""}else this.parent instanceof Hd?(e=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.bottom="bottom"==i?"0":"",this.dom.box.style.top="top"==i?"0":""):(e=this.parent.height,this.dom.box.style.top="".concat(this.parent.top,"px"),this.dom.box.style.bottom="");this.dom.box.style.height="".concat(e,"px")}}]),e}(Pd);$d.prototype.baseClassName="vis-item vis-background",$d.prototype.stack=!1,$d.prototype.show=qd.prototype.show,$d.prototype.hide=qd.prototype.hide,$d.prototype.repositionX=qd.prototype.repositionX;Ph("div.vis-tooltip{position:absolute;visibility:hidden;padding:5px;white-space:nowrap;font-family:verdana;font-size:14px;color:#000;background-color:#f5f4ed;-moz-border-radius:3px;-webkit-border-radius:3px;border-radius:3px;border:1px solid #808074;box-shadow:3px 3px 10px rgba(0,0,0,.2);pointer-events:none;z-index:5}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRvb2x0aXAuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLGdCQUNFLGlCQUFrQixDQUNsQixpQkFBa0IsQ0FDbEIsV0FBWSxDQUNaLGtCQUFtQixDQUVuQixtQkFBb0IsQ0FDcEIsY0FBYyxDQUNkLFVBQWEsQ0FDYix3QkFBeUIsQ0FFekIsc0JBQXVCLENBQ3ZCLHlCQUEwQixDQUMxQixpQkFBa0IsQ0FDbEIsd0JBQXlCLENBRXpCLHNDQUEyQyxDQUMzQyxtQkFBb0IsQ0FFcEIsU0FDRiIsImZpbGUiOiJ0b29sdGlwLmNzcyIsInNvdXJjZXNDb250ZW50IjpbImRpdi52aXMtdG9vbHRpcCB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICBwYWRkaW5nOiA1cHg7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG5cbiAgZm9udC1mYW1pbHk6IHZlcmRhbmE7XG4gIGZvbnQtc2l6ZToxNHB4O1xuICBjb2xvcjojMDAwMDAwO1xuICBiYWNrZ3JvdW5kLWNvbG9yOiAjZjVmNGVkO1xuXG4gIC1tb3otYm9yZGVyLXJhZGl1czogM3B4O1xuICAtd2Via2l0LWJvcmRlci1yYWRpdXM6IDNweDtcbiAgYm9yZGVyLXJhZGl1czogM3B4O1xuICBib3JkZXI6IDFweCBzb2xpZCAjODA4MDc0O1xuXG4gIGJveC1zaGFkb3c6IDNweCAzcHggMTBweCByZ2JhKDAsIDAsIDAsIDAuMik7XG4gIHBvaW50ZXItZXZlbnRzOiBub25lO1xuXG4gIHotaW5kZXg6IDU7XG59XG4iXX0= */");var tu=function(){function t(e,i){ba(this,t),this.container=e,this.overflowMethod=i||"cap",this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-tooltip",this.container.appendChild(this.frame)}return xa(t,[{key:"setPosition",value:function(t,e){this.x=Nn(t),this.y=Nn(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),!0===t){var e=this.frame.clientHeight,i=this.frame.clientWidth,n=this.frame.parentNode.clientHeight,o=this.frame.parentNode.clientWidth,s=0,r=0;if("flip"==this.overflowMethod||"none"==this.overflowMethod){var a=!1,l=!0;"flip"==this.overflowMethod&&(this.y-e<this.padding&&(l=!1),this.x+i>o-this.padding&&(a=!0)),s=a?this.x-i:this.x,r=l?this.y-e:this.y}else(r=this.y-e)+e+this.padding>n&&(r=n-e-this.padding),r<this.padding&&(r=this.padding),(s=this.x)+i+this.padding>o&&(s=o-i-this.padding),s<this.padding&&(s=this.padding);this.frame.style.left=s+"px",this.frame.style.top=r+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.left="0",this.frame.style.top="0",this.frame.style.visibility="hidden"}},{key:"destroy",value:function(){this.frame.parentNode.removeChild(this.frame)}}]),t}(),eu=Ce.every,iu=$i("every"),nu=on("every");V({target:"Array",proto:!0,forced:!iu||!nu},{every:function(t){return eu(this,t,arguments.length>1?arguments[1]:void 0)}});var ou=Pi("Array").every,su=Array.prototype,ru=function(t){var e=t.every;return t===su||t instanceof Array&&e===su.every?ou:e},au=function(t){function e(t,i,n){var o;ba(this,e);var s=Ao({},{fitOnDoubleClick:!0},n,{editable:!1});if((o=ar(this,dr(e).call(this,t,i,s))).props={content:{width:0,height:0}},!t||null==t.uiItems)throw new Error('Property "uiItems" missing in item '+t.id);return o.id=Id(),o.group=t.group,o._setupRange(),o.emitter=o.data.eventEmitter,o.range=o.data.range,o.attached=!1,o.isCluster=!0,o.data.isCluster=!0,o}return pr(e,t),xa(e,[{key:"hasItems",value:function(){return this.data.uiItems&&this.data.uiItems.length&&this.attached}},{key:"setUiItems",value:function(t){this.detach(),this.data.uiItems=t,this._setupRange(),this.attach()}},{key:"isVisible",value:function(t){var e=this.data.end?this.data.end-this.data.start:0,i=this.width*t.getMillisecondsPerPixel(),n=Math.max(e,this.data.start.getTime()+i);return this.data.start<t.end&&n>t.start&&this.hasItems()}},{key:"getData",value:function(){return{isCluster:!0,id:this.id,items:this.data.items||[],data:this.data}}},{key:"redraw",value:function(t){var e,i,n,o,s,r,a,l,h=[va(e=this._createDomElement).call(e,this),va(i=this._appendDomElement).call(i,this),va(n=this._updateDirtyDomComponents).call(n,this),va(o=function(){this.dirty&&(a=this._getDomComponentsSizes())}).call(o,this),va(s=function(){var t;this.dirty&&va(t=this._updateDomComponentsSizes).call(t,this)(a)}).call(s,this),va(r=this._repaintDomAdditionals).call(r,this)];return t?h:(mn(h).call(h,(function(t){l=t()})),l)}},{key:"show",value:function(){this.displayed||this.redraw()}},{key:"hide",value:function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),this.options.showStipes&&(t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot)),this.displayed=!1}}},{key:"repositionX",value:function(){var t=this.conversion.toScreen(this.data.start),e=this.data.end?this.conversion.toScreen(this.data.end):0;if(e)this.repositionXWithRanges(t,e);else{var i=void 0===this.data.align?this.options.align:this.data.align;this.repositionXWithoutRanges(t,i)}this.options.showStipes&&(this.dom.line.style.display=this._isStipeVisible()?"block":"none",this.dom.dot.style.display=this._isStipeVisible()?"block":"none",this._isStipeVisible()&&this.repositionStype(t,e))}},{key:"repositionStype",value:function(t,e){this.dom.line.style.display="block",this.dom.dot.style.display="block";var i=this.dom.line.offsetWidth,n=this.dom.dot.offsetWidth;if(e){var o=i+t+(e-t)/2,s=o-n/2,r=this.options.rtl?-1*o:o,a=this.options.rtl?-1*s:s;this.dom.line.style.transform="translateX(".concat(r,"px)"),this.dom.dot.style.transform="translateX(".concat(a,"px)")}else{var l=this.options.rtl?-1*t:t,h=this.options.rtl?-1*(t-n/2):t-n/2;this.dom.line.style.transform="translateX(".concat(l,"px)"),this.dom.dot.style.transform="translateX(".concat(h,"px)")}}},{key:"repositionXWithoutRanges",value:function(t,e){"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px"):(this.left=t,this.dom.box.style.left=this.left+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px")}},{key:"repositionXWithRanges",value:function(t,e){var i=Math.round(Math.max(e-t+.5,1));this.options.rtl?this.right=t:this.left=t,this.width=Math.max(i,this.minWidth||0),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=i+"px"}},{key:"repositionY",value:function(){var t=this.options.orientation.item,e=this.dom.box;if(e.style.top="top"==t?(this.top||0)+"px":(this.parent.height-this.top-this.height||0)+"px",this.options.showStipes){if("top"==t)this.dom.line.style.top="0",this.dom.line.style.height=this.parent.top+this.top+1+"px",this.dom.line.style.bottom="";else{var i=this.parent.itemSet.props.height,n=i-this.parent.top-this.parent.height+this.top;this.dom.line.style.top=i-n+"px",this.dom.line.style.bottom="0"}this.dom.dot.style.top=-this.dom.dot.offsetHeight/2+"px"}}},{key:"getWidthLeft",value:function(){return this.width/2}},{key:"getWidthRight",value:function(){return this.width/2}},{key:"move",value:function(){this.repositionX(),this.repositionY()}},{key:"attach",value:function(){var t,e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this.data.uiItems);!(e=(o=s.next()).done);e=!0){o.value.cluster=this}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}this.data.items=Ln(t=this.data.uiItems).call(t,(function(t){return t.data})),this.attached=!0,this.dirty=!0}},{key:"detach",value:function(){var t=arguments.length>0&&void 0!==arguments[0]&&arguments[0];if(this.hasItems()){var e=!0,i=!1,n=void 0;try{for(var o,s=Fi(this.data.uiItems);!(e=(o=s.next()).done);e=!0){var r=o.value;delete r.cluster}}catch(t){i=!0,n=t}finally{try{e||null==s.return||s.return()}finally{if(i)throw n}}this.attached=!1,t&&this.group&&(this.group.remove(this),this.group=null),this.data.items=[],this.dirty=!0}}},{key:"_onDoubleClick",value:function(){this._fit()}},{key:"_setupRange",value:function(){var t,e,i,n=Ln(t=this.data.uiItems).call(t,(function(t){return{start:t.data.start.valueOf(),end:t.data.end?t.data.end.valueOf():t.data.start.valueOf()}}));this.data.min=Math.min.apply(Math,mo(Ln(n).call(n,(function(t){return Math.min(t.start,t.end||t.start)})))),this.data.max=Math.max.apply(Math,mo(Ln(n).call(n,(function(t){return Math.max(t.start,t.end||t.start)}))));var o=Ln(e=this.data.uiItems).call(e,(function(t){return t.center})),s=da(o).call(o,(function(t,e){return t+e}),0)/this.data.uiItems.length;rr(i=this.data.uiItems).call(i,(function(t){return t.data.end}))?(this.data.start=new Date(this.data.min),this.data.end=new Date(this.data.max)):(this.data.start=new Date(s),this.data.end=null)}},{key:"_getUiItems",value:function(){var t,e=this;return this.data.uiItems&&this.data.uiItems.length?En(t=this.data.uiItems).call(t,(function(t){return t.cluster===e})):[]}},{key:"_createDomElement",value:function(){if(!this.dom){var t;if(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.options.showStipes&&(this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-cluster-line",this.dom.line.style.display="none",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-cluster-dot",this.dom.dot.style.display="none"),this.options.fitOnDoubleClick)this.dom.box.ondblclick=va(t=e.prototype._onDoubleClick).call(t,this);this.dom.box["vis-item"]=this,this.dirty=!0}}},{key:"_appendDomElement",value:function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}var e=this.parent.dom.background;if(this.options.showStipes){if(!this.dom.line.parentNode){if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}}this.displayed=!0}},{key:"_updateDirtyDomComponents",value:function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.baseClassName+" "+(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+" vis-readonly";this.dom.box.className="vis-item "+t,this.options.showStipes&&(this.dom.line.className="vis-item vis-cluster-line "+(this.selected?" vis-selected":""),this.dom.dot.className="vis-item vis-cluster-dot "+(this.selected?" vis-selected":"")),this.data.end&&(this.dom.content.style.maxWidth="none")}}},{key:"_getDomComponentsSizes",value:function(){var t={previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}};return this.options.showStipes&&(t.dot={height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},t.line={width:this.dom.line.offsetWidth}),t}},{key:"_updateDomComponentsSizes",value:function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.data.end?this.minWidth=t.box.width:this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1}},{key:"_repaintDomAdditionals",value:function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box)}},{key:"_isStipeVisible",value:function(){return this.minWidth>=this.width||!this.data.end}},{key:"_getFitRange",value:function(){var t=.05*(this.data.max-this.data.min)/2;return{fitStart:this.data.min-t,fitEnd:this.data.max+t}}},{key:"_fit",value:function(){if(this.emitter){var t=this._getFitRange(),e=t.fitStart,i=t.fitEnd,n={start:new Date(e),end:new Date(i),animation:!0};this.emitter.emit("fit",n)}}},{key:"_getItemData",value:function(){return this.data}}]),e}(Pd);au.prototype.baseClassName="vis-item vis-range vis-cluster";var lu="__ungrouped__",hu=function(){function t(e){ba(this,t),this.itemSet=e,this.groups={},this.cache={},this.cache[-1]=[]}return xa(t,[{key:"createClusterItem",value:function(t,e,i){return new au(t,e,i)}},{key:"setItems",value:function(t,e){this.items=t||[],this.dataChanged=!0,this.applyOnChangedLevel=!1,e&&e.applyOnChangedLevel&&(this.applyOnChangedLevel=e.applyOnChangedLevel)}},{key:"updateData",value:function(){this.dataChanged=!0,this.applyOnChangedLevel=!1}},{key:"getClusters",value:function(t,e,i){var n="boolean"==typeof i?{}:i,o=n.maxItems,s=n.clusterCriteria;s||(s=function(){return!0}),o=o||1;var r=-1,a=0;if(e>0){if(e>=1)return[];r=Math.abs(Math.round(Math.log(100/e)/Math.log(2))),a=Math.abs(Math.pow(2,r))}if(this.dataChanged){var l=r!=this.cacheLevel;(!this.applyOnChangedLevel||l)&&(this._dropLevelsCache(),this._filterData())}this.cacheLevel=r;var h=this.cache[r];if(!h){for(var d in h=[],this.groups)if(this.groups.hasOwnProperty(d))for(var u=this.groups[d],c=u.length,p=0;p<c;){for(var m=u[p],f=1,g=p-1;g>=0&&m.center-u[g].center<a/2;)!u[g].cluster&&s(m.data,u[g].data)&&f++,g--;for(var v=p+1;v<u.length&&u[v].center-m.center<a/2;)s(m.data,u[v].data)&&f++,v++;for(var b=h.length-1;b>=0&&m.center-h[b].center<a;)m.group==h[b].group&&s(m.data,h[b].data)&&f++,b--;if(f>o){for(var y=f-o+1,x=[],w=p;x.length<y&&w<u.length;)s(u[w].data,u[w].data)&&x.push(u[w]),w++;var _=this.itemSet.getGroupId(m.data),k=this.itemSet.groups[_]||this.itemSet.groups[lu],D=this._getClusterForItems(x,k,t,i);h.push(D),p+=y}else delete m.cluster,p+=1}this.cache[r]=h}return h}},{key:"_filterData",value:function(){var t={};this.groups=t;for(var e=0,i=On(this.items);e<i.length;e++){var n=i[e],o=n.parent?n.parent.groupId:"",s=t[o];s||(s=[],t[o]=s),s.push(n),n.data.start&&(n.data.end?n.center=(n.data.start.valueOf()+n.data.end.valueOf())/2:n.center=n.data.start.valueOf())}for(var r in t){var a;if(t.hasOwnProperty(r))wr(a=t[r]).call(a,(function(t,e){return t.center-e.center}))}this.dataChanged=!1}},{key:"_getClusterForItems",value:function(t,e,i,n){var o,s,r=Ln(o=i||[]).call(o,(function(t){var e;return{cluster:t,itemsIds:new Ar(Ln(e=t.data.uiItems).call(e,(function(t){return t.id})))}}));if(r.length){var a=!0,l=!1,h=void 0;try{for(var d,u=function(){var e=d.value;if(e.itemsIds.size===t.length&&ru(t).call(t,(function(t){return e.itemsIds.has(t.id)})))return s=e.cluster,"break"},c=Fi(r);!(a=(d=c.next()).done);a=!0){if("break"===u())break}}catch(t){l=!0,h=t}finally{try{a||null==c.return||c.return()}finally{if(l)throw h}}}if(s)return s.setUiItems(t),s.group!==e&&(s.group&&s.group.remove(s),e&&(e.add(s),s.group=e)),s;var p=n.titleTemplate||"",m={toScreen:this.itemSet.body.util.toScreen,toTime:this.itemSet.body.util.toTime},f=p.replace(/{count}/,t.length),g='<div title="'+f+'">'+t.length+"</div>",v=Ao({},n,this.itemSet.options),b={content:g,title:f,group:e,uiItems:t,eventEmitter:this.itemSet.body.emitter,range:this.itemSet.body.range};return s=this.createClusterItem(b,m,v),e&&(e.add(s),s.group=e),s.attach(),s}},{key:"_dropLevelsCache",value:function(){this.cache={},this.cacheLevel=-1,this.cache[this.cacheLevel]=[]}}]),t}();Ph('.vis-itemset{position:relative;padding:0;margin:0;box-sizing:border-box}.vis-itemset .vis-background,.vis-itemset .vis-foreground{position:absolute;width:100%;height:100%;overflow:visible}.vis-axis{position:absolute;width:100%;height:0;left:0;z-index:1}.vis-foreground .vis-group{position:relative;box-sizing:border-box;border-bottom:1px solid #bfbfbf}.vis-foreground .vis-group:last-child{border-bottom:none}.vis-nesting-group{cursor:pointer}.vis-label.vis-nested-group.vis-group-level-unknown-but-gte1{background:#f5f5f5}.vis-label.vis-nested-group.vis-group-level-0{background-color:#fff}.vis-ltr .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-left:0}.vis-rtl .vis-label.vis-nested-group.vis-group-level-0 .vis-inner{padding-right:0}.vis-label.vis-nested-group.vis-group-level-1{background-color:rgba(0,0,0,.05)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-left:15px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-1 .vis-inner{padding-right:15px}.vis-label.vis-nested-group.vis-group-level-2{background-color:rgba(0,0,0,.1)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-left:30px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-2 .vis-inner{padding-right:30px}.vis-label.vis-nested-group.vis-group-level-3{background-color:rgba(0,0,0,.15)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-left:45px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-3 .vis-inner{padding-right:45px}.vis-label.vis-nested-group.vis-group-level-4{background-color:rgba(0,0,0,.2)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-left:60px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-4 .vis-inner{padding-right:60px}.vis-label.vis-nested-group.vis-group-level-5{background-color:rgba(0,0,0,.25)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-left:75px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-5 .vis-inner{padding-right:75px}.vis-label.vis-nested-group.vis-group-level-6{background-color:rgba(0,0,0,.3)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-left:90px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-6 .vis-inner{padding-right:90px}.vis-label.vis-nested-group.vis-group-level-7{background-color:rgba(0,0,0,.35)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-left:105px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-7 .vis-inner{padding-right:105px}.vis-label.vis-nested-group.vis-group-level-8{background-color:rgba(0,0,0,.4)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-left:120px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-8 .vis-inner{padding-right:120px}.vis-label.vis-nested-group.vis-group-level-9{background-color:rgba(0,0,0,.45)}.vis-ltr .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-left:135px}.vis-rtl .vis-label.vis-nested-group.vis-group-level-9 .vis-inner{padding-right:135px}.vis-label.vis-nested-group{background-color:rgba(0,0,0,.5)}.vis-ltr .vis-label.vis-nested-group .vis-inner{padding-left:150px}.vis-rtl .vis-label.vis-nested-group .vis-inner{padding-right:150px}.vis-group-level-unknown-but-gte1{border:1px solid red}.vis-label.vis-nesting-group:before{display:inline-block;width:15px}.vis-label.vis-nesting-group.expanded:before{content:"\\25BC"}.vis-label.vis-nesting-group.collapsed:before{content:"\\25B6"}.vis-rtl .vis-label.vis-nesting-group.collapsed:before{content:"\\25C0"}.vis-ltr .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-left:15px}.vis-rtl .vis-label:not(.vis-nesting-group):not(.vis-group-level-0){padding-right:15px}.vis-overlay{position:absolute;top:0;left:0;width:100%;height:100%;z-index:10}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIml0ZW1zZXQuY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLGFBQ0UsaUJBQWtCLENBQ2xCLFNBQVUsQ0FDVixRQUFTLENBRVQscUJBQ0YsQ0FFQSwwREFFRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLFdBQVksQ0FDWixnQkFDRixDQUVBLFVBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxRQUFTLENBQ1QsTUFBTyxDQUNQLFNBQ0YsQ0FFQSwyQkFDRSxpQkFBa0IsQ0FDbEIscUJBQXNCLENBQ3RCLCtCQUNGLENBRUEsc0NBQ0Usa0JBQ0YsQ0FFQSxtQkFDRSxjQUNGLENBRUEsNkRBQ0Usa0JBQ0YsQ0FDQSw4Q0FDRSxxQkFDRixDQUNBLGtFQUNFLGNBQ0YsQ0FDQSxrRUFDRSxlQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxpQkFDRixDQUNBLGtFQUNFLGtCQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBQ0EsOENBQ0UsK0JBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBQ0EsOENBQ0UsZ0NBQ0YsQ0FDQSxrRUFDRSxrQkFDRixDQUNBLGtFQUNFLG1CQUNGLENBR0EsNEJBQ0UsK0JBQ0YsQ0FDQSxnREFDRSxrQkFDRixDQUNBLGdEQUNFLG1CQUNGLENBRUEsa0NBQ0Usb0JBQ0YsQ0FHQSxvQ0FFRSxvQkFBcUIsQ0FDckIsVUFDRixDQUNBLDZDQUNFLGVBQ0YsQ0FDQSw4Q0FDRSxlQUNGLENBQ0EsdURBQ0UsZUFDRixDQUVBLG9FQUNFLGlCQUNGLENBQ0Esb0VBQ0Usa0JBQ0YsQ0FFQSxhQUNFLGlCQUFrQixDQUNsQixLQUFNLENBQ04sTUFBTyxDQUNQLFVBQVcsQ0FDWCxXQUFZLENBQ1osVUFDRiIsImZpbGUiOiJpdGVtc2V0LmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy1pdGVtc2V0IHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG5cbiAgYm94LXNpemluZzogYm9yZGVyLWJveDtcbn1cblxuLnZpcy1pdGVtc2V0IC52aXMtYmFja2dyb3VuZCxcbi52aXMtaXRlbXNldCAudmlzLWZvcmVncm91bmQge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIG92ZXJmbG93OiB2aXNpYmxlO1xufVxuXG4udmlzLWF4aXMge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDA7XG4gIGxlZnQ6IDA7XG4gIHotaW5kZXg6IDE7XG59XG5cbi52aXMtZm9yZWdyb3VuZCAudmlzLWdyb3VwIHtcbiAgcG9zaXRpb246IHJlbGF0aXZlO1xuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xuICBib3JkZXItYm90dG9tOiAxcHggc29saWQgI2JmYmZiZjtcbn1cblxuLnZpcy1mb3JlZ3JvdW5kIC52aXMtZ3JvdXA6bGFzdC1jaGlsZCB7XG4gIGJvcmRlci1ib3R0b206IG5vbmU7XG59XG5cbi52aXMtbmVzdGluZy1ncm91cCB7XG4gIGN1cnNvcjogcG9pbnRlcjtcbn1cblxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC11bmtub3duLWJ1dC1ndGUxIHtcbiAgYmFja2dyb3VuZDogI2Y1ZjVmNTtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMCB7XG4gIGJhY2tncm91bmQtY29sb3I6ICNmZmZmZmY7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMCAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogMDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMSB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4wNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTEgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTVweDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMSAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogMTVweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMiB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4xKTtcbn1cbi52aXMtbHRyIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMiAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1sZWZ0OiAzMHB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0yIC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiAzMHB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0zIHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjE1KTtcbn1cbi52aXMtbHRyIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtMyAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1sZWZ0OiA0NXB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC0zIC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiA0NXB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC00IHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjIpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC00IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDYwcHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTQgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDYwcHg7XG59XG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTUge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuMjUpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC01IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDc1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTUgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDc1cHg7XG59XG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTYge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuMyk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTYgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogOTBweDtcbn1cbi52aXMtcnRsIC52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtNiAudmlzLWlubmVyIHtcbiAgcGFkZGluZy1yaWdodDogOTBweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtNyB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC4zNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTcgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTA1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTcgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDEwNXB4O1xufVxuLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IHtcbiAgYmFja2dyb3VuZC1jb2xvcjogcmdiYSgwLCAwLCAwLCAwLjQpO1xufVxuLnZpcy1sdHIgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLWxlZnQ6IDEyMHB4O1xufVxuLnZpcy1ydGwgLnZpcy1sYWJlbC52aXMtbmVzdGVkLWdyb3VwLnZpcy1ncm91cC1sZXZlbC04IC52aXMtaW5uZXIge1xuICBwYWRkaW5nLXJpZ2h0OiAxMjBweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RlZC1ncm91cC52aXMtZ3JvdXAtbGV2ZWwtOSB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMCwgMCwgMCwgMC40NSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTkgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTM1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAudmlzLWdyb3VwLWxldmVsLTkgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDEzNXB4O1xufVxuLyogZGVmYXVsdCB0YWtlcyBvdmVyIGJlZ2lubmluZyB3aXRoIGxldmVsLTEwICh0aGF0cyB3aHkgd2UgYWRkIC52aXMtbmVzdGVkLWdyb3VwXG4gIHRvIHRoZSBzZWxlY3RvcnMgYWJvdmUsIHRvIGhhdmUgaGlnaGVyIHNwZWNpZml0eSB0aGFuIHRoZXNlIHJ1bGVzIGZvciB0aGUgZGVmYXVsdHMpICovXG4udmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAge1xuICBiYWNrZ3JvdW5kLWNvbG9yOiByZ2JhKDAsIDAsIDAsIDAuNSk7XG59XG4udmlzLWx0ciAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctbGVmdDogMTUwcHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0ZWQtZ3JvdXAgLnZpcy1pbm5lciB7XG4gIHBhZGRpbmctcmlnaHQ6IDE1MHB4O1xufVxuXG4udmlzLWdyb3VwLWxldmVsLXVua25vd24tYnV0LWd0ZTEge1xuICBib3JkZXI6IDFweCBzb2xpZCByZWQ7XG59XG5cbi8qIGV4cGFuZGVkL2NvbGxhcHNlZCBpbmRpY2F0b3JzICovXG4udmlzLWxhYmVsLnZpcy1uZXN0aW5nLWdyb3VwOmJlZm9yZSxcbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXA6YmVmb3JlIHtcbiAgZGlzcGxheTogaW5saW5lLWJsb2NrO1xuICB3aWR0aDogMTVweDtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXAuZXhwYW5kZWQ6YmVmb3JlIHtcbiAgY29udGVudDogXCJcXDI1QkNcIjtcbn1cbi52aXMtbGFiZWwudmlzLW5lc3RpbmctZ3JvdXAuY29sbGFwc2VkOmJlZm9yZSB7XG4gIGNvbnRlbnQ6IFwiXFwyNUI2XCI7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsLnZpcy1uZXN0aW5nLWdyb3VwLmNvbGxhcHNlZDpiZWZvcmUge1xuICBjb250ZW50OiBcIlxcMjVDMFwiO1xufVxuLyogY29tcGVuc2F0ZSBtaXNzaW5nIGV4cGFuZGVkL2NvbGxhcHNlZCBpbmRpY2F0b3IsIGJ1dCBvbmx5IGF0IGxldmVscyA+IDAgKi9cbi52aXMtbHRyIC52aXMtbGFiZWw6bm90KC52aXMtbmVzdGluZy1ncm91cCk6bm90KC52aXMtZ3JvdXAtbGV2ZWwtMCkge1xuICBwYWRkaW5nLWxlZnQ6IDE1cHg7XG59XG4udmlzLXJ0bCAudmlzLWxhYmVsOm5vdCgudmlzLW5lc3RpbmctZ3JvdXApOm5vdCgudmlzLWdyb3VwLWxldmVsLTApIHtcbiAgcGFkZGluZy1yaWdodDogMTVweDtcbn1cblxuLnZpcy1vdmVybGF5IHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICB0b3A6IDA7XG4gIGxlZnQ6IDA7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDEwMCU7XG4gIHotaW5kZXg6IDEwO1xufSJdfQ== */');Ph(".vis-labelset{overflow:hidden}.vis-labelset,.vis-labelset .vis-label{position:relative;box-sizing:border-box}.vis-labelset .vis-label{left:0;top:0;width:100%;color:#4d4d4d;border-bottom:1px solid #bfbfbf}.vis-labelset .vis-label.draggable{cursor:pointer}.vis-group-is-dragging{background:rgba(0,0,0,.1)}.vis-labelset .vis-label:last-child{border-bottom:none}.vis-labelset .vis-label .vis-inner{display:inline-block;padding:5px}.vis-labelset .vis-label .vis-inner.vis-hidden{padding:0}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImxhYmVsc2V0LmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxjQUdFLGVBR0YsQ0FFQSx1Q0FQRSxpQkFBa0IsQ0FJbEIscUJBV0YsQ0FSQSx5QkFFRSxNQUFPLENBQ1AsS0FBTSxDQUNOLFVBQVcsQ0FDWCxhQUFjLENBTWQsK0JBSEYsQ0FNQSxtQ0FDRSxjQUNGLENBRUEsdUJBQ0UseUJBQ0YsQ0FFQSxvQ0FDRSxrQkFDRixDQUVBLG9DQUNFLG9CQUFxQixDQUNyQixXQUNGLENBRUEsK0NBQ0UsU0FDRiIsImZpbGUiOiJsYWJlbHNldC5jc3MiLCJzb3VyY2VzQ29udGVudCI6WyJcbi52aXMtbGFiZWxzZXQge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG5cbiAgb3ZlcmZsb3c6IGhpZGRlbjtcblxuICBib3gtc2l6aW5nOiBib3JkZXItYm94O1xufVxuXG4udmlzLWxhYmVsc2V0IC52aXMtbGFiZWwge1xuICBwb3NpdGlvbjogcmVsYXRpdmU7XG4gIGxlZnQ6IDA7XG4gIHRvcDogMDtcbiAgd2lkdGg6IDEwMCU7XG4gIGNvbG9yOiAjNGQ0ZDRkO1xuXG4gIGJveC1zaXppbmc6IGJvcmRlci1ib3g7XG59XG5cbi52aXMtbGFiZWxzZXQgLnZpcy1sYWJlbCB7XG4gIGJvcmRlci1ib3R0b206IDFweCBzb2xpZCAjYmZiZmJmO1xufVxuXG4udmlzLWxhYmVsc2V0IC52aXMtbGFiZWwuZHJhZ2dhYmxlIHtcbiAgY3Vyc29yOiBwb2ludGVyO1xufVxuXG4udmlzLWdyb3VwLWlzLWRyYWdnaW5nIHtcbiAgYmFja2dyb3VuZDogcmdiYSgwLCAwLCAwLCAuMSk7XG59XG5cbi52aXMtbGFiZWxzZXQgLnZpcy1sYWJlbDpsYXN0LWNoaWxkIHtcbiAgYm9yZGVyLWJvdHRvbTogbm9uZTtcbn1cblxuLnZpcy1sYWJlbHNldCAudmlzLWxhYmVsIC52aXMtaW5uZXIge1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG4gIHBhZGRpbmc6IDVweDtcbn1cblxuLnZpcy1sYWJlbHNldCAudmlzLWxhYmVsIC52aXMtaW5uZXIudmlzLWhpZGRlbiB7XG4gIHBhZGRpbmc6IDA7XG59XG4iXX0= */");var du="__background__",uu=function(t){function e(t,i){var n;ba(this,e),(n=ar(this,dr(e).call(this))).body=t,n.defaultOptions={type:null,orientation:{item:"bottom"},align:"auto",stack:!0,stackSubgroups:!0,groupOrderSwap:function(t,e,i){var n=e.order;e.order=t.order,t.order=n},groupOrder:"order",selectable:!0,multiselect:!1,longSelectPressTime:251,itemsAlwaysDraggable:{item:!1,range:!1},editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1,overrideItems:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:Hh.snap,onDropObjectOnItem:function(t,e,i){i(e)},onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20},showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip",delay:500},tooltipOnItemUpdateTime:!1},n.options=za.extend({},n.defaultOptions),n.options.rtl=i.rtl,n.options.onTimeout=i.onTimeout,n.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},n.dom={},n.props={},n.hammer=null;var o=Bs(n);return n.itemsData=null,n.groupsData=null,n.itemsSettingTime=null,n.initialItemSetDrawn=!1,n.userContinueNotBail=null,n.sequentialSelection=!1,n.itemListeners={add:function(t,e,i){o._onAdd(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()},update:function(t,e,i){o._onUpdate(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()},remove:function(t,e,i){o._onRemove(e.items),o.options.cluster&&o.clusterGenerator.setItems(o.items,{applyOnChangedLevel:!1}),o.redraw()}},n.groupListeners={add:function(t,e,i){if(o._onAddGroups(e.items),o.groupsData&&o.groupsData.length>0){var n,s=o.groupsData.getDataSet();mn(n=s.get()).call(n,(function(t){if(t.nestedGroups){var e;0!=t.showNested&&(t.showNested=!0);var n=[];mn(e=t.nestedGroups).call(e,(function(e){var i=s.get(e);i&&(i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),n=Pn(n).call(n,i))})),s.update(n,i)}}))}},update:function(t,e,i){o._onUpdateGroups(e.items)},remove:function(t,e,i){o._onRemoveGroups(e.items)}},n.items={},n.groups={},n.groupIds=[],n.selection=[],n.popup=null,n.popupTimer=null,n.touchParams={},n.groupTouchParams={group:null,isDragging:!1},n._create(),n.setOptions(i),n.clusters=[],n}return pr(e,t),xa(e,[{key:"_create",value:function(){var t,e,i,n,o,s,r,a,l,h,d,u,c,p,m,f=this,g=document.createElement("div");g.className="vis-itemset",g["vis-itemset"]=this,this.dom.frame=g;var v=document.createElement("div");v.className="vis-background",g.appendChild(v),this.dom.background=v;var b=document.createElement("div");b.className="vis-foreground",g.appendChild(b),this.dom.foreground=b;var y=document.createElement("div");y.className="vis-axis",this.dom.axis=y;var x=document.createElement("div");x.className="vis-labelset",this.dom.labelSet=x,this._updateUngrouped();var w=new Hd(du,null,this);w.show(),this.groups[du]=w,this.hammer=new Vh(this.body.dom.centerContainer),this.hammer.on("hammer.input",(function(t){t.isFirst&&f._onTouch(t)})),this.hammer.on("panstart",va(t=this._onDragStart).call(t,this)),this.hammer.on("panmove",va(e=this._onDrag).call(e,this)),this.hammer.on("panend",va(i=this._onDragEnd).call(i,this)),this.hammer.get("pan").set({threshold:5,direction:Vh.ALL}),this.hammer.on("tap",va(n=this._onSelectItem).call(n,this)),this.hammer.on("press",va(o=this._onMultiSelectItem).call(o,this)),this.hammer.on("doubletap",va(s=this._onAddItem).call(s,this)),this.options.rtl?this.groupHammer=new Vh(this.body.dom.rightContainer):this.groupHammer=new Vh(this.body.dom.leftContainer),this.groupHammer.on("tap",va(r=this._onGroupClick).call(r,this)),this.groupHammer.on("panstart",va(a=this._onGroupDragStart).call(a,this)),this.groupHammer.on("panmove",va(l=this._onGroupDrag).call(l,this)),this.groupHammer.on("panend",va(h=this._onGroupDragEnd).call(h,this)),this.groupHammer.get("pan").set({threshold:5,direction:Vh.DIRECTION_VERTICAL}),this.body.dom.centerContainer.addEventListener("mouseover",va(d=this._onMouseOver).call(d,this)),this.body.dom.centerContainer.addEventListener("mouseout",va(u=this._onMouseOut).call(u,this)),this.body.dom.centerContainer.addEventListener("mousemove",va(c=this._onMouseMove).call(c,this)),this.body.dom.centerContainer.addEventListener("contextmenu",va(p=this._onDragEnd).call(p,this)),this.body.dom.centerContainer.addEventListener("mousewheel",va(m=this._onMouseWheel).call(m,this)),this.show()}},{key:"setOptions",value:function(t){var e=this;if(t){var i,n;za.selectiveExtend(["type","rtl","align","order","stack","stackSubgroups","selectable","multiselect","sequentialSelection","multiselectPerGroup","longSelectPressTime","groupOrder","dataAttributes","template","groupTemplate","visibleFrameTemplate","hide","snap","groupOrderSwap","showTooltips","tooltip","tooltipOnItemUpdateTime","groupHeightMode","onTimeout"],this.options,t),"itemsAlwaysDraggable"in t&&("boolean"==typeof t.itemsAlwaysDraggable?(this.options.itemsAlwaysDraggable.item=t.itemsAlwaysDraggable,this.options.itemsAlwaysDraggable.range=!1):"object"===zo(t.itemsAlwaysDraggable)&&(za.selectiveExtend(["item","range"],this.options.itemsAlwaysDraggable,t.itemsAlwaysDraggable),this.options.itemsAlwaysDraggable.item||(this.options.itemsAlwaysDraggable.range=!1))),"sequentialSelection"in t&&"boolean"==typeof t.sequentialSelection&&(this.options.sequentialSelection=t.sequentialSelection),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===zo(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===zo(t.margin)&&(za.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===zo(t.margin.item)&&za.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),mn(i=["locale","locales"]).call(i,(function(i){i in t&&(e.options[i]=t[i])})),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable,this.options.editable.overrideItems=!1):"object"===zo(t.editable)&&za.selectiveExtend(["updateTime","updateGroup","add","remove","overrideItems"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===zo(t.groupEditable)&&za.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));mn(n=["onDropObjectOnItem","onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"]).call(n,(function(i){var n=t[i];if(n){var o;if("function"!=typeof n)throw new Error(Pn(o="option ".concat(i," must be a function ")).call(o,i,"(item, callback)"));e.options[i]=n}})),t.cluster?(Ao(this.options,{cluster:t.cluster}),this.clusterGenerator||(this.clusterGenerator=new hu(this)),this.clusterGenerator.setItems(this.items,{applyOnChangedLevel:!1}),this.markDirty({refreshItems:!0,restackGroups:!0}),this.redraw()):this.clusterGenerator?(this._detachAllClusters(),this.clusters=[],this.clusterGenerator=null,this.options.cluster=void 0,this.markDirty({refreshItems:!0,restackGroups:!0}),this.redraw()):this.markDirty()}}},{key:"markDirty",value:function(t){this.groupIds=[],t&&(t.refreshItems&&mn(za).call(za,this.items,(function(t){t.dirty=!0,t.displayed&&t.redraw()})),t.restackGroups&&mn(za).call(za,this.groups,(function(t,e){e!==du&&(t.stackDirty=!0)})))}},{key:"destroy",value:function(){this.clearPopupTimer(),this.hide(),this.setItems(null),this.setGroups(null),this.hammer&&this.hammer.destroy(),this.groupHammer&&this.groupHammer.destroy(),this.hammer=null,this.body=null,this.conversion=null}},{key:"hide",value:function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)}},{key:"show",value:function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||(this.options.rtl?this.body.dom.right.appendChild(this.dom.labelSet):this.body.dom.left.appendChild(this.dom.labelSet))}},{key:"setPopupTimer",value:function(t){if(this.clearPopupTimer(),t){var e=this.options.tooltip.delay||"number"==typeof this.options.tooltip.delay?this.options.tooltip.delay:500;this.popupTimer=qr((function(){t.show()}),e)}}},{key:"clearPopupTimer",value:function(){null!=this.popupTimer&&(clearTimeout(this.popupTimer),this.popupTimer=null)}},{key:"setSelection",value:function(t){var e;null==t&&(t=[]),Zo(t)||(t=[t]);var i=En(e=this.selection).call(e,(function(e){return-1===Fo(t).call(t,e)})),n=!0,o=!1,s=void 0;try{for(var r,a=Fi(i);!(n=(r=a.next()).done);n=!0){var l=r.value,h=this.getItemById(l);h&&h.unselect()}}catch(t){o=!0,s=t}finally{try{n||null==a.return||a.return()}finally{if(o)throw s}}this.selection=mo(t);var d=!0,u=!1,c=void 0;try{for(var p,m=Fi(t);!(d=(p=m.next()).done);d=!0){var f=p.value,g=this.getItemById(f);g&&g.select()}}catch(t){u=!0,c=t}finally{try{d||null==m.return||m.return()}finally{if(u)throw c}}}},{key:"getSelection",value:function(){var t;return Pn(t=this.selection).call(t,[])}},{key:"getVisibleItems",value:function(){var t,e,i=this.body.range.getRange();this.options.rtl?(t=this.body.util.toScreen(i.start),e=this.body.util.toScreen(i.end)):(e=this.body.util.toScreen(i.start),t=this.body.util.toScreen(i.end));var n=[];for(var o in this.groups)if(this.groups.hasOwnProperty(o)){var s=this.groups[o],r=s.isVisible?s.visibleItems:[],a=!0,l=!1,h=void 0;try{for(var d,u=Fi(r);!(a=(d=u.next()).done);a=!0){var c=d.value;this.options.rtl?c.right<e&&c.right+c.width>t&&n.push(c.id):c.left<t&&c.left+c.width>e&&n.push(c.id)}}catch(t){l=!0,h=t}finally{try{a||null==u.return||u.return()}finally{if(l)throw h}}}return n}},{key:"getVisibleGroups",value:function(){var t=[];for(var e in this.groups){if(this.groups.hasOwnProperty(e))this.groups[e].isVisible&&t.push(e)}return t}},{key:"getItemById",value:function(t){var e;return this.items[t]||yd(e=this.clusters).call(e,(function(e){return e.id===t}))}},{key:"_deselect",value:function(t){for(var e=this.selection,i=0,n=e.length;i<n;i++)if(e[i]==t){Hr(e).call(e,i,1);break}}},{key:"redraw",value:function(){var t=this,e=this.options.margin,i=this.body.range,n=za.option.asSize,o=this.options,s=o.orientation.item,r=!1,a=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,a.className="vis-itemset",this.options.cluster&&this._clusterItems(),r=this._orderGroups()||r;var l=i.end-i.start,h=l!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth,d=i.start!=this.lastRangeStart,u=o.stack!=this.lastStack,c=o.stackSubgroups!=this.lastStackSubgroups,p=h||d||u||c;this.lastVisibleInterval=l,this.lastRangeStart=i.start,this.lastStack=o.stack,this.lastStackSubgroups=o.stackSubgroups,this.props.lastWidth=this.props.width;var m=this._firstGroup(),f={item:e.item,axis:e.axis},g={item:e.item,axis:e.item.vertical/2},v=0,b=e.axis+e.item.vertical;this.groups[du].redraw(i,g,p);var y={},x=0;return mn(za).call(za,this.groups,(function(t,e){if(e!==du){var n=t==m?f:g;y[e]=t.redraw(i,n,p,!0),x=y[e].length}})),x>0&&function(){for(var e={},i=function(t){mn(za).call(za,y,(function(i,n){e[n]=i[t]()}))},n=0;n<x;n++)i(n);mn(za).call(za,t.groups,(function(t,i){if(i!==du){var n=e[i];r=n||r,v+=t.height}})),v=Math.max(v,b)}(),v=Math.max(v,b),a.style.height=n(v),this.props.width=a.offsetWidth,this.props.height=v,this.dom.axis.style.top=n("top"==s?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",this.hammer.get("press").set({time:this.options.longSelectPressTime}),this.initialItemSetDrawn=!0,r=this._isResized()||r}},{key:"_firstGroup",value:function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t];return this.groups[e]||this.groups.__ungrouped__||null}},{key:"_updateUngrouped",value:function(){var t,e,i=this.groups.__ungrouped__;if(this.groupsData){if(i)for(e in i.dispose(),delete this.groups.__ungrouped__,this.items)if(this.items.hasOwnProperty(e)){(t=this.items[e]).parent&&t.parent.remove(t);var n=this.getGroupId(t.data),o=this.groups[n];o&&o.add(t)||t.hide()}}else if(!i){for(e in i=new jd(null,null,this),this.groups.__ungrouped__=i,this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}}},{key:"getLabelSet",value:function(){return this.dom.labelSet}},{key:"setItems",value:function(t){this.itemsSettingTime=new Date;var e,i=this,n=this.itemsData;if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=Ua(t)}else this.itemsData=null;if(n&&(mn(za).call(za,this.itemListeners,(function(t,e){n.off(e,t)})),n.dispose(),e=n.getIds(),this._onRemove(e)),this.itemsData){var o=this.id;mn(za).call(za,this.itemListeners,(function(t,e){i.itemsData.on(e,t,o)})),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})}},{key:"getItems",value:function(){return null!=this.itemsData?this.itemsData.rawDS:null}},{key:"setGroups",value:function(t){var e,i=this;if(this.groupsData&&(mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.off(e,t)})),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n,o=this.groupsData;this.groupsData instanceof Xa&&(o=this.groupsData.getDataSet()),mn(n=o.get()).call(n,(function(t){var e;t.nestedGroups&&mn(e=t.nestedGroups).call(e,(function(e){var i=o.get(e);i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),o.update(i)}))}));var s=this.id;mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.on(e,t,s)})),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.options.cluster&&(this.clusterGenerator.updateData(),this._clusterItems(),this.markDirty({refreshItems:!0,restackGroups:!0})),this.body.emitter.emit("_change",{queue:!0})}},{key:"getGroups",value:function(){return this.groupsData}},{key:"removeItem",value:function(t){var e=this,i=this.itemsData.get(t);i&&this.options.onRemove(i,(function(i){i&&e.itemsData.remove(t)}))}},{key:"_getType",value:function(t){return t.type||this.options.type||(t.end?"range":"box")}},{key:"getGroupId",value:function(t){return"background"==this._getType(t)&&null==t.group?du:this.groupsData?t.group:"__ungrouped__"}},{key:"_onUpdate",value:function(t){var i=this,n=this;mn(t).call(t,(function(t){var o,s=n.itemsData.get(t),r=n.items[t],a=s?n._getType(s):null,l=e.types[a];if(r&&(l&&r instanceof l?n._updateItem(r,s):(o=r.selected,n._removeItem(r),r=null)),!r&&s){if(!l)throw new TypeError('Unknown item type "'.concat(a,'"'));(r=new l(s,n.conversion,n.options)).id=t,n._addItem(r),o&&(i.selection.push(t),r.select())}})),this._order(),this.options.cluster&&(this.clusterGenerator.setItems(this.items,{applyOnChangedLevel:!1}),this._clusterItems()),this.body.emitter.emit("_change",{queue:!0})}},{key:"_onRemove",value:function(t){var e=0,i=this;mn(t).call(t,(function(t){var n=i.items[t];n&&(e++,i._removeItem(n))})),e&&(this._order(),this.body.emitter.emit("_change",{queue:!0}))}},{key:"_order",value:function(){mn(za).call(za,this.groups,(function(t){t.order()}))}},{key:"_onUpdateGroups",value:function(t){this._onAddGroups(t)}},{key:"_onAddGroups",value:function(t){var e=this;mn(t).call(t,(function(t){var i=e.groupsData.get(t),n=e.groups[t];if(n)n.setData(i);else{if("__ungrouped__"==t||t==du)throw new Error("Illegal group id. ".concat(t," is a reserved id."));var o=Oi(e.options);for(var s in za.extend(o,{height:null}),n=new jd(t,i,e),e.groups[t]=n,e.items)if(e.items.hasOwnProperty(s)){var r=e.items[s];r.data.group==t&&n.add(r)}n.order(),n.show()}})),this.body.emitter.emit("_change",{queue:!0})}},{key:"_onRemoveGroups",value:function(t){var e=this;mn(t).call(t,(function(t){var i=e.groups[t];i&&(i.dispose(),delete e.groups[t])})),this.options.cluster&&(this.clusterGenerator.updateData(),this._clusterItems()),this.markDirty({restackGroups:!!this.options.cluster}),this.body.emitter.emit("_change",{queue:!0})}},{key:"_orderGroups",value:function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder});t=this._orderNestedGroups(t);var e=!za.equalArray(t,this.groupIds);if(e){var i=this.groups;mn(t).call(t,(function(t){i[t].hide()})),mn(t).call(t,(function(t){i[t].show()})),this.groupIds=t}return e}return!1}},{key:"_orderNestedGroups",value:function(t){var e=this;return function t(e,i){var n=[];return mn(i).call(i,(function(i){if(n.push(i),e.groupsData.get(i).nestedGroups){var o,s=Ln(o=e.groupsData.get({filter:function(t){return t.nestedInGroup==i},order:e.options.groupOrder})).call(o,(function(t){return t.id}));n=Pn(n).call(n,t(e,s))}})),n}(this,En(t).call(t,(function(t){return!e.groupsData.get(t).nestedInGroup})))}},{key:"_addItem",value:function(t){this.items[t.id]=t;var e=this.getGroupId(t.data),i=this.groups[e];i?i&&i.data&&i.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1,i&&i.add(t)}},{key:"_updateItem",value:function(t,e){t.setData(e);var i=this.getGroupId(t.data),n=this.groups[i];n?n&&n.data&&n.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1}},{key:"_removeItem",value:function(t){var e,i;t.hide(),delete this.items[t.id];var n=Fo(e=this.selection).call(e,t.id);-1!=n&&Hr(i=this.selection).call(i,n,1),t.parent&&t.parent.remove(t),null!=this.popup&&this.popup.hide()}},{key:"_constructByEndArray",value:function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof qd&&e.push(t[i]);return e}},{key:"_onTouch",value:function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null}},{key:"_getGroupIndex",value:function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e}},{key:"_onDragStart",value:function(t){var e=this;if(!this.touchParams.itemIsDragging){var i,n=this.touchParams.item||null,o=this;if(n&&(n.selected||this.options.itemsAlwaysDraggable.item)){if(this.options.editable.overrideItems&&!this.options.editable.updateTime&&!this.options.editable.updateGroup)return;if(null!=n.editable&&!n.editable.updateTime&&!n.editable.updateGroup&&!this.options.editable.overrideItems)return;var s=this.touchParams.dragLeftItem,r=this.touchParams.dragRightItem;if(this.touchParams.itemIsDragging=!0,this.touchParams.selectedItem=n,s)i={item:s,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(n.data)},this.touchParams.itemProps=[i];else if(r)i={item:r,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(n.data)},this.touchParams.itemProps=[i];else if(this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey))this._onDragStartAddItem(t);else{this.groupIds.length<1&&this.redraw();var a=this._getGroupIndex(n.data.group),l=this.options.itemsAlwaysDraggable.item&&!n.selected?[n.id]:this.getSelection();this.touchParams.itemProps=Ln(l).call(l,(function(i){var n=o.items[i],s=o._getGroupIndex(n.data.group);return{item:n,initialX:t.center.x,groupOffset:a-s,data:e._cloneItemData(n.data)}}))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)}}},{key:"_onDragStartAddItem",value:function(t){var e=this.options.snap||null,i=this.dom.frame.getBoundingClientRect(),n=this.options.rtl?i.right-t.center.x+10:t.center.x-i.left-10,o=this.body.util.toTime(n),s=this.body.util.getScale(),r=this.body.util.getStep(),a=e?e(o,s,r):o,l={type:"range",start:a,end:a,content:"new item"},h=Id();l[this.itemsData.idProp]=h;var d=this.groupFromTarget(t);d&&(l.group=d.groupId);var u=new qd(l,this.conversion,this.options);u.id=h,u.data=this._cloneItemData(l),this._addItem(u),this.touchParams.selectedItem=u;var c={item:u,initialX:t.center.x,data:u.data};this.options.rtl?c.dragLeft=!0:c.dragRight=!0,this.touchParams.itemProps=[c],t.stopPropagation()}},{key:"_onDrag",value:function(t){var e=this;if(null!=this.popup&&this.options.showTooltips&&!this.popup.hidden){var i=this.body.dom.centerContainer,n=i.getBoundingClientRect();this.popup.setPosition(t.center.x-n.left+i.offsetLeft,t.center.y-n.top+i.offsetTop),this.popup.show()}if(this.touchParams.itemProps){var o;t.stopPropagation();var s=this,r=this.options.snap||null,a=this.body.dom.root.offsetLeft,l=this.options.rtl?a+this.body.domProps.right.width:a+this.body.domProps.left.width,h=this.body.util.getScale(),d=this.body.util.getStep(),u=this.touchParams.selectedItem,c=(this.options.editable.overrideItems||null==u.editable)&&this.options.editable.updateGroup||!this.options.editable.overrideItems&&null!=u.editable&&u.editable.updateGroup,p=null;if(c&&u&&null!=u.data.group){var m=s.groupFromTarget(t);m&&(p=this._getGroupIndex(m.groupId))}mn(o=this.touchParams.itemProps).call(o,(function(i){var n,o,a,m,f,g=s.body.util.toTime(t.center.x-l),v=s.body.util.toTime(i.initialX-l);n=e.options.rtl?-(g-v):g-v;var b=e._cloneItemData(i.item.data);if(null==i.item.editable||i.item.editable.updateTime||i.item.editable.updateGroup||s.options.editable.overrideItems){if((e.options.editable.overrideItems||null==u.editable)&&e.options.editable.updateTime||!e.options.editable.overrideItems&&null!=u.editable&&u.editable.updateTime)if(i.dragLeft)e.options.rtl?null!=b.end&&(a=za.convert(i.data.end,"Date"),f=new Date(a.valueOf()+n),b.end=r?r(f,h,d):f):null!=b.start&&(o=za.convert(i.data.start,"Date"),m=new Date(o.valueOf()+n),b.start=r?r(m,h,d):m);else if(i.dragRight)e.options.rtl?null!=b.start&&(o=za.convert(i.data.start,"Date"),m=new Date(o.valueOf()+n),b.start=r?r(m,h,d):m):null!=b.end&&(a=za.convert(i.data.end,"Date"),f=new Date(a.valueOf()+n),b.end=r?r(f,h,d):f);else if(null!=b.start)if(o=za.convert(i.data.start,"Date").valueOf(),m=new Date(o+n),null!=b.end){var y=(a=za.convert(i.data.end,"Date")).valueOf()-o.valueOf();b.start=r?r(m,h,d):m,b.end=new Date(b.start.valueOf()+y)}else b.start=r?r(m,h,d):m;if(c&&!i.dragLeft&&!i.dragRight&&null!=p&&null!=b.group){var x=p-i.groupOffset;x=Math.max(0,x),x=Math.min(s.groupIds.length-1,x),b.group=s.groupIds[x]}b=e._cloneItemData(b),s.options.onMoving(b,(function(t){t&&i.item.setData(e._cloneItemData(t,"Date"))}))}})),this.body.emitter.emit("_change")}}},{key:"_moveToGroup",value:function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var n=t.parent;n.remove(t),n.order(),t.data.group=i.groupId,i.add(t),i.order()}}},{key:"_onDragEnd",value:function(t){var e=this;if(this.touchParams.itemIsDragging=!1,this.touchParams.itemProps){t.stopPropagation();var i=this,n=this.touchParams.itemProps;this.touchParams.itemProps=null,mn(n).call(n,(function(t){var n=t.item.id;if(null!=i.itemsData.get(n)){var o=e._cloneItemData(t.item.data);i.options.onMove(o,(function(o){o?(o[e.itemsData.idProp]=n,e.itemsData.update(o)):(t.item.setData(t.data),i.body.emitter.emit("_change"))}))}else i.options.onAdd(t.item.data,(function(e){i._removeItem(t.item),e&&i.itemsData.add(e),i.body.emitter.emit("_change")}))}))}}},{key:"_onGroupClick",value:function(t){var e=this,i=this.groupFromTarget(t);qr((function(){e.toggleGroupShowNested(i)}),1)}},{key:"toggleGroupShowNested",value:function(t){var e,i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:void 0;if(t&&t.nestedGroups){var n=this.groupsData.getDataSet();t.showNested=null!=i?!!i:!t.showNested;var o=n.get(t.groupId);o.showNested=t.showNested;for(var s=t.nestedGroups,r=s;r.length>0;){var a=r;r=[];for(var l=0;l<a.length;l++){var h=n.get(a[l]);h.nestedGroups&&(r=Pn(r).call(r,h.nestedGroups))}r.length>0&&(s=Pn(s).call(s,r))}var d=Ln(e=n.get(s)).call(e,(function(t){return null==t.visible&&(t.visible=!0),t.visible=!!o.showNested,t}));n.update(Pn(d).call(d,o)),o.showNested?(za.removeClassName(t.dom.label,"collapsed"),za.addClassName(t.dom.label,"expanded")):(za.removeClassName(t.dom.label,"expanded"),za.addClassName(t.dom.label,"collapsed"))}}},{key:"toggleGroupDragClassName",value:function(t){t.dom.label.classList.toggle("vis-group-is-dragging"),t.dom.foreground.classList.toggle("vis-group-is-dragging")}},{key:"_onGroupDragStart",value:function(t){this.groupTouchParams.isDragging||this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.isDragging=!0,this.toggleGroupDragClassName(this.groupTouchParams.group),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))}},{key:"_onGroupDrag",value:function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupsData;this.groupsData instanceof Xa&&(e=this.groupsData.getDataSet());var i=this.groupFromTarget(t);if(i&&i.height!=this.groupTouchParams.group.height){var n=i.top<this.groupTouchParams.group.top,o=t.center?t.center.y:t.clientY,s=i.dom.foreground.getBoundingClientRect(),r=this.groupTouchParams.group.height;if(n){if(s.top+r<o)return}else{var a=i.height;if(s.top+a-r>o)return}}if(i&&i!=this.groupTouchParams.group){var l=e.get(i.groupId),h=e.get(this.groupTouchParams.group.groupId);h&&l&&(this.options.groupOrderSwap(h,l,e),e.update(h),e.update(l));var d=e.getIds({order:this.options.groupOrder});if(!za.equalArray(d,this.groupTouchParams.originalOrder))for(var u=this.groupTouchParams.originalOrder,c=this.groupTouchParams.group.groupId,p=Math.min(u.length,d.length),m=0,f=0,g=0;m<p;){for(;m+f<p&&m+g<p&&d[m+f]==u[m+g];)m++;if(m+f>=p)break;if(d[m+f]==c)f=1;else if(u[m+g]==c)g=1;else{var v=Fo(d).call(d,u[m+g]),b=e.get(d[m+f]),y=e.get(u[m+g]);this.options.groupOrderSwap(b,y,e),e.update(b),e.update(y);var x=d[m+f];d[m+f]=u[m+g],d[v]=x,m++}}}}}},{key:"_onGroupDragEnd",value:function(t){if(this.groupTouchParams.isDragging=!1,this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,n=e.groupsData.getDataSet(),o=za.extend({},n.get(i));e.options.onMoveGroup(o,(function(t){if(t)t[n._idProp]=i,n.update(t);else{var o=n.getIds({order:e.options.groupOrder});if(!za.equalArray(o,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,r=Math.min(s.length,o.length),a=0;a<r;){for(;a<r&&o[a]==s[a];)a++;if(a>=r)break;var l=Fo(o).call(o,s[a]),h=n.get(o[a]),d=n.get(s[a]);e.options.groupOrderSwap(h,d,n),n.update(h),n.update(d);var u=o[a];o[a]=s[a],o[l]=u,a++}}})),e.body.emitter.emit("groupDragged",{groupId:i}),this.toggleGroupDragClassName(this.groupTouchParams.group),this.groupTouchParams.group=null}}},{key:"_onSelectItem",value:function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)this._onMultiSelectItem(t);else{var n=this.getSelection(),o=this.itemFromTarget(t),s=o&&o.selectable?[o.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||n.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}}}},{key:"_onMouseOver",value:function(t){var e=this.itemFromTarget(t);if(e&&e!==this.itemFromRelatedTarget(t)){var i=e.getTitle();if(this.options.showTooltips&&i){null==this.popup&&(this.popup=new tu(this.body.dom.root,this.options.tooltip.overflowMethod||"flip")),this.popup.setText(i);var n=this.body.dom.centerContainer,o=n.getBoundingClientRect();this.popup.setPosition(t.clientX-o.left+n.offsetLeft,t.clientY-o.top+n.offsetTop),this.setPopupTimer(this.popup)}else this.clearPopupTimer(),null!=this.popup&&this.popup.hide();this.body.emitter.emit("itemover",{item:e.id,event:t})}}},{key:"_onMouseOut",value:function(t){var e=this.itemFromTarget(t);e&&(e!==this.itemFromRelatedTarget(t)&&(this.clearPopupTimer(),null!=this.popup&&this.popup.hide(),this.body.emitter.emit("itemout",{item:e.id,event:t})))}},{key:"_onMouseMove",value:function(t){if(this.itemFromTarget(t)&&(null!=this.popupTimer&&this.setPopupTimer(this.popup),this.options.showTooltips&&this.options.tooltip.followMouse&&this.popup&&!this.popup.hidden)){var e=this.body.dom.centerContainer,i=e.getBoundingClientRect();this.popup.setPosition(t.clientX-i.left+e.offsetLeft,t.clientY-i.top+e.offsetTop),this.popup.show()}}},{key:"_onMouseWheel",value:function(t){this.touchParams.itemIsDragging&&this._onDragEnd(t)}},{key:"_onUpdateItem",value:function(t){if(this.options.selectable&&(this.options.editable.updateTime||this.options.editable.updateGroup)){var e=this;if(t){var i=e.itemsData.get(t.id);this.options.onUpdate(i,(function(t){t&&e.itemsData.update(t)}))}}}},{key:"_onDropObjectOnItem",value:function(t){var e=this.itemFromTarget(t),i=JSON.parse(t.dataTransfer.getData("text"));this.options.onDropObjectOnItem(i,e)}},{key:"_onAddItem",value:function(t){if(this.options.selectable&&this.options.editable.add){var e,i,n=this,o=this.options.snap||null,s=this.dom.frame.getBoundingClientRect(),r=this.options.rtl?s.right-t.center.x:t.center.x-s.left,a=this.body.util.toTime(r),l=this.body.util.getScale(),h=this.body.util.getStep();"drop"==t.type?((i=JSON.parse(t.dataTransfer.getData("text"))).content=i.content?i.content:"new item",i.start=i.start?i.start:o?o(a,l,h):a,i.type=i.type||"box",i[this.itemsData.idProp]=i.id||Id(),"range"!=i.type||i.end||(e=this.body.util.toTime(r+this.props.width/5),i.end=o?o(e,l,h):e)):((i={start:o?o(a,l,h):a,content:"new item"})[this.itemsData.idProp]=Id(),"range"===this.options.type&&(e=this.body.util.toTime(r+this.props.width/5),i.end=o?o(e,l,h):e));var d=this.groupFromTarget(t);d&&(i.group=d.groupId),i=this._cloneItemData(i),this.options.onAdd(i,(function(e){e&&(n.itemsData.add(e),"drop"==t.type&&n.setSelection([e.id]))}))}}},{key:"_onMultiSelectItem",value:function(t){var i=this;if(this.options.selectable){var n=this.itemFromTarget(t);if(n){var o=this.options.multiselect?this.getSelection():[];if((t.srcEvent&&t.srcEvent.shiftKey||!1||this.options.sequentialSelection)&&this.options.multiselect){var s=this.itemsData.get(n.id).group,r=void 0;this.options.multiselectPerGroup&&o.length>0&&(r=this.itemsData.get(o[0]).group),this.options.multiselectPerGroup&&null!=r&&r!=s||o.push(n.id);var a=e._getItemRange(this.itemsData.get(o));if(!this.options.multiselectPerGroup||r==s)for(var l in o=[],this.items)if(this.items.hasOwnProperty(l)){var h=this.items[l],d=h.data.start,u=void 0!==h.data.end?h.data.end:d;!(d>=a.min&&u<=a.max)||this.options.multiselectPerGroup&&r!=this.itemsData.get(h.id).group||h instanceof $d||o.push(h.id)}}else{var c=Fo(o).call(o,n.id);-1==c?o.push(n.id):Hr(o).call(o,c,1)}var p=En(o).call(o,(function(t){return i.getItemById(t).selectable}));this.setSelection(p),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}}},{key:"itemFromElement",value:function(t){for(var e=t;e;){if(e.hasOwnProperty("vis-item"))return e["vis-item"];e=e.parentNode}return null}},{key:"itemFromTarget",value:function(t){return this.itemFromElement(t.target)}},{key:"itemFromRelatedTarget",value:function(t){return this.itemFromElement(t.relatedTarget)}},{key:"groupFromTarget",value:function(t){var e=t.center?t.center.y:t.clientY,i=this.groupIds;i.length<=0&&this.groupsData&&(i=this.groupsData.getIds({order:this.options.groupOrder}));for(var n=0;n<i.length;n++){var o=i[n],s=this.groups[o],r=s.dom.foreground,a=r.getBoundingClientRect();if(e>=a.top&&e<a.top+r.offsetHeight)return s;if("top"===this.options.orientation.item){if(n===this.groupIds.length-1&&e>a.top)return s}else if(0===n&&e<a.top+r.offset)return s}return null}},{key:"_cloneItemData",value:function(t,e){var i=za.extend({},t);return e||(e=this.itemsData.type),null!=i.start&&(i.start=za.convert(i.start,e&&e.start||"Date")),null!=i.end&&(i.end=za.convert(i.end,e&&e.end||"Date")),i}},{key:"_clusterItems",value:function(){if(this.options.cluster){var t=this.body.range.conversion(this.body.domProps.center.width).scale,e=this.clusterGenerator.getClusters(this.clusters,t,this.options.cluster);if(this.clusters!=e){if(this._detachAllClusters(),e){var i=!0,n=!1,o=void 0;try{for(var s,r=Fi(e);!(i=(s=r.next()).done);i=!0){s.value.attach()}}catch(t){n=!0,o=t}finally{try{i||null==r.return||r.return()}finally{if(n)throw o}}this.clusters=e}this._updateClusters(e)}}}},{key:"_detachAllClusters",value:function(){if(this.options.cluster&&this.clusters&&this.clusters.length){var t=!0,e=!1,i=void 0;try{for(var n,o=Fi(this.clusters);!(t=(n=o.next()).done);t=!0){n.value.detach()}}catch(t){e=!0,i=t}finally{try{t||null==o.return||o.return()}finally{if(e)throw i}}}}},{key:"_updateClusters",value:function(t){if(this.clusters&&this.clusters.length){var e,i=new Ar(Ln(t).call(t,(function(t){return t.id}))),n=En(e=this.clusters).call(e,(function(t){return!i.has(t.id)})),o=!1,s=!0,r=!1,a=void 0;try{for(var l,h=Fi(n);!(s=(l=h.next()).done);s=!0){var d,u,c=l.value,p=Fo(d=this.selection).call(d,c.id);if(-1!==p)c.unselect(),Hr(u=this.selection).call(u,p,1),o=!0}}catch(t){r=!0,a=t}finally{try{s||null==h.return||h.return()}finally{if(r)throw a}}if(o){var m=this.getSelection();this.body.emitter.emit("select",{items:m,event:event})}}this.clusters=t||[]}}],[{key:"_getItemRange",value:function(t){var e=null,i=null;return mn(t).call(t,(function(t){(null==i||t.start<i)&&(i=t.start),null!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)})),{min:i,max:e}}},{key:"itemSetFromTarget",value:function(t){for(var e=t.target;e;){if(e.hasOwnProperty("vis-itemset"))return e["vis-itemset"];e=e.parentNode}return null}}]),e}(Ja);uu.types={background:$d,box:Jd,range:qd,point:Kd},uu.prototype._onAdd=uu.prototype._onUpdate;var cu,pu=!1,mu="background: #FFeeee; color: #dd0000",fu=function(){function t(){ba(this,t)}return xa(t,null,[{key:"validate",value:function(e,i,n){pu=!1,cu=i;var o=i;return void 0!==n&&(o=i[n]),t.parse(e,o,[]),pu}},{key:"parse",value:function(e,i,n){for(var o in e)e.hasOwnProperty(o)&&t.check(o,e,i,n)}},{key:"check",value:function(e,i,n,o){if(void 0!==n[e]||void 0!==n.__any__){var s=e,r=!0;void 0===n[e]&&void 0!==n.__any__&&(s="__any__",r="object"===t.getType(i[e]));var a=n[s];r&&void 0!==a.__type__&&(a=a.__type__),t.checkFields(e,i,n,s,a,o)}else t.getSuggestion(e,n,o)}},{key:"checkFields",value:function(e,i,n,o,s,r){var a=function(i){console.log("%c"+i+t.printLocation(r,e),mu)},l=t.getType(i[e]),h=s[l];void 0!==h?"array"===t.getType(h)&&-1===Fo(h).call(h,i[e])?(a('Invalid option detected in "'+e+'". Allowed values are:'+t.print(h)+' not "'+i[e]+'". '),pu=!0):"object"===l&&"__any__"!==o&&(r=za.copyAndExtendArray(r,e),t.parse(i[e],n[o],r)):void 0===s.any&&(a('Invalid type received for "'+e+'". Expected: '+t.print(Yi(s))+". Received ["+l+'] "'+i[e]+'"'),pu=!0)}},{key:"getType",value:function(t){var e=zo(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Zo(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":!0===t._isAMomentObject?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,n){var o,s=t.findInOptions(e,i,n,!1),r=t.findInOptions(e,cu,[],!0);o=void 0!==s.indexMatch?" in "+t.printLocation(s.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+s.indexMatch+'"?\n\n':r.distance<=4&&s.distance>r.distance?" in "+t.printLocation(s.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(r.path,r.closestMatch,""):s.distance<=8?'. Did you mean "'+s.closestMatch+'"?'+t.printLocation(s.path,e):". Did you mean one of these: "+t.print(Yi(i))+t.printLocation(n,e),console.log('%cUnknown option detected: "'+e+'"'+o,mu),pu=!0}},{key:"findInOptions",value:function(e,i,n){var o=arguments.length>3&&void 0!==arguments[3]&&arguments[3],s=1e9,r="",a=[],l=e.toLowerCase(),h=void 0;for(var d in i){var u=void 0;if(void 0!==i[d].__type__&&!0===o){var c=t.findInOptions(e,i[d],za.copyAndExtendArray(n,d));s>c.distance&&(r=c.closestMatch,a=c.path,s=c.distance,h=c.indexMatch)}else{var p;-1!==Fo(p=d.toLowerCase()).call(p,l)&&(h=d),s>(u=t.levenshteinDistance(e,d))&&(r=d,a=za.copyArray(n),s=u)}}return{closestMatch:r,path:a,distance:s,indexMatch:h}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"Problem value found at: \n",n="\n\n"+i+"options = {\n",o=0;o<t.length;o++){for(var s=0;s<o+1;s++)n+="  ";n+=t[o]+": {\n"}for(var r=0;r<t.length+1;r++)n+="  ";n+=e+"\n";for(var a=0;a<t.length+1;a++){for(var l=0;l<t.length-a;l++)n+="  ";n+="}\n"}return n+"\n\n"}},{key:"print",value:function(t){return Vs(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,n,o=[];for(i=0;i<=e.length;i++)o[i]=[i];for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}(),gu="string",vu="boolean",bu="number",yu="object",xu={configure:{enabled:{boolean:vu},filter:{boolean:vu,function:"function"},container:{dom:"dom"},__type__:{object:yu,boolean:vu,function:"function"}},align:{string:gu},alignCurrentTime:{string:gu,undefined:"undefined"},rtl:{boolean:vu,undefined:"undefined"},rollingMode:{follow:{boolean:vu},offset:{number:bu,undefined:"undefined"},__type__:{object:yu}},onTimeout:{timeoutMs:{number:bu},callback:{function:"function"},__type__:{object:yu}},verticalScroll:{boolean:vu,undefined:"undefined"},horizontalScroll:{boolean:vu,undefined:"undefined"},autoResize:{boolean:vu},throttleRedraw:{number:bu},clickToUse:{boolean:vu},dataAttributes:{string:gu,array:"array"},editable:{add:{boolean:vu,undefined:"undefined"},remove:{boolean:vu,undefined:"undefined"},updateGroup:{boolean:vu,undefined:"undefined"},updateTime:{boolean:vu,undefined:"undefined"},overrideItems:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},end:{number:bu,date:"date",string:gu,moment:"moment"},format:{minorLabels:{millisecond:{string:gu,undefined:"undefined"},second:{string:gu,undefined:"undefined"},minute:{string:gu,undefined:"undefined"},hour:{string:gu,undefined:"undefined"},weekday:{string:gu,undefined:"undefined"},day:{string:gu,undefined:"undefined"},week:{string:gu,undefined:"undefined"},month:{string:gu,undefined:"undefined"},year:{string:gu,undefined:"undefined"},__type__:{object:yu,function:"function"}},majorLabels:{millisecond:{string:gu,undefined:"undefined"},second:{string:gu,undefined:"undefined"},minute:{string:gu,undefined:"undefined"},hour:{string:gu,undefined:"undefined"},weekday:{string:gu,undefined:"undefined"},day:{string:gu,undefined:"undefined"},week:{string:gu,undefined:"undefined"},month:{string:gu,undefined:"undefined"},year:{string:gu,undefined:"undefined"},__type__:{object:yu,function:"function"}},__type__:{object:yu}},moment:{function:"function"},groupHeightMode:{string:gu},groupOrder:{string:gu,function:"function"},groupEditable:{add:{boolean:vu,undefined:"undefined"},remove:{boolean:vu,undefined:"undefined"},order:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},groupOrderSwap:{function:"function"},height:{string:gu,number:bu},hiddenDates:{start:{date:"date",number:bu,string:gu,moment:"moment"},end:{date:"date",number:bu,string:gu,moment:"moment"},repeat:{string:gu},__type__:{object:yu,array:"array"}},itemsAlwaysDraggable:{item:{boolean:vu,undefined:"undefined"},range:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},limitSize:{boolean:vu},locale:{string:gu},locales:{__any__:{any:"any"},__type__:{object:yu}},longSelectPressTime:{number:bu},margin:{axis:{number:bu},item:{horizontal:{number:bu,undefined:"undefined"},vertical:{number:bu,undefined:"undefined"},__type__:{object:yu,number:bu}},__type__:{object:yu,number:bu}},max:{date:"date",number:bu,string:gu,moment:"moment"},maxHeight:{number:bu,string:gu},maxMinorChars:{number:bu},min:{date:"date",number:bu,string:gu,moment:"moment"},minHeight:{number:bu,string:gu},moveable:{boolean:vu},multiselect:{boolean:vu},multiselectPerGroup:{boolean:vu},onAdd:{function:"function"},onDropObjectOnItem:{function:"function"},onUpdate:{function:"function"},onMove:{function:"function"},onMoving:{function:"function"},onRemove:{function:"function"},onAddGroup:{function:"function"},onMoveGroup:{function:"function"},onRemoveGroup:{function:"function"},onInitialDrawComplete:{function:"function"},order:{function:"function"},orientation:{axis:{string:gu,undefined:"undefined"},item:{string:gu,undefined:"undefined"},__type__:{string:gu,object:yu}},selectable:{boolean:vu},sequentialSelection:{boolean:vu},showCurrentTime:{boolean:vu},showMajorLabels:{boolean:vu},showMinorLabels:{boolean:vu},showWeekScale:{boolean:vu},stack:{boolean:vu},stackSubgroups:{boolean:vu},cluster:{maxItems:{number:bu,undefined:"undefined"},titleTemplate:{string:gu,undefined:"undefined"},clusterCriteria:{function:"function",undefined:"undefined"},showStipes:{boolean:vu,undefined:"undefined"},fitOnDoubleClick:{boolean:vu,undefined:"undefined"},__type__:{boolean:vu,object:yu}},snap:{function:"function",null:"null"},start:{date:"date",number:bu,string:gu,moment:"moment"},template:{function:"function"},loadingScreenTemplate:{function:"function"},groupTemplate:{function:"function"},visibleFrameTemplate:{string:gu,function:"function"},showTooltips:{boolean:vu},tooltip:{followMouse:{boolean:vu},overflowMethod:{string:["cap","flip","none"]},delay:{number:bu},template:{function:"function"},__type__:{object:yu}},tooltipOnItemUpdateTime:{template:{function:"function"},__type__:{boolean:vu,object:yu}},timeAxis:{scale:{string:gu,undefined:"undefined"},step:{number:bu,undefined:"undefined"},__type__:{object:yu}},type:{string:gu},width:{string:gu,number:bu},preferZoom:{boolean:vu},zoomable:{boolean:vu},zoomKey:{string:["ctrlKey","altKey","shiftKey","metaKey",""]},zoomFriction:{number:bu},zoomMax:{number:bu},zoomMin:{number:bu},__type__:{object:yu}},wu={global:{align:["center","left","right"],alignCurrentTime:["none","year","month","quarter","week","isoWeek","day","date","hour","minute","second"],direction:!1,autoResize:!0,clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},groupHeightMode:["auto","fixed","fitItems"],groupsDraggable:!1,height:"",locale:"",longSelectPressTime:251,margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},preferZoom:!1,selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,stackSubgroups:!0,cluster:!1,start:"",showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip",delay:[500,0,99999,100]},tooltipOnItemUpdateTime:!1,type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","shiftKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};V({target:"Array",proto:!0},{fill:function(t){for(var e=Lt(this),i=$(e.length),n=arguments.length,o=it(n>1?arguments[1]:void 0,i),s=n>2?arguments[2]:void 0,r=void 0===s?i:it(s,i);r>o;)e[o++]=t;return e}});var _u=Pi("Array").fill,ku=Array.prototype,Du=function(t){var e=t.fill;return t===ku||t instanceof Array&&e===ku.fill?_u:e},Iu={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"},Cu=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;ba(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return xa(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){if("string"==typeof t)return Iu[t]}},{key:"setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];if("none"!==t){var i,n=this._isColorString(t);if(void 0!==n&&(t=n),!0===za.isString(t)){if(!0===za.isValidRGB(t)){var o=t.substr(4).substr(0,t.length-5).split(",");i={r:o[0],g:o[1],b:o[2],a:1}}else if(!0===za.isValidRGBA(t)){var s=t.substr(5).substr(0,t.length-6).split(",");i={r:s[0],g:s[1],b:s[2],a:s[3]}}else if(!0===za.isValidHex(t)){var r=za.hexToRGB(t);i={r:r.r,g:r.g,b:r.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var a=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:a}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+Vs(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this,e=!(arguments.length>0&&void 0!==arguments[0])||arguments[0];!0===e&&(this.previousColor=za.extend({},this.color)),!0===this.applied&&this.updateCallback(this.initialColor),this.frame.style.display="none",qr((function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)}),0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];!0===e&&(this.initialColor=za.extend({},t)),this.color=t;var i=za.RGBToHSV(t.r,t.g,t.b),n=2*Math.PI,o=this.r*i.s,s=this.centerCoordinates.x+o*Math.sin(n*i.h),r=this.centerCoordinates.y+o*Math.cos(n*i.h);this.colorPickerSelector.style.left=s-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=r-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=za.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=za.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.color,e=za.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var n=this.colorPickerCanvas.clientWidth,o=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,n,o),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),Du(i).call(i),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){var t,e,i,n;if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var o=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(o.webkitBackingStorePixelRatio||o.mozBackingStorePixelRatio||o.msBackingStorePixelRatio||o.oBackingStorePixelRatio||o.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var s=document.createElement("DIV");s.style.color="red",s.style.fontWeight="bold",s.style.padding="10px",s.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(s)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(t){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(t){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var r=this;this.opacityRange.onchange=function(){r._setOpacity(this.value)},this.opacityRange.oninput=function(){r._setOpacity(this.value)},this.brightnessRange.onchange=function(){r._setBrightness(this.value)},this.brightnessRange.oninput=function(){r._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=va(t=this._hide).call(t,this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=va(e=this._apply).call(e,this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=va(i=this._save).call(i,this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=va(n=this._loadLast).call(n,this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new Vh(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),jh(this.hammer,(function(e){t._moveSelector(e)})),this.hammer.on("tap",(function(e){t._moveSelector(e)})),this.hammer.on("panstart",(function(e){t._moveSelector(e)})),this.hammer.on("panmove",(function(e){t._moveSelector(e)})),this.hammer.on("panend",(function(e){t._moveSelector(e)}))}},{key:"_generateHueCircle",value:function(){if(!1===this.generated){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e,i,n,o,s=this.colorPickerCanvas.clientWidth,r=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,s,r),this.centerCoordinates={x:.5*s,y:.5*r},this.r=.49*s;var a,l=2*Math.PI/360,h=1/this.r;for(n=0;n<360;n++)for(o=0;o<this.r;o++)e=this.centerCoordinates.x+o*Math.sin(l*n),i=this.centerCoordinates.y+o*Math.cos(l*n),a=za.HSVToRGB(n*(1/360),o*h,1),t.fillStyle="rgb("+a.r+","+a.g+","+a.b+")",t.fillRect(e-.5,i-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,s,r)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,n=t.center.y-e.top,o=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,a=n-o,l=Math.atan2(r,a),h=.98*Math.min(Math.sqrt(r*r+a*a),s),d=Math.cos(l)*h+o,u=Math.sin(l)*h+s;this.colorPickerSelector.style.top=d-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var c=l/(2*Math.PI);c=c<0?c+1:c;var p=h/this.r,m=za.RGBToHSV(this.color.r,this.color.g,this.color.b);m.h=c,m.s=p;var f=za.HSVToRGB(m.h,m.s,m.v);f.a=this.color.a,this.color=f,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();Ph('div.vis-configuration{position:relative;display:block;float:left;font-size:12px}div.vis-configuration-wrapper{display:block;width:700px}div.vis-configuration-wrapper:after{clear:both;content:"";display:block}div.vis-configuration.vis-config-option-container{display:block;width:495px;background-color:#fff;border:2px solid #f7f8fa;border-radius:4px;margin-top:20px;left:10px;padding-left:5px}div.vis-configuration.vis-config-button{display:block;width:495px;height:25px;vertical-align:middle;line-height:25px;background-color:#f7f8fa;border:2px solid #ceced0;border-radius:4px;margin-top:20px;left:10px;padding-left:5px;cursor:pointer;margin-bottom:30px}div.vis-configuration.vis-config-button.hover{background-color:#4588e6;border:2px solid #214373;color:#fff}div.vis-configuration.vis-config-item{display:block;float:left;width:495px;height:25px;vertical-align:middle;line-height:25px}div.vis-configuration.vis-config-item.vis-config-s2{left:10px;background-color:#f7f8fa;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s3{left:20px;background-color:#e4e9f0;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-item.vis-config-s4{left:30px;background-color:#cfd8e6;padding-left:5px;border-radius:3px}div.vis-configuration.vis-config-header{font-size:18px;font-weight:700}div.vis-configuration.vis-config-label{width:120px;height:25px;line-height:25px}div.vis-configuration.vis-config-label.vis-config-s3{width:110px}div.vis-configuration.vis-config-label.vis-config-s4{width:100px}div.vis-configuration.vis-config-colorBlock{top:1px;width:30px;height:19px;border:1px solid #444;border-radius:2px;padding:0;margin:0;cursor:pointer}input.vis-configuration.vis-config-checkbox{left:-5px}input.vis-configuration.vis-config-rangeinput{position:relative;top:-5px;width:60px;padding:1px;margin:0;pointer-events:none}input.vis-configuration.vis-config-range{-webkit-appearance:none;border:0 solid #fff;background-color:transparent;width:300px;height:20px}input.vis-configuration.vis-config-range::-webkit-slider-runnable-track{width:300px;height:5px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-webkit-slider-thumb{-webkit-appearance:none;border:1px solid #14334b;height:17px;width:17px;border-radius:50%;background:#3876c2;background:-moz-linear-gradient(top,#3876c2 0,#385380 100%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#3876c2),color-stop(100%,#385380));background:-webkit-linear-gradient(top,#3876c2,#385380);background:-o-linear-gradient(top,#3876c2 0,#385380 100%);background:-ms-linear-gradient(top,#3876c2 0,#385380 100%);background:linear-gradient(180deg,#3876c2 0,#385380);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#3876c2",endColorstr="#385380",GradientType=0);box-shadow:0 0 1px 0 #111927;margin-top:-7px}input.vis-configuration.vis-config-range:focus{outline:none}input.vis-configuration.vis-config-range:focus::-webkit-slider-runnable-track{background:#9d9d9d;background:-moz-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#9d9d9d),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#9d9d9d,#c8c8c8 99%);background:-o-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#9d9d9d 0,#c8c8c8 99%);background:linear-gradient(180deg,#9d9d9d 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#9d9d9d",endColorstr="#c8c8c8",GradientType=0)}input.vis-configuration.vis-config-range::-moz-range-track{width:300px;height:10px;background:#dedede;background:-moz-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-webkit-gradient(linear,left top,left bottom,color-stop(0,#dedede),color-stop(99%,#c8c8c8));background:-webkit-linear-gradient(top,#dedede,#c8c8c8 99%);background:-o-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:-ms-linear-gradient(top,#dedede 0,#c8c8c8 99%);background:linear-gradient(180deg,#dedede 0,#c8c8c8 99%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#dedede",endColorstr="#c8c8c8",GradientType=0);border:1px solid #999;box-shadow:0 0 3px 0 #aaa;border-radius:3px}input.vis-configuration.vis-config-range::-moz-range-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:-moz-focusring{outline:1px solid #fff;outline-offset:-1px}input.vis-configuration.vis-config-range::-ms-track{width:300px;height:5px;background:transparent;border-color:transparent;border-width:6px 0;color:transparent}input.vis-configuration.vis-config-range::-ms-fill-lower{background:#777;border-radius:10px}input.vis-configuration.vis-config-range::-ms-fill-upper{background:#ddd;border-radius:10px}input.vis-configuration.vis-config-range::-ms-thumb{border:none;height:16px;width:16px;border-radius:50%;background:#385380}input.vis-configuration.vis-config-range:focus::-ms-fill-lower{background:#888}input.vis-configuration.vis-config-range:focus::-ms-fill-upper{background:#ccc}.vis-configuration-popup{position:absolute;background:rgba(57,76,89,.85);border:2px solid #f2faff;line-height:30px;height:30px;width:150px;text-align:center;color:#fff;font-size:14px;border-radius:4px;-webkit-transition:opacity .3s ease-in-out;-moz-transition:opacity .3s ease-in-out;transition:opacity .3s ease-in-out}.vis-configuration-popup:after,.vis-configuration-popup:before{left:100%;top:50%;border:solid transparent;content:" ";height:0;width:0;position:absolute;pointer-events:none}.vis-configuration-popup:after{border-color:rgba(136,183,213,0) rgba(136,183,213,0) rgba(136,183,213,0) rgba(57,76,89,.85);border-width:8px;margin-top:-8px}.vis-configuration-popup:before{border-color:rgba(194,225,245,0) rgba(194,225,245,0) rgba(194,225,245,0) #f2faff;border-width:12px;margin-top:-12px}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImNvbmZpZ3VyYXRpb24uY3NzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLHNCQUNJLGlCQUFpQixDQUNqQixhQUFhLENBQ2IsVUFBVSxDQUNWLGNBQ0osQ0FFQSw4QkFDSSxhQUFhLENBQ2IsV0FDSixDQUVBLG9DQUNFLFVBQVcsQ0FDWCxVQUFXLENBQ1gsYUFDRixDQUVBLGtEQUNJLGFBQWEsQ0FDYixXQUFXLENBQ1gscUJBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixpQkFBaUIsQ0FDakIsZUFBZSxDQUNmLFNBQVMsQ0FDVCxnQkFDSixDQUVBLHdDQUNJLGFBQWEsQ0FDYixXQUFXLENBQ1gsV0FBVyxDQUNYLHFCQUFzQixDQUN0QixnQkFBZ0IsQ0FDaEIsd0JBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixpQkFBaUIsQ0FDakIsZUFBZSxDQUNmLFNBQVMsQ0FDVCxnQkFBZ0IsQ0FDaEIsY0FBZSxDQUNmLGtCQUNKLENBRUEsOENBQ0ksd0JBQXlCLENBQ3pCLHdCQUF3QixDQUN4QixVQUNKLENBRUEsc0NBQ0ksYUFBYSxDQUNiLFVBQVUsQ0FDVixXQUFXLENBQ1gsV0FBVyxDQUNYLHFCQUFzQixDQUN0QixnQkFDSixDQUdBLG9EQUNJLFNBQVMsQ0FDVCx3QkFBeUIsQ0FDekIsZ0JBQWdCLENBQ2hCLGlCQUNKLENBQ0Esb0RBQ0ksU0FBUyxDQUNULHdCQUF5QixDQUN6QixnQkFBZ0IsQ0FDaEIsaUJBQ0osQ0FDQSxvREFDSSxTQUFTLENBQ1Qsd0JBQXlCLENBQ3pCLGdCQUFnQixDQUNoQixpQkFDSixDQUVBLHdDQUNJLGNBQWMsQ0FDZCxlQUNKLENBRUEsdUNBQ0ksV0FBVyxDQUNYLFdBQVcsQ0FDWCxnQkFDSixDQUVBLHFEQUNJLFdBQ0osQ0FDQSxxREFDSSxXQUNKLENBRUEsNENBQ0ksT0FBTyxDQUNQLFVBQVUsQ0FDVixXQUFXLENBQ1gscUJBQXdCLENBQ3hCLGlCQUFpQixDQUNqQixTQUFXLENBQ1gsUUFBVSxDQUNWLGNBQ0osQ0FFQSw0Q0FDSSxTQUNKLENBR0EsOENBQ0ksaUJBQWlCLENBQ2pCLFFBQVEsQ0FDUixVQUFVLENBRVYsV0FBVyxDQUNYLFFBQVEsQ0FDUixtQkFDSixDQUVBLHlDQUVJLHVCQUF3QixDQUd4QixtQkFBdUIsQ0FDdkIsNEJBQThCLENBRzlCLFdBQVksQ0FDWixXQUNKLENBQ0Esd0VBQ0ksV0FBWSxDQUNaLFVBQVcsQ0FDWCxrQkFBbUIsQ0FDbkIsMERBQStELENBQy9ELHNHQUE0RyxDQUM1RywyREFBaUUsQ0FDakUsd0RBQTRELENBQzVELHlEQUE2RCxDQUM3RCx3REFBK0QsQ0FDL0QsK0dBQW1ILENBRW5ILHFCQUF5QixDQUN6Qix5QkFBbUMsQ0FDbkMsaUJBQ0osQ0FDQSwrREFDSSx1QkFBd0IsQ0FDeEIsd0JBQXlCLENBQ3pCLFdBQVksQ0FDWixVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGtCQUFtQixDQUNuQiwyREFBZ0UsQ0FDaEUsdUdBQTZHLENBQzdHLHVEQUFrRSxDQUNsRSx5REFBNkQsQ0FDN0QsMERBQThELENBQzlELG9EQUFnRSxDQUNoRSwrR0FBbUgsQ0FDbkgsNEJBQW1DLENBQ25DLGVBQ0osQ0FDQSwrQ0FDSSxZQUNKLENBQ0EsOEVBQ0ksa0JBQW1CLENBQ25CLDBEQUE4RCxDQUM5RCxzR0FBNEcsQ0FDNUcsMkRBQWlFLENBQ2pFLHdEQUE0RCxDQUM1RCx5REFBNkQsQ0FDN0Qsd0RBQStELENBQy9ELCtHQUNKLENBRUEsMkRBQ0ksV0FBWSxDQUNaLFdBQVksQ0FDWixrQkFBbUIsQ0FDbkIsMERBQStELENBQy9ELHNHQUE0RyxDQUM1RywyREFBaUUsQ0FDakUsd0RBQTRELENBQzVELHlEQUE2RCxDQUM3RCx3REFBK0QsQ0FDL0QsK0dBQW1ILENBRW5ILHFCQUF5QixDQUN6Qix5QkFBbUMsQ0FDbkMsaUJBQ0osQ0FDQSwyREFDSSxXQUFZLENBQ1osV0FBWSxDQUNaLFVBQVcsQ0FFWCxpQkFBa0IsQ0FDbEIsa0JBQ0osQ0FHQSx3REFDSSxzQkFBd0IsQ0FDeEIsbUJBQ0osQ0FFQSxvREFDSSxXQUFZLENBQ1osVUFBVyxDQUdYLHNCQUF1QixDQUd2Qix3QkFBeUIsQ0FDekIsa0JBQW1CLENBR25CLGlCQUNKLENBQ0EseURBQ0ksZUFBZ0IsQ0FDaEIsa0JBQ0osQ0FDQSx5REFDSSxlQUFnQixDQUNoQixrQkFDSixDQUNBLG9EQUNJLFdBQVksQ0FDWixXQUFZLENBQ1osVUFBVyxDQUNYLGlCQUFrQixDQUNsQixrQkFDSixDQUNBLCtEQUNJLGVBQ0osQ0FDQSwrREFDSSxlQUNKLENBRUEseUJBQ0ksaUJBQWtCLENBQ2xCLDZCQUFrQyxDQUNsQyx3QkFBeUIsQ0FDekIsZ0JBQWdCLENBQ2hCLFdBQVcsQ0FDWCxXQUFXLENBQ1gsaUJBQWlCLENBQ2pCLFVBQWMsQ0FDZCxjQUFjLENBQ2QsaUJBQWlCLENBQ2pCLDBDQUE0QyxDQUM1Qyx1Q0FBeUMsQ0FDekMsa0NBQ0osQ0FDQSwrREFDSSxTQUFVLENBQ1YsT0FBUSxDQUNSLHdCQUF5QixDQUN6QixXQUFZLENBQ1osUUFBUyxDQUNULE9BQVEsQ0FDUixpQkFBa0IsQ0FDbEIsbUJBQ0osQ0FFQSwrQkFFSSwyRkFBeUMsQ0FDekMsZ0JBQWlCLENBQ2pCLGVBQ0osQ0FDQSxnQ0FFSSxnRkFBMEIsQ0FDMUIsaUJBQWtCLENBQ2xCLGdCQUNKIiwiZmlsZSI6ImNvbmZpZ3VyYXRpb24uY3NzIiwic291cmNlc0NvbnRlbnQiOlsiZGl2LnZpcy1jb25maWd1cmF0aW9uIHtcbiAgICBwb3NpdGlvbjpyZWxhdGl2ZTtcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIGZsb2F0OmxlZnQ7XG4gICAgZm9udC1zaXplOjEycHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi13cmFwcGVyIHtcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIHdpZHRoOjcwMHB4O1xufVxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24td3JhcHBlcjo6YWZ0ZXIge1xuICBjbGVhcjogYm90aDtcbiAgY29udGVudDogXCJcIjtcbiAgZGlzcGxheTogYmxvY2s7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLW9wdGlvbi1jb250YWluZXJ7XG4gICAgZGlzcGxheTpibG9jaztcbiAgICB3aWR0aDo0OTVweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjZmZmZmZmO1xuICAgIGJvcmRlcjoycHggc29saWQgI2Y3ZjhmYTtcbiAgICBib3JkZXItcmFkaXVzOjRweDtcbiAgICBtYXJnaW4tdG9wOjIwcHg7XG4gICAgbGVmdDoxMHB4O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWJ1dHRvbntcbiAgICBkaXNwbGF5OmJsb2NrO1xuICAgIHdpZHRoOjQ5NXB4O1xuICAgIGhlaWdodDoyNXB4O1xuICAgIHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7XG4gICAgbGluZS1oZWlnaHQ6MjVweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjZjdmOGZhO1xuICAgIGJvcmRlcjoycHggc29saWQgI2NlY2VkMDtcbiAgICBib3JkZXItcmFkaXVzOjRweDtcbiAgICBtYXJnaW4tdG9wOjIwcHg7XG4gICAgbGVmdDoxMHB4O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG4gICAgY3Vyc29yOiBwb2ludGVyO1xuICAgIG1hcmdpbi1ib3R0b206MzBweDtcbn1cblxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctYnV0dG9uLmhvdmVye1xuICAgIGJhY2tncm91bmQtY29sb3I6ICM0NTg4ZTY7XG4gICAgYm9yZGVyOjJweCBzb2xpZCAjMjE0MzczO1xuICAgIGNvbG9yOiNmZmZmZmY7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWl0ZW17XG4gICAgZGlzcGxheTpibG9jaztcbiAgICBmbG9hdDpsZWZ0O1xuICAgIHdpZHRoOjQ5NXB4O1xuICAgIGhlaWdodDoyNXB4O1xuICAgIHZlcnRpY2FsLWFsaWduOiBtaWRkbGU7XG4gICAgbGluZS1oZWlnaHQ6MjVweDtcbn1cblxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1pdGVtLnZpcy1jb25maWctczJ7XG4gICAgbGVmdDoxMHB4O1xuICAgIGJhY2tncm91bmQtY29sb3I6ICNmN2Y4ZmE7XG4gICAgcGFkZGluZy1sZWZ0OjVweDtcbiAgICBib3JkZXItcmFkaXVzOjNweDtcbn1cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWl0ZW0udmlzLWNvbmZpZy1zM3tcbiAgICBsZWZ0OjIwcHg7XG4gICAgYmFja2dyb3VuZC1jb2xvcjogI2U0ZTlmMDtcbiAgICBwYWRkaW5nLWxlZnQ6NXB4O1xuICAgIGJvcmRlci1yYWRpdXM6M3B4O1xufVxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctaXRlbS52aXMtY29uZmlnLXM0e1xuICAgIGxlZnQ6MzBweDtcbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjY2ZkOGU2O1xuICAgIHBhZGRpbmctbGVmdDo1cHg7XG4gICAgYm9yZGVyLXJhZGl1czozcHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWhlYWRlcntcbiAgICBmb250LXNpemU6MThweDtcbiAgICBmb250LXdlaWdodDogYm9sZDtcbn1cblxuZGl2LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctbGFiZWx7XG4gICAgd2lkdGg6MTIwcHg7XG4gICAgaGVpZ2h0OjI1cHg7XG4gICAgbGluZS1oZWlnaHQ6IDI1cHg7XG59XG5cbmRpdi52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLWxhYmVsLnZpcy1jb25maWctczN7XG4gICAgd2lkdGg6MTEwcHg7XG59XG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1sYWJlbC52aXMtY29uZmlnLXM0e1xuICAgIHdpZHRoOjEwMHB4O1xufVxuXG5kaXYudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1jb2xvckJsb2Nre1xuICAgIHRvcDoxcHg7XG4gICAgd2lkdGg6MzBweDtcbiAgICBoZWlnaHQ6MTlweDtcbiAgICBib3JkZXI6MXB4IHNvbGlkICM0NDQ0NDQ7XG4gICAgYm9yZGVyLXJhZGl1czoycHg7XG4gICAgcGFkZGluZzowcHg7XG4gICAgbWFyZ2luOjBweDtcbiAgICBjdXJzb3I6cG9pbnRlcjtcbn1cblxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1jaGVja2JveCB7XG4gICAgbGVmdDotNXB4O1xufVxuXG5cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2VpbnB1dHtcbiAgICBwb3NpdGlvbjpyZWxhdGl2ZTtcbiAgICB0b3A6LTVweDtcbiAgICB3aWR0aDo2MHB4O1xuICAgIC8qaGVpZ2h0OjEzcHg7Ki9cbiAgICBwYWRkaW5nOjFweDtcbiAgICBtYXJnaW46MDtcbiAgICBwb2ludGVyLWV2ZW50czpub25lO1xufVxuXG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdle1xuICAgIC8qcmVtb3ZlcyBkZWZhdWx0IHdlYmtpdCBzdHlsZXMqL1xuICAgIC13ZWJraXQtYXBwZWFyYW5jZTogbm9uZTtcblxuICAgIC8qZml4IGZvciBGRiB1bmFibGUgdG8gYXBwbHkgZm9jdXMgc3R5bGUgYnVnICovXG4gICAgYm9yZGVyOiAwcHggc29saWQgd2hpdGU7XG4gICAgYmFja2dyb3VuZC1jb2xvcjpyZ2JhKDAsMCwwLDApO1xuXG4gICAgLypyZXF1aXJlZCBmb3IgcHJvcGVyIHRyYWNrIHNpemluZyBpbiBGRiovXG4gICAgd2lkdGg6IDMwMHB4O1xuICAgIGhlaWdodDoyMHB4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LXdlYmtpdC1zbGlkZXItcnVubmFibGUtdHJhY2sge1xuICAgIHdpZHRoOiAzMDBweDtcbiAgICBoZWlnaHQ6IDVweDtcbiAgICBiYWNrZ3JvdW5kOiAjZGVkZWRlOyAvKiBPbGQgYnJvd3NlcnMgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbW96LWxpbmVhci1ncmFkaWVudCh0b3AsICAjZGVkZWRlIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsI2RlZGVkZSksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgI2RlZGVkZSAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgI2RlZGVkZSAwJSwgI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICNkZWRlZGUgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjZGVkZWRlIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyNkZWRlZGUnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cblxuICAgIGJvcmRlcjogMXB4IHNvbGlkICM5OTk5OTk7XG4gICAgYm94LXNoYWRvdzogI2FhYWFhYSAwcHggMHB4IDNweCAwcHg7XG4gICAgYm9yZGVyLXJhZGl1czogM3B4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LXdlYmtpdC1zbGlkZXItdGh1bWIge1xuICAgIC13ZWJraXQtYXBwZWFyYW5jZTogbm9uZTtcbiAgICBib3JkZXI6IDFweCBzb2xpZCAjMTQzMzRiO1xuICAgIGhlaWdodDogMTdweDtcbiAgICB3aWR0aDogMTdweDtcbiAgICBib3JkZXItcmFkaXVzOiA1MCU7XG4gICAgYmFja2dyb3VuZDogIzM4NzZjMjsgLyogT2xkIGJyb3dzZXJzICovXG4gICAgYmFja2dyb3VuZDogLW1vei1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwgIzM4NTM4MCAxMDAlKTsgLyogRkYzLjYrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1ncmFkaWVudChsaW5lYXIsIGxlZnQgdG9wLCBsZWZ0IGJvdHRvbSwgY29sb3Itc3RvcCgwJSwjMzg3NmMyKSwgY29sb3Itc3RvcCgxMDAlLCMzODUzODApKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwjMzg1MzgwIDEwMCUpOyAvKiBDaHJvbWUxMCssU2FmYXJpNS4xKyAqL1xuICAgIGJhY2tncm91bmQ6IC1vLWxpbmVhci1ncmFkaWVudCh0b3AsICAjMzg3NmMyIDAlLCMzODUzODAgMTAwJSk7IC8qIE9wZXJhIDExLjEwKyAqL1xuICAgIGJhY2tncm91bmQ6IC1tcy1saW5lYXItZ3JhZGllbnQodG9wLCAgIzM4NzZjMiAwJSwjMzg1MzgwIDEwMCUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjMzg3NmMyIDAlLCMzODUzODAgMTAwJSk7IC8qIFczQyAqL1xuICAgIGZpbHRlcjogcHJvZ2lkOkRYSW1hZ2VUcmFuc2Zvcm0uTWljcm9zb2Z0LmdyYWRpZW50KCBzdGFydENvbG9yc3RyPScjMzg3NmMyJywgZW5kQ29sb3JzdHI9JyMzODUzODAnLEdyYWRpZW50VHlwZT0wICk7IC8qIElFNi05ICovXG4gICAgYm94LXNoYWRvdzogIzExMTkyNyAwcHggMHB4IDFweCAwcHg7XG4gICAgbWFyZ2luLXRvcDogLTdweDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Zm9jdXMge1xuICAgIG91dGxpbmU6IG5vbmU7XG59XG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOmZvY3VzOjotd2Via2l0LXNsaWRlci1ydW5uYWJsZS10cmFjayB7XG4gICAgYmFja2dyb3VuZDogIzlkOWQ5ZDsgLyogT2xkIGJyb3dzZXJzICovXG4gICAgYmFja2dyb3VuZDogLW1vei1saW5lYXItZ3JhZGllbnQodG9wLCAjOWQ5ZDlkIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsIzlkOWQ5ZCksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgIzlkOWQ5ZCAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgICM5ZDlkOWQgMCUsI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICM5ZDlkOWQgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjOWQ5ZDlkIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyM5ZDlkOWQnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cbn1cblxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1vei1yYW5nZS10cmFjayB7XG4gICAgd2lkdGg6IDMwMHB4O1xuICAgIGhlaWdodDogMTBweDtcbiAgICBiYWNrZ3JvdW5kOiAjZGVkZWRlOyAvKiBPbGQgYnJvd3NlcnMgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbW96LWxpbmVhci1ncmFkaWVudCh0b3AsICAjZGVkZWRlIDAlLCAjYzhjOGM4IDk5JSk7IC8qIEZGMy42KyAqL1xuICAgIGJhY2tncm91bmQ6IC13ZWJraXQtZ3JhZGllbnQobGluZWFyLCBsZWZ0IHRvcCwgbGVmdCBib3R0b20sIGNvbG9yLXN0b3AoMCUsI2RlZGVkZSksIGNvbG9yLXN0b3AoOTklLCNjOGM4YzgpKTsgLyogQ2hyb21lLFNhZmFyaTQrICovXG4gICAgYmFja2dyb3VuZDogLXdlYmtpdC1saW5lYXItZ3JhZGllbnQodG9wLCAgI2RlZGVkZSAwJSwjYzhjOGM4IDk5JSk7IC8qIENocm9tZTEwKyxTYWZhcmk1LjErICovXG4gICAgYmFja2dyb3VuZDogLW8tbGluZWFyLWdyYWRpZW50KHRvcCwgI2RlZGVkZSAwJSwgI2M4YzhjOCA5OSUpOyAvKiBPcGVyYSAxMS4xMCsgKi9cbiAgICBiYWNrZ3JvdW5kOiAtbXMtbGluZWFyLWdyYWRpZW50KHRvcCwgICNkZWRlZGUgMCUsI2M4YzhjOCA5OSUpOyAvKiBJRTEwKyAqL1xuICAgIGJhY2tncm91bmQ6IGxpbmVhci1ncmFkaWVudCh0byBib3R0b20sICAjZGVkZWRlIDAlLCNjOGM4YzggOTklKTsgLyogVzNDICovXG4gICAgZmlsdGVyOiBwcm9naWQ6RFhJbWFnZVRyYW5zZm9ybS5NaWNyb3NvZnQuZ3JhZGllbnQoIHN0YXJ0Q29sb3JzdHI9JyNkZWRlZGUnLCBlbmRDb2xvcnN0cj0nI2M4YzhjOCcsR3JhZGllbnRUeXBlPTAgKTsgLyogSUU2LTkgKi9cblxuICAgIGJvcmRlcjogMXB4IHNvbGlkICM5OTk5OTk7XG4gICAgYm94LXNoYWRvdzogI2FhYWFhYSAwcHggMHB4IDNweCAwcHg7XG4gICAgYm9yZGVyLXJhZGl1czogM3B4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1vei1yYW5nZS10aHVtYiB7XG4gICAgYm9yZGVyOiBub25lO1xuICAgIGhlaWdodDogMTZweDtcbiAgICB3aWR0aDogMTZweDtcblxuICAgIGJvcmRlci1yYWRpdXM6IDUwJTtcbiAgICBiYWNrZ3JvdW5kOiAgIzM4NTM4MDtcbn1cblxuLypoaWRlIHRoZSBvdXRsaW5lIGJlaGluZCB0aGUgYm9yZGVyKi9cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6LW1vei1mb2N1c3Jpbmd7XG4gICAgb3V0bGluZTogMXB4IHNvbGlkIHdoaXRlO1xuICAgIG91dGxpbmUtb2Zmc2V0OiAtMXB4O1xufVxuXG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOjotbXMtdHJhY2sge1xuICAgIHdpZHRoOiAzMDBweDtcbiAgICBoZWlnaHQ6IDVweDtcblxuICAgIC8qcmVtb3ZlIGJnIGNvbG91ciBmcm9tIHRoZSB0cmFjaywgd2UnbGwgdXNlIG1zLWZpbGwtbG93ZXIgYW5kIG1zLWZpbGwtdXBwZXIgaW5zdGVhZCAqL1xuICAgIGJhY2tncm91bmQ6IHRyYW5zcGFyZW50O1xuXG4gICAgLypsZWF2ZSByb29tIGZvciB0aGUgbGFyZ2VyIHRodW1iIHRvIG92ZXJmbG93IHdpdGggYSB0cmFuc3BhcmVudCBib3JkZXIgKi9cbiAgICBib3JkZXItY29sb3I6IHRyYW5zcGFyZW50O1xuICAgIGJvcmRlci13aWR0aDogNnB4IDA7XG5cbiAgICAvKnJlbW92ZSBkZWZhdWx0IHRpY2sgbWFya3MqL1xuICAgIGNvbG9yOiB0cmFuc3BhcmVudDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Oi1tcy1maWxsLWxvd2VyIHtcbiAgICBiYWNrZ3JvdW5kOiAjNzc3O1xuICAgIGJvcmRlci1yYWRpdXM6IDEwcHg7XG59XG5pbnB1dC52aXMtY29uZmlndXJhdGlvbi52aXMtY29uZmlnLXJhbmdlOjotbXMtZmlsbC11cHBlciB7XG4gICAgYmFja2dyb3VuZDogI2RkZDtcbiAgICBib3JkZXItcmFkaXVzOiAxMHB4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTo6LW1zLXRodW1iIHtcbiAgICBib3JkZXI6IG5vbmU7XG4gICAgaGVpZ2h0OiAxNnB4O1xuICAgIHdpZHRoOiAxNnB4O1xuICAgIGJvcmRlci1yYWRpdXM6IDUwJTtcbiAgICBiYWNrZ3JvdW5kOiAgIzM4NTM4MDtcbn1cbmlucHV0LnZpcy1jb25maWd1cmF0aW9uLnZpcy1jb25maWctcmFuZ2U6Zm9jdXM6Oi1tcy1maWxsLWxvd2VyIHtcbiAgICBiYWNrZ3JvdW5kOiAjODg4O1xufVxuaW5wdXQudmlzLWNvbmZpZ3VyYXRpb24udmlzLWNvbmZpZy1yYW5nZTpmb2N1czo6LW1zLWZpbGwtdXBwZXIge1xuICAgIGJhY2tncm91bmQ6ICNjY2M7XG59XG5cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cCB7XG4gICAgcG9zaXRpb246IGFic29sdXRlO1xuICAgIGJhY2tncm91bmQ6IHJnYmEoNTcsIDc2LCA4OSwgMC44NSk7XG4gICAgYm9yZGVyOiAycHggc29saWQgI2YyZmFmZjtcbiAgICBsaW5lLWhlaWdodDozMHB4O1xuICAgIGhlaWdodDozMHB4O1xuICAgIHdpZHRoOjE1MHB4O1xuICAgIHRleHQtYWxpZ246Y2VudGVyO1xuICAgIGNvbG9yOiAjZmZmZmZmO1xuICAgIGZvbnQtc2l6ZToxNHB4O1xuICAgIGJvcmRlci1yYWRpdXM6NHB4O1xuICAgIC13ZWJraXQtdHJhbnNpdGlvbjogb3BhY2l0eSAwLjNzIGVhc2UtaW4tb3V0O1xuICAgIC1tb3otdHJhbnNpdGlvbjogb3BhY2l0eSAwLjNzIGVhc2UtaW4tb3V0O1xuICAgIHRyYW5zaXRpb246IG9wYWNpdHkgMC4zcyBlYXNlLWluLW91dDtcbn1cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cDphZnRlciwgLnZpcy1jb25maWd1cmF0aW9uLXBvcHVwOmJlZm9yZSB7XG4gICAgbGVmdDogMTAwJTtcbiAgICB0b3A6IDUwJTtcbiAgICBib3JkZXI6IHNvbGlkIHRyYW5zcGFyZW50O1xuICAgIGNvbnRlbnQ6IFwiIFwiO1xuICAgIGhlaWdodDogMDtcbiAgICB3aWR0aDogMDtcbiAgICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gICAgcG9pbnRlci1ldmVudHM6IG5vbmU7XG59XG5cbi52aXMtY29uZmlndXJhdGlvbi1wb3B1cDphZnRlciB7XG4gICAgYm9yZGVyLWNvbG9yOiByZ2JhKDEzNiwgMTgzLCAyMTMsIDApO1xuICAgIGJvcmRlci1sZWZ0LWNvbG9yOiByZ2JhKDU3LCA3NiwgODksIDAuODUpO1xuICAgIGJvcmRlci13aWR0aDogOHB4O1xuICAgIG1hcmdpbi10b3A6IC04cHg7XG59XG4udmlzLWNvbmZpZ3VyYXRpb24tcG9wdXA6YmVmb3JlIHtcbiAgICBib3JkZXItY29sb3I6IHJnYmEoMTk0LCAyMjUsIDI0NSwgMCk7XG4gICAgYm9yZGVyLWxlZnQtY29sb3I6ICNmMmZhZmY7XG4gICAgYm9yZGVyLXdpZHRoOiAxMnB4O1xuICAgIG1hcmdpbi10b3A6IC0xMnB4O1xufSJdfQ== */');var Lu=function(){function t(e,i,n){var o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;ba(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},za.extend(this.options,this.defaultOptions),this.configureOptions=n,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new Cu(o),this.wrapper=void 0}return xa(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;if("string"==typeof t)this.options.filter=t;else if(t instanceof Array)this.options.filter=t.join();else if("object"===zo(t)){if(null==t)throw new TypeError("options cannot be null");void 0!==t.container&&(this.options.container=t.container),void 0!==En(t)&&(this.options.filter=En(t)),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)}else"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0);!1===En(this.options)&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,!0===this.options.enabled&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){this._clean(),this.changedOptions=[];var t=En(this.options),e=0,i=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,i=!1,"function"==typeof t?i=(i=t(n,[]))||this._handleObject(this.configureOptions[n],[n],!0):!0!==t&&-1===Fo(t).call(t,n)||(i=!0),!1!==i&&(this.allowCreation=!0,e>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),e++);this._makeButton(),this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){if(!0===this.allowCreation){var e=document.createElement("div");e.className="vis-configuration vis-config-item vis-config-s"+t.length;for(var i=arguments.length,n=new Array(i>1?i-1:0),o=1;o<i;o++)n[o-1]=arguments[o];return mn(n).call(n,(function(t){e.appendChild(t)})),this.domElements.push(e),this.domElements.length}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=document.createElement("div");return n.className="vis-configuration vis-config-label vis-config-s"+e.length,n.innerHTML=!0===i?"<i><b>"+t+":</b></i>":t+":",n}},{key:"_makeDropdown",value:function(t,e,i){var n=document.createElement("select");n.className="vis-configuration vis-config-select";var o=0;void 0!==e&&-1!==Fo(t).call(t,e)&&(o=Fo(t).call(t,e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===o&&(r.selected="selected"),r.innerHTML=t[s],n.appendChild(r)}var a=this;n.onchange=function(){a._update(this.value,i)};var l=this._makeLabel(i[i.length-1],i);this._makeItem(i,l,n)}},{key:"_makeRange",value:function(t,e,i){var n=t[0],o=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=o,a.max=s}catch(t){}a.step=r;var l="",h=0;if(void 0!==e){e<0&&1.2*e<o?(a.min=Math.ceil(1.2*e),h=a.min,l="range increased"):e/1.2<o&&(a.min=Math.ceil(e/1.2),h=a.min,l="range increased"),1.2*e>s&&1!==s&&(a.max=Math.ceil(1.2*e),h=a.max,l="range increased"),a.value=e}else a.value=n;var d=document.createElement("input");d.className="vis-configuration vis-config-rangeinput",d.value=Number(a.value);var u=this;a.onchange=function(){d.value=this.value,u._update(Number(this.value),i)},a.oninput=function(){d.value=this.value};var c=this._makeLabel(i[i.length-1],i),p=this._makeItem(i,c,a,d);""!==l&&this.popupHistory[p]!==h&&(this.popupHistory[p]=h,this._setupPopup(l,p))}},{key:"_makeButton",value:function(){var t=this;if(!0===this.options.showButton){var e=document.createElement("div");e.className="vis-configuration vis-config-button",e.innerHTML="generate options",e.onclick=function(){t._printOptions()},e.onmouseover=function(){e.className="vis-configuration vis-config-button hover"},e.onmouseout=function(){e.className="vis-configuration vis-config-button"},this.optionsContainer=document.createElement("div"),this.optionsContainer.className="vis-configuration vis-config-option-container",this.domElements.push(this.optionsContainer),this.domElements.push(e)}}},{key:"_setupPopup",value:function(t,e){var i=this;if(!0===this.initialized&&!0===this.allowCreation&&this.popupCounter<this.popupLimit){var n=document.createElement("div");n.id="vis-configuration-popup",n.className="vis-configuration-popup",n.innerHTML=t,n.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:n,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index].getBoundingClientRect();this.popupDiv.html.style.left=e.left+"px",this.popupDiv.html.style.top=e.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=qr((function(){t.popupDiv.html.style.opacity=0}),1500),this.popupDiv.deleteTimeout=qr((function(){t._removePopup()}),1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var n=document.createElement("input");n.type="checkbox",n.className="vis-configuration vis-config-checkbox",n.checked=t,void 0!==e&&(n.checked=e,e!==t&&("object"===zo(t)?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var o=this;n.onchange=function(){o._update(this.checked,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,n)}},{key:"_makeTextInput",value:function(t,e,i){var n=document.createElement("input");n.type="text",n.className="vis-configuration vis-config-text",n.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var o=this;n.onchange=function(){o._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,n)}},{key:"_makeColorField",value:function(t,e,i){var n=this,o=t[1],s=document.createElement("div");"none"!==(e=void 0===e?o:e)?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?o:e,s.onclick=function(){n._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var n=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback((function(t){var o="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=o,n._update(o,i)})),this.colorPicker.setCloseCallback((function(){e.onclick=function(){n._showColorPicker(t,e,i)}}))}},{key:"_handleObject",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],n=!1,o=En(this.options),s=!1;for(var r in t)if(t.hasOwnProperty(r)){n=!0;var a=t[r],l=za.copyAndExtendArray(e,r);if("function"==typeof o&&!1===(n=o(r,e))&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,n=this._handleObject(a,l,!0),this.allowCreation=!1===i),!1!==n){s=!0;var h=this._getValue(l);if(a instanceof Array)this._handleArray(a,h,l);else if("string"==typeof a)this._makeTextInput(a,h,l);else if("boolean"==typeof a)this._makeCheckbox(a,h,l);else if(a instanceof Object){var d=!0;if(-1!==Fo(e).call(e,"physics")&&this.moduleOptions.physics.solver!==r&&(d=!1),!0===d)if(void 0!==a.enabled){var u=za.copyAndExtendArray(l,"enabled"),c=this._getValue(u);if(!0===c){var p=this._makeLabel(r,l,!0);this._makeItem(l,p),s=this._handleObject(a,l)||s}else this._makeCheckbox(a,c,l)}else{var m=this._makeLabel(r,l,!0);this._makeItem(l,m),s=this._handleObject(a,l)||s}}else console.error("dont know how to handle",a,r,l)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},n=i;t="false"!==(t="true"===t||t)&&t;for(var o=0;o<e.length;o++)"global"!==e[o]&&(void 0===n[e[o]]&&(n[e[o]]={}),o!==e.length-1?n=n[e[o]]:n[e[o]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+Vs(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}(),Gu=function(t){function e(t,i,n,o){var s,r,a,l,h,d,u,c;if(ba(this,e),(c=ar(this,dr(e).call(this))).initTime=new Date,c.itemsDone=!1,!(Bs(c)instanceof e))throw new SyntaxError("Constructor must be called with the new operator");if(!(Zo(n)||n instanceof Aa||n instanceof Xa)&&n instanceof Object){var p=o;o=n,n=p}o&&o.throttleRedraw&&console.warn('Timeline option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var m=Bs(c);if(c.defaultOptions={autoResize:!0,longSelectPressTime:251,orientation:{axis:"bottom",item:"bottom"},moment:Ta},c.options=za.deepExtend({},c.defaultOptions),c._create(t),!o||o&&void 0===o.rtl){var f;c.dom.root.style.visibility="hidden";for(var g=c.dom.root;!f&&g;)f=window.getComputedStyle(g,null).direction,g=g.parentElement;c.options.rtl=f&&"rtl"==f.toLowerCase()}else c.options.rtl=o.rtl;o&&(o.rollingMode&&(c.options.rollingMode=o.rollingMode),o.onInitialDrawComplete&&(c.options.onInitialDrawComplete=o.onInitialDrawComplete),o.onTimeout&&(c.options.onTimeout=o.onTimeout),o.loadingScreenTemplate&&(c.options.loadingScreenTemplate=o.loadingScreenTemplate));var v=document.createElement("div");if(c.options.loadingScreenTemplate){var b,y=va(b=c.options.loadingScreenTemplate).call(b,Bs(c)),x=y(c.dom.loadingScreen);x instanceof Object&&!(x instanceof Element)?y(v):x instanceof Element?(v.innerHTML="",v.appendChild(x)):null!=x&&(v.innerHTML=x)}return c.dom.loadingScreen.appendChild(v),c.components=[],c.body={dom:c.dom,domProps:c.props,emitter:{on:va(s=c.on).call(s,Bs(c)),off:va(r=c.off).call(r,Bs(c)),emit:va(a=c.emit).call(a,Bs(c))},hiddenDates:[],util:{getScale:function(){return m.timeAxis.step.scale},getStep:function(){return m.timeAxis.step.step},toScreen:va(l=m._toScreen).call(l,m),toGlobalScreen:va(h=m._toGlobalScreen).call(h,m),toTime:va(d=m._toTime).call(d,m),toGlobalTime:va(u=m._toGlobalTime).call(u,m)}},c.range=new ml(c.body,c.options),c.components.push(c.range),c.body.range=c.range,c.timeAxis=new Jh(c.body,c.options),c.timeAxis2=null,c.components.push(c.timeAxis),c.currentTime=new pd(c.body,c.options),c.components.push(c.currentTime),c.itemSet=new uu(c.body,c.options),c.components.push(c.itemSet),c.itemsData=null,c.groupsData=null,c.dom.root.onclick=function(t){m.emit("click",m.getEventProperties(t))},c.dom.root.ondblclick=function(t){m.emit("doubleClick",m.getEventProperties(t))},c.dom.root.oncontextmenu=function(t){m.emit("contextmenu",m.getEventProperties(t))},c.dom.root.onmouseover=function(t){m.emit("mouseOver",m.getEventProperties(t))},window.PointerEvent?(c.dom.root.onpointerdown=function(t){m.emit("mouseDown",m.getEventProperties(t))},c.dom.root.onpointermove=function(t){m.emit("mouseMove",m.getEventProperties(t))},c.dom.root.onpointerup=function(t){m.emit("mouseUp",m.getEventProperties(t))}):(c.dom.root.onmousemove=function(t){m.emit("mouseMove",m.getEventProperties(t))},c.dom.root.onmousedown=function(t){m.emit("mouseDown",m.getEventProperties(t))},c.dom.root.onmouseup=function(t){m.emit("mouseUp",m.getEventProperties(t))}),c.initialFitDone=!1,c.on("changed",(function(){if(null!=m.itemsData){if(!m.initialFitDone&&!m.options.rollingMode)if(m.initialFitDone=!0,null!=m.options.start||null!=m.options.end){if(null==m.options.start||null==m.options.end)var t=m.getItemRange();var e=null!=m.options.start?m.options.start:t.min,i=null!=m.options.end?m.options.end:t.max;m.setWindow(e,i,{animation:!1})}else m.fit({animation:!1});m.initialDrawDone||!m.initialRangeChangeDone&&(m.options.start||m.options.end)&&!m.options.rollingMode||(m.initialDrawDone=!0,m.itemSet.initialDrawDone=!0,m.dom.root.style.visibility="visible",m.dom.loadingScreen.parentNode.removeChild(m.dom.loadingScreen),m.options.onInitialDrawComplete&&qr((function(){return m.options.onInitialDrawComplete()}),0))}})),c.on("destroyTimeline",(function(){m.destroy()})),o&&c.setOptions(o),c.body.emitter.on("fit",(function(t){c._onFit(t),c.redraw()})),n&&c.setGroups(n),i&&c.setItems(i),c._redraw(),c}return pr(e,t),xa(e,[{key:"_createConfigurator",value:function(){return new Lu(this,this.dom.container,wu)}},{key:"redraw",value:function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()}},{key:"setOptions",value:function(t){if(!0===fu.validate(t,xu)&&console.log("%cErrors have been found in the supplied options object.",mu),cd.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var e=this.itemsData;if(e){var i=this.getSelection();this.setItems(null),this.setItems(e.rawDS),this.setSelection(i)}}}},{key:"setItems",value:function(t){var e;this.itemsDone=!1,e=t?Ua(t instanceof Aa||t instanceof Xa?t:new Aa(t)):null,this.itemsData&&this.itemsData.dispose(),this.itemsData=e,this.itemSet&&this.itemSet.setItems(null!=e?e.rawDS:null)}},{key:"setGroups",value:function(t){var e;if(t){var i=function(t){return!1!==t.visible};e=t instanceof Aa||t instanceof Xa?new Xa(t,{filter:i}):new Aa(En(t).call(t,i))}else e=null;this.groupsData instanceof Xa&&this.groupsData.setData(null),this.groupsData=e,this.itemSet.setGroups(e)}},{key:"setData",value:function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)}},{key:"setSelection",value:function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)}},{key:"getSelection",value:function(){return this.itemSet&&this.itemSet.getSelection()||[]}},{key:"focus",value:function(t,e){if(this.itemsData&&null!=t){var i=Zo(t)?t:[t],n=this.itemsData.get(i),o=null,s=null;if(mn(n).call(n,(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===o||e<o)&&(o=e),(null===s||i>s)&&(s=i)})),null!==o&&null!==s){var r=this,a=this.itemSet.items[i[0]],l=-1*this._getScrollTop(),h=null,d=function(){var t=Qu(r,a);t.shouldScroll&&t.itemTop!=h.itemTop&&(r._setScrollTop(-t.scrollOffset),r._redraw())},u=!e||void 0===e.zoom||e.zoom,c=(o+s)/2,p=u?1.1*(s-o):Math.max(this.range.end-this.range.start,1.1*(s-o)),m=!e||void 0===e.animation||e.animation;m||(h={shouldScroll:!1,scrollOffset:-1,itemTop:-1}),this.range.setRange(c-p/2,c+p/2,{animation:m},(function(){d(),qr(d,100)}),(function(t,e,i){var n=Qu(r,a);if(!1!==n&&(h||(h=n),h.itemTop!=n.itemTop||h.shouldScroll)){h.itemTop!=n.itemTop&&n.shouldScroll&&(h=n,l=-1*r._getScrollTop());var o=l,s=h.scrollOffset,d=i?s:o+(s-o)*t;r._setScrollTop(-d),e||r._redraw()}}))}}}},{key:"fit",value:function(t,e){var i,n=!t||void 0===t.animation||t.animation;1===this.itemsData.length&&void 0===this.itemsData.get()[0].end?(i=this.getDataRange(),this.moveTo(i.min.valueOf(),{animation:n},e)):(i=this.getItemRange(),this.range.setRange(i.min,i.max,{animation:n},e))}},{key:"getItemRange",value:function(){var t=this,e=this.getDataRange(),i=null!==e.min?e.min.valueOf():null,n=null!==e.max?e.max.valueOf():null,o=null,s=null;if(null!=i&&null!=n){var r=n-i;r<=0&&(r=10);var a=r/this.props.center.width,l={},h=0;if(mn(za).call(za,this.itemSet.items,(function(t,e){if(t.groupShowing){l[e]=t.redraw(!0),h=l[e].length}})),h>0)for(var d=function(t){mn(za).call(za,l,(function(e){e[t]()}))},u=0;u<h;u++)d(u);if(mn(za).call(za,this.itemSet.items,(function(e){var r,l,h=Mu(e),d=Su(e);t.options.rtl?(r=h-(e.getWidthRight()+10)*a,l=d+(e.getWidthLeft()+10)*a):(r=h-(e.getWidthLeft()+10)*a,l=d+(e.getWidthRight()+10)*a),r<i&&(i=r,o=e),l>n&&(n=l,s=e)})),o&&s){var c=o.getWidthLeft()+10,p=s.getWidthRight()+10,m=this.props.center.width-c-p;m>0&&(this.options.rtl?(i=Mu(o)-p*r/m,n=Su(s)+c*r/m):(i=Mu(o)-c*r/m,n=Su(s)+p*r/m))}}return{min:null!=i?new Date(i):null,max:null!=n?new Date(n):null}}},{key:"getDataRange",value:function(){var t,e=null,i=null;this.itemsData&&mn(t=this.itemsData).call(t,(function(t){var n=za.convert(t.start,"Date").valueOf(),o=za.convert(null!=t.end?t.end:t.start,"Date").valueOf();(null===e||n<e)&&(e=n),(null===i||o>i)&&(i=o)}));return{min:null!=e?new Date(e):null,max:null!=i?new Date(i):null}}},{key:"getEventProperties",value:function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,n=this.dom.centerContainer.getBoundingClientRect(),o=this.options.rtl?n.right-e:e-n.left,s=i-n.top,r=this.itemSet.itemFromTarget(t),a=this.itemSet.groupFromTarget(t),l=ud.customTimeFromTarget(t),h=this.itemSet.options.snap||null,d=this.body.util.getScale(),u=this.body.util.getStep(),c=this._toTime(o),p=h?h(c,d,u):c,m=za.getTarget(t),f=null;return null!=r?f="item":null!=l?f="custom-time":za.hasParent(m,this.timeAxis.dom.foreground)||this.timeAxis2&&za.hasParent(m,this.timeAxis2.dom.foreground)?f="axis":za.hasParent(m,this.itemSet.dom.labelSet)?f="group-label":za.hasParent(m,this.currentTime.bar)?f="current-time":za.hasParent(m,this.dom.center)&&(f="background"),{event:t,item:r?r.id:null,isCluster:!!r&&!!r.isCluster,items:r?r.items||[]:null,group:a?a.groupId:null,customTime:l?l.options.id:null,what:f,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:s,time:c,snappedTime:p}}},{key:"toggleRollingMode",value:function(){this.range.rolling?this.range.stopRolling():(null==this.options.rollingMode&&this.setOptions(this.options),this.range.startRolling())}},{key:"_redraw",value:function(){cd.prototype._redraw.call(this)}},{key:"_onFit",value:function(t){var e=t.start,i=t.end,n=t.animation;i?this.range.setRange(e,i,{animation:n}):this.moveTo(e.valueOf(),{animation:n})}}]),e}(cd);function Mu(t){return za.convert(t.data.start,"Date").valueOf()}function Su(t){var e=null!=t.data.end?t.data.end:t.data.start;return za.convert(e,"Date").valueOf()}function Qu(t,e){if(!e.parent)return!1;var i=t.options.rtl?t.props.rightContainer.height:t.props.leftContainer.height,n=t.props.center.height,o=e.parent,s=o.top,r=!0,a=t.timeAxis.options.orientation.axis,l=function(){return"bottom"==a?o.height-e.top-e.height:e.top},h=-1*t._getScrollTop(),d=s+l(),u=e.height;return d<h?s+i<=s+l()+u&&(s+=l()-t.itemSet.options.margin.item.vertical):d+u>h+i?s+=l()+u-i+t.itemSet.options.margin.item.vertical:r=!1,{shouldScroll:r,scrollOffset:s=Math.min(s,n-i),itemTop:d}}var Nu=function(){function t(e,i,n,o,s,r){var a=arguments.length>6&&void 0!==arguments[6]&&arguments[6],l=arguments.length>7&&void 0!==arguments[7]&&arguments[7];if(ba(this,t),this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=s,this.majorCharHeight=r,this._start=e,this._end=i,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=a,this.autoScaleStart=n,this.autoScaleEnd=o,this.formattingFunction=l,n||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>h.magnitudefactor*h.minorSteps[h.minorStepIdx]*.5?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};n&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}return xa(t,[{key:"setCharHeight",value:function(t){this.majorCharHeight=t}},{key:"setHeight",value:function(t){this.containerHeight=t}},{key:"determineScale",value:function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var n=0;i<0&&(n=i);for(var o=!1,s=n;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){if(this.magnitudefactor*this.minorSteps[r]>=e){o=!0,this.minorStepIdx=r;break}}if(!0===o)break}}},{key:"is_major",value:function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])==0}},{key:"getStep",value:function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]}},{key:"getFirstMajor",value:function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)}},{key:"formatValue",value:function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?"".concat(e):"string"==typeof e?e:t.toPrecision(5)}},{key:"getLines",value:function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,n=this._start+i;this._end-n>1e-5;n+=e)n!=this._start&&t.push({major:this.is_major(n),y:this.convertValue(n),val:this.formatValue(n)});return t}},{key:"followScale",value:function(t){var e=this.minorStepIdx,i=this._start,n=this._end,o=this,s=function(){o.magnitudefactor*=2},r=function(){o.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e||s(),s()):(this.minorStepIdx=2,1==e||r(),r()));for(var a=t.convertValue(0),l=t.getStep()*t.scale,h=!1,d=0;!h&&d++<5;){this.scale=l/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var u=this.containerHeight/this.scale;this._start=i,this._end=this._start+u;var c=this._end*this.scale,p=this.magnitudefactor*this.majorSteps[this.minorStepIdx],m=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var f=a-c;this._end+=f/this.scale,this._start=this._end-u}else this.autoScaleStart?(this._start-=m/this.scale,this._end=this._start+u):(this._start+=p-m/this.scale,this._end=this._start+u);if(!this.autoScaleEnd&&this._end>n+1e-5)r(),h=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),h=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&u<n-i?(s(),h=!1):h=!0}}}},{key:"convertValue",value:function(t){return this.containerHeight-(t-this._start)*this.scale}},{key:"screenToValue",value:function(t){return(this.containerHeight-t)/this.scale+this._start}}]),t}();Ph(".vis-panel.vis-background.vis-horizontal .vis-grid.vis-horizontal{position:absolute;width:100%;height:0;border-bottom:1px solid}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-minor{border-color:#e5e5e5}.vis-panel.vis-background.vis-horizontal .vis-grid.vis-major{border-color:#bfbfbf}.vis-data-axis .vis-y-axis.vis-major{width:100%;position:absolute;color:#4d4d4d;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-major.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-minor{position:absolute;width:100%;color:#bebebe;white-space:nowrap}.vis-data-axis .vis-y-axis.vis-minor.vis-measure{padding:0;margin:0;border:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title{position:absolute;color:#4d4d4d;white-space:nowrap;bottom:20px;text-align:center}.vis-data-axis .vis-y-axis.vis-title.vis-measure{padding:0;margin:0;visibility:hidden;width:auto}.vis-data-axis .vis-y-axis.vis-title.vis-left{bottom:0;-webkit-transform-origin:left top;-moz-transform-origin:left top;-ms-transform-origin:left top;-o-transform-origin:left top;transform-origin:left bottom;-webkit-transform:rotate(-90deg);-moz-transform:rotate(-90deg);-ms-transform:rotate(-90deg);-o-transform:rotate(-90deg);transform:rotate(-90deg)}.vis-data-axis .vis-y-axis.vis-title.vis-right{bottom:0;-webkit-transform-origin:right bottom;-moz-transform-origin:right bottom;-ms-transform-origin:right bottom;-o-transform-origin:right bottom;transform-origin:right bottom;-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.vis-legend{background-color:rgba(247,252,255,.65);padding:5px;border:1px solid #b3b3b3;box-shadow:2px 2px 10px hsla(0,0%,60.4%,.55)}.vis-legend-text{white-space:nowrap;display:inline-block}\n/*# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbImRhdGFheGlzLmNzcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxrRUFDRSxpQkFBa0IsQ0FDbEIsVUFBVyxDQUNYLFFBQVMsQ0FDVCx1QkFDRixDQUVBLDZEQUNFLG9CQUNGLENBRUEsNkRBQ0Usb0JBQ0YsQ0FHQSxxQ0FDRSxVQUFXLENBQ1gsaUJBQWtCLENBQ2xCLGFBQWMsQ0FDZCxrQkFDRixDQUVBLGlEQUNFLFNBQVUsQ0FDVixRQUFTLENBQ1QsUUFBUyxDQUNULGlCQUFrQixDQUNsQixVQUNGLENBR0EscUNBQ0UsaUJBQWtCLENBQ2xCLFVBQVcsQ0FDWCxhQUFjLENBQ2Qsa0JBQ0YsQ0FFQSxpREFDRSxTQUFVLENBQ1YsUUFBUyxDQUNULFFBQVMsQ0FDVCxpQkFBa0IsQ0FDbEIsVUFDRixDQUVBLHFDQUNFLGlCQUFrQixDQUNsQixhQUFjLENBQ2Qsa0JBQW1CLENBQ25CLFdBQVksQ0FDWixpQkFDRixDQUVBLGlEQUNFLFNBQVUsQ0FDVixRQUFTLENBQ1QsaUJBQWtCLENBQ2xCLFVBQ0YsQ0FFQSw4Q0FDRSxRQUFTLENBQ1QsaUNBQWtDLENBQ2xDLDhCQUErQixDQUMvQiw2QkFBOEIsQ0FDOUIsNEJBQTZCLENBQzdCLDRCQUE2QixDQUM3QixnQ0FBaUMsQ0FDakMsNkJBQThCLENBQzlCLDRCQUE2QixDQUM3QiwyQkFBNEIsQ0FDNUIsd0JBQ0YsQ0FFQSwrQ0FDRSxRQUFTLENBQ1QscUNBQXNDLENBQ3RDLGtDQUFtQyxDQUNuQyxpQ0FBa0MsQ0FDbEMsZ0NBQWlDLENBQ2pDLDZCQUE4QixDQUM5QiwrQkFBZ0MsQ0FDaEMsNEJBQTZCLENBQzdCLDJCQUE0QixDQUM1QiwwQkFBMkIsQ0FDM0IsdUJBQ0YsQ0FFQSxZQUNFLHNDQUEyQyxDQUMzQyxXQUFZLENBQ1osd0JBQXlCLENBQ3pCLDRDQUNGLENBRUEsaUJBRUUsa0JBQW1CLENBQ25CLG9CQUNGIiwiZmlsZSI6ImRhdGFheGlzLmNzcyIsInNvdXJjZXNDb250ZW50IjpbIlxuLnZpcy1wYW5lbC52aXMtYmFja2dyb3VuZC52aXMtaG9yaXpvbnRhbCAudmlzLWdyaWQudmlzLWhvcml6b250YWwge1xuICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gIHdpZHRoOiAxMDAlO1xuICBoZWlnaHQ6IDA7XG4gIGJvcmRlci1ib3R0b206IDFweCBzb2xpZDtcbn1cblxuLnZpcy1wYW5lbC52aXMtYmFja2dyb3VuZC52aXMtaG9yaXpvbnRhbCAudmlzLWdyaWQudmlzLW1pbm9yIHtcbiAgYm9yZGVyLWNvbG9yOiAjZTVlNWU1O1xufVxuXG4udmlzLXBhbmVsLnZpcy1iYWNrZ3JvdW5kLnZpcy1ob3Jpem9udGFsIC52aXMtZ3JpZC52aXMtbWFqb3Ige1xuICBib3JkZXItY29sb3I6ICNiZmJmYmY7XG59XG5cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLW1ham9yIHtcbiAgd2lkdGg6IDEwMCU7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgY29sb3I6ICM0ZDRkNGQ7XG4gIHdoaXRlLXNwYWNlOiBub3dyYXA7XG59XG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy1tYWpvci52aXMtbWVhc3VyZSB7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgYm9yZGVyOiAwO1xuICB2aXNpYmlsaXR5OiBoaWRkZW47XG4gIHdpZHRoOiBhdXRvO1xufVxuXG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy1taW5vciB7XG4gIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgd2lkdGg6IDEwMCU7XG4gIGNvbG9yOiAjYmViZWJlO1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xufVxuXG4udmlzLWRhdGEtYXhpcyAudmlzLXktYXhpcy52aXMtbWlub3IudmlzLW1lYXN1cmUge1xuICBwYWRkaW5nOiAwO1xuICBtYXJnaW46IDA7XG4gIGJvcmRlcjogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICB3aWR0aDogYXV0bztcbn1cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLXRpdGxlIHtcbiAgcG9zaXRpb246IGFic29sdXRlO1xuICBjb2xvcjogIzRkNGQ0ZDtcbiAgd2hpdGUtc3BhY2U6IG5vd3JhcDtcbiAgYm90dG9tOiAyMHB4O1xuICB0ZXh0LWFsaWduOiBjZW50ZXI7XG59XG5cbi52aXMtZGF0YS1heGlzIC52aXMteS1heGlzLnZpcy10aXRsZS52aXMtbWVhc3VyZSB7XG4gIHBhZGRpbmc6IDA7XG4gIG1hcmdpbjogMDtcbiAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICB3aWR0aDogYXV0bztcbn1cblxuLnZpcy1kYXRhLWF4aXMgLnZpcy15LWF4aXMudmlzLXRpdGxlLnZpcy1sZWZ0IHtcbiAgYm90dG9tOiAwO1xuICAtd2Via2l0LXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICAtbW96LXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICAtbXMtdHJhbnNmb3JtLW9yaWdpbjogbGVmdCB0b3A7XG4gIC1vLXRyYW5zZm9ybS1vcmlnaW46IGxlZnQgdG9wO1xuICB0cmFuc2Zvcm0tb3JpZ2luOiBsZWZ0IGJvdHRvbTtcbiAgLXdlYmtpdC10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICAtbW96LXRyYW5zZm9ybTogcm90YXRlKC05MGRlZyk7XG4gIC1tcy10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICAtby10cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xuICB0cmFuc2Zvcm06IHJvdGF0ZSgtOTBkZWcpO1xufVxuXG4udmlzLWRhdGEtYXhpcyAudmlzLXktYXhpcy52aXMtdGl0bGUudmlzLXJpZ2h0IHtcbiAgYm90dG9tOiAwO1xuICAtd2Via2l0LXRyYW5zZm9ybS1vcmlnaW46IHJpZ2h0IGJvdHRvbTtcbiAgLW1vei10cmFuc2Zvcm0tb3JpZ2luOiByaWdodCBib3R0b207XG4gIC1tcy10cmFuc2Zvcm0tb3JpZ2luOiByaWdodCBib3R0b207XG4gIC1vLXRyYW5zZm9ybS1vcmlnaW46IHJpZ2h0IGJvdHRvbTtcbiAgdHJhbnNmb3JtLW9yaWdpbjogcmlnaHQgYm90dG9tO1xuICAtd2Via2l0LXRyYW5zZm9ybTogcm90YXRlKDkwZGVnKTtcbiAgLW1vei10cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7XG4gIC1tcy10cmFuc2Zvcm06IHJvdGF0ZSg5MGRlZyk7XG4gIC1vLXRyYW5zZm9ybTogcm90YXRlKDkwZGVnKTtcbiAgdHJhbnNmb3JtOiByb3RhdGUoOTBkZWcpO1xufVxuXG4udmlzLWxlZ2VuZCB7XG4gIGJhY2tncm91bmQtY29sb3I6IHJnYmEoMjQ3LCAyNTIsIDI1NSwgMC42NSk7XG4gIHBhZGRpbmc6IDVweDtcbiAgYm9yZGVyOiAxcHggc29saWQgI2IzYjNiMztcbiAgYm94LXNoYWRvdzogMnB4IDJweCAxMHB4IHJnYmEoMTU0LCAxNTQsIDE1NCwgMC41NSk7XG59XG5cbi52aXMtbGVnZW5kLXRleHQge1xuICAvKmZvbnQtc2l6ZTogMTBweDsqL1xuICB3aGl0ZS1zcGFjZTogbm93cmFwO1xuICBkaXNwbGF5OiBpbmxpbmUtYmxvY2tcbn0iXX0= */");var Fu=function(t){function e(t,i,n,o){var s;ba(this,e),(s=ar(this,dr(e).call(this))).id=Id(),s.body=t,s.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return"".concat(Pa(t.toPrecision(3)))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return"".concat(Pa(t.toPrecision(3)))},title:{text:void 0,style:void 0}}},s.linegraphOptions=o,s.linegraphSVG=n,s.props={},s.DOMelements={lines:{},labels:{},title:{}},s.dom={},s.scale=void 0,s.range={start:0,end:0},s.options=za.extend({},s.defaultOptions),s.conversionFactor=1,s.setOptions(i),s.width=Number("".concat(s.options.width).replace("px","")),s.minWidth=s.width,s.height=s.linegraphSVG.getBoundingClientRect().height,s.hidden=!1,s.stepPixels=25,s.zeroCrossing=-1,s.amountOfSteps=-1,s.lineOffset=0,s.master=!0,s.masterAxis=null,s.svgElements={},s.iconsRemoved=!1,s.groups={},s.amountOfGroups=0,s._create(),null==s.scale&&s._redrawLabels(),s.framework={svg:s.svg,svgElements:s.svgElements,options:s.options,groups:s.groups};var r=Bs(s);return s.body.emitter.on("verticalDrag",(function(){r.dom.lineContainer.style.top="".concat(r.body.domProps.scrollTop,"px")})),s}return pr(e,t),xa(e,[{key:"addGroup",value:function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1}},{key:"updateGroup",value:function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e}},{key:"removeGroup",value:function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)}},{key:"setOptions",value:function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);za.selectiveDeepExtend(["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"],this.options,t),this.minWidth=Number("".concat(this.options.width).replace("px","")),!0===e&&this.dom.frame&&(this.hide(),this.show())}}},{key:"_create",value:function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.dom.lineContainer.style.visibility="visible",this.dom.lineContainer.style.display="block",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)}},{key:"_redrawGroupIcons",value:function(){var t;Cs(this.svgElements);var e=this.options.iconWidth,i=11.5;t="left"===this.options.orientation?4:this.width-e-4;var n=Yi(this.groups);wr(n).call(n,(function(t,e){return t<e?-1:1}));var o=!0,s=!1,r=void 0;try{for(var a,l=Fi(n);!(o=(a=l.next()).done);o=!0){var h=a.value;!0!==this.groups[h].visible||void 0!==this.linegraphOptions.visibility[h]&&!0!==this.linegraphOptions.visibility[h]||(this.groups[h].getLegend(e,15,this.framework,t,i),i+=19)}}catch(t){s=!0,r=t}finally{try{o||null==l.return||l.return()}finally{if(s)throw r}}Ls(this.svgElements),this.iconsRemoved=!1}},{key:"_cleanupIcons",value:function(){!1===this.iconsRemoved&&(Cs(this.svgElements),Ls(this.svgElements),this.iconsRemoved=!0)}},{key:"show",value:function(){this.hidden=!1,this.dom.frame.parentNode||("left"===this.options.orientation?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.right.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer),this.dom.lineContainer.style.display="block"}},{key:"hide",value:function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.style.display="none"}},{key:"setRange",value:function(t,e){this.range.start=t,this.range.end=e}},{key:"redraw",value:function(){var t=!1,e=0;for(var i in this.dom.lineContainer.style.top="".concat(this.body.domProps.scrollTop,"px"),this.groups)this.groups.hasOwnProperty(i)&&(!0!==this.groups[i].visible||void 0!==this.linegraphOptions.visibility[i]&&!0!==this.linegraphOptions.visibility[i]||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height="".concat(this.height,"px"),this.width=!0===this.options.visible?Number("".concat(this.options.width).replace("px","")):0;var n=this.props,o=this.dom.frame;o.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels,l=this.body.dom.backgroundHorizontal.offsetWidth;n.minorLabelHeight=r?n.minorCharHeight:0,n.majorLabelHeight=a?n.majorCharHeight:0,n.minorLineWidth=l-this.lineOffset-this.width+2*this.options.minorLinesOffset,n.minorLineHeight=1,n.majorLineWidth=l-this.lineOffset-this.width+2*this.options.majorLinesOffset,n.majorLineHeight=1,"left"===s?(o.style.top="0",o.style.left="0",o.style.bottom="",o.style.width="".concat(this.width,"px"),o.style.height="".concat(this.height,"px"),this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(o.style.top="",o.style.bottom="0",o.style.left="0",o.style.width="".concat(this.width,"px"),o.style.height="".concat(this.height,"px"),this.props.width=this.body.domProps.right.width,this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,!0===this.options.icons?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t}},{key:"_redrawLabels",value:function(){var t=this,e=!1;Cs(this.DOMelements.lines),Cs(this.DOMelements.labels);var i=this.options.orientation,n=null!=this.options[i].range?this.options[i].range:{},o=!0;null!=n.max&&(this.range.end=n.max,o=!1);var s=!0;null!=n.min&&(this.range.start=n.min,s=!1),this.scale=new Nu(this.range.start,this.range.end,s,o,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),!1===this.master&&null!=this.masterAxis?(this.scale.followScale(this.masterAxis.scale),this.dom.lineContainer.style.display="none"):this.dom.lineContainer.style.display="block",this.maxLabelSize=0;var r=this.scale.getLines();mn(r).call(r,(function(e){var n=e.y,o=e.major;t.options.showMinorLabels&&!1===o&&t._redrawLabel(n-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),o&&n>=0&&t._redrawLabel(n-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),!0===t.master&&(o?t._redrawLine(n,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(n,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))}));var a=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(a=this.props.titleCharHeight);var l=!0===this.options.icons?Math.max(this.options.iconWidth,a)+this.options.labelOffsetX+15:a+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-l&&!0===this.options.visible?(this.width=this.maxLabelSize+l,this.options.width="".concat(this.width,"px"),Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-l&&!0===this.options.visible&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+l),this.options.width="".concat(this.width,"px"),Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),this.redraw(),e=!0):(Ls(this.DOMelements.lines),Ls(this.DOMelements.labels),e=!1),e}},{key:"convertValue",value:function(t){return this.scale.convertValue(t)}},{key:"screenToValue",value:function(t){return this.scale.screenToValue(t)}},{key:"_redrawLabel",value:function(t,e,i,n,o){var s=Ss("div",this.DOMelements.labels,this.dom.frame);s.className=n,s.innerHTML=e,"left"===i?(s.style.left="-".concat(this.options.labelOffsetX,"px"),s.style.textAlign="right"):(s.style.right="-".concat(this.options.labelOffsetX,"px"),s.style.textAlign="left"),s.style.top="".concat(t-.5*o+this.options.labelOffsetY,"px"),e+="";var r=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*r&&(this.maxLabelSize=e.length*r)}},{key:"_redrawLine",value:function(t,e,i,n,o){if(!0===this.master){var s=Ss("div",this.DOMelements.lines,this.dom.lineContainer);s.className=i,s.innerHTML="","left"===e?s.style.left="".concat(this.width-n,"px"):s.style.right="".concat(this.width-n,"px"),s.style.width="".concat(o,"px"),s.style.top="".concat(t,"px")}}},{key:"_redrawTitle",value:function(t){if(Cs(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=Ss("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-".concat(t),e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&za.addCssText(e,this.options[t].title.style),"left"===t?e.style.left="".concat(this.props.titleCharHeight,"px"):e.style.right="".concat(this.props.titleCharHeight,"px"),e.style.width="".concat(this.height,"px")}Ls(this.DOMelements.title)}},{key:"_calculateCharSize",value:function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),n=document.createElement("div");n.className="vis-y-axis vis-major vis-measure",n.appendChild(i),this.dom.frame.appendChild(n),this.props.majorCharHeight=n.clientHeight,this.props.majorCharWidth=n.clientWidth,this.dom.frame.removeChild(n)}if(!("titleCharHeight"in this.props)){var o=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(o),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}}}]),e}(Ja);function Zu(t,e){}function Bu(t,e){return{style:(e=void 0===e?{}:e).style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function Ou(t,e){}function Wu(t,e){}function Au(t,e,i,n){this.id=e;this.options=za.selectiveBridgeObject(["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"],i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=n,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible||t.visible}function Xu(t,e,i,n){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=za.extend({},this.defaultOptions),this.linegraphOptions=n,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}Zu.draw=function(t,e,i,n){n=n||0;for(var o=function(t,e){var i=void 0;t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender);e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender);return i}(i,e),s=0;s<t.length;s++)if(o){var r=o(t[s],e);!0!==r&&"object"!==zo(r)||Qs(t[s].screen_x+n,t[s].screen_y,Bu(e,r),i.svgElements,i.svg,t[s].label)}else Qs(t[s].screen_x+n,t[s].screen_y,Bu(e),i.svgElements,i.svg,t[s].label)},Zu.drawIcon=function(t,e,i,n,o,s){var r=.5*o,a=Ms("rect",s.svgElements,s.svg);a.setAttributeNS(null,"x",e),a.setAttributeNS(null,"y",i-r),a.setAttributeNS(null,"width",n),a.setAttributeNS(null,"height",2*r),a.setAttributeNS(null,"class","vis-outline"),Qs(e+.5*n,i,Bu(t),s.svgElements,s.svg)},Ou.drawIcon=function(t,e,i,n,o,s){var r=.5*o,a=Ms("rect",s.svgElements,s.svg);a.setAttributeNS(null,"x",e),a.setAttributeNS(null,"y",i-r),a.setAttributeNS(null,"width",n),a.setAttributeNS(null,"height",2*r),a.setAttributeNS(null,"class","vis-outline");var l=Math.round(.3*n),h=t.options.barChart.width/l,d=Math.round(.4*o),u=Math.round(.75*o),c=Math.round((n-2*l)/3);if(Ns(e+.5*l+c,i+r-d-1,l,d,t.className+" vis-bar",s.svgElements,s.svg,t.style),Ns(e+1.5*l+c+2,i+r-u-1,l,u,t.className+" vis-bar",s.svgElements,s.svg,t.style),1==t.options.drawPoints.enabled){var p={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/h,className:t.className};Qs(e+.5*l+c,i+r-d-1,p,s.svgElements,s.svg),Qs(e+1.5*l+c+2,i+r-u-1,p,s.svgElements,s.svg)}},Ou.draw=function(t,e,i){var n,o,s,r,a,l,h=[],d={},u=0;for(a=0;a<t.length;a++)if("bar"===(r=i.groups[t[a]]).options.style&&!0===r.visible&&(void 0===i.options.groups.visibility[t[a]]||!0===i.options.groups.visibility[t[a]]))for(l=0;l<e[t[a]].length;l++)h.push({screen_x:e[t[a]][l].screen_x,screen_end:e[t[a]][l].screen_end,screen_y:e[t[a]][l].screen_y,x:e[t[a]][l].x,end:e[t[a]][l].end,y:e[t[a]][l].y,groupId:t[a],label:e[t[a]][l].label}),u+=1;if(0!==u)for(wr(h).call(h,(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x})),Ou._getDataIntersections(d,h),a=0;a<h.length;a++){var c=null!=(r=i.groups[h[a].groupId]).options.barChart.minWidth?r.options.barChart.minWidth:.1*r.options.barChart.width,p=0;if(void 0===d[o=h[a].screen_x])a+1<h.length&&(n=Math.abs(h[a+1].screen_x-o)),s=Ou._getSafeDrawData(n,r,c);else{var m=a+(d[o].amount-d[o].resolved);m<h.length&&(n=Math.abs(h[m].screen_x-o)),s=Ou._getSafeDrawData(n,r,c),d[o].resolved+=1,!0===r.options.stack&&!0!==r.options.excludeFromStacking?h[a].screen_y<r.zeroPosition?(p=d[o].accumulatedNegative,d[o].accumulatedNegative+=r.zeroPosition-h[a].screen_y):(p=d[o].accumulatedPositive,d[o].accumulatedPositive+=r.zeroPosition-h[a].screen_y):!0===r.options.barChart.sideBySide&&(s.width=s.width/d[o].amount,s.offset+=d[o].resolved*s.width-.5*s.width*(d[o].amount+1))}var f=s.width,g=h[a].screen_x;if(null!=h[a].screen_end?g+=.5*(f=h[a].screen_end-h[a].screen_x):g+=s.offset,Ns(g,h[a].screen_y-p,f,r.zeroPosition-h[a].screen_y,r.className+" vis-bar",i.svgElements,i.svg,r.style),!0===r.options.drawPoints.enabled){var v={screen_x:h[a].screen_x,screen_y:h[a].screen_y-p,x:h[a].x,y:h[a].y,groupId:h[a].groupId,label:h[a].label};Zu.draw([v],r,i,s.offset)}}},Ou._getDataIntersections=function(t,e){for(var i,n=0;n<e.length;n++)n+1<e.length&&(i=Math.abs(e[n+1].screen_x-e[n].screen_x)),n>0&&(i=Math.min(i,Math.abs(e[n-1].screen_x-e[n].screen_x))),0===i&&(void 0===t[e[n].screen_x]&&(t[e[n].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[n].screen_x].amount+=1)},Ou._getSafeDrawData=function(t,e,i){var n,o;return t<e.options.barChart.width&&t>0?(n=t<i?i:t,o=0,"left"===e.options.barChart.align?o-=.5*t:"right"===e.options.barChart.align&&(o+=.5*t)):(n=e.options.barChart.width,o=0,"left"===e.options.barChart.align?o-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(o+=.5*e.options.barChart.width)),{width:n,offset:o}},Ou.getStackedYRange=function(t,e,i,n,o){if(t.length>0){wr(t).call(t,(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}));var s={};Ou._getDataIntersections(s,t),e[n]=Ou._getStackedYRange(s,t),e[n].yAxisOrientation=o,i.push(n)}},Ou._getStackedYRange=function(t,e){for(var i,n=e[0].screen_y,o=e[0].screen_y,s=0;s<e.length;s++)void 0===t[i=e[s].screen_x]?(n=n>e[s].screen_y?e[s].screen_y:n,o=o<e[s].screen_y?e[s].screen_y:o):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(n=(n=n>t[r].accumulatedNegative?t[r].accumulatedNegative:n)>t[r].accumulatedPositive?t[r].accumulatedPositive:n,o=(o=o<t[r].accumulatedNegative?t[r].accumulatedNegative:o)<t[r].accumulatedPositive?t[r].accumulatedPositive:o);return{min:n,max:o}},Wu.calcPath=function(t,e){if(null!=t&&t.length>0){return 1==e.options.interpolation.enabled?Wu._catmullRom(t,e):Wu._linear(t)}},Wu.drawIcon=function(t,e,i,n,o,s){var r,a,l=.5*o,h=Ms("rect",s.svgElements,s.svg);(h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-l),h.setAttributeNS(null,"width",n),h.setAttributeNS(null,"height",2*l),h.setAttributeNS(null,"class","vis-outline"),(r=Ms("path",s.svgElements,s.svg)).setAttributeNS(null,"class",t.className),void 0!==t.style&&r.setAttributeNS(null,"style",t.style),r.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+n)+","+i),1==t.options.shaded.enabled&&(a=Ms("path",s.svgElements,s.svg),"top"==t.options.shaded.orientation?a.setAttributeNS(null,"d","M"+e+", "+(i-l)+"L"+e+","+i+" L"+(e+n)+","+i+" L"+(e+n)+","+(i-l)):a.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+l)+" L"+(e+n)+","+(i+l)+"L"+(e+n)+","+i),a.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&a.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled)&&Qs(e+.5*n,i,{style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className},s.svgElements,s.svg)},Wu.drawShading=function(t,e,i,n){if(1==e.options.shaded.enabled){var o,s=Number(n.svg.style.height.replace("px","")),r=Ms("path",n.svgElements,n.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var l=0;l="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),o="group"==e.options.shaded.orientation&&null!=i&&null!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+l+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",o)}},Wu.draw=function(t,e,i){if(null!=t&&null!=t){var n=Ms("path",i.svgElements,i.svg);n.setAttributeNS(null,"class",e.className),void 0!==e.style&&n.setAttributeNS(null,"style",e.style);var o="L";1==e.options.interpolation.enabled&&(o="C"),n.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,o,!1))}},Wu.serializePath=function(t,e,i){if(t.length<2)return"";var n,o=e;if(i)for(n=t.length-2;n>0;n--)o+=t[n][0]+","+t[n][1]+" ";else for(n=1;n<t.length;n++)o+=t[n][0]+","+t[n][1]+" ";return o},Wu._catmullRomUniform=function(t){var e,i,n,o,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var l=t.length,h=0;h<l-1;h++)e=0==h?t[0]:t[h-1],i=t[h],n=t[h+1],o=h+2<l?t[h+2]:n,s={screen_x:(-e.screen_x+6*i.screen_x+n.screen_x)*(1/6),screen_y:(-e.screen_y+6*i.screen_y+n.screen_y)*(1/6)},r={screen_x:(i.screen_x+6*n.screen_x-o.screen_x)*(1/6),screen_y:(i.screen_y+6*n.screen_y-o.screen_y)*(1/6)},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([n.screen_x,n.screen_y]);return a},Wu._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var n,o,s,r,a,l,h,d,u,c,p,m,f,g,v,b,y,x,w,_=[];_.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var k=t.length,D=0;D<k-1;D++)n=0==D?t[0]:t[D-1],o=t[D],s=t[D+1],r=D+2<k?t[D+2]:s,h=Math.sqrt(Math.pow(n.screen_x-o.screen_x,2)+Math.pow(n.screen_y-o.screen_y,2)),d=Math.sqrt(Math.pow(o.screen_x-s.screen_x,2)+Math.pow(o.screen_y-s.screen_y,2)),u=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),g=Math.pow(u,i),b=Math.pow(u,2*i),v=Math.pow(d,i),y=Math.pow(d,2*i),w=Math.pow(h,i),c=2*(x=Math.pow(h,2*i))+3*w*v+y,p=2*b+3*g*v+y,(m=3*w*(w+v))>0&&(m=1/m),(f=3*g*(g+v))>0&&(f=1/f),a={screen_x:(-y*n.screen_x+c*o.screen_x+x*s.screen_x)*m,screen_y:(-y*n.screen_y+c*o.screen_y+x*s.screen_y)*m},l={screen_x:(b*o.screen_x+p*s.screen_x-y*r.screen_x)*f,screen_y:(b*o.screen_y+p*s.screen_y-y*r.screen_y)*f},0==a.screen_x&&0==a.screen_y&&(a=o),0==l.screen_x&&0==l.screen_y&&(l=s),_.push([a.screen_x,a.screen_y]),_.push([l.screen_x,l.screen_y]),_.push([s.screen_x,s.screen_y]);return _},Wu._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},Au.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==wr(this.options)&&za.insertSort(this.itemsData,(function(t,e){return t.x>e.x?1:-1}))):this.itemsData=[]},Au.prototype.getItems=function(){return this.itemsData},Au.prototype.setZeroPosition=function(t){this.zeroPosition=t},Au.prototype.setOptions=function(t){if(void 0!==t){za.selectiveDeepExtend(["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"],this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),za.mergeOptions(this.options,t,"interpolation"),za.mergeOptions(this.options,t,"drawPoints"),za.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==zo(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},Au.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible||t.visible,this.style=t.style,this.setOptions(t.options)},Au.prototype.getLegend=function(t,e,i,n,o){null!=i&&null!=i||(i={svg:document.createElementNS("http://www.w3.org/2000/svg","svg"),svgElements:{},options:this.options,groups:[this]});switch(null!=n&&null!=n||(n=0),null!=o&&null!=o||(o=.5*e),this.options.style){case"line":Wu.drawIcon(this,n,o,t,e,i);break;case"points":case"point":Zu.drawIcon(this,n,o,t,e,i);break;case"bar":Ou.drawIcon(this,n,o,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},Au.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,n=0;n<t.length;n++)e=e>t[n].y?t[n].y:e,i=i<t[n].y?t[n].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},Xu.prototype=new Ja,Xu.prototype.clear=function(){this.groups={},this.amountOfGroups=0},Xu.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},Xu.prototype.updateGroup=function(t,e){this.groups[t]=e},Xu.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},Xu.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},Xu.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},Xu.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},Xu.prototype.setOptions=function(t){za.selectiveDeepExtend(["enabled","orientation","icons","left","right"],this.options,t)},Xu.prototype.redraw=function(){var t=0,e=Yi(this.groups);wr(e).call(e,(function(t,e){return t<e?-1:1}));for(var i=0;i<e.length;i++){var n=e[i];1!=this.groups[n].visible||void 0!==this.linegraphOptions.visibility[n]&&1!=this.linegraphOptions.visibility[n]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var o=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+o+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());var s="";for(i=0;i<e.length;i++)n=e[i],1!=this.groups[n].visible||void 0!==this.linegraphOptions.visibility[n]&&1!=this.linegraphOptions.visibility[n]||(s+=this.groups[n].content+"<br />");this.dom.textArea.innerHTML=s,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},Xu.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=Yi(this.groups);wr(t).call(t,(function(t,e){return t<e?-1:1})),Gs(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),n=i,o=this.options.iconSize,s=.75*this.options.iconSize,r=i+.5*s+3;this.svg.style.width=o+5+i+"px";for(var a=0;a<t.length;a++){var l=t[a];1!=this.groups[l].visible||void 0!==this.linegraphOptions.visibility[l]&&1!=this.linegraphOptions.visibility[l]||(this.groups[l].getLegend(o,s,this.framework,n,r),r+=s+this.options.iconSpacing)}}};function Tu(t,e){this.id=Id(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=za.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,n){i._onAdd(e.items)},update:function(t,e,n){i._onUpdate(e.items)},remove:function(t,e,n){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,n){i._onAddGroups(e.items)},update:function(t,e,n){i._onUpdateGroups(e.items)},remove:function(t,e,n){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",(function(){i.lastStart=i.body.range.start,i.svg.style.left=za.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)})),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}Tu.prototype=new Ja,Tu.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new Fu(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new Fu(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new Xu(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new Xu(this.body,this.options.legend,"right",this.options.groups),this.show()},Tu.prototype.setOptions=function(t){if(t){void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&Nn((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),za.selectiveDeepExtend(["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"],this.options,t),za.mergeOptions(this.options,t,"interpolation"),za.mergeOptions(this.options,t,"drawPoints"),za.mergeOptions(this.options,t,"shaded"),za.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==zo(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty("__ungrouped__")&&this.groups.__ungrouped__.setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},Tu.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},Tu.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},Tu.prototype.setItems=function(t){var e,i=this,n=this.itemsData;if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=Ua(t)}else this.itemsData=null;if(n&&(mn(za).call(za,this.itemListeners,(function(t,e){n.off(e,t)})),n.dispose(),e=n.getIds(),this._onRemove(e)),this.itemsData){var o=this.id;mn(za).call(za,this.itemListeners,(function(t,e){i.itemsData.on(e,t,o)})),e=this.itemsData.getIds(),this._onAdd(e)}},Tu.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.off(e,t)})),e=this.groupsData.getIds(),this.groupsData=null;for(var n=0;n<e.length;n++)this._removeGroup(e[n])}if(t){if(!(t instanceof Aa||t instanceof Xa))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.id;mn(za).call(za,this.groupListeners,(function(t,e){i.groupsData.on(e,t,o)})),e=this.groupsData.getIds(),this._onAddGroups(e)}},Tu.prototype._onUpdate=function(t){this._updateAllGroupData(t)},Tu.prototype._onAdd=function(t){this._onUpdate(t)},Tu.prototype._onRemove=function(t){this._onUpdate(t)},Tu.prototype._onUpdateGroups=function(t){this._updateAllGroupData(null,t)},Tu.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},Tu.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},Tu.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},Tu.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new Au(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},Tu.prototype._updateAllGroupData=function(t,e){if(null!=this.itemsData){var i={},n=this.itemsData.get(),o=this.itemsData.idProp,s={};t&&Ln(t).call(t,(function(t){s[t]=t}));for(var r={},a=0;a<n.length;a++){var l=n[a],h=l.group;null==h&&(h="__ungrouped__"),r.hasOwnProperty(h)?r[h]++:r[h]=1}var d={};if(!e&&t)for(h in this.groups)if(this.groups.hasOwnProperty(h)){var u=(f=this.groups[h]).getItems();i[h]=En(u).call(u,(function(t){return d[t[o]]=t[o],t[o]!==s[t[o]]}));var c=r[h];r[h]-=i[h].length,i[h].length<c&&(i[h][c-1]={})}for(a=0;a<n.length;a++)if(null==(h=(l=n[a]).group)&&(h="__ungrouped__"),e||!t||l[o]===s[l[o]]||!d.hasOwnProperty(l[o])){i.hasOwnProperty(h)||(i[h]=new Array(r[h]));var p=za.bridgeObject(l);p.x=za.convert(l.x,"Date"),p.end=za.convert(l.end,"Date"),p.orginalY=l.y,p.y=Number(l.y),p[o]=l[o];var m=i[h].length-r[h]--;i[h][m]=p}for(h in this.groups)this.groups.hasOwnProperty(h)&&(i.hasOwnProperty(h)||(i[h]=new Array(0)));for(h in i)if(i.hasOwnProperty(h))if(0==i[h].length)this.groups.hasOwnProperty(h)&&this._removeGroup(h);else{var f=void 0;null!=this.groupsData&&(f=this.groupsData.get(h)),null==f&&(f={id:h,content:this.options.defaultGroup+h}),this._updateGroup(f,h),this.groups[h].setItems(i[h])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},Tu.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e,i=this.body.range.end-this.body.range.start,n=i!=this.lastVisibleInterval;(this.lastVisibleInterval=i,1==t)&&(this.svg.style.width=za.option.asSize(3*this.props.width),this.svg.style.left=za.option.asSize(-this.props.width),-1==Fo(e=this.options.height+"").call(e,"%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0));if(1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==n||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,s=this.body.range.end-this.body.range.start;if(0!=this.props.width){var r=o*(this.props.width/s);this.svg.style.left=-this.props.width-r+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},Tu.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}za.insertSort(t,(function(t,e){var i=t.zIndex,n=e.zIndex;return void 0===i&&(i=0),void 0===n&&(n=0),i==n?0:i<n?-1:1}));for(var n=new Array(t.length),o=0;o<t.length;o++)n[o]=t[o].id;return n},Tu.prototype._updateGraph=function(){if(Cs(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},n=this.body.util.toGlobalTime(-this.body.domProps.root.width),o=this.body.util.toGlobalTime(2*this.body.domProps.root.width),s=this._getSortedGroupIds();if(s.length>0){var r={};for(this._getRelevantData(s,r,n,o),this._applySampling(s,r),e=0;e<s.length;e++)this._convertXcoordinates(r[s[e]]);if(this._getYRanges(s,r,i),1==this._updateYAxis(s,i))return Ls(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var a=void 0;for(e=0;e<s.length;e++)t=this.groups[s[e]],!0===this.options.stack&&"line"===this.options.style&&(null!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(null!=a&&(this._stack(r[t.id],r[a.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==a.options.shaded.orientation?(a.options.shaded.orientation="group",a.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=a.id))),a=t)),this._convertYcoordinates(r[s[e]],t);var l={};for(e=0;e<s.length;e++)if("line"===(t=this.groups[s[e]]).options.style&&1==t.options.shaded.enabled){var h=r[s[e]];if(null==h||0==h.length)continue;if(l.hasOwnProperty(s[e])||(l[s[e]]=Wu.calcPath(h,t)),"group"===t.options.shaded.orientation){var d=t.options.shaded.groupId;if(-1===Fo(s).call(s,d)){console.log(t.id+": Unknown shading group target given:"+d);continue}l.hasOwnProperty(d)||(l[d]=Wu.calcPath(r[d],this.groups[d])),Wu.drawShading(l[s[e]],t,l[d],this.framework)}else Wu.drawShading(l[s[e]],t,void 0,this.framework)}for(Ou.draw(s,r,this.framework),e=0;e<s.length;e++)if(t=this.groups[s[e]],r[s[e]].length>0)switch(t.options.style){case"line":l.hasOwnProperty(s[e])||(l[s[e]]=Wu.calcPath(r[s[e]],t)),Wu.draw(l[s[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||Zu.draw(r[s[e]],t,this.framework)}}}return Ls(this.svgElements),!1},Tu.prototype._stack=function(t,e){var i,n,o,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var l=i;l<e.length;l++){if(e[l].x===t[a].x){s=e[l],r=e[l],i=l;break}if(e[l].x>t[a].x){r=e[l],s=0==l?r:e[l-1],i=l;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),n=r.x-s.x,o=r.y-s.y,t[a].y=0==n?t[a].orginalY+r.y:t[a].orginalY+o/n*(t[a].x-s.x)+s.y}},Tu.prototype._getRelevantData=function(t,e,i,n){var o,s,r,a;if(t.length>0)for(s=0;s<t.length;s++){var l=(o=this.groups[t[s]]).getItems();if(1==wr(o.options)){var h=function(t,e){return t.getTime()==e.getTime()?0:t<e?-1:1},d=Math.max(0,za.binarySearchValue(l,i,"x","before",h)),u=Math.min(l.length,za.binarySearchValue(l,n,"x","after",h)+1);u<=0&&(u=l.length);var c=new Array(u-d);for(r=d;r<u;r++)a=o.itemsData[r],c[r-d]=a;e[t[s]]=c}else e[t[s]]=o.itemsData}},Tu.prototype._applySampling=function(t,e){if(t.length>0)for(var i=0;i<t.length;i++)if(1==this.groups[t[i]].options.sampling){var n=e[t[i]];if(n.length>0){var o,s=n.length,r=s/(this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x));o=Math.min(Math.ceil(.2*s),Math.max(1,Math.round(r)));for(var a=new Array(s),l=0;l<s;l+=o){a[Math.round(l/o)]=n[l]}e[t[i]]=Hr(a).call(a,0,Math.round(s/o))}}},Tu.prototype._getYRanges=function(t,e,i){var n,o,s,r,a=[],l=[];if(t.length>0){for(s=0;s<t.length;s++)n=e[t[s]],r=this.groups[t[s]].options,n.length>0&&(o=this.groups[t[s]],!0===r.stack&&"bar"===r.style?"left"===r.yAxisOrientation?a=Pn(a).call(a,n):l=Pn(l).call(l,n):i[t[s]]=o.getYRange(n,t[s]));Ou.getStackedYRange(a,i,t,"__barStackLeft","left"),Ou.getStackedYRange(l,i,t,"__barStackRight","right")}},Tu.prototype._updateYAxis=function(t,e){var i,n,o=!1,s=!1,r=!1,a=1e9,l=1e9,h=-1e9,d=-1e9;if(t.length>0){for(var u=0;u<t.length;u++){var c=this.groups[t[u]];c&&"right"!=c.options.yAxisOrientation?(s=!0,a=1e9,h=-1e9):c&&c.options.yAxisOrientation&&(r=!0,l=1e9,d=-1e9)}for(u=0;u<t.length;u++)e.hasOwnProperty(t[u])&&!0!==e[t[u]].ignore&&(i=e[t[u]].min,n=e[t[u]].max,"right"!=e[t[u]].yAxisOrientation?(s=!0,a=a>i?i:a,h=h<n?n:h):(r=!0,l=l>i?i:l,d=d<n?n:d));1==s&&this.yAxisLeft.setRange(a,h),1==r&&this.yAxisRight.setRange(l,d)}o=this._toggleAxisVisiblity(s,this.yAxisLeft)||o,o=this._toggleAxisVisiblity(r,this.yAxisRight)||o,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(this.yAxisLeft.lineOffset=1==r?this.yAxisRight.width:0,o=this.yAxisLeft.redraw()||o,o=this.yAxisRight.redraw()||o):o=this.yAxisRight.redraw()||o;var p=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"];for(u=0;u<p.length;u++)-1!=Fo(t).call(t,p[u])&&Hr(t).call(t,Fo(t).call(t,p[u]),1);return o},Tu.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},Tu.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y,null!=t[i].end?t[i].screen_end=e(t[i].end)+this.props.width:t[i].screen_end=void 0},Tu.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,n=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var o=0;o<t.length;o++)t[o].screen_y=Math.round(i.convertValue(t[o].y));e.setZeroPosition(Math.min(n,i.convertValue(0)))};var Yu="boolean",Eu={configure:{enabled:{boolean:Yu},filter:{boolean:Yu,function:"function"},container:{dom:"dom"},__type__:{object:"object",boolean:Yu,function:"function"}},alignCurrentTime:{string:"string",undefined:"undefined"},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:"string"},sort:{boolean:Yu},sampling:{boolean:Yu},stack:{boolean:Yu},graphHeight:{string:"string",number:"number"},shaded:{enabled:{boolean:Yu},orientation:{string:["bottom","top","zero","group"]},groupId:{object:"object"},__type__:{boolean:Yu,object:"object"}},style:{string:["line","bar","points"]},barChart:{width:{number:"number"},minWidth:{number:"number"},sideBySide:{boolean:Yu},align:{string:["left","center","right"]},__type__:{object:"object"}},interpolation:{enabled:{boolean:Yu},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:"number"},__type__:{object:"object",boolean:Yu}},drawPoints:{enabled:{boolean:Yu},onRender:{function:"function"},size:{number:"number"},style:{string:["square","circle"]},__type__:{object:"object",boolean:Yu,function:"function"}},dataAxis:{showMinorLabels:{boolean:Yu},showMajorLabels:{boolean:Yu},showWeekScale:{boolean:Yu},icons:{boolean:Yu},width:{string:"string",number:"number"},visible:{boolean:Yu},alignZeros:{boolean:Yu},left:{range:{min:{number:"number",undefined:"undefined"},max:{number:"number",undefined:"undefined"},__type__:{object:"object"}},format:{function:"function"},title:{text:{string:"string",number:"number",undefined:"undefined"},style:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},right:{range:{min:{number:"number",undefined:"undefined"},max:{number:"number",undefined:"undefined"},__type__:{object:"object"}},format:{function:"function"},title:{text:{string:"string",number:"number",undefined:"undefined"},style:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},__type__:{object:"object"}},legend:{enabled:{boolean:Yu},icons:{boolean:Yu},left:{visible:{boolean:Yu},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:"object"}},right:{visible:{boolean:Yu},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:"object"}},__type__:{object:"object",boolean:Yu}},groups:{visibility:{any:"any"},__type__:{object:"object"}},autoResize:{boolean:Yu},throttleRedraw:{number:"number"},clickToUse:{boolean:Yu},end:{number:"number",date:"date",string:"string",moment:"moment"},format:{minorLabels:{millisecond:{string:"string",undefined:"undefined"},second:{string:"string",undefined:"undefined"},minute:{string:"string",undefined:"undefined"},hour:{string:"string",undefined:"undefined"},weekday:{string:"string",undefined:"undefined"},day:{string:"string",undefined:"undefined"},week:{string:"string",undefined:"undefined"},month:{string:"string",undefined:"undefined"},quarter:{string:"string",undefined:"undefined"},year:{string:"string",undefined:"undefined"},__type__:{object:"object"}},majorLabels:{millisecond:{string:"string",undefined:"undefined"},second:{string:"string",undefined:"undefined"},minute:{string:"string",undefined:"undefined"},hour:{string:"string",undefined:"undefined"},weekday:{string:"string",undefined:"undefined"},day:{string:"string",undefined:"undefined"},week:{string:"string",undefined:"undefined"},month:{string:"string",undefined:"undefined"},quarter:{string:"string",undefined:"undefined"},year:{string:"string",undefined:"undefined"},__type__:{object:"object"}},__type__:{object:"object"}},moment:{function:"function"},height:{string:"string",number:"number"},hiddenDates:{start:{date:"date",number:"number",string:"string",moment:"moment"},end:{date:"date",number:"number",string:"string",moment:"moment"},repeat:{string:"string"},__type__:{object:"object",array:"array"}},locale:{string:"string"},locales:{__any__:{any:"any"},__type__:{object:"object"}},max:{date:"date",number:"number",string:"string",moment:"moment"},maxHeight:{number:"number",string:"string"},maxMinorChars:{number:"number"},min:{date:"date",number:"number",string:"string",moment:"moment"},minHeight:{number:"number",string:"string"},moveable:{boolean:Yu},multiselect:{boolean:Yu},orientation:{string:"string"},showCurrentTime:{boolean:Yu},showMajorLabels:{boolean:Yu},showMinorLabels:{boolean:Yu},showWeekScale:{boolean:Yu},start:{date:"date",number:"number",string:"string",moment:"moment"},timeAxis:{scale:{string:"string",undefined:"undefined"},step:{number:"number",undefined:"undefined"},__type__:{object:"object"}},width:{string:"string",number:"number"},zoomable:{boolean:Yu},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:"number"},zoomMin:{number:"number"},zIndex:{number:"number"},__type__:{object:"object"}},Ru={global:{alignCurrentTime:["none","year","month","quarter","week","isoWeek","day","date","hour","minute","second"],sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,showWeekScale:!1,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",quarter:"[Q]Q",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",quarter:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,showWeekScale:!1,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};function Uu(t,e,i,n){var o,s,r,a,l,h,d;if(!(Zo(i)||i instanceof Aa||i instanceof Xa)&&i instanceof Object){var u=n;n=i,i=u}n&&n.throttleRedraw&&console.warn('Graph2d option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var c=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:Ta,width:null,height:null,maxHeight:null,minHeight:null},this.options=za.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:va(o=this.on).call(o,this),off:va(s=this.off).call(s,this),emit:va(r=this.emit).call(r,this)},hiddenDates:[],util:{toScreen:va(a=c._toScreen).call(a,c),toGlobalScreen:va(l=c._toGlobalScreen).call(l,c),toTime:va(h=c._toTime).call(h,c),toGlobalTime:va(d=c._toGlobalTime).call(d,c)}},this.range=new ml(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new Jh(this.body),this.components.push(this.timeAxis),this.currentTime=new pd(this.body),this.components.push(this.currentTime),this.linegraph=new Tu(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",(function(t){c.emit("click",c.getEventProperties(t))})),this.on("doubletap",(function(t){c.emit("doubleClick",c.getEventProperties(t))})),this.dom.root.oncontextmenu=function(t){c.emit("contextmenu",c.getEventProperties(t))},this.initialFitDone=!1,this.on("changed",(function(){if(null!=c.itemsData){if(!c.initialFitDone&&!c.options.rollingMode)if(c.initialFitDone=!0,null!=c.options.start||null!=c.options.end){if(null==c.options.start||null==c.options.end)var t=c.getItemRange();var e=null!=c.options.start?c.options.start:t.min,i=null!=c.options.end?c.options.end:t.max;c.setWindow(e,i,{animation:!1})}else c.fit({animation:!1});c.initialDrawDone||!c.initialRangeChangeDone&&(c.options.start||c.options.end)&&!c.options.rollingMode||(c.initialDrawDone=!0,c.dom.root.style.visibility="visible",c.dom.loadingScreen.parentNode.removeChild(c.dom.loadingScreen),c.options.onInitialDrawComplete&&qr((function(){return c.options.onInitialDrawComplete()}),0))}})),n&&this.setOptions(n),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}Uu.prototype=new cd,Uu.prototype.setOptions=function(t){!0===fu.validate(t,Eu)&&console.log("%cErrors have been found in the supplied options object.",mu),cd.prototype.setOptions.call(this,t)},Uu.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?Ua(t instanceof Aa||t instanceof Xa?t:new Aa(t)):null,this.itemsData&&this.itemsData.dispose(),this.itemsData=e,this.linegraph&&this.linegraph.setItems(null!=e?e.rawDS:null),i)if(null!=this.options.start||null!=this.options.end){var n=null!=this.options.start?this.options.start:null,o=null!=this.options.end?this.options.end:null;this.setWindow(n,o,{animation:!1})}else this.fit({animation:!1})},Uu.prototype.setGroups=function(t){var e;e=t?t instanceof Aa||t instanceof Xa?t:new Aa(t):null,this.groupsData=e,this.linegraph.setGroups(e)},Uu.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},Uu.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]&&(this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]))},Uu.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var n=0;n<this.linegraph.groups[i].itemsData.length;n++){var o=this.linegraph.groups[i].itemsData[n],s=za.convert(o.x,"Date").valueOf();t=null==t||t>s?s:t,e=null==e||e<s?s:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},Uu.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,n=e-za.getAbsoluteLeft(this.dom.centerContainer),o=i-za.getAbsoluteTop(this.dom.centerContainer),s=this._toTime(n),r=ud.customTimeFromTarget(t),a=za.getTarget(t),l=null;za.hasParent(a,this.timeAxis.dom.foreground)||this.timeAxis2&&za.hasParent(a,this.timeAxis2.dom.foreground)?l="axis":za.hasParent(a,this.linegraph.yAxisLeft.dom.frame)||za.hasParent(a,this.linegraph.yAxisRight.dom.frame)?l="data-axis":za.hasParent(a,this.linegraph.legendLeft.dom.frame)||za.hasParent(a,this.linegraph.legendRight.dom.frame)?l="legend":null!=r?l="custom-time":za.hasParent(a,this.currentTime.bar)?l="current-time":za.hasParent(a,this.dom.center)&&(l="background");var h=[],d=this.linegraph.yAxisLeft,u=this.linegraph.yAxisRight;return!d.hidden&&this.itemsData.length>0&&h.push(d.screenToValue(o)),!u.hidden&&this.itemsData.length>0&&h.push(u.screenToValue(o)),{event:t,customTime:r?r.options.id:null,what:l,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:n,y:o,time:s,value:h}},Uu.prototype._createConfigurator=function(){return new Lu(this,this.dom.container,Ru)};var zu=Fs();Ta.locale(zu);var Vu={Core:cd,DateUtil:pl,Range:ml,stack:zd,TimeStep:Hh,components:{items:{Item:Pd,BackgroundItem:$d,BoxItem:Jd,PointItem:Kd,RangeItem:qd},BackgroundGroup:Hd,Component:Ja,CurrentTime:pd,CustomTime:ud,DataAxis:Fu,DataScale:Nu,GraphGroup:Au,Group:jd,ItemSet:uu,Legend:Xu,LineGraph:Tu,TimeAxis:Jh}};t.DOMutil=Zs,t.DataSet=Aa,t.DataView=Xa,t.Graph2d=Uu,t.Hammer=Vh,t.Queue=Za,t.Timeline=Gu,t.keycharm=qh,t.moment=Ta,t.timeline=Vu,t.util=Is,Object.defineProperty(t,"__esModule",{value:!0})}));
-//# sourceMappingURL=vis-timeline-graph2d.min.js.map
+"use strict";!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.vis=e():t.vis=e()}(this,function(){return function(t){function e(o){if(i[o])return i[o].exports;var n=i[o]={i:o,l:!1,exports:{}};return t[o].call(n.exports,n,n.exports,e),n.l=!0,n.exports}var i={};return e.m=t,e.c=i,e.d=function(t,i,o){e.o(t,i)||Object.defineProperty(t,i,{configurable:!1,enumerable:!0,get:o})},e.n=function(t){var i=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(i,"a",i),i},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="",e(e.s=85)}([function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e,i,o){var n=!1;!0===o&&(n=null===e[i]&&void 0!==t[i]),n?delete t[i]:t[i]=e[i]}var s=i(86),r=o(s),a=i(43),h=o(a),d=i(4),l=o(d),u=i(1),p=o(u),c=i(3),m=i(126);e.isNumber=function(t){return t instanceof Number||"number"==typeof t},e.recursiveDOMDelete=function(t){if(t)for(;!0===t.hasChildNodes();)e.recursiveDOMDelete(t.firstChild),t.removeChild(t.firstChild)},e.giveRange=function(t,e,i,o){if(e==t)return.5;var n=1/(e-t);return Math.max(0,(o-t)*n)},e.isString=function(t){return t instanceof String||"string"==typeof t},e.isDate=function(t){if(t instanceof Date)return!0;if(e.isString(t)){if(f.exec(t))return!0;if(!isNaN(Date.parse(t)))return!0}return!1},e.randomUUID=function(){return m.v4()},e.assignAllKeys=function(t,e){for(var i in t)t.hasOwnProperty(i)&&"object"!==(0,p.default)(t[i])&&(t[i]=e)},e.fillIfDefined=function(t,i){var o=arguments.length>2&&void 0!==arguments[2]&&arguments[2];for(var s in t)void 0!==i[s]&&(null===i[s]||"object"!==(0,p.default)(i[s])?n(t,i,s,o):"object"===(0,p.default)(t[s])&&e.fillIfDefined(t[s],i[s],o))},e.extend=function(t,e){for(var i=1;i<arguments.length;i++){var o=arguments[i];for(var n in o)o.hasOwnProperty(n)&&(t[n]=o[n])}return t},e.selectiveExtend=function(t,e,i){if(!Array.isArray(t))throw new Error("Array with property names expected as first argument");for(var o=2;o<arguments.length;o++)for(var n=arguments[o],s=0;s<t.length;s++){var r=t[s];n&&n.hasOwnProperty(r)&&(e[r]=n[r])}return e},e.selectiveDeepExtend=function(t,i,o){var s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var r=0;r<t.length;r++){var a=t[r];if(o.hasOwnProperty(a))if(o[a]&&o[a].constructor===Object)void 0===i[a]&&(i[a]={}),i[a].constructor===Object?e.deepExtend(i[a],o[a],!1,s):n(i,o,a,s);else{if(Array.isArray(o[a]))throw new TypeError("Arrays are not supported by deepExtend");n(i,o,a,s)}}return i},e.selectiveNotDeepExtend=function(t,i,o){var s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];if(Array.isArray(o))throw new TypeError("Arrays are not supported by deepExtend");for(var r in o)if(o.hasOwnProperty(r)&&-1===t.indexOf(r))if(o[r]&&o[r].constructor===Object)void 0===i[r]&&(i[r]={}),i[r].constructor===Object?e.deepExtend(i[r],o[r]):n(i,o,r,s);else if(Array.isArray(o[r])){i[r]=[];for(var a=0;a<o[r].length;a++)i[r].push(o[r][a])}else n(i,o,r,s);return i},e.deepExtend=function(t,i){var o=arguments.length>2&&void 0!==arguments[2]&&arguments[2],s=arguments.length>3&&void 0!==arguments[3]&&arguments[3];for(var r in i)if(i.hasOwnProperty(r)||!0===o)if(i[r]&&i[r].constructor===Object)void 0===t[r]&&(t[r]={}),t[r].constructor===Object?e.deepExtend(t[r],i[r],o):n(t,i,r,s);else if(Array.isArray(i[r])){t[r]=[];for(var a=0;a<i[r].length;a++)t[r].push(i[r][a])}else n(t,i,r,s);return t},e.equalArray=function(t,e){if(t.length!=e.length)return!1;for(var i=0,o=t.length;i<o;i++)if(t[i]!=e[i])return!1;return!0},e.convert=function(t,i){var o;if(void 0!==t){if(null===t)return null;if(!i)return t;if("string"!=typeof i&&!(i instanceof String))throw new Error("Type must be a string");switch(i){case"boolean":case"Boolean":return Boolean(t);case"number":case"Number":return e.isString(t)&&!isNaN(Date.parse(t))?c(t).valueOf():Number(t.valueOf());case"string":case"String":return String(t);case"Date":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return new Date(t.valueOf());if(c.isMoment(t))return new Date(t.valueOf());if(e.isString(t))return o=f.exec(t),o?new Date(Number(o[1])):c(new Date(t)).toDate();throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"Moment":if(e.isNumber(t))return c(t);if(t instanceof Date)return c(t.valueOf());if(c.isMoment(t))return c(t);if(e.isString(t))return o=f.exec(t),c(o?Number(o[1]):t);throw new Error("Cannot convert object of type "+e.getType(t)+" to type Date");case"ISODate":if(e.isNumber(t))return new Date(t);if(t instanceof Date)return t.toISOString();if(c.isMoment(t))return t.toDate().toISOString();if(e.isString(t))return o=f.exec(t),o?new Date(Number(o[1])).toISOString():c(t).format();throw new Error("Cannot convert object of type "+e.getType(t)+" to type ISODate");case"ASPDate":if(e.isNumber(t))return"/Date("+t+")/";if(t instanceof Date)return"/Date("+t.valueOf()+")/";if(e.isString(t)){o=f.exec(t);return"/Date("+(o?new Date(Number(o[1])).valueOf():new Date(t).valueOf())+")/"}throw new Error("Cannot convert object of type "+e.getType(t)+" to type ASPDate");default:throw new Error('Unknown type "'+i+'"')}}};var f=/^\/?Date\((\-?\d+)/i;e.getType=function(t){var e=void 0===t?"undefined":(0,p.default)(t);return"object"==e?null===t?"null":t instanceof Boolean?"Boolean":t instanceof Number?"Number":t instanceof String?"String":Array.isArray(t)?"Array":t instanceof Date?"Date":"Object":"number"==e?"Number":"boolean"==e?"Boolean":"string"==e?"String":void 0===e?"undefined":e},e.copyAndExtendArray=function(t,e){for(var i=[],o=0;o<t.length;o++)i.push(t[o]);return i.push(e),i},e.copyArray=function(t){for(var e=[],i=0;i<t.length;i++)e.push(t[i]);return e},e.getAbsoluteLeft=function(t){return t.getBoundingClientRect().left},e.getAbsoluteRight=function(t){return t.getBoundingClientRect().right},e.getAbsoluteTop=function(t){return t.getBoundingClientRect().top},e.addClassName=function(t,e){var i=t.className.split(" "),o=e.split(" ");i=i.concat(o.filter(function(t){return i.indexOf(t)<0})),t.className=i.join(" ")},e.removeClassName=function(t,e){var i=t.className.split(" "),o=e.split(" ");i=i.filter(function(t){return o.indexOf(t)<0}),t.className=i.join(" ")},e.forEach=function(t,e){var i,o;if(Array.isArray(t))for(i=0,o=t.length;i<o;i++)e(t[i],i,t);else for(i in t)t.hasOwnProperty(i)&&e(t[i],i,t)},e.toArray=function(t){var e=[];for(var i in t)t.hasOwnProperty(i)&&e.push(t[i]);return e},e.updateProperty=function(t,e,i){return t[e]!==i&&(t[e]=i,!0)},e.throttle=function(t){var e=!1;return function(){e||(e=!0,requestAnimationFrame(function(){e=!1,t()}))}},e.addEventListener=function(t,e,i,o){t.addEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.addEventListener(e,i,o)):t.attachEvent("on"+e,i)},e.removeEventListener=function(t,e,i,o){t.removeEventListener?(void 0===o&&(o=!1),"mousewheel"===e&&navigator.userAgent.indexOf("Firefox")>=0&&(e="DOMMouseScroll"),t.removeEventListener(e,i,o)):t.detachEvent("on"+e,i)},e.preventDefault=function(t){t||(t=window.event),t.preventDefault?t.preventDefault():t.returnValue=!1},e.getTarget=function(t){t||(t=window.event);var e;return t.target?e=t.target:t.srcElement&&(e=t.srcElement),void 0!=e.nodeType&&3==e.nodeType&&(e=e.parentNode),e},e.hasParent=function(t,e){for(var i=t;i;){if(i===e)return!0;i=i.parentNode}return!1},e.option={},e.option.asBoolean=function(t,e){return"function"==typeof t&&(t=t()),null!=t?0!=t:e||null},e.option.asNumber=function(t,e){return"function"==typeof t&&(t=t()),null!=t?Number(t)||e||null:e||null},e.option.asString=function(t,e){return"function"==typeof t&&(t=t()),null!=t?String(t):e||null},e.option.asSize=function(t,i){return"function"==typeof t&&(t=t()),e.isString(t)?t:e.isNumber(t)?t+"px":i||null},e.option.asElement=function(t,e){return"function"==typeof t&&(t=t()),t||e||null},e.hexToRGB=function(t){var e=/^#?([a-f\d])([a-f\d])([a-f\d])$/i;t=t.replace(e,function(t,e,i,o){return e+e+i+i+o+o});var i=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(t);return i?{r:parseInt(i[1],16),g:parseInt(i[2],16),b:parseInt(i[3],16)}:null},e.overrideOpacity=function(t,i){var o;return-1!=t.indexOf("rgba")?t:-1!=t.indexOf("rgb")?(o=t.substr(t.indexOf("(")+1).replace(")","").split(","),"rgba("+o[0]+","+o[1]+","+o[2]+","+i+")"):(o=e.hexToRGB(t),null==o?t:"rgba("+o.r+","+o.g+","+o.b+","+i+")")},e.RGBToHex=function(t,e,i){return"#"+((1<<24)+(t<<16)+(e<<8)+i).toString(16).slice(1)},e.parseColor=function(t){var i;if(!0===e.isString(t)){if(!0===e.isValidRGB(t)){var o=t.substr(4).substr(0,t.length-5).split(",").map(function(t){return parseInt(t)});t=e.RGBToHex(o[0],o[1],o[2])}if(!0===e.isValidHex(t)){var n=e.hexToHSV(t),s={h:n.h,s:.8*n.s,v:Math.min(1,1.02*n.v)},r={h:n.h,s:Math.min(1,1.25*n.s),v:.8*n.v},a=e.HSVToHex(r.h,r.s,r.v),h=e.HSVToHex(s.h,s.s,s.v);i={background:t,border:a,highlight:{background:h,border:a},hover:{background:h,border:a}}}else i={background:t,border:t,highlight:{background:t,border:t},hover:{background:t,border:t}}}else i={},i.background=t.background||void 0,i.border=t.border||void 0,e.isString(t.highlight)?i.highlight={border:t.highlight,background:t.highlight}:(i.highlight={},i.highlight.background=t.highlight&&t.highlight.background||void 0,i.highlight.border=t.highlight&&t.highlight.border||void 0),e.isString(t.hover)?i.hover={border:t.hover,background:t.hover}:(i.hover={},i.hover.background=t.hover&&t.hover.background||void 0,i.hover.border=t.hover&&t.hover.border||void 0);return i},e.RGBToHSV=function(t,e,i){t/=255,e/=255,i/=255;var o=Math.min(t,Math.min(e,i)),n=Math.max(t,Math.max(e,i));if(o==n)return{h:0,s:0,v:o};var s=t==o?e-i:i==o?t-e:i-t;return{h:60*((t==o?3:i==o?1:5)-s/(n-o))/360,s:(n-o)/n,v:n}};var g={split:function(t){var e={};return t.split(";").forEach(function(t){if(""!=t.trim()){var i=t.split(":"),o=i[0].trim(),n=i[1].trim();e[o]=n}}),e},join:function(t){return(0,l.default)(t).map(function(e){return e+": "+t[e]}).join("; ")}};e.addCssText=function(t,i){var o=g.split(t.style.cssText),n=g.split(i),s=e.extend(o,n);t.style.cssText=g.join(s)},e.removeCssText=function(t,e){var i=g.split(t.style.cssText),o=g.split(e);for(var n in o)o.hasOwnProperty(n)&&delete i[n];t.style.cssText=g.join(i)},e.HSVToRGB=function(t,e,i){var o,n,s,r=Math.floor(6*t),a=6*t-r,h=i*(1-e),d=i*(1-a*e),l=i*(1-(1-a)*e);switch(r%6){case 0:o=i,n=l,s=h;break;case 1:o=d,n=i,s=h;break;case 2:o=h,n=i,s=l;break;case 3:o=h,n=d,s=i;break;case 4:o=l,n=h,s=i;break;case 5:o=i,n=h,s=d}return{r:Math.floor(255*o),g:Math.floor(255*n),b:Math.floor(255*s)}},e.HSVToHex=function(t,i,o){var n=e.HSVToRGB(t,i,o);return e.RGBToHex(n.r,n.g,n.b)},e.hexToHSV=function(t){var i=e.hexToRGB(t);return e.RGBToHSV(i.r,i.g,i.b)},e.isValidHex=function(t){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(t)},e.isValidRGB=function(t){return t=t.replace(" ",""),/rgb\((\d{1,3}),(\d{1,3}),(\d{1,3})\)/i.test(t)},e.isValidRGBA=function(t){return t=t.replace(" ",""),/rgba\((\d{1,3}),(\d{1,3}),(\d{1,3}),(.{1,3})\)/i.test(t)},e.selectiveBridgeObject=function(t,i){if(null!==i&&"object"===(void 0===i?"undefined":(0,p.default)(i))){for(var o=(0,h.default)(i),n=0;n<t.length;n++)i.hasOwnProperty(t[n])&&"object"==(0,p.default)(i[t[n]])&&(o[t[n]]=e.bridgeObject(i[t[n]]));return o}return null},e.bridgeObject=function(t){if(null!==t&&"object"===(void 0===t?"undefined":(0,p.default)(t))){var i=(0,h.default)(t);if(t instanceof Element)i=t;else{i=(0,h.default)(t);for(var o in t)t.hasOwnProperty(o)&&"object"==(0,p.default)(t[o])&&(i[o]=e.bridgeObject(t[o]))}return i}return null},e.insertSort=function(t,e){for(var i=0;i<t.length;i++){for(var o=t[i],n=i;n>0&&e(o,t[n-1])<0;n--)t[n]=t[n-1];t[n]=o}return t},e.mergeOptions=function(t,e,i){var o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},n=function(t){return null!==t&&void 0!==t},s=function(t){return null!==t&&"object"===(void 0===t?"undefined":(0,p.default)(t))};if(!s(t))throw new Error("Parameter mergeTarget must be an object");if(!s(e))throw new Error("Parameter options must be an object");if(!n(i))throw new Error("Parameter option must have a value");if(!s(o))throw new Error("Parameter globalOptions must be an object");var r=e[i],a=s(o)&&!function(t){for(var e in t)if(t.hasOwnProperty(e))return!1;return!0}(o),d=a?o[i]:void 0,l=d?d.enabled:void 0;if(void 0!==r){if("boolean"==typeof r)return s(t[i])||(t[i]={}),void(t[i].enabled=r);if(null===r&&!s(t[i])){if(!n(d))return;t[i]=(0,h.default)(d)}if(s(r)){var u=!0;void 0!==r.enabled?u=r.enabled:void 0!==l&&(u=d.enabled),function(t,e,i){s(t[i])||(t[i]={});var o=e[i],n=t[i];for(var r in o)o.hasOwnProperty(r)&&(n[r]=o[r])}(t,e,i),t[i].enabled=u}}},e.binarySearchCustom=function(t,e,i,o){for(var n=0,s=0,r=t.length-1;s<=r&&n<1e4;){var a=Math.floor((s+r)/2),h=t[a],d=void 0===o?h[i]:h[i][o],l=e(d);if(0==l)return a;-1==l?s=a+1:r=a-1,n++}return-1},e.binarySearchValue=function(t,e,i,o,n){var s,r,a,h,d=0,l=0,u=t.length-1;for(n=void 0!=n?n:function(t,e){return t==e?0:t<e?-1:1};l<=u&&d<1e4;){if(h=Math.floor(.5*(u+l)),s=t[Math.max(0,h-1)][i],r=t[h][i],a=t[Math.min(t.length-1,h+1)][i],0==n(r,e))return h;if(n(s,e)<0&&n(r,e)>0)return"before"==o?Math.max(0,h-1):h;if(n(r,e)<0&&n(a,e)>0)return"before"==o?h:Math.min(t.length-1,h+1);n(r,e)<0?l=h+1:u=h-1,d++}return-1},e.easingFunctions={linear:function(t){return t},easeInQuad:function(t){return t*t},easeOutQuad:function(t){return t*(2-t)},easeInOutQuad:function(t){return t<.5?2*t*t:(4-2*t)*t-1},easeInCubic:function(t){return t*t*t},easeOutCubic:function(t){return--t*t*t+1},easeInOutCubic:function(t){return t<.5?4*t*t*t:(t-1)*(2*t-2)*(2*t-2)+1},easeInQuart:function(t){return t*t*t*t},easeOutQuart:function(t){return 1- --t*t*t*t},easeInOutQuart:function(t){return t<.5?8*t*t*t*t:1-8*--t*t*t*t},easeInQuint:function(t){return t*t*t*t*t},easeOutQuint:function(t){return 1+--t*t*t*t*t},easeInOutQuint:function(t){return t<.5?16*t*t*t*t*t:1+16*--t*t*t*t*t}},e.getScrollBarWidth=function(){var t=document.createElement("p");t.style.width="100%",t.style.height="200px";var e=document.createElement("div");e.style.position="absolute",e.style.top="0px",e.style.left="0px",e.style.visibility="hidden",e.style.width="200px",e.style.height="150px",e.style.overflow="hidden",e.appendChild(t),document.body.appendChild(e);var i=t.offsetWidth;e.style.overflow="scroll";var o=t.offsetWidth;return i==o&&(o=e.clientWidth),document.body.removeChild(e),i-o},e.topMost=function(t,e){var i=void 0;Array.isArray(e)||(e=[e]);var o=!0,n=!1,s=void 0;try{for(var a,h=(0,r.default)(t);!(o=(a=h.next()).done);o=!0){var d=a.value;if(d){i=d[e[0]];for(var l=1;l<e.length;l++)i&&(i=i[e[l]]);if(void 0!==i)break}}}catch(t){n=!0,s=t}finally{try{!o&&h.return&&h.return()}finally{if(n)throw s}}return i}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}e.__esModule=!0;var n=i(110),s=o(n),r=i(112),a=o(r),h="function"==typeof a.default&&"symbol"==typeof s.default?function(t){return typeof t}:function(t){return t&&"function"==typeof a.default&&t.constructor===a.default&&t!==a.default.prototype?"symbol":typeof t};e.default="function"==typeof a.default&&"symbol"===h(s.default)?function(t){return void 0===t?"undefined":h(t)}:function(t){return t&&"function"==typeof a.default&&t.constructor===a.default&&t!==a.default.prototype?"symbol":void 0===t?"undefined":h(t)}},function(t,e){var i=t.exports={version:"2.5.1"};"number"==typeof __e&&(__e=i)},function(t,e,i){t.exports="undefined"!=typeof window&&window.moment||i(123)},function(t,e,i){t.exports={default:i(107),__esModule:!0}},function(t,e,i){function o(t,e){this.options=null,this.props=null}var n=i(0);o.prototype.setOptions=function(t){t&&n.extend(this.options,t)},o.prototype.redraw=function(){return!1},o.prototype.destroy=function(){},o.prototype._isResized=function(){var t=this.props._previousWidth!==this.props.width||this.props._previousHeight!==this.props.height;return this.props._previousWidth=this.props.width,this.props._previousHeight=this.props.height,t},t.exports=o},function(t,e){var i=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=i)},function(t,e,i){var o=i(40)("wks"),n=i(27),s=i(6).Symbol,r="function"==typeof s;(t.exports=function(t){return o[t]||(o[t]=r&&s[t]||(r?s:n)("Symbol."+t))}).store=o},function(t,e,i){var o=i(18),n=i(57),s=i(36),r=Object.defineProperty;e.f=i(9)?Object.defineProperty:function(t,e,i){if(o(t),e=s(e,!0),o(i),n)try{return r(t,e,i)}catch(t){}if("get"in i||"set"in i)throw TypeError("Accessors not supported!");return"value"in i&&(t[e]=i.value),t}},function(t,e,i){t.exports=!i(19)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,e){var i={}.hasOwnProperty;t.exports=function(t,e){return i.call(t,e)}},function(t,e,i){e.prepareElements=function(t){for(var e in t)t.hasOwnProperty(e)&&(t[e].redundant=t[e].used,t[e].used=[])},e.cleanupElements=function(t){for(var e in t)if(t.hasOwnProperty(e)&&t[e].redundant){for(var i=0;i<t[e].redundant.length;i++)t[e].redundant[i].parentNode.removeChild(t[e].redundant[i]);t[e].redundant=[]}},e.resetElements=function(t){e.prepareElements(t),e.cleanupElements(t),e.prepareElements(t)},e.getSVGElement=function(t,e,i){var o;return e.hasOwnProperty(t)?e[t].redundant.length>0?(o=e[t].redundant[0],e[t].redundant.shift()):(o=document.createElementNS("http://www.w3.org/2000/svg",t),i.appendChild(o)):(o=document.createElementNS("http://www.w3.org/2000/svg",t),e[t]={used:[],redundant:[]},i.appendChild(o)),e[t].used.push(o),o},e.getDOMElement=function(t,e,i,o){var n;return e.hasOwnProperty(t)?e[t].redundant.length>0?(n=e[t].redundant[0],e[t].redundant.shift()):(n=document.createElement(t),void 0!==o?i.insertBefore(n,o):i.appendChild(n)):(n=document.createElement(t),e[t]={used:[],redundant:[]},void 0!==o?i.insertBefore(n,o):i.appendChild(n)),e[t].used.push(n),n},e.drawPoint=function(t,i,o,n,s,r){var a;if("circle"==o.style?(a=e.getSVGElement("circle",n,s),a.setAttributeNS(null,"cx",t),a.setAttributeNS(null,"cy",i),a.setAttributeNS(null,"r",.5*o.size)):(a=e.getSVGElement("rect",n,s),a.setAttributeNS(null,"x",t-.5*o.size),a.setAttributeNS(null,"y",i-.5*o.size),a.setAttributeNS(null,"width",o.size),a.setAttributeNS(null,"height",o.size)),void 0!==o.styles&&a.setAttributeNS(null,"style",o.styles),a.setAttributeNS(null,"class",o.className+" vis-point"),r){var h=e.getSVGElement("text",n,s);r.xOffset&&(t+=r.xOffset),r.yOffset&&(i+=r.yOffset),r.content&&(h.textContent=r.content),r.className&&h.setAttributeNS(null,"class",r.className+" vis-label"),h.setAttributeNS(null,"x",t),h.setAttributeNS(null,"y",i)}return a},e.drawBar=function(t,i,o,n,s,r,a,h){if(0!=n){n<0&&(n*=-1,i-=n);var d=e.getSVGElement("rect",r,a);d.setAttributeNS(null,"x",t-.5*o),d.setAttributeNS(null,"y",i),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",n),d.setAttributeNS(null,"class",s),h&&d.setAttributeNS(null,"style",h)}}},function(t,e,i){function o(){var t=function(){};return{on:t,off:t,destroy:t,emit:t,get:function(e){return{set:t}}}}if("undefined"!=typeof window){var n=i(130),s=window.Hammer||i(131);t.exports=n(s,{preventDefault:"mouse"})}else t.exports=function(){return o()}},function(t,e,i){var o=i(91),n=i(34);t.exports=function(t){return o(n(t))}},function(t,e,i){var o=i(8),n=i(25);t.exports=i(9)?function(t,e,i){return o.f(t,e,n(1,i))}:function(t,e,i){return t[e]=i,t}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){if(t&&!Array.isArray(t)&&(e=t,t=null),this._options=e||{},this._data={},this.length=0,this._fieldId=this._options.fieldId||"id",this._type={},this._options.type)for(var i=(0,l.default)(this._options.type),o=0,n=i.length;o<n;o++){var s=i[o],r=this._options.type[s];this._type[s]="Date"==r||"ISODate"==r||"ASPDate"==r?"Date":r}this._subscribers={},t&&this.add(t),this.setOptions(e)}var s=i(16),r=o(s),a=i(1),h=o(a),d=i(4),l=o(d),u=i(0),p=i(65);n.prototype.setOptions=function(t){t&&void 0!==t.queue&&(!1===t.queue?this._queue&&(this._queue.destroy(),delete this._queue):(this._queue||(this._queue=p.extend(this,{replace:["add","update","remove"]})),"object"===(0,h.default)(t.queue)&&this._queue.setOptions(t.queue)))},n.prototype.on=function(t,e){var i=this._subscribers[t];i||(i=[],this._subscribers[t]=i),i.push({callback:e})},n.prototype.off=function(t,e){var i=this._subscribers[t];i&&(this._subscribers[t]=i.filter(function(t){return t.callback!=e}))},n.prototype._trigger=function(t,e,i){if("*"==t)throw new Error("Cannot trigger event *");var o=[];t in this._subscribers&&(o=o.concat(this._subscribers[t])),"*"in this._subscribers&&(o=o.concat(this._subscribers["*"]));for(var n=0,s=o.length;n<s;n++){var r=o[n];r.callback&&r.callback(t,e,i||null)}},n.prototype.add=function(t,e){var i,o=[],n=this;if(Array.isArray(t))for(var s=0,r=t.length;s<r;s++)i=n._addItem(t[s]),o.push(i);else{if(!t||"object"!==(void 0===t?"undefined":(0,h.default)(t)))throw new Error("Unknown dataType");i=n._addItem(t),o.push(i)}return o.length&&this._trigger("add",{items:o},e),o},n.prototype.update=function(t,e){var i=[],o=[],n=[],s=[],r=this,a=r._fieldId,d=function(t){var e=t[a];if(r._data[e]){var h=u.extend({},r._data[e]);e=r._updateItem(t),o.push(e),s.push(t),n.push(h)}else e=r._addItem(t),i.push(e)};if(Array.isArray(t))for(var l=0,p=t.length;l<p;l++)t[l]&&"object"===(0,h.default)(t[l])?d(t[l]):console.warn("Ignoring input item, which is not an object at index "+l);else{if(!t||"object"!==(void 0===t?"undefined":(0,h.default)(t)))throw new Error("Unknown dataType");d(t)}if(i.length&&this._trigger("add",{items:i},e),o.length){var c={items:o,oldData:n,data:s};this._trigger("update",c,e)}return i.concat(o)},n.prototype.get=function(t){var e,i,o,n=this,s=u.getType(arguments[0]);"String"==s||"Number"==s?(e=arguments[0],o=arguments[1]):"Array"==s?(i=arguments[0],o=arguments[1]):o=arguments[0];var r;if(o&&o.returnType){r=-1==["Array","Object"].indexOf(o.returnType)?"Array":o.returnType}else r="Array";var a,h,d,p,c,m=o&&o.type||this._options.type,f=o&&o.filter,g=[];if(void 0!=e)(a=n._getItem(e,m))&&f&&!f(a)&&(a=null);else if(void 0!=i)for(p=0,c=i.length;p<c;p++)a=n._getItem(i[p],m),f&&!f(a)||g.push(a);else for(h=(0,l.default)(this._data),p=0,c=h.length;p<c;p++)d=h[p],a=n._getItem(d,m),f&&!f(a)||g.push(a);if(o&&o.order&&void 0==e&&this._sort(g,o.order),o&&o.fields){var v=o.fields;if(void 0!=e)a=this._filterFields(a,v);else for(p=0,c=g.length;p<c;p++)g[p]=this._filterFields(g[p],v)}if("Object"==r){var y,b={};for(p=0,c=g.length;p<c;p++)y=g[p],b[y.id]=y;return b}return void 0!=e?a:g},n.prototype.getIds=function(t){var e,i,o,n,s,r=this._data,a=t&&t.filter,h=t&&t.order,d=t&&t.type||this._options.type,u=(0,l.default)(r),p=[];if(a)if(h){for(s=[],e=0,i=u.length;e<i;e++)o=u[e],n=this._getItem(o,d),a(n)&&s.push(n);for(this._sort(s,h),e=0,i=s.length;e<i;e++)p.push(s[e][this._fieldId])}else for(e=0,i=u.length;e<i;e++)o=u[e],n=this._getItem(o,d),a(n)&&p.push(n[this._fieldId]);else if(h){for(s=[],e=0,i=u.length;e<i;e++)o=u[e],s.push(r[o]);for(this._sort(s,h),e=0,i=s.length;e<i;e++)p.push(s[e][this._fieldId])}else for(e=0,i=u.length;e<i;e++)o=u[e],n=r[o],p.push(n[this._fieldId]);return p},n.prototype.getDataSet=function(){return this},n.prototype.forEach=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=this._data,d=(0,l.default)(h);if(e&&e.order){var u=this.get(e);for(i=0,o=u.length;i<o;i++)n=u[i],s=n[this._fieldId],t(n,s)}else for(i=0,o=d.length;i<o;i++)s=d[i],n=this._getItem(s,a),r&&!r(n)||t(n,s)},n.prototype.map=function(t,e){var i,o,n,s,r=e&&e.filter,a=e&&e.type||this._options.type,h=[],d=this._data,u=(0,l.default)(d);for(i=0,o=u.length;i<o;i++)n=u[i],s=this._getItem(n,a),r&&!r(s)||h.push(t(s,n));return e&&e.order&&this._sort(h,e.order),h},n.prototype._filterFields=function(t,e){if(!t)return t;var i,o,n={},s=(0,l.default)(t),r=s.length;if(Array.isArray(e))for(i=0;i<r;i++)o=s[i],-1!=e.indexOf(o)&&(n[o]=t[o]);else for(i=0;i<r;i++)o=s[i],e.hasOwnProperty(o)&&(n[e[o]]=t[o]);return n},n.prototype._sort=function(t,e){if(u.isString(e)){var i=e;t.sort(function(t,e){var o=t[i],n=e[i];return o>n?1:o<n?-1:0})}else{if("function"!=typeof e)throw new TypeError("Order must be a function or a string");t.sort(e)}},n.prototype.remove=function(t,e){var i,o,n,s,r=[],a=[],h=[];for(h=Array.isArray(t)?t:[t],i=0,o=h.length;i<o;i++)(s=this._remove(h[i]))&&void 0!=(n=s[this._fieldId])&&(r.push(n),a.push(s));return r.length&&this._trigger("remove",{items:r,oldData:a},e),r},n.prototype._remove=function(t){var e,i;return u.isNumber(t)||u.isString(t)?i=t:t&&"object"===(void 0===t?"undefined":(0,h.default)(t))&&(i=t[this._fieldId]),void 0!==i&&this._data[i]?(e=this._data[i],delete this._data[i],this.length--,e):null},n.prototype.clear=function(t){var e,i,o=(0,l.default)(this._data),n=[];for(e=0,i=o.length;e<i;e++)n.push(this._data[o[e]]);return this._data={},this.length=0,this._trigger("remove",{items:o,oldData:n},t),o},n.prototype.max=function(t){var e,i,o=this._data,n=(0,l.default)(o),s=null,r=null;for(e=0,i=n.length;e<i;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d>r)&&(s=h,r=d)}return s},n.prototype.min=function(t){var e,i,o=this._data,n=(0,l.default)(o),s=null,r=null;for(e=0,i=n.length;e<i;e++){var a=n[e],h=o[a],d=h[t];null!=d&&(!s||d<r)&&(s=h,r=d)}return s},n.prototype.distinct=function(t){var e,i,o,n=this._data,s=(0,l.default)(n),r=[],a=this._options.type&&this._options.type[t]||null,h=0;for(e=0,o=s.length;e<o;e++){var d=s[e],p=n[d],c=p[t],m=!1;for(i=0;i<h;i++)if(r[i]==c){m=!0;break}m||void 0===c||(r[h]=c,h++)}if(a)for(e=0,o=r.length;e<o;e++)r[e]=u.convert(r[e],a);return r},n.prototype._addItem=function(t){var e=t[this._fieldId];if(void 0!=e){if(this._data[e])throw new Error("Cannot add item: item with id "+e+" already exists")}else e=u.randomUUID(),t[this._fieldId]=e;var i,o,n={},s=(0,l.default)(t);for(i=0,o=s.length;i<o;i++){var r=s[i],a=this._type[r];n[r]=u.convert(t[r],a)}return this._data[e]=n,this.length++,e},n.prototype._getItem=function(t,e){var i,o,n,s,r=this._data[t];if(!r)return null;var a={},h=(0,l.default)(r);if(e)for(n=0,s=h.length;n<s;n++)i=h[n],o=r[i],a[i]=u.convert(o,e[i]);else for(n=0,s=h.length;n<s;n++)i=h[n],o=r[i],a[i]=o;return a[this._fieldId]||(a[this._fieldId]=r.id),a},n.prototype._updateItem=function(t){var e=t[this._fieldId];if(void 0==e)throw new Error("Cannot update item: item has no id (item: "+(0,r.default)(t)+")");var i=this._data[e];if(!i)throw new Error("Cannot update item: no item with id "+e+" found");for(var o=(0,l.default)(t),n=0,s=o.length;n<s;n++){var a=o[n],h=this._type[a];i[a]=u.convert(t[a],h)}return e},t.exports=n},function(t,e,i){t.exports={default:i(128),__esModule:!0}},function(t,e,i){var o=i(6),n=i(2),s=i(92),r=i(14),a=function(t,e,i){var h,d,l,u=t&a.F,p=t&a.G,c=t&a.S,m=t&a.P,f=t&a.B,g=t&a.W,v=p?n:n[e]||(n[e]={}),y=v.prototype,b=p?o:c?o[e]:(o[e]||{}).prototype;p&&(i=e);for(h in i)(d=!u&&b&&void 0!==b[h])&&h in v||(l=d?b[h]:i[h],v[h]=p&&"function"!=typeof b[h]?i[h]:f&&d?s(l,o):g&&b[h]==l?function(t){var e=function(e,i,o){if(this instanceof t){switch(arguments.length){case 0:return new t;case 1:return new t(e);case 2:return new t(e,i)}return new t(e,i,o)}return t.apply(this,arguments)};return e.prototype=t.prototype,e}(l):m&&"function"==typeof l?s(Function.call,l):l,m&&((v.virtual||(v.virtual={}))[h]=l,t&a.R&&y&&!y[h]&&r(y,h,l)))};a.F=1,a.G=2,a.S=4,a.P=8,a.B=16,a.W=32,a.U=64,a.R=128,t.exports=a},function(t,e,i){var o=i(24);t.exports=function(t){if(!o(t))throw TypeError(t+" is not an object!");return t}},function(t,e){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t,e,i){function o(t,e){this._data=null,this._ids={},this.length=0,this._options=e||{},this._fieldId="id",this._subscribers={};var i=this;this.listener=function(){i._onEvent.apply(i,arguments)},this.setData(t)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(15);o.prototype.setData=function(t){var e,i,o,n,s;if(this._data){for(this._data.off&&this._data.off("*",this.listener),e=this._data.getIds({filter:this._options&&this._options.filter}),s=[],o=0,n=e.length;o<n;o++)s.push(this._data._data[e[o]]);this._ids={},this.length=0,this._trigger("remove",{items:e,oldData:s})}if(this._data=t,this._data){for(this._fieldId=this._options.fieldId||this._data&&this._data.options&&this._data.options.fieldId||"id",e=this._data.getIds({filter:this._options&&this._options.filter}),o=0,n=e.length;o<n;o++)i=e[o],this._ids[i]=!0;this.length=e.length,this._trigger("add",{items:e}),this._data.on&&this._data.on("*",this.listener)}},o.prototype.refresh=function(){var t,e,i,o=this._data.getIds({filter:this._options&&this._options.filter}),n=(0,s.default)(this._ids),r={},a=[],h=[],d=[];for(e=0,i=o.length;e<i;e++)t=o[e],r[t]=!0,this._ids[t]||(a.push(t),this._ids[t]=!0);for(e=0,i=n.length;e<i;e++)t=n[e],r[t]||(h.push(t),d.push(this._data._data[t]),delete this._ids[t]);this.length+=a.length-h.length,a.length&&this._trigger("add",{items:a}),h.length&&this._trigger("remove",{items:h,oldData:d})},o.prototype.get=function(t){var e,i,o,n=this,s=r.getType(arguments[0]);"String"==s||"Number"==s||"Array"==s?(e=arguments[0],i=arguments[1],o=arguments[2]):(i=arguments[0],o=arguments[1]);var a=r.extend({},this._options,i);this._options.filter&&i&&i.filter&&(a.filter=function(t){return n._options.filter(t)&&i.filter(t)});var h=[];return void 0!=e&&h.push(e),h.push(a),h.push(o),this._data&&this._data.get.apply(this._data,h)},o.prototype.getIds=function(t){var e;if(this._data){var i,o=this._options.filter;i=t&&t.filter?o?function(e){return o(e)&&t.filter(e)}:t.filter:o,e=this._data.getIds({filter:i,order:t&&t.order})}else e=[];return e},o.prototype.map=function(t,e){var i=[];if(this._data){var o,n=this._options.filter;o=e&&e.filter?n?function(t){return n(t)&&e.filter(t)}:e.filter:n,i=this._data.map(t,{filter:o,order:e&&e.order})}else i=[];return i},o.prototype.getDataSet=function(){for(var t=this;t instanceof o;)t=t._data;return t||null},o.prototype._onEvent=function(t,e,i){var o,n,s,r,a=e&&e.items,h=[],d=[],l=[],u=[],p=[],c=[];if(a&&this._data){switch(t){case"add":for(o=0,n=a.length;o<n;o++)s=a[o],(r=this.get(s))&&(this._ids[s]=!0,h.push(s));break;case"update":for(o=0,n=a.length;o<n;o++)s=a[o],r=this.get(s),r?this._ids[s]?(d.push(s),p.push(e.data[o]),u.push(e.oldData[o])):(this._ids[s]=!0,h.push(s)):this._ids[s]&&(delete this._ids[s],l.push(s),c.push(e.oldData[o]));break;case"remove":for(o=0,n=a.length;o<n;o++)s=a[o],this._ids[s]&&(delete this._ids[s],l.push(s),c.push(e.oldData[o]))}this.length+=h.length-l.length,h.length&&this._trigger("add",{items:h},i),d.length&&this._trigger("update",{items:d,oldData:u,data:p},i),l.length&&this._trigger("remove",{items:l,oldData:c},i)}},o.prototype.on=a.prototype.on,o.prototype.off=a.prototype.off,o.prototype._trigger=a.prototype._trigger,o.prototype.subscribe=o.prototype.on,o.prototype.unsubscribe=o.prototype.off,t.exports=o},function(t,e,i){e.convertHiddenOptions=function(t,i,o){if(o&&!Array.isArray(o))return e.convertHiddenOptions(t,i,[o]);if(i.hiddenDates=[],o&&1==Array.isArray(o)){for(var n=0;n<o.length;n++)if(void 0===o[n].repeat){var s={};s.start=t(o[n].start).toDate().valueOf(),s.end=t(o[n].end).toDate().valueOf(),i.hiddenDates.push(s)}i.hiddenDates.sort(function(t,e){return t.start-e.start})}},e.updateHiddenDates=function(t,i,o){if(o&&!Array.isArray(o))return e.updateHiddenDates(t,i,[o]);if(o&&void 0!==i.domProps.centerContainer.width){e.convertHiddenOptions(t,i,o);for(var n=t(i.range.start),s=t(i.range.end),r=i.range.end-i.range.start,a=r/i.domProps.centerContainer.width,h=0;h<o.length;h++)if(void 0!==o[h].repeat){var d=t(o[h].start),l=t(o[h].end);if("Invalid Date"==d._d)throw new Error("Supplied start date is not valid: "+o[h].start);if("Invalid Date"==l._d)throw new Error("Supplied end date is not valid: "+o[h].end);var u=l-d;if(u>=4*a){var p=0,c=s.clone();switch(o[h].repeat){case"daily":d.day()!=l.day()&&(p=1),d.dayOfYear(n.dayOfYear()),d.year(n.year()),d.subtract(7,"days"),l.dayOfYear(n.dayOfYear()),l.year(n.year()),l.subtract(7-p,"days"),c.add(1,"weeks");break;case"weekly":var m=l.diff(d,"days"),f=d.day();d.date(n.date()),d.month(n.month()),d.year(n.year()),l=d.clone(),d.day(f),l.day(f),l.add(m,"days"),d.subtract(1,"weeks"),l.subtract(1,"weeks"),c.add(1,"weeks");break;case"monthly":d.month()!=l.month()&&(p=1),d.month(n.month()),d.year(n.year()),d.subtract(1,"months"),l.month(n.month()),l.year(n.year()),l.subtract(1,"months"),l.add(p,"months"),c.add(1,"months");break;case"yearly":d.year()!=l.year()&&(p=1),d.year(n.year()),d.subtract(1,"years"),l.year(n.year()),l.subtract(1,"years"),l.add(p,"years"),c.add(1,"years");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}for(;d<c;)switch(i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()}),o[h].repeat){case"daily":d.add(1,"days"),l.add(1,"days");break;case"weekly":d.add(1,"weeks"),l.add(1,"weeks");break;case"monthly":d.add(1,"months"),l.add(1,"months");break;case"yearly":d.add(1,"y"),l.add(1,"y");break;default:return void console.log("Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:",o[h].repeat)}i.hiddenDates.push({start:d.valueOf(),end:l.valueOf()})}}e.removeDuplicates(i);var g=e.isHidden(i.range.start,i.hiddenDates),v=e.isHidden(i.range.end,i.hiddenDates),y=i.range.start,b=i.range.end;1==g.hidden&&(y=1==i.range.startToFront?g.startDate-1:g.endDate+1),1==v.hidden&&(b=1==i.range.endToFront?v.startDate-1:v.endDate+1),1!=g.hidden&&1!=v.hidden||i.range._applyRange(y,b)}},e.removeDuplicates=function(t){for(var e=t.hiddenDates,i=[],o=0;o<e.length;o++)for(var n=0;n<e.length;n++)o!=n&&1!=e[n].remove&&1!=e[o].remove&&(e[n].start>=e[o].start&&e[n].end<=e[o].end?e[n].remove=!0:e[n].start>=e[o].start&&e[n].start<=e[o].end?(e[o].end=e[n].end,e[n].remove=!0):e[n].end>=e[o].start&&e[n].end<=e[o].end&&(e[o].start=e[n].start,e[n].remove=!0));for(o=0;o<e.length;o++)!0!==e[o].remove&&i.push(e[o]);t.hiddenDates=i,t.hiddenDates.sort(function(t,e){return t.start-e.start})},e.printDates=function(t){for(var e=0;e<t.length;e++)console.log(e,new Date(t[e].start),new Date(t[e].end),t[e].start,t[e].end,t[e].remove)},e.stepOverHiddenDates=function(t,e,i){for(var o=!1,n=e.current.valueOf(),s=0;s<e.hiddenDates.length;s++){var r=e.hiddenDates[s].start,a=e.hiddenDates[s].end;if(n>=r&&n<a){o=!0;break}}if(1==o&&n<e._end.valueOf()&&n!=i){var h=t(i),d=t(a);h.year()!=d.year()?e.switchedYear=!0:h.month()!=d.month()?e.switchedMonth=!0:h.dayOfYear()!=d.dayOfYear()&&(e.switchedDay=!0),e.current=d}},e.toScreen=function(t,i,o){var n;if(0==t.body.hiddenDates.length)return n=t.range.conversion(o),(i.valueOf()-n.offset)*n.scale;var s=e.isHidden(i,t.body.hiddenDates);1==s.hidden&&(i=s.startDate);var r=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end);if(i<t.range.start){n=t.range.conversion(o,r);var a=e.getHiddenDurationBeforeStart(t.body.hiddenDates,i,n.offset);return i=t.options.moment(i).toDate().valueOf(),i+=a,-(n.offset-i.valueOf())*n.scale}if(i>t.range.end){var h={start:t.range.start,end:i};return i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,h,i),n=t.range.conversion(o,r),(i.valueOf()-n.offset)*n.scale}return i=e.correctTimeForHidden(t.options.moment,t.body.hiddenDates,t.range,i),n=t.range.conversion(o,r),(i.valueOf()-n.offset)*n.scale},e.toTime=function(t,i,o){if(0==t.body.hiddenDates.length){var n=t.range.conversion(o);return new Date(i/n.scale+n.offset)}var s=e.getHiddenDurationBetween(t.body.hiddenDates,t.range.start,t.range.end),r=t.range.end-t.range.start-s,a=r*i/o,h=e.getAccumulatedHiddenDuration(t.body.hiddenDates,t.range,a);return new Date(h+a+t.range.start)},e.getHiddenDurationBetween=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&r<i&&(o+=r-s)}return o},e.getHiddenDurationBeforeStart=function(t,e,i){for(var o=0,n=0;n<t.length;n++){var s=t[n].start,r=t[n].end;s>=e&&r<=i&&(o+=r-s)}return o},e.correctTimeForHidden=function(t,i,o,n){return n=t(n).toDate().valueOf(),n-=e.getHiddenDurationBefore(t,i,o,n)},e.getHiddenDurationBefore=function(t,e,i,o){var n=0;o=t(o).toDate().valueOf();for(var s=0;s<e.length;s++){var r=e[s].start,a=e[s].end;r>=i.start&&a<i.end&&o>=a&&(n+=a-r)}return n},e.getAccumulatedHiddenDuration=function(t,e,i){for(var o=0,n=0,s=e.start,r=0;r<t.length;r++){var a=t[r].start,h=t[r].end;if(a>=e.start&&h<e.end){if(n+=a-s,s=h,n>=i)break;o+=h-a}}return o},e.snapAwayFromHidden=function(t,i,o,n){var s=e.isHidden(i,t);return 1==s.hidden?o<0?1==n?s.startDate-(s.endDate-i)-1:s.startDate-1:1==n?s.endDate+(i-s.startDate)+1:s.endDate+1:i},e.isHidden=function(t,e){for(var i=0;i<e.length;i++){var o=e[i].start,n=e[i].end;if(t>=o&&t<n)return{hidden:!0,startDate:o,endDate:n}}return{hidden:!1,startDate:o,endDate:n}}},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e,i){this.id=null,this.parent=null,this.data=t,this.dom=null,this.conversion=e||{},this.options=i||{},this.selected=!1,this.displayed=!1,this.groupShowing=!0,this.dirty=!0,this.top=null,this.right=null,this.left=null,this.width=null,this.height=null,this.editable=null,this._updateEditStatus()}var s=i(1),r=o(s),a=i(4),h=o(a),d=i(12),l=i(0),u=i(3);n.prototype.stack=!0,n.prototype.select=function(){this.selected=!0,this.dirty=!0,this.displayed&&this.redraw()},n.prototype.unselect=function(){this.selected=!1,this.dirty=!0,this.displayed&&this.redraw()},n.prototype.setData=function(t){void 0!=t.group&&this.data.group!=t.group&&null!=this.parent&&this.parent.itemSet._moveToGroup(this,t.group),this.parent&&(this.parent.stackDirty=!0),void 0!=t.subgroup&&this.data.subgroup!=t.subgroup&&null!=this.parent&&this.parent.changeSubgroup(this,this.data.subgroup,t.subgroup),this.data=t,this._updateEditStatus(),this.dirty=!0,this.displayed&&this.redraw()},n.prototype.setParent=function(t){this.displayed?(this.hide(),this.parent=t,this.parent&&this.show()):this.parent=t},n.prototype.isVisible=function(t){return!1},n.prototype.show=function(){return!1},n.prototype.hide=function(){return!1},n.prototype.redraw=function(){},n.prototype.repositionX=function(){},n.prototype.repositionY=function(){},n.prototype._repaintDragCenter=function(){if(this.selected&&this.options.editable.updateTime&&!this.dom.dragCenter){var t=this,e=document.createElement("div");e.className="vis-drag-center",e.dragCenterItem=this;var i=new d(e);i.on("tap",function(e){t.parent.itemSet.body.emitter.emit("click",{event:e,item:t.id})}),i.on("doubletap",function(e){e.stopPropagation(),t.parent.itemSet._onUpdateItem(t),t.parent.itemSet.body.emitter.emit("doubleClick",{event:e,item:t.id})}),this.dom.box?this.dom.dragLeft?this.dom.box.insertBefore(e,this.dom.dragLeft):this.dom.box.appendChild(e):this.dom.point&&this.dom.point.appendChild(e),this.dom.dragCenter=e}else!this.selected&&this.dom.dragCenter&&(this.dom.dragCenter.parentNode&&this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter),this.dom.dragCenter=null)},n.prototype._repaintDeleteButton=function(t){var e=(this.options.editable.overrideItems||null==this.editable)&&this.options.editable.remove||!this.options.editable.overrideItems&&null!=this.editable&&this.editable.remove;if(this.selected&&e&&!this.dom.deleteButton){var i=this,o=document.createElement("div");this.options.rtl?o.className="vis-delete-rtl":o.className="vis-delete",o.title="Delete this item",new d(o).on("tap",function(t){t.stopPropagation(),i.parent.removeFromDataSet(i)}),t.appendChild(o),this.dom.deleteButton=o}else!this.selected&&this.dom.deleteButton&&(this.dom.deleteButton.parentNode&&this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton),this.dom.deleteButton=null)},n.prototype._repaintOnItemUpdateTimeTooltip=function(t){if(this.options.tooltipOnItemUpdateTime){var e=(this.options.editable.updateTime||!0===this.data.editable)&&!1!==this.data.editable;if(this.selected&&e&&!this.dom.onItemUpdateTimeTooltip){var i=document.createElement("div");i.className="vis-onUpdateTime-tooltip",t.appendChild(i),this.dom.onItemUpdateTimeTooltip=i}else!this.selected&&this.dom.onItemUpdateTimeTooltip&&(this.dom.onItemUpdateTimeTooltip.parentNode&&this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip),this.dom.onItemUpdateTimeTooltip=null);if(this.dom.onItemUpdateTimeTooltip){this.dom.onItemUpdateTimeTooltip.style.visibility=this.parent.itemSet.touchParams.itemIsDragging?"visible":"hidden",this.options.rtl?this.dom.onItemUpdateTimeTooltip.style.right=this.dom.content.style.right:this.dom.onItemUpdateTimeTooltip.style.left=this.dom.content.style.left;var o,n=this.parent.itemSet.body.domProps.scrollTop;o="top"==this.options.orientation.item?this.top:this.parent.height-this.top-this.height;o+this.parent.top-50<-n?(this.dom.onItemUpdateTimeTooltip.style.bottom="",this.dom.onItemUpdateTimeTooltip.style.top=this.height+2+"px"):(this.dom.onItemUpdateTimeTooltip.style.top="",this.dom.onItemUpdateTimeTooltip.style.bottom=this.height+2+"px");var s,r;this.options.tooltipOnItemUpdateTime&&this.options.tooltipOnItemUpdateTime.template?(r=this.options.tooltipOnItemUpdateTime.template.bind(this),s=r(this.data)):(s="start: "+u(this.data.start).format("MM/DD/YYYY hh:mm"),this.data.end&&(s+="<br> end: "+u(this.data.end).format("MM/DD/YYYY hh:mm"))),this.dom.onItemUpdateTimeTooltip.innerHTML=s}}},n.prototype._updateContents=function(t){var e,i,o,n,s=this.parent.itemSet.itemsData.get(this.id),r=this.dom.box||this.dom.point,a=r.getElementsByClassName("vis-item-visible-frame")[0];if(this.options.visibleFrameTemplate?(n=this.options.visibleFrameTemplate.bind(this),o=n(s,r)):o="",a)if(o instanceof Object&&!(o instanceof Element))n(s,a);else if(this._contentToString(this.itemVisibleFrameContent)!==this._contentToString(o)){if(o instanceof Element)a.innerHTML="",a.appendChild(o);else if(void 0!=o)a.innerHTML=o;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.itemVisibleFrameContent=o}if(this.options.template?(i=this.options.template.bind(this),e=i(s,t,this.data)):e=this.data.content,e instanceof Object&&!(e instanceof Element))i(s,t);else if(this._contentToString(this.content)!==this._contentToString(e)){if(e instanceof Element)t.innerHTML="",t.appendChild(e);else if(void 0!=e)t.innerHTML=e;else if("background"!=this.data.type||void 0!==this.data.content)throw new Error('Property "content" missing in item '+this.id);this.content=e}},n.prototype._updateDataAttributes=function(t){if(this.options.dataAttributes&&this.options.dataAttributes.length>0){var e=[];if(Array.isArray(this.options.dataAttributes))e=this.options.dataAttributes;else{if("all"!=this.options.dataAttributes)return;e=(0,h.default)(this.data)}for(var i=0;i<e.length;i++){var o=e[i],n=this.data[o];null!=n?t.setAttribute("data-"+o,n):t.removeAttribute("data-"+o)}}},n.prototype._updateStyle=function(t){this.style&&(l.removeCssText(t,this.style),this.style=null),this.data.style&&(l.addCssText(t,this.data.style),this.style=this.data.style)},n.prototype._contentToString=function(t){return"string"==typeof t?t:t&&"outerHTML"in t?t.outerHTML:t},n.prototype._updateEditStatus=function(){this.options&&("boolean"==typeof this.options.editable?this.editable={updateTime:this.options.editable,updateGroup:this.options.editable,remove:this.options.editable}:"object"===(0,r.default)(this.options.editable)&&(this.editable={},l.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.options.editable))),this.options&&this.options.editable&&!0===this.options.editable.overrideItems||this.data&&("boolean"==typeof this.data.editable?this.editable={updateTime:this.data.editable,updateGroup:this.data.editable,remove:this.data.editable}:"object"===(0,r.default)(this.data.editable)&&(this.editable={},l.selectiveExtend(["updateTime","updateGroup","remove"],this.editable,this.data.editable)))},n.prototype.getWidthLeft=function(){return 0},n.prototype.getWidthRight=function(){return 0},n.prototype.getTitle=function(){return this.data.title},t.exports=n},function(t,e){t.exports={}},function(t,e){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,e){t.exports=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}}},function(t,e,i){var o=i(60),n=i(41);t.exports=Object.keys||function(t){return o(t,n)}},function(t,e){var i=0,o=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++i+o).toString(36))}},function(t,e,i){function o(t,e){this.dom={foreground:null,lines:[],majorTexts:[],minorTexts:[],redundant:{lines:[],majorTexts:[],minorTexts:[]}},this.props={range:{start:0,end:0,minimumStep:0},lineTop:0},this.defaultOptions={orientation:{axis:"bottom"},showMinorLabels:!0,showMajorLabels:!0,maxMinorChars:7,format:h.FORMAT,moment:l,timeAxis:null},this.options=r.extend({},this.defaultOptions),this.body=t,this._create(),this.setOptions(e)}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(5),h=i(49),d=i(21),l=i(3);o.prototype=new a,o.prototype.setOptions=function(t){t&&(r.selectiveExtend(["showMinorLabels","showMajorLabels","maxMinorChars","hiddenDates","timeAxis","moment","rtl"],this.options,t),r.selectiveDeepExtend(["format"],this.options,t),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.axis=t.orientation:"object"===(0,s.default)(t.orientation)&&"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis)),"locale"in t&&("function"==typeof l.locale?l.locale(t.locale):l.lang(t.locale)))},o.prototype._create=function(){this.dom.foreground=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.foreground.className="vis-time-axis vis-foreground",this.dom.background.className="vis-time-axis vis-background"},o.prototype.destroy=function(){this.dom.foreground.parentNode&&this.dom.foreground.parentNode.removeChild(this.dom.foreground),this.dom.background.parentNode&&this.dom.background.parentNode.removeChild(this.dom.background),this.body=null},o.prototype.redraw=function(){var t=this.props,e=this.dom.foreground,i=this.dom.background,o="top"==this.options.orientation.axis?this.body.dom.top:this.body.dom.bottom,n=e.parentNode!==o;this._calculateCharSize();var s=this.options.showMinorLabels&&"none"!==this.options.orientation.axis,r=this.options.showMajorLabels&&"none"!==this.options.orientation.axis;t.minorLabelHeight=s?t.minorCharHeight:0,t.majorLabelHeight=r?t.majorCharHeight:0,t.height=t.minorLabelHeight+t.majorLabelHeight,t.width=e.offsetWidth,t.minorLineHeight=this.body.domProps.root.height-t.majorLabelHeight-("top"==this.options.orientation.axis?this.body.domProps.bottom.height:this.body.domProps.top.height),t.minorLineWidth=1,t.majorLineHeight=t.minorLineHeight+t.majorLabelHeight,t.majorLineWidth=1;var a=e.nextSibling,h=i.nextSibling;return e.parentNode&&e.parentNode.removeChild(e),i.parentNode&&i.parentNode.removeChild(i),e.style.height=this.props.height+"px",this._repaintLabels(),a?o.insertBefore(e,a):o.appendChild(e),h?this.body.dom.backgroundVertical.insertBefore(i,h):this.body.dom.backgroundVertical.appendChild(i),this._isResized()||n},o.prototype._repaintLabels=function(){var t=this.options.orientation.axis,e=r.convert(this.body.range.start,"Number"),i=r.convert(this.body.range.end,"Number"),o=this.body.util.toTime((this.props.minorCharWidth||10)*this.options.maxMinorChars).valueOf(),n=o-d.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this.body.range,o);n-=this.body.util.toTime(0).valueOf();var s=new h(new Date(e),new Date(i),n,this.body.hiddenDates,this.options);s.setMoment(this.options.moment),this.options.format&&s.setFormat(this.options.format),this.options.timeAxis&&s.setScale(this.options.timeAxis),this.step=s;var a=this.dom;a.redundant.lines=a.lines,a.redundant.majorTexts=a.majorTexts,a.redundant.minorTexts=a.minorTexts,a.lines=[],a.majorTexts=[],a.minorTexts=[];var l,p,c,m,f,g,v,y,b,_=0,w=void 0,x=0;for(s.start(),l=s.getCurrent(),c=this.body.util.toScreen(l);s.hasNext()&&x<1e3;){switch(x++,m=s.isMajor(),b=s.getClassName(),y=s.getLabelMinor(),l,p=c,s.next(),l=s.getCurrent(),s.isMajor(),c=this.body.util.toScreen(l),g=_,_=c-p,s.scale){case"week":f=!0;break;default:f=_>=.4*g}if(this.options.showMinorLabels&&f){var D=this._repaintMinorText(p,y,t,b);D.style.width=_+"px"}m&&this.options.showMajorLabels?(p>0&&(void 0==w&&(w=p),D=this._repaintMajorText(p,s.getLabelMajor(),t,b)),v=this._repaintMajorLine(p,_,t,b)):f?v=this._repaintMinorLine(p,_,t,b):v&&(v.style.width=parseInt(v.style.width)+_+"px")}if(1e3!==x||u||(console.warn("Something is wrong with the Timeline scale. Limited drawing of grid lines to 1000 lines."),u=!0),this.options.showMajorLabels){var S=this.body.util.toTime(0),k=s.getLabelMajor(S),C=k.length*(this.props.majorCharWidth||10)+10;(void 0==w||C<w)&&this._repaintMajorText(0,k,t,b)}r.forEach(this.dom.redundant,function(t){for(;t.length;){var e=t.pop();e&&e.parentNode&&e.parentNode.removeChild(e)}})},o.prototype._repaintMinorText=function(t,e,i,o){var n=this.dom.redundant.minorTexts.shift();if(!n){var s=document.createTextNode("");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return this.dom.minorTexts.push(n),n.innerHTML=e,n.style.top="top"==i?this.props.majorLabelHeight+"px":"0",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",n.className="vis-text vis-minor "+o,n},o.prototype._repaintMajorText=function(t,e,i,o){var n=this.dom.redundant.majorTexts.shift();if(!n){var s=document.createElement("div");n=document.createElement("div"),n.appendChild(s),this.dom.foreground.appendChild(n)}return n.childNodes[0].innerHTML=e,n.className="vis-text vis-major "+o,n.style.top="top"==i?"0":this.props.minorLabelHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t+"px"):n.style.left=t+"px",this.dom.majorTexts.push(n),n},o.prototype._repaintMinorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return n.style.top="top"==i?s.majorLabelHeight+"px":this.body.domProps.top.height+"px",n.style.height=s.minorLineHeight+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-minor "+o):(n.style.left=t-s.minorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-minor "+o),n.style.width=e+"px",n},o.prototype._repaintMajorLine=function(t,e,i,o){var n=this.dom.redundant.lines.shift();n||(n=document.createElement("div"),this.dom.background.appendChild(n)),this.dom.lines.push(n);var s=this.props;return n.style.top="top"==i?"0":this.body.domProps.top.height+"px",this.options.rtl?(n.style.left="",n.style.right=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical-rtl vis-major "+o):(n.style.left=t-s.majorLineWidth/2+"px",n.className="vis-grid vis-vertical vis-major "+o),n.style.height=s.majorLineHeight+"px",n.style.width=e+"px",n},o.prototype._calculateCharSize=function(){this.dom.measureCharMinor||(this.dom.measureCharMinor=document.createElement("DIV"),this.dom.measureCharMinor.className="vis-text vis-minor vis-measure",this.dom.measureCharMinor.style.position="absolute",this.dom.measureCharMinor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMinor)),this.props.minorCharHeight=this.dom.measureCharMinor.clientHeight,this.props.minorCharWidth=this.dom.measureCharMinor.clientWidth,this.dom.measureCharMajor||(this.dom.measureCharMajor=document.createElement("DIV"),this.dom.measureCharMajor.className="vis-text vis-major vis-measure",this.dom.measureCharMajor.style.position="absolute",this.dom.measureCharMajor.appendChild(document.createTextNode("0")),this.dom.foreground.appendChild(this.dom.measureCharMajor)),this.props.majorCharHeight=this.dom.measureCharMajor.clientHeight,this.props.majorCharWidth=this.dom.measureCharMajor.clientWidth};var u=!1;t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={moment:a,locales:h,locale:"en",id:void 0,title:void 0},this.options=s.extend({},this.defaultOptions),e&&e.time?this.customTime=e.time:this.customTime=new Date,this.eventParams={},this.setOptions(e),this._create()}var n=i(12),s=i(0),r=i(5),a=i(3),h=i(69);o.prototype=new r,o.prototype.setOptions=function(t){t&&s.selectiveExtend(["moment","locale","locales","id"],this.options,t)},o.prototype._create=function(){function t(t){this.body.range._onMouseWheel(t)}var e=document.createElement("div");e["custom-time"]=this,e.className="vis-custom-time "+(this.options.id||""),e.style.position="absolute",e.style.top="0px",e.style.height="100%",this.bar=e;var i=document.createElement("div");i.style.position="relative",i.style.top="0px",i.style.left="-10px",i.style.height="100%",i.style.width="20px",i.addEventListener?(i.addEventListener("mousewheel",t.bind(this),!1),i.addEventListener("DOMMouseScroll",t.bind(this),!1)):i.attachEvent("onmousewheel",t.bind(this)),e.appendChild(i),this.hammer=new n(i),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:n.DIRECTION_HORIZONTAL})},o.prototype.destroy=function(){this.hide(),this.hammer.destroy(),this.hammer=null,this.body=null},o.prototype.redraw=function(){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar));var e=this.body.util.toScreen(this.customTime),i=this.options.locales[this.options.locale];i||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),i=this.options.locales.en);var o=this.options.title;return void 0===o?(o=i.time+": "+this.options.moment(this.customTime).format("dddd, MMMM Do YYYY, H:mm:ss"),o=o.charAt(0).toUpperCase()+o.substring(1)):"function"==typeof o&&(o=o.call(this.customTime)),this.bar.style.left=e+"px",this.bar.title=o,!1},o.prototype.hide=function(){this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar)},o.prototype.setCustomTime=function(t){this.customTime=s.convert(t,"Date"),this.redraw()},o.prototype.getCustomTime=function(){return new Date(this.customTime.valueOf())},o.prototype.setCustomTitle=function(t){this.options.title=t},o.prototype._onDragStart=function(t){this.eventParams.dragging=!0,this.eventParams.customTime=this.customTime,t.stopPropagation()},o.prototype._onDrag=function(t){if(this.eventParams.dragging){var e=this.body.util.toScreen(this.eventParams.customTime)+t.deltaX,i=this.body.util.toTime(e);this.setCustomTime(i),this.body.emitter.emit("timechange",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation()}},o.prototype._onDragEnd=function(t){this.eventParams.dragging&&(this.body.emitter.emit("timechanged",{id:this.options.id,time:new Date(this.customTime.valueOf()),event:t}),t.stopPropagation())},o.customTimeFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("custom-time"))return e["custom-time"];e=e.parentNode}return null},t.exports=o},function(t,e,i){e.__esModule=!0,e.default=function(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}},function(t,e,i){e.__esModule=!0;var o=i(134),n=function(t){return t&&t.__esModule?t:{default:t}}(o);e.default=function(){function t(t,e){for(var i=0;i<e.length;i++){var o=e[i];o.enumerable=o.enumerable||!1,o.configurable=!0,"value"in o&&(o.writable=!0),(0,n.default)(t,o.key,o)}}return function(e,i,o){return i&&t(e.prototype,i),o&&t(e,o),e}}()},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0}),e.printStyle=void 0;var n=i(16),s=o(n),r=i(1),a=o(r),h=i(4),d=o(h),l=i(30),u=o(l),p=i(31),c=o(p),m=i(0),f=!1,g=void 0,v="background: #FFeeee; color: #dd0000",y=function(){function t(){(0,u.default)(this,t)}return(0,c.default)(t,null,[{key:"validate",value:function(e,i,o){f=!1,g=i;var n=i;return void 0!==o&&(n=i[o]),t.parse(e,n,[]),f}},{key:"parse",value:function(e,i,o){for(var n in e)e.hasOwnProperty(n)&&t.check(n,e,i,o)}},{key:"check",value:function(e,i,o,n){if(void 0===o[e]&&void 0===o.__any__)return void t.getSuggestion(e,o,n);var s=e,r=!0;void 0===o[e]&&void 0!==o.__any__&&(s="__any__",r="object"===t.getType(i[e]));var a=o[s];r&&void 0!==a.__type__&&(a=a.__type__),t.checkFields(e,i,o,s,a,n)}},{key:"checkFields",value:function(e,i,o,n,s,r){var a=function(i){console.log("%c"+i+t.printLocation(r,e),v)},h=t.getType(i[e]),l=s[h];void 0!==l?"array"===t.getType(l)&&-1===l.indexOf(i[e])?(a('Invalid option detected in "'+e+'". Allowed values are:'+t.print(l)+' not "'+i[e]+'". '),f=!0):"object"===h&&"__any__"!==n&&(r=m.copyAndExtendArray(r,e),t.parse(i[e],o[n],r)):void 0===s.any&&(a('Invalid type received for "'+e+'". Expected: '+t.print((0,d.default)(s))+". Received ["+h+'] "'+i[e]+'"'),f=!0)}},{key:"getType",value:function(t){var e=void 0===t?"undefined":(0,a.default)(t);return"object"===e?null===t?"null":t instanceof Boolean?"boolean":t instanceof Number?"number":t instanceof String?"string":Array.isArray(t)?"array":t instanceof Date?"date":void 0!==t.nodeType?"dom":!0===t._isAMomentObject?"moment":"object":"number"===e?"number":"boolean"===e?"boolean":"string"===e?"string":void 0===e?"undefined":e}},{key:"getSuggestion",value:function(e,i,o){var n=t.findInOptions(e,i,o,!1),s=t.findInOptions(e,g,[],!0),r=void 0;r=void 0!==n.indexMatch?" in "+t.printLocation(n.path,e,"")+'Perhaps it was incomplete? Did you mean: "'+n.indexMatch+'"?\n\n':s.distance<=4&&n.distance>s.distance?" in "+t.printLocation(n.path,e,"")+"Perhaps it was misplaced? Matching option found at: "+t.printLocation(s.path,s.closestMatch,""):n.distance<=8?'. Did you mean "'+n.closestMatch+'"?'+t.printLocation(n.path,e):". Did you mean one of these: "+t.print((0,d.default)(i))+t.printLocation(o,e),console.log('%cUnknown option detected: "'+e+'"'+r,v),f=!0}},{key:"findInOptions",value:function(e,i,o){var n=arguments.length>3&&void 0!==arguments[3]&&arguments[3],s=1e9,r="",a=[],h=e.toLowerCase(),d=void 0;for(var l in i){var u=void 0;if(void 0!==i[l].__type__&&!0===n){var p=t.findInOptions(e,i[l],m.copyAndExtendArray(o,l));s>p.distance&&(r=p.closestMatch,a=p.path,s=p.distance,d=p.indexMatch)}else-1!==l.toLowerCase().indexOf(h)&&(d=l),u=t.levenshteinDistance(e,l),s>u&&(r=l,a=m.copyArray(o),s=u)}return{closestMatch:r,path:a,distance:s,indexMatch:d}}},{key:"printLocation",value:function(t,e){for(var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"Problem value found at: \n",o="\n\n"+i+"options = {\n",n=0;n<t.length;n++){for(var s=0;s<n+1;s++)o+="  ";o+=t[n]+": {\n"}for(var r=0;r<t.length+1;r++)o+="  ";o+=e+"\n";for(var a=0;a<t.length+1;a++){for(var h=0;h<t.length-a;h++)o+="  ";o+="}\n"}return o+"\n\n"}},{key:"print",value:function(t){return(0,s.default)(t).replace(/(\")|(\[)|(\])|(,"__type__")/g,"").replace(/(\,)/g,", ")}},{key:"levenshteinDistance",value:function(t,e){if(0===t.length)return e.length;if(0===e.length)return t.length;var i,o=[];for(i=0;i<=e.length;i++)o[i]=[i];var n;for(n=0;n<=t.length;n++)o[0][n]=n;for(i=1;i<=e.length;i++)for(n=1;n<=t.length;n++)e.charAt(i-1)==t.charAt(n-1)?o[i][n]=o[i-1][n-1]:o[i][n]=Math.min(o[i-1][n-1]+1,Math.min(o[i][n-1]+1,o[i-1][n]+1));return o[e.length][t.length]}}]),t}();e.default=y,e.printStyle=v},function(t,e){var i={}.toString;t.exports=function(t){return i.call(t).slice(8,-1)}},function(t,e){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on  "+t);return t}},function(t,e){t.exports=!0},function(t,e,i){var o=i(24);t.exports=function(t,e){if(!o(t))return t;var i,n;if(e&&"function"==typeof(i=t.toString)&&!o(n=i.call(t)))return n;if("function"==typeof(i=t.valueOf)&&!o(n=i.call(t)))return n;if(!e&&"function"==typeof(i=t.toString)&&!o(n=i.call(t)))return n;throw TypeError("Can't convert object to primitive value")}},function(t,e,i){var o=i(18),n=i(95),s=i(41),r=i(39)("IE_PROTO"),a=function(){},h=function(){var t,e=i(58)("iframe"),o=s.length;for(e.style.display="none",i(99).appendChild(e),e.src="javascript:",t=e.contentWindow.document,t.open(),t.write("<script>document.F=Object<\/script>"),t.close(),h=t.F;o--;)delete h.prototype[s[o]];return h()};t.exports=Object.create||function(t,e){var i;return null!==t?(a.prototype=o(t),i=new a,a.prototype=null,i[r]=t):i=h(),void 0===e?i:n(i,e)}},function(t,e){var i=Math.ceil,o=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?o:i)(t)}},function(t,e,i){var o=i(40)("keys"),n=i(27);t.exports=function(t){return o[t]||(o[t]=n(t))}},function(t,e,i){var o=i(6),n=o["__core-js_shared__"]||(o["__core-js_shared__"]={});t.exports=function(t){return n[t]||(n[t]={})}},function(t,e){t.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(t,e,i){var o=i(8).f,n=i(10),s=i(7)("toStringTag");t.exports=function(t,e,i){t&&!n(t=i?t:t.prototype,s)&&o(t,s,{configurable:!0,value:e})}},function(t,e,i){t.exports={default:i(105),__esModule:!0}},function(t,e,i){e.f=i(7)},function(t,e,i){var o=i(6),n=i(2),s=i(35),r=i(44),a=i(8).f;t.exports=function(t){var e=n.Symbol||(n.Symbol=s?{}:o.Symbol||{});"_"==t.charAt(0)||t in e||a(e,t,{value:r.f(t)})}},function(t,e){e.f={}.propertyIsEnumerable},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){var i=c().hours(0).minutes(0).seconds(0).milliseconds(0),o=i.clone().add(-3,"days").valueOf(),n=i.clone().add(3,"days").valueOf();this.millisecondsPerPixelCache=void 0,void 0===e?(this.start=o,this.end=n):(this.start=e.start||o,this.end=e.end||n),this.rolling=!1,this.body=t,this.deltaDifference=0,this.scaleOffset=0,this.startToFront=!1,this.endToFront=!0,this.defaultOptions={rtl:!1,start:null,end:null,moment:c,direction:"horizontal",moveable:!0,zoomable:!0,min:null,max:null,zoomMin:10,zoomMax:31536e10,rollingMode:{follow:!1,offset:.5}},this.options=p.extend({},this.defaultOptions),this.props={touch:{}},this.animationTimer=null,this.body.emitter.on("panstart",this._onDragStart.bind(this)),this.body.emitter.on("panmove",this._onDrag.bind(this)),this.body.emitter.on("panend",this._onDragEnd.bind(this)),this.body.emitter.on("mousewheel",this._onMouseWheel.bind(this)),this.body.emitter.on("touch",this._onTouch.bind(this)),this.body.emitter.on("pinch",this._onPinch.bind(this)),this.body.dom.rollingModeBtn.addEventListener("click",this.startRolling.bind(this)),this.setOptions(e)}function s(t){if("horizontal"!=t&&"vertical"!=t)throw new TypeError('Unknown direction "'+t+'". Choose "horizontal" or "vertical".')}var r=i(4),a=o(r),h=i(16),d=o(h),l=i(1),u=o(l),p=i(0),c=i(3),m=i(5),f=i(21);n.prototype=new m,n.prototype.setOptions=function(t){if(t){var e=["animation","direction","min","max","zoomMin","zoomMax","moveable","zoomable","moment","activate","hiddenDates","zoomKey","rtl","showCurrentTime","rollingMode","horizontalScroll"];p.selectiveExtend(e,this.options,t),t.rollingMode&&t.rollingMode.follow&&this.startRolling(),("start"in t||"end"in t)&&this.setRange(t.start,t.end)}},n.prototype.startRolling=function(){function t(){e.stopRolling(),e.rolling=!0;var i=e.end-e.start,o=p.convert(new Date,"Date").valueOf(),n=o-i*e.options.rollingMode.offset,s=o+i*(1-e.options.rollingMode.offset),r={animation:!1};e.setRange(n,s,r),i=1/e.conversion(e.body.domProps.center.width).scale/10,i<30&&(i=30),i>1e3&&(i=1e3),e.body.dom.rollingModeBtn.style.visibility="hidden",e.currentTimeTimer=setTimeout(t,i)}var e=this;t()},n.prototype.stopRolling=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),this.rolling=!1,this.body.dom.rollingModeBtn.style.visibility="visible")},n.prototype.setRange=function(t,e,i,o,n){i||(i={}),!0!==i.byUser&&(i.byUser=!1);var s=this,r=void 0!=t?p.convert(t,"Date").valueOf():null,h=void 0!=e?p.convert(e,"Date").valueOf():null;if(this._cancelAnimation(),this.millisecondsPerPixelCache=void 0,i.animation){var l=this.start,c=this.end,m="object"===(0,u.default)(i.animation)&&"duration"in i.animation?i.animation.duration:500,g="object"===(0,u.default)(i.animation)&&"easingFunction"in i.animation?i.animation.easingFunction:"easeInOutQuad",v=p.easingFunctions[g];if(!v)throw new Error("Unknown easing function "+(0,d.default)(g)+". Choose from: "+(0,a.default)(p.easingFunctions).join(", "));var y=(new Date).valueOf(),b=!1;return function t(){if(!s.props.touch.dragging){var e=(new Date).valueOf(),a=e-y,d=v(a/m),u=a>m,p=u||null===r?r:l+(r-l)*d,g=u||null===h?h:c+(h-c)*d;_=s._applyRange(p,g),f.updateHiddenDates(s.options.moment,s.body,s.options.hiddenDates),b=b||_;var w={start:new Date(s.start),end:new Date(s.end),byUser:i.byUser,event:i.event};if(n&&n(d,_,u),_&&s.body.emitter.emit("rangechange",w),u){if(b&&(s.body.emitter.emit("rangechanged",w),o))return o()}else s.animationTimer=setTimeout(t,20)}}()}var _=this._applyRange(r,h);if(f.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),_){var w={start:new Date(this.start),end:new Date(this.end),byUser:i.byUser,event:i.event};if(this.body.emitter.emit("rangechange",w),clearTimeout(s.timeoutID),s.timeoutID=setTimeout(function(){s.body.emitter.emit("rangechanged",w)},200),o)return o()}},n.prototype.getMillisecondsPerPixel=function(){return void 0===this.millisecondsPerPixelCache&&(this.millisecondsPerPixelCache=(this.end-this.start)/this.body.dom.center.clientWidth),this.millisecondsPerPixelCache},n.prototype._cancelAnimation=function(){this.animationTimer&&(clearTimeout(this.animationTimer),this.animationTimer=null)},n.prototype._applyRange=function(t,e){var i,o=null!=t?p.convert(t,"Date").valueOf():this.start,n=null!=e?p.convert(e,"Date").valueOf():this.end,s=null!=this.options.max?p.convert(this.options.max,"Date").valueOf():null,r=null!=this.options.min?p.convert(this.options.min,"Date").valueOf():null;if(isNaN(o)||null===o)throw new Error('Invalid start "'+t+'"');if(isNaN(n)||null===n)throw new Error('Invalid end "'+e+'"');if(n<o&&(n=o),null!==r&&o<r&&(i=r-o,o+=i,n+=i,null!=s&&n>s&&(n=s)),null!==s&&n>s&&(i=n-s,o-=i,n-=i,null!=r&&o<r&&(o=r)),null!==this.options.zoomMin){var a=parseFloat(this.options.zoomMin);if(a<0&&(a=0),n-o<a){this.end-this.start===a&&o>=this.start-.5&&n<=this.end?(o=this.start,n=this.end):(i=a-(n-o),o-=i/2,n+=i/2)}}if(null!==this.options.zoomMax){var h=parseFloat(this.options.zoomMax);h<0&&(h=0),n-o>h&&(this.end-this.start===h&&o<this.start&&n>this.end?(o=this.start,n=this.end):(i=n-o-h,o+=i/2,n-=i/2))}var d=this.start!=o||this.end!=n;return o>=this.start&&o<=this.end||n>=this.start&&n<=this.end||this.start>=o&&this.start<=n||this.end>=o&&this.end<=n||this.body.emitter.emit("checkRangedItems"),this.start=o,this.end=n,d},n.prototype.getRange=function(){return{start:this.start,end:this.end}},n.prototype.conversion=function(t,e){return n.conversion(this.start,this.end,t,e)},n.conversion=function(t,e,i,o){return void 0===o&&(o=0),0!=i&&e-t!=0?{offset:t,scale:i/(e-t-o)}:{offset:0,scale:1}},n.prototype._onDragStart=function(t){this.deltaDifference=0,this.previousDelta=0,this.options.moveable&&this._isInsideRange(t)&&this.props.touch.allowDragging&&(this.stopRolling(),this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.dragging=!0,this.body.dom.root&&(this.body.dom.root.style.cursor="move"))},n.prototype._onDrag=function(t){if(t&&this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging){var e=this.options.direction;s(e);var i="horizontal"==e?t.deltaX:t.deltaY;i-=this.deltaDifference;var o=this.props.touch.end-this.props.touch.start;o-=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end);var n,r="horizontal"==e?this.body.domProps.center.width:this.body.domProps.center.height;n=this.options.rtl?i/r*o:-i/r*o;var a=this.props.touch.start+n,h=this.props.touch.end+n,d=f.snapAwayFromHidden(this.body.hiddenDates,a,this.previousDelta-i,!0),l=f.snapAwayFromHidden(this.body.hiddenDates,h,this.previousDelta-i,!0);if(d!=a||l!=h)return this.deltaDifference+=i,this.props.touch.start=d,this.props.touch.end=l,void this._onDrag(t);this.previousDelta=i,this._applyRange(a,h);var u=new Date(this.start),p=new Date(this.end);this.body.emitter.emit("rangechange",{start:u,end:p,byUser:!0,event:t}),this.body.emitter.emit("panmove")}},n.prototype._onDragEnd=function(t){this.props.touch.dragging&&this.options.moveable&&this.props.touch.allowDragging&&(this.props.touch.dragging=!1,this.body.dom.root&&(this.body.dom.root.style.cursor="auto"),this.body.emitter.emit("rangechanged",{start:new Date(this.start),end:new Date(this.end),byUser:!0,event:t}))},n.prototype._onMouseWheel=function(t){var e=0;if(t.wheelDelta?e=t.wheelDelta/120:t.detail&&(e=-t.detail/3),!(this.options.zoomKey&&!t[this.options.zoomKey]&&this.options.zoomable||!this.options.zoomable&&this.options.moveable)&&this.options.zoomable&&this.options.moveable&&this._isInsideRange(t)&&e){var i;i=e<0?1-e/5:1/(1+e/5);var o;if(this.rolling)o=this.start+(this.end-this.start)*this.options.rollingMode.offset;else{var n=this.getPointer({x:t.clientX,y:t.clientY},this.body.dom.center);o=this._pointerToDate(n)}this.zoom(i,o,e,t),t.preventDefault()}},n.prototype._onTouch=function(t){this.props.touch.start=this.start,this.props.touch.end=this.end,this.props.touch.allowDragging=!0,this.props.touch.center=null,this.scaleOffset=0,this.deltaDifference=0,p.preventDefault(t)},n.prototype._onPinch=function(t){if(this.options.zoomable&&this.options.moveable){p.preventDefault(t),this.props.touch.allowDragging=!1,this.props.touch.center||(this.props.touch.center=this.getPointer(t.center,this.body.dom.center)),this.stopRolling();var e=1/(t.scale+this.scaleOffset),i=this._pointerToDate(this.props.touch.center),o=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),n=f.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,i),s=o-n,r=i-n+(this.props.touch.start-(i-n))*e,a=i+s+(this.props.touch.end-(i+s))*e;this.startToFront=1-e<=0,this.endToFront=e-1<=0;var h=f.snapAwayFromHidden(this.body.hiddenDates,r,1-e,!0),d=f.snapAwayFromHidden(this.body.hiddenDates,a,e-1,!0);h==r&&d==a||(this.props.touch.start=h,this.props.touch.end=d,this.scaleOffset=1-t.scale,r=h,a=d);var l={animation:!1,byUser:!0,event:t};this.setRange(r,a,l),this.startToFront=!1,this.endToFront=!0}},n.prototype._isInsideRange=function(t){var e,i=t.center?t.center.x:t.clientX;e=this.options.rtl?i-p.getAbsoluteLeft(this.body.dom.centerContainer):p.getAbsoluteRight(this.body.dom.centerContainer)-i;var o=this.body.util.toTime(e);return o>=this.start&&o<=this.end},n.prototype._pointerToDate=function(t){var e,i=this.options.direction;if(s(i),"horizontal"==i)return this.body.util.toTime(t.x).valueOf();var o=this.body.domProps.center.height;return e=this.conversion(o),t.y/e.scale+e.offset},n.prototype.getPointer=function(t,e){return this.options.rtl?{x:p.getAbsoluteRight(e)-t.x,y:t.y-p.getAbsoluteTop(e)}:{x:t.x-p.getAbsoluteLeft(e),y:t.y-p.getAbsoluteTop(e)}},n.prototype.zoom=function(t,e,i,o){null==e&&(e=(this.start+this.end)/2);var n=f.getHiddenDurationBetween(this.body.hiddenDates,this.start,this.end),s=f.getHiddenDurationBefore(this.options.moment,this.body.hiddenDates,this,e),r=n-s,a=e-s+(this.start-(e-s))*t,h=e+r+(this.end-(e+r))*t;this.startToFront=!(i>0),this.endToFront=!(-i>0);var d=f.snapAwayFromHidden(this.body.hiddenDates,a,i,!0),l=f.snapAwayFromHidden(this.body.hiddenDates,h,-i,!0);d==a&&l==h||(a=d,h=l);var u={animation:!1,byUser:!0,event:o};this.setRange(a,h,u),this.startToFront=!1,this.endToFront=!0},n.prototype.move=function(t){var e=this.end-this.start,i=this.start+e*t,o=this.end+e*t;this.start=i,this.end=o},n.prototype.moveTo=function(t){var e=(this.start+this.end)/2,i=e-t,o=this.start-i,n=this.end-i,s={animation:!1,byUser:!0,event:null};this.setRange(o,n,s)},t.exports=n},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(){}var s=i(16),r=o(s),a=i(1),h=o(a),d=i(66),l=i(12),u=i(67),p=i(0),c=i(28),m=i(132),f=i(21),g=i(29);d(n.prototype),n.prototype._create=function(t){function e(t){this.isActive()&&this.emit("mousewheel",t);var e=0,i=0;if("detail"in t&&(i=-1*t.detail),"wheelDelta"in t&&(i=t.wheelDelta),"wheelDeltaY"in t&&(i=t.wheelDeltaY),"wheelDeltaX"in t&&(e=-1*t.wheelDeltaX),"axis"in t&&t.axis===t.HORIZONTAL_AXIS&&(e=-1*i,i=0),"deltaY"in t&&(i=-1*t.deltaY),"deltaX"in t&&(e=t.deltaX),this.options.zoomKey&&!t[this.options.zoomKey])if(t.preventDefault(),this.options.verticalScroll&&Math.abs(i)>=Math.abs(e)){var o=this.props.scrollTop,n=o+i;this.isActive()&&(this._setScrollTop(n),this._redraw(),this.emit("scroll",t))}else if(this.options.horizontalScroll){var s=Math.abs(e)>=Math.abs(i)?e:i,r=s/120*(this.range.end-this.range.start)/20,a=this.range.start+r,h=this.range.end+r,d={animation:!1,byUser:!0,event:t};this.range.setRange(a,h,d)}}function i(t){if(s.options.verticalScroll&&(t.preventDefault(),s.isActive())){var e=-t.target.scrollTop;s._setScrollTop(e),s._redraw(),s.emit("scrollSide",t)}}function o(t){if(t.preventDefault&&t.preventDefault(),!(!t.target.className.indexOf("vis")>-1||a))return t.dataTransfer.dropEffect="move",a=!0,!1}function n(t){t.preventDefault&&t.preventDefault(),t.stopPropagation&&t.stopPropagation();try{var e=JSON.parse(t.dataTransfer.getData("text"));if(!e||!e.content)return}catch(t){return!1}return a=!1,t.center={x:t.clientX,y:t.clientY},"item"!==e.target?s.itemSet._onAddItem(t):s.itemSet._onDropObjectOnItem(t),s.emit("drop",s.getEventProperties(t)),!1}this.dom={},this.dom.container=t,this.dom.root=document.createElement("div"),this.dom.background=document.createElement("div"),this.dom.backgroundVertical=document.createElement("div"),this.dom.backgroundHorizontal=document.createElement("div"),this.dom.centerContainer=document.createElement("div"),this.dom.leftContainer=document.createElement("div"),this.dom.rightContainer=document.createElement("div"),this.dom.center=document.createElement("div"),this.dom.left=document.createElement("div"),this.dom.right=document.createElement("div"),this.dom.top=document.createElement("div"),this.dom.bottom=document.createElement("div"),this.dom.shadowTop=document.createElement("div"),this.dom.shadowBottom=document.createElement("div"),this.dom.shadowTopLeft=document.createElement("div"),this.dom.shadowBottomLeft=document.createElement("div"),this.dom.shadowTopRight=document.createElement("div"),this.dom.shadowBottomRight=document.createElement("div"),this.dom.rollingModeBtn=document.createElement("div"),this.dom.root.className="vis-timeline",this.dom.background.className="vis-panel vis-background",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical",this.dom.backgroundHorizontal.className="vis-panel vis-background vis-horizontal",this.dom.centerContainer.className="vis-panel vis-center",this.dom.leftContainer.className="vis-panel vis-left",this.dom.rightContainer.className="vis-panel vis-right",this.dom.top.className="vis-panel vis-top",this.dom.bottom.className="vis-panel vis-bottom",this.dom.left.className="vis-content",this.dom.center.className="vis-content",this.dom.right.className="vis-content",this.dom.shadowTop.className="vis-shadow vis-top",this.dom.shadowBottom.className="vis-shadow vis-bottom",this.dom.shadowTopLeft.className="vis-shadow vis-top",this.dom.shadowBottomLeft.className="vis-shadow vis-bottom",this.dom.shadowTopRight.className="vis-shadow vis-top",this.dom.shadowBottomRight.className="vis-shadow vis-bottom",this.dom.rollingModeBtn.className="vis-rolling-mode-btn",this.dom.root.appendChild(this.dom.background),this.dom.root.appendChild(this.dom.backgroundVertical),this.dom.root.appendChild(this.dom.backgroundHorizontal),this.dom.root.appendChild(this.dom.centerContainer),this.dom.root.appendChild(this.dom.leftContainer),this.dom.root.appendChild(this.dom.rightContainer),this.dom.root.appendChild(this.dom.top),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.bottom),this.dom.root.appendChild(this.dom.rollingModeBtn),this.dom.centerContainer.appendChild(this.dom.center),this.dom.leftContainer.appendChild(this.dom.left),this.dom.rightContainer.appendChild(this.dom.right),this.dom.centerContainer.appendChild(this.dom.shadowTop),this.dom.centerContainer.appendChild(this.dom.shadowBottom),this.dom.leftContainer.appendChild(this.dom.shadowTopLeft),this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft),this.dom.rightContainer.appendChild(this.dom.shadowTopRight),this.dom.rightContainer.appendChild(this.dom.shadowBottomRight),this.props={root:{},background:{},centerContainer:{},leftContainer:{},rightContainer:{},center:{},left:{},right:{},top:{},bottom:{},border:{},scrollTop:0,scrollTopMin:0},this.on("rangechange",function(){!0===this.initialDrawDone&&this._redraw()}.bind(this)),this.on("rangechanged",function(){this.initialRangeChangeDone||(this.initialRangeChangeDone=!0)}.bind(this)),this.on("touch",this._onTouch.bind(this)),this.on("panmove",this._onDrag.bind(this));var s=this;this._origRedraw=this._redraw.bind(this),this._redraw=p.throttle(this._origRedraw),this.on("_change",function(t){s.itemSet&&s.itemSet.initialItemSetDrawn&&t&&1==t.queue?s._redraw():s._origRedraw()}),this.hammer=new l(this.dom.root);var r=this.hammer.get("pinch").set({enable:!0});u.disablePreventDefaultVertically(r),this.hammer.get("pan").set({threshold:5,direction:l.DIRECTION_HORIZONTAL}),this.listeners={},["tap","doubletap","press","pinch","pan","panstart","panmove","panend"].forEach(function(t){var e=function(e){s.isActive()&&s.emit(t,e)};s.hammer.on(t,e),s.listeners[t]=e}),u.onTouch(this.hammer,function(t){s.emit("touch",t)}.bind(this)),u.onRelease(this.hammer,function(t){s.emit("release",t)}.bind(this)),this.dom.centerContainer.addEventListener?(this.dom.centerContainer.addEventListener("mousewheel",e.bind(this),!1),this.dom.centerContainer.addEventListener("DOMMouseScroll",e.bind(this),!1)):this.dom.centerContainer.attachEvent("onmousewheel",e.bind(this)),this.dom.left.parentNode.addEventListener("scroll",i.bind(this)),this.dom.right.parentNode.addEventListener("scroll",i.bind(this));var a=!1;if(this.dom.center.addEventListener("dragover",o.bind(this),!1),this.dom.center.addEventListener("drop",n.bind(this),!1),this.customTimes=[],this.touch={},this.redrawCount=0,this.initialDrawDone=!1,this.initialRangeChangeDone=!1,!t)throw new Error("No container provided");t.appendChild(this.dom.root)},n.prototype.setOptions=function(t){if(t){var e=["width","height","minHeight","maxHeight","autoResize","start","end","clickToUse","dataAttributes","hiddenDates","locale","locales","moment","rtl","zoomKey","horizontalScroll","verticalScroll"];if(p.selectiveExtend(e,this.options,t),this.dom.rollingModeBtn.style.visibility="hidden",this.options.rtl&&(this.dom.container.style.direction="rtl",this.dom.backgroundVertical.className="vis-panel vis-background vis-vertical-rtl"),this.options.verticalScroll&&(this.options.rtl?this.dom.rightContainer.className="vis-panel vis-right vis-vertical-scroll":this.dom.leftContainer.className="vis-panel vis-left vis-vertical-scroll"),"object"!==(0,h.default)(this.options.orientation)&&(this.options.orientation={item:void 0,axis:void 0}),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation={item:t.orientation,axis:t.orientation}:"object"===(0,h.default)(t.orientation)&&("item"in t.orientation&&(this.options.orientation.item=t.orientation.item),"axis"in t.orientation&&(this.options.orientation.axis=t.orientation.axis))),"both"===this.options.orientation.axis){if(!this.timeAxis2){var i=this.timeAxis2=new c(this.body);i.setOptions=function(t){var e=t?p.extend({},t):{};e.orientation="top",c.prototype.setOptions.call(i,e)},this.components.push(i)}}else if(this.timeAxis2){var o=this.components.indexOf(this.timeAxis2);-1!==o&&this.components.splice(o,1),this.timeAxis2.destroy(),this.timeAxis2=null}if("function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),"hiddenDates"in this.options&&f.convertHiddenOptions(this.options.moment,this.body,this.options.hiddenDates),"clickToUse"in t&&(t.clickToUse?this.activator||(this.activator=new m(this.dom.root)):this.activator&&(this.activator.destroy(),delete this.activator)),"showCustomTime"in t)throw new Error("Option `showCustomTime` is deprecated. Create a custom time bar via timeline.addCustomTime(time [, id])");this._initAutoResize()}if(this.components.forEach(function(e){return e.setOptions(t)}),"configure"in t){this.configurator||(this.configurator=this._createConfigurator()),this.configurator.setOptions(t.configure);var n=p.deepExtend({},this.options);this.components.forEach(function(t){p.deepExtend(n,t.options)}),this.configurator.setModuleOptions({global:n})}this._redraw()},n.prototype.isActive=function(){return!this.activator||this.activator.active},n.prototype.destroy=function(){this.setItems(null),this.setGroups(null),this.off(),this._stopAutoResize(),this.dom.root.parentNode&&this.dom.root.parentNode.removeChild(this.dom.root),this.dom=null,this.activator&&(this.activator.destroy(),delete this.activator);for(var t in this.listeners)this.listeners.hasOwnProperty(t)&&delete this.listeners[t];this.listeners=null,this.hammer=null,this.components.forEach(function(t){return t.destroy()}),this.body=null},n.prototype.setCustomTime=function(t,e){var i=this.customTimes.filter(function(t){return e===t.options.id});if(0===i.length)throw new Error("No custom time bar found with id "+(0,r.default)(e));i.length>0&&i[0].setCustomTime(t)},n.prototype.getCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+(0,r.default)(t));return e[0].getCustomTime()},n.prototype.setCustomTimeTitle=function(t,e){var i=this.customTimes.filter(function(t){return t.options.id===e});if(0===i.length)throw new Error("No custom time bar found with id "+(0,r.default)(e));if(i.length>0)return i[0].setCustomTitle(t)},n.prototype.getEventProperties=function(t){return{event:t}},n.prototype.addCustomTime=function(t,e){var i=void 0!==t?p.convert(t,"Date").valueOf():new Date;if(this.customTimes.some(function(t){return t.options.id===e}))throw new Error("A custom time with id "+(0,r.default)(e)+" already exists");var o=new g(this.body,p.extend({},this.options,{time:i,id:e}));return this.customTimes.push(o),this.components.push(o),this._redraw(),e},n.prototype.removeCustomTime=function(t){var e=this.customTimes.filter(function(e){return e.options.id===t});if(0===e.length)throw new Error("No custom time bar found with id "+(0,r.default)(t));e.forEach(function(t){this.customTimes.splice(this.customTimes.indexOf(t),1),this.components.splice(this.components.indexOf(t),1),t.destroy()}.bind(this))},n.prototype.getVisibleItems=function(){return this.itemSet&&this.itemSet.getVisibleItems()||[]},n.prototype.fit=function(t,e){var i=this.getDataRange();if(null!==i.min||null!==i.max){var o=i.max-i.min,n=new Date(i.min.valueOf()-.01*o),s=new Date(i.max.valueOf()+.01*o),r=!t||void 0===t.animation||t.animation;this.range.setRange(n,s,{animation:r},e)}},n.prototype.getDataRange=function(){throw new Error("Cannot invoke abstract method getDataRange")},n.prototype.setWindow=function(t,e,i,o){"function"==typeof arguments[2]&&(o=arguments[2],i={});var n,s;1==arguments.length?(s=arguments[0],n=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:n})):2==arguments.length&&"function"==typeof arguments[1]?(s=arguments[0],o=arguments[1],n=void 0===s.animation||s.animation,this.range.setRange(s.start,s.end,{animation:n},o)):(n=!i||void 0===i.animation||i.animation,this.range.setRange(t,e,{animation:n},o))},n.prototype.moveTo=function(t,e,i){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.range.end-this.range.start,n=p.convert(t,"Date").valueOf(),s=n-o/2,r=n+o/2,a=!e||void 0===e.animation||e.animation;this.range.setRange(s,r,{animation:a},i)},n.prototype.getWindow=function(){var t=this.range.getRange();return{start:new Date(t.start),end:new Date(t.end)}},n.prototype.zoomIn=function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.getWindow(),n=o.start.valueOf(),s=o.end.valueOf(),r=s-n,a=r/(1+t),h=(r-a)/2,d=n+h,l=s-h;this.setWindow(d,l,e,i)}},n.prototype.zoomOut=function(t,e,i){if(!(!t||t<0||t>1)){"function"==typeof arguments[1]&&(i=arguments[1],e={});var o=this.getWindow(),n=o.start.valueOf(),s=o.end.valueOf(),r=s-n,a=n-r*t/2,h=s+r*t/2;this.setWindow(a,h,e,i)}},n.prototype.redraw=function(){this._redraw()},n.prototype._redraw=function(){this.redrawCount++;var t=!1,e=this.options,i=this.props,o=this.dom;if(o&&o.container&&0!=o.root.offsetWidth){f.updateHiddenDates(this.options.moment,this.body,this.options.hiddenDates),"top"==e.orientation?(p.addClassName(o.root,"vis-top"),p.removeClassName(o.root,"vis-bottom")):(p.removeClassName(o.root,"vis-top"),p.addClassName(o.root,"vis-bottom")),o.root.style.maxHeight=p.option.asSize(e.maxHeight,""),o.root.style.minHeight=p.option.asSize(e.minHeight,""),o.root.style.width=p.option.asSize(e.width,""),i.border.left=(o.centerContainer.offsetWidth-o.centerContainer.clientWidth)/2,i.border.right=i.border.left,i.border.top=(o.centerContainer.offsetHeight-o.centerContainer.clientHeight)/2,i.border.bottom=i.border.top,i.borderRootHeight=o.root.offsetHeight-o.root.clientHeight,i.borderRootWidth=o.root.offsetWidth-o.root.clientWidth,0===o.centerContainer.clientHeight&&(i.border.left=i.border.top,i.border.right=i.border.left),0===o.root.clientHeight&&(i.borderRootWidth=i.borderRootHeight),i.center.height=o.center.offsetHeight,i.left.height=o.left.offsetHeight,i.right.height=o.right.offsetHeight,i.top.height=o.top.clientHeight||-i.border.top,i.bottom.height=o.bottom.clientHeight||-i.border.bottom;var n=Math.max(i.left.height,i.center.height,i.right.height),s=i.top.height+n+i.bottom.height+i.borderRootHeight+i.border.top+i.border.bottom;o.root.style.height=p.option.asSize(e.height,s+"px"),i.root.height=o.root.offsetHeight,i.background.height=i.root.height-i.borderRootHeight;var r=i.root.height-i.top.height-i.bottom.height-i.borderRootHeight;i.centerContainer.height=r,i.leftContainer.height=r,i.rightContainer.height=i.leftContainer.height,i.root.width=o.root.offsetWidth,i.background.width=i.root.width-i.borderRootWidth,this.initialDrawDone||(i.scrollbarWidth=p.getScrollBarWidth()),e.verticalScroll?e.rtl?(i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth+i.scrollbarWidth||-i.border.right):(i.left.width=o.leftContainer.clientWidth+i.scrollbarWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right):(i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right),this._setDOM();var a=this._updateScrollTop();"top"!=e.orientation.item&&(a+=Math.max(i.centerContainer.height-i.center.height-i.border.top-i.border.bottom,0)),o.center.style.top=a+"px";var h=0==i.scrollTop?"hidden":"",d=i.scrollTop==i.scrollTopMin?"hidden":"";o.shadowTop.style.visibility=h,o.shadowBottom.style.visibility=d,o.shadowTopLeft.style.visibility=h,o.shadowBottomLeft.style.visibility=d,o.shadowTopRight.style.visibility=h,o.shadowBottomRight.style.visibility=d,e.verticalScroll&&(o.rightContainer.className="vis-panel vis-right vis-vertical-scroll",o.leftContainer.className="vis-panel vis-left vis-vertical-scroll",o.shadowTopRight.style.visibility="hidden",o.shadowBottomRight.style.visibility="hidden",o.shadowTopLeft.style.visibility="hidden",o.shadowBottomLeft.style.visibility="hidden",o.left.style.top="0px",o.right.style.top="0px"),(!e.verticalScroll||i.center.height<i.centerContainer.height)&&(o.left.style.top=a+"px",o.right.style.top=a+"px",o.rightContainer.className=o.rightContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),o.leftContainer.className=o.leftContainer.className.replace(new RegExp("(?:^|\\s)vis-vertical-scroll(?:\\s|$)")," "),i.left.width=o.leftContainer.clientWidth||-i.border.left,i.right.width=o.rightContainer.clientWidth||-i.border.right,this._setDOM());var u=i.center.height>i.centerContainer.height;this.hammer.get("pan").set({direction:u?l.DIRECTION_ALL:l.DIRECTION_HORIZONTAL}),this.components.forEach(function(e){t=e.redraw()||t});if(t){if(this.redrawCount<5)return void this.body.emitter.emit("_change");console.log("WARNING: infinite loop in redraw?")}else this.redrawCount=0;this.body.emitter.emit("changed")}},n.prototype._setDOM=function(){var t=this.props,e=this.dom;t.leftContainer.width=t.left.width,t.rightContainer.width=t.right.width;var i=t.root.width-t.left.width-t.right.width-t.borderRootWidth;t.center.width=i,t.centerContainer.width=i,t.top.width=i,t.bottom.width=i,e.background.style.height=t.background.height+"px",e.backgroundVertical.style.height=t.background.height+"px",e.backgroundHorizontal.style.height=t.centerContainer.height+"px",e.centerContainer.style.height=t.centerContainer.height+"px",e.leftContainer.style.height=t.leftContainer.height+"px",e.rightContainer.style.height=t.rightContainer.height+"px",e.background.style.width=t.background.width+"px",e.backgroundVertical.style.width=t.centerContainer.width+"px",e.backgroundHorizontal.style.width=t.background.width+"px",e.centerContainer.style.width=t.center.width+"px",e.top.style.width=t.top.width+"px",e.bottom.style.width=t.bottom.width+"px",e.background.style.left="0",e.background.style.top="0",e.backgroundVertical.style.left=t.left.width+t.border.left+"px",e.backgroundVertical.style.top="0",e.backgroundHorizontal.style.left="0",e.backgroundHorizontal.style.top=t.top.height+"px",e.centerContainer.style.left=t.left.width+"px",e.centerContainer.style.top=t.top.height+"px",e.leftContainer.style.left="0",e.leftContainer.style.top=t.top.height+"px",e.rightContainer.style.left=t.left.width+t.center.width+"px",e.rightContainer.style.top=t.top.height+"px",e.top.style.left=t.left.width+"px",e.top.style.top="0",e.bottom.style.left=t.left.width+"px",e.bottom.style.top=t.top.height+t.centerContainer.height+"px",e.center.style.left="0",e.left.style.left="0",e.right.style.left="0"},n.prototype.repaint=function(){throw new Error("Function repaint is deprecated. Use redraw instead.")},n.prototype.setCurrentTime=function(t){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");this.currentTime.setCurrentTime(t)},n.prototype.getCurrentTime=function(){if(!this.currentTime)throw new Error("Option showCurrentTime must be true");return this.currentTime.getCurrentTime()},n.prototype._toTime=function(t){return f.toTime(this,t,this.props.center.width)},n.prototype._toGlobalTime=function(t){return f.toTime(this,t,this.props.root.width)},n.prototype._toScreen=function(t){return f.toScreen(this,t,this.props.center.width)},n.prototype._toGlobalScreen=function(t){return f.toScreen(this,t,this.props.root.width)},n.prototype._initAutoResize=function(){1==this.options.autoResize?this._startAutoResize():this._stopAutoResize()},n.prototype._startAutoResize=function(){var t=this;this._stopAutoResize(),this._onResize=function(){if(1!=t.options.autoResize)return void t._stopAutoResize();t.dom.root&&(t.dom.root.offsetWidth==t.props.lastWidth&&t.dom.root.offsetHeight==t.props.lastHeight||(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight,t.props.scrollbarWidth=p.getScrollBarWidth(),t.body.emitter.emit("_change")))},p.addEventListener(window,"resize",this._onResize),t.dom.root&&(t.props.lastWidth=t.dom.root.offsetWidth,t.props.lastHeight=t.dom.root.offsetHeight),this.watchTimer=setInterval(this._onResize,1e3)},n.prototype._stopAutoResize=function(){this.watchTimer&&(clearInterval(this.watchTimer),this.watchTimer=void 0),this._onResize&&(p.removeEventListener(window,"resize",this._onResize),this._onResize=null)},n.prototype._onTouch=function(t){this.touch.allowDragging=!0,this.touch.initialScrollTop=this.props.scrollTop},n.prototype._onPinch=function(t){this.touch.allowDragging=!1},n.prototype._onDrag=function(t){if(t&&this.touch.allowDragging){var e=t.deltaY,i=this._getScrollTop(),o=this._setScrollTop(this.touch.initialScrollTop+e);this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),o!=i&&this.emit("verticalDrag")}},n.prototype._setScrollTop=function(t){return this.props.scrollTop=t,this._updateScrollTop(),this.props.scrollTop},n.prototype._updateScrollTop=function(){var t=Math.min(this.props.centerContainer.height-this.props.center.height,0);return t!=this.props.scrollTopMin&&("top"!=this.options.orientation.item&&(this.props.scrollTop+=t-this.props.scrollTopMin),this.props.scrollTopMin=t),this.props.scrollTop>0&&(this.props.scrollTop=0),this.props.scrollTop<t&&(this.props.scrollTop=t),this.options.verticalScroll&&(this.dom.left.parentNode.scrollTop=-this.props.scrollTop,this.dom.right.parentNode.scrollTop=-this.props.scrollTop),this.props.scrollTop},n.prototype._getScrollTop=function(){return this.props.scrollTop},n.prototype._createConfigurator=function(){throw new Error("Cannot invoke abstract method _createConfigurator")},t.exports=n},function(t,e,i){function o(t,e,i,s,r){this.moment=n,this.current=this.moment(),this._start=this.moment(),this._end=this.moment(),this.autoScale=!0,this.scale="day",this.step=1,this.setRange(t,e,i),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,Array.isArray(s)?this.hiddenDates=s:this.hiddenDates=void 0!=s?[s]:[],this.format=o.FORMAT,this.options=r||{}}var n=i(3),s=i(21),r=i(0);o.FORMAT={minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},o.prototype.setMoment=function(t){this.moment=t,this.current=this.moment(this.current.valueOf()),this._start=this.moment(this._start.valueOf()),this._end=this.moment(this._end.valueOf())},o.prototype.setFormat=function(t){var e=r.deepExtend({},o.FORMAT);this.format=r.deepExtend(e,t)},o.prototype.setRange=function(t,e,i){if(!(t instanceof Date&&e instanceof Date))throw"No legal start or end date in method setRange";this._start=void 0!=t?this.moment(t.valueOf()):new Date,this._end=void 0!=e?this.moment(e.valueOf()):new Date,this.autoScale&&this.setMinimumStep(i)},o.prototype.start=function(){this.current=this._start.clone(),this.roundToMinor()},o.prototype.roundToMinor=function(){switch("week"==this.scale&&this.current.weekday(0),this.scale){case"year":this.current.year(this.step*Math.floor(this.current.year()/this.step)),this.current.month(0);case"month":this.current.date(1);case"week":case"day":case"weekday":this.current.hours(0);case"hour":this.current.minutes(0);case"minute":this.current.seconds(0);case"second":this.current.milliseconds(0)}if(1!=this.step)switch(this.scale){case"millisecond":this.current.subtract(this.current.milliseconds()%this.step,"milliseconds");break;case"second":this.current.subtract(this.current.seconds()%this.step,"seconds");break;case"minute":this.current.subtract(this.current.minutes()%this.step,"minutes");break;case"hour":this.current.subtract(this.current.hours()%this.step,"hours");break;case"weekday":case"day":this.current.subtract((this.current.date()-1)%this.step,"day");break;case"week":this.current.subtract(this.current.week()%this.step,"week");break;case"month":this.current.subtract(this.current.month()%this.step,"month");break;case"year":this.current.subtract(this.current.year()%this.step,"year")}},o.prototype.hasNext=function(){return this.current.valueOf()<=this._end.valueOf()},o.prototype.next=function(){var t=this.current.valueOf();switch(this.scale){case"millisecond":this.current.add(this.step,"millisecond");break;case"second":this.current.add(this.step,"second");break;case"minute":this.current.add(this.step,"minute");break;case"hour":this.current.add(this.step,"hour"),this.current.month()<6?this.current.subtract(this.current.hours()%this.step,"hour"):this.current.hours()%this.step!=0&&this.current.add(this.step-this.current.hours()%this.step,"hour");break;case"weekday":case"day":this.current.add(this.step,"day");break;case"week":if(0!==this.current.weekday())this.current.weekday(0),this.current.add(this.step,"week");else if(!1===this.options.showMajorLabels)this.current.add(this.step,"week");else{var e=this.current.clone();e.add(1,"week"),e.isSame(this.current,"month")?this.current.add(this.step,"week"):(this.current.add(this.step,"week"),this.current.date(1))}break;case"month":this.current.add(this.step,"month");break;case"year":this.current.add(this.step,"year")}if(1!=this.step)switch(this.scale){case"millisecond":this.current.milliseconds()>0&&this.current.milliseconds()<this.step&&this.current.milliseconds(0);break;case"second":this.current.seconds()>0&&this.current.seconds()<this.step&&this.current.seconds(0);break;case"minute":this.current.minutes()>0&&this.current.minutes()<this.step&&this.current.minutes(0);break;case"hour":this.current.hours()>0&&this.current.hours()<this.step&&this.current.hours(0);break;case"weekday":case"day":this.current.date()<this.step+1&&this.current.date(1);break;case"week":this.current.week()<this.step&&this.current.week(1);break;case"month":this.current.month()<this.step&&this.current.month(0)}this.current.valueOf()==t&&(this.current=this._end.clone()),this.switchedDay=!1,this.switchedMonth=!1,this.switchedYear=!1,s.stepOverHiddenDates(this.moment,this,t)},o.prototype.getCurrent=function(){return this.current},o.prototype.setScale=function(t){t&&"string"==typeof t.scale&&(this.scale=t.scale,this.step=t.step>0?t.step:1,this.autoScale=!1)},o.prototype.setAutoScale=function(t){this.autoScale=t},o.prototype.setMinimumStep=function(t){if(void 0!=t){31104e9>t&&(this.scale="year",this.step=1e3),15552e9>t&&(this.scale="year",this.step=500),31104e8>t&&(this.scale="year",this.step=100),15552e8>t&&(this.scale="year",this.step=50),31104e7>t&&(this.scale="year",this.step=10),15552e7>t&&(this.scale="year",this.step=5),31104e6>t&&(this.scale="year",this.step=1),7776e6>t&&(this.scale="month",this.step=3),2592e6>t&&(this.scale="month",this.step=1),432e6>t&&(this.scale="day",this.step=5),1728e5>t&&(this.scale="day",this.step=2),864e5>t&&(this.scale="day",this.step=1),432e5>t&&(this.scale="weekday",this.step=1),144e5>t&&(this.scale="hour",this.step=4),36e5>t&&(this.scale="hour",this.step=1),9e5>t&&(this.scale="minute",this.step=15),6e5>t&&(this.scale="minute",this.step=10),3e5>t&&(this.scale="minute",this.step=5),6e4>t&&(this.scale="minute",this.step=1),15e3>t&&(this.scale="second",this.step=15),1e4>t&&(this.scale="second",this.step=10),5e3>t&&(this.scale="second",this.step=5),1e3>t&&(this.scale="second",this.step=1),200>t&&(this.scale="millisecond",this.step=200),100>t&&(this.scale="millisecond",this.step=100),50>t&&(this.scale="millisecond",this.step=50),10>t&&(this.scale="millisecond",this.step=10),5>t&&(this.scale="millisecond",this.step=5),1>t&&(this.scale="millisecond",this.step=1)}},o.snap=function(t,e,i){var o=n(t);if("year"==e){var s=o.year()+Math.round(o.month()/12);o.year(Math.round(s/i)*i),o.month(0),o.date(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("month"==e)o.date()>15?(o.date(1),o.add(1,"month")):o.date(1),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("week"==e)o.weekday()>2?(o.weekday(0),o.add(1,"week")):o.weekday(0),o.hours(0),o.minutes(0),o.seconds(0),o.milliseconds(0);else if("day"==e){switch(i){case 5:case 2:o.hours(24*Math.round(o.hours()/24));break;default:o.hours(12*Math.round(o.hours()/12))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("weekday"==e){switch(i){case 5:case 2:o.hours(12*Math.round(o.hours()/12));break;default:o.hours(6*Math.round(o.hours()/6))}o.minutes(0),o.seconds(0),o.milliseconds(0)}else if("hour"==e){switch(i){case 4:o.minutes(60*Math.round(o.minutes()/60));break;default:o.minutes(30*Math.round(o.minutes()/30))}o.seconds(0),o.milliseconds(0)}else if("minute"==e){switch(i){case 15:case 10:o.minutes(5*Math.round(o.minutes()/5)),o.seconds(0);break;case 5:o.seconds(60*Math.round(o.seconds()/60));break;default:o.seconds(30*Math.round(o.seconds()/30))}o.milliseconds(0)}else if("second"==e)switch(i){case 15:case 10:o.seconds(5*Math.round(o.seconds()/5)),o.milliseconds(0);break;case 5:o.milliseconds(1e3*Math.round(o.milliseconds()/1e3));break;default:o.milliseconds(500*Math.round(o.milliseconds()/500))}else if("millisecond"==e){var r=i>5?i/2:1;o.milliseconds(Math.round(o.milliseconds()/r)*r)}return o},o.prototype.isMajor=function(){if(1==this.switchedYear)switch(this.scale){case"year":case"month":case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedMonth)switch(this.scale){case"week":case"weekday":case"day":case"hour":case"minute":case"second":case"millisecond":return!0;default:return!1}else if(1==this.switchedDay)switch(this.scale){case"millisecond":case"second":case"minute":case"hour":return!0;default:return!1}var t=this.moment(this.current);switch(this.scale){case"millisecond":return 0==t.milliseconds();case"second":return 0==t.seconds();case"minute":return 0==t.hours()&&0==t.minutes();case"hour":return 0==t.hours();case"weekday":case"day":case"week":return 1==t.date();case"month":return 0==t.month();case"year":default:return!1}},o.prototype.getLabelMinor=function(t){if(void 0==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.minorLabels)return this.format.minorLabels(t,this.scale,this.step);var e=this.format.minorLabels[this.scale];switch(this.scale){case"week":if(this.isMajor()&&0!==t.weekday())return"";default:return e&&e.length>0?this.moment(t).format(e):""}},o.prototype.getLabelMajor=function(t){if(void 0==t&&(t=this.current),t instanceof Date&&(t=this.moment(t)),"function"==typeof this.format.majorLabels)return this.format.majorLabels(t,this.scale,this.step);var e=this.format.majorLabels[this.scale];return e&&e.length>0?this.moment(t).format(e):""},o.prototype.getClassName=function(){function t(t){return t/a%2==0?" vis-even":" vis-odd"}function e(t){return t.isSame(new Date,"day")?" vis-today":t.isSame(n().add(1,"day"),"day")?" vis-tomorrow":t.isSame(n().add(-1,"day"),"day")?" vis-yesterday":""}function i(t){return t.isSame(new Date,"week")?" vis-current-week":""}function o(t){return t.isSame(new Date,"month")?" vis-current-month":""}var n=this.moment,s=this.moment(this.current),r=s.locale?s.locale("en"):s.lang("en"),a=this.step,h=[];switch(this.scale){case"millisecond":h.push(e(r)),h.push(t(r.milliseconds()));break;case"second":h.push(e(r)),h.push(t(r.seconds()));break;case"minute":h.push(e(r)),h.push(t(r.minutes()));break;case"hour":h.push("vis-h"+r.hours()+(4==this.step?"-h"+(r.hours()+4):"")),h.push(e(r)),h.push(t(r.hours()));break;case"weekday":h.push("vis-"+r.format("dddd").toLowerCase()),h.push(e(r)),h.push(i(r)),h.push(t(r.date()));break;case"day":h.push("vis-day"+r.date()),h.push("vis-"+r.format("MMMM").toLowerCase()),h.push(e(r)),h.push(o(r)),h.push(this.step<=2?e(r):""),h.push(this.step<=2?"vis-"+r.format("dddd").toLowerCase():""),h.push(t(r.date()-1));break;case"week":h.push("vis-week"+r.format("w")),h.push(i(r)),h.push(t(r.week()));break;case"month":h.push("vis-"+r.format("MMMM").toLowerCase()),h.push(o(r)),h.push(t(r.month()));break;case"year":h.push("vis-year"+r.year()),h.push(function(t){return t.isSame(new Date,"year")?" vis-current-year":""}(r)),h.push(t(r.year()))}return h.filter(String).join(" ")},t.exports=o},function(t,e,i){function o(t,e){this.body=t,this.defaultOptions={rtl:!1,showCurrentTime:!0,moment:r,locales:a,locale:"en"},this.options=n.extend({},this.defaultOptions),this.offset=0,this._create(),this.setOptions(e)}var n=i(0),s=i(5),r=i(3),a=i(69);o.prototype=new s,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-current-time",t.style.position="absolute",t.style.top="0px",t.style.height="100%",this.bar=t},o.prototype.destroy=function(){this.options.showCurrentTime=!1,this.redraw(),this.body=null},o.prototype.setOptions=function(t){t&&n.selectiveExtend(["rtl","showCurrentTime","moment","locale","locales"],this.options,t)},o.prototype.redraw=function(){if(this.options.showCurrentTime){var t=this.body.dom.backgroundVertical;this.bar.parentNode!=t&&(this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),t.appendChild(this.bar),this.start());var e=this.options.moment((new Date).valueOf()+this.offset),i=this.body.util.toScreen(e),o=this.options.locales[this.options.locale];o||(this.warned||(console.log("WARNING: options.locales['"+this.options.locale+"'] not found. See http://visjs.org/docs/timeline/#Localization"),this.warned=!0),o=this.options.locales.en);var n=o.current+" "+o.time+": "+e.format("dddd, MMMM Do YYYY, H:mm:ss");n=n.charAt(0).toUpperCase()+n.substring(1),this.options.rtl?this.bar.style.right=i+"px":this.bar.style.left=i+"px",this.bar.title=n}else this.bar.parentNode&&this.bar.parentNode.removeChild(this.bar),this.stop();return!1},o.prototype.start=function(){function t(){e.stop();var i=e.body.range.conversion(e.body.domProps.center.width).scale,o=1/i/10;o<30&&(o=30),o>1e3&&(o=1e3),e.redraw(),e.body.emitter.emit("currentTimeTick"),e.currentTimeTimer=setTimeout(t,o)}var e=this;t()},o.prototype.stop=function(){void 0!==this.currentTimeTimer&&(clearTimeout(this.currentTimeTimer),delete this.currentTimeTimer)},o.prototype.setCurrentTime=function(t){var e=n.convert(t,"Date").valueOf(),i=(new Date).valueOf();this.offset=e-i,this.redraw()},o.prototype.getCurrentTime=function(){return new Date((new Date).valueOf()+this.offset)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.groupId=t,this.subgroups={},this.subgroupStack={},this.subgroupStackAll=!1,this.doInnerStack=!1,this.subgroupIndex=0,this.subgroupOrderer=e&&e.subgroupOrder,this.itemSet=i,this.isVisible=null,this.stackDirty=!0,e&&e.nestedGroups&&(this.nestedGroups=e.nestedGroups,0==e.showNested?this.showNested=!1:this.showNested=!0),e&&e.subgroupStack)if("boolean"==typeof e.subgroupStack)this.doInnerStack=e.subgroupStack,this.subgroupStackAll=e.subgroupStack;else for(var o in e.subgroupStack)this.subgroupStack[o]=e.subgroupStack[o],this.doInnerStack=this.doInnerStack||e.subgroupStack[o];this.nestedInGroup=null,this.dom={},this.props={label:{width:0,height:0}},this.className=null,this.items={},this.visibleItems=[],this.itemsInRange=[],this.orderedItems={byStart:[],byEnd:[]},this.checkRangedItems=!1;var n=this;this.itemSet.body.emitter.on("checkRangedItems",function(){n.checkRangedItems=!0}),this._create(),this.setData(e)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(71);o.prototype._create=function(){var t=document.createElement("div");this.itemSet.options.groupEditable.order?t.className="vis-label draggable":t.className="vis-label",this.dom.label=t;var e=document.createElement("div");e.className="vis-inner",t.appendChild(e),this.dom.inner=e;var i=document.createElement("div");i.className="vis-group",i["timeline-group"]=this,this.dom.foreground=i,this.dom.background=document.createElement("div"),this.dom.background.className="vis-group",this.dom.axis=document.createElement("div"),this.dom.axis.className="vis-group",this.dom.marker=document.createElement("div"),this.dom.marker.style.visibility="hidden",this.dom.marker.style.position="absolute",this.dom.marker.innerHTML="",this.dom.background.appendChild(this.dom.marker)},o.prototype.setData=function(t){var e,i;if(this.itemSet.options&&this.itemSet.options.groupTemplate?(i=this.itemSet.options.groupTemplate.bind(this),e=i(t,this.dom.inner)):e=t&&t.content,e instanceof Element){for(this.dom.inner.appendChild(e);this.dom.inner.firstChild;)this.dom.inner.removeChild(this.dom.inner.firstChild);this.dom.inner.appendChild(e)}else e instanceof Object?i(t,this.dom.inner):this.dom.inner.innerHTML=void 0!==e&&null!==e?e:this.groupId||"";if(this.dom.label.title=t&&t.title||"",this.dom.inner.firstChild?r.removeClassName(this.dom.inner,"vis-hidden"):r.addClassName(this.dom.inner,"vis-hidden"),t&&t.nestedGroups){this.nestedGroups&&this.nestedGroups==t.nestedGroups||(this.nestedGroups=t.nestedGroups),void 0===t.showNested&&void 0!==this.showNested||(0==t.showNested?this.showNested=!1:this.showNested=!0),r.addClassName(this.dom.label,"vis-nesting-group");var o=this.itemSet.options.rtl?"collapsed-rtl":"collapsed";this.showNested?(r.removeClassName(this.dom.label,o),r.addClassName(this.dom.label,"expanded")):(r.removeClassName(this.dom.label,"expanded"),r.addClassName(this.dom.label,o))}else this.nestedGroups&&(this.nestedGroups=null,o=this.itemSet.options.rtl?"collapsed-rtl":"collapsed",r.removeClassName(this.dom.label,o),r.removeClassName(this.dom.label,"expanded"),r.removeClassName(this.dom.label,"vis-nesting-group"));t&&t.nestedInGroup&&(r.addClassName(this.dom.label,"vis-nested-group"),this.itemSet.options&&this.itemSet.options.rtl?this.dom.inner.style.paddingRight="30px":this.dom.inner.style.paddingLeft="30px");var n=t&&t.className||null;n!=this.className&&(this.className&&(r.removeClassName(this.dom.label,this.className),r.removeClassName(this.dom.foreground,this.className),r.removeClassName(this.dom.background,this.className),r.removeClassName(this.dom.axis,this.className)),r.addClassName(this.dom.label,n),r.addClassName(this.dom.foreground,n),r.addClassName(this.dom.background,n),r.addClassName(this.dom.axis,n),this.className=n),this.style&&(r.removeCssText(this.dom.label,this.style),this.style=null),t&&t.style&&(r.addCssText(this.dom.label,t.style),this.style=t.style)},o.prototype.getLabelWidth=function(){return this.props.label.width},o.prototype._didMarkerHeightChange=function(){var t=this.dom.marker.clientHeight;if(t!=this.lastMarkerHeight){this.lastMarkerHeight=t;var e={},i=0;r.forEach(this.items,function(t,o){if(t.dirty=!0,t.displayed){e[o]=t.redraw(!0),i=e[o].length}});if(i>0)for(var o=0;o<i;o++)r.forEach(e,function(t){t[o]()});return!0}},o.prototype._calculateGroupSizeAndPosition=function(){var t=this.dom.foreground.offsetTop,e=this.dom.foreground.offsetLeft,i=this.dom.foreground.offsetWidth;this.top=t,this.right=e,this.width=i},o.prototype._redrawItems=function(t,e,i,o){if(t||this.stackDirty||this.isVisible&&!e){var n={},s=null;if("function"==typeof this.itemSet.options.order){var h=this,d={},l=0;if(r.forEach(this.items,function(t,e){t.displayed||(d[e]=t.redraw(!0),l=d[e].length,h.visibleItems.push(t))}),l>0)for(var u=0;u<l;u++)r.forEach(d,function(t){t[u]()});if(r.forEach(this.items,function(t){t.repositionX(!1)}),this.doInnerStack&&this.itemSet.options.stackSubgroups){for(s in this.subgroups)n[s]=this.subgroups[s].items.slice().sort(function(t,e){return h.itemSet.options.order(t.data,e.data)});a.stackSubgroupsWithInnerStack(n,i,this.subgroups)}else{var p=this.orderedItems.byStart.slice().sort(function(t,e){return h.itemSet.options.order(t.data,e.data)});a.stack(p,i,!0)}this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,o)}else if(this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,o),this.itemSet.options.stack)if(this.doInnerStack&&this.itemSet.options.stackSubgroups){for(s in this.subgroups)n[s]=this.subgroups[s].items;a.stackSubgroupsWithInnerStack(n,i,this.subgroups)}else a.stack(this.visibleItems,i,!0);else a.nostack(this.visibleItems,i,this.subgroups,this.itemSet.options.stackSubgroups);this.stackDirty=!1}},o.prototype._didResize=function(t,e){t=r.updateProperty(this,"height",e)||t;var i=this.dom.inner.clientWidth,o=this.dom.inner.clientHeight;return t=r.updateProperty(this.props.label,"width",i)||t,t=r.updateProperty(this.props.label,"height",o)||t},o.prototype._applyGroupHeight=function(t){this.dom.background.style.height=t+"px",this.dom.foreground.style.height=t+"px",this.dom.label.style.height=t+"px"},o.prototype._updateItemsVerticalPosition=function(t){for(var e=0,i=this.visibleItems.length;e<i;e++){var o=this.visibleItems[e];o.repositionY(t),this.isVisible||"__background__"==this.groupId||o.displayed&&o.hide()}},o.prototype.redraw=function(t,e,i,o){var n,s=!1,r=this.isVisible,a=[function(){i=this._didMarkerHeightChange.bind(this)}.bind(this),this._updateSubGroupHeights.bind(this,e),this._calculateGroupSizeAndPosition.bind(this),function(){this.isVisible=this._isGroupVisible.bind(this)(t,e)}.bind(this),function(){this._redrawItems.bind(this)(i,r,e,t)}.bind(this),this._updateSubgroupsSizes.bind(this),function(){n=this._calculateHeight.bind(this)(e)}.bind(this),this._calculateGroupSizeAndPosition.bind(this),function(){s=this._didResize.bind(this)(s,n)}.bind(this),function(){this._applyGroupHeight.bind(this)(n)}.bind(this),function(){this._updateItemsVerticalPosition.bind(this)(e)}.bind(this),function(){return!this.isVisible&&this.height&&(s=!1),s}];if(o)return a;var h;return a.forEach(function(t){h=t()}),h},o.prototype._updateSubGroupHeights=function(t){if((0,s.default)(this.subgroups).length>0){var e=this;this.resetSubgroups(),r.forEach(this.visibleItems,function(i){void 0!==i.data.subgroup&&(e.subgroups[i.data.subgroup].height=Math.max(e.subgroups[i.data.subgroup].height,i.height+t.item.vertical),e.subgroups[i.data.subgroup].visible=!0)})}},o.prototype._isGroupVisible=function(t,e){return this.top<=t.body.domProps.centerContainer.height-t.body.domProps.scrollTop+e.axis&&this.top+this.height+e.axis>=-t.body.domProps.scrollTop},o.prototype._calculateHeight=function(t){var e,i=this.visibleItems;if(i.length>0){var o=i[0].top,n=i[0].top+i[0].height;if(r.forEach(i,function(t){o=Math.min(o,t.top),n=Math.max(n,t.top+t.height)}),o>t.axis){var s=o-t.axis;n-=s,r.forEach(i,function(t){t.top-=s})}e=n+t.item.vertical/2}else e=0;return e=Math.max(e,this.props.label.height)},o.prototype.show=function(){this.dom.label.parentNode||this.itemSet.dom.labelSet.appendChild(this.dom.label),this.dom.foreground.parentNode||this.itemSet.dom.foreground.appendChild(this.dom.foreground),this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background),this.dom.axis.parentNode||this.itemSet.dom.axis.appendChild(this.dom.axis)},o.prototype.hide=function(){var t=this.dom.label;t.parentNode&&t.parentNode.removeChild(t);var e=this.dom.foreground;e.parentNode&&e.parentNode.removeChild(e);var i=this.dom.background;i.parentNode&&i.parentNode.removeChild(i);var o=this.dom.axis;o.parentNode&&o.parentNode.removeChild(o)},o.prototype.add=function(t){if(this.items[t.id]=t,t.setParent(this),this.stackDirty=!0,void 0!==t.data.subgroup&&(this._addToSubgroup(t),this.orderSubgroups()),-1==this.visibleItems.indexOf(t)){var e=this.itemSet.body.range;this._checkIfVisible(t,this.visibleItems,e)}},o.prototype._addToSubgroup=function(t,e){e=e||t.data.subgroup,void 0!=e&&void 0===this.subgroups[e]&&(this.subgroups[e]={height:0,top:0,start:t.data.start,end:t.data.end||t.data.start,visible:!1,index:this.subgroupIndex,items:[],stack:this.subgroupStackAll||this.subgroupStack[e]||!1},this.subgroupIndex++),new Date(t.data.start)<new Date(this.subgroups[e].start)&&(this.subgroups[e].start=t.data.start);var i=t.data.end||t.data.start;new Date(i)>new Date(this.subgroups[e].end)&&(this.subgroups[e].end=i),this.subgroups[e].items.push(t)},o.prototype._updateSubgroupsSizes=function(){var t=this;if(t.subgroups)for(var e in t.subgroups){var i=t.subgroups[e].items[0].data.end||t.subgroups[e].items[0].data.start,o=t.subgroups[e].items[0].data.start,n=i-1;t.subgroups[e].items.forEach(function(t){new Date(t.data.start)<new Date(o)&&(o=t.data.start);var e=t.data.end||t.data.start;new Date(e)>new Date(n)&&(n=e)}),t.subgroups[e].start=o,t.subgroups[e].end=new Date(n-1)}},o.prototype.orderSubgroups=function(){if(void 0!==this.subgroupOrderer){var t,e=[];if("string"==typeof this.subgroupOrderer){for(t in this.subgroups)e.push({subgroup:t,sortField:this.subgroups[t].items[0].data[this.subgroupOrderer]});e.sort(function(t,e){return t.sortField-e.sortField})}else if("function"==typeof this.subgroupOrderer){for(t in this.subgroups)e.push(this.subgroups[t].items[0].data);e.sort(this.subgroupOrderer)}if(e.length>0)for(var i=0;i<e.length;i++)this.subgroups[e[i].subgroup].index=i}},o.prototype.resetSubgroups=function(){for(var t in this.subgroups)this.subgroups.hasOwnProperty(t)&&(this.subgroups[t].visible=!1,this.subgroups[t].height=0)},o.prototype.remove=function(t){delete this.items[t.id],t.setParent(null),this.stackDirty=!0;var e=this.visibleItems.indexOf(t);-1!=e&&this.visibleItems.splice(e,1),void 0!==t.data.subgroup&&(this._removeFromSubgroup(t),this.orderSubgroups())},o.prototype._removeFromSubgroup=function(t,e){if(void 0!=(e=e||t.data.subgroup)){var i=this.subgroups[e];if(i){var o=i.items.indexOf(t);o>=0&&(i.items.splice(o,1),i.items.length?this._updateSubgroupsSizes():delete this.subgroups[e])}}},o.prototype.removeFromDataSet=function(t){this.itemSet.removeItem(t.id)},o.prototype.order=function(){for(var t=r.toArray(this.items),e=[],i=[],o=0;o<t.length;o++)void 0!==t[o].data.end&&i.push(t[o]),e.push(t[o]);this.orderedItems={byStart:e,byEnd:i},a.orderByStart(this.orderedItems.byStart),a.orderByEnd(this.orderedItems.byEnd)},o.prototype._updateItemsInRange=function(t,e,i){var o=[],n={},s=(i.end-i.start)/4,a=i.start-s,h=i.end+s,d=function(t){return t<a?-1:t<=h?0:1};if(e.length>0)for(var l=0;l<e.length;l++)this._checkIfVisibleWithReference(e[l],o,n,i);var u=r.binarySearchCustom(t.byStart,d,"data","start");if(this._traceVisible(u,t.byStart,o,n,function(t){return t.data.start<a||t.data.start>h}),1==this.checkRangedItems)for(this.checkRangedItems=!1,l=0;l<t.byEnd.length;l++)this._checkIfVisibleWithReference(t.byEnd[l],o,n,i);else{var p=r.binarySearchCustom(t.byEnd,d,"data","end");this._traceVisible(p,t.byEnd,o,n,function(t){return t.data.end<a||t.data.end>h})}var c={},m=0;for(l=0;l<o.length;l++){var f=o[l];if(!f.displayed){c[l]=f.redraw(!0),m=c[l].length}}if(m>0)for(var g=0;g<m;g++)r.forEach(c,function(t){t[g]()});for(l=0;l<o.length;l++)o[l].repositionX();return o},o.prototype._traceVisible=function(t,e,i,o,n){if(-1!=t){var s,r;for(s=t;s>=0&&(r=e[s],!n(r));s--)void 0===o[r.id]&&(o[r.id]=!0,i.push(r));for(s=t+1;s<e.length&&(r=e[s],!n(r));s++)void 0===o[r.id]&&(o[r.id]=!0,i.push(r))}},o.prototype._checkIfVisible=function(t,e,i){t.isVisible(i)?(t.displayed||t.show(),t.repositionX(),e.push(t)):t.displayed&&t.hide()},o.prototype._checkIfVisibleWithReference=function(t,e,i,o){t.isVisible(o)?void 0===i[t.id]&&(i[t.id]=!0,e.push(t)):t.displayed&&t.hide()},o.prototype.changeSubgroup=function(t,e,i){this._removeFromSubgroup(t,e),this._addToSubgroup(t,i),this.orderSubgroups()},t.exports=o},function(t,e,i){function o(t,e,i){r.call(this,t,e,i),this.width=0,this.height=0,this.top=0,this.left=0}var n=i(43),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(51);o.prototype=(0,s.default)(r.prototype),o.prototype.redraw=function(t,e,i){this.visibleItems=this._updateItemsInRange(this.orderedItems,this.visibleItems,t),this.width=this.dom.background.offsetWidth,this.dom.background.style.height="0";for(var o=0,n=this.visibleItems.length;o<n;o++){this.visibleItems[o].repositionY(e)}return!1},o.prototype.show=function(){this.dom.background.parentNode||this.itemSet.dom.background.appendChild(this.dom.background)},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,this.options=i,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-range",o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.visibleFrame=document.createElement("div"),this.dom.visibleFrame.className="vis-item-visible-frame",this.dom.box.appendChild(this.dom.visibleFrame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dom.box["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className=this.baseClassName+e,this.dom.content.style.maxWidth="none"}},o.prototype._getDomComponentsSizes=function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.frame).overflow,{content:{width:this.dom.content.offsetWidth},box:{height:this.dom.box.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.props.content.width=t.content.width,this.height=t.box.height,this.dom.content.style.maxWidth="",this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDeleteButton(this.dom.box),this._repaintDragCenter(),this._repaintDragLeft(),this._repaintDragRight()},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes.bind(this)())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom.box;t.parentNode&&t.parentNode.removeChild(t),this.displayed=!1}},o.prototype.repositionX=function(t){var e,i,o=this.parent.width,n=this.conversion.toScreen(this.data.start),s=this.conversion.toScreen(this.data.end),r=void 0===this.data.align?this.options.align:this.data.align;!1===this.data.limitSize||void 0!==t&&!0!==t||(n<-o&&(n=-o),s>2*o&&(s=2*o));var a=Math.max(s-n+.5,1);switch(this.overflow?(this.options.rtl?this.right=n:this.left=n,this.width=a+this.props.content.width,i=this.props.content.width):(this.options.rtl?this.right=n:this.left=n,this.width=a,i=Math.min(s-n,this.props.content.width)),this.options.rtl?this.dom.box.style.right=this.right+"px":this.dom.box.style.left=this.left+"px",this.dom.box.style.width=a+"px",r){case"left":this.options.rtl?this.dom.content.style.right="0":this.dom.content.style.left="0";break;case"right":this.options.rtl?this.dom.content.style.right=Math.max(a-i,0)+"px":this.dom.content.style.left=Math.max(a-i,0)+"px";break;case"center":this.options.rtl?this.dom.content.style.right=Math.max((a-i)/2,0)+"px":this.dom.content.style.left=Math.max((a-i)/2,0)+"px";break;default:e=this.overflow?s>0?Math.max(-n,0):-i:n<0?-n:0,this.options.rtl?this.dom.content.style.right=e+"px":(this.dom.content.style.left=e+"px",this.dom.content.style.width="calc(100% - "+e+"px)")}},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box;e.style.top="top"==t?this.top+"px":this.parent.height-this.top-this.height+"px"},o.prototype._repaintDragLeft=function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.options.editable.updateTime&&!this.dom.dragLeft){var t=document.createElement("div");t.className="vis-drag-left",t.dragLeftItem=this,this.dom.box.appendChild(t),this.dom.dragLeft=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragLeft||(this.dom.dragLeft.parentNode&&this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft),this.dom.dragLeft=null)},o.prototype._repaintDragRight=function(){if((this.selected||this.options.itemsAlwaysDraggable.range)&&this.options.editable.updateTime&&!this.dom.dragRight){var t=document.createElement("div");t.className="vis-drag-right",t.dragRightItem=this,this.dom.box.appendChild(t),this.dom.dragRight=t}else this.selected||this.options.itemsAlwaysDraggable.range||!this.dom.dragRight||(this.dom.dragRight.parentNode&&this.dom.dragRight.parentNode.removeChild(this.dom.dragRight),this.dom.dragRight=null)},t.exports=o},function(t,e,i){function o(t,e){}function n(t,e){return e=void 0===e?{}:e,{style:e.style||t.options.drawPoints.style,styles:e.styles||t.options.drawPoints.styles,size:e.size||t.options.drawPoints.size,className:e.className||t.className}}function s(t,e){var i=void 0;return t.options&&t.options.drawPoints&&t.options.drawPoints.onRender&&"function"==typeof t.options.drawPoints.onRender&&(i=t.options.drawPoints.onRender),e.group.options&&e.group.options.drawPoints&&e.group.options.drawPoints.onRender&&"function"==typeof e.group.options.drawPoints.onRender&&(i=e.group.options.drawPoints.onRender),i}var r=i(1),a=function(t){return t&&t.__esModule?t:{default:t}}(r),h=i(11);o.draw=function(t,e,i,o){o=o||0;for(var r=s(i,e),d=0;d<t.length;d++)if(r){var l=r(t[d],e);!0!==l&&"object"!==(void 0===l?"undefined":(0,a.default)(l))||h.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e,l),i.svgElements,i.svg,t[d].label)}else h.drawPoint(t[d].screen_x+o,t[d].screen_y,n(e),i.svgElements,i.svg,t[d].label)},o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,d=h.getSVGElement("rect",r.svgElements,r.svg);d.setAttributeNS(null,"x",e),d.setAttributeNS(null,"y",i-a),d.setAttributeNS(null,"width",o),d.setAttributeNS(null,"height",2*a),d.setAttributeNS(null,"class","vis-outline"),h.drawPoint(e+.5*o,i,n(t),r.svgElements,r.svg)},t.exports=o},function(t,e,i){i(88);for(var o=i(6),n=i(14),s=i(23),r=i(7)("toStringTag"),a="CSSRuleList,CSSStyleDeclaration,CSSValueList,ClientRectList,DOMRectList,DOMStringList,DOMTokenList,DataTransferItemList,FileList,HTMLAllCollection,HTMLCollection,HTMLFormElement,HTMLSelectElement,MediaList,MimeTypeArray,NamedNodeMap,NodeList,PaintRequestList,Plugin,PluginArray,SVGLengthList,SVGNumberList,SVGPathSegList,SVGPointList,SVGStringList,SVGTransformList,SourceBufferList,StyleSheetList,TextTrackCueList,TextTrackList,TouchList".split(","),h=0;h<a.length;h++){var d=a[h],l=o[d],u=l&&l.prototype;u&&!u[r]&&n(u,r,d),s[d]=s.Array}},function(t,e,i){var o=i(35),n=i(17),s=i(59),r=i(14),a=i(10),h=i(23),d=i(94),l=i(42),u=i(100),p=i(7)("iterator"),c=!([].keys&&"next"in[].keys()),m=function(){return this};t.exports=function(t,e,i,f,g,v,y){d(i,e,f);var b,_,w,x=function(t){if(!c&&t in C)return C[t];switch(t){case"keys":case"values":return function(){return new i(this,t)}}return function(){return new i(this,t)}},D=e+" Iterator",S="values"==g,k=!1,C=t.prototype,T=C[p]||C["@@iterator"]||g&&C[g],M=T||x(g),O=g?S?x("entries"):M:void 0,E="Array"==e?C.entries||T:T;if(E&&(w=u(E.call(new t)))!==Object.prototype&&w.next&&(l(w,D,!0),o||a(w,p)||r(w,p,m)),S&&T&&"values"!==T.name&&(k=!0,M=function(){return T.call(this)}),o&&!y||!c&&!k&&C[p]||r(C,p,M),h[e]=M,h[D]=m,g)if(b={values:S?M:x("values"),keys:v?M:x("keys"),entries:O},y)for(_ in b)_ in C||s(C,_,b[_]);else n(n.P+n.F*(c||k),e,b);return b}},function(t,e,i){t.exports=!i(9)&&!i(19)(function(){return 7!=Object.defineProperty(i(58)("div"),"a",{get:function(){return 7}}).a})},function(t,e,i){var o=i(24),n=i(6).document,s=o(n)&&o(n.createElement);t.exports=function(t){return s?n.createElement(t):{}}},function(t,e,i){t.exports=i(14)},function(t,e,i){var o=i(10),n=i(13),s=i(96)(!1),r=i(39)("IE_PROTO");t.exports=function(t,e){var i,a=n(t),h=0,d=[];for(i in a)i!=r&&o(a,i)&&d.push(i);for(;e.length>h;)o(a,i=e[h++])&&(~s(d,i)||d.push(i));return d}},function(t,e,i){var o=i(34);t.exports=function(t){return Object(o(t))}},function(t,e,i){var o=i(101)(!0);i(56)(String,"String",function(t){this._t=String(t),this._i=0},function(){var t,e=this._t,i=this._i;return i>=e.length?{value:void 0,done:!0}:(t=o(e,i),this._i+=t.length,{value:t,done:!1})})},function(t,e){e.f=Object.getOwnPropertySymbols},function(t,e,i){var o=i(60),n=i(41).concat("length","prototype");e.f=Object.getOwnPropertyNames||function(t){return o(t,n)}},function(t,e,i){function o(t){this.delay=null,this.max=1/0,this._queue=[],this._timeout=null,this._extended=null,this.setOptions(t)}o.prototype.setOptions=function(t){t&&void 0!==t.delay&&(this.delay=t.delay),t&&void 0!==t.max&&(this.max=t.max),this._flushIfNeeded()},o.extend=function(t,e){var i=new o(e);if(void 0!==t.flush)throw new Error("Target object already has a property flush");t.flush=function(){i.flush()};var n=[{name:"flush",original:void 0}];if(e&&e.replace)for(var s=0;s<e.replace.length;s++){var r=e.replace[s];n.push({name:r,original:t[r]}),i.replace(t,r)}return i._extended={object:t,methods:n},i},o.prototype.destroy=function(){if(this.flush(),this._extended){for(var t=this._extended.object,e=this._extended.methods,i=0;i<e.length;i++){var o=e[i];o.original?t[o.name]=o.original:delete t[o.name]}this._extended=null}},o.prototype.replace=function(t,e){var i=this,o=t[e];if(!o)throw new Error("Method "+e+" undefined");t[e]=function(){for(var t=[],e=0;e<arguments.length;e++)t[e]=arguments[e];i.queue({args:t,fn:o,context:this})}},o.prototype.queue=function(t){"function"==typeof t?this._queue.push({fn:t}):this._queue.push(t),this._flushIfNeeded()},o.prototype._flushIfNeeded=function(){if(this._queue.length>this.max&&this.flush(),clearTimeout(this._timeout),this.queue.length>0&&"number"==typeof this.delay){var t=this;this._timeout=setTimeout(function(){t.flush()},this.delay)}},o.prototype.flush=function(){for(;this._queue.length>0;){var t=this._queue.shift();t.fn.apply(t.context||t.fn,t.args||[])}},t.exports=o},function(t,e){function i(t){if(t)return o(t)}function o(t){for(var e in i.prototype)t[e]=i.prototype[e];return t}t.exports=i,i.prototype.on=i.prototype.addEventListener=function(t,e){return this._callbacks=this._callbacks||{},(this._callbacks[t]=this._callbacks[t]||[]).push(e),this},i.prototype.once=function(t,e){function i(){o.off(t,i),e.apply(this,arguments)}var o=this;return this._callbacks=this._callbacks||{},i.fn=e,this.on(t,i),this},i.prototype.off=i.prototype.removeListener=i.prototype.removeAllListeners=i.prototype.removeEventListener=function(t,e){if(this._callbacks=this._callbacks||{},0==arguments.length)return this._callbacks={},this;var i=this._callbacks[t];if(!i)return this;if(1==arguments.length)return delete this._callbacks[t],this;for(var o,n=0;n<i.length;n++)if((o=i[n])===e||o.fn===e){i.splice(n,1);break}return this},i.prototype.emit=function(t){this._callbacks=this._callbacks||{};var e=[].slice.call(arguments,1),i=this._callbacks[t];if(i){i=i.slice(0);for(var o=0,n=i.length;o<n;++o)i[o].apply(this,e)}return this},i.prototype.listeners=function(t){return this._callbacks=this._callbacks||{},this._callbacks[t]||[]},i.prototype.hasListeners=function(t){return!!this.listeners(t).length}},function(t,e,i){e.onTouch=function(t,e){e.inputHandler=function(t){t.isFirst&&e(t)},t.on("hammer.input",e.inputHandler)},e.onRelease=function(t,e){return e.inputHandler=function(t){t.isFinal&&e(t)},t.on("hammer.input",e.inputHandler)},e.offTouch=function(t,e){t.off("hammer.input",e.inputHandler)},e.offRelease=e.offTouch,e.disablePreventDefaultVertically=function(t){return t.getTouchAction=function(){return["pan-y"]},t}},function(t,e,i){var o,n,s;!function(i,r){n=[],o=r,void 0!==(s="function"==typeof o?o.apply(e,n):o)&&(t.exports=s)}(0,function(){function t(t){var e,i=t&&t.preventDefault||!1,o=t&&t.container||window,n={},s={keydown:{},keyup:{}},r={};for(e=97;e<=122;e++)r[String.fromCharCode(e)]={code:e-97+65,shift:!1};for(e=65;e<=90;e++)r[String.fromCharCode(e)]={code:e,shift:!0};for(e=0;e<=9;e++)r[""+e]={code:48+e,shift:!1};for(e=1;e<=12;e++)r["F"+e]={code:111+e,shift:!1};for(e=0;e<=9;e++)r["num"+e]={code:96+e,shift:!1};r["num*"]={code:106,shift:!1},r["num+"]={code:107,shift:!1},r["num-"]={code:109,shift:!1},r["num/"]={code:111,shift:!1},r["num."]={code:110,shift:!1},r.left={code:37,shift:!1},r.up={code:38,shift:!1},r.right={code:39,shift:!1},r.down={code:40,shift:!1},r.space={code:32,shift:!1},r.enter={code:13,shift:!1},r.shift={code:16,shift:void 0},r.esc={code:27,shift:!1},r.backspace={code:8,shift:!1},r.tab={code:9,shift:!1},r.ctrl={code:17,shift:!1},r.alt={code:18,shift:!1},r.delete={code:46,shift:!1},r.pageup={code:33,shift:!1},r.pagedown={code:34,shift:!1},r["="]={code:187,shift:!1},r["-"]={code:189,shift:!1},r["]"]={code:221,shift:!1},r["["]={code:219,shift:!1};var a=function(t){d(t,"keydown")},h=function(t){d(t,"keyup")},d=function(t,e){if(void 0!==s[e][t.keyCode]){for(var o=s[e][t.keyCode],n=0;n<o.length;n++)void 0===o[n].shift?o[n].fn(t):1==o[n].shift&&1==t.shiftKey?o[n].fn(t):0==o[n].shift&&0==t.shiftKey&&o[n].fn(t);1==i&&t.preventDefault()}};return n.bind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);void 0===s[i][r[t].code]&&(s[i][r[t].code]=[]),s[i][r[t].code].push({fn:e,shift:r[t].shift})},n.bindAll=function(t,e){void 0===e&&(e="keydown");for(var i in r)r.hasOwnProperty(i)&&n.bind(i,t,e)},n.getKey=function(t){for(var e in r)if(r.hasOwnProperty(e)){if(1==t.shiftKey&&1==r[e].shift&&t.keyCode==r[e].code)return e;if(0==t.shiftKey&&0==r[e].shift&&t.keyCode==r[e].code)return e;if(t.keyCode==r[e].code&&"shift"==e)return e}return"unknown key, currently not supported"},n.unbind=function(t,e,i){if(void 0===i&&(i="keydown"),void 0===r[t])throw new Error("unsupported key: "+t);if(void 0!==e){var o=[],n=s[i][r[t].code];if(void 0!==n)for(var a=0;a<n.length;a++)n[a].fn==e&&n[a].shift==r[t].shift||o.push(s[i][r[t].code][a]);s[i][r[t].code]=o}else s[i][r[t].code]=[]},n.reset=function(){s={keydown:{},keyup:{}}},n.destroy=function(){s={keydown:{},keyup:{}},o.removeEventListener("keydown",a,!0),o.removeEventListener("keyup",h,!0)},o.addEventListener("keydown",a,!0),o.addEventListener("keyup",h,!0),n}return t})},function(t,e,i){e.en={current:"current",time:"time"},e.en_EN=e.en,e.en_US=e.en,e.it={current:"attuale",time:"tempo"},e.it_IT=e.it,e.it_CH=e.it,e.nl={current:"huidige",time:"tijd"},e.nl_NL=e.nl,e.nl_BE=e.nl,e.de={current:"Aktuelle",time:"Zeit"},e.de_DE=e.de,e.fr={current:"actuel",time:"heure"},e.fr_FR=e.fr,e.fr_CA=e.fr,e.fr_BE=e.fr,e.es={current:"corriente",time:"hora"},e.es_ES=e.es},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}function n(t,e){this.body=t,this.defaultOptions={type:null,orientation:{item:"bottom"},align:"auto",stack:!0,stackSubgroups:!0,groupOrderSwap:function(t,e,i){var o=e.order;e.order=t.order,t.order=o},groupOrder:"order",selectable:!0,multiselect:!1,itemsAlwaysDraggable:{item:!1,range:!1},editable:{updateTime:!1,updateGroup:!1,add:!1,remove:!1,overrideItems:!1},groupEditable:{order:!1,add:!1,remove:!1},snap:c.snap,onDropObjectOnItem:function(t,e,i){i(e)},onAdd:function(t,e){e(t)},onUpdate:function(t,e){e(t)},onMove:function(t,e){e(t)},onRemove:function(t,e){e(t)},onMoving:function(t,e){e(t)},onAddGroup:function(t,e){e(t)},onMoveGroup:function(t,e){e(t)},onRemoveGroup:function(t,e){e(t)},margin:{item:{horizontal:10,vertical:10},axis:20},showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip"},tooltipOnItemUpdateTime:!1},this.options=l.extend({},this.defaultOptions),this.options.rtl=e.rtl,this.itemOptions={type:{start:"Date",end:"Date"}},this.conversion={toScreen:t.util.toScreen,toTime:t.util.toTime},this.dom={},this.props={},this.hammer=null;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){if(i._onAddGroups(e.items),i.groupsData&&i.groupsData.length>0){var n=i.groupsData.getDataSet();n.get().forEach(function(t){if(t.nestedGroups){0!=t.showNested&&(t.showNested=!0);var e=[];t.nestedGroups.forEach(function(i){var o=n.get(i);o&&(o.nestedInGroup=t.id,0==t.showNested&&(o.visible=!1),e=e.concat(o))}),n.update(e,o)}})}},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.groups={},this.groupIds=[],this.selection=[],this.popup=null,this.touchParams={},this.groupTouchParams={},this._create(),this.setOptions(e)}var s=i(43),r=o(s),a=i(1),h=o(a),d=i(12),l=i(0),u=i(15),p=i(20),c=i(49),m=i(5),f=i(51),g=i(52),v=i(72),y=i(73),b=i(53),_=i(74),w=i(133).default,x="__ungrouped__",D="__background__";n.prototype=new m,n.types={background:_,box:v,range:b,point:y},n.prototype._create=function(){var t=document.createElement("div");t.className="vis-itemset",t["timeline-itemset"]=this,this.dom.frame=t;var e=document.createElement("div");e.className="vis-background",t.appendChild(e),this.dom.background=e;var i=document.createElement("div");i.className="vis-foreground",t.appendChild(i),this.dom.foreground=i;var o=document.createElement("div");o.className="vis-axis",this.dom.axis=o;var n=document.createElement("div");n.className="vis-labelset",this.dom.labelSet=n,this._updateUngrouped();var s=new g(D,null,this);s.show(),this.groups[D]=s,this.hammer=new d(this.body.dom.centerContainer),this.hammer.on("hammer.input",function(t){t.isFirst&&this._onTouch(t)}.bind(this)),this.hammer.on("panstart",this._onDragStart.bind(this)),this.hammer.on("panmove",this._onDrag.bind(this)),this.hammer.on("panend",this._onDragEnd.bind(this)),this.hammer.get("pan").set({threshold:5,direction:d.DIRECTION_HORIZONTAL}),this.hammer.on("tap",this._onSelectItem.bind(this)),this.hammer.on("press",this._onMultiSelectItem.bind(this)),this.hammer.on("doubletap",this._onAddItem.bind(this)),this.options.rtl?this.groupHammer=new d(this.body.dom.rightContainer):this.groupHammer=new d(this.body.dom.leftContainer),this.groupHammer.on("tap",this._onGroupClick.bind(this)),this.groupHammer.on("panstart",this._onGroupDragStart.bind(this)),this.groupHammer.on("panmove",this._onGroupDrag.bind(this)),this.groupHammer.on("panend",this._onGroupDragEnd.bind(this)),this.groupHammer.get("pan").set({threshold:5,direction:d.DIRECTION_VERTICAL}),this.body.dom.centerContainer.addEventListener("mouseover",this._onMouseOver.bind(this)),this.body.dom.centerContainer.addEventListener("mouseout",this._onMouseOut.bind(this)),this.body.dom.centerContainer.addEventListener("mousemove",this._onMouseMove.bind(this)),this.body.dom.centerContainer.addEventListener("contextmenu",this._onDragEnd.bind(this)),this.body.dom.centerContainer.addEventListener("mousewheel",this._onMouseWheel.bind(this)),this.show()},n.prototype.setOptions=function(t){if(t){var e=["type","rtl","align","order","stack","stackSubgroups","selectable","multiselect","multiselectPerGroup","groupOrder","dataAttributes","template","groupTemplate","visibleFrameTemplate","hide","snap","groupOrderSwap","showTooltips","tooltip","tooltipOnItemUpdateTime"];l.selectiveExtend(e,this.options,t),"itemsAlwaysDraggable"in t&&("boolean"==typeof t.itemsAlwaysDraggable?(this.options.itemsAlwaysDraggable.item=t.itemsAlwaysDraggable,this.options.itemsAlwaysDraggable.range=!1):"object"===(0,h.default)(t.itemsAlwaysDraggable)&&(l.selectiveExtend(["item","range"],this.options.itemsAlwaysDraggable,t.itemsAlwaysDraggable),this.options.itemsAlwaysDraggable.item||(this.options.itemsAlwaysDraggable.range=!1))),"orientation"in t&&("string"==typeof t.orientation?this.options.orientation.item="top"===t.orientation?"top":"bottom":"object"===(0,h.default)(t.orientation)&&"item"in t.orientation&&(this.options.orientation.item=t.orientation.item)),"margin"in t&&("number"==typeof t.margin?(this.options.margin.axis=t.margin,this.options.margin.item.horizontal=t.margin,this.options.margin.item.vertical=t.margin):"object"===(0,h.default)(t.margin)&&(l.selectiveExtend(["axis"],this.options.margin,t.margin),"item"in t.margin&&("number"==typeof t.margin.item?(this.options.margin.item.horizontal=t.margin.item,this.options.margin.item.vertical=t.margin.item):"object"===(0,h.default)(t.margin.item)&&l.selectiveExtend(["horizontal","vertical"],this.options.margin.item,t.margin.item)))),"editable"in t&&("boolean"==typeof t.editable?(this.options.editable.updateTime=t.editable,this.options.editable.updateGroup=t.editable,this.options.editable.add=t.editable,this.options.editable.remove=t.editable,this.options.editable.overrideItems=!1):"object"===(0,h.default)(t.editable)&&l.selectiveExtend(["updateTime","updateGroup","add","remove","overrideItems"],this.options.editable,t.editable)),"groupEditable"in t&&("boolean"==typeof t.groupEditable?(this.options.groupEditable.order=t.groupEditable,this.options.groupEditable.add=t.groupEditable,this.options.groupEditable.remove=t.groupEditable):"object"===(0,h.default)(t.groupEditable)&&l.selectiveExtend(["order","add","remove"],this.options.groupEditable,t.groupEditable));["onDropObjectOnItem","onAdd","onUpdate","onRemove","onMove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup"].forEach(function(e){var i=t[e];if(i){if(!(i instanceof Function))throw new Error("option "+e+" must be a function "+e+"(item, callback)");this.options[e]=i}}.bind(this)),this.markDirty()}},n.prototype.markDirty=function(t){this.groupIds=[],t&&t.refreshItems&&l.forEach(this.items,function(t){t.dirty=!0,t.displayed&&t.redraw()})},n.prototype.destroy=function(){this.hide(),this.setItems(null),this.setGroups(null),this.hammer=null,this.body=null,this.conversion=null},n.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.axis.parentNode&&this.dom.axis.parentNode.removeChild(this.dom.axis),this.dom.labelSet.parentNode&&this.dom.labelSet.parentNode.removeChild(this.dom.labelSet)},n.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame),this.dom.axis.parentNode||this.body.dom.backgroundVertical.appendChild(this.dom.axis),this.dom.labelSet.parentNode||(this.options.rtl?this.body.dom.right.appendChild(this.dom.labelSet):this.body.dom.left.appendChild(this.dom.labelSet))},n.prototype.setSelection=function(t){var e,i,o,n;for(void 0==t&&(t=[]),Array.isArray(t)||(t=[t]),e=0,i=this.selection.length;e<i;e++)o=this.selection[e],(n=this.items[o])&&n.unselect();for(this.selection=[],e=0,i=t.length;e<i;e++)o=t[e],(n=this.items[o])&&(this.selection.push(o),n.select())},n.prototype.getSelection=function(){return this.selection.concat([])},n.prototype.getVisibleItems=function(){var t,e,i=this.body.range.getRange();this.options.rtl?(t=this.body.util.toScreen(i.start),e=this.body.util.toScreen(i.end)):(e=this.body.util.toScreen(i.start),t=this.body.util.toScreen(i.end));var o=[];for(var n in this.groups)if(this.groups.hasOwnProperty(n))for(var s=this.groups[n],r=s.isVisible?s.visibleItems:[],a=0;a<r.length;a++){var h=r[a];this.options.rtl?h.right<e&&h.right+h.width>t&&o.push(h.id):h.left<t&&h.left+h.width>e&&o.push(h.id)}return o},n.prototype._deselect=function(t){for(var e=this.selection,i=0,o=e.length;i<o;i++)if(e[i]==t){e.splice(i,1);break}},n.prototype.redraw=function(){var t=this.options.margin,e=this.body.range,i=l.option.asSize,o=this.options,n=o.orientation.item,s=!1,r=this.dom.frame;this.props.top=this.body.domProps.top.height+this.body.domProps.border.top,this.options.rtl?this.props.right=this.body.domProps.right.width+this.body.domProps.border.right:this.props.left=this.body.domProps.left.width+this.body.domProps.border.left,r.className="vis-itemset",s=this._orderGroups()||s;var a=e.end-e.start,h=a!=this.lastVisibleInterval||this.props.width!=this.props.lastWidth,d=e.start!=this.lastRangeStart,u=o.stack!=this.lastStack,p=o.stackSubgroups!=this.lastStackSubgroups,c=h||d||u||p;this.lastVisibleInterval=a,this.lastRangeStart=e.start,this.lastStack=o.stack,this.lastStackSubgroups=o.stackSubgroups,this.props.lastWidth=this.props.width;var m=this._firstGroup(),f={item:t.item,axis:t.axis},g={item:t.item,axis:t.item.vertical/2},v=0,y=t.axis+t.item.vertical;this.groups[D].redraw(e,g,c);var b={},_=0;if(l.forEach(this.groups,function(t,i){if(i!==D){var o=t==m?f:g;b[i]=t.redraw(e,o,c,!0),_=b[i].length}}),_>0){for(var w={},x=0;x<_;x++)l.forEach(b,function(t,e){w[e]=t[x]()});l.forEach(this.groups,function(t,e){if(e!==D){var i=w[e];s=i||s,v+=t.height}}),v=Math.max(v,y)}return v=Math.max(v,y),r.style.height=i(v),this.props.width=r.offsetWidth,this.props.height=v,this.dom.axis.style.top=i("top"==n?this.body.domProps.top.height+this.body.domProps.border.top:this.body.domProps.top.height+this.body.domProps.centerContainer.height),this.options.rtl?this.dom.axis.style.right="0":this.dom.axis.style.left="0",this.initialItemSetDrawn=!0,s=this._isResized()||s},n.prototype._firstGroup=function(){var t="top"==this.options.orientation.item?0:this.groupIds.length-1,e=this.groupIds[t];return this.groups[e]||this.groups[x]||null},n.prototype._updateUngrouped=function(){var t,e,i=this.groups[x];if(this.groupsData){if(i){i.hide(),delete this.groups[x];for(e in this.items)if(this.items.hasOwnProperty(e)){t=this.items[e],t.parent&&t.parent.remove(t);var o=this._getGroupId(t.data),n=this.groups[o];n&&n.add(t)||t.hide()}}}else if(!i){i=new f(null,null,this),this.groups[x]=i;for(e in this.items)this.items.hasOwnProperty(e)&&(t=this.items[e],i.add(t));i.show()}},n.prototype.getLabelSet=function(){return this.dom.labelSet},n.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof u||t instanceof p))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(l.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;l.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e),this._updateUngrouped()}this.body.emitter.emit("_change",{queue:!0})},n.prototype.getItems=function(){return this.itemsData},n.prototype.setGroups=function(t){var e,i=this;if(this.groupsData&&(l.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null,this._onRemoveGroups(e)),t){if(!(t instanceof u||t instanceof p))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var o=this.groupsData;this.groupsData instanceof p&&(o=this.groupsData.getDataSet()),o.get().forEach(function(t){t.nestedGroups&&t.nestedGroups.forEach(function(e){var i=o.get(e);i.nestedInGroup=t.id,0==t.showNested&&(i.visible=!1),o.update(i)})});var n=this.id;l.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}this._updateUngrouped(),this._order(),this.body.emitter.emit("_change",{queue:!0})},n.prototype.getGroups=function(){return this.groupsData},n.prototype.removeItem=function(t){var e=this.itemsData.get(t),i=this.itemsData.getDataSet();e&&this.options.onRemove(e,function(e){e&&i.remove(t)})},n.prototype._getType=function(t){return t.type||this.options.type||(t.end?"range":"box")},n.prototype._getGroupId=function(t){return"background"==this._getType(t)&&void 0==t.group?D:this.groupsData?t.group:x},n.prototype._onUpdate=function(t){var e=this;t.forEach(function(t){var i,o=e.itemsData.get(t,e.itemOptions),s=e.items[t],r=o?e._getType(o):null,a=n.types[r];if(s&&(a&&s instanceof a?e._updateItem(s,o):(i=s.selected,e._removeItem(s),s=null)),!s&&o){if(!a)throw"rangeoverflow"==r?new TypeError('Item type "rangeoverflow" is deprecated. Use css styling instead: .vis-item.vis-range .vis-item-content {overflow: visible;}'):new TypeError('Unknown item type "'+r+'"');s=new a(o,e.conversion,e.options),s.id=t,e._addItem(s),i&&(this.selection.push(t),s.select())}}.bind(this)),this._order(),this.body.emitter.emit("_change",{queue:!0})},n.prototype._onAdd=n.prototype._onUpdate,n.prototype._onRemove=function(t){var e=0,i=this;t.forEach(function(t){var o=i.items[t];o&&(e++,i._removeItem(o))}),e&&(this._order(),this.body.emitter.emit("_change",{queue:!0}))},n.prototype._order=function(){l.forEach(this.groups,function(t){t.order()})},n.prototype._onUpdateGroups=function(t){this._onAddGroups(t)},n.prototype._onAddGroups=function(t){var e=this;t.forEach(function(t){var i=e.groupsData.get(t),o=e.groups[t];if(o)o.setData(i);else{if(t==x||t==D)throw new Error("Illegal group id. "+t+" is a reserved id.");var n=(0,r.default)(e.options);l.extend(n,{height:null}),o=new f(t,i,e),e.groups[t]=o;for(var s in e.items)if(e.items.hasOwnProperty(s)){var a=e.items[s];a.data.group==t&&o.add(a)}o.order(),o.show()}}),this.body.emitter.emit("_change",{queue:!0})},n.prototype._onRemoveGroups=function(t){var e=this.groups;t.forEach(function(t){var i=e[t];i&&(i.hide(),delete e[t])}),this.markDirty(),this.body.emitter.emit("_change",{queue:!0})},n.prototype._orderGroups=function(){if(this.groupsData){var t=this.groupsData.getIds({order:this.options.groupOrder});t=this._orderNestedGroups(t);var e=!l.equalArray(t,this.groupIds);if(e){var i=this.groups;t.forEach(function(t){i[t].hide()}),t.forEach(function(t){i[t].show()}),this.groupIds=t}return e}return!1},n.prototype._orderNestedGroups=function(t){var e=[];return t.forEach(function(t){var i=this.groupsData.get(t);if(i.nestedInGroup||e.push(t),i.nestedGroups){var o=this.groupsData.get({filter:function(e){return e.nestedInGroup==t},order:this.options.groupOrder}),n=o.map(function(t){return t.id});e=e.concat(n)}},this),e},n.prototype._addItem=function(t){this.items[t.id]=t;var e=this._getGroupId(t.data),i=this.groups[e];i?i&&i.data&&i.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1,i&&i.add(t)},n.prototype._updateItem=function(t,e){t.setData(e);var i=this._getGroupId(t.data),o=this.groups[i];o?o&&o.data&&o.data.showNested&&(t.groupShowing=!0):t.groupShowing=!1},n.prototype._removeItem=function(t){t.hide(),delete this.items[t.id];var e=this.selection.indexOf(t.id);-1!=e&&this.selection.splice(e,1),t.parent&&t.parent.remove(t)},n.prototype._constructByEndArray=function(t){for(var e=[],i=0;i<t.length;i++)t[i]instanceof b&&e.push(t[i]);return e},n.prototype._onTouch=function(t){this.touchParams.item=this.itemFromTarget(t),this.touchParams.dragLeftItem=t.target.dragLeftItem||!1,this.touchParams.dragRightItem=t.target.dragRightItem||!1,this.touchParams.itemProps=null},n.prototype._getGroupIndex=function(t){for(var e=0;e<this.groupIds.length;e++)if(t==this.groupIds[e])return e},n.prototype._onDragStart=function(t){if(!this.touchParams.itemIsDragging){var e,i=this.touchParams.item||null,o=this;if(i&&(i.selected||this.options.itemsAlwaysDraggable.item)){if(this.options.editable.overrideItems&&!this.options.editable.updateTime&&!this.options.editable.updateGroup)return;if(null!=i.editable&&!i.editable.updateTime&&!i.editable.updateGroup&&!this.options.editable.overrideItems)return;var n=this.touchParams.dragLeftItem,s=this.touchParams.dragRightItem;if(this.touchParams.itemIsDragging=!0,this.touchParams.selectedItem=i,n)e={item:n,initialX:t.center.x,dragLeft:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(s)e={item:s,initialX:t.center.x,dragRight:!0,data:this._cloneItemData(i.data)},this.touchParams.itemProps=[e];else if(this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey))this._onDragStartAddItem(t);else{this.groupIds.length<1&&this.redraw();var r=this._getGroupIndex(i.data.group),a=this.options.itemsAlwaysDraggable.item&&!i.selected?[i.id]:this.getSelection();this.touchParams.itemProps=a.map(function(e){var i=o.items[e],n=o._getGroupIndex(i.data.group);return{item:i,initialX:t.center.x,groupOffset:r-n,data:this._cloneItemData(i.data)}}.bind(this))}t.stopPropagation()}else this.options.editable.add&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey)&&this._onDragStartAddItem(t)}},n.prototype._onDragStartAddItem=function(t){var e,i,o=this.options.snap||null;this.options.rtl?(e=l.getAbsoluteRight(this.dom.frame),i=e-t.center.x+10):(e=l.getAbsoluteLeft(this.dom.frame),i=t.center.x-e-10);var n=this.body.util.toTime(i),s=this.body.util.getScale(),r=this.body.util.getStep(),a=o?o(n,s,r):n,h=a,d={type:"range",start:a,end:h,content:"new item"},u=l.randomUUID();d[this.itemsData._fieldId]=u;var p=this.groupFromTarget(t);p&&(d.group=p.groupId);var c=new b(d,this.conversion,this.options);c.id=u,c.data=this._cloneItemData(d),this._addItem(c),this.touchParams.selectedItem=c;var m={item:c,initialX:t.center.x,data:c.data};this.options.rtl?m.dragLeft=!0:m.dragRight=!0,this.touchParams.itemProps=[m],t.stopPropagation()},n.prototype._onDrag=function(t){if(this.touchParams.itemProps){t.stopPropagation();var e,i=this,o=this.options.snap||null;e=this.options.rtl?this.body.dom.root.offsetLeft+this.body.domProps.right.width:this.body.dom.root.offsetLeft+this.body.domProps.left.width;var n=this.body.util.getScale(),s=this.body.util.getStep(),r=this.touchParams.selectedItem,a=(this.options.editable.overrideItems||null==r.editable)&&this.options.editable.updateGroup||!this.options.editable.overrideItems&&null!=r.editable&&r.editable.updateGroup,h=null;if(a&&r&&void 0!=r.data.group){var d=i.groupFromTarget(t);d&&(h=this._getGroupIndex(d.groupId))}this.touchParams.itemProps.forEach(function(d){var u,p,c,m,f,g=i.body.util.toTime(t.center.x-e),v=i.body.util.toTime(d.initialX-e);u=this.options.rtl?-(g-v):g-v;var y=this._cloneItemData(d.item.data);if(null==d.item.editable||d.item.editable.updateTime||d.item.editable.updateGroup||i.options.editable.overrideItems){if((this.options.editable.overrideItems||null==r.editable)&&this.options.editable.updateTime||!this.options.editable.overrideItems&&null!=r.editable&&r.editable.updateTime)if(d.dragLeft)this.options.rtl?void 0!=y.end&&(c=l.convert(d.data.end,"Date"),f=new Date(c.valueOf()+u),y.end=o?o(f,n,s):f):void 0!=y.start&&(p=l.convert(d.data.start,"Date"),m=new Date(p.valueOf()+u),y.start=o?o(m,n,s):m);else if(d.dragRight)this.options.rtl?void 0!=y.start&&(p=l.convert(d.data.start,"Date"),m=new Date(p.valueOf()+u),y.start=o?o(m,n,s):m):void 0!=y.end&&(c=l.convert(d.data.end,"Date"),f=new Date(c.valueOf()+u),y.end=o?o(f,n,s):f);else if(void 0!=y.start)if(p=l.convert(d.data.start,"Date").valueOf(),m=new Date(p+u),void 0!=y.end){c=l.convert(d.data.end,"Date");var b=c.valueOf()-p.valueOf();y.start=o?o(m,n,s):m,y.end=new Date(y.start.valueOf()+b)}else y.start=o?o(m,n,s):m;if(a&&!d.dragLeft&&!d.dragRight&&null!=h&&void 0!=y.group){var _=h-d.groupOffset;_=Math.max(0,_),_=Math.min(i.groupIds.length-1,_),y.group=i.groupIds[_]}y=this._cloneItemData(y),i.options.onMoving(y,function(t){t&&d.item.setData(this._cloneItemData(t,"Date"))}.bind(this))}}.bind(this)),this.body.emitter.emit("_change")}},n.prototype._moveToGroup=function(t,e){var i=this.groups[e];if(i&&i.groupId!=t.data.group){var o=t.parent;o.remove(t),o.order(),t.data.group=i.groupId,i.add(t),i.order()}},n.prototype._onDragEnd=function(t){if(this.touchParams.itemIsDragging=!1,this.touchParams.itemProps){t.stopPropagation();var e=this,i=this.itemsData.getDataSet(),o=this.touchParams.itemProps;this.touchParams.itemProps=null,o.forEach(function(t){var o=t.item.id;if(null!=e.itemsData.get(o,e.itemOptions)){var n=this._cloneItemData(t.item.data);e.options.onMove(n,function(n){n?(n[i._fieldId]=o,i.update(n)):(t.item.setData(t.data),e.body.emitter.emit("_change"))})}else e.options.onAdd(t.item.data,function(i){e._removeItem(t.item),i&&e.itemsData.getDataSet().add(i),e.body.emitter.emit("_change")})}.bind(this))}},n.prototype._onGroupClick=function(t){var e=this.groupFromTarget(t);if(e&&e.nestedGroups){var i=this.groupsData.getDataSet(),o=i.get(e.groupId);void 0==o.showNested&&(o.showNested=!0),o.showNested=!o.showNested;var n=i.get(e.nestedGroups).map(function(t){return t.visible=o.showNested,t});if(i.update(n.concat(o)),o.showNested)l.removeClassName(e.dom.label,"collapsed"),l.addClassName(e.dom.label,"expanded");else{l.removeClassName(e.dom.label,"expanded");var s=this.options.rtl?"collapsed-rtl":"collapsed";l.addClassName(e.dom.label,s)}}},n.prototype._onGroupDragStart=function(t){this.options.groupEditable.order&&(this.groupTouchParams.group=this.groupFromTarget(t),this.groupTouchParams.group&&(t.stopPropagation(),this.groupTouchParams.originalOrder=this.groupsData.getIds({order:this.options.groupOrder})))},n.prototype._onGroupDrag=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this.groupsData;this.groupsData instanceof p&&(e=this.groupsData.getDataSet());var i=this.groupFromTarget(t);if(i&&i.height!=this.groupTouchParams.group.height){var o=i.top<this.groupTouchParams.group.top,n=t.center?t.center.y:t.clientY,s=l.getAbsoluteTop(i.dom.foreground),r=this.groupTouchParams.group.height;if(o){if(s+r<n)return}else{if(s+i.height-r>n)return}}if(i&&i!=this.groupTouchParams.group){var a=e.get(i.groupId),h=e.get(this.groupTouchParams.group.groupId);h&&a&&(this.options.groupOrderSwap(h,a,e),e.update(h),e.update(a));var d=e.getIds({order:this.options.groupOrder});if(!l.equalArray(d,this.groupTouchParams.originalOrder))for(var u=this.groupTouchParams.originalOrder,c=this.groupTouchParams.group.groupId,m=Math.min(u.length,d.length),f=0,g=0,v=0;f<m;){for(;f+g<m&&f+v<m&&d[f+g]==u[f+v];)f++;if(f+g>=m)break;if(d[f+g]==c)g=1;else if(u[f+v]==c)v=1;else{var y=d.indexOf(u[f+v]),b=e.get(d[f+g]),_=e.get(u[f+v]);this.options.groupOrderSwap(b,_,e),e.update(b),e.update(_);var w=d[f+g];d[f+g]=u[f+v],d[y]=w,f++}}}}},n.prototype._onGroupDragEnd=function(t){if(this.options.groupEditable.order&&this.groupTouchParams.group){t.stopPropagation();var e=this,i=e.groupTouchParams.group.groupId,o=e.groupsData.getDataSet(),n=l.extend({},o.get(i));e.options.onMoveGroup(n,function(t){if(t)t[o._fieldId]=i,o.update(t);else{var n=o.getIds({order:e.options.groupOrder});if(!l.equalArray(n,e.groupTouchParams.originalOrder))for(var s=e.groupTouchParams.originalOrder,r=Math.min(s.length,n.length),a=0;a<r;){for(;a<r&&n[a]==s[a];)a++;if(a>=r)break;var h=n.indexOf(s[a]),d=o.get(n[a]),u=o.get(s[a]);e.options.groupOrderSwap(d,u,o),o.update(d),o.update(u);var p=n[a];n[a]=s[a],n[h]=p,a++}}}),e.body.emitter.emit("groupDragged",{groupId:i})}},n.prototype._onSelectItem=function(t){if(this.options.selectable){var e=t.srcEvent&&(t.srcEvent.ctrlKey||t.srcEvent.metaKey),i=t.srcEvent&&t.srcEvent.shiftKey;if(e||i)return void this._onMultiSelectItem(t);var o=this.getSelection(),n=this.itemFromTarget(t),s=n?[n.id]:[];this.setSelection(s);var r=this.getSelection();(r.length>0||o.length>0)&&this.body.emitter.emit("select",{items:r,event:t})}},n.prototype._onMouseOver=function(t){var e=this.itemFromTarget(t);if(e){if(e!==this.itemFromRelatedTarget(t)){var i=e.getTitle();if(this.options.showTooltips&&i){null==this.popup&&(this.popup=new w(this.body.dom.root,this.options.tooltip.overflowMethod||"flip")),this.popup.setText(i);var o=this.body.dom.centerContainer;this.popup.setPosition(t.clientX-l.getAbsoluteLeft(o)+o.offsetLeft,t.clientY-l.getAbsoluteTop(o)+o.offsetTop),this.popup.show()}else null!=this.popup&&this.popup.hide();this.body.emitter.emit("itemover",{item:e.id,event:t})}}},n.prototype._onMouseOut=function(t){var e=this.itemFromTarget(t);if(e){e!==this.itemFromRelatedTarget(t)&&(null!=this.popup&&this.popup.hide(),this.body.emitter.emit("itemout",{item:e.id,event:t}))}},n.prototype._onMouseMove=function(t){if(this.itemFromTarget(t)&&this.options.showTooltips&&this.options.tooltip.followMouse&&this.popup&&!this.popup.hidden){var e=this.body.dom.centerContainer;this.popup.setPosition(t.clientX-l.getAbsoluteLeft(e)+e.offsetLeft,t.clientY-l.getAbsoluteTop(e)+e.offsetTop),this.popup.show()}},n.prototype._onMouseWheel=function(t){this.touchParams.itemIsDragging&&this._onDragEnd(t)},n.prototype._onUpdateItem=function(t){if(this.options.selectable&&this.options.editable.add){var e=this;if(t){var i=e.itemsData.get(t.id);this.options.onUpdate(i,function(t){t&&e.itemsData.getDataSet().update(t)})}}},n.prototype._onDropObjectOnItem=function(t){var e=this.itemFromTarget(t),i=JSON.parse(t.dataTransfer.getData("text"));this.options.onDropObjectOnItem(i,e)},n.prototype._onAddItem=function(t){if(this.options.selectable&&this.options.editable.add){var e,i,o=this,n=this.options.snap||null;this.options.rtl?(e=l.getAbsoluteRight(this.dom.frame),i=e-t.center.x):(e=l.getAbsoluteLeft(this.dom.frame),i=t.center.x-e);var s,r,a=this.body.util.toTime(i),h=this.body.util.getScale(),d=this.body.util.getStep();"drop"==t.type?(r=JSON.parse(t.dataTransfer.getData("text")),r.content=r.content?r.content:"new item",r.start=r.start?r.start:n?n(a,h,d):a,r.type=r.type||"box",r[this.itemsData._fieldId]=r.id||l.randomUUID(),"range"!=r.type||r.end||(s=this.body.util.toTime(i+this.props.width/5),r.end=n?n(s,h,d):s)):(r={start:n?n(a,h,d):a,content:"new item"},r[this.itemsData._fieldId]=l.randomUUID(),"range"===this.options.type&&(s=this.body.util.toTime(i+this.props.width/5),r.end=n?n(s,h,d):s));var u=this.groupFromTarget(t);u&&(r.group=u.groupId),r=this._cloneItemData(r),this.options.onAdd(r,function(e){e&&(o.itemsData.getDataSet().add(e),"drop"==t.type&&o.setSelection([e.id]))})}},n.prototype._onMultiSelectItem=function(t){if(this.options.selectable){var e=this.itemFromTarget(t);if(e){var i=this.options.multiselect?this.getSelection():[];if((t.srcEvent&&t.srcEvent.shiftKey||!1)&&this.options.multiselect){var o=this.itemsData.get(e.id).group,s=void 0;this.options.multiselectPerGroup&&i.length>0&&(s=this.itemsData.get(i[0]).group),this.options.multiselectPerGroup&&void 0!=s&&s!=o||i.push(e.id);var r=n._getItemRange(this.itemsData.get(i,this.itemOptions));if(!this.options.multiselectPerGroup||s==o){i=[];for(var a in this.items)if(this.items.hasOwnProperty(a)){var h=this.items[a],d=h.data.start,l=void 0!==h.data.end?h.data.end:d;!(d>=r.min&&l<=r.max)||this.options.multiselectPerGroup&&s!=this.itemsData.get(h.id).group||h instanceof _||i.push(h.id)}}}else{var u=i.indexOf(e.id);-1==u?i.push(e.id):i.splice(u,1)}this.setSelection(i),this.body.emitter.emit("select",{items:this.getSelection(),event:t})}}},n._getItemRange=function(t){var e=null,i=null;return t.forEach(function(t){(null==i||t.start<i)&&(i=t.start),void 0!=t.end?(null==e||t.end>e)&&(e=t.end):(null==e||t.start>e)&&(e=t.start)}),{min:i,max:e}},n.prototype.itemFromElement=function(t){for(var e=t;e;){if(e.hasOwnProperty("timeline-item"))return e["timeline-item"];e=e.parentNode}return null},n.prototype.itemFromTarget=function(t){return this.itemFromElement(t.target)},n.prototype.itemFromRelatedTarget=function(t){return this.itemFromElement(t.relatedTarget)},n.prototype.groupFromTarget=function(t){var e=t.center?t.center.y:t.clientY,i=this.groupIds;i.length<=0&&this.groupsData&&(i=this.groupsData.getIds({order:this.options.groupOrder}));for(var o=0;o<i.length;o++){var n=i[o],s=this.groups[n],r=s.dom.foreground,a=l.getAbsoluteTop(r);if(e>a&&e<a+r.offsetHeight)return s;if("top"===this.options.orientation.item){if(o===this.groupIds.length-1&&e>a)return s}else if(0===o&&e<a+r.offset)return s}return null},n.itemSetFromTarget=function(t){for(var e=t.target;e;){if(e.hasOwnProperty("timeline-itemset"))return e["timeline-itemset"];e=e.parentNode}return null},n.prototype._cloneItemData=function(t,e){var i=l.extend({},t);return e||(e=this.itemsData.getDataSet()._options.type),void 0!=i.start&&(i.start=l.convert(i.start,e&&e.start||"Date")),void 0!=i.end&&(i.end=l.convert(i.end,e&&e.end||"Date")),i},t.exports=n},function(t,e,i){e.orderByStart=function(t){t.sort(function(t,e){return t.data.start-e.data.start})},e.orderByEnd=function(t){t.sort(function(t,e){return("end"in t.data?t.data.end:t.data.start)-("end"in e.data?e.data.end:e.data.start)})},e.stack=function(t,i,o){if(o)for(var n=0;n<t.length;n++)t[n].top=null;for(var n=0;n<t.length;n++){var s=t[n];if(s.stack&&null===s.top){s.top=i.axis;do{for(var r=null,a=0,h=t.length;a<h;a++){var d=t[a];if(null!==d.top&&d!==s&&d.stack&&e.collision(s,d,i.item,d.options.rtl)){r=d;break}}null!=r&&(s.top=r.top+r.height+i.item.vertical)}while(r)}}},e.substack=function(t,i,o){for(var n=0;n<t.length;n++)t[n].top=null;var s=o.height;for(n=0;n<t.length;n++){var r=t[n];if(r.stack&&null===r.top){r.top=r.baseTop;do{for(var a=null,h=0,d=t.length;h<d;h++){var l=t[h];if(null!==l.top&&l!==r&&e.collision(r,l,i.item,l.options.rtl)){a=l;break}}null!=a&&(r.top=a.top+a.height+i.item.vertical),r.top+r.height>s&&(s=r.top+r.height)}while(a)}}o.height=s-o.top+.5*i.item.vertical},e.nostack=function(t,i,o,n){for(var s=0;s<t.length;s++)if(void 0==t[s].data.subgroup)t[s].top=i.item.vertical;else if(void 0!==t[s].data.subgroup&&n){var r=0;for(var a in o)o.hasOwnProperty(a)&&1==o[a].visible&&o[a].index<o[t[s].data.subgroup].index&&(r+=o[a].height,o[t[s].data.subgroup].top=r);t[s].top=r+.5*i.item.vertical}n||e.stackSubgroups(t,i,o)},e.stackSubgroups=function(t,i,o){for(var n in o)if(o.hasOwnProperty(n)){o[n].top=0;do{var s=null;for(var r in o)if(null!==o[r].top&&r!==n&&o[n].index>o[r].index&&e.collisionByTimes(o[n],o[r])){s=o[r];break}null!=s&&(o[n].top=s.top+s.height)}while(s)}for(var a=0;a<t.length;a++)void 0!==t[a].data.subgroup&&(t[a].top=o[t[a].data.subgroup].top+.5*i.item.vertical)},e.stackSubgroupsWithInnerStack=function(t,i,o){var n=!1,s=[];for(var r in o)o[r].hasOwnProperty("index")?s[o[r].index]=r:s.push(r);for(var a=0;a<s.length;a++)if(r=s[a],o.hasOwnProperty(r)){n=n||o[r].stack,o[r].top=0;for(var h in o)o[h].visible&&o[r].index>o[h].index&&(o[r].top+=o[h].height);for(var d=t[r],l=0;l<d.length;l++)void 0!==d[l].data.subgroup&&(d[l].top=o[d[l].data.subgroup].top+.5*i.item.vertical,o[r].stack&&(d[l].baseTop=d[l].top));n&&o[r].stack&&e.substack(t[r],i,o[r])}},e.collision=function(t,e,i,o){return o?t.right-i.horizontal+.001<e.right+e.width&&t.right+t.width+i.horizontal-.001>e.right&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top:t.left-i.horizontal+.001<e.left+e.width&&t.left+t.width+i.horizontal-.001>e.left&&t.top-i.vertical+.001<e.top+e.height&&t.top+t.height+i.vertical-.001>e.top},e.collisionByTimes=function(t,e){return t.start<=e.start&&t.end>=e.start&&t.top<e.top+e.height&&t.top+t.height>e.top||e.start<=t.start&&e.end>=t.start&&e.top<t.top+t.height&&e.top+e.height>t.top}},function(t,e,i){function o(t,e,i){if(this.props={dot:{width:0,height:0},line:{width:0,height:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=this.options.align,i=this.width*t.getMillisecondsPerPixel();return"right"==e?this.data.start.getTime()>t.start&&this.data.start.getTime()-i<t.end:"left"==e?this.data.start.getTime()+i>t.start&&this.data.start.getTime()<t.end:this.data.start.getTime()+i/2>t.start&&this.data.start.getTime()-i/2<t.end},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("DIV"),this.dom.content=document.createElement("DIV"),this.dom.content.className="vis-item-content",this.dom.box.appendChild(this.dom.content),this.dom.line=document.createElement("DIV"),this.dom.line.className="vis-line",this.dom.dot=document.createElement("DIV"),this.dom.dot.className="vis-dot",this.dom.box["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.box)}if(!this.dom.line.parentNode){var e=this.parent.dom.background;if(!e)throw new Error("Cannot redraw item: parent has no background container element");e.appendChild(this.dom.line)}if(!this.dom.dot.parentNode){var i=this.parent.dom.axis;if(!e)throw new Error("Cannot redraw item: parent has no axis container element");i.appendChild(this.dom.dot)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.box),this._updateStyle(this.dom.box);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.box.className="vis-item vis-box"+e,this.dom.line.className="vis-item vis-line"+e,this.dom.dot.className="vis-item vis-dot"+e}},o.prototype._getDomComponentsSizes=function(){return{previous:{right:this.dom.box.style.right,left:this.dom.box.style.left},dot:{height:this.dom.dot.offsetHeight,width:this.dom.dot.offsetWidth},line:{width:this.dom.line.offsetWidth},box:{width:this.dom.box.offsetWidth,height:this.dom.box.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.options.rtl?this.dom.box.style.right="0px":this.dom.box.style.left="0px",this.props.dot.height=t.dot.height,this.props.dot.width=t.dot.width,this.props.line.width=t.line.width,this.width=t.box.width,this.height=t.box.height,this.options.rtl?this.dom.box.style.right=t.previous.right:this.dom.box.style.left=t.previous.left,this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.box),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.box)},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){if(this.displayed){var t=this.dom;t.box.parentNode&&t.box.parentNode.removeChild(t.box),t.line.parentNode&&t.line.parentNode.removeChild(t.line),t.dot.parentNode&&t.dot.parentNode.removeChild(t.dot),this.displayed=!1}},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start),e=this.options.align;"right"==e?this.options.rtl?(this.right=t-this.width,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t-this.width,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width+"px",this.dom.dot.style.left=t-this.props.line.width/2-this.props.dot.width/2+"px"):"left"==e?this.options.rtl?(this.right=t,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t+"px",this.dom.dot.style.right=t+this.props.line.width/2-this.props.dot.width/2+"px"):(this.left=t,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t+"px",this.dom.dot.style.left=t+this.props.line.width/2-this.props.dot.width/2+"px"):this.options.rtl?(this.right=t-this.width/2,this.dom.box.style.right=this.right+"px",this.dom.line.style.right=t-this.props.line.width+"px",this.dom.dot.style.right=t-this.props.dot.width/2+"px"):(this.left=t-this.width/2,this.dom.box.style.left=this.left+"px",this.dom.line.style.left=t-this.props.line.width/2+"px",this.dom.dot.style.left=t-this.props.dot.width/2+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.box,i=this.dom.line,o=this.dom.dot;if("top"==t)e.style.top=(this.top||0)+"px",i.style.top="0",i.style.height=this.parent.top+this.top+1+"px",i.style.bottom="";else{var n=this.parent.itemSet.props.height,s=n-this.parent.top-this.parent.height+this.top;e.style.top=(this.parent.height-this.top-this.height||0)+"px",i.style.top=n-s+"px",i.style.bottom="0"}o.style.top=-this.props.dot.height/2+"px"},o.prototype.getWidthLeft=function(){return this.width/2},o.prototype.getWidthRight=function(){return this.width/2},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={dot:{top:0,width:0,height:0},content:{height:0,marginLeft:0,marginRight:0}},this.options=i,t&&void 0==t.start)throw new Error('Property "start" missing in item '+t);n.call(this,t,e,i)}var n=i(22);o.prototype=new n(null,null,null),o.prototype.isVisible=function(t){var e=this.width*t.getMillisecondsPerPixel();return this.data.start.getTime()+e>t.start&&this.data.start<t.end},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.point=document.createElement("div"),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.point.appendChild(this.dom.content),this.dom.dot=document.createElement("div"),this.dom.point.appendChild(this.dom.dot),this.dom.point["timeline-item"]=this,this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.point.parentNode){var t=this.parent.dom.foreground;if(!t)throw new Error("Cannot redraw item: parent has no foreground container element");t.appendChild(this.dom.point)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.point),this._updateStyle(this.dom.point);var t=this.editable.updateTime||this.editable.updateGroup,e=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"")+(t?" vis-editable":" vis-readonly");this.dom.point.className="vis-item vis-point"+e,this.dom.dot.className="vis-item vis-dot"+e}},o.prototype._getDomComponentsSizes=function(){return{dot:{width:this.dom.dot.offsetWidth,height:this.dom.dot.offsetHeight},content:{width:this.dom.content.offsetWidth,height:this.dom.content.offsetHeight},point:{width:this.dom.point.offsetWidth,height:this.dom.point.offsetHeight}}},o.prototype._updateDomComponentsSizes=function(t){this.props.dot.width=t.dot.width,this.props.dot.height=t.dot.height,this.props.content.height=t.content.height,this.options.rtl?this.dom.content.style.marginRight=2*this.props.dot.width+"px":this.dom.content.style.marginLeft=2*this.props.dot.width+"px",this.width=t.point.width,this.height=t.point.height,this.dom.dot.style.top=(this.height-this.props.dot.height)/2+"px",this.options.rtl?this.dom.dot.style.right=this.props.dot.width/2+"px":this.dom.dot.style.left=this.props.dot.width/2+"px",this.dirty=!1},o.prototype._repaintDomAdditionals=function(){this._repaintOnItemUpdateTimeTooltip(this.dom.point),this._repaintDragCenter(),this._repaintDeleteButton(this.dom.point)},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=function(){this.displayed||this.redraw()},o.prototype.hide=function(){this.displayed&&(this.dom.point.parentNode&&this.dom.point.parentNode.removeChild(this.dom.point),this.displayed=!1)},o.prototype.repositionX=function(){var t=this.conversion.toScreen(this.data.start);this.options.rtl?(this.right=t-this.props.dot.width,this.dom.point.style.right=this.right+"px"):(this.left=t-this.props.dot.width,this.dom.point.style.left=this.left+"px")},o.prototype.repositionY=function(){var t=this.options.orientation.item,e=this.dom.point;e.style.top="top"==t?this.top+"px":this.parent.height-this.top-this.height+"px"},o.prototype.getWidthLeft=function(){return this.props.dot.width},o.prototype.getWidthRight=function(){return this.props.dot.width},t.exports=o},function(t,e,i){function o(t,e,i){if(this.props={content:{width:0}},this.overflow=!1,t){if(void 0==t.start)throw new Error('Property "start" missing in item '+t.id);if(void 0==t.end)throw new Error('Property "end" missing in item '+t.id)}n.call(this,t,e,i)}var n=i(22),s=i(52),r=i(53);o.prototype=new n(null,null,null),o.prototype.baseClassName="vis-item vis-background",o.prototype.stack=!1,o.prototype.isVisible=function(t){return this.data.start<t.end&&this.data.end>t.start},o.prototype._createDomElement=function(){this.dom||(this.dom={},this.dom.box=document.createElement("div"),this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-item-overflow",this.dom.box.appendChild(this.dom.frame),this.dom.content=document.createElement("div"),this.dom.content.className="vis-item-content",this.dom.frame.appendChild(this.dom.content),this.dirty=!0)},o.prototype._appendDomElement=function(){if(!this.parent)throw new Error("Cannot redraw item: no parent attached");if(!this.dom.box.parentNode){var t=this.parent.dom.background;if(!t)throw new Error("Cannot redraw item: parent has no background container element");t.appendChild(this.dom.box)}this.displayed=!0},o.prototype._updateDirtyDomComponents=function(){if(this.dirty){this._updateContents(this.dom.content),this._updateDataAttributes(this.dom.content),this._updateStyle(this.dom.box);var t=(this.data.className?" "+this.data.className:"")+(this.selected?" vis-selected":"");this.dom.box.className=this.baseClassName+t}},o.prototype._getDomComponentsSizes=function(){return this.overflow="hidden"!==window.getComputedStyle(this.dom.content).overflow,{content:{width:this.dom.content.offsetWidth}}},o.prototype._updateDomComponentsSizes=function(t){this.props.content.width=t.content.width,this.height=0,this.dirty=!1},o.prototype._repaintDomAdditionals=function(){},o.prototype.redraw=function(t){var e,i=[this._createDomElement.bind(this),this._appendDomElement.bind(this),this._updateDirtyDomComponents.bind(this),function(){this.dirty&&(e=this._getDomComponentsSizes.bind(this)())}.bind(this),function(){this.dirty&&this._updateDomComponentsSizes.bind(this)(e)}.bind(this),this._repaintDomAdditionals.bind(this)];if(t)return i;var o;return i.forEach(function(t){o=t()}),o},o.prototype.show=r.prototype.show,o.prototype.hide=r.prototype.hide,o.prototype.repositionX=r.prototype.repositionX,o.prototype.repositionY=function(t){var e,i=this.options.orientation.item;if(void 0!==this.data.subgroup){var o=this.data.subgroup;this.dom.box.style.height=this.parent.subgroups[o].height+"px",this.dom.box.style.top="top"==i?this.parent.top+this.parent.subgroups[o].top+"px":this.parent.top+this.parent.height-this.parent.subgroups[o].top-this.parent.subgroups[o].height+"px",this.dom.box.style.bottom=""}else this.parent instanceof s?(e=Math.max(this.parent.height,this.parent.itemSet.body.domProps.center.height,this.parent.itemSet.body.domProps.centerContainer.height),this.dom.box.style.bottom="bottom"==i?"0":"",this.dom.box.style.top="top"==i?"0":""):(e=this.parent.height,this.dom.box.style.top=this.parent.top+"px",this.dom.box.style.bottom="");this.dom.box.style.height=e+"px"},t.exports=o},function(t,e,i){Object.defineProperty(e,"__esModule",{value:!0});var o="string",n="boolean",s="number",r="object",a={configure:{enabled:{boolean:n},filter:{boolean:n,function:"function"},container:{dom:"dom"},__type__:{object:r,boolean:n,function:"function"}},align:{string:o},rtl:{boolean:n,undefined:"undefined"},rollingMode:{follow:{boolean:n},offset:{number:s,undefined:"undefined"},__type__:{object:r}},verticalScroll:{boolean:n,undefined:"undefined"},horizontalScroll:{boolean:n,undefined:"undefined"},autoResize:{boolean:n},throttleRedraw:{number:s},clickToUse:{boolean:n},dataAttributes:{string:o,array:"array"},editable:{add:{boolean:n,undefined:"undefined"},remove:{boolean:n,undefined:"undefined"},updateGroup:{boolean:n,undefined:"undefined"},updateTime:{boolean:n,undefined:"undefined"},overrideItems:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},end:{number:s,date:"date",string:o,moment:"moment"},format:{minorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},week:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r,function:"function"}},majorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},week:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r,function:"function"}},__type__:{object:r}},moment:{function:"function"},groupOrder:{string:o,function:"function"},groupEditable:{add:{boolean:n,undefined:"undefined"},remove:{boolean:n,undefined:"undefined"},order:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},groupOrderSwap:{function:"function"},height:{string:o,number:s},hiddenDates:{start:{date:"date",number:s,string:o,moment:"moment"},end:{date:"date",number:s,string:o,moment:"moment"},repeat:{string:o},__type__:{object:r,array:"array"}},itemsAlwaysDraggable:{item:{boolean:n,undefined:"undefined"},range:{boolean:n,undefined:"undefined"},__type__:{boolean:n,object:r}},limitSize:{boolean:n},locale:{string:o},locales:{__any__:{any:"any"},__type__:{object:r}},margin:{axis:{number:s},item:{horizontal:{number:s,undefined:"undefined"},vertical:{number:s,undefined:"undefined"},__type__:{object:r,number:s}},__type__:{object:r,number:s}},max:{date:"date",number:s,string:o,moment:"moment"},maxHeight:{number:s,string:o},maxMinorChars:{number:s},min:{date:"date",number:s,string:o,moment:"moment"},minHeight:{number:s,string:o},moveable:{boolean:n},multiselect:{boolean:n},multiselectPerGroup:{boolean:n},onAdd:{function:"function"},onDropObjectOnItem:{function:"function"},onUpdate:{function:"function"},onMove:{function:"function"},onMoving:{function:"function"},onRemove:{function:"function"},onAddGroup:{function:"function"},onMoveGroup:{function:"function"},onRemoveGroup:{function:"function"},onInitialDrawComplete:{function:"function"},order:{function:"function"},orientation:{axis:{string:o,undefined:"undefined"},item:{string:o,undefined:"undefined"},__type__:{string:o,object:r}},selectable:{boolean:n},showCurrentTime:{boolean:n},showMajorLabels:{boolean:n},showMinorLabels:{boolean:n},stack:{boolean:n},stackSubgroups:{boolean:n},snap:{function:"function",null:"null"},start:{date:"date",number:s,string:o,moment:"moment"},template:{function:"function"},groupTemplate:{function:"function"},visibleFrameTemplate:{string:o,function:"function"},showTooltips:{boolean:n},tooltip:{followMouse:{boolean:n},overflowMethod:{string:["cap","flip"]},__type__:{object:r}},tooltipOnItemUpdateTime:{template:{function:"function"},__type__:{boolean:n,object:r}},timeAxis:{scale:{string:o,undefined:"undefined"},step:{number:s,undefined:"undefined"},__type__:{object:r}},type:{string:o},width:{string:o,number:s},zoomable:{boolean:n},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:s},zoomMin:{number:s},__type__:{object:r}},h={global:{align:["center","left","right"],direction:!1,autoResize:!0,clickToUse:!1,editable:{add:!1,remove:!1,updateGroup:!1,updateTime:!1},end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",week:"w",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",week:"MMMM YYYY",month:"YYYY",year:""}},groupsDraggable:!1,height:"",locale:"",margin:{axis:[20,0,100,1],item:{horizontal:[10,0,100,1],vertical:[10,0,100,1]}},max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!1,multiselect:!1,multiselectPerGroup:!1,orientation:{axis:["both","bottom","top"],item:["bottom","top"]},selectable:!0,showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,stack:!0,stackSubgroups:!0,start:"",showTooltips:!0,tooltip:{followMouse:!1,overflowMethod:"flip"},tooltipOnItemUpdateTime:!1,type:["box","point","range","background"],width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1]}};e.allOptions=a,e.configureOptions=h},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(16),s=o(n),r=i(1),a=o(r),h=i(30),d=o(h),l=i(31),u=o(l),p=i(0),c=i(137).default,m=function(){function t(e,i,o){var n=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1;(0,d.default)(this,t),this.parent=e,this.changedOptions=[],this.container=i,this.allowCreation=!1,this.options={},this.initialized=!1,this.popupCounter=0,this.defaultOptions={enabled:!1,filter:!0,container:void 0,showButton:!0},p.extend(this.options,this.defaultOptions),this.configureOptions=o,this.moduleOptions={},this.domElements=[],this.popupDiv={},this.popupLimit=5,this.popupHistory={},this.colorPicker=new c(n),this.wrapper=void 0}return(0,u.default)(t,[{key:"setOptions",value:function(t){if(void 0!==t){this.popupHistory={},this._removePopup();var e=!0;"string"==typeof t?this.options.filter=t:t instanceof Array?this.options.filter=t.join():"object"===(void 0===t?"undefined":(0,a.default)(t))?(void 0!==t.container&&(this.options.container=t.container),void 0!==t.filter&&(this.options.filter=t.filter),void 0!==t.showButton&&(this.options.showButton=t.showButton),void 0!==t.enabled&&(e=t.enabled)):"boolean"==typeof t?(this.options.filter=!0,e=t):"function"==typeof t&&(this.options.filter=t,e=!0),!1===this.options.filter&&(e=!1),this.options.enabled=e}this._clean()}},{key:"setModuleOptions",value:function(t){this.moduleOptions=t,!0===this.options.enabled&&(this._clean(),void 0!==this.options.container&&(this.container=this.options.container),this._create())}},{key:"_create",value:function(){var t=this;this._clean(),this.changedOptions=[];var e=this.options.filter,i=0,o=!1;for(var n in this.configureOptions)this.configureOptions.hasOwnProperty(n)&&(this.allowCreation=!1,o=!1,"function"==typeof e?(o=e(n,[]),o=o||this._handleObject(this.configureOptions[n],[n],!0)):!0!==e&&-1===e.indexOf(n)||(o=!0),!1!==o&&(this.allowCreation=!0,i>0&&this._makeItem([]),this._makeHeader(n),this._handleObject(this.configureOptions[n],[n])),i++);if(!0===this.options.showButton){var s=document.createElement("div");s.className="vis-configuration vis-config-button",s.innerHTML="generate options",s.onclick=function(){t._printOptions()},s.onmouseover=function(){s.className="vis-configuration vis-config-button hover"},s.onmouseout=function(){s.className="vis-configuration vis-config-button"},this.optionsContainer=document.createElement("div"),this.optionsContainer.className="vis-configuration vis-config-option-container",this.domElements.push(this.optionsContainer),this.domElements.push(s)}this._push()}},{key:"_push",value:function(){this.wrapper=document.createElement("div"),this.wrapper.className="vis-configuration-wrapper",this.container.appendChild(this.wrapper);for(var t=0;t<this.domElements.length;t++)this.wrapper.appendChild(this.domElements[t]);this._showPopupIfNeeded()}},{key:"_clean",value:function(){for(var t=0;t<this.domElements.length;t++)this.wrapper.removeChild(this.domElements[t]);void 0!==this.wrapper&&(this.container.removeChild(this.wrapper),this.wrapper=void 0),this.domElements=[],this._removePopup()}},{key:"_getValue",value:function(t){for(var e=this.moduleOptions,i=0;i<t.length;i++){if(void 0===e[t[i]]){e=void 0;break}e=e[t[i]]}return e}},{key:"_makeItem",value:function(t){if(!0===this.allowCreation){var e=document.createElement("div");e.className="vis-configuration vis-config-item vis-config-s"+t.length;for(var i=arguments.length,o=Array(i>1?i-1:0),n=1;n<i;n++)o[n-1]=arguments[n];return o.forEach(function(t){e.appendChild(t)}),this.domElements.push(e),this.domElements.length}return 0}},{key:"_makeHeader",value:function(t){var e=document.createElement("div");e.className="vis-configuration vis-config-header",e.innerHTML=t,this._makeItem([],e)}},{key:"_makeLabel",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],o=document.createElement("div");return o.className="vis-configuration vis-config-label vis-config-s"+e.length,o.innerHTML=!0===i?"<i><b>"+t+":</b></i>":t+":",o}},{key:"_makeDropdown",value:function(t,e,i){var o=document.createElement("select");o.className="vis-configuration vis-config-select";var n=0;void 0!==e&&-1!==t.indexOf(e)&&(n=t.indexOf(e));for(var s=0;s<t.length;s++){var r=document.createElement("option");r.value=t[s],s===n&&(r.selected="selected"),r.innerHTML=t[s],o.appendChild(r)}var a=this;o.onchange=function(){a._update(this.value,i)};var h=this._makeLabel(i[i.length-1],i);this._makeItem(i,h,o)}},{key:"_makeRange",value:function(t,e,i){var o=t[0],n=t[1],s=t[2],r=t[3],a=document.createElement("input");a.className="vis-configuration vis-config-range";try{a.type="range",a.min=n,a.max=s}catch(t){}a.step=r;var h="",d=0;if(void 0!==e){e<0&&1.2*e<n?(a.min=Math.ceil(1.2*e),d=a.min,h="range increased"):e/1.2<n&&(a.min=Math.ceil(e/1.2),d=a.min,h="range increased"),1.2*e>s&&1!==s&&(a.max=Math.ceil(1.2*e),d=a.max,h="range increased"),a.value=e}else a.value=o;var l=document.createElement("input");l.className="vis-configuration vis-config-rangeinput",l.value=a.value;var u=this;a.onchange=function(){l.value=this.value,u._update(Number(this.value),i)},a.oninput=function(){l.value=this.value};var p=this._makeLabel(i[i.length-1],i),c=this._makeItem(i,p,a,l);""!==h&&this.popupHistory[c]!==d&&(this.popupHistory[c]=d,this._setupPopup(h,c))}},{key:"_setupPopup",value:function(t,e){var i=this;if(!0===this.initialized&&!0===this.allowCreation&&this.popupCounter<this.popupLimit){var o=document.createElement("div");o.id="vis-configuration-popup",o.className="vis-configuration-popup",o.innerHTML=t,o.onclick=function(){i._removePopup()},this.popupCounter+=1,this.popupDiv={html:o,index:e}}}},{key:"_removePopup",value:function(){void 0!==this.popupDiv.html&&(this.popupDiv.html.parentNode.removeChild(this.popupDiv.html),clearTimeout(this.popupDiv.hideTimeout),clearTimeout(this.popupDiv.deleteTimeout),this.popupDiv={})}},{key:"_showPopupIfNeeded",value:function(){var t=this;if(void 0!==this.popupDiv.html){var e=this.domElements[this.popupDiv.index],i=e.getBoundingClientRect();this.popupDiv.html.style.left=i.left+"px",this.popupDiv.html.style.top=i.top-30+"px",document.body.appendChild(this.popupDiv.html),this.popupDiv.hideTimeout=setTimeout(function(){t.popupDiv.html.style.opacity=0},1500),this.popupDiv.deleteTimeout=setTimeout(function(){t._removePopup()},1800)}}},{key:"_makeCheckbox",value:function(t,e,i){var o=document.createElement("input");o.type="checkbox",o.className="vis-configuration vis-config-checkbox",o.checked=t,void 0!==e&&(o.checked=e,e!==t&&("object"===(void 0===t?"undefined":(0,a.default)(t))?e!==t.enabled&&this.changedOptions.push({path:i,value:e}):this.changedOptions.push({path:i,value:e})));var n=this;o.onchange=function(){n._update(this.checked,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeTextInput",value:function(t,e,i){var o=document.createElement("input");o.type="text",o.className="vis-configuration vis-config-text",o.value=e,e!==t&&this.changedOptions.push({path:i,value:e});var n=this;o.onchange=function(){n._update(this.value,i)};var s=this._makeLabel(i[i.length-1],i);this._makeItem(i,s,o)}},{key:"_makeColorField",value:function(t,e,i){var o=this,n=t[1],s=document.createElement("div");e=void 0===e?n:e,"none"!==e?(s.className="vis-configuration vis-config-colorBlock",s.style.backgroundColor=e):s.className="vis-configuration vis-config-colorBlock none",e=void 0===e?n:e,s.onclick=function(){o._showColorPicker(e,s,i)};var r=this._makeLabel(i[i.length-1],i);this._makeItem(i,r,s)}},{key:"_showColorPicker",value:function(t,e,i){var o=this;e.onclick=function(){},this.colorPicker.insertTo(e),this.colorPicker.show(),this.colorPicker.setColor(t),this.colorPicker.setUpdateCallback(function(t){var n="rgba("+t.r+","+t.g+","+t.b+","+t.a+")";e.style.backgroundColor=n,o._update(n,i)}),this.colorPicker.setCloseCallback(function(){e.onclick=function(){o._showColorPicker(t,e,i)}})}},{key:"_handleObject",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],i=arguments.length>2&&void 0!==arguments[2]&&arguments[2],o=!1,n=this.options.filter,s=!1;for(var r in t)if(t.hasOwnProperty(r)){o=!0;var a=t[r],h=p.copyAndExtendArray(e,r);if("function"==typeof n&&!1===(o=n(r,e))&&!(a instanceof Array)&&"string"!=typeof a&&"boolean"!=typeof a&&a instanceof Object&&(this.allowCreation=!1,o=this._handleObject(a,h,!0),this.allowCreation=!1===i),!1!==o){s=!0;var d=this._getValue(h);if(a instanceof Array)this._handleArray(a,d,h);else if("string"==typeof a)this._makeTextInput(a,d,h);else if("boolean"==typeof a)this._makeCheckbox(a,d,h);else if(a instanceof Object){var l=!0;if(-1!==e.indexOf("physics")&&this.moduleOptions.physics.solver!==r&&(l=!1),!0===l)if(void 0!==a.enabled){var u=p.copyAndExtendArray(h,"enabled"),c=this._getValue(u);if(!0===c){var m=this._makeLabel(r,h,!0);this._makeItem(h,m),s=this._handleObject(a,h)||s}else this._makeCheckbox(a,c,h)}else{var f=this._makeLabel(r,h,!0);this._makeItem(h,f),s=this._handleObject(a,h)||s}}else console.error("dont know how to handle",a,r,h)}}return s}},{key:"_handleArray",value:function(t,e,i){"string"==typeof t[0]&&"color"===t[0]?(this._makeColorField(t,e,i),t[1]!==e&&this.changedOptions.push({path:i,value:e})):"string"==typeof t[0]?(this._makeDropdown(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:e})):"number"==typeof t[0]&&(this._makeRange(t,e,i),t[0]!==e&&this.changedOptions.push({path:i,value:Number(e)}))}},{key:"_update",value:function(t,e){var i=this._constructOptions(t,e);this.parent.body&&this.parent.body.emitter&&this.parent.body.emitter.emit&&this.parent.body.emitter.emit("configChange",i),this.initialized=!0,this.parent.setOptions(i)}},{key:"_constructOptions",value:function(t,e){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},o=i;t="true"===t||t,t="false"!==t&&t;for(var n=0;n<e.length;n++)"global"!==e[n]&&(void 0===o[e[n]]&&(o[e[n]]={}),n!==e.length-1?o=o[e[n]]:o[e[n]]=t);return i}},{key:"_printOptions",value:function(){var t=this.getOptions();this.optionsContainer.innerHTML="<pre>var options = "+(0,s.default)(t,null,2)+"</pre>"}},{key:"getOptions",value:function(){for(var t={},e=0;e<this.changedOptions.length;e++)this._constructOptions(this.changedOptions[e].value,this.changedOptions[e].path,t);return t}}]),t}();e.default=m},function(t,e,i){function o(t,e){this.id=r.randomUUID(),this.body=t,this.defaultOptions={yAxisOrientation:"left",defaultGroup:"default",sort:!0,sampling:!0,stack:!1,graphHeight:"400px",shaded:{enabled:!1,orientation:"bottom"},style:"line",barChart:{width:50,sideBySide:!1,align:"center"},interpolation:{enabled:!0,parametrization:"centripetal",alpha:.5},drawPoints:{enabled:!0,size:6,style:"square"},dataAxis:{},legend:{},groups:{visibility:{}}},this.options=r.extend({},this.defaultOptions),this.dom={},this.props={},this.hammer=null,this.groups={},this.abortedGraphUpdate=!1,this.updateSVGheight=!1,this.updateSVGheightOnResize=!1,this.forceGraphUpdate=!0;var i=this;this.itemsData=null,this.groupsData=null,this.itemListeners={add:function(t,e,o){i._onAdd(e.items)},update:function(t,e,o){i._onUpdate(e.items)},remove:function(t,e,o){i._onRemove(e.items)}},this.groupListeners={add:function(t,e,o){i._onAddGroups(e.items)},update:function(t,e,o){i._onUpdateGroups(e.items)},remove:function(t,e,o){i._onRemoveGroups(e.items)}},this.items={},this.selection=[],this.lastStart=this.body.range.start,this.touchParams={},this.svgElements={},this.setOptions(e),this.groupsUsingDefaultStyles=[0],this.body.emitter.on("rangechanged",function(){i.lastStart=i.body.range.start,i.svg.style.left=r.option.asSize(-i.props.width),i.forceGraphUpdate=!0,i.redraw.call(i)}),this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups}}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(15),d=i(20),l=i(5),u=i(78),p=i(80),c=i(83),m=i(81),f=i(82),g=i(54);o.prototype=new l,o.prototype._create=function(){var t=document.createElement("div");t.className="vis-line-graph",this.dom.frame=t,this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="relative",this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",this.svg.style.display="block",t.appendChild(this.svg),this.options.dataAxis.orientation="left",this.yAxisLeft=new u(this.body,this.options.dataAxis,this.svg,this.options.groups),this.options.dataAxis.orientation="right",this.yAxisRight=new u(this.body,this.options.dataAxis,this.svg,this.options.groups),delete this.options.dataAxis.orientation,this.legendLeft=new c(this.body,this.options.legend,"left",this.options.groups),this.legendRight=new c(this.body,this.options.legend,"right",this.options.groups),this.show()},o.prototype.setOptions=function(t){if(t){var e=["sampling","defaultGroup","stack","height","graphHeight","yAxisOrientation","style","barChart","dataAxis","sort","groups"];void 0===t.graphHeight&&void 0!==t.height?(this.updateSVGheight=!0,this.updateSVGheightOnResize=!0):void 0!==this.body.domProps.centerContainer.height&&void 0!==t.graphHeight&&parseInt((t.graphHeight+"").replace("px",""))<this.body.domProps.centerContainer.height&&(this.updateSVGheight=!0),r.selectiveDeepExtend(e,this.options,t),r.mergeOptions(this.options,t,"interpolation"),r.mergeOptions(this.options,t,"drawPoints"),r.mergeOptions(this.options,t,"shaded"),r.mergeOptions(this.options,t,"legend"),t.interpolation&&"object"==(0,s.default)(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5)),this.yAxisLeft&&void 0!==t.dataAxis&&(this.yAxisLeft.setOptions(this.options.dataAxis),this.yAxisRight.setOptions(this.options.dataAxis)),this.legendLeft&&void 0!==t.legend&&(this.legendLeft.setOptions(this.options.legend),this.legendRight.setOptions(this.options.legend)),this.groups.hasOwnProperty("__ungrouped__")&&this.groups.__ungrouped__.setOptions(t)}this.dom.frame&&(this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0}))},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setItems=function(t){var e,i=this,o=this.itemsData;if(t){if(!(t instanceof h||t instanceof d))throw new TypeError("Data must be an instance of DataSet or DataView");this.itemsData=t}else this.itemsData=null;if(o&&(r.forEach(this.itemListeners,function(t,e){o.off(e,t)}),e=o.getIds(),this._onRemove(e)),this.itemsData){var n=this.id;r.forEach(this.itemListeners,function(t,e){i.itemsData.on(e,t,n)}),e=this.itemsData.getIds(),this._onAdd(e)}},o.prototype.setGroups=function(t){var e,i=this;if(this.groupsData){r.forEach(this.groupListeners,function(t,e){i.groupsData.off(e,t)}),e=this.groupsData.getIds(),this.groupsData=null;for(var o=0;o<e.length;o++)this._removeGroup(e[o])}if(t){if(!(t instanceof h||t instanceof d))throw new TypeError("Data must be an instance of DataSet or DataView");this.groupsData=t}else this.groupsData=null;if(this.groupsData){var n=this.id;r.forEach(this.groupListeners,function(t,e){i.groupsData.on(e,t,n)}),e=this.groupsData.getIds(),this._onAddGroups(e)}},o.prototype._onUpdate=function(t){this._updateAllGroupData(t)},o.prototype._onAdd=function(t){this._onUpdate(t)},o.prototype._onRemove=function(t){this._onUpdate(t)},o.prototype._onUpdateGroups=function(t){this._updateAllGroupData(null,t)},o.prototype._onAddGroups=function(t){this._onUpdateGroups(t)},o.prototype._onRemoveGroups=function(t){for(var e=0;e<t.length;e++)this._removeGroup(t[e]);this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})},o.prototype._removeGroup=function(t){this.groups.hasOwnProperty(t)&&("right"==this.groups[t].options.yAxisOrientation?(this.yAxisRight.removeGroup(t),this.legendRight.removeGroup(t),this.legendRight.redraw()):(this.yAxisLeft.removeGroup(t),this.legendLeft.removeGroup(t),this.legendLeft.redraw()),delete this.groups[t])},o.prototype._updateGroup=function(t,e){this.groups.hasOwnProperty(e)?(this.groups[e].update(t),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.updateGroup(e,this.groups[e]),this.legendRight.updateGroup(e,this.groups[e]),this.yAxisLeft.removeGroup(e),this.legendLeft.removeGroup(e)):(this.yAxisLeft.updateGroup(e,this.groups[e]),this.legendLeft.updateGroup(e,this.groups[e]),this.yAxisRight.removeGroup(e),this.legendRight.removeGroup(e))):(this.groups[e]=new p(t,e,this.options,this.groupsUsingDefaultStyles),"right"==this.groups[e].options.yAxisOrientation?(this.yAxisRight.addGroup(e,this.groups[e]),this.legendRight.addGroup(e,this.groups[e])):(this.yAxisLeft.addGroup(e,this.groups[e]),this.legendLeft.addGroup(e,this.groups[e]))),this.legendLeft.redraw(),this.legendRight.redraw()},o.prototype._updateAllGroupData=function(t,e){if(null!=this.itemsData){var i={},o=this.itemsData.get(),n=this.itemsData._fieldId,s={};t&&t.map(function(t){s[t]=t});for(var a={},h=0;h<o.length;h++){var d=o[h],l=d.group;null!==l&&void 0!==l||(l="__ungrouped__"),a.hasOwnProperty(l)?a[l]++:a[l]=1}var u={};if(!e&&t)for(l in this.groups)if(this.groups.hasOwnProperty(l)){g=this.groups[l];var p=g.getItems();i[l]=p.filter(function(t){return u[t[n]]=t[n],t[n]!==s[t[n]]});var c=a[l];a[l]-=i[l].length,i[l].length<c&&(i[l][c-1]={})}for(h=0;h<o.length;h++)if(d=o[h],l=d.group,null!==l&&void 0!==l||(l="__ungrouped__"),e||!t||d[n]===s[d[n]]||!u.hasOwnProperty(d[n])){i.hasOwnProperty(l)||(i[l]=new Array(a[l]));var m=r.bridgeObject(d);m.x=r.convert(d.x,"Date"),m.end=r.convert(d.end,"Date"),m.orginalY=d.y,m.y=Number(d.y),m[n]=d[n];var f=i[l].length-a[l]--;i[l][f]=m}for(l in this.groups)this.groups.hasOwnProperty(l)&&(i.hasOwnProperty(l)||(i[l]=new Array(0)));for(l in i)if(i.hasOwnProperty(l))if(0==i[l].length)this.groups.hasOwnProperty(l)&&this._removeGroup(l);else{var g=void 0;void 0!=this.groupsData&&(g=this.groupsData.get(l)),void 0==g&&(g={id:l,content:this.options.defaultGroup+l}),this._updateGroup(g,l),this.groups[l].setItems(i[l])}this.forceGraphUpdate=!0,this.body.emitter.emit("_change",{queue:!0})}},o.prototype.redraw=function(){var t=!1;this.props.width=this.dom.frame.offsetWidth,this.props.height=this.body.domProps.centerContainer.height-this.body.domProps.border.top-this.body.domProps.border.bottom,t=this._isResized()||t;var e=this.body.range.end-this.body.range.start,i=e!=this.lastVisibleInterval;if(this.lastVisibleInterval=e,1==t&&(this.svg.style.width=r.option.asSize(3*this.props.width),this.svg.style.left=r.option.asSize(-this.props.width),-1==(this.options.height+"").indexOf("%")&&1!=this.updateSVGheightOnResize||(this.updateSVGheight=!0)),1==this.updateSVGheight?(this.options.graphHeight!=this.props.height+"px"&&(this.options.graphHeight=this.props.height+"px",this.svg.style.height=this.props.height+"px"),this.updateSVGheight=!1):this.svg.style.height=(""+this.options.graphHeight).replace("px","")+"px",1==t||1==i||1==this.abortedGraphUpdate||1==this.forceGraphUpdate)t=this._updateGraph()||t,this.forceGraphUpdate=!1;else if(0!=this.lastStart){var o=this.body.range.start-this.lastStart,n=this.body.range.end-this.body.range.start;if(0!=this.props.width){var s=this.props.width/n,a=o*s;this.svg.style.left=-this.props.width-a+"px"}}return this.legendLeft.redraw(),this.legendRight.redraw(),t},o.prototype._getSortedGroupIds=function(){var t=[];for(var e in this.groups)if(this.groups.hasOwnProperty(e)){var i=this.groups[e];1!=i.visible||void 0!==this.options.groups.visibility[e]&&1!=this.options.groups.visibility[e]||t.push({id:e,zIndex:i.options.zIndex})}r.insertSort(t,function(t,e){var i=t.zIndex,o=e.zIndex;return void 0===i&&(i=0),void 0===o&&(o=0),i==o?0:i<o?-1:1});for(var o=new Array(t.length),n=0;n<t.length;n++)o[n]=t[n].id;return o},o.prototype._updateGraph=function(){if(a.prepareElements(this.svgElements),0!=this.props.width&&null!=this.itemsData){var t,e,i={},o=this.body.util.toGlobalTime(-this.body.domProps.root.width),n=this.body.util.toGlobalTime(2*this.body.domProps.root.width),s=this._getSortedGroupIds();if(s.length>0){var r={};for(this._getRelevantData(s,r,o,n),this._applySampling(s,r),e=0;e<s.length;e++)this._convertXcoordinates(r[s[e]]);if(this._getYRanges(s,r,i),1==this._updateYAxis(s,i))return a.cleanupElements(this.svgElements),this.abortedGraphUpdate=!0,!0;this.abortedGraphUpdate=!1;var h=void 0;for(e=0;e<s.length;e++)t=this.groups[s[e]],!0===this.options.stack&&"line"===this.options.style&&(void 0!=t.options.excludeFromStacking&&t.options.excludeFromStacking||(void 0!=h&&(this._stack(r[t.id],r[h.id]),1==t.options.shaded.enabled&&"group"!==t.options.shaded.orientation&&("top"==t.options.shaded.orientation&&"group"!==h.options.shaded.orientation?(h.options.shaded.orientation="group",h.options.shaded.groupId=t.id):(t.options.shaded.orientation="group",t.options.shaded.groupId=h.id))),h=t)),this._convertYcoordinates(r[s[e]],t);var d={};for(e=0;e<s.length;e++)if(t=this.groups[s[e]],"line"===t.options.style&&1==t.options.shaded.enabled){var l=r[s[e]];if(null==l||0==l.length)continue;if(d.hasOwnProperty(s[e])||(d[s[e]]=f.calcPath(l,t)),"group"===t.options.shaded.orientation){var u=t.options.shaded.groupId;if(-1===s.indexOf(u)){console.log(t.id+": Unknown shading group target given:"+u);continue}d.hasOwnProperty(u)||(d[u]=f.calcPath(r[u],this.groups[u])),f.drawShading(d[s[e]],t,d[u],this.framework)}else f.drawShading(d[s[e]],t,void 0,this.framework)}for(m.draw(s,r,this.framework),e=0;e<s.length;e++)if(t=this.groups[s[e]],r[s[e]].length>0)switch(t.options.style){case"line":d.hasOwnProperty(s[e])||(d[s[e]]=f.calcPath(r[s[e]],t)),f.draw(d[s[e]],t,this.framework);case"point":case"points":"point"!=t.options.style&&"points"!=t.options.style&&1!=t.options.drawPoints.enabled||g.draw(r[s[e]],t,this.framework)}}}return a.cleanupElements(this.svgElements),!1},o.prototype._stack=function(t,e){var i,o,n,s,r;i=0;for(var a=0;a<t.length;a++){s=void 0,r=void 0;for(var h=i;h<e.length;h++){if(e[h].x===t[a].x){s=e[h],r=e[h],i=h;break}if(e[h].x>t[a].x){r=e[h],s=0==h?r:e[h-1],i=h;break}}void 0===r&&(s=e[e.length-1],r=e[e.length-1]),o=r.x-s.x,n=r.y-s.y,t[a].y=0==o?t[a].orginalY+r.y:t[a].orginalY+n/o*(t[a].x-s.x)+s.y}},o.prototype._getRelevantData=function(t,e,i,o){var n,s,a,h;if(t.length>0)for(s=0;s<t.length;s++){n=this.groups[t[s]];var d=n.getItems();if(1==n.options.sort){var l=function(t,e){return t.getTime()==e.getTime()?0:t<e?-1:1},u=Math.max(0,r.binarySearchValue(d,i,"x","before",l)),p=Math.min(d.length,r.binarySearchValue(d,o,"x","after",l)+1);p<=0&&(p=d.length);var c=new Array(p-u);for(a=u;a<p;a++)h=n.itemsData[a],c[a-u]=h;e[t[s]]=c}else e[t[s]]=n.itemsData}},o.prototype._applySampling=function(t,e){var i;if(t.length>0)for(var o=0;o<t.length;o++)if(i=this.groups[t[o]],1==i.options.sampling){var n=e[t[o]];if(n.length>0){var s=1,r=n.length,a=this.body.util.toGlobalScreen(n[n.length-1].x)-this.body.util.toGlobalScreen(n[0].x),h=r/a;s=Math.min(Math.ceil(.2*r),Math.max(1,Math.round(h)));for(var d=new Array(r),l=0;l<r;l+=s){var u=Math.round(l/s);d[u]=n[l]}e[t[o]]=d.splice(0,Math.round(r/s))}}},o.prototype._getYRanges=function(t,e,i){var o,n,s,r,a=[],h=[];if(t.length>0){for(s=0;s<t.length;s++)o=e[t[s]],r=this.groups[t[s]].options,o.length>0&&(n=this.groups[t[s]],!0===r.stack&&"bar"===r.style?"left"===r.yAxisOrientation?a=a.concat(o):h=h.concat(o):i[t[s]]=n.getYRange(o,t[s]));m.getStackedYRange(a,i,t,"__barStackLeft","left"),m.getStackedYRange(h,i,t,"__barStackRight","right")}},o.prototype._updateYAxis=function(t,e){var i,o,n=!1,s=!1,r=!1,a=1e9,h=1e9,d=-1e9,l=-1e9;if(t.length>0){for(var u=0;u<t.length;u++){var p=this.groups[t[u]];p&&"right"!=p.options.yAxisOrientation?(s=!0,a=1e9,d=-1e9):p&&p.options.yAxisOrientation&&(r=!0,h=1e9,l=-1e9)}for(u=0;u<t.length;u++)e.hasOwnProperty(t[u])&&!0!==e[t[u]].ignore&&(i=e[t[u]].min,o=e[t[u]].max,"right"!=e[t[u]].yAxisOrientation?(s=!0,a=a>i?i:a,d=d<o?o:d):(r=!0,h=h>i?i:h,l=l<o?o:l));1==s&&this.yAxisLeft.setRange(a,d),1==r&&this.yAxisRight.setRange(h,l)}n=this._toggleAxisVisiblity(s,this.yAxisLeft)||n,n=this._toggleAxisVisiblity(r,this.yAxisRight)||n,1==r&&1==s?(this.yAxisLeft.drawIcons=!0,this.yAxisRight.drawIcons=!0):(this.yAxisLeft.drawIcons=!1,this.yAxisRight.drawIcons=!1),this.yAxisRight.master=!s,this.yAxisRight.masterAxis=this.yAxisLeft,0==this.yAxisRight.master?(this.yAxisLeft.lineOffset=1==r?this.yAxisRight.width:0,n=this.yAxisLeft.redraw()||n,n=this.yAxisRight.redraw()||n):n=this.yAxisRight.redraw()||n;var c=["__barStackLeft","__barStackRight","__lineStackLeft","__lineStackRight"];for(u=0;u<c.length;u++)-1!=t.indexOf(c[u])&&t.splice(t.indexOf(c[u]),1);return n},o.prototype._toggleAxisVisiblity=function(t,e){var i=!1;return 0==t?e.dom.frame.parentNode&&0==e.hidden&&(e.hide(),i=!0):e.dom.frame.parentNode||1!=e.hidden||(e.show(),i=!0),i},o.prototype._convertXcoordinates=function(t){for(var e=this.body.util.toScreen,i=0;i<t.length;i++)t[i].screen_x=e(t[i].x)+this.props.width,t[i].screen_y=t[i].y,void 0!=t[i].end?t[i].screen_end=e(t[i].end)+this.props.width:t[i].screen_end=void 0},o.prototype._convertYcoordinates=function(t,e){var i=this.yAxisLeft,o=Number(this.svg.style.height.replace("px",""));"right"==e.options.yAxisOrientation&&(i=this.yAxisRight);for(var n=0;n<t.length;n++)t[n].screen_y=Math.round(i.convertValue(t[n].y));e.setZeroPosition(Math.min(o,i.convertValue(0)))},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=r.randomUUID(),this.body=t,this.defaultOptions={orientation:"left",showMinorLabels:!0,showMajorLabels:!0,icons:!1,majorLinesOffset:7,minorLinesOffset:4,labelOffsetX:10,labelOffsetY:2,iconWidth:20,width:"40px",visible:!0,alignZeros:!0,left:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}},right:{range:{min:void 0,max:void 0},format:function(t){return""+parseFloat(t.toPrecision(3))},title:{text:void 0,style:void 0}}},this.linegraphOptions=o,this.linegraphSVG=i,this.props={},this.DOMelements={lines:{},labels:{},title:{}},this.dom={},this.scale=void 0,this.range={start:0,end:0},this.options=r.extend({},this.defaultOptions),this.conversionFactor=1,this.setOptions(e),this.width=Number((""+this.options.width).replace("px","")),this.minWidth=this.width,this.height=this.linegraphSVG.getBoundingClientRect().height,this.hidden=!1,this.stepPixels=25,this.zeroCrossing=-1,this.amountOfSteps=-1,this.lineOffset=0,this.master=!0,this.masterAxis=null,this.svgElements={},this.iconsRemoved=!1,this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups};var n=this;this.body.emitter.on("verticalDrag",function(){n.dom.lineContainer.style.top=n.body.domProps.scrollTop+"px"})}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(5),d=i(79);o.prototype=new h,o.prototype.addGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1},o.prototype.updateGroup=function(t,e){this.groups.hasOwnProperty(t)||(this.amountOfGroups+=1),this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype.setOptions=function(t){if(t){var e=!1;this.options.orientation!=t.orientation&&void 0!==t.orientation&&(e=!0);var i=["orientation","showMinorLabels","showMajorLabels","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","width","visible","left","right","alignZeros"];r.selectiveDeepExtend(i,this.options,t),this.minWidth=Number((""+this.options.width).replace("px","")),!0===e&&this.dom.frame&&(this.hide(),this.show())}},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.style.width=this.options.width,this.dom.frame.style.height=this.height,this.dom.lineContainer=document.createElement("div"),this.dom.lineContainer.style.width="100%",this.dom.lineContainer.style.height=this.height,this.dom.lineContainer.style.position="relative",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.height="100%",this.svg.style.width="100%",this.svg.style.display="block",this.dom.frame.appendChild(this.svg)},o.prototype._redrawGroupIcons=function(){a.prepareElements(this.svgElements);var t,e=this.options.iconWidth,i=11.5;t="left"===this.options.orientation?4:this.width-e-4;var o=(0,s.default)(this.groups);o.sort(function(t,e){return t<e?-1:1});for(var n=0;n<o.length;n++){var r=o[n];!0!==this.groups[r].visible||void 0!==this.linegraphOptions.visibility[r]&&!0!==this.linegraphOptions.visibility[r]||(this.groups[r].getLegend(e,15,this.framework,t,i),i+=19)}a.cleanupElements(this.svgElements),this.iconsRemoved=!1},o.prototype._cleanupIcons=function(){!1===this.iconsRemoved&&(a.prepareElements(this.svgElements),a.cleanupElements(this.svgElements),this.iconsRemoved=!0)},o.prototype.show=function(){this.hidden=!1,this.dom.frame.parentNode||("left"===this.options.orientation?this.body.dom.left.appendChild(this.dom.frame):this.body.dom.right.appendChild(this.dom.frame)),this.dom.lineContainer.parentNode||this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer)},o.prototype.hide=function(){this.hidden=!0,this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame),this.dom.lineContainer.parentNode&&this.dom.lineContainer.parentNode.removeChild(this.dom.lineContainer)},o.prototype.setRange=function(t,e){this.range.start=t,this.range.end=e},o.prototype.redraw=function(){var t=!1,e=0;this.dom.lineContainer.style.top=this.body.domProps.scrollTop+"px";for(var i in this.groups)this.groups.hasOwnProperty(i)&&(!0!==this.groups[i].visible||void 0!==this.linegraphOptions.visibility[i]&&!0!==this.linegraphOptions.visibility[i]||e++);if(0===this.amountOfGroups||0===e)this.hide();else{this.show(),this.height=Number(this.linegraphSVG.style.height.replace("px","")),this.dom.lineContainer.style.height=this.height+"px",this.width=!0===this.options.visible?Number((""+this.options.width).replace("px","")):0;var o=this.props,n=this.dom.frame;n.className="vis-data-axis",this._calculateCharSize();var s=this.options.orientation,r=this.options.showMinorLabels,a=this.options.showMajorLabels;o.minorLabelHeight=r?o.minorCharHeight:0,o.majorLabelHeight=a?o.majorCharHeight:0,o.minorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.minorLinesOffset,o.minorLineHeight=1,o.majorLineWidth=this.body.dom.backgroundHorizontal.offsetWidth-this.lineOffset-this.width+2*this.options.majorLinesOffset,o.majorLineHeight=1,"left"===s?(n.style.top="0",n.style.left="0",n.style.bottom="",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.left.width,this.props.height=this.body.domProps.left.height):(n.style.top="",n.style.bottom="0",n.style.left="0",n.style.width=this.width+"px",n.style.height=this.height+"px",this.props.width=this.body.domProps.right.width,this.props.height=this.body.domProps.right.height),t=this._redrawLabels(),t=this._isResized()||t,!0===this.options.icons?this._redrawGroupIcons():this._cleanupIcons(),this._redrawTitle(s)}return t},o.prototype._redrawLabels=function(){var t=this,e=!1;a.prepareElements(this.DOMelements.lines),a.prepareElements(this.DOMelements.labels);var i=this.options.orientation,o=void 0!=this.options[i].range?this.options[i].range:{},n=!0;void 0!=o.max&&(this.range.end=o.max,n=!1);var s=!0;void 0!=o.min&&(this.range.start=o.min,s=!1),this.scale=new d(this.range.start,this.range.end,s,n,this.dom.frame.offsetHeight,this.props.majorCharHeight,this.options.alignZeros,this.options[i].format),!1===this.master&&void 0!=this.masterAxis&&this.scale.followScale(this.masterAxis.scale),this.maxLabelSize=0,this.scale.getLines().forEach(function(e){var o=e.y,n=e.major;t.options.showMinorLabels&&!1===n&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-minor",t.props.minorCharHeight),n&&o>=0&&t._redrawLabel(o-2,e.val,i,"vis-y-axis vis-major",t.props.majorCharHeight),!0===t.master&&(n?t._redrawLine(o,i,"vis-grid vis-horizontal vis-major",t.options.majorLinesOffset,t.props.majorLineWidth):t._redrawLine(o,i,"vis-grid vis-horizontal vis-minor",t.options.minorLinesOffset,t.props.minorLineWidth))});var r=0;void 0!==this.options[i].title&&void 0!==this.options[i].title.text&&(r=this.props.titleCharHeight);var h=!0===this.options.icons?Math.max(this.options.iconWidth,r)+this.options.labelOffsetX+15:r+this.options.labelOffsetX+15;return this.maxLabelSize>this.width-h&&!0===this.options.visible?(this.width=this.maxLabelSize+h,this.options.width=this.width+"px",a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):this.maxLabelSize<this.width-h&&!0===this.options.visible&&this.width>this.minWidth?(this.width=Math.max(this.minWidth,this.maxLabelSize+h),this.options.width=this.width+"px",a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),this.redraw(),e=!0):(a.cleanupElements(this.DOMelements.lines),a.cleanupElements(this.DOMelements.labels),e=!1),e},o.prototype.convertValue=function(t){return this.scale.convertValue(t)},o.prototype.screenToValue=function(t){return this.scale.screenToValue(t)},o.prototype._redrawLabel=function(t,e,i,o,n){var s=a.getDOMElement("div",this.DOMelements.labels,this.dom.frame);s.className=o,s.innerHTML=e,"left"===i?(s.style.left="-"+this.options.labelOffsetX+"px",s.style.textAlign="right"):(s.style.right="-"+this.options.labelOffsetX+"px",s.style.textAlign="left"),s.style.top=t-.5*n+this.options.labelOffsetY+"px",e+="";var r=Math.max(this.props.majorCharWidth,this.props.minorCharWidth);this.maxLabelSize<e.length*r&&(this.maxLabelSize=e.length*r)},o.prototype._redrawLine=function(t,e,i,o,n){if(!0===this.master){var s=a.getDOMElement("div",this.DOMelements.lines,this.dom.lineContainer);s.className=i,s.innerHTML="","left"===e?s.style.left=this.width-o+"px":s.style.right=this.width-o+"px",s.style.width=n+"px",s.style.top=t+"px"}},o.prototype._redrawTitle=function(t){if(a.prepareElements(this.DOMelements.title),void 0!==this.options[t].title&&void 0!==this.options[t].title.text){var e=a.getDOMElement("div",this.DOMelements.title,this.dom.frame);e.className="vis-y-axis vis-title vis-"+t,e.innerHTML=this.options[t].title.text,void 0!==this.options[t].title.style&&r.addCssText(e,this.options[t].title.style),"left"===t?e.style.left=this.props.titleCharHeight+"px":e.style.right=this.props.titleCharHeight+"px",e.style.width=this.height+"px"}a.cleanupElements(this.DOMelements.title)},o.prototype._calculateCharSize=function(){if(!("minorCharHeight"in this.props)){var t=document.createTextNode("0"),e=document.createElement("div");e.className="vis-y-axis vis-minor vis-measure",e.appendChild(t),this.dom.frame.appendChild(e),this.props.minorCharHeight=e.clientHeight,this.props.minorCharWidth=e.clientWidth,this.dom.frame.removeChild(e)}if(!("majorCharHeight"in this.props)){var i=document.createTextNode("0"),o=document.createElement("div");o.className="vis-y-axis vis-major vis-measure",o.appendChild(i),this.dom.frame.appendChild(o),this.props.majorCharHeight=o.clientHeight,this.props.majorCharWidth=o.clientWidth,this.dom.frame.removeChild(o)}if(!("titleCharHeight"in this.props)){var n=document.createTextNode("0"),s=document.createElement("div");s.className="vis-y-axis vis-title vis-measure",s.appendChild(n),this.dom.frame.appendChild(s),this.props.titleCharHeight=s.clientHeight,this.props.titleCharWidth=s.clientWidth,this.dom.frame.removeChild(s)}},t.exports=o},function(t,e,i){function o(t,e,i,o,n,s){var r=arguments.length>6&&void 0!==arguments[6]&&arguments[6],a=arguments.length>7&&void 0!==arguments[7]&&arguments[7];if(this.majorSteps=[1,2,5,10],this.minorSteps=[.25,.5,1,2],this.customLines=null,this.containerHeight=n,this.majorCharHeight=s,this._start=t,this._end=e,this.scale=1,this.minorStepIdx=-1,this.magnitudefactor=1,this.determineScale(),this.zeroAlign=r,this.autoScaleStart=i,this.autoScaleEnd=o,this.formattingFunction=a,i||o){var h=this,d=function(t){var e=t-t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx]);return t%(h.magnitudefactor*h.minorSteps[h.minorStepIdx])>h.magnitudefactor*h.minorSteps[h.minorStepIdx]*.5?e+h.magnitudefactor*h.minorSteps[h.minorStepIdx]:e};i&&(this._start-=2*this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._start=d(this._start)),o&&(this._end+=this.magnitudefactor*this.minorSteps[this.minorStepIdx],this._end=d(this._end)),this.determineScale()}}o.prototype.setCharHeight=function(t){this.majorCharHeight=t},o.prototype.setHeight=function(t){this.containerHeight=t},o.prototype.determineScale=function(){var t=this._end-this._start;this.scale=this.containerHeight/t;var e=this.majorCharHeight/this.scale,i=t>0?Math.round(Math.log(t)/Math.LN10):0;this.minorStepIdx=-1,this.magnitudefactor=Math.pow(10,i);var o=0;i<0&&(o=i);for(var n=!1,s=o;Math.abs(s)<=Math.abs(i);s++){this.magnitudefactor=Math.pow(10,s);for(var r=0;r<this.minorSteps.length;r++){if(this.magnitudefactor*this.minorSteps[r]>=e){n=!0,this.minorStepIdx=r;break}}if(!0===n)break}},o.prototype.is_major=function(t){return t%(this.magnitudefactor*this.majorSteps[this.minorStepIdx])==0},o.prototype.getStep=function(){return this.magnitudefactor*this.minorSteps[this.minorStepIdx]},o.prototype.getFirstMajor=function(){var t=this.magnitudefactor*this.majorSteps[this.minorStepIdx];return this.convertValue(this._start+(t-this._start%t)%t)},o.prototype.formatValue=function(t){var e=t.toPrecision(5);return"function"==typeof this.formattingFunction&&(e=this.formattingFunction(t)),"number"==typeof e?""+e:"string"==typeof e?e:t.toPrecision(5)},o.prototype.getLines=function(){for(var t=[],e=this.getStep(),i=(e-this._start%e)%e,o=this._start+i;this._end-o>1e-5;o+=e)o!=this._start&&t.push({major:this.is_major(o),y:this.convertValue(o),val:this.formatValue(o)});return t},o.prototype.followScale=function(t){var e=this.minorStepIdx,i=this._start,o=this._end,n=this,s=function(){n.magnitudefactor*=2},r=function(){n.magnitudefactor/=2};t.minorStepIdx<=1&&this.minorStepIdx<=1||t.minorStepIdx>1&&this.minorStepIdx>1||(t.minorStepIdx<this.minorStepIdx?(this.minorStepIdx=1,2==e?s():(s(),s())):(this.minorStepIdx=2,1==e?r():(r(),r())));for(var a=t.convertValue(0),h=t.getStep()*t.scale,d=!1,l=0;!d&&l++<5;){this.scale=h/(this.minorSteps[this.minorStepIdx]*this.magnitudefactor);var u=this.containerHeight/this.scale;this._start=i,this._end=this._start+u;var p=this._end*this.scale,c=this.magnitudefactor*this.majorSteps[this.minorStepIdx],m=this.getFirstMajor()-t.getFirstMajor();if(this.zeroAlign){var f=a-p;this._end+=f/this.scale,this._start=this._end-u}else this.autoScaleStart?(this._start-=m/this.scale,this._end=this._start+u):(this._start+=c-m/this.scale,this._end=this._start+u);if(!this.autoScaleEnd&&this._end>o+1e-5)r(),d=!1;else{if(!this.autoScaleStart&&this._start<i-1e-5){if(!(this.zeroAlign&&i>=0)){r(),d=!1;continue}console.warn("Can't adhere to given 'min' range, due to zeroalign")}this.autoScaleStart&&this.autoScaleEnd&&u<o-i?(s(),d=!1):d=!0}}},o.prototype.convertValue=function(t){return this.containerHeight-(t-this._start)*this.scale},o.prototype.screenToValue=function(t){return(this.containerHeight-t)/this.scale+this._start},t.exports=o},function(t,e,i){function o(t,e,i,o){this.id=e;var n=["sampling","style","sort","yAxisOrientation","barChart","drawPoints","shaded","interpolation","zIndex","excludeFromStacking","excludeFromLegend"];this.options=r.selectiveBridgeObject(n,i),this.usingDefaultStyle=void 0===t.className,this.groupsUsingDefaultStyles=o,this.zeroPosition=0,this.update(t),1==this.usingDefaultStyle&&(this.groupsUsingDefaultStyles[0]+=1),this.itemsData=[],this.visible=void 0===t.visible||t.visible}var n=i(1),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(81),h=i(82),d=i(54);o.prototype.setItems=function(t){null!=t?(this.itemsData=t,1==this.options.sort&&r.insertSort(this.itemsData,function(t,e){return t.x>e.x?1:-1})):this.itemsData=[]},o.prototype.getItems=function(){return this.itemsData},o.prototype.setZeroPosition=function(t){this.zeroPosition=t},o.prototype.setOptions=function(t){if(void 0!==t){var e=["sampling","style","sort","yAxisOrientation","barChart","zIndex","excludeFromStacking","excludeFromLegend"];r.selectiveDeepExtend(e,this.options,t),"function"==typeof t.drawPoints&&(t.drawPoints={onRender:t.drawPoints}),r.mergeOptions(this.options,t,"interpolation"),r.mergeOptions(this.options,t,"drawPoints"),r.mergeOptions(this.options,t,"shaded"),t.interpolation&&"object"==(0,s.default)(t.interpolation)&&t.interpolation.parametrization&&("uniform"==t.interpolation.parametrization?this.options.interpolation.alpha=0:"chordal"==t.interpolation.parametrization?this.options.interpolation.alpha=1:(this.options.interpolation.parametrization="centripetal",this.options.interpolation.alpha=.5))}},o.prototype.update=function(t){this.group=t,this.content=t.content||"graph",this.className=t.className||this.className||"vis-graph-group"+this.groupsUsingDefaultStyles[0]%10,this.visible=void 0===t.visible||t.visible,this.style=t.style,this.setOptions(t.options)},o.prototype.getLegend=function(t,e,i,o,n){if(void 0==i||null==i){i={svg:document.createElementNS("http://www.w3.org/2000/svg","svg"),svgElements:{},options:this.options,groups:[this]}}switch(void 0!=o&&null!=o||(o=0),void 0!=n&&null!=n||(n=.5*e),this.options.style){case"line":h.drawIcon(this,o,n,t,e,i);break;case"points":case"point":d.drawIcon(this,o,n,t,e,i);break;case"bar":a.drawIcon(this,o,n,t,e,i)}return{icon:i.svg,label:this.content,orientation:this.options.yAxisOrientation}},o.prototype.getYRange=function(t){for(var e=t[0].y,i=t[0].y,o=0;o<t.length;o++)e=e>t[o].y?t[o].y:e,i=i<t[o].y?t[o].y:i;return{min:e,max:i,yAxisOrientation:this.options.yAxisOrientation}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(11),s=i(54);o.drawIcon=function(t,e,i,o,s,r){var a=.5*s,h=n.getSVGElement("rect",r.svgElements,r.svg);h.setAttributeNS(null,"x",e),h.setAttributeNS(null,"y",i-a),h.setAttributeNS(null,"width",o),h.setAttributeNS(null,"height",2*a),h.setAttributeNS(null,"class","vis-outline");var d=Math.round(.3*o),l=t.options.barChart.width,u=l/d,p=Math.round(.4*s),c=Math.round(.75*s),m=Math.round((o-2*d)/3);if(n.drawBar(e+.5*d+m,i+a-p-1,d,p,t.className+" vis-bar",r.svgElements,r.svg,t.style),n.drawBar(e+1.5*d+m+2,i+a-c-1,d,c,t.className+" vis-bar",r.svgElements,r.svg,t.style),1==t.options.drawPoints.enabled){var f={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size/u,className:t.className};n.drawPoint(e+.5*d+m,i+a-p-1,f,r.svgElements,r.svg),n.drawPoint(e+1.5*d+m+2,i+a-c-1,f,r.svgElements,r.svg)}},o.draw=function(t,e,i){var r,a,h,d,l,u,p=[],c={},m=0;for(l=0;l<t.length;l++)if(d=i.groups[t[l]],"bar"===d.options.style&&!0===d.visible&&(void 0===i.options.groups.visibility[t[l]]||!0===i.options.groups.visibility[t[l]]))for(u=0;u<e[t[l]].length;u++)p.push({screen_x:e[t[l]][u].screen_x,screen_end:e[t[l]][u].screen_end,screen_y:e[t[l]][u].screen_y,x:e[t[l]][u].x,end:e[t[l]][u].end,y:e[t[l]][u].y,groupId:t[l],label:e[t[l]][u].label}),m+=1;if(0!==m)for(p.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x}),o._getDataIntersections(c,p),l=0;l<p.length;l++){d=i.groups[p[l].groupId];var f=void 0!=d.options.barChart.minWidth?d.options.barChart.minWidth:.1*d.options.barChart.width;a=p[l].screen_x;var g=0;if(void 0===c[a])l+1<p.length&&(r=Math.abs(p[l+1].screen_x-a)),h=o._getSafeDrawData(r,d,f);else{var v=l+(c[a].amount-c[a].resolved);v<p.length&&(r=Math.abs(p[v].screen_x-a)),h=o._getSafeDrawData(r,d,f),c[a].resolved+=1,!0===d.options.stack&&!0!==d.options.excludeFromStacking?p[l].screen_y<d.zeroPosition?(g=c[a].accumulatedNegative,c[a].accumulatedNegative+=d.zeroPosition-p[l].screen_y):(g=c[a].accumulatedPositive,c[a].accumulatedPositive+=d.zeroPosition-p[l].screen_y):!0===d.options.barChart.sideBySide&&(h.width=h.width/c[a].amount,h.offset+=c[a].resolved*h.width-.5*h.width*(c[a].amount+1))}var y=h.width,b=p[l].screen_x;if(void 0!=p[l].screen_end?(y=p[l].screen_end-p[l].screen_x,b+=.5*y):b+=h.offset,n.drawBar(b,p[l].screen_y-g,y,d.zeroPosition-p[l].screen_y,d.className+" vis-bar",i.svgElements,i.svg,d.style),!0===d.options.drawPoints.enabled){var _={screen_x:p[l].screen_x,screen_y:p[l].screen_y-g,x:p[l].x,y:p[l].y,groupId:p[l].groupId,label:p[l].label};s.draw([_],d,i,h.offset)}}},o._getDataIntersections=function(t,e){for(var i,o=0;o<e.length;o++)o+1<e.length&&(i=Math.abs(e[o+1].screen_x-e[o].screen_x)),o>0&&(i=Math.min(i,Math.abs(e[o-1].screen_x-e[o].screen_x))),0===i&&(void 0===t[e[o].screen_x]&&(t[e[o].screen_x]={amount:0,resolved:0,accumulatedPositive:0,accumulatedNegative:0}),t[e[o].screen_x].amount+=1)},o._getSafeDrawData=function(t,e,i){var o,n;return t<e.options.barChart.width&&t>0?(o=t<i?i:t,n=0,"left"===e.options.barChart.align?n-=.5*t:"right"===e.options.barChart.align&&(n+=.5*t)):(o=e.options.barChart.width,n=0,"left"===e.options.barChart.align?n-=.5*e.options.barChart.width:"right"===e.options.barChart.align&&(n+=.5*e.options.barChart.width)),{width:o,offset:n}},o.getStackedYRange=function(t,e,i,n,s){if(t.length>0){t.sort(function(t,e){return t.screen_x===e.screen_x?t.groupId<e.groupId?-1:1:t.screen_x-e.screen_x});var r={};o._getDataIntersections(r,t),e[n]=o._getStackedYRange(r,t),e[n].yAxisOrientation=s,i.push(n)}},o._getStackedYRange=function(t,e){for(var i,o=e[0].screen_y,n=e[0].screen_y,s=0;s<e.length;s++)i=e[s].screen_x,void 0===t[i]?(o=o>e[s].screen_y?e[s].screen_y:o,n=n<e[s].screen_y?e[s].screen_y:n):e[s].screen_y<0?t[i].accumulatedNegative+=e[s].screen_y:t[i].accumulatedPositive+=e[s].screen_y;for(var r in t)t.hasOwnProperty(r)&&(o=o>t[r].accumulatedNegative?t[r].accumulatedNegative:o,o=o>t[r].accumulatedPositive?t[r].accumulatedPositive:o,n=n<t[r].accumulatedNegative?t[r].accumulatedNegative:n,n=n<t[r].accumulatedPositive?t[r].accumulatedPositive:n);return{min:o,max:n}},t.exports=o},function(t,e,i){function o(t,e){}var n=i(11);o.calcPath=function(t,e){if(null!=t&&t.length>0){return 1==e.options.interpolation.enabled?o._catmullRom(t,e):o._linear(t)}},o.drawIcon=function(t,e,i,o,s,r){var a,h,d=.5*s,l=n.getSVGElement("rect",r.svgElements,r.svg);if(l.setAttributeNS(null,"x",e),l.setAttributeNS(null,"y",i-d),l.setAttributeNS(null,"width",o),l.setAttributeNS(null,"height",2*d),l.setAttributeNS(null,"class","vis-outline"),a=n.getSVGElement("path",r.svgElements,r.svg),a.setAttributeNS(null,"class",t.className),void 0!==t.style&&a.setAttributeNS(null,"style",t.style),a.setAttributeNS(null,"d","M"+e+","+i+" L"+(e+o)+","+i),1==t.options.shaded.enabled&&(h=n.getSVGElement("path",r.svgElements,r.svg),"top"==t.options.shaded.orientation?h.setAttributeNS(null,"d","M"+e+", "+(i-d)+"L"+e+","+i+" L"+(e+o)+","+i+" L"+(e+o)+","+(i-d)):h.setAttributeNS(null,"d","M"+e+","+i+" L"+e+","+(i+d)+" L"+(e+o)+","+(i+d)+"L"+(e+o)+","+i),h.setAttributeNS(null,"class",t.className+" vis-icon-fill"),void 0!==t.options.shaded.style&&""!==t.options.shaded.style&&h.setAttributeNS(null,"style",t.options.shaded.style)),1==t.options.drawPoints.enabled){var u={style:t.options.drawPoints.style,styles:t.options.drawPoints.styles,size:t.options.drawPoints.size,className:t.className};n.drawPoint(e+.5*o,i,u,r.svgElements,r.svg)}},o.drawShading=function(t,e,i,o){if(1==e.options.shaded.enabled){var s=Number(o.svg.style.height.replace("px","")),r=n.getSVGElement("path",o.svgElements,o.svg),a="L";1==e.options.interpolation.enabled&&(a="C");var h,d=0;d="top"==e.options.shaded.orientation?0:"bottom"==e.options.shaded.orientation?s:Math.min(Math.max(0,e.zeroPosition),s),h="group"==e.options.shaded.orientation&&null!=i&&void 0!=i?"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" L"+i[i.length-1][0]+","+i[i.length-1][1]+" "+this.serializePath(i,a,!0)+i[0][0]+","+i[0][1]+" Z":"M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,a,!1)+" V"+d+" H"+t[0][0]+" Z",r.setAttributeNS(null,"class",e.className+" vis-fill"),void 0!==e.options.shaded.style&&r.setAttributeNS(null,"style",e.options.shaded.style),r.setAttributeNS(null,"d",h)}},o.draw=function(t,e,i){if(null!=t&&void 0!=t){var o=n.getSVGElement("path",i.svgElements,i.svg);o.setAttributeNS(null,"class",e.className),void 0!==e.style&&o.setAttributeNS(null,"style",e.style);var s="L";1==e.options.interpolation.enabled&&(s="C"),o.setAttributeNS(null,"d","M"+t[0][0]+","+t[0][1]+" "+this.serializePath(t,s,!1))}},o.serializePath=function(t,e,i){if(t.length<2)return"";var o,n=e;if(i)for(o=t.length-2;o>0;o--)n+=t[o][0]+","+t[o][1]+" ";else for(o=1;o<t.length;o++)n+=t[o][0]+","+t[o][1]+" ";return n},o._catmullRomUniform=function(t){var e,i,o,n,s,r,a=[];a.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var h=t.length,d=0;d<h-1;d++)e=0==d?t[0]:t[d-1],i=t[d],o=t[d+1],n=d+2<h?t[d+2]:o,s={screen_x:(-e.screen_x+6*i.screen_x+o.screen_x)*(1/6),screen_y:(-e.screen_y+6*i.screen_y+o.screen_y)*(1/6)},r={screen_x:(i.screen_x+6*o.screen_x-n.screen_x)*(1/6),screen_y:(i.screen_y+6*o.screen_y-n.screen_y)*(1/6)},a.push([s.screen_x,s.screen_y]),a.push([r.screen_x,r.screen_y]),a.push([o.screen_x,o.screen_y]);return a},o._catmullRom=function(t,e){var i=e.options.interpolation.alpha;if(0==i||void 0===i)return this._catmullRomUniform(t);var o,n,s,r,a,h,d,l,u,p,c,m,f,g,v,y,b,_,w,x=[];x.push([Math.round(t[0].screen_x),Math.round(t[0].screen_y)]);for(var D=t.length,S=0;S<D-1;S++)o=0==S?t[0]:t[S-1],n=t[S],s=t[S+1],r=S+2<D?t[S+2]:s,d=Math.sqrt(Math.pow(o.screen_x-n.screen_x,2)+Math.pow(o.screen_y-n.screen_y,2)),l=Math.sqrt(Math.pow(n.screen_x-s.screen_x,2)+Math.pow(n.screen_y-s.screen_y,2)),u=Math.sqrt(Math.pow(s.screen_x-r.screen_x,2)+Math.pow(s.screen_y-r.screen_y,2)),g=Math.pow(u,i),y=Math.pow(u,2*i),v=Math.pow(l,i),b=Math.pow(l,2*i),w=Math.pow(d,i),_=Math.pow(d,2*i),p=2*_+3*w*v+b,c=2*y+3*g*v+b,m=3*w*(w+v),m>0&&(m=1/m),f=3*g*(g+v),f>0&&(f=1/f),a={screen_x:(-b*o.screen_x+p*n.screen_x+_*s.screen_x)*m,screen_y:(-b*o.screen_y+p*n.screen_y+_*s.screen_y)*m},h={screen_x:(y*n.screen_x+c*s.screen_x-b*r.screen_x)*f,screen_y:(y*n.screen_y+c*s.screen_y-b*r.screen_y)*f},0==a.screen_x&&0==a.screen_y&&(a=n),0==h.screen_x&&0==h.screen_y&&(h=s),x.push([a.screen_x,a.screen_y]),x.push([h.screen_x,h.screen_y]),x.push([s.screen_x,s.screen_y]);return x},o._linear=function(t){for(var e=[],i=0;i<t.length;i++)e.push([t[i].screen_x,t[i].screen_y]);return e},t.exports=o},function(t,e,i){function o(t,e,i,o){this.body=t,this.defaultOptions={enabled:!1,icons:!0,iconSize:20,iconSpacing:6,left:{visible:!0,position:"top-left"},right:{visible:!0,position:"top-right"}},this.side=i,this.options=r.extend({},this.defaultOptions),this.linegraphOptions=o,this.svgElements={},this.dom={},this.groups={},this.amountOfGroups=0,this._create(),this.framework={svg:this.svg,svgElements:this.svgElements,options:this.options,groups:this.groups},this.setOptions(e)}var n=i(4),s=function(t){return t&&t.__esModule?t:{default:t}}(n),r=i(0),a=i(11),h=i(5);o.prototype=new h,o.prototype.clear=function(){this.groups={},this.amountOfGroups=0},o.prototype.addGroup=function(t,e){1!=e.options.excludeFromLegend&&(this.groups.hasOwnProperty(t)||(this.groups[t]=e),this.amountOfGroups+=1)},o.prototype.updateGroup=function(t,e){this.groups[t]=e},o.prototype.removeGroup=function(t){this.groups.hasOwnProperty(t)&&(delete this.groups[t],this.amountOfGroups-=1)},o.prototype._create=function(){this.dom.frame=document.createElement("div"),this.dom.frame.className="vis-legend",this.dom.frame.style.position="absolute",this.dom.frame.style.top="10px",this.dom.frame.style.display="block",this.dom.textArea=document.createElement("div"),this.dom.textArea.className="vis-legend-text",this.dom.textArea.style.position="relative",this.dom.textArea.style.top="0px",this.svg=document.createElementNS("http://www.w3.org/2000/svg","svg"),this.svg.style.position="absolute",this.svg.style.top="0px",this.svg.style.width=this.options.iconSize+5+"px",this.svg.style.height="100%",this.dom.frame.appendChild(this.svg),this.dom.frame.appendChild(this.dom.textArea)},o.prototype.hide=function(){this.dom.frame.parentNode&&this.dom.frame.parentNode.removeChild(this.dom.frame)},o.prototype.show=function(){this.dom.frame.parentNode||this.body.dom.center.appendChild(this.dom.frame)},o.prototype.setOptions=function(t){var e=["enabled","orientation","icons","left","right"];r.selectiveDeepExtend(e,this.options,t)},o.prototype.redraw=function(){var t=0,e=(0,s.default)(this.groups);e.sort(function(t,e){return t<e?-1:1});for(var i=0;i<e.length;i++){var o=e[i];1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||t++}if(0==this.options[this.side].visible||0==this.amountOfGroups||0==this.options.enabled||0==t)this.hide();else{if(this.show(),"top-left"==this.options[this.side].position||"bottom-left"==this.options[this.side].position?(this.dom.frame.style.left="4px",this.dom.frame.style.textAlign="left",this.dom.textArea.style.textAlign="left",this.dom.textArea.style.left=this.options.iconSize+15+"px",this.dom.textArea.style.right="",this.svg.style.left="0px",this.svg.style.right=""):(this.dom.frame.style.right="4px",this.dom.frame.style.textAlign="right",this.dom.textArea.style.textAlign="right",this.dom.textArea.style.right=this.options.iconSize+15+"px",this.dom.textArea.style.left="",this.svg.style.right="0px",this.svg.style.left=""),"top-left"==this.options[this.side].position||"top-right"==this.options[this.side].position)this.dom.frame.style.top=4-Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.bottom="";else{var n=this.body.domProps.center.height-this.body.domProps.centerContainer.height;this.dom.frame.style.bottom=4+n+Number(this.body.dom.center.style.top.replace("px",""))+"px",this.dom.frame.style.top=""}0==this.options.icons?(this.dom.frame.style.width=this.dom.textArea.offsetWidth+10+"px",this.dom.textArea.style.right="",this.dom.textArea.style.left="",this.svg.style.width="0px"):(this.dom.frame.style.width=this.options.iconSize+15+this.dom.textArea.offsetWidth+10+"px",this.drawLegendIcons());var r="";for(i=0;i<e.length;i++)o=e[i],1!=this.groups[o].visible||void 0!==this.linegraphOptions.visibility[o]&&1!=this.linegraphOptions.visibility[o]||(r+=this.groups[o].content+"<br />");this.dom.textArea.innerHTML=r,this.dom.textArea.style.lineHeight=.75*this.options.iconSize+this.options.iconSpacing+"px"}},o.prototype.drawLegendIcons=function(){if(this.dom.frame.parentNode){var t=(0,s.default)(this.groups);t.sort(function(t,e){return t<e?-1:1}),a.resetElements(this.svgElements);var e=window.getComputedStyle(this.dom.frame).paddingTop,i=Number(e.replace("px","")),o=i,n=this.options.iconSize,r=.75*this.options.iconSize,h=i+.5*r+3;this.svg.style.width=n+5+i+"px";for(var d=0;d<t.length;d++){var l=t[d];1!=this.groups[l].visible||void 0!==this.linegraphOptions.visibility[l]&&1!=this.linegraphOptions.visibility[l]||(this.groups[l].getLegend(n,r,this.framework,o,h),h+=r+this.options.iconSpacing)}}},t.exports=o},function(t,e,i){Object.defineProperty(e,"__esModule",{value:!0});var o="string",n="boolean",s="number",r="object",a={configure:{enabled:{boolean:n},filter:{boolean:n,function:"function"},container:{dom:"dom"},__type__:{object:r,boolean:n,function:"function"}},yAxisOrientation:{string:["left","right"]},defaultGroup:{string:o},sort:{boolean:n},sampling:{boolean:n},stack:{boolean:n},graphHeight:{string:o,number:s},shaded:{enabled:{boolean:n},orientation:{string:["bottom","top","zero","group"]},groupId:{object:r},__type__:{boolean:n,object:r}},style:{string:["line","bar","points"]},barChart:{width:{number:s},minWidth:{number:s},sideBySide:{boolean:n},align:{string:["left","center","right"]},__type__:{object:r}},interpolation:{enabled:{boolean:n},parametrization:{string:["centripetal","chordal","uniform"]},alpha:{number:s},__type__:{object:r,boolean:n}},drawPoints:{enabled:{boolean:n},onRender:{function:"function"},size:{number:s},style:{string:["square","circle"]},__type__:{object:r,boolean:n,function:"function"}},dataAxis:{showMinorLabels:{boolean:n},showMajorLabels:{boolean:n},icons:{boolean:n},width:{string:o,number:s},visible:{boolean:n},alignZeros:{boolean:n},left:{range:{min:{number:s,undefined:"undefined"},max:{number:s,undefined:"undefined"},__type__:{object:r}},format:{function:"function"},title:{text:{string:o,number:s,undefined:"undefined"},style:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},right:{range:{min:{number:s,undefined:"undefined"},max:{number:s,undefined:"undefined"},__type__:{object:r}},format:{function:"function"},title:{text:{string:o,number:s,undefined:"undefined"},style:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},__type__:{object:r}},legend:{enabled:{boolean:n},icons:{boolean:n},left:{visible:{boolean:n},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:r}},right:{visible:{boolean:n},position:{string:["top-right","bottom-right","top-left","bottom-left"]},__type__:{object:r}},__type__:{object:r,boolean:n}},groups:{visibility:{any:"any"},__type__:{object:r}},autoResize:{boolean:n},throttleRedraw:{number:s},clickToUse:{boolean:n},end:{number:s,date:"date",string:o,moment:"moment"},format:{minorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r}},majorLabels:{millisecond:{string:o,undefined:"undefined"},second:{string:o,undefined:"undefined"},minute:{string:o,undefined:"undefined"},hour:{string:o,undefined:"undefined"},weekday:{string:o,undefined:"undefined"},day:{string:o,undefined:"undefined"},month:{string:o,undefined:"undefined"},year:{string:o,undefined:"undefined"},__type__:{object:r}},__type__:{object:r}},moment:{function:"function"},height:{string:o,number:s},hiddenDates:{start:{date:"date",number:s,string:o,moment:"moment"},end:{date:"date",number:s,string:o,moment:"moment"},repeat:{string:o},__type__:{object:r,array:"array"}},locale:{string:o},locales:{__any__:{any:"any"},__type__:{object:r}},max:{date:"date",number:s,string:o,moment:"moment"},maxHeight:{number:s,string:o},maxMinorChars:{number:s},min:{date:"date",number:s,string:o,moment:"moment"},minHeight:{number:s,string:o},moveable:{boolean:n},multiselect:{boolean:n},orientation:{string:o},showCurrentTime:{boolean:n},showMajorLabels:{boolean:n},showMinorLabels:{boolean:n},start:{date:"date",number:s,string:o,moment:"moment"},timeAxis:{scale:{string:o,undefined:"undefined"},step:{number:s,undefined:"undefined"},__type__:{object:r}},width:{string:o,number:s},zoomable:{boolean:n},zoomKey:{string:["ctrlKey","altKey","metaKey",""]},zoomMax:{number:s},zoomMin:{number:s},zIndex:{number:s},__type__:{object:r}},h={global:{sort:!0,sampling:!0,stack:!1,shaded:{enabled:!1,orientation:["zero","top","bottom","group"]},style:["line","bar","points"],barChart:{width:[50,5,100,5],minWidth:[50,5,100,5],sideBySide:!1,align:["left","center","right"]},interpolation:{enabled:!0,parametrization:["centripetal","chordal","uniform"]},drawPoints:{enabled:!0,size:[6,2,30,1],style:["square","circle"]},dataAxis:{showMinorLabels:!0,showMajorLabels:!0,icons:!1,width:[40,0,200,1],visible:!0,alignZeros:!0,left:{title:{text:"",style:""}},right:{title:{text:"",style:""}}},legend:{enabled:!1,icons:!0,left:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]},right:{visible:!0,position:["top-right","bottom-right","top-left","bottom-left"]}},autoResize:!0,clickToUse:!1,end:"",format:{minorLabels:{millisecond:"SSS",second:"s",minute:"HH:mm",hour:"HH:mm",weekday:"ddd D",day:"D",month:"MMM",year:"YYYY"},majorLabels:{millisecond:"HH:mm:ss",second:"D MMMM HH:mm",minute:"ddd D MMMM",hour:"ddd D MMMM",weekday:"MMMM YYYY",day:"MMMM YYYY",month:"YYYY",year:""}},height:"",locale:"",max:"",maxHeight:"",maxMinorChars:[7,0,20,1],min:"",minHeight:"",moveable:!0,orientation:["both","bottom","top"],showCurrentTime:!1,showMajorLabels:!0,showMinorLabels:!0,start:"",width:"100%",zoomable:!0,zoomKey:["ctrlKey","altKey","metaKey",""],zoomMax:[31536e10,10,31536e10,1],zoomMin:[10,10,31536e10,1],zIndex:0}};e.allOptions=a,e.configureOptions=h},function(t,e,i){e.util=i(0),e.DOMutil=i(11),e.DataSet=i(15),e.DataView=i(20),e.Queue=i(65),e.Timeline=i(129),e.Graph2d=i(138),e.timeline={Core:i(48),DateUtil:i(21),Range:i(47),stack:i(71),TimeStep:i(49),components:{items:{Item:i(22),BackgroundItem:i(74),BoxItem:i(72),PointItem:i(73),RangeItem:i(53)},BackgroundGroup:i(52),Component:i(5),CurrentTime:i(50),CustomTime:i(29),DataAxis:i(78),DataScale:i(79),GraphGroup:i(80),Group:i(51),ItemSet:i(70),Legend:i(83),LineGraph:i(77),TimeAxis:i(28)}},e.moment=i(3),e.Hammer=i(12),e.keycharm=i(68)},function(t,e,i){t.exports={default:i(87),__esModule:!0}},function(t,e,i){i(55),i(62),t.exports=i(102)},function(t,e,i){var o=i(89),n=i(90),s=i(23),r=i(13);t.exports=i(56)(Array,"Array",function(t,e){this._t=r(t),this._i=0,this._k=e},function(){var t=this._t,e=this._k,i=this._i++;return!t||i>=t.length?(this._t=void 0,n(1)):"keys"==e?n(0,i):"values"==e?n(0,t[i]):n(0,[i,t[i]])},"values"),s.Arguments=s.Array,o("keys"),o("values"),o("entries")},function(t,e){t.exports=function(){}},function(t,e){t.exports=function(t,e){return{value:e,done:!!t}}},function(t,e,i){var o=i(33);t.exports=Object("z").propertyIsEnumerable(0)?Object:function(t){return"String"==o(t)?t.split(""):Object(t)}},function(t,e,i){var o=i(93);t.exports=function(t,e,i){if(o(t),void 0===e)return t;switch(i){case 1:return function(i){return t.call(e,i)};case 2:return function(i,o){return t.call(e,i,o)};case 3:return function(i,o,n){return t.call(e,i,o,n)}}return function(){return t.apply(e,arguments)}}},function(t,e){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,e,i){var o=i(37),n=i(25),s=i(42),r={};i(14)(r,i(7)("iterator"),function(){return this}),t.exports=function(t,e,i){t.prototype=o(r,{next:n(1,i)}),s(t,e+" Iterator")}},function(t,e,i){var o=i(8),n=i(18),s=i(26);t.exports=i(9)?Object.defineProperties:function(t,e){n(t);for(var i,r=s(e),a=r.length,h=0;a>h;)o.f(t,i=r[h++],e[i]);return t}},function(t,e,i){var o=i(13),n=i(97),s=i(98);t.exports=function(t){return function(e,i,r){var a,h=o(e),d=n(h.length),l=s(r,d);if(t&&i!=i){for(;d>l;)if((a=h[l++])!=a)return!0}else for(;d>l;l++)if((t||l in h)&&h[l]===i)return t||l||0;return!t&&-1}}},function(t,e,i){var o=i(38),n=Math.min;t.exports=function(t){return t>0?n(o(t),9007199254740991):0}},function(t,e,i){var o=i(38),n=Math.max,s=Math.min;t.exports=function(t,e){return t=o(t),t<0?n(t+e,0):s(t,e)}},function(t,e,i){var o=i(6).document;t.exports=o&&o.documentElement},function(t,e,i){var o=i(10),n=i(61),s=i(39)("IE_PROTO"),r=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=n(t),o(t,s)?t[s]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?r:null}},function(t,e,i){var o=i(38),n=i(34);t.exports=function(t){return function(e,i){var s,r,a=String(n(e)),h=o(i),d=a.length;return h<0||h>=d?t?"":void 0:(s=a.charCodeAt(h),s<55296||s>56319||h+1===d||(r=a.charCodeAt(h+1))<56320||r>57343?t?a.charAt(h):s:t?a.slice(h,h+2):r-56320+(s-55296<<10)+65536)}}},function(t,e,i){var o=i(18),n=i(103);t.exports=i(2).getIterator=function(t){var e=n(t);if("function"!=typeof e)throw TypeError(t+" is not iterable!");return o(e.call(t))}},function(t,e,i){var o=i(104),n=i(7)("iterator"),s=i(23);t.exports=i(2).getIteratorMethod=function(t){if(void 0!=t)return t[n]||t["@@iterator"]||s[o(t)]}},function(t,e,i){var o=i(33),n=i(7)("toStringTag"),s="Arguments"==o(function(){return arguments}()),r=function(t,e){try{return t[e]}catch(t){}};t.exports=function(t){var e,i,a;return void 0===t?"Undefined":null===t?"Null":"string"==typeof(i=r(e=Object(t),n))?i:s?o(e):"Object"==(a=o(e))&&"function"==typeof e.callee?"Arguments":a}},function(t,e,i){i(106);var o=i(2).Object;t.exports=function(t,e){return o.create(t,e)}},function(t,e,i){var o=i(17);o(o.S,"Object",{create:i(37)})},function(t,e,i){i(108),t.exports=i(2).Object.keys},function(t,e,i){var o=i(61),n=i(26);i(109)("keys",function(){return function(t){return n(o(t))}})},function(t,e,i){var o=i(17),n=i(2),s=i(19);t.exports=function(t,e){var i=(n.Object||{})[t]||Object[t],r={};r[t]=e(i),o(o.S+o.F*s(function(){i(1)}),"Object",r)}},function(t,e,i){t.exports={default:i(111),__esModule:!0}},function(t,e,i){i(62),i(55),t.exports=i(44).f("iterator")},function(t,e,i){t.exports={default:i(113),__esModule:!0}},function(t,e,i){i(114),i(120),i(121),i(122),t.exports=i(2).Symbol},function(t,e,i){var o=i(6),n=i(10),s=i(9),r=i(17),a=i(59),h=i(115).KEY,d=i(19),l=i(40),u=i(42),p=i(27),c=i(7),m=i(44),f=i(45),g=i(116),v=i(117),y=i(18),b=i(13),_=i(36),w=i(25),x=i(37),D=i(118),S=i(119),k=i(8),C=i(26),T=S.f,M=k.f,O=D.f,E=o.Symbol,P=o.JSON,I=P&&P.stringify,N=c("_hidden"),A=c("toPrimitive"),R={}.propertyIsEnumerable,L=l("symbol-registry"),F=l("symbols"),H=l("op-symbols"),j=Object.prototype,Y="function"==typeof E,G=o.QObject,z=!G||!G.prototype||!G.prototype.findChild,W=s&&d(function(){return 7!=x(M({},"a",{get:function(){return M(this,"a",{value:7}).a}})).a})?function(t,e,i){var o=T(j,e);o&&delete j[e],M(t,e,i),o&&t!==j&&M(j,e,o)}:M,V=function(t){var e=F[t]=x(E.prototype);return e._k=t,e},B=Y&&"symbol"==typeof E.iterator?function(t){return"symbol"==typeof t}:function(t){return t instanceof E},U=function(t,e,i){return t===j&&U(H,e,i),y(t),e=_(e,!0),y(i),n(F,e)?(i.enumerable?(n(t,N)&&t[N][e]&&(t[N][e]=!1),i=x(i,{enumerable:w(0,!1)})):(n(t,N)||M(t,N,w(1,{})),t[N][e]=!0),W(t,e,i)):M(t,e,i)},q=function(t,e){y(t);for(var i,o=g(e=b(e)),n=0,s=o.length;s>n;)U(t,i=o[n++],e[i]);return t},X=function(t,e){return void 0===e?x(t):q(x(t),e)},Z=function(t){var e=R.call(this,t=_(t,!0));return!(this===j&&n(F,t)&&!n(H,t))&&(!(e||!n(this,t)||!n(F,t)||n(this,N)&&this[N][t])||e)},K=function(t,e){if(t=b(t),e=_(e,!0),t!==j||!n(F,e)||n(H,e)){var i=T(t,e);return!i||!n(F,e)||n(t,N)&&t[N][e]||(i.enumerable=!0),i}},J=function(t){for(var e,i=O(b(t)),o=[],s=0;i.length>s;)n(F,e=i[s++])||e==N||e==h||o.push(e);return o},$=function(t){for(var e,i=t===j,o=O(i?H:b(t)),s=[],r=0;o.length>r;)!n(F,e=o[r++])||i&&!n(j,e)||s.push(F[e]);return s};Y||(E=function(){if(this instanceof E)throw TypeError("Symbol is not a constructor!");var t=p(arguments.length>0?arguments[0]:void 0),e=function(i){this===j&&e.call(H,i),n(this,N)&&n(this[N],t)&&(this[N][t]=!1),W(this,t,w(1,i))};return s&&z&&W(j,t,{configurable:!0,set:e}),V(t)},a(E.prototype,"toString",function(){return this._k}),S.f=K,k.f=U,i(64).f=D.f=J,i(46).f=Z,i(63).f=$,s&&!i(35)&&a(j,"propertyIsEnumerable",Z,!0),m.f=function(t){return V(c(t))}),r(r.G+r.W+r.F*!Y,{Symbol:E});for(var Q="hasInstance,isConcatSpreadable,iterator,match,replace,search,species,split,toPrimitive,toStringTag,unscopables".split(","),tt=0;Q.length>tt;)c(Q[tt++]);for(var et=C(c.store),it=0;et.length>it;)f(et[it++]);r(r.S+r.F*!Y,"Symbol",{for:function(t){return n(L,t+="")?L[t]:L[t]=E(t)},keyFor:function(t){if(!B(t))throw TypeError(t+" is not a symbol!");for(var e in L)if(L[e]===t)return e},useSetter:function(){z=!0},useSimple:function(){z=!1}}),r(r.S+r.F*!Y,"Object",{create:X,defineProperty:U,defineProperties:q,getOwnPropertyDescriptor:K,getOwnPropertyNames:J,getOwnPropertySymbols:$}),P&&r(r.S+r.F*(!Y||d(function(){var t=E();return"[null]"!=I([t])||"{}"!=I({a:t})||"{}"!=I(Object(t))})),"JSON",{stringify:function(t){if(void 0!==t&&!B(t)){for(var e,i,o=[t],n=1;arguments.length>n;)o.push(arguments[n++]);return e=o[1],"function"==typeof e&&(i=e),!i&&v(e)||(e=function(t,e){if(i&&(e=i.call(this,t,e)),!B(e))return e}),o[1]=e,I.apply(P,o)}}}),E.prototype[A]||i(14)(E.prototype,A,E.prototype.valueOf),u(E,"Symbol"),u(Math,"Math",!0),u(o.JSON,"JSON",!0)},function(t,e,i){var o=i(27)("meta"),n=i(24),s=i(10),r=i(8).f,a=0,h=Object.isExtensible||function(){return!0},d=!i(19)(function(){return h(Object.preventExtensions({}))}),l=function(t){r(t,o,{value:{i:"O"+ ++a,w:{}}})},u=function(t,e){if(!n(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!s(t,o)){if(!h(t))return"F";if(!e)return"E";l(t)}return t[o].i},p=function(t,e){if(!s(t,o)){if(!h(t))return!0;if(!e)return!1;l(t)}return t[o].w},c=function(t){return d&&m.NEED&&h(t)&&!s(t,o)&&l(t),t},m=t.exports={KEY:o,NEED:!1,fastKey:u,getWeak:p,onFreeze:c}},function(t,e,i){var o=i(26),n=i(63),s=i(46);t.exports=function(t){var e=o(t),i=n.f;if(i)for(var r,a=i(t),h=s.f,d=0;a.length>d;)h.call(t,r=a[d++])&&e.push(r);return e}},function(t,e,i){var o=i(33);t.exports=Array.isArray||function(t){return"Array"==o(t)}},function(t,e,i){var o=i(13),n=i(64).f,s={}.toString,r="object"==typeof window&&window&&Object.getOwnPropertyNames?Object.getOwnPropertyNames(window):[],a=function(t){try{return n(t)}catch(t){return r.slice()}};t.exports.f=function(t){return r&&"[object Window]"==s.call(t)?a(t):n(o(t))}},function(t,e,i){var o=i(46),n=i(25),s=i(13),r=i(36),a=i(10),h=i(57),d=Object.getOwnPropertyDescriptor;e.f=i(9)?d:function(t,e){if(t=s(t),e=r(e,!0),h)try{return d(t,e)}catch(t){}if(a(t,e))return n(!o.f.call(t,e),t[e])}},function(t,e){},function(t,e,i){i(45)("asyncIterator")},function(t,e,i){i(45)("observable")},function(t,e,i){(function(t){!function(e,i){t.exports=i()}(0,function(){function e(){return To.apply(null,arguments)}function i(t){return t instanceof Array||"[object Array]"===Object.prototype.toString.call(t)}function o(t){return null!=t&&"[object Object]"===Object.prototype.toString.call(t)}function n(t){if(Object.getOwnPropertyNames)return 0===Object.getOwnPropertyNames(t).length;var e;for(e in t)if(t.hasOwnProperty(e))return!1;return!0}function s(t){return void 0===t}function r(t){return"number"==typeof t||"[object Number]"===Object.prototype.toString.call(t)}function a(t){return t instanceof Date||"[object Date]"===Object.prototype.toString.call(t)}function h(t,e){var i,o=[];for(i=0;i<t.length;++i)o.push(e(t[i],i));return o}function d(t,e){return Object.prototype.hasOwnProperty.call(t,e)}function l(t,e){for(var i in e)d(e,i)&&(t[i]=e[i]);return d(e,"toString")&&(t.toString=e.toString),d(e,"valueOf")&&(t.valueOf=e.valueOf),t}function u(t,e,i,o){return Se(t,e,i,o,!0).utc()}function p(){return{empty:!1,unusedTokens:[],unusedInput:[],overflow:-2,charsLeftOver:0,nullInput:!1,invalidMonth:null,invalidFormat:!1,userInvalidated:!1,iso:!1,parsedDateParts:[],meridiem:null,rfc2822:!1,weekdayMismatch:!1}}function c(t){return null==t._pf&&(t._pf=p()),t._pf}function m(t){if(null==t._isValid){var e=c(t),i=Mo.call(e.parsedDateParts,function(t){return null!=t}),o=!isNaN(t._d.getTime())&&e.overflow<0&&!e.empty&&!e.invalidMonth&&!e.invalidWeekday&&!e.weekdayMismatch&&!e.nullInput&&!e.invalidFormat&&!e.userInvalidated&&(!e.meridiem||e.meridiem&&i);if(t._strict&&(o=o&&0===e.charsLeftOver&&0===e.unusedTokens.length&&void 0===e.bigHour),null!=Object.isFrozen&&Object.isFrozen(t))return o;t._isValid=o}return t._isValid}function f(t){var e=u(NaN);return null!=t?l(c(e),t):c(e).userInvalidated=!0,e}function g(t,e){var i,o,n;if(s(e._isAMomentObject)||(t._isAMomentObject=e._isAMomentObject),s(e._i)||(t._i=e._i),s(e._f)||(t._f=e._f),s(e._l)||(t._l=e._l),s(e._strict)||(t._strict=e._strict),s(e._tzm)||(t._tzm=e._tzm),s(e._isUTC)||(t._isUTC=e._isUTC),s(e._offset)||(t._offset=e._offset),s(e._pf)||(t._pf=c(e)),s(e._locale)||(t._locale=e._locale),Oo.length>0)for(i=0;i<Oo.length;i++)o=Oo[i],n=e[o],s(n)||(t[o]=n);return t}function v(t){g(this,t),this._d=new Date(null!=t._d?t._d.getTime():NaN),this.isValid()||(this._d=new Date(NaN)),!1===Eo&&(Eo=!0,e.updateOffset(this),Eo=!1)}function y(t){return t instanceof v||null!=t&&null!=t._isAMomentObject}function b(t){return t<0?Math.ceil(t)||0:Math.floor(t)}function _(t){var e=+t,i=0;return 0!==e&&isFinite(e)&&(i=b(e)),i}function w(t,e,i){var o,n=Math.min(t.length,e.length),s=Math.abs(t.length-e.length),r=0;for(o=0;o<n;o++)(i&&t[o]!==e[o]||!i&&_(t[o])!==_(e[o]))&&r++;return r+s}function x(t){!1===e.suppressDeprecationWarnings&&"undefined"!=typeof console&&console.warn&&console.warn("Deprecation warning: "+t)}function D(t,i){var o=!0;return l(function(){if(null!=e.deprecationHandler&&e.deprecationHandler(null,t),o){for(var n,s=[],r=0;r<arguments.length;r++){if(n="","object"==typeof arguments[r]){n+="\n["+r+"] ";for(var a in arguments[0])n+=a+": "+arguments[0][a]+", ";n=n.slice(0,-2)}else n=arguments[r];s.push(n)}x(t+"\nArguments: "+Array.prototype.slice.call(s).join("")+"\n"+(new Error).stack),o=!1}return i.apply(this,arguments)},i)}function S(t,i){null!=e.deprecationHandler&&e.deprecationHandler(t,i),Po[t]||(x(i),Po[t]=!0)}function k(t){return t instanceof Function||"[object Function]"===Object.prototype.toString.call(t)}function C(t){var e,i;for(i in t)e=t[i],k(e)?this[i]=e:this["_"+i]=e;this._config=t,this._dayOfMonthOrdinalParseLenient=new RegExp((this._dayOfMonthOrdinalParse.source||this._ordinalParse.source)+"|"+/\d{1,2}/.source)}function T(t,e){var i,n=l({},t);for(i in e)d(e,i)&&(o(t[i])&&o(e[i])?(n[i]={},l(n[i],t[i]),l(n[i],e[i])):null!=e[i]?n[i]=e[i]:delete n[i]);for(i in t)d(t,i)&&!d(e,i)&&o(t[i])&&(n[i]=l({},n[i]));return n}function M(t){null!=t&&this.set(t)}function O(t,e,i){var o=this._calendar[t]||this._calendar.sameElse;return k(o)?o.call(e,i):o}function E(t){var e=this._longDateFormat[t],i=this._longDateFormat[t.toUpperCase()];return e||!i?e:(this._longDateFormat[t]=i.replace(/MMMM|MM|DD|dddd/g,function(t){return t.slice(1)}),this._longDateFormat[t])}function P(){return this._invalidDate}function I(t){return this._ordinal.replace("%d",t)}function N(t,e,i,o){var n=this._relativeTime[i];return k(n)?n(t,e,i,o):n.replace(/%d/i,t)}function A(t,e){var i=this._relativeTime[t>0?"future":"past"];return k(i)?i(e):i.replace(/%s/i,e)}function R(t,e){var i=t.toLowerCase();Fo[i]=Fo[i+"s"]=Fo[e]=t}function L(t){return"string"==typeof t?Fo[t]||Fo[t.toLowerCase()]:void 0}function F(t){var e,i,o={};for(i in t)d(t,i)&&(e=L(i))&&(o[e]=t[i]);return o}function H(t,e){Ho[t]=e}function j(t){var e=[];for(var i in t)e.push({unit:i,priority:Ho[i]});return e.sort(function(t,e){return t.priority-e.priority}),e}function Y(t,e,i){var o=""+Math.abs(t),n=e-o.length;return(t>=0?i?"+":"":"-")+Math.pow(10,Math.max(0,n)).toString().substr(1)+o}function G(t,e,i,o){var n=o;"string"==typeof o&&(n=function(){return this[o]()}),t&&(zo[t]=n),e&&(zo[e[0]]=function(){return Y(n.apply(this,arguments),e[1],e[2])}),i&&(zo[i]=function(){return this.localeData().ordinal(n.apply(this,arguments),t)})}function z(t){return t.match(/\[[\s\S]/)?t.replace(/^\[|\]$/g,""):t.replace(/\\/g,"")}function W(t){var e,i,o=t.match(jo);for(e=0,i=o.length;e<i;e++)zo[o[e]]?o[e]=zo[o[e]]:o[e]=z(o[e]);return function(e){var n,s="";for(n=0;n<i;n++)s+=k(o[n])?o[n].call(e,t):o[n];return s}}function V(t,e){return t.isValid()?(e=B(e,t.localeData()),Go[e]=Go[e]||W(e),Go[e](t)):t.localeData().invalidDate()}function B(t,e){function i(t){return e.longDateFormat(t)||t}var o=5;for(Yo.lastIndex=0;o>=0&&Yo.test(t);)t=t.replace(Yo,i),Yo.lastIndex=0,o-=1;return t}function U(t,e,i){an[t]=k(e)?e:function(t,o){return t&&i?i:e}}function q(t,e){return d(an,t)?an[t](e._strict,e._locale):new RegExp(X(t))}function X(t){return Z(t.replace("\\","").replace(/\\(\[)|\\(\])|\[([^\]\[]*)\]|\\(.)/g,function(t,e,i,o,n){return e||i||o||n}))}function Z(t){return t.replace(/[-\/\\^$*+?.()|[\]{}]/g,"\\$&")}function K(t,e){var i,o=e;for("string"==typeof t&&(t=[t]),r(e)&&(o=function(t,i){i[e]=_(t)}),i=0;i<t.length;i++)hn[t[i]]=o}function J(t,e){K(t,function(t,i,o,n){o._w=o._w||{},e(t,o._w,o,n)})}function $(t,e,i){null!=e&&d(hn,t)&&hn[t](e,i._a,i,t)}function Q(t){return tt(t)?366:365}function tt(t){return t%4==0&&t%100!=0||t%400==0}function et(){return tt(this.year())}function it(t,i){return function(o){return null!=o?(nt(this,t,o),e.updateOffset(this,i),this):ot(this,t)}}function ot(t,e){return t.isValid()?t._d["get"+(t._isUTC?"UTC":"")+e]():NaN}function nt(t,e,i){t.isValid()&&!isNaN(i)&&("FullYear"===e&&tt(t.year())?t._d["set"+(t._isUTC?"UTC":"")+e](i,t.month(),ht(i,t.month())):t._d["set"+(t._isUTC?"UTC":"")+e](i))}function st(t){return t=L(t),k(this[t])?this[t]():this}function rt(t,e){if("object"==typeof t){t=F(t);for(var i=j(t),o=0;o<i.length;o++)this[i[o].unit](t[i[o].unit])}else if(t=L(t),k(this[t]))return this[t](e);return this}function at(t,e){return(t%e+e)%e}function ht(t,e){if(isNaN(t)||isNaN(e))return NaN;var i=at(e,12);return t+=(e-i)/12,1===i?tt(t)?29:28:31-i%7%2}function dt(t,e){return t?i(this._months)?this._months[t.month()]:this._months[(this._months.isFormat||_n).test(e)?"format":"standalone"][t.month()]:i(this._months)?this._months:this._months.standalone}function lt(t,e){return t?i(this._monthsShort)?this._monthsShort[t.month()]:this._monthsShort[_n.test(e)?"format":"standalone"][t.month()]:i(this._monthsShort)?this._monthsShort:this._monthsShort.standalone}function ut(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._monthsParse)for(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[],o=0;o<12;++o)s=u([2e3,o]),this._shortMonthsParse[o]=this.monthsShort(s,"").toLocaleLowerCase(),this._longMonthsParse[o]=this.months(s,"").toLocaleLowerCase();return i?"MMM"===e?(n=yn.call(this._shortMonthsParse,r),-1!==n?n:null):(n=yn.call(this._longMonthsParse,r),-1!==n?n:null):"MMM"===e?-1!==(n=yn.call(this._shortMonthsParse,r))?n:(n=yn.call(this._longMonthsParse,r),-1!==n?n:null):-1!==(n=yn.call(this._longMonthsParse,r))?n:(n=yn.call(this._shortMonthsParse,r),-1!==n?n:null)}function pt(t,e,i){var o,n,s;if(this._monthsParseExact)return ut.call(this,t,e,i);for(this._monthsParse||(this._monthsParse=[],this._longMonthsParse=[],this._shortMonthsParse=[]),o=0;o<12;o++){if(n=u([2e3,o]),i&&!this._longMonthsParse[o]&&(this._longMonthsParse[o]=new RegExp("^"+this.months(n,"").replace(".","")+"$","i"),this._shortMonthsParse[o]=new RegExp("^"+this.monthsShort(n,"").replace(".","")+"$","i")),i||this._monthsParse[o]||(s="^"+this.months(n,"")+"|^"+this.monthsShort(n,""),this._monthsParse[o]=new RegExp(s.replace(".",""),"i")),i&&"MMMM"===e&&this._longMonthsParse[o].test(t))return o;if(i&&"MMM"===e&&this._shortMonthsParse[o].test(t))return o;if(!i&&this._monthsParse[o].test(t))return o}}function ct(t,e){var i;if(!t.isValid())return t;if("string"==typeof e)if(/^\d+$/.test(e))e=_(e);else if(e=t.localeData().monthsParse(e),!r(e))return t;return i=Math.min(t.date(),ht(t.year(),e)),t._d["set"+(t._isUTC?"UTC":"")+"Month"](e,i),t}function mt(t){return null!=t?(ct(this,t),e.updateOffset(this,!0),this):ot(this,"Month")}function ft(){return ht(this.year(),this.month())}function gt(t){return this._monthsParseExact?(d(this,"_monthsRegex")||yt.call(this),t?this._monthsShortStrictRegex:this._monthsShortRegex):(d(this,"_monthsShortRegex")||(this._monthsShortRegex=Dn),this._monthsShortStrictRegex&&t?this._monthsShortStrictRegex:this._monthsShortRegex)}function vt(t){return this._monthsParseExact?(d(this,"_monthsRegex")||yt.call(this),t?this._monthsStrictRegex:this._monthsRegex):(d(this,"_monthsRegex")||(this._monthsRegex=Sn),this._monthsStrictRegex&&t?this._monthsStrictRegex:this._monthsRegex)}function yt(){function t(t,e){return e.length-t.length}var e,i,o=[],n=[],s=[];for(e=0;e<12;e++)i=u([2e3,e]),o.push(this.monthsShort(i,"")),n.push(this.months(i,"")),s.push(this.months(i,"")),s.push(this.monthsShort(i,""));for(o.sort(t),n.sort(t),s.sort(t),e=0;e<12;e++)o[e]=Z(o[e]),n[e]=Z(n[e]);for(e=0;e<24;e++)s[e]=Z(s[e]);this._monthsRegex=new RegExp("^("+s.join("|")+")","i"),this._monthsShortRegex=this._monthsRegex,this._monthsStrictRegex=new RegExp("^("+n.join("|")+")","i"),this._monthsShortStrictRegex=new RegExp("^("+o.join("|")+")","i")}function bt(t,e,i,o,n,s,r){var a=new Date(t,e,i,o,n,s,r);return t<100&&t>=0&&isFinite(a.getFullYear())&&a.setFullYear(t),a}function _t(t){var e=new Date(Date.UTC.apply(null,arguments));return t<100&&t>=0&&isFinite(e.getUTCFullYear())&&e.setUTCFullYear(t),e}function wt(t,e,i){var o=7+e-i;return-(7+_t(t,0,o).getUTCDay()-e)%7+o-1}function xt(t,e,i,o,n){var s,r,a=(7+i-o)%7,h=wt(t,o,n),d=1+7*(e-1)+a+h;return d<=0?(s=t-1,r=Q(s)+d):d>Q(t)?(s=t+1,r=d-Q(t)):(s=t,r=d),{year:s,dayOfYear:r}}function Dt(t,e,i){var o,n,s=wt(t.year(),e,i),r=Math.floor((t.dayOfYear()-s-1)/7)+1;return r<1?(n=t.year()-1,o=r+St(n,e,i)):r>St(t.year(),e,i)?(o=r-St(t.year(),e,i),n=t.year()+1):(n=t.year(),o=r),{week:o,year:n}}function St(t,e,i){var o=wt(t,e,i),n=wt(t+1,e,i);return(Q(t)-o+n)/7}function kt(t){return Dt(t,this._week.dow,this._week.doy).week}function Ct(){return this._week.dow}function Tt(){return this._week.doy}function Mt(t){var e=this.localeData().week(this);return null==t?e:this.add(7*(t-e),"d")}function Ot(t){var e=Dt(this,1,4).week;return null==t?e:this.add(7*(t-e),"d")}function Et(t,e){return"string"!=typeof t?t:isNaN(t)?(t=e.weekdaysParse(t),"number"==typeof t?t:null):parseInt(t,10)}function Pt(t,e){return"string"==typeof t?e.weekdaysParse(t)%7||7:isNaN(t)?null:t}function It(t,e){return t?i(this._weekdays)?this._weekdays[t.day()]:this._weekdays[this._weekdays.isFormat.test(e)?"format":"standalone"][t.day()]:i(this._weekdays)?this._weekdays:this._weekdays.standalone}function Nt(t){return t?this._weekdaysShort[t.day()]:this._weekdaysShort}function At(t){return t?this._weekdaysMin[t.day()]:this._weekdaysMin}function Rt(t,e,i){var o,n,s,r=t.toLocaleLowerCase();if(!this._weekdaysParse)for(this._weekdaysParse=[],this._shortWeekdaysParse=[],this._minWeekdaysParse=[],o=0;o<7;++o)s=u([2e3,1]).day(o),this._minWeekdaysParse[o]=this.weekdaysMin(s,"").toLocaleLowerCase(),this._shortWeekdaysParse[o]=this.weekdaysShort(s,"").toLocaleLowerCase(),this._weekdaysParse[o]=this.weekdays(s,"").toLocaleLowerCase();return i?"dddd"===e?(n=yn.call(this._weekdaysParse,r),-1!==n?n:null):"ddd"===e?(n=yn.call(this._shortWeekdaysParse,r),-1!==n?n:null):(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):"dddd"===e?-1!==(n=yn.call(this._weekdaysParse,r))?n:-1!==(n=yn.call(this._shortWeekdaysParse,r))?n:(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):"ddd"===e?-1!==(n=yn.call(this._shortWeekdaysParse,r))?n:-1!==(n=yn.call(this._weekdaysParse,r))?n:(n=yn.call(this._minWeekdaysParse,r),-1!==n?n:null):-1!==(n=yn.call(this._minWeekdaysParse,r))?n:-1!==(n=yn.call(this._weekdaysParse,r))?n:(n=yn.call(this._shortWeekdaysParse,r),-1!==n?n:null)}function Lt(t,e,i){var o,n,s;if(this._weekdaysParseExact)return Rt.call(this,t,e,i);for(this._weekdaysParse||(this._weekdaysParse=[],this._minWeekdaysParse=[],this._shortWeekdaysParse=[],this._fullWeekdaysParse=[]),o=0;o<7;o++){if(n=u([2e3,1]).day(o),i&&!this._fullWeekdaysParse[o]&&(this._fullWeekdaysParse[o]=new RegExp("^"+this.weekdays(n,"").replace(".",".?")+"$","i"),this._shortWeekdaysParse[o]=new RegExp("^"+this.weekdaysShort(n,"").replace(".",".?")+"$","i"),this._minWeekdaysParse[o]=new RegExp("^"+this.weekdaysMin(n,"").replace(".",".?")+"$","i")),this._weekdaysParse[o]||(s="^"+this.weekdays(n,"")+"|^"+this.weekdaysShort(n,"")+"|^"+this.weekdaysMin(n,""),this._weekdaysParse[o]=new RegExp(s.replace(".",""),"i")),i&&"dddd"===e&&this._fullWeekdaysParse[o].test(t))return o;if(i&&"ddd"===e&&this._shortWeekdaysParse[o].test(t))return o;if(i&&"dd"===e&&this._minWeekdaysParse[o].test(t))return o;if(!i&&this._weekdaysParse[o].test(t))return o}}function Ft(t){if(!this.isValid())return null!=t?this:NaN;var e=this._isUTC?this._d.getUTCDay():this._d.getDay();return null!=t?(t=Et(t,this.localeData()),this.add(t-e,"d")):e}function Ht(t){if(!this.isValid())return null!=t?this:NaN;var e=(this.day()+7-this.localeData()._week.dow)%7;return null==t?e:this.add(t-e,"d")}function jt(t){if(!this.isValid())return null!=t?this:NaN;if(null!=t){var e=Pt(t,this.localeData());return this.day(this.day()%7?e:e-7)}return this.day()||7}function Yt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysStrictRegex:this._weekdaysRegex):(d(this,"_weekdaysRegex")||(this._weekdaysRegex=On),this._weekdaysStrictRegex&&t?this._weekdaysStrictRegex:this._weekdaysRegex)}function Gt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex):(d(this,"_weekdaysShortRegex")||(this._weekdaysShortRegex=En),this._weekdaysShortStrictRegex&&t?this._weekdaysShortStrictRegex:this._weekdaysShortRegex)}function zt(t){return this._weekdaysParseExact?(d(this,"_weekdaysRegex")||Wt.call(this),t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex):(d(this,"_weekdaysMinRegex")||(this._weekdaysMinRegex=Pn),this._weekdaysMinStrictRegex&&t?this._weekdaysMinStrictRegex:this._weekdaysMinRegex)}function Wt(){function t(t,e){return e.length-t.length}var e,i,o,n,s,r=[],a=[],h=[],d=[];for(e=0;e<7;e++)i=u([2e3,1]).day(e),o=this.weekdaysMin(i,""),n=this.weekdaysShort(i,""),s=this.weekdays(i,""),r.push(o),a.push(n),h.push(s),d.push(o),d.push(n),d.push(s);for(r.sort(t),a.sort(t),h.sort(t),d.sort(t),e=0;e<7;e++)a[e]=Z(a[e]),h[e]=Z(h[e]),d[e]=Z(d[e]);this._weekdaysRegex=new RegExp("^("+d.join("|")+")","i"),this._weekdaysShortRegex=this._weekdaysRegex,this._weekdaysMinRegex=this._weekdaysRegex,this._weekdaysStrictRegex=new RegExp("^("+h.join("|")+")","i"),this._weekdaysShortStrictRegex=new RegExp("^("+a.join("|")+")","i"),this._weekdaysMinStrictRegex=new RegExp("^("+r.join("|")+")","i")}function Vt(){return this.hours()%12||12}function Bt(){return this.hours()||24}function Ut(t,e){G(t,0,0,function(){return this.localeData().meridiem(this.hours(),this.minutes(),e)})}function qt(t,e){return e._meridiemParse}function Xt(t){return"p"===(t+"").toLowerCase().charAt(0)}function Zt(t,e,i){return t>11?i?"pm":"PM":i?"am":"AM"}function Kt(t){return t?t.toLowerCase().replace("_","-"):t}function Jt(t){for(var e,i,o,n,s=0;s<t.length;){for(n=Kt(t[s]).split("-"),e=n.length,i=Kt(t[s+1]),i=i?i.split("-"):null;e>0;){if(o=$t(n.slice(0,e).join("-")))return o;if(i&&i.length>=e&&w(n,i,!0)>=e-1)break;e--}s++}return null}function $t(e){var i=null;if(!Ln[e]&&void 0!==t&&t&&t.exports)try{i=In._abbr;!function(){var t=new Error('Cannot find module "./locale"');throw t.code="MODULE_NOT_FOUND",t}(),Qt(i)}catch(t){}return Ln[e]}function Qt(t,e){var i;return t&&(i=s(e)?ie(t):te(t,e))&&(In=i),In._abbr}function te(t,e){if(null!==e){var i=Rn;if(e.abbr=t,null!=Ln[t])S("defineLocaleOverride","use moment.updateLocale(localeName, config) to change an existing locale. moment.defineLocale(localeName, config) should only be used for creating a new locale See http://momentjs.com/guides/#/warnings/define-locale/ for more info."),i=Ln[t]._config;else if(null!=e.parentLocale){if(null==Ln[e.parentLocale])return Fn[e.parentLocale]||(Fn[e.parentLocale]=[]),Fn[e.parentLocale].push({name:t,config:e}),null;i=Ln[e.parentLocale]._config}return Ln[t]=new M(T(i,e)),Fn[t]&&Fn[t].forEach(function(t){te(t.name,t.config)}),Qt(t),Ln[t]}return delete Ln[t],null}function ee(t,e){if(null!=e){var i,o=Rn;null!=Ln[t]&&(o=Ln[t]._config),e=T(o,e),i=new M(e),i.parentLocale=Ln[t],Ln[t]=i,Qt(t)}else null!=Ln[t]&&(null!=Ln[t].parentLocale?Ln[t]=Ln[t].parentLocale:null!=Ln[t]&&delete Ln[t]);return Ln[t]}function ie(t){var e;if(t&&t._locale&&t._locale._abbr&&(t=t._locale._abbr),!t)return In;if(!i(t)){if(e=$t(t))return e;t=[t]}return Jt(t)}function oe(){return Io(Ln)}function ne(t){var e,i=t._a;return i&&-2===c(t).overflow&&(e=i[ln]<0||i[ln]>11?ln:i[un]<1||i[un]>ht(i[dn],i[ln])?un:i[pn]<0||i[pn]>24||24===i[pn]&&(0!==i[cn]||0!==i[mn]||0!==i[fn])?pn:i[cn]<0||i[cn]>59?cn:i[mn]<0||i[mn]>59?mn:i[fn]<0||i[fn]>999?fn:-1,c(t)._overflowDayOfYear&&(e<dn||e>un)&&(e=un),c(t)._overflowWeeks&&-1===e&&(e=gn),c(t)._overflowWeekday&&-1===e&&(e=vn),c(t).overflow=e),t}function se(t,e,i){return null!=t?t:null!=e?e:i}function re(t){var i=new Date(e.now());return t._useUTC?[i.getUTCFullYear(),i.getUTCMonth(),i.getUTCDate()]:[i.getFullYear(),i.getMonth(),i.getDate()]}function ae(t){var e,i,o,n,s=[];if(!t._d){for(o=re(t),t._w&&null==t._a[un]&&null==t._a[ln]&&he(t),null!=t._dayOfYear&&(n=se(t._a[dn],o[dn]),(t._dayOfYear>Q(n)||0===t._dayOfYear)&&(c(t)._overflowDayOfYear=!0),i=_t(n,0,t._dayOfYear),t._a[ln]=i.getUTCMonth(),t._a[un]=i.getUTCDate()),e=0;e<3&&null==t._a[e];++e)t._a[e]=s[e]=o[e];for(;e<7;e++)t._a[e]=s[e]=null==t._a[e]?2===e?1:0:t._a[e];24===t._a[pn]&&0===t._a[cn]&&0===t._a[mn]&&0===t._a[fn]&&(t._nextDay=!0,t._a[pn]=0),t._d=(t._useUTC?_t:bt).apply(null,s),null!=t._tzm&&t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),t._nextDay&&(t._a[pn]=24),t._w&&void 0!==t._w.d&&t._w.d!==t._d.getDay()&&(c(t).weekdayMismatch=!0)}}function he(t){var e,i,o,n,s,r,a,h;if(e=t._w,null!=e.GG||null!=e.W||null!=e.E)s=1,r=4,i=se(e.GG,t._a[dn],Dt(ke(),1,4).year),o=se(e.W,1),((n=se(e.E,1))<1||n>7)&&(h=!0);else{s=t._locale._week.dow,r=t._locale._week.doy;var d=Dt(ke(),s,r);i=se(e.gg,t._a[dn],d.year),o=se(e.w,d.week),null!=e.d?((n=e.d)<0||n>6)&&(h=!0):null!=e.e?(n=e.e+s,(e.e<0||e.e>6)&&(h=!0)):n=s}o<1||o>St(i,s,r)?c(t)._overflowWeeks=!0:null!=h?c(t)._overflowWeekday=!0:(a=xt(i,o,n,s,r),t._a[dn]=a.year,t._dayOfYear=a.dayOfYear)}function de(t){var e,i,o,n,s,r,a=t._i,h=Hn.exec(a)||jn.exec(a);if(h){for(c(t).iso=!0,e=0,i=Gn.length;e<i;e++)if(Gn[e][1].exec(h[1])){n=Gn[e][0],o=!1!==Gn[e][2];break}if(null==n)return void(t._isValid=!1);if(h[3]){for(e=0,i=zn.length;e<i;e++)if(zn[e][1].exec(h[3])){s=(h[2]||" ")+zn[e][0];break}if(null==s)return void(t._isValid=!1)}if(!o&&null!=s)return void(t._isValid=!1);if(h[4]){if(!Yn.exec(h[4]))return void(t._isValid=!1);r="Z"}t._f=n+(s||"")+(r||""),ve(t)}else t._isValid=!1}function le(t,e,i,o,n,s){var r=[ue(t),xn.indexOf(e),parseInt(i,10),parseInt(o,10),parseInt(n,10)];return s&&r.push(parseInt(s,10)),r}function ue(t){var e=parseInt(t,10);return e<=49?2e3+e:e<=999?1900+e:e}function pe(t){return t.replace(/\([^)]*\)|[\n\t]/g," ").replace(/(\s\s+)/g," ").trim()}function ce(t,e,i){if(t){if(Tn.indexOf(t)!==new Date(e[0],e[1],e[2]).getDay())return c(i).weekdayMismatch=!0,i._isValid=!1,!1}return!0}function me(t,e,i){if(t)return Bn[t];if(e)return 0;var o=parseInt(i,10),n=o%100;return(o-n)/100*60+n}function fe(t){var e=Vn.exec(pe(t._i));if(e){var i=le(e[4],e[3],e[2],e[5],e[6],e[7]);if(!ce(e[1],i,t))return;t._a=i,t._tzm=me(e[8],e[9],e[10]),t._d=_t.apply(null,t._a),t._d.setUTCMinutes(t._d.getUTCMinutes()-t._tzm),c(t).rfc2822=!0}else t._isValid=!1}function ge(t){var i=Wn.exec(t._i);if(null!==i)return void(t._d=new Date(+i[1]));de(t),!1===t._isValid&&(delete t._isValid,fe(t),!1===t._isValid&&(delete t._isValid,e.createFromInputFallback(t)))}function ve(t){if(t._f===e.ISO_8601)return void de(t);if(t._f===e.RFC_2822)return void fe(t);t._a=[],c(t).empty=!0;var i,o,n,s,r,a=""+t._i,h=a.length,d=0;for(n=B(t._f,t._locale).match(jo)||[],i=0;i<n.length;i++)s=n[i],o=(a.match(q(s,t))||[])[0],o&&(r=a.substr(0,a.indexOf(o)),r.length>0&&c(t).unusedInput.push(r),a=a.slice(a.indexOf(o)+o.length),d+=o.length),zo[s]?(o?c(t).empty=!1:c(t).unusedTokens.push(s),$(s,o,t)):t._strict&&!o&&c(t).unusedTokens.push(s);c(t).charsLeftOver=h-d,a.length>0&&c(t).unusedInput.push(a),t._a[pn]<=12&&!0===c(t).bigHour&&t._a[pn]>0&&(c(t).bigHour=void 0),c(t).parsedDateParts=t._a.slice(0),c(t).meridiem=t._meridiem,t._a[pn]=ye(t._locale,t._a[pn],t._meridiem),ae(t),ne(t)}function ye(t,e,i){var o;return null==i?e:null!=t.meridiemHour?t.meridiemHour(e,i):null!=t.isPM?(o=t.isPM(i),o&&e<12&&(e+=12),o||12!==e||(e=0),e):e}function be(t){var e,i,o,n,s;if(0===t._f.length)return c(t).invalidFormat=!0,void(t._d=new Date(NaN));for(n=0;n<t._f.length;n++)s=0,e=g({},t),null!=t._useUTC&&(e._useUTC=t._useUTC),e._f=t._f[n],ve(e),m(e)&&(s+=c(e).charsLeftOver,s+=10*c(e).unusedTokens.length,c(e).score=s,(null==o||s<o)&&(o=s,i=e));l(t,i||e)}function _e(t){if(!t._d){var e=F(t._i);t._a=h([e.year,e.month,e.day||e.date,e.hour,e.minute,e.second,e.millisecond],function(t){return t&&parseInt(t,10)}),ae(t)}}function we(t){var e=new v(ne(xe(t)));return e._nextDay&&(e.add(1,"d"),e._nextDay=void 0),e}function xe(t){var e=t._i,o=t._f;return t._locale=t._locale||ie(t._l),null===e||void 0===o&&""===e?f({nullInput:!0}):("string"==typeof e&&(t._i=e=t._locale.preparse(e)),y(e)?new v(ne(e)):(a(e)?t._d=e:i(o)?be(t):o?ve(t):De(t),m(t)||(t._d=null),t))}function De(t){var n=t._i;s(n)?t._d=new Date(e.now()):a(n)?t._d=new Date(n.valueOf()):"string"==typeof n?ge(t):i(n)?(t._a=h(n.slice(0),function(t){return parseInt(t,10)}),ae(t)):o(n)?_e(t):r(n)?t._d=new Date(n):e.createFromInputFallback(t)}function Se(t,e,s,r,a){var h={};return!0!==s&&!1!==s||(r=s,s=void 0),(o(t)&&n(t)||i(t)&&0===t.length)&&(t=void 0),h._isAMomentObject=!0,h._useUTC=h._isUTC=a,h._l=s,h._i=t,h._f=e,h._strict=r,we(h)}function ke(t,e,i,o){return Se(t,e,i,o,!1)}function Ce(t,e){var o,n;if(1===e.length&&i(e[0])&&(e=e[0]),!e.length)return ke();for(o=e[0],n=1;n<e.length;++n)e[n].isValid()&&!e[n][t](o)||(o=e[n]);return o}function Te(){return Ce("isBefore",[].slice.call(arguments,0))}function Me(){return Ce("isAfter",[].slice.call(arguments,0))}function Oe(t){for(var e in t)if(-1===yn.call(Zn,e)||null!=t[e]&&isNaN(t[e]))return!1;for(var i=!1,o=0;o<Zn.length;++o)if(t[Zn[o]]){if(i)return!1;parseFloat(t[Zn[o]])!==_(t[Zn[o]])&&(i=!0)}return!0}function Ee(){return this._isValid}function Pe(){return Ke(NaN)}function Ie(t){var e=F(t),i=e.year||0,o=e.quarter||0,n=e.month||0,s=e.week||0,r=e.day||0,a=e.hour||0,h=e.minute||0,d=e.second||0,l=e.millisecond||0;this._isValid=Oe(e),this._milliseconds=+l+1e3*d+6e4*h+1e3*a*60*60,this._days=+r+7*s,this._months=+n+3*o+12*i,this._data={},this._locale=ie(),this._bubble()}function Ne(t){return t instanceof Ie}function Ae(t){return t<0?-1*Math.round(-1*t):Math.round(t)}function Re(t,e){G(t,0,0,function(){var t=this.utcOffset(),i="+";return t<0&&(t=-t,i="-"),i+Y(~~(t/60),2)+e+Y(~~t%60,2)})}function Le(t,e){var i=(e||"").match(t);if(null===i)return null;var o=i[i.length-1]||[],n=(o+"").match(Kn)||["-",0,0],s=60*n[1]+_(n[2]);return 0===s?0:"+"===n[0]?s:-s}function Fe(t,i){var o,n;return i._isUTC?(o=i.clone(),n=(y(t)||a(t)?t.valueOf():ke(t).valueOf())-o.valueOf(),o._d.setTime(o._d.valueOf()+n),e.updateOffset(o,!1),o):ke(t).local()}function He(t){return 15*-Math.round(t._d.getTimezoneOffset()/15)}function je(t,i,o){var n,s=this._offset||0;if(!this.isValid())return null!=t?this:NaN;if(null!=t){if("string"==typeof t){if(null===(t=Le(nn,t)))return this}else Math.abs(t)<16&&!o&&(t*=60);return!this._isUTC&&i&&(n=He(this)),this._offset=t,this._isUTC=!0,null!=n&&this.add(n,"m"),s!==t&&(!i||this._changeInProgress?ei(this,Ke(t-s,"m"),1,!1):this._changeInProgress||(this._changeInProgress=!0,e.updateOffset(this,!0),this._changeInProgress=null)),this}return this._isUTC?s:He(this)}function Ye(t,e){return null!=t?("string"!=typeof t&&(t=-t),this.utcOffset(t,e),this):-this.utcOffset()}function Ge(t){return this.utcOffset(0,t)}function ze(t){return this._isUTC&&(this.utcOffset(0,t),this._isUTC=!1,t&&this.subtract(He(this),"m")),this}function We(){if(null!=this._tzm)this.utcOffset(this._tzm,!1,!0);else if("string"==typeof this._i){var t=Le(on,this._i);null!=t?this.utcOffset(t):this.utcOffset(0,!0)}return this}function Ve(t){return!!this.isValid()&&(t=t?ke(t).utcOffset():0,(this.utcOffset()-t)%60==0)}function Be(){return this.utcOffset()>this.clone().month(0).utcOffset()||this.utcOffset()>this.clone().month(5).utcOffset()}function Ue(){if(!s(this._isDSTShifted))return this._isDSTShifted;var t={};if(g(t,this),t=xe(t),t._a){var e=t._isUTC?u(t._a):ke(t._a);this._isDSTShifted=this.isValid()&&w(t._a,e.toArray())>0}else this._isDSTShifted=!1;return this._isDSTShifted}function qe(){return!!this.isValid()&&!this._isUTC}function Xe(){return!!this.isValid()&&this._isUTC}function Ze(){return!!this.isValid()&&(this._isUTC&&0===this._offset)}function Ke(t,e){var i,o,n,s=t,a=null;return Ne(t)?s={ms:t._milliseconds,d:t._days,M:t._months}:r(t)?(s={},e?s[e]=t:s.milliseconds=t):(a=Jn.exec(t))?(i="-"===a[1]?-1:1,s={y:0,d:_(a[un])*i,h:_(a[pn])*i,m:_(a[cn])*i,s:_(a[mn])*i,ms:_(Ae(1e3*a[fn]))*i}):(a=$n.exec(t))?(i="-"===a[1]?-1:(a[1],1),s={y:Je(a[2],i),M:Je(a[3],i),w:Je(a[4],i),d:Je(a[5],i),h:Je(a[6],i),m:Je(a[7],i),s:Je(a[8],i)}):null==s?s={}:"object"==typeof s&&("from"in s||"to"in s)&&(n=Qe(ke(s.from),ke(s.to)),s={},s.ms=n.milliseconds,s.M=n.months),o=new Ie(s),Ne(t)&&d(t,"_locale")&&(o._locale=t._locale),o}function Je(t,e){var i=t&&parseFloat(t.replace(",","."));return(isNaN(i)?0:i)*e}function $e(t,e){var i={milliseconds:0,months:0};return i.months=e.month()-t.month()+12*(e.year()-t.year()),t.clone().add(i.months,"M").isAfter(e)&&--i.months,i.milliseconds=+e-+t.clone().add(i.months,"M"),i}function Qe(t,e){var i;return t.isValid()&&e.isValid()?(e=Fe(e,t),t.isBefore(e)?i=$e(t,e):(i=$e(e,t),i.milliseconds=-i.milliseconds,i.months=-i.months),i):{milliseconds:0,months:0}}function ti(t,e){return function(i,o){var n,s;return null===o||isNaN(+o)||(S(e,"moment()."+e+"(period, number) is deprecated. Please use moment()."+e+"(number, period). See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info."),s=i,i=o,o=s),i="string"==typeof i?+i:i,n=Ke(i,o),ei(this,n,t),this}}function ei(t,i,o,n){var s=i._milliseconds,r=Ae(i._days),a=Ae(i._months);t.isValid()&&(n=null==n||n,a&&ct(t,ot(t,"Month")+a*o),r&&nt(t,"Date",ot(t,"Date")+r*o),s&&t._d.setTime(t._d.valueOf()+s*o),n&&e.updateOffset(t,r||a))}function ii(t,e){var i=t.diff(e,"days",!0);return i<-6?"sameElse":i<-1?"lastWeek":i<0?"lastDay":i<1?"sameDay":i<2?"nextDay":i<7?"nextWeek":"sameElse"}function oi(t,i){var o=t||ke(),n=Fe(o,this).startOf("day"),s=e.calendarFormat(this,n)||"sameElse",r=i&&(k(i[s])?i[s].call(this,o):i[s]);return this.format(r||this.localeData().calendar(s,this,ke(o)))}function ni(){return new v(this)}function si(t,e){var i=y(t)?t:ke(t);return!(!this.isValid()||!i.isValid())&&(e=L(s(e)?"millisecond":e),"millisecond"===e?this.valueOf()>i.valueOf():i.valueOf()<this.clone().startOf(e).valueOf())}function ri(t,e){var i=y(t)?t:ke(t);return!(!this.isValid()||!i.isValid())&&(e=L(s(e)?"millisecond":e),"millisecond"===e?this.valueOf()<i.valueOf():this.clone().endOf(e).valueOf()<i.valueOf())}function ai(t,e,i,o){return o=o||"()",("("===o[0]?this.isAfter(t,i):!this.isBefore(t,i))&&(")"===o[1]?this.isBefore(e,i):!this.isAfter(e,i))}function hi(t,e){var i,o=y(t)?t:ke(t);return!(!this.isValid()||!o.isValid())&&(e=L(e||"millisecond"),"millisecond"===e?this.valueOf()===o.valueOf():(i=o.valueOf(),this.clone().startOf(e).valueOf()<=i&&i<=this.clone().endOf(e).valueOf()))}function di(t,e){return this.isSame(t,e)||this.isAfter(t,e)}function li(t,e){return this.isSame(t,e)||this.isBefore(t,e)}function ui(t,e,i){var o,n,s;if(!this.isValid())return NaN;if(o=Fe(t,this),!o.isValid())return NaN;switch(n=6e4*(o.utcOffset()-this.utcOffset()),e=L(e)){case"year":s=pi(this,o)/12;break;case"month":s=pi(this,o);break;case"quarter":s=pi(this,o)/3;break;case"second":s=(this-o)/1e3;break;case"minute":s=(this-o)/6e4;break;case"hour":s=(this-o)/36e5;break;case"day":s=(this-o-n)/864e5;break;case"week":s=(this-o-n)/6048e5;break;default:s=this-o}return i?s:b(s)}function pi(t,e){var i,o,n=12*(e.year()-t.year())+(e.month()-t.month()),s=t.clone().add(n,"months");return e-s<0?(i=t.clone().add(n-1,"months"),o=(e-s)/(s-i)):(i=t.clone().add(n+1,"months"),o=(e-s)/(i-s)),-(n+o)||0}function ci(){return this.clone().locale("en").format("ddd MMM DD YYYY HH:mm:ss [GMT]ZZ")}function mi(){if(!this.isValid())return null;var t=this.clone().utc();return t.year()<0||t.year()>9999?V(t,"YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]"):k(Date.prototype.toISOString)?this.toDate().toISOString():V(t,"YYYY-MM-DD[T]HH:mm:ss.SSS[Z]")}function fi(){if(!this.isValid())return"moment.invalid(/* "+this._i+" */)";var t="moment",e="";this.isLocal()||(t=0===this.utcOffset()?"moment.utc":"moment.parseZone",e="Z");var i="["+t+'("]',o=0<=this.year()&&this.year()<=9999?"YYYY":"YYYYYY",n=e+'[")]';return this.format(i+o+"-MM-DD[T]HH:mm:ss.SSS"+n)}function gi(t){t||(t=this.isUtc()?e.defaultFormatUtc:e.defaultFormat);var i=V(this,t);return this.localeData().postformat(i)}function vi(t,e){return this.isValid()&&(y(t)&&t.isValid()||ke(t).isValid())?Ke({to:this,from:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function yi(t){return this.from(ke(),t)}function bi(t,e){return this.isValid()&&(y(t)&&t.isValid()||ke(t).isValid())?Ke({from:this,to:t}).locale(this.locale()).humanize(!e):this.localeData().invalidDate()}function _i(t){return this.to(ke(),t)}function wi(t){var e;return void 0===t?this._locale._abbr:(e=ie(t),null!=e&&(this._locale=e),this)}function xi(){return this._locale}function Di(t){switch(t=L(t)){case"year":this.month(0);case"quarter":case"month":this.date(1);case"week":case"isoWeek":case"day":case"date":this.hours(0);case"hour":this.minutes(0);case"minute":this.seconds(0);case"second":this.milliseconds(0)}return"week"===t&&this.weekday(0),"isoWeek"===t&&this.isoWeekday(1),"quarter"===t&&this.month(3*Math.floor(this.month()/3)),this}function Si(t){return void 0===(t=L(t))||"millisecond"===t?this:("date"===t&&(t="day"),this.startOf(t).add(1,"isoWeek"===t?"week":t).subtract(1,"ms"))}function ki(){return this._d.valueOf()-6e4*(this._offset||0)}function Ci(){return Math.floor(this.valueOf()/1e3)}function Ti(){return new Date(this.valueOf())}function Mi(){var t=this;return[t.year(),t.month(),t.date(),t.hour(),t.minute(),t.second(),t.millisecond()]}function Oi(){var t=this;return{years:t.year(),months:t.month(),date:t.date(),hours:t.hours(),minutes:t.minutes(),seconds:t.seconds(),milliseconds:t.milliseconds()}}function Ei(){return this.isValid()?this.toISOString():null}function Pi(){return m(this)}function Ii(){return l({},c(this))}function Ni(){return c(this).overflow}function Ai(){return{input:this._i,format:this._f,locale:this._locale,isUTC:this._isUTC,strict:this._strict}}function Ri(t,e){G(0,[t,t.length],0,e)}function Li(t){return Yi.call(this,t,this.week(),this.weekday(),this.localeData()._week.dow,this.localeData()._week.doy)}function Fi(t){return Yi.call(this,t,this.isoWeek(),this.isoWeekday(),1,4)}function Hi(){return St(this.year(),1,4)}function ji(){var t=this.localeData()._week;return St(this.year(),t.dow,t.doy)}function Yi(t,e,i,o,n){var s;return null==t?Dt(this,o,n).year:(s=St(t,o,n),e>s&&(e=s),Gi.call(this,t,e,i,o,n))}function Gi(t,e,i,o,n){var s=xt(t,e,i,o,n),r=_t(s.year,0,s.dayOfYear);return this.year(r.getUTCFullYear()),this.month(r.getUTCMonth()),this.date(r.getUTCDate()),this}function zi(t){return null==t?Math.ceil((this.month()+1)/3):this.month(3*(t-1)+this.month()%3)}function Wi(t){var e=Math.round((this.clone().startOf("day")-this.clone().startOf("year"))/864e5)+1;return null==t?e:this.add(t-e,"d")}function Vi(t,e){e[fn]=_(1e3*("0."+t))}function Bi(){return this._isUTC?"UTC":""}function Ui(){return this._isUTC?"Coordinated Universal Time":""}function qi(t){return ke(1e3*t)}function Xi(){return ke.apply(null,arguments).parseZone()}function Zi(t){return t}function Ki(t,e,i,o){var n=ie(),s=u().set(o,e);return n[i](s,t)}function Ji(t,e,i){if(r(t)&&(e=t,t=void 0),t=t||"",null!=e)return Ki(t,e,i,"month");var o,n=[];for(o=0;o<12;o++)n[o]=Ki(t,o,i,"month");return n}function $i(t,e,i,o){"boolean"==typeof t?(r(e)&&(i=e,e=void 0),e=e||""):(e=t,i=e,t=!1,r(e)&&(i=e,e=void 0),e=e||"");var n=ie(),s=t?n._week.dow:0;if(null!=i)return Ki(e,(i+s)%7,o,"day");var a,h=[];for(a=0;a<7;a++)h[a]=Ki(e,(a+s)%7,o,"day");return h}function Qi(t,e){return Ji(t,e,"months")}function to(t,e){return Ji(t,e,"monthsShort")}function eo(t,e,i){return $i(t,e,i,"weekdays")}function io(t,e,i){return $i(t,e,i,"weekdaysShort")}function oo(t,e,i){return $i(t,e,i,"weekdaysMin")}function no(){var t=this._data;return this._milliseconds=ds(this._milliseconds),this._days=ds(this._days),this._months=ds(this._months),t.milliseconds=ds(t.milliseconds),t.seconds=ds(t.seconds),t.minutes=ds(t.minutes),t.hours=ds(t.hours),t.months=ds(t.months),t.years=ds(t.years),this}function so(t,e,i,o){var n=Ke(e,i);return t._milliseconds+=o*n._milliseconds,t._days+=o*n._days,t._months+=o*n._months,t._bubble()}function ro(t,e){return so(this,t,e,1)}function ao(t,e){return so(this,t,e,-1)}function ho(t){return t<0?Math.floor(t):Math.ceil(t)}function lo(){var t,e,i,o,n,s=this._milliseconds,r=this._days,a=this._months,h=this._data;return s>=0&&r>=0&&a>=0||s<=0&&r<=0&&a<=0||(s+=864e5*ho(po(a)+r),r=0,a=0),h.milliseconds=s%1e3,t=b(s/1e3),h.seconds=t%60,e=b(t/60),h.minutes=e%60,i=b(e/60),h.hours=i%24,r+=b(i/24),n=b(uo(r)),a+=n,r-=ho(po(n)),o=b(a/12),a%=12,h.days=r,h.months=a,h.years=o,this}function uo(t){return 4800*t/146097}function po(t){return 146097*t/4800}function co(t){if(!this.isValid())return NaN;var e,i,o=this._milliseconds;if("month"===(t=L(t))||"year"===t)return e=this._days+o/864e5,i=this._months+uo(e),"month"===t?i:i/12;switch(e=this._days+Math.round(po(this._months)),t){case"week":return e/7+o/6048e5;case"day":return e+o/864e5;case"hour":return 24*e+o/36e5;case"minute":return 1440*e+o/6e4;case"second":return 86400*e+o/1e3;case"millisecond":return Math.floor(864e5*e)+o;default:throw new Error("Unknown unit "+t)}}function mo(){return this.isValid()?this._milliseconds+864e5*this._days+this._months%12*2592e6+31536e6*_(this._months/12):NaN}function fo(t){return function(){return this.as(t)}}function go(){return Ke(this)}function vo(t){return t=L(t),this.isValid()?this[t+"s"]():NaN}function yo(t){return function(){return this.isValid()?this._data[t]:NaN}}function bo(){return b(this.days()/7)}function _o(t,e,i,o,n){return n.relativeTime(e||1,!!i,t,o)}function wo(t,e,i){var o=Ke(t).abs(),n=ks(o.as("s")),s=ks(o.as("m")),r=ks(o.as("h")),a=ks(o.as("d")),h=ks(o.as("M")),d=ks(o.as("y")),l=n<=Cs.ss&&["s",n]||n<Cs.s&&["ss",n]||s<=1&&["m"]||s<Cs.m&&["mm",s]||r<=1&&["h"]||r<Cs.h&&["hh",r]||a<=1&&["d"]||a<Cs.d&&["dd",a]||h<=1&&["M"]||h<Cs.M&&["MM",h]||d<=1&&["y"]||["yy",d];return l[2]=e,l[3]=+t>0,l[4]=i,_o.apply(null,l)}function xo(t){return void 0===t?ks:"function"==typeof t&&(ks=t,!0)}function Do(t,e){return void 0!==Cs[t]&&(void 0===e?Cs[t]:(Cs[t]=e,"s"===t&&(Cs.ss=e-1),!0))}function So(t){if(!this.isValid())return this.localeData().invalidDate();var e=this.localeData(),i=wo(this,!t,e);return t&&(i=e.pastFuture(+this,i)),e.postformat(i)}function ko(t){return(t>0)-(t<0)||+t}function Co(){if(!this.isValid())return this.localeData().invalidDate();var t,e,i,o=Ts(this._milliseconds)/1e3,n=Ts(this._days),s=Ts(this._months);t=b(o/60),e=b(t/60),o%=60,t%=60,i=b(s/12),s%=12;var r=i,a=s,h=n,d=e,l=t,u=o?o.toFixed(3).replace(/\.?0+$/,""):"",p=this.asSeconds();if(!p)return"P0D";var c=p<0?"-":"",m=ko(this._months)!==ko(p)?"-":"",f=ko(this._days)!==ko(p)?"-":"",g=ko(this._milliseconds)!==ko(p)?"-":"";return c+"P"+(r?m+r+"Y":"")+(a?m+a+"M":"")+(h?f+h+"D":"")+(d||l||u?"T":"")+(d?g+d+"H":"")+(l?g+l+"M":"")+(u?g+u+"S":"")}var To,Mo;Mo=Array.prototype.some?Array.prototype.some:function(t){for(var e=Object(this),i=e.length>>>0,o=0;o<i;o++)if(o in e&&t.call(this,e[o],o,e))return!0;return!1};var Oo=e.momentProperties=[],Eo=!1,Po={};e.suppressDeprecationWarnings=!1,e.deprecationHandler=null;var Io;Io=Object.keys?Object.keys:function(t){var e,i=[];for(e in t)d(t,e)&&i.push(e);return i};var No={sameDay:"[Today at] LT",nextDay:"[Tomorrow at] LT",nextWeek:"dddd [at] LT",lastDay:"[Yesterday at] LT",lastWeek:"[Last] dddd [at] LT",sameElse:"L"},Ao={LTS:"h:mm:ss A",LT:"h:mm A",L:"MM/DD/YYYY",LL:"MMMM D, YYYY",LLL:"MMMM D, YYYY h:mm A",LLLL:"dddd, MMMM D, YYYY h:mm A"},Ro=/\d{1,2}/,Lo={future:"in %s",past:"%s ago",s:"a few seconds",ss:"%d seconds",m:"a minute",mm:"%d minutes",h:"an hour",hh:"%d hours",d:"a day",dd:"%d days",M:"a month",MM:"%d months",y:"a year",yy:"%d years"},Fo={},Ho={},jo=/(\[[^\[]*\])|(\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g,Yo=/(\[[^\[]*\])|(\\)?(LTS|LT|LL?L?L?|l{1,4})/g,Go={},zo={},Wo=/\d/,Vo=/\d\d/,Bo=/\d{3}/,Uo=/\d{4}/,qo=/[+-]?\d{6}/,Xo=/\d\d?/,Zo=/\d\d\d\d?/,Ko=/\d\d\d\d\d\d?/,Jo=/\d{1,3}/,$o=/\d{1,4}/,Qo=/[+-]?\d{1,6}/,tn=/\d+/,en=/[+-]?\d+/,on=/Z|[+-]\d\d:?\d\d/gi,nn=/Z|[+-]\d\d(?::?\d\d)?/gi,sn=/[+-]?\d+(\.\d{1,3})?/,rn=/[0-9]*['a-z\u00A0-\u05FF\u0700-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+|[\u0600-\u06FF\/]+(\s*?[\u0600-\u06FF]+){1,2}/i,an={},hn={},dn=0,ln=1,un=2,pn=3,cn=4,mn=5,fn=6,gn=7,vn=8;G("Y",0,0,function(){var t=this.year();return t<=9999?""+t:"+"+t}),G(0,["YY",2],0,function(){return this.year()%100}),G(0,["YYYY",4],0,"year"),G(0,["YYYYY",5],0,"year"),G(0,["YYYYYY",6,!0],0,"year"),R("year","y"),H("year",1),U("Y",en),U("YY",Xo,Vo),U("YYYY",$o,Uo),U("YYYYY",Qo,qo),U("YYYYYY",Qo,qo),K(["YYYYY","YYYYYY"],dn),K("YYYY",function(t,i){i[dn]=2===t.length?e.parseTwoDigitYear(t):_(t)}),K("YY",function(t,i){i[dn]=e.parseTwoDigitYear(t)}),K("Y",function(t,e){e[dn]=parseInt(t,10)}),e.parseTwoDigitYear=function(t){return _(t)+(_(t)>68?1900:2e3)};var yn,bn=it("FullYear",!0);yn=Array.prototype.indexOf?Array.prototype.indexOf:function(t){var e;for(e=0;e<this.length;++e)if(this[e]===t)return e;return-1},G("M",["MM",2],"Mo",function(){return this.month()+1}),G("MMM",0,0,function(t){return this.localeData().monthsShort(this,t)}),G("MMMM",0,0,function(t){return this.localeData().months(this,t)}),R("month","M"),H("month",8),U("M",Xo),U("MM",Xo,Vo),U("MMM",function(t,e){return e.monthsShortRegex(t)}),U("MMMM",function(t,e){return e.monthsRegex(t)}),K(["M","MM"],function(t,e){e[ln]=_(t)-1}),K(["MMM","MMMM"],function(t,e,i,o){var n=i._locale.monthsParse(t,o,i._strict);null!=n?e[ln]=n:c(i).invalidMonth=t});var _n=/D[oD]?(\[[^\[\]]*\]|\s)+MMMM?/,wn="January_February_March_April_May_June_July_August_September_October_November_December".split("_"),xn="Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"),Dn=rn,Sn=rn;G("w",["ww",2],"wo","week"),G("W",["WW",2],"Wo","isoWeek"),R("week","w"),R("isoWeek","W"),H("week",5),H("isoWeek",5),U("w",Xo),U("ww",Xo,Vo),U("W",Xo),U("WW",Xo,Vo),J(["w","ww","W","WW"],function(t,e,i,o){e[o.substr(0,1)]=_(t)});var kn={dow:0,doy:6};G("d",0,"do","day"),G("dd",0,0,function(t){return this.localeData().weekdaysMin(this,t)}),G("ddd",0,0,function(t){return this.localeData().weekdaysShort(this,t)}),G("dddd",0,0,function(t){return this.localeData().weekdays(this,t)}),G("e",0,0,"weekday"),G("E",0,0,"isoWeekday"),R("day","d"),R("weekday","e"),R("isoWeekday","E"),H("day",11),H("weekday",11),H("isoWeekday",11),U("d",Xo),U("e",Xo),U("E",Xo),U("dd",function(t,e){return e.weekdaysMinRegex(t)}),U("ddd",function(t,e){return e.weekdaysShortRegex(t)}),U("dddd",function(t,e){return e.weekdaysRegex(t)}),J(["dd","ddd","dddd"],function(t,e,i,o){var n=i._locale.weekdaysParse(t,o,i._strict);null!=n?e.d=n:c(i).invalidWeekday=t}),J(["d","e","E"],function(t,e,i,o){e[o]=_(t)});var Cn="Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),Tn="Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"),Mn="Su_Mo_Tu_We_Th_Fr_Sa".split("_"),On=rn,En=rn,Pn=rn;G("H",["HH",2],0,"hour"),G("h",["hh",2],0,Vt),G("k",["kk",2],0,Bt),G("hmm",0,0,function(){return""+Vt.apply(this)+Y(this.minutes(),2)}),G("hmmss",0,0,function(){return""+Vt.apply(this)+Y(this.minutes(),2)+Y(this.seconds(),2)}),G("Hmm",0,0,function(){return""+this.hours()+Y(this.minutes(),2)}),G("Hmmss",0,0,function(){return""+this.hours()+Y(this.minutes(),2)+Y(this.seconds(),2)}),Ut("a",!0),Ut("A",!1),R("hour","h"),H("hour",13),U("a",qt),U("A",qt),U("H",Xo),U("h",Xo),U("k",Xo),U("HH",Xo,Vo),U("hh",Xo,Vo),U("kk",Xo,Vo),U("hmm",Zo),U("hmmss",Ko),U("Hmm",Zo),U("Hmmss",Ko),K(["H","HH"],pn),K(["k","kk"],function(t,e,i){var o=_(t);e[pn]=24===o?0:o}),K(["a","A"],function(t,e,i){i._isPm=i._locale.isPM(t),i._meridiem=t}),K(["h","hh"],function(t,e,i){e[pn]=_(t),c(i).bigHour=!0}),K("hmm",function(t,e,i){var o=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o)),c(i).bigHour=!0}),K("hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o,2)),e[mn]=_(t.substr(n)),c(i).bigHour=!0}),K("Hmm",function(t,e,i){var o=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o))}),K("Hmmss",function(t,e,i){var o=t.length-4,n=t.length-2;e[pn]=_(t.substr(0,o)),e[cn]=_(t.substr(o,2)),e[mn]=_(t.substr(n))});var In,Nn=/[ap]\.?m?\.?/i,An=it("Hours",!0),Rn={calendar:No,longDateFormat:Ao,invalidDate:"Invalid date",ordinal:"%d",dayOfMonthOrdinalParse:Ro,relativeTime:Lo,months:wn,monthsShort:xn,week:kn,weekdays:Cn,weekdaysMin:Mn,weekdaysShort:Tn,meridiemParse:Nn},Ln={},Fn={},Hn=/^\s*((?:[+-]\d{6}|\d{4})-(?:\d\d-\d\d|W\d\d-\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?::\d\d(?::\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,jn=/^\s*((?:[+-]\d{6}|\d{4})(?:\d\d\d\d|W\d\d\d|W\d\d|\d\d\d|\d\d))(?:(T| )(\d\d(?:\d\d(?:\d\d(?:[.,]\d+)?)?)?)([\+\-]\d\d(?::?\d\d)?|\s*Z)?)?$/,Yn=/Z|[+-]\d\d(?::?\d\d)?/,Gn=[["YYYYYY-MM-DD",/[+-]\d{6}-\d\d-\d\d/],["YYYY-MM-DD",/\d{4}-\d\d-\d\d/],["GGGG-[W]WW-E",/\d{4}-W\d\d-\d/],["GGGG-[W]WW",/\d{4}-W\d\d/,!1],["YYYY-DDD",/\d{4}-\d{3}/],["YYYY-MM",/\d{4}-\d\d/,!1],["YYYYYYMMDD",/[+-]\d{10}/],["YYYYMMDD",/\d{8}/],["GGGG[W]WWE",/\d{4}W\d{3}/],["GGGG[W]WW",/\d{4}W\d{2}/,!1],["YYYYDDD",/\d{7}/]],zn=[["HH:mm:ss.SSSS",/\d\d:\d\d:\d\d\.\d+/],["HH:mm:ss,SSSS",/\d\d:\d\d:\d\d,\d+/],["HH:mm:ss",/\d\d:\d\d:\d\d/],["HH:mm",/\d\d:\d\d/],["HHmmss.SSSS",/\d\d\d\d\d\d\.\d+/],["HHmmss,SSSS",/\d\d\d\d\d\d,\d+/],["HHmmss",/\d\d\d\d\d\d/],["HHmm",/\d\d\d\d/],["HH",/\d\d/]],Wn=/^\/?Date\((\-?\d+)/i,Vn=/^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\s)?(\d{1,2})\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s(\d{2,4})\s(\d\d):(\d\d)(?::(\d\d))?\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\d{4}))$/,Bn={UT:0,GMT:0,EDT:-240,EST:-300,CDT:-300,CST:-360,MDT:-360,MST:-420,PDT:-420,PST:-480};e.createFromInputFallback=D("value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are discouraged and will be removed in an upcoming major release. Please refer to http://momentjs.com/guides/#/warnings/js-date/ for more info.",function(t){t._d=new Date(t._i+(t._useUTC?" UTC":""))}),e.ISO_8601=function(){},e.RFC_2822=function(){};var Un=D("moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var t=ke.apply(null,arguments);return this.isValid()&&t.isValid()?t<this?this:t:f()}),qn=D("moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/",function(){var t=ke.apply(null,arguments);return this.isValid()&&t.isValid()?t>this?this:t:f()}),Xn=function(){return Date.now?Date.now():+new Date},Zn=["year","quarter","month","week","day","hour","minute","second","millisecond"];Re("Z",":"),Re("ZZ",""),U("Z",nn),U("ZZ",nn),K(["Z","ZZ"],function(t,e,i){i._useUTC=!0,i._tzm=Le(nn,t)});var Kn=/([\+\-]|\d\d)/gi;e.updateOffset=function(){};var Jn=/^(\-|\+)?(?:(\d*)[. ])?(\d+)\:(\d+)(?:\:(\d+)(\.\d*)?)?$/,$n=/^(-|\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;Ke.fn=Ie.prototype,Ke.invalid=Pe;var Qn=ti(1,"add"),ts=ti(-1,"subtract");e.defaultFormat="YYYY-MM-DDTHH:mm:ssZ",e.defaultFormatUtc="YYYY-MM-DDTHH:mm:ss[Z]";var es=D("moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.",function(t){return void 0===t?this.localeData():this.locale(t)});G(0,["gg",2],0,function(){return this.weekYear()%100}),G(0,["GG",2],0,function(){return this.isoWeekYear()%100}),Ri("gggg","weekYear"),Ri("ggggg","weekYear"),Ri("GGGG","isoWeekYear"),Ri("GGGGG","isoWeekYear"),R("weekYear","gg"),R("isoWeekYear","GG"),H("weekYear",1),H("isoWeekYear",1),U("G",en),U("g",en),U("GG",Xo,Vo),U("gg",Xo,Vo),U("GGGG",$o,Uo),U("gggg",$o,Uo),U("GGGGG",Qo,qo),U("ggggg",Qo,qo),J(["gggg","ggggg","GGGG","GGGGG"],function(t,e,i,o){e[o.substr(0,2)]=_(t)}),J(["gg","GG"],function(t,i,o,n){i[n]=e.parseTwoDigitYear(t)}),G("Q",0,"Qo","quarter"),R("quarter","Q"),H("quarter",7),U("Q",Wo),K("Q",function(t,e){e[ln]=3*(_(t)-1)}),G("D",["DD",2],"Do","date"),R("date","D"),H("date",9),U("D",Xo),U("DD",Xo,Vo),U("Do",function(t,e){return t?e._dayOfMonthOrdinalParse||e._ordinalParse:e._dayOfMonthOrdinalParseLenient}),K(["D","DD"],un),K("Do",function(t,e){e[un]=_(t.match(Xo)[0],10)});var is=it("Date",!0);G("DDD",["DDDD",3],"DDDo","dayOfYear"),R("dayOfYear","DDD"),H("dayOfYear",4),U("DDD",Jo),U("DDDD",Bo),K(["DDD","DDDD"],function(t,e,i){i._dayOfYear=_(t)}),G("m",["mm",2],0,"minute"),R("minute","m"),H("minute",14),U("m",Xo),U("mm",Xo,Vo),K(["m","mm"],cn);var os=it("Minutes",!1);G("s",["ss",2],0,"second"),R("second","s"),H("second",15),U("s",Xo),U("ss",Xo,Vo),K(["s","ss"],mn);var ns=it("Seconds",!1);G("S",0,0,function(){return~~(this.millisecond()/100)}),G(0,["SS",2],0,function(){return~~(this.millisecond()/10)}),G(0,["SSS",3],0,"millisecond"),G(0,["SSSS",4],0,function(){return 10*this.millisecond()}),G(0,["SSSSS",5],0,function(){return 100*this.millisecond()}),G(0,["SSSSSS",6],0,function(){return 1e3*this.millisecond()}),G(0,["SSSSSSS",7],0,function(){return 1e4*this.millisecond()}),G(0,["SSSSSSSS",8],0,function(){return 1e5*this.millisecond()}),G(0,["SSSSSSSSS",9],0,function(){return 1e6*this.millisecond()}),R("millisecond","ms"),H("millisecond",16),U("S",Jo,Wo),U("SS",Jo,Vo),U("SSS",Jo,Bo);var ss;for(ss="SSSS";ss.length<=9;ss+="S")U(ss,tn);for(ss="S";ss.length<=9;ss+="S")K(ss,Vi);var rs=it("Milliseconds",!1);G("z",0,0,"zoneAbbr"),G("zz",0,0,"zoneName");var as=v.prototype;as.add=Qn,as.calendar=oi,as.clone=ni,as.diff=ui,as.endOf=Si,as.format=gi,as.from=vi,as.fromNow=yi,as.to=bi,as.toNow=_i,as.get=st,as.invalidAt=Ni,as.isAfter=si,as.isBefore=ri,as.isBetween=ai,as.isSame=hi,as.isSameOrAfter=di,as.isSameOrBefore=li,as.isValid=Pi,as.lang=es,as.locale=wi,as.localeData=xi,as.max=qn,as.min=Un,as.parsingFlags=Ii,as.set=rt,as.startOf=Di,as.subtract=ts,as.toArray=Mi,as.toObject=Oi,as.toDate=Ti,as.toISOString=mi,as.inspect=fi,as.toJSON=Ei,as.toString=ci,as.unix=Ci,as.valueOf=ki,as.creationData=Ai,as.year=bn,as.isLeapYear=et,as.weekYear=Li,as.isoWeekYear=Fi,as.quarter=as.quarters=zi,as.month=mt,as.daysInMonth=ft,as.week=as.weeks=Mt,as.isoWeek=as.isoWeeks=Ot,as.weeksInYear=ji,as.isoWeeksInYear=Hi,as.date=is,as.day=as.days=Ft,as.weekday=Ht,as.isoWeekday=jt,as.dayOfYear=Wi,as.hour=as.hours=An,as.minute=as.minutes=os,as.second=as.seconds=ns,as.millisecond=as.milliseconds=rs,as.utcOffset=je,as.utc=Ge,as.local=ze,as.parseZone=We,as.hasAlignedHourOffset=Ve,as.isDST=Be,as.isLocal=qe,as.isUtcOffset=Xe,as.isUtc=Ze,as.isUTC=Ze,as.zoneAbbr=Bi,as.zoneName=Ui,as.dates=D("dates accessor is deprecated. Use date instead.",is),as.months=D("months accessor is deprecated. Use month instead",mt),as.years=D("years accessor is deprecated. Use year instead",bn),as.zone=D("moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/",Ye),as.isDSTShifted=D("isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information",Ue);var hs=M.prototype;hs.calendar=O,hs.longDateFormat=E,hs.invalidDate=P,hs.ordinal=I,hs.preparse=Zi,hs.postformat=Zi,hs.relativeTime=N,hs.pastFuture=A,hs.set=C,hs.months=dt,hs.monthsShort=lt,hs.monthsParse=pt,hs.monthsRegex=vt,hs.monthsShortRegex=gt,hs.week=kt,hs.firstDayOfYear=Tt,hs.firstDayOfWeek=Ct,hs.weekdays=It,hs.weekdaysMin=At,hs.weekdaysShort=Nt,hs.weekdaysParse=Lt,hs.weekdaysRegex=Yt,hs.weekdaysShortRegex=Gt,hs.weekdaysMinRegex=zt,hs.isPM=Xt,hs.meridiem=Zt,Qt("en",{dayOfMonthOrdinalParse:/\d{1,2}(th|st|nd|rd)/,ordinal:function(t){var e=t%10;return t+(1===_(t%100/10)?"th":1===e?"st":2===e?"nd":3===e?"rd":"th")}}),e.lang=D("moment.lang is deprecated. Use moment.locale instead.",Qt),e.langData=D("moment.langData is deprecated. Use moment.localeData instead.",ie);var ds=Math.abs,ls=fo("ms"),us=fo("s"),ps=fo("m"),cs=fo("h"),ms=fo("d"),fs=fo("w"),gs=fo("M"),vs=fo("y"),ys=yo("milliseconds"),bs=yo("seconds"),_s=yo("minutes"),ws=yo("hours"),xs=yo("days"),Ds=yo("months"),Ss=yo("years"),ks=Math.round,Cs={ss:44,s:45,m:45,h:22,d:26,M:11},Ts=Math.abs,Ms=Ie.prototype;return Ms.isValid=Ee,Ms.abs=no,Ms.add=ro,Ms.subtract=ao,Ms.as=co,Ms.asMilliseconds=ls,Ms.asSeconds=us,Ms.asMinutes=ps,Ms.asHours=cs,Ms.asDays=ms,Ms.asWeeks=fs,Ms.asMonths=gs,Ms.asYears=vs,Ms.valueOf=mo,Ms._bubble=lo,Ms.clone=go,Ms.get=vo,Ms.milliseconds=ys,Ms.seconds=bs,Ms.minutes=_s,Ms.hours=ws,Ms.days=xs,Ms.weeks=bo,Ms.months=Ds,Ms.years=Ss,Ms.humanize=So,Ms.toISOString=Co,Ms.toString=Co,Ms.toJSON=Co,Ms.locale=wi,Ms.localeData=xi,Ms.toIsoString=D("toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)",Co),Ms.lang=es,G("X",0,0,"unix"),G("x",0,0,"valueOf"),U("x",en),U("X",sn),K("X",function(t,e,i){i._d=new Date(1e3*parseFloat(t,10))}),K("x",function(t,e,i){i._d=new Date(_(t))}),e.version="2.19.1",function(t){To=t}(ke),e.fn=as,e.min=Te,e.max=Me,e.now=Xn,e.utc=u,e.unix=qi,e.months=Qi,e.isDate=a,e.locale=Qt,e.invalid=f,e.duration=Ke,e.isMoment=y,e.weekdays=eo,e.parseZone=Xi,e.localeData=ie,e.isDuration=Ne,e.monthsShort=to,e.weekdaysMin=oo,e.defineLocale=te,e.updateLocale=ee,e.locales=oe,e.weekdaysShort=io,e.normalizeUnits=L,e.relativeTimeRounding=xo,e.relativeTimeThreshold=Do,e.calendarFormat=ii,e.prototype=as,e})}).call(e,i(124)(t))},function(t,e){t.exports=function(t){return t.webpackPolyfill||(t.deprecate=function(){},t.paths=[],t.children||(t.children=[]),Object.defineProperty(t,"loaded",{enumerable:!0,get:function(){return t.l}}),Object.defineProperty(t,"id",{enumerable:!0,get:function(){return t.i}}),t.webpackPolyfill=1),t}},function(t,e){function i(t){throw new Error("Cannot find module '"+t+"'.")}i.keys=function(){return[]},i.resolve=i,t.exports=i,i.id=125},function(t,e,i){(function(e){function i(t,e,i){var o=e&&i||0,n=0;for(e=e||[],t.toLowerCase().replace(/[0-9a-f]{2}/g,function(t){n<16&&(e[o+n++]=u[t])});n<16;)e[o+n++]=0;return e}function o(t,e){var i=e||0,o=l;return o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+"-"+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]+o[t[i++]]}function n(t,e,i){var n=e&&i||0,s=e||[];t=t||{};var r=void 0!==t.clockseq?t.clockseq:f,a=void 0!==t.msecs?t.msecs:(new Date).getTime(),h=void 0!==t.nsecs?t.nsecs:v+1,d=a-g+(h-v)/1e4;if(d<0&&void 0===t.clockseq&&(r=r+1&16383),(d<0||a>g)&&void 0===t.nsecs&&(h=0),h>=1e4)throw new Error("uuid.v1(): Can't create more than 10M uuids/sec");g=a,v=h,f=r,a+=122192928e5;var l=(1e4*(268435455&a)+h)%4294967296;s[n++]=l>>>24&255,s[n++]=l>>>16&255,s[n++]=l>>>8&255,s[n++]=255&l;var u=a/4294967296*1e4&268435455;s[n++]=u>>>8&255,s[n++]=255&u,s[n++]=u>>>24&15|16,s[n++]=u>>>16&255,s[n++]=r>>>8|128,s[n++]=255&r;for(var p=t.node||m,c=0;c<6;c++)s[n+c]=p[c];return e||o(s)}function s(t,e,i){var n=e&&i||0;"string"==typeof t&&(e="binary"==t?new Array(16):null,t=null),t=t||{};var s=t.random||(t.rng||r)();if(s[6]=15&s[6]|64,s[8]=63&s[8]|128,e)for(var a=0;a<16;a++)e[n+a]=s[a];return e||o(s)}var r,a="undefined"!=typeof window?window:void 0!==e?e:null;if(a&&a.crypto&&crypto.getRandomValues){var h=new Uint8Array(16);r=function(){return crypto.getRandomValues(h),h}}if(!r){var d=new Array(16);r=function(){for(var t,e=0;e<16;e++)0==(3&e)&&(t=4294967296*Math.random()),d[e]=t>>>((3&e)<<3)&255;return d}}for(var l=[],u={},p=0;p<256;p++)l[p]=(p+256).toString(16).substr(1),u[l[p]]=p;var c=r(),m=[1|c[0],c[1],c[2],c[3],c[4],c[5]],f=16383&(c[6]<<8|c[7]),g=0,v=0,y=s;y.v1=n,y.v4=s,y.parse=i,y.unparse=o,t.exports=y}).call(e,i(127))},function(t,e){var i;i=function(){return this}();try{i=i||Function("return this")()||(0,eval)("this")}catch(t){"object"==typeof window&&(i=window)}t.exports=i},function(t,e,i){var o=i(2),n=o.JSON||(o.JSON={stringify:JSON.stringify});t.exports=function(t){return n.stringify.apply(n,arguments)}},function(t,e,i){function o(t,e,i,n){if(!(this instanceof o))throw new SyntaxError("Constructor must be called with the new operator");if(!(Array.isArray(i)||i instanceof d||i instanceof l)&&i instanceof Object){var s=n;n=i,i=s}n&&n.throttleRedraw&&console.warn('Timeline option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var r=this;if(this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:a,width:null,height:null,maxHeight:null,minHeight:null},this.options=h.deepExtend({},this.defaultOptions),this._create(t),!n||n&&void 0===n.rtl){this.dom.root.style.visibility="hidden";for(var p,f=this.dom.root;!p&&f;)p=window.getComputedStyle(f,null).direction,f=f.parentElement;this.options.rtl=p&&"rtl"==p.toLowerCase()}else this.options.rtl=n.rtl;this.options.rollingMode=n&&n.rollingMode,this.options.onInitialDrawComplete=n&&n.onInitialDrawComplete,this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{getScale:function(){return r.timeAxis.step.scale},getStep:function(){return r.timeAxis.step.step},toScreen:r._toScreen.bind(r),toGlobalScreen:r._toGlobalScreen.bind(r),toTime:r._toTime.bind(r),toGlobalTime:r._toGlobalTime.bind(r)}},this.range=new u(this.body,this.options),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new c(this.body,this.options),this.timeAxis2=null,this.components.push(this.timeAxis),this.currentTime=new m(this.body,this.options),this.components.push(this.currentTime),this.itemSet=new g(this.body,this.options),this.components.push(this.itemSet),this.itemsData=null,this.groupsData=null,this.dom.root.onclick=function(t){r.emit("click",r.getEventProperties(t))},this.dom.root.ondblclick=function(t){r.emit("doubleClick",r.getEventProperties(t))},this.dom.root.oncontextmenu=function(t){r.emit("contextmenu",r.getEventProperties(t))},this.dom.root.onmouseover=function(t){r.emit("mouseOver",r.getEventProperties(t))},window.PointerEvent?(this.dom.root.onpointerdown=function(t){r.emit("mouseDown",r.getEventProperties(t))},this.dom.root.onpointermove=function(t){r.emit("mouseMove",r.getEventProperties(t))},this.dom.root.onpointerup=function(t){r.emit("mouseUp",r.getEventProperties(t))}):(this.dom.root.onmousemove=function(t){r.emit("mouseMove",r.getEventProperties(t))},this.dom.root.onmousedown=function(t){r.emit("mouseDown",r.getEventProperties(t))},this.dom.root.onmouseup=function(t){r.emit("mouseUp",r.getEventProperties(t))}),this.initialFitDone=!1,this.on("changed",function(){if(null!=this.itemsData&&!this.options.rollingMode){if(!r.initialFitDone)if(r.initialFitDone=!0,void 0!=r.options.start||void 0!=r.options.end){if(void 0==r.options.start||void 0==r.options.end)var t=r.getItemRange();var e=void 0!=r.options.start?r.options.start:t.min,i=void 0!=r.options.end?r.options.end:t.max;r.setWindow(e,i,{animation:!1})}else r.fit({animation:!1});!r.initialDrawDone&&r.initialRangeChangeDone&&(r.initialDrawDone=!0,r.dom.root.style.visibility="visible",r.options.onInitialDrawComplete&&setTimeout(function(){return r.options.onInitialDrawComplete()},0))}}),n&&this.setOptions(n),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}function n(t){return h.convert(t.data.start,"Date").valueOf()}function s(t){var e=void 0!=t.data.end?t.data.end:t.data.start;return h.convert(e,"Date").valueOf()}function r(t,e){var i=t.props.leftContainer.height,o=t.props.left.height,n=e.parent,s=n.top,r=!0,a=t.timeAxis.options.orientation.axis,h=function(){return"bottom"==a?n.height-e.top-e.height:e.top},d=-1*t._getScrollTop(),l=s+h(),u=e.height;return l<d?s+i<=s+h()+u&&(s+=h()-t.itemSet.options.margin.item.vertical):l+u>d+i?s+=h()+u-i+t.itemSet.options.margin.item.vertical:r=!1,s=Math.min(s,o-i),{shouldScroll:r,scrollOffset:s,itemTop:l}}var a=i(3),h=i(0),d=i(15),l=i(20),u=i(47),p=i(48),c=i(28),m=i(50),f=i(29),g=i(70),v=i(32).printStyle,y=i(75).allOptions,b=i(75).configureOptions,_=i(76).default,w=i(32).default;o.prototype=new p,o.prototype._createConfigurator=function(){return new _(this,this.dom.container,b)},o.prototype.redraw=function(){this.itemSet&&this.itemSet.markDirty({refreshItems:!0}),this._redraw()},o.prototype.setOptions=function(t){if(!0===w.validate(t,y)&&console.log("%cErrors have been found in the supplied options object.",v),p.prototype.setOptions.call(this,t),"type"in t&&t.type!==this.options.type){this.options.type=t.type;var e=this.itemsData;if(e){var i=this.getSelection();this.setItems(null),this.setItems(e),this.setSelection(i)}}},o.prototype.setItems=function(t){var e;e=t?t instanceof d||t instanceof l?t:new d(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.itemSet&&this.itemSet.setItems(e)},o.prototype.setGroups=function(t){var e;if(t){var i=function(t){return!1!==t.visible};e=t instanceof d||t instanceof l?new l(t,{filter:i}):new d(t.filter(i))}else e=null;this.groupsData=e,this.itemSet.setGroups(e)},o.prototype.setData=function(t){t&&t.groups&&this.setGroups(t.groups),t&&t.items&&this.setItems(t.items)},o.prototype.setSelection=function(t,e){this.itemSet&&this.itemSet.setSelection(t),e&&e.focus&&this.focus(t,e)},o.prototype.getSelection=function(){return this.itemSet&&this.itemSet.getSelection()||[]},o.prototype.focus=function(t,e){if(this.itemsData&&void 0!=t){var i=Array.isArray(t)?t:[t],o=this.itemsData.getDataSet().get(i,{type:{start:"Date",end:"Date"}}),n=null,s=null;if(o.forEach(function(t){var e=t.start.valueOf(),i="end"in t?t.end.valueOf():t.start.valueOf();(null===n||e<n)&&(n=e),(null===s||i>s)&&(s=i)}),null!==n&&null!==s){var a=this,h=this.itemSet.items[i[0]],d=-1*this._getScrollTop(),l=null,u=function(t,e,i){var o=r(a,h);if(l||(l=o),l.itemTop!=o.itemTop||l.shouldScroll){l.itemTop!=o.itemTop&&o.shouldScroll&&(l=o,d=-1*a._getScrollTop());var n=d,s=l.scrollOffset,u=i?s:n+(s-n)*t;a._setScrollTop(-u),e||a._redraw()}},p=function(){var t=r(a,h);t.shouldScroll&&t.itemTop!=l.itemTop&&(a._setScrollTop(-t.scrollOffset),a._redraw())},c=function(){p(),setTimeout(p,100)},m=(n+s)/2,f=Math.max(this.range.end-this.range.start,1.1*(s-n)),g=!e||void 0===e.animation||e.animation;g||(l={shouldScroll:!1,scrollOffset:-1,itemTop:-1}),this.range.setRange(m-f/2,m+f/2,{animation:g},c,u)}}},o.prototype.fit=function(t,e){var i,o=!t||void 0===t.animation||t.animation,n=this.itemsData&&this.itemsData.getDataSet();1===n.length&&void 0===n.get()[0].end?(i=this.getDataRange(),this.moveTo(i.min.valueOf(),{animation:o},e)):(i=this.getItemRange(),this.range.setRange(i.min,i.max,{animation:o},e))},o.prototype.getItemRange=function(){var t=this.getDataRange(),e=null!==t.min?t.min.valueOf():null,i=null!==t.max?t.max.valueOf():null,o=null,r=null;if(null!=e&&null!=i){var a=i-e;a<=0&&(a=10);var d=a/this.props.center.width,l={},u=0;h.forEach(this.itemSet.items,function(t,e){if(t.groupShowing){l[e]=t.redraw(!0),u=l[e].length}});if(u>0)for(var p=0;p<u;p++)h.forEach(l,function(t){t[p]()});if(h.forEach(this.itemSet.items,function(t){var a,h,l=n(t),u=s(t);this.options.rtl?(a=l-(t.getWidthRight()+10)*d,h=u+(t.getWidthLeft()+10)*d):(a=l-(t.getWidthLeft()+10)*d,h=u+(t.getWidthRight()+10)*d),a<e&&(e=a,o=t),h>i&&(i=h,r=t)}.bind(this)),o&&r){var c=o.getWidthLeft()+10,m=r.getWidthRight()+10,f=this.props.center.width-c-m;f>0&&(this.options.rtl?(e=n(o)-m*a/f,i=s(r)+c*a/f):(e=n(o)-c*a/f,i=s(r)+m*a/f))}}return{min:null!=e?new Date(e):null,max:null!=i?new Date(i):null}},o.prototype.getDataRange=function(){var t=null,e=null,i=this.itemsData&&this.itemsData.getDataSet();return i&&i.forEach(function(i){var o=h.convert(i.start,"Date").valueOf(),n=h.convert(void 0!=i.end?i.end:i.start,"Date").valueOf();(null===t||o<t)&&(t=o),(null===e||n>e)&&(e=n)}),{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},o.prototype.getEventProperties=function(t){var e,i=t.center?t.center.x:t.clientX,o=t.center?t.center.y:t.clientY;e=this.options.rtl?h.getAbsoluteRight(this.dom.centerContainer)-i:i-h.getAbsoluteLeft(this.dom.centerContainer);var n=o-h.getAbsoluteTop(this.dom.centerContainer),s=this.itemSet.itemFromTarget(t),r=this.itemSet.groupFromTarget(t),a=f.customTimeFromTarget(t),d=this.itemSet.options.snap||null,l=this.body.util.getScale(),u=this.body.util.getStep(),p=this._toTime(e),c=d?d(p,l,u):p,m=h.getTarget(t),g=null;return null!=s?g="item":null!=a?g="custom-time":h.hasParent(m,this.timeAxis.dom.foreground)?g="axis":this.timeAxis2&&h.hasParent(m,this.timeAxis2.dom.foreground)?g="axis":h.hasParent(m,this.itemSet.dom.labelSet)?g="group-label":h.hasParent(m,this.currentTime.bar)?g="current-time":h.hasParent(m,this.dom.center)&&(g="background"),{event:t,item:s?s.id:null,group:r?r.groupId:null,what:g,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:e,y:n,time:p,snappedTime:c}},o.prototype.toggleRollingMode=function(){this.range.rolling?this.range.stopRolling():(void 0==this.options.rollingMode&&this.setOptions(this.options),this.range.startRolling())},t.exports=o},function(t,e,i){var o,n,s;!function(i){n=[],o=i,void 0!==(s="function"==typeof o?o.apply(e,n):o)&&(t.exports=s)}(function(){var t=null;return function e(i,o){function n(t){return t.match(/[^ ]+/g)}function s(e){if("hammer.input"!==e.type){if(e.srcEvent._handled||(e.srcEvent._handled={}),e.srcEvent._handled[e.type])return;e.srcEvent._handled[e.type]=!0}var i=!1;e.stopPropagation=function(){i=!0};var o=e.srcEvent.stopPropagation.bind(e.srcEvent);"function"==typeof o&&(e.srcEvent.stopPropagation=function(){o(),e.stopPropagation()}),e.firstTarget=t;for(var n=t;n&&!i;){var s=n.hammer;if(s)for(var r,a=0;a<s.length;a++)if(r=s[a]._handlers[e.type])for(var h=0;h<r.length&&!i;h++)r[h](e);n=n.parentNode}}var r=o||{preventDefault:!1};if(i.Manager){var a=i,h=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a(t,o),o)};return a.assign(h,a),h.Manager=function(t,i){var o=Object.create(r);return i&&a.assign(o,i),e(new a.Manager(t,o),o)},h}var d=Object.create(i),l=i.element;return l.hammer||(l.hammer=[]),l.hammer.push(d),i.on("hammer.input",function(e){!0!==r.preventDefault&&r.preventDefault!==e.pointerType||e.preventDefault(),e.isFirst&&(t=e.target)}),d._handlers={},d.on=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o||(d._handlers[t]=o=[],i.on(t,s)),o.push(e)}),d},d.off=function(t,e){return n(t).forEach(function(t){var o=d._handlers[t];o&&(o=e?o.filter(function(t){return t!==e}):[],o.length>0?d._handlers[t]=o:(i.off(t,s),delete d._handlers[t]))}),d},d.emit=function(e,o){t=o.target,i.emit(e,o)},d.destroy=function(){var t=i.element.hammer,e=t.indexOf(d);-1!==e&&t.splice(e,1),t.length||delete i.element.hammer,d._handlers={},i.destroy()},d}})},function(t,e,i){var o;/*! Hammer.JS - v2.0.7 - 2016-04-22
+ * http://hammerjs.github.io/
+ *
+ * Copyright (c) 2016 Jorik Tangelder;
+ * Licensed under the MIT license */
+!function(n,s,r,a){function h(t,e,i){return setTimeout(c(t,i),e)}function d(t,e,i){return!!Array.isArray(t)&&(l(t,i[e],i),!0)}function l(t,e,i){var o;if(t)if(t.forEach)t.forEach(e,i);else if(t.length!==a)for(o=0;o<t.length;)e.call(i,t[o],o,t),o++;else for(o in t)t.hasOwnProperty(o)&&e.call(i,t[o],o,t)}function u(t,e,i){var o="DEPRECATED METHOD: "+e+"\n"+i+" AT \n";return function(){var e=new Error("get-stack-trace"),i=e&&e.stack?e.stack.replace(/^[^\(]+?[\n$]/gm,"").replace(/^\s+at\s+/gm,"").replace(/^Object.<anonymous>\s*\(/gm,"{anonymous}()@"):"Unknown Stack Trace",s=n.console&&(n.console.warn||n.console.log);return s&&s.call(n.console,o,i),t.apply(this,arguments)}}function p(t,e,i){var o,n=e.prototype;o=t.prototype=Object.create(n),o.constructor=t,o._super=n,i&&mt(o,i)}function c(t,e){return function(){return t.apply(e,arguments)}}function m(t,e){return typeof t==vt?t.apply(e?e[0]||a:a,e):t}function f(t,e){return t===a?e:t}function g(t,e,i){l(_(e),function(e){t.addEventListener(e,i,!1)})}function v(t,e,i){l(_(e),function(e){t.removeEventListener(e,i,!1)})}function y(t,e){for(;t;){if(t==e)return!0;t=t.parentNode}return!1}function b(t,e){return t.indexOf(e)>-1}function _(t){return t.trim().split(/\s+/g)}function w(t,e,i){if(t.indexOf&&!i)return t.indexOf(e);for(var o=0;o<t.length;){if(i&&t[o][i]==e||!i&&t[o]===e)return o;o++}return-1}function x(t){return Array.prototype.slice.call(t,0)}function D(t,e,i){for(var o=[],n=[],s=0;s<t.length;){var r=e?t[s][e]:t[s];w(n,r)<0&&o.push(t[s]),n[s]=r,s++}return i&&(o=e?o.sort(function(t,i){return t[e]>i[e]}):o.sort()),o}function S(t,e){for(var i,o,n=e[0].toUpperCase()+e.slice(1),s=0;s<ft.length;){if(i=ft[s],(o=i?i+n:e)in t)return o;s++}return a}function k(){return Dt++}function C(t){var e=t.ownerDocument||t;return e.defaultView||e.parentWindow||n}function T(t,e){var i=this;this.manager=t,this.callback=e,this.element=t.element,this.target=t.options.inputTarget,this.domHandler=function(e){m(t.options.enable,[t])&&i.handler(e)},this.init()}function M(t){var e=t.options.inputClass;return new(e||(Ct?z:Tt?B:kt?q:G))(t,O)}function O(t,e,i){var o=i.pointers.length,n=i.changedPointers.length,s=e&Ot&&o-n==0,r=e&(Pt|It)&&o-n==0;i.isFirst=!!s,i.isFinal=!!r,s&&(t.session={}),i.eventType=e,E(t,i),t.emit("hammer.input",i),t.recognize(i),t.session.prevInput=i}function E(t,e){var i=t.session,o=e.pointers,n=o.length;i.firstInput||(i.firstInput=N(e)),n>1&&!i.firstMultiple?i.firstMultiple=N(e):1===n&&(i.firstMultiple=!1);var s=i.firstInput,r=i.firstMultiple,a=r?r.center:s.center,h=e.center=A(o);e.timeStamp=_t(),e.deltaTime=e.timeStamp-s.timeStamp,e.angle=H(a,h),e.distance=F(a,h),P(i,e),e.offsetDirection=L(e.deltaX,e.deltaY);var d=R(e.deltaTime,e.deltaX,e.deltaY);e.overallVelocityX=d.x,e.overallVelocityY=d.y,e.overallVelocity=bt(d.x)>bt(d.y)?d.x:d.y,e.scale=r?Y(r.pointers,o):1,e.rotation=r?j(r.pointers,o):0,e.maxPointers=i.prevInput?e.pointers.length>i.prevInput.maxPointers?e.pointers.length:i.prevInput.maxPointers:e.pointers.length,I(i,e);var l=t.element;y(e.srcEvent.target,l)&&(l=e.srcEvent.target),e.target=l}function P(t,e){var i=e.center,o=t.offsetDelta||{},n=t.prevDelta||{},s=t.prevInput||{};e.eventType!==Ot&&s.eventType!==Pt||(n=t.prevDelta={x:s.deltaX||0,y:s.deltaY||0},o=t.offsetDelta={x:i.x,y:i.y}),e.deltaX=n.x+(i.x-o.x),e.deltaY=n.y+(i.y-o.y)}function I(t,e){var i,o,n,s,r=t.lastInterval||e,h=e.timeStamp-r.timeStamp;if(e.eventType!=It&&(h>Mt||r.velocity===a)){var d=e.deltaX-r.deltaX,l=e.deltaY-r.deltaY,u=R(h,d,l);o=u.x,n=u.y,i=bt(u.x)>bt(u.y)?u.x:u.y,s=L(d,l),t.lastInterval=e}else i=r.velocity,o=r.velocityX,n=r.velocityY,s=r.direction;e.velocity=i,e.velocityX=o,e.velocityY=n,e.direction=s}function N(t){for(var e=[],i=0;i<t.pointers.length;)e[i]={clientX:yt(t.pointers[i].clientX),clientY:yt(t.pointers[i].clientY)},i++;return{timeStamp:_t(),pointers:e,center:A(e),deltaX:t.deltaX,deltaY:t.deltaY}}function A(t){var e=t.length;if(1===e)return{x:yt(t[0].clientX),y:yt(t[0].clientY)};for(var i=0,o=0,n=0;n<e;)i+=t[n].clientX,o+=t[n].clientY,n++;return{x:yt(i/e),y:yt(o/e)}}function R(t,e,i){return{x:e/t||0,y:i/t||0}}function L(t,e){return t===e?Nt:bt(t)>=bt(e)?t<0?At:Rt:e<0?Lt:Ft}function F(t,e,i){i||(i=Gt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return Math.sqrt(o*o+n*n)}function H(t,e,i){i||(i=Gt);var o=e[i[0]]-t[i[0]],n=e[i[1]]-t[i[1]];return 180*Math.atan2(n,o)/Math.PI}function j(t,e){return H(e[1],e[0],zt)+H(t[1],t[0],zt)}function Y(t,e){return F(e[0],e[1],zt)/F(t[0],t[1],zt)}function G(){this.evEl=Vt,this.evWin=Bt,this.pressed=!1,T.apply(this,arguments)}function z(){this.evEl=Xt,this.evWin=Zt,T.apply(this,arguments),this.store=this.manager.session.pointerEvents=[]}function W(){this.evTarget=Jt,this.evWin=$t,this.started=!1,T.apply(this,arguments)}function V(t,e){var i=x(t.touches),o=x(t.changedTouches);return e&(Pt|It)&&(i=D(i.concat(o),"identifier",!0)),[i,o]}function B(){this.evTarget=te,this.targetIds={},T.apply(this,arguments)}function U(t,e){var i=x(t.touches),o=this.targetIds;if(e&(Ot|Et)&&1===i.length)return o[i[0].identifier]=!0,[i,i];var n,s,r=x(t.changedTouches),a=[],h=this.target;if(s=i.filter(function(t){return y(t.target,h)}),e===Ot)for(n=0;n<s.length;)o[s[n].identifier]=!0,n++;for(n=0;n<r.length;)o[r[n].identifier]&&a.push(r[n]),e&(Pt|It)&&delete o[r[n].identifier],n++;return a.length?[D(s.concat(a),"identifier",!0),a]:void 0}function q(){T.apply(this,arguments);var t=c(this.handler,this);this.touch=new B(this.manager,t),this.mouse=new G(this.manager,t),this.primaryTouch=null,this.lastTouches=[]}function X(t,e){t&Ot?(this.primaryTouch=e.changedPointers[0].identifier,Z.call(this,e)):t&(Pt|It)&&Z.call(this,e)}function Z(t){var e=t.changedPointers[0];if(e.identifier===this.primaryTouch){var i={x:e.clientX,y:e.clientY};this.lastTouches.push(i);var o=this.lastTouches,n=function(){var t=o.indexOf(i);t>-1&&o.splice(t,1)};setTimeout(n,ee)}}function K(t){for(var e=t.srcEvent.clientX,i=t.srcEvent.clientY,o=0;o<this.lastTouches.length;o++){var n=this.lastTouches[o],s=Math.abs(e-n.x),r=Math.abs(i-n.y);if(s<=ie&&r<=ie)return!0}return!1}function J(t,e){this.manager=t,this.set(e)}function $(t){if(b(t,ae))return ae;var e=b(t,he),i=b(t,de);return e&&i?ae:e||i?e?he:de:b(t,re)?re:se}function Q(t){this.options=mt({},this.defaults,t||{}),this.id=k(),this.manager=null,this.options.enable=f(this.options.enable,!0),this.state=ue,this.simultaneous={},this.requireFail=[]}function tt(t){return t&ge?"cancel":t&me?"end":t&ce?"move":t&pe?"start":""}function et(t){return t==Ft?"down":t==Lt?"up":t==At?"left":t==Rt?"right":""}function it(t,e){var i=e.manager;return i?i.get(t):t}function ot(){Q.apply(this,arguments)}function nt(){ot.apply(this,arguments),this.pX=null,this.pY=null}function st(){ot.apply(this,arguments)}function rt(){Q.apply(this,arguments),this._timer=null,this._input=null}function at(){ot.apply(this,arguments)}function ht(){ot.apply(this,arguments)}function dt(){Q.apply(this,arguments),this.pTime=!1,this.pCenter=!1,this._timer=null,this._input=null,this.count=0}function lt(t,e){return e=e||{},e.recognizers=f(e.recognizers,lt.defaults.preset),new ut(t,e)}function ut(t,e){this.options=mt({},lt.defaults,e||{}),this.options.inputTarget=this.options.inputTarget||t,this.handlers={},this.session={},this.recognizers=[],this.oldCssProps={},this.element=t,this.input=M(this),this.touchAction=new J(this,this.options.touchAction),pt(this,!0),l(this.options.recognizers,function(t){var e=this.add(new t[0](t[1]));t[2]&&e.recognizeWith(t[2]),t[3]&&e.requireFailure(t[3])},this)}function pt(t,e){var i=t.element;if(i.style){var o;l(t.options.cssProps,function(n,s){o=S(i.style,s),e?(t.oldCssProps[o]=i.style[o],i.style[o]=n):i.style[o]=t.oldCssProps[o]||""}),e||(t.oldCssProps={})}}function ct(t,e){var i=s.createEvent("Event");i.initEvent(t,!0,!0),i.gesture=e,e.target.dispatchEvent(i)}var mt,ft=["","webkit","Moz","MS","ms","o"],gt=s.createElement("div"),vt="function",yt=Math.round,bt=Math.abs,_t=Date.now;mt="function"!=typeof Object.assign?function(t){if(t===a||null===t)throw new TypeError("Cannot convert undefined or null to object");for(var e=Object(t),i=1;i<arguments.length;i++){var o=arguments[i];if(o!==a&&null!==o)for(var n in o)o.hasOwnProperty(n)&&(e[n]=o[n])}return e}:Object.assign;var wt=u(function(t,e,i){for(var o=Object.keys(e),n=0;n<o.length;)(!i||i&&t[o[n]]===a)&&(t[o[n]]=e[o[n]]),n++;return t},"extend","Use `assign`."),xt=u(function(t,e){return wt(t,e,!0)},"merge","Use `assign`."),Dt=1,St=/mobile|tablet|ip(ad|hone|od)|android/i,kt="ontouchstart"in n,Ct=S(n,"PointerEvent")!==a,Tt=kt&&St.test(navigator.userAgent),Mt=25,Ot=1,Et=2,Pt=4,It=8,Nt=1,At=2,Rt=4,Lt=8,Ft=16,Ht=At|Rt,jt=Lt|Ft,Yt=Ht|jt,Gt=["x","y"],zt=["clientX","clientY"];T.prototype={handler:function(){},init:function(){this.evEl&&g(this.element,this.evEl,this.domHandler),this.evTarget&&g(this.target,this.evTarget,this.domHandler),this.evWin&&g(C(this.element),this.evWin,this.domHandler)},destroy:function(){this.evEl&&v(this.element,this.evEl,this.domHandler),this.evTarget&&v(this.target,this.evTarget,this.domHandler),this.evWin&&v(C(this.element),this.evWin,this.domHandler)}};var Wt={mousedown:Ot,mousemove:Et,mouseup:Pt},Vt="mousedown",Bt="mousemove mouseup";p(G,T,{handler:function(t){var e=Wt[t.type];e&Ot&&0===t.button&&(this.pressed=!0),e&Et&&1!==t.which&&(e=Pt),this.pressed&&(e&Pt&&(this.pressed=!1),this.callback(this.manager,e,{pointers:[t],changedPointers:[t],pointerType:"mouse",srcEvent:t}))}});var Ut={pointerdown:Ot,pointermove:Et,pointerup:Pt,pointercancel:It,pointerout:It},qt={2:"touch",3:"pen",4:"mouse",5:"kinect"},Xt="pointerdown",Zt="pointermove pointerup pointercancel";n.MSPointerEvent&&!n.PointerEvent&&(Xt="MSPointerDown",Zt="MSPointerMove MSPointerUp MSPointerCancel"),p(z,T,{handler:function(t){var e=this.store,i=!1,o=t.type.toLowerCase().replace("ms",""),n=Ut[o],s=qt[t.pointerType]||t.pointerType,r="touch"==s,a=w(e,t.pointerId,"pointerId");n&Ot&&(0===t.button||r)?a<0&&(e.push(t),a=e.length-1):n&(Pt|It)&&(i=!0),a<0||(e[a]=t,this.callback(this.manager,n,{pointers:e,changedPointers:[t],pointerType:s,srcEvent:t}),i&&e.splice(a,1))}});var Kt={touchstart:Ot,touchmove:Et,touchend:Pt,touchcancel:It},Jt="touchstart",$t="touchstart touchmove touchend touchcancel";p(W,T,{handler:function(t){var e=Kt[t.type];if(e===Ot&&(this.started=!0),this.started){var i=V.call(this,t,e);e&(Pt|It)&&i[0].length-i[1].length==0&&(this.started=!1),this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}}});var Qt={touchstart:Ot,touchmove:Et,touchend:Pt,touchcancel:It},te="touchstart touchmove touchend touchcancel";p(B,T,{handler:function(t){var e=Qt[t.type],i=U.call(this,t,e);i&&this.callback(this.manager,e,{pointers:i[0],changedPointers:i[1],pointerType:"touch",srcEvent:t})}});var ee=2500,ie=25;p(q,T,{handler:function(t,e,i){var o="touch"==i.pointerType,n="mouse"==i.pointerType;if(!(n&&i.sourceCapabilities&&i.sourceCapabilities.firesTouchEvents)){if(o)X.call(this,e,i);else if(n&&K.call(this,i))return;this.callback(t,e,i)}},destroy:function(){this.touch.destroy(),this.mouse.destroy()}});var oe=S(gt.style,"touchAction"),ne=oe!==a,se="auto",re="manipulation",ae="none",he="pan-x",de="pan-y",le=function(){if(!ne)return!1;var t={},e=n.CSS&&n.CSS.supports;return["auto","manipulation","pan-y","pan-x","pan-x pan-y","none"].forEach(function(i){t[i]=!e||n.CSS.supports("touch-action",i)}),t}();J.prototype={set:function(t){"compute"==t&&(t=this.compute()),ne&&this.manager.element.style&&le[t]&&(this.manager.element.style[oe]=t),this.actions=t.toLowerCase().trim()},update:function(){this.set(this.manager.options.touchAction)},compute:function(){var t=[];return l(this.manager.recognizers,function(e){m(e.options.enable,[e])&&(t=t.concat(e.getTouchAction()))}),$(t.join(" "))},preventDefaults:function(t){var e=t.srcEvent,i=t.offsetDirection;if(this.manager.session.prevented)return void e.preventDefault();var o=this.actions,n=b(o,ae)&&!le[ae],s=b(o,de)&&!le[de],r=b(o,he)&&!le[he];if(n){var a=1===t.pointers.length,h=t.distance<2,d=t.deltaTime<250;if(a&&h&&d)return}return r&&s?void 0:n||s&&i&Ht||r&&i&jt?this.preventSrc(e):void 0},preventSrc:function(t){this.manager.session.prevented=!0,t.preventDefault()}};var ue=1,pe=2,ce=4,me=8,fe=me,ge=16;Q.prototype={defaults:{},set:function(t){return mt(this.options,t),this.manager&&this.manager.touchAction.update(),this},recognizeWith:function(t){if(d(t,"recognizeWith",this))return this;var e=this.simultaneous;return t=it(t,this),e[t.id]||(e[t.id]=t,t.recognizeWith(this)),this},dropRecognizeWith:function(t){return d(t,"dropRecognizeWith",this)?this:(t=it(t,this),delete this.simultaneous[t.id],this)},requireFailure:function(t){if(d(t,"requireFailure",this))return this;var e=this.requireFail;return t=it(t,this),-1===w(e,t)&&(e.push(t),t.requireFailure(this)),this},dropRequireFailure:function(t){if(d(t,"dropRequireFailure",this))return this;t=it(t,this);var e=w(this.requireFail,t);return e>-1&&this.requireFail.splice(e,1),this},hasRequireFailures:function(){return this.requireFail.length>0},canRecognizeWith:function(t){return!!this.simultaneous[t.id]},emit:function(t){function e(e){i.manager.emit(e,t)}var i=this,o=this.state;o<me&&e(i.options.event+tt(o)),e(i.options.event),t.additionalEvent&&e(t.additionalEvent),o>=me&&e(i.options.event+tt(o))},tryEmit:function(t){if(this.canEmit())return this.emit(t);this.state=32},canEmit:function(){for(var t=0;t<this.requireFail.length;){if(!(this.requireFail[t].state&(32|ue)))return!1;t++}return!0},recognize:function(t){var e=mt({},t);if(!m(this.options.enable,[this,e]))return this.reset(),void(this.state=32);this.state&(fe|ge|32)&&(this.state=ue),this.state=this.process(e),this.state&(pe|ce|me|ge)&&this.tryEmit(e)},process:function(t){},getTouchAction:function(){},reset:function(){}},p(ot,Q,{defaults:{pointers:1},attrTest:function(t){var e=this.options.pointers;return 0===e||t.pointers.length===e},process:function(t){var e=this.state,i=t.eventType,o=e&(pe|ce),n=this.attrTest(t);return o&&(i&It||!n)?e|ge:o||n?i&Pt?e|me:e&pe?e|ce:pe:32}}),p(nt,ot,{defaults:{event:"pan",threshold:10,pointers:1,direction:Yt},getTouchAction:function(){var t=this.options.direction,e=[];return t&Ht&&e.push(de),t&jt&&e.push(he),e},directionTest:function(t){var e=this.options,i=!0,o=t.distance,n=t.direction,s=t.deltaX,r=t.deltaY;return n&e.direction||(e.direction&Ht?(n=0===s?Nt:s<0?At:Rt,i=s!=this.pX,o=Math.abs(t.deltaX)):(n=0===r?Nt:r<0?Lt:Ft,i=r!=this.pY,o=Math.abs(t.deltaY))),t.direction=n,i&&o>e.threshold&&n&e.direction},attrTest:function(t){return ot.prototype.attrTest.call(this,t)&&(this.state&pe||!(this.state&pe)&&this.directionTest(t))},emit:function(t){this.pX=t.deltaX,this.pY=t.deltaY;var e=et(t.direction);e&&(t.additionalEvent=this.options.event+e),this._super.emit.call(this,t)}}),p(st,ot,{defaults:{event:"pinch",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.scale-1)>this.options.threshold||this.state&pe)},emit:function(t){if(1!==t.scale){var e=t.scale<1?"in":"out";t.additionalEvent=this.options.event+e}this._super.emit.call(this,t)}}),p(rt,Q,{defaults:{event:"press",pointers:1,time:251,threshold:9},getTouchAction:function(){return[se]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime>e.time;if(this._input=t,!o||!i||t.eventType&(Pt|It)&&!n)this.reset();else if(t.eventType&Ot)this.reset(),this._timer=h(function(){this.state=fe,this.tryEmit()},e.time,this);else if(t.eventType&Pt)return fe;return 32},reset:function(){clearTimeout(this._timer)},emit:function(t){this.state===fe&&(t&&t.eventType&Pt?this.manager.emit(this.options.event+"up",t):(this._input.timeStamp=_t(),this.manager.emit(this.options.event,this._input)))}}),p(at,ot,{defaults:{event:"rotate",threshold:0,pointers:2},getTouchAction:function(){return[ae]},attrTest:function(t){return this._super.attrTest.call(this,t)&&(Math.abs(t.rotation)>this.options.threshold||this.state&pe)}}),p(ht,ot,{defaults:{event:"swipe",threshold:10,velocity:.3,direction:Ht|jt,pointers:1},getTouchAction:function(){return nt.prototype.getTouchAction.call(this)},attrTest:function(t){var e,i=this.options.direction;return i&(Ht|jt)?e=t.overallVelocity:i&Ht?e=t.overallVelocityX:i&jt&&(e=t.overallVelocityY),this._super.attrTest.call(this,t)&&i&t.offsetDirection&&t.distance>this.options.threshold&&t.maxPointers==this.options.pointers&&bt(e)>this.options.velocity&&t.eventType&Pt},emit:function(t){var e=et(t.offsetDirection);e&&this.manager.emit(this.options.event+e,t),this.manager.emit(this.options.event,t)}}),p(dt,Q,{defaults:{event:"tap",pointers:1,taps:1,interval:300,time:250,threshold:9,posThreshold:10},getTouchAction:function(){return[re]},process:function(t){var e=this.options,i=t.pointers.length===e.pointers,o=t.distance<e.threshold,n=t.deltaTime<e.time;if(this.reset(),t.eventType&Ot&&0===this.count)return this.failTimeout();if(o&&n&&i){if(t.eventType!=Pt)return this.failTimeout();var s=!this.pTime||t.timeStamp-this.pTime<e.interval,r=!this.pCenter||F(this.pCenter,t.center)<e.posThreshold;this.pTime=t.timeStamp,this.pCenter=t.center,r&&s?this.count+=1:this.count=1,this._input=t;if(0===this.count%e.taps)return this.hasRequireFailures()?(this._timer=h(function(){this.state=fe,this.tryEmit()},e.interval,this),pe):fe}return 32},failTimeout:function(){return this._timer=h(function(){this.state=32},this.options.interval,this),32},reset:function(){clearTimeout(this._timer)},emit:function(){this.state==fe&&(this._input.tapCount=this.count,this.manager.emit(this.options.event,this._input))}}),lt.VERSION="2.0.7",lt.defaults={domEvents:!1,touchAction:"compute",enable:!0,inputTarget:null,inputClass:null,preset:[[at,{enable:!1}],[st,{enable:!1},["rotate"]],[ht,{direction:Ht}],[nt,{direction:Ht},["swipe"]],[dt],[dt,{event:"doubletap",taps:2},["tap"]],[rt]],cssProps:{userSelect:"none",touchSelect:"none",touchCallout:"none",contentZooming:"none",userDrag:"none",tapHighlightColor:"rgba(0,0,0,0)"}};ut.prototype={set:function(t){return mt(this.options,t),t.touchAction&&this.touchAction.update(),t.inputTarget&&(this.input.destroy(),this.input.target=t.inputTarget,this.input.init()),this},stop:function(t){this.session.stopped=t?2:1},recognize:function(t){var e=this.session;if(!e.stopped){this.touchAction.preventDefaults(t);var i,o=this.recognizers,n=e.curRecognizer;(!n||n&&n.state&fe)&&(n=e.curRecognizer=null);for(var s=0;s<o.length;)i=o[s],2===e.stopped||n&&i!=n&&!i.canRecognizeWith(n)?i.reset():i.recognize(t),!n&&i.state&(pe|ce|me)&&(n=e.curRecognizer=i),s++}},get:function(t){if(t instanceof Q)return t;for(var e=this.recognizers,i=0;i<e.length;i++)if(e[i].options.event==t)return e[i];return null},add:function(t){if(d(t,"add",this))return this;var e=this.get(t.options.event);return e&&this.remove(e),this.recognizers.push(t),t.manager=this,this.touchAction.update(),t},remove:function(t){if(d(t,"remove",this))return this;if(t=this.get(t)){var e=this.recognizers,i=w(e,t);-1!==i&&(e.splice(i,1),this.touchAction.update())}return this},on:function(t,e){if(t!==a&&e!==a){var i=this.handlers;return l(_(t),function(t){i[t]=i[t]||[],i[t].push(e)}),this}},off:function(t,e){if(t!==a){var i=this.handlers;return l(_(t),function(t){e?i[t]&&i[t].splice(w(i[t],e),1):delete i[t]}),this}},emit:function(t,e){this.options.domEvents&&ct(t,e);var i=this.handlers[t]&&this.handlers[t].slice();if(i&&i.length){e.type=t,e.preventDefault=function(){e.srcEvent.preventDefault()};for(var o=0;o<i.length;)i[o](e),o++}},destroy:function(){this.element&&pt(this,!1),this.handlers={},this.session={},this.input.destroy(),this.element=null}},mt(lt,{INPUT_START:Ot,INPUT_MOVE:Et,INPUT_END:Pt,INPUT_CANCEL:It,STATE_POSSIBLE:ue,STATE_BEGAN:pe,STATE_CHANGED:ce,STATE_ENDED:me,STATE_RECOGNIZED:fe,STATE_CANCELLED:ge,STATE_FAILED:32,DIRECTION_NONE:Nt,DIRECTION_LEFT:At,DIRECTION_RIGHT:Rt,DIRECTION_UP:Lt,DIRECTION_DOWN:Ft,DIRECTION_HORIZONTAL:Ht,DIRECTION_VERTICAL:jt,DIRECTION_ALL:Yt,Manager:ut,Input:T,TouchAction:J,TouchInput:B,MouseInput:G,PointerEventInput:z,TouchMouseInput:q,SingleTouchInput:W,Recognizer:Q,AttrRecognizer:ot,Tap:dt,Pan:nt,Swipe:ht,Pinch:st,Rotate:at,Press:rt,on:g,off:v,each:l,merge:xt,extend:wt,assign:mt,inherit:p,bindFn:c,prefixed:S}),(void 0!==n?n:"undefined"!=typeof self?self:{}).Hammer=lt,(o=function(){return lt}.call(e,i,e,t))!==a&&(t.exports=o)}(window,document)},function(t,e,i){function o(t){this.active=!1,this.dom={container:t},this.dom.overlay=document.createElement("div"),this.dom.overlay.className="vis-overlay",this.dom.container.appendChild(this.dom.overlay),this.hammer=a(this.dom.overlay),this.hammer.on("tap",this._onTapOverlay.bind(this));var e=this;["tap","doubletap","press","pinch","pan","panstart","panmove","panend"].forEach(function(t){e.hammer.on(t,function(t){t.stopPropagation()})}),document&&document.body&&(this.onClick=function(i){n(i.target,t)||e.deactivate()},document.body.addEventListener("click",this.onClick)),void 0!==this.keycharm&&this.keycharm.destroy(),this.keycharm=s(),this.escListener=this.deactivate.bind(this)}function n(t,e){for(;t;){if(t===e)return!0;t=t.parentNode}return!1}var s=i(68),r=i(66),a=i(12),h=i(0);r(o.prototype),o.current=null,o.prototype.destroy=function(){this.deactivate(),this.dom.overlay.parentNode.removeChild(this.dom.overlay),this.onClick&&document.body.removeEventListener("click",this.onClick),this.hammer.destroy(),this.hammer=null},o.prototype.activate=function(){o.current&&o.current.deactivate(),o.current=this,this.active=!0,this.dom.overlay.style.display="none",h.addClassName(this.dom.container,"vis-active"),this.emit("change"),this.emit("activate"),this.keycharm.bind("esc",this.escListener)},o.prototype.deactivate=function(){this.active=!1,this.dom.overlay.style.display="",h.removeClassName(this.dom.container,"vis-active"),this.keycharm.unbind("esc",this.escListener),this.emit("change"),this.emit("deactivate")},o.prototype._onTapOverlay=function(t){this.activate(),t.stopPropagation()},t.exports=o},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(30),s=o(n),r=i(31),a=o(r),h=function(){function t(e,i){(0,s.default)(this,t),this.container=e,this.overflowMethod=i||"cap",this.x=0,this.y=0,this.padding=5,this.hidden=!1,this.frame=document.createElement("div"),this.frame.className="vis-tooltip",this.container.appendChild(this.frame)}return(0,a.default)(t,[{key:"setPosition",value:function(t,e){this.x=parseInt(t),this.y=parseInt(e)}},{key:"setText",value:function(t){t instanceof Element?(this.frame.innerHTML="",this.frame.appendChild(t)):this.frame.innerHTML=t}},{key:"show",value:function(t){if(void 0===t&&(t=!0),!0===t){var e=this.frame.clientHeight,i=this.frame.clientWidth,o=this.frame.parentNode.clientHeight,n=this.frame.parentNode.clientWidth,s=0,r=0;if("flip"==this.overflowMethod){var a=!1,h=!0;this.y-e<this.padding&&(h=!1),this.x+i>n-this.padding&&(a=!0),s=a?this.x-i:this.x,r=h?this.y-e:this.y}else r=this.y-e,r+e+this.padding>o&&(r=o-e-this.padding),r<this.padding&&(r=this.padding),s=this.x,s+i+this.padding>n&&(s=n-i-this.padding),s<this.padding&&(s=this.padding);this.frame.style.left=s+"px",this.frame.style.top=r+"px",this.frame.style.visibility="visible",this.hidden=!1}else this.hide()}},{key:"hide",value:function(){this.hidden=!0,this.frame.style.left="0",this.frame.style.top="0",this.frame.style.visibility="hidden"}},{key:"destroy",value:function(){this.frame.parentNode.removeChild(this.frame)}}]),t}();e.default=h},function(t,e,i){t.exports={default:i(135),__esModule:!0}},function(t,e,i){i(136);var o=i(2).Object;t.exports=function(t,e,i){return o.defineProperty(t,e,i)}},function(t,e,i){var o=i(17);o(o.S+o.F*!i(9),"Object",{defineProperty:i(8).f})},function(t,e,i){function o(t){return t&&t.__esModule?t:{default:t}}Object.defineProperty(e,"__esModule",{value:!0});var n=i(16),s=o(n),r=i(30),a=o(r),h=i(31),d=o(h),l=i(12),u=i(67),p=i(0),c=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;(0,a.default)(this,t),this.pixelRatio=e,this.generated=!1,this.centerCoordinates={x:144.5,y:144.5},this.r=289*.49,this.color={r:255,g:255,b:255,a:1},this.hueCircle=void 0,this.initialColor={r:255,g:255,b:255,a:1},this.previousColor=void 0,this.applied=!1,this.updateCallback=function(){},this.closeCallback=function(){},this._create()}return(0,d.default)(t,[{key:"insertTo",value:function(t){void 0!==this.hammer&&(this.hammer.destroy(),this.hammer=void 0),this.container=t,this.container.appendChild(this.frame),this._bindHammer(),this._setSize()}},{key:"setUpdateCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker update callback is not a function.");this.updateCallback=t}},{key:"setCloseCallback",value:function(t){if("function"!=typeof t)throw new Error("Function attempted to set as colorPicker closing callback is not a function.");this.closeCallback=t}},{key:"_isColorString",value:function(t){var e={black:"#000000",navy:"#000080",darkblue:"#00008B",mediumblue:"#0000CD",blue:"#0000FF",darkgreen:"#006400",green:"#008000",teal:"#008080",darkcyan:"#008B8B",deepskyblue:"#00BFFF",darkturquoise:"#00CED1",mediumspringgreen:"#00FA9A",lime:"#00FF00",springgreen:"#00FF7F",aqua:"#00FFFF",cyan:"#00FFFF",midnightblue:"#191970",dodgerblue:"#1E90FF",lightseagreen:"#20B2AA",forestgreen:"#228B22",seagreen:"#2E8B57",darkslategray:"#2F4F4F",limegreen:"#32CD32",mediumseagreen:"#3CB371",turquoise:"#40E0D0",royalblue:"#4169E1",steelblue:"#4682B4",darkslateblue:"#483D8B",mediumturquoise:"#48D1CC",indigo:"#4B0082",darkolivegreen:"#556B2F",cadetblue:"#5F9EA0",cornflowerblue:"#6495ED",mediumaquamarine:"#66CDAA",dimgray:"#696969",slateblue:"#6A5ACD",olivedrab:"#6B8E23",slategray:"#708090",lightslategray:"#778899",mediumslateblue:"#7B68EE",lawngreen:"#7CFC00",chartreuse:"#7FFF00",aquamarine:"#7FFFD4",maroon:"#800000",purple:"#800080",olive:"#808000",gray:"#808080",skyblue:"#87CEEB",lightskyblue:"#87CEFA",blueviolet:"#8A2BE2",darkred:"#8B0000",darkmagenta:"#8B008B",saddlebrown:"#8B4513",darkseagreen:"#8FBC8F",lightgreen:"#90EE90",mediumpurple:"#9370D8",darkviolet:"#9400D3",palegreen:"#98FB98",darkorchid:"#9932CC",yellowgreen:"#9ACD32",sienna:"#A0522D",brown:"#A52A2A",darkgray:"#A9A9A9",lightblue:"#ADD8E6",greenyellow:"#ADFF2F",paleturquoise:"#AFEEEE",lightsteelblue:"#B0C4DE",powderblue:"#B0E0E6",firebrick:"#B22222",darkgoldenrod:"#B8860B",mediumorchid:"#BA55D3",rosybrown:"#BC8F8F",darkkhaki:"#BDB76B",silver:"#C0C0C0",mediumvioletred:"#C71585",indianred:"#CD5C5C",peru:"#CD853F",chocolate:"#D2691E",tan:"#D2B48C",lightgrey:"#D3D3D3",palevioletred:"#D87093",thistle:"#D8BFD8",orchid:"#DA70D6",goldenrod:"#DAA520",crimson:"#DC143C",gainsboro:"#DCDCDC",plum:"#DDA0DD",burlywood:"#DEB887",lightcyan:"#E0FFFF",lavender:"#E6E6FA",darksalmon:"#E9967A",violet:"#EE82EE",palegoldenrod:"#EEE8AA",lightcoral:"#F08080",khaki:"#F0E68C",aliceblue:"#F0F8FF",honeydew:"#F0FFF0",azure:"#F0FFFF",sandybrown:"#F4A460",wheat:"#F5DEB3",beige:"#F5F5DC",whitesmoke:"#F5F5F5",mintcream:"#F5FFFA",ghostwhite:"#F8F8FF",salmon:"#FA8072",antiquewhite:"#FAEBD7",linen:"#FAF0E6",lightgoldenrodyellow:"#FAFAD2",oldlace:"#FDF5E6",red:"#FF0000",fuchsia:"#FF00FF",magenta:"#FF00FF",deeppink:"#FF1493",orangered:"#FF4500",tomato:"#FF6347",hotpink:"#FF69B4",coral:"#FF7F50",darkorange:"#FF8C00",lightsalmon:"#FFA07A",orange:"#FFA500",lightpink:"#FFB6C1",pink:"#FFC0CB",gold:"#FFD700",peachpuff:"#FFDAB9",navajowhite:"#FFDEAD",moccasin:"#FFE4B5",bisque:"#FFE4C4",mistyrose:"#FFE4E1",blanchedalmond:"#FFEBCD",papayawhip:"#FFEFD5",lavenderblush:"#FFF0F5",seashell:"#FFF5EE",cornsilk:"#FFF8DC",lemonchiffon:"#FFFACD",floralwhite:"#FFFAF0",snow:"#FFFAFA",yellow:"#FFFF00",lightyellow:"#FFFFE0",ivory:"#FFFFF0",white:"#FFFFFF"};if("string"==typeof t)return e[t]}},{key:"setColor",value:function(t){var e=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];if("none"!==t){var i=void 0,o=this._isColorString(t);if(void 0!==o&&(t=o),!0===p.isString(t)){if(!0===p.isValidRGB(t)){var n=t.substr(4).substr(0,t.length-5).split(",");i={r:n[0],g:n[1],b:n[2],a:1}}else if(!0===p.isValidRGBA(t)){var r=t.substr(5).substr(0,t.length-6).split(",");i={r:r[0],g:r[1],b:r[2],a:r[3]}}else if(!0===p.isValidHex(t)){var a=p.hexToRGB(t);i={r:a.r,g:a.g,b:a.b,a:1}}}else if(t instanceof Object&&void 0!==t.r&&void 0!==t.g&&void 0!==t.b){var h=void 0!==t.a?t.a:"1.0";i={r:t.r,g:t.g,b:t.b,a:h}}if(void 0===i)throw new Error("Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: "+(0,s.default)(t));this._setColor(i,e)}}},{key:"show",value:function(){void 0!==this.closeCallback&&(this.closeCallback(),this.closeCallback=void 0),this.applied=!1,this.frame.style.display="block",this._generateHueCircle()}},{key:"_hide",value:function(){var t=this;!0===(!(arguments.length>0&&void 0!==arguments[0])||arguments[0])&&(this.previousColor=p.extend({},this.color)),!0===this.applied&&this.updateCallback(this.initialColor),this.frame.style.display="none",setTimeout(function(){void 0!==t.closeCallback&&(t.closeCallback(),t.closeCallback=void 0)},0)}},{key:"_save",value:function(){this.updateCallback(this.color),this.applied=!1,this._hide()}},{key:"_apply",value:function(){this.applied=!0,this.updateCallback(this.color),this._updatePicker(this.color)}},{key:"_loadLast",value:function(){void 0!==this.previousColor?this.setColor(this.previousColor,!1):alert("There is no last color to load...")}},{key:"_setColor",value:function(t){!0===(!(arguments.length>1&&void 0!==arguments[1])||arguments[1])&&(this.initialColor=p.extend({},t)),this.color=t;var e=p.RGBToHSV(t.r,t.g,t.b),i=2*Math.PI,o=this.r*e.s,n=this.centerCoordinates.x+o*Math.sin(i*e.h),s=this.centerCoordinates.y+o*Math.cos(i*e.h);this.colorPickerSelector.style.left=n-.5*this.colorPickerSelector.clientWidth+"px",this.colorPickerSelector.style.top=s-.5*this.colorPickerSelector.clientHeight+"px",this._updatePicker(t)}},{key:"_setOpacity",value:function(t){this.color.a=t/100,this._updatePicker(this.color)}},{key:"_setBrightness",value:function(t){var e=p.RGBToHSV(this.color.r,this.color.g,this.color.b);e.v=t/100;var i=p.HSVToRGB(e.h,e.s,e.v);i.a=this.color.a,this.color=i,this._updatePicker()}},{key:"_updatePicker",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:this.color,e=p.RGBToHSV(t.r,t.g,t.b),i=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(i.webkitBackingStorePixelRatio||i.mozBackingStorePixelRatio||i.msBackingStorePixelRatio||i.oBackingStorePixelRatio||i.backingStorePixelRatio||1)),i.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var o=this.colorPickerCanvas.clientWidth,n=this.colorPickerCanvas.clientHeight;i.clearRect(0,0,o,n),i.putImageData(this.hueCircle,0,0),i.fillStyle="rgba(0,0,0,"+(1-e.v)+")",i.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),i.fill(),this.brightnessRange.value=100*e.v,this.opacityRange.value=100*t.a,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}},{key:"_setSize",value:function(){this.colorPickerCanvas.style.width="100%",this.colorPickerCanvas.style.height="100%",this.colorPickerCanvas.width=289*this.pixelRatio,this.colorPickerCanvas.height=289*this.pixelRatio}},{key:"_create",value:function(){if(this.frame=document.createElement("div"),this.frame.className="vis-color-picker",this.colorPickerDiv=document.createElement("div"),this.colorPickerSelector=document.createElement("div"),this.colorPickerSelector.className="vis-selector",this.colorPickerDiv.appendChild(this.colorPickerSelector),this.colorPickerCanvas=document.createElement("canvas"),this.colorPickerDiv.appendChild(this.colorPickerCanvas),this.colorPickerCanvas.getContext){var t=this.colorPickerCanvas.getContext("2d");this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1),this.colorPickerCanvas.getContext("2d").setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0)}else{var e=document.createElement("DIV");e.style.color="red",e.style.fontWeight="bold",e.style.padding="10px",e.innerHTML="Error: your browser does not support HTML canvas",this.colorPickerCanvas.appendChild(e)}this.colorPickerDiv.className="vis-color",this.opacityDiv=document.createElement("div"),this.opacityDiv.className="vis-opacity",this.brightnessDiv=document.createElement("div"),this.brightnessDiv.className="vis-brightness",this.arrowDiv=document.createElement("div"),this.arrowDiv.className="vis-arrow",this.opacityRange=document.createElement("input");try{this.opacityRange.type="range",this.opacityRange.min="0",this.opacityRange.max="100"}catch(t){}this.opacityRange.value="100",this.opacityRange.className="vis-range",this.brightnessRange=document.createElement("input");try{this.brightnessRange.type="range",this.brightnessRange.min="0",this.brightnessRange.max="100"}catch(t){}this.brightnessRange.value="100",this.brightnessRange.className="vis-range",this.opacityDiv.appendChild(this.opacityRange),this.brightnessDiv.appendChild(this.brightnessRange);var i=this;this.opacityRange.onchange=function(){i._setOpacity(this.value)},this.opacityRange.oninput=function(){i._setOpacity(this.value)},this.brightnessRange.onchange=function(){i._setBrightness(this.value)},this.brightnessRange.oninput=function(){i._setBrightness(this.value)},this.brightnessLabel=document.createElement("div"),this.brightnessLabel.className="vis-label vis-brightness",this.brightnessLabel.innerHTML="brightness:",this.opacityLabel=document.createElement("div"),this.opacityLabel.className="vis-label vis-opacity",this.opacityLabel.innerHTML="opacity:",this.newColorDiv=document.createElement("div"),this.newColorDiv.className="vis-new-color",this.newColorDiv.innerHTML="new",this.initialColorDiv=document.createElement("div"),this.initialColorDiv.className="vis-initial-color",this.initialColorDiv.innerHTML="initial",this.cancelButton=document.createElement("div"),this.cancelButton.className="vis-button vis-cancel",this.cancelButton.innerHTML="cancel",this.cancelButton.onclick=this._hide.bind(this,!1),this.applyButton=document.createElement("div"),this.applyButton.className="vis-button vis-apply",this.applyButton.innerHTML="apply",this.applyButton.onclick=this._apply.bind(this),this.saveButton=document.createElement("div"),this.saveButton.className="vis-button vis-save",this.saveButton.innerHTML="save",this.saveButton.onclick=this._save.bind(this),this.loadButton=document.createElement("div"),this.loadButton.className="vis-button vis-load",this.loadButton.innerHTML="load last",this.loadButton.onclick=this._loadLast.bind(this),this.frame.appendChild(this.colorPickerDiv),this.frame.appendChild(this.arrowDiv),this.frame.appendChild(this.brightnessLabel),this.frame.appendChild(this.brightnessDiv),this.frame.appendChild(this.opacityLabel),this.frame.appendChild(this.opacityDiv),this.frame.appendChild(this.newColorDiv),this.frame.appendChild(this.initialColorDiv),this.frame.appendChild(this.cancelButton),this.frame.appendChild(this.applyButton),this.frame.appendChild(this.saveButton),this.frame.appendChild(this.loadButton)}},{key:"_bindHammer",value:function(){var t=this;this.drag={},this.pinch={},this.hammer=new l(this.colorPickerCanvas),this.hammer.get("pinch").set({enable:!0}),u.onTouch(this.hammer,function(e){t._moveSelector(e)}),this.hammer.on("tap",function(e){t._moveSelector(e)}),this.hammer.on("panstart",function(e){t._moveSelector(e)}),this.hammer.on("panmove",function(e){t._moveSelector(e)}),this.hammer.on("panend",function(e){t._moveSelector(e)})}},{key:"_generateHueCircle",value:function(){if(!1===this.generated){var t=this.colorPickerCanvas.getContext("2d");void 0===this.pixelRation&&(this.pixelRatio=(window.devicePixelRatio||1)/(t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1)),t.setTransform(this.pixelRatio,0,0,this.pixelRatio,0,0);var e=this.colorPickerCanvas.clientWidth,i=this.colorPickerCanvas.clientHeight;t.clearRect(0,0,e,i);var o=void 0,n=void 0,s=void 0,r=void 0;this.centerCoordinates={x:.5*e,y:.5*i},this.r=.49*e;var a=2*Math.PI/360,h=1/this.r,d=void 0;for(s=0;s<360;s++)for(r=0;r<this.r;r++)o=this.centerCoordinates.x+r*Math.sin(a*s),n=this.centerCoordinates.y+r*Math.cos(a*s),d=p.HSVToRGB(s*(1/360),r*h,1),t.fillStyle="rgb("+d.r+","+d.g+","+d.b+")",t.fillRect(o-.5,n-.5,2,2);t.strokeStyle="rgba(0,0,0,1)",t.circle(this.centerCoordinates.x,this.centerCoordinates.y,this.r),t.stroke(),this.hueCircle=t.getImageData(0,0,e,i)}this.generated=!0}},{key:"_moveSelector",value:function(t){var e=this.colorPickerDiv.getBoundingClientRect(),i=t.center.x-e.left,o=t.center.y-e.top,n=.5*this.colorPickerDiv.clientHeight,s=.5*this.colorPickerDiv.clientWidth,r=i-s,a=o-n,h=Math.atan2(r,a),d=.98*Math.min(Math.sqrt(r*r+a*a),s),l=Math.cos(h)*d+n,u=Math.sin(h)*d+s;this.colorPickerSelector.style.top=l-.5*this.colorPickerSelector.clientHeight+"px",this.colorPickerSelector.style.left=u-.5*this.colorPickerSelector.clientWidth+"px";var c=h/(2*Math.PI);c=c<0?c+1:c;var m=d/this.r,f=p.RGBToHSV(this.color.r,this.color.g,this.color.b);f.h=c,f.s=m;var g=p.HSVToRGB(f.h,f.s,f.v);g.a=this.color.a,this.color=g,this.initialColorDiv.style.backgroundColor="rgba("+this.initialColor.r+","+this.initialColor.g+","+this.initialColor.b+","+this.initialColor.a+")",this.newColorDiv.style.backgroundColor="rgba("+this.color.r+","+this.color.g+","+this.color.b+","+this.color.a+")"}}]),t}();e.default=c},function(t,e,i){function o(t,e,i,o){if(!(Array.isArray(i)||i instanceof r||i instanceof a)&&i instanceof Object){var d=o;o=i,i=d}o&&o.throttleRedraw&&console.warn('Graph2d option "throttleRedraw" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.');var p=this;this.defaultOptions={start:null,end:null,autoResize:!0,orientation:{axis:"bottom",item:"bottom"},moment:n,width:null,height:null,maxHeight:null,minHeight:null},this.options=s.deepExtend({},this.defaultOptions),this._create(t),this.components=[],this.body={dom:this.dom,domProps:this.props,emitter:{on:this.on.bind(this),off:this.off.bind(this),emit:this.emit.bind(this)},hiddenDates:[],util:{toScreen:p._toScreen.bind(p),toGlobalScreen:p._toGlobalScreen.bind(p),toTime:p._toTime.bind(p),toGlobalTime:p._toGlobalTime.bind(p)}},this.range=new h(this.body),this.components.push(this.range),this.body.range=this.range,this.timeAxis=new l(this.body),this.components.push(this.timeAxis),this.currentTime=new u(this.body),this.components.push(this.currentTime),this.linegraph=new c(this.body),this.components.push(this.linegraph),this.itemsData=null,this.groupsData=null,this.on("tap",function(t){p.emit("click",p.getEventProperties(t))}),this.on("doubletap",function(t){p.emit("doubleClick",p.getEventProperties(t))}),this.dom.root.oncontextmenu=function(t){p.emit("contextmenu",p.getEventProperties(t))},o&&this.setOptions(o),i&&this.setGroups(i),e&&this.setItems(e),this._redraw()}var n=i(3),s=i(0),r=i(15),a=i(20),h=i(47),d=i(48),l=i(28),u=i(50),p=i(29),c=i(77),m=i(32).printStyle,f=i(84).allOptions,g=i(84).configureOptions,v=i(76).default,y=i(32).default;o.prototype=new d,o.prototype.setOptions=function(t){!0===y.validate(t,f)&&console.log("%cErrors have been found in the supplied options object.",m),d.prototype.setOptions.call(this,t)},o.prototype.setItems=function(t){var e,i=null==this.itemsData;if(e=t?t instanceof r||t instanceof a?t:new r(t,{type:{start:"Date",end:"Date"}}):null,this.itemsData=e,this.linegraph&&this.linegraph.setItems(e),i)if(void 0!=this.options.start||void 0!=this.options.end){var o=void 0!=this.options.start?this.options.start:null,n=void 0!=this.options.end?this.options.end:null;this.setWindow(o,n,{animation:!1})}else this.fit({animation:!1})},o.prototype.setGroups=function(t){var e;e=t?t instanceof r||t instanceof a?t:new r(t):null,this.groupsData=e,this.linegraph.setGroups(e)},o.prototype.getLegend=function(t,e,i){return void 0===e&&(e=15),void 0===i&&(i=15),void 0!==this.linegraph.groups[t]?this.linegraph.groups[t].getLegend(e,i):"cannot find group:'"+t+"'"},o.prototype.isGroupVisible=function(t){return void 0!==this.linegraph.groups[t]&&(this.linegraph.groups[t].visible&&(void 0===this.linegraph.options.groups.visibility[t]||1==this.linegraph.options.groups.visibility[t]))},o.prototype.getDataRange=function(){var t=null,e=null;for(var i in this.linegraph.groups)if(this.linegraph.groups.hasOwnProperty(i)&&1==this.linegraph.groups[i].visible)for(var o=0;o<this.linegraph.groups[i].itemsData.length;o++){var n=this.linegraph.groups[i].itemsData[o],r=s.convert(n.x,"Date").valueOf();t=null==t?r:t>r?r:t,e=null==e?r:e<r?r:e}return{min:null!=t?new Date(t):null,max:null!=e?new Date(e):null}},o.prototype.getEventProperties=function(t){var e=t.center?t.center.x:t.clientX,i=t.center?t.center.y:t.clientY,o=e-s.getAbsoluteLeft(this.dom.centerContainer),n=i-s.getAbsoluteTop(this.dom.centerContainer),r=this._toTime(o),a=p.customTimeFromTarget(t),h=s.getTarget(t),d=null;s.hasParent(h,this.timeAxis.dom.foreground)?d="axis":this.timeAxis2&&s.hasParent(h,this.timeAxis2.dom.foreground)?d="axis":s.hasParent(h,this.linegraph.yAxisLeft.dom.frame)?d="data-axis":s.hasParent(h,this.linegraph.yAxisRight.dom.frame)?d="data-axis":s.hasParent(h,this.linegraph.legendLeft.dom.frame)?d="legend":s.hasParent(h,this.linegraph.legendRight.dom.frame)?d="legend":null!=a?d="custom-time":s.hasParent(h,this.currentTime.bar)?d="current-time":s.hasParent(h,this.dom.center)&&(d="background");var l=[],u=this.linegraph.yAxisLeft,c=this.linegraph.yAxisRight;return!u.hidden&&this.itemsData.length>0&&l.push(u.screenToValue(n)),!c.hidden&&this.itemsData.length>0&&l.push(c.screenToValue(n)),{event:t,what:d,pageX:t.srcEvent?t.srcEvent.pageX:t.pageX,pageY:t.srcEvent?t.srcEvent.pageY:t.pageY,x:o,y:n,time:r,value:l}},o.prototype._createConfigurator=function(){return new v(this,this.dom.container,g)},t.exports=o}])});
\ No newline at end of file
diff --git a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map b/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
deleted file mode 100644
index 779880dcc11af..0000000000000
--- a/core/src/main/resources/org/apache/spark/ui/static/vis-timeline-graph2d.min.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"vis-timeline-graph2d.min.js","sources":["../../node_modules/moment/moment.js","../../node_modules/core-js-pure/internals/global.js","../../node_modules/moment/locale/de.js","../../node_modules/moment/locale/es.js","../../node_modules/moment/locale/fr.js","../../node_modules/moment/locale/it.js","../../node_modules/moment/locale/ja.js","../../node_modules/moment/locale/nl.js","../../node_modules/moment/locale/pl.js","../../node_modules/moment/locale/ru.js","../../node_modules/moment/locale/uk.js","../../node_modules/core-js-pure/internals/fails.js","../../node_modules/core-js-pure/internals/descriptors.js","../../node_modules/core-js-pure/internals/object-property-is-enumerable.js","../../node_modules/core-js-pure/internals/create-property-descriptor.js","../../node_modules/core-js-pure/internals/classof-raw.js","../../node_modules/core-js-pure/internals/indexed-object.js","../../node_modules/core-js-pure/internals/require-object-coercible.js","../../node_modules/core-js-pure/internals/to-indexed-object.js","../../node_modules/core-js-pure/internals/is-object.js","../../node_modules/core-js-pure/internals/to-primitive.js","../../node_modules/core-js-pure/internals/has.js","../../node_modules/core-js-pure/internals/document-create-element.js","../../node_modules/core-js-pure/internals/ie8-dom-define.js","../../node_modules/core-js-pure/internals/object-get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/is-forced.js","../../node_modules/core-js-pure/internals/path.js","../../node_modules/core-js-pure/internals/a-function.js","../../node_modules/core-js-pure/internals/function-bind-context.js","../../node_modules/core-js-pure/internals/an-object.js","../../node_modules/core-js-pure/internals/object-define-property.js","../../node_modules/core-js-pure/internals/create-non-enumerable-property.js","../../node_modules/core-js-pure/internals/export.js","../../node_modules/core-js-pure/modules/es.object.define-property.js","../../node_modules/core-js-pure/es/object/define-property.js","../../node_modules/core-js-pure/stable/object/define-property.js","../../node_modules/core-js-pure/internals/to-integer.js","../../node_modules/core-js-pure/internals/to-length.js","../../node_modules/core-js-pure/internals/to-absolute-index.js","../../node_modules/core-js-pure/internals/array-includes.js","../../node_modules/core-js-pure/internals/hidden-keys.js","../../node_modules/core-js-pure/internals/object-keys-internal.js","../../node_modules/core-js-pure/internals/enum-bug-keys.js","../../node_modules/core-js-pure/internals/object-keys.js","../../node_modules/core-js-pure/internals/object-define-properties.js","../../node_modules/core-js-pure/modules/es.object.define-properties.js","../../node_modules/core-js-pure/es/object/define-properties.js","../../node_modules/core-js-pure/internals/get-built-in.js","../../node_modules/core-js-pure/internals/object-get-own-property-names.js","../../node_modules/core-js-pure/internals/object-get-own-property-symbols.js","../../node_modules/core-js-pure/internals/own-keys.js","../../node_modules/core-js-pure/internals/create-property.js","../../node_modules/core-js-pure/modules/es.object.get-own-property-descriptors.js","../../node_modules/core-js-pure/es/object/get-own-property-descriptors.js","../../node_modules/core-js-pure/modules/es.object.get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/object-create.js","../../node_modules/core-js-pure/es/object/get-own-property-descriptor.js","../../node_modules/core-js-pure/internals/native-symbol.js","../../node_modules/core-js-pure/internals/use-symbol-as-uid.js","../../node_modules/core-js-pure/internals/is-array.js","../../node_modules/core-js-pure/internals/to-object.js","../../node_modules/core-js-pure/internals/html.js","../../node_modules/core-js-pure/internals/shared-store.js","../../node_modules/core-js-pure/internals/set-global.js","../../node_modules/core-js-pure/internals/shared.js","../../node_modules/core-js-pure/internals/uid.js","../../node_modules/core-js-pure/internals/shared-key.js","../../node_modules/core-js-pure/internals/object-get-own-property-names-external.js","../../node_modules/core-js-pure/internals/redefine.js","../../node_modules/core-js-pure/internals/well-known-symbol.js","../../node_modules/core-js-pure/internals/well-known-symbol-wrapped.js","../../node_modules/core-js-pure/internals/define-well-known-symbol.js","../../node_modules/core-js-pure/internals/to-string-tag-support.js","../../node_modules/core-js-pure/internals/classof.js","../../node_modules/core-js-pure/internals/object-to-string.js","../../node_modules/core-js-pure/internals/set-to-string-tag.js","../../node_modules/core-js-pure/internals/inspect-source.js","../../node_modules/core-js-pure/internals/internal-state.js","../../node_modules/core-js-pure/internals/native-weak-map.js","../../node_modules/core-js-pure/internals/array-species-create.js","../../node_modules/core-js-pure/internals/array-iteration.js","../../node_modules/core-js-pure/modules/es.symbol.js","../../node_modules/core-js-pure/es/object/get-own-property-symbols.js","../../node_modules/core-js-pure/internals/iterators-core.js","../../node_modules/core-js-pure/internals/iterators.js","../../node_modules/core-js-pure/internals/correct-prototype-getter.js","../../node_modules/core-js-pure/internals/object-get-prototype-of.js","../../node_modules/core-js-pure/internals/create-iterator-constructor.js","../../node_modules/core-js-pure/internals/object-set-prototype-of.js","../../node_modules/core-js-pure/internals/a-possible-prototype.js","../../node_modules/core-js-pure/internals/define-iterator.js","../../node_modules/core-js-pure/modules/es.array.iterator.js","../../node_modules/core-js-pure/internals/dom-iterables.js","../../node_modules/core-js-pure/modules/web.dom-collections.iterator.js","../../node_modules/core-js-pure/internals/string-multibyte.js","../../node_modules/core-js-pure/modules/es.string.iterator.js","../../node_modules/core-js-pure/internals/get-iterator-method.js","../../node_modules/core-js-pure/internals/get-iterator.js","../../node_modules/core-js-pure/modules/es.object.create.js","../../node_modules/core-js-pure/es/object/create.js","../../node_modules/core-js-pure/stable/object/create.js","../../node_modules/core-js-pure/features/object/define-property.js","../../node_modules/@babel/runtime-corejs3/helpers/defineProperty.js","../../node_modules/core-js-pure/modules/es.object.keys.js","../../node_modules/core-js-pure/es/object/keys.js","../../node_modules/core-js-pure/internals/string-trim-forced.js","../../node_modules/core-js-pure/internals/whitespaces.js","../../node_modules/core-js-pure/internals/string-trim.js","../../node_modules/core-js-pure/modules/es.string.trim.js","../../node_modules/core-js-pure/internals/entry-virtual.js","../../node_modules/core-js-pure/es/string/virtual/trim.js","../../node_modules/core-js-pure/es/instance/trim.js","../../node_modules/core-js-pure/internals/array-method-is-strict.js","../../node_modules/core-js-pure/internals/array-method-uses-to-length.js","../../node_modules/core-js-pure/internals/array-for-each.js","../../node_modules/core-js-pure/modules/es.array.for-each.js","../../node_modules/core-js-pure/es/array/virtual/for-each.js","../../node_modules/core-js-pure/internals/engine-v8-version.js","../../node_modules/core-js-pure/stable/instance/for-each.js","../../node_modules/core-js-pure/internals/engine-user-agent.js","../../node_modules/core-js-pure/internals/array-method-has-species-support.js","../../node_modules/core-js-pure/modules/es.array.map.js","../../node_modules/core-js-pure/es/array/virtual/map.js","../../node_modules/core-js-pure/es/instance/map.js","../../node_modules/core-js-pure/internals/number-parse-int.js","../../node_modules/core-js-pure/modules/es.parse-int.js","../../node_modules/core-js-pure/es/parse-int.js","../../node_modules/core-js-pure/internals/object-to-array.js","../../node_modules/core-js-pure/modules/es.object.values.js","../../node_modules/core-js-pure/es/object/values.js","../../node_modules/core-js-pure/modules/es.array.filter.js","../../node_modules/core-js-pure/es/array/virtual/filter.js","../../node_modules/core-js-pure/es/instance/filter.js","../../node_modules/core-js-pure/modules/es.array.concat.js","../../node_modules/core-js-pure/es/array/virtual/concat.js","../../node_modules/core-js-pure/es/instance/concat.js","../../node_modules/core-js-pure/modules/es.array.is-array.js","../../node_modules/core-js-pure/es/array/is-array.js","../../node_modules/core-js-pure/features/array/is-array.js","../../node_modules/@babel/runtime-corejs3/helpers/arrayWithoutHoles.js","../../node_modules/core-js-pure/internals/call-with-safe-iteration-closing.js","../../node_modules/core-js-pure/internals/is-array-iterator-method.js","../../node_modules/core-js-pure/internals/check-correctness-of-iteration.js","../../node_modules/core-js-pure/modules/es.array.from.js","../../node_modules/core-js-pure/internals/array-from.js","../../node_modules/core-js-pure/es/array/from.js","../../node_modules/core-js-pure/internals/is-iterable.js","../../node_modules/@babel/runtime-corejs3/helpers/iterableToArray.js","../../node_modules/@babel/runtime-corejs3/helpers/nonIterableSpread.js","../../node_modules/@babel/runtime-corejs3/helpers/toConsumableArray.js","../../node_modules/core-js-pure/modules/es.array.slice.js","../../node_modules/core-js-pure/es/array/virtual/slice.js","../../node_modules/core-js-pure/es/instance/slice.js","../../node_modules/core-js-pure/modules/es.object.get-prototype-of.js","../../node_modules/core-js-pure/es/object/get-prototype-of.js","../../node_modules/core-js-pure/stable/object/get-prototype-of.js","../../node_modules/core-js-pure/modules/es.array.index-of.js","../../node_modules/core-js-pure/es/array/virtual/index-of.js","../../node_modules/core-js-pure/es/instance/index-of.js","../../node_modules/core-js-pure/stable/array/is-array.js","../../node_modules/core-js-pure/internals/object-assign.js","../../node_modules/core-js-pure/modules/es.object.assign.js","../../node_modules/core-js-pure/es/object/assign.js","../../node_modules/core-js-pure/modules/es.symbol.iterator.js","../../node_modules/core-js-pure/es/symbol/iterator.js","../../node_modules/core-js-pure/features/symbol/iterator.js","../../node_modules/core-js-pure/modules/es.symbol.async-iterator.js","../../node_modules/core-js-pure/modules/es.symbol.has-instance.js","../../node_modules/core-js-pure/modules/es.symbol.is-concat-spreadable.js","../../node_modules/core-js-pure/modules/es.symbol.match.js","../../node_modules/core-js-pure/modules/es.symbol.match-all.js","../../node_modules/core-js-pure/modules/es.symbol.replace.js","../../node_modules/core-js-pure/modules/es.symbol.search.js","../../node_modules/core-js-pure/modules/es.symbol.species.js","../../node_modules/core-js-pure/modules/es.symbol.split.js","../../node_modules/core-js-pure/modules/es.symbol.to-primitive.js","../../node_modules/core-js-pure/modules/es.symbol.to-string-tag.js","../../node_modules/core-js-pure/modules/es.symbol.unscopables.js","../../node_modules/core-js-pure/modules/es.math.to-string-tag.js","../../node_modules/core-js-pure/modules/es.json.to-string-tag.js","../../node_modules/core-js-pure/es/symbol/index.js","../../node_modules/core-js-pure/modules/esnext.symbol.async-dispose.js","../../node_modules/core-js-pure/modules/esnext.symbol.dispose.js","../../node_modules/core-js-pure/modules/esnext.symbol.observable.js","../../node_modules/core-js-pure/modules/esnext.symbol.pattern-match.js","../../node_modules/core-js-pure/modules/esnext.symbol.replace-all.js","../../node_modules/core-js-pure/features/symbol/index.js","../../node_modules/@babel/runtime-corejs3/helpers/typeof.js","../../node_modules/@babel/runtime-corejs3/helpers/arrayWithHoles.js","../../node_modules/@babel/runtime-corejs3/helpers/iterableToArrayLimit.js","../../node_modules/@babel/runtime-corejs3/helpers/nonIterableRest.js","../../node_modules/@babel/runtime-corejs3/helpers/slicedToArray.js","../../node_modules/core-js-pure/modules/es.date.now.js","../../node_modules/core-js-pure/es/date/now.js","../../node_modules/vis-util/esnext/esm/vis-util.js","../../lib/DOMutil.js","../../node_modules/@babel/runtime-corejs3/helpers/assertThisInitialized.js","../../node_modules/core-js-pure/es/array/virtual/entries.js","../../node_modules/core-js-pure/stable/instance/entries.js","../../node_modules/core-js-pure/modules/es.json.stringify.js","../../node_modules/core-js-pure/es/json/stringify.js","../../node_modules/core-js-pure/es/array/virtual/values.js","../../node_modules/core-js-pure/stable/instance/values.js","../../node_modules/core-js-pure/es/array/virtual/keys.js","../../node_modules/core-js-pure/stable/instance/keys.js","../../node_modules/core-js-pure/modules/es.array.some.js","../../node_modules/core-js-pure/es/array/virtual/some.js","../../node_modules/core-js-pure/es/instance/some.js","../../node_modules/@babel/runtime-corejs3/helpers/possibleConstructorReturn.js","../../node_modules/core-js-pure/features/object/get-prototype-of.js","../../node_modules/core-js-pure/modules/es.object.set-prototype-of.js","../../node_modules/core-js-pure/es/object/set-prototype-of.js","../../node_modules/@babel/runtime-corejs3/helpers/getPrototypeOf.js","../../node_modules/core-js-pure/features/object/create.js","../../node_modules/@babel/runtime-corejs3/helpers/setPrototypeOf.js","../../node_modules/@babel/runtime-corejs3/helpers/inherits.js","../../node_modules/core-js-pure/modules/es.array.sort.js","../../node_modules/core-js-pure/es/array/virtual/sort.js","../../node_modules/core-js-pure/es/instance/sort.js","../../node_modules/core-js-pure/internals/freezing.js","../../node_modules/core-js-pure/internals/internal-metadata.js","../../node_modules/core-js-pure/internals/iterate.js","../../node_modules/core-js-pure/internals/an-instance.js","../../node_modules/core-js-pure/internals/collection.js","../../node_modules/core-js-pure/internals/redefine-all.js","../../node_modules/core-js-pure/internals/set-species.js","../../node_modules/core-js-pure/internals/collection-strong.js","../../node_modules/core-js-pure/modules/es.set.js","../../node_modules/core-js-pure/es/set/index.js","../../node_modules/core-js-pure/modules/es.map.js","../../node_modules/core-js-pure/es/map/index.js","../../node_modules/@babel/runtime-corejs3/node_modules/regenerator-runtime/runtime.js","../../node_modules/core-js-pure/stable/symbol/iterator.js","../../node_modules/core-js-pure/modules/es.array.splice.js","../../node_modules/core-js-pure/es/array/virtual/splice.js","../../node_modules/core-js-pure/es/instance/splice.js","../../node_modules/core-js-pure/modules/web.timers.js","../../node_modules/core-js-pure/stable/set-timeout.js","../../node_modules/core-js-pure/internals/flatten-into-array.js","../../node_modules/core-js-pure/modules/es.array.flat-map.js","../../node_modules/core-js-pure/es/array/virtual/flat-map.js","../../node_modules/core-js-pure/es/instance/flat-map.js","../../node_modules/core-js-pure/internals/array-reduce.js","../../node_modules/core-js-pure/modules/es.array.reduce.js","../../node_modules/core-js-pure/es/array/virtual/reduce.js","../../node_modules/core-js-pure/es/instance/reduce.js","../../node_modules/core-js-pure/internals/function-bind.js","../../node_modules/core-js-pure/modules/es.function.bind.js","../../node_modules/core-js-pure/es/function/virtual/bind.js","../../node_modules/core-js-pure/es/instance/bind.js","../../node_modules/@babel/runtime-corejs3/helpers/classCallCheck.js","../../node_modules/@babel/runtime-corejs3/helpers/createClass.js","../../node_modules/vis-data/esnext/esm/vis-data.js","../../lib/module/moment.js","../../lib/util.js","../../node_modules/core-js-pure/internals/number-parse-float.js","../../node_modules/core-js-pure/modules/es.parse-float.js","../../node_modules/core-js-pure/es/parse-float.js","../../lib/timeline/component/Component.js","../../node_modules/core-js-pure/modules/es.string.repeat.js","../../node_modules/core-js-pure/internals/string-repeat.js","../../node_modules/core-js-pure/es/string/virtual/repeat.js","../../node_modules/core-js-pure/es/instance/repeat.js","../../lib/timeline/DateUtil.js","../../lib/timeline/Range.js","../../node_modules/core-js-pure/stable/set-interval.js","../../node_modules/component-emitter/index.js","../../node_modules/propagating-hammerjs/propagating.js","../../node_modules/@egjs/hammerjs/dist/hammer.esm.js","../../lib/module/hammer.js","../../lib/hammerUtil.js","../../lib/timeline/TimeStep.js","../../node_modules/style-inject/dist/style-inject.es.js","../../lib/timeline/component/TimeAxis.js","../../node_modules/keycharm/keycharm.js","../../lib/shared/Activator.js","../../lib/timeline/locales.js","../../lib/timeline/component/CustomTime.js","../../lib/timeline/Core.js","../../lib/timeline/component/CurrentTime.js","../../node_modules/core-js-pure/modules/es.array.find.js","../../node_modules/core-js-pure/es/array/virtual/find.js","../../node_modules/core-js-pure/es/instance/find.js","../../node_modules/uuid/dist/esm-browser/rng.js","../../node_modules/uuid/dist/esm-browser/bytesToUuid.js","../../node_modules/uuid/dist/esm-browser/v4.js","../../node_modules/core-js-pure/modules/es.array.includes.js","../../node_modules/core-js-pure/es/array/virtual/includes.js","../../node_modules/core-js-pure/internals/is-regexp.js","../../node_modules/core-js-pure/internals/not-a-regexp.js","../../node_modules/core-js-pure/internals/correct-is-regexp-logic.js","../../node_modules/core-js-pure/modules/es.string.includes.js","../../node_modules/core-js-pure/es/string/virtual/includes.js","../../node_modules/core-js-pure/es/instance/includes.js","../../lib/timeline/Stack.js","../../lib/timeline/component/Group.js","../../lib/timeline/component/BackgroundGroup.js","../../lib/timeline/component/item/Item.js","../../lib/timeline/component/item/BoxItem.js","../../lib/timeline/component/item/PointItem.js","../../lib/timeline/component/item/RangeItem.js","../../lib/timeline/component/item/BackgroundItem.js","../../lib/shared/Popup.js","../../node_modules/core-js-pure/modules/es.array.every.js","../../node_modules/core-js-pure/es/array/virtual/every.js","../../node_modules/core-js-pure/es/instance/every.js","../../lib/timeline/component/item/ClusterItem.js","../../lib/timeline/component/ClusterGenerator.js","../../lib/timeline/component/ItemSet.js","../../lib/shared/Validator.js","../../lib/timeline/optionsTimeline.js","../../node_modules/core-js-pure/modules/es.array.fill.js","../../node_modules/core-js-pure/internals/array-fill.js","../../node_modules/core-js-pure/es/array/virtual/fill.js","../../node_modules/core-js-pure/es/instance/fill.js","../../lib/shared/ColorPicker.js","../../lib/shared/Configurator.js","../../lib/timeline/Timeline.js","../../lib/timeline/component/DataScale.js","../../lib/timeline/component/DataAxis.js","../../lib/timeline/component/graph2d_types/points.js","../../lib/timeline/component/graph2d_types/bar.js","../../lib/timeline/component/graph2d_types/line.js","../../lib/timeline/component/GraphGroup.js","../../lib/timeline/component/Legend.js","../../lib/timeline/component/LineGraph.js","../../lib/timeline/optionsGraph2d.js","../../lib/timeline/Graph2d.js","../../lib/entry-standalone.js"],"sourcesContent":["//! moment.js\n\n;(function (global, factory) {\n    typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() :\n    typeof define === 'function' && define.amd ? define(factory) :\n    global.moment = factory()\n}(this, (function () { 'use strict';\n\n    var hookCallback;\n\n    function hooks () {\n        return hookCallback.apply(null, arguments);\n    }\n\n    // This is done to register the method called with moment()\n    // without creating circular dependencies.\n    function setHookCallback (callback) {\n        hookCallback = callback;\n    }\n\n    function isArray(input) {\n        return input instanceof Array || Object.prototype.toString.call(input) === '[object Array]';\n    }\n\n    function isObject(input) {\n        // IE8 will treat undefined and null as object if it wasn't for\n        // input != null\n        return input != null && Object.prototype.toString.call(input) === '[object Object]';\n    }\n\n    function isObjectEmpty(obj) {\n        if (Object.getOwnPropertyNames) {\n            return (Object.getOwnPropertyNames(obj).length === 0);\n        } else {\n            var k;\n            for (k in obj) {\n                if (obj.hasOwnProperty(k)) {\n                    return false;\n                }\n            }\n            return true;\n        }\n    }\n\n    function isUndefined(input) {\n        return input === void 0;\n    }\n\n    function isNumber(input) {\n        return typeof input === 'number' || Object.prototype.toString.call(input) === '[object Number]';\n    }\n\n    function isDate(input) {\n        return input instanceof Date || Object.prototype.toString.call(input) === '[object Date]';\n    }\n\n    function map(arr, fn) {\n        var res = [], i;\n        for (i = 0; i < arr.length; ++i) {\n            res.push(fn(arr[i], i));\n        }\n        return res;\n    }\n\n    function hasOwnProp(a, b) {\n        return Object.prototype.hasOwnProperty.call(a, b);\n    }\n\n    function extend(a, b) {\n        for (var i in b) {\n            if (hasOwnProp(b, i)) {\n                a[i] = b[i];\n            }\n        }\n\n        if (hasOwnProp(b, 'toString')) {\n            a.toString = b.toString;\n        }\n\n        if (hasOwnProp(b, 'valueOf')) {\n            a.valueOf = b.valueOf;\n        }\n\n        return a;\n    }\n\n    function createUTC (input, format, locale, strict) {\n        return createLocalOrUTC(input, format, locale, strict, true).utc();\n    }\n\n    function defaultParsingFlags() {\n        // We need to deep clone this object.\n        return {\n            empty           : false,\n            unusedTokens    : [],\n            unusedInput     : [],\n            overflow        : -2,\n            charsLeftOver   : 0,\n            nullInput       : false,\n            invalidMonth    : null,\n            invalidFormat   : false,\n            userInvalidated : false,\n            iso             : false,\n            parsedDateParts : [],\n            meridiem        : null,\n            rfc2822         : false,\n            weekdayMismatch : false\n        };\n    }\n\n    function getParsingFlags(m) {\n        if (m._pf == null) {\n            m._pf = defaultParsingFlags();\n        }\n        return m._pf;\n    }\n\n    var some;\n    if (Array.prototype.some) {\n        some = Array.prototype.some;\n    } else {\n        some = function (fun) {\n            var t = Object(this);\n            var len = t.length >>> 0;\n\n            for (var i = 0; i < len; i++) {\n                if (i in t && fun.call(this, t[i], i, t)) {\n                    return true;\n                }\n            }\n\n            return false;\n        };\n    }\n\n    function isValid(m) {\n        if (m._isValid == null) {\n            var flags = getParsingFlags(m);\n            var parsedParts = some.call(flags.parsedDateParts, function (i) {\n                return i != null;\n            });\n            var isNowValid = !isNaN(m._d.getTime()) &&\n                flags.overflow < 0 &&\n                !flags.empty &&\n                !flags.invalidMonth &&\n                !flags.invalidWeekday &&\n                !flags.weekdayMismatch &&\n                !flags.nullInput &&\n                !flags.invalidFormat &&\n                !flags.userInvalidated &&\n                (!flags.meridiem || (flags.meridiem && parsedParts));\n\n            if (m._strict) {\n                isNowValid = isNowValid &&\n                    flags.charsLeftOver === 0 &&\n                    flags.unusedTokens.length === 0 &&\n                    flags.bigHour === undefined;\n            }\n\n            if (Object.isFrozen == null || !Object.isFrozen(m)) {\n                m._isValid = isNowValid;\n            }\n            else {\n                return isNowValid;\n            }\n        }\n        return m._isValid;\n    }\n\n    function createInvalid (flags) {\n        var m = createUTC(NaN);\n        if (flags != null) {\n            extend(getParsingFlags(m), flags);\n        }\n        else {\n            getParsingFlags(m).userInvalidated = true;\n        }\n\n        return m;\n    }\n\n    // Plugins that add properties should also add the key here (null value),\n    // so we can properly clone ourselves.\n    var momentProperties = hooks.momentProperties = [];\n\n    function copyConfig(to, from) {\n        var i, prop, val;\n\n        if (!isUndefined(from._isAMomentObject)) {\n            to._isAMomentObject = from._isAMomentObject;\n        }\n        if (!isUndefined(from._i)) {\n            to._i = from._i;\n        }\n        if (!isUndefined(from._f)) {\n            to._f = from._f;\n        }\n        if (!isUndefined(from._l)) {\n            to._l = from._l;\n        }\n        if (!isUndefined(from._strict)) {\n            to._strict = from._strict;\n        }\n        if (!isUndefined(from._tzm)) {\n            to._tzm = from._tzm;\n        }\n        if (!isUndefined(from._isUTC)) {\n            to._isUTC = from._isUTC;\n        }\n        if (!isUndefined(from._offset)) {\n            to._offset = from._offset;\n        }\n        if (!isUndefined(from._pf)) {\n            to._pf = getParsingFlags(from);\n        }\n        if (!isUndefined(from._locale)) {\n            to._locale = from._locale;\n        }\n\n        if (momentProperties.length > 0) {\n            for (i = 0; i < momentProperties.length; i++) {\n                prop = momentProperties[i];\n                val = from[prop];\n                if (!isUndefined(val)) {\n                    to[prop] = val;\n                }\n            }\n        }\n\n        return to;\n    }\n\n    var updateInProgress = false;\n\n    // Moment prototype object\n    function Moment(config) {\n        copyConfig(this, config);\n        this._d = new Date(config._d != null ? config._d.getTime() : NaN);\n        if (!this.isValid()) {\n            this._d = new Date(NaN);\n        }\n        // Prevent infinite loop in case updateOffset creates new moment\n        // objects.\n        if (updateInProgress === false) {\n            updateInProgress = true;\n            hooks.updateOffset(this);\n            updateInProgress = false;\n        }\n    }\n\n    function isMoment (obj) {\n        return obj instanceof Moment || (obj != null && obj._isAMomentObject != null);\n    }\n\n    function absFloor (number) {\n        if (number < 0) {\n            // -0 -> 0\n            return Math.ceil(number) || 0;\n        } else {\n            return Math.floor(number);\n        }\n    }\n\n    function toInt(argumentForCoercion) {\n        var coercedNumber = +argumentForCoercion,\n            value = 0;\n\n        if (coercedNumber !== 0 && isFinite(coercedNumber)) {\n            value = absFloor(coercedNumber);\n        }\n\n        return value;\n    }\n\n    // compare two arrays, return the number of differences\n    function compareArrays(array1, array2, dontConvert) {\n        var len = Math.min(array1.length, array2.length),\n            lengthDiff = Math.abs(array1.length - array2.length),\n            diffs = 0,\n            i;\n        for (i = 0; i < len; i++) {\n            if ((dontConvert && array1[i] !== array2[i]) ||\n                (!dontConvert && toInt(array1[i]) !== toInt(array2[i]))) {\n                diffs++;\n            }\n        }\n        return diffs + lengthDiff;\n    }\n\n    function warn(msg) {\n        if (hooks.suppressDeprecationWarnings === false &&\n                (typeof console !==  'undefined') && console.warn) {\n            console.warn('Deprecation warning: ' + msg);\n        }\n    }\n\n    function deprecate(msg, fn) {\n        var firstTime = true;\n\n        return extend(function () {\n            if (hooks.deprecationHandler != null) {\n                hooks.deprecationHandler(null, msg);\n            }\n            if (firstTime) {\n                var args = [];\n                var arg;\n                for (var i = 0; i < arguments.length; i++) {\n                    arg = '';\n                    if (typeof arguments[i] === 'object') {\n                        arg += '\\n[' + i + '] ';\n                        for (var key in arguments[0]) {\n                            arg += key + ': ' + arguments[0][key] + ', ';\n                        }\n                        arg = arg.slice(0, -2); // Remove trailing comma and space\n                    } else {\n                        arg = arguments[i];\n                    }\n                    args.push(arg);\n                }\n                warn(msg + '\\nArguments: ' + Array.prototype.slice.call(args).join('') + '\\n' + (new Error()).stack);\n                firstTime = false;\n            }\n            return fn.apply(this, arguments);\n        }, fn);\n    }\n\n    var deprecations = {};\n\n    function deprecateSimple(name, msg) {\n        if (hooks.deprecationHandler != null) {\n            hooks.deprecationHandler(name, msg);\n        }\n        if (!deprecations[name]) {\n            warn(msg);\n            deprecations[name] = true;\n        }\n    }\n\n    hooks.suppressDeprecationWarnings = false;\n    hooks.deprecationHandler = null;\n\n    function isFunction(input) {\n        return input instanceof Function || Object.prototype.toString.call(input) === '[object Function]';\n    }\n\n    function set (config) {\n        var prop, i;\n        for (i in config) {\n            prop = config[i];\n            if (isFunction(prop)) {\n                this[i] = prop;\n            } else {\n                this['_' + i] = prop;\n            }\n        }\n        this._config = config;\n        // Lenient ordinal parsing accepts just a number in addition to\n        // number + (possibly) stuff coming from _dayOfMonthOrdinalParse.\n        // TODO: Remove \"ordinalParse\" fallback in next major release.\n        this._dayOfMonthOrdinalParseLenient = new RegExp(\n            (this._dayOfMonthOrdinalParse.source || this._ordinalParse.source) +\n                '|' + (/\\d{1,2}/).source);\n    }\n\n    function mergeConfigs(parentConfig, childConfig) {\n        var res = extend({}, parentConfig), prop;\n        for (prop in childConfig) {\n            if (hasOwnProp(childConfig, prop)) {\n                if (isObject(parentConfig[prop]) && isObject(childConfig[prop])) {\n                    res[prop] = {};\n                    extend(res[prop], parentConfig[prop]);\n                    extend(res[prop], childConfig[prop]);\n                } else if (childConfig[prop] != null) {\n                    res[prop] = childConfig[prop];\n                } else {\n                    delete res[prop];\n                }\n            }\n        }\n        for (prop in parentConfig) {\n            if (hasOwnProp(parentConfig, prop) &&\n                    !hasOwnProp(childConfig, prop) &&\n                    isObject(parentConfig[prop])) {\n                // make sure changes to properties don't modify parent config\n                res[prop] = extend({}, res[prop]);\n            }\n        }\n        return res;\n    }\n\n    function Locale(config) {\n        if (config != null) {\n            this.set(config);\n        }\n    }\n\n    var keys;\n\n    if (Object.keys) {\n        keys = Object.keys;\n    } else {\n        keys = function (obj) {\n            var i, res = [];\n            for (i in obj) {\n                if (hasOwnProp(obj, i)) {\n                    res.push(i);\n                }\n            }\n            return res;\n        };\n    }\n\n    var defaultCalendar = {\n        sameDay : '[Today at] LT',\n        nextDay : '[Tomorrow at] LT',\n        nextWeek : 'dddd [at] LT',\n        lastDay : '[Yesterday at] LT',\n        lastWeek : '[Last] dddd [at] LT',\n        sameElse : 'L'\n    };\n\n    function calendar (key, mom, now) {\n        var output = this._calendar[key] || this._calendar['sameElse'];\n        return isFunction(output) ? output.call(mom, now) : output;\n    }\n\n    var defaultLongDateFormat = {\n        LTS  : 'h:mm:ss A',\n        LT   : 'h:mm A',\n        L    : 'MM/DD/YYYY',\n        LL   : 'MMMM D, YYYY',\n        LLL  : 'MMMM D, YYYY h:mm A',\n        LLLL : 'dddd, MMMM D, YYYY h:mm A'\n    };\n\n    function longDateFormat (key) {\n        var format = this._longDateFormat[key],\n            formatUpper = this._longDateFormat[key.toUpperCase()];\n\n        if (format || !formatUpper) {\n            return format;\n        }\n\n        this._longDateFormat[key] = formatUpper.replace(/MMMM|MM|DD|dddd/g, function (val) {\n            return val.slice(1);\n        });\n\n        return this._longDateFormat[key];\n    }\n\n    var defaultInvalidDate = 'Invalid date';\n\n    function invalidDate () {\n        return this._invalidDate;\n    }\n\n    var defaultOrdinal = '%d';\n    var defaultDayOfMonthOrdinalParse = /\\d{1,2}/;\n\n    function ordinal (number) {\n        return this._ordinal.replace('%d', number);\n    }\n\n    var defaultRelativeTime = {\n        future : 'in %s',\n        past   : '%s ago',\n        s  : 'a few seconds',\n        ss : '%d seconds',\n        m  : 'a minute',\n        mm : '%d minutes',\n        h  : 'an hour',\n        hh : '%d hours',\n        d  : 'a day',\n        dd : '%d days',\n        M  : 'a month',\n        MM : '%d months',\n        y  : 'a year',\n        yy : '%d years'\n    };\n\n    function relativeTime (number, withoutSuffix, string, isFuture) {\n        var output = this._relativeTime[string];\n        return (isFunction(output)) ?\n            output(number, withoutSuffix, string, isFuture) :\n            output.replace(/%d/i, number);\n    }\n\n    function pastFuture (diff, output) {\n        var format = this._relativeTime[diff > 0 ? 'future' : 'past'];\n        return isFunction(format) ? format(output) : format.replace(/%s/i, output);\n    }\n\n    var aliases = {};\n\n    function addUnitAlias (unit, shorthand) {\n        var lowerCase = unit.toLowerCase();\n        aliases[lowerCase] = aliases[lowerCase + 's'] = aliases[shorthand] = unit;\n    }\n\n    function normalizeUnits(units) {\n        return typeof units === 'string' ? aliases[units] || aliases[units.toLowerCase()] : undefined;\n    }\n\n    function normalizeObjectUnits(inputObject) {\n        var normalizedInput = {},\n            normalizedProp,\n            prop;\n\n        for (prop in inputObject) {\n            if (hasOwnProp(inputObject, prop)) {\n                normalizedProp = normalizeUnits(prop);\n                if (normalizedProp) {\n                    normalizedInput[normalizedProp] = inputObject[prop];\n                }\n            }\n        }\n\n        return normalizedInput;\n    }\n\n    var priorities = {};\n\n    function addUnitPriority(unit, priority) {\n        priorities[unit] = priority;\n    }\n\n    function getPrioritizedUnits(unitsObj) {\n        var units = [];\n        for (var u in unitsObj) {\n            units.push({unit: u, priority: priorities[u]});\n        }\n        units.sort(function (a, b) {\n            return a.priority - b.priority;\n        });\n        return units;\n    }\n\n    function zeroFill(number, targetLength, forceSign) {\n        var absNumber = '' + Math.abs(number),\n            zerosToFill = targetLength - absNumber.length,\n            sign = number >= 0;\n        return (sign ? (forceSign ? '+' : '') : '-') +\n            Math.pow(10, Math.max(0, zerosToFill)).toString().substr(1) + absNumber;\n    }\n\n    var formattingTokens = /(\\[[^\\[]*\\])|(\\\\)?([Hh]mm(ss)?|Mo|MM?M?M?|Do|DDDo|DD?D?D?|ddd?d?|do?|w[o|w]?|W[o|W]?|Qo?|YYYYYY|YYYYY|YYYY|YY|gg(ggg?)?|GG(GGG?)?|e|E|a|A|hh?|HH?|kk?|mm?|ss?|S{1,9}|x|X|zz?|ZZ?|.)/g;\n\n    var localFormattingTokens = /(\\[[^\\[]*\\])|(\\\\)?(LTS|LT|LL?L?L?|l{1,4})/g;\n\n    var formatFunctions = {};\n\n    var formatTokenFunctions = {};\n\n    // token:    'M'\n    // padded:   ['MM', 2]\n    // ordinal:  'Mo'\n    // callback: function () { this.month() + 1 }\n    function addFormatToken (token, padded, ordinal, callback) {\n        var func = callback;\n        if (typeof callback === 'string') {\n            func = function () {\n                return this[callback]();\n            };\n        }\n        if (token) {\n            formatTokenFunctions[token] = func;\n        }\n        if (padded) {\n            formatTokenFunctions[padded[0]] = function () {\n                return zeroFill(func.apply(this, arguments), padded[1], padded[2]);\n            };\n        }\n        if (ordinal) {\n            formatTokenFunctions[ordinal] = function () {\n                return this.localeData().ordinal(func.apply(this, arguments), token);\n            };\n        }\n    }\n\n    function removeFormattingTokens(input) {\n        if (input.match(/\\[[\\s\\S]/)) {\n            return input.replace(/^\\[|\\]$/g, '');\n        }\n        return input.replace(/\\\\/g, '');\n    }\n\n    function makeFormatFunction(format) {\n        var array = format.match(formattingTokens), i, length;\n\n        for (i = 0, length = array.length; i < length; i++) {\n            if (formatTokenFunctions[array[i]]) {\n                array[i] = formatTokenFunctions[array[i]];\n            } else {\n                array[i] = removeFormattingTokens(array[i]);\n            }\n        }\n\n        return function (mom) {\n            var output = '', i;\n            for (i = 0; i < length; i++) {\n                output += isFunction(array[i]) ? array[i].call(mom, format) : array[i];\n            }\n            return output;\n        };\n    }\n\n    // format date using native date object\n    function formatMoment(m, format) {\n        if (!m.isValid()) {\n            return m.localeData().invalidDate();\n        }\n\n        format = expandFormat(format, m.localeData());\n        formatFunctions[format] = formatFunctions[format] || makeFormatFunction(format);\n\n        return formatFunctions[format](m);\n    }\n\n    function expandFormat(format, locale) {\n        var i = 5;\n\n        function replaceLongDateFormatTokens(input) {\n            return locale.longDateFormat(input) || input;\n        }\n\n        localFormattingTokens.lastIndex = 0;\n        while (i >= 0 && localFormattingTokens.test(format)) {\n            format = format.replace(localFormattingTokens, replaceLongDateFormatTokens);\n            localFormattingTokens.lastIndex = 0;\n            i -= 1;\n        }\n\n        return format;\n    }\n\n    var match1         = /\\d/;            //       0 - 9\n    var match2         = /\\d\\d/;          //      00 - 99\n    var match3         = /\\d{3}/;         //     000 - 999\n    var match4         = /\\d{4}/;         //    0000 - 9999\n    var match6         = /[+-]?\\d{6}/;    // -999999 - 999999\n    var match1to2      = /\\d\\d?/;         //       0 - 99\n    var match3to4      = /\\d\\d\\d\\d?/;     //     999 - 9999\n    var match5to6      = /\\d\\d\\d\\d\\d\\d?/; //   99999 - 999999\n    var match1to3      = /\\d{1,3}/;       //       0 - 999\n    var match1to4      = /\\d{1,4}/;       //       0 - 9999\n    var match1to6      = /[+-]?\\d{1,6}/;  // -999999 - 999999\n\n    var matchUnsigned  = /\\d+/;           //       0 - inf\n    var matchSigned    = /[+-]?\\d+/;      //    -inf - inf\n\n    var matchOffset    = /Z|[+-]\\d\\d:?\\d\\d/gi; // +00:00 -00:00 +0000 -0000 or Z\n    var matchShortOffset = /Z|[+-]\\d\\d(?::?\\d\\d)?/gi; // +00 -00 +00:00 -00:00 +0000 -0000 or Z\n\n    var matchTimestamp = /[+-]?\\d+(\\.\\d{1,3})?/; // 123456789 123456789.123\n\n    // any word (or two) characters or numbers including two/three word month in arabic.\n    // includes scottish gaelic two word and hyphenated months\n    var matchWord = /[0-9]{0,256}['a-z\\u00A0-\\u05FF\\u0700-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFF07\\uFF10-\\uFFEF]{1,256}|[\\u0600-\\u06FF\\/]{1,256}(\\s*?[\\u0600-\\u06FF]{1,256}){1,2}/i;\n\n    var regexes = {};\n\n    function addRegexToken (token, regex, strictRegex) {\n        regexes[token] = isFunction(regex) ? regex : function (isStrict, localeData) {\n            return (isStrict && strictRegex) ? strictRegex : regex;\n        };\n    }\n\n    function getParseRegexForToken (token, config) {\n        if (!hasOwnProp(regexes, token)) {\n            return new RegExp(unescapeFormat(token));\n        }\n\n        return regexes[token](config._strict, config._locale);\n    }\n\n    // Code from http://stackoverflow.com/questions/3561493/is-there-a-regexp-escape-function-in-javascript\n    function unescapeFormat(s) {\n        return regexEscape(s.replace('\\\\', '').replace(/\\\\(\\[)|\\\\(\\])|\\[([^\\]\\[]*)\\]|\\\\(.)/g, function (matched, p1, p2, p3, p4) {\n            return p1 || p2 || p3 || p4;\n        }));\n    }\n\n    function regexEscape(s) {\n        return s.replace(/[-\\/\\\\^$*+?.()|[\\]{}]/g, '\\\\$&');\n    }\n\n    var tokens = {};\n\n    function addParseToken (token, callback) {\n        var i, func = callback;\n        if (typeof token === 'string') {\n            token = [token];\n        }\n        if (isNumber(callback)) {\n            func = function (input, array) {\n                array[callback] = toInt(input);\n            };\n        }\n        for (i = 0; i < token.length; i++) {\n            tokens[token[i]] = func;\n        }\n    }\n\n    function addWeekParseToken (token, callback) {\n        addParseToken(token, function (input, array, config, token) {\n            config._w = config._w || {};\n            callback(input, config._w, config, token);\n        });\n    }\n\n    function addTimeToArrayFromToken(token, input, config) {\n        if (input != null && hasOwnProp(tokens, token)) {\n            tokens[token](input, config._a, config, token);\n        }\n    }\n\n    var YEAR = 0;\n    var MONTH = 1;\n    var DATE = 2;\n    var HOUR = 3;\n    var MINUTE = 4;\n    var SECOND = 5;\n    var MILLISECOND = 6;\n    var WEEK = 7;\n    var WEEKDAY = 8;\n\n    // FORMATTING\n\n    addFormatToken('Y', 0, 0, function () {\n        var y = this.year();\n        return y <= 9999 ? '' + y : '+' + y;\n    });\n\n    addFormatToken(0, ['YY', 2], 0, function () {\n        return this.year() % 100;\n    });\n\n    addFormatToken(0, ['YYYY',   4],       0, 'year');\n    addFormatToken(0, ['YYYYY',  5],       0, 'year');\n    addFormatToken(0, ['YYYYYY', 6, true], 0, 'year');\n\n    // ALIASES\n\n    addUnitAlias('year', 'y');\n\n    // PRIORITIES\n\n    addUnitPriority('year', 1);\n\n    // PARSING\n\n    addRegexToken('Y',      matchSigned);\n    addRegexToken('YY',     match1to2, match2);\n    addRegexToken('YYYY',   match1to4, match4);\n    addRegexToken('YYYYY',  match1to6, match6);\n    addRegexToken('YYYYYY', match1to6, match6);\n\n    addParseToken(['YYYYY', 'YYYYYY'], YEAR);\n    addParseToken('YYYY', function (input, array) {\n        array[YEAR] = input.length === 2 ? hooks.parseTwoDigitYear(input) : toInt(input);\n    });\n    addParseToken('YY', function (input, array) {\n        array[YEAR] = hooks.parseTwoDigitYear(input);\n    });\n    addParseToken('Y', function (input, array) {\n        array[YEAR] = parseInt(input, 10);\n    });\n\n    // HELPERS\n\n    function daysInYear(year) {\n        return isLeapYear(year) ? 366 : 365;\n    }\n\n    function isLeapYear(year) {\n        return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0;\n    }\n\n    // HOOKS\n\n    hooks.parseTwoDigitYear = function (input) {\n        return toInt(input) + (toInt(input) > 68 ? 1900 : 2000);\n    };\n\n    // MOMENTS\n\n    var getSetYear = makeGetSet('FullYear', true);\n\n    function getIsLeapYear () {\n        return isLeapYear(this.year());\n    }\n\n    function makeGetSet (unit, keepTime) {\n        return function (value) {\n            if (value != null) {\n                set$1(this, unit, value);\n                hooks.updateOffset(this, keepTime);\n                return this;\n            } else {\n                return get(this, unit);\n            }\n        };\n    }\n\n    function get (mom, unit) {\n        return mom.isValid() ?\n            mom._d['get' + (mom._isUTC ? 'UTC' : '') + unit]() : NaN;\n    }\n\n    function set$1 (mom, unit, value) {\n        if (mom.isValid() && !isNaN(value)) {\n            if (unit === 'FullYear' && isLeapYear(mom.year()) && mom.month() === 1 && mom.date() === 29) {\n                mom._d['set' + (mom._isUTC ? 'UTC' : '') + unit](value, mom.month(), daysInMonth(value, mom.month()));\n            }\n            else {\n                mom._d['set' + (mom._isUTC ? 'UTC' : '') + unit](value);\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function stringGet (units) {\n        units = normalizeUnits(units);\n        if (isFunction(this[units])) {\n            return this[units]();\n        }\n        return this;\n    }\n\n\n    function stringSet (units, value) {\n        if (typeof units === 'object') {\n            units = normalizeObjectUnits(units);\n            var prioritized = getPrioritizedUnits(units);\n            for (var i = 0; i < prioritized.length; i++) {\n                this[prioritized[i].unit](units[prioritized[i].unit]);\n            }\n        } else {\n            units = normalizeUnits(units);\n            if (isFunction(this[units])) {\n                return this[units](value);\n            }\n        }\n        return this;\n    }\n\n    function mod(n, x) {\n        return ((n % x) + x) % x;\n    }\n\n    var indexOf;\n\n    if (Array.prototype.indexOf) {\n        indexOf = Array.prototype.indexOf;\n    } else {\n        indexOf = function (o) {\n            // I know\n            var i;\n            for (i = 0; i < this.length; ++i) {\n                if (this[i] === o) {\n                    return i;\n                }\n            }\n            return -1;\n        };\n    }\n\n    function daysInMonth(year, month) {\n        if (isNaN(year) || isNaN(month)) {\n            return NaN;\n        }\n        var modMonth = mod(month, 12);\n        year += (month - modMonth) / 12;\n        return modMonth === 1 ? (isLeapYear(year) ? 29 : 28) : (31 - modMonth % 7 % 2);\n    }\n\n    // FORMATTING\n\n    addFormatToken('M', ['MM', 2], 'Mo', function () {\n        return this.month() + 1;\n    });\n\n    addFormatToken('MMM', 0, 0, function (format) {\n        return this.localeData().monthsShort(this, format);\n    });\n\n    addFormatToken('MMMM', 0, 0, function (format) {\n        return this.localeData().months(this, format);\n    });\n\n    // ALIASES\n\n    addUnitAlias('month', 'M');\n\n    // PRIORITY\n\n    addUnitPriority('month', 8);\n\n    // PARSING\n\n    addRegexToken('M',    match1to2);\n    addRegexToken('MM',   match1to2, match2);\n    addRegexToken('MMM',  function (isStrict, locale) {\n        return locale.monthsShortRegex(isStrict);\n    });\n    addRegexToken('MMMM', function (isStrict, locale) {\n        return locale.monthsRegex(isStrict);\n    });\n\n    addParseToken(['M', 'MM'], function (input, array) {\n        array[MONTH] = toInt(input) - 1;\n    });\n\n    addParseToken(['MMM', 'MMMM'], function (input, array, config, token) {\n        var month = config._locale.monthsParse(input, token, config._strict);\n        // if we didn't find a month name, mark the date as invalid.\n        if (month != null) {\n            array[MONTH] = month;\n        } else {\n            getParsingFlags(config).invalidMonth = input;\n        }\n    });\n\n    // LOCALES\n\n    var MONTHS_IN_FORMAT = /D[oD]?(\\[[^\\[\\]]*\\]|\\s)+MMMM?/;\n    var defaultLocaleMonths = 'January_February_March_April_May_June_July_August_September_October_November_December'.split('_');\n    function localeMonths (m, format) {\n        if (!m) {\n            return isArray(this._months) ? this._months :\n                this._months['standalone'];\n        }\n        return isArray(this._months) ? this._months[m.month()] :\n            this._months[(this._months.isFormat || MONTHS_IN_FORMAT).test(format) ? 'format' : 'standalone'][m.month()];\n    }\n\n    var defaultLocaleMonthsShort = 'Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec'.split('_');\n    function localeMonthsShort (m, format) {\n        if (!m) {\n            return isArray(this._monthsShort) ? this._monthsShort :\n                this._monthsShort['standalone'];\n        }\n        return isArray(this._monthsShort) ? this._monthsShort[m.month()] :\n            this._monthsShort[MONTHS_IN_FORMAT.test(format) ? 'format' : 'standalone'][m.month()];\n    }\n\n    function handleStrictParse(monthName, format, strict) {\n        var i, ii, mom, llc = monthName.toLocaleLowerCase();\n        if (!this._monthsParse) {\n            // this is not used\n            this._monthsParse = [];\n            this._longMonthsParse = [];\n            this._shortMonthsParse = [];\n            for (i = 0; i < 12; ++i) {\n                mom = createUTC([2000, i]);\n                this._shortMonthsParse[i] = this.monthsShort(mom, '').toLocaleLowerCase();\n                this._longMonthsParse[i] = this.months(mom, '').toLocaleLowerCase();\n            }\n        }\n\n        if (strict) {\n            if (format === 'MMM') {\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._longMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        } else {\n            if (format === 'MMM') {\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._longMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._longMonthsParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortMonthsParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        }\n    }\n\n    function localeMonthsParse (monthName, format, strict) {\n        var i, mom, regex;\n\n        if (this._monthsParseExact) {\n            return handleStrictParse.call(this, monthName, format, strict);\n        }\n\n        if (!this._monthsParse) {\n            this._monthsParse = [];\n            this._longMonthsParse = [];\n            this._shortMonthsParse = [];\n        }\n\n        // TODO: add sorting\n        // Sorting makes sure if one month (or abbr) is a prefix of another\n        // see sorting in computeMonthsParse\n        for (i = 0; i < 12; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, i]);\n            if (strict && !this._longMonthsParse[i]) {\n                this._longMonthsParse[i] = new RegExp('^' + this.months(mom, '').replace('.', '') + '$', 'i');\n                this._shortMonthsParse[i] = new RegExp('^' + this.monthsShort(mom, '').replace('.', '') + '$', 'i');\n            }\n            if (!strict && !this._monthsParse[i]) {\n                regex = '^' + this.months(mom, '') + '|^' + this.monthsShort(mom, '');\n                this._monthsParse[i] = new RegExp(regex.replace('.', ''), 'i');\n            }\n            // test the regex\n            if (strict && format === 'MMMM' && this._longMonthsParse[i].test(monthName)) {\n                return i;\n            } else if (strict && format === 'MMM' && this._shortMonthsParse[i].test(monthName)) {\n                return i;\n            } else if (!strict && this._monthsParse[i].test(monthName)) {\n                return i;\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function setMonth (mom, value) {\n        var dayOfMonth;\n\n        if (!mom.isValid()) {\n            // No op\n            return mom;\n        }\n\n        if (typeof value === 'string') {\n            if (/^\\d+$/.test(value)) {\n                value = toInt(value);\n            } else {\n                value = mom.localeData().monthsParse(value);\n                // TODO: Another silent failure?\n                if (!isNumber(value)) {\n                    return mom;\n                }\n            }\n        }\n\n        dayOfMonth = Math.min(mom.date(), daysInMonth(mom.year(), value));\n        mom._d['set' + (mom._isUTC ? 'UTC' : '') + 'Month'](value, dayOfMonth);\n        return mom;\n    }\n\n    function getSetMonth (value) {\n        if (value != null) {\n            setMonth(this, value);\n            hooks.updateOffset(this, true);\n            return this;\n        } else {\n            return get(this, 'Month');\n        }\n    }\n\n    function getDaysInMonth () {\n        return daysInMonth(this.year(), this.month());\n    }\n\n    var defaultMonthsShortRegex = matchWord;\n    function monthsShortRegex (isStrict) {\n        if (this._monthsParseExact) {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                computeMonthsParse.call(this);\n            }\n            if (isStrict) {\n                return this._monthsShortStrictRegex;\n            } else {\n                return this._monthsShortRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_monthsShortRegex')) {\n                this._monthsShortRegex = defaultMonthsShortRegex;\n            }\n            return this._monthsShortStrictRegex && isStrict ?\n                this._monthsShortStrictRegex : this._monthsShortRegex;\n        }\n    }\n\n    var defaultMonthsRegex = matchWord;\n    function monthsRegex (isStrict) {\n        if (this._monthsParseExact) {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                computeMonthsParse.call(this);\n            }\n            if (isStrict) {\n                return this._monthsStrictRegex;\n            } else {\n                return this._monthsRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_monthsRegex')) {\n                this._monthsRegex = defaultMonthsRegex;\n            }\n            return this._monthsStrictRegex && isStrict ?\n                this._monthsStrictRegex : this._monthsRegex;\n        }\n    }\n\n    function computeMonthsParse () {\n        function cmpLenRev(a, b) {\n            return b.length - a.length;\n        }\n\n        var shortPieces = [], longPieces = [], mixedPieces = [],\n            i, mom;\n        for (i = 0; i < 12; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, i]);\n            shortPieces.push(this.monthsShort(mom, ''));\n            longPieces.push(this.months(mom, ''));\n            mixedPieces.push(this.months(mom, ''));\n            mixedPieces.push(this.monthsShort(mom, ''));\n        }\n        // Sorting makes sure if one month (or abbr) is a prefix of another it\n        // will match the longer piece.\n        shortPieces.sort(cmpLenRev);\n        longPieces.sort(cmpLenRev);\n        mixedPieces.sort(cmpLenRev);\n        for (i = 0; i < 12; i++) {\n            shortPieces[i] = regexEscape(shortPieces[i]);\n            longPieces[i] = regexEscape(longPieces[i]);\n        }\n        for (i = 0; i < 24; i++) {\n            mixedPieces[i] = regexEscape(mixedPieces[i]);\n        }\n\n        this._monthsRegex = new RegExp('^(' + mixedPieces.join('|') + ')', 'i');\n        this._monthsShortRegex = this._monthsRegex;\n        this._monthsStrictRegex = new RegExp('^(' + longPieces.join('|') + ')', 'i');\n        this._monthsShortStrictRegex = new RegExp('^(' + shortPieces.join('|') + ')', 'i');\n    }\n\n    function createDate (y, m, d, h, M, s, ms) {\n        // can't just apply() to create a date:\n        // https://stackoverflow.com/q/181348\n        var date;\n        // the date constructor remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            date = new Date(y + 400, m, d, h, M, s, ms);\n            if (isFinite(date.getFullYear())) {\n                date.setFullYear(y);\n            }\n        } else {\n            date = new Date(y, m, d, h, M, s, ms);\n        }\n\n        return date;\n    }\n\n    function createUTCDate (y) {\n        var date;\n        // the Date.UTC function remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            var args = Array.prototype.slice.call(arguments);\n            // preserve leap years using a full 400 year cycle, then reset\n            args[0] = y + 400;\n            date = new Date(Date.UTC.apply(null, args));\n            if (isFinite(date.getUTCFullYear())) {\n                date.setUTCFullYear(y);\n            }\n        } else {\n            date = new Date(Date.UTC.apply(null, arguments));\n        }\n\n        return date;\n    }\n\n    // start-of-first-week - start-of-year\n    function firstWeekOffset(year, dow, doy) {\n        var // first-week day -- which january is always in the first week (4 for iso, 1 for other)\n            fwd = 7 + dow - doy,\n            // first-week day local weekday -- which local weekday is fwd\n            fwdlw = (7 + createUTCDate(year, 0, fwd).getUTCDay() - dow) % 7;\n\n        return -fwdlw + fwd - 1;\n    }\n\n    // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_a_date_given_the_year.2C_week_number_and_weekday\n    function dayOfYearFromWeeks(year, week, weekday, dow, doy) {\n        var localWeekday = (7 + weekday - dow) % 7,\n            weekOffset = firstWeekOffset(year, dow, doy),\n            dayOfYear = 1 + 7 * (week - 1) + localWeekday + weekOffset,\n            resYear, resDayOfYear;\n\n        if (dayOfYear <= 0) {\n            resYear = year - 1;\n            resDayOfYear = daysInYear(resYear) + dayOfYear;\n        } else if (dayOfYear > daysInYear(year)) {\n            resYear = year + 1;\n            resDayOfYear = dayOfYear - daysInYear(year);\n        } else {\n            resYear = year;\n            resDayOfYear = dayOfYear;\n        }\n\n        return {\n            year: resYear,\n            dayOfYear: resDayOfYear\n        };\n    }\n\n    function weekOfYear(mom, dow, doy) {\n        var weekOffset = firstWeekOffset(mom.year(), dow, doy),\n            week = Math.floor((mom.dayOfYear() - weekOffset - 1) / 7) + 1,\n            resWeek, resYear;\n\n        if (week < 1) {\n            resYear = mom.year() - 1;\n            resWeek = week + weeksInYear(resYear, dow, doy);\n        } else if (week > weeksInYear(mom.year(), dow, doy)) {\n            resWeek = week - weeksInYear(mom.year(), dow, doy);\n            resYear = mom.year() + 1;\n        } else {\n            resYear = mom.year();\n            resWeek = week;\n        }\n\n        return {\n            week: resWeek,\n            year: resYear\n        };\n    }\n\n    function weeksInYear(year, dow, doy) {\n        var weekOffset = firstWeekOffset(year, dow, doy),\n            weekOffsetNext = firstWeekOffset(year + 1, dow, doy);\n        return (daysInYear(year) - weekOffset + weekOffsetNext) / 7;\n    }\n\n    // FORMATTING\n\n    addFormatToken('w', ['ww', 2], 'wo', 'week');\n    addFormatToken('W', ['WW', 2], 'Wo', 'isoWeek');\n\n    // ALIASES\n\n    addUnitAlias('week', 'w');\n    addUnitAlias('isoWeek', 'W');\n\n    // PRIORITIES\n\n    addUnitPriority('week', 5);\n    addUnitPriority('isoWeek', 5);\n\n    // PARSING\n\n    addRegexToken('w',  match1to2);\n    addRegexToken('ww', match1to2, match2);\n    addRegexToken('W',  match1to2);\n    addRegexToken('WW', match1to2, match2);\n\n    addWeekParseToken(['w', 'ww', 'W', 'WW'], function (input, week, config, token) {\n        week[token.substr(0, 1)] = toInt(input);\n    });\n\n    // HELPERS\n\n    // LOCALES\n\n    function localeWeek (mom) {\n        return weekOfYear(mom, this._week.dow, this._week.doy).week;\n    }\n\n    var defaultLocaleWeek = {\n        dow : 0, // Sunday is the first day of the week.\n        doy : 6  // The week that contains Jan 6th is the first week of the year.\n    };\n\n    function localeFirstDayOfWeek () {\n        return this._week.dow;\n    }\n\n    function localeFirstDayOfYear () {\n        return this._week.doy;\n    }\n\n    // MOMENTS\n\n    function getSetWeek (input) {\n        var week = this.localeData().week(this);\n        return input == null ? week : this.add((input - week) * 7, 'd');\n    }\n\n    function getSetISOWeek (input) {\n        var week = weekOfYear(this, 1, 4).week;\n        return input == null ? week : this.add((input - week) * 7, 'd');\n    }\n\n    // FORMATTING\n\n    addFormatToken('d', 0, 'do', 'day');\n\n    addFormatToken('dd', 0, 0, function (format) {\n        return this.localeData().weekdaysMin(this, format);\n    });\n\n    addFormatToken('ddd', 0, 0, function (format) {\n        return this.localeData().weekdaysShort(this, format);\n    });\n\n    addFormatToken('dddd', 0, 0, function (format) {\n        return this.localeData().weekdays(this, format);\n    });\n\n    addFormatToken('e', 0, 0, 'weekday');\n    addFormatToken('E', 0, 0, 'isoWeekday');\n\n    // ALIASES\n\n    addUnitAlias('day', 'd');\n    addUnitAlias('weekday', 'e');\n    addUnitAlias('isoWeekday', 'E');\n\n    // PRIORITY\n    addUnitPriority('day', 11);\n    addUnitPriority('weekday', 11);\n    addUnitPriority('isoWeekday', 11);\n\n    // PARSING\n\n    addRegexToken('d',    match1to2);\n    addRegexToken('e',    match1to2);\n    addRegexToken('E',    match1to2);\n    addRegexToken('dd',   function (isStrict, locale) {\n        return locale.weekdaysMinRegex(isStrict);\n    });\n    addRegexToken('ddd',   function (isStrict, locale) {\n        return locale.weekdaysShortRegex(isStrict);\n    });\n    addRegexToken('dddd',   function (isStrict, locale) {\n        return locale.weekdaysRegex(isStrict);\n    });\n\n    addWeekParseToken(['dd', 'ddd', 'dddd'], function (input, week, config, token) {\n        var weekday = config._locale.weekdaysParse(input, token, config._strict);\n        // if we didn't get a weekday name, mark the date as invalid\n        if (weekday != null) {\n            week.d = weekday;\n        } else {\n            getParsingFlags(config).invalidWeekday = input;\n        }\n    });\n\n    addWeekParseToken(['d', 'e', 'E'], function (input, week, config, token) {\n        week[token] = toInt(input);\n    });\n\n    // HELPERS\n\n    function parseWeekday(input, locale) {\n        if (typeof input !== 'string') {\n            return input;\n        }\n\n        if (!isNaN(input)) {\n            return parseInt(input, 10);\n        }\n\n        input = locale.weekdaysParse(input);\n        if (typeof input === 'number') {\n            return input;\n        }\n\n        return null;\n    }\n\n    function parseIsoWeekday(input, locale) {\n        if (typeof input === 'string') {\n            return locale.weekdaysParse(input) % 7 || 7;\n        }\n        return isNaN(input) ? null : input;\n    }\n\n    // LOCALES\n    function shiftWeekdays (ws, n) {\n        return ws.slice(n, 7).concat(ws.slice(0, n));\n    }\n\n    var defaultLocaleWeekdays = 'Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday'.split('_');\n    function localeWeekdays (m, format) {\n        var weekdays = isArray(this._weekdays) ? this._weekdays :\n            this._weekdays[(m && m !== true && this._weekdays.isFormat.test(format)) ? 'format' : 'standalone'];\n        return (m === true) ? shiftWeekdays(weekdays, this._week.dow)\n            : (m) ? weekdays[m.day()] : weekdays;\n    }\n\n    var defaultLocaleWeekdaysShort = 'Sun_Mon_Tue_Wed_Thu_Fri_Sat'.split('_');\n    function localeWeekdaysShort (m) {\n        return (m === true) ? shiftWeekdays(this._weekdaysShort, this._week.dow)\n            : (m) ? this._weekdaysShort[m.day()] : this._weekdaysShort;\n    }\n\n    var defaultLocaleWeekdaysMin = 'Su_Mo_Tu_We_Th_Fr_Sa'.split('_');\n    function localeWeekdaysMin (m) {\n        return (m === true) ? shiftWeekdays(this._weekdaysMin, this._week.dow)\n            : (m) ? this._weekdaysMin[m.day()] : this._weekdaysMin;\n    }\n\n    function handleStrictParse$1(weekdayName, format, strict) {\n        var i, ii, mom, llc = weekdayName.toLocaleLowerCase();\n        if (!this._weekdaysParse) {\n            this._weekdaysParse = [];\n            this._shortWeekdaysParse = [];\n            this._minWeekdaysParse = [];\n\n            for (i = 0; i < 7; ++i) {\n                mom = createUTC([2000, 1]).day(i);\n                this._minWeekdaysParse[i] = this.weekdaysMin(mom, '').toLocaleLowerCase();\n                this._shortWeekdaysParse[i] = this.weekdaysShort(mom, '').toLocaleLowerCase();\n                this._weekdaysParse[i] = this.weekdays(mom, '').toLocaleLowerCase();\n            }\n        }\n\n        if (strict) {\n            if (format === 'dddd') {\n                ii = indexOf.call(this._weekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else if (format === 'ddd') {\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        } else {\n            if (format === 'dddd') {\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else if (format === 'ddd') {\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            } else {\n                ii = indexOf.call(this._minWeekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._weekdaysParse, llc);\n                if (ii !== -1) {\n                    return ii;\n                }\n                ii = indexOf.call(this._shortWeekdaysParse, llc);\n                return ii !== -1 ? ii : null;\n            }\n        }\n    }\n\n    function localeWeekdaysParse (weekdayName, format, strict) {\n        var i, mom, regex;\n\n        if (this._weekdaysParseExact) {\n            return handleStrictParse$1.call(this, weekdayName, format, strict);\n        }\n\n        if (!this._weekdaysParse) {\n            this._weekdaysParse = [];\n            this._minWeekdaysParse = [];\n            this._shortWeekdaysParse = [];\n            this._fullWeekdaysParse = [];\n        }\n\n        for (i = 0; i < 7; i++) {\n            // make the regex if we don't have it already\n\n            mom = createUTC([2000, 1]).day(i);\n            if (strict && !this._fullWeekdaysParse[i]) {\n                this._fullWeekdaysParse[i] = new RegExp('^' + this.weekdays(mom, '').replace('.', '\\\\.?') + '$', 'i');\n                this._shortWeekdaysParse[i] = new RegExp('^' + this.weekdaysShort(mom, '').replace('.', '\\\\.?') + '$', 'i');\n                this._minWeekdaysParse[i] = new RegExp('^' + this.weekdaysMin(mom, '').replace('.', '\\\\.?') + '$', 'i');\n            }\n            if (!this._weekdaysParse[i]) {\n                regex = '^' + this.weekdays(mom, '') + '|^' + this.weekdaysShort(mom, '') + '|^' + this.weekdaysMin(mom, '');\n                this._weekdaysParse[i] = new RegExp(regex.replace('.', ''), 'i');\n            }\n            // test the regex\n            if (strict && format === 'dddd' && this._fullWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (strict && format === 'ddd' && this._shortWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (strict && format === 'dd' && this._minWeekdaysParse[i].test(weekdayName)) {\n                return i;\n            } else if (!strict && this._weekdaysParse[i].test(weekdayName)) {\n                return i;\n            }\n        }\n    }\n\n    // MOMENTS\n\n    function getSetDayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        var day = this._isUTC ? this._d.getUTCDay() : this._d.getDay();\n        if (input != null) {\n            input = parseWeekday(input, this.localeData());\n            return this.add(input - day, 'd');\n        } else {\n            return day;\n        }\n    }\n\n    function getSetLocaleDayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        var weekday = (this.day() + 7 - this.localeData()._week.dow) % 7;\n        return input == null ? weekday : this.add(input - weekday, 'd');\n    }\n\n    function getSetISODayOfWeek (input) {\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n\n        // behaves the same as moment#day except\n        // as a getter, returns 7 instead of 0 (1-7 range instead of 0-6)\n        // as a setter, sunday should belong to the previous week.\n\n        if (input != null) {\n            var weekday = parseIsoWeekday(input, this.localeData());\n            return this.day(this.day() % 7 ? weekday : weekday - 7);\n        } else {\n            return this.day() || 7;\n        }\n    }\n\n    var defaultWeekdaysRegex = matchWord;\n    function weekdaysRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysStrictRegex;\n            } else {\n                return this._weekdaysRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                this._weekdaysRegex = defaultWeekdaysRegex;\n            }\n            return this._weekdaysStrictRegex && isStrict ?\n                this._weekdaysStrictRegex : this._weekdaysRegex;\n        }\n    }\n\n    var defaultWeekdaysShortRegex = matchWord;\n    function weekdaysShortRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysShortStrictRegex;\n            } else {\n                return this._weekdaysShortRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysShortRegex')) {\n                this._weekdaysShortRegex = defaultWeekdaysShortRegex;\n            }\n            return this._weekdaysShortStrictRegex && isStrict ?\n                this._weekdaysShortStrictRegex : this._weekdaysShortRegex;\n        }\n    }\n\n    var defaultWeekdaysMinRegex = matchWord;\n    function weekdaysMinRegex (isStrict) {\n        if (this._weekdaysParseExact) {\n            if (!hasOwnProp(this, '_weekdaysRegex')) {\n                computeWeekdaysParse.call(this);\n            }\n            if (isStrict) {\n                return this._weekdaysMinStrictRegex;\n            } else {\n                return this._weekdaysMinRegex;\n            }\n        } else {\n            if (!hasOwnProp(this, '_weekdaysMinRegex')) {\n                this._weekdaysMinRegex = defaultWeekdaysMinRegex;\n            }\n            return this._weekdaysMinStrictRegex && isStrict ?\n                this._weekdaysMinStrictRegex : this._weekdaysMinRegex;\n        }\n    }\n\n\n    function computeWeekdaysParse () {\n        function cmpLenRev(a, b) {\n            return b.length - a.length;\n        }\n\n        var minPieces = [], shortPieces = [], longPieces = [], mixedPieces = [],\n            i, mom, minp, shortp, longp;\n        for (i = 0; i < 7; i++) {\n            // make the regex if we don't have it already\n            mom = createUTC([2000, 1]).day(i);\n            minp = this.weekdaysMin(mom, '');\n            shortp = this.weekdaysShort(mom, '');\n            longp = this.weekdays(mom, '');\n            minPieces.push(minp);\n            shortPieces.push(shortp);\n            longPieces.push(longp);\n            mixedPieces.push(minp);\n            mixedPieces.push(shortp);\n            mixedPieces.push(longp);\n        }\n        // Sorting makes sure if one weekday (or abbr) is a prefix of another it\n        // will match the longer piece.\n        minPieces.sort(cmpLenRev);\n        shortPieces.sort(cmpLenRev);\n        longPieces.sort(cmpLenRev);\n        mixedPieces.sort(cmpLenRev);\n        for (i = 0; i < 7; i++) {\n            shortPieces[i] = regexEscape(shortPieces[i]);\n            longPieces[i] = regexEscape(longPieces[i]);\n            mixedPieces[i] = regexEscape(mixedPieces[i]);\n        }\n\n        this._weekdaysRegex = new RegExp('^(' + mixedPieces.join('|') + ')', 'i');\n        this._weekdaysShortRegex = this._weekdaysRegex;\n        this._weekdaysMinRegex = this._weekdaysRegex;\n\n        this._weekdaysStrictRegex = new RegExp('^(' + longPieces.join('|') + ')', 'i');\n        this._weekdaysShortStrictRegex = new RegExp('^(' + shortPieces.join('|') + ')', 'i');\n        this._weekdaysMinStrictRegex = new RegExp('^(' + minPieces.join('|') + ')', 'i');\n    }\n\n    // FORMATTING\n\n    function hFormat() {\n        return this.hours() % 12 || 12;\n    }\n\n    function kFormat() {\n        return this.hours() || 24;\n    }\n\n    addFormatToken('H', ['HH', 2], 0, 'hour');\n    addFormatToken('h', ['hh', 2], 0, hFormat);\n    addFormatToken('k', ['kk', 2], 0, kFormat);\n\n    addFormatToken('hmm', 0, 0, function () {\n        return '' + hFormat.apply(this) + zeroFill(this.minutes(), 2);\n    });\n\n    addFormatToken('hmmss', 0, 0, function () {\n        return '' + hFormat.apply(this) + zeroFill(this.minutes(), 2) +\n            zeroFill(this.seconds(), 2);\n    });\n\n    addFormatToken('Hmm', 0, 0, function () {\n        return '' + this.hours() + zeroFill(this.minutes(), 2);\n    });\n\n    addFormatToken('Hmmss', 0, 0, function () {\n        return '' + this.hours() + zeroFill(this.minutes(), 2) +\n            zeroFill(this.seconds(), 2);\n    });\n\n    function meridiem (token, lowercase) {\n        addFormatToken(token, 0, 0, function () {\n            return this.localeData().meridiem(this.hours(), this.minutes(), lowercase);\n        });\n    }\n\n    meridiem('a', true);\n    meridiem('A', false);\n\n    // ALIASES\n\n    addUnitAlias('hour', 'h');\n\n    // PRIORITY\n    addUnitPriority('hour', 13);\n\n    // PARSING\n\n    function matchMeridiem (isStrict, locale) {\n        return locale._meridiemParse;\n    }\n\n    addRegexToken('a',  matchMeridiem);\n    addRegexToken('A',  matchMeridiem);\n    addRegexToken('H',  match1to2);\n    addRegexToken('h',  match1to2);\n    addRegexToken('k',  match1to2);\n    addRegexToken('HH', match1to2, match2);\n    addRegexToken('hh', match1to2, match2);\n    addRegexToken('kk', match1to2, match2);\n\n    addRegexToken('hmm', match3to4);\n    addRegexToken('hmmss', match5to6);\n    addRegexToken('Hmm', match3to4);\n    addRegexToken('Hmmss', match5to6);\n\n    addParseToken(['H', 'HH'], HOUR);\n    addParseToken(['k', 'kk'], function (input, array, config) {\n        var kInput = toInt(input);\n        array[HOUR] = kInput === 24 ? 0 : kInput;\n    });\n    addParseToken(['a', 'A'], function (input, array, config) {\n        config._isPm = config._locale.isPM(input);\n        config._meridiem = input;\n    });\n    addParseToken(['h', 'hh'], function (input, array, config) {\n        array[HOUR] = toInt(input);\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('hmm', function (input, array, config) {\n        var pos = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos));\n        array[MINUTE] = toInt(input.substr(pos));\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('hmmss', function (input, array, config) {\n        var pos1 = input.length - 4;\n        var pos2 = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos1));\n        array[MINUTE] = toInt(input.substr(pos1, 2));\n        array[SECOND] = toInt(input.substr(pos2));\n        getParsingFlags(config).bigHour = true;\n    });\n    addParseToken('Hmm', function (input, array, config) {\n        var pos = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos));\n        array[MINUTE] = toInt(input.substr(pos));\n    });\n    addParseToken('Hmmss', function (input, array, config) {\n        var pos1 = input.length - 4;\n        var pos2 = input.length - 2;\n        array[HOUR] = toInt(input.substr(0, pos1));\n        array[MINUTE] = toInt(input.substr(pos1, 2));\n        array[SECOND] = toInt(input.substr(pos2));\n    });\n\n    // LOCALES\n\n    function localeIsPM (input) {\n        // IE8 Quirks Mode & IE7 Standards Mode do not allow accessing strings like arrays\n        // Using charAt should be more compatible.\n        return ((input + '').toLowerCase().charAt(0) === 'p');\n    }\n\n    var defaultLocaleMeridiemParse = /[ap]\\.?m?\\.?/i;\n    function localeMeridiem (hours, minutes, isLower) {\n        if (hours > 11) {\n            return isLower ? 'pm' : 'PM';\n        } else {\n            return isLower ? 'am' : 'AM';\n        }\n    }\n\n\n    // MOMENTS\n\n    // Setting the hour should keep the time, because the user explicitly\n    // specified which hour they want. So trying to maintain the same hour (in\n    // a new timezone) makes sense. Adding/subtracting hours does not follow\n    // this rule.\n    var getSetHour = makeGetSet('Hours', true);\n\n    var baseConfig = {\n        calendar: defaultCalendar,\n        longDateFormat: defaultLongDateFormat,\n        invalidDate: defaultInvalidDate,\n        ordinal: defaultOrdinal,\n        dayOfMonthOrdinalParse: defaultDayOfMonthOrdinalParse,\n        relativeTime: defaultRelativeTime,\n\n        months: defaultLocaleMonths,\n        monthsShort: defaultLocaleMonthsShort,\n\n        week: defaultLocaleWeek,\n\n        weekdays: defaultLocaleWeekdays,\n        weekdaysMin: defaultLocaleWeekdaysMin,\n        weekdaysShort: defaultLocaleWeekdaysShort,\n\n        meridiemParse: defaultLocaleMeridiemParse\n    };\n\n    // internal storage for locale config files\n    var locales = {};\n    var localeFamilies = {};\n    var globalLocale;\n\n    function normalizeLocale(key) {\n        return key ? key.toLowerCase().replace('_', '-') : key;\n    }\n\n    // pick the locale from the array\n    // try ['en-au', 'en-gb'] as 'en-au', 'en-gb', 'en', as in move through the list trying each\n    // substring from most specific to least, but move to the next array item if it's a more specific variant than the current root\n    function chooseLocale(names) {\n        var i = 0, j, next, locale, split;\n\n        while (i < names.length) {\n            split = normalizeLocale(names[i]).split('-');\n            j = split.length;\n            next = normalizeLocale(names[i + 1]);\n            next = next ? next.split('-') : null;\n            while (j > 0) {\n                locale = loadLocale(split.slice(0, j).join('-'));\n                if (locale) {\n                    return locale;\n                }\n                if (next && next.length >= j && compareArrays(split, next, true) >= j - 1) {\n                    //the next array item is better than a shallower substring of this one\n                    break;\n                }\n                j--;\n            }\n            i++;\n        }\n        return globalLocale;\n    }\n\n    function loadLocale(name) {\n        var oldLocale = null;\n        // TODO: Find a better way to register and load all the locales in Node\n        if (!locales[name] && (typeof module !== 'undefined') &&\n                module && module.exports) {\n            try {\n                oldLocale = globalLocale._abbr;\n                var aliasedRequire = require;\n                aliasedRequire('./locale/' + name);\n                getSetGlobalLocale(oldLocale);\n            } catch (e) {}\n        }\n        return locales[name];\n    }\n\n    // This function will load locale and then set the global locale.  If\n    // no arguments are passed in, it will simply return the current global\n    // locale key.\n    function getSetGlobalLocale (key, values) {\n        var data;\n        if (key) {\n            if (isUndefined(values)) {\n                data = getLocale(key);\n            }\n            else {\n                data = defineLocale(key, values);\n            }\n\n            if (data) {\n                // moment.duration._locale = moment._locale = data;\n                globalLocale = data;\n            }\n            else {\n                if ((typeof console !==  'undefined') && console.warn) {\n                    //warn user if arguments are passed but the locale could not be set\n                    console.warn('Locale ' + key +  ' not found. Did you forget to load it?');\n                }\n            }\n        }\n\n        return globalLocale._abbr;\n    }\n\n    function defineLocale (name, config) {\n        if (config !== null) {\n            var locale, parentConfig = baseConfig;\n            config.abbr = name;\n            if (locales[name] != null) {\n                deprecateSimple('defineLocaleOverride',\n                        'use moment.updateLocale(localeName, config) to change ' +\n                        'an existing locale. moment.defineLocale(localeName, ' +\n                        'config) should only be used for creating a new locale ' +\n                        'See http://momentjs.com/guides/#/warnings/define-locale/ for more info.');\n                parentConfig = locales[name]._config;\n            } else if (config.parentLocale != null) {\n                if (locales[config.parentLocale] != null) {\n                    parentConfig = locales[config.parentLocale]._config;\n                } else {\n                    locale = loadLocale(config.parentLocale);\n                    if (locale != null) {\n                        parentConfig = locale._config;\n                    } else {\n                        if (!localeFamilies[config.parentLocale]) {\n                            localeFamilies[config.parentLocale] = [];\n                        }\n                        localeFamilies[config.parentLocale].push({\n                            name: name,\n                            config: config\n                        });\n                        return null;\n                    }\n                }\n            }\n            locales[name] = new Locale(mergeConfigs(parentConfig, config));\n\n            if (localeFamilies[name]) {\n                localeFamilies[name].forEach(function (x) {\n                    defineLocale(x.name, x.config);\n                });\n            }\n\n            // backwards compat for now: also set the locale\n            // make sure we set the locale AFTER all child locales have been\n            // created, so we won't end up with the child locale set.\n            getSetGlobalLocale(name);\n\n\n            return locales[name];\n        } else {\n            // useful for testing\n            delete locales[name];\n            return null;\n        }\n    }\n\n    function updateLocale(name, config) {\n        if (config != null) {\n            var locale, tmpLocale, parentConfig = baseConfig;\n            // MERGE\n            tmpLocale = loadLocale(name);\n            if (tmpLocale != null) {\n                parentConfig = tmpLocale._config;\n            }\n            config = mergeConfigs(parentConfig, config);\n            locale = new Locale(config);\n            locale.parentLocale = locales[name];\n            locales[name] = locale;\n\n            // backwards compat for now: also set the locale\n            getSetGlobalLocale(name);\n        } else {\n            // pass null for config to unupdate, useful for tests\n            if (locales[name] != null) {\n                if (locales[name].parentLocale != null) {\n                    locales[name] = locales[name].parentLocale;\n                } else if (locales[name] != null) {\n                    delete locales[name];\n                }\n            }\n        }\n        return locales[name];\n    }\n\n    // returns locale data\n    function getLocale (key) {\n        var locale;\n\n        if (key && key._locale && key._locale._abbr) {\n            key = key._locale._abbr;\n        }\n\n        if (!key) {\n            return globalLocale;\n        }\n\n        if (!isArray(key)) {\n            //short-circuit everything else\n            locale = loadLocale(key);\n            if (locale) {\n                return locale;\n            }\n            key = [key];\n        }\n\n        return chooseLocale(key);\n    }\n\n    function listLocales() {\n        return keys(locales);\n    }\n\n    function checkOverflow (m) {\n        var overflow;\n        var a = m._a;\n\n        if (a && getParsingFlags(m).overflow === -2) {\n            overflow =\n                a[MONTH]       < 0 || a[MONTH]       > 11  ? MONTH :\n                a[DATE]        < 1 || a[DATE]        > daysInMonth(a[YEAR], a[MONTH]) ? DATE :\n                a[HOUR]        < 0 || a[HOUR]        > 24 || (a[HOUR] === 24 && (a[MINUTE] !== 0 || a[SECOND] !== 0 || a[MILLISECOND] !== 0)) ? HOUR :\n                a[MINUTE]      < 0 || a[MINUTE]      > 59  ? MINUTE :\n                a[SECOND]      < 0 || a[SECOND]      > 59  ? SECOND :\n                a[MILLISECOND] < 0 || a[MILLISECOND] > 999 ? MILLISECOND :\n                -1;\n\n            if (getParsingFlags(m)._overflowDayOfYear && (overflow < YEAR || overflow > DATE)) {\n                overflow = DATE;\n            }\n            if (getParsingFlags(m)._overflowWeeks && overflow === -1) {\n                overflow = WEEK;\n            }\n            if (getParsingFlags(m)._overflowWeekday && overflow === -1) {\n                overflow = WEEKDAY;\n            }\n\n            getParsingFlags(m).overflow = overflow;\n        }\n\n        return m;\n    }\n\n    // Pick the first defined of two or three arguments.\n    function defaults(a, b, c) {\n        if (a != null) {\n            return a;\n        }\n        if (b != null) {\n            return b;\n        }\n        return c;\n    }\n\n    function currentDateArray(config) {\n        // hooks is actually the exported moment object\n        var nowValue = new Date(hooks.now());\n        if (config._useUTC) {\n            return [nowValue.getUTCFullYear(), nowValue.getUTCMonth(), nowValue.getUTCDate()];\n        }\n        return [nowValue.getFullYear(), nowValue.getMonth(), nowValue.getDate()];\n    }\n\n    // convert an array to a date.\n    // the array should mirror the parameters below\n    // note: all values past the year are optional and will default to the lowest possible value.\n    // [year, month, day , hour, minute, second, millisecond]\n    function configFromArray (config) {\n        var i, date, input = [], currentDate, expectedWeekday, yearToUse;\n\n        if (config._d) {\n            return;\n        }\n\n        currentDate = currentDateArray(config);\n\n        //compute day of the year from weeks and weekdays\n        if (config._w && config._a[DATE] == null && config._a[MONTH] == null) {\n            dayOfYearFromWeekInfo(config);\n        }\n\n        //if the day of the year is set, figure out what it is\n        if (config._dayOfYear != null) {\n            yearToUse = defaults(config._a[YEAR], currentDate[YEAR]);\n\n            if (config._dayOfYear > daysInYear(yearToUse) || config._dayOfYear === 0) {\n                getParsingFlags(config)._overflowDayOfYear = true;\n            }\n\n            date = createUTCDate(yearToUse, 0, config._dayOfYear);\n            config._a[MONTH] = date.getUTCMonth();\n            config._a[DATE] = date.getUTCDate();\n        }\n\n        // Default to current date.\n        // * if no year, month, day of month are given, default to today\n        // * if day of month is given, default month and year\n        // * if month is given, default only year\n        // * if year is given, don't default anything\n        for (i = 0; i < 3 && config._a[i] == null; ++i) {\n            config._a[i] = input[i] = currentDate[i];\n        }\n\n        // Zero out whatever was not defaulted, including time\n        for (; i < 7; i++) {\n            config._a[i] = input[i] = (config._a[i] == null) ? (i === 2 ? 1 : 0) : config._a[i];\n        }\n\n        // Check for 24:00:00.000\n        if (config._a[HOUR] === 24 &&\n                config._a[MINUTE] === 0 &&\n                config._a[SECOND] === 0 &&\n                config._a[MILLISECOND] === 0) {\n            config._nextDay = true;\n            config._a[HOUR] = 0;\n        }\n\n        config._d = (config._useUTC ? createUTCDate : createDate).apply(null, input);\n        expectedWeekday = config._useUTC ? config._d.getUTCDay() : config._d.getDay();\n\n        // Apply timezone offset from input. The actual utcOffset can be changed\n        // with parseZone.\n        if (config._tzm != null) {\n            config._d.setUTCMinutes(config._d.getUTCMinutes() - config._tzm);\n        }\n\n        if (config._nextDay) {\n            config._a[HOUR] = 24;\n        }\n\n        // check for mismatching day of week\n        if (config._w && typeof config._w.d !== 'undefined' && config._w.d !== expectedWeekday) {\n            getParsingFlags(config).weekdayMismatch = true;\n        }\n    }\n\n    function dayOfYearFromWeekInfo(config) {\n        var w, weekYear, week, weekday, dow, doy, temp, weekdayOverflow;\n\n        w = config._w;\n        if (w.GG != null || w.W != null || w.E != null) {\n            dow = 1;\n            doy = 4;\n\n            // TODO: We need to take the current isoWeekYear, but that depends on\n            // how we interpret now (local, utc, fixed offset). So create\n            // a now version of current config (take local/utc/offset flags, and\n            // create now).\n            weekYear = defaults(w.GG, config._a[YEAR], weekOfYear(createLocal(), 1, 4).year);\n            week = defaults(w.W, 1);\n            weekday = defaults(w.E, 1);\n            if (weekday < 1 || weekday > 7) {\n                weekdayOverflow = true;\n            }\n        } else {\n            dow = config._locale._week.dow;\n            doy = config._locale._week.doy;\n\n            var curWeek = weekOfYear(createLocal(), dow, doy);\n\n            weekYear = defaults(w.gg, config._a[YEAR], curWeek.year);\n\n            // Default to current week.\n            week = defaults(w.w, curWeek.week);\n\n            if (w.d != null) {\n                // weekday -- low day numbers are considered next week\n                weekday = w.d;\n                if (weekday < 0 || weekday > 6) {\n                    weekdayOverflow = true;\n                }\n            } else if (w.e != null) {\n                // local weekday -- counting starts from beginning of week\n                weekday = w.e + dow;\n                if (w.e < 0 || w.e > 6) {\n                    weekdayOverflow = true;\n                }\n            } else {\n                // default to beginning of week\n                weekday = dow;\n            }\n        }\n        if (week < 1 || week > weeksInYear(weekYear, dow, doy)) {\n            getParsingFlags(config)._overflowWeeks = true;\n        } else if (weekdayOverflow != null) {\n            getParsingFlags(config)._overflowWeekday = true;\n        } else {\n            temp = dayOfYearFromWeeks(weekYear, week, weekday, dow, doy);\n            config._a[YEAR] = temp.year;\n            config._dayOfYear = temp.dayOfYear;\n        }\n    }\n\n    // iso 8601 regex\n    // 0000-00-00 0000-W00 or 0000-W00-0 + T + 00 or 00:00 or 00:00:00 or 00:00:00.000 + +00:00 or +0000 or +00)\n    var extendedIsoRegex = /^\\s*((?:[+-]\\d{6}|\\d{4})-(?:\\d\\d-\\d\\d|W\\d\\d-\\d|W\\d\\d|\\d\\d\\d|\\d\\d))(?:(T| )(\\d\\d(?::\\d\\d(?::\\d\\d(?:[.,]\\d+)?)?)?)([\\+\\-]\\d\\d(?::?\\d\\d)?|\\s*Z)?)?$/;\n    var basicIsoRegex = /^\\s*((?:[+-]\\d{6}|\\d{4})(?:\\d\\d\\d\\d|W\\d\\d\\d|W\\d\\d|\\d\\d\\d|\\d\\d))(?:(T| )(\\d\\d(?:\\d\\d(?:\\d\\d(?:[.,]\\d+)?)?)?)([\\+\\-]\\d\\d(?::?\\d\\d)?|\\s*Z)?)?$/;\n\n    var tzRegex = /Z|[+-]\\d\\d(?::?\\d\\d)?/;\n\n    var isoDates = [\n        ['YYYYYY-MM-DD', /[+-]\\d{6}-\\d\\d-\\d\\d/],\n        ['YYYY-MM-DD', /\\d{4}-\\d\\d-\\d\\d/],\n        ['GGGG-[W]WW-E', /\\d{4}-W\\d\\d-\\d/],\n        ['GGGG-[W]WW', /\\d{4}-W\\d\\d/, false],\n        ['YYYY-DDD', /\\d{4}-\\d{3}/],\n        ['YYYY-MM', /\\d{4}-\\d\\d/, false],\n        ['YYYYYYMMDD', /[+-]\\d{10}/],\n        ['YYYYMMDD', /\\d{8}/],\n        // YYYYMM is NOT allowed by the standard\n        ['GGGG[W]WWE', /\\d{4}W\\d{3}/],\n        ['GGGG[W]WW', /\\d{4}W\\d{2}/, false],\n        ['YYYYDDD', /\\d{7}/]\n    ];\n\n    // iso time formats and regexes\n    var isoTimes = [\n        ['HH:mm:ss.SSSS', /\\d\\d:\\d\\d:\\d\\d\\.\\d+/],\n        ['HH:mm:ss,SSSS', /\\d\\d:\\d\\d:\\d\\d,\\d+/],\n        ['HH:mm:ss', /\\d\\d:\\d\\d:\\d\\d/],\n        ['HH:mm', /\\d\\d:\\d\\d/],\n        ['HHmmss.SSSS', /\\d\\d\\d\\d\\d\\d\\.\\d+/],\n        ['HHmmss,SSSS', /\\d\\d\\d\\d\\d\\d,\\d+/],\n        ['HHmmss', /\\d\\d\\d\\d\\d\\d/],\n        ['HHmm', /\\d\\d\\d\\d/],\n        ['HH', /\\d\\d/]\n    ];\n\n    var aspNetJsonRegex = /^\\/?Date\\((\\-?\\d+)/i;\n\n    // date from iso format\n    function configFromISO(config) {\n        var i, l,\n            string = config._i,\n            match = extendedIsoRegex.exec(string) || basicIsoRegex.exec(string),\n            allowTime, dateFormat, timeFormat, tzFormat;\n\n        if (match) {\n            getParsingFlags(config).iso = true;\n\n            for (i = 0, l = isoDates.length; i < l; i++) {\n                if (isoDates[i][1].exec(match[1])) {\n                    dateFormat = isoDates[i][0];\n                    allowTime = isoDates[i][2] !== false;\n                    break;\n                }\n            }\n            if (dateFormat == null) {\n                config._isValid = false;\n                return;\n            }\n            if (match[3]) {\n                for (i = 0, l = isoTimes.length; i < l; i++) {\n                    if (isoTimes[i][1].exec(match[3])) {\n                        // match[2] should be 'T' or space\n                        timeFormat = (match[2] || ' ') + isoTimes[i][0];\n                        break;\n                    }\n                }\n                if (timeFormat == null) {\n                    config._isValid = false;\n                    return;\n                }\n            }\n            if (!allowTime && timeFormat != null) {\n                config._isValid = false;\n                return;\n            }\n            if (match[4]) {\n                if (tzRegex.exec(match[4])) {\n                    tzFormat = 'Z';\n                } else {\n                    config._isValid = false;\n                    return;\n                }\n            }\n            config._f = dateFormat + (timeFormat || '') + (tzFormat || '');\n            configFromStringAndFormat(config);\n        } else {\n            config._isValid = false;\n        }\n    }\n\n    // RFC 2822 regex: For details see https://tools.ietf.org/html/rfc2822#section-3.3\n    var rfc2822 = /^(?:(Mon|Tue|Wed|Thu|Fri|Sat|Sun),?\\s)?(\\d{1,2})\\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\\s(\\d{2,4})\\s(\\d\\d):(\\d\\d)(?::(\\d\\d))?\\s(?:(UT|GMT|[ECMP][SD]T)|([Zz])|([+-]\\d{4}))$/;\n\n    function extractFromRFC2822Strings(yearStr, monthStr, dayStr, hourStr, minuteStr, secondStr) {\n        var result = [\n            untruncateYear(yearStr),\n            defaultLocaleMonthsShort.indexOf(monthStr),\n            parseInt(dayStr, 10),\n            parseInt(hourStr, 10),\n            parseInt(minuteStr, 10)\n        ];\n\n        if (secondStr) {\n            result.push(parseInt(secondStr, 10));\n        }\n\n        return result;\n    }\n\n    function untruncateYear(yearStr) {\n        var year = parseInt(yearStr, 10);\n        if (year <= 49) {\n            return 2000 + year;\n        } else if (year <= 999) {\n            return 1900 + year;\n        }\n        return year;\n    }\n\n    function preprocessRFC2822(s) {\n        // Remove comments and folding whitespace and replace multiple-spaces with a single space\n        return s.replace(/\\([^)]*\\)|[\\n\\t]/g, ' ').replace(/(\\s\\s+)/g, ' ').replace(/^\\s\\s*/, '').replace(/\\s\\s*$/, '');\n    }\n\n    function checkWeekday(weekdayStr, parsedInput, config) {\n        if (weekdayStr) {\n            // TODO: Replace the vanilla JS Date object with an indepentent day-of-week check.\n            var weekdayProvided = defaultLocaleWeekdaysShort.indexOf(weekdayStr),\n                weekdayActual = new Date(parsedInput[0], parsedInput[1], parsedInput[2]).getDay();\n            if (weekdayProvided !== weekdayActual) {\n                getParsingFlags(config).weekdayMismatch = true;\n                config._isValid = false;\n                return false;\n            }\n        }\n        return true;\n    }\n\n    var obsOffsets = {\n        UT: 0,\n        GMT: 0,\n        EDT: -4 * 60,\n        EST: -5 * 60,\n        CDT: -5 * 60,\n        CST: -6 * 60,\n        MDT: -6 * 60,\n        MST: -7 * 60,\n        PDT: -7 * 60,\n        PST: -8 * 60\n    };\n\n    function calculateOffset(obsOffset, militaryOffset, numOffset) {\n        if (obsOffset) {\n            return obsOffsets[obsOffset];\n        } else if (militaryOffset) {\n            // the only allowed military tz is Z\n            return 0;\n        } else {\n            var hm = parseInt(numOffset, 10);\n            var m = hm % 100, h = (hm - m) / 100;\n            return h * 60 + m;\n        }\n    }\n\n    // date and time from ref 2822 format\n    function configFromRFC2822(config) {\n        var match = rfc2822.exec(preprocessRFC2822(config._i));\n        if (match) {\n            var parsedArray = extractFromRFC2822Strings(match[4], match[3], match[2], match[5], match[6], match[7]);\n            if (!checkWeekday(match[1], parsedArray, config)) {\n                return;\n            }\n\n            config._a = parsedArray;\n            config._tzm = calculateOffset(match[8], match[9], match[10]);\n\n            config._d = createUTCDate.apply(null, config._a);\n            config._d.setUTCMinutes(config._d.getUTCMinutes() - config._tzm);\n\n            getParsingFlags(config).rfc2822 = true;\n        } else {\n            config._isValid = false;\n        }\n    }\n\n    // date from iso format or fallback\n    function configFromString(config) {\n        var matched = aspNetJsonRegex.exec(config._i);\n\n        if (matched !== null) {\n            config._d = new Date(+matched[1]);\n            return;\n        }\n\n        configFromISO(config);\n        if (config._isValid === false) {\n            delete config._isValid;\n        } else {\n            return;\n        }\n\n        configFromRFC2822(config);\n        if (config._isValid === false) {\n            delete config._isValid;\n        } else {\n            return;\n        }\n\n        // Final attempt, use Input Fallback\n        hooks.createFromInputFallback(config);\n    }\n\n    hooks.createFromInputFallback = deprecate(\n        'value provided is not in a recognized RFC2822 or ISO format. moment construction falls back to js Date(), ' +\n        'which is not reliable across all browsers and versions. Non RFC2822/ISO date formats are ' +\n        'discouraged and will be removed in an upcoming major release. Please refer to ' +\n        'http://momentjs.com/guides/#/warnings/js-date/ for more info.',\n        function (config) {\n            config._d = new Date(config._i + (config._useUTC ? ' UTC' : ''));\n        }\n    );\n\n    // constant that refers to the ISO standard\n    hooks.ISO_8601 = function () {};\n\n    // constant that refers to the RFC 2822 form\n    hooks.RFC_2822 = function () {};\n\n    // date from string and format string\n    function configFromStringAndFormat(config) {\n        // TODO: Move this to another part of the creation flow to prevent circular deps\n        if (config._f === hooks.ISO_8601) {\n            configFromISO(config);\n            return;\n        }\n        if (config._f === hooks.RFC_2822) {\n            configFromRFC2822(config);\n            return;\n        }\n        config._a = [];\n        getParsingFlags(config).empty = true;\n\n        // This array is used to make a Date, either with `new Date` or `Date.UTC`\n        var string = '' + config._i,\n            i, parsedInput, tokens, token, skipped,\n            stringLength = string.length,\n            totalParsedInputLength = 0;\n\n        tokens = expandFormat(config._f, config._locale).match(formattingTokens) || [];\n\n        for (i = 0; i < tokens.length; i++) {\n            token = tokens[i];\n            parsedInput = (string.match(getParseRegexForToken(token, config)) || [])[0];\n            // console.log('token', token, 'parsedInput', parsedInput,\n            //         'regex', getParseRegexForToken(token, config));\n            if (parsedInput) {\n                skipped = string.substr(0, string.indexOf(parsedInput));\n                if (skipped.length > 0) {\n                    getParsingFlags(config).unusedInput.push(skipped);\n                }\n                string = string.slice(string.indexOf(parsedInput) + parsedInput.length);\n                totalParsedInputLength += parsedInput.length;\n            }\n            // don't parse if it's not a known token\n            if (formatTokenFunctions[token]) {\n                if (parsedInput) {\n                    getParsingFlags(config).empty = false;\n                }\n                else {\n                    getParsingFlags(config).unusedTokens.push(token);\n                }\n                addTimeToArrayFromToken(token, parsedInput, config);\n            }\n            else if (config._strict && !parsedInput) {\n                getParsingFlags(config).unusedTokens.push(token);\n            }\n        }\n\n        // add remaining unparsed input length to the string\n        getParsingFlags(config).charsLeftOver = stringLength - totalParsedInputLength;\n        if (string.length > 0) {\n            getParsingFlags(config).unusedInput.push(string);\n        }\n\n        // clear _12h flag if hour is <= 12\n        if (config._a[HOUR] <= 12 &&\n            getParsingFlags(config).bigHour === true &&\n            config._a[HOUR] > 0) {\n            getParsingFlags(config).bigHour = undefined;\n        }\n\n        getParsingFlags(config).parsedDateParts = config._a.slice(0);\n        getParsingFlags(config).meridiem = config._meridiem;\n        // handle meridiem\n        config._a[HOUR] = meridiemFixWrap(config._locale, config._a[HOUR], config._meridiem);\n\n        configFromArray(config);\n        checkOverflow(config);\n    }\n\n\n    function meridiemFixWrap (locale, hour, meridiem) {\n        var isPm;\n\n        if (meridiem == null) {\n            // nothing to do\n            return hour;\n        }\n        if (locale.meridiemHour != null) {\n            return locale.meridiemHour(hour, meridiem);\n        } else if (locale.isPM != null) {\n            // Fallback\n            isPm = locale.isPM(meridiem);\n            if (isPm && hour < 12) {\n                hour += 12;\n            }\n            if (!isPm && hour === 12) {\n                hour = 0;\n            }\n            return hour;\n        } else {\n            // this is not supposed to happen\n            return hour;\n        }\n    }\n\n    // date from string and array of format strings\n    function configFromStringAndArray(config) {\n        var tempConfig,\n            bestMoment,\n\n            scoreToBeat,\n            i,\n            currentScore;\n\n        if (config._f.length === 0) {\n            getParsingFlags(config).invalidFormat = true;\n            config._d = new Date(NaN);\n            return;\n        }\n\n        for (i = 0; i < config._f.length; i++) {\n            currentScore = 0;\n            tempConfig = copyConfig({}, config);\n            if (config._useUTC != null) {\n                tempConfig._useUTC = config._useUTC;\n            }\n            tempConfig._f = config._f[i];\n            configFromStringAndFormat(tempConfig);\n\n            if (!isValid(tempConfig)) {\n                continue;\n            }\n\n            // if there is any input that was not parsed add a penalty for that format\n            currentScore += getParsingFlags(tempConfig).charsLeftOver;\n\n            //or tokens\n            currentScore += getParsingFlags(tempConfig).unusedTokens.length * 10;\n\n            getParsingFlags(tempConfig).score = currentScore;\n\n            if (scoreToBeat == null || currentScore < scoreToBeat) {\n                scoreToBeat = currentScore;\n                bestMoment = tempConfig;\n            }\n        }\n\n        extend(config, bestMoment || tempConfig);\n    }\n\n    function configFromObject(config) {\n        if (config._d) {\n            return;\n        }\n\n        var i = normalizeObjectUnits(config._i);\n        config._a = map([i.year, i.month, i.day || i.date, i.hour, i.minute, i.second, i.millisecond], function (obj) {\n            return obj && parseInt(obj, 10);\n        });\n\n        configFromArray(config);\n    }\n\n    function createFromConfig (config) {\n        var res = new Moment(checkOverflow(prepareConfig(config)));\n        if (res._nextDay) {\n            // Adding is smart enough around DST\n            res.add(1, 'd');\n            res._nextDay = undefined;\n        }\n\n        return res;\n    }\n\n    function prepareConfig (config) {\n        var input = config._i,\n            format = config._f;\n\n        config._locale = config._locale || getLocale(config._l);\n\n        if (input === null || (format === undefined && input === '')) {\n            return createInvalid({nullInput: true});\n        }\n\n        if (typeof input === 'string') {\n            config._i = input = config._locale.preparse(input);\n        }\n\n        if (isMoment(input)) {\n            return new Moment(checkOverflow(input));\n        } else if (isDate(input)) {\n            config._d = input;\n        } else if (isArray(format)) {\n            configFromStringAndArray(config);\n        } else if (format) {\n            configFromStringAndFormat(config);\n        }  else {\n            configFromInput(config);\n        }\n\n        if (!isValid(config)) {\n            config._d = null;\n        }\n\n        return config;\n    }\n\n    function configFromInput(config) {\n        var input = config._i;\n        if (isUndefined(input)) {\n            config._d = new Date(hooks.now());\n        } else if (isDate(input)) {\n            config._d = new Date(input.valueOf());\n        } else if (typeof input === 'string') {\n            configFromString(config);\n        } else if (isArray(input)) {\n            config._a = map(input.slice(0), function (obj) {\n                return parseInt(obj, 10);\n            });\n            configFromArray(config);\n        } else if (isObject(input)) {\n            configFromObject(config);\n        } else if (isNumber(input)) {\n            // from milliseconds\n            config._d = new Date(input);\n        } else {\n            hooks.createFromInputFallback(config);\n        }\n    }\n\n    function createLocalOrUTC (input, format, locale, strict, isUTC) {\n        var c = {};\n\n        if (locale === true || locale === false) {\n            strict = locale;\n            locale = undefined;\n        }\n\n        if ((isObject(input) && isObjectEmpty(input)) ||\n                (isArray(input) && input.length === 0)) {\n            input = undefined;\n        }\n        // object construction must be done this way.\n        // https://github.com/moment/moment/issues/1423\n        c._isAMomentObject = true;\n        c._useUTC = c._isUTC = isUTC;\n        c._l = locale;\n        c._i = input;\n        c._f = format;\n        c._strict = strict;\n\n        return createFromConfig(c);\n    }\n\n    function createLocal (input, format, locale, strict) {\n        return createLocalOrUTC(input, format, locale, strict, false);\n    }\n\n    var prototypeMin = deprecate(\n        'moment().min is deprecated, use moment.max instead. http://momentjs.com/guides/#/warnings/min-max/',\n        function () {\n            var other = createLocal.apply(null, arguments);\n            if (this.isValid() && other.isValid()) {\n                return other < this ? this : other;\n            } else {\n                return createInvalid();\n            }\n        }\n    );\n\n    var prototypeMax = deprecate(\n        'moment().max is deprecated, use moment.min instead. http://momentjs.com/guides/#/warnings/min-max/',\n        function () {\n            var other = createLocal.apply(null, arguments);\n            if (this.isValid() && other.isValid()) {\n                return other > this ? this : other;\n            } else {\n                return createInvalid();\n            }\n        }\n    );\n\n    // Pick a moment m from moments so that m[fn](other) is true for all\n    // other. This relies on the function fn to be transitive.\n    //\n    // moments should either be an array of moment objects or an array, whose\n    // first element is an array of moment objects.\n    function pickBy(fn, moments) {\n        var res, i;\n        if (moments.length === 1 && isArray(moments[0])) {\n            moments = moments[0];\n        }\n        if (!moments.length) {\n            return createLocal();\n        }\n        res = moments[0];\n        for (i = 1; i < moments.length; ++i) {\n            if (!moments[i].isValid() || moments[i][fn](res)) {\n                res = moments[i];\n            }\n        }\n        return res;\n    }\n\n    // TODO: Use [].sort instead?\n    function min () {\n        var args = [].slice.call(arguments, 0);\n\n        return pickBy('isBefore', args);\n    }\n\n    function max () {\n        var args = [].slice.call(arguments, 0);\n\n        return pickBy('isAfter', args);\n    }\n\n    var now = function () {\n        return Date.now ? Date.now() : +(new Date());\n    };\n\n    var ordering = ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second', 'millisecond'];\n\n    function isDurationValid(m) {\n        for (var key in m) {\n            if (!(indexOf.call(ordering, key) !== -1 && (m[key] == null || !isNaN(m[key])))) {\n                return false;\n            }\n        }\n\n        var unitHasDecimal = false;\n        for (var i = 0; i < ordering.length; ++i) {\n            if (m[ordering[i]]) {\n                if (unitHasDecimal) {\n                    return false; // only allow non-integers for smallest unit\n                }\n                if (parseFloat(m[ordering[i]]) !== toInt(m[ordering[i]])) {\n                    unitHasDecimal = true;\n                }\n            }\n        }\n\n        return true;\n    }\n\n    function isValid$1() {\n        return this._isValid;\n    }\n\n    function createInvalid$1() {\n        return createDuration(NaN);\n    }\n\n    function Duration (duration) {\n        var normalizedInput = normalizeObjectUnits(duration),\n            years = normalizedInput.year || 0,\n            quarters = normalizedInput.quarter || 0,\n            months = normalizedInput.month || 0,\n            weeks = normalizedInput.week || normalizedInput.isoWeek || 0,\n            days = normalizedInput.day || 0,\n            hours = normalizedInput.hour || 0,\n            minutes = normalizedInput.minute || 0,\n            seconds = normalizedInput.second || 0,\n            milliseconds = normalizedInput.millisecond || 0;\n\n        this._isValid = isDurationValid(normalizedInput);\n\n        // representation for dateAddRemove\n        this._milliseconds = +milliseconds +\n            seconds * 1e3 + // 1000\n            minutes * 6e4 + // 1000 * 60\n            hours * 1000 * 60 * 60; //using 1000 * 60 * 60 instead of 36e5 to avoid floating point rounding errors https://github.com/moment/moment/issues/2978\n        // Because of dateAddRemove treats 24 hours as different from a\n        // day when working around DST, we need to store them separately\n        this._days = +days +\n            weeks * 7;\n        // It is impossible to translate months into days without knowing\n        // which months you are are talking about, so we have to store\n        // it separately.\n        this._months = +months +\n            quarters * 3 +\n            years * 12;\n\n        this._data = {};\n\n        this._locale = getLocale();\n\n        this._bubble();\n    }\n\n    function isDuration (obj) {\n        return obj instanceof Duration;\n    }\n\n    function absRound (number) {\n        if (number < 0) {\n            return Math.round(-1 * number) * -1;\n        } else {\n            return Math.round(number);\n        }\n    }\n\n    // FORMATTING\n\n    function offset (token, separator) {\n        addFormatToken(token, 0, 0, function () {\n            var offset = this.utcOffset();\n            var sign = '+';\n            if (offset < 0) {\n                offset = -offset;\n                sign = '-';\n            }\n            return sign + zeroFill(~~(offset / 60), 2) + separator + zeroFill(~~(offset) % 60, 2);\n        });\n    }\n\n    offset('Z', ':');\n    offset('ZZ', '');\n\n    // PARSING\n\n    addRegexToken('Z',  matchShortOffset);\n    addRegexToken('ZZ', matchShortOffset);\n    addParseToken(['Z', 'ZZ'], function (input, array, config) {\n        config._useUTC = true;\n        config._tzm = offsetFromString(matchShortOffset, input);\n    });\n\n    // HELPERS\n\n    // timezone chunker\n    // '+10:00' > ['10',  '00']\n    // '-1530'  > ['-15', '30']\n    var chunkOffset = /([\\+\\-]|\\d\\d)/gi;\n\n    function offsetFromString(matcher, string) {\n        var matches = (string || '').match(matcher);\n\n        if (matches === null) {\n            return null;\n        }\n\n        var chunk   = matches[matches.length - 1] || [];\n        var parts   = (chunk + '').match(chunkOffset) || ['-', 0, 0];\n        var minutes = +(parts[1] * 60) + toInt(parts[2]);\n\n        return minutes === 0 ?\n          0 :\n          parts[0] === '+' ? minutes : -minutes;\n    }\n\n    // Return a moment from input, that is local/utc/zone equivalent to model.\n    function cloneWithOffset(input, model) {\n        var res, diff;\n        if (model._isUTC) {\n            res = model.clone();\n            diff = (isMoment(input) || isDate(input) ? input.valueOf() : createLocal(input).valueOf()) - res.valueOf();\n            // Use low-level api, because this fn is low-level api.\n            res._d.setTime(res._d.valueOf() + diff);\n            hooks.updateOffset(res, false);\n            return res;\n        } else {\n            return createLocal(input).local();\n        }\n    }\n\n    function getDateOffset (m) {\n        // On Firefox.24 Date#getTimezoneOffset returns a floating point.\n        // https://github.com/moment/moment/pull/1871\n        return -Math.round(m._d.getTimezoneOffset() / 15) * 15;\n    }\n\n    // HOOKS\n\n    // This function will be called whenever a moment is mutated.\n    // It is intended to keep the offset in sync with the timezone.\n    hooks.updateOffset = function () {};\n\n    // MOMENTS\n\n    // keepLocalTime = true means only change the timezone, without\n    // affecting the local hour. So 5:31:26 +0300 --[utcOffset(2, true)]-->\n    // 5:31:26 +0200 It is possible that 5:31:26 doesn't exist with offset\n    // +0200, so we adjust the time as needed, to be valid.\n    //\n    // Keeping the time actually adds/subtracts (one hour)\n    // from the actual represented time. That is why we call updateOffset\n    // a second time. In case it wants us to change the offset again\n    // _changeInProgress == true case, then we have to adjust, because\n    // there is no such time in the given timezone.\n    function getSetOffset (input, keepLocalTime, keepMinutes) {\n        var offset = this._offset || 0,\n            localAdjust;\n        if (!this.isValid()) {\n            return input != null ? this : NaN;\n        }\n        if (input != null) {\n            if (typeof input === 'string') {\n                input = offsetFromString(matchShortOffset, input);\n                if (input === null) {\n                    return this;\n                }\n            } else if (Math.abs(input) < 16 && !keepMinutes) {\n                input = input * 60;\n            }\n            if (!this._isUTC && keepLocalTime) {\n                localAdjust = getDateOffset(this);\n            }\n            this._offset = input;\n            this._isUTC = true;\n            if (localAdjust != null) {\n                this.add(localAdjust, 'm');\n            }\n            if (offset !== input) {\n                if (!keepLocalTime || this._changeInProgress) {\n                    addSubtract(this, createDuration(input - offset, 'm'), 1, false);\n                } else if (!this._changeInProgress) {\n                    this._changeInProgress = true;\n                    hooks.updateOffset(this, true);\n                    this._changeInProgress = null;\n                }\n            }\n            return this;\n        } else {\n            return this._isUTC ? offset : getDateOffset(this);\n        }\n    }\n\n    function getSetZone (input, keepLocalTime) {\n        if (input != null) {\n            if (typeof input !== 'string') {\n                input = -input;\n            }\n\n            this.utcOffset(input, keepLocalTime);\n\n            return this;\n        } else {\n            return -this.utcOffset();\n        }\n    }\n\n    function setOffsetToUTC (keepLocalTime) {\n        return this.utcOffset(0, keepLocalTime);\n    }\n\n    function setOffsetToLocal (keepLocalTime) {\n        if (this._isUTC) {\n            this.utcOffset(0, keepLocalTime);\n            this._isUTC = false;\n\n            if (keepLocalTime) {\n                this.subtract(getDateOffset(this), 'm');\n            }\n        }\n        return this;\n    }\n\n    function setOffsetToParsedOffset () {\n        if (this._tzm != null) {\n            this.utcOffset(this._tzm, false, true);\n        } else if (typeof this._i === 'string') {\n            var tZone = offsetFromString(matchOffset, this._i);\n            if (tZone != null) {\n                this.utcOffset(tZone);\n            }\n            else {\n                this.utcOffset(0, true);\n            }\n        }\n        return this;\n    }\n\n    function hasAlignedHourOffset (input) {\n        if (!this.isValid()) {\n            return false;\n        }\n        input = input ? createLocal(input).utcOffset() : 0;\n\n        return (this.utcOffset() - input) % 60 === 0;\n    }\n\n    function isDaylightSavingTime () {\n        return (\n            this.utcOffset() > this.clone().month(0).utcOffset() ||\n            this.utcOffset() > this.clone().month(5).utcOffset()\n        );\n    }\n\n    function isDaylightSavingTimeShifted () {\n        if (!isUndefined(this._isDSTShifted)) {\n            return this._isDSTShifted;\n        }\n\n        var c = {};\n\n        copyConfig(c, this);\n        c = prepareConfig(c);\n\n        if (c._a) {\n            var other = c._isUTC ? createUTC(c._a) : createLocal(c._a);\n            this._isDSTShifted = this.isValid() &&\n                compareArrays(c._a, other.toArray()) > 0;\n        } else {\n            this._isDSTShifted = false;\n        }\n\n        return this._isDSTShifted;\n    }\n\n    function isLocal () {\n        return this.isValid() ? !this._isUTC : false;\n    }\n\n    function isUtcOffset () {\n        return this.isValid() ? this._isUTC : false;\n    }\n\n    function isUtc () {\n        return this.isValid() ? this._isUTC && this._offset === 0 : false;\n    }\n\n    // ASP.NET json date format regex\n    var aspNetRegex = /^(\\-|\\+)?(?:(\\d*)[. ])?(\\d+)\\:(\\d+)(?:\\:(\\d+)(\\.\\d*)?)?$/;\n\n    // from http://docs.closure-library.googlecode.com/git/closure_goog_date_date.js.source.html\n    // somewhat more in line with 4.4.3.2 2004 spec, but allows decimal anywhere\n    // and further modified to allow for strings containing both week and day\n    var isoRegex = /^(-|\\+)?P(?:([-+]?[0-9,.]*)Y)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)W)?(?:([-+]?[0-9,.]*)D)?(?:T(?:([-+]?[0-9,.]*)H)?(?:([-+]?[0-9,.]*)M)?(?:([-+]?[0-9,.]*)S)?)?$/;\n\n    function createDuration (input, key) {\n        var duration = input,\n            // matching against regexp is expensive, do it on demand\n            match = null,\n            sign,\n            ret,\n            diffRes;\n\n        if (isDuration(input)) {\n            duration = {\n                ms : input._milliseconds,\n                d  : input._days,\n                M  : input._months\n            };\n        } else if (isNumber(input)) {\n            duration = {};\n            if (key) {\n                duration[key] = input;\n            } else {\n                duration.milliseconds = input;\n            }\n        } else if (!!(match = aspNetRegex.exec(input))) {\n            sign = (match[1] === '-') ? -1 : 1;\n            duration = {\n                y  : 0,\n                d  : toInt(match[DATE])                         * sign,\n                h  : toInt(match[HOUR])                         * sign,\n                m  : toInt(match[MINUTE])                       * sign,\n                s  : toInt(match[SECOND])                       * sign,\n                ms : toInt(absRound(match[MILLISECOND] * 1000)) * sign // the millisecond decimal point is included in the match\n            };\n        } else if (!!(match = isoRegex.exec(input))) {\n            sign = (match[1] === '-') ? -1 : 1;\n            duration = {\n                y : parseIso(match[2], sign),\n                M : parseIso(match[3], sign),\n                w : parseIso(match[4], sign),\n                d : parseIso(match[5], sign),\n                h : parseIso(match[6], sign),\n                m : parseIso(match[7], sign),\n                s : parseIso(match[8], sign)\n            };\n        } else if (duration == null) {// checks for null or undefined\n            duration = {};\n        } else if (typeof duration === 'object' && ('from' in duration || 'to' in duration)) {\n            diffRes = momentsDifference(createLocal(duration.from), createLocal(duration.to));\n\n            duration = {};\n            duration.ms = diffRes.milliseconds;\n            duration.M = diffRes.months;\n        }\n\n        ret = new Duration(duration);\n\n        if (isDuration(input) && hasOwnProp(input, '_locale')) {\n            ret._locale = input._locale;\n        }\n\n        return ret;\n    }\n\n    createDuration.fn = Duration.prototype;\n    createDuration.invalid = createInvalid$1;\n\n    function parseIso (inp, sign) {\n        // We'd normally use ~~inp for this, but unfortunately it also\n        // converts floats to ints.\n        // inp may be undefined, so careful calling replace on it.\n        var res = inp && parseFloat(inp.replace(',', '.'));\n        // apply sign while we're at it\n        return (isNaN(res) ? 0 : res) * sign;\n    }\n\n    function positiveMomentsDifference(base, other) {\n        var res = {};\n\n        res.months = other.month() - base.month() +\n            (other.year() - base.year()) * 12;\n        if (base.clone().add(res.months, 'M').isAfter(other)) {\n            --res.months;\n        }\n\n        res.milliseconds = +other - +(base.clone().add(res.months, 'M'));\n\n        return res;\n    }\n\n    function momentsDifference(base, other) {\n        var res;\n        if (!(base.isValid() && other.isValid())) {\n            return {milliseconds: 0, months: 0};\n        }\n\n        other = cloneWithOffset(other, base);\n        if (base.isBefore(other)) {\n            res = positiveMomentsDifference(base, other);\n        } else {\n            res = positiveMomentsDifference(other, base);\n            res.milliseconds = -res.milliseconds;\n            res.months = -res.months;\n        }\n\n        return res;\n    }\n\n    // TODO: remove 'name' arg after deprecation is removed\n    function createAdder(direction, name) {\n        return function (val, period) {\n            var dur, tmp;\n            //invert the arguments, but complain about it\n            if (period !== null && !isNaN(+period)) {\n                deprecateSimple(name, 'moment().' + name  + '(period, number) is deprecated. Please use moment().' + name + '(number, period). ' +\n                'See http://momentjs.com/guides/#/warnings/add-inverted-param/ for more info.');\n                tmp = val; val = period; period = tmp;\n            }\n\n            val = typeof val === 'string' ? +val : val;\n            dur = createDuration(val, period);\n            addSubtract(this, dur, direction);\n            return this;\n        };\n    }\n\n    function addSubtract (mom, duration, isAdding, updateOffset) {\n        var milliseconds = duration._milliseconds,\n            days = absRound(duration._days),\n            months = absRound(duration._months);\n\n        if (!mom.isValid()) {\n            // No op\n            return;\n        }\n\n        updateOffset = updateOffset == null ? true : updateOffset;\n\n        if (months) {\n            setMonth(mom, get(mom, 'Month') + months * isAdding);\n        }\n        if (days) {\n            set$1(mom, 'Date', get(mom, 'Date') + days * isAdding);\n        }\n        if (milliseconds) {\n            mom._d.setTime(mom._d.valueOf() + milliseconds * isAdding);\n        }\n        if (updateOffset) {\n            hooks.updateOffset(mom, days || months);\n        }\n    }\n\n    var add      = createAdder(1, 'add');\n    var subtract = createAdder(-1, 'subtract');\n\n    function getCalendarFormat(myMoment, now) {\n        var diff = myMoment.diff(now, 'days', true);\n        return diff < -6 ? 'sameElse' :\n                diff < -1 ? 'lastWeek' :\n                diff < 0 ? 'lastDay' :\n                diff < 1 ? 'sameDay' :\n                diff < 2 ? 'nextDay' :\n                diff < 7 ? 'nextWeek' : 'sameElse';\n    }\n\n    function calendar$1 (time, formats) {\n        // We want to compare the start of today, vs this.\n        // Getting start-of-today depends on whether we're local/utc/offset or not.\n        var now = time || createLocal(),\n            sod = cloneWithOffset(now, this).startOf('day'),\n            format = hooks.calendarFormat(this, sod) || 'sameElse';\n\n        var output = formats && (isFunction(formats[format]) ? formats[format].call(this, now) : formats[format]);\n\n        return this.format(output || this.localeData().calendar(format, this, createLocal(now)));\n    }\n\n    function clone () {\n        return new Moment(this);\n    }\n\n    function isAfter (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input);\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() > localInput.valueOf();\n        } else {\n            return localInput.valueOf() < this.clone().startOf(units).valueOf();\n        }\n    }\n\n    function isBefore (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input);\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() < localInput.valueOf();\n        } else {\n            return this.clone().endOf(units).valueOf() < localInput.valueOf();\n        }\n    }\n\n    function isBetween (from, to, units, inclusivity) {\n        var localFrom = isMoment(from) ? from : createLocal(from),\n            localTo = isMoment(to) ? to : createLocal(to);\n        if (!(this.isValid() && localFrom.isValid() && localTo.isValid())) {\n            return false;\n        }\n        inclusivity = inclusivity || '()';\n        return (inclusivity[0] === '(' ? this.isAfter(localFrom, units) : !this.isBefore(localFrom, units)) &&\n            (inclusivity[1] === ')' ? this.isBefore(localTo, units) : !this.isAfter(localTo, units));\n    }\n\n    function isSame (input, units) {\n        var localInput = isMoment(input) ? input : createLocal(input),\n            inputMs;\n        if (!(this.isValid() && localInput.isValid())) {\n            return false;\n        }\n        units = normalizeUnits(units) || 'millisecond';\n        if (units === 'millisecond') {\n            return this.valueOf() === localInput.valueOf();\n        } else {\n            inputMs = localInput.valueOf();\n            return this.clone().startOf(units).valueOf() <= inputMs && inputMs <= this.clone().endOf(units).valueOf();\n        }\n    }\n\n    function isSameOrAfter (input, units) {\n        return this.isSame(input, units) || this.isAfter(input, units);\n    }\n\n    function isSameOrBefore (input, units) {\n        return this.isSame(input, units) || this.isBefore(input, units);\n    }\n\n    function diff (input, units, asFloat) {\n        var that,\n            zoneDelta,\n            output;\n\n        if (!this.isValid()) {\n            return NaN;\n        }\n\n        that = cloneWithOffset(input, this);\n\n        if (!that.isValid()) {\n            return NaN;\n        }\n\n        zoneDelta = (that.utcOffset() - this.utcOffset()) * 6e4;\n\n        units = normalizeUnits(units);\n\n        switch (units) {\n            case 'year': output = monthDiff(this, that) / 12; break;\n            case 'month': output = monthDiff(this, that); break;\n            case 'quarter': output = monthDiff(this, that) / 3; break;\n            case 'second': output = (this - that) / 1e3; break; // 1000\n            case 'minute': output = (this - that) / 6e4; break; // 1000 * 60\n            case 'hour': output = (this - that) / 36e5; break; // 1000 * 60 * 60\n            case 'day': output = (this - that - zoneDelta) / 864e5; break; // 1000 * 60 * 60 * 24, negate dst\n            case 'week': output = (this - that - zoneDelta) / 6048e5; break; // 1000 * 60 * 60 * 24 * 7, negate dst\n            default: output = this - that;\n        }\n\n        return asFloat ? output : absFloor(output);\n    }\n\n    function monthDiff (a, b) {\n        // difference in months\n        var wholeMonthDiff = ((b.year() - a.year()) * 12) + (b.month() - a.month()),\n            // b is in (anchor - 1 month, anchor + 1 month)\n            anchor = a.clone().add(wholeMonthDiff, 'months'),\n            anchor2, adjust;\n\n        if (b - anchor < 0) {\n            anchor2 = a.clone().add(wholeMonthDiff - 1, 'months');\n            // linear across the month\n            adjust = (b - anchor) / (anchor - anchor2);\n        } else {\n            anchor2 = a.clone().add(wholeMonthDiff + 1, 'months');\n            // linear across the month\n            adjust = (b - anchor) / (anchor2 - anchor);\n        }\n\n        //check for negative zero, return zero if negative zero\n        return -(wholeMonthDiff + adjust) || 0;\n    }\n\n    hooks.defaultFormat = 'YYYY-MM-DDTHH:mm:ssZ';\n    hooks.defaultFormatUtc = 'YYYY-MM-DDTHH:mm:ss[Z]';\n\n    function toString () {\n        return this.clone().locale('en').format('ddd MMM DD YYYY HH:mm:ss [GMT]ZZ');\n    }\n\n    function toISOString(keepOffset) {\n        if (!this.isValid()) {\n            return null;\n        }\n        var utc = keepOffset !== true;\n        var m = utc ? this.clone().utc() : this;\n        if (m.year() < 0 || m.year() > 9999) {\n            return formatMoment(m, utc ? 'YYYYYY-MM-DD[T]HH:mm:ss.SSS[Z]' : 'YYYYYY-MM-DD[T]HH:mm:ss.SSSZ');\n        }\n        if (isFunction(Date.prototype.toISOString)) {\n            // native implementation is ~50x faster, use it when we can\n            if (utc) {\n                return this.toDate().toISOString();\n            } else {\n                return new Date(this.valueOf() + this.utcOffset() * 60 * 1000).toISOString().replace('Z', formatMoment(m, 'Z'));\n            }\n        }\n        return formatMoment(m, utc ? 'YYYY-MM-DD[T]HH:mm:ss.SSS[Z]' : 'YYYY-MM-DD[T]HH:mm:ss.SSSZ');\n    }\n\n    /**\n     * Return a human readable representation of a moment that can\n     * also be evaluated to get a new moment which is the same\n     *\n     * @link https://nodejs.org/dist/latest/docs/api/util.html#util_custom_inspect_function_on_objects\n     */\n    function inspect () {\n        if (!this.isValid()) {\n            return 'moment.invalid(/* ' + this._i + ' */)';\n        }\n        var func = 'moment';\n        var zone = '';\n        if (!this.isLocal()) {\n            func = this.utcOffset() === 0 ? 'moment.utc' : 'moment.parseZone';\n            zone = 'Z';\n        }\n        var prefix = '[' + func + '(\"]';\n        var year = (0 <= this.year() && this.year() <= 9999) ? 'YYYY' : 'YYYYYY';\n        var datetime = '-MM-DD[T]HH:mm:ss.SSS';\n        var suffix = zone + '[\")]';\n\n        return this.format(prefix + year + datetime + suffix);\n    }\n\n    function format (inputString) {\n        if (!inputString) {\n            inputString = this.isUtc() ? hooks.defaultFormatUtc : hooks.defaultFormat;\n        }\n        var output = formatMoment(this, inputString);\n        return this.localeData().postformat(output);\n    }\n\n    function from (time, withoutSuffix) {\n        if (this.isValid() &&\n                ((isMoment(time) && time.isValid()) ||\n                 createLocal(time).isValid())) {\n            return createDuration({to: this, from: time}).locale(this.locale()).humanize(!withoutSuffix);\n        } else {\n            return this.localeData().invalidDate();\n        }\n    }\n\n    function fromNow (withoutSuffix) {\n        return this.from(createLocal(), withoutSuffix);\n    }\n\n    function to (time, withoutSuffix) {\n        if (this.isValid() &&\n                ((isMoment(time) && time.isValid()) ||\n                 createLocal(time).isValid())) {\n            return createDuration({from: this, to: time}).locale(this.locale()).humanize(!withoutSuffix);\n        } else {\n            return this.localeData().invalidDate();\n        }\n    }\n\n    function toNow (withoutSuffix) {\n        return this.to(createLocal(), withoutSuffix);\n    }\n\n    // If passed a locale key, it will set the locale for this\n    // instance.  Otherwise, it will return the locale configuration\n    // variables for this instance.\n    function locale (key) {\n        var newLocaleData;\n\n        if (key === undefined) {\n            return this._locale._abbr;\n        } else {\n            newLocaleData = getLocale(key);\n            if (newLocaleData != null) {\n                this._locale = newLocaleData;\n            }\n            return this;\n        }\n    }\n\n    var lang = deprecate(\n        'moment().lang() is deprecated. Instead, use moment().localeData() to get the language configuration. Use moment().locale() to change languages.',\n        function (key) {\n            if (key === undefined) {\n                return this.localeData();\n            } else {\n                return this.locale(key);\n            }\n        }\n    );\n\n    function localeData () {\n        return this._locale;\n    }\n\n    var MS_PER_SECOND = 1000;\n    var MS_PER_MINUTE = 60 * MS_PER_SECOND;\n    var MS_PER_HOUR = 60 * MS_PER_MINUTE;\n    var MS_PER_400_YEARS = (365 * 400 + 97) * 24 * MS_PER_HOUR;\n\n    // actual modulo - handles negative numbers (for dates before 1970):\n    function mod$1(dividend, divisor) {\n        return (dividend % divisor + divisor) % divisor;\n    }\n\n    function localStartOfDate(y, m, d) {\n        // the date constructor remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            return new Date(y + 400, m, d) - MS_PER_400_YEARS;\n        } else {\n            return new Date(y, m, d).valueOf();\n        }\n    }\n\n    function utcStartOfDate(y, m, d) {\n        // Date.UTC remaps years 0-99 to 1900-1999\n        if (y < 100 && y >= 0) {\n            // preserve leap years using a full 400 year cycle, then reset\n            return Date.UTC(y + 400, m, d) - MS_PER_400_YEARS;\n        } else {\n            return Date.UTC(y, m, d);\n        }\n    }\n\n    function startOf (units) {\n        var time;\n        units = normalizeUnits(units);\n        if (units === undefined || units === 'millisecond' || !this.isValid()) {\n            return this;\n        }\n\n        var startOfDate = this._isUTC ? utcStartOfDate : localStartOfDate;\n\n        switch (units) {\n            case 'year':\n                time = startOfDate(this.year(), 0, 1);\n                break;\n            case 'quarter':\n                time = startOfDate(this.year(), this.month() - this.month() % 3, 1);\n                break;\n            case 'month':\n                time = startOfDate(this.year(), this.month(), 1);\n                break;\n            case 'week':\n                time = startOfDate(this.year(), this.month(), this.date() - this.weekday());\n                break;\n            case 'isoWeek':\n                time = startOfDate(this.year(), this.month(), this.date() - (this.isoWeekday() - 1));\n                break;\n            case 'day':\n            case 'date':\n                time = startOfDate(this.year(), this.month(), this.date());\n                break;\n            case 'hour':\n                time = this._d.valueOf();\n                time -= mod$1(time + (this._isUTC ? 0 : this.utcOffset() * MS_PER_MINUTE), MS_PER_HOUR);\n                break;\n            case 'minute':\n                time = this._d.valueOf();\n                time -= mod$1(time, MS_PER_MINUTE);\n                break;\n            case 'second':\n                time = this._d.valueOf();\n                time -= mod$1(time, MS_PER_SECOND);\n                break;\n        }\n\n        this._d.setTime(time);\n        hooks.updateOffset(this, true);\n        return this;\n    }\n\n    function endOf (units) {\n        var time;\n        units = normalizeUnits(units);\n        if (units === undefined || units === 'millisecond' || !this.isValid()) {\n            return this;\n        }\n\n        var startOfDate = this._isUTC ? utcStartOfDate : localStartOfDate;\n\n        switch (units) {\n            case 'year':\n                time = startOfDate(this.year() + 1, 0, 1) - 1;\n                break;\n            case 'quarter':\n                time = startOfDate(this.year(), this.month() - this.month() % 3 + 3, 1) - 1;\n                break;\n            case 'month':\n                time = startOfDate(this.year(), this.month() + 1, 1) - 1;\n                break;\n            case 'week':\n                time = startOfDate(this.year(), this.month(), this.date() - this.weekday() + 7) - 1;\n                break;\n            case 'isoWeek':\n                time = startOfDate(this.year(), this.month(), this.date() - (this.isoWeekday() - 1) + 7) - 1;\n                break;\n            case 'day':\n            case 'date':\n                time = startOfDate(this.year(), this.month(), this.date() + 1) - 1;\n                break;\n            case 'hour':\n                time = this._d.valueOf();\n                time += MS_PER_HOUR - mod$1(time + (this._isUTC ? 0 : this.utcOffset() * MS_PER_MINUTE), MS_PER_HOUR) - 1;\n                break;\n            case 'minute':\n                time = this._d.valueOf();\n                time += MS_PER_MINUTE - mod$1(time, MS_PER_MINUTE) - 1;\n                break;\n            case 'second':\n                time = this._d.valueOf();\n                time += MS_PER_SECOND - mod$1(time, MS_PER_SECOND) - 1;\n                break;\n        }\n\n        this._d.setTime(time);\n        hooks.updateOffset(this, true);\n        return this;\n    }\n\n    function valueOf () {\n        return this._d.valueOf() - ((this._offset || 0) * 60000);\n    }\n\n    function unix () {\n        return Math.floor(this.valueOf() / 1000);\n    }\n\n    function toDate () {\n        return new Date(this.valueOf());\n    }\n\n    function toArray () {\n        var m = this;\n        return [m.year(), m.month(), m.date(), m.hour(), m.minute(), m.second(), m.millisecond()];\n    }\n\n    function toObject () {\n        var m = this;\n        return {\n            years: m.year(),\n            months: m.month(),\n            date: m.date(),\n            hours: m.hours(),\n            minutes: m.minutes(),\n            seconds: m.seconds(),\n            milliseconds: m.milliseconds()\n        };\n    }\n\n    function toJSON () {\n        // new Date(NaN).toJSON() === null\n        return this.isValid() ? this.toISOString() : null;\n    }\n\n    function isValid$2 () {\n        return isValid(this);\n    }\n\n    function parsingFlags () {\n        return extend({}, getParsingFlags(this));\n    }\n\n    function invalidAt () {\n        return getParsingFlags(this).overflow;\n    }\n\n    function creationData() {\n        return {\n            input: this._i,\n            format: this._f,\n            locale: this._locale,\n            isUTC: this._isUTC,\n            strict: this._strict\n        };\n    }\n\n    // FORMATTING\n\n    addFormatToken(0, ['gg', 2], 0, function () {\n        return this.weekYear() % 100;\n    });\n\n    addFormatToken(0, ['GG', 2], 0, function () {\n        return this.isoWeekYear() % 100;\n    });\n\n    function addWeekYearFormatToken (token, getter) {\n        addFormatToken(0, [token, token.length], 0, getter);\n    }\n\n    addWeekYearFormatToken('gggg',     'weekYear');\n    addWeekYearFormatToken('ggggg',    'weekYear');\n    addWeekYearFormatToken('GGGG',  'isoWeekYear');\n    addWeekYearFormatToken('GGGGG', 'isoWeekYear');\n\n    // ALIASES\n\n    addUnitAlias('weekYear', 'gg');\n    addUnitAlias('isoWeekYear', 'GG');\n\n    // PRIORITY\n\n    addUnitPriority('weekYear', 1);\n    addUnitPriority('isoWeekYear', 1);\n\n\n    // PARSING\n\n    addRegexToken('G',      matchSigned);\n    addRegexToken('g',      matchSigned);\n    addRegexToken('GG',     match1to2, match2);\n    addRegexToken('gg',     match1to2, match2);\n    addRegexToken('GGGG',   match1to4, match4);\n    addRegexToken('gggg',   match1to4, match4);\n    addRegexToken('GGGGG',  match1to6, match6);\n    addRegexToken('ggggg',  match1to6, match6);\n\n    addWeekParseToken(['gggg', 'ggggg', 'GGGG', 'GGGGG'], function (input, week, config, token) {\n        week[token.substr(0, 2)] = toInt(input);\n    });\n\n    addWeekParseToken(['gg', 'GG'], function (input, week, config, token) {\n        week[token] = hooks.parseTwoDigitYear(input);\n    });\n\n    // MOMENTS\n\n    function getSetWeekYear (input) {\n        return getSetWeekYearHelper.call(this,\n                input,\n                this.week(),\n                this.weekday(),\n                this.localeData()._week.dow,\n                this.localeData()._week.doy);\n    }\n\n    function getSetISOWeekYear (input) {\n        return getSetWeekYearHelper.call(this,\n                input, this.isoWeek(), this.isoWeekday(), 1, 4);\n    }\n\n    function getISOWeeksInYear () {\n        return weeksInYear(this.year(), 1, 4);\n    }\n\n    function getWeeksInYear () {\n        var weekInfo = this.localeData()._week;\n        return weeksInYear(this.year(), weekInfo.dow, weekInfo.doy);\n    }\n\n    function getSetWeekYearHelper(input, week, weekday, dow, doy) {\n        var weeksTarget;\n        if (input == null) {\n            return weekOfYear(this, dow, doy).year;\n        } else {\n            weeksTarget = weeksInYear(input, dow, doy);\n            if (week > weeksTarget) {\n                week = weeksTarget;\n            }\n            return setWeekAll.call(this, input, week, weekday, dow, doy);\n        }\n    }\n\n    function setWeekAll(weekYear, week, weekday, dow, doy) {\n        var dayOfYearData = dayOfYearFromWeeks(weekYear, week, weekday, dow, doy),\n            date = createUTCDate(dayOfYearData.year, 0, dayOfYearData.dayOfYear);\n\n        this.year(date.getUTCFullYear());\n        this.month(date.getUTCMonth());\n        this.date(date.getUTCDate());\n        return this;\n    }\n\n    // FORMATTING\n\n    addFormatToken('Q', 0, 'Qo', 'quarter');\n\n    // ALIASES\n\n    addUnitAlias('quarter', 'Q');\n\n    // PRIORITY\n\n    addUnitPriority('quarter', 7);\n\n    // PARSING\n\n    addRegexToken('Q', match1);\n    addParseToken('Q', function (input, array) {\n        array[MONTH] = (toInt(input) - 1) * 3;\n    });\n\n    // MOMENTS\n\n    function getSetQuarter (input) {\n        return input == null ? Math.ceil((this.month() + 1) / 3) : this.month((input - 1) * 3 + this.month() % 3);\n    }\n\n    // FORMATTING\n\n    addFormatToken('D', ['DD', 2], 'Do', 'date');\n\n    // ALIASES\n\n    addUnitAlias('date', 'D');\n\n    // PRIORITY\n    addUnitPriority('date', 9);\n\n    // PARSING\n\n    addRegexToken('D',  match1to2);\n    addRegexToken('DD', match1to2, match2);\n    addRegexToken('Do', function (isStrict, locale) {\n        // TODO: Remove \"ordinalParse\" fallback in next major release.\n        return isStrict ?\n          (locale._dayOfMonthOrdinalParse || locale._ordinalParse) :\n          locale._dayOfMonthOrdinalParseLenient;\n    });\n\n    addParseToken(['D', 'DD'], DATE);\n    addParseToken('Do', function (input, array) {\n        array[DATE] = toInt(input.match(match1to2)[0]);\n    });\n\n    // MOMENTS\n\n    var getSetDayOfMonth = makeGetSet('Date', true);\n\n    // FORMATTING\n\n    addFormatToken('DDD', ['DDDD', 3], 'DDDo', 'dayOfYear');\n\n    // ALIASES\n\n    addUnitAlias('dayOfYear', 'DDD');\n\n    // PRIORITY\n    addUnitPriority('dayOfYear', 4);\n\n    // PARSING\n\n    addRegexToken('DDD',  match1to3);\n    addRegexToken('DDDD', match3);\n    addParseToken(['DDD', 'DDDD'], function (input, array, config) {\n        config._dayOfYear = toInt(input);\n    });\n\n    // HELPERS\n\n    // MOMENTS\n\n    function getSetDayOfYear (input) {\n        var dayOfYear = Math.round((this.clone().startOf('day') - this.clone().startOf('year')) / 864e5) + 1;\n        return input == null ? dayOfYear : this.add((input - dayOfYear), 'd');\n    }\n\n    // FORMATTING\n\n    addFormatToken('m', ['mm', 2], 0, 'minute');\n\n    // ALIASES\n\n    addUnitAlias('minute', 'm');\n\n    // PRIORITY\n\n    addUnitPriority('minute', 14);\n\n    // PARSING\n\n    addRegexToken('m',  match1to2);\n    addRegexToken('mm', match1to2, match2);\n    addParseToken(['m', 'mm'], MINUTE);\n\n    // MOMENTS\n\n    var getSetMinute = makeGetSet('Minutes', false);\n\n    // FORMATTING\n\n    addFormatToken('s', ['ss', 2], 0, 'second');\n\n    // ALIASES\n\n    addUnitAlias('second', 's');\n\n    // PRIORITY\n\n    addUnitPriority('second', 15);\n\n    // PARSING\n\n    addRegexToken('s',  match1to2);\n    addRegexToken('ss', match1to2, match2);\n    addParseToken(['s', 'ss'], SECOND);\n\n    // MOMENTS\n\n    var getSetSecond = makeGetSet('Seconds', false);\n\n    // FORMATTING\n\n    addFormatToken('S', 0, 0, function () {\n        return ~~(this.millisecond() / 100);\n    });\n\n    addFormatToken(0, ['SS', 2], 0, function () {\n        return ~~(this.millisecond() / 10);\n    });\n\n    addFormatToken(0, ['SSS', 3], 0, 'millisecond');\n    addFormatToken(0, ['SSSS', 4], 0, function () {\n        return this.millisecond() * 10;\n    });\n    addFormatToken(0, ['SSSSS', 5], 0, function () {\n        return this.millisecond() * 100;\n    });\n    addFormatToken(0, ['SSSSSS', 6], 0, function () {\n        return this.millisecond() * 1000;\n    });\n    addFormatToken(0, ['SSSSSSS', 7], 0, function () {\n        return this.millisecond() * 10000;\n    });\n    addFormatToken(0, ['SSSSSSSS', 8], 0, function () {\n        return this.millisecond() * 100000;\n    });\n    addFormatToken(0, ['SSSSSSSSS', 9], 0, function () {\n        return this.millisecond() * 1000000;\n    });\n\n\n    // ALIASES\n\n    addUnitAlias('millisecond', 'ms');\n\n    // PRIORITY\n\n    addUnitPriority('millisecond', 16);\n\n    // PARSING\n\n    addRegexToken('S',    match1to3, match1);\n    addRegexToken('SS',   match1to3, match2);\n    addRegexToken('SSS',  match1to3, match3);\n\n    var token;\n    for (token = 'SSSS'; token.length <= 9; token += 'S') {\n        addRegexToken(token, matchUnsigned);\n    }\n\n    function parseMs(input, array) {\n        array[MILLISECOND] = toInt(('0.' + input) * 1000);\n    }\n\n    for (token = 'S'; token.length <= 9; token += 'S') {\n        addParseToken(token, parseMs);\n    }\n    // MOMENTS\n\n    var getSetMillisecond = makeGetSet('Milliseconds', false);\n\n    // FORMATTING\n\n    addFormatToken('z',  0, 0, 'zoneAbbr');\n    addFormatToken('zz', 0, 0, 'zoneName');\n\n    // MOMENTS\n\n    function getZoneAbbr () {\n        return this._isUTC ? 'UTC' : '';\n    }\n\n    function getZoneName () {\n        return this._isUTC ? 'Coordinated Universal Time' : '';\n    }\n\n    var proto = Moment.prototype;\n\n    proto.add               = add;\n    proto.calendar          = calendar$1;\n    proto.clone             = clone;\n    proto.diff              = diff;\n    proto.endOf             = endOf;\n    proto.format            = format;\n    proto.from              = from;\n    proto.fromNow           = fromNow;\n    proto.to                = to;\n    proto.toNow             = toNow;\n    proto.get               = stringGet;\n    proto.invalidAt         = invalidAt;\n    proto.isAfter           = isAfter;\n    proto.isBefore          = isBefore;\n    proto.isBetween         = isBetween;\n    proto.isSame            = isSame;\n    proto.isSameOrAfter     = isSameOrAfter;\n    proto.isSameOrBefore    = isSameOrBefore;\n    proto.isValid           = isValid$2;\n    proto.lang              = lang;\n    proto.locale            = locale;\n    proto.localeData        = localeData;\n    proto.max               = prototypeMax;\n    proto.min               = prototypeMin;\n    proto.parsingFlags      = parsingFlags;\n    proto.set               = stringSet;\n    proto.startOf           = startOf;\n    proto.subtract          = subtract;\n    proto.toArray           = toArray;\n    proto.toObject          = toObject;\n    proto.toDate            = toDate;\n    proto.toISOString       = toISOString;\n    proto.inspect           = inspect;\n    proto.toJSON            = toJSON;\n    proto.toString          = toString;\n    proto.unix              = unix;\n    proto.valueOf           = valueOf;\n    proto.creationData      = creationData;\n    proto.year       = getSetYear;\n    proto.isLeapYear = getIsLeapYear;\n    proto.weekYear    = getSetWeekYear;\n    proto.isoWeekYear = getSetISOWeekYear;\n    proto.quarter = proto.quarters = getSetQuarter;\n    proto.month       = getSetMonth;\n    proto.daysInMonth = getDaysInMonth;\n    proto.week           = proto.weeks        = getSetWeek;\n    proto.isoWeek        = proto.isoWeeks     = getSetISOWeek;\n    proto.weeksInYear    = getWeeksInYear;\n    proto.isoWeeksInYear = getISOWeeksInYear;\n    proto.date       = getSetDayOfMonth;\n    proto.day        = proto.days             = getSetDayOfWeek;\n    proto.weekday    = getSetLocaleDayOfWeek;\n    proto.isoWeekday = getSetISODayOfWeek;\n    proto.dayOfYear  = getSetDayOfYear;\n    proto.hour = proto.hours = getSetHour;\n    proto.minute = proto.minutes = getSetMinute;\n    proto.second = proto.seconds = getSetSecond;\n    proto.millisecond = proto.milliseconds = getSetMillisecond;\n    proto.utcOffset            = getSetOffset;\n    proto.utc                  = setOffsetToUTC;\n    proto.local                = setOffsetToLocal;\n    proto.parseZone            = setOffsetToParsedOffset;\n    proto.hasAlignedHourOffset = hasAlignedHourOffset;\n    proto.isDST                = isDaylightSavingTime;\n    proto.isLocal              = isLocal;\n    proto.isUtcOffset          = isUtcOffset;\n    proto.isUtc                = isUtc;\n    proto.isUTC                = isUtc;\n    proto.zoneAbbr = getZoneAbbr;\n    proto.zoneName = getZoneName;\n    proto.dates  = deprecate('dates accessor is deprecated. Use date instead.', getSetDayOfMonth);\n    proto.months = deprecate('months accessor is deprecated. Use month instead', getSetMonth);\n    proto.years  = deprecate('years accessor is deprecated. Use year instead', getSetYear);\n    proto.zone   = deprecate('moment().zone is deprecated, use moment().utcOffset instead. http://momentjs.com/guides/#/warnings/zone/', getSetZone);\n    proto.isDSTShifted = deprecate('isDSTShifted is deprecated. See http://momentjs.com/guides/#/warnings/dst-shifted/ for more information', isDaylightSavingTimeShifted);\n\n    function createUnix (input) {\n        return createLocal(input * 1000);\n    }\n\n    function createInZone () {\n        return createLocal.apply(null, arguments).parseZone();\n    }\n\n    function preParsePostFormat (string) {\n        return string;\n    }\n\n    var proto$1 = Locale.prototype;\n\n    proto$1.calendar        = calendar;\n    proto$1.longDateFormat  = longDateFormat;\n    proto$1.invalidDate     = invalidDate;\n    proto$1.ordinal         = ordinal;\n    proto$1.preparse        = preParsePostFormat;\n    proto$1.postformat      = preParsePostFormat;\n    proto$1.relativeTime    = relativeTime;\n    proto$1.pastFuture      = pastFuture;\n    proto$1.set             = set;\n\n    proto$1.months            =        localeMonths;\n    proto$1.monthsShort       =        localeMonthsShort;\n    proto$1.monthsParse       =        localeMonthsParse;\n    proto$1.monthsRegex       = monthsRegex;\n    proto$1.monthsShortRegex  = monthsShortRegex;\n    proto$1.week = localeWeek;\n    proto$1.firstDayOfYear = localeFirstDayOfYear;\n    proto$1.firstDayOfWeek = localeFirstDayOfWeek;\n\n    proto$1.weekdays       =        localeWeekdays;\n    proto$1.weekdaysMin    =        localeWeekdaysMin;\n    proto$1.weekdaysShort  =        localeWeekdaysShort;\n    proto$1.weekdaysParse  =        localeWeekdaysParse;\n\n    proto$1.weekdaysRegex       =        weekdaysRegex;\n    proto$1.weekdaysShortRegex  =        weekdaysShortRegex;\n    proto$1.weekdaysMinRegex    =        weekdaysMinRegex;\n\n    proto$1.isPM = localeIsPM;\n    proto$1.meridiem = localeMeridiem;\n\n    function get$1 (format, index, field, setter) {\n        var locale = getLocale();\n        var utc = createUTC().set(setter, index);\n        return locale[field](utc, format);\n    }\n\n    function listMonthsImpl (format, index, field) {\n        if (isNumber(format)) {\n            index = format;\n            format = undefined;\n        }\n\n        format = format || '';\n\n        if (index != null) {\n            return get$1(format, index, field, 'month');\n        }\n\n        var i;\n        var out = [];\n        for (i = 0; i < 12; i++) {\n            out[i] = get$1(format, i, field, 'month');\n        }\n        return out;\n    }\n\n    // ()\n    // (5)\n    // (fmt, 5)\n    // (fmt)\n    // (true)\n    // (true, 5)\n    // (true, fmt, 5)\n    // (true, fmt)\n    function listWeekdaysImpl (localeSorted, format, index, field) {\n        if (typeof localeSorted === 'boolean') {\n            if (isNumber(format)) {\n                index = format;\n                format = undefined;\n            }\n\n            format = format || '';\n        } else {\n            format = localeSorted;\n            index = format;\n            localeSorted = false;\n\n            if (isNumber(format)) {\n                index = format;\n                format = undefined;\n            }\n\n            format = format || '';\n        }\n\n        var locale = getLocale(),\n            shift = localeSorted ? locale._week.dow : 0;\n\n        if (index != null) {\n            return get$1(format, (index + shift) % 7, field, 'day');\n        }\n\n        var i;\n        var out = [];\n        for (i = 0; i < 7; i++) {\n            out[i] = get$1(format, (i + shift) % 7, field, 'day');\n        }\n        return out;\n    }\n\n    function listMonths (format, index) {\n        return listMonthsImpl(format, index, 'months');\n    }\n\n    function listMonthsShort (format, index) {\n        return listMonthsImpl(format, index, 'monthsShort');\n    }\n\n    function listWeekdays (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdays');\n    }\n\n    function listWeekdaysShort (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdaysShort');\n    }\n\n    function listWeekdaysMin (localeSorted, format, index) {\n        return listWeekdaysImpl(localeSorted, format, index, 'weekdaysMin');\n    }\n\n    getSetGlobalLocale('en', {\n        dayOfMonthOrdinalParse: /\\d{1,2}(th|st|nd|rd)/,\n        ordinal : function (number) {\n            var b = number % 10,\n                output = (toInt(number % 100 / 10) === 1) ? 'th' :\n                (b === 1) ? 'st' :\n                (b === 2) ? 'nd' :\n                (b === 3) ? 'rd' : 'th';\n            return number + output;\n        }\n    });\n\n    // Side effect imports\n\n    hooks.lang = deprecate('moment.lang is deprecated. Use moment.locale instead.', getSetGlobalLocale);\n    hooks.langData = deprecate('moment.langData is deprecated. Use moment.localeData instead.', getLocale);\n\n    var mathAbs = Math.abs;\n\n    function abs () {\n        var data           = this._data;\n\n        this._milliseconds = mathAbs(this._milliseconds);\n        this._days         = mathAbs(this._days);\n        this._months       = mathAbs(this._months);\n\n        data.milliseconds  = mathAbs(data.milliseconds);\n        data.seconds       = mathAbs(data.seconds);\n        data.minutes       = mathAbs(data.minutes);\n        data.hours         = mathAbs(data.hours);\n        data.months        = mathAbs(data.months);\n        data.years         = mathAbs(data.years);\n\n        return this;\n    }\n\n    function addSubtract$1 (duration, input, value, direction) {\n        var other = createDuration(input, value);\n\n        duration._milliseconds += direction * other._milliseconds;\n        duration._days         += direction * other._days;\n        duration._months       += direction * other._months;\n\n        return duration._bubble();\n    }\n\n    // supports only 2.0-style add(1, 's') or add(duration)\n    function add$1 (input, value) {\n        return addSubtract$1(this, input, value, 1);\n    }\n\n    // supports only 2.0-style subtract(1, 's') or subtract(duration)\n    function subtract$1 (input, value) {\n        return addSubtract$1(this, input, value, -1);\n    }\n\n    function absCeil (number) {\n        if (number < 0) {\n            return Math.floor(number);\n        } else {\n            return Math.ceil(number);\n        }\n    }\n\n    function bubble () {\n        var milliseconds = this._milliseconds;\n        var days         = this._days;\n        var months       = this._months;\n        var data         = this._data;\n        var seconds, minutes, hours, years, monthsFromDays;\n\n        // if we have a mix of positive and negative values, bubble down first\n        // check: https://github.com/moment/moment/issues/2166\n        if (!((milliseconds >= 0 && days >= 0 && months >= 0) ||\n                (milliseconds <= 0 && days <= 0 && months <= 0))) {\n            milliseconds += absCeil(monthsToDays(months) + days) * 864e5;\n            days = 0;\n            months = 0;\n        }\n\n        // The following code bubbles up values, see the tests for\n        // examples of what that means.\n        data.milliseconds = milliseconds % 1000;\n\n        seconds           = absFloor(milliseconds / 1000);\n        data.seconds      = seconds % 60;\n\n        minutes           = absFloor(seconds / 60);\n        data.minutes      = minutes % 60;\n\n        hours             = absFloor(minutes / 60);\n        data.hours        = hours % 24;\n\n        days += absFloor(hours / 24);\n\n        // convert days to months\n        monthsFromDays = absFloor(daysToMonths(days));\n        months += monthsFromDays;\n        days -= absCeil(monthsToDays(monthsFromDays));\n\n        // 12 months -> 1 year\n        years = absFloor(months / 12);\n        months %= 12;\n\n        data.days   = days;\n        data.months = months;\n        data.years  = years;\n\n        return this;\n    }\n\n    function daysToMonths (days) {\n        // 400 years have 146097 days (taking into account leap year rules)\n        // 400 years have 12 months === 4800\n        return days * 4800 / 146097;\n    }\n\n    function monthsToDays (months) {\n        // the reverse of daysToMonths\n        return months * 146097 / 4800;\n    }\n\n    function as (units) {\n        if (!this.isValid()) {\n            return NaN;\n        }\n        var days;\n        var months;\n        var milliseconds = this._milliseconds;\n\n        units = normalizeUnits(units);\n\n        if (units === 'month' || units === 'quarter' || units === 'year') {\n            days = this._days + milliseconds / 864e5;\n            months = this._months + daysToMonths(days);\n            switch (units) {\n                case 'month':   return months;\n                case 'quarter': return months / 3;\n                case 'year':    return months / 12;\n            }\n        } else {\n            // handle milliseconds separately because of floating point math errors (issue #1867)\n            days = this._days + Math.round(monthsToDays(this._months));\n            switch (units) {\n                case 'week'   : return days / 7     + milliseconds / 6048e5;\n                case 'day'    : return days         + milliseconds / 864e5;\n                case 'hour'   : return days * 24    + milliseconds / 36e5;\n                case 'minute' : return days * 1440  + milliseconds / 6e4;\n                case 'second' : return days * 86400 + milliseconds / 1000;\n                // Math.floor prevents floating point math errors here\n                case 'millisecond': return Math.floor(days * 864e5) + milliseconds;\n                default: throw new Error('Unknown unit ' + units);\n            }\n        }\n    }\n\n    // TODO: Use this.as('ms')?\n    function valueOf$1 () {\n        if (!this.isValid()) {\n            return NaN;\n        }\n        return (\n            this._milliseconds +\n            this._days * 864e5 +\n            (this._months % 12) * 2592e6 +\n            toInt(this._months / 12) * 31536e6\n        );\n    }\n\n    function makeAs (alias) {\n        return function () {\n            return this.as(alias);\n        };\n    }\n\n    var asMilliseconds = makeAs('ms');\n    var asSeconds      = makeAs('s');\n    var asMinutes      = makeAs('m');\n    var asHours        = makeAs('h');\n    var asDays         = makeAs('d');\n    var asWeeks        = makeAs('w');\n    var asMonths       = makeAs('M');\n    var asQuarters     = makeAs('Q');\n    var asYears        = makeAs('y');\n\n    function clone$1 () {\n        return createDuration(this);\n    }\n\n    function get$2 (units) {\n        units = normalizeUnits(units);\n        return this.isValid() ? this[units + 's']() : NaN;\n    }\n\n    function makeGetter(name) {\n        return function () {\n            return this.isValid() ? this._data[name] : NaN;\n        };\n    }\n\n    var milliseconds = makeGetter('milliseconds');\n    var seconds      = makeGetter('seconds');\n    var minutes      = makeGetter('minutes');\n    var hours        = makeGetter('hours');\n    var days         = makeGetter('days');\n    var months       = makeGetter('months');\n    var years        = makeGetter('years');\n\n    function weeks () {\n        return absFloor(this.days() / 7);\n    }\n\n    var round = Math.round;\n    var thresholds = {\n        ss: 44,         // a few seconds to seconds\n        s : 45,         // seconds to minute\n        m : 45,         // minutes to hour\n        h : 22,         // hours to day\n        d : 26,         // days to month\n        M : 11          // months to year\n    };\n\n    // helper function for moment.fn.from, moment.fn.fromNow, and moment.duration.fn.humanize\n    function substituteTimeAgo(string, number, withoutSuffix, isFuture, locale) {\n        return locale.relativeTime(number || 1, !!withoutSuffix, string, isFuture);\n    }\n\n    function relativeTime$1 (posNegDuration, withoutSuffix, locale) {\n        var duration = createDuration(posNegDuration).abs();\n        var seconds  = round(duration.as('s'));\n        var minutes  = round(duration.as('m'));\n        var hours    = round(duration.as('h'));\n        var days     = round(duration.as('d'));\n        var months   = round(duration.as('M'));\n        var years    = round(duration.as('y'));\n\n        var a = seconds <= thresholds.ss && ['s', seconds]  ||\n                seconds < thresholds.s   && ['ss', seconds] ||\n                minutes <= 1             && ['m']           ||\n                minutes < thresholds.m   && ['mm', minutes] ||\n                hours   <= 1             && ['h']           ||\n                hours   < thresholds.h   && ['hh', hours]   ||\n                days    <= 1             && ['d']           ||\n                days    < thresholds.d   && ['dd', days]    ||\n                months  <= 1             && ['M']           ||\n                months  < thresholds.M   && ['MM', months]  ||\n                years   <= 1             && ['y']           || ['yy', years];\n\n        a[2] = withoutSuffix;\n        a[3] = +posNegDuration > 0;\n        a[4] = locale;\n        return substituteTimeAgo.apply(null, a);\n    }\n\n    // This function allows you to set the rounding function for relative time strings\n    function getSetRelativeTimeRounding (roundingFunction) {\n        if (roundingFunction === undefined) {\n            return round;\n        }\n        if (typeof(roundingFunction) === 'function') {\n            round = roundingFunction;\n            return true;\n        }\n        return false;\n    }\n\n    // This function allows you to set a threshold for relative time strings\n    function getSetRelativeTimeThreshold (threshold, limit) {\n        if (thresholds[threshold] === undefined) {\n            return false;\n        }\n        if (limit === undefined) {\n            return thresholds[threshold];\n        }\n        thresholds[threshold] = limit;\n        if (threshold === 's') {\n            thresholds.ss = limit - 1;\n        }\n        return true;\n    }\n\n    function humanize (withSuffix) {\n        if (!this.isValid()) {\n            return this.localeData().invalidDate();\n        }\n\n        var locale = this.localeData();\n        var output = relativeTime$1(this, !withSuffix, locale);\n\n        if (withSuffix) {\n            output = locale.pastFuture(+this, output);\n        }\n\n        return locale.postformat(output);\n    }\n\n    var abs$1 = Math.abs;\n\n    function sign(x) {\n        return ((x > 0) - (x < 0)) || +x;\n    }\n\n    function toISOString$1() {\n        // for ISO strings we do not use the normal bubbling rules:\n        //  * milliseconds bubble up until they become hours\n        //  * days do not bubble at all\n        //  * months bubble up until they become years\n        // This is because there is no context-free conversion between hours and days\n        // (think of clock changes)\n        // and also not between days and months (28-31 days per month)\n        if (!this.isValid()) {\n            return this.localeData().invalidDate();\n        }\n\n        var seconds = abs$1(this._milliseconds) / 1000;\n        var days         = abs$1(this._days);\n        var months       = abs$1(this._months);\n        var minutes, hours, years;\n\n        // 3600 seconds -> 60 minutes -> 1 hour\n        minutes           = absFloor(seconds / 60);\n        hours             = absFloor(minutes / 60);\n        seconds %= 60;\n        minutes %= 60;\n\n        // 12 months -> 1 year\n        years  = absFloor(months / 12);\n        months %= 12;\n\n\n        // inspired by https://github.com/dordille/moment-isoduration/blob/master/moment.isoduration.js\n        var Y = years;\n        var M = months;\n        var D = days;\n        var h = hours;\n        var m = minutes;\n        var s = seconds ? seconds.toFixed(3).replace(/\\.?0+$/, '') : '';\n        var total = this.asSeconds();\n\n        if (!total) {\n            // this is the same as C#'s (Noda) and python (isodate)...\n            // but not other JS (goog.date)\n            return 'P0D';\n        }\n\n        var totalSign = total < 0 ? '-' : '';\n        var ymSign = sign(this._months) !== sign(total) ? '-' : '';\n        var daysSign = sign(this._days) !== sign(total) ? '-' : '';\n        var hmsSign = sign(this._milliseconds) !== sign(total) ? '-' : '';\n\n        return totalSign + 'P' +\n            (Y ? ymSign + Y + 'Y' : '') +\n            (M ? ymSign + M + 'M' : '') +\n            (D ? daysSign + D + 'D' : '') +\n            ((h || m || s) ? 'T' : '') +\n            (h ? hmsSign + h + 'H' : '') +\n            (m ? hmsSign + m + 'M' : '') +\n            (s ? hmsSign + s + 'S' : '');\n    }\n\n    var proto$2 = Duration.prototype;\n\n    proto$2.isValid        = isValid$1;\n    proto$2.abs            = abs;\n    proto$2.add            = add$1;\n    proto$2.subtract       = subtract$1;\n    proto$2.as             = as;\n    proto$2.asMilliseconds = asMilliseconds;\n    proto$2.asSeconds      = asSeconds;\n    proto$2.asMinutes      = asMinutes;\n    proto$2.asHours        = asHours;\n    proto$2.asDays         = asDays;\n    proto$2.asWeeks        = asWeeks;\n    proto$2.asMonths       = asMonths;\n    proto$2.asQuarters     = asQuarters;\n    proto$2.asYears        = asYears;\n    proto$2.valueOf        = valueOf$1;\n    proto$2._bubble        = bubble;\n    proto$2.clone          = clone$1;\n    proto$2.get            = get$2;\n    proto$2.milliseconds   = milliseconds;\n    proto$2.seconds        = seconds;\n    proto$2.minutes        = minutes;\n    proto$2.hours          = hours;\n    proto$2.days           = days;\n    proto$2.weeks          = weeks;\n    proto$2.months         = months;\n    proto$2.years          = years;\n    proto$2.humanize       = humanize;\n    proto$2.toISOString    = toISOString$1;\n    proto$2.toString       = toISOString$1;\n    proto$2.toJSON         = toISOString$1;\n    proto$2.locale         = locale;\n    proto$2.localeData     = localeData;\n\n    proto$2.toIsoString = deprecate('toIsoString() is deprecated. Please use toISOString() instead (notice the capitals)', toISOString$1);\n    proto$2.lang = lang;\n\n    // Side effect imports\n\n    // FORMATTING\n\n    addFormatToken('X', 0, 0, 'unix');\n    addFormatToken('x', 0, 0, 'valueOf');\n\n    // PARSING\n\n    addRegexToken('x', matchSigned);\n    addRegexToken('X', matchTimestamp);\n    addParseToken('X', function (input, array, config) {\n        config._d = new Date(parseFloat(input, 10) * 1000);\n    });\n    addParseToken('x', function (input, array, config) {\n        config._d = new Date(toInt(input));\n    });\n\n    // Side effect imports\n\n\n    hooks.version = '2.24.0';\n\n    setHookCallback(createLocal);\n\n    hooks.fn                    = proto;\n    hooks.min                   = min;\n    hooks.max                   = max;\n    hooks.now                   = now;\n    hooks.utc                   = createUTC;\n    hooks.unix                  = createUnix;\n    hooks.months                = listMonths;\n    hooks.isDate                = isDate;\n    hooks.locale                = getSetGlobalLocale;\n    hooks.invalid               = createInvalid;\n    hooks.duration              = createDuration;\n    hooks.isMoment              = isMoment;\n    hooks.weekdays              = listWeekdays;\n    hooks.parseZone             = createInZone;\n    hooks.localeData            = getLocale;\n    hooks.isDuration            = isDuration;\n    hooks.monthsShort           = listMonthsShort;\n    hooks.weekdaysMin           = listWeekdaysMin;\n    hooks.defineLocale          = defineLocale;\n    hooks.updateLocale          = updateLocale;\n    hooks.locales               = listLocales;\n    hooks.weekdaysShort         = listWeekdaysShort;\n    hooks.normalizeUnits        = normalizeUnits;\n    hooks.relativeTimeRounding  = getSetRelativeTimeRounding;\n    hooks.relativeTimeThreshold = getSetRelativeTimeThreshold;\n    hooks.calendarFormat        = getCalendarFormat;\n    hooks.prototype             = proto;\n\n    // currently HTML5 input type only supports 24-hour formats\n    hooks.HTML5_FMT = {\n        DATETIME_LOCAL: 'YYYY-MM-DDTHH:mm',             // <input type=\"datetime-local\" />\n        DATETIME_LOCAL_SECONDS: 'YYYY-MM-DDTHH:mm:ss',  // <input type=\"datetime-local\" step=\"1\" />\n        DATETIME_LOCAL_MS: 'YYYY-MM-DDTHH:mm:ss.SSS',   // <input type=\"datetime-local\" step=\"0.001\" />\n        DATE: 'YYYY-MM-DD',                             // <input type=\"date\" />\n        TIME: 'HH:mm',                                  // <input type=\"time\" />\n        TIME_SECONDS: 'HH:mm:ss',                       // <input type=\"time\" step=\"1\" />\n        TIME_MS: 'HH:mm:ss.SSS',                        // <input type=\"time\" step=\"0.001\" />\n        WEEK: 'GGGG-[W]WW',                             // <input type=\"week\" />\n        MONTH: 'YYYY-MM'                                // <input type=\"month\" />\n    };\n\n    return hooks;\n\n})));\n","var check = function (it) {\n  return it && it.Math == Math && it;\n};\n\n// https://github.com/zloirock/core-js/issues/86#issuecomment-115759028\nmodule.exports =\n  // eslint-disable-next-line no-undef\n  check(typeof globalThis == 'object' && globalThis) ||\n  check(typeof window == 'object' && window) ||\n  check(typeof self == 'object' && self) ||\n  check(typeof global == 'object' && global) ||\n  // eslint-disable-next-line no-new-func\n  Function('return this')();\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function processRelativeTime(number, withoutSuffix, key, isFuture) {\n        var format = {\n            'm': ['eine Minute', 'einer Minute'],\n            'h': ['eine Stunde', 'einer Stunde'],\n            'd': ['ein Tag', 'einem Tag'],\n            'dd': [number + ' Tage', number + ' Tagen'],\n            'M': ['ein Monat', 'einem Monat'],\n            'MM': [number + ' Monate', number + ' Monaten'],\n            'y': ['ein Jahr', 'einem Jahr'],\n            'yy': [number + ' Jahre', number + ' Jahren']\n        };\n        return withoutSuffix ? format[key][0] : format[key][1];\n    }\n\n    var de = moment.defineLocale('de', {\n        months : 'Januar_Februar_März_April_Mai_Juni_Juli_August_September_Oktober_November_Dezember'.split('_'),\n        monthsShort : 'Jan._Feb._März_Apr._Mai_Juni_Juli_Aug._Sep._Okt._Nov._Dez.'.split('_'),\n        monthsParseExact : true,\n        weekdays : 'Sonntag_Montag_Dienstag_Mittwoch_Donnerstag_Freitag_Samstag'.split('_'),\n        weekdaysShort : 'So._Mo._Di._Mi._Do._Fr._Sa.'.split('_'),\n        weekdaysMin : 'So_Mo_Di_Mi_Do_Fr_Sa'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT: 'HH:mm',\n            LTS: 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D. MMMM YYYY',\n            LLL : 'D. MMMM YYYY HH:mm',\n            LLLL : 'dddd, D. MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[heute um] LT [Uhr]',\n            sameElse: 'L',\n            nextDay: '[morgen um] LT [Uhr]',\n            nextWeek: 'dddd [um] LT [Uhr]',\n            lastDay: '[gestern um] LT [Uhr]',\n            lastWeek: '[letzten] dddd [um] LT [Uhr]'\n        },\n        relativeTime : {\n            future : 'in %s',\n            past : 'vor %s',\n            s : 'ein paar Sekunden',\n            ss : '%d Sekunden',\n            m : processRelativeTime,\n            mm : '%d Minuten',\n            h : processRelativeTime,\n            hh : '%d Stunden',\n            d : processRelativeTime,\n            dd : processRelativeTime,\n            M : processRelativeTime,\n            MM : processRelativeTime,\n            y : processRelativeTime,\n            yy : processRelativeTime\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}\\./,\n        ordinal : '%d.',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return de;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsShortDot = 'ene._feb._mar._abr._may._jun._jul._ago._sep._oct._nov._dic.'.split('_'),\n        monthsShort = 'ene_feb_mar_abr_may_jun_jul_ago_sep_oct_nov_dic'.split('_');\n\n    var monthsParse = [/^ene/i, /^feb/i, /^mar/i, /^abr/i, /^may/i, /^jun/i, /^jul/i, /^ago/i, /^sep/i, /^oct/i, /^nov/i, /^dic/i];\n    var monthsRegex = /^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre|ene\\.?|feb\\.?|mar\\.?|abr\\.?|may\\.?|jun\\.?|jul\\.?|ago\\.?|sep\\.?|oct\\.?|nov\\.?|dic\\.?)/i;\n\n    var es = moment.defineLocale('es', {\n        months : 'enero_febrero_marzo_abril_mayo_junio_julio_agosto_septiembre_octubre_noviembre_diciembre'.split('_'),\n        monthsShort : function (m, format) {\n            if (!m) {\n                return monthsShortDot;\n            } else if (/-MMM-/.test(format)) {\n                return monthsShort[m.month()];\n            } else {\n                return monthsShortDot[m.month()];\n            }\n        },\n        monthsRegex : monthsRegex,\n        monthsShortRegex : monthsRegex,\n        monthsStrictRegex : /^(enero|febrero|marzo|abril|mayo|junio|julio|agosto|septiembre|octubre|noviembre|diciembre)/i,\n        monthsShortStrictRegex : /^(ene\\.?|feb\\.?|mar\\.?|abr\\.?|may\\.?|jun\\.?|jul\\.?|ago\\.?|sep\\.?|oct\\.?|nov\\.?|dic\\.?)/i,\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n        weekdays : 'domingo_lunes_martes_miércoles_jueves_viernes_sábado'.split('_'),\n        weekdaysShort : 'dom._lun._mar._mié._jue._vie._sáb.'.split('_'),\n        weekdaysMin : 'do_lu_ma_mi_ju_vi_sá'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'H:mm',\n            LTS : 'H:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D [de] MMMM [de] YYYY',\n            LLL : 'D [de] MMMM [de] YYYY H:mm',\n            LLLL : 'dddd, D [de] MMMM [de] YYYY H:mm'\n        },\n        calendar : {\n            sameDay : function () {\n                return '[hoy a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            nextDay : function () {\n                return '[mañana a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            nextWeek : function () {\n                return 'dddd [a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            lastDay : function () {\n                return '[ayer a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            lastWeek : function () {\n                return '[el] dddd [pasado a la' + ((this.hours() !== 1) ? 's' : '') + '] LT';\n            },\n            sameElse : 'L'\n        },\n        relativeTime : {\n            future : 'en %s',\n            past : 'hace %s',\n            s : 'unos segundos',\n            ss : '%d segundos',\n            m : 'un minuto',\n            mm : '%d minutos',\n            h : 'una hora',\n            hh : '%d horas',\n            d : 'un día',\n            dd : '%d días',\n            M : 'un mes',\n            MM : '%d meses',\n            y : 'un año',\n            yy : '%d años'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}º/,\n        ordinal : '%dº',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return es;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var fr = moment.defineLocale('fr', {\n        months : 'janvier_février_mars_avril_mai_juin_juillet_août_septembre_octobre_novembre_décembre'.split('_'),\n        monthsShort : 'janv._févr._mars_avr._mai_juin_juil._août_sept._oct._nov._déc.'.split('_'),\n        monthsParseExact : true,\n        weekdays : 'dimanche_lundi_mardi_mercredi_jeudi_vendredi_samedi'.split('_'),\n        weekdaysShort : 'dim._lun._mar._mer._jeu._ven._sam.'.split('_'),\n        weekdaysMin : 'di_lu_ma_me_je_ve_sa'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay : '[Aujourd’hui à] LT',\n            nextDay : '[Demain à] LT',\n            nextWeek : 'dddd [à] LT',\n            lastDay : '[Hier à] LT',\n            lastWeek : 'dddd [dernier à] LT',\n            sameElse : 'L'\n        },\n        relativeTime : {\n            future : 'dans %s',\n            past : 'il y a %s',\n            s : 'quelques secondes',\n            ss : '%d secondes',\n            m : 'une minute',\n            mm : '%d minutes',\n            h : 'une heure',\n            hh : '%d heures',\n            d : 'un jour',\n            dd : '%d jours',\n            M : 'un mois',\n            MM : '%d mois',\n            y : 'un an',\n            yy : '%d ans'\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}(er|)/,\n        ordinal : function (number, period) {\n            switch (period) {\n                // TODO: Return 'e' when day of month > 1. Move this case inside\n                // block for masculine words below.\n                // See https://github.com/moment/moment/issues/3375\n                case 'D':\n                    return number + (number === 1 ? 'er' : '');\n\n                // Words with masculine grammatical gender: mois, trimestre, jour\n                default:\n                case 'M':\n                case 'Q':\n                case 'DDD':\n                case 'd':\n                    return number + (number === 1 ? 'er' : 'e');\n\n                // Words with feminine grammatical gender: semaine\n                case 'w':\n                case 'W':\n                    return number + (number === 1 ? 're' : 'e');\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return fr;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var it = moment.defineLocale('it', {\n        months : 'gennaio_febbraio_marzo_aprile_maggio_giugno_luglio_agosto_settembre_ottobre_novembre_dicembre'.split('_'),\n        monthsShort : 'gen_feb_mar_apr_mag_giu_lug_ago_set_ott_nov_dic'.split('_'),\n        weekdays : 'domenica_lunedì_martedì_mercoledì_giovedì_venerdì_sabato'.split('_'),\n        weekdaysShort : 'dom_lun_mar_mer_gio_ven_sab'.split('_'),\n        weekdaysMin : 'do_lu_ma_me_gi_ve_sa'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD/MM/YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[Oggi alle] LT',\n            nextDay: '[Domani alle] LT',\n            nextWeek: 'dddd [alle] LT',\n            lastDay: '[Ieri alle] LT',\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[la scorsa] dddd [alle] LT';\n                    default:\n                        return '[lo scorso] dddd [alle] LT';\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : function (s) {\n                return ((/^[0-9].+$/).test(s) ? 'tra' : 'in') + ' ' + s;\n            },\n            past : '%s fa',\n            s : 'alcuni secondi',\n            ss : '%d secondi',\n            m : 'un minuto',\n            mm : '%d minuti',\n            h : 'un\\'ora',\n            hh : '%d ore',\n            d : 'un giorno',\n            dd : '%d giorni',\n            M : 'un mese',\n            MM : '%d mesi',\n            y : 'un anno',\n            yy : '%d anni'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}º/,\n        ordinal: '%dº',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return it;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var ja = moment.defineLocale('ja', {\n        months : '一月_二月_三月_四月_五月_六月_七月_八月_九月_十月_十一月_十二月'.split('_'),\n        monthsShort : '1月_2月_3月_4月_5月_6月_7月_8月_9月_10月_11月_12月'.split('_'),\n        weekdays : '日曜日_月曜日_火曜日_水曜日_木曜日_金曜日_土曜日'.split('_'),\n        weekdaysShort : '日_月_火_水_木_金_土'.split('_'),\n        weekdaysMin : '日_月_火_水_木_金_土'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'YYYY/MM/DD',\n            LL : 'YYYY年M月D日',\n            LLL : 'YYYY年M月D日 HH:mm',\n            LLLL : 'YYYY年M月D日 dddd HH:mm',\n            l : 'YYYY/MM/DD',\n            ll : 'YYYY年M月D日',\n            lll : 'YYYY年M月D日 HH:mm',\n            llll : 'YYYY年M月D日(ddd) HH:mm'\n        },\n        meridiemParse: /午前|午後/i,\n        isPM : function (input) {\n            return input === '午後';\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 12) {\n                return '午前';\n            } else {\n                return '午後';\n            }\n        },\n        calendar : {\n            sameDay : '[今日] LT',\n            nextDay : '[明日] LT',\n            nextWeek : function (now) {\n                if (now.week() < this.week()) {\n                    return '[来週]dddd LT';\n                } else {\n                    return 'dddd LT';\n                }\n            },\n            lastDay : '[昨日] LT',\n            lastWeek : function (now) {\n                if (this.week() < now.week()) {\n                    return '[先週]dddd LT';\n                } else {\n                    return 'dddd LT';\n                }\n            },\n            sameElse : 'L'\n        },\n        dayOfMonthOrdinalParse : /\\d{1,2}日/,\n        ordinal : function (number, period) {\n            switch (period) {\n                case 'd':\n                case 'D':\n                case 'DDD':\n                    return number + '日';\n                default:\n                    return number;\n            }\n        },\n        relativeTime : {\n            future : '%s後',\n            past : '%s前',\n            s : '数秒',\n            ss : '%d秒',\n            m : '1分',\n            mm : '%d分',\n            h : '1時間',\n            hh : '%d時間',\n            d : '1日',\n            dd : '%d日',\n            M : '1ヶ月',\n            MM : '%dヶ月',\n            y : '1年',\n            yy : '%d年'\n        }\n    });\n\n    return ja;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsShortWithDots = 'jan._feb._mrt._apr._mei_jun._jul._aug._sep._okt._nov._dec.'.split('_'),\n        monthsShortWithoutDots = 'jan_feb_mrt_apr_mei_jun_jul_aug_sep_okt_nov_dec'.split('_');\n\n    var monthsParse = [/^jan/i, /^feb/i, /^maart|mrt.?$/i, /^apr/i, /^mei$/i, /^jun[i.]?$/i, /^jul[i.]?$/i, /^aug/i, /^sep/i, /^okt/i, /^nov/i, /^dec/i];\n    var monthsRegex = /^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december|jan\\.?|feb\\.?|mrt\\.?|apr\\.?|ju[nl]\\.?|aug\\.?|sep\\.?|okt\\.?|nov\\.?|dec\\.?)/i;\n\n    var nl = moment.defineLocale('nl', {\n        months : 'januari_februari_maart_april_mei_juni_juli_augustus_september_oktober_november_december'.split('_'),\n        monthsShort : function (m, format) {\n            if (!m) {\n                return monthsShortWithDots;\n            } else if (/-MMM-/.test(format)) {\n                return monthsShortWithoutDots[m.month()];\n            } else {\n                return monthsShortWithDots[m.month()];\n            }\n        },\n\n        monthsRegex: monthsRegex,\n        monthsShortRegex: monthsRegex,\n        monthsStrictRegex: /^(januari|februari|maart|april|mei|ju[nl]i|augustus|september|oktober|november|december)/i,\n        monthsShortStrictRegex: /^(jan\\.?|feb\\.?|mrt\\.?|apr\\.?|mei|ju[nl]\\.?|aug\\.?|sep\\.?|okt\\.?|nov\\.?|dec\\.?)/i,\n\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n\n        weekdays : 'zondag_maandag_dinsdag_woensdag_donderdag_vrijdag_zaterdag'.split('_'),\n        weekdaysShort : 'zo._ma._di._wo._do._vr._za.'.split('_'),\n        weekdaysMin : 'zo_ma_di_wo_do_vr_za'.split('_'),\n        weekdaysParseExact : true,\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD-MM-YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[vandaag om] LT',\n            nextDay: '[morgen om] LT',\n            nextWeek: 'dddd [om] LT',\n            lastDay: '[gisteren om] LT',\n            lastWeek: '[afgelopen] dddd [om] LT',\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'over %s',\n            past : '%s geleden',\n            s : 'een paar seconden',\n            ss : '%d seconden',\n            m : 'één minuut',\n            mm : '%d minuten',\n            h : 'één uur',\n            hh : '%d uur',\n            d : 'één dag',\n            dd : '%d dagen',\n            M : 'één maand',\n            MM : '%d maanden',\n            y : 'één jaar',\n            yy : '%d jaar'\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}(ste|de)/,\n        ordinal : function (number) {\n            return number + ((number === 1 || number === 8 || number >= 20) ? 'ste' : 'de');\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return nl;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    var monthsNominative = 'styczeń_luty_marzec_kwiecień_maj_czerwiec_lipiec_sierpień_wrzesień_październik_listopad_grudzień'.split('_'),\n        monthsSubjective = 'stycznia_lutego_marca_kwietnia_maja_czerwca_lipca_sierpnia_września_października_listopada_grudnia'.split('_');\n    function plural(n) {\n        return (n % 10 < 5) && (n % 10 > 1) && ((~~(n / 10) % 10) !== 1);\n    }\n    function translate(number, withoutSuffix, key) {\n        var result = number + ' ';\n        switch (key) {\n            case 'ss':\n                return result + (plural(number) ? 'sekundy' : 'sekund');\n            case 'm':\n                return withoutSuffix ? 'minuta' : 'minutę';\n            case 'mm':\n                return result + (plural(number) ? 'minuty' : 'minut');\n            case 'h':\n                return withoutSuffix  ? 'godzina'  : 'godzinę';\n            case 'hh':\n                return result + (plural(number) ? 'godziny' : 'godzin');\n            case 'MM':\n                return result + (plural(number) ? 'miesiące' : 'miesięcy');\n            case 'yy':\n                return result + (plural(number) ? 'lata' : 'lat');\n        }\n    }\n\n    var pl = moment.defineLocale('pl', {\n        months : function (momentToFormat, format) {\n            if (!momentToFormat) {\n                return monthsNominative;\n            } else if (format === '') {\n                // Hack: if format empty we know this is used to generate\n                // RegExp by moment. Give then back both valid forms of months\n                // in RegExp ready format.\n                return '(' + monthsSubjective[momentToFormat.month()] + '|' + monthsNominative[momentToFormat.month()] + ')';\n            } else if (/D MMMM/.test(format)) {\n                return monthsSubjective[momentToFormat.month()];\n            } else {\n                return monthsNominative[momentToFormat.month()];\n            }\n        },\n        monthsShort : 'sty_lut_mar_kwi_maj_cze_lip_sie_wrz_paź_lis_gru'.split('_'),\n        weekdays : 'niedziela_poniedziałek_wtorek_środa_czwartek_piątek_sobota'.split('_'),\n        weekdaysShort : 'ndz_pon_wt_śr_czw_pt_sob'.split('_'),\n        weekdaysMin : 'Nd_Pn_Wt_Śr_Cz_Pt_So'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY',\n            LLL : 'D MMMM YYYY HH:mm',\n            LLLL : 'dddd, D MMMM YYYY HH:mm'\n        },\n        calendar : {\n            sameDay: '[Dziś o] LT',\n            nextDay: '[Jutro o] LT',\n            nextWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[W niedzielę o] LT';\n\n                    case 2:\n                        return '[We wtorek o] LT';\n\n                    case 3:\n                        return '[W środę o] LT';\n\n                    case 6:\n                        return '[W sobotę o] LT';\n\n                    default:\n                        return '[W] dddd [o] LT';\n                }\n            },\n            lastDay: '[Wczoraj o] LT',\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                        return '[W zeszłą niedzielę o] LT';\n                    case 3:\n                        return '[W zeszłą środę o] LT';\n                    case 6:\n                        return '[W zeszłą sobotę o] LT';\n                    default:\n                        return '[W zeszły] dddd [o] LT';\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'za %s',\n            past : '%s temu',\n            s : 'kilka sekund',\n            ss : translate,\n            m : translate,\n            mm : translate,\n            h : translate,\n            hh : translate,\n            d : '1 dzień',\n            dd : '%d dni',\n            M : 'miesiąc',\n            MM : translate,\n            y : 'rok',\n            yy : translate\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}\\./,\n        ordinal : '%d.',\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return pl;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function plural(word, num) {\n        var forms = word.split('_');\n        return num % 10 === 1 && num % 100 !== 11 ? forms[0] : (num % 10 >= 2 && num % 10 <= 4 && (num % 100 < 10 || num % 100 >= 20) ? forms[1] : forms[2]);\n    }\n    function relativeTimeWithPlural(number, withoutSuffix, key) {\n        var format = {\n            'ss': withoutSuffix ? 'секунда_секунды_секунд' : 'секунду_секунды_секунд',\n            'mm': withoutSuffix ? 'минута_минуты_минут' : 'минуту_минуты_минут',\n            'hh': 'час_часа_часов',\n            'dd': 'день_дня_дней',\n            'MM': 'месяц_месяца_месяцев',\n            'yy': 'год_года_лет'\n        };\n        if (key === 'm') {\n            return withoutSuffix ? 'минута' : 'минуту';\n        }\n        else {\n            return number + ' ' + plural(format[key], +number);\n        }\n    }\n    var monthsParse = [/^янв/i, /^фев/i, /^мар/i, /^апр/i, /^ма[йя]/i, /^июн/i, /^июл/i, /^авг/i, /^сен/i, /^окт/i, /^ноя/i, /^дек/i];\n\n    // http://new.gramota.ru/spravka/rules/139-prop : § 103\n    // Сокращения месяцев: http://new.gramota.ru/spravka/buro/search-answer?s=242637\n    // CLDR data:          http://www.unicode.org/cldr/charts/28/summary/ru.html#1753\n    var ru = moment.defineLocale('ru', {\n        months : {\n            format: 'января_февраля_марта_апреля_мая_июня_июля_августа_сентября_октября_ноября_декабря'.split('_'),\n            standalone: 'январь_февраль_март_апрель_май_июнь_июль_август_сентябрь_октябрь_ноябрь_декабрь'.split('_')\n        },\n        monthsShort : {\n            // по CLDR именно \"июл.\" и \"июн.\", но какой смысл менять букву на точку ?\n            format: 'янв._февр._мар._апр._мая_июня_июля_авг._сент._окт._нояб._дек.'.split('_'),\n            standalone: 'янв._февр._март_апр._май_июнь_июль_авг._сент._окт._нояб._дек.'.split('_')\n        },\n        weekdays : {\n            standalone: 'воскресенье_понедельник_вторник_среда_четверг_пятница_суббота'.split('_'),\n            format: 'воскресенье_понедельник_вторник_среду_четверг_пятницу_субботу'.split('_'),\n            isFormat: /\\[ ?[Вв] ?(?:прошлую|следующую|эту)? ?\\] ?dddd/\n        },\n        weekdaysShort : 'вс_пн_вт_ср_чт_пт_сб'.split('_'),\n        weekdaysMin : 'вс_пн_вт_ср_чт_пт_сб'.split('_'),\n        monthsParse : monthsParse,\n        longMonthsParse : monthsParse,\n        shortMonthsParse : monthsParse,\n\n        // полные названия с падежами, по три буквы, для некоторых, по 4 буквы, сокращения с точкой и без точки\n        monthsRegex: /^(январ[ья]|янв\\.?|феврал[ья]|февр?\\.?|марта?|мар\\.?|апрел[ья]|апр\\.?|ма[йя]|июн[ья]|июн\\.?|июл[ья]|июл\\.?|августа?|авг\\.?|сентябр[ья]|сент?\\.?|октябр[ья]|окт\\.?|ноябр[ья]|нояб?\\.?|декабр[ья]|дек\\.?)/i,\n\n        // копия предыдущего\n        monthsShortRegex: /^(январ[ья]|янв\\.?|феврал[ья]|февр?\\.?|марта?|мар\\.?|апрел[ья]|апр\\.?|ма[йя]|июн[ья]|июн\\.?|июл[ья]|июл\\.?|августа?|авг\\.?|сентябр[ья]|сент?\\.?|октябр[ья]|окт\\.?|ноябр[ья]|нояб?\\.?|декабр[ья]|дек\\.?)/i,\n\n        // полные названия с падежами\n        monthsStrictRegex: /^(январ[яь]|феврал[яь]|марта?|апрел[яь]|ма[яй]|июн[яь]|июл[яь]|августа?|сентябр[яь]|октябр[яь]|ноябр[яь]|декабр[яь])/i,\n\n        // Выражение, которое соотвествует только сокращённым формам\n        monthsShortStrictRegex: /^(янв\\.|февр?\\.|мар[т.]|апр\\.|ма[яй]|июн[ья.]|июл[ья.]|авг\\.|сент?\\.|окт\\.|нояб?\\.|дек\\.)/i,\n        longDateFormat : {\n            LT : 'H:mm',\n            LTS : 'H:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY г.',\n            LLL : 'D MMMM YYYY г., H:mm',\n            LLLL : 'dddd, D MMMM YYYY г., H:mm'\n        },\n        calendar : {\n            sameDay: '[Сегодня, в] LT',\n            nextDay: '[Завтра, в] LT',\n            lastDay: '[Вчера, в] LT',\n            nextWeek: function (now) {\n                if (now.week() !== this.week()) {\n                    switch (this.day()) {\n                        case 0:\n                            return '[В следующее] dddd, [в] LT';\n                        case 1:\n                        case 2:\n                        case 4:\n                            return '[В следующий] dddd, [в] LT';\n                        case 3:\n                        case 5:\n                        case 6:\n                            return '[В следующую] dddd, [в] LT';\n                    }\n                } else {\n                    if (this.day() === 2) {\n                        return '[Во] dddd, [в] LT';\n                    } else {\n                        return '[В] dddd, [в] LT';\n                    }\n                }\n            },\n            lastWeek: function (now) {\n                if (now.week() !== this.week()) {\n                    switch (this.day()) {\n                        case 0:\n                            return '[В прошлое] dddd, [в] LT';\n                        case 1:\n                        case 2:\n                        case 4:\n                            return '[В прошлый] dddd, [в] LT';\n                        case 3:\n                        case 5:\n                        case 6:\n                            return '[В прошлую] dddd, [в] LT';\n                    }\n                } else {\n                    if (this.day() === 2) {\n                        return '[Во] dddd, [в] LT';\n                    } else {\n                        return '[В] dddd, [в] LT';\n                    }\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'через %s',\n            past : '%s назад',\n            s : 'несколько секунд',\n            ss : relativeTimeWithPlural,\n            m : relativeTimeWithPlural,\n            mm : relativeTimeWithPlural,\n            h : 'час',\n            hh : relativeTimeWithPlural,\n            d : 'день',\n            dd : relativeTimeWithPlural,\n            M : 'месяц',\n            MM : relativeTimeWithPlural,\n            y : 'год',\n            yy : relativeTimeWithPlural\n        },\n        meridiemParse: /ночи|утра|дня|вечера/i,\n        isPM : function (input) {\n            return /^(дня|вечера)$/.test(input);\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 4) {\n                return 'ночи';\n            } else if (hour < 12) {\n                return 'утра';\n            } else if (hour < 17) {\n                return 'дня';\n            } else {\n                return 'вечера';\n            }\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}-(й|го|я)/,\n        ordinal: function (number, period) {\n            switch (period) {\n                case 'M':\n                case 'd':\n                case 'DDD':\n                    return number + '-й';\n                case 'D':\n                    return number + '-го';\n                case 'w':\n                case 'W':\n                    return number + '-я';\n                default:\n                    return number;\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 4  // The week that contains Jan 4th is the first week of the year.\n        }\n    });\n\n    return ru;\n\n})));\n","//! moment.js locale configuration\n\n;(function (global, factory) {\n   typeof exports === 'object' && typeof module !== 'undefined'\n       && typeof require === 'function' ? factory(require('../moment')) :\n   typeof define === 'function' && define.amd ? define(['../moment'], factory) :\n   factory(global.moment)\n}(this, (function (moment) { 'use strict';\n\n\n    function plural(word, num) {\n        var forms = word.split('_');\n        return num % 10 === 1 && num % 100 !== 11 ? forms[0] : (num % 10 >= 2 && num % 10 <= 4 && (num % 100 < 10 || num % 100 >= 20) ? forms[1] : forms[2]);\n    }\n    function relativeTimeWithPlural(number, withoutSuffix, key) {\n        var format = {\n            'ss': withoutSuffix ? 'секунда_секунди_секунд' : 'секунду_секунди_секунд',\n            'mm': withoutSuffix ? 'хвилина_хвилини_хвилин' : 'хвилину_хвилини_хвилин',\n            'hh': withoutSuffix ? 'година_години_годин' : 'годину_години_годин',\n            'dd': 'день_дні_днів',\n            'MM': 'місяць_місяці_місяців',\n            'yy': 'рік_роки_років'\n        };\n        if (key === 'm') {\n            return withoutSuffix ? 'хвилина' : 'хвилину';\n        }\n        else if (key === 'h') {\n            return withoutSuffix ? 'година' : 'годину';\n        }\n        else {\n            return number + ' ' + plural(format[key], +number);\n        }\n    }\n    function weekdaysCaseReplace(m, format) {\n        var weekdays = {\n            'nominative': 'неділя_понеділок_вівторок_середа_четвер_п’ятниця_субота'.split('_'),\n            'accusative': 'неділю_понеділок_вівторок_середу_четвер_п’ятницю_суботу'.split('_'),\n            'genitive': 'неділі_понеділка_вівторка_середи_четверга_п’ятниці_суботи'.split('_')\n        };\n\n        if (m === true) {\n            return weekdays['nominative'].slice(1, 7).concat(weekdays['nominative'].slice(0, 1));\n        }\n        if (!m) {\n            return weekdays['nominative'];\n        }\n\n        var nounCase = (/(\\[[ВвУу]\\]) ?dddd/).test(format) ?\n            'accusative' :\n            ((/\\[?(?:минулої|наступної)? ?\\] ?dddd/).test(format) ?\n                'genitive' :\n                'nominative');\n        return weekdays[nounCase][m.day()];\n    }\n    function processHoursFunction(str) {\n        return function () {\n            return str + 'о' + (this.hours() === 11 ? 'б' : '') + '] LT';\n        };\n    }\n\n    var uk = moment.defineLocale('uk', {\n        months : {\n            'format': 'січня_лютого_березня_квітня_травня_червня_липня_серпня_вересня_жовтня_листопада_грудня'.split('_'),\n            'standalone': 'січень_лютий_березень_квітень_травень_червень_липень_серпень_вересень_жовтень_листопад_грудень'.split('_')\n        },\n        monthsShort : 'січ_лют_бер_квіт_трав_черв_лип_серп_вер_жовт_лист_груд'.split('_'),\n        weekdays : weekdaysCaseReplace,\n        weekdaysShort : 'нд_пн_вт_ср_чт_пт_сб'.split('_'),\n        weekdaysMin : 'нд_пн_вт_ср_чт_пт_сб'.split('_'),\n        longDateFormat : {\n            LT : 'HH:mm',\n            LTS : 'HH:mm:ss',\n            L : 'DD.MM.YYYY',\n            LL : 'D MMMM YYYY р.',\n            LLL : 'D MMMM YYYY р., HH:mm',\n            LLLL : 'dddd, D MMMM YYYY р., HH:mm'\n        },\n        calendar : {\n            sameDay: processHoursFunction('[Сьогодні '),\n            nextDay: processHoursFunction('[Завтра '),\n            lastDay: processHoursFunction('[Вчора '),\n            nextWeek: processHoursFunction('[У] dddd ['),\n            lastWeek: function () {\n                switch (this.day()) {\n                    case 0:\n                    case 3:\n                    case 5:\n                    case 6:\n                        return processHoursFunction('[Минулої] dddd [').call(this);\n                    case 1:\n                    case 2:\n                    case 4:\n                        return processHoursFunction('[Минулого] dddd [').call(this);\n                }\n            },\n            sameElse: 'L'\n        },\n        relativeTime : {\n            future : 'за %s',\n            past : '%s тому',\n            s : 'декілька секунд',\n            ss : relativeTimeWithPlural,\n            m : relativeTimeWithPlural,\n            mm : relativeTimeWithPlural,\n            h : 'годину',\n            hh : relativeTimeWithPlural,\n            d : 'день',\n            dd : relativeTimeWithPlural,\n            M : 'місяць',\n            MM : relativeTimeWithPlural,\n            y : 'рік',\n            yy : relativeTimeWithPlural\n        },\n        // M. E.: those two are virtually unused but a user might want to implement them for his/her website for some reason\n        meridiemParse: /ночі|ранку|дня|вечора/,\n        isPM: function (input) {\n            return /^(дня|вечора)$/.test(input);\n        },\n        meridiem : function (hour, minute, isLower) {\n            if (hour < 4) {\n                return 'ночі';\n            } else if (hour < 12) {\n                return 'ранку';\n            } else if (hour < 17) {\n                return 'дня';\n            } else {\n                return 'вечора';\n            }\n        },\n        dayOfMonthOrdinalParse: /\\d{1,2}-(й|го)/,\n        ordinal: function (number, period) {\n            switch (period) {\n                case 'M':\n                case 'd':\n                case 'DDD':\n                case 'w':\n                case 'W':\n                    return number + '-й';\n                case 'D':\n                    return number + '-го';\n                default:\n                    return number;\n            }\n        },\n        week : {\n            dow : 1, // Monday is the first day of the week.\n            doy : 7  // The week that contains Jan 7th is the first week of the year.\n        }\n    });\n\n    return uk;\n\n})));\n","module.exports = function (exec) {\n  try {\n    return !!exec();\n  } catch (error) {\n    return true;\n  }\n};\n","var fails = require('../internals/fails');\n\n// Thank's IE8 for his funny defineProperty\nmodule.exports = !fails(function () {\n  return Object.defineProperty({}, 1, { get: function () { return 7; } })[1] != 7;\n});\n","'use strict';\nvar nativePropertyIsEnumerable = {}.propertyIsEnumerable;\nvar getOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;\n\n// Nashorn ~ JDK8 bug\nvar NASHORN_BUG = getOwnPropertyDescriptor && !nativePropertyIsEnumerable.call({ 1: 2 }, 1);\n\n// `Object.prototype.propertyIsEnumerable` method implementation\n// https://tc39.github.io/ecma262/#sec-object.prototype.propertyisenumerable\nexports.f = NASHORN_BUG ? function propertyIsEnumerable(V) {\n  var descriptor = getOwnPropertyDescriptor(this, V);\n  return !!descriptor && descriptor.enumerable;\n} : nativePropertyIsEnumerable;\n","module.exports = function (bitmap, value) {\n  return {\n    enumerable: !(bitmap & 1),\n    configurable: !(bitmap & 2),\n    writable: !(bitmap & 4),\n    value: value\n  };\n};\n","var toString = {}.toString;\n\nmodule.exports = function (it) {\n  return toString.call(it).slice(8, -1);\n};\n","var fails = require('../internals/fails');\nvar classof = require('../internals/classof-raw');\n\nvar split = ''.split;\n\n// fallback for non-array-like ES3 and non-enumerable old V8 strings\nmodule.exports = fails(function () {\n  // throws an error in rhino, see https://github.com/mozilla/rhino/issues/346\n  // eslint-disable-next-line no-prototype-builtins\n  return !Object('z').propertyIsEnumerable(0);\n}) ? function (it) {\n  return classof(it) == 'String' ? split.call(it, '') : Object(it);\n} : Object;\n","// `RequireObjectCoercible` abstract operation\n// https://tc39.github.io/ecma262/#sec-requireobjectcoercible\nmodule.exports = function (it) {\n  if (it == undefined) throw TypeError(\"Can't call method on \" + it);\n  return it;\n};\n","// toObject with fallback for non-array-like ES3 strings\nvar IndexedObject = require('../internals/indexed-object');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\nmodule.exports = function (it) {\n  return IndexedObject(requireObjectCoercible(it));\n};\n","module.exports = function (it) {\n  return typeof it === 'object' ? it !== null : typeof it === 'function';\n};\n","var isObject = require('../internals/is-object');\n\n// `ToPrimitive` abstract operation\n// https://tc39.github.io/ecma262/#sec-toprimitive\n// instead of the ES6 spec version, we didn't implement @@toPrimitive case\n// and the second argument - flag - preferred type is a string\nmodule.exports = function (input, PREFERRED_STRING) {\n  if (!isObject(input)) return input;\n  var fn, val;\n  if (PREFERRED_STRING && typeof (fn = input.toString) == 'function' && !isObject(val = fn.call(input))) return val;\n  if (typeof (fn = input.valueOf) == 'function' && !isObject(val = fn.call(input))) return val;\n  if (!PREFERRED_STRING && typeof (fn = input.toString) == 'function' && !isObject(val = fn.call(input))) return val;\n  throw TypeError(\"Can't convert object to primitive value\");\n};\n","var hasOwnProperty = {}.hasOwnProperty;\n\nmodule.exports = function (it, key) {\n  return hasOwnProperty.call(it, key);\n};\n","var global = require('../internals/global');\nvar isObject = require('../internals/is-object');\n\nvar document = global.document;\n// typeof document.createElement is 'object' in old IE\nvar EXISTS = isObject(document) && isObject(document.createElement);\n\nmodule.exports = function (it) {\n  return EXISTS ? document.createElement(it) : {};\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar createElement = require('../internals/document-create-element');\n\n// Thank's IE8 for his funny defineProperty\nmodule.exports = !DESCRIPTORS && !fails(function () {\n  return Object.defineProperty(createElement('div'), 'a', {\n    get: function () { return 7; }\n  }).a != 7;\n});\n","var DESCRIPTORS = require('../internals/descriptors');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar toPrimitive = require('../internals/to-primitive');\nvar has = require('../internals/has');\nvar IE8_DOM_DEFINE = require('../internals/ie8-dom-define');\n\nvar nativeGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;\n\n// `Object.getOwnPropertyDescriptor` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptor\nexports.f = DESCRIPTORS ? nativeGetOwnPropertyDescriptor : function getOwnPropertyDescriptor(O, P) {\n  O = toIndexedObject(O);\n  P = toPrimitive(P, true);\n  if (IE8_DOM_DEFINE) try {\n    return nativeGetOwnPropertyDescriptor(O, P);\n  } catch (error) { /* empty */ }\n  if (has(O, P)) return createPropertyDescriptor(!propertyIsEnumerableModule.f.call(O, P), O[P]);\n};\n","var fails = require('../internals/fails');\n\nvar replacement = /#|\\.prototype\\./;\n\nvar isForced = function (feature, detection) {\n  var value = data[normalize(feature)];\n  return value == POLYFILL ? true\n    : value == NATIVE ? false\n    : typeof detection == 'function' ? fails(detection)\n    : !!detection;\n};\n\nvar normalize = isForced.normalize = function (string) {\n  return String(string).replace(replacement, '.').toLowerCase();\n};\n\nvar data = isForced.data = {};\nvar NATIVE = isForced.NATIVE = 'N';\nvar POLYFILL = isForced.POLYFILL = 'P';\n\nmodule.exports = isForced;\n","module.exports = {};\n","module.exports = function (it) {\n  if (typeof it != 'function') {\n    throw TypeError(String(it) + ' is not a function');\n  } return it;\n};\n","var aFunction = require('../internals/a-function');\n\n// optional / simple context binding\nmodule.exports = function (fn, that, length) {\n  aFunction(fn);\n  if (that === undefined) return fn;\n  switch (length) {\n    case 0: return function () {\n      return fn.call(that);\n    };\n    case 1: return function (a) {\n      return fn.call(that, a);\n    };\n    case 2: return function (a, b) {\n      return fn.call(that, a, b);\n    };\n    case 3: return function (a, b, c) {\n      return fn.call(that, a, b, c);\n    };\n  }\n  return function (/* ...args */) {\n    return fn.apply(that, arguments);\n  };\n};\n","var isObject = require('../internals/is-object');\n\nmodule.exports = function (it) {\n  if (!isObject(it)) {\n    throw TypeError(String(it) + ' is not an object');\n  } return it;\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar IE8_DOM_DEFINE = require('../internals/ie8-dom-define');\nvar anObject = require('../internals/an-object');\nvar toPrimitive = require('../internals/to-primitive');\n\nvar nativeDefineProperty = Object.defineProperty;\n\n// `Object.defineProperty` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperty\nexports.f = DESCRIPTORS ? nativeDefineProperty : function defineProperty(O, P, Attributes) {\n  anObject(O);\n  P = toPrimitive(P, true);\n  anObject(Attributes);\n  if (IE8_DOM_DEFINE) try {\n    return nativeDefineProperty(O, P, Attributes);\n  } catch (error) { /* empty */ }\n  if ('get' in Attributes || 'set' in Attributes) throw TypeError('Accessors not supported');\n  if ('value' in Attributes) O[P] = Attributes.value;\n  return O;\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar definePropertyModule = require('../internals/object-define-property');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\n\nmodule.exports = DESCRIPTORS ? function (object, key, value) {\n  return definePropertyModule.f(object, key, createPropertyDescriptor(1, value));\n} : function (object, key, value) {\n  object[key] = value;\n  return object;\n};\n","'use strict';\nvar global = require('../internals/global');\nvar getOwnPropertyDescriptor = require('../internals/object-get-own-property-descriptor').f;\nvar isForced = require('../internals/is-forced');\nvar path = require('../internals/path');\nvar bind = require('../internals/function-bind-context');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\n\nvar wrapConstructor = function (NativeConstructor) {\n  var Wrapper = function (a, b, c) {\n    if (this instanceof NativeConstructor) {\n      switch (arguments.length) {\n        case 0: return new NativeConstructor();\n        case 1: return new NativeConstructor(a);\n        case 2: return new NativeConstructor(a, b);\n      } return new NativeConstructor(a, b, c);\n    } return NativeConstructor.apply(this, arguments);\n  };\n  Wrapper.prototype = NativeConstructor.prototype;\n  return Wrapper;\n};\n\n/*\n  options.target      - name of the target object\n  options.global      - target is the global object\n  options.stat        - export as static methods of target\n  options.proto       - export as prototype methods of target\n  options.real        - real prototype method for the `pure` version\n  options.forced      - export even if the native feature is available\n  options.bind        - bind methods to the target, required for the `pure` version\n  options.wrap        - wrap constructors to preventing global pollution, required for the `pure` version\n  options.unsafe      - use the simple assignment of property instead of delete + defineProperty\n  options.sham        - add a flag to not completely full polyfills\n  options.enumerable  - export as enumerable property\n  options.noTargetGet - prevent calling a getter on target\n*/\nmodule.exports = function (options, source) {\n  var TARGET = options.target;\n  var GLOBAL = options.global;\n  var STATIC = options.stat;\n  var PROTO = options.proto;\n\n  var nativeSource = GLOBAL ? global : STATIC ? global[TARGET] : (global[TARGET] || {}).prototype;\n\n  var target = GLOBAL ? path : path[TARGET] || (path[TARGET] = {});\n  var targetPrototype = target.prototype;\n\n  var FORCED, USE_NATIVE, VIRTUAL_PROTOTYPE;\n  var key, sourceProperty, targetProperty, nativeProperty, resultProperty, descriptor;\n\n  for (key in source) {\n    FORCED = isForced(GLOBAL ? key : TARGET + (STATIC ? '.' : '#') + key, options.forced);\n    // contains in native\n    USE_NATIVE = !FORCED && nativeSource && has(nativeSource, key);\n\n    targetProperty = target[key];\n\n    if (USE_NATIVE) if (options.noTargetGet) {\n      descriptor = getOwnPropertyDescriptor(nativeSource, key);\n      nativeProperty = descriptor && descriptor.value;\n    } else nativeProperty = nativeSource[key];\n\n    // export native or implementation\n    sourceProperty = (USE_NATIVE && nativeProperty) ? nativeProperty : source[key];\n\n    if (USE_NATIVE && typeof targetProperty === typeof sourceProperty) continue;\n\n    // bind timers to global for call from export context\n    if (options.bind && USE_NATIVE) resultProperty = bind(sourceProperty, global);\n    // wrap global constructors for prevent changs in this version\n    else if (options.wrap && USE_NATIVE) resultProperty = wrapConstructor(sourceProperty);\n    // make static versions for prototype methods\n    else if (PROTO && typeof sourceProperty == 'function') resultProperty = bind(Function.call, sourceProperty);\n    // default case\n    else resultProperty = sourceProperty;\n\n    // add a flag to not completely full polyfills\n    if (options.sham || (sourceProperty && sourceProperty.sham) || (targetProperty && targetProperty.sham)) {\n      createNonEnumerableProperty(resultProperty, 'sham', true);\n    }\n\n    target[key] = resultProperty;\n\n    if (PROTO) {\n      VIRTUAL_PROTOTYPE = TARGET + 'Prototype';\n      if (!has(path, VIRTUAL_PROTOTYPE)) {\n        createNonEnumerableProperty(path, VIRTUAL_PROTOTYPE, {});\n      }\n      // export virtual prototype methods\n      path[VIRTUAL_PROTOTYPE][key] = sourceProperty;\n      // export real prototype methods\n      if (options.real && targetPrototype && !targetPrototype[key]) {\n        createNonEnumerableProperty(targetPrototype, key, sourceProperty);\n      }\n    }\n  }\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar objectDefinePropertyModile = require('../internals/object-define-property');\n\n// `Object.defineProperty` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperty\n$({ target: 'Object', stat: true, forced: !DESCRIPTORS, sham: !DESCRIPTORS }, {\n  defineProperty: objectDefinePropertyModile.f\n});\n","require('../../modules/es.object.define-property');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar defineProperty = module.exports = function defineProperty(it, key, desc) {\n  return Object.defineProperty(it, key, desc);\n};\n\nif (Object.defineProperty.sham) defineProperty.sham = true;\n","var parent = require('../../es/object/define-property');\n\nmodule.exports = parent;\n","var ceil = Math.ceil;\nvar floor = Math.floor;\n\n// `ToInteger` abstract operation\n// https://tc39.github.io/ecma262/#sec-tointeger\nmodule.exports = function (argument) {\n  return isNaN(argument = +argument) ? 0 : (argument > 0 ? floor : ceil)(argument);\n};\n","var toInteger = require('../internals/to-integer');\n\nvar min = Math.min;\n\n// `ToLength` abstract operation\n// https://tc39.github.io/ecma262/#sec-tolength\nmodule.exports = function (argument) {\n  return argument > 0 ? min(toInteger(argument), 0x1FFFFFFFFFFFFF) : 0; // 2 ** 53 - 1 == 9007199254740991\n};\n","var toInteger = require('../internals/to-integer');\n\nvar max = Math.max;\nvar min = Math.min;\n\n// Helper for a popular repeating case of the spec:\n// Let integer be ? ToInteger(index).\n// If integer < 0, let result be max((length + integer), 0); else let result be min(integer, length).\nmodule.exports = function (index, length) {\n  var integer = toInteger(index);\n  return integer < 0 ? max(integer + length, 0) : min(integer, length);\n};\n","var toIndexedObject = require('../internals/to-indexed-object');\nvar toLength = require('../internals/to-length');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\n\n// `Array.prototype.{ indexOf, includes }` methods implementation\nvar createMethod = function (IS_INCLUDES) {\n  return function ($this, el, fromIndex) {\n    var O = toIndexedObject($this);\n    var length = toLength(O.length);\n    var index = toAbsoluteIndex(fromIndex, length);\n    var value;\n    // Array#includes uses SameValueZero equality algorithm\n    // eslint-disable-next-line no-self-compare\n    if (IS_INCLUDES && el != el) while (length > index) {\n      value = O[index++];\n      // eslint-disable-next-line no-self-compare\n      if (value != value) return true;\n    // Array#indexOf ignores holes, Array#includes - not\n    } else for (;length > index; index++) {\n      if ((IS_INCLUDES || index in O) && O[index] === el) return IS_INCLUDES || index || 0;\n    } return !IS_INCLUDES && -1;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.includes` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.includes\n  includes: createMethod(true),\n  // `Array.prototype.indexOf` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.indexof\n  indexOf: createMethod(false)\n};\n","module.exports = {};\n","var has = require('../internals/has');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar indexOf = require('../internals/array-includes').indexOf;\nvar hiddenKeys = require('../internals/hidden-keys');\n\nmodule.exports = function (object, names) {\n  var O = toIndexedObject(object);\n  var i = 0;\n  var result = [];\n  var key;\n  for (key in O) !has(hiddenKeys, key) && has(O, key) && result.push(key);\n  // Don't enum bug & hidden keys\n  while (names.length > i) if (has(O, key = names[i++])) {\n    ~indexOf(result, key) || result.push(key);\n  }\n  return result;\n};\n","// IE8- don't enum bug keys\nmodule.exports = [\n  'constructor',\n  'hasOwnProperty',\n  'isPrototypeOf',\n  'propertyIsEnumerable',\n  'toLocaleString',\n  'toString',\n  'valueOf'\n];\n","var internalObjectKeys = require('../internals/object-keys-internal');\nvar enumBugKeys = require('../internals/enum-bug-keys');\n\n// `Object.keys` method\n// https://tc39.github.io/ecma262/#sec-object.keys\nmodule.exports = Object.keys || function keys(O) {\n  return internalObjectKeys(O, enumBugKeys);\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar definePropertyModule = require('../internals/object-define-property');\nvar anObject = require('../internals/an-object');\nvar objectKeys = require('../internals/object-keys');\n\n// `Object.defineProperties` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperties\nmodule.exports = DESCRIPTORS ? Object.defineProperties : function defineProperties(O, Properties) {\n  anObject(O);\n  var keys = objectKeys(Properties);\n  var length = keys.length;\n  var index = 0;\n  var key;\n  while (length > index) definePropertyModule.f(O, key = keys[index++], Properties[key]);\n  return O;\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar defineProperties = require('../internals/object-define-properties');\n\n// `Object.defineProperties` method\n// https://tc39.github.io/ecma262/#sec-object.defineproperties\n$({ target: 'Object', stat: true, forced: !DESCRIPTORS, sham: !DESCRIPTORS }, {\n  defineProperties: defineProperties\n});\n","require('../../modules/es.object.define-properties');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar defineProperties = module.exports = function defineProperties(T, D) {\n  return Object.defineProperties(T, D);\n};\n\nif (Object.defineProperties.sham) defineProperties.sham = true;\n","var path = require('../internals/path');\nvar global = require('../internals/global');\n\nvar aFunction = function (variable) {\n  return typeof variable == 'function' ? variable : undefined;\n};\n\nmodule.exports = function (namespace, method) {\n  return arguments.length < 2 ? aFunction(path[namespace]) || aFunction(global[namespace])\n    : path[namespace] && path[namespace][method] || global[namespace] && global[namespace][method];\n};\n","var internalObjectKeys = require('../internals/object-keys-internal');\nvar enumBugKeys = require('../internals/enum-bug-keys');\n\nvar hiddenKeys = enumBugKeys.concat('length', 'prototype');\n\n// `Object.getOwnPropertyNames` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertynames\nexports.f = Object.getOwnPropertyNames || function getOwnPropertyNames(O) {\n  return internalObjectKeys(O, hiddenKeys);\n};\n","exports.f = Object.getOwnPropertySymbols;\n","var getBuiltIn = require('../internals/get-built-in');\nvar getOwnPropertyNamesModule = require('../internals/object-get-own-property-names');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar anObject = require('../internals/an-object');\n\n// all object keys, includes non-enumerable and symbols\nmodule.exports = getBuiltIn('Reflect', 'ownKeys') || function ownKeys(it) {\n  var keys = getOwnPropertyNamesModule.f(anObject(it));\n  var getOwnPropertySymbols = getOwnPropertySymbolsModule.f;\n  return getOwnPropertySymbols ? keys.concat(getOwnPropertySymbols(it)) : keys;\n};\n","'use strict';\nvar toPrimitive = require('../internals/to-primitive');\nvar definePropertyModule = require('../internals/object-define-property');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\n\nmodule.exports = function (object, key, value) {\n  var propertyKey = toPrimitive(key);\n  if (propertyKey in object) definePropertyModule.f(object, propertyKey, createPropertyDescriptor(0, value));\n  else object[propertyKey] = value;\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar ownKeys = require('../internals/own-keys');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar getOwnPropertyDescriptorModule = require('../internals/object-get-own-property-descriptor');\nvar createProperty = require('../internals/create-property');\n\n// `Object.getOwnPropertyDescriptors` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptors\n$({ target: 'Object', stat: true, sham: !DESCRIPTORS }, {\n  getOwnPropertyDescriptors: function getOwnPropertyDescriptors(object) {\n    var O = toIndexedObject(object);\n    var getOwnPropertyDescriptor = getOwnPropertyDescriptorModule.f;\n    var keys = ownKeys(O);\n    var result = {};\n    var index = 0;\n    var key, descriptor;\n    while (keys.length > index) {\n      descriptor = getOwnPropertyDescriptor(O, key = keys[index++]);\n      if (descriptor !== undefined) createProperty(result, key, descriptor);\n    }\n    return result;\n  }\n});\n","require('../../modules/es.object.get-own-property-descriptors');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getOwnPropertyDescriptors;\n","var $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar nativeGetOwnPropertyDescriptor = require('../internals/object-get-own-property-descriptor').f;\nvar DESCRIPTORS = require('../internals/descriptors');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeGetOwnPropertyDescriptor(1); });\nvar FORCED = !DESCRIPTORS || FAILS_ON_PRIMITIVES;\n\n// `Object.getOwnPropertyDescriptor` method\n// https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptor\n$({ target: 'Object', stat: true, forced: FORCED, sham: !DESCRIPTORS }, {\n  getOwnPropertyDescriptor: function getOwnPropertyDescriptor(it, key) {\n    return nativeGetOwnPropertyDescriptor(toIndexedObject(it), key);\n  }\n});\n","var anObject = require('../internals/an-object');\nvar defineProperties = require('../internals/object-define-properties');\nvar enumBugKeys = require('../internals/enum-bug-keys');\nvar hiddenKeys = require('../internals/hidden-keys');\nvar html = require('../internals/html');\nvar documentCreateElement = require('../internals/document-create-element');\nvar sharedKey = require('../internals/shared-key');\n\nvar GT = '>';\nvar LT = '<';\nvar PROTOTYPE = 'prototype';\nvar SCRIPT = 'script';\nvar IE_PROTO = sharedKey('IE_PROTO');\n\nvar EmptyConstructor = function () { /* empty */ };\n\nvar scriptTag = function (content) {\n  return LT + SCRIPT + GT + content + LT + '/' + SCRIPT + GT;\n};\n\n// Create object with fake `null` prototype: use ActiveX Object with cleared prototype\nvar NullProtoObjectViaActiveX = function (activeXDocument) {\n  activeXDocument.write(scriptTag(''));\n  activeXDocument.close();\n  var temp = activeXDocument.parentWindow.Object;\n  activeXDocument = null; // avoid memory leak\n  return temp;\n};\n\n// Create object with fake `null` prototype: use iframe Object with cleared prototype\nvar NullProtoObjectViaIFrame = function () {\n  // Thrash, waste and sodomy: IE GC bug\n  var iframe = documentCreateElement('iframe');\n  var JS = 'java' + SCRIPT + ':';\n  var iframeDocument;\n  iframe.style.display = 'none';\n  html.appendChild(iframe);\n  // https://github.com/zloirock/core-js/issues/475\n  iframe.src = String(JS);\n  iframeDocument = iframe.contentWindow.document;\n  iframeDocument.open();\n  iframeDocument.write(scriptTag('document.F=Object'));\n  iframeDocument.close();\n  return iframeDocument.F;\n};\n\n// Check for document.domain and active x support\n// No need to use active x approach when document.domain is not set\n// see https://github.com/es-shims/es5-shim/issues/150\n// variation of https://github.com/kitcambridge/es5-shim/commit/4f738ac066346\n// avoid IE GC bug\nvar activeXDocument;\nvar NullProtoObject = function () {\n  try {\n    /* global ActiveXObject */\n    activeXDocument = document.domain && new ActiveXObject('htmlfile');\n  } catch (error) { /* ignore */ }\n  NullProtoObject = activeXDocument ? NullProtoObjectViaActiveX(activeXDocument) : NullProtoObjectViaIFrame();\n  var length = enumBugKeys.length;\n  while (length--) delete NullProtoObject[PROTOTYPE][enumBugKeys[length]];\n  return NullProtoObject();\n};\n\nhiddenKeys[IE_PROTO] = true;\n\n// `Object.create` method\n// https://tc39.github.io/ecma262/#sec-object.create\nmodule.exports = Object.create || function create(O, Properties) {\n  var result;\n  if (O !== null) {\n    EmptyConstructor[PROTOTYPE] = anObject(O);\n    result = new EmptyConstructor();\n    EmptyConstructor[PROTOTYPE] = null;\n    // add \"__proto__\" for Object.getPrototypeOf polyfill\n    result[IE_PROTO] = O;\n  } else result = NullProtoObject();\n  return Properties === undefined ? result : defineProperties(result, Properties);\n};\n","require('../../modules/es.object.get-own-property-descriptor');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nvar getOwnPropertyDescriptor = module.exports = function getOwnPropertyDescriptor(it, key) {\n  return Object.getOwnPropertyDescriptor(it, key);\n};\n\nif (Object.getOwnPropertyDescriptor.sham) getOwnPropertyDescriptor.sham = true;\n","var fails = require('../internals/fails');\n\nmodule.exports = !!Object.getOwnPropertySymbols && !fails(function () {\n  // Chrome 38 Symbol has incorrect toString conversion\n  // eslint-disable-next-line no-undef\n  return !String(Symbol());\n});\n","var NATIVE_SYMBOL = require('../internals/native-symbol');\n\nmodule.exports = NATIVE_SYMBOL\n  // eslint-disable-next-line no-undef\n  && !Symbol.sham\n  // eslint-disable-next-line no-undef\n  && typeof Symbol.iterator == 'symbol';\n","var classof = require('../internals/classof-raw');\n\n// `IsArray` abstract operation\n// https://tc39.github.io/ecma262/#sec-isarray\nmodule.exports = Array.isArray || function isArray(arg) {\n  return classof(arg) == 'Array';\n};\n","var requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `ToObject` abstract operation\n// https://tc39.github.io/ecma262/#sec-toobject\nmodule.exports = function (argument) {\n  return Object(requireObjectCoercible(argument));\n};\n","var getBuiltIn = require('../internals/get-built-in');\n\nmodule.exports = getBuiltIn('document', 'documentElement');\n","var global = require('../internals/global');\nvar setGlobal = require('../internals/set-global');\n\nvar SHARED = '__core-js_shared__';\nvar store = global[SHARED] || setGlobal(SHARED, {});\n\nmodule.exports = store;\n","var global = require('../internals/global');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\n\nmodule.exports = function (key, value) {\n  try {\n    createNonEnumerableProperty(global, key, value);\n  } catch (error) {\n    global[key] = value;\n  } return value;\n};\n","var IS_PURE = require('../internals/is-pure');\nvar store = require('../internals/shared-store');\n\n(module.exports = function (key, value) {\n  return store[key] || (store[key] = value !== undefined ? value : {});\n})('versions', []).push({\n  version: '3.6.4',\n  mode: IS_PURE ? 'pure' : 'global',\n  copyright: '© 2020 Denis Pushkarev (zloirock.ru)'\n});\n","var id = 0;\nvar postfix = Math.random();\n\nmodule.exports = function (key) {\n  return 'Symbol(' + String(key === undefined ? '' : key) + ')_' + (++id + postfix).toString(36);\n};\n","var shared = require('../internals/shared');\nvar uid = require('../internals/uid');\n\nvar keys = shared('keys');\n\nmodule.exports = function (key) {\n  return keys[key] || (keys[key] = uid(key));\n};\n","var toIndexedObject = require('../internals/to-indexed-object');\nvar nativeGetOwnPropertyNames = require('../internals/object-get-own-property-names').f;\n\nvar toString = {}.toString;\n\nvar windowNames = typeof window == 'object' && window && Object.getOwnPropertyNames\n  ? Object.getOwnPropertyNames(window) : [];\n\nvar getWindowNames = function (it) {\n  try {\n    return nativeGetOwnPropertyNames(it);\n  } catch (error) {\n    return windowNames.slice();\n  }\n};\n\n// fallback for IE11 buggy Object.getOwnPropertyNames with iframe and window\nmodule.exports.f = function getOwnPropertyNames(it) {\n  return windowNames && toString.call(it) == '[object Window]'\n    ? getWindowNames(it)\n    : nativeGetOwnPropertyNames(toIndexedObject(it));\n};\n","var createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\n\nmodule.exports = function (target, key, value, options) {\n  if (options && options.enumerable) target[key] = value;\n  else createNonEnumerableProperty(target, key, value);\n};\n","var global = require('../internals/global');\nvar shared = require('../internals/shared');\nvar has = require('../internals/has');\nvar uid = require('../internals/uid');\nvar NATIVE_SYMBOL = require('../internals/native-symbol');\nvar USE_SYMBOL_AS_UID = require('../internals/use-symbol-as-uid');\n\nvar WellKnownSymbolsStore = shared('wks');\nvar Symbol = global.Symbol;\nvar createWellKnownSymbol = USE_SYMBOL_AS_UID ? Symbol : Symbol && Symbol.withoutSetter || uid;\n\nmodule.exports = function (name) {\n  if (!has(WellKnownSymbolsStore, name)) {\n    if (NATIVE_SYMBOL && has(Symbol, name)) WellKnownSymbolsStore[name] = Symbol[name];\n    else WellKnownSymbolsStore[name] = createWellKnownSymbol('Symbol.' + name);\n  } return WellKnownSymbolsStore[name];\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nexports.f = wellKnownSymbol;\n","var path = require('../internals/path');\nvar has = require('../internals/has');\nvar wrappedWellKnownSymbolModule = require('../internals/well-known-symbol-wrapped');\nvar defineProperty = require('../internals/object-define-property').f;\n\nmodule.exports = function (NAME) {\n  var Symbol = path.Symbol || (path.Symbol = {});\n  if (!has(Symbol, NAME)) defineProperty(Symbol, NAME, {\n    value: wrappedWellKnownSymbolModule.f(NAME)\n  });\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\nvar test = {};\n\ntest[TO_STRING_TAG] = 'z';\n\nmodule.exports = String(test) === '[object z]';\n","var TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar classofRaw = require('../internals/classof-raw');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n// ES3 wrong here\nvar CORRECT_ARGUMENTS = classofRaw(function () { return arguments; }()) == 'Arguments';\n\n// fallback for IE11 Script Access Denied error\nvar tryGet = function (it, key) {\n  try {\n    return it[key];\n  } catch (error) { /* empty */ }\n};\n\n// getting tag from ES6+ `Object.prototype.toString`\nmodule.exports = TO_STRING_TAG_SUPPORT ? classofRaw : function (it) {\n  var O, tag, result;\n  return it === undefined ? 'Undefined' : it === null ? 'Null'\n    // @@toStringTag case\n    : typeof (tag = tryGet(O = Object(it), TO_STRING_TAG)) == 'string' ? tag\n    // builtinTag case\n    : CORRECT_ARGUMENTS ? classofRaw(O)\n    // ES3 arguments fallback\n    : (result = classofRaw(O)) == 'Object' && typeof O.callee == 'function' ? 'Arguments' : result;\n};\n","'use strict';\nvar TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar classof = require('../internals/classof');\n\n// `Object.prototype.toString` method implementation\n// https://tc39.github.io/ecma262/#sec-object.prototype.tostring\nmodule.exports = TO_STRING_TAG_SUPPORT ? {}.toString : function toString() {\n  return '[object ' + classof(this) + ']';\n};\n","var TO_STRING_TAG_SUPPORT = require('../internals/to-string-tag-support');\nvar defineProperty = require('../internals/object-define-property').f;\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\nvar toString = require('../internals/object-to-string');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n\nmodule.exports = function (it, TAG, STATIC, SET_METHOD) {\n  if (it) {\n    var target = STATIC ? it : it.prototype;\n    if (!has(target, TO_STRING_TAG)) {\n      defineProperty(target, TO_STRING_TAG, { configurable: true, value: TAG });\n    }\n    if (SET_METHOD && !TO_STRING_TAG_SUPPORT) {\n      createNonEnumerableProperty(target, 'toString', toString);\n    }\n  }\n};\n","var store = require('../internals/shared-store');\n\nvar functionToString = Function.toString;\n\n// this helper broken in `3.4.1-3.4.4`, so we can't use `shared` helper\nif (typeof store.inspectSource != 'function') {\n  store.inspectSource = function (it) {\n    return functionToString.call(it);\n  };\n}\n\nmodule.exports = store.inspectSource;\n","var NATIVE_WEAK_MAP = require('../internals/native-weak-map');\nvar global = require('../internals/global');\nvar isObject = require('../internals/is-object');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar objectHas = require('../internals/has');\nvar sharedKey = require('../internals/shared-key');\nvar hiddenKeys = require('../internals/hidden-keys');\n\nvar WeakMap = global.WeakMap;\nvar set, get, has;\n\nvar enforce = function (it) {\n  return has(it) ? get(it) : set(it, {});\n};\n\nvar getterFor = function (TYPE) {\n  return function (it) {\n    var state;\n    if (!isObject(it) || (state = get(it)).type !== TYPE) {\n      throw TypeError('Incompatible receiver, ' + TYPE + ' required');\n    } return state;\n  };\n};\n\nif (NATIVE_WEAK_MAP) {\n  var store = new WeakMap();\n  var wmget = store.get;\n  var wmhas = store.has;\n  var wmset = store.set;\n  set = function (it, metadata) {\n    wmset.call(store, it, metadata);\n    return metadata;\n  };\n  get = function (it) {\n    return wmget.call(store, it) || {};\n  };\n  has = function (it) {\n    return wmhas.call(store, it);\n  };\n} else {\n  var STATE = sharedKey('state');\n  hiddenKeys[STATE] = true;\n  set = function (it, metadata) {\n    createNonEnumerableProperty(it, STATE, metadata);\n    return metadata;\n  };\n  get = function (it) {\n    return objectHas(it, STATE) ? it[STATE] : {};\n  };\n  has = function (it) {\n    return objectHas(it, STATE);\n  };\n}\n\nmodule.exports = {\n  set: set,\n  get: get,\n  has: has,\n  enforce: enforce,\n  getterFor: getterFor\n};\n","var global = require('../internals/global');\nvar inspectSource = require('../internals/inspect-source');\n\nvar WeakMap = global.WeakMap;\n\nmodule.exports = typeof WeakMap === 'function' && /native code/.test(inspectSource(WeakMap));\n","var isObject = require('../internals/is-object');\nvar isArray = require('../internals/is-array');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar SPECIES = wellKnownSymbol('species');\n\n// `ArraySpeciesCreate` abstract operation\n// https://tc39.github.io/ecma262/#sec-arrayspeciescreate\nmodule.exports = function (originalArray, length) {\n  var C;\n  if (isArray(originalArray)) {\n    C = originalArray.constructor;\n    // cross-realm fallback\n    if (typeof C == 'function' && (C === Array || isArray(C.prototype))) C = undefined;\n    else if (isObject(C)) {\n      C = C[SPECIES];\n      if (C === null) C = undefined;\n    }\n  } return new (C === undefined ? Array : C)(length === 0 ? 0 : length);\n};\n","var bind = require('../internals/function-bind-context');\nvar IndexedObject = require('../internals/indexed-object');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar arraySpeciesCreate = require('../internals/array-species-create');\n\nvar push = [].push;\n\n// `Array.prototype.{ forEach, map, filter, some, every, find, findIndex }` methods implementation\nvar createMethod = function (TYPE) {\n  var IS_MAP = TYPE == 1;\n  var IS_FILTER = TYPE == 2;\n  var IS_SOME = TYPE == 3;\n  var IS_EVERY = TYPE == 4;\n  var IS_FIND_INDEX = TYPE == 6;\n  var NO_HOLES = TYPE == 5 || IS_FIND_INDEX;\n  return function ($this, callbackfn, that, specificCreate) {\n    var O = toObject($this);\n    var self = IndexedObject(O);\n    var boundFunction = bind(callbackfn, that, 3);\n    var length = toLength(self.length);\n    var index = 0;\n    var create = specificCreate || arraySpeciesCreate;\n    var target = IS_MAP ? create($this, length) : IS_FILTER ? create($this, 0) : undefined;\n    var value, result;\n    for (;length > index; index++) if (NO_HOLES || index in self) {\n      value = self[index];\n      result = boundFunction(value, index, O);\n      if (TYPE) {\n        if (IS_MAP) target[index] = result; // map\n        else if (result) switch (TYPE) {\n          case 3: return true;              // some\n          case 5: return value;             // find\n          case 6: return index;             // findIndex\n          case 2: push.call(target, value); // filter\n        } else if (IS_EVERY) return false;  // every\n      }\n    }\n    return IS_FIND_INDEX ? -1 : IS_SOME || IS_EVERY ? IS_EVERY : target;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.forEach` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.foreach\n  forEach: createMethod(0),\n  // `Array.prototype.map` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.map\n  map: createMethod(1),\n  // `Array.prototype.filter` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.filter\n  filter: createMethod(2),\n  // `Array.prototype.some` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.some\n  some: createMethod(3),\n  // `Array.prototype.every` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.every\n  every: createMethod(4),\n  // `Array.prototype.find` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.find\n  find: createMethod(5),\n  // `Array.prototype.findIndex` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.findIndex\n  findIndex: createMethod(6)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar global = require('../internals/global');\nvar getBuiltIn = require('../internals/get-built-in');\nvar IS_PURE = require('../internals/is-pure');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar NATIVE_SYMBOL = require('../internals/native-symbol');\nvar USE_SYMBOL_AS_UID = require('../internals/use-symbol-as-uid');\nvar fails = require('../internals/fails');\nvar has = require('../internals/has');\nvar isArray = require('../internals/is-array');\nvar isObject = require('../internals/is-object');\nvar anObject = require('../internals/an-object');\nvar toObject = require('../internals/to-object');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar toPrimitive = require('../internals/to-primitive');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar nativeObjectCreate = require('../internals/object-create');\nvar objectKeys = require('../internals/object-keys');\nvar getOwnPropertyNamesModule = require('../internals/object-get-own-property-names');\nvar getOwnPropertyNamesExternal = require('../internals/object-get-own-property-names-external');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar getOwnPropertyDescriptorModule = require('../internals/object-get-own-property-descriptor');\nvar definePropertyModule = require('../internals/object-define-property');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar redefine = require('../internals/redefine');\nvar shared = require('../internals/shared');\nvar sharedKey = require('../internals/shared-key');\nvar hiddenKeys = require('../internals/hidden-keys');\nvar uid = require('../internals/uid');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar wrappedWellKnownSymbolModule = require('../internals/well-known-symbol-wrapped');\nvar defineWellKnownSymbol = require('../internals/define-well-known-symbol');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar InternalStateModule = require('../internals/internal-state');\nvar $forEach = require('../internals/array-iteration').forEach;\n\nvar HIDDEN = sharedKey('hidden');\nvar SYMBOL = 'Symbol';\nvar PROTOTYPE = 'prototype';\nvar TO_PRIMITIVE = wellKnownSymbol('toPrimitive');\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(SYMBOL);\nvar ObjectPrototype = Object[PROTOTYPE];\nvar $Symbol = global.Symbol;\nvar $stringify = getBuiltIn('JSON', 'stringify');\nvar nativeGetOwnPropertyDescriptor = getOwnPropertyDescriptorModule.f;\nvar nativeDefineProperty = definePropertyModule.f;\nvar nativeGetOwnPropertyNames = getOwnPropertyNamesExternal.f;\nvar nativePropertyIsEnumerable = propertyIsEnumerableModule.f;\nvar AllSymbols = shared('symbols');\nvar ObjectPrototypeSymbols = shared('op-symbols');\nvar StringToSymbolRegistry = shared('string-to-symbol-registry');\nvar SymbolToStringRegistry = shared('symbol-to-string-registry');\nvar WellKnownSymbolsStore = shared('wks');\nvar QObject = global.QObject;\n// Don't use setters in Qt Script, https://github.com/zloirock/core-js/issues/173\nvar USE_SETTER = !QObject || !QObject[PROTOTYPE] || !QObject[PROTOTYPE].findChild;\n\n// fallback for old Android, https://code.google.com/p/v8/issues/detail?id=687\nvar setSymbolDescriptor = DESCRIPTORS && fails(function () {\n  return nativeObjectCreate(nativeDefineProperty({}, 'a', {\n    get: function () { return nativeDefineProperty(this, 'a', { value: 7 }).a; }\n  })).a != 7;\n}) ? function (O, P, Attributes) {\n  var ObjectPrototypeDescriptor = nativeGetOwnPropertyDescriptor(ObjectPrototype, P);\n  if (ObjectPrototypeDescriptor) delete ObjectPrototype[P];\n  nativeDefineProperty(O, P, Attributes);\n  if (ObjectPrototypeDescriptor && O !== ObjectPrototype) {\n    nativeDefineProperty(ObjectPrototype, P, ObjectPrototypeDescriptor);\n  }\n} : nativeDefineProperty;\n\nvar wrap = function (tag, description) {\n  var symbol = AllSymbols[tag] = nativeObjectCreate($Symbol[PROTOTYPE]);\n  setInternalState(symbol, {\n    type: SYMBOL,\n    tag: tag,\n    description: description\n  });\n  if (!DESCRIPTORS) symbol.description = description;\n  return symbol;\n};\n\nvar isSymbol = USE_SYMBOL_AS_UID ? function (it) {\n  return typeof it == 'symbol';\n} : function (it) {\n  return Object(it) instanceof $Symbol;\n};\n\nvar $defineProperty = function defineProperty(O, P, Attributes) {\n  if (O === ObjectPrototype) $defineProperty(ObjectPrototypeSymbols, P, Attributes);\n  anObject(O);\n  var key = toPrimitive(P, true);\n  anObject(Attributes);\n  if (has(AllSymbols, key)) {\n    if (!Attributes.enumerable) {\n      if (!has(O, HIDDEN)) nativeDefineProperty(O, HIDDEN, createPropertyDescriptor(1, {}));\n      O[HIDDEN][key] = true;\n    } else {\n      if (has(O, HIDDEN) && O[HIDDEN][key]) O[HIDDEN][key] = false;\n      Attributes = nativeObjectCreate(Attributes, { enumerable: createPropertyDescriptor(0, false) });\n    } return setSymbolDescriptor(O, key, Attributes);\n  } return nativeDefineProperty(O, key, Attributes);\n};\n\nvar $defineProperties = function defineProperties(O, Properties) {\n  anObject(O);\n  var properties = toIndexedObject(Properties);\n  var keys = objectKeys(properties).concat($getOwnPropertySymbols(properties));\n  $forEach(keys, function (key) {\n    if (!DESCRIPTORS || $propertyIsEnumerable.call(properties, key)) $defineProperty(O, key, properties[key]);\n  });\n  return O;\n};\n\nvar $create = function create(O, Properties) {\n  return Properties === undefined ? nativeObjectCreate(O) : $defineProperties(nativeObjectCreate(O), Properties);\n};\n\nvar $propertyIsEnumerable = function propertyIsEnumerable(V) {\n  var P = toPrimitive(V, true);\n  var enumerable = nativePropertyIsEnumerable.call(this, P);\n  if (this === ObjectPrototype && has(AllSymbols, P) && !has(ObjectPrototypeSymbols, P)) return false;\n  return enumerable || !has(this, P) || !has(AllSymbols, P) || has(this, HIDDEN) && this[HIDDEN][P] ? enumerable : true;\n};\n\nvar $getOwnPropertyDescriptor = function getOwnPropertyDescriptor(O, P) {\n  var it = toIndexedObject(O);\n  var key = toPrimitive(P, true);\n  if (it === ObjectPrototype && has(AllSymbols, key) && !has(ObjectPrototypeSymbols, key)) return;\n  var descriptor = nativeGetOwnPropertyDescriptor(it, key);\n  if (descriptor && has(AllSymbols, key) && !(has(it, HIDDEN) && it[HIDDEN][key])) {\n    descriptor.enumerable = true;\n  }\n  return descriptor;\n};\n\nvar $getOwnPropertyNames = function getOwnPropertyNames(O) {\n  var names = nativeGetOwnPropertyNames(toIndexedObject(O));\n  var result = [];\n  $forEach(names, function (key) {\n    if (!has(AllSymbols, key) && !has(hiddenKeys, key)) result.push(key);\n  });\n  return result;\n};\n\nvar $getOwnPropertySymbols = function getOwnPropertySymbols(O) {\n  var IS_OBJECT_PROTOTYPE = O === ObjectPrototype;\n  var names = nativeGetOwnPropertyNames(IS_OBJECT_PROTOTYPE ? ObjectPrototypeSymbols : toIndexedObject(O));\n  var result = [];\n  $forEach(names, function (key) {\n    if (has(AllSymbols, key) && (!IS_OBJECT_PROTOTYPE || has(ObjectPrototype, key))) {\n      result.push(AllSymbols[key]);\n    }\n  });\n  return result;\n};\n\n// `Symbol` constructor\n// https://tc39.github.io/ecma262/#sec-symbol-constructor\nif (!NATIVE_SYMBOL) {\n  $Symbol = function Symbol() {\n    if (this instanceof $Symbol) throw TypeError('Symbol is not a constructor');\n    var description = !arguments.length || arguments[0] === undefined ? undefined : String(arguments[0]);\n    var tag = uid(description);\n    var setter = function (value) {\n      if (this === ObjectPrototype) setter.call(ObjectPrototypeSymbols, value);\n      if (has(this, HIDDEN) && has(this[HIDDEN], tag)) this[HIDDEN][tag] = false;\n      setSymbolDescriptor(this, tag, createPropertyDescriptor(1, value));\n    };\n    if (DESCRIPTORS && USE_SETTER) setSymbolDescriptor(ObjectPrototype, tag, { configurable: true, set: setter });\n    return wrap(tag, description);\n  };\n\n  redefine($Symbol[PROTOTYPE], 'toString', function toString() {\n    return getInternalState(this).tag;\n  });\n\n  redefine($Symbol, 'withoutSetter', function (description) {\n    return wrap(uid(description), description);\n  });\n\n  propertyIsEnumerableModule.f = $propertyIsEnumerable;\n  definePropertyModule.f = $defineProperty;\n  getOwnPropertyDescriptorModule.f = $getOwnPropertyDescriptor;\n  getOwnPropertyNamesModule.f = getOwnPropertyNamesExternal.f = $getOwnPropertyNames;\n  getOwnPropertySymbolsModule.f = $getOwnPropertySymbols;\n\n  wrappedWellKnownSymbolModule.f = function (name) {\n    return wrap(wellKnownSymbol(name), name);\n  };\n\n  if (DESCRIPTORS) {\n    // https://github.com/tc39/proposal-Symbol-description\n    nativeDefineProperty($Symbol[PROTOTYPE], 'description', {\n      configurable: true,\n      get: function description() {\n        return getInternalState(this).description;\n      }\n    });\n    if (!IS_PURE) {\n      redefine(ObjectPrototype, 'propertyIsEnumerable', $propertyIsEnumerable, { unsafe: true });\n    }\n  }\n}\n\n$({ global: true, wrap: true, forced: !NATIVE_SYMBOL, sham: !NATIVE_SYMBOL }, {\n  Symbol: $Symbol\n});\n\n$forEach(objectKeys(WellKnownSymbolsStore), function (name) {\n  defineWellKnownSymbol(name);\n});\n\n$({ target: SYMBOL, stat: true, forced: !NATIVE_SYMBOL }, {\n  // `Symbol.for` method\n  // https://tc39.github.io/ecma262/#sec-symbol.for\n  'for': function (key) {\n    var string = String(key);\n    if (has(StringToSymbolRegistry, string)) return StringToSymbolRegistry[string];\n    var symbol = $Symbol(string);\n    StringToSymbolRegistry[string] = symbol;\n    SymbolToStringRegistry[symbol] = string;\n    return symbol;\n  },\n  // `Symbol.keyFor` method\n  // https://tc39.github.io/ecma262/#sec-symbol.keyfor\n  keyFor: function keyFor(sym) {\n    if (!isSymbol(sym)) throw TypeError(sym + ' is not a symbol');\n    if (has(SymbolToStringRegistry, sym)) return SymbolToStringRegistry[sym];\n  },\n  useSetter: function () { USE_SETTER = true; },\n  useSimple: function () { USE_SETTER = false; }\n});\n\n$({ target: 'Object', stat: true, forced: !NATIVE_SYMBOL, sham: !DESCRIPTORS }, {\n  // `Object.create` method\n  // https://tc39.github.io/ecma262/#sec-object.create\n  create: $create,\n  // `Object.defineProperty` method\n  // https://tc39.github.io/ecma262/#sec-object.defineproperty\n  defineProperty: $defineProperty,\n  // `Object.defineProperties` method\n  // https://tc39.github.io/ecma262/#sec-object.defineproperties\n  defineProperties: $defineProperties,\n  // `Object.getOwnPropertyDescriptor` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertydescriptors\n  getOwnPropertyDescriptor: $getOwnPropertyDescriptor\n});\n\n$({ target: 'Object', stat: true, forced: !NATIVE_SYMBOL }, {\n  // `Object.getOwnPropertyNames` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertynames\n  getOwnPropertyNames: $getOwnPropertyNames,\n  // `Object.getOwnPropertySymbols` method\n  // https://tc39.github.io/ecma262/#sec-object.getownpropertysymbols\n  getOwnPropertySymbols: $getOwnPropertySymbols\n});\n\n// Chrome 38 and 39 `Object.getOwnPropertySymbols` fails on primitives\n// https://bugs.chromium.org/p/v8/issues/detail?id=3443\n$({ target: 'Object', stat: true, forced: fails(function () { getOwnPropertySymbolsModule.f(1); }) }, {\n  getOwnPropertySymbols: function getOwnPropertySymbols(it) {\n    return getOwnPropertySymbolsModule.f(toObject(it));\n  }\n});\n\n// `JSON.stringify` method behavior with symbols\n// https://tc39.github.io/ecma262/#sec-json.stringify\nif ($stringify) {\n  var FORCED_JSON_STRINGIFY = !NATIVE_SYMBOL || fails(function () {\n    var symbol = $Symbol();\n    // MS Edge converts symbol values to JSON as {}\n    return $stringify([symbol]) != '[null]'\n      // WebKit converts symbol values to JSON as null\n      || $stringify({ a: symbol }) != '{}'\n      // V8 throws on boxed symbols\n      || $stringify(Object(symbol)) != '{}';\n  });\n\n  $({ target: 'JSON', stat: true, forced: FORCED_JSON_STRINGIFY }, {\n    // eslint-disable-next-line no-unused-vars\n    stringify: function stringify(it, replacer, space) {\n      var args = [it];\n      var index = 1;\n      var $replacer;\n      while (arguments.length > index) args.push(arguments[index++]);\n      $replacer = replacer;\n      if (!isObject(replacer) && it === undefined || isSymbol(it)) return; // IE8 returns string on undefined\n      if (!isArray(replacer)) replacer = function (key, value) {\n        if (typeof $replacer == 'function') value = $replacer.call(this, key, value);\n        if (!isSymbol(value)) return value;\n      };\n      args[1] = replacer;\n      return $stringify.apply(null, args);\n    }\n  });\n}\n\n// `Symbol.prototype[@@toPrimitive]` method\n// https://tc39.github.io/ecma262/#sec-symbol.prototype-@@toprimitive\nif (!$Symbol[PROTOTYPE][TO_PRIMITIVE]) {\n  createNonEnumerableProperty($Symbol[PROTOTYPE], TO_PRIMITIVE, $Symbol[PROTOTYPE].valueOf);\n}\n// `Symbol.prototype[@@toStringTag]` property\n// https://tc39.github.io/ecma262/#sec-symbol.prototype-@@tostringtag\nsetToStringTag($Symbol, SYMBOL);\n\nhiddenKeys[HIDDEN] = true;\n","require('../../modules/es.symbol');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getOwnPropertySymbols;\n","'use strict';\nvar getPrototypeOf = require('../internals/object-get-prototype-of');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar has = require('../internals/has');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar IS_PURE = require('../internals/is-pure');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar BUGGY_SAFARI_ITERATORS = false;\n\nvar returnThis = function () { return this; };\n\n// `%IteratorPrototype%` object\n// https://tc39.github.io/ecma262/#sec-%iteratorprototype%-object\nvar IteratorPrototype, PrototypeOfArrayIteratorPrototype, arrayIterator;\n\nif ([].keys) {\n  arrayIterator = [].keys();\n  // Safari 8 has buggy iterators w/o `next`\n  if (!('next' in arrayIterator)) BUGGY_SAFARI_ITERATORS = true;\n  else {\n    PrototypeOfArrayIteratorPrototype = getPrototypeOf(getPrototypeOf(arrayIterator));\n    if (PrototypeOfArrayIteratorPrototype !== Object.prototype) IteratorPrototype = PrototypeOfArrayIteratorPrototype;\n  }\n}\n\nif (IteratorPrototype == undefined) IteratorPrototype = {};\n\n// 25.1.2.1.1 %IteratorPrototype%[@@iterator]()\nif (!IS_PURE && !has(IteratorPrototype, ITERATOR)) {\n  createNonEnumerableProperty(IteratorPrototype, ITERATOR, returnThis);\n}\n\nmodule.exports = {\n  IteratorPrototype: IteratorPrototype,\n  BUGGY_SAFARI_ITERATORS: BUGGY_SAFARI_ITERATORS\n};\n","module.exports = {};\n","var fails = require('../internals/fails');\n\nmodule.exports = !fails(function () {\n  function F() { /* empty */ }\n  F.prototype.constructor = null;\n  return Object.getPrototypeOf(new F()) !== F.prototype;\n});\n","var has = require('../internals/has');\nvar toObject = require('../internals/to-object');\nvar sharedKey = require('../internals/shared-key');\nvar CORRECT_PROTOTYPE_GETTER = require('../internals/correct-prototype-getter');\n\nvar IE_PROTO = sharedKey('IE_PROTO');\nvar ObjectPrototype = Object.prototype;\n\n// `Object.getPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.getprototypeof\nmodule.exports = CORRECT_PROTOTYPE_GETTER ? Object.getPrototypeOf : function (O) {\n  O = toObject(O);\n  if (has(O, IE_PROTO)) return O[IE_PROTO];\n  if (typeof O.constructor == 'function' && O instanceof O.constructor) {\n    return O.constructor.prototype;\n  } return O instanceof Object ? ObjectPrototype : null;\n};\n","'use strict';\nvar IteratorPrototype = require('../internals/iterators-core').IteratorPrototype;\nvar create = require('../internals/object-create');\nvar createPropertyDescriptor = require('../internals/create-property-descriptor');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar Iterators = require('../internals/iterators');\n\nvar returnThis = function () { return this; };\n\nmodule.exports = function (IteratorConstructor, NAME, next) {\n  var TO_STRING_TAG = NAME + ' Iterator';\n  IteratorConstructor.prototype = create(IteratorPrototype, { next: createPropertyDescriptor(1, next) });\n  setToStringTag(IteratorConstructor, TO_STRING_TAG, false, true);\n  Iterators[TO_STRING_TAG] = returnThis;\n  return IteratorConstructor;\n};\n","var anObject = require('../internals/an-object');\nvar aPossiblePrototype = require('../internals/a-possible-prototype');\n\n// `Object.setPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.setprototypeof\n// Works with __proto__ only. Old v8 can't work with null proto objects.\n/* eslint-disable no-proto */\nmodule.exports = Object.setPrototypeOf || ('__proto__' in {} ? function () {\n  var CORRECT_SETTER = false;\n  var test = {};\n  var setter;\n  try {\n    setter = Object.getOwnPropertyDescriptor(Object.prototype, '__proto__').set;\n    setter.call(test, []);\n    CORRECT_SETTER = test instanceof Array;\n  } catch (error) { /* empty */ }\n  return function setPrototypeOf(O, proto) {\n    anObject(O);\n    aPossiblePrototype(proto);\n    if (CORRECT_SETTER) setter.call(O, proto);\n    else O.__proto__ = proto;\n    return O;\n  };\n}() : undefined);\n","var isObject = require('../internals/is-object');\n\nmodule.exports = function (it) {\n  if (!isObject(it) && it !== null) {\n    throw TypeError(\"Can't set \" + String(it) + ' as a prototype');\n  } return it;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar createIteratorConstructor = require('../internals/create-iterator-constructor');\nvar getPrototypeOf = require('../internals/object-get-prototype-of');\nvar setPrototypeOf = require('../internals/object-set-prototype-of');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar redefine = require('../internals/redefine');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar IS_PURE = require('../internals/is-pure');\nvar Iterators = require('../internals/iterators');\nvar IteratorsCore = require('../internals/iterators-core');\n\nvar IteratorPrototype = IteratorsCore.IteratorPrototype;\nvar BUGGY_SAFARI_ITERATORS = IteratorsCore.BUGGY_SAFARI_ITERATORS;\nvar ITERATOR = wellKnownSymbol('iterator');\nvar KEYS = 'keys';\nvar VALUES = 'values';\nvar ENTRIES = 'entries';\n\nvar returnThis = function () { return this; };\n\nmodule.exports = function (Iterable, NAME, IteratorConstructor, next, DEFAULT, IS_SET, FORCED) {\n  createIteratorConstructor(IteratorConstructor, NAME, next);\n\n  var getIterationMethod = function (KIND) {\n    if (KIND === DEFAULT && defaultIterator) return defaultIterator;\n    if (!BUGGY_SAFARI_ITERATORS && KIND in IterablePrototype) return IterablePrototype[KIND];\n    switch (KIND) {\n      case KEYS: return function keys() { return new IteratorConstructor(this, KIND); };\n      case VALUES: return function values() { return new IteratorConstructor(this, KIND); };\n      case ENTRIES: return function entries() { return new IteratorConstructor(this, KIND); };\n    } return function () { return new IteratorConstructor(this); };\n  };\n\n  var TO_STRING_TAG = NAME + ' Iterator';\n  var INCORRECT_VALUES_NAME = false;\n  var IterablePrototype = Iterable.prototype;\n  var nativeIterator = IterablePrototype[ITERATOR]\n    || IterablePrototype['@@iterator']\n    || DEFAULT && IterablePrototype[DEFAULT];\n  var defaultIterator = !BUGGY_SAFARI_ITERATORS && nativeIterator || getIterationMethod(DEFAULT);\n  var anyNativeIterator = NAME == 'Array' ? IterablePrototype.entries || nativeIterator : nativeIterator;\n  var CurrentIteratorPrototype, methods, KEY;\n\n  // fix native\n  if (anyNativeIterator) {\n    CurrentIteratorPrototype = getPrototypeOf(anyNativeIterator.call(new Iterable()));\n    if (IteratorPrototype !== Object.prototype && CurrentIteratorPrototype.next) {\n      if (!IS_PURE && getPrototypeOf(CurrentIteratorPrototype) !== IteratorPrototype) {\n        if (setPrototypeOf) {\n          setPrototypeOf(CurrentIteratorPrototype, IteratorPrototype);\n        } else if (typeof CurrentIteratorPrototype[ITERATOR] != 'function') {\n          createNonEnumerableProperty(CurrentIteratorPrototype, ITERATOR, returnThis);\n        }\n      }\n      // Set @@toStringTag to native iterators\n      setToStringTag(CurrentIteratorPrototype, TO_STRING_TAG, true, true);\n      if (IS_PURE) Iterators[TO_STRING_TAG] = returnThis;\n    }\n  }\n\n  // fix Array#{values, @@iterator}.name in V8 / FF\n  if (DEFAULT == VALUES && nativeIterator && nativeIterator.name !== VALUES) {\n    INCORRECT_VALUES_NAME = true;\n    defaultIterator = function values() { return nativeIterator.call(this); };\n  }\n\n  // define iterator\n  if ((!IS_PURE || FORCED) && IterablePrototype[ITERATOR] !== defaultIterator) {\n    createNonEnumerableProperty(IterablePrototype, ITERATOR, defaultIterator);\n  }\n  Iterators[NAME] = defaultIterator;\n\n  // export additional methods\n  if (DEFAULT) {\n    methods = {\n      values: getIterationMethod(VALUES),\n      keys: IS_SET ? defaultIterator : getIterationMethod(KEYS),\n      entries: getIterationMethod(ENTRIES)\n    };\n    if (FORCED) for (KEY in methods) {\n      if (BUGGY_SAFARI_ITERATORS || INCORRECT_VALUES_NAME || !(KEY in IterablePrototype)) {\n        redefine(IterablePrototype, KEY, methods[KEY]);\n      }\n    } else $({ target: NAME, proto: true, forced: BUGGY_SAFARI_ITERATORS || INCORRECT_VALUES_NAME }, methods);\n  }\n\n  return methods;\n};\n","'use strict';\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar Iterators = require('../internals/iterators');\nvar InternalStateModule = require('../internals/internal-state');\nvar defineIterator = require('../internals/define-iterator');\n\nvar ARRAY_ITERATOR = 'Array Iterator';\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(ARRAY_ITERATOR);\n\n// `Array.prototype.entries` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.entries\n// `Array.prototype.keys` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.keys\n// `Array.prototype.values` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.values\n// `Array.prototype[@@iterator]` method\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@iterator\n// `CreateArrayIterator` internal method\n// https://tc39.github.io/ecma262/#sec-createarrayiterator\nmodule.exports = defineIterator(Array, 'Array', function (iterated, kind) {\n  setInternalState(this, {\n    type: ARRAY_ITERATOR,\n    target: toIndexedObject(iterated), // target\n    index: 0,                          // next index\n    kind: kind                         // kind\n  });\n// `%ArrayIteratorPrototype%.next` method\n// https://tc39.github.io/ecma262/#sec-%arrayiteratorprototype%.next\n}, function () {\n  var state = getInternalState(this);\n  var target = state.target;\n  var kind = state.kind;\n  var index = state.index++;\n  if (!target || index >= target.length) {\n    state.target = undefined;\n    return { value: undefined, done: true };\n  }\n  if (kind == 'keys') return { value: index, done: false };\n  if (kind == 'values') return { value: target[index], done: false };\n  return { value: [index, target[index]], done: false };\n}, 'values');\n\n// argumentsList[@@iterator] is %ArrayProto_values%\n// https://tc39.github.io/ecma262/#sec-createunmappedargumentsobject\n// https://tc39.github.io/ecma262/#sec-createmappedargumentsobject\nIterators.Arguments = Iterators.Array;\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('keys');\naddToUnscopables('values');\naddToUnscopables('entries');\n","// iterable DOM collections\n// flag - `iterable` interface - 'entries', 'keys', 'values', 'forEach' methods\nmodule.exports = {\n  CSSRuleList: 0,\n  CSSStyleDeclaration: 0,\n  CSSValueList: 0,\n  ClientRectList: 0,\n  DOMRectList: 0,\n  DOMStringList: 0,\n  DOMTokenList: 1,\n  DataTransferItemList: 0,\n  FileList: 0,\n  HTMLAllCollection: 0,\n  HTMLCollection: 0,\n  HTMLFormElement: 0,\n  HTMLSelectElement: 0,\n  MediaList: 0,\n  MimeTypeArray: 0,\n  NamedNodeMap: 0,\n  NodeList: 1,\n  PaintRequestList: 0,\n  Plugin: 0,\n  PluginArray: 0,\n  SVGLengthList: 0,\n  SVGNumberList: 0,\n  SVGPathSegList: 0,\n  SVGPointList: 0,\n  SVGStringList: 0,\n  SVGTransformList: 0,\n  SourceBufferList: 0,\n  StyleSheetList: 0,\n  TextTrackCueList: 0,\n  TextTrackList: 0,\n  TouchList: 0\n};\n","require('./es.array.iterator');\nvar DOMIterables = require('../internals/dom-iterables');\nvar global = require('../internals/global');\nvar classof = require('../internals/classof');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar Iterators = require('../internals/iterators');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar TO_STRING_TAG = wellKnownSymbol('toStringTag');\n\nfor (var COLLECTION_NAME in DOMIterables) {\n  var Collection = global[COLLECTION_NAME];\n  var CollectionPrototype = Collection && Collection.prototype;\n  if (CollectionPrototype && classof(CollectionPrototype) !== TO_STRING_TAG) {\n    createNonEnumerableProperty(CollectionPrototype, TO_STRING_TAG, COLLECTION_NAME);\n  }\n  Iterators[COLLECTION_NAME] = Iterators.Array;\n}\n","var toInteger = require('../internals/to-integer');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `String.prototype.{ codePointAt, at }` methods implementation\nvar createMethod = function (CONVERT_TO_STRING) {\n  return function ($this, pos) {\n    var S = String(requireObjectCoercible($this));\n    var position = toInteger(pos);\n    var size = S.length;\n    var first, second;\n    if (position < 0 || position >= size) return CONVERT_TO_STRING ? '' : undefined;\n    first = S.charCodeAt(position);\n    return first < 0xD800 || first > 0xDBFF || position + 1 === size\n      || (second = S.charCodeAt(position + 1)) < 0xDC00 || second > 0xDFFF\n        ? CONVERT_TO_STRING ? S.charAt(position) : first\n        : CONVERT_TO_STRING ? S.slice(position, position + 2) : (first - 0xD800 << 10) + (second - 0xDC00) + 0x10000;\n  };\n};\n\nmodule.exports = {\n  // `String.prototype.codePointAt` method\n  // https://tc39.github.io/ecma262/#sec-string.prototype.codepointat\n  codeAt: createMethod(false),\n  // `String.prototype.at` method\n  // https://github.com/mathiasbynens/String.prototype.at\n  charAt: createMethod(true)\n};\n","'use strict';\nvar charAt = require('../internals/string-multibyte').charAt;\nvar InternalStateModule = require('../internals/internal-state');\nvar defineIterator = require('../internals/define-iterator');\n\nvar STRING_ITERATOR = 'String Iterator';\nvar setInternalState = InternalStateModule.set;\nvar getInternalState = InternalStateModule.getterFor(STRING_ITERATOR);\n\n// `String.prototype[@@iterator]` method\n// https://tc39.github.io/ecma262/#sec-string.prototype-@@iterator\ndefineIterator(String, 'String', function (iterated) {\n  setInternalState(this, {\n    type: STRING_ITERATOR,\n    string: String(iterated),\n    index: 0\n  });\n// `%StringIteratorPrototype%.next` method\n// https://tc39.github.io/ecma262/#sec-%stringiteratorprototype%.next\n}, function next() {\n  var state = getInternalState(this);\n  var string = state.string;\n  var index = state.index;\n  var point;\n  if (index >= string.length) return { value: undefined, done: true };\n  point = charAt(string, index);\n  state.index += point.length;\n  return { value: point, done: false };\n});\n","var classof = require('../internals/classof');\nvar Iterators = require('../internals/iterators');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar ITERATOR = wellKnownSymbol('iterator');\n\nmodule.exports = function (it) {\n  if (it != undefined) return it[ITERATOR]\n    || it['@@iterator']\n    || Iterators[classof(it)];\n};\n","var anObject = require('../internals/an-object');\nvar getIteratorMethod = require('../internals/get-iterator-method');\n\nmodule.exports = function (it) {\n  var iteratorMethod = getIteratorMethod(it);\n  if (typeof iteratorMethod != 'function') {\n    throw TypeError(String(it) + ' is not iterable');\n  } return anObject(iteratorMethod.call(it));\n};\n","var $ = require('../internals/export');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar create = require('../internals/object-create');\n\n// `Object.create` method\n// https://tc39.github.io/ecma262/#sec-object.create\n$({ target: 'Object', stat: true, sham: !DESCRIPTORS }, {\n  create: create\n});\n","require('../../modules/es.object.create');\nvar path = require('../../internals/path');\n\nvar Object = path.Object;\n\nmodule.exports = function create(P, D) {\n  return Object.create(P, D);\n};\n","var parent = require('../../es/object/create');\n\nmodule.exports = parent;\n","var parent = require('../../es/object/define-property');\n\nmodule.exports = parent;\n","var _Object$defineProperty = require(\"../core-js/object/define-property\");\n\nfunction _defineProperty(obj, key, value) {\n  if (key in obj) {\n    _Object$defineProperty(obj, key, {\n      value: value,\n      enumerable: true,\n      configurable: true,\n      writable: true\n    });\n  } else {\n    obj[key] = value;\n  }\n\n  return obj;\n}\n\nmodule.exports = _defineProperty;","var $ = require('../internals/export');\nvar toObject = require('../internals/to-object');\nvar nativeKeys = require('../internals/object-keys');\nvar fails = require('../internals/fails');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeKeys(1); });\n\n// `Object.keys` method\n// https://tc39.github.io/ecma262/#sec-object.keys\n$({ target: 'Object', stat: true, forced: FAILS_ON_PRIMITIVES }, {\n  keys: function keys(it) {\n    return nativeKeys(toObject(it));\n  }\n});\n","require('../../modules/es.object.keys');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.keys;\n","var fails = require('../internals/fails');\nvar whitespaces = require('../internals/whitespaces');\n\nvar non = '\\u200B\\u0085\\u180E';\n\n// check that a method works with the correct list\n// of whitespaces and has a correct name\nmodule.exports = function (METHOD_NAME) {\n  return fails(function () {\n    return !!whitespaces[METHOD_NAME]() || non[METHOD_NAME]() != non || whitespaces[METHOD_NAME].name !== METHOD_NAME;\n  });\n};\n","// a string of all valid unicode whitespaces\n// eslint-disable-next-line max-len\nmodule.exports = '\\u0009\\u000A\\u000B\\u000C\\u000D\\u0020\\u00A0\\u1680\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200A\\u202F\\u205F\\u3000\\u2028\\u2029\\uFEFF';\n","var requireObjectCoercible = require('../internals/require-object-coercible');\nvar whitespaces = require('../internals/whitespaces');\n\nvar whitespace = '[' + whitespaces + ']';\nvar ltrim = RegExp('^' + whitespace + whitespace + '*');\nvar rtrim = RegExp(whitespace + whitespace + '*$');\n\n// `String.prototype.{ trim, trimStart, trimEnd, trimLeft, trimRight }` methods implementation\nvar createMethod = function (TYPE) {\n  return function ($this) {\n    var string = String(requireObjectCoercible($this));\n    if (TYPE & 1) string = string.replace(ltrim, '');\n    if (TYPE & 2) string = string.replace(rtrim, '');\n    return string;\n  };\n};\n\nmodule.exports = {\n  // `String.prototype.{ trimLeft, trimStart }` methods\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trimstart\n  start: createMethod(1),\n  // `String.prototype.{ trimRight, trimEnd }` methods\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trimend\n  end: createMethod(2),\n  // `String.prototype.trim` method\n  // https://tc39.github.io/ecma262/#sec-string.prototype.trim\n  trim: createMethod(3)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $trim = require('../internals/string-trim').trim;\nvar forcedStringTrimMethod = require('../internals/string-trim-forced');\n\n// `String.prototype.trim` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.trim\n$({ target: 'String', proto: true, forced: forcedStringTrimMethod('trim') }, {\n  trim: function trim() {\n    return $trim(this);\n  }\n});\n","var path = require('../internals/path');\n\nmodule.exports = function (CONSTRUCTOR) {\n  return path[CONSTRUCTOR + 'Prototype'];\n};\n","require('../../../modules/es.string.trim');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').trim;\n","var trim = require('../string/virtual/trim');\n\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.trim;\n  return typeof it === 'string' || it === StringPrototype\n    || (it instanceof String && own === StringPrototype.trim) ? trim : own;\n};\n","'use strict';\nvar fails = require('../internals/fails');\n\nmodule.exports = function (METHOD_NAME, argument) {\n  var method = [][METHOD_NAME];\n  return !!method && fails(function () {\n    // eslint-disable-next-line no-useless-call,no-throw-literal\n    method.call(null, argument || function () { throw 1; }, 1);\n  });\n};\n","var DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar has = require('../internals/has');\n\nvar defineProperty = Object.defineProperty;\nvar cache = {};\n\nvar thrower = function (it) { throw it; };\n\nmodule.exports = function (METHOD_NAME, options) {\n  if (has(cache, METHOD_NAME)) return cache[METHOD_NAME];\n  if (!options) options = {};\n  var method = [][METHOD_NAME];\n  var ACCESSORS = has(options, 'ACCESSORS') ? options.ACCESSORS : false;\n  var argument0 = has(options, 0) ? options[0] : thrower;\n  var argument1 = has(options, 1) ? options[1] : undefined;\n\n  return cache[METHOD_NAME] = !!method && !fails(function () {\n    if (ACCESSORS && !DESCRIPTORS) return true;\n    var O = { length: -1 };\n\n    if (ACCESSORS) defineProperty(O, 1, { enumerable: true, get: thrower });\n    else O[1] = 1;\n\n    method.call(O, argument0, argument1);\n  });\n};\n","'use strict';\nvar $forEach = require('../internals/array-iteration').forEach;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('forEach');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('forEach');\n\n// `Array.prototype.forEach` method implementation\n// https://tc39.github.io/ecma262/#sec-array.prototype.foreach\nmodule.exports = (!STRICT_METHOD || !USES_TO_LENGTH) ? function forEach(callbackfn /* , thisArg */) {\n  return $forEach(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n} : [].forEach;\n","'use strict';\nvar $ = require('../internals/export');\nvar forEach = require('../internals/array-for-each');\n\n// `Array.prototype.forEach` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.foreach\n$({ target: 'Array', proto: true, forced: [].forEach != forEach }, {\n  forEach: forEach\n});\n","require('../../../modules/es.array.for-each');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').forEach;\n","var global = require('../internals/global');\nvar userAgent = require('../internals/engine-user-agent');\n\nvar process = global.process;\nvar versions = process && process.versions;\nvar v8 = versions && versions.v8;\nvar match, version;\n\nif (v8) {\n  match = v8.split('.');\n  version = match[0] + match[1];\n} else if (userAgent) {\n  match = userAgent.match(/Edge\\/(\\d+)/);\n  if (!match || match[1] >= 74) {\n    match = userAgent.match(/Chrome\\/(\\d+)/);\n    if (match) version = match[1];\n  }\n}\n\nmodule.exports = version && +version;\n","require('../../modules/web.dom-collections.iterator');\nvar forEach = require('../array/virtual/for-each');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.forEach;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.forEach)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? forEach : own;\n};\n","var getBuiltIn = require('../internals/get-built-in');\n\nmodule.exports = getBuiltIn('navigator', 'userAgent') || '';\n","var fails = require('../internals/fails');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar V8_VERSION = require('../internals/engine-v8-version');\n\nvar SPECIES = wellKnownSymbol('species');\n\nmodule.exports = function (METHOD_NAME) {\n  // We can't use this feature detection in V8 since it causes\n  // deoptimization and serious performance degradation\n  // https://github.com/zloirock/core-js/issues/677\n  return V8_VERSION >= 51 || !fails(function () {\n    var array = [];\n    var constructor = array.constructor = {};\n    constructor[SPECIES] = function () {\n      return { foo: 1 };\n    };\n    return array[METHOD_NAME](Boolean).foo !== 1;\n  });\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $map = require('../internals/array-iteration').map;\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('map');\n// FF49- issue\nvar USES_TO_LENGTH = arrayMethodUsesToLength('map');\n\n// `Array.prototype.map` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.map\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  map: function map(callbackfn /* , thisArg */) {\n    return $map(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.map');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').map;\n","var map = require('../array/virtual/map');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.map;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.map) ? map : own;\n};\n","var global = require('../internals/global');\nvar trim = require('../internals/string-trim').trim;\nvar whitespaces = require('../internals/whitespaces');\n\nvar $parseInt = global.parseInt;\nvar hex = /^[+-]?0[Xx]/;\nvar FORCED = $parseInt(whitespaces + '08') !== 8 || $parseInt(whitespaces + '0x16') !== 22;\n\n// `parseInt` method\n// https://tc39.github.io/ecma262/#sec-parseint-string-radix\nmodule.exports = FORCED ? function parseInt(string, radix) {\n  var S = trim(String(string));\n  return $parseInt(S, (radix >>> 0) || (hex.test(S) ? 16 : 10));\n} : $parseInt;\n","var $ = require('../internals/export');\nvar parseIntImplementation = require('../internals/number-parse-int');\n\n// `parseInt` method\n// https://tc39.github.io/ecma262/#sec-parseint-string-radix\n$({ global: true, forced: parseInt != parseIntImplementation }, {\n  parseInt: parseIntImplementation\n});\n","require('../modules/es.parse-int');\nvar path = require('../internals/path');\n\nmodule.exports = path.parseInt;\n","var DESCRIPTORS = require('../internals/descriptors');\nvar objectKeys = require('../internals/object-keys');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar propertyIsEnumerable = require('../internals/object-property-is-enumerable').f;\n\n// `Object.{ entries, values }` methods implementation\nvar createMethod = function (TO_ENTRIES) {\n  return function (it) {\n    var O = toIndexedObject(it);\n    var keys = objectKeys(O);\n    var length = keys.length;\n    var i = 0;\n    var result = [];\n    var key;\n    while (length > i) {\n      key = keys[i++];\n      if (!DESCRIPTORS || propertyIsEnumerable.call(O, key)) {\n        result.push(TO_ENTRIES ? [key, O[key]] : O[key]);\n      }\n    }\n    return result;\n  };\n};\n\nmodule.exports = {\n  // `Object.entries` method\n  // https://tc39.github.io/ecma262/#sec-object.entries\n  entries: createMethod(true),\n  // `Object.values` method\n  // https://tc39.github.io/ecma262/#sec-object.values\n  values: createMethod(false)\n};\n","var $ = require('../internals/export');\nvar $values = require('../internals/object-to-array').values;\n\n// `Object.values` method\n// https://tc39.github.io/ecma262/#sec-object.values\n$({ target: 'Object', stat: true }, {\n  values: function values(O) {\n    return $values(O);\n  }\n});\n","require('../../modules/es.object.values');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.values;\n","'use strict';\nvar $ = require('../internals/export');\nvar $filter = require('../internals/array-iteration').filter;\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('filter');\n// Edge 14- issue\nvar USES_TO_LENGTH = arrayMethodUsesToLength('filter');\n\n// `Array.prototype.filter` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.filter\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  filter: function filter(callbackfn /* , thisArg */) {\n    return $filter(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.filter');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').filter;\n","var filter = require('../array/virtual/filter');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.filter;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.filter) ? filter : own;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar isArray = require('../internals/is-array');\nvar isObject = require('../internals/is-object');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar createProperty = require('../internals/create-property');\nvar arraySpeciesCreate = require('../internals/array-species-create');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar V8_VERSION = require('../internals/engine-v8-version');\n\nvar IS_CONCAT_SPREADABLE = wellKnownSymbol('isConcatSpreadable');\nvar MAX_SAFE_INTEGER = 0x1FFFFFFFFFFFFF;\nvar MAXIMUM_ALLOWED_INDEX_EXCEEDED = 'Maximum allowed index exceeded';\n\n// We can't use this feature detection in V8 since it causes\n// deoptimization and serious performance degradation\n// https://github.com/zloirock/core-js/issues/679\nvar IS_CONCAT_SPREADABLE_SUPPORT = V8_VERSION >= 51 || !fails(function () {\n  var array = [];\n  array[IS_CONCAT_SPREADABLE] = false;\n  return array.concat()[0] !== array;\n});\n\nvar SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('concat');\n\nvar isConcatSpreadable = function (O) {\n  if (!isObject(O)) return false;\n  var spreadable = O[IS_CONCAT_SPREADABLE];\n  return spreadable !== undefined ? !!spreadable : isArray(O);\n};\n\nvar FORCED = !IS_CONCAT_SPREADABLE_SUPPORT || !SPECIES_SUPPORT;\n\n// `Array.prototype.concat` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.concat\n// with adding support of @@isConcatSpreadable and @@species\n$({ target: 'Array', proto: true, forced: FORCED }, {\n  concat: function concat(arg) { // eslint-disable-line no-unused-vars\n    var O = toObject(this);\n    var A = arraySpeciesCreate(O, 0);\n    var n = 0;\n    var i, k, length, len, E;\n    for (i = -1, length = arguments.length; i < length; i++) {\n      E = i === -1 ? O : arguments[i];\n      if (isConcatSpreadable(E)) {\n        len = toLength(E.length);\n        if (n + len > MAX_SAFE_INTEGER) throw TypeError(MAXIMUM_ALLOWED_INDEX_EXCEEDED);\n        for (k = 0; k < len; k++, n++) if (k in E) createProperty(A, n, E[k]);\n      } else {\n        if (n >= MAX_SAFE_INTEGER) throw TypeError(MAXIMUM_ALLOWED_INDEX_EXCEEDED);\n        createProperty(A, n++, E);\n      }\n    }\n    A.length = n;\n    return A;\n  }\n});\n","require('../../../modules/es.array.concat');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').concat;\n","var concat = require('../array/virtual/concat');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.concat;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.concat) ? concat : own;\n};\n","var $ = require('../internals/export');\nvar isArray = require('../internals/is-array');\n\n// `Array.isArray` method\n// https://tc39.github.io/ecma262/#sec-array.isarray\n$({ target: 'Array', stat: true }, {\n  isArray: isArray\n});\n","require('../../modules/es.array.is-array');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Array.isArray;\n","var parent = require('../../es/array/is-array');\n\nmodule.exports = parent;\n","var _Array$isArray = require(\"../core-js/array/is-array\");\n\nfunction _arrayWithoutHoles(arr) {\n  if (_Array$isArray(arr)) {\n    for (var i = 0, arr2 = new Array(arr.length); i < arr.length; i++) {\n      arr2[i] = arr[i];\n    }\n\n    return arr2;\n  }\n}\n\nmodule.exports = _arrayWithoutHoles;","var anObject = require('../internals/an-object');\n\n// call something on iterator step with safe closing on error\nmodule.exports = function (iterator, fn, value, ENTRIES) {\n  try {\n    return ENTRIES ? fn(anObject(value)[0], value[1]) : fn(value);\n  // 7.4.6 IteratorClose(iterator, completion)\n  } catch (error) {\n    var returnMethod = iterator['return'];\n    if (returnMethod !== undefined) anObject(returnMethod.call(iterator));\n    throw error;\n  }\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\nvar Iterators = require('../internals/iterators');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar ArrayPrototype = Array.prototype;\n\n// check on default Array iterator\nmodule.exports = function (it) {\n  return it !== undefined && (Iterators.Array === it || ArrayPrototype[ITERATOR] === it);\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar ITERATOR = wellKnownSymbol('iterator');\nvar SAFE_CLOSING = false;\n\ntry {\n  var called = 0;\n  var iteratorWithReturn = {\n    next: function () {\n      return { done: !!called++ };\n    },\n    'return': function () {\n      SAFE_CLOSING = true;\n    }\n  };\n  iteratorWithReturn[ITERATOR] = function () {\n    return this;\n  };\n  // eslint-disable-next-line no-throw-literal\n  Array.from(iteratorWithReturn, function () { throw 2; });\n} catch (error) { /* empty */ }\n\nmodule.exports = function (exec, SKIP_CLOSING) {\n  if (!SKIP_CLOSING && !SAFE_CLOSING) return false;\n  var ITERATION_SUPPORT = false;\n  try {\n    var object = {};\n    object[ITERATOR] = function () {\n      return {\n        next: function () {\n          return { done: ITERATION_SUPPORT = true };\n        }\n      };\n    };\n    exec(object);\n  } catch (error) { /* empty */ }\n  return ITERATION_SUPPORT;\n};\n","var $ = require('../internals/export');\nvar from = require('../internals/array-from');\nvar checkCorrectnessOfIteration = require('../internals/check-correctness-of-iteration');\n\nvar INCORRECT_ITERATION = !checkCorrectnessOfIteration(function (iterable) {\n  Array.from(iterable);\n});\n\n// `Array.from` method\n// https://tc39.github.io/ecma262/#sec-array.from\n$({ target: 'Array', stat: true, forced: INCORRECT_ITERATION }, {\n  from: from\n});\n","'use strict';\nvar bind = require('../internals/function-bind-context');\nvar toObject = require('../internals/to-object');\nvar callWithSafeIterationClosing = require('../internals/call-with-safe-iteration-closing');\nvar isArrayIteratorMethod = require('../internals/is-array-iterator-method');\nvar toLength = require('../internals/to-length');\nvar createProperty = require('../internals/create-property');\nvar getIteratorMethod = require('../internals/get-iterator-method');\n\n// `Array.from` method implementation\n// https://tc39.github.io/ecma262/#sec-array.from\nmodule.exports = function from(arrayLike /* , mapfn = undefined, thisArg = undefined */) {\n  var O = toObject(arrayLike);\n  var C = typeof this == 'function' ? this : Array;\n  var argumentsLength = arguments.length;\n  var mapfn = argumentsLength > 1 ? arguments[1] : undefined;\n  var mapping = mapfn !== undefined;\n  var iteratorMethod = getIteratorMethod(O);\n  var index = 0;\n  var length, result, step, iterator, next, value;\n  if (mapping) mapfn = bind(mapfn, argumentsLength > 2 ? arguments[2] : undefined, 2);\n  // if the target is not iterable or it's an array with the default iterator - use a simple case\n  if (iteratorMethod != undefined && !(C == Array && isArrayIteratorMethod(iteratorMethod))) {\n    iterator = iteratorMethod.call(O);\n    next = iterator.next;\n    result = new C();\n    for (;!(step = next.call(iterator)).done; index++) {\n      value = mapping ? callWithSafeIterationClosing(iterator, mapfn, [step.value, index], true) : step.value;\n      createProperty(result, index, value);\n    }\n  } else {\n    length = toLength(O.length);\n    result = new C(length);\n    for (;length > index; index++) {\n      value = mapping ? mapfn(O[index], index) : O[index];\n      createProperty(result, index, value);\n    }\n  }\n  result.length = index;\n  return result;\n};\n","require('../../modules/es.string.iterator');\nrequire('../../modules/es.array.from');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Array.from;\n","var classof = require('../internals/classof');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar Iterators = require('../internals/iterators');\n\nvar ITERATOR = wellKnownSymbol('iterator');\n\nmodule.exports = function (it) {\n  var O = Object(it);\n  return O[ITERATOR] !== undefined\n    || '@@iterator' in O\n    // eslint-disable-next-line no-prototype-builtins\n    || Iterators.hasOwnProperty(classof(O));\n};\n","var _Array$from = require(\"../core-js/array/from\");\n\nvar _isIterable = require(\"../core-js/is-iterable\");\n\nfunction _iterableToArray(iter) {\n  if (_isIterable(Object(iter)) || Object.prototype.toString.call(iter) === \"[object Arguments]\") return _Array$from(iter);\n}\n\nmodule.exports = _iterableToArray;","function _nonIterableSpread() {\n  throw new TypeError(\"Invalid attempt to spread non-iterable instance\");\n}\n\nmodule.exports = _nonIterableSpread;","var arrayWithoutHoles = require(\"./arrayWithoutHoles\");\n\nvar iterableToArray = require(\"./iterableToArray\");\n\nvar nonIterableSpread = require(\"./nonIterableSpread\");\n\nfunction _toConsumableArray(arr) {\n  return arrayWithoutHoles(arr) || iterableToArray(arr) || nonIterableSpread();\n}\n\nmodule.exports = _toConsumableArray;","'use strict';\nvar $ = require('../internals/export');\nvar isObject = require('../internals/is-object');\nvar isArray = require('../internals/is-array');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toLength = require('../internals/to-length');\nvar toIndexedObject = require('../internals/to-indexed-object');\nvar createProperty = require('../internals/create-property');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('slice');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('slice', { ACCESSORS: true, 0: 0, 1: 2 });\n\nvar SPECIES = wellKnownSymbol('species');\nvar nativeSlice = [].slice;\nvar max = Math.max;\n\n// `Array.prototype.slice` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.slice\n// fallback for not array-like ES3 strings and DOM objects\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  slice: function slice(start, end) {\n    var O = toIndexedObject(this);\n    var length = toLength(O.length);\n    var k = toAbsoluteIndex(start, length);\n    var fin = toAbsoluteIndex(end === undefined ? length : end, length);\n    // inline `ArraySpeciesCreate` for usage native `Array#slice` where it's possible\n    var Constructor, result, n;\n    if (isArray(O)) {\n      Constructor = O.constructor;\n      // cross-realm fallback\n      if (typeof Constructor == 'function' && (Constructor === Array || isArray(Constructor.prototype))) {\n        Constructor = undefined;\n      } else if (isObject(Constructor)) {\n        Constructor = Constructor[SPECIES];\n        if (Constructor === null) Constructor = undefined;\n      }\n      if (Constructor === Array || Constructor === undefined) {\n        return nativeSlice.call(O, k, fin);\n      }\n    }\n    result = new (Constructor === undefined ? Array : Constructor)(max(fin - k, 0));\n    for (n = 0; k < fin; k++, n++) if (k in O) createProperty(result, n, O[k]);\n    result.length = n;\n    return result;\n  }\n});\n","require('../../../modules/es.array.slice');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').slice;\n","var slice = require('../array/virtual/slice');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.slice;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.slice) ? slice : own;\n};\n","var $ = require('../internals/export');\nvar fails = require('../internals/fails');\nvar toObject = require('../internals/to-object');\nvar nativeGetPrototypeOf = require('../internals/object-get-prototype-of');\nvar CORRECT_PROTOTYPE_GETTER = require('../internals/correct-prototype-getter');\n\nvar FAILS_ON_PRIMITIVES = fails(function () { nativeGetPrototypeOf(1); });\n\n// `Object.getPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.getprototypeof\n$({ target: 'Object', stat: true, forced: FAILS_ON_PRIMITIVES, sham: !CORRECT_PROTOTYPE_GETTER }, {\n  getPrototypeOf: function getPrototypeOf(it) {\n    return nativeGetPrototypeOf(toObject(it));\n  }\n});\n\n","require('../../modules/es.object.get-prototype-of');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.getPrototypeOf;\n","var parent = require('../../es/object/get-prototype-of');\n\nmodule.exports = parent;\n","'use strict';\nvar $ = require('../internals/export');\nvar $indexOf = require('../internals/array-includes').indexOf;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar nativeIndexOf = [].indexOf;\n\nvar NEGATIVE_ZERO = !!nativeIndexOf && 1 / [1].indexOf(1, -0) < 0;\nvar STRICT_METHOD = arrayMethodIsStrict('indexOf');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('indexOf', { ACCESSORS: true, 1: 0 });\n\n// `Array.prototype.indexOf` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.indexof\n$({ target: 'Array', proto: true, forced: NEGATIVE_ZERO || !STRICT_METHOD || !USES_TO_LENGTH }, {\n  indexOf: function indexOf(searchElement /* , fromIndex = 0 */) {\n    return NEGATIVE_ZERO\n      // convert -0 to +0\n      ? nativeIndexOf.apply(this, arguments) || 0\n      : $indexOf(this, searchElement, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.index-of');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').indexOf;\n","var indexOf = require('../array/virtual/index-of');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.indexOf;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.indexOf) ? indexOf : own;\n};\n","var parent = require('../../es/array/is-array');\n\nmodule.exports = parent;\n","'use strict';\nvar DESCRIPTORS = require('../internals/descriptors');\nvar fails = require('../internals/fails');\nvar objectKeys = require('../internals/object-keys');\nvar getOwnPropertySymbolsModule = require('../internals/object-get-own-property-symbols');\nvar propertyIsEnumerableModule = require('../internals/object-property-is-enumerable');\nvar toObject = require('../internals/to-object');\nvar IndexedObject = require('../internals/indexed-object');\n\nvar nativeAssign = Object.assign;\nvar defineProperty = Object.defineProperty;\n\n// `Object.assign` method\n// https://tc39.github.io/ecma262/#sec-object.assign\nmodule.exports = !nativeAssign || fails(function () {\n  // should have correct order of operations (Edge bug)\n  if (DESCRIPTORS && nativeAssign({ b: 1 }, nativeAssign(defineProperty({}, 'a', {\n    enumerable: true,\n    get: function () {\n      defineProperty(this, 'b', {\n        value: 3,\n        enumerable: false\n      });\n    }\n  }), { b: 2 })).b !== 1) return true;\n  // should work with symbols and should have deterministic property order (V8 bug)\n  var A = {};\n  var B = {};\n  // eslint-disable-next-line no-undef\n  var symbol = Symbol();\n  var alphabet = 'abcdefghijklmnopqrst';\n  A[symbol] = 7;\n  alphabet.split('').forEach(function (chr) { B[chr] = chr; });\n  return nativeAssign({}, A)[symbol] != 7 || objectKeys(nativeAssign({}, B)).join('') != alphabet;\n}) ? function assign(target, source) { // eslint-disable-line no-unused-vars\n  var T = toObject(target);\n  var argumentsLength = arguments.length;\n  var index = 1;\n  var getOwnPropertySymbols = getOwnPropertySymbolsModule.f;\n  var propertyIsEnumerable = propertyIsEnumerableModule.f;\n  while (argumentsLength > index) {\n    var S = IndexedObject(arguments[index++]);\n    var keys = getOwnPropertySymbols ? objectKeys(S).concat(getOwnPropertySymbols(S)) : objectKeys(S);\n    var length = keys.length;\n    var j = 0;\n    var key;\n    while (length > j) {\n      key = keys[j++];\n      if (!DESCRIPTORS || propertyIsEnumerable.call(S, key)) T[key] = S[key];\n    }\n  } return T;\n} : nativeAssign;\n","var $ = require('../internals/export');\nvar assign = require('../internals/object-assign');\n\n// `Object.assign` method\n// https://tc39.github.io/ecma262/#sec-object.assign\n$({ target: 'Object', stat: true, forced: Object.assign !== assign }, {\n  assign: assign\n});\n","require('../../modules/es.object.assign');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.assign;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.iterator` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.iterator\ndefineWellKnownSymbol('iterator');\n","require('../../modules/es.symbol.iterator');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar WrappedWellKnownSymbolModule = require('../../internals/well-known-symbol-wrapped');\n\nmodule.exports = WrappedWellKnownSymbolModule.f('iterator');\n","var parent = require('../../es/symbol/iterator');\n\nmodule.exports = parent;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.asyncIterator` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.asynciterator\ndefineWellKnownSymbol('asyncIterator');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.hasInstance` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.hasinstance\ndefineWellKnownSymbol('hasInstance');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.isConcatSpreadable` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.isconcatspreadable\ndefineWellKnownSymbol('isConcatSpreadable');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.match` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.match\ndefineWellKnownSymbol('match');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.matchAll` well-known symbol\ndefineWellKnownSymbol('matchAll');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.replace` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.replace\ndefineWellKnownSymbol('replace');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.search` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.search\ndefineWellKnownSymbol('search');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.species` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.species\ndefineWellKnownSymbol('species');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.split` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.split\ndefineWellKnownSymbol('split');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.toPrimitive` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.toprimitive\ndefineWellKnownSymbol('toPrimitive');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.toStringTag` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.tostringtag\ndefineWellKnownSymbol('toStringTag');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.unscopables` well-known symbol\n// https://tc39.github.io/ecma262/#sec-symbol.unscopables\ndefineWellKnownSymbol('unscopables');\n","var setToStringTag = require('../internals/set-to-string-tag');\n\n// Math[@@toStringTag] property\n// https://tc39.github.io/ecma262/#sec-math-@@tostringtag\nsetToStringTag(Math, 'Math', true);\n","var global = require('../internals/global');\nvar setToStringTag = require('../internals/set-to-string-tag');\n\n// JSON[@@toStringTag] property\n// https://tc39.github.io/ecma262/#sec-json-@@tostringtag\nsetToStringTag(global.JSON, 'JSON', true);\n","require('../../modules/es.array.concat');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.symbol');\nrequire('../../modules/es.symbol.async-iterator');\nrequire('../../modules/es.symbol.description');\nrequire('../../modules/es.symbol.has-instance');\nrequire('../../modules/es.symbol.is-concat-spreadable');\nrequire('../../modules/es.symbol.iterator');\nrequire('../../modules/es.symbol.match');\nrequire('../../modules/es.symbol.match-all');\nrequire('../../modules/es.symbol.replace');\nrequire('../../modules/es.symbol.search');\nrequire('../../modules/es.symbol.species');\nrequire('../../modules/es.symbol.split');\nrequire('../../modules/es.symbol.to-primitive');\nrequire('../../modules/es.symbol.to-string-tag');\nrequire('../../modules/es.symbol.unscopables');\nrequire('../../modules/es.math.to-string-tag');\nrequire('../../modules/es.json.to-string-tag');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Symbol;\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.asyncDispose` well-known symbol\n// https://github.com/tc39/proposal-using-statement\ndefineWellKnownSymbol('asyncDispose');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.dispose` well-known symbol\n// https://github.com/tc39/proposal-using-statement\ndefineWellKnownSymbol('dispose');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.observable` well-known symbol\n// https://github.com/tc39/proposal-observable\ndefineWellKnownSymbol('observable');\n","var defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\n// `Symbol.patternMatch` well-known symbol\n// https://github.com/tc39/proposal-pattern-matching\ndefineWellKnownSymbol('patternMatch');\n","// TODO: remove from `core-js@4`\nvar defineWellKnownSymbol = require('../internals/define-well-known-symbol');\n\ndefineWellKnownSymbol('replaceAll');\n","var parent = require('../../es/symbol');\nrequire('../../modules/esnext.symbol.async-dispose');\nrequire('../../modules/esnext.symbol.dispose');\nrequire('../../modules/esnext.symbol.observable');\nrequire('../../modules/esnext.symbol.pattern-match');\n// TODO: Remove from `core-js@4`\nrequire('../../modules/esnext.symbol.replace-all');\n\nmodule.exports = parent;\n","var _Symbol$iterator = require(\"../core-js/symbol/iterator\");\n\nvar _Symbol = require(\"../core-js/symbol\");\n\nfunction _typeof(obj) {\n  \"@babel/helpers - typeof\";\n\n  if (typeof _Symbol === \"function\" && typeof _Symbol$iterator === \"symbol\") {\n    module.exports = _typeof = function _typeof(obj) {\n      return typeof obj;\n    };\n  } else {\n    module.exports = _typeof = function _typeof(obj) {\n      return obj && typeof _Symbol === \"function\" && obj.constructor === _Symbol && obj !== _Symbol.prototype ? \"symbol\" : typeof obj;\n    };\n  }\n\n  return _typeof(obj);\n}\n\nmodule.exports = _typeof;","var _Array$isArray = require(\"../core-js/array/is-array\");\n\nfunction _arrayWithHoles(arr) {\n  if (_Array$isArray(arr)) return arr;\n}\n\nmodule.exports = _arrayWithHoles;","var _getIterator = require(\"../core-js/get-iterator\");\n\nvar _isIterable = require(\"../core-js/is-iterable\");\n\nfunction _iterableToArrayLimit(arr, i) {\n  if (!(_isIterable(Object(arr)) || Object.prototype.toString.call(arr) === \"[object Arguments]\")) {\n    return;\n  }\n\n  var _arr = [];\n  var _n = true;\n  var _d = false;\n  var _e = undefined;\n\n  try {\n    for (var _i = _getIterator(arr), _s; !(_n = (_s = _i.next()).done); _n = true) {\n      _arr.push(_s.value);\n\n      if (i && _arr.length === i) break;\n    }\n  } catch (err) {\n    _d = true;\n    _e = err;\n  } finally {\n    try {\n      if (!_n && _i[\"return\"] != null) _i[\"return\"]();\n    } finally {\n      if (_d) throw _e;\n    }\n  }\n\n  return _arr;\n}\n\nmodule.exports = _iterableToArrayLimit;","function _nonIterableRest() {\n  throw new TypeError(\"Invalid attempt to destructure non-iterable instance\");\n}\n\nmodule.exports = _nonIterableRest;","var arrayWithHoles = require(\"./arrayWithHoles\");\n\nvar iterableToArrayLimit = require(\"./iterableToArrayLimit\");\n\nvar nonIterableRest = require(\"./nonIterableRest\");\n\nfunction _slicedToArray(arr, i) {\n  return arrayWithHoles(arr) || iterableToArrayLimit(arr, i) || nonIterableRest();\n}\n\nmodule.exports = _slicedToArray;","var $ = require('../internals/export');\n\n// `Date.now` method\n// https://tc39.github.io/ecma262/#sec-date.now\n$({ target: 'Date', stat: true }, {\n  now: function now() {\n    return new Date().getTime();\n  }\n});\n","require('../../modules/es.date.now');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Date.now;\n","/**\n * vis-util\n * https://github.com/visjs/vis-util\n *\n * utilitie collection for visjs\n *\n * @version 4.0.0\n * @date    2020-02-29T15:35:13.178Z\n *\n * @copyright (c) 2011-2017 Almende B.V, http://almende.com\n * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs\n *\n * @license\n * vis.js is dual licensed under both\n *\n *   1. The Apache 2.0 License\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *   and\n *\n *   2. The MIT License\n *      http://opensource.org/licenses/MIT\n *\n * vis.js may be distributed under either license.\n */\n\n/**\r\n * Seedable, fast and reasonably good (not crypto but more than okay for our\r\n * needs) random number generator.\r\n *\r\n * @remarks\r\n * Adapted from {@link https://web.archive.org/web/20110429100736/http://baagoe.com:80/en/RandomMusings/javascript}.\r\n * Original algorithm created by Johannes Baagøe \\<baagoe\\@baagoe.com\\> in 2010.\r\n */\r\n/**\r\n * Create a seeded pseudo random generator based on Alea by Johannes Baagøe.\r\n *\r\n * @param seed - All supplied arguments will be used as a seed. In case nothing\r\n * is supplied the current time will be used to seed the generator.\r\n *\r\n * @returns A ready to use seeded generator.\r\n */\r\nfunction Alea(...seed) {\r\n    return AleaImplementation(seed.length ? seed : [Date.now()]);\r\n}\r\n/**\r\n * An implementation of [[Alea]] without user input validation.\r\n *\r\n * @param seed - The data that will be used to seed the generator.\r\n *\r\n * @returns A ready to use seeded generator.\r\n */\r\nfunction AleaImplementation(seed) {\r\n    let [s0, s1, s2] = mashSeed(seed);\r\n    let c = 1;\r\n    const random = () => {\r\n        const t = 2091639 * s0 + c * 2.3283064365386963e-10; // 2^-32\r\n        s0 = s1;\r\n        s1 = s2;\r\n        return (s2 = t - (c = t | 0));\r\n    };\r\n    random.uint32 = () => random() * 0x100000000; // 2^32\r\n    random.fract53 = () => random() + ((random() * 0x200000) | 0) * 1.1102230246251565e-16; // 2^-53\r\n    random.algorithm = \"Alea\";\r\n    random.seed = seed;\r\n    random.version = \"0.9\";\r\n    return random;\r\n}\r\n/**\r\n * Turn arbitrary data into values [[AleaImplementation]] can use to generate\r\n * random numbers.\r\n *\r\n * @param seed - Arbitrary data that will be used as the seed.\r\n *\r\n * @returns Three numbers to use as initial values for [[AleaImplementation]].\r\n */\r\nfunction mashSeed(...seed) {\r\n    const mash = Mash();\r\n    let s0 = mash(\" \");\r\n    let s1 = mash(\" \");\r\n    let s2 = mash(\" \");\r\n    for (let i = 0; i < seed.length; i++) {\r\n        s0 -= mash(seed[i]);\r\n        if (s0 < 0) {\r\n            s0 += 1;\r\n        }\r\n        s1 -= mash(seed[i]);\r\n        if (s1 < 0) {\r\n            s1 += 1;\r\n        }\r\n        s2 -= mash(seed[i]);\r\n        if (s2 < 0) {\r\n            s2 += 1;\r\n        }\r\n    }\r\n    return [s0, s1, s2];\r\n}\r\n/**\r\n * Create a new mash function.\r\n *\r\n * @returns A nonpure function that takes arbitrary [[Mashable]] data and turns\r\n * them into numbers.\r\n */\r\nfunction Mash() {\r\n    let n = 0xefc8249d;\r\n    return function (data) {\r\n        const string = data.toString();\r\n        for (let i = 0; i < string.length; i++) {\r\n            n += string.charCodeAt(i);\r\n            let h = 0.02519603282416938 * n;\r\n            n = h >>> 0;\r\n            h -= n;\r\n            h *= n;\r\n            n = h >>> 0;\r\n            h -= n;\r\n            n += h * 0x100000000; // 2^32\r\n        }\r\n        return (n >>> 0) * 2.3283064365386963e-10; // 2^-32\r\n    };\r\n}\n\n// utility functions\r\n// parse ASP.Net Date pattern,\r\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\r\n// code from http://momentjs.com/\r\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\r\n// Color REs\r\nconst fullHexRE = /^#?([a-f\\d]{2})([a-f\\d]{2})([a-f\\d]{2})$/i;\r\nconst shortHexRE = /^#?([a-f\\d])([a-f\\d])([a-f\\d])$/i;\r\nconst rgbRE = /^rgb\\( *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *\\)$/i;\r\nconst rgbaRE = /^rgba\\( *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *(1?\\d{1,2}|2[0-4]\\d|25[0-5]) *, *([01]|0?\\.\\d+) *\\)$/i;\r\n/**\r\n * Test whether given object is a number.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if number, false otherwise.\r\n */\r\nfunction isNumber(value) {\r\n    return value instanceof Number || typeof value === \"number\";\r\n}\r\n/**\r\n * Remove everything in the DOM object.\r\n *\r\n * @param DOMobject - Node whose child nodes will be recursively deleted.\r\n */\r\nfunction recursiveDOMDelete(DOMobject) {\r\n    if (DOMobject) {\r\n        while (DOMobject.hasChildNodes() === true) {\r\n            const child = DOMobject.firstChild;\r\n            if (child) {\r\n                recursiveDOMDelete(child);\r\n                DOMobject.removeChild(child);\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Test whether given object is a string.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if string, false otherwise.\r\n */\r\nfunction isString(value) {\r\n    return value instanceof String || typeof value === \"string\";\r\n}\r\n/**\r\n * Test whether given object is a object (not primitive or null).\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if not null object, false otherwise.\r\n */\r\nfunction isObject(value) {\r\n    return typeof value === \"object\" && value !== null;\r\n}\r\n/**\r\n * Test whether given object is a Date, or a String containing a Date.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if Date instance or string date representation, false otherwise.\r\n */\r\nfunction isDate(value) {\r\n    if (value instanceof Date) {\r\n        return true;\r\n    }\r\n    else if (isString(value)) {\r\n        // test whether this string contains a date\r\n        const match = ASPDateRegex.exec(value);\r\n        if (match) {\r\n            return true;\r\n        }\r\n        else if (!isNaN(Date.parse(value))) {\r\n            return true;\r\n        }\r\n    }\r\n    return false;\r\n}\r\n/**\r\n * Copy property from b to a if property present in a.\r\n * If property in b explicitly set to null, delete it if `allowDeletion` set.\r\n *\r\n * Internal helper routine, should not be exported. Not added to `exports` for that reason.\r\n *\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param prop - Name of property to copy from b to a.\r\n * @param allowDeletion - If true, delete property in a if explicitly set to null in b.\r\n */\r\nfunction copyOrDelete(a, b, prop, allowDeletion) {\r\n    let doDeletion = false;\r\n    if (allowDeletion === true) {\r\n        doDeletion = b[prop] === null && a[prop] !== undefined;\r\n    }\r\n    if (doDeletion) {\r\n        delete a[prop];\r\n    }\r\n    else {\r\n        a[prop] = b[prop]; // Remember, this is a reference copy!\r\n    }\r\n}\r\n/**\r\n * Fill an object with a possibly partially defined other object.\r\n *\r\n * Only copies values for the properties already present in a.\r\n * That means an object is not created on a property if only the b object has it.\r\n *\r\n * @param a - The object that will have it's properties updated.\r\n * @param b - The object with property updates.\r\n * @param allowDeletion - If true, delete properties in a that are explicitly set to null in b.\r\n */\r\nfunction fillIfDefined(a, b, allowDeletion = false) {\r\n    // NOTE: iteration of properties of a\r\n    // NOTE: prototype properties iterated over as well\r\n    for (const prop in a) {\r\n        if (b[prop] !== undefined) {\r\n            if (b[prop] === null || typeof b[prop] !== \"object\") {\r\n                // Note: typeof null === 'object'\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n            else {\r\n                const aProp = a[prop];\r\n                const bProp = b[prop];\r\n                if (isObject(aProp) && isObject(bProp)) {\r\n                    fillIfDefined(aProp, bProp, allowDeletion);\r\n                }\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Copy the values of all of the enumerable own properties from one or more source objects to a\r\n * target object. Returns the target object.\r\n *\r\n * @param target - The target object to copy to.\r\n * @param source - The source object from which to copy properties.\r\n *\r\n * @returns The target object.\r\n */\r\nconst extend = Object.assign;\r\n/**\r\n * Extend object a with selected properties of object b or a series of objects.\r\n *\r\n * @remarks\r\n * Only properties with defined values are copied.\r\n *\r\n * @param props - Properties to be copied to a.\r\n * @param a - The target.\r\n * @param others - The sources.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveExtend(props, a, ...others) {\r\n    if (!Array.isArray(props)) {\r\n        throw new Error(\"Array with property names expected as first argument\");\r\n    }\r\n    for (const other of others) {\r\n        for (let p = 0; p < props.length; p++) {\r\n            const prop = props[p];\r\n            if (other && Object.prototype.hasOwnProperty.call(other, prop)) {\r\n                a[prop] = other[prop];\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Extend object a with selected properties of object b.\r\n * Only properties with defined values are copied.\r\n *\r\n * @remarks\r\n * Previous version of this routine implied that multiple source objects could\r\n * be used; however, the implementation was **wrong**. Since multiple (\\>1)\r\n * sources weren't used anywhere in the `vis.js` code, this has been removed\r\n *\r\n * @param props - Names of first-level properties to copy over.\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param allowDeletion - If true, delete property in a if explicitly set to null in b.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveDeepExtend(props, a, b, allowDeletion = false) {\r\n    // TODO: add support for Arrays to deepExtend\r\n    if (Array.isArray(b)) {\r\n        throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n    }\r\n    for (let p = 0; p < props.length; p++) {\r\n        const prop = props[p];\r\n        if (Object.prototype.hasOwnProperty.call(b, prop)) {\r\n            if (b[prop] && b[prop].constructor === Object) {\r\n                if (a[prop] === undefined) {\r\n                    a[prop] = {};\r\n                }\r\n                if (a[prop].constructor === Object) {\r\n                    deepExtend(a[prop], b[prop], false, allowDeletion);\r\n                }\r\n                else {\r\n                    copyOrDelete(a, b, prop, allowDeletion);\r\n                }\r\n            }\r\n            else if (Array.isArray(b[prop])) {\r\n                throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Extend object `a` with properties of object `b`, ignoring properties which\r\n * are explicitly specified to be excluded.\r\n *\r\n * @remarks\r\n * The properties of `b` are considered for copying. Properties which are\r\n * themselves objects are are also extended. Only properties with defined\r\n * values are copied.\r\n *\r\n * @param propsToExclude - Names of properties which should *not* be copied.\r\n * @param a - Object to extend.\r\n * @param b - Object to take properties from for extension.\r\n * @param allowDeletion - If true, delete properties in a that are explicitly\r\n * set to null in b.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction selectiveNotDeepExtend(propsToExclude, a, b, allowDeletion = false) {\r\n    // TODO: add support for Arrays to deepExtend\r\n    // NOTE: array properties have an else-below; apparently, there is a problem here.\r\n    if (Array.isArray(b)) {\r\n        throw new TypeError(\"Arrays are not supported by deepExtend\");\r\n    }\r\n    for (const prop in b) {\r\n        if (!Object.prototype.hasOwnProperty.call(b, prop)) {\r\n            continue;\r\n        } // Handle local properties only\r\n        if (propsToExclude.indexOf(prop) !== -1) {\r\n            continue;\r\n        } // In exclusion list, skip\r\n        if (b[prop] && b[prop].constructor === Object) {\r\n            if (a[prop] === undefined) {\r\n                a[prop] = {};\r\n            }\r\n            if (a[prop].constructor === Object) {\r\n                deepExtend(a[prop], b[prop]); // NOTE: allowDeletion not propagated!\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n        else if (Array.isArray(b[prop])) {\r\n            a[prop] = [];\r\n            for (let i = 0; i < b[prop].length; i++) {\r\n                a[prop].push(b[prop][i]);\r\n            }\r\n        }\r\n        else {\r\n            copyOrDelete(a, b, prop, allowDeletion);\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Deep extend an object a with the properties of object b.\r\n *\r\n * @param a - Target object.\r\n * @param b - Source object.\r\n * @param protoExtend - If true, the prototype values will also be extended.\r\n * (That is the options objects that inherit from others will also get the\r\n * inherited options).\r\n * @param allowDeletion - If true, the values of fields that are null will be deleted.\r\n *\r\n * @returns Argument a.\r\n */\r\nfunction deepExtend(a, b, protoExtend = false, allowDeletion = false) {\r\n    for (const prop in b) {\r\n        if (Object.prototype.hasOwnProperty.call(b, prop) || protoExtend === true) {\r\n            if (typeof b[prop] === \"object\" &&\r\n                b[prop] !== null &&\r\n                Object.getPrototypeOf(b[prop]) === Object.prototype) {\r\n                if (a[prop] === undefined) {\r\n                    a[prop] = deepExtend({}, b[prop], protoExtend); // NOTE: allowDeletion not propagated!\r\n                }\r\n                else if (typeof a[prop] === \"object\" &&\r\n                    a[prop] !== null &&\r\n                    Object.getPrototypeOf(a[prop]) === Object.prototype) {\r\n                    deepExtend(a[prop], b[prop], protoExtend); // NOTE: allowDeletion not propagated!\r\n                }\r\n                else {\r\n                    copyOrDelete(a, b, prop, allowDeletion);\r\n                }\r\n            }\r\n            else if (Array.isArray(b[prop])) {\r\n                a[prop] = b[prop].slice();\r\n            }\r\n            else {\r\n                copyOrDelete(a, b, prop, allowDeletion);\r\n            }\r\n        }\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * Test whether all elements in two arrays are equal.\r\n *\r\n * @param a - First array.\r\n * @param b - Second array.\r\n *\r\n * @returns True if both arrays have the same length and same elements (1 = '1').\r\n */\r\nfunction equalArray(a, b) {\r\n    if (a.length !== b.length) {\r\n        return false;\r\n    }\r\n    for (let i = 0, len = a.length; i < len; i++) {\r\n        if (a[i] != b[i]) {\r\n            return false;\r\n        }\r\n    }\r\n    return true;\r\n}\r\n/**\r\n * Get the type of an object, for example exports.getType([]) returns 'Array'.\r\n *\r\n * @param object - Input value of unknown type.\r\n *\r\n * @returns Detected type.\r\n */\r\nfunction getType(object) {\r\n    const type = typeof object;\r\n    if (type === \"object\") {\r\n        if (object === null) {\r\n            return \"null\";\r\n        }\r\n        if (object instanceof Boolean) {\r\n            return \"Boolean\";\r\n        }\r\n        if (object instanceof Number) {\r\n            return \"Number\";\r\n        }\r\n        if (object instanceof String) {\r\n            return \"String\";\r\n        }\r\n        if (Array.isArray(object)) {\r\n            return \"Array\";\r\n        }\r\n        if (object instanceof Date) {\r\n            return \"Date\";\r\n        }\r\n        return \"Object\";\r\n    }\r\n    if (type === \"number\") {\r\n        return \"Number\";\r\n    }\r\n    if (type === \"boolean\") {\r\n        return \"Boolean\";\r\n    }\r\n    if (type === \"string\") {\r\n        return \"String\";\r\n    }\r\n    if (type === undefined) {\r\n        return \"undefined\";\r\n    }\r\n    return type;\r\n}\r\n/**\r\n * Used to extend an array and copy it. This is used to propagate paths recursively.\r\n *\r\n * @param arr - First part.\r\n * @param newValue - The value to be aadded into the array.\r\n *\r\n * @returns A new array with all items from arr and newValue (which is last).\r\n */\r\nfunction copyAndExtendArray(arr, newValue) {\r\n    return [...arr, newValue];\r\n}\r\n/**\r\n * Used to extend an array and copy it. This is used to propagate paths recursively.\r\n *\r\n * @param arr - The array to be copied.\r\n *\r\n * @returns Shallow copy of arr.\r\n */\r\nfunction copyArray(arr) {\r\n    return arr.slice();\r\n}\r\n/**\r\n * Retrieve the absolute left value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute left position of this element in the browser page.\r\n */\r\nfunction getAbsoluteLeft(elem) {\r\n    return elem.getBoundingClientRect().left;\r\n}\r\n/**\r\n * Retrieve the absolute right value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute right position of this element in the browser page.\r\n */\r\nfunction getAbsoluteRight(elem) {\r\n    return elem.getBoundingClientRect().right;\r\n}\r\n/**\r\n * Retrieve the absolute top value of a DOM element.\r\n *\r\n * @param elem - A dom element, for example a div.\r\n *\r\n * @returns The absolute top position of this element in the browser page.\r\n */\r\nfunction getAbsoluteTop(elem) {\r\n    return elem.getBoundingClientRect().top;\r\n}\r\n/**\r\n * Add a className to the given elements style.\r\n *\r\n * @param elem - The element to which the classes will be added.\r\n * @param classNames - Space separated list of classes.\r\n */\r\nfunction addClassName(elem, classNames) {\r\n    let classes = elem.className.split(\" \");\r\n    const newClasses = classNames.split(\" \");\r\n    classes = classes.concat(newClasses.filter(function (className) {\r\n        return classes.indexOf(className) < 0;\r\n    }));\r\n    elem.className = classes.join(\" \");\r\n}\r\n/**\r\n * Remove a className from the given elements style.\r\n *\r\n * @param elem - The element from which the classes will be removed.\r\n * @param classNames - Space separated list of classes.\r\n */\r\nfunction removeClassName(elem, classNames) {\r\n    let classes = elem.className.split(\" \");\r\n    const oldClasses = classNames.split(\" \");\r\n    classes = classes.filter(function (className) {\r\n        return oldClasses.indexOf(className) < 0;\r\n    });\r\n    elem.className = classes.join(\" \");\r\n}\r\n/**\r\n * For each method for both arrays and objects.\r\n * In case of an array, the built-in Array.forEach() is applied (**No, it's not!**).\r\n * In case of an Object, the method loops over all properties of the object.\r\n *\r\n * @param object - An Object or Array to be iterated over.\r\n * @param callback - Array.forEach-like callback.\r\n */\r\nfunction forEach(object, callback) {\r\n    if (Array.isArray(object)) {\r\n        // array\r\n        const len = object.length;\r\n        for (let i = 0; i < len; i++) {\r\n            callback(object[i], i, object);\r\n        }\r\n    }\r\n    else {\r\n        // object\r\n        for (const key in object) {\r\n            if (Object.prototype.hasOwnProperty.call(object, key)) {\r\n                callback(object[key], key, object);\r\n            }\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert an object into an array: all objects properties are put into the array. The resulting array is unordered.\r\n *\r\n * @param o - Object that contains the properties and methods.\r\n *\r\n * @returns An array of unordered values.\r\n */\r\nconst toArray = Object.values;\r\n/**\r\n * Update a property in an object.\r\n *\r\n * @param object - The object whose property will be updated.\r\n * @param key - Name of the property to be updated.\r\n * @param value - The new value to be assigned.\r\n *\r\n * @returns Whether the value was updated (true) or already strictly the same in the original object (false).\r\n */\r\nfunction updateProperty(object, key, value) {\r\n    if (object[key] !== value) {\r\n        object[key] = value;\r\n        return true;\r\n    }\r\n    else {\r\n        return false;\r\n    }\r\n}\r\n/**\r\n * Throttle the given function to be only executed once per animation frame.\r\n *\r\n * @param fn - The original function.\r\n *\r\n * @returns The throttled function.\r\n */\r\nfunction throttle(fn) {\r\n    let scheduled = false;\r\n    return () => {\r\n        if (!scheduled) {\r\n            scheduled = true;\r\n            requestAnimationFrame(() => {\r\n                scheduled = false;\r\n                fn();\r\n            });\r\n        }\r\n    };\r\n}\r\n/**\r\n * Add and event listener. Works for all browsers.\r\n *\r\n * @param element - The element to bind the event listener to.\r\n * @param action - Same as Element.addEventListener(action, —, —).\r\n * @param listener - Same as Element.addEventListener(—, listener, —).\r\n * @param useCapture - Same as Element.addEventListener(—, —, useCapture).\r\n */\r\nfunction addEventListener(element, action, listener, useCapture) {\r\n    if (element.addEventListener) {\r\n        if (useCapture === undefined) {\r\n            useCapture = false;\r\n        }\r\n        if (action === \"mousewheel\" &&\r\n            navigator.userAgent.indexOf(\"Firefox\") >= 0) {\r\n            action = \"DOMMouseScroll\"; // For Firefox\r\n        }\r\n        element.addEventListener(action, listener, useCapture);\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        element.attachEvent(\"on\" + action, listener); // IE browsers\r\n    }\r\n}\r\n/**\r\n * Remove an event listener from an element.\r\n *\r\n * @param element - The element to bind the event listener to.\r\n * @param action - Same as Element.removeEventListener(action, —, —).\r\n * @param listener - Same as Element.removeEventListener(—, listener, —).\r\n * @param useCapture - Same as Element.removeEventListener(—, —, useCapture).\r\n */\r\nfunction removeEventListener(element, action, listener, useCapture) {\r\n    if (element.removeEventListener) {\r\n        // non-IE browsers\r\n        if (useCapture === undefined) {\r\n            useCapture = false;\r\n        }\r\n        if (action === \"mousewheel\" &&\r\n            navigator.userAgent.indexOf(\"Firefox\") >= 0) {\r\n            action = \"DOMMouseScroll\"; // For Firefox\r\n        }\r\n        element.removeEventListener(action, listener, useCapture);\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        element.detachEvent(\"on\" + action, listener); // IE browsers\r\n    }\r\n}\r\n/**\r\n * Cancels the event's default action if it is cancelable, without stopping further propagation of the event.\r\n *\r\n * @param event - The event whose default action should be prevented.\r\n */\r\nfunction preventDefault(event) {\r\n    if (!event) {\r\n        event = window.event;\r\n    }\r\n    if (!event) ;\r\n    else if (event.preventDefault) {\r\n        event.preventDefault(); // non-IE browsers\r\n    }\r\n    else {\r\n        // @TODO: IE types? Does anyone care?\r\n        event.returnValue = false; // IE browsers\r\n    }\r\n}\r\n/**\r\n * Get HTML element which is the target of the event.\r\n *\r\n * @param event - The event.\r\n *\r\n * @returns The element or null if not obtainable.\r\n */\r\nfunction getTarget(event = window.event) {\r\n    // code from http://www.quirksmode.org/js/events_properties.html\r\n    // @TODO: EventTarget can be almost anything, is it okay to return only Elements?\r\n    let target = null;\r\n    if (!event) ;\r\n    else if (event.target) {\r\n        target = event.target;\r\n    }\r\n    else if (event.srcElement) {\r\n        target = event.srcElement;\r\n    }\r\n    if (!(target instanceof Element)) {\r\n        return null;\r\n    }\r\n    if (target.nodeType != null && target.nodeType == 3) {\r\n        // defeat Safari bug\r\n        target = target.parentNode;\r\n        if (!(target instanceof Element)) {\r\n            return null;\r\n        }\r\n    }\r\n    return target;\r\n}\r\n/**\r\n * Check if given element contains given parent somewhere in the DOM tree.\r\n *\r\n * @param element - The element to be tested.\r\n * @param parent - The ancestor (not necessarily parent) of the element.\r\n *\r\n * @returns True if parent is an ancestor of the element, false otherwise.\r\n */\r\nfunction hasParent(element, parent) {\r\n    let elem = element;\r\n    while (elem) {\r\n        if (elem === parent) {\r\n            return true;\r\n        }\r\n        else if (elem.parentNode) {\r\n            elem = elem.parentNode;\r\n        }\r\n        else {\r\n            return false;\r\n        }\r\n    }\r\n    return false;\r\n}\r\nconst option = {\r\n    /**\r\n     * Convert a value into a boolean.\r\n     *\r\n     * @param value - Value to be converted intoboolean, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding boolean value, if none then the default value, if none then null.\r\n     */\r\n    asBoolean(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return value != false;\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a number.\r\n     *\r\n     * @param value - Value to be converted intonumber, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding **boxed** number value, if none then the default value, if none then null.\r\n     */\r\n    asNumber(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return Number(value) || defaultValue || null;\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a string.\r\n     *\r\n     * @param value - Value to be converted intostring, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding **boxed** string value, if none then the default value, if none then null.\r\n     */\r\n    asString(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (value != null) {\r\n            return String(value);\r\n        }\r\n        return defaultValue || null;\r\n    },\r\n    /**\r\n     * Convert a value into a size.\r\n     *\r\n     * @param value - Value to be converted intosize, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns Corresponding string value (number + 'px'), if none then the default value, if none then null.\r\n     */\r\n    asSize(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        if (isString(value)) {\r\n            return value;\r\n        }\r\n        else if (isNumber(value)) {\r\n            return value + \"px\";\r\n        }\r\n        else {\r\n            return defaultValue || null;\r\n        }\r\n    },\r\n    /**\r\n     * Convert a value into a DOM Element.\r\n     *\r\n     * @param value - Value to be converted into DOM Element, a function will be executed as `(() => unknown)`.\r\n     * @param defaultValue - If the value or the return value of the function == null then this will be returned.\r\n     *\r\n     * @returns The DOM Element, if none then the default value, if none then null.\r\n     */\r\n    asElement(value, defaultValue) {\r\n        if (typeof value == \"function\") {\r\n            value = value();\r\n        }\r\n        return value || defaultValue || null;\r\n    }\r\n};\r\n/**\r\n * Convert hex color string into RGB color object.\r\n *\r\n * @remarks\r\n * {@link http://stackoverflow.com/questions/5623838/rgb-to-hex-and-hex-to-rgb}\r\n *\r\n * @param hex - Hex color string (3 or 6 digits, with or without #).\r\n *\r\n * @returns RGB color object.\r\n */\r\nfunction hexToRGB(hex) {\r\n    let result;\r\n    switch (hex.length) {\r\n        case 3:\r\n        case 4:\r\n            result = shortHexRE.exec(hex);\r\n            return result\r\n                ? {\r\n                    r: parseInt(result[1] + result[1], 16),\r\n                    g: parseInt(result[2] + result[2], 16),\r\n                    b: parseInt(result[3] + result[3], 16)\r\n                }\r\n                : null;\r\n        case 6:\r\n        case 7:\r\n            result = fullHexRE.exec(hex);\r\n            return result\r\n                ? {\r\n                    r: parseInt(result[1], 16),\r\n                    g: parseInt(result[2], 16),\r\n                    b: parseInt(result[3], 16)\r\n                }\r\n                : null;\r\n        default:\r\n            return null;\r\n    }\r\n}\r\n/**\r\n * This function takes string color in hex or RGB format and adds the opacity, RGBA is passed through unchanged.\r\n *\r\n * @param color - The color string (hex, RGB, RGBA).\r\n * @param opacity - The new opacity.\r\n *\r\n * @returns RGBA string, for example 'rgba(255, 0, 127, 0.3)'.\r\n */\r\nfunction overrideOpacity(color, opacity) {\r\n    if (color.indexOf(\"rgba\") !== -1) {\r\n        return color;\r\n    }\r\n    else if (color.indexOf(\"rgb\") !== -1) {\r\n        const rgb = color\r\n            .substr(color.indexOf(\"(\") + 1)\r\n            .replace(\")\", \"\")\r\n            .split(\",\");\r\n        return \"rgba(\" + rgb[0] + \",\" + rgb[1] + \",\" + rgb[2] + \",\" + opacity + \")\";\r\n    }\r\n    else {\r\n        const rgb = hexToRGB(color);\r\n        if (rgb == null) {\r\n            return color;\r\n        }\r\n        else {\r\n            return \"rgba(\" + rgb.r + \",\" + rgb.g + \",\" + rgb.b + \",\" + opacity + \")\";\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert RGB \\<0, 255\\> into hex color string.\r\n *\r\n * @param red - Red channel.\r\n * @param green - Green channel.\r\n * @param blue - Blue channel.\r\n *\r\n * @returns Hex color string (for example: '#0acdc0').\r\n */\r\nfunction RGBToHex(red, green, blue) {\r\n    return (\"#\" + ((1 << 24) + (red << 16) + (green << 8) + blue).toString(16).slice(1));\r\n}\r\n/**\r\n * Parse a color property into an object with border, background, and highlight colors.\r\n *\r\n * @param inputColor - Shorthand color string or input color object.\r\n * @param defaultColor - Full color object to fill in missing values in inputColor.\r\n *\r\n * @returns Color object.\r\n */\r\nfunction parseColor(inputColor, defaultColor) {\r\n    if (isString(inputColor)) {\r\n        let colorStr = inputColor;\r\n        if (isValidRGB(colorStr)) {\r\n            const rgb = colorStr\r\n                .substr(4)\r\n                .substr(0, colorStr.length - 5)\r\n                .split(\",\")\r\n                .map(function (value) {\r\n                return parseInt(value);\r\n            });\r\n            colorStr = RGBToHex(rgb[0], rgb[1], rgb[2]);\r\n        }\r\n        if (isValidHex(colorStr) === true) {\r\n            const hsv = hexToHSV(colorStr);\r\n            const lighterColorHSV = {\r\n                h: hsv.h,\r\n                s: hsv.s * 0.8,\r\n                v: Math.min(1, hsv.v * 1.02)\r\n            };\r\n            const darkerColorHSV = {\r\n                h: hsv.h,\r\n                s: Math.min(1, hsv.s * 1.25),\r\n                v: hsv.v * 0.8\r\n            };\r\n            const darkerColorHex = HSVToHex(darkerColorHSV.h, darkerColorHSV.s, darkerColorHSV.v);\r\n            const lighterColorHex = HSVToHex(lighterColorHSV.h, lighterColorHSV.s, lighterColorHSV.v);\r\n            return {\r\n                background: colorStr,\r\n                border: darkerColorHex,\r\n                highlight: {\r\n                    background: lighterColorHex,\r\n                    border: darkerColorHex\r\n                },\r\n                hover: {\r\n                    background: lighterColorHex,\r\n                    border: darkerColorHex\r\n                }\r\n            };\r\n        }\r\n        else {\r\n            return {\r\n                background: colorStr,\r\n                border: colorStr,\r\n                highlight: {\r\n                    background: colorStr,\r\n                    border: colorStr\r\n                },\r\n                hover: {\r\n                    background: colorStr,\r\n                    border: colorStr\r\n                }\r\n            };\r\n        }\r\n    }\r\n    else {\r\n        if (defaultColor) {\r\n            const color = {\r\n                background: inputColor.background || defaultColor.background,\r\n                border: inputColor.border || defaultColor.border,\r\n                highlight: isString(inputColor.highlight)\r\n                    ? {\r\n                        border: inputColor.highlight,\r\n                        background: inputColor.highlight\r\n                    }\r\n                    : {\r\n                        background: (inputColor.highlight && inputColor.highlight.background) ||\r\n                            defaultColor.highlight.background,\r\n                        border: (inputColor.highlight && inputColor.highlight.border) ||\r\n                            defaultColor.highlight.border\r\n                    },\r\n                hover: isString(inputColor.hover)\r\n                    ? {\r\n                        border: inputColor.hover,\r\n                        background: inputColor.hover\r\n                    }\r\n                    : {\r\n                        border: (inputColor.hover && inputColor.hover.border) ||\r\n                            defaultColor.hover.border,\r\n                        background: (inputColor.hover && inputColor.hover.background) ||\r\n                            defaultColor.hover.background\r\n                    }\r\n            };\r\n            return color;\r\n        }\r\n        else {\r\n            const color = {\r\n                background: inputColor.background || undefined,\r\n                border: inputColor.border || undefined,\r\n                highlight: isString(inputColor.highlight)\r\n                    ? {\r\n                        border: inputColor.highlight,\r\n                        background: inputColor.highlight\r\n                    }\r\n                    : {\r\n                        background: (inputColor.highlight && inputColor.highlight.background) ||\r\n                            undefined,\r\n                        border: (inputColor.highlight && inputColor.highlight.border) ||\r\n                            undefined\r\n                    },\r\n                hover: isString(inputColor.hover)\r\n                    ? {\r\n                        border: inputColor.hover,\r\n                        background: inputColor.hover\r\n                    }\r\n                    : {\r\n                        border: (inputColor.hover && inputColor.hover.border) || undefined,\r\n                        background: (inputColor.hover && inputColor.hover.background) || undefined\r\n                    }\r\n            };\r\n            return color;\r\n        }\r\n    }\r\n}\r\n/**\r\n * Convert RGB \\<0, 255\\> into HSV object.\r\n *\r\n * @remarks\r\n * {@link http://www.javascripter.net/faq/rgb2hsv.htm}\r\n *\r\n * @param red - Red channel.\r\n * @param green - Green channel.\r\n * @param blue - Blue channel.\r\n *\r\n * @returns HSV color object.\r\n */\r\nfunction RGBToHSV(red, green, blue) {\r\n    red = red / 255;\r\n    green = green / 255;\r\n    blue = blue / 255;\r\n    const minRGB = Math.min(red, Math.min(green, blue));\r\n    const maxRGB = Math.max(red, Math.max(green, blue));\r\n    // Black-gray-white\r\n    if (minRGB === maxRGB) {\r\n        return { h: 0, s: 0, v: minRGB };\r\n    }\r\n    // Colors other than black-gray-white:\r\n    const d = red === minRGB ? green - blue : blue === minRGB ? red - green : blue - red;\r\n    const h = red === minRGB ? 3 : blue === minRGB ? 1 : 5;\r\n    const hue = (60 * (h - d / (maxRGB - minRGB))) / 360;\r\n    const saturation = (maxRGB - minRGB) / maxRGB;\r\n    const value = maxRGB;\r\n    return { h: hue, s: saturation, v: value };\r\n}\r\nconst cssUtil = {\r\n    // split a string with css styles into an object with key/values\r\n    split(cssText) {\r\n        const styles = {};\r\n        cssText.split(\";\").forEach((style) => {\r\n            if (style.trim() != \"\") {\r\n                const parts = style.split(\":\");\r\n                const key = parts[0].trim();\r\n                const value = parts[1].trim();\r\n                styles[key] = value;\r\n            }\r\n        });\r\n        return styles;\r\n    },\r\n    // build a css text string from an object with key/values\r\n    join(styles) {\r\n        return Object.keys(styles)\r\n            .map(function (key) {\r\n            return key + \": \" + styles[key];\r\n        })\r\n            .join(\"; \");\r\n    }\r\n};\r\n/**\r\n * Append a string with css styles to an element.\r\n *\r\n * @param element - The element that will receive new styles.\r\n * @param cssText - The styles to be appended.\r\n */\r\nfunction addCssText(element, cssText) {\r\n    const currentStyles = cssUtil.split(element.style.cssText);\r\n    const newStyles = cssUtil.split(cssText);\r\n    const styles = {\r\n        ...currentStyles,\r\n        ...newStyles\r\n    };\r\n    element.style.cssText = cssUtil.join(styles);\r\n}\r\n/**\r\n * Remove a string with css styles from an element.\r\n *\r\n * @param element - The element from which styles should be removed.\r\n * @param cssText - The styles to be removed.\r\n */\r\nfunction removeCssText(element, cssText) {\r\n    const styles = cssUtil.split(element.style.cssText);\r\n    const removeStyles = cssUtil.split(cssText);\r\n    for (const key in removeStyles) {\r\n        if (Object.prototype.hasOwnProperty.call(removeStyles, key)) {\r\n            delete styles[key];\r\n        }\r\n    }\r\n    element.style.cssText = cssUtil.join(styles);\r\n}\r\n/**\r\n * Convert HSV \\<0, 1\\> into RGB color object.\r\n *\r\n * @remarks\r\n * {@link https://gist.github.com/mjijackson/5311256}\r\n *\r\n * @param h - Hue.\r\n * @param s - Saturation.\r\n * @param v - Value.\r\n *\r\n * @returns RGB color object.\r\n */\r\nfunction HSVToRGB(h, s, v) {\r\n    let r;\r\n    let g;\r\n    let b;\r\n    const i = Math.floor(h * 6);\r\n    const f = h * 6 - i;\r\n    const p = v * (1 - s);\r\n    const q = v * (1 - f * s);\r\n    const t = v * (1 - (1 - f) * s);\r\n    switch (i % 6) {\r\n        case 0:\r\n            (r = v), (g = t), (b = p);\r\n            break;\r\n        case 1:\r\n            (r = q), (g = v), (b = p);\r\n            break;\r\n        case 2:\r\n            (r = p), (g = v), (b = t);\r\n            break;\r\n        case 3:\r\n            (r = p), (g = q), (b = v);\r\n            break;\r\n        case 4:\r\n            (r = t), (g = p), (b = v);\r\n            break;\r\n        case 5:\r\n            (r = v), (g = p), (b = q);\r\n            break;\r\n    }\r\n    return {\r\n        r: Math.floor(r * 255),\r\n        g: Math.floor(g * 255),\r\n        b: Math.floor(b * 255)\r\n    };\r\n}\r\n/**\r\n * Convert HSV \\<0, 1\\> into hex color string.\r\n *\r\n * @param h - Hue.\r\n * @param s - Saturation.\r\n * @param v - Value.\r\n *\r\n * @returns Hex color string.\r\n */\r\nfunction HSVToHex(h, s, v) {\r\n    const rgb = HSVToRGB(h, s, v);\r\n    return RGBToHex(rgb.r, rgb.g, rgb.b);\r\n}\r\n/**\r\n * Convert hex color string into HSV \\<0, 1\\>.\r\n *\r\n * @param hex - Hex color string.\r\n *\r\n * @returns HSV color object.\r\n */\r\nfunction hexToHSV(hex) {\r\n    const rgb = hexToRGB(hex);\r\n    if (!rgb) {\r\n        throw new TypeError(`'${hex}' is not a valid color.`);\r\n    }\r\n    return RGBToHSV(rgb.r, rgb.g, rgb.b);\r\n}\r\n/**\r\n * Validate hex color string.\r\n *\r\n * @param hex - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidHex(hex) {\r\n    const isOk = /(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(hex);\r\n    return isOk;\r\n}\r\n/**\r\n * Validate RGB color string.\r\n *\r\n * @param rgb - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidRGB(rgb) {\r\n    return rgbRE.test(rgb);\r\n}\r\n/**\r\n * Validate RGBA color string.\r\n *\r\n * @param rgba - Unknown string that may contain a color.\r\n *\r\n * @returns True if the string is valid, false otherwise.\r\n */\r\nfunction isValidRGBA(rgba) {\r\n    return rgbaRE.test(rgba);\r\n}\r\n/**\r\n * This recursively redirects the prototype of JSON objects to the referenceObject.\r\n * This is used for default options.\r\n *\r\n * @param fields - Names of properties to be bridged.\r\n * @param referenceObject - The original object.\r\n *\r\n * @returns A new object inheriting from the referenceObject.\r\n */\r\nfunction selectiveBridgeObject(fields, referenceObject) {\r\n    if (referenceObject !== null && typeof referenceObject === \"object\") {\r\n        // !!! typeof null === 'object'\r\n        const objectTo = Object.create(referenceObject);\r\n        for (let i = 0; i < fields.length; i++) {\r\n            if (Object.prototype.hasOwnProperty.call(referenceObject, fields[i])) {\r\n                if (typeof referenceObject[fields[i]] == \"object\") {\r\n                    objectTo[fields[i]] = bridgeObject(referenceObject[fields[i]]);\r\n                }\r\n            }\r\n        }\r\n        return objectTo;\r\n    }\r\n    else {\r\n        return null;\r\n    }\r\n}\r\n/**\r\n * This recursively redirects the prototype of JSON objects to the referenceObject.\r\n * This is used for default options.\r\n *\r\n * @param referenceObject - The original object.\r\n *\r\n * @returns The Element if the referenceObject is an Element, or a new object inheriting from the referenceObject.\r\n */\r\nfunction bridgeObject(referenceObject) {\r\n    if (referenceObject === null || typeof referenceObject !== \"object\") {\r\n        return null;\r\n    }\r\n    if (referenceObject instanceof Element) {\r\n        // Avoid bridging DOM objects\r\n        return referenceObject;\r\n    }\r\n    const objectTo = Object.create(referenceObject);\r\n    for (const i in referenceObject) {\r\n        if (Object.prototype.hasOwnProperty.call(referenceObject, i)) {\r\n            if (typeof referenceObject[i] == \"object\") {\r\n                objectTo[i] = bridgeObject(referenceObject[i]);\r\n            }\r\n        }\r\n    }\r\n    return objectTo;\r\n}\r\n/**\r\n * This method provides a stable sort implementation, very fast for presorted data.\r\n *\r\n * @param a - The array to be sorted (in-place).\r\n * @param compare - An order comparator.\r\n *\r\n * @returns The argument a.\r\n */\r\nfunction insertSort(a, compare) {\r\n    for (let i = 0; i < a.length; i++) {\r\n        const k = a[i];\r\n        let j;\r\n        for (j = i; j > 0 && compare(k, a[j - 1]) < 0; j--) {\r\n            a[j] = a[j - 1];\r\n        }\r\n        a[j] = k;\r\n    }\r\n    return a;\r\n}\r\n/**\r\n * This is used to set the options of subobjects in the options object.\r\n *\r\n * A requirement of these subobjects is that they have an 'enabled' element\r\n * which is optional for the user but mandatory for the program.\r\n *\r\n * The added value here of the merge is that option 'enabled' is set as required.\r\n *\r\n * @param mergeTarget - Either this.options or the options used for the groups.\r\n * @param options - Options.\r\n * @param option - Option key in the options argument.\r\n * @param globalOptions - Global options, passed in to determine value of option 'enabled'.\r\n */\r\nfunction mergeOptions(mergeTarget, options, option, globalOptions = {}) {\r\n    // Local helpers\r\n    const isPresent = function (obj) {\r\n        return obj !== null && obj !== undefined;\r\n    };\r\n    const isObject = function (obj) {\r\n        return obj !== null && typeof obj === \"object\";\r\n    };\r\n    // https://stackoverflow.com/a/34491287/1223531\r\n    const isEmpty = function (obj) {\r\n        for (const x in obj) {\r\n            if (Object.prototype.hasOwnProperty.call(obj, x)) {\r\n                return false;\r\n            }\r\n        }\r\n        return true;\r\n    };\r\n    // Guards\r\n    if (!isObject(mergeTarget)) {\r\n        throw new Error(\"Parameter mergeTarget must be an object\");\r\n    }\r\n    if (!isObject(options)) {\r\n        throw new Error(\"Parameter options must be an object\");\r\n    }\r\n    if (!isPresent(option)) {\r\n        throw new Error(\"Parameter option must have a value\");\r\n    }\r\n    if (!isObject(globalOptions)) {\r\n        throw new Error(\"Parameter globalOptions must be an object\");\r\n    }\r\n    //\r\n    // Actual merge routine, separated from main logic\r\n    // Only a single level of options is merged. Deeper levels are ref'd. This may actually be an issue.\r\n    //\r\n    const doMerge = function (target, options, option) {\r\n        if (!isObject(target[option])) {\r\n            target[option] = {};\r\n        }\r\n        const src = options[option];\r\n        const dst = target[option];\r\n        for (const prop in src) {\r\n            if (Object.prototype.hasOwnProperty.call(src, prop)) {\r\n                dst[prop] = src[prop];\r\n            }\r\n        }\r\n    };\r\n    // Local initialization\r\n    const srcOption = options[option];\r\n    const globalPassed = isObject(globalOptions) && !isEmpty(globalOptions);\r\n    const globalOption = globalPassed ? globalOptions[option] : undefined;\r\n    const globalEnabled = globalOption ? globalOption.enabled : undefined;\r\n    /////////////////////////////////////////\r\n    // Main routine\r\n    /////////////////////////////////////////\r\n    if (srcOption === undefined) {\r\n        return; // Nothing to do\r\n    }\r\n    if (typeof srcOption === \"boolean\") {\r\n        if (!isObject(mergeTarget[option])) {\r\n            mergeTarget[option] = {};\r\n        }\r\n        mergeTarget[option].enabled = srcOption;\r\n        return;\r\n    }\r\n    if (srcOption === null && !isObject(mergeTarget[option])) {\r\n        // If possible, explicit copy from globals\r\n        if (isPresent(globalOption)) {\r\n            mergeTarget[option] = Object.create(globalOption);\r\n        }\r\n        else {\r\n            return; // Nothing to do\r\n        }\r\n    }\r\n    if (!isObject(srcOption)) {\r\n        return;\r\n    }\r\n    //\r\n    // Ensure that 'enabled' is properly set. It is required internally\r\n    // Note that the value from options will always overwrite the existing value\r\n    //\r\n    let enabled = true; // default value\r\n    if (srcOption.enabled !== undefined) {\r\n        enabled = srcOption.enabled;\r\n    }\r\n    else {\r\n        // Take from globals, if present\r\n        if (globalEnabled !== undefined) {\r\n            enabled = globalOption.enabled;\r\n        }\r\n    }\r\n    doMerge(mergeTarget, options, option);\r\n    mergeTarget[option].enabled = enabled;\r\n}\r\n/**\r\n * This function does a binary search for a visible item in a sorted list. If we find a visible item, the code that uses\r\n * this function will then iterate in both directions over this sorted list to find all visible items.\r\n *\r\n * @param orderedItems - Items ordered by start.\r\n * @param comparator - -1 is lower, 0 is equal, 1 is higher.\r\n * @param field - Property name on an item (That is item[field]).\r\n * @param field2 - Second property name on an item (That is item[field][field2]).\r\n *\r\n * @returns Index of the found item or -1 if nothing was found.\r\n */\r\nfunction binarySearchCustom(orderedItems, comparator, field, field2) {\r\n    const maxIterations = 10000;\r\n    let iteration = 0;\r\n    let low = 0;\r\n    let high = orderedItems.length - 1;\r\n    while (low <= high && iteration < maxIterations) {\r\n        const middle = Math.floor((low + high) / 2);\r\n        const item = orderedItems[middle];\r\n        const value = field2 === undefined ? item[field] : item[field][field2];\r\n        const searchResult = comparator(value);\r\n        if (searchResult == 0) {\r\n            // jihaa, found a visible item!\r\n            return middle;\r\n        }\r\n        else if (searchResult == -1) {\r\n            // it is too small --> increase low\r\n            low = middle + 1;\r\n        }\r\n        else {\r\n            // it is too big --> decrease high\r\n            high = middle - 1;\r\n        }\r\n        iteration++;\r\n    }\r\n    return -1;\r\n}\r\n/**\r\n * This function does a binary search for a specific value in a sorted array.\r\n * If it does not exist but is in between of two values, we return either the\r\n * one before or the one after, depending on user input If it is found, we\r\n * return the index, else -1.\r\n *\r\n * @param orderedItems - Sorted array.\r\n * @param target - The searched value.\r\n * @param field - Name of the property in items to be searched.\r\n * @param sidePreference - If the target is between two values, should the index of the before or the after be returned?\r\n * @param comparator - An optional comparator, returning -1, 0, 1 for \\<, ===, \\>.\r\n *\r\n * @returns The index of found value or -1 if nothing was found.\r\n */\r\nfunction binarySearchValue(orderedItems, target, field, sidePreference, comparator) {\r\n    const maxIterations = 10000;\r\n    let iteration = 0;\r\n    let low = 0;\r\n    let high = orderedItems.length - 1;\r\n    let prevValue;\r\n    let value;\r\n    let nextValue;\r\n    let middle;\r\n    comparator =\r\n        comparator != undefined\r\n            ? comparator\r\n            : function (a, b) {\r\n                return a == b ? 0 : a < b ? -1 : 1;\r\n            };\r\n    while (low <= high && iteration < maxIterations) {\r\n        // get a new guess\r\n        middle = Math.floor(0.5 * (high + low));\r\n        prevValue = orderedItems[Math.max(0, middle - 1)][field];\r\n        value = orderedItems[middle][field];\r\n        nextValue =\r\n            orderedItems[Math.min(orderedItems.length - 1, middle + 1)][field];\r\n        if (comparator(value, target) == 0) {\r\n            // we found the target\r\n            return middle;\r\n        }\r\n        else if (comparator(prevValue, target) < 0 &&\r\n            comparator(value, target) > 0) {\r\n            // target is in between of the previous and the current\r\n            return sidePreference == \"before\" ? Math.max(0, middle - 1) : middle;\r\n        }\r\n        else if (comparator(value, target) < 0 &&\r\n            comparator(nextValue, target) > 0) {\r\n            // target is in between of the current and the next\r\n            return sidePreference == \"before\"\r\n                ? middle\r\n                : Math.min(orderedItems.length - 1, middle + 1);\r\n        }\r\n        else {\r\n            // didnt find the target, we need to change our boundaries.\r\n            if (comparator(value, target) < 0) {\r\n                // it is too small --> increase low\r\n                low = middle + 1;\r\n            }\r\n            else {\r\n                // it is too big --> decrease high\r\n                high = middle - 1;\r\n            }\r\n        }\r\n        iteration++;\r\n    }\r\n    // didnt find anything. Return -1.\r\n    return -1;\r\n}\r\n/*\r\n * Easing Functions.\r\n * Only considering the t value for the range [0, 1] => [0, 1].\r\n *\r\n * Inspiration: from http://gizma.com/easing/\r\n * https://gist.github.com/gre/1650294\r\n */\r\nconst easingFunctions = {\r\n    /**\r\n     * Provides no easing and no acceleration.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    linear(t) {\r\n        return t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuad(t) {\r\n        return t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuad(t) {\r\n        return t * (2 - t);\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuad(t) {\r\n        return t < 0.5 ? 2 * t * t : -1 + (4 - 2 * t) * t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInCubic(t) {\r\n        return t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutCubic(t) {\r\n        return --t * t * t + 1;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutCubic(t) {\r\n        return t < 0.5 ? 4 * t * t * t : (t - 1) * (2 * t - 2) * (2 * t - 2) + 1;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuart(t) {\r\n        return t * t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuart(t) {\r\n        return 1 - --t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuart(t) {\r\n        return t < 0.5 ? 8 * t * t * t * t : 1 - 8 * --t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate from zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInQuint(t) {\r\n        return t * t * t * t * t;\r\n    },\r\n    /**\r\n     * Decelerate to zero velocity.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeOutQuint(t) {\r\n        return 1 + --t * t * t * t * t;\r\n    },\r\n    /**\r\n     * Accelerate until halfway, then decelerate.\r\n     *\r\n     * @param t - Time.\r\n     *\r\n     * @returns Value at time t.\r\n     */\r\n    easeInOutQuint(t) {\r\n        return t < 0.5 ? 16 * t * t * t * t * t : 1 + 16 * --t * t * t * t * t;\r\n    }\r\n};\r\n/**\r\n * Experimentaly compute the width of the scrollbar for this browser.\r\n *\r\n * @returns The width in pixels.\r\n */\r\nfunction getScrollBarWidth() {\r\n    const inner = document.createElement(\"p\");\r\n    inner.style.width = \"100%\";\r\n    inner.style.height = \"200px\";\r\n    const outer = document.createElement(\"div\");\r\n    outer.style.position = \"absolute\";\r\n    outer.style.top = \"0px\";\r\n    outer.style.left = \"0px\";\r\n    outer.style.visibility = \"hidden\";\r\n    outer.style.width = \"200px\";\r\n    outer.style.height = \"150px\";\r\n    outer.style.overflow = \"hidden\";\r\n    outer.appendChild(inner);\r\n    document.body.appendChild(outer);\r\n    const w1 = inner.offsetWidth;\r\n    outer.style.overflow = \"scroll\";\r\n    let w2 = inner.offsetWidth;\r\n    if (w1 == w2) {\r\n        w2 = outer.clientWidth;\r\n    }\r\n    document.body.removeChild(outer);\r\n    return w1 - w2;\r\n}\r\n// @TODO: This doesn't work properly.\r\n// It works only for single property objects,\r\n// otherwise it combines all of the types in a union.\r\n// export function topMost<K1 extends string, V1> (\r\n//   pile: Record<K1, undefined | V1>[],\r\n//   accessors: K1 | [K1]\r\n// ): undefined | V1\r\n// export function topMost<K1 extends string, K2 extends string, V1, V2> (\r\n//   pile: Record<K1, undefined | V1 | Record<K2, undefined | V2>>[],\r\n//   accessors: [K1, K2]\r\n// ): undefined | V1 | V2\r\n// export function topMost<K1 extends string, K2 extends string, K3 extends string, V1, V2, V3> (\r\n//   pile: Record<K1, undefined | V1 | Record<K2, undefined | V2 | Record<K3, undefined | V3>>>[],\r\n//   accessors: [K1, K2, K3]\r\n// ): undefined | V1 | V2 | V3\r\n/**\r\n * Get the top most property value from a pile of objects.\r\n *\r\n * @param pile - Array of objects, no required format.\r\n * @param accessors - Array of property names.\r\n * For example `object['foo']['bar']` → `['foo', 'bar']`.\r\n *\r\n * @returns Value of the property with given accessors path from the first pile item where it's not undefined.\r\n */\r\nfunction topMost(pile, accessors) {\r\n    let candidate;\r\n    if (!Array.isArray(accessors)) {\r\n        accessors = [accessors];\r\n    }\r\n    for (const member of pile) {\r\n        if (member) {\r\n            candidate = member[accessors[0]];\r\n            for (let i = 1; i < accessors.length; i++) {\r\n                if (candidate) {\r\n                    candidate = candidate[accessors[i]];\r\n                }\r\n            }\r\n            if (typeof candidate !== \"undefined\") {\r\n                break;\r\n            }\r\n        }\r\n    }\r\n    return candidate;\r\n}\n\nexport { Alea, HSVToHex, HSVToRGB, RGBToHSV, RGBToHex, addClassName, addCssText, addEventListener, binarySearchCustom, binarySearchValue, bridgeObject, copyAndExtendArray, copyArray, deepExtend, easingFunctions, equalArray, extend, fillIfDefined, forEach, getAbsoluteLeft, getAbsoluteRight, getAbsoluteTop, getScrollBarWidth, getTarget, getType, hasParent, hexToHSV, hexToRGB, insertSort, isDate, isNumber, isObject, isString, isValidHex, isValidRGB, isValidRGBA, mergeOptions, option, overrideOpacity, parseColor, preventDefault, recursiveDOMDelete, removeClassName, removeCssText, removeEventListener, selectiveBridgeObject, selectiveDeepExtend, selectiveExtend, selectiveNotDeepExtend, throttle, toArray, topMost, updateProperty };\n//# sourceMappingURL=vis-util.js.map\n","// DOM utility methods\n\n/**\n * this prepares the JSON container for allocating SVG elements\n * @param {Object} JSONcontainer\n * @private\n */\nexport function prepareElements(JSONcontainer) {\n  // cleanup the redundant svgElements;\n  for (var elementType in JSONcontainer) {\n    if (JSONcontainer.hasOwnProperty(elementType)) {\n      JSONcontainer[elementType].redundant = JSONcontainer[elementType].used;\n      JSONcontainer[elementType].used = [];\n    }\n  }\n}\n\n/**\n * this cleans up all the unused SVG elements. By asking for the parentNode, we only need to supply the JSON container from\n * which to remove the redundant elements.\n *\n * @param {Object} JSONcontainer\n * @private\n */\nexport function cleanupElements(JSONcontainer) {\n  // cleanup the redundant svgElements;\n  for (var elementType in JSONcontainer) {\n    if (JSONcontainer.hasOwnProperty(elementType)) {\n      if (JSONcontainer[elementType].redundant) {\n        for (var i = 0; i < JSONcontainer[elementType].redundant.length; i++) {\n          JSONcontainer[elementType].redundant[i].parentNode.removeChild(JSONcontainer[elementType].redundant[i]);\n        }\n        JSONcontainer[elementType].redundant = [];\n      }\n    }\n  }\n}\n\n/**\n * Ensures that all elements are removed first up so they can be recreated cleanly\n * @param {Object} JSONcontainer\n */\nexport function resetElements(JSONcontainer) {\n  prepareElements(JSONcontainer);\n  cleanupElements(JSONcontainer);\n  prepareElements(JSONcontainer);\n}\n\n/**\n * Allocate or generate an SVG element if needed. Store a reference to it in the JSON container and draw it in the svgContainer\n * the JSON container and the SVG container have to be supplied so other svg containers (like the legend) can use this.\n *\n * @param {string} elementType\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @returns {Element}\n * @private\n */\nexport function getSVGElement(elementType, JSONcontainer, svgContainer) {\n  var element;\n  // allocate SVG element, if it doesnt yet exist, create one.\n  if (JSONcontainer.hasOwnProperty(elementType)) { // this element has been created before\n    // check if there is an redundant element\n    if (JSONcontainer[elementType].redundant.length > 0) {\n      element = JSONcontainer[elementType].redundant[0];\n      JSONcontainer[elementType].redundant.shift();\n    }\n    else {\n      // create a new element and add it to the SVG\n      element = document.createElementNS('http://www.w3.org/2000/svg', elementType);\n      svgContainer.appendChild(element);\n    }\n  }\n  else {\n    // create a new element and add it to the SVG, also create a new object in the svgElements to keep track of it.\n    element = document.createElementNS('http://www.w3.org/2000/svg', elementType);\n    JSONcontainer[elementType] = {used: [], redundant: []};\n    svgContainer.appendChild(element);\n  }\n  JSONcontainer[elementType].used.push(element);\n  return element;\n}\n\n\n/**\n * Allocate or generate an SVG element if needed. Store a reference to it in the JSON container and draw it in the svgContainer\n * the JSON container and the SVG container have to be supplied so other svg containers (like the legend) can use this.\n *\n * @param {string} elementType\n * @param {Object} JSONcontainer\n * @param {Element} DOMContainer\n * @param {Element} insertBefore\n * @returns {*}\n */\nexport function getDOMElement(elementType, JSONcontainer, DOMContainer, insertBefore) {\n  var element;\n  // allocate DOM element, if it doesnt yet exist, create one.\n  if (JSONcontainer.hasOwnProperty(elementType)) { // this element has been created before\n    // check if there is an redundant element\n    if (JSONcontainer[elementType].redundant.length > 0) {\n      element = JSONcontainer[elementType].redundant[0];\n      JSONcontainer[elementType].redundant.shift();\n    }\n    else {\n      // create a new element and add it to the SVG\n      element = document.createElement(elementType);\n      if (insertBefore !== undefined) {\n        DOMContainer.insertBefore(element, insertBefore);\n      }\n      else {\n        DOMContainer.appendChild(element);\n      }\n    }\n  }\n  else {\n    // create a new element and add it to the SVG, also create a new object in the svgElements to keep track of it.\n    element = document.createElement(elementType);\n    JSONcontainer[elementType] = {used: [], redundant: []};\n    if (insertBefore !== undefined) {\n      DOMContainer.insertBefore(element, insertBefore);\n    }\n    else {\n      DOMContainer.appendChild(element);\n    }\n  }\n  JSONcontainer[elementType].used.push(element);\n  return element;\n}\n\n\n\n\n/**\n * Draw a point object. This is a separate function because it can also be called by the legend.\n * The reason the JSONcontainer and the target SVG svgContainer have to be supplied is so the legend can use these functions\n * as well.\n *\n * @param {number} x\n * @param {number} y\n * @param {Object} groupTemplate: A template containing the necessary information to draw the datapoint e.g., {style: 'circle', size: 5, className: 'className' }\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @param {Object} labelObj\n * @returns {vis.PointItem}\n */\nexport function drawPoint(x, y, groupTemplate, JSONcontainer, svgContainer, labelObj) {\n  var point;\n  if (groupTemplate.style == 'circle') {\n    point = getSVGElement('circle', JSONcontainer, svgContainer);\n    point.setAttributeNS(null, \"cx\", x);\n    point.setAttributeNS(null, \"cy\", y);\n    point.setAttributeNS(null, \"r\", 0.5 * groupTemplate.size);\n  }\n  else {\n    point = getSVGElement('rect', JSONcontainer, svgContainer);\n    point.setAttributeNS(null, \"x\", x - 0.5 * groupTemplate.size);\n    point.setAttributeNS(null, \"y\", y - 0.5 * groupTemplate.size);\n    point.setAttributeNS(null, \"width\", groupTemplate.size);\n    point.setAttributeNS(null, \"height\", groupTemplate.size);\n  }\n\n  if (groupTemplate.styles !== undefined) {\n    point.setAttributeNS(null, \"style\", groupTemplate.styles);\n  }\n  point.setAttributeNS(null, \"class\", groupTemplate.className + \" vis-point\");\n  //handle label\n\n\n  if (labelObj) {\n    var label = getSVGElement('text', JSONcontainer, svgContainer);\n    if (labelObj.xOffset) {\n      x = x + labelObj.xOffset;\n    }\n\n    if (labelObj.yOffset) {\n      y = y + labelObj.yOffset;\n    }\n    if (labelObj.content) {\n      label.textContent = labelObj.content;\n    }\n\n    if (labelObj.className) {\n      label.setAttributeNS(null, \"class\", labelObj.className  + \" vis-label\");\n    }\n    label.setAttributeNS(null, \"x\", x);\n    label.setAttributeNS(null, \"y\", y);\n  }\n\n  return point;\n}\n\n/**\n * draw a bar SVG element centered on the X coordinate\n *\n * @param {number} x\n * @param {number} y\n * @param {number} width\n * @param {number} height\n * @param {string} className\n * @param {Object} JSONcontainer\n * @param {Object} svgContainer\n * @param {string} style\n */\nexport function drawBar (x, y, width, height, className, JSONcontainer, svgContainer, style) {\n  if (height != 0) {\n    if (height < 0) {\n      height *= -1;\n      y -= height;\n    }\n    var rect = getSVGElement('rect',JSONcontainer, svgContainer);\n    rect.setAttributeNS(null, \"x\", x - 0.5 * width);\n    rect.setAttributeNS(null, \"y\", y);\n    rect.setAttributeNS(null, \"width\", width);\n    rect.setAttributeNS(null, \"height\", height);\n    rect.setAttributeNS(null, \"class\", className);\n    if (style) {\n      rect.setAttributeNS(null, \"style\", style);\n    }\n  }\n}\n\n/**\n * get default language\n * @returns {string}\n */\nexport function getNavigatorLanguage() {\n  try {\n    if (!navigator) return 'en';\n    if (navigator.languages && navigator.languages.length) {\n      return navigator.languages;\n    } else {\n      return navigator.userLanguage || navigator.language || navigator.browserLanguage || 'en';\n    }\n  } \n  catch(error) {\n    return 'en';\n  }\n}\n","function _assertThisInitialized(self) {\n  if (self === void 0) {\n    throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");\n  }\n\n  return self;\n}\n\nmodule.exports = _assertThisInitialized;","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').entries;\n","require('../../modules/web.dom-collections.iterator');\nvar entries = require('../array/virtual/entries');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.entries;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.entries)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? entries : own;\n};\n","var $ = require('../internals/export');\nvar getBuiltIn = require('../internals/get-built-in');\nvar fails = require('../internals/fails');\n\nvar $stringify = getBuiltIn('JSON', 'stringify');\nvar re = /[\\uD800-\\uDFFF]/g;\nvar low = /^[\\uD800-\\uDBFF]$/;\nvar hi = /^[\\uDC00-\\uDFFF]$/;\n\nvar fix = function (match, offset, string) {\n  var prev = string.charAt(offset - 1);\n  var next = string.charAt(offset + 1);\n  if ((low.test(match) && !hi.test(next)) || (hi.test(match) && !low.test(prev))) {\n    return '\\\\u' + match.charCodeAt(0).toString(16);\n  } return match;\n};\n\nvar FORCED = fails(function () {\n  return $stringify('\\uDF06\\uD834') !== '\"\\\\udf06\\\\ud834\"'\n    || $stringify('\\uDEAD') !== '\"\\\\udead\"';\n});\n\nif ($stringify) {\n  // https://github.com/tc39/proposal-well-formed-stringify\n  $({ target: 'JSON', stat: true, forced: FORCED }, {\n    // eslint-disable-next-line no-unused-vars\n    stringify: function stringify(it, replacer, space) {\n      var result = $stringify.apply(null, arguments);\n      return typeof result == 'string' ? result.replace(re, fix) : result;\n    }\n  });\n}\n","require('../../modules/es.json.stringify');\nvar core = require('../../internals/path');\n\nif (!core.JSON) core.JSON = { stringify: JSON.stringify };\n\n// eslint-disable-next-line no-unused-vars\nmodule.exports = function stringify(it, replacer, space) {\n  return core.JSON.stringify.apply(null, arguments);\n};\n","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').values;\n","require('../../modules/web.dom-collections.iterator');\nvar values = require('../array/virtual/values');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.values;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.values)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? values : own;\n};\n","require('../../../modules/es.array.iterator');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').keys;\n","require('../../modules/web.dom-collections.iterator');\nvar keys = require('../array/virtual/keys');\nvar classof = require('../../internals/classof');\nvar ArrayPrototype = Array.prototype;\n\nvar DOMIterables = {\n  DOMTokenList: true,\n  NodeList: true\n};\n\nmodule.exports = function (it) {\n  var own = it.keys;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.keys)\n    // eslint-disable-next-line no-prototype-builtins\n    || DOMIterables.hasOwnProperty(classof(it)) ? keys : own;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $some = require('../internals/array-iteration').some;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('some');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('some');\n\n// `Array.prototype.some` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.some\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  some: function some(callbackfn /* , thisArg */) {\n    return $some(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.some');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').some;\n","var some = require('../array/virtual/some');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.some;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.some) ? some : own;\n};\n","var _typeof = require(\"../helpers/typeof\");\n\nvar assertThisInitialized = require(\"./assertThisInitialized\");\n\nfunction _possibleConstructorReturn(self, call) {\n  if (call && (_typeof(call) === \"object\" || typeof call === \"function\")) {\n    return call;\n  }\n\n  return assertThisInitialized(self);\n}\n\nmodule.exports = _possibleConstructorReturn;","var parent = require('../../es/object/get-prototype-of');\n\nmodule.exports = parent;\n","var $ = require('../internals/export');\nvar setPrototypeOf = require('../internals/object-set-prototype-of');\n\n// `Object.setPrototypeOf` method\n// https://tc39.github.io/ecma262/#sec-object.setprototypeof\n$({ target: 'Object', stat: true }, {\n  setPrototypeOf: setPrototypeOf\n});\n","require('../../modules/es.object.set-prototype-of');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Object.setPrototypeOf;\n","var _Object$getPrototypeOf = require(\"../core-js/object/get-prototype-of\");\n\nvar _Object$setPrototypeOf = require(\"../core-js/object/set-prototype-of\");\n\nfunction _getPrototypeOf(o) {\n  module.exports = _getPrototypeOf = _Object$setPrototypeOf ? _Object$getPrototypeOf : function _getPrototypeOf(o) {\n    return o.__proto__ || _Object$getPrototypeOf(o);\n  };\n  return _getPrototypeOf(o);\n}\n\nmodule.exports = _getPrototypeOf;","var parent = require('../../es/object/create');\n\nmodule.exports = parent;\n","var _Object$setPrototypeOf = require(\"../core-js/object/set-prototype-of\");\n\nfunction _setPrototypeOf(o, p) {\n  module.exports = _setPrototypeOf = _Object$setPrototypeOf || function _setPrototypeOf(o, p) {\n    o.__proto__ = p;\n    return o;\n  };\n\n  return _setPrototypeOf(o, p);\n}\n\nmodule.exports = _setPrototypeOf;","var _Object$create = require(\"../core-js/object/create\");\n\nvar setPrototypeOf = require(\"./setPrototypeOf\");\n\nfunction _inherits(subClass, superClass) {\n  if (typeof superClass !== \"function\" && superClass !== null) {\n    throw new TypeError(\"Super expression must either be null or a function\");\n  }\n\n  subClass.prototype = _Object$create(superClass && superClass.prototype, {\n    constructor: {\n      value: subClass,\n      writable: true,\n      configurable: true\n    }\n  });\n  if (superClass) setPrototypeOf(subClass, superClass);\n}\n\nmodule.exports = _inherits;","'use strict';\nvar $ = require('../internals/export');\nvar aFunction = require('../internals/a-function');\nvar toObject = require('../internals/to-object');\nvar fails = require('../internals/fails');\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\n\nvar test = [];\nvar nativeSort = test.sort;\n\n// IE8-\nvar FAILS_ON_UNDEFINED = fails(function () {\n  test.sort(undefined);\n});\n// V8 bug\nvar FAILS_ON_NULL = fails(function () {\n  test.sort(null);\n});\n// Old WebKit\nvar STRICT_METHOD = arrayMethodIsStrict('sort');\n\nvar FORCED = FAILS_ON_UNDEFINED || !FAILS_ON_NULL || !STRICT_METHOD;\n\n// `Array.prototype.sort` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.sort\n$({ target: 'Array', proto: true, forced: FORCED }, {\n  sort: function sort(comparefn) {\n    return comparefn === undefined\n      ? nativeSort.call(toObject(this))\n      : nativeSort.call(toObject(this), aFunction(comparefn));\n  }\n});\n","require('../../../modules/es.array.sort');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').sort;\n","var sort = require('../array/virtual/sort');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.sort;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.sort) ? sort : own;\n};\n","var fails = require('../internals/fails');\n\nmodule.exports = !fails(function () {\n  return Object.isExtensible(Object.preventExtensions({}));\n});\n","var hiddenKeys = require('../internals/hidden-keys');\nvar isObject = require('../internals/is-object');\nvar has = require('../internals/has');\nvar defineProperty = require('../internals/object-define-property').f;\nvar uid = require('../internals/uid');\nvar FREEZING = require('../internals/freezing');\n\nvar METADATA = uid('meta');\nvar id = 0;\n\nvar isExtensible = Object.isExtensible || function () {\n  return true;\n};\n\nvar setMetadata = function (it) {\n  defineProperty(it, METADATA, { value: {\n    objectID: 'O' + ++id, // object ID\n    weakData: {}          // weak collections IDs\n  } });\n};\n\nvar fastKey = function (it, create) {\n  // return a primitive with prefix\n  if (!isObject(it)) return typeof it == 'symbol' ? it : (typeof it == 'string' ? 'S' : 'P') + it;\n  if (!has(it, METADATA)) {\n    // can't set metadata to uncaught frozen object\n    if (!isExtensible(it)) return 'F';\n    // not necessary to add metadata\n    if (!create) return 'E';\n    // add missing metadata\n    setMetadata(it);\n  // return object ID\n  } return it[METADATA].objectID;\n};\n\nvar getWeakData = function (it, create) {\n  if (!has(it, METADATA)) {\n    // can't set metadata to uncaught frozen object\n    if (!isExtensible(it)) return true;\n    // not necessary to add metadata\n    if (!create) return false;\n    // add missing metadata\n    setMetadata(it);\n  // return the store of weak collections IDs\n  } return it[METADATA].weakData;\n};\n\n// add metadata on freeze-family methods calling\nvar onFreeze = function (it) {\n  if (FREEZING && meta.REQUIRED && isExtensible(it) && !has(it, METADATA)) setMetadata(it);\n  return it;\n};\n\nvar meta = module.exports = {\n  REQUIRED: false,\n  fastKey: fastKey,\n  getWeakData: getWeakData,\n  onFreeze: onFreeze\n};\n\nhiddenKeys[METADATA] = true;\n","var anObject = require('../internals/an-object');\nvar isArrayIteratorMethod = require('../internals/is-array-iterator-method');\nvar toLength = require('../internals/to-length');\nvar bind = require('../internals/function-bind-context');\nvar getIteratorMethod = require('../internals/get-iterator-method');\nvar callWithSafeIterationClosing = require('../internals/call-with-safe-iteration-closing');\n\nvar Result = function (stopped, result) {\n  this.stopped = stopped;\n  this.result = result;\n};\n\nvar iterate = module.exports = function (iterable, fn, that, AS_ENTRIES, IS_ITERATOR) {\n  var boundFunction = bind(fn, that, AS_ENTRIES ? 2 : 1);\n  var iterator, iterFn, index, length, result, next, step;\n\n  if (IS_ITERATOR) {\n    iterator = iterable;\n  } else {\n    iterFn = getIteratorMethod(iterable);\n    if (typeof iterFn != 'function') throw TypeError('Target is not iterable');\n    // optimisation for array iterators\n    if (isArrayIteratorMethod(iterFn)) {\n      for (index = 0, length = toLength(iterable.length); length > index; index++) {\n        result = AS_ENTRIES\n          ? boundFunction(anObject(step = iterable[index])[0], step[1])\n          : boundFunction(iterable[index]);\n        if (result && result instanceof Result) return result;\n      } return new Result(false);\n    }\n    iterator = iterFn.call(iterable);\n  }\n\n  next = iterator.next;\n  while (!(step = next.call(iterator)).done) {\n    result = callWithSafeIterationClosing(iterator, boundFunction, step.value, AS_ENTRIES);\n    if (typeof result == 'object' && result && result instanceof Result) return result;\n  } return new Result(false);\n};\n\niterate.stop = function (result) {\n  return new Result(true, result);\n};\n","module.exports = function (it, Constructor, name) {\n  if (!(it instanceof Constructor)) {\n    throw TypeError('Incorrect ' + (name ? name + ' ' : '') + 'invocation');\n  } return it;\n};\n","'use strict';\nvar $ = require('./export');\nvar global = require('../internals/global');\nvar InternalMetadataModule = require('../internals/internal-metadata');\nvar fails = require('../internals/fails');\nvar createNonEnumerableProperty = require('../internals/create-non-enumerable-property');\nvar iterate = require('../internals/iterate');\nvar anInstance = require('../internals/an-instance');\nvar isObject = require('../internals/is-object');\nvar setToStringTag = require('../internals/set-to-string-tag');\nvar defineProperty = require('../internals/object-define-property').f;\nvar forEach = require('../internals/array-iteration').forEach;\nvar DESCRIPTORS = require('../internals/descriptors');\nvar InternalStateModule = require('../internals/internal-state');\n\nvar setInternalState = InternalStateModule.set;\nvar internalStateGetterFor = InternalStateModule.getterFor;\n\nmodule.exports = function (CONSTRUCTOR_NAME, wrapper, common) {\n  var IS_MAP = CONSTRUCTOR_NAME.indexOf('Map') !== -1;\n  var IS_WEAK = CONSTRUCTOR_NAME.indexOf('Weak') !== -1;\n  var ADDER = IS_MAP ? 'set' : 'add';\n  var NativeConstructor = global[CONSTRUCTOR_NAME];\n  var NativePrototype = NativeConstructor && NativeConstructor.prototype;\n  var exported = {};\n  var Constructor;\n\n  if (!DESCRIPTORS || typeof NativeConstructor != 'function'\n    || !(IS_WEAK || NativePrototype.forEach && !fails(function () { new NativeConstructor().entries().next(); }))\n  ) {\n    // create collection constructor\n    Constructor = common.getConstructor(wrapper, CONSTRUCTOR_NAME, IS_MAP, ADDER);\n    InternalMetadataModule.REQUIRED = true;\n  } else {\n    Constructor = wrapper(function (target, iterable) {\n      setInternalState(anInstance(target, Constructor, CONSTRUCTOR_NAME), {\n        type: CONSTRUCTOR_NAME,\n        collection: new NativeConstructor()\n      });\n      if (iterable != undefined) iterate(iterable, target[ADDER], target, IS_MAP);\n    });\n\n    var getInternalState = internalStateGetterFor(CONSTRUCTOR_NAME);\n\n    forEach(['add', 'clear', 'delete', 'forEach', 'get', 'has', 'set', 'keys', 'values', 'entries'], function (KEY) {\n      var IS_ADDER = KEY == 'add' || KEY == 'set';\n      if (KEY in NativePrototype && !(IS_WEAK && KEY == 'clear')) {\n        createNonEnumerableProperty(Constructor.prototype, KEY, function (a, b) {\n          var collection = getInternalState(this).collection;\n          if (!IS_ADDER && IS_WEAK && !isObject(a)) return KEY == 'get' ? undefined : false;\n          var result = collection[KEY](a === 0 ? 0 : a, b);\n          return IS_ADDER ? this : result;\n        });\n      }\n    });\n\n    IS_WEAK || defineProperty(Constructor.prototype, 'size', {\n      configurable: true,\n      get: function () {\n        return getInternalState(this).collection.size;\n      }\n    });\n  }\n\n  setToStringTag(Constructor, CONSTRUCTOR_NAME, false, true);\n\n  exported[CONSTRUCTOR_NAME] = Constructor;\n  $({ global: true, forced: true }, exported);\n\n  if (!IS_WEAK) common.setStrong(Constructor, CONSTRUCTOR_NAME, IS_MAP);\n\n  return Constructor;\n};\n","var redefine = require('../internals/redefine');\n\nmodule.exports = function (target, src, options) {\n  for (var key in src) {\n    if (options && options.unsafe && target[key]) target[key] = src[key];\n    else redefine(target, key, src[key], options);\n  } return target;\n};\n","'use strict';\nvar getBuiltIn = require('../internals/get-built-in');\nvar definePropertyModule = require('../internals/object-define-property');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\nvar DESCRIPTORS = require('../internals/descriptors');\n\nvar SPECIES = wellKnownSymbol('species');\n\nmodule.exports = function (CONSTRUCTOR_NAME) {\n  var Constructor = getBuiltIn(CONSTRUCTOR_NAME);\n  var defineProperty = definePropertyModule.f;\n\n  if (DESCRIPTORS && Constructor && !Constructor[SPECIES]) {\n    defineProperty(Constructor, SPECIES, {\n      configurable: true,\n      get: function () { return this; }\n    });\n  }\n};\n","'use strict';\nvar defineProperty = require('../internals/object-define-property').f;\nvar create = require('../internals/object-create');\nvar redefineAll = require('../internals/redefine-all');\nvar bind = require('../internals/function-bind-context');\nvar anInstance = require('../internals/an-instance');\nvar iterate = require('../internals/iterate');\nvar defineIterator = require('../internals/define-iterator');\nvar setSpecies = require('../internals/set-species');\nvar DESCRIPTORS = require('../internals/descriptors');\nvar fastKey = require('../internals/internal-metadata').fastKey;\nvar InternalStateModule = require('../internals/internal-state');\n\nvar setInternalState = InternalStateModule.set;\nvar internalStateGetterFor = InternalStateModule.getterFor;\n\nmodule.exports = {\n  getConstructor: function (wrapper, CONSTRUCTOR_NAME, IS_MAP, ADDER) {\n    var C = wrapper(function (that, iterable) {\n      anInstance(that, C, CONSTRUCTOR_NAME);\n      setInternalState(that, {\n        type: CONSTRUCTOR_NAME,\n        index: create(null),\n        first: undefined,\n        last: undefined,\n        size: 0\n      });\n      if (!DESCRIPTORS) that.size = 0;\n      if (iterable != undefined) iterate(iterable, that[ADDER], that, IS_MAP);\n    });\n\n    var getInternalState = internalStateGetterFor(CONSTRUCTOR_NAME);\n\n    var define = function (that, key, value) {\n      var state = getInternalState(that);\n      var entry = getEntry(that, key);\n      var previous, index;\n      // change existing entry\n      if (entry) {\n        entry.value = value;\n      // create new entry\n      } else {\n        state.last = entry = {\n          index: index = fastKey(key, true),\n          key: key,\n          value: value,\n          previous: previous = state.last,\n          next: undefined,\n          removed: false\n        };\n        if (!state.first) state.first = entry;\n        if (previous) previous.next = entry;\n        if (DESCRIPTORS) state.size++;\n        else that.size++;\n        // add to index\n        if (index !== 'F') state.index[index] = entry;\n      } return that;\n    };\n\n    var getEntry = function (that, key) {\n      var state = getInternalState(that);\n      // fast case\n      var index = fastKey(key);\n      var entry;\n      if (index !== 'F') return state.index[index];\n      // frozen object case\n      for (entry = state.first; entry; entry = entry.next) {\n        if (entry.key == key) return entry;\n      }\n    };\n\n    redefineAll(C.prototype, {\n      // 23.1.3.1 Map.prototype.clear()\n      // 23.2.3.2 Set.prototype.clear()\n      clear: function clear() {\n        var that = this;\n        var state = getInternalState(that);\n        var data = state.index;\n        var entry = state.first;\n        while (entry) {\n          entry.removed = true;\n          if (entry.previous) entry.previous = entry.previous.next = undefined;\n          delete data[entry.index];\n          entry = entry.next;\n        }\n        state.first = state.last = undefined;\n        if (DESCRIPTORS) state.size = 0;\n        else that.size = 0;\n      },\n      // 23.1.3.3 Map.prototype.delete(key)\n      // 23.2.3.4 Set.prototype.delete(value)\n      'delete': function (key) {\n        var that = this;\n        var state = getInternalState(that);\n        var entry = getEntry(that, key);\n        if (entry) {\n          var next = entry.next;\n          var prev = entry.previous;\n          delete state.index[entry.index];\n          entry.removed = true;\n          if (prev) prev.next = next;\n          if (next) next.previous = prev;\n          if (state.first == entry) state.first = next;\n          if (state.last == entry) state.last = prev;\n          if (DESCRIPTORS) state.size--;\n          else that.size--;\n        } return !!entry;\n      },\n      // 23.2.3.6 Set.prototype.forEach(callbackfn, thisArg = undefined)\n      // 23.1.3.5 Map.prototype.forEach(callbackfn, thisArg = undefined)\n      forEach: function forEach(callbackfn /* , that = undefined */) {\n        var state = getInternalState(this);\n        var boundFunction = bind(callbackfn, arguments.length > 1 ? arguments[1] : undefined, 3);\n        var entry;\n        while (entry = entry ? entry.next : state.first) {\n          boundFunction(entry.value, entry.key, this);\n          // revert to the last existing entry\n          while (entry && entry.removed) entry = entry.previous;\n        }\n      },\n      // 23.1.3.7 Map.prototype.has(key)\n      // 23.2.3.7 Set.prototype.has(value)\n      has: function has(key) {\n        return !!getEntry(this, key);\n      }\n    });\n\n    redefineAll(C.prototype, IS_MAP ? {\n      // 23.1.3.6 Map.prototype.get(key)\n      get: function get(key) {\n        var entry = getEntry(this, key);\n        return entry && entry.value;\n      },\n      // 23.1.3.9 Map.prototype.set(key, value)\n      set: function set(key, value) {\n        return define(this, key === 0 ? 0 : key, value);\n      }\n    } : {\n      // 23.2.3.1 Set.prototype.add(value)\n      add: function add(value) {\n        return define(this, value = value === 0 ? 0 : value, value);\n      }\n    });\n    if (DESCRIPTORS) defineProperty(C.prototype, 'size', {\n      get: function () {\n        return getInternalState(this).size;\n      }\n    });\n    return C;\n  },\n  setStrong: function (C, CONSTRUCTOR_NAME, IS_MAP) {\n    var ITERATOR_NAME = CONSTRUCTOR_NAME + ' Iterator';\n    var getInternalCollectionState = internalStateGetterFor(CONSTRUCTOR_NAME);\n    var getInternalIteratorState = internalStateGetterFor(ITERATOR_NAME);\n    // add .keys, .values, .entries, [@@iterator]\n    // 23.1.3.4, 23.1.3.8, 23.1.3.11, 23.1.3.12, 23.2.3.5, 23.2.3.8, 23.2.3.10, 23.2.3.11\n    defineIterator(C, CONSTRUCTOR_NAME, function (iterated, kind) {\n      setInternalState(this, {\n        type: ITERATOR_NAME,\n        target: iterated,\n        state: getInternalCollectionState(iterated),\n        kind: kind,\n        last: undefined\n      });\n    }, function () {\n      var state = getInternalIteratorState(this);\n      var kind = state.kind;\n      var entry = state.last;\n      // revert to the last existing entry\n      while (entry && entry.removed) entry = entry.previous;\n      // get next entry\n      if (!state.target || !(state.last = entry = entry ? entry.next : state.state.first)) {\n        // or finish the iteration\n        state.target = undefined;\n        return { value: undefined, done: true };\n      }\n      // return step by kind\n      if (kind == 'keys') return { value: entry.key, done: false };\n      if (kind == 'values') return { value: entry.value, done: false };\n      return { value: [entry.key, entry.value], done: false };\n    }, IS_MAP ? 'entries' : 'values', !IS_MAP, true);\n\n    // add [@@species], 23.1.2.2, 23.2.2.2\n    setSpecies(CONSTRUCTOR_NAME);\n  }\n};\n","'use strict';\nvar collection = require('../internals/collection');\nvar collectionStrong = require('../internals/collection-strong');\n\n// `Set` constructor\n// https://tc39.github.io/ecma262/#sec-set-objects\nmodule.exports = collection('Set', function (init) {\n  return function Set() { return init(this, arguments.length ? arguments[0] : undefined); };\n}, collectionStrong);\n","require('../../modules/es.set');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Set;\n","'use strict';\nvar collection = require('../internals/collection');\nvar collectionStrong = require('../internals/collection-strong');\n\n// `Map` constructor\n// https://tc39.github.io/ecma262/#sec-map-objects\nmodule.exports = collection('Map', function (init) {\n  return function Map() { return init(this, arguments.length ? arguments[0] : undefined); };\n}, collectionStrong);\n","require('../../modules/es.map');\nrequire('../../modules/es.object.to-string');\nrequire('../../modules/es.string.iterator');\nrequire('../../modules/web.dom-collections.iterator');\nvar path = require('../../internals/path');\n\nmodule.exports = path.Map;\n","/**\n * Copyright (c) 2014-present, Facebook, Inc.\n *\n * This source code is licensed under the MIT license found in the\n * LICENSE file in the root directory of this source tree.\n */\n\nvar runtime = (function (exports) {\n  \"use strict\";\n\n  var Op = Object.prototype;\n  var hasOwn = Op.hasOwnProperty;\n  var undefined; // More compressible than void 0.\n  var $Symbol = typeof Symbol === \"function\" ? Symbol : {};\n  var iteratorSymbol = $Symbol.iterator || \"@@iterator\";\n  var asyncIteratorSymbol = $Symbol.asyncIterator || \"@@asyncIterator\";\n  var toStringTagSymbol = $Symbol.toStringTag || \"@@toStringTag\";\n\n  function wrap(innerFn, outerFn, self, tryLocsList) {\n    // If outerFn provided and outerFn.prototype is a Generator, then outerFn.prototype instanceof Generator.\n    var protoGenerator = outerFn && outerFn.prototype instanceof Generator ? outerFn : Generator;\n    var generator = Object.create(protoGenerator.prototype);\n    var context = new Context(tryLocsList || []);\n\n    // The ._invoke method unifies the implementations of the .next,\n    // .throw, and .return methods.\n    generator._invoke = makeInvokeMethod(innerFn, self, context);\n\n    return generator;\n  }\n  exports.wrap = wrap;\n\n  // Try/catch helper to minimize deoptimizations. Returns a completion\n  // record like context.tryEntries[i].completion. This interface could\n  // have been (and was previously) designed to take a closure to be\n  // invoked without arguments, but in all the cases we care about we\n  // already have an existing method we want to call, so there's no need\n  // to create a new function object. We can even get away with assuming\n  // the method takes exactly one argument, since that happens to be true\n  // in every case, so we don't have to touch the arguments object. The\n  // only additional allocation required is the completion record, which\n  // has a stable shape and so hopefully should be cheap to allocate.\n  function tryCatch(fn, obj, arg) {\n    try {\n      return { type: \"normal\", arg: fn.call(obj, arg) };\n    } catch (err) {\n      return { type: \"throw\", arg: err };\n    }\n  }\n\n  var GenStateSuspendedStart = \"suspendedStart\";\n  var GenStateSuspendedYield = \"suspendedYield\";\n  var GenStateExecuting = \"executing\";\n  var GenStateCompleted = \"completed\";\n\n  // Returning this object from the innerFn has the same effect as\n  // breaking out of the dispatch switch statement.\n  var ContinueSentinel = {};\n\n  // Dummy constructor functions that we use as the .constructor and\n  // .constructor.prototype properties for functions that return Generator\n  // objects. For full spec compliance, you may wish to configure your\n  // minifier not to mangle the names of these two functions.\n  function Generator() {}\n  function GeneratorFunction() {}\n  function GeneratorFunctionPrototype() {}\n\n  // This is a polyfill for %IteratorPrototype% for environments that\n  // don't natively support it.\n  var IteratorPrototype = {};\n  IteratorPrototype[iteratorSymbol] = function () {\n    return this;\n  };\n\n  var getProto = Object.getPrototypeOf;\n  var NativeIteratorPrototype = getProto && getProto(getProto(values([])));\n  if (NativeIteratorPrototype &&\n      NativeIteratorPrototype !== Op &&\n      hasOwn.call(NativeIteratorPrototype, iteratorSymbol)) {\n    // This environment has a native %IteratorPrototype%; use it instead\n    // of the polyfill.\n    IteratorPrototype = NativeIteratorPrototype;\n  }\n\n  var Gp = GeneratorFunctionPrototype.prototype =\n    Generator.prototype = Object.create(IteratorPrototype);\n  GeneratorFunction.prototype = Gp.constructor = GeneratorFunctionPrototype;\n  GeneratorFunctionPrototype.constructor = GeneratorFunction;\n  GeneratorFunctionPrototype[toStringTagSymbol] =\n    GeneratorFunction.displayName = \"GeneratorFunction\";\n\n  // Helper for defining the .next, .throw, and .return methods of the\n  // Iterator interface in terms of a single ._invoke method.\n  function defineIteratorMethods(prototype) {\n    [\"next\", \"throw\", \"return\"].forEach(function(method) {\n      prototype[method] = function(arg) {\n        return this._invoke(method, arg);\n      };\n    });\n  }\n\n  exports.isGeneratorFunction = function(genFun) {\n    var ctor = typeof genFun === \"function\" && genFun.constructor;\n    return ctor\n      ? ctor === GeneratorFunction ||\n        // For the native GeneratorFunction constructor, the best we can\n        // do is to check its .name property.\n        (ctor.displayName || ctor.name) === \"GeneratorFunction\"\n      : false;\n  };\n\n  exports.mark = function(genFun) {\n    if (Object.setPrototypeOf) {\n      Object.setPrototypeOf(genFun, GeneratorFunctionPrototype);\n    } else {\n      genFun.__proto__ = GeneratorFunctionPrototype;\n      if (!(toStringTagSymbol in genFun)) {\n        genFun[toStringTagSymbol] = \"GeneratorFunction\";\n      }\n    }\n    genFun.prototype = Object.create(Gp);\n    return genFun;\n  };\n\n  // Within the body of any async function, `await x` is transformed to\n  // `yield regeneratorRuntime.awrap(x)`, so that the runtime can test\n  // `hasOwn.call(value, \"__await\")` to determine if the yielded value is\n  // meant to be awaited.\n  exports.awrap = function(arg) {\n    return { __await: arg };\n  };\n\n  function AsyncIterator(generator, PromiseImpl) {\n    function invoke(method, arg, resolve, reject) {\n      var record = tryCatch(generator[method], generator, arg);\n      if (record.type === \"throw\") {\n        reject(record.arg);\n      } else {\n        var result = record.arg;\n        var value = result.value;\n        if (value &&\n            typeof value === \"object\" &&\n            hasOwn.call(value, \"__await\")) {\n          return PromiseImpl.resolve(value.__await).then(function(value) {\n            invoke(\"next\", value, resolve, reject);\n          }, function(err) {\n            invoke(\"throw\", err, resolve, reject);\n          });\n        }\n\n        return PromiseImpl.resolve(value).then(function(unwrapped) {\n          // When a yielded Promise is resolved, its final value becomes\n          // the .value of the Promise<{value,done}> result for the\n          // current iteration.\n          result.value = unwrapped;\n          resolve(result);\n        }, function(error) {\n          // If a rejected Promise was yielded, throw the rejection back\n          // into the async generator function so it can be handled there.\n          return invoke(\"throw\", error, resolve, reject);\n        });\n      }\n    }\n\n    var previousPromise;\n\n    function enqueue(method, arg) {\n      function callInvokeWithMethodAndArg() {\n        return new PromiseImpl(function(resolve, reject) {\n          invoke(method, arg, resolve, reject);\n        });\n      }\n\n      return previousPromise =\n        // If enqueue has been called before, then we want to wait until\n        // all previous Promises have been resolved before calling invoke,\n        // so that results are always delivered in the correct order. If\n        // enqueue has not been called before, then it is important to\n        // call invoke immediately, without waiting on a callback to fire,\n        // so that the async generator function has the opportunity to do\n        // any necessary setup in a predictable way. This predictability\n        // is why the Promise constructor synchronously invokes its\n        // executor callback, and why async functions synchronously\n        // execute code before the first await. Since we implement simple\n        // async functions in terms of async generators, it is especially\n        // important to get this right, even though it requires care.\n        previousPromise ? previousPromise.then(\n          callInvokeWithMethodAndArg,\n          // Avoid propagating failures to Promises returned by later\n          // invocations of the iterator.\n          callInvokeWithMethodAndArg\n        ) : callInvokeWithMethodAndArg();\n    }\n\n    // Define the unified helper method that is used to implement .next,\n    // .throw, and .return (see defineIteratorMethods).\n    this._invoke = enqueue;\n  }\n\n  defineIteratorMethods(AsyncIterator.prototype);\n  AsyncIterator.prototype[asyncIteratorSymbol] = function () {\n    return this;\n  };\n  exports.AsyncIterator = AsyncIterator;\n\n  // Note that simple async functions are implemented on top of\n  // AsyncIterator objects; they just return a Promise for the value of\n  // the final result produced by the iterator.\n  exports.async = function(innerFn, outerFn, self, tryLocsList, PromiseImpl) {\n    if (PromiseImpl === void 0) PromiseImpl = Promise;\n\n    var iter = new AsyncIterator(\n      wrap(innerFn, outerFn, self, tryLocsList),\n      PromiseImpl\n    );\n\n    return exports.isGeneratorFunction(outerFn)\n      ? iter // If outerFn is a generator, return the full iterator.\n      : iter.next().then(function(result) {\n          return result.done ? result.value : iter.next();\n        });\n  };\n\n  function makeInvokeMethod(innerFn, self, context) {\n    var state = GenStateSuspendedStart;\n\n    return function invoke(method, arg) {\n      if (state === GenStateExecuting) {\n        throw new Error(\"Generator is already running\");\n      }\n\n      if (state === GenStateCompleted) {\n        if (method === \"throw\") {\n          throw arg;\n        }\n\n        // Be forgiving, per 25.3.3.3.3 of the spec:\n        // https://people.mozilla.org/~jorendorff/es6-draft.html#sec-generatorresume\n        return doneResult();\n      }\n\n      context.method = method;\n      context.arg = arg;\n\n      while (true) {\n        var delegate = context.delegate;\n        if (delegate) {\n          var delegateResult = maybeInvokeDelegate(delegate, context);\n          if (delegateResult) {\n            if (delegateResult === ContinueSentinel) continue;\n            return delegateResult;\n          }\n        }\n\n        if (context.method === \"next\") {\n          // Setting context._sent for legacy support of Babel's\n          // function.sent implementation.\n          context.sent = context._sent = context.arg;\n\n        } else if (context.method === \"throw\") {\n          if (state === GenStateSuspendedStart) {\n            state = GenStateCompleted;\n            throw context.arg;\n          }\n\n          context.dispatchException(context.arg);\n\n        } else if (context.method === \"return\") {\n          context.abrupt(\"return\", context.arg);\n        }\n\n        state = GenStateExecuting;\n\n        var record = tryCatch(innerFn, self, context);\n        if (record.type === \"normal\") {\n          // If an exception is thrown from innerFn, we leave state ===\n          // GenStateExecuting and loop back for another invocation.\n          state = context.done\n            ? GenStateCompleted\n            : GenStateSuspendedYield;\n\n          if (record.arg === ContinueSentinel) {\n            continue;\n          }\n\n          return {\n            value: record.arg,\n            done: context.done\n          };\n\n        } else if (record.type === \"throw\") {\n          state = GenStateCompleted;\n          // Dispatch the exception by looping back around to the\n          // context.dispatchException(context.arg) call above.\n          context.method = \"throw\";\n          context.arg = record.arg;\n        }\n      }\n    };\n  }\n\n  // Call delegate.iterator[context.method](context.arg) and handle the\n  // result, either by returning a { value, done } result from the\n  // delegate iterator, or by modifying context.method and context.arg,\n  // setting context.delegate to null, and returning the ContinueSentinel.\n  function maybeInvokeDelegate(delegate, context) {\n    var method = delegate.iterator[context.method];\n    if (method === undefined) {\n      // A .throw or .return when the delegate iterator has no .throw\n      // method always terminates the yield* loop.\n      context.delegate = null;\n\n      if (context.method === \"throw\") {\n        // Note: [\"return\"] must be used for ES3 parsing compatibility.\n        if (delegate.iterator[\"return\"]) {\n          // If the delegate iterator has a return method, give it a\n          // chance to clean up.\n          context.method = \"return\";\n          context.arg = undefined;\n          maybeInvokeDelegate(delegate, context);\n\n          if (context.method === \"throw\") {\n            // If maybeInvokeDelegate(context) changed context.method from\n            // \"return\" to \"throw\", let that override the TypeError below.\n            return ContinueSentinel;\n          }\n        }\n\n        context.method = \"throw\";\n        context.arg = new TypeError(\n          \"The iterator does not provide a 'throw' method\");\n      }\n\n      return ContinueSentinel;\n    }\n\n    var record = tryCatch(method, delegate.iterator, context.arg);\n\n    if (record.type === \"throw\") {\n      context.method = \"throw\";\n      context.arg = record.arg;\n      context.delegate = null;\n      return ContinueSentinel;\n    }\n\n    var info = record.arg;\n\n    if (! info) {\n      context.method = \"throw\";\n      context.arg = new TypeError(\"iterator result is not an object\");\n      context.delegate = null;\n      return ContinueSentinel;\n    }\n\n    if (info.done) {\n      // Assign the result of the finished delegate to the temporary\n      // variable specified by delegate.resultName (see delegateYield).\n      context[delegate.resultName] = info.value;\n\n      // Resume execution at the desired location (see delegateYield).\n      context.next = delegate.nextLoc;\n\n      // If context.method was \"throw\" but the delegate handled the\n      // exception, let the outer generator proceed normally. If\n      // context.method was \"next\", forget context.arg since it has been\n      // \"consumed\" by the delegate iterator. If context.method was\n      // \"return\", allow the original .return call to continue in the\n      // outer generator.\n      if (context.method !== \"return\") {\n        context.method = \"next\";\n        context.arg = undefined;\n      }\n\n    } else {\n      // Re-yield the result returned by the delegate method.\n      return info;\n    }\n\n    // The delegate iterator is finished, so forget it and continue with\n    // the outer generator.\n    context.delegate = null;\n    return ContinueSentinel;\n  }\n\n  // Define Generator.prototype.{next,throw,return} in terms of the\n  // unified ._invoke helper method.\n  defineIteratorMethods(Gp);\n\n  Gp[toStringTagSymbol] = \"Generator\";\n\n  // A Generator should always return itself as the iterator object when the\n  // @@iterator function is called on it. Some browsers' implementations of the\n  // iterator prototype chain incorrectly implement this, causing the Generator\n  // object to not be returned from this call. This ensures that doesn't happen.\n  // See https://github.com/facebook/regenerator/issues/274 for more details.\n  Gp[iteratorSymbol] = function() {\n    return this;\n  };\n\n  Gp.toString = function() {\n    return \"[object Generator]\";\n  };\n\n  function pushTryEntry(locs) {\n    var entry = { tryLoc: locs[0] };\n\n    if (1 in locs) {\n      entry.catchLoc = locs[1];\n    }\n\n    if (2 in locs) {\n      entry.finallyLoc = locs[2];\n      entry.afterLoc = locs[3];\n    }\n\n    this.tryEntries.push(entry);\n  }\n\n  function resetTryEntry(entry) {\n    var record = entry.completion || {};\n    record.type = \"normal\";\n    delete record.arg;\n    entry.completion = record;\n  }\n\n  function Context(tryLocsList) {\n    // The root entry object (effectively a try statement without a catch\n    // or a finally block) gives us a place to store values thrown from\n    // locations where there is no enclosing try statement.\n    this.tryEntries = [{ tryLoc: \"root\" }];\n    tryLocsList.forEach(pushTryEntry, this);\n    this.reset(true);\n  }\n\n  exports.keys = function(object) {\n    var keys = [];\n    for (var key in object) {\n      keys.push(key);\n    }\n    keys.reverse();\n\n    // Rather than returning an object with a next method, we keep\n    // things simple and return the next function itself.\n    return function next() {\n      while (keys.length) {\n        var key = keys.pop();\n        if (key in object) {\n          next.value = key;\n          next.done = false;\n          return next;\n        }\n      }\n\n      // To avoid creating an additional object, we just hang the .value\n      // and .done properties off the next function object itself. This\n      // also ensures that the minifier will not anonymize the function.\n      next.done = true;\n      return next;\n    };\n  };\n\n  function values(iterable) {\n    if (iterable) {\n      var iteratorMethod = iterable[iteratorSymbol];\n      if (iteratorMethod) {\n        return iteratorMethod.call(iterable);\n      }\n\n      if (typeof iterable.next === \"function\") {\n        return iterable;\n      }\n\n      if (!isNaN(iterable.length)) {\n        var i = -1, next = function next() {\n          while (++i < iterable.length) {\n            if (hasOwn.call(iterable, i)) {\n              next.value = iterable[i];\n              next.done = false;\n              return next;\n            }\n          }\n\n          next.value = undefined;\n          next.done = true;\n\n          return next;\n        };\n\n        return next.next = next;\n      }\n    }\n\n    // Return an iterator with no values.\n    return { next: doneResult };\n  }\n  exports.values = values;\n\n  function doneResult() {\n    return { value: undefined, done: true };\n  }\n\n  Context.prototype = {\n    constructor: Context,\n\n    reset: function(skipTempReset) {\n      this.prev = 0;\n      this.next = 0;\n      // Resetting context._sent for legacy support of Babel's\n      // function.sent implementation.\n      this.sent = this._sent = undefined;\n      this.done = false;\n      this.delegate = null;\n\n      this.method = \"next\";\n      this.arg = undefined;\n\n      this.tryEntries.forEach(resetTryEntry);\n\n      if (!skipTempReset) {\n        for (var name in this) {\n          // Not sure about the optimal order of these conditions:\n          if (name.charAt(0) === \"t\" &&\n              hasOwn.call(this, name) &&\n              !isNaN(+name.slice(1))) {\n            this[name] = undefined;\n          }\n        }\n      }\n    },\n\n    stop: function() {\n      this.done = true;\n\n      var rootEntry = this.tryEntries[0];\n      var rootRecord = rootEntry.completion;\n      if (rootRecord.type === \"throw\") {\n        throw rootRecord.arg;\n      }\n\n      return this.rval;\n    },\n\n    dispatchException: function(exception) {\n      if (this.done) {\n        throw exception;\n      }\n\n      var context = this;\n      function handle(loc, caught) {\n        record.type = \"throw\";\n        record.arg = exception;\n        context.next = loc;\n\n        if (caught) {\n          // If the dispatched exception was caught by a catch block,\n          // then let that catch block handle the exception normally.\n          context.method = \"next\";\n          context.arg = undefined;\n        }\n\n        return !! caught;\n      }\n\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        var record = entry.completion;\n\n        if (entry.tryLoc === \"root\") {\n          // Exception thrown outside of any try block that could handle\n          // it, so set the completion value of the entire function to\n          // throw the exception.\n          return handle(\"end\");\n        }\n\n        if (entry.tryLoc <= this.prev) {\n          var hasCatch = hasOwn.call(entry, \"catchLoc\");\n          var hasFinally = hasOwn.call(entry, \"finallyLoc\");\n\n          if (hasCatch && hasFinally) {\n            if (this.prev < entry.catchLoc) {\n              return handle(entry.catchLoc, true);\n            } else if (this.prev < entry.finallyLoc) {\n              return handle(entry.finallyLoc);\n            }\n\n          } else if (hasCatch) {\n            if (this.prev < entry.catchLoc) {\n              return handle(entry.catchLoc, true);\n            }\n\n          } else if (hasFinally) {\n            if (this.prev < entry.finallyLoc) {\n              return handle(entry.finallyLoc);\n            }\n\n          } else {\n            throw new Error(\"try statement without catch or finally\");\n          }\n        }\n      }\n    },\n\n    abrupt: function(type, arg) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.tryLoc <= this.prev &&\n            hasOwn.call(entry, \"finallyLoc\") &&\n            this.prev < entry.finallyLoc) {\n          var finallyEntry = entry;\n          break;\n        }\n      }\n\n      if (finallyEntry &&\n          (type === \"break\" ||\n           type === \"continue\") &&\n          finallyEntry.tryLoc <= arg &&\n          arg <= finallyEntry.finallyLoc) {\n        // Ignore the finally entry if control is not jumping to a\n        // location outside the try/catch block.\n        finallyEntry = null;\n      }\n\n      var record = finallyEntry ? finallyEntry.completion : {};\n      record.type = type;\n      record.arg = arg;\n\n      if (finallyEntry) {\n        this.method = \"next\";\n        this.next = finallyEntry.finallyLoc;\n        return ContinueSentinel;\n      }\n\n      return this.complete(record);\n    },\n\n    complete: function(record, afterLoc) {\n      if (record.type === \"throw\") {\n        throw record.arg;\n      }\n\n      if (record.type === \"break\" ||\n          record.type === \"continue\") {\n        this.next = record.arg;\n      } else if (record.type === \"return\") {\n        this.rval = this.arg = record.arg;\n        this.method = \"return\";\n        this.next = \"end\";\n      } else if (record.type === \"normal\" && afterLoc) {\n        this.next = afterLoc;\n      }\n\n      return ContinueSentinel;\n    },\n\n    finish: function(finallyLoc) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.finallyLoc === finallyLoc) {\n          this.complete(entry.completion, entry.afterLoc);\n          resetTryEntry(entry);\n          return ContinueSentinel;\n        }\n      }\n    },\n\n    \"catch\": function(tryLoc) {\n      for (var i = this.tryEntries.length - 1; i >= 0; --i) {\n        var entry = this.tryEntries[i];\n        if (entry.tryLoc === tryLoc) {\n          var record = entry.completion;\n          if (record.type === \"throw\") {\n            var thrown = record.arg;\n            resetTryEntry(entry);\n          }\n          return thrown;\n        }\n      }\n\n      // The context.catch method must only be called with a location\n      // argument that corresponds to a known catch block.\n      throw new Error(\"illegal catch attempt\");\n    },\n\n    delegateYield: function(iterable, resultName, nextLoc) {\n      this.delegate = {\n        iterator: values(iterable),\n        resultName: resultName,\n        nextLoc: nextLoc\n      };\n\n      if (this.method === \"next\") {\n        // Deliberately forget the last sent value so that we don't\n        // accidentally pass it on to the delegate.\n        this.arg = undefined;\n      }\n\n      return ContinueSentinel;\n    }\n  };\n\n  // Regardless of whether this script is executing as a CommonJS module\n  // or not, return the runtime object so that we can declare the variable\n  // regeneratorRuntime in the outer scope, which allows this module to be\n  // injected easily by `bin/regenerator --include-runtime script.js`.\n  return exports;\n\n}(\n  // If this script is executing as a CommonJS module, use module.exports\n  // as the regeneratorRuntime namespace. Otherwise create a new empty\n  // object. Either way, the resulting object will be used to initialize\n  // the regeneratorRuntime variable at the top of this file.\n  typeof module === \"object\" ? module.exports : {}\n));\n\ntry {\n  regeneratorRuntime = runtime;\n} catch (accidentalStrictMode) {\n  // This module should not be running in strict mode, so the above\n  // assignment should always work unless something is misconfigured. Just\n  // in case runtime.js accidentally runs in strict mode, we can escape\n  // strict mode using a global Function call. This could conceivably fail\n  // if a Content Security Policy forbids using Function, but in that case\n  // the proper solution is to fix the accidental strict mode problem. If\n  // you've misconfigured your bundler to force strict mode and applied a\n  // CSP to forbid Function, and you're not willing to fix either of those\n  // problems, please detail your unique predicament in a GitHub issue.\n  Function(\"r\", \"regeneratorRuntime = r\")(runtime);\n}\n","var parent = require('../../es/symbol/iterator');\n\nmodule.exports = parent;\n","'use strict';\nvar $ = require('../internals/export');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toInteger = require('../internals/to-integer');\nvar toLength = require('../internals/to-length');\nvar toObject = require('../internals/to-object');\nvar arraySpeciesCreate = require('../internals/array-species-create');\nvar createProperty = require('../internals/create-property');\nvar arrayMethodHasSpeciesSupport = require('../internals/array-method-has-species-support');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar HAS_SPECIES_SUPPORT = arrayMethodHasSpeciesSupport('splice');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('splice', { ACCESSORS: true, 0: 0, 1: 2 });\n\nvar max = Math.max;\nvar min = Math.min;\nvar MAX_SAFE_INTEGER = 0x1FFFFFFFFFFFFF;\nvar MAXIMUM_ALLOWED_LENGTH_EXCEEDED = 'Maximum allowed length exceeded';\n\n// `Array.prototype.splice` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.splice\n// with adding support of @@species\n$({ target: 'Array', proto: true, forced: !HAS_SPECIES_SUPPORT || !USES_TO_LENGTH }, {\n  splice: function splice(start, deleteCount /* , ...items */) {\n    var O = toObject(this);\n    var len = toLength(O.length);\n    var actualStart = toAbsoluteIndex(start, len);\n    var argumentsLength = arguments.length;\n    var insertCount, actualDeleteCount, A, k, from, to;\n    if (argumentsLength === 0) {\n      insertCount = actualDeleteCount = 0;\n    } else if (argumentsLength === 1) {\n      insertCount = 0;\n      actualDeleteCount = len - actualStart;\n    } else {\n      insertCount = argumentsLength - 2;\n      actualDeleteCount = min(max(toInteger(deleteCount), 0), len - actualStart);\n    }\n    if (len + insertCount - actualDeleteCount > MAX_SAFE_INTEGER) {\n      throw TypeError(MAXIMUM_ALLOWED_LENGTH_EXCEEDED);\n    }\n    A = arraySpeciesCreate(O, actualDeleteCount);\n    for (k = 0; k < actualDeleteCount; k++) {\n      from = actualStart + k;\n      if (from in O) createProperty(A, k, O[from]);\n    }\n    A.length = actualDeleteCount;\n    if (insertCount < actualDeleteCount) {\n      for (k = actualStart; k < len - actualDeleteCount; k++) {\n        from = k + actualDeleteCount;\n        to = k + insertCount;\n        if (from in O) O[to] = O[from];\n        else delete O[to];\n      }\n      for (k = len; k > len - actualDeleteCount + insertCount; k--) delete O[k - 1];\n    } else if (insertCount > actualDeleteCount) {\n      for (k = len - actualDeleteCount; k > actualStart; k--) {\n        from = k + actualDeleteCount - 1;\n        to = k + insertCount - 1;\n        if (from in O) O[to] = O[from];\n        else delete O[to];\n      }\n    }\n    for (k = 0; k < insertCount; k++) {\n      O[k + actualStart] = arguments[k + 2];\n    }\n    O.length = len - actualDeleteCount + insertCount;\n    return A;\n  }\n});\n","require('../../../modules/es.array.splice');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').splice;\n","var splice = require('../array/virtual/splice');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.splice;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.splice) ? splice : own;\n};\n","var $ = require('../internals/export');\nvar global = require('../internals/global');\nvar userAgent = require('../internals/engine-user-agent');\n\nvar slice = [].slice;\nvar MSIE = /MSIE .\\./.test(userAgent); // <- dirty ie9- check\n\nvar wrap = function (scheduler) {\n  return function (handler, timeout /* , ...arguments */) {\n    var boundArgs = arguments.length > 2;\n    var args = boundArgs ? slice.call(arguments, 2) : undefined;\n    return scheduler(boundArgs ? function () {\n      // eslint-disable-next-line no-new-func\n      (typeof handler == 'function' ? handler : Function(handler)).apply(this, args);\n    } : handler, timeout);\n  };\n};\n\n// ie9- setTimeout & setInterval additional parameters fix\n// https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#timers\n$({ global: true, bind: true, forced: MSIE }, {\n  // `setTimeout` method\n  // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-settimeout\n  setTimeout: wrap(global.setTimeout),\n  // `setInterval` method\n  // https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#dom-setinterval\n  setInterval: wrap(global.setInterval)\n});\n","require('../modules/web.timers');\nvar path = require('../internals/path');\n\nmodule.exports = path.setTimeout;\n","'use strict';\nvar isArray = require('../internals/is-array');\nvar toLength = require('../internals/to-length');\nvar bind = require('../internals/function-bind-context');\n\n// `FlattenIntoArray` abstract operation\n// https://tc39.github.io/proposal-flatMap/#sec-FlattenIntoArray\nvar flattenIntoArray = function (target, original, source, sourceLen, start, depth, mapper, thisArg) {\n  var targetIndex = start;\n  var sourceIndex = 0;\n  var mapFn = mapper ? bind(mapper, thisArg, 3) : false;\n  var element;\n\n  while (sourceIndex < sourceLen) {\n    if (sourceIndex in source) {\n      element = mapFn ? mapFn(source[sourceIndex], sourceIndex, original) : source[sourceIndex];\n\n      if (depth > 0 && isArray(element)) {\n        targetIndex = flattenIntoArray(target, original, element, toLength(element.length), targetIndex, depth - 1) - 1;\n      } else {\n        if (targetIndex >= 0x1FFFFFFFFFFFFF) throw TypeError('Exceed the acceptable array length');\n        target[targetIndex] = element;\n      }\n\n      targetIndex++;\n    }\n    sourceIndex++;\n  }\n  return targetIndex;\n};\n\nmodule.exports = flattenIntoArray;\n","'use strict';\nvar $ = require('../internals/export');\nvar flattenIntoArray = require('../internals/flatten-into-array');\nvar toObject = require('../internals/to-object');\nvar toLength = require('../internals/to-length');\nvar aFunction = require('../internals/a-function');\nvar arraySpeciesCreate = require('../internals/array-species-create');\n\n// `Array.prototype.flatMap` method\n// https://github.com/tc39/proposal-flatMap\n$({ target: 'Array', proto: true }, {\n  flatMap: function flatMap(callbackfn /* , thisArg */) {\n    var O = toObject(this);\n    var sourceLen = toLength(O.length);\n    var A;\n    aFunction(callbackfn);\n    A = arraySpeciesCreate(O, 0);\n    A.length = flattenIntoArray(A, O, O, sourceLen, 0, 1, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n    return A;\n  }\n});\n","require('../../../modules/es.array.flat-map');\nrequire('../../../modules/es.array.unscopables.flat-map');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').flatMap;\n","var flatMap = require('../array/virtual/flat-map');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.flatMap;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.flatMap) ? flatMap : own;\n};\n","var aFunction = require('../internals/a-function');\nvar toObject = require('../internals/to-object');\nvar IndexedObject = require('../internals/indexed-object');\nvar toLength = require('../internals/to-length');\n\n// `Array.prototype.{ reduce, reduceRight }` methods implementation\nvar createMethod = function (IS_RIGHT) {\n  return function (that, callbackfn, argumentsLength, memo) {\n    aFunction(callbackfn);\n    var O = toObject(that);\n    var self = IndexedObject(O);\n    var length = toLength(O.length);\n    var index = IS_RIGHT ? length - 1 : 0;\n    var i = IS_RIGHT ? -1 : 1;\n    if (argumentsLength < 2) while (true) {\n      if (index in self) {\n        memo = self[index];\n        index += i;\n        break;\n      }\n      index += i;\n      if (IS_RIGHT ? index < 0 : length <= index) {\n        throw TypeError('Reduce of empty array with no initial value');\n      }\n    }\n    for (;IS_RIGHT ? index >= 0 : length > index; index += i) if (index in self) {\n      memo = callbackfn(memo, self[index], index, O);\n    }\n    return memo;\n  };\n};\n\nmodule.exports = {\n  // `Array.prototype.reduce` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.reduce\n  left: createMethod(false),\n  // `Array.prototype.reduceRight` method\n  // https://tc39.github.io/ecma262/#sec-array.prototype.reduceright\n  right: createMethod(true)\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar $reduce = require('../internals/array-reduce').left;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('reduce');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('reduce', { 1: 0 });\n\n// `Array.prototype.reduce` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.reduce\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  reduce: function reduce(callbackfn /* , initialValue */) {\n    return $reduce(this, callbackfn, arguments.length, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.reduce');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').reduce;\n","var reduce = require('../array/virtual/reduce');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.reduce;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.reduce) ? reduce : own;\n};\n","'use strict';\nvar aFunction = require('../internals/a-function');\nvar isObject = require('../internals/is-object');\n\nvar slice = [].slice;\nvar factories = {};\n\nvar construct = function (C, argsLength, args) {\n  if (!(argsLength in factories)) {\n    for (var list = [], i = 0; i < argsLength; i++) list[i] = 'a[' + i + ']';\n    // eslint-disable-next-line no-new-func\n    factories[argsLength] = Function('C,a', 'return new C(' + list.join(',') + ')');\n  } return factories[argsLength](C, args);\n};\n\n// `Function.prototype.bind` method implementation\n// https://tc39.github.io/ecma262/#sec-function.prototype.bind\nmodule.exports = Function.bind || function bind(that /* , ...args */) {\n  var fn = aFunction(this);\n  var partArgs = slice.call(arguments, 1);\n  var boundFunction = function bound(/* args... */) {\n    var args = partArgs.concat(slice.call(arguments));\n    return this instanceof boundFunction ? construct(fn, args.length, args) : fn.apply(that, args);\n  };\n  if (isObject(fn.prototype)) boundFunction.prototype = fn.prototype;\n  return boundFunction;\n};\n","var $ = require('../internals/export');\nvar bind = require('../internals/function-bind');\n\n// `Function.prototype.bind` method\n// https://tc39.github.io/ecma262/#sec-function.prototype.bind\n$({ target: 'Function', proto: true }, {\n  bind: bind\n});\n","require('../../../modules/es.function.bind');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Function').bind;\n","var bind = require('../function/virtual/bind');\n\nvar FunctionPrototype = Function.prototype;\n\nmodule.exports = function (it) {\n  var own = it.bind;\n  return it === FunctionPrototype || (it instanceof Function && own === FunctionPrototype.bind) ? bind : own;\n};\n","function _classCallCheck(instance, Constructor) {\n  if (!(instance instanceof Constructor)) {\n    throw new TypeError(\"Cannot call a class as a function\");\n  }\n}\n\nmodule.exports = _classCallCheck;","var _Object$defineProperty = require(\"../core-js/object/define-property\");\n\nfunction _defineProperties(target, props) {\n  for (var i = 0; i < props.length; i++) {\n    var descriptor = props[i];\n    descriptor.enumerable = descriptor.enumerable || false;\n    descriptor.configurable = true;\n    if (\"value\" in descriptor) descriptor.writable = true;\n\n    _Object$defineProperty(target, descriptor.key, descriptor);\n  }\n}\n\nfunction _createClass(Constructor, protoProps, staticProps) {\n  if (protoProps) _defineProperties(Constructor.prototype, protoProps);\n  if (staticProps) _defineProperties(Constructor, staticProps);\n  return Constructor;\n}\n\nmodule.exports = _createClass;","/**\n * vis-data\n * http://visjs.org/\n *\n * Manage unstructured data using DataSet. Add, update, and remove data, and listen for changes in the data.\n *\n * @version 6.5.0\n * @date    2020-03-07T21:44:22.378Z\n *\n * @copyright (c) 2011-2017 Almende B.V, http://almende.com\n * @copyright (c) 2017-2019 visjs contributors, https://github.com/visjs\n *\n * @license\n * vis.js is dual licensed under both\n *\n *   1. The Apache 2.0 License\n *      http://www.apache.org/licenses/LICENSE-2.0\n *\n *   and\n *\n *   2. The MIT License\n *      http://opensource.org/licenses/MIT\n *\n * vis.js may be distributed under either license.\n */\n\nimport { isNumber, isString, getType, deepExtend } from 'vis-util/esnext';\nimport moment from 'moment';\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\n/**\r\n * Create new data pipe.\r\n *\r\n * @param from - The source data set or data view.\r\n *\r\n * @remarks\r\n * Example usage:\r\n * ```typescript\r\n * interface AppItem {\r\n *   whoami: string;\r\n *   appData: unknown;\r\n *   visData: VisItem;\r\n * }\r\n * interface VisItem {\r\n *   id: number;\r\n *   label: string;\r\n *   color: string;\r\n *   x: number;\r\n *   y: number;\r\n * }\r\n *\r\n * const ds1 = new DataSet<AppItem, \"whoami\">([], { fieldId: \"whoami\" });\r\n * const ds2 = new DataSet<VisItem, \"id\">();\r\n *\r\n * const pipe = createNewDataPipeFrom(ds1)\r\n *   .filter((item): boolean => item.enabled === true)\r\n *   .map<VisItem, \"id\">((item): VisItem => item.visData)\r\n *   .to(ds2);\r\n *\r\n * pipe.start();\r\n * ```\r\n *\r\n * @returns A factory whose methods can be used to configure the pipe.\r\n */\r\nfunction createNewDataPipeFrom(from) {\r\n    return new DataPipeUnderConstruction(from);\r\n}\r\n/**\r\n * Internal implementation of the pipe. This should be accessible only through\r\n * `createNewDataPipeFrom` from the outside.\r\n *\r\n * @typeparam SI - Source item type.\r\n * @typeparam SP - Source item type's id property name.\r\n * @typeparam TI - Target item type.\r\n * @typeparam TP - Target item type's id property name.\r\n */\r\nclass SimpleDataPipe {\r\n    /**\r\n     * Create a new data pipe.\r\n     *\r\n     * @param _source - The data set or data view that will be observed.\r\n     * @param _transformers - An array of transforming functions to be used to\r\n     * filter or transform the items in the pipe.\r\n     * @param _target - The data set or data view that will receive the items.\r\n     */\r\n    constructor(_source, _transformers, _target) {\r\n        this._source = _source;\r\n        this._transformers = _transformers;\r\n        this._target = _target;\r\n        /**\r\n         * Bound listeners for use with `DataInterface['on' | 'off']`.\r\n         */\r\n        this._listeners = {\r\n            add: this._add.bind(this),\r\n            remove: this._remove.bind(this),\r\n            update: this._update.bind(this)\r\n        };\r\n    }\r\n    /** @inheritdoc */\r\n    all() {\r\n        this._target.update(this._transformItems(this._source.get()));\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    start() {\r\n        this._source.on(\"add\", this._listeners.add);\r\n        this._source.on(\"remove\", this._listeners.remove);\r\n        this._source.on(\"update\", this._listeners.update);\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    stop() {\r\n        this._source.off(\"add\", this._listeners.add);\r\n        this._source.off(\"remove\", this._listeners.remove);\r\n        this._source.off(\"update\", this._listeners.update);\r\n        return this;\r\n    }\r\n    /**\r\n     * Apply the transformers to the items.\r\n     *\r\n     * @param items - The items to be transformed.\r\n     *\r\n     * @returns The transformed items.\r\n     */\r\n    _transformItems(items) {\r\n        return this._transformers.reduce((items, transform) => {\r\n            return transform(items);\r\n        }, items);\r\n    }\r\n    /**\r\n     * Handle an add event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the ids of the added items.\r\n     */\r\n    _add(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.add(this._transformItems(this._source.get(payload.items)));\r\n    }\r\n    /**\r\n     * Handle an update event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the ids of the updated items.\r\n     */\r\n    _update(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.update(this._transformItems(this._source.get(payload.items)));\r\n    }\r\n    /**\r\n     * Handle a remove event.\r\n     *\r\n     * @param _name - Ignored.\r\n     * @param payload - The payload containing the data of the removed items.\r\n     */\r\n    _remove(_name, payload) {\r\n        if (payload == null) {\r\n            return;\r\n        }\r\n        this._target.remove(this._transformItems(payload.oldData));\r\n    }\r\n}\r\n/**\r\n * Internal implementation of the pipe factory. This should be accessible\r\n * only through `createNewDataPipeFrom` from the outside.\r\n *\r\n * @typeparam TI - Target item type.\r\n * @typeparam TP - Target item type's id property name.\r\n */\r\nclass DataPipeUnderConstruction {\r\n    /**\r\n     * Create a new data pipe factory. This is an internal constructor that\r\n     * should never be called from outside of this file.\r\n     *\r\n     * @param _source - The source data set or data view for this pipe.\r\n     */\r\n    constructor(_source) {\r\n        this._source = _source;\r\n        /**\r\n         * Array transformers used to transform items within the pipe. This is typed\r\n         * as any for the sake of simplicity.\r\n         */\r\n        this._transformers = [];\r\n    }\r\n    /**\r\n     * Filter the items.\r\n     *\r\n     * @param callback - A filtering function that returns true if given item\r\n     * should be piped and false if not.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    filter(callback) {\r\n        this._transformers.push((input) => input.filter(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Map each source item to a new type.\r\n     *\r\n     * @param callback - A mapping function that takes a source item and returns\r\n     * corresponding mapped item.\r\n     *\r\n     * @typeparam TI - Target item type.\r\n     * @typeparam TP - Target item type's id property name.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    map(callback) {\r\n        this._transformers.push((input) => input.map(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Map each source item to zero or more items of a new type.\r\n     *\r\n     * @param callback - A mapping function that takes a source item and returns\r\n     * an array of corresponding mapped items.\r\n     *\r\n     * @typeparam TI - Target item type.\r\n     * @typeparam TP - Target item type's id property name.\r\n     *\r\n     * @returns This factory for further configuration.\r\n     */\r\n    flatMap(callback) {\r\n        this._transformers.push((input) => input.flatMap(callback));\r\n        return this;\r\n    }\r\n    /**\r\n     * Connect this pipe to given data set.\r\n     *\r\n     * @param target - The data set that will receive the items from this pipe.\r\n     *\r\n     * @returns The pipe connected between given data sets and performing\r\n     * configured transformation on the processed items.\r\n     */\r\n    to(target) {\r\n        return new SimpleDataPipe(this._source, this._transformers, target);\r\n    }\r\n}\n\nfunction createCommonjsModule(fn, module) {\n\treturn module = { exports: {} }, fn(module, module.exports), module.exports;\n}\n\nvar rngBrowser = createCommonjsModule(function (module) {\n// Unique ID creation requires a high quality random # generator.  In the\n// browser this is a little complicated due to unknown quality of Math.random()\n// and inconsistent support for the `crypto` API.  We do the best we can via\n// feature-detection\n\n// getRandomValues needs to be invoked in a context where \"this\" is a Crypto\n// implementation. Also, find the complete implementation of crypto on IE11.\nvar getRandomValues = (typeof(crypto) != 'undefined' && crypto.getRandomValues && crypto.getRandomValues.bind(crypto)) ||\n                      (typeof(msCrypto) != 'undefined' && typeof window.msCrypto.getRandomValues == 'function' && msCrypto.getRandomValues.bind(msCrypto));\n\nif (getRandomValues) {\n  // WHATWG crypto RNG - http://wiki.whatwg.org/wiki/Crypto\n  var rnds8 = new Uint8Array(16); // eslint-disable-line no-undef\n\n  module.exports = function whatwgRNG() {\n    getRandomValues(rnds8);\n    return rnds8;\n  };\n} else {\n  // Math.random()-based (RNG)\n  //\n  // If all else fails, use Math.random().  It's fast, but is of unspecified\n  // quality.\n  var rnds = new Array(16);\n\n  module.exports = function mathRNG() {\n    for (var i = 0, r; i < 16; i++) {\n      if ((i & 0x03) === 0) r = Math.random() * 0x100000000;\n      rnds[i] = r >>> ((i & 0x03) << 3) & 0xff;\n    }\n\n    return rnds;\n  };\n}\n});\n\n/**\n * Convert array of 16 byte values to UUID string format of the form:\n * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n */\nvar byteToHex = [];\nfor (var i = 0; i < 256; ++i) {\n  byteToHex[i] = (i + 0x100).toString(16).substr(1);\n}\n\nfunction bytesToUuid(buf, offset) {\n  var i = offset || 0;\n  var bth = byteToHex;\n  // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4\n  return ([\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]], '-',\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]],\n    bth[buf[i++]], bth[buf[i++]]\n  ]).join('');\n}\n\nvar bytesToUuid_1 = bytesToUuid;\n\nfunction v4(options, buf, offset) {\n  var i = buf && offset || 0;\n\n  if (typeof(options) == 'string') {\n    buf = options === 'binary' ? new Array(16) : null;\n    options = null;\n  }\n  options = options || {};\n\n  var rnds = options.random || (options.rng || rngBrowser)();\n\n  // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`\n  rnds[6] = (rnds[6] & 0x0f) | 0x40;\n  rnds[8] = (rnds[8] & 0x3f) | 0x80;\n\n  // Copy bytes to buffer, if provided\n  if (buf) {\n    for (var ii = 0; ii < 16; ++ii) {\n      buf[i + ii] = rnds[ii];\n    }\n  }\n\n  return buf || bytesToUuid_1(rnds);\n}\n\nvar v4_1 = v4;\n\n// utility functions\r\n// parse ASP.Net Date pattern,\r\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\r\n// code from http://momentjs.com/\r\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\r\n/**\r\n * Test whether given object is a Moment date.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if Moment instance, false otherwise.\r\n */\r\nfunction isMoment(value) {\r\n    return moment.isMoment(value);\r\n}\r\n/**\r\n * Convert an object into another type\r\n *\r\n * @param object - Value of unknown type.\r\n * @param type - Name of the desired type.\r\n *\r\n * @returns Object in the desired type.\r\n * @throws Error\r\n */\r\nfunction convert(object, type) {\r\n    let match;\r\n    if (object === undefined) {\r\n        return undefined;\r\n    }\r\n    if (object === null) {\r\n        return null;\r\n    }\r\n    if (!type) {\r\n        return object;\r\n    }\r\n    if (!(typeof type === \"string\") && !(type instanceof String)) {\r\n        throw new Error(\"Type must be a string\");\r\n    }\r\n    //noinspection FallthroughInSwitchStatementJS\r\n    switch (type) {\r\n        case \"boolean\":\r\n        case \"Boolean\":\r\n            return Boolean(object);\r\n        case \"number\":\r\n        case \"Number\":\r\n            if (isString(object) && !isNaN(Date.parse(object))) {\r\n                return moment(object).valueOf();\r\n            }\r\n            else {\r\n                // @TODO: I don't think that Number and String constructors are a good idea.\r\n                // This could also fail if the object doesn't have valueOf method or if it's redefined.\r\n                // For example: Object.create(null) or { valueOf: 7 }.\r\n                return Number(object.valueOf());\r\n            }\r\n        case \"string\":\r\n        case \"String\":\r\n            return String(object);\r\n        case \"Date\":\r\n            if (isNumber(object)) {\r\n                return new Date(object);\r\n            }\r\n            if (object instanceof Date) {\r\n                return new Date(object.valueOf());\r\n            }\r\n            else if (isMoment(object)) {\r\n                return new Date(object.valueOf());\r\n            }\r\n            if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return new Date(Number(match[1])); // parse number\r\n                }\r\n                else {\r\n                    return moment(new Date(object)).toDate(); // parse string\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" + getType(object) + \" to type Date\");\r\n            }\r\n        case \"Moment\":\r\n            if (isNumber(object)) {\r\n                return moment(object);\r\n            }\r\n            if (object instanceof Date) {\r\n                return moment(object.valueOf());\r\n            }\r\n            else if (isMoment(object)) {\r\n                return moment(object);\r\n            }\r\n            if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return moment(Number(match[1])); // parse number\r\n                }\r\n                else {\r\n                    return moment(object); // parse string\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" + getType(object) + \" to type Date\");\r\n            }\r\n        case \"ISODate\":\r\n            if (isNumber(object)) {\r\n                return new Date(object);\r\n            }\r\n            else if (object instanceof Date) {\r\n                return object.toISOString();\r\n            }\r\n            else if (isMoment(object)) {\r\n                return object.toDate().toISOString();\r\n            }\r\n            else if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    return new Date(Number(match[1])).toISOString(); // parse number\r\n                }\r\n                else {\r\n                    return moment(object).format(); // ISO 8601\r\n                }\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" +\r\n                    getType(object) +\r\n                    \" to type ISODate\");\r\n            }\r\n        case \"ASPDate\":\r\n            if (isNumber(object)) {\r\n                return \"/Date(\" + object + \")/\";\r\n            }\r\n            else if (object instanceof Date || isMoment(object)) {\r\n                return \"/Date(\" + object.valueOf() + \")/\";\r\n            }\r\n            else if (isString(object)) {\r\n                match = ASPDateRegex.exec(object);\r\n                let value;\r\n                if (match) {\r\n                    // object is an ASP date\r\n                    value = new Date(Number(match[1])).valueOf(); // parse number\r\n                }\r\n                else {\r\n                    value = new Date(object).valueOf(); // parse string\r\n                }\r\n                return \"/Date(\" + value + \")/\";\r\n            }\r\n            else {\r\n                throw new Error(\"Cannot convert object of type \" +\r\n                    getType(object) +\r\n                    \" to type ASPDate\");\r\n            }\r\n        default:\r\n            const never = type;\r\n            throw new Error(`Unknown type ${never}`);\r\n    }\r\n}\n\n/**\r\n * Determine whether a value can be used as an id.\r\n *\r\n * @param value - Input value of unknown type.\r\n *\r\n * @returns True if the value is valid id, false otherwise.\r\n */\r\nfunction isId(value) {\r\n    return typeof value === \"string\" || typeof value === \"number\";\r\n}\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\n/**\r\n * A queue.\r\n *\r\n * @typeParam T - The type of method names to be replaced by queued versions.\r\n */\r\nclass Queue {\r\n    /**\r\n     * Construct a new Queue.\r\n     *\r\n     * @param options - Queue configuration.\r\n     */\r\n    constructor(options) {\r\n        this._queue = [];\r\n        this._timeout = null;\r\n        this._extended = null;\r\n        // options\r\n        this.delay = null;\r\n        this.max = Infinity;\r\n        this.setOptions(options);\r\n    }\r\n    /**\r\n     * Update the configuration of the queue.\r\n     *\r\n     * @param options - Queue configuration.\r\n     */\r\n    setOptions(options) {\r\n        if (options && typeof options.delay !== \"undefined\") {\r\n            this.delay = options.delay;\r\n        }\r\n        if (options && typeof options.max !== \"undefined\") {\r\n            this.max = options.max;\r\n        }\r\n        this._flushIfNeeded();\r\n    }\r\n    /**\r\n     * Extend an object with queuing functionality.\r\n     * The object will be extended with a function flush, and the methods provided in options.replace will be replaced with queued ones.\r\n     *\r\n     * @param object - The object to be extended.\r\n     * @param options - Additional options.\r\n     *\r\n     * @returns The created queue.\r\n     */\r\n    static extend(object, options) {\r\n        const queue = new Queue(options);\r\n        if (object.flush !== undefined) {\r\n            throw new Error(\"Target object already has a property flush\");\r\n        }\r\n        object.flush = () => {\r\n            queue.flush();\r\n        };\r\n        const methods = [\r\n            {\r\n                name: \"flush\",\r\n                original: undefined\r\n            }\r\n        ];\r\n        if (options && options.replace) {\r\n            for (let i = 0; i < options.replace.length; i++) {\r\n                const name = options.replace[i];\r\n                methods.push({\r\n                    name: name,\r\n                    // @TODO: better solution?\r\n                    original: object[name]\r\n                });\r\n                // @TODO: better solution?\r\n                queue.replace(object, name);\r\n            }\r\n        }\r\n        queue._extended = {\r\n            object: object,\r\n            methods: methods\r\n        };\r\n        return queue;\r\n    }\r\n    /**\r\n     * Destroy the queue. The queue will first flush all queued actions, and in case it has extended an object, will restore the original object.\r\n     */\r\n    destroy() {\r\n        this.flush();\r\n        if (this._extended) {\r\n            const object = this._extended.object;\r\n            const methods = this._extended.methods;\r\n            for (let i = 0; i < methods.length; i++) {\r\n                const method = methods[i];\r\n                if (method.original) {\r\n                    // @TODO: better solution?\r\n                    object[method.name] = method.original;\r\n                }\r\n                else {\r\n                    // @TODO: better solution?\r\n                    delete object[method.name];\r\n                }\r\n            }\r\n            this._extended = null;\r\n        }\r\n    }\r\n    /**\r\n     * Replace a method on an object with a queued version.\r\n     *\r\n     * @param object - Object having the method.\r\n     * @param method - The method name.\r\n     */\r\n    replace(object, method) {\r\n        /* eslint-disable-next-line @typescript-eslint/no-this-alias */\r\n        const me = this;\r\n        const original = object[method];\r\n        if (!original) {\r\n            throw new Error(\"Method \" + method + \" undefined\");\r\n        }\r\n        object[method] = function (...args) {\r\n            // add this call to the queue\r\n            me.queue({\r\n                args: args,\r\n                fn: original,\r\n                context: this\r\n            });\r\n        };\r\n    }\r\n    /**\r\n     * Queue a call.\r\n     *\r\n     * @param entry - The function or entry to be queued.\r\n     */\r\n    queue(entry) {\r\n        if (typeof entry === \"function\") {\r\n            this._queue.push({ fn: entry });\r\n        }\r\n        else {\r\n            this._queue.push(entry);\r\n        }\r\n        this._flushIfNeeded();\r\n    }\r\n    /**\r\n     * Check whether the queue needs to be flushed.\r\n     */\r\n    _flushIfNeeded() {\r\n        // flush when the maximum is exceeded.\r\n        if (this._queue.length > this.max) {\r\n            this.flush();\r\n        }\r\n        // flush after a period of inactivity when a delay is configured\r\n        if (this._timeout != null) {\r\n            clearTimeout(this._timeout);\r\n            this._timeout = null;\r\n        }\r\n        if (this.queue.length > 0 && typeof this.delay === \"number\") {\r\n            this._timeout = setTimeout(() => {\r\n                this.flush();\r\n            }, this.delay);\r\n        }\r\n    }\r\n    /**\r\n     * Flush all queued calls\r\n     */\r\n    flush() {\r\n        this._queue.splice(0).forEach((entry) => {\r\n            entry.fn.apply(entry.context || entry.fn, entry.args || []);\r\n        });\r\n    }\r\n}\n\n/* eslint-disable @typescript-eslint/member-ordering */\r\n/**\r\n * [[DataSet]] code that can be reused in [[DataView]] or other similar implementations of [[DataInterface]].\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataSetPart {\r\n    constructor() {\r\n        this._subscribers = {\r\n            \"*\": [],\r\n            add: [],\r\n            remove: [],\r\n            update: []\r\n        };\r\n        /**\r\n         * @deprecated Use on instead (PS: DataView.subscribe === DataView.on).\r\n         */\r\n        this.subscribe = DataSetPart.prototype.on;\r\n        /**\r\n         * @deprecated Use off instead (PS: DataView.unsubscribe === DataView.off).\r\n         */\r\n        this.unsubscribe = DataSetPart.prototype.off;\r\n    }\r\n    /**\r\n     * Trigger an event\r\n     *\r\n     * @param event - Event name.\r\n     * @param payload - Event payload.\r\n     * @param senderId - Id of the sender.\r\n     */\r\n    _trigger(event, payload, senderId) {\r\n        if (event === \"*\") {\r\n            throw new Error(\"Cannot trigger event *\");\r\n        }\r\n        [...this._subscribers[event], ...this._subscribers[\"*\"]].forEach((subscriber) => {\r\n            subscriber(event, payload, senderId != null ? senderId : null);\r\n        });\r\n    }\r\n    /**\r\n     * Subscribe to an event, add an event listener.\r\n     *\r\n     * @remarks Non-function callbacks are ignored.\r\n     *\r\n     * @param event - Event name.\r\n     * @param callback - Callback method.\r\n     */\r\n    on(event, callback) {\r\n        if (typeof callback === \"function\") {\r\n            this._subscribers[event].push(callback);\r\n        }\r\n        // @TODO: Maybe throw for invalid callbacks?\r\n    }\r\n    /**\r\n     * Unsubscribe from an event, remove an event listener.\r\n     *\r\n     * @remarks If the same callback was subscribed more than once **all** occurences will be removed.\r\n     *\r\n     * @param event - Event name.\r\n     * @param callback - Callback method.\r\n     */\r\n    off(event, callback) {\r\n        this._subscribers[event] = this._subscribers[event].filter((subscriber) => subscriber !== callback);\r\n    }\r\n}\n\n/**\r\n * Data stream\r\n *\r\n * @remarks\r\n * [[DataStream]] offers an always up to date stream of items from a [[DataSet]] or [[DataView]].\r\n * That means that the stream is evaluated at the time of iteration, conversion to another data type or when [[cache]] is called, not when the [[DataStream]] was created.\r\n * Multiple invocations of for example [[toItemArray]] may yield different results (if the data source like for example [[DataSet]] gets modified).\r\n *\r\n * @typeparam Item - The item type this stream is going to work with.\r\n */\r\nclass DataStream {\r\n    /**\r\n     * Create a new data stream.\r\n     *\r\n     * @param _pairs - The id, item pairs.\r\n     */\r\n    constructor(_pairs) {\r\n        this._pairs = _pairs;\r\n    }\r\n    /**\r\n     * Return an iterable of key, value pairs for every entry in the stream.\r\n     */\r\n    *[Symbol.iterator]() {\r\n        for (const [id, item] of this._pairs) {\r\n            yield [id, item];\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of key, value pairs for every entry in the stream.\r\n     */\r\n    *entries() {\r\n        for (const [id, item] of this._pairs) {\r\n            yield [id, item];\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of keys in the stream.\r\n     */\r\n    *keys() {\r\n        for (const [id] of this._pairs) {\r\n            yield id;\r\n        }\r\n    }\r\n    /**\r\n     * Return an iterable of values in the stream.\r\n     */\r\n    *values() {\r\n        for (const [, item] of this._pairs) {\r\n            yield item;\r\n        }\r\n    }\r\n    /**\r\n     * Return an array containing all the ids in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all ids from this stream.\r\n     */\r\n    toIdArray() {\r\n        return [...this._pairs].map((pair) => pair[0]);\r\n    }\r\n    /**\r\n     * Return an array containing all the items in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all items from this stream.\r\n     */\r\n    toItemArray() {\r\n        return [...this._pairs].map((pair) => pair[1]);\r\n    }\r\n    /**\r\n     * Return an array containing all the entries in this stream.\r\n     *\r\n     * @remarks\r\n     * The array may contain duplicities.\r\n     *\r\n     * @returns The array with all entries from this stream.\r\n     */\r\n    toEntryArray() {\r\n        return [...this._pairs];\r\n    }\r\n    /**\r\n     * Return an object map containing all the items in this stream accessible by ids.\r\n     *\r\n     * @remarks\r\n     * In case of duplicate ids (coerced to string so `7 == '7'`) the last encoutered appears in the returned object.\r\n     *\r\n     * @returns The object map of all id → item pairs from this stream.\r\n     */\r\n    toObjectMap() {\r\n        const map = Object.create(null);\r\n        for (const [id, item] of this._pairs) {\r\n            map[id] = item;\r\n        }\r\n        return map;\r\n    }\r\n    /**\r\n     * Return a map containing all the items in this stream accessible by ids.\r\n     *\r\n     * @returns The map of all id → item pairs from this stream.\r\n     */\r\n    toMap() {\r\n        return new Map(this._pairs);\r\n    }\r\n    /**\r\n     * Return a set containing all the (unique) ids in this stream.\r\n     *\r\n     * @returns The set of all ids from this stream.\r\n     */\r\n    toIdSet() {\r\n        return new Set(this.toIdArray());\r\n    }\r\n    /**\r\n     * Return a set containing all the (unique) items in this stream.\r\n     *\r\n     * @returns The set of all items from this stream.\r\n     */\r\n    toItemSet() {\r\n        return new Set(this.toItemArray());\r\n    }\r\n    /**\r\n     * Cache the items from this stream.\r\n     *\r\n     * @remarks\r\n     * This method allows for items to be fetched immediatelly and used (possibly multiple times) later.\r\n     * It can also be used to optimize performance as [[DataStream]] would otherwise reevaluate everything upon each iteration.\r\n     *\r\n     * ## Example\r\n     * ```javascript\r\n     * const ds = new DataSet([…])\r\n     *\r\n     * const cachedStream = ds.stream()\r\n     *   .filter(…)\r\n     *   .sort(…)\r\n     *   .map(…)\r\n     *   .cached(…) // Data are fetched, processed and cached here.\r\n     *\r\n     * ds.clear()\r\n     * chachedStream // Still has all the items.\r\n     * ```\r\n     *\r\n     * @returns A new [[DataStream]] with cached items (detached from the original [[DataSet]]).\r\n     */\r\n    cache() {\r\n        return new DataStream([...this._pairs]);\r\n    }\r\n    /**\r\n     * Get the distinct values of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @typeparam T - The type of the distinct value.\r\n     *\r\n     * @returns A set of all distinct properties.\r\n     */\r\n    distinct(callback) {\r\n        const set = new Set();\r\n        for (const [id, item] of this._pairs) {\r\n            set.add(callback(item, id));\r\n        }\r\n        return set;\r\n    }\r\n    /**\r\n     * Filter the items of the stream.\r\n     *\r\n     * @param callback - The function that decides whether an item will be included.\r\n     *\r\n     * @returns A new data stream with the filtered items.\r\n     */\r\n    filter(callback) {\r\n        const pairs = this._pairs;\r\n        return new DataStream({\r\n            *[Symbol.iterator]() {\r\n                for (const [id, item] of pairs) {\r\n                    if (callback(item, id)) {\r\n                        yield [id, item];\r\n                    }\r\n                }\r\n            }\r\n        });\r\n    }\r\n    /**\r\n     * Execute a callback for each item of the stream.\r\n     *\r\n     * @param callback - The function that will be invoked for each item.\r\n     */\r\n    forEach(callback) {\r\n        for (const [id, item] of this._pairs) {\r\n            callback(item, id);\r\n        }\r\n    }\r\n    /**\r\n     * Map the items into a different type.\r\n     *\r\n     * @param callback - The function that does the conversion.\r\n     *\r\n     * @typeparam Mapped - The type of the item after mapping.\r\n     *\r\n     * @returns A new data stream with the mapped items.\r\n     */\r\n    map(callback) {\r\n        const pairs = this._pairs;\r\n        return new DataStream({\r\n            *[Symbol.iterator]() {\r\n                for (const [id, item] of pairs) {\r\n                    yield [id, callback(item, id)];\r\n                }\r\n            }\r\n        });\r\n    }\r\n    /**\r\n     * Get the item with the maximum value of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @returns The item with the maximum if found otherwise null.\r\n     */\r\n    max(callback) {\r\n        const iter = this._pairs[Symbol.iterator]();\r\n        let curr = iter.next();\r\n        if (curr.done) {\r\n            return null;\r\n        }\r\n        let maxItem = curr.value[1];\r\n        let maxValue = callback(curr.value[1], curr.value[0]);\r\n        while (!(curr = iter.next()).done) {\r\n            const [id, item] = curr.value;\r\n            const value = callback(item, id);\r\n            if (value > maxValue) {\r\n                maxValue = value;\r\n                maxItem = item;\r\n            }\r\n        }\r\n        return maxItem;\r\n    }\r\n    /**\r\n     * Get the item with the minimum value of given property.\r\n     *\r\n     * @param callback - The function that picks and possibly converts the property.\r\n     *\r\n     * @returns The item with the minimum if found otherwise null.\r\n     */\r\n    min(callback) {\r\n        const iter = this._pairs[Symbol.iterator]();\r\n        let curr = iter.next();\r\n        if (curr.done) {\r\n            return null;\r\n        }\r\n        let minItem = curr.value[1];\r\n        let minValue = callback(curr.value[1], curr.value[0]);\r\n        while (!(curr = iter.next()).done) {\r\n            const [id, item] = curr.value;\r\n            const value = callback(item, id);\r\n            if (value < minValue) {\r\n                minValue = value;\r\n                minItem = item;\r\n            }\r\n        }\r\n        return minItem;\r\n    }\r\n    /**\r\n     * Reduce the items into a single value.\r\n     *\r\n     * @param callback - The function that does the reduction.\r\n     * @param accumulator - The initial value of the accumulator.\r\n     *\r\n     * @typeparam T - The type of the accumulated value.\r\n     *\r\n     * @returns The reduced value.\r\n     */\r\n    reduce(callback, accumulator) {\r\n        for (const [id, item] of this._pairs) {\r\n            accumulator = callback(accumulator, item, id);\r\n        }\r\n        return accumulator;\r\n    }\r\n    /**\r\n     * Sort the items.\r\n     *\r\n     * @param callback - Item comparator.\r\n     *\r\n     * @returns A new stream with sorted items.\r\n     */\r\n    sort(callback) {\r\n        return new DataStream({\r\n            [Symbol.iterator]: () => [...this._pairs]\r\n                .sort(([idA, itemA], [idB, itemB]) => callback(itemA, itemB, idA, idB))[Symbol.iterator]()\r\n        });\r\n    }\r\n}\n\n/* eslint @typescript-eslint/member-ordering: [\"error\", { \"classes\": [\"field\", \"constructor\", \"method\"] }] */\r\nconst warnTypeCorectionDeprecation = () => {\r\n    console.warn(\"Type coercion has been deprecated. \" +\r\n        \"Please, use data pipes instead. \" +\r\n        \"See https://visjs.github.io/vis-data/data/datapipe.html#TypeCoercion for more details with working migration example.\");\r\n};\r\n/**\r\n * # DataSet\r\n *\r\n * Vis.js comes with a flexible DataSet, which can be used to hold and manipulate unstructured data and listen for changes in the data. The DataSet is key/value based. Data items can be added, updated and removed from the DataSet, and one can subscribe to changes in the DataSet. The data in the DataSet can be filtered and ordered, and fields (like dates) can be converted to a specific type. Data can be normalized when appending it to the DataSet as well.\r\n *\r\n * ## Example\r\n *\r\n * The following example shows how to use a DataSet.\r\n *\r\n * ```javascript\r\n * // create a DataSet\r\n * var options = {};\r\n * var data = new vis.DataSet(options);\r\n *\r\n * // add items\r\n * // note that the data items can contain different properties and data formats\r\n * data.add([\r\n *   {id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true},\r\n *   {id: 2, text: 'item 2', date: '2013-06-23', group: 2},\r\n *   {id: 3, text: 'item 3', date: '2013-06-25', group: 2},\r\n *   {id: 4, text: 'item 4'}\r\n * ]);\r\n *\r\n * // subscribe to any change in the DataSet\r\n * data.on('*', function (event, properties, senderId) {\r\n *   console.log('event', event, properties);\r\n * });\r\n *\r\n * // update an existing item\r\n * data.update({id: 2, group: 1});\r\n *\r\n * // remove an item\r\n * data.remove(4);\r\n *\r\n * // get all ids\r\n * var ids = data.getIds();\r\n * console.log('ids', ids);\r\n *\r\n * // get a specific item\r\n * var item1 = data.get(1);\r\n * console.log('item1', item1);\r\n *\r\n * // retrieve a filtered subset of the data\r\n * var items = data.get({\r\n *   filter: function (item) {\r\n *     return item.group == 1;\r\n *   }\r\n * });\r\n * console.log('filtered items', items);\r\n * ```\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataSet extends DataSetPart {\r\n    /**\r\n     * Construct a new DataSet.\r\n     *\r\n     * @param data - Initial data or options.\r\n     * @param options - Options (type error if data is also options).\r\n     */\r\n    constructor(data, options) {\r\n        super();\r\n        // correctly read optional arguments\r\n        if (data && !Array.isArray(data)) {\r\n            options = data;\r\n            data = [];\r\n        }\r\n        this._options = options || {};\r\n        this._data = new Map(); // map with data indexed by id\r\n        this.length = 0; // number of items in the DataSet\r\n        this._idProp = this._options.fieldId || \"id\"; // name of the field containing id\r\n        this._type = {}; // internal field types (NOTE: this can differ from this._options.type)\r\n        // all variants of a Date are internally stored as Date, so we can convert\r\n        // from everything to everything (also from ISODate to Number for example)\r\n        if (this._options.type) {\r\n            warnTypeCorectionDeprecation();\r\n            const fields = Object.keys(this._options.type);\r\n            for (let i = 0, len = fields.length; i < len; i++) {\r\n                const field = fields[i];\r\n                const value = this._options.type[field];\r\n                if (value == \"Date\" || value == \"ISODate\" || value == \"ASPDate\") {\r\n                    this._type[field] = \"Date\";\r\n                }\r\n                else {\r\n                    this._type[field] = value;\r\n                }\r\n            }\r\n        }\r\n        // add initial data when provided\r\n        if (data && data.length) {\r\n            this.add(data);\r\n        }\r\n        this.setOptions(options);\r\n    }\r\n    /**\r\n     * Set new options.\r\n     *\r\n     * @param options - The new options.\r\n     */\r\n    setOptions(options) {\r\n        if (options && options.queue !== undefined) {\r\n            if (options.queue === false) {\r\n                // delete queue if loaded\r\n                if (this._queue) {\r\n                    this._queue.destroy();\r\n                    delete this._queue;\r\n                }\r\n            }\r\n            else {\r\n                // create queue and update its options\r\n                if (!this._queue) {\r\n                    this._queue = Queue.extend(this, {\r\n                        replace: [\"add\", \"update\", \"remove\"]\r\n                    });\r\n                }\r\n                if (options.queue && typeof options.queue === \"object\") {\r\n                    this._queue.setOptions(options.queue);\r\n                }\r\n            }\r\n        }\r\n    }\r\n    /**\r\n     * Add a data item or an array with items.\r\n     *\r\n     * After the items are added to the DataSet, the DataSet will trigger an event `add`. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet()\r\n     *\r\n     * // add items\r\n     * const ids = data.add([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { text: 'item without an id' }\r\n     * ])\r\n     *\r\n     * console.log(ids) // [1, 2, '<UUIDv4>']\r\n     * ```\r\n     *\r\n     * @param data - Items to be added (ids will be generated if missing).\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns addedIds - Array with the ids (generated if not present) of the added items.\r\n     *\r\n     * @throws When an item with the same id as any of the added items already exists.\r\n     */\r\n    add(data, senderId) {\r\n        const addedIds = [];\r\n        let id;\r\n        if (Array.isArray(data)) {\r\n            // Array\r\n            const idsToAdd = data.map(d => d[this._idProp]);\r\n            if (idsToAdd.some(id => this._data.has(id))) {\r\n                throw new Error(\"A duplicate id was found in the parameter array.\");\r\n            }\r\n            for (let i = 0, len = data.length; i < len; i++) {\r\n                id = this._addItem(data[i]);\r\n                addedIds.push(id);\r\n            }\r\n        }\r\n        else if (data && typeof data === \"object\") {\r\n            // Single item\r\n            id = this._addItem(data);\r\n            addedIds.push(id);\r\n        }\r\n        else {\r\n            throw new Error(\"Unknown dataType\");\r\n        }\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        return addedIds;\r\n    }\r\n    /**\r\n     * Update existing items. When an item does not exist, it will be created.\r\n     *\r\n     * @remarks\r\n     * The provided properties will be merged in the existing item. When an item does not exist, it will be created.\r\n     *\r\n     * After the items are updated, the DataSet will trigger an event `add` for the added items, and an event `update`. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' }\r\n     * ])\r\n     *\r\n     * // update items\r\n     * const ids = data.update([\r\n     *   { id: 2, text: 'item 2 (updated)' },\r\n     *   { id: 4, text: 'item 4 (new)' }\r\n     * ])\r\n     *\r\n     * console.log(ids) // [2, 4]\r\n     * ```\r\n     *\r\n     * ## Warning for TypeScript users\r\n     * This method may introduce partial items into the data set. Use add or updateOnly instead for better type safety.\r\n     *\r\n     * @param data - Items to be updated (if the id is already present) or added (if the id is missing).\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns updatedIds - The ids of the added (these may be newly generated if there was no id in the item from the data) or updated items.\r\n     *\r\n     * @throws When the supplied data is neither an item nor an array of items.\r\n     */\r\n    update(data, senderId) {\r\n        const addedIds = [];\r\n        const updatedIds = [];\r\n        const oldData = [];\r\n        const updatedData = [];\r\n        const idProp = this._idProp;\r\n        const addOrUpdate = (item) => {\r\n            const origId = item[idProp];\r\n            if (origId != null && this._data.has(origId)) {\r\n                const fullItem = item; // it has an id, therefore it is a fullitem\r\n                const oldItem = Object.assign({}, this._data.get(origId));\r\n                // update item\r\n                const id = this._updateItem(fullItem);\r\n                updatedIds.push(id);\r\n                updatedData.push(fullItem);\r\n                oldData.push(oldItem);\r\n            }\r\n            else {\r\n                // add new item\r\n                const id = this._addItem(item);\r\n                addedIds.push(id);\r\n            }\r\n        };\r\n        if (Array.isArray(data)) {\r\n            // Array\r\n            for (let i = 0, len = data.length; i < len; i++) {\r\n                if (data[i] && typeof data[i] === \"object\") {\r\n                    addOrUpdate(data[i]);\r\n                }\r\n                else {\r\n                    console.warn(\"Ignoring input item, which is not an object at index \" + i);\r\n                }\r\n            }\r\n        }\r\n        else if (data && typeof data === \"object\") {\r\n            // Single item\r\n            addOrUpdate(data);\r\n        }\r\n        else {\r\n            throw new Error(\"Unknown dataType\");\r\n        }\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        if (updatedIds.length) {\r\n            const props = { items: updatedIds, oldData: oldData, data: updatedData };\r\n            // TODO: remove deprecated property 'data' some day\r\n            //Object.defineProperty(props, 'data', {\r\n            //  'get': (function() {\r\n            //    console.warn('Property data is deprecated. Use DataSet.get(ids) to retrieve the new data, use the oldData property on this object to get the old data');\r\n            //    return updatedData;\r\n            //  }).bind(this)\r\n            //});\r\n            this._trigger(\"update\", props, senderId);\r\n        }\r\n        return addedIds.concat(updatedIds);\r\n    }\r\n    /**\r\n     * Update existing items. When an item does not exist, an error will be thrown.\r\n     *\r\n     * @remarks\r\n     * The provided properties will be deeply merged into the existing item.\r\n     * When an item does not exist (id not present in the data set or absent), an error will be thrown and nothing will be changed.\r\n     *\r\n     * After the items are updated, the DataSet will trigger an event `update`.\r\n     * When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     *\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' },\r\n     * ])\r\n     *\r\n     * // update items\r\n     * const ids = data.update([\r\n     *   { id: 2, text: 'item 2 (updated)' }, // works\r\n     *   // { id: 4, text: 'item 4 (new)' }, // would throw\r\n     *   // { text: 'item 4 (new)' }, // would also throw\r\n     * ])\r\n     *\r\n     * console.log(ids) // [2]\r\n     * ```\r\n     *\r\n     * @param data - Updates (the id and optionally other props) to the items in this data set.\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns updatedIds - The ids of the updated items.\r\n     *\r\n     * @throws When the supplied data is neither an item nor an array of items, when the ids are missing.\r\n     */\r\n    updateOnly(data, senderId) {\r\n        if (!Array.isArray(data)) {\r\n            data = [data];\r\n        }\r\n        const updateEventData = data\r\n            .map((update) => {\r\n            const oldData = this._data.get(update[this._idProp]);\r\n            if (oldData == null) {\r\n                throw new Error(\"Updating non-existent items is not allowed.\");\r\n            }\r\n            return { oldData, update };\r\n        })\r\n            .map(({ oldData, update }) => {\r\n            const id = oldData[this._idProp];\r\n            const updatedData = deepExtend(deepExtend({}, oldData), update);\r\n            this._data.set(id, updatedData);\r\n            return {\r\n                id,\r\n                oldData: oldData,\r\n                updatedData\r\n            };\r\n        });\r\n        if (updateEventData.length) {\r\n            const props = {\r\n                items: updateEventData.map((value) => value.id),\r\n                oldData: updateEventData.map((value) => value.oldData),\r\n                data: updateEventData.map((value) => value.updatedData)\r\n            };\r\n            // TODO: remove deprecated property 'data' some day\r\n            //Object.defineProperty(props, 'data', {\r\n            //  'get': (function() {\r\n            //    console.warn('Property data is deprecated. Use DataSet.get(ids) to retrieve the new data, use the oldData property on this object to get the old data');\r\n            //    return updatedData;\r\n            //  }).bind(this)\r\n            //});\r\n            this._trigger(\"update\", props, senderId);\r\n            return props.items;\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    get(first, second) {\r\n        // @TODO: Woudn't it be better to split this into multiple methods?\r\n        // parse the arguments\r\n        let id = undefined;\r\n        let ids = undefined;\r\n        let options = undefined;\r\n        if (isId(first)) {\r\n            // get(id [, options])\r\n            id = first;\r\n            options = second;\r\n        }\r\n        else if (Array.isArray(first)) {\r\n            // get(ids [, options])\r\n            ids = first;\r\n            options = second;\r\n        }\r\n        else {\r\n            // get([, options])\r\n            options = first;\r\n        }\r\n        // determine the return type\r\n        const returnType = options && options.returnType === \"Object\" ? \"Object\" : \"Array\";\r\n        // @TODO: WTF is this? Or am I missing something?\r\n        // var returnType\r\n        // if (options && options.returnType) {\r\n        //   var allowedValues = ['Array', 'Object']\r\n        //   returnType =\r\n        //     allowedValues.indexOf(options.returnType) == -1\r\n        //       ? 'Array'\r\n        //       : options.returnType\r\n        // } else {\r\n        //   returnType = 'Array'\r\n        // }\r\n        // build options\r\n        const type = (options && options.type) || this._options.type;\r\n        const filter = options && options.filter;\r\n        const items = [];\r\n        let item = null;\r\n        let itemIds = null;\r\n        let itemId = null;\r\n        // convert items\r\n        if (id != null) {\r\n            // return a single item\r\n            item = this._getItem(id, type);\r\n            if (item && filter && !filter(item)) {\r\n                item = null;\r\n            }\r\n        }\r\n        else if (ids != null) {\r\n            // return a subset of items\r\n            for (let i = 0, len = ids.length; i < len; i++) {\r\n                item = this._getItem(ids[i], type);\r\n                if (item != null && (!filter || filter(item))) {\r\n                    items.push(item);\r\n                }\r\n            }\r\n        }\r\n        else {\r\n            // return all items\r\n            itemIds = [...this._data.keys()];\r\n            for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                itemId = itemIds[i];\r\n                item = this._getItem(itemId, type);\r\n                if (item != null && (!filter || filter(item))) {\r\n                    items.push(item);\r\n                }\r\n            }\r\n        }\r\n        // order the results\r\n        if (options && options.order && id == undefined) {\r\n            this._sort(items, options.order);\r\n        }\r\n        // filter fields of the items\r\n        if (options && options.fields) {\r\n            const fields = options.fields;\r\n            if (id != undefined && item != null) {\r\n                item = this._filterFields(item, fields);\r\n            }\r\n            else {\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    items[i] = this._filterFields(items[i], fields);\r\n                }\r\n            }\r\n        }\r\n        // return the results\r\n        if (returnType == \"Object\") {\r\n            const result = {};\r\n            for (let i = 0, len = items.length; i < len; i++) {\r\n                const resultant = items[i];\r\n                // @TODO: Shoudn't this be this._fieldId?\r\n                // result[resultant.id] = resultant\r\n                const id = resultant[this._idProp];\r\n                result[id] = resultant;\r\n            }\r\n            return result;\r\n        }\r\n        else {\r\n            if (id != null) {\r\n                // a single item\r\n                return item;\r\n            }\r\n            else {\r\n                // just return our array\r\n                return items;\r\n            }\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getIds(options) {\r\n        const data = this._data;\r\n        const filter = options && options.filter;\r\n        const order = options && options.order;\r\n        const type = (options && options.type) || this._options.type;\r\n        const itemIds = [...data.keys()];\r\n        const ids = [];\r\n        let item;\r\n        let items;\r\n        if (filter) {\r\n            // get filtered items\r\n            if (order) {\r\n                // create ordered list\r\n                items = [];\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = this._getItem(id, type);\r\n                    if (filter(item)) {\r\n                        items.push(item);\r\n                    }\r\n                }\r\n                this._sort(items, order);\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    ids.push(items[i][this._idProp]);\r\n                }\r\n            }\r\n            else {\r\n                // create unordered list\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = this._getItem(id, type);\r\n                    if (filter(item)) {\r\n                        ids.push(item[this._idProp]);\r\n                    }\r\n                }\r\n            }\r\n        }\r\n        else {\r\n            // get all items\r\n            if (order) {\r\n                // create an ordered list\r\n                items = [];\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    items.push(data.get(id));\r\n                }\r\n                this._sort(items, order);\r\n                for (let i = 0, len = items.length; i < len; i++) {\r\n                    ids.push(items[i][this._idProp]);\r\n                }\r\n            }\r\n            else {\r\n                // create unordered list\r\n                for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                    const id = itemIds[i];\r\n                    item = data.get(id);\r\n                    ids.push(item[this._idProp]);\r\n                }\r\n            }\r\n        }\r\n        return ids;\r\n    }\r\n    /** @inheritdoc */\r\n    getDataSet() {\r\n        return this;\r\n    }\r\n    /** @inheritdoc */\r\n    forEach(callback, options) {\r\n        const filter = options && options.filter;\r\n        const type = (options && options.type) || this._options.type;\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        if (options && options.order) {\r\n            // execute forEach on ordered list\r\n            const items = this.get(options);\r\n            for (let i = 0, len = items.length; i < len; i++) {\r\n                const item = items[i];\r\n                const id = item[this._idProp];\r\n                callback(item, id);\r\n            }\r\n        }\r\n        else {\r\n            // unordered\r\n            for (let i = 0, len = itemIds.length; i < len; i++) {\r\n                const id = itemIds[i];\r\n                const item = this._getItem(id, type);\r\n                if (!filter || filter(item)) {\r\n                    callback(item, id);\r\n                }\r\n            }\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    map(callback, options) {\r\n        const filter = options && options.filter;\r\n        const type = (options && options.type) || this._options.type;\r\n        const mappedItems = [];\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        // convert and filter items\r\n        for (let i = 0, len = itemIds.length; i < len; i++) {\r\n            const id = itemIds[i];\r\n            const item = this._getItem(id, type);\r\n            if (!filter || filter(item)) {\r\n                mappedItems.push(callback(item, id));\r\n            }\r\n        }\r\n        // order items\r\n        if (options && options.order) {\r\n            this._sort(mappedItems, options.order);\r\n        }\r\n        return mappedItems;\r\n    }\r\n    /**\r\n     * Filter the fields of an item.\r\n     *\r\n     * @param item - The item whose fields should be filtered.\r\n     * @param fields - The names of the fields that will be kept.\r\n     *\r\n     * @typeParam K - Field name type.\r\n     *\r\n     * @returns The item without any additional fields.\r\n     */\r\n    _filterFields(item, fields) {\r\n        if (!item) {\r\n            // item is null\r\n            return item;\r\n        }\r\n        return (Array.isArray(fields)\r\n            ? // Use the supplied array\r\n                fields\r\n            : // Use the keys of the supplied object\r\n                Object.keys(fields)).reduce((filteredItem, field) => {\r\n            filteredItem[field] = item[field];\r\n            return filteredItem;\r\n        }, {});\r\n    }\r\n    /**\r\n     * Sort the provided array with items.\r\n     *\r\n     * @param items - Items to be sorted in place.\r\n     * @param order - A field name or custom sort function.\r\n     *\r\n     * @typeParam T - The type of the items in the items array.\r\n     */\r\n    _sort(items, order) {\r\n        if (typeof order === \"string\") {\r\n            // order by provided field name\r\n            const name = order; // field name\r\n            items.sort((a, b) => {\r\n                // @TODO: How to treat missing properties?\r\n                const av = a[name];\r\n                const bv = b[name];\r\n                return av > bv ? 1 : av < bv ? -1 : 0;\r\n            });\r\n        }\r\n        else if (typeof order === \"function\") {\r\n            // order by sort function\r\n            items.sort(order);\r\n        }\r\n        else {\r\n            // TODO: extend order by an Object {field:string, direction:string}\r\n            //       where direction can be 'asc' or 'desc'\r\n            throw new TypeError(\"Order must be a function or a string\");\r\n        }\r\n    }\r\n    /**\r\n     * Remove an item or multiple items by “reference” (only the id is used) or by id.\r\n     *\r\n     * The method ignores removal of non-existing items, and returns an array containing the ids of the items which are actually removed from the DataSet.\r\n     *\r\n     * After the items are removed, the DataSet will trigger an event `remove` for the removed items. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * ## Example\r\n     * ```javascript\r\n     * // create a DataSet\r\n     * const data = new vis.DataSet([\r\n     *   { id: 1, text: 'item 1' },\r\n     *   { id: 2, text: 'item 2' },\r\n     *   { id: 3, text: 'item 3' }\r\n     * ])\r\n     *\r\n     * // remove items\r\n     * const ids = data.remove([2, { id: 3 }, 4])\r\n     *\r\n     * console.log(ids) // [2, 3]\r\n     * ```\r\n     *\r\n     * @param id - One or more items or ids of items to be removed.\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns The ids of the removed items.\r\n     */\r\n    remove(id, senderId) {\r\n        const removedIds = [];\r\n        const removedItems = [];\r\n        // force everything to be an array for simplicity\r\n        const ids = Array.isArray(id) ? id : [id];\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            const item = this._remove(ids[i]);\r\n            if (item) {\r\n                const itemId = item[this._idProp];\r\n                if (itemId != null) {\r\n                    removedIds.push(itemId);\r\n                    removedItems.push(item);\r\n                }\r\n            }\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems }, senderId);\r\n        }\r\n        return removedIds;\r\n    }\r\n    /**\r\n     * Remove an item by its id or reference.\r\n     *\r\n     * @param id - Id of an item or the item itself.\r\n     *\r\n     * @returns The removed item if removed, null otherwise.\r\n     */\r\n    _remove(id) {\r\n        // @TODO: It origianlly returned the item although the docs say id.\r\n        // The code expects the item, so probably an error in the docs.\r\n        let ident;\r\n        // confirm the id to use based on the args type\r\n        if (isId(id)) {\r\n            ident = id;\r\n        }\r\n        else if (id && typeof id === \"object\") {\r\n            ident = id[this._idProp]; // look for the identifier field using ._idProp\r\n        }\r\n        // do the removing if the item is found\r\n        if (ident != null && this._data.has(ident)) {\r\n            const item = this._data.get(ident) || null;\r\n            this._data.delete(ident);\r\n            --this.length;\r\n            return item;\r\n        }\r\n        return null;\r\n    }\r\n    /**\r\n     * Clear the entire data set.\r\n     *\r\n     * After the items are removed, the [[DataSet]] will trigger an event `remove` for all removed items. When a `senderId` is provided, this id will be passed with the triggered event to all subscribers.\r\n     *\r\n     * @param senderId - Sender id.\r\n     *\r\n     * @returns removedIds - The ids of all removed items.\r\n     */\r\n    clear(senderId) {\r\n        const ids = [...this._data.keys()];\r\n        const items = [];\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            items.push(this._data.get(ids[i]));\r\n        }\r\n        this._data.clear();\r\n        this.length = 0;\r\n        this._trigger(\"remove\", { items: ids, oldData: items }, senderId);\r\n        return ids;\r\n    }\r\n    /**\r\n     * Find the item with maximum value of a specified field.\r\n     *\r\n     * @param field - Name of the property that should be searched for max value.\r\n     *\r\n     * @returns Item containing max value, or null if no items.\r\n     */\r\n    max(field) {\r\n        let max = null;\r\n        let maxField = null;\r\n        for (const item of this._data.values()) {\r\n            const itemField = item[field];\r\n            if (typeof itemField === \"number\" &&\r\n                (maxField == null || itemField > maxField)) {\r\n                max = item;\r\n                maxField = itemField;\r\n            }\r\n        }\r\n        return max || null;\r\n    }\r\n    /**\r\n     * Find the item with minimum value of a specified field.\r\n     *\r\n     * @param field - Name of the property that should be searched for min value.\r\n     *\r\n     * @returns Item containing min value, or null if no items.\r\n     */\r\n    min(field) {\r\n        let min = null;\r\n        let minField = null;\r\n        for (const item of this._data.values()) {\r\n            const itemField = item[field];\r\n            if (typeof itemField === \"number\" &&\r\n                (minField == null || itemField < minField)) {\r\n                min = item;\r\n                minField = itemField;\r\n            }\r\n        }\r\n        return min || null;\r\n    }\r\n    /**\r\n     * Find all distinct values of a specified field\r\n     *\r\n     * @param prop - The property name whose distinct values should be returned.\r\n     *\r\n     * @returns Unordered array containing all distinct values. Items without specified property are ignored.\r\n     */\r\n    distinct(prop) {\r\n        const data = this._data;\r\n        const itemIds = [...data.keys()];\r\n        const values = [];\r\n        const fieldType = (this._options.type && this._options.type[prop]) || null;\r\n        let count = 0;\r\n        for (let i = 0, len = itemIds.length; i < len; i++) {\r\n            const id = itemIds[i];\r\n            const item = data.get(id);\r\n            const value = item[prop];\r\n            let exists = false;\r\n            for (let j = 0; j < count; j++) {\r\n                if (values[j] == value) {\r\n                    exists = true;\r\n                    break;\r\n                }\r\n            }\r\n            if (!exists && value !== undefined) {\r\n                values[count] = value;\r\n                count++;\r\n            }\r\n        }\r\n        if (fieldType) {\r\n            for (let i = 0, len = values.length; i < len; i++) {\r\n                values[i] = convert(values[i], fieldType);\r\n            }\r\n        }\r\n        return values;\r\n    }\r\n    /**\r\n     * Add a single item. Will fail when an item with the same id already exists.\r\n     *\r\n     * @param item - A new item to be added.\r\n     *\r\n     * @returns Added item's id. An id is generated when it is not present in the item.\r\n     */\r\n    _addItem(item) {\r\n        let id = item[this._idProp];\r\n        if (id != null) {\r\n            // check whether this id is already taken\r\n            if (this._data.has(id)) {\r\n                // item already exists\r\n                throw new Error(\"Cannot add item: item with id \" + id + \" already exists\");\r\n            }\r\n        }\r\n        else {\r\n            // generate an id\r\n            id = v4_1();\r\n            item[this._idProp] = id;\r\n        }\r\n        const d = {};\r\n        const fields = Object.keys(item);\r\n        for (let i = 0, len = fields.length; i < len; i++) {\r\n            const field = fields[i];\r\n            const fieldType = this._type[field]; // type may be undefined\r\n            d[field] = convert(item[field], fieldType);\r\n        }\r\n        this._data.set(id, d);\r\n        ++this.length;\r\n        return id;\r\n    }\r\n    /**\r\n     * Get an item. Fields can be converted to a specific type\r\n     *\r\n     * @param id - Id of the requested item.\r\n     * @param types - Property name to type name object map of type converstions.\r\n     *\r\n     * @returns The item, optionally after type conversion.\r\n     */\r\n    _getItem(id, types) {\r\n        // @TODO: I have no idea how to type this.\r\n        // get the item from the dataset\r\n        const raw = this._data.get(id);\r\n        if (!raw) {\r\n            return null;\r\n        }\r\n        // convert the items field types\r\n        let converted;\r\n        const fields = Object.keys(raw);\r\n        if (types) {\r\n            warnTypeCorectionDeprecation();\r\n            converted = {};\r\n            for (let i = 0, len = fields.length; i < len; i++) {\r\n                const field = fields[i];\r\n                const value = raw[field];\r\n                converted[field] = convert(value, types[field]);\r\n            }\r\n        }\r\n        else {\r\n            // no field types specified, no converting needed\r\n            converted = { ...raw };\r\n        }\r\n        if (converted[this._idProp] == null) {\r\n            converted[this._idProp] = raw.id;\r\n        }\r\n        return converted;\r\n    }\r\n    /**\r\n     * Update a single item: merge with existing item.\r\n     * Will fail when the item has no id, or when there does not exist an item with the same id.\r\n     *\r\n     * @param item - The new item\r\n     *\r\n     * @returns The id of the updated item.\r\n     */\r\n    _updateItem(item) {\r\n        const id = item[this._idProp];\r\n        if (id == null) {\r\n            throw new Error(\"Cannot update item: item has no id (item: \" +\r\n                JSON.stringify(item) +\r\n                \")\");\r\n        }\r\n        const d = this._data.get(id);\r\n        if (!d) {\r\n            // item doesn't exist\r\n            throw new Error(\"Cannot update item: no item with id \" + id + \" found\");\r\n        }\r\n        // merge with current item\r\n        const fields = Object.keys(item);\r\n        for (let i = 0, len = fields.length; i < len; i++) {\r\n            const field = fields[i];\r\n            const fieldType = this._type[field]; // type may be undefined\r\n            d[field] = convert(item[field], fieldType);\r\n        }\r\n        return id;\r\n    }\r\n    /** @inheritdoc */\r\n    stream(ids) {\r\n        if (ids) {\r\n            const data = this._data;\r\n            return new DataStream({\r\n                *[Symbol.iterator]() {\r\n                    for (const id of ids) {\r\n                        const item = data.get(id);\r\n                        if (item != null) {\r\n                            yield [id, item];\r\n                        }\r\n                    }\r\n                }\r\n            });\r\n        }\r\n        else {\r\n            return new DataStream({\r\n                [Symbol.iterator]: this._data.entries.bind(this._data)\r\n            });\r\n        }\r\n    }\r\n}\n\n/**\r\n * DataView\r\n *\r\n * A DataView offers a filtered and/or formatted view on a DataSet. One can subscribe to changes in a DataView, and easily get filtered or formatted data without having to specify filters and field types all the time.\r\n *\r\n * ## Example\r\n * ```javascript\r\n * // create a DataSet\r\n * var data = new vis.DataSet();\r\n * data.add([\r\n *   {id: 1, text: 'item 1', date: new Date(2013, 6, 20), group: 1, first: true},\r\n *   {id: 2, text: 'item 2', date: '2013-06-23', group: 2},\r\n *   {id: 3, text: 'item 3', date: '2013-06-25', group: 2},\r\n *   {id: 4, text: 'item 4'}\r\n * ]);\r\n *\r\n * // create a DataView\r\n * // the view will only contain items having a property group with value 1,\r\n * // and will only output fields id, text, and date.\r\n * var view = new vis.DataView(data, {\r\n *   filter: function (item) {\r\n *     return (item.group == 1);\r\n *   },\r\n *   fields: ['id', 'text', 'date']\r\n * });\r\n *\r\n * // subscribe to any change in the DataView\r\n * view.on('*', function (event, properties, senderId) {\r\n *   console.log('event', event, properties);\r\n * });\r\n *\r\n * // update an item in the data set\r\n * data.update({id: 2, group: 1});\r\n *\r\n * // get all ids in the view\r\n * var ids = view.getIds();\r\n * console.log('ids', ids); // will output [1, 2]\r\n *\r\n * // get all items in the view\r\n * var items = view.get();\r\n * ```\r\n *\r\n * @typeParam Item - Item type that may or may not have an id.\r\n * @typeParam IdProp - Name of the property that contains the id.\r\n */\r\nclass DataView extends DataSetPart {\r\n    /**\r\n     * Create a DataView.\r\n     *\r\n     * @param data - The instance containing data (directly or indirectly).\r\n     * @param options - Options to configure this data view.\r\n     */\r\n    constructor(data, options) {\r\n        super();\r\n        /** @inheritdoc */\r\n        this.length = 0;\r\n        this._ids = new Set(); // ids of the items currently in memory (just contains a boolean true)\r\n        this._options = options || {};\r\n        this._listener = this._onEvent.bind(this);\r\n        this.setData(data);\r\n    }\r\n    // TODO: implement a function .config() to dynamically update things like configured filter\r\n    // and trigger changes accordingly\r\n    /**\r\n     * Set a data source for the view.\r\n     *\r\n     * @param data - The instance containing data (directly or indirectly).\r\n     *\r\n     * @remarks\r\n     * Note that when the data view is bound to a data set it won't be garbage\r\n     * collected unless the data set is too. Use `dataView.setData(null)` or\r\n     * `dataView.dispose()` to enable garbage collection before you lose the last\r\n     * reference.\r\n     */\r\n    setData(data) {\r\n        if (this._data) {\r\n            // unsubscribe from current dataset\r\n            if (this._data.off) {\r\n                this._data.off(\"*\", this._listener);\r\n            }\r\n            // trigger a remove of all items in memory\r\n            const ids = this._data.getIds({ filter: this._options.filter });\r\n            const items = this._data.get(ids);\r\n            this._ids.clear();\r\n            this.length = 0;\r\n            this._trigger(\"remove\", { items: ids, oldData: items });\r\n        }\r\n        if (data != null) {\r\n            this._data = data;\r\n            // trigger an add of all added items\r\n            const ids = this._data.getIds({ filter: this._options.filter });\r\n            for (let i = 0, len = ids.length; i < len; i++) {\r\n                const id = ids[i];\r\n                this._ids.add(id);\r\n            }\r\n            this.length = ids.length;\r\n            this._trigger(\"add\", { items: ids });\r\n        }\r\n        else {\r\n            this._data = new DataSet();\r\n        }\r\n        // subscribe to new dataset\r\n        if (this._data.on) {\r\n            this._data.on(\"*\", this._listener);\r\n        }\r\n    }\r\n    /**\r\n     * Refresh the DataView.\r\n     * Useful when the DataView has a filter function containing a variable parameter.\r\n     */\r\n    refresh() {\r\n        const ids = this._data.getIds({\r\n            filter: this._options.filter\r\n        });\r\n        const oldIds = [...this._ids];\r\n        const newIds = {};\r\n        const addedIds = [];\r\n        const removedIds = [];\r\n        const removedItems = [];\r\n        // check for additions\r\n        for (let i = 0, len = ids.length; i < len; i++) {\r\n            const id = ids[i];\r\n            newIds[id] = true;\r\n            if (!this._ids.has(id)) {\r\n                addedIds.push(id);\r\n                this._ids.add(id);\r\n            }\r\n        }\r\n        // check for removals\r\n        for (let i = 0, len = oldIds.length; i < len; i++) {\r\n            const id = oldIds[i];\r\n            const item = this._data.get(id);\r\n            if (item == null) {\r\n                // @TODO: Investigate.\r\n                // Doesn't happen during tests or examples.\r\n                // Is it really impossible or could it eventually happen?\r\n                // How to handle it if it does? The types guarantee non-nullable items.\r\n                console.error(\"If you see this, report it please.\");\r\n            }\r\n            else if (!newIds[id]) {\r\n                removedIds.push(id);\r\n                removedItems.push(item);\r\n                this._ids.delete(id);\r\n            }\r\n        }\r\n        this.length += addedIds.length - removedIds.length;\r\n        // trigger events\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds });\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems });\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    get(first, second) {\r\n        if (this._data == null) {\r\n            return null;\r\n        }\r\n        // parse the arguments\r\n        let ids = null;\r\n        let options;\r\n        if (isId(first) || Array.isArray(first)) {\r\n            ids = first;\r\n            options = second;\r\n        }\r\n        else {\r\n            options = first;\r\n        }\r\n        // extend the options with the default options and provided options\r\n        const viewOptions = Object.assign({}, this._options, options);\r\n        // create a combined filter method when needed\r\n        const thisFilter = this._options.filter;\r\n        const optionsFilter = options && options.filter;\r\n        if (thisFilter && optionsFilter) {\r\n            viewOptions.filter = (item) => {\r\n                return thisFilter(item) && optionsFilter(item);\r\n            };\r\n        }\r\n        if (ids == null) {\r\n            return this._data.get(viewOptions);\r\n        }\r\n        else {\r\n            return this._data.get(ids, viewOptions);\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getIds(options) {\r\n        if (this._data.length) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options != null ? options.filter : null;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = (item) => {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            return this._data.getIds({\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    forEach(callback, options) {\r\n        if (this._data) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options && options.filter;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = function (item) {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            this._data.forEach(callback, {\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    map(callback, options) {\r\n        if (this._data) {\r\n            const defaultFilter = this._options.filter;\r\n            const optionsFilter = options && options.filter;\r\n            let filter;\r\n            if (optionsFilter) {\r\n                if (defaultFilter) {\r\n                    filter = (item) => {\r\n                        return defaultFilter(item) && optionsFilter(item);\r\n                    };\r\n                }\r\n                else {\r\n                    filter = optionsFilter;\r\n                }\r\n            }\r\n            else {\r\n                filter = defaultFilter;\r\n            }\r\n            return this._data.map(callback, {\r\n                filter: filter,\r\n                order: options && options.order\r\n            });\r\n        }\r\n        else {\r\n            return [];\r\n        }\r\n    }\r\n    /** @inheritdoc */\r\n    getDataSet() {\r\n        return this._data.getDataSet();\r\n    }\r\n    /** @inheritdoc */\r\n    stream(ids) {\r\n        return this._data.stream(ids || {\r\n            [Symbol.iterator]: this._ids.keys.bind(this._ids)\r\n        });\r\n    }\r\n    /**\r\n     * Render the instance unusable prior to garbage collection.\r\n     *\r\n     * @remarks\r\n     * The intention of this method is to help discover scenarios where the data\r\n     * view is being used when the programmer thinks it has been garbage collected\r\n     * already. It's stricter version of `dataView.setData(null)`.\r\n     */\r\n    dispose() {\r\n        var _a;\r\n        if ((_a = this._data) === null || _a === void 0 ? void 0 : _a.off) {\r\n            this._data.off(\"*\", this._listener);\r\n        }\r\n        const message = \"This data view has already been disposed of.\";\r\n        Object.defineProperty(this, \"_data\", {\r\n            get: () => {\r\n                throw new Error(message);\r\n            },\r\n            set: () => {\r\n                throw new Error(message);\r\n            },\r\n            configurable: false\r\n        });\r\n    }\r\n    /**\r\n     * Event listener. Will propagate all events from the connected data set to the subscribers of the DataView, but will filter the items and only trigger when there are changes in the filtered data set.\r\n     *\r\n     * @param event - The name of the event.\r\n     * @param params - Parameters of the event.\r\n     * @param senderId - Id supplied by the sender.\r\n     */\r\n    _onEvent(event, params, senderId) {\r\n        if (!params || !params.items || !this._data) {\r\n            return;\r\n        }\r\n        const ids = params.items;\r\n        const addedIds = [];\r\n        const updatedIds = [];\r\n        const removedIds = [];\r\n        const oldItems = [];\r\n        const updatedItems = [];\r\n        const removedItems = [];\r\n        switch (event) {\r\n            case \"add\":\r\n                // filter the ids of the added items\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    const item = this.get(id);\r\n                    if (item) {\r\n                        this._ids.add(id);\r\n                        addedIds.push(id);\r\n                    }\r\n                }\r\n                break;\r\n            case \"update\":\r\n                // determine the event from the views viewpoint: an updated\r\n                // item can be added, updated, or removed from this view.\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    const item = this.get(id);\r\n                    if (item) {\r\n                        if (this._ids.has(id)) {\r\n                            updatedIds.push(id);\r\n                            updatedItems.push(params.data[i]);\r\n                            oldItems.push(params.oldData[i]);\r\n                        }\r\n                        else {\r\n                            this._ids.add(id);\r\n                            addedIds.push(id);\r\n                        }\r\n                    }\r\n                    else {\r\n                        if (this._ids.has(id)) {\r\n                            this._ids.delete(id);\r\n                            removedIds.push(id);\r\n                            removedItems.push(params.oldData[i]);\r\n                        }\r\n                    }\r\n                }\r\n                break;\r\n            case \"remove\":\r\n                // filter the ids of the removed items\r\n                for (let i = 0, len = ids.length; i < len; i++) {\r\n                    const id = ids[i];\r\n                    if (this._ids.has(id)) {\r\n                        this._ids.delete(id);\r\n                        removedIds.push(id);\r\n                        removedItems.push(params.oldData[i]);\r\n                    }\r\n                }\r\n                break;\r\n        }\r\n        this.length += addedIds.length - removedIds.length;\r\n        if (addedIds.length) {\r\n            this._trigger(\"add\", { items: addedIds }, senderId);\r\n        }\r\n        if (updatedIds.length) {\r\n            this._trigger(\"update\", { items: updatedIds, oldData: oldItems, data: updatedItems }, senderId);\r\n        }\r\n        if (removedIds.length) {\r\n            this._trigger(\"remove\", { items: removedIds, oldData: removedItems }, senderId);\r\n        }\r\n    }\r\n}\n\nexport { DataSet, DataStream, DataView, Queue, createNewDataPipeFrom };\n//# sourceMappingURL=vis-data.js.map\n","// first check if moment.js is already loaded in the browser window, if so,\n// use this instance. Else, load via commonjs.\n//\n// Note: This doesn't work in ESM.\nmodule.exports = (typeof window !== 'undefined') && window['moment'] || require('moment');\n","// utility functions\n\nexport * from \"vis-util/esnext\";\nimport * as util from \"vis-util/esnext\";\nimport { getType, isNumber, isString } from \"vis-util/esnext\";\nimport { DataSet, createNewDataPipeFrom } from \"vis-data/esnext\";\n\nimport moment from \"moment\";\n\n// parse ASP.Net Date pattern,\n// for example '/Date(1198908717056)/' or '/Date(1198908717056-0700)/'\n// code from http://momentjs.com/\nconst ASPDateRegex = /^\\/?Date\\((-?\\d+)/i;\n\n/**\n * Convert an object into another type\n *\n * @param object - Value of unknown type.\n * @param type - Name of the desired type.\n *\n * @returns Object in the desired type.\n * @throws Error\n */\nexport function convert(object, type) {\n  let match;\n\n  if (object === undefined) {\n    return undefined;\n  }\n  if (object === null) {\n    return null;\n  }\n\n  if (!type) {\n    return object;\n  }\n  if (!(typeof type === \"string\") && !(type instanceof String)) {\n    throw new Error(\"Type must be a string\");\n  }\n\n  //noinspection FallthroughInSwitchStatementJS\n  switch (type) {\n    case \"boolean\":\n    case \"Boolean\":\n      return Boolean(object);\n\n    case \"number\":\n    case \"Number\":\n      if (isString(object) && !isNaN(Date.parse(object))) {\n        return moment(object).valueOf();\n      } else {\n        // @TODO: I don't think that Number and String constructors are a good idea.\n        // This could also fail if the object doesn't have valueOf method or if it's redefined.\n        // For example: Object.create(null) or { valueOf: 7 }.\n        return Number(object.valueOf());\n      }\n    case \"string\":\n    case \"String\":\n      return String(object);\n\n    case \"Date\":\n      if (isNumber(object)) {\n        return new Date(object);\n      }\n      if (object instanceof Date) {\n        return new Date(object.valueOf());\n      } else if (moment.isMoment(object)) {\n        return new Date(object.valueOf());\n      }\n      if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return new Date(Number(match[1])); // parse number\n        } else {\n          return moment(new Date(object)).toDate(); // parse string\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" + getType(object) + \" to type Date\"\n        );\n      }\n\n    case \"Moment\":\n      if (isNumber(object)) {\n        return moment(object);\n      }\n      if (object instanceof Date) {\n        return moment(object.valueOf());\n      } else if (moment.isMoment(object)) {\n        return moment(object);\n      }\n      if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return moment(Number(match[1])); // parse number\n        } else {\n          return moment(object); // parse string\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" + getType(object) + \" to type Date\"\n        );\n      }\n\n    case \"ISODate\":\n      if (isNumber(object)) {\n        return new Date(object);\n      } else if (object instanceof Date) {\n        return object.toISOString();\n      } else if (moment.isMoment(object)) {\n        return object.toDate().toISOString();\n      } else if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        if (match) {\n          // object is an ASP date\n          return new Date(Number(match[1])).toISOString(); // parse number\n        } else {\n          return moment(object).format(); // ISO 8601\n        }\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" +\n            getType(object) +\n            \" to type ISODate\"\n        );\n      }\n\n    case \"ASPDate\":\n      if (isNumber(object)) {\n        return \"/Date(\" + object + \")/\";\n      } else if (object instanceof Date || moment.isMoment(object)) {\n        return \"/Date(\" + object.valueOf() + \")/\";\n      } else if (isString(object)) {\n        match = ASPDateRegex.exec(object);\n        let value;\n        if (match) {\n          // object is an ASP date\n          value = new Date(Number(match[1])).valueOf(); // parse number\n        } else {\n          value = new Date(object).valueOf(); // parse string\n        }\n        return \"/Date(\" + value + \")/\";\n      } else {\n        throw new Error(\n          \"Cannot convert object of type \" +\n            getType(object) +\n            \" to type ASPDate\"\n        );\n      }\n\n    default:\n      throw new Error(`Unknown type ${type}`);\n  }\n}\n\n/**\n * Create a Data Set like wrapper to seamlessly coerce data types.\n *\n * @param rawDS - The Data Set with raw uncoerced data.\n * @param type - A record assigning a data type to property name.\n *\n * @remarks\n * The write operations (`add`, `remove`, `update` and `updateOnly`) write into\n * the raw (uncoerced) data set. These values are then picked up by a pipe\n * which coerces the values using the [[convert]] function and feeds them into\n * the coerced data set. When querying (`forEach`, `get`, `getIds`, `off` and\n * `on`) the values are then fetched from the coerced data set and already have\n * the required data types. The values are coerced only once when inserted and\n * then the same value is returned each time until it is updated or deleted.\n *\n * For example: `typeCoercedDataSet.add({ id: 7, start: \"2020-01-21\" })` would\n * result in `typeCoercedDataSet.get(7)` returning `{ id: 7, start: moment(new\n * Date(\"2020-01-21\")).toDate() }`.\n *\n * Use the dispose method prior to throwing a reference to this away. Otherwise\n * the pipe connecting the two Data Sets will keep the unaccessible coerced\n * Data Set alive and updated as long as the raw Data Set exists.\n *\n * @returns A Data Set like object that saves data into the raw Data Set and\n * retrieves them from the coerced Data Set.\n */\nexport function typeCoerceDataSet(\n  rawDS,\n  type = { start: \"Date\", end: \"Date\" }\n) {\n  const idProp = rawDS._idProp;\n  const coercedDS = new DataSet({ fieldId: idProp });\n\n  const pipe = createNewDataPipeFrom(rawDS)\n    .map(item =>\n      Object.keys(item).reduce((acc, key) => {\n        acc[key] = convert(item[key], type[key]);\n        return acc;\n      }, {})\n    )\n    .to(coercedDS);\n\n  pipe.all().start();\n\n  return {\n    // Write only.\n    add: (...args) => rawDS.getDataSet().add(...args),\n    remove: (...args) => rawDS.getDataSet().remove(...args),\n    update: (...args) => rawDS.getDataSet().update(...args),\n    updateOnly: (...args) => rawDS.getDataSet().updateOnly(...args),\n\n    // Read only.\n    forEach: coercedDS.forEach.bind(coercedDS),\n    get: coercedDS.get.bind(coercedDS),\n    getIds: coercedDS.getIds.bind(coercedDS),\n    off: coercedDS.off.bind(coercedDS),\n    on: coercedDS.on.bind(coercedDS),\n\n    get length() {\n      return coercedDS.length;\n    },\n\n    // Non standard.\n    idProp,\n    type,\n\n    rawDS,\n    coercedDS,\n    dispose: () => pipe.stop()\n  };\n}\n\nexport default {\n  ...util,\n  convert\n};\n","var global = require('../internals/global');\nvar trim = require('../internals/string-trim').trim;\nvar whitespaces = require('../internals/whitespaces');\n\nvar $parseFloat = global.parseFloat;\nvar FORCED = 1 / $parseFloat(whitespaces + '-0') !== -Infinity;\n\n// `parseFloat` method\n// https://tc39.github.io/ecma262/#sec-parsefloat-string\nmodule.exports = FORCED ? function parseFloat(string) {\n  var trimmedString = trim(String(string));\n  var result = $parseFloat(trimmedString);\n  return result === 0 && trimmedString.charAt(0) == '-' ? -0 : result;\n} : $parseFloat;\n","var $ = require('../internals/export');\nvar parseFloatImplementation = require('../internals/number-parse-float');\n\n// `parseFloat` method\n// https://tc39.github.io/ecma262/#sec-parsefloat-string\n$({ global: true, forced: parseFloat != parseFloatImplementation }, {\n  parseFloat: parseFloatImplementation\n});\n","require('../modules/es.parse-float');\nvar path = require('../internals/path');\n\nmodule.exports = path.parseFloat;\n","import util from '../../util';\n\n/** Prototype for visual components */\nexport default class Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} [body]\n * @param {Object} [options]\n */\n  constructor(body, options) {  // eslint-disable-line no-unused-vars\n    this.options = null;\n    this.props = null;\n  }\n\n  /**\n   * Set options for the component. The new options will be merged into the\n   * current options.\n   * @param {Object} options\n   */\n  setOptions(options) {\n    if (options) {\n      util.extend(this.options, options);\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    // should be implemented by the component\n    return false;\n  }\n\n  /**\n   * Destroy the component. Cleanup DOM and event listeners\n   */\n  destroy() {\n    // should be implemented by the component\n  }\n\n  /**\n   * Test whether the component is resized since the last time _isResized() was\n   * called.\n   * @return {Boolean} Returns true if the component is resized\n   * @protected\n   */\n  _isResized() {\n    const resized = (\n      this.props._previousWidth !== this.props.width ||\n      this.props._previousHeight !== this.props.height\n    );\n\n    this.props._previousWidth = this.props.width;\n    this.props._previousHeight = this.props.height;\n\n    return resized;\n  }\n}\n","var $ = require('../internals/export');\nvar repeat = require('../internals/string-repeat');\n\n// `String.prototype.repeat` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.repeat\n$({ target: 'String', proto: true }, {\n  repeat: repeat\n});\n","'use strict';\nvar toInteger = require('../internals/to-integer');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\n\n// `String.prototype.repeat` method implementation\n// https://tc39.github.io/ecma262/#sec-string.prototype.repeat\nmodule.exports = ''.repeat || function repeat(count) {\n  var str = String(requireObjectCoercible(this));\n  var result = '';\n  var n = toInteger(count);\n  if (n < 0 || n == Infinity) throw RangeError('Wrong number of repetitions');\n  for (;n > 0; (n >>>= 1) && (str += str)) if (n & 1) result += str;\n  return result;\n};\n","require('../../../modules/es.string.repeat');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').repeat;\n","var repeat = require('../string/virtual/repeat');\n\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.repeat;\n  return typeof it === 'string' || it === StringPrototype\n    || (it instanceof String && own === StringPrototype.repeat) ? repeat : own;\n};\n","\n/**\n * used in Core to convert the options into a volatile variable\n * \n * @param {function} moment\n * @param {Object} body\n * @param {Array | Object} hiddenDates\n * @returns {number}\n */\nexport function convertHiddenOptions(moment, body, hiddenDates) {\n  if (hiddenDates && !Array.isArray(hiddenDates)) {\n    return convertHiddenOptions(moment, body, [hiddenDates])\n  }\n\n  body.hiddenDates = [];\n  if (hiddenDates) {\n    if (Array.isArray(hiddenDates) == true) {\n      for (let i = 0; i < hiddenDates.length; i++) {\n        if (hiddenDates[i].repeat === undefined) {\n          const dateItem = {};\n          dateItem.start = moment(hiddenDates[i].start).toDate().valueOf();\n          dateItem.end = moment(hiddenDates[i].end).toDate().valueOf();\n          body.hiddenDates.push(dateItem);\n        }\n      }\n      body.hiddenDates.sort((a, b) => a.start - b.start); // sort by start time\n    }\n  }\n}\n\n/**\n * create new entrees for the repeating hidden dates\n *\n * @param {function} moment\n * @param {Object} body\n * @param {Array | Object} hiddenDates\n * @returns {null}\n */\nexport function updateHiddenDates(moment, body, hiddenDates) {\n  if (hiddenDates && !Array.isArray(hiddenDates)) {\n    return updateHiddenDates(moment, body, [hiddenDates])\n  }\n\n  if (hiddenDates && body.domProps.centerContainer.width !== undefined) {\n    convertHiddenOptions(moment, body, hiddenDates);\n\n    const start = moment(body.range.start);\n    const end = moment(body.range.end);\n\n    const totalRange = (body.range.end - body.range.start);\n    const pixelTime = totalRange / body.domProps.centerContainer.width;\n\n    for (let i = 0; i < hiddenDates.length; i++) {\n      if (hiddenDates[i].repeat !== undefined) {\n        const startDate = moment(hiddenDates[i].start);\n        let endDate = moment(hiddenDates[i].end);\n\n        if (startDate._d == \"Invalid Date\") {\n          throw new Error(`Supplied start date is not valid: ${hiddenDates[i].start}`);\n        }\n        if (endDate._d == \"Invalid Date\") {\n          throw new Error(`Supplied end date is not valid: ${hiddenDates[i].end}`);\n        }\n\n        const duration = endDate - startDate;\n        if (duration >= 4 * pixelTime) {\n\n          let offset = 0;\n          const runUntil = end.clone();\n          switch (hiddenDates[i].repeat) {\n            case \"daily\": // case of time\n              if (startDate.day() != endDate.day()) {\n                offset = 1;\n              }\n              startDate.dayOfYear(start.dayOfYear());\n              startDate.year(start.year());\n              startDate.subtract(7,'days');\n\n              endDate.dayOfYear(start.dayOfYear());\n              endDate.year(start.year());\n              endDate.subtract(7 - offset,'days');\n\n              runUntil.add(1, 'weeks');\n              break;\n            case \"weekly\": {\n              const dayOffset = endDate.diff(startDate,'days');\n              const day = startDate.day();\n\n              // set the start date to the range.start\n              startDate.date(start.date());\n              startDate.month(start.month());\n              startDate.year(start.year());\n              endDate = startDate.clone();\n\n              // force\n              startDate.day(day);\n              endDate.day(day);\n              endDate.add(dayOffset,'days');\n\n              startDate.subtract(1,'weeks');\n              endDate.subtract(1,'weeks');\n\n              runUntil.add(1, 'weeks');\n              break;\n            }\n            case \"monthly\":\n              if (startDate.month() != endDate.month()) {\n                offset = 1;\n              }\n              startDate.month(start.month());\n              startDate.year(start.year());\n              startDate.subtract(1,'months');\n\n              endDate.month(start.month());\n              endDate.year(start.year());\n              endDate.subtract(1,'months');\n              endDate.add(offset,'months');\n\n              runUntil.add(1, 'months');\n              break;\n            case \"yearly\":\n              if (startDate.year() != endDate.year()) {\n                offset = 1;\n              }\n              startDate.year(start.year());\n              startDate.subtract(1,'years');\n              endDate.year(start.year());\n              endDate.subtract(1,'years');\n              endDate.add(offset,'years');\n\n              runUntil.add(1, 'years');\n              break;\n            default:\n              console.log(\"Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:\", hiddenDates[i].repeat);\n              return;\n          }\n          while (startDate < runUntil) {\n            body.hiddenDates.push({start: startDate.valueOf(), end: endDate.valueOf()});\n            switch (hiddenDates[i].repeat) {\n              case \"daily\":\n                startDate.add(1, 'days');\n                endDate.add(1, 'days');\n                break;\n              case \"weekly\":\n                startDate.add(1, 'weeks');\n                endDate.add(1, 'weeks');\n                break;\n              case \"monthly\":\n                startDate.add(1, 'months');\n                endDate.add(1, 'months');\n                break;\n              case \"yearly\":\n                startDate.add(1, 'y');\n                endDate.add(1, 'y');\n                break;\n              default:\n                console.log(\"Wrong repeat format, allowed are: daily, weekly, monthly, yearly. Given:\", hiddenDates[i].repeat);\n                return;\n            }\n          }\n          body.hiddenDates.push({start: startDate.valueOf(), end: endDate.valueOf()});\n        }\n      }\n    }\n    // remove duplicates, merge where possible\n    removeDuplicates(body);\n    // ensure the new positions are not on hidden dates\n    const startHidden = getIsHidden(body.range.start, body.hiddenDates);\n    const endHidden = getIsHidden(body.range.end,body.hiddenDates);\n    let rangeStart = body.range.start;\n    let rangeEnd = body.range.end;\n    if (startHidden.hidden == true) {rangeStart = body.range.startToFront == true ? startHidden.startDate - 1 : startHidden.endDate + 1;}\n    if (endHidden.hidden == true)   {rangeEnd   = body.range.endToFront == true ?   endHidden.startDate - 1   : endHidden.endDate + 1;}\n    if (startHidden.hidden == true || endHidden.hidden == true) {\n      body.range._applyRange(rangeStart, rangeEnd);\n    }\n  }\n\n}\n\n/**\n * remove duplicates from the hidden dates list. Duplicates are evil. They mess everything up.\n * Scales with N^2\n *\n * @param {Object} body\n */\nexport function removeDuplicates(body) {\n  const hiddenDates = body.hiddenDates;\n  const safeDates = [];\n  for (var i = 0; i < hiddenDates.length; i++) {\n    for (let j = 0; j < hiddenDates.length; j++) {\n      if (i != j && hiddenDates[j].remove != true && hiddenDates[i].remove != true) {\n        // j inside i\n        if (hiddenDates[j].start >= hiddenDates[i].start && hiddenDates[j].end <= hiddenDates[i].end) {\n          hiddenDates[j].remove = true;\n        }\n        // j start inside i\n        else if (hiddenDates[j].start >= hiddenDates[i].start && hiddenDates[j].start <= hiddenDates[i].end) {\n          hiddenDates[i].end = hiddenDates[j].end;\n          hiddenDates[j].remove = true;\n        }\n        // j end inside i\n        else if (hiddenDates[j].end >= hiddenDates[i].start && hiddenDates[j].end <= hiddenDates[i].end) {\n          hiddenDates[i].start = hiddenDates[j].start;\n          hiddenDates[j].remove = true;\n        }\n      }\n    }\n  }\n\n  for (i = 0; i < hiddenDates.length; i++) {\n    if (hiddenDates[i].remove !== true) {\n      safeDates.push(hiddenDates[i]);\n    }\n  }\n\n  body.hiddenDates = safeDates;\n  body.hiddenDates.sort((a, b) => a.start - b.start); // sort by start time\n}\n\n/**\n * Prints dates to console\n * @param {array} dates\n */\nexport function printDates(dates) {\n  for (let i =0; i < dates.length; i++) {\n    console.log(i, new Date(dates[i].start),new Date(dates[i].end), dates[i].start, dates[i].end, dates[i].remove);\n  }\n}\n\n/**\n * Used in TimeStep to avoid the hidden times.\n * @param {function} moment\n * @param {TimeStep} timeStep\n * @param {Date} previousTime\n */\nexport function stepOverHiddenDates(moment, timeStep, previousTime) {\n  let stepInHidden = false;\n  const currentValue = timeStep.current.valueOf();\n  for (let i = 0; i < timeStep.hiddenDates.length; i++) {\n    const startDate = timeStep.hiddenDates[i].start;\n    var endDate = timeStep.hiddenDates[i].end;\n    if (currentValue >= startDate && currentValue < endDate) {\n      stepInHidden = true;\n      break;\n    }\n  }\n\n  if (stepInHidden == true && currentValue < timeStep._end.valueOf() && currentValue != previousTime) {\n    const prevValue = moment(previousTime);\n    const newValue = moment(endDate);\n    //check if the next step should be major\n    if (prevValue.year() != newValue.year()) {timeStep.switchedYear = true;}\n    else if (prevValue.month() != newValue.month()) {timeStep.switchedMonth = true;}\n    else if (prevValue.dayOfYear() != newValue.dayOfYear()) {timeStep.switchedDay = true;}\n\n    timeStep.current = newValue;\n  }\n}\n\n///**\n// * Used in TimeStep to avoid the hidden times.\n// * @param timeStep\n// * @param previousTime\n// */\n//checkFirstStep = function(timeStep) {\n//  var stepInHidden = false;\n//  var currentValue = timeStep.current.valueOf();\n//  for (var i = 0; i < timeStep.hiddenDates.length; i++) {\n//    var startDate = timeStep.hiddenDates[i].start;\n//    var endDate = timeStep.hiddenDates[i].end;\n//    if (currentValue >= startDate && currentValue < endDate) {\n//      stepInHidden = true;\n//      break;\n//    }\n//  }\n//\n//  if (stepInHidden == true && currentValue <= timeStep._end.valueOf()) {\n//    var newValue = moment(endDate);\n//    timeStep.current = newValue.toDate();\n//  }\n//};\n\n/**\n * replaces the Core toScreen methods\n *\n * @param {timeline.Core} Core\n * @param {Date} time\n * @param {number} width\n * @returns {number}\n */\nexport function toScreen(Core, time, width) {\n  let conversion;\n  if (Core.body.hiddenDates.length == 0) {\n      conversion = Core.range.conversion(width);\n      return (time.valueOf() - conversion.offset) * conversion.scale;\n    } else {\n      const hidden = getIsHidden(time, Core.body.hiddenDates);\n      if (hidden.hidden == true) {\n        time = hidden.startDate;\n      }\n\n      const duration = getHiddenDurationBetween(Core.body.hiddenDates, Core.range.start, Core.range.end);\n      if (time < Core.range.start) {\n        conversion = Core.range.conversion(width, duration);\n        const hiddenBeforeStart = getHiddenDurationBeforeStart(Core.body.hiddenDates, time, conversion.offset);\n        time = Core.options.moment(time).toDate().valueOf();\n        time = time + hiddenBeforeStart;\n        return -(conversion.offset - time.valueOf()) * conversion.scale;\n        \n      } else if (time > Core.range.end) {\n        const rangeAfterEnd = {start: Core.range.start, end: time};\n        time = correctTimeForHidden(Core.options.moment, Core.body.hiddenDates, rangeAfterEnd, time);\n        conversion = Core.range.conversion(width, duration);\n        return (time.valueOf() - conversion.offset) * conversion.scale;\n\n      } else {\n        time = correctTimeForHidden(Core.options.moment, Core.body.hiddenDates, Core.range, time);\n        conversion = Core.range.conversion(width, duration);\n        return (time.valueOf() - conversion.offset) * conversion.scale;\n      }\n    }\n  }\n\n/**\n * Replaces the core toTime methods\n *\n * @param {timeline.Core} Core\n * @param {number} x\n * @param {number} width\n * @returns {Date}\n */\nexport function toTime(Core, x, width) {\n  if (Core.body.hiddenDates.length == 0) {\n    const conversion = Core.range.conversion(width);\n    return new Date(x / conversion.scale + conversion.offset);\n  }\n  else {\n    const hiddenDuration = getHiddenDurationBetween(Core.body.hiddenDates, Core.range.start, Core.range.end);\n    const totalDuration = Core.range.end - Core.range.start - hiddenDuration;\n    const partialDuration = totalDuration * x / width;\n    const accumulatedHiddenDuration = getAccumulatedHiddenDuration(Core.body.hiddenDates, Core.range, partialDuration);\n\n    return new Date(accumulatedHiddenDuration + partialDuration + Core.range.start);\n  }\n}\n\n/**\n * Support function\n *\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {number} start\n * @param {number} end\n * @returns {number}\n */\nexport function getHiddenDurationBetween(hiddenDates, start, end) {\n  let duration = 0;\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= start && endDate < end) {\n      duration += endDate - startDate;\n    }\n  }\n  return duration;\n}\n\n/**\n * Support function\n *\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {number} start\n * @param {number} end\n * @returns {number}\n */\nexport function getHiddenDurationBeforeStart(hiddenDates, start, end) {\n  let duration = 0;\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n\n    if (startDate >= start && endDate <= end) {\n      duration += endDate - startDate;\n    }\n  }\n  return duration;\n}\n\n/**\n * Support function\n * @param {function} moment\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {Date} time\n * @returns {number}\n */\nexport function correctTimeForHidden(moment, hiddenDates, range, time) {\n  time = moment(time).toDate().valueOf();\n  time -= getHiddenDurationBefore(moment, hiddenDates,range,time);\n  return time;\n}\n\n/**\n * Support function\n * @param {function} moment\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {Date} time\n * @returns {number}\n */\nexport function getHiddenDurationBefore(moment, hiddenDates, range, time) {\n  let timeOffset = 0;\n  time = moment(time).toDate().valueOf();\n\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= range.start && endDate < range.end) {\n      if (time >= endDate) {\n        timeOffset += (endDate - startDate);\n      }\n    }\n  }\n  return timeOffset;\n}\n\n/**\n * sum the duration from start to finish, including the hidden duration,\n * until the required amount has been reached, return the accumulated hidden duration\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {{start: number, end: number}} range\n * @param {number} [requiredDuration=0]\n * @returns {number}\n */\nexport function getAccumulatedHiddenDuration(hiddenDates, range, requiredDuration) {\n  let hiddenDuration = 0;\n  let duration = 0;\n  let previousPoint = range.start;\n  //printDates(hiddenDates)\n  for (let i = 0; i < hiddenDates.length; i++) {\n    const startDate = hiddenDates[i].start;\n    const endDate = hiddenDates[i].end;\n    // if time after the cutout, and the\n    if (startDate >= range.start && endDate < range.end) {\n      duration += startDate - previousPoint;\n      previousPoint = endDate;\n      if (duration >= requiredDuration) {\n        break;\n      }\n      else {\n        hiddenDuration += endDate - startDate;\n      }\n    }\n  }\n\n  return hiddenDuration;\n}\n\n/**\n * used to step over to either side of a hidden block. Correction is disabled on tablets, might be set to true\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @param {Date} time\n * @param {number} direction\n * @param {boolean} correctionEnabled\n * @returns {Date|number}\n */\nexport function snapAwayFromHidden(hiddenDates, time, direction, correctionEnabled) {\n  const isHidden = getIsHidden(time, hiddenDates);\n  if (isHidden.hidden == true) {\n    if (direction < 0) {\n      if (correctionEnabled == true) {\n        return isHidden.startDate - (isHidden.endDate - time) - 1;\n      }\n      else {\n        return isHidden.startDate - 1;\n      }\n    }\n    else {\n      if (correctionEnabled == true) {\n        return isHidden.endDate + (time - isHidden.startDate) + 1;\n      }\n      else {\n        return isHidden.endDate + 1;\n      }\n    }\n  }\n  else {\n    return time;\n  }\n\n}\n\n/**\n * Check if a time is hidden\n *\n * @param {Date} time\n * @param {Array.<{start: Window.start, end: *}>} hiddenDates\n * @returns {{hidden: boolean, startDate: Window.start, endDate: *}}\n */\nexport function getIsHidden(time, hiddenDates) {\n  for (let i = 0; i < hiddenDates.length; i++) {\n    var startDate = hiddenDates[i].start;\n    var endDate = hiddenDates[i].end;\n\n    if (time >= startDate && time < endDate) { // if the start is entering a hidden zone\n      return {hidden: true, startDate, endDate};\n    }\n  }\n  return {hidden: false, startDate, endDate};\n}\n","import util from '../util';\nimport moment from '../module/moment';\nimport Component from './component/Component';\nimport * as DateUtil  from './DateUtil';\n\n/**\n * A Range controls a numeric range with a start and end value.\n * The Range adjusts the range based on mouse events or programmatic changes,\n * and triggers events when the range is changing or has been changed.\n */\nexport default class Range extends Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter}} body\n * @param {Object} [options]    See description at Range.setOptions\n * @constructor Range\n * @extends Component\n */\n  constructor(body, options) {\n    super();\n    const now = moment().hours(0).minutes(0).seconds(0).milliseconds(0);\n    const start = now.clone().add(-3, 'days').valueOf();\n    const end = now.clone().add(3, 'days').valueOf(); \n    this.millisecondsPerPixelCache = undefined;\n    \n    if(options === undefined) {\n      this.start = start;\n      this.end = end;\n    } else {\n      this.start = options.start || start;\n      this.end = options.end || end\n    }\n\n    this.rolling = false;\n\n    this.body = body;\n    this.deltaDifference = 0;\n    this.scaleOffset = 0;\n    this.startToFront = false;\n    this.endToFront = true;\n\n    // default options\n    this.defaultOptions = {\n      rtl: false,\n      start: null,\n      end: null,\n      moment,\n      direction: 'horizontal', // 'horizontal' or 'vertical'\n      moveable: true,\n      zoomable: true,\n      min: null,\n      max: null,\n      zoomMin: 10,                                // milliseconds\n      zoomMax: 1000 * 60 * 60 * 24 * 365 * 10000,  // milliseconds\n      rollingMode: {\n        follow: false,\n        offset: 0.5\n      }\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.props = {\n      touch: {}\n    };\n    this.animationTimer = null;\n\n    // drag listeners for dragging\n    this.body.emitter.on('panstart', this._onDragStart.bind(this));\n    this.body.emitter.on('panmove',  this._onDrag.bind(this));\n    this.body.emitter.on('panend',   this._onDragEnd.bind(this));\n\n    // mouse wheel for zooming\n    this.body.emitter.on('mousewheel', this._onMouseWheel.bind(this));\n\n    // pinch to zoom\n    this.body.emitter.on('touch', this._onTouch.bind(this));\n    this.body.emitter.on('pinch', this._onPinch.bind(this));\n\n    // on click of rolling mode button\n    this.body.dom.rollingModeBtn.addEventListener('click', this.startRolling.bind(this));\n\n    this.setOptions(options);\n  }\n\n  /**\n   * Set options for the range controller\n   * @param {Object} options      Available options:\n   *                              {number | Date | String} start  Start date for the range\n   *                              {number | Date | String} end    End date for the range\n   *                              {number} min    Minimum value for start\n   *                              {number} max    Maximum value for end\n   *                              {number} zoomMin    Set a minimum value for\n   *                                                  (end - start).\n   *                              {number} zoomMax    Set a maximum value for\n   *                                                  (end - start).\n   *                              {boolean} moveable Enable moving of the range\n   *                                                 by dragging. True by default\n   *                              {boolean} zoomable Enable zooming of the range\n   *                                                 by pinching/scrolling. True by default\n   */\n  setOptions(options) {\n    if (options) {\n      // copy the options that we know\n      const fields = [\n        'animation', 'direction', 'min', 'max', 'zoomMin', 'zoomMax', 'moveable', 'zoomable',\n        'moment', 'activate', 'hiddenDates', 'zoomKey', 'zoomFriction', 'rtl', 'showCurrentTime', 'rollingMode', 'horizontalScroll'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n\n      if (options.rollingMode && options.rollingMode.follow) {\n        this.startRolling();\n      }\n      if ('start' in options || 'end' in options) {\n        // apply a new range. both start and end are optional\n        this.setRange(options.start, options.end);\n      }\n    }\n  }\n\n  /**\n   * Start auto refreshing the current time bar\n   */\n  startRolling() {\n    const me = this;\n\n    /**\n     *  Updates the current time.\n     */\n    function update () {\n      me.stopRolling();\n      me.rolling = true;\n\n\n      let interval = me.end - me.start;\n      const t = util.convert(new Date(), 'Date').valueOf();\n      const rollingModeOffset = me.options.rollingMode && me.options.rollingMode.offset || 0.5\n\n      const start = t - interval * (rollingModeOffset);\n      const end = t + interval * (1 - rollingModeOffset);\n\n      const options = {\n        animation: false\n      };\n      me.setRange(start, end, options);\n\n      // determine interval to refresh\n      const scale = me.conversion(me.body.domProps.center.width).scale;\n      interval = 1 / scale / 10;\n      if (interval < 30)   interval = 30;\n      if (interval > 1000) interval = 1000;\n\n      me.body.dom.rollingModeBtn.style.visibility = \"hidden\";\n      // start a renderTimer to adjust for the new time\n      me.currentTimeTimer = setTimeout(update, interval);\n    }\n\n    update();\n  }\n\n  /**\n   * Stop auto refreshing the current time bar\n   */\n  stopRolling() {\n    if (this.currentTimeTimer !== undefined) {\n      clearTimeout(this.currentTimeTimer);\n      this.rolling = false;\n      this.body.dom.rollingModeBtn.style.visibility = \"visible\";\n    }\n  }\n\n  /**\n   * Set a new start and end range\n   * @param {Date | number | string} start\n   * @param {Date | number | string} end\n   * @param {Object} options      Available options:\n   *                              {boolean | {duration: number, easingFunction: string}} [animation=false]\n   *                                    If true, the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                              {boolean} [byUser=false]\n   *                              {Event}  event  Mouse event\n   * @param {Function} callback     a callback function to be executed at the end of this function  \n   * @param {Function} frameCallback    a callback function executed each frame of the range animation.\n   *                                    The callback will be passed three parameters:\n   *                                    {number} easeCoefficient    an easing coefficent\n   *                                    {boolean} willDraw          If true the caller will redraw after the callback completes\n   *                                    {boolean} done              If true then animation is ending after the current frame\n   * @return {void}\n   */\n  setRange(start, end, options, callback, frameCallback) {\n    if (!options) {\n      options = {};\n    }\n    if (options.byUser !== true) {\n      options.byUser = false;\n    }\n    const me = this;\n    const finalStart = start != undefined ? util.convert(start, 'Date').valueOf() : null;\n    const finalEnd   = end != undefined   ? util.convert(end, 'Date').valueOf()   : null;\n    this._cancelAnimation();\n    this.millisecondsPerPixelCache = undefined;\n\n    if (options.animation) { // true or an Object\n      const initStart = this.start;\n      const initEnd = this.end;\n      const duration = (typeof options.animation === 'object' && 'duration' in options.animation) ? options.animation.duration : 500;\n      const easingName = (typeof options.animation === 'object' && 'easingFunction' in options.animation) ? options.animation.easingFunction : 'easeInOutQuad';\n      const easingFunction = util.easingFunctions[easingName];\n      if (!easingFunction) {\n        throw new Error(`Unknown easing function ${JSON.stringify(easingName)}. Choose from: ${Object.keys(util.easingFunctions).join(', ')}`);\n      }\n\n      const initTime = Date.now();\n      let anyChanged = false;\n\n      const next = () => {\n        if (!me.props.touch.dragging) {\n          const now = Date.now();\n          const time = now - initTime;\n          const ease = easingFunction(time / duration);\n          const done = time > duration;\n          const s = (done || finalStart === null) ? finalStart : initStart + (finalStart - initStart) * ease;\n          const e = (done || finalEnd   === null) ? finalEnd   : initEnd   + (finalEnd   - initEnd)   * ease;\n\n          changed = me._applyRange(s, e);\n          DateUtil.updateHiddenDates(me.options.moment, me.body, me.options.hiddenDates);\n          anyChanged = anyChanged || changed;\n\n          const params = {\n            start: new Date(me.start), \n            end: new Date(me.end), \n            byUser: options.byUser,\n            event: options.event\n          };\n\n          if (frameCallback) { frameCallback(ease, changed, done); }\n\n          if (changed) {          \n            me.body.emitter.emit('rangechange', params);\n          }\n\n          if (done) {\n            if (anyChanged) {\n              me.body.emitter.emit('rangechanged', params);\n              if (callback) { return callback() }\n            }\n          }\n          else {\n            // animate with as high as possible frame rate, leave 20 ms in between\n            // each to prevent the browser from blocking\n            me.animationTimer = setTimeout(next, 20);\n          }\n        }\n      };\n\n      return next();\n    }\n    else {\n      var changed = this._applyRange(finalStart, finalEnd);\n      DateUtil.updateHiddenDates(this.options.moment, this.body, this.options.hiddenDates);\n      if (changed) {\n        const params = {\n          start: new Date(this.start), \n          end: new Date(this.end), \n          byUser: options.byUser, \n          event: options.event\n        };\n\n        this.body.emitter.emit('rangechange', params);\n        clearTimeout( me.timeoutID );\n        me.timeoutID = setTimeout( () => {\n          me.body.emitter.emit('rangechanged', params);\n        }, 200 );\n        if (callback) { return callback() }\n      }\n    }\n  }\n\n  /**\n   * Get the number of milliseconds per pixel.\n   *\n   * @returns {undefined|number}\n   */\n  getMillisecondsPerPixel() {\n    if (this.millisecondsPerPixelCache === undefined) {\n      this.millisecondsPerPixelCache = (this.end - this.start) / this.body.dom.center.clientWidth;\n    }\n    return this.millisecondsPerPixelCache;\n  }\n\n  /**\n   * Stop an animation\n   * @private\n   */\n  _cancelAnimation() {\n    if (this.animationTimer) {\n      clearTimeout(this.animationTimer);\n      this.animationTimer = null;\n    }\n  }\n\n  /**\n   * Set a new start and end range. This method is the same as setRange, but\n   * does not trigger a range change and range changed event, and it returns\n   * true when the range is changed\n   * @param {number} [start]\n   * @param {number} [end]\n   * @return {boolean} changed\n   * @private\n   */\n  _applyRange(start, end) {\n    let newStart = (start != null) ? util.convert(start, 'Date').valueOf() : this.start;\n    let newEnd   = (end != null)   ? util.convert(end, 'Date').valueOf()   : this.end;\n    const max = (this.options.max != null) ? util.convert(this.options.max, 'Date').valueOf() : null;\n    const min = (this.options.min != null) ? util.convert(this.options.min, 'Date').valueOf() : null;\n    let diff;\n\n    // check for valid number\n    if (isNaN(newStart) || newStart === null) {\n      throw new Error(`Invalid start \"${start}\"`);\n    }\n    if (isNaN(newEnd) || newEnd === null) {\n      throw new Error(`Invalid end \"${end}\"`);\n    }\n\n    // prevent end < start\n    if (newEnd < newStart) {\n      newEnd = newStart;\n    }\n\n    // prevent start < min\n    if (min !== null) {\n      if (newStart < min) {\n        diff = (min - newStart);\n        newStart += diff;\n        newEnd += diff;\n\n        // prevent end > max\n        if (max != null) {\n          if (newEnd > max) {\n            newEnd = max;\n          }\n        }\n      }\n    }\n\n    // prevent end > max\n    if (max !== null) {\n      if (newEnd > max) {\n        diff = (newEnd - max);\n        newStart -= diff;\n        newEnd -= diff;\n\n        // prevent start < min\n        if (min != null) {\n          if (newStart < min) {\n            newStart = min;\n          }\n        }\n      }\n    }\n\n    // prevent (end-start) < zoomMin\n    if (this.options.zoomMin !== null) {\n      let zoomMin = parseFloat(this.options.zoomMin);\n      if (zoomMin < 0) {\n        zoomMin = 0;\n      }\n      if ((newEnd - newStart) < zoomMin) {\n        // compensate for a scale of 0.5 ms\n        const compensation = 0.5;\n        if ((this.end - this.start) === zoomMin && newStart >= this.start - compensation && newEnd <= this.end) {\n          // ignore this action, we are already zoomed to the minimum\n          newStart = this.start;\n          newEnd = this.end;\n        }\n        else {\n          // zoom to the minimum\n          diff = (zoomMin - (newEnd - newStart));\n          newStart -= diff / 2;\n          newEnd += diff / 2;\n        }\n      }\n    }\n\n    // prevent (end-start) > zoomMax\n    if (this.options.zoomMax !== null) {\n      let zoomMax = parseFloat(this.options.zoomMax);\n      if (zoomMax < 0) {\n        zoomMax = 0;\n      }\n\n      if ((newEnd - newStart) > zoomMax) {\n        if ((this.end - this.start) === zoomMax && newStart < this.start && newEnd > this.end) {\n          // ignore this action, we are already zoomed to the maximum\n          newStart = this.start;\n          newEnd = this.end;\n        }\n        else {\n          // zoom to the maximum\n          diff = ((newEnd - newStart) - zoomMax);\n          newStart += diff / 2;\n          newEnd -= diff / 2;\n        }\n      }\n    }\n\n    const changed = (this.start != newStart || this.end != newEnd);\n\n    // if the new range does NOT overlap with the old range, emit checkRangedItems to avoid not showing ranged items (ranged meaning has end time, not necessarily of type Range)\n    if (!((newStart >= this.start && newStart   <= this.end) || (newEnd   >= this.start && newEnd   <= this.end)) &&\n        !((this.start >= newStart && this.start <= newEnd)   || (this.end >= newStart   && this.end <= newEnd) )) {\n      this.body.emitter.emit('checkRangedItems');\n    }\n\n    this.start = newStart;\n    this.end = newEnd;\n    return changed;\n  }\n\n  /**\n   * Retrieve the current range.\n   * @return {Object} An object with start and end properties\n   */\n  getRange() {\n    return {\n      start: this.start,\n      end: this.end\n    };\n  }\n\n  /**\n   * Calculate the conversion offset and scale for current range, based on\n   * the provided width\n   * @param {number} width\n   * @param {number} [totalHidden=0]\n   * @returns {{offset: number, scale: number}} conversion\n   */\n  conversion(width, totalHidden) {\n    return Range.conversion(this.start, this.end, width, totalHidden);\n  }\n\n  /**\n   * Static method to calculate the conversion offset and scale for a range,\n   * based on the provided start, end, and width\n   * @param {number} start\n   * @param {number} end\n   * @param {number} width\n   * @param {number} [totalHidden=0]\n   * @returns {{offset: number, scale: number}} conversion\n   */\n  static conversion(start, end, width, totalHidden) {\n    if (totalHidden === undefined) {\n      totalHidden = 0;\n    }\n    if (width != 0 && (end - start != 0)) {\n      return {\n        offset: start,\n        scale: width / (end - start - totalHidden)\n      }\n    }\n    else {\n      return {\n        offset: 0,\n        scale: 1\n      };\n    }\n  }\n\n  /**\n   * Start dragging horizontally or vertically\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    this.deltaDifference = 0;\n    this.previousDelta = 0;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // only start dragging when the mouse is inside the current range\n    if (!this._isInsideRange(event)) return;\n\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    this.stopRolling();\n\n    this.props.touch.start = this.start;\n    this.props.touch.end = this.end;\n    this.props.touch.dragging = true;\n\n    if (this.body.dom.root) {\n      this.body.dom.root.style.cursor = 'move';\n    }\n  }\n\n  /**\n   * Perform dragging operation\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!event) return;\n\n    if (!this.props.touch.dragging) return;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // TODO: this may be redundant in hammerjs2\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    const direction = this.options.direction;\n    validateDirection(direction);\n    let delta = (direction == 'horizontal') ? event.deltaX : event.deltaY;\n    delta -= this.deltaDifference;\n    let interval = (this.props.touch.end - this.props.touch.start);\n\n    // normalize dragging speed if cutout is in between.\n    const duration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    interval -= duration;\n\n    const width = (direction == 'horizontal') ? this.body.domProps.center.width : this.body.domProps.center.height;\n    let diffRange;\n    if (this.options.rtl) {\n      diffRange = delta / width * interval;\n    } else {\n      diffRange = -delta / width * interval;\n    }\n\n    const newStart = this.props.touch.start + diffRange;\n    const newEnd = this.props.touch.end + diffRange;\n\n    // snapping times away from hidden zones\n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, this.previousDelta-delta, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, this.previousDelta-delta, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      this.deltaDifference += delta;\n      this.props.touch.start = safeStart;\n      this.props.touch.end = safeEnd;\n      this._onDrag(event);\n      return;\n    }\n\n    this.previousDelta = delta;\n    this._applyRange(newStart, newEnd);\n\n\n    const startDate = new Date(this.start);\n    const endDate = new Date(this.end);\n\n    // fire a rangechange event\n    this.body.emitter.emit('rangechange', {\n      start: startDate,\n      end:   endDate,\n      byUser: true,\n      event\n    });\n\n    // fire a panmove event\n    this.body.emitter.emit('panmove');\n  }\n\n  /**\n   * Stop dragging operation\n   * @param {event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    if (!this.props.touch.dragging) return;\n\n    // only allow dragging when configured as movable\n    if (!this.options.moveable) return;\n\n    // TODO: this may be redundant in hammerjs2\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.props.touch.allowDragging) return;\n\n    this.props.touch.dragging = false;\n    if (this.body.dom.root) {\n      this.body.dom.root.style.cursor = 'auto';\n    }\n\n    // fire a rangechanged event\n    this.body.emitter.emit('rangechanged', {\n      start: new Date(this.start),\n      end:   new Date(this.end),\n      byUser: true,\n      event\n    });\n  }\n\n  /**\n   * Event handler for mouse wheel event, used to zoom\n   * Code from http://adomas.org/javascript-mouse-wheel/\n   * @param {Event} event\n   * @private\n   */\n  _onMouseWheel(event) {\n    // retrieve delta\n    let delta = 0;\n    if (event.wheelDelta) { /* IE/Opera. */\n      delta = event.wheelDelta / 120;\n    } else if (event.detail) { /* Mozilla case. */\n      // In Mozilla, sign of delta is different than in IE.\n      // Also, delta is multiple of 3.\n      delta = -event.detail / 3;\n    } else if (event.deltaY) {\n      delta = -event.deltaY / 3;\n    } \n\n    // don't allow zoom when the according key is pressed and the zoomKey option or not zoomable but movable\n    if ((this.options.zoomKey && !event[this.options.zoomKey] && this.options.zoomable) \n      || (!this.options.zoomable && this.options.moveable)) {\n      return;\n    }\n\n    // only allow zooming when configured as zoomable and moveable\n    if (!(this.options.zoomable && this.options.moveable)) return;\n    \n    // only zoom when the mouse is inside the current range\n    if (!this._isInsideRange(event)) return;\n\n    // If delta is nonzero, handle it.\n    // Basically, delta is now positive if wheel was scrolled up,\n    // and negative, if wheel was scrolled down.\n    if (delta) {\n      // perform the zoom action. Delta is normally 1 or -1\n\n      // adjust a negative delta such that zooming in with delta 0.1\n      // equals zooming out with a delta -0.1\n\n      const zoomFriction = this.options.zoomFriction || 5;\n      let scale;\n      if (delta < 0) {\n        scale = 1 - (delta / zoomFriction);\n      }\n      else {\n        scale = 1 / (1 + (delta / zoomFriction)) ;\n      }\n\n      // calculate center, the date to zoom around\n      let pointerDate;\n      if (this.rolling) {\n        const rollingModeOffset = this.options.rollingMode && this.options.rollingMode.offset || 0.5;\n        pointerDate = this.start + ((this.end - this.start) * rollingModeOffset);\n      } else {\n        const pointer = this.getPointer({x: event.clientX, y: event.clientY}, this.body.dom.center);\n        pointerDate = this._pointerToDate(pointer);\n      }\n      this.zoom(scale, pointerDate, delta, event);\n\n      // Prevent default actions caused by mouse wheel\n      // (else the page and timeline both scroll)\n      event.preventDefault();\n    }\n  }\n\n  /**\n   * Start of a touch gesture\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {  // eslint-disable-line no-unused-vars\n    this.props.touch.start = this.start;\n    this.props.touch.end = this.end;\n    this.props.touch.allowDragging = true;\n    this.props.touch.center = null;\n    this.props.touch.centerDate = null;\n    this.scaleOffset = 0;\n    this.deltaDifference = 0;\n    // Disable the browser default handling of this event.\n    util.preventDefault(event);\n  }\n\n  /**\n   * Handle pinch event\n   * @param {Event} event\n   * @private\n   */\n  _onPinch(event) {\n    // only allow zooming when configured as zoomable and moveable\n    if (!(this.options.zoomable && this.options.moveable)) return;\n\n    // Disable the browser default handling of this event.\n    util.preventDefault(event);\n\n    this.props.touch.allowDragging = false;\n\n    if (!this.props.touch.center) {\n      this.props.touch.center = this.getPointer(event.center, this.body.dom.center);\n      this.props.touch.centerDate = this._pointerToDate(this.props.touch.center);\n    }\n\n    this.stopRolling();\n    const scale = 1 / (event.scale + this.scaleOffset);\n    const centerDate = this.props.touch.centerDate;\n\n    const hiddenDuration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    const hiddenDurationBefore = DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this, centerDate);\n    const hiddenDurationAfter = hiddenDuration - hiddenDurationBefore;\n\n    // calculate new start and end\n    let newStart = (centerDate - hiddenDurationBefore) + (this.props.touch.start - (centerDate - hiddenDurationBefore)) * scale;\n    let newEnd = (centerDate + hiddenDurationAfter) + (this.props.touch.end - (centerDate + hiddenDurationAfter)) * scale;\n\n    // snapping times away from hidden zones\n    this.startToFront = 1 - scale <= 0; // used to do the right auto correction with periodic hidden times\n    this.endToFront = scale - 1 <= 0;   // used to do the right auto correction with periodic hidden times\n    \n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, 1 - scale, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, scale - 1, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      this.props.touch.start = safeStart;\n      this.props.touch.end = safeEnd;\n      this.scaleOffset = 1 - event.scale;\n      newStart = safeStart;\n      newEnd = safeEnd;\n    }\n    \n    const options = {\n      animation: false,\n      byUser: true,\n      event\n    };\n    this.setRange(newStart, newEnd, options);\n\n    this.startToFront = false; // revert to default\n    this.endToFront = true; // revert to default\n  }\n\n  /**\n   * Test whether the mouse from a mouse event is inside the visible window,\n   * between the current start and end date\n   * @param {Object} event\n   * @return {boolean} Returns true when inside the visible window\n   * @private\n   */\n  _isInsideRange(event) {\n    // calculate the time where the mouse is, check whether inside\n    // and no scroll action should happen.\n    const clientX = event.center ? event.center.x : event.clientX;\n    const centerContainerRect = this.body.dom.centerContainer.getBoundingClientRect();\n    const x = this.options.rtl ? clientX - centerContainerRect.left : centerContainerRect.right - clientX;\n    const time = this.body.util.toTime(x);\n\n    return time >= this.start && time <= this.end;\n  }\n\n  /**\n   * Helper function to calculate the center date for zooming\n   * @param {{x: number, y: number}} pointer\n   * @return {number} date\n   * @private\n   */\n  _pointerToDate(pointer) {\n    let conversion;\n    const direction = this.options.direction;\n\n    validateDirection(direction);\n\n    if (direction == 'horizontal') {\n      return this.body.util.toTime(pointer.x).valueOf();\n    }\n    else {\n      const height = this.body.domProps.center.height;\n      conversion = this.conversion(height);\n      return pointer.y / conversion.scale + conversion.offset;\n    }\n  }\n\n  /**\n   * Get the pointer location relative to the location of the dom element\n   * @param {{x: number, y: number}} touch\n   * @param {Element} element   HTML DOM element\n   * @return {{x: number, y: number}} pointer\n   * @private\n   */\n  getPointer(touch, element) {\n    const elementRect = element.getBoundingClientRect();\n    if (this.options.rtl) {\n      return {\n        x: elementRect.right - touch.x,\n        y: touch.y - elementRect.top\n      };\n    } else {\n      return {\n        x: touch.x - elementRect.left,\n        y: touch.y - elementRect.top\n      };\n    }\n  }\n\n  /**\n   * Zoom the range the given scale in or out. Start and end date will\n   * be adjusted, and the timeline will be redrawn. You can optionally give a\n   * date around which to zoom.\n   * For example, try scale = 0.9 or 1.1\n   * @param {number} scale      Scaling factor. Values above 1 will zoom out,\n   *                            values below 1 will zoom in.\n   * @param {number} [center]   Value representing a date around which will\n   *                            be zoomed.\n   * @param {number} delta\n   * @param {Event} event\n   */\n  zoom(scale, center, delta, event) {\n    // if centerDate is not provided, take it half between start Date and end Date\n    if (center == null) {\n      center = (this.start + this.end) / 2;\n    }\n\n    const hiddenDuration = DateUtil.getHiddenDurationBetween(this.body.hiddenDates, this.start, this.end);\n    const hiddenDurationBefore = DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this, center);\n    const hiddenDurationAfter = hiddenDuration - hiddenDurationBefore;\n\n    // calculate new start and end\n    let newStart = (center-hiddenDurationBefore) + (this.start - (center-hiddenDurationBefore)) * scale;\n    let newEnd   = (center+hiddenDurationAfter) + (this.end - (center+hiddenDurationAfter)) * scale;\n\n    // snapping times away from hidden zones\n    this.startToFront = delta > 0 ? false : true; // used to do the right autocorrection with periodic hidden times\n    this.endToFront = -delta  > 0 ? false : true; // used to do the right autocorrection with periodic hidden times\n    const safeStart = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newStart, delta, true);\n    const safeEnd = DateUtil.snapAwayFromHidden(this.body.hiddenDates, newEnd, -delta, true);\n    if (safeStart != newStart || safeEnd != newEnd) {\n      newStart = safeStart;\n      newEnd = safeEnd;\n    }\n\n    const options = {\n      animation: false,\n      byUser: true,\n      event\n    };\n    this.setRange(newStart, newEnd, options);\n\n    this.startToFront = false; // revert to default\n    this.endToFront = true; // revert to default\n  }\n\n  /**\n   * Move the range with a given delta to the left or right. Start and end\n   * value will be adjusted. For example, try delta = 0.1 or -0.1\n   * @param {number}  delta     Moving amount. Positive value will move right,\n   *                            negative value will move left\n   */\n  move(delta) {\n    // zoom start Date and end Date relative to the centerDate\n    const diff = (this.end - this.start);\n\n    // apply new values\n    const newStart = this.start + diff * delta;\n    const newEnd = this.end + diff * delta;\n\n    // TODO: reckon with min and max range\n\n    this.start = newStart;\n    this.end = newEnd;\n  }\n\n  /**\n   * Move the range to a new center point\n   * @param {number} moveTo      New center point of the range\n   */\n  moveTo(moveTo) {\n    const center = (this.start + this.end) / 2;\n\n    const diff = center - moveTo;\n\n    // calculate new start and end\n    const newStart = this.start - diff;\n    const newEnd = this.end - diff;\n\n    const options = {\n      animation: false,\n      byUser: true,\n      event: null\n    };\n    this.setRange(newStart, newEnd, options);\n  }\n}\n\n/**\n * Test whether direction has a valid value\n * @param {string} direction    'horizontal' or 'vertical'\n */\nfunction validateDirection (direction) {\n  if (direction != 'horizontal' && direction != 'vertical') {\n    throw new TypeError(`Unknown direction \"${direction}\". Choose \"horizontal\" or \"vertical\".`);\n  }\n}\n","require('../modules/web.timers');\nvar path = require('../internals/path');\n\nmodule.exports = path.setInterval;\n","\r\n/**\r\n * Expose `Emitter`.\r\n */\r\n\r\nif (typeof module !== 'undefined') {\r\n  module.exports = Emitter;\r\n}\r\n\r\n/**\r\n * Initialize a new `Emitter`.\r\n *\r\n * @api public\r\n */\r\n\r\nfunction Emitter(obj) {\r\n  if (obj) return mixin(obj);\r\n};\r\n\r\n/**\r\n * Mixin the emitter properties.\r\n *\r\n * @param {Object} obj\r\n * @return {Object}\r\n * @api private\r\n */\r\n\r\nfunction mixin(obj) {\r\n  for (var key in Emitter.prototype) {\r\n    obj[key] = Emitter.prototype[key];\r\n  }\r\n  return obj;\r\n}\r\n\r\n/**\r\n * Listen on the given `event` with `fn`.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.on =\r\nEmitter.prototype.addEventListener = function(event, fn){\r\n  this._callbacks = this._callbacks || {};\r\n  (this._callbacks['$' + event] = this._callbacks['$' + event] || [])\r\n    .push(fn);\r\n  return this;\r\n};\r\n\r\n/**\r\n * Adds an `event` listener that will be invoked a single\r\n * time then automatically removed.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.once = function(event, fn){\r\n  function on() {\r\n    this.off(event, on);\r\n    fn.apply(this, arguments);\r\n  }\r\n\r\n  on.fn = fn;\r\n  this.on(event, on);\r\n  return this;\r\n};\r\n\r\n/**\r\n * Remove the given callback for `event` or all\r\n * registered callbacks.\r\n *\r\n * @param {String} event\r\n * @param {Function} fn\r\n * @return {Emitter}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.off =\r\nEmitter.prototype.removeListener =\r\nEmitter.prototype.removeAllListeners =\r\nEmitter.prototype.removeEventListener = function(event, fn){\r\n  this._callbacks = this._callbacks || {};\r\n\r\n  // all\r\n  if (0 == arguments.length) {\r\n    this._callbacks = {};\r\n    return this;\r\n  }\r\n\r\n  // specific event\r\n  var callbacks = this._callbacks['$' + event];\r\n  if (!callbacks) return this;\r\n\r\n  // remove all handlers\r\n  if (1 == arguments.length) {\r\n    delete this._callbacks['$' + event];\r\n    return this;\r\n  }\r\n\r\n  // remove specific handler\r\n  var cb;\r\n  for (var i = 0; i < callbacks.length; i++) {\r\n    cb = callbacks[i];\r\n    if (cb === fn || cb.fn === fn) {\r\n      callbacks.splice(i, 1);\r\n      break;\r\n    }\r\n  }\r\n\r\n  // Remove event specific arrays for event types that no\r\n  // one is subscribed for to avoid memory leak.\r\n  if (callbacks.length === 0) {\r\n    delete this._callbacks['$' + event];\r\n  }\r\n\r\n  return this;\r\n};\r\n\r\n/**\r\n * Emit `event` with the given args.\r\n *\r\n * @param {String} event\r\n * @param {Mixed} ...\r\n * @return {Emitter}\r\n */\r\n\r\nEmitter.prototype.emit = function(event){\r\n  this._callbacks = this._callbacks || {};\r\n\r\n  var args = new Array(arguments.length - 1)\r\n    , callbacks = this._callbacks['$' + event];\r\n\r\n  for (var i = 1; i < arguments.length; i++) {\r\n    args[i - 1] = arguments[i];\r\n  }\r\n\r\n  if (callbacks) {\r\n    callbacks = callbacks.slice(0);\r\n    for (var i = 0, len = callbacks.length; i < len; ++i) {\r\n      callbacks[i].apply(this, args);\r\n    }\r\n  }\r\n\r\n  return this;\r\n};\r\n\r\n/**\r\n * Return array of callbacks for `event`.\r\n *\r\n * @param {String} event\r\n * @return {Array}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.listeners = function(event){\r\n  this._callbacks = this._callbacks || {};\r\n  return this._callbacks['$' + event] || [];\r\n};\r\n\r\n/**\r\n * Check if this emitter has `event` handlers.\r\n *\r\n * @param {String} event\r\n * @return {Boolean}\r\n * @api public\r\n */\r\n\r\nEmitter.prototype.hasListeners = function(event){\r\n  return !! this.listeners(event).length;\r\n};\r\n","'use strict';\n\n(function (factory) {\n  if (typeof define === 'function' && define.amd) {\n    // AMD. Register as an anonymous module.\n    define([], factory);\n  } else if (typeof exports === 'object') {\n    // Node. Does not work with strict CommonJS, but\n    // only CommonJS-like environments that support module.exports,\n    // like Node.\n    module.exports = factory();\n  } else {\n    // Browser globals (root is window)\n    window.propagating = factory();\n  }\n}(function () {\n  var _firstTarget = null; // singleton, will contain the target element where the touch event started\n\n  /**\n   * Extend an Hammer.js instance with event propagation.\n   *\n   * Features:\n   * - Events emitted by hammer will propagate in order from child to parent\n   *   elements.\n   * - Events are extended with a function `event.stopPropagation()` to stop\n   *   propagation to parent elements.\n   * - An option `preventDefault` to stop all default browser behavior.\n   *\n   * Usage:\n   *   var hammer = propagatingHammer(new Hammer(element));\n   *   var hammer = propagatingHammer(new Hammer(element), {preventDefault: true});\n   *\n   * @param {Hammer.Manager} hammer   An hammer instance.\n   * @param {Object} [options]        Available options:\n   *                                  - `preventDefault: true | false | 'mouse' | 'touch' | 'pen'`.\n   *                                    Enforce preventing the default browser behavior.\n   *                                    Cannot be set to `false`.\n   * @return {Hammer.Manager} Returns the same hammer instance with extended\n   *                          functionality\n   */\n  return function propagating(hammer, options) {\n    var _options = options || {\n      preventDefault: false\n    };\n\n    if (hammer.Manager) {\n      // This looks like the Hammer constructor.\n      // Overload the constructors with our own.\n      var Hammer = hammer;\n\n      var PropagatingHammer = function(element, options) {\n        var o = Object.create(_options);\n        if (options) Hammer.assign(o, options);\n        return propagating(new Hammer(element, o), o);\n      };\n      Hammer.assign(PropagatingHammer, Hammer);\n\n      PropagatingHammer.Manager = function (element, options) {\n        var o = Object.create(_options);\n        if (options) Hammer.assign(o, options);\n        return propagating(new Hammer.Manager(element, o), o);\n      };\n\n      return PropagatingHammer;\n    }\n\n    // create a wrapper object which will override the functions\n    // `on`, `off`, `destroy`, and `emit` of the hammer instance\n    var wrapper = Object.create(hammer);\n\n    // attach to DOM element\n    var element = hammer.element;\n\n    if(!element.hammer) element.hammer = [];\n    element.hammer.push(wrapper);\n\n    // register an event to catch the start of a gesture and store the\n    // target in a singleton\n    hammer.on('hammer.input', function (event) {\n      if (_options.preventDefault === true || (_options.preventDefault === event.pointerType)) {\n        event.preventDefault();\n      }\n      if (event.isFirst) {\n        _firstTarget = event.target;\n      }\n    });\n\n    /** @type {Object.<String, Array.<function>>} */\n    wrapper._handlers = {};\n\n    /**\n     * Register a handler for one or multiple events\n     * @param {String} events    A space separated string with events\n     * @param {function} handler A callback function, called as handler(event)\n     * @returns {Hammer.Manager} Returns the hammer instance\n     */\n    wrapper.on = function (events, handler) {\n      // register the handler\n      split(events).forEach(function (event) {\n        var _handlers = wrapper._handlers[event];\n        if (!_handlers) {\n          wrapper._handlers[event] = _handlers = [];\n\n          // register the static, propagated handler\n          hammer.on(event, propagatedHandler);\n        }\n        _handlers.push(handler);\n      });\n\n      return wrapper;\n    };\n\n    /**\n     * Unregister a handler for one or multiple events\n     * @param {String} events      A space separated string with events\n     * @param {function} [handler] Optional. The registered handler. If not\n     *                             provided, all handlers for given events\n     *                             are removed.\n     * @returns {Hammer.Manager}   Returns the hammer instance\n     */\n    wrapper.off = function (events, handler) {\n      // unregister the handler\n      split(events).forEach(function (event) {\n        var _handlers = wrapper._handlers[event];\n        if (_handlers) {\n          _handlers = handler ? _handlers.filter(function (h) {\n            return h !== handler;\n          }) : [];\n\n          if (_handlers.length > 0) {\n            wrapper._handlers[event] = _handlers;\n          }\n          else {\n            // remove static, propagated handler\n            hammer.off(event, propagatedHandler);\n            delete wrapper._handlers[event];\n          }\n        }\n      });\n\n      return wrapper;\n    };\n\n    /**\n     * Emit to the event listeners\n     * @param {string} eventType\n     * @param {Event} event\n     */\n    wrapper.emit = function(eventType, event) {\n      _firstTarget = event.target;\n      hammer.emit(eventType, event);\n    };\n\n    wrapper.destroy = function () {\n      // Detach from DOM element\n      var hammers = hammer.element.hammer;\n      var idx = hammers.indexOf(wrapper);\n      if(idx !== -1) hammers.splice(idx,1);\n      if(!hammers.length) delete hammer.element.hammer;\n\n      // clear all handlers\n      wrapper._handlers = {};\n\n      // call original hammer destroy\n      hammer.destroy();\n    };\n\n    // split a string with space separated words\n    function split(events) {\n      return events.match(/[^ ]+/g);\n    }\n\n    /**\n     * A static event handler, applying event propagation.\n     * @param {Object} event\n     */\n    function propagatedHandler(event) {\n      // let only a single hammer instance handle this event\n      if (event.type !== 'hammer.input') {\n        // it is possible that the same srcEvent is used with multiple hammer events,\n        // we keep track on which events are handled in an object _handled\n        if (!event.srcEvent._handled) {\n          event.srcEvent._handled = {};\n        }\n\n        if (event.srcEvent._handled[event.type]) {\n          return;\n        }\n        else {\n          event.srcEvent._handled[event.type] = true;\n        }\n      }\n\n      // attach a stopPropagation function to the event\n      var stopped = false;\n      event.stopPropagation = function () {\n        stopped = true;\n      };\n\n      //wrap the srcEvent's stopPropagation to also stop hammer propagation:\n      var srcStop = event.srcEvent.stopPropagation.bind(event.srcEvent);\n      if(typeof srcStop == \"function\") {\n        event.srcEvent.stopPropagation = function(){\n          srcStop();\n          event.stopPropagation();\n        }\n      }\n\n      // attach firstTarget property to the event\n      event.firstTarget = _firstTarget;\n\n      // propagate over all elements (until stopped)\n      var elem = _firstTarget;\n      while (elem && !stopped) {\n        var elemHammer = elem.hammer;\n        if(elemHammer){\n          var _handlers;\n          for(var k = 0; k < elemHammer.length; k++){\n            _handlers = elemHammer[k]._handlers[event.type];\n            if(_handlers) for (var i = 0; i < _handlers.length && !stopped; i++) {\n              _handlers[i](event);\n            }\n          }\n        }\n        elem = elem.parentNode;\n      }\n    }\n\n    return wrapper;\n  };\n}));\n","/*! Hammer.JS - v2.0.17-rc - 2019-12-16\n * http://naver.github.io/egjs\n *\n * Forked By Naver egjs\n * Copyright (c) hammerjs\n * Licensed under the MIT license */\nfunction _extends() {\n  _extends = Object.assign || function (target) {\n    for (var i = 1; i < arguments.length; i++) {\n      var source = arguments[i];\n\n      for (var key in source) {\n        if (Object.prototype.hasOwnProperty.call(source, key)) {\n          target[key] = source[key];\n        }\n      }\n    }\n\n    return target;\n  };\n\n  return _extends.apply(this, arguments);\n}\n\nfunction _inheritsLoose(subClass, superClass) {\n  subClass.prototype = Object.create(superClass.prototype);\n  subClass.prototype.constructor = subClass;\n  subClass.__proto__ = superClass;\n}\n\nfunction _assertThisInitialized(self) {\n  if (self === void 0) {\n    throw new ReferenceError(\"this hasn't been initialised - super() hasn't been called\");\n  }\n\n  return self;\n}\n\n/**\n * @private\n * extend object.\n * means that properties in dest will be overwritten by the ones in src.\n * @param {Object} target\n * @param {...Object} objects_to_assign\n * @returns {Object} target\n */\nvar assign;\n\nif (typeof Object.assign !== 'function') {\n  assign = function assign(target) {\n    if (target === undefined || target === null) {\n      throw new TypeError('Cannot convert undefined or null to object');\n    }\n\n    var output = Object(target);\n\n    for (var index = 1; index < arguments.length; index++) {\n      var source = arguments[index];\n\n      if (source !== undefined && source !== null) {\n        for (var nextKey in source) {\n          if (source.hasOwnProperty(nextKey)) {\n            output[nextKey] = source[nextKey];\n          }\n        }\n      }\n    }\n\n    return output;\n  };\n} else {\n  assign = Object.assign;\n}\n\nvar assign$1 = assign;\n\nvar VENDOR_PREFIXES = ['', 'webkit', 'Moz', 'MS', 'ms', 'o'];\nvar TEST_ELEMENT = typeof document === \"undefined\" ? {\n  style: {}\n} : document.createElement('div');\nvar TYPE_FUNCTION = 'function';\nvar round = Math.round,\n    abs = Math.abs;\nvar now = Date.now;\n\n/**\n * @private\n * get the prefixed property\n * @param {Object} obj\n * @param {String} property\n * @returns {String|Undefined} prefixed\n */\n\nfunction prefixed(obj, property) {\n  var prefix;\n  var prop;\n  var camelProp = property[0].toUpperCase() + property.slice(1);\n  var i = 0;\n\n  while (i < VENDOR_PREFIXES.length) {\n    prefix = VENDOR_PREFIXES[i];\n    prop = prefix ? prefix + camelProp : property;\n\n    if (prop in obj) {\n      return prop;\n    }\n\n    i++;\n  }\n\n  return undefined;\n}\n\n/* eslint-disable no-new-func, no-nested-ternary */\nvar win;\n\nif (typeof window === \"undefined\") {\n  // window is undefined in node.js\n  win = {};\n} else {\n  win = window;\n}\n\nvar PREFIXED_TOUCH_ACTION = prefixed(TEST_ELEMENT.style, 'touchAction');\nvar NATIVE_TOUCH_ACTION = PREFIXED_TOUCH_ACTION !== undefined;\nfunction getTouchActionProps() {\n  if (!NATIVE_TOUCH_ACTION) {\n    return false;\n  }\n\n  var touchMap = {};\n  var cssSupports = win.CSS && win.CSS.supports;\n  ['auto', 'manipulation', 'pan-y', 'pan-x', 'pan-x pan-y', 'none'].forEach(function (val) {\n    // If css.supports is not supported but there is native touch-action assume it supports\n    // all values. This is the case for IE 10 and 11.\n    return touchMap[val] = cssSupports ? win.CSS.supports('touch-action', val) : true;\n  });\n  return touchMap;\n}\n\nvar TOUCH_ACTION_COMPUTE = 'compute';\nvar TOUCH_ACTION_AUTO = 'auto';\nvar TOUCH_ACTION_MANIPULATION = 'manipulation'; // not implemented\n\nvar TOUCH_ACTION_NONE = 'none';\nvar TOUCH_ACTION_PAN_X = 'pan-x';\nvar TOUCH_ACTION_PAN_Y = 'pan-y';\nvar TOUCH_ACTION_MAP = getTouchActionProps();\n\nvar MOBILE_REGEX = /mobile|tablet|ip(ad|hone|od)|android/i;\nvar SUPPORT_TOUCH = 'ontouchstart' in win;\nvar SUPPORT_POINTER_EVENTS = prefixed(win, 'PointerEvent') !== undefined;\nvar SUPPORT_ONLY_TOUCH = SUPPORT_TOUCH && MOBILE_REGEX.test(navigator.userAgent);\nvar INPUT_TYPE_TOUCH = 'touch';\nvar INPUT_TYPE_PEN = 'pen';\nvar INPUT_TYPE_MOUSE = 'mouse';\nvar INPUT_TYPE_KINECT = 'kinect';\nvar COMPUTE_INTERVAL = 25;\nvar INPUT_START = 1;\nvar INPUT_MOVE = 2;\nvar INPUT_END = 4;\nvar INPUT_CANCEL = 8;\nvar DIRECTION_NONE = 1;\nvar DIRECTION_LEFT = 2;\nvar DIRECTION_RIGHT = 4;\nvar DIRECTION_UP = 8;\nvar DIRECTION_DOWN = 16;\nvar DIRECTION_HORIZONTAL = DIRECTION_LEFT | DIRECTION_RIGHT;\nvar DIRECTION_VERTICAL = DIRECTION_UP | DIRECTION_DOWN;\nvar DIRECTION_ALL = DIRECTION_HORIZONTAL | DIRECTION_VERTICAL;\nvar PROPS_XY = ['x', 'y'];\nvar PROPS_CLIENT_XY = ['clientX', 'clientY'];\n\n/**\n * @private\n * walk objects and arrays\n * @param {Object} obj\n * @param {Function} iterator\n * @param {Object} context\n */\nfunction each(obj, iterator, context) {\n  var i;\n\n  if (!obj) {\n    return;\n  }\n\n  if (obj.forEach) {\n    obj.forEach(iterator, context);\n  } else if (obj.length !== undefined) {\n    i = 0;\n\n    while (i < obj.length) {\n      iterator.call(context, obj[i], i, obj);\n      i++;\n    }\n  } else {\n    for (i in obj) {\n      obj.hasOwnProperty(i) && iterator.call(context, obj[i], i, obj);\n    }\n  }\n}\n\n/**\n * @private\n * let a boolean value also be a function that must return a boolean\n * this first item in args will be used as the context\n * @param {Boolean|Function} val\n * @param {Array} [args]\n * @returns {Boolean}\n */\n\nfunction boolOrFn(val, args) {\n  if (typeof val === TYPE_FUNCTION) {\n    return val.apply(args ? args[0] || undefined : undefined, args);\n  }\n\n  return val;\n}\n\n/**\n * @private\n * small indexOf wrapper\n * @param {String} str\n * @param {String} find\n * @returns {Boolean} found\n */\nfunction inStr(str, find) {\n  return str.indexOf(find) > -1;\n}\n\n/**\n * @private\n * when the touchActions are collected they are not a valid value, so we need to clean things up. *\n * @param {String} actions\n * @returns {*}\n */\n\nfunction cleanTouchActions(actions) {\n  // none\n  if (inStr(actions, TOUCH_ACTION_NONE)) {\n    return TOUCH_ACTION_NONE;\n  }\n\n  var hasPanX = inStr(actions, TOUCH_ACTION_PAN_X);\n  var hasPanY = inStr(actions, TOUCH_ACTION_PAN_Y); // if both pan-x and pan-y are set (different recognizers\n  // for different directions, e.g. horizontal pan but vertical swipe?)\n  // we need none (as otherwise with pan-x pan-y combined none of these\n  // recognizers will work, since the browser would handle all panning\n\n  if (hasPanX && hasPanY) {\n    return TOUCH_ACTION_NONE;\n  } // pan-x OR pan-y\n\n\n  if (hasPanX || hasPanY) {\n    return hasPanX ? TOUCH_ACTION_PAN_X : TOUCH_ACTION_PAN_Y;\n  } // manipulation\n\n\n  if (inStr(actions, TOUCH_ACTION_MANIPULATION)) {\n    return TOUCH_ACTION_MANIPULATION;\n  }\n\n  return TOUCH_ACTION_AUTO;\n}\n\n/**\n * @private\n * Touch Action\n * sets the touchAction property or uses the js alternative\n * @param {Manager} manager\n * @param {String} value\n * @constructor\n */\n\nvar TouchAction =\n/*#__PURE__*/\nfunction () {\n  function TouchAction(manager, value) {\n    this.manager = manager;\n    this.set(value);\n  }\n  /**\n   * @private\n   * set the touchAction value on the element or enable the polyfill\n   * @param {String} value\n   */\n\n\n  var _proto = TouchAction.prototype;\n\n  _proto.set = function set(value) {\n    // find out the touch-action by the event handlers\n    if (value === TOUCH_ACTION_COMPUTE) {\n      value = this.compute();\n    }\n\n    if (NATIVE_TOUCH_ACTION && this.manager.element.style && TOUCH_ACTION_MAP[value]) {\n      this.manager.element.style[PREFIXED_TOUCH_ACTION] = value;\n    }\n\n    this.actions = value.toLowerCase().trim();\n  };\n  /**\n   * @private\n   * just re-set the touchAction value\n   */\n\n\n  _proto.update = function update() {\n    this.set(this.manager.options.touchAction);\n  };\n  /**\n   * @private\n   * compute the value for the touchAction property based on the recognizer's settings\n   * @returns {String} value\n   */\n\n\n  _proto.compute = function compute() {\n    var actions = [];\n    each(this.manager.recognizers, function (recognizer) {\n      if (boolOrFn(recognizer.options.enable, [recognizer])) {\n        actions = actions.concat(recognizer.getTouchAction());\n      }\n    });\n    return cleanTouchActions(actions.join(' '));\n  };\n  /**\n   * @private\n   * this method is called on each input cycle and provides the preventing of the browser behavior\n   * @param {Object} input\n   */\n\n\n  _proto.preventDefaults = function preventDefaults(input) {\n    var srcEvent = input.srcEvent;\n    var direction = input.offsetDirection; // if the touch action did prevented once this session\n\n    if (this.manager.session.prevented) {\n      srcEvent.preventDefault();\n      return;\n    }\n\n    var actions = this.actions;\n    var hasNone = inStr(actions, TOUCH_ACTION_NONE) && !TOUCH_ACTION_MAP[TOUCH_ACTION_NONE];\n    var hasPanY = inStr(actions, TOUCH_ACTION_PAN_Y) && !TOUCH_ACTION_MAP[TOUCH_ACTION_PAN_Y];\n    var hasPanX = inStr(actions, TOUCH_ACTION_PAN_X) && !TOUCH_ACTION_MAP[TOUCH_ACTION_PAN_X];\n\n    if (hasNone) {\n      // do not prevent defaults if this is a tap gesture\n      var isTapPointer = input.pointers.length === 1;\n      var isTapMovement = input.distance < 2;\n      var isTapTouchTime = input.deltaTime < 250;\n\n      if (isTapPointer && isTapMovement && isTapTouchTime) {\n        return;\n      }\n    }\n\n    if (hasPanX && hasPanY) {\n      // `pan-x pan-y` means browser handles all scrolling/panning, do not prevent\n      return;\n    }\n\n    if (hasNone || hasPanY && direction & DIRECTION_HORIZONTAL || hasPanX && direction & DIRECTION_VERTICAL) {\n      return this.preventSrc(srcEvent);\n    }\n  };\n  /**\n   * @private\n   * call preventDefault to prevent the browser's default behavior (scrolling in most cases)\n   * @param {Object} srcEvent\n   */\n\n\n  _proto.preventSrc = function preventSrc(srcEvent) {\n    this.manager.session.prevented = true;\n    srcEvent.preventDefault();\n  };\n\n  return TouchAction;\n}();\n\n/**\n * @private\n * find if a node is in the given parent\n * @method hasParent\n * @param {HTMLElement} node\n * @param {HTMLElement} parent\n * @return {Boolean} found\n */\nfunction hasParent(node, parent) {\n  while (node) {\n    if (node === parent) {\n      return true;\n    }\n\n    node = node.parentNode;\n  }\n\n  return false;\n}\n\n/**\n * @private\n * get the center of all the pointers\n * @param {Array} pointers\n * @return {Object} center contains `x` and `y` properties\n */\n\nfunction getCenter(pointers) {\n  var pointersLength = pointers.length; // no need to loop when only one touch\n\n  if (pointersLength === 1) {\n    return {\n      x: round(pointers[0].clientX),\n      y: round(pointers[0].clientY)\n    };\n  }\n\n  var x = 0;\n  var y = 0;\n  var i = 0;\n\n  while (i < pointersLength) {\n    x += pointers[i].clientX;\n    y += pointers[i].clientY;\n    i++;\n  }\n\n  return {\n    x: round(x / pointersLength),\n    y: round(y / pointersLength)\n  };\n}\n\n/**\n * @private\n * create a simple clone from the input used for storage of firstInput and firstMultiple\n * @param {Object} input\n * @returns {Object} clonedInputData\n */\n\nfunction simpleCloneInputData(input) {\n  // make a simple copy of the pointers because we will get a reference if we don't\n  // we only need clientXY for the calculations\n  var pointers = [];\n  var i = 0;\n\n  while (i < input.pointers.length) {\n    pointers[i] = {\n      clientX: round(input.pointers[i].clientX),\n      clientY: round(input.pointers[i].clientY)\n    };\n    i++;\n  }\n\n  return {\n    timeStamp: now(),\n    pointers: pointers,\n    center: getCenter(pointers),\n    deltaX: input.deltaX,\n    deltaY: input.deltaY\n  };\n}\n\n/**\n * @private\n * calculate the absolute distance between two points\n * @param {Object} p1 {x, y}\n * @param {Object} p2 {x, y}\n * @param {Array} [props] containing x and y keys\n * @return {Number} distance\n */\n\nfunction getDistance(p1, p2, props) {\n  if (!props) {\n    props = PROPS_XY;\n  }\n\n  var x = p2[props[0]] - p1[props[0]];\n  var y = p2[props[1]] - p1[props[1]];\n  return Math.sqrt(x * x + y * y);\n}\n\n/**\n * @private\n * calculate the angle between two coordinates\n * @param {Object} p1\n * @param {Object} p2\n * @param {Array} [props] containing x and y keys\n * @return {Number} angle\n */\n\nfunction getAngle(p1, p2, props) {\n  if (!props) {\n    props = PROPS_XY;\n  }\n\n  var x = p2[props[0]] - p1[props[0]];\n  var y = p2[props[1]] - p1[props[1]];\n  return Math.atan2(y, x) * 180 / Math.PI;\n}\n\n/**\n * @private\n * get the direction between two points\n * @param {Number} x\n * @param {Number} y\n * @return {Number} direction\n */\n\nfunction getDirection(x, y) {\n  if (x === y) {\n    return DIRECTION_NONE;\n  }\n\n  if (abs(x) >= abs(y)) {\n    return x < 0 ? DIRECTION_LEFT : DIRECTION_RIGHT;\n  }\n\n  return y < 0 ? DIRECTION_UP : DIRECTION_DOWN;\n}\n\nfunction computeDeltaXY(session, input) {\n  var center = input.center; // let { offsetDelta:offset = {}, prevDelta = {}, prevInput = {} } = session;\n  // jscs throwing error on defalut destructured values and without defaults tests fail\n\n  var offset = session.offsetDelta || {};\n  var prevDelta = session.prevDelta || {};\n  var prevInput = session.prevInput || {};\n\n  if (input.eventType === INPUT_START || prevInput.eventType === INPUT_END) {\n    prevDelta = session.prevDelta = {\n      x: prevInput.deltaX || 0,\n      y: prevInput.deltaY || 0\n    };\n    offset = session.offsetDelta = {\n      x: center.x,\n      y: center.y\n    };\n  }\n\n  input.deltaX = prevDelta.x + (center.x - offset.x);\n  input.deltaY = prevDelta.y + (center.y - offset.y);\n}\n\n/**\n * @private\n * calculate the velocity between two points. unit is in px per ms.\n * @param {Number} deltaTime\n * @param {Number} x\n * @param {Number} y\n * @return {Object} velocity `x` and `y`\n */\nfunction getVelocity(deltaTime, x, y) {\n  return {\n    x: x / deltaTime || 0,\n    y: y / deltaTime || 0\n  };\n}\n\n/**\n * @private\n * calculate the scale factor between two pointersets\n * no scale is 1, and goes down to 0 when pinched together, and bigger when pinched out\n * @param {Array} start array of pointers\n * @param {Array} end array of pointers\n * @return {Number} scale\n */\n\nfunction getScale(start, end) {\n  return getDistance(end[0], end[1], PROPS_CLIENT_XY) / getDistance(start[0], start[1], PROPS_CLIENT_XY);\n}\n\n/**\n * @private\n * calculate the rotation degrees between two pointersets\n * @param {Array} start array of pointers\n * @param {Array} end array of pointers\n * @return {Number} rotation\n */\n\nfunction getRotation(start, end) {\n  return getAngle(end[1], end[0], PROPS_CLIENT_XY) + getAngle(start[1], start[0], PROPS_CLIENT_XY);\n}\n\n/**\n * @private\n * velocity is calculated every x ms\n * @param {Object} session\n * @param {Object} input\n */\n\nfunction computeIntervalInputData(session, input) {\n  var last = session.lastInterval || input;\n  var deltaTime = input.timeStamp - last.timeStamp;\n  var velocity;\n  var velocityX;\n  var velocityY;\n  var direction;\n\n  if (input.eventType !== INPUT_CANCEL && (deltaTime > COMPUTE_INTERVAL || last.velocity === undefined)) {\n    var deltaX = input.deltaX - last.deltaX;\n    var deltaY = input.deltaY - last.deltaY;\n    var v = getVelocity(deltaTime, deltaX, deltaY);\n    velocityX = v.x;\n    velocityY = v.y;\n    velocity = abs(v.x) > abs(v.y) ? v.x : v.y;\n    direction = getDirection(deltaX, deltaY);\n    session.lastInterval = input;\n  } else {\n    // use latest velocity info if it doesn't overtake a minimum period\n    velocity = last.velocity;\n    velocityX = last.velocityX;\n    velocityY = last.velocityY;\n    direction = last.direction;\n  }\n\n  input.velocity = velocity;\n  input.velocityX = velocityX;\n  input.velocityY = velocityY;\n  input.direction = direction;\n}\n\n/**\n* @private\n * extend the data with some usable properties like scale, rotate, velocity etc\n * @param {Object} manager\n * @param {Object} input\n */\n\nfunction computeInputData(manager, input) {\n  var session = manager.session;\n  var pointers = input.pointers;\n  var pointersLength = pointers.length; // store the first input to calculate the distance and direction\n\n  if (!session.firstInput) {\n    session.firstInput = simpleCloneInputData(input);\n  } // to compute scale and rotation we need to store the multiple touches\n\n\n  if (pointersLength > 1 && !session.firstMultiple) {\n    session.firstMultiple = simpleCloneInputData(input);\n  } else if (pointersLength === 1) {\n    session.firstMultiple = false;\n  }\n\n  var firstInput = session.firstInput,\n      firstMultiple = session.firstMultiple;\n  var offsetCenter = firstMultiple ? firstMultiple.center : firstInput.center;\n  var center = input.center = getCenter(pointers);\n  input.timeStamp = now();\n  input.deltaTime = input.timeStamp - firstInput.timeStamp;\n  input.angle = getAngle(offsetCenter, center);\n  input.distance = getDistance(offsetCenter, center);\n  computeDeltaXY(session, input);\n  input.offsetDirection = getDirection(input.deltaX, input.deltaY);\n  var overallVelocity = getVelocity(input.deltaTime, input.deltaX, input.deltaY);\n  input.overallVelocityX = overallVelocity.x;\n  input.overallVelocityY = overallVelocity.y;\n  input.overallVelocity = abs(overallVelocity.x) > abs(overallVelocity.y) ? overallVelocity.x : overallVelocity.y;\n  input.scale = firstMultiple ? getScale(firstMultiple.pointers, pointers) : 1;\n  input.rotation = firstMultiple ? getRotation(firstMultiple.pointers, pointers) : 0;\n  input.maxPointers = !session.prevInput ? input.pointers.length : input.pointers.length > session.prevInput.maxPointers ? input.pointers.length : session.prevInput.maxPointers;\n  computeIntervalInputData(session, input); // find the correct target\n\n  var target = manager.element;\n  var srcEvent = input.srcEvent;\n  var srcEventTarget;\n\n  if (srcEvent.composedPath) {\n    srcEventTarget = srcEvent.composedPath()[0];\n  } else if (srcEvent.path) {\n    srcEventTarget = srcEvent.path[0];\n  } else {\n    srcEventTarget = srcEvent.target;\n  }\n\n  if (hasParent(srcEventTarget, target)) {\n    target = srcEventTarget;\n  }\n\n  input.target = target;\n}\n\n/**\n * @private\n * handle input events\n * @param {Manager} manager\n * @param {String} eventType\n * @param {Object} input\n */\n\nfunction inputHandler(manager, eventType, input) {\n  var pointersLen = input.pointers.length;\n  var changedPointersLen = input.changedPointers.length;\n  var isFirst = eventType & INPUT_START && pointersLen - changedPointersLen === 0;\n  var isFinal = eventType & (INPUT_END | INPUT_CANCEL) && pointersLen - changedPointersLen === 0;\n  input.isFirst = !!isFirst;\n  input.isFinal = !!isFinal;\n\n  if (isFirst) {\n    manager.session = {};\n  } // source event is the normalized value of the domEvents\n  // like 'touchstart, mouseup, pointerdown'\n\n\n  input.eventType = eventType; // compute scale, rotation etc\n\n  computeInputData(manager, input); // emit secret event\n\n  manager.emit('hammer.input', input);\n  manager.recognize(input);\n  manager.session.prevInput = input;\n}\n\n/**\n * @private\n * split string on whitespace\n * @param {String} str\n * @returns {Array} words\n */\nfunction splitStr(str) {\n  return str.trim().split(/\\s+/g);\n}\n\n/**\n * @private\n * addEventListener with multiple events at once\n * @param {EventTarget} target\n * @param {String} types\n * @param {Function} handler\n */\n\nfunction addEventListeners(target, types, handler) {\n  each(splitStr(types), function (type) {\n    target.addEventListener(type, handler, false);\n  });\n}\n\n/**\n * @private\n * removeEventListener with multiple events at once\n * @param {EventTarget} target\n * @param {String} types\n * @param {Function} handler\n */\n\nfunction removeEventListeners(target, types, handler) {\n  each(splitStr(types), function (type) {\n    target.removeEventListener(type, handler, false);\n  });\n}\n\n/**\n * @private\n * get the window object of an element\n * @param {HTMLElement} element\n * @returns {DocumentView|Window}\n */\nfunction getWindowForElement(element) {\n  var doc = element.ownerDocument || element;\n  return doc.defaultView || doc.parentWindow || window;\n}\n\n/**\n * @private\n * create new input type manager\n * @param {Manager} manager\n * @param {Function} callback\n * @returns {Input}\n * @constructor\n */\n\nvar Input =\n/*#__PURE__*/\nfunction () {\n  function Input(manager, callback) {\n    var self = this;\n    this.manager = manager;\n    this.callback = callback;\n    this.element = manager.element;\n    this.target = manager.options.inputTarget; // smaller wrapper around the handler, for the scope and the enabled state of the manager,\n    // so when disabled the input events are completely bypassed.\n\n    this.domHandler = function (ev) {\n      if (boolOrFn(manager.options.enable, [manager])) {\n        self.handler(ev);\n      }\n    };\n\n    this.init();\n  }\n  /**\n   * @private\n   * should handle the inputEvent data and trigger the callback\n   * @virtual\n   */\n\n\n  var _proto = Input.prototype;\n\n  _proto.handler = function handler() {};\n  /**\n   * @private\n   * bind the events\n   */\n\n\n  _proto.init = function init() {\n    this.evEl && addEventListeners(this.element, this.evEl, this.domHandler);\n    this.evTarget && addEventListeners(this.target, this.evTarget, this.domHandler);\n    this.evWin && addEventListeners(getWindowForElement(this.element), this.evWin, this.domHandler);\n  };\n  /**\n   * @private\n   * unbind the events\n   */\n\n\n  _proto.destroy = function destroy() {\n    this.evEl && removeEventListeners(this.element, this.evEl, this.domHandler);\n    this.evTarget && removeEventListeners(this.target, this.evTarget, this.domHandler);\n    this.evWin && removeEventListeners(getWindowForElement(this.element), this.evWin, this.domHandler);\n  };\n\n  return Input;\n}();\n\n/**\n * @private\n * find if a array contains the object using indexOf or a simple polyFill\n * @param {Array} src\n * @param {String} find\n * @param {String} [findByKey]\n * @return {Boolean|Number} false when not found, or the index\n */\nfunction inArray(src, find, findByKey) {\n  if (src.indexOf && !findByKey) {\n    return src.indexOf(find);\n  } else {\n    var i = 0;\n\n    while (i < src.length) {\n      if (findByKey && src[i][findByKey] == find || !findByKey && src[i] === find) {\n        // do not use === here, test fails\n        return i;\n      }\n\n      i++;\n    }\n\n    return -1;\n  }\n}\n\nvar POINTER_INPUT_MAP = {\n  pointerdown: INPUT_START,\n  pointermove: INPUT_MOVE,\n  pointerup: INPUT_END,\n  pointercancel: INPUT_CANCEL,\n  pointerout: INPUT_CANCEL\n}; // in IE10 the pointer types is defined as an enum\n\nvar IE10_POINTER_TYPE_ENUM = {\n  2: INPUT_TYPE_TOUCH,\n  3: INPUT_TYPE_PEN,\n  4: INPUT_TYPE_MOUSE,\n  5: INPUT_TYPE_KINECT // see https://twitter.com/jacobrossi/status/480596438489890816\n\n};\nvar POINTER_ELEMENT_EVENTS = 'pointerdown';\nvar POINTER_WINDOW_EVENTS = 'pointermove pointerup pointercancel'; // IE10 has prefixed support, and case-sensitive\n\nif (win.MSPointerEvent && !win.PointerEvent) {\n  POINTER_ELEMENT_EVENTS = 'MSPointerDown';\n  POINTER_WINDOW_EVENTS = 'MSPointerMove MSPointerUp MSPointerCancel';\n}\n/**\n * @private\n * Pointer events input\n * @constructor\n * @extends Input\n */\n\n\nvar PointerEventInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(PointerEventInput, _Input);\n\n  function PointerEventInput() {\n    var _this;\n\n    var proto = PointerEventInput.prototype;\n    proto.evEl = POINTER_ELEMENT_EVENTS;\n    proto.evWin = POINTER_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.store = _this.manager.session.pointerEvents = [];\n    return _this;\n  }\n  /**\n   * @private\n   * handle mouse events\n   * @param {Object} ev\n   */\n\n\n  var _proto = PointerEventInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var store = this.store;\n    var removePointer = false;\n    var eventTypeNormalized = ev.type.toLowerCase().replace('ms', '');\n    var eventType = POINTER_INPUT_MAP[eventTypeNormalized];\n    var pointerType = IE10_POINTER_TYPE_ENUM[ev.pointerType] || ev.pointerType;\n    var isTouch = pointerType === INPUT_TYPE_TOUCH; // get index of the event in the store\n\n    var storeIndex = inArray(store, ev.pointerId, 'pointerId'); // start and mouse must be down\n\n    if (eventType & INPUT_START && (ev.button === 0 || isTouch)) {\n      if (storeIndex < 0) {\n        store.push(ev);\n        storeIndex = store.length - 1;\n      }\n    } else if (eventType & (INPUT_END | INPUT_CANCEL)) {\n      removePointer = true;\n    } // it not found, so the pointer hasn't been down (so it's probably a hover)\n\n\n    if (storeIndex < 0) {\n      return;\n    } // update the event in the store\n\n\n    store[storeIndex] = ev;\n    this.callback(this.manager, eventType, {\n      pointers: store,\n      changedPointers: [ev],\n      pointerType: pointerType,\n      srcEvent: ev\n    });\n\n    if (removePointer) {\n      // remove from the store\n      store.splice(storeIndex, 1);\n    }\n  };\n\n  return PointerEventInput;\n}(Input);\n\n/**\n * @private\n * convert array-like objects to real arrays\n * @param {Object} obj\n * @returns {Array}\n */\nfunction toArray(obj) {\n  return Array.prototype.slice.call(obj, 0);\n}\n\n/**\n * @private\n * unique array with objects based on a key (like 'id') or just by the array's value\n * @param {Array} src [{id:1},{id:2},{id:1}]\n * @param {String} [key]\n * @param {Boolean} [sort=False]\n * @returns {Array} [{id:1},{id:2}]\n */\n\nfunction uniqueArray(src, key, sort) {\n  var results = [];\n  var values = [];\n  var i = 0;\n\n  while (i < src.length) {\n    var val = key ? src[i][key] : src[i];\n\n    if (inArray(values, val) < 0) {\n      results.push(src[i]);\n    }\n\n    values[i] = val;\n    i++;\n  }\n\n  if (sort) {\n    if (!key) {\n      results = results.sort();\n    } else {\n      results = results.sort(function (a, b) {\n        return a[key] > b[key];\n      });\n    }\n  }\n\n  return results;\n}\n\nvar TOUCH_INPUT_MAP = {\n  touchstart: INPUT_START,\n  touchmove: INPUT_MOVE,\n  touchend: INPUT_END,\n  touchcancel: INPUT_CANCEL\n};\nvar TOUCH_TARGET_EVENTS = 'touchstart touchmove touchend touchcancel';\n/**\n * @private\n * Multi-user touch events input\n * @constructor\n * @extends Input\n */\n\nvar TouchInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(TouchInput, _Input);\n\n  function TouchInput() {\n    var _this;\n\n    TouchInput.prototype.evTarget = TOUCH_TARGET_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.targetIds = {}; // this.evTarget = TOUCH_TARGET_EVENTS;\n\n    return _this;\n  }\n\n  var _proto = TouchInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var type = TOUCH_INPUT_MAP[ev.type];\n    var touches = getTouches.call(this, ev, type);\n\n    if (!touches) {\n      return;\n    }\n\n    this.callback(this.manager, type, {\n      pointers: touches[0],\n      changedPointers: touches[1],\n      pointerType: INPUT_TYPE_TOUCH,\n      srcEvent: ev\n    });\n  };\n\n  return TouchInput;\n}(Input);\n\nfunction getTouches(ev, type) {\n  var allTouches = toArray(ev.touches);\n  var targetIds = this.targetIds; // when there is only one touch, the process can be simplified\n\n  if (type & (INPUT_START | INPUT_MOVE) && allTouches.length === 1) {\n    targetIds[allTouches[0].identifier] = true;\n    return [allTouches, allTouches];\n  }\n\n  var i;\n  var targetTouches;\n  var changedTouches = toArray(ev.changedTouches);\n  var changedTargetTouches = [];\n  var target = this.target; // get target touches from touches\n\n  targetTouches = allTouches.filter(function (touch) {\n    return hasParent(touch.target, target);\n  }); // collect touches\n\n  if (type === INPUT_START) {\n    i = 0;\n\n    while (i < targetTouches.length) {\n      targetIds[targetTouches[i].identifier] = true;\n      i++;\n    }\n  } // filter changed touches to only contain touches that exist in the collected target ids\n\n\n  i = 0;\n\n  while (i < changedTouches.length) {\n    if (targetIds[changedTouches[i].identifier]) {\n      changedTargetTouches.push(changedTouches[i]);\n    } // cleanup removed touches\n\n\n    if (type & (INPUT_END | INPUT_CANCEL)) {\n      delete targetIds[changedTouches[i].identifier];\n    }\n\n    i++;\n  }\n\n  if (!changedTargetTouches.length) {\n    return;\n  }\n\n  return [// merge targetTouches with changedTargetTouches so it contains ALL touches, including 'end' and 'cancel'\n  uniqueArray(targetTouches.concat(changedTargetTouches), 'identifier', true), changedTargetTouches];\n}\n\nvar MOUSE_INPUT_MAP = {\n  mousedown: INPUT_START,\n  mousemove: INPUT_MOVE,\n  mouseup: INPUT_END\n};\nvar MOUSE_ELEMENT_EVENTS = 'mousedown';\nvar MOUSE_WINDOW_EVENTS = 'mousemove mouseup';\n/**\n * @private\n * Mouse events input\n * @constructor\n * @extends Input\n */\n\nvar MouseInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(MouseInput, _Input);\n\n  function MouseInput() {\n    var _this;\n\n    var proto = MouseInput.prototype;\n    proto.evEl = MOUSE_ELEMENT_EVENTS;\n    proto.evWin = MOUSE_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.pressed = false; // mousedown state\n\n    return _this;\n  }\n  /**\n   * @private\n   * handle mouse events\n   * @param {Object} ev\n   */\n\n\n  var _proto = MouseInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var eventType = MOUSE_INPUT_MAP[ev.type]; // on start we want to have the left mouse button down\n\n    if (eventType & INPUT_START && ev.button === 0) {\n      this.pressed = true;\n    }\n\n    if (eventType & INPUT_MOVE && ev.which !== 1) {\n      eventType = INPUT_END;\n    } // mouse must be down\n\n\n    if (!this.pressed) {\n      return;\n    }\n\n    if (eventType & INPUT_END) {\n      this.pressed = false;\n    }\n\n    this.callback(this.manager, eventType, {\n      pointers: [ev],\n      changedPointers: [ev],\n      pointerType: INPUT_TYPE_MOUSE,\n      srcEvent: ev\n    });\n  };\n\n  return MouseInput;\n}(Input);\n\n/**\n * @private\n * Combined touch and mouse input\n *\n * Touch has a higher priority then mouse, and while touching no mouse events are allowed.\n * This because touch devices also emit mouse events while doing a touch.\n *\n * @constructor\n * @extends Input\n */\n\nvar DEDUP_TIMEOUT = 2500;\nvar DEDUP_DISTANCE = 25;\n\nfunction setLastTouch(eventData) {\n  var _eventData$changedPoi = eventData.changedPointers,\n      touch = _eventData$changedPoi[0];\n\n  if (touch.identifier === this.primaryTouch) {\n    var lastTouch = {\n      x: touch.clientX,\n      y: touch.clientY\n    };\n    var lts = this.lastTouches;\n    this.lastTouches.push(lastTouch);\n\n    var removeLastTouch = function removeLastTouch() {\n      var i = lts.indexOf(lastTouch);\n\n      if (i > -1) {\n        lts.splice(i, 1);\n      }\n    };\n\n    setTimeout(removeLastTouch, DEDUP_TIMEOUT);\n  }\n}\n\nfunction recordTouches(eventType, eventData) {\n  if (eventType & INPUT_START) {\n    this.primaryTouch = eventData.changedPointers[0].identifier;\n    setLastTouch.call(this, eventData);\n  } else if (eventType & (INPUT_END | INPUT_CANCEL)) {\n    setLastTouch.call(this, eventData);\n  }\n}\n\nfunction isSyntheticEvent(eventData) {\n  var x = eventData.srcEvent.clientX;\n  var y = eventData.srcEvent.clientY;\n\n  for (var i = 0; i < this.lastTouches.length; i++) {\n    var t = this.lastTouches[i];\n    var dx = Math.abs(x - t.x);\n    var dy = Math.abs(y - t.y);\n\n    if (dx <= DEDUP_DISTANCE && dy <= DEDUP_DISTANCE) {\n      return true;\n    }\n  }\n\n  return false;\n}\n\nvar TouchMouseInput =\n/*#__PURE__*/\nfunction () {\n  var TouchMouseInput =\n  /*#__PURE__*/\n  function (_Input) {\n    _inheritsLoose(TouchMouseInput, _Input);\n\n    function TouchMouseInput(_manager, callback) {\n      var _this;\n\n      _this = _Input.call(this, _manager, callback) || this;\n\n      _this.handler = function (manager, inputEvent, inputData) {\n        var isTouch = inputData.pointerType === INPUT_TYPE_TOUCH;\n        var isMouse = inputData.pointerType === INPUT_TYPE_MOUSE;\n\n        if (isMouse && inputData.sourceCapabilities && inputData.sourceCapabilities.firesTouchEvents) {\n          return;\n        } // when we're in a touch event, record touches to  de-dupe synthetic mouse event\n\n\n        if (isTouch) {\n          recordTouches.call(_assertThisInitialized(_assertThisInitialized(_this)), inputEvent, inputData);\n        } else if (isMouse && isSyntheticEvent.call(_assertThisInitialized(_assertThisInitialized(_this)), inputData)) {\n          return;\n        }\n\n        _this.callback(manager, inputEvent, inputData);\n      };\n\n      _this.touch = new TouchInput(_this.manager, _this.handler);\n      _this.mouse = new MouseInput(_this.manager, _this.handler);\n      _this.primaryTouch = null;\n      _this.lastTouches = [];\n      return _this;\n    }\n    /**\n     * @private\n     * handle mouse and touch events\n     * @param {Hammer} manager\n     * @param {String} inputEvent\n     * @param {Object} inputData\n     */\n\n\n    var _proto = TouchMouseInput.prototype;\n\n    /**\n     * @private\n     * remove the event listeners\n     */\n    _proto.destroy = function destroy() {\n      this.touch.destroy();\n      this.mouse.destroy();\n    };\n\n    return TouchMouseInput;\n  }(Input);\n\n  return TouchMouseInput;\n}();\n\n/**\n * @private\n * create new input type manager\n * called by the Manager constructor\n * @param {Hammer} manager\n * @returns {Input}\n */\n\nfunction createInputInstance(manager) {\n  var Type; // let inputClass = manager.options.inputClass;\n\n  var inputClass = manager.options.inputClass;\n\n  if (inputClass) {\n    Type = inputClass;\n  } else if (SUPPORT_POINTER_EVENTS) {\n    Type = PointerEventInput;\n  } else if (SUPPORT_ONLY_TOUCH) {\n    Type = TouchInput;\n  } else if (!SUPPORT_TOUCH) {\n    Type = MouseInput;\n  } else {\n    Type = TouchMouseInput;\n  }\n\n  return new Type(manager, inputHandler);\n}\n\n/**\n * @private\n * if the argument is an array, we want to execute the fn on each entry\n * if it aint an array we don't want to do a thing.\n * this is used by all the methods that accept a single and array argument.\n * @param {*|Array} arg\n * @param {String} fn\n * @param {Object} [context]\n * @returns {Boolean}\n */\n\nfunction invokeArrayArg(arg, fn, context) {\n  if (Array.isArray(arg)) {\n    each(arg, context[fn], context);\n    return true;\n  }\n\n  return false;\n}\n\nvar STATE_POSSIBLE = 1;\nvar STATE_BEGAN = 2;\nvar STATE_CHANGED = 4;\nvar STATE_ENDED = 8;\nvar STATE_RECOGNIZED = STATE_ENDED;\nvar STATE_CANCELLED = 16;\nvar STATE_FAILED = 32;\n\n/**\n * @private\n * get a unique id\n * @returns {number} uniqueId\n */\nvar _uniqueId = 1;\nfunction uniqueId() {\n  return _uniqueId++;\n}\n\n/**\n * @private\n * get a recognizer by name if it is bound to a manager\n * @param {Recognizer|String} otherRecognizer\n * @param {Recognizer} recognizer\n * @returns {Recognizer}\n */\nfunction getRecognizerByNameIfManager(otherRecognizer, recognizer) {\n  var manager = recognizer.manager;\n\n  if (manager) {\n    return manager.get(otherRecognizer);\n  }\n\n  return otherRecognizer;\n}\n\n/**\n * @private\n * get a usable string, used as event postfix\n * @param {constant} state\n * @returns {String} state\n */\n\nfunction stateStr(state) {\n  if (state & STATE_CANCELLED) {\n    return 'cancel';\n  } else if (state & STATE_ENDED) {\n    return 'end';\n  } else if (state & STATE_CHANGED) {\n    return 'move';\n  } else if (state & STATE_BEGAN) {\n    return 'start';\n  }\n\n  return '';\n}\n\n/**\n * @private\n * Recognizer flow explained; *\n * All recognizers have the initial state of POSSIBLE when a input session starts.\n * The definition of a input session is from the first input until the last input, with all it's movement in it. *\n * Example session for mouse-input: mousedown -> mousemove -> mouseup\n *\n * On each recognizing cycle (see Manager.recognize) the .recognize() method is executed\n * which determines with state it should be.\n *\n * If the recognizer has the state FAILED, CANCELLED or RECOGNIZED (equals ENDED), it is reset to\n * POSSIBLE to give it another change on the next cycle.\n *\n *               Possible\n *                  |\n *            +-----+---------------+\n *            |                     |\n *      +-----+-----+               |\n *      |           |               |\n *   Failed      Cancelled          |\n *                          +-------+------+\n *                          |              |\n *                      Recognized       Began\n *                                         |\n *                                      Changed\n *                                         |\n *                                  Ended/Recognized\n */\n\n/**\n * @private\n * Recognizer\n * Every recognizer needs to extend from this class.\n * @constructor\n * @param {Object} options\n */\n\nvar Recognizer =\n/*#__PURE__*/\nfunction () {\n  function Recognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    this.options = _extends({\n      enable: true\n    }, options);\n    this.id = uniqueId();\n    this.manager = null; // default is enable true\n\n    this.state = STATE_POSSIBLE;\n    this.simultaneous = {};\n    this.requireFail = [];\n  }\n  /**\n   * @private\n   * set options\n   * @param {Object} options\n   * @return {Recognizer}\n   */\n\n\n  var _proto = Recognizer.prototype;\n\n  _proto.set = function set(options) {\n    assign$1(this.options, options); // also update the touchAction, in case something changed about the directions/enabled state\n\n    this.manager && this.manager.touchAction.update();\n    return this;\n  };\n  /**\n   * @private\n   * recognize simultaneous with an other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.recognizeWith = function recognizeWith(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'recognizeWith', this)) {\n      return this;\n    }\n\n    var simultaneous = this.simultaneous;\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n\n    if (!simultaneous[otherRecognizer.id]) {\n      simultaneous[otherRecognizer.id] = otherRecognizer;\n      otherRecognizer.recognizeWith(this);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * drop the simultaneous link. it doesnt remove the link on the other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.dropRecognizeWith = function dropRecognizeWith(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'dropRecognizeWith', this)) {\n      return this;\n    }\n\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n    delete this.simultaneous[otherRecognizer.id];\n    return this;\n  };\n  /**\n   * @private\n   * recognizer can only run when an other is failing\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.requireFailure = function requireFailure(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'requireFailure', this)) {\n      return this;\n    }\n\n    var requireFail = this.requireFail;\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n\n    if (inArray(requireFail, otherRecognizer) === -1) {\n      requireFail.push(otherRecognizer);\n      otherRecognizer.requireFailure(this);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * drop the requireFailure link. it does not remove the link on the other recognizer.\n   * @param {Recognizer} otherRecognizer\n   * @returns {Recognizer} this\n   */\n\n\n  _proto.dropRequireFailure = function dropRequireFailure(otherRecognizer) {\n    if (invokeArrayArg(otherRecognizer, 'dropRequireFailure', this)) {\n      return this;\n    }\n\n    otherRecognizer = getRecognizerByNameIfManager(otherRecognizer, this);\n    var index = inArray(this.requireFail, otherRecognizer);\n\n    if (index > -1) {\n      this.requireFail.splice(index, 1);\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * has require failures boolean\n   * @returns {boolean}\n   */\n\n\n  _proto.hasRequireFailures = function hasRequireFailures() {\n    return this.requireFail.length > 0;\n  };\n  /**\n   * @private\n   * if the recognizer can recognize simultaneous with an other recognizer\n   * @param {Recognizer} otherRecognizer\n   * @returns {Boolean}\n   */\n\n\n  _proto.canRecognizeWith = function canRecognizeWith(otherRecognizer) {\n    return !!this.simultaneous[otherRecognizer.id];\n  };\n  /**\n   * @private\n   * You should use `tryEmit` instead of `emit` directly to check\n   * that all the needed recognizers has failed before emitting.\n   * @param {Object} input\n   */\n\n\n  _proto.emit = function emit(input) {\n    var self = this;\n    var state = this.state;\n\n    function emit(event) {\n      self.manager.emit(event, input);\n    } // 'panstart' and 'panmove'\n\n\n    if (state < STATE_ENDED) {\n      emit(self.options.event + stateStr(state));\n    }\n\n    emit(self.options.event); // simple 'eventName' events\n\n    if (input.additionalEvent) {\n      // additional event(panleft, panright, pinchin, pinchout...)\n      emit(input.additionalEvent);\n    } // panend and pancancel\n\n\n    if (state >= STATE_ENDED) {\n      emit(self.options.event + stateStr(state));\n    }\n  };\n  /**\n   * @private\n   * Check that all the require failure recognizers has failed,\n   * if true, it emits a gesture event,\n   * otherwise, setup the state to FAILED.\n   * @param {Object} input\n   */\n\n\n  _proto.tryEmit = function tryEmit(input) {\n    if (this.canEmit()) {\n      return this.emit(input);\n    } // it's failing anyway\n\n\n    this.state = STATE_FAILED;\n  };\n  /**\n   * @private\n   * can we emit?\n   * @returns {boolean}\n   */\n\n\n  _proto.canEmit = function canEmit() {\n    var i = 0;\n\n    while (i < this.requireFail.length) {\n      if (!(this.requireFail[i].state & (STATE_FAILED | STATE_POSSIBLE))) {\n        return false;\n      }\n\n      i++;\n    }\n\n    return true;\n  };\n  /**\n   * @private\n   * update the recognizer\n   * @param {Object} inputData\n   */\n\n\n  _proto.recognize = function recognize(inputData) {\n    // make a new copy of the inputData\n    // so we can change the inputData without messing up the other recognizers\n    var inputDataClone = assign$1({}, inputData); // is is enabled and allow recognizing?\n\n    if (!boolOrFn(this.options.enable, [this, inputDataClone])) {\n      this.reset();\n      this.state = STATE_FAILED;\n      return;\n    } // reset when we've reached the end\n\n\n    if (this.state & (STATE_RECOGNIZED | STATE_CANCELLED | STATE_FAILED)) {\n      this.state = STATE_POSSIBLE;\n    }\n\n    this.state = this.process(inputDataClone); // the recognizer has recognized a gesture\n    // so trigger an event\n\n    if (this.state & (STATE_BEGAN | STATE_CHANGED | STATE_ENDED | STATE_CANCELLED)) {\n      this.tryEmit(inputDataClone);\n    }\n  };\n  /**\n   * @private\n   * return the state of the recognizer\n   * the actual recognizing happens in this method\n   * @virtual\n   * @param {Object} inputData\n   * @returns {constant} STATE\n   */\n\n  /* jshint ignore:start */\n\n\n  _proto.process = function process(inputData) {};\n  /* jshint ignore:end */\n\n  /**\n   * @private\n   * return the preferred touch-action\n   * @virtual\n   * @returns {Array}\n   */\n\n\n  _proto.getTouchAction = function getTouchAction() {};\n  /**\n   * @private\n   * called when the gesture isn't allowed to recognize\n   * like when another is being recognized or it is disabled\n   * @virtual\n   */\n\n\n  _proto.reset = function reset() {};\n\n  return Recognizer;\n}();\n\n/**\n * @private\n * A tap is recognized when the pointer is doing a small tap/click. Multiple taps are recognized if they occur\n * between the given interval and position. The delay option can be used to recognize multi-taps without firing\n * a single tap.\n *\n * The eventData from the emitted event contains the property `tapCount`, which contains the amount of\n * multi-taps being recognized.\n * @constructor\n * @extends Recognizer\n */\n\nvar TapRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(TapRecognizer, _Recognizer);\n\n  function TapRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _Recognizer.call(this, _extends({\n      event: 'tap',\n      pointers: 1,\n      taps: 1,\n      interval: 300,\n      // max time between the multi-tap taps\n      time: 250,\n      // max time of the pointer to be down (like finger on the screen)\n      threshold: 9,\n      // a minimal movement is ok, but keep it low\n      posThreshold: 10\n    }, options)) || this; // previous time and center,\n    // used for tap counting\n\n    _this.pTime = false;\n    _this.pCenter = false;\n    _this._timer = null;\n    _this._input = null;\n    _this.count = 0;\n    return _this;\n  }\n\n  var _proto = TapRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_MANIPULATION];\n  };\n\n  _proto.process = function process(input) {\n    var _this2 = this;\n\n    var options = this.options;\n    var validPointers = input.pointers.length === options.pointers;\n    var validMovement = input.distance < options.threshold;\n    var validTouchTime = input.deltaTime < options.time;\n    this.reset();\n\n    if (input.eventType & INPUT_START && this.count === 0) {\n      return this.failTimeout();\n    } // we only allow little movement\n    // and we've reached an end event, so a tap is possible\n\n\n    if (validMovement && validTouchTime && validPointers) {\n      if (input.eventType !== INPUT_END) {\n        return this.failTimeout();\n      }\n\n      var validInterval = this.pTime ? input.timeStamp - this.pTime < options.interval : true;\n      var validMultiTap = !this.pCenter || getDistance(this.pCenter, input.center) < options.posThreshold;\n      this.pTime = input.timeStamp;\n      this.pCenter = input.center;\n\n      if (!validMultiTap || !validInterval) {\n        this.count = 1;\n      } else {\n        this.count += 1;\n      }\n\n      this._input = input; // if tap count matches we have recognized it,\n      // else it has began recognizing...\n\n      var tapCount = this.count % options.taps;\n\n      if (tapCount === 0) {\n        // no failing requirements, immediately trigger the tap event\n        // or wait as long as the multitap interval to trigger\n        if (!this.hasRequireFailures()) {\n          return STATE_RECOGNIZED;\n        } else {\n          this._timer = setTimeout(function () {\n            _this2.state = STATE_RECOGNIZED;\n\n            _this2.tryEmit();\n          }, options.interval);\n          return STATE_BEGAN;\n        }\n      }\n    }\n\n    return STATE_FAILED;\n  };\n\n  _proto.failTimeout = function failTimeout() {\n    var _this3 = this;\n\n    this._timer = setTimeout(function () {\n      _this3.state = STATE_FAILED;\n    }, this.options.interval);\n    return STATE_FAILED;\n  };\n\n  _proto.reset = function reset() {\n    clearTimeout(this._timer);\n  };\n\n  _proto.emit = function emit() {\n    if (this.state === STATE_RECOGNIZED) {\n      this._input.tapCount = this.count;\n      this.manager.emit(this.options.event, this._input);\n    }\n  };\n\n  return TapRecognizer;\n}(Recognizer);\n\n/**\n * @private\n * This recognizer is just used as a base for the simple attribute recognizers.\n * @constructor\n * @extends Recognizer\n */\n\nvar AttrRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(AttrRecognizer, _Recognizer);\n\n  function AttrRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _Recognizer.call(this, _extends({\n      pointers: 1\n    }, options)) || this;\n  }\n  /**\n   * @private\n   * Used to check if it the recognizer receives valid input, like input.distance > 10.\n   * @memberof AttrRecognizer\n   * @param {Object} input\n   * @returns {Boolean} recognized\n   */\n\n\n  var _proto = AttrRecognizer.prototype;\n\n  _proto.attrTest = function attrTest(input) {\n    var optionPointers = this.options.pointers;\n    return optionPointers === 0 || input.pointers.length === optionPointers;\n  };\n  /**\n   * @private\n   * Process the input and return the state for the recognizer\n   * @memberof AttrRecognizer\n   * @param {Object} input\n   * @returns {*} State\n   */\n\n\n  _proto.process = function process(input) {\n    var state = this.state;\n    var eventType = input.eventType;\n    var isRecognized = state & (STATE_BEGAN | STATE_CHANGED);\n    var isValid = this.attrTest(input); // on cancel input and we've recognized before, return STATE_CANCELLED\n\n    if (isRecognized && (eventType & INPUT_CANCEL || !isValid)) {\n      return state | STATE_CANCELLED;\n    } else if (isRecognized || isValid) {\n      if (eventType & INPUT_END) {\n        return state | STATE_ENDED;\n      } else if (!(state & STATE_BEGAN)) {\n        return STATE_BEGAN;\n      }\n\n      return state | STATE_CHANGED;\n    }\n\n    return STATE_FAILED;\n  };\n\n  return AttrRecognizer;\n}(Recognizer);\n\n/**\n * @private\n * direction cons to string\n * @param {constant} direction\n * @returns {String}\n */\n\nfunction directionStr(direction) {\n  if (direction === DIRECTION_DOWN) {\n    return 'down';\n  } else if (direction === DIRECTION_UP) {\n    return 'up';\n  } else if (direction === DIRECTION_LEFT) {\n    return 'left';\n  } else if (direction === DIRECTION_RIGHT) {\n    return 'right';\n  }\n\n  return '';\n}\n\n/**\n * @private\n * Pan\n * Recognized when the pointer is down and moved in the allowed direction.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar PanRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(PanRecognizer, _AttrRecognizer);\n\n  function PanRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _AttrRecognizer.call(this, _extends({\n      event: 'pan',\n      threshold: 10,\n      pointers: 1,\n      direction: DIRECTION_ALL\n    }, options)) || this;\n    _this.pX = null;\n    _this.pY = null;\n    return _this;\n  }\n\n  var _proto = PanRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    var direction = this.options.direction;\n    var actions = [];\n\n    if (direction & DIRECTION_HORIZONTAL) {\n      actions.push(TOUCH_ACTION_PAN_Y);\n    }\n\n    if (direction & DIRECTION_VERTICAL) {\n      actions.push(TOUCH_ACTION_PAN_X);\n    }\n\n    return actions;\n  };\n\n  _proto.directionTest = function directionTest(input) {\n    var options = this.options;\n    var hasMoved = true;\n    var distance = input.distance;\n    var direction = input.direction;\n    var x = input.deltaX;\n    var y = input.deltaY; // lock to axis?\n\n    if (!(direction & options.direction)) {\n      if (options.direction & DIRECTION_HORIZONTAL) {\n        direction = x === 0 ? DIRECTION_NONE : x < 0 ? DIRECTION_LEFT : DIRECTION_RIGHT;\n        hasMoved = x !== this.pX;\n        distance = Math.abs(input.deltaX);\n      } else {\n        direction = y === 0 ? DIRECTION_NONE : y < 0 ? DIRECTION_UP : DIRECTION_DOWN;\n        hasMoved = y !== this.pY;\n        distance = Math.abs(input.deltaY);\n      }\n    }\n\n    input.direction = direction;\n    return hasMoved && distance > options.threshold && direction & options.direction;\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return AttrRecognizer.prototype.attrTest.call(this, input) && ( // replace with a super call\n    this.state & STATE_BEGAN || !(this.state & STATE_BEGAN) && this.directionTest(input));\n  };\n\n  _proto.emit = function emit(input) {\n    this.pX = input.deltaX;\n    this.pY = input.deltaY;\n    var direction = directionStr(input.direction);\n\n    if (direction) {\n      input.additionalEvent = this.options.event + direction;\n    }\n\n    _AttrRecognizer.prototype.emit.call(this, input);\n  };\n\n  return PanRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Swipe\n * Recognized when the pointer is moving fast (velocity), with enough distance in the allowed direction.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar SwipeRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(SwipeRecognizer, _AttrRecognizer);\n\n  function SwipeRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'swipe',\n      threshold: 10,\n      velocity: 0.3,\n      direction: DIRECTION_HORIZONTAL | DIRECTION_VERTICAL,\n      pointers: 1\n    }, options)) || this;\n  }\n\n  var _proto = SwipeRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return PanRecognizer.prototype.getTouchAction.call(this);\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    var direction = this.options.direction;\n    var velocity;\n\n    if (direction & (DIRECTION_HORIZONTAL | DIRECTION_VERTICAL)) {\n      velocity = input.overallVelocity;\n    } else if (direction & DIRECTION_HORIZONTAL) {\n      velocity = input.overallVelocityX;\n    } else if (direction & DIRECTION_VERTICAL) {\n      velocity = input.overallVelocityY;\n    }\n\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && direction & input.offsetDirection && input.distance > this.options.threshold && input.maxPointers === this.options.pointers && abs(velocity) > this.options.velocity && input.eventType & INPUT_END;\n  };\n\n  _proto.emit = function emit(input) {\n    var direction = directionStr(input.offsetDirection);\n\n    if (direction) {\n      this.manager.emit(this.options.event + direction, input);\n    }\n\n    this.manager.emit(this.options.event, input);\n  };\n\n  return SwipeRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Pinch\n * Recognized when two or more pointers are moving toward (zoom-in) or away from each other (zoom-out).\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar PinchRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(PinchRecognizer, _AttrRecognizer);\n\n  function PinchRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'pinch',\n      threshold: 0,\n      pointers: 2\n    }, options)) || this;\n  }\n\n  var _proto = PinchRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_NONE];\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && (Math.abs(input.scale - 1) > this.options.threshold || this.state & STATE_BEGAN);\n  };\n\n  _proto.emit = function emit(input) {\n    if (input.scale !== 1) {\n      var inOut = input.scale < 1 ? 'in' : 'out';\n      input.additionalEvent = this.options.event + inOut;\n    }\n\n    _AttrRecognizer.prototype.emit.call(this, input);\n  };\n\n  return PinchRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Rotate\n * Recognized when two or more pointer are moving in a circular motion.\n * @constructor\n * @extends AttrRecognizer\n */\n\nvar RotateRecognizer =\n/*#__PURE__*/\nfunction (_AttrRecognizer) {\n  _inheritsLoose(RotateRecognizer, _AttrRecognizer);\n\n  function RotateRecognizer(options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return _AttrRecognizer.call(this, _extends({\n      event: 'rotate',\n      threshold: 0,\n      pointers: 2\n    }, options)) || this;\n  }\n\n  var _proto = RotateRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_NONE];\n  };\n\n  _proto.attrTest = function attrTest(input) {\n    return _AttrRecognizer.prototype.attrTest.call(this, input) && (Math.abs(input.rotation) > this.options.threshold || this.state & STATE_BEGAN);\n  };\n\n  return RotateRecognizer;\n}(AttrRecognizer);\n\n/**\n * @private\n * Press\n * Recognized when the pointer is down for x ms without any movement.\n * @constructor\n * @extends Recognizer\n */\n\nvar PressRecognizer =\n/*#__PURE__*/\nfunction (_Recognizer) {\n  _inheritsLoose(PressRecognizer, _Recognizer);\n\n  function PressRecognizer(options) {\n    var _this;\n\n    if (options === void 0) {\n      options = {};\n    }\n\n    _this = _Recognizer.call(this, _extends({\n      event: 'press',\n      pointers: 1,\n      time: 251,\n      // minimal time of the pointer to be pressed\n      threshold: 9\n    }, options)) || this;\n    _this._timer = null;\n    _this._input = null;\n    return _this;\n  }\n\n  var _proto = PressRecognizer.prototype;\n\n  _proto.getTouchAction = function getTouchAction() {\n    return [TOUCH_ACTION_AUTO];\n  };\n\n  _proto.process = function process(input) {\n    var _this2 = this;\n\n    var options = this.options;\n    var validPointers = input.pointers.length === options.pointers;\n    var validMovement = input.distance < options.threshold;\n    var validTime = input.deltaTime > options.time;\n    this._input = input; // we only allow little movement\n    // and we've reached an end event, so a tap is possible\n\n    if (!validMovement || !validPointers || input.eventType & (INPUT_END | INPUT_CANCEL) && !validTime) {\n      this.reset();\n    } else if (input.eventType & INPUT_START) {\n      this.reset();\n      this._timer = setTimeout(function () {\n        _this2.state = STATE_RECOGNIZED;\n\n        _this2.tryEmit();\n      }, options.time);\n    } else if (input.eventType & INPUT_END) {\n      return STATE_RECOGNIZED;\n    }\n\n    return STATE_FAILED;\n  };\n\n  _proto.reset = function reset() {\n    clearTimeout(this._timer);\n  };\n\n  _proto.emit = function emit(input) {\n    if (this.state !== STATE_RECOGNIZED) {\n      return;\n    }\n\n    if (input && input.eventType & INPUT_END) {\n      this.manager.emit(this.options.event + \"up\", input);\n    } else {\n      this._input.timeStamp = now();\n      this.manager.emit(this.options.event, this._input);\n    }\n  };\n\n  return PressRecognizer;\n}(Recognizer);\n\nvar defaults = {\n  /**\n   * @private\n   * set if DOM events are being triggered.\n   * But this is slower and unused by simple implementations, so disabled by default.\n   * @type {Boolean}\n   * @default false\n   */\n  domEvents: false,\n\n  /**\n   * @private\n   * The value for the touchAction property/fallback.\n   * When set to `compute` it will magically set the correct value based on the added recognizers.\n   * @type {String}\n   * @default compute\n   */\n  touchAction: TOUCH_ACTION_COMPUTE,\n\n  /**\n   * @private\n   * @type {Boolean}\n   * @default true\n   */\n  enable: true,\n\n  /**\n   * @private\n   * EXPERIMENTAL FEATURE -- can be removed/changed\n   * Change the parent input target element.\n   * If Null, then it is being set the to main element.\n   * @type {Null|EventTarget}\n   * @default null\n   */\n  inputTarget: null,\n\n  /**\n   * @private\n   * force an input class\n   * @type {Null|Function}\n   * @default null\n   */\n  inputClass: null,\n\n  /**\n   * @private\n   * Some CSS properties can be used to improve the working of Hammer.\n   * Add them to this method and they will be set when creating a new Manager.\n   * @namespace\n   */\n  cssProps: {\n    /**\n     * @private\n     * Disables text selection to improve the dragging gesture. Mainly for desktop browsers.\n     * @type {String}\n     * @default 'none'\n     */\n    userSelect: \"none\",\n\n    /**\n     * @private\n     * Disable the Windows Phone grippers when pressing an element.\n     * @type {String}\n     * @default 'none'\n     */\n    touchSelect: \"none\",\n\n    /**\n     * @private\n     * Disables the default callout shown when you touch and hold a touch target.\n     * On iOS, when you touch and hold a touch target such as a link, Safari displays\n     * a callout containing information about the link. This property allows you to disable that callout.\n     * @type {String}\n     * @default 'none'\n     */\n    touchCallout: \"none\",\n\n    /**\n     * @private\n     * Specifies whether zooming is enabled. Used by IE10>\n     * @type {String}\n     * @default 'none'\n     */\n    contentZooming: \"none\",\n\n    /**\n     * @private\n     * Specifies that an entire element should be draggable instead of its contents. Mainly for desktop browsers.\n     * @type {String}\n     * @default 'none'\n     */\n    userDrag: \"none\",\n\n    /**\n     * @private\n     * Overrides the highlight color shown when the user taps a link or a JavaScript\n     * clickable element in iOS. This property obeys the alpha value, if specified.\n     * @type {String}\n     * @default 'rgba(0,0,0,0)'\n     */\n    tapHighlightColor: \"rgba(0,0,0,0)\"\n  }\n};\n/**\n * @private\n * Default recognizer setup when calling `Hammer()`\n * When creating a new Manager these will be skipped.\n * This is separated with other defaults because of tree-shaking.\n * @type {Array}\n */\n\nvar preset = [[RotateRecognizer, {\n  enable: false\n}], [PinchRecognizer, {\n  enable: false\n}, ['rotate']], [SwipeRecognizer, {\n  direction: DIRECTION_HORIZONTAL\n}], [PanRecognizer, {\n  direction: DIRECTION_HORIZONTAL\n}, ['swipe']], [TapRecognizer], [TapRecognizer, {\n  event: 'doubletap',\n  taps: 2\n}, ['tap']], [PressRecognizer]];\n\nvar STOP = 1;\nvar FORCED_STOP = 2;\n/**\n * @private\n * add/remove the css properties as defined in manager.options.cssProps\n * @param {Manager} manager\n * @param {Boolean} add\n */\n\nfunction toggleCssProps(manager, add) {\n  var element = manager.element;\n\n  if (!element.style) {\n    return;\n  }\n\n  var prop;\n  each(manager.options.cssProps, function (value, name) {\n    prop = prefixed(element.style, name);\n\n    if (add) {\n      manager.oldCssProps[prop] = element.style[prop];\n      element.style[prop] = value;\n    } else {\n      element.style[prop] = manager.oldCssProps[prop] || \"\";\n    }\n  });\n\n  if (!add) {\n    manager.oldCssProps = {};\n  }\n}\n/**\n * @private\n * trigger dom event\n * @param {String} event\n * @param {Object} data\n */\n\n\nfunction triggerDomEvent(event, data) {\n  var gestureEvent = document.createEvent(\"Event\");\n  gestureEvent.initEvent(event, true, true);\n  gestureEvent.gesture = data;\n  data.target.dispatchEvent(gestureEvent);\n}\n/**\n* @private\n * Manager\n * @param {HTMLElement} element\n * @param {Object} [options]\n * @constructor\n */\n\n\nvar Manager =\n/*#__PURE__*/\nfunction () {\n  function Manager(element, options) {\n    var _this = this;\n\n    this.options = assign$1({}, defaults, options || {});\n    this.options.inputTarget = this.options.inputTarget || element;\n    this.handlers = {};\n    this.session = {};\n    this.recognizers = [];\n    this.oldCssProps = {};\n    this.element = element;\n    this.input = createInputInstance(this);\n    this.touchAction = new TouchAction(this, this.options.touchAction);\n    toggleCssProps(this, true);\n    each(this.options.recognizers, function (item) {\n      var recognizer = _this.add(new item[0](item[1]));\n\n      item[2] && recognizer.recognizeWith(item[2]);\n      item[3] && recognizer.requireFailure(item[3]);\n    }, this);\n  }\n  /**\n   * @private\n   * set options\n   * @param {Object} options\n   * @returns {Manager}\n   */\n\n\n  var _proto = Manager.prototype;\n\n  _proto.set = function set(options) {\n    assign$1(this.options, options); // Options that need a little more setup\n\n    if (options.touchAction) {\n      this.touchAction.update();\n    }\n\n    if (options.inputTarget) {\n      // Clean up existing event listeners and reinitialize\n      this.input.destroy();\n      this.input.target = options.inputTarget;\n      this.input.init();\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * stop recognizing for this session.\n   * This session will be discarded, when a new [input]start event is fired.\n   * When forced, the recognizer cycle is stopped immediately.\n   * @param {Boolean} [force]\n   */\n\n\n  _proto.stop = function stop(force) {\n    this.session.stopped = force ? FORCED_STOP : STOP;\n  };\n  /**\n   * @private\n   * run the recognizers!\n   * called by the inputHandler function on every movement of the pointers (touches)\n   * it walks through all the recognizers and tries to detect the gesture that is being made\n   * @param {Object} inputData\n   */\n\n\n  _proto.recognize = function recognize(inputData) {\n    var session = this.session;\n\n    if (session.stopped) {\n      return;\n    } // run the touch-action polyfill\n\n\n    this.touchAction.preventDefaults(inputData);\n    var recognizer;\n    var recognizers = this.recognizers; // this holds the recognizer that is being recognized.\n    // so the recognizer's state needs to be BEGAN, CHANGED, ENDED or RECOGNIZED\n    // if no recognizer is detecting a thing, it is set to `null`\n\n    var curRecognizer = session.curRecognizer; // reset when the last recognizer is recognized\n    // or when we're in a new session\n\n    if (!curRecognizer || curRecognizer && curRecognizer.state & STATE_RECOGNIZED) {\n      session.curRecognizer = null;\n      curRecognizer = null;\n    }\n\n    var i = 0;\n\n    while (i < recognizers.length) {\n      recognizer = recognizers[i]; // find out if we are allowed try to recognize the input for this one.\n      // 1.   allow if the session is NOT forced stopped (see the .stop() method)\n      // 2.   allow if we still haven't recognized a gesture in this session, or the this recognizer is the one\n      //      that is being recognized.\n      // 3.   allow if the recognizer is allowed to run simultaneous with the current recognized recognizer.\n      //      this can be setup with the `recognizeWith()` method on the recognizer.\n\n      if (session.stopped !== FORCED_STOP && ( // 1\n      !curRecognizer || recognizer === curRecognizer || // 2\n      recognizer.canRecognizeWith(curRecognizer))) {\n        // 3\n        recognizer.recognize(inputData);\n      } else {\n        recognizer.reset();\n      } // if the recognizer has been recognizing the input as a valid gesture, we want to store this one as the\n      // current active recognizer. but only if we don't already have an active recognizer\n\n\n      if (!curRecognizer && recognizer.state & (STATE_BEGAN | STATE_CHANGED | STATE_ENDED)) {\n        session.curRecognizer = recognizer;\n        curRecognizer = recognizer;\n      }\n\n      i++;\n    }\n  };\n  /**\n   * @private\n   * get a recognizer by its event name.\n   * @param {Recognizer|String} recognizer\n   * @returns {Recognizer|Null}\n   */\n\n\n  _proto.get = function get(recognizer) {\n    if (recognizer instanceof Recognizer) {\n      return recognizer;\n    }\n\n    var recognizers = this.recognizers;\n\n    for (var i = 0; i < recognizers.length; i++) {\n      if (recognizers[i].options.event === recognizer) {\n        return recognizers[i];\n      }\n    }\n\n    return null;\n  };\n  /**\n   * @private add a recognizer to the manager\n   * existing recognizers with the same event name will be removed\n   * @param {Recognizer} recognizer\n   * @returns {Recognizer|Manager}\n   */\n\n\n  _proto.add = function add(recognizer) {\n    if (invokeArrayArg(recognizer, \"add\", this)) {\n      return this;\n    } // remove existing\n\n\n    var existing = this.get(recognizer.options.event);\n\n    if (existing) {\n      this.remove(existing);\n    }\n\n    this.recognizers.push(recognizer);\n    recognizer.manager = this;\n    this.touchAction.update();\n    return recognizer;\n  };\n  /**\n   * @private\n   * remove a recognizer by name or instance\n   * @param {Recognizer|String} recognizer\n   * @returns {Manager}\n   */\n\n\n  _proto.remove = function remove(recognizer) {\n    if (invokeArrayArg(recognizer, \"remove\", this)) {\n      return this;\n    }\n\n    var targetRecognizer = this.get(recognizer); // let's make sure this recognizer exists\n\n    if (recognizer) {\n      var recognizers = this.recognizers;\n      var index = inArray(recognizers, targetRecognizer);\n\n      if (index !== -1) {\n        recognizers.splice(index, 1);\n        this.touchAction.update();\n      }\n    }\n\n    return this;\n  };\n  /**\n   * @private\n   * bind event\n   * @param {String} events\n   * @param {Function} handler\n   * @returns {EventEmitter} this\n   */\n\n\n  _proto.on = function on(events, handler) {\n    if (events === undefined || handler === undefined) {\n      return this;\n    }\n\n    var handlers = this.handlers;\n    each(splitStr(events), function (event) {\n      handlers[event] = handlers[event] || [];\n      handlers[event].push(handler);\n    });\n    return this;\n  };\n  /**\n   * @private unbind event, leave emit blank to remove all handlers\n   * @param {String} events\n   * @param {Function} [handler]\n   * @returns {EventEmitter} this\n   */\n\n\n  _proto.off = function off(events, handler) {\n    if (events === undefined) {\n      return this;\n    }\n\n    var handlers = this.handlers;\n    each(splitStr(events), function (event) {\n      if (!handler) {\n        delete handlers[event];\n      } else {\n        handlers[event] && handlers[event].splice(inArray(handlers[event], handler), 1);\n      }\n    });\n    return this;\n  };\n  /**\n   * @private emit event to the listeners\n   * @param {String} event\n   * @param {Object} data\n   */\n\n\n  _proto.emit = function emit(event, data) {\n    // we also want to trigger dom events\n    if (this.options.domEvents) {\n      triggerDomEvent(event, data);\n    } // no handlers, so skip it all\n\n\n    var handlers = this.handlers[event] && this.handlers[event].slice();\n\n    if (!handlers || !handlers.length) {\n      return;\n    }\n\n    data.type = event;\n\n    data.preventDefault = function () {\n      data.srcEvent.preventDefault();\n    };\n\n    var i = 0;\n\n    while (i < handlers.length) {\n      handlers[i](data);\n      i++;\n    }\n  };\n  /**\n   * @private\n   * destroy the manager and unbinds all events\n   * it doesn't unbind dom events, that is the user own responsibility\n   */\n\n\n  _proto.destroy = function destroy() {\n    this.element && toggleCssProps(this, false);\n    this.handlers = {};\n    this.session = {};\n    this.input.destroy();\n    this.element = null;\n  };\n\n  return Manager;\n}();\n\nvar SINGLE_TOUCH_INPUT_MAP = {\n  touchstart: INPUT_START,\n  touchmove: INPUT_MOVE,\n  touchend: INPUT_END,\n  touchcancel: INPUT_CANCEL\n};\nvar SINGLE_TOUCH_TARGET_EVENTS = 'touchstart';\nvar SINGLE_TOUCH_WINDOW_EVENTS = 'touchstart touchmove touchend touchcancel';\n/**\n * @private\n * Touch events input\n * @constructor\n * @extends Input\n */\n\nvar SingleTouchInput =\n/*#__PURE__*/\nfunction (_Input) {\n  _inheritsLoose(SingleTouchInput, _Input);\n\n  function SingleTouchInput() {\n    var _this;\n\n    var proto = SingleTouchInput.prototype;\n    proto.evTarget = SINGLE_TOUCH_TARGET_EVENTS;\n    proto.evWin = SINGLE_TOUCH_WINDOW_EVENTS;\n    _this = _Input.apply(this, arguments) || this;\n    _this.started = false;\n    return _this;\n  }\n\n  var _proto = SingleTouchInput.prototype;\n\n  _proto.handler = function handler(ev) {\n    var type = SINGLE_TOUCH_INPUT_MAP[ev.type]; // should we handle the touch events?\n\n    if (type === INPUT_START) {\n      this.started = true;\n    }\n\n    if (!this.started) {\n      return;\n    }\n\n    var touches = normalizeSingleTouches.call(this, ev, type); // when done, reset the started state\n\n    if (type & (INPUT_END | INPUT_CANCEL) && touches[0].length - touches[1].length === 0) {\n      this.started = false;\n    }\n\n    this.callback(this.manager, type, {\n      pointers: touches[0],\n      changedPointers: touches[1],\n      pointerType: INPUT_TYPE_TOUCH,\n      srcEvent: ev\n    });\n  };\n\n  return SingleTouchInput;\n}(Input);\n\nfunction normalizeSingleTouches(ev, type) {\n  var all = toArray(ev.touches);\n  var changed = toArray(ev.changedTouches);\n\n  if (type & (INPUT_END | INPUT_CANCEL)) {\n    all = uniqueArray(all.concat(changed), 'identifier', true);\n  }\n\n  return [all, changed];\n}\n\n/**\n * @private\n * wrap a method with a deprecation warning and stack trace\n * @param {Function} method\n * @param {String} name\n * @param {String} message\n * @returns {Function} A new function wrapping the supplied method.\n */\nfunction deprecate(method, name, message) {\n  var deprecationMessage = \"DEPRECATED METHOD: \" + name + \"\\n\" + message + \" AT \\n\";\n  return function () {\n    var e = new Error('get-stack-trace');\n    var stack = e && e.stack ? e.stack.replace(/^[^\\(]+?[\\n$]/gm, '').replace(/^\\s+at\\s+/gm, '').replace(/^Object.<anonymous>\\s*\\(/gm, '{anonymous}()@') : 'Unknown Stack Trace';\n    var log = window.console && (window.console.warn || window.console.log);\n\n    if (log) {\n      log.call(window.console, deprecationMessage, stack);\n    }\n\n    return method.apply(this, arguments);\n  };\n}\n\n/**\n * @private\n * extend object.\n * means that properties in dest will be overwritten by the ones in src.\n * @param {Object} dest\n * @param {Object} src\n * @param {Boolean} [merge=false]\n * @returns {Object} dest\n */\n\nvar extend = deprecate(function (dest, src, merge) {\n  var keys = Object.keys(src);\n  var i = 0;\n\n  while (i < keys.length) {\n    if (!merge || merge && dest[keys[i]] === undefined) {\n      dest[keys[i]] = src[keys[i]];\n    }\n\n    i++;\n  }\n\n  return dest;\n}, 'extend', 'Use `assign`.');\n\n/**\n * @private\n * merge the values from src in the dest.\n * means that properties that exist in dest will not be overwritten by src\n * @param {Object} dest\n * @param {Object} src\n * @returns {Object} dest\n */\n\nvar merge = deprecate(function (dest, src) {\n  return extend(dest, src, true);\n}, 'merge', 'Use `assign`.');\n\n/**\n * @private\n * simple class inheritance\n * @param {Function} child\n * @param {Function} base\n * @param {Object} [properties]\n */\n\nfunction inherit(child, base, properties) {\n  var baseP = base.prototype;\n  var childP;\n  childP = child.prototype = Object.create(baseP);\n  childP.constructor = child;\n  childP._super = baseP;\n\n  if (properties) {\n    assign$1(childP, properties);\n  }\n}\n\n/**\n * @private\n * simple function bind\n * @param {Function} fn\n * @param {Object} context\n * @returns {Function}\n */\nfunction bindFn(fn, context) {\n  return function boundFn() {\n    return fn.apply(context, arguments);\n  };\n}\n\n/**\n * @private\n * Simple way to create a manager with a default set of recognizers.\n * @param {HTMLElement} element\n * @param {Object} [options]\n * @constructor\n */\n\nvar Hammer =\n/*#__PURE__*/\nfunction () {\n  var Hammer =\n  /**\n    * @private\n    * @const {string}\n    */\n  function Hammer(element, options) {\n    if (options === void 0) {\n      options = {};\n    }\n\n    return new Manager(element, _extends({\n      recognizers: preset.concat()\n    }, options));\n  };\n\n  Hammer.VERSION = \"2.0.17-rc\";\n  Hammer.DIRECTION_ALL = DIRECTION_ALL;\n  Hammer.DIRECTION_DOWN = DIRECTION_DOWN;\n  Hammer.DIRECTION_LEFT = DIRECTION_LEFT;\n  Hammer.DIRECTION_RIGHT = DIRECTION_RIGHT;\n  Hammer.DIRECTION_UP = DIRECTION_UP;\n  Hammer.DIRECTION_HORIZONTAL = DIRECTION_HORIZONTAL;\n  Hammer.DIRECTION_VERTICAL = DIRECTION_VERTICAL;\n  Hammer.DIRECTION_NONE = DIRECTION_NONE;\n  Hammer.DIRECTION_DOWN = DIRECTION_DOWN;\n  Hammer.INPUT_START = INPUT_START;\n  Hammer.INPUT_MOVE = INPUT_MOVE;\n  Hammer.INPUT_END = INPUT_END;\n  Hammer.INPUT_CANCEL = INPUT_CANCEL;\n  Hammer.STATE_POSSIBLE = STATE_POSSIBLE;\n  Hammer.STATE_BEGAN = STATE_BEGAN;\n  Hammer.STATE_CHANGED = STATE_CHANGED;\n  Hammer.STATE_ENDED = STATE_ENDED;\n  Hammer.STATE_RECOGNIZED = STATE_RECOGNIZED;\n  Hammer.STATE_CANCELLED = STATE_CANCELLED;\n  Hammer.STATE_FAILED = STATE_FAILED;\n  Hammer.Manager = Manager;\n  Hammer.Input = Input;\n  Hammer.TouchAction = TouchAction;\n  Hammer.TouchInput = TouchInput;\n  Hammer.MouseInput = MouseInput;\n  Hammer.PointerEventInput = PointerEventInput;\n  Hammer.TouchMouseInput = TouchMouseInput;\n  Hammer.SingleTouchInput = SingleTouchInput;\n  Hammer.Recognizer = Recognizer;\n  Hammer.AttrRecognizer = AttrRecognizer;\n  Hammer.Tap = TapRecognizer;\n  Hammer.Pan = PanRecognizer;\n  Hammer.Swipe = SwipeRecognizer;\n  Hammer.Pinch = PinchRecognizer;\n  Hammer.Rotate = RotateRecognizer;\n  Hammer.Press = PressRecognizer;\n  Hammer.on = addEventListeners;\n  Hammer.off = removeEventListeners;\n  Hammer.each = each;\n  Hammer.merge = merge;\n  Hammer.extend = extend;\n  Hammer.bindFn = bindFn;\n  Hammer.assign = assign$1;\n  Hammer.inherit = inherit;\n  Hammer.bindFn = bindFn;\n  Hammer.prefixed = prefixed;\n  Hammer.toArray = toArray;\n  Hammer.inArray = inArray;\n  Hammer.uniqueArray = uniqueArray;\n  Hammer.splitStr = splitStr;\n  Hammer.boolOrFn = boolOrFn;\n  Hammer.hasParent = hasParent;\n  Hammer.addEventListeners = addEventListeners;\n  Hammer.removeEventListeners = removeEventListeners;\n  Hammer.defaults = assign$1({}, defaults, {\n    preset: preset\n  });\n  return Hammer;\n}();\n\n//  style loader but by script tag, not by the loader.\n\nvar defaults$1 = Hammer.defaults;\n\nexport default Hammer;\nexport { INPUT_START, INPUT_MOVE, INPUT_END, INPUT_CANCEL, STATE_POSSIBLE, STATE_BEGAN, STATE_CHANGED, STATE_ENDED, STATE_RECOGNIZED, STATE_CANCELLED, STATE_FAILED, DIRECTION_NONE, DIRECTION_LEFT, DIRECTION_RIGHT, DIRECTION_UP, DIRECTION_DOWN, DIRECTION_HORIZONTAL, DIRECTION_VERTICAL, DIRECTION_ALL, Manager, Input, TouchAction, TouchInput, MouseInput, PointerEventInput, TouchMouseInput, SingleTouchInput, Recognizer, AttrRecognizer, TapRecognizer as Tap, PanRecognizer as Pan, SwipeRecognizer as Swipe, PinchRecognizer as Pinch, RotateRecognizer as Rotate, PressRecognizer as Press, addEventListeners as on, removeEventListeners as off, each, merge, extend, assign$1 as assign, inherit, bindFn, prefixed, toArray, inArray, uniqueArray, splitStr, boolOrFn, hasParent, addEventListeners, removeEventListeners, defaults$1 as defaults };\n//# sourceMappingURL=hammer.esm.js.map\n","import PropagatingHammer from 'propagating-hammerjs';\nimport Hammer from '@egjs/hammerjs';\n\n/**\n * Setup a mock hammer.js object, for unit testing.\n *\n * Inspiration: https://github.com/uber/deck.gl/pull/658\n *\n * @returns {{on: noop, off: noop, destroy: noop, emit: noop, get: get}}\n */\nfunction hammerMock() {\n  const noop = () => {};\n\n  return {\n    on: noop,\n    off: noop,\n    destroy: noop,\n    emit: noop,\n\n    get(m) {\t//eslint-disable-line no-unused-vars\n      return {\n        set: noop\n      };\n    }\n  };\n}\n\nlet modifiedHammer\n\nif (typeof window !== 'undefined') {\n  const OurHammer = window['Hammer'] || Hammer;\n  modifiedHammer = PropagatingHammer(OurHammer, {\n    preventDefault: 'mouse'\n  });\n} else {\n  modifiedHammer = () => // hammer.js is only available in a browser, not in node.js. Replacing it with a mock object.\n  hammerMock()\n}\n\nexport default modifiedHammer;\n","/**\n * Register a touch event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\nexport function onTouch (hammer, callback) {\n  callback.inputHandler = function (event) {\n    if (event.isFirst) {\n      callback(event);\n    }\n  };\n\n  hammer.on('hammer.input', callback.inputHandler);\n}\n\n/**\n * Register a release event, taking place after a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n * @returns {*}\n */\nexport function onRelease (hammer, callback) {\n  callback.inputHandler = function (event) {\n    if (event.isFinal) {\n      callback(event);\n    }\n  };\n\n  return hammer.on('hammer.input', callback.inputHandler);\n}\n\n\n/**\n * Unregister a touch event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\nexport function offTouch (hammer, callback) {\n  hammer.off('hammer.input', callback.inputHandler);\n}\n\n/**\n * Unregister a release event, taking place before a gesture\n * @param {Hammer} hammer       A hammer instance\n * @param {function} callback   Callback, called as callback(event)\n */\n\nexport const offRelease = offTouch;\n\n/**\n * Hack the PinchRecognizer such that it doesn't prevent default behavior\n * for vertical panning.\n *\n * Yeah ... this is quite a hack ... see https://github.com/hammerjs/hammer.js/issues/932\n *\n * @param {Hammer.Pinch} pinchRecognizer\n * @return {Hammer.Pinch} returns the pinchRecognizer\n */\nexport function disablePreventDefaultVertically (pinchRecognizer) {\n  const TOUCH_ACTION_PAN_Y = 'pan-y';\n\n  pinchRecognizer.getTouchAction = function() {\n    // default method returns [TOUCH_ACTION_NONE]\n    return [TOUCH_ACTION_PAN_Y];\n  };\n\n  return pinchRecognizer;\n}\n","import moment from '../module/moment';\nimport * as DateUtil  from './DateUtil';\nimport util from '../util';\n\n/**\n * The class TimeStep is an iterator for dates. You provide a start date and an\n * end date. The class itself determines the best scale (step size) based on the\n * provided start Date, end Date, and minimumStep.\n *\n * If minimumStep is provided, the step size is chosen as close as possible\n * to the minimumStep but larger than minimumStep. If minimumStep is not\n * provided, the scale is set to 1 DAY.\n * The minimumStep should correspond with the onscreen size of about 6 characters\n *\n * Alternatively, you can set a scale by hand.\n * After creation, you can initialize the class by executing first(). Then you\n * can iterate from the start date to the end date via next(). You can check if\n * the end date is reached with the function hasNext(). After each step, you can\n * retrieve the current date via getCurrent().\n * The TimeStep has scales ranging from milliseconds, seconds, minutes, hours,\n * days, to years.\n *\n * Version: 1.2\n *\n */\nclass TimeStep {\n  /**\n    * @param {Date} [start]         The start date, for example new Date(2010, 9, 21)\n    *                               or new Date(2010, 9, 21, 23, 45, 00)\n    * @param {Date} [end]           The end date\n    * @param {number} [minimumStep] Optional. Minimum step size in milliseconds\n    * @param {Date|Array.<Date>} [hiddenDates] Optional.\n    * @param {{showMajorLabels: boolean, showWeekScale: boolean}} [options] Optional.\n    * @constructor  TimeStep\n    */\n  constructor(start, end, minimumStep, hiddenDates, options) {\n    this.moment = (options && options.moment) || moment;\n    this.options = options ? options : {};\n\n    // variables\n    this.current = this.moment();\n    this._start = this.moment();\n    this._end = this.moment();\n\n    this.autoScale  = true;\n    this.scale = 'day';\n    this.step = 1;\n\n    // initialize the range\n    this.setRange(start, end, minimumStep);\n\n    // hidden Dates options\n    this.switchedDay = false;\n    this.switchedMonth = false;\n    this.switchedYear = false;\n    if (Array.isArray(hiddenDates)) {\n      this.hiddenDates = hiddenDates;\n    }\n    else if (hiddenDates != undefined) {\n      this.hiddenDates = [hiddenDates];\n    }\n    else {\n      this.hiddenDates = [];\n    }\n\n    this.format = TimeStep.FORMAT; // default formatting\n  }\n\n  /**\n   * Set custom constructor function for moment. Can be used to set dates\n   * to UTC or to set a utcOffset.\n   * @param {function} moment\n   */\n  setMoment(moment) {\n    this.moment = moment;\n\n    // update the date properties, can have a new utcOffset\n    this.current = this.moment(this.current.valueOf());\n    this._start = this.moment(this._start.valueOf());\n    this._end = this.moment(this._end.valueOf());\n  }\n\n  /**\n   * Set custom formatting for the minor an major labels of the TimeStep.\n   * Both `minorLabels` and `majorLabels` are an Object with properties:\n   * 'millisecond', 'second', 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'.\n   * @param {{minorLabels: Object, majorLabels: Object}} format\n   */\n  setFormat(format) {\n    const defaultFormat = util.deepExtend({}, TimeStep.FORMAT);\n    this.format = util.deepExtend(defaultFormat, format);\n  }\n\n  /**\n   * Set a new range\n   * If minimumStep is provided, the step size is chosen as close as possible\n   * to the minimumStep but larger than minimumStep. If minimumStep is not\n   * provided, the scale is set to 1 DAY.\n   * The minimumStep should correspond with the onscreen size of about 6 characters\n   * @param {Date} [start]      The start date and time.\n   * @param {Date} [end]        The end date and time.\n   * @param {int} [minimumStep] Optional. Minimum step size in milliseconds\n   */\n  setRange(start, end, minimumStep) {\n    if (!(start instanceof Date) || !(end instanceof Date)) {\n      throw  \"No legal start or end date in method setRange\";\n    }\n\n    this._start = (start != undefined) ? this.moment(start.valueOf()) : Date.now();\n    this._end = (end != undefined) ? this.moment(end.valueOf()) : Date.now();\n\n    if (this.autoScale) {\n      this.setMinimumStep(minimumStep);\n    }\n  }\n\n  /**\n   * Set the range iterator to the start date.\n   */\n  start() {\n    this.current = this._start.clone();\n    this.roundToMinor();\n  }\n\n  /**\n   * Round the current date to the first minor date value\n   * This must be executed once when the current date is set to start Date\n   */\n  roundToMinor() {\n    // round to floor\n    // to prevent year & month scales rounding down to the first day of week we perform this separately\n    if (this.scale == 'week') {\n      this.current.weekday(0);\n    }\n    // IMPORTANT: we have no breaks in this switch! (this is no bug)\n    // noinspection FallThroughInSwitchStatementJS\n    switch (this.scale) {\n      case 'year':\n        this.current.year(this.step * Math.floor(this.current.year() / this.step));\n        this.current.month(0);\n      case 'month':        this.current.date(1);          // eslint-disable-line no-fallthrough\n      case 'week':                                        // eslint-disable-line no-fallthrough\n      case 'day':                                         // eslint-disable-line no-fallthrough\n      case 'weekday':      this.current.hours(0);         // eslint-disable-line no-fallthrough\n      case 'hour':         this.current.minutes(0);       // eslint-disable-line no-fallthrough\n      case 'minute':       this.current.seconds(0);       // eslint-disable-line no-fallthrough\n      case 'second':       this.current.milliseconds(0);  // eslint-disable-line no-fallthrough\n      //case 'millisecond': // nothing to do for milliseconds\n    }\n\n    if (this.step != 1) {\n      // round down to the first minor value that is a multiple of the current step size\n      let  priorCurrent = this.current.clone();\n      switch (this.scale) {        \n        case 'millisecond':  this.current.subtract(this.current.milliseconds() % this.step, 'milliseconds');  break;\n        case 'second':       this.current.subtract(this.current.seconds() % this.step, 'seconds'); break;\n        case 'minute':       this.current.subtract(this.current.minutes() % this.step, 'minutes'); break;\n        case 'hour':         this.current.subtract(this.current.hours() % this.step, 'hours'); break;\n        case 'weekday':      // intentional fall through\n        case 'day':          this.current.subtract((this.current.date() - 1) % this.step, 'day'); break;\n        case 'week':         this.current.subtract(this.current.week() % this.step, 'week'); break;\n        case 'month':        this.current.subtract(this.current.month() % this.step, 'month');  break;\n        case 'year':         this.current.subtract(this.current.year() % this.step, 'year'); break;\n        default: break;\n      }\n      if (!priorCurrent.isSame(this.current)) {\n          this.current = this.moment(DateUtil.snapAwayFromHidden(this.hiddenDates, this.current.valueOf(), -1, true));\n      }\n    }\n  }\n\n  /**\n   * Check if the there is a next step\n   * @return {boolean}  true if the current date has not passed the end date\n   */\n  hasNext() {\n    return (this.current.valueOf() <= this._end.valueOf());\n  }\n\n  /**\n   * Do the next step\n   */\n  next() {\n    const prev = this.current.valueOf();\n\n    // Two cases, needed to prevent issues with switching daylight savings\n    // (end of March and end of October)\n    switch (this.scale) {\n      case 'millisecond':  this.current.add(this.step, 'millisecond'); break;\n      case 'second':       this.current.add(this.step, 'second'); break;\n      case 'minute':       this.current.add(this.step, 'minute'); break;\n      case 'hour':\n        this.current.add(this.step, 'hour');\n\n        if (this.current.month() < 6) {\n          this.current.subtract(this.current.hours() % this.step, 'hour');\n        } else {\n          if (this.current.hours() % this.step !== 0) {\n            this.current.add(this.step - this.current.hours() % this.step, 'hour');\n          }\n        }\n        break;\n      case 'weekday':      // intentional fall through\n      case 'day':          this.current.add(this.step, 'day'); break;\n      case 'week':\n        if (this.current.weekday() !== 0){ // we had a month break not correlating with a week's start before\n          this.current.weekday(0); // switch back to week cycles\n          this.current.add(this.step, 'week');\n        } else if(this.options.showMajorLabels === false) {\n          this.current.add(this.step, 'week'); // the default case\n        } else { // first day of the week\n          const nextWeek = this.current.clone();\n          nextWeek.add(1, 'week');\n          if(nextWeek.isSame(this.current, 'month')){ // is the first day of the next week in the same month?\n            this.current.add(this.step, 'week'); // the default case\n          } else { // inject a step at each first day of the month\n            this.current.add(this.step, 'week');\n            this.current.date(1);\n          }\n        }\n        break;\n      case 'month':        this.current.add(this.step, 'month'); break;\n      case 'year':         this.current.add(this.step, 'year'); break;\n      default: break;\n    }\n\n    if (this.step != 1) {\n      // round down to the correct major value\n      switch (this.scale) {\n        case 'millisecond':  if(this.current.milliseconds() > 0 && this.current.milliseconds() < this.step) this.current.milliseconds(0);  break;\n        case 'second':       if(this.current.seconds() > 0 && this.current.seconds() < this.step) this.current.seconds(0);  break;\n        case 'minute':       if(this.current.minutes() > 0 && this.current.minutes() < this.step) this.current.minutes(0); break;\n        case 'hour':         if(this.current.hours() > 0 && this.current.hours() < this.step) this.current.hours(0);  break;\n        case 'weekday':      // intentional fall through\n        case 'day':          if(this.current.date() < this.step+1) this.current.date(1); break;\n        case 'week':         if(this.current.week() < this.step) this.current.week(1); break; // week numbering starts at 1, not 0\n        case 'month':        if(this.current.month() < this.step) this.current.month(0);  break;\n        case 'year':         break; // nothing to do for year\n        default:             break;\n      }\n    }\n\n    // safety mechanism: if current time is still unchanged, move to the end\n    if (this.current.valueOf() == prev) {\n      this.current = this._end.clone();\n    }\n\n    // Reset switches for year, month and day. Will get set to true where appropriate in DateUtil.stepOverHiddenDates\n    this.switchedDay = false;\n    this.switchedMonth = false;\n    this.switchedYear = false;\n\n    DateUtil.stepOverHiddenDates(this.moment, this, prev);\n  }\n\n  /**\n   * Get the current datetime\n   * @return {Moment}  current The current date\n   */\n  getCurrent() {\n    return this.current.clone();\n  }\n\n  /**\n   * Set a custom scale. Autoscaling will be disabled.\n   * For example setScale('minute', 5) will result\n   * in minor steps of 5 minutes, and major steps of an hour.\n   *\n   * @param {{scale: string, step: number}} params\n   *                               An object containing two properties:\n   *                               - A string 'scale'. Choose from 'millisecond', 'second',\n   *                                 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'.\n   *                               - A number 'step'. A step size, by default 1.\n   *                                 Choose for example 1, 2, 5, or 10.\n   */\n  setScale(params) {\n    if (params && typeof params.scale == 'string') {\n      this.scale = params.scale;\n      this.step = params.step > 0 ? params.step : 1;\n      this.autoScale = false;\n    }\n  }\n\n  /**\n   * Enable or disable autoscaling\n   * @param {boolean} enable  If true, autoascaling is set true\n   */\n  setAutoScale(enable) {\n    this.autoScale = enable;\n  }\n\n  /**\n   * Automatically determine the scale that bests fits the provided minimum step\n   * @param {number} [minimumStep]  The minimum step size in milliseconds\n   */\n  setMinimumStep(minimumStep) {\n    if (minimumStep == undefined) {\n      return;\n    }\n\n    //var b = asc + ds;\n\n    const stepYear       = (1000 * 60 * 60 * 24 * 30 * 12);\n    const stepMonth      = (1000 * 60 * 60 * 24 * 30);\n    const stepDay        = (1000 * 60 * 60 * 24);\n    const stepHour       = (1000 * 60 * 60);\n    const stepMinute     = (1000 * 60);\n    const stepSecond     = (1000);\n    const stepMillisecond= (1);\n\n    // find the smallest step that is larger than the provided minimumStep\n    if (stepYear*1000 > minimumStep)        {this.scale = 'year';        this.step = 1000;}\n    if (stepYear*500 > minimumStep)         {this.scale = 'year';        this.step = 500;}\n    if (stepYear*100 > minimumStep)         {this.scale = 'year';        this.step = 100;}\n    if (stepYear*50 > minimumStep)          {this.scale = 'year';        this.step = 50;}\n    if (stepYear*10 > minimumStep)          {this.scale = 'year';        this.step = 10;}\n    if (stepYear*5 > minimumStep)           {this.scale = 'year';        this.step = 5;}\n    if (stepYear > minimumStep)             {this.scale = 'year';        this.step = 1;}\n    if (stepMonth*3 > minimumStep)          {this.scale = 'month';       this.step = 3;}\n    if (stepMonth > minimumStep)            {this.scale = 'month';       this.step = 1;}\n    if (stepDay*7 > minimumStep && this.options.showWeekScale)            {this.scale = 'week';        this.step = 1;}\n    if (stepDay*2 > minimumStep)            {this.scale = 'day';         this.step = 2;}\n    if (stepDay > minimumStep)              {this.scale = 'day';         this.step = 1;}\n    if (stepDay/2 > minimumStep)            {this.scale = 'weekday';     this.step = 1;}\n    if (stepHour*4 > minimumStep)           {this.scale = 'hour';        this.step = 4;}\n    if (stepHour > minimumStep)             {this.scale = 'hour';        this.step = 1;}\n    if (stepMinute*15 > minimumStep)        {this.scale = 'minute';      this.step = 15;}\n    if (stepMinute*10 > minimumStep)        {this.scale = 'minute';      this.step = 10;}\n    if (stepMinute*5 > minimumStep)         {this.scale = 'minute';      this.step = 5;}\n    if (stepMinute > minimumStep)           {this.scale = 'minute';      this.step = 1;}\n    if (stepSecond*15 > minimumStep)        {this.scale = 'second';      this.step = 15;}\n    if (stepSecond*10 > minimumStep)        {this.scale = 'second';      this.step = 10;}\n    if (stepSecond*5 > minimumStep)         {this.scale = 'second';      this.step = 5;}\n    if (stepSecond > minimumStep)           {this.scale = 'second';      this.step = 1;}\n    if (stepMillisecond*200 > minimumStep)  {this.scale = 'millisecond'; this.step = 200;}\n    if (stepMillisecond*100 > minimumStep)  {this.scale = 'millisecond'; this.step = 100;}\n    if (stepMillisecond*50 > minimumStep)   {this.scale = 'millisecond'; this.step = 50;}\n    if (stepMillisecond*10 > minimumStep)   {this.scale = 'millisecond'; this.step = 10;}\n    if (stepMillisecond*5 > minimumStep)    {this.scale = 'millisecond'; this.step = 5;}\n    if (stepMillisecond > minimumStep)      {this.scale = 'millisecond'; this.step = 1;}\n  }\n\n  /**\n   * Snap a date to a rounded value.\n   * The snap intervals are dependent on the current scale and step.\n   * Static function\n   * @param {Date} date    the date to be snapped.\n   * @param {string} scale Current scale, can be 'millisecond', 'second',\n   *                       'minute', 'hour', 'weekday, 'day', 'week', 'month', 'year'.\n   * @param {number} step  Current step (1, 2, 4, 5, ...\n   * @return {Date} snappedDate\n   */\n  static snap(date, scale, step) {\n    const clone = moment(date);\n\n    if (scale == 'year') {\n      const year = clone.year() + Math.round(clone.month() / 12);\n      clone.year(Math.round(year / step) * step);\n      clone.month(0);\n      clone.date(0);\n      clone.hours(0);\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'month') {\n      if (clone.date() > 15) {\n        clone.date(1);\n        clone.add(1, 'month');\n        // important: first set Date to 1, after that change the month.\n      }\n      else {\n        clone.date(1);\n      }\n\n      clone.hours(0);\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'week') {\n        if (clone.weekday() > 2) { // doing it the momentjs locale aware way\n            clone.weekday(0);\n            clone.add(1, 'week');\n        }\n        else {\n            clone.weekday(0);\n        }\n\n        clone.hours(0);\n        clone.minutes(0);\n        clone.seconds(0);\n        clone.milliseconds(0);\n    }\n    else if (scale == 'day') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 5:\n        case 2:\n          clone.hours(Math.round(clone.hours() / 24) * 24); break;\n        default:\n          clone.hours(Math.round(clone.hours() / 12) * 12); break;\n      }\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'weekday') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 5:\n        case 2:\n          clone.hours(Math.round(clone.hours() / 12) * 12); break;\n        default:\n          clone.hours(Math.round(clone.hours() / 6) * 6); break;\n      }\n      clone.minutes(0);\n      clone.seconds(0);\n      clone.milliseconds(0);\n    }\n    else if (scale == 'hour') {\n      switch (step) {\n        case 4:\n          clone.minutes(Math.round(clone.minutes() / 60) * 60); break;\n        default:\n          clone.minutes(Math.round(clone.minutes() / 30) * 30); break;\n      }\n      clone.seconds(0);\n      clone.milliseconds(0);\n    } else if (scale == 'minute') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 15:\n        case 10:\n          clone.minutes(Math.round(clone.minutes() / 5) * 5);\n          clone.seconds(0);\n          break;\n        case 5:\n          clone.seconds(Math.round(clone.seconds() / 60) * 60); break;\n        default:\n          clone.seconds(Math.round(clone.seconds() / 30) * 30); break;\n      }\n      clone.milliseconds(0);\n    }\n    else if (scale == 'second') {\n      //noinspection FallthroughInSwitchStatementJS\n      switch (step) {\n        case 15:\n        case 10:\n          clone.seconds(Math.round(clone.seconds() / 5) * 5);\n          clone.milliseconds(0);\n          break;\n        case 5:\n          clone.milliseconds(Math.round(clone.milliseconds() / 1000) * 1000); break;\n        default:\n          clone.milliseconds(Math.round(clone.milliseconds() / 500) * 500); break;\n      }\n    }\n    else if (scale == 'millisecond') {\n      const _step = step > 5 ? step / 2 : 1;\n      clone.milliseconds(Math.round(clone.milliseconds() / _step) * _step);\n    }\n\n    return clone;\n  }\n\n  /**\n   * Check if the current value is a major value (for example when the step\n   * is DAY, a major value is each first day of the MONTH)\n   * @return {boolean} true if current date is major, else false.\n   */\n  isMajor() {\n    if (this.switchedYear == true) {\n      switch (this.scale) {\n        case 'year':\n        case 'month':\n        case 'week':\n        case 'weekday':\n        case 'day':\n        case 'hour':\n        case 'minute':\n        case 'second':\n        case 'millisecond':\n          return true;\n        default:\n          return false;\n      }\n    }\n    else if (this.switchedMonth == true) {\n      switch (this.scale) {\n        case 'week':\n        case 'weekday':\n        case 'day':\n        case 'hour':\n        case 'minute':\n        case 'second':\n        case 'millisecond':\n          return true;\n        default:\n          return false;\n      }\n    }\n    else if (this.switchedDay == true) {\n      switch (this.scale) {\n        case 'millisecond':\n        case 'second':\n        case 'minute':\n        case 'hour':\n          return true;\n        default:\n          return false;\n      }\n    }\n\n    const date = this.moment(this.current);\n    switch (this.scale) {\n      case 'millisecond':\n        return (date.milliseconds() == 0);\n      case 'second':\n        return (date.seconds() == 0);\n      case 'minute':\n        return (date.hours() == 0) && (date.minutes() == 0);\n      case 'hour':\n        return (date.hours() == 0);\n      case 'weekday': // intentional fall through\n      case 'day':\n        return (date.date() == 1);\n      case 'week':\n        return (date.date() == 1);\n      case 'month':\n        return (date.month() == 0);\n      case 'year':\n        return false;\n      default:\n        return false;\n    }\n  }\n\n  /**\n   * Returns formatted text for the minor axislabel, depending on the current\n   * date and the scale. For example when scale is MINUTE, the current time is\n   * formatted as \"hh:mm\".\n   * @param {Date} [date=this.current] custom date. if not provided, current date is taken\n   * @returns {String}\n   */\n  getLabelMinor(date) {\n    if (date == undefined) {\n      date = this.current;\n    }\n    if (date instanceof Date) {\n      date = this.moment(date)\n    }\n\n    if (typeof(this.format.minorLabels) === \"function\") {\n      return this.format.minorLabels(date, this.scale, this.step);\n    }\n\n    const format = this.format.minorLabels[this.scale];\n    // noinspection FallThroughInSwitchStatementJS\n    switch (this.scale) {\n      case 'week':\n        // Don't draw the minor label if this date is the first day of a month AND if it's NOT the start of the week.\n        // The 'date' variable may actually be the 'next' step when called from TimeAxis' _repaintLabels.\n        if(date.date() === 1 && date.weekday() !== 0){\n            return \"\";\n        }\n      default: // eslint-disable-line no-fallthrough\n        return (format && format.length > 0) ? this.moment(date).format(format) : '';\n    }\n  }\n\n  /**\n   * Returns formatted text for the major axis label, depending on the current\n   * date and the scale. For example when scale is MINUTE, the major scale is\n   * hours, and the hour will be formatted as \"hh\".\n   * @param {Date} [date=this.current] custom date. if not provided, current date is taken\n   * @returns {String}\n   */\n  getLabelMajor(date) {\n    if (date == undefined) {\n      date = this.current;\n    }\n    if (date instanceof Date) {\n      date = this.moment(date)\n    }\n\n    if (typeof(this.format.majorLabels) === \"function\") {\n      return this.format.majorLabels(date, this.scale, this.step);\n    }\n\n    const format = this.format.majorLabels[this.scale];\n    return (format && format.length > 0) ? this.moment(date).format(format) : '';\n  }\n\n  /**\n   * get class name\n   * @return {string} class name\n   */\n  getClassName() {\n    const _moment = this.moment;\n    const m = this.moment(this.current);\n    const current = m.locale ? m.locale('en') : m.lang('en'); // old versions of moment have .lang() function\n    const step = this.step;\n    const classNames = [];\n\n    /**\n     *\n     * @param {number} value\n     * @returns {String}\n     */\n    function even(value) {\n      return (value / step % 2 == 0) ? ' vis-even' : ' vis-odd';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function today(date) {\n      if (date.isSame(Date.now(), 'day')) {\n        return ' vis-today';\n      }\n      if (date.isSame(_moment().add(1, 'day'), 'day')) {\n        return ' vis-tomorrow';\n      }\n      if (date.isSame(_moment().add(-1, 'day'), 'day')) {\n        return ' vis-yesterday';\n      }\n      return '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentWeek(date) {\n      return date.isSame(Date.now(), 'week') ? ' vis-current-week' : '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentMonth(date) {\n      return date.isSame(Date.now(), 'month') ? ' vis-current-month' : '';\n    }\n\n    /**\n     *\n     * @param {Date} date\n     * @returns {String}\n     */\n    function currentYear(date) {\n      return date.isSame(Date.now(), 'year') ? ' vis-current-year' : '';\n    }\n\n    switch (this.scale) {\n      case 'millisecond':\n        classNames.push(today(current));\n        classNames.push(even(current.milliseconds()));\n        break;\n      case 'second':\n        classNames.push(today(current));\n        classNames.push(even(current.seconds()));\n        break;\n      case 'minute':\n        classNames.push(today(current));\n        classNames.push(even(current.minutes()));\n        break;\n      case 'hour':\n        classNames.push(`vis-h${current.hours()}${this.step == 4 ? '-h' + (current.hours() + 4) : ''}`);\n        classNames.push(today(current));\n        classNames.push(even(current.hours()));\n        break;\n      case 'weekday':\n        classNames.push(`vis-${current.format('dddd').toLowerCase()}`);\n        classNames.push(today(current));\n        classNames.push(currentWeek(current));\n        classNames.push(even(current.date()));\n        break;\n      case 'day':\n        classNames.push(`vis-day${current.date()}`);\n        classNames.push(`vis-${current.format('MMMM').toLowerCase()}`);\n        classNames.push(today(current));\n        classNames.push(currentMonth(current));\n        classNames.push(this.step <= 2 ? today(current) : '');\n        classNames.push(this.step <= 2 ? `vis-${current.format('dddd').toLowerCase()}` : '');\n        classNames.push(even(current.date() - 1));\n        break;\n      case 'week':\n        classNames.push(`vis-week${current.format('w')}`);\n        classNames.push(currentWeek(current));\n        classNames.push(even(current.week()));\n        break;\n      case 'month':\n        classNames.push(`vis-${current.format('MMMM').toLowerCase()}`);\n        classNames.push(currentMonth(current));\n        classNames.push(even(current.month()));\n        break;\n      case 'year':\n        classNames.push(`vis-year${current.year()}`);\n        classNames.push(currentYear(current));\n        classNames.push(even(current.year()));\n        break;\n    }\n    return classNames.filter(String).join(\" \");\n  }\n}\n\n// Time formatting\nTimeStep.FORMAT = {\n  minorLabels: {\n    millisecond:'SSS',\n    second:     's',\n    minute:     'HH:mm',\n    hour:       'HH:mm',\n    weekday:    'ddd D',\n    day:        'D',\n    week:       'w',\n    month:      'MMM',\n    year:       'YYYY'\n  },\n  majorLabels: {\n    millisecond:'HH:mm:ss',\n    second:     'D MMMM HH:mm',\n    minute:     'ddd D MMMM',\n    hour:       'ddd D MMMM',\n    weekday:    'MMMM YYYY',\n    day:        'MMMM YYYY',\n    week:       'MMMM YYYY',\n    month:      'YYYY',\n    year:       ''\n  }\n};\n\nexport default TimeStep;\n","function styleInject(css, ref) {\n  if ( ref === void 0 ) ref = {};\n  var insertAt = ref.insertAt;\n\n  if (!css || typeof document === 'undefined') { return; }\n\n  var head = document.head || document.getElementsByTagName('head')[0];\n  var style = document.createElement('style');\n  style.type = 'text/css';\n\n  if (insertAt === 'top') {\n    if (head.firstChild) {\n      head.insertBefore(style, head.firstChild);\n    } else {\n      head.appendChild(style);\n    }\n  } else {\n    head.appendChild(style);\n  }\n\n  if (style.styleSheet) {\n    style.styleSheet.cssText = css;\n  } else {\n    style.appendChild(document.createTextNode(css));\n  }\n}\n\nexport default styleInject;\n","import util from '../../util';\nimport Component from'./Component';\nimport TimeStep from '../TimeStep';\nimport * as DateUtil  from '../DateUtil';\nimport moment from '../../module/moment';\n\nimport './css/timeaxis.css';\n\n/** A horizontal time axis */\nclass TimeAxis extends Component {\n/**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} body\n * @param {Object} [options]        See TimeAxis.setOptions for the available\n *                                  options.\n * @constructor TimeAxis\n * @extends Component\n */\n  constructor(body, options) {\n    super();\n    this.dom = {\n      foreground: null,\n      lines: [],\n      majorTexts: [],\n      minorTexts: [],\n      redundant: {\n        lines: [],\n        majorTexts: [],\n        minorTexts: []\n      }\n    };\n    this.props = {\n      range: {\n        start: 0,\n        end: 0,\n        minimumStep: 0\n      },\n      lineTop: 0\n    };\n\n    this.defaultOptions = {\n      orientation: {\n        axis: 'bottom'\n      },  // axis orientation: 'top' or 'bottom'\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      maxMinorChars: 7,\n      format: TimeStep.FORMAT,\n      moment,\n      timeAxis: null\n    };\n    this.options = util.extend({}, this.defaultOptions);\n\n    this.body = body;\n\n    // create the HTML DOM\n    this._create();\n\n    this.setOptions(options);\n  }\n\n  /**\n   * Set options for the TimeAxis.\n   * Parameters will be merged in current options.\n   * @param {Object} options  Available options:\n   *                          {string} [orientation.axis]\n   *                          {boolean} [showMinorLabels]\n   *                          {boolean} [showMajorLabels]\n   *                          {boolean} [showWeekScale]\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend([\n        'showMinorLabels',\n        'showMajorLabels',\n        'showWeekScale',\n        'maxMinorChars',\n        'hiddenDates',\n        'timeAxis',\n        'moment',\n        'rtl'\n      ], this.options, options);\n\n      // deep copy the format options\n      util.selectiveDeepExtend(['format'], this.options, options);\n\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation.axis = options.orientation;\n        }\n        else if (typeof options.orientation === 'object' && 'axis' in options.orientation) {\n          this.options.orientation.axis = options.orientation.axis;\n        }\n      }\n\n      // apply locale to moment.js\n      // TODO: not so nice, this is applied globally to moment.js\n      if ('locale' in options) {\n        if (typeof moment.locale === 'function') {\n          // moment.js 2.8.1+\n          moment.locale(options.locale);\n        }\n        else {\n          moment.lang(options.locale);\n        }\n      }\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the TimeAxis\n   */\n  _create() {\n    this.dom.foreground = document.createElement('div');\n    this.dom.background = document.createElement('div');\n\n    this.dom.foreground.className = 'vis-time-axis vis-foreground';\n    this.dom.background.className = 'vis-time-axis vis-background';\n  }\n\n  /**\n   * Destroy the TimeAxis\n   */\n  destroy() {\n    // remove from DOM\n    if (this.dom.foreground.parentNode) {\n      this.dom.foreground.parentNode.removeChild(this.dom.foreground);\n    }\n    if (this.dom.background.parentNode) {\n      this.dom.background.parentNode.removeChild(this.dom.background);\n    }\n\n    this.body = null;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const props = this.props;\n    const foreground = this.dom.foreground;\n    const background = this.dom.background;\n\n    // determine the correct parent DOM element (depending on option orientation)\n    const parent = (this.options.orientation.axis == 'top') ? this.body.dom.top : this.body.dom.bottom;\n    const parentChanged = (foreground.parentNode !== parent);\n\n    // calculate character width and height\n    this._calculateCharSize();\n\n    // TODO: recalculate sizes only needed when parent is resized or options is changed\n    const showMinorLabels = this.options.showMinorLabels && this.options.orientation.axis !== 'none';\n    const showMajorLabels = this.options.showMajorLabels && this.options.orientation.axis !== 'none';\n\n    // determine the width and height of the elemens for the axis\n    props.minorLabelHeight = showMinorLabels ? props.minorCharHeight : 0;\n    props.majorLabelHeight = showMajorLabels ? props.majorCharHeight : 0;\n    props.height = props.minorLabelHeight + props.majorLabelHeight;\n    props.width = foreground.offsetWidth;\n\n    props.minorLineHeight = this.body.domProps.root.height - props.majorLabelHeight -\n        (this.options.orientation.axis == 'top' ? this.body.domProps.bottom.height : this.body.domProps.top.height);\n    props.minorLineWidth = 1; // TODO: really calculate width\n    props.majorLineHeight = props.minorLineHeight + props.majorLabelHeight;\n    props.majorLineWidth = 1; // TODO: really calculate width\n\n    //  take foreground and background offline while updating (is almost twice as fast)\n    const foregroundNextSibling = foreground.nextSibling;\n    const backgroundNextSibling = background.nextSibling;\n    foreground.parentNode && foreground.parentNode.removeChild(foreground);\n    background.parentNode && background.parentNode.removeChild(background);\n\n    foreground.style.height = `${this.props.height}px`;\n\n    this._repaintLabels();\n\n    // put DOM online again (at the same place)\n    if (foregroundNextSibling) {\n      parent.insertBefore(foreground, foregroundNextSibling);\n    }\n    else {\n      parent.appendChild(foreground)\n    }\n    if (backgroundNextSibling) {\n      this.body.dom.backgroundVertical.insertBefore(background, backgroundNextSibling);\n    }\n    else {\n      this.body.dom.backgroundVertical.appendChild(background)\n    }\n    return this._isResized() || parentChanged;\n  }\n\n  /**\n   * Repaint major and minor text labels and vertical grid lines\n   * @private\n   */\n  _repaintLabels() {\n    const orientation = this.options.orientation.axis;\n\n    // calculate range and step (step such that we have space for 7 characters per label)\n    const start = util.convert(this.body.range.start, 'Number');\n    const end = util.convert(this.body.range.end, 'Number');\n    const timeLabelsize = this.body.util.toTime((this.props.minorCharWidth || 10) * this.options.maxMinorChars).valueOf();\n    let minimumStep = timeLabelsize - DateUtil.getHiddenDurationBefore(this.options.moment, this.body.hiddenDates, this.body.range, timeLabelsize);\n    minimumStep -= this.body.util.toTime(0).valueOf();\n\n    const step = new TimeStep(new Date(start), new Date(end), minimumStep, this.body.hiddenDates, this.options);\n    step.setMoment(this.options.moment);\n    if (this.options.format) {\n      step.setFormat(this.options.format);\n    }\n    if (this.options.timeAxis) {\n      step.setScale(this.options.timeAxis);\n    }\n    this.step = step;\n\n    // Move all DOM elements to a \"redundant\" list, where they\n    // can be picked for re-use, and clear the lists with lines and texts.\n    // At the end of the function _repaintLabels, left over elements will be cleaned up\n    const dom = this.dom;\n    dom.redundant.lines = dom.lines;\n    dom.redundant.majorTexts = dom.majorTexts;\n    dom.redundant.minorTexts = dom.minorTexts;\n    dom.lines = [];\n    dom.majorTexts = [];\n    dom.minorTexts = [];\n\n    let current;\n    let next;\n    let x;\n    let xNext;\n    let isMajor;\n    let showMinorGrid;\n    let width = 0;\n    let prevWidth;\n    let line;\n    let xFirstMajorLabel = undefined;\n    let count = 0;\n    const MAX = 1000;\n    let className;\n\n    step.start();\n    next = step.getCurrent();\n    xNext = this.body.util.toScreen(next);\n    while (step.hasNext() && count < MAX) {\n      count++;\n\n      isMajor = step.isMajor();\n      className = step.getClassName();\n\n      current = next;\n      x = xNext;\n\n      step.next();\n      next = step.getCurrent();\n      xNext = this.body.util.toScreen(next);\n\n      prevWidth = width;\n      width = xNext - x;\n      switch (step.scale) {\n        case 'week':         showMinorGrid = true; break;\n        default:             showMinorGrid = (width >= prevWidth * 0.4); break; // prevent displaying of the 31th of the month on a scale of 5 days\n      }\n\n      if (this.options.showMinorLabels && showMinorGrid) {\n        var label = this._repaintMinorText(x, step.getLabelMinor(current), orientation, className);\n        label.style.width = `${width}px`; // set width to prevent overflow\n      }\n\n      if (isMajor && this.options.showMajorLabels) {\n        if (x > 0) {\n          if (xFirstMajorLabel == undefined) {\n            xFirstMajorLabel = x;\n          }\n          label = this._repaintMajorText(x, step.getLabelMajor(current), orientation, className);\n        }\n        line = this._repaintMajorLine(x, width, orientation, className);\n      }\n      else { // minor line\n        if (showMinorGrid) {\n          line = this._repaintMinorLine(x, width, orientation, className);\n        }\n        else {\n          if (line) {\n            // adjust the width of the previous grid\n            line.style.width = `${parseInt(line.style.width) + width}px`;\n          }\n        }\n      }\n    }\n\n    if (count === MAX && !warnedForOverflow) {\n        console.warn(`Something is wrong with the Timeline scale. Limited drawing of grid lines to ${MAX} lines.`);\n        warnedForOverflow = true;\n    }\n\n    // create a major label on the left when needed\n    if (this.options.showMajorLabels) {\n      const leftTime = this.body.util.toTime(0); // upper bound estimation\n      const leftText = step.getLabelMajor(leftTime);\n      const widthText = leftText.length * (this.props.majorCharWidth || 10) + 10;\n\n      if (xFirstMajorLabel == undefined || widthText < xFirstMajorLabel) {\n        this._repaintMajorText(0, leftText, orientation, className);\n      }\n    }\n\n    // Cleanup leftover DOM elements from the redundant list\n    util.forEach(this.dom.redundant, arr => {\n      while (arr.length) {\n        const elem = arr.pop();\n        if (elem && elem.parentNode) {\n          elem.parentNode.removeChild(elem);\n        }\n      }\n    });\n  }\n\n  /**\n   * Create a minor label for the axis at position x\n   * @param {number} x\n   * @param {string} text\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the HTML element of the created label\n   * @private\n   */\n  _repaintMinorText(x, text, orientation, className) {\n    // reuse redundant label\n    let label = this.dom.redundant.minorTexts.shift();\n\n    if (!label) {\n      // create new label\n      const content = document.createTextNode('');\n      label = document.createElement('div');\n      label.appendChild(content);\n      this.dom.foreground.appendChild(label);\n    }\n    this.dom.minorTexts.push(label);\n    label.innerHTML = text;\n\n\n    let y = (orientation == 'top') ? this.props.majorLabelHeight : 0;\n    this._setXY(label, x, y);\n\n    label.className = `vis-text vis-minor ${className}`;\n    //label.title = title;  // TODO: this is a heavy operation\n\n    return label;\n  }\n\n  /**\n   * Create a Major label for the axis at position x\n   * @param {number} x\n   * @param {string} text\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the HTML element of the created label\n   * @private\n   */\n  _repaintMajorText(x, text, orientation, className) {\n    // reuse redundant label\n    let label = this.dom.redundant.majorTexts.shift();\n\n    if (!label) {\n      // create label\n      const content = document.createElement('div');\n      label = document.createElement('div');\n      label.appendChild(content);\n      this.dom.foreground.appendChild(label);\n    }\n\n    label.childNodes[0].innerHTML = text;\n    label.className = `vis-text vis-major ${className}`;\n    //label.title = title; // TODO: this is a heavy operation\n\n    let y = (orientation == 'top') ? 0 : this.props.minorLabelHeight;\n    this._setXY(label, x, y);\n\n    this.dom.majorTexts.push(label);\n    return label;\n  }\n\n  /**\n   * sets xy\n   * @param {string} label\n   * @param {number} x\n   * @param {number} y\n   * @private\n   */\n  _setXY(label, x, y) {\n    // If rtl is true, inverse x.\n    const directionX = this.options.rtl ? (x * -1) : x;\n    label.style.transform = `translate(${directionX}px, ${y}px)`;\n  }\n\n  /**\n   * Create a minor line for the axis at position x\n   * @param {number} left\n   * @param {number} width\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the created line\n   * @private\n   */\n  _repaintMinorLine(left, width, orientation, className) {\n    // reuse redundant line\n    let line = this.dom.redundant.lines.shift();\n    if (!line) {\n      // create vertical line\n      line = document.createElement('div');\n      this.dom.background.appendChild(line);\n    }\n    this.dom.lines.push(line);\n\n    const props = this.props;\n    \n    line.style.width = `${width}px`;\n    line.style.height = `${props.minorLineHeight}px`;\n\n    let y = (orientation == 'top') ? props.majorLabelHeight : this.body.domProps.top.height;\n    let x = left - props.minorLineWidth / 2;\n\n    this._setXY(line, x, y);\n    line.className = `vis-grid ${this.options.rtl ?  'vis-vertical-rtl' : 'vis-vertical'} vis-minor ${className}`;\n\n    return line;\n  }\n\n  /**\n   * Create a Major line for the axis at position x\n   * @param {number} left\n   * @param {number} width\n   * @param {string} orientation   \"top\" or \"bottom\" (default)\n   * @param {string} className\n   * @return {Element} Returns the created line\n   * @private\n   */\n  _repaintMajorLine(left, width, orientation, className) {\n    // reuse redundant line\n    let line = this.dom.redundant.lines.shift();\n    if (!line) {\n      // create vertical line\n      line = document.createElement('div');\n      this.dom.background.appendChild(line);\n    }\n    this.dom.lines.push(line);\n\n    const props = this.props;\n    \n    line.style.width = `${width}px`;\n    line.style.height = `${props.majorLineHeight}px`;\n\n    let y = (orientation == 'top') ? 0 : this.body.domProps.top.height;\n    let x = left - props.majorLineWidth / 2;\n\n    this._setXY(line, x, y);\n    line.className = `vis-grid ${this.options.rtl ?  'vis-vertical-rtl' : 'vis-vertical'} vis-major ${className}`;\n\n    return line;\n  }\n\n  /**\n   * Determine the size of text on the axis (both major and minor axis).\n   * The size is calculated only once and then cached in this.props.\n   * @private\n   */\n  _calculateCharSize() {\n    // Note: We calculate char size with every redraw. Size may change, for\n    // example when any of the timelines parents had display:none for example.\n\n    // determine the char width and height on the minor axis\n    if (!this.dom.measureCharMinor) {\n      this.dom.measureCharMinor = document.createElement('DIV');\n      this.dom.measureCharMinor.className = 'vis-text vis-minor vis-measure';\n      this.dom.measureCharMinor.style.position = 'absolute';\n\n      this.dom.measureCharMinor.appendChild(document.createTextNode('0'));\n      this.dom.foreground.appendChild(this.dom.measureCharMinor);\n    }\n    this.props.minorCharHeight = this.dom.measureCharMinor.clientHeight;\n    this.props.minorCharWidth = this.dom.measureCharMinor.clientWidth;\n\n    // determine the char width and height on the major axis\n    if (!this.dom.measureCharMajor) {\n      this.dom.measureCharMajor = document.createElement('DIV');\n      this.dom.measureCharMajor.className = 'vis-text vis-major vis-measure';\n      this.dom.measureCharMajor.style.position = 'absolute';\n\n      this.dom.measureCharMajor.appendChild(document.createTextNode('0'));\n      this.dom.foreground.appendChild(this.dom.measureCharMajor);\n    }\n    this.props.majorCharHeight = this.dom.measureCharMajor.clientHeight;\n    this.props.majorCharWidth = this.dom.measureCharMajor.clientWidth;\n  }\n}\n\n\nvar warnedForOverflow = false;\n\nexport default TimeAxis;\n","\"use strict\";\r\n/**\r\n * Created by Alex on 11/6/2014.\r\n */\r\n\r\n// https://github.com/umdjs/umd/blob/master/returnExports.js#L40-L60\r\n// if the module has no dependencies, the above pattern can be simplified to\r\n(function (root, factory) {\r\n  if (typeof define === 'function' && define.amd) {\r\n    // AMD. Register as an anonymous module.\r\n    define([], factory);\r\n  } else if (typeof exports === 'object') {\r\n    // Node. Does not work with strict CommonJS, but\r\n    // only CommonJS-like environments that support module.exports,\r\n    // like Node.\r\n    module.exports = factory();\r\n  } else {\r\n    // Browser globals (root is window)\r\n    root.keycharm = factory();\r\n  }\r\n}(this, function () {\r\n\r\n  function keycharm(options) {\r\n    var preventDefault = options && options.preventDefault || false;\r\n\r\n    var container = options && options.container || window;\r\n\r\n    var _exportFunctions = {};\r\n    var _bound = {keydown:{}, keyup:{}};\r\n    var _keys = {};\r\n    var i;\r\n\r\n    // a - z\r\n    for (i = 97; i <= 122; i++) {_keys[String.fromCharCode(i)] = {code:65 + (i - 97), shift: false};}\r\n    // A - Z\r\n    for (i = 65; i <= 90; i++) {_keys[String.fromCharCode(i)] = {code:i, shift: true};}\r\n    // 0 - 9\r\n    for (i = 0;  i <= 9;   i++) {_keys['' + i] = {code:48 + i, shift: false};}\r\n    // F1 - F12\r\n    for (i = 1;  i <= 12;   i++) {_keys['F' + i] = {code:111 + i, shift: false};}\r\n    // num0 - num9\r\n    for (i = 0;  i <= 9;   i++) {_keys['num' + i] = {code:96 + i, shift: false};}\r\n\r\n    // numpad misc\r\n    _keys['num*'] = {code:106, shift: false};\r\n    _keys['num+'] = {code:107, shift: false};\r\n    _keys['num-'] = {code:109, shift: false};\r\n    _keys['num/'] = {code:111, shift: false};\r\n    _keys['num.'] = {code:110, shift: false};\r\n    // arrows\r\n    _keys['left']  = {code:37, shift: false};\r\n    _keys['up']    = {code:38, shift: false};\r\n    _keys['right'] = {code:39, shift: false};\r\n    _keys['down']  = {code:40, shift: false};\r\n    // extra keys\r\n    _keys['space'] = {code:32, shift: false};\r\n    _keys['enter'] = {code:13, shift: false};\r\n    _keys['shift'] = {code:16, shift: undefined};\r\n    _keys['esc']   = {code:27, shift: false};\r\n    _keys['backspace'] = {code:8, shift: false};\r\n    _keys['tab']       = {code:9, shift: false};\r\n    _keys['ctrl']      = {code:17, shift: false};\r\n    _keys['alt']       = {code:18, shift: false};\r\n    _keys['delete']    = {code:46, shift: false};\r\n    _keys['pageup']    = {code:33, shift: false};\r\n    _keys['pagedown']  = {code:34, shift: false};\r\n    // symbols\r\n    _keys['=']     = {code:187, shift: false};\r\n    _keys['-']     = {code:189, shift: false};\r\n    _keys[']']     = {code:221, shift: false};\r\n    _keys['[']     = {code:219, shift: false};\r\n\r\n\r\n\r\n    var down = function(event) {handleEvent(event,'keydown');};\r\n    var up = function(event) {handleEvent(event,'keyup');};\r\n\r\n    // handle the actualy bound key with the event\r\n    var handleEvent = function(event,type) {\r\n      if (_bound[type][event.keyCode] !== undefined) {\r\n        var bound = _bound[type][event.keyCode];\r\n        for (var i = 0; i < bound.length; i++) {\r\n          if (bound[i].shift === undefined) {\r\n            bound[i].fn(event);\r\n          }\r\n          else if (bound[i].shift == true && event.shiftKey == true) {\r\n            bound[i].fn(event);\r\n          }\r\n          else if (bound[i].shift == false && event.shiftKey == false) {\r\n            bound[i].fn(event);\r\n          }\r\n        }\r\n\r\n        if (preventDefault == true) {\r\n          event.preventDefault();\r\n        }\r\n      }\r\n    };\r\n\r\n    // bind a key to a callback\r\n    _exportFunctions.bind = function(key, callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      if (_keys[key] === undefined) {\r\n        throw new Error(\"unsupported key: \" + key);\r\n      }\r\n      if (_bound[type][_keys[key].code] === undefined) {\r\n        _bound[type][_keys[key].code] = [];\r\n      }\r\n      _bound[type][_keys[key].code].push({fn:callback, shift:_keys[key].shift});\r\n    };\r\n\r\n\r\n    // bind all keys to a call back (demo purposes)\r\n    _exportFunctions.bindAll = function(callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      for (var key in _keys) {\r\n        if (_keys.hasOwnProperty(key)) {\r\n          _exportFunctions.bind(key,callback,type);\r\n        }\r\n      }\r\n    };\r\n\r\n    // get the key label from an event\r\n    _exportFunctions.getKey = function(event) {\r\n      for (var key in _keys) {\r\n        if (_keys.hasOwnProperty(key)) {\r\n          if (event.shiftKey == true && _keys[key].shift == true && event.keyCode == _keys[key].code) {\r\n            return key;\r\n          }\r\n          else if (event.shiftKey == false && _keys[key].shift == false && event.keyCode == _keys[key].code) {\r\n            return key;\r\n          }\r\n          else if (event.keyCode == _keys[key].code && key == 'shift') {\r\n            return key;\r\n          }\r\n        }\r\n      }\r\n      return \"unknown key, currently not supported\";\r\n    };\r\n\r\n    // unbind either a specific callback from a key or all of them (by leaving callback undefined)\r\n    _exportFunctions.unbind = function(key, callback, type) {\r\n      if (type === undefined) {\r\n        type = 'keydown';\r\n      }\r\n      if (_keys[key] === undefined) {\r\n        throw new Error(\"unsupported key: \" + key);\r\n      }\r\n      if (callback !== undefined) {\r\n        var newBindings = [];\r\n        var bound = _bound[type][_keys[key].code];\r\n        if (bound !== undefined) {\r\n          for (var i = 0; i < bound.length; i++) {\r\n            if (!(bound[i].fn == callback && bound[i].shift == _keys[key].shift)) {\r\n              newBindings.push(_bound[type][_keys[key].code][i]);\r\n            }\r\n          }\r\n        }\r\n        _bound[type][_keys[key].code] = newBindings;\r\n      }\r\n      else {\r\n        _bound[type][_keys[key].code] = [];\r\n      }\r\n    };\r\n\r\n    // reset all bound variables.\r\n    _exportFunctions.reset = function() {\r\n      _bound = {keydown:{}, keyup:{}};\r\n    };\r\n\r\n    // unbind all listeners and reset all variables.\r\n    _exportFunctions.destroy = function() {\r\n      _bound = {keydown:{}, keyup:{}};\r\n      container.removeEventListener('keydown', down, true);\r\n      container.removeEventListener('keyup', up, true);\r\n    };\r\n\r\n    // create listeners.\r\n    container.addEventListener('keydown',down,true);\r\n    container.addEventListener('keyup',up,true);\r\n\r\n    // return the public functions.\r\n    return _exportFunctions;\r\n  }\r\n\r\n  return keycharm;\r\n}));\r\n\r\n\r\n","import keycharm from 'keycharm';\nimport Emitter from 'component-emitter';\nimport Hammer from '../module/hammer';\nimport util from '../util';\n\nimport './activator.css';\n\n/**\n * Turn an element into an clickToUse element.\n * When not active, the element has a transparent overlay. When the overlay is\n * clicked, the mode is changed to active.\n * When active, the element is displayed with a blue border around it, and\n * the interactive contents of the element can be used. When clicked outside\n * the element, the elements mode is changed to inactive.\n * @param {Element} container\n * @constructor Activator\n */\nfunction Activator(container) {\n  this.active = false;\n\n  this.dom = {\n    container: container\n  };\n\n  this.dom.overlay = document.createElement('div');\n  this.dom.overlay.className = 'vis-overlay';\n\n  this.dom.container.appendChild(this.dom.overlay);\n\n  this.hammer = Hammer(this.dom.overlay);\n  this.hammer.on('tap', this._onTapOverlay.bind(this));\n\n  // block all touch events (except tap)\n  var me = this;\n  var events = [\n    'tap', 'doubletap', 'press',\n    'pinch',\n    'pan', 'panstart', 'panmove', 'panend'\n  ];\n  events.forEach(function (event) {\n    me.hammer.on(event, function (event) {\n      event.stopPropagation();\n    });\n  });\n\n  // attach a click event to the window, in order to deactivate when clicking outside the timeline\n  if (document && document.body) {\n    this.onClick = function (event) {\n      if (!_hasParent(event.target, container)) {\n        me.deactivate();\n      }\n    };\n    document.body.addEventListener('click', this.onClick);\n  }\n\n  if (this.keycharm !== undefined) {\n    this.keycharm.destroy();\n  }\n  this.keycharm = keycharm();\n\n  // keycharm listener only bounded when active)\n  this.escListener = this.deactivate.bind(this);\n}\n\n// turn into an event emitter\nEmitter(Activator.prototype);\n\n// The currently active activator\nActivator.current = null;\n\n/**\n * Destroy the activator. Cleans up all created DOM and event listeners\n */\nActivator.prototype.destroy = function () {\n  this.deactivate();\n\n  // remove dom\n  this.dom.overlay.parentNode.removeChild(this.dom.overlay);\n\n  // remove global event listener\n  if (this.onClick) {\n    document.body.removeEventListener('click', this.onClick);\n  }\n  // remove keycharm\n  if (this.keycharm !== undefined) {\n    this.keycharm.destroy();\n  }\n  this.keycharm = null;\n  // cleanup hammer instances\n  this.hammer.destroy();\n  this.hammer = null;\n  // FIXME: cleaning up hammer instances doesn't work (Timeline not removed from memory)\n};\n\n/**\n * Activate the element\n * Overlay is hidden, element is decorated with a blue shadow border\n */\nActivator.prototype.activate = function () {\n  // we allow only one active activator at a time\n  if (Activator.current) {\n    Activator.current.deactivate();\n  }\n  Activator.current = this;\n\n  this.active = true;\n  this.dom.overlay.style.display = 'none';\n  util.addClassName(this.dom.container, 'vis-active');\n\n  this.emit('change');\n  this.emit('activate');\n\n  // ugly hack: bind ESC after emitting the events, as the Network rebinds all\n  // keyboard events on a 'change' event\n  this.keycharm.bind('esc', this.escListener);\n};\n\n/**\n * Deactivate the element\n * Overlay is displayed on top of the element\n */\nActivator.prototype.deactivate = function () {\n  if (Activator.current === this) {\n    Activator.current = null;\n  }\n\n  this.active = false;\n  this.dom.overlay.style.display = '';\n  util.removeClassName(this.dom.container, 'vis-active');\n  this.keycharm.unbind('esc', this.escListener);\n\n  this.emit('change');\n  this.emit('deactivate');\n};\n\n/**\n * Handle a tap event: activate the container\n * @param {Event}  event   The event\n * @private\n */\nActivator.prototype._onTapOverlay = function (event) {\n  // activate the container\n  this.activate();\n  event.stopPropagation();\n};\n\n/**\n * Test whether the element has the requested parent element somewhere in\n * its chain of parent nodes.\n * @param {HTMLElement} element\n * @param {HTMLElement} parent\n * @returns {boolean} Returns true when the parent is found somewhere in the\n *                    chain of parent nodes.\n * @private\n */\nfunction _hasParent(element, parent) {\n  while (element) {\n    if (element === parent) {\n      return true\n    }\n    element = element.parentNode;\n  }\n  return false;\n}\n\nexport default Activator;\n","/*\n * IMPORTANT: Locales for Moment has to be imported in the legacy and standalone\n * entry points. For the peer build it's users responsibility to do so.\n */\n\n// English\nconst en = {\n  current: 'current',\n  time: 'time',\n  deleteSelected: 'Delete selected',\n};\nconst en_EN = en;\nconst en_US = en;\n\n// Italiano\nconst it = {\n  current: 'attuale',\n  time: 'tempo',\n  deleteSelected: 'Cancella la selezione',\n};\nconst it_IT = it;\nconst it_CH = it;\n\n// Dutch\nconst nl = {\n  current: 'huidige',\n  time: 'tijd',\n  deleteSelected: 'Selectie verwijderen'\n};\nconst nl_NL = nl;\nconst nl_BE = nl;\n\n// German\nconst de = {\n  current: 'Aktuelle',\n  time: 'Zeit',\n  deleteSelected: 'L\\u00f6sche Auswahl',\n};\nconst de_DE = de;\n\n// French\nconst fr = {\n  current: 'actuel',\n  time: 'heure',\n  deleteSelected: 'Effacer la selection',\n};\nconst fr_FR = fr;\nconst fr_CA = fr;\nconst fr_BE = fr;\n\n// Espanol\nconst es = {\n  current: 'corriente',\n  time: 'hora',\n  deleteSelected: 'Eliminar selecci\\u00f3n',\n};\nconst es_ES = es;\n\n// Ukrainian\nconst uk = {\n  current: 'поточний',\n  time: 'час',\n  deleteSelected: 'Видалити обране',\n};\nconst uk_UA = uk;\n\n// Russian\nconst ru = {\n  current: 'текущее',\n  time: 'время',\n  deleteSelected: 'Удалить выбранное',\n};\nconst ru_RU = ru;\n\n// Polish\nconst pl = {\n  current: 'aktualny',\n  time: 'czas',\n  deleteSelected: 'Usuń wybrane',\n};\nconst pl_PL = pl;\n\n// Japanese\nconst ja = {\n  current: '現在',\n  time: '時刻',\n  deleteSelected: '選択されたものを削除',\n};\nconst ja_JP = ja;\n\nconst locales = {\n  en,\n  en_EN,\n  en_US,\n  it,\n  it_IT,\n  it_CH,\n  nl,\n  nl_NL,\n  nl_BE,\n  de,\n  de_DE,\n  fr,\n  fr_FR,\n  fr_CA,\n  fr_BE,\n  es,\n  es_ES,\n  uk,\n  uk_UA,\n  ru,\n  ru_RU,\n  pl,\n  pl_PL,\n  ja,\n  ja_JP,\n}\n\nexport default locales\n","import Hammer from '../../module/hammer';\nimport util from '../../util';\nimport Component from './Component';\nimport moment from '../../module/moment';\nimport locales from '../locales';\n\nimport './css/customtime.css';\n\n/** A custom time bar */\nclass CustomTime extends Component {\n /**\n * @param {{range: Range, dom: Object}} body\n * @param {Object} [options]        Available parameters:\n *                                  {number | string} id\n *                                  {string} locales\n *                                  {string} locale\n * @constructor CustomTime\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n\n    // default options\n    this.defaultOptions = {\n      moment,\n      locales,\n      locale: 'en',\n      id: undefined,\n      title: undefined\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.setOptions(options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n\n    if (options && options.time != null) {\n      this.customTime = options.time;\n    } else {\n      this.customTime = new Date();\n    }\n\n    this.eventParams = {}; // stores state parameters while dragging the bar\n\n    // create the DOM\n    this._create();\n  }\n\n  /**\n   * Set options for the component. Options will be merged in current options.\n   * @param {Object} options  Available parameters:\n   *                                  {number | string} id\n   *                                  {string} locales\n   *                                  {string} locale\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend(['moment', 'locale', 'locales', 'id', 'title', 'rtl', 'snap'], this.options, options);\n    }\n  }\n\n  /**\n   * Create the DOM for the custom time\n   * @private\n   */\n  _create() {\n    const bar = document.createElement('div');\n    bar['custom-time'] = this;\n    bar.className = `vis-custom-time ${this.options.id || ''}`;\n    bar.style.position = 'absolute';\n    bar.style.top = '0px';\n    bar.style.height = '100%';\n    this.bar = bar;\n\n    const drag = document.createElement('div');\n    drag.style.position = 'relative';\n    drag.style.top = '0px';\n    if(this.options.rtl) {\n      drag.style.right = '-10px';\n    } else  {\n       drag.style.left = '-10px';\n    }\n    drag.style.height = '100%';\n    drag.style.width = '20px';\n\n    /**\n     *\n     * @param {WheelEvent} e\n     */\n    function onMouseWheel (e) {\n      this.body.range._onMouseWheel(e);\n    }\n\n    if (drag.addEventListener) {\n      // IE9, Chrome, Safari, Opera\n      drag.addEventListener(\"mousewheel\", onMouseWheel.bind(this), false);\n      // Firefox\n      drag.addEventListener(\"DOMMouseScroll\", onMouseWheel.bind(this), false);\n    } else {\n      // IE 6/7/8\n      drag.attachEvent(\"onmousewheel\", onMouseWheel.bind(this));\n    }\n\n    bar.appendChild(drag);\n    // attach event listeners\n    this.hammer = new Hammer(drag);\n    this.hammer.on('panstart', this._onDragStart.bind(this));\n    this.hammer.on('panmove',  this._onDrag.bind(this));\n    this.hammer.on('panend',   this._onDragEnd.bind(this));\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_ALL});\n  }\n\n  /**\n   * Destroy the CustomTime bar\n   */\n  destroy() {\n    this.hide();\n\n    this.hammer.destroy();\n    this.hammer = null;\n\n    this.body = null;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const parent = this.body.dom.backgroundVertical;\n    if (this.bar.parentNode != parent) {\n      // attach to the dom\n      if (this.bar.parentNode) {\n        this.bar.parentNode.removeChild(this.bar);\n      }\n      parent.appendChild(this.bar);\n    }\n\n    const x = this.body.util.toScreen(this.customTime);\n\n    let locale = this.options.locales[this.options.locale];\n    if (!locale) {\n      if (!this.warned) {\n        console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n        this.warned = true;\n      }\n      locale = this.options.locales['en']; // fall back on english when not available\n    }\n\n    let title = this.options.title;\n    // To hide the title completely use empty string ''.\n    if (title === undefined) {\n      title = `${locale.time}: ${this.options.moment(this.customTime).format('dddd, MMMM Do YYYY, H:mm:ss')}`;\n      title = title.charAt(0).toUpperCase() + title.substring(1);\n    } else if (typeof title === \"function\") {\n      title = title.call(this, this.customTime);\n    }\n\n    this.options.rtl ? this.bar.style.right = `${x}px` : this.bar.style.left = `${x}px`;\n    this.bar.title = title;\n\n    return false;\n  }\n\n  /**\n   * Remove the CustomTime from the DOM\n   */\n  hide() {\n    // remove the line from the DOM\n    if (this.bar.parentNode) {\n      this.bar.parentNode.removeChild(this.bar);\n    }\n  }\n\n  /**\n   * Set custom time.\n   * @param {Date | number | string} time\n   */\n  setCustomTime(time) {\n    this.customTime = util.convert(time, 'Date');\n    this.redraw();\n  }\n\n  /**\n   * Retrieve the current custom time.\n   * @return {Date} customTime\n   */\n  getCustomTime() {\n    return new Date(this.customTime.valueOf());\n  }\n\n  /**\n   * Set custom marker.\n   * @param {string} [title] Title of the custom marker\n   * @param {boolean} [editable] Make the custom marker editable.\n   */\n  setCustomMarker(title, editable) {\n    const marker = document.createElement('div');\n    marker.className = `vis-custom-time-marker`;\n    marker.innerHTML = title;\n    marker.style.position = 'absolute';\n\n    if (editable) {\n      marker.setAttribute('contenteditable', 'true');\n      marker.addEventListener('pointerdown', function () {\n        marker.focus();\n      });\n      marker.addEventListener('input', this._onMarkerChange.bind(this));\n      // The editable div element has no change event, so here emulates the change event.\n      marker.title = title;\n      marker.addEventListener('blur', function (event) {\n        if (this.title != event.target.innerHTML) {\n          this._onMarkerChanged(event);\n          this.title = event.target.innerHTML;\n        }\n      }.bind(this));\n    }\n\n    this.bar.appendChild(marker);\n  }\n\n  /**\n    * Set custom title.\n    * @param {Date | number | string} title\n    */\n  setCustomTitle(title) {\n    this.options.title = title;\n  }\n\n  /**\n   * Start moving horizontally\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    this.eventParams.dragging = true;\n    this.eventParams.customTime = this.customTime;\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform moving operating.\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!this.eventParams.dragging) return;\n\n    let deltaX = this.options.rtl ? (-1) * event.deltaX : event.deltaX;\n\n    const x = this.body.util.toScreen(this.eventParams.customTime) + deltaX;\n    const time = this.body.util.toTime(x);\n\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const snap = this.options.snap;\n\n    const snappedTime = snap ? snap(time, scale, step) : time;\n\n    this.setCustomTime(snappedTime);\n\n    // fire a timechange event\n    this.body.emitter.emit('timechange', {\n      id: this.options.id,\n      time: new Date(this.customTime.valueOf()),\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Stop moving operating.\n   * @param {Event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    if (!this.eventParams.dragging) return;\n\n    // fire a timechanged event\n    this.body.emitter.emit('timechanged', {\n      id: this.options.id,\n      time: new Date(this.customTime.valueOf()),\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform input operating.\n   * @param {Event} event\n   * @private\n   */\n  _onMarkerChange(event) {\n    this.body.emitter.emit('markerchange', {\n      id: this.options.id,\n      title: event.target.innerHTML,\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Perform change operating.\n   * @param {Event} event\n   * @private\n   */\n  _onMarkerChanged(event) {\n    this.body.emitter.emit('markerchanged', {\n      id: this.options.id,\n      title: event.target.innerHTML,\n      event\n    });\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Find a custom time from an event target:\n   * searches for the attribute 'custom-time' in the event target's element tree\n   * @param {Event} event\n   * @return {CustomTime | null} customTime\n   */\n  static customTimeFromTarget(event) {\n    let target = event.target;\n    while (target) {\n      if (target.hasOwnProperty('custom-time')) {\n        return target['custom-time'];\n      }\n      target = target.parentNode;\n    }\n\n    return null;\n  }\n}\n\nexport default CustomTime;\n","import Emitter from 'component-emitter';\nimport Hammer from '../module/hammer';\nimport * as hammerUtil from '../hammerUtil';\nimport util from '../util';\nimport TimeAxis from './component/TimeAxis';\nimport Activator from '../shared/Activator';\nimport * as DateUtil  from './DateUtil';\nimport CustomTime from './component/CustomTime';\n\nimport './component/css/animation.css';\nimport './component/css/currenttime.css';\nimport './component/css/panel.css';\nimport './component/css/pathStyles.css';\nimport './component/css/timeline.css';\nimport '../shared/bootstrap.css';\n\n/**\n * Create a timeline visualization\n * @constructor Core\n */\nclass Core {\n  /**\n   * Create the main DOM for the Core: a root panel containing left, right,\n   * top, bottom, content, and background panel.\n   * @param {Element} container  The container element where the Core will\n   *                             be attached.\n   * @protected\n   */\n  _create(container) {\n    this.dom = {};\n\n    this.dom.container = container;\n    this.dom.container.style.position = 'relative';\n\n    this.dom.root                 = document.createElement('div');\n    this.dom.background           = document.createElement('div');\n    this.dom.backgroundVertical   = document.createElement('div');\n    this.dom.backgroundHorizontal = document.createElement('div');\n    this.dom.centerContainer      = document.createElement('div');\n    this.dom.leftContainer        = document.createElement('div');\n    this.dom.rightContainer       = document.createElement('div');\n    this.dom.center               = document.createElement('div');\n    this.dom.left                 = document.createElement('div');\n    this.dom.right                = document.createElement('div');\n    this.dom.top                  = document.createElement('div');\n    this.dom.bottom               = document.createElement('div');\n    this.dom.shadowTop            = document.createElement('div');\n    this.dom.shadowBottom         = document.createElement('div');\n    this.dom.shadowTopLeft        = document.createElement('div');\n    this.dom.shadowBottomLeft     = document.createElement('div');\n    this.dom.shadowTopRight       = document.createElement('div');\n    this.dom.shadowBottomRight    = document.createElement('div');\n    this.dom.rollingModeBtn       = document.createElement('div');\n    this.dom.loadingScreen        = document.createElement('div');\n\n    this.dom.root.className                 = 'vis-timeline';\n    this.dom.background.className           = 'vis-panel vis-background';\n    this.dom.backgroundVertical.className   = 'vis-panel vis-background vis-vertical';\n    this.dom.backgroundHorizontal.className = 'vis-panel vis-background vis-horizontal';\n    this.dom.centerContainer.className      = 'vis-panel vis-center';\n    this.dom.leftContainer.className        = 'vis-panel vis-left';\n    this.dom.rightContainer.className       = 'vis-panel vis-right';\n    this.dom.top.className                  = 'vis-panel vis-top';\n    this.dom.bottom.className               = 'vis-panel vis-bottom';\n    this.dom.left.className                 = 'vis-content';\n    this.dom.center.className               = 'vis-content';\n    this.dom.right.className                = 'vis-content';\n    this.dom.shadowTop.className            = 'vis-shadow vis-top';\n    this.dom.shadowBottom.className         = 'vis-shadow vis-bottom';\n    this.dom.shadowTopLeft.className        = 'vis-shadow vis-top';\n    this.dom.shadowBottomLeft.className     = 'vis-shadow vis-bottom';\n    this.dom.shadowTopRight.className       = 'vis-shadow vis-top';\n    this.dom.shadowBottomRight.className    = 'vis-shadow vis-bottom';\n    this.dom.rollingModeBtn.className       = 'vis-rolling-mode-btn';\n    this.dom.loadingScreen.className        = 'vis-loading-screen';\n\n    this.dom.root.appendChild(this.dom.background);\n    this.dom.root.appendChild(this.dom.backgroundVertical);\n    this.dom.root.appendChild(this.dom.backgroundHorizontal);\n    this.dom.root.appendChild(this.dom.centerContainer);\n    this.dom.root.appendChild(this.dom.leftContainer);\n    this.dom.root.appendChild(this.dom.rightContainer);\n    this.dom.root.appendChild(this.dom.top);\n    this.dom.root.appendChild(this.dom.bottom);\n    this.dom.root.appendChild(this.dom.rollingModeBtn);\n\n    this.dom.centerContainer.appendChild(this.dom.center);\n    this.dom.leftContainer.appendChild(this.dom.left);\n    this.dom.rightContainer.appendChild(this.dom.right);\n    this.dom.centerContainer.appendChild(this.dom.shadowTop);\n    this.dom.centerContainer.appendChild(this.dom.shadowBottom);\n    this.dom.leftContainer.appendChild(this.dom.shadowTopLeft);\n    this.dom.leftContainer.appendChild(this.dom.shadowBottomLeft);\n    this.dom.rightContainer.appendChild(this.dom.shadowTopRight);\n    this.dom.rightContainer.appendChild(this.dom.shadowBottomRight);\n    \n    // size properties of each of the panels\n    this.props = {\n      root: {},\n      background: {},\n      centerContainer: {},\n      leftContainer: {},\n      rightContainer: {},\n      center: {},\n      left: {},\n      right: {},\n      top: {},\n      bottom: {},\n      border: {},\n      scrollTop: 0,\n      scrollTopMin: 0\n    };\n\n    this.on('rangechange', () => {\n      if (this.initialDrawDone === true) {\n        this._redraw();\n      }\n    });\n    this.on('rangechanged', () => {\n      if (!this.initialRangeChangeDone) {\n        this.initialRangeChangeDone = true;\n      }\n    });\n    this.on('touch',       this._onTouch.bind(this));\n    this.on('panmove',         this._onDrag.bind(this));\n\n    const me = this;\n    this._origRedraw = this._redraw.bind(this);\n    this._redraw = util.throttle(this._origRedraw);\n\n    this.on('_change', properties => {\n      if (me.itemSet && me.itemSet.initialItemSetDrawn && properties && properties.queue == true) {\n        me._redraw()\n      } else {\n        me._origRedraw();\n      }\n    });\n\n    // create event listeners for all interesting events, these events will be\n    // emitted via emitter\n    this.hammer = new Hammer(this.dom.root);\n    const pinchRecognizer = this.hammer.get('pinch').set({enable: true});\n    pinchRecognizer && hammerUtil.disablePreventDefaultVertically(pinchRecognizer);\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_ALL});\n    this.listeners = {};\n\n    const events = [\n      'tap', 'doubletap', 'press',\n      'pinch',\n      'pan', 'panstart', 'panmove', 'panend'\n        // TODO: cleanup\n      //'touch', 'pinch',\n      //'tap', 'doubletap', 'hold',\n      //'dragstart', 'drag', 'dragend',\n      //'mousewheel', 'DOMMouseScroll' // DOMMouseScroll is needed for Firefox\n    ];\n    events.forEach(type => {\n      const listener = event => {\n        if (me.isActive()) {\n          me.emit(type, event);\n        }\n      };\n      me.hammer.on(type, listener);\n      me.listeners[type] = listener;\n    });\n\n    // emulate a touch event (emitted before the start of a pan, pinch, tap, or press)\n    hammerUtil.onTouch(this.hammer, event => {\n      me.emit('touch', event);\n    });\n\n    // emulate a release event (emitted after a pan, pinch, tap, or press)\n    hammerUtil.onRelease(this.hammer, event => {\n      me.emit('release', event);\n    });\n\n    /**\n     *\n     * @param {WheelEvent} event\n     */\n    function onMouseWheel(event) {\n\n      // Reasonable default wheel deltas\n      const LINE_HEIGHT = 40;\n      const PAGE_HEIGHT = 800;\n\n      if (this.isActive()) {\n        this.emit('mousewheel', event);\n      }\n\n      // deltaX and deltaY normalization from jquery.mousewheel.js\n      let deltaX = 0;\n      let deltaY = 0;\n\n      // Old school scrollwheel delta\n      if ( 'detail'      in event ) { deltaY = event.detail * -1;      }\n      if ( 'wheelDelta'  in event ) { deltaY = event.wheelDelta;       }\n      if ( 'wheelDeltaY' in event ) { deltaY = event.wheelDeltaY;      }\n      if ( 'wheelDeltaX' in event ) { deltaX = event.wheelDeltaX * -1; }\n\n      // Firefox < 17 horizontal scrolling related to DOMMouseScroll event\n      if ( 'axis' in event && event.axis === event.HORIZONTAL_AXIS ) {\n        deltaX = deltaY * -1;\n        deltaY = 0;\n      }\n\n      // New school wheel delta (wheel event)\n      if ( 'deltaY' in event ) {\n        deltaY = event.deltaY * -1;\n      }\n      if ( 'deltaX' in event ) {\n        deltaX = event.deltaX;\n      }\n\n      // Normalize deltas\n      if (event.deltaMode) {\n          if (event.deltaMode === 1) {   // delta in LINE units\n            deltaX *= LINE_HEIGHT;\n            deltaY *= LINE_HEIGHT;\n          } else {                       // delta in PAGE units\n            deltaX *= LINE_HEIGHT;\n            deltaY *= PAGE_HEIGHT;\n          }\n        }\n        // Prevent scrolling when zooming (no zoom key, or pressing zoom key)\n      if (this.options.preferZoom) {\n        if (!this.options.zoomKey || event[this.options.zoomKey]) return;\n      } else {\n        if (this.options.zoomKey && event[this.options.zoomKey]) return\n      }\n      // Don't preventDefault if you can't scroll\n      if (!this.options.verticalScroll && !this.options.horizontalScroll) return;\n\n      if (this.options.verticalScroll && Math.abs(deltaY) >= Math.abs(deltaX)) {\n        const current = this.props.scrollTop;\n        const adjusted = current + deltaY;\n\n        if (this.isActive()) {\n          const newScrollTop = this._setScrollTop(adjusted);\n\n          if (newScrollTop !== current) {\n            this._redraw();\n            this.emit('scroll', event);\n\n            // Prevent default actions caused by mouse wheel\n            // (else the page and timeline both scroll)\n            event.preventDefault();\n          }\n        }\n      } else if (this.options.horizontalScroll) {\n        const delta = Math.abs(deltaX) >= Math.abs(deltaY) ? deltaX : deltaY;\n\n        // calculate a single scroll jump relative to the range scale\n        const diff = (delta / 120) * (this.range.end - this.range.start) / 20;\n        // calculate new start and end\n        const newStart = this.range.start + diff;\n        const newEnd = this.range.end + diff;\n\n        const options = {\n          animation: false,\n          byUser: true,\n          event\n        };\n        this.range.setRange(newStart, newEnd, options);\n\n        event.preventDefault();\n      }\n    }\n\n    // Add modern wheel event listener\n    const wheelType = \"onwheel\" in document.createElement(\"div\") ? \"wheel\" : // Modern browsers support \"wheel\"\n    document.onmousewheel !== undefined ? \"mousewheel\" : // Webkit and IE support at least \"mousewheel\"\n    \n    // DOMMouseScroll - Older Firefox versions use \"DOMMouseScroll\"\n    // onmousewheel - All the use \"onmousewheel\"\n    this.dom.centerContainer.addEventListener ? \"DOMMouseScroll\" :  \"onmousewheel\"\n    this.dom.top.addEventListener ? \"DOMMouseScroll\" : \"onmousewheel\"\n    this.dom.bottom.addEventListener ? \"DOMMouseScroll\" : \"onmousewheel\"\n    this.dom.centerContainer.addEventListener(wheelType, onMouseWheel.bind(this), false);\n    this.dom.top.addEventListener(wheelType, onMouseWheel.bind(this), false);\n    this.dom.bottom.addEventListener(wheelType, onMouseWheel.bind(this), false);\n\n\n    /**\n     *\n     * @param {scroll} event\n     */\n    function onMouseScrollSide(event) {\n      if (!me.options.verticalScroll) return;\n      \n      if (me._isProgramaticallyScrolled) {\n        me._isProgramaticallyScrolled = false;\n        return;\n      }\n      \n      event.preventDefault();\n      if (me.isActive()) {\n        const adjusted = -event.target.scrollTop;\n        me._setScrollTop(adjusted);\n        me._redraw();\n        me.emit('scrollSide', event);\n      }\n    }\n\n    this.dom.left.parentNode.addEventListener('scroll', onMouseScrollSide.bind(this));\n    this.dom.right.parentNode.addEventListener('scroll', onMouseScrollSide.bind(this));\n\n    let itemAddedToTimeline = false;\n\n    /**\n     *\n     * @param {dragover} event\n     * @returns {boolean}\n     */\n    function handleDragOver(event) {\n      if (event.preventDefault) {\n        me.emit('dragover', me.getEventProperties(event));\n        event.preventDefault(); // Necessary. Allows us to drop.\n      }\n\n      // make sure your target is a timeline element\n      if (!(event.target.className.indexOf(\"timeline\") > -1)) return;\n\n      // make sure only one item is added every time you're over the timeline\n      if (itemAddedToTimeline) return;\n\n      event.dataTransfer.dropEffect = 'move';  \n      itemAddedToTimeline = true;\n      return false;\n    }\n\n    /**\n     *\n     * @param {drop} event\n     * @returns {boolean}\n     */\n    function handleDrop(event) {\n      // prevent redirect to blank page - Firefox\n      if(event.preventDefault) { event.preventDefault(); }\n      if(event.stopPropagation) { event.stopPropagation(); }\n      // return when dropping non-timeline items\n      try {\n        var itemData = JSON.parse(event.dataTransfer.getData(\"text\"))\n        if (!itemData || !itemData.content) return\n      } catch (err) {\n        return false;\n      }\n\n      itemAddedToTimeline = false;\n      event.center = {\n        x: event.clientX,\n        y: event.clientY\n      };\n\n      if (itemData.target !== 'item') {\n        me.itemSet._onAddItem(event);\n      } else {\n        me.itemSet._onDropObjectOnItem(event);\n      }\n      me.emit('drop', me.getEventProperties(event))\n      return false;\n    }\n\n    this.dom.center.addEventListener('dragover', handleDragOver.bind(this), false);\n    this.dom.center.addEventListener('drop', handleDrop.bind(this), false);\n\n    this.customTimes = [];\n\n    // store state information needed for touch events\n    this.touch = {};\n\n    this.redrawCount = 0;\n    this.initialDrawDone = false;\n    this.initialRangeChangeDone = false;\n\n    // attach the root panel to the provided container\n    if (!container) throw new Error('No container provided');\n    container.appendChild(this.dom.root);\n    container.appendChild(this.dom.loadingScreen);\n  }\n\n  /**\n   * Set options. Options will be passed to all components loaded in the Timeline.\n   * @param {Object} [options]\n   *                           {String} orientation\n   *                              Vertical orientation for the Timeline,\n   *                              can be 'bottom' (default) or 'top'.\n   *                           {string | number} width\n   *                              Width for the timeline, a number in pixels or\n   *                              a css string like '1000px' or '75%'. '100%' by default.\n   *                           {string | number} height\n   *                              Fixed height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'. If undefined,\n   *                              The Timeline will automatically size such that\n   *                              its contents fit.\n   *                           {string | number} minHeight\n   *                              Minimum height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'.\n   *                           {string | number} maxHeight\n   *                              Maximum height for the Timeline, a number in pixels or\n   *                              a css string like '400px' or '75%'.\n   *                           {number | Date | string} start\n   *                              Start date for the visible window\n   *                           {number | Date | string} end\n   *                              End date for the visible window\n   */\n  setOptions(options) {\n    if (options) {\n      // copy the known options\n      const fields = [\n        'width', 'height', 'minHeight', 'maxHeight', 'autoResize',\n        'start', 'end', 'clickToUse', 'dataAttributes', 'hiddenDates',\n        'locale', 'locales', 'moment', 'preferZoom', 'rtl', 'zoomKey',\n        'horizontalScroll', 'verticalScroll', 'longSelectPressTime'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n      this.dom.rollingModeBtn.style.visibility = 'hidden';\n\n      if (this.options.rtl) {\n        this.dom.container.style.direction = \"rtl\";\n        this.dom.backgroundVertical.className = 'vis-panel vis-background vis-vertical-rtl';    \n      }\n\n      if (this.options.verticalScroll) {\n        if (this.options.rtl) {\n          this.dom.rightContainer.className = 'vis-panel vis-right vis-vertical-scroll';\n        } else {\n          this.dom.leftContainer.className = 'vis-panel vis-left vis-vertical-scroll';\n        }\n      }\n\n      if (typeof this.options.orientation !== 'object') {\n        this.options.orientation = {item:undefined,axis:undefined};\n      }\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation = {\n            item: options.orientation,\n            axis: options.orientation\n          };\n        }\n        else if (typeof options.orientation === 'object') {\n          if ('item' in options.orientation) {\n            this.options.orientation.item = options.orientation.item;\n          }\n          if ('axis' in options.orientation) {\n            this.options.orientation.axis = options.orientation.axis;\n          }\n        }\n      }\n\n      if (this.options.orientation.axis === 'both') {\n        if (!this.timeAxis2) {\n          const timeAxis2 = this.timeAxis2 = new TimeAxis(this.body);\n          timeAxis2.setOptions = options => {\n            const _options = options ? util.extend({}, options) : {};\n            _options.orientation = 'top'; // override the orientation option, always top\n            TimeAxis.prototype.setOptions.call(timeAxis2, _options);\n          };\n          this.components.push(timeAxis2);\n        }\n      }\n      else {\n        if (this.timeAxis2) {\n          const index = this.components.indexOf(this.timeAxis2);\n          if (index !== -1) {\n            this.components.splice(index, 1);\n          }\n          this.timeAxis2.destroy();\n          this.timeAxis2 = null;\n        }\n      }\n\n      // if the graph2d's drawPoints is a function delegate the callback to the onRender property\n      if (typeof options.drawPoints == 'function') {\n        options.drawPoints = {\n            onRender: options.drawPoints\n        };\n      }\n\n      if ('hiddenDates' in this.options) {\n        DateUtil.convertHiddenOptions(this.options.moment, this.body, this.options.hiddenDates);\n      }\n\n      if ('clickToUse' in options) {\n        if (options.clickToUse) {\n          if (!this.activator) {\n            this.activator = new Activator(this.dom.root);\n          }\n        }\n        else {\n          if (this.activator) {\n            this.activator.destroy();\n            delete this.activator;\n          }\n        }\n      }\n\n      // enable/disable autoResize\n      this._initAutoResize();\n    }\n\n    // propagate options to all components\n    this.components.forEach(component => component.setOptions(options));\n\n    // enable/disable configure\n    if ('configure' in options) {\n      if (!this.configurator) {\n        this.configurator = this._createConfigurator();\n      }\n\n      this.configurator.setOptions(options.configure);\n\n      // collect the settings of all components, and pass them to the configuration system\n      const appliedOptions = util.deepExtend({}, this.options);\n      this.components.forEach(component => {\n        util.deepExtend(appliedOptions, component.options);\n      });\n      this.configurator.setModuleOptions({global: appliedOptions});\n    }\n\n    this._redraw();\n  }\n\n  /**\n   * Returns true when the Timeline is active.\n   * @returns {boolean}\n   */\n  isActive() {\n    return !this.activator || this.activator.active;\n  }\n\n  /**\n   * Destroy the Core, clean up all DOM elements and event listeners.\n   */\n  destroy() {\n    // unbind datasets\n    this.setItems(null);\n    this.setGroups(null);\n\n    // remove all event listeners\n    this.off();\n\n    // stop checking for changed size\n    this._stopAutoResize();\n\n    // remove from DOM\n    if (this.dom.root.parentNode) {\n      this.dom.root.parentNode.removeChild(this.dom.root);\n    }\n    this.dom = null;\n\n    // remove Activator\n    if (this.activator) {\n      this.activator.destroy();\n      delete this.activator;\n    }\n\n    // cleanup hammer touch events\n    for (const event in this.listeners) {\n      if (this.listeners.hasOwnProperty(event)) {\n        delete this.listeners[event];\n      }\n    }\n    this.listeners = null;\n    this.hammer && this.hammer.destroy();\n    this.hammer = null;\n\n    // give all components the opportunity to cleanup\n    this.components.forEach(component => component.destroy());\n\n    this.body = null;\n  }\n\n  /**\n   * Set a custom time bar\n   * @param {Date} time\n   * @param {number} [id=undefined] Optional id of the custom time bar to be adjusted.\n   */\n  setCustomTime(time, id) {\n    const customTimes = this.customTimes.filter(component => id === component.options.id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n\n    if (customTimes.length > 0) {\n      customTimes[0].setCustomTime(time);\n    }\n  }\n\n  /**\n   * Retrieve the current custom time.\n   * @param {number} [id=undefined]    Id of the custom time bar.\n   * @return {Date | undefined} customTime\n   */\n  getCustomTime(id) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    return customTimes[0].getCustomTime();\n  }\n\n  /**\n   * Set a custom marker for the custom time bar.\n   * @param {string} [title] Title of the custom marker.\n   * @param {number} [id=undefined] Id of the custom marker.\n   * @param {boolean} [editable=false] Make the custom marker editable.\n   */\n  setCustomTimeMarker(title, id, editable) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    if (customTimes.length > 0) {\n      customTimes[0].setCustomMarker(title, editable);\n    }\n  }\n\n  /**\n   * Set a custom title for the custom time bar.\n   * @param {string} [title] Custom title\n   * @param {number} [id=undefined]    Id of the custom time bar.\n   * @returns {*}\n   */\n  setCustomTimeTitle(title, id) {\n    const customTimes = this.customTimes.filter(component => component.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n    if (customTimes.length > 0) {\n      return customTimes[0].setCustomTitle(title);\n    }\n  }\n\n  /**\n   * Retrieve meta information from an event.\n   * Should be overridden by classes extending Core\n   * @param {Event} event\n   * @return {Object} An object with related information.\n   */\n  getEventProperties(event) {\n    return { event };\n  }\n\n  /**\n   * Add custom vertical bar\n   * @param {Date | string | number} [time]  A Date, unix timestamp, or\n   *                                         ISO date string. Time point where\n   *                                         the new bar should be placed.\n   *                                         If not provided, `new Date()` will\n   *                                         be used.\n   * @param {number | string} [id=undefined] Id of the new bar. Optional\n   * @return {number | string}               Returns the id of the new bar\n   */\n  addCustomTime(time, id) {\n    const timestamp = time !== undefined\n        ? util.convert(time, 'Date')\n        : new Date();\n\n    const exists = this.customTimes.some(customTime => customTime.options.id === id);\n    if (exists) {\n      throw new Error(`A custom time with id ${JSON.stringify(id)} already exists`);\n    }\n\n    const customTime = new CustomTime(this.body, util.extend({}, this.options, {\n      time : timestamp,\n      id,\n      snap: this.itemSet.options.snap\n    }));\n\n    this.customTimes.push(customTime);\n    this.components.push(customTime);\n    this._redraw();\n\n    return id;\n  }\n\n  /**\n   * Remove previously added custom bar\n   * @param {int} id ID of the custom bar to be removed\n   * [at]returns {boolean} True if the bar exists and is removed, false otherwise\n   */\n  removeCustomTime(id) {\n    const customTimes = this.customTimes.filter(bar => bar.options.id === id);\n\n    if (customTimes.length === 0) {\n      throw new Error(`No custom time bar found with id ${JSON.stringify(id)}`)\n    }\n\n    customTimes.forEach(customTime => {\n      this.customTimes.splice(this.customTimes.indexOf(customTime), 1);\n      this.components.splice(this.components.indexOf(customTime), 1);\n      customTime.destroy();\n    })\n  }\n\n  /**\n   * Get the id's of the currently visible items.\n   * @returns {Array} The ids of the visible items\n   */\n  getVisibleItems() {\n    return this.itemSet && this.itemSet.getVisibleItems() || [];\n  }\n\n  /**\n   * Get the id's of the currently visible groups.\n   * @returns {Array} The ids of the visible groups\n   */\n  getVisibleGroups() {\n    return this.itemSet && this.itemSet.getVisibleGroups() || [];\n  }\n\n  /**\n   * Set Core window such that it fits all items\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  fit(options, callback) {\n    const range = this.getDataRange();\n\n    // skip range set if there is no min and max date\n    if (range.min === null && range.max === null) {\n      return;\n    }\n\n    // apply a margin of 1% left and right of the data\n    const interval = range.max - range.min;\n    const min = new Date(range.min.valueOf() - interval * 0.01);\n    const max = new Date(range.max.valueOf() + interval * 0.01);\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n    this.range.setRange(min, max, { animation }, callback);\n  }\n\n  /**\n   * Calculate the data range of the items start and end dates\n   * [at]returns {{min: [Date], max: [Date]}}\n   * @protected\n   */\n  getDataRange() {\n    // must be implemented by Timeline and Graph2d\n    throw new Error('Cannot invoke abstract method getDataRange');\n  }\n\n  /**\n   * Set the visible window. Both parameters are optional, you can change only\n   * start or only end. Syntax:\n   *\n   *     TimeLine.setWindow(start, end)\n   *     TimeLine.setWindow(start, end, options)\n   *     TimeLine.setWindow(range)\n   *\n   * Where start and end can be a Date, number, or string, and range is an\n   * object with properties start and end.\n   *\n   * @param {Date | number | string | Object} [start] Start date of visible window\n   * @param {Date | number | string} [end]            End date of visible window\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  setWindow(start, end, options, callback) {\n    if (typeof arguments[2] == \"function\") {\n      callback = arguments[2];\n      options = {};\n    }\n    let animation;\n    let range;\n    if (arguments.length == 1) {\n      range = arguments[0];\n      animation = (range.animation !== undefined) ? range.animation : true;\n      this.range.setRange(range.start, range.end, { animation });\n    }\n    else if (arguments.length == 2 && typeof arguments[1] == \"function\") {\n      range = arguments[0];\n      callback = arguments[1];\n      animation = (range.animation !== undefined) ? range.animation : true;\n      this.range.setRange(range.start, range.end, { animation }, callback);\n    }\n    else {\n      animation = (options && options.animation !== undefined) ? options.animation : true;\n      this.range.setRange(start, end, { animation }, callback);\n    }\n  }\n\n  /**\n   * Move the window such that given time is centered on screen.\n   * @param {Date | number | string} time\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  moveTo(time, options, callback) {\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const interval = this.range.end - this.range.start;\n    const t = util.convert(time, 'Date').valueOf();\n\n    const start = t - interval / 2;\n    const end = t + interval / 2;\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n\n    this.range.setRange(start, end, { animation }, callback);\n  }\n\n  /**\n   * Get the visible window\n   * @return {{start: Date, end: Date}}   Visible range\n   */\n  getWindow() {\n    const range = this.range.getRange();\n    return {\n      start: new Date(range.start),\n      end: new Date(range.end)\n    };\n  }\n\n  /**\n   * Zoom in the window such that given time is centered on screen.\n   * @param {number} percentage - must be between [0..1]\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  zoomIn(percentage, options, callback) {\n    if (!percentage || percentage < 0 || percentage > 1) return;\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const range = this.getWindow();\n    const start = range.start.valueOf();\n    const end = range.end.valueOf();\n    const interval = end - start;\n    const newInterval = interval / (1 + percentage);\n    const distance = (interval - newInterval) / 2;\n    const newStart = start + distance;\n    const newEnd = end - distance;\n\n    this.setWindow(newStart, newEnd, options, callback);\n  }\n\n  /**\n   * Zoom out the window such that given time is centered on screen.\n   * @param {number} percentage - must be between [0..1]\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback] a callback funtion to be executed at the end of this function\n   */\n  zoomOut(percentage, options, callback) {\n    if (!percentage || percentage < 0 || percentage > 1) return\n    if (typeof arguments[1] == \"function\") {\n      callback = arguments[1];\n      options = {};\n    }\n    const range = this.getWindow();\n    const start = range.start.valueOf();\n    const end = range.end.valueOf();\n    const interval = end - start;\n    const newStart = start - interval * percentage / 2;\n    const newEnd = end + interval * percentage / 2;\n\n    this.setWindow(newStart, newEnd, options, callback);\n  }\n\n  /**\n   * Force a redraw. Can be overridden by implementations of Core\n   *\n   * Note: this function will be overridden on construction with a trottled version\n   */\n  redraw() {\n    this._redraw();\n  }\n\n  /**\n   * Redraw for internal use. Redraws all components. See also the public\n   * method redraw.\n   * @protected\n   */\n  _redraw() {\n    this.redrawCount++;\n    const dom = this.dom;\n    \n    if (!dom || !dom.container || dom.root.offsetWidth == 0) return; // when destroyed, or invisible\n    \n    let resized = false;\n    const options = this.options;\n    const props = this.props;\n\n    DateUtil.updateHiddenDates(this.options.moment, this.body, this.options.hiddenDates);\n\n    // update class names\n    if (options.orientation == 'top') {\n      util.addClassName(dom.root, 'vis-top');\n      util.removeClassName(dom.root, 'vis-bottom');\n    }\n    else {\n      util.removeClassName(dom.root, 'vis-top');\n      util.addClassName(dom.root, 'vis-bottom');\n    }\n\n    if (options.rtl) {\n      util.addClassName(dom.root, 'vis-rtl');\n      util.removeClassName(dom.root, 'vis-ltr');\n    } else {\n      util.addClassName(dom.root, 'vis-ltr');\n      util.removeClassName(dom.root, 'vis-rtl');\n    }\n\n    // update root width and height options\n    dom.root.style.maxHeight = util.option.asSize(options.maxHeight, '');\n    dom.root.style.minHeight = util.option.asSize(options.minHeight, '');\n    dom.root.style.width = util.option.asSize(options.width, '');\n    const rootOffsetWidth = dom.root.offsetWidth;\n\n    // calculate border widths\n    props.border.left = 1\n    props.border.right = 1\n    props.border.top = 1\n    props.border.bottom = 1\n\n    // calculate the heights. If any of the side panels is empty, we set the height to\n    // minus the border width, such that the border will be invisible\n    props.center.height = dom.center.offsetHeight;\n    props.left.height   = dom.left.offsetHeight;\n    props.right.height  = dom.right.offsetHeight;\n    props.top.height    = dom.top.clientHeight    || -props.border.top;\n    props.bottom.height = dom.bottom.clientHeight || -props.border.bottom;\n\n    // TODO: compensate borders when any of the panels is empty.\n\n    // apply auto height\n    // TODO: only calculate autoHeight when needed (else we cause an extra reflow/repaint of the DOM)\n    const contentHeight = Math.max(props.left.height, props.center.height, props.right.height);\n    const autoHeight = props.top.height + contentHeight + props.bottom.height + props.border.top + props.border.bottom;\n    dom.root.style.height = util.option.asSize(options.height, `${autoHeight}px`);\n\n    // calculate heights of the content panels\n    props.root.height = dom.root.offsetHeight;\n    props.background.height = props.root.height;\n    const containerHeight = props.root.height - props.top.height - props.bottom.height;\n    props.centerContainer.height  = containerHeight;\n    props.leftContainer.height    = containerHeight;\n    props.rightContainer.height   = props.leftContainer.height;\n\n    // calculate the widths of the panels\n    props.root.width = rootOffsetWidth;\n    props.background.width = props.root.width;\n\n    if (!this.initialDrawDone) {\n      props.scrollbarWidth = util.getScrollBarWidth();\n    }\n\n    const leftContainerClientWidth = dom.leftContainer.clientWidth;\n    const rightContainerClientWidth = dom.rightContainer.clientWidth;\n\n    if (options.verticalScroll) {\n      if (options.rtl) {\n        props.left.width = leftContainerClientWidth || -props.border.left;\n        props.right.width = rightContainerClientWidth + props.scrollbarWidth || -props.border.right;\n      } else {\n        props.left.width = leftContainerClientWidth + props.scrollbarWidth || -props.border.left;\n        props.right.width = rightContainerClientWidth || -props.border.right;\n      }\n    } else {\n      props.left.width = leftContainerClientWidth  || -props.border.left;\n      props.right.width = rightContainerClientWidth || -props.border.right;\n    }\n\n    this._setDOM();\n\n    // update the scrollTop, feasible range for the offset can be changed\n    // when the height of the Core or of the contents of the center changed\n    let offset = this._updateScrollTop();\n\n    // reposition the scrollable contents\n    if (options.orientation.item != 'top') {\n      offset += Math.max(props.centerContainer.height - props.center.height -\n        props.border.top - props.border.bottom, 0);\n    }\n    dom.center.style.transform  = `translateY(${offset}px)`;\n\n    // show shadows when vertical scrolling is available\n    const visibilityTop = props.scrollTop == 0 ? 'hidden' : '';\n    const visibilityBottom = props.scrollTop == props.scrollTopMin ? 'hidden' : '';\n    dom.shadowTop.style.visibility          = visibilityTop;\n    dom.shadowBottom.style.visibility       = visibilityBottom;\n    dom.shadowTopLeft.style.visibility      = visibilityTop;\n    dom.shadowBottomLeft.style.visibility   = visibilityBottom;\n    dom.shadowTopRight.style.visibility     = visibilityTop;\n    dom.shadowBottomRight.style.visibility  = visibilityBottom;\n\n    if (options.verticalScroll) {\n      dom.rightContainer.className = 'vis-panel vis-right vis-vertical-scroll';\n      dom.leftContainer.className = 'vis-panel vis-left vis-vertical-scroll';\n\n      dom.shadowTopRight.style.visibility     = \"hidden\";\n      dom.shadowBottomRight.style.visibility  = \"hidden\";\n      dom.shadowTopLeft.style.visibility      = \"hidden\";\n      dom.shadowBottomLeft.style.visibility   = \"hidden\";\n\n      dom.left.style.top    = '0px';\n      dom.right.style.top   = '0px';\n    }\n\n    if (!options.verticalScroll ||  props.center.height < props.centerContainer.height) {\n      dom.left.style.top    = `${offset}px`;\n      dom.right.style.top   = `${offset}px`;\n      dom.rightContainer.className = dom.rightContainer.className.replace(new RegExp('(?:^|\\\\s)'+ 'vis-vertical-scroll' + '(?:\\\\s|$)'), ' ');\n      dom.leftContainer.className = dom.leftContainer.className.replace(new RegExp('(?:^|\\\\s)'+ 'vis-vertical-scroll' + '(?:\\\\s|$)'), ' ');\n      props.left.width = leftContainerClientWidth  || -props.border.left;\n      props.right.width = rightContainerClientWidth || -props.border.right;\n      this._setDOM();\n    }\n\n    // enable/disable vertical panning\n    const contentsOverflow = props.center.height > props.centerContainer.height;\n    this.hammer.get('pan').set({\n      direction: contentsOverflow ? Hammer.DIRECTION_ALL : Hammer.DIRECTION_HORIZONTAL\n    });\n\n    // set the long press time\n    this.hammer.get('press').set({\n      time: this.options.longSelectPressTime\n    });\n\n    // redraw all components\n    this.components.forEach(component => {\n      resized = component.redraw() || resized;\n    });\n    const MAX_REDRAW = 5;\n    if (resized) {\n      if (this.redrawCount < MAX_REDRAW) {\n        this.body.emitter.emit('_change');\n        return;\n      }\n      else {\n        console.log('WARNING: infinite loop in redraw?');\n      }\n    } else {\n      this.redrawCount = 0;\n    }\n\n    //Emit public 'changed' event for UI updates, see issue #1592\n    this.body.emitter.emit(\"changed\");\n  }\n\n  /**\n   * sets the basic DOM components needed for the timeline\\graph2d\n   */\n  _setDOM() {\n    const props = this.props;\n    const dom = this.dom;\n\n    props.leftContainer.width = props.left.width;\n    props.rightContainer.width = props.right.width;\n    const centerWidth = props.root.width - props.left.width - props.right.width;\n    props.center.width          = centerWidth;\n    props.centerContainer.width = centerWidth;\n    props.top.width             = centerWidth;\n    props.bottom.width          = centerWidth;\n\n    // resize the panels\n    dom.background.style.height           = `${props.background.height}px`;\n    dom.backgroundVertical.style.height   = `${props.background.height}px`;\n    dom.backgroundHorizontal.style.height = `${props.centerContainer.height}px`;\n    dom.centerContainer.style.height      = `${props.centerContainer.height}px`;\n    dom.leftContainer.style.height        = `${props.leftContainer.height}px`;\n    dom.rightContainer.style.height       = `${props.rightContainer.height}px`;\n\n    dom.background.style.width            = `${props.background.width}px`;\n    dom.backgroundVertical.style.width    = `${props.centerContainer.width}px`;\n    dom.backgroundHorizontal.style.width  = `${props.background.width}px`;\n    dom.centerContainer.style.width       = `${props.center.width}px`;\n    dom.top.style.width                   = `${props.top.width}px`;\n    dom.bottom.style.width                = `${props.bottom.width}px`;\n\n    // reposition the panels\n    dom.background.style.left           = '0';\n    dom.background.style.top            = '0';\n    dom.backgroundVertical.style.left   = `${props.left.width + props.border.left}px`;\n    dom.backgroundVertical.style.top    = '0';\n    dom.backgroundHorizontal.style.left = '0';\n    dom.backgroundHorizontal.style.top  = `${props.top.height}px`;\n    dom.centerContainer.style.left      = `${props.left.width}px`;\n    dom.centerContainer.style.top       = `${props.top.height}px`;\n    dom.leftContainer.style.left        = '0';\n    dom.leftContainer.style.top         = `${props.top.height}px`;\n    dom.rightContainer.style.left       = `${props.left.width + props.center.width}px`;\n    dom.rightContainer.style.top        = `${props.top.height}px`;\n    dom.top.style.left                  = `${props.left.width}px`;\n    dom.top.style.top                   = '0';\n    dom.bottom.style.left               = `${props.left.width}px`;\n    dom.bottom.style.top                = `${props.top.height + props.centerContainer.height}px`;\n    dom.center.style.left               = '0';\n    dom.left.style.left                 = '0';\n    dom.right.style.left                = '0';\n  }\n\n  /**\n   * Set a current time. This can be used for example to ensure that a client's\n   * time is synchronized with a shared server time.\n   * Only applicable when option `showCurrentTime` is true.\n   * @param {Date | string | number} time     A Date, unix timestamp, or\n   *                                          ISO date string.\n   */\n  setCurrentTime(time) {\n    if (!this.currentTime) {\n      throw new Error('Option showCurrentTime must be true');\n    }\n\n    this.currentTime.setCurrentTime(time);\n  }\n\n  /**\n   * Get the current time.\n   * Only applicable when option `showCurrentTime` is true.\n   * @return {Date} Returns the current time.\n   */\n  getCurrentTime() {\n    if (!this.currentTime) {\n      throw new Error('Option showCurrentTime must be true');\n    }\n\n    return this.currentTime.getCurrentTime();\n  }\n\n  /**\n   * Convert a position on screen (pixels) to a datetime\n   * @param {int}     x    Position on the screen in pixels\n   * @return {Date}   time The datetime the corresponds with given position x\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toTime(x) {\n    return DateUtil.toTime(this, x, this.props.center.width);\n  }\n\n  /**\n   * Convert a position on the global screen (pixels) to a datetime\n   * @param {int}     x    Position on the screen in pixels\n   * @return {Date}   time The datetime the corresponds with given position x\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toGlobalTime(x) {\n    return DateUtil.toTime(this, x, this.props.root.width);\n    //var conversion = this.range.conversion(this.props.root.width);\n    //return new Date(x / conversion.scale + conversion.offset);\n  }\n\n  /**\n   * Convert a datetime (Date object) into a position on the screen\n   * @param {Date}   time A date\n   * @return {int}   x    The position on the screen in pixels which corresponds\n   *                      with the given date.\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toScreen(time) {\n    return DateUtil.toScreen(this, time, this.props.center.width);\n  }\n\n  /**\n   * Convert a datetime (Date object) into a position on the root\n   * This is used to get the pixel density estimate for the screen, not the center panel\n   * @param {Date}   time A date\n   * @return {int}   x    The position on root in pixels which corresponds\n   *                      with the given date.\n   * @protected\n   * TODO: move this function to Range\n   */\n  _toGlobalScreen(time) {\n    return DateUtil.toScreen(this, time, this.props.root.width);\n    //var conversion = this.range.conversion(this.props.root.width);\n    //return (time.valueOf() - conversion.offset) * conversion.scale;\n  }\n\n  /**\n   * Initialize watching when option autoResize is true\n   * @private\n   */\n  _initAutoResize() {\n    if (this.options.autoResize == true) {\n      this._startAutoResize();\n    }\n    else {\n      this._stopAutoResize();\n    }\n  }\n\n  /**\n   * Watch for changes in the size of the container. On resize, the Panel will\n   * automatically redraw itself.\n   * @private\n   */\n  _startAutoResize() {\n    const me = this;\n\n    this._stopAutoResize();\n\n    this._onResize = () => {\n      if (me.options.autoResize != true) {\n        // stop watching when the option autoResize is changed to false\n        me._stopAutoResize();\n        return;\n      }\n\n      if (me.dom.root) {\n        const rootOffsetHeight = me.dom.root.offsetHeight;\n        const rootOffsetWidth = me.dom.root.offsetWidth;\n        // check whether the frame is resized\n        // Note: we compare offsetWidth here, not clientWidth. For some reason,\n        // IE does not restore the clientWidth from 0 to the actual width after\n        // changing the timeline's container display style from none to visible\n        if ((rootOffsetWidth != me.props.lastWidth) ||\n          (rootOffsetHeight != me.props.lastHeight)) {\n          me.props.lastWidth = rootOffsetWidth;\n          me.props.lastHeight = rootOffsetHeight;\n          me.props.scrollbarWidth = util.getScrollBarWidth();\n\n          me.body.emitter.emit('_change');\n        }\n      }\n    };\n\n    // add event listener to window resize\n    util.addEventListener(window, 'resize', this._onResize);\n\n    //Prevent initial unnecessary redraw\n    if (me.dom.root) {\n        me.props.lastWidth = me.dom.root.offsetWidth;\n        me.props.lastHeight = me.dom.root.offsetHeight;\n    }\n\n    this.watchTimer = setInterval(this._onResize, 1000);\n  }\n\n  /**\n   * Stop watching for a resize of the frame.\n   * @private\n   */\n  _stopAutoResize() {\n    if (this.watchTimer) {\n      clearInterval(this.watchTimer);\n      this.watchTimer = undefined;\n    }\n\n    // remove event listener on window.resize\n    if (this._onResize) {\n      util.removeEventListener(window, 'resize', this._onResize);\n      this._onResize = null;\n    }\n  }\n\n  /**\n   * Start moving the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {  // eslint-disable-line no-unused-vars\n    this.touch.allowDragging = true;\n    this.touch.initialScrollTop = this.props.scrollTop;\n  }\n\n  /**\n   * Start moving the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onPinch(event) {  // eslint-disable-line no-unused-vars\n    this.touch.allowDragging = false;\n  }\n\n  /**\n   * Move the timeline vertically\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (!event) return\n    // refuse to drag when we where pinching to prevent the timeline make a jump\n    // when releasing the fingers in opposite order from the touch screen\n    if (!this.touch.allowDragging) return;\n\n    const delta = event.deltaY;\n\n    const oldScrollTop = this._getScrollTop();\n    const newScrollTop = this._setScrollTop(this.touch.initialScrollTop + delta);\n\n    if (this.options.verticalScroll) {\n      this.dom.left.parentNode.scrollTop = -this.props.scrollTop;\n      this.dom.right.parentNode.scrollTop = -this.props.scrollTop;\n    }\n\n    if (newScrollTop != oldScrollTop) {\n      this.emit(\"verticalDrag\");\n    }\n  }\n\n  /**\n   * Apply a scrollTop\n   * @param {number} scrollTop\n   * @returns {number} scrollTop  Returns the applied scrollTop\n   * @private\n   */\n  _setScrollTop(scrollTop) {\n    this.props.scrollTop = scrollTop;\n    this._updateScrollTop();\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Update the current scrollTop when the height of  the containers has been changed\n   * @returns {number} scrollTop  Returns the applied scrollTop\n   * @private\n   */\n  _updateScrollTop() {\n    // recalculate the scrollTopMin\n    const scrollTopMin = Math.min(this.props.centerContainer.height - this.props.center.height, 0); // is negative or zero\n    if (scrollTopMin != this.props.scrollTopMin) {\n      // in case of bottom orientation, change the scrollTop such that the contents\n      // do not move relative to the time axis at the bottom\n      if (this.options.orientation.item != 'top') {\n        this.props.scrollTop += (scrollTopMin - this.props.scrollTopMin);\n      }\n      this.props.scrollTopMin = scrollTopMin;\n    }\n\n    // limit the scrollTop to the feasible scroll range\n    if (this.props.scrollTop > 0) this.props.scrollTop = 0;\n    if (this.props.scrollTop < scrollTopMin) this.props.scrollTop = scrollTopMin;\n\n    if (this.options.verticalScroll) {\n      this.dom.left.parentNode.scrollTop = -this.props.scrollTop;\n      this.dom.right.parentNode.scrollTop = -this.props.scrollTop;\n    }\n    this._isProgramaticallyScrolled = true;\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Get the current scrollTop\n   * @returns {number} scrollTop\n   * @private\n   */\n  _getScrollTop() {\n    return this.props.scrollTop;\n  }\n\n  /**\n   * Load a configurator\n   * [at]returns {Object}\n   * @private\n   */\n  _createConfigurator() {\n    throw new Error('Cannot invoke abstract method _createConfigurator');\n  }\n}\n\n// turn Core into an event emitter\nEmitter(Core.prototype);\n\nexport default Core;\n","import util from '../../util';\nimport Component from './Component';\nimport moment from '../../module/moment';\nimport locales from '../locales';\n\n/**\n * A current time bar\n */\nclass CurrentTime extends Component {\n/**\n * @param {{range: Range, dom: Object, domProps: Object}} body\n * @param {Object} [options]        Available parameters:\n *                                  {Boolean} [showCurrentTime]\n *                                  {String}  [alignCurrentTime]\n * @constructor CurrentTime\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n\n    // default options\n    this.defaultOptions = {\n      rtl: false,\n      showCurrentTime: true,\n      alignCurrentTime: undefined,\n\n      moment,\n      locales,\n      locale: 'en'\n    };\n    this.options = util.extend({}, this.defaultOptions);\n    this.setOptions(options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n    this.offset = 0;\n\n    this._create();\n\n  }\n\n  /**\n   * Create the HTML DOM for the current time bar\n   * @private\n   */\n  _create() {\n    const bar = document.createElement('div');\n    bar.className = 'vis-current-time';\n    bar.style.position = 'absolute';\n    bar.style.top = '0px';\n    bar.style.height = '100%';\n\n    this.bar = bar;\n  }\n\n  /**\n   * Destroy the CurrentTime bar\n   */\n  destroy() {\n    this.options.showCurrentTime = false;\n    this.redraw(); // will remove the bar from the DOM and stop refreshing\n\n    this.body = null;\n  }\n\n  /**\n   * Set options for the component. Options will be merged in current options.\n   * @param {Object} options  Available parameters:\n   *                          {boolean} [showCurrentTime]\n   *                          {String}  [alignCurrentTime]\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      util.selectiveExtend(['rtl', 'showCurrentTime', 'alignCurrentTime', 'moment', 'locale', 'locales'], this.options, options);\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    if (this.options.showCurrentTime) {\n      const parent = this.body.dom.backgroundVertical;\n      if (this.bar.parentNode != parent) {\n        // attach to the dom\n        if (this.bar.parentNode) {\n          this.bar.parentNode.removeChild(this.bar);\n        }\n        parent.appendChild(this.bar);\n\n        this.start();\n      }\n\n      let now = this.options.moment(Date.now() + this.offset);\n\n      if (this.options.alignCurrentTime) {\n        now = now.startOf(this.options.alignCurrentTime);\n      }\n\n      const x = this.body.util.toScreen(now);\n\n      let locale = this.options.locales[this.options.locale];\n      if (!locale) {\n        if (!this.warned) {\n          console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n          this.warned = true;\n        }\n        locale = this.options.locales['en']; // fall back on english when not available\n      }\n      let title = `${locale.current} ${locale.time}: ${now.format('dddd, MMMM Do YYYY, H:mm:ss')}`;\n      title = title.charAt(0).toUpperCase() + title.substring(1);\n\n      if (this.options.rtl) {\n        this.bar.style.transform = `translateX(${x * -1}px)`;\n      } else {\n        this.bar.style.transform = `translateX(${x}px)`;\n      }\n      this.bar.title = title;\n    }\n    else {\n      // remove the line from the DOM\n      if (this.bar.parentNode) {\n        this.bar.parentNode.removeChild(this.bar);\n      }\n      this.stop();\n    }\n\n    return false;\n  }\n\n  /**\n   * Start auto refreshing the current time bar\n   */\n  start() {\n    const me = this;\n\n      /**\n       *  Updates the current time.\n       */\n      function update () {\n      me.stop();\n\n      // determine interval to refresh\n      const scale = me.body.range.conversion(me.body.domProps.center.width).scale;\n      let interval = 1 / scale / 10;\n      if (interval < 30)   interval = 30;\n      if (interval > 1000) interval = 1000;\n\n      me.redraw();\n      me.body.emitter.emit('currentTimeTick');\n\n      // start a renderTimer to adjust for the new time\n      me.currentTimeTimer = setTimeout(update, interval);\n    }\n\n    update();\n  }\n\n  /**\n   * Stop auto refreshing the current time bar\n   */\n  stop() {\n    if (this.currentTimeTimer !== undefined) {\n      clearTimeout(this.currentTimeTimer);\n      delete this.currentTimeTimer;\n    }\n  }\n\n  /**\n   * Set a current time. This can be used for example to ensure that a client's\n   * time is synchronized with a shared server time.\n   * @param {Date | string | number} time     A Date, unix timestamp, or\n   *                                          ISO date string.\n   */\n  setCurrentTime(time) {\n    const t = util.convert(time, 'Date').valueOf();\n    const now = Date.now();\n    this.offset = t - now;\n    this.redraw();\n  }\n\n  /**\n   * Get the current time.\n   * @return {Date} Returns the current time.\n   */\n  getCurrentTime() {\n    return new Date(Date.now() + this.offset);\n  }\n}\n\nexport default CurrentTime;\n","'use strict';\nvar $ = require('../internals/export');\nvar $find = require('../internals/array-iteration').find;\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar FIND = 'find';\nvar SKIPS_HOLES = true;\n\nvar USES_TO_LENGTH = arrayMethodUsesToLength(FIND);\n\n// Shouldn't skip holes\nif (FIND in []) Array(1)[FIND](function () { SKIPS_HOLES = false; });\n\n// `Array.prototype.find` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.find\n$({ target: 'Array', proto: true, forced: SKIPS_HOLES || !USES_TO_LENGTH }, {\n  find: function find(callbackfn /* , that = undefined */) {\n    return $find(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables(FIND);\n","require('../../../modules/es.array.find');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').find;\n","var find = require('../array/virtual/find');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.find;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.find) ? find : own;\n};\n","// Unique ID creation requires a high quality random # generator. In the browser we therefore\n// require the crypto API and do not support built-in fallback to lower quality random number\n// generators (like Math.random()).\n// getRandomValues needs to be invoked in a context where \"this\" is a Crypto implementation. Also,\n// find the complete implementation of crypto (msCrypto) on IE11.\nvar getRandomValues = typeof crypto != 'undefined' && crypto.getRandomValues && crypto.getRandomValues.bind(crypto) || typeof msCrypto != 'undefined' && typeof msCrypto.getRandomValues == 'function' && msCrypto.getRandomValues.bind(msCrypto);\nvar rnds8 = new Uint8Array(16); // eslint-disable-line no-undef\n\nexport default function rng() {\n  if (!getRandomValues) {\n    throw new Error('crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported');\n  }\n\n  return getRandomValues(rnds8);\n}","/**\n * Convert array of 16 byte values to UUID string format of the form:\n * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX\n */\nvar byteToHex = [];\n\nfor (var i = 0; i < 256; ++i) {\n  byteToHex[i] = (i + 0x100).toString(16).substr(1);\n}\n\nfunction bytesToUuid(buf, offset) {\n  var i = offset || 0;\n  var bth = byteToHex; // join used to fix memory issue caused by concatenation: https://bugs.chromium.org/p/v8/issues/detail?id=3175#c4\n\n  return [bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], '-', bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]], bth[buf[i++]]].join('');\n}\n\nexport default bytesToUuid;","import rng from './rng.js';\nimport bytesToUuid from './bytesToUuid.js';\n\nfunction v4(options, buf, offset) {\n  var i = buf && offset || 0;\n\n  if (typeof options == 'string') {\n    buf = options === 'binary' ? new Array(16) : null;\n    options = null;\n  }\n\n  options = options || {};\n  var rnds = options.random || (options.rng || rng)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`\n\n  rnds[6] = rnds[6] & 0x0f | 0x40;\n  rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided\n\n  if (buf) {\n    for (var ii = 0; ii < 16; ++ii) {\n      buf[i + ii] = rnds[ii];\n    }\n  }\n\n  return buf || bytesToUuid(rnds);\n}\n\nexport default v4;","'use strict';\nvar $ = require('../internals/export');\nvar $includes = require('../internals/array-includes').includes;\nvar addToUnscopables = require('../internals/add-to-unscopables');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar USES_TO_LENGTH = arrayMethodUsesToLength('indexOf', { ACCESSORS: true, 1: 0 });\n\n// `Array.prototype.includes` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.includes\n$({ target: 'Array', proto: true, forced: !USES_TO_LENGTH }, {\n  includes: function includes(el /* , fromIndex = 0 */) {\n    return $includes(this, el, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('includes');\n","require('../../../modules/es.array.includes');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').includes;\n","var isObject = require('../internals/is-object');\nvar classof = require('../internals/classof-raw');\nvar wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar MATCH = wellKnownSymbol('match');\n\n// `IsRegExp` abstract operation\n// https://tc39.github.io/ecma262/#sec-isregexp\nmodule.exports = function (it) {\n  var isRegExp;\n  return isObject(it) && ((isRegExp = it[MATCH]) !== undefined ? !!isRegExp : classof(it) == 'RegExp');\n};\n","var isRegExp = require('../internals/is-regexp');\n\nmodule.exports = function (it) {\n  if (isRegExp(it)) {\n    throw TypeError(\"The method doesn't accept regular expressions\");\n  } return it;\n};\n","var wellKnownSymbol = require('../internals/well-known-symbol');\n\nvar MATCH = wellKnownSymbol('match');\n\nmodule.exports = function (METHOD_NAME) {\n  var regexp = /./;\n  try {\n    '/./'[METHOD_NAME](regexp);\n  } catch (e) {\n    try {\n      regexp[MATCH] = false;\n      return '/./'[METHOD_NAME](regexp);\n    } catch (f) { /* empty */ }\n  } return false;\n};\n","'use strict';\nvar $ = require('../internals/export');\nvar notARegExp = require('../internals/not-a-regexp');\nvar requireObjectCoercible = require('../internals/require-object-coercible');\nvar correctIsRegExpLogic = require('../internals/correct-is-regexp-logic');\n\n// `String.prototype.includes` method\n// https://tc39.github.io/ecma262/#sec-string.prototype.includes\n$({ target: 'String', proto: true, forced: !correctIsRegExpLogic('includes') }, {\n  includes: function includes(searchString /* , position = 0 */) {\n    return !!~String(requireObjectCoercible(this))\n      .indexOf(notARegExp(searchString), arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.string.includes');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('String').includes;\n","var arrayIncludes = require('../array/virtual/includes');\nvar stringIncludes = require('../string/virtual/includes');\n\nvar ArrayPrototype = Array.prototype;\nvar StringPrototype = String.prototype;\n\nmodule.exports = function (it) {\n  var own = it.includes;\n  if (it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.includes)) return arrayIncludes;\n  if (typeof it === 'string' || it === StringPrototype || (it instanceof String && own === StringPrototype.includes)) {\n    return stringIncludes;\n  } return own;\n};\n","// Utility functions for ordering and stacking of items\nconst EPSILON = 0.001; // used when checking collisions, to prevent round-off errors\n\n/**\n * Order items by their start data\n * @param {Item[]} items\n */\nexport function orderByStart(items) {\n  items.sort((a, b) => a.data.start - b.data.start);\n}\n\n/**\n * Order items by their end date. If they have no end date, their start date\n * is used.\n * @param {Item[]} items\n */\nexport function orderByEnd(items) {\n  items.sort((a, b) => {\n    const aTime = ('end' in a.data) ? a.data.end : a.data.start;\n    const bTime = ('end' in b.data) ? b.data.end : b.data.start;\n\n    return aTime - bTime;\n  });\n}\n\n/**\n * Adjust vertical positions of the items such that they don't overlap each\n * other.\n * @param {Item[]} items\n *            All visible items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {boolean} [force=false]\n *            If true, all items will be repositioned. If false (default), only\n *            items having a top===null will be re-stacked\n * @param {function} shouldBailItemsRedrawFunction\n *            bailing function\n * @return {boolean} shouldBail\n */\nexport function stack(items, margin, force, shouldBailItemsRedrawFunction) {\n  if (force) {\n    // reset top position of all items\n    for (var i = 0; i < items.length; i++) {\n      items[i].top = null;\n    }\n  }\n\n  // calculate new, non-overlapping positions\n  for (var i = 0; i < items.length; i++) {  // eslint-disable-line no-redeclare\n    const item = items[i];\n    if (item.stack && item.top === null) {\n      // initialize top position\n      item.top = margin.axis;\n      var shouldBail = false;\n\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (let j = 0, jj = items.length; j < jj; j++) {\n          const other = items[j];\n          shouldBail = shouldBailItemsRedrawFunction() || false;\n\n          if (shouldBail) { return true; }\n\n          if (other.top !== null && other !== item && other.stack && collision(item, other, margin.item, other.options.rtl)) {\n            collidingItem = other;\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the items above the colliding element\n          item.top = collidingItem.top + collidingItem.height + margin.item.vertical;\n        }\n      } while (collidingItem);\n    }\n  }\n  return shouldBail;\n}\n\n/**\n * Adjust vertical positions of the items within a single subgroup such that they\n * don't overlap each other.\n * @param {Item[]} items\n *            All items withina subgroup\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroup} subgroup\n *            The subgroup that is being stacked\n */\nexport function substack(items, margin, subgroup) {\n  for (var i = 0; i < items.length; i++) {\n    items[i].top = null;\n  }\n\n  // Set the initial height\n  let subgroupHeight = subgroup.height;\n\n  // calculate new, non-overlapping positions\n  for (i = 0; i < items.length; i++) {\n    const item = items[i];\n\n    if (item.stack && item.top === null) {\n      // initialize top position\n      item.top = item.baseTop;//margin.axis + item.baseTop;\n\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (let j = 0, jj = items.length; j < jj; j++) {\n          const other = items[j];\n          if (other.top !== null && other !== item /*&& other.stack*/ && collision(item, other, margin.item, other.options.rtl)) {\n            collidingItem = other;\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the items above the colliding element\n          item.top = collidingItem.top + collidingItem.height + margin.item.vertical;// + item.baseTop;\n        }\n\n        if (item.top + item.height > subgroupHeight) {\n          subgroupHeight = item.top + item.height;\n        }\n      } while (collidingItem);\n    }\n  }\n\n  // Set the new height\n  subgroup.height = subgroupHeight - subgroup.top + 0.5 * margin.item.vertical;\n}\n\n/**\n * Adjust vertical positions of the items without stacking them\n * @param {Item[]} items\n *            All visible items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n * @param {boolean} isStackSubgroups\n */\nexport function nostack(items, margin, subgroups, isStackSubgroups) {\n  for (let i = 0; i < items.length; i++) {\n    if (items[i].data.subgroup == undefined) {\n      items[i].top = margin.item.vertical;\n    } else if (items[i].data.subgroup !== undefined && isStackSubgroups) {\n      let newTop = 0;\n      for (const subgroup in subgroups) {\n        if (subgroups.hasOwnProperty(subgroup)) {\n          if (subgroups[subgroup].visible == true && subgroups[subgroup].index < subgroups[items[i].data.subgroup].index) {\n            newTop += subgroups[subgroup].height;\n            subgroups[items[i].data.subgroup].top = newTop;\n          }\n        }\n      }\n      items[i].top = newTop + 0.5 * margin.item.vertical;\n    }\n  }\n  if (!isStackSubgroups) {\n    stackSubgroups(items, margin, subgroups)\n  }\n}\n\n/**\n * Adjust vertical positions of the subgroups such that they don't overlap each\n * other.\n * @param {Array.<timeline.Item>} items\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n */\nexport function stackSubgroups(items, margin, subgroups) {\n  for (const subgroup in subgroups) {\n    if (subgroups.hasOwnProperty(subgroup)) {\n\n\n      subgroups[subgroup].top = 0;\n      do {\n        // TODO: optimize checking for overlap. when there is a gap without items,\n        //       you only need to check for items from the next item on, not from zero\n        var collidingItem = null;\n        for (const otherSubgroup in subgroups) {\n          if (subgroups[otherSubgroup].top !== null && otherSubgroup !== subgroup && subgroups[subgroup].index > subgroups[otherSubgroup].index && collisionByTimes(subgroups[subgroup], subgroups[otherSubgroup])) {\n            collidingItem = subgroups[otherSubgroup];\n            break;\n          }\n        }\n\n        if (collidingItem != null) {\n          // There is a collision. Reposition the subgroups above the colliding element\n          subgroups[subgroup].top = collidingItem.top + collidingItem.height;\n        }\n      } while (collidingItem);\n    }\n  }\n  for (let i = 0; i < items.length; i++) {\n    if (items[i].data.subgroup !== undefined) {\n      items[i].top = subgroups[items[i].data.subgroup].top + 0.5 * margin.item.vertical;\n    }\n  }\n}\n\n/**\n * Adjust vertical positions of the subgroups such that they don't overlap each\n * other, then stacks the contents of each subgroup individually.\n * @param {Item[]} subgroupItems\n *            All the items in a subgroup\n * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n *            Margins between items and between items and the axis.\n * @param {subgroups[]} subgroups\n *            All subgroups\n */\nexport function stackSubgroupsWithInnerStack(subgroupItems, margin, subgroups) {\n  let doSubStack = false;\n\n  // Run subgroups in their order (if any)\n  const subgroupOrder = [];\n\n  for(var subgroup in subgroups) {\n    if (subgroups[subgroup].hasOwnProperty(\"index\")) {\n      subgroupOrder[subgroups[subgroup].index] = subgroup;\n    }\n    else {\n      subgroupOrder.push(subgroup);\n    }\n  }\n\n  for(let j = 0; j < subgroupOrder.length; j++) {\n    subgroup = subgroupOrder[j];\n    if (subgroups.hasOwnProperty(subgroup)) {\n\n      doSubStack = doSubStack || subgroups[subgroup].stack;\n      subgroups[subgroup].top = 0;\n\n      for (const otherSubgroup in subgroups) {\n        if (subgroups[otherSubgroup].visible && subgroups[subgroup].index > subgroups[otherSubgroup].index) {\n          subgroups[subgroup].top += subgroups[otherSubgroup].height;\n        }\n      }\n\n      const items = subgroupItems[subgroup];\n      for(let i = 0; i < items.length; i++) {\n        if (items[i].data.subgroup !== undefined) {\n          items[i].top = subgroups[items[i].data.subgroup].top + 0.5 * margin.item.vertical;\n\n          if (subgroups[subgroup].stack) {\n            items[i].baseTop = items[i].top;\n          }\n        }\n      }\n\n      if (doSubStack && subgroups[subgroup].stack) {\n        substack(subgroupItems[subgroup], margin, subgroups[subgroup]);\n      }\n    }\n  }\n}\n\n/**\n * Test if the two provided items collide\n * The items must have parameters left, width, top, and height.\n * @param {Item} a          The first item\n * @param {Item} b          The second item\n * @param {{horizontal: number, vertical: number}} margin\n *                          An object containing a horizontal and vertical\n *                          minimum required margin.\n * @param {boolean} rtl\n * @return {boolean}        true if a and b collide, else false\n */\nexport function collision(a, b, margin, rtl) {\n  if (rtl) {\n    return  ((a.right - margin.horizontal + EPSILON)  < (b.right + b.width) &&\n    (a.right + a.width + margin.horizontal - EPSILON) > b.right &&\n    (a.top - margin.vertical + EPSILON)              < (b.top + b.height) &&\n    (a.top + a.height + margin.vertical - EPSILON)   > b.top);\n  } else {\n    return ((a.left - margin.horizontal + EPSILON)   < (b.left + b.width) &&\n    (a.left + a.width + margin.horizontal - EPSILON) > b.left &&\n    (a.top - margin.vertical + EPSILON)              < (b.top + b.height) &&\n    (a.top + a.height + margin.vertical - EPSILON)   > b.top);\n  }\n}\n\n/**\n * Test if the two provided objects collide\n * The objects must have parameters start, end, top, and height.\n * @param {Object} a          The first Object\n * @param {Object} b          The second Object\n * @return {boolean}        true if a and b collide, else false\n */\nexport function collisionByTimes(a, b) {\n\n  // Check for overlap by time and height. Abutting is OK and\n  // not considered a collision while overlap is considered a collision.\n  const timeOverlap = a.start < b.end && a.end > b.start;\n  const heightOverlap = a.top < (b.top + b.height) && (a.top + a.height) > b.top;\n  return timeOverlap && heightOverlap;\n}\n","import util from '../../util';\nimport * as stack from '../Stack';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n\n/**\n * @constructor Group\n */\nclass Group {\n  /**\n * @param {number | string} groupId\n * @param {Object} data\n * @param {ItemSet} itemSet\n * @constructor Group\n */\n  constructor(groupId, data, itemSet) {\n    this.groupId = groupId;\n    this.subgroups = {};\n    this.subgroupStack = {};\n    this.subgroupStackAll = false;\n    this.subgroupVisibility = {};\n    this.doInnerStack = false;\n    this.shouldBailStackItems = false;\n    this.subgroupIndex = 0;\n    this.subgroupOrderer = data && data.subgroupOrder;\n    this.itemSet = itemSet;\n    this.isVisible = null;\n    this.stackDirty = true; // if true, items will be restacked on next redraw\n\n    // This is a stack of functions (`() => void`) that will be executed before\n    // the instance is disposed off (method `dispose`). Anything that needs to\n    // be manually disposed off before garbage collection happens (or so that\n    // garbage collection can happen) should be added to this stack.\n    this._disposeCallbacks = [];\n\n    if (data && data.nestedGroups) {\n      this.nestedGroups = data.nestedGroups;\n      if (data.showNested == false) {\n        this.showNested = false;\n      } else {\n        this.showNested = true;\n      }\n    }\n\n    if (data && data.subgroupStack) {\n      if (typeof data.subgroupStack === \"boolean\") {\n        this.doInnerStack = data.subgroupStack;\n        this.subgroupStackAll = data.subgroupStack;\n      }\n      else {\n        // We might be doing stacking on specific sub groups, but only\n        // if at least one is set to do stacking\n        for(const key in data.subgroupStack) {\n          this.subgroupStack[key] = data.subgroupStack[key];\n          this.doInnerStack = this.doInnerStack || data.subgroupStack[key];\n        }\n      }\n    }\n\n    if (data && data.heightMode) {\n      this.heightMode = data.heightMode;\n    } else {\n      this.heightMode = itemSet.options.groupHeightMode;\n    }\n\n    this.nestedInGroup = null;\n\n    this.dom = {};\n    this.props = {\n      label: {\n        width: 0,\n        height: 0\n      }\n    };\n    this.className = null;\n\n    this.items = {};        // items filtered by groupId of this group\n    this.visibleItems = []; // items currently visible in window\n    this.itemsInRange = []; // items currently in range\n    this.orderedItems = {\n      byStart: [],\n      byEnd: []\n    };\n    this.checkRangedItems = false; // needed to refresh the ranged items if the window is programatically changed with NO overlap.\n\n    const handleCheckRangedItems = () => {\n      this.checkRangedItems = true;\n    };\n    this.itemSet.body.emitter.on(\"checkRangedItems\", handleCheckRangedItems);\n    this._disposeCallbacks.push(() => {\n      this.itemSet.body.emitter.off(\"checkRangedItems\", handleCheckRangedItems);\n    });\n\n    this._create();\n\n    this.setData(data);\n  }\n\n  /**\n   * Create DOM elements for the group\n   * @private\n   */\n  _create() {\n    const label = document.createElement('div');\n    if (this.itemSet.options.groupEditable.order) {\n      label.className = 'vis-label draggable';\n    } else {\n      label.className = 'vis-label';\n    }\n    this.dom.label = label;\n\n    const inner = document.createElement('div');\n    inner.className = 'vis-inner';\n    label.appendChild(inner);\n    this.dom.inner = inner;\n\n    const foreground = document.createElement('div');\n    foreground.className = 'vis-group';\n    foreground['vis-group'] = this;\n    this.dom.foreground = foreground;\n\n    this.dom.background = document.createElement('div');\n    this.dom.background.className = 'vis-group';\n\n    this.dom.axis = document.createElement('div');\n    this.dom.axis.className = 'vis-group';\n\n    // create a hidden marker to detect when the Timelines container is attached\n    // to the DOM, or the style of a parent of the Timeline is changed from\n    // display:none is changed to visible.\n    this.dom.marker = document.createElement('div');\n    this.dom.marker.style.visibility = 'hidden';\n    this.dom.marker.style.position = 'absolute';\n    this.dom.marker.innerHTML = '';\n    this.dom.background.appendChild(this.dom.marker);\n  }\n\n  /**\n   * Set the group data for this group\n   * @param {Object} data   Group data, can contain properties content and className\n   */\n  setData(data) {\n    if (this.itemSet.groupTouchParams.isDragging) return;\n\n    // update contents\n    let content;\n    let templateFunction;\n\n    if (data && data.subgroupVisibility) {\n      for (const key in data.subgroupVisibility) {\n        this.subgroupVisibility[key] = data.subgroupVisibility[key];\n      }\n    }\n\n    if (this.itemSet.options && this.itemSet.options.groupTemplate) {\n      templateFunction = this.itemSet.options.groupTemplate.bind(this);\n      content = templateFunction(data, this.dom.inner);\n    } else {\n      content = data && data.content;\n    }\n\n    if (content instanceof Element) {\n      while (this.dom.inner.firstChild) {\n        this.dom.inner.removeChild(this.dom.inner.firstChild);\n      }\n      this.dom.inner.appendChild(content);\n    } else if (content instanceof Object && content.isReactComponent) {\n      // Do nothing. Component was rendered into the node be ReactDOM.render.\n      // That branch is necessary for evasion of a second call templateFunction.\n      // Supports only React < 16(due to the asynchronous nature of React 16).\n    } else if (content instanceof Object) {\n      templateFunction(data, this.dom.inner);\n    } else if (content !== undefined && content !== null) {\n      this.dom.inner.innerHTML = content;\n    } else {\n      this.dom.inner.innerHTML = this.groupId || ''; // groupId can be null\n    }\n\n    // update title\n    this.dom.label.title = data && data.title || '';\n    if (!this.dom.inner.firstChild) {\n      util.addClassName(this.dom.inner, 'vis-hidden');\n    }\n    else {\n      util.removeClassName(this.dom.inner, 'vis-hidden');\n    }\n\n    if (data && data.nestedGroups) {\n      if (!this.nestedGroups || this.nestedGroups != data.nestedGroups) {\n        this.nestedGroups = data.nestedGroups;\n      }\n\n      if (data.showNested !== undefined || this.showNested === undefined) {\n        if (data.showNested == false) {\n          this.showNested = false;\n        } else {\n          this.showNested = true;\n        }\n      }\n\n      util.addClassName(this.dom.label, 'vis-nesting-group');\n      if (this.showNested) {\n        util.removeClassName(this.dom.label, 'collapsed');\n        util.addClassName(this.dom.label, 'expanded');\n      } else {\n        util.removeClassName(this.dom.label, 'expanded');\n        util.addClassName(this.dom.label, 'collapsed');\n      }\n    } else if (this.nestedGroups) {\n      this.nestedGroups = null;\n      util.removeClassName(this.dom.label, 'collapsed');\n      util.removeClassName(this.dom.label, 'expanded');\n      util.removeClassName(this.dom.label, 'vis-nesting-group');\n    }\n\n    if (data && (data.treeLevel|| data.nestedInGroup)) {\n      util.addClassName(this.dom.label, 'vis-nested-group');\n      if (data.treeLevel) {\n        util.addClassName(this.dom.label, 'vis-group-level-' + data.treeLevel);\n      } else {\n        // Nesting level is unknown, but we're sure it's at least 1\n        util.addClassName(this.dom.label, 'vis-group-level-unknown-but-gte1');\n      }\n    } else {\n      util.addClassName(this.dom.label, 'vis-group-level-0');\n    }\n    \n    // update className\n    const className = data && data.className || null;\n    if (className != this.className) {\n      if (this.className) {\n        util.removeClassName(this.dom.label, this.className);\n        util.removeClassName(this.dom.foreground, this.className);\n        util.removeClassName(this.dom.background, this.className);\n        util.removeClassName(this.dom.axis, this.className);\n      }\n      util.addClassName(this.dom.label, className);\n      util.addClassName(this.dom.foreground, className);\n      util.addClassName(this.dom.background, className);\n      util.addClassName(this.dom.axis, className);\n      this.className = className;\n    }\n\n    // update style\n    if (this.style) {\n      util.removeCssText(this.dom.label, this.style);\n      this.style = null;\n    }\n    if (data && data.style) {\n      util.addCssText(this.dom.label, data.style);\n      this.style = data.style;\n    }\n  }\n\n  /**\n   * Get the width of the group label\n   * @return {number} width\n   */\n  getLabelWidth() {\n    return this.props.label.width;\n  }\n\n  /**\n   * check if group has had an initial height hange\n   * @returns {boolean} \n   */\n  _didMarkerHeightChange() {\n    const markerHeight = this.dom.marker.clientHeight;\n    if (markerHeight != this.lastMarkerHeight) {\n      this.lastMarkerHeight = markerHeight;\n      const redrawQueue = {};\n      let redrawQueueLength = 0;\n\n      util.forEach(this.items, (item, key) => {\n        item.dirty = true;\n        if (item.displayed) {\n          const returnQueue = true;\n          redrawQueue[key] = item.redraw(returnQueue);\n          redrawQueueLength = redrawQueue[key].length;\n        }\n      })\n\n      const needRedraw = redrawQueueLength > 0;\n      if (needRedraw) {\n        // redraw all regular items\n        for (let i = 0; i < redrawQueueLength; i++) {\n          util.forEach(redrawQueue, fns => {\n            fns[i]();\n          });\n        }\n      }\n      return true;\n    } else {\n      return false;\n    }\n  }\n\n  /**\n   * calculate group dimentions and position\n   * @param {number} pixels\n   */\n  _calculateGroupSizeAndPosition() {\n    const { offsetTop, offsetLeft, offsetWidth } = this.dom.foreground;\n    this.top = offsetTop;\n    this.right = offsetLeft;\n    this.width = offsetWidth;\n  }\n\n  /**\n   * checks if should bail redraw of items\n   * @returns {boolean} should bail \n   */\n  _shouldBailItemsRedraw() {\n    const me = this;\n    const timeoutOptions = this.itemSet.options.onTimeout;\n    const bailOptions = {\n      relativeBailingTime: this.itemSet.itemsSettingTime,\n      bailTimeMs: timeoutOptions && timeoutOptions.timeoutMs,\n      userBailFunction: timeoutOptions && timeoutOptions.callback,\n      shouldBailStackItems: this.shouldBailStackItems\n    };\n    let bail = null;\n    if (!this.itemSet.initialDrawDone) {\n      if (bailOptions.shouldBailStackItems) { return true; }\n      if (Math.abs(Date.now() - new Date(bailOptions.relativeBailingTime)) > bailOptions.bailTimeMs) {\n        if (bailOptions.userBailFunction && this.itemSet.userContinueNotBail == null) {\n          bailOptions.userBailFunction(didUserContinue => {\n            me.itemSet.userContinueNotBail = didUserContinue;\n            bail = !didUserContinue;\n          })\n        } else if (me.itemSet.userContinueNotBail == false) {\n          bail = true;\n        } else {\n          bail = false;\n        }\n      }\n    }\n\n    return bail;\n  }\n\n  /**\n   * redraws items\n   * @param {boolean} forceRestack\n   * @param {boolean} lastIsVisible\n   * @param {number} margin\n   * @param {object} range\n   * @private\n   */\n  _redrawItems(forceRestack, lastIsVisible, margin, range) {\n    const restack = forceRestack || this.stackDirty || this.isVisible && !lastIsVisible;\n\n    // if restacking, reposition visible items vertically\n    if (restack) {\n      const orderedItems = {\n        byEnd: this.orderedItems.byEnd.filter(item => !item.isCluster),\n        byStart: this.orderedItems.byStart.filter(item => !item.isCluster)\n      }\n\n      const orderedClusters = {\n        byEnd: [...new Set(this.orderedItems.byEnd.map(item => item.cluster).filter(item => !!item))],\n        byStart: [...new Set(this.orderedItems.byStart.map(item => item.cluster).filter(item => !!item))],\n      }\n\n     /**\n     * Get all visible items in range\n     * @return {array} items\n     */\n      const getVisibleItems = () => {\n        const visibleItems = this._updateItemsInRange(orderedItems, this.visibleItems.filter(item => !item.isCluster), range);\n        const visibleClusters = this._updateClustersInRange(orderedClusters, this.visibleItems.filter(item => item.isCluster), range);\n        return [...visibleItems, ...visibleClusters];\n      }\n\n      /**\n       * Get visible items grouped by subgroup\n       * @param {function} orderFn An optional function to order items inside the subgroups\n       * @return {Object}\n       */\n      const getVisibleItemsGroupedBySubgroup = orderFn => {\n        let visibleSubgroupsItems = {};\n        for (const subgroup in this.subgroups) {\n          const items = this.visibleItems.filter(item => item.data.subgroup === subgroup);\n          visibleSubgroupsItems[subgroup] = orderFn ? items.sort((a, b) => orderFn(a.data, b.data)) : items;\n        }\n        return visibleSubgroupsItems;\n      };\n\n      if (typeof this.itemSet.options.order === 'function') {\n        // a custom order function\n        //show all items\n        const me = this;\n        if (this.doInnerStack && this.itemSet.options.stackSubgroups) {\n          // Order the items within each subgroup\n          const visibleSubgroupsItems = getVisibleItemsGroupedBySubgroup(this.itemSet.options.order);\n          stack.stackSubgroupsWithInnerStack(visibleSubgroupsItems, margin, this.subgroups);\n          this.visibleItems = getVisibleItems();\n          this._updateSubGroupHeights(margin);\n        }\n        else {\n          this.visibleItems = getVisibleItems();\n          this._updateSubGroupHeights(margin);\n          // order all items and force a restacking\n           // order all items outside clusters and force a restacking\n          const customOrderedItems = this.visibleItems\n                                  .slice()\n                                  .filter(item => item.isCluster || (!item.isCluster && !item.cluster))\n                                  .sort((a, b) => {\n                                      return me.itemSet.options.order(a.data, b.data);\n                                  });\n          this.shouldBailStackItems = stack.stack(customOrderedItems, margin, true, this._shouldBailItemsRedraw.bind(this));\n        }\n      } else {\n        // no custom order function, lazy stacking\n        this.visibleItems = getVisibleItems();\n        this._updateSubGroupHeights(margin);\n\n        if (this.itemSet.options.stack) {\n          if (this.doInnerStack && this.itemSet.options.stackSubgroups) {\n            const visibleSubgroupsItems = getVisibleItemsGroupedBySubgroup();\n            stack.stackSubgroupsWithInnerStack(visibleSubgroupsItems, margin, this.subgroups);\n          }\n          else {\n            // TODO: ugly way to access options...\n            this.shouldBailStackItems = stack.stack(this.visibleItems, margin, true, this._shouldBailItemsRedraw.bind(this));\n          }\n        } else {\n          // no stacking\n          stack.nostack(this.visibleItems, margin, this.subgroups, this.itemSet.options.stackSubgroups);\n        }\n      }\n\n      for (let i = 0; i < this.visibleItems.length; i++) {\n        this.visibleItems[i].repositionX();\n        if (this.subgroupVisibility[this.visibleItems[i].data.subgroup] !== undefined) {\n          if (!this.subgroupVisibility[this.visibleItems[i].data.subgroup]) {\n            this.visibleItems[i].hide();\n          }\n        }\n      }\n\n      if (this.itemSet.options.cluster) {\n        util.forEach(this.items, item => {\n          if (item.cluster && item.displayed) {\n            item.hide();\n          }\n        });\n      }\n\n      if (this.shouldBailStackItems) {\n        this.itemSet.body.emitter.emit('destroyTimeline')\n      }\n      this.stackDirty = false;\n    }\n  }\n\n  /**\n   * check if group resized\n   * @param {boolean} resized\n   * @param {number} height\n   * @return {boolean} did resize\n   */\n  _didResize(resized, height) {\n    resized = util.updateProperty(this, 'height', height) || resized;\n    // recalculate size of label\n    const labelWidth = this.dom.inner.clientWidth;\n    const labelHeight = this.dom.inner.clientHeight;\n    resized = util.updateProperty(this.props.label, 'width', labelWidth) || resized;\n    resized = util.updateProperty(this.props.label, 'height', labelHeight) || resized;\n    return resized;\n  }\n\n  /**\n   * apply group height\n   * @param {number} height\n   */\n  _applyGroupHeight(height) {\n    this.dom.background.style.height = `${height}px`;\n    this.dom.foreground.style.height = `${height}px`;\n    this.dom.label.style.height = `${height}px`;\n  }\n\n  /**\n   * update vertical position of items after they are re-stacked and the height of the group is calculated\n   * @param {number} margin\n   */\n  _updateItemsVerticalPosition(margin) {\n    for (let i = 0, ii = this.visibleItems.length; i < ii; i++) {\n      const item = this.visibleItems[i];\n      item.repositionY(margin);\n      if (!this.isVisible && this.groupId != ReservedGroupIds.BACKGROUND) {\n        if (item.displayed) item.hide();\n      }\n    }\n  }\n\n  /**\n   * Repaint this group\n   * @param {{start: number, end: number}} range\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @param {boolean} [forceRestack=false]  Force restacking of all items\n   * @param {boolean} [returnQueue=false]  return the queue or if the group resized\n   * @return {boolean} Returns true if the group is resized or the redraw queue if returnQueue=true\n   */\n  redraw(range, margin, forceRestack, returnQueue) {\n    let resized = false;\n    const lastIsVisible = this.isVisible;\n    let height;\n\n    const queue = [\n      () => {\n        forceRestack = this._didMarkerHeightChange.call(this) || forceRestack;\n      },\n      \n      // recalculate the height of the subgroups\n      this._updateSubGroupHeights.bind(this, margin),\n\n      // calculate actual size and position\n      this._calculateGroupSizeAndPosition.bind(this),\n\n      () => {\n        this.isVisible = this._isGroupVisible.bind(this)(range, margin);\n      },\n      \n      () => {\n        this._redrawItems.bind(this)(forceRestack, lastIsVisible, margin, range)\n      },\n\n      // update subgroups\n      this._updateSubgroupsSizes.bind(this),\n\n      () => {\n        height = this._calculateHeight.bind(this)(margin);\n      },\n\n      // calculate actual size and position again\n      this._calculateGroupSizeAndPosition.bind(this),\n\n      () => {\n        resized = this._didResize.bind(this)(resized, height)\n      },\n\n      () => {\n        this._applyGroupHeight.bind(this)(height)\n      },\n\n      () => {\n        this._updateItemsVerticalPosition.bind(this)(margin)\n      },\n\n      (() => {\n        if (!this.isVisible && this.height) {\n          resized = false;\n        }\n        return resized\n      }).bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * recalculate the height of the subgroups\n   *\n   * @param {{item: timeline.Item}} margin\n   * @private\n   */\n  _updateSubGroupHeights(margin) {\n    if (Object.keys(this.subgroups).length > 0) {\n      const me = this;\n\n      this._resetSubgroups();\n\n      util.forEach(this.visibleItems, item => {\n        if (item.data.subgroup !== undefined) {\n          me.subgroups[item.data.subgroup].height = Math.max(me.subgroups[item.data.subgroup].height, item.height + margin.item.vertical);\n          me.subgroups[item.data.subgroup].visible = typeof this.subgroupVisibility[item.data.subgroup] === 'undefined' ? true : Boolean(this.subgroupVisibility[item.data.subgroup]);\n        }\n      });\n    }\n  }\n\n  /**\n   * check if group is visible\n   *\n   * @param {timeline.Range} range\n   * @param {{axis: timeline.DataAxis}} margin\n   * @returns {boolean} is visible\n   * @private\n   */\n  _isGroupVisible(range, margin) {\n    return (this.top <= range.body.domProps.centerContainer.height - range.body.domProps.scrollTop + margin.axis)\n    && (this.top + this.height + margin.axis >= - range.body.domProps.scrollTop);\n  }\n\n  /**\n   * recalculate the height of the group\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @returns {number} Returns the height\n   * @private\n   */\n  _calculateHeight(margin) {\n    // recalculate the height of the group\n    let height;\n\n    let items;\n\n    if (this.heightMode === 'fixed') {\n      items = util.toArray(this.items);\n    } else {\n      // default or 'auto'\n      items = this.visibleItems;\n    }\n\n    if (items.length > 0) {\n      let min = items[0].top;\n      let max = items[0].top + items[0].height;\n      util.forEach(items, item => {\n        min = Math.min(min, item.top);\n        max = Math.max(max, (item.top + item.height));\n      });\n      if (min > margin.axis) {\n        // there is an empty gap between the lowest item and the axis\n        const offset = min - margin.axis;\n        max -= offset;\n        util.forEach(items, item => {\n          item.top -= offset;\n        });\n      }\n      height = Math.ceil(max + margin.item.vertical / 2);\n      if (this.heightMode !== \"fitItems\") {\n        height = Math.max(height, this.props.label.height);\n      }\n    }\n    else {\n      height = 0 || this.props.label.height;\n    }\n    return height;\n  }\n\n  /**\n   * Show this group: attach to the DOM\n   */\n  show() {\n    if (!this.dom.label.parentNode) {\n      this.itemSet.dom.labelSet.appendChild(this.dom.label);\n    }\n\n    if (!this.dom.foreground.parentNode) {\n      this.itemSet.dom.foreground.appendChild(this.dom.foreground);\n    }\n\n    if (!this.dom.background.parentNode) {\n      this.itemSet.dom.background.appendChild(this.dom.background);\n    }\n\n    if (!this.dom.axis.parentNode) {\n      this.itemSet.dom.axis.appendChild(this.dom.axis);\n    }\n  }\n\n  /**\n   * Hide this group: remove from the DOM\n   */\n  hide() {\n    const label = this.dom.label;\n    if (label.parentNode) {\n      label.parentNode.removeChild(label);\n    }\n\n    const foreground = this.dom.foreground;\n    if (foreground.parentNode) {\n      foreground.parentNode.removeChild(foreground);\n    }\n\n    const background = this.dom.background;\n    if (background.parentNode) {\n      background.parentNode.removeChild(background);\n    }\n\n    const axis = this.dom.axis;\n    if (axis.parentNode) {\n      axis.parentNode.removeChild(axis);\n    }\n  }\n\n  /**\n   * Add an item to the group\n   * @param {Item} item\n   */\n  add(item) {\n    this.items[item.id] = item;\n    item.setParent(this);\n    this.stackDirty = true;\n    // add to\n    if (item.data.subgroup !== undefined) {\n      this._addToSubgroup(item);\n      this.orderSubgroups();\n    }\n\n    if (!this.visibleItems.includes(item)) {\n      const range = this.itemSet.body.range; // TODO: not nice accessing the range like this\n      this._checkIfVisible(item, this.visibleItems, range);\n    }\n  }\n\n  /**\n   * add item to subgroup\n   * @param {object} item\n   * @param {string} subgroupId\n   */\n  _addToSubgroup(item, subgroupId=item.data.subgroup) {\n    if (subgroupId != undefined && this.subgroups[subgroupId] === undefined) {\n      this.subgroups[subgroupId] = {\n        height: 0,\n        top: 0,\n        start: item.data.start,\n        end: item.data.end || item.data.start,\n        visible: false,\n        index: this.subgroupIndex,\n        items: [],\n        stack: this.subgroupStackAll || this.subgroupStack[subgroupId] || false\n      };\n      this.subgroupIndex++;\n    }\n\n\n    if (new Date(item.data.start) < new Date(this.subgroups[subgroupId].start)) {\n      this.subgroups[subgroupId].start = item.data.start;\n    }\n\n    const itemEnd = item.data.end || item.data.start;\n    if (new Date(itemEnd) > new Date(this.subgroups[subgroupId].end)) {\n      this.subgroups[subgroupId].end = itemEnd;\n    }\n\n    this.subgroups[subgroupId].items.push(item);\n  }\n\n  /**\n   * update subgroup sizes\n   */\n  _updateSubgroupsSizes() {\n    const me = this;\n    if (me.subgroups) {\n      for (const subgroup in me.subgroups) {\n        const initialEnd = me.subgroups[subgroup].items[0].data.end || me.subgroups[subgroup].items[0].data.start;\n        let newStart = me.subgroups[subgroup].items[0].data.start;\n        let newEnd = initialEnd - 1;\n\n        me.subgroups[subgroup].items.forEach(item => {\n          if (new Date(item.data.start) < new Date(newStart)) {\n            newStart = item.data.start;\n          }\n\n          const itemEnd = item.data.end || item.data.start;\n          if (new Date(itemEnd) > new Date(newEnd)) {\n            newEnd = itemEnd;\n          }\n        })\n\n        me.subgroups[subgroup].start = newStart;\n        me.subgroups[subgroup].end = new Date(newEnd - 1) // -1 to compensate for colliding end to start subgroups;\n\n      }\n    }\n  }\n\n  /**\n   * order subgroups\n   */\n  orderSubgroups() {\n    if (this.subgroupOrderer !== undefined) {\n      const sortArray = [];\n      if (typeof this.subgroupOrderer == 'string') {\n        for (const subgroup in this.subgroups) {\n          sortArray.push({subgroup, sortField: this.subgroups[subgroup].items[0].data[this.subgroupOrderer]})\n        }\n        sortArray.sort((a, b) => a.sortField - b.sortField)\n      }\n      else if (typeof this.subgroupOrderer == 'function') {\n        for (const subgroup in this.subgroups) {\n          sortArray.push(this.subgroups[subgroup].items[0].data);\n        }\n        sortArray.sort(this.subgroupOrderer);\n      }\n\n      if (sortArray.length > 0) {\n        for (let i = 0; i < sortArray.length; i++) {\n          this.subgroups[sortArray[i].subgroup].index = i;\n        }\n      }\n    }\n  }\n\n  /**\n   * add item to subgroup\n   */\n  _resetSubgroups() {\n    for (const subgroup in this.subgroups) {\n      if (this.subgroups.hasOwnProperty(subgroup)) {\n        this.subgroups[subgroup].visible = false;\n        this.subgroups[subgroup].height = 0;\n      }\n    }\n  }\n\n  /**\n   * Remove an item from the group\n   * @param {Item} item\n   */\n  remove(item) {\n    delete this.items[item.id];\n    item.setParent(null);\n    this.stackDirty = true;\n\n    // remove from visible items\n    const index = this.visibleItems.indexOf(item);\n    if (index != -1) this.visibleItems.splice(index, 1);\n\n    if(item.data.subgroup !== undefined){\n      this._removeFromSubgroup(item);\n      this.orderSubgroups();\n    }\n  }\n\n  /**\n   * remove item from subgroup\n   * @param {object} item\n   * @param {string} subgroupId\n   */\n  _removeFromSubgroup(item, subgroupId=item.data.subgroup) {\n    if (subgroupId != undefined) {\n      const subgroup = this.subgroups[subgroupId];\n      if (subgroup){\n        const itemIndex = subgroup.items.indexOf(item);\n        //  Check the item is actually in this subgroup. How should items not in the group be handled?\n        if (itemIndex >= 0) {\n          subgroup.items.splice(itemIndex,1);\n          if (!subgroup.items.length){\n            delete this.subgroups[subgroupId];\n          } else {\n            this._updateSubgroupsSizes();\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * Remove an item from the corresponding DataSet\n   * @param {Item} item\n   */\n  removeFromDataSet(item) {\n    this.itemSet.removeItem(item.id);\n  }\n\n  /**\n   * Reorder the items\n   */\n  order() {\n    const array = util.toArray(this.items);\n    const startArray = [];\n    const endArray = [];\n\n    for (let i = 0; i < array.length; i++) {\n      if (array[i].data.end !== undefined) {\n        endArray.push(array[i]);\n      }\n      startArray.push(array[i]);\n    }\n    this.orderedItems = {\n      byStart: startArray,\n      byEnd: endArray\n    };\n\n    stack.orderByStart(this.orderedItems.byStart);\n    stack.orderByEnd(this.orderedItems.byEnd);\n  }\n\n  /**\n   * Update the visible items\n   * @param {{byStart: Item[], byEnd: Item[]}} orderedItems   All items ordered by start date and by end date\n   * @param {Item[]} oldVisibleItems                          The previously visible items.\n   * @param {{start: number, end: number}} range              Visible range\n   * @return {Item[]} visibleItems                            The new visible items.\n   * @private\n   */\n  _updateItemsInRange(orderedItems, oldVisibleItems, range) {\n    const visibleItems = [];\n    const visibleItemsLookup = {}; // we keep this to quickly look up if an item already exists in the list without using indexOf on visibleItems\n\n    if (!this.isVisible && this.groupId != ReservedGroupIds.BACKGROUND) {\n      for (let i = 0; i < oldVisibleItems.length; i++) {\n        var item = oldVisibleItems[i];\n        if (item.displayed) item.hide();\n      }\n      return visibleItems;\n    } \n\n    const interval = (range.end - range.start) / 4;\n    const lowerBound = range.start - interval;\n    const upperBound = range.end + interval;\n\n    // this function is used to do the binary search for items having start date only.\n    const startSearchFunction = value => {\n      if      (value < lowerBound)  {return -1;}\n      else if (value <= upperBound) {return  0;}\n      else                          {return  1;}\n    };\n\n    // this function is used to do the binary search for items having start and end dates (range).\n    const endSearchFunction = value => {\n      if      (value < lowerBound)  {return -1;}\n      else                          {return  0;}\n    }\n\n    // first check if the items that were in view previously are still in view.\n    // IMPORTANT: this handles the case for the items with startdate before the window and enddate after the window!\n    // also cleans up invisible items.\n    if (oldVisibleItems.length > 0) {\n      for (let i = 0; i < oldVisibleItems.length; i++) {\n        this._checkIfVisibleWithReference(oldVisibleItems[i], visibleItems, visibleItemsLookup, range);\n      }\n    }\n\n    // we do a binary search for the items that have only start values.\n    const initialPosByStart = util.binarySearchCustom(orderedItems.byStart, startSearchFunction, 'data','start');\n\n    // trace the visible items from the inital start pos both ways until an invisible item is found, we only look at the start values.\n    this._traceVisible(initialPosByStart, orderedItems.byStart, visibleItems, visibleItemsLookup, item => item.data.start < lowerBound || item.data.start > upperBound);\n\n    // if the window has changed programmatically without overlapping the old window, the ranged items with start < lowerBound and end > upperbound are not shown.\n    // We therefore have to brute force check all items in the byEnd list\n    if (this.checkRangedItems == true) {\n      this.checkRangedItems = false;\n      for (let i = 0; i < orderedItems.byEnd.length; i++) {\n        this._checkIfVisibleWithReference(orderedItems.byEnd[i], visibleItems, visibleItemsLookup, range);\n      }\n    }\n    else {\n      // we do a binary search for the items that have defined end times.\n      const initialPosByEnd = util.binarySearchCustom(orderedItems.byEnd, endSearchFunction, 'data','end');\n\n      // trace the visible items from the inital start pos both ways until an invisible item is found, we only look at the end values.\n      this._traceVisible(initialPosByEnd, orderedItems.byEnd, visibleItems, visibleItemsLookup, item => item.data.end < lowerBound || item.data.start > upperBound);\n    }\n\n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n\n    for (let i = 0; i < visibleItems.length; i++) {\n      const item = visibleItems[i];\n      if (!item.displayed) {\n        const returnQueue = true;\n        redrawQueue[i] = item.redraw(returnQueue);\n        redrawQueueLength = redrawQueue[i].length;\n      }\n    }\n\n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      // redraw all regular items\n      for (let j = 0; j < redrawQueueLength; j++) {\n        util.forEach(redrawQueue, fns => {\n          fns[j]();\n        });\n      }\n    }\n\n    for (let i = 0; i < visibleItems.length; i++) {\n      visibleItems[i].repositionX();\n    }\n\n    return visibleItems;\n  }\n\n  /**\n   * trace visible items in group\n   * @param {number} initialPos\n   * @param {array} items\n   * @param {aray} visibleItems\n   * @param {object} visibleItemsLookup\n   * @param {function} breakCondition\n   */\n  _traceVisible(initialPos, items, visibleItems, visibleItemsLookup, breakCondition) {\n    if (initialPos != -1) {\n      for (let i = initialPos; i >= 0; i--) {\n        let item = items[i];\n        if (breakCondition(item)) {\n          break;\n        }\n        else {\n          if (!(item.isCluster  && !item.hasItems()) && !item.cluster) {\n            if (visibleItemsLookup[item.id] === undefined) {\n              visibleItemsLookup[item.id] = true;\n              visibleItems.push(item);\n            }\n          }\n        }\n      }\n\n      for (let i = initialPos + 1; i < items.length; i++) {\n        let item = items[i];\n        if (breakCondition(item)) {\n          break;\n        }\n        else {\n          if (!(item.isCluster && !item.hasItems()) && !item.cluster) {\n            if (visibleItemsLookup[item.id] === undefined) {\n              visibleItemsLookup[item.id] = true;\n              visibleItems.push(item);\n            }\n          }\n        }\n      }\n    }\n  }\n\n  /**\n   * this function is very similar to the _checkIfInvisible() but it does not\n   * return booleans, hides the item if it should not be seen and always adds to\n   * the visibleItems.\n   * this one is for brute forcing and hiding.\n   *\n   * @param {Item} item\n   * @param {Array} visibleItems\n   * @param {{start:number, end:number}} range\n   * @private\n   */\n  _checkIfVisible(item, visibleItems, range) {\n      if (item.isVisible(range)) {\n        if (!item.displayed) item.show();\n        // reposition item horizontally\n        item.repositionX();\n        visibleItems.push(item);\n      }\n      else {\n        if (item.displayed) item.hide();\n      }\n  }\n\n  /**\n   * this function is very similar to the _checkIfInvisible() but it does not\n   * return booleans, hides the item if it should not be seen and always adds to\n   * the visibleItems.\n   * this one is for brute forcing and hiding.\n   *\n   * @param {Item} item\n   * @param {Array.<timeline.Item>} visibleItems\n   * @param {Object<number, boolean>} visibleItemsLookup\n   * @param {{start:number, end:number}} range\n   * @private\n   */\n  _checkIfVisibleWithReference(item, visibleItems, visibleItemsLookup, range) {\n    if (item.isVisible(range)) {\n      if (visibleItemsLookup[item.id] === undefined) {\n        visibleItemsLookup[item.id] = true;\n        visibleItems.push(item);\n      }\n    }\n    else {\n      if (item.displayed) item.hide();\n    }\n  }\n\n  /**\n   * Update the visible items\n   * @param {array} orderedClusters \n   * @param {array} oldVisibleClusters                         \n   * @param {{start: number, end: number}} range             \n   * @return {Item[]} visibleItems                            \n   * @private\n   */\n  _updateClustersInRange(orderedClusters, oldVisibleClusters, range) {\n    // Clusters can overlap each other so we cannot use binary search here\n    const visibleClusters = [];\n    const visibleClustersLookup = {}; // we keep this to quickly look up if an item already exists in the list without using indexOf on visibleItems\n  \n    if (oldVisibleClusters.length > 0) {\n      for (let i = 0; i < oldVisibleClusters.length; i++) {\n        this._checkIfVisibleWithReference(oldVisibleClusters[i], visibleClusters, visibleClustersLookup, range);\n      }\n    }\n  \n    for (let i = 0; i < orderedClusters.byStart.length; i++) {\n      this._checkIfVisibleWithReference(orderedClusters.byStart[i], visibleClusters, visibleClustersLookup, range);\n    }\n  \n    for (let i = 0; i < orderedClusters.byEnd.length; i++) {\n      this._checkIfVisibleWithReference(orderedClusters.byEnd[i], visibleClusters, visibleClustersLookup, range);\n    }\n  \n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n  \n    for (let i = 0; i < visibleClusters.length; i++) {\n      const item = visibleClusters[i];\n      if (!item.displayed) {\n        const returnQueue = true;\n        redrawQueue[i] = item.redraw(returnQueue);\n        redrawQueueLength = redrawQueue[i].length;\n      }\n    }\n  \n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      // redraw all regular items\n      for (var j = 0; j < redrawQueueLength; j++) {\n        util.forEach(redrawQueue, function (fns) {\n          fns[j]();\n        });\n      }\n    }\n  \n    for (let i = 0; i < visibleClusters.length; i++) {\n      visibleClusters[i].repositionX();\n    }\n    \n    return visibleClusters;\n  }\n\n  /**\n   * change item subgroup\n   * @param {object} item\n   * @param {string} oldSubgroup\n   * @param {string} newSubgroup\n   */\n  changeSubgroup(item, oldSubgroup, newSubgroup) {\n    this._removeFromSubgroup(item, oldSubgroup);\n    this._addToSubgroup(item, newSubgroup);\n    this.orderSubgroups();\n  }\n\n  /**\n   * Call this method before you lose the last reference to an instance of this.\n   * It will remove listeners etc.\n   */\n  dispose() {\n    this.hide();\n\n    let disposeCallback;\n    while ((disposeCallback = this._disposeCallbacks.pop())) {\n      disposeCallback();\n    }\n  }\n}\n\nexport default Group;\n","import Group from './Group'\n\n/**\n * @constructor BackgroundGroup\n * @extends Group\n */\nclass BackgroundGroup extends Group {\n  /**\n * @param {number | string} groupId\n * @param {Object} data\n * @param {ItemSet} itemSet\n */\n  constructor(groupId, data, itemSet) {\n    super(groupId, data, itemSet)\n    // Group.call(this, groupId, data, itemSet);\n\n    this.width = 0;\n    this.height = 0;\n    this.top = 0;\n    this.left = 0;\n  }\n\n  /**\n   * Repaint this group\n   * @param {{start: number, end: number}} range\n   * @param {{item: {horizontal: number, vertical: number}, axis: number}} margin\n   * @param {boolean} [forceRestack=false]  Force restacking of all items\n   * @return {boolean} Returns true if the group is resized\n   */\n  redraw(range, margin, forceRestack) {  // eslint-disable-line no-unused-vars\n    const resized = false;\n\n    this.visibleItems = this._updateItemsInRange(this.orderedItems, this.visibleItems, range);\n\n    // calculate actual size\n    this.width = this.dom.background.offsetWidth;\n\n    // apply new height (just always zero for BackgroundGroup\n    this.dom.background.style.height  = '0';\n\n    // update vertical position of items after they are re-stacked and the height of the group is calculated\n    for (let i = 0, ii = this.visibleItems.length; i < ii; i++) {\n      const item = this.visibleItems[i];\n      item.repositionY(margin);\n    }\n\n    return resized;\n  }\n\n  /**\n   * Show this group: attach to the DOM\n   */\n  show() {\n    if (!this.dom.background.parentNode) {\n      this.itemSet.dom.background.appendChild(this.dom.background);\n    }\n  }\n}\n\nexport default BackgroundGroup;","import Hammer from '../../../module/hammer';\nimport util from '../../../util';\nimport moment from '../../../module/moment';\nimport locales from '../../locales';\n\nimport '../css/item.css';\n\n/**\n * Item\n */\nclass Item {\n  /**\n * @constructor Item\n * @param {Object} data             Object containing (optional) parameters type,\n *                                  start, end, content, group, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} options          Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    this.id = null;\n    this.parent = null;\n    this.data = data;\n    this.dom = null;\n    this.conversion = conversion || {};\n    this.defaultOptions = {\n      locales,\n      locale: 'en'\n    };\n    this.options = util.extend({}, this.defaultOptions, options);\n    this.options.locales = util.extend({}, locales, this.options.locales);\n    const defaultLocales = this.defaultOptions.locales[this.defaultOptions.locale];\n    Object.keys(this.options.locales).forEach(locale => {\n      this.options.locales[locale] = util.extend(\n        {},\n        defaultLocales,\n        this.options.locales[locale]\n      );\n    });\n    this.selected = false;\n    this.displayed = false;\n    this.groupShowing = true;\n    this.selectable = (options && options.selectable) || false;\n    this.dirty = true;\n\n    this.top = null;\n    this.right = null;\n    this.left = null;\n    this.width = null;\n    this.height = null;\n\n    this.setSelectability(data);\n\n    this.editable = null;\n    this._updateEditStatus();\n  }\n\n  /**\n   * Select current item\n   */\n  select() {\n    if (this.selectable) {\n      this.selected = true;\n      this.dirty = true;\n      if (this.displayed) this.redraw();\n    }\n  }\n\n  /**\n   * Unselect current item\n   */\n  unselect() {\n    this.selected = false;\n    this.dirty = true;\n    if (this.displayed) this.redraw();\n  }\n\n  /**\n   * Set data for the item. Existing data will be updated. The id should not\n   * be changed. When the item is displayed, it will be redrawn immediately.\n   * @param {Object} data\n   */\n  setData(data) {\n    const groupChanged = data.group != undefined && this.data.group != data.group;\n    if (groupChanged && this.parent != null) {\n      this.parent.itemSet._moveToGroup(this, data.group);\n    }\n\n    this.setSelectability(data);\n\n    if (this.parent) {\n      this.parent.stackDirty = true;\n    }\n    \n    const subGroupChanged = data.subgroup != undefined && this.data.subgroup != data.subgroup;\n    if (subGroupChanged && this.parent != null) {\n      this.parent.changeSubgroup(this, this.data.subgroup, data.subgroup);\n    }\n\n    this.data = data;\n    this._updateEditStatus();\n    this.dirty = true;\n    if (this.displayed) this.redraw();\n  }\n\n  /**\n   * Set whether the item can be selected.\n   * Can only be set/unset if the timeline's `selectable` configuration option is `true`.\n   * @param {Object} data `data` from `constructor` and `setData`\n   */\n  setSelectability(data) {\n    if (data) {\n      this.selectable = typeof data.selectable === 'undefined' ? true : Boolean(data.selectable);\n    }\n  }\n\n  /**\n   * Set a parent for the item\n   * @param {Group} parent\n   */\n  setParent(parent) {\n    if (this.displayed) {\n      this.hide();\n      this.parent = parent;\n      if (this.parent) {\n        this.show();\n      }\n    }\n    else {\n      this.parent = parent;\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {  // eslint-disable-line no-unused-vars\n    return false;\n  }\n\n  /**\n   * Show the Item in the DOM (when not already visible)\n   * @return {Boolean} changed\n   */\n  show() {\n    return false;\n  }\n\n  /**\n   * Hide the Item from the DOM (when visible)\n   * @return {Boolean} changed\n   */\n  hide() {\n    return false;\n  }\n\n  /**\n   * Repaint the item\n   */\n  redraw() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Reposition the Item horizontally\n   */\n  repositionX() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Reposition the Item vertically\n   */\n  repositionY() {\n    // should be implemented by the item\n  }\n\n  /**\n   * Repaint a drag area on the center of the item when the item is selected\n   * @protected\n   */\n  _repaintDragCenter() {\n    if (this.selected && this.editable.updateTime && !this.dom.dragCenter) {\n      const me = this;\n      // create and show drag area\n      const dragCenter = document.createElement('div');\n      dragCenter.className = 'vis-drag-center';\n      dragCenter.dragCenterItem = this;\n      this.hammerDragCenter = new Hammer(dragCenter);\n\n      this.hammerDragCenter.on('tap', event => {\n        me.parent.itemSet.body.emitter.emit('click',  {\n          event,\n          item: me.id\n        });\n      });\n      this.hammerDragCenter.on('doubletap', event => {\n        event.stopPropagation();\n        me.parent.itemSet._onUpdateItem(me);\n        me.parent.itemSet.body.emitter.emit('doubleClick', {\n          event,\n          item: me.id\n        });\n      });\n      this.hammerDragCenter.on('panstart', event => {\n        // do not allow this event to propagate to the Range\n        event.stopPropagation();\n        me.parent.itemSet._onDragStart(event);\n      })\n      this.hammerDragCenter.on('panmove',  me.parent.itemSet._onDrag.bind(me.parent.itemSet));\n      this.hammerDragCenter.on('panend',   me.parent.itemSet._onDragEnd.bind(me.parent.itemSet));\n\n      if (this.dom.box) {\n        if (this.dom.dragLeft) {\n          this.dom.box.insertBefore(dragCenter, this.dom.dragLeft);\n        }\n        else {\n          this.dom.box.appendChild(dragCenter);\n        }\n      }\n      else if (this.dom.point) {\n        this.dom.point.appendChild(dragCenter);\n      }\n      \n      this.dom.dragCenter = dragCenter;\n    }\n    else if (!this.selected && this.dom.dragCenter) {\n      // delete drag area\n      if (this.dom.dragCenter.parentNode) {\n        this.dom.dragCenter.parentNode.removeChild(this.dom.dragCenter);\n      }\n      this.dom.dragCenter = null;\n      \n      if (this.hammerDragCenter) {\n        this.hammerDragCenter.destroy();\n        this.hammerDragCenter = null;\n      }\n    }\n  }\n\n  /**\n   * Repaint a delete button on the top right of the item when the item is selected\n   * @param {HTMLElement} anchor\n   * @protected\n   */\n  _repaintDeleteButton(anchor) {\n    const editable = ((this.options.editable.overrideItems || this.editable == null) && this.options.editable.remove) ||\n                   (!this.options.editable.overrideItems && this.editable != null && this.editable.remove);\n\n    if (this.selected && editable && !this.dom.deleteButton) {\n      // create and show button\n      const me = this;\n\n      const deleteButton = document.createElement('div');\n\n      if (this.options.rtl) {\n        deleteButton.className = 'vis-delete-rtl';\n      } else {\n        deleteButton.className = 'vis-delete';\n      }\n      let optionsLocale = this.options.locales[this.options.locale];\n      if (!optionsLocale) {\n        if (!this.warned) {\n          console.warn(`WARNING: options.locales['${this.options.locale}'] not found. See https://visjs.github.io/vis-timeline/docs/timeline/#Localization`);\n          this.warned = true;\n        }\n        optionsLocale = this.options.locales['en']; // fall back on english when not available\n      }\n      deleteButton.title = optionsLocale.deleteSelected;\n\n      // TODO: be able to destroy the delete button\n      this.hammerDeleteButton = new Hammer(deleteButton).on('tap', event => {\n        event.stopPropagation();\n        me.parent.removeFromDataSet(me);\n      });\n\n      anchor.appendChild(deleteButton);\n      this.dom.deleteButton = deleteButton;\n    }\n    else if (!this.selected && this.dom.deleteButton) {\n      // remove button\n      if (this.dom.deleteButton.parentNode) {\n        this.dom.deleteButton.parentNode.removeChild(this.dom.deleteButton);\n      }\n      this.dom.deleteButton = null;\n\n      if (this.hammerDeleteButton) {\n        this.hammerDeleteButton.destroy();\n        this.hammerDeleteButton = null;\n      }\n    }\n  }\n\n  /**\n   * Repaint a onChange tooltip on the top right of the item when the item is selected\n   * @param {HTMLElement} anchor\n   * @protected\n   */\n  _repaintOnItemUpdateTimeTooltip(anchor) {\n    if (!this.options.tooltipOnItemUpdateTime) return;\n\n    const editable = (this.options.editable.updateTime || \n                    this.data.editable === true) &&\n                   this.data.editable !== false;\n\n    if (this.selected && editable && !this.dom.onItemUpdateTimeTooltip) {\n      const onItemUpdateTimeTooltip = document.createElement('div');\n\n      onItemUpdateTimeTooltip.className = 'vis-onUpdateTime-tooltip';\n      anchor.appendChild(onItemUpdateTimeTooltip);\n      this.dom.onItemUpdateTimeTooltip = onItemUpdateTimeTooltip;\n\n    } else if (!this.selected && this.dom.onItemUpdateTimeTooltip) {\n      // remove button\n      if (this.dom.onItemUpdateTimeTooltip.parentNode) {\n        this.dom.onItemUpdateTimeTooltip.parentNode.removeChild(this.dom.onItemUpdateTimeTooltip);\n      }\n      this.dom.onItemUpdateTimeTooltip = null;\n    }\n\n    // position onChange tooltip\n    if (this.dom.onItemUpdateTimeTooltip) {\n\n      // only show when editing\n      this.dom.onItemUpdateTimeTooltip.style.visibility = this.parent.itemSet.touchParams.itemIsDragging ? 'visible' : 'hidden';\n      \n      // position relative to item's content\n      this.dom.onItemUpdateTimeTooltip.style.transform = 'translateX(-50%)';\n      this.dom.onItemUpdateTimeTooltip.style.left = '50%';\n\n      // position above or below the item depending on the item's position in the window\n      const tooltipOffset = 50; // TODO: should be tooltip height (depends on template)\n      const scrollTop = this.parent.itemSet.body.domProps.scrollTop;\n\n        // TODO: this.top for orientation:true is actually the items distance from the bottom... \n        // (should be this.bottom)\n      let itemDistanceFromTop; \n      if (this.options.orientation.item == 'top') {\n        itemDistanceFromTop = this.top;\n      } else {\n        itemDistanceFromTop = (this.parent.height - this.top - this.height)\n      }\n      const isCloseToTop = itemDistanceFromTop + this.parent.top - tooltipOffset < -scrollTop;\n\n      if (isCloseToTop) {\n        this.dom.onItemUpdateTimeTooltip.style.bottom = \"\";\n        this.dom.onItemUpdateTimeTooltip.style.top = `${this.height + 2}px`;\n      } else {\n        this.dom.onItemUpdateTimeTooltip.style.top = \"\";\n        this.dom.onItemUpdateTimeTooltip.style.bottom = `${this.height + 2}px`;\n      }\n      \n      // handle tooltip content\n      let content;\n      let templateFunction;\n\n      if (this.options.tooltipOnItemUpdateTime && this.options.tooltipOnItemUpdateTime.template) {\n        templateFunction = this.options.tooltipOnItemUpdateTime.template.bind(this);\n        content = templateFunction(this.data);\n      } else {\n        content = `start: ${moment(this.data.start).format('MM/DD/YYYY hh:mm')}`;\n        if (this.data.end) { \n          content += `<br> end: ${moment(this.data.end).format('MM/DD/YYYY hh:mm')}`;\n        }\n      }\n      this.dom.onItemUpdateTimeTooltip.innerHTML = content;\n    }\n  }\n\n   /**\n   * get item data\n   * @return {object}\n   * @private\n   */\n  _getItemData() {\n    return this.parent.itemSet.itemsData.get(this.id);\n  }\n\n  /**\n   * Set HTML contents for the item\n   * @param {Element} element   HTML element to fill with the contents\n   * @private\n   */\n  _updateContents(element) {\n    let content;\n    let changed;\n    let templateFunction;\n    let itemVisibleFrameContent;\n    let visibleFrameTemplateFunction; \n    const itemData = this._getItemData(); // get a clone of the data from the dataset\n\n    const frameElement = this.dom.box || this.dom.point;\n    const itemVisibleFrameContentElement = frameElement.getElementsByClassName('vis-item-visible-frame')[0];\n\n    if (this.options.visibleFrameTemplate) {\n      visibleFrameTemplateFunction = this.options.visibleFrameTemplate.bind(this);\n      itemVisibleFrameContent = visibleFrameTemplateFunction(itemData, itemVisibleFrameContentElement);\n    } else {\n      itemVisibleFrameContent = '';\n    }\n    \n    if (itemVisibleFrameContentElement) {\n      if ((itemVisibleFrameContent instanceof Object) && !(itemVisibleFrameContent instanceof Element)) {\n        visibleFrameTemplateFunction(itemData, itemVisibleFrameContentElement)\n      } else {\n         changed = this._contentToString(this.itemVisibleFrameContent) !== this._contentToString(itemVisibleFrameContent);\n         if (changed) {\n          // only replace the content when changed\n          if (itemVisibleFrameContent instanceof Element) {\n            itemVisibleFrameContentElement.innerHTML = '';\n            itemVisibleFrameContentElement.appendChild(itemVisibleFrameContent);\n          }\n          else if (itemVisibleFrameContent != undefined) {\n            itemVisibleFrameContentElement.innerHTML = itemVisibleFrameContent;\n          }\n          else {\n            if (!(this.data.type == 'background' && this.data.content === undefined)) {\n              throw new Error(`Property \"content\" missing in item ${this.id}`);\n            }\n          }\n\n          this.itemVisibleFrameContent = itemVisibleFrameContent;\n         }\n      }\n    }\n\n    if (this.options.template) {\n      templateFunction = this.options.template.bind(this);\n      content = templateFunction(itemData, element, this.data);\n    } else {\n      content = this.data.content;\n    }\n\n    if ((content instanceof Object) && !(content instanceof Element)) {\n      templateFunction(itemData, element)\n    } else {\n      changed = this._contentToString(this.content) !== this._contentToString(content);\n      if (changed) {\n        // only replace the content when changed\n        if (content instanceof Element) {\n          element.innerHTML = '';\n          element.appendChild(content);\n        }\n        else if (content != undefined) {\n          element.innerHTML = content;\n        }\n        else {\n          if (!(this.data.type == 'background' && this.data.content === undefined)) {\n            throw new Error(`Property \"content\" missing in item ${this.id}`);\n          }\n        }\n        this.content = content;\n      }\n    }\n  }\n\n  /**\n   * Process dataAttributes timeline option and set as data- attributes on dom.content\n   * @param {Element} element   HTML element to which the attributes will be attached\n   * @private\n   */\n  _updateDataAttributes(element) {\n   if (this.options.dataAttributes && this.options.dataAttributes.length > 0) {\n     let attributes = [];\n\n     if (Array.isArray(this.options.dataAttributes)) {\n       attributes = this.options.dataAttributes;\n     }\n     else if (this.options.dataAttributes == 'all') {\n       attributes = Object.keys(this.data);\n     }\n     else {\n       return;\n     }\n\n     for (const name of attributes) {\n       const value = this.data[name];\n\n       if (value != null) {\n         element.setAttribute(`data-${name}`, value);\n       }\n       else {\n         element.removeAttribute(`data-${name}`);\n       }\n     }\n   }\n }\n\n  /**\n   * Update custom styles of the element\n   * @param {Element} element\n   * @private\n   */\n  _updateStyle(element) {\n    // remove old styles\n    if (this.style) {\n      util.removeCssText(element, this.style);\n      this.style = null;\n    }\n\n    // append new styles\n    if (this.data.style) {\n      util.addCssText(element, this.data.style);\n      this.style = this.data.style;\n    }\n  }\n\n  /**\n   * Stringify the items contents\n   * @param {string | Element | undefined} content\n   * @returns {string | undefined}\n   * @private\n   */\n  _contentToString(content) {\n    if (typeof content === 'string') return content;\n    if (content && 'outerHTML' in content) return content.outerHTML;\n    return content;\n  }\n\n  /**\n   * Update the editability of this item.\n   */\n  _updateEditStatus() {\n    if (this.options) {\n      if(typeof this.options.editable === 'boolean') {\n        this.editable = {\n          updateTime: this.options.editable,\n          updateGroup: this.options.editable,\n          remove: this.options.editable\n        };\n      } else if(typeof this.options.editable === 'object') {\n          this.editable = {};\n          util.selectiveExtend(['updateTime', 'updateGroup', 'remove'], this.editable, this.options.editable);\n      }\n    }\n    // Item data overrides, except if options.editable.overrideItems is set.\n    if (!this.options || !(this.options.editable) || (this.options.editable.overrideItems !== true)) {\n      if (this.data) {\n        if (typeof this.data.editable === 'boolean') {\n          this.editable = {\n            updateTime: this.data.editable,\n            updateGroup: this.data.editable,\n            remove: this.data.editable\n          }\n        } else if (typeof this.data.editable === 'object') {\n          // TODO: in timeline.js 5.0, we should change this to not reset options from the timeline configuration.\n          // Basically just remove the next line...\n          this.editable = {};\n          util.selectiveExtend(['updateTime', 'updateGroup', 'remove'], this.editable, this.data.editable);\n        }\n      }\n    }\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return 0;\n  }\n\n  /**\n   * Return the width of the item right from the max of its start and end date\n   * @return {number}\n   */\n  getWidthRight() {\n    return 0;\n  }\n\n  /**\n   * Return the title of the item\n   * @return {string | undefined}\n   */\n  getTitle() {\n    if (this.options.tooltip && this.options.tooltip.template) {\n      const templateFunction = this.options.tooltip.template.bind(this);\n      return templateFunction(this._getItemData(), this.data);\n    }\n\n    return this.data.title;\n  }\n}\n\nItem.prototype.stack = true;\n\nexport default Item;\n","import Item from './Item';\n\n/**\n * @constructor BoxItem\n * @extends Item\n */\nclass BoxItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      dot: {\n        width: 0,\n        height: 0\n      },\n      line: {\n        width: 0,\n        height: 0\n      }\n    };\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {{start: number, end: number}} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    let isVisible;\n    const align = this.data.align || this.options.align;\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n\n    if (align == 'right') {\n      isVisible = (this.data.start.getTime() > range.start) && (this.data.start.getTime() - widthInMs < range.end);\n    }\n    else if (align == 'left') {\n      isVisible = (this.data.start.getTime() + widthInMs > range.start) && (this.data.start.getTime() < range.end);\n    }\n    else {\n      // default or 'center'\n      isVisible = (this.data.start.getTime() + widthInMs / 2 > range.start ) && (this.data.start.getTime() - widthInMs/2 < range.end);\n    }\n    return isVisible;\n  }\n\n   /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // create main box\n      this.dom.box = document.createElement('DIV');\n\n      // contents box (inside the background box). used for making margins\n      this.dom.content = document.createElement('DIV');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.box.appendChild(this.dom.content);\n\n      // line to axis\n      this.dom.line = document.createElement('DIV');\n      this.dom.line.className = 'vis-line';\n\n      // dot on axis\n      this.dom.dot = document.createElement('DIV');\n      this.dom.dot.className = 'vis-dot';\n\n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) throw new Error('Cannot redraw item: parent has no foreground container element');\n      foreground.appendChild(this.dom.box);\n    }\n    if (!this.dom.line.parentNode) {\n      var background = this.parent.dom.background;\n      if (!background) throw new Error('Cannot redraw item: parent has no background container element');\n      background.appendChild(this.dom.line);\n    }\n    if (!this.dom.dot.parentNode) {\n      const axis = this.parent.dom.axis;\n      if (!background) throw new Error('Cannot redraw item: parent has no axis container element');\n      axis.appendChild(this.dom.dot);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM element\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n\n      // update class\n      const className = (this.data.className ? ' ' + this.data.className : '') +\n        (this.selected ? ' vis-selected' : '') +\n        (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.box.className = `vis-item vis-box${className}`;\n      this.dom.line.className = `vis-item vis-line${className}`;\n      this.dom.dot.className = `vis-item vis-dot${className}`;\n    }\n  }\n\n  /**\n   * get DOM components sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    return {\n      previous: {\n        right: this.dom.box.style.right,\n        left: this.dom.box.style.left\n      },\n      dot: {\n        height: this.dom.dot.offsetHeight,\n        width: this.dom.dot.offsetWidth\n      },\n      line: {\n        width: this.dom.line.offsetWidth\n      },\n      box: {\n        width: this.dom.box.offsetWidth,\n        height: this.dom.box.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    if (this.options.rtl) {\n      this.dom.box.style.right = \"0px\";\n    } else {\n      this.dom.box.style.left = \"0px\";\n    }\n\n    // recalculate size\n    this.props.dot.height = sizes.dot.height;\n    this.props.dot.width = sizes.dot.width;\n    this.props.line.width = sizes.line.width;\n    this.width = sizes.box.width;\n    this.height = sizes.box.height;\n\n    // restore previous position\n    if (this.options.rtl) {\n      this.dom.box.style.right = sizes.previous.right;\n    } else {\n      this.dom.box.style.left = sizes.previous.left;\n    }\n\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n    this._repaintDragCenter();\n    this._repaintDeleteButton(this.dom.box);\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      const dom = this.dom;\n\n      if (dom.box.remove) dom.box.remove();\n      else if (dom.box.parentNode) dom.box.parentNode.removeChild(dom.box); // IE11\n\n      if (dom.line.remove) dom.line.remove();\n      else if (dom.line.parentNode) dom.line.parentNode.removeChild(dom.line); // IE11\n      \n      if (dom.dot.remove) dom.dot.remove();\n      else if (dom.dot.parentNode) dom.dot.parentNode.removeChild(dom.dot); // IE11\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item XY\n   */\n  repositionXY() {\n    const rtl = this.options.rtl;\n\n    const repositionXY = (element, x, y, rtl = false) => {\n      if (x === undefined && y === undefined) return;\n      // If rtl invert the number.\n      const directionX = rtl ? (x * -1) : x;\n\n      //no y. translate x\n      if (y === undefined) {\n        element.style.transform = `translateX(${directionX}px)`;\n        return;\n      }\n\n      //no x. translate y\n      if (x === undefined) {\n        element.style.transform = `translateY(${y}px)`;\n        return;\n      }\n\n      element.style.transform = `translate(${directionX}px, ${y}px)`;\n    }\n    repositionXY(this.dom.box, this.boxX, this.boxY, rtl);\n    repositionXY(this.dom.dot, this.dotX, this.dotY, rtl);\n    repositionXY(this.dom.line, this.lineX, this.lineY, rtl);\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @Override\n   */\n  repositionX() {\n    const start = this.conversion.toScreen(this.data.start);\n    const align = this.data.align === undefined ? this.options.align : this.data.align\n    const lineWidth = this.props.line.width;\n    const dotWidth = this.props.dot.width;\n    \n    if (align == 'right') {\n      // calculate right position of the box\n      this.boxX = start - this.width;\n      this.lineX = start - lineWidth;\n      this.dotX = start - lineWidth / 2 - dotWidth / 2;\n    }\n    else if (align == 'left') {\n      // calculate left position of the box\n      this.boxX = start;\n      this.lineX = start;\n      this.dotX = start + lineWidth / 2 - dotWidth / 2;\n    }\n    else {\n      // default or 'center'\n      this.boxX = start - this.width / 2;\n      this.lineX = this.options.rtl ? start - lineWidth : start - lineWidth / 2;\n      this.dotX = start - dotWidth / 2;\n    }\n\n    if (this.options.rtl)\n      this.right = this.boxX;\n    else\n      this.left = this.boxX;\n\n    this.repositionXY();\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    const lineStyle = this.dom.line.style;\n\n    if (orientation == 'top') {\n      const lineHeight = this.parent.top + this.top + 1;\n\n      this.boxY = this.top || 0;\n      lineStyle.height = `${lineHeight}px`;\n      lineStyle.bottom = '';\n      lineStyle.top = '0';\n    }\n    else { // orientation 'bottom'\n      const itemSetHeight = this.parent.itemSet.props.height; // TODO: this is nasty\n      const lineHeight = itemSetHeight - this.parent.top - this.parent.height + this.top;\n\n      this.boxY = this.parent.height - this.top - (this.height || 0);\n      lineStyle.height = `${lineHeight}px`;\n      lineStyle.top = '';\n      lineStyle.bottom = '0';\n    }\n\n    this.dotY = -this.props.dot.height / 2;\n\n    this.repositionXY();\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.width / 2;\n  }\n\n  /**\n   * Return the width of the item right from its start date\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.width / 2;\n  }\n}\n\nexport default BoxItem;\n","import Item from './Item';\n\n/**\n * @constructor PointItem\n * @extends Item\n */\nclass PointItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      dot: {\n        top: 0,\n        width: 0,\n        height: 0\n      },\n      content: {\n        height: 0,\n        marginLeft: 0,\n        marginRight: 0\n      }\n    };\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {{start: number, end: number}} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n    \n    return (this.data.start.getTime() + widthInMs > range.start ) && (this.data.start < range.end);\n  }\n\n  /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // background box\n      this.dom.point = document.createElement('div');\n      // className is updated in redraw()\n\n      // contents box, right from the dot\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.point.appendChild(this.dom.content);\n\n      // dot at start\n      this.dom.dot = document.createElement('div');\n      this.dom.point.appendChild(this.dom.dot);\n\n      // attach this item as attribute\n      this.dom.point['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.point.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n      foreground.appendChild(this.dom.point);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.point);\n      this._updateStyle(this.dom.point);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n      // update class\n      const className = (this.data.className ? ' ' + this.data.className : '') +\n          (this.selected ? ' vis-selected' : '') +\n          (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.point.className  = `vis-item vis-point${className}`;\n      this.dom.dot.className  = `vis-item vis-dot${className}`;\n    }\n  }\n\n  /**\n   * get DOM component sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    return {\n      dot:  {\n        width: this.dom.dot.offsetWidth,\n        height: this.dom.dot.offsetHeight\n      },\n      content: {\n        width: this.dom.content.offsetWidth,\n        height: this.dom.content.offsetHeight\n      },\n      point: {\n        width: this.dom.point.offsetWidth,\n        height: this.dom.point.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {array} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    // recalculate size of dot and contents\n    this.props.dot.width = sizes.dot.width;\n    this.props.dot.height = sizes.dot.height;\n    this.props.content.height = sizes.content.height;\n\n    // resize contents\n    if (this.options.rtl) {\n      this.dom.content.style.marginRight = `${2 * this.props.dot.width}px`;\n    } else {\n      this.dom.content.style.marginLeft = `${2 * this.props.dot.width}px`;\n    }\n    //this.dom.content.style.marginRight = ... + 'px'; // TODO: margin right\n\n    // recalculate size\n    this.width = sizes.point.width;\n    this.height = sizes.point.height;\n\n    // reposition the dot\n    this.dom.dot.style.top = `${(this.height - this.props.dot.height) / 2}px`;\n    \n    const dotWidth = this.props.dot.width;\n    const translateX = this.options.rtl ? (dotWidth / 2) * -1 : dotWidth / 2;\n    this.dom.dot.style.transform = `translateX(${translateX}px`;\n    this.dirty = false;\n  }\n\n  /**\n   * Repain DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.point);\n    this._repaintDragCenter();\n    this._repaintDeleteButton(this.dom.point);\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  \n  /**\n   * Reposition XY\n   */\n  repositionXY() {\n    const rtl = this.options.rtl;\n    \n    const repositionXY = (element, x, y, rtl = false) => {\n      if (x === undefined && y === undefined) return;\n      // If rtl invert the number.\n      const directionX = rtl ? (x * -1) : x;\n\n      //no y. translate x\n      if (y === undefined) {\n        element.style.transform = `translateX(${directionX}px)`;\n        return;\n      }\n\n      //no x. translate y\n      if (x === undefined) {\n        element.style.transform = `translateY(${y}px)`;\n        return;\n      }\n\n      element.style.transform = `translate(${directionX}px, ${y}px)`;\n    }\n    repositionXY(this.dom.point, this.pointX, this.pointY, rtl);\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      if (this.dom.point.parentNode) {\n        this.dom.point.parentNode.removeChild(this.dom.point);\n      }\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @Override\n   */\n  repositionX() {\n    const start = this.conversion.toScreen(this.data.start);\n\n    this.pointX = start;\n    if (this.options.rtl) {\n      this.right = start - this.props.dot.width;\n    } else {\n      this.left = start - this.props.dot.width;\n    }\n\n    this.repositionXY();\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    if (orientation == 'top') {\n      this.pointY = this.top;\n    }\n    else {\n      this.pointY = this.parent.height - this.top - this.height\n    }\n\n    this.repositionXY();\n  }\n\n  /**\n   * Return the width of the item left from its start date\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.props.dot.width;\n  }\n\n  /**\n   * Return the width of the item right from  its start date\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.props.dot.width;\n  }\n}\n\nexport default PointItem;\n","import Item from './Item';\n\n/**\n * @constructor RangeItem\n * @extends Item\n */\nclass RangeItem extends Item {\n  /**\n * @param {Object} data             Object containing parameters start, end\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe options\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      content: {\n        width: 0\n      }\n    };\n    this.overflow = false; // if contents can overflow (css styling), this flag is set to true\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data.id}`);\n      }\n      if (data.end == undefined) {\n        throw new Error(`Property \"end\" missing in item ${data.id}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   *\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    if (this.cluster) {\n      return false;\n    }\n    // determine visibility\n    return (this.data.start < range.end) && (this.data.end > range.start);\n  }\n\n  /**\n   * create DOM elements\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n        // background box\n      this.dom.box = document.createElement('div');\n      // className is updated in redraw()\n\n      // frame box (to prevent the item contents from overflowing)\n      this.dom.frame = document.createElement('div');\n      this.dom.frame.className = 'vis-item-overflow';\n      this.dom.box.appendChild(this.dom.frame);\n    \n      // visible frame box (showing the frame that is always visible)\n      this.dom.visibleFrame = document.createElement('div');\n      this.dom.visibleFrame.className = 'vis-item-visible-frame';\n      this.dom.box.appendChild(this.dom.visibleFrame);\n\n      // contents box\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.frame.appendChild(this.dom.content);\n\n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n\n  }\n\n  /**\n   * append element to DOM\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n      foreground.appendChild(this.dom.box);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update dirty DOM components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // update dirty DOM. An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n\n      const editable = (this.editable.updateTime || this.editable.updateGroup);\n\n      // update class\n      const className = (this.data.className ? (' ' + this.data.className) : '') +\n          (this.selected ? ' vis-selected' : '') + \n          (editable ? ' vis-editable' : ' vis-readonly');\n      this.dom.box.className = this.baseClassName + className;\n\n      // turn off max-width to be able to calculate the real width\n      // this causes an extra browser repaint/reflow, but so be it\n      this.dom.content.style.maxWidth = 'none';\n    }\n  }\n\n  /**\n   * get DOM component sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    // determine from css whether this box has overflow\n    this.overflow = window.getComputedStyle(this.dom.frame).overflow !== 'hidden';\n    this.whiteSpace = window.getComputedStyle(this.dom.content).whiteSpace !== 'nowrap';\n    return {\n      content: {\n        width: this.dom.content.offsetWidth,\n      },\n      box: {\n        height: this.dom.box.offsetHeight\n      }\n    }\n  }\n\n  /**\n   * update DOM component sizes\n   * @param {array} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    this.props.content.width = sizes.content.width;\n    this.height = sizes.box.height;\n    this.dom.content.style.maxWidth = '';\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additional components\n   * @private\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n    this._repaintDeleteButton(this.dom.box);\n    this._repaintDragCenter();\n    this._repaintDragLeft();\n    this._repaintDragRight();\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      // update dirty DOM \n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes.bind(this)();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Show the item in the DOM (when not already visible). The items DOM will\n   * be created when needed.\n   * @param {boolean} [returnQueue=false]  whether to return a queue of functions to execute instead of just executing them\n   * @return {boolean} the redraw queue if returnQueue=true\n   */\n  show(returnQueue) {\n    if (!this.displayed) {\n      return this.redraw(returnQueue);\n    }\n  }\n\n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      const box = this.dom.box;\n\n      if (box.parentNode) {\n        box.parentNode.removeChild(box);\n      }\n\n      this.displayed = false;\n    }\n  }\n\n  /**\n   * Reposition the item horizontally\n   * @param {boolean} [limitSize=true] If true (default), the width of the range\n   *                                   item will be limited, as the browser cannot\n   *                                   display very wide divs. This means though\n   *                                   that the applied left and width may\n   *                                   not correspond to the ranges start and end\n   * @Override\n   */\n  repositionX(limitSize) {\n    const parentWidth = this.parent.width;\n    let start = this.conversion.toScreen(this.data.start);\n    let end = this.conversion.toScreen(this.data.end);\n    const align = this.data.align === undefined ? this.options.align : this.data.align;\n    let contentStartPosition;\n    let contentWidth;\n\n    // limit the width of the range, as browsers cannot draw very wide divs\n    // unless limitSize: false is explicitly set in item data\n    if (this.data.limitSize !== false && (limitSize === undefined || limitSize === true)) {\n      if (start < -parentWidth) {\n        start = -parentWidth;\n      }\n      if (end > 2 * parentWidth) {\n        end = 2 * parentWidth;\n      }\n    }\n\n    //round to 3 decimals to compensate floating-point values rounding\n    const boxWidth = Math.max(Math.round((end - start) * 1000) / 1000, 1);\n\n    if (this.overflow) {\n      if (this.options.rtl) {\n        this.right = start;\n      } else {\n        this.left = start;\n      }\n      this.width = boxWidth + this.props.content.width;\n      contentWidth = this.props.content.width;\n\n      // Note: The calculation of width is an optimistic calculation, giving\n      //       a width which will not change when moving the Timeline\n      //       So no re-stacking needed, which is nicer for the eye;\n    }\n    else {\n      if (this.options.rtl) {\n        this.right = start;\n      } else {\n        this.left = start;\n      }\n      this.width = boxWidth;\n      contentWidth = Math.min(end - start, this.props.content.width);\n    }\n\n    if (this.options.rtl) {\n      this.dom.box.style.transform = `translateX(${this.right * -1}px)`;\n    } else {\n      this.dom.box.style.transform = `translateX(${this.left}px)`;\n    }\n    this.dom.box.style.width = `${boxWidth}px`;\n    if (this.whiteSpace) {\n        this.height = this.dom.box.offsetHeight;\n    }\n\n    switch (align) {\n      case 'left':\n        this.dom.content.style.transform = 'translateX(0)';\n        break;\n\n      case 'right':\n        if (this.options.rtl) {\n          const translateX = Math.max((boxWidth - contentWidth), 0) * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${Math.max((boxWidth - contentWidth), 0)}px)`;\n        }\n        break;\n\n      case 'center':\n        if (this.options.rtl) {\n          const translateX = Math.max((boxWidth - contentWidth) / 2, 0) * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${Math.max((boxWidth - contentWidth) / 2, 0)}px)`;\n        }\n        \n        break;\n\n      default: // 'auto'\n        // when range exceeds left of the window, position the contents at the left of the visible area\n        if (this.overflow) {\n          if (end > 0) {\n            contentStartPosition = Math.max(-start, 0);\n          }\n          else {\n            contentStartPosition = -contentWidth; // ensure it's not visible anymore\n          }\n        }\n        else {\n          if (start < 0) {\n            contentStartPosition = -start;\n          }\n          else {\n            contentStartPosition = 0;\n          }\n        }\n        if (this.options.rtl) {\n          const translateX = contentStartPosition * -1;\n          this.dom.content.style.transform = `translateX(${translateX}px)`;\n        } else {\n          this.dom.content.style.transform = `translateX(${contentStartPosition}px)`;\n          // this.dom.content.style.width = `calc(100% - ${contentStartPosition}px)`;\n        }\n    }\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY() {\n    const orientation = this.options.orientation.item;\n    const box = this.dom.box;\n\n    if (orientation == 'top') {\n      box.style.top = `${this.top}px`;\n    }\n    else {\n      box.style.top = `${this.parent.height - this.top - this.height}px`;\n    }\n  }\n\n  /**\n   * Repaint a drag area on the left side of the range when the range is selected\n   * @protected\n   */\n  _repaintDragLeft() {\n    if ((this.selected || this.options.itemsAlwaysDraggable.range) && this.editable.updateTime && !this.dom.dragLeft) {\n      // create and show drag area\n      const dragLeft = document.createElement('div');\n      dragLeft.className = 'vis-drag-left';\n      dragLeft.dragLeftItem = this;\n\n      this.dom.box.appendChild(dragLeft);\n      this.dom.dragLeft = dragLeft;\n    }\n    else if (!this.selected && !this.options.itemsAlwaysDraggable.range && this.dom.dragLeft) {\n      // delete drag area\n      if (this.dom.dragLeft.parentNode) {\n        this.dom.dragLeft.parentNode.removeChild(this.dom.dragLeft);\n      }\n      this.dom.dragLeft = null;\n    }\n  }\n\n  /**\n   * Repaint a drag area on the right side of the range when the range is selected\n   * @protected\n   */\n  _repaintDragRight() {\n    if ((this.selected || this.options.itemsAlwaysDraggable.range) && this.editable.updateTime && !this.dom.dragRight) {\n      // create and show drag area\n      const dragRight = document.createElement('div');\n      dragRight.className = 'vis-drag-right';\n      dragRight.dragRightItem = this;\n\n      this.dom.box.appendChild(dragRight);\n      this.dom.dragRight = dragRight;\n    }\n    else if (!this.selected && !this.options.itemsAlwaysDraggable.range && this.dom.dragRight) {\n      // delete drag area\n      if (this.dom.dragRight.parentNode) {\n        this.dom.dragRight.parentNode.removeChild(this.dom.dragRight);\n      }\n      this.dom.dragRight = null;\n    }\n  }\n}\n\nRangeItem.prototype.baseClassName = 'vis-item vis-range';\n\nexport default RangeItem;\n","import Item from './Item';\nimport BackgroundGroup from '../BackgroundGroup';\nimport RangeItem from './RangeItem';\n\n/**\n * @constructor BackgroundItem\n * @extends Item\n */\nclass BackgroundItem extends Item {\n  /**\n * @constructor BackgroundItem\n * @param {Object} data             Object containing parameters start, end\n *                                  content, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} [options]        Configuration options\n *                                  // TODO: describe options\n * // TODO: implement support for the BackgroundItem just having a start, then being displayed as a sort of an annotation\n */\n  constructor(data, conversion, options) {\n    super(data, conversion, options)\n    this.props = {\n      content: {\n        width: 0\n      }\n    };\n    this.overflow = false; // if contents can overflow (css styling), this flag is set to true\n\n    // validate data\n    if (data) {\n      if (data.start == undefined) {\n        throw new Error(`Property \"start\" missing in item ${data.id}`);\n      }\n      if (data.end == undefined) {\n        throw new Error(`Property \"end\" missing in item ${data.id}`);\n      }\n    }\n  }\n\n  /**\n   * Check whether this item is visible inside given range\n   * @param {timeline.Range} range with a timestamp for start and end\n   * @returns {boolean} True if visible\n   */\n  isVisible(range) {\n    // determine visibility\n    return (this.data.start < range.end) && (this.data.end > range.start); \n  }\n\n  /**\n   * create DOM element\n   * @private\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n\n      // background box\n      this.dom.box = document.createElement('div');\n      // className is updated in redraw()\n\n      // frame box (to prevent the item contents from overflowing\n      this.dom.frame = document.createElement('div');\n      this.dom.frame.className = 'vis-item-overflow';\n      this.dom.box.appendChild(this.dom.frame);\n\n      // contents box\n      this.dom.content = document.createElement('div');\n      this.dom.content.className = 'vis-item-content';\n      this.dom.frame.appendChild(this.dom.content);\n\n      // Note: we do NOT attach this item as attribute to the DOM,\n      //       such that background items cannot be selected\n      //this.dom.box['vis-item'] = this;\n\n      this.dirty = true;\n    }\n  }\n\n  /**\n   * append DOM element\n   * @private\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n    if (!this.dom.box.parentNode) {\n      const background = this.parent.dom.background;\n      if (!background) {\n        throw new Error('Cannot redraw item: parent has no background container element');\n      }\n      background.appendChild(this.dom.box);\n    }\n    this.displayed = true;\n  }\n\n  /**\n   * update DOM Dirty components\n   * @private\n   */\n  _updateDirtyDomComponents() {\n    // update dirty DOM. An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.content);\n      this._updateStyle(this.dom.box);\n\n      // update class\n      const className = (this.data.className ? (' ' + this.data.className) : '') +\n          (this.selected ? ' vis-selected' : '');\n      this.dom.box.className = this.baseClassName + className;\n    }\n  }\n\n  /**\n   * get DOM components sizes\n   * @return {object}\n   * @private\n   */\n  _getDomComponentsSizes() {\n    // determine from css whether this box has overflow\n    this.overflow = window.getComputedStyle(this.dom.content).overflow !== 'hidden';\n    return {\n      content: {\n        width: this.dom.content.offsetWidth\n      }\n    }\n  }\n\n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   * @private\n   */\n  _updateDomComponentsSizes(sizes) {\n    // recalculate size\n    this.props.content.width = sizes.content.width;\n    this.height = 0; // set height zero, so this item will be ignored when stacking items\n\n    this.dirty = false;\n  }\n\n  /**\n   * repaint DOM additionals\n   * @private\n   */\n  _repaintDomAdditionals() {\n  }\n\n  /**\n   * Repaint the item\n   * @param {boolean} [returnQueue=false]  return the queue\n   * @return {boolean} the redraw result or the redraw queue if returnQueue=true\n   */\n  redraw(returnQueue) {\n    let sizes;\n    const queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n\n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n\n      this._updateDirtyDomComponents.bind(this),\n\n      () => {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes.bind(this)();\n        }\n      },\n\n      () => {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      },\n\n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n\n    if (returnQueue) {\n      return queue;\n    } else {\n      let result;\n      queue.forEach(fn => {\n        result = fn();\n      });\n      return result;\n    }\n  }\n\n  /**\n   * Reposition the item vertically\n   * @Override\n   */\n  repositionY(margin) {  // eslint-disable-line no-unused-vars\n    let height;\n    const orientation = this.options.orientation.item;\n\n    // special positioning for subgroups\n    if (this.data.subgroup !== undefined) {\n      // TODO: instead of calculating the top position of the subgroups here for every BackgroundItem, calculate the top of the subgroup once in Itemset\n      const itemSubgroup = this.data.subgroup;\n\n      this.dom.box.style.height = `${this.parent.subgroups[itemSubgroup].height}px`;\n\n      if (orientation == 'top') { \n        this.dom.box.style.top = `${this.parent.top + this.parent.subgroups[itemSubgroup].top}px`;\n      } else {\n        this.dom.box.style.top = `${this.parent.top + this.parent.height - this.parent.subgroups[itemSubgroup].top - this.parent.subgroups[itemSubgroup].height}px`;\n      }\n      this.dom.box.style.bottom = '';\n    }\n    // and in the case of no subgroups:\n    else {\n      // we want backgrounds with groups to only show in groups.\n      if (this.parent instanceof BackgroundGroup) {\n        // if the item is not in a group:\n        height = Math.max(this.parent.height,\n            this.parent.itemSet.body.domProps.center.height,\n            this.parent.itemSet.body.domProps.centerContainer.height);\n        this.dom.box.style.bottom = orientation == 'bottom' ? '0' : '';\n        this.dom.box.style.top = orientation == 'top' ? '0' : '';\n      }\n      else {\n        height = this.parent.height;\n        // same alignment for items when orientation is top or bottom\n        this.dom.box.style.top = `${this.parent.top}px`;\n        this.dom.box.style.bottom = '';\n      }\n    }\n    this.dom.box.style.height = `${height}px`;\n  }\n}\n\nBackgroundItem.prototype.baseClassName = 'vis-item vis-background';\n\nBackgroundItem.prototype.stack = false;\n\n/**\n * Show the item in the DOM (when not already visible). The items DOM will\n * be created when needed.\n */\nBackgroundItem.prototype.show = RangeItem.prototype.show;\n\n/**\n * Hide the item from the DOM (when visible)\n * @return {Boolean} changed\n */\nBackgroundItem.prototype.hide = RangeItem.prototype.hide;\n\n/**\n * Reposition the item horizontally\n * @Override\n */\nBackgroundItem.prototype.repositionX = RangeItem.prototype.repositionX;\n\nexport default BackgroundItem;\n","import './tooltip.css';\n\n/**\n * Popup is a class to create a popup window with some text\n */\nclass Popup {\n  /**\n   * @param {Element} container       The container object.\n   * @param {string}  overflowMethod  How the popup should act to overflowing ('flip', 'cap' or 'none')\n   */\n  constructor(container, overflowMethod) {\n    this.container = container;\n    this.overflowMethod = overflowMethod || 'cap';\n\n    this.x = 0;\n    this.y = 0;\n    this.padding = 5;\n    this.hidden = false;\n\n    // create the frame\n    this.frame = document.createElement('div');\n    this.frame.className = 'vis-tooltip';\n    this.container.appendChild(this.frame);\n  }\n\n  /**\n   * @param {number} x   Horizontal position of the popup window\n   * @param {number} y   Vertical position of the popup window\n   */\n  setPosition(x, y) {\n    this.x = parseInt(x);\n    this.y = parseInt(y);\n  }\n\n  /**\n   * Set the content for the popup window. This can be HTML code or text.\n   * @param {string | Element} content\n   */\n  setText(content) {\n    if (content instanceof Element) {\n      this.frame.innerHTML = '';\n      this.frame.appendChild(content);\n    }\n    else {\n      this.frame.innerHTML = content; // string containing text or HTML\n    }\n  }\n\n  /**\n   * Show the popup window\n   * @param {boolean} [doShow]    Show or hide the window\n   */\n  show(doShow) {\n    if (doShow === undefined) {\n      doShow = true;\n    }\n\n    if (doShow === true) {\n      var height = this.frame.clientHeight;\n      var width = this.frame.clientWidth;\n      var maxHeight = this.frame.parentNode.clientHeight;\n      var maxWidth = this.frame.parentNode.clientWidth;\n\n      var left = 0, top = 0;\n\n      if (this.overflowMethod == 'flip' || this.overflowMethod == 'none') {\n        let isLeft = false, isTop = true; // Where around the position it's located\n\n        if (this.overflowMethod == 'flip') {\n          if (this.y - height < this.padding) {\n            isTop = false;\n          }\n\n          if (this.x + width > maxWidth - this.padding) {\n            isLeft = true;\n          }\n        }\n\n        if (isLeft) {\n          left = this.x - width;\n        } else {\n          left = this.x;\n        }\n\n        if (isTop) {\n          top = this.y - height;\n        } else {\n          top = this.y;\n        }\n      } else { // this.overflowMethod == 'cap'\n        top = (this.y - height);\n        if (top + height + this.padding > maxHeight) {\n          top = maxHeight - height - this.padding;\n        }\n        if (top < this.padding) {\n          top = this.padding;\n        }\n\n        left = this.x;\n        if (left + width + this.padding > maxWidth) {\n          left = maxWidth - width - this.padding;\n        }\n        if (left < this.padding) {\n          left = this.padding;\n        }\n      }\n\n      this.frame.style.left = left + \"px\";\n      this.frame.style.top = top + \"px\";\n      this.frame.style.visibility = \"visible\";\n      this.hidden = false;\n    }\n    else {\n      this.hide();\n    }\n  }\n\n  /**\n   * Hide the popup window\n   */\n  hide() {\n    this.hidden = true;\n    this.frame.style.left = \"0\";\n    this.frame.style.top = \"0\";\n    this.frame.style.visibility = \"hidden\";\n  }\n\n  /**\n   * Remove the popup window\n   */\n  destroy() {\n    this.frame.parentNode.removeChild(this.frame); // Remove element from DOM\n  }\n}\n\nexport default Popup;\n","'use strict';\nvar $ = require('../internals/export');\nvar $every = require('../internals/array-iteration').every;\nvar arrayMethodIsStrict = require('../internals/array-method-is-strict');\nvar arrayMethodUsesToLength = require('../internals/array-method-uses-to-length');\n\nvar STRICT_METHOD = arrayMethodIsStrict('every');\nvar USES_TO_LENGTH = arrayMethodUsesToLength('every');\n\n// `Array.prototype.every` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.every\n$({ target: 'Array', proto: true, forced: !STRICT_METHOD || !USES_TO_LENGTH }, {\n  every: function every(callbackfn /* , thisArg */) {\n    return $every(this, callbackfn, arguments.length > 1 ? arguments[1] : undefined);\n  }\n});\n","require('../../../modules/es.array.every');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').every;\n","var every = require('../array/virtual/every');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.every;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.every) ? every : own;\n};\n","import { v4 as randomUUID } from \"uuid\";\nimport Item from './Item';\n\n/**\n * ClusterItem\n */\nclass ClusterItem extends Item {\n  /**\n * @constructor Item\n * @param {Object} data             Object containing (optional) parameters type,\n *                                  start, end, content, group, className.\n * @param {{toScreen: function, toTime: function}} conversion\n *                                  Conversion functions from time to screen and vice versa\n * @param {Object} options          Configuration options\n *                                  // TODO: describe available options\n */\n  constructor(data, conversion, options) {\n    const modifiedOptions = Object.assign({}, {fitOnDoubleClick: true}, options, {editable: false});\n    super(data, conversion, modifiedOptions);\n\n    this.props = {\n      content: {\n        width: 0,\n        height: 0\n      },\n    };\n    \n    if (!data || data.uiItems == undefined) {\n      throw new Error('Property \"uiItems\" missing in item ' + data.id);\n    }\n  \n    this.id = randomUUID();\n    this.group = data.group;\n    this._setupRange();\n  \n    this.emitter = this.data.eventEmitter;\n    this.range = this.data.range;\n    this.attached = false;\n    this.isCluster = true;\n    this.data.isCluster = true;\n  }\n\n  /**\n   * check if there are items\n   * @return {boolean}\n   */\n  hasItems() {\n    return this.data.uiItems && this.data.uiItems.length && this.attached;\n  }\n  \n  /**\n   * set UI items\n   * @param {array} items\n   */\n  setUiItems(items) {\n    this.detach();\n  \n    this.data.uiItems = items;\n  \n    this._setupRange();\n  \n    this.attach();\n  }\n  \n  /**\n   * check is visible\n   * @param {object} range\n   * @return {boolean}\n   */\n  isVisible(range) {\n    const rangeWidth = this.data.end ? this.data.end - this.data.start : 0;\n    const widthInMs = this.width * range.getMillisecondsPerPixel();\n    const end = Math.max(rangeWidth, this.data.start.getTime() + widthInMs);\n    return (this.data.start < range.end) && (end > range.start) && this.hasItems();\n  }\n  \n  /**\n   * get cluster data\n   * @return {object}\n   */\n  getData() {\n    return {\n      isCluster: true,\n      id: this.id,\n      items: this.data.items || [],\n      data: this.data\n    }\n  }\n  \n  /**\n   * redraw cluster item\n   * @param {boolean} returnQueue\n   * @return {boolean}\n   */\n  redraw (returnQueue) {\n    var sizes\n    var queue = [\n      // create item DOM\n      this._createDomElement.bind(this),\n  \n      // append DOM to parent DOM\n      this._appendDomElement.bind(this),\n  \n      // update dirty DOM\n      this._updateDirtyDomComponents.bind(this),\n  \n      (function () {\n        if (this.dirty) {\n          sizes = this._getDomComponentsSizes();\n        }\n      }).bind(this),\n  \n      (function () {\n        if (this.dirty) {\n          this._updateDomComponentsSizes.bind(this)(sizes);\n        }\n      }).bind(this),\n  \n      // repaint DOM additionals\n      this._repaintDomAdditionals.bind(this)\n    ];\n  \n    if (returnQueue) {\n      return queue;\n    } else {\n      var result;\n      queue.forEach(function (fn) {\n        result = fn();\n      });\n      return result;\n    }\n  }\n  \n  /**\n   * show cluster item\n   */\n  show() {\n    if (!this.displayed) {\n      this.redraw();\n    }\n  }\n  \n  /**\n   * Hide the item from the DOM (when visible)\n   */\n  hide() {\n    if (this.displayed) {\n      var dom = this.dom;\n      if (dom.box.parentNode) {\n        dom.box.parentNode.removeChild(dom.box);\n      }\n\n      if (this.options.showStipes) {\n        if (dom.line.parentNode)  {\n          dom.line.parentNode.removeChild(dom.line);\n        }\n        if (dom.dot.parentNode)  {\n          dom.dot.parentNode.removeChild(dom.dot);\n        }\n      }\n      this.displayed = false;\n    }\n  }\n  \n  /**\n   * reposition item x axis\n   */\n  repositionX() {\n    let start = this.conversion.toScreen(this.data.start);\n    let end = this.data.end ? this.conversion.toScreen(this.data.end) : 0;\n    if (end) {\n      this.repositionXWithRanges(start, end);\n    } else {\n      let align = this.data.align === undefined ? this.options.align : this.data.align;\n      this.repositionXWithoutRanges(start, align);\n    }\n  \n    if (this.options.showStipes) {\n      this.dom.line.style.display = this._isStipeVisible() ? 'block' : 'none';\n      this.dom.dot.style.display = this._isStipeVisible() ? 'block' : 'none';\n\n      if (this._isStipeVisible()) {\n        this.repositionStype(start, end);\n      }\n    }\n  }\n\n  /**\n   * reposition item stype\n   * @param {date} start\n   * @param {date} end\n   */\n  repositionStype(start, end) {\n    this.dom.line.style.display = 'block';\n    this.dom.dot.style.display = 'block';\n    const lineOffsetWidth = this.dom.line.offsetWidth;\n    const dotOffsetWidth = this.dom.dot.offsetWidth;\n\n    if (end) {\n      const lineOffset = lineOffsetWidth + start + (end - start) / 2;\n      const dotOffset = lineOffset - dotOffsetWidth / 2;\n      const lineOffsetDirection = this.options.rtl ? lineOffset * -1 : lineOffset;\n      const dotOffsetDirection = this.options.rtl ? dotOffset * -1 : dotOffset;\n\n      this.dom.line.style.transform = `translateX(${lineOffsetDirection}px)`;\n      this.dom.dot.style.transform = `translateX(${dotOffsetDirection}px)`;\n    } else {\n      const lineOffsetDirection = this.options.rtl ? (start * -1) : start;\n      const dotOffsetDirection = this.options.rtl ? ((start - dotOffsetWidth / 2) * -1) : (start - dotOffsetWidth / 2);\n\n      this.dom.line.style.transform = `translateX(${lineOffsetDirection}px)`;\n      this.dom.dot.style.transform = `translateX(${dotOffsetDirection}px)`;\n    }\n  }\n  \n  /**\n   * reposition x without ranges\n   * @param {date} start\n   * @param {string} align\n   */\n  repositionXWithoutRanges(start, align) {\n    // calculate left position of the box\n    if (align == 'right') {\n      if (this.options.rtl) {\n        this.right = start - this.width;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start - this.width;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    } else if (align == 'left') {\n      if (this.options.rtl) {\n        this.right = start;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    } else {\n      // default or 'center'\n      if (this.options.rtl) {\n        this.right = start - this.width / 2;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.right = this.right + 'px';\n      } else {\n        this.left = start - this.width / 2;\n  \n        // reposition box, line, and dot\n        this.dom.box.style.left = this.left + 'px';\n      }\n    }\n  }\n  \n  /**\n   * reposition x with ranges\n   * @param {date} start\n   * @param {date} end\n   */\n  repositionXWithRanges(start, end) {\n    let boxWidth = Math.round(Math.max(end - start + 0.5, 1));\n  \n    if (this.options.rtl) {\n      this.right = start;\n    } else {\n      this.left = start;\n    }\n  \n    this.width = Math.max(boxWidth, this.minWidth || 0);\n  \n    if (this.options.rtl) {\n      this.dom.box.style.right = this.right + 'px';\n    } else {\n      this.dom.box.style.left = this.left + 'px';\n    }\n  \n    this.dom.box.style.width = boxWidth + 'px';\n  }\n  \n  /**\n   * reposition item y axis\n   */\n  repositionY() {\n    var orientation = this.options.orientation.item;\n    var box = this.dom.box;\n    if (orientation == 'top') {\n      box.style.top = (this.top || 0) + 'px';\n    } else { \n      // orientation 'bottom'\n      box.style.top = (this.parent.height - this.top - this.height || 0) + 'px';\n    }\n  \n    if (this.options.showStipes) {\n      if (orientation == 'top') {\n        this.dom.line.style.top    = '0';\n        this.dom.line.style.height = (this.parent.top + this.top + 1) + 'px';\n        this.dom.line.style.bottom = '';\n      }\n      else { \n        // orientation 'bottom'\n        var itemSetHeight = this.parent.itemSet.props.height;\n        var lineHeight = itemSetHeight - this.parent.top - this.parent.height + this.top;\n        this.dom.line.style.top    = (itemSetHeight - lineHeight) + 'px';\n        this.dom.line.style.bottom = '0';\n      }\n    \n      this.dom.dot.style.top = (-this.dom.dot.offsetHeight / 2) + 'px';\n    }\n  }\n\n  /**\n   * get width left\n   * @return {number}\n   */\n  getWidthLeft() {\n    return this.width / 2;\n  }\n  \n  /**\n   * get width right\n   * @return {number}\n   */\n  getWidthRight() {\n    return this.width / 2;\n  }\n  \n  /**\n   * move cluster item\n   */\n  move() {\n    this.repositionX();\n    this.repositionY();\n  }\n  \n  /**\n   * attach\n   */\n  attach() {\n    for (let item of this.data.uiItems) {\n      item.cluster = this;\n    }\n  \n    this.data.items = this.data.uiItems.map(item => item.data);\n\n    this.attached = true;\n    this.dirty = true;\n  }\n  \n  /**\n   * detach\n   * @param {boolean} detachFromParent\n   * @return {void}\n   */\n  detach(detachFromParent = false) {\n    if (!this.hasItems()) {\n      return;\n    }\n  \n    for (let item of this.data.uiItems) {\n      delete item.cluster;\n    }\n  \n    this.attached = false;\n  \n    if (detachFromParent && this.group) {\n      this.group.remove(this);\n      this.group = null;\n    }\n  \n    this.data.items = [];\n    this.dirty = true;\n  }\n  \n  /**\n   * handle on double click\n   */\n  _onDoubleClick() {\n   this._fit();\n  }\n  \n  /**\n   * set range\n   */\n  _setupRange() {\n    const stats = this.data.uiItems.map(item => ({\n      start: item.data.start.valueOf(),\n      end: item.data.end ? item.data.end.valueOf() : item.data.start.valueOf(),\n    }));\n  \n    this.data.min = Math.min(...stats.map(s => Math.min(s.start, s.end || s.start)));\n    this.data.max = Math.max(...stats.map(s => Math.max(s.start, s.end || s.start)));\n    const centers = this.data.uiItems.map(item => item.center);\n    const avg = centers.reduce((sum, value) => sum + value, 0) / this.data.uiItems.length;\n  \n    if (this.data.uiItems.some(item => item.data.end)) {\n      // contains ranges\n      this.data.start = new Date(this.data.min);\n      this.data.end = new Date(this.data.max);\n    } else {\n      this.data.start = new Date(avg);\n      this.data.end = null;\n    }\n  }\n  \n  /**\n   * get UI items\n   * @return {array}\n   */\n  _getUiItems() {\n    if (this.data.uiItems && this.data.uiItems.length) {\n      return this.data.uiItems.filter(item => item.cluster === this);\n    }\n  \n    return [];\n  }\n  \n  /**\n   * create DOM element\n   */\n  _createDomElement() {\n    if (!this.dom) {\n      // create DOM\n      this.dom = {};\n  \n      // create main box\n      this.dom.box = document.createElement('DIV');\n  \n      // contents box (inside the background box). used for making margins\n      this.dom.content = document.createElement('DIV');\n      this.dom.content.className = 'vis-item-content';\n      \n      this.dom.box.appendChild(this.dom.content);\n  \n      if (this.options.showStipes) {\n        // line to axis\n        this.dom.line = document.createElement('DIV');\n        this.dom.line.className = 'vis-cluster-line';\n        this.dom.line.style.display = 'none';\n  \n        // dot on axis\n        this.dom.dot = document.createElement('DIV');\n        this.dom.dot.className = 'vis-cluster-dot';\n        this.dom.dot.style.display = 'none';\n      }\n  \n      if (this.options.fitOnDoubleClick) {\n        this.dom.box.ondblclick = ClusterItem.prototype._onDoubleClick.bind(this);\n      }\n      \n      // attach this item as attribute\n      this.dom.box['vis-item'] = this;\n  \n      this.dirty = true;\n    }\n  }\n  \n  /**\n   * append element to DOM\n   */\n  _appendDomElement() {\n    if (!this.parent) {\n      throw new Error('Cannot redraw item: no parent attached');\n    }\n  \n    if (!this.dom.box.parentNode) {\n      const foreground = this.parent.dom.foreground;\n      if (!foreground) {\n        throw new Error('Cannot redraw item: parent has no foreground container element');\n      }\n  \n      foreground.appendChild(this.dom.box);\n    }\n  \n    const background = this.parent.dom.background;\n  \n    if (this.options.showStipes) {\n      if (!this.dom.line.parentNode) {\n        if (!background) throw new Error('Cannot redraw item: parent has no background container element');\n        background.appendChild(this.dom.line);\n      }\n    \n      if (!this.dom.dot.parentNode) {\n        var axis = this.parent.dom.axis;\n        if (!background) throw new Error('Cannot redraw item: parent has no axis container element');\n        axis.appendChild(this.dom.dot);\n      }\n    }\n  \n    this.displayed = true;\n  }\n  \n  /**\n   * update dirty DOM components\n   */\n  _updateDirtyDomComponents() {\n    // An item is marked dirty when:\n    // - the item is not yet rendered\n    // - the item's data is changed\n    // - the item is selected/deselected\n    if (this.dirty) {\n      this._updateContents(this.dom.content);\n      this._updateDataAttributes(this.dom.box);\n      this._updateStyle(this.dom.box);\n  \n      // update class\n      const className = this.baseClassName + ' ' + (this.data.className ? ' ' + this.data.className : '') +\n        (this.selected ? ' vis-selected' : '') + ' vis-readonly';\n      this.dom.box.className = 'vis-item ' + className;\n  \n      if (this.options.showStipes) {\n        this.dom.line.className = 'vis-item vis-cluster-line ' +  (this.selected ? ' vis-selected' : '');\n        this.dom.dot.className  = 'vis-item vis-cluster-dot ' +  (this.selected ? ' vis-selected' : '');\n      }\n  \n      if (this.data.end) {\n        // turn off max-width to be able to calculate the real width\n        // this causes an extra browser repaint/reflow, but so be it\n        this.dom.content.style.maxWidth = 'none';\n      }\n    }\n  }\n  \n  /**\n   * get DOM components sizes\n   * @return {object}\n   */\n  _getDomComponentsSizes() {\n    const sizes = {\n      previous: {\n        right: this.dom.box.style.right,\n        left: this.dom.box.style.left\n      },\n      box: {\n        width: this.dom.box.offsetWidth,\n        height: this.dom.box.offsetHeight\n      },\n    };\n\n    if (this.options.showStipes) {\n      sizes.dot = {\n        height: this.dom.dot.offsetHeight,\n        width: this.dom.dot.offsetWidth\n      };\n      sizes.line = {\n        width: this.dom.line.offsetWidth\n      };\n    }\n\n    return sizes;\n  }\n  \n  /**\n   * update DOM components sizes\n   * @param {object} sizes\n   */\n  _updateDomComponentsSizes(sizes) {\n    if (this.options.rtl) {\n      this.dom.box.style.right = \"0px\";\n    } else {\n      this.dom.box.style.left = \"0px\";\n    }\n  \n    // recalculate size\n    if (!this.data.end) {\n      this.width = sizes.box.width;\n    } else {\n      this.minWidth = sizes.box.width;\n    }\n  \n    this.height = sizes.box.height;\n  \n    // restore previous position\n    if (this.options.rtl) {\n      this.dom.box.style.right = sizes.previous.right;\n    } else {\n      this.dom.box.style.left = sizes.previous.left;\n    }\n  \n    this.dirty = false;\n  }\n  \n  /**\n   * repaint DOM additional components\n   */\n  _repaintDomAdditionals() {\n    this._repaintOnItemUpdateTimeTooltip(this.dom.box);\n  }\n  \n  /**\n   * check is stripe visible\n   * @return {number}\n   * @private\n   */\n  _isStipeVisible() {\n    return this.minWidth >= this.width || !this.data.end;\n  }\n  \n  /**\n   * get fit range\n   * @return {object}\n   * @private\n   */\n  _getFitRange() {\n    const offset = 0.05*(this.data.max - this.data.min) / 2;\n      return {\n        fitStart: this.data.min - offset,\n        fitEnd: this.data.max + offset\n      };\n  }\n  \n   /**\n   * fit\n   * @private\n   */\n  _fit() {\n    if (this.emitter) {\n      const {fitStart, fitEnd} = this._getFitRange();\n  \n      const fitArgs = {\n        start: new Date(fitStart),\n        end: new Date(fitEnd),\n        animation: true\n      };\n  \n      this.emitter.emit('fit', fitArgs);\n    }\n  }\n\n   /**\n   * get item data\n   * @return {object}\n   * @private\n   */\n  _getItemData() {\n    return this.data;\n  }\n}\n\nClusterItem.prototype.baseClassName = 'vis-item vis-range vis-cluster';\nexport default ClusterItem;\n","import ClusterItem from './item/ClusterItem';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n/**\n * An Cluster generator generates cluster items\n */\nexport default class ClusterGenerator {\n    /**\n     * @param {ItemSet} itemSet itemsSet instance\n     * @constructor ClusterGenerator\n    */\n    constructor(itemSet) {\n        this.itemSet = itemSet;\n        this.groups = {};\n        this.cache = {};\n        this.cache[-1] = [];\n    }\n\n    /**\n     * @param {Object} itemData             Object containing parameters start content, className.\n     * @param {{toScreen: function, toTime: function}} conversion\n     *                                  Conversion functions from time to screen and vice versa\n     * @param {Object} [options]        Configuration options\n     * @return {Object} newItem\n    */\n    createClusterItem(itemData, conversion, options) {\n        const newItem = new ClusterItem(itemData, conversion, options);\n        return newItem;\n    }\n\n    /**\n     * Set the items to be clustered.\n     * This will clear cached clusters.\n     * @param {Item[]} items\n     * @param {Object} [options]  Available options:\n     *                            {boolean} applyOnChangedLevel\n     *                                If true (default), the changed data is applied\n     *                                as soon the cluster level changes. If false,\n     *                                The changed data is applied immediately\n     */\n    setItems(items, options) {\n        this.items = items || [];\n        this.dataChanged = true;\n        this.applyOnChangedLevel = false;\n\n        if (options && options.applyOnChangedLevel) {\n            this.applyOnChangedLevel = options.applyOnChangedLevel;\n        }\n    }\n\n    /**\n     * Update the current data set: clear cache, and recalculate the clustering for\n     * the current level\n     */\n    updateData() {\n        this.dataChanged = true;\n        this.applyOnChangedLevel = false;\n    }\n\n    /**\n     * Cluster the items which are too close together\n     * @param {array} oldClusters \n     * @param {number} scale      The scale of the current window : (windowWidth / (endDate - startDate)) \n     * @param {{maxItems: number, clusterCriteria: function, titleTemplate: string}} options             \n     * @return {array} clusters\n    */\n    getClusters(oldClusters, scale, options) {\n        let { maxItems, clusterCriteria } = typeof options === \"boolean\" ? {} : options;\n    \n        if (!clusterCriteria) {\n            clusterCriteria = () => true;\n        }\n\n        maxItems = maxItems || 1;\n\n        let level = -1;\n        let granularity = 2;\n        let timeWindow = 0;\n\n        if (scale > 0) {\n            if (scale >= 1) {\n                return [];\n            }\n\n            level = Math.abs(Math.round(Math.log(100 / scale) / Math.log(granularity)));\n            timeWindow = Math.abs(Math.pow(granularity, level));\n        }\n\n        // clear the cache when and re-generate groups the data when needed.\n        if (this.dataChanged) {\n            const levelChanged = (level != this.cacheLevel);\n            const applyDataNow = this.applyOnChangedLevel ? levelChanged : true;\n            if (applyDataNow) {\n                this._dropLevelsCache();\n                this._filterData();\n            }\n        }\n\n        this.cacheLevel = level;\n        let clusters = this.cache[level];\n        if (!clusters) {\n            clusters = [];\n            for (let groupName in this.groups) {\n                if (this.groups.hasOwnProperty(groupName)) {\n                    const items = this.groups[groupName];\n                    const iMax = items.length;\n                    let i = 0;\n                    while (i < iMax) {\n                        // find all items around current item, within the timeWindow\n                        let item = items[i];\n                        let neighbors = 1; // start at 1, to include itself)\n\n                        // loop through items left from the current item\n                        let j = i - 1;\n                        while (j >= 0 && (item.center - items[j].center) < timeWindow / 2) {\n                            if (!items[j].cluster && clusterCriteria(item.data, items[j].data)) {\n                                neighbors++;\n                            }\n                            j--;\n                        }\n\n                        // loop through items right from the current item\n                        let k = i + 1;\n                        while (k < items.length && (items[k].center - item.center) < timeWindow / 2) {\n                            if (clusterCriteria(item.data, items[k].data)) {\n                                neighbors++;\n                            }\n                            k++;\n                        }\n\n                        // loop through the created clusters\n                        let l = clusters.length - 1;\n                        while (l >= 0 && (item.center - clusters[l].center) < timeWindow) {\n                            if (item.group == clusters[l].group && clusterCriteria(item.data, clusters[l].data)) {\n                                neighbors++;\n                            }\n                            l--;\n                        }\n\n                        // aggregate until the number of items is within maxItems\n                        if (neighbors > maxItems) {\n                            // too busy in this window.\n                            const num = neighbors - maxItems + 1;\n                            const clusterItems = [];\n\n                            // append the items to the cluster,\n                            // and calculate the average start for the cluster\n                            let m = i;\n                            while (clusterItems.length < num && m < items.length) {\n                                if (clusterCriteria(items[m].data, items[m].data)) {\n                                    clusterItems.push(items[m]);\n                                }\n                                m++;\n                            }\n\n                            const groupId = this.itemSet.getGroupId(item.data);\n                            const group = this.itemSet.groups[groupId] || this.itemSet.groups[ReservedGroupIds.UNGROUPED];\n                            let cluster = this._getClusterForItems(clusterItems, group, oldClusters, options);\n                            clusters.push(cluster);\n\n                            i += num;\n                        } else {\n                            delete item.cluster;\n                            i += 1;\n                        }\n                    }\n                }\n            }\n\n            this.cache[level] = clusters;\n        }\n\n        return clusters;\n    }\n\n    /**\n     * Filter the items per group.\n     * @private\n     */\n    _filterData() {\n        // filter per group\n        const groups = {};\n        this.groups = groups;\n\n        // split the items per group\n        for (const item of Object.values(this.items)) {\n            // put the item in the correct group\n            const groupName = item.parent ? item.parent.groupId : '';\n            let group = groups[groupName];\n            if (!group) {\n                group = [];\n                groups[groupName] = group;\n            }\n            group.push(item);\n\n            // calculate the center of the item\n            if (item.data.start) {\n                if (item.data.end) {\n                    // range\n                    item.center = (item.data.start.valueOf() + item.data.end.valueOf()) / 2;\n                } else {\n                    // box, dot\n                    item.center = item.data.start.valueOf();\n                }\n            }\n        }\n\n        // sort the items per group\n        for (let currentGroupName in groups) {\n            if (groups.hasOwnProperty(currentGroupName)) {\n                groups[currentGroupName].sort((a, b) => a.center - b.center);\n            }\n        }\n\n        this.dataChanged = false;\n    }\n\n    /**\n     * Create new cluster or return existing\n     * @private\n     * @param {array} clusterItems    \n     * @param {object} group \n     * @param {array} oldClusters \n     * @param {object} options \n     * @returns {object} cluster\n     */\n    _getClusterForItems(clusterItems, group, oldClusters, options) {\n        const oldClustersLookup = (oldClusters || []).map(cluster => ({\n            cluster,\n            itemsIds: new Set(cluster.data.uiItems.map(item => item.id))\n        }));\n        let cluster;\n        if (oldClustersLookup.length) {\n            for (let oldClusterData of oldClustersLookup) {\n                if (oldClusterData.itemsIds.size === clusterItems.length \n                    && clusterItems.every(clusterItem => oldClusterData.itemsIds.has(clusterItem.id))) {\n                    cluster = oldClusterData.cluster;\n                    break;\n                }\n            }\n        }\n\n        if (cluster) {\n            cluster.setUiItems(clusterItems);\n            if (cluster.group !== group) {\n                if (cluster.group) {\n                    cluster.group.remove(cluster);    \n                }\n\n                if (group) {\n                    group.add(cluster);    \n                    cluster.group = group;\n                }\n            }\n            return cluster;\n        }\n\n        let titleTemplate = options.titleTemplate || '';\n        const conversion = {\n            toScreen: this.itemSet.body.util.toScreen,\n            toTime: this.itemSet.body.util.toTime\n        };\n\n        const title = titleTemplate.replace(/{count}/, clusterItems.length);\n        const clusterContent = '<div title=\"' + title + '\">' + clusterItems.length + '</div>';\n        const clusterOptions = Object.assign({}, options, this.itemSet.options);\n        const data = {\n            'content': clusterContent,\n            'title': title,\n            'group': group,\n            'uiItems': clusterItems,\n            'eventEmitter': this.itemSet.body.emitter,\n            'range': this.itemSet.body.range\n        };\n        cluster = this.createClusterItem(data,\n        conversion,\n        clusterOptions);\n\n        if (group) {\n            group.add(cluster);\n            cluster.group = group;\n        }\n\n        cluster.attach();\n\n        return cluster;\n    }\n\n    /**\n     * Drop cache\n     * @private\n     */\n    _dropLevelsCache() {\n        this.cache = {};\n        this.cacheLevel = -1;\n        this.cache[this.cacheLevel] = [];\n    }\n}\n","import { v4 as randomUUID } from \"uuid\";\nimport Hammer from '../../module/hammer';\nimport util, { typeCoerceDataSet } from '../../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport TimeStep from '../TimeStep';\nimport Component from './Component';\nimport Group from './Group';\nimport BackgroundGroup from './BackgroundGroup';\nimport BoxItem from './item/BoxItem';\nimport PointItem from './item/PointItem';\nimport RangeItem from './item/RangeItem';\nimport BackgroundItem from './item/BackgroundItem';\nimport Popup from '../../shared/Popup';\nimport ClusterGenerator from './ClusterGenerator';\n\nimport './css/itemset.css';\nimport './css/labelset.css';\n\nconst UNGROUPED = '__ungrouped__';   // reserved group id for ungrouped items\nconst BACKGROUND = '__background__'; // reserved group id for background items without group\n\nexport const ReservedGroupIds = {\n  UNGROUPED,\n  BACKGROUND\n}\n\n/**\n * An ItemSet holds a set of items and ranges which can be displayed in a\n * range. The width is determined by the parent of the ItemSet, and the height\n * is determined by the size of the items.\n */\nclass ItemSet extends Component {\n  /**\n * @param {{dom: Object, domProps: Object, emitter: Emitter, range: Range}} body\n * @param {Object} [options]      See ItemSet.setOptions for the available options.\n * @constructor ItemSet\n * @extends Component\n */\n  constructor(body, options) {\n    super()\n    this.body = body;\n    this.defaultOptions = {\n      type: null,  // 'box', 'point', 'range', 'background'\n      orientation: {\n        item: 'bottom'   // item orientation: 'top' or 'bottom'\n      },\n      align: 'auto', // alignment of box items\n      stack: true,\n      stackSubgroups: true,\n      groupOrderSwap(fromGroup, toGroup, groups) {  // eslint-disable-line no-unused-vars\n        const targetOrder = toGroup.order;\n        toGroup.order = fromGroup.order;\n        fromGroup.order = targetOrder;\n      },\n      groupOrder: 'order',\n\n      selectable: true,\n      multiselect: false,\n      longSelectPressTime: 251,\n      itemsAlwaysDraggable: {\n        item: false,\n        range: false,\n      },\n\n      editable: {\n        updateTime: false,\n        updateGroup: false,\n        add: false,\n        remove: false,\n        overrideItems: false\n      },\n\n      groupEditable: {\n        order: false,\n        add: false,\n        remove: false\n      },\n\n      snap: TimeStep.snap,\n\n      // Only called when `objectData.target === 'item'.\n      onDropObjectOnItem(objectData, item, callback) {\n        callback(item)\n      },\n      onAdd(item, callback) {\n        callback(item);\n      },\n      onUpdate(item, callback) {\n        callback(item);\n      },\n      onMove(item, callback) {\n        callback(item);\n      },\n      onRemove(item, callback) {\n        callback(item);\n      },\n      onMoving(item, callback) {\n        callback(item);\n      },\n      onAddGroup(item, callback) {\n        callback(item);\n      },\n      onMoveGroup(item, callback) {\n        callback(item);\n      },\n      onRemoveGroup(item, callback) {\n        callback(item);\n      },\n\n      margin: {\n        item: {\n          horizontal: 10,\n          vertical: 10\n        },\n        axis: 20\n      },\n\n      showTooltips: true,\n\n      tooltip: {\n        followMouse: false,\n        overflowMethod: 'flip',\n        delay: 500\n      },\n\n      tooltipOnItemUpdateTime: false\n    };\n\n    // options is shared by this ItemSet and all its items\n    this.options = util.extend({}, this.defaultOptions);\n    this.options.rtl = options.rtl;\n    this.options.onTimeout = options.onTimeout;\n\n    this.conversion = {\n      toScreen: body.util.toScreen,\n      toTime: body.util.toTime\n    };\n    this.dom = {};\n    this.props = {};\n    this.hammer = null;\n    \n    const me = this;\n    this.itemsData = null;    // DataSet\n    this.groupsData = null;   // DataSet\n    this.itemsSettingTime = null;\n    this.initialItemSetDrawn = false;\n    this.userContinueNotBail = null;  \n\n    this.sequentialSelection = false;\n    \n    // listeners for the DataSet of the items\n    this.itemListeners = {\n      'add'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onAdd(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      },\n      'update'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onUpdate(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      },\n      'remove'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onRemove(params.items);\n        if (me.options.cluster) {\n          me.clusterGenerator.setItems(me.items, { applyOnChangedLevel: false });\n        }\n        me.redraw();\n      }\n    };\n\n    // listeners for the DataSet of the groups\n    this.groupListeners = {\n      'add'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onAddGroups(params.items);\n        \n        if (me.groupsData && me.groupsData.length > 0) {\n            const groupsData = me.groupsData.getDataSet();\n            groupsData.get().forEach(groupData => {\n            if (groupData.nestedGroups) {\n              if (groupData.showNested != false) {\n                groupData.showNested = true;\n              }\n              let updatedGroups = [];\n              groupData.nestedGroups.forEach(nestedGroupId => {\n                const updatedNestedGroup = groupsData.get(nestedGroupId);\n                if (!updatedNestedGroup) { return; }\n                updatedNestedGroup.nestedInGroup = groupData.id;\n                if (groupData.showNested == false) {\n                  updatedNestedGroup.visible = false;\n                }\n                updatedGroups = updatedGroups.concat(updatedNestedGroup);\n              });\n              groupsData.update(updatedGroups, senderId);\n            }\n          });\n        }\n      },\n      'update'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onUpdateGroups(params.items);\n      },\n      'remove'(event, params, senderId) {  // eslint-disable-line no-unused-vars\n        me._onRemoveGroups(params.items);\n      }\n    };\n\n    this.items = {};      // object with an Item for every data item\n    this.groups = {};     // Group object for every group\n    this.groupIds = [];\n\n    this.selection = [];  // list with the ids of all selected nodes\n\n    this.popup = null;\n    this.popupTimer = null;\n\n    this.touchParams = {}; // stores properties while dragging\n    this.groupTouchParams = {\n      group: null,\n      isDragging: false\n    };\n  \n    // create the HTML DOM\n    this._create();\n\n    this.setOptions(options);\n    this.clusters = [];\n  }\n\n  /**\n   * Create the HTML DOM for the ItemSet\n   */\n  _create() {\n    const frame = document.createElement('div');\n    frame.className = 'vis-itemset';\n    frame['vis-itemset'] = this;\n    this.dom.frame = frame;\n\n    // create background panel\n    const background = document.createElement('div');\n    background.className = 'vis-background';\n    frame.appendChild(background);\n    this.dom.background = background;\n\n    // create foreground panel\n    const foreground = document.createElement('div');\n    foreground.className = 'vis-foreground';\n    frame.appendChild(foreground);\n    this.dom.foreground = foreground;\n\n    // create axis panel\n    const axis = document.createElement('div');\n    axis.className = 'vis-axis';\n    this.dom.axis = axis;\n\n    // create labelset\n    const labelSet = document.createElement('div');\n    labelSet.className = 'vis-labelset';\n    this.dom.labelSet = labelSet;\n\n    // create ungrouped Group\n    this._updateUngrouped();\n\n    // create background Group\n    const backgroundGroup = new BackgroundGroup(BACKGROUND, null, this);\n    backgroundGroup.show();\n    this.groups[BACKGROUND] = backgroundGroup;\n\n    // attach event listeners\n    // Note: we bind to the centerContainer for the case where the height\n    //       of the center container is larger than of the ItemSet, so we\n    //       can click in the empty area to create a new item or deselect an item.\n    this.hammer = new Hammer(this.body.dom.centerContainer);\n\n    // drag items when selected\n    this.hammer.on('hammer.input', event => {\n      if (event.isFirst) {\n        this._onTouch(event);\n      }\n    });\n    this.hammer.on('panstart', this._onDragStart.bind(this));\n    this.hammer.on('panmove',  this._onDrag.bind(this));\n    this.hammer.on('panend',   this._onDragEnd.bind(this));\n    this.hammer.get('pan').set({threshold:5, direction: Hammer.ALL});\n\n    // single select (or unselect) when tapping an item\n    this.hammer.on('tap',  this._onSelectItem.bind(this));\n\n    // multi select when holding mouse/touch, or on ctrl+click\n    this.hammer.on('press', this._onMultiSelectItem.bind(this));\n\n    // add item on doubletap\n    this.hammer.on('doubletap', this._onAddItem.bind(this));\n\n    if (this.options.rtl) {\n      this.groupHammer = new Hammer(this.body.dom.rightContainer);\n    } else {\n      this.groupHammer = new Hammer(this.body.dom.leftContainer);\n    }\n    \n    this.groupHammer.on('tap',      this._onGroupClick.bind(this));\n    this.groupHammer.on('panstart', this._onGroupDragStart.bind(this));\n    this.groupHammer.on('panmove',  this._onGroupDrag.bind(this));\n    this.groupHammer.on('panend',   this._onGroupDragEnd.bind(this));\n    this.groupHammer.get('pan').set({threshold:5, direction: Hammer.DIRECTION_VERTICAL});\n    \n    this.body.dom.centerContainer.addEventListener('mouseover', this._onMouseOver.bind(this));\n    this.body.dom.centerContainer.addEventListener('mouseout', this._onMouseOut.bind(this));\n    this.body.dom.centerContainer.addEventListener('mousemove', this._onMouseMove.bind(this));\n    // right-click on timeline \n    this.body.dom.centerContainer.addEventListener('contextmenu', this._onDragEnd.bind(this));\n\n    this.body.dom.centerContainer.addEventListener('mousewheel', this._onMouseWheel.bind(this));\n\n    // attach to the DOM\n    this.show();\n  }\n\n  /**\n   * Set options for the ItemSet. Existing options will be extended/overwritten.\n   * @param {Object} [options] The following options are available:\n   *                           {string} type\n   *                              Default type for the items. Choose from 'box'\n   *                              (default), 'point', 'range', or 'background'.\n   *                              The default style can be overwritten by\n   *                              individual items.\n   *                           {string} align\n   *                              Alignment for the items, only applicable for\n   *                              BoxItem. Choose 'center' (default), 'left', or\n   *                              'right'.\n   *                           {string} orientation.item\n   *                              Orientation of the item set. Choose 'top' or\n   *                              'bottom' (default).\n   *                           {Function} groupOrder\n   *                              A sorting function for ordering groups\n   *                           {boolean} stack\n   *                              If true (default), items will be stacked on\n   *                              top of each other.\n   *                           {number} margin.axis\n   *                              Margin between the axis and the items in pixels.\n   *                              Default is 20.\n   *                           {number} margin.item.horizontal\n   *                              Horizontal margin between items in pixels.\n   *                              Default is 10.\n   *                           {number} margin.item.vertical\n   *                              Vertical Margin between items in pixels.\n   *                              Default is 10.\n   *                           {number} margin.item\n   *                              Margin between items in pixels in both horizontal\n   *                              and vertical direction. Default is 10.\n   *                           {number} margin\n   *                              Set margin for both axis and items in pixels.\n   *                           {boolean} selectable\n   *                              If true (default), items can be selected.\n   *                           {boolean} multiselect\n   *                              If true, multiple items can be selected.\n   *                              False by default.\n   *                           {boolean} editable\n   *                              Set all editable options to true or false\n   *                           {boolean} editable.updateTime\n   *                              Allow dragging an item to an other moment in time\n   *                           {boolean} editable.updateGroup\n   *                              Allow dragging an item to an other group\n   *                           {boolean} editable.add\n   *                              Allow creating new items on double tap\n   *                           {boolean} editable.remove\n   *                              Allow removing items by clicking the delete button\n   *                              top right of a selected item.\n   *                           {Function(item: Item, callback: Function)} onAdd\n   *                              Callback function triggered when an item is about to be added:\n   *                              when the user double taps an empty space in the Timeline.\n   *                           {Function(item: Item, callback: Function)} onUpdate\n   *                              Callback function fired when an item is about to be updated.\n   *                              This function typically has to show a dialog where the user\n   *                              change the item. If not implemented, nothing happens.\n   *                           {Function(item: Item, callback: Function)} onMove\n   *                              Fired when an item has been moved. If not implemented,\n   *                              the move action will be accepted.\n   *                           {Function(item: Item, callback: Function)} onRemove\n   *                              Fired when an item is about to be deleted.\n   *                              If not implemented, the item will be always removed.\n   */\n  setOptions(options) {\n    if (options) {\n      // copy all options that we know\n      const fields = [\n        'type', 'rtl', 'align', 'order', 'stack', 'stackSubgroups', 'selectable', 'multiselect', 'sequentialSelection',\n        'multiselectPerGroup', 'longSelectPressTime', 'groupOrder', 'dataAttributes', 'template', 'groupTemplate', 'visibleFrameTemplate',\n        'hide', 'snap', 'groupOrderSwap', 'showTooltips', 'tooltip', 'tooltipOnItemUpdateTime', 'groupHeightMode', 'onTimeout'\n      ];\n      util.selectiveExtend(fields, this.options, options);\n\n      if ('itemsAlwaysDraggable' in options) {\n        if (typeof options.itemsAlwaysDraggable === 'boolean') {\n          this.options.itemsAlwaysDraggable.item = options.itemsAlwaysDraggable;\n          this.options.itemsAlwaysDraggable.range = false;\n        }\n        else if (typeof options.itemsAlwaysDraggable === 'object') {\n          util.selectiveExtend(['item', 'range'], this.options.itemsAlwaysDraggable, options.itemsAlwaysDraggable);\n          // only allow range always draggable when item is always draggable as well\n          if (! this.options.itemsAlwaysDraggable.item) {\n            this.options.itemsAlwaysDraggable.range = false;\n          }\n        }\n      }\n\n      if ('sequentialSelection' in options) {\n        if (typeof options.sequentialSelection === 'boolean') {\n          this.options.sequentialSelection = options.sequentialSelection;\n        }\n      }\n\n      if ('orientation' in options) {\n        if (typeof options.orientation === 'string') {\n          this.options.orientation.item = options.orientation === 'top' ? 'top' : 'bottom';\n        }\n        else if (typeof options.orientation === 'object' && 'item' in options.orientation) {\n          this.options.orientation.item = options.orientation.item;\n        }\n      }\n\n      if ('margin' in options) {\n        if (typeof options.margin === 'number') {\n          this.options.margin.axis = options.margin;\n          this.options.margin.item.horizontal = options.margin;\n          this.options.margin.item.vertical = options.margin;\n        }\n        else if (typeof options.margin === 'object') {\n          util.selectiveExtend(['axis'], this.options.margin, options.margin);\n          if ('item' in options.margin) {\n            if (typeof options.margin.item === 'number') {\n              this.options.margin.item.horizontal = options.margin.item;\n              this.options.margin.item.vertical = options.margin.item;\n            }\n            else if (typeof options.margin.item === 'object') {\n              util.selectiveExtend(['horizontal', 'vertical'], this.options.margin.item, options.margin.item);\n            }\n          }\n        }\n      }\n\n      ['locale', 'locales'].forEach(key => {\n        if (key in options) {\n          this.options[key] = options[key];\n        }\n      });\n\n      if ('editable' in options) {\n        if (typeof options.editable === 'boolean') {\n          this.options.editable.updateTime    = options.editable;\n          this.options.editable.updateGroup   = options.editable;\n          this.options.editable.add           = options.editable;\n          this.options.editable.remove        = options.editable;\n          this.options.editable.overrideItems = false;\n        }\n        else if (typeof options.editable === 'object') {\n          util.selectiveExtend(['updateTime', 'updateGroup', 'add', 'remove', 'overrideItems'], this.options.editable, options.editable);\n        }\n      }\n\n      if ('groupEditable' in options) {\n        if (typeof options.groupEditable === 'boolean') {\n          this.options.groupEditable.order  = options.groupEditable;\n          this.options.groupEditable.add    = options.groupEditable;\n          this.options.groupEditable.remove = options.groupEditable;\n        }\n        else if (typeof options.groupEditable === 'object') {\n          util.selectiveExtend(['order', 'add', 'remove'], this.options.groupEditable, options.groupEditable);\n        }\n      }\n\n      // callback functions\n      const addCallback = name => {\n        const fn = options[name];\n        if (fn) {\n          if (!(typeof fn === 'function')) {\n            throw new Error(`option ${name} must be a function ${name}(item, callback)`);\n          }\n          this.options[name] = fn;\n        }\n      };\n      ['onDropObjectOnItem', 'onAdd', 'onUpdate', 'onRemove', 'onMove', 'onMoving', 'onAddGroup', 'onMoveGroup', 'onRemoveGroup'].forEach(addCallback);\n\n      if (options.cluster) {\n        Object.assign(this.options, {\n          cluster: options.cluster\n        });\n        if (!this.clusterGenerator) {\n          this.clusterGenerator = new ClusterGenerator(this);\n        } \n        this.clusterGenerator.setItems(this.items, { applyOnChangedLevel: false });\n        this.markDirty({ refreshItems: true, restackGroups: true });\n\n        this.redraw();\n      } else if (this.clusterGenerator) {\n        this._detachAllClusters();\n        this.clusters = [];\n        this.clusterGenerator = null;\n        this.options.cluster = undefined;\n        this.markDirty({ refreshItems: true, restackGroups: true });\n\n        this.redraw();\n      } else {\n        // force the itemSet to refresh: options like orientation and margins may be changed\n        this.markDirty();\n      }\n    }\n  }\n\n  /**\n   * Mark the ItemSet dirty so it will refresh everything with next redraw.\n   * Optionally, all items can be marked as dirty and be refreshed.\n   * @param {{refreshItems: boolean}} [options]\n   */\n  markDirty(options) {\n    this.groupIds = [];\n\n    if (options) {\n      if (options.refreshItems) {\n        util.forEach(this.items, item => {\n          item.dirty = true;\n          if (item.displayed) item.redraw();\n        });\n      }\n      \n      if (options.restackGroups) {\n        util.forEach(this.groups, (group, key) => {\n          if (key === BACKGROUND) return;\n          group.stackDirty = true;\n        });\n      }\n    }\n  }\n\n  /**\n   * Destroy the ItemSet\n   */\n  destroy() {\n    this.clearPopupTimer();\n    this.hide();\n    this.setItems(null);\n    this.setGroups(null);\n\n    this.hammer && this.hammer.destroy();\n    this.groupHammer && this.groupHammer.destroy();\n    this.hammer = null;\n\n    this.body = null;\n    this.conversion = null;\n  }\n\n  /**\n   * Hide the component from the DOM\n   */\n  hide() {\n    // remove the frame containing the items\n    if (this.dom.frame.parentNode) {\n      this.dom.frame.parentNode.removeChild(this.dom.frame);\n    }\n\n    // remove the axis with dots\n    if (this.dom.axis.parentNode) {\n      this.dom.axis.parentNode.removeChild(this.dom.axis);\n    }\n\n    // remove the labelset containing all group labels\n    if (this.dom.labelSet.parentNode) {\n      this.dom.labelSet.parentNode.removeChild(this.dom.labelSet);\n    }\n  }\n\n  /**\n   * Show the component in the DOM (when not already visible).\n   */\n  show() {\n    // show frame containing the items\n    if (!this.dom.frame.parentNode) {\n      this.body.dom.center.appendChild(this.dom.frame);\n    }\n\n    // show axis with dots\n    if (!this.dom.axis.parentNode) {\n      this.body.dom.backgroundVertical.appendChild(this.dom.axis);\n    }\n\n    // show labelset containing labels\n    if (!this.dom.labelSet.parentNode) {\n      if (this.options.rtl) {\n        this.body.dom.right.appendChild(this.dom.labelSet);\n      } else {\n        this.body.dom.left.appendChild(this.dom.labelSet);\n      }\n    }\n  }\n\n  /**\n   * Activates the popup timer to show the given popup after a fixed time.\n   * @param {Popup} popup\n   */\n  setPopupTimer(popup) {\n    this.clearPopupTimer();\n    if (popup) {\n      const delay = this.options.tooltip.delay || typeof this.options.tooltip.delay === 'number' ?\n            this.options.tooltip.delay :\n            500;\n      this.popupTimer = setTimeout(\n        function () {\n          popup.show()\n        }, delay);\n    }\n  }\n\n  /**\n   * Clears the popup timer for the tooltip.\n   */\n  clearPopupTimer() {\n    if (this.popupTimer != null) {\n        clearTimeout(this.popupTimer);\n        this.popupTimer = null;\n    }\n  }\n  \n  /**\n   * Set selected items by their id. Replaces the current selection\n   * Unknown id's are silently ignored.\n   * @param {string[] | string} [ids] An array with zero or more id's of the items to be\n   *                                  selected, or a single item id. If ids is undefined\n   *                                  or an empty array, all items will be unselected.\n   */\n  setSelection(ids) {\n    if (ids == undefined) { \n      ids = [];\n    }\n    \n    if (!Array.isArray(ids)) {\n      ids = [ids];\n    }\n  \n\n    const idsToDeselect = this.selection.filter(id => ids.indexOf(id) === -1);\n\n    // unselect currently selected items\n    for (let selectedId of idsToDeselect) {\n      const item = this.getItemById(selectedId);\n      if (item) {\n        item.unselect();\n      }\n    }\n  \n    // select items\n    this.selection = [ ...ids ];\n    for (let id of ids) {\n      const item = this.getItemById(id);\n      if (item) {\n        item.select();\n      }\n    }\n  }\n\n  /**\n   * Get the selected items by their id\n   * @return {Array} ids  The ids of the selected items\n   */\n  getSelection() {\n    return this.selection.concat([]);\n  }\n\n  /**\n   * Get the id's of the currently visible items.\n   * @returns {Array} The ids of the visible items\n   */\n  getVisibleItems() {\n    const range = this.body.range.getRange();\n    let right;\n    let left;\n\n    if (this.options.rtl) { \n      right  = this.body.util.toScreen(range.start);\n      left = this.body.util.toScreen(range.end);\n    } else {\n      left  = this.body.util.toScreen(range.start);\n      right = this.body.util.toScreen(range.end);\n    }\n\n    const ids = [];\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        const group = this.groups[groupId];\n        const rawVisibleItems = group.isVisible ? group.visibleItems : [];\n\n        // filter the \"raw\" set with visibleItems into a set which is really\n        // visible by pixels\n        for (const item of rawVisibleItems) {\n          // TODO: also check whether visible vertically\n          if (this.options.rtl) { \n            if ((item.right < left) && (item.right + item.width > right)) {\n              ids.push(item.id);\n            }\n          } else {\n            if ((item.left < right) && (item.left + item.width > left)) {\n              ids.push(item.id);\n            }\n          }\n        }\n      }\n    }\n\n    return ids;\n  }\n\n  /**\n   * Get the id's of the currently visible groups.\n   * @returns {Array} The ids of the visible groups\n   */\n  getVisibleGroups() {\n    const ids = [];\n\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        const group = this.groups[groupId];\n        if (group.isVisible) {\n          ids.push(groupId)\n        }\n      }\n    }\n\n    return ids;\n  }\n  \n  /**\n   * get item by id\n   * @param {string} id\n   * @return {object} item\n   */\n  getItemById(id) {\n    return this.items[id] || this.clusters.find(cluster => cluster.id === id);\n  } \n\n  /**\n   * Deselect a selected item\n   * @param {string | number} id\n   * @private\n   */\n  _deselect(id) {\n    const selection = this.selection;\n    for (let i = 0, ii = selection.length; i < ii; i++) {\n      if (selection[i] == id) { // non-strict comparison!\n        selection.splice(i, 1);\n        break;\n      }\n    }\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    const margin = this.options.margin;\n    const range = this.body.range;\n    const asSize = util.option.asSize;\n    const options = this.options;\n    const orientation = options.orientation.item;\n    let resized = false;\n    const frame = this.dom.frame;\n\n    // recalculate absolute position (before redrawing groups)\n    this.props.top = this.body.domProps.top.height + this.body.domProps.border.top;\n\n    if (this.options.rtl) {\n      this.props.right = this.body.domProps.right.width + this.body.domProps.border.right;\n    } else {\n      this.props.left = this.body.domProps.left.width + this.body.domProps.border.left;\n    }\n\n    // update class name\n    frame.className = 'vis-itemset';\n\n    if (this.options.cluster) {\n      this._clusterItems();\n    }\n\n    // reorder the groups (if needed)\n    resized = this._orderGroups() || resized;\n\n    // check whether zoomed (in that case we need to re-stack everything)\n    // TODO: would be nicer to get this as a trigger from Range\n    const visibleInterval = range.end - range.start;\n    const zoomed = (visibleInterval != this.lastVisibleInterval) || (this.props.width != this.props.lastWidth);\n    const scrolled = range.start != this.lastRangeStart;\n    const changedStackOption = options.stack != this.lastStack;\n    const changedStackSubgroupsOption = options.stackSubgroups != this.lastStackSubgroups;\n    const forceRestack = (zoomed || scrolled || changedStackOption || changedStackSubgroupsOption);\n    this.lastVisibleInterval = visibleInterval;\n    this.lastRangeStart = range.start;\n    this.lastStack = options.stack;\n    this.lastStackSubgroups = options.stackSubgroups;\n\n    this.props.lastWidth = this.props.width;\n    const firstGroup = this._firstGroup();\n    const firstMargin = {\n      item: margin.item,\n      axis: margin.axis\n    };\n    const nonFirstMargin = {\n      item: margin.item,\n      axis: margin.item.vertical / 2\n    };\n    let height = 0;\n    const minHeight = margin.axis + margin.item.vertical;\n\n    // redraw the background group\n    this.groups[BACKGROUND].redraw(range, nonFirstMargin, forceRestack);\n\n    const redrawQueue = {};\n    let redrawQueueLength = 0;\n\n    // collect redraw functions\n    util.forEach(this.groups, (group, key) => {\n      if (key === BACKGROUND) return;\n      const groupMargin = group == firstGroup ? firstMargin : nonFirstMargin;\n      const returnQueue = true;\n      redrawQueue[key] = group.redraw(range, groupMargin, forceRestack, returnQueue);\n      redrawQueueLength = redrawQueue[key].length;\n    });\n\n    const needRedraw = redrawQueueLength > 0;\n    if (needRedraw) {\n      const redrawResults = {};\n\n      for (let i = 0; i < redrawQueueLength; i++) {\n        util.forEach(redrawQueue, (fns, key) => {\n          redrawResults[key] = fns[i]();\n        });\n      }\n\n      // redraw all regular groups\n      util.forEach(this.groups, (group, key) => {\n        if (key === BACKGROUND) return;\n        const groupResized = redrawResults[key];\n        resized = groupResized || resized;\n        height += group.height;\n      });\n      height = Math.max(height, minHeight);\n    }\n\n    height = Math.max(height, minHeight);\n\n    // update frame height\n    frame.style.height  = asSize(height);\n\n    // calculate actual size\n    this.props.width = frame.offsetWidth;\n    this.props.height = height;\n\n    // reposition axis\n    this.dom.axis.style.top = asSize((orientation == 'top') ?\n        (this.body.domProps.top.height + this.body.domProps.border.top) :\n        (this.body.domProps.top.height + this.body.domProps.centerContainer.height));\n    if (this.options.rtl) {\n      this.dom.axis.style.right = '0';\n    } else {\n      this.dom.axis.style.left = '0';\n    }\n\n    this.hammer.get('press').set({time: this.options.longSelectPressTime});\n\n    this.initialItemSetDrawn = true;\n    // check if this component is resized\n    resized = this._isResized() || resized;\n\n    return resized;\n  }\n\n  /**\n   * Get the first group, aligned with the axis\n   * @return {Group | null} firstGroup\n   * @private\n   */\n  _firstGroup() {\n    const firstGroupIndex = (this.options.orientation.item == 'top') ? 0 : (this.groupIds.length - 1);\n    const firstGroupId = this.groupIds[firstGroupIndex];\n    const firstGroup = this.groups[firstGroupId] || this.groups[UNGROUPED];\n\n    return firstGroup || null;\n  }\n\n  /**\n   * Create or delete the group holding all ungrouped items. This group is used when\n   * there are no groups specified.\n   * @protected\n   */\n  _updateUngrouped() {\n    let ungrouped = this.groups[UNGROUPED];\n    let item;\n    let itemId;\n\n    if (this.groupsData) {\n      // remove the group holding all ungrouped items\n      if (ungrouped) {\n        ungrouped.dispose();\n        delete this.groups[UNGROUPED];\n\n        for (itemId in this.items) {\n          if (this.items.hasOwnProperty(itemId)) {\n            item = this.items[itemId];\n            item.parent && item.parent.remove(item);\n            const groupId = this.getGroupId(item.data);\n            const group = this.groups[groupId];\n            group && group.add(item) || item.hide();\n          }\n        }\n      }\n    }\n    else {\n      // create a group holding all (unfiltered) items\n      if (!ungrouped) {\n        const id = null;\n        const data = null;\n        ungrouped = new Group(id, data, this);\n        this.groups[UNGROUPED] = ungrouped;\n\n        for (itemId in this.items) {\n          if (this.items.hasOwnProperty(itemId)) {\n            item = this.items[itemId];\n            ungrouped.add(item);\n          }\n        }\n\n        ungrouped.show();\n      }\n    }\n  }\n\n  /**\n   * Get the element for the labelset\n   * @return {HTMLElement} labelSet\n   */\n  getLabelSet() {\n    return this.dom.labelSet;\n  }\n\n  /**\n   * Set items\n   * @param {vis.DataSet | null} items\n   */\n  setItems(items) {\n    this.itemsSettingTime = new Date();\n    const me = this;\n    let ids;\n    const oldItemsData = this.itemsData;\n\n    // replace the dataset\n    if (!items) {\n      this.itemsData = null;\n    }\n    else if (items instanceof DataSet || items instanceof DataView) {\n      this.itemsData = typeCoerceDataSet(items);\n    }\n    else {\n      throw new TypeError('Data must be an instance of DataSet or DataView');\n    }\n\n    if (oldItemsData) {\n      // unsubscribe from old dataset\n      util.forEach(this.itemListeners, (callback, event) => {\n        oldItemsData.off(event, callback);\n      });\n\n      // stop maintaining a coerced version of the old data set\n      oldItemsData.dispose()\n\n      // remove all drawn items\n      ids = oldItemsData.getIds();\n      this._onRemove(ids);\n    }\n\n    if (this.itemsData) {\n      // subscribe to new dataset\n      const id = this.id;\n      util.forEach(this.itemListeners, (callback, event) => {\n        me.itemsData.on(event, callback, id);\n      });\n\n      // add all new items\n      ids = this.itemsData.getIds();\n      this._onAdd(ids);\n\n      // update the group holding all ungrouped items\n      this._updateUngrouped();\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Get the current items\n   * @returns {vis.DataSet | null}\n   */\n  getItems() {\n    return this.itemsData != null ? this.itemsData.rawDS : null;\n  }\n\n  /**\n   * Set groups\n   * @param {vis.DataSet} groups\n   */\n  setGroups(groups) {\n    const me = this;\n    let ids;\n\n    // unsubscribe from current dataset\n    if (this.groupsData) {\n      util.forEach(this.groupListeners, (callback, event) => {\n        me.groupsData.off(event, callback);\n      });\n\n      // remove all drawn groups\n      ids = this.groupsData.getIds();\n      this.groupsData = null;\n      this._onRemoveGroups(ids); // note: this will cause a redraw\n    }\n\n    // replace the dataset\n    if (!groups) {\n      this.groupsData = null;\n    }\n    else if (groups instanceof DataSet || groups instanceof DataView) {\n      this.groupsData = groups;\n    }\n    else {\n      throw new TypeError('Data must be an instance of DataSet or DataView');\n    }\n\n    if (this.groupsData) {\n      // go over all groups nesting\n      let groupsData = this.groupsData;\n      if (this.groupsData instanceof DataView) {\n        groupsData = this.groupsData.getDataSet()\n      }\n\n      groupsData.get().forEach(group => {\n        if (group.nestedGroups) {\n          group.nestedGroups.forEach(nestedGroupId => {\n            const updatedNestedGroup = groupsData.get(nestedGroupId);\n            updatedNestedGroup.nestedInGroup = group.id;\n            if (group.showNested == false) {\n              updatedNestedGroup.visible = false;\n            }\n            groupsData.update(updatedNestedGroup);\n          })\n        }\n      });\n\n      // subscribe to new dataset\n      const id = this.id;\n      util.forEach(this.groupListeners, (callback, event) => {\n        me.groupsData.on(event, callback, id);\n      });\n\n      // draw all ms\n      ids = this.groupsData.getIds();\n      this._onAddGroups(ids);\n    }\n\n    // update the group holding all ungrouped items\n    this._updateUngrouped();\n\n    // update the order of all items in each group\n    this._order();\n\n    if (this.options.cluster) {\n      this.clusterGenerator.updateData();\n      this._clusterItems();\n      this.markDirty({ refreshItems: true, restackGroups: true });\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Get the current groups\n   * @returns {vis.DataSet | null} groups\n   */\n  getGroups() {\n    return this.groupsData;\n  }\n\n  /**\n   * Remove an item by its id\n   * @param {string | number} id\n   */\n  removeItem(id) {\n    const item = this.itemsData.get(id);\n\n    if (item) {\n      // confirm deletion\n      this.options.onRemove(item, item => {\n        if (item) {\n          // remove by id here, it is possible that an item has no id defined\n          // itself, so better not delete by the item itself\n          this.itemsData.remove(id);\n        }\n      });\n    }\n  }\n\n  /**\n   * Get the time of an item based on it's data and options.type\n   * @param {Object} itemData\n   * @returns {string} Returns the type\n   * @private\n   */\n  _getType(itemData) {\n    return itemData.type || this.options.type || (itemData.end ? 'range' : 'box');\n  }\n\n  /**\n   * Get the group id for an item\n   * @param {Object} itemData\n   * @returns {string} Returns the groupId\n   * @private\n   */\n  getGroupId(itemData) {\n    const type = this._getType(itemData);\n    if (type == 'background' && itemData.group == undefined) {\n     return BACKGROUND;\n    }\n    else {\n      return this.groupsData ? itemData.group : UNGROUPED;\n    }\n  }\n\n  /**\n   * Handle updated items\n   * @param {number[]} ids\n   * @protected\n   */\n  _onUpdate(ids) {\n    const me = this;\n\n    ids.forEach(id => {\n      const itemData = me.itemsData.get(id);\n      let item = me.items[id];\n      const type = itemData ? me._getType(itemData) : null;\n\n      const constructor = ItemSet.types[type];\n      let selected;\n\n      if (item) {\n        // update item   \t\n        if (!constructor || !(item instanceof constructor)) {\n          // item type has changed, delete the item and recreate it\n          selected = item.selected; // preserve selection of this item\n          me._removeItem(item);\n          item = null;\n        }\n        else {\n          me._updateItem(item, itemData);\n        }\n      }\n\n      if (!item && itemData) {\n        // create item\n        if (constructor) {\n          item = new constructor(itemData, me.conversion, me.options);\n          item.id = id; // TODO: not so nice setting id afterwards\n\n          me._addItem(item);\n          if (selected) {\n            this.selection.push(id);\n            item.select();\n          }\n        }\n        else {\n          throw new TypeError(`Unknown item type \"${type}\"`);\n        }\n      }\n    });\n\n    this._order();\n    \n    if (this.options.cluster) {\n      this.clusterGenerator.setItems(this.items, { applyOnChangedLevel: false });\n      this._clusterItems();\n    }\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Handle removed items\n   * @param {number[]} ids\n   * @protected\n   */\n  _onRemove(ids) {\n    let count = 0;\n    const me = this;\n    ids.forEach(id => {\n      const item = me.items[id];\n      if (item) {\n        count++;\n        me._removeItem(item);\n      }\n    });\n\n    if (count) {\n      // update order\n      this._order();\n      this.body.emitter.emit('_change', {queue: true});\n    }\n  }\n\n  /**\n   * Update the order of item in all groups\n   * @private\n   */\n  _order() {\n    // reorder the items in all groups\n    // TODO: optimization: only reorder groups affected by the changed items\n    util.forEach(this.groups, group => {\n      group.order();\n    });\n  }\n\n  /**\n   * Handle updated groups\n   * @param {number[]} ids\n   * @private\n   */\n  _onUpdateGroups(ids) {\n    this._onAddGroups(ids);\n  }\n\n  /**\n   * Handle changed groups (added or updated)\n   * @param {number[]} ids\n   * @private\n   */\n  _onAddGroups(ids) {\n    const me = this;\n\n    ids.forEach(id => {\n      const groupData = me.groupsData.get(id);\n      let group = me.groups[id];\n\n      if (!group) {\n        // check for reserved ids\n        if (id == UNGROUPED || id == BACKGROUND) {\n          throw new Error(`Illegal group id. ${id} is a reserved id.`);\n        }\n\n        const groupOptions = Object.create(me.options);\n        util.extend(groupOptions, {\n          height: null\n        });\n\n        group = new Group(id, groupData, me);\n        me.groups[id] = group;\n\n        // add items with this groupId to the new group\n        for (const itemId in me.items) {\n          if (me.items.hasOwnProperty(itemId)) {\n            const item = me.items[itemId];\n            if (item.data.group == id) {\n              group.add(item);\n            }\n          }\n        }\n\n        group.order();\n        group.show();\n      }\n      else {\n        // update group\n        group.setData(groupData);\n      }\n    });\n\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Handle removed groups\n   * @param {number[]} ids\n   * @private\n   */\n  _onRemoveGroups(ids) {\n    ids.forEach(id => {\n      const group = this.groups[id];\n\n      if (group) {\n        group.dispose();\n        delete this.groups[id];\n      }\n    });\n\n    if (this.options.cluster) {\n      this.clusterGenerator.updateData();\n      this._clusterItems();\n    } \n\n    this.markDirty({ restackGroups: !!this.options.cluster });\n    this.body.emitter.emit('_change', {queue: true});\n  }\n\n  /**\n   * Reorder the groups if needed\n   * @return {boolean} changed\n   * @private\n   */\n  _orderGroups() {\n    if (this.groupsData) {\n      // reorder the groups\n      let groupIds = this.groupsData.getIds({\n        order: this.options.groupOrder\n      });\n\n      groupIds = this._orderNestedGroups(groupIds);\n\n      const changed = !util.equalArray(groupIds, this.groupIds);\n      if (changed) {\n        // hide all groups, removes them from the DOM\n        const groups = this.groups;\n        groupIds.forEach(groupId => {\n          groups[groupId].hide();\n        });\n\n        // show the groups again, attach them to the DOM in correct order\n        groupIds.forEach(groupId => {\n          groups[groupId].show();\n        });\n\n        this.groupIds = groupIds;\n      }\n\n      return changed;\n    }\n    else {\n      return false;\n    }\n  }\n\n  /**\n   * Reorder the nested groups\n   *\n   * @param {Array.<number>} groupIds\n   * @returns {Array.<number>}\n   * @private\n   */\n  _orderNestedGroups(groupIds) {\n    /**\n     * Recursively order nested groups\n     *\n     * @param {ItemSet} t\n     * @param {Array.<number>} groupIds\n     * @returns {Array.<number>}\n     * @private\n     */\n    function getOrderedNestedGroups(t, groupIds) {\n      let result = [];\n      groupIds.forEach(groupId => {\n        result.push(groupId);\n        const groupData = t.groupsData.get(groupId);\n        if (groupData.nestedGroups) {\n          const nestedGroupIds = t.groupsData.get({\n            filter(nestedGroup) {\n              return nestedGroup.nestedInGroup == groupId;\n            },\n            order: t.options.groupOrder\n          }).map(nestedGroup => nestedGroup.id);\n          result = result.concat(getOrderedNestedGroups(t, nestedGroupIds));\n        }\n      });\n\n      return result;\n    }\n\n    const topGroupIds = groupIds.filter(groupId => !this.groupsData.get(groupId).nestedInGroup);\n\n    return getOrderedNestedGroups(this, topGroupIds);\n  }\n\n  /**\n   * Add a new item\n   * @param {Item} item\n   * @private\n   */\n  _addItem(item) {\n    this.items[item.id] = item;\n\n    // add to group\n    const groupId = this.getGroupId(item.data);\n    const group = this.groups[groupId];\n\n    if (!group) {\n      item.groupShowing = false;\n    } else if (group && group.data && group.data.showNested) {\n      item.groupShowing = true;\n    }\n\n    if (group) group.add(item);\n  }\n\n  /**\n   * Update an existing item\n   * @param {Item} item\n   * @param {Object} itemData\n   * @private\n   */\n  _updateItem(item, itemData) {\n    // update the items data (will redraw the item when displayed)\n    item.setData(itemData);\n\n    const groupId = this.getGroupId(item.data);\n    const group = this.groups[groupId];\n    if (!group) {\n      item.groupShowing = false;\n    } else if (group && group.data && group.data.showNested) {\n      item.groupShowing = true;\n    }\n  }\n\n  /**\n   * Delete an item from the ItemSet: remove it from the DOM, from the map\n   * with items, and from the map with visible items, and from the selection\n   * @param {Item} item\n   * @private\n   */\n  _removeItem(item) {\n    // remove from DOM\n    item.hide();\n\n    // remove from items\n    delete this.items[item.id];\n\n    // remove from selection\n    const index = this.selection.indexOf(item.id);\n    if (index != -1) this.selection.splice(index, 1);\n\n    // remove from group\n    item.parent && item.parent.remove(item);\n\n    // remove Tooltip from DOM\n    if (this.popup != null) {\n      this.popup.hide();\n    }\n  }\n\n  /**\n   * Create an array containing all items being a range (having an end date)\n   * @param {Array.<Object>} array\n   * @returns {Array}\n   * @private\n   */\n  _constructByEndArray(array) {\n    const endArray = [];\n\n    for (let i = 0; i < array.length; i++) {\n      if (array[i] instanceof RangeItem) {\n        endArray.push(array[i]);\n      }\n    }\n    return endArray;\n  }\n\n  /**\n   * Register the clicked item on touch, before dragStart is initiated.\n   *\n   * dragStart is initiated from a mousemove event, AFTER the mouse/touch is\n   * already moving. Therefore, the mouse/touch can sometimes be above an other\n   * DOM element than the item itself.\n   *\n   * @param {Event} event\n   * @private\n   */\n  _onTouch(event) {\n    // store the touched item, used in _onDragStart\n    this.touchParams.item = this.itemFromTarget(event);\n    this.touchParams.dragLeftItem = event.target.dragLeftItem || false;\n    this.touchParams.dragRightItem = event.target.dragRightItem || false;\n    this.touchParams.itemProps = null;\n  }\n\n  /**\n   * Given an group id, returns the index it has.\n   *\n   * @param {number} groupId\n   * @returns {number} index / groupId\n   * @private\n   */\n  _getGroupIndex(groupId) {\n      for (let i = 0; i < this.groupIds.length; i++) {\n          if (groupId == this.groupIds[i])\n              return i;\n      }\n  }\n\n  /**\n   * Start dragging the selected events\n   * @param {Event} event\n   * @private\n   */\n  _onDragStart(event) {\n    if (this.touchParams.itemIsDragging) { return; }\n    const item = this.touchParams.item || null;\n    const me = this;\n    let props;\n\n    if (item && (item.selected || this.options.itemsAlwaysDraggable.item)) {\n\n      if (this.options.editable.overrideItems &&\n          !this.options.editable.updateTime &&\n          !this.options.editable.updateGroup) {\n        return;\n      }\n\n      // override options.editable\n      if ((item.editable != null && !item.editable.updateTime && !item.editable.updateGroup)\n          && !this.options.editable.overrideItems) {\n        return;\n      }\n\n      const dragLeftItem = this.touchParams.dragLeftItem;\n      const dragRightItem = this.touchParams.dragRightItem;\n      this.touchParams.itemIsDragging = true;\n      this.touchParams.selectedItem = item;\n\n      if (dragLeftItem) {\n        props = {\n          item: dragLeftItem,\n          initialX: event.center.x,\n          dragLeft:  true,\n          data: this._cloneItemData(item.data)\n        };\n\n        this.touchParams.itemProps = [props];\n      } else if (dragRightItem) {\n        props = {\n          item: dragRightItem,\n          initialX: event.center.x,\n          dragRight: true,\n          data: this._cloneItemData(item.data)\n        };\n\n        this.touchParams.itemProps = [props];\n      } else if (this.options.editable.add && (event.srcEvent.ctrlKey || event.srcEvent.metaKey)) {\n        // create a new range item when dragging with ctrl key down\n        this._onDragStartAddItem(event);\n      } else {\n        if(this.groupIds.length < 1) {\n          // Mitigates a race condition if _onDragStart() is\n          // called after markDirty() without redraw() being called between.\n          this.redraw();\n        }\n        \n        const baseGroupIndex = this._getGroupIndex(item.data.group);\n\n        const itemsToDrag = (this.options.itemsAlwaysDraggable.item && !item.selected) ? [item.id] : this.getSelection();\n\n        this.touchParams.itemProps = itemsToDrag.map(id => {\n          const item = me.items[id];\n          const groupIndex = me._getGroupIndex(item.data.group);\n          return {\n            item,\n            initialX: event.center.x,\n            groupOffset: baseGroupIndex-groupIndex,\n            data: this._cloneItemData(item.data)\n          };\n        });\n      }\n\n      event.stopPropagation();\n    } else if (this.options.editable.add && (event.srcEvent.ctrlKey || event.srcEvent.metaKey)) {\n      // create a new range item when dragging with ctrl key down\n      this._onDragStartAddItem(event);\n    }\n  }\n\n  /**\n   * Start creating a new range item by dragging.\n   * @param {Event} event\n   * @private\n   */\n  _onDragStartAddItem(event) {\n    const snap = this.options.snap || null;\n    const frameRect = this.dom.frame.getBoundingClientRect()\n\n    // plus (if rtl) 10 to compensate for the drag starting as soon as you've moved 10px\n    const x = this.options.rtl ? frameRect.right - event.center.x  + 10 : event.center.x - frameRect.left - 10;\n\n    const time = this.body.util.toTime(x);\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const start = snap ? snap(time, scale, step) : time;\n    const end = start;\n\n    const itemData = {\n      type: 'range',\n      start,\n      end,\n      content: 'new item'\n    };\n\n    const id = randomUUID();\n    itemData[this.itemsData.idProp] = id;\n\n    const group = this.groupFromTarget(event);\n    if (group) {\n      itemData.group = group.groupId;\n    }\n    const newItem = new RangeItem(itemData, this.conversion, this.options);\n    newItem.id = id; // TODO: not so nice setting id afterwards\n    newItem.data = this._cloneItemData(itemData);\n    this._addItem(newItem);\n    this.touchParams.selectedItem = newItem;\n    \n    const props = {\n      item: newItem,\n      initialX: event.center.x,\n      data: newItem.data\n    };\n\n    if (this.options.rtl) {\n      props.dragLeft = true;\n    } else {\n      props.dragRight = true;\n    }\n    this.touchParams.itemProps = [props];\n\n    event.stopPropagation();\n  }\n\n  /**\n   * Drag selected items\n   * @param {Event} event\n   * @private\n   */\n  _onDrag(event) {\n    if (this.popup != null && this.options.showTooltips && !this.popup.hidden) {\n      // this.popup.hide();\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.center.x - containerRect.left + container.offsetLeft,\n        event.center.y - containerRect.top + container.offsetTop\n      );\n      this.popup.show(); // redraw\n    }\n    \n    if (this.touchParams.itemProps) {\n      event.stopPropagation();\n\n      const me = this;\n      const snap = this.options.snap || null;\n      const domRootOffsetLeft = this.body.dom.root.offsetLeft;\n      const xOffset = this.options.rtl ? domRootOffsetLeft + this.body.domProps.right.width : domRootOffsetLeft + this.body.domProps.left.width;\n      const scale = this.body.util.getScale();\n      const step = this.body.util.getStep();\n\n      //only calculate the new group for the item that's actually dragged\n      const selectedItem = this.touchParams.selectedItem;\n      const updateGroupAllowed = ((this.options.editable.overrideItems || selectedItem.editable == null) && this.options.editable.updateGroup) ||\n                               (!this.options.editable.overrideItems && selectedItem.editable != null && selectedItem.editable.updateGroup);\n      let newGroupBase = null;\n      if (updateGroupAllowed && selectedItem) {\n        if (selectedItem.data.group != undefined) {\n          // drag from one group to another\n          const group = me.groupFromTarget(event);\n          if (group) {\n            //we know the offset for all items, so the new group for all items\n            //will be relative to this one.\n            newGroupBase = this._getGroupIndex(group.groupId);\n          }\n        }\n      }\n\n      // move\n      this.touchParams.itemProps.forEach(props => {\n        const current = me.body.util.toTime(event.center.x - xOffset);\n        const initial = me.body.util.toTime(props.initialX - xOffset);\n        let offset;\n        let initialStart;\n        let initialEnd;\n        let start;\n        let end;\n\n        if (this.options.rtl) {\n          offset = -(current - initial); // ms\n        } else {\n          offset = (current - initial); // ms\n        }\n\n        let itemData = this._cloneItemData(props.item.data); // clone the data\n        if (props.item.editable != null\n          && !props.item.editable.updateTime\n          && !props.item.editable.updateGroup\n          && !me.options.editable.overrideItems) {\n          return;\n        }\n\n        const updateTimeAllowed = ((this.options.editable.overrideItems || selectedItem.editable == null) && this.options.editable.updateTime) ||\n                                 (!this.options.editable.overrideItems && selectedItem.editable != null && selectedItem.editable.updateTime);\n        if (updateTimeAllowed) {\n          if (props.dragLeft) {\n            // drag left side of a range item\n            if (this.options.rtl) {\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                end = new Date(initialEnd.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.end = snap ? snap(end, scale, step) : end;\n              }\n            } else {\n              if (itemData.start != undefined) {\n                initialStart = util.convert(props.data.start, 'Date');\n                start = new Date(initialStart.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            }\n          }\n          else if (props.dragRight) {\n            // drag right side of a range item\n            if (this.options.rtl) {\n              if (itemData.start != undefined) {\n                initialStart = util.convert(props.data.start, 'Date');\n                start = new Date(initialStart.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            } else {\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                end = new Date(initialEnd.valueOf() + offset);\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.end = snap ? snap(end, scale, step) : end;\n              }\n            }\n          }\n          else {\n            // drag both start and end\n            if (itemData.start != undefined) {\n\n              initialStart = util.convert(props.data.start, 'Date').valueOf();\n              start = new Date(initialStart + offset);\n\n              if (itemData.end != undefined) {\n                initialEnd = util.convert(props.data.end, 'Date');\n                const duration  = initialEnd.valueOf() - initialStart.valueOf();\n\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n                itemData.end   = new Date(itemData.start.valueOf() + duration);\n              }\n              else {\n                // TODO: pass a Moment instead of a Date to snap(). (Breaking change)\n                itemData.start = snap ? snap(start, scale, step) : start;\n              }\n            }\n          }\n        }\n\n        if (updateGroupAllowed && (!props.dragLeft && !props.dragRight) && newGroupBase!=null) {\n          if (itemData.group != undefined) {\n            let newOffset = newGroupBase - props.groupOffset;\n\n            //make sure we stay in bounds\n            newOffset = Math.max(0, newOffset);\n            newOffset = Math.min(me.groupIds.length-1, newOffset);\n            itemData.group = me.groupIds[newOffset];\n          }\n        }\n\n        // confirm moving the item\n        itemData = this._cloneItemData(itemData);  // convert start and end to the correct type\n        me.options.onMoving(itemData, itemData => {\n          if (itemData) {\n            props.item.setData(this._cloneItemData(itemData, 'Date'));\n          }\n        });\n      });\n      \n      this.body.emitter.emit('_change');\n    }\n  }\n\n  /**\n   * Move an item to another group\n   * @param {Item} item\n   * @param {string | number} groupId\n   * @private\n   */\n  _moveToGroup(item, groupId) {\n    const group = this.groups[groupId];\n    if (group && group.groupId != item.data.group) {\n      const oldGroup = item.parent;\n      oldGroup.remove(item);\n      oldGroup.order();\n      \n      item.data.group = group.groupId;\n      \n      group.add(item);\n      group.order();\n    }\n  }\n\n  /**\n   * End of dragging selected items\n   * @param {Event} event\n   * @private\n   */\n  _onDragEnd(event) {\n    this.touchParams.itemIsDragging = false;\n    if (this.touchParams.itemProps) {\n      event.stopPropagation();\n\n      const me = this;\n      const itemProps = this.touchParams.itemProps;\n      this.touchParams.itemProps = null;\n\n      itemProps.forEach(props => {\n        const id = props.item.id;\n        const exists = me.itemsData.get(id) != null;\n\n        if (!exists) {\n          // add a new item\n          me.options.onAdd(props.item.data, itemData => {\n            me._removeItem(props.item); // remove temporary item\n            if (itemData) {\n              me.itemsData.add(itemData);\n            }\n\n            // force re-stacking of all items next redraw\n            me.body.emitter.emit('_change');\n          });\n        }\n        else {\n          // update existing item\n          const itemData = this._cloneItemData(props.item.data); // convert start and end to the correct type\n          me.options.onMove(itemData, itemData => {\n            if (itemData) {\n              // apply changes\n              itemData[this.itemsData.idProp] = id; // ensure the item contains its id (can be undefined)\n              this.itemsData.update(itemData);\n            }\n            else {\n              // restore original values\n              props.item.setData(props.data);\n\n              me.body.emitter.emit('_change');\n            }\n          });\n        }\n      });\n    }\n  }\n\n  /**\n   * On group click\n   * @param {Event} event\n   * @private\n   */\n  _onGroupClick(event) {\n    const group = this.groupFromTarget(event);\n    setTimeout(() => {\n      this.toggleGroupShowNested(group);\n    }, 1)\n  }\n  \n  /**\n   * Toggle show nested\n   * @param {object} group\n   * @param {boolean} force\n   */\n  toggleGroupShowNested(group, force = undefined) {\n\n    if (!group || !group.nestedGroups) return;\n\n    const groupsData = this.groupsData.getDataSet();\n\n    if (force != undefined) {\n      group.showNested = !!force;\n    } else {\n      group.showNested = !group.showNested;\n    }\n\n    let nestingGroup = groupsData.get(group.groupId);\n    nestingGroup.showNested = group.showNested;\n\n    let fullNestedGroups = group.nestedGroups;\n    let nextLevel = fullNestedGroups;\n    while (nextLevel.length > 0) {\n      let current = nextLevel;\n      nextLevel = [];\n      for (let i = 0; i < current.length; i++) {\n        let node = groupsData.get(current[i]);\n        if (node.nestedGroups) {\n          nextLevel = nextLevel.concat(node.nestedGroups);\n        }\n      }\n      if (nextLevel.length > 0) {\n        fullNestedGroups = fullNestedGroups.concat(nextLevel);\n      }\n    }\n    let nestedGroups = groupsData.get(fullNestedGroups).map(function (nestedGroup) {\n        if (nestedGroup.visible == undefined) {\n          nestedGroup.visible = true;\n        }\n        nestedGroup.visible = !!nestingGroup.showNested;\n        return nestedGroup;\n      });\n\n    groupsData.update(nestedGroups.concat(nestingGroup));\n\n    if (nestingGroup.showNested) {\n      util.removeClassName(group.dom.label, 'collapsed');\n      util.addClassName(group.dom.label, 'expanded');\n    } else {\n      util.removeClassName(group.dom.label, 'expanded');\n      util.addClassName(group.dom.label, 'collapsed');\n    }\n  }\n  \n  /**\n   * Toggle group drag classname\n   * @param {object} group\n   */\n  toggleGroupDragClassName(group) {\n    group.dom.label.classList.toggle('vis-group-is-dragging');\n    group.dom.foreground.classList.toggle('vis-group-is-dragging');\n  }\n  \n  /**\n   * on drag start\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onGroupDragStart(event) {\n    if (this.groupTouchParams.isDragging) return;\n\n    if (this.options.groupEditable.order) {\n      this.groupTouchParams.group = this.groupFromTarget(event);\n      \n      if (this.groupTouchParams.group) {\n        event.stopPropagation();      \n        \n        this.groupTouchParams.isDragging = true;\n        this.toggleGroupDragClassName(this.groupTouchParams.group);\n        \n        this.groupTouchParams.originalOrder = this.groupsData.getIds({\n          order: this.options.groupOrder\n        });\n      }\n    }\n  }\n\n  /**\n   * on drag\n   * @param {Event} event\n   * @return {void}\n   * @private\n   */\n  _onGroupDrag(event) {\n      if (this.options.groupEditable.order && this.groupTouchParams.group) {\n          event.stopPropagation();\n          \n      let groupsData = this.groupsData;\n      if (this.groupsData instanceof DataView) {\n        groupsData = this.groupsData.getDataSet()\n      }\n          // drag from one group to another\n          const group = this.groupFromTarget(event);\n          \n          // try to avoid toggling when groups differ in height\n          if (group && group.height != this.groupTouchParams.group.height) {\n              const movingUp = (group.top < this.groupTouchParams.group.top);\n              const clientY = event.center ? event.center.y : event.clientY;\n              const targetGroup = group.dom.foreground.getBoundingClientRect()\n              const draggedGroupHeight = this.groupTouchParams.group.height;\n              if (movingUp) {\n                  // skip swapping the groups when the dragged group is not below clientY afterwards\n                  if (targetGroup.top + draggedGroupHeight < clientY) {\n                      return;\n                  }\n              } else {\n                  const targetGroupHeight = group.height;\n                  // skip swapping the groups when the dragged group is not below clientY afterwards\n                  if (targetGroup.top + targetGroupHeight - draggedGroupHeight > clientY) {\n                      return;\n                  }\n              }\n          }\n          \n          if (group && group != this.groupTouchParams.group) {\n              const targetGroup = groupsData.get(group.groupId);\n              const draggedGroup = groupsData.get(this.groupTouchParams.group.groupId);\n              \n              // switch groups\n              if (draggedGroup && targetGroup) {\n                  this.options.groupOrderSwap(draggedGroup, targetGroup, groupsData);\n                  groupsData.update(draggedGroup);\n                  groupsData.update(targetGroup);\n              }\n              \n              // fetch current order of groups\n              const newOrder = groupsData.getIds({\n                order: this.options.groupOrder\n              });\n\n              \n              // in case of changes since _onGroupDragStart\n              if (!util.equalArray(newOrder, this.groupTouchParams.originalOrder)) {\n                  const origOrder = this.groupTouchParams.originalOrder;\n                  const draggedId = this.groupTouchParams.group.groupId;\n                  const numGroups = Math.min(origOrder.length, newOrder.length);\n                  let curPos = 0;\n                  let newOffset = 0;\n                  let orgOffset = 0;\n                  while (curPos < numGroups) {\n                      // as long as the groups are where they should be step down along the groups order\n                      while ((curPos+newOffset) < numGroups \n                          && (curPos+orgOffset) < numGroups \n                          && newOrder[curPos+newOffset] == origOrder[curPos+orgOffset]) {\n                          curPos++;\n                      }\n                      \n                      // all ok\n                      if (curPos+newOffset >= numGroups) {\n                          break;\n                      }\n                      \n                      // not all ok\n                      // if dragged group was move upwards everything below should have an offset\n                      if (newOrder[curPos+newOffset] == draggedId) {\n                          newOffset = 1;\n\n                      }\n                      // if dragged group was move downwards everything above should have an offset\n                      else if (origOrder[curPos+orgOffset] == draggedId) {\n                          orgOffset = 1;\n\n                      } \n                      // found a group (apart from dragged group) that has the wrong position -> switch with the \n                      // group at the position where other one should be, fix index arrays and continue\n                      else {\n                          const slippedPosition = newOrder.indexOf(origOrder[curPos+orgOffset]);\n                          const switchGroup = groupsData.get(newOrder[curPos+newOffset]);\n                          const shouldBeGroup = groupsData.get(origOrder[curPos+orgOffset]);\n                          this.options.groupOrderSwap(switchGroup, shouldBeGroup, groupsData);\n                          groupsData.update(switchGroup);\n                          groupsData.update(shouldBeGroup);\n                          \n                          const switchGroupId = newOrder[curPos+newOffset];\n                          newOrder[curPos+newOffset] = origOrder[curPos+orgOffset];\n                          newOrder[slippedPosition] = switchGroupId;\n                          \n                          curPos++;\n                      }\n                  }\n              }\n              \n          }\n      }\n  }\n\n  /**\n   * on drag end\n   * @param {Event} event\n   * @return {void}\n   * @private\n   */\n  _onGroupDragEnd(event) {\n    this.groupTouchParams.isDragging = false;\n\n    if (this.options.groupEditable.order && this.groupTouchParams.group) {\n      event.stopPropagation();\n          \n      // update existing group\n      const me = this;\n      const id = me.groupTouchParams.group.groupId;\n      const dataset = me.groupsData.getDataSet();\n      const groupData = util.extend({}, dataset.get(id)); // clone the data\n      me.options.onMoveGroup(groupData, groupData => {\n        if (groupData) {\n          // apply changes\n          groupData[dataset._idProp] = id; // ensure the group contains its id (can be undefined)\n          dataset.update(groupData);\n        }\n        else {\n\n          // fetch current order of groups\n          const newOrder = dataset.getIds({\n              order: me.options.groupOrder\n          });\n\n          // restore original order\n          if (!util.equalArray(newOrder, me.groupTouchParams.originalOrder)) {\n            const origOrder = me.groupTouchParams.originalOrder;\n            const numGroups = Math.min(origOrder.length, newOrder.length);\n            let curPos = 0;\n            while (curPos < numGroups) {\n              // as long as the groups are where they should be step down along the groups order\n              while (curPos < numGroups && newOrder[curPos] == origOrder[curPos]) {\n                curPos++;\n              }\n\n              // all ok\n              if (curPos >= numGroups) {\n                break;\n              }\n\n              // found a group that has the wrong position -> switch with the\n              // group at the position where other one should be, fix index arrays and continue\n              const slippedPosition = newOrder.indexOf(origOrder[curPos]);\n              const switchGroup = dataset.get(newOrder[curPos]);\n              const shouldBeGroup = dataset.get(origOrder[curPos]);\n              me.options.groupOrderSwap(switchGroup, shouldBeGroup, dataset);\n              dataset.update(switchGroup);\n              dataset.update(shouldBeGroup);\n\n              const switchGroupId = newOrder[curPos];\n              newOrder[curPos] = origOrder[curPos];\n              newOrder[slippedPosition] = switchGroupId;\n\n              curPos++;\n            }\n          }\n        }\n      });\n\n      me.body.emitter.emit('groupDragged', { groupId: id });\n      this.toggleGroupDragClassName(this.groupTouchParams.group);\n      this.groupTouchParams.group = null;\n    }\n  }\n\n  /**\n   * Handle selecting/deselecting an item when tapping it\n   * @param {Event} event\n   * @private\n   */\n  _onSelectItem(event) {\n    if (!this.options.selectable) return;\n\n    const ctrlKey  = event.srcEvent && (event.srcEvent.ctrlKey || event.srcEvent.metaKey);\n    const shiftKey = event.srcEvent && event.srcEvent.shiftKey;\n    if (ctrlKey || shiftKey) {\n      this._onMultiSelectItem(event);\n      return;\n    }\n\n    const oldSelection = this.getSelection();\n\n    const item = this.itemFromTarget(event);\n    const selection = item && item.selectable ? [item.id] : [];\n    this.setSelection(selection);\n\n    const newSelection = this.getSelection();\n\n    // emit a select event,\n    // except when old selection is empty and new selection is still empty\n    if (newSelection.length > 0 || oldSelection.length > 0) {\n      this.body.emitter.emit('select', {\n        items: newSelection,\n        event\n      });\n    }\n  }\n\n  /**\n   * Handle hovering an item\n   * @param {Event} event\n   * @private\n   */\n  _onMouseOver(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    // Item we just left\n    const related = this.itemFromRelatedTarget(event);\n    if (item === related) {\n      // We haven't changed item, just element in the item\n      return;\n    }\n\n    const title = item.getTitle();\n    if (this.options.showTooltips && title) {\n      if (this.popup == null) {\n        this.popup = new Popup(this.body.dom.root,\n            this.options.tooltip.overflowMethod || 'flip');\n      }\n\n      this.popup.setText(title);\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.clientX - containerRect.left + container.offsetLeft,\n        event.clientY - containerRect.top + container.offsetTop\n      );\n      this.setPopupTimer(this.popup);\n    } else {\n      // Hovering over item without a title, hide popup\n      // Needed instead of _just_ in _onMouseOut due to #2572\n      this.clearPopupTimer();\n      if (this.popup != null) {\n        this.popup.hide();\n      }\n    }\n\n    this.body.emitter.emit('itemover', {\n      item: item.id,\n      event\n    });\n  }\n\n  /**\n   * on mouse start\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onMouseOut(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    // Item we are going to\n    const related = this.itemFromRelatedTarget(event);\n    if (item === related) {\n      // We aren't changing item, just element in the item\n      return;\n    }\n\n    this.clearPopupTimer();\n    if (this.popup != null) {\n      this.popup.hide();\n    }\n\n    this.body.emitter.emit('itemout', {\n      item: item.id,\n      event\n    });\n  }\n\n  /**\n   * on mouse move\n   * @param {Event} event\n   * @return {void}   \n   * @private\n   */\n  _onMouseMove(event) {\n    const item = this.itemFromTarget(event);\n    if (!item) return;\n\n    if (this.popupTimer != null) {\n      // restart timer\n      this.setPopupTimer(this.popup);\n    }\n    \n    if (this.options.showTooltips && this.options.tooltip.followMouse && this.popup && !this.popup.hidden) {\n      const container = this.body.dom.centerContainer;\n      const containerRect = container.getBoundingClientRect()\n      this.popup.setPosition(\n        event.clientX - containerRect.left + container.offsetLeft,\n        event.clientY - containerRect.top + container.offsetTop\n      );\n      this.popup.show(); // Redraw\n    }\n  }\n\n  /**\n   * Handle mousewheel\n   * @param {Event}  event   The event\n   * @private\n   */\n  _onMouseWheel(event) {\n    if (this.touchParams.itemIsDragging) {\n      this._onDragEnd(event);\n    }\n  }\n\n  /**\n   * Handle updates of an item on double tap\n   * @param {timeline.Item}  item   The item\n   * @private\n   */\n  _onUpdateItem(item) {\n    if (!this.options.selectable) return;\n    if (!this.options.editable.updateTime && !this.options.editable.updateGroup) return;\n\n    const me = this;\n   \n    if (item) {\n      // execute async handler to update the item (or cancel it)\n      const itemData = me.itemsData.get(item.id); // get a clone of the data from the dataset\n      this.options.onUpdate(itemData, itemData => {\n        if (itemData) {\n          me.itemsData.update(itemData);\n        }\n      });\n    }\n  }\n\n  /**\n   * Handle drop event of data on item\n   * Only called when `objectData.target === 'item'.\n   * @param {Event} event The event \n   * @private\n   */\n  _onDropObjectOnItem(event) {\n    const item = this.itemFromTarget(event);\n    const objectData = JSON.parse(event.dataTransfer.getData(\"text\"));\n    this.options.onDropObjectOnItem(objectData, item)\n  }\n\n  /**\n   * Handle creation of an item on double tap or drop of a drag event\n   * @param {Event} event   The event\n   * @private\n   */\n  _onAddItem(event) {\n    if (!this.options.selectable) return;\n    if (!this.options.editable.add) return;\n\n    const me = this;\n    const snap = this.options.snap || null;\n\n    // add item\n    const frameRect = this.dom.frame.getBoundingClientRect()\n    const x = this.options.rtl ? frameRect.right - event.center.x : event.center.x - frameRect.left;\n    const start = this.body.util.toTime(x);\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    let end;\n\n    let newItemData;\n    if (event.type == 'drop') {\n      newItemData = JSON.parse(event.dataTransfer.getData(\"text\"));\n      newItemData.content = newItemData.content ? newItemData.content : 'new item';\n      newItemData.start = newItemData.start ? newItemData.start : (snap ? snap(start, scale, step) : start);\n      newItemData.type = newItemData.type || 'box';\n      newItemData[this.itemsData.idProp] = newItemData.id || randomUUID();\n\n      if (newItemData.type == 'range' && !newItemData.end) {\n        end = this.body.util.toTime(x + this.props.width / 5);\n        newItemData.end = snap ? snap(end, scale, step) : end;\n      }\n    } else {\n      newItemData = {\n        start: snap ? snap(start, scale, step) : start,\n        content: 'new item'\n      };\n      newItemData[this.itemsData.idProp] = randomUUID();\n\n      // when default type is a range, add a default end date to the new item\n      if (this.options.type === 'range') {\n        end = this.body.util.toTime(x + this.props.width / 5);\n        newItemData.end = snap ? snap(end, scale, step) : end;\n      }\n    }\n\n    const group = this.groupFromTarget(event);\n    if (group) {\n      newItemData.group = group.groupId;\n    }\n\n    // execute async handler to customize (or cancel) adding an item\n    newItemData = this._cloneItemData(newItemData);     // convert start and end to the correct type\n    this.options.onAdd(newItemData, item => {\n      if (item) {\n        me.itemsData.add(item);\n        if (event.type == 'drop') {\n          me.setSelection([item.id]);\n        }\n        // TODO: need to trigger a redraw?\n      }\n    });\n  }\n\n  /**\n   * Handle selecting/deselecting multiple items when holding an item\n   * @param {Event} event\n   * @private\n   */\n  _onMultiSelectItem(event) {\n    if (!this.options.selectable) return;\n\n    const item = this.itemFromTarget(event);\n\n    if (item) {\n      // multi select items (if allowed)\n\n      let selection = this.options.multiselect\n        ? this.getSelection() // take current selection\n        : [];                 // deselect current selection\n\n      const shiftKey = event.srcEvent && event.srcEvent.shiftKey || false;\n\n      if ((shiftKey || this.options.sequentialSelection) && this.options.multiselect) {\n        // select all items between the old selection and the tapped item\n        const itemGroup = this.itemsData.get(item.id).group;\n\n        // when filtering get the group of the last selected item\n        let lastSelectedGroup = undefined;\n        if (this.options.multiselectPerGroup) {\n          if (selection.length > 0) {\n            lastSelectedGroup = this.itemsData.get(selection[0]).group;\n          }\n        }\n\n        // determine the selection range\n        if (!this.options.multiselectPerGroup || lastSelectedGroup == undefined || lastSelectedGroup == itemGroup) {\n          selection.push(item.id);\n        }\n        const range = ItemSet._getItemRange(this.itemsData.get(selection));\n        \n        if (!this.options.multiselectPerGroup || lastSelectedGroup == itemGroup) {\n          // select all items within the selection range\n          selection = [];\n          for (const id in this.items) {\n            if (this.items.hasOwnProperty(id)) {\n              const _item = this.items[id];\n              const start = _item.data.start;\n              const end = (_item.data.end !== undefined) ? _item.data.end : start;\n\n              if (start >= range.min &&\n                  end <= range.max &&\n                  (!this.options.multiselectPerGroup || lastSelectedGroup == this.itemsData.get(_item.id).group) &&\n                  !(_item instanceof BackgroundItem)) {\n                selection.push(_item.id); // do not use id but item.id, id itself is stringified\n              }\n            }\n          }\n        }\n      }\n      else {\n        // add/remove this item from the current selection\n        const index = selection.indexOf(item.id);\n        if (index == -1) {\n          // item is not yet selected -> select it\n          selection.push(item.id);\n        }\n        else {\n          // item is already selected -> deselect it\n          selection.splice(index, 1);\n        }\n      }\n\n      const filteredSelection = selection.filter(item => this.getItemById(item).selectable);\n\n      this.setSelection(filteredSelection);\n\n      this.body.emitter.emit('select', {\n        items: this.getSelection(),\n        event\n      });\n    }\n  }\n\n  /**\n   * Calculate the time range of a list of items\n   * @param {Array.<Object>} itemsData\n   * @return {{min: Date, max: Date}} Returns the range of the provided items\n   * @private\n   */\n  static _getItemRange(itemsData) {\n    let max = null;\n    let min = null;\n\n    itemsData.forEach(data => {\n      if (min == null || data.start < min) {\n        min = data.start;\n      }\n\n      if (data.end != undefined) {\n        if (max == null || data.end > max) {\n          max = data.end;\n        }\n      }\n      else {\n        if (max == null || data.start > max) {\n          max = data.start;\n        }\n      }\n    });\n\n    return {\n      min,\n      max\n    }\n  }\n\n  /**\n   * Find an item from an element:\n   * searches for the attribute 'vis-item' in the element's tree\n   * @param {HTMLElement} element\n   * @return {Item | null} item\n   */\n  itemFromElement(element) {\n    let cur = element;\n    while (cur) {\n      if (cur.hasOwnProperty('vis-item')) {\n        return cur['vis-item'];\n      }\n      cur = cur.parentNode;\n    }\n\n    return null;\n  }\n\n  /**\n   * Find an item from an event target:\n   * searches for the attribute 'vis-item' in the event target's element tree\n   * @param {Event} event\n   * @return {Item | null} item\n   */\n  itemFromTarget(event) {\n    return this.itemFromElement(event.target);\n  }\n\n  /**\n   * Find an item from an event's related target:\n   * searches for the attribute 'vis-item' in the related target's element tree\n   * @param {Event} event\n   * @return {Item | null} item\n   */\n  itemFromRelatedTarget(event) {\n    return this.itemFromElement(event.relatedTarget);\n  }\n\n  /**\n   * Find the Group from an event target:\n   * searches for the attribute 'vis-group' in the event target's element tree\n   * @param {Event} event\n   * @return {Group | null} group\n   */\n  groupFromTarget(event) {\n    const clientY = event.center ? event.center.y : event.clientY;\n    let groupIds = this.groupIds;\n    \n    if (groupIds.length <= 0 && this.groupsData) {\n      groupIds = this.groupsData.getIds({\n        order: this.options.groupOrder\n      });\n    }\n    \n    for (let i = 0; i < groupIds.length; i++) {\n      const groupId = groupIds[i];\n      const group = this.groups[groupId];\n      const foreground = group.dom.foreground;\n      const foregroundRect = foreground.getBoundingClientRect()\n      if (clientY >= foregroundRect.top && clientY < foregroundRect.top + foreground.offsetHeight) {\n        return group;\n      }\n\n      if (this.options.orientation.item === 'top') {\n        if (i === this.groupIds.length - 1 && clientY > foregroundRect.top) {\n          return group;\n        }\n      }\n      else {\n        if (i === 0 && clientY < foregroundRect.top + foreground.offset) {\n          return group;\n        }\n      }\n    }\n\n    return null;\n  }\n\n  /**\n   * Find the ItemSet from an event target:\n   * searches for the attribute 'vis-itemset' in the event target's element tree\n   * @param {Event} event\n   * @return {ItemSet | null} item\n   */\n  static itemSetFromTarget(event) {\n    let target = event.target;\n    while (target) {\n      if (target.hasOwnProperty('vis-itemset')) {\n        return target['vis-itemset'];\n      }\n      target = target.parentNode;\n    }\n\n    return null;\n  }\n\n  /**\n   * Clone the data of an item, and \"normalize\" it: convert the start and end date\n   * to the type (Date, Moment, ...) configured in the DataSet. If not configured,\n   * start and end are converted to Date.\n   * @param {Object} itemData, typically `item.data`\n   * @param {string} [type]  Optional Date type. If not provided, the type from the DataSet is taken\n   * @return {Object} The cloned object\n   * @private\n   */\n  _cloneItemData(itemData, type) {\n    const clone = util.extend({}, itemData);\n\n    if (!type) {\n      // convert start and end date to the type (Date, Moment, ...) configured in the DataSet\n      type = this.itemsData.type;\n    }\n\n    if (clone.start != undefined) {\n      clone.start = util.convert(clone.start, type && type.start || 'Date');\n    }\n    if (clone.end != undefined) {\n      clone.end = util.convert(clone.end , type && type.end || 'Date');\n    }\n\n    return clone;\n  }\n\n  /**\n   * cluster items\n   * @return {void}   \n   * @private\n   */\n  _clusterItems() {\n    if (!this.options.cluster) {\n      return;\n    }\n\n    const { scale } = this.body.range.conversion(this.body.domProps.center.width);\n    const clusters = this.clusterGenerator.getClusters(this.clusters, scale, this.options.cluster);\n\n    if (this.clusters != clusters) {\n      this._detachAllClusters();\n\n      if (clusters) {\n        for (let cluster of clusters) {\n          cluster.attach();\n        }\n        this.clusters = clusters;\n      }\n\n      this._updateClusters(clusters);\n    }\n  }\n\n  /**\n   * detach all cluster items\n   * @private\n   */\n  _detachAllClusters() {\n    if (this.options.cluster) {\n      if (this.clusters && this.clusters.length) {\n        for (let cluster of this.clusters) {\n          cluster.detach();\n        }\n      }\n    }\n  }\n\n  /**\n   * update clusters\n   * @param {array} clusters\n   * @private\n   */\n  _updateClusters(clusters) {\n    if (this.clusters && this.clusters.length) {\n      const newClustersIds = new Set(clusters.map(cluster => cluster.id));\n      const clustersToUnselect = this.clusters.filter(cluster => !newClustersIds.has(cluster.id));\n      let selectionChanged = false;\n      for (let cluster of clustersToUnselect) {\n        const selectedIdx = this.selection.indexOf(cluster.id);\n        if (selectedIdx !== -1) {\n          cluster.unselect();\n          this.selection.splice(selectedIdx, 1);\n          selectionChanged = true;\n        }\n      }\n\n      if (selectionChanged) {\n        const newSelection = this.getSelection();\n        this.body.emitter.emit('select', {\n          items: newSelection,\n          event: event\n        });\n      }\n    }\n\n    this.clusters = clusters || [];\n  }\n}\n\n// available item types will be registered here\nItemSet.types = {\n  background: BackgroundItem,\n  box: BoxItem,\n  range: RangeItem,\n  point: PointItem\n};\n\n/**\n * Handle added items\n * @param {number[]} ids\n * @protected\n */\nItemSet.prototype._onAdd = ItemSet.prototype._onUpdate;\n\nexport default ItemSet;\n","import util from '../util';\n\nlet errorFound = false;\nlet allOptions;\nlet printStyle = 'background: #FFeeee; color: #dd0000';\n/**\n *  Used to validate options.\n */\nclass Validator {\n  /**\n   * @ignore\n   */\n  constructor() {\n  }\n\n  /**\n   * Main function to be called\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {Object} subObject\n   * @returns {boolean}\n   * @static\n   */\n  static validate(options, referenceOptions, subObject) {\n    errorFound = false;\n    allOptions = referenceOptions;\n    let usedOptions = referenceOptions;\n    if (subObject !== undefined) {\n      usedOptions = referenceOptions[subObject];\n    }\n    Validator.parse(options, usedOptions, []);\n    return errorFound;\n  }\n\n\n  /**\n   * Will traverse an object recursively and check every value\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {array} path    | where to look for the actual option\n   * @static\n   */\n  static parse(options, referenceOptions, path) {\n    for (let option in options) {\n      if (options.hasOwnProperty(option)) {\n        Validator.check(option, options, referenceOptions, path);\n      }\n    }\n  }\n\n\n  /**\n   * Check every value. If the value is an object, call the parse function on that object.\n   * @param {string} option\n   * @param {Object} options\n   * @param {Object} referenceOptions\n   * @param {array} path    | where to look for the actual option\n   * @static\n   */\n  static check(option, options, referenceOptions, path) {\n    if (referenceOptions[option] === undefined && referenceOptions.__any__ === undefined) {\n      Validator.getSuggestion(option, referenceOptions, path);\n      return;\n    }\n\n    let referenceOption = option;\n    let is_object = true;\n\n    if (referenceOptions[option] === undefined && referenceOptions.__any__ !== undefined) {\n      // NOTE: This only triggers if the __any__ is in the top level of the options object.\n      //       THAT'S A REALLY BAD PLACE TO ALLOW IT!!!!\n      // TODO: Examine if needed, remove if possible\n\n      // __any__ is a wildcard. Any value is accepted and will be further analysed by reference.\n      referenceOption = '__any__';\n\n      // if the any-subgroup is not a predefined object in the configurator,\n      // we do not look deeper into the object.\n      is_object = (Validator.getType(options[option]) === 'object');\n    }\n    else {\n      // Since all options in the reference are objects, we can check whether \n      // they are supposed to be the object to look for the __type__ field.\n      // if this is an object, we check if the correct type has been supplied to account for shorthand options.\n    }\n\n    let refOptionObj = referenceOptions[referenceOption];\n    if (is_object && refOptionObj.__type__ !== undefined) {\n      refOptionObj = refOptionObj.__type__;\n    }\n\n    Validator.checkFields(option, options, referenceOptions, referenceOption, refOptionObj, path);\n  }\n\n  /**\n   *\n   * @param {string}  option           | the option property\n   * @param {Object}  options          | The supplied options object\n   * @param {Object}  referenceOptions | The reference options containing all options and their allowed formats\n   * @param {string}  referenceOption  | Usually this is the same as option, except when handling an __any__ tag.\n   * @param {string}  refOptionObj     | This is the type object from the reference options\n   * @param {Array}   path             | where in the object is the option\n   * @static\n   */\n  static checkFields(option, options, referenceOptions, referenceOption, refOptionObj, path) {\n    let log = function(message) {\n      console.log('%c' + message + Validator.printLocation(path, option), printStyle);\n    };\n\n    let optionType = Validator.getType(options[option]);\n    let refOptionType = refOptionObj[optionType];\n\n    if (refOptionType !== undefined) {\n      // if the type is correct, we check if it is supposed to be one of a few select values\n      if (Validator.getType(refOptionType) === 'array' && refOptionType.indexOf(options[option]) === -1) {\n        log('Invalid option detected in \"' + option + '\".' +\n          ' Allowed values are:' + Validator.print(refOptionType) +\n          ' not \"' + options[option] + '\". ');\n        errorFound = true;\n      }\n      else if (optionType === 'object' && referenceOption !== \"__any__\") {\n        path = util.copyAndExtendArray(path, option);\n        Validator.parse(options[option], referenceOptions[referenceOption], path);\n      }\n    }\n    else if (refOptionObj['any'] === undefined) {\n      // type of the field is incorrect and the field cannot be any\n      log('Invalid type received for \"' + option +\n        '\". Expected: ' + Validator.print(Object.keys(refOptionObj)) +\n        '. Received ['  + optionType + '] \"' + options[option] + '\"');\n      errorFound = true;\n    }\n  }\n\n  /**\n   *\n   * @param {Object|boolean|number|string|Array.<number>|Date|Node|Moment|undefined|null} object\n   * @returns {string}\n   * @static\n   */\n  static getType(object) {\n    var type = typeof object;\n\n    if (type === 'object') {\n      if (object === null) {\n        return 'null';\n      }\n      if (object instanceof Boolean) {\n        return 'boolean';\n      }\n      if (object instanceof Number) {\n        return 'number';\n      }\n      if (object instanceof String) {\n        return 'string';\n      }\n      if (Array.isArray(object)) {\n        return 'array';\n      }\n      if (object instanceof Date) {\n        return 'date';\n      }\n      if (object.nodeType !== undefined) {\n        return 'dom';\n      }\n      if (object._isAMomentObject === true) {\n        return 'moment';\n      }\n      return 'object';\n    }\n    else if (type === 'number') {\n      return 'number';\n    }\n    else if (type === 'boolean') {\n      return 'boolean';\n    }\n    else if (type === 'string') {\n      return 'string';\n    }\n    else if (type === undefined) {\n      return 'undefined';\n    }\n    return type;\n  }\n\n  /**\n   * @param {string} option\n   * @param {Object} options\n   * @param {Array.<string>} path\n   * @static\n   */\n  static getSuggestion(option, options, path) {\n    let localSearch = Validator.findInOptions(option,options,path,false);\n    let globalSearch = Validator.findInOptions(option,allOptions,[],true);\n\n    let localSearchThreshold = 8;\n    let globalSearchThreshold = 4;\n\n    let msg;\n    if (localSearch.indexMatch !== undefined) {\n      msg = ' in ' + Validator.printLocation(localSearch.path, option,'') +\n        'Perhaps it was incomplete? Did you mean: \"' + localSearch.indexMatch + '\"?\\n\\n';\n    }\n    else if (globalSearch.distance <= globalSearchThreshold && localSearch.distance > globalSearch.distance) {\n      msg = ' in ' + Validator.printLocation(localSearch.path, option,'') +\n        'Perhaps it was misplaced? Matching option found at: ' +\n        Validator.printLocation(globalSearch.path, globalSearch.closestMatch,'');\n    }\n    else if (localSearch.distance <= localSearchThreshold) {\n      msg = '. Did you mean \"' + localSearch.closestMatch + '\"?' +\n        Validator.printLocation(localSearch.path, option);\n    }\n    else {\n      msg = '. Did you mean one of these: ' + Validator.print(Object.keys(options)) +\n      Validator.printLocation(path, option);\n    }\n\n    console.log('%cUnknown option detected: \"' + option + '\"' + msg, printStyle);\n    errorFound = true;\n  }\n\n  /**\n   * traverse the options in search for a match.\n   * @param {string} option\n   * @param {Object} options\n   * @param {Array} path    | where to look for the actual option\n   * @param {boolean} [recursive=false]\n   * @returns {{closestMatch: string, path: Array, distance: number}}\n   * @static\n   */\n  static findInOptions(option, options, path, recursive = false) {\n    let min = 1e9;\n    let closestMatch = '';\n    let closestMatchPath = [];\n    let lowerCaseOption = option.toLowerCase();\n    let indexMatch = undefined;\n    for (let op in options) {  // eslint-disable-line guard-for-in\n      let distance;\n      if (options[op].__type__ !== undefined && recursive === true) {\n        let result = Validator.findInOptions(option, options[op], util.copyAndExtendArray(path,op));\n        if (min > result.distance) {\n          closestMatch = result.closestMatch;\n          closestMatchPath = result.path;\n          min = result.distance;\n          indexMatch = result.indexMatch;\n        }\n      }\n      else {\n        if (op.toLowerCase().indexOf(lowerCaseOption) !== -1) {\n          indexMatch = op;\n        }\n        distance = Validator.levenshteinDistance(option, op);\n        if (min > distance) {\n          closestMatch = op;\n          closestMatchPath = util.copyArray(path);\n          min = distance;\n        }\n      }\n    }\n    return {closestMatch:closestMatch, path:closestMatchPath, distance:min, indexMatch: indexMatch};\n  }\n\n  /**\n   * @param {Array.<string>} path\n   * @param {Object} option\n   * @param {string} prefix\n   * @returns {String}\n   * @static\n   */\n  static printLocation(path, option, prefix = 'Problem value found at: \\n') {\n    let str = '\\n\\n' + prefix + 'options = {\\n';\n    for (let i = 0; i < path.length; i++) {\n      for (let j = 0; j < i + 1; j++) {\n        str += '  ';\n      }\n      str += path[i] + ': {\\n'\n    }\n    for (let j = 0; j < path.length + 1; j++) {\n      str += '  ';\n    }\n    str += option + '\\n';\n    for (let i = 0; i < path.length + 1; i++) {\n      for (let j = 0; j < path.length - i; j++) {\n        str += '  ';\n      }\n      str += '}\\n'\n    }\n    return str + '\\n\\n';\n  }\n\n  /**\n   * @param {Object} options\n   * @returns {String}\n   * @static\n   */\n  static print(options) {\n    return JSON.stringify(options).replace(/(\\\")|(\\[)|(\\])|(,\"__type__\")/g, \"\").replace(/(\\,)/g, ', ')\n  }\n\n\n  /**\n   *  Compute the edit distance between the two given strings\n   * http://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#JavaScript\n   *\n   * Copyright (c) 2011 Andrei Mackenzie\n   *\n   * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n   *\n   * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n   *\n   * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n   *\n   * @param {string} a\n   * @param {string} b\n   * @returns {Array.<Array.<number>>}}\n   * @static\n   */\n  static levenshteinDistance(a, b) {\n    if (a.length === 0) return b.length;\n    if (b.length === 0) return a.length;\n\n    var matrix = [];\n\n    // increment along the first column of each row\n    var i;\n    for (i = 0; i <= b.length; i++) {\n      matrix[i] = [i];\n    }\n\n    // increment each column in the first row\n    var j;\n    for (j = 0; j <= a.length; j++) {\n      matrix[0][j] = j;\n    }\n\n    // Fill in the rest of the matrix\n    for (i = 1; i <= b.length; i++) {\n      for (j = 1; j <= a.length; j++) {\n        if (b.charAt(i - 1) == a.charAt(j - 1)) {\n          matrix[i][j] = matrix[i - 1][j - 1];\n        } else {\n          matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, // substitution\n            Math.min(matrix[i][j - 1] + 1, // insertion\n              matrix[i - 1][j] + 1)); // deletion\n        }\n      }\n    }\n\n    return matrix[b.length][a.length];\n  }\n}\n\n\nexport {\n  Validator,\n  printStyle\n};\n","/**\n * This object contains all possible options. It will check if the types are correct, if required if the option is one\n * of the allowed values.\n *\n * __any__ means that the name of the property does not matter.\n * __type__ is a required field for all objects and contains the allowed types of all objects\n */\nlet string = 'string';\nlet bool = 'boolean';\nlet number = 'number';\nlet array = 'array';\nlet date = 'date';\nlet object = 'object'; // should only be in a __type__ property\nlet dom = 'dom';\nlet moment = 'moment';\nlet any = 'any';\n\nlet allOptions = {\n  configure: {\n    enabled: { 'boolean': bool},\n    filter: { 'boolean': bool,'function': 'function'},\n    container: {dom},\n    __type__: {object, 'boolean': bool,'function': 'function'}\n  },\n\n  //globals :\n  align: {string},\n  alignCurrentTime: {string, 'undefined': 'undefined'},\n  rtl: { 'boolean': bool, 'undefined': 'undefined'},\n  rollingMode: {\n    follow: { 'boolean': bool },\n    offset: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  onTimeout: {\n    timeoutMs: {number},\n    callback: {'function': 'function'},\n    __type__: {object}\n  },\n  verticalScroll: { 'boolean': bool, 'undefined': 'undefined'},\n  horizontalScroll: { 'boolean': bool, 'undefined': 'undefined'},\n  autoResize: { 'boolean': bool},\n  throttleRedraw: {number}, // TODO: DEPRICATED see https://github.com/almende/vis/issues/2511\n  clickToUse: { 'boolean': bool},\n  dataAttributes: {string, array},\n  editable: {\n    add: { 'boolean': bool, 'undefined': 'undefined'},\n    remove: { 'boolean': bool, 'undefined': 'undefined'},\n    updateGroup: { 'boolean': bool, 'undefined': 'undefined'},\n    updateTime: { 'boolean': bool, 'undefined': 'undefined'},\n    overrideItems: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  end: {number, date, string, moment},\n  format: {\n    minorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object, 'function': 'function'}\n    },\n    majorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object, 'function': 'function'}\n    },\n    __type__: {object}\n  },\n  moment: {'function': 'function'},\n  groupHeightMode: {string},\n  groupOrder: {string, 'function': 'function'},\n  groupEditable: {\n    add: { 'boolean': bool, 'undefined': 'undefined'},\n    remove: { 'boolean': bool, 'undefined': 'undefined'},\n    order: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  groupOrderSwap: {'function': 'function'},\n  height: {string, number},\n  hiddenDates: {\n    start: {date, number, string, moment},\n    end: {date, number, string, moment},\n    repeat: {string},\n    __type__: {object, array}\n  },\n  itemsAlwaysDraggable: {\n    item: { 'boolean': bool, 'undefined': 'undefined'},\n    range: { 'boolean': bool, 'undefined': 'undefined'},\n    __type__: { 'boolean': bool, object}\n  },\n  limitSize: {'boolean': bool},\n  locale:{string},\n  locales:{\n    __any__: {any},\n    __type__: {object}\n  },\n  longSelectPressTime: {number},\n  margin: {\n    axis: {number},\n    item: {\n      horizontal: {number,'undefined': 'undefined'},\n      vertical: {number,'undefined': 'undefined'},\n      __type__: {object,number}\n    },\n    __type__: {object,number}\n  },\n  max: {date, number, string, moment},\n  maxHeight: {number, string},\n  maxMinorChars: {number},\n  min: {date, number, string, moment},\n  minHeight: {number, string},\n  moveable: { 'boolean': bool},\n  multiselect: { 'boolean': bool},\n  multiselectPerGroup: { 'boolean': bool},\n  onAdd: {'function': 'function'},\n  onDropObjectOnItem: {'function': 'function'},\n  onUpdate: {'function': 'function'},\n  onMove: {'function': 'function'},\n  onMoving: {'function': 'function'},\n  onRemove: {'function': 'function'},\n  onAddGroup: {'function': 'function'},\n  onMoveGroup: {'function': 'function'},\n  onRemoveGroup: {'function': 'function'},\n  onInitialDrawComplete: {'function': 'function'},\n  order: {'function': 'function'},\n  orientation: {\n    axis: {string,'undefined': 'undefined'},\n    item: {string,'undefined': 'undefined'},\n    __type__: {string, object}\n  },\n  selectable: { 'boolean': bool},\n  sequentialSelection: { 'boolean': bool },\n  showCurrentTime: { 'boolean': bool},\n  showMajorLabels: { 'boolean': bool},\n  showMinorLabels: { 'boolean': bool},\n  showWeekScale: { 'boolean': bool},\n  stack: { 'boolean': bool},\n  stackSubgroups: { 'boolean': bool},\n  cluster: { \n    maxItems: {'number': number, 'undefined': 'undefined'},\n    titleTemplate: {'string': string, 'undefined': 'undefined'},\n    clusterCriteria: { 'function': 'function', 'undefined': 'undefined'},\n    showStipes: {'boolean': bool, 'undefined': 'undefined'},\n    fitOnDoubleClick: {'boolean': bool, 'undefined': 'undefined'},\n    __type__: {'boolean': bool, object}\n  },\n  snap: {'function': 'function', 'null': 'null'},\n  start: {date, number, string, moment},\n  template: {'function': 'function'},\n  loadingScreenTemplate: {'function': 'function'},\n  groupTemplate: {'function': 'function'},\n  visibleFrameTemplate: {string, 'function': 'function'},\n  showTooltips: { 'boolean': bool},\n  tooltip: {\n    followMouse: { 'boolean': bool },\n    overflowMethod: { 'string': ['cap', 'flip', 'none'] },\n    delay: {number},\n    template: {'function': 'function'},\n    __type__: {object}\n  },\n  tooltipOnItemUpdateTime: {\n    template: {'function': 'function'},\n    __type__: { 'boolean': bool, object}\n  },\n  timeAxis: {\n    scale: {string,'undefined': 'undefined'},\n    step: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  type: {string},\n  width: {string, number},\n  preferZoom: { 'boolean': bool},\n  zoomable: { 'boolean': bool},\n  zoomKey: {string: ['ctrlKey', 'altKey', 'shiftKey', 'metaKey', '']},\n  zoomFriction: {number},\n  zoomMax: {number},\n  zoomMin: {number},\n\n  __type__: {object}\n};\n\nlet configureOptions = {\n  global: {\n    align:  ['center', 'left', 'right'],\n    alignCurrentTime: ['none', 'year', 'month', 'quarter', 'week', 'isoWeek', 'day', 'date', 'hour', 'minute', 'second'],\n    direction:  false,\n    autoResize: true,\n    clickToUse: false,\n    // dataAttributes: ['all'], // FIXME: can be 'all' or string[]\n      editable: {\n      add: false,\n      remove: false,\n      updateGroup: false,\n      updateTime: false\n    },\n    end: '',\n    format: {\n      minorLabels: {\n        millisecond:'SSS',\n        second:     's',\n        minute:     'HH:mm',\n        hour:       'HH:mm',\n        weekday:    'ddd D',\n        day:        'D',\n        week:       'w',\n        month:      'MMM',\n        year:       'YYYY'\n      },\n      majorLabels: {\n        millisecond:'HH:mm:ss',\n        second:     'D MMMM HH:mm',\n        minute:     'ddd D MMMM',\n        hour:       'ddd D MMMM',\n        weekday:    'MMMM YYYY',\n        day:        'MMMM YYYY',\n        week:       'MMMM YYYY',\n        month:      'YYYY',\n        year:       ''\n      }\n    },\n    groupHeightMode: ['auto', 'fixed', 'fitItems'],\n    //groupOrder: {string, 'function': 'function'},\n    groupsDraggable: false,\n    height: '',\n    //hiddenDates: {object, array},\n    locale: '',\n    longSelectPressTime: 251,\n    margin: {\n      axis: [20, 0, 100, 1],\n      item: {\n        horizontal: [10, 0, 100, 1],\n        vertical: [10, 0, 100, 1]\n      }\n    },\n    max: '',\n    maxHeight: '',\n    maxMinorChars: [7, 0, 20, 1],\n    min: '',\n    minHeight: '',\n    moveable: false,\n    multiselect: false,\n    multiselectPerGroup: false,\n    //onAdd: {'function': 'function'},\n    //onUpdate: {'function': 'function'},\n    //onMove: {'function': 'function'},\n    //onMoving: {'function': 'function'},\n    //onRename: {'function': 'function'},\n    //order: {'function': 'function'},\n    orientation: {\n      axis: ['both', 'bottom', 'top'],\n      item: ['bottom', 'top']\n    },\n    preferZoom: false,\n    selectable: true,\n    showCurrentTime: false,\n    showMajorLabels: true,\n    showMinorLabels: true,\n    stack: true,\n    stackSubgroups: true,\n    cluster: false,\n    //snap: {'function': 'function', nada},\n    start: '',\n    //template: {'function': 'function'},\n    //timeAxis: {\n    //  scale: ['millisecond', 'second', 'minute', 'hour', 'weekday', 'day', 'week', 'month', 'year'],\n    //  step: [1, 1, 10, 1]\n    //},\n    showTooltips: true,\n    tooltip: {\n      followMouse: false,\n      overflowMethod: 'flip',\n      delay: [500, 0, 99999, 100],\n    },\n    tooltipOnItemUpdateTime: false,\n    type: ['box', 'point', 'range', 'background'],\n    width: '100%',\n    zoomable: true,\n    zoomKey: ['ctrlKey', 'altKey', 'shiftKey', 'metaKey', ''],\n    zoomMax: [315360000000000, 10, 315360000000000, 1],\n    zoomMin: [10, 10, 315360000000000, 1]\n  }\n};\n\nexport {allOptions, configureOptions};\n","var $ = require('../internals/export');\nvar fill = require('../internals/array-fill');\nvar addToUnscopables = require('../internals/add-to-unscopables');\n\n// `Array.prototype.fill` method\n// https://tc39.github.io/ecma262/#sec-array.prototype.fill\n$({ target: 'Array', proto: true }, {\n  fill: fill\n});\n\n// https://tc39.github.io/ecma262/#sec-array.prototype-@@unscopables\naddToUnscopables('fill');\n","'use strict';\nvar toObject = require('../internals/to-object');\nvar toAbsoluteIndex = require('../internals/to-absolute-index');\nvar toLength = require('../internals/to-length');\n\n// `Array.prototype.fill` method implementation\n// https://tc39.github.io/ecma262/#sec-array.prototype.fill\nmodule.exports = function fill(value /* , start = 0, end = @length */) {\n  var O = toObject(this);\n  var length = toLength(O.length);\n  var argumentsLength = arguments.length;\n  var index = toAbsoluteIndex(argumentsLength > 1 ? arguments[1] : undefined, length);\n  var end = argumentsLength > 2 ? arguments[2] : undefined;\n  var endPos = end === undefined ? length : toAbsoluteIndex(end, length);\n  while (endPos > index) O[index++] = value;\n  return O;\n};\n","require('../../../modules/es.array.fill');\nvar entryVirtual = require('../../../internals/entry-virtual');\n\nmodule.exports = entryVirtual('Array').fill;\n","var fill = require('../array/virtual/fill');\n\nvar ArrayPrototype = Array.prototype;\n\nmodule.exports = function (it) {\n  var own = it.fill;\n  return it === ArrayPrototype || (it instanceof Array && own === ArrayPrototype.fill) ? fill : own;\n};\n","import Hammer from '../module/hammer';\nimport * as hammerUtil from '../hammerUtil';\nimport util from '../util';\n\n\nvar htmlColors = {black: '#000000', navy: '#000080', darkblue: '#00008B', mediumblue: '#0000CD', blue: '#0000FF', darkgreen: '#006400', green: '#008000', teal: '#008080', darkcyan: '#008B8B', deepskyblue: '#00BFFF', darkturquoise: '#00CED1', mediumspringgreen: '#00FA9A', lime: '#00FF00', springgreen: '#00FF7F', aqua: '#00FFFF', cyan: '#00FFFF', midnightblue: '#191970', dodgerblue: '#1E90FF', lightseagreen: '#20B2AA', forestgreen: '#228B22', seagreen: '#2E8B57', darkslategray: '#2F4F4F', limegreen: '#32CD32', mediumseagreen: '#3CB371', turquoise: '#40E0D0', royalblue: '#4169E1', steelblue: '#4682B4', darkslateblue: '#483D8B', mediumturquoise: '#48D1CC', indigo: '#4B0082', darkolivegreen: '#556B2F', cadetblue: '#5F9EA0', cornflowerblue: '#6495ED', mediumaquamarine: '#66CDAA', dimgray: '#696969', slateblue: '#6A5ACD', olivedrab: '#6B8E23', slategray: '#708090', lightslategray: '#778899', mediumslateblue: '#7B68EE', lawngreen: '#7CFC00', chartreuse: '#7FFF00', aquamarine: '#7FFFD4', maroon: '#800000', purple: '#800080', olive: '#808000', gray: '#808080', skyblue: '#87CEEB', lightskyblue: '#87CEFA', blueviolet: '#8A2BE2', darkred: '#8B0000', darkmagenta: '#8B008B', saddlebrown: '#8B4513', darkseagreen: '#8FBC8F', lightgreen: '#90EE90', mediumpurple: '#9370D8', darkviolet: '#9400D3', palegreen: '#98FB98', darkorchid: '#9932CC', yellowgreen: '#9ACD32', sienna: '#A0522D', brown: '#A52A2A', darkgray: '#A9A9A9', lightblue: '#ADD8E6', greenyellow: '#ADFF2F', paleturquoise: '#AFEEEE', lightsteelblue: '#B0C4DE', powderblue: '#B0E0E6', firebrick: '#B22222', darkgoldenrod: '#B8860B', mediumorchid: '#BA55D3', rosybrown: '#BC8F8F', darkkhaki: '#BDB76B', silver: '#C0C0C0', mediumvioletred: '#C71585', indianred: '#CD5C5C', peru: '#CD853F', chocolate: '#D2691E', tan: '#D2B48C', lightgrey: '#D3D3D3', palevioletred: '#D87093', thistle: '#D8BFD8', orchid: '#DA70D6', goldenrod: '#DAA520', crimson: '#DC143C', gainsboro: '#DCDCDC', plum: '#DDA0DD', burlywood: '#DEB887', lightcyan: '#E0FFFF', lavender: '#E6E6FA', darksalmon: '#E9967A', violet: '#EE82EE', palegoldenrod: '#EEE8AA', lightcoral: '#F08080', khaki: '#F0E68C', aliceblue: '#F0F8FF', honeydew: '#F0FFF0', azure: '#F0FFFF', sandybrown: '#F4A460', wheat: '#F5DEB3', beige: '#F5F5DC', whitesmoke: '#F5F5F5', mintcream: '#F5FFFA', ghostwhite: '#F8F8FF', salmon: '#FA8072', antiquewhite: '#FAEBD7', linen: '#FAF0E6', lightgoldenrodyellow: '#FAFAD2', oldlace: '#FDF5E6', red: '#FF0000', fuchsia: '#FF00FF', magenta: '#FF00FF', deeppink: '#FF1493', orangered: '#FF4500', tomato: '#FF6347', hotpink: '#FF69B4', coral: '#FF7F50', darkorange: '#FF8C00', lightsalmon: '#FFA07A', orange: '#FFA500', lightpink: '#FFB6C1', pink: '#FFC0CB', gold: '#FFD700', peachpuff: '#FFDAB9', navajowhite: '#FFDEAD', moccasin: '#FFE4B5', bisque: '#FFE4C4', mistyrose: '#FFE4E1', blanchedalmond: '#FFEBCD', papayawhip: '#FFEFD5', lavenderblush: '#FFF0F5', seashell: '#FFF5EE', cornsilk: '#FFF8DC', lemonchiffon: '#FFFACD', floralwhite: '#FFFAF0', snow: '#FFFAFA', yellow: '#FFFF00', lightyellow: '#FFFFE0', ivory: '#FFFFF0', white: '#FFFFFF'};\n\n/**\n * @param {number} [pixelRatio=1]\n */\nclass ColorPicker {\n  /**\n   * @param {number} [pixelRatio=1]\n   */\n  constructor(pixelRatio = 1) {\n    this.pixelRatio = pixelRatio;\n    this.generated = false;\n    this.centerCoordinates = {x:289/2, y:289/2};\n    this.r = 289 * 0.49;\n    this.color = {r:255,g:255,b:255,a:1.0};\n    this.hueCircle = undefined;\n    this.initialColor = {r:255,g:255,b:255,a:1.0};\n    this.previousColor= undefined;\n    this.applied = false;\n\n    // bound by\n    this.updateCallback = () => {};\n    this.closeCallback = () => {};\n\n    // create all DOM elements\n    this._create();\n  }\n\n\n  /**\n   * this inserts the colorPicker into a div from the DOM\n   * @param {Element} container\n   */\n  insertTo(container) {\n    if (this.hammer !== undefined) {\n      this.hammer.destroy();\n      this.hammer = undefined;\n    }\n    this.container = container;\n    this.container.appendChild(this.frame);\n    this._bindHammer();\n\n    this._setSize();\n  }\n\n  /**\n   * the callback is executed on apply and save. Bind it to the application\n   * @param {function} callback\n   */\n  setUpdateCallback(callback) {\n    if (typeof callback === 'function') {\n      this.updateCallback = callback;\n    }\n    else {\n      throw new Error(\"Function attempted to set as colorPicker update callback is not a function.\");\n    }\n  }\n\n  /**\n   * the callback is executed on apply and save. Bind it to the application\n   * @param {function} callback\n   */\n  setCloseCallback(callback) {\n    if (typeof callback === 'function') {\n      this.closeCallback = callback;\n    }\n    else {\n      throw new Error(\"Function attempted to set as colorPicker closing callback is not a function.\");\n    }\n  }\n\n  /**\n   *\n   * @param {string} color\n   * @returns {String}\n   * @private\n   */\n  _isColorString(color) {\n    if (typeof color === 'string') {\n      return htmlColors[color];\n    }\n  }\n\n\n  /**\n   * Set the color of the colorPicker\n   * Supported formats:\n   * 'red'                   --> HTML color string\n   * '#ffffff'               --> hex string\n   * 'rgb(255,255,255)'      --> rgb string\n   * 'rgba(255,255,255,1.0)' --> rgba string\n   * {r:255,g:255,b:255}     --> rgb object\n   * {r:255,g:255,b:255,a:1.0} --> rgba object\n   * @param {string|Object} color\n   * @param {boolean} [setInitial=true]\n   */\n  setColor(color, setInitial = true) {\n    if (color === 'none') {\n      return;\n    }\n\n    let rgba;\n\n    // if a html color shorthand is used, convert to hex\n    var htmlColor = this._isColorString(color);\n    if (htmlColor !== undefined) {\n      color = htmlColor;\n    }\n\n    // check format\n    if (util.isString(color) === true) {\n      if (util.isValidRGB(color) === true) {\n        let rgbaArray = color.substr(4).substr(0, color.length - 5).split(',');\n        rgba = {r:rgbaArray[0], g:rgbaArray[1], b:rgbaArray[2], a:1.0};\n      }\n      else if (util.isValidRGBA(color) === true) {\n        let rgbaArray = color.substr(5).substr(0, color.length - 6).split(',');\n        rgba = {r:rgbaArray[0], g:rgbaArray[1], b:rgbaArray[2], a:rgbaArray[3]};\n      }\n      else if (util.isValidHex(color) === true) {\n        let rgbObj = util.hexToRGB(color);\n        rgba = {r:rgbObj.r, g:rgbObj.g, b:rgbObj.b, a:1.0};\n      }\n    }\n    else {\n      if (color instanceof Object) {\n        if (color.r !== undefined && color.g !== undefined && color.b !== undefined) {\n          let alpha = color.a !== undefined ? color.a : '1.0';\n          rgba = {r:color.r, g:color.g, b:color.b, a:alpha};\n        }\n      }\n    }\n\n    // set color\n    if (rgba === undefined) {\n      throw new Error(\"Unknown color passed to the colorPicker. Supported are strings: rgb, hex, rgba. Object: rgb ({r:r,g:g,b:b,[a:a]}). Supplied: \" + JSON.stringify(color));\n    }\n    else {\n      this._setColor(rgba, setInitial);\n    }\n  }\n\n\n  /**\n   * this shows the color picker.\n   * The hue circle is constructed once and stored.\n   */\n  show() {\n    if (this.closeCallback !== undefined) {\n      this.closeCallback();\n      this.closeCallback = undefined;\n    }\n\n    this.applied = false;\n    this.frame.style.display = 'block';\n    this._generateHueCircle();\n  }\n\n  // ------------------------------------------ PRIVATE ----------------------------- //\n\n  /**\n   * Hide the picker. Is called by the cancel button.\n   * Optional boolean to store the previous color for easy access later on.\n   * @param {boolean} [storePrevious=true]\n   * @private\n   */\n  _hide(storePrevious = true) {\n    // store the previous color for next time;\n    if (storePrevious === true) {\n      this.previousColor = util.extend({}, this.color);\n    }\n\n    if (this.applied === true) {\n      this.updateCallback(this.initialColor);\n    }\n\n    this.frame.style.display = 'none';\n\n    // call the closing callback, restoring the onclick method.\n    // this is in a setTimeout because it will trigger the show again before the click is done.\n    setTimeout(() => {\n      if (this.closeCallback !== undefined) {\n        this.closeCallback();\n        this.closeCallback = undefined;\n      }\n    },0);\n  }\n\n\n  /**\n   * bound to the save button. Saves and hides.\n   * @private\n   */\n  _save() {\n    this.updateCallback(this.color);\n    this.applied = false;\n    this._hide();\n  }\n\n\n  /**\n   * Bound to apply button. Saves but does not close. Is undone by the cancel button.\n   * @private\n   */\n  _apply() {\n    this.applied = true;\n    this.updateCallback(this.color);\n    this._updatePicker(this.color);\n  }\n\n\n  /**\n   * load the color from the previous session.\n   * @private\n   */\n  _loadLast() {\n    if (this.previousColor !== undefined) {\n      this.setColor(this.previousColor, false);\n    }\n    else {\n      alert(\"There is no last color to load...\");\n    }\n  }\n\n\n  /**\n   * set the color, place the picker\n   * @param {Object} rgba\n   * @param {boolean} [setInitial=true]\n   * @private\n   */\n  _setColor(rgba, setInitial = true) {\n    // store the initial color\n    if (setInitial === true) {\n      this.initialColor = util.extend({}, rgba);\n    }\n\n    this.color = rgba;\n    let hsv = util.RGBToHSV(rgba.r, rgba.g, rgba.b);\n\n    let angleConvert = 2 * Math.PI;\n    let radius = this.r * hsv.s;\n    let x = this.centerCoordinates.x + radius * Math.sin(angleConvert * hsv.h);\n    let y = this.centerCoordinates.y + radius * Math.cos(angleConvert * hsv.h);\n\n    this.colorPickerSelector.style.left = x - 0.5 * this.colorPickerSelector.clientWidth + 'px';\n    this.colorPickerSelector.style.top = y - 0.5 * this.colorPickerSelector.clientHeight + 'px';\n\n    this._updatePicker(rgba);\n  }\n\n\n  /**\n   * bound to opacity control\n   * @param {number} value\n   * @private\n   */\n  _setOpacity(value) {\n    this.color.a = value / 100;\n    this._updatePicker(this.color);\n  }\n\n\n  /**\n   * bound to brightness control\n   * @param {number} value\n   * @private\n   */\n  _setBrightness(value) {\n    let hsv = util.RGBToHSV(this.color.r, this.color.g, this.color.b);\n    hsv.v = value / 100;\n    let rgba = util.HSVToRGB(hsv.h, hsv.s, hsv.v);\n    rgba['a'] = this.color.a;\n    this.color = rgba;\n    this._updatePicker();\n  }\n\n\n  /**\n   * update the color picker. A black circle overlays the hue circle to mimic the brightness decreasing.\n   * @param {Object} rgba\n   * @private\n   */\n  _updatePicker(rgba = this.color) {\n    let hsv = util.RGBToHSV(rgba.r, rgba.g, rgba.b);\n    let ctx = this.colorPickerCanvas.getContext('2d');\n    if (this.pixelRation === undefined) {\n      this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n      ctx.mozBackingStorePixelRatio ||\n      ctx.msBackingStorePixelRatio ||\n      ctx.oBackingStorePixelRatio ||\n      ctx.backingStorePixelRatio || 1);\n    }\n    ctx.setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n\n    // clear the canvas\n    let w = this.colorPickerCanvas.clientWidth;\n    let h = this.colorPickerCanvas.clientHeight;\n    ctx.clearRect(0, 0, w, h);\n\n    ctx.putImageData(this.hueCircle, 0,0);\n    ctx.fillStyle = 'rgba(0,0,0,' + (1- hsv.v) + ')';\n    ctx.circle(this.centerCoordinates.x, this.centerCoordinates.y, this.r);\n    ctx.fill();\n\n    this.brightnessRange.value = 100 * hsv.v;\n    this.opacityRange.value    = 100 * rgba.a;\n\n    this.initialColorDiv.style.backgroundColor = 'rgba(' + this.initialColor.r + ',' + this.initialColor.g + ',' + this.initialColor.b + ',' + this.initialColor.a + ')';\n    this.newColorDiv.style.backgroundColor = 'rgba(' + this.color.r + ',' + this.color.g + ',' + this.color.b + ',' + this.color.a + ')';\n  }\n\n\n  /**\n   * used by create to set the size of the canvas.\n   * @private\n   */\n  _setSize() {\n    this.colorPickerCanvas.style.width = '100%';\n    this.colorPickerCanvas.style.height = '100%';\n\n    this.colorPickerCanvas.width = 289 * this.pixelRatio;\n    this.colorPickerCanvas.height = 289 * this.pixelRatio;\n  }\n\n\n  /**\n   * create all dom elements\n   * TODO: cleanup, lots of similar dom elements\n   * @private\n   */\n  _create() {\n    this.frame = document.createElement('div');\n    this.frame.className = 'vis-color-picker';\n\n    this.colorPickerDiv = document.createElement('div');\n    this.colorPickerSelector = document.createElement('div');\n    this.colorPickerSelector.className = 'vis-selector';\n    this.colorPickerDiv.appendChild(this.colorPickerSelector);\n\n    this.colorPickerCanvas = document.createElement('canvas');\n    this.colorPickerDiv.appendChild(this.colorPickerCanvas);\n\n    if (!this.colorPickerCanvas.getContext) {\n      let noCanvas = document.createElement( 'DIV' );\n      noCanvas.style.color = 'red';\n      noCanvas.style.fontWeight =  'bold' ;\n      noCanvas.style.padding =  '10px';\n      noCanvas.innerHTML =  'Error: your browser does not support HTML canvas';\n      this.colorPickerCanvas.appendChild(noCanvas);\n    }\n    else {\n      let ctx = this.colorPickerCanvas.getContext(\"2d\");\n      this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n      ctx.mozBackingStorePixelRatio ||\n      ctx.msBackingStorePixelRatio ||\n      ctx.oBackingStorePixelRatio ||\n      ctx.backingStorePixelRatio || 1);\n      this.colorPickerCanvas.getContext(\"2d\").setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n    }\n\n    this.colorPickerDiv.className = 'vis-color';\n\n    this.opacityDiv = document.createElement('div');\n    this.opacityDiv.className = 'vis-opacity';\n\n    this.brightnessDiv = document.createElement('div');\n    this.brightnessDiv.className = 'vis-brightness';\n\n    this.arrowDiv = document.createElement('div');\n    this.arrowDiv.className = 'vis-arrow';\n\n    this.opacityRange = document.createElement('input');\n    try {\n      this.opacityRange.type = 'range'; // Not supported on IE9\n      this.opacityRange.min = '0';\n      this.opacityRange.max = '100';\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    this.opacityRange.value = '100';\n    this.opacityRange.className = 'vis-range';\n\n    this.brightnessRange = document.createElement('input');\n    try {\n      this.brightnessRange.type = 'range'; // Not supported on IE9\n      this.brightnessRange.min = '0';\n      this.brightnessRange.max = '100';\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    this.brightnessRange.value = '100';\n    this.brightnessRange.className = 'vis-range';\n\n    this.opacityDiv.appendChild(this.opacityRange);\n    this.brightnessDiv.appendChild(this.brightnessRange);\n\n    var me = this;\n    this.opacityRange.onchange = function () {me._setOpacity(this.value);};\n    this.opacityRange.oninput  = function () {me._setOpacity(this.value);};\n    this.brightnessRange.onchange = function () {me._setBrightness(this.value);};\n    this.brightnessRange.oninput  = function () {me._setBrightness(this.value);};\n\n    this.brightnessLabel = document.createElement(\"div\");\n    this.brightnessLabel.className = \"vis-label vis-brightness\";\n    this.brightnessLabel.innerHTML = 'brightness:';\n\n    this.opacityLabel = document.createElement(\"div\");\n    this.opacityLabel.className = \"vis-label vis-opacity\";\n    this.opacityLabel.innerHTML = 'opacity:';\n\n    this.newColorDiv = document.createElement(\"div\");\n    this.newColorDiv.className = \"vis-new-color\";\n    this.newColorDiv.innerHTML = 'new';\n\n    this.initialColorDiv = document.createElement(\"div\");\n    this.initialColorDiv.className = \"vis-initial-color\";\n    this.initialColorDiv.innerHTML = 'initial';\n\n    this.cancelButton = document.createElement(\"div\");\n    this.cancelButton.className = \"vis-button vis-cancel\";\n    this.cancelButton.innerHTML = 'cancel';\n    this.cancelButton.onclick = this._hide.bind(this, false);\n\n    this.applyButton = document.createElement(\"div\");\n    this.applyButton.className = \"vis-button vis-apply\";\n    this.applyButton.innerHTML = 'apply';\n    this.applyButton.onclick = this._apply.bind(this);\n\n    this.saveButton = document.createElement(\"div\");\n    this.saveButton.className = \"vis-button vis-save\";\n    this.saveButton.innerHTML = 'save';\n    this.saveButton.onclick = this._save.bind(this);\n\n    this.loadButton = document.createElement(\"div\");\n    this.loadButton.className = \"vis-button vis-load\";\n    this.loadButton.innerHTML = 'load last';\n    this.loadButton.onclick = this._loadLast.bind(this);\n\n    this.frame.appendChild(this.colorPickerDiv);\n    this.frame.appendChild(this.arrowDiv);\n    this.frame.appendChild(this.brightnessLabel);\n    this.frame.appendChild(this.brightnessDiv);\n    this.frame.appendChild(this.opacityLabel);\n    this.frame.appendChild(this.opacityDiv);\n    this.frame.appendChild(this.newColorDiv);\n    this.frame.appendChild(this.initialColorDiv);\n\n    this.frame.appendChild(this.cancelButton);\n    this.frame.appendChild(this.applyButton);\n    this.frame.appendChild(this.saveButton);\n    this.frame.appendChild(this.loadButton);\n  }\n\n\n  /**\n   * bind hammer to the color picker\n   * @private\n   */\n  _bindHammer() {\n    this.drag = {};\n    this.pinch = {};\n    this.hammer = new Hammer(this.colorPickerCanvas);\n    this.hammer.get('pinch').set({enable: true});\n\n    hammerUtil.onTouch(this.hammer, (event) => {this._moveSelector(event)});\n    this.hammer.on('tap',       (event) => {this._moveSelector(event)});\n    this.hammer.on('panstart',  (event) => {this._moveSelector(event)});\n    this.hammer.on('panmove',   (event) => {this._moveSelector(event)});\n    this.hammer.on('panend',    (event) => {this._moveSelector(event)});\n  }\n\n\n  /**\n   * generate the hue circle. This is relatively heavy (200ms) and is done only once on the first time it is shown.\n   * @private\n   */\n  _generateHueCircle() {\n    if (this.generated === false) {\n      let ctx = this.colorPickerCanvas.getContext('2d');\n      if (this.pixelRation === undefined) {\n        this.pixelRatio = (window.devicePixelRatio || 1) / (ctx.webkitBackingStorePixelRatio ||\n        ctx.mozBackingStorePixelRatio ||\n        ctx.msBackingStorePixelRatio ||\n        ctx.oBackingStorePixelRatio ||\n        ctx.backingStorePixelRatio || 1);\n      }\n      ctx.setTransform(this.pixelRatio, 0, 0, this.pixelRatio, 0, 0);\n\n      // clear the canvas\n      let w = this.colorPickerCanvas.clientWidth;\n      let h = this.colorPickerCanvas.clientHeight;\n      ctx.clearRect(0, 0, w, h);\n\n\n      // draw hue circle\n      let x, y, hue, sat;\n      this.centerCoordinates = {x: w * 0.5, y: h * 0.5};\n      this.r = 0.49 * w;\n      let angleConvert = (2 * Math.PI) / 360;\n      let hfac = 1 / 360;\n      let sfac = 1 / this.r;\n      let rgb;\n      for (hue = 0; hue < 360; hue++) {\n        for (sat = 0; sat < this.r; sat++) {\n          x = this.centerCoordinates.x + sat * Math.sin(angleConvert * hue);\n          y = this.centerCoordinates.y + sat * Math.cos(angleConvert * hue);\n          rgb = util.HSVToRGB(hue * hfac, sat * sfac, 1);\n          ctx.fillStyle = 'rgb(' + rgb.r + ',' + rgb.g + ',' + rgb.b + ')';\n          ctx.fillRect(x - 0.5, y - 0.5, 2, 2);\n        }\n      }\n      ctx.strokeStyle = 'rgba(0,0,0,1)';\n      ctx.circle(this.centerCoordinates.x, this.centerCoordinates.y, this.r);\n      ctx.stroke();\n\n      this.hueCircle = ctx.getImageData(0,0,w,h);\n    }\n    this.generated = true;\n  }\n\n\n  /**\n   * move the selector. This is called by hammer functions.\n   *\n   * @param {Event}  event   The event\n   * @private\n   */\n  _moveSelector(event) {\n    let rect = this.colorPickerDiv.getBoundingClientRect();\n    let left = event.center.x - rect.left;\n    let top = event.center.y - rect.top;\n\n    let centerY = 0.5 * this.colorPickerDiv.clientHeight;\n    let centerX = 0.5 * this.colorPickerDiv.clientWidth;\n\n    let x = left - centerX;\n    let y = top - centerY;\n\n    let angle = Math.atan2(x,y);\n    let radius = 0.98 * Math.min(Math.sqrt(x * x + y * y), centerX);\n\n    let newTop = Math.cos(angle) * radius + centerY;\n    let newLeft = Math.sin(angle) * radius + centerX;\n\n    this.colorPickerSelector.style.top = newTop - 0.5 * this.colorPickerSelector.clientHeight + 'px';\n    this.colorPickerSelector.style.left = newLeft - 0.5 * this.colorPickerSelector.clientWidth + 'px';\n\n    // set color\n    let h = angle / (2 * Math.PI);\n    h = h < 0 ? h + 1 : h;\n    let s = radius / this.r;\n    let hsv = util.RGBToHSV(this.color.r, this.color.g, this.color.b);\n    hsv.h = h;\n    hsv.s = s;\n    let rgba = util.HSVToRGB(hsv.h, hsv.s, hsv.v);\n    rgba['a'] = this.color.a;\n    this.color = rgba;\n\n    // update previews\n    this.initialColorDiv.style.backgroundColor = 'rgba(' + this.initialColor.r + ',' + this.initialColor.g + ',' + this.initialColor.b + ',' + this.initialColor.a + ')';\n    this.newColorDiv.style.backgroundColor = 'rgba(' + this.color.r + ',' + this.color.g + ',' + this.color.b + ',' + this.color.a + ')';\n  }\n}\n\nexport default ColorPicker;\n","import util from '../util';\nimport ColorPicker from './ColorPicker';\n\nimport './configuration.css';\n\n/**\n * The way this works is for all properties of this.possible options, you can supply the property name in any form to list the options.\n * Boolean options are recognised as Boolean\n * Number options should be written as array: [default value, min value, max value, stepsize]\n * Colors should be written as array: ['color', '#ffffff']\n * Strings with should be written as array: [option1, option2, option3, ..]\n *\n * The options are matched with their counterparts in each of the modules and the values used in the configuration are\n */\nclass Configurator {\n  /**\n   * @param {Object} parentModule        | the location where parentModule.setOptions() can be called\n   * @param {Object} defaultContainer    | the default container of the module\n   * @param {Object} configureOptions    | the fully configured and predefined options set found in allOptions.js\n   * @param {number} pixelRatio          | canvas pixel ratio\n   */\n  constructor(parentModule, defaultContainer, configureOptions, pixelRatio = 1) {\n    this.parent = parentModule;\n    this.changedOptions = [];\n    this.container = defaultContainer;\n    this.allowCreation = false;\n\n    this.options = {};\n    this.initialized = false;\n    this.popupCounter = 0;\n    this.defaultOptions = {\n      enabled: false,\n      filter: true,\n      container: undefined,\n      showButton: true\n    };\n    util.extend(this.options, this.defaultOptions);\n\n    this.configureOptions = configureOptions;\n    this.moduleOptions = {};\n    this.domElements = [];\n    this.popupDiv = {};\n    this.popupLimit = 5;\n    this.popupHistory = {};\n    this.colorPicker = new ColorPicker(pixelRatio);\n    this.wrapper = undefined;\n  }\n\n\n  /**\n   * refresh all options.\n   * Because all modules parse their options by themselves, we just use their options. We copy them here.\n   *\n   * @param {Object} options\n   */\n  setOptions(options) {\n    if (options !== undefined) {\n      // reset the popup history because the indices may have been changed.\n      this.popupHistory = {};\n      this._removePopup();\n\n      let enabled = true;\n      if (typeof options === 'string') {\n        this.options.filter = options;\n      }\n      else if (options instanceof Array) {\n        this.options.filter = options.join();\n      }\n      else if (typeof options === 'object') {\n        if (options == null) {\n          throw new TypeError('options cannot be null');\n        }\n        if (options.container !== undefined) {\n          this.options.container = options.container;\n        }\n        if (options.filter !== undefined) {\n          this.options.filter = options.filter;\n        }\n        if (options.showButton !== undefined) {\n          this.options.showButton = options.showButton;\n        }\n        if (options.enabled !== undefined) {\n          enabled = options.enabled;\n        }\n      }\n      else if (typeof options === 'boolean') {\n        this.options.filter = true;\n        enabled = options;\n      }\n      else if (typeof options === 'function') {\n        this.options.filter = options;\n        enabled = true;\n      }\n      if (this.options.filter === false) {\n        enabled = false;\n      }\n\n      this.options.enabled = enabled;\n    }\n    this._clean();\n  }\n\n  /**\n   *\n   * @param {Object} moduleOptions\n   */\n  setModuleOptions(moduleOptions) {\n    this.moduleOptions = moduleOptions;\n    if (this.options.enabled === true) {\n      this._clean();\n      if (this.options.container !== undefined) {\n        this.container = this.options.container;\n      }\n      this._create();\n    }\n  }\n\n  /**\n   * Create all DOM elements\n   * @private\n   */\n  _create() {\n    this._clean();\n    this.changedOptions = [];\n\n    let filter = this.options.filter;\n    let counter = 0;\n    let show = false;\n    for (let option in this.configureOptions) {\n      if (this.configureOptions.hasOwnProperty(option)) {\n        this.allowCreation = false;\n        show = false;\n        if (typeof filter === 'function') {\n          show = filter(option,[]);\n          show = show || this._handleObject(this.configureOptions[option], [option], true);\n        }\n        else if (filter === true || filter.indexOf(option) !== -1) {\n          show = true;\n        }\n\n        if (show !== false) {\n          this.allowCreation = true;\n\n          // linebreak between categories\n          if (counter > 0) {\n            this._makeItem([]);\n          }\n          // a header for the category\n          this._makeHeader(option);\n\n          // get the sub options\n          this._handleObject(this.configureOptions[option], [option]);\n        }\n        counter++;\n      }\n    }\n    this._makeButton();\n    this._push();\n    //~ this.colorPicker.insertTo(this.container);\n  }\n\n\n  /**\n   * draw all DOM elements on the screen\n   * @private\n   */\n  _push() {\n    this.wrapper = document.createElement('div');\n    this.wrapper.className = 'vis-configuration-wrapper';\n    this.container.appendChild(this.wrapper);\n    for (var i = 0; i < this.domElements.length; i++) {\n      this.wrapper.appendChild(this.domElements[i]);\n    }\n\n    this._showPopupIfNeeded()\n  }\n\n\n  /**\n   * delete all DOM elements\n   * @private\n   */\n  _clean() {\n    for (var i = 0; i < this.domElements.length; i++) {\n      this.wrapper.removeChild(this.domElements[i]);\n    }\n\n    if (this.wrapper !== undefined) {\n      this.container.removeChild(this.wrapper);\n      this.wrapper = undefined;\n    }\n    this.domElements = [];\n\n    this._removePopup();\n  }\n\n\n  /**\n   * get the value from the actualOptions if it exists\n   * @param {array} path    | where to look for the actual option\n   * @returns {*}\n   * @private\n   */\n  _getValue(path) {\n    let base = this.moduleOptions;\n    for (let i = 0; i < path.length; i++) {\n      if (base[path[i]] !== undefined) {\n        base = base[path[i]];\n      }\n      else {\n        base = undefined;\n        break;\n      }\n    }\n    return base;\n  }\n\n\n  /**\n   * all option elements are wrapped in an item\n   * @param {Array} path    | where to look for the actual option\n   * @param {Array.<Element>} domElements\n   * @returns {number}\n   * @private\n   */\n  _makeItem(path, ...domElements) {\n    if (this.allowCreation === true) {\n      let item = document.createElement('div');\n      item.className = 'vis-configuration vis-config-item vis-config-s' + path.length;\n      domElements.forEach((element) => {\n        item.appendChild(element);\n      });\n      this.domElements.push(item);\n      return this.domElements.length;\n    }\n    return 0;\n  }\n\n\n  /**\n   * header for major subjects\n   * @param {string} name\n   * @private\n   */\n  _makeHeader(name) {\n    let div = document.createElement('div');\n    div.className = 'vis-configuration vis-config-header';\n    div.innerHTML = name;\n    this._makeItem([],div);\n  }\n\n\n  /**\n   * make a label, if it is an object label, it gets different styling.\n   * @param {string} name\n   * @param {array} path    | where to look for the actual option\n   * @param {string} objectLabel\n   * @returns {HTMLElement}\n   * @private\n   */\n  _makeLabel(name, path, objectLabel = false) {\n    let div = document.createElement('div');\n    div.className = 'vis-configuration vis-config-label vis-config-s' + path.length;\n    if (objectLabel === true) {\n      div.innerHTML = '<i><b>' + name + ':</b></i>';\n    }\n    else {\n      div.innerHTML = name + ':';\n    }\n    return div;\n  }\n\n\n  /**\n   * make a dropdown list for multiple possible string optoins\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeDropdown(arr, value, path) {\n    let select = document.createElement('select');\n    select.className = 'vis-configuration vis-config-select';\n    let selectedValue = 0;\n    if (value !== undefined) {\n      if (arr.indexOf(value) !== -1) {\n        selectedValue = arr.indexOf(value);\n      }\n    }\n\n    for (let i = 0; i < arr.length; i++) {\n      let option = document.createElement('option');\n      option.value = arr[i];\n      if (i === selectedValue) {\n        option.selected = 'selected';\n      }\n      option.innerHTML = arr[i];\n      select.appendChild(option);\n    }\n\n    let me = this;\n    select.onchange = function () {me._update(this.value, path);};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, select);\n  }\n\n\n  /**\n   * make a range object for numeric options\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeRange(arr, value, path) {\n    let defaultValue = arr[0];\n    let min = arr[1];\n    let max = arr[2];\n    let step = arr[3];\n    let range = document.createElement('input');\n    range.className = 'vis-configuration vis-config-range';\n    try {\n      range.type = 'range'; // not supported on IE9\n      range.min = min;\n      range.max = max;\n    }\n    // TODO: Add some error handling and remove this lint exception\n    catch (err) {}  // eslint-disable-line no-empty\n    range.step = step;\n\n    // set up the popup settings in case they are needed.\n    let popupString = '';\n    let popupValue = 0;\n\n    if (value !== undefined) {\n      let factor = 1.20;\n      if (value < 0 && value * factor < min) {\n        range.min = Math.ceil(value * factor);\n        popupValue = range.min;\n        popupString = 'range increased';\n      }\n      else if (value / factor < min) {\n        range.min = Math.ceil(value / factor);\n        popupValue = range.min;\n        popupString = 'range increased';\n      }\n      if (value * factor > max && max !== 1) {\n        range.max = Math.ceil(value * factor);\n        popupValue = range.max;\n        popupString = 'range increased';\n      }\n      range.value = value;\n    }\n    else {\n      range.value = defaultValue;\n    }\n\n    let input = document.createElement('input');\n    input.className = 'vis-configuration vis-config-rangeinput';\n    input.value = Number(range.value);\n\n    var me = this;\n    range.onchange = function () {input.value = this.value; me._update(Number(this.value), path);};\n    range.oninput  = function () {input.value = this.value; };\n\n    let label = this._makeLabel(path[path.length-1], path);\n    let itemIndex = this._makeItem(path, label, range, input);\n\n    // if a popup is needed AND it has not been shown for this value, show it.\n    if (popupString !== '' && this.popupHistory[itemIndex] !== popupValue) {\n      this.popupHistory[itemIndex] = popupValue;\n      this._setupPopup(popupString, itemIndex);\n    }\n  }\n\n  /**\n   * make a button object\n   * @private\n   */\n  _makeButton() {\n    if (this.options.showButton === true) {\n      let generateButton = document.createElement('div');\n      generateButton.className = 'vis-configuration vis-config-button';\n      generateButton.innerHTML = 'generate options';\n      generateButton.onclick =     () => {this._printOptions();};\n      generateButton.onmouseover = () => {generateButton.className = 'vis-configuration vis-config-button hover';};\n      generateButton.onmouseout =  () => {generateButton.className = 'vis-configuration vis-config-button';};\n\n      this.optionsContainer = document.createElement('div');\n      this.optionsContainer.className = 'vis-configuration vis-config-option-container';\n\n      this.domElements.push(this.optionsContainer);\n      this.domElements.push(generateButton);\n    }\n  }\n\n\n  /**\n   * prepare the popup\n   * @param {string} string\n   * @param {number} index\n   * @private\n   */\n  _setupPopup(string, index) {\n    if (this.initialized === true && this.allowCreation === true && this.popupCounter < this.popupLimit) {\n      let div = document.createElement(\"div\");\n      div.id = \"vis-configuration-popup\";\n      div.className = \"vis-configuration-popup\";\n      div.innerHTML = string;\n      div.onclick = () => {this._removePopup()};\n      this.popupCounter += 1;\n      this.popupDiv = {html:div, index:index};\n    }\n  }\n\n\n  /**\n   * remove the popup from the dom\n   * @private\n   */\n  _removePopup() {\n    if (this.popupDiv.html !== undefined) {\n      this.popupDiv.html.parentNode.removeChild(this.popupDiv.html);\n      clearTimeout(this.popupDiv.hideTimeout);\n      clearTimeout(this.popupDiv.deleteTimeout);\n      this.popupDiv = {};\n    }\n  }\n\n\n  /**\n   * Show the popup if it is needed.\n   * @private\n   */\n  _showPopupIfNeeded() {\n    if (this.popupDiv.html !== undefined) {\n      let correspondingElement = this.domElements[this.popupDiv.index];\n      let rect = correspondingElement.getBoundingClientRect();\n      this.popupDiv.html.style.left = rect.left + \"px\";\n      this.popupDiv.html.style.top = rect.top - 30 + \"px\"; // 30 is the height;\n      document.body.appendChild(this.popupDiv.html)\n      this.popupDiv.hideTimeout = setTimeout(() => {\n        this.popupDiv.html.style.opacity = 0;\n      },1500);\n      this.popupDiv.deleteTimeout = setTimeout(() => {\n        this._removePopup();\n      },1800)\n    }\n  }\n\n  /**\n   * make a checkbox for boolean options.\n   * @param {number} defaultValue\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeCheckbox(defaultValue, value, path) {\n    var checkbox = document.createElement('input');\n    checkbox.type = 'checkbox';\n    checkbox.className = 'vis-configuration vis-config-checkbox';\n    checkbox.checked = defaultValue;\n    if (value !== undefined) {\n      checkbox.checked = value;\n      if (value !== defaultValue) {\n        if (typeof defaultValue === 'object') {\n          if (value !== defaultValue.enabled) {\n            this.changedOptions.push({path:path, value:value});\n          }\n        }\n        else {\n          this.changedOptions.push({path:path, value:value});\n        }\n      }\n    }\n\n    let me = this;\n    checkbox.onchange = function() {me._update(this.checked, path)};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, checkbox);\n  }\n\n  /**\n   * make a text input field for string options.\n   * @param {number} defaultValue\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeTextInput(defaultValue, value, path) {\n    var checkbox = document.createElement('input');\n    checkbox.type = 'text';\n    checkbox.className = 'vis-configuration vis-config-text';\n    checkbox.value = value;\n    if (value !== defaultValue) {\n      this.changedOptions.push({path:path, value:value});\n    }\n\n    let me = this;\n    checkbox.onchange = function() {me._update(this.value, path)};\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path, label, checkbox);\n  }\n\n\n  /**\n   * make a color field with a color picker for color fields\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _makeColorField(arr, value, path) {\n    let defaultColor = arr[1];\n    let div = document.createElement('div');\n    value = value === undefined ? defaultColor : value;\n\n    if (value !== 'none') {\n      div.className = 'vis-configuration vis-config-colorBlock';\n      div.style.backgroundColor = value;\n    }\n    else {\n      div.className = 'vis-configuration vis-config-colorBlock none';\n    }\n\n    value = value === undefined ? defaultColor : value;\n    div.onclick = () => {\n      this._showColorPicker(value,div,path);\n    };\n\n    let label = this._makeLabel(path[path.length-1], path);\n    this._makeItem(path,label, div);\n  }\n\n\n  /**\n   * used by the color buttons to call the color picker.\n   * @param {number} value\n   * @param {HTMLElement} div\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _showColorPicker(value, div, path) {\n    // clear the callback from this div\n    div.onclick = function() {};\n\n    this.colorPicker.insertTo(div);\n    this.colorPicker.show();\n\n    this.colorPicker.setColor(value);\n    this.colorPicker.setUpdateCallback((color) => {\n      let colorString = 'rgba(' + color.r + ',' + color.g + ',' + color.b + ',' + color.a + ')';\n      div.style.backgroundColor = colorString;\n      this._update(colorString,path);\n    });\n\n    // on close of the colorpicker, restore the callback.\n    this.colorPicker.setCloseCallback(() => {\n      div.onclick = () => {\n        this._showColorPicker(value,div,path);\n      };\n    });\n  }\n\n\n  /**\n   * parse an object and draw the correct items\n   * @param {Object} obj\n   * @param {array} [path=[]]    | where to look for the actual option\n   * @param {boolean} [checkOnly=false]\n   * @returns {boolean}\n   * @private\n   */\n  _handleObject(obj, path = [], checkOnly = false) {\n    let show = false;\n    let filter = this.options.filter;\n    let visibleInSet = false;\n    for (let subObj in obj) {\n      if (obj.hasOwnProperty(subObj)) {\n        show = true;\n        let item = obj[subObj];\n        let newPath = util.copyAndExtendArray(path, subObj);\n        if (typeof filter === 'function') {\n          show = filter(subObj,path);\n\n          // if needed we must go deeper into the object.\n          if (show === false) {\n            if (!(item instanceof Array) && typeof item !== 'string' && typeof item !== 'boolean' && item instanceof Object) {\n              this.allowCreation = false;\n              show = this._handleObject(item, newPath, true);\n              this.allowCreation = checkOnly === false;\n            }\n          }\n        }\n\n        if (show !== false) {\n          visibleInSet = true;\n          let value = this._getValue(newPath);\n\n          if (item instanceof Array) {\n            this._handleArray(item, value, newPath);\n          }\n          else if (typeof item === 'string') {\n            this._makeTextInput(item, value, newPath);\n          }\n          else if (typeof item === 'boolean') {\n            this._makeCheckbox(item, value, newPath);\n          }\n          else if (item instanceof Object) {\n            // collapse the physics options that are not enabled\n            let draw = true;\n            if (path.indexOf('physics') !== -1) {\n              if (this.moduleOptions.physics.solver !== subObj) {\n                draw = false;\n              }\n            }\n\n            if (draw === true) {\n              // initially collapse options with an disabled enabled option.\n              if (item.enabled !== undefined) {\n                let enabledPath = util.copyAndExtendArray(newPath, 'enabled');\n                let enabledValue = this._getValue(enabledPath);\n                if (enabledValue === true) {\n                  let label = this._makeLabel(subObj, newPath, true);\n                  this._makeItem(newPath, label);\n                  visibleInSet = this._handleObject(item, newPath) || visibleInSet;\n                }\n                else {\n                  this._makeCheckbox(item, enabledValue, newPath);\n                }\n              }\n              else {\n                let label = this._makeLabel(subObj, newPath, true);\n                this._makeItem(newPath, label);\n                visibleInSet = this._handleObject(item, newPath) || visibleInSet;\n              }\n            }\n          }\n          else {\n            console.error('dont know how to handle', item, subObj, newPath);\n          }\n        }\n      }\n    }\n    return visibleInSet;\n  }\n\n\n  /**\n   * handle the array type of option\n   * @param {Array.<number>} arr\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _handleArray(arr, value, path) {\n    if (typeof arr[0] === 'string' && arr[0] === 'color') {\n      this._makeColorField(arr, value, path);\n      if (arr[1] !== value) {this.changedOptions.push({path:path, value:value});}\n    }\n    else if (typeof arr[0] === 'string') {\n      this._makeDropdown(arr, value, path);\n      if (arr[0] !== value) {this.changedOptions.push({path:path, value:value});}\n    }\n    else if (typeof arr[0] === 'number') {\n      this._makeRange(arr, value, path);\n      if (arr[0] !== value) {this.changedOptions.push({path:path, value:Number(value)});}\n    }\n  }\n\n\n\n  /**\n   * called to update the network with the new settings.\n   * @param {number} value\n   * @param {array} path    | where to look for the actual option\n   * @private\n   */\n  _update(value, path) {\n    let options = this._constructOptions(value,path);\n\n    if (this.parent.body && this.parent.body.emitter && this.parent.body.emitter.emit) {\n      this.parent.body.emitter.emit(\"configChange\", options);\n    }\n    this.initialized = true;\n    this.parent.setOptions(options);\n  }\n\n\n  /**\n   *\n   * @param {string|Boolean} value\n   * @param {Array.<string>} path\n   * @param {{}} optionsObj\n   * @returns {{}}\n   * @private\n   */\n  _constructOptions(value, path, optionsObj = {}) {\n    let pointer = optionsObj;\n\n    // when dropdown boxes can be string or boolean, we typecast it into correct types\n    value = value === 'true'  ? true  : value;\n    value = value === 'false' ? false : value;\n\n    for (let i = 0; i < path.length; i++) {\n      if (path[i] !== 'global') {\n        if (pointer[path[i]] === undefined) {\n          pointer[path[i]] = {};\n        }\n        if (i !== path.length - 1) {\n          pointer = pointer[path[i]];\n        }\n        else {\n          pointer[path[i]] = value;\n        }\n      }\n    }\n    return optionsObj;\n  }\n\n  /**\n   * @private\n   */\n  _printOptions() {\n    let options = this.getOptions();\n    this.optionsContainer.innerHTML = '<pre>var options = ' + JSON.stringify(options, null, 2) + '</pre>';\n  }\n\n  /**\n   *\n   * @returns {{}} options\n   */\n  getOptions() {\n    let options = {};\n    for (var i = 0; i < this.changedOptions.length; i++) {\n      this._constructOptions(this.changedOptions[i].value, this.changedOptions[i].path, options)\n    }\n    return options;\n  }\n}\n\n\nexport default Configurator;\n","import moment from '../module/moment';\nimport util, { typeCoerceDataSet } from '../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Range from './Range';\nimport Core from './Core';\nimport TimeAxis from './component/TimeAxis';\nimport CurrentTime from './component/CurrentTime';\nimport CustomTime from './component/CustomTime';\nimport ItemSet from './component/ItemSet';\n\nimport { Validator } from '../shared/Validator';\nimport { printStyle } from '../shared/Validator';\nimport { allOptions } from './optionsTimeline';\nimport { configureOptions } from './optionsTimeline';\n\nimport Configurator from '../shared/Configurator';\n\n/**\n * Create a timeline visualization\n * @extends Core\n */\nexport default class Timeline extends Core {\n  /**\n * @param {HTMLElement} container\n * @param {vis.DataSet | vis.DataView | Array} [items]\n * @param {vis.DataSet | vis.DataView | Array} [groups]\n * @param {Object} [options]  See Timeline.setOptions for the available options.\n * @constructor Timeline\n */\n  constructor(container, items, groups, options) {\n    super()\n    this.initTime = new Date();\n    this.itemsDone = false;\n\n    if (!(this instanceof Timeline)) {\n      throw new SyntaxError('Constructor must be called with the new operator');\n    }\n\n    // if the third element is options, the forth is groups (optionally);\n    if (!(Array.isArray(groups) || groups instanceof DataSet || groups instanceof DataView) && groups instanceof Object) {\n      const forthArgument = options;\n      options = groups;\n      groups = forthArgument;\n    }\n\n    // TODO: REMOVE THIS in the next MAJOR release\n    // see https://github.com/almende/vis/issues/2511\n    if (options && options.throttleRedraw) {\n      console.warn(\"Timeline option \\\"throttleRedraw\\\" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.\");\n    }\n\n    const me = this;\n    this.defaultOptions = {\n      autoResize: true,\n      longSelectPressTime: 251,\n      orientation: {\n        axis: 'bottom',   // axis orientation: 'bottom', 'top', or 'both'\n        item: 'bottom'    // not relevant\n      },\n      moment,\n    };\n    this.options = util.deepExtend({}, this.defaultOptions);\n\n    // Create the DOM, props, and emitter\n    this._create(container);\n    if (!options || (options && typeof options.rtl == \"undefined\")) {\n      this.dom.root.style.visibility = 'hidden';\n      let directionFromDom;\n      let domNode = this.dom.root;\n      while (!directionFromDom && domNode) {\n        directionFromDom = window.getComputedStyle(domNode, null).direction;\n        domNode = domNode.parentElement;\n      }\n      this.options.rtl = (directionFromDom && (directionFromDom.toLowerCase() == \"rtl\"));\n    } else {\n      this.options.rtl = options.rtl;\n    }\n\n    if (options) {\n      if (options.rollingMode) { this.options.rollingMode = options.rollingMode; }\n      if (options.onInitialDrawComplete) { this.options.onInitialDrawComplete = options.onInitialDrawComplete; }\n      if (options.onTimeout) { this.options.onTimeout = options.onTimeout; }\n      if (options.loadingScreenTemplate) { this.options.loadingScreenTemplate = options.loadingScreenTemplate; }\n    }\n\n    // Prepare loading screen\n    const loadingScreenFragment = document.createElement('div');\n    if (this.options.loadingScreenTemplate) {\n      const templateFunction = this.options.loadingScreenTemplate.bind(this);\n      const loadingScreen = templateFunction(this.dom.loadingScreen);\n      if ((loadingScreen instanceof Object) && !(loadingScreen instanceof Element)) {\n        templateFunction(loadingScreenFragment)\n      } else {\n        if (loadingScreen instanceof Element) {\n          loadingScreenFragment.innerHTML = '';\n          loadingScreenFragment.appendChild(loadingScreen);\n        }\n        else if (loadingScreen != undefined) {\n          loadingScreenFragment.innerHTML = loadingScreen;\n        }\n      }\n    }\n    this.dom.loadingScreen.appendChild(loadingScreenFragment);\n\n    // all components listed here will be repainted automatically\n    this.components = [];\n\n    this.body = {\n      dom: this.dom,\n      domProps: this.props,\n      emitter: {\n        on: this.on.bind(this),\n        off: this.off.bind(this),\n        emit: this.emit.bind(this)\n      },\n      hiddenDates: [],\n      util: {\n        getScale() {\n          return me.timeAxis.step.scale;\n        },\n        getStep() {\n          return me.timeAxis.step.step;\n        },\n\n        toScreen: me._toScreen.bind(me),\n        toGlobalScreen: me._toGlobalScreen.bind(me), // this refers to the root.width\n        toTime: me._toTime.bind(me),\n        toGlobalTime : me._toGlobalTime.bind(me)\n      }\n    };\n\n    // range\n    this.range = new Range(this.body, this.options);\n    this.components.push(this.range);\n    this.body.range = this.range;\n\n    // time axis\n    this.timeAxis = new TimeAxis(this.body, this.options);\n    this.timeAxis2 = null; // used in case of orientation option 'both'\n    this.components.push(this.timeAxis);\n\n    // current time bar\n    this.currentTime = new CurrentTime(this.body, this.options);\n    this.components.push(this.currentTime);\n\n    // item set\n    this.itemSet = new ItemSet(this.body, this.options);\n    this.components.push(this.itemSet);\n\n    this.itemsData = null;      // DataSet\n    this.groupsData = null;     // DataSet\n\n    this.dom.root.onclick = event => {\n      me.emit('click', me.getEventProperties(event))\n    };\n    this.dom.root.ondblclick = event => {\n      me.emit('doubleClick', me.getEventProperties(event))\n    };\n    this.dom.root.oncontextmenu = event => {\n      me.emit('contextmenu', me.getEventProperties(event))\n    };\n    this.dom.root.onmouseover = event => {\n      me.emit('mouseOver', me.getEventProperties(event))\n    };\n    if(window.PointerEvent) {\n      this.dom.root.onpointerdown = event => {\n        me.emit('mouseDown', me.getEventProperties(event))\n      };\n      this.dom.root.onpointermove = event => {\n        me.emit('mouseMove', me.getEventProperties(event))\n      };\n      this.dom.root.onpointerup = event => {\n        me.emit('mouseUp', me.getEventProperties(event))\n      };\n    } else {\n      this.dom.root.onmousemove = event => {\n        me.emit('mouseMove', me.getEventProperties(event))\n      };\n      this.dom.root.onmousedown = event => {\n        me.emit('mouseDown', me.getEventProperties(event))\n      };\n      this.dom.root.onmouseup = event => {\n        me.emit('mouseUp', me.getEventProperties(event))\n      };\n    }\n\n    //Single time autoscale/fit\n    this.initialFitDone = false;\n    this.on('changed', () => {\n      if (me.itemsData == null) return;\n      if (!me.initialFitDone && !me.options.rollingMode) {\n        me.initialFitDone = true;\n        if (me.options.start != undefined || me.options.end != undefined) {\n          if (me.options.start == undefined || me.options.end == undefined) {\n            var range = me.getItemRange();\n          }\n\n          const start = me.options.start != undefined ? me.options.start : range.min;\n          const end   = me.options.end   != undefined ? me.options.end   : range.max;\n          me.setWindow(start, end, {animation: false});\n        } else {\n          me.fit({animation: false});\n        }\n      }\n\n      if (!me.initialDrawDone && (me.initialRangeChangeDone || (!me.options.start && !me.options.end) \n        || me.options.rollingMode)) {\n        me.initialDrawDone = true;\n        me.itemSet.initialDrawDone = true;\n        me.dom.root.style.visibility = 'visible';\n        me.dom.loadingScreen.parentNode.removeChild(me.dom.loadingScreen);\n        if (me.options.onInitialDrawComplete) {\n          setTimeout(() => {\n            return me.options.onInitialDrawComplete();\n          }, 0)\n        }\n      }\n    });\n\n    this.on('destroyTimeline', () => {\n      me.destroy()\n    });\n\n    // apply options\n    if (options) {\n      this.setOptions(options);\n    }\n\n    this.body.emitter.on('fit', (args) => {\n      this._onFit(args);\n      this.redraw();\n    });\n\n    // IMPORTANT: THIS HAPPENS BEFORE SET ITEMS!\n    if (groups) {\n      this.setGroups(groups);\n    }\n\n    // create itemset\n    if (items) {\n      this.setItems(items);\n    }\n\n    // draw for the first time\n    this._redraw();\n  }\n\n  /**\n   * Load a configurator\n   * @return {Object}\n   * @private\n   */\n  _createConfigurator() {\n    return new Configurator(this, this.dom.container, configureOptions);\n  }\n\n  /**\n   * Force a redraw. The size of all items will be recalculated.\n   * Can be useful to manually redraw when option autoResize=false and the window\n   * has been resized, or when the items CSS has been changed.\n   *\n   * Note: this function will be overridden on construction with a trottled version\n   */\n  redraw() {\n    this.itemSet && this.itemSet.markDirty({refreshItems: true});\n    this._redraw();\n  }\n\n  /**\n   * Remove an item from the group\n   * @param {object} options\n   */\n  setOptions(options) {\n    // validate options\n    let errorFound = Validator.validate(options, allOptions);\n\n    if (errorFound === true) {\n      console.log('%cErrors have been found in the supplied options object.', printStyle);\n    }\n\n    Core.prototype.setOptions.call(this, options);\n\n    if ('type' in options) {\n      if (options.type !== this.options.type) {\n        this.options.type = options.type;\n\n        // force recreation of all items\n        const itemsData = this.itemsData;\n        if (itemsData) {\n          const selection = this.getSelection();\n          this.setItems(null);            // remove all\n          this.setItems(itemsData.rawDS); // add all\n          this.setSelection(selection);   // restore selection\n        }\n      }\n    }\n  }\n\n  /**\n   * Set items\n   * @param {vis.DataSet | Array | null} items\n   */\n  setItems(items) {\n    this.itemsDone = false;\n    \n    // convert to type DataSet when needed\n    let newDataSet;\n    if (!items) {\n      newDataSet = null;\n    }\n    else if (items instanceof DataSet || items instanceof DataView) {\n      newDataSet = typeCoerceDataSet(items);\n    }\n    else {\n      // turn an array into a dataset\n      newDataSet = typeCoerceDataSet(new DataSet(items));\n    }\n\n    // set items\n    if (this.itemsData) {\n      // stop maintaining a coerced version of the old data set\n      this.itemsData.dispose();\n    }\n    this.itemsData = newDataSet;\n    this.itemSet && this.itemSet.setItems(newDataSet != null ? newDataSet.rawDS : null);\n  }\n\n  /**\n   * Set groups\n   * @param {vis.DataSet | Array} groups\n   */\n  setGroups(groups) {\n    // convert to type DataSet when needed\n    let newDataSet;\n    if (!groups) {\n      newDataSet = null;\n    }\n    else {\n      const filter = group => group.visible !== false;\n      if (groups instanceof DataSet || groups instanceof DataView) {\n        newDataSet = new DataView(groups,{filter});\n      }\n      else {\n        // turn an array into a dataset\n        newDataSet = new DataSet(groups.filter(filter));\n      }\n    }\n\n    // This looks weird but it's necessary to prevent memory leaks.\n    //\n    // The problem is that the DataView will exist as long as the DataSet it's\n    // connected to. This will force it to swap the groups DataSet for it's own\n    // DataSet. In this arrangement it will become unreferenced from the outside\n    // and garbage collected.\n    //\n    // IMPORTANT NOTE: If `this.groupsData` is a DataView was created in this\n    // method. Even if the original is a DataView already a new one has been\n    // created and assigned to `this.groupsData`. In case this changes in the\n    // future it will be necessary to rework this!!!!\n    if (this.groupsData instanceof DataView) {\n      this.groupsData.setData(null);\n    }\n    this.groupsData = newDataSet;\n    this.itemSet.setGroups(newDataSet);\n  }\n\n  /**\n   * Set both items and groups in one go\n   * @param {{items: (Array | vis.DataSet), groups: (Array | vis.DataSet)}} data\n   */\n  setData(data) {\n    if (data && data.groups) {\n      this.setGroups(data.groups);\n    }\n\n    if (data && data.items) {\n      this.setItems(data.items);\n    }\n  }\n\n  /**\n   * Set selected items by their id. Replaces the current selection\n   * Unknown id's are silently ignored.\n   * @param {string[] | string} [ids]  An array with zero or more id's of the items to be\n   *                                selected. If ids is an empty array, all items will be\n   *                                unselected.\n   * @param {Object} [options]      Available options:\n   *                                `focus: boolean`\n   *                                    If true, focus will be set to the selected item(s)\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                                    Only applicable when option focus is true.\n   */\n  setSelection(ids, options) {\n    this.itemSet && this.itemSet.setSelection(ids);\n\n    if (options && options.focus) {\n      this.focus(ids, options);\n    }\n  }\n\n  /**\n   * Get the selected items by their id\n   * @return {Array} ids  The ids of the selected items\n   */\n  getSelection() {\n    return this.itemSet && this.itemSet.getSelection() || [];\n  }\n\n  /**\n   * Adjust the visible window such that the selected item (or multiple items)\n   * are centered on screen.\n   * @param {string | String[]} id     An item id or array with item ids\n   * @param {Object} [options]      Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   *                                `zoom: boolean`\n   *                                    If true (default), the timeline will\n   *                                    zoom on the element after focus it.\n   */\n  focus(id, options) {\n    if (!this.itemsData || id == undefined) return;\n\n    const ids = Array.isArray(id) ? id : [id];\n\n    // get the specified item(s)\n    const itemsData = this.itemsData.get(ids);\n\n    // calculate minimum start and maximum end of specified items\n    let start = null;\n    let end = null;\n    itemsData.forEach(itemData => {\n      const s = itemData.start.valueOf();\n      const e = 'end' in itemData ? itemData.end.valueOf() : itemData.start.valueOf();\n\n      if (start === null || s < start) {\n        start = s;\n      }\n\n      if (end === null || e > end) {\n        end = e;\n      }\n    });\n\n\n    if (start !== null && end !== null) {\n      const me = this;\n      // Use the first item for the vertical focus\n      const item = this.itemSet.items[ids[0]];\n      let startPos = this._getScrollTop() * -1;\n      let initialVerticalScroll = null;\n\n      // Setup a handler for each frame of the vertical scroll\n      const verticalAnimationFrame = (ease, willDraw, done) => {\n        const verticalScroll = getItemVerticalScroll(me, item);\n\n        if (verticalScroll === false) {\n          return; // We don't need to scroll, so do nothing\n        }\n\n        if(!initialVerticalScroll) {\n          initialVerticalScroll = verticalScroll;\n        }\n\n        if(initialVerticalScroll.itemTop == verticalScroll.itemTop && !initialVerticalScroll.shouldScroll) {\n          return; // We don't need to scroll, so do nothing\n        }\n        else if(initialVerticalScroll.itemTop != verticalScroll.itemTop && verticalScroll.shouldScroll) {\n          // The redraw shifted elements, so reset the animation to correct\n          initialVerticalScroll = verticalScroll;\n          startPos = me._getScrollTop() * -1;\n        }      \n\n        const from = startPos;\n        const to = initialVerticalScroll.scrollOffset;\n        const scrollTop = done ? to : (from + (to - from) * ease);\n\n        me._setScrollTop(-scrollTop);\n\n        if(!willDraw) {\n          me._redraw();\n        }\n      };\n\n      // Enforces the final vertical scroll position\n      const setFinalVerticalPosition = () => {\n        const finalVerticalScroll = getItemVerticalScroll(me, item);\n\n        if (finalVerticalScroll.shouldScroll && finalVerticalScroll.itemTop != initialVerticalScroll.itemTop) {\n          me._setScrollTop(-finalVerticalScroll.scrollOffset);\n          me._redraw();\n        }\n      };\n\n      // Perform one last check at the end to make sure the final vertical\n      // position is correct\n      const finalVerticalCallback = () => {\n        // Double check we ended at the proper scroll position\n        setFinalVerticalPosition();\n\n        // Let the redraw settle and finalize the position.      \n        setTimeout(setFinalVerticalPosition, 100);\n      };\n\n      // calculate the new middle and interval for the window\n      const zoom = options && options.zoom !== undefined ? options.zoom : true;\n      const middle = (start + end) / 2;\n      const interval = zoom ? (end - start) * 1.1 : Math.max(this.range.end - this.range.start, (end - start) * 1.1);\n\n      const animation = options && options.animation !== undefined ? options.animation : true;\n\n      if (!animation) {\n        // We aren't animating so set a default so that the final callback forces the vertical location\n        initialVerticalScroll = { shouldScroll: false, scrollOffset: -1, itemTop: -1 };\n      }\n\n      this.range.setRange(middle - interval / 2, middle + interval / 2, { animation }, finalVerticalCallback, verticalAnimationFrame);  \n    }\n  }\n\n  /**\n   * Set Timeline window such that it fits all items\n   * @param {Object} [options]  Available options:\n   *                                `animation: boolean | {duration: number, easingFunction: string}`\n   *                                    If true (default), the range is animated\n   *                                    smoothly to the new window. An object can be\n   *                                    provided to specify duration and easing function.\n   *                                    Default duration is 500 ms, and default easing\n   *                                    function is 'easeInOutQuad'.\n   * @param {function} [callback]\n   */\n  fit(options, callback) {\n    const animation = (options && options.animation !== undefined) ? options.animation : true;\n    let range;\n\n    if (this.itemsData.length === 1 && this.itemsData.get()[0].end === undefined) {\n      // a single item -> don't fit, just show a range around the item from -4 to +3 days\n      range = this.getDataRange();\n      this.moveTo(range.min.valueOf(), {animation}, callback);\n    }\n    else {\n      // exactly fit the items (plus a small margin)\n      range = this.getItemRange();\n      this.range.setRange(range.min, range.max, { animation }, callback);\n    }\n  }\n\n  /**\n   * Determine the range of the items, taking into account their actual width\n   * and a margin of 10 pixels on both sides.\n   *\n   * @returns {{min: Date, max: Date}}\n   */\n  getItemRange() {\n    // get a rough approximation for the range based on the items start and end dates\n    const range = this.getDataRange();\n    let min = range.min !== null ? range.min.valueOf() : null;\n    let max = range.max !== null ? range.max.valueOf() : null;\n    let minItem = null;\n    let maxItem = null;\n\n    if (min != null && max != null) {\n      let interval = (max - min); // ms\n      if (interval <= 0) {\n        interval = 10;\n      }\n      const factor = interval / this.props.center.width;\n\n      const redrawQueue = {};\n      let redrawQueueLength = 0;\n\n      // collect redraw functions\n      util.forEach(this.itemSet.items, (item, key) => {\n        if (item.groupShowing) {\n          const returnQueue = true;\n          redrawQueue[key] = item.redraw(returnQueue);\n          redrawQueueLength = redrawQueue[key].length;\n        }\n      })\n\n      const needRedraw = redrawQueueLength > 0;\n      if (needRedraw) {\n        // redraw all regular items\n        for (let i = 0; i < redrawQueueLength; i++) {\n          util.forEach(redrawQueue, fns => {\n            fns[i]();\n          });\n        }\n      }\n\n       // calculate the date of the left side and right side of the items given\n      util.forEach(this.itemSet.items, item => {\n        const start = getStart(item);\n        const end = getEnd(item);\n        let startSide;\n        let endSide;\n\n        if (this.options.rtl) {\n          startSide  = start - (item.getWidthRight()  + 10) * factor;\n          endSide = end   + (item.getWidthLeft() + 10) * factor;\n        } else {\n          startSide  = start - (item.getWidthLeft()  + 10) * factor;\n          endSide = end   + (item.getWidthRight() + 10) * factor;\n        }\n\n        if (startSide < min) {\n          min = startSide;\n          minItem = item;\n        }\n        if (endSide > max) {\n          max = endSide;\n          maxItem = item;\n        }\n      });\n\n      if (minItem && maxItem) {\n        const lhs = minItem.getWidthLeft() + 10;\n        const rhs = maxItem.getWidthRight() + 10;\n        const delta = this.props.center.width - lhs - rhs;  // px\n\n        if (delta > 0) {\n          if (this.options.rtl) {\n            min = getStart(minItem) - rhs * interval / delta; // ms\n            max = getEnd(maxItem)   + lhs * interval / delta; // ms\n          } else {\n            min = getStart(minItem) - lhs * interval / delta; // ms\n            max = getEnd(maxItem)   + rhs * interval / delta; // ms\n          }\n        }\n      }\n    }\n\n    return {\n      min: min != null ? new Date(min) : null,\n      max: max != null ? new Date(max) : null\n    }\n  }\n\n  /**\n   * Calculate the data range of the items start and end dates\n   * @returns {{min: Date, max: Date}}\n   */\n  getDataRange() {\n    let min = null;\n    let max = null;\n\n    if (this.itemsData) {\n      this.itemsData.forEach(item => {\n        const start = util.convert(item.start, 'Date').valueOf();\n        const end   = util.convert(item.end != undefined ? item.end : item.start, 'Date').valueOf();\n        if (min === null || start < min) {\n          min = start;\n        }\n        if (max === null || end > max) {\n          max = end;\n        }\n      });\n    }\n\n    return {\n      min: min != null ? new Date(min) : null,\n      max: max != null ? new Date(max) : null\n    }\n  }\n\n  /**\n   * Generate Timeline related information from an event\n   * @param {Event} event\n   * @return {Object} An object with related information, like on which area\n   *                  The event happened, whether clicked on an item, etc.\n   */\n  getEventProperties(event) {\n    const clientX = event.center ? event.center.x : event.clientX;\n    const clientY = event.center ? event.center.y : event.clientY;\n    const centerContainerRect = this.dom.centerContainer.getBoundingClientRect();\n    const x = this.options.rtl ? centerContainerRect.right - clientX : clientX - centerContainerRect.left;\n    const y = clientY - centerContainerRect.top;\n    \n    const item  = this.itemSet.itemFromTarget(event);\n    const group = this.itemSet.groupFromTarget(event);\n    const customTime = CustomTime.customTimeFromTarget(event);\n\n    const snap = this.itemSet.options.snap || null;\n    const scale = this.body.util.getScale();\n    const step = this.body.util.getStep();\n    const time = this._toTime(x);\n    const snappedTime = snap ? snap(time, scale, step) : time;\n\n    const element = util.getTarget(event);\n    let what = null;\n    if (item != null)                                                    {what = 'item';}\n    else if (customTime != null)                                         {what = 'custom-time';}\n    else if (util.hasParent(element, this.timeAxis.dom.foreground))      {what = 'axis';}\n    else if (this.timeAxis2 && util.hasParent(element, this.timeAxis2.dom.foreground)) {what = 'axis';}\n    else if (util.hasParent(element, this.itemSet.dom.labelSet))         {what = 'group-label';}\n    else if (util.hasParent(element, this.currentTime.bar))              {what = 'current-time';}\n    else if (util.hasParent(element, this.dom.center))                   {what = 'background';}\n\n    return {\n      event,\n      item: item ? item.id : null,\n      isCluster: item ? !!item.isCluster: false,\n      items: item ? item.items || []: null,\n      group: group ? group.groupId : null,\n      customTime: customTime ? customTime.options.id : null,\n      what,\n      pageX: event.srcEvent ? event.srcEvent.pageX : event.pageX,\n      pageY: event.srcEvent ? event.srcEvent.pageY : event.pageY,\n      x,\n      y,\n      time,\n      snappedTime\n    }\n  }\n\n  /**\n   * Toggle Timeline rolling mode\n   */\n  toggleRollingMode() {\n    if (this.range.rolling) {\n      this.range.stopRolling();\n    } else {\n      if (this.options.rollingMode == undefined) {\n        this.setOptions(this.options)\n      }\n      this.range.startRolling();\n    }\n  }\n\n  /**\n   * redraw\n   * @private\n   */\n  _redraw() {\n    Core.prototype._redraw.call(this);\n  }\n\n  /**\n   * on fit callback\n   * @param {object} args\n   * @private\n   */\n  _onFit(args) {\n    const { start, end, animation } = args;\n    if (!end) {\n      this.moveTo(start.valueOf(), {\n        animation\n      });\n    } else {\n      this.range.setRange(start, end, {\n        animation: animation\n      });\n    }\n  }\n}\n\n/**\n *\n * @param {timeline.Item} item\n * @returns {number}\n */\nfunction getStart(item) {\n  return util.convert(item.data.start, 'Date').valueOf()\n}\n\n/**\n *\n * @param {timeline.Item} item\n * @returns {number}\n */\nfunction getEnd(item) {\n  const end = item.data.end != undefined ? item.data.end : item.data.start;\n  return util.convert(end, 'Date').valueOf();\n}\n\n/**\n * @param {vis.Timeline} timeline\n * @param {timeline.Item} item\n * @return {{shouldScroll: bool, scrollOffset: number, itemTop: number}}\n */\nfunction getItemVerticalScroll(timeline, item) {\n  if (!item.parent) {\n    // The item no longer exists, so ignore this focus.\n    return false;\n  }\n\n  const itemsetHeight = timeline.options.rtl ? timeline.props.rightContainer.height : timeline.props.leftContainer.height;\n  const contentHeight = timeline.props.center.height;\n  \n  const group = item.parent;\n  let offset = group.top;\n  let shouldScroll = true;\n  const orientation = timeline.timeAxis.options.orientation.axis;\n  \n  const itemTop = () => {\n  if (orientation == \"bottom\") {\n      return group.height - item.top - item.height;\n    }\n    else {\n      return item.top;\n    }\n  };\n\n  const currentScrollHeight = timeline._getScrollTop() * -1;\n  const targetOffset = offset + itemTop();\n  const height = item.height;\n\n  if (targetOffset < currentScrollHeight) {\n    if (offset + itemsetHeight <= offset + itemTop() + height) {\n      offset += itemTop() - timeline.itemSet.options.margin.item.vertical;\n    }\n  }\n  else if (targetOffset + height > currentScrollHeight + itemsetHeight) {\n    offset += itemTop() + height - itemsetHeight + timeline.itemSet.options.margin.item.vertical;\n  }\n  else {\n    shouldScroll = false;\n  }\n\n  offset = Math.min(offset, contentHeight - itemsetHeight);\n\n  return { shouldScroll, scrollOffset: offset, itemTop: targetOffset };\n}\n","/** DataScale */\nclass DataScale {\n  /**\n   *\n   * @param {number} start\n   * @param {number} end\n   * @param {boolean} autoScaleStart\n   * @param {boolean} autoScaleEnd\n   * @param {number} containerHeight\n   * @param {number} majorCharHeight\n   * @param {boolean} zeroAlign\n   * @param {function} formattingFunction\n   * @constructor DataScale\n   */\n  constructor(\n    start,\n    end,\n    autoScaleStart,\n    autoScaleEnd,\n    containerHeight,\n    majorCharHeight,\n    zeroAlign = false,\n    formattingFunction=false) {\n    this.majorSteps = [1, 2, 5, 10];\n    this.minorSteps = [0.25, 0.5, 1, 2];\n    this.customLines = null;\n\n    this.containerHeight = containerHeight;\n    this.majorCharHeight = majorCharHeight;\n    this._start = start;\n    this._end = end;\n\n    this.scale = 1;\n    this.minorStepIdx = -1;\n    this.magnitudefactor = 1;\n    this.determineScale();\n\n    this.zeroAlign = zeroAlign;\n    this.autoScaleStart = autoScaleStart;\n    this.autoScaleEnd = autoScaleEnd;\n\n    this.formattingFunction = formattingFunction;\n\n    if (autoScaleStart || autoScaleEnd) {\n      const me = this;\n      const roundToMinor = value => {\n        const rounded = value - (value % (me.magnitudefactor * me.minorSteps[me.minorStepIdx]));\n        if (value % (me.magnitudefactor * me.minorSteps[me.minorStepIdx]) > 0.5 * (me.magnitudefactor * me.minorSteps[me.minorStepIdx])) {\n          return rounded + (me.magnitudefactor * me.minorSteps[me.minorStepIdx]);\n        }\n        else {\n          return rounded;\n        }\n      };\n      if (autoScaleStart) {\n        this._start -= this.magnitudefactor * 2 * this.minorSteps[this.minorStepIdx];\n        this._start = roundToMinor(this._start);\n      }\n\n      if (autoScaleEnd) {\n        this._end += this.magnitudefactor * this.minorSteps[this.minorStepIdx];\n        this._end = roundToMinor(this._end);\n      }\n      this.determineScale();\n    }\n  }\n\n  /**\n   * set chart height\n   * @param {number} majorCharHeight \n   */\n  setCharHeight(majorCharHeight) {\n    this.majorCharHeight = majorCharHeight;\n  }\n\n  /**\n   * set height\n   * @param {number} containerHeight \n   */\n  setHeight(containerHeight) {\n    this.containerHeight = containerHeight;\n  }\n\n  /**\n   * determine scale\n   */\n  determineScale() {\n    const range = this._end - this._start;\n    this.scale = this.containerHeight / range;\n    const minimumStepValue = this.majorCharHeight / this.scale;\n    const orderOfMagnitude = (range > 0)\n        ? Math.round(Math.log(range) / Math.LN10)\n        : 0;\n\n    this.minorStepIdx = -1;\n    this.magnitudefactor = Math.pow(10, orderOfMagnitude);\n\n    let start = 0;\n    if (orderOfMagnitude < 0) {\n      start = orderOfMagnitude;\n    }\n\n    let solutionFound = false;\n    for (let l = start; Math.abs(l) <= Math.abs(orderOfMagnitude); l++) {\n      this.magnitudefactor = Math.pow(10, l);\n      for (let j = 0; j < this.minorSteps.length; j++) {\n        const stepSize = this.magnitudefactor * this.minorSteps[j];\n        if (stepSize >= minimumStepValue) {\n          solutionFound = true;\n          this.minorStepIdx = j;\n          break;\n        }\n      }\n      if (solutionFound === true) {\n        break;\n      }\n    }\n  }\n\n  /**\n   * returns if value is major\n   * @param {number} value\n   * @returns {boolean} \n   */\n  is_major(value) {\n    return (value % (this.magnitudefactor * this.majorSteps[this.minorStepIdx]) === 0);\n  }\n\n  /**\n   * returns step size\n   * @returns {number} \n   */\n  getStep() {\n    return this.magnitudefactor * this.minorSteps[this.minorStepIdx];\n  }\n\n  /**\n   * returns first major\n   * @returns {number} \n   */\n  getFirstMajor() {\n    const majorStep = this.magnitudefactor * this.majorSteps[this.minorStepIdx];\n    return this.convertValue(this._start + ((majorStep - (this._start % majorStep)) % majorStep));\n  }\n\n  /**\n   * returns first major\n   * @param {date} current\n   * @returns {date} formatted date\n   */\n  formatValue(current) {\n    let returnValue = current.toPrecision(5);\n    if (typeof this.formattingFunction === 'function') {\n      returnValue = this.formattingFunction(current);\n    }\n\n    if (typeof returnValue === 'number') {\n      return `${returnValue}`;\n    }\n    else if (typeof returnValue === 'string') {\n      return returnValue;\n    }\n    else {\n      return current.toPrecision(5);\n    }\n\n  }\n\n  /**\n   * returns lines\n   * @returns {object} lines\n   */\n  getLines() {\n    const lines = [];\n    const step = this.getStep();\n    const bottomOffset = (step - (this._start % step)) % step;\n    for (let i = (this._start + bottomOffset); this._end-i > 0.00001; i += step) {\n      if (i != this._start) { //Skip the bottom line\n        lines.push({major: this.is_major(i), y: this.convertValue(i), val: this.formatValue(i)});\n      }\n    }\n    return lines;\n  }\n\n  /**\n   * follow scale\n   * @param {object} other\n   */\n  followScale(other) {\n    const oldStepIdx = this.minorStepIdx;\n    const oldStart = this._start;\n    const oldEnd = this._end;\n\n    const me = this;\n    const increaseMagnitude = () => {\n      me.magnitudefactor *= 2;\n    };\n    const decreaseMagnitude = () => {\n      me.magnitudefactor /= 2;\n    };\n\n    if ((other.minorStepIdx <= 1 && this.minorStepIdx <= 1) || (other.minorStepIdx > 1 && this.minorStepIdx > 1)) {\n      //easy, no need to change stepIdx nor multiplication factor\n    } else if (other.minorStepIdx < this.minorStepIdx) {\n      //I'm 5, they are 4 per major.\n      this.minorStepIdx = 1;\n      if (oldStepIdx == 2) {\n        increaseMagnitude();\n      } else {\n        increaseMagnitude();\n        increaseMagnitude();\n      }\n    } else {\n      //I'm 4, they are 5 per major\n      this.minorStepIdx = 2;\n      if (oldStepIdx == 1) {\n        decreaseMagnitude();\n      } else {\n        decreaseMagnitude();\n        decreaseMagnitude();\n      }\n    }\n\n    //Get masters stats:\n    const otherZero = other.convertValue(0);\n    const otherStep = other.getStep() * other.scale;\n\n    let done = false;\n    let count = 0;\n    //Loop until magnitude is correct for given constrains.\n    while (!done && count++ <5) {\n\n      //Get my stats:\n      this.scale = otherStep / (this.minorSteps[this.minorStepIdx] * this.magnitudefactor);\n      const newRange = this.containerHeight / this.scale;\n\n      //For the case the magnitudefactor has changed:\n      this._start = oldStart;\n      this._end = this._start + newRange;\n\n      const myOriginalZero = this._end * this.scale;\n      const majorStep = this.magnitudefactor * this.majorSteps[this.minorStepIdx];\n      const majorOffset = this.getFirstMajor() - other.getFirstMajor();\n\n      if (this.zeroAlign) {\n        const zeroOffset = otherZero - myOriginalZero;\n        this._end += (zeroOffset / this.scale);\n        this._start = this._end - newRange;\n      } else {\n        if (!this.autoScaleStart) {\n          this._start += majorStep - (majorOffset / this.scale);\n          this._end = this._start + newRange;\n        } else {\n          this._start -= majorOffset / this.scale;\n          this._end = this._start + newRange;\n        }\n      }\n      if (!this.autoScaleEnd && this._end > oldEnd+0.00001) {\n        //Need to decrease magnitude to prevent scale overshoot! (end)\n        decreaseMagnitude();\n        done = false;\n        continue;\n      }\n      if (!this.autoScaleStart && this._start < oldStart-0.00001) {\n        if (this.zeroAlign && oldStart >= 0) {\n          console.warn(\"Can't adhere to given 'min' range, due to zeroalign\");\n        } else {\n          //Need to decrease magnitude to prevent scale overshoot! (start)\n          decreaseMagnitude();\n          done = false;\n          continue;\n        }\n      }\n      if (this.autoScaleStart && this.autoScaleEnd && newRange < (oldEnd-oldStart)){\n        increaseMagnitude();\n        done = false;\n        continue;\n      }\n      done = true;\n    }\n  }\n\n  /**\n   * convert value\n   * @param {number} value\n   * @returns {number} \n   */\n  convertValue(value) {\n    return this.containerHeight - ((value - this._start) * this.scale);\n  }\n\n  /**\n   * returns screen to value\n   * @param {number} pixels\n   * @returns {number} \n   */\n  screenToValue(pixels) {\n    return ((this.containerHeight - pixels) / this.scale) + this._start;\n  }\n}\n\nexport default DataScale;","import { v4 as randomUUID } from \"uuid\";import util from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport Component from './Component';\nimport DataScale from './DataScale';\n\nimport './css/dataaxis.css';\n\n/** A horizontal time axis */\nclass DataAxis extends Component {\n  /**\n * @param {Object} body\n * @param {Object} [options]        See DataAxis.setOptions for the available\n *                                  options.\n * @param {SVGElement} svg\n * @param {timeline.LineGraph.options} linegraphOptions\n * @constructor DataAxis\n * @extends Component\n */\n  constructor(body, options, svg, linegraphOptions) {\n    super()\n    this.id = randomUUID();\n    this.body = body;\n\n    this.defaultOptions = {\n      orientation: 'left',  // supported: 'left', 'right'\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      icons: false,\n      majorLinesOffset: 7,\n      minorLinesOffset: 4,\n      labelOffsetX: 10,\n      labelOffsetY: 2,\n      iconWidth: 20,\n      width: '40px',\n      visible: true,\n      alignZeros: true,\n      left: {\n        range: {min: undefined, max: undefined},\n        format(value) {\n          return `${parseFloat(value.toPrecision(3))}`;\n        },\n        title: {text: undefined, style: undefined}\n      },\n      right: {\n        range: {min: undefined, max: undefined},\n        format(value) {\n          return `${parseFloat(value.toPrecision(3))}`;\n        },\n        title: {text: undefined, style: undefined}\n      }\n    };\n\n    this.linegraphOptions = linegraphOptions;\n    this.linegraphSVG = svg;\n    this.props = {};\n    this.DOMelements = { // dynamic elements\n      lines: {},\n      labels: {},\n      title: {}\n    };\n\n    this.dom = {};\n    this.scale = undefined;\n    this.range = {start: 0, end: 0};\n\n    this.options = util.extend({}, this.defaultOptions);\n    this.conversionFactor = 1;\n\n    this.setOptions(options);\n    this.width = Number((`${this.options.width}`).replace(\"px\", \"\"));\n    this.minWidth = this.width;\n    this.height = this.linegraphSVG.getBoundingClientRect().height;\n    this.hidden = false;\n\n    this.stepPixels = 25;\n    this.zeroCrossing = -1;\n    this.amountOfSteps = -1;\n\n    this.lineOffset = 0;\n    this.master = true;\n    this.masterAxis = null;\n    this.svgElements = {};\n    this.iconsRemoved = false;\n\n    this.groups = {};\n    this.amountOfGroups = 0;\n\n    // create the HTML DOM\n    this._create();\n    if (this.scale == undefined) {\n      this._redrawLabels();\n    }\n    this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n\n    const me = this;\n    this.body.emitter.on(\"verticalDrag\", () => {\n      me.dom.lineContainer.style.top = `${me.body.domProps.scrollTop}px`;\n    });\n  }\n\n  /**\n   * Adds group to data axis\n   * @param {string} label \n   * @param {object} graphOptions\n   */\n  addGroup(label, graphOptions) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.groups[label] = graphOptions;\n    }\n    this.amountOfGroups += 1;\n  }\n\n  /**\n   * updates group of data axis\n   * @param {string} label \n   * @param {object} graphOptions\n   */\n  updateGroup(label, graphOptions) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.amountOfGroups += 1;\n    }\n    this.groups[label] = graphOptions;\n  }\n\n  /**\n   * removes group of data axis\n   * @param {string} label \n   */\n  removeGroup(label) {\n    if (this.groups.hasOwnProperty(label)) {\n      delete this.groups[label];\n      this.amountOfGroups -= 1;\n    }\n  }\n\n  /**\n   * sets options\n   * @param {object} options\n   */\n  setOptions(options) {\n    if (options) {\n      let redraw = false;\n      if (this.options.orientation != options.orientation && options.orientation !== undefined) {\n        redraw = true;\n      }\n      const fields = [\n        'orientation',\n        'showMinorLabels',\n        'showMajorLabels',\n        'icons',\n        'majorLinesOffset',\n        'minorLinesOffset',\n        'labelOffsetX',\n        'labelOffsetY',\n        'iconWidth',\n        'width',\n        'visible',\n        'left',\n        'right',\n        'alignZeros'\n      ];\n      util.selectiveDeepExtend(fields, this.options, options);\n\n      this.minWidth = Number((`${this.options.width}`).replace(\"px\", \"\"));\n      if (redraw === true && this.dom.frame) {\n        this.hide();\n        this.show();\n      }\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  _create() {\n    this.dom.frame = document.createElement('div');\n    this.dom.frame.style.width = this.options.width;\n    this.dom.frame.style.height = this.height;\n\n    this.dom.lineContainer = document.createElement('div');\n    this.dom.lineContainer.style.width = '100%';\n    this.dom.lineContainer.style.height = this.height;\n    this.dom.lineContainer.style.position = 'relative';\n    this.dom.lineContainer.style.visibility = 'visible';\n    this.dom.lineContainer.style.display = 'block';\n\n    // create svg element for graph drawing.\n    this.svg = document.createElementNS('http://www.w3.org/2000/svg', \"svg\");\n    this.svg.style.position = \"absolute\";\n    this.svg.style.top = '0px';\n    this.svg.style.height = '100%';\n    this.svg.style.width = '100%';\n    this.svg.style.display = \"block\";\n    this.dom.frame.appendChild(this.svg);\n  }\n\n  /**\n   * redraws groups icons\n   */\n  _redrawGroupIcons() {\n    DOMutil.prepareElements(this.svgElements);\n\n    let x;\n    const iconWidth = this.options.iconWidth;\n    const iconHeight = 15;\n    const iconOffset = 4;\n    let y = iconOffset + 0.5 * iconHeight;\n\n    if (this.options.orientation === 'left') {\n      x = iconOffset;\n    }\n    else {\n      x = this.width - iconWidth - iconOffset;\n    }\n\n    const groupArray = Object.keys(this.groups);\n    groupArray.sort((a, b) => a < b ? -1 : 1)\n\n    for (const groupId of groupArray) {\n      if (this.groups[groupId].visible === true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] === true)) {\n        this.groups[groupId].getLegend(iconWidth, iconHeight, this.framework, x, y);\n        y += iconHeight + iconOffset;\n      }\n    }\n\n    DOMutil.cleanupElements(this.svgElements);\n    this.iconsRemoved = false;\n  }\n\n  /**\n   * Cleans up icons\n   */\n  _cleanupIcons() {\n    if (this.iconsRemoved === false) {\n      DOMutil.prepareElements(this.svgElements);\n      DOMutil.cleanupElements(this.svgElements);\n      this.iconsRemoved = true;\n    }\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  show() {\n    this.hidden = false;\n    if (!this.dom.frame.parentNode) {\n      if (this.options.orientation === 'left') {\n        this.body.dom.left.appendChild(this.dom.frame);\n      }\n      else {\n        this.body.dom.right.appendChild(this.dom.frame);\n      }\n    }\n\n    if (!this.dom.lineContainer.parentNode) {\n      this.body.dom.backgroundHorizontal.appendChild(this.dom.lineContainer);\n    }\n    this.dom.lineContainer.style.display = 'block';\n  }\n\n  /**\n   * Create the HTML DOM for the DataAxis\n   */\n  hide() {\n    this.hidden = true;\n    if (this.dom.frame.parentNode) {\n      this.dom.frame.parentNode.removeChild(this.dom.frame);\n    }\n\n    this.dom.lineContainer.style.display = 'none';\n  }\n\n  /**\n   * Set a range (start and end)\n   * @param {number} start\n   * @param {number} end\n   */\n  setRange(start, end) {\n    this.range.start = start;\n    this.range.end = end;\n  }\n\n  /**\n   * Repaint the component\n   * @return {boolean} Returns true if the component is resized\n   */\n  redraw() {\n    let resized = false;\n    let activeGroups = 0;\n\n    // Make sure the line container adheres to the vertical scrolling.\n    this.dom.lineContainer.style.top = `${this.body.domProps.scrollTop}px`;\n\n    for (const groupId in this.groups) {\n      if (this.groups.hasOwnProperty(groupId)) {\n        if (this.groups[groupId].visible === true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] === true)) {\n          activeGroups++;\n        }\n      }\n    }\n    if (this.amountOfGroups === 0 || activeGroups === 0) {\n      this.hide();\n    }\n    else {\n      this.show();\n      this.height = Number(this.linegraphSVG.style.height.replace(\"px\", \"\"));\n\n      // svg offsetheight did not work in firefox and explorer...\n      this.dom.lineContainer.style.height = `${this.height}px`;\n      this.width = this.options.visible === true ? Number((`${this.options.width}`).replace(\"px\", \"\")) : 0;\n\n      const props = this.props;\n      const frame = this.dom.frame;\n\n      // update classname\n      frame.className = 'vis-data-axis';\n\n      // calculate character width and height\n      this._calculateCharSize();\n\n      const orientation = this.options.orientation;\n      const showMinorLabels = this.options.showMinorLabels;\n      const showMajorLabels = this.options.showMajorLabels;\n\n      const backgroundHorizontalOffsetWidth = this.body.dom.backgroundHorizontal.offsetWidth;\n\n      // determine the width and height of the elements for the axis\n      props.minorLabelHeight = showMinorLabels ? props.minorCharHeight : 0;\n      props.majorLabelHeight = showMajorLabels ? props.majorCharHeight : 0;\n\n      props.minorLineWidth = backgroundHorizontalOffsetWidth - this.lineOffset - this.width + 2 * this.options.minorLinesOffset;\n      props.minorLineHeight = 1;\n      props.majorLineWidth = backgroundHorizontalOffsetWidth - this.lineOffset - this.width + 2 * this.options.majorLinesOffset;\n      props.majorLineHeight = 1;\n\n      //  take frame offline while updating (is almost twice as fast)\n      if (orientation === 'left') {\n        frame.style.top = '0';\n        frame.style.left = '0';\n        frame.style.bottom = '';\n        frame.style.width = `${this.width}px`;\n        frame.style.height = `${this.height}px`;\n        this.props.width = this.body.domProps.left.width;\n        this.props.height = this.body.domProps.left.height;\n      }\n      else { // right\n        frame.style.top = '';\n        frame.style.bottom = '0';\n        frame.style.left = '0';\n        frame.style.width = `${this.width}px`;\n        frame.style.height = `${this.height}px`;\n        this.props.width = this.body.domProps.right.width;\n        this.props.height = this.body.domProps.right.height;\n      }\n\n      resized = this._redrawLabels();\n      resized = this._isResized() || resized;\n\n      if (this.options.icons === true) {\n        this._redrawGroupIcons();\n      }\n      else {\n        this._cleanupIcons();\n      }\n\n      this._redrawTitle(orientation);\n    }\n    return resized;\n  }\n\n  /**\n   * Repaint major and minor text labels and vertical grid lines\n   *\n   * @returns {boolean}\n   * @private\n   */\n  _redrawLabels() {\n    let resized = false;\n    DOMutil.prepareElements(this.DOMelements.lines);\n    DOMutil.prepareElements(this.DOMelements.labels);\n    const orientation = this.options['orientation'];\n    const customRange = this.options[orientation].range != undefined ? this.options[orientation].range : {};\n\n    //Override range with manual options:\n    let autoScaleEnd = true;\n    if (customRange.max != undefined) {\n      this.range.end = customRange.max;\n      autoScaleEnd = false;\n    }\n    let autoScaleStart = true;\n    if (customRange.min != undefined) {\n      this.range.start = customRange.min;\n      autoScaleStart = false;\n    }\n\n    this.scale = new DataScale(\n      this.range.start,\n      this.range.end,\n      autoScaleStart,\n      autoScaleEnd,\n      this.dom.frame.offsetHeight,\n      this.props.majorCharHeight,\n      this.options.alignZeros,\n      this.options[orientation].format\n    );\n\n    if (this.master === false && this.masterAxis != undefined) {\n      this.scale.followScale(this.masterAxis.scale);\n      this.dom.lineContainer.style.display = 'none';\n    } else {\n      this.dom.lineContainer.style.display = 'block';\n    }\n\n    //Is updated in side-effect of _redrawLabel():\n    this.maxLabelSize = 0;\n\n    const lines = this.scale.getLines();\n    lines.forEach(\n      line=> {\n        const y = line.y;\n        const isMajor = line.major;\n        if (this.options['showMinorLabels'] && isMajor === false) {\n          this._redrawLabel(y - 2, line.val, orientation, 'vis-y-axis vis-minor', this.props.minorCharHeight);\n        }\n        if (isMajor) {\n          if (y >= 0) {\n            this._redrawLabel(y - 2, line.val, orientation, 'vis-y-axis vis-major', this.props.majorCharHeight);\n          }\n        }\n        if (this.master === true) {\n          if (isMajor) {\n            this._redrawLine(y, orientation, 'vis-grid vis-horizontal vis-major', this.options.majorLinesOffset, this.props.majorLineWidth);\n          }\n          else {\n            this._redrawLine(y, orientation, 'vis-grid vis-horizontal vis-minor', this.options.minorLinesOffset, this.props.minorLineWidth);\n          }\n        }\n      });\n\n    // Note that title is rotated, so we're using the height, not width!\n    let titleWidth = 0;\n    if (this.options[orientation].title !== undefined && this.options[orientation].title.text !== undefined) {\n      titleWidth = this.props.titleCharHeight;\n    }\n    const offset = this.options.icons === true ? Math.max(this.options.iconWidth, titleWidth) + this.options.labelOffsetX + 15 : titleWidth + this.options.labelOffsetX + 15;\n\n    // this will resize the yAxis to accommodate the labels.\n    if (this.maxLabelSize > (this.width - offset) && this.options.visible === true) {\n      this.width = this.maxLabelSize + offset;\n      this.options.width = `${this.width}px`;\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      this.redraw();\n      resized = true;\n    }\n    // this will resize the yAxis if it is too big for the labels.\n    else if (this.maxLabelSize < (this.width - offset) && this.options.visible === true && this.width > this.minWidth) {\n      this.width = Math.max(this.minWidth, this.maxLabelSize + offset);\n      this.options.width = `${this.width}px`;\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      this.redraw();\n      resized = true;\n    }\n    else {\n      DOMutil.cleanupElements(this.DOMelements.lines);\n      DOMutil.cleanupElements(this.DOMelements.labels);\n      resized = false;\n    }\n\n    return resized;\n  }\n\n  /**\n   * converts value\n   * @param {number} value\n   * @returns {number} converted number\n   */\n  convertValue(value) {\n    return this.scale.convertValue(value);\n  }\n\n  /**\n   * converts value\n   * @param {number} x\n   * @returns {number} screen value\n   */\n  screenToValue(x) {\n    return this.scale.screenToValue(x);\n  }\n\n  /**\n   * Create a label for the axis at position x\n   *\n   * @param {number} y\n   * @param {string} text\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   * @param {string} className\n   * @param {number} characterHeight\n   * @private\n   */\n  _redrawLabel(y, text, orientation, className, characterHeight) {\n    // reuse redundant label\n    const label = DOMutil.getDOMElement('div', this.DOMelements.labels, this.dom.frame); //this.dom.redundant.labels.shift();\n    label.className = className;\n    label.innerHTML = text;\n    if (orientation === 'left') {\n      label.style.left = `-${this.options.labelOffsetX}px`;\n      label.style.textAlign = \"right\";\n    }\n    else {\n      label.style.right = `-${this.options.labelOffsetX}px`;\n      label.style.textAlign = \"left\";\n    }\n\n    label.style.top = `${y - 0.5 * characterHeight + this.options.labelOffsetY}px`;\n\n    text += '';\n\n    const largestWidth = Math.max(this.props.majorCharWidth, this.props.minorCharWidth);\n    if (this.maxLabelSize < text.length * largestWidth) {\n      this.maxLabelSize = text.length * largestWidth;\n    }\n  }\n\n  /**\n   * Create a minor line for the axis at position y\n   * @param {number} y\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   * @param {string} className\n   * @param {number} offset\n   * @param {number} width\n   */\n  _redrawLine(y, orientation, className, offset, width) {\n    if (this.master === true) {\n      const line = DOMutil.getDOMElement('div', this.DOMelements.lines, this.dom.lineContainer);  //this.dom.redundant.lines.shift();\n      line.className = className;\n      line.innerHTML = '';\n\n      if (orientation === 'left') {\n        line.style.left = `${this.width - offset}px`;\n      }\n      else {\n        line.style.right = `${this.width - offset}px`;\n      }\n\n      line.style.width = `${width}px`;\n      line.style.top = `${y}px`;\n    }\n  }\n\n  /**\n   * Create a title for the axis\n   * @private\n   * @param {'top'|'right'|'bottom'|'left'} orientation\n   */\n  _redrawTitle(orientation) {\n    DOMutil.prepareElements(this.DOMelements.title);\n\n    // Check if the title is defined for this axes\n    if (this.options[orientation].title !== undefined && this.options[orientation].title.text !== undefined) {\n      const title = DOMutil.getDOMElement('div', this.DOMelements.title, this.dom.frame);\n      title.className = `vis-y-axis vis-title vis-${orientation}`;\n      title.innerHTML = this.options[orientation].title.text;\n\n      // Add style - if provided\n      if (this.options[orientation].title.style !== undefined) {\n        util.addCssText(title, this.options[orientation].title.style);\n      }\n\n      if (orientation === 'left') {\n        title.style.left = `${this.props.titleCharHeight}px`;\n      }\n      else {\n        title.style.right = `${this.props.titleCharHeight}px`;\n      }\n\n      title.style.width = `${this.height}px`;\n    }\n\n    // we need to clean up in case we did not use all elements.\n    DOMutil.cleanupElements(this.DOMelements.title);\n  }\n\n  /**\n   * Determine the size of text on the axis (both major and minor axis).\n   * The size is calculated only once and then cached in this.props.\n   * @private\n   */\n  _calculateCharSize() {\n    // determine the char width and height on the minor axis\n    if (!('minorCharHeight' in this.props)) {\n      const textMinor = document.createTextNode('0');\n      const measureCharMinor = document.createElement('div');\n      measureCharMinor.className = 'vis-y-axis vis-minor vis-measure';\n      measureCharMinor.appendChild(textMinor);\n      this.dom.frame.appendChild(measureCharMinor);\n\n      this.props.minorCharHeight = measureCharMinor.clientHeight;\n      this.props.minorCharWidth = measureCharMinor.clientWidth;\n\n      this.dom.frame.removeChild(measureCharMinor);\n    }\n\n    if (!('majorCharHeight' in this.props)) {\n      const textMajor = document.createTextNode('0');\n      const measureCharMajor = document.createElement('div');\n      measureCharMajor.className = 'vis-y-axis vis-major vis-measure';\n      measureCharMajor.appendChild(textMajor);\n      this.dom.frame.appendChild(measureCharMajor);\n\n      this.props.majorCharHeight = measureCharMajor.clientHeight;\n      this.props.majorCharWidth = measureCharMajor.clientWidth;\n\n      this.dom.frame.removeChild(measureCharMajor);\n    }\n\n    if (!('titleCharHeight' in this.props)) {\n      const textTitle = document.createTextNode('0');\n      const measureCharTitle = document.createElement('div');\n      measureCharTitle.className = 'vis-y-axis vis-title vis-measure';\n      measureCharTitle.appendChild(textTitle);\n      this.dom.frame.appendChild(measureCharTitle);\n\n      this.props.titleCharHeight = measureCharTitle.clientHeight;\n      this.props.titleCharWidth = measureCharTitle.clientWidth;\n\n      this.dom.frame.removeChild(measureCharTitle);\n    }\n  }\n}\n\nexport default DataAxis;\n","import * as DOMutil from '../../../DOMutil';\n\n/**\n *\n * @param {number | string} groupId\n * @param {Object} options   // TODO: Describe options\n *\n * @constructor Points\n */\nfunction Points(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\n/**\n * draw the data points\n *\n * @param {Array} dataset\n * @param {GraphGroup} group\n * @param {Object} framework            | SVG DOM element\n * @param {number} [offset]\n */\nPoints.draw = function (dataset, group, framework, offset) {\n  offset = offset || 0;\n  var callback = getCallback(framework, group);\n\n  for (var i = 0; i < dataset.length; i++) {\n    if (!callback) {\n      // draw the point the simple way.\n      DOMutil.drawPoint(dataset[i].screen_x + offset, dataset[i].screen_y, getGroupTemplate(group), framework.svgElements, framework.svg, dataset[i].label);\n    }\n    else {\n      var callbackResult = callback(dataset[i], group); // result might be true, false or an object\n      if (callbackResult === true || typeof callbackResult === 'object') {\n        DOMutil.drawPoint(dataset[i].screen_x + offset, dataset[i].screen_y, getGroupTemplate(group, callbackResult), framework.svgElements, framework.svg, dataset[i].label);\n      }\n    }\n  }\n};\n\nPoints.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n  var fillHeight = iconHeight * 0.5;\n\n  var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n  outline.setAttributeNS(null, \"x\", x);\n  outline.setAttributeNS(null, \"y\", y - fillHeight);\n  outline.setAttributeNS(null, \"width\", iconWidth);\n  outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n  outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n  //Don't call callback on icon\n  DOMutil.drawPoint(x + 0.5 * iconWidth, y, getGroupTemplate(group), framework.svgElements, framework.svg);\n};\n\n/**\n *\n * @param {vis.Group} group\n * @param {any} callbackResult\n * @returns {{style: *, styles: (*|string), size: *, className: *}}\n */\nfunction getGroupTemplate(group, callbackResult) {\n  callbackResult = (typeof callbackResult === 'undefined') ? {} : callbackResult;\n  return {\n    style: callbackResult.style || group.options.drawPoints.style,\n    styles: callbackResult.styles || group.options.drawPoints.styles,\n    size: callbackResult.size || group.options.drawPoints.size,\n    className: callbackResult.className || group.className\n  };\n}\n\n/**\n *\n * @param {Object} framework            | SVG DOM element\n * @param {vis.Group} group\n * @returns {function}\n */\nfunction getCallback(framework, group) {\n  var callback = undefined;\n  // check for the graph2d onRender\n  if (framework.options && framework.options.drawPoints && framework.options.drawPoints.onRender && typeof framework.options.drawPoints.onRender == 'function') {\n    callback = framework.options.drawPoints.onRender;\n  }\n\n  // override it with the group onRender if defined\n  if (group.group.options && group.group.options.drawPoints && group.group.options.drawPoints.onRender && typeof group.group.options.drawPoints.onRender == 'function') {\n    callback = group.group.options.drawPoints.onRender;\n  }\n  return callback;\n}\n\nexport default Points;\n","import * as DOMutil from '../../../DOMutil';\nimport Points from './points';\n\n/**\n *\n * @param {vis.GraphGroup.id} groupId\n * @param {Object} options   // TODO: Describe options\n * @constructor Bargraph\n */\nfunction Bargraph(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\nBargraph.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n  var fillHeight = iconHeight * 0.5;\n  var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n  outline.setAttributeNS(null, \"x\", x);\n  outline.setAttributeNS(null, \"y\", y - fillHeight);\n  outline.setAttributeNS(null, \"width\", iconWidth);\n  outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n  outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n  var barWidth = Math.round(0.3 * iconWidth);\n  var originalWidth = group.options.barChart.width;\n  var scale = originalWidth / barWidth;\n  var bar1Height = Math.round(0.4 * iconHeight);\n  var bar2Height = Math.round(0.75 * iconHeight);\n\n  var offset = Math.round((iconWidth - (2 * barWidth)) / 3);\n\n  DOMutil.drawBar(x + 0.5 * barWidth + offset, y + fillHeight - bar1Height - 1, barWidth, bar1Height, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n  DOMutil.drawBar(x + 1.5 * barWidth + offset + 2, y + fillHeight - bar2Height - 1, barWidth, bar2Height, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n\n  if (group.options.drawPoints.enabled == true) {\n    var groupTemplate = {\n      style: group.options.drawPoints.style,\n      styles: group.options.drawPoints.styles,\n      size: (group.options.drawPoints.size / scale),\n      className: group.className\n    };\n    DOMutil.drawPoint(x + 0.5 * barWidth + offset, y + fillHeight - bar1Height - 1, groupTemplate, framework.svgElements, framework.svg);\n    DOMutil.drawPoint(x + 1.5 * barWidth + offset + 2, y + fillHeight - bar2Height - 1, groupTemplate, framework.svgElements, framework.svg);\n  }\n};\n\n/**\n * draw a bar graph\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {Object} processedGroupData\n * @param {{svg: Object, svgElements: Array.<Object>, options: Object, groups: Array.<vis.Group>}} framework\n */\nBargraph.draw = function (groupIds, processedGroupData, framework) {\n  var combinedData = [];\n  var intersections = {};\n  var coreDistance;\n  var key, drawData;\n  var group;\n  var i, j;\n  var barPoints = 0;\n\n  // combine all barchart data\n  for (i = 0; i < groupIds.length; i++) {\n    group = framework.groups[groupIds[i]];\n    if (group.options.style === 'bar') {\n      if (group.visible === true && (framework.options.groups.visibility[groupIds[i]] === undefined || framework.options.groups.visibility[groupIds[i]] === true)) {\n        for (j = 0; j < processedGroupData[groupIds[i]].length; j++) {\n          combinedData.push({\n            screen_x: processedGroupData[groupIds[i]][j].screen_x,\n            screen_end: processedGroupData[groupIds[i]][j].screen_end,\n            screen_y: processedGroupData[groupIds[i]][j].screen_y,\n            x: processedGroupData[groupIds[i]][j].x,\n            end: processedGroupData[groupIds[i]][j].end,\n            y: processedGroupData[groupIds[i]][j].y,\n            groupId: groupIds[i],\n            label: processedGroupData[groupIds[i]][j].label\n          });\n          barPoints += 1;\n        }\n      }\n    }\n  }\n\n  if (barPoints === 0) {\n    return;\n  }\n\n  // sort by time and by group\n  combinedData.sort(function (a, b) {\n    if (a.screen_x === b.screen_x) {\n      return a.groupId < b.groupId ? -1 : 1;\n    }\n    else {\n      return a.screen_x - b.screen_x;\n    }\n  });\n\n  // get intersections\n  Bargraph._getDataIntersections(intersections, combinedData);\n\n  // plot barchart\n  for (i = 0; i < combinedData.length; i++) {\n    group = framework.groups[combinedData[i].groupId];\n    var minWidth = group.options.barChart.minWidth != undefined ? group.options.barChart.minWidth : 0.1 * group.options.barChart.width;\n\n    key = combinedData[i].screen_x;\n    var heightOffset = 0;\n    if (intersections[key] === undefined) {\n      if (i + 1 < combinedData.length) {\n        coreDistance = Math.abs(combinedData[i + 1].screen_x - key);\n      }\n      drawData = Bargraph._getSafeDrawData(coreDistance, group, minWidth);\n    }\n    else {\n      var nextKey = i + (intersections[key].amount - intersections[key].resolved);\n      if (nextKey < combinedData.length) {\n        coreDistance = Math.abs(combinedData[nextKey].screen_x - key);\n      }\n      drawData = Bargraph._getSafeDrawData(coreDistance, group, minWidth);\n      intersections[key].resolved += 1;\n\n      if (group.options.stack === true && group.options.excludeFromStacking !== true) {\n        if (combinedData[i].screen_y < group.zeroPosition) {\n          heightOffset = intersections[key].accumulatedNegative;\n          intersections[key].accumulatedNegative += group.zeroPosition - combinedData[i].screen_y;\n        }\n        else {\n          heightOffset = intersections[key].accumulatedPositive;\n          intersections[key].accumulatedPositive += group.zeroPosition - combinedData[i].screen_y;\n        }\n      }\n      else if (group.options.barChart.sideBySide === true) {\n        drawData.width = drawData.width / intersections[key].amount;\n        drawData.offset += (intersections[key].resolved) * drawData.width - (0.5 * drawData.width * (intersections[key].amount + 1));\n      }\n    }\n    \n    let dataWidth = drawData.width;\n    let start = combinedData[i].screen_x;\n\n    // are we drawing explicit boxes? (we supplied an end value)\n    if (combinedData[i].screen_end != undefined){\n      dataWidth = combinedData[i].screen_end - combinedData[i].screen_x;\n      start += (dataWidth * 0.5);\n    }\n    else {\n      start += drawData.offset\n    }\n\n    DOMutil.drawBar(start, combinedData[i].screen_y - heightOffset, dataWidth, group.zeroPosition - combinedData[i].screen_y, group.className + ' vis-bar', framework.svgElements, framework.svg, group.style);\n\n    // draw points\n    if (group.options.drawPoints.enabled === true) {\n      let pointData = {\n        screen_x: combinedData[i].screen_x,\n        screen_y: combinedData[i].screen_y - heightOffset,\n        x: combinedData[i].x,\n        y: combinedData[i].y,\n        groupId: combinedData[i].groupId,\n        label: combinedData[i].label\n      };\n      Points.draw([pointData], group, framework, drawData.offset);\n      //DOMutil.drawPoint(combinedData[i].x + drawData.offset, combinedData[i].y, group, framework.svgElements, framework.svg);\n    }\n  }\n};\n\n\n/**\n * Fill the intersections object with counters of how many datapoints share the same x coordinates\n * @param {Object} intersections\n * @param {Array.<Object>} combinedData\n * @private\n */\nBargraph._getDataIntersections = function (intersections, combinedData) {\n  // get intersections\n  var coreDistance;\n  for (var i = 0; i < combinedData.length; i++) {\n    if (i + 1 < combinedData.length) {\n      coreDistance = Math.abs(combinedData[i + 1].screen_x - combinedData[i].screen_x);\n    }\n    if (i > 0) {\n      coreDistance = Math.min(coreDistance, Math.abs(combinedData[i - 1].screen_x - combinedData[i].screen_x));\n    }\n    if (coreDistance === 0) {\n      if (intersections[combinedData[i].screen_x] === undefined) {\n        intersections[combinedData[i].screen_x] = {\n          amount: 0,\n          resolved: 0,\n          accumulatedPositive: 0,\n          accumulatedNegative: 0\n        };\n      }\n      intersections[combinedData[i].screen_x].amount += 1;\n    }\n  }\n};\n\n\n/**\n * Get the width and offset for bargraphs based on the coredistance between datapoints\n *\n * @param {number} coreDistance\n * @param {vis.Group} group\n * @param {number} minWidth\n * @returns {{width: number, offset: number}}\n * @private\n */\nBargraph._getSafeDrawData = function (coreDistance, group, minWidth) {\n  var width, offset;\n  if (coreDistance < group.options.barChart.width && coreDistance > 0) {\n    width = coreDistance < minWidth ? minWidth : coreDistance\n\n    offset = 0; // recalculate offset with the new width;\n    if (group.options.barChart.align === 'left') {\n      offset -= 0.5 * coreDistance;\n    }\n    else if (group.options.barChart.align === 'right') {\n      offset += 0.5 * coreDistance;\n    }\n  }\n  else {\n    // default settings\n    width = group.options.barChart.width;\n    offset = 0;\n    if (group.options.barChart.align === 'left') {\n      offset -= 0.5 * group.options.barChart.width;\n    }\n    else if (group.options.barChart.align === 'right') {\n      offset += 0.5 * group.options.barChart.width;\n    }\n  }\n\n  return {width: width, offset: offset};\n};\n\nBargraph.getStackedYRange = function (combinedData, groupRanges, groupIds, groupLabel, orientation) {\n  if (combinedData.length > 0) {\n    // sort by time and by group\n    combinedData.sort(function (a, b) {\n      if (a.screen_x === b.screen_x) {\n        return a.groupId < b.groupId ? -1 : 1;\n      }\n      else {\n        return a.screen_x - b.screen_x;\n      }\n    });\n    var intersections = {};\n\n    Bargraph._getDataIntersections(intersections, combinedData);\n    groupRanges[groupLabel] = Bargraph._getStackedYRange(intersections, combinedData);\n    groupRanges[groupLabel].yAxisOrientation = orientation;\n    groupIds.push(groupLabel);\n  }\n};\n\nBargraph._getStackedYRange = function (intersections, combinedData) {\n  var key;\n  var yMin = combinedData[0].screen_y;\n  var yMax = combinedData[0].screen_y;\n  for (var i = 0; i < combinedData.length; i++) {\n    key = combinedData[i].screen_x;\n    if (intersections[key] === undefined) {\n      yMin = yMin > combinedData[i].screen_y ? combinedData[i].screen_y : yMin;\n      yMax = yMax < combinedData[i].screen_y ? combinedData[i].screen_y : yMax;\n    }\n    else {\n      if (combinedData[i].screen_y < 0) {\n        intersections[key].accumulatedNegative += combinedData[i].screen_y;\n      }\n      else {\n        intersections[key].accumulatedPositive += combinedData[i].screen_y;\n      }\n    }\n  }\n  for (var xpos in intersections) {\n    if (intersections.hasOwnProperty(xpos)) {\n      yMin = yMin > intersections[xpos].accumulatedNegative ? intersections[xpos].accumulatedNegative : yMin;\n      yMin = yMin > intersections[xpos].accumulatedPositive ? intersections[xpos].accumulatedPositive : yMin;\n      yMax = yMax < intersections[xpos].accumulatedNegative ? intersections[xpos].accumulatedNegative : yMax;\n      yMax = yMax < intersections[xpos].accumulatedPositive ? intersections[xpos].accumulatedPositive : yMax;\n    }\n  }\n\n  return {min: yMin, max: yMax};\n};\n\nexport default Bargraph;","import * as DOMutil from '../../../DOMutil';\n\n/**\n *\n * @param {vis.GraphGroup.id} groupId\n * @param {Object} options   // TODO: Describe options\n * @constructor Line\n */\nfunction Line(groupId, options) {  // eslint-disable-line no-unused-vars\n}\n\nLine.calcPath = function (dataset, group) {\n    if (dataset != null) {\n        if (dataset.length > 0) {\n            var d = [];\n\n            // construct path from dataset\n            if (group.options.interpolation.enabled == true) {\n                d = Line._catmullRom(dataset, group);\n            }\n            else {\n                d = Line._linear(dataset);\n            }\n            return d;\n        }\n    }\n};\n\nLine.drawIcon = function (group, x, y, iconWidth, iconHeight, framework) {\n    var fillHeight = iconHeight * 0.5;\n    var path, fillPath;\n\n    var outline = DOMutil.getSVGElement(\"rect\", framework.svgElements, framework.svg);\n    outline.setAttributeNS(null, \"x\", x);\n    outline.setAttributeNS(null, \"y\", y - fillHeight);\n    outline.setAttributeNS(null, \"width\", iconWidth);\n    outline.setAttributeNS(null, \"height\", 2 * fillHeight);\n    outline.setAttributeNS(null, \"class\", \"vis-outline\");\n\n    path = DOMutil.getSVGElement(\"path\", framework.svgElements, framework.svg);\n    path.setAttributeNS(null, \"class\", group.className);\n    if (group.style !== undefined) {\n        path.setAttributeNS(null, \"style\", group.style);\n    }\n\n    path.setAttributeNS(null, \"d\", \"M\" + x + \",\" + y + \" L\" + (x + iconWidth) + \",\" + y + \"\");\n    if (group.options.shaded.enabled == true) {\n        fillPath = DOMutil.getSVGElement(\"path\", framework.svgElements, framework.svg);\n        if (group.options.shaded.orientation == 'top') {\n            fillPath.setAttributeNS(null, \"d\", \"M\" + x + \", \" + (y - fillHeight) +\n              \"L\" + x + \",\" + y + \" L\" + (x + iconWidth) + \",\" + y + \" L\" + (x + iconWidth) + \",\" + (y - fillHeight));\n        }\n        else {\n            fillPath.setAttributeNS(null, \"d\", \"M\" + x + \",\" + y + \" \" +\n              \"L\" + x + \",\" + (y + fillHeight) + \" \" +\n              \"L\" + (x + iconWidth) + \",\" + (y + fillHeight) +\n              \"L\" + (x + iconWidth) + \",\" + y);\n        }\n        fillPath.setAttributeNS(null, \"class\", group.className + \" vis-icon-fill\");\n        if (group.options.shaded.style !== undefined && group.options.shaded.style !== \"\") {\n            fillPath.setAttributeNS(null, \"style\", group.options.shaded.style);\n        }\n    }\n\n    if (group.options.drawPoints.enabled == true) {\n        var groupTemplate = {\n            style: group.options.drawPoints.style,\n            styles: group.options.drawPoints.styles,\n            size: group.options.drawPoints.size,\n            className: group.className\n        };\n        DOMutil.drawPoint(x + 0.5 * iconWidth, y, groupTemplate, framework.svgElements, framework.svg);\n    }\n};\n\nLine.drawShading = function (pathArray, group, subPathArray, framework) {\n    // append shading to the path\n    if (group.options.shaded.enabled == true) {\n        var svgHeight = Number(framework.svg.style.height.replace('px',''));\n        var fillPath = DOMutil.getSVGElement('path', framework.svgElements, framework.svg);\n        var type = \"L\";\n        if (group.options.interpolation.enabled == true){\n            type = \"C\";\n        }\n        var dFill;\n        var zero = 0;\n        if (group.options.shaded.orientation == 'top') {\n            zero = 0;\n        }\n        else if (group.options.shaded.orientation == 'bottom') {\n            zero = svgHeight;\n        }\n        else {\n            zero = Math.min(Math.max(0, group.zeroPosition), svgHeight);\n        }\n        if (group.options.shaded.orientation == 'group' && (subPathArray != null && subPathArray != undefined)) {\n            dFill = 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" +\n                    this.serializePath(pathArray,type,false) +\n                    ' L'+ subPathArray[subPathArray.length-1][0]+ \",\" + subPathArray[subPathArray.length-1][1] + \" \" +\n                    this.serializePath(subPathArray,type,true) +\n                    subPathArray[0][0]+ \",\"+subPathArray[0][1] + \" Z\";\n        }\n        else {\n            dFill = 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" +\n                    this.serializePath(pathArray,type,false) +\n                    ' V' + zero + ' H'+ pathArray[0][0] + \" Z\";\n        }\n\n        fillPath.setAttributeNS(null, 'class', group.className + ' vis-fill');\n        if (group.options.shaded.style !== undefined) {\n            fillPath.setAttributeNS(null, 'style', group.options.shaded.style);\n        }\n        fillPath.setAttributeNS(null, 'd', dFill);\n    }\n};\n\n/**\n * draw a line graph\n *\n * @param {Array.<Object>} pathArray\n * @param {vis.Group} group\n * @param {{svg: Object, svgElements: Array.<Object>, options: Object, groups: Array.<vis.Group>}} framework\n */\nLine.draw = function (pathArray, group, framework) {\n    if (pathArray != null && pathArray != undefined) {\n        var path = DOMutil.getSVGElement('path', framework.svgElements, framework.svg);\n        path.setAttributeNS(null, \"class\", group.className);\n        if (group.style !== undefined) {\n            path.setAttributeNS(null, \"style\", group.style);\n        }\n\n        var type = \"L\";\n        if (group.options.interpolation.enabled == true){\n            type = \"C\";\n        }\n        // copy properties to path for drawing.\n        path.setAttributeNS(null, 'd', 'M' + pathArray[0][0]+ \",\"+pathArray[0][1] + \" \" + this.serializePath(pathArray,type,false));\n    }\n};\n\nLine.serializePath = function(pathArray,type,inverse){\n    if (pathArray.length < 2){\n        //Too little data to create a path.\n        return \"\";\n    }\n    var d = type;\n    var i;\n    if (inverse){\n        for (i = pathArray.length-2; i > 0; i--){\n            d += pathArray[i][0] + \",\" + pathArray[i][1] + \" \";\n        }\n    }\n    else {\n        for (i = 1; i < pathArray.length; i++){\n            d += pathArray[i][0] + \",\" + pathArray[i][1] + \" \";\n        }\n    }\n    return d;\n};\n\n/**\n * This uses an uniform parametrization of the interpolation algorithm:\n * 'On the Parameterization of Catmull-Rom Curves' by Cem Yuksel et al.\n * @param {Array.<Object>} data\n * @returns {string}\n * @private\n */\nLine._catmullRomUniform = function (data) {\n    // catmull rom\n    var p0, p1, p2, p3, bp1, bp2;\n    var d = [];\n    d.push( [ Math.round(data[0].screen_x) , Math.round(data[0].screen_y) ]);\n    var normalization = 1 / 6;\n    var length = data.length;\n    for (var i = 0; i < length - 1; i++) {\n\n        p0 = (i == 0) ? data[0] : data[i - 1];\n        p1 = data[i];\n        p2 = data[i + 1];\n        p3 = (i + 2 < length) ? data[i + 2] : p2;\n\n\n        // Catmull-Rom to Cubic Bezier conversion matrix\n        //    0       1       0       0\n        //  -1/6      1      1/6      0\n        //    0      1/6      1     -1/6\n        //    0       0       1       0\n\n        //    bp0 = { x: p1.x,                               y: p1.y };\n        bp1 = {\n            screen_x: ((-p0.screen_x + 6 * p1.screen_x + p2.screen_x) * normalization),\n            screen_y: ((-p0.screen_y + 6 * p1.screen_y + p2.screen_y) * normalization)\n        };\n        bp2 = {\n            screen_x: (( p1.screen_x + 6 * p2.screen_x - p3.screen_x) * normalization),\n            screen_y: (( p1.screen_y + 6 * p2.screen_y - p3.screen_y) * normalization)\n        };\n        //    bp0 = { x: p2.x,                               y: p2.y };\n\n        d.push( [ bp1.screen_x , bp1.screen_y ]);\n        d.push( [ bp2.screen_x , bp2.screen_y ]);\n        d.push( [ p2.screen_x  , p2.screen_y  ]);\n    }\n\n    return d;\n};\n\n/**\n * This uses either the chordal or centripetal parameterization of the catmull-rom algorithm.\n * By default, the centripetal parameterization is used because this gives the nicest results.\n * These parameterizations are relatively heavy because the distance between 4 points have to be calculated.\n *\n * One optimization can be used to reuse distances since this is a sliding window approach.\n * @param {Array.<Object>} data\n * @param {vis.GraphGroup} group\n * @returns {string}\n * @private\n */\nLine._catmullRom = function (data, group) {\n    var alpha = group.options.interpolation.alpha;\n    if (alpha == 0 || alpha === undefined) {\n        return this._catmullRomUniform(data);\n    }\n    else {\n        var p0, p1, p2, p3, bp1, bp2, d1, d2, d3, A, B, N, M;\n        var d3powA, d2powA, d3pow2A, d2pow2A, d1pow2A, d1powA;\n        var d = [];\n        d.push( [ Math.round(data[0].screen_x) , Math.round(data[0].screen_y) ]);\n        var length = data.length;\n        for (var i = 0; i < length - 1; i++) {\n\n            p0 = (i == 0) ? data[0] : data[i - 1];\n            p1 = data[i];\n            p2 = data[i + 1];\n            p3 = (i + 2 < length) ? data[i + 2] : p2;\n\n            d1 = Math.sqrt(Math.pow(p0.screen_x - p1.screen_x, 2) + Math.pow(p0.screen_y - p1.screen_y, 2));\n            d2 = Math.sqrt(Math.pow(p1.screen_x - p2.screen_x, 2) + Math.pow(p1.screen_y - p2.screen_y, 2));\n            d3 = Math.sqrt(Math.pow(p2.screen_x - p3.screen_x, 2) + Math.pow(p2.screen_y - p3.screen_y, 2));\n\n            // Catmull-Rom to Cubic Bezier conversion matrix\n\n            // A = 2d1^2a + 3d1^a * d2^a + d3^2a\n            // B = 2d3^2a + 3d3^a * d2^a + d2^2a\n\n            // [   0             1            0          0          ]\n            // [   -d2^2a /N     A/N          d1^2a /N   0          ]\n            // [   0             d3^2a /M     B/M        -d2^2a /M  ]\n            // [   0             0            1          0          ]\n\n            d3powA = Math.pow(d3, alpha);\n            d3pow2A = Math.pow(d3, 2 * alpha);\n            d2powA = Math.pow(d2, alpha);\n            d2pow2A = Math.pow(d2, 2 * alpha);\n            d1powA = Math.pow(d1, alpha);\n            d1pow2A = Math.pow(d1, 2 * alpha);\n\n            A = 2 * d1pow2A + 3 * d1powA * d2powA + d2pow2A;\n            B = 2 * d3pow2A + 3 * d3powA * d2powA + d2pow2A;\n            N = 3 * d1powA * (d1powA + d2powA);\n            if (N > 0) {\n                N = 1 / N;\n            }\n            M = 3 * d3powA * (d3powA + d2powA);\n            if (M > 0) {\n                M = 1 / M;\n            }\n\n            bp1 = {\n                screen_x: ((-d2pow2A * p0.screen_x + A * p1.screen_x + d1pow2A * p2.screen_x) * N),\n                screen_y: ((-d2pow2A * p0.screen_y + A * p1.screen_y + d1pow2A * p2.screen_y) * N)\n            };\n\n            bp2 = {\n                screen_x: (( d3pow2A * p1.screen_x + B * p2.screen_x - d2pow2A * p3.screen_x) * M),\n                screen_y: (( d3pow2A * p1.screen_y + B * p2.screen_y - d2pow2A * p3.screen_y) * M)\n            };\n\n            if (bp1.screen_x == 0 && bp1.screen_y == 0) {\n                bp1 = p1;\n            }\n            if (bp2.screen_x == 0 && bp2.screen_y == 0) {\n                bp2 = p2;\n            }\n            d.push( [ bp1.screen_x , bp1.screen_y ]);\n            d.push( [ bp2.screen_x , bp2.screen_y ]);\n            d.push( [ p2.screen_x  , p2.screen_y  ]);\n        }\n\n        return d;\n    }\n};\n\n/**\n * this generates the SVG path for a linear drawing between datapoints.\n * @param {Array.<Object>} data\n * @returns {string}\n * @private\n */\nLine._linear = function (data) {\n    // linear\n    var d = [];\n    for (var i = 0; i < data.length; i++) {\n        d.push([ data[i].screen_x , data[i].screen_y ]);\n    }\n    return d;\n};\n\nexport default Line;\n","import util from '../../util';\nimport Bars from './graph2d_types/bar';\nimport Lines from './graph2d_types/line';\nimport Points from './graph2d_types/points';\n\n/**\n * /**\n * @param {object} group            | the object of the group from the dataset\n * @param {string} groupId          | ID of the group\n * @param {object} options          | the default options\n * @param {array} groupsUsingDefaultStyles  | this array has one entree.\n *                                            It is passed as an array so it is passed by reference.\n *                                            It enumerates through the default styles\n * @constructor GraphGroup\n */\nfunction GraphGroup(group, groupId, options, groupsUsingDefaultStyles) {\n  this.id = groupId;\n  var fields = ['sampling', 'style', 'sort', 'yAxisOrientation', 'barChart', 'drawPoints', 'shaded', 'interpolation', 'zIndex','excludeFromStacking', 'excludeFromLegend'];\n  this.options = util.selectiveBridgeObject(fields, options);\n  this.usingDefaultStyle = group.className === undefined;\n  this.groupsUsingDefaultStyles = groupsUsingDefaultStyles;\n  this.zeroPosition = 0;\n  this.update(group);\n  if (this.usingDefaultStyle == true) {\n    this.groupsUsingDefaultStyles[0] += 1;\n  }\n  this.itemsData = [];\n  this.visible = group.visible === undefined ? true : group.visible;\n}\n\n/**\n * this loads a reference to all items in this group into this group.\n * @param {array} items\n */\nGraphGroup.prototype.setItems = function (items) {\n  if (items != null) {\n    this.itemsData = items;\n    if (this.options.sort == true) {\n      util.insertSort(this.itemsData,function (a, b) {\n        return a.x > b.x ? 1 : -1;\n      });\n    }\n  }\n  else {\n    this.itemsData = [];\n  }\n};\n\nGraphGroup.prototype.getItems = function () {\n  return this.itemsData;\n};\n\n/**\n * this is used for barcharts and shading, this way, we only have to calculate it once.\n * @param {number} pos\n */\nGraphGroup.prototype.setZeroPosition = function (pos) {\n  this.zeroPosition = pos;\n};\n\n/**\n * set the options of the graph group over the default options.\n * @param {Object} options\n */\nGraphGroup.prototype.setOptions = function (options) {\n  if (options !== undefined) {\n    var fields = ['sampling', 'style', 'sort', 'yAxisOrientation', 'barChart', 'zIndex','excludeFromStacking', 'excludeFromLegend'];\n    util.selectiveDeepExtend(fields, this.options, options);\n\n    // if the group's drawPoints is a function delegate the callback to the onRender property\n    if (typeof options.drawPoints == 'function') {\n      options.drawPoints = {\n        onRender: options.drawPoints\n      }\n    }\n\n    util.mergeOptions(this.options, options, 'interpolation');\n    util.mergeOptions(this.options, options, 'drawPoints');\n    util.mergeOptions(this.options, options, 'shaded');\n\n    if (options.interpolation) {\n      if (typeof options.interpolation == 'object') {\n        if (options.interpolation.parametrization) {\n          if (options.interpolation.parametrization == 'uniform') {\n            this.options.interpolation.alpha = 0;\n          }\n          else if (options.interpolation.parametrization == 'chordal') {\n            this.options.interpolation.alpha = 1.0;\n          }\n          else {\n            this.options.interpolation.parametrization = 'centripetal';\n            this.options.interpolation.alpha = 0.5;\n          }\n        }\n      }\n    }\n  }\n};\n\n\n/**\n * this updates the current group class with the latest group dataset entree, used in _updateGroup in linegraph\n * @param {vis.Group} group\n */\nGraphGroup.prototype.update = function (group) {\n  this.group = group;\n  this.content = group.content || 'graph';\n  this.className = group.className || this.className || 'vis-graph-group' + this.groupsUsingDefaultStyles[0] % 10;\n  this.visible = group.visible === undefined ? true : group.visible;\n  this.style = group.style;\n  this.setOptions(group.options);\n};\n\n/**\n * return the legend entree for this group.\n *\n * @param {number} iconWidth\n * @param {number} iconHeight\n * @param {{svg: (*|Element), svgElements: Object, options: Object, groups: Array.<Object>}} framework\n * @param {number} x\n * @param {number} y\n * @returns {{icon: (*|Element), label: (*|string), orientation: *}}\n */\nGraphGroup.prototype.getLegend = function (iconWidth, iconHeight, framework, x, y) {\n  if (framework == undefined || framework == null) {\n    var svg = document.createElementNS('http://www.w3.org/2000/svg', \"svg\");\n    framework = {svg: svg, svgElements:{}, options: this.options, groups: [this]}\n  }\n  if (x == undefined || x == null){\n    x = 0;\n  }\n  if (y == undefined || y == null){\n    y = 0.5 * iconHeight;\n  }\n  switch (this.options.style){\n    case \"line\":\n      Lines.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n    case \"points\": //explicit no break\n    case \"point\":\n      Points.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n    case \"bar\":\n      Bars.drawIcon(this, x, y, iconWidth, iconHeight, framework);\n      break;\n  }\n  return {icon: framework.svg, label: this.content, orientation: this.options.yAxisOrientation};\n};\n\nGraphGroup.prototype.getYRange = function (groupData) {\n  var yMin = groupData[0].y;\n  var yMax = groupData[0].y;\n  for (var j = 0; j < groupData.length; j++) {\n    yMin = yMin > groupData[j].y ? groupData[j].y : yMin;\n    yMax = yMax < groupData[j].y ? groupData[j].y : yMax;\n  }\n  return {min: yMin, max: yMax, yAxisOrientation: this.options.yAxisOrientation};\n};\n\nexport default GraphGroup;\n","import util from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport Component from './Component';\n\n/**\n * Legend for Graph2d\n *\n * @param {vis.Graph2d.body} body\n * @param {vis.Graph2d.options} options\n * @param {number} side\n * @param {vis.LineGraph.options} linegraphOptions\n * @constructor Legend\n * @extends Component\n */\nfunction Legend(body, options, side, linegraphOptions) {\n  this.body = body;\n  this.defaultOptions = {\n    enabled: false,\n    icons: true,\n    iconSize: 20,\n    iconSpacing: 6,\n    left: {\n      visible: true,\n      position: 'top-left' // top/bottom - left,center,right\n    },\n    right: {\n      visible: true,\n      position: 'top-right' // top/bottom - left,center,right\n    }\n  };\n\n  this.side = side;\n  this.options = util.extend({}, this.defaultOptions);\n  this.linegraphOptions = linegraphOptions;\n\n  this.svgElements = {};\n  this.dom = {};\n  this.groups = {};\n  this.amountOfGroups = 0;\n  this._create();\n  this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n\n  this.setOptions(options);\n}\n\nLegend.prototype = new Component();\n\nLegend.prototype.clear = function() {\n  this.groups = {};\n  this.amountOfGroups = 0;\n};\n\nLegend.prototype.addGroup = function(label, graphOptions) {\n\n  // Include a group only if the group option 'excludeFromLegend: false' is not set.\n  if (graphOptions.options.excludeFromLegend != true) {\n    if (!this.groups.hasOwnProperty(label)) {\n      this.groups[label] = graphOptions;\n    }\n    this.amountOfGroups += 1;\n  }\n};\n\nLegend.prototype.updateGroup = function(label, graphOptions) {\n  this.groups[label] = graphOptions;\n};\n\nLegend.prototype.removeGroup = function(label) {\n  if (this.groups.hasOwnProperty(label)) {\n    delete this.groups[label];\n    this.amountOfGroups -= 1;\n  }\n};\n\nLegend.prototype._create = function() {\n  this.dom.frame = document.createElement('div');\n  this.dom.frame.className = 'vis-legend';\n  this.dom.frame.style.position = \"absolute\";\n  this.dom.frame.style.top = \"10px\";\n  this.dom.frame.style.display = \"block\";\n\n  this.dom.textArea = document.createElement('div');\n  this.dom.textArea.className = 'vis-legend-text';\n  this.dom.textArea.style.position = \"relative\";\n  this.dom.textArea.style.top = \"0px\";\n\n  this.svg = document.createElementNS('http://www.w3.org/2000/svg',\"svg\");\n  this.svg.style.position = 'absolute';\n  this.svg.style.top = 0 +'px';\n  this.svg.style.width = this.options.iconSize + 5 + 'px';\n  this.svg.style.height = '100%';\n\n  this.dom.frame.appendChild(this.svg);\n  this.dom.frame.appendChild(this.dom.textArea);\n};\n\n/**\n * Hide the component from the DOM\n */\nLegend.prototype.hide = function() {\n  // remove the frame containing the items\n  if (this.dom.frame.parentNode) {\n    this.dom.frame.parentNode.removeChild(this.dom.frame);\n  }\n};\n\n/**\n * Show the component in the DOM (when not already visible).\n */\nLegend.prototype.show = function() {\n  // show frame containing the items\n  if (!this.dom.frame.parentNode) {\n    this.body.dom.center.appendChild(this.dom.frame);\n  }\n};\n\nLegend.prototype.setOptions = function(options) {\n  var fields = ['enabled','orientation','icons','left','right'];\n  util.selectiveDeepExtend(fields, this.options, options);\n};\n\nLegend.prototype.redraw = function() {\n  var activeGroups = 0;\n  var groupArray = Object.keys(this.groups);\n  groupArray.sort(function (a,b) {\n    return (a < b ? -1 : 1);\n  })\n\n  for (var i = 0; i < groupArray.length; i++) {\n    var groupId = groupArray[i];\n    if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n      activeGroups++;\n    }\n  }\n\n  if (this.options[this.side].visible == false || this.amountOfGroups == 0 || this.options.enabled == false || activeGroups == 0) {\n    this.hide();\n  }\n  else {\n    this.show();\n    if (this.options[this.side].position == 'top-left' || this.options[this.side].position == 'bottom-left') {\n      this.dom.frame.style.left = '4px';\n      this.dom.frame.style.textAlign = \"left\";\n      this.dom.textArea.style.textAlign = \"left\";\n      this.dom.textArea.style.left = (this.options.iconSize + 15) + 'px';\n      this.dom.textArea.style.right = '';\n      this.svg.style.left = 0 +'px';\n      this.svg.style.right = '';\n    }\n    else {\n      this.dom.frame.style.right = '4px';\n      this.dom.frame.style.textAlign = \"right\";\n      this.dom.textArea.style.textAlign = \"right\";\n      this.dom.textArea.style.right = (this.options.iconSize + 15) + 'px';\n      this.dom.textArea.style.left = '';\n      this.svg.style.right = 0 +'px';\n      this.svg.style.left = '';\n    }\n\n    if (this.options[this.side].position == 'top-left' || this.options[this.side].position == 'top-right') {\n      this.dom.frame.style.top = 4 - Number(this.body.dom.center.style.top.replace(\"px\",\"\")) + 'px';\n      this.dom.frame.style.bottom = '';\n    }\n    else {\n      var scrollableHeight = this.body.domProps.center.height - this.body.domProps.centerContainer.height;\n      this.dom.frame.style.bottom = 4 + scrollableHeight + Number(this.body.dom.center.style.top.replace(\"px\",\"\")) + 'px';\n      this.dom.frame.style.top = '';\n    }\n\n    if (this.options.icons == false) {\n      this.dom.frame.style.width = this.dom.textArea.offsetWidth + 10 + 'px';\n      this.dom.textArea.style.right = '';\n      this.dom.textArea.style.left = '';\n      this.svg.style.width = '0px';\n    }\n    else {\n      this.dom.frame.style.width = this.options.iconSize + 15 + this.dom.textArea.offsetWidth + 10 + 'px'\n      this.drawLegendIcons();\n    }\n\n    var content = '';\n    for (i = 0; i < groupArray.length; i++) {\n      groupId = groupArray[i];\n      if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n        content += this.groups[groupId].content + '<br />';\n      }\n    }\n    this.dom.textArea.innerHTML = content;\n    this.dom.textArea.style.lineHeight = ((0.75 * this.options.iconSize) + this.options.iconSpacing) + 'px';\n  }\n};\n\nLegend.prototype.drawLegendIcons = function() {\n  if (this.dom.frame.parentNode) {\n    var groupArray = Object.keys(this.groups);\n    groupArray.sort(function (a,b) {\n      return (a < b ? -1 : 1);\n    });\n\n    // this resets the elements so the order is maintained\n    DOMutil.resetElements(this.svgElements);\n\n    var padding = window.getComputedStyle(this.dom.frame).paddingTop;\n    var iconOffset = Number(padding.replace('px',''));\n    var x = iconOffset;\n    var iconWidth = this.options.iconSize;\n    var iconHeight = 0.75 * this.options.iconSize;\n    var y = iconOffset + 0.5 * iconHeight + 3;\n\n    this.svg.style.width = iconWidth + 5 + iconOffset + 'px';\n\n    for (var i = 0; i < groupArray.length; i++) {\n      var groupId = groupArray[i];\n      if (this.groups[groupId].visible == true && (this.linegraphOptions.visibility[groupId] === undefined || this.linegraphOptions.visibility[groupId] == true)) {\n        this.groups[groupId].getLegend(iconWidth, iconHeight, this.framework, x, y);\n        y += iconHeight + this.options.iconSpacing;\n      }\n    }\n  }\n};\n\nexport default Legend;\n","import { v4 as randomUUID } from \"uuid\";\nimport util, { typeCoerceDataSet } from '../../util';\nimport * as DOMutil from '../../DOMutil';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Component from './Component';\nimport DataAxis from './DataAxis';\nimport GraphGroup from './GraphGroup';\nimport Legend from './Legend';\nimport Bars from './graph2d_types/bar';\nimport Lines from './graph2d_types/line';\nimport Points from './graph2d_types/points';\n\nvar UNGROUPED = '__ungrouped__'; // reserved group id for ungrouped items\n\n/**\n * This is the constructor of the LineGraph. It requires a Timeline body and options.\n *\n * @param {vis.Timeline.body} body\n * @param {Object} options\n * @constructor LineGraph\n * @extends Component\n */\nfunction LineGraph(body, options) {\n  this.id = randomUUID();\n  this.body = body;\n\n  this.defaultOptions = {\n    yAxisOrientation: 'left',\n    defaultGroup: 'default',\n    sort: true,\n    sampling: true,\n    stack: false,\n    graphHeight: '400px',\n    shaded: {\n      enabled: false,\n      orientation: 'bottom' // top, bottom, zero\n    },\n    style: 'line', // line, bar\n    barChart: {\n      width: 50,\n      sideBySide: false,\n      align: 'center' // left, center, right\n    },\n    interpolation: {\n      enabled: true,\n      parametrization: 'centripetal', // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n      alpha: 0.5\n    },\n    drawPoints: {\n      enabled: true,\n      size: 6,\n      style: 'square' // square, circle\n    },\n    dataAxis: {}, //Defaults are done on DataAxis level\n    legend: {}, //Defaults are done on Legend level\n    groups: {\n      visibility: {}\n    }\n  };\n\n  // options is shared by this lineGraph and all its items\n  this.options = util.extend({}, this.defaultOptions);\n  this.dom = {};\n  this.props = {};\n  this.hammer = null;\n  this.groups = {};\n  this.abortedGraphUpdate = false;\n  this.updateSVGheight = false;\n  this.updateSVGheightOnResize = false;\n  this.forceGraphUpdate = true;\n\n  var me = this;\n  this.itemsData = null;    // DataSet\n  this.groupsData = null;   // DataSet\n\n  // listeners for the DataSet of the items\n  this.itemListeners = {\n    'add': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onAdd(params.items);\n    },\n    'update': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onUpdate(params.items);\n    },\n    'remove': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onRemove(params.items);\n    }\n  };\n\n  // listeners for the DataSet of the groups\n  this.groupListeners = {\n    'add': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onAddGroups(params.items);\n    },\n    'update': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onUpdateGroups(params.items);\n    },\n    'remove': function (event, params, senderId) {  // eslint-disable-line no-unused-vars\n      me._onRemoveGroups(params.items);\n    }\n  };\n\n  this.items = {};      // object with an Item for every data item\n  this.selection = [];  // list with the ids of all selected nodes\n  this.lastStart = this.body.range.start;\n  this.touchParams = {}; // stores properties while dragging\n\n  this.svgElements = {};\n  this.setOptions(options);\n  this.groupsUsingDefaultStyles = [0];\n  this.body.emitter.on('rangechanged', function () {\n    me.lastStart = me.body.range.start;\n    me.svg.style.left = util.option.asSize(-me.props.width);\n\n    me.forceGraphUpdate = true;\n    //Is this local redraw necessary? (Core also does a change event!)\n    me.redraw.call(me);\n  });\n\n  // create the HTML DOM\n  this._create();\n  this.framework = {svg: this.svg, svgElements: this.svgElements, options: this.options, groups: this.groups};\n}\n\nLineGraph.prototype = new Component();\n\n/**\n * Create the HTML DOM for the ItemSet\n */\nLineGraph.prototype._create = function () {\n  var frame = document.createElement('div');\n  frame.className = 'vis-line-graph';\n  this.dom.frame = frame;\n\n  // create svg element for graph drawing.\n  this.svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg');\n  this.svg.style.position = 'relative';\n  this.svg.style.height = ('' + this.options.graphHeight).replace('px', '') + 'px';\n  this.svg.style.display = 'block';\n  frame.appendChild(this.svg);\n\n  // data axis\n  this.options.dataAxis.orientation = 'left';\n  this.yAxisLeft = new DataAxis(this.body, this.options.dataAxis, this.svg, this.options.groups);\n\n  this.options.dataAxis.orientation = 'right';\n  this.yAxisRight = new DataAxis(this.body, this.options.dataAxis, this.svg, this.options.groups);\n  delete this.options.dataAxis.orientation;\n\n  // legends\n  this.legendLeft = new Legend(this.body, this.options.legend, 'left', this.options.groups);\n  this.legendRight = new Legend(this.body, this.options.legend, 'right', this.options.groups);\n\n  this.show();\n};\n\n/**\n * set the options of the LineGraph. the mergeOptions is used for subObjects that have an enabled element.\n * @param {object} options\n */\nLineGraph.prototype.setOptions = function (options) {\n  if (options) {\n    var fields = ['sampling', 'defaultGroup', 'stack', 'height', 'graphHeight', 'yAxisOrientation', 'style', 'barChart', 'dataAxis', 'sort', 'groups'];\n    if (options.graphHeight === undefined && options.height !== undefined) {\n      this.updateSVGheight = true;\n      this.updateSVGheightOnResize = true;\n    }\n    else if (this.body.domProps.centerContainer.height !== undefined && options.graphHeight !== undefined) {\n      if (parseInt((options.graphHeight + '').replace(\"px\", '')) < this.body.domProps.centerContainer.height) {\n        this.updateSVGheight = true;\n      }\n    }\n    util.selectiveDeepExtend(fields, this.options, options);\n    util.mergeOptions(this.options, options, 'interpolation');\n    util.mergeOptions(this.options, options, 'drawPoints');\n    util.mergeOptions(this.options, options, 'shaded');\n    util.mergeOptions(this.options, options, 'legend');\n\n    if (options.interpolation) {\n      if (typeof options.interpolation == 'object') {\n        if (options.interpolation.parametrization) {\n          if (options.interpolation.parametrization == 'uniform') {\n            this.options.interpolation.alpha = 0;\n          }\n          else if (options.interpolation.parametrization == 'chordal') {\n            this.options.interpolation.alpha = 1.0;\n          }\n          else {\n            this.options.interpolation.parametrization = 'centripetal';\n            this.options.interpolation.alpha = 0.5;\n          }\n        }\n      }\n    }\n\n    if (this.yAxisLeft) {\n      if (options.dataAxis !== undefined) {\n        this.yAxisLeft.setOptions(this.options.dataAxis);\n        this.yAxisRight.setOptions(this.options.dataAxis);\n      }\n    }\n\n    if (this.legendLeft) {\n      if (options.legend !== undefined) {\n        this.legendLeft.setOptions(this.options.legend);\n        this.legendRight.setOptions(this.options.legend);\n      }\n    }\n\n    if (this.groups.hasOwnProperty(UNGROUPED)) {\n      this.groups[UNGROUPED].setOptions(options);\n    }\n  }\n\n  // this is used to redraw the graph if the visibility of the groups is changed.\n  if (this.dom.frame) { //not on initial run?\n    this.forceGraphUpdate=true;\n    this.body.emitter.emit(\"_change\",{queue: true});\n  }\n};\n\n/**\n * Hide the component from the DOM\n */\nLineGraph.prototype.hide = function () {\n  // remove the frame containing the items\n  if (this.dom.frame.parentNode) {\n    this.dom.frame.parentNode.removeChild(this.dom.frame);\n  }\n};\n\n\n/**\n * Show the component in the DOM (when not already visible).\n */\nLineGraph.prototype.show = function () {\n  // show frame containing the items\n  if (!this.dom.frame.parentNode) {\n    this.body.dom.center.appendChild(this.dom.frame);\n  }\n};\n\n\n/**\n * Set items\n * @param {vis.DataSet | null} items\n */\nLineGraph.prototype.setItems = function (items) {\n  var me = this,\n    ids,\n    oldItemsData = this.itemsData;\n\n  // replace the dataset\n  if (!items) {\n    this.itemsData = null;\n  }\n  else if (items instanceof DataSet || items instanceof DataView) {\n    this.itemsData = typeCoerceDataSet(items);\n  }\n  else {\n    throw new TypeError('Data must be an instance of DataSet or DataView');\n  }\n\n  if (oldItemsData) {\n    // unsubscribe from old dataset\n    util.forEach(this.itemListeners, function (callback, event) {\n      oldItemsData.off(event, callback);\n    });\n\n    // stop maintaining a coerced version of the old data set\n    oldItemsData.dispose()\n\n    // remove all drawn items\n    ids = oldItemsData.getIds();\n    this._onRemove(ids);\n  }\n\n  if (this.itemsData) {\n    // subscribe to new dataset\n    var id = this.id;\n    util.forEach(this.itemListeners, function (callback, event) {\n      me.itemsData.on(event, callback, id);\n    });\n\n    // add all new items\n    ids = this.itemsData.getIds();\n    this._onAdd(ids);\n  }\n};\n\n\n/**\n * Set groups\n * @param {vis.DataSet} groups\n */\nLineGraph.prototype.setGroups = function (groups) {\n  var me = this;\n  var ids;\n\n  // unsubscribe from current dataset\n  if (this.groupsData) {\n    util.forEach(this.groupListeners, function (callback, event) {\n      me.groupsData.off(event, callback);\n    });\n\n    // remove all drawn groups\n    ids = this.groupsData.getIds();\n    this.groupsData = null;\n    for (var i = 0; i < ids.length; i++) {\n      this._removeGroup(ids[i]);\n    }\n  }\n\n  // replace the dataset\n  if (!groups) {\n    this.groupsData = null;\n  }\n  else if (groups instanceof DataSet || groups instanceof DataView) {\n    this.groupsData = groups;\n  }\n  else {\n    throw new TypeError('Data must be an instance of DataSet or DataView');\n  }\n\n  if (this.groupsData) {\n    // subscribe to new dataset\n    var id = this.id;\n    util.forEach(this.groupListeners, function (callback, event) {\n      me.groupsData.on(event, callback, id);\n    });\n\n    // draw all ms\n    ids = this.groupsData.getIds();\n    this._onAddGroups(ids);\n  }\n};\n\nLineGraph.prototype._onUpdate = function (ids) {\n  this._updateAllGroupData(ids);\n};\nLineGraph.prototype._onAdd = function (ids) {\n  this._onUpdate(ids);\n};\nLineGraph.prototype._onRemove = function (ids) {\n  this._onUpdate(ids);\n};\nLineGraph.prototype._onUpdateGroups = function (groupIds) {\n  this._updateAllGroupData(null, groupIds);\n};\nLineGraph.prototype._onAddGroups = function (groupIds) {\n  this._onUpdateGroups(groupIds);\n};\n\n/**\n * this cleans the group out off the legends and the dataaxis, updates the ungrouped and updates the graph\n * @param {Array} groupIds\n * @private\n */\nLineGraph.prototype._onRemoveGroups = function (groupIds) {\n  for (var i = 0; i < groupIds.length; i++) {\n    this._removeGroup(groupIds[i]);\n  }\n  this.forceGraphUpdate = true;\n  this.body.emitter.emit(\"_change\",{queue: true});\n};\n\n/**\n * this cleans the group out off the legends and the dataaxis\n * @param {vis.GraphGroup.id} groupId\n * @private\n */\nLineGraph.prototype._removeGroup = function (groupId) {\n  if (this.groups.hasOwnProperty(groupId)) {\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.removeGroup(groupId);\n      this.legendRight.removeGroup(groupId);\n      this.legendRight.redraw();\n    }\n    else {\n      this.yAxisLeft.removeGroup(groupId);\n      this.legendLeft.removeGroup(groupId);\n      this.legendLeft.redraw();\n    }\n    delete this.groups[groupId];\n  }\n};\n\n/**\n * update a group object with the group dataset entree\n *\n * @param {vis.GraphGroup} group\n * @param {vis.GraphGroup.id} groupId\n * @private\n */\nLineGraph.prototype._updateGroup = function (group, groupId) {\n  if (!this.groups.hasOwnProperty(groupId)) {\n    this.groups[groupId] = new GraphGroup(group, groupId, this.options, this.groupsUsingDefaultStyles);\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.addGroup(groupId, this.groups[groupId]);\n      this.legendRight.addGroup(groupId, this.groups[groupId]);\n    }\n    else {\n      this.yAxisLeft.addGroup(groupId, this.groups[groupId]);\n      this.legendLeft.addGroup(groupId, this.groups[groupId]);\n    }\n  }\n  else {\n    this.groups[groupId].update(group);\n    if (this.groups[groupId].options.yAxisOrientation == 'right') {\n      this.yAxisRight.updateGroup(groupId, this.groups[groupId]);\n      this.legendRight.updateGroup(groupId, this.groups[groupId]);\n      //If yAxisOrientation changed, clean out the group from the other axis.\n      this.yAxisLeft.removeGroup(groupId);\n      this.legendLeft.removeGroup(groupId);\n    }\n    else {\n      this.yAxisLeft.updateGroup(groupId, this.groups[groupId]);\n      this.legendLeft.updateGroup(groupId, this.groups[groupId]);\n      //If yAxisOrientation changed, clean out the group from the other axis.\n      this.yAxisRight.removeGroup(groupId);\n      this.legendRight.removeGroup(groupId);\n    }\n  }\n  this.legendLeft.redraw();\n  this.legendRight.redraw();\n};\n\n\n/**\n * this updates all groups, it is used when there is an update the the itemset.\n *\n * @param  {Array} ids\n * @param  {Array} groupIds\n * @private\n */\nLineGraph.prototype._updateAllGroupData = function (ids, groupIds) {\n  if (this.itemsData != null) {\n    var groupsContent = {};\n    var items = this.itemsData.get();\n    var fieldId = this.itemsData.idProp;\n    var idMap = {};\n    if (ids){\n      ids.map(function (id) {\n        idMap[id] = id;\n      });\n    }\n\n    //pre-Determine array sizes, for more efficient memory claim\n    var groupCounts = {};\n    for (var i = 0; i < items.length; i++) {\n      var item = items[i];\n      var groupId = item.group;\n      if (groupId === null || groupId === undefined) {\n        groupId = UNGROUPED;\n      }\n      groupCounts.hasOwnProperty(groupId) ? groupCounts[groupId]++ : groupCounts[groupId] = 1;\n    }\n\n    //Pre-load arrays from existing groups if items are not changed (not in ids)\n    var existingItemsMap = {};\n    if (!groupIds && ids) {\n      for (groupId in this.groups) {\n        if (this.groups.hasOwnProperty(groupId)) {\n          group = this.groups[groupId];\n          var existing_items = group.getItems();\n\n          groupsContent[groupId] = existing_items.filter(function (item) {\n            existingItemsMap[item[fieldId]] = item[fieldId];\n            return (item[fieldId] !== idMap[item[fieldId]]);\n          });\n          var newLength = groupCounts[groupId];\n          groupCounts[groupId] -= groupsContent[groupId].length;\n          if (groupsContent[groupId].length < newLength) {\n            groupsContent[groupId][newLength - 1] = {};\n          }\n        }\n      }\n    }\n\n    //Now insert data into the arrays.\n    for (i = 0; i < items.length; i++) {\n      item = items[i];\n      groupId = item.group;\n      if (groupId === null || groupId === undefined) {\n        groupId = UNGROUPED;\n      }\n      if (!groupIds && ids && (item[fieldId] !== idMap[item[fieldId]]) && existingItemsMap.hasOwnProperty(item[fieldId])) {\n        continue;\n      }\n      if (!groupsContent.hasOwnProperty(groupId)) {\n        groupsContent[groupId] = new Array(groupCounts[groupId]);\n      }\n      //Copy data (because of unmodifiable DataView input.\n      var extended = util.bridgeObject(item);\n      extended.x = util.convert(item.x, 'Date');\n      extended.end = util.convert(item.end, 'Date');\n      extended.orginalY = item.y; //real Y\n      extended.y = Number(item.y);\n      extended[fieldId] = item[fieldId];\n\n      var index= groupsContent[groupId].length - groupCounts[groupId]--;\n      groupsContent[groupId][index] = extended;\n    }\n\n    //Make sure all groups are present, to allow removal of old groups\n    for (groupId in this.groups){\n      if (this.groups.hasOwnProperty(groupId)){\n        if (!groupsContent.hasOwnProperty(groupId)) {\n          groupsContent[groupId] = new Array(0);\n        }\n      }\n    }\n\n    //Update legendas, style and axis\n    for (groupId in groupsContent) {\n      if (groupsContent.hasOwnProperty(groupId)) {\n        if (groupsContent[groupId].length == 0) {\n          if (this.groups.hasOwnProperty(groupId)) {\n            this._removeGroup(groupId);\n          }\n        } else {\n          var group = undefined;\n          if (this.groupsData != undefined) {\n            group = this.groupsData.get(groupId);\n          }\n          if (group == undefined) {\n            group = {id: groupId, content: this.options.defaultGroup + groupId};\n          }\n          this._updateGroup(group, groupId);\n          this.groups[groupId].setItems(groupsContent[groupId]);\n        }\n      }\n    }\n    this.forceGraphUpdate = true;\n    this.body.emitter.emit(\"_change\",{queue: true});\n  }\n};\n\n/**\n * Redraw the component, mandatory function\n * @return {boolean} Returns true if the component is resized\n */\nLineGraph.prototype.redraw = function () {\n  var resized = false;\n\n  // calculate actual size and position\n  this.props.width = this.dom.frame.offsetWidth;\n  this.props.height = this.body.domProps.centerContainer.height\n    - this.body.domProps.border.top\n    - this.body.domProps.border.bottom;\n\n  // check if this component is resized\n  resized = this._isResized() || resized;\n\n  // check whether zoomed (in that case we need to re-stack everything)\n  var visibleInterval = this.body.range.end - this.body.range.start;\n  var zoomed = (visibleInterval != this.lastVisibleInterval);\n  this.lastVisibleInterval = visibleInterval;\n\n\n  // the svg element is three times as big as the width, this allows for fully dragging left and right\n  // without reloading the graph. the controls for this are bound to events in the constructor\n  if (resized == true) {\n    this.svg.style.width = util.option.asSize(3 * this.props.width);\n    this.svg.style.left = util.option.asSize(-this.props.width);\n\n    // if the height of the graph is set as proportional, change the height of the svg\n    if ((this.options.height + '').indexOf(\"%\") != -1 || this.updateSVGheightOnResize == true) {\n      this.updateSVGheight = true;\n    }\n  }\n\n  // update the height of the graph on each redraw of the graph.\n  if (this.updateSVGheight == true) {\n    if (this.options.graphHeight != this.props.height + 'px') {\n      this.options.graphHeight = this.props.height + 'px';\n      this.svg.style.height = this.props.height + 'px';\n    }\n    this.updateSVGheight = false;\n  }\n  else {\n    this.svg.style.height = ('' + this.options.graphHeight).replace('px', '') + 'px';\n  }\n\n  // zoomed is here to ensure that animations are shown correctly.\n  if (resized == true || zoomed == true || this.abortedGraphUpdate == true || this.forceGraphUpdate == true) {\n    resized = this._updateGraph() || resized;\n    this.forceGraphUpdate = false;\n  }\n  else {\n    // move the whole svg while dragging\n    if (this.lastStart != 0) {\n      var offset = this.body.range.start - this.lastStart;\n      var range = this.body.range.end - this.body.range.start;\n      if (this.props.width != 0) {\n        var rangePerPixelInv = this.props.width / range;\n        var xOffset = offset * rangePerPixelInv;\n        this.svg.style.left = (-this.props.width - xOffset) + 'px';\n      }\n    }\n  }\n  this.legendLeft.redraw();\n  this.legendRight.redraw();\n  return resized;\n};\n\n\nLineGraph.prototype._getSortedGroupIds = function(){\n  // getting group Ids\n  var grouplist = [];\n  for (var groupId in this.groups) {\n    if (this.groups.hasOwnProperty(groupId)) {\n      var group = this.groups[groupId];\n      if (group.visible == true && (this.options.groups.visibility[groupId] === undefined || this.options.groups.visibility[groupId] == true)) {\n        grouplist.push({id:groupId,zIndex:group.options.zIndex});\n      }\n    }\n  }\n  util.insertSort(grouplist,function(a,b){\n    var az = a.zIndex;\n    var bz = b.zIndex;\n    if (az === undefined) az=0;\n    if (bz === undefined) bz=0;\n    return az==bz? 0: (az<bz ? -1: 1);\n  });\n  var groupIds = new Array(grouplist.length);\n  for (var i=0; i< grouplist.length; i++){\n    groupIds[i] = grouplist[i].id;\n  }\n  return groupIds;\n};\n\n/**\n * Update and redraw the graph.\n *\n * @returns {boolean}\n * @private\n */\nLineGraph.prototype._updateGraph = function () {\n  // reset the svg elements\n  DOMutil.prepareElements(this.svgElements);\n  if (this.props.width != 0 && this.itemsData != null) {\n    var group, i;\n    var groupRanges = {};\n    var changeCalled = false;\n    // this is the range of the SVG canvas\n    var minDate = this.body.util.toGlobalTime(-this.body.domProps.root.width);\n    var maxDate = this.body.util.toGlobalTime(2 * this.body.domProps.root.width);\n\n    // getting group Ids\n    var groupIds = this._getSortedGroupIds();\n    if (groupIds.length > 0) {\n      var groupsData = {};\n\n      // fill groups data, this only loads the data we require based on the timewindow\n      this._getRelevantData(groupIds, groupsData, minDate, maxDate);\n\n      // apply sampling, if disabled, it will pass through this function.\n      this._applySampling(groupIds, groupsData);\n\n      // we transform the X coordinates to detect collisions\n      for (i = 0; i < groupIds.length; i++) {\n        this._convertXcoordinates(groupsData[groupIds[i]]);\n      }\n\n      // now all needed data has been collected we start the processing.\n      this._getYRanges(groupIds, groupsData, groupRanges);\n\n      // update the Y axis first, we use this data to draw at the correct Y points\n      changeCalled = this._updateYAxis(groupIds, groupRanges);\n\n      //  at changeCalled, abort this update cycle as the graph needs another update with new Width input from the Redraw container.\n      //  Cleanup SVG elements on abort.\n      if (changeCalled == true) {\n        DOMutil.cleanupElements(this.svgElements);\n        this.abortedGraphUpdate = true;\n        return true;\n      }\n      this.abortedGraphUpdate = false;\n\n      // With the yAxis scaled correctly, use this to get the Y values of the points.\n      var below = undefined;\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (this.options.stack === true && this.options.style === 'line') {\n          if (group.options.excludeFromStacking == undefined || !group.options.excludeFromStacking) {\n            if (below != undefined) {\n              this._stack(groupsData[group.id], groupsData[below.id]);\n              if (group.options.shaded.enabled == true && group.options.shaded.orientation !== \"group\"){\n                if (group.options.shaded.orientation == \"top\" && below.options.shaded.orientation !== \"group\"){\n                  below.options.shaded.orientation=\"group\";\n                  below.options.shaded.groupId=group.id;\n                } else {\n                  group.options.shaded.orientation=\"group\";\n                  group.options.shaded.groupId=below.id;\n                }\n              }\n            }\n            below = group;\n          }\n        }\n        this._convertYcoordinates(groupsData[groupIds[i]], group);\n      }\n\n      //Precalculate paths and draw shading if appropriate. This will make sure the shading is always behind any lines.\n      var paths = {};\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (group.options.style === 'line' && group.options.shaded.enabled == true) {\n          var dataset = groupsData[groupIds[i]];\n          if (dataset == null || dataset.length == 0) {\n            continue;\n          }\n          if (!paths.hasOwnProperty(groupIds[i])) {\n            paths[groupIds[i]] = Lines.calcPath(dataset, group);\n          }\n          if (group.options.shaded.orientation === \"group\") {\n            var subGroupId = group.options.shaded.groupId;\n            if (groupIds.indexOf(subGroupId) === -1) {\n              console.log(group.id + \": Unknown shading group target given:\" + subGroupId);\n              continue;\n            }\n            if (!paths.hasOwnProperty(subGroupId)) {\n              paths[subGroupId] = Lines.calcPath(groupsData[subGroupId], this.groups[subGroupId]);\n            }\n            Lines.drawShading(paths[groupIds[i]], group, paths[subGroupId], this.framework);\n          }\n          else {\n            Lines.drawShading(paths[groupIds[i]], group, undefined, this.framework);\n          }\n        }\n      }\n\n      // draw the groups, calculating paths if still necessary.\n      Bars.draw(groupIds, groupsData, this.framework);\n      for (i = 0; i < groupIds.length; i++) {\n        group = this.groups[groupIds[i]];\n        if (groupsData[groupIds[i]].length > 0) {\n          switch (group.options.style) {\n            case \"line\":\n              if (!paths.hasOwnProperty(groupIds[i])) {\n                paths[groupIds[i]] = Lines.calcPath(groupsData[groupIds[i]], group);\n              }\n              Lines.draw(paths[groupIds[i]], group, this.framework);\n            // eslint-disable-line no-fallthrough\n            case \"point\":\n            // eslint-disable-line no-fallthrough\n            case \"points\":\n              if (group.options.style == \"point\" || group.options.style == \"points\" || group.options.drawPoints.enabled == true) {\n                Points.draw(groupsData[groupIds[i]], group, this.framework);\n              }\n              break;\n            case \"bar\":\n            // bar needs to be drawn enmasse\n            // eslint-disable-line no-fallthrough\n            default:\n            //do nothing...\n          }\n        }\n\n      }\n    }\n  }\n\n  // cleanup unused svg elements\n  DOMutil.cleanupElements(this.svgElements);\n  return false;\n};\n\nLineGraph.prototype._stack = function (data, subData) {\n  var index, dx, dy, subPrevPoint, subNextPoint;\n  index = 0;\n  // for each data point we look for a matching on in the set below\n  for (var j = 0; j < data.length; j++) {\n    subPrevPoint = undefined;\n    subNextPoint = undefined;\n    // we look for time matches or a before-after point\n    for (var k = index; k < subData.length; k++) {\n      // if times match exactly\n      if (subData[k].x === data[j].x) {\n        subPrevPoint = subData[k];\n        subNextPoint = subData[k];\n        index = k;\n        break;\n      }\n      else if (subData[k].x > data[j].x) { // overshoot\n        subNextPoint = subData[k];\n        if (k == 0) {\n          subPrevPoint = subNextPoint;\n        }\n        else {\n          subPrevPoint = subData[k - 1];\n        }\n        index = k;\n        break;\n      }\n    }\n    // in case the last data point has been used, we assume it stays like this.\n    if (subNextPoint === undefined) {\n      subPrevPoint = subData[subData.length - 1];\n      subNextPoint = subData[subData.length - 1];\n    }\n    // linear interpolation\n    dx = subNextPoint.x - subPrevPoint.x;\n    dy = subNextPoint.y - subPrevPoint.y;\n    if (dx == 0) {\n      data[j].y = data[j].orginalY + subNextPoint.y;\n    }\n    else {\n      data[j].y = data[j].orginalY + (dy / dx) * (data[j].x - subPrevPoint.x) + subPrevPoint.y; // ax + b where b is data[j].y\n    }\n  }\n}\n\n\n/**\n * first select and preprocess the data from the datasets.\n * the groups have their preselection of data, we now loop over this data to see\n * what data we need to draw. Sorted data is much faster.\n * more optimization is possible by doing the sampling before and using the binary search\n * to find the end date to determine the increment.\n *\n * @param {array}  groupIds\n * @param {object} groupsData\n * @param {date}   minDate\n * @param {date}   maxDate\n * @private\n */\nLineGraph.prototype._getRelevantData = function (groupIds, groupsData, minDate, maxDate) {\n  var group, i, j, item;\n  if (groupIds.length > 0) {\n    for (i = 0; i < groupIds.length; i++) {\n      group = this.groups[groupIds[i]];\n      var itemsData = group.getItems();\n      // optimization for sorted data\n      if (group.options.sort == true) {\n        var dateComparator = function (a, b) {\n          return a.getTime() == b.getTime() ? 0 : a < b ? -1 : 1\n        };\n        var first = Math.max(0, util.binarySearchValue(itemsData, minDate, 'x', 'before', dateComparator));\n        var last = Math.min(itemsData.length, util.binarySearchValue(itemsData, maxDate, 'x', 'after', dateComparator) + 1);\n        if (last <= 0) {\n          last = itemsData.length;\n        }\n        var dataContainer = new Array(last-first);\n        for (j = first; j < last; j++) {\n          item = group.itemsData[j];\n          dataContainer[j-first] = item;\n        }\n        groupsData[groupIds[i]] = dataContainer;\n      }\n      else {\n        // If unsorted data, all data is relevant, just returning entire structure\n        groupsData[groupIds[i]] = group.itemsData;\n      }\n    }\n  }\n};\n\n\n/**\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {vis.DataSet} groupsData\n * @private\n */\nLineGraph.prototype._applySampling = function (groupIds, groupsData) {\n  var group;\n  if (groupIds.length > 0) {\n    for (var i = 0; i < groupIds.length; i++) {\n      group = this.groups[groupIds[i]];\n      if (group.options.sampling == true) {\n        var dataContainer = groupsData[groupIds[i]];\n        if (dataContainer.length > 0) {\n          var increment = 1;\n          var amountOfPoints = dataContainer.length;\n\n          // the global screen is used because changing the width of the yAxis may affect the increment, resulting in an endless loop\n          // of width changing of the yAxis.\n          //TODO: This assumes sorted data, but that's not guaranteed!\n          var xDistance = this.body.util.toGlobalScreen(dataContainer[dataContainer.length - 1].x) - this.body.util.toGlobalScreen(dataContainer[0].x);\n          var pointsPerPixel = amountOfPoints / xDistance;\n          increment = Math.min(Math.ceil(0.2 * amountOfPoints), Math.max(1, Math.round(pointsPerPixel)));\n\n          var sampledData = new Array(amountOfPoints);\n          for (var j = 0; j < amountOfPoints; j += increment) {\n            var idx = Math.round(j/increment);\n            sampledData[idx]=dataContainer[j];\n          }\n          groupsData[groupIds[i]] = sampledData.splice(0,Math.round(amountOfPoints/increment));\n        }\n      }\n    }\n  }\n};\n\n\n/**\n *\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {vis.DataSet} groupsData\n * @param {object} groupRanges  | this is being filled here\n * @private\n */\nLineGraph.prototype._getYRanges = function (groupIds, groupsData, groupRanges) {\n  var groupData, group, i;\n  var combinedDataLeft = [];\n  var combinedDataRight = [];\n  var options;\n  if (groupIds.length > 0) {\n    for (i = 0; i < groupIds.length; i++) {\n      groupData = groupsData[groupIds[i]];\n      options = this.groups[groupIds[i]].options;\n      if (groupData.length > 0) {\n        group = this.groups[groupIds[i]];\n        // if bar graphs are stacked, their range need to be handled differently and accumulated over all groups.\n        if (options.stack === true && options.style === 'bar') {\n          if (options.yAxisOrientation === 'left') {\n            combinedDataLeft = combinedDataLeft.concat(groupData);\n          }\n          else {\n            combinedDataRight = combinedDataRight.concat(groupData);\n          }\n        }\n        else {\n          groupRanges[groupIds[i]] = group.getYRange(groupData, groupIds[i]);\n        }\n      }\n    }\n\n    // if bar graphs are stacked, their range need to be handled differently and accumulated over all groups.\n    Bars.getStackedYRange(combinedDataLeft, groupRanges, groupIds, '__barStackLeft', 'left');\n    Bars.getStackedYRange(combinedDataRight, groupRanges, groupIds, '__barStackRight', 'right');\n  }\n};\n\n\n/**\n * this sets the Y ranges for the Y axis. It also determines which of the axis should be shown or hidden.\n * @param {Array.<vis.GraphGroup.id>} groupIds\n * @param {Object} groupRanges\n * @returns {boolean} resized\n * @private\n */\nLineGraph.prototype._updateYAxis = function (groupIds, groupRanges) {\n  var resized = false;\n  var yAxisLeftUsed = false;\n  var yAxisRightUsed = false;\n  var minLeft = 1e9, minRight = 1e9, maxLeft = -1e9, maxRight = -1e9, minVal, maxVal;\n  // if groups are present\n  if (groupIds.length > 0) {\n    // this is here to make sure that if there are no items in the axis but there are groups, that there is no infinite draw/redraw loop.\n    for (var i = 0; i < groupIds.length; i++) {\n      var group = this.groups[groupIds[i]];\n      if (group && group.options.yAxisOrientation != 'right') {\n        yAxisLeftUsed = true;\n        minLeft = 1e9;\n        maxLeft = -1e9;\n      }\n      else if (group && group.options.yAxisOrientation) {\n        yAxisRightUsed = true;\n        minRight = 1e9;\n        maxRight = -1e9;\n      }\n    }\n\n    // if there are items:\n    for (i = 0; i < groupIds.length; i++) {\n      if (groupRanges.hasOwnProperty(groupIds[i])) {\n        if (groupRanges[groupIds[i]].ignore !== true) {\n          minVal = groupRanges[groupIds[i]].min;\n          maxVal = groupRanges[groupIds[i]].max;\n\n          if (groupRanges[groupIds[i]].yAxisOrientation != 'right') {\n            yAxisLeftUsed = true;\n            minLeft = minLeft > minVal ? minVal : minLeft;\n            maxLeft = maxLeft < maxVal ? maxVal : maxLeft;\n          }\n          else {\n            yAxisRightUsed = true;\n            minRight = minRight > minVal ? minVal : minRight;\n            maxRight = maxRight < maxVal ? maxVal : maxRight;\n          }\n        }\n      }\n    }\n\n    if (yAxisLeftUsed == true) {\n      this.yAxisLeft.setRange(minLeft, maxLeft);\n    }\n    if (yAxisRightUsed == true) {\n      this.yAxisRight.setRange(minRight, maxRight);\n    }\n  }\n  resized = this._toggleAxisVisiblity(yAxisLeftUsed, this.yAxisLeft) || resized;\n  resized = this._toggleAxisVisiblity(yAxisRightUsed, this.yAxisRight) || resized;\n\n  if (yAxisRightUsed == true && yAxisLeftUsed == true) {\n    this.yAxisLeft.drawIcons = true;\n    this.yAxisRight.drawIcons = true;\n  }\n  else {\n    this.yAxisLeft.drawIcons = false;\n    this.yAxisRight.drawIcons = false;\n  }\n  this.yAxisRight.master = !yAxisLeftUsed;\n  this.yAxisRight.masterAxis = this.yAxisLeft;\n\n  if (this.yAxisRight.master == false) {\n    if (yAxisRightUsed == true) {\n      this.yAxisLeft.lineOffset = this.yAxisRight.width;\n    }\n    else {\n      this.yAxisLeft.lineOffset = 0;\n    }\n\n    resized = this.yAxisLeft.redraw() || resized;\n    resized = this.yAxisRight.redraw() || resized;\n  }\n  else {\n    resized = this.yAxisRight.redraw() || resized;\n  }\n\n  // clean the accumulated lists\n  var tempGroups = ['__barStackLeft', '__barStackRight', '__lineStackLeft', '__lineStackRight'];\n  for (i = 0; i < tempGroups.length; i++) {\n    if (groupIds.indexOf(tempGroups[i]) != -1) {\n      groupIds.splice(groupIds.indexOf(tempGroups[i]), 1);\n    }\n  }\n\n  return resized;\n};\n\n\n/**\n * This shows or hides the Y axis if needed. If there is a change, the changed event is emitted by the updateYAxis function\n *\n * @param {boolean} axisUsed\n * @param {vis.DataAxis}  axis\n * @returns {boolean}\n * @private\n */\nLineGraph.prototype._toggleAxisVisiblity = function (axisUsed, axis) {\n  var changed = false;\n  if (axisUsed == false) {\n    if (axis.dom.frame.parentNode && axis.hidden == false) {\n      axis.hide();\n      changed = true;\n    }\n  }\n  else {\n    if (!axis.dom.frame.parentNode && axis.hidden == true) {\n      axis.show();\n      changed = true;\n    }\n  }\n  return changed;\n};\n\n\n/**\n * This uses the DataAxis object to generate the correct X coordinate on the SVG window. It uses the\n * util function toScreen to get the x coordinate from the timestamp. It also pre-filters the data and get the minMax ranges for\n * the yAxis.\n *\n * @param {Array.<Object>} datapoints\n * @private\n */\nLineGraph.prototype._convertXcoordinates = function (datapoints) {\n  var toScreen = this.body.util.toScreen;\n  for (var i = 0; i < datapoints.length; i++) {\n    datapoints[i].screen_x = toScreen(datapoints[i].x) + this.props.width;\n    datapoints[i].screen_y = datapoints[i].y; //starting point for range calculations\n    if (datapoints[i].end != undefined) {\n      datapoints[i].screen_end = toScreen(datapoints[i].end) + this.props.width;\n    }\n    else {\n      datapoints[i].screen_end = undefined;\n    }\n  }\n};\n\n\n/**\n * This uses the DataAxis object to generate the correct X coordinate on the SVG window. It uses the\n * util function toScreen to get the x coordinate from the timestamp. It also pre-filters the data and get the minMax ranges for\n * the yAxis.\n *\n * @param {Array.<Object>} datapoints\n * @param {vis.GraphGroup} group\n * @private\n */\nLineGraph.prototype._convertYcoordinates = function (datapoints, group) {\n  var axis = this.yAxisLeft;\n  var svgHeight = Number(this.svg.style.height.replace('px', ''));\n  if (group.options.yAxisOrientation == 'right') {\n    axis = this.yAxisRight;\n  }\n  for (var i = 0; i < datapoints.length; i++) {\n    datapoints[i].screen_y = Math.round(axis.convertValue(datapoints[i].y));\n  }\n  group.setZeroPosition(Math.min(svgHeight, axis.convertValue(0)));\n};\n\n\nexport default LineGraph;\n","/**\n * This object contains all possible options. It will check if the types are correct, if required if the option is one\n * of the allowed values.\n *\n * __any__ means that the name of the property does not matter.\n * __type__ is a required field for all objects and contains the allowed types of all objects\n */\nlet string = 'string';\nlet bool = 'boolean';\nlet number = 'number';\nlet array = 'array';\nlet date = 'date';\nlet object = 'object'; // should only be in a __type__ property\nlet dom = 'dom';\nlet moment = 'moment';\nlet any = 'any';\n\n\nlet allOptions = {\n  configure: {\n    enabled: {'boolean': bool},\n    filter: {'boolean': bool,'function': 'function'},\n    container: {dom},\n    __type__: {object,'boolean': bool,'function': 'function'}\n  },\n\n  //globals :\n  alignCurrentTime: {string, 'undefined': 'undefined'},  \n  yAxisOrientation: {string:['left','right']},\n  defaultGroup: {string},\n  sort: {'boolean': bool},\n  sampling: {'boolean': bool},\n  stack:{'boolean': bool},\n  graphHeight: {string, number},\n  shaded: {\n    enabled: {'boolean': bool},\n    orientation: {string:['bottom','top','zero','group']}, // top, bottom, zero, group\n    groupId: {object},\n    __type__: {'boolean': bool,object}\n  },\n  style: {string:['line','bar','points']}, // line, bar\n  barChart: {\n    width: {number},\n    minWidth: {number},\n    sideBySide: {'boolean': bool},\n    align: {string:['left','center','right']},\n    __type__: {object}\n  },\n  interpolation: {\n    enabled: {'boolean': bool},\n    parametrization: {string:['centripetal', 'chordal','uniform']}, // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n    alpha: {number},\n    __type__: {object,'boolean': bool}\n  },\n  drawPoints: {\n    enabled: {'boolean': bool},\n    onRender: { 'function': 'function' },\n    size: {number},\n    style: {string:['square','circle']}, // square, circle\n    __type__: {object,'boolean': bool,'function': 'function'}\n  },\n  dataAxis: {\n    showMinorLabels: {'boolean': bool},\n    showMajorLabels: {'boolean': bool},\n    showWeekScale: {'boolean': bool},\n    icons: {'boolean': bool},\n    width: {string, number},\n    visible: {'boolean': bool},\n    alignZeros: {'boolean': bool},\n    left:{\n      range: {min:{number,'undefined': 'undefined'},max:{number,'undefined': 'undefined'},__type__: {object}},\n      format: {'function': 'function'},\n      title: {text:{string,number,'undefined': 'undefined'},style:{string,'undefined': 'undefined'},__type__: {object}},\n      __type__: {object}\n    },\n    right:{\n      range: {min:{number,'undefined': 'undefined'},max:{number,'undefined': 'undefined'},__type__: {object}},\n      format: {'function': 'function'},\n      title: {text:{string,number,'undefined': 'undefined'},style:{string,'undefined': 'undefined'},__type__: {object}},\n      __type__: {object}\n    },\n    __type__: {object}\n  },\n  legend: {\n    enabled: {'boolean': bool},\n    icons: {'boolean': bool},\n    left: {\n      visible: {'boolean': bool},\n      position: {string:['top-right','bottom-right','top-left','bottom-left']},\n      __type__: {object}\n    },\n    right: {\n      visible: {'boolean': bool},\n      position: {string:['top-right','bottom-right','top-left','bottom-left']},\n      __type__: {object}\n    },\n    __type__: {object,'boolean': bool}\n  },\n  groups: {\n    visibility: {any},\n    __type__: {object}\n  },\n\n  autoResize: {'boolean': bool},\n  throttleRedraw: {number}, // TODO: DEPRICATED see https://github.com/almende/vis/issues/2511\n  clickToUse: {'boolean': bool},\n  end: {number, date, string, moment},\n  format: {\n    minorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      quarter: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object}\n    },\n    majorLabels: {\n      millisecond: {string,'undefined': 'undefined'},\n      second: {string,'undefined': 'undefined'},\n      minute: {string,'undefined': 'undefined'},\n      hour: {string,'undefined': 'undefined'},\n      weekday: {string,'undefined': 'undefined'},\n      day: {string,'undefined': 'undefined'},\n      week: {string,'undefined': 'undefined'},\n      month: {string,'undefined': 'undefined'},\n      quarter: {string,'undefined': 'undefined'},\n      year: {string,'undefined': 'undefined'},\n      __type__: {object}\n    },\n    __type__: {object}\n  },\n  moment: {'function': 'function'},\n  height: {string, number},\n  hiddenDates: {\n    start: {date, number, string, moment},\n    end: {date, number, string, moment},\n    repeat: {string},\n    __type__: {object, array}\n  },\n  locale:{string},\n  locales:{\n    __any__: {any},\n    __type__: {object}\n  },\n  max: {date, number, string, moment},\n  maxHeight: {number, string},\n  maxMinorChars: {number},\n  min: {date, number, string, moment},\n  minHeight: {number, string},\n  moveable: {'boolean': bool},\n  multiselect: {'boolean': bool},\n  orientation: {string},\n  showCurrentTime: {'boolean': bool},\n  showMajorLabels: {'boolean': bool},\n  showMinorLabels: {'boolean': bool},\n  showWeekScale: {'boolean': bool},\n  start: {date, number, string, moment},\n  timeAxis: {\n    scale: {string,'undefined': 'undefined'},\n    step: {number,'undefined': 'undefined'},\n    __type__: {object}\n  },\n  width: {string, number},\n  zoomable: {'boolean': bool},\n  zoomKey: {string: ['ctrlKey', 'altKey', 'metaKey', '']},\n  zoomMax: {number},\n  zoomMin: {number},\n  zIndex: {number},\n  __type__: {object}\n};\n\nlet configureOptions = {\n  global: {\n    alignCurrentTime: ['none', 'year', 'month', 'quarter', 'week', 'isoWeek', 'day', 'date', 'hour', 'minute', 'second'],   \n    //yAxisOrientation: ['left','right'], // TDOO: enable as soon as Grahp2d doesn't crash when changing this on the fly\n    sort: true,\n    sampling: true,\n    stack:false,\n    shaded: {\n      enabled: false,\n      orientation: ['zero','top','bottom','group'] // zero, top, bottom\n    },\n    style: ['line','bar','points'], // line, bar\n    barChart: {\n      width: [50,5,100,5],\n      minWidth: [50,5,100,5],\n      sideBySide: false,\n      align: ['left','center','right'] // left, center, right\n    },\n    interpolation: {\n      enabled: true,\n      parametrization: ['centripetal','chordal','uniform'] // uniform (alpha = 0.0), chordal (alpha = 1.0), centripetal (alpha = 0.5)\n    },\n    drawPoints: {\n      enabled: true,\n      size: [6,2,30,1],\n      style: ['square', 'circle'] // square, circle\n    },\n    dataAxis: {\n      showMinorLabels: true,\n      showMajorLabels: true,\n      showWeekScale: false,\n      icons: false,\n      width: [40,0,200,1],\n      visible: true,\n      alignZeros: true,\n      left:{\n        //range: {min:'undefined': 'undefined'ined,max:'undefined': 'undefined'ined},\n        //format: function (value) {return value;},\n        title: {text:'',style:''}\n      },\n      right:{\n        //range: {min:'undefined': 'undefined'ined,max:'undefined': 'undefined'ined},\n        //format: function (value) {return value;},\n        title: {text:'',style:''}\n      }\n    },\n    legend: {\n      enabled: false,\n      icons: true,\n      left: {\n        visible: true,\n        position: ['top-right','bottom-right','top-left','bottom-left'] // top/bottom - left,right\n      },\n      right: {\n        visible: true,\n        position: ['top-right','bottom-right','top-left','bottom-left'] // top/bottom - left,right\n      }\n    },\n\n    autoResize: true,\n    clickToUse: false,\n    end: '',\n    format: {\n      minorLabels: {\n        millisecond:'SSS',\n        second:     's',\n        minute:     'HH:mm',\n        hour:       'HH:mm',\n        weekday:    'ddd D',\n        day:        'D',\n        week:       'w',\n        month:      'MMM',\n        quarter:    '[Q]Q',\n        year:       'YYYY'\n      },\n      majorLabels: {\n        millisecond:'HH:mm:ss',\n        second:     'D MMMM HH:mm',\n        minute:     'ddd D MMMM',\n        hour:       'ddd D MMMM',\n        weekday:    'MMMM YYYY',\n        day:        'MMMM YYYY',\n        week:       'MMMM YYYY',\n        month:      'YYYY',\n        quarter:    'YYYY',\n        year:       ''\n      }\n    },\n\n    height: '',\n    locale: '',\n    max: '',\n    maxHeight: '',\n    maxMinorChars: [7, 0, 20, 1],\n    min: '',\n    minHeight: '',\n    moveable:true,\n    orientation: ['both', 'bottom', 'top'],\n    showCurrentTime: false,\n    showMajorLabels: true,\n    showMinorLabels: true,\n    showWeekScale: false,\n    start: '',\n    width: '100%',\n    zoomable: true,\n    zoomKey: ['ctrlKey', 'altKey', 'metaKey', ''],\n    zoomMax: [315360000000000, 10, 315360000000000, 1],\n    zoomMin: [10, 10, 315360000000000, 1],\n    zIndex: 0\n  }\n};\n\nexport {allOptions, configureOptions};\n","import moment from '../module/moment';\nimport util, { typeCoerceDataSet } from '../util';\nimport { DataSet }  from 'vis-data/esnext';\nimport { DataView } from 'vis-data/esnext';\nimport Range from './Range';\nimport Core from './Core';\nimport TimeAxis from './component/TimeAxis';\nimport CurrentTime from './component/CurrentTime';\nimport CustomTime from './component/CustomTime';\nimport LineGraph from './component/LineGraph';\n\nimport { Validator } from '../shared/Validator';\nimport { printStyle } from '../shared/Validator';\nimport { allOptions } from './optionsGraph2d';\nimport { configureOptions } from './optionsGraph2d';\n\nimport Configurator from '../shared/Configurator';\n\n/**\n * Create a timeline visualization\n * @param {HTMLElement} container\n * @param {vis.DataSet | Array} [items]\n * @param {vis.DataSet | Array | vis.DataView | Object} [groups]\n * @param {Object} [options]  See Graph2d.setOptions for the available options.\n * @constructor Graph2d\n * @extends Core\n */\nfunction Graph2d (container, items, groups, options) {\n  // if the third element is options, the forth is groups (optionally);\n  if (!(Array.isArray(groups) || groups instanceof DataSet || groups instanceof DataView) && groups instanceof Object) {\n    var forthArgument = options;\n    options = groups;\n    groups = forthArgument;\n  }\n\n  // TODO: REMOVE THIS in the next MAJOR release\n  // see https://github.com/almende/vis/issues/2511\n  if (options && options.throttleRedraw) {\n    console.warn(\"Graph2d option \\\"throttleRedraw\\\" is DEPRICATED and no longer supported. It will be removed in the next MAJOR release.\");\n  }\n\n  var me = this;\n  this.defaultOptions = {\n    start: null,\n    end:   null,\n\n    autoResize: true,\n\n    orientation: {\n      axis: 'bottom',   // axis orientation: 'bottom', 'top', or 'both'\n      item: 'bottom'    // not relevant for Graph2d\n    },\n\n    moment: moment,\n\n    width: null,\n    height: null,\n    maxHeight: null,\n    minHeight: null\n  };\n  this.options = util.deepExtend({}, this.defaultOptions);\n\n  // Create the DOM, props, and emitter\n  this._create(container);\n\n  // all components listed here will be repainted automatically\n  this.components = [];\n\n  this.body = {\n    dom: this.dom,\n    domProps: this.props,\n    emitter: {\n      on: this.on.bind(this),\n      off: this.off.bind(this),\n      emit: this.emit.bind(this)\n    },\n    hiddenDates: [],\n    util: {\n      toScreen: me._toScreen.bind(me),\n      toGlobalScreen: me._toGlobalScreen.bind(me), // this refers to the root.width\n      toTime: me._toTime.bind(me),\n      toGlobalTime : me._toGlobalTime.bind(me)\n    }\n  };\n\n  // range\n  this.range = new Range(this.body);\n  this.components.push(this.range);\n  this.body.range = this.range;\n\n  // time axis\n  this.timeAxis = new TimeAxis(this.body);\n  this.components.push(this.timeAxis);\n  //this.body.util.snap = this.timeAxis.snap.bind(this.timeAxis);\n\n  // current time bar\n  this.currentTime = new CurrentTime(this.body);\n  this.components.push(this.currentTime);\n\n  // item set\n  this.linegraph = new LineGraph(this.body);\n\n  this.components.push(this.linegraph);\n\n  this.itemsData = null;      // DataSet\n  this.groupsData = null;     // DataSet\n\n\n  this.on('tap', function (event) {\n    me.emit('click', me.getEventProperties(event))\n  });\n  this.on('doubletap', function (event) {\n    me.emit('doubleClick', me.getEventProperties(event))\n  });\n  this.dom.root.oncontextmenu = function (event) {\n    me.emit('contextmenu', me.getEventProperties(event))\n  };\n  \n  //Single time autoscale/fit\n  this.initialFitDone = false;\n  this.on('changed', function (){\n    if (me.itemsData == null) return;\n    if (!me.initialFitDone && !me.options.rollingMode) {\n      me.initialFitDone = true;\n      if (me.options.start != undefined || me.options.end != undefined) {\n        if (me.options.start == undefined || me.options.end == undefined) {\n          var range = me.getItemRange();\n        }\n\n        var start = me.options.start != undefined ? me.options.start : range.min;\n        var end   = me.options.end   != undefined ? me.options.end   : range.max;\n        me.setWindow(start, end, {animation: false});\n      } else {\n        me.fit({animation: false});\n      }\n    }\n\n    if (!me.initialDrawDone && (me.initialRangeChangeDone || (!me.options.start && !me.options.end) \n      || me.options.rollingMode)) {\n      me.initialDrawDone = true;\n      me.dom.root.style.visibility = 'visible';\n      me.dom.loadingScreen.parentNode.removeChild(me.dom.loadingScreen);\n      if (me.options.onInitialDrawComplete) {\n        setTimeout(() => {\n          return me.options.onInitialDrawComplete();\n        }, 0)\n      }\n    }\n  });\n  \n  // apply options\n  if (options) {\n    this.setOptions(options);\n  }\n\n  // IMPORTANT: THIS HAPPENS BEFORE SET ITEMS!\n  if (groups) {\n    this.setGroups(groups);\n  }\n\n  // create itemset\n  if (items) {\n    this.setItems(items);\n  }\n\n  // draw for the first time\n  this._redraw();\n}\n\n// Extend the functionality from Core\nGraph2d.prototype = new Core();\n\nGraph2d.prototype.setOptions = function (options) {\n  // validate options\n  let errorFound = Validator.validate(options, allOptions);\n  if (errorFound === true) {\n    console.log('%cErrors have been found in the supplied options object.', printStyle);\n  }\n\n  Core.prototype.setOptions.call(this, options);\n};\n\n/**\n * Set items\n * @param {vis.DataSet | Array | null} items\n */\nGraph2d.prototype.setItems = function(items) {\n  var initialLoad = (this.itemsData == null);\n\n  // convert to type DataSet when needed\n  var newDataSet;\n  if (!items) {\n    newDataSet = null;\n  }\n  else if (items instanceof DataSet || items instanceof DataView) {\n    newDataSet = typeCoerceDataSet(items);\n  }\n  else {\n    // turn an array into a dataset\n    newDataSet = typeCoerceDataSet(new DataSet(items));\n  }\n\n  // set items\n  if (this.itemsData) {\n    // stop maintaining a coerced version of the old data set\n    this.itemsData.dispose()\n  }\n  this.itemsData = newDataSet;\n  this.linegraph && this.linegraph.setItems(newDataSet != null ? newDataSet.rawDS : null);\n\n  if (initialLoad) {\n    if (this.options.start != undefined || this.options.end != undefined) {\n      var start = this.options.start != undefined ? this.options.start : null;\n      var end   = this.options.end != undefined   ? this.options.end : null;\n      this.setWindow(start, end, {animation: false});\n    }\n    else {\n      this.fit({animation: false});\n    }\n  }\n};\n\n/**\n * Set groups\n * @param {vis.DataSet | Array} groups\n */\nGraph2d.prototype.setGroups = function(groups) {\n  // convert to type DataSet when needed\n  var newDataSet;\n  if (!groups) {\n    newDataSet = null;\n  }\n  else if (groups instanceof DataSet || groups instanceof DataView) {\n    newDataSet = groups;\n  }\n  else {\n    // turn an array into a dataset\n    newDataSet = new DataSet(groups);\n  }\n\n  this.groupsData = newDataSet;\n  this.linegraph.setGroups(newDataSet);\n};\n\n/**\n * Returns an object containing an SVG element with the icon of the group (size determined by iconWidth and iconHeight), the label of the group (content) and the yAxisOrientation of the group (left or right).\n * @param {vis.GraphGroup.id} groupId\n * @param {number} width\n * @param {number} height\n * @returns {{icon: SVGElement, label: string, orientation: string}|string}\n */\nGraph2d.prototype.getLegend = function(groupId, width, height) {\n  if (width  === undefined) {width  = 15;}\n  if (height === undefined) {height = 15;}\n  if (this.linegraph.groups[groupId] !== undefined) {\n    return this.linegraph.groups[groupId].getLegend(width,height);\n  }\n  else {\n    return \"cannot find group:'\" +  groupId + \"'\";\n  }\n};\n\n/**\n * This checks if the visible option of the supplied group (by ID) is true or false.\n * @param {vis.GraphGroup.id} groupId\n * @returns {boolean}\n */\nGraph2d.prototype.isGroupVisible = function(groupId) {\n  if (this.linegraph.groups[groupId] !== undefined) {\n    return (this.linegraph.groups[groupId].visible && (this.linegraph.options.groups.visibility[groupId] === undefined || this.linegraph.options.groups.visibility[groupId] == true));\n  }\n  else {\n    return false;\n  }\n};\n\n\n/**\n * Get the data range of the item set.\n * @returns {{min: Date, max: Date}} range  A range with a start and end Date.\n *                                          When no minimum is found, min==null\n *                                          When no maximum is found, max==null\n */\nGraph2d.prototype.getDataRange = function() {\n  var min = null;\n  var max = null;\n\n  // calculate min from start filed\n  for (var groupId in this.linegraph.groups) {\n    if (this.linegraph.groups.hasOwnProperty(groupId)) {\n      if (this.linegraph.groups[groupId].visible == true) {\n        for (var i = 0; i < this.linegraph.groups[groupId].itemsData.length; i++) {\n          var item = this.linegraph.groups[groupId].itemsData[i];\n          var value = util.convert(item.x, 'Date').valueOf();\n          min = min == null ? value : min > value ? value : min;\n          max = max == null ? value : max < value ? value : max;\n        }\n      }\n    }\n  }\n\n  return {\n    min: (min != null) ? new Date(min) : null,\n    max: (max != null) ? new Date(max) : null\n  };\n};\n\n\n/**\n * Generate Timeline related information from an event\n * @param {Event} event\n * @return {Object} An object with related information, like on which area\n *                  The event happened, whether clicked on an item, etc.\n */\nGraph2d.prototype.getEventProperties = function (event) {\n  var clientX = event.center ? event.center.x : event.clientX;\n  var clientY = event.center ? event.center.y : event.clientY;\n  var x = clientX - util.getAbsoluteLeft(this.dom.centerContainer);\n  var y = clientY - util.getAbsoluteTop(this.dom.centerContainer);\n  var time = this._toTime(x);\n\n  var customTime = CustomTime.customTimeFromTarget(event);\n\n  var element = util.getTarget(event);\n  var what = null;\n  if (util.hasParent(element, this.timeAxis.dom.foreground))              {what = 'axis';}\n  else if (this.timeAxis2 && util.hasParent(element, this.timeAxis2.dom.foreground)) {what = 'axis';}\n  else if (util.hasParent(element, this.linegraph.yAxisLeft.dom.frame))   {what = 'data-axis';}\n  else if (util.hasParent(element, this.linegraph.yAxisRight.dom.frame))  {what = 'data-axis';}\n  else if (util.hasParent(element, this.linegraph.legendLeft.dom.frame))  {what = 'legend';}\n  else if (util.hasParent(element, this.linegraph.legendRight.dom.frame)) {what = 'legend';}\n  else if (customTime != null)                {what = 'custom-time';}\n  else if (util.hasParent(element, this.currentTime.bar))                 {what = 'current-time';}\n  else if (util.hasParent(element, this.dom.center))                      {what = 'background';}\n\n  var value = [];\n  var yAxisLeft = this.linegraph.yAxisLeft;\n  var yAxisRight = this.linegraph.yAxisRight;\n  if (!yAxisLeft.hidden && this.itemsData.length > 0) {\n    value.push(yAxisLeft.screenToValue(y));\n  }\n  if (!yAxisRight.hidden && this.itemsData.length > 0) {\n    value.push(yAxisRight.screenToValue(y));\n  }\n\n  return {\n    event: event,\n    customTime: customTime ? customTime.options.id : null,\n    what: what,\n    pageX: event.srcEvent ? event.srcEvent.pageX : event.pageX,\n    pageY: event.srcEvent ? event.srcEvent.pageY : event.pageY,\n    x: x,\n    y: y,\n    time: time,\n    value: value\n  }\n};\n\n/**\n * Load a configurator\n * @return {Object}\n * @private\n */\nGraph2d.prototype._createConfigurator = function () {\n  return new Configurator(this, this.dom.container, configureOptions);\n};\n\n\nexport default Graph2d;\n","// locales\nimport \"moment/locale/de\";\nimport \"moment/locale/es\";\nimport \"moment/locale/fr\";\nimport \"moment/locale/it\";\nimport \"moment/locale/ja\";\nimport \"moment/locale/nl\";\nimport \"moment/locale/pl\";\nimport \"moment/locale/ru\";\nimport \"moment/locale/uk\";\n\n// utils\nimport * as util from \"vis-util/esnext\";\nimport * as DOMutil from \"./DOMutil\";\n\n// data\nimport { DataSet, DataView, Queue } from \"vis-data/esnext\";\n\n// Timeline\nimport Timeline from \"./timeline/Timeline\";\nimport Graph2d from \"./timeline/Graph2d\";\n\nimport Core from \"./timeline/Core\";\nimport * as DateUtil from \"./timeline/DateUtil\";\nimport Range from \"./timeline/Range\";\nimport * as stack from \"./timeline/Stack\";\nimport TimeStep from \"./timeline/TimeStep\";\n\nimport Item from \"./timeline/component/item/Item\";\nimport BackgroundItem from \"./timeline/component/item/BackgroundItem\";\nimport BoxItem from \"./timeline/component/item/BoxItem\";\nimport PointItem from \"./timeline/component/item/PointItem\";\nimport RangeItem from \"./timeline/component/item/RangeItem\";\n\nimport BackgroundGroup from \"./timeline/component/BackgroundGroup\";\nimport Component from \"./timeline/component/Component\";\nimport CurrentTime from \"./timeline/component/CurrentTime\";\nimport CustomTime from \"./timeline/component/CustomTime\";\nimport DataAxis from \"./timeline/component/DataAxis\";\nimport DataScale from \"./timeline/component/DataScale\";\nimport GraphGroup from \"./timeline/component/GraphGroup\";\nimport Group from \"./timeline/component/Group\";\nimport ItemSet from \"./timeline/component/ItemSet\";\nimport Legend from \"./timeline/component/Legend\";\nimport LineGraph from \"./timeline/component/LineGraph\";\nimport TimeAxis from \"./timeline/component/TimeAxis\";\n\n// bundled external libraries\nimport moment from \"./module/moment\";\nimport Hammer from \"./module/hammer\";\nimport keycharm from \"keycharm\";\n\n// TODO: This should probably be moved somewhere else to ensure that both builds\n// behave the same way.\nconst defaultLanguage = DOMutil.getNavigatorLanguage();\nmoment.locale(defaultLanguage);\n\nconst timeline = {\n  Core,\n  DateUtil,\n  Range,\n  stack,\n  TimeStep,\n\n  components: {\n    items: {\n      Item,\n      BackgroundItem,\n      BoxItem,\n      PointItem,\n      RangeItem\n    },\n\n    BackgroundGroup,\n    Component,\n    CurrentTime,\n    CustomTime,\n    DataAxis,\n    DataScale,\n    GraphGroup,\n    Group,\n    ItemSet,\n    Legend,\n    LineGraph,\n    TimeAxis\n  }\n};\n\nexport {\n  DOMutil,\n  DataSet,\n  DataView,\n  Graph2d,\n  Hammer,\n  Queue,\n  Timeline,\n  keycharm,\n  moment,\n  timeline,\n  util\n};\n"],"names":["module","hookCallback","some","hooks","apply","arguments","isArray","input","Array","Object","prototype","toString","call","isObject","isUndefined","isNumber","isDate","Date","map","arr","fn","i","res","length","push","hasOwnProp","a","b","hasOwnProperty","extend","valueOf","createUTC","format","locale","strict","createLocalOrUTC","utc","getParsingFlags","m","_pf","empty","unusedTokens","unusedInput","overflow","charsLeftOver","nullInput","invalidMonth","invalidFormat","userInvalidated","iso","parsedDateParts","meridiem","rfc2822","weekdayMismatch","isValid","_isValid","flags","parsedParts","isNowValid","isNaN","_d","getTime","invalidWeekday","_strict","undefined","bigHour","isFrozen","createInvalid","NaN","fun","t","this","len","momentProperties","copyConfig","to","from","prop","val","_isAMomentObject","_i","_f","_l","_tzm","_isUTC","_offset","_locale","updateInProgress","Moment","config","updateOffset","isMoment","obj","absFloor","number","Math","ceil","floor","toInt","argumentForCoercion","coercedNumber","value","isFinite","compareArrays","array1","array2","dontConvert","min","lengthDiff","abs","diffs","warn","msg","suppressDeprecationWarnings","console","deprecate","firstTime","deprecationHandler","arg","args","key","slice","join","Error","stack","keys","deprecations","deprecateSimple","name","isFunction","Function","mergeConfigs","parentConfig","childConfig","Locale","set","aliases","addUnitAlias","unit","shorthand","lowerCase","toLowerCase","normalizeUnits","units","normalizeObjectUnits","inputObject","normalizedProp","normalizedInput","priorities","addUnitPriority","priority","zeroFill","targetLength","forceSign","absNumber","zerosToFill","pow","max","substr","formattingTokens","localFormattingTokens","formatFunctions","formatTokenFunctions","addFormatToken","token","padded","ordinal","callback","func","localeData","formatMoment","expandFormat","array","match","replace","mom","output","makeFormatFunction","invalidDate","replaceLongDateFormatTokens","longDateFormat","lastIndex","test","match1","match2","match3","match4","match6","match1to2","match3to4","match5to6","match1to3","match1to4","match1to6","matchUnsigned","matchSigned","matchOffset","matchShortOffset","matchWord","regexes","addRegexToken","regex","strictRegex","isStrict","getParseRegexForToken","RegExp","regexEscape","matched","p1","p2","p3","p4","s","tokens","addParseToken","addWeekParseToken","_w","addTimeToArrayFromToken","_a","daysInYear","year","isLeapYear","y","parseTwoDigitYear","parseInt","indexOf","getSetYear","makeGetSet","keepTime","set$1","get","month","date","daysInMonth","x","modMonth","o","monthsShort","months","monthsShortRegex","monthsRegex","monthsParse","MONTHS_IN_FORMAT","defaultLocaleMonths","split","defaultLocaleMonthsShort","handleStrictParse","monthName","ii","llc","toLocaleLowerCase","_monthsParse","_longMonthsParse","_shortMonthsParse","setMonth","dayOfMonth","getSetMonth","defaultMonthsShortRegex","defaultMonthsRegex","computeMonthsParse","cmpLenRev","shortPieces","longPieces","mixedPieces","sort","_monthsRegex","_monthsShortRegex","_monthsStrictRegex","_monthsShortStrictRegex","createDate","d","h","M","ms","getFullYear","setFullYear","createUTCDate","UTC","getUTCFullYear","setUTCFullYear","firstWeekOffset","dow","doy","fwd","getUTCDay","dayOfYearFromWeeks","week","weekday","resYear","resDayOfYear","dayOfYear","weekOfYear","resWeek","weekOffset","weeksInYear","weekOffsetNext","shiftWeekdays","ws","n","concat","weekdaysMin","weekdaysShort","weekdays","weekdaysMinRegex","weekdaysShortRegex","weekdaysRegex","weekdaysParse","defaultLocaleWeekdays","defaultLocaleWeekdaysShort","defaultLocaleWeekdaysMin","handleStrictParse$1","weekdayName","_weekdaysParse","_shortWeekdaysParse","_minWeekdaysParse","day","defaultWeekdaysRegex","defaultWeekdaysShortRegex","defaultWeekdaysMinRegex","computeWeekdaysParse","minp","shortp","longp","minPieces","_weekdaysRegex","_weekdaysShortRegex","_weekdaysMinRegex","_weekdaysStrictRegex","_weekdaysShortStrictRegex","_weekdaysMinStrictRegex","hFormat","hours","lowercase","minutes","matchMeridiem","_meridiemParse","seconds","kInput","_isPm","isPM","_meridiem","pos","pos1","pos2","globalLocale","getSetHour","baseConfig","calendar","sameDay","nextDay","nextWeek","lastDay","lastWeek","sameElse","LTS","LT","L","LL","LLL","LLLL","dayOfMonthOrdinalParse","relativeTime","future","past","ss","mm","hh","dd","MM","yy","meridiemParse","locales","localeFamilies","normalizeLocale","loadLocale","oldLocale","exports","_abbr","require","getSetGlobalLocale","e","values","data","getLocale","defineLocale","abbr","_config","parentLocale","forEach","names","j","next","chooseLocale","checkOverflow","_overflowDayOfYear","_overflowWeeks","_overflowWeekday","defaults","c","configFromArray","currentDate","expectedWeekday","yearToUse","nowValue","now","_useUTC","getUTCMonth","getUTCDate","getMonth","getDate","currentDateArray","w","weekYear","temp","weekdayOverflow","GG","W","E","createLocal","_week","curWeek","gg","_dayOfYear","dayOfYearFromWeekInfo","_nextDay","getDay","setUTCMinutes","getUTCMinutes","extendedIsoRegex","basicIsoRegex","tzRegex","isoDates","isoTimes","aspNetJsonRegex","configFromISO","l","allowTime","dateFormat","timeFormat","tzFormat","string","exec","configFromStringAndFormat","untruncateYear","yearStr","obsOffsets","UT","GMT","EDT","EST","CDT","CST","MDT","MST","PDT","PST","configFromRFC2822","monthStr","dayStr","hourStr","minuteStr","secondStr","result","parsedArray","weekdayStr","parsedInput","checkWeekday","obsOffset","militaryOffset","numOffset","hm","calculateOffset","ISO_8601","RFC_2822","skipped","stringLength","totalParsedInputLength","hour","isPm","meridiemHour","meridiemFixWrap","prepareConfig","preparse","tempConfig","bestMoment","scoreToBeat","currentScore","score","configFromStringAndArray","createFromInputFallback","configFromString","minute","second","millisecond","configFromObject","configFromInput","isUTC","getOwnPropertyNames","k","isObjectEmpty","add","prototypeMin","other","prototypeMax","pickBy","moments","ordering","Duration","duration","years","quarters","quarter","weeks","isoWeek","days","milliseconds","unitHasDecimal","parseFloat","isDurationValid","_milliseconds","_days","_months","_data","_bubble","isDuration","absRound","round","offset","separator","utcOffset","sign","offsetFromString","chunkOffset","matcher","matches","parts","cloneWithOffset","model","diff","clone","setTime","local","getDateOffset","getTimezoneOffset","isUtc","aspNetRegex","isoRegex","createDuration","ret","diffRes","base","parseIso","isBefore","positiveMomentsDifference","inp","isAfter","createAdder","direction","period","tmp","addSubtract","isAdding","invalid","subtract","monthDiff","wholeMonthDiff","anchor","newLocaleData","defaultFormat","defaultFormatUtc","lang","mod$1","dividend","divisor","localStartOfDate","utcStartOfDate","addWeekYearFormatToken","getter","getSetWeekYearHelper","weeksTarget","setWeekAll","dayOfYearData","isoWeekYear","_dayOfMonthOrdinalParse","_ordinalParse","_dayOfMonthOrdinalParseLenient","getSetDayOfMonth","getSetMinute","getSetSecond","parseMs","getSetMillisecond","proto","preParsePostFormat","time","formats","sod","startOf","calendarFormat","asFloat","that","zoneDelta","endOf","startOfDate","isoWeekday","inputString","postformat","withoutSuffix","humanize","fromNow","toNow","invalidAt","localInput","isBetween","inclusivity","localFrom","localTo","isSame","inputMs","isSameOrAfter","isSameOrBefore","parsingFlags","prioritized","unitsObj","u","getPrioritizedUnits","toArray","toObject","toDate","toISOString","keepOffset","inspect","zone","isLocal","prefix","suffix","toJSON","unix","creationData","isoWeeks","weekInfo","isoWeeksInYear","parseWeekday","parseIsoWeekday","keepLocalTime","keepMinutes","localAdjust","_changeInProgress","parseZone","tZone","hasAlignedHourOffset","isDST","isUtcOffset","zoneAbbr","zoneName","dates","isDSTShifted","_isDSTShifted","proto$1","get$1","index","field","setter","listMonthsImpl","out","listWeekdaysImpl","localeSorted","shift","_calendar","_longDateFormat","formatUpper","toUpperCase","_invalidDate","_ordinal","isFuture","_relativeTime","pastFuture","source","isFormat","_monthsShort","_monthsParseExact","firstDayOfYear","firstDayOfWeek","_weekdays","_weekdaysMin","_weekdaysShort","_weekdaysParseExact","_fullWeekdaysParse","charAt","isLower","langData","mathAbs","addSubtract$1","absCeil","daysToMonths","monthsToDays","makeAs","alias","as","asMilliseconds","asSeconds","asMinutes","asHours","asDays","asWeeks","asMonths","asQuarters","asYears","makeGetter","thresholds","substituteTimeAgo","relativeTime$1","posNegDuration","abs$1","toISOString$1","Y","D","toFixed","total","totalSign","ymSign","daysSign","hmsSign","proto$2","monthsFromDays","withSuffix","toIsoString","version","updateLocale","tmpLocale","relativeTimeRounding","roundingFunction","relativeTimeThreshold","threshold","limit","myMoment","HTML5_FMT","DATETIME_LOCAL","DATETIME_LOCAL_SECONDS","DATETIME_LOCAL_MS","DATE","TIME","TIME_SECONDS","TIME_MS","WEEK","MONTH","factory","check","moment","processRelativeTime","monthsParseExact","weekdaysParseExact","require$$0","monthsShortDot","monthsStrictRegex","monthsShortStrictRegex","longMonthsParse","shortMonthsParse","ll","lll","llll","monthsShortWithDots","monthsShortWithoutDots","monthsNominative","monthsSubjective","plural","translate","momentToFormat","relativeTimeWithPlural","num","forms","standalone","processHoursFunction","str","it","globalThis","window","self","global","error","fails","defineProperty","nativePropertyIsEnumerable","propertyIsEnumerable","getOwnPropertyDescriptor","V","descriptor","enumerable","bitmap","configurable","writable","classof","TypeError","IndexedObject","requireObjectCoercible","PREFERRED_STRING","document","EXISTS","createElement","DESCRIPTORS","nativeGetOwnPropertyDescriptor","O","P","toIndexedObject","toPrimitive","IE8_DOM_DEFINE","has","createPropertyDescriptor","propertyIsEnumerableModule","f","replacement","isForced","feature","detection","normalize","POLYFILL","NATIVE","String","aFunction","nativeDefineProperty","Attributes","anObject","object","definePropertyModule","wrapConstructor","NativeConstructor","Wrapper","options","USE_NATIVE","VIRTUAL_PROTOTYPE","sourceProperty","targetProperty","nativeProperty","resultProperty","TARGET","target","GLOBAL","STATIC","stat","PROTO","nativeSource","path","targetPrototype","forced","noTargetGet","bind","wrap","sham","createNonEnumerableProperty","real","objectDefinePropertyModile","desc","parent","argument","toInteger","integer","createMethod","IS_INCLUDES","$this","el","fromIndex","toLength","toAbsoluteIndex","includes","hiddenKeys","internalObjectKeys","enumBugKeys","defineProperties","Properties","objectKeys","T","variable","namespace","method","getOwnPropertySymbols","getBuiltIn","getOwnPropertyNamesModule","getOwnPropertySymbolsModule","propertyKey","getOwnPropertyDescriptors","getOwnPropertyDescriptorModule","ownKeys","createProperty","FAILS_ON_PRIMITIVES","activeXDocument","Symbol","NATIVE_SYMBOL","iterator","setGlobal","store","mode","copyright","id","postfix","random","shared","uid","IE_PROTO","sharedKey","EmptyConstructor","scriptTag","content","NullProtoObject","domain","ActiveXObject","iframeDocument","iframe","write","close","parentWindow","NullProtoObjectViaActiveX","documentCreateElement","style","display","html","appendChild","src","contentWindow","open","F","create","nativeGetOwnPropertyNames","windowNames","getWindowNames","WellKnownSymbolsStore","createWellKnownSymbol","USE_SYMBOL_AS_UID","withoutSetter","wellKnownSymbol","NAME","wrappedWellKnownSymbolModule","TO_STRING_TAG","CORRECT_ARGUMENTS","classofRaw","TO_STRING_TAG_SUPPORT","tag","tryGet","callee","TAG","SET_METHOD","functionToString","inspectSource","WeakMap","NATIVE_WEAK_MAP","wmget","wmhas","wmset","metadata","STATE","objectHas","enforce","getterFor","TYPE","state","type","SPECIES","originalArray","C","constructor","IS_MAP","IS_FILTER","IS_SOME","IS_EVERY","IS_FIND_INDEX","NO_HOLES","callbackfn","specificCreate","boundFunction","arraySpeciesCreate","filter","every","find","findIndex","$forEach","HIDDEN","TO_PRIMITIVE","setInternalState","InternalStateModule","getInternalState","ObjectPrototype","$Symbol","$stringify","getOwnPropertyNamesExternal","AllSymbols","ObjectPrototypeSymbols","StringToSymbolRegistry","SymbolToStringRegistry","QObject","USE_SETTER","findChild","setSymbolDescriptor","nativeObjectCreate","ObjectPrototypeDescriptor","description","symbol","isSymbol","$defineProperty","$defineProperties","properties","$getOwnPropertySymbols","$propertyIsEnumerable","$getOwnPropertyDescriptor","$getOwnPropertyNames","IS_OBJECT_PROTOTYPE","redefine","defineWellKnownSymbol","keyFor","sym","useSetter","useSimple","FORCED_JSON_STRINGIFY","$","stringify","replacer","space","$replacer","setToStringTag","IteratorPrototype","PrototypeOfArrayIteratorPrototype","arrayIterator","getPrototypeOf","CORRECT_PROTOTYPE_GETTER","BUGGY_SAFARI_ITERATORS","returnThis","setPrototypeOf","CORRECT_SETTER","aPossiblePrototype","__proto__","IteratorsCore","ITERATOR","Iterable","IteratorConstructor","DEFAULT","IS_SET","FORCED","Iterators","createIteratorConstructor","CurrentIteratorPrototype","methods","KEY","getIterationMethod","KIND","defaultIterator","IterablePrototype","INCORRECT_VALUES_NAME","nativeIterator","anyNativeIterator","entries","defineIterator","iterated","kind","done","Arguments","COLLECTION_NAME","CSSRuleList","CSSStyleDeclaration","CSSValueList","ClientRectList","DOMRectList","DOMStringList","DOMTokenList","DataTransferItemList","FileList","HTMLAllCollection","HTMLCollection","HTMLFormElement","HTMLSelectElement","MediaList","MimeTypeArray","NamedNodeMap","NodeList","PaintRequestList","Plugin","PluginArray","SVGLengthList","SVGNumberList","SVGPathSegList","SVGPointList","SVGStringList","SVGTransformList","SourceBufferList","StyleSheetList","TextTrackCueList","TextTrackList","TouchList","Collection","CollectionPrototype","CONVERT_TO_STRING","first","S","position","size","charCodeAt","codeAt","point","iteratorMethod","getIteratorMethod","_Object$defineProperty","nativeKeys","METHOD_NAME","whitespace","whitespaces","ltrim","rtrim","start","end","trim","$trim","CONSTRUCTOR","entryVirtual","StringPrototype","own","cache","thrower","ACCESSORS","argument0","argument1","STRICT_METHOD","arrayMethodIsStrict","USES_TO_LENGTH","arrayMethodUsesToLength","ArrayPrototype","DOMIterables","process","versions","v8","userAgent","V8_VERSION","foo","Boolean","$map","HAS_SPECIES_SUPPORT","arrayMethodHasSpeciesSupport","$parseInt","hex","radix","parseIntImplementation","TO_ENTRIES","$values","$filter","IS_CONCAT_SPREADABLE","IS_CONCAT_SPREADABLE_SUPPORT","SPECIES_SUPPORT","isConcatSpreadable","spreadable","A","_Array$isArray","arr2","ENTRIES","returnMethod","SAFE_CLOSING","called","iteratorWithReturn","INCORRECT_ITERATION","SKIP_CLOSING","ITERATION_SUPPORT","checkCorrectnessOfIteration","iterable","arrayLike","step","argumentsLength","mapfn","mapping","isArrayIteratorMethod","callWithSafeIterationClosing","iter","_isIterable","_Array$from","arrayWithoutHoles","iterableToArray","nonIterableSpread","nativeSlice","Constructor","fin","nativeGetPrototypeOf","$indexOf","nativeIndexOf","NEGATIVE_ZERO","searchElement","nativeAssign","assign","B","chr","WrappedWellKnownSymbolModule","JSON","_typeof","_Symbol","_Symbol$iterator","_arr","_n","_e","_s","_getIterator","err","arrayWithHoles","iterableToArrayLimit","nonIterableRest","AleaImplementation","seed","mash","Mash","s0","s1","s2","mashSeed","uint32","fract53","algorithm","_Date$now","prepareElements","JSONcontainer","elementType","redundant","used","cleanupElements","parentNode","removeChild","resetElements","getSVGElement","svgContainer","element","createElementNS","getDOMElement","DOMContainer","insertBefore","drawPoint","groupTemplate","labelObj","setAttributeNS","styles","className","label","xOffset","yOffset","textContent","drawBar","width","height","rect","getNavigatorLanguage","navigator","languages","userLanguage","language","browserLanguage","ReferenceError","re","low","hi","fix","prev","core","$some","assertThisInitialized","_getPrototypeOf","_Object$setPrototypeOf","_Object$getPrototypeOf","_setPrototypeOf","p","subClass","superClass","_Object$create","nativeSort","FAILS_ON_UNDEFINED","FAILS_ON_NULL","comparefn","isExtensible","preventExtensions","METADATA","setMetadata","objectID","weakData","meta","REQUIRED","fastKey","getWeakData","onFreeze","FREEZING","Result","stopped","AS_ENTRIES","IS_ITERATOR","iterFn","stop","require$$1","internalStateGetterFor","CONSTRUCTOR_NAME","wrapper","common","IS_WEAK","ADDER","NativePrototype","exported","anInstance","collection","iterate","IS_ADDER","getConstructor","InternalMetadataModule","setStrong","unsafe","last","define","previous","entry","getEntry","removed","redefineAll","clear","ITERATOR_NAME","getInternalCollectionState","getInternalIteratorState","setSpecies","init","collectionStrong","Set","Map","runtime","Op","hasOwn","iteratorSymbol","asyncIteratorSymbol","asyncIterator","toStringTagSymbol","toStringTag","innerFn","outerFn","tryLocsList","protoGenerator","Generator","generator","context","Context","_invoke","doneResult","delegate","delegateResult","maybeInvokeDelegate","ContinueSentinel","sent","_sent","dispatchException","abrupt","record","tryCatch","makeInvokeMethod","GeneratorFunction","GeneratorFunctionPrototype","getProto","NativeIteratorPrototype","Gp","defineIteratorMethods","AsyncIterator","PromiseImpl","previousPromise","callInvokeWithMethodAndArg","resolve","reject","invoke","__await","then","unwrapped","info","resultName","nextLoc","pushTryEntry","locs","tryLoc","catchLoc","finallyLoc","afterLoc","tryEntries","resetTryEntry","completion","reset","displayName","isGeneratorFunction","genFun","ctor","mark","awrap","async","Promise","reverse","pop","skipTempReset","rootRecord","rval","exception","handle","loc","caught","hasCatch","hasFinally","finallyEntry","complete","finish","thrown","delegateYield","regeneratorRuntime","accidentalStrictMode","splice","deleteCount","insertCount","actualDeleteCount","actualStart","MSIE","scheduler","handler","timeout","boundArgs","setTimeout","setInterval","flattenIntoArray","original","sourceLen","depth","mapper","thisArg","targetIndex","sourceIndex","mapFn","flatMap","IS_RIGHT","memo","$reduce","left","right","reduce","factories","construct","argsLength","list","partArgs","FunctionPrototype","instance","_defineProperties","props","protoProps","staticProps","createNewDataPipeFrom","DataPipeUnderConstruction","SimpleDataPipe","_source","_transformers","_target","_listeners","_bindInstanceProperty","_add","remove","_remove","update","_update","_transformItems","on","off","items","_reduceInstanceProperty","transform","_name","payload","oldData","_filterInstanceProperty","_mapInstanceProperty","_flatMapInstanceProperty","ASPDateRegex","convert","isString","parse","Number","getType","typeCoerceDataSet","rawDS","idProp","_idProp","coercedDS","DataSet","fieldId","pipe","item","acc","all","getDataSet","updateOnly","getIds","dispose","util","$parseFloat","Infinity","trimmedString","parseFloatImplementation","Component","body","resized","_previousWidth","_previousHeight","repeat","count","RangeError","convertHiddenOptions","hiddenDates","_repeatInstanceProperty","dateItem","updateHiddenDates","domProps","centerContainer","range","pixelTime","startDate","endDate","runUntil","dayOffset","log","removeDuplicates","startHidden","getIsHidden","endHidden","rangeStart","rangeEnd","hidden","startToFront","endToFront","_applyRange","safeDates","stepOverHiddenDates","timeStep","previousTime","stepInHidden","currentValue","current","_end","prevValue","newValue","switchedYear","switchedMonth","switchedDay","toScreen","Core","conversion","scale","getHiddenDurationBetween","hiddenBeforeStart","getHiddenDurationBeforeStart","rangeAfterEnd","correctTimeForHidden","toTime","hiddenDuration","partialDuration","accumulatedHiddenDuration","getAccumulatedHiddenDuration","getHiddenDurationBefore","timeOffset","requiredDuration","previousPoint","snapAwayFromHidden","correctionEnabled","isHidden","Range","millisecondsPerPixelCache","rolling","deltaDifference","scaleOffset","defaultOptions","rtl","moveable","zoomable","zoomMin","zoomMax","rollingMode","follow","_this","touch","animationTimer","emitter","_onDragStart","_onDrag","_onDragEnd","_onMouseWheel","_onTouch","_onPinch","dom","rollingModeBtn","addEventListener","startRolling","setOptions","selectiveExtend","setRange","me","stopRolling","interval","rollingModeOffset","animation","center","visibility","currentTimeTimer","_setTimeout","clearTimeout","frameCallback","byUser","finalStart","finalEnd","_cancelAnimation","initStart","initEnd","easingName","easingFunction","easingFunctions","_JSON$stringify","_Object$keys","initTime","anyChanged","dragging","ease","changed","DateUtil","params","event","emit","timeoutID","clientWidth","newStart","newEnd","_parseFloat","totalHidden","previousDelta","_isInsideRange","allowDragging","root","cursor","validateDirection","delta","deltaX","deltaY","diffRange","safeStart","safeEnd","wheelDelta","detail","zoomKey","pointerDate","zoomFriction","pointer","getPointer","clientX","clientY","_pointerToDate","zoom","preventDefault","centerDate","hiddenDurationBefore","hiddenDurationAfter","centerContainerRect","getBoundingClientRect","elementRect","top","moveTo","Emitter","mixin","_callbacks","once","removeListener","removeAllListeners","removeEventListener","cb","callbacks","listeners","hasListeners","_firstTarget","propagating","hammer","_options","Manager","Hammer","PropagatingHammer","events","propagatedHandler","srcEvent","_handled","stopPropagation","srcStop","firstTarget","elem","elemHammer","_handlers","pointerType","isFirst","eventType","destroy","hammers","idx","assign$1","nextKey","modifiedHammer","noop","onTouch","inputHandler","TimeStep","minimumStep","_start","autoScale","FORMAT","deepExtend","setMinimumStep","roundToMinor","priorCurrent","showMajorLabels","enable","stepYear","stepMonth","stepDay","showWeekScale","stepHour","stepMinute","stepSecond","stepMillisecond","minorLabels","majorLabels","_moment","classNames","even","today","currentWeek","currentMonth","currentYear","_step","styleInject","css","ref","insertAt","head","getElementsByTagName","firstChild","styleSheet","cssText","createTextNode","TimeAxis","foreground","lines","majorTexts","minorTexts","lineTop","orientation","axis","showMinorLabels","maxMinorChars","timeAxis","_create","selectiveDeepExtend","background","bottom","parentChanged","_calculateCharSize","minorLabelHeight","minorCharHeight","majorLabelHeight","majorCharHeight","offsetWidth","minorLineHeight","minorLineWidth","majorLineHeight","majorLineWidth","foregroundNextSibling","nextSibling","backgroundNextSibling","_repaintLabels","backgroundVertical","_isResized","timeLabelsize","minorCharWidth","setMoment","setFormat","setScale","xNext","isMajor","showMinorGrid","prevWidth","line","xFirstMajorLabel","getCurrent","hasNext","getClassName","_repaintMinorText","getLabelMinor","_repaintMajorText","getLabelMajor","_repaintMajorLine","_repaintMinorLine","_parseInt","warnedForOverflow","leftTime","leftText","widthText","majorCharWidth","text","innerHTML","_setXY","childNodes","directionX","measureCharMinor","clientHeight","measureCharMajor","container","_exportFunctions","_bound","keydown","keyup","_keys","fromCharCode","code","down","handleEvent","up","keyCode","bound","shiftKey","bindAll","getKey","unbind","newBindings","Activator","active","overlay","_onTapOverlay","onClick","_hasParent","deactivate","keycharm","escListener","activate","addClassName","removeClassName","en","deleteSelected","nl","de","fr","es","uk","ru","pl","ja","en_EN","en_US","it_IT","it_CH","nl_NL","nl_BE","de_DE","fr_FR","fr_CA","fr_BE","es_ES","uk_UA","ru_RU","pl_PL","ja_JP","CustomTime","title","defaultLocales","customTime","eventParams","bar","drag","onMouseWheel","attachEvent","DIRECTION_ALL","hide","warned","substring","redraw","editable","marker","setAttribute","focus","_onMarkerChange","_onMarkerChanged","getScale","getStep","snap","snappedTime","setCustomTime","backgroundHorizontal","leftContainer","rightContainer","shadowTop","shadowBottom","shadowTopLeft","shadowBottomLeft","shadowTopRight","shadowBottomRight","loadingScreen","border","scrollTop","scrollTopMin","initialDrawDone","_redraw","initialRangeChangeDone","_origRedraw","throttle","itemSet","initialItemSetDrawn","queue","pinchRecognizer","getTouchAction","hammerUtil","isActive","wheelDeltaY","wheelDeltaX","HORIZONTAL_AXIS","deltaMode","preferZoom","verticalScroll","horizontalScroll","adjusted","_setScrollTop","listener","isFinal","wheelType","onmousewheel","onMouseScrollSide","_isProgramaticallyScrolled","itemAddedToTimeline","handleDragOver","getEventProperties","_indexOfInstanceProperty","dataTransfer","dropEffect","handleDrop","itemData","getData","_onAddItem","_onDropObjectOnItem","customTimes","redrawCount","timeAxis2","components","drawPoints","onRender","clickToUse","activator","_initAutoResize","component","configurator","_createConfigurator","configure","appliedOptions","setModuleOptions","setItems","setGroups","_stopAutoResize","getCustomTime","setCustomMarker","setCustomTitle","timestamp","_someInstanceProperty","_this2","getVisibleItems","getVisibleGroups","getDataRange","getRange","percentage","getWindow","newInterval","distance","setWindow","maxHeight","option","asSize","minHeight","rootOffsetWidth","offsetHeight","contentHeight","autoHeight","containerHeight","scrollbarWidth","getScrollBarWidth","leftContainerClientWidth","rightContainerClientWidth","_setDOM","_updateScrollTop","visibilityTop","visibilityBottom","contentsOverflow","DIRECTION_HORIZONTAL","longSelectPressTime","centerWidth","currentTime","setCurrentTime","getCurrentTime","autoResize","_startAutoResize","_onResize","rootOffsetHeight","lastWidth","lastHeight","watchTimer","_setInterval","clearInterval","initialScrollTop","oldScrollTop","_getScrollTop","newScrollTop","CurrentTime","showCurrentTime","alignCurrentTime","$find","SKIPS_HOLES","getRandomValues","crypto","msCrypto","rnds8","Uint8Array","rng","byteToHex","v4","buf","rnds","bth","bytesToUuid","$includes","MATCH","isRegExp","regexp","correctIsRegExpLogic","searchString","notARegExp","arrayIncludes","stringIncludes","orderByStart","orderByEnd","margin","force","shouldBailItemsRedrawFunction","shouldBail","collidingItem","jj","collision","vertical","substack","subgroup","subgroupHeight","baseTop","nostack","subgroups","isStackSubgroups","newTop","visible","stackSubgroups","otherSubgroup","collisionByTimes","stackSubgroupsWithInnerStack","subgroupItems","doSubStack","subgroupOrder","horizontal","timeOverlap","heightOverlap","ReservedGroupIds","Group","groupId","subgroupStack","subgroupStackAll","subgroupVisibility","doInnerStack","shouldBailStackItems","subgroupIndex","subgroupOrderer","isVisible","stackDirty","_disposeCallbacks","nestedGroups","showNested","heightMode","groupHeightMode","nestedInGroup","visibleItems","itemsInRange","orderedItems","byStart","byEnd","checkRangedItems","handleCheckRangedItems","setData","groupEditable","order","inner","groupTouchParams","isDragging","templateFunction","Element","isReactComponent","treeLevel","removeCssText","addCssText","markerHeight","lastMarkerHeight","redrawQueue","redrawQueueLength","dirty","displayed","fns","offsetTop","offsetLeft","timeoutOptions","onTimeout","bailOptions","relativeBailingTime","itemsSettingTime","bailTimeMs","timeoutMs","userBailFunction","bail","userContinueNotBail","didUserContinue","forceRestack","lastIsVisible","isCluster","orderedClusters","cluster","_updateItemsInRange","visibleClusters","_updateClustersInRange","getVisibleItemsGroupedBySubgroup","orderFn","visibleSubgroupsItems","_sortInstanceProperty","_updateSubGroupHeights","customOrderedItems","_shouldBailItemsRedraw","repositionX","updateProperty","labelWidth","labelHeight","repositionY","returnQueue","_this3","_didMarkerHeightChange","_calculateGroupSizeAndPosition","_isGroupVisible","_redrawItems","_updateSubgroupsSizes","_calculateHeight","_didResize","_applyGroupHeight","_updateItemsVerticalPosition","_resetSubgroups","_this4","labelSet","setParent","_addToSubgroup","orderSubgroups","_includesInstanceProperty","_checkIfVisible","subgroupId","itemEnd","initialEnd","sortArray","sortField","_spliceInstanceProperty","_removeFromSubgroup","itemIndex","removeItem","startArray","endArray","oldVisibleItems","visibleItemsLookup","lowerBound","upperBound","_checkIfVisibleWithReference","initialPosByStart","binarySearchCustom","_traceVisible","initialPosByEnd","initialPos","breakCondition","hasItems","show","oldVisibleClusters","visibleClustersLookup","oldSubgroup","newSubgroup","disposeCallback","BackgroundGroup","Item","selected","groupShowing","selectable","setSelectability","_updateEditStatus","group","_moveToGroup","changeSubgroup","updateTime","dragCenter","dragCenterItem","hammerDragCenter","_onUpdateItem","box","dragLeft","overrideItems","deleteButton","optionsLocale","hammerDeleteButton","removeFromDataSet","tooltipOnItemUpdateTime","onItemUpdateTimeTooltip","touchParams","itemIsDragging","template","itemsData","itemVisibleFrameContent","visibleFrameTemplateFunction","_getItemData","itemVisibleFrameContentElement","getElementsByClassName","visibleFrameTemplate","_contentToString","dataAttributes","attributes","removeAttribute","outerHTML","updateGroup","tooltip","BoxItem","dot","align","widthInMs","getMillisecondsPerPixel","_updateContents","_updateDataAttributes","_updateStyle","sizes","_repaintOnItemUpdateTimeTooltip","_repaintDragCenter","_repaintDeleteButton","_createDomElement","_appendDomElement","_updateDirtyDomComponents","_getDomComponentsSizes","_updateDomComponentsSizes","_repaintDomAdditionals","repositionXY","boxX","boxY","dotX","dotY","lineX","lineY","lineWidth","dotWidth","lineStyle","lineHeight","PointItem","marginLeft","marginRight","translateX","pointX","pointY","RangeItem","frame","visibleFrame","baseClassName","maxWidth","getComputedStyle","whiteSpace","_repaintDragLeft","_repaintDragRight","limitSize","contentStartPosition","contentWidth","parentWidth","boxWidth","itemsAlwaysDraggable","dragLeftItem","dragRight","dragRightItem","BackgroundItem","itemSubgroup","Popup","overflowMethod","padding","doShow","isLeft","isTop","$every","ClusterItem","modifiedOptions","_Object$assign","fitOnDoubleClick","uiItems","randomUUID","_setupRange","eventEmitter","attached","detach","attach","rangeWidth","showStipes","repositionXWithRanges","repositionXWithoutRanges","_isStipeVisible","repositionStype","lineOffsetWidth","dotOffsetWidth","lineOffset","dotOffset","lineOffsetDirection","dotOffsetDirection","minWidth","itemSetHeight","detachFromParent","_fit","stats","centers","avg","sum","ondblclick","_onDoubleClick","fitStart","fitEnd","_getFitRange","fitArgs","ClusterGenerator","groups","dataChanged","applyOnChangedLevel","oldClusters","maxItems","clusterCriteria","level","timeWindow","levelChanged","cacheLevel","_dropLevelsCache","_filterData","clusters","groupName","iMax","neighbors","clusterItems","getGroupId","_getClusterForItems","_Object$values2","currentGroupName","oldClustersLookup","itemsIds","oldClusterData","_everyInstanceProperty","clusterItem","setUiItems","titleTemplate","clusterContent","clusterOptions","createClusterItem","BACKGROUND","ItemSet","groupOrderSwap","fromGroup","toGroup","targetOrder","groupOrder","multiselect","onDropObjectOnItem","objectData","onAdd","onUpdate","onMove","onRemove","onMoving","onAddGroup","onMoveGroup","onRemoveGroup","showTooltips","followMouse","delay","groupsData","sequentialSelection","itemListeners","senderId","_onAdd","clusterGenerator","_onUpdate","_onRemove","groupListeners","_onAddGroups","groupData","updatedGroups","nestedGroupId","updatedNestedGroup","_concatInstanceProperty","_onUpdateGroups","_onRemoveGroups","groupIds","selection","popup","popupTimer","_updateUngrouped","backgroundGroup","ALL","_onSelectItem","_onMultiSelectItem","groupHammer","_onGroupClick","_onGroupDragStart","_onGroupDrag","_onGroupDragEnd","DIRECTION_VERTICAL","_onMouseOver","_onMouseOut","_onMouseMove","markDirty","refreshItems","restackGroups","_detachAllClusters","clearPopupTimer","ids","idsToDeselect","selectedId","getItemById","unselect","select","rawVisibleItems","_findInstanceProperty","_clusterItems","_orderGroups","visibleInterval","zoomed","lastVisibleInterval","scrolled","lastRangeStart","changedStackOption","lastStack","changedStackSubgroupsOption","lastStackSubgroups","firstGroup","_firstGroup","firstMargin","nonFirstMargin","groupMargin","redrawResults","groupResized","firstGroupIndex","firstGroupId","itemId","ungrouped","oldItemsData","DataView","_order","updateData","_this5","_getType","types","_updateItem","_removeItem","_addItem","_this6","groupOptions","_this7","_orderNestedGroups","equalArray","getOrderedNestedGroups","nestedGroupIds","nestedGroup","_this8","itemFromTarget","itemProps","selectedItem","initialX","_cloneItemData","ctrlKey","metaKey","_onDragStartAddItem","baseGroupIndex","_getGroupIndex","itemsToDrag","getSelection","groupIndex","groupOffset","_this9","frameRect","groupFromTarget","newItem","containerRect","setPosition","domRootOffsetLeft","updateGroupAllowed","newGroupBase","initialStart","initial","_this10","newOffset","oldGroup","_this11","_this12","toggleGroupShowNested","nestingGroup","fullNestedGroups","nextLevel","node","classList","toggle","toggleGroupDragClassName","originalOrder","movingUp","targetGroup","draggedGroupHeight","targetGroupHeight","draggedGroup","newOrder","origOrder","draggedId","numGroups","curPos","orgOffset","slippedPosition","switchGroup","shouldBeGroup","switchGroupId","dataset","oldSelection","setSelection","newSelection","itemFromRelatedTarget","getTitle","setText","setPopupTimer","newItemData","itemGroup","lastSelectedGroup","multiselectPerGroup","_getItemRange","_item","filteredSelection","_this13","cur","itemFromElement","relatedTarget","foregroundRect","getClusters","_updateClusters","newClustersIds","clustersToUnselect","selectionChanged","selectedIdx","allOptions","errorFound","printStyle","Validator","referenceOptions","subObject","usedOptions","__any__","referenceOption","is_object","refOptionObj","__type__","checkFields","getSuggestion","message","printLocation","optionType","refOptionType","print","copyAndExtendArray","nodeType","localSearch","findInOptions","globalSearch","indexMatch","closestMatch","recursive","closestMatchPath","lowerCaseOption","op","levenshteinDistance","copyArray","matrix","bool","enabled","throttleRedraw","any","onInitialDrawComplete","loadingScreenTemplate","configureOptions","groupsDraggable","fill","endPos","htmlColors","black","navy","darkblue","mediumblue","blue","darkgreen","green","teal","darkcyan","deepskyblue","darkturquoise","mediumspringgreen","lime","springgreen","aqua","cyan","midnightblue","dodgerblue","lightseagreen","forestgreen","seagreen","darkslategray","limegreen","mediumseagreen","turquoise","royalblue","steelblue","darkslateblue","mediumturquoise","indigo","darkolivegreen","cadetblue","cornflowerblue","mediumaquamarine","dimgray","slateblue","olivedrab","slategray","lightslategray","mediumslateblue","lawngreen","chartreuse","aquamarine","maroon","purple","olive","gray","skyblue","lightskyblue","blueviolet","darkred","darkmagenta","saddlebrown","darkseagreen","lightgreen","mediumpurple","darkviolet","palegreen","darkorchid","yellowgreen","sienna","brown","darkgray","lightblue","greenyellow","paleturquoise","lightsteelblue","powderblue","firebrick","darkgoldenrod","mediumorchid","rosybrown","darkkhaki","silver","mediumvioletred","indianred","peru","chocolate","tan","lightgrey","palevioletred","thistle","orchid","goldenrod","crimson","gainsboro","plum","burlywood","lightcyan","lavender","darksalmon","violet","palegoldenrod","lightcoral","khaki","aliceblue","honeydew","azure","sandybrown","wheat","beige","whitesmoke","mintcream","ghostwhite","salmon","antiquewhite","linen","lightgoldenrodyellow","oldlace","red","fuchsia","magenta","deeppink","orangered","tomato","hotpink","coral","darkorange","lightsalmon","orange","lightpink","pink","gold","peachpuff","navajowhite","moccasin","bisque","mistyrose","blanchedalmond","papayawhip","lavenderblush","seashell","cornsilk","lemonchiffon","floralwhite","snow","yellow","lightyellow","ivory","white","ColorPicker","pixelRatio","generated","centerCoordinates","r","color","g","hueCircle","initialColor","previousColor","applied","updateCallback","closeCallback","_bindHammer","_setSize","setInitial","rgba","htmlColor","_isColorString","isValidRGB","rgbaArray","isValidRGBA","isValidHex","rgbObj","hexToRGB","alpha","_setColor","_generateHueCircle","storePrevious","_hide","_updatePicker","setColor","alert","hsv","RGBToHSV","angleConvert","PI","radius","sin","cos","colorPickerSelector","v","HSVToRGB","ctx","colorPickerCanvas","getContext","pixelRation","devicePixelRatio","webkitBackingStorePixelRatio","mozBackingStorePixelRatio","msBackingStorePixelRatio","oBackingStorePixelRatio","backingStorePixelRatio","setTransform","clearRect","putImageData","fillStyle","circle","brightnessRange","opacityRange","initialColorDiv","backgroundColor","newColorDiv","colorPickerDiv","noCanvas","fontWeight","opacityDiv","brightnessDiv","arrowDiv","onchange","_setOpacity","oninput","_setBrightness","brightnessLabel","opacityLabel","cancelButton","onclick","applyButton","_apply","saveButton","_save","loadButton","_loadLast","pinch","_moveSelector","hue","sat","rgb","sfac","fillRect","strokeStyle","stroke","getImageData","centerY","centerX","angle","atan2","sqrt","newLeft","Configurator","parentModule","defaultContainer","changedOptions","allowCreation","initialized","popupCounter","showButton","moduleOptions","domElements","popupDiv","popupLimit","popupHistory","colorPicker","_removePopup","_clean","counter","_handleObject","_makeItem","_makeHeader","_makeButton","_push","_showPopupIfNeeded","div","objectLabel","selectedValue","_makeLabel","defaultValue","popupString","popupValue","_setupPopup","generateButton","_printOptions","onmouseover","onmouseout","optionsContainer","hideTimeout","deleteTimeout","opacity","checkbox","checked","defaultColor","_showColorPicker","insertTo","setUpdateCallback","colorString","setCloseCallback","checkOnly","visibleInSet","subObj","newPath","_getValue","_handleArray","_makeTextInput","_makeCheckbox","draw","physics","solver","enabledPath","enabledValue","_makeColorField","_makeDropdown","_makeRange","_constructOptions","optionsObj","getOptions","Timeline","itemsDone","_assertThisInitialized","SyntaxError","forthArgument","directionFromDom","domNode","parentElement","loadingScreenFragment","_toScreen","toGlobalScreen","_toGlobalScreen","_toTime","toGlobalTime","_toGlobalTime","oncontextmenu","PointerEvent","onpointerdown","onpointermove","onpointerup","onmousemove","onmousedown","onmouseup","initialFitDone","getItemRange","fit","_onFit","validate","newDataSet","startPos","initialVerticalScroll","setFinalVerticalPosition","finalVerticalScroll","getItemVerticalScroll","shouldScroll","itemTop","scrollOffset","middle","willDraw","minItem","maxItem","factor","startSide","endSide","getStart","getEnd","getWidthRight","getWidthLeft","lhs","rhs","customTimeFromTarget","getTarget","what","hasParent","pageX","pageY","timeline","itemsetHeight","currentScrollHeight","targetOffset","DataScale","autoScaleStart","autoScaleEnd","zeroAlign","formattingFunction","majorSteps","minorSteps","customLines","minorStepIdx","magnitudefactor","determineScale","rounded","minimumStepValue","orderOfMagnitude","LN10","solutionFound","majorStep","convertValue","returnValue","toPrecision","bottomOffset","major","is_major","formatValue","oldStepIdx","oldStart","oldEnd","increaseMagnitude","decreaseMagnitude","otherZero","otherStep","newRange","myOriginalZero","majorOffset","getFirstMajor","zeroOffset","pixels","DataAxis","svg","linegraphOptions","icons","majorLinesOffset","minorLinesOffset","labelOffsetX","labelOffsetY","iconWidth","alignZeros","linegraphSVG","DOMelements","labels","conversionFactor","stepPixels","zeroCrossing","amountOfSteps","master","masterAxis","svgElements","iconsRemoved","amountOfGroups","_redrawLabels","framework","lineContainer","graphOptions","DOMutil","iconOffset","groupArray","getLegend","iconHeight","activeGroups","backgroundHorizontalOffsetWidth","_redrawGroupIcons","_cleanupIcons","_redrawTitle","customRange","followScale","maxLabelSize","getLines","_redrawLabel","_redrawLine","titleWidth","titleCharHeight","screenToValue","characterHeight","textAlign","largestWidth","textMinor","textMajor","textTitle","measureCharTitle","titleCharWidth","Points","getGroupTemplate","callbackResult","Bargraph","Line","GraphGroup","groupsUsingDefaultStyles","selectiveBridgeObject","usingDefaultStyle","zeroPosition","Legend","side","iconSize","iconSpacing","getCallback","screen_x","screen_y","drawIcon","fillHeight","outline","barWidth","barChart","bar1Height","bar2Height","processedGroupData","coreDistance","drawData","combinedData","intersections","barPoints","screen_end","_getDataIntersections","heightOffset","_getSafeDrawData","amount","resolved","excludeFromStacking","accumulatedNegative","accumulatedPositive","sideBySide","dataWidth","pointData","getStackedYRange","groupRanges","groupLabel","_getStackedYRange","yAxisOrientation","yMin","yMax","xpos","calcPath","interpolation","_catmullRom","_linear","fillPath","shaded","drawShading","pathArray","subPathArray","dFill","svgHeight","zero","serializePath","inverse","_catmullRomUniform","p0","bp1","bp2","d1","d2","d3","N","d3powA","d2powA","d3pow2A","d2pow2A","d1pow2A","d1powA","insertSort","getItems","setZeroPosition","mergeOptions","parametrization","Lines","Bars","icon","getYRange","addGroup","excludeFromLegend","removeGroup","textArea","scrollableHeight","drawLegendIcons","paddingTop","LineGraph","defaultGroup","sampling","graphHeight","dataAxis","legend","abortedGraphUpdate","updateSVGheight","updateSVGheightOnResize","forceGraphUpdate","lastStart","yAxisLeft","yAxisRight","legendLeft","legendRight","_removeGroup","_updateAllGroupData","_updateGroup","groupsContent","idMap","groupCounts","existingItemsMap","existing_items","newLength","extended","bridgeObject","orginalY","_updateGraph","_getSortedGroupIds","grouplist","zIndex","az","bz","minDate","maxDate","_getRelevantData","_applySampling","_convertXcoordinates","_getYRanges","_updateYAxis","below","_stack","_convertYcoordinates","paths","subGroupId","subData","dx","dy","subPrevPoint","subNextPoint","dateComparator","binarySearchValue","dataContainer","increment","amountOfPoints","pointsPerPixel","sampledData","combinedDataLeft","combinedDataRight","minVal","maxVal","yAxisLeftUsed","yAxisRightUsed","minLeft","minRight","maxLeft","maxRight","ignore","_toggleAxisVisiblity","drawIcons","tempGroups","axisUsed","datapoints","Graph2d","linegraph","initialLoad","isGroupVisible","getAbsoluteLeft","getAbsoluteTop","defaultLanguage"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;8jBAGmEA,UAG1D,eAEDC,EA6GAC,WA3GKC,WACEF,EAAaG,MAAM,KAAMC,oBAS3BC,EAAQC,UACNA,aAAiBC,OAAmD,mBAA1CC,OAAOC,UAAUC,SAASC,KAAKL,YAG3DM,EAASN,UAGE,MAATA,GAA2D,oBAA1CE,OAAOC,UAAUC,SAASC,KAAKL,YAiBlDO,EAAYP,eACA,IAAVA,WAGFQ,EAASR,SACU,iBAAVA,GAAgE,oBAA1CE,OAAOC,UAAUC,SAASC,KAAKL,YAG9DS,EAAOT,UACLA,aAAiBU,MAAkD,kBAA1CR,OAAOC,UAAUC,SAASC,KAAKL,YAG1DW,EAAIC,EAAKC,OACAC,EAAVC,EAAM,OACLD,EAAI,EAAGA,EAAIF,EAAII,SAAUF,EAC1BC,EAAIE,KAAKJ,EAAGD,EAAIE,GAAIA,WAEjBC,WAGFG,EAAWC,EAAGC,UACZlB,OAAOC,UAAUkB,eAAehB,KAAKc,EAAGC,YAG1CE,EAAOH,EAAGC,OACV,IAAIN,KAAKM,EACNF,EAAWE,EAAGN,KACdK,EAAEL,GAAKM,EAAEN,WAIbI,EAAWE,EAAG,cACdD,EAAEf,SAAWgB,EAAEhB,UAGfc,EAAWE,EAAG,aACdD,EAAEI,QAAUH,EAAEG,SAGXJ,WAGFK,EAAWxB,EAAOyB,EAAQC,EAAQC,UAChCC,GAAiB5B,EAAOyB,EAAQC,EAAQC,GAAQ,GAAME,eAuBxDC,EAAgBC,UACR,MAATA,EAAEC,MACFD,EAAEC,IApBC,CACHC,OAAkB,EAClBC,aAAkB,GAClBC,YAAkB,GAClBC,UAAmB,EACnBC,cAAkB,EAClBC,WAAkB,EAClBC,aAAkB,KAClBC,eAAkB,EAClBC,iBAAkB,EAClBC,KAAkB,EAClBC,gBAAkB,GAClBC,SAAkB,KAClBC,SAAkB,EAClBC,iBAAkB,IAQff,EAAEC,aAqBJe,EAAQhB,MACK,MAAdA,EAAEiB,SAAkB,KAChBC,EAAQnB,EAAgBC,GACxBmB,EAAcvD,EAAKU,KAAK4C,EAAMN,iBAAiB,SAAU7B,UAC7C,MAALA,KAEPqC,GAAcC,MAAMrB,EAAEsB,GAAGC,YACzBL,EAAMb,SAAW,IAChBa,EAAMhB,QACNgB,EAAMV,eACNU,EAAMM,iBACNN,EAAMH,kBACNG,EAAMX,YACNW,EAAMT,gBACNS,EAAMR,mBACLQ,EAAML,UAAaK,EAAML,UAAYM,MAEvCnB,EAAEyB,UACFL,EAAaA,GACe,IAAxBF,EAAMZ,eACwB,IAA9BY,EAAMf,aAAalB,aACDyC,IAAlBR,EAAMS,SAGS,MAAnBxD,OAAOyD,UAAqBzD,OAAOyD,SAAS5B,UAIrCoB,EAHPpB,EAAEiB,SAAWG,SAMdpB,EAAEiB,kBAGJY,EAAeX,OAChBlB,EAAIP,EAAUqC,YACL,MAATZ,EACA3B,EAAOQ,EAAgBC,GAAIkB,GAG3BnB,EAAgBC,GAAGU,iBAAkB,EAGlCV,EA3DPpC,EADAM,MAAME,UAAUR,KACTM,MAAME,UAAUR,KAEhB,SAAUmE,WACTC,EAAI7D,OAAO8D,MACXC,EAAMF,EAAE/C,SAAW,EAEdF,EAAI,EAAGA,EAAImD,EAAKnD,OACjBA,KAAKiD,GAAKD,EAAIzD,KAAK2D,KAAMD,EAAEjD,GAAIA,EAAGiD,UAC3B,SAIR,OAoDXG,EAAmBtE,EAAMsE,iBAAmB,YAEvCC,EAAWC,EAAIC,OAChBvD,EAAGwD,EAAMC,KAERhE,EAAY8D,EAAKG,oBAClBJ,EAAGI,iBAAmBH,EAAKG,kBAE1BjE,EAAY8D,EAAKI,MAClBL,EAAGK,GAAKJ,EAAKI,IAEZlE,EAAY8D,EAAKK,MAClBN,EAAGM,GAAKL,EAAKK,IAEZnE,EAAY8D,EAAKM,MAClBP,EAAGO,GAAKN,EAAKM,IAEZpE,EAAY8D,EAAKb,WAClBY,EAAGZ,QAAUa,EAAKb,SAEjBjD,EAAY8D,EAAKO,QAClBR,EAAGQ,KAAOP,EAAKO,MAEdrE,EAAY8D,EAAKQ,UAClBT,EAAGS,OAASR,EAAKQ,QAEhBtE,EAAY8D,EAAKS,WAClBV,EAAGU,QAAUT,EAAKS,SAEjBvE,EAAY8D,EAAKrC,OAClBoC,EAAGpC,IAAMF,EAAgBuC,IAExB9D,EAAY8D,EAAKU,WAClBX,EAAGW,QAAUV,EAAKU,SAGlBb,EAAiBlD,OAAS,MACrBF,EAAI,EAAGA,EAAIoD,EAAiBlD,OAAQF,IAGhCP,EADLgE,EAAMF,EADNC,EAAOJ,EAAiBpD,OAGpBsD,EAAGE,GAAQC,UAKhBH,MAGPY,GAAmB,WAGdC,EAAOC,GACZf,EAAWH,KAAMkB,QACZ7B,GAAK,IAAI3C,KAAkB,MAAbwE,EAAO7B,GAAa6B,EAAO7B,GAAGC,UAAYO,KACxDG,KAAKjB,iBACDM,GAAK,IAAI3C,KAAKmD,OAIE,IAArBmB,IACAA,GAAmB,EACnBpF,EAAMuF,aAAanB,MACnBgB,GAAmB,YAIlBI,EAAUC,UACRA,aAAeJ,GAAkB,MAAPI,GAAuC,MAAxBA,EAAIb,0BAG/Cc,EAAUC,UACXA,EAAS,EAEFC,KAAKC,KAAKF,IAAW,EAErBC,KAAKE,MAAMH,YAIjBI,EAAMC,OACPC,GAAiBD,EACjBE,EAAQ,SAEU,IAAlBD,GAAuBE,SAASF,KAChCC,EAAQR,EAASO,IAGdC,WAIFE,EAAcC,EAAQC,EAAQC,OAI/BrF,EAHAmD,EAAMuB,KAAKY,IAAIH,EAAOjF,OAAQkF,EAAOlF,QACrCqF,EAAab,KAAKc,IAAIL,EAAOjF,OAASkF,EAAOlF,QAC7CuF,EAAQ,MAEPzF,EAAI,EAAGA,EAAImD,EAAKnD,KACZqF,GAAeF,EAAOnF,KAAOoF,EAAOpF,KACnCqF,GAAeR,EAAMM,EAAOnF,MAAQ6E,EAAMO,EAAOpF,MACnDyF,WAGDA,EAAQF,WAGVG,EAAKC,IACgC,IAAtC7G,EAAM8G,6BACmB,oBAAbC,SAA6BA,QAAQH,MACjDG,QAAQH,KAAK,wBAA0BC,YAItCG,EAAUH,EAAK5F,OAChBgG,GAAY,SAETvF,GAAO,cACsB,MAA5B1B,EAAMkH,oBACNlH,EAAMkH,mBAAmB,KAAML,GAE/BI,EAAW,SAEPE,EADAC,EAAO,GAEFlG,EAAI,EAAGA,EAAIhB,UAAUkB,OAAQF,IAAK,IACvCiG,EAAM,GACsB,iBAAjBjH,UAAUgB,GAAiB,KAE7B,IAAImG,KADTF,GAAO,MAAQjG,EAAI,KACHhB,UAAU,GACtBiH,GAAOE,EAAM,KAAOnH,UAAU,GAAGmH,GAAO,KAE5CF,EAAMA,EAAIG,MAAM,GAAI,QAEpBH,EAAMjH,UAAUgB,GAEpBkG,EAAK/F,KAAK8F,GAEdP,EAAKC,EAAM,gBAAkBxG,MAAME,UAAU+G,MAAM7G,KAAK2G,GAAMG,KAAK,IAAM,MAAQ,IAAIC,OAASC,OAC9FR,GAAY,SAEThG,EAAGhB,MAAMmE,KAAMlE,aACvBe,OAyEHyG,EAtEAC,EAAe,YAEVC,EAAgBC,EAAMhB,GACK,MAA5B7G,EAAMkH,oBACNlH,EAAMkH,mBAAmBW,EAAMhB,GAE9Bc,EAAaE,KACdjB,EAAKC,GACLc,EAAaE,IAAQ,YAOpBC,EAAW1H,UACTA,aAAiB2H,UAAsD,sBAA1CzH,OAAOC,UAAUC,SAASC,KAAKL,YAsB9D4H,EAAaC,EAAcC,OACIxD,EAAhCvD,EAAMO,EAAO,GAAIuG,OAChBvD,KAAQwD,EACL5G,EAAW4G,EAAaxD,KACpBhE,EAASuH,EAAavD,KAAUhE,EAASwH,EAAYxD,KACrDvD,EAAIuD,GAAQ,GACZhD,EAAOP,EAAIuD,GAAOuD,EAAavD,IAC/BhD,EAAOP,EAAIuD,GAAOwD,EAAYxD,KACF,MAArBwD,EAAYxD,GACnBvD,EAAIuD,GAAQwD,EAAYxD,UAEjBvD,EAAIuD,QAIlBA,KAAQuD,EACL3G,EAAW2G,EAAcvD,KACpBpD,EAAW4G,EAAaxD,IACzBhE,EAASuH,EAAavD,MAE1BvD,EAAIuD,GAAQhD,EAAO,GAAIP,EAAIuD,YAG5BvD,WAGFgH,EAAO7C,GACE,MAAVA,QACK8C,IAAI9C,GAtDjBtF,EAAM8G,6BAA8B,EACpC9G,EAAMkH,mBAAqB,KA4DvBQ,EADApH,OAAOoH,KACApH,OAAOoH,KAEP,SAAUjC,OACTvE,EAAGC,EAAM,OACRD,KAAKuE,EACFnE,EAAWmE,EAAKvE,IAChBC,EAAIE,KAAKH,UAGVC,OAoFXkH,EAAU,YAELC,EAAcC,EAAMC,OACrBC,EAAYF,EAAKG,cACrBL,EAAQI,GAAaJ,EAAQI,EAAY,KAAOJ,EAAQG,GAAaD,WAGhEI,EAAeC,SACI,iBAAVA,EAAqBP,EAAQO,IAAUP,EAAQO,EAAMF,oBAAiB7E,WAG/EgF,EAAqBC,OAEtBC,EACArE,EAFAsE,EAAkB,OAIjBtE,KAAQoE,EACLxH,EAAWwH,EAAapE,KACxBqE,EAAiBJ,EAAejE,MAE5BsE,EAAgBD,GAAkBD,EAAYpE,WAKnDsE,MAGPC,EAAa,YAERC,EAAgBX,EAAMY,GAC3BF,EAAWV,GAAQY,WAcdC,EAASzD,EAAQ0D,EAAcC,OAChCC,EAAY,GAAK3D,KAAKc,IAAIf,GAC1B6D,EAAcH,EAAeE,EAAUnI,cAChCuE,GAAU,EACL2D,EAAY,IAAM,GAAM,KACpC1D,KAAK6D,IAAI,GAAI7D,KAAK8D,IAAI,EAAGF,IAAchJ,WAAWmJ,OAAO,GAAKJ,MAGlEK,EAAmB,uLAEnBC,EAAwB,6CAExBC,EAAkB,GAElBC,EAAuB,YAMlBC,EAAgBC,EAAOC,EAAQC,EAASC,OACzCC,EAAOD,EACa,iBAAbA,IACPC,EAAO,kBACIjG,KAAKgG,OAGhBH,IACAF,EAAqBE,GAASI,GAE9BH,IACAH,EAAqBG,EAAO,IAAM,kBACvBd,EAASiB,EAAKpK,MAAMmE,KAAMlE,WAAYgK,EAAO,GAAIA,EAAO,MAGnEC,IACAJ,EAAqBI,GAAW,kBACrB/F,KAAKkG,aAAaH,QAAQE,EAAKpK,MAAMmE,KAAMlE,WAAY+J,cAiCjEM,EAAapI,EAAGN,UAChBM,EAAEgB,WAIPtB,EAAS2I,EAAa3I,EAAQM,EAAEmI,cAChCR,EAAgBjI,GAAUiI,EAAgBjI,aA3BlBA,OACoBX,EAAGE,EARnBhB,EAQxBqK,EAAQ5I,EAAO6I,MAAMd,OAEpB1I,EAAI,EAAGE,EAASqJ,EAAMrJ,OAAQF,EAAIE,EAAQF,IACvC6I,EAAqBU,EAAMvJ,IAC3BuJ,EAAMvJ,GAAK6I,EAAqBU,EAAMvJ,IAEtCuJ,EAAMvJ,IAdcd,EAccqK,EAAMvJ,IAbtCwJ,MAAM,YACLtK,EAAMuK,QAAQ,WAAY,IAE9BvK,EAAMuK,QAAQ,MAAO,WAcrB,SAAUC,OACI1J,EAAb2J,EAAS,OACR3J,EAAI,EAAGA,EAAIE,EAAQF,IACpB2J,GAAU/C,EAAW2C,EAAMvJ,IAAMuJ,EAAMvJ,GAAGT,KAAKmK,EAAK/I,GAAU4I,EAAMvJ,UAEjE2J,GAW0CC,CAAmBjJ,GAEjEiI,EAAgBjI,GAAQM,IANpBA,EAAEmI,aAAaS,uBASrBP,EAAa3I,EAAQC,OACtBZ,EAAI,WAEC8J,EAA4B5K,UAC1B0B,EAAOmJ,eAAe7K,IAAUA,MAG3CyJ,EAAsBqB,UAAY,EAC3BhK,GAAK,GAAK2I,EAAsBsB,KAAKtJ,IACxCA,EAASA,EAAO8I,QAAQd,EAAuBmB,GAC/CnB,EAAsBqB,UAAY,EAClChK,GAAK,SAGFW,MAGPuJ,EAAiB,KACjBC,EAAiB,OACjBC,EAAiB,QACjBC,EAAiB,QACjBC,EAAiB,aACjBC,EAAiB,QACjBC,EAAiB,YACjBC,GAAiB,gBACjBC,GAAiB,UACjBC,GAAiB,UACjBC,GAAiB,eAEjBC,GAAiB,MACjBC,GAAiB,WAEjBC,GAAiB,qBACjBC,GAAmB,0BAMnBC,GAAY,wJAEZC,GAAU,YAELC,GAAepC,EAAOqC,EAAOC,GAClCH,GAAQnC,GAASnC,EAAWwE,GAASA,EAAQ,SAAUE,EAAUlC,UACrDkC,GAAYD,EAAeA,EAAcD,YAIhDG,GAAuBxC,EAAO3E,UAC9BhE,EAAW8K,GAASnC,GAIlBmC,GAAQnC,GAAO3E,EAAO1B,QAAS0B,EAAOH,SAHlC,IAAIuH,OAQRC,GAR8B1C,EAQhBU,QAAQ,KAAM,IAAIA,QAAQ,uCAAuC,SAAUiC,EAASC,EAAIC,EAAIC,EAAIC,UAC1GH,GAAMC,GAAMC,GAAMC,gBAIxBL,GAAYM,UACVA,EAAEtC,QAAQ,yBAA0B,YAG3CuC,GAAS,YAEJC,GAAelD,EAAOG,OACvBlJ,EAAGmJ,EAAOD,MACO,iBAAVH,IACPA,EAAQ,CAACA,IAETrJ,EAASwJ,KACTC,EAAO,SAAUjK,EAAOqK,GACpBA,EAAML,GAAYrE,EAAM3F,KAG3Bc,EAAI,EAAGA,EAAI+I,EAAM7I,OAAQF,IAC1BgM,GAAOjD,EAAM/I,IAAMmJ,WAIlB+C,GAAmBnD,EAAOG,GAC/B+C,GAAclD,GAAO,SAAU7J,EAAOqK,EAAOnF,EAAQ2E,GACjD3E,EAAO+H,GAAK/H,EAAO+H,IAAM,GACzBjD,EAAShK,EAAOkF,EAAO+H,GAAI/H,EAAQ2E,eAIlCqD,GAAwBrD,EAAO7J,EAAOkF,GAC9B,MAATlF,GAAiBkB,EAAW4L,GAAQjD,IACpCiD,GAAOjD,GAAO7J,EAAOkF,EAAOiI,GAAIjI,EAAQ2E,YA0DvCuD,GAAWC,UACTC,GAAWD,GAAQ,IAAM,aAG3BC,GAAWD,UACRA,EAAO,GAAM,GAAKA,EAAO,KAAQ,GAAMA,EAAO,KAAQ,EA/ClEzD,EAAe,IAAK,EAAG,GAAG,eAClB2D,EAAIvJ,KAAKqJ,cACNE,GAAK,KAAO,GAAKA,EAAI,IAAMA,KAGtC3D,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAKqJ,OAAS,OAGzBzD,EAAe,EAAG,CAAC,OAAU,GAAU,EAAG,QAC1CA,EAAe,EAAG,CAAC,QAAU,GAAU,EAAG,QAC1CA,EAAe,EAAG,CAAC,SAAU,GAAG,GAAO,EAAG,QAI1C1B,EAAa,OAAQ,KAIrBY,EAAgB,OAAQ,GAIxBmD,GAAc,IAAUL,IACxBK,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,OAAUR,GAAWN,GACnCc,GAAc,QAAUP,GAAWN,GACnCa,GAAc,SAAUP,GAAWN,GAEnC2B,GAAc,CAAC,QAAS,UAzCb,GA0CXA,GAAc,QAAQ,SAAU/M,EAAOqK,GACnCA,EA3CO,GA2CwB,IAAjBrK,EAAMgB,OAAepB,EAAM4N,kBAAkBxN,GAAS2F,EAAM3F,MAE9E+M,GAAc,MAAM,SAAU/M,EAAOqK,GACjCA,EA9CO,GA8COzK,EAAM4N,kBAAkBxN,MAE1C+M,GAAc,KAAK,SAAU/M,EAAOqK,GAChCA,EAjDO,GAiDOoD,SAASzN,EAAO,OAelCJ,EAAM4N,kBAAoB,SAAUxN,UACzB2F,EAAM3F,IAAU2F,EAAM3F,GAAS,GAAK,KAAO,UAsElD0N,GAjEAC,GAAaC,GAAW,YAAY,YAM/BA,GAAYzF,EAAM0F,UAChB,SAAU/H,UACA,MAATA,GACAgI,GAAM9J,KAAMmE,EAAMrC,GAClBlG,EAAMuF,aAAanB,KAAM6J,GAClB7J,MAEA+J,GAAI/J,KAAMmE,aAKpB4F,GAAKvD,EAAKrC,UACRqC,EAAIzH,UACPyH,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,KAAUtE,aAGpDiK,GAAOtD,EAAKrC,EAAMrC,GACnB0E,EAAIzH,YAAcK,MAAM0C,KACX,aAATqC,GAAuBmF,GAAW9C,EAAI6C,SAA2B,IAAhB7C,EAAIwD,SAAgC,KAAfxD,EAAIyD,OAC1EzD,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,GAAMrC,EAAO0E,EAAIwD,QAASE,GAAYpI,EAAO0E,EAAIwD,UAG5FxD,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAMsD,GAAMrC,aAqDpDoI,GAAYb,EAAMW,MACnB5K,MAAMiK,IAASjK,MAAM4K,UACdnK,QAvBCsK,EAyBRC,GAAeJ,GAzBPG,EAyBc,IAxBRA,GAAKA,SAyBvBd,IAASW,EAAQI,GAAY,GACT,IAAbA,EAAkBd,GAAWD,GAAQ,GAAK,GAAO,GAAKe,EAAW,EAAI,EApB5EV,GADAzN,MAAME,UAAUuN,QACNzN,MAAME,UAAUuN,QAEhB,SAAUW,OAEZvN,MACCA,EAAI,EAAGA,EAAIkD,KAAKhD,SAAUF,KACvBkD,KAAKlD,KAAOuN,SACLvN,SAGP,GAehB8I,EAAe,IAAK,CAAC,KAAM,GAAI,MAAM,kBAC1B5F,KAAKgK,QAAU,KAG1BpE,EAAe,MAAO,EAAG,GAAG,SAAUnI,UAC3BuC,KAAKkG,aAAaoE,YAAYtK,KAAMvC,MAG/CmI,EAAe,OAAQ,EAAG,GAAG,SAAUnI,UAC5BuC,KAAKkG,aAAaqE,OAAOvK,KAAMvC,MAK1CyG,EAAa,QAAS,KAItBY,EAAgB,QAAS,GAIzBmD,GAAc,IAAQZ,GACtBY,GAAc,KAAQZ,EAAWJ,GACjCgB,GAAc,OAAQ,SAAUG,EAAU1K,UAC/BA,EAAO8M,iBAAiBpC,MAEnCH,GAAc,QAAQ,SAAUG,EAAU1K,UAC/BA,EAAO+M,YAAYrC,MAG9BW,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,GACxCA,EAlMQ,GAkMO1E,EAAM3F,GAAS,KAGlC+M,GAAc,CAAC,MAAO,SAAS,SAAU/M,EAAOqK,EAAOnF,EAAQ2E,OACvDmE,EAAQ9I,EAAOH,QAAQ2J,YAAY1O,EAAO6J,EAAO3E,EAAO1B,SAE/C,MAATwK,EACA3D,EAzMI,GAyMW2D,EAEflM,EAAgBoD,GAAQ3C,aAAevC,SAM3C2O,GAAmB,gCACnBC,GAAsB,wFAAwFC,MAAM,KAUpHC,GAA2B,kDAAkDD,MAAM,cAU9EE,GAAkBC,EAAWvN,EAAQE,OACtCb,EAAGmO,EAAIzE,EAAK0E,EAAMF,EAAUG,wBAC3BnL,KAAKoL,sBAEDA,aAAe,QACfC,iBAAmB,QACnBC,kBAAoB,GACpBxO,EAAI,EAAGA,EAAI,KAAMA,EAClB0J,EAAMhJ,EAAU,CAAC,IAAMV,SAClBwO,kBAAkBxO,GAAKkD,KAAKsK,YAAY9D,EAAK,IAAI2E,yBACjDE,iBAAiBvO,GAAKkD,KAAKuK,OAAO/D,EAAK,IAAI2E,2BAIpDxN,EACe,QAAXF,GAEe,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,IACvBD,EAAK,MAGT,KADfA,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,IACtBD,EAAK,KAGb,QAAXxN,GAEY,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,MAK3B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,IAF9BD,EAGa,MAGZ,KADZA,EAAKvB,GAAQrN,KAAK2D,KAAKqL,iBAAkBH,MAK1B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKsL,kBAAmBJ,IAF/BD,EAGa,cA6C3BM,GAAU/E,EAAK1E,OAChB0J,MAEChF,EAAIzH,iBAEEyH,KAGU,iBAAV1E,KACH,QAAQiF,KAAKjF,GACbA,EAAQH,EAAMG,YAITtF,EAFLsF,EAAQ0E,EAAIN,aAAawE,YAAY5I,WAG1B0E,SAKnBgF,EAAahK,KAAKY,IAAIoE,EAAIyD,OAAQC,GAAY1D,EAAI6C,OAAQvH,IAC1D0E,EAAInH,GAAG,OAASmH,EAAI3F,OAAS,MAAQ,IAAM,SAASiB,EAAO0J,GACpDhF,WAGFiF,GAAa3J,UACL,MAATA,GACAyJ,GAASvL,KAAM8B,GACflG,EAAMuF,aAAanB,MAAM,GAClBA,MAEA+J,GAAI/J,KAAM,aAQrB0L,GAA0B3D,GAoB1B4D,GAAqB5D,YAoBhB6D,cACIC,EAAU1O,EAAGC,UACXA,EAAEJ,OAASG,EAAEH,WAIpBF,EAAG0J,EADHsF,EAAc,GAAIC,EAAa,GAAIC,EAAc,OAEhDlP,EAAI,EAAGA,EAAI,GAAIA,IAEhB0J,EAAMhJ,EAAU,CAAC,IAAMV,IACvBgP,EAAY7O,KAAK+C,KAAKsK,YAAY9D,EAAK,KACvCuF,EAAW9O,KAAK+C,KAAKuK,OAAO/D,EAAK,KACjCwF,EAAY/O,KAAK+C,KAAKuK,OAAO/D,EAAK,KAClCwF,EAAY/O,KAAK+C,KAAKsK,YAAY9D,EAAK,SAI3CsF,EAAYG,KAAKJ,GACjBE,EAAWE,KAAKJ,GAChBG,EAAYC,KAAKJ,GACZ/O,EAAI,EAAGA,EAAI,GAAIA,IAChBgP,EAAYhP,GAAKyL,GAAYuD,EAAYhP,IACzCiP,EAAWjP,GAAKyL,GAAYwD,EAAWjP,QAEtCA,EAAI,EAAGA,EAAI,GAAIA,IAChBkP,EAAYlP,GAAKyL,GAAYyD,EAAYlP,SAGxCoP,aAAe,IAAI5D,OAAO,KAAO0D,EAAY7I,KAAK,KAAO,IAAK,UAC9DgJ,kBAAoBnM,KAAKkM,kBACzBE,mBAAqB,IAAI9D,OAAO,KAAOyD,EAAW5I,KAAK,KAAO,IAAK,UACnEkJ,wBAA0B,IAAI/D,OAAO,KAAOwD,EAAY3I,KAAK,KAAO,IAAK,cAGzEmJ,GAAY/C,EAAGxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,OAG/BzC,SAEAV,EAAI,KAAOA,GAAK,GAEhBU,EAAO,IAAIvN,KAAK6M,EAAI,IAAKxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,GACpC3K,SAASkI,EAAK0C,gBACd1C,EAAK2C,YAAYrD,IAGrBU,EAAO,IAAIvN,KAAK6M,EAAGxL,EAAGwO,EAAGC,EAAGC,EAAG5D,EAAG6D,GAG/BzC,WAGF4C,GAAetD,OAChBU,KAEAV,EAAI,KAAOA,GAAK,EAAG,KACfvG,EAAO/G,MAAME,UAAU+G,MAAM7G,KAAKP,WAEtCkH,EAAK,GAAKuG,EAAI,IACdU,EAAO,IAAIvN,KAAKA,KAAKoQ,IAAIjR,MAAM,KAAMmH,IACjCjB,SAASkI,EAAK8C,mBACd9C,EAAK+C,eAAezD,QAGxBU,EAAO,IAAIvN,KAAKA,KAAKoQ,IAAIjR,MAAM,KAAMC,mBAGlCmO,WAIFgD,GAAgB5D,EAAM6D,EAAKC,OAE5BC,EAAM,EAAIF,EAAMC,UAEP,EAAIN,GAAcxD,EAAM,EAAG+D,GAAKC,YAAcH,GAAO,EAElDE,EAAM,WAIjBE,GAAmBjE,EAAMkE,EAAMC,EAASN,EAAKC,OAI9CM,EAASC,EADTC,EAAY,EAAI,GAAKJ,EAAO,IAFZ,EAAIC,EAAUN,GAAO,EACxBD,GAAgB5D,EAAM6D,EAAKC,UAIxCQ,GAAa,EAEbD,EAAetE,GADfqE,EAAUpE,EAAO,GACoBsE,EAC9BA,EAAYvE,GAAWC,IAC9BoE,EAAUpE,EAAO,EACjBqE,EAAeC,EAAYvE,GAAWC,KAEtCoE,EAAUpE,EACVqE,EAAeC,GAGZ,CACHtE,KAAMoE,EACNE,UAAWD,YAIVE,GAAWpH,EAAK0G,EAAKC,OAGtBU,EAASJ,EAFTK,EAAab,GAAgBzG,EAAI6C,OAAQ6D,EAAKC,GAC9CI,EAAO/L,KAAKE,OAAO8E,EAAImH,YAAcG,EAAa,GAAK,GAAK,SAG5DP,EAAO,EAEPM,EAAUN,EAAOQ,GADjBN,EAAUjH,EAAI6C,OAAS,EACe6D,EAAKC,GACpCI,EAAOQ,GAAYvH,EAAI6C,OAAQ6D,EAAKC,IAC3CU,EAAUN,EAAOQ,GAAYvH,EAAI6C,OAAQ6D,EAAKC,GAC9CM,EAAUjH,EAAI6C,OAAS,IAEvBoE,EAAUjH,EAAI6C,OACdwE,EAAUN,GAGP,CACHA,KAAMM,EACNxE,KAAMoE,YAILM,GAAY1E,EAAM6D,EAAKC,OACxBW,EAAab,GAAgB5D,EAAM6D,EAAKC,GACxCa,EAAiBf,GAAgB5D,EAAO,EAAG6D,EAAKC,UAC5C/D,GAAWC,GAAQyE,EAAaE,GAAkB,WAoJrDC,GAAeC,EAAIC,UACjBD,EAAGhL,MAAMiL,EAAG,GAAGC,OAAOF,EAAGhL,MAAM,EAAGiL,IAhJ7CvI,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,QACrCA,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,WAIrC1B,EAAa,OAAQ,KACrBA,EAAa,UAAW,KAIxBY,EAAgB,OAAQ,GACxBA,EAAgB,UAAW,GAI3BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAE/B+B,GAAkB,CAAC,IAAK,KAAM,IAAK,OAAO,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GACrE0H,EAAK1H,EAAMN,OAAO,EAAG,IAAM5D,EAAM3F,MAsCrC4J,EAAe,IAAK,EAAG,KAAM,OAE7BA,EAAe,KAAM,EAAG,GAAG,SAAUnI,UAC1BuC,KAAKkG,aAAamI,YAAYrO,KAAMvC,MAG/CmI,EAAe,MAAO,EAAG,GAAG,SAAUnI,UAC3BuC,KAAKkG,aAAaoI,cAActO,KAAMvC,MAGjDmI,EAAe,OAAQ,EAAG,GAAG,SAAUnI,UAC5BuC,KAAKkG,aAAaqI,SAASvO,KAAMvC,MAG5CmI,EAAe,IAAK,EAAG,EAAG,WAC1BA,EAAe,IAAK,EAAG,EAAG,cAI1B1B,EAAa,MAAO,KACpBA,EAAa,UAAW,KACxBA,EAAa,aAAc,KAG3BY,EAAgB,MAAO,IACvBA,EAAgB,UAAW,IAC3BA,EAAgB,aAAc,IAI9BmD,GAAc,IAAQZ,GACtBY,GAAc,IAAQZ,GACtBY,GAAc,IAAQZ,GACtBY,GAAc,MAAQ,SAAUG,EAAU1K,UAC/BA,EAAO8Q,iBAAiBpG,MAEnCH,GAAc,OAAS,SAAUG,EAAU1K,UAChCA,EAAO+Q,mBAAmBrG,MAErCH,GAAc,QAAU,SAAUG,EAAU1K,UACjCA,EAAOgR,cAActG,MAGhCY,GAAkB,CAAC,KAAM,MAAO,SAAS,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,OAChE2H,EAAUtM,EAAOH,QAAQ4N,cAAc3S,EAAO6J,EAAO3E,EAAO1B,SAEjD,MAAXgO,EACAD,EAAKhB,EAAIiB,EAET1P,EAAgBoD,GAAQ3B,eAAiBvD,KAIjDgN,GAAkB,CAAC,IAAK,IAAK,MAAM,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GAC9D0H,EAAK1H,GAASlE,EAAM3F,UAkCpB4S,GAAwB,2DAA2D/D,MAAM,KAQzFgE,GAA6B,8BAA8BhE,MAAM,KAMjEiE,GAA2B,uBAAuBjE,MAAM,cAMnDkE,GAAoBC,EAAavR,EAAQE,OAC1Cb,EAAGmO,EAAIzE,EAAK0E,EAAM8D,EAAY7D,wBAC7BnL,KAAKiP,wBACDA,eAAiB,QACjBC,oBAAsB,QACtBC,kBAAoB,GAEpBrS,EAAI,EAAGA,EAAI,IAAKA,EACjB0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,QAC1BqS,kBAAkBrS,GAAKkD,KAAKqO,YAAY7H,EAAK,IAAI2E,yBACjD+D,oBAAoBpS,GAAKkD,KAAKsO,cAAc9H,EAAK,IAAI2E,yBACrD8D,eAAenS,GAAKkD,KAAKuO,SAAS/H,EAAK,IAAI2E,2BAIpDxN,EACe,SAAXF,GAEe,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,IACpBD,EAAK,KACN,QAAXxN,GAEQ,KADfwN,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,IACzBD,EAAK,MAGT,KADfA,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IACvBD,EAAK,KAGb,SAAXxN,GAEY,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAK3B,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,MAK7B,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IAN/BD,EAOa,KACN,QAAXxN,GAEK,KADZwN,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,MAKhC,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAKxB,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,IAN/BD,EAOa,MAGZ,KADZA,EAAKvB,GAAQrN,KAAK2D,KAAKmP,kBAAmBjE,MAK9B,KADZD,EAAKvB,GAAQrN,KAAK2D,KAAKiP,eAAgB/D,MAKxB,KADfD,EAAKvB,GAAQrN,KAAK2D,KAAKkP,oBAAqBhE,IANjCD,EAOa,SAqFhCoE,GAAuBtH,GAoBvBuH,GAA4BvH,GAoB5BwH,GAA0BxH,YAqBrByH,cACI3D,EAAU1O,EAAGC,UACXA,EAAEJ,OAASG,EAAEH,WAIpBF,EAAG0J,EAAKiJ,EAAMC,EAAQC,EADtBC,EAAY,GAAI9D,EAAc,GAAIC,EAAa,GAAIC,EAAc,OAEhElP,EAAI,EAAGA,EAAI,EAAGA,IAEf0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,GAC/B2S,EAAOzP,KAAKqO,YAAY7H,EAAK,IAC7BkJ,EAAS1P,KAAKsO,cAAc9H,EAAK,IACjCmJ,EAAQ3P,KAAKuO,SAAS/H,EAAK,IAC3BoJ,EAAU3S,KAAKwS,GACf3D,EAAY7O,KAAKyS,GACjB3D,EAAW9O,KAAK0S,GAChB3D,EAAY/O,KAAKwS,GACjBzD,EAAY/O,KAAKyS,GACjB1D,EAAY/O,KAAK0S,OAIrBC,EAAU3D,KAAKJ,GACfC,EAAYG,KAAKJ,GACjBE,EAAWE,KAAKJ,GAChBG,EAAYC,KAAKJ,GACZ/O,EAAI,EAAGA,EAAI,EAAGA,IACfgP,EAAYhP,GAAKyL,GAAYuD,EAAYhP,IACzCiP,EAAWjP,GAAKyL,GAAYwD,EAAWjP,IACvCkP,EAAYlP,GAAKyL,GAAYyD,EAAYlP,SAGxC+S,eAAiB,IAAIvH,OAAO,KAAO0D,EAAY7I,KAAK,KAAO,IAAK,UAChE2M,oBAAsB9P,KAAK6P,oBAC3BE,kBAAoB/P,KAAK6P,oBAEzBG,qBAAuB,IAAI1H,OAAO,KAAOyD,EAAW5I,KAAK,KAAO,IAAK,UACrE8M,0BAA4B,IAAI3H,OAAO,KAAOwD,EAAY3I,KAAK,KAAO,IAAK,UAC3E+M,wBAA0B,IAAI5H,OAAO,KAAOsH,EAAUzM,KAAK,KAAO,IAAK,cAKvEgN,YACEnQ,KAAKoQ,QAAU,IAAM,YA6BvBxR,GAAUiH,EAAOwK,GACtBzK,EAAeC,EAAO,EAAG,GAAG,kBACjB7F,KAAKkG,aAAatH,SAASoB,KAAKoQ,QAASpQ,KAAKsQ,UAAWD,eAgB/DE,GAAenI,EAAU1K,UACvBA,EAAO8S,eAzClB5K,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,QAClCA,EAAe,IAAK,CAAC,KAAM,GAAI,EAAGuK,IAClCvK,EAAe,IAAK,CAAC,KAAM,GAAI,qBALpB5F,KAAKoQ,SAAW,MAO3BxK,EAAe,MAAO,EAAG,GAAG,iBACjB,GAAKuK,GAAQtU,MAAMmE,MAAQgF,EAAShF,KAAKsQ,UAAW,MAG/D1K,EAAe,QAAS,EAAG,GAAG,iBACnB,GAAKuK,GAAQtU,MAAMmE,MAAQgF,EAAShF,KAAKsQ,UAAW,GACvDtL,EAAShF,KAAKyQ,UAAW,MAGjC7K,EAAe,MAAO,EAAG,GAAG,iBACjB,GAAK5F,KAAKoQ,QAAUpL,EAAShF,KAAKsQ,UAAW,MAGxD1K,EAAe,QAAS,EAAG,GAAG,iBACnB,GAAK5F,KAAKoQ,QAAUpL,EAAShF,KAAKsQ,UAAW,GAChDtL,EAAShF,KAAKyQ,UAAW,MASjC7R,GAAS,KAAK,GACdA,GAAS,KAAK,GAIdsF,EAAa,OAAQ,KAGrBY,EAAgB,OAAQ,IAQxBmD,GAAc,IAAMsI,IACpBtI,GAAc,IAAMsI,IACpBtI,GAAc,IAAMZ,GACpBY,GAAc,IAAMZ,GACpBY,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,KAAMZ,EAAWJ,GAE/BgB,GAAc,MAAOX,GACrBW,GAAc,QAASV,IACvBU,GAAc,MAAOX,GACrBW,GAAc,QAASV,IAEvBwB,GAAc,CAAC,IAAK,MA3+BT,GA4+BXA,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,OAC3CwP,EAAS/O,EAAM3F,GACnBqK,EA9+BO,GA8+BkB,KAAXqK,EAAgB,EAAIA,KAEtC3H,GAAc,CAAC,IAAK,MAAM,SAAU/M,EAAOqK,EAAOnF,GAC9CA,EAAOyP,MAAQzP,EAAOH,QAAQ6P,KAAK5U,GACnCkF,EAAO2P,UAAY7U,KAEvB+M,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,GAC/CmF,EAr/BO,GAq/BO1E,EAAM3F,GACpB8B,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,OAAO,SAAU/M,EAAOqK,EAAOnF,OACrC4P,EAAM9U,EAAMgB,OAAS,EACzBqJ,EA1/BO,GA0/BO1E,EAAM3F,EAAMuJ,OAAO,EAAGuL,IACpCzK,EA1/BS,GA0/BO1E,EAAM3F,EAAMuJ,OAAOuL,IACnChT,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,SAAS,SAAU/M,EAAOqK,EAAOnF,OACvC6P,EAAO/U,EAAMgB,OAAS,EACtBgU,EAAOhV,EAAMgB,OAAS,EAC1BqJ,EAjgCO,GAigCO1E,EAAM3F,EAAMuJ,OAAO,EAAGwL,IACpC1K,EAjgCS,GAigCO1E,EAAM3F,EAAMuJ,OAAOwL,EAAM,IACzC1K,EAjgCS,GAigCO1E,EAAM3F,EAAMuJ,OAAOyL,IACnClT,EAAgBoD,GAAQxB,SAAU,KAEtCqJ,GAAc,OAAO,SAAU/M,EAAOqK,EAAOnF,OACrC4P,EAAM9U,EAAMgB,OAAS,EACzBqJ,EAxgCO,GAwgCO1E,EAAM3F,EAAMuJ,OAAO,EAAGuL,IACpCzK,EAxgCS,GAwgCO1E,EAAM3F,EAAMuJ,OAAOuL,OAEvC/H,GAAc,SAAS,SAAU/M,EAAOqK,EAAOnF,OACvC6P,EAAO/U,EAAMgB,OAAS,EACtBgU,EAAOhV,EAAMgB,OAAS,EAC1BqJ,EA9gCO,GA8gCO1E,EAAM3F,EAAMuJ,OAAO,EAAGwL,IACpC1K,EA9gCS,GA8gCO1E,EAAM3F,EAAMuJ,OAAOwL,EAAM,IACzC1K,EA9gCS,GA8gCO1E,EAAM3F,EAAMuJ,OAAOyL,WAoDnCC,GAzBAC,GAAatH,GAAW,SAAS,GAEjCuH,GAAa,CACbC,SAj2CkB,CAClBC,QAAU,gBACVC,QAAU,mBACVC,SAAW,eACXC,QAAU,oBACVC,SAAW,sBACXC,SAAW,KA41CX7K,eAp1CwB,CACxB8K,IAAO,YACPC,GAAO,SACPC,EAAO,aACPC,GAAO,eACPC,IAAO,sBACPC,KAAO,6BA+0CPrL,YA7zCqB,eA8zCrBZ,QAxzCiB,KAyzCjBkM,uBAxzCgC,UAyzChCC,aAnzCsB,CACtBC,OAAS,QACTC,KAAS,SACTvJ,EAAK,gBACLwJ,GAAK,aACLtU,EAAK,WACLuU,GAAK,aACL9F,EAAK,UACL+F,GAAK,WACLhG,EAAK,QACLiG,GAAK,UACL/F,EAAK,UACLgG,GAAK,YACLlJ,EAAK,SACLmJ,GAAK,YAuyCLnI,OAAQK,GACRN,YAAaQ,GAEbyC,KA9gBoB,CACpBL,IAAM,EACNC,IAAM,GA8gBNoB,SAAUK,GACVP,YAAaS,GACbR,cAAeO,GAEf8D,cAnC6B,iBAuC7BC,GAAU,GACVC,GAAiB,YAGZC,GAAgB7P,UACdA,EAAMA,EAAIqB,cAAciC,QAAQ,IAAK,KAAOtD,WA8B9C8P,GAAWtP,OACZuP,EAAY,SAEXJ,GAAQnP,IACLhI,GAAUA,EAAOwX,YAEjBD,EAAY/B,GAAaiC,MACJC,IAErBC,GAAmBJ,GACrB,MAAOK,WAENT,GAAQnP,YAMV2P,GAAoBnQ,EAAKqQ,OAC1BC,SACAtQ,KAEIsQ,EADAhX,EAAY+W,GACLE,GAAUvQ,GAGVwQ,GAAaxQ,EAAKqQ,IAKzBrC,GAAesC,EAGU,oBAAb5Q,SAA6BA,QAAQH,MAE7CG,QAAQH,KAAK,UAAYS,EAAO,2CAKrCgO,GAAaiC,eAGfO,GAAchQ,EAAMvC,MACV,OAAXA,EAAiB,KACbxD,EAAQmG,EAAesN,MAC3BjQ,EAAOwS,KAAOjQ,EACO,MAAjBmP,GAAQnP,GACRD,EAAgB,uBACR,2OAIRK,EAAe+O,GAAQnP,GAAMkQ,aAC1B,GAA2B,MAAvBzS,EAAO0S,gBACsB,MAAhChB,GAAQ1R,EAAO0S,cACf/P,EAAe+O,GAAQ1R,EAAO0S,cAAcD,YACzC,IAEW,OADdjW,EAASqV,GAAW7R,EAAO0S,sBAIlBf,GAAe3R,EAAO0S,gBACvBf,GAAe3R,EAAO0S,cAAgB,IAE1Cf,GAAe3R,EAAO0S,cAAc3W,KAAK,CACrCwG,KAAMA,EACNvC,OAAQA,IAEL,KATP2C,EAAenG,EAAOiW,eAalCf,GAAQnP,GAAQ,IAAIM,EAAOH,EAAaC,EAAc3C,IAElD2R,GAAepP,IACfoP,GAAepP,GAAMoQ,SAAQ,SAAU1J,GACnCsJ,GAAatJ,EAAE1G,KAAM0G,EAAEjJ,WAO/BkS,GAAmB3P,GAGZmP,GAAQnP,iBAGRmP,GAAQnP,GACR,cAiCN+P,GAAWvQ,OACZvF,KAEAuF,GAAOA,EAAIlC,SAAWkC,EAAIlC,QAAQmS,QAClCjQ,EAAMA,EAAIlC,QAAQmS,QAGjBjQ,SACMgO,OAGNlV,EAAQkH,GAAM,IAEfvF,EAASqV,GAAW9P,UAETvF,EAEXuF,EAAM,CAACA,mBArKO6Q,WACPC,EAAGC,EAAMtW,EAAQmN,EAAxB/N,EAAI,EAEDA,EAAIgX,EAAM9W,QAAQ,KAErB+W,GADAlJ,EAAQiI,GAAgBgB,EAAMhX,IAAI+N,MAAM,MAC9B7N,OAEVgX,GADAA,EAAOlB,GAAgBgB,EAAMhX,EAAI,KACnBkX,EAAKnJ,MAAM,KAAO,KACzBkJ,EAAI,GAAG,IACVrW,EAASqV,GAAWlI,EAAM3H,MAAM,EAAG6Q,GAAG5Q,KAAK,aAEhCzF,KAEPsW,GAAQA,EAAKhX,QAAU+W,GAAK/R,EAAc6I,EAAOmJ,GAAM,IAASD,EAAI,QAIxEA,IAEJjX,WAEGmU,GAmJAgD,CAAahR,YAOfiR,GAAenW,OAChBK,EACAjB,EAAIY,EAAEoL,UAENhM,IAAsC,IAAjCW,EAAgBC,GAAGK,WACxBA,EACIjB,EApwCA,GAowCiB,GAAKA,EApwCtB,GAowCuC,GApwCvC,EAqwCAA,EApwCD,GAowCkB,GAAKA,EApwCvB,GAowCwC+M,GAAY/M,EAtwCpD,GAswC6DA,EArwC5D,IACD,EAqwCCA,EApwCD,GAowCkB,GAAKA,EApwCvB,GAowCwC,IAAmB,KAAZA,EApwC/C,KAowCgF,IAAdA,EAnwChE,IAmwCiG,IAAdA,EAlwCnF,IAkwCyH,IAAnBA,EAjwCjG,IAHP,EAqwCCA,EApwCC,GAowCgB,GAAKA,EApwCrB,GAowCsC,GApwCtC,EAqwCDA,EApwCC,GAowCgB,GAAKA,EApwCrB,GAowCsC,GApwCtC,EAqwCDA,EApwCM,GAowCW,GAAKA,EApwChB,GAowCiC,IApwCjC,GAqwCL,EAEDW,EAAgBC,GAAGoW,qBAAuB/V,EA7wC3C,GA6wC8DA,EA3wC9D,KA4wCCA,EA5wCD,GA8wCCN,EAAgBC,GAAGqW,iBAAgC,IAAdhW,IACrCA,EA1wCD,GA4wCCN,EAAgBC,GAAGsW,mBAAkC,IAAdjW,IACvCA,EA5wCE,GA+wCNN,EAAgBC,GAAGK,SAAWA,GAG3BL,WAIFuW,GAASnX,EAAGC,EAAGmX,UACX,MAALpX,EACOA,EAEF,MAALC,EACOA,EAEJmX,WAgBFC,GAAiBtT,OAClBpE,EAAGmN,EAAkBwK,EAAaC,EAAiBC,EAA1C3Y,EAAQ,OAEjBkF,EAAO7B,QAIXoV,WApBsBvT,OAElB0T,EAAW,IAAIlY,KAAKd,EAAMiZ,cAC1B3T,EAAO4T,QACA,CAACF,EAAS7H,iBAAkB6H,EAASG,cAAeH,EAASI,cAEjE,CAACJ,EAASjI,cAAeiI,EAASK,WAAYL,EAASM,WAchDC,CAAiBjU,GAG3BA,EAAO+H,IAAyB,MAAnB/H,EAAOiI,GA7zCjB,IA6zCyD,MAApBjI,EAAOiI,GA9zC3C,aAy3CmBjI,OACvBkU,EAAGC,EAAU9H,EAAMC,EAASN,EAAKC,EAAKmI,EAAMC,KAGpC,OADZH,EAAIlU,EAAO+H,IACLuM,IAAqB,MAAPJ,EAAEK,GAAoB,MAAPL,EAAEM,EACjCxI,EAAM,EACNC,EAAM,EAMNkI,EAAWf,GAASc,EAAEI,GAAItU,EAAOiI,GAt4C9B,GAs4CwCyE,GAAW+H,KAAe,EAAG,GAAGtM,MAC3EkE,EAAO+G,GAASc,EAAEK,EAAG,KACrBjI,EAAU8G,GAASc,EAAEM,EAAG,IACV,GAAKlI,EAAU,KACzB+H,GAAkB,OAEnB,CACHrI,EAAMhM,EAAOH,QAAQ6U,MAAM1I,IAC3BC,EAAMjM,EAAOH,QAAQ6U,MAAMzI,QAEvB0I,EAAUjI,GAAW+H,KAAezI,EAAKC,GAE7CkI,EAAWf,GAASc,EAAEU,GAAI5U,EAAOiI,GAl5C9B,GAk5CwC0M,EAAQxM,MAGnDkE,EAAO+G,GAASc,EAAEA,EAAGS,EAAQtI,MAElB,MAAP6H,EAAE7I,IAEFiB,EAAU4H,EAAE7I,GACE,GAAKiB,EAAU,KACzB+H,GAAkB,GAER,MAAPH,EAAE/B,GAET7F,EAAU4H,EAAE/B,EAAInG,GACZkI,EAAE/B,EAAI,GAAK+B,EAAE/B,EAAI,KACjBkC,GAAkB,IAItB/H,EAAUN,EAGdK,EAAO,GAAKA,EAAOQ,GAAYsH,EAAUnI,EAAKC,GAC9CrP,EAAgBoD,GAAQkT,gBAAiB,EACf,MAAnBmB,EACPzX,EAAgBoD,GAAQmT,kBAAmB,GAE3CiB,EAAOhI,GAAmB+H,EAAU9H,EAAMC,EAASN,EAAKC,GACxDjM,EAAOiI,GA96CJ,GA86CemM,EAAKjM,KACvBnI,EAAO6U,WAAaT,EAAK3H,WA/GzBqI,CAAsB9U,GAID,MAArBA,EAAO6U,aACPpB,EAAYL,GAASpT,EAAOiI,GAr0CzB,GAq0CmCsL,EAr0CnC,KAu0CCvT,EAAO6U,WAAa3M,GAAWuL,IAAoC,IAAtBzT,EAAO6U,cACpDjY,EAAgBoD,GAAQiT,oBAAqB,GAGjDlK,EAAO4C,GAAc8H,EAAW,EAAGzT,EAAO6U,YAC1C7U,EAAOiI,GA30CH,GA20Cec,EAAK8K,cACxB7T,EAAOiI,GA30CJ,GA20Cec,EAAK+K,cAQtBlY,EAAI,EAAGA,EAAI,GAAqB,MAAhBoE,EAAOiI,GAAGrM,KAAcA,EACzCoE,EAAOiI,GAAGrM,GAAKd,EAAMc,GAAK2X,EAAY3X,QAInCA,EAAI,EAAGA,IACVoE,EAAOiI,GAAGrM,GAAKd,EAAMc,GAAsB,MAAhBoE,EAAOiI,GAAGrM,GAAqB,IAANA,EAAU,EAAI,EAAKoE,EAAOiI,GAAGrM,GAI7D,KAApBoE,EAAOiI,GA51CJ,IA61CuB,IAAtBjI,EAAOiI,GA51CN,IA61CqB,IAAtBjI,EAAOiI,GA51CN,IA61C0B,IAA3BjI,EAAOiI,GA51CD,KA61CVjI,EAAO+U,UAAW,EAClB/U,EAAOiI,GAj2CJ,GAi2Ce,GAGtBjI,EAAO7B,IAAM6B,EAAO4T,QAAUjI,GAAgBP,IAAYzQ,MAAM,KAAMG,GACtE0Y,EAAkBxT,EAAO4T,QAAU5T,EAAO7B,GAAGgO,YAAcnM,EAAO7B,GAAG6W,SAIlD,MAAfhV,EAAON,MACPM,EAAO7B,GAAG8W,cAAcjV,EAAO7B,GAAG+W,gBAAkBlV,EAAON,MAG3DM,EAAO+U,WACP/U,EAAOiI,GA92CJ,GA82Ce,IAIlBjI,EAAO+H,SAA6B,IAAhB/H,EAAO+H,GAAGsD,GAAqBrL,EAAO+H,GAAGsD,IAAMmI,IACnE5W,EAAgBoD,GAAQpC,iBAAkB,QA+D9CuX,GAAmB,mJACnBC,GAAgB,8IAEhBC,GAAU,wBAEVC,GAAW,CACX,CAAC,eAAgB,uBACjB,CAAC,aAAc,mBACf,CAAC,eAAgB,kBACjB,CAAC,aAAc,eAAe,GAC9B,CAAC,WAAY,eACb,CAAC,UAAW,cAAc,GAC1B,CAAC,aAAc,cACf,CAAC,WAAY,UAEZ,aAAc,eACf,CAAC,YAAa,eAAe,GAC7B,CAAC,UAAW,UAIZC,GAAW,CACX,CAAC,gBAAiB,uBAClB,CAAC,gBAAiB,sBAClB,CAAC,WAAY,kBACb,CAAC,QAAS,aACV,CAAC,cAAe,qBAChB,CAAC,cAAe,oBAChB,CAAC,SAAU,gBACX,CAAC,OAAQ,YACT,CAAC,KAAM,SAGPC,GAAkB,+BAGbC,GAAczV,OACfpE,EAAG8Z,EAGHC,EAAWC,EAAYC,EAAYC,EAFnCC,EAAS/V,EAAOT,GAChB6F,EAAQ+P,GAAiBa,KAAKD,IAAWX,GAAcY,KAAKD,MAG5D3Q,EAAO,KACPxI,EAAgBoD,GAAQxC,KAAM,EAEzB5B,EAAI,EAAG8Z,EAAIJ,GAASxZ,OAAQF,EAAI8Z,EAAG9Z,OAChC0Z,GAAS1Z,GAAG,GAAGoa,KAAK5Q,EAAM,IAAK,CAC/BwQ,EAAaN,GAAS1Z,GAAG,GACzB+Z,GAA+B,IAAnBL,GAAS1Z,GAAG,YAId,MAAdga,cACA5V,EAAOlC,UAAW,MAGlBsH,EAAM,GAAI,KACLxJ,EAAI,EAAG8Z,EAAIH,GAASzZ,OAAQF,EAAI8Z,EAAG9Z,OAChC2Z,GAAS3Z,GAAG,GAAGoa,KAAK5Q,EAAM,IAAK,CAE/ByQ,GAAczQ,EAAM,IAAM,KAAOmQ,GAAS3Z,GAAG,YAInC,MAAdia,cACA7V,EAAOlC,UAAW,OAIrB6X,GAA2B,MAAdE,cACd7V,EAAOlC,UAAW,MAGlBsH,EAAM,GAAI,KACNiQ,GAAQW,KAAK5Q,EAAM,gBAGnBpF,EAAOlC,UAAW,GAFlBgY,EAAW,IAMnB9V,EAAOR,GAAKoW,GAAcC,GAAc,KAAOC,GAAY,IAC3DG,GAA0BjW,QAE1BA,EAAOlC,UAAW,MAKtBH,GAAU,mMAkBLuY,GAAeC,OAChBhO,EAAOI,SAAS4N,EAAS,WACzBhO,GAAQ,GACD,IAAOA,EACPA,GAAQ,IACR,KAAOA,EAEXA,MAsBPiO,GAAa,CACbC,GAAI,EACJC,IAAK,EACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,IACLC,KAAK,cAiBAC,GAAkB/W,OAxEQmW,EAASa,EAAUC,EAAQC,EAASC,EAAWC,EAC1EC,EAwEAjS,EAAQzH,GAAQqY,KAAuBhW,EAAOT,GA7CzC8F,QAAQ,oBAAqB,KAAKA,QAAQ,WAAY,KAAKA,QAAQ,SAAU,IAAIA,QAAQ,SAAU,QA8CxGD,EAAO,KACHkS,GA3EuBnB,EA2EiB/Q,EAAM,GA3Ed4R,EA2EkB5R,EAAM,GA3Ed6R,EA2EkB7R,EAAM,GA3EhB8R,EA2EoB9R,EAAM,GA3EjB+R,EA2EqB/R,EAAM,GA3EhBgS,EA2EoBhS,EAAM,GA1EpGiS,EAAS,CACTnB,GAAeC,GACfvM,GAAyBpB,QAAQwO,GACjCzO,SAAS0O,EAAQ,IACjB1O,SAAS2O,EAAS,IAClB3O,SAAS4O,EAAW,KAGpBC,GACAC,EAAOtb,KAAKwM,SAAS6O,EAAW,KAG7BC,gBAkBWE,EAAYC,EAAaxX,UACvCuX,GAEsB5J,GAA2BnF,QAAQ+O,KACrC,IAAI/b,KAAKgc,EAAY,GAAIA,EAAY,GAAIA,EAAY,IAAIxC,WAEzEpY,EAAgBoD,GAAQpC,iBAAkB,EAC1CoC,EAAOlC,UAAW,GACX,GAqCN2Z,CAAarS,EAAM,GAAIkS,EAAatX,UAIzCA,EAAOiI,GAAKqP,EACZtX,EAAON,cAvBUgY,EAAWC,EAAgBC,MAC5CF,SACOtB,GAAWsB,GACf,GAAIC,SAEA,MAEHE,EAAKtP,SAASqP,EAAW,IACzB/a,EAAIgb,EAAK,WAAUA,EAAKhb,GAAK,IACtB,GAAKA,EAcFib,CAAgB1S,EAAM,GAAIA,EAAM,GAAIA,EAAM,KAExDpF,EAAO7B,GAAKwN,GAAchR,MAAM,KAAMqF,EAAOiI,IAC7CjI,EAAO7B,GAAG8W,cAAcjV,EAAO7B,GAAG+W,gBAAkBlV,EAAON,MAE3D9C,EAAgBoD,GAAQrC,SAAU,OAElCqC,EAAOlC,UAAW,WAgDjBmY,GAA0BjW,MAE3BA,EAAOR,KAAO9E,EAAMqd,YAIpB/X,EAAOR,KAAO9E,EAAMsd,UAIxBhY,EAAOiI,GAAK,GACZrL,EAAgBoD,GAAQjD,OAAQ,MAI5BnB,EAAG4b,EAAa5P,EAAQjD,EAAOsT,EAD/BlC,EAAS,GAAK/V,EAAOT,GAErB2Y,EAAenC,EAAOja,OACtBqc,EAAyB,MAE7BvQ,EAAS1C,EAAalF,EAAOR,GAAIQ,EAAOH,SAASuF,MAAMd,IAAqB,GAEvE1I,EAAI,EAAGA,EAAIgM,EAAO9L,OAAQF,IAC3B+I,EAAQiD,EAAOhM,IACf4b,GAAezB,EAAO3Q,MAAM+B,GAAsBxC,EAAO3E,KAAY,IAAI,OAIrEiY,EAAUlC,EAAO1R,OAAO,EAAG0R,EAAOvN,QAAQgP,KAC9B1b,OAAS,GACjBc,EAAgBoD,GAAQ/C,YAAYlB,KAAKkc,GAE7ClC,EAASA,EAAO/T,MAAM+T,EAAOvN,QAAQgP,GAAeA,EAAY1b,QAChEqc,GAA0BX,EAAY1b,QAGtC2I,EAAqBE,IACjB6S,EACA5a,EAAgBoD,GAAQjD,OAAQ,EAGhCH,EAAgBoD,GAAQhD,aAAajB,KAAK4I,GAE9CqD,GAAwBrD,EAAO6S,EAAaxX,IAEvCA,EAAO1B,UAAYkZ,GACxB5a,EAAgBoD,GAAQhD,aAAajB,KAAK4I,GAKlD/H,EAAgBoD,GAAQ7C,cAAgB+a,EAAeC,EACnDpC,EAAOja,OAAS,GAChBc,EAAgBoD,GAAQ/C,YAAYlB,KAAKga,GAIzC/V,EAAOiI,GA7sDJ,IA6sDgB,KACiB,IAApCrL,EAAgBoD,GAAQxB,SACxBwB,EAAOiI,GA/sDJ,GA+sDe,IAClBrL,EAAgBoD,GAAQxB,aAAUD,GAGtC3B,EAAgBoD,GAAQvC,gBAAkBuC,EAAOiI,GAAGjG,MAAM,GAC1DpF,EAAgBoD,GAAQtC,SAAWsC,EAAO2P,UAE1C3P,EAAOiI,GAttDA,YA6tDezL,EAAQ4b,EAAM1a,OAChC2a,SAEY,MAAZ3a,EAEO0a,EAEgB,MAAvB5b,EAAO8b,aACA9b,EAAO8b,aAAaF,EAAM1a,GACX,MAAflB,EAAOkT,OAEd2I,EAAO7b,EAAOkT,KAAKhS,KACP0a,EAAO,KACfA,GAAQ,IAEPC,GAAiB,KAATD,IACTA,EAAO,GAEJA,GAGAA,EA5BOG,CAAgBvY,EAAOH,QAASG,EAAOiI,GAttDlD,GAstD4DjI,EAAO2P,WAE1E2D,GAAgBtT,GAChBgT,GAAchT,QA7DV+W,GAAkB/W,QAJlByV,GAAczV,YAmKbwY,GAAexY,OAChBlF,EAAQkF,EAAOT,GACfhD,EAASyD,EAAOR,UAEpBQ,EAAOH,QAAUG,EAAOH,SAAWyS,GAAUtS,EAAOP,IAEtC,OAAV3E,QAA8ByD,IAAXhC,GAAkC,KAAVzB,EACpC4D,EAAc,CAACtB,WAAW,KAGhB,iBAAVtC,IACPkF,EAAOT,GAAKzE,EAAQkF,EAAOH,QAAQ4Y,SAAS3d,IAG5CoF,EAASpF,GACF,IAAIiF,EAAOiT,GAAclY,KACzBS,EAAOT,GACdkF,EAAO7B,GAAKrD,EACLD,EAAQ0B,YAtFWyD,OAC1B0Y,EACAC,EAEAC,EACAhd,EACAid,KAEqB,IAArB7Y,EAAOR,GAAG1D,cACVc,EAAgBoD,GAAQ1C,eAAgB,OACxC0C,EAAO7B,GAAK,IAAI3C,KAAKmD,UAIpB/C,EAAI,EAAGA,EAAIoE,EAAOR,GAAG1D,OAAQF,IAC9Bid,EAAe,EACfH,EAAazZ,EAAW,GAAIe,GACN,MAAlBA,EAAO4T,UACP8E,EAAW9E,QAAU5T,EAAO4T,SAEhC8E,EAAWlZ,GAAKQ,EAAOR,GAAG5D,GAC1Bqa,GAA0ByC,GAErB7a,EAAQ6a,KAKbG,GAAgBjc,EAAgB8b,GAAYvb,cAG5C0b,GAAkE,GAAlDjc,EAAgB8b,GAAY1b,aAAalB,OAEzDc,EAAgB8b,GAAYI,MAAQD,GAEjB,MAAfD,GAAuBC,EAAeD,KACtCA,EAAcC,EACdF,EAAaD,IAIrBtc,EAAO4D,EAAQ2Y,GAAcD,GA8CzBK,CAAyB/Y,GAClBzD,EACP0Z,GAA0BjW,YAYTA,OACjBlF,EAAQkF,EAAOT,GACflE,EAAYP,GACZkF,EAAO7B,GAAK,IAAI3C,KAAKd,EAAMiZ,OACpBpY,EAAOT,GACdkF,EAAO7B,GAAK,IAAI3C,KAAKV,EAAMuB,WACH,iBAAVvB,WAxPIkF,OAClBsH,EAAUkO,GAAgBQ,KAAKhW,EAAOT,IAE1B,OAAZ+H,GAKJmO,GAAczV,IACU,IAApBA,EAAOlC,kBACAkC,EAAOlC,SAKlBiZ,GAAkB/W,IACM,IAApBA,EAAOlC,kBACAkC,EAAOlC,SAMlBpD,EAAMse,wBAAwBhZ,MAnB1BA,EAAO7B,GAAK,IAAI3C,MAAM8L,EAAQ,IAqP9B2R,CAAiBjZ,GACVnF,EAAQC,IACfkF,EAAOiI,GAAKxM,EAAIX,EAAMkH,MAAM,IAAI,SAAU7B,UAC/BoI,SAASpI,EAAK,OAEzBmT,GAAgBtT,IACT5E,EAASN,YAtEEkF,OAClBA,EAAO7B,QAIPvC,EAAI2H,EAAqBvD,EAAOT,IACpCS,EAAOiI,GAAKxM,EAAI,CAACG,EAAEuM,KAAMvM,EAAEkN,MAAOlN,EAAEsS,KAAOtS,EAAEmN,KAAMnN,EAAEwc,KAAMxc,EAAEsd,OAAQtd,EAAEud,OAAQvd,EAAEwd,cAAc,SAAUjZ,UAC9FA,GAAOoI,SAASpI,EAAK,OAGhCmT,GAAgBtT,IA6DZqZ,CAAiBrZ,GACV1E,EAASR,GAEhBkF,EAAO7B,GAAK,IAAI3C,KAAKV,GAErBJ,EAAMse,wBAAwBhZ,GA7B9BsZ,CAAgBtZ,GAGfnC,EAAQmC,KACTA,EAAO7B,GAAK,MAGT6B,aA0BFtD,GAAkB5B,EAAOyB,EAAQC,EAAQC,EAAQ8c,OAlElD1d,EAmEAwX,EAAI,UAEO,IAAX7W,IAA8B,IAAXA,IACnBC,EAASD,EACTA,OAAS+B,IAGRnD,EAASN,aA5iFKqF,MACfnF,OAAOwe,2BAC4C,IAA3Cxe,OAAOwe,oBAAoBrZ,GAAKrE,WAEpC2d,MACCA,KAAKtZ,KACFA,EAAIhE,eAAesd,UACZ,SAGR,EAkiFaC,CAAc5e,IAC7BD,EAAQC,IAA2B,IAAjBA,EAAMgB,UAC7BhB,OAAQyD,GAIZ8U,EAAE/T,kBAAmB,EACrB+T,EAAEO,QAAUP,EAAE1T,OAAS4Z,EACvBlG,EAAE5T,GAAKjD,EACP6W,EAAE9T,GAAKzE,EACPuY,EAAE7T,GAAKjD,EACP8W,EAAE/U,QAAU7B,GArFRZ,EAAM,IAAIkE,EAAOiT,GAAcwF,GAuFXnF,MAtFhB0B,WAEJlZ,EAAI8d,IAAI,EAAG,KACX9d,EAAIkZ,cAAWxW,GAGZ1C,WAmFF4Y,GAAa3Z,EAAOyB,EAAQC,EAAQC,UAClCC,GAAiB5B,EAAOyB,EAAQC,EAAQC,GAAQ,GAxQ3D/B,EAAMse,wBAA0BtX,EAC5B,kVAIA,SAAU1B,GACNA,EAAO7B,GAAK,IAAI3C,KAAKwE,EAAOT,IAAMS,EAAO4T,QAAU,OAAS,QAKpElZ,EAAMqd,SAAW,aAGjBrd,EAAMsd,SAAW,iBA6Pb4B,GAAelY,EACf,sGACA,eACQmY,EAAQpF,GAAY9Z,MAAM,KAAMC,kBAChCkE,KAAKjB,WAAagc,EAAMhc,UACjBgc,EAAQ/a,KAAOA,KAAO+a,EAEtBnb,OAKfob,GAAepY,EACf,sGACA,eACQmY,EAAQpF,GAAY9Z,MAAM,KAAMC,kBAChCkE,KAAKjB,WAAagc,EAAMhc,UACjBgc,EAAQ/a,KAAOA,KAAO+a,EAEtBnb,gBAUVqb,GAAOpe,EAAIqe,OACZne,EAAKD,KACc,IAAnBoe,EAAQle,QAAgBjB,EAAQmf,EAAQ,MACxCA,EAAUA,EAAQ,KAEjBA,EAAQle,cACF2Y,SAEX5Y,EAAMme,EAAQ,GACTpe,EAAI,EAAGA,EAAIoe,EAAQle,SAAUF,EACzBoe,EAAQpe,GAAGiC,YAAamc,EAAQpe,GAAGD,GAAIE,KACxCA,EAAMme,EAAQpe,WAGfC,MAoBPoe,GAAW,CAAC,OAAQ,UAAW,QAAS,OAAQ,MAAO,OAAQ,SAAU,SAAU,wBAgC9EC,GAAUC,OACXzW,EAAkBH,EAAqB4W,GACvCC,EAAQ1W,EAAgByE,MAAQ,EAChCkS,EAAW3W,EAAgB4W,SAAW,EACtCjR,EAAS3F,EAAgBoF,OAAS,EAClCyR,EAAQ7W,EAAgB2I,MAAQ3I,EAAgB8W,SAAW,EAC3DC,EAAO/W,EAAgBwK,KAAO,EAC9BgB,EAAQxL,EAAgB0U,MAAQ,EAChChJ,EAAU1L,EAAgBwV,QAAU,EACpC3J,EAAU7L,EAAgByV,QAAU,EACpCuB,EAAehX,EAAgB0V,aAAe,OAE7Ctb,kBA1CgBjB,OAChB,IAAIkF,KAAOlF,MAC2B,IAAjC2L,GAAQrN,KAAK8e,GAAUlY,IAA0B,MAAVlF,EAAEkF,IAAiB7D,MAAMrB,EAAEkF,WAC7D,UAIX4Y,GAAiB,EACZ/e,EAAI,EAAGA,EAAIqe,GAASne,SAAUF,KAC/BiB,EAAEod,GAASre,IAAK,IACZ+e,SACO,EAEPC,WAAW/d,EAAEod,GAASre,OAAS6E,EAAM5D,EAAEod,GAASre,OAChD+e,GAAiB,UAKtB,EAuBSE,CAAgBnX,QAG3BoX,eAAiBJ,EACR,IAAVnL,EACU,IAAVH,EACQ,IAARF,EAAe,GAAK,QAGnB6L,OAASN,EACF,EAARF,OAICS,SAAW3R,EACD,EAAXgR,EACQ,GAARD,OAECa,MAAQ,QAERpb,QAAUyS,UAEV4I,mBAGAC,GAAYhb,UACVA,aAAe+Z,YAGjBkB,GAAU/a,UACXA,EAAS,GACyB,EAA3BC,KAAK+a,OAAO,EAAIhb,GAEhBC,KAAK+a,MAAMhb,YAMjBib,GAAQ3W,EAAO4W,GACpB7W,EAAeC,EAAO,EAAG,GAAG,eACpB2W,EAASxc,KAAK0c,YACdC,EAAO,WACPH,EAAS,IACTA,GAAUA,EACVG,EAAO,KAEJA,EAAO3X,KAAYwX,EAAS,IAAK,GAAKC,EAAYzX,IAAYwX,EAAU,GAAI,MAI3FA,GAAO,IAAK,KACZA,GAAO,KAAM,IAIbvU,GAAc,IAAMH,IACpBG,GAAc,KAAMH,IACpBiB,GAAc,CAAC,IAAK,OAAO,SAAU/M,EAAOqK,EAAOnF,GAC/CA,EAAO4T,SAAU,EACjB5T,EAAON,KAAOgc,GAAiB9U,GAAkB9L,UAQjD6gB,GAAc,2BAETD,GAAiBE,EAAS7F,OAC3B8F,GAAW9F,GAAU,IAAI3Q,MAAMwW,MAEnB,OAAZC,SACO,SAIPC,IADUD,EAAQA,EAAQ/f,OAAS,IAAM,IACtB,IAAIsJ,MAAMuW,KAAgB,CAAC,IAAK,EAAG,GACtDvM,EAAuB,GAAX0M,EAAM,GAAWrb,EAAMqb,EAAM,WAE1B,IAAZ1M,EACL,EACa,MAAb0M,EAAM,GAAa1M,GAAWA,WAI3B2M,GAAgBjhB,EAAOkhB,OACxBngB,EAAKogB,SACLD,EAAMrc,QACN9D,EAAMmgB,EAAME,QACZD,GAAQ/b,EAASpF,IAAUS,EAAOT,GAASA,EAAMuB,UAAYoY,GAAY3Z,GAAOuB,WAAaR,EAAIQ,UAEjGR,EAAIsC,GAAGge,QAAQtgB,EAAIsC,GAAG9B,UAAY4f,GAClCvhB,EAAMuF,aAAapE,GAAK,GACjBA,GAEA4Y,GAAY3Z,GAAOshB,iBAIzBC,GAAexf,UAGgC,IAA5CyD,KAAK+a,MAAMxe,EAAEsB,GAAGme,oBAAsB,aAqJzCC,aACEzd,KAAKjB,WAAYiB,KAAKa,QAA2B,IAAjBb,KAAKc,QA/IhDlF,EAAMuF,aAAe,iBAmJjBuc,GAAc,2DAKdC,GAAW,+KAENC,GAAgB5hB,EAAOiH,OAIxB0Z,EACAkB,EACAC,EAiFmBC,EAAMhD,EACzBhe,EAvFAse,EAAWrf,EAEXsK,EAAQ,YAKR+V,GAAWrgB,GACXqf,EAAW,CACP3O,GAAK1Q,EAAMggB,cACXzP,EAAKvQ,EAAMigB,MACXxP,EAAKzQ,EAAMkgB,SAER1f,EAASR,IAChBqf,EAAW,GACPpY,EACAoY,EAASpY,GAAOjH,EAEhBqf,EAASO,aAAe5f,IAElBsK,EAAQoX,GAAYxG,KAAKlb,KACnC2gB,EAAqB,MAAbrW,EAAM,IAAe,EAAI,EACjC+U,EAAW,CACP9R,EAAK,EACLgD,EAAK5K,EAAM2E,EA7xEZ,IA6xEmDqW,EAClDnQ,EAAK7K,EAAM2E,EA7xEZ,IA6xEmDqW,EAClD5e,EAAK4D,EAAM2E,EA7xEV,IA6xEiDqW,EAClD9T,EAAKlH,EAAM2E,EA7xEV,IA6xEiDqW,EAClDjQ,GAAK/K,EAAM2a,GAA8B,IAArBhW,EA7xEd,KA6xE4CqW,KAE5CrW,EAAQqX,GAASzG,KAAKlb,KAChC2gB,EAAqB,MAAbrW,EAAM,IAAe,EAAI,EACjC+U,EAAW,CACP9R,EAAIyU,GAAS1X,EAAM,GAAIqW,GACvBlQ,EAAIuR,GAAS1X,EAAM,GAAIqW,GACvBvH,EAAI4I,GAAS1X,EAAM,GAAIqW,GACvBpQ,EAAIyR,GAAS1X,EAAM,GAAIqW,GACvBnQ,EAAIwR,GAAS1X,EAAM,GAAIqW,GACvB5e,EAAIigB,GAAS1X,EAAM,GAAIqW,GACvB9T,EAAImV,GAAS1X,EAAM,GAAIqW,KAER,MAAZtB,EACPA,EAAW,GACgB,iBAAbA,IAA0B,SAAUA,GAAY,OAAQA,KA2CnD0C,EA1CSpI,GAAY0F,EAAShb,MA0CxB0a,EA1C+BpF,GAAY0F,EAASjb,IAA7E0d,EA4CEC,EAAKhf,WAAagc,EAAMhc,WAI9Bgc,EAAQkC,GAAgBlC,EAAOgD,GAC3BA,EAAKE,SAASlD,GACdhe,EAAMmhB,GAA0BH,EAAMhD,KAEtChe,EAAMmhB,GAA0BnD,EAAOgD,IACnCnC,cAAgB7e,EAAI6e,aACxB7e,EAAIwN,QAAUxN,EAAIwN,QAGfxN,GAZI,CAAC6e,aAAc,EAAGrR,OAAQ,IA3CjC8Q,EAAW,IACF3O,GAAKoR,EAAQlC,aACtBP,EAAS5O,EAAIqR,EAAQvT,QAGzBsT,EAAM,IAAIzC,GAASC,GAEfgB,GAAWrgB,IAAUkB,EAAWlB,EAAO,aACvC6hB,EAAI9c,QAAU/E,EAAM+E,SAGjB8c,WAMFG,GAAUG,EAAKxB,OAIhB5f,EAAMohB,GAAOrC,WAAWqC,EAAI5X,QAAQ,IAAK,aAErCnH,MAAMrC,GAAO,EAAIA,GAAO4f,WAG3BuB,GAA0BH,EAAMhD,OACjChe,EAAM,UAEVA,EAAIwN,OAASwQ,EAAM/Q,QAAU+T,EAAK/T,QACC,IAA9B+Q,EAAM1R,OAAS0U,EAAK1U,QACrB0U,EAAKX,QAAQvC,IAAI9d,EAAIwN,OAAQ,KAAK6T,QAAQrD,MACxChe,EAAIwN,OAGVxN,EAAI6e,cAAgBb,GAAUgD,EAAKX,QAAQvC,IAAI9d,EAAIwN,OAAQ,KAEpDxN,WAsBFshB,GAAYC,EAAW7a,UACrB,SAAUlD,EAAKge,OACTC,SAEM,OAAXD,GAAoBnf,OAAOmf,KAC3B/a,EAAgBC,EAAM,YAAcA,EAAQ,uDAAyDA,EAAO,kGAE5G+a,EAAMje,EAAKA,EAAMge,EAAQA,EAASC,GAKtCC,GAAYze,KADN4d,GADNrd,EAAqB,iBAARA,GAAoBA,EAAMA,EACbge,GACHD,GAChBte,eAINye,GAAajY,EAAK6U,EAAUqD,EAAUvd,OACvCya,EAAeP,EAASW,cACxBL,EAAOW,GAASjB,EAASY,OACzB1R,EAAS+R,GAASjB,EAASa,SAE1B1V,EAAIzH,YAKToC,EAA+B,MAAhBA,GAA8BA,EAEzCoJ,GACAgB,GAAS/E,EAAKuD,GAAIvD,EAAK,SAAW+D,EAASmU,GAE3C/C,GACA7R,GAAMtD,EAAK,OAAQuD,GAAIvD,EAAK,QAAUmV,EAAO+C,GAE7C9C,GACApV,EAAInH,GAAGge,QAAQ7W,EAAInH,GAAG9B,UAAYqe,EAAe8C,GAEjDvd,GACAvF,EAAMuF,aAAaqF,EAAKmV,GAAQpR,IApFxCqT,GAAe/gB,GAAKue,GAASjf,UAC7ByhB,GAAee,0BAtVJf,GAAe/d,UA6atBgb,GAAWwD,GAAY,EAAG,OAC1BO,GAAWP,IAAa,EAAG,qBA0HtBQ,GAAW1hB,EAAGC,OAEf0hB,EAA0C,IAAvB1hB,EAAEiM,OAASlM,EAAEkM,SAAiBjM,EAAE4M,QAAU7M,EAAE6M,SAE/D+U,EAAS5hB,EAAEigB,QAAQvC,IAAIiE,EAAgB,kBAclCA,GAXL1hB,EAAI2hB,EAAS,GAGH3hB,EAAI2hB,IAAWA,EAFf5hB,EAAEigB,QAAQvC,IAAIiE,EAAiB,EAAG,YAMlC1hB,EAAI2hB,IAFJ5hB,EAAEigB,QAAQvC,IAAIiE,EAAiB,EAAG,UAETC,MAIF,WA6FhCrhB,GAAQuF,OACT+b,cAEQvf,IAARwD,EACOjD,KAAKe,QAAQmS,OAGC,OADrB8L,EAAgBxL,GAAUvQ,WAEjBlC,QAAUie,GAEZhf,MApGfpE,EAAMqjB,cAAgB,uBACtBrjB,EAAMsjB,iBAAmB,6BAuGrBC,GAAOvc,EACP,mJACA,SAAUK,eACMxD,IAARwD,EACOjD,KAAKkG,aAELlG,KAAKtC,OAAOuF,eAKtBiD,YACElG,KAAKe,iBASPqe,GAAMC,EAAUC,UACbD,EAAWC,EAAUA,GAAWA,WAGnCC,GAAiBhW,EAAGxL,EAAGwO,UAExBhD,EAAI,KAAOA,GAAK,EAET,IAAI7M,KAAK6M,EAAI,IAAKxL,EAAGwO,GAXb,YAaR,IAAI7P,KAAK6M,EAAGxL,EAAGwO,GAAGhP,mBAIxBiiB,GAAejW,EAAGxL,EAAGwO,UAEtBhD,EAAI,KAAOA,GAAK,EAET7M,KAAKoQ,IAAIvD,EAAI,IAAKxL,EAAGwO,GArBb,YAuBR7P,KAAKoQ,IAAIvD,EAAGxL,EAAGwO,YAuKrBkT,GAAwB5Z,EAAO6Z,GACpC9Z,EAAe,EAAG,CAACC,EAAOA,EAAM7I,QAAS,EAAG0iB,YA+DvCC,GAAqB3jB,EAAOuR,EAAMC,EAASN,EAAKC,OACjDyS,SACS,MAAT5jB,EACO4R,GAAW5N,KAAMkN,EAAKC,GAAK9D,MAG9BkE,GADJqS,EAAc7R,GAAY/R,EAAOkR,EAAKC,MAElCI,EAAOqS,GAEJC,GAAWxjB,KAAK2D,KAAMhE,EAAOuR,EAAMC,EAASN,EAAKC,aAIvD0S,GAAWxK,EAAU9H,EAAMC,EAASN,EAAKC,OAC1C2S,EAAgBxS,GAAmB+H,EAAU9H,EAAMC,EAASN,EAAKC,GACjElD,EAAO4C,GAAciT,EAAczW,KAAM,EAAGyW,EAAcnS,uBAEzDtE,KAAKY,EAAK8C,uBACV/C,MAAMC,EAAK8K,oBACX9K,KAAKA,EAAK+K,cACRhV,KA5FX4F,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAKqV,WAAa,OAG7BzP,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,kBACrB5F,KAAK+f,cAAgB,OAOhCN,GAAuB,OAAY,YACnCA,GAAuB,QAAY,YACnCA,GAAuB,OAAS,eAChCA,GAAuB,QAAS,eAIhCvb,EAAa,WAAY,MACzBA,EAAa,cAAe,MAI5BY,EAAgB,WAAY,GAC5BA,EAAgB,cAAe,GAK/BmD,GAAc,IAAUL,IACxBK,GAAc,IAAUL,IACxBK,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,KAAUZ,EAAWJ,GACnCgB,GAAc,OAAUR,GAAWN,GACnCc,GAAc,OAAUR,GAAWN,GACnCc,GAAc,QAAUP,GAAWN,GACnCa,GAAc,QAAUP,GAAWN,GAEnC4B,GAAkB,CAAC,OAAQ,QAAS,OAAQ,UAAU,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GACjF0H,EAAK1H,EAAMN,OAAO,EAAG,IAAM5D,EAAM3F,MAGrCgN,GAAkB,CAAC,KAAM,OAAO,SAAUhN,EAAOuR,EAAMrM,EAAQ2E,GAC3D0H,EAAK1H,GAASjK,EAAM4N,kBAAkBxN,MAqD1C4J,EAAe,IAAK,EAAG,KAAM,WAI7B1B,EAAa,UAAW,KAIxBY,EAAgB,UAAW,GAI3BmD,GAAc,IAAKjB,GACnB+B,GAAc,KAAK,SAAU/M,EAAOqK,GAChCA,EAz8FQ,GAy8F4B,GAApB1E,EAAM3F,GAAS,MAWnC4J,EAAe,IAAK,CAAC,KAAM,GAAI,KAAM,QAIrC1B,EAAa,OAAQ,KAGrBY,EAAgB,OAAQ,GAIxBmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/BgB,GAAc,MAAM,SAAUG,EAAU1K,UAE7B0K,EACJ1K,EAAOsiB,yBAA2BtiB,EAAOuiB,cAC1CviB,EAAOwiB,kCAGbnX,GAAc,CAAC,IAAK,MAv+FT,GAw+FXA,GAAc,MAAM,SAAU/M,EAAOqK,GACjCA,EAz+FO,GAy+FO1E,EAAM3F,EAAMsK,MAAMe,GAAW,WAK3C8Y,GAAmBvW,GAAW,QAAQ,GAI1ChE,EAAe,MAAO,CAAC,OAAQ,GAAI,OAAQ,aAI3C1B,EAAa,YAAa,OAG1BY,EAAgB,YAAa,GAI7BmD,GAAc,MAAQT,IACtBS,GAAc,OAAQf,GACtB6B,GAAc,CAAC,MAAO,SAAS,SAAU/M,EAAOqK,EAAOnF,GACnDA,EAAO6U,WAAapU,EAAM3F,MAc9B4J,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,UAIlC1B,EAAa,SAAU,KAIvBY,EAAgB,SAAU,IAI1BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/B8B,GAAc,CAAC,IAAK,MA1hGP,OA8hGTqX,GAAexW,GAAW,WAAW,GAIzChE,EAAe,IAAK,CAAC,KAAM,GAAI,EAAG,UAIlC1B,EAAa,SAAU,KAIvBY,EAAgB,SAAU,IAI1BmD,GAAc,IAAMZ,GACpBY,GAAc,KAAMZ,EAAWJ,GAC/B8B,GAAc,CAAC,IAAK,MA/iGP,OAkmGTlD,GA/CAwa,GAAezW,GAAW,WAAW,OAIzChE,EAAe,IAAK,EAAG,GAAG,oBACZ5F,KAAKsa,cAAgB,QAGnC1U,EAAe,EAAG,CAAC,KAAM,GAAI,GAAG,oBAClB5F,KAAKsa,cAAgB,OAGnC1U,EAAe,EAAG,CAAC,MAAO,GAAI,EAAG,eACjCA,EAAe,EAAG,CAAC,OAAQ,GAAI,GAAG,kBACF,GAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,QAAS,GAAI,GAAG,kBACH,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,SAAU,GAAI,GAAG,kBACJ,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,UAAW,GAAI,GAAG,kBACL,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,WAAY,GAAI,GAAG,kBACN,IAArB5F,KAAKsa,iBAEhB1U,EAAe,EAAG,CAAC,YAAa,GAAI,GAAG,kBACP,IAArB5F,KAAKsa,iBAMhBpW,EAAa,cAAe,MAI5BY,EAAgB,cAAe,IAI/BmD,GAAc,IAAQT,GAAWR,GACjCiB,GAAc,KAAQT,GAAWP,GACjCgB,GAAc,MAAQT,GAAWN,GAG5BrB,GAAQ,OAAQA,GAAM7I,QAAU,EAAG6I,IAAS,IAC7CoC,GAAcpC,GAAO8B,aAGhB2Y,GAAQtkB,EAAOqK,GACpBA,EAvmGc,GAumGO1E,EAAuB,KAAhB,KAAO3F,QAGlC6J,GAAQ,IAAKA,GAAM7I,QAAU,EAAG6I,IAAS,IAC1CkD,GAAclD,GAAOya,QAIrBC,GAAoB3W,GAAW,gBAAgB,GAInDhE,EAAe,IAAM,EAAG,EAAG,YAC3BA,EAAe,KAAM,EAAG,EAAG,gBAYvB4a,GAAQvf,EAAO9E,mBAsFVskB,GAAoBxJ,UAClBA,EArFXuJ,GAAM3F,IAAoBA,GAC1B2F,GAAMpP,kBAjuBesP,EAAMC,OAGnB9L,EAAM6L,GAAQ/K,KACdiL,EAAM3D,GAAgBpI,EAAK7U,MAAM6gB,QAAQ,OACzCpjB,EAAS7B,EAAMklB,eAAe9gB,KAAM4gB,IAAQ,WAE5Cna,EAASka,IAAYjd,EAAWid,EAAQljB,IAAWkjB,EAAQljB,GAAQpB,KAAK2D,KAAM6U,GAAO8L,EAAQljB,WAE1FuC,KAAKvC,OAAOgJ,GAAUzG,KAAKkG,aAAakL,SAAS3T,EAAQuC,KAAM2V,GAAYd,MAytBtF2L,GAAMpD,wBArtBK,IAAInc,EAAOjB,OAstBtBwgB,GAAMrD,cAvpBSnhB,EAAOwI,EAAOuc,OACrBC,EACAC,EACAxa,MAECzG,KAAKjB,iBACCc,SAGXmhB,EAAO/D,GAAgBjhB,EAAOgE,OAEpBjB,iBACCc,WAGXohB,EAAoD,KAAvCD,EAAKtE,YAAc1c,KAAK0c,aAErClY,EAAQD,EAAeC,QAGd,OAAQiC,EAASoY,GAAU7e,KAAMghB,GAAQ,aACzC,QAASva,EAASoY,GAAU7e,KAAMghB,aAClC,UAAWva,EAASoY,GAAU7e,KAAMghB,GAAQ,YAC5C,SAAUva,GAAUzG,KAAOghB,GAAQ,cACnC,SAAUva,GAAUzG,KAAOghB,GAAQ,cACnC,OAAQva,GAAUzG,KAAOghB,GAAQ,eACjC,MAAOva,GAAUzG,KAAOghB,EAAOC,GAAa,gBAC5C,OAAQxa,GAAUzG,KAAOghB,EAAOC,GAAa,qBACzCxa,EAASzG,KAAOghB,SAGtBD,EAAUta,EAASnF,EAASmF,IAynBvC+Z,GAAMU,eA5ZU1c,OACRkc,UAEUjhB,KADd+E,EAAQD,EAAeC,KACc,gBAAVA,IAA4BxE,KAAKjB,iBACjDiB,SAGPmhB,EAAcnhB,KAAKa,OAAS2e,GAAiBD,UAEzC/a,OACC,OACDkc,EAAOS,EAAYnhB,KAAKqJ,OAAS,EAAG,EAAG,GAAK,YAE3C,UACDqX,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAUhK,KAAKgK,QAAU,EAAI,EAAG,GAAK,YAEzE,QACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAU,EAAG,GAAK,YAEtD,OACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAASjK,KAAKwN,UAAY,GAAK,YAEjF,UACDkT,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,QAAUjK,KAAKohB,aAAe,GAAK,GAAK,YAE1F,UACA,OACDV,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAAS,GAAK,YAEhE,OACDyW,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GA3GM,KA2GgBtB,GAAMsB,GAAQ1gB,KAAKa,OAAS,EA5G1C,IA4G8Cb,KAAK0c,aA3GrD,MA2GkG,YAEvG,SACDgE,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAhHQ,IAgHgBtB,GAAMsB,EAhHtB,KAgH6C,YAEpD,SACDA,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GArHQ,IAqHgBtB,GAAMsB,EArHtB,KAqH6C,cAIxDrhB,GAAGge,QAAQqD,GAChB9kB,EAAMuF,aAAanB,MAAM,GAClBA,MAgXXwgB,GAAM/iB,gBA/iBW4jB,GACRA,IACDA,EAAcrhB,KAAKyd,QAAU7hB,EAAMsjB,iBAAmBtjB,EAAMqjB,mBAE5DxY,EAASN,EAAanG,KAAMqhB,UACzBrhB,KAAKkG,aAAaob,WAAW7a,IA2iBxC+Z,GAAMngB,cAxiBSqgB,EAAMa,UACbvhB,KAAKjB,YACCqC,EAASsf,IAASA,EAAK3hB,WACxB4W,GAAY+K,GAAM3hB,WAChB6e,GAAe,CAACxd,GAAIJ,KAAMK,KAAMqgB,IAAOhjB,OAAOsC,KAAKtC,UAAU8jB,UAAUD,GAEvEvhB,KAAKkG,aAAaS,eAmiBjC6Z,GAAMiB,iBA/hBYF,UACPvhB,KAAKK,KAAKsV,KAAe4L,IA+hBpCf,GAAMpgB,YA5hBOsgB,EAAMa,UACXvhB,KAAKjB,YACCqC,EAASsf,IAASA,EAAK3hB,WACxB4W,GAAY+K,GAAM3hB,WAChB6e,GAAe,CAACvd,KAAML,KAAMI,GAAIsgB,IAAOhjB,OAAOsC,KAAKtC,UAAU8jB,UAAUD,GAEvEvhB,KAAKkG,aAAaS,eAuhBjC6Z,GAAMkB,eAnhBUH,UACLvhB,KAAKI,GAAGuV,KAAe4L,IAmhBlCf,GAAMzW,aAxiGcvF,UAEZd,EAAW1D,KADfwE,EAAQD,EAAeC,KAEZxE,KAAKwE,KAETxE,MAoiGXwgB,GAAMmB,4BAvUK7jB,EAAgBkC,MAAM5B,UAwUjCoiB,GAAMpC,iBA5tBYpiB,EAAOwI,OACjBod,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YACjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,UAAYqkB,EAAWrkB,UAE5BqkB,EAAWrkB,UAAYyC,KAAKod,QAAQyD,QAAQrc,GAAOjH,YAotBlEijB,GAAMvC,kBAhtBajiB,EAAOwI,OAClBod,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YACjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,UAAYqkB,EAAWrkB,UAE5ByC,KAAKod,QAAQ8D,MAAM1c,GAAOjH,UAAYqkB,EAAWrkB,YAwsBhEijB,GAAMqB,mBApsBcxhB,EAAMD,EAAIoE,EAAOsd,OAC7BC,EAAY3gB,EAASf,GAAQA,EAAOsV,GAAYtV,GAChD2hB,EAAU5gB,EAAShB,GAAMA,EAAKuV,GAAYvV,YACxCJ,KAAKjB,WAAagjB,EAAUhjB,WAAaijB,EAAQjjB,aAI5B,OAD3B+iB,EAAcA,GAAe,MACT,GAAa9hB,KAAKoe,QAAQ2D,EAAWvd,IAAUxE,KAAKie,SAAS8D,EAAWvd,MACpE,MAAnBsd,EAAY,GAAa9hB,KAAKie,SAAS+D,EAASxd,IAAUxE,KAAKoe,QAAQ4D,EAASxd,KA6rBzFgc,GAAMyB,gBA1rBWjmB,EAAOwI,OAEhB0d,EADAN,EAAaxgB,EAASpF,GAASA,EAAQ2Z,GAAY3Z,YAEjDgE,KAAKjB,YAAa6iB,EAAW7iB,aAIrB,iBADdyF,EAAQD,EAAeC,IAAU,eAEtBxE,KAAKzC,YAAcqkB,EAAWrkB,WAErC2kB,EAAUN,EAAWrkB,UACdyC,KAAKod,QAAQyD,QAAQrc,GAAOjH,WAAa2kB,GAAWA,GAAWliB,KAAKod,QAAQ8D,MAAM1c,GAAOjH,aAgrBxGijB,GAAM2B,uBA5qBkBnmB,EAAOwI,UACpBxE,KAAKiiB,OAAOjmB,EAAOwI,IAAUxE,KAAKoe,QAAQpiB,EAAOwI,IA4qB5Dgc,GAAM4B,wBAzqBmBpmB,EAAOwI,UACrBxE,KAAKiiB,OAAOjmB,EAAOwI,IAAUxE,KAAKie,SAASjiB,EAAOwI,IAyqB7Dgc,GAAMzhB,0BAtVKA,EAAQiB,OAuVnBwgB,GAAMrB,KAAoBA,GAC1BqB,GAAM9iB,OAAoBA,GAC1B8iB,GAAMta,WAAoBA,GAC1Bsa,GAAMlb,IAAoB0V,GAC1BwF,GAAMpe,IAAoB0Y,GAC1B0F,GAAM6B,+BAxVK/kB,EAAO,GAAIQ,EAAgBkC,QAyVtCwgB,GAAMxc,aA9iGcQ,EAAO1C,MACF,iBAAV0C,UAEH8d,WApTiBC,OACrB/d,EAAQ,OACP,IAAIge,KAAKD,EACV/d,EAAMvH,KAAK,CAACkH,KAAMqe,EAAGzd,SAAUF,EAAW2d,YAE9Che,EAAMyH,MAAK,SAAU9O,EAAGC,UACbD,EAAE4H,SAAW3H,EAAE2H,YAEnBP,EA4Seie,CADlBje,EAAQC,EAAqBD,IAEpB1H,EAAI,EAAGA,EAAIwlB,EAAYtlB,OAAQF,SAC/BwlB,EAAYxlB,GAAGqH,MAAMK,EAAM8d,EAAYxlB,GAAGqH,eAI/CT,EAAW1D,KADfwE,EAAQD,EAAeC,YAEZxE,KAAKwE,GAAO1C,UAGpB9B,MAkiGXwgB,GAAMK,iBAleYrc,OACVkc,UAEUjhB,KADd+E,EAAQD,EAAeC,KACc,gBAAVA,IAA4BxE,KAAKjB,iBACjDiB,SAGPmhB,EAAcnhB,KAAKa,OAAS2e,GAAiBD,UAEzC/a,OACC,OACDkc,EAAOS,EAAYnhB,KAAKqJ,OAAQ,EAAG,aAElC,UACDqX,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAUhK,KAAKgK,QAAU,EAAG,aAEhE,QACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAS,aAE7C,OACD0W,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,OAASjK,KAAKwN,qBAEhE,UACDkT,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,QAAUjK,KAAKohB,aAAe,cAEhF,UACA,OACDV,EAAOS,EAAYnhB,KAAKqJ,OAAQrJ,KAAKgK,QAAShK,KAAKiK,kBAElD,OACDyW,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,GAAQ1gB,KAAKa,OAAS,EA5D5B,IA4DgCb,KAAK0c,aA3DvC,gBA6DL,SACDgE,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,EAhEN,eAkEP,SACDA,EAAO1gB,KAAKX,GAAG9B,UACfmjB,GAAQtB,GAAMsB,EArEN,iBAyEXrhB,GAAGge,QAAQqD,GAChB9kB,EAAMuF,aAAanB,MAAM,GAClBA,MAsbXwgB,GAAM5B,SAAoBA,GAC1B4B,GAAMkC,uBAvXE3kB,EAAIiC,WACD,CAACjC,EAAEsL,OAAQtL,EAAEiM,QAASjM,EAAEkM,OAAQlM,EAAEub,OAAQvb,EAAEqc,SAAUrc,EAAEsc,SAAUtc,EAAEuc,gBAuX/EkG,GAAMmC,wBAnXE5kB,EAAIiC,WACD,CACHsb,MAAOvd,EAAEsL,OACTkB,OAAQxM,EAAEiM,QACVC,KAAMlM,EAAEkM,OACRmG,MAAOrS,EAAEqS,QACTE,QAASvS,EAAEuS,UACXG,QAAS1S,EAAE0S,UACXmL,aAAc7d,EAAE6d,iBA4WxB4E,GAAMoC,yBA7XK,IAAIlmB,KAAKsD,KAAKzC,YA8XzBijB,GAAMqC,qBArnBeC,OACZ9iB,KAAKjB,iBACC,SAEPlB,GAAqB,IAAfilB,EACN/kB,EAAIF,EAAMmC,KAAKod,QAAQvf,MAAQmC,YAC/BjC,EAAEsL,OAAS,GAAKtL,EAAEsL,OAAS,KACpBlD,EAAapI,EAAGF,EAAM,iCAAmC,gCAEhE6F,EAAWhH,KAAKP,UAAU0mB,aAEtBhlB,EACOmC,KAAK4iB,SAASC,cAEd,IAAInmB,KAAKsD,KAAKzC,UAA+B,GAAnByC,KAAK0c,YAAmB,KAAMmG,cAActc,QAAQ,IAAKJ,EAAapI,EAAG,MAG3GoI,EAAapI,EAAGF,EAAM,+BAAiC,+BAqmBlE2iB,GAAMuC,uBA3lBG/iB,KAAKjB,gBACC,qBAAuBiB,KAAKS,GAAK,WAExCwF,EAAO,SACP+c,EAAO,GACNhjB,KAAKijB,YACNhd,EAA4B,IAArBjG,KAAK0c,YAAoB,aAAe,mBAC/CsG,EAAO,SAEPE,EAAS,IAAMjd,EAAO,MACtBoD,EAAQ,GAAKrJ,KAAKqJ,QAAUrJ,KAAKqJ,QAAU,KAAQ,OAAS,SAE5D8Z,EAASH,EAAO,cAEbhjB,KAAKvC,OAAOylB,EAAS7Z,EAHb,wBAG+B8Z,IA8kBlD3C,GAAM4C,yBAzWKpjB,KAAKjB,UAAYiB,KAAK6iB,cAAgB,MA0WjDrC,GAAMpkB,2BA3nBK4D,KAAKod,QAAQ1f,OAAO,MAAMD,OAAO,qCA4nB5C+iB,GAAM6C,uBAtYK7hB,KAAKE,MAAM1B,KAAKzC,UAAY,MAuYvCijB,GAAMjjB,0BA3YKyC,KAAKX,GAAG9B,UAAmC,KAArByC,KAAKc,SAAW,IA4YjD0f,GAAM8C,8BA7VK,CACHtnB,MAAOgE,KAAKS,GACZhD,OAAQuC,KAAKU,GACbhD,OAAQsC,KAAKe,QACb0Z,MAAOza,KAAKa,OACZlD,OAAQqC,KAAKR,UAyVrBghB,GAAMnX,KAAaM,GACnB6W,GAAMlX,6BAtmGKA,GAAWtJ,KAAKqJ,SAumG3BmX,GAAMnL,kBApSmBrZ,UACd2jB,GAAqBtjB,KAAK2D,KACzBhE,EACAgE,KAAKuN,OACLvN,KAAKwN,UACLxN,KAAKkG,aAAa0P,MAAM1I,IACxBlN,KAAKkG,aAAa0P,MAAMzI,MA+RpCqT,GAAMT,qBA5RsB/jB,UACjB2jB,GAAqBtjB,KAAK2D,KACzBhE,EAAOgE,KAAK0b,UAAW1b,KAAKohB,aAAc,EAAG,IA2RzDZ,GAAMhF,QAAUgF,GAAMjF,kBAnOEvf,UACJ,MAATA,EAAgBwF,KAAKC,MAAMzB,KAAKgK,QAAU,GAAK,GAAKhK,KAAKgK,MAAoB,GAAbhO,EAAQ,GAASgE,KAAKgK,QAAU,IAmO3GwW,GAAMxW,MAAcyB,GACpB+U,GAAMtW,8BAx1FKA,GAAYlK,KAAKqJ,OAAQrJ,KAAKgK,UAy1FzCwW,GAAMjT,KAAiBiT,GAAM/E,eAznFRzf,OACbuR,EAAOvN,KAAKkG,aAAaqH,KAAKvN,aAClB,MAAThE,EAAgBuR,EAAOvN,KAAK6a,IAAqB,GAAhB7e,EAAQuR,GAAW,MAwnF/DiT,GAAM9E,QAAiB8E,GAAM+C,kBArnFLvnB,OAChBuR,EAAOK,GAAW5N,KAAM,EAAG,GAAGuN,YAClB,MAATvR,EAAgBuR,EAAOvN,KAAK6a,IAAqB,GAAhB7e,EAAQuR,GAAW,MAonF/DiT,GAAMzS,2BAxREyV,EAAWxjB,KAAKkG,aAAa0P,aAC1B7H,GAAY/N,KAAKqJ,OAAQma,EAAStW,IAAKsW,EAASrW,MAwR3DqT,GAAMiD,iCA7RK1V,GAAY/N,KAAKqJ,OAAQ,EAAG,IA8RvCmX,GAAMvW,KAAakW,GACnBK,GAAMpR,IAAaoR,GAAM7E,cA55EC3f,OACjBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,QAE9BuP,EAAMpP,KAAKa,OAASb,KAAKX,GAAGgO,YAAcrN,KAAKX,GAAG6W,gBACzC,MAATla,GACAA,WAjKcA,EAAO0B,SACJ,iBAAV1B,EACAA,EAGNoD,MAAMpD,GAKU,iBADrBA,EAAQ0B,EAAOiR,cAAc3S,IAElBA,EAGJ,KARIyN,SAASzN,EAAO,IA2Jf0nB,CAAa1nB,EAAOgE,KAAKkG,cAC1BlG,KAAK6a,IAAI7e,EAAQoT,EAAK,MAEtBA,GAo5EfoR,GAAMhT,iBAh5E0BxR,OACvBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,QAE9B2N,GAAWxN,KAAKoP,MAAQ,EAAIpP,KAAKkG,aAAa0P,MAAM1I,KAAO,SAC/C,MAATlR,EAAgBwR,EAAUxN,KAAK6a,IAAI7e,EAAQwR,EAAS,MA44E/DgT,GAAMY,oBAz4EuBplB,OACpBgE,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,OAOrB,MAAT7D,EAAe,KACXwR,WAzKaxR,EAAO0B,SACP,iBAAV1B,EACA0B,EAAOiR,cAAc3S,GAAS,GAAK,EAEvCoD,MAAMpD,GAAS,KAAOA,EAqKX2nB,CAAgB3nB,EAAOgE,KAAKkG,qBACnClG,KAAKoP,IAAIpP,KAAKoP,MAAQ,EAAI5B,EAAUA,EAAU,UAE9CxN,KAAKoP,OAAS,GA63E7BoR,GAAM7S,mBApLoB3R,OAClB2R,EAAYnM,KAAK+a,OAAOvc,KAAKod,QAAQyD,QAAQ,OAAS7gB,KAAKod,QAAQyD,QAAQ,SAAW,OAAS,SACnF,MAAT7kB,EAAgB2R,EAAY3N,KAAK6a,IAAK7e,EAAQ2R,EAAY,MAmLrE6S,GAAMlH,KAAOkH,GAAMpQ,MAAQc,GAC3BsP,GAAMpG,OAASoG,GAAMlQ,QAAU8P,GAC/BI,GAAMnG,OAASmG,GAAM/P,QAAU4P,GAC/BG,GAAMlG,YAAckG,GAAM5E,aAAe2E,GACzCC,GAAM9D,mBAxkCiB1gB,EAAO4nB,EAAeC,OAErCC,EADAtH,EAASxc,KAAKc,SAAW,MAExBd,KAAKjB,iBACU,MAAT/C,EAAgBgE,KAAOH,OAErB,MAAT7D,EAAe,IACM,iBAAVA,MAEO,QADdA,EAAQ4gB,GAAiB9U,GAAkB9L,WAEhCgE,UAEJwB,KAAKc,IAAItG,GAAS,KAAO6nB,IAChC7nB,GAAgB,WAEfgE,KAAKa,QAAU+iB,IAChBE,EAAcvG,GAAcvd,YAE3Bc,QAAU9E,OACV6E,QAAS,EACK,MAAfijB,QACKjJ,IAAIiJ,EAAa,KAEtBtH,IAAWxgB,KACN4nB,GAAiB5jB,KAAK+jB,kBACvBtF,GAAYze,KAAM4d,GAAe5hB,EAAQwgB,EAAQ,KAAM,GAAG,GAClDxc,KAAK+jB,yBACRA,mBAAoB,EACzBnoB,EAAMuF,aAAanB,MAAM,QACpB+jB,kBAAoB,OAG1B/jB,YAEAA,KAAKa,OAAS2b,EAASe,GAAcvd,OAuiCpDwgB,GAAM3iB,aArhCmB+lB,UACd5jB,KAAK0c,UAAU,EAAGkH,IAqhC7BpD,GAAMlD,eAlhCqBsG,UACnB5jB,KAAKa,cACA6b,UAAU,EAAGkH,QACb/iB,QAAS,EAEV+iB,QACKhF,SAASrB,GAAcvd,MAAO,MAGpCA,MA0gCXwgB,GAAMwD,wBAtgCe,MAAbhkB,KAAKY,UACA8b,UAAU1c,KAAKY,MAAM,GAAO,QAC9B,GAAuB,iBAAZZ,KAAKS,GAAiB,KAChCwjB,EAAQrH,GAAiB/U,GAAa7H,KAAKS,IAClC,MAATwjB,OACKvH,UAAUuH,QAGVvH,UAAU,GAAG,UAGnB1c,MA4/BXwgB,GAAM0D,8BAz/ByBloB,WACtBgE,KAAKjB,YAGV/C,EAAQA,EAAQ2Z,GAAY3Z,GAAO0gB,YAAc,GAEzC1c,KAAK0c,YAAc1gB,GAAS,IAAO,IAo/B/CwkB,GAAM2D,wBA/+BEnkB,KAAK0c,YAAc1c,KAAKod,QAAQpT,MAAM,GAAG0S,aACzC1c,KAAK0c,YAAc1c,KAAKod,QAAQpT,MAAM,GAAG0S,aA++BjD8D,GAAMyC,2BAr9BKjjB,KAAKjB,YAAaiB,KAAKa,QAs9BlC2f,GAAM4D,+BAl9BKpkB,KAAKjB,WAAYiB,KAAKa,QAm9BjC2f,GAAM/C,MAAuBA,GAC7B+C,GAAM/F,MAAuBgD,GAC7B+C,GAAM6D,2BA7EKrkB,KAAKa,OAAS,MAAQ,IA8EjC2f,GAAM8D,2BA1EKtkB,KAAKa,OAAS,6BAA+B,IA2ExD2f,GAAM+D,MAAS3hB,EAAU,kDAAmDud,IAC5EK,GAAMjW,OAAS3H,EAAU,mDAAoD6I,IAC7E+U,GAAMlF,MAAS1Y,EAAU,iDAAkD+G,IAC3E6W,GAAMwC,KAASpgB,EAAU,qHAjjCJ5G,EAAO4nB,UACX,MAAT5nB,GACqB,iBAAVA,IACPA,GAASA,QAGR0gB,UAAU1gB,EAAO4nB,GAEf5jB,OAECA,KAAK0c,eAwiCrB8D,GAAMgE,aAAe5hB,EAAU,0HAp/BtBrG,EAAYyD,KAAKykB,sBACXzkB,KAAKykB,kBAGZlQ,EAAI,MAERpU,EAAWoU,EAAGvU,OACduU,EAAImF,GAAcnF,IAEZpL,GAAI,KACF4R,EAAQxG,EAAE1T,OAASrD,EAAU+W,EAAEpL,IAAMwM,GAAYpB,EAAEpL,SAClDsb,cAAgBzkB,KAAKjB,WACtBiD,EAAcuS,EAAEpL,GAAI4R,EAAM2H,WAAa,YAEtC+B,eAAgB,SAGlBzkB,KAAKykB,qBAi/BZC,GAAU3gB,EAAO5H,mBAiCZwoB,GAAOlnB,EAAQmnB,EAAOC,EAAOC,OAC9BpnB,EAAS8V,KACT3V,EAAML,IAAYwG,IAAI8gB,EAAQF,UAC3BlnB,EAAOmnB,GAAOhnB,EAAKJ,YAGrBsnB,GAAgBtnB,EAAQmnB,EAAOC,MAChCroB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,GAEN,MAATmnB,SACOD,GAAMlnB,EAAQmnB,EAAOC,EAAO,aAGnC/nB,EACAkoB,EAAM,OACLloB,EAAI,EAAGA,EAAI,GAAIA,IAChBkoB,EAAIloB,GAAK6nB,GAAMlnB,EAAQX,EAAG+nB,EAAO,gBAE9BG,WAWFC,GAAkBC,EAAcznB,EAAQmnB,EAAOC,GACxB,kBAAjBK,GACH1oB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,KAGnBmnB,EADAnnB,EAASynB,EAETA,GAAe,EAEX1oB,EAASiB,KACTmnB,EAAQnnB,EACRA,OAASgC,GAGbhC,EAASA,GAAU,QAUnBX,EAPAY,EAAS8V,KACT2R,EAAQD,EAAexnB,EAAOkY,MAAM1I,IAAM,KAEjC,MAAT0X,SACOD,GAAMlnB,GAASmnB,EAAQO,GAAS,EAAGN,EAAO,WAIjDG,EAAM,OACLloB,EAAI,EAAGA,EAAI,EAAGA,IACfkoB,EAAIloB,GAAK6nB,GAAMlnB,GAASX,EAAIqoB,GAAS,EAAGN,EAAO,cAE5CG,EAlGXN,GAAQtT,kBAzgHWnO,EAAKuD,EAAKqO,OACrBpO,EAASzG,KAAKolB,UAAUniB,IAAQjD,KAAKolB,UAAL,gBAC7B1hB,EAAW+C,GAAUA,EAAOpK,KAAKmK,EAAKqO,GAAOpO,GAwgHxDie,GAAQ7d,wBA5/GiB5D,OACjBxF,EAASuC,KAAKqlB,gBAAgBpiB,GAC9BqiB,EAActlB,KAAKqlB,gBAAgBpiB,EAAIsiB,sBAEvC9nB,IAAW6nB,EACJ7nB,QAGN4nB,gBAAgBpiB,GAAOqiB,EAAY/e,QAAQ,oBAAoB,SAAUhG,UACnEA,EAAI2C,MAAM,MAGdlD,KAAKqlB,gBAAgBpiB,KAi/GhCyhB,GAAQ/d,8BA3+GG3G,KAAKwlB,cA4+GhBd,GAAQ3e,iBAt+GUxE,UACPvB,KAAKylB,SAASlf,QAAQ,KAAMhF,IAs+GvCmjB,GAAQ/K,SAAkB8G,GAC1BiE,GAAQpD,WAAkBb,GAC1BiE,GAAQxS,sBAp9Ge3Q,EAAQggB,EAAetK,EAAQyO,OAC9Cjf,EAASzG,KAAK2lB,cAAc1O,UACxBvT,EAAW+C,GACfA,EAAOlF,EAAQggB,EAAetK,EAAQyO,GACtCjf,EAAOF,QAAQ,MAAOhF,IAi9G9BmjB,GAAQkB,oBA98GazI,EAAM1W,OACnBhJ,EAASuC,KAAK2lB,cAAcxI,EAAO,EAAI,SAAW,eAC/CzZ,EAAWjG,GAAUA,EAAOgJ,GAAUhJ,EAAO8I,QAAQ,MAAOE,IA68GvEie,GAAQ1gB,aA7lHM9C,OACNZ,EAAMxD,MACLA,KAAKoE,EAEFwC,EADJpD,EAAOY,EAAOpE,SAELA,GAAKwD,OAEL,IAAMxD,GAAKwD,OAGnBqT,QAAUzS,OAIVgf,+BAAiC,IAAI5X,QACrCtI,KAAKggB,wBAAwB6F,QAAU7lB,KAAKigB,cAAc4F,QACvD,IAAO,UAAWA,SA+kH9BnB,GAAQna,gBAxhGexM,EAAGN,UACjBM,EAIEhC,EAAQiE,KAAKkc,SAAWlc,KAAKkc,QAAQne,EAAEiM,SAC1ChK,KAAKkc,SAASlc,KAAKkc,QAAQ4J,UAAYnb,IAAkB5D,KAAKtJ,GAAU,SAAW,cAAcM,EAAEiM,SAJ5FjO,EAAQiE,KAAKkc,SAAWlc,KAAKkc,QAChClc,KAAKkc,QAAL,YAshGZwI,GAAQpa,qBA/gGoBvM,EAAGN,UACtBM,EAIEhC,EAAQiE,KAAK+lB,cAAgB/lB,KAAK+lB,aAAahoB,EAAEiM,SACpDhK,KAAK+lB,aAAapb,GAAiB5D,KAAKtJ,GAAU,SAAW,cAAcM,EAAEiM,SAJtEjO,EAAQiE,KAAK+lB,cAAgB/lB,KAAK+lB,aACrC/lB,KAAK+lB,aAAL,YA6gGZrB,GAAQha,qBA99FoBM,EAAWvN,EAAQE,OACvCb,EAAG0J,EAAK0B,KAERlI,KAAKgmB,yBACEjb,GAAkB1O,KAAK2D,KAAMgL,EAAWvN,EAAQE,OAGtDqC,KAAKoL,oBACDA,aAAe,QACfC,iBAAmB,QACnBC,kBAAoB,IAMxBxO,EAAI,EAAGA,EAAI,GAAIA,IAAK,IAErB0J,EAAMhJ,EAAU,CAAC,IAAMV,IACnBa,IAAWqC,KAAKqL,iBAAiBvO,UAC5BuO,iBAAiBvO,GAAK,IAAIwL,OAAO,IAAMtI,KAAKuK,OAAO/D,EAAK,IAAID,QAAQ,IAAK,IAAM,IAAK,UACpF+E,kBAAkBxO,GAAK,IAAIwL,OAAO,IAAMtI,KAAKsK,YAAY9D,EAAK,IAAID,QAAQ,IAAK,IAAM,IAAK,MAE9F5I,GAAWqC,KAAKoL,aAAatO,KAC9BoL,EAAQ,IAAMlI,KAAKuK,OAAO/D,EAAK,IAAM,KAAOxG,KAAKsK,YAAY9D,EAAK,SAC7D4E,aAAatO,GAAK,IAAIwL,OAAOJ,EAAM3B,QAAQ,IAAK,IAAK,MAG1D5I,GAAqB,SAAXF,GAAqBuC,KAAKqL,iBAAiBvO,GAAGiK,KAAKiE,UACtDlO,EACJ,GAAIa,GAAqB,QAAXF,GAAoBuC,KAAKsL,kBAAkBxO,GAAGiK,KAAKiE,UAC7DlO,EACJ,IAAKa,GAAUqC,KAAKoL,aAAatO,GAAGiK,KAAKiE,UACrClO,IA87FnB4nB,GAAQja,qBA33FcrC,UACdpI,KAAKgmB,mBACA9oB,EAAW8C,KAAM,iBAClB4L,GAAmBvP,KAAK2D,MAExBoI,EACOpI,KAAKoM,mBAELpM,KAAKkM,eAGXhP,EAAW8C,KAAM,uBACbkM,aAAeP,IAEjB3L,KAAKoM,oBAAsBhE,EAC9BpI,KAAKoM,mBAAqBpM,KAAKkM,eA62F3CwY,GAAQla,0BAh5FmBpC,UACnBpI,KAAKgmB,mBACA9oB,EAAW8C,KAAM,iBAClB4L,GAAmBvP,KAAK2D,MAExBoI,EACOpI,KAAKqM,wBAELrM,KAAKmM,oBAGXjP,EAAW8C,KAAM,4BACbmM,kBAAoBT,IAEtB1L,KAAKqM,yBAA2BjE,EACnCpI,KAAKqM,wBAA0BrM,KAAKmM,oBAk4FhDuY,GAAQnX,cAxsFa/G,UACVoH,GAAWpH,EAAKxG,KAAK4V,MAAM1I,IAAKlN,KAAK4V,MAAMzI,KAAKI,MAwsF3DmX,GAAQuB,iCA3rFGjmB,KAAK4V,MAAMzI,KA4rFtBuX,GAAQwB,iCAhsFGlmB,KAAK4V,MAAM1I,KAksFtBwX,GAAQnW,kBAplFiBxQ,EAAGN,OACpB8Q,EAAWxS,EAAQiE,KAAKmmB,WAAanmB,KAAKmmB,UAC1CnmB,KAAKmmB,UAAWpoB,IAAW,IAANA,GAAciC,KAAKmmB,UAAUL,SAAS/e,KAAKtJ,GAAW,SAAW,qBAC5E,IAANM,EAAckQ,GAAcM,EAAUvO,KAAK4V,MAAM1I,KAClDnP,EAAKwQ,EAASxQ,EAAEqR,OAASb,GAilFpCmW,GAAQrW,qBAvkFoBtQ,UACV,IAANA,EAAckQ,GAAcjO,KAAKomB,aAAcpmB,KAAK4V,MAAM1I,KAC3DnP,EAAKiC,KAAKomB,aAAaroB,EAAEqR,OAASpP,KAAKomB,cAskFlD1B,GAAQpW,uBA9kFsBvQ,UACZ,IAANA,EAAckQ,GAAcjO,KAAKqmB,eAAgBrmB,KAAK4V,MAAM1I,KAC7DnP,EAAKiC,KAAKqmB,eAAetoB,EAAEqR,OAASpP,KAAKqmB,gBA6kFpD3B,GAAQ/V,uBApgFsBK,EAAavR,EAAQE,OAC3Cb,EAAG0J,EAAK0B,KAERlI,KAAKsmB,2BACEvX,GAAoB1S,KAAK2D,KAAMgP,EAAavR,EAAQE,OAG1DqC,KAAKiP,sBACDA,eAAiB,QACjBE,kBAAoB,QACpBD,oBAAsB,QACtBqX,mBAAqB,IAGzBzpB,EAAI,EAAGA,EAAI,EAAGA,IAAK,IAGpB0J,EAAMhJ,EAAU,CAAC,IAAM,IAAI4R,IAAItS,GAC3Ba,IAAWqC,KAAKumB,mBAAmBzpB,UAC9BypB,mBAAmBzpB,GAAK,IAAIwL,OAAO,IAAMtI,KAAKuO,SAAS/H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,UAC5F2I,oBAAoBpS,GAAK,IAAIwL,OAAO,IAAMtI,KAAKsO,cAAc9H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,UAClG4I,kBAAkBrS,GAAK,IAAIwL,OAAO,IAAMtI,KAAKqO,YAAY7H,EAAK,IAAID,QAAQ,IAAK,QAAU,IAAK,MAElGvG,KAAKiP,eAAenS,KACrBoL,EAAQ,IAAMlI,KAAKuO,SAAS/H,EAAK,IAAM,KAAOxG,KAAKsO,cAAc9H,EAAK,IAAM,KAAOxG,KAAKqO,YAAY7H,EAAK,SACpGyI,eAAenS,GAAK,IAAIwL,OAAOJ,EAAM3B,QAAQ,IAAK,IAAK,MAG5D5I,GAAqB,SAAXF,GAAqBuC,KAAKumB,mBAAmBzpB,GAAGiK,KAAKiI,UACxDlS,EACJ,GAAIa,GAAqB,QAAXF,GAAoBuC,KAAKkP,oBAAoBpS,GAAGiK,KAAKiI,UAC/DlS,EACJ,GAAIa,GAAqB,OAAXF,GAAmBuC,KAAKmP,kBAAkBrS,GAAGiK,KAAKiI,UAC5DlS,EACJ,IAAKa,GAAUqC,KAAKiP,eAAenS,GAAGiK,KAAKiI,UACvClS,IAm+EnB4nB,GAAQhW,uBAr7EgBtG,UAChBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKgQ,qBAELhQ,KAAK6P,iBAGX3S,EAAW8C,KAAM,yBACb6P,eAAiBR,IAEnBrP,KAAKgQ,sBAAwB5H,EAChCpI,KAAKgQ,qBAAuBhQ,KAAK6P,iBAu6E7C6U,GAAQjW,4BAl6EqBrG,UACrBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKiQ,0BAELjQ,KAAK8P,sBAGX5S,EAAW8C,KAAM,8BACb8P,oBAAsBR,IAExBtP,KAAKiQ,2BAA6B7H,EACrCpI,KAAKiQ,0BAA4BjQ,KAAK8P,sBAo5ElD4U,GAAQlW,0BA/4EmBpG,UACnBpI,KAAKsmB,qBACAppB,EAAW8C,KAAM,mBAClBwP,GAAqBnT,KAAK2D,MAE1BoI,EACOpI,KAAKkQ,wBAELlQ,KAAK+P,oBAGX7S,EAAW8C,KAAM,4BACb+P,kBAAoBR,IAEtBvP,KAAKkQ,yBAA2B9H,EACnCpI,KAAKkQ,wBAA0BlQ,KAAK+P,oBAk4EhD2U,GAAQ9T,cAtuEa5U,SAGgC,OAAxCA,EAAQ,IAAIsI,cAAckiB,OAAO,IAouE9C9B,GAAQ9lB,kBAhuEiBwR,EAAOE,EAASmW,UACjCrW,EAAQ,GACDqW,EAAU,KAAO,KAEjBA,EAAU,KAAO,MAwzEhCrT,GAAmB,KAAM,CACrBnB,uBAAwB,uBACxBlM,QAAU,SAAUxE,OACZnE,EAAImE,EAAS,UAKVA,GAJoC,IAA7BI,EAAMJ,EAAS,IAAM,IAAa,KACrC,IAANnE,EAAW,KACL,IAANA,EAAW,KACL,IAANA,EAAW,KAAO,SAO/BxB,EAAMujB,KAAOvc,EAAU,wDAAyDwQ,IAChFxX,EAAM8qB,SAAW9jB,EAAU,gEAAiE4Q,QAExFmT,GAAUnlB,KAAKc,aAmBVskB,GAAevL,EAAUrf,EAAO8F,EAAOwc,OACxCvD,EAAQ6C,GAAe5hB,EAAO8F,UAElCuZ,EAASW,eAAiBsC,EAAYvD,EAAMiB,cAC5CX,EAASY,OAAiBqC,EAAYvD,EAAMkB,MAC5CZ,EAASa,SAAiBoC,EAAYvD,EAAMmB,QAErCb,EAASe,mBAaXyK,GAAStlB,UACVA,EAAS,EACFC,KAAKE,MAAMH,GAEXC,KAAKC,KAAKF,YAmDhBulB,GAAcnL,UAGL,KAAPA,EAAc,gBAGhBoL,GAAcxc,UAEH,OAATA,EAAkB,cAkDpByc,GAAQC,UACN,kBACIjnB,KAAKknB,GAAGD,QAInBE,GAAiBH,GAAO,MACxBI,GAAiBJ,GAAO,KACxBK,GAAiBL,GAAO,KACxBM,GAAiBN,GAAO,KACxBO,GAAiBP,GAAO,KACxBQ,GAAiBR,GAAO,KACxBS,GAAiBT,GAAO,KACxBU,GAAiBV,GAAO,KACxBW,GAAiBX,GAAO,cAWnBY,GAAWnkB,UACT,kBACIzD,KAAKjB,UAAYiB,KAAKmc,MAAM1Y,GAAQ5D,SAI/C+b,GAAegM,GAAW,gBAC1BnX,GAAemX,GAAW,WAC1BtX,GAAesX,GAAW,WAC1BxX,GAAewX,GAAW,SAC1BjM,GAAeiM,GAAW,QAC1Brd,GAAeqd,GAAW,UAC1BtM,GAAesM,GAAW,SAM1BrL,GAAQ/a,KAAK+a,MACbsL,GAAa,CACbxV,GAAI,GACJxJ,EAAI,GACJ9K,EAAI,GACJyO,EAAI,GACJD,EAAI,GACJE,EAAI,aAICqb,GAAkB7Q,EAAQ1V,EAAQggB,EAAemE,EAAUhoB,UACzDA,EAAOwU,aAAa3Q,GAAU,IAAKggB,EAAetK,EAAQyO,YAG5DqC,GAAgBC,EAAgBzG,EAAe7jB,OAChD2d,EAAWuC,GAAeoK,GAAgB1lB,MAC1CmO,EAAW8L,GAAMlB,EAAS6L,GAAG,MAC7B5W,EAAWiM,GAAMlB,EAAS6L,GAAG,MAC7B9W,EAAWmM,GAAMlB,EAAS6L,GAAG,MAC7BvL,EAAWY,GAAMlB,EAAS6L,GAAG,MAC7B3c,EAAWgS,GAAMlB,EAAS6L,GAAG,MAC7B5L,EAAWiB,GAAMlB,EAAS6L,GAAG,MAE7B/pB,EAAIsT,GAAWoX,GAAWxV,IAAM,CAAC,IAAK5B,IAClCA,EAAUoX,GAAWhf,GAAO,CAAC,KAAM4H,IACnCH,GAAW,GAAiB,CAAC,MAC7BA,EAAUuX,GAAW9pB,GAAO,CAAC,KAAMuS,IACnCF,GAAW,GAAiB,CAAC,MAC7BA,EAAUyX,GAAWrb,GAAO,CAAC,KAAM4D,IACnCuL,GAAW,GAAiB,CAAC,MAC7BA,EAAUkM,GAAWtb,GAAO,CAAC,KAAMoP,IACnCpR,GAAW,GAAiB,CAAC,MAC7BA,EAAUsd,GAAWpb,GAAO,CAAC,KAAMlC,IACnC+Q,GAAW,GAAiB,CAAC,MAAkB,CAAC,KAAMA,UAE9Dne,EAAE,GAAKokB,EACPpkB,EAAE,IAAM6qB,EAAiB,EACzB7qB,EAAE,GAAKO,EACAoqB,GAAkBjsB,MAAM,KAAMsB,OA6CrC8qB,GAAQzmB,KAAKc,aAERqa,GAAKxS,UACDA,EAAI,IAAMA,EAAI,KAAQA,WAG1B+d,SAQAloB,KAAKjB,iBACCiB,KAAKkG,aAAaS,kBAMzB2J,EAASF,EAHTK,EAAUwX,GAAMjoB,KAAKgc,eAAiB,IACtCL,EAAesM,GAAMjoB,KAAKic,OAC1B1R,EAAe0d,GAAMjoB,KAAKkc,SAI9B5L,EAAoBhP,EAASmP,EAAU,IACvCL,EAAoB9O,EAASgP,EAAU,IACvCG,GAAW,GACXH,GAAW,OAQP6X,EALK7mB,EAASiJ,EAAS,IAMvBkC,EALJlC,GAAU,GAMN6d,EAAIzM,EACJnP,EAAI4D,EACJrS,EAAIuS,EACJzH,EAAI4H,EAAUA,EAAQ4X,QAAQ,GAAG9hB,QAAQ,SAAU,IAAM,GACzD+hB,EAAQtoB,KAAKonB,gBAEZkB,QAGM,UAGPC,EAAYD,EAAQ,EAAI,IAAM,GAC9BE,EAAS7L,GAAK3c,KAAKkc,WAAaS,GAAK2L,GAAS,IAAM,GACpDG,EAAW9L,GAAK3c,KAAKic,SAAWU,GAAK2L,GAAS,IAAM,GACpDI,EAAU/L,GAAK3c,KAAKgc,iBAAmBW,GAAK2L,GAAS,IAAM,UAExDC,EAAY,KACdJ,EAAIK,EAASL,EAAI,IAAM,KACvB1b,EAAI+b,EAAS/b,EAAI,IAAM,KACvB2b,EAAIK,EAAWL,EAAI,IAAM,KACxB5b,GAAKzO,GAAK8K,EAAK,IAAM,KACtB2D,EAAIkc,EAAUlc,EAAI,IAAM,KACxBzO,EAAI2qB,EAAU3qB,EAAI,IAAM,KACxB8K,EAAI6f,EAAU7f,EAAI,IAAM,QAG7B8f,GAAUvN,GAASjf,iBAEvBwsB,GAAQ5pB,0BA3tDGiB,KAAKhB,UA4tDhB2pB,GAAQrmB,mBAvVAiR,EAAiBvT,KAAKmc,kBAErBH,cAAgB2K,GAAQ3mB,KAAKgc,oBAC7BC,MAAgB0K,GAAQ3mB,KAAKic,YAC7BC,QAAgByK,GAAQ3mB,KAAKkc,SAElC3I,EAAKqI,aAAgB+K,GAAQpT,EAAKqI,cAClCrI,EAAK9C,QAAgBkW,GAAQpT,EAAK9C,SAClC8C,EAAKjD,QAAgBqW,GAAQpT,EAAKjD,SAClCiD,EAAKnD,MAAgBuW,GAAQpT,EAAKnD,OAClCmD,EAAKhJ,OAAgBoc,GAAQpT,EAAKhJ,QAClCgJ,EAAK+H,MAAgBqL,GAAQpT,EAAK+H,OAE3Btb,MA2UX2oB,GAAQ9N,aA7TQ7e,EAAO8F,UACZ8kB,GAAc5mB,KAAMhE,EAAO8F,EAAO,IA6T7C6mB,GAAQ/J,kBAzTa5iB,EAAO8F,UACjB8kB,GAAc5mB,KAAMhE,EAAO8F,GAAQ,IAyT9C6mB,GAAQzB,YApPK1iB,OACJxE,KAAKjB,iBACCc,QAEP8b,EACApR,EACAqR,EAAe5b,KAAKgc,iBAIV,WAFdxX,EAAQD,EAAeC,KAEY,YAAVA,GAAiC,SAAVA,SAC5CmX,EAAO3b,KAAKic,MAAQL,EAAe,MACnCrR,EAASvK,KAAKkc,QAAU4K,GAAanL,GAC7BnX,OACC,eAAkB+F,MAClB,iBAAkBA,EAAS,MAC3B,cAAkBA,EAAS,eAIpCoR,EAAO3b,KAAKic,MAAQza,KAAK+a,MAAMwK,GAAa/mB,KAAKkc,UACzC1X,OACC,cAAkBmX,EAAO,EAAQC,EAAe,WAChD,aAAkBD,EAAeC,EAAe,UAChD,cAAyB,GAAPD,EAAeC,EAAe,SAChD,gBAAyB,KAAPD,EAAeC,EAAe,QAChD,gBAAyB,MAAPD,EAAeC,EAAe,QAEhD,qBAAsBpa,KAAKE,MAAa,MAAPia,GAAgBC,gBACvC,IAAIxY,MAAM,gBAAkBoB,KAwNvDmkB,GAAQxB,eAAiBA,GACzBwB,GAAQvB,UAAiBA,GACzBuB,GAAQtB,UAAiBA,GACzBsB,GAAQrB,QAAiBA,GACzBqB,GAAQpB,OAAiBA,GACzBoB,GAAQnB,QAAiBA,GACzBmB,GAAQlB,SAAiBA,GACzBkB,GAAQjB,WAAiBA,GACzBiB,GAAQhB,QAAiBA,GACzBgB,GAAQprB,0BA1NCyC,KAAKjB,UAINiB,KAAKgc,cACQ,MAAbhc,KAAKic,MACJjc,KAAKkc,QAAU,GAAM,OACK,QAA3Bva,EAAM3B,KAAKkc,QAAU,IANdrc,KA0Nf8oB,GAAQvM,uBApTA3L,EAASH,EAASF,EAAOkL,EAAOsN,EAJhChN,EAAe5b,KAAKgc,cACpBL,EAAe3b,KAAKic,MACpB1R,EAAevK,KAAKkc,QACpB3I,EAAevT,KAAKmc,aAKjBP,GAAgB,GAAKD,GAAQ,GAAKpR,GAAU,GAC1CqR,GAAgB,GAAKD,GAAQ,GAAKpR,GAAU,IACjDqR,GAAuD,MAAvCiL,GAAQE,GAAaxc,GAAUoR,GAC/CA,EAAO,EACPpR,EAAS,GAKbgJ,EAAKqI,aAAeA,EAAe,IAEnCnL,EAAoBnP,EAASsa,EAAe,KAC5CrI,EAAK9C,QAAeA,EAAU,GAE9BH,EAAoBhP,EAASmP,EAAU,IACvC8C,EAAKjD,QAAeA,EAAU,GAE9BF,EAAoB9O,EAASgP,EAAU,IACvCiD,EAAKnD,MAAeA,EAAQ,GAE5BuL,GAAQra,EAAS8O,EAAQ,IAGzBwY,EAAiBtnB,EAASwlB,GAAanL,IACvCpR,GAAUqe,EACVjN,GAAQkL,GAAQE,GAAa6B,IAG7BtN,EAAQha,EAASiJ,EAAS,IAC1BA,GAAU,GAEVgJ,EAAKoI,KAASA,EACdpI,EAAKhJ,OAASA,EACdgJ,EAAK+H,MAASA,EAEPtb,MA8QX2oB,GAAQvL,wBAhMGQ,GAAe5d,OAiM1B2oB,GAAQ5e,aA9LQvF,UACZA,EAAQD,EAAeC,GAChBxE,KAAKjB,UAAYiB,KAAKwE,EAAQ,OAAS3E,KA6LlD8oB,GAAQ/M,aAAiBA,GACzB+M,GAAQlY,QAAiBA,GACzBkY,GAAQrY,QAAiBA,GACzBqY,GAAQvY,MAAiBA,GACzBuY,GAAQhN,KAAiBA,GACzBgN,GAAQlN,wBAhLGna,EAAStB,KAAK2b,OAAS,IAiLlCgN,GAAQpe,OAAiBA,GACzBoe,GAAQrN,MAAiBA,GACzBqN,GAAQnH,kBA3GWqH,OACV7oB,KAAKjB,iBACCiB,KAAKkG,aAAaS,kBAGzBjJ,EAASsC,KAAKkG,aACdO,EAASshB,GAAe/nB,MAAO6oB,EAAYnrB,UAE3CmrB,IACApiB,EAAS/I,EAAOkoB,YAAY5lB,KAAMyG,IAG/B/I,EAAO4jB,WAAW7a,IAgG7BkiB,GAAQ9F,YAAiBqF,GACzBS,GAAQvsB,SAAiB8rB,GACzBS,GAAQvF,OAAiB8E,GACzBS,GAAQjrB,OAAiBA,GACzBirB,GAAQziB,WAAiBA,GAEzByiB,GAAQG,YAAclmB,EAAU,sFAAuFslB,IACvHS,GAAQxJ,KAAOA,GAMfvZ,EAAe,IAAK,EAAG,EAAG,QAC1BA,EAAe,IAAK,EAAG,EAAG,WAI1BqC,GAAc,IAAKL,IACnBK,GAAc,IAlzHO,wBAmzHrBc,GAAc,KAAK,SAAU/M,EAAOqK,EAAOnF,GACvCA,EAAO7B,GAAK,IAAI3C,KAA6B,IAAxBof,WAAW9f,EAAO,QAE3C+M,GAAc,KAAK,SAAU/M,EAAOqK,EAAOnF,GACvCA,EAAO7B,GAAK,IAAI3C,KAAKiF,EAAM3F,OAM/BJ,EAAMmtB,QAAU,SAz7IZrtB,EA27IYia,GAEhB/Z,EAAMiB,GAAwB2jB,GAC9B5kB,EAAMwG,mBAj0DEY,EAAO,GAAGE,MAAM7G,KAAKP,UAAW,UAE7Bmf,GAAO,WAAYjY,IAg0D9BpH,EAAM0J,mBA5zDEtC,EAAO,GAAGE,MAAM7G,KAAKP,UAAW,UAE7Bmf,GAAO,UAAWjY,IA2zD7BpH,EAAMiZ,IAxzDI,kBACCnY,KAAKmY,IAAMnY,KAAKmY,OAAU,IAAInY,MAwzDzCd,EAAMiC,IAAwBL,EAC9B5B,EAAMynB,cAnjBernB,UACV2Z,GAAoB,IAAR3Z,IAmjBvBJ,EAAM2O,gBAjce9M,EAAQmnB,UAClBG,GAAetnB,EAAQmnB,EAAO,WAiczChpB,EAAMa,OAAwBA,EAC9Bb,EAAM8B,OAAwB0V,GAC9BxX,EAAM+iB,QAAwB/e,EAC9BhE,EAAMyf,SAAwBuC,GAC9BhiB,EAAMwF,SAAwBA,EAC9BxF,EAAM2S,kBA/biB2W,EAAcznB,EAAQmnB,UAClCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,aA+bzDhpB,EAAMooB,4BAtjBKrO,GAAY9Z,MAAM,KAAMC,WAAWkoB,aAujB9CpoB,EAAMsK,WAAwBsN,GAC9B5X,EAAMygB,WAAwBA,GAC9BzgB,EAAM0O,qBAvcoB7M,EAAQmnB,UACvBG,GAAetnB,EAAQmnB,EAAO,gBAuczChpB,EAAMyS,qBA5boB6W,EAAcznB,EAAQmnB,UACrCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,gBA4bzDhpB,EAAM6X,aAAwBA,GAC9B7X,EAAMotB,sBA9kFgBvlB,EAAMvC,MACV,MAAVA,EAAgB,KACZxD,EAAQurB,EAAWplB,EAAesN,GAGrB,OADjB8X,EAAYlW,GAAWtP,MAEnBI,EAAeolB,EAAUtV,UAG7BjW,EAAS,IAAIqG,EADb7C,EAAS0C,EAAaC,EAAc3C,KAE7B0S,aAAehB,GAAQnP,GAC9BmP,GAAQnP,GAAQ/F,EAGhB0V,GAAmB3P,QAGE,MAAjBmP,GAAQnP,KAC0B,MAA9BmP,GAAQnP,GAAMmQ,aACdhB,GAAQnP,GAAQmP,GAAQnP,GAAMmQ,aACN,MAAjBhB,GAAQnP,WACRmP,GAAQnP,WAIpBmP,GAAQnP,IAsjFnB7H,EAAMgX,0BA1hFKtP,EAAKsP,KA2hFhBhX,EAAM0S,uBApcsB4W,EAAcznB,EAAQmnB,UACvCK,GAAiBC,EAAcznB,EAAQmnB,EAAO,kBAoczDhpB,EAAM2I,eAAwBA,EAC9B3I,EAAMstB,8BA/L+BC,eACR1pB,IAArB0pB,EACO5M,GAEsB,mBAAtB4M,IACP5M,GAAQ4M,GACD,IA0LfvtB,EAAMwtB,+BApLgCC,EAAWC,eACf7pB,IAA1BooB,GAAWwB,UAGD5pB,IAAV6pB,EACOzB,GAAWwB,IAEtBxB,GAAWwB,GAAaC,EACN,MAAdD,IACAxB,GAAWxV,GAAKiX,EAAQ,IAErB,KA0KX1tB,EAAMklB,wBA73CqByI,EAAU1U,OAC7BsI,EAAOoM,EAASpM,KAAKtI,EAAK,QAAQ,UAC/BsI,GAAQ,EAAI,WACXA,GAAQ,EAAI,WACZA,EAAO,EAAI,UACXA,EAAO,EAAI,UACXA,EAAO,EAAI,UACXA,EAAO,EAAI,WAAa,YAu3CpCvhB,EAAMO,UAAwBqkB,GAG9B5kB,EAAM4tB,UAAY,CACdC,eAAgB,mBAChBC,uBAAwB,sBACxBC,kBAAmB,0BACnBC,KAAM,aACNC,KAAM,QACNC,aAAc,WACdC,QAAS,eACTC,KAAM,aACNC,MAAO,WAGJruB,EAp/IyEsuB,MCHhFC,qBCOK,SAAUC,YAGNC,EAAoB9oB,EAAQggB,EAAete,EAAKyiB,OACjDjoB,EAAS,GACJ,CAAC,cAAe,kBAChB,CAAC,cAAe,kBAChB,CAAC,UAAW,gBACX,CAAC8D,EAAS,QAASA,EAAS,YAC7B,CAAC,YAAa,kBACb,CAACA,EAAS,UAAWA,EAAS,cAC/B,CAAC,WAAY,iBACZ,CAACA,EAAS,SAAUA,EAAS,mBAEhCggB,EAAgB9jB,EAAOwF,GAAK,GAAKxF,EAAOwF,GAAK,GAG/CmnB,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,qFAAqFM,MAAM,KACpGP,YAAc,6DAA6DO,MAAM,KACjFyf,kBAAmB,EACnB/b,SAAW,8DAA8D1D,MAAM,KAC/EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAI,QACJD,IAAK,WACLE,EAAI,aACJC,GAAK,eACLC,IAAM,qBACNC,KAAO,4BAEXZ,SAAW,CACPC,QAAS,sBACTK,SAAU,IACVJ,QAAS,uBACTC,SAAU,qBACVC,QAAS,wBACTC,SAAU,gCAEdS,aAAe,CACXC,OAAS,QACTC,KAAO,SACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAIssB,EACJ/X,GAAK,aACL9F,EAAI6d,EACJ9X,GAAK,aACLhG,EAAI8d,EACJ7X,GAAK6X,EACL5d,EAAI4d,EACJ5X,GAAK4X,EACL9gB,EAAI8gB,EACJ3X,GAAK2X,GAETpY,uBAAwB,YACxBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAhEwB+c,CAAQM,wBCGzC,SAAUJ,OAGXK,EAAiB,8DAA8D5f,MAAM,KACrFP,EAAc,kDAAkDO,MAAM,KAEtEH,EAAc,CAAC,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,SAClHD,EAAc,mLAET2f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,2FAA2FM,MAAM,KAC1GP,YAAc,SAAUvM,EAAGN,UAClBM,EAEM,QAAQgJ,KAAKtJ,GACb6M,EAAYvM,EAAEiM,SAEdygB,EAAe1sB,EAAEiM,SAJjBygB,GAOfhgB,YAAcA,EACdD,iBAAmBC,EACnBigB,kBAAoB,+FACpBC,uBAAyB,0FACzBjgB,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EACnB6D,SAAW,uDAAuD1D,MAAM,KACxEyD,cAAgB,qCAAqCzD,MAAM,KAC3DwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,OACLD,IAAM,UACNE,EAAI,aACJC,GAAK,wBACLC,IAAM,6BACNC,KAAO,oCAEXZ,SAAW,CACPC,QAAU,iBACC,aAAiC,IAAjBrR,KAAKoQ,QAAiB,IAAM,IAAM,QAE7DkB,QAAU,iBACC,gBAAoC,IAAjBtR,KAAKoQ,QAAiB,IAAM,IAAM,QAEhEmB,SAAW,iBACA,cAAkC,IAAjBvR,KAAKoQ,QAAiB,IAAM,IAAM,QAE9DoB,QAAU,iBACC,cAAkC,IAAjBxR,KAAKoQ,QAAiB,IAAM,IAAM,QAE9DqB,SAAW,iBACA,0BAA8C,IAAjBzR,KAAKoQ,QAAiB,IAAM,IAAM,QAE1EsB,SAAW,KAEfQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,gBACJwJ,GAAK,cACLtU,EAAI,YACJuU,GAAK,aACL9F,EAAI,WACJ+F,GAAK,WACLhG,EAAI,SACJiG,GAAK,UACL/F,EAAI,SACJgG,GAAK,WACLlJ,EAAI,SACJmJ,GAAK,WAETT,uBAAyB,WACzBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAhFwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,uFAAuFM,MAAM,KACtGP,YAAc,iEAAiEO,MAAM,KACrFyf,kBAAmB,EACnB/b,SAAW,sDAAsD1D,MAAM,KACvEyD,cAAgB,qCAAqCzD,MAAM,KAC3DwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAU,qBACVC,QAAU,gBACVC,SAAW,cACXC,QAAU,cACVC,SAAW,sBACXC,SAAW,KAEfQ,aAAe,CACXC,OAAS,UACTC,KAAO,YACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAI,aACJuU,GAAK,aACL9F,EAAI,YACJ+F,GAAK,YACLhG,EAAI,UACJiG,GAAK,WACL/F,EAAI,UACJgG,GAAK,UACLlJ,EAAI,QACJmJ,GAAK,UAETT,uBAAwB,eACxBlM,QAAU,SAAUxE,EAAQgd,UAChBA,OAIC,WACMhd,GAAqB,IAAXA,EAAe,KAAO,gBAItC,QACA,QACA,UACA,WACMA,GAAqB,IAAXA,EAAe,KAAO,SAGtC,QACA,WACMA,GAAqB,IAAXA,EAAe,KAAO,OAGnDgM,KAAO,CACHL,IAAM,EACNC,IAAM,MAvEwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,gGAAgGM,MAAM,KAC/GP,YAAc,kDAAkDO,MAAM,KACtE0D,SAAW,2DAA2D1D,MAAM,KAC5EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAS,iBACTC,QAAS,mBACTC,SAAU,iBACVC,QAAS,iBACTC,SAAU,kBACEzR,KAAKoP,YACJ,QACM,2CAEA,+BAGnBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,SAAUtJ,UACN,YAAa9B,KAAK8B,GAAK,MAAQ,MAAQ,IAAMA,GAE1DuJ,KAAO,QACPvJ,EAAI,iBACJwJ,GAAK,aACLtU,EAAI,YACJuU,GAAK,YACL9F,EAAI,SACJ+F,GAAK,SACLhG,EAAI,YACJiG,GAAK,YACL/F,EAAI,UACJgG,GAAK,UACLlJ,EAAI,UACJmJ,GAAK,WAETT,uBAAyB,WACzBlM,QAAS,MACTwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAzDwB+c,CAAQM,wBCGzC,SAAUJ,GAGNA,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,wCAAwCM,MAAM,KACvDP,YAAc,yCAAyCO,MAAM,KAC7D0D,SAAW,8BAA8B1D,MAAM,KAC/CyD,cAAgB,gBAAgBzD,MAAM,KACtCwD,YAAc,gBAAgBxD,MAAM,KACpChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,YACLC,IAAM,kBACNC,KAAO,uBACP4E,EAAI,aACJkU,GAAK,YACLC,IAAM,kBACNC,KAAO,wBAEXrY,cAAe,SACf/B,KAAO,SAAU5U,SACI,OAAVA,GAEX4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,GACA,KAEA,MAGflI,SAAW,CACPC,QAAU,UACVC,QAAU,UACVC,SAAW,SAAUsD,UACbA,EAAItH,OAASvN,KAAKuN,OACX,cAEA,WAGfiE,QAAU,UACVC,SAAW,SAAUoD,UACb7U,KAAKuN,OAASsH,EAAItH,OACX,cAEA,WAGfmE,SAAW,KAEfO,uBAAyB,WACzBlM,QAAU,SAAUxE,EAAQgd,UAChBA,OACC,QACA,QACA,aACMhd,EAAS,mBAETA,IAGnB2Q,aAAe,CACXC,OAAS,MACTC,KAAO,MACPvJ,EAAI,KACJwJ,GAAK,MACLtU,EAAI,KACJuU,GAAK,MACL9F,EAAI,MACJ+F,GAAK,OACLhG,EAAI,KACJiG,GAAK,MACL/F,EAAI,MACJgG,GAAK,OACLlJ,EAAI,KACJmJ,GAAK,UAhFyBwX,CAAQM,wBCGzC,SAAUJ,OAGXa,EAAsB,6DAA6DpgB,MAAM,KACzFqgB,EAAyB,kDAAkDrgB,MAAM,KAEjFH,EAAc,CAAC,QAAS,QAAS,iBAAkB,QAAS,SAAU,cAAe,cAAe,QAAS,QAAS,QAAS,QAAS,SACxID,EAAc,qKAET2f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,0FAA0FM,MAAM,KACzGP,YAAc,SAAUvM,EAAGN,UAClBM,EAEM,QAAQgJ,KAAKtJ,GACbytB,EAAuBntB,EAAEiM,SAEzBihB,EAAoBltB,EAAEiM,SAJtBihB,GAQfxgB,YAAaA,EACbD,iBAAkBC,EAClBigB,kBAAmB,4FACnBC,uBAAwB,mFAExBjgB,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EAEnB6D,SAAW,6DAA6D1D,MAAM,KAC9EyD,cAAgB,8BAA8BzD,MAAM,KACpDwD,YAAc,uBAAuBxD,MAAM,KAC3C0f,oBAAqB,EACrB1jB,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,0BAEXZ,SAAW,CACPC,QAAS,kBACTC,QAAS,iBACTC,SAAU,eACVC,QAAS,mBACTC,SAAU,2BACVC,SAAU,KAEdQ,aAAe,CACXC,OAAS,UACTC,KAAO,aACPvJ,EAAI,oBACJwJ,GAAK,cACLtU,EAAI,aACJuU,GAAK,aACL9F,EAAI,UACJ+F,GAAK,SACLhG,EAAI,UACJiG,GAAK,WACL/F,EAAI,YACJgG,GAAK,aACLlJ,EAAI,WACJmJ,GAAK,WAETT,uBAAwB,kBACxBlM,QAAU,SAAUxE,UACTA,GAAsB,IAAXA,GAA2B,IAAXA,GAAgBA,GAAU,GAAM,MAAQ,OAE9EgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA3EwB+c,CAAQM,wBCGzC,SAAUJ,OAGXe,EAAmB,mGAAmGtgB,MAAM,KAC5HugB,EAAmB,qGAAqGvgB,MAAM,cACzHwgB,EAAOld,UACJA,EAAI,GAAK,GAAOA,EAAI,GAAK,MAAWA,EAAI,IAAM,IAAQ,WAEzDmd,EAAU/pB,EAAQggB,EAAete,OAClCsV,EAAShX,EAAS,WACd0B,OACC,YACMsV,GAAU8S,EAAO9pB,GAAU,UAAY,cAC7C,WACMggB,EAAgB,SAAW,aACjC,YACMhJ,GAAU8S,EAAO9pB,GAAU,SAAW,aAC5C,WACMggB,EAAiB,UAAa,cACpC,YACMhJ,GAAU8S,EAAO9pB,GAAU,UAAY,cAC7C,YACMgX,GAAU8S,EAAO9pB,GAAU,WAAa,gBAC9C,YACMgX,GAAU8S,EAAO9pB,GAAU,OAAS,QAI9C6oB,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,SAAUghB,EAAgB9tB,UAC1B8tB,EAEiB,KAAX9tB,EAIA,IAAM2tB,EAAiBG,EAAevhB,SAAW,IAAMmhB,EAAiBI,EAAevhB,SAAW,IAClG,SAASjD,KAAKtJ,GACd2tB,EAAiBG,EAAevhB,SAEhCmhB,EAAiBI,EAAevhB,SAThCmhB,GAYf7gB,YAAc,kDAAkDO,MAAM,KACtE0D,SAAW,6DAA6D1D,MAAM,KAC9EyD,cAAgB,2BAA2BzD,MAAM,KACjDwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,cACLC,IAAM,oBACNC,KAAO,2BAEXZ,SAAW,CACPC,QAAS,cACTC,QAAS,eACTC,SAAU,kBACEvR,KAAKoP,YACJ,QACM,0BAEN,QACM,wBAEN,QACM,sBAEN,QACM,gCAGA,oBAGnBoC,QAAS,iBACTC,SAAU,kBACEzR,KAAKoP,YACJ,QACM,iCACN,QACM,6BACN,QACM,uCAEA,2BAGnBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,eACJwJ,GAAKiZ,EACLvtB,EAAIutB,EACJhZ,GAAKgZ,EACL9e,EAAI8e,EACJ/Y,GAAK+Y,EACL/e,EAAI,UACJiG,GAAK,SACL/F,EAAI,UACJgG,GAAK6Y,EACL/hB,EAAI,MACJmJ,GAAK4Y,GAETrZ,uBAAwB,YACxBlM,QAAU,MACVwH,KAAO,CACHL,IAAM,EACNC,IAAM,MAlHwB+c,CAAQM,wBCGzC,SAAUJ,YAONoB,EAAuBjqB,EAAQggB,EAAete,OAJjCwoB,EACdC,QAYQ,MAARzoB,EACOse,EAAgB,SAAW,SAG3BhgB,EAAS,KAjBFkqB,GAiB6BlqB,EAhB3CmqB,EAIS,IACHnK,EAAgB,yBAA2B,4BAC3CA,EAAgB,sBAAwB,yBACxC,oBACA,mBACA,0BACA,gBAM8Bte,GAhBvB4H,MAAM,KAChB4gB,EAAM,IAAO,GAAKA,EAAM,KAAQ,GAAKC,EAAM,GAAMD,EAAM,IAAM,GAAKA,EAAM,IAAM,IAAMA,EAAM,IAAM,IAAMA,EAAM,KAAO,IAAMC,EAAM,GAAKA,EAAM,QAkBjJhhB,EAAc,CAAC,QAAS,QAAS,QAAS,QAAS,WAAY,QAAS,QAAS,QAAS,QAAS,QAAS,QAAS,SAKhH0f,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,CACL9M,OAAQ,oFAAoFoN,MAAM,KAClG8gB,WAAY,kFAAkF9gB,MAAM,MAExGP,YAAc,CAEV7M,OAAQ,gEAAgEoN,MAAM,KAC9E8gB,WAAY,gEAAgE9gB,MAAM,MAEtF0D,SAAW,CACPod,WAAY,gEAAgE9gB,MAAM,KAClFpN,OAAQ,gEAAgEoN,MAAM,KAC9Eib,SAAU,kDAEdxX,cAAgB,uBAAuBzD,MAAM,KAC7CwD,YAAc,uBAAuBxD,MAAM,KAC3CH,YAAcA,EACdkgB,gBAAkBlgB,EAClBmgB,iBAAmBngB,EAGnBD,YAAa,2MAGbD,iBAAkB,2MAGlBkgB,kBAAmB,wHAGnBC,uBAAwB,6FACxB9jB,eAAiB,CACb+K,GAAK,OACLD,IAAM,UACNE,EAAI,aACJC,GAAK,iBACLC,IAAM,uBACNC,KAAO,8BAEXZ,SAAW,CACPC,QAAS,kBACTC,QAAS,iBACTE,QAAS,gBACTD,SAAU,SAAUsD,MACZA,EAAItH,SAAWvN,KAAKuN,cAcD,IAAfvN,KAAKoP,MACE,oBAEA,0BAhBHpP,KAAKoP,YACJ,QACM,kCACN,OACA,OACA,QACM,kCACN,OACA,OACA,QACM,+BAUvBqC,SAAU,SAAUoD,MACZA,EAAItH,SAAWvN,KAAKuN,cAcD,IAAfvN,KAAKoP,MACE,oBAEA,0BAhBHpP,KAAKoP,YACJ,QACM,gCACN,OACA,OACA,QACM,gCACN,OACA,OACA,QACM,6BAUvBsC,SAAU,KAEdQ,aAAe,CACXC,OAAS,WACTC,KAAO,WACPvJ,EAAI,mBACJwJ,GAAKmZ,EACLztB,EAAIytB,EACJlZ,GAAKkZ,EACLhf,EAAI,MACJ+F,GAAKiZ,EACLjf,EAAI,OACJiG,GAAKgZ,EACL/e,EAAI,QACJgG,GAAK+Y,EACLjiB,EAAI,MACJmJ,GAAK8Y,GAET7Y,cAAe,wBACf/B,KAAO,SAAU5U,SACN,iBAAiB+K,KAAK/K,IAEjC4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,EACA,OACAA,EAAO,GACP,OACAA,EAAO,GACP,MAEA,UAGfrH,uBAAwB,mBACxBlM,QAAS,SAAUxE,EAAQgd,UACfA,OACC,QACA,QACA,aACMhd,EAAS,SACf,WACMA,EAAS,UACf,QACA,WACMA,EAAS,oBAETA,IAGnBgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA1KwB+c,CAAQM,wBCGzC,SAAUJ,YAONoB,EAAuBjqB,EAAQggB,EAAete,OAJjCwoB,EACdC,QAYQ,MAARzoB,EACOse,EAAgB,UAAY,UAEtB,MAARte,EACEse,EAAgB,SAAW,SAG3BhgB,EAAS,KApBFkqB,GAoB6BlqB,EAnB3CmqB,EAIS,IACHnK,EAAgB,yBAA2B,4BAC3CA,EAAgB,yBAA2B,4BAC3CA,EAAgB,sBAAwB,yBACxC,mBACA,2BACA,kBAS8Bte,GAnBvB4H,MAAM,KAChB4gB,EAAM,IAAO,GAAKA,EAAM,KAAQ,GAAKC,EAAM,GAAMD,EAAM,IAAM,GAAKA,EAAM,IAAM,IAAMA,EAAM,IAAM,IAAMA,EAAM,KAAO,IAAMC,EAAM,GAAKA,EAAM,aA0C5IE,EAAqBC,UACnB,kBACIA,EAAM,KAAwB,KAAjB7rB,KAAKoQ,QAAiB,IAAM,IAAM,QAIrDga,EAAO3W,aAAa,KAAM,CAC/BlJ,OAAS,QACK,yFAAyFM,MAAM,gBAC3F,iGAAiGA,MAAM,MAEzHP,YAAc,yDAAyDO,MAAM,KAC7E0D,kBAjCyBxQ,EAAGN,OACxB8Q,EAAW,YACG,0DAA0D1D,MAAM,gBAChE,0DAA0DA,MAAM,cAClE,4DAA4DA,MAAM,aAGxE,IAAN9M,EACOwQ,EAAQ,WAAerL,MAAM,EAAG,GAAGkL,OAAOG,EAAQ,WAAerL,MAAM,EAAG,IAEhFnF,EASEwQ,EALS,qBAAsBxH,KAAKtJ,GACvC,aACE,sCAAuCsJ,KAAKtJ,GAC1C,WACA,cACkBM,EAAEqR,OARjBb,EAAQ,YAuBnBD,cAAgB,uBAAuBzD,MAAM,KAC7CwD,YAAc,uBAAuBxD,MAAM,KAC3ChE,eAAiB,CACb+K,GAAK,QACLD,IAAM,WACNE,EAAI,aACJC,GAAK,iBACLC,IAAM,wBACNC,KAAO,+BAEXZ,SAAW,CACPC,QAASua,EAAqB,cAC9Bta,QAASsa,EAAqB,YAC9Bpa,QAASoa,EAAqB,WAC9Bra,SAAUqa,EAAqB,cAC/Bna,SAAU,kBACEzR,KAAKoP,YACJ,OACA,OACA,OACA,SACMwc,EAAqB,oBAAoBvvB,KAAK2D,WACpD,OACA,OACA,SACM4rB,EAAqB,qBAAqBvvB,KAAK2D,QAGlE0R,SAAU,KAEdQ,aAAe,CACXC,OAAS,QACTC,KAAO,UACPvJ,EAAI,kBACJwJ,GAAKmZ,EACLztB,EAAIytB,EACJlZ,GAAKkZ,EACLhf,EAAI,SACJ+F,GAAKiZ,EACLjf,EAAI,OACJiG,GAAKgZ,EACL/e,EAAI,SACJgG,GAAK+Y,EACLjiB,EAAI,MACJmJ,GAAK8Y,GAGT7Y,cAAe,wBACf/B,KAAM,SAAU5U,SACL,iBAAiB+K,KAAK/K,IAEjC4C,SAAW,SAAU0a,EAAMc,EAAQqM,UAC3BnN,EAAO,EACA,OACAA,EAAO,GACP,QACAA,EAAO,GACP,MAEA,UAGfrH,uBAAwB,iBACxBlM,QAAS,SAAUxE,EAAQgd,UACfA,OACC,QACA,QACA,UACA,QACA,WACMhd,EAAS,SACf,WACMA,EAAS,qBAETA,IAGnBgM,KAAO,CACHL,IAAM,EACNC,IAAM,MA9IwB+c,CAAQM,MTJtC,SAAUsB,UACbA,GAAMA,EAAGtqB,MAAQA,MAAQsqB,MAMhC3B,EAA2B,iBAAd4B,YAA0BA,aACvC5B,EAAuB,iBAAV6B,QAAsBA,SACnC7B,EAAqB,iBAAR8B,MAAoBA,OACjC9B,EAAuB,iBAAV+B,GAAsBA,IAEnCvoB,SAAS,cAATA,KUZe,SAAUuT,eAEdA,IACT,MAAOiV,UACA,OCDOC,GAAM,kBACwD,GAAvElwB,OAAOmwB,eAAe,GAAI,EAAG,CAAEtiB,IAAK,kBAAqB,KAAQ,MCHtEuiB,EAA6B,GAAGC,qBAChCC,EAA2BtwB,OAAOswB,8BAGpBA,IAA6BF,EAA2BjwB,KAAK,GAAK,GAAK,GAI/D,SAA8BowB,OAClDC,EAAaF,EAAyBxsB,KAAMysB,WACvCC,GAAcA,EAAWC,YAChCL,KCZa,SAAUM,EAAQ9qB,SAC1B,CACL6qB,aAAuB,EAATC,GACdC,eAAyB,EAATD,GAChBE,WAAqB,EAATF,GACZ9qB,MAAOA,ICLP1F,EAAW,GAAGA,WAED,SAAU0vB,UAClB1vB,EAASC,KAAKyvB,GAAI5oB,MAAM,GAAI,ICAjC2H,EAAQ,GAAGA,QAGEuhB,GAAM,kBAGblwB,OAAO,KAAKqwB,qBAAqB,MACtC,SAAUT,SACS,UAAfiB,EAAQjB,GAAkBjhB,EAAMxO,KAAKyvB,EAAI,IAAM5vB,OAAO4vB,IAC3D5vB,SCVa,SAAU4vB,MACfrsB,MAANqsB,EAAiB,MAAMkB,UAAU,wBAA0BlB,UACxDA,KCAQ,SAAUA,UAClBmB,EAAcC,EAAuBpB,OCL7B,SAAUA,SACJ,iBAAPA,EAAyB,OAAPA,EAA4B,mBAAPA,KCKtC,SAAU9vB,EAAOmxB,OAC3B7wB,EAASN,GAAQ,OAAOA,MACzBa,EAAI0D,KACJ4sB,GAAoD,mBAAxBtwB,EAAKb,EAAMI,YAA4BE,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,KAC3E,mBAAvB1D,EAAKb,EAAMuB,WAA2BjB,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,MACpF4sB,GAAoD,mBAAxBtwB,EAAKb,EAAMI,YAA4BE,EAASiE,EAAM1D,EAAGR,KAAKL,IAAS,OAAOuE,QACzGysB,UAAU,4CCZd3vB,EAAiB,GAAGA,iBAEP,SAAUyuB,EAAI7oB,UACtB5F,EAAehB,KAAKyvB,EAAI7oB,ICA7BmqB,EAAWlB,EAAOkB,SAElBC,EAAS/wB,EAAS8wB,IAAa9wB,EAAS8wB,EAASE,iBAEpC,SAAUxB,UAClBuB,EAASD,EAASE,cAAcxB,GAAM,OCH7ByB,IAAgBnB,GAAM,kBAG9B,GAFDlwB,OAAOmwB,eAAeiB,EAAc,OAAQ,IAAK,CACtDvjB,IAAK,kBAAqB,KACzB5M,KCADqwB,EAAiCtxB,OAAOswB,8BAIhCe,EAAcC,EAAiC,SAAkCC,EAAGC,MAC9FD,EAAIE,EAAgBF,GACpBC,EAAIE,EAAYF,GAAG,GACfG,EAAgB,WACXL,EAA+BC,EAAGC,GACzC,MAAOvB,OACL2B,EAAIL,EAAGC,GAAI,OAAOK,GAA0BC,EAA2BC,EAAE5xB,KAAKoxB,EAAGC,GAAID,EAAEC,MChBzFQ,EAAc,kBAEdC,EAAW,SAAUC,EAASC,OAC5BvsB,EAAQyR,EAAK+a,EAAUF,WACpBtsB,GAASysB,GACZzsB,GAAS0sB,IACW,mBAAbH,EAA0BjC,EAAMiC,KACrCA,IAGJC,EAAYH,EAASG,UAAY,SAAUrX,UACtCwX,OAAOxX,GAAQ1Q,QAAQ2nB,EAAa,KAAK5pB,eAG9CiP,EAAO4a,EAAS5a,KAAO,GACvBib,EAASL,EAASK,OAAS,IAC3BD,EAAWJ,EAASI,SAAW,MAElBJ,ICpBA,KCAA,SAAUrC,MACR,mBAANA,QACHkB,UAAUyB,OAAO3C,GAAM,6BACtBA,KCAM,SAAUjvB,EAAImkB,EAAMhkB,MACnC0xB,EAAU7xB,QACG4C,IAATuhB,EAAoB,OAAOnkB,SACvBG,QACD,SAAU,kBACNH,EAAGR,KAAK2kB,SAEZ,SAAU,SAAU7jB,UAChBN,EAAGR,KAAK2kB,EAAM7jB,SAElB,SAAU,SAAUA,EAAGC,UACnBP,EAAGR,KAAK2kB,EAAM7jB,EAAGC,SAErB,SAAU,SAAUD,EAAGC,EAAGmX,UACtB1X,EAAGR,KAAK2kB,EAAM7jB,EAAGC,EAAGmX,WAGxB,kBACE1X,EAAGhB,MAAMmlB,EAAMllB,eCnBT,SAAUgwB,OACpBxvB,EAASwvB,SACNkB,UAAUyB,OAAO3C,GAAM,4BACtBA,GCAP6C,EAAuBzyB,OAAOmwB,oBAItBkB,EAAcoB,EAAuB,SAAwBlB,EAAGC,EAAGkB,MAC7EC,EAASpB,GACTC,EAAIE,EAAYF,GAAG,GACnBmB,EAASD,GACLf,EAAgB,WACXc,EAAqBlB,EAAGC,EAAGkB,GAClC,MAAOzC,OACL,QAASyC,GAAc,QAASA,EAAY,MAAM5B,UAAU,iCAC5D,UAAW4B,IAAYnB,EAAEC,GAAKkB,EAAW9sB,OACtC2rB,MCdQF,EAAc,SAAUuB,EAAQ7rB,EAAKnB,UAC7CitB,EAAqBd,EAAEa,EAAQ7rB,EAAK8qB,EAAyB,EAAGjsB,KACrE,SAAUgtB,EAAQ7rB,EAAKnB,UACzBgtB,EAAO7rB,GAAOnB,EACPgtB,GCNLtC,EAA2BhC,EAA2DyD,EAOtFe,EAAkB,SAAUC,OAC1BC,EAAU,SAAU/xB,EAAGC,EAAGmX,MACxBvU,gBAAgBivB,EAAmB,QAC7BnzB,UAAUkB,aACX,SAAU,IAAIiyB,OACd,SAAU,IAAIA,EAAkB9xB,QAChC,SAAU,IAAI8xB,EAAkB9xB,EAAGC,UACjC,IAAI6xB,EAAkB9xB,EAAGC,EAAGmX,UAC9B0a,EAAkBpzB,MAAMmE,KAAMlE,mBAEzCozB,EAAQ/yB,UAAY8yB,EAAkB9yB,UAC/B+yB,KAiBQ,SAAUC,EAAStJ,OAWtBuJ,EAAYC,EACpBpsB,EAAKqsB,EAAgBC,EAAgBC,EAAgBC,EAAgB/C,EAXrEgD,EAASP,EAAQQ,OACjBC,EAAST,EAAQjD,OACjB2D,EAASV,EAAQW,KACjBC,EAAQZ,EAAQ3O,MAEhBwP,EAAeJ,EAAS1D,EAAS2D,EAAS3D,EAAOwD,IAAWxD,EAAOwD,IAAW,IAAIvzB,UAElFwzB,EAASC,EAASK,EAAOA,EAAKP,KAAYO,EAAKP,GAAU,IACzDQ,EAAkBP,EAAOxzB,cAKxB8G,KAAO4iB,EAGVuJ,GAFSjB,EAASyB,EAAS3sB,EAAMysB,GAAUG,EAAS,IAAM,KAAO5sB,EAAKksB,EAAQgB,SAEtDH,GAAgBlC,EAAIkC,EAAc/sB,GAE1DssB,EAAiBI,EAAO1sB,GAEpBmsB,IAEFI,EAFkBL,EAAQiB,aAC1B1D,EAAaF,EAAyBwD,EAAc/sB,KACrBypB,EAAW5qB,MACpBkuB,EAAa/sB,IAGrCqsB,EAAkBF,GAAcI,EAAkBA,EAAiB3J,EAAO5iB,GAEtEmsB,UAAqBG,UAA0BD,IAGnBG,EAA5BN,EAAQkB,MAAQjB,EAA6BiB,EAAKf,EAAgBpD,GAE7DiD,EAAQmB,MAAQlB,EAA6BJ,EAAgBM,GAE7DS,GAAkC,mBAAlBT,EAA+Ce,EAAK1sB,SAAStH,KAAMizB,GAEtEA,GAGlBH,EAAQoB,MAASjB,GAAkBA,EAAeiB,MAAUhB,GAAkBA,EAAegB,OAC/FC,EAA4Bf,EAAgB,QAAQ,GAGtDE,EAAO1sB,GAAOwsB,EAEVM,IAEGjC,EAAImC,EADTZ,EAAoBK,EAAS,cAE3Bc,EAA4BP,EAAMZ,EAAmB,IAGvDY,EAAKZ,GAAmBpsB,GAAOqsB,EAE3BH,EAAQsB,MAAQP,IAAoBA,EAAgBjtB,IACtDutB,EAA4BN,EAAiBjtB,EAAKqsB,QCvFxD,CAAEK,OAAQ,SAAUG,MAAM,EAAMK,QAAS5C,EAAagD,MAAOhD,GAAe,CAC5ElB,eAAgBqE,EAA2BzC,6BCJzC/xB,EAAS+zB,EAAK/zB,OAEdmwB,EAAiB5wB,UAAiB,SAAwBqwB,EAAI7oB,EAAK0tB,UAC9Dz0B,EAAOmwB,eAAeP,EAAI7oB,EAAK0tB,IAGpCz0B,EAAOmwB,eAAekE,OAAMlE,EAAekE,MAAO,QCPrCK,ECFbnvB,EAAOD,KAAKC,KACZC,EAAQF,KAAKE,QAIA,SAAUmvB,UAClBzxB,MAAMyxB,GAAYA,GAAY,GAAKA,EAAW,EAAInvB,EAAQD,GAAMovB,ICJrEzuB,EAAMZ,KAAKY,MAIE,SAAUyuB,UAClBA,EAAW,EAAIzuB,EAAI0uB,EAAUD,GAAW,kBAAoB,GCLjEvrB,GAAM9D,KAAK8D,IACXlD,GAAMZ,KAAKY,OAKE,SAAUwiB,EAAO5nB,OAC5B+zB,EAAUD,EAAUlM,UACjBmM,EAAU,EAAIzrB,GAAIyrB,EAAU/zB,EAAQ,GAAKoF,GAAI2uB,EAAS/zB,ICL3Dg0B,GAAe,SAAUC,UACpB,SAAUC,EAAOC,EAAIC,OAItBtvB,EAHA2rB,EAAIE,EAAgBuD,GACpBl0B,EAASq0B,EAAS5D,EAAEzwB,QACpB4nB,EAAQ0M,GAAgBF,EAAWp0B,MAInCi0B,GAAeE,GAAMA,GAAI,KAAOn0B,EAAS4nB,OAC3C9iB,EAAQ2rB,EAAE7I,OAEG9iB,EAAO,OAAO,OAEtB,KAAM9E,EAAS4nB,EAAOA,QACtBqM,GAAerM,KAAS6I,IAAMA,EAAE7I,KAAWuM,EAAI,OAAOF,GAAerM,GAAS,SAC3EqM,IAAgB,OAIb,CAGfM,SAAUP,IAAa,GAGvBtnB,QAASsnB,IAAa,OC9BP,GCEbtnB,GAAU8gB,GAAuC9gB,WAGpC,SAAUolB,EAAQhb,OAI7B7Q,EAHAwqB,EAAIE,EAAgBmB,GACpBhyB,EAAI,EACJyb,EAAS,OAERtV,KAAOwqB,GAAIK,EAAI0D,GAAYvuB,IAAQ6qB,EAAIL,EAAGxqB,IAAQsV,EAAOtb,KAAKgG,QAE5D6Q,EAAM9W,OAASF,GAAOgxB,EAAIL,EAAGxqB,EAAM6Q,EAAMhX,SAC7C4M,GAAQ6O,EAAQtV,IAAQsV,EAAOtb,KAAKgG,WAEhCsV,MCdQ,CACf,cACA,iBACA,gBACA,uBACA,iBACA,WACA,cCHerc,OAAOoH,MAAQ,SAAcmqB,UACrCgE,GAAmBhE,EAAGiE,QCCdnE,EAAcrxB,OAAOy1B,iBAAmB,SAA0BlE,EAAGmE,GACpF/C,EAASpB,WAILxqB,EAHAK,EAAOuuB,GAAWD,GAClB50B,EAASsG,EAAKtG,OACd4nB,EAAQ,EAEL5nB,EAAS4nB,GAAOmK,EAAqBd,EAAER,EAAGxqB,EAAMK,EAAKshB,KAAUgN,EAAW3uB,WAC1EwqB,KCRP,CAAEkC,OAAQ,SAAUG,MAAM,EAAMK,QAAS5C,EAAagD,MAAOhD,GAAe,CAC5EoE,iBAAkBA,+BCJhBz1B,EAAS+zB,EAAK/zB,OAEdy1B,EAAmBl2B,UAAiB,SAA0Bq2B,EAAG1J,UAC5DlsB,EAAOy1B,iBAAiBG,EAAG1J,IAGhClsB,EAAOy1B,iBAAiBpB,OAAMoB,EAAiBpB,MAAO,MCNtD7B,GAAY,SAAUqD,SACE,mBAAZA,EAAyBA,OAAWtyB,MAGnC,SAAUuyB,EAAWC,UAC7Bn2B,UAAUkB,OAAS,EAAI0xB,GAAUuB,EAAK+B,KAAetD,GAAUxC,EAAO8F,IACzE/B,EAAK+B,IAAc/B,EAAK+B,GAAWC,IAAW/F,EAAO8F,IAAc9F,EAAO8F,GAAWC,ICNvFT,GAAaE,GAAYtjB,OAAO,SAAU,mBAIlClS,OAAOwe,qBAAuB,SAA6B+S,UAC9DgE,GAAmBhE,EAAG+D,YCRnBt1B,OAAOg2B,0BCMFC,GAAW,UAAW,YAAc,SAAiBrG,OAChExoB,EAAO8uB,GAA0BnE,EAAEY,EAAS/C,IAC5CoG,EAAwBG,GAA4BpE,SACjDiE,EAAwB5uB,EAAK8K,OAAO8jB,EAAsBpG,IAAOxoB,MCJzD,SAAUwrB,EAAQ7rB,EAAKnB,OAClCwwB,EAAc1E,EAAY3qB,GAC1BqvB,KAAexD,EAAQC,EAAqBd,EAAEa,EAAQwD,EAAavE,EAAyB,EAAGjsB,IAC9FgtB,EAAOwD,GAAexwB,KCC3B,CAAE6tB,OAAQ,SAAUG,MAAM,EAAMS,MAAOhD,GAAe,CACtDgF,0BAA2B,SAAmCzD,WAMxD7rB,EAAKypB,EALLe,EAAIE,EAAgBmB,GACpBtC,EAA2BgG,EAA+BvE,EAC1D3qB,EAAOmvB,GAAQhF,GACflV,EAAS,GACTqM,EAAQ,EAELthB,EAAKtG,OAAS4nB,QAEAnlB,KADnBitB,EAAaF,EAAyBiB,EAAGxqB,EAAMK,EAAKshB,QACtB8N,GAAena,EAAQtV,EAAKypB,UAErDnU,KClBX,OAAiB0X,EAAK/zB,OAAOq2B,0BCAzB/E,GAAiChD,EAA2DyD,EAG5F0E,GAAsBvG,GAAM,WAAcoB,GAA+B,QAK3E,CAAEmC,OAAQ,SAAUG,MAAM,EAAMK,QAJpB5C,GAAeoF,GAIqBpC,MAAOhD,GAAe,CACtEf,yBAA0B,SAAkCV,EAAI7oB,UACvDuqB,GAA+BG,EAAgB7B,GAAK7oB,UCsC3D2vB,yBChDA12B,EAAS+zB,EAAK/zB,OAEdswB,EAA2B/wB,UAAiB,SAAkCqwB,EAAI7oB,UAC7E/G,EAAOswB,yBAAyBV,EAAI7oB,IAGzC/G,EAAOswB,yBAAyB+D,OAAM/D,EAAyB+D,MAAO,WCPvDr0B,OAAOg2B,wBAA0B9F,GAAM,kBAGhDqC,OAAOoE,gBCHAC,KAEXD,OAAOtC,MAEkB,iBAAnBsC,OAAOE,YCFF92B,MAAMF,SAAW,SAAiBgH,SAC1B,SAAhBgqB,EAAQhqB,OCDA,SAAU8tB,UAClB30B,OAAOgxB,EAAuB2D,QCHtBsB,GAAW,WAAY,sBCE5BjG,EADC,uBCAI,SAAUjpB,EAAKnB,OAE5B0uB,EAA4BtE,EAAQjpB,EAAKnB,GACzC,MAAOqqB,GACPD,EAAOjpB,GAAOnB,SACPA,EDJmBkxB,CADjB,qBACmC,uBED/Cv3B,UAAiB,SAAUwH,EAAKnB,UACxBmxB,GAAMhwB,KAASgwB,GAAMhwB,QAAiBxD,IAAVqC,EAAsBA,EAAQ,MAChE,WAAY,IAAI7E,KAAK,CACtB8rB,QAAS,QACTmK,KAAgB,OAChBC,UAAW,4CCRTC,GAAK,EACLC,GAAU7xB,KAAK8xB,YAEF,SAAUrwB,SAClB,UAAYwrB,YAAehvB,IAARwD,EAAoB,GAAKA,GAAO,QAAUmwB,GAAKC,IAASj3B,SAAS,KCDzFkH,GAAOiwB,GAAO,WAED,SAAUtwB,UAClBK,GAAKL,KAASK,GAAKL,GAAOuwB,GAAIvwB,KXMnCwwB,GAAWC,GAAU,YAErBC,GAAmB,aAEnBC,GAAY,SAAUC,SACjBjiB,WAAmBiiB,EARnB,cA2CLC,GAAkB,eAGlBlB,GAAkBxF,SAAS2G,QAAU,IAAIC,cAAc,YACvD,MAAO7H,IA1BoB,IAIzB8H,EAFAC,EAyBJJ,GAAkBlB,GApCY,SAAUA,GACxCA,EAAgBuB,MAAMP,GAAU,KAChChB,EAAgBwB,YACZ9e,EAAOsd,EAAgByB,aAAan4B,cACxC02B,EAAkB,KACXtd,EA+B6Bgf,CAA0B1B,MAzB1DsB,EAASK,EAAsB,WAG5BC,MAAMC,QAAU,OACvBC,GAAKC,YAAYT,GAEjBA,EAAOU,IAAMnG,OALJ,gBAMTwF,EAAiBC,EAAOW,cAAczH,UACvB0H,OACfb,EAAeE,MAAMP,GAAU,sBAC/BK,EAAeG,QACRH,EAAec,WAelB/3B,EAAS00B,GAAY10B,OAClBA,YAAiB82B,GAAe,UAAYpC,GAAY10B,WACxD82B,MAGTtC,GAAWiC,KAAY,EAIvB,OAAiBv3B,OAAO84B,QAAU,SAAgBvH,EAAGmE,OAC/CrZ,SACM,OAANkV,GACFkG,GAAgB,UAAc9E,EAASpB,GACvClV,EAAS,IAAIob,GACbA,GAAgB,UAAc,KAE9Bpb,EAAOkb,IAAYhG,GACdlV,EAASub,UACMr0B,IAAfmyB,EAA2BrZ,EAASoZ,GAAiBpZ,EAAQqZ,IY3ElEqD,GAA4BzK,GAAsDyD,EAElF7xB,GAAW,GAAGA,SAEd84B,GAA+B,iBAAVlJ,QAAsBA,QAAU9vB,OAAOwe,oBAC5Dxe,OAAOwe,oBAAoBsR,QAAU,SAWtB,SAA6BF,UACvCoJ,IAAoC,mBAArB94B,GAASC,KAAKyvB,GAVjB,SAAUA,cAEpBmJ,GAA0BnJ,GACjC,MAAOK,UACA+I,GAAYhyB,SAOjBiyB,CAAerJ,GACfmJ,GAA0BtH,EAAgB7B,SClB/B,SAAU6D,EAAQ1sB,EAAKnB,EAAOqtB,GACzCA,GAAWA,EAAQxC,WAAYgD,EAAO1sB,GAAOnB,EAC5C0uB,EAA4Bb,EAAQ1sB,EAAKnB,ICG5CszB,GAAwB7B,GAAO,OAC/BV,GAAS3G,EAAO2G,OAChBwC,GAAwBC,GAAoBzC,GAASA,IAAUA,GAAO0C,eAAiB/B,MAE1E,SAAU/vB,UACpBqqB,EAAIsH,GAAuB3xB,KAC1BqvB,IAAiBhF,EAAI+E,GAAQpvB,GAAO2xB,GAAsB3xB,GAAQovB,GAAOpvB,GACxE2xB,GAAsB3xB,GAAQ4xB,GAAsB,UAAY5xB,IAC9D2xB,GAAsB3xB,UCbrB+xB,ICCRnJ,GAAiB7B,EAA+CyD,KAEnD,SAAUwH,OACrB5C,EAAS5C,EAAK4C,SAAW5C,EAAK4C,OAAS,IACtC/E,EAAI+E,EAAQ4C,IAAOpJ,GAAewG,EAAQ4C,EAAM,CACnD3zB,MAAO4zB,GAA6BzH,EAAEwH,MCLtC1uB,GAAO,GAEXA,GAHoByuB,GAAgB,gBAGd,IAEtB,OAAkC,eAAjB/G,OAAO1nB,ICHpB4uB,GAAgBH,GAAgB,eAEhCI,GAAuE,aAAnDC,EAAW,kBAAqB/5B,UAArB,OAUlBg6B,GAAwBD,EAAa,SAAU/J,OAC1D2B,EAAGsI,EAAKxd,cACE9Y,IAAPqsB,EAAmB,YAAqB,OAAPA,EAAc,OAEM,iBAAhDiK,EAXD,SAAUjK,EAAI7oB,cAEhB6oB,EAAG7oB,GACV,MAAOkpB,KAQS6J,CAAOvI,EAAIvxB,OAAO4vB,GAAK6J,KAA8BI,EAEnEH,GAAoBC,EAAWpI,GAEH,WAA3BlV,EAASsd,EAAWpI,KAAsC,mBAAZA,EAAEwI,OAAuB,YAAc1d,MClB3Eud,GAAwB,GAAG15B,SAAW,iBAC9C,WAAa2wB,GAAQ/sB,MAAQ,KCNlCqsB,GAAiB7B,EAA+CyD,EAMhE0H,GAAgBH,GAAgB,kBAEnB,SAAU1J,EAAIoK,EAAKrG,EAAQsG,MACtCrK,EAAI,KACF6D,EAASE,EAAS/D,EAAKA,EAAG3vB,UACzB2xB,EAAI6B,EAAQgG,KACftJ,GAAesD,EAAQgG,GAAe,CAAE9I,cAAc,EAAM/qB,MAAOo0B,IAEjEC,IAAeL,IACjBtF,EAA4Bb,EAAQ,WAAYvzB,MCdlDg6B,GAAmBzyB,SAASvH,SAGE,mBAAvB62B,GAAMoD,gBACfpD,GAAMoD,cAAgB,SAAUvK,UACvBsK,GAAiB/5B,KAAKyvB,KAIjC,ICFI9nB,GAAK+F,GAAK+jB,MDEGmF,GAAMoD,cERnBC,GAAUpK,EAAOoK,WAEe,mBAAZA,IAA0B,cAAcvvB,KAAKsvB,GAAcC,KDG/EA,GAAUpK,EAAOoK,QAgBrB,GAAIC,GAAiB,KACftD,GAAQ,IAAIqD,GACZE,GAAQvD,GAAMlpB,IACd0sB,GAAQxD,GAAMnF,IACd4I,GAAQzD,GAAMjvB,IAClBA,GAAM,SAAU8nB,EAAI6K,UAClBD,GAAMr6B,KAAK42B,GAAOnH,EAAI6K,GACfA,GAET5sB,GAAM,SAAU+hB,UACP0K,GAAMn6B,KAAK42B,GAAOnH,IAAO,IAElCgC,GAAM,SAAUhC,UACP2K,GAAMp6B,KAAK42B,GAAOnH,QAEtB,KACD8K,GAAQlD,GAAU,SACtBlC,GAAWoF,KAAS,EACpB5yB,GAAM,SAAU8nB,EAAI6K,UAClBnG,EAA4B1E,EAAI8K,GAAOD,GAChCA,GAET5sB,GAAM,SAAU+hB,UACP+K,EAAU/K,EAAI8K,IAAS9K,EAAG8K,IAAS,IAE5C9I,GAAM,SAAUhC,UACP+K,EAAU/K,EAAI8K,KAIzB,OAAiB,CACf5yB,IAAKA,GACL+F,IAAKA,GACL+jB,IAAKA,GACLgJ,QA/CY,SAAUhL,UACfgC,GAAIhC,GAAM/hB,GAAI+hB,GAAM9nB,GAAI8nB,EAAI,KA+CnCiL,UA5Cc,SAAUC,UACjB,SAAUlL,OACXmL,MACC36B,EAASwvB,KAAQmL,EAAQltB,GAAI+hB,IAAKoL,OAASF,QACxChK,UAAU,0BAA4BgK,EAAO,oBAC5CC,KEhBTE,GAAU3B,GAAgB,cAIb,SAAU4B,EAAep6B,OACpCq6B,SACAt7B,GAAQq7B,KAGM,mBAFhBC,EAAID,EAAcE,cAEaD,IAAMp7B,QAASF,GAAQs7B,EAAEl7B,WAC/CG,EAAS+6B,IAEN,QADVA,EAAIA,EAAEF,OACUE,OAAI53B,GAH+C43B,OAAI53B,GAKlE,SAAWA,IAAN43B,EAAkBp7B,MAAQo7B,GAAc,IAAXr6B,EAAe,EAAIA,ICZ5DC,GAAO,GAAGA,KAGV+zB,GAAe,SAAUgG,OACvBO,EAAiB,GAARP,EACTQ,EAAoB,GAARR,EACZS,EAAkB,GAART,EACVU,EAAmB,GAARV,EACXW,EAAwB,GAARX,EAChBY,EAAmB,GAARZ,GAAaW,SACrB,SAAUzG,EAAO2G,EAAY7W,EAAM8W,WAQpCh2B,EAAOyW,EAPPkV,EAAI9K,GAASuO,GACbjF,EAAOgB,EAAcQ,GACrBsK,EAAgB1H,EAAKwH,EAAY7W,EAAM,GACvChkB,EAASq0B,EAASpF,EAAKjvB,QACvB4nB,EAAQ,EACRoQ,EAAS8C,GAAkBE,GAC3BrI,EAAS4H,EAASvC,EAAO9D,EAAOl0B,GAAUw6B,EAAYxC,EAAO9D,EAAO,QAAKzxB,EAEvEzC,EAAS4nB,EAAOA,IAAS,IAAIgT,GAAYhT,KAASqH,KAEtD1T,EAASwf,EADTj2B,EAAQmqB,EAAKrH,GACiBA,EAAO6I,GACjCuJ,MACEO,EAAQ5H,EAAO/K,GAASrM,OACvB,GAAIA,EAAQ,OAAQye,QAClB,SAAU,OACV,SAAUl1B,OACV,SAAU8iB,OACV,EAAG3nB,GAAKZ,KAAKszB,EAAQ7tB,QACrB,GAAI41B,EAAU,OAAO,SAGzBC,GAAiB,EAAIF,GAAWC,EAAWA,EAAW/H,OAIhD,CAGf9b,QAASmd,GAAa,GAGtBr0B,IAAKq0B,GAAa,GAGlBiH,OAAQjH,GAAa,GAGrBr1B,KAAMq1B,GAAa,GAGnBkH,MAAOlH,GAAa,GAGpBmH,KAAMnH,GAAa,GAGnBoH,UAAWpH,GAAa,IC3BtBqH,GAAW7N,GAAwC3W,QAEnDykB,GAAS5E,GAAU,UAGnB6E,GAAe/C,GAAgB,eAC/BgD,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAJ9B,UAKT4B,GAAkBz8B,OAAM,UACxB08B,GAAU1M,EAAO2G,OACjBgG,GAAa1G,GAAW,OAAQ,aAChC3E,GAAiCgF,EAA+BvE,EAChEU,GAAuBI,EAAqBd,EAC5CgH,GAA4B6D,GAA4B7K,EACxD3B,GAA6B0B,EAA2BC,EACxD8K,GAAaxF,GAAO,WACpByF,GAAyBzF,GAAO,cAChC0F,GAAyB1F,GAAO,6BAChC2F,GAAyB3F,GAAO,6BAChC6B,GAAwB7B,GAAO,OAC/B4F,GAAUjN,EAAOiN,QAEjBC,IAAcD,KAAYA,GAAO,YAAgBA,GAAO,UAAYE,UAGpEC,GAAsB/L,GAAenB,GAAM,kBAGpC,GAFFmN,GAAmB5K,GAAqB,GAAI,IAAK,CACtD5kB,IAAK,kBAAqB4kB,GAAqB3uB,KAAM,IAAK,CAAE8B,MAAO,IAAK3E,MACtEA,KACD,SAAUswB,EAAGC,EAAGkB,OACf4K,EAA4BhM,GAA+BmL,GAAiBjL,GAC5E8L,UAAkCb,GAAgBjL,GACtDiB,GAAqBlB,EAAGC,EAAGkB,GACvB4K,GAA6B/L,IAAMkL,IACrChK,GAAqBgK,GAAiBjL,EAAG8L,IAEzC7K,GAEA2B,GAAO,SAAUyF,EAAK0D,OACpBC,EAASX,GAAWhD,GAAOwD,GAAmBX,GAAO,kBACzDJ,GAAiBkB,EAAQ,CACvBxC,KAtCS,SAuCTnB,IAAKA,EACL0D,YAAaA,IAEVlM,IAAamM,EAAOD,YAAcA,GAChCC,GAGLC,GAAWrE,GAAoB,SAAUxJ,SACvB,iBAANA,GACZ,SAAUA,UACL5vB,OAAO4vB,aAAe8M,IAG3BgB,GAAkB,SAAwBnM,EAAGC,EAAGkB,GAC9CnB,IAAMkL,IAAiBiB,GAAgBZ,GAAwBtL,EAAGkB,GACtEC,EAASpB,OACLxqB,EAAM2qB,EAAYF,GAAG,UACzBmB,EAASD,GACLd,EAAIiL,GAAY91B,IACb2rB,EAAWjC,YAIVmB,EAAIL,EAAG6K,KAAW7K,EAAE6K,IAAQr1B,KAAMwqB,EAAE6K,IAAQr1B,IAAO,GACvD2rB,EAAa2K,GAAmB3K,EAAY,CAAEjC,WAAYoB,EAAyB,GAAG,OAJjFD,EAAIL,EAAG6K,KAAS3J,GAAqBlB,EAAG6K,GAAQvK,EAAyB,EAAG,KACjFN,EAAE6K,IAAQr1B,IAAO,GAIVq2B,GAAoB7L,EAAGxqB,EAAK2rB,IAC9BD,GAAqBlB,EAAGxqB,EAAK2rB,IAGpCiL,GAAoB,SAA0BpM,EAAGmE,GACnD/C,EAASpB,OACLqM,EAAanM,EAAgBiE,GAC7BtuB,EAAOuuB,GAAWiI,GAAY1rB,OAAO2rB,GAAuBD,WAChEzB,GAAS/0B,GAAM,SAAUL,GAClBsqB,IAAeyM,GAAsB39B,KAAKy9B,EAAY72B,IAAM22B,GAAgBnM,EAAGxqB,EAAK62B,EAAW72B,OAE/FwqB,GAOLuM,GAAwB,SAA8BvN,OACpDiB,EAAIE,EAAYnB,GAAG,GACnBE,EAAaL,GAA2BjwB,KAAK2D,KAAM0tB,WACnD1tB,OAAS24B,IAAmB7K,EAAIiL,GAAYrL,KAAOI,EAAIkL,GAAwBtL,QAC5Ef,IAAemB,EAAI9tB,KAAM0tB,KAAOI,EAAIiL,GAAYrL,IAAMI,EAAI9tB,KAAMs4B,KAAWt4B,KAAKs4B,IAAQ5K,KAAKf,IAGlGsN,GAA4B,SAAkCxM,EAAGC,OAC/D5B,EAAK6B,EAAgBF,GACrBxqB,EAAM2qB,EAAYF,GAAG,MACrB5B,IAAO6M,KAAmB7K,EAAIiL,GAAY91B,IAAS6qB,EAAIkL,GAAwB/1B,QAC/EypB,EAAac,GAA+B1B,EAAI7oB,UAChDypB,IAAcoB,EAAIiL,GAAY91B,IAAU6qB,EAAIhC,EAAIwM,KAAWxM,EAAGwM,IAAQr1B,KACxEypB,EAAWC,YAAa,GAEnBD,IAGLwN,GAAuB,SAA6BzM,OAClD3Z,EAAQmhB,GAA0BtH,EAAgBF,IAClDlV,EAAS,UACb8f,GAASvkB,GAAO,SAAU7Q,GACnB6qB,EAAIiL,GAAY91B,IAAS6qB,EAAI0D,GAAYvuB,IAAMsV,EAAOtb,KAAKgG,MAE3DsV,GAGLwhB,GAAyB,SAA+BtM,OACtD0M,EAAsB1M,IAAMkL,GAC5B7kB,EAAQmhB,GAA0BkF,EAAsBnB,GAAyBrL,EAAgBF,IACjGlV,EAAS,UACb8f,GAASvkB,GAAO,SAAU7Q,IACpB6qB,EAAIiL,GAAY91B,IAAUk3B,IAAuBrM,EAAI6K,GAAiB11B,IACxEsV,EAAOtb,KAAK87B,GAAW91B,OAGpBsV,GAkHT,GA7GKua,KAcHsH,IAbAxB,GAAU,cACJ54B,gBAAgB44B,GAAS,MAAM5L,UAAU,mCACzCyM,EAAe39B,UAAUkB,aAA2ByC,IAAjB3D,UAAU,GAA+B2yB,OAAO3yB,UAAU,SAA7B2D,EAChEs2B,EAAMvC,GAAIiG,GACV3U,EAAS,SAAUhjB,GACjB9B,OAAS24B,IAAiB7T,EAAOzoB,KAAK28B,GAAwBl3B,GAC9DgsB,EAAI9tB,KAAMs4B,KAAWxK,EAAI9tB,KAAKs4B,IAASvC,KAAM/1B,KAAKs4B,IAAQvC,IAAO,GACrEuD,GAAoBt5B,KAAM+1B,EAAKhI,EAAyB,EAAGjsB,YAEzDyrB,GAAe6L,IAAYE,GAAoBX,GAAiB5C,EAAK,CAAElJ,cAAc,EAAM7oB,IAAK8gB,IAC7FwL,GAAKyF,EAAK0D,KAGH,UAAa,YAAY,kBAChCf,GAAiB14B,MAAM+1B,OAGhCqE,GAASxB,GAAS,iBAAiB,SAAUa,UACpCnJ,GAAKkD,GAAIiG,GAAcA,MAGhCzL,EAA2BC,EAAI+L,GAC/BjL,EAAqBd,EAAI2L,GACzBpH,EAA+BvE,EAAIgM,GACnC7H,GAA0BnE,EAAI6K,GAA4B7K,EAAIiM,GAC9D7H,GAA4BpE,EAAI8L,GAEhCrE,GAA6BzH,EAAI,SAAUxqB,UAClC6sB,GAAKkF,GAAgB/xB,GAAOA,IAGjC8pB,GAEFoB,GAAqBiK,GAAO,UAAa,cAAe,CACtD/L,cAAc,EACd9iB,IAAK,kBACI2uB,GAAiB14B,MAAMy5B,kBASpC,CAAEvN,QAAQ,EAAMoE,MAAM,EAAMH,QAAS2C,GAAevC,MAAOuC,IAAiB,CAC5ED,OAAQ+F,KAGVP,GAASxG,GAAWuD,KAAwB,SAAU3xB,GACpD42B,GAAsB52B,QAGtB,CAAEksB,OAjLS,SAiLOG,MAAM,EAAMK,QAAS2C,IAAiB,KAGjD,SAAU7vB,OACXgU,EAASwX,OAAOxrB,MAChB6qB,EAAImL,GAAwBhiB,GAAS,OAAOgiB,GAAuBhiB,OACnEyiB,EAASd,GAAQ3hB,UACrBgiB,GAAuBhiB,GAAUyiB,EACjCR,GAAuBQ,GAAUziB,EAC1ByiB,GAITY,OAAQ,SAAgBC,OACjBZ,GAASY,GAAM,MAAMvN,UAAUuN,EAAM,uBACtCzM,EAAIoL,GAAwBqB,GAAM,OAAOrB,GAAuBqB,IAEtEC,UAAW,WAAcpB,IAAa,GACtCqB,UAAW,WAAcrB,IAAa,OAGtC,CAAEzJ,OAAQ,SAAUG,MAAM,EAAMK,QAAS2C,GAAevC,MAAOhD,GAAe,CAG9EyH,OA3HY,SAAgBvH,EAAGmE,eACTnyB,IAAfmyB,EAA2B2H,GAAmB9L,GAAKoM,GAAkBN,GAAmB9L,GAAImE,IA6HnGvF,eAAgBuN,GAGhBjI,iBAAkBkI,GAGlBrN,yBAA0ByN,OAG1B,CAAEtK,OAAQ,SAAUG,MAAM,EAAMK,QAAS2C,IAAiB,CAG1DpY,oBAAqBwf,GAGrBhI,sBAAuB6H,OAKvB,CAAEpK,OAAQ,SAAUG,MAAM,EAAMK,OAAQ/D,GAAM,WAAciG,GAA4BpE,EAAE,OAAU,CACpGiE,sBAAuB,SAA+BpG,UAC7CuG,GAA4BpE,EAAEtL,GAASmJ,OAM9C+M,GAAY,KACV6B,IAAyB5H,IAAiB1G,GAAM,eAC9CsN,EAASd,WAEkB,UAAxBC,GAAW,CAACa,KAEe,MAA7Bb,GAAW,CAAE17B,EAAGu8B,KAEc,MAA9Bb,GAAW38B,OAAOw9B,OAGzBiB,EAAE,CAAEhL,OAAQ,OAAQG,MAAM,EAAMK,OAAQuK,IAAyB,CAE/DE,UAAW,SAAmB9O,EAAI+O,EAAUC,WAGtCC,EAFA/3B,EAAO,CAAC8oB,GACRlH,EAAQ,EAEL9oB,UAAUkB,OAAS4nB,GAAO5hB,EAAK/F,KAAKnB,UAAU8oB,SACrDmW,EAAYF,GACPv+B,EAASu+B,SAAoBp7B,IAAPqsB,KAAoB6N,GAAS7N,UACnD/vB,GAAQ8+B,KAAWA,EAAW,SAAU53B,EAAKnB,MACxB,mBAAbi5B,IAAyBj5B,EAAQi5B,EAAU1+B,KAAK2D,KAAMiD,EAAKnB,KACjE63B,GAAS73B,GAAQ,OAAOA,IAE/BkB,EAAK,GAAK63B,EACHhC,GAAWh9B,MAAM,KAAMmH,MAO/B41B,GAAO,UAAYL,KACtB/H,EAA4BoI,GAAO,UAAaL,GAAcK,GAAO,UAAYr7B,SAInFy9B,GAAepC,GA7QF,UA+QbpH,GAAW8G,KAAU,ECnTrB,ICWI2C,GAAmBC,GAAmCC,MDXzClL,EAAK/zB,OAAOg2B,yBEHZ,OCEC9F,GAAM,oBACb2I,YACTA,EAAE54B,UAAUm7B,YAAc,KACnBp7B,OAAOk/B,eAAe,IAAIrG,KAASA,EAAE54B,aCA1Cs3B,GAAWC,GAAU,YACrBiF,GAAkBz8B,OAAOC,aAIZk/B,GAA2Bn/B,OAAOk/B,eAAiB,SAAU3N,UAC5EA,EAAI9K,GAAS8K,GACTK,EAAIL,EAAGgG,IAAkBhG,EAAEgG,IACH,mBAAjBhG,EAAE6J,aAA6B7J,aAAaA,EAAE6J,YAChD7J,EAAE6J,YAAYn7B,UACdsxB,aAAavxB,OAASy8B,GAAkB,MHP/C2C,IADW9F,GAAgB,aACF,GAQzB,GAAGlyB,OAGC,SAFN63B,GAAgB,GAAG73B,SAIjB43B,GAAoCE,GAAeA,GAAeD,QACxBj/B,OAAOC,YAAW8+B,GAAoBC,IAHlDI,IAAyB,GAOlC77B,MAArBw7B,KAAgCA,GAAoB,IAOxD,OAAiB,CACfA,kBAAmBA,GACnBK,uBAAwBA,IIlCtBL,GAAoBzQ,GAAuCyQ,kBAM3DM,GAAa,kBAAqBv7B,SCArB9D,OAAOs/B,iBAAmB,aAAe,GAAK,eAGzD1W,EAFA2W,GAAiB,EACjB10B,EAAO,QAGT+d,EAAS5oB,OAAOswB,yBAAyBtwB,OAAOC,UAAW,aAAa6H,KACjE3H,KAAK0K,EAAM,IAClB00B,EAAiB10B,aAAgB9K,MACjC,MAAOkwB,WACF,SAAwBsB,EAAGjN,UAChCqO,EAASpB,GCfI,SAAU3B,OACpBxvB,EAASwvB,IAAc,OAAPA,QACbkB,UAAU,aAAeyB,OAAO3C,GAAM,mBDc5C4P,CAAmBlb,GACfib,EAAgB3W,EAAOzoB,KAAKoxB,EAAGjN,GAC9BiN,EAAEkO,UAAYnb,EACZiN,GAdoD,QAgBzDhuB,GEVFw7B,GAAoBW,GAAcX,kBAClCK,GAAyBM,GAAcN,uBACvCO,GAAWrG,GAAgB,YAK3B+F,GAAa,kBAAqBv7B,SAErB,SAAU87B,EAAUrG,EAAMsG,EAAqB/nB,EAAMgoB,EAASC,EAAQC,IHbtE,SAAUH,EAAqBtG,EAAMzhB,OAChD2hB,EAAgBF,EAAO,YAC3BsG,EAAoB5/B,UAAY64B,GAAOiG,GAAmB,CAAEjnB,KAAM+Z,EAAyB,EAAG/Z,KAC9FgnB,GAAee,EAAqBpG,GAAe,GAAO,GAC1DwG,GAAUxG,GAAiB4F,GGU3Ba,CAA0BL,EAAqBtG,EAAMzhB,OAoBjDqoB,EAA0BC,EAASC,EAlBnCC,EAAqB,SAAUC,MAC7BA,IAAST,GAAWU,EAAiB,OAAOA,MAC3CpB,IAA0BmB,KAAQE,EAAmB,OAAOA,EAAkBF,UAC3EA,OAZD,WACE,aACC,iBAaa,kBAA4B,IAAIV,EAAoB/7B,KAAMy8B,WACxE,kBAAqB,IAAIV,EAAoB/7B,QAGpD21B,EAAgBF,EAAO,YACvBmH,GAAwB,EACxBD,EAAoBb,EAAS3/B,UAC7B0gC,EAAiBF,EAAkBd,KAClCc,EAAkB,eAClBX,GAAWW,EAAkBX,GAC9BU,GAAmBpB,IAA0BuB,GAAkBL,EAAmBR,GAClFc,EAA4B,SAARrH,GAAkBkH,EAAkBI,SAA4BF,KAIpFC,IACFT,EAA2BjB,GAAe0B,EAAkBzgC,KAAK,IAAIy/B,IACjEb,KAAsB/+B,OAAOC,WAAakgC,EAAyBroB,OASrEgnB,GAAeqB,EAA0B1G,GAAe,GAAM,GACjDwG,GAAUxG,GAAiB4F,KAzCjC,UA8CPS,GAAqBa,GA9Cd,WA8CgCA,EAAep5B,OACxDm5B,GAAwB,EACxBF,EAAkB,kBAA2BG,EAAexgC,KAAK2D,QAI/D,GAAwB28B,EAAkBd,MAAca,GAC1DlM,EAA4BmM,EAAmBd,GAAUa,GAE3DP,GAAU1G,GAAQiH,EAGdV,KACFM,EAAU,CACRhpB,OAAQkpB,EA5DD,UA6DPl5B,KAAM24B,EAASS,EAAkBF,EA9D5B,QA+DLO,QAASP,EA7DD,YA+DNN,EAAQ,IAAKK,KAAOD,GAClBhB,KAA0BsB,GAA2BL,KAAOI,GAC9DvC,GAASuC,EAAmBJ,EAAKD,EAAQC,SAEtC5B,EAAE,CAAEhL,OAAQ8F,EAAMjV,OAAO,EAAM2P,OAAQmL,IAA0BsB,GAAyBN,UAG5FA,GChFL9D,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAFtB,kBAcJiG,GAAe/gC,MAAO,SAAS,SAAUghC,EAAUC,GAClE1E,GAAiBx4B,KAAM,CACrBk3B,KAhBiB,iBAiBjBvH,OAAQhC,EAAgBsP,GACxBrY,MAAO,EACPsY,KAAMA,OAIP,eACGjG,EAAQyB,GAAiB14B,MACzB2vB,EAASsH,EAAMtH,OACfuN,EAAOjG,EAAMiG,KACbtY,EAAQqS,EAAMrS,eACb+K,GAAU/K,GAAS+K,EAAO3yB,QAC7Bi6B,EAAMtH,YAASlwB,EACR,CAAEqC,WAAOrC,EAAW09B,MAAM,IAEvB,QAARD,EAAuB,CAAEp7B,MAAO8iB,EAAOuY,MAAM,GACrC,UAARD,EAAyB,CAAEp7B,MAAO6tB,EAAO/K,GAAQuY,MAAM,GACpD,CAAEr7B,MAAO,CAAC8iB,EAAO+K,EAAO/K,IAASuY,MAAM,KAC7C,aAKOC,UAAYjB,GAAUlgC,MC7ChC,ICMI05B,GAAgBH,GAAgB,eAEpC,IAAK,IAAI6H,KDRQ,CACfC,YAAa,EACbC,oBAAqB,EACrBC,aAAc,EACdC,eAAgB,EAChBC,YAAa,EACbC,cAAe,EACfC,aAAc,EACdC,qBAAsB,EACtBC,SAAU,EACVC,kBAAmB,EACnBC,eAAgB,EAChBC,gBAAiB,EACjBC,kBAAmB,EACnBC,UAAW,EACXC,cAAe,EACfC,aAAc,EACdC,SAAU,EACVC,iBAAkB,EAClBC,OAAQ,EACRC,YAAa,EACbC,cAAe,EACfC,cAAe,EACfC,eAAgB,EAChBC,aAAc,EACdC,cAAe,EACfC,iBAAkB,EAClBC,iBAAkB,EAClBC,eAAgB,EAChBC,iBAAkB,EAClBC,cAAe,EACfC,UAAW,GCvB6B,KACpCC,GAAanT,EAAOmR,IACpBiC,GAAsBD,IAAcA,GAAWljC,UAC/CmjC,IAAuBvS,GAAQuS,MAAyB3J,IAC1DnF,EAA4B8O,GAAqB3J,GAAe0H,IAElElB,GAAUkB,IAAmBlB,GAAUlgC,MCZzC,IAAI+0B,GAAe,SAAUuO,UACpB,SAAUrO,EAAOpgB,OAIlB0uB,EAAOnlB,EAHPolB,EAAIhR,OAAOvB,EAAuBgE,IAClCwO,EAAW5O,EAAUhgB,GACrB6uB,EAAOF,EAAEziC,cAET0iC,EAAW,GAAKA,GAAYC,EAAaJ,EAAoB,QAAK9/B,GACtE+/B,EAAQC,EAAEG,WAAWF,IACN,OAAUF,EAAQ,OAAUE,EAAW,IAAMC,IACtDtlB,EAASolB,EAAEG,WAAWF,EAAW,IAAM,OAAUrlB,EAAS,MAC1DklB,EAAoBE,EAAEjZ,OAAOkZ,GAAYF,EACzCD,EAAoBE,EAAEv8B,MAAMw8B,EAAUA,EAAW,GAA+BrlB,EAAS,OAAlCmlB,EAAQ,OAAU,IAA0B,QCdzGhZ,GDkBa,CAGfqZ,OAAQ7O,IAAa,GAGrBxK,OAAQwK,IAAa,ICxB+BxK,OAKlDgS,GAAmBC,GAAoBz0B,IACvC00B,GAAmBD,GAAoB1B,UAFrB,mBAMtBiG,GAAevO,OAAQ,UAAU,SAAUwO,GACzCzE,GAAiBx4B,KAAM,CACrBk3B,KARkB,kBASlBjgB,OAAQwX,OAAOwO,GACfrY,MAAO,OAIR,eAIGkb,EAHA7I,EAAQyB,GAAiB14B,MACzBiX,EAASggB,EAAMhgB,OACf2N,EAAQqS,EAAMrS,aAEdA,GAAS3N,EAAOja,OAAe,CAAE8E,WAAOrC,EAAW09B,MAAM,IAC7D2C,EAAQtZ,GAAOvP,EAAQ2N,GACvBqS,EAAMrS,OAASkb,EAAM9iC,OACd,CAAE8E,MAAOg+B,EAAO3C,MAAM,OCvB/B,IAAItB,GAAWrG,GAAgB,eAEd,SAAU1J,MACfrsB,MAANqsB,EAAiB,OAAOA,EAAG+P,KAC1B/P,EAAG,eACHqQ,GAAUpP,GAAQjB,QCNR,SAAUA,OACrBiU,EAAiBC,GAAkBlU,MACV,mBAAlBiU,QACH/S,UAAUyB,OAAO3C,GAAM,2BACtB+C,EAASkR,EAAe1jC,KAAKyvB,OCDtC,CAAE6D,OAAQ,SAAUG,MAAM,EAAMS,MAAOhD,GAAe,CACtDyH,OAAQA,KCJV,IAAI94B,GAAS+zB,EAAK/zB,UAED,SAAgBwxB,EAAGtF,UAC3BlsB,GAAO84B,OAAOtH,EAAGtF,OCJTwI,MCAAA,ECejB,OAfA,SAAyBvvB,EAAK4B,EAAKnB,UAC7BmB,KAAO5B,EACT4+B,GAAuB5+B,EAAK4B,EAAK,CAC/BnB,MAAOA,EACP6qB,YAAY,EACZE,cAAc,EACdC,UAAU,IAGZzrB,EAAI4B,GAAOnB,EAGNT,GCTLsxB,GAAsBvG,GAAM,WAAc8T,GAAW,QAIvD,CAAEvQ,OAAQ,SAAUG,MAAM,EAAMK,OAAQwC,IAAuB,CAC/DrvB,KAAM,SAAcwoB,UACXoU,GAAWvd,GAASmJ,OCR/B,ICI2BqU,MDJVlQ,EAAK/zB,OAAOoH,QEDZ,gDCCb88B,GAAa,IAAMC,GAAc,IACjCC,GAAQh4B,OAAO,IAAM83B,GAAaA,GAAa,KAC/CG,GAAQj4B,OAAO83B,GAAaA,GAAa,MAGzCpP,GAAe,SAAUgG,UACpB,SAAU9F,OACXja,EAASwX,OAAOvB,EAAuBgE,WAChC,EAAP8F,IAAU/f,EAASA,EAAO1Q,QAAQ+5B,GAAO,KAClC,EAAPtJ,IAAU/f,EAASA,EAAO1Q,QAAQg6B,GAAO,KACtCtpB,OAIM,CAGfupB,MAAOxP,GAAa,GAGpByP,IAAKzP,GAAa,GAGlB0P,KAAM1P,GAAa,ICxBjB2P,GAAQnW,GAAoCkW,OAK9C,CAAE/Q,OAAQ,SAAUnP,OAAO,EAAM2P,QHARgQ,GGAuC,OHCzD/T,GAAM,mBACFiU,GAAYF,OANf,OAAA,MAMqCA,OAAyBE,GAAYF,IAAa18B,OAAS08B,QGF7B,CAC3EO,KAAM,kBACGC,GAAM3gC,SCPjB,OAAiB,SAAU4gC,UAClB3Q,EAAK2Q,EAAc,iBCAXC,GAAa,UAAUH,KCDpCI,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAG4U,WACQ,iBAAP5U,GAAmBA,IAAOgV,IAClChV,aAAc2C,QAAUsS,IAAQD,GAAgBJ,KAAQA,GAAOK,MCJtD,SAAUZ,EAAatP,OAClCoB,EAAS,GAAGkO,WACPlO,GAAU7F,GAAM,WAEvB6F,EAAO51B,KAAK,KAAMw0B,GAAY,iBAAoB,GAAM,OCHxDxE,GAAiBnwB,OAAOmwB,eACxB2U,GAAQ,GAERC,GAAU,SAAUnV,SAAYA,MAEnB,SAAUqU,EAAahR,MAClCrB,EAAIkT,GAAOb,GAAc,OAAOa,GAAMb,GACrChR,IAASA,EAAU,QACpB8C,EAAS,GAAGkO,GACZe,IAAYpT,EAAIqB,EAAS,cAAeA,EAAQ+R,UAChDC,EAAYrT,EAAIqB,EAAS,GAAKA,EAAQ,GAAK8R,GAC3CG,EAAYtT,EAAIqB,EAAS,GAAKA,EAAQ,QAAK1vB,SAExCuhC,GAAMb,KAAiBlO,IAAW7F,GAAM,cACzC8U,IAAc3T,EAAa,OAAO,MAClCE,EAAI,CAAEzwB,QAAS,GAEfkkC,EAAW7U,GAAeoB,EAAG,EAAG,CAAEd,YAAY,EAAM5iB,IAAKk3B,KACxDxT,EAAE,GAAK,EAEZwE,EAAO51B,KAAKoxB,EAAG0T,EAAWC,OCvB1B/I,GAAW7N,GAAwC3W,QAInDwtB,GAAgBC,GAAoB,WACpCC,GAAiBC,GAAwB,cAI1BH,IAAkBE,GAEjC,GAAG1tB,QAFgD,SAAiBgkB,UAC/DQ,GAASr4B,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCLxE,CAAEkwB,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ,GAAGtc,SAAWA,IAAW,CACjEA,QAASA,KCJX,ICGIvN,GAAOyiB,MDHM8X,GAAa,SAAShtB,QEAnC4tB,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGjY,eACNiY,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe5tB,SAE1E6tB,GAAarkC,eAAe0vB,GAAQjB,IAAOjY,GAAUktB,MCZ3C5O,GAAW,YAAa,cAAgB,GFCrDwP,GAAUzV,EAAOyV,QACjBC,GAAWD,IAAWA,GAAQC,SAC9BC,GAAKD,IAAYA,GAASC,GAG1BA,GAEF9Y,IADAziB,GAAQu7B,GAAGh3B,MAAM,MACD,GAAKvE,GAAM,GAClBw7B,OACTx7B,GAAQw7B,GAAUx7B,MAAM,iBACVA,GAAM,IAAM,MACxBA,GAAQw7B,GAAUx7B,MAAM,oBACbyiB,GAAUziB,GAAM,IAI/B,OAAiByiB,KAAYA,GGfzBoO,GAAU3B,GAAgB,cAEb,SAAU2K,UAIlB4B,IAAc,KAAO3V,GAAM,eAC5B/lB,EAAQ,UACMA,EAAMixB,YAAc,IAC1BH,IAAW,iBACd,CAAE6K,IAAK,IAE2B,IAApC37B,EAAM85B,GAAa8B,SAASD,QCdnCE,GAAO1X,GAAwC7tB,IAI/CwlC,GAAsBC,GAA6B,OAEnDb,GAAiBC,GAAwB,SAK3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnF5kC,IAAK,SAAak7B,UACTqK,GAAKliC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCZxE,OAAiBohC,GAAa,SAASlkC,ICDnC8kC,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGnvB,WACNmvB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe9kC,IAAOA,GAAMokC,GCL1FL,GAAOlW,GAAoCkW,KAG3C2B,GAAYnW,EAAOziB,SACnB64B,GAAM,iBACqC,IAAlCD,GAAUhC,GAAc,OAAmD,KAApCgC,GAAUhC,GAAc,QAIlD,SAAkBppB,EAAQsrB,OAC9C9C,EAAIiB,GAAKjS,OAAOxX,WACborB,GAAU5C,EAAI8C,IAAU,IAAOD,GAAIv7B,KAAK04B,GAAK,GAAK,MACvD4C,KCRF,CAAEnW,QAAQ,EAAMiE,OAAQ1mB,UAAY+4B,IAA0B,CAC9D/4B,SAAU+4B,KCHZ,OAAiBvS,EAAKxmB,SCAlB8iB,GAAuB/B,EAAsDyD,EAG7E+C,GAAe,SAAUyR,UACpB,SAAU3W,WAMX7oB,EALAwqB,EAAIE,EAAgB7B,GACpBxoB,EAAOuuB,GAAWpE,GAClBzwB,EAASsG,EAAKtG,OACdF,EAAI,EACJyb,EAAS,GAENvb,EAASF,GACdmG,EAAMK,EAAKxG,KACNywB,IAAehB,GAAqBlwB,KAAKoxB,EAAGxqB,IAC/CsV,EAAOtb,KAAKwlC,EAAa,CAACx/B,EAAKwqB,EAAExqB,IAAQwqB,EAAExqB,WAGxCsV,ICnBPmqB,GDuBa,CAGf3F,QAAS/L,IAAa,GAGtB1d,OAAQ0d,IAAa,IC7B+B1d,SAIpD,CAAEqc,OAAQ,SAAUG,MAAM,GAAQ,CAClCxc,OAAQ,SAAgBma,UACfiV,GAAQjV,MCJnB,OAAiBwC,EAAK/zB,OAAOoX,OCDzBqvB,GAAUnY,GAAwCyN,OAIlDkK,GAAsBC,GAA6B,UAEnDb,GAAiBC,GAAwB,YAK3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnFtJ,OAAQ,SAAgBJ,UACf8K,GAAQ3iC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCZ3E,OAAiBohC,GAAa,SAAS5I,OCDnCwJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmM,cACNnM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAexJ,OAAUA,GAAS8I,GCOhG6B,GAAuBpN,GAAgB,sBAOvCqN,GAA+Bd,IAAc,KAAO3V,GAAM,eACxD/lB,EAAQ,UACZA,EAAMu8B,KAAwB,EACvBv8B,EAAM+H,SAAS,KAAO/H,KAG3By8B,GAAkBV,GAA6B,UAE/CW,GAAqB,SAAUtV,OAC5BnxB,EAASmxB,GAAI,OAAO,MACrBuV,EAAavV,EAAEmV,gBACGnjC,IAAfujC,IAA6BA,EAAajnC,GAAQ0xB,MAQzD,CAAEkC,OAAQ,QAASnP,OAAO,EAAM2P,QALpB0S,KAAiCC,IAKK,CAClD10B,OAAQ,SAAgBrL,OAIlBjG,EAAG6d,EAAG3d,EAAQiD,EAAKyV,EAHnB+X,EAAI9K,GAAS3iB,MACbijC,EAAIjL,GAAmBvK,EAAG,GAC1Btf,EAAI,MAEHrR,GAAK,EAAGE,EAASlB,UAAUkB,OAAQF,EAAIE,EAAQF,OAClD4Y,GAAW,IAAP5Y,EAAW2wB,EAAI3xB,UAAUgB,GACzBimC,GAAmBrtB,GAAI,IAErBvH,GADJlO,EAAMoxB,EAAS3b,EAAE1Y,SAlCF,iBAmCiB,MAAMgwB,UAlCT,sCAmCxBrS,EAAI,EAAGA,EAAI1a,EAAK0a,IAAKxM,IAASwM,KAAKjF,GAAGgd,GAAeuQ,EAAG90B,EAAGuH,EAAEiF,QAC7D,IACDxM,GAtCW,iBAsCY,MAAM6e,UArCJ,kCAsC7B0F,GAAeuQ,EAAG90B,IAAKuH,UAG3ButB,EAAEjmC,OAASmR,EACJ80B,KCtDX,OAAiBpC,GAAa,SAASzyB,OCDnCqzB,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG1d,cACN0d,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAerzB,OAAUA,GAAS2yB,KCDlG,CAAEpR,OAAQ,QAASG,MAAM,GAAQ,CACjC/zB,QAASA,KCHX,OAAiBk0B,EAAKh0B,MAAMF,WCDX60B,GCUjB,OAVA,SAA4Bh0B,MACtBsmC,GAAetmC,GAAM,KAClB,IAAIE,EAAI,EAAGqmC,EAAO,IAAIlnC,MAAMW,EAAII,QAASF,EAAIF,EAAII,OAAQF,IAC5DqmC,EAAKrmC,GAAKF,EAAIE,UAGTqmC,OCLM,SAAUpQ,EAAUl2B,EAAIiF,EAAOshC,cAErCA,EAAUvmC,EAAGgyB,EAAS/sB,GAAO,GAAIA,EAAM,IAAMjF,EAAGiF,GAEvD,MAAOqqB,OACHkX,EAAetQ,EAAQ,kBACNtzB,IAAjB4jC,GAA4BxU,EAASwU,EAAahnC,KAAK02B,IACrD5G,ICPN0P,GAAWrG,GAAgB,YAC3BiM,GAAiBxlC,MAAME,aAGV,SAAU2vB,eACXrsB,IAAPqsB,IAAqBqQ,GAAUlgC,QAAU6vB,GAAM2V,GAAe5F,MAAc/P,ICNjF+P,GAAWrG,GAAgB,YAC3B8N,IAAe,EAEnB,QACMC,GAAS,EACTC,GAAqB,CACvBxvB,KAAM,iBACG,CAAEmpB,OAAQoG,cAET,WACRD,IAAe,IAGnBE,GAAmB3H,IAAY,kBACtB77B,MAGT/D,MAAMoE,KAAKmjC,IAAoB,iBAAoB,KACnD,MAAOrX,IAET,IClBIsX,IDkBa,SAAUvsB,EAAMwsB,OAC1BA,IAAiBJ,GAAc,OAAO,MACvCK,GAAoB,UAElB7U,EAAS,GACbA,EAAO+M,IAAY,iBACV,CACL7nB,KAAM,iBACG,CAAEmpB,KAAMwG,GAAoB,MAIzCzsB,EAAK4X,GACL,MAAO3C,WACFwX,EChCkBC,EAA4B,SAAUC,GAC/D5nC,MAAMoE,KAAKwjC,QAKX,CAAElU,OAAQ,QAASG,MAAM,EAAMK,OAAQsT,IAAuB,CAC9DpjC,KCAe,SAAcyjC,OAQzB9mC,EAAQub,EAAQwrB,EAAMhR,EAAU/e,EAAMlS,EAPtC2rB,EAAI9K,GAASmhB,GACbzM,EAAmB,mBAARr3B,KAAqBA,KAAO/D,MACvC+nC,EAAkBloC,UAAUkB,OAC5BinC,EAAQD,EAAkB,EAAIloC,UAAU,QAAK2D,EAC7CykC,OAAoBzkC,IAAVwkC,EACVlE,EAAiBC,GAAkBvS,GACnC7I,EAAQ,KAERsf,IAASD,EAAQ5T,EAAK4T,EAAOD,EAAkB,EAAIloC,UAAU,QAAK2D,EAAW,IAE3DA,MAAlBsgC,GAAiC1I,GAAKp7B,OAASkoC,GAAsBpE,OAUvExnB,EAAS,IAAI8e,EADbr6B,EAASq0B,EAAS5D,EAAEzwB,SAEdA,EAAS4nB,EAAOA,IACpB9iB,EAAQoiC,EAAUD,EAAMxW,EAAE7I,GAAQA,GAAS6I,EAAE7I,GAC7C8N,GAAena,EAAQqM,EAAO9iB,YAXhCkS,GADA+e,EAAWgN,EAAe1jC,KAAKoxB,IACfzZ,KAChBuE,EAAS,IAAI8e,IACL0M,EAAO/vB,EAAK3X,KAAK02B,IAAWoK,KAAMvY,IACxC9iB,EAAQoiC,EAAUE,GAA6BrR,EAAUkR,EAAO,CAACF,EAAKjiC,MAAO8iB,IAAQ,GAAQmf,EAAKjiC,MAClG4wB,GAAena,EAAQqM,EAAO9iB,UAUlCyW,EAAOvb,OAAS4nB,EACTrM,KCnCT,OAAiB0X,EAAKh0B,MAAMoE,KCAxBw7B,GAAWrG,GAAgB,eAEd,SAAU1J,OACrB2B,EAAIvxB,OAAO4vB,eACQrsB,IAAhBguB,EAAEoO,KACJ,eAAgBpO,GAEhB0O,GAAU9+B,eAAe0vB,GAAQU,KCHxC,OAJA,SAA0B4W,MACpBC,GAAYpoC,OAAOmoC,KAAmD,uBAAzCnoC,OAAOC,UAAUC,SAASC,KAAKgoC,GAAgC,OAAOE,GAAYF,ICDrH,OAJA,iBACQ,IAAIrX,UAAU,oDCStB,OAJA,SAA4BpwB,UACnB4nC,GAAkB5nC,IAAQ6nC,GAAgB7nC,IAAQ8nC,MCKvDvC,GAAsBC,GAA6B,SACnDb,GAAiBC,GAAwB,QAAS,CAAEN,WAAW,IAAS,IAAM,IAE9E/J,GAAU3B,GAAgB,WAC1BmP,GAAc,GAAGzhC,MACjBoC,GAAM9D,KAAK8D,MAKb,CAAEqqB,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnFr+B,MAAO,SAAes9B,EAAOC,OAMvBmE,EAAarsB,EAAQpK,EALrBsf,EAAIE,EAAgB3tB,MACpBhD,EAASq0B,EAAS5D,EAAEzwB,QACpB2d,EAAI2W,GAAgBkP,EAAOxjC,GAC3B6nC,EAAMvT,QAAwB7xB,IAARghC,EAAoBzjC,EAASyjC,EAAKzjC,MAGxDjB,GAAQ0xB,KAGgB,mBAF1BmX,EAAcnX,EAAE6J,cAEyBsN,IAAgB3oC,QAASF,GAAQ6oC,EAAYzoC,WAE3EG,EAASsoC,IAEE,QADpBA,EAAcA,EAAYzN,OACAyN,OAAcnlC,GAHxCmlC,OAAcnlC,EAKZmlC,IAAgB3oC,YAAyBwD,IAAhBmlC,UACpBD,GAAYtoC,KAAKoxB,EAAG9S,EAAGkqB,OAGlCtsB,EAAS,SAAqB9Y,IAAhBmlC,EAA4B3oC,MAAQ2oC,GAAat/B,GAAIu/B,EAAMlqB,EAAG,IACvExM,EAAI,EAAGwM,EAAIkqB,EAAKlqB,IAAKxM,IAASwM,KAAK8S,GAAGiF,GAAena,EAAQpK,EAAGsf,EAAE9S,WACvEpC,EAAOvb,OAASmR,EACToK,KC3CX,OAAiBsoB,GAAa,SAAS39B,MCDnCu+B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG5oB,aACN4oB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAev+B,MAASA,GAAQ69B,GCA9FpO,GAAsBvG,GAAM,WAAc0Y,GAAqB,QAIjE,CAAEnV,OAAQ,SAAUG,MAAM,EAAMK,OAAQwC,GAAqBpC,MAAO8K,IAA4B,CAChGD,eAAgB,SAAwBtP,UAC/BgZ,GAAqBniB,GAASmJ,OCTzC,OAAiBmE,EAAK/zB,OAAOk/B,kBCDZxK,GCAbmU,GAAWva,GAAuC9gB,QAIlDs7B,GAAgB,GAAGt7B,QAEnBu7B,KAAkBD,IAAiB,EAAI,CAAC,GAAGt7B,QAAQ,GAAI,GAAK,EAC5D23B,GAAgBC,GAAoB,WACpCC,GAAiBC,GAAwB,UAAW,CAAEN,WAAW,IAAS,MAI5E,CAAEvR,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ8U,KAAkB5D,KAAkBE,IAAkB,CAC9F73B,QAAS,SAAiBw7B,UACjBD,GAEHD,GAAcnpC,MAAMmE,KAAMlE,YAAc,EACxCipC,GAAS/kC,KAAMklC,EAAeppC,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MChB5E,OAAiBohC,GAAa,SAASn3B,QCDnC+3B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGpiB,eACNoiB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe/3B,QAAWA,GAAUq3B,MCJrFnQ,GCObuU,GAAejpC,OAAOkpC,OACtB/Y,GAAiBnwB,OAAOmwB,mBAIV8Y,IAAgB/Y,GAAM,cAElCmB,GAQiB,IARF4X,GAAa,CAAE/nC,EAAG,GAAK+nC,GAAa9Y,GAAe,GAAI,IAAK,CAC7EM,YAAY,EACZ5iB,IAAK,WACHsiB,GAAersB,KAAM,IAAK,CACxB8B,MAAO,EACP6qB,YAAY,OAGd,CAAEvvB,EAAG,KAAMA,EAAS,OAAO,MAE3B6lC,EAAI,GACJoC,EAAI,GAEJ3L,EAAS7G,gBAEboQ,EAAEvJ,GAAU,EADG,uBAEN7uB,MAAM,IAAIgJ,SAAQ,SAAUyxB,GAAOD,EAAEC,GAAOA,KACf,GAA/BH,GAAa,GAAIlC,GAAGvJ,IAHZ,wBAG4B7H,GAAWsT,GAAa,GAAIE,IAAIliC,KAAK,OAC7E,SAAgBwsB,EAAQ9J,WACvBiM,EAAInP,GAASgN,GACbqU,EAAkBloC,UAAUkB,OAC5B4nB,EAAQ,EACRsN,EAAwBG,GAA4BpE,EACpD1B,EAAuByB,EAA2BC,EAC/C+V,EAAkBpf,WAKnB3hB,EAJAw8B,EAAIxS,EAAcnxB,UAAU8oB,MAC5BthB,EAAO4uB,EAAwBL,GAAW4N,GAAGrxB,OAAO8jB,EAAsBuN,IAAM5N,GAAW4N,GAC3FziC,EAASsG,EAAKtG,OACd+W,EAAI,EAED/W,EAAS+W,GACd9Q,EAAMK,EAAKyQ,KACNwZ,IAAehB,EAAqBlwB,KAAKojC,EAAGx8B,KAAM6uB,EAAE7uB,GAAOw8B,EAAEx8B,WAE7D6uB,GACPqT,KC9CF,CAAExV,OAAQ,SAAUG,MAAM,EAAMK,OAAQj0B,OAAOkpC,SAAWA,IAAU,CACpEA,OAAQA,KCHV,OAAiBnV,EAAK/zB,OAAOkpC,OCC7B/K,GAAsB,YCCtB,OAAiBkL,GAA6BtX,EAAE,eCH/B2C,sCCEjByJ,GAAsB,wDCAtBA,GAAsB,eCAtBA,GAAsB,sBCAtBA,GAAsB,SCDtBA,GAAsB,YCCtBA,GAAsB,WCAtBA,GAAsB,UCAtBA,GAAsB,WCAtBA,GAAsB,SCAtBA,GAAsB,eCAtBA,GAAsB,eCAtBA,GAAsB,eCAtBW,GAAex5B,KAAM,QAAQ,GCC7Bw5B,GAAe9O,EAAOsZ,KAAM,QAAQ,eCgBpC,OAAiBvV,EAAK4C,OCjBtBwH,GAAsB,gBCAtBA,GAAsB,WCAtBA,GAAsB,cCAtBA,GAAsB,gBCDtBA,GAAsB,cCKtB,OAAiBzJ,8BCJR6U,EAAQpkC,UAIb5F,UAAiBgqC,EADI,mBAAZC,IAAsD,iBAArBC,GACf,SAAiBtkC,iBAC5BA,GAGW,SAAiBA,UACnCA,GAA0B,mBAAZqkC,IAA0BrkC,EAAIi2B,cAAgBoO,IAAWrkC,IAAQqkC,GAAQvpC,UAAY,gBAAkBkF,GAIzHokC,EAAQpkC,GAGjB5F,UAAiBgqC,KCdjB,OAJA,SAAyB7oC,MACnBsmC,GAAetmC,GAAM,OAAOA,GC+BlC,OA9BA,SAA+BA,EAAKE,MAC5BwnC,GAAYpoC,OAAOU,KAAiD,uBAAxCV,OAAOC,UAAUC,SAASC,KAAKO,QAI7DgpC,EAAO,GACPC,GAAK,EACLxmC,GAAK,EACLymC,OAAKrmC,UAGF,IAA4BsmC,EAAxBtlC,EAAKulC,GAAappC,KAAYipC,GAAME,EAAKtlC,EAAGuT,QAAQmpB,QAC3DyI,EAAK3oC,KAAK8oC,EAAGjkC,QAEThF,GAAK8oC,EAAK5oC,SAAWF,GAHyC+oC,GAAK,IAKzE,MAAOI,GACP5mC,GAAK,EACLymC,EAAKG,cAGEJ,GAAsB,MAAhBplC,EAAE,QAAoBA,EAAE,oBAE/BpB,EAAI,MAAMymC,UAIXF,IC3BT,OAJA,iBACQ,IAAI5Y,UAAU,yDCStB,OAJA,SAAwBpwB,EAAKE,UACpBopC,GAAetpC,IAAQupC,GAAqBvpC,EAAKE,IAAMspC,QCH9D,CAAEzW,OAAQ,OAAQG,MAAM,GAAQ,CAChCjb,IAAK,kBACI,IAAInY,MAAO4C,aCHtB,OAAiB2wB,EAAKvzB,KAAKmY,6IC4C3B,SAASwxB,GAAmBC,SA+B5B,mBACQC,EAAOC,KAETC,EAAKF,EAAK,KACVG,EAAKH,EAAK,KACVI,EAAKJ,EAAK,KAELzpC,EAAI,EAAGA,EAAIhB,UAAKkB,OAAQF,KAC/B2pC,GAAMF,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP2pC,GAAM,IAERC,GAAMH,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP4pC,GAAM,IAERC,GAAMJ,EAAUzpC,uBAAAA,mBAAAA,KACP,IACP6pC,GAAM,SAIH,CAACF,EAAIC,EAAIC,GApDGC,CAASN,aAAvBG,OAAIC,OAAIC,OACTpyB,EAAI,EAEF+e,EAAc,eACZvzB,EAAI,QAAU0mC,EAAS,uBAAJlyB,SACzBkyB,EAAKC,EACLA,EAAKC,EACGA,EAAK5mC,GAAKwU,EAAQ,EAAJxU,WAGxBuzB,EAAOuT,OAAS,kBAAyB,WAAXvT,KAE9BA,EAAOwT,QAAU,kBACfxT,IAAyC,uBAAjB,QAAXA,IAAuB,IAEtCA,EAAOyT,UAAY,OACnBzT,EAAOgT,KAAOA,EACdhT,EAAOvK,QAAU,MAEVuK,EA+CT,SAASkT,SACHr4B,EAAI,kBAED,SAASoF,WACR0D,EAAS1D,EAAKnX,WACXU,EAAI,EAAGA,EAAIma,EAAOja,OAAQF,IAAK,KAElC0P,EAAI,oBADR2B,GAAK8I,EAAO2oB,WAAW9iC,IAGvB0P,GADA2B,EAAI3B,IAAM,EAGV2B,GADA3B,GAAK2B,KACK,EAEVA,GAAS,YADT3B,GAAK2B,UAGY,wBAAXA,IAAM;;;;;;;;;;;;;;;;;;;;;;;;;;sCA7FMm4B,2BAAAA,yBACfD,GAAmBC,EAAKtpC,OAASspC,EAAO,CAACU,ytSC9B3C,SAASC,GAAgBC,OAEzB,IAAIC,KAAeD,EAClBA,EAAc7pC,eAAe8pC,KAC/BD,EAAcC,GAAaC,UAAYF,EAAcC,GAAaE,KAClEH,EAAcC,GAAaE,KAAO,IAYjC,SAASC,GAAgBJ,OAEzB,IAAIC,KAAeD,KAClBA,EAAc7pC,eAAe8pC,IAC3BD,EAAcC,GAAaC,UAAW,KACnC,IAAItqC,EAAI,EAAGA,EAAIoqC,EAAcC,GAAaC,UAAUpqC,OAAQF,IAC/DoqC,EAAcC,GAAaC,UAAUtqC,GAAGyqC,WAAWC,YAAYN,EAAcC,GAAaC,UAAUtqC,IAEtGoqC,EAAcC,GAAaC,UAAY,IAUxC,SAASK,GAAcP,GAC5BD,GAAgBC,GAChBI,GAAgBJ,GAChBD,GAAgBC,GAaX,SAASQ,GAAcP,EAAaD,EAAeS,OACpDC,SAEAV,EAAc7pC,eAAe8pC,GAE3BD,EAAcC,GAAaC,UAAUpqC,OAAS,GAChD4qC,EAAUV,EAAcC,GAAaC,UAAU,GAC/CF,EAAcC,GAAaC,UAAUjiB,UAIrCyiB,EAAUxa,SAASya,gBAAgB,6BAA8BV,GACjEQ,EAAahT,YAAYiT,KAK3BA,EAAUxa,SAASya,gBAAgB,6BAA8BV,GACjED,EAAcC,GAAe,CAACE,KAAM,GAAID,UAAW,IACnDO,EAAahT,YAAYiT,IAE3BV,EAAcC,GAAaE,KAAKpqC,KAAK2qC,GAC9BA,EAcF,SAASE,GAAcX,EAAaD,EAAea,EAAcC,OAClEJ,SAEAV,EAAc7pC,eAAe8pC,GAE3BD,EAAcC,GAAaC,UAAUpqC,OAAS,GAChD4qC,EAAUV,EAAcC,GAAaC,UAAU,GAC/CF,EAAcC,GAAaC,UAAUjiB,UAIrCyiB,EAAUxa,SAASE,cAAc6Z,QACZ1nC,IAAjBuoC,EACFD,EAAaC,aAAaJ,EAASI,GAGnCD,EAAapT,YAAYiT,KAM7BA,EAAUxa,SAASE,cAAc6Z,GACjCD,EAAcC,GAAe,CAACE,KAAM,GAAID,UAAW,SAC9B3nC,IAAjBuoC,EACFD,EAAaC,aAAaJ,EAASI,GAGnCD,EAAapT,YAAYiT,IAG7BV,EAAcC,GAAaE,KAAKpqC,KAAK2qC,GAC9BA,EAmBF,SAASK,GAAU99B,EAAGZ,EAAG2+B,EAAehB,EAAeS,EAAcQ,OACtErI,KACuB,UAAvBoI,EAAc1T,QAChBsL,EAAQ4H,GAAc,SAAUR,EAAeS,IACzCS,eAAe,KAAM,KAAMj+B,GACjC21B,EAAMsI,eAAe,KAAM,KAAM7+B,GACjCu2B,EAAMsI,eAAe,KAAM,IAAK,GAAMF,EAAcvI,SAGpDG,EAAQ4H,GAAc,OAAQR,EAAeS,IACvCS,eAAe,KAAM,IAAKj+B,EAAI,GAAM+9B,EAAcvI,MACxDG,EAAMsI,eAAe,KAAM,IAAK7+B,EAAI,GAAM2+B,EAAcvI,MACxDG,EAAMsI,eAAe,KAAM,QAASF,EAAcvI,MAClDG,EAAMsI,eAAe,KAAM,SAAUF,EAAcvI,YAGxBlgC,IAAzByoC,EAAcG,QAChBvI,EAAMsI,eAAe,KAAM,QAASF,EAAcG,QAEpDvI,EAAMsI,eAAe,KAAM,QAASF,EAAcI,UAAY,cAI1DH,EAAU,KACRI,EAAQb,GAAc,OAAQR,EAAeS,GAC7CQ,EAASK,UACXr+B,GAAQg+B,EAASK,SAGfL,EAASM,UACXl/B,GAAQ4+B,EAASM,SAEfN,EAAStU,UACX0U,EAAMG,YAAcP,EAAStU,SAG3BsU,EAASG,WACXC,EAAMH,eAAe,KAAM,QAASD,EAASG,UAAa,cAE5DC,EAAMH,eAAe,KAAM,IAAKj+B,GAChCo+B,EAAMH,eAAe,KAAM,IAAK7+B,UAG3Bu2B,EAeF,SAAS6I,GAASx+B,EAAGZ,EAAGq/B,EAAOC,EAAQP,EAAWpB,EAAeS,EAAcnT,MACtE,GAAVqU,EAAa,CACXA,EAAS,IAEXt/B,GADAs/B,IAAW,OAGTC,EAAOpB,GAAc,OAAOR,EAAeS,GAC/CmB,EAAKV,eAAe,KAAM,IAAKj+B,EAAI,GAAMy+B,GACzCE,EAAKV,eAAe,KAAM,IAAK7+B,GAC/Bu/B,EAAKV,eAAe,KAAM,QAASQ,GACnCE,EAAKV,eAAe,KAAM,SAAUS,GACpCC,EAAKV,eAAe,KAAM,QAASE,GAC/B9T,GACFsU,EAAKV,eAAe,KAAM,QAAS5T,IASlC,SAASuU,gBAEPC,UACDA,UAAUC,WAAaD,UAAUC,UAAUjsC,OACtCgsC,UAAUC,UAEVD,UAAUE,cAAgBF,UAAUG,UAAYH,UAAUI,iBAAmB,KAJ/D,KAOzB,MAAMjd,SACG,sLCnOX,OARA,SAAgCF,WACjB,IAATA,QACI,IAAIod,eAAe,oEAGpBpd,MCFQ4U,GAAa,SAAS9D,QCAnC0E,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGiR,eACNjR,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe1E,SAE1E2E,GAAarkC,eAAe0vB,GAAQjB,IAAOiR,GAAUgE,GCVxDlI,GAAa1G,GAAW,OAAQ,aAChCmX,GAAK,mBACLC,GAAM,oBACNC,GAAK,oBAELC,GAAM,SAAUnjC,EAAOkW,EAAQvF,OAC7ByyB,EAAOzyB,EAAOuP,OAAOhK,EAAS,GAC9BxI,EAAOiD,EAAOuP,OAAOhK,EAAS,UAC7B+sB,GAAIxiC,KAAKT,KAAWkjC,GAAGziC,KAAKiN,IAAWw1B,GAAGziC,KAAKT,KAAWijC,GAAIxiC,KAAK2iC,GAC/D,MAAQpjC,EAAMs5B,WAAW,GAAGxjC,SAAS,IACrCkK,GAGP41B,GAAS9P,GAAM,iBACqB,qBAA/ByM,GAAW,iBACY,cAAzBA,GAAW,aAGdA,IAEF8B,EAAE,CAAEhL,OAAQ,OAAQG,MAAM,EAAMK,OAAQ+L,IAAU,CAEhDtB,UAAW,SAAmB9O,EAAI+O,EAAUC,OACtCviB,EAASsgB,GAAWh9B,MAAM,KAAMC,iBACZ,iBAAVyc,EAAqBA,EAAOhS,QAAQ+iC,GAAIG,IAAOlxB,KCzB9DoxB,EAAKnE,OAAMmE,EAAKnE,KAAO,CAAE5K,UAAW4K,KAAK5K,YAG9C,OAAiB,SAAmB9O,EAAI+O,EAAUC,UACzC6O,EAAKnE,KAAK5K,UAAU/+B,MAAM,KAAMC,eCJxB+kC,GAAa,SAASvtB,OCAnCmuB,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGxY,cACNwY,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAenuB,QAE1EouB,GAAarkC,eAAe0vB,GAAQjB,IAAOxY,GAASytB,MCX1CF,GAAa,SAASv9B,KCAnCm+B,GAAiBxlC,MAAME,UAEvBulC,GAAe,CACjB9D,cAAc,EACdU,UAAU,MAGK,SAAUxS,OACrBiV,EAAMjV,EAAGxoB,YACNwoB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAen+B,MAE1Eo+B,GAAarkC,eAAe0vB,GAAQjB,IAAOxoB,GAAOy9B,GCZrD6I,GAAQpf,GAAwC7uB,KAIhD0lC,GAAgBC,GAAoB,QACpCC,GAAiBC,GAAwB,UAI3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7E5lC,KAAM,SAAck8B,UACX+R,GAAM5pC,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCVzE,OAAiBohC,GAAa,SAASllC,KCDnC8lC,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGnwB,YACNmwB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe9lC,KAAQA,GAAOolC,GCMhG,OARA,SAAoC9U,EAAM5vB,UACpCA,GAA2B,WAAlBopC,GAAQppC,IAAsC,mBAATA,EAI3CwtC,GAAsB5d,GAHpB5vB,MCJMu0B,KCGf,CAAEjB,OAAQ,SAAUG,MAAM,GAAQ,CAClC0L,eAAgBA,KCHlB,OAAiBvL,EAAK/zB,OAAOs/B,0CCCpBsO,EAAgBz/B,UACvB5O,UAAiBquC,EAAkBC,GAAyBC,GAAyB,SAAyB3/B,UACrGA,EAAEsxB,WAAaqO,GAAuB3/B,IAExCy/B,EAAgBz/B,GAGzB5O,UAAiBquC,QCTAlZ,8BCARqZ,EAAgB5/B,EAAG6/B,UAC1BzuC,UAAiBwuC,EAAkBF,IAA0B,SAAyB1/B,EAAG6/B,UACvF7/B,EAAEsxB,UAAYuO,EACP7/B,GAGF4/B,EAAgB5/B,EAAG6/B,GAG5BzuC,UAAiBwuC,KCQjB,OAfA,SAAmBE,EAAUC,MACD,mBAAfA,GAA4C,OAAfA,QAChC,IAAIpd,UAAU,sDAGtBmd,EAAShuC,UAAYkuC,GAAeD,GAAcA,EAAWjuC,UAAW,CACtEm7B,YAAa,CACXx1B,MAAOqoC,EACPrd,UAAU,EACVD,cAAc,KAGdud,GAAY5O,GAAe2O,EAAUC,ICTvCrjC,GAAO,GACPujC,GAAavjC,GAAKkF,KAGlBs+B,GAAqBne,GAAM,WAC7BrlB,GAAKkF,UAAKxM,MAGR+qC,GAAgBpe,GAAM,WACxBrlB,GAAKkF,KAAK,SAGRo1B,GAAgBC,GAAoB,UAMtC,CAAE3R,OAAQ,QAASnP,OAAO,EAAM2P,OAJrBoa,KAAuBC,KAAkBnJ,IAIF,CAClDp1B,KAAM,SAAcw+B,eACGhrC,IAAdgrC,EACHH,GAAWjuC,KAAKsmB,GAAS3iB,OACzBsqC,GAAWjuC,KAAKsmB,GAAS3iB,MAAO0uB,EAAU+b,OC1BlD,OAAiB5J,GAAa,SAAS50B,KCDnCw1B,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG7f,YACN6f,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAex1B,KAAQA,GAAO80B,OCJ9E3U,GAAM,kBACflwB,OAAOwuC,aAAaxuC,OAAOyuC,kBAAkB,8BCAlDte,EAAiB7B,EAA+CyD,EAIhE2c,EAAWpX,GAAI,QACfJ,EAAK,EAELsX,EAAexuC,OAAOwuC,cAAgB,kBACjC,GAGLG,EAAc,SAAU/e,GAC1BO,EAAeP,EAAI8e,EAAU,CAAE9oC,MAAO,CACpCgpC,SAAU,OAAQ1X,EAClB2X,SAAU,OAoCVC,EAAOvvC,UAAiB,CAC1BwvC,UAAU,EACVC,QAlCY,SAAUpf,EAAIkJ,OAErB14B,EAASwvB,GAAK,MAAoB,iBAANA,EAAiBA,GAAmB,iBAANA,EAAiB,IAAM,KAAOA,MACxFgC,EAAIhC,EAAI8e,GAAW,KAEjBF,EAAa5e,GAAK,MAAO,QAEzBkJ,EAAQ,MAAO,IAEpB6V,EAAY/e,UAELA,EAAG8e,GAAUE,UAwBtBK,YArBgB,SAAUrf,EAAIkJ,OACzBlH,EAAIhC,EAAI8e,GAAW,KAEjBF,EAAa5e,GAAK,OAAO,MAEzBkJ,EAAQ,OAAO,EAEpB6V,EAAY/e,UAELA,EAAG8e,GAAUG,UAatBK,SATa,SAAUtf,UACnBuf,IAAYL,EAAKC,UAAYP,EAAa5e,KAAQgC,EAAIhC,EAAI8e,IAAWC,EAAY/e,GAC9EA,IAUT0F,GAAWoZ,IAAY,8ECrDnBU,EAAS,SAAUC,EAAShzB,QACzBgzB,QAAUA,OACVhzB,OAASA,IAGF9c,UAAiB,SAAUooC,EAAUhnC,EAAImkB,EAAMwqB,EAAYC,OAEnE1Y,EAAU2Y,EAAQ9mB,EAAO5nB,EAAQub,EAAQvE,EAAM+vB,EAD/ChM,EAAgB1H,EAAKxzB,EAAImkB,EAAMwqB,EAAa,EAAI,MAGhDC,EACF1Y,EAAW8Q,MACN,IAEgB,mBADrB6H,EAAS1L,GAAkB6D,IACM,MAAM7W,UAAU,6BAE7CmX,GAAsBuH,GAAS,KAC5B9mB,EAAQ,EAAG5nB,EAASq0B,EAASwS,EAAS7mC,QAASA,EAAS4nB,EAAOA,QAClErM,EAASizB,EACLzT,EAAclJ,EAASkV,EAAOF,EAASjf,IAAQ,GAAImf,EAAK,IACxDhM,EAAc8L,EAASjf,MACbrM,aAAkB+yB,EAAQ,OAAO/yB,SACxC,IAAI+yB,GAAO,GAEtBvY,EAAW2Y,EAAOrvC,KAAKwnC,OAGzB7vB,EAAO+e,EAAS/e,OACP+vB,EAAO/vB,EAAK3X,KAAK02B,IAAWoK,SAEd,iBADrB5kB,EAAS6rB,GAA6BrR,EAAUgF,EAAegM,EAAKjiC,MAAO0pC,KAC1CjzB,GAAUA,aAAkB+yB,EAAQ,OAAO/yB,SACrE,IAAI+yB,GAAO,KAGdK,KAAO,SAAUpzB,UAChB,IAAI+yB,GAAO,EAAM/yB,WCzCT,SAAUuT,EAAI8Y,EAAanhC,QACpCqoB,aAAc8Y,SACZ5X,UAAU,cAAgBvpB,EAAOA,EAAO,IAAM,IAAM,qBACnDqoB,GCOPO,GAAiB7B,EAA+CyD,EAChEpa,GAAU+3B,GAAwC/3B,QAIlD2kB,GAAmBC,GAAoBz0B,IACvC6nC,GAAyBpT,GAAoB1B,aAEhC,SAAU+U,EAAkBC,EAASC,OAOhDpH,EANArN,GAA8C,IAArCuU,EAAiBpiC,QAAQ,OAClCuiC,GAAgD,IAAtCH,EAAiBpiC,QAAQ,QACnCwiC,EAAQ3U,EAAS,MAAQ,MACzBtI,EAAoB/C,EAAO4f,GAC3BK,EAAkBld,GAAqBA,EAAkB9yB,UACzDiwC,EAAW,MAGV7e,GAA2C,mBAArB0B,IACpBgd,GAAWE,EAAgBt4B,UAAYuY,GAAM,gBAAkB6C,GAAoB8N,UAAU/oB,WAK7F,CACL4wB,EAAcmH,GAAQ,SAAUpc,EAAQkU,GACtCrL,GAAiB6T,GAAW1c,EAAQiV,EAAakH,GAAmB,CAClE5U,KAAM4U,EACNQ,WAAY,IAAIrd,IAEFxvB,MAAZokC,GAAuB0I,GAAQ1I,EAAUlU,EAAOuc,GAAQvc,EAAQ4H,UAGlEmB,EAAmBmT,GAAuBC,GAE9Cj4B,GAAQ,CAAC,MAAO,QAAS,SAAU,UAAW,MAAO,MAAO,MAAO,OAAQ,SAAU,YAAY,SAAU0oB,OACrGiQ,EAAkB,OAAPjQ,GAAuB,OAAPA,EAC3BA,KAAO4P,KAAqBF,GAAkB,SAAP1P,IACzC/L,EAA4BoU,EAAYzoC,UAAWogC,GAAK,SAAUp/B,EAAGC,OAC/DkvC,EAAa5T,EAAiB14B,MAAMssC,eACnCE,GAAYP,IAAY3vC,EAASa,GAAI,MAAc,OAAPo/B,QAAe98B,MAC5D8Y,EAAS+zB,EAAW/P,GAAW,IAANp/B,EAAU,EAAIA,EAAGC,UACvCovC,EAAWxsC,KAAOuY,QAK/B0zB,GAAW5f,GAAeuY,EAAYzoC,UAAW,OAAQ,CACvD0wB,cAAc,EACd9iB,IAAK,kBACI2uB,EAAiB14B,MAAMssC,WAAW3M,aA5B7CiF,EAAcoH,EAAOS,eAAeV,EAASD,EAAkBvU,EAAQ2U,GACvEQ,GAAuBzB,UAAW,SAgCpCjQ,GAAe4J,EAAakH,GAAkB,GAAO,GAErDM,EAASN,GAAoBlH,EAC7BjK,EAAE,CAAEzO,QAAQ,EAAMiE,QAAQ,GAAQic,GAE7BH,GAASD,EAAOW,UAAU/H,EAAakH,EAAkBvU,GAEvDqN,MCrEQ,SAAUjV,EAAQiF,EAAKzF,OACjC,IAAIlsB,KAAO2xB,EACVzF,GAAWA,EAAQyd,QAAUjd,EAAO1sB,GAAM0sB,EAAO1sB,GAAO2xB,EAAI3xB,GAC3Dm3B,GAASzK,EAAQ1sB,EAAK2xB,EAAI3xB,GAAMksB,UAC9BQ,GCAPwH,GAAU3B,GAAgB,WCL1BnJ,GAAiB7B,EAA+CyD,EAShEid,GAAUU,GAA0CV,QAGpD1S,GAAmBC,GAAoBz0B,IACvC6nC,GAAyBpT,GAAoB1B,aAEhC,CACf0V,eAAgB,SAAUV,EAASD,EAAkBvU,EAAQ2U,OACvD7U,EAAI0U,GAAQ,SAAU/qB,EAAM6iB,GAC9BwI,GAAWrrB,EAAMqW,EAAGyU,GACpBtT,GAAiBxX,EAAM,CACrBkW,KAAM4U,EACNlnB,MAAOoQ,GAAO,MACdwK,WAAO//B,EACPotC,UAAMptC,EACNkgC,KAAM,IAEHpS,IAAavM,EAAK2e,KAAO,GACdlgC,MAAZokC,GAAuB0I,GAAQ1I,EAAU7iB,EAAKkrB,GAAQlrB,EAAMuW,MAG9DmB,EAAmBmT,GAAuBC,GAE1CgB,EAAS,SAAU9rB,EAAM/d,EAAKnB,OAG5BirC,EAAUnoB,EAFVqS,EAAQyB,EAAiB1X,GACzBgsB,EAAQC,EAASjsB,EAAM/d,UAGvB+pC,EACFA,EAAMlrC,MAAQA,GAGdm1B,EAAM4V,KAAOG,EAAQ,CACnBpoB,MAAOA,EAAQsmB,GAAQjoC,GAAK,GAC5BA,IAAKA,EACLnB,MAAOA,EACPirC,SAAUA,EAAW9V,EAAM4V,KAC3B74B,UAAMvU,EACNytC,SAAS,GAENjW,EAAMuI,QAAOvI,EAAMuI,MAAQwN,GAC5BD,IAAUA,EAAS/4B,KAAOg5B,GAC1Bzf,EAAa0J,EAAM0I,OAClB3e,EAAK2e,OAEI,MAAV/a,IAAeqS,EAAMrS,MAAMA,GAASooB,IACjChsB,GAGPisB,EAAW,SAAUjsB,EAAM/d,OAIzB+pC,EAHA/V,EAAQyB,EAAiB1X,GAEzB4D,EAAQsmB,GAAQjoC,MAEN,MAAV2hB,EAAe,OAAOqS,EAAMrS,MAAMA,OAEjCooB,EAAQ/V,EAAMuI,MAAOwN,EAAOA,EAAQA,EAAMh5B,QACzCg5B,EAAM/pC,KAAOA,EAAK,OAAO+pC,UAIjCG,GAAY9V,EAAEl7B,UAAW,CAGvBixC,MAAO,mBAEDnW,EAAQyB,EADD14B,MAEPuT,EAAO0jB,EAAMrS,MACbooB,EAAQ/V,EAAMuI,MACXwN,GACLA,EAAME,SAAU,EACZF,EAAMD,WAAUC,EAAMD,SAAWC,EAAMD,SAAS/4B,UAAOvU,UACpD8T,EAAKy5B,EAAMpoB,OAClBooB,EAAQA,EAAMh5B,KAEhBijB,EAAMuI,MAAQvI,EAAM4V,UAAOptC,EACvB8tB,EAAa0J,EAAM0I,KAAO,EAXnB3/B,KAYD2/B,KAAO,UAIT,SAAU18B,OAEdg0B,EAAQyB,EADD14B,MAEPgtC,EAAQC,EAFDjtC,KAEgBiD,MACvB+pC,EAAO,KACLh5B,EAAOg5B,EAAMh5B,KACb01B,EAAOsD,EAAMD,gBACV9V,EAAMrS,MAAMooB,EAAMpoB,OACzBooB,EAAME,SAAU,EACZxD,IAAMA,EAAK11B,KAAOA,GAClBA,IAAMA,EAAK+4B,SAAWrD,GACtBzS,EAAMuI,OAASwN,IAAO/V,EAAMuI,MAAQxrB,GACpCijB,EAAM4V,MAAQG,IAAO/V,EAAM4V,KAAOnD,GAClCnc,EAAa0J,EAAM0I,OAZd3/B,KAaC2/B,eACDqN,GAIbn5B,QAAS,SAAiBgkB,WAGpBmV,EAFA/V,EAAQyB,EAAiB14B,MACzB+3B,EAAgB1H,EAAKwH,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,EAAW,GAE/EutC,EAAQA,EAAQA,EAAMh5B,KAAOijB,EAAMuI,WACxCzH,EAAciV,EAAMlrC,MAAOkrC,EAAM/pC,IAAKjD,MAE/BgtC,GAASA,EAAME,SAASF,EAAQA,EAAMD,UAKjDjf,IAAK,SAAa7qB,WACPgqC,EAASjtC,KAAMiD,MAI5BkqC,GAAY9V,EAAEl7B,UAAWo7B,EAAS,CAEhCxtB,IAAK,SAAa9G,OACZ+pC,EAAQC,EAASjtC,KAAMiD,UACpB+pC,GAASA,EAAMlrC,OAGxBkC,IAAK,SAAaf,EAAKnB,UACdgrC,EAAO9sC,KAAc,IAARiD,EAAY,EAAIA,EAAKnB,KAEzC,CAEF+Y,IAAK,SAAa/Y,UACTgrC,EAAO9sC,KAAM8B,EAAkB,IAAVA,EAAc,EAAIA,EAAOA,MAGrDyrB,GAAalB,GAAegL,EAAEl7B,UAAW,OAAQ,CACnD4N,IAAK,kBACI2uB,EAAiB14B,MAAM2/B,QAG3BtI,GAETsV,UAAW,SAAUtV,EAAGyU,EAAkBvU,OACpC8V,EAAgBvB,EAAmB,YACnCwB,EAA6BzB,GAAuBC,GACpDyB,EAA2B1B,GAAuBwB,GAGtDrQ,GAAe3F,EAAGyU,GAAkB,SAAU7O,EAAUC,GACtD1E,GAAiBx4B,KAAM,CACrBk3B,KAAMmW,EACN1d,OAAQsN,EACRhG,MAAOqW,EAA2BrQ,GAClCC,KAAMA,EACN2P,UAAMptC,OAEP,mBACGw3B,EAAQsW,EAAyBvtC,MACjCk9B,EAAOjG,EAAMiG,KACb8P,EAAQ/V,EAAM4V,KAEXG,GAASA,EAAME,SAASF,EAAQA,EAAMD,gBAExC9V,EAAMtH,SAAYsH,EAAM4V,KAAOG,EAAQA,EAAQA,EAAMh5B,KAAOijB,EAAMA,MAAMuI,OAMjE,QAARtC,EAAuB,CAAEp7B,MAAOkrC,EAAM/pC,IAAKk6B,MAAM,GACzC,UAARD,EAAyB,CAAEp7B,MAAOkrC,EAAMlrC,MAAOq7B,MAAM,GAClD,CAAEr7B,MAAO,CAACkrC,EAAM/pC,IAAK+pC,EAAMlrC,OAAQq7B,MAAM,IAN9ClG,EAAMtH,YAASlwB,EACR,CAAEqC,WAAOrC,EAAW09B,MAAM,MAMlC5F,EAAS,UAAY,UAAWA,GAAQ,GD5K9B,SAAUuU,OACrBlH,EAAczS,GAAW2Z,GACzBzf,EAAiB0C,EAAqBd,EAEtCV,GAAeqX,IAAgBA,EAAYzN,KAC7C9K,EAAeuY,EAAazN,GAAS,CACnCtK,cAAc,EACd9iB,IAAK,kBAAqB/J,QCwK5BwtC,CAAW1B,SCjLEQ,GAAW,OAAO,SAAUmB,UACpC,kBAAwBA,EAAKztC,KAAMlE,UAAUkB,OAASlB,UAAU,QAAK2D,MAC3EiuC,ICFczd,EAAK0d,SCALrB,GAAW,OAAO,SAAUmB,UACpC,kBAAwBA,EAAKztC,KAAMlE,UAAUkB,OAASlB,UAAU,QAAK2D,MAC3EiuC,ICFczd,EAAK2d,2BCClBC,EAAW,SAAU56B,OAGnB66B,EAAK5xC,OAAOC,UACZ4xC,EAASD,EAAGzwC,eAEZu7B,EAA4B,mBAAX/F,OAAwBA,OAAS,GAClDmb,EAAiBpV,EAAQ7F,UAAY,aACrCkb,EAAsBrV,EAAQsV,eAAiB,kBAC/CC,EAAoBvV,EAAQwV,aAAe,yBAEtC9d,EAAK+d,EAASC,EAASriB,EAAMsiB,OAEhCC,EAAiBF,GAAWA,EAAQnyC,qBAAqBsyC,EAAYH,EAAUG,EAC/EC,EAAYxyC,OAAO84B,OAAOwZ,EAAeryC,WACzCwyC,EAAU,IAAIC,EAAQL,GAAe,WAIzCG,EAAUG,iBAqMcR,EAASpiB,EAAM0iB,OACnC1X,EA9KuB,wBAgLpB,SAAgBhF,EAAQlvB,MA9KT,cA+KhBk0B,QACI,IAAI7zB,MAAM,mCA/KE,cAkLhB6zB,EAA6B,IAChB,UAAXhF,QACIlvB,SAKD+rC,QAGTH,EAAQ1c,OAASA,EACjB0c,EAAQ5rC,IAAMA,IAED,KACPgsC,EAAWJ,EAAQI,YACnBA,EAAU,KACRC,EAAiBC,EAAoBF,EAAUJ,MAC/CK,EAAgB,IACdA,IAAmBE,EAAkB,gBAClCF,MAIY,SAAnBL,EAAQ1c,OAGV0c,EAAQQ,KAAOR,EAAQS,MAAQT,EAAQ5rC,SAElC,GAAuB,UAAnB4rC,EAAQ1c,OAAoB,IAjNhB,mBAkNjBgF,QACFA,EAhNc,YAiNR0X,EAAQ5rC,IAGhB4rC,EAAQU,kBAAkBV,EAAQ5rC,SAEN,WAAnB4rC,EAAQ1c,QACjB0c,EAAQW,OAAO,SAAUX,EAAQ5rC,KAGnCk0B,EA3NkB,gBA6NdsY,EAASC,EAASnB,EAASpiB,EAAM0iB,MACjB,WAAhBY,EAAOrY,KAAmB,IAG5BD,EAAQ0X,EAAQxR,KAhOA,YAFK,iBAsOjBoS,EAAOxsC,MAAQmsC,iBAIZ,CACLptC,MAAOytC,EAAOxsC,IACdo6B,KAAMwR,EAAQxR,MAGS,UAAhBoS,EAAOrY,OAChBD,EA9OgB,YAiPhB0X,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAMwsC,EAAOxsC,OA7QP0sC,CAAiBpB,EAASpiB,EAAM0iB,GAE7CD,WAcAc,EAAS3yC,EAAIwE,EAAK0B,aAEhB,CAAEm0B,KAAM,SAAUn0B,IAAKlG,EAAGR,KAAKgF,EAAK0B,IAC3C,MAAOkjC,SACA,CAAE/O,KAAM,QAASn0B,IAAKkjC,IAhBjChzB,EAAQqd,KAAOA,MA2BX4e,EAAmB,YAMdT,cACAiB,cACAC,SAIL1U,EAAoB,GACxBA,EAAkB+S,GAAkB,kBAC3BhuC,UAGL4vC,EAAW1zC,OAAOk/B,eAClByU,EAA0BD,GAAYA,EAASA,EAASt8B,EAAO,MAC/Du8B,GACAA,IAA4B/B,GAC5BC,EAAO1xC,KAAKwzC,EAAyB7B,KAGvC/S,EAAoB4U,OAGlBC,EAAKH,EAA2BxzC,UAClCsyC,EAAUtyC,UAAYD,OAAO84B,OAAOiG,YAQ7B8U,EAAsB5zC,IAC5B,OAAQ,QAAS,UAAU0X,SAAQ,SAASoe,GAC3C91B,EAAU81B,GAAU,SAASlvB,UACpB/C,KAAK6uC,QAAQ5c,EAAQlvB,gBAoCzBitC,EAActB,EAAWuB,OAgC5BC,OAgCCrB,iBA9BY5c,EAAQlvB,YACdotC,WACA,IAAIF,GAAY,SAASG,EAASC,aAnCpCC,EAAOre,EAAQlvB,EAAKqtC,EAASC,OAChCd,EAASC,EAASd,EAAUzc,GAASyc,EAAW3rC,MAChC,UAAhBwsC,EAAOrY,KAEJ,KACD3e,EAASg3B,EAAOxsC,IAChBjB,EAAQyW,EAAOzW,aACfA,GACiB,iBAAVA,GACPisC,EAAO1xC,KAAKyF,EAAO,WACdmuC,EAAYG,QAAQtuC,EAAMyuC,SAASC,MAAK,SAAS1uC,GACtDwuC,EAAO,OAAQxuC,EAAOsuC,EAASC,MAC9B,SAASpK,GACVqK,EAAO,QAASrK,EAAKmK,EAASC,MAI3BJ,EAAYG,QAAQtuC,GAAO0uC,MAAK,SAASC,GAI9Cl4B,EAAOzW,MAAQ2uC,EACfL,EAAQ73B,MACP,SAAS4T,UAGHmkB,EAAO,QAASnkB,EAAOikB,EAASC,MAvBzCA,EAAOd,EAAOxsC,KAiCZutC,CAAOre,EAAQlvB,EAAKqtC,EAASC,aAI1BH,EAaLA,EAAkBA,EAAgBM,KAChCL,EAGAA,GACEA,cAkHDlB,EAAoBF,EAAUJ,OACjC1c,EAAS8c,EAAShc,SAAS4b,EAAQ1c,gBAtSrCxyB,IAuSEwyB,EAAsB,IAGxB0c,EAAQI,SAAW,KAEI,UAAnBJ,EAAQ1c,OAAoB,IAE1B8c,EAAShc,SAAT,SAGF4b,EAAQ1c,OAAS,SACjB0c,EAAQ5rC,SAlTZtD,EAmTIwvC,EAAoBF,EAAUJ,GAEP,UAAnBA,EAAQ1c,eAGHid,EAIXP,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAM,IAAIiqB,UAChB,yDAGGkiB,MAGLK,EAASC,EAASvd,EAAQ8c,EAAShc,SAAU4b,EAAQ5rC,QAErC,UAAhBwsC,EAAOrY,YACTyX,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAMwsC,EAAOxsC,IACrB4rC,EAAQI,SAAW,KACZG,MAGLwB,EAAOnB,EAAOxsC,WAEZ2tC,EAOFA,EAAKvT,MAGPwR,EAAQI,EAAS4B,YAAcD,EAAK5uC,MAGpC6sC,EAAQ36B,KAAO+6B,EAAS6B,QAQD,WAAnBjC,EAAQ1c,SACV0c,EAAQ1c,OAAS,OACjB0c,EAAQ5rC,SAtWVtD,GAgXFkvC,EAAQI,SAAW,KACZG,GANEwB,GA3BP/B,EAAQ1c,OAAS,QACjB0c,EAAQ5rC,IAAM,IAAIiqB,UAAU,oCAC5B2hB,EAAQI,SAAW,KACZG,YAoDF2B,EAAaC,OAChB9D,EAAQ,CAAE+D,OAAQD,EAAK,IAEvB,KAAKA,IACP9D,EAAMgE,SAAWF,EAAK,IAGpB,KAAKA,IACP9D,EAAMiE,WAAaH,EAAK,GACxB9D,EAAMkE,SAAWJ,EAAK,SAGnBK,WAAWl0C,KAAK+vC,YAGdoE,EAAcpE,OACjBuC,EAASvC,EAAMqE,YAAc,GACjC9B,EAAOrY,KAAO,gBACPqY,EAAOxsC,IACdiqC,EAAMqE,WAAa9B,WAGZX,EAAQL,QAIV4C,WAAa,CAAC,CAAEJ,OAAQ,SAC7BxC,EAAY16B,QAAQg9B,EAAc7wC,WAC7BsxC,OAAM,YA8BJh+B,EAAOuwB,MACVA,EAAU,KACR9D,EAAiB8D,EAASmK,MAC1BjO,SACKA,EAAe1jC,KAAKwnC,MAGA,mBAAlBA,EAAS7vB,YACX6vB,MAGJzkC,MAAMykC,EAAS7mC,QAAS,KACvBF,GAAK,EAAGkX,EAAO,SAASA,WACjBlX,EAAI+mC,EAAS7mC,WAChB+wC,EAAO1xC,KAAKwnC,EAAU/mC,UACxBkX,EAAKlS,MAAQ+hC,EAAS/mC,GACtBkX,EAAKmpB,MAAO,EACLnpB,SAIXA,EAAKlS,WAtdTrC,EAudIuU,EAAKmpB,MAAO,EAELnpB,UAGFA,EAAKA,KAAOA,SAKhB,CAAEA,KAAM86B,YAIRA,UACA,CAAEhtC,WAtePrC,EAseyB09B,MAAM,UA5ZnCuS,EAAkBvzC,UAAY2zC,EAAGxY,YAAcqY,EAC/CA,EAA2BrY,YAAcoY,EACzCC,EAA2BxB,GACzBuB,EAAkB6B,YAAc,oBAYlCt+B,EAAQu+B,oBAAsB,SAASC,OACjCC,EAAyB,mBAAXD,GAAyBA,EAAOna,oBAC3Coa,IACHA,IAAShC,GAG2B,uBAAnCgC,EAAKH,aAAeG,EAAKjuC,QAIhCwP,EAAQ0+B,KAAO,SAASF,UAClBv1C,OAAOs/B,eACTt/B,OAAOs/B,eAAeiW,EAAQ9B,IAE9B8B,EAAO9V,UAAYgU,EACbxB,KAAqBsD,IACzBA,EAAOtD,GAAqB,sBAGhCsD,EAAOt1C,UAAYD,OAAO84B,OAAO8a,GAC1B2B,GAOTx+B,EAAQ2+B,MAAQ,SAAS7uC,SAChB,CAAEwtC,QAASxtC,IAsEpBgtC,EAAsBC,EAAc7zC,WACpC6zC,EAAc7zC,UAAU8xC,GAAuB,kBACtCjuC,MAETiT,EAAQ+8B,cAAgBA,EAKxB/8B,EAAQ4+B,MAAQ,SAASxD,EAASC,EAASriB,EAAMsiB,EAAa0B,QACxC,IAAhBA,IAAwBA,EAAc6B,aAEtCzN,EAAO,IAAI2L,EACb1f,EAAK+d,EAASC,EAASriB,EAAMsiB,GAC7B0B,UAGKh9B,EAAQu+B,oBAAoBlD,GAC/BjK,EACAA,EAAKrwB,OAAOw8B,MAAK,SAASj4B,UACjBA,EAAO4kB,KAAO5kB,EAAOzW,MAAQuiC,EAAKrwB,WAuKjD+7B,EAAsBD,GAEtBA,EAAG3B,GAAqB,YAOxB2B,EAAG9B,GAAkB,kBACZhuC,MAGT8vC,EAAG1zC,SAAW,iBACL,sBAkCT6W,EAAQ3P,KAAO,SAASwrB,OAClBxrB,EAAO,OACN,IAAIL,KAAO6rB,EACdxrB,EAAKrG,KAAKgG,UAEZK,EAAKyuC,UAIE,SAAS/9B,SACP1Q,EAAKtG,QAAQ,KACdiG,EAAMK,EAAK0uC,SACX/uC,KAAO6rB,SACT9a,EAAKlS,MAAQmB,EACb+Q,EAAKmpB,MAAO,EACLnpB,SAOXA,EAAKmpB,MAAO,EACLnpB,IAsCXf,EAAQK,OAASA,EAMjBs7B,EAAQzyC,UAAY,CAClBm7B,YAAasX,EAEb0C,MAAO,SAASW,WACTvI,KAAO,OACP11B,KAAO,OAGPm7B,KAAOnvC,KAAKovC,WAjfjB3vC,OAkfK09B,MAAO,OACP4R,SAAW,UAEX9c,OAAS,YACTlvB,SAtfLtD,OAwfK0xC,WAAWt9B,QAAQu9B,IAEnBa,MACE,IAAIxuC,KAAQzD,KAEQ,MAAnByD,EAAK+iB,OAAO,IACZunB,EAAO1xC,KAAK2D,KAAMyD,KACjBrE,OAAOqE,EAAKP,MAAM,WAChBO,QAhgBXhE,IAsgBFksC,KAAM,gBACCxO,MAAO,MAGR+U,EADYlyC,KAAKmxC,WAAW,GACLE,cACH,UAApBa,EAAWhb,WACPgb,EAAWnvC,WAGZ/C,KAAKmyC,MAGd9C,kBAAmB,SAAS+C,MACtBpyC,KAAKm9B,WACDiV,MAGJzD,EAAU3uC,cACLqyC,EAAOC,EAAKC,UACnBhD,EAAOrY,KAAO,QACdqY,EAAOxsC,IAAMqvC,EACbzD,EAAQ36B,KAAOs+B,EAEXC,IAGF5D,EAAQ1c,OAAS,OACjB0c,EAAQ5rC,SAjiBZtD,KAoiBY8yC,MAGP,IAAIz1C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,GACxByyC,EAASvC,EAAMqE,cAEE,SAAjBrE,EAAM+D,cAIDsB,EAAO,UAGZrF,EAAM+D,QAAU/wC,KAAK0pC,KAAM,KACzB8I,EAAWzE,EAAO1xC,KAAK2wC,EAAO,YAC9ByF,EAAa1E,EAAO1xC,KAAK2wC,EAAO,iBAEhCwF,GAAYC,EAAY,IACtBzyC,KAAK0pC,KAAOsD,EAAMgE,gBACbqB,EAAOrF,EAAMgE,UAAU,GACzB,GAAIhxC,KAAK0pC,KAAOsD,EAAMiE,kBACpBoB,EAAOrF,EAAMiE,iBAGjB,GAAIuB,MACLxyC,KAAK0pC,KAAOsD,EAAMgE,gBACbqB,EAAOrF,EAAMgE,UAAU,OAG3B,CAAA,IAAIyB,QAMH,IAAIrvC,MAAM,6CALZpD,KAAK0pC,KAAOsD,EAAMiE,kBACboB,EAAOrF,EAAMiE,gBAU9B3B,OAAQ,SAASpY,EAAMn0B,OAChB,IAAIjG,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAM+D,QAAU/wC,KAAK0pC,MACrBqE,EAAO1xC,KAAK2wC,EAAO,eACnBhtC,KAAK0pC,KAAOsD,EAAMiE,WAAY,KAC5ByB,EAAe1F,SAKnB0F,IACU,UAATxb,GACS,aAATA,IACDwb,EAAa3B,QAAUhuC,GACvBA,GAAO2vC,EAAazB,aAGtByB,EAAe,UAGbnD,EAASmD,EAAeA,EAAarB,WAAa,UACtD9B,EAAOrY,KAAOA,EACdqY,EAAOxsC,IAAMA,EAET2vC,QACGzgB,OAAS,YACTje,KAAO0+B,EAAazB,WAClB/B,GAGFlvC,KAAK2yC,SAASpD,IAGvBoD,SAAU,SAASpD,EAAQ2B,MACL,UAAhB3B,EAAOrY,WACHqY,EAAOxsC,UAGK,UAAhBwsC,EAAOrY,MACS,aAAhBqY,EAAOrY,UACJljB,KAAOu7B,EAAOxsC,IACM,WAAhBwsC,EAAOrY,WACXib,KAAOnyC,KAAK+C,IAAMwsC,EAAOxsC,SACzBkvB,OAAS,cACTje,KAAO,OACa,WAAhBu7B,EAAOrY,MAAqBga,SAChCl9B,KAAOk9B,GAGPhC,GAGT0D,OAAQ,SAAS3B,OACV,IAAIn0C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAMiE,aAAeA,cAClB0B,SAAS3F,EAAMqE,WAAYrE,EAAMkE,UACtCE,EAAcpE,GACPkC,UAKJ,SAAS6B,OACX,IAAIj0C,EAAIkD,KAAKmxC,WAAWn0C,OAAS,EAAGF,GAAK,IAAKA,EAAG,KAChDkwC,EAAQhtC,KAAKmxC,WAAWr0C,MACxBkwC,EAAM+D,SAAWA,EAAQ,KACvBxB,EAASvC,EAAMqE,cACC,UAAhB9B,EAAOrY,KAAkB,KACvB2b,EAAStD,EAAOxsC,IACpBquC,EAAcpE,UAET6F,SAML,IAAIzvC,MAAM,0BAGlB0vC,cAAe,SAASjP,EAAU8M,EAAYC,eACvC7B,SAAW,CACdhc,SAAUzf,EAAOuwB,GACjB8M,WAAYA,EACZC,QAASA,GAGS,SAAhB5wC,KAAKiyB,cAGFlvB,SA1qBPtD,GA6qBOyvC,IAQJj8B,EA1rBM,CAisBgBxX,EAAOwX,aAIpC8/B,mBAAqBlF,EACrB,MAAOmF,GAUPrvC,SAAS,IAAK,yBAAdA,CAAwCkqC,UCrtBzBjd,GCSbuR,GAAsBC,GAA6B,UACnDb,GAAiBC,GAAwB,SAAU,CAAEN,WAAW,IAAS,IAAM,IAE/E57B,GAAM9D,KAAK8D,IACXlD,GAAMZ,KAAKY,MAOb,CAAEutB,OAAQ,QAASnP,OAAO,EAAM2P,QAASgS,KAAwBZ,IAAkB,CACnF0R,OAAQ,SAAgBzS,EAAO0S,OAKzBC,EAAaC,EAAmBnQ,EAAGtoB,EAAGta,EAAMD,EAJ5CqtB,EAAI9K,GAAS3iB,MACbC,EAAMoxB,EAAS5D,EAAEzwB,QACjBq2C,EAAc/hB,GAAgBkP,EAAOvgC,GACrC+jC,EAAkBloC,UAAUkB,UAER,IAApBgnC,EACFmP,EAAcC,EAAoB,EACL,IAApBpP,GACTmP,EAAc,EACdC,EAAoBnzC,EAAMozC,IAE1BF,EAAcnP,EAAkB,EAChCoP,EAAoBhxC,GAAIkD,GAAIwrB,EAAUoiB,GAAc,GAAIjzC,EAAMozC,IAE5DpzC,EAAMkzC,EAAcC,EAtBL,uBAuBXpmB,UAtB0B,uCAwBlCiW,EAAIjL,GAAmBvK,EAAG2lB,GACrBz4B,EAAI,EAAGA,EAAIy4B,EAAmBz4B,KACjCta,EAAOgzC,EAAc14B,KACT8S,GAAGiF,GAAeuQ,EAAGtoB,EAAG8S,EAAEptB,OAExC4iC,EAAEjmC,OAASo2C,EACPD,EAAcC,EAAmB,KAC9Bz4B,EAAI04B,EAAa14B,EAAI1a,EAAMmzC,EAAmBz4B,IAEjDva,EAAKua,EAAIw4B,GADT9yC,EAAOsa,EAAIy4B,KAEC3lB,EAAGA,EAAErtB,GAAMqtB,EAAEptB,UACbotB,EAAErtB,OAEXua,EAAI1a,EAAK0a,EAAI1a,EAAMmzC,EAAoBD,EAAax4B,WAAY8S,EAAE9S,EAAI,QACtE,GAAIw4B,EAAcC,MAClBz4B,EAAI1a,EAAMmzC,EAAmBz4B,EAAI04B,EAAa14B,IAEjDva,EAAKua,EAAIw4B,EAAc,GADvB9yC,EAAOsa,EAAIy4B,EAAoB,KAEnB3lB,EAAGA,EAAErtB,GAAMqtB,EAAEptB,UACbotB,EAAErtB,OAGbua,EAAI,EAAGA,EAAIw4B,EAAax4B,IAC3B8S,EAAE9S,EAAI04B,GAAev3C,UAAU6e,EAAI,UAErC8S,EAAEzwB,OAASiD,EAAMmzC,EAAoBD,EAC9BlQ,KChEX,OAAiBpC,GAAa,SAASoS,OCDnCxR,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmnB,cACNnnB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAewR,OAAUA,GAASlS,GCFhG79B,GAAQ,GAAGA,MACXowC,GAAO,WAAWvsC,KAAK+6B,IAEvBxR,GAAO,SAAUijB,UACZ,SAAUC,EAASC,OACpBC,EAAY53C,UAAUkB,OAAS,EAC/BgG,EAAO0wC,EAAYxwC,GAAM7G,KAAKP,UAAW,QAAK2D,SAC3C8zC,EAAUG,EAAY,YAER,mBAAXF,EAAwBA,EAAU7vC,SAAS6vC,IAAU33C,MAAMmE,KAAMgD,IACvEwwC,EAASC,OAMf,CAAEvnB,QAAQ,EAAMmE,MAAM,EAAMF,OAAQmjB,IAAQ,CAG5CK,WAAYrjB,GAAKpE,EAAOynB,YAGxBC,YAAatjB,GAAKpE,EAAO0nB,eCvB3B,OAAiB3jB,EAAK0jB,WCIlBE,GAAmB,SAAUlkB,EAAQmkB,EAAUjuB,EAAQkuB,EAAWvT,EAAOwT,EAAOC,EAAQC,WAItFtM,EAHAuM,EAAc3T,EACd4T,EAAc,EACdC,IAAQJ,GAAS5jB,EAAK4jB,EAAQC,EAAS,GAGpCE,EAAcL,GAAW,IAC1BK,KAAevuB,EAAQ,IACzB+hB,EAAUyM,EAAQA,EAAMxuB,EAAOuuB,GAAcA,EAAaN,GAAYjuB,EAAOuuB,GAEzEJ,EAAQ,GAAKj4C,GAAQ6rC,GACvBuM,EAAcN,GAAiBlkB,EAAQmkB,EAAUlM,EAASvW,EAASuW,EAAQ5qC,QAASm3C,EAAaH,EAAQ,GAAK,MACzG,IACDG,GAAe,iBAAkB,MAAMnnB,UAAU,sCACrD2C,EAAOwkB,GAAevM,EAGxBuM,IAEFC,WAEKD,MAGQN,KCrBf,CAAElkB,OAAQ,QAASnP,OAAO,GAAQ,CAClC8zB,QAAS,SAAiBzc,OAGpBoL,EAFAxV,EAAI9K,GAAS3iB,MACb+zC,EAAY1iB,EAAS5D,EAAEzwB,eAE3B0xB,EAAUmJ,IACVoL,EAAIjL,GAAmBvK,EAAG,IACxBzwB,OAAS62C,GAAiB5Q,EAAGxV,EAAGA,EAAGsmB,EAAW,EAAG,EAAGlc,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,GACjGwjC,KCdX,OAAiBpC,GAAa,SAASyT,QCFnC7S,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGwoB,eACNxoB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAe6S,QAAWA,GAAUvT,GCAlG/P,GAAe,SAAUujB,UACpB,SAAUvzB,EAAM6W,EAAYmM,EAAiBwQ,GAClD9lB,EAAUmJ,OACNpK,EAAI9K,GAAS3B,GACbiL,EAAOgB,EAAcQ,GACrBzwB,EAASq0B,EAAS5D,EAAEzwB,QACpB4nB,EAAQ2vB,EAAWv3C,EAAS,EAAI,EAChCF,EAAIy3C,GAAY,EAAI,KACpBvQ,EAAkB,EAAG,OAAa,IAChCpf,KAASqH,EAAM,CACjBuoB,EAAOvoB,EAAKrH,GACZA,GAAS9nB,WAGX8nB,GAAS9nB,EACLy3C,EAAW3vB,EAAQ,EAAI5nB,GAAU4nB,QAC7BoI,UAAU,oDAGdunB,EAAW3vB,GAAS,EAAI5nB,EAAS4nB,EAAOA,GAAS9nB,EAAO8nB,KAASqH,IACrEuoB,EAAO3c,EAAW2c,EAAMvoB,EAAKrH,GAAQA,EAAO6I,WAEvC+mB,IC1BPC,GD8Ba,CAGfC,KAAM1jB,IAAa,GAGnB2jB,MAAO3jB,IAAa,ICpC6B0jB,KAI/CrT,GAAgBC,GAAoB,UACpCC,GAAiBC,GAAwB,SAAU,GAAK,MAI1D,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7EqT,OAAQ,SAAgB/c,UACf4c,GAAQz0C,KAAM63B,EAAY/7B,UAAUkB,OAAQlB,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCV7F,OAAiBohC,GAAa,SAAS+T,OCDnCnT,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAG8oB,cACN9oB,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAemT,OAAUA,GAAS7T,GCFhG79B,GAAQ,GAAGA,MACX2xC,GAAY,GAEZC,GAAY,SAAUzd,EAAG0d,EAAY/xC,QACjC+xC,KAAcF,IAAY,KACzB,IAAIG,EAAO,GAAIl4C,EAAI,EAAGA,EAAIi4C,EAAYj4C,IAAKk4C,EAAKl4C,GAAK,KAAOA,EAAI,IAErE+3C,GAAUE,GAAcpxC,SAAS,MAAO,gBAAkBqxC,EAAK7xC,KAAK,KAAO,YACpE0xC,GAAUE,GAAY1d,EAAGr0B,OAKnBW,SAAS0sB,MAAQ,SAAcrP,OAC1CnkB,EAAK6xB,EAAU1uB,MACfi1C,EAAW/xC,GAAM7G,KAAKP,UAAW,GACjCi8B,EAAgB,eACd/0B,EAAOiyC,EAAS7mC,OAAOlL,GAAM7G,KAAKP,mBAC/BkE,gBAAgB+3B,EAAgB+c,GAAUj4C,EAAImG,EAAKhG,OAAQgG,GAAQnG,EAAGhB,MAAMmlB,EAAMhe,WAEvF1G,EAASO,EAAGV,aAAY47B,EAAc57B,UAAYU,EAAGV,WAClD47B,KCpBP,CAAEpI,OAAQ,WAAYnP,OAAO,GAAQ,CACrC6P,KAAMA,KCHR,OAAiBwQ,GAAa,YAAYxQ,KCDtC6kB,GAAoBvxC,SAASxH,aAEhB,SAAU2vB,OACrBiV,EAAMjV,EAAGuE,YACNvE,IAAOopB,IAAsBppB,aAAcnoB,UAAYo9B,IAAQmU,GAAkB7kB,KAAQA,GAAO0Q,GCAzG,OANA,SAAyBoU,EAAUvQ,QAC3BuQ,aAAoBvQ,SAClB,IAAI5X,UAAU,sCCAxB,SAASooB,GAAkBzlB,EAAQ0lB,OAC5B,IAAIv4C,EAAI,EAAGA,EAAIu4C,EAAMr4C,OAAQF,IAAK,KACjC4vB,EAAa2oB,EAAMv4C,GACvB4vB,EAAWC,WAAaD,EAAWC,aAAc,EACjDD,EAAWG,cAAe,EACtB,UAAWH,IAAYA,EAAWI,UAAW,GAEjDmT,GAAuBtQ,EAAQjD,EAAWzpB,IAAKypB,IAUnD,OANA,SAAsBkY,EAAa0Q,EAAYC,UACzCD,GAAYF,GAAkBxQ,EAAYzoC,UAAWm5C,GACrDC,GAAaH,GAAkBxQ,EAAa2Q,GACzC3Q,qJCwDO4Q,GAGdn1C,UACO,IAAIo1C,GAA0Bp1C,OAcjCq1C,yBAwBeC,EACAC,EACAC,6BAFAF,QAAAA,OACAC,cAAAA,OACAC,QAAAA,OAjBFC,WAAqC,CACpDj7B,IAAKk7B,UAAKC,aAAUh2C,MACpBi2C,OAAQF,UAAKG,gBAAal2C,MAC1Bm2C,OAAQJ,UAAKK,gBAAap2C,4DAmBrB61C,QAAQM,OAAOn2C,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,QAC/C/J,iDAKF21C,QAAQW,GAAG,MAAOt2C,KAAK81C,WAAWj7B,UAClC86B,QAAQW,GAAG,SAAUt2C,KAAK81C,WAAWG,aACrCN,QAAQW,GAAG,SAAUt2C,KAAK81C,WAAWK,QAEnCn2C,gDAKF21C,QAAQY,IAAI,MAAOv2C,KAAK81C,WAAWj7B,UACnC86B,QAAQY,IAAI,SAAUv2C,KAAK81C,WAAWG,aACtCN,QAAQY,IAAI,SAAUv2C,KAAK81C,WAAWK,QAEpCn2C,6CAUew2C,gBACfC,UAAKb,uBAAqB,SAACY,EAAOE,UAChCA,EAAUF,KAChBA,gCAUHG,EACAC,GAEe,MAAXA,QAICf,QAAQh7B,IAAI7a,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,IAAI6sC,EAAQJ,yCAU/DG,EACAC,GAEe,MAAXA,QAICf,QAAQM,OAAOn2C,KAAKq2C,gBAAgBr2C,KAAK21C,QAAQ5rC,IAAI6sC,EAAQJ,yCAUlEG,EACAC,GAEe,MAAXA,QAICf,QAAQI,OAAOj2C,KAAKq2C,gBAAgBO,EAAQC,mBAW/CpB,yBAgBgCE,mBAAAA,QAAAA,OARnBC,cAAoC,6CAmBnD5vC,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqB86C,GAAA96C,QAAAA,EAAagK,MACpDhG,iCAePgG,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqB+6C,GAAA/6C,QAAAA,EAAUgK,MAChDhG,qCAeRgG,eAEK4vC,cAAc34C,MAAK,SAACjB,UAAqBg7C,GAAAh7C,QAAAA,EAAcgK,MACpDhG,gCAWA2vB,UACD,IAAI+lB,GAAe11C,KAAK21C,QAAS31C,KAAK41C,cAAejmB,8vsBCjS5B,oBAAX3D,QAA2BA,OAAM,QAAcxB,2ICQxE,IAAMysB,GAAe,qBAWd,SAASC,GAAQpoB,EAAQoI,OAC1B5wB,UAEW7G,IAAXqvB,MAGW,OAAXA,SACK,SAGJoI,SACIpI,KAEa,iBAAToI,KAAwBA,aAAgBzI,cAC7C,IAAIrrB,MAAM,gCAIV8zB,OACD,cACA,iBACI+K,QAAQnT,OAEZ,aACA,gBACCqoB,GAASroB,KAAY1vB,MAAM1C,KAAK06C,MAAMtoB,IACjC1E,EAAO0E,GAAQvxB,UAKf85C,OAAOvoB,EAAOvxB,eAEpB,aACA,gBACIkxB,OAAOK,OAEX,UACCtyB,GAASsyB,UACJ,IAAIpyB,KAAKoyB,MAEdA,aAAkBpyB,YACb,IAAIA,KAAKoyB,EAAOvxB,WAClB,GAAI6sB,EAAOhpB,SAAS0tB,UAClB,IAAIpyB,KAAKoyB,EAAOvxB,cAErB45C,GAASroB,UACXxoB,EAAQ2wC,GAAa//B,KAAK4X,IAGjB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAEtB8jB,EAAO,IAAI1tB,KAAKoyB,IAASlM,eAG5B,IAAIxf,MACR,iCAAmCk0C,GAAQxoB,GAAU,qBAItD,YACCtyB,GAASsyB,UACJ1E,EAAO0E,MAEZA,aAAkBpyB,YACb0tB,EAAO0E,EAAOvxB,WAChB,GAAI6sB,EAAOhpB,SAAS0tB,UAClB1E,EAAO0E,MAEZqoB,GAASroB,UACXxoB,EAAQ2wC,GAAa//B,KAAK4X,GAGjB1E,EAFL9jB,EAEY+wC,OAAO/wC,EAAM,IAEbwoB,SAGV,IAAI1rB,MACR,iCAAmCk0C,GAAQxoB,GAAU,qBAItD,aACCtyB,GAASsyB,UACJ,IAAIpyB,KAAKoyB,GACX,GAAIA,aAAkBpyB,YACpBoyB,EAAOjM,cACT,GAAIuH,EAAOhpB,SAAS0tB,UAClBA,EAAOlM,SAASC,cAClB,GAAIs0B,GAASroB,UAClBxoB,EAAQ2wC,GAAa//B,KAAK4X,IAGjB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAAKuc,cAE3BuH,EAAO0E,GAAQrxB,eAGlB,IAAI2F,MACR,iCACEk0C,GAAQxoB,GACR,wBAIH,aACCtyB,GAASsyB,SACJ,SAAWA,EAAS,KACtB,GAAIA,aAAkBpyB,MAAQ0tB,EAAOhpB,SAAS0tB,SAC5C,SAAWA,EAAOvxB,UAAY,KAChC,GAAI45C,GAASroB,SASX,WARPxoB,EAAQ2wC,GAAa//B,KAAK4X,IAIhB,IAAIpyB,KAAK26C,OAAO/wC,EAAM,KAAK/I,UAE3B,IAAIb,KAAKoyB,GAAQvxB,WAED,WAEpB,IAAI6F,MACR,iCACEk0C,GAAQxoB,GACR,kCAKA,IAAI1rB,6BAAsB8zB,MA8B/B,SAASqgB,GACdC,mBACAtgB,yDAAO,CAAEsJ,MAAO,OAAQC,IAAK,QAEvBgX,EAASD,EAAME,QACfC,EAAY,IAAIC,GAAQ,CAAEC,QAASJ,IAEnCK,EAAOf,KAAAvB,GAAsBgC,YAC5B,SAAAO,gBACHtB,QAAYsB,YAAa,SAACC,EAAK/0C,UAC7B+0C,EAAI/0C,GAAOi0C,GAAQa,EAAK90C,GAAMi0B,EAAKj0B,IAC5B+0C,IACN,OAEJ53C,GAAGu3C,UAENG,EAAKG,MAAMzX,QAEJ,CAEL3lB,IAAK,0BAAa28B,EAAMU,cAAar9B,wBACrCo7B,OAAQ,0BAAauB,EAAMU,cAAajC,2BACxCE,OAAQ,0BAAaqB,EAAMU,cAAa/B,2BACxCgC,WAAY,0BAAaX,EAAMU,cAAaC,+BAG5CtkC,QAASkiC,QAAA4B,WAAuBA,GAChC5tC,IAAKgsC,KAAA4B,EAAU5tC,YAAS4tC,GACxBS,OAAQrC,KAAA4B,EAAUS,eAAYT,GAC9BpB,IAAKR,KAAA4B,EAAUpB,YAASoB,GACxBrB,GAAIP,KAAA4B,EAAUrB,WAAQqB,uBAGbA,EAAU36C,QAInBy6C,OAAAA,EACAvgB,KAAAA,EAEAsgB,MAAAA,EACAG,UAAAA,EACAU,QAAS,kBAAMP,EAAKnM,uRAKnB2M,IACHpB,QAAAA,KCtOExW,GAAOlW,GAAoCkW,KAG3C6X,GAAcrsB,EAAOpQ,cACZ,EAAIy8B,GAAYlY,GAAc,QAAWmY,EAAAA,EAI5B,SAAoBvhC,OACxCwhC,EAAgB/X,GAAKjS,OAAOxX,IAC5BsB,EAASggC,GAAYE,UACP,IAAXlgC,GAA2C,KAA3BkgC,EAAcjyB,OAAO,IAAa,EAAIjO,GAC3DggC,KCRF,CAAErsB,QAAQ,EAAMiE,OAAQrU,YAAc48B,IAA4B,CAClE58B,WAAY48B,KCHd,OAAiBzoB,EAAKnU,WCAD68B,yBAKPC,EAAMzpB,mBACXA,QAAU,UACVkmB,MAAQ,mDAQJlmB,GACLA,GACFmpB,GAAKh7C,OAAO0C,KAAKmvB,QAASA,2CAUrB,8EAiBD0pB,EACJ74C,KAAKq1C,MAAMyD,iBAAmB94C,KAAKq1C,MAAMzM,OACzC5oC,KAAKq1C,MAAM0D,kBAAoB/4C,KAAKq1C,MAAMxM,mBAGvCwM,MAAMyD,eAAiB94C,KAAKq1C,MAAMzM,WAClCyM,MAAM0D,gBAAkB/4C,KAAKq1C,MAAMxM,OAEjCgQ,aClDT,CAAElpB,OAAQ,SAAUnP,OAAO,GAAQ,CACnCw4B,OCAe,GAAGA,QAAU,SAAgBC,OACxCptB,EAAM4C,OAAOvB,EAAuBltB,OACpCuY,EAAS,GACTpK,EAAI2iB,EAAUmoB,MACd9qC,EAAI,GAAKA,GAAKqqC,EAAAA,EAAU,MAAMU,WAAW,oCACvC/qC,EAAI,GAAIA,KAAO,KAAO0d,GAAOA,GAAc,EAAJ1d,IAAOoK,GAAUsT,UACvDtT,KCTT,OAAiBsoB,GAAa,UAAUmY,OCDpClY,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAGktB,aACQ,iBAAPltB,GAAmBA,IAAOgV,IAClChV,aAAc2C,QAAUsS,IAAQD,GAAgBkY,OAAUA,GAASjY,GCEpE,SAASoY,GAAqB/uB,EAAQwuB,EAAMQ,MAC7CA,IAAgBlW,GAAckW,UACzBD,GAAqB/uB,EAAQwuB,EAAM,CAACQ,OAG7CR,EAAKQ,YAAc,GACfA,GACgC,GAA9BlW,GAAckW,GAAsB,WAC7Bt8C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,YACR2C,IAA1B45C,GAAAD,EAAYt8C,IAAyB,KACjCw8C,EAAW,GACjBA,EAAS9Y,MAAQpW,EAAOgvB,EAAYt8C,GAAG0jC,OAAO5d,SAASrlB,UACvD+7C,EAAS7Y,IAAMrW,EAAOgvB,EAAYt8C,GAAG2jC,KAAK7d,SAASrlB,UACnDq7C,EAAKQ,YAAYn8C,KAAKq8C,QAG1BV,EAAKQ,qBAAiB,SAACj8C,EAAGC,UAAMD,EAAEqjC,MAAQpjC,EAAEojC,UAa3C,SAAS+Y,GAAkBnvB,EAAQwuB,EAAMQ,MAC1CA,IAAgBlW,GAAckW,UACzBG,GAAkBnvB,EAAQwuB,EAAM,CAACQ,OAGtCA,QAAuD35C,IAAxCm5C,EAAKY,SAASC,gBAAgB7Q,MAAqB,CACpEuQ,GAAqB/uB,EAAQwuB,EAAMQ,WAE7B5Y,EAAQpW,EAAOwuB,EAAKc,MAAMlZ,OAC1BC,EAAMrW,EAAOwuB,EAAKc,MAAMjZ,KAGxBkZ,GADcf,EAAKc,MAAMjZ,IAAMmY,EAAKc,MAAMlZ,OACjBoY,EAAKY,SAASC,gBAAgB7Q,MAEpD9rC,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,YACR2C,IAA1B45C,GAAAD,EAAYt8C,IAAyB,KACjC88C,EAAYxvB,EAAOgvB,EAAYt8C,GAAG0jC,OACpCqZ,EAAUzvB,EAAOgvB,EAAYt8C,GAAG2jC,QAEhB,gBAAhBmZ,EAAUv6C,SACN,IAAI+D,kDAA2Cg2C,EAAYt8C,GAAG0jC,WAEpD,gBAAdqZ,EAAQx6C,SACJ,IAAI+D,gDAAyCg2C,EAAYt8C,GAAG2jC,SAGnDoZ,EAAUD,GACX,EAAID,EAAW,KAEzBn9B,EAAS,EACPs9B,EAAWrZ,EAAIrjB,kBACbg8B,EAAYt8C,SACb,QACC88C,EAAUxqC,OAASyqC,EAAQzqC,QAC7BoN,EAAS,GAEXo9B,EAAUjsC,UAAU6yB,EAAM7yB,aAC1BisC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,QAErBi7B,EAAQlsC,UAAU6yB,EAAM7yB,aACxBksC,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAIpC,EAAO,QAE5Bs9B,EAASj/B,IAAI,EAAG,mBAEb,aACGk/B,EAAYF,EAAQ18B,KAAKy8B,EAAU,QACnCxqC,EAAMwqC,EAAUxqC,MAGtBwqC,EAAU3vC,KAAKu2B,EAAMv2B,QACrB2vC,EAAU5vC,MAAMw2B,EAAMx2B,SACtB4vC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBwwC,EAAUD,EAAUx8B,QAGpBw8B,EAAUxqC,IAAIA,GACdyqC,EAAQzqC,IAAIA,GACZyqC,EAAQh/B,IAAIk/B,EAAU,QAEtBH,EAAUh7B,SAAS,EAAE,SACrBi7B,EAAQj7B,SAAS,EAAE,SAEnBk7B,EAASj/B,IAAI,EAAG,mBAGb,UACC++B,EAAU5vC,SAAW6vC,EAAQ7vC,UAC/BwS,EAAS,GAEXo9B,EAAU5vC,MAAMw2B,EAAMx2B,SACtB4vC,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,UAErBi7B,EAAQ7vC,MAAMw2B,EAAMx2B,SACpB6vC,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAE,UACnBi7B,EAAQh/B,IAAI2B,EAAO,UAEnBs9B,EAASj/B,IAAI,EAAG,oBAEb,SACC++B,EAAUvwC,QAAUwwC,EAAQxwC,SAC9BmT,EAAS,GAEXo9B,EAAUvwC,KAAKm3B,EAAMn3B,QACrBuwC,EAAUh7B,SAAS,EAAE,SACrBi7B,EAAQxwC,KAAKm3B,EAAMn3B,QACnBwwC,EAAQj7B,SAAS,EAAE,SACnBi7B,EAAQh/B,IAAI2B,EAAO,SAEnBs9B,EAASj/B,IAAI,EAAG,mCAGhBlY,QAAQq3C,IAAI,8EAA4EZ,EAAYt8C,UAGjG88C,EAAYE,UACjBlB,EAAKQ,YAAYn8C,KAAK,CAACujC,MAAOoZ,EAAUr8C,UAAWkjC,IAAKoZ,EAAQt8C,eACxD67C,EAAYt8C,SACb,QACH88C,EAAU/+B,IAAI,EAAG,QACjBg/B,EAAQh/B,IAAI,EAAG,kBAEZ,SACH++B,EAAU/+B,IAAI,EAAG,SACjBg/B,EAAQh/B,IAAI,EAAG,mBAEZ,UACH++B,EAAU/+B,IAAI,EAAG,UACjBg/B,EAAQh/B,IAAI,EAAG,oBAEZ,SACH++B,EAAU/+B,IAAI,EAAG,KACjBg/B,EAAQh/B,IAAI,EAAG,+BAGflY,QAAQq3C,IAAI,8EAA4EZ,EAAYt8C,KAI1G87C,EAAKQ,YAAYn8C,KAAK,CAACujC,MAAOoZ,EAAUr8C,UAAWkjC,IAAKoZ,EAAQt8C,aAKtE08C,GAAiBrB,OAEXsB,EAAcC,GAAYvB,EAAKc,MAAMlZ,MAAOoY,EAAKQ,aACjDgB,EAAYD,GAAYvB,EAAKc,MAAMjZ,IAAImY,EAAKQ,aAC9CiB,EAAazB,EAAKc,MAAMlZ,MACxB8Z,EAAW1B,EAAKc,MAAMjZ,IACA,GAAtByZ,EAAYK,SAAiBF,EAAwC,GAA3BzB,EAAKc,MAAMc,aAAuBN,EAAYN,UAAY,EAAIM,EAAYL,QAAU,GAC1G,GAApBO,EAAUG,SAAmBD,EAAsC,GAAzB1B,EAAKc,MAAMe,WAAuBL,EAAUR,UAAY,EAAMQ,EAAUP,QAAU,GACtG,GAAtBK,EAAYK,QAAsC,GAApBH,EAAUG,QAC1C3B,EAAKc,MAAMgB,YAAYL,EAAYC,IAYlC,SAASL,GAAiBrB,aACzBQ,EAAcR,EAAKQ,YACnBuB,EAAY,GACT79C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,QACjC,IAAIiX,EAAI,EAAGA,EAAIqlC,EAAYp8C,OAAQ+W,IAClCjX,GAAKiX,GAA8B,GAAzBqlC,EAAYrlC,GAAGkiC,QAA2C,GAAzBmD,EAAYt8C,GAAGm5C,SAExDmD,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG2jC,IACvF2Y,EAAYrlC,GAAGkiC,QAAS,EAGjBmD,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAGysB,OAAS4Y,EAAYt8C,GAAG2jC,KAC9F2Y,EAAYt8C,GAAG2jC,IAAM2Y,EAAYrlC,GAAG0sB,IACpC2Y,EAAYrlC,GAAGkiC,QAAS,GAGjBmD,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG0jC,OAAS4Y,EAAYrlC,GAAG0sB,KAAO2Y,EAAYt8C,GAAG2jC,MAC1F2Y,EAAYt8C,GAAG0jC,MAAQ4Y,EAAYrlC,GAAGysB,MACtC4Y,EAAYrlC,GAAGkiC,QAAS,QAM3Bn5C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,KACJ,IAA1Bs8C,EAAYt8C,GAAGm5C,QACjB0E,EAAU19C,KAAKm8C,EAAYt8C,IAI/B87C,EAAKQ,YAAcuB,OACnB/B,EAAKQ,qBAAiB,SAACj8C,EAAGC,UAAMD,EAAEqjC,MAAQpjC,EAAEojC,SAmBvC,SAASoa,GAAoBxwB,EAAQywB,EAAUC,WAChDC,GAAe,EACbC,EAAeH,EAASI,QAAQ19C,UAC7BT,EAAI,EAAGA,EAAI+9C,EAASzB,YAAYp8C,OAAQF,IAAK,KAC9C88C,EAAYiB,EAASzB,YAAYt8C,GAAG0jC,MACtCqZ,EAAUgB,EAASzB,YAAYt8C,GAAG2jC,OAClCua,GAAgBpB,GAAaoB,EAAenB,EAAS,CACvDkB,GAAe,YAKC,GAAhBA,GAAwBC,EAAeH,EAASK,KAAK39C,WAAay9C,GAAgBF,EAAc,KAC5FK,EAAY/wB,EAAO0wB,GACnBM,EAAWhxB,EAAOyvB,GAEpBsB,EAAU9xC,QAAU+xC,EAAS/xC,OAASwxC,EAASQ,cAAe,EACzDF,EAAUnxC,SAAWoxC,EAASpxC,QAAU6wC,EAASS,eAAgB,EACjEH,EAAUxtC,aAAeytC,EAASztC,cAAcktC,EAASU,aAAc,GAEhFV,EAASI,QAAUG,GAmChB,SAASI,GAASC,EAAM/6B,EAAMkoB,OAC/B8S,KACgC,GAAhCD,EAAK7C,KAAKQ,YAAYp8C,cACtB0+C,EAAaD,EAAK/B,MAAMgC,WAAW9S,IAC3BloB,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,UAEnDpB,EAASJ,GAAYz5B,EAAM+6B,EAAK7C,KAAKQ,aACtB,GAAjBmB,EAAOA,SACT75B,EAAO65B,EAAOX,eAGVv+B,EAAWugC,GAAyBH,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAMlZ,MAAOib,EAAK/B,MAAMjZ,QAC1F/f,EAAO+6B,EAAK/B,MAAMlZ,MAAO,CAC3Bkb,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,OACpCwgC,EAAoBC,GAA6BL,EAAK7C,KAAKQ,YAAa14B,EAAMg7B,EAAWl/B,eAC/FkE,EAAO+6B,EAAKtsB,QAAQ/E,OAAO1J,GAAMkC,SAASrlB,UAC1CmjB,GAAcm7B,IACLH,EAAWl/B,OAASkE,EAAKnjB,WAAam+C,EAAWC,MAErD,GAAIj7B,EAAO+6B,EAAK/B,MAAMjZ,IAAK,KAC1Bsb,EAAgB,CAACvb,MAAOib,EAAK/B,MAAMlZ,MAAOC,IAAK/f,UACrDA,EAAOs7B,GAAqBP,EAAKtsB,QAAQ/E,OAAQqxB,EAAK7C,KAAKQ,YAAa2C,EAAer7B,GACvFg7B,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,IAClCqF,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,aAGzDj7B,EAAOs7B,GAAqBP,EAAKtsB,QAAQ/E,OAAQqxB,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAOh5B,GACpFg7B,EAAaD,EAAK/B,MAAMgC,WAAW9S,EAAOvtB,IAClCqF,EAAKnjB,UAAYm+C,EAAWl/B,QAAUk/B,EAAWC,MAa1D,SAASM,GAAOR,EAAMtxC,EAAGy+B,MACM,GAAhC6S,EAAK7C,KAAKQ,YAAYp8C,OAAa,KAC/B0+C,EAAaD,EAAK/B,MAAMgC,WAAW9S,UAClC,IAAIlsC,KAAKyN,EAAIuxC,EAAWC,MAAQD,EAAWl/B,YAG5C0/B,EAAiBN,GAAyBH,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAMlZ,MAAOib,EAAK/B,MAAMjZ,KAE9F0b,GADgBV,EAAK/B,MAAMjZ,IAAMgb,EAAK/B,MAAMlZ,MAAQ0b,GAClB/xC,EAAIy+B,EACtCwT,EAA4BC,GAA6BZ,EAAK7C,KAAKQ,YAAaqC,EAAK/B,MAAOyC,UAE3F,IAAIz/C,KAAK0/C,EAA4BD,EAAkBV,EAAK/B,MAAMlZ,OAYtE,SAASob,GAAyBxC,EAAa5Y,EAAOC,WACvDplB,EAAW,EACNve,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAapZ,GAASqZ,EAAUpZ,IAClCplB,GAAYw+B,EAAUD,UAGnBv+B,EAWF,SAASygC,GAA6B1C,EAAa5Y,EAAOC,WAC3DplB,EAAW,EACNve,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAapZ,GAASqZ,GAAWpZ,IACnCplB,GAAYw+B,EAAUD,UAGnBv+B,EAWF,SAAS2gC,GAAqB5xB,EAAQgvB,EAAaM,EAAOh5B,UAC/DA,EAAO0J,EAAO1J,GAAMkC,SAASrlB,UAC7BmjB,GAAQ47B,GAAwBlyB,EAAQgvB,EAAYM,EAAMh5B,GAYrD,SAAS47B,GAAwBlyB,EAAQgvB,EAAaM,EAAOh5B,OAC9D67B,EAAa,EACjB77B,EAAO0J,EAAO1J,GAAMkC,SAASrlB,cAExB,IAAIT,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,IAE3BmZ,GAAaF,EAAMlZ,OAASqZ,EAAUH,EAAMjZ,KAC1C/f,GAAQm5B,IACV0C,GAAe1C,EAAUD,UAIxB2C,EAWF,SAASF,GAA6BjD,EAAaM,EAAO8C,WAC3DN,EAAiB,EACjB7gC,EAAW,EACXohC,EAAgB/C,EAAMlZ,MAEjB1jC,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACrC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,OAE3BmZ,GAAaF,EAAMlZ,OAASqZ,EAAUH,EAAMjZ,IAAK,IACnDplB,GAAYu+B,EAAY6C,EACxBA,EAAgB5C,EACZx+B,GAAYmhC,QAIdN,GAAkBrC,EAAUD,UAK3BsC,EAWF,SAASQ,GAAmBtD,EAAa14B,EAAMpC,EAAWq+B,OACzDC,EAAWzC,GAAYz5B,EAAM04B,UACZ,GAAnBwD,EAASrC,OACPj8B,EAAY,EACW,GAArBq+B,EACKC,EAAShD,WAAagD,EAAS/C,QAAUn5B,GAAQ,EAGjDk8B,EAAShD,UAAY,EAIL,GAArB+C,EACKC,EAAS/C,SAAWn5B,EAAOk8B,EAAShD,WAAa,EAGjDgD,EAAS/C,QAAU,EAKvBn5B,EAYJ,SAASy5B,GAAYz5B,EAAM04B,OAC3B,IAAIt8C,EAAI,EAAGA,EAAIs8C,EAAYp8C,OAAQF,IAAK,KACvC88C,EAAYR,EAAYt8C,GAAG0jC,MAC3BqZ,EAAUT,EAAYt8C,GAAG2jC,OAEzB/f,GAAQk5B,GAAal5B,EAAOm5B,QACvB,CAACU,QAAQ,EAAMX,UAAAA,EAAWC,QAAAA,SAG9B,CAACU,QAAQ,EAAOX,UAAAA,EAAWC,QAAAA,oHA9R7B,SAAoBt1B,OACpB,IAAIznB,EAAG,EAAGA,EAAIynB,EAAMvnB,OAAQF,IAC/B6F,QAAQq3C,IAAIl9C,EAAG,IAAIJ,KAAK6nB,EAAMznB,GAAG0jC,OAAO,IAAI9jC,KAAK6nB,EAAMznB,GAAG2jC,KAAMlc,EAAMznB,GAAG0jC,MAAOjc,EAAMznB,GAAG2jC,IAAKlc,EAAMznB,GAAGm5C,4OCxNtF4G,0BAOPjE,EAAMzpB,kEAEVta,EAAMuV,KAASha,MAAM,GAAGE,QAAQ,GAAGG,QAAQ,GAAGmL,aAAa,GAC3D4kB,EAAQ3rB,EAAIuI,QAAQvC,KAAK,EAAG,QAAQtd,UACpCkjC,EAAM5rB,EAAIuI,QAAQvC,IAAI,EAAG,QAAQtd,mBAClCu/C,+BAA4Br9C,OAElBA,IAAZ0vB,KACIqR,MAAQA,IACRC,IAAMA,MAEND,MAAQrR,EAAQqR,OAASA,IACzBC,IAAMtR,EAAQsR,KAAOA,KAGvBsc,SAAU,IAEVnE,KAAOA,IACPoE,gBAAkB,IAClBC,YAAc,IACdzC,cAAe,IACfC,YAAa,IAGbyC,eAAiB,CACpBC,KAAK,EACL3c,MAAO,KACPC,IAAK,KACLrW,OAAAA,GACA9L,UAAW,aACX8+B,UAAU,EACVC,UAAU,EACVj7C,IAAK,KACLkD,IAAK,KACLg4C,QAAS,GACTC,QAAS,SACTC,YAAa,CACXC,QAAQ,EACRjhC,OAAQ,OAGP2S,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/B7H,MAAQ,CACXsI,MAAO,MAEJC,eAAiB,OAGjBhF,KAAKiF,QAAQvH,GAAG,WAAYP,OAAK+H,+BACjClF,KAAKiF,QAAQvH,GAAG,UAAYP,OAAKgI,0BACjCnF,KAAKiF,QAAQvH,GAAG,SAAYP,OAAKiI,6BAGjCpF,KAAKiF,QAAQvH,GAAG,aAAcP,OAAKkI,gCAGnCrF,KAAKiF,QAAQvH,GAAG,QAASP,OAAKmI,2BAC9BtF,KAAKiF,QAAQvH,GAAG,QAASP,OAAKoI,2BAG9BvF,KAAKwF,IAAIC,eAAeC,iBAAiB,QAASvI,OAAKwI,+BAEvDC,WAAWrvB,2DAmBPA,MACLA,EAAS,CAMXmpB,GAAKmG,gBAJU,CACb,YAAa,YAAa,MAAO,MAAO,UAAW,UAAW,WAAY,WAC1E,SAAU,WAAY,cAAe,UAAW,eAAgB,MAAO,kBAAmB,cAAe,oBAE9Ez+C,KAAKmvB,QAASA,GAEvCA,EAAQquB,aAAeruB,EAAQquB,YAAYC,aACxCc,gBAEH,UAAWpvB,GAAW,QAASA,SAE5BuvB,SAASvvB,EAAQqR,MAAOrR,EAAQsR,iDASnCke,EAAK3+C,eAKFm2C,IACPwI,EAAGC,cACHD,EAAG5B,SAAU,MAGT8B,EAAWF,EAAGle,IAAMke,EAAGne,MACrBzgC,EAAIu4C,GAAKpB,QAAQ,IAAIx6C,KAAQ,QAAQa,UACrCuhD,EAAoBH,EAAGxvB,QAAQquB,aAAemB,EAAGxvB,QAAQquB,YAAYhhC,QAAU,GAE/EgkB,EAAQzgC,EAAI8+C,EAAYC,EACxBre,EAAM1gC,EAAI8+C,GAAY,EAAIC,GAKhCH,EAAGD,SAASle,EAAOC,EAHH,CACdse,WAAW,KAMbF,EAAW,EADGF,EAAGjD,WAAWiD,EAAG/F,KAAKY,SAASwF,OAAOpW,OAAO+S,MACpC,IACR,KAAMkD,EAAW,IAC5BA,EAAW,MAAMA,EAAW,KAEhCF,EAAG/F,KAAKwF,IAAIC,eAAe7pB,MAAMyqB,WAAa,SAE9CN,EAAGO,iBAAmBC,GAAWhJ,EAAQ0I,GAG3C1I,8CAO8B12C,IAA1BO,KAAKk/C,mBACPE,aAAap/C,KAAKk/C,uBACbnC,SAAU,OACVnE,KAAKwF,IAAIC,eAAe7pB,MAAMyqB,WAAa,4CAyB3Cze,EAAOC,EAAKtR,EAASnpB,EAAUq5C,GACjClwB,IACHA,EAAU,KAEW,IAAnBA,EAAQmwB,SACVnwB,EAAQmwB,QAAS,OAEbX,EAAK3+C,KACLu/C,EAAsB9/C,MAAT+gC,EAAqB8X,GAAKpB,QAAQ1W,EAAO,QAAQjjC,UAAY,KAC1EiiD,EAAoB//C,MAAPghC,EAAqB6X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAAc,aAC3EkiD,wBACA3C,+BAA4Br9C,EAE7B0vB,EAAQ4vB,UAAW,OACfW,EAAY1/C,KAAKwgC,MACjBmf,EAAU3/C,KAAKygC,IACfplB,EAAyC,WAA7BoqB,GAAOtW,EAAQ4vB,YAA0B,aAAc5vB,EAAQ4vB,UAAa5vB,EAAQ4vB,UAAU1jC,SAAW,IACrHukC,EAA2C,WAA7Bna,GAAOtW,EAAQ4vB,YAA0B,mBAAoB5vB,EAAQ4vB,UAAa5vB,EAAQ4vB,UAAUc,eAAiB,gBACnIA,EAAiBvH,GAAKwH,gBAAgBF,OACvCC,QACG,IAAIz8C,6CAAiC28C,GAAeH,8BAA6BI,GAAY1H,GAAKwH,iBAAiB38C,KAAK,YAG1H88C,EAAWjZ,KACbkZ,GAAa,SAEJ,SAAPlsC,QACC2qC,EAAGtJ,MAAMsI,MAAMwC,SAAU,KAEtBz/B,EADMsmB,KACOiZ,EACbG,EAAOP,EAAen/B,EAAOrF,GAC7B8hB,EAAOzc,EAAOrF,EACdxS,EAAKs0B,GAAuB,OAAfoiB,EAAuBA,EAAaG,GAAaH,EAAaG,GAAaU,EACxF/sC,EAAK8pB,GAAuB,OAAfqiB,EAAuBA,EAAaG,GAAaH,EAAaG,GAAaS,EAE9FC,EAAU1B,EAAGjE,YAAY7xC,EAAGwK,GAC5BitC,GAA2B3B,EAAGxvB,QAAQ/E,OAAQu0B,EAAG/F,KAAM+F,EAAGxvB,QAAQiqB,aAClE8G,EAAaA,GAAcG,MAErBE,EAAS,CACb/f,MAAO,IAAI9jC,KAAKiiD,EAAGne,OACnBC,IAAK,IAAI/jC,KAAKiiD,EAAGle,KACjB6e,OAAQnwB,EAAQmwB,OAChBkB,MAAOrxB,EAAQqxB,UAGbnB,GAAiBA,EAAce,EAAMC,EAASljB,GAE9CkjB,GACF1B,EAAG/F,KAAKiF,QAAQ4C,KAAK,cAAeF,GAGlCpjB,MACE+iB,IACFvB,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgBF,GACjCv6C,UAAmBA,SAMzB24C,EAAGf,eAAiBuB,GAAWnrC,EAAM,KAKpCA,OAGHqsC,EAAUrgD,KAAK06C,YAAY6E,EAAYC,MAC3Cc,GAA2BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aACpEiH,EAAS,KACLE,EAAS,CACb/f,MAAO,IAAI9jC,KAAKsD,KAAKwgC,OACrBC,IAAK,IAAI/jC,KAAKsD,KAAKygC,KACnB6e,OAAQnwB,EAAQmwB,OAChBkB,MAAOrxB,EAAQqxB,eAGZ5H,KAAKiF,QAAQ4C,KAAK,cAAeF,GACtCnB,aAAcT,EAAG+B,WACjB/B,EAAG+B,UAAYvB,IAAY,WACzBR,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgBF,KACpC,KACCv6C,SAAmBA,mEAWYvG,IAAnCO,KAAK88C,iCACFA,2BAA6B98C,KAAKygC,IAAMzgC,KAAKwgC,OAASxgC,KAAK44C,KAAKwF,IAAIY,OAAO2B,aAE3E3gD,KAAK88C,qEAQR98C,KAAK49C,iBACPwB,aAAap/C,KAAK49C,qBACbA,eAAiB,0CAadpd,EAAOC,OAKbtjB,EAJAyjC,EAAqB,MAATpgB,EAAiB8X,GAAKpB,QAAQ1W,EAAO,QAAQjjC,UAAYyC,KAAKwgC,MAC1EqgB,EAAmB,MAAPpgB,EAAiB6X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAAcyC,KAAKygC,IACxEn7B,EAA2B,MAApBtF,KAAKmvB,QAAQ7pB,IAAegzC,GAAKpB,QAAQl3C,KAAKmvB,QAAQ7pB,IAAK,QAAQ/H,UAAY,KACtF6E,EAA2B,MAApBpC,KAAKmvB,QAAQ/sB,IAAek2C,GAAKpB,QAAQl3C,KAAKmvB,QAAQ/sB,IAAK,QAAQ7E,UAAY,QAIxF6B,MAAMwhD,IAA0B,OAAbA,QACf,IAAIx9C,+BAAwBo9B,WAEhCphC,MAAMyhD,IAAsB,OAAXA,QACb,IAAIz9C,6BAAsBq9B,WAI9BogB,EAASD,IACXC,EAASD,GAIC,OAARx+C,GACEw+C,EAAWx+C,IAEbw+C,GADAzjC,EAAQ/a,EAAMw+C,EAEdC,GAAU1jC,EAGC,MAAP7X,GACEu7C,EAASv7C,IACXu7C,EAASv7C,IAOL,OAARA,GACEu7C,EAASv7C,IAEXs7C,GADAzjC,EAAQ0jC,EAASv7C,EAEjBu7C,GAAU1jC,EAGC,MAAP/a,GACEw+C,EAAWx+C,IACbw+C,EAAWx+C,IAOU,OAAzBpC,KAAKmvB,QAAQmuB,QAAkB,KAC7BA,EAAUwD,GAAW9gD,KAAKmvB,QAAQmuB,YAClCA,EAAU,IACZA,EAAU,GAEPuD,EAASD,EAAYtD,EAAS,CAG5Bt9C,KAAKygC,IAAMzgC,KAAKwgC,QAAW8c,GAAWsD,GAAY5gD,KAAKwgC,MADvC,IAC+DqgB,GAAU7gD,KAAKygC,KAEjGmgB,EAAW5gD,KAAKwgC,MAChBqgB,EAAS7gD,KAAKygC,MAKdmgB,IADAzjC,EAAQmgC,GAAWuD,EAASD,IACT,EACnBC,GAAU1jC,EAAO,OAMM,OAAzBnd,KAAKmvB,QAAQouB,QAAkB,KAC7BA,EAAUuD,GAAW9gD,KAAKmvB,QAAQouB,SAClCA,EAAU,IACZA,EAAU,GAGPsD,EAASD,EAAYrD,IACnBv9C,KAAKygC,IAAMzgC,KAAKwgC,QAAW+c,GAAWqD,EAAW5gD,KAAKwgC,OAASqgB,EAAS7gD,KAAKygC,KAEhFmgB,EAAW5gD,KAAKwgC,MAChBqgB,EAAS7gD,KAAKygC,MAKdmgB,IADAzjC,EAAS0jC,EAASD,EAAYrD,GACX,EACnBsD,GAAU1jC,EAAO,QAKjBkjC,EAAWrgD,KAAKwgC,OAASogB,GAAY5gD,KAAKygC,KAAOogB,SAGhDD,GAAY5gD,KAAKwgC,OAASogB,GAAc5gD,KAAKygC,KAASogB,GAAY7gD,KAAKwgC,OAASqgB,GAAY7gD,KAAKygC,KACjGzgC,KAAKwgC,OAASogB,GAAY5gD,KAAKwgC,OAASqgB,GAAc7gD,KAAKygC,KAAOmgB,GAAc5gD,KAAKygC,KAAOogB,QAC5FjI,KAAKiF,QAAQ4C,KAAK,yBAGpBjgB,MAAQogB,OACRngB,IAAMogB,EACJR,2CAQA,CACL7f,MAAOxgC,KAAKwgC,MACZC,IAAKzgC,KAAKygC,wCAWHmI,EAAOmY,UACTlE,EAAMnB,WAAW17C,KAAKwgC,MAAOxgC,KAAKygC,IAAKmI,EAAOmY,wCAmC1CP,QACNxD,gBAAkB,OAClBgE,cAAgB,EAGhBhhD,KAAKmvB,QAAQiuB,UAGbp9C,KAAKihD,eAAeT,IAIpBxgD,KAAKq1C,MAAMsI,MAAMuD,qBAEjBtC,mBAEAvJ,MAAMsI,MAAMnd,MAAQxgC,KAAKwgC,WACzB6U,MAAMsI,MAAMld,IAAMzgC,KAAKygC,SACvB4U,MAAMsI,MAAMwC,UAAW,EAExBngD,KAAK44C,KAAKwF,IAAI+C,YACXvI,KAAKwF,IAAI+C,KAAK3sB,MAAM4sB,OAAS,yCAS9BZ,MACDA,GAEAxgD,KAAKq1C,MAAMsI,MAAMwC,UAGjBngD,KAAKmvB,QAAQiuB,UAKbp9C,KAAKq1C,MAAMsI,MAAMuD,mBAEhB5iC,EAAYte,KAAKmvB,QAAQ7Q,UAC/B+iC,GAAkB/iC,OACdgjC,EAAsB,cAAbhjC,EAA6BkiC,EAAMe,OAASf,EAAMgB,OAC/DF,GAASthD,KAAKg9C,oBACV6B,EAAY7+C,KAAKq1C,MAAMsI,MAAMld,IAAMzgC,KAAKq1C,MAAMsI,MAAMnd,MAIxDqe,GADiByB,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,SAIvFghB,EADE7Y,EAAsB,cAAbtqB,EAA6Bte,KAAK44C,KAAKY,SAASwF,OAAOpW,MAAQ5oC,KAAK44C,KAAKY,SAASwF,OAAOnW,OAGtG4Y,EADEzhD,KAAKmvB,QAAQguB,IACHmE,EAAQ1Y,EAAQiW,GAEfyC,EAAQ1Y,EAAQiW,MAGzB+B,EAAW5gD,KAAKq1C,MAAMsI,MAAMnd,MAAQihB,EACpCZ,EAAS7gD,KAAKq1C,MAAMsI,MAAMld,IAAMghB,EAGhCC,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAU5gD,KAAKghD,cAAcM,GAAO,GACnGK,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,EAAQ7gD,KAAKghD,cAAcM,GAAO,MACjGI,GAAad,GAAYe,GAAWd,cACjC7D,iBAAmBsE,OACnBjM,MAAMsI,MAAMnd,MAAQkhB,OACpBrM,MAAMsI,MAAMld,IAAMkhB,YAClB5D,QAAQyC,QAIVQ,cAAgBM,OAChB5G,YAAYkG,EAAUC,OAGrBjH,EAAY,IAAIl9C,KAAKsD,KAAKwgC,OAC1BqZ,EAAU,IAAIn9C,KAAKsD,KAAKygC,UAGzBmY,KAAKiF,QAAQ4C,KAAK,cAAe,CACpCjgB,MAAOoZ,EACPnZ,IAAOoZ,EACPyF,QAAQ,EACRkB,MAAAA,SAIG5H,KAAKiF,QAAQ4C,KAAK,+CAQdD,GACJxgD,KAAKq1C,MAAMsI,MAAMwC,UAGjBngD,KAAKmvB,QAAQiuB,UAKbp9C,KAAKq1C,MAAMsI,MAAMuD,qBAEjB7L,MAAMsI,MAAMwC,UAAW,EACxBngD,KAAK44C,KAAKwF,IAAI+C,YACXvI,KAAKwF,IAAI+C,KAAK3sB,MAAM4sB,OAAS,aAI/BxI,KAAKiF,QAAQ4C,KAAK,eAAgB,CACrCjgB,MAAO,IAAI9jC,KAAKsD,KAAKwgC,OACrBC,IAAO,IAAI/jC,KAAKsD,KAAKygC,KACrB6e,QAAQ,EACRkB,MAAAA,2CAUUA,OAERc,EAAQ,KACRd,EAAMoB,WACRN,EAAQd,EAAMoB,WAAa,IAClBpB,EAAMqB,OAGfP,GAASd,EAAMqB,OAAS,EACfrB,EAAMgB,SACfF,GAASd,EAAMgB,OAAS,KAIrBxhD,KAAKmvB,QAAQ2yB,UAAYtB,EAAMxgD,KAAKmvB,QAAQ2yB,UAAY9hD,KAAKmvB,QAAQkuB,WACnEr9C,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,WAKvCp9C,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,UAGvCp9C,KAAKihD,eAAeT,IAKrBc,EAAO,KAOL3F,EASAoG,EAVEC,EAAehiD,KAAKmvB,QAAQ6yB,cAAgB,KAGhDrG,EADE2F,EAAQ,EACF,EAAKA,EAAQU,EAGb,GAAK,EAAKV,EAAQU,GAKxBhiD,KAAK+8C,QAAS,KACV+B,EAAoB9+C,KAAKmvB,QAAQquB,aAAex9C,KAAKmvB,QAAQquB,YAAYhhC,QAAU,GACzFulC,EAAc/hD,KAAKwgC,OAAUxgC,KAAKygC,IAAMzgC,KAAKwgC,OAASse,MACjD,KACCmD,EAAUjiD,KAAKkiD,WAAW,CAAC/3C,EAAGq2C,EAAM2B,QAAS54C,EAAGi3C,EAAM4B,SAAUpiD,KAAK44C,KAAKwF,IAAIY,QACpF+C,EAAc/hD,KAAKqiD,eAAeJ,QAE/BK,KAAK3G,EAAOoG,EAAaT,EAAOd,GAIrCA,EAAM+B,mDASD/B,QACFnL,MAAMsI,MAAMnd,MAAQxgC,KAAKwgC,WACzB6U,MAAMsI,MAAMld,IAAMzgC,KAAKygC,SACvB4U,MAAMsI,MAAMuD,eAAgB,OAC5B7L,MAAMsI,MAAMqB,OAAS,UACrB3J,MAAMsI,MAAM6E,WAAa,UACzBvF,YAAc,OACdD,gBAAkB,EAEvB1E,GAAKiK,eAAe/B,oCAQbA,MAEDxgD,KAAKmvB,QAAQkuB,UAAYr9C,KAAKmvB,QAAQiuB,UAG5C9E,GAAKiK,eAAe/B,QAEfnL,MAAMsI,MAAMuD,eAAgB,EAE5BlhD,KAAKq1C,MAAMsI,MAAMqB,cACf3J,MAAMsI,MAAMqB,OAASh/C,KAAKkiD,WAAW1B,EAAMxB,OAAQh/C,KAAK44C,KAAKwF,IAAIY,aACjE3J,MAAMsI,MAAM6E,WAAaxiD,KAAKqiD,eAAeriD,KAAKq1C,MAAMsI,MAAMqB,cAGhEJ,kBACCjD,EAAQ,GAAK6E,EAAM7E,MAAQ37C,KAAKi9C,aAChCuF,EAAaxiD,KAAKq1C,MAAMsI,MAAM6E,WAE9BtG,EAAiBoE,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,KAC3FgiB,EAAuBnC,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAMwiD,GAC1GE,EAAsBxG,EAAiBuG,EAGzC7B,EAAY4B,EAAaC,GAAyBziD,KAAKq1C,MAAMsI,MAAMnd,OAASgiB,EAAaC,IAAyB9G,EAClHkF,EAAU2B,EAAaE,GAAwB1iD,KAAKq1C,MAAMsI,MAAMld,KAAO+hB,EAAaE,IAAwB/G,OAG3GnB,aAAe,EAAImB,GAAS,OAC5BlB,WAAakB,EAAQ,GAAK,MAEzB+F,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAU,EAAIjF,GAAO,GACpFgG,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,EAAQlF,EAAQ,GAAG,GAClF+F,GAAad,GAAYe,GAAWd,SACjCxL,MAAMsI,MAAMnd,MAAQkhB,OACpBrM,MAAMsI,MAAMld,IAAMkhB,OAClB1E,YAAc,EAAIuD,EAAM7E,MAC7BiF,EAAWc,EACXb,EAASc,OAGLxyB,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9B,SAASkC,EAAUC,EAAQ1xB,QAE3BqrB,cAAe,OACfC,YAAa,0CAUL+F,OAGP2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDQ,EAAsB3iD,KAAK44C,KAAKwF,IAAI3E,gBAAgBmJ,wBACpDz4C,EAAInK,KAAKmvB,QAAQguB,IAAMgF,EAAUQ,EAAoBjO,KAAOiO,EAAoBhO,MAAQwN,EACxFzhC,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,UAE5BuW,GAAQ1gB,KAAKwgC,OAAS9f,GAAQ1gB,KAAKygC,2CAS7BwhB,OACTvG,EACEp9B,EAAYte,KAAKmvB,QAAQ7Q,aAE/B+iC,GAAkB/iC,GAED,cAAbA,SACKte,KAAK44C,KAAKN,KAAK2D,OAAOgG,EAAQ93C,GAAG5M,cAGlCsrC,EAAS7oC,KAAK44C,KAAKY,SAASwF,OAAOnW,cACzC6S,EAAa17C,KAAK07C,WAAW7S,GACtBoZ,EAAQ14C,EAAImyC,EAAWC,MAAQD,EAAWl/B,0CAW1CmhC,EAAO/V,OACVib,EAAcjb,EAAQgb,+BACxB5iD,KAAKmvB,QAAQguB,IACR,CACLhzC,EAAG04C,EAAYlO,MAAQgJ,EAAMxzC,EAC7BZ,EAAGo0C,EAAMp0C,EAAIs5C,EAAYC,KAGpB,CACL34C,EAAGwzC,EAAMxzC,EAAI04C,EAAYnO,KACzBnrC,EAAGo0C,EAAMp0C,EAAIs5C,EAAYC,kCAiB1BnH,EAAOqD,EAAQsC,EAAOd,GAEX,MAAVxB,IACFA,GAAUh/C,KAAKwgC,MAAQxgC,KAAKygC,KAAO,OAG/Byb,EAAiBoE,GAAkCtgD,KAAK44C,KAAKQ,YAAap5C,KAAKwgC,MAAOxgC,KAAKygC,KAC3FgiB,EAAuBnC,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAMg/C,GAC1G0D,EAAsBxG,EAAiBuG,EAGzC7B,EAAY5B,EAAOyD,GAAyBziD,KAAKwgC,OAASwe,EAAOyD,IAAyB9G,EAC1FkF,EAAY7B,EAAO0D,GAAwB1iD,KAAKygC,KAAOue,EAAO0D,IAAwB/G,OAGrFnB,eAAe8G,EAAQ,QACvB7G,cAAc6G,EAAS,OACtBI,EAAYpB,GAA4BtgD,KAAK44C,KAAKQ,YAAawH,EAAUU,GAAO,GAChFK,EAAUrB,GAA4BtgD,KAAK44C,KAAKQ,YAAayH,GAASS,GAAO,GAC/EI,GAAad,GAAYe,GAAWd,IACtCD,EAAWc,EACXb,EAASc,OAGLxyB,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9B,SAASkC,EAAUC,EAAQ1xB,QAE3BqrB,cAAe,OACfC,YAAa,+BASf6G,OAEGnkC,EAAQnd,KAAKygC,IAAMzgC,KAAKwgC,MAGxBogB,EAAW5gD,KAAKwgC,MAAQrjB,EAAOmkC,EAC/BT,EAAS7gD,KAAKygC,IAAMtjB,EAAOmkC,OAI5B9gB,MAAQogB,OACRngB,IAAMogB,iCAONkC,OAGC5lC,GAFUnd,KAAKwgC,MAAQxgC,KAAKygC,KAAO,EAEnBsiB,EAGhBnC,EAAW5gD,KAAKwgC,MAAQrjB,EACxB0jC,EAAS7gD,KAAKygC,IAAMtjB,OAOrBuhC,SAASkC,EAAUC,EALR,CACd9B,WAAW,EACXO,QAAQ,EACRkB,MAAO,4CA/aOhgB,EAAOC,EAAKmI,EAAOmY,eACfthD,IAAhBshD,IACFA,EAAc,GAEH,GAATnY,GAAenI,EAAMD,GAAS,EACzB,CACLhkB,OAAQgkB,EACRmb,MAAO/S,GAASnI,EAAMD,EAAQugB,IAIzB,CACLvkC,OAAQ,EACRm/B,MAAO,UAtcoBhD,IAk3BnC,SAAS0I,GAAmB/iC,MACT,cAAbA,GAA0C,YAAbA,QACzB,IAAI0O,uCAAgC1O,4CC33B9C,OAAiB2R,EAAK2jB,uCCYboP,EAAQ3hD,MACXA,EAAK,gBAWIA,OACR,IAAI4B,KAAO+/C,EAAQ7mD,UACtBkF,EAAI4B,GAAO+/C,EAAQ7mD,UAAU8G,UAExB5B,EAfS4hD,CAAM5hD,GAVtB5F,UAAiBunD,EAqCnBA,EAAQ7mD,UAAUm6C,GAClB0M,EAAQ7mD,UAAUmiD,iBAAmB,SAASkC,EAAO3jD,eAC9CqmD,WAAaljD,KAAKkjD,YAAc,IACpCljD,KAAKkjD,WAAW,IAAM1C,GAASxgD,KAAKkjD,WAAW,IAAM1C,IAAU,IAC7DvjD,KAAKJ,GACDmD,MAaTgjD,EAAQ7mD,UAAUgnD,KAAO,SAAS3C,EAAO3jD,YAC9By5C,SACFC,IAAIiK,EAAOlK,GAChBz5C,EAAGhB,MAAMmE,KAAMlE,kBAGjBw6C,EAAGz5C,GAAKA,OACHy5C,GAAGkK,EAAOlK,GACRt2C,MAaTgjD,EAAQ7mD,UAAUo6C,IAClByM,EAAQ7mD,UAAUinD,eAClBJ,EAAQ7mD,UAAUknD,mBAClBL,EAAQ7mD,UAAUmnD,oBAAsB,SAAS9C,EAAO3jD,WACjDqmD,WAAaljD,KAAKkjD,YAAc,GAGjC,GAAKpnD,UAAUkB,mBACZkmD,WAAa,GACXljD,SAcLujD,EAVAC,EAAYxjD,KAAKkjD,WAAW,IAAM1C,OACjCgD,EAAW,OAAOxjD,QAGnB,GAAKlE,UAAUkB,qBACVgD,KAAKkjD,WAAW,IAAM1C,GACtBxgD,SAKJ,IAAIlD,EAAI,EAAGA,EAAI0mD,EAAUxmD,OAAQF,QACpCymD,EAAKC,EAAU1mD,MACJD,GAAM0mD,EAAG1mD,KAAOA,EAAI,CAC7B2mD,EAAUvQ,OAAOn2C,EAAG,gBAOC,IAArB0mD,EAAUxmD,eACLgD,KAAKkjD,WAAW,IAAM1C,GAGxBxgD,MAWTgjD,EAAQ7mD,UAAUskD,KAAO,SAASD,QAC3B0C,WAAaljD,KAAKkjD,YAAc,WAEjClgD,EAAO,IAAI/G,MAAMH,UAAUkB,OAAS,GACpCwmD,EAAYxjD,KAAKkjD,WAAW,IAAM1C,GAE7B1jD,EAAI,EAAGA,EAAIhB,UAAUkB,OAAQF,IACpCkG,EAAKlG,EAAI,GAAKhB,UAAUgB,MAGtB0mD,EAEG,CAAI1mD,EAAI,MAAR,IAAWmD,GADhBujD,EAAYA,EAAUtgD,MAAM,IACIlG,OAAQF,EAAImD,IAAOnD,EACjD0mD,EAAU1mD,GAAGjB,MAAMmE,KAAMgD,UAItBhD,MAWTgjD,EAAQ7mD,UAAUsnD,UAAY,SAASjD,eAChC0C,WAAaljD,KAAKkjD,YAAc,GAC9BljD,KAAKkjD,WAAW,IAAM1C,IAAU,IAWzCwC,EAAQ7mD,UAAUunD,aAAe,SAASlD,WAC9BxgD,KAAKyjD,UAAUjD,GAAOxjD,mCC7J5B2mD,EANFloD,WAMEkoD,EAAe,KAwBZ,SAASC,EAAYC,EAAQ10B,OAC9B20B,EAAW30B,GAAW,CACxBozB,gBAAgB,MAGdsB,EAAOE,QAAS,KAGdC,EAASH,EAETI,EAAoB,SAASrc,EAASzY,OACpC9kB,EAAInO,OAAO84B,OAAO8uB,UAClB30B,GAAS60B,EAAO5e,OAAO/6B,EAAG8kB,GACvBy0B,EAAY,IAAII,EAAOpc,EAASv9B,GAAIA,WAE7C25C,EAAO5e,OAAO6e,EAAmBD,GAEjCC,EAAkBF,QAAU,SAAUnc,EAASzY,OACzC9kB,EAAInO,OAAO84B,OAAO8uB,UAClB30B,GAAS60B,EAAO5e,OAAO/6B,EAAG8kB,GACvBy0B,EAAY,IAAII,EAAOD,QAAQnc,EAASv9B,GAAIA,IAG9C45C,MAKLlY,EAAU7vC,OAAO84B,OAAO6uB,GAGxBjc,EAAUic,EAAOjc,iBAiGZ/8B,EAAMq5C,UACNA,EAAO59C,MAAM,mBAOb69C,EAAkB3D,MAEN,iBAAfA,EAAMtpB,KAAyB,IAG5BspB,EAAM4D,SAASC,WAClB7D,EAAM4D,SAASC,SAAW,IAGxB7D,EAAM4D,SAASC,SAAS7D,EAAMtpB,aAIhCspB,EAAM4D,SAASC,SAAS7D,EAAMtpB,OAAQ,MAKtCqU,GAAU,EACdiV,EAAM8D,gBAAkB,WACtB/Y,GAAU,OAIRgZ,EAAU/D,EAAM4D,SAASE,gBAAgBj0B,KAAKmwB,EAAM4D,UACnC,mBAAXG,IACR/D,EAAM4D,SAASE,gBAAkB,WAC/BC,IACA/D,EAAM8D,oBAKV9D,EAAMgE,YAAcb,UAGhBc,EAAOd,EACJc,IAASlZ,GAAS,KACnBmZ,EAAaD,EAAKZ,UACnBa,UACGC,EACIhqC,EAAI,EAAGA,EAAI+pC,EAAW1nD,OAAQ2d,OACpCgqC,EAAYD,EAAW/pC,GAAGgqC,UAAUnE,EAAMtpB,MAC5B,IAAK,IAAIp6B,EAAI,EAAGA,EAAI6nD,EAAU3nD,SAAWuuC,EAASzuC,IAC9D6nD,EAAU7nD,GAAG0jD,GAInBiE,EAAOA,EAAKld,mBAvJZK,EAAQic,SAAQjc,EAAQic,OAAS,IACrCjc,EAAQic,OAAO5mD,KAAK8uC,GAIpB8X,EAAOvN,GAAG,gBAAgB,SAAUkK,IACF,IAA5BsD,EAASvB,gBAA4BuB,EAASvB,iBAAmB/B,EAAMoE,aACzEpE,EAAM+B,iBAEJ/B,EAAMqE,UACRlB,EAAenD,EAAM7wB,WAKzBoc,EAAQ4Y,UAAY,GAQpB5Y,EAAQuK,GAAK,SAAU4N,EAAQ1Q,UAE7B3oC,EAAMq5C,GAAQrwC,SAAQ,SAAU2sC,OAC1BmE,EAAY5Y,EAAQ4Y,UAAUnE,GAC7BmE,IACH5Y,EAAQ4Y,UAAUnE,GAASmE,EAAY,GAGvCd,EAAOvN,GAAGkK,EAAO2D,IAEnBQ,EAAU1nD,KAAKu2C,MAGVzH,GAWTA,EAAQwK,IAAM,SAAU2N,EAAQ1Q,UAE9B3oC,EAAMq5C,GAAQrwC,SAAQ,SAAU2sC,OAC1BmE,EAAY5Y,EAAQ4Y,UAAUnE,GAC9BmE,KACFA,EAAYnR,EAAUmR,EAAU1sB,QAAO,SAAUzrB,UACxCA,IAAMgnC,KACV,IAESx2C,OAAS,EACrB+uC,EAAQ4Y,UAAUnE,GAASmE,GAI3Bd,EAAOtN,IAAIiK,EAAO2D,UACXpY,EAAQ4Y,UAAUnE,QAKxBzU,GAQTA,EAAQ0U,KAAO,SAASqE,EAAWtE,GACjCmD,EAAenD,EAAM7wB,OACrBk0B,EAAOpD,KAAKqE,EAAWtE,IAGzBzU,EAAQgZ,QAAU,eAEZC,EAAUnB,EAAOjc,QAAQic,OACzBoB,EAAMD,EAAQt7C,QAAQqiC,IACd,IAATkZ,GAAYD,EAAQ/R,OAAOgS,EAAI,GAC9BD,EAAQhoD,eAAe6mD,EAAOjc,QAAQic,OAG1C9X,EAAQ4Y,UAAY,GAGpBd,EAAOkB,WAgEFhZ;;;;;;;6aCpMX,OAAAmZ,GAvBA,mBAAWhpD,OAAPkpC,OACO,SAAAzV,MACHA,MAAAA,QACI,IAAA3C,UAAN,sDAGEvmB,EAASvK,OAAbyzB,GACS/K,EAAT,EAAoBA,EAAQ9oB,UAA5BkB,OAA8C4nB,IAAS,KAC/CiB,EAAS/pB,UAAf8oB,MACIiB,MAAAA,MACG,IAALs/B,KAAAt/B,EACMA,EAAAA,eAAJs/B,KACE1+C,EAAAA,GAAkBof,EAAlBpf,WAKRA,GAGOvK,OAATkpC,utnBCFEggB,ygCAEkB,oBAAXp5B,OAETo5B,GAAiBnB,GADCj4B,OAAM,QAAcg4B,GACQ,CAC5CzB,eAAgB,UAGlB6C,GAAiB,iBAtBV,CACL9O,GAHI+O,EAAO,aAIX9O,IAAK8O,EACLN,QAASM,EACT5E,KAAM4E,EAENt7C,aAAIhM,SACK,CACLiG,IAAKqhD,KAXb,IACQA,UA4BOD,GClCR,SAASE,GAASzB,EAAQ79C,GAC/BA,EAASu/C,aAAe,SAAU/E,GAC5BA,EAAMqE,SACR7+C,EAASw6C,IAIbqD,EAAOvN,GAAG,eAAgBtwC,EAASu/C,kBCa/BC,yBAUQhlB,EAAOC,EAAKglB,EAAarM,EAAajqB,mBAC3C/E,OAAU+E,GAAWA,EAAQ/E,QAAWA,QACxC+E,QAAUA,GAAoB,QAG9B8rB,QAAUj7C,KAAKoqB,cACfs7B,OAAS1lD,KAAKoqB,cACd8wB,KAAOl7C,KAAKoqB,cAEZu7B,WAAa,OACbhK,MAAQ,WACR5X,KAAO,OAGP2a,SAASle,EAAOC,EAAKglB,QAGrBlK,aAAc,OACdD,eAAgB,OAChBD,cAAe,EAChBnY,GAAckW,QACXA,YAAcA,OAGdA,YADiB35C,MAAf25C,EACY,CAACA,GAGD,QAGhB37C,OAAS+nD,EAASI,oDAQfx7B,QACHA,OAASA,OAGT6wB,QAAUj7C,KAAKoqB,OAAOpqB,KAAKi7C,QAAQ19C,gBACnCmoD,OAAS1lD,KAAKoqB,OAAOpqB,KAAK0lD,OAAOnoD,gBACjC29C,KAAOl7C,KAAKoqB,OAAOpqB,KAAKk7C,KAAK39C,6CAS1BE,OACFwhB,EAAgBq5B,GAAKuN,WAAW,GAAIL,EAASI,aAC9CnoD,OAAS66C,GAAKuN,WAAW5mC,EAAexhB,oCAatC+iC,EAAOC,EAAKglB,QACbjlB,aAAiB9jC,MAAW+jC,aAAe/jC,WACxC,qDAGJgpD,OAAmBjmD,MAAT+gC,EAAsBxgC,KAAKoqB,OAAOoW,EAAMjjC,WAAaypC,UAC/DkU,KAAez7C,MAAPghC,EAAoBzgC,KAAKoqB,OAAOqW,EAAIljC,WAAaypC,KAE1DhnC,KAAK2lD,gBACFG,eAAeL,wCAQjBxK,QAAUj7C,KAAK0lD,OAAOtoC,aACtB2oC,6DAUa,QAAd/lD,KAAK27C,YACFV,QAAQztC,QAAQ,GAIfxN,KAAK27C,WACN,YACEV,QAAQ5xC,KAAKrJ,KAAK+jC,KAAOviC,KAAKE,MAAM1B,KAAKi7C,QAAQ5xC,OAASrJ,KAAK+jC,YAC/DkX,QAAQjxC,MAAM,OAChB,aAAqBixC,QAAQhxC,KAAK,OAClC,WACA,UACA,eAAqBgxC,QAAQ7qC,MAAM,OACnC,YAAqB6qC,QAAQ3qC,QAAQ,OACrC,cAAqB2qC,QAAQxqC,QAAQ,OACrC,cAAqBwqC,QAAQr/B,aAAa,MAIhC,GAAb5b,KAAK+jC,KAAW,KAEbiiB,EAAehmD,KAAKi7C,QAAQ79B,eACzBpd,KAAK27C,WACN,mBAAqBV,QAAQr8B,SAAS5e,KAAKi7C,QAAQr/B,eAAiB5b,KAAK+jC,KAAM,0BAC/E,cAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQxqC,UAAYzQ,KAAK+jC,KAAM,qBAC1E,cAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ3qC,UAAYtQ,KAAK+jC,KAAM,qBAC1E,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,mBACxE,cACA,WAAqBkX,QAAQr8B,UAAU5e,KAAKi7C,QAAQhxC,OAAS,GAAKjK,KAAK+jC,KAAM,iBAC7E,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ1tC,OAASvN,KAAK+jC,KAAM,kBACvE,aAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQjxC,QAAUhK,KAAK+jC,KAAM,mBACxE,YAAqBkX,QAAQr8B,SAAS5e,KAAKi7C,QAAQ5xC,OAASrJ,KAAK+jC,KAAM,QAGzEiiB,EAAa/jC,OAAOjiB,KAAKi7C,gBACrBA,QAAUj7C,KAAKoqB,OAAOk2B,GAA4BtgD,KAAKo5C,YAAap5C,KAAKi7C,QAAQ19C,WAAY,GAAG,+CAUnGyC,KAAKi7C,QAAQ19C,WAAayC,KAAKk7C,KAAK39C,6CAOtCmsC,EAAO1pC,KAAKi7C,QAAQ19C,iBAIlByC,KAAK27C,WACN,mBAAqBV,QAAQpgC,IAAI7a,KAAK+jC,KAAM,yBAC5C,cAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,oBAC5C,cAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,oBAC5C,YACEkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,QAExB/jC,KAAKi7C,QAAQjxC,QAAU,OACpBixC,QAAQr8B,SAAS5e,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,QAEpD/jC,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,MAAS,QAClCkX,QAAQpgC,IAAI7a,KAAK+jC,KAAO/jC,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,KAAM,kBAIhE,cACA,WAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,iBAC5C,UAC4B,IAA3B/jC,KAAKi7C,QAAQztC,eACVytC,QAAQztC,QAAQ,QAChBytC,QAAQpgC,IAAI7a,KAAK+jC,KAAM,aACvB,IAAoC,IAAjC/jC,KAAKmvB,QAAQ82B,qBAChBhL,QAAQpgC,IAAI7a,KAAK+jC,KAAM,YACvB,KACCxyB,EAAWvR,KAAKi7C,QAAQ79B,QAC9B7L,EAASsJ,IAAI,EAAG,QACbtJ,EAAS0Q,OAAOjiB,KAAKi7C,QAAS,cAC1BA,QAAQpgC,IAAI7a,KAAK+jC,KAAM,cAEvBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,aACvBkX,QAAQhxC,KAAK,cAInB,aAAqBgxC,QAAQpgC,IAAI7a,KAAK+jC,KAAM,mBAC5C,YAAqBkX,QAAQpgC,IAAI7a,KAAK+jC,KAAM,WAIlC,GAAb/jC,KAAK+jC,YAEC/jC,KAAK27C,WACN,cAAmB37C,KAAKi7C,QAAQr/B,eAAiB,GAAK5b,KAAKi7C,QAAQr/B,eAAiB5b,KAAK+jC,MAAM/jC,KAAKi7C,QAAQr/B,aAAa,aACzH,SAAmB5b,KAAKi7C,QAAQxqC,UAAY,GAAKzQ,KAAKi7C,QAAQxqC,UAAYzQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQxqC,QAAQ,aAC1G,SAAmBzQ,KAAKi7C,QAAQ3qC,UAAY,GAAKtQ,KAAKi7C,QAAQ3qC,UAAYtQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ3qC,QAAQ,aAC1G,OAAmBtQ,KAAKi7C,QAAQ7qC,QAAU,GAAKpQ,KAAKi7C,QAAQ7qC,QAAUpQ,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ7qC,MAAM,aACpG,cACA,MAAmBpQ,KAAKi7C,QAAQhxC,OAASjK,KAAK+jC,KAAK,GAAG/jC,KAAKi7C,QAAQhxC,KAAK,aACxE,OAAmBjK,KAAKi7C,QAAQ1tC,OAASvN,KAAK+jC,MAAM/jC,KAAKi7C,QAAQ1tC,KAAK,aACtE,QAAmBvN,KAAKi7C,QAAQjxC,QAAUhK,KAAK+jC,MAAM/jC,KAAKi7C,QAAQjxC,MAAM,GAO7EhK,KAAKi7C,QAAQ19C,WAAamsC,SACvBuR,QAAUj7C,KAAKk7C,KAAK99B,cAItBm+B,aAAc,OACdD,eAAgB,OAChBD,cAAe,EAEpBiF,GAA6BtgD,KAAKoqB,OAAQpqB,KAAM0pC,+CAQzC1pC,KAAKi7C,QAAQ79B,yCAebmjC,GACHA,GAAiC,iBAAhBA,EAAO5E,aACrBA,MAAQ4E,EAAO5E,WACf5X,KAAOwc,EAAOxc,KAAO,EAAIwc,EAAOxc,KAAO,OACvC4hB,WAAY,wCAQRO,QACNP,UAAYO,yCAOJT,MACMhmD,MAAfgmD,GAeAU,QAAgBV,SAA0B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAeV,SAA2B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAeV,SAA2B9J,MAAQ,YAAoB5X,KAAO,KAC7EoiB,QAAcV,SAA4B9J,MAAQ,YAAoB5X,KAAO,IAC7EoiB,QAAcV,SAA4B9J,MAAQ,YAAoB5X,KAAO,IAC7EoiB,QAAaV,SAA6B9J,MAAQ,YAAoB5X,KAAO,GAdzD,QAeT0hB,SAA+B9J,MAAQ,YAAoB5X,KAAO,GAC7EqiB,OAAcX,SAA4B9J,MAAQ,aAAoB5X,KAAO,GAfzD,OAgBR0hB,SAA8B9J,MAAQ,aAAoB5X,KAAO,GAC7EsiB,OAAYZ,GAAezlD,KAAKmvB,QAAQm3B,qBAAgC3K,MAAQ,YAAoB5X,KAAO,GAC3GsiB,OAAYZ,SAA8B9J,MAAQ,WAAoB5X,KAAO,GAjBzD,MAkBV0hB,SAAgC9J,MAAQ,WAAoB5X,KAAO,GAC7EsiB,MAAYZ,SAA8B9J,MAAQ,eAAoB5X,KAAO,GAC7EwiB,MAAad,SAA6B9J,MAAQ,YAAoB5X,KAAO,GAnBzD,KAoBT0hB,SAA+B9J,MAAQ,YAAoB5X,KAAO,GAC7EyiB,IAAgBf,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7EyiB,IAAgBf,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7EyiB,IAAef,SAA2B9J,MAAQ,cAAoB5X,KAAO,GAtBzD,IAuBP0hB,SAA6B9J,MAAQ,cAAoB5X,KAAO,GAC7E0iB,KAAgBhB,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7E0iB,IAAgBhB,SAA0B9J,MAAQ,cAAoB5X,KAAO,IAC7E0iB,IAAehB,SAA2B9J,MAAQ,cAAoB5X,KAAO,GAzBzD,IA0BP0hB,SAA6B9J,MAAQ,cAAoB5X,KAAO,GAC7E2iB,IAAsBjB,SAAoB9J,MAAQ,mBAAoB5X,KAAO,KAC7E2iB,IAAsBjB,SAAoB9J,MAAQ,mBAAoB5X,KAAO,KAC7E2iB,GAAqBjB,SAAqB9J,MAAQ,mBAAoB5X,KAAO,IAC7E2iB,GAAqBjB,SAAqB9J,MAAQ,mBAAoB5X,KAAO,IAC7E2iB,EAAoBjB,SAAsB9J,MAAQ,mBAAoB5X,KAAO,GA9BzD,EA+BF0hB,SAAwB9J,MAAQ,mBAAoB5X,KAAO,yCAqIxD,GAArB/jC,KAAKq7C,oBACCr7C,KAAK27C,WACN,WACA,YACA,WACA,cACA,UACA,WACA,aACA,aACA,qBACI,iBAEA,OAGR,GAA0B,GAAtB37C,KAAKs7C,qBACJt7C,KAAK27C,WACN,WACA,cACA,UACA,WACA,aACA,aACA,qBACI,iBAEA,OAGR,GAAwB,GAApB37C,KAAKu7C,mBACJv7C,KAAK27C,WACN,kBACA,aACA,aACA,cACI,iBAEA,MAIP1xC,EAAOjK,KAAKoqB,OAAOpqB,KAAKi7C,gBACtBj7C,KAAK27C,WACN,qBAC4B,GAAvB1xC,EAAK2R,mBACV,gBACuB,GAAlB3R,EAAKwG,cACV,gBACqB,GAAhBxG,EAAKmG,SAAoC,GAAlBnG,EAAKqG,cACjC,cACqB,GAAhBrG,EAAKmG,YACV,cACA,UAEA,cACoB,GAAfnG,EAAKA,WACV,eACqB,GAAhBA,EAAKD,YACV,sBAGI,yCAWCC,MACAxK,MAARwK,IACFA,EAAOjK,KAAKi7C,SAEVhxC,aAAgBvN,OAClBuN,EAAOjK,KAAKoqB,OAAOngB,IAGmB,mBAA7BjK,KAAKvC,OAAOkpD,mBACd3mD,KAAKvC,OAAOkpD,YAAY18C,EAAMjK,KAAK27C,MAAO37C,KAAK+jC,UAGlDtmC,EAASuC,KAAKvC,OAAOkpD,YAAY3mD,KAAK27C,cAEpC37C,KAAK27C,WACN,UAGgB,IAAhB1xC,EAAKA,QAAmC,IAAnBA,EAAKuD,gBAClB,kBAGH/P,GAAUA,EAAOT,OAAS,EAAKgD,KAAKoqB,OAAOngB,GAAMxM,OAAOA,GAAU,0CAWlEwM,MACAxK,MAARwK,IACFA,EAAOjK,KAAKi7C,SAEVhxC,aAAgBvN,OAClBuN,EAAOjK,KAAKoqB,OAAOngB,IAGmB,mBAA7BjK,KAAKvC,OAAOmpD,mBACd5mD,KAAKvC,OAAOmpD,YAAY38C,EAAMjK,KAAK27C,MAAO37C,KAAK+jC,UAGlDtmC,EAASuC,KAAKvC,OAAOmpD,YAAY5mD,KAAK27C,cACpCl+C,GAAUA,EAAOT,OAAS,EAAKgD,KAAKoqB,OAAOngB,GAAMxM,OAAOA,GAAU,gDAQpEopD,EAAU7mD,KAAKoqB,OACfrsB,EAAIiC,KAAKoqB,OAAOpqB,KAAKi7C,SACrBA,EAAUl9C,EAAEL,OAASK,EAAEL,OAAO,MAAQK,EAAEohB,KAAK,MAC7C4kB,EAAO/jC,KAAK+jC,KACZ+iB,EAAa,YAOVC,EAAKjlD,UACJA,EAAQiiC,EAAO,GAAK,EAAK,YAAc,oBAQxCijB,EAAM/8C,UACTA,EAAKgY,OAAO+kB,KAAY,OACnB,aAEL/8B,EAAKgY,OAAO4kC,IAAUhsC,IAAI,EAAG,OAAQ,OAChC,gBAEL5Q,EAAKgY,OAAO4kC,IAAUhsC,KAAK,EAAG,OAAQ,OACjC,iBAEF,YAQAosC,EAAYh9C,UACZA,EAAKgY,OAAO+kB,KAAY,QAAU,oBAAsB,YAQxDkgB,EAAaj9C,UACbA,EAAKgY,OAAO+kB,KAAY,SAAW,qBAAuB,UAY3DhnC,KAAK27C,WACN,cACHmL,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQr/B,2BAE1B,SACHkrC,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQxqC,sBAE1B,SACHq2C,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ3qC,sBAE1B,OACHw2C,EAAW7pD,yBAAag+C,EAAQ7qC,iBAAuB,GAAbpQ,KAAK+jC,KAAY,MAAQkX,EAAQ7qC,QAAU,GAAK,KAC1F02C,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ7qC,oBAE1B,UACH02C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAKgqD,EAAYhM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQhxC,mBAE1B,MACH68C,EAAW7pD,sBAAeg+C,EAAQhxC,SAClC68C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAK+pD,EAAM/L,IACtB6L,EAAW7pD,KAAKiqD,EAAajM,IAC7B6L,EAAW7pD,KAAK+C,KAAK+jC,MAAQ,EAAIijB,EAAM/L,GAAW,IAClD6L,EAAW7pD,KAAK+C,KAAK+jC,MAAQ,gBAAWkX,EAAQx9C,OAAO,QAAQ6G,eAAkB,IACjFwiD,EAAW7pD,KAAK8pD,EAAK9L,EAAQhxC,OAAS,cAEnC,OACH68C,EAAW7pD,uBAAgBg+C,EAAQx9C,OAAO,OAC1CqpD,EAAW7pD,KAAKgqD,EAAYhM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ1tC,mBAE1B,QACHu5C,EAAW7pD,mBAAYg+C,EAAQx9C,OAAO,QAAQ6G,gBAC9CwiD,EAAW7pD,KAAKiqD,EAAajM,IAC7B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQjxC,oBAE1B,OACH88C,EAAW7pD,uBAAgBg+C,EAAQ5xC,SACnCy9C,EAAW7pD,cAjDMgN,UACZA,EAAKgY,OAAO+kB,KAAY,QAAU,oBAAsB,GAgD7CmgB,CAAYlM,IAC5B6L,EAAW7pD,KAAK8pD,EAAK9L,EAAQ5xC,gBAG1BytC,GAAAgQ,QAAAA,EAAkBr4B,QAAQtrB,KAAK,oCApW5B8G,EAAM0xC,EAAO5X,OACjB3mB,EAAQgN,GAAOngB,MAER,QAAT0xC,EAAiB,KACbtyC,EAAO+T,EAAM/T,OAAS7H,KAAK+a,MAAMa,EAAMpT,QAAU,IACvDoT,EAAM/T,KAAK7H,KAAK+a,MAAMlT,EAAO06B,GAAQA,GACrC3mB,EAAMpT,MAAM,GACZoT,EAAMnT,KAAK,GACXmT,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,SAAT+/B,EACHv+B,EAAMnT,OAAS,IACjBmT,EAAMnT,KAAK,GACXmT,EAAMvC,IAAI,EAAG,UAIbuC,EAAMnT,KAAK,GAGbmT,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,QAAT+/B,EACDv+B,EAAM5P,UAAY,GAClB4P,EAAM5P,QAAQ,GACd4P,EAAMvC,IAAI,EAAG,SAGbuC,EAAM5P,QAAQ,GAGlB4P,EAAMhN,MAAM,GACZgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAElB,GAAa,OAAT+/B,EAAgB,QAEf5X,QACD,OACA,EACH3mB,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,mBAEvCgN,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,KAE3CgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,WAAT+/B,EAAoB,QAEnB5X,QACD,OACA,EACH3mB,EAAMhN,MAAuC,GAAjC5O,KAAK+a,MAAMa,EAAMhN,QAAU,mBAEvCgN,EAAMhN,MAAsC,EAAhC5O,KAAK+a,MAAMa,EAAMhN,QAAU,IAE3CgN,EAAM9M,QAAQ,GACd8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QAEhB,GAAa,QAAT+/B,EAAiB,QAChB5X,QACD,EACH3mB,EAAM9M,QAA2C,GAAnC9O,KAAK+a,MAAMa,EAAM9M,UAAY,mBAE3C8M,EAAM9M,QAA2C,GAAnC9O,KAAK+a,MAAMa,EAAM9M,UAAY,KAE/C8M,EAAM3M,QAAQ,GACd2M,EAAMxB,aAAa,QACd,GAAa,UAAT+/B,EAAmB,QAEpB5X,QACD,QACA,GACH3mB,EAAM9M,QAA0C,EAAlC9O,KAAK+a,MAAMa,EAAM9M,UAAY,IAC3C8M,EAAM3M,QAAQ,cAEX,EACH2M,EAAM3M,QAA2C,GAAnCjP,KAAK+a,MAAMa,EAAM3M,UAAY,mBAE3C2M,EAAM3M,QAA2C,GAAnCjP,KAAK+a,MAAMa,EAAM3M,UAAY,KAE/C2M,EAAMxB,aAAa,QAEhB,GAAa,UAAT+/B,SAEC5X,QACD,QACA,GACH3mB,EAAM3M,QAA0C,EAAlCjP,KAAK+a,MAAMa,EAAM3M,UAAY,IAC3C2M,EAAMxB,aAAa,cAEhB,EACHwB,EAAMxB,aAAuD,IAA1Cpa,KAAK+a,MAAMa,EAAMxB,eAAiB,oBAErDwB,EAAMxB,aAAsD,IAAzCpa,KAAK+a,MAAMa,EAAMxB,eAAiB,WAGtD,GAAa,eAAT+/B,EAAwB,KACzByL,EAAQrjB,EAAO,EAAIA,EAAO,EAAI,EACpC3mB,EAAMxB,aAAapa,KAAK+a,MAAMa,EAAMxB,eAAiBwrC,GAASA,UAGzDhqC,WC/cX,SAASiqC,GAAYC,EAAKC,QACX,IAARA,IAAiBA,EAAM,QACxBC,EAAWD,EAAIC,YAEdF,GAA2B,oBAAbl6B,cAEfq6B,EAAOr6B,SAASq6B,MAAQr6B,SAASs6B,qBAAqB,QAAQ,GAC9DlzB,EAAQpH,SAASE,cAAc,SACnCkH,EAAM0C,KAAO,WAEI,QAAbswB,GACEC,EAAKE,WACPF,EAAKzf,aAAaxT,EAAOizB,EAAKE,YAKhCF,EAAK9yB,YAAYH,GAGfA,EAAMozB,WACRpzB,EAAMozB,WAAWC,QAAUP,EAE3B9yB,EAAMG,YAAYvH,SAAS06B,eAAeR,KDkrB9C9B,GAASI,OAAS,CAChBe,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZX,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZX,KAAY,smFErtBV0+C,0BAQQnP,EAAMzpB,yDAEXivB,IAAM,CACT4J,WAAY,KACZC,MAAO,GACPC,WAAY,GACZC,WAAY,GACZ/gB,UAAW,CACT6gB,MAAO,GACPC,WAAY,GACZC,WAAY,OAGX9S,MAAQ,CACXqE,MAAO,CACLlZ,MAAO,EACPC,IAAK,EACLglB,YAAa,GAEf2C,QAAS,KAGNlL,eAAiB,CACpBmL,YAAa,CACXC,KAAM,UAERC,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfkC,cAAe,EACf/qD,OAAQ+nD,GAASI,OACjBx7B,OAAAA,GACAq+B,SAAU,QAEPt5B,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAE/BtE,KAAOA,IAGP8P,YAEAlK,WAAWrvB,2DAYPA,GACLA,IAEFmpB,GAAKmG,gBAAgB,CACnB,kBACA,kBACA,gBACA,gBACA,cACA,WACA,SACA,OACCz+C,KAAKmvB,QAASA,GAGjBmpB,GAAKqQ,oBAAoB,CAAC,UAAW3oD,KAAKmvB,QAASA,GAE/C,gBAAiBA,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAEF,WAA/B5iB,GAAOtW,EAAQk5B,cAA4B,SAAUl5B,EAAQk5B,mBAC/Dl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAAYC,OAMpD,WAAYn5B,IACe,mBAAlB/E,GAAO1sB,OAEhB0sB,GAAO1sB,OAAOyxB,EAAQzxB,QAGtB0sB,GAAOjL,KAAKgQ,EAAQzxB,iDAUrB0gD,IAAI4J,WAAa56B,SAASE,cAAc,YACxC8wB,IAAIwK,WAAax7B,SAASE,cAAc,YAExC8wB,IAAI4J,WAAW1f,UAAY,oCAC3B8V,IAAIwK,WAAWtgB,UAAY,iEAQ5BtoC,KAAKo+C,IAAI4J,WAAWzgB,iBACjB6W,IAAI4J,WAAWzgB,WAAWC,YAAYxnC,KAAKo+C,IAAI4J,YAElDhoD,KAAKo+C,IAAIwK,WAAWrhB,iBACjB6W,IAAIwK,WAAWrhB,WAAWC,YAAYxnC,KAAKo+C,IAAIwK,iBAGjDhQ,KAAO,0CAQNvD,EAAQr1C,KAAKq1C,MACb2S,EAAahoD,KAAKo+C,IAAI4J,WACtBY,EAAa5oD,KAAKo+C,IAAIwK,WAGtBh4B,EAA2C,OAAjC5wB,KAAKmvB,QAAQk5B,YAAYC,KAAiBtoD,KAAK44C,KAAKwF,IAAI0E,IAAM9iD,KAAK44C,KAAKwF,IAAIyK,OACtFC,EAAiBd,EAAWzgB,aAAe3W,OAG5Cm4B,yBAGCR,EAAkBvoD,KAAKmvB,QAAQo5B,iBAAqD,SAAlCvoD,KAAKmvB,QAAQk5B,YAAYC,KAC3ErC,EAAkBjmD,KAAKmvB,QAAQ82B,iBAAqD,SAAlCjmD,KAAKmvB,QAAQk5B,YAAYC,KAGjFjT,EAAM2T,iBAAmBT,EAAkBlT,EAAM4T,gBAAkB,EACnE5T,EAAM6T,iBAAmBjD,EAAkB5Q,EAAM8T,gBAAkB,EACnE9T,EAAMxM,OAASwM,EAAM2T,iBAAmB3T,EAAM6T,iBAC9C7T,EAAMzM,MAAQof,EAAWoB,YAEzB/T,EAAMgU,gBAAkBrpD,KAAK44C,KAAKY,SAAS2H,KAAKtY,OAASwM,EAAM6T,kBACzB,OAAjClpD,KAAKmvB,QAAQk5B,YAAYC,KAAgBtoD,KAAK44C,KAAKY,SAASqP,OAAOhgB,OAAS7oC,KAAK44C,KAAKY,SAASsJ,IAAIja,QACxGwM,EAAMiU,eAAiB,EACvBjU,EAAMkU,gBAAkBlU,EAAMgU,gBAAkBhU,EAAM6T,iBACtD7T,EAAMmU,eAAiB,MAGjBC,EAAwBzB,EAAW0B,YACnCC,EAAwBf,EAAWc,mBACzC1B,EAAWzgB,YAAcygB,EAAWzgB,WAAWC,YAAYwgB,GAC3DY,EAAWrhB,YAAcqhB,EAAWrhB,WAAWC,YAAYohB,GAE3DZ,EAAWxzB,MAAMqU,iBAAY7oC,KAAKq1C,MAAMxM,kBAEnC+gB,iBAGDH,EACF74B,EAAOoX,aAAaggB,EAAYyB,GAGhC74B,EAAO+D,YAAYqzB,GAEjB2B,OACG/Q,KAAKwF,IAAIyL,mBAAmB7hB,aAAa4gB,EAAYe,QAGrD/Q,KAAKwF,IAAIyL,mBAAmBl1B,YAAYi0B,GAExC5oD,KAAK8pD,cAAgBhB,+CAQtBT,EAAcroD,KAAKmvB,QAAQk5B,YAAYC,KAGvC9nB,EAAQ8X,GAAKpB,QAAQl3C,KAAK44C,KAAKc,MAAMlZ,MAAO,UAC5CC,EAAM6X,GAAKpB,QAAQl3C,KAAK44C,KAAKc,MAAMjZ,IAAK,UACxCspB,EAAgB/pD,KAAK44C,KAAKN,KAAK2D,QAAQj8C,KAAKq1C,MAAM2U,gBAAkB,IAAMhqD,KAAKmvB,QAAQq5B,eAAejrD,UACxGkoD,EAAcsE,EAAgBzJ,GAAiCtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAKQ,YAAap5C,KAAK44C,KAAKc,MAAOqQ,GAChItE,GAAezlD,KAAK44C,KAAKN,KAAK2D,OAAO,GAAG1+C,cAElCwmC,EAAO,IAAIyhB,GAAS,IAAI9oD,KAAK8jC,GAAQ,IAAI9jC,KAAK+jC,GAAMglB,EAAazlD,KAAK44C,KAAKQ,YAAap5C,KAAKmvB,SACnG4U,EAAKkmB,UAAUjqD,KAAKmvB,QAAQ/E,QACxBpqB,KAAKmvB,QAAQ1xB,QACfsmC,EAAKmmB,UAAUlqD,KAAKmvB,QAAQ1xB,QAE1BuC,KAAKmvB,QAAQs5B,UACf1kB,EAAKomB,SAASnqD,KAAKmvB,QAAQs5B,eAExB1kB,KAAOA,MAaRkX,EACAjnC,EACA7J,EACAigD,EACAC,EACAC,EAbElM,EAAMp+C,KAAKo+C,IACjBA,EAAIhX,UAAU6gB,MAAQ7J,EAAI6J,MAC1B7J,EAAIhX,UAAU8gB,WAAa9J,EAAI8J,WAC/B9J,EAAIhX,UAAU+gB,WAAa/J,EAAI+J,WAC/B/J,EAAI6J,MAAQ,GACZ7J,EAAI8J,WAAa,GACjB9J,EAAI+J,WAAa,OASboC,EACAC,EAIAliB,EANAM,EAAQ,EAGR6hB,OAAmBhrD,EACnBw5C,EAAQ,MAIZlV,EAAKvD,QACLxsB,EAAO+vB,EAAK2mB,aACZN,EAAQpqD,KAAK44C,KAAKN,KAAKkD,SAASxnC,GACzB+vB,EAAK4mB,WAAa1R,EANb,KAM0B,QACpCA,IAEAoR,EAAUtmB,EAAKsmB,UACf/hB,EAAYvE,EAAK6mB,eAEjB3P,EAAUjnC,EACV7J,EAAIigD,EAEJrmB,EAAK/vB,OACLA,EAAO+vB,EAAK2mB,aAGZH,EAAY3hB,EACZA,GAHAwhB,EAAQpqD,KAAK44C,KAAKN,KAAKkD,SAASxnC,IAGhB7J,EACR45B,EAAK4X,WACN,OAAgB2O,GAAgB,gBAChBA,EAAiB1hB,GAAqB,GAAZ2hB,KAG7CvqD,KAAKmvB,QAAQo5B,iBAAmB+B,EAAe,KAC7C/hB,EAAQvoC,KAAK6qD,kBAAkB1gD,EAAG45B,EAAK+mB,cAAc7P,GAAUoN,EAAa/f,GAChFC,EAAM/T,MAAMoU,gBAAWA,QAGrByhB,GAAWrqD,KAAKmvB,QAAQ82B,iBACtB97C,EAAI,IACkB1K,MAApBgrD,IACFA,EAAmBtgD,GAErBo+B,EAAQvoC,KAAK+qD,kBAAkB5gD,EAAG45B,EAAKinB,cAAc/P,GAAUoN,EAAa/f,IAE9EkiB,EAAOxqD,KAAKirD,kBAAkB9gD,EAAGy+B,EAAOyf,EAAa/f,IAGjDgiB,EACFE,EAAOxqD,KAAKkrD,kBAAkB/gD,EAAGy+B,EAAOyf,EAAa/f,GAGjDkiB,IAEFA,EAAKh2B,MAAMoU,gBAAWuiB,GAASX,EAAKh2B,MAAMoU,OAASA,YA/C/C,MAqDRqQ,GAAkBmS,KAClBzoD,QAAQH,4FAtDA,gBAuDR4oD,IAAoB,GAIpBprD,KAAKmvB,QAAQ82B,gBAAiB,KAC1BoF,EAAWrrD,KAAK44C,KAAKN,KAAK2D,OAAO,GACjCqP,EAAWvnB,EAAKinB,cAAcK,GAC9BE,EAAYD,EAAStuD,QAAUgD,KAAKq1C,MAAMmW,gBAAkB,IAAM,IAEhD/rD,MAApBgrD,GAAiCc,EAAYd,SAC1CM,kBAAkB,EAAGO,EAAUjD,EAAa/f,MAKrDgQ,SAAAA,GAAat4C,KAAKo+C,IAAIhX,WAAW,SAAAxqC,QACxBA,EAAII,QAAQ,KACXynD,EAAO7nD,EAAIo1C,MACbyS,GAAQA,EAAKld,YACfkd,EAAKld,WAAWC,YAAYid,iDAelBt6C,EAAGshD,EAAMpD,EAAa/f,OAElCC,EAAQvoC,KAAKo+C,IAAIhX,UAAU+gB,WAAWhjC,YAErCojB,EAAO,KAEJ1U,EAAUzG,SAAS06B,eAAe,KACxCvf,EAAQnb,SAASE,cAAc,QACzBqH,YAAYd,QACbuqB,IAAI4J,WAAWrzB,YAAY4T,QAE7B6V,IAAI+J,WAAWlrD,KAAKsrC,GACzBA,EAAMmjB,UAAYD,MAGdliD,EAAoB,OAAf8+C,EAAwBroD,KAAKq1C,MAAM6T,iBAAmB,cAC1DyC,OAAOpjB,EAAOp+B,EAAGZ,GAEtBg/B,EAAMD,uCAAkCA,GAGjCC,4CAYSp+B,EAAGshD,EAAMpD,EAAa/f,OAElCC,EAAQvoC,KAAKo+C,IAAIhX,UAAU8gB,WAAW/iC,YAErCojB,EAAO,KAEJ1U,EAAUzG,SAASE,cAAc,QACvCib,EAAQnb,SAASE,cAAc,QACzBqH,YAAYd,QACbuqB,IAAI4J,WAAWrzB,YAAY4T,GAGlCA,EAAMqjB,WAAW,GAAGF,UAAYD,EAChCljB,EAAMD,uCAAkCA,OAGpC/+B,EAAoB,OAAf8+C,EAAwB,EAAIroD,KAAKq1C,MAAM2T,6BAC3C2C,OAAOpjB,EAAOp+B,EAAGZ,QAEjB60C,IAAI8J,WAAWjrD,KAAKsrC,GAClBA,iCAUFA,EAAOp+B,EAAGZ,SAETsiD,EAAa7rD,KAAKmvB,QAAQguB,KAAY,EAALhzC,EAAUA,EACjDo+B,EAAM/T,MAAMkiB,mCAAyBmV,kBAAiBtiD,mDAYtCmrC,EAAM9L,EAAOyf,EAAa/f,SAEtCkiB,EAAOxqD,KAAKo+C,IAAIhX,UAAU6gB,MAAM9iC,QAC/BqlC,IAEHA,EAAOp9B,SAASE,cAAc,YACzB8wB,IAAIwK,WAAWj0B,YAAY61B,SAE7BpM,IAAI6J,MAAMhrD,KAAKutD,OAEdnV,EAAQr1C,KAAKq1C,MAEnBmV,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMqU,iBAAYwM,EAAMgU,0BAEzB9/C,EAAoB,OAAf8+C,EAAwBhT,EAAM6T,iBAAmBlpD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAC7E1+B,EAAIuqC,EAAOW,EAAMiU,eAAiB,cAEjCqC,OAAOnB,EAAMrgD,EAAGZ,GACrBihD,EAAKliB,kCAAwBtoC,KAAKmvB,QAAQguB,IAAO,mBAAqB,sCAA4B7U,GAE3FkiB,4CAYS9V,EAAM9L,EAAOyf,EAAa/f,SAEtCkiB,EAAOxqD,KAAKo+C,IAAIhX,UAAU6gB,MAAM9iC,QAC/BqlC,IAEHA,EAAOp9B,SAASE,cAAc,YACzB8wB,IAAIwK,WAAWj0B,YAAY61B,SAE7BpM,IAAI6J,MAAMhrD,KAAKutD,OAEdnV,EAAQr1C,KAAKq1C,MAEnBmV,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMqU,iBAAYwM,EAAMkU,0BAEzBhgD,EAAoB,OAAf8+C,EAAwB,EAAIroD,KAAK44C,KAAKY,SAASsJ,IAAIja,OACxD1+B,EAAIuqC,EAAOW,EAAMmU,eAAiB,cAEjCmC,OAAOnB,EAAMrgD,EAAGZ,GACrBihD,EAAKliB,kCAAwBtoC,KAAKmvB,QAAQguB,IAAO,mBAAqB,sCAA4B7U,GAE3FkiB,+CAaFxqD,KAAKo+C,IAAI0N,wBACP1N,IAAI0N,iBAAmB1+B,SAASE,cAAc,YAC9C8wB,IAAI0N,iBAAiBxjB,UAAY,sCACjC8V,IAAI0N,iBAAiBt3B,MAAMkL,SAAW,gBAEtC0e,IAAI0N,iBAAiBn3B,YAAYvH,SAAS06B,eAAe,WACzD1J,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI0N,wBAEtCzW,MAAM4T,gBAAkBjpD,KAAKo+C,IAAI0N,iBAAiBC,kBAClD1W,MAAM2U,eAAiBhqD,KAAKo+C,IAAI0N,iBAAiBnL,YAGjD3gD,KAAKo+C,IAAI4N,wBACP5N,IAAI4N,iBAAmB5+B,SAASE,cAAc,YAC9C8wB,IAAI4N,iBAAiB1jB,UAAY,sCACjC8V,IAAI4N,iBAAiBx3B,MAAMkL,SAAW,gBAEtC0e,IAAI4N,iBAAiBr3B,YAAYvH,SAAS06B,eAAe,WACzD1J,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI4N,wBAEtC3W,MAAM8T,gBAAkBnpD,KAAKo+C,IAAI4N,iBAAiBD,kBAClD1W,MAAMmW,eAAiBxrD,KAAKo+C,IAAI4N,iBAAiBrL,mBAtenChI,IA2enByS,IAAoB,sBCrepB3vD,mBAOgB0zB,OAQZryB,EAPAylD,EAAiBpzB,GAAWA,EAAQozB,iBAAkB,EAEtD0J,EAAY98B,GAAWA,EAAQ88B,WAAajgC,OAE5CkgC,EAAmB,GACnBC,EAAS,CAACC,QAAQ,GAAIC,MAAM,IAC5BC,EAAQ,OAIPxvD,EAAI,GAAIA,GAAK,IAAKA,IAAMwvD,EAAM79B,OAAO89B,aAAazvD,IAAM,CAAC0vD,KAAW1vD,EAAI,GAAV,GAAeqoB,OAAO,OAEpFroB,EAAI,GAAIA,GAAK,GAAIA,IAAMwvD,EAAM79B,OAAO89B,aAAazvD,IAAM,CAAC0vD,KAAK1vD,EAAGqoB,OAAO,OAEvEroB,EAAI,EAAIA,GAAK,EAAKA,IAAMwvD,EAAM,GAAKxvD,GAAK,CAAC0vD,KAAK,GAAK1vD,EAAGqoB,OAAO,OAE7DroB,EAAI,EAAIA,GAAK,GAAMA,IAAMwvD,EAAM,IAAMxvD,GAAK,CAAC0vD,KAAK,IAAM1vD,EAAGqoB,OAAO,OAEhEroB,EAAI,EAAIA,GAAK,EAAKA,IAAMwvD,EAAM,MAAQxvD,GAAK,CAAC0vD,KAAK,GAAK1vD,EAAGqoB,OAAO,GAGrEmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAClCmnC,EAAM,QAAU,CAACE,KAAK,IAAKrnC,OAAO,GAElCmnC,EAAK,KAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,GAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,KAAY,CAACE,KAAK,GAAIrnC,OAAO,GAElCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,MAAY,CAACE,KAAK,GAAIrnC,WAAO1lB,GAClC6sD,EAAK,IAAY,CAACE,KAAK,GAAIrnC,OAAO,GAClCmnC,EAAK,UAAgB,CAACE,KAAK,EAAGrnC,OAAO,GACrCmnC,EAAK,IAAgB,CAACE,KAAK,EAAGrnC,OAAO,GACrCmnC,EAAK,KAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,IAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,OAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,OAAgB,CAACE,KAAK,GAAIrnC,OAAO,GACtCmnC,EAAK,SAAgB,CAACE,KAAK,GAAIrnC,OAAO,GAEtCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,GACnCmnC,EAAM,KAAW,CAACE,KAAK,IAAKrnC,OAAO,OAI/BsnC,EAAO,SAASjM,GAAQkM,EAAYlM,EAAM,YAC1CmM,EAAK,SAASnM,GAAQkM,EAAYlM,EAAM,UAGxCkM,EAAc,SAASlM,EAAMtpB,WACKz3B,IAAhC0sD,EAAOj1B,GAAMspB,EAAMoM,SAAwB,SACzCC,EAAQV,EAAOj1B,GAAMspB,EAAMoM,SACtB9vD,EAAI,EAAGA,EAAI+vD,EAAM7vD,OAAQF,UACT2C,IAAnBotD,EAAM/vD,GAAGqoB,OAGc,GAAlB0nC,EAAM/vD,GAAGqoB,OAAmC,GAAlBq7B,EAAMsM,UAGd,GAAlBD,EAAM/vD,GAAGqoB,OAAoC,GAAlBq7B,EAAMsM,WALxCD,EAAM/vD,GAAGD,GAAG2jD,GAUM,GAAlB+B,GACF/B,EAAM+B,0BAMZ2J,EAAiB77B,KAAO,SAASptB,EAAK+C,EAAUkxB,WACjCz3B,IAATy3B,IACFA,EAAO,gBAEUz3B,IAAf6sD,EAAMrpD,SACF,IAAIG,MAAM,oBAAsBH,QAEFxD,IAAlC0sD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,QAC1BL,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQ,IAElCL,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAMvvD,KAAK,CAACJ,GAAGmJ,EAAUmf,MAAMmnC,EAAMrpD,GAAKkiB,SAKpE+mC,EAAiBa,QAAU,SAAS/mD,EAAUkxB,OAIvC,IAAIj0B,UAHIxD,IAATy3B,IACFA,EAAO,WAEOo1B,EACVA,EAAMjvD,eAAe4F,IACvBipD,EAAiB77B,KAAKptB,EAAI+C,EAASkxB,IAMzCg1B,EAAiBc,OAAS,SAASxM,OAC5B,IAAIv9C,KAAOqpD,KACVA,EAAMjvD,eAAe4F,GAAM,IACP,GAAlBu9C,EAAMsM,UAAwC,GAApBR,EAAMrpD,GAAKkiB,OAAiBq7B,EAAMoM,SAAWN,EAAMrpD,GAAKupD,YAC7EvpD,EAEJ,GAAsB,GAAlBu9C,EAAMsM,UAAyC,GAApBR,EAAMrpD,GAAKkiB,OAAkBq7B,EAAMoM,SAAWN,EAAMrpD,GAAKupD,YACpFvpD,EAEJ,GAAIu9C,EAAMoM,SAAWN,EAAMrpD,GAAKupD,MAAe,SAAPvpD,SACpCA,QAIN,wCAITipD,EAAiBe,OAAS,SAAShqD,EAAK+C,EAAUkxB,WACnCz3B,IAATy3B,IACFA,EAAO,gBAEUz3B,IAAf6sD,EAAMrpD,SACF,IAAIG,MAAM,oBAAsBH,WAEvBxD,IAAbuG,EAAwB,KACtBknD,EAAc,GACdL,EAAQV,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,cACtB/sD,IAAVotD,MACG,IAAI/vD,EAAI,EAAGA,EAAI+vD,EAAM7vD,OAAQF,IAC1B+vD,EAAM/vD,GAAGD,IAAMmJ,GAAY6mD,EAAM/vD,GAAGqoB,OAASmnC,EAAMrpD,GAAKkiB,OAC5D+nC,EAAYjwD,KAAKkvD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAM1vD,IAIrDqvD,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQU,OAGhCf,EAAOj1B,GAAMo1B,EAAMrpD,GAAKupD,MAAQ,IAKpCN,EAAiB5a,MAAQ,WACvB6a,EAAS,CAACC,QAAQ,GAAIC,MAAM,KAI9BH,EAAiBnH,QAAU,WACzBoH,EAAS,CAACC,QAAQ,GAAIC,MAAM,IAC5BJ,EAAU3I,oBAAoB,UAAWmJ,GAAM,GAC/CR,EAAU3I,oBAAoB,QAASqJ,GAAI,IAI7CV,EAAU3N,iBAAiB,UAAUmO,GAAK,GAC1CR,EAAU3N,iBAAiB,QAAQqO,GAAG,GAG/BT,MCzKX,SAASiB,GAAUlB,gBACZmB,QAAS,OAEThP,IAAM,CACT6N,UAAWA,QAGR7N,IAAIiP,QAAUjgC,SAASE,cAAc,YACrC8wB,IAAIiP,QAAQ/kB,UAAY,mBAExB8V,IAAI6N,UAAUt3B,YAAY30B,KAAKo+C,IAAIiP,cAEnCxJ,OAASG,GAAOhkD,KAAKo+C,IAAIiP,cACzBxJ,OAAOvN,GAAG,MAAOP,UAAKuX,sBAAmBttD,WAG1C2+C,EAAK3+C,KACLkkD,EAAS,CACX,MAAO,YAAa,QACpB,QACA,MAAO,WAAY,UAAW,aAEhCA,QAAAA,GAAe,SAAU1D,GACvB7B,EAAGkF,OAAOvN,GAAGkK,GAAO,SAAUA,GAC5BA,EAAM8D,wBAKNl3B,UAAYA,SAASwrB,YAClB2U,QAAU,SAAU/M,IA4G7B,SAAoB5Y,EAAShX,QACpBgX,GAAS,IACVA,IAAYhX,SACP,EAETgX,EAAUA,EAAQL,kBAEb,GAlHEimB,CAAWhN,EAAM7wB,OAAQs8B,IAC5BtN,EAAG8O,cAGPrgC,SAASwrB,KAAK0F,iBAAiB,QAASt+C,KAAKutD,eAGzB9tD,IAAlBO,KAAK0tD,eACFA,SAAS3I,eAEX2I,SAAWA,UAGXC,YAAc5X,UAAK0X,mBAAgBztD,kwBAIlCmtD,GAAUhxD,WAGlBgxD,GAAUlS,QAAU,KAKpBkS,GAAUhxD,UAAU4oD,QAAU,gBACvB0I,kBAGArP,IAAIiP,QAAQ9lB,WAAWC,YAAYxnC,KAAKo+C,IAAIiP,SAG7CrtD,KAAKutD,SACPngC,SAASwrB,KAAK0K,oBAAoB,QAAStjD,KAAKutD,cAG5B9tD,IAAlBO,KAAK0tD,eACFA,SAAS3I,eAEX2I,SAAW,UAEX7J,OAAOkB,eACPlB,OAAS,MAQhBsJ,GAAUhxD,UAAUyxD,SAAW,iBAEzBT,GAAUlS,SACZkS,GAAUlS,QAAQwS,aAEpBN,GAAUlS,QAAUj7C,UAEfotD,QAAS,OACThP,IAAIiP,QAAQ74B,MAAMC,QAAU,OACjC6jB,GAAKuV,aAAa7tD,KAAKo+C,IAAI6N,UAAW,mBAEjCxL,KAAK,eACLA,KAAK,sBAILiN,iBAAc,MAAO1tD,KAAK2tD,cAOjCR,GAAUhxD,UAAUsxD,WAAa,WAC3BN,GAAUlS,UAAYj7C,OACxBmtD,GAAUlS,QAAU,WAGjBmS,QAAS,OACThP,IAAIiP,QAAQ74B,MAAMC,QAAU,GACjC6jB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI6N,UAAW,mBACpCyB,SAAST,OAAO,MAAOjtD,KAAK2tD,kBAE5BlN,KAAK,eACLA,KAAK,eAQZ0M,GAAUhxD,UAAUmxD,cAAgB,SAAU9M,QAEvCoN,WACLpN,EAAM8D,mBCzIR,IAAMyJ,GAAK,CACT9S,QAAS,UACTv6B,KAAM,OACNstC,eAAgB,mBAMZliC,GAAK,CACTmvB,QAAS,UACTv6B,KAAM,QACNstC,eAAgB,yBAMZC,GAAK,CACThT,QAAS,UACTv6B,KAAM,OACNstC,eAAgB,wBAMZE,GAAK,CACTjT,QAAS,WACTv6B,KAAM,OACNstC,eAAgB,kBAKZG,GAAK,CACTlT,QAAS,SACTv6B,KAAM,QACNstC,eAAgB,wBAOZI,GAAK,CACTnT,QAAS,YACTv6B,KAAM,OACNstC,eAAgB,sBAKZK,GAAK,CACTpT,QAAS,WACTv6B,KAAM,MACNstC,eAAgB,mBAKZM,GAAK,CACTrT,QAAS,UACTv6B,KAAM,QACNstC,eAAgB,qBAKZO,GAAK,CACTtT,QAAS,WACTv6B,KAAM,OACNstC,eAAgB,gBAKZQ,GAAK,CACTvT,QAAS,KACTv6B,KAAM,KACNstC,eAAgB,cAIZp7C,GAAU,CACdm7C,GAAAA,GACAU,MAjFYV,GAkFZW,MAjFYX,GAkFZjiC,GAAAA,GACA6iC,MA3EY7iC,GA4EZ8iC,MA3EY9iC,GA4EZmiC,GAAAA,GACAY,MArEYZ,GAsEZa,MArEYb,GAsEZC,GAAAA,GACAa,MA/DYb,GAgEZC,GAAAA,GACAa,MAzDYb,GA0DZc,MAzDYd,GA0DZe,MAzDYf,GA0DZC,GAAAA,GACAe,MAnDYf,GAoDZC,GAAAA,GACAe,MA7CYf,GA8CZC,GAAAA,GACAe,MAvCYf,GAwCZC,GAAAA,GACAe,MAjCYf,GAkCZC,GAAAA,GACAe,MA3BYf,yiCC/ERgB,0BAUQ5W,EAAMzpB,oDAEXypB,KAAOA,IAGPsE,eAAiB,CACpB9yB,OAAAA,GACAxX,QAAAA,GACAlV,OAAQ,KACR01B,QAAI3zB,EACJgwD,WAAOhwD,KAEJ0vB,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/BsB,WAAWrvB,KACXA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS8qC,EAAKvuB,QAAQvc,aACvD88C,EAAiBhS,EAAKR,eAAetqC,QAAQ8qC,EAAKR,eAAex/C,uBAC3DggD,EAAKvuB,QAAQvc,kBAAiB,SAAAlV,KACnCyxB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,OAIrByxB,GAA2B,MAAhBA,EAAQzO,OAChBivC,WAAaxgC,EAAQzO,OAErBivC,WAAa,IAAIjzD,OAGnBkzD,YAAc,KAGdlH,kEAUIv5B,GACLA,GAEFmpB,GAAKmG,gBAAgB,CAAC,SAAU,SAAU,UAAW,KAAM,QAAS,MAAO,QAASz+C,KAAKmvB,QAASA,+CAS9F0gC,EAAMziC,SAASE,cAAc,OACnCuiC,EAAI,eAAiB7vD,KACrB6vD,EAAIvnB,oCAA+BtoC,KAAKmvB,QAAQiE,IAAM,IACtDy8B,EAAIr7B,MAAMkL,SAAW,WACrBmwB,EAAIr7B,MAAMsuB,IAAM,MAChB+M,EAAIr7B,MAAMqU,OAAS,YACdgnB,IAAMA,MAELC,EAAO1iC,SAASE,cAAc,gBAe3ByiC,EAAc18C,QAChBulC,KAAKc,MAAMuE,cAAc5qC,GAfhCy8C,EAAKt7B,MAAMkL,SAAW,WACtBowB,EAAKt7B,MAAMsuB,IAAM,MACd9iD,KAAKmvB,QAAQguB,IACd2S,EAAKt7B,MAAMmgB,MAAQ,QAElBmb,EAAKt7B,MAAMkgB,KAAO,QAErBob,EAAKt7B,MAAMqU,OAAS,OACpBinB,EAAKt7B,MAAMoU,MAAQ,OAUfknB,EAAKxR,kBAEPwR,EAAKxR,iBAAiB,aAAcvI,GAAAga,QAAAA,EAAkB/vD,OAAO,GAE7D8vD,EAAKxR,iBAAiB,iBAAkBvI,GAAAga,QAAAA,EAAkB/vD,OAAO,IAGjE8vD,EAAKE,YAAY,eAAgBja,GAAAga,QAAAA,EAAkB/vD,OAGrD6vD,EAAIl7B,YAAYm7B,QAEXjM,OAAS,IAAIG,GAAO8L,QACpBjM,OAAOvN,GAAG,WAAYP,UAAK+H,qBAAkB99C,YAC7C6jD,OAAOvN,GAAG,UAAYP,UAAKgI,gBAAa/9C,YACxC6jD,OAAOvN,GAAG,SAAYP,UAAKiI,mBAAgBh+C,YAC3C6jD,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOiM,uDAOtDC,YAEArM,OAAOkB,eACPlB,OAAS,UAETjL,KAAO,0CAQNhoB,EAAS5wB,KAAK44C,KAAKwF,IAAIyL,mBACzB7pD,KAAK6vD,IAAItoB,YAAc3W,IAErB5wB,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,KAEvCj/B,EAAO+D,YAAY30B,KAAK6vD,UAGpB1lD,EAAInK,KAAK44C,KAAKN,KAAKkD,SAASx7C,KAAK2vD,YAEnCjyD,EAASsC,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QAC1CA,IACEsC,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhBzyD,EAASsC,KAAKmvB,QAAQvc,QAAb,UAGP68C,EAAQzvD,KAAKmvB,QAAQsgC,WAEXhwD,IAAVgwD,EAEFA,GADAA,iBAAW/xD,EAAOgjB,mBAAS1gB,KAAKmvB,QAAQ/E,OAAOpqB,KAAK2vD,YAAYlyD,OAAO,iCACzD+oB,OAAO,GAAGjB,cAAgBkqC,EAAMW,UAAU,GAC9B,mBAAVX,IAChBA,EAAQA,EAAMpzD,KAAK2D,KAAMA,KAAK2vD,yBAG3BxgC,QAAQguB,IAAMn9C,KAAK6vD,IAAIr7B,MAAMmgB,gBAAWxqC,QAAQnK,KAAK6vD,IAAIr7B,MAAMkgB,eAAUvqC,aACzE0lD,IAAIJ,MAAQA,GAEV,iCAQHzvD,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,2CAQ3BnvC,QACPivC,WAAarX,GAAKpB,QAAQx2B,EAAM,aAChC2vC,wDAQE,IAAI3zD,KAAKsD,KAAK2vD,WAAWpyD,mDAQlBkyD,EAAOa,WACfC,EAASnjC,SAASE,cAAc,QACtCijC,EAAOjoB,mCACPioB,EAAO7E,UAAY+D,EACnBc,EAAO/7B,MAAMkL,SAAW,WAEpB4wB,KACFC,EAAOC,aAAa,kBAAmB,QACvCD,EAAOjS,iBAAiB,eAAe,WACrCiS,EAAOE,WAETF,EAAOjS,iBAAiB,QAASvI,UAAK2a,wBAAqB1wD,OAE3DuwD,EAAOd,MAAQA,EACfc,EAAOjS,iBAAiB,OAAQvI,cAAUyK,GACpCxgD,KAAKyvD,OAASjP,EAAM7wB,OAAO+7B,iBACxBiF,iBAAiBnQ,QACjBiP,MAAQjP,EAAM7wB,OAAO+7B,oBAEvB1rD,aAGJ6vD,IAAIl7B,YAAY47B,0CAORd,QACRtgC,QAAQsgC,MAAQA,uCAQVjP,QACNoP,YAAYzP,UAAW,OACvByP,YAAYD,WAAa3vD,KAAK2vD,WAEnCnP,EAAM8D,kDAQA9D,MACDxgD,KAAK4vD,YAAYzP,cAElBoB,EAASvhD,KAAKmvB,QAAQguB,KAAQ,EAAKqD,EAAMe,OAASf,EAAMe,OAEtDp3C,EAAInK,KAAK44C,KAAKN,KAAKkD,SAASx7C,KAAK4vD,YAAYD,YAAcpO,EAC3D7gC,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAE7BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBC,EAAO9wD,KAAKmvB,QAAQ2hC,KAEpBC,EAAcD,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,OAEhDswC,cAAcD,QAGdnY,KAAKiF,QAAQ4C,KAAK,aAAc,CACnCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjB1S,KAAM,IAAIhkB,KAAKsD,KAAK2vD,WAAWpyD,WAC/BijD,MAAAA,IAGFA,EAAM8D,sDAQG9D,GACJxgD,KAAK4vD,YAAYzP,gBAGjBvH,KAAKiF,QAAQ4C,KAAK,cAAe,CACpCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjB1S,KAAM,IAAIhkB,KAAKsD,KAAK2vD,WAAWpyD,WAC/BijD,MAAAA,IAGFA,EAAM8D,2DAQQ9D,QACT5H,KAAKiF,QAAQ4C,KAAK,eAAgB,CACrCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjBq8B,MAAOjP,EAAM7wB,OAAO+7B,UACpBlL,MAAAA,IAGFA,EAAM8D,2DAQS9D,QACV5H,KAAKiF,QAAQ4C,KAAK,gBAAiB,CACtCrtB,GAAIpzB,KAAKmvB,QAAQiE,GACjBq8B,MAAOjP,EAAM7wB,OAAO+7B,UACpBlL,MAAAA,IAGFA,EAAM8D,iEASoB9D,WACtB7wB,EAAS6wB,EAAM7wB,OACZA,GAAQ,IACTA,EAAOtyB,eAAe,sBACjBsyB,EAAO,eAEhBA,EAASA,EAAO4X,kBAGX,YA9UcoR,klWCWnB8C,iFAQIwQ,yBACD7N,IAAM,QAENA,IAAI6N,UAAYA,OAChB7N,IAAI6N,UAAUz3B,MAAMkL,SAAW,gBAE/B0e,IAAI+C,KAAuB/zB,SAASE,cAAc,YAClD8wB,IAAIwK,WAAuBx7B,SAASE,cAAc,YAClD8wB,IAAIyL,mBAAuBz8B,SAASE,cAAc,YAClD8wB,IAAI6S,qBAAuB7jC,SAASE,cAAc,YAClD8wB,IAAI3E,gBAAuBrsB,SAASE,cAAc,YAClD8wB,IAAI8S,cAAuB9jC,SAASE,cAAc,YAClD8wB,IAAI+S,eAAuB/jC,SAASE,cAAc,YAClD8wB,IAAIY,OAAuB5xB,SAASE,cAAc,YAClD8wB,IAAI1J,KAAuBtnB,SAASE,cAAc,YAClD8wB,IAAIzJ,MAAuBvnB,SAASE,cAAc,YAClD8wB,IAAI0E,IAAuB11B,SAASE,cAAc,YAClD8wB,IAAIyK,OAAuBz7B,SAASE,cAAc,YAClD8wB,IAAIgT,UAAuBhkC,SAASE,cAAc,YAClD8wB,IAAIiT,aAAuBjkC,SAASE,cAAc,YAClD8wB,IAAIkT,cAAuBlkC,SAASE,cAAc,YAClD8wB,IAAImT,iBAAuBnkC,SAASE,cAAc,YAClD8wB,IAAIoT,eAAuBpkC,SAASE,cAAc,YAClD8wB,IAAIqT,kBAAuBrkC,SAASE,cAAc,YAClD8wB,IAAIC,eAAuBjxB,SAASE,cAAc,YAClD8wB,IAAIsT,cAAuBtkC,SAASE,cAAc,YAElD8wB,IAAI+C,KAAK7Y,UAA4B,oBACrC8V,IAAIwK,WAAWtgB,UAAsB,gCACrC8V,IAAIyL,mBAAmBvhB,UAAc,6CACrC8V,IAAI6S,qBAAqB3oB,UAAY,+CACrC8V,IAAI3E,gBAAgBnR,UAAiB,4BACrC8V,IAAI8S,cAAc5oB,UAAmB,0BACrC8V,IAAI+S,eAAe7oB,UAAkB,2BACrC8V,IAAI0E,IAAIxa,UAA6B,yBACrC8V,IAAIyK,OAAOvgB,UAA0B,4BACrC8V,IAAI1J,KAAKpM,UAA4B,mBACrC8V,IAAIY,OAAO1W,UAA0B,mBACrC8V,IAAIzJ,MAAMrM,UAA2B,mBACrC8V,IAAIgT,UAAU9oB,UAAuB,0BACrC8V,IAAIiT,aAAa/oB,UAAoB,6BACrC8V,IAAIkT,cAAchpB,UAAmB,0BACrC8V,IAAImT,iBAAiBjpB,UAAgB,6BACrC8V,IAAIoT,eAAelpB,UAAkB,0BACrC8V,IAAIqT,kBAAkBnpB,UAAe,6BACrC8V,IAAIC,eAAe/V,UAAkB,4BACrC8V,IAAIsT,cAAcppB,UAAmB,0BAErC8V,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIwK,iBAC9BxK,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIyL,yBAC9BzL,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI6S,2BAC9B7S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI3E,sBAC9B2E,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI8S,oBAC9B9S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI+S,qBAC9B/S,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAI0E,UAC9B1E,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIyK,aAC9BzK,IAAI+C,KAAKxsB,YAAY30B,KAAKo+C,IAAIC,qBAE9BD,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIY,aACzCZ,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAI1J,WACvC0J,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIzJ,YACxCyJ,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIgT,gBACzChT,IAAI3E,gBAAgB9kB,YAAY30B,KAAKo+C,IAAIiT,mBACzCjT,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAIkT,oBACvClT,IAAI8S,cAAcv8B,YAAY30B,KAAKo+C,IAAImT,uBACvCnT,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIoT,qBACxCpT,IAAI+S,eAAex8B,YAAY30B,KAAKo+C,IAAIqT,wBAGxCpc,MAAQ,CACX8L,KAAM,GACNyH,WAAY,GACZnP,gBAAiB,GACjByX,cAAe,GACfC,eAAgB,GAChBnS,OAAQ,GACRtK,KAAM,GACNC,MAAO,GACPmO,IAAK,GACL+F,OAAQ,GACR8I,OAAQ,GACRC,UAAW,EACXC,aAAc,QAGXvb,GAAG,eAAe,YACQ,IAAzBoH,EAAKoU,iBACPpU,EAAKqU,kBAGJzb,GAAG,gBAAgB,WACjBoH,EAAKsU,yBACRtU,EAAKsU,wBAAyB,WAG7B1b,GAAG,QAAeP,UAAKmI,iBAAcl+C,YACrCs2C,GAAG,UAAmBP,UAAKgI,gBAAa/9C,WAEvC2+C,EAAK3+C,UACNiyD,YAAclc,UAAKgc,gBAAa/xD,WAChC+xD,QAAUzZ,GAAK4Z,SAASlyD,KAAKiyD,kBAE7B3b,GAAG,WAAW,SAAAxc,GACb6kB,EAAGwT,SAAWxT,EAAGwT,QAAQC,qBAAuBt4B,GAAkC,GAApBA,EAAWu4B,MAC3E1T,EAAGoT,UAEHpT,EAAGsT,sBAMFpO,OAAS,IAAIG,GAAOhkD,KAAKo+C,IAAI+C,UAC5BmR,EAAkBtyD,KAAK6jD,OAAO95C,IAAI,SAAS/F,IAAI,CAACkiD,QAAQ,IAC9DoM,GRpFG,SAA0CA,GAG/CA,EAAgBC,eAAiB,iBAExB,CAJkB,UQmFNC,CAA2CF,QACzDzO,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOiM,qBACtDxM,UAAY,OR3HMI,EAAQ79C,EQ6HzBk+C,EAAS,CACb,MAAO,YAAa,QACpB,QACA,MAAO,WAAY,UAAW,mBA+BvB6L,EAAavP,GAMhBxgD,KAAKyyD,iBACFhS,KAAK,aAAcD,OAItBe,EAAS,EACTC,EAAS,KAGR,WAAiBhB,IAAUgB,GAAyB,EAAhBhB,EAAMqB,QAC1C,eAAiBrB,IAAUgB,EAAShB,EAAMoB,YAC1C,gBAAiBpB,IAAUgB,EAAShB,EAAMkS,aAC1C,gBAAiBlS,IAAUe,GAA8B,EAArBf,EAAMmS,aAG1C,SAAUnS,GAASA,EAAM8H,OAAS9H,EAAMoS,kBAC3CrR,GAAmB,EAAVC,EACTA,EAAS,GAIN,WAAYhB,IACfgB,GAAyB,EAAhBhB,EAAMgB,QAEZ,WAAYhB,IACfe,EAASf,EAAMe,QAIbf,EAAMqS,YACkB,IAApBrS,EAAMqS,WACRtR,GAlCc,GAmCdC,GAnCc,KAqCdD,GArCc,GAsCdC,GArCc,MAyChBxhD,KAAKmvB,QAAQ2jC,gBACV9yD,KAAKmvB,QAAQ2yB,SAAWtB,EAAMxgD,KAAKmvB,QAAQ2yB,SAAU,eAEtD9hD,KAAKmvB,QAAQ2yB,SAAWtB,EAAMxgD,KAAKmvB,QAAQ2yB,SAAU,UAGtD9hD,KAAKmvB,QAAQ4jC,gBAAmB/yD,KAAKmvB,QAAQ6jC,oBAE9ChzD,KAAKmvB,QAAQ4jC,gBAAkBvxD,KAAKc,IAAIk/C,IAAWhgD,KAAKc,IAAIi/C,GAAS,KACjEtG,EAAUj7C,KAAKq1C,MAAMuc,UACrBqB,EAAWhY,EAAUuG,KAEvBxhD,KAAKyyD,WACczyD,KAAKkzD,cAAcD,KAEnBhY,SACd8W,eACAtR,KAAK,SAAUD,GAIpBA,EAAM+B,uBAGL,GAAIviD,KAAKmvB,QAAQ6jC,iBAAkB,KAIlC71C,GAHQ3b,KAAKc,IAAIi/C,IAAW//C,KAAKc,IAAIk/C,GAAUD,EAASC,GAGxC,KAAQxhD,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,OAAS,GAE7DogB,EAAW5gD,KAAK05C,MAAMlZ,MAAQrjB,EAC9B0jC,EAAS7gD,KAAK05C,MAAMjZ,IAAMtjB,EAE1BgS,EAAU,CACd4vB,WAAW,EACXO,QAAQ,EACRkB,MAAAA,QAEG9G,MAAMgF,SAASkC,EAAUC,EAAQ1xB,GAEtCqxB,EAAM+B,qBA7GV2B,QAAAA,GAAe,SAAAhtB,OACPi8B,EAAW,SAAA3S,GACX7B,EAAG8T,YACL9T,EAAG8B,KAAKvpB,EAAMspB,IAGlB7B,EAAGkF,OAAOvN,GAAGpf,EAAMi8B,GACnBxU,EAAG8E,UAAUvsB,GAAQi8B,KAIvBX,GAAmBxyD,KAAK6jD,QAAQ,SAAArD,GAC9B7B,EAAG8B,KAAK,QAASD,MRnJIqD,EQuJF7jD,KAAK6jD,QRvJK79C,EQuJG,SAAAw6C,GAChC7B,EAAG8B,KAAK,UAAWD,KRvJd+E,aAAe,SAAU/E,GAC5BA,EAAM4S,SACRptD,EAASw6C,IAINqD,EAAOvN,GAAG,eAAgBtwC,EAASu/C,kBQkPlC8N,EAAY,YAAajmC,SAASE,cAAc,OAAS,aACrC7tB,IAA1B2tB,SAASkmC,aAA6B,kBAIjClV,IAAI3E,gBAAgB6E,iBAAmB,iBAAoB,wBAYvDiV,EAAkB/S,MACpB7B,EAAGxvB,QAAQ4jC,kBAEZpU,EAAG6U,2BACL7U,EAAG6U,4BAA6B,UAIlChT,EAAM+B,iBACF5D,EAAG8T,WAAY,KACXQ,GAAYzS,EAAM7wB,OAAOiiC,UAC/BjT,EAAGuU,cAAcD,GACjBtU,EAAGoT,UACHpT,EAAG8B,KAAK,aAAcD,SAxBrBpC,IAAI0E,IAAIxE,sBACRF,IAAIyK,OAAOvK,sBACXF,IAAI3E,gBAAgB6E,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QACzEo+C,IAAI0E,IAAIxE,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QAC7Do+C,IAAIyK,OAAOvK,iBAAiB+U,EAAWtd,GAAAga,QAAAA,EAAkB/vD,OAAO,QAwBhEo+C,IAAI1J,KAAKnN,WAAW+W,iBAAiB,SAAUvI,GAAAwd,QAAAA,EAAuBvzD,YACtEo+C,IAAIzJ,MAAMpN,WAAW+W,iBAAiB,SAAUvI,GAAAwd,QAAAA,EAAuBvzD,WAExEyzD,GAAsB,WAOjBC,EAAelT,YAClBA,EAAM+B,iBACR5D,EAAG8B,KAAK,WAAY9B,EAAGgV,mBAAmBnT,IAC1CA,EAAM+B,kBAIFqR,KAAApT,EAAM7wB,OAAO2Y,kBAAkB,aAAe,IAGhDmrB,SAEJjT,EAAMqT,aAAaC,WAAa,OAChCL,GAAsB,GACf,WAQAM,EAAWvT,GAEfA,EAAM+B,gBAAkB/B,EAAM+B,iBAC9B/B,EAAM8D,iBAAmB9D,EAAM8D,0BAG5B0P,EAAWxuB,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,aAChDD,IAAaA,EAASngC,QAAS,OACpC,MAAOoS,UACA,SAGTwtB,GAAsB,EACtBjT,EAAMxB,OAAS,CACb70C,EAAGq2C,EAAM2B,QACT54C,EAAGi3C,EAAM4B,SAGa,SAApB4R,EAASrkC,OACXgvB,EAAGwT,QAAQ+B,WAAW1T,GAEtB7B,EAAGwT,QAAQgC,oBAAoB3T,GAEjC7B,EAAG8B,KAAK,OAAQ9B,EAAGgV,mBAAmBnT,KAC/B,UAGJpC,IAAIY,OAAOV,iBAAiB,WAAYvI,GAAA2d,QAAAA,EAAoB1zD,OAAO,QACnEo+C,IAAIY,OAAOV,iBAAiB,OAAQvI,GAAAge,QAAAA,EAAgB/zD,OAAO,QAE3Do0D,YAAc,QAGdzW,MAAQ,QAER0W,YAAc,OACdvC,iBAAkB,OAClBE,wBAAyB,GAGzB/F,EAAW,MAAM,IAAI7oD,MAAM,yBAChC6oD,EAAUt3B,YAAY30B,KAAKo+C,IAAI+C,MAC/B8K,EAAUt3B,YAAY30B,KAAKo+C,IAAIsT,kDA4BtBviC,YACLA,EAAS,IAQXmpB,GAAKmG,gBANU,CACb,QAAS,SAAU,YAAa,YAAa,aAC7C,QAAS,MAAO,aAAc,iBAAkB,cAChD,SAAU,UAAW,SAAU,aAAc,MAAO,UACpD,mBAAoB,iBAAkB,uBAEXz+C,KAAKmvB,QAASA,QACtCivB,IAAIC,eAAe7pB,MAAMyqB,WAAa,SAEvCj/C,KAAKmvB,QAAQguB,WACViB,IAAI6N,UAAUz3B,MAAMlW,UAAY,WAChC8/B,IAAIyL,mBAAmBvhB,UAAY,6CAGtCtoC,KAAKmvB,QAAQ4jC,iBACX/yD,KAAKmvB,QAAQguB,SACViB,IAAI+S,eAAe7oB,UAAY,+CAE/B8V,IAAI8S,cAAc5oB,UAAY,0CAIC,WAApC7C,GAAOzlC,KAAKmvB,QAAQk5B,oBACjBl5B,QAAQk5B,YAAc,CAACtQ,UAAKt4C,EAAU6oD,UAAK7oD,IAE9C,gBAAiB0vB,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAc,CACzBtQ,KAAM5oB,EAAQk5B,YACdC,KAAMn5B,EAAQk5B,aAGsB,WAA/B5iB,GAAOtW,EAAQk5B,eAClB,SAAUl5B,EAAQk5B,mBACfl5B,QAAQk5B,YAAYtQ,KAAO5oB,EAAQk5B,YAAYtQ,MAElD,SAAU5oB,EAAQk5B,mBACfl5B,QAAQk5B,YAAYC,KAAOn5B,EAAQk5B,YAAYC,QAKpB,SAAlCtoD,KAAKmvB,QAAQk5B,YAAYC,UACtBtoD,KAAKs0D,UAAW,KACbA,EAAYt0D,KAAKs0D,UAAY,IAAIvM,GAAS/nD,KAAK44C,MACrD0b,EAAU9V,WAAa,SAAArvB,OACf20B,EAAW30B,EAAUmpB,GAAKh7C,OAAO,GAAI6xB,GAAW,GACtD20B,EAASuE,YAAc,MACvBN,GAAS5rD,UAAUqiD,WAAWniD,KAAKi4D,EAAWxQ,SAE3CyQ,WAAWt3D,KAAKq3D,YAInBt0D,KAAKs0D,UAAW,SACZ1vC,EAAQgvC,UAAKW,mBAAmBv0D,KAAKs0D,eAC5B,IAAX1vC,YACG2vC,mBAAkB3vC,EAAO,QAE3B0vC,UAAUvP,eACVuP,UAAY,KAKY,mBAAtBnlC,EAAQqlC,aACjBrlC,EAAQqlC,WAAa,CACjBC,SAAUtlC,EAAQqlC,aAIpB,gBAAiBx0D,KAAKmvB,SACxBmxB,GAA8BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aAGzE,eAAgBjqB,IACdA,EAAQulC,WACL10D,KAAK20D,iBACHA,UAAY,IAAIxH,GAAUntD,KAAKo+C,IAAI+C,OAItCnhD,KAAK20D,iBACFA,UAAU5P,iBACR/kD,KAAK20D,iBAMbC,+BAIFL,oBAAmB,SAAAM,UAAaA,EAAUrW,WAAWrvB,MAGtD,cAAeA,EAAS,OACrBnvB,KAAK80D,oBACHA,aAAe90D,KAAK+0D,4BAGtBD,aAAatW,WAAWrvB,EAAQ6lC,eAG/BC,EAAiB3c,GAAKuN,WAAW,GAAI7lD,KAAKmvB,mBAC3ColC,oBAAmB,SAAAM,GACtBvc,GAAKuN,WAAWoP,EAAgBJ,EAAU1lC,iBAEvC2lC,aAAaI,iBAAiB,CAAChpC,OAAQ+oC,SAGzClD,oDAQG/xD,KAAK20D,WAAa30D,KAAK20D,UAAUvH,mDA8BpC,IAAM5M,UAtBN2U,SAAS,WACTC,UAAU,WAGV7e,WAGA8e,kBAGDr1D,KAAKo+C,IAAI+C,KAAK5Z,iBACX6W,IAAI+C,KAAK5Z,WAAWC,YAAYxnC,KAAKo+C,IAAI+C,WAE3C/C,IAAM,KAGPp+C,KAAK20D,iBACFA,UAAU5P,iBACR/kD,KAAK20D,WAIM30D,KAAKyjD,UACnBzjD,KAAKyjD,UAAUpmD,eAAemjD,WACzBxgD,KAAKyjD,UAAUjD,QAGrBiD,UAAY,UACZI,QAAU7jD,KAAK6jD,OAAOkB,eACtBlB,OAAS,eAGT0Q,oBAAmB,SAAAM,UAAaA,EAAU9P,kBAE1CnM,KAAO,2CAQAl4B,EAAM0S,SACZghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAazhC,IAAOyhC,EAAU1lC,QAAQiE,SAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,KAGjEghC,EAAYp3D,OAAS,GACvBo3D,EAAY,GAAGpD,cAActwC,yCASnB0S,SACNghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,YAE9DghC,EAAY,GAAGkB,4DASJ7F,EAAOr8B,EAAIk9B,SACvB8D,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,KAEjEghC,EAAYp3D,OAAS,GACvBo3D,EAAY,GAAGmB,gBAAgB9F,EAAOa,8CAUvBb,EAAOr8B,SAClBghC,EAActd,UAAKsd,qBAAmB,SAAAS,UAAaA,EAAU1lC,QAAQiE,KAAOA,QAEvD,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,QAEjEghC,EAAYp3D,OAAS,SAChBo3D,EAAY,GAAGoB,eAAe/F,8CAUtBjP,SACV,CAAEA,MAAAA,yCAaG9/B,EAAM0S,SACZqiC,OAAqBh2D,IAATihB,EACZ43B,GAAKpB,QAAQx2B,EAAM,QACnB,IAAIhkB,QAEKg5D,UAAKtB,qBAAiB,SAAAzE,UAAcA,EAAWxgC,QAAQiE,KAAOA,WAErE,IAAIhwB,sCAA+B28C,GAAe3sB,2BAGpDu8B,EAAa,IAAIH,GAAWxvD,KAAK44C,KAAMN,GAAKh7C,OAAO,GAAI0C,KAAKmvB,QAAS,CACzEzO,KAAO+0C,EACPriC,GAAAA,EACA09B,KAAM9wD,KAAKmyD,QAAQhjC,QAAQ2hC,oBAGxBsD,YAAYn3D,KAAK0yD,QACjB4E,WAAWt3D,KAAK0yD,QAChBoC,UAEE3+B,2CAQQA,gBACTghC,EAActd,UAAKsd,qBAAmB,SAAAvE,UAAOA,EAAI1gC,QAAQiE,KAAOA,QAE3C,IAAvBghC,EAAYp3D,aACR,IAAIoG,iDAA0C28C,GAAe3sB,QAGrEghC,QAAAA,GAAoB,SAAAzE,oBAClBgG,EAAKvB,oBAAmBR,KAAA+B,EAAKvB,oBAAoBzE,GAAa,QAC9DgG,EAAKpB,mBAAkBX,KAAA+B,EAAKpB,mBAAmB5E,GAAa,GAC5DA,EAAW5K,8DASN/kD,KAAKmyD,SAAWnyD,KAAKmyD,QAAQyD,mBAAqB,qDAQlD51D,KAAKmyD,SAAWnyD,KAAKmyD,QAAQ0D,oBAAsB,+BAcxD1mC,EAASnpB,OACL0zC,EAAQ15C,KAAK81D,kBAGD,OAAdpc,EAAMt3C,KAA8B,OAAds3C,EAAMp0C,SAK1Bu5C,EAAWnF,EAAMp0C,IAAMo0C,EAAMt3C,IAC7BA,EAAM,IAAI1F,KAAKg9C,EAAMt3C,IAAI7E,UAAuB,IAAXshD,GACrCv5C,EAAM,IAAI5I,KAAKg9C,EAAMp0C,IAAI/H,UAAuB,IAAXshD,GACrCE,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eACpErF,MAAMgF,SAASt8C,EAAKkD,EAAK,CAAEy5C,UAAAA,GAAa/4C,iDAUvC,IAAI5C,MAAM,gFAyBRo9B,EAAOC,EAAKtR,EAASnpB,OAKzB+4C,EACArF,EALuB,mBAAhB59C,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,IAIY,GAApBrzB,UAAUkB,QAEZ+hD,OAAiCt/C,KADjCi6C,EAAQ59C,UAAU,IACCijD,WAA2BrF,EAAMqF,eAC/CrF,MAAMgF,SAAShF,EAAMlZ,MAAOkZ,EAAMjZ,IAAK,CAAEse,UAAAA,KAEnB,GAApBjjD,UAAUkB,QAAsC,mBAAhBlB,UAAU,IACjD49C,EAAQ59C,UAAU,GAClBkK,EAAWlK,UAAU,GACrBijD,OAAiCt/C,IAApBi6C,EAAMqF,WAA2BrF,EAAMqF,eAC/CrF,MAAMgF,SAAShF,EAAMlZ,MAAOkZ,EAAMjZ,IAAK,CAAEse,UAAAA,GAAa/4C,KAG3D+4C,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eAC9DrF,MAAMgF,SAASle,EAAOC,EAAK,CAAEse,UAAAA,GAAa/4C,mCAgB5C0a,EAAMyO,EAASnpB,GACO,mBAAhBlK,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAEN0vB,EAAW7+C,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,MACvCzgC,EAAIu4C,GAAKpB,QAAQx2B,EAAM,QAAQnjB,UAE/BijC,EAAQzgC,EAAI8+C,EAAW,EACvBpe,EAAM1gC,EAAI8+C,EAAW,EACrBE,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,eAEpErF,MAAMgF,SAASle,EAAOC,EAAK,CAAEse,UAAAA,GAAa/4C,2CAQzC0zC,EAAQ15C,KAAK05C,MAAMqc,iBAClB,CACLv1B,MAAO,IAAI9jC,KAAKg9C,EAAMlZ,OACtBC,IAAK,IAAI/jC,KAAKg9C,EAAMjZ,qCAgBjBu1B,EAAY7mC,EAASnpB,SACrBgwD,GAAcA,EAAa,GAAKA,EAAa,IACvB,mBAAhBl6D,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAENuqB,EAAQ15C,KAAKi2D,YACbz1B,EAAQkZ,EAAMlZ,MAAMjjC,UACpBkjC,EAAMiZ,EAAMjZ,IAAIljC,UAChBshD,EAAWpe,EAAMD,EACjB01B,EAAcrX,GAAY,EAAImX,GAC9BG,GAAYtX,EAAWqX,GAAe,EACtCtV,EAAWpgB,EAAQ21B,EACnBtV,EAASpgB,EAAM01B,OAEhBC,UAAUxV,EAAUC,EAAQ1xB,EAASnpB,oCAepCgwD,EAAY7mC,EAASnpB,SACtBgwD,GAAcA,EAAa,GAAKA,EAAa,IACvB,mBAAhBl6D,UAAU,KACnBkK,EAAWlK,UAAU,GACrBqzB,EAAU,QAENuqB,EAAQ15C,KAAKi2D,YACbz1B,EAAQkZ,EAAMlZ,MAAMjjC,UACpBkjC,EAAMiZ,EAAMjZ,IAAIljC,UAChBshD,EAAWpe,EAAMD,EACjBogB,EAAWpgB,EAAQqe,EAAWmX,EAAa,EAC3CnV,EAASpgB,EAAMoe,EAAWmX,EAAa,OAExCI,UAAUxV,EAAUC,EAAQ1xB,EAASnpB,0CASrC+rD,uDASAsC,kBACCjW,EAAMp+C,KAAKo+C,OAEZA,GAAQA,EAAI6N,WAAqC,GAAxB7N,EAAI+C,KAAKiI,iBAEnCvQ,GAAU,EACR1pB,EAAUnvB,KAAKmvB,QACfkmB,EAAQr1C,KAAKq1C,MAEnBiL,GAA2BtgD,KAAKmvB,QAAQ/E,OAAQpqB,KAAK44C,KAAM54C,KAAKmvB,QAAQiqB,aAG7C,OAAvBjqB,EAAQk5B,aACV/P,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,gBAG/B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,WAC/B7I,GAAKuV,aAAazP,EAAI+C,KAAM,eAG1BhyB,EAAQguB,KACV7E,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,aAE/B7I,GAAKuV,aAAazP,EAAI+C,KAAM,WAC5B7I,GAAKwV,gBAAgB1P,EAAI+C,KAAM,YAIjC/C,EAAI+C,KAAK3sB,MAAM6hC,UAAY/d,GAAKge,OAAOC,OAAOpnC,EAAQknC,UAAW,IACjEjY,EAAI+C,KAAK3sB,MAAMgiC,UAAYle,GAAKge,OAAOC,OAAOpnC,EAAQqnC,UAAW,IACjEpY,EAAI+C,KAAK3sB,MAAMoU,MAAQ0P,GAAKge,OAAOC,OAAOpnC,EAAQyZ,MAAO,QACnD6tB,EAAkBrY,EAAI+C,KAAKiI,YAGjC/T,EAAMsc,OAAOjd,KAAO,EACpBW,EAAMsc,OAAOhd,MAAQ,EACrBU,EAAMsc,OAAO7O,IAAM,EACnBzN,EAAMsc,OAAO9I,OAAS,EAItBxT,EAAM2J,OAAOnW,OAASuV,EAAIY,OAAO0X,aACjCrhB,EAAMX,KAAK7L,OAAWuV,EAAI1J,KAAKgiB,aAC/BrhB,EAAMV,MAAM9L,OAAUuV,EAAIzJ,MAAM+hB,aAChCrhB,EAAMyN,IAAIja,OAAYuV,EAAI0E,IAAIiJ,eAAoB1W,EAAMsc,OAAO7O,IAC/DzN,EAAMwT,OAAOhgB,OAASuV,EAAIyK,OAAOkD,eAAiB1W,EAAMsc,OAAO9I,WAMzD8N,EAAgBn1D,KAAK8D,IAAI+vC,EAAMX,KAAK7L,OAAQwM,EAAM2J,OAAOnW,OAAQwM,EAAMV,MAAM9L,QAC7E+tB,EAAavhB,EAAMyN,IAAIja,OAAS8tB,EAAgBthB,EAAMwT,OAAOhgB,OAASwM,EAAMsc,OAAO7O,IAAMzN,EAAMsc,OAAO9I,OAC5GzK,EAAI+C,KAAK3sB,MAAMqU,OAASyP,GAAKge,OAAOC,OAAOpnC,EAAQ0Z,iBAAW+tB,SAG9DvhB,EAAM8L,KAAKtY,OAASuV,EAAI+C,KAAKuV,aAC7BrhB,EAAMuT,WAAW/f,OAASwM,EAAM8L,KAAKtY,WAC/BguB,EAAkBxhB,EAAM8L,KAAKtY,OAASwM,EAAMyN,IAAIja,OAASwM,EAAMwT,OAAOhgB,OAC5EwM,EAAMoE,gBAAgB5Q,OAAUguB,EAChCxhB,EAAM6b,cAAcroB,OAAYguB,EAChCxhB,EAAM8b,eAAetoB,OAAWwM,EAAM6b,cAAcroB,OAGpDwM,EAAM8L,KAAKvY,MAAQ6tB,EACnBphB,EAAMuT,WAAWhgB,MAAQyM,EAAM8L,KAAKvY,MAE/B5oC,KAAK8xD,kBACRzc,EAAMyhB,eAAiBxe,GAAKye,yBAGxBC,EAA2B5Y,EAAI8S,cAAcvQ,YAC7CsW,EAA4B7Y,EAAI+S,eAAexQ,YAEjDxxB,EAAQ4jC,eACN5jC,EAAQguB,KACV9H,EAAMX,KAAK9L,MAAQouB,IAA6B3hB,EAAMsc,OAAOjd,KAC7DW,EAAMV,MAAM/L,MAAQquB,EAA4B5hB,EAAMyhB,iBAAmBzhB,EAAMsc,OAAOhd,QAEtFU,EAAMX,KAAK9L,MAAQouB,EAA2B3hB,EAAMyhB,iBAAmBzhB,EAAMsc,OAAOjd,KACpFW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,QAGjEU,EAAMX,KAAK9L,MAAQouB,IAA8B3hB,EAAMsc,OAAOjd,KAC9DW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,YAG5DuiB,cAID16C,EAASxc,KAAKm3D,mBAGc,OAA5BhoC,EAAQk5B,YAAYtQ,OACtBv7B,GAAUhb,KAAK8D,IAAI+vC,EAAMoE,gBAAgB5Q,OAASwM,EAAM2J,OAAOnW,OAC7DwM,EAAMsc,OAAO7O,IAAMzN,EAAMsc,OAAO9I,OAAQ,IAE5CzK,EAAIY,OAAOxqB,MAAMkiB,+BAA2Bl6B,aAGtC46C,EAAmC,GAAnB/hB,EAAMuc,UAAiB,SAAW,GAClDyF,EAAmBhiB,EAAMuc,WAAavc,EAAMwc,aAAe,SAAW,GAC5EzT,EAAIgT,UAAU58B,MAAMyqB,WAAsBmY,EAC1ChZ,EAAIiT,aAAa78B,MAAMyqB,WAAmBoY,EAC1CjZ,EAAIkT,cAAc98B,MAAMyqB,WAAkBmY,EAC1ChZ,EAAImT,iBAAiB/8B,MAAMyqB,WAAeoY,EAC1CjZ,EAAIoT,eAAeh9B,MAAMyqB,WAAiBmY,EAC1ChZ,EAAIqT,kBAAkBj9B,MAAMyqB,WAAcoY,EAEtCloC,EAAQ4jC,iBACV3U,EAAI+S,eAAe7oB,UAAY,0CAC/B8V,EAAI8S,cAAc5oB,UAAY,yCAE9B8V,EAAIoT,eAAeh9B,MAAMyqB,WAAiB,SAC1Cb,EAAIqT,kBAAkBj9B,MAAMyqB,WAAc,SAC1Cb,EAAIkT,cAAc98B,MAAMyqB,WAAkB,SAC1Cb,EAAImT,iBAAiB/8B,MAAMyqB,WAAe,SAE1Cb,EAAI1J,KAAKlgB,MAAMsuB,IAAS,MACxB1E,EAAIzJ,MAAMngB,MAAMsuB,IAAQ,SAGrB3zB,EAAQ4jC,gBAAmB1d,EAAM2J,OAAOnW,OAASwM,EAAMoE,gBAAgB5Q,UAC1EuV,EAAI1J,KAAKlgB,MAAMsuB,cAAYtmC,QAC3B4hC,EAAIzJ,MAAMngB,MAAMsuB,cAAWtmC,QAC3B4hC,EAAI+S,eAAe7oB,UAAY8V,EAAI+S,eAAe7oB,UAAU/hC,QAAQ,IAAI+B,OAAO,yCAAmD,KAClI81C,EAAI8S,cAAc5oB,UAAY8V,EAAI8S,cAAc5oB,UAAU/hC,QAAQ,IAAI+B,OAAO,yCAAmD,KAChI+sC,EAAMX,KAAK9L,MAAQouB,IAA8B3hB,EAAMsc,OAAOjd,KAC9DW,EAAMV,MAAM/L,MAAQquB,IAA8B5hB,EAAMsc,OAAOhd,WAC1DuiB,eAIDI,EAAmBjiB,EAAM2J,OAAOnW,OAASwM,EAAMoE,gBAAgB5Q,YAChEgb,OAAO95C,IAAI,OAAO/F,IAAI,CACzBsa,UAAWg5C,EAAmBtT,GAAOiM,cAAgBjM,GAAOuT,4BAIzD1T,OAAO95C,IAAI,SAAS/F,IAAI,CAC3B0c,KAAM1gB,KAAKmvB,QAAQqoC,gCAIhBjD,oBAAmB,SAAAM,GACtBhc,EAAUgc,EAAUxE,UAAYxX,QAG9BA,EAAS,IACP74C,KAAKq0D,YAFQ,mBAGVzb,KAAKiF,QAAQ4C,KAAK,WAIvB99C,QAAQq3C,IAAI,+CAGTqa,YAAc,OAIhBzb,KAAKiF,QAAQ4C,KAAK,kDAOjBpL,EAAQr1C,KAAKq1C,MACb+I,EAAMp+C,KAAKo+C,IAEjB/I,EAAM6b,cAActoB,MAAQyM,EAAMX,KAAK9L,MACvCyM,EAAM8b,eAAevoB,MAAQyM,EAAMV,MAAM/L,UACnC6uB,EAAcpiB,EAAM8L,KAAKvY,MAAQyM,EAAMX,KAAK9L,MAAQyM,EAAMV,MAAM/L,MACtEyM,EAAM2J,OAAOpW,MAAiB6uB,EAC9BpiB,EAAMoE,gBAAgB7Q,MAAQ6uB,EAC9BpiB,EAAMyN,IAAIla,MAAoB6uB,EAC9BpiB,EAAMwT,OAAOjgB,MAAiB6uB,EAG9BrZ,EAAIwK,WAAWp0B,MAAMqU,iBAAsBwM,EAAMuT,WAAW/f,aAC5DuV,EAAIyL,mBAAmBr1B,MAAMqU,iBAAcwM,EAAMuT,WAAW/f,aAC5DuV,EAAI6S,qBAAqBz8B,MAAMqU,iBAAYwM,EAAMoE,gBAAgB5Q,aACjEuV,EAAI3E,gBAAgBjlB,MAAMqU,iBAAiBwM,EAAMoE,gBAAgB5Q,aACjEuV,EAAI8S,cAAc18B,MAAMqU,iBAAmBwM,EAAM6b,cAAcroB,aAC/DuV,EAAI+S,eAAe38B,MAAMqU,iBAAkBwM,EAAM8b,eAAetoB,aAEhEuV,EAAIwK,WAAWp0B,MAAMoU,gBAAsByM,EAAMuT,WAAWhgB,YAC5DwV,EAAIyL,mBAAmBr1B,MAAMoU,gBAAcyM,EAAMoE,gBAAgB7Q,YACjEwV,EAAI6S,qBAAqBz8B,MAAMoU,gBAAYyM,EAAMuT,WAAWhgB,YAC5DwV,EAAI3E,gBAAgBjlB,MAAMoU,gBAAiByM,EAAM2J,OAAOpW,YACxDwV,EAAI0E,IAAItuB,MAAMoU,gBAA6ByM,EAAMyN,IAAIla,YACrDwV,EAAIyK,OAAOr0B,MAAMoU,gBAA0ByM,EAAMwT,OAAOjgB,YAGxDwV,EAAIwK,WAAWp0B,MAAMkgB,KAAiB,IACtC0J,EAAIwK,WAAWp0B,MAAMsuB,IAAiB,IACtC1E,EAAIyL,mBAAmBr1B,MAAMkgB,eAAYW,EAAMX,KAAK9L,MAAQyM,EAAMsc,OAAOjd,WACzE0J,EAAIyL,mBAAmBr1B,MAAMsuB,IAAS,IACtC1E,EAAI6S,qBAAqBz8B,MAAMkgB,KAAO,IACtC0J,EAAI6S,qBAAqBz8B,MAAMsuB,cAAUzN,EAAMyN,IAAIja,aACnDuV,EAAI3E,gBAAgBjlB,MAAMkgB,eAAeW,EAAMX,KAAK9L,YACpDwV,EAAI3E,gBAAgBjlB,MAAMsuB,cAAezN,EAAMyN,IAAIja,aACnDuV,EAAI8S,cAAc18B,MAAMkgB,KAAc,IACtC0J,EAAI8S,cAAc18B,MAAMsuB,cAAiBzN,EAAMyN,IAAIja,aACnDuV,EAAI+S,eAAe38B,MAAMkgB,eAAgBW,EAAMX,KAAK9L,MAAQyM,EAAM2J,OAAOpW,YACzEwV,EAAI+S,eAAe38B,MAAMsuB,cAAgBzN,EAAMyN,IAAIja,aACnDuV,EAAI0E,IAAItuB,MAAMkgB,eAA2BW,EAAMX,KAAK9L,YACpDwV,EAAI0E,IAAItuB,MAAMsuB,IAAwB,IACtC1E,EAAIyK,OAAOr0B,MAAMkgB,eAAwBW,EAAMX,KAAK9L,YACpDwV,EAAIyK,OAAOr0B,MAAMsuB,cAAwBzN,EAAMyN,IAAIja,OAASwM,EAAMoE,gBAAgB5Q,aAClFuV,EAAIY,OAAOxqB,MAAMkgB,KAAqB,IACtC0J,EAAI1J,KAAKlgB,MAAMkgB,KAAuB,IACtC0J,EAAIzJ,MAAMngB,MAAMkgB,KAAsB,2CAUzBh0B,OACR1gB,KAAK03D,kBACF,IAAIt0D,MAAM,4CAGbs0D,YAAYC,eAAej3C,gDAS3B1gB,KAAK03D,kBACF,IAAIt0D,MAAM,8CAGXpD,KAAK03D,YAAYE,iDAUlBztD,UACCm2C,GAAgBtgD,KAAMmK,EAAGnK,KAAKq1C,MAAM2J,OAAOpW,6CAUtCz+B,UACLm2C,GAAgBtgD,KAAMmK,EAAGnK,KAAKq1C,MAAM8L,KAAKvY,yCAaxCloB,UACD4/B,GAAkBtgD,KAAM0gB,EAAM1gB,KAAKq1C,MAAM2J,OAAOpW,+CAYzCloB,UACP4/B,GAAkBtgD,KAAM0gB,EAAM1gB,KAAKq1C,MAAM8L,KAAKvY,iDAUtB,GAA3B5oC,KAAKmvB,QAAQ0oC,gBACVC,wBAGAzC,iEAUD1W,EAAK3+C,UAENq1D,uBAEA0C,UAAY,cACc,GAAzBpZ,EAAGxvB,QAAQ0oC,eAMXlZ,EAAGP,IAAI+C,KAAM,KACT6W,EAAmBrZ,EAAGP,IAAI+C,KAAKuV,aAC/BD,EAAkB9X,EAAGP,IAAI+C,KAAKiI,YAK/BqN,GAAmB9X,EAAGtJ,MAAM4iB,WAC9BD,GAAoBrZ,EAAGtJ,MAAM6iB,aAC9BvZ,EAAGtJ,MAAM4iB,UAAYxB,EACrB9X,EAAGtJ,MAAM6iB,WAAaF,EACtBrZ,EAAGtJ,MAAMyhB,eAAiBxe,GAAKye,oBAE/BpY,EAAG/F,KAAKiF,QAAQ4C,KAAK,kBAjBvB9B,EAAG0W,mBAuBP/c,GAAKgG,iBAAiBtyB,OAAQ,SAAUhsB,KAAK+3D,WAGzCpZ,EAAGP,IAAI+C,OACPxC,EAAGtJ,MAAM4iB,UAAYtZ,EAAGP,IAAI+C,KAAKiI,YACjCzK,EAAGtJ,MAAM6iB,WAAavZ,EAAGP,IAAI+C,KAAKuV,mBAGjCyB,WAAaC,GAAYp4D,KAAK+3D,UAAW,+CAQ1C/3D,KAAKm4D,aACPE,cAAcr4D,KAAKm4D,iBACdA,gBAAa14D,GAIhBO,KAAK+3D,YACPzf,GAAKgL,oBAAoBt3B,OAAQ,SAAUhsB,KAAK+3D,gBAC3CA,UAAY,uCASZvX,QACF7C,MAAMuD,eAAgB,OACtBvD,MAAM2a,iBAAmBt4D,KAAKq1C,MAAMuc,2CAQlCpR,QACF7C,MAAMuD,eAAgB,kCAQrBV,MACDA,GAGAxgD,KAAK29C,MAAMuD,mBAEVI,EAAQd,EAAMgB,OAEd+W,EAAev4D,KAAKw4D,gBACpBC,EAAez4D,KAAKkzD,cAAclzD,KAAK29C,MAAM2a,iBAAmBhX,GAElEthD,KAAKmvB,QAAQ4jC,sBACV3U,IAAI1J,KAAKnN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,eAC5CxT,IAAIzJ,MAAMpN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,WAGhD6G,GAAgBF,QACb9X,KAAK,uDAUAmR,eACPvc,MAAMuc,UAAYA,OAClBuF,mBACEn3D,KAAKq1C,MAAMuc,yDAUZC,EAAerwD,KAAKY,IAAIpC,KAAKq1C,MAAMoE,gBAAgB5Q,OAAS7oC,KAAKq1C,MAAM2J,OAAOnW,OAAQ,UACxFgpB,GAAgB7xD,KAAKq1C,MAAMwc,eAGQ,OAAjC7xD,KAAKmvB,QAAQk5B,YAAYtQ,YACtB1C,MAAMuc,WAAcC,EAAe7xD,KAAKq1C,MAAMwc,mBAEhDxc,MAAMwc,aAAeA,GAIxB7xD,KAAKq1C,MAAMuc,UAAY,IAAG5xD,KAAKq1C,MAAMuc,UAAY,GACjD5xD,KAAKq1C,MAAMuc,UAAYC,IAAc7xD,KAAKq1C,MAAMuc,UAAYC,GAE5D7xD,KAAKmvB,QAAQ4jC,sBACV3U,IAAI1J,KAAKnN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,eAC5CxT,IAAIzJ,MAAMpN,WAAWqqB,WAAa5xD,KAAKq1C,MAAMuc,gBAE/C4B,4BAA6B,EAC3BxzD,KAAKq1C,MAAMuc,yDASX5xD,KAAKq1C,MAAMuc,8DASZ,IAAIxuD,MAAM,iEAKZq4C,GAAKt/C,eCz2CPu8D,0BASQ9f,EAAMzpB,oDAEXypB,KAAOA,IAGPsE,eAAiB,CACpBC,KAAK,EACLwb,iBAAiB,EACjBC,sBAAkBn5D,EAElB2qB,OAAAA,GACAxX,QAAAA,GACAlV,OAAQ,QAELyxB,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/BsB,WAAWrvB,KACXA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS8qC,EAAKvuB,QAAQvc,aACvD88C,EAAiBhS,EAAKR,eAAetqC,QAAQ8qC,EAAKR,eAAex/C,uBAC3DggD,EAAKvuB,QAAQvc,kBAAiB,SAAAlV,KACnCyxB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,SAGpB8e,OAAS,IAETksC,qEASCmH,EAAMziC,SAASE,cAAc,OACnCuiC,EAAIvnB,UAAY,mBAChBunB,EAAIr7B,MAAMkL,SAAW,WACrBmwB,EAAIr7B,MAAMsuB,IAAM,MAChB+M,EAAIr7B,MAAMqU,OAAS,YAEdgnB,IAAMA,yCAON1gC,QAAQwpC,iBAAkB,OAC1BtI,cAEAzX,KAAO,wCASHzpB,GACLA,GAEFmpB,GAAKmG,gBAAgB,CAAC,MAAO,kBAAmB,mBAAoB,SAAU,SAAU,WAAYz+C,KAAKmvB,QAASA,uCAShHnvB,KAAKmvB,QAAQwpC,gBAAiB,SAC1B/nC,EAAS5wB,KAAK44C,KAAKwF,IAAIyL,mBACzB7pD,KAAK6vD,IAAItoB,YAAc3W,IAErB5wB,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,KAEvCj/B,EAAO+D,YAAY30B,KAAK6vD,UAEnBrvB,aAGH3rB,EAAM7U,KAAKmvB,QAAQ/E,OAAO4c,KAAahnC,KAAKwc,QAE5Cxc,KAAKmvB,QAAQypC,mBACf/jD,EAAMA,EAAIgM,QAAQ7gB,KAAKmvB,QAAQypC,uBAG3BzuD,EAAInK,KAAK44C,KAAKN,KAAKkD,SAAS3mC,GAE9BnX,EAASsC,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QAC1CA,IACEsC,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhBzyD,EAASsC,KAAKmvB,QAAQvc,QAAb,QAEP68C,sBAAW/xD,EAAOu9C,qBAAWv9C,EAAOgjB,mBAAS7L,EAAIpX,OAAO,gCAC5DgyD,EAAQA,EAAMjpC,OAAO,GAAGjB,cAAgBkqC,EAAMW,UAAU,GAEpDpwD,KAAKmvB,QAAQguB,SACV0S,IAAIr7B,MAAMkiB,gCAA+B,EAALvsC,cAEpC0lD,IAAIr7B,MAAMkiB,+BAA0BvsC,cAEtC0lD,IAAIJ,MAAQA,OAIbzvD,KAAK6vD,IAAItoB,iBACNsoB,IAAItoB,WAAWC,YAAYxnC,KAAK6vD,UAElClkB,cAGA,sCAODgT,EAAK3+C,eAKAm2C,IACTwI,EAAGhT,WAICkT,EAAW,EADDF,EAAG/F,KAAKc,MAAMgC,WAAWiD,EAAG/F,KAAKY,SAASwF,OAAOpW,OAAO+S,MAC3C,GACvBkD,EAAW,KAAMA,EAAW,IAC5BA,EAAW,MAAMA,EAAW,KAEhCF,EAAG0R,SACH1R,EAAG/F,KAAKiF,QAAQ4C,KAAK,mBAGrB9B,EAAGO,iBAAmBC,GAAWhJ,EAAQ0I,GAG3C1I,uCAO8B12C,IAA1BO,KAAKk/C,mBACPE,aAAap/C,KAAKk/C,yBACXl/C,KAAKk/C,yDAUDx+B,OACP3gB,EAAIu4C,GAAKpB,QAAQx2B,EAAM,QAAQnjB,UAC/BsX,EAAMmyB,UACPxqB,OAASzc,EAAI8U,OACbw7C,yDAQE,IAAI3zD,KAAKsqC,KAAahnC,KAAKwc,eA3LZm8B,ICNtBkgB,GAAQruC,GAAwC2N,KAKhD2gC,IAAc,EAEdv3B,GAAiBC,GAHV,QAAA,QAMC,IAAIvlC,MAAM,GAAN,MAAe,WAAc68D,IAAc,OAIzD,CAAEnpC,OAAQ,QAASnP,OAAO,EAAM2P,OAAQ2oC,KAAgBv3B,IAAkB,CAC1EpJ,KAAM,SAAcN,UACXghC,GAAM74D,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCfzE,OAAiBohC,GAAa,SAAS1I,KCDnCsJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGqM,YACNrM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAetJ,KAAQA,GAAO4I,GCD5Fg4B,GAAmC,oBAAVC,QAAyBA,OAAOD,iBAAmBC,OAAOD,gBAAgB1oC,KAAK2oC,SAA8B,oBAAZC,UAA8D,mBAA5BA,SAASF,iBAAiCE,SAASF,gBAAgB1oC,KAAK4oC,UACpOC,GAAQ,IAAIC,WAAW,IAEZ,SAASC,SACjBL,SACG,IAAI31D,MAAM,mHAGX21D,GAAgBG,ICPzB,IAFA,IAAIG,GAAY,GAEPv8D,GAAI,EAAGA,GAAI,MAAOA,GACzBu8D,GAAUv8D,KAAMA,GAAI,KAAOV,SAAS,IAAImJ,OAAO,GCJjD,SAAS+zD,GAAGnqC,EAASoqC,EAAK/8C,OACpB1f,EAAIy8D,GAAO/8C,GAAU,EAEH,iBAAX2S,IACToqC,EAAkB,WAAZpqC,EAAuB,IAAIlzB,MAAM,IAAM,KAC7CkzB,EAAU,UAIRqqC,GADJrqC,EAAUA,GAAW,IACFmE,SAAWnE,EAAQiqC,KAAOA,SAE7CI,EAAK,GAAe,GAAVA,EAAK,GAAY,GAC3BA,EAAK,GAAe,GAAVA,EAAK,GAAY,IAEvBD,MACG,IAAItuD,EAAK,EAAGA,EAAK,KAAMA,EAC1BsuD,EAAIz8D,EAAImO,GAAMuuD,EAAKvuD,UAIhBsuD,GDbT,SAAqBA,EAAK/8C,OACpB1f,EAAI0f,GAAU,EACdi9C,EAAMJ,SAEH,CAACI,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO,IAAK28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,MAAO28D,EAAIF,EAAIz8D,OAAOqG,KAAK,ICSnQu2D,CAAYF,GCrB5B,IAAIG,GAAYnvC,GAAuC+G,SAInDgQ,GAAiBC,GAAwB,UAAW,CAAEN,WAAW,IAAS,MAI5E,CAAEvR,OAAQ,QAASnP,OAAO,EAAM2P,QAASoR,IAAkB,CAC3DhQ,SAAU,SAAkBJ,UACnBwoC,GAAU35D,KAAMmxB,EAAIr1B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCTrE,OAAiBohC,GAAa,SAAStP,SCCnCqoC,GAAQpkC,GAAgB,YCFX,SAAU1J,MDMV,SAAUA,OACrB+tC,SACGv9D,EAASwvB,UAAmCrsB,KAA1Bo6D,EAAW/tC,EAAG8tC,OAA0BC,EAA0B,UAAf9sC,EAAQjB,ICPhF+tC,CAAS/tC,SACLkB,UAAU,wDACTlB,GCHP8tC,GAAQpkC,GAAgB,WCM1B,CAAE7F,OAAQ,SAAUnP,OAAO,EAAM2P,QDJlB,SAAUgQ,OACrB25B,EAAS,cAEL35B,GAAa25B,GACnB,MAAOzmD,cAELymD,EAAOF,KAAS,EACT,MAAMz5B,GAAa25B,GAC1B,MAAO7rC,YACF,ECLiC8rC,CAAqB,aAAe,CAC9ExoC,SAAU,SAAkByoC,YAChBvrC,OAAOvB,EAAuBltB,OACrC0J,QAAQuwD,GAAWD,GAAel+D,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCR/E,OAAiBohC,GAAa,UAAUtP,SCApCkQ,GAAiBxlC,MAAME,UACvB2kC,GAAkBrS,OAAOtyB,aAEZ,SAAU2vB,OACrBiV,EAAMjV,EAAGyF,gBACTzF,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAelQ,SAAkB2oC,GAC5E,iBAAPpuC,GAAmBA,IAAOgV,IAAoBhV,aAAc2C,QAAUsS,IAAQD,GAAgBvP,SAChG4oC,GACAp5B,GCJJ,SAASq5B,GAAa5jB,MAC3BA,QAAAA,GAAW,SAACr5C,EAAGC,UAAMD,EAAEoW,KAAKitB,MAAQpjC,EAAEmW,KAAKitB,SAQtC,SAAS65B,GAAW7jB,MACzBA,QAAAA,GAAW,SAACr5C,EAAGC,UACE,QAASD,EAAEoW,KAAQpW,EAAEoW,KAAKktB,IAAMtjC,EAAEoW,KAAKitB,QACvC,QAASpjC,EAAEmW,KAAQnW,EAAEmW,KAAKktB,IAAMrjC,EAAEmW,KAAKitB,UAoBnD,SAASn9B,GAAMmzC,EAAO8jB,EAAQC,EAAOC,MACtCD,MAEG,IAAIz9D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAChC05C,EAAM15C,GAAGgmD,IAAM,SAKVhmD,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC/Bi7C,EAAOvB,EAAM15C,MACfi7C,EAAK10C,OAAsB,OAAb00C,EAAK+K,IAAc,CAEnC/K,EAAK+K,IAAMwX,EAAOhS,SACdmS,GAAa,IAEd,SAGGC,EAAgB,KACX3mD,EAAI,EAAG4mD,EAAKnkB,EAAMx5C,OAAQ+W,EAAI4mD,EAAI5mD,IAAK,KACxCgH,EAAQy7B,EAAMziC,MACpB0mD,EAAaD,MAAmC,SAEvB,KAEP,OAAdz/C,EAAM+nC,KAAgB/nC,IAAUg9B,GAAQh9B,EAAM1X,OAASu3D,GAAU7iB,EAAMh9B,EAAOu/C,EAAOviB,KAAMh9B,EAAMoU,QAAQguB,KAAM,CACjHud,EAAgB3/C,SAKC,MAAjB2/C,IAEF3iB,EAAK+K,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,OAASyxB,EAAOviB,KAAK8iB,gBAE7DH,WAGND,EAaF,SAASK,GAAStkB,EAAO8jB,EAAQS,OACjC,IAAIj+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAChC05C,EAAM15C,GAAGgmD,IAAM,SAIbkY,EAAiBD,EAASlyB,WAGzB/rC,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC3Bi7C,EAAOvB,EAAM15C,MAEfi7C,EAAK10C,OAAsB,OAAb00C,EAAK+K,IAAc,CAEnC/K,EAAK+K,IAAM/K,EAAKkjB,UAEb,SAGGP,EAAgB,KACX3mD,EAAI,EAAG4mD,EAAKnkB,EAAMx5C,OAAQ+W,EAAI4mD,EAAI5mD,IAAK,KACxCgH,EAAQy7B,EAAMziC,MACF,OAAdgH,EAAM+nC,KAAgB/nC,IAAUg9B,GAA2B6iB,GAAU7iB,EAAMh9B,EAAOu/C,EAAOviB,KAAMh9B,EAAMoU,QAAQguB,KAAM,CACrHud,EAAgB3/C,SAKC,MAAjB2/C,IAEF3iB,EAAK+K,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,OAASyxB,EAAOviB,KAAK8iB,UAGhE9iB,EAAK+K,IAAM/K,EAAKlP,OAASmyB,IAC3BA,EAAiBjjB,EAAK+K,IAAM/K,EAAKlP,cAE5B6xB,IAKbK,EAASlyB,OAASmyB,EAAiBD,EAASjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,SAa/D,SAASK,GAAQ1kB,EAAO8jB,EAAQa,EAAWC,OAC3C,IAAIt+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,OACF2C,MAA1B+2C,EAAM15C,GAAGyW,KAAKwnD,SAChBvkB,EAAM15C,GAAGgmD,IAAMwX,EAAOviB,KAAK8iB,cACtB,QAA+Bp7D,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,UAA0BK,EAAkB,KAC/DC,EAAS,MACR,IAAMN,KAAYI,EACjBA,EAAU99D,eAAe09D,IACQ,GAA/BI,EAAUJ,GAAUO,SAAmBH,EAAUJ,GAAUn2C,MAAQu2C,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUn2C,QACvGy2C,GAAUF,EAAUJ,GAAUlyB,OAC9BsyB,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAMuY,GAI9C7kB,EAAM15C,GAAGgmD,IAAMuY,EAAS,GAAMf,EAAOviB,KAAK8iB,SAGzCO,GACHG,GAAe/kB,EAAO8jB,EAAQa,GAY3B,SAASI,GAAe/kB,EAAO8jB,EAAQa,OACvC,IAAMJ,KAAYI,KACjBA,EAAU99D,eAAe09D,GAAW,CAGtCI,EAAUJ,GAAUjY,IAAM,IACvB,KAGG4X,EAAgB,SACf,IAAMc,KAAiBL,KACW,OAAjCA,EAAUK,GAAe1Y,KAAgB0Y,IAAkBT,GAAYI,EAAUJ,GAAUn2C,MAAQu2C,EAAUK,GAAe52C,OAAS62C,GAAiBN,EAAUJ,GAAWI,EAAUK,IAAiB,CACxMd,EAAgBS,EAAUK,SAKT,MAAjBd,IAEFS,EAAUJ,GAAUjY,IAAM4X,EAAc5X,IAAM4X,EAAc7xB,cAEvD6xB,OAGR,IAAI59D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,SACD2C,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,WAChBvkB,EAAM15C,GAAGgmD,IAAMqY,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,UAexE,SAASa,GAA6BC,EAAerB,EAAQa,OAC9DS,GAAa,EAGXC,EAAgB,OAElB,IAAId,KAAYI,EACdA,EAAUJ,GAAU19D,eAAe,SACrCw+D,EAAcV,EAAUJ,GAAUn2C,OAASm2C,EAG3Cc,EAAc5+D,KAAK89D,OAInB,IAAIhnD,EAAI,EAAGA,EAAI8nD,EAAc7+D,OAAQ+W,OACvCgnD,EAAWc,EAAc9nD,GACrBonD,EAAU99D,eAAe09D,GAAW,KAKjC,IAAMS,KAHXI,EAAaA,GAAcT,EAAUJ,GAAU13D,MAC/C83D,EAAUJ,GAAUjY,IAAM,EAEEqY,EACtBA,EAAUK,GAAeF,SAAWH,EAAUJ,GAAUn2C,MAAQu2C,EAAUK,GAAe52C,QAC3Fu2C,EAAUJ,GAAUjY,KAAOqY,EAAUK,GAAe3yB,gBAIlD2N,EAAQmlB,EAAcZ,GACpBj+D,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,SACA2C,IAA3B+2C,EAAM15C,GAAGyW,KAAKwnD,WAChBvkB,EAAM15C,GAAGgmD,IAAMqY,EAAU3kB,EAAM15C,GAAGyW,KAAKwnD,UAAUjY,IAAM,GAAMwX,EAAOviB,KAAK8iB,SAErEM,EAAUJ,GAAU13D,QACtBmzC,EAAM15C,GAAGm+D,QAAUzkB,EAAM15C,GAAGgmD,MAK9B8Y,GAAcT,EAAUJ,GAAU13D,OACpCy3D,GAASa,EAAcZ,GAAWT,EAAQa,EAAUJ,KAiBrD,SAASH,GAAUz9D,EAAGC,EAAGk9D,EAAQnd,UAClCA,EACQhgD,EAAEw3C,MAAQ2lB,EAAOwB,WAlRf,KAkRyC1+D,EAAEu3C,MAAQv3C,EAAEwrC,OAChEzrC,EAAEw3C,MAAQx3C,EAAEyrC,MAAQ0xB,EAAOwB,WAnRhB,KAmRwC1+D,EAAEu3C,OACrDx3C,EAAE2lD,IAAMwX,EAAOO,SApRJ,KAoRwCz9D,EAAE0lD,IAAM1lD,EAAEyrC,QAC7D1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAASyxB,EAAOO,SArRf,KAqRuCz9D,EAAE0lD,IAE5C3lD,EAAEu3C,KAAO4lB,EAAOwB,WAvRb,KAuRwC1+D,EAAEs3C,KAAOt3C,EAAEwrC,OAC9DzrC,EAAEu3C,KAAOv3C,EAAEyrC,MAAQ0xB,EAAOwB,WAxRf,KAwRuC1+D,EAAEs3C,MACpDv3C,EAAE2lD,IAAMwX,EAAOO,SAzRJ,KAyRwCz9D,EAAE0lD,IAAM1lD,EAAEyrC,QAC7D1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAASyxB,EAAOO,SA1Rf,KA0RuCz9D,EAAE0lD,IAWlD,SAAS2Y,GAAiBt+D,EAAGC,OAI5B2+D,EAAc5+D,EAAEqjC,MAAQpjC,EAAEqjC,KAAOtjC,EAAEsjC,IAAMrjC,EAAEojC,MAC3Cw7B,EAAgB7+D,EAAE2lD,IAAO1lD,EAAE0lD,IAAM1lD,EAAEyrC,QAAY1rC,EAAE2lD,IAAM3lD,EAAE0rC,OAAUzrC,EAAE0lD,WACpEiZ,GAAeC,0LCtSXC,GAFM,iBAWbC,yBAOQC,EAAS5oD,EAAM4+C,iCACpBgK,QAAUA,OACVhB,UAAY,QACZiB,cAAgB,QAChBC,kBAAmB,OACnBC,mBAAqB,QACrBC,cAAe,OACfC,sBAAuB,OACvBC,cAAgB,OAChBC,gBAAkBnpD,GAAQA,EAAKsoD,mBAC/B1J,QAAUA,OACVwK,UAAY,UACZC,YAAa,OAMbC,kBAAoB,GAErBtpD,GAAQA,EAAKupD,oBACVA,aAAevpD,EAAKupD,aACF,GAAnBvpD,EAAKwpD,gBACFA,YAAa,OAEbA,YAAa,GAIlBxpD,GAAQA,EAAK6oD,iBACmB,kBAAvB7oD,EAAK6oD,mBACTG,aAAehpD,EAAK6oD,mBACpBC,iBAAmB9oD,EAAK6oD,uBAKzB,IAAMn5D,KAAOsQ,EAAK6oD,mBACfA,cAAcn5D,GAAOsQ,EAAK6oD,cAAcn5D,QACxCs5D,aAAev8D,KAAKu8D,cAAgBhpD,EAAK6oD,cAAcn5D,GAK9DsQ,GAAQA,EAAKypD,gBACVA,WAAazpD,EAAKypD,gBAElBA,WAAa7K,EAAQhjC,QAAQ8tC,qBAG/BC,cAAgB,UAEhB9e,IAAM,QACN/I,MAAQ,CACX9M,MAAO,CACLK,MAAO,EACPC,OAAQ,SAGPP,UAAY,UAEZkO,MAAQ,QACR2mB,aAAe,QACfC,aAAe,QACfC,aAAe,CAClBC,QAAS,GACTC,MAAO,SAEJC,kBAAmB,MAElBC,EAAyB,WAC7B/f,EAAK8f,kBAAmB,QAErBrL,QAAQvZ,KAAKiF,QAAQvH,GAAG,mBAAoBmnB,QAC5CZ,kBAAkB5/D,MAAK,WAC1BygD,EAAKyU,QAAQvZ,KAAKiF,QAAQtH,IAAI,mBAAoBknB,WAG/C/U,eAEAgV,QAAQnqD,oDAQPg1B,EAAQnb,SAASE,cAAc,OACjCttB,KAAKmyD,QAAQhjC,QAAQwuC,cAAcC,MACrCr1B,EAAMD,UAAY,sBAElBC,EAAMD,UAAY,iBAEf8V,IAAI7V,MAAQA,MAEXs1B,EAAQzwC,SAASE,cAAc,OACrCuwC,EAAMv1B,UAAY,YAClBC,EAAM5T,YAAYkpC,QACbzf,IAAIyf,MAAQA,MAEX7V,EAAa56B,SAASE,cAAc,OAC1C06B,EAAW1f,UAAY,YACvB0f,EAAW,aAAehoD,UACrBo+C,IAAI4J,WAAaA,OAEjB5J,IAAIwK,WAAax7B,SAASE,cAAc,YACxC8wB,IAAIwK,WAAWtgB,UAAY,iBAE3B8V,IAAIkK,KAAOl7B,SAASE,cAAc,YAClC8wB,IAAIkK,KAAKhgB,UAAY,iBAKrB8V,IAAImS,OAASnjC,SAASE,cAAc,YACpC8wB,IAAImS,OAAO/7B,MAAMyqB,WAAa,cAC9Bb,IAAImS,OAAO/7B,MAAMkL,SAAW,gBAC5B0e,IAAImS,OAAO7E,UAAY,QACvBtN,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAImS,wCAOnCh9C,OACFvT,KAAKmyD,QAAQ2L,iBAAiBC,gBAG9BlqC,EACAmqC,OAEAzqD,GAAQA,EAAK+oD,uBACV,IAAMr5D,KAAOsQ,EAAK+oD,wBAChBA,mBAAmBr5D,GAAOsQ,EAAK+oD,mBAAmBr5D,MAIvDjD,KAAKmyD,QAAQhjC,SAAWnvB,KAAKmyD,QAAQhjC,QAAQ+Y,cAE/CrU,GADAmqC,EAAmBjoB,UAAKoc,QAAQhjC,QAAQ+Y,sBAAmBloC,OAChCuT,EAAMvT,KAAKo+C,IAAIyf,YAE1ChqC,EAAUtgB,GAAQA,EAAKsgB,WAGrBA,aAAmBoqC,QAAS,MACvBj+D,KAAKo+C,IAAIyf,MAAMlW,iBACfvJ,IAAIyf,MAAMr2B,YAAYxnC,KAAKo+C,IAAIyf,MAAMlW,iBAEvCvJ,IAAIyf,MAAMlpC,YAAYd,QAClBA,aAAmB33B,QAAU23B,EAAQqqC,mBAIrCrqC,aAAmB33B,OAC5B8hE,EAAiBzqD,EAAMvT,KAAKo+C,IAAIyf,YAE3Bzf,IAAIyf,MAAMnS,UADN73B,MAAAA,EACkBA,EAEA7zB,KAAKm8D,SAAW,SAIxC/d,IAAI7V,MAAMknB,MAAQl8C,GAAQA,EAAKk8C,OAAS,GACxCzvD,KAAKo+C,IAAIyf,MAAMlW,WAIlBrP,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIyf,MAAO,cAHrCvlB,GAAKuV,aAAa7tD,KAAKo+C,IAAIyf,MAAO,cAMhCtqD,GAAQA,EAAKupD,cACV98D,KAAK88D,cAAgB98D,KAAK88D,cAAgBvpD,EAAKupD,oBAC7CA,aAAevpD,EAAKupD,mBAGHr9D,IAApB8T,EAAKwpD,iBAAgDt9D,IAApBO,KAAK+8D,aACjB,GAAnBxpD,EAAKwpD,gBACFA,YAAa,OAEbA,YAAa,GAItBzkB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,qBAC9BvoC,KAAK+8D,YACPzkB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,aACrC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,cAElC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,YACrC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,eAE3BvoC,KAAK88D,oBACTA,aAAe,KACpBxkB,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,aACrC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,YACrC+P,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAO,sBAGnCh1B,IAASA,EAAK4qD,WAAY5qD,EAAK2pD,gBACjC5kB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,oBAC9Bh1B,EAAK4qD,UACP7lB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,mBAAqBh1B,EAAK4qD,WAG5D7lB,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,qCAGpC+P,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAO,yBAI9BD,EAAY/0B,GAAQA,EAAK+0B,WAAa,KACxCA,GAAatoC,KAAKsoC,YAChBtoC,KAAKsoC,YACPgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI7V,MAAOvoC,KAAKsoC,WAC1CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAI4J,WAAYhoD,KAAKsoC,WAC/CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIwK,WAAY5oD,KAAKsoC,WAC/CgQ,GAAKwV,gBAAgB9tD,KAAKo+C,IAAIkK,KAAMtoD,KAAKsoC,YAE3CgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAI7V,MAAOD,GAClCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAI4J,WAAY1f,GACvCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAIwK,WAAYtgB,GACvCgQ,GAAKuV,aAAa7tD,KAAKo+C,IAAIkK,KAAMhgB,QAC5BA,UAAYA,GAIftoC,KAAKw0B,QACP8jB,GAAK8lB,cAAcp+D,KAAKo+C,IAAI7V,MAAOvoC,KAAKw0B,YACnCA,MAAQ,MAEXjhB,GAAQA,EAAKihB,QACf8jB,GAAK+lB,WAAWr+D,KAAKo+C,IAAI7V,MAAOh1B,EAAKihB,YAChCA,MAAQjhB,EAAKihB,uDASbx0B,KAAKq1C,MAAM9M,MAAMK,2DAQlB01B,EAAet+D,KAAKo+C,IAAImS,OAAOxE,gBACjCuS,GAAgBt+D,KAAKu+D,iBAAkB,MACpCA,iBAAmBD,MAClBE,EAAc,GAChBC,EAAoB,QAExBnmB,SAAAA,GAAat4C,KAAKw2C,OAAO,SAACuB,EAAM90C,MAC9B80C,EAAK2mB,OAAQ,EACT3mB,EAAK4mB,UAAW,CAElBH,EAAYv7D,GAAO80C,EAAKsY,QADJ,GAEpBoO,EAAoBD,EAAYv7D,GAAKjG,WAItByhE,EAAoB,qBAG5B3hE,MACPw7C,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI9hE,SAFCA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,UAMJ,SAEA,iEASsCkD,KAAKo+C,IAAI4J,WAAhD6W,IAAAA,UAAWC,IAAAA,WAAY1V,IAAAA,iBAC1BtG,IAAM+b,OACNlqB,MAAQmqB,OACRl2B,MAAQwgB,uDAQPzK,EAAK3+C,KACL++D,EAAiB/+D,KAAKmyD,QAAQhjC,QAAQ6vC,UACtCC,EAAc,CAClBC,oBAAqBl/D,KAAKmyD,QAAQgN,iBAClCC,WAAYL,GAAkBA,EAAeM,UAC7CC,iBAAkBP,GAAkBA,EAAe/4D,SACnDw2D,qBAAsBx8D,KAAKw8D,sBAEzB+C,EAAO,SACNv/D,KAAKmyD,QAAQL,gBAAiB,IAC7BmN,EAAYzC,4BAA+B,EAC3Ch7D,KAAKc,IAAI0kC,KAAa,IAAItqC,KAAKuiE,EAAYC,sBAAwBD,EAAYG,aAC7EH,EAAYK,kBAAwD,MAApCt/D,KAAKmyD,QAAQqN,oBAC/CP,EAAYK,kBAAiB,SAAAG,GAC3B9gB,EAAGwT,QAAQqN,oBAAsBC,EACjCF,GAAQE,KAGVF,EAD2C,GAAlC5gB,EAAGwT,QAAQqN,4BAQnBD,uCAWIG,EAAcC,EAAerF,EAAQ5gB,iBAChCgmB,GAAgB1/D,KAAK48D,YAAc58D,KAAK28D,YAAcgD,EAGzD,iBACLtC,EAAe,CACnBE,MAAOzmB,UAAKumB,aAAaE,eAAa,SAAAxlB,UAASA,EAAK6nB,aACpDtC,QAASxmB,UAAKumB,aAAaC,iBAAe,SAAAvlB,UAASA,EAAK6nB,cAGpDC,EAAkB,CACtBtC,SAAW,OAAQzmB,eAAKumB,aAAaE,eAAU,SAAAxlB,UAAQA,EAAK+nB,oBAAgB,SAAA/nB,WAAUA,OACtFulB,WAAa,OAAQxmB,eAAKumB,aAAaC,iBAAY,SAAAvlB,UAAQA,EAAK+nB,oBAAgB,SAAA/nB,WAAUA,QAOtF6d,EAAkB,qBAChBuH,EAAexH,EAAKoK,oBAAoB1C,EAAcvmB,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAASA,EAAK6nB,aAAYlmB,GACzGsmB,EAAkBrK,EAAKsK,uBAAuBJ,EAAiB/oB,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAAQA,EAAK6nB,aAAYlmB,6BAC5GyjB,MAAiB6C,KAQxBE,EAAmC,SAAAC,OACnCC,EAAwB,cACjBrF,SACHvkB,EAAQM,KAAA6e,EAAKwH,sBAAoB,SAAAplB,UAAQA,EAAKxkC,KAAKwnD,WAAaA,KACtEqF,EAAsBrF,GAAYoF,EAAUE,GAAA7pB,QAAAA,GAAW,SAACr5C,EAAGC,UAAM+iE,EAAQhjE,EAAEoW,KAAMnW,EAAEmW,SAASijC,OAFzF,IAAMukB,KAAYpF,EAAKwF,YAAjBJ,UAIJqF,MAGiC,mBAA/BpgE,KAAKmyD,QAAQhjC,QAAQyuC,MAAsB,KAG9Cjf,EAAK3+C,QACPA,KAAKu8D,cAAgBv8D,KAAKmyD,QAAQhjC,QAAQosC,eAAgB,CAG5Dl4D,GAD8B68D,EAAiClgE,KAAKmyD,QAAQhjC,QAAQyuC,OAC1BtD,EAAQt6D,KAAKm7D,gBAClEgC,aAAevH,SACf0K,uBAAuBhG,OAEzB,kBACE6C,aAAevH,SACf0K,uBAAuBhG,OAGtBiG,EAAqBF,oBAAKlD,+BAEA,SAAAplB,UAAQA,EAAK6nB,YAAe7nB,EAAK6nB,YAAc7nB,EAAK+nB,oBACtD,SAAC3iE,EAAGC,UACCuhD,EAAGwT,QAAQhjC,QAAQyuC,MAAMzgE,EAAEoW,KAAMnW,EAAEmW,cAEjEipD,qBAAuBn5D,GAAYk9D,EAAoBjG,GAAQ,EAAMvkB,UAAKyqB,+BAA4BxgE,YAExG,eAEAm9D,aAAevH,SACf0K,uBAAuBhG,GAExBt6D,KAAKmyD,QAAQhjC,QAAQ9rB,SACnBrD,KAAKu8D,cAAgBv8D,KAAKmyD,QAAQhjC,QAAQosC,eAE5Cl4D,GAD8B68D,IAC4B5F,EAAQt6D,KAAKm7D,qBAIlEqB,qBAAuBn5D,GAAYrD,KAAKm9D,aAAc7C,GAAQ,EAAMvkB,UAAKyqB,+BAA4BxgE,YAI5GqD,GAAcrD,KAAKm9D,aAAc7C,EAAQt6D,KAAKm7D,UAAWn7D,KAAKmyD,QAAQhjC,QAAQosC,oBAI7E,IAAIz+D,EAAI,EAAGA,EAAIkD,KAAKm9D,aAAangE,OAAQF,SACvCqgE,aAAargE,GAAG2jE,mBAC+ChhE,IAAhEO,KAAKs8D,mBAAmBt8D,KAAKm9D,aAAargE,GAAGyW,KAAKwnD,YAC/C/6D,KAAKs8D,mBAAmBt8D,KAAKm9D,aAAargE,GAAGyW,KAAKwnD,gBAChDoC,aAAargE,GAAGozD,QAKvBlwD,KAAKmyD,QAAQhjC,QAAQ2wC,YACvBxnB,SAAAA,GAAat4C,KAAKw2C,OAAO,SAAAuB,GACnBA,EAAK+nB,SAAW/nB,EAAK4mB,WACvB5mB,EAAKmY,UAKPlwD,KAAKw8D,2BACFrK,QAAQvZ,KAAKiF,QAAQ4C,KAAK,wBAE5Bmc,YAAa,sCAUX/jB,EAAShQ,GAClBgQ,EAAUP,GAAKooB,eAAe1gE,KAAM,SAAU6oC,IAAWgQ,MAEnD8nB,EAAa3gE,KAAKo+C,IAAIyf,MAAMld,YAC5BigB,EAAc5gE,KAAKo+C,IAAIyf,MAAM9R,oBACnClT,EAAUP,GAAKooB,eAAe1gE,KAAKq1C,MAAM9M,MAAO,QAASo4B,IAAe9nB,EACxEA,EAAUP,GAAKooB,eAAe1gE,KAAKq1C,MAAM9M,MAAO,SAAUq4B,IAAgB/nB,4CAQ1DhQ,QACXuV,IAAIwK,WAAWp0B,MAAMqU,iBAAYA,aACjCuV,IAAI4J,WAAWxzB,MAAMqU,iBAAYA,aACjCuV,IAAI7V,MAAM/T,MAAMqU,iBAAYA,6DAONyxB,OACtB,IAAIx9D,EAAI,EAAGmO,EAAKjL,KAAKm9D,aAAangE,OAAQF,EAAImO,EAAInO,IAAK,KACpDi7C,EAAO/3C,KAAKm9D,aAAargE,GAC/Bi7C,EAAK8oB,YAAYvG,GACZt6D,KAAK28D,WAAa38D,KAAKm8D,SAAWF,IACjClkB,EAAK4mB,WAAW5mB,EAAKmY,uCAaxBxW,EAAO4gB,EAAQoF,EAAcoB,iBAG9Bj4B,EAsDEtwB,SAxDFsgC,GAAU,EACR8mB,EAAgB3/D,KAAK28D,UAGrBtK,EAAQ,CACZ,WACEqN,EAAeqB,EAAKC,uBAAuB3kE,KAAK0kE,IAASrB,aAItDY,+BAA4BtgE,KAAMs6D,aAGlC2G,uCAAoCjhE,MAEzC,iBACE+gE,EAAKpE,UAAY5mB,KAAAgrB,EAAKG,wBAAqBH,EAA1BhrB,CAAgC2D,EAAO4gB,IAG1D,sBACEyG,EAAKI,qBAAkBJ,GAAMrB,EAAcC,EAAerF,EAAQ5gB,cAI/D0nB,8BAA2BphE,MAEhC,iBACE6oC,EAASkN,KAAAgrB,EAAKM,yBAAsBN,EAA3BhrB,CAAiCukB,cAIvC2G,uCAAoCjhE,MAEzC,iBACE64C,EAAU9C,KAAAgrB,EAAKO,mBAAgBP,EAArBhrB,CAA2B8C,EAAShQ,IAGhD,sBACEk4B,EAAKQ,0BAAuBR,GAAMl4B,IAGpC,sBACEk4B,EAAKS,qCAAkCT,GAAMzG,IAG/CvkB,KAAC,kBACMgrB,EAAKpE,WAAaoE,EAAKl4B,SAC1BgQ,GAAU,GAELA,WACD74C,cAGN8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,kDAUY+hD,iBACjBta,GAAYhgD,KAAKm7D,WAAWn+D,OAAS,EAAG,KACpC2hD,EAAK3+C,UAENyhE,qBAELnpB,SAAAA,GAAat4C,KAAKm9D,cAAc,SAAAplB,QACHt4C,IAAvBs4C,EAAKxkC,KAAKwnD,WACZpc,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUlyB,OAASrnC,KAAK8D,IAAIq5C,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUlyB,OAAQkP,EAAKlP,OAASyxB,EAAOviB,KAAK8iB,UACtHlc,EAAGwc,UAAUpjB,EAAKxkC,KAAKwnD,UAAUO,aAAiE,IAAhDoG,EAAKpF,mBAAmBvkB,EAAKxkC,KAAKwnD,WAAmC94B,QAAQy/B,EAAKpF,mBAAmBvkB,EAAKxkC,KAAKwnD,wDAczJrhB,EAAO4gB,UACbt6D,KAAK8iD,KAAOpJ,EAAMd,KAAKY,SAASC,gBAAgB5Q,OAAS6Q,EAAMd,KAAKY,SAASoY,UAAY0I,EAAOhS,MACpGtoD,KAAK8iD,IAAM9iD,KAAK6oC,OAASyxB,EAAOhS,OAAU5O,EAAMd,KAAKY,SAASoY,mDASnD0I,OAEXzxB,EAEA2N,MAGFA,EADsB,UAApBx2C,KAAKg9D,WACC1kB,GAAK51B,QAAQ1iB,KAAKw2C,OAGlBx2C,KAAKm9D,cAGLngE,OAAS,EAAG,KAChBoF,EAAMo0C,EAAM,GAAGsM,IACfx9C,EAAMkxC,EAAM,GAAGsM,IAAMtM,EAAM,GAAG3N,aAClCyP,SAAAA,GAAa9B,GAAO,SAAAuB,GAClB31C,EAAMZ,KAAKY,IAAIA,EAAK21C,EAAK+K,KACzBx9C,EAAM9D,KAAK8D,IAAIA,EAAMyyC,EAAK+K,IAAM/K,EAAKlP,WAEnCzmC,EAAMk4D,EAAOhS,KAAM,KAEf9rC,EAASpa,EAAMk4D,EAAOhS,KAC5BhjD,GAAOkX,KACP87B,SAAAA,GAAa9B,GAAO,SAAAuB,GAClBA,EAAK+K,KAAOtmC,KAGhBqsB,EAASrnC,KAAKC,KAAK6D,EAAMg1D,EAAOviB,KAAK8iB,SAAW,GACxB,aAApB76D,KAAKg9D,aACPn0B,EAASrnC,KAAK8D,IAAIujC,EAAQ7oC,KAAKq1C,MAAM9M,MAAMM,cAI7CA,EAAc7oC,KAAKq1C,MAAM9M,MAAMM,cAE1BA,iCAOF7oC,KAAKo+C,IAAI7V,MAAMhB,iBACb4qB,QAAQ/T,IAAIujB,SAAShtC,YAAY30B,KAAKo+C,IAAI7V,OAG5CvoC,KAAKo+C,IAAI4J,WAAWzgB,iBAClB4qB,QAAQ/T,IAAI4J,WAAWrzB,YAAY30B,KAAKo+C,IAAI4J,YAG9ChoD,KAAKo+C,IAAIwK,WAAWrhB,iBAClB4qB,QAAQ/T,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAIwK,YAG9C5oD,KAAKo+C,IAAIkK,KAAK/gB,iBACZ4qB,QAAQ/T,IAAIkK,KAAK3zB,YAAY30B,KAAKo+C,IAAIkK,yCAQvC/f,EAAQvoC,KAAKo+C,IAAI7V,MACnBA,EAAMhB,YACRgB,EAAMhB,WAAWC,YAAYe,OAGzByf,EAAahoD,KAAKo+C,IAAI4J,WACxBA,EAAWzgB,YACbygB,EAAWzgB,WAAWC,YAAYwgB,OAG9BY,EAAa5oD,KAAKo+C,IAAIwK,WACxBA,EAAWrhB,YACbqhB,EAAWrhB,WAAWC,YAAYohB,OAG9BN,EAAOtoD,KAAKo+C,IAAIkK,KAClBA,EAAK/gB,YACP+gB,EAAK/gB,WAAWC,YAAY8gB,+BAQ5BvQ,iBACGvB,MAAMuB,EAAK3kB,IAAM2kB,EACtBA,EAAK6pB,UAAU5hE,WACV48D,YAAa,OAESn9D,IAAvBs4C,EAAKxkC,KAAKwnD,gBACP8G,eAAe9pB,QACf+pB,mBAGFC,UAAK5E,qBAAsBplB,GAAO,KAC/B2B,EAAQ15C,KAAKmyD,QAAQvZ,KAAKc,WAC3BsoB,gBAAgBjqB,EAAM/3C,KAAKm9D,aAAczjB,2CASnC3B,OAAMkqB,yDAAWlqB,EAAKxkC,KAAKwnD,SACtBt7D,MAAdwiE,QAA0DxiE,IAA/BO,KAAKm7D,UAAU8G,UACvC9G,UAAU8G,GAAc,CAC3Bp5B,OAAQ,EACRia,IAAK,EACLtiB,MAAOuX,EAAKxkC,KAAKitB,MACjBC,IAAKsX,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MAChC86B,SAAS,EACT12C,MAAO5kB,KAAKy8D,cACZjmB,MAAO,GACPnzC,MAAOrD,KAAKq8D,kBAAoBr8D,KAAKo8D,cAAc6F,KAAe,QAE/DxF,iBAIH,IAAI//D,KAAKq7C,EAAKxkC,KAAKitB,OAAS,IAAI9jC,KAAKsD,KAAKm7D,UAAU8G,GAAYzhC,cAC7D26B,UAAU8G,GAAYzhC,MAAQuX,EAAKxkC,KAAKitB,WAGzC0hC,EAAUnqB,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MACvC,IAAI9jC,KAAKwlE,GAAW,IAAIxlE,KAAKsD,KAAKm7D,UAAU8G,GAAYxhC,YACrD06B,UAAU8G,GAAYxhC,IAAMyhC,QAG9B/G,UAAU8G,GAAYzrB,MAAMv5C,KAAK86C,uDAOhC4G,EAAK3+C,QACP2+C,EAAGwc,UAAW,gBACLJ,SACHoH,EAAaxjB,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKktB,KAAOke,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKitB,MAChGogB,EAAWjC,EAAGwc,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKitB,MAChDqgB,EAASshB,EAAa,OAE1BxjB,EAAGwc,UAAUJ,GAAUvkB,eAAc,SAAAuB,GAC/B,IAAIr7C,KAAKq7C,EAAKxkC,KAAKitB,OAAS,IAAI9jC,KAAKkkD,KACvCA,EAAW7I,EAAKxkC,KAAKitB,WAGjB0hC,EAAUnqB,EAAKxkC,KAAKktB,KAAOsX,EAAKxkC,KAAKitB,MACvC,IAAI9jC,KAAKwlE,GAAW,IAAIxlE,KAAKmkD,KAC/BA,EAASqhB,MAIbvjB,EAAGwc,UAAUJ,GAAUv6B,MAAQogB,EAC/BjC,EAAGwc,UAAUJ,GAAUt6B,IAAM,IAAI/jC,KAAKmkD,EAAS,QAjB5C,IAAMka,KAAYpc,EAAGwc,YAAfJ,qDA2BgBt7D,IAAzBO,KAAK08D,gBAA+B,KAChC0F,EAAY,MACiB,iBAAxBpiE,KAAK08D,gBAA6B,KACtC,IAAM3B,KAAY/6D,KAAKm7D,UAC1BiH,EAAUnlE,KAAK,CAAC89D,SAAAA,EAAUsH,UAAWriE,KAAKm7D,UAAUJ,GAAUvkB,MAAM,GAAGjjC,KAAKvT,KAAK08D,sBAEnF0F,QAAAA,GAAe,SAACjlE,EAAGC,UAAMD,EAAEklE,UAAYjlE,EAAEilE,kBAEtC,GAAmC,mBAAxBriE,KAAK08D,gBAA+B,KAC7C,IAAM3B,KAAY/6D,KAAKm7D,UAC1BiH,EAAUnlE,KAAK+C,KAAKm7D,UAAUJ,GAAUvkB,MAAM,GAAGjjC,SAEnD6uD,QAAAA,EAAepiE,KAAK08D,oBAGlB0F,EAAUplE,OAAS,MAChB,IAAIF,EAAI,EAAGA,EAAIslE,EAAUplE,OAAQF,SAC/Bq+D,UAAUiH,EAAUtlE,GAAGi+D,UAAUn2C,MAAQ9nB,iDAU/C,IAAMi+D,KAAY/6D,KAAKm7D,UACtBn7D,KAAKm7D,UAAU99D,eAAe09D,UAC3BI,UAAUJ,GAAUO,SAAU,OAC9BH,UAAUJ,GAAUlyB,OAAS,kCASjCkP,kBACE/3C,KAAKw2C,MAAMuB,EAAK3kB,IACvB2kB,EAAK6pB,UAAU,WACVhF,YAAa,MAGZh4C,EAAQgvC,UAAKuJ,qBAAqBplB,IAC1B,GAAVnzB,GAAa09C,UAAKnF,qBAAoBv4C,EAAO,QAEvBnlB,IAAvBs4C,EAAKxkC,KAAKwnD,gBACNwH,oBAAoBxqB,QACpB+pB,8DASW/pB,OAAMkqB,yDAAWlqB,EAAKxkC,KAAKwnD,YAC3Bt7D,MAAdwiE,EAAyB,KACrBlH,EAAW/6D,KAAKm7D,UAAU8G,MAC5BlH,EAAS,SACLyH,EAAY5O,KAAAmH,EAASvkB,cAAcuB,MAErCyqB,GAAa,OACfzH,EAASvkB,cAAagsB,EAAU,GAC3BzH,EAASvkB,MAAMx5C,YAGbokE,+BAFEphE,KAAKm7D,UAAU8G,+CAadlqB,QACXoa,QAAQsQ,WAAW1qB,EAAK3kB,4CAOvB/sB,EAAQiyC,GAAK51B,QAAQ1iB,KAAKw2C,OAC1BksB,EAAa,GACbC,EAAW,GAER7lE,EAAI,EAAGA,EAAIuJ,EAAMrJ,OAAQF,SACN2C,IAAtB4G,EAAMvJ,GAAGyW,KAAKktB,KAChBkiC,EAAS1lE,KAAKoJ,EAAMvJ,IAEtB4lE,EAAWzlE,KAAKoJ,EAAMvJ,SAEnBugE,aAAe,CAClBC,QAASoF,EACTnF,MAAOoF,GAGTt/D,GAAmBrD,KAAKq9D,aAAaC,SACrCj6D,GAAiBrD,KAAKq9D,aAAaE,mDAWjBF,EAAcuF,EAAiBlpB,OAC3CyjB,EAAe,GACf0F,EAAqB,OAEtB7iE,KAAK28D,WAAa38D,KAAKm8D,SAAWF,GAA6B,KAC7D,IAAIn/D,EAAI,EAAGA,EAAI8lE,EAAgB5lE,OAAQF,IAAK,KAC3Ci7C,EAAO6qB,EAAgB9lE,GACvBi7C,EAAK4mB,WAAW5mB,EAAKmY,cAEpBiN,MAGHte,GAAYnF,EAAMjZ,IAAMiZ,EAAMlZ,OAAS,EACvCsiC,EAAappB,EAAMlZ,MAAQqe,EAC3BkkB,EAAarpB,EAAMjZ,IAAMoe,KAkB3B+jB,EAAgB5lE,OAAS,MACtB,IAAIF,EAAI,EAAGA,EAAI8lE,EAAgB5lE,OAAQF,SACrCkmE,6BAA6BJ,EAAgB9lE,GAAIqgE,EAAc0F,EAAoBnpB,OAKtFupB,EAAoB3qB,GAAK4qB,mBAAmB7F,EAAaC,SAtBnC,SAAAx7D,UACjBA,EAAQghE,GAAsB,EAC9BhhE,GAASihE,EAAqB,EACA,IAmBoD,OAAO,iBAG/FI,cAAcF,EAAmB5F,EAAaC,QAASH,EAAc0F,GAAoB,SAAA9qB,UAAQA,EAAKxkC,KAAKitB,MAAQsiC,GAAc/qB,EAAKxkC,KAAKitB,MAAQuiC,KAI3H,GAAzB/iE,KAAKw9D,iBAA0B,MAC5BA,kBAAmB,MACnB,IAAI1gE,EAAI,EAAGA,EAAIugE,EAAaE,MAAMvgE,OAAQF,SACxCkmE,6BAA6B3F,EAAaE,MAAMzgE,GAAIqgE,EAAc0F,EAAoBnpB,OAG1F,KAEG0pB,EAAkB9qB,GAAK4qB,mBAAmB7F,EAAaE,OA9BrC,SAAAz7D,UACfA,EAAQghE,GAAsB,EACA,IA4BgD,OAAO,YAGzFK,cAAcC,EAAiB/F,EAAaE,MAAOJ,EAAc0F,GAAoB,SAAA9qB,UAAQA,EAAKxkC,KAAKktB,IAAMqiC,GAAc/qB,EAAKxkC,KAAKitB,MAAQuiC,aAG9IvE,EAAc,GAChBC,EAAoB,EAEf3hE,EAAI,EAAGA,EAAIqgE,EAAangE,OAAQF,IAAK,KACtCi7C,EAAOolB,EAAargE,OACrBi7C,EAAK4mB,UAAW,CAEnBH,EAAY1hE,GAAKi7C,EAAKsY,QADF,GAEpBoO,EAAoBD,EAAY1hE,GAAGE,WAIpByhE,EAAoB,qBAG5B1qD,MACPukC,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI7qD,SAFCA,EAAI,EAAGA,EAAI0qD,EAAmB1qD,MAA9BA,OAON,IAAIjX,EAAI,EAAGA,EAAIqgE,EAAangE,OAAQF,IACvCqgE,EAAargE,GAAG2jE,qBAGXtD,wCAWKkG,EAAY7sB,EAAO2mB,EAAc0F,EAAoBS,OAC9C,GAAfD,EAAkB,KACf,IAAIvmE,EAAIumE,EAAYvmE,GAAK,EAAGA,IAAK,KAChCi7C,EAAOvB,EAAM15C,MACbwmE,EAAevrB,SAIXA,EAAK6nB,YAAe7nB,EAAKwrB,YAAgBxrB,EAAK+nB,cACdrgE,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,QAMrB,IAAIj7C,EAAIumE,EAAa,EAAGvmE,EAAI05C,EAAMx5C,OAAQF,IAAK,KAC9Ci7C,EAAOvB,EAAM15C,MACbwmE,EAAevrB,SAIXA,EAAK6nB,YAAc7nB,EAAKwrB,YAAgBxrB,EAAK+nB,cACbrgE,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,8CAmBdA,EAAMolB,EAAczjB,GAC5B3B,EAAK4kB,UAAUjjB,IACZ3B,EAAK4mB,WAAW5mB,EAAKyrB,OAE1BzrB,EAAK0oB,cACLtD,EAAalgE,KAAK86C,IAGdA,EAAK4mB,WAAW5mB,EAAKmY,4DAgBFnY,EAAMolB,EAAc0F,EAAoBnpB,GAC/D3B,EAAK4kB,UAAUjjB,QACmBj6C,IAAhCojE,EAAmB9qB,EAAK3kB,MAC1ByvC,EAAmB9qB,EAAK3kB,KAAM,EAC9B+pC,EAAalgE,KAAK86C,IAIhBA,EAAK4mB,WAAW5mB,EAAKmY,sDAYN2P,EAAiB4D,EAAoB/pB,OAEpDsmB,EAAkB,GAClB0D,EAAwB,MAE1BD,EAAmBzmE,OAAS,MACzB,IAAIF,EAAI,EAAGA,EAAI2mE,EAAmBzmE,OAAQF,SACxCkmE,6BAA6BS,EAAmB3mE,GAAIkjE,EAAiB0D,EAAuBhqB,OAIhG,IAAI58C,EAAI,EAAGA,EAAI+iE,EAAgBvC,QAAQtgE,OAAQF,SAC7CkmE,6BAA6BnD,EAAgBvC,QAAQxgE,GAAIkjE,EAAiB0D,EAAuBhqB,OAGnG,IAAI58C,EAAI,EAAGA,EAAI+iE,EAAgBtC,MAAMvgE,OAAQF,SAC3CkmE,6BAA6BnD,EAAgBtC,MAAMzgE,GAAIkjE,EAAiB0D,EAAuBhqB,WAGhG8kB,EAAc,GAChBC,EAAoB,EAEf3hE,EAAI,EAAGA,EAAIkjE,EAAgBhjE,OAAQF,IAAK,KACzCi7C,EAAOioB,EAAgBljE,OACxBi7C,EAAK4mB,UAAW,CAEnBH,EAAY1hE,GAAKi7C,EAAKsY,QADF,GAEpBoO,EAAoBD,EAAY1hE,GAAGE,WAIpByhE,EAAoB,MAGhC,IAAI1qD,EAAI,EAAGA,EAAI0qD,EAAmB1qD,OACrCukC,SAAAA,GAAakmB,GAAa,SAAUI,GAClCA,EAAI7qD,YAKL,IAAIjX,EAAI,EAAGA,EAAIkjE,EAAgBhjE,OAAQF,IAC1CkjE,EAAgBljE,GAAG2jE,qBAGdT,yCASMjoB,EAAM4rB,EAAaC,QAC3BrB,oBAAoBxqB,EAAM4rB,QAC1B9B,eAAe9pB,EAAM6rB,QACrB9B,uDAUD+B,WAFC3T,OAGG2T,EAAkB7jE,KAAK68D,kBAAkB7qB,OAC/C6xB,aChoCAC,0BAMQ3H,EAAS5oD,EAAM4+C,sDACnBgK,EAAS5oD,EAAM4+C,KAGhBvpB,MAAQ,IACRC,OAAS,IACTia,IAAM,IACNpO,KAAO,sDAUPgF,EAAO4gB,EAAQoF,QAGfvC,aAAen9D,KAAK+/D,oBAAoB//D,KAAKq9D,aAAcr9D,KAAKm9D,aAAczjB,QAG9E9Q,MAAQ5oC,KAAKo+C,IAAIwK,WAAWQ,iBAG5BhL,IAAIwK,WAAWp0B,MAAMqU,OAAU,QAG/B,IAAI/rC,EAAI,EAAGmO,EAAKjL,KAAKm9D,aAAangE,OAAQF,EAAImO,EAAInO,IAAK,CAC7CkD,KAAKm9D,aAAargE,GAC1B+jE,YAAYvG,UAbH,iCAuBXt6D,KAAKo+C,IAAIwK,WAAWrhB,iBAClB4qB,QAAQ/T,IAAIwK,WAAWj0B,YAAY30B,KAAKo+C,IAAIwK,mBAhDzBsT,imVCIxB6H,yBAUQxwD,EAAMmoC,EAAYvsB,gCACvBiE,GAAK,UACLxC,OAAS,UACTrd,KAAOA,OACP6qC,IAAM,UACN1C,WAAaA,GAAc,QAC3BwB,eAAiB,CACpBtqC,QAAAA,GACAlV,OAAQ,WAELyxB,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,eAAgB/tB,QAC/CA,QAAQvc,QAAU0lC,GAAKh7C,OAAO,GAAIsV,GAAS5S,KAAKmvB,QAAQvc,aACvD88C,EAAiB1vD,KAAKk9C,eAAetqC,QAAQ5S,KAAKk9C,eAAex/C,gBAC3DsC,KAAKmvB,QAAQvc,kBAAiB,SAAAlV,GACxCggD,EAAKvuB,QAAQvc,QAAQlV,GAAU46C,GAAKh7C,OAClC,GACAoyD,EACAhS,EAAKvuB,QAAQvc,QAAQlV,YAGpBsmE,UAAW,OACXrF,WAAY,OACZsF,cAAe,OACfC,WAAc/0C,GAAWA,EAAQ+0C,aAAe,OAChDxF,OAAQ,OAER5b,IAAM,UACNnO,MAAQ,UACRD,KAAO,UACP9L,MAAQ,UACRC,OAAS,UAETs7B,iBAAiB5wD,QAEjB+8C,SAAW,UACX8T,gEAODpkE,KAAKkkE,kBACFF,UAAW,OACXtF,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,kDAQtB2T,UAAW,OACXtF,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,yCAQnB98C,GAC6B9T,MAAd8T,EAAK8wD,OAAsBrkE,KAAKuT,KAAK8wD,OAAS9wD,EAAK8wD,OACrC,MAAfrkE,KAAK4wB,aAClBA,OAAOuhC,QAAQmS,aAAatkE,KAAMuT,EAAK8wD,YAGzCF,iBAAiB5wD,GAElBvT,KAAK4wB,cACFA,OAAOgsC,YAAa,GAGcn9D,MAAjB8T,EAAKwnD,UAAyB/6D,KAAKuT,KAAKwnD,UAAYxnD,EAAKwnD,UAC3C,MAAf/6D,KAAK4wB,aACrBA,OAAO2zC,eAAevkE,KAAMA,KAAKuT,KAAKwnD,SAAUxnD,EAAKwnD,eAGvDxnD,KAAOA,OACP6wD,yBACA1F,OAAQ,EACT1+D,KAAK2+D,WAAW3+D,KAAKqwD,kDAQV98C,GACXA,SACG2wD,gBAAwC,IAApB3wD,EAAK2wD,YAAoCjiC,QAAQ1uB,EAAK2wD,+CAQzEtzC,GACJ5wB,KAAK2+D,gBACFzO,YACAt/B,OAASA,EACV5wB,KAAK4wB,aACF4yC,aAIF5yC,OAASA,oCASR8oB,UACD,wCAQA,wCAQA,kKA6BH15C,KAAKgkE,UAAYhkE,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAIqmB,WAAY,SAC/D9lB,EAAK3+C,KAELykE,EAAar3C,SAASE,cAAc,OAC1Cm3C,EAAWn8B,UAAY,kBACvBm8B,EAAWC,eAAiB1kE,UACvB2kE,iBAAmB,IAAI3gB,GAAOygB,QAE9BE,iBAAiBruB,GAAG,OAAO,SAAAkK,GAC9B7B,EAAG/tB,OAAOuhC,QAAQvZ,KAAKiF,QAAQ4C,KAAK,QAAU,CAC5CD,MAAAA,EACAzI,KAAM4G,EAAGvrB,aAGRuxC,iBAAiBruB,GAAG,aAAa,SAAAkK,GACpCA,EAAM8D,kBACN3F,EAAG/tB,OAAOuhC,QAAQyS,cAAcjmB,GAChCA,EAAG/tB,OAAOuhC,QAAQvZ,KAAKiF,QAAQ4C,KAAK,cAAe,CACjDD,MAAAA,EACAzI,KAAM4G,EAAGvrB,aAGRuxC,iBAAiBruB,GAAG,YAAY,SAAAkK,GAEnCA,EAAM8D,kBACN3F,EAAG/tB,OAAOuhC,QAAQrU,aAAa0C,WAE5BmkB,iBAAiBruB,GAAG,UAAYP,KAAA4I,EAAG/tB,OAAOuhC,QAAQpU,gBAAaY,EAAG/tB,OAAOuhC,eACzEwS,iBAAiBruB,GAAG,SAAYP,KAAA4I,EAAG/tB,OAAOuhC,QAAQnU,mBAAgBW,EAAG/tB,OAAOuhC,UAE7EnyD,KAAKo+C,IAAIymB,IACP7kE,KAAKo+C,IAAI0mB,cACN1mB,IAAIymB,IAAI78B,aAAay8B,EAAYzkE,KAAKo+C,IAAI0mB,eAG1C1mB,IAAIymB,IAAIlwC,YAAY8vC,GAGpBzkE,KAAKo+C,IAAIte,YACXse,IAAIte,MAAMnL,YAAY8vC,QAGxBrmB,IAAIqmB,WAAaA,OAEdzkE,KAAKgkE,UAAYhkE,KAAKo+C,IAAIqmB,aAE9BzkE,KAAKo+C,IAAIqmB,WAAWl9B,iBACjB6W,IAAIqmB,WAAWl9B,WAAWC,YAAYxnC,KAAKo+C,IAAIqmB,iBAEjDrmB,IAAIqmB,WAAa,KAElBzkE,KAAK2kE,wBACFA,iBAAiB5f,eACjB4f,iBAAmB,oDAUT5lD,OACbuxC,GAAatwD,KAAKmvB,QAAQmhC,SAASyU,eAAkC,MAAjB/kE,KAAKswD,WAAqBtwD,KAAKmvB,QAAQmhC,SAASra,SACzFj2C,KAAKmvB,QAAQmhC,SAASyU,eAAkC,MAAjB/kE,KAAKswD,UAAoBtwD,KAAKswD,SAASra,UAE3Fj2C,KAAKgkE,UAAY1T,IAAatwD,KAAKo+C,IAAI4mB,aAAc,KAEjDrmB,EAAK3+C,KAELglE,EAAe53C,SAASE,cAAc,OAExCttB,KAAKmvB,QAAQguB,IACf6nB,EAAa18B,UAAY,iBAEzB08B,EAAa18B,UAAY,iBAEvB28B,EAAgBjlE,KAAKmvB,QAAQvc,QAAQ5S,KAAKmvB,QAAQzxB,QACjDunE,IACEjlE,KAAKmwD,SACRxtD,QAAQH,yCAAkCxC,KAAKmvB,QAAQzxB,mGAClDyyD,QAAS,GAEhB8U,EAAgBjlE,KAAKmvB,QAAQvc,QAAb,IAElBoyD,EAAavV,MAAQwV,EAAcjX,oBAG9BkX,mBAAqB,IAAIlhB,GAAOghB,GAAc1uB,GAAG,OAAO,SAAAkK,GAC3DA,EAAM8D,kBACN3F,EAAG/tB,OAAOu0C,kBAAkBxmB,MAG9B5/B,EAAO4V,YAAYqwC,QACd5mB,IAAI4mB,aAAeA,OAEhBhlE,KAAKgkE,UAAYhkE,KAAKo+C,IAAI4mB,eAE9BhlE,KAAKo+C,IAAI4mB,aAAaz9B,iBACnB6W,IAAI4mB,aAAaz9B,WAAWC,YAAYxnC,KAAKo+C,IAAI4mB,mBAEnD5mB,IAAI4mB,aAAe,KAEpBhlE,KAAKklE,0BACFA,mBAAmBngB,eACnBmgB,mBAAqB,+DAUAnmD,MACzB/e,KAAKmvB,QAAQi2C,6BAEZ9U,GAAYtwD,KAAKmvB,QAAQmhC,SAASkU,aACD,IAAvBxkE,KAAKuT,KAAK+8C,YACY,IAAvBtwD,KAAKuT,KAAK+8C,YAErBtwD,KAAKgkE,UAAY1T,IAAatwD,KAAKo+C,IAAIinB,wBAAyB,KAC5DA,EAA0Bj4C,SAASE,cAAc,OAEvD+3C,EAAwB/8B,UAAY,2BACpCvpB,EAAO4V,YAAY0wC,QACdjnB,IAAIinB,wBAA0BA,OAEzBrlE,KAAKgkE,UAAYhkE,KAAKo+C,IAAIinB,0BAEhCrlE,KAAKo+C,IAAIinB,wBAAwB99B,iBAC9B6W,IAAIinB,wBAAwB99B,WAAWC,YAAYxnC,KAAKo+C,IAAIinB,8BAE9DjnB,IAAIinB,wBAA0B,SAIjCrlE,KAAKo+C,IAAIinB,wBAAyB,MAG/BjnB,IAAIinB,wBAAwB7wC,MAAMyqB,WAAaj/C,KAAK4wB,OAAOuhC,QAAQmT,YAAYC,eAAiB,UAAY,cAG5GnnB,IAAIinB,wBAAwB7wC,MAAMkiB,UAAY,wBAC9C0H,IAAIinB,wBAAwB7wC,MAAMkgB,KAAO,UAyB1C7gB,IArBE+9B,EAAY5xD,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASoY,cAKf,OAAjC5xD,KAAKmvB,QAAQk5B,YAAYtQ,KACL/3C,KAAK8iD,IAEJ9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,QAEnB7oC,KAAK4wB,OAAOkyB,IAXjC,IAWwD8O,QAGvExT,IAAIinB,wBAAwB7wC,MAAMq0B,OAAS,QAC3CzK,IAAIinB,wBAAwB7wC,MAAMsuB,cAAS9iD,KAAK6oC,OAAS,eAEzDuV,IAAIinB,wBAAwB7wC,MAAMsuB,IAAM,QACxC1E,IAAIinB,wBAAwB7wC,MAAMq0B,iBAAY7oD,KAAK6oC,OAAS,SAO/D7oC,KAAKmvB,QAAQi2C,yBAA2BplE,KAAKmvB,QAAQi2C,wBAAwBI,SAE/E3xC,EADmBkiB,UAAK5mB,QAAQi2C,wBAAwBI,iBAAcxlE,KAC5Dg+D,CAAiBh+D,KAAKuT,WAEhCsgB,mBAAoBzJ,GAAOpqB,KAAKuT,KAAKitB,OAAO/iC,OAAO,qBAC/CuC,KAAKuT,KAAKktB,MACZ5M,uBAAwBzJ,GAAOpqB,KAAKuT,KAAKktB,KAAKhjC,OAAO,2BAGpD2gD,IAAIinB,wBAAwB3Z,UAAY73B,kDAUxC7zB,KAAK4wB,OAAOuhC,QAAQsT,UAAU17D,IAAI/J,KAAKozB,4CAQhCwU,OACV/T,EAEAmqC,EACA0H,EACAC,MACE3R,EAAWh0D,KAAK4lE,eAGhBC,GADe7lE,KAAKo+C,IAAIymB,KAAO7kE,KAAKo+C,IAAIte,OACMgmC,uBAAuB,0BAA0B,GAEjG9lE,KAAKmvB,QAAQ42C,qBAEfL,GADAC,EAA+B5vB,UAAK5mB,QAAQ42C,6BAA0B/lE,OACfg0D,EAAU6R,GAEjEH,EAA0B,MAGxBG,KACGH,aAAmCxpE,UAAawpE,aAAmCzH,SACtF0H,EAA6B3R,EAAU6R,WAE5B7lE,KAAKgmE,iBAAiBhmE,KAAK0lE,2BAA6B1lE,KAAKgmE,iBAAiBN,GAC3E,IAERA,aAAmCzH,QACrC4H,EAA+Bna,UAAY,GAC3Cma,EAA+BlxC,YAAY+wC,QAExC,GAA+BjmE,MAA3BimE,EACPG,EAA+Bna,UAAYga,UAGnB,cAAlB1lE,KAAKuT,KAAK2jB,WAA8Cz3B,IAAtBO,KAAKuT,KAAKsgB,cAC1C,IAAIzwB,mDAA4CpD,KAAKozB,UAI1DsyC,wBAA0BA,EAKjC1lE,KAAKmvB,QAAQq2C,SAEf3xC,GADAmqC,EAAmBjoB,UAAK5mB,QAAQq2C,iBAAcxlE,OACnBg0D,EAAUpsB,EAAS5nC,KAAKuT,MAEnDsgB,EAAU7zB,KAAKuT,KAAKsgB,WAGjBA,aAAmB33B,UAAa23B,aAAmBoqC,SACtDD,EAAiBhK,EAAUpsB,WAEjB5nC,KAAKgmE,iBAAiBhmE,KAAK6zB,WAAa7zB,KAAKgmE,iBAAiBnyC,GAC3D,IAEPA,aAAmBoqC,QACrBr2B,EAAQ8jB,UAAY,GACpB9jB,EAAQjT,YAAYd,QAEjB,GAAep0B,MAAXo0B,EACP+T,EAAQ8jB,UAAY73B,UAGI,cAAlB7zB,KAAKuT,KAAK2jB,WAA8Cz3B,IAAtBO,KAAKuT,KAAKsgB,cAC1C,IAAIzwB,mDAA4CpD,KAAKozB,UAG1DS,QAAUA,iDAUC+T,MACjB5nC,KAAKmvB,QAAQ82C,gBAAkBjmE,KAAKmvB,QAAQ82C,eAAejpE,OAAS,EAAG,KACrEkpE,EAAa,MAEbhjC,GAAcljC,KAAKmvB,QAAQ82C,gBAC7BC,EAAalmE,KAAKmvB,QAAQ82C,mBAEvB,CAAA,GAAmC,OAA/BjmE,KAAKmvB,QAAQ82C,sBACpBC,EAAalmB,GAAYhgD,KAAKuT,gDAMb2yD,+BAAY,KAApBziE,UACH3B,EAAQ9B,KAAKuT,KAAK9P,GAEX,MAAT3B,EACF8lC,EAAQ4oB,4BAAqB/sD,GAAQ3B,GAGrC8lC,EAAQu+B,+BAAwB1iE,8HAW1BmkC,GAEP5nC,KAAKw0B,QACP8jB,GAAK8lB,cAAcx2B,EAAS5nC,KAAKw0B,YAC5BA,MAAQ,MAIXx0B,KAAKuT,KAAKihB,QACZ8jB,GAAK+lB,WAAWz2B,EAAS5nC,KAAKuT,KAAKihB,YAC9BA,MAAQx0B,KAAKuT,KAAKihB,gDAUVX,SACQ,iBAAZA,EAA6BA,EACpCA,GAAW,cAAeA,EAAgBA,EAAQuyC,UAC/CvyC,8CAOH7zB,KAAKmvB,UAC6B,kBAA1BnvB,KAAKmvB,QAAQmhC,cAChBA,SAAW,CACdkU,WAAYxkE,KAAKmvB,QAAQmhC,SACzB+V,YAAarmE,KAAKmvB,QAAQmhC,SAC1Bra,OAAQj2C,KAAKmvB,QAAQmhC,UAEkB,WAAjC7qB,GAAOzlC,KAAKmvB,QAAQmhC,iBACrBA,SAAW,GAChBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,UAAWz+C,KAAKswD,SAAUtwD,KAAKmvB,QAAQmhC,YAI3FtwD,KAAKmvB,SAAanvB,KAAKmvB,QAAQmhC,WAAsD,IAAxCtwD,KAAKmvB,QAAQmhC,SAASyU,eAClE/kE,KAAKuT,OAC2B,kBAAvBvT,KAAKuT,KAAK+8C,cACdA,SAAW,CACdkU,WAAYxkE,KAAKuT,KAAK+8C,SACtB+V,YAAarmE,KAAKuT,KAAK+8C,SACvBra,OAAQj2C,KAAKuT,KAAK+8C,UAEmB,WAA9B7qB,GAAOzlC,KAAKuT,KAAK+8C,iBAGrBA,SAAW,GAChBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,UAAWz+C,KAAKswD,SAAUtwD,KAAKuT,KAAK+8C,0DAWtF,iDAQA,kDAQHtwD,KAAKmvB,QAAQm3C,SAAWtmE,KAAKmvB,QAAQm3C,QAAQd,SACtBzvB,UAAK5mB,QAAQm3C,QAAQd,iBAAcxlE,KACrDg+D,CAAiBh+D,KAAK4lE,eAAgB5lE,KAAKuT,MAG7CvT,KAAKuT,KAAKk8C,eAIrBsU,GAAK5nE,UAAUkH,OAAQ,MCrkBjBkjE,0BASQhzD,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXmxB,IAAK,CACH59B,MAAO,EACPC,OAAQ,GAEV2hB,KAAM,CACJ5hB,MAAO,EACPC,OAAQ,IAIRt1B,GACgB9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,kEAUhDmmC,MACJ15C,KAAK8/D,eACA,MAGLnD,EACE8J,EAAQzmE,KAAKuT,KAAKkzD,OAASzmE,KAAKmvB,QAAQs3C,MACxCC,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,iCAGnChK,EADW,SAAT8J,EACWzmE,KAAKuT,KAAKitB,MAAMlhC,UAAYo6C,EAAMlZ,OAAWxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMjZ,IAExF,QAATgmC,EACMzmE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMlZ,OAAWxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYo6C,EAAMjZ,IAI3FzgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAY,EAAIhtB,EAAMlZ,OAAYxgC,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAU,EAAIhtB,EAAMjZ,IAEtHk8B,8CAQF38D,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAGjC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIvqB,cAG7BuqB,IAAIoM,KAAOp9B,SAASE,cAAc,YAClC8wB,IAAIoM,KAAKliB,UAAY,gBAGrB8V,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIooB,IAAIl+B,UAAY,eAGpB8V,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,EAAY,MAAM,IAAI5kD,MAAM,kEACjC4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,SAE7B7kE,KAAKo+C,IAAIoM,KAAKjjB,WAAY,KACzBqhB,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,eAC5BA,EAAY,MAAM,IAAIxlD,MAAM,kEACjCwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIoM,UAE7BxqD,KAAKo+C,IAAIooB,IAAIj/B,WAAY,KACtB+gB,EAAOtoD,KAAK4wB,OAAOwtB,IAAIkK,SACxBM,EAAY,MAAM,IAAIxlD,MAAM,4DACjCklD,EAAK3zB,YAAY30B,KAAKo+C,IAAIooB,UAEvB7H,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAErBvU,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAGtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAClEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC3BlS,IAAIymB,IAAIv8B,oCAA+BA,QACvC8V,IAAIoM,KAAKliB,qCAAgCA,QACzC8V,IAAIooB,IAAIl+B,oCAA+BA,2DAUvC,CACLyE,SAAU,CACR4H,MAAO30C,KAAKo+C,IAAIymB,IAAIrwC,MAAMmgB,MAC1BD,KAAM10C,KAAKo+C,IAAIymB,IAAIrwC,MAAMkgB,MAE3B8xB,IAAK,CACH39B,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,aACrB9tB,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,aAEtBoB,KAAM,CACJ5hB,MAAO5oC,KAAKo+C,IAAIoM,KAAKpB,aAEvByb,IAAK,CACHj8B,MAAO5oC,KAAKo+C,IAAIymB,IAAIzb,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,iEAUDqQ,GACpB/mE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ,WAEtByJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO,WAIvBW,MAAMmxB,IAAI39B,OAASk+B,EAAMP,IAAI39B,YAC7BwM,MAAMmxB,IAAI59B,MAAQm+B,EAAMP,IAAI59B,WAC5ByM,MAAMmV,KAAK5hB,MAAQm+B,EAAMvc,KAAK5hB,WAC9BA,MAAQm+B,EAAMlC,IAAIj8B,WAClBC,OAASk+B,EAAMlC,IAAIh8B,OAGpB7oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQoyB,EAAMh6B,SAAS4H,WAErCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAOqyB,EAAMh6B,SAAS2H,UAGtCgqB,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIymB,UACzCoC,0BACAC,qBAAqBlnE,KAAKo+C,IAAIymB,oCAQ9B/D,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,WACM21D,EAAK+I,QACPqI,EAAQpR,EAAK2R,2BAIjB,iBACM3R,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,gCAUNuoD,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,qCAQjB9gE,KAAK2+D,UAAW,KACZvgB,EAAMp+C,KAAKo+C,IAEbA,EAAIymB,IAAI5uB,OAAQmI,EAAIymB,IAAI5uB,SACnBmI,EAAIymB,IAAIt9B,YAAY6W,EAAIymB,IAAIt9B,WAAWC,YAAY4W,EAAIymB,KAE5DzmB,EAAIoM,KAAKvU,OAAQmI,EAAIoM,KAAKvU,SACrBmI,EAAIoM,KAAKjjB,YAAY6W,EAAIoM,KAAKjjB,WAAWC,YAAY4W,EAAIoM,MAE9DpM,EAAIooB,IAAIvwB,OAAQmI,EAAIooB,IAAIvwB,SACnBmI,EAAIooB,IAAIj/B,YAAY6W,EAAIooB,IAAIj/B,WAAWC,YAAY4W,EAAIooB,UAE3D7H,WAAY,8CAQbxhB,EAAMn9C,KAAKmvB,QAAQguB,IAEnBsqB,EAAe,SAAC7/B,EAASz9B,EAAGZ,SAAG4zC,kEACzB19C,IAAN0K,QAAyB1K,IAAN8J,OAEjBsiD,EAAa1O,GAAY,EAALhzC,EAAUA,EAcpCy9B,EAAQpT,MAAMkiB,eAXJj3C,IAAN8J,OAMM9J,IAAN0K,2BAKmC0hD,kBAAiBtiD,8BAJdA,8BANAsiD,WAY5C4b,EAAaznE,KAAKo+C,IAAIymB,IAAK7kE,KAAK0nE,KAAM1nE,KAAK2nE,KAAMxqB,GACjDsqB,EAAaznE,KAAKo+C,IAAIooB,IAAKxmE,KAAK4nE,KAAM5nE,KAAK6nE,KAAM1qB,GACjDsqB,EAAaznE,KAAKo+C,IAAIoM,KAAMxqD,KAAK8nE,MAAO9nE,KAAK+nE,MAAO5qB,6CAQ9C3c,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CimC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,MACvEuB,EAAYhoE,KAAKq1C,MAAMmV,KAAK5hB,MAC5Bq/B,EAAWjoE,KAAKq1C,MAAMmxB,IAAI59B,MAEnB,SAAT69B,QAEGiB,KAAOlnC,EAAQxgC,KAAK4oC,WACpBk/B,MAAQtnC,EAAQwnC,OAChBJ,KAAOpnC,EAAQwnC,EAAY,EAAIC,EAAW,GAE/B,QAATxB,QAEFiB,KAAOlnC,OACPsnC,MAAQtnC,OACRonC,KAAOpnC,EAAQwnC,EAAY,EAAIC,EAAW,SAI1CP,KAAOlnC,EAAQxgC,KAAK4oC,MAAQ,OAC5Bk/B,MAAQ9nE,KAAKmvB,QAAQguB,IAAM3c,EAAQwnC,EAAYxnC,EAAQwnC,EAAY,OACnEJ,KAAOpnC,EAAQynC,EAAW,GAG7BjoE,KAAKmvB,QAAQguB,IACfn9C,KAAK20C,MAAQ30C,KAAK0nE,KAElB1nE,KAAK00C,KAAO10C,KAAK0nE,UAEdD,yDAQCpf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvCmwB,EAAYloE,KAAKo+C,IAAIoM,KAAKh2B,SAEb,OAAf6zB,EAAsB,KAClB8f,EAAanoE,KAAK4wB,OAAOkyB,IAAM9iD,KAAK8iD,IAAM,OAE3C6kB,KAAO3nE,KAAK8iD,KAAO,EACxBolB,EAAUr/B,iBAAYs/B,QACtBD,EAAUrf,OAAS,GACnBqf,EAAUplB,IAAM,QAEb,KAEGqlB,EADgBnoE,KAAK4wB,OAAOuhC,QAAQ9c,MAAMxM,OACb7oC,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,SAE1E6kB,KAAO3nE,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,KAAO9iD,KAAK6oC,QAAU,GAC5Dq/B,EAAUr/B,iBAAYs/B,QACtBD,EAAUplB,IAAM,GAChBolB,EAAUrf,OAAS,SAGhBgf,MAAQ7nE,KAAKq1C,MAAMmxB,IAAI39B,OAAS,OAEhC4+B,6DAQEznE,KAAK4oC,MAAQ,iDAQb5oC,KAAK4oC,MAAQ,SAvYFm7B,ICAhBqE,0BASQ70D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXmxB,IAAK,CACH1jB,IAAK,EACLla,MAAO,EACPC,OAAQ,GAEVhV,QAAS,CACPgV,OAAQ,EACRw/B,WAAY,EACZC,YAAa,IAIb/0D,GACgB9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,kEAUhDmmC,MACJ15C,KAAK8/D,eACA,MAGH4G,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,iCAE7B3mE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,EAAYhtB,EAAMlZ,OAAYxgC,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,gDAQrFzgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIte,MAAQ1S,SAASE,cAAc,YAInC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIte,MAAMnL,YAAY30B,KAAKo+C,IAAIvqB,cAG/BuqB,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIte,MAAMnL,YAAY30B,KAAKo+C,IAAIooB,UAG/BpoB,IAAIte,MAAM,YAAc9/B,UAExB0+D,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIte,MAAMyH,WAAY,KACxBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAElB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIte,YAE7B6+B,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIte,YAC/BgnC,aAAa9mE,KAAKo+C,IAAIte,WAErBwwB,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAEtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAChEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC7BlS,IAAIte,MAAMwI,sCAAkCA,QAC5C8V,IAAIooB,IAAIl+B,oCAAgCA,2DAUxC,CACLk+B,IAAM,CACJ59B,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,cAEvB7iC,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,YACxBvgB,OAAQ7oC,KAAKo+C,IAAIvqB,QAAQ6iC,cAE3B52B,MAAO,CACL8I,MAAO5oC,KAAKo+C,IAAIte,MAAMspB,YACtBvgB,OAAQ7oC,KAAKo+C,IAAIte,MAAM42B,iEAUHqQ,QAEnB1xB,MAAMmxB,IAAI59B,MAAQm+B,EAAMP,IAAI59B,WAC5ByM,MAAMmxB,IAAI39B,OAASk+B,EAAMP,IAAI39B,YAC7BwM,MAAMxhB,QAAQgV,OAASk+B,EAAMlzC,QAAQgV,OAGtC7oC,KAAKmvB,QAAQguB,SACViB,IAAIvqB,QAAQW,MAAM8zC,sBAAiB,EAAItoE,KAAKq1C,MAAMmxB,IAAI59B,iBAEtDwV,IAAIvqB,QAAQW,MAAM6zC,qBAAgB,EAAIroE,KAAKq1C,MAAMmxB,IAAI59B,iBAKvDA,MAAQm+B,EAAMjnC,MAAM8I,WACpBC,OAASk+B,EAAMjnC,MAAM+I,YAGrBuV,IAAIooB,IAAIhyC,MAAMsuB,eAAU9iD,KAAK6oC,OAAS7oC,KAAKq1C,MAAMmxB,IAAI39B,QAAU,YAE9Do/B,EAAWjoE,KAAKq1C,MAAMmxB,IAAI59B,MAC1B2/B,EAAavoE,KAAKmvB,QAAQguB,IAAO8qB,EAAW,GAAM,EAAIA,EAAW,OAClE7pB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0B6xB,aACxC7J,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIte,YACzCmnC,0BACAC,qBAAqBlnE,KAAKo+C,IAAIte,sCAQ9BghC,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,WACM21D,EAAK+I,QACPqI,EAAQpR,EAAK2R,2BAIjB,iBACM3R,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,8CASH4kC,EAAMn9C,KAAKmvB,QAAQguB,IAEnBsqB,EAAe,SAAC7/B,EAASz9B,EAAGZ,SAAG4zC,kEACzB19C,IAAN0K,QAAyB1K,IAAN8J,OAEjBsiD,EAAa1O,GAAY,EAALhzC,EAAUA,EAcpCy9B,EAAQpT,MAAMkiB,eAXJj3C,IAAN8J,OAMM9J,IAAN0K,2BAKmC0hD,kBAAiBtiD,8BAJdA,8BANAsiD,WAY5C4b,EAAaznE,KAAKo+C,IAAIte,MAAO9/B,KAAKwoE,OAAQxoE,KAAKyoE,OAAQtrB,gCASpD2jB,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,kCAQjB9gE,KAAK2+D,YACH3+D,KAAKo+C,IAAIte,MAAMyH,iBACZ6W,IAAIte,MAAMyH,WAAWC,YAAYxnC,KAAKo+C,IAAIte,YAG5C6+B,WAAY,6CASbn+B,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,YAE5CgoC,OAAShoC,EACVxgC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,EAAQxgC,KAAKq1C,MAAMmxB,IAAI59B,WAE/B8L,KAAOlU,EAAQxgC,KAAKq1C,MAAMmxB,IAAI59B,WAGhC6+B,yDAQCpf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,UAEtC0wB,OADY,OAAfpgB,EACYroD,KAAK8iD,IAGL9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,YAGhD4+B,6DAQEznE,KAAKq1C,MAAMmxB,IAAI59B,qDAQf5oC,KAAKq1C,MAAMmxB,IAAI59B,aArUFm7B,ICAlB2E,0BASQn1D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,MAGNxqC,UAAW,EAEZmV,EAAM,IACU9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,EAAK6f,QAE3C3zB,MAAZ8T,EAAKktB,UACD,IAAIr9B,+CAAwCmQ,EAAK6f,mEAWnDsmB,UACJ15C,KAAK8/D,UAID9/D,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASzgC,KAAKuT,KAAKktB,IAAMiZ,EAAMlZ,mDAQ1DxgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAIjC8wB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,yBACtB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIuqB,YAG7BvqB,IAAIwqB,aAAex7C,SAASE,cAAc,YAC1C8wB,IAAIwqB,aAAatgC,UAAY,8BAC7B8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIwqB,mBAG7BxqB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIvqB,cAG/BuqB,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDAUV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAElB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,UAE7BlG,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAErBvU,EAAYtwD,KAAKswD,SAASkU,YAAcxkE,KAAKswD,SAAS+V,YAGtD/9B,GAAatoC,KAAKuT,KAAK+0B,UAAa,IAAMtoC,KAAKuT,KAAK+0B,UAAa,KAClEtoC,KAAKgkE,SAAW,gBAAkB,KAClC1T,EAAW,gBAAkB,sBAC7BlS,IAAIymB,IAAIv8B,UAAYtoC,KAAK6oE,cAAgBvgC,OAIzC8V,IAAIvqB,QAAQW,MAAMs0C,SAAW,qEAW/B1qE,SAAgE,WAArD4tB,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIuqB,OAAOvqE,cACnD4qE,WAAsE,WAAzDh9C,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIvqB,SAASm1C,WACrD,CACLn1C,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,aAE1Byb,IAAK,CACHh8B,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,iEAUDqQ,QACnB1xB,MAAMxhB,QAAQ+U,MAAQm+B,EAAMlzC,QAAQ+U,WACpCC,OAASk+B,EAAMlC,IAAIh8B,YACnBuV,IAAIvqB,QAAQW,MAAMs0C,SAAW,QAC7BpK,OAAQ,wDAQRsI,gCAAgChnE,KAAKo+C,IAAIymB,UACzCqC,qBAAqBlnE,KAAKo+C,IAAIymB,UAC9BoC,0BACAgC,wBACAC,mDAQApI,eACDiG,EA8BExuD,SA7BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC,iBACM21D,EAAK+I,QACPqI,EAAQhxB,KAAA4f,EAAK2R,+BAA4B3R,EAAjC5f,KAIZ,iBACM4f,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,gCAUNuoD,OACE9gE,KAAK2+D,iBACD3+D,KAAKqwD,OAAOyQ,qCAQjB9gE,KAAK2+D,UAAW,KACZkG,EAAM7kE,KAAKo+C,IAAIymB,IAEjBA,EAAIt9B,YACNs9B,EAAIt9B,WAAWC,YAAYq9B,QAGxBlG,WAAY,uCAaTwK,OAKNC,EACAC,EALEC,EAActpE,KAAK4wB,OAAOgY,MAC5BpI,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CC,EAAMzgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKktB,KACvCgmC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,OAMjD,IAAxBzmE,KAAKuT,KAAK41D,gBAAsC1pE,IAAd0pE,IAAyC,IAAdA,IAC3D3oC,GAAS8oC,IACX9oC,GAAS8oC,GAEP7oC,EAAM,EAAI6oC,IACZ7oC,EAAM,EAAI6oC,QAKRC,EAAW/nE,KAAK8D,IAAI9D,KAAK+a,MAAsB,KAAfkkB,EAAMD,IAAiB,IAAM,UAE/DxgC,KAAK5B,UACH4B,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAEToI,MAAQ2gC,EAAWvpE,KAAKq1C,MAAMxhB,QAAQ+U,MAC3CygC,EAAerpE,KAAKq1C,MAAMxhB,QAAQ+U,QAO9B5oC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAEToI,MAAQ2gC,EACbF,EAAe7nE,KAAKY,IAAIq+B,EAAMD,EAAOxgC,KAAKq1C,MAAMxhB,QAAQ+U,QAGtD5oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMkiB,gCAAwC,EAAd12C,KAAK20C,kBAE7CyJ,IAAIymB,IAAIrwC,MAAMkiB,+BAA0B12C,KAAK00C,iBAE/C0J,IAAIymB,IAAIrwC,MAAMoU,gBAAW2gC,QAC1BvpE,KAAKgpE,kBACAngC,OAAS7oC,KAAKo+C,IAAIymB,IAAInO,cAGvB+P,OACD,YACEroB,IAAIvqB,QAAQW,MAAMkiB,UAAY,0BAGhC,WACC12C,KAAKmvB,QAAQguB,IAAK,KACdorB,GAAuD,EAA1C/mE,KAAK8D,IAAKikE,EAAWF,EAAe,QAClDjrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0Bl1C,KAAK8D,IAAKikE,EAAWF,EAAe,oBAIpF,YACCrpE,KAAKmvB,QAAQguB,IAAK,KACdorB,GAA2D,EAA9C/mE,KAAK8D,KAAKikE,EAAWF,GAAgB,EAAG,QACtDjrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0Bl1C,KAAK8D,KAAKikE,EAAWF,GAAgB,EAAG,2BASvFD,EAFAppE,KAAK5B,SACHqiC,EAAM,EACej/B,KAAK8D,KAAKk7B,EAAO,IAGhB6oC,EAItB7oC,EAAQ,GACcA,EAGD,EAGvBxgC,KAAKmvB,QAAQguB,IAAK,KACdorB,GAAqC,EAAxBa,OACdhrB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B6xB,mBAE5CnqB,IAAIvqB,QAAQW,MAAMkiB,+BAA0B0yB,oDAWjD/gB,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvC8sB,EAAM7kE,KAAKo+C,IAAIymB,IAGnBA,EAAIrwC,MAAMsuB,cADO,OAAfuF,EACiBroD,KAAK8iD,IAGL9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,4DASrD7oC,KAAKgkE,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAU15C,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAI0mB,SAAU,KAE1GA,EAAW13C,SAASE,cAAc,OACxCw3C,EAASx8B,UAAY,gBACrBw8B,EAAS2E,aAAezpE,UAEnBo+C,IAAIymB,IAAIlwC,YAAYmwC,QACpB1mB,IAAI0mB,SAAWA,OAEZ9kE,KAAKgkE,UAAahkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAS15C,KAAKo+C,IAAI0mB,WAE1E9kE,KAAKo+C,IAAI0mB,SAASv9B,iBACf6W,IAAI0mB,SAASv9B,WAAWC,YAAYxnC,KAAKo+C,IAAI0mB,eAE/C1mB,IAAI0mB,SAAW,sDASjB9kE,KAAKgkE,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAU15C,KAAKswD,SAASkU,aAAexkE,KAAKo+C,IAAIsrB,UAAW,KAE3GA,EAAYt8C,SAASE,cAAc,OACzCo8C,EAAUphC,UAAY,iBACtBohC,EAAUC,cAAgB3pE,UAErBo+C,IAAIymB,IAAIlwC,YAAY+0C,QACpBtrB,IAAIsrB,UAAYA,OAEb1pE,KAAKgkE,UAAahkE,KAAKmvB,QAAQq6C,qBAAqB9vB,QAAS15C,KAAKo+C,IAAIsrB,YAE1E1pE,KAAKo+C,IAAIsrB,UAAUniC,iBAChB6W,IAAIsrB,UAAUniC,WAAWC,YAAYxnC,KAAKo+C,IAAIsrB,gBAEhDtrB,IAAIsrB,UAAY,aA5ZH3F,IAiaxB2E,GAAUvsE,UAAU0sE,cAAgB,yBC/Z9Be,0BAWQr2D,EAAMmoC,EAAYvsB,kDACtB5b,EAAMmoC,EAAYvsB,KACnBkmB,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,MAGNxqC,UAAW,EAGZmV,EAAM,IACU9T,MAAd8T,EAAKitB,YACD,IAAIp9B,iDAA0CmQ,EAAK6f,QAE3C3zB,MAAZ8T,EAAKktB,UACD,IAAIr9B,+CAAwCmQ,EAAK6f,mEAUnDsmB,UAEA15C,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASzgC,KAAKuT,KAAKktB,IAAMiZ,EAAMlZ,kDAQ1DxgC,KAAKo+C,WAEHA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAIjC8wB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,yBACtB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIuqB,YAG7BvqB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBACxB8V,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIvqB,cAM/B6qC,OAAQ,mDASV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAEbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBqhB,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,eAC9BA,QACG,IAAIxlD,MAAM,kEAElBwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIymB,UAE7BlG,WAAY,yDAYb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIvqB,cAC/BizC,aAAa9mE,KAAKo+C,IAAIymB,SAGrBv8B,GAAatoC,KAAKuT,KAAK+0B,UAAa,IAAMtoC,KAAKuT,KAAK+0B,UAAa,KAClEtoC,KAAKgkE,SAAW,gBAAkB,SAClC5lB,IAAIymB,IAAIv8B,UAAYtoC,KAAK6oE,cAAgBvgC,gEAW3ClqC,SAAkE,WAAvD4tB,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIvqB,SAASz1B,SACnD,CACLy1B,QAAS,CACP+U,MAAO5oC,KAAKo+C,IAAIvqB,QAAQu1B,gEAUJ2d,QAEnB1xB,MAAMxhB,QAAQ+U,MAAQm+B,EAAMlzC,QAAQ+U,WACpCC,OAAS,OAET61B,OAAQ,mFAeRoC,eACDiG,EA6BExuD,SA5BA85C,EAAQ,WAEP8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,MAE5B+1C,UAAKsxB,kCAA+BrnE,MAEpC,iBACM21D,EAAK+I,QACPqI,EAAQhxB,KAAA4f,EAAK2R,+BAA4B3R,EAAjC5f,KAIZ,iBACM4f,EAAK+I,YACP/I,EAAK4R,kCAA+B5R,GAAMoR,cAKzCS,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAAx1D,GACZ0b,EAAS1b,OAEJ0b,uCAQC+hD,OACNzxB,EACEwf,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,aAGlBt4C,IAAvBO,KAAKuT,KAAKwnD,SAAwB,KAE9B8O,EAAe7pE,KAAKuT,KAAKwnD,cAE1B3c,IAAIymB,IAAIrwC,MAAMqU,iBAAY7oC,KAAK4wB,OAAOuqC,UAAU0O,GAAchhC,kBAG5DuV,IAAIymB,IAAIrwC,MAAMsuB,cADF,OAAfuF,EAC0BroD,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOuqC,UAAU0O,GAAc/mB,IAEtD9iD,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK4wB,OAAOuqC,UAAU0O,GAAc/mB,IAAM9iD,KAAK4wB,OAAOuqC,UAAU0O,GAAchhC,kBAE9IuV,IAAIymB,IAAIrwC,MAAMq0B,OAAS,QAKxB7oD,KAAK4wB,kBAAkBkzC,IAEzBj7B,EAASrnC,KAAK8D,IAAItF,KAAK4wB,OAAOiY,OAC1B7oC,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASwF,OAAOnW,OACzC7oC,KAAK4wB,OAAOuhC,QAAQvZ,KAAKY,SAASC,gBAAgB5Q,aACjDuV,IAAIymB,IAAIrwC,MAAMq0B,OAAwB,UAAfR,EAA0B,IAAM,QACvDjK,IAAIymB,IAAIrwC,MAAMsuB,IAAqB,OAAfuF,EAAuB,IAAM,KAGtDxf,EAAS7oC,KAAK4wB,OAAOiY,YAEhBuV,IAAIymB,IAAIrwC,MAAMsuB,cAAS9iD,KAAK4wB,OAAOkyB,eACnC1E,IAAIymB,IAAIrwC,MAAMq0B,OAAS,SAG3BzK,IAAIymB,IAAIrwC,MAAMqU,iBAAYA,eArONk7B,IAyO7B6F,GAAeztE,UAAU0sE,cAAgB,0BAEzCe,GAAeztE,UAAUkH,OAAQ,EAMjCumE,GAAeztE,UAAUqnE,KAAOkF,GAAUvsE,UAAUqnE,KAMpDoG,GAAeztE,UAAU+zD,KAAOwY,GAAUvsE,UAAU+zD,KAMpD0Z,GAAeztE,UAAUskE,YAAciI,GAAUvsE,UAAUskE,qzCChQrDqJ,yBAKQ7d,EAAW8d,mBAChB9d,UAAYA,OACZ8d,eAAiBA,GAAkB,WAEnC5/D,EAAI,OACJZ,EAAI,OACJygE,QAAU,OACVzvB,QAAS,OAGTouB,MAAQv7C,SAASE,cAAc,YAC/Bq7C,MAAMrgC,UAAY,mBAClB2jB,UAAUt3B,YAAY30B,KAAK2oE,sDAOtBx+D,EAAGZ,QACRY,EAAIghD,GAAShhD,QACbZ,EAAI4hD,GAAS5hD,mCAOZsqB,GACFA,aAAmBoqC,cAChB0K,MAAMjd,UAAY,QAClBid,MAAMh0C,YAAYd,SAGlB80C,MAAMjd,UAAY73B,+BAQtBo2C,WACYxqE,IAAXwqE,IACFA,GAAS,IAGI,IAAXA,EAAiB,KACfphC,EAAS7oC,KAAK2oE,MAAM5c,aACpBnjB,EAAQ5oC,KAAK2oE,MAAMhoB,YACnB0V,EAAYr2D,KAAK2oE,MAAMphC,WAAWwkB,aAClC+c,EAAW9oE,KAAK2oE,MAAMphC,WAAWoZ,YAEjCjM,EAAO,EAAGoO,EAAM,KAEO,QAAvB9iD,KAAK+pE,gBAAmD,QAAvB/pE,KAAK+pE,eAA0B,KAC9DG,GAAS,EAAOC,GAAQ,EAED,QAAvBnqE,KAAK+pE,iBACH/pE,KAAKuJ,EAAIs/B,EAAS7oC,KAAKgqE,UACzBG,GAAQ,GAGNnqE,KAAKmK,EAAIy+B,EAAQkgC,EAAW9oE,KAAKgqE,UACnCE,GAAS,IAKXx1B,EADEw1B,EACKlqE,KAAKmK,EAAIy+B,EAET5oC,KAAKmK,EAIZ24C,EADEqnB,EACInqE,KAAKuJ,EAAIs/B,EAET7oC,KAAKuJ,OAGbu5C,EAAO9iD,KAAKuJ,EAAIs/B,GACNA,EAAS7oC,KAAKgqE,QAAU3T,IAChCvT,EAAMuT,EAAYxtB,EAAS7oC,KAAKgqE,SAE9BlnB,EAAM9iD,KAAKgqE,UACblnB,EAAM9iD,KAAKgqE,UAGbt1B,EAAO10C,KAAKmK,GACDy+B,EAAQ5oC,KAAKgqE,QAAUlB,IAChCp0B,EAAOo0B,EAAWlgC,EAAQ5oC,KAAKgqE,SAE7Bt1B,EAAO10C,KAAKgqE,UACdt1B,EAAO10C,KAAKgqE,cAIXrB,MAAMn0C,MAAMkgB,KAAOA,EAAO,UAC1Bi0B,MAAMn0C,MAAMsuB,IAAMA,EAAM,UACxB6lB,MAAMn0C,MAAMyqB,WAAa,eACzB1E,QAAS,YAGT2V,2CAQF3V,QAAS,OACTouB,MAAMn0C,MAAMkgB,KAAO,SACnBi0B,MAAMn0C,MAAMsuB,IAAM,SAClB6lB,MAAMn0C,MAAMyqB,WAAa,gDAOzB0pB,MAAMphC,WAAWC,YAAYxnC,KAAK2oE,gBCjIvCyB,GAAS5/C,GAAwC0N,MAIjDmJ,GAAgBC,GAAoB,SACpCC,GAAiBC,GAAwB,WAI3C,CAAE7R,OAAQ,QAASnP,OAAO,EAAM2P,QAASkR,KAAkBE,IAAkB,CAC7ErJ,MAAO,SAAeL,UACbuyC,GAAOpqE,KAAM63B,EAAY/7B,UAAUkB,OAAS,EAAIlB,UAAU,QAAK2D,MCV1E,OAAiBohC,GAAa,SAAS3I,MCDnCuJ,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGoM,aACNpM,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAevJ,MAASA,GAAQ6I,GCA5FspC,0BAUQ92D,EAAMmoC,EAAYvsB,wBACtBm7C,EAAkBC,GAAc,GAAI,CAACC,kBAAkB,GAAOr7C,EAAS,CAACmhC,UAAU,kCAClF/8C,EAAMmoC,EAAY4uB,KAEnBj1B,MAAQ,CACXxhB,QAAS,CACP+U,MAAO,EACPC,OAAQ,KAIPt1B,GAAwB9T,MAAhB8T,EAAKk3D,cACV,IAAIrnE,MAAM,sCAAwCmQ,EAAK6f,aAG1DA,GAAKs3C,OACLrG,MAAQ9wD,EAAK8wD,QACbsG,gBAEA9sB,QAAUH,EAAKnqC,KAAKq3D,eACpBlxB,MAAQgE,EAAKnqC,KAAKmmC,QAClBmxB,UAAW,IACXjL,WAAY,IACZrsD,KAAKqsD,WAAY,iEAQf5/D,KAAKuT,KAAKk3D,SAAWzqE,KAAKuT,KAAKk3D,QAAQztE,QAAUgD,KAAK6qE,4CAOpDr0B,QACJs0B,cAEAv3D,KAAKk3D,QAAUj0B,OAEfm0B,mBAEAI,2CAQGrxB,OACFsxB,EAAahrE,KAAKuT,KAAKktB,IAAMzgC,KAAKuT,KAAKktB,IAAMzgC,KAAKuT,KAAKitB,MAAQ,EAC/DkmC,EAAY1mE,KAAK4oC,MAAQ8Q,EAAMitB,0BAC/BlmC,EAAMj/B,KAAK8D,IAAI0lE,EAAYhrE,KAAKuT,KAAKitB,MAAMlhC,UAAYonE,UACrD1mE,KAAKuT,KAAKitB,MAAQkZ,EAAMjZ,KAASA,EAAMiZ,EAAMlZ,OAAUxgC,KAAKujE,mDAQ7D,CACL3D,WAAW,EACXxsC,GAAIpzB,KAAKozB,GACTojB,MAAOx2C,KAAKuT,KAAKijC,OAAS,GAC1BjjC,KAAMvT,KAAKuT,qCASPutD,mBACFiG,EA8BExuD,EA7BF85C,EAAQ,WAEL8U,0BAAuBnnE,gBAGvBonE,0BAAuBpnE,gBAGvBqnE,kCAA+BrnE,MAEpC+1C,KAAC,WACK/1C,KAAK0+D,QACPqI,EAAQ/mE,KAAKsnE,mCAETtnE,MAER+1C,KAAC,iBACK/1C,KAAK0+D,iBACF6I,kCAA+BvnE,MAAM+mE,YAEtC/mE,gBAGHwnE,+BAA4BxnE,cAG/B8gE,EACKzO,MAGPA,QAAAA,GAAc,SAAUx1D,GACtB0b,EAAS1b,OAEJ0b,kCAQJvY,KAAK2+D,gBACHtO,2CAQHrwD,KAAK2+D,UAAW,KACdvgB,EAAMp+C,KAAKo+C,IACXA,EAAIymB,IAAIt9B,YACV6W,EAAIymB,IAAIt9B,WAAWC,YAAY4W,EAAIymB,KAGjC7kE,KAAKmvB,QAAQ87C,aACX7sB,EAAIoM,KAAKjjB,YACX6W,EAAIoM,KAAKjjB,WAAWC,YAAY4W,EAAIoM,MAElCpM,EAAIooB,IAAIj/B,YACV6W,EAAIooB,IAAIj/B,WAAWC,YAAY4W,EAAIooB,WAGlC7H,WAAY,6CAQfn+B,EAAQxgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKitB,OAC3CC,EAAMzgC,KAAKuT,KAAKktB,IAAMzgC,KAAK07C,WAAWF,SAASx7C,KAAKuT,KAAKktB,KAAO,KAChEA,OACGyqC,sBAAsB1qC,EAAOC,OAC7B,KACDgmC,OAA4BhnE,IAApBO,KAAKuT,KAAKkzD,MAAsBzmE,KAAKmvB,QAAQs3C,MAAQzmE,KAAKuT,KAAKkzD,WACtE0E,yBAAyB3qC,EAAOimC,GAGnCzmE,KAAKmvB,QAAQ87C,kBACV7sB,IAAIoM,KAAKh2B,MAAMC,QAAUz0B,KAAKorE,kBAAoB,QAAU,YAC5DhtB,IAAIooB,IAAIhyC,MAAMC,QAAUz0B,KAAKorE,kBAAoB,QAAU,OAE5DprE,KAAKorE,wBACFC,gBAAgB7qC,EAAOC,4CAUlBD,EAAOC,QAChB2d,IAAIoM,KAAKh2B,MAAMC,QAAU,aACzB2pB,IAAIooB,IAAIhyC,MAAMC,QAAU,YACvB62C,EAAkBtrE,KAAKo+C,IAAIoM,KAAKpB,YAChCmiB,EAAiBvrE,KAAKo+C,IAAIooB,IAAIpd,eAEhC3oB,EAAK,KACD+qC,EAAaF,EAAkB9qC,GAASC,EAAMD,GAAS,EACvDirC,EAAYD,EAAaD,EAAiB,EAC1CG,EAAsB1rE,KAAKmvB,QAAQguB,KAAoB,EAAdquB,EAAkBA,EAC3DG,EAAqB3rE,KAAKmvB,QAAQguB,KAAmB,EAAbsuB,EAAiBA,OAE1DrtB,IAAIoM,KAAKh2B,MAAMkiB,+BAA0Bg1B,cACzCttB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0Bi1B,aACxC,KACCD,EAAsB1rE,KAAKmvB,QAAQguB,KAAgB,EAAT3c,EAAcA,EACxDmrC,EAAqB3rE,KAAKmvB,QAAQguB,KAAuC,GAA/B3c,EAAQ+qC,EAAiB,GAAY/qC,EAAQ+qC,EAAiB,OAEzGntB,IAAIoM,KAAKh2B,MAAMkiB,+BAA0Bg1B,cACzCttB,IAAIooB,IAAIhyC,MAAMkiB,+BAA0Bi1B,2DASxBnrC,EAAOimC,GAEjB,SAATA,EACEzmE,KAAKmvB,QAAQguB,UACVxI,MAAQnU,EAAQxgC,KAAK4oC,WAGrBwV,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,EAAQxgC,KAAK4oC,WAGpBwV,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,MAEtB,QAAT+xB,EACLzmE,KAAKmvB,QAAQguB,UACVxI,MAAQnU,OAGR4d,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,OAGP4d,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,MAIpC10C,KAAKmvB,QAAQguB,UACVxI,MAAQnU,EAAQxgC,KAAK4oC,MAAQ,OAG7BwV,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,YAEnCD,KAAOlU,EAAQxgC,KAAK4oC,MAAQ,OAG5BwV,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,oDAUtBlU,EAAOC,OACvB8oC,EAAW/nE,KAAK+a,MAAM/a,KAAK8D,IAAIm7B,EAAMD,EAAQ,GAAK,IAElDxgC,KAAKmvB,QAAQguB,SACVxI,MAAQnU,OAERkU,KAAOlU,OAGToI,MAAQpnC,KAAK8D,IAAIikE,EAAUvpE,KAAK4rE,UAAY,GAE7C5rE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ30C,KAAK20C,MAAQ,UAEnCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO10C,KAAK00C,KAAO,UAGnC0J,IAAIymB,IAAIrwC,MAAMoU,MAAQ2gC,EAAW,+CAOlClhB,EAAcroD,KAAKmvB,QAAQk5B,YAAYtQ,KACvC8sB,EAAM7kE,KAAKo+C,IAAIymB,OAEjBA,EAAIrwC,MAAMsuB,IADO,OAAfuF,GACeroD,KAAK8iD,KAAO,GAAK,MAGjB9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,IAAM9iD,KAAK6oC,QAAU,GAAK,KAGnE7oC,KAAKmvB,QAAQ87C,WAAY,IACR,OAAf5iB,OACGjK,IAAIoM,KAAKh2B,MAAMsuB,IAAS,SACxB1E,IAAIoM,KAAKh2B,MAAMqU,OAAU7oC,KAAK4wB,OAAOkyB,IAAM9iD,KAAK8iD,IAAM,EAAK,UAC3D1E,IAAIoM,KAAKh2B,MAAMq0B,OAAS,OAE1B,KAECgjB,EAAgB7rE,KAAK4wB,OAAOuhC,QAAQ9c,MAAMxM,OAC1Cs/B,EAAa0D,EAAgB7rE,KAAK4wB,OAAOkyB,IAAM9iD,KAAK4wB,OAAOiY,OAAS7oC,KAAK8iD,SACxE1E,IAAIoM,KAAKh2B,MAAMsuB,IAAU+oB,EAAgB1D,EAAc,UACvD/pB,IAAIoM,KAAKh2B,MAAMq0B,OAAS,SAG1BzK,IAAIooB,IAAIhyC,MAAMsuB,KAAQ9iD,KAAKo+C,IAAIooB,IAAI9P,aAAe,EAAK,oDASvD12D,KAAK4oC,MAAQ,iDAQb5oC,KAAK4oC,MAAQ,sCAOf63B,mBACAI,2FAOY7gE,KAAKuT,KAAKk3D,qCAAS,SAC7B3K,QAAU9/D,6FAGZuT,KAAKijC,MAAQO,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAKxkC,aAEhDs3D,UAAW,OACXnM,OAAQ,uCAQRoN,6DACA9rE,KAAKujE,sDAIOvjE,KAAKuT,KAAKk3D,qCAAS,KAA3B1yB,iBACAA,EAAK+nB,gGAGT+K,UAAW,EAEZiB,GAAoB9rE,KAAKqkE,aACtBA,MAAMpuB,OAAOj2C,WACbqkE,MAAQ,WAGV9wD,KAAKijC,MAAQ,QACbkoB,OAAQ,iDAOTqN,uDAOEC,EAAQj1B,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,SAAS,CAC3CvX,MAAOuX,EAAKxkC,KAAKitB,MAAMjjC,UACvBkjC,IAAKsX,EAAKxkC,KAAKktB,IAAMsX,EAAKxkC,KAAKktB,IAAIljC,UAAYw6C,EAAKxkC,KAAKitB,MAAMjjC,mBAG5DgW,KAAKnR,IAAMZ,KAAKY,UAALZ,QAAYu1C,GAAAi1B,QAAAA,GAAU,SAAAnjE,UAAKrH,KAAKY,IAAIyG,EAAE23B,MAAO33B,EAAE43B,KAAO53B,EAAE23B,iBACnEjtB,KAAKjO,IAAM9D,KAAK8D,UAAL9D,QAAYu1C,GAAAi1B,QAAAA,GAAU,SAAAnjE,UAAKrH,KAAK8D,IAAIuD,EAAE23B,MAAO33B,EAAE43B,KAAO53B,EAAE23B,gBAClEyrC,EAAUl1B,UAAKxjC,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAKiH,UAC7CktB,EAAMz1B,GAAAw1B,QAAAA,GAAe,SAACE,EAAKrqE,UAAUqqE,EAAMrqE,IAAO,GAAK9B,KAAKuT,KAAKk3D,QAAQztE,OAE3E04D,UAAKniD,KAAKk3D,iBAAa,SAAA1yB,UAAQA,EAAKxkC,KAAKktB,aAEtCltB,KAAKitB,MAAQ,IAAI9jC,KAAKsD,KAAKuT,KAAKnR,UAChCmR,KAAKktB,IAAM,IAAI/jC,KAAKsD,KAAKuT,KAAKjO,YAE9BiO,KAAKitB,MAAQ,IAAI9jC,KAAKwvE,QACtB34D,KAAKktB,IAAM,gEASdzgC,KAAKuT,KAAKk3D,SAAWzqE,KAAKuT,KAAKk3D,QAAQztE,OAClC85C,UAAKvjC,KAAKk3D,iBAAe,SAAA1yB,UAAQA,EAAK+nB,UAAYnK,KAGpD,mDAOF31D,KAAKo+C,IAAK,eAERA,IAAM,QAGNA,IAAIymB,IAAMz3C,SAASE,cAAc,YAGjC8wB,IAAIvqB,QAAUzG,SAASE,cAAc,YACrC8wB,IAAIvqB,QAAQyU,UAAY,wBAExB8V,IAAIymB,IAAIlwC,YAAY30B,KAAKo+C,IAAIvqB,SAE9B7zB,KAAKmvB,QAAQ87C,kBAEV7sB,IAAIoM,KAAOp9B,SAASE,cAAc,YAClC8wB,IAAIoM,KAAKliB,UAAY,wBACrB8V,IAAIoM,KAAKh2B,MAAMC,QAAU,YAGzB2pB,IAAIooB,IAAMp5C,SAASE,cAAc,YACjC8wB,IAAIooB,IAAIl+B,UAAY,uBACpB8V,IAAIooB,IAAIhyC,MAAMC,QAAU,QAG3Bz0B,KAAKmvB,QAAQq7C,sBACVpsB,IAAIymB,IAAIuH,WAAar2B,KAAAs0B,EAAYluE,UAAUkwE,uBAAoBrsE,WAIjEo+C,IAAIymB,IAAI,YAAc7kE,UAEtB0+D,OAAQ,mDAQV1+D,KAAK4wB,aACF,IAAIxtB,MAAM,8CAGbpD,KAAKo+C,IAAIymB,IAAIt9B,WAAY,KACtBygB,EAAahoD,KAAK4wB,OAAOwtB,IAAI4J,eAC9BA,QACG,IAAI5kD,MAAM,kEAGlB4kD,EAAWrzB,YAAY30B,KAAKo+C,IAAIymB,SAG5Bjc,EAAa5oD,KAAK4wB,OAAOwtB,IAAIwK,cAE/B5oD,KAAKmvB,QAAQ87C,WAAY,KACtBjrE,KAAKo+C,IAAIoM,KAAKjjB,WAAY,KACxBqhB,EAAY,MAAM,IAAIxlD,MAAM,kEACjCwlD,EAAWj0B,YAAY30B,KAAKo+C,IAAIoM,UAG7BxqD,KAAKo+C,IAAIooB,IAAIj/B,WAAY,KACxB+gB,EAAOtoD,KAAK4wB,OAAOwtB,IAAIkK,SACtBM,EAAY,MAAM,IAAIxlD,MAAM,4DACjCklD,EAAK3zB,YAAY30B,KAAKo+C,IAAIooB,WAIzB7H,WAAY,yDAWb3+D,KAAK0+D,MAAO,MACTkI,gBAAgB5mE,KAAKo+C,IAAIvqB,cACzBgzC,sBAAsB7mE,KAAKo+C,IAAIymB,UAC/BiC,aAAa9mE,KAAKo+C,IAAIymB,SAGrBv8B,EAAYtoC,KAAK6oE,cAAgB,KAAO7oE,KAAKuT,KAAK+0B,UAAY,IAAMtoC,KAAKuT,KAAK+0B,UAAY,KAC7FtoC,KAAKgkE,SAAW,gBAAkB,IAAM,qBACtC5lB,IAAIymB,IAAIv8B,UAAY,YAAcA,EAEnCtoC,KAAKmvB,QAAQ87C,kBACV7sB,IAAIoM,KAAKliB,UAAY,8BAAiCtoC,KAAKgkE,SAAW,gBAAkB,SACxF5lB,IAAIooB,IAAIl+B,UAAa,6BAAgCtoC,KAAKgkE,SAAW,gBAAkB,KAG1FhkE,KAAKuT,KAAKktB,WAGP2d,IAAIvqB,QAAQW,MAAMs0C,SAAW,8DAUhC/B,EAAQ,CACZh6B,SAAU,CACR4H,MAAO30C,KAAKo+C,IAAIymB,IAAIrwC,MAAMmgB,MAC1BD,KAAM10C,KAAKo+C,IAAIymB,IAAIrwC,MAAMkgB,MAE3BmwB,IAAK,CACHj8B,MAAO5oC,KAAKo+C,IAAIymB,IAAIzb,YACpBvgB,OAAQ7oC,KAAKo+C,IAAIymB,IAAInO,sBAIrB12D,KAAKmvB,QAAQ87C,aACflE,EAAMP,IAAM,CACV39B,OAAQ7oC,KAAKo+C,IAAIooB,IAAI9P,aACrB9tB,MAAO5oC,KAAKo+C,IAAIooB,IAAIpd,aAEtB2d,EAAMvc,KAAO,CACX5hB,MAAO5oC,KAAKo+C,IAAIoM,KAAKpB,cAIlB2d,oDAOiBA,GACpB/mE,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQ,WAEtByJ,IAAIymB,IAAIrwC,MAAMkgB,KAAO,MAIvB10C,KAAKuT,KAAKktB,SAGRmrC,SAAW7E,EAAMlC,IAAIj8B,WAFrBA,MAAQm+B,EAAMlC,IAAIj8B,WAKpBC,OAASk+B,EAAMlC,IAAIh8B,OAGpB7oC,KAAKmvB,QAAQguB,SACViB,IAAIymB,IAAIrwC,MAAMmgB,MAAQoyB,EAAMh6B,SAAS4H,WAErCyJ,IAAIymB,IAAIrwC,MAAMkgB,KAAOqyB,EAAMh6B,SAAS2H,UAGtCgqB,OAAQ,wDAORsI,gCAAgChnE,KAAKo+C,IAAIymB,sDASvC7kE,KAAK4rE,UAAY5rE,KAAK4oC,QAAU5oC,KAAKuT,KAAKktB,+CAS3CjkB,EAAS,KAAMxc,KAAKuT,KAAKjO,IAAMtF,KAAKuT,KAAKnR,KAAO,QAC7C,CACLkqE,SAAUtsE,KAAKuT,KAAKnR,IAAMoa,EAC1B+vD,OAAQvsE,KAAKuT,KAAKjO,IAAMkX,qCASxBxc,KAAK69C,QAAS,OACW79C,KAAKwsE,eAAzBF,IAAAA,SAAUC,IAAAA,OAEXE,EAAU,CACdjsC,MAAO,IAAI9jC,KAAK4vE,GAChB7rC,IAAK,IAAI/jC,KAAK6vE,GACdxtB,WAAW,QAGRlB,QAAQ4C,KAAK,MAAOgsB,kDAUpBzsE,KAAKuT,YA5nBUwwD,IAgoB1BsG,GAAYluE,UAAU0sE,cAAgB,iCCpoBtC,IAGa5M,GAHK,gBAWGyQ,yBAKLva,mBACHA,QAAUA,OACVwa,OAAS,QACT3rC,MAAQ,QACRA,OAAO,GAAK,wDAUHgzB,EAAUtY,EAAYvsB,UACpB,IAAIk7C,GAAYrW,EAAUtY,EAAYvsB,oCAcjDqnB,EAAOrnB,QACPqnB,MAAQA,GAAS,QACjBo2B,aAAc,OACdC,qBAAsB,EAEvB19C,GAAWA,EAAQ09C,2BACdA,oBAAsB19C,EAAQ09C,+DASlCD,aAAc,OACdC,qBAAsB,sCAUnBC,EAAanxB,EAAOxsB,SAC2B,kBAAZA,EAAwB,GAAKA,EAAlE49C,IAAAA,SAAUC,IAAAA,gBAEXA,IACDA,EAAkB,kBAAM,IAG5BD,EAAWA,GAAY,MAEnBE,GAAS,EAETC,EAAa,KAEbvxB,EAAQ,EAAG,IACPA,GAAS,QACF,GAGXsxB,EAAQzrE,KAAKc,IAAId,KAAK+a,MAAM/a,KAAKw4C,IAAI,IAAM2B,GAASn6C,KAAKw4C,IAR3C,KASdkzB,EAAa1rE,KAAKc,IAAId,KAAK6D,IATb,EAS8B4nE,OAI5CjtE,KAAK4sE,YAAa,KACZO,EAAgBF,GAASjtE,KAAKotE,aACfptE,KAAK6sE,qBAAsBM,UAEvCE,wBACAC,oBAIRF,WAAaH,MACdM,EAAWvtE,KAAKghC,MAAMisC,OACrBM,EAAU,KAEN,IAAIC,KADTD,EAAW,GACWvtE,KAAK2sE,UACnB3sE,KAAK2sE,OAAOtvE,eAAemwE,WACrBh3B,EAAQx2C,KAAK2sE,OAAOa,GACpBC,EAAOj3B,EAAMx5C,OACfF,EAAI,EACDA,EAAI2wE,GAAM,SAET11B,EAAOvB,EAAM15C,GACb4wE,EAAY,EAGZ35D,EAAIjX,EAAI,EACLiX,GAAK,GAAMgkC,EAAKiH,OAASxI,EAAMziC,GAAGirC,OAAUkuB,EAAa,IACvD12B,EAAMziC,GAAG+rD,SAAWkN,EAAgBj1B,EAAKxkC,KAAMijC,EAAMziC,GAAGR,OACzDm6D,IAEJ35D,YAIA4G,EAAI7d,EAAI,EACL6d,EAAI67B,EAAMx5C,QAAWw5C,EAAM77B,GAAGqkC,OAASjH,EAAKiH,OAAUkuB,EAAa,GAClEF,EAAgBj1B,EAAKxkC,KAAMijC,EAAM77B,GAAGpH,OACpCm6D,IAEJ/yD,YAIA/D,EAAI22D,EAASvwE,OAAS,EACnB4Z,GAAK,GAAMmhC,EAAKiH,OAASuuB,EAAS32D,GAAGooC,OAAUkuB,GAC9Cn1B,EAAKssB,OAASkJ,EAAS32D,GAAGytD,OAAS2I,EAAgBj1B,EAAKxkC,KAAMg6D,EAAS32D,GAAGrD,OAC1Em6D,IAEJ92D,OAIA82D,EAAYX,EAAU,SAEhBthD,EAAMiiD,EAAYX,EAAW,EAC7BY,EAAe,GAIjB5vE,EAAIjB,EACD6wE,EAAa3wE,OAASyuB,GAAO1tB,EAAIy4C,EAAMx5C,QACtCgwE,EAAgBx2B,EAAMz4C,GAAGwV,KAAMijC,EAAMz4C,GAAGwV,OACxCo6D,EAAa1wE,KAAKu5C,EAAMz4C,IAE5BA,QAGEo+D,EAAUn8D,KAAKmyD,QAAQyb,WAAW71B,EAAKxkC,MACvC8wD,EAAQrkE,KAAKmyD,QAAQwa,OAAOxQ,IAAYn8D,KAAKmyD,QAAQwa,OAAO1Q,IAC9D6D,EAAU9/D,KAAK6tE,oBAAoBF,EAActJ,EAAOyI,EAAa39C,GACzEo+C,EAAStwE,KAAK6iE,GAEdhjE,GAAK2uB,cAEEssB,EAAK+nB,QACZhjE,GAAK,OAMhBkkC,MAAMisC,GAASM,SAGjBA,4CASDZ,EAAS,QACVA,OAASA,gBAGKmB,GAAc9tE,KAAKw2C,sBAAQ,KAAnCuB,OAEDy1B,EAAYz1B,EAAKnnB,OAASmnB,EAAKnnB,OAAOurC,QAAU,GAClDkI,EAAQsI,EAAOa,GACdnJ,IACDA,EAAQ,GACRsI,EAAOa,GAAanJ,GAExBA,EAAMpnE,KAAK86C,GAGPA,EAAKxkC,KAAKitB,QACNuX,EAAKxkC,KAAKktB,IAEVsX,EAAKiH,QAAUjH,EAAKxkC,KAAKitB,MAAMjjC,UAAYw6C,EAAKxkC,KAAKktB,IAAIljC,WAAa,EAGtEw6C,EAAKiH,OAASjH,EAAKxkC,KAAKitB,MAAMjjC,eAMrC,IAAIwwE,KAAoBpB,EAAQ,UAC7BA,EAAOtvE,eAAe0wE,QACtBpB,EAAOoB,YAAuB,SAAC5wE,EAAGC,UAAMD,EAAE6hD,OAAS5hD,EAAE4hD,eAIxD4tB,aAAc,8CAYHe,EAActJ,EAAOyI,EAAa39C,SAK9C2wC,EAJEkO,EAAoBj3B,KAAC+1B,GAAe,YAAQ,SAAAhN,eAAY,CAC1DA,QAAAA,EACAmO,SAAU,OAAQl3B,KAAA+oB,EAAQvsD,KAAKk3D,iBAAY,SAAA1yB,UAAQA,EAAK3kB,cAGxD46C,EAAkBhxE,OAAQ,uDACjBkxE,aACDA,EAAeD,SAAStuC,OAASguC,EAAa3wE,QAC3CmxE,GAAAR,QAAAA,GAAmB,SAAAS,UAAeF,EAAeD,SAASngD,IAAIsgD,EAAYh7C,cAC7E0sC,EAAUoO,EAAepO,sBAHNkO,+BAAmB,kBAItC,6FAKRlO,SACAA,EAAQuO,WAAWV,GACf7N,EAAQuE,QAAUA,IACdvE,EAAQuE,OACRvE,EAAQuE,MAAMpuB,OAAO6pB,GAGrBuE,IACAA,EAAMxpD,IAAIilD,GACVA,EAAQuE,MAAQA,IAGjBvE,MAGPwO,EAAgBn/C,EAAQm/C,eAAiB,GACvC5yB,EAAa,CACfF,SAAUx7C,KAAKmyD,QAAQvZ,KAAKN,KAAKkD,SACjCS,OAAQj8C,KAAKmyD,QAAQvZ,KAAKN,KAAK2D,QAG7BwT,EAAQ6e,EAAc/nE,QAAQ,UAAWonE,EAAa3wE,QACtDuxE,EAAiB,eAAiB9e,EAAQ,KAAOke,EAAa3wE,OAAS,SACvEwxE,EAAiBjE,GAAc,GAAIp7C,EAASnvB,KAAKmyD,QAAQhjC,SACzD5b,EAAO,SACEg7D,QACF9e,QACA4U,UACEsJ,eACK3tE,KAAKmyD,QAAQvZ,KAAKiF,cACzB79C,KAAKmyD,QAAQvZ,KAAKc,cAE/BomB,EAAU9/D,KAAKyuE,kBAAkBl7D,EACjCmoC,EACA8yB,GAEInK,IACAA,EAAMxpD,IAAIilD,GACVA,EAAQuE,MAAQA,GAGpBvE,EAAQiL,SAEDjL,kDAQF9+B,MAAQ,QACRosC,YAAc,OACdpsC,MAAMhhC,KAAKotE,YAAc,ssaC3RtC,IACMsB,GAAa,iBAYbC,0BAOQ/1B,EAAMzpB,kDAEXypB,KAAOA,IACPsE,eAAiB,CACpBhmB,KAAM,KACNmxB,YAAa,CACXtQ,KAAM,UAER0uB,MAAO,OACPpjE,OAAO,EACPk4D,gBAAgB,EAChBqT,wBAAeC,EAAWC,EAASnC,OAC3BoC,EAAcD,EAAQlR,MAC5BkR,EAAQlR,MAAQiR,EAAUjR,MAC1BiR,EAAUjR,MAAQmR,GAEpBC,WAAY,QAEZ9K,YAAY,EACZ+K,aAAa,EACbzX,oBAAqB,IACrBgS,qBAAsB,CACpBzxB,MAAM,EACN2B,OAAO,GAGT4W,SAAU,CACRkU,YAAY,EACZ6B,aAAa,EACbxrD,KAAK,EACLo7B,QAAQ,EACR8uB,eAAe,GAGjBpH,cAAe,CACbC,OAAO,EACP/iD,KAAK,EACLo7B,QAAQ,GAGV6a,KAAMtL,GAASsL,KAGfoe,4BAAmBC,EAAYp3B,EAAM/xC,GACnCA,EAAS+xC,IAEXq3B,eAAMr3B,EAAM/xC,GACVA,EAAS+xC,IAEXs3B,kBAASt3B,EAAM/xC,GACbA,EAAS+xC,IAEXu3B,gBAAOv3B,EAAM/xC,GACXA,EAAS+xC,IAEXw3B,kBAASx3B,EAAM/xC,GACbA,EAAS+xC,IAEXy3B,kBAASz3B,EAAM/xC,GACbA,EAAS+xC,IAEX03B,oBAAW13B,EAAM/xC,GACfA,EAAS+xC,IAEX23B,qBAAY33B,EAAM/xC,GAChBA,EAAS+xC,IAEX43B,uBAAc53B,EAAM/xC,GAClBA,EAAS+xC,IAGXuiB,OAAQ,CACNviB,KAAM,CACJ+jB,WAAY,GACZjB,SAAU,IAEZvS,KAAM,IAGRsnB,cAAc,EAEdtJ,QAAS,CACPuJ,aAAa,EACb9F,eAAgB,OAChB+F,MAAO,KAGT1K,yBAAyB,KAItBj2C,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/B/tB,QAAQguB,IAAMhuB,EAAQguB,MACtBhuB,QAAQ6vC,UAAY7vC,EAAQ6vC,YAE5BtjB,WAAa,CAChBF,SAAU5C,EAAKN,KAAKkD,SACpBS,OAAQrD,EAAKN,KAAK2D,UAEfmC,IAAM,KACN/I,MAAQ,KACRwO,OAAS,SAERlF,iBACD8mB,UAAY,OACZsK,WAAa,OACb5Q,iBAAmB,OACnB/M,qBAAsB,IACtBoN,oBAAsB,OAEtBwQ,qBAAsB,IAGtBC,cAAgB,cACbzvB,EAAOD,EAAQ2vB,GACnBvxB,EAAGwxB,OAAO5vB,EAAO/J,OACbmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,0BAEI7P,EAAOD,EAAQ2vB,GACtBvxB,EAAG0xB,UAAU9vB,EAAO/J,OAChBmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,0BAEI7P,EAAOD,EAAQ2vB,GACtBvxB,EAAG2xB,UAAU/vB,EAAO/J,OAChBmI,EAAGxvB,QAAQ2wC,SACbnhB,EAAGyxB,iBAAiBjb,SAASxW,EAAGnI,MAAO,CAAEq2B,qBAAqB,IAEhEluB,EAAG0R,aAKFkgB,eAAiB,cACd/vB,EAAOD,EAAQ2vB,MACnBvxB,EAAG6xB,aAAajwB,EAAO/J,OAEnBmI,EAAGoxB,YAAcpxB,EAAGoxB,WAAW/yE,OAAS,EAAG,OACrC+yE,EAAapxB,EAAGoxB,WAAW73B,kBACjC63B,EAAWhmE,eAAc,SAAA0mE,MACrBA,EAAU3T,aAAc,OACE,GAAxB2T,EAAU1T,aACZ0T,EAAU1T,YAAa,OAErB2T,EAAgB,QACpBD,EAAU3T,sBAAqB,SAAA6T,OACvBC,EAAqBb,EAAWhmE,IAAI4mE,GACrCC,IACLA,EAAmB1T,cAAgBuT,EAAUr9C,GACjB,GAAxBq9C,EAAU1T,aACZ6T,EAAmBtV,SAAU,GAE/BoV,EAAgBG,GAAAH,QAAAA,EAAqBE,OAEvCb,EAAW55B,OAAOu6B,EAAeR,yBAKhC1vB,EAAOD,EAAQ2vB,GACtBvxB,EAAGmyB,gBAAgBvwB,EAAO/J,wBAEnBgK,EAAOD,EAAQ2vB,GACtBvxB,EAAGoyB,gBAAgBxwB,EAAO/J,WAIzBA,MAAQ,KACRm2B,OAAS,KACTqE,SAAW,KAEXC,UAAY,KAEZC,MAAQ,OACRC,WAAa,OAEb7L,YAAc,KACdxH,iBAAmB,CACtBuG,MAAO,KACPtG,YAAY,KAITrV,YAEAlK,WAAWrvB,KACXo+C,SAAW,mGAOV5E,EAAQv7C,SAASE,cAAc,OACrCq7C,EAAMrgC,UAAY,cAClBqgC,EAAM,eAAiB3oE,UAClBo+C,IAAIuqB,MAAQA,MAGX/f,EAAax7B,SAASE,cAAc,OAC1Cs7B,EAAWtgB,UAAY,iBACvBqgC,EAAMh0C,YAAYi0B,QACbxK,IAAIwK,WAAaA,MAGhBZ,EAAa56B,SAASE,cAAc,OAC1C06B,EAAW1f,UAAY,iBACvBqgC,EAAMh0C,YAAYqzB,QACb5J,IAAI4J,WAAaA,MAGhBM,EAAOl7B,SAASE,cAAc,OACpCg7B,EAAKhgB,UAAY,gBACZ8V,IAAIkK,KAAOA,MAGVqZ,EAAWv0C,SAASE,cAAc,OACxCq0C,EAASr5B,UAAY,oBAChB8V,IAAIujB,SAAWA,OAGfyP,uBAGCC,EAAkB,IAAIvN,GAAgB4K,GAAY,KAAM1uE,MAC9DqxE,EAAgB7N,YACXmJ,OAAO+B,IAAc2C,OAMrBxtB,OAAS,IAAIG,GAAOhkD,KAAK44C,KAAKwF,IAAI3E,sBAGlCoK,OAAOvN,GAAG,gBAAgB,SAAAkK,GACzBA,EAAMqE,SACR8Q,EAAKzX,SAASsC,WAGbqD,OAAOvN,GAAG,WAAYP,UAAK+H,qBAAkB99C,YAC7C6jD,OAAOvN,GAAG,UAAYP,UAAKgI,gBAAa/9C,YACxC6jD,OAAOvN,GAAG,SAAYP,UAAKiI,mBAAgBh+C,YAC3C6jD,OAAO95C,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAOstB,WAGtDztB,OAAOvN,GAAG,MAAQP,UAAKw7B,sBAAmBvxE,YAG1C6jD,OAAOvN,GAAG,QAASP,UAAKy7B,2BAAwBxxE,YAGhD6jD,OAAOvN,GAAG,YAAaP,UAAKme,mBAAgBl0D,OAE7CA,KAAKmvB,QAAQguB,SACVs0B,YAAc,IAAIztB,GAAOhkD,KAAK44C,KAAKwF,IAAI+S,qBAEvCsgB,YAAc,IAAIztB,GAAOhkD,KAAK44C,KAAKwF,IAAI8S,oBAGzCugB,YAAYn7B,GAAG,MAAYP,UAAK27B,sBAAmB1xE,YACnDyxE,YAAYn7B,GAAG,WAAYP,UAAK47B,0BAAuB3xE,YACvDyxE,YAAYn7B,GAAG,UAAYP,UAAK67B,qBAAkB5xE,YAClDyxE,YAAYn7B,GAAG,SAAYP,UAAK87B,wBAAqB7xE,YACrDyxE,YAAY1nE,IAAI,OAAO/F,IAAI,CAACqlB,UAAU,EAAG/K,UAAW0lC,GAAO8tB,0BAE3Dl5B,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,YAAavI,UAAKg8B,qBAAkB/xE,YAC9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,WAAYvI,UAAKi8B,oBAAiBhyE,YAC5E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,YAAavI,UAAKk8B,qBAAkBjyE,YAE9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,cAAevI,UAAKiI,mBAAgBh+C,YAE9E44C,KAAKwF,IAAI3E,gBAAgB6E,iBAAiB,aAAcvI,UAAKkI,sBAAmBj+C,YAGhFwjE,0CAmEIr0C,iBACLA,EAAS,SAOXmpB,GAAKmG,gBALU,CACb,OAAQ,MAAO,QAAS,QAAS,QAAS,iBAAkB,aAAc,cAAe,sBACzF,sBAAuB,sBAAuB,aAAc,iBAAkB,WAAY,gBAAiB,uBAC3G,OAAQ,OAAQ,iBAAkB,eAAgB,UAAW,0BAA2B,kBAAmB,aAEhFz+C,KAAKmvB,QAASA,GAEvC,yBAA0BA,IACgB,kBAAjCA,EAAQq6C,2BACZr6C,QAAQq6C,qBAAqBzxB,KAAO5oB,EAAQq6C,0BAC5Cr6C,QAAQq6C,qBAAqB9vB,OAAQ,GAEK,WAAxCjU,GAAOtW,EAAQq6C,wBACtBlxB,GAAKmG,gBAAgB,CAAC,OAAQ,SAAUz+C,KAAKmvB,QAAQq6C,qBAAsBr6C,EAAQq6C,sBAE7ExpE,KAAKmvB,QAAQq6C,qBAAqBzxB,YACjC5oB,QAAQq6C,qBAAqB9vB,OAAQ,KAK5C,wBAAyBvqB,GACgB,kBAAhCA,EAAQ6gD,2BACZ7gD,QAAQ6gD,oBAAsB7gD,EAAQ6gD,qBAI3C,gBAAiB7gD,IACgB,iBAAxBA,EAAQk5B,iBACZl5B,QAAQk5B,YAAYtQ,KAA+B,QAAxB5oB,EAAQk5B,YAAwB,MAAQ,SAElC,WAA/B5iB,GAAOtW,EAAQk5B,cAA4B,SAAUl5B,EAAQk5B,mBAC/Dl5B,QAAQk5B,YAAYtQ,KAAO5oB,EAAQk5B,YAAYtQ,OAIpD,WAAY5oB,IACgB,iBAAnBA,EAAQmrC,aACZnrC,QAAQmrC,OAAOhS,KAAOn5B,EAAQmrC,YAC9BnrC,QAAQmrC,OAAOviB,KAAK+jB,WAAa3sC,EAAQmrC,YACzCnrC,QAAQmrC,OAAOviB,KAAK8iB,SAAW1rC,EAAQmrC,QAEX,WAA1B70B,GAAOtW,EAAQmrC,UACtBhiB,GAAKmG,gBAAgB,CAAC,QAASz+C,KAAKmvB,QAAQmrC,OAAQnrC,EAAQmrC,QACxD,SAAUnrC,EAAQmrC,SACe,iBAAxBnrC,EAAQmrC,OAAOviB,WACnB5oB,QAAQmrC,OAAOviB,KAAK+jB,WAAa3sC,EAAQmrC,OAAOviB,UAChD5oB,QAAQmrC,OAAOviB,KAAK8iB,SAAW1rC,EAAQmrC,OAAOviB,MAEb,WAA/BtS,GAAOtW,EAAQmrC,OAAOviB,OAC7BO,GAAKmG,gBAAgB,CAAC,aAAc,YAAaz+C,KAAKmvB,QAAQmrC,OAAOviB,KAAM5oB,EAAQmrC,OAAOviB,eAMjG,SAAU,oBAAmB,SAAA90C,GACxBA,KAAOksB,IACT4xC,EAAK5xC,QAAQlsB,GAAOksB,EAAQlsB,OAI5B,aAAcksB,IACgB,kBAArBA,EAAQmhC,eACZnhC,QAAQmhC,SAASkU,WAAgBr1C,EAAQmhC,cACzCnhC,QAAQmhC,SAAS+V,YAAgBl3C,EAAQmhC,cACzCnhC,QAAQmhC,SAASz1C,IAAgBsU,EAAQmhC,cACzCnhC,QAAQmhC,SAASra,OAAgB9mB,EAAQmhC,cACzCnhC,QAAQmhC,SAASyU,eAAgB,GAEH,WAA5Bt/B,GAAOtW,EAAQmhC,WACtBhY,GAAKmG,gBAAgB,CAAC,aAAc,cAAe,MAAO,SAAU,iBAAkBz+C,KAAKmvB,QAAQmhC,SAAUnhC,EAAQmhC,WAIrH,kBAAmBnhC,IACgB,kBAA1BA,EAAQwuC,oBACZxuC,QAAQwuC,cAAcC,MAASzuC,EAAQwuC,mBACvCxuC,QAAQwuC,cAAc9iD,IAASsU,EAAQwuC,mBACvCxuC,QAAQwuC,cAAc1nB,OAAS9mB,EAAQwuC,eAEJ,WAAjCl4B,GAAOtW,EAAQwuC,gBACtBrlB,GAAKmG,gBAAgB,CAAC,QAAS,MAAO,UAAWz+C,KAAKmvB,QAAQwuC,cAAexuC,EAAQwuC,sBAcxF,qBAAsB,QAAS,WAAY,WAAY,SAAU,WAAY,aAAc,cAAe,0BATvF,SAAAl6D,OACZ5G,EAAKsyB,EAAQ1rB,MACf5G,EAAI,UACc,mBAAPA,QACL,IAAIuG,4BAAgBK,kCAA2BA,uBAEvDs9D,EAAK5xC,QAAQ1rB,GAAQ5G,MAKrBsyB,EAAQ2wC,YACI9/D,KAAKmvB,QAAS,CAC1B2wC,QAAS3wC,EAAQ2wC,UAEd9/D,KAAKowE,wBACHA,iBAAmB,IAAI1D,GAAiB1sE,YAE1CowE,iBAAiBjb,SAASn1D,KAAKw2C,MAAO,CAAEq2B,qBAAqB,SAC7DqF,UAAU,CAAEC,cAAc,EAAMC,eAAe,SAE/C/hB,UACIrwD,KAAKowE,uBACTiC,0BACA9E,SAAW,QACX6C,iBAAmB,UACnBjhD,QAAQ2wC,aAAUrgE,OAClByyE,UAAU,CAAEC,cAAc,EAAMC,eAAe,SAE/C/hB,eAGA6hB,+CAUD/iD,QACH6hD,SAAW,GAEZ7hD,IACEA,EAAQgjD,iBACV75B,SAAAA,GAAat4C,KAAKw2C,OAAO,SAAAuB,GACvBA,EAAK2mB,OAAQ,EACT3mB,EAAK4mB,WAAW5mB,EAAKsY,YAIzBlhC,EAAQijD,kBACV95B,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAACtI,EAAOphE,GAC5BA,IAAQyrE,KACZrK,EAAMzH,YAAa,8CAUpB0V,uBACApiB,YACAiF,SAAS,WACTC,UAAU,WAEVvR,QAAU7jD,KAAK6jD,OAAOkB,eACtB0sB,aAAezxE,KAAKyxE,YAAY1sB,eAChClB,OAAS,UAETjL,KAAO,UACP8C,WAAa,oCAQd17C,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,OAI7C3oE,KAAKo+C,IAAIkK,KAAK/gB,iBACX6W,IAAIkK,KAAK/gB,WAAWC,YAAYxnC,KAAKo+C,IAAIkK,MAI5CtoD,KAAKo+C,IAAIujB,SAASp6B,iBACf6W,IAAIujB,SAASp6B,WAAWC,YAAYxnC,KAAKo+C,IAAIujB,yCAS/C3hE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,OAIvC3oE,KAAKo+C,IAAIkK,KAAK/gB,iBACZqR,KAAKwF,IAAIyL,mBAAmBl1B,YAAY30B,KAAKo+C,IAAIkK,MAInDtoD,KAAKo+C,IAAIujB,SAASp6B,aACjBvnC,KAAKmvB,QAAQguB,SACVvE,KAAKwF,IAAIzJ,MAAMhgB,YAAY30B,KAAKo+C,IAAIujB,eAEpC/oB,KAAKwF,IAAI1J,KAAK/f,YAAY30B,KAAKo+C,IAAIujB,iDAShCuP,WACPoB,kBACDpB,EAAO,KACHpB,EAAQ9vE,KAAKmvB,QAAQm3C,QAAQwJ,OAA+C,iBAA/B9vE,KAAKmvB,QAAQm3C,QAAQwJ,MAClE9vE,KAAKmvB,QAAQm3C,QAAQwJ,MACrB,SACDqB,WAAahyB,IAChB,WACE+xB,EAAM1N,SACLsM,8CAQgB,MAAnB9vE,KAAKmxE,aACL/xB,aAAap/C,KAAKmxE,iBACbA,WAAa,2CAWXoB,SACA9yE,MAAP8yE,IACFA,EAAM,IAGHrvC,GAAcqvC,KACjBA,EAAM,CAACA,QAIHC,EAAgB17B,UAAKm6B,mBAAiB,SAAA79C,UAA2B,IAArBwgC,GAAA2e,QAAAA,EAAYn/C,4CAGvCo/C,+BAAe,KAA7BC,UACD16B,EAAO/3C,KAAK0yE,YAAYD,GAC1B16B,GACFA,EAAK46B,mGAKJ1B,aAAiBsB,6CACPA,+BAAK,KAAXn/C,UACD2kB,EAAO/3C,KAAK0yE,YAAYt/C,GAC1B2kB,GACFA,EAAK66B,iJAUF/B,UAAKI,kBAAiB,kDASzBt8B,EACAD,EAFEgF,EAAQ15C,KAAK44C,KAAKc,MAAMqc,WAI1B/1D,KAAKmvB,QAAQguB,KACfxI,EAAS30C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMlZ,OACvCkU,EAAO10C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMjZ,OAErCiU,EAAQ10C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMlZ,OACtCmU,EAAQ30C,KAAK44C,KAAKN,KAAKkD,SAAS9B,EAAMjZ,UAGlC8xC,EAAM,OACP,IAAMpW,KAAWn8D,KAAK2sE,UACrB3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KACjCkI,EAAQrkE,KAAK2sE,OAAOxQ,GACpB0W,EAAkBxO,EAAM1H,UAAY0H,EAAMlH,aAAe,yCAI5C0V,+BAAiB,KAAzB96B,UAEL/3C,KAAKmvB,QAAQguB,IACVpF,EAAKpD,MAAQD,GAAUqD,EAAKpD,MAAQoD,EAAKnP,MAAQ+L,GACpD49B,EAAIt1E,KAAK86C,EAAK3kB,IAGX2kB,EAAKrD,KAAOC,GAAWoD,EAAKrD,KAAOqD,EAAKnP,MAAQ8L,GACnD69B,EAAIt1E,KAAK86C,EAAK3kB,+FAOjBm/C,iDAQDA,EAAM,OAEP,IAAMpW,KAAWn8D,KAAK2sE,OAAQ,IAC7B3sE,KAAK2sE,OAAOtvE,eAAe8+D,GACfn8D,KAAK2sE,OAAOxQ,GAChBQ,WACR4V,EAAIt1E,KAAKk/D,UAKRoW,sCAQGn/C,gBACHpzB,KAAKw2C,MAAMpjB,IAAO0/C,UAAKvF,kBAAc,SAAAzN,UAAWA,EAAQ1sC,KAAOA,uCAQ9DA,WACF69C,EAAYjxE,KAAKixE,UACdn0E,EAAI,EAAGmO,EAAKgmE,EAAUj0E,OAAQF,EAAImO,EAAInO,OACzCm0E,EAAUn0E,IAAMs2B,EAAI,IACtB69C,QAAAA,EAAiBn0E,EAAG,sDAWlBw9D,EAASt6D,KAAKmvB,QAAQmrC,OACtB5gB,EAAQ15C,KAAK44C,KAAKc,MAClB6c,EAASje,GAAKge,OAAOC,OACrBpnC,EAAUnvB,KAAKmvB,QACfk5B,EAAcl5B,EAAQk5B,YAAYtQ,KACpCc,GAAU,EACR8vB,EAAQ3oE,KAAKo+C,IAAIuqB,WAGlBtzB,MAAMyN,IAAM9iD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAEvE9iD,KAAKmvB,QAAQguB,SACV9H,MAAMV,MAAQ30C,KAAK44C,KAAKY,SAAS7E,MAAM/L,MAAQ5oC,KAAK44C,KAAKY,SAASmY,OAAOhd,WAEzEU,MAAMX,KAAO10C,KAAK44C,KAAKY,SAAS9E,KAAK9L,MAAQ5oC,KAAK44C,KAAKY,SAASmY,OAAOjd,KAI9Ei0B,EAAMrgC,UAAY,cAEdtoC,KAAKmvB,QAAQ2wC,cACViT,gBAIPl6B,EAAU74C,KAAKgzE,gBAAkBn6B,MAI3Bo6B,EAAkBv5B,EAAMjZ,IAAMiZ,EAAMlZ,MACpC0yC,EAAUD,GAAmBjzE,KAAKmzE,qBAAyBnzE,KAAKq1C,MAAMzM,OAAS5oC,KAAKq1C,MAAM4iB,UAC1Fmb,EAAW15B,EAAMlZ,OAASxgC,KAAKqzE,eAC/BC,EAAqBnkD,EAAQ9rB,OAASrD,KAAKuzE,UAC3CC,EAA8BrkD,EAAQosC,gBAAkBv7D,KAAKyzE,mBAC7D/T,EAAgBwT,GAAUE,GAAYE,GAAsBE,OAC7DL,oBAAsBF,OACtBI,eAAiB35B,EAAMlZ,WACvB+yC,UAAYpkD,EAAQ9rB,WACpBowE,mBAAqBtkD,EAAQosC,oBAE7BlmB,MAAM4iB,UAAYj4D,KAAKq1C,MAAMzM,UAC5B8qC,EAAa1zE,KAAK2zE,cAClBC,EAAc,CAClB77B,KAAMuiB,EAAOviB,KACbuQ,KAAMgS,EAAOhS,MAETurB,EAAiB,CACrB97B,KAAMuiB,EAAOviB,KACbuQ,KAAMgS,EAAOviB,KAAK8iB,SAAW,GAE3BhyB,EAAS,EACP2tB,EAAY8D,EAAOhS,KAAOgS,EAAOviB,KAAK8iB,cAGvC8R,OAAO+B,IAAYre,OAAO3W,EAAOm6B,EAAgBnU,OAEhDlB,EAAc,GAChBC,EAAoB,YAGxBnmB,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAACtI,EAAOphE,MAC5BA,IAAQyrE,QACNoF,EAAczP,GAASqP,EAAaE,EAAcC,EAExDrV,EAAYv7D,GAAOohE,EAAMhU,OAAO3W,EAAOo6B,EAAapU,GADhC,GAEpBjB,EAAoBD,EAAYv7D,GAAKjG,WAGpByhE,EAAoB,sBAE/BsV,EAAgB,cAEbj3E,MACPw7C,SAAAA,GAAakmB,GAAa,SAACI,EAAK37D,GAC9B8wE,EAAc9wE,GAAO27D,EAAI9hE,SAFpBA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,MAOTw7C,SAAAA,GAAaopB,EAAKiL,QAAQ,SAACtI,EAAOphE,MAC5BA,IAAQyrE,QACNsF,EAAeD,EAAc9wE,GACnC41C,EAAUm7B,GAAgBn7B,EAC1BhQ,GAAUw7B,EAAMx7B,WAElBA,EAASrnC,KAAK8D,IAAIujC,EAAQ2tB,MAG5B3tB,EAASrnC,KAAK8D,IAAIujC,EAAQ2tB,GAG1BmS,EAAMn0C,MAAMqU,OAAU0tB,EAAO1tB,QAGxBwM,MAAMzM,MAAQ+/B,EAAMvf,iBACpB/T,MAAMxM,OAASA,OAGfuV,IAAIkK,KAAK9zB,MAAMsuB,IAAMyT,EAAuB,OAAflO,EAC7BroD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAC1D9iD,KAAK44C,KAAKY,SAASsJ,IAAIja,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,QACpE7oC,KAAKmvB,QAAQguB,SACViB,IAAIkK,KAAK9zB,MAAMmgB,MAAQ,SAEvByJ,IAAIkK,KAAK9zB,MAAMkgB,KAAO,SAGxBmP,OAAO95C,IAAI,SAAS/F,IAAI,CAAC0c,KAAM1gB,KAAKmvB,QAAQqoC,2BAE5CpF,qBAAsB,EAE3BvZ,EAAU74C,KAAK8pD,cAAgBjR,4CAWzBo7B,EAAoD,OAAjCj0E,KAAKmvB,QAAQk5B,YAAYtQ,KAAiB,EAAK/3C,KAAKgxE,SAASh0E,OAAS,EACzFk3E,EAAel0E,KAAKgxE,SAASiD,UAChBj0E,KAAK2sE,OAAOuH,IAAiBl0E,KAAK2sE,OAAL,eAE3B,oDAUjB50B,EACAo8B,EAFAC,EAAYp0E,KAAK2sE,OAAL,iBAIZ3sE,KAAK+vE,eAEHqE,MAIGD,KAHLC,EAAU/7B,iBACHr4C,KAAK2sE,OAAL,cAEQ3sE,KAAKw2C,SACdx2C,KAAKw2C,MAAMn5C,eAAe82E,GAAS,EACrCp8B,EAAO/3C,KAAKw2C,MAAM29B,IACbvjD,QAAUmnB,EAAKnnB,OAAOqlB,OAAO8B,OAC5BokB,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GAC1BkI,GAASA,EAAMxpD,IAAIk9B,IAASA,EAAKmY,iBAOlCkkB,EAAW,KAMTD,KAHLC,EAAY,IAAIlY,GAFL,KACE,KACmBl8D,WAC3B2sE,OAAL,cAAyByH,EAEVp0E,KAAKw2C,MACdx2C,KAAKw2C,MAAMn5C,eAAe82E,KAC5Bp8B,EAAO/3C,KAAKw2C,MAAM29B,GAClBC,EAAUv5D,IAAIk9B,IAIlBq8B,EAAU5Q,qDAUPxjE,KAAKo+C,IAAIujB,0CAOTnrB,QACF2oB,iBAAmB,IAAIziE,SAExB61E,EADE5zB,EAAK3+C,KAELq0E,EAAer0E,KAAKylE,aAGrBjvB,EAGA,CAAA,KAAIA,aAAiBoB,IAAWpB,aAAiB89B,UAI9C,IAAItnD,UAAU,wDAHfy4C,UAAYluB,GAAkBf,aAH9BivB,UAAY,QASf4O,OAEF/7B,SAAAA,GAAat4C,KAAKiwE,eAAe,SAACjqE,EAAUw6C,GAC1C6zB,EAAa99B,IAAIiK,EAAOx6C,MAI1BquE,EAAah8B,UAGbk6B,EAAM8B,EAAaj8B,cACdk4B,UAAUiC,IAGbvyE,KAAKylE,UAAW,KAEZryC,EAAKpzB,KAAKozB,MAChBklB,SAAAA,GAAat4C,KAAKiwE,eAAe,SAACjqE,EAAUw6C,GAC1C7B,EAAG8mB,UAAUnvB,GAAGkK,EAAOx6C,EAAUotB,MAInCm/C,EAAMvyE,KAAKylE,UAAUrtB,cAChB+3B,OAAOoC,QAGPnB,wBAGFx4B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,8CAQjB,MAAlBryD,KAAKylE,UAAoBzlE,KAAKylE,UAAUjuB,MAAQ,uCAO/Cm1B,OAEJ4F,EADE5zB,EAAK3+C,QAIPA,KAAK+vE,gBACPz3B,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAACvqE,EAAUw6C,GAC3C7B,EAAGoxB,WAAWx5B,IAAIiK,EAAOx6C,MAI3BusE,EAAMvyE,KAAK+vE,WAAW33B,cACjB23B,WAAa,UACbgB,gBAAgBwB,IAIlB5F,EAGA,CAAA,KAAIA,aAAkB/0B,IAAW+0B,aAAkB2H,UAIhD,IAAItnD,UAAU,wDAHf+iD,WAAapD,YAHboD,WAAa,QAShB/vE,KAAK+vE,WAAY,OAEfA,EAAa/vE,KAAK+vE,WAClB/vE,KAAK+vE,sBAAsBuE,KAC7BvE,EAAa/vE,KAAK+vE,WAAW73B,mBAG/B63B,EAAWhmE,eAAc,SAAAs6D,SACnBA,EAAMvH,mBACRuH,EAAMvH,sBAAqB,SAAA6T,OACnBC,EAAqBb,EAAWhmE,IAAI4mE,GAC1CC,EAAmB1T,cAAgBmH,EAAMjxC,GACjB,GAApBixC,EAAMtH,aACR6T,EAAmBtV,SAAU,GAE/ByU,EAAW55B,OAAOy6B,aAMlBx9C,EAAKpzB,KAAKozB,MAChBklB,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAACvqE,EAAUw6C,GAC3C7B,EAAGoxB,WAAWz5B,GAAGkK,EAAOx6C,EAAUotB,MAIpCm/C,EAAMvyE,KAAK+vE,WAAW33B,cACjBo4B,aAAa+B,QAIfnB,wBAGAmD,SAEDv0E,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBoE,kBACjBzB,qBACAb,UAAU,CAAEC,cAAc,EAAMC,eAAe,UAGjDx5B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,+CAQnCryD,KAAK+vE,8CAOH38C,cACH2kB,EAAO/3C,KAAKylE,UAAU17D,IAAIqpB,GAE5B2kB,QAEG5oB,QAAQogD,SAASx3B,GAAM,SAAAA,GACtBA,GAGF08B,EAAKhP,UAAUxvB,OAAO7iB,uCAYrB4gC,UACAA,EAAS98B,MAAQl3B,KAAKmvB,QAAQ+H,OAAS88B,EAASvzB,IAAM,QAAU,0CAS9DuzB,SAEG,cADCh0D,KAAK00E,SAAS1gB,IACmBv0D,MAAlBu0D,EAASqQ,MAC7BqK,GAGC1uE,KAAK+vE,WAAa/b,EAASqQ,MA1lCtB,kDAmmCNkO,cACF5zB,EAAK3+C,QAEXuyE,QAAAA,GAAY,SAAAn/C,OAMN4wC,EALEhQ,EAAWrV,EAAG8mB,UAAU17D,IAAIqpB,GAC9B2kB,EAAO4G,EAAGnI,MAAMpjB,GACd8D,EAAO88B,EAAWrV,EAAG+1B,SAAS1gB,GAAY,KAE1C18B,EAAcq3C,EAAQgG,MAAMz9C,MAG9B6gB,IAEGzgB,GAAiBygB,aAAgBzgB,EAOpCqnB,EAAGi2B,YAAY78B,EAAMic,IALrBgQ,EAAWjsB,EAAKisB,SAChBrlB,EAAGk2B,YAAY98B,GACfA,EAAO,QAONA,GAAQic,EAAU,KAEjB18B,QAWI,IAAItK,uCAAgCkK,SAV1C6gB,EAAO,IAAIzgB,EAAY08B,EAAUrV,EAAGjD,WAAYiD,EAAGxvB,UAC9CiE,GAAKA,EAEVurB,EAAGm2B,SAAS/8B,GACRisB,IACF+Q,EAAK9D,UAAUh0E,KAAKm2B,GACpB2kB,EAAK66B,mBASR2B,SAEDv0E,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBjb,SAASn1D,KAAKw2C,MAAO,CAAEq2B,qBAAqB,SAC7DkG,sBAGFn6B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,sCAQlCkgB,OACJt5B,EAAQ,EACN0F,EAAK3+C,QACXuyE,QAAAA,GAAY,SAAAn/C,OACJ2kB,EAAO4G,EAAGnI,MAAMpjB,GAClB2kB,IACFkB,IACA0F,EAAGk2B,YAAY98B,OAIfkB,SAEGs7B,cACA37B,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,yCAW5C/Z,SAAAA,GAAat4C,KAAK2sE,QAAQ,SAAAtI,GACxBA,EAAMzG,mDASM2U,QACT/B,aAAa+B,wCAQPA,OACL5zB,EAAK3+C,QAEXuyE,QAAAA,GAAY,SAAAn/C,OACJq9C,EAAY9xB,EAAGoxB,WAAWhmE,IAAIqpB,GAChCixC,EAAQ1lB,EAAGguB,OAAOv5C,MAEjBixC,EA6BHA,EAAM3G,QAAQ+S,OA7BJ,IA/sCA,iBAitCNr9C,GAAmBA,GAAMs7C,SACrB,IAAItrE,kCAA2BgwB,6BAGjC4hD,EAAe3qC,GAAcsU,EAAGxvB,aASjC,IAAMglD,KARX77B,GAAKh7C,OAAO03E,EAAc,CACxBnsC,OAAQ,OAGVw7B,EAAQ,IAAInI,GAAM9oC,EAAIq9C,EAAW9xB,GACjCA,EAAGguB,OAAOv5C,GAAMixC,EAGK1lB,EAAGnI,SAClBmI,EAAGnI,MAAMn5C,eAAe82E,GAAS,KAC7Bp8B,EAAO4G,EAAGnI,MAAM29B,GAClBp8B,EAAKxkC,KAAK8wD,OAASjxC,GACrBixC,EAAMxpD,IAAIk9B,GAKhBssB,EAAMzG,QACNyG,EAAMb,gBAQL5qB,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,4CAQ5BkgB,iBACdA,QAAAA,GAAY,SAAAn/C,OACJixC,EAAQ4Q,EAAKtI,OAAOv5C,GAEtBixC,IACFA,EAAMhsB,iBACC48B,EAAKtI,OAAOv5C,OAInBpzB,KAAKmvB,QAAQ2wC,eACVsQ,iBAAiBoE,kBACjBzB,sBAGFb,UAAU,CAAEE,gBAAiBpyE,KAAKmvB,QAAQ2wC,eAC1ClnB,KAAKiF,QAAQ4C,KAAK,UAAW,CAAC4R,OAAO,8CAStCryD,KAAK+vE,WAAY,KAEfiB,EAAWhxE,KAAK+vE,WAAW33B,OAAO,CACpCwlB,MAAO59D,KAAKmvB,QAAQ6/C,aAGtBgC,EAAWhxE,KAAKk1E,mBAAmBlE,OAE7B3wB,GAAW/H,GAAK68B,WAAWnE,EAAUhxE,KAAKgxE,aAC5C3wB,EAAS,KAELssB,EAAS3sE,KAAK2sE,UACpBqE,QAAAA,GAAiB,SAAA7U,GACfwQ,EAAOxQ,GAASjM,aAIlB8gB,QAAAA,GAAiB,SAAA7U,GACfwQ,EAAOxQ,GAASqH,eAGbwN,SAAWA,SAGX3wB,SAGA,6CAWQ2wB,8BASRoE,EAAuBr1E,EAAGixE,OAC7Bz4D,EAAS,aACby4D,QAAAA,GAAiB,SAAA7U,MACf5jD,EAAOtb,KAAKk/D,GACMp8D,EAAEgwE,WAAWhmE,IAAIoyD,GACrBW,aAAc,OACpBuY,EAAiBt+B,KAAAh3C,EAAEgwE,WAAWhmE,IAAI,CACtCkuB,gBAAOq9C,UACEA,EAAYpY,eAAiBf,GAEtCyB,MAAO79D,EAAEovB,QAAQ6/C,sBACZ,SAAAsG,UAAeA,EAAYliD,MAClC7a,EAASs4D,GAAAt4D,QAAAA,EAAc68D,EAAuBr1E,EAAGs1E,QAI9C98D,EAKF68D,CAAuBp1E,KAFV82C,GAAAk6B,QAAAA,GAAgB,SAAA7U,UAAYoZ,EAAKxF,WAAWhmE,IAAIoyD,GAASe,mDAUtEnlB,QACFvB,MAAMuB,EAAK3kB,IAAM2kB,MAGhBokB,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GAErBkI,EAEMA,GAASA,EAAM9wD,MAAQ8wD,EAAM9wD,KAAKwpD,aAC3ChlB,EAAKksB,cAAe,GAFpBlsB,EAAKksB,cAAe,EAKlBI,GAAOA,EAAMxpD,IAAIk9B,uCASXA,EAAMic,GAEhBjc,EAAK2lB,QAAQ1J,OAEPmI,EAAUn8D,KAAK4tE,WAAW71B,EAAKxkC,MAC/B8wD,EAAQrkE,KAAK2sE,OAAOxQ,GACrBkI,EAEMA,GAASA,EAAM9wD,MAAQ8wD,EAAM9wD,KAAKwpD,aAC3ChlB,EAAKksB,cAAe,GAFpBlsB,EAAKksB,cAAe,sCAYZlsB,WAEVA,EAAKmY,cAGElwD,KAAKw2C,MAAMuB,EAAK3kB,QAGjBxO,EAAQgvC,UAAKqd,kBAAkBl5B,EAAK3kB,KAC5B,GAAVxO,GAAa09C,UAAK2O,kBAAiBrsD,EAAO,GAG9CmzB,EAAKnnB,QAAUmnB,EAAKnnB,OAAOqlB,OAAO8B,GAGhB,MAAd/3C,KAAKkxE,YACFA,MAAMhhB,oDAUM7pD,WACbs8D,EAAW,GAER7lE,EAAI,EAAGA,EAAIuJ,EAAMrJ,OAAQF,IAC5BuJ,EAAMvJ,aAAc4rE,IACtB/F,EAAS1lE,KAAKoJ,EAAMvJ,WAGjB6lE,mCAaAniB,QAEF8kB,YAAYvtB,KAAO/3C,KAAKw1E,eAAeh1B,QACvC8kB,YAAYmE,aAAejpB,EAAM7wB,OAAO85C,eAAgB,OACxDnE,YAAYqE,cAAgBnpB,EAAM7wB,OAAOg6C,gBAAiB,OAC1DrE,YAAYmQ,UAAY,4CAUhBtZ,OACN,IAAIr/D,EAAI,EAAGA,EAAIkD,KAAKgxE,SAASh0E,OAAQF,OAClCq/D,GAAWn8D,KAAKgxE,SAASl0E,GACzB,OAAOA,uCASN0jD,kBACPxgD,KAAKslE,YAAYC,oBAGjBlwB,EAFE0C,EAAO/3C,KAAKslE,YAAYvtB,MAAQ,KAChC4G,EAAK3+C,QAGP+3C,IAASA,EAAKisB,UAAYhkE,KAAKmvB,QAAQq6C,qBAAqBzxB,MAAO,IAEjE/3C,KAAKmvB,QAAQmhC,SAASyU,gBACrB/kE,KAAKmvB,QAAQmhC,SAASkU,aACtBxkE,KAAKmvB,QAAQmhC,SAAS+V,sBAKL,MAAjBtuB,EAAKuY,WAAqBvY,EAAKuY,SAASkU,aAAezsB,EAAKuY,SAAS+V,cAClErmE,KAAKmvB,QAAQmhC,SAASyU,yBAIxB0E,EAAezpE,KAAKslE,YAAYmE,aAChCE,EAAgB3pE,KAAKslE,YAAYqE,sBAClCrE,YAAYC,gBAAiB,OAC7BD,YAAYoQ,aAAe39B,EAE5B0xB,EACFp0B,EAAQ,CACN0C,KAAM0xB,EACNkM,SAAUn1B,EAAMxB,OAAO70C,EACvB26D,UAAW,EACXvxD,KAAMvT,KAAK41E,eAAe79B,EAAKxkC,YAG5B+xD,YAAYmQ,UAAY,CAACpgC,QACzB,GAAIs0B,EACTt0B,EAAQ,CACN0C,KAAM4xB,EACNgM,SAAUn1B,EAAMxB,OAAO70C,EACvBu/D,WAAW,EACXn2D,KAAMvT,KAAK41E,eAAe79B,EAAKxkC,YAG5B+xD,YAAYmQ,UAAY,CAACpgC,QACzB,GAAIr1C,KAAKmvB,QAAQmhC,SAASz1C,MAAQ2lC,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,cAE3EC,oBAAoBv1B,OACpB,CACFxgD,KAAKgxE,SAASh0E,OAAS,QAGnBqzD,aAGD2lB,EAAiBh2E,KAAKi2E,eAAel+B,EAAKxkC,KAAK8wD,OAE/C6R,EAAel2E,KAAKmvB,QAAQq6C,qBAAqBzxB,OAASA,EAAKisB,SAAY,CAACjsB,EAAK3kB,IAAMpzB,KAAKm2E,oBAE7F7Q,YAAYmQ,UAAY1+B,GAAAm/B,QAAAA,GAAgB,SAAA9iD,OACrC2kB,EAAO4G,EAAGnI,MAAMpjB,GAChBgjD,EAAaz3B,EAAGs3B,eAAel+B,EAAKxkC,KAAK8wD,aACxC,CACLtsB,KAAAA,EACA49B,SAAUn1B,EAAMxB,OAAO70C,EACvBksE,YAAaL,EAAeI,EAC5B7iE,KAAM+iE,EAAKV,eAAe79B,EAAKxkC,UAKrCitC,EAAM8D,uBACGtkD,KAAKmvB,QAAQmhC,SAASz1C,MAAQ2lC,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,eAE3EC,oBAAoBv1B,gDASTA,OACZsQ,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAC5BylB,EAAYv2E,KAAKo+C,IAAIuqB,MAAM/lB,wBAG3Bz4C,EAAInK,KAAKmvB,QAAQguB,IAAMo5B,EAAU5hC,MAAQ6L,EAAMxB,OAAO70C,EAAK,GAAKq2C,EAAMxB,OAAO70C,EAAIosE,EAAU7hC,KAAO,GAElGh0B,EAAO1gB,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAC7BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBrwB,EAAQswB,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,EAGzCszC,EAAW,CACf98B,KAAM,QACNsJ,MAAAA,EACAC,IALUD,EAMV3M,QAAS,YAGLT,EAAKs3C,KACX1W,EAASh0D,KAAKylE,UAAUhuB,QAAUrkB,MAE5BixC,EAAQrkE,KAAKw2E,gBAAgBh2B,GAC/B6jB,IACFrQ,EAASqQ,MAAQA,EAAMlI,aAEnBsa,EAAU,IAAI/N,GAAU1U,EAAUh0D,KAAK07C,WAAY17C,KAAKmvB,SAC9DsnD,EAAQrjD,GAAKA,EACbqjD,EAAQljE,KAAOvT,KAAK41E,eAAe5hB,QAC9B8gB,SAAS2B,QACTnR,YAAYoQ,aAAee,MAE1BphC,EAAQ,CACZ0C,KAAM0+B,EACNd,SAAUn1B,EAAMxB,OAAO70C,EACvBoJ,KAAMkjE,EAAQljE,MAGZvT,KAAKmvB,QAAQguB,IACf9H,EAAMyvB,UAAW,EAEjBzvB,EAAMq0B,WAAY,OAEfpE,YAAYmQ,UAAY,CAACpgC,GAE9BmL,EAAM8D,kDAQA9D,iBACY,MAAdxgD,KAAKkxE,OAAiBlxE,KAAKmvB,QAAQygD,eAAiB5vE,KAAKkxE,MAAM32B,OAAQ,KAEnE0R,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAMxB,OAAO70C,EAAIusE,EAAchiC,KAAOuX,EAAU6S,WAChDte,EAAMxB,OAAOz1C,EAAImtE,EAAc5zB,IAAMmJ,EAAU4S,gBAE5CqS,MAAM1N,UAGTxjE,KAAKslE,YAAYmQ,UAAW,OAC9Bj1B,EAAM8D,sBAEA3F,EAAK3+C,KACL8wD,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAC5B8lB,EAAoB52E,KAAK44C,KAAKwF,IAAI+C,KAAK2d,WACvCt2B,EAAUxoC,KAAKmvB,QAAQguB,IAAMy5B,EAAoB52E,KAAK44C,KAAKY,SAAS7E,MAAM/L,MAAQguC,EAAoB52E,KAAK44C,KAAKY,SAAS9E,KAAK9L,MAC9H+S,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UAGtB6kB,EAAe11E,KAAKslE,YAAYoQ,aAChCmB,GAAuB72E,KAAKmvB,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,WAAqBtwD,KAAKmvB,QAAQmhC,SAAS+V,cACjGrmE,KAAKmvB,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,UAAoBolB,EAAaplB,SAAS+V,YACrHyQ,EAAe,QACfD,GAAsBnB,GACOj2E,MAA3Bi2E,EAAaniE,KAAK8wD,MAAoB,KAElCA,EAAQ1lB,EAAG63B,gBAAgBh2B,GAC7B6jB,IAGFyS,EAAe92E,KAAKi2E,eAAe5R,EAAMlI,oBAM1CmJ,YAAYmQ,mBAAkB,SAAApgC,OAG7B74B,EACAu6D,EACA5U,EACA3hC,EACAC,EANEwa,EAAU0D,EAAG/F,KAAKN,KAAK2D,OAAOuE,EAAMxB,OAAO70C,EAAIq+B,GAC/CwuC,EAAUr4B,EAAG/F,KAAKN,KAAK2D,OAAO5G,EAAMsgC,SAAWntC,GAQnDhsB,EADEy6D,EAAK9nD,QAAQguB,MACJlC,EAAU+7B,GAEX/7B,EAAU+7B,MAGlBhjB,EAAWijB,EAAKrB,eAAevgC,EAAM0C,KAAKxkC,SACnB,MAAvB8hC,EAAM0C,KAAKuY,UACTjb,EAAM0C,KAAKuY,SAASkU,YACpBnvB,EAAM0C,KAAKuY,SAAS+V,aACpB1nB,EAAGxvB,QAAQmhC,SAASyU,mBAIEkS,EAAK9nD,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,WAAqB2mB,EAAK9nD,QAAQmhC,SAASkU,aAChGyS,EAAK9nD,QAAQmhC,SAASyU,eAA0C,MAAzB2Q,EAAaplB,UAAoBolB,EAAaplB,SAASkU,cAEnHnvB,EAAMyvB,SAEJmS,EAAK9nD,QAAQguB,IACK19C,MAAhBu0D,EAASvzB,MACX0hC,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,QAC1CA,EAAM,IAAI/jC,KAAKylE,EAAW5kE,UAAYif,GAEtCw3C,EAASvzB,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,GAG3BhhC,MAAlBu0D,EAASxzB,QACXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAC9CA,EAAQ,IAAI9jC,KAAKq6E,EAAax5E,UAAYif,GAE1Cw3C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,QAIpD,GAAI6U,EAAMq0B,UAETuN,EAAK9nD,QAAQguB,IACO19C,MAAlBu0D,EAASxzB,QACXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAC9CA,EAAQ,IAAI9jC,KAAKq6E,EAAax5E,UAAYif,GAE1Cw3C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,GAGjC/gC,MAAhBu0D,EAASvzB,MACX0hC,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,QAC1CA,EAAM,IAAI/jC,KAAKylE,EAAW5kE,UAAYif,GAEtCw3C,EAASvzB,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,WAM7BhhC,MAAlBu0D,EAASxzB,SAEXu2C,EAAez+B,GAAKpB,QAAQ7B,EAAM9hC,KAAKitB,MAAO,QAAQjjC,UACtDijC,EAAQ,IAAI9jC,KAAKq6E,EAAev6D,GAEZ/c,MAAhBu0D,EAASvzB,IAAkB,KAEvBplB,GADN8mD,EAAa7pB,GAAKpB,QAAQ7B,EAAM9hC,KAAKktB,IAAK,SACbljC,UAAYw5E,EAAax5E,UAGtDy2D,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EACnDwzB,EAASvzB,IAAQ,IAAI/jC,KAAKs3D,EAASxzB,MAAMjjC,UAAY8d,QAIrD24C,EAASxzB,MAAQswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,KAMvDq2C,IAAwBxhC,EAAMyvB,WAAazvB,EAAMq0B,WAA4B,MAAdoN,GAC3Cr3E,MAAlBu0D,EAASqQ,MAAoB,KAC3B6S,EAAYJ,EAAezhC,EAAMghC,YAGrCa,EAAY11E,KAAK8D,IAAI,EAAG4xE,GACxBA,EAAY11E,KAAKY,IAAIu8C,EAAGqyB,SAASh0E,OAAO,EAAGk6E,GAC3CljB,EAASqQ,MAAQ1lB,EAAGqyB,SAASkG,GAKjCljB,EAAWijB,EAAKrB,eAAe5hB,GAC/BrV,EAAGxvB,QAAQqgD,SAASxb,GAAU,SAAAA,GACxBA,GACF3e,EAAM0C,KAAK2lB,QAAQuZ,EAAKrB,eAAe5hB,EAAU,qBAKlDpb,KAAKiF,QAAQ4C,KAAK,iDAUd1I,EAAMokB,OACXkI,EAAQrkE,KAAK2sE,OAAOxQ,MACtBkI,GAASA,EAAMlI,SAAWpkB,EAAKxkC,KAAK8wD,MAAO,KACvC8S,EAAWp/B,EAAKnnB,OACtBumD,EAASlhC,OAAO8B,GAChBo/B,EAASvZ,QAET7lB,EAAKxkC,KAAK8wD,MAAQA,EAAMlI,QAExBkI,EAAMxpD,IAAIk9B,GACVssB,EAAMzG,4CASCpd,sBACJ8kB,YAAYC,gBAAiB,EAC9BvlE,KAAKslE,YAAYmQ,UAAW,CAC9Bj1B,EAAM8D,sBAEA3F,EAAK3+C,KACLy1E,EAAYz1E,KAAKslE,YAAYmQ,eAC9BnQ,YAAYmQ,UAAY,QAE7BA,QAAAA,GAAkB,SAAApgC,OACVjiB,EAAKiiB,EAAM0C,KAAK3kB,MACiB,MAAxBurB,EAAG8mB,UAAU17D,IAAIqpB,GAc3B,KAEG4gC,EAAWojB,EAAKxB,eAAevgC,EAAM0C,KAAKxkC,MAChDorC,EAAGxvB,QAAQmgD,OAAOtb,GAAU,SAAAA,GACtBA,GAEFA,EAASojB,EAAK3R,UAAUhuB,QAAUrkB,EAClCgkD,EAAK3R,UAAUtvB,OAAO6d,KAItB3e,EAAM0C,KAAK2lB,QAAQroB,EAAM9hC,MAEzBorC,EAAG/F,KAAKiF,QAAQ4C,KAAK,oBAvBzB9B,EAAGxvB,QAAQigD,MAAM/5B,EAAM0C,KAAKxkC,MAAM,SAAAygD,GAChCrV,EAAGk2B,YAAYx/B,EAAM0C,MACjBic,GACFrV,EAAG8mB,UAAU5qD,IAAIm5C,GAInBrV,EAAG/F,KAAKiF,QAAQ4C,KAAK,wDA6BjBD,cACN6jB,EAAQrkE,KAAKw2E,gBAAgBh2B,OACxB,WACT62B,EAAKC,sBAAsBjT,KAC1B,iDAQiBA,SAAO9J,8DAAQ96D,KAE9B4kE,GAAUA,EAAMvH,kBAEfiT,EAAa/vE,KAAK+vE,WAAW73B,aAGjCmsB,EAAMtH,WADKt9D,MAAT86D,IACmBA,GAED8J,EAAMtH,eAGxBwa,EAAexH,EAAWhmE,IAAIs6D,EAAMlI,SACxCob,EAAaxa,WAAasH,EAAMtH,mBAE5Bya,EAAmBnT,EAAMvH,aACzB2a,EAAYD,EACTC,EAAUz6E,OAAS,GAAG,KACvBi+C,EAAUw8B,EACdA,EAAY,OACP,IAAI36E,EAAI,EAAGA,EAAIm+C,EAAQj+C,OAAQF,IAAK,KACnC46E,EAAO3H,EAAWhmE,IAAIkxC,EAAQn+C,IAC9B46E,EAAK5a,eACP2a,EAAY5G,GAAA4G,QAAAA,EAAiBC,EAAK5a,eAGlC2a,EAAUz6E,OAAS,IACrBw6E,EAAmB3G,GAAA2G,QAAAA,EAAwBC,QAG3C3a,EAAe/lB,KAAAg5B,EAAWhmE,IAAIytE,YAAsB,SAAUlC,UACnC71E,MAAvB61E,EAAYha,UACdga,EAAYha,SAAU,GAExBga,EAAYha,UAAYic,EAAaxa,WAC9BuY,KAGXvF,EAAW55B,OAAO06B,GAAA/T,QAAAA,EAAoBya,IAElCA,EAAaxa,YACfzkB,GAAKwV,gBAAgBuW,EAAMjmB,IAAI7V,MAAO,aACtC+P,GAAKuV,aAAawW,EAAMjmB,IAAI7V,MAAO,cAEnC+P,GAAKwV,gBAAgBuW,EAAMjmB,IAAI7V,MAAO,YACtC+P,GAAKuV,aAAawW,EAAMjmB,IAAI7V,MAAO,gEAQd87B,GACvBA,EAAMjmB,IAAI7V,MAAMovC,UAAUC,OAAO,yBACjCvT,EAAMjmB,IAAI4J,WAAW2vB,UAAUC,OAAO,mEAStBp3B,GACZxgD,KAAK89D,iBAAiBC,YAEtB/9D,KAAKmvB,QAAQwuC,cAAcC,aACxBE,iBAAiBuG,MAAQrkE,KAAKw2E,gBAAgBh2B,GAE/CxgD,KAAK89D,iBAAiBuG,QACxB7jB,EAAM8D,uBAEDwZ,iBAAiBC,YAAa,OAC9B8Z,yBAAyB73E,KAAK89D,iBAAiBuG,YAE/CvG,iBAAiBga,cAAgB93E,KAAK+vE,WAAW33B,OAAO,CAC3DwlB,MAAO59D,KAAKmvB,QAAQ6/C,oDAYfxuB,MACLxgD,KAAKmvB,QAAQwuC,cAAcC,OAAS59D,KAAK89D,iBAAiBuG,MAAO,CACjE7jB,EAAM8D,sBAENyrB,EAAa/vE,KAAK+vE,WAClB/vE,KAAK+vE,sBAAsBuE,KAC7BvE,EAAa/vE,KAAK+vE,WAAW73B,kBAGrBmsB,EAAQrkE,KAAKw2E,gBAAgBh2B,MAG/B6jB,GAASA,EAAMx7B,QAAU7oC,KAAK89D,iBAAiBuG,MAAMx7B,OAAQ,KACvDkvC,EAAY1T,EAAMvhB,IAAM9iD,KAAK89D,iBAAiBuG,MAAMvhB,IACpDV,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChD41B,EAAc3T,EAAMjmB,IAAI4J,WAAWpF,wBACnCq1B,EAAqBj4E,KAAK89D,iBAAiBuG,MAAMx7B,UACnDkvC,MAEIC,EAAYl1B,IAAMm1B,EAAqB71B,aAGxC,KACG81B,EAAoB7T,EAAMx7B,UAE5BmvC,EAAYl1B,IAAMo1B,EAAoBD,EAAqB71B,aAMnEiiB,GAASA,GAASrkE,KAAK89D,iBAAiBuG,MAAO,KACzC2T,EAAcjI,EAAWhmE,IAAIs6D,EAAMlI,SACnCgc,EAAepI,EAAWhmE,IAAI/J,KAAK89D,iBAAiBuG,MAAMlI,SAG5Dgc,GAAgBH,SACX7oD,QAAQy/C,eAAeuJ,EAAcH,EAAajI,GACvDA,EAAW55B,OAAOgiC,GAClBpI,EAAW55B,OAAO6hC,QAIhBI,EAAWrI,EAAW33B,OAAO,CACjCwlB,MAAO59D,KAAKmvB,QAAQ6/C,iBAKjB12B,GAAK68B,WAAWiD,EAAUp4E,KAAK89D,iBAAiBga,uBAC3CO,EAAYr4E,KAAK89D,iBAAiBga,cAClCQ,EAAYt4E,KAAK89D,iBAAiBuG,MAAMlI,QACxCoc,EAAY/2E,KAAKY,IAAIi2E,EAAUr7E,OAAQo7E,EAASp7E,QAClDw7E,EAAS,EACTtB,EAAY,EACZuB,EAAY,EACTD,EAASD,GAAW,MAEfC,EAAOtB,EAAaqB,GACpBC,EAAOC,EAAaF,GACrBH,EAASI,EAAOtB,IAAcmB,EAAUG,EAAOC,IAClDD,OAIAA,EAAOtB,GAAaqB,WAMpBH,EAASI,EAAOtB,IAAcoB,EAC9BpB,EAAY,OAIX,GAAImB,EAAUG,EAAOC,IAAcH,EACpCG,EAAY,MAKX,KACKC,EAAkB9kB,GAAAwkB,QAAAA,EAAiBC,EAAUG,EAAOC,IACpDE,EAAc5I,EAAWhmE,IAAIquE,EAASI,EAAOtB,IAC7C0B,EAAgB7I,EAAWhmE,IAAIsuE,EAAUG,EAAOC,SACjDtpD,QAAQy/C,eAAe+J,EAAaC,EAAe7I,GACxDA,EAAW55B,OAAOwiC,GAClB5I,EAAW55B,OAAOyiC,OAEZC,EAAgBT,EAASI,EAAOtB,GACtCkB,EAASI,EAAOtB,GAAamB,EAAUG,EAAOC,GAC9CL,EAASM,GAAmBG,EAE5BL,gDAeRh4B,WACTsd,iBAAiBC,YAAa,EAE/B/9D,KAAKmvB,QAAQwuC,cAAcC,OAAS59D,KAAK89D,iBAAiBuG,MAAO,CACnE7jB,EAAM8D,sBAGA3F,EAAK3+C,KACLozB,EAAKurB,EAAGmf,iBAAiBuG,MAAMlI,QAC/B2c,EAAUn6B,EAAGoxB,WAAW73B,aACxBu4B,EAAYn4B,GAAKh7C,OAAO,GAAIw7E,EAAQ/uE,IAAIqpB,IAC9CurB,EAAGxvB,QAAQugD,YAAYe,GAAW,SAAAA,MAC5BA,EAEFA,EAAUqI,EAAQphC,SAAWtkB,EAC7B0lD,EAAQ3iC,OAAOs6B,OAEZ,KAGG2H,EAAWU,EAAQ1gC,OAAO,CAC5BwlB,MAAOjf,EAAGxvB,QAAQ6/C,iBAIjB12B,GAAK68B,WAAWiD,EAAUz5B,EAAGmf,iBAAiBga,uBAC3CO,EAAY15B,EAAGmf,iBAAiBga,cAChCS,EAAY/2E,KAAKY,IAAIi2E,EAAUr7E,OAAQo7E,EAASp7E,QAClDw7E,EAAS,EACNA,EAASD,GAAW,MAElBC,EAASD,GAAaH,EAASI,IAAWH,EAAUG,IACzDA,OAIEA,GAAUD,YAMRG,EAAkB9kB,GAAAwkB,QAAAA,EAAiBC,EAAUG,IAC7CG,EAAcG,EAAQ/uE,IAAIquE,EAASI,IACnCI,EAAgBE,EAAQ/uE,IAAIsuE,EAAUG,IAC5C75B,EAAGxvB,QAAQy/C,eAAe+J,EAAaC,EAAeE,GACtDA,EAAQ3iC,OAAOwiC,GACfG,EAAQ3iC,OAAOyiC,OAETC,EAAgBT,EAASI,GAC/BJ,EAASI,GAAUH,EAAUG,GAC7BJ,EAASM,GAAmBG,EAE5BL,SAMR75B,EAAG/F,KAAKiF,QAAQ4C,KAAK,eAAgB,CAAE0b,QAAS/oC,SAC3CykD,yBAAyB73E,KAAK89D,iBAAiBuG,YAC/CvG,iBAAiBuG,MAAQ,4CASpB7jB,MACPxgD,KAAKmvB,QAAQ+0C,gBAEZ2R,EAAWr1B,EAAM4D,WAAa5D,EAAM4D,SAASyxB,SAAWr1B,EAAM4D,SAAS0xB,SACvEhpB,EAAWtM,EAAM4D,UAAY5D,EAAM4D,SAAS0I,YAC9C+oB,GAAW/oB,OACR0kB,mBAAmBhxB,YAIpBu4B,EAAe/4E,KAAKm2E,eAEpBp+B,EAAO/3C,KAAKw1E,eAAeh1B,GAC3BywB,EAAYl5B,GAAQA,EAAKmsB,WAAa,CAACnsB,EAAK3kB,IAAM,QACnD4lD,aAAa/H,OAEZgI,EAAej5E,KAAKm2E,gBAItB8C,EAAaj8E,OAAS,GAAK+7E,EAAa/7E,OAAS,SAC9C47C,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOyiC,EACPz4B,MAAAA,2CAUOA,OACLzI,EAAO/3C,KAAKw1E,eAAeh1B,MAC5BzI,GAIDA,IADY/3C,KAAKk5E,sBAAsB14B,QAMrCiP,EAAQ1X,EAAKohC,cACfn5E,KAAKmvB,QAAQygD,cAAgBngB,EAAO,CACpB,MAAdzvD,KAAKkxE,aACFA,MAAQ,IAAIpH,GAAM9pE,KAAK44C,KAAKwF,IAAI+C,KACjCnhD,KAAKmvB,QAAQm3C,QAAQyD,gBAAkB,cAGxCmH,MAAMkI,QAAQ3pB,OACbxD,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAM2B,QAAUu0B,EAAchiC,KAAOuX,EAAU6S,WAC/Cte,EAAM4B,QAAUs0B,EAAc5zB,IAAMmJ,EAAU4S,gBAE3Cwa,cAAcr5E,KAAKkxE,iBAInBoB,kBACa,MAAdtyE,KAAKkxE,YACFA,MAAMhhB,YAIVtX,KAAKiF,QAAQ4C,KAAK,WAAY,CACjC1I,KAAMA,EAAK3kB,GACXotB,MAAAA,yCAUQA,OACJzI,EAAO/3C,KAAKw1E,eAAeh1B,GAC5BzI,IAIDA,IADY/3C,KAAKk5E,sBAAsB14B,UAMtC8xB,kBACa,MAAdtyE,KAAKkxE,YACFA,MAAMhhB,YAGRtX,KAAKiF,QAAQ4C,KAAK,UAAW,CAChC1I,KAAMA,EAAK3kB,GACXotB,MAAAA,2CAUSA,MACExgD,KAAKw1E,eAAeh1B,KAGV,MAAnBxgD,KAAKmxE,iBAEFkI,cAAcr5E,KAAKkxE,OAGtBlxE,KAAKmvB,QAAQygD,cAAgB5vE,KAAKmvB,QAAQm3C,QAAQuJ,aAAe7vE,KAAKkxE,QAAUlxE,KAAKkxE,MAAM32B,QAAQ,KAC/F0R,EAAYjsD,KAAK44C,KAAKwF,IAAI3E,gBAC1Bi9B,EAAgBzqB,EAAUrJ,6BAC3BsuB,MAAMyF,YACTn2B,EAAM2B,QAAUu0B,EAAchiC,KAAOuX,EAAU6S,WAC/Cte,EAAM4B,QAAUs0B,EAAc5zB,IAAMmJ,EAAU4S,gBAE3CqS,MAAM1N,8CASDhjB,GACRxgD,KAAKslE,YAAYC,qBACdvnB,WAAWwC,yCASNzI,MACP/3C,KAAKmvB,QAAQ+0C,aACblkE,KAAKmvB,QAAQmhC,SAASkU,YAAexkE,KAAKmvB,QAAQmhC,SAAS+V,kBAE1D1nB,EAAK3+C,QAEP+3C,EAAM,KAEFic,EAAWrV,EAAG8mB,UAAU17D,IAAIguC,EAAK3kB,SAClCjE,QAAQkgD,SAASrb,GAAU,SAAAA,GAC1BA,GACFrV,EAAG8mB,UAAUtvB,OAAO6d,oDAYRxT,OACZzI,EAAO/3C,KAAKw1E,eAAeh1B,GAC3B2uB,EAAa3pC,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,cACpD9kC,QAAQ+/C,mBAAmBC,EAAYp3B,sCAQnCyI,MACJxgD,KAAKmvB,QAAQ+0C,YACblkE,KAAKmvB,QAAQmhC,SAASz1C,SAWvB4lB,EAEA64C,EAXE36B,EAAK3+C,KACL8wD,EAAO9wD,KAAKmvB,QAAQ2hC,MAAQ,KAG5BylB,EAAYv2E,KAAKo+C,IAAIuqB,MAAM/lB,wBAC3Bz4C,EAAInK,KAAKmvB,QAAQguB,IAAMo5B,EAAU5hC,MAAQ6L,EAAMxB,OAAO70C,EAAIq2C,EAAMxB,OAAO70C,EAAIosE,EAAU7hC,KACrFlU,EAAQxgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,GAC9BwxC,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UAIV,QAAdrQ,EAAMtpB,OACRoiD,EAAc9zC,KAAK4R,MAAMoJ,EAAMqT,aAAaI,QAAQ,UACxCpgC,QAAUylD,EAAYzlD,QAAUylD,EAAYzlD,QAAU,WAClEylD,EAAY94C,MAAQ84C,EAAY94C,MAAQ84C,EAAY94C,MAASswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EAC/F84C,EAAYpiD,KAAOoiD,EAAYpiD,MAAQ,MACvCoiD,EAAYt5E,KAAKylE,UAAUhuB,QAAU6hC,EAAYlmD,IAAMs3C,KAE/B,SAApB4O,EAAYpiD,MAAoBoiD,EAAY74C,MAC9CA,EAAMzgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,EAAInK,KAAKq1C,MAAMzM,MAAQ,GACnD0wC,EAAY74C,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,MAGpD64C,EAAc,CACZ94C,MAAOswB,EAAOA,EAAKtwB,EAAOmb,EAAO5X,GAAQvD,EACzC3M,QAAS,aAEC7zB,KAAKylE,UAAUhuB,QAAUizB,KAGX,UAAtB1qE,KAAKmvB,QAAQ+H,OACfuJ,EAAMzgC,KAAK44C,KAAKN,KAAK2D,OAAO9xC,EAAInK,KAAKq1C,MAAMzM,MAAQ,GACnD0wC,EAAY74C,IAAMqwB,EAAOA,EAAKrwB,EAAKkb,EAAO5X,GAAQtD,QAIhD4jC,EAAQrkE,KAAKw2E,gBAAgBh2B,GAC/B6jB,IACFiV,EAAYjV,MAAQA,EAAMlI,SAI5Bmd,EAAct5E,KAAK41E,eAAe0D,QAC7BnqD,QAAQigD,MAAMkK,GAAa,SAAAvhC,GAC1BA,IACF4G,EAAG8mB,UAAU5qD,IAAIk9B,GACC,QAAdyI,EAAMtpB,MACRynB,EAAGq6B,aAAa,CAACjhC,EAAK3kB,qDAYXotB,iBACZxgD,KAAKmvB,QAAQ+0C,gBAEZnsB,EAAO/3C,KAAKw1E,eAAeh1B,MAE7BzI,EAAM,KAGJk5B,EAAYjxE,KAAKmvB,QAAQ8/C,YACzBjvE,KAAKm2E,eACL,OAEa31B,EAAM4D,UAAY5D,EAAM4D,SAAS0I,WAAY,GAE7C9sD,KAAKmvB,QAAQ6gD,sBAAwBhwE,KAAKmvB,QAAQ8/C,YAAa,KAExEsK,EAAYv5E,KAAKylE,UAAU17D,IAAIguC,EAAK3kB,IAAIixC,MAG1CmV,OAAoB/5E,EACpBO,KAAKmvB,QAAQsqD,qBACXxI,EAAUj0E,OAAS,IACrBw8E,EAAoBx5E,KAAKylE,UAAU17D,IAAIknE,EAAU,IAAI5M,OAKpDrkE,KAAKmvB,QAAQsqD,qBAA4Ch6E,MAArB+5E,GAAkCA,GAAqBD,GAC9FtI,EAAUh0E,KAAK86C,EAAK3kB,QAEhBsmB,EAAQi1B,EAAQ+K,cAAc15E,KAAKylE,UAAU17D,IAAIknE,QAElDjxE,KAAKmvB,QAAQsqD,qBAAuBD,GAAqBD,MAGvD,IAAMnmD,KADX69C,EAAY,GACKjxE,KAAKw2C,SAChBx2C,KAAKw2C,MAAMn5C,eAAe+1B,GAAK,KAC3BumD,EAAQ35E,KAAKw2C,MAAMpjB,GACnBoN,EAAQm5C,EAAMpmE,KAAKitB,MACnBC,OAA0BhhC,IAAnBk6E,EAAMpmE,KAAKktB,IAAqBk5C,EAAMpmE,KAAKktB,IAAMD,IAE1DA,GAASkZ,EAAMt3C,KACfq+B,GAAOiZ,EAAMp0C,MACXtF,KAAKmvB,QAAQsqD,qBAAuBD,GAAqBx5E,KAAKylE,UAAU17D,IAAI4vE,EAAMvmD,IAAIixC,OACtFsV,aAAiB/P,IACrBqH,EAAUh0E,KAAK08E,EAAMvmD,SAM1B,KAEGxO,EAAQgvC,GAAAqd,QAAAA,EAAkBl5B,EAAK3kB,KACvB,GAAVxO,EAEFqsD,EAAUh0E,KAAK86C,EAAK3kB,OAIpB69C,QAAAA,EAAiBrsD,EAAO,OAItBg1D,EAAoB9iC,GAAAm6B,QAAAA,GAAiB,SAAAl5B,UAAQ8hC,EAAKnH,YAAY36B,GAAMmsB,mBAErE8U,aAAaY,QAEbhhC,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOx2C,KAAKm2E,eACZ31B,MAAAA,8CA4CU5Y,WACVkyC,EAAMlyC,EACHkyC,GAAK,IACNA,EAAIz8E,eAAe,mBACdy8E,EAAI,YAEbA,EAAMA,EAAIvyC,kBAGL,4CASMiZ,UACNxgD,KAAK+5E,gBAAgBv5B,EAAM7wB,sDASd6wB,UACbxgD,KAAK+5E,gBAAgBv5B,EAAMw5B,uDASpBx5B,OACR4B,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAClD4uB,EAAWhxE,KAAKgxE,SAEhBA,EAASh0E,QAAU,GAAKgD,KAAK+vE,aAC/BiB,EAAWhxE,KAAK+vE,WAAW33B,OAAO,CAChCwlB,MAAO59D,KAAKmvB,QAAQ6/C,kBAInB,IAAIlyE,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KAClCq/D,EAAU6U,EAASl0E,GACnBunE,EAAQrkE,KAAK2sE,OAAOxQ,GACpBnU,EAAaqc,EAAMjmB,IAAI4J,WACvBiyB,EAAiBjyB,EAAWpF,2BAC9BR,GAAW63B,EAAen3B,KAAOV,EAAU63B,EAAen3B,IAAMkF,EAAW0O,oBACtE2N,KAG6B,QAAlCrkE,KAAKmvB,QAAQk5B,YAAYtQ,SACvBj7C,IAAMkD,KAAKgxE,SAASh0E,OAAS,GAAKolD,EAAU63B,EAAen3B,WACtDuhB,UAIC,IAANvnE,GAAWslD,EAAU63B,EAAen3B,IAAMkF,EAAWxrC,cAChD6nD,SAKN,4CA8BMrQ,EAAU98B,OACjB9Z,EAAQk7B,GAAKh7C,OAAO,GAAI02D,UAEzB98B,IAEHA,EAAOl3B,KAAKylE,UAAUvuC,MAGLz3B,MAAf2d,EAAMojB,QACRpjB,EAAMojB,MAAQ8X,GAAKpB,QAAQ95B,EAAMojB,MAAOtJ,GAAQA,EAAKsJ,OAAS,SAE/C/gC,MAAb2d,EAAMqjB,MACRrjB,EAAMqjB,IAAM6X,GAAKpB,QAAQ95B,EAAMqjB,IAAMvJ,GAAQA,EAAKuJ,KAAO,SAGpDrjB,6CASFpd,KAAKmvB,QAAQ2wC,aAIVnkB,EAAU37C,KAAK44C,KAAKc,MAAMgC,WAAW17C,KAAK44C,KAAKY,SAASwF,OAAOpW,OAA/D+S,MACF4xB,EAAWvtE,KAAKowE,iBAAiB8J,YAAYl6E,KAAKutE,SAAU5xB,EAAO37C,KAAKmvB,QAAQ2wC,YAElF9/D,KAAKutE,UAAYA,EAAU,SACxB8E,qBAED9E,EAAU,2CACQA,+BAAU,SACpBxC,iGAELwC,SAAWA,OAGb4M,gBAAgB5M,qDASnBvtE,KAAKmvB,QAAQ2wC,SACX9/D,KAAKutE,UAAYvtE,KAAKutE,SAASvwE,OAAQ,2CACrBgD,KAAKutE,sCAAU,SACzBzC,sIAWAyC,MACVvtE,KAAKutE,UAAYvtE,KAAKutE,SAASvwE,OAAQ,OACnCo9E,EAAiB,OAAQrjC,GAAAw2B,QAAAA,GAAa,SAAAzN,UAAWA,EAAQ1sC,OACzDinD,EAAqBvjC,UAAKy2B,kBAAgB,SAAAzN,UAAYsa,EAAetsD,IAAIgyC,EAAQ1sC,OACnFknD,GAAmB,wCACHD,+BAAoB,SAA/Bva,UACDya,EAAc3mB,UAAKqd,kBAAkBnR,EAAQ1sC,QAC9B,IAAjBmnD,EACFza,EAAQ6S,qBACH1B,kBAAiBsJ,EAAa,GACnCD,GAAmB,wFAInBA,EAAkB,KACdrB,EAAej5E,KAAKm2E,oBACrBv9B,KAAKiF,QAAQ4C,KAAK,SAAU,CAC/BjK,MAAOyiC,EACPz4B,MAAOA,cAKR+sB,SAAWA,GAAY,2CA3NT9H,OACfngE,EAAM,KACNlD,EAAM,eAEVqjE,QAAAA,GAAkB,SAAAlyD,IACL,MAAPnR,GAAemR,EAAKitB,MAAQp+B,KAC9BA,EAAMmR,EAAKitB,OAGG/gC,MAAZ8T,EAAKktB,KACI,MAAPn7B,GAAeiO,EAAKktB,IAAMn7B,KAC5BA,EAAMiO,EAAKktB,MAIF,MAAPn7B,GAAeiO,EAAKitB,MAAQl7B,KAC9BA,EAAMiO,EAAKitB,UAKV,CACLp+B,IAAAA,EACAkD,IAAAA,6CAwFqBk7C,WACnB7wB,EAAS6wB,EAAM7wB,OACZA,GAAQ,IACTA,EAAOtyB,eAAe,sBACjBsyB,EAAO,eAEhBA,EAASA,EAAO4X,kBAGX,YA9/EWoR,IAsmFtBg2B,GAAQgG,MAAQ,CACd/rB,WAAYghB,GACZ/E,IAAK0B,GACL7sB,MAAOgvB,GACP5oC,MAAOsoC,IAQTuG,GAAQxyE,UAAUg0E,OAASxB,GAAQxyE,UAAUk0E,UChpF7C,IACImK,GADAC,IAAa,EAEbC,GAAa,sCAIXC,uFAeYxrD,EAASyrD,EAAkBC,GACzCJ,IAAa,EACbD,GAAaI,MACTE,EAAcF,cACAn7E,IAAdo7E,IACFC,EAAcF,EAAiBC,IAEjCF,EAAUvjC,MAAMjoB,EAAS2rD,EAAa,IAC/BL,iCAWItrD,EAASyrD,EAAkB3qD,OACjC,IAAIqmC,KAAUnnC,EACbA,EAAQ9xB,eAAei5D,IACzBqkB,EAAUxwD,MAAMmsC,EAAQnnC,EAASyrD,EAAkB3qD,iCAc5CqmC,EAAQnnC,EAASyrD,EAAkB3qD,WACbxwB,IAA7Bm7E,EAAiBtkB,SAAsD72D,IAA7Bm7E,EAAiBG,aAK3DC,EAAkB1kB,EAClB2kB,GAAY,OAEiBx7E,IAA7Bm7E,EAAiBtkB,SAAsD72D,IAA7Bm7E,EAAiBG,UAM7DC,EAAkB,UAIlBC,EAAoD,WAAvCN,EAAUrjC,QAAQnoB,EAAQmnC,SAQrC4kB,EAAeN,EAAiBI,GAChCC,QAAuCx7E,IAA1By7E,EAAaC,WAC5BD,EAAeA,EAAaC,UAG9BR,EAAUS,YAAY9kB,EAAQnnC,EAASyrD,EAAkBI,EAAiBE,EAAcjrD,QA9BtF0qD,EAAUU,cAAc/kB,EAAQskB,EAAkB3qD,uCA2CnCqmC,EAAQnnC,EAASyrD,EAAkBI,EAAiBE,EAAcjrD,OAC/E+pB,EAAM,SAASshC,GACjB34E,QAAQq3C,IAAI,KAAOshC,EAAUX,EAAUY,cAActrD,EAAMqmC,GAASokB,KAGlEc,EAAab,EAAUrjC,QAAQnoB,EAAQmnC,IACvCmlB,EAAgBP,EAAaM,QAEX/7E,IAAlBg8E,EAEuC,UAArCd,EAAUrjC,QAAQmkC,KAA0E,IAA5C7nB,GAAA6nB,QAAAA,EAAsBtsD,EAAQmnC,KAChFtc,EAAI,+BAAiCsc,EAAS,yBACnBqkB,EAAUe,MAAMD,GACzC,SAAWtsD,EAAQmnC,GAAU,OAC/BmkB,IAAa,GAES,WAAfe,GAA+C,YAApBR,IAClC/qD,EAAOqoB,GAAKqjC,mBAAmB1rD,EAAMqmC,GACrCqkB,EAAUvjC,MAAMjoB,EAAQmnC,GAASskB,EAAiBI,GAAkB/qD,SAGvCxwB,IAAxBy7E,EAAY,MAEnBlhC,EAAI,8BAAgCsc,EAClC,gBAAkBqkB,EAAUe,MAAM17B,GAAYk7B,IAC9C,eAAkBM,EAAa,MAAQrsD,EAAQmnC,GAAU,KAC3DmkB,IAAa,mCAUF3rD,OACToI,KAAcpI,SAEL,WAAToI,EACa,OAAXpI,EACK,OAELA,aAAkBmT,QACb,UAELnT,aAAkBuoB,OACb,SAELvoB,aAAkBL,OACb,SAELyU,GAAcpU,GACT,QAELA,aAAkBpyB,KACb,YAEe+C,IAApBqvB,EAAO8sD,SACF,OAEuB,IAA5B9sD,EAAOtuB,iBACF,SAEF,SAES,WAAT02B,EACA,SAES,YAATA,EACA,UAES,WAATA,EACA,cAESz3B,IAATy3B,EACA,YAEFA,wCASYo/B,EAAQnnC,EAASc,OAOhCxtB,EANAo5E,EAAclB,EAAUmB,cAAcxlB,EAAOnnC,EAAQc,GAAK,GAC1D8rD,EAAepB,EAAUmB,cAAcxlB,EAAOkkB,GAAW,IAAG,GAO9D/3E,OAD6BhD,IAA3Bo8E,EAAYG,WACR,OAASrB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,EAAO,IAC9D,6CAA+CulB,EAAYG,WAAa,SAEnED,EAAa5lB,UAPM,GAO+B0lB,EAAY1lB,SAAW4lB,EAAa5lB,SACvF,OAASwkB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,EAAO,IAC9D,uDACAqkB,EAAUY,cAAcQ,EAAa9rD,KAAM8rD,EAAaE,aAAa,IAEhEJ,EAAY1lB,UAbM,EAcnB,mBAAqB0lB,EAAYI,aAAe,KACpDtB,EAAUY,cAAcM,EAAY5rD,KAAMqmC,GAGtC,gCAAkCqkB,EAAUe,MAAM17B,GAAY7wB,IACpEwrD,EAAUY,cAActrD,EAAMqmC,GAGhC3zD,QAAQq3C,IAAI,+BAAiCsc,EAAS,IAAM7zD,EAAKi4E,IACjED,IAAa,wCAYMnkB,EAAQnnC,EAASc,OAAMisD,0DACtC95E,EAAM,IACN65E,EAAe,GACfE,EAAmB,GACnBC,EAAkB9lB,EAAOhyD,cACzB03E,OAAav8E,MACZ,IAAI48E,KAAMltD,EAAS,KAClBgnC,iBACyB12D,IAAzB0vB,EAAQktD,GAAIlB,WAAwC,IAAde,EAAoB,KACxD3jE,EAASoiE,EAAUmB,cAAcxlB,EAAQnnC,EAAQktD,GAAK/jC,GAAKqjC,mBAAmB1rD,EAAKosD,IACnFj6E,EAAMmW,EAAO49C,WACf8lB,EAAe1jE,EAAO0jE,aACtBE,EAAmB5jE,EAAO0X,KAC1B7tB,EAAMmW,EAAO49C,SACb6lB,EAAazjE,EAAOyjE,gBAGnB,QACgD,IAA/CpoB,KAAAyoB,EAAG/3E,sBAAsB83E,KAC3BJ,EAAaK,GAGXj6E,GADJ+zD,EAAWwkB,EAAU2B,oBAAoBhmB,EAAQ+lB,MAE/CJ,EAAeI,EACfF,EAAmB7jC,GAAKikC,UAAUtsD,GAClC7tB,EAAM+zD,UAIL,CAAC8lB,aAAaA,EAAchsD,KAAKksD,EAAkBhmB,SAAS/zD,EAAK45E,WAAYA,yCAUjE/rD,EAAMqmC,WAAQpzC,yDAAS,6BACtC2I,EAAM,OAAS3I,EAAS,gBACnBpmB,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IAAK,KAC/B,IAAIiX,EAAI,EAAGA,EAAIjX,EAAI,EAAGiX,IACzB8X,GAAO,KAETA,GAAOoE,EAAKnzB,GAAK,YAEd,IAAIiX,EAAI,EAAGA,EAAIkc,EAAKjzB,OAAS,EAAG+W,IACnC8X,GAAO,KAETA,GAAOyqC,EAAS,SACX,IAAIx5D,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAS,EAAGF,IAAK,KACnC,IAAIiX,EAAI,EAAGA,EAAIkc,EAAKjzB,OAASF,EAAGiX,IACnC8X,GAAO,KAETA,GAAO,aAEFA,EAAM,qCAQFsD,UACJ4wB,GAAe5wB,GAAS5oB,QAAQ,gCAAiC,IAAIA,QAAQ,QAAS,kDAqBpEpJ,EAAGC,MACX,IAAbD,EAAEH,OAAc,OAAOI,EAAEJ,UACZ,IAAbI,EAAEJ,OAAc,OAAOG,EAAEH,WAKzBF,EAMAiX,EATAyoE,EAAS,OAIR1/E,EAAI,EAAGA,GAAKM,EAAEJ,OAAQF,IACzB0/E,EAAO1/E,GAAK,CAACA,OAKViX,EAAI,EAAGA,GAAK5W,EAAEH,OAAQ+W,IACzByoE,EAAO,GAAGzoE,GAAKA,MAIZjX,EAAI,EAAGA,GAAKM,EAAEJ,OAAQF,QACpBiX,EAAI,EAAGA,GAAK5W,EAAEH,OAAQ+W,IACrB3W,EAAEopB,OAAO1pB,EAAI,IAAMK,EAAEqpB,OAAOzS,EAAI,GAClCyoE,EAAO1/E,GAAGiX,GAAKyoE,EAAO1/E,EAAI,GAAGiX,EAAI,GAEjCyoE,EAAO1/E,GAAGiX,GAAKvS,KAAKY,IAAIo6E,EAAO1/E,EAAI,GAAGiX,EAAI,GAAK,EAC7CvS,KAAKY,IAAIo6E,EAAO1/E,GAAGiX,EAAI,GAAK,EAC1ByoE,EAAO1/E,EAAI,GAAGiX,GAAK,WAKtByoE,EAAOp/E,EAAEJ,QAAQG,EAAEH,iBCrV1Bia,GAAS,SACTwlE,GAAO,UACPl7E,GAAS,SAGTutB,GAAS,SAKT0rD,GAAa,CACfxlB,UAAW,CACT0nB,QAAS,SAAaD,IACtBxkD,OAAQ,SAAawkD,YAAiB,YACtCxwB,UAAW,CAAC7N,IARN,OASN+8B,SAAU,CAACrsD,OAAAA,WAAmB2tD,YAAiB,aAIjDhW,MAAO,CAACxvD,OAAAA,IACR2hD,iBAAkB,CAAC3hD,OAAAA,aAAqB,aACxCkmC,IAAK,SAAas/B,aAAmB,aACrCj/B,YAAa,CACXC,OAAQ,SAAag/B,IACrBjgE,OAAQ,CAACjb,OAAAA,aAAoB,aAC7B45E,SAAU,CAACrsD,OAAAA,KAEbkwC,UAAW,CACTK,UAAW,CAAC99D,OAAAA,IACZyE,SAAU,UAAa,YACvBm1E,SAAU,CAACrsD,OAAAA,KAEbikC,eAAgB,SAAa0pB,aAAmB,aAChDzpB,iBAAkB,SAAaypB,aAAmB,aAClD5kB,WAAY,SAAa4kB,IACzBE,eAAgB,CAACp7E,OAAAA,IACjBmzD,WAAY,SAAa+nB,IACzBxW,eAAgB,CAAChvD,OAAAA,GAAQ5Q,MAlCf,SAmCViqD,SAAU,CACRz1C,IAAK,SAAa4hE,aAAmB,aACrCxmC,OAAQ,SAAawmC,aAAmB,aACxCpW,YAAa,SAAaoW,aAAmB,aAC7CjY,WAAY,SAAaiY,aAAmB,aAC5C1X,cAAe,SAAa0X,aAAmB,aAC/CtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B2R,IAAK,CAACl/B,OAAAA,GAAQ0I,KA1CL,OA0CWgN,OAAAA,GAAQmT,OAvCjB,UAwCX3sB,OAAQ,CACNkpD,YAAa,CACXrsC,YAAa,CAACrD,OAAAA,aAAoB,aAClCoD,OAAQ,CAACpD,OAAAA,aAAoB,aAC7BmD,OAAQ,CAACnD,OAAAA,aAAoB,aAC7BqC,KAAM,CAACrC,OAAAA,aAAoB,aAC3BzJ,QAAS,CAACyJ,OAAAA,aAAoB,aAC9B7H,IAAK,CAAC6H,OAAAA,aAAoB,aAC1B1J,KAAM,CAAC0J,OAAAA,aAAoB,aAC3BjN,MAAO,CAACiN,OAAAA,aAAoB,aAC5B5N,KAAM,CAAC4N,OAAAA,aAAoB,aAC3BkkE,SAAU,CAACrsD,OAAAA,YAAoB,aAEjC83B,YAAa,CACXtsC,YAAa,CAACrD,OAAAA,aAAoB,aAClCoD,OAAQ,CAACpD,OAAAA,aAAoB,aAC7BmD,OAAQ,CAACnD,OAAAA,aAAoB,aAC7BqC,KAAM,CAACrC,OAAAA,aAAoB,aAC3BzJ,QAAS,CAACyJ,OAAAA,aAAoB,aAC9B7H,IAAK,CAAC6H,OAAAA,aAAoB,aAC1B1J,KAAM,CAAC0J,OAAAA,aAAoB,aAC3BjN,MAAO,CAACiN,OAAAA,aAAoB,aAC5B5N,KAAM,CAAC4N,OAAAA,aAAoB,aAC3BkkE,SAAU,CAACrsD,OAAAA,YAAoB,aAEjCqsD,SAAU,CAACrsD,OAAAA,KAEb1E,OAAQ,UAAa,YACrB6yC,gBAAiB,CAAChmD,OAAAA,IAClB+3D,WAAY,CAAC/3D,OAAAA,YAAoB,YACjC0mD,cAAe,CACb9iD,IAAK,SAAa4hE,aAAmB,aACrCxmC,OAAQ,SAAawmC,aAAmB,aACxC7e,MAAO,SAAa6e,aAAmB,aACvCtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B8/C,eAAgB,UAAa,YAC7B/lC,OAAQ,CAAC5xB,OAAAA,GAAQ1V,OAAAA,IACjB63C,YAAa,CACX5Y,MAAO,CAACv2B,KAlFD,OAkFO1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OA/ErB,UAgFTqW,IAAK,CAACx2B,KAnFC,OAmFK1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAhFnB,UAiFT4uB,OAAQ,CAAC/hC,OAAAA,IACTkkE,SAAU,CAACrsD,OAAAA,GAAQzoB,MAtFX,UAwFVmjE,qBAAsB,CACpBzxB,KAAM,SAAa0kC,aAAmB,aACtC/iC,MAAO,SAAa+iC,aAAmB,aACvCtB,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/Bq6C,UAAW,SAAYsT,IACvB/+E,OAAO,CAACuZ,OAAAA,IACRrE,QAAQ,CACNmoE,QAAS,CAAC6B,IA3FJ,OA4FNzB,SAAU,CAACrsD,OAAAA,KAEb0oC,oBAAqB,CAACj2D,OAAAA,IACtB+4D,OAAQ,CACNhS,KAAM,CAAC/mD,OAAAA,IACPw2C,KAAM,CACJ+jB,WAAY,CAACv6D,OAAAA,aAAoB,aACjCs5D,SAAU,CAACt5D,OAAAA,aAAoB,aAC/B45E,SAAU,CAACrsD,OAAAA,GAAOvtB,OAAAA,KAEpB45E,SAAU,CAACrsD,OAAAA,GAAOvtB,OAAAA,KAEpB+D,IAAK,CAAC2E,KA5GG,OA4GG1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAzGjB,UA0GXisC,UAAW,CAAC90D,OAAAA,GAAQ0V,OAAAA,IACpBuxC,cAAe,CAACjnD,OAAAA,IAChBa,IAAK,CAAC6H,KA/GG,OA+GG1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OA5GjB,UA6GXosC,UAAW,CAACj1D,OAAAA,GAAQ0V,OAAAA,IACpBmmC,SAAU,SAAaq/B,IACvBxN,YAAa,SAAawN,IAC1BhD,oBAAqB,SAAagD,IAClCrN,MAAO,UAAa,YACpBF,mBAAoB,UAAa,YACjCG,SAAU,UAAa,YACvBC,OAAQ,UAAa,YACrBE,SAAU,UAAa,YACvBD,SAAU,UAAa,YACvBE,WAAY,UAAa,YACzBC,YAAa,UAAa,YAC1BC,cAAe,UAAa,YAC5BkN,sBAAuB,UAAa,YACpCjf,MAAO,UAAa,YACpBvV,YAAa,CACXC,KAAM,CAACrxC,OAAAA,aAAoB,aAC3B8gC,KAAM,CAAC9gC,OAAAA,aAAoB,aAC3BkkE,SAAU,CAAClkE,OAAAA,GAAQ6X,OAAAA,KAErBo1C,WAAY,SAAauY,IACzBzM,oBAAqB,SAAayM,IAClC9jB,gBAAiB,SAAa8jB,IAC9Bx2B,gBAAiB,SAAaw2B,IAC9Bl0B,gBAAiB,SAAak0B,IAC9Bn2B,cAAe,SAAam2B,IAC5Bp5E,MAAO,SAAao5E,IACpBlhB,eAAgB,SAAakhB,IAC7B3c,QAAS,CACPiN,SAAU,QAAWxrE,aAAqB,aAC1C+sE,cAAe,QAAWr3D,aAAqB,aAC/C+1D,gBAAiB,UAAc,qBAAyB,aACxD/B,WAAY,SAAYwR,aAAmB,aAC3CjS,iBAAkB,SAAYiS,aAAmB,aACjDtB,SAAU,SAAYsB,GAAM3tD,OAAAA,KAE9BgiC,KAAM,UAAa,gBAAoB,QACvCtwB,MAAO,CAACv2B,KArJC,OAqJK1I,OAAAA,GAAQ0V,OAAAA,GAAQmT,OAlJnB,UAmJXo7C,SAAU,UAAa,YACvBsX,sBAAuB,UAAa,YACpC50C,cAAe,UAAa,YAC5B69B,qBAAsB,CAAC9uD,OAAAA,YAAoB,YAC3C24D,aAAc,SAAa6M,IAC3BnW,QAAS,CACPuJ,YAAa,SAAa4M,IAC1B1S,eAAgB,QAAY,CAAC,MAAO,OAAQ,SAC5C+F,MAAO,CAACvuE,OAAAA,IACRikE,SAAU,UAAa,YACvB2V,SAAU,CAACrsD,OAAAA,KAEbs2C,wBAAyB,CACvBI,SAAU,UAAa,YACvB2V,SAAU,SAAasB,GAAM3tD,OAAAA,KAE/B25B,SAAU,CACR9M,MAAO,CAAC1kC,OAAAA,aAAoB,aAC5B8sB,KAAM,CAACxiC,OAAAA,aAAoB,aAC3B45E,SAAU,CAACrsD,OAAAA,KAEboI,KAAM,CAACjgB,OAAAA,IACP2xB,MAAO,CAAC3xB,OAAAA,GAAQ1V,OAAAA,IAChBuxD,WAAY,SAAa2pB,IACzBp/B,SAAU,SAAao/B,IACvB36B,QAAS,CAAC7qC,OAAQ,CAAC,UAAW,SAAU,WAAY,UAAW,KAC/D+qC,aAAc,CAACzgD,OAAAA,IACfg8C,QAAS,CAACh8C,OAAAA,IACV+7C,QAAS,CAAC/7C,OAAAA,IAEV45E,SAAU,CAACrsD,OAAAA,KAGTiuD,GAAmB,CACrB7wD,OAAQ,CACNu6C,MAAQ,CAAC,SAAU,OAAQ,SAC3B7N,iBAAkB,CAAC,OAAQ,OAAQ,QAAS,UAAW,OAAQ,UAAW,MAAO,OAAQ,OAAQ,SAAU,UAC3Gt6C,WAAY,EACZu5C,YAAY,EACZnD,YAAY,EAEVpE,SAAU,CACVz1C,KAAK,EACLo7B,QAAQ,EACRowB,aAAa,EACb7B,YAAY,GAEd/jC,IAAK,GACLhjC,OAAQ,CACNkpD,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZX,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZX,KAAY,KAGhB4zD,gBAAiB,CAAC,OAAQ,QAAS,YAEnC+f,iBAAiB,EACjBn0C,OAAQ,GAERnrC,OAAQ,GACR85D,oBAAqB,IACrB8C,OAAQ,CACNhS,KAAM,CAAC,GAAI,EAAG,IAAK,GACnBvQ,KAAM,CACJ+jB,WAAY,CAAC,GAAI,EAAG,IAAK,GACzBjB,SAAU,CAAC,GAAI,EAAG,IAAK,KAG3Bv1D,IAAK,GACL+wD,UAAW,GACX7N,cAAe,CAAC,EAAG,EAAG,GAAI,GAC1BpmD,IAAK,GACLo0D,UAAW,GACXpZ,UAAU,EACV6xB,aAAa,EACbwK,qBAAqB,EAOrBpxB,YAAa,CACXC,KAAM,CAAC,OAAQ,SAAU,OACzBvQ,KAAM,CAAC,SAAU,QAEnB+a,YAAY,EACZoR,YAAY,EACZvL,iBAAiB,EACjB1S,iBAAiB,EACjBsC,iBAAiB,EACjBllD,OAAO,EACPk4D,gBAAgB,EAChBuE,SAAS,EAETt/B,MAAO,GAMPovC,cAAc,EACdtJ,QAAS,CACPuJ,aAAa,EACb9F,eAAgB,OAChB+F,MAAO,CAAC,IAAK,EAAG,MAAO,MAEzB1K,yBAAyB,EACzBluC,KAAM,CAAC,MAAO,QAAS,QAAS,cAChC0R,MAAO,OACPyU,UAAU,EACVyE,QAAS,CAAC,UAAW,SAAU,WAAY,UAAW,IACtDvE,QAAS,CAAC,SAAiB,GAAI,SAAiB,GAChDD,QAAS,CAAC,GAAI,GAAI,SAAiB,OC9RrC,CAAE3tB,OAAQ,QAASnP,OAAO,GAAQ,CAClCy8D,KCAe,SAAcn7E,WACzB2rB,EAAI9K,GAAS3iB,MACbhD,EAASq0B,EAAS5D,EAAEzwB,QACpBgnC,EAAkBloC,UAAUkB,OAC5B4nB,EAAQ0M,GAAgB0S,EAAkB,EAAIloC,UAAU,QAAK2D,EAAWzC,GACxEyjC,EAAMuD,EAAkB,EAAIloC,UAAU,QAAK2D,EAC3Cy9E,OAAiBz9E,IAARghC,EAAoBzjC,EAASs0B,GAAgBmP,EAAKzjC,GACxDkgF,EAASt4D,GAAO6I,EAAE7I,KAAW9iB,SAC7B2rB,KCZT,OAAiBoT,GAAa,SAASo8C,KCDnCx7C,GAAiBxlC,MAAME,aAEV,SAAU2vB,OACrBiV,EAAMjV,EAAGmxD,YACNnxD,IAAO2V,IAAmB3V,aAAc7vB,OAAS8kC,IAAQU,GAAew7C,KAAQA,GAAOl8C,GCD5Fo8C,GAAa,CAACC,MAAO,UAAWC,KAAM,UAAWC,SAAU,UAAWC,WAAY,UAAWC,KAAM,UAAWC,UAAW,UAAWC,MAAO,UAAWC,KAAM,UAAWC,SAAU,UAAWC,YAAa,UAAWC,cAAe,UAAWC,kBAAmB,UAAWC,KAAM,UAAWC,YAAa,UAAWC,KAAM,UAAWC,KAAM,UAAWC,aAAc,UAAWC,WAAY,UAAWC,cAAe,UAAWC,YAAa,UAAWC,SAAU,UAAWC,cAAe,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,UAAW,UAAWC,UAAW,UAAWC,UAAW,UAAWC,cAAe,UAAWC,gBAAiB,UAAWC,OAAQ,UAAWC,eAAgB,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,iBAAkB,UAAWC,QAAS,UAAWC,UAAW,UAAWC,UAAW,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,gBAAiB,UAAWC,UAAW,UAAWC,WAAY,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,OAAQ,UAAWC,MAAO,UAAWC,KAAM,UAAWC,QAAS,UAAWC,aAAc,UAAWC,WAAY,UAAWC,QAAS,UAAWC,YAAa,UAAWC,YAAa,UAAWC,aAAc,UAAWC,WAAY,UAAWC,aAAc,UAAWC,WAAY,UAAWC,UAAW,UAAWC,WAAY,UAAWC,YAAa,UAAWC,OAAQ,UAAWC,MAAO,UAAWC,SAAU,UAAWC,UAAW,UAAWC,YAAa,UAAWC,cAAe,UAAWC,eAAgB,UAAWC,WAAY,UAAWC,UAAW,UAAWC,cAAe,UAAWC,aAAc,UAAWC,UAAW,UAAWC,UAAW,UAAWC,OAAQ,UAAWC,gBAAiB,UAAWC,UAAW,UAAWC,KAAM,UAAWC,UAAW,UAAWC,IAAK,UAAWC,UAAW,UAAWC,cAAe,UAAWC,QAAS,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,QAAS,UAAWC,UAAW,UAAWC,KAAM,UAAWC,UAAW,UAAWC,UAAW,UAAWC,SAAU,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,cAAe,UAAWC,WAAY,UAAWC,MAAO,UAAWC,UAAW,UAAWC,SAAU,UAAWC,MAAO,UAAWC,WAAY,UAAWC,MAAO,UAAWC,MAAO,UAAWC,WAAY,UAAWC,UAAW,UAAWC,WAAY,UAAWC,OAAQ,UAAWC,aAAc,UAAWC,MAAO,UAAWC,qBAAsB,UAAWC,QAAS,UAAWC,IAAK,UAAWC,QAAS,UAAWC,QAAS,UAAWC,SAAU,UAAWC,UAAW,UAAWC,OAAQ,UAAWC,QAAS,UAAWC,MAAO,UAAWC,WAAY,UAAWC,YAAa,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,KAAM,UAAWC,KAAM,UAAWC,UAAW,UAAWC,YAAa,UAAWC,SAAU,UAAWC,OAAQ,UAAWC,UAAW,UAAWC,eAAgB,UAAWC,WAAY,UAAWC,cAAe,UAAWC,SAAU,UAAWC,SAAU,UAAWC,aAAc,UAAWC,YAAa,UAAWC,KAAM,UAAWC,OAAQ,UAAWC,YAAa,UAAWC,MAAO,UAAWC,MAAO,WAK1/FC,+BAIQC,yDAAa,kBAClBA,WAAaA,OACbC,WAAY,OACZC,kBAAoB,CAACh8E,EAAE,MAAOZ,EAAE,YAChC68E,EAAI,IAAM,SACVC,MAAQ,CAACD,EAAE,IAAIE,EAAE,IAAIlpF,EAAE,IAAID,EAAE,QAC7BopF,eAAY9mF,OACZ+mF,aAAe,CAACJ,EAAE,IAAIE,EAAE,IAAIlpF,EAAE,IAAID,EAAE,QACpCspF,mBAAehnF,OACfinF,SAAU,OAGVC,eAAiB,kBACjBC,cAAgB,kBAGhBl+B,sDAQEuD,QACaxsD,IAAhBO,KAAK6jD,cACFA,OAAOkB,eACPlB,YAASpkD,QAEXwsD,UAAYA,OACZA,UAAUt3B,YAAY30B,KAAK2oE,YAC3Bke,mBAEAC,qDAOW9gF,MACQ,mBAAbA,QAIH,IAAI5C,MAAM,oFAHXujF,eAAiB3gF,2CAWTA,MACS,mBAAbA,QAIH,IAAI5C,MAAM,qFAHXwjF,cAAgB5gF,yCAaVqgF,MACQ,iBAAVA,SACFlJ,GAAWkJ,oCAiBbA,OAAOU,gEACA,SAAVV,OAIAW,EAGAC,EAAYjnF,KAAKknF,eAAeb,WAClB5mF,IAAdwnF,IACFZ,EAAQY,IAImB,IAAzB3uC,GAAKnB,SAASkvC,QACe,IAA3B/tC,GAAK6uC,WAAWd,GAAiB,KAC/Be,EAAYf,EAAM9gF,OAAO,GAAGA,OAAO,EAAG8gF,EAAMrpF,OAAS,GAAG6N,MAAM,KAClEm8E,EAAO,CAACZ,EAAEgB,EAAU,GAAId,EAAEc,EAAU,GAAIhqF,EAAEgqF,EAAU,GAAIjqF,EAAE,QAEvD,IAAgC,IAA5Bm7C,GAAK+uC,YAAYhB,GAAiB,KACrCe,EAAYf,EAAM9gF,OAAO,GAAGA,OAAO,EAAG8gF,EAAMrpF,OAAS,GAAG6N,MAAM,KAClEm8E,EAAO,CAACZ,EAAEgB,EAAU,GAAId,EAAEc,EAAU,GAAIhqF,EAAEgqF,EAAU,GAAIjqF,EAAEiqF,EAAU,SAEjE,IAA+B,IAA3B9uC,GAAKgvC,WAAWjB,GAAiB,KACpCkB,EAASjvC,GAAKkvC,SAASnB,GAC3BW,EAAO,CAACZ,EAAEmB,EAAOnB,EAAGE,EAAEiB,EAAOjB,EAAGlpF,EAAEmqF,EAAOnqF,EAAGD,EAAE,YAI5CkpF,aAAiBnqF,aACHuD,IAAZ4mF,EAAMD,QAA+B3mF,IAAZ4mF,EAAMC,QAA+B7mF,IAAZ4mF,EAAMjpF,EAAiB,KACvEqqF,OAAoBhoF,IAAZ4mF,EAAMlpF,EAAkBkpF,EAAMlpF,EAAI,MAC9C6pF,EAAO,CAACZ,EAAEC,EAAMD,EAAGE,EAAED,EAAMC,EAAGlpF,EAAEipF,EAAMjpF,EAAGD,EAAEsqF,WAMpChoF,IAATunF,QACI,IAAI5jF,MAAM,gIAAkI28C,GAAesmC,SAG5JqB,UAAUV,EAAMD,wCAUItnF,IAAvBO,KAAK4mF,qBACFA,qBACAA,mBAAgBnnF,QAGlBinF,SAAU,OACV/d,MAAMn0C,MAAMC,QAAU,aACtBkzD,gEAWDC,8DAEkB,IAAlBA,SACGnB,cAAgBnuC,GAAKh7C,OAAO,GAAI0C,KAAKqmF,SAGvB,IAAjBrmF,KAAK0mF,cACFC,eAAe3mF,KAAKwmF,mBAGtB7d,MAAMn0C,MAAMC,QAAU,WAIhB,gBACkBh1B,IAAvBi+C,EAAKkpC,gBACPlpC,EAAKkpC,gBACLlpC,EAAKkpC,mBAAgBnnF,KAEvB,wCASGknF,eAAe3mF,KAAKqmF,YACpBK,SAAU,OACVmB,8CASAnB,SAAU,OACVC,eAAe3mF,KAAKqmF,YACpByB,cAAc9nF,KAAKqmF,gDASG5mF,IAAvBO,KAAKymF,mBACFsB,SAAS/nF,KAAKymF,eAAe,GAGlCuB,MAAM,uEAWAhB,OAAMD,8DAEK,IAAfA,SACGP,aAAeluC,GAAKh7C,OAAO,GAAI0pF,SAGjCX,MAAQW,MACTiB,EAAM3vC,GAAK4vC,SAASlB,EAAKZ,EAAGY,EAAKV,EAAGU,EAAK5pF,GAEzC+qF,EAAe,EAAI3mF,KAAK4mF,GACxBC,EAASroF,KAAKomF,EAAI6B,EAAIp/E,EACtBsB,EAAInK,KAAKmmF,kBAAkBh8E,EAAIk+E,EAAS7mF,KAAK8mF,IAAIH,EAAeF,EAAIz7E,GACpEjD,EAAIvJ,KAAKmmF,kBAAkB58E,EAAI8+E,EAAS7mF,KAAK+mF,IAAIJ,EAAeF,EAAIz7E,QAEnEg8E,oBAAoBh0D,MAAMkgB,KAAOvqC,EAAI,GAAMnK,KAAKwoF,oBAAoB7nC,YAAc,UAClF6nC,oBAAoBh0D,MAAMsuB,IAAMv5C,EAAI,GAAMvJ,KAAKwoF,oBAAoBz8B,aAAe,UAElF+7B,cAAcd,uCASTllF,QACLukF,MAAMlpF,EAAI2E,EAAQ,SAClBgmF,cAAc9nF,KAAKqmF,8CASXvkF,OACTmmF,EAAM3vC,GAAK4vC,SAASloF,KAAKqmF,MAAMD,EAAGpmF,KAAKqmF,MAAMC,EAAGtmF,KAAKqmF,MAAMjpF,GAC/D6qF,EAAIQ,EAAI3mF,EAAQ,QACZklF,EAAO1uC,GAAKowC,SAAST,EAAIz7E,EAAGy7E,EAAIp/E,EAAGo/E,EAAIQ,GAC3CzB,EAAI,EAAQhnF,KAAKqmF,MAAMlpF,OAClBkpF,MAAQW,OACRc,4DASOd,yDAAOhnF,KAAKqmF,MACpB4B,EAAM3vC,GAAK4vC,SAASlB,EAAKZ,EAAGY,EAAKV,EAAGU,EAAK5pF,GACzCurF,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACnBppF,IAArBO,KAAK8oF,mBACF7C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,IAEhCT,EAAIU,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OAGxD7wE,EAAIpV,KAAK4oF,kBAAkBjoC,YAC3Bn0C,EAAIxM,KAAK4oF,kBAAkB78B,aAC/B48B,EAAIW,UAAU,EAAG,EAAGl0E,EAAG5I,GAEvBm8E,EAAIY,aAAavpF,KAAKumF,UAAW,EAAE,GACnCoC,EAAIa,UAAY,eAAiB,EAAGvB,EAAIQ,GAAK,IAC7CE,EAAIc,OAAOzpF,KAAKmmF,kBAAkBh8E,EAAGnK,KAAKmmF,kBAAkB58E,EAAGvJ,KAAKomF,MACpEuC,QAAAA,QAEKe,gBAAgB5nF,MAAQ,IAAMmmF,EAAIQ,OAClCkB,aAAa7nF,MAAW,IAAMklF,EAAK7pF,OAEnCysF,gBAAgBp1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKwmF,aAAaJ,EAAI,IAAMpmF,KAAKwmF,aAAaF,EAAI,IAAMtmF,KAAKwmF,aAAappF,EAAI,IAAM4C,KAAKwmF,aAAarpF,EAAI,SAC5J2sF,YAAYt1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKqmF,MAAMD,EAAI,IAAMpmF,KAAKqmF,MAAMC,EAAI,IAAMtmF,KAAKqmF,MAAMjpF,EAAI,IAAM4C,KAAKqmF,MAAMlpF,EAAI,4CAS5HyrF,kBAAkBp0D,MAAMoU,MAAQ,YAChCggD,kBAAkBp0D,MAAMqU,OAAS,YAEjC+/C,kBAAkBhgD,MAAQ,IAAM5oC,KAAKimF,gBACrC2C,kBAAkB//C,OAAS,IAAM7oC,KAAKimF,iEAUtCtd,MAAQv7C,SAASE,cAAc,YAC/Bq7C,MAAMrgC,UAAY,wBAElByhD,eAAiB38D,SAASE,cAAc,YACxCk7D,oBAAsBp7D,SAASE,cAAc,YAC7Ck7D,oBAAoBlgD,UAAY,oBAChCyhD,eAAep1D,YAAY30B,KAAKwoF,0BAEhCI,kBAAoBx7D,SAASE,cAAc,eAC3Cy8D,eAAep1D,YAAY30B,KAAK4oF,mBAEhC5oF,KAAK4oF,kBAAkBC,WAQvB,KACCF,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACvC5C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,QACzBR,kBAAkBC,WAAW,MAAMQ,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OAf1D,KAClC+D,EAAW58D,SAASE,cAAe,OACvC08D,EAASx1D,MAAM6xD,MAAQ,MACvB2D,EAASx1D,MAAMy1D,WAAc,OAC7BD,EAASx1D,MAAMw1C,QAAW,OAC1BggB,EAASt+B,UAAa,wDACjBk9B,kBAAkBj0D,YAAYq1D,QAYhCD,eAAezhD,UAAY,iBAE3B4hD,WAAa98D,SAASE,cAAc,YACpC48D,WAAW5hD,UAAY,mBAEvB6hD,cAAgB/8D,SAASE,cAAc,YACvC68D,cAAc7hD,UAAY,sBAE1B8hD,SAAWh9D,SAASE,cAAc,YAClC88D,SAAS9hD,UAAY,iBAErBqhD,aAAev8D,SAASE,cAAc,kBAEpCq8D,aAAazyD,KAAO,aACpByyD,aAAavnF,IAAM,SACnBunF,aAAarkF,IAAM,YAGnB2gC,SACF0jD,aAAa7nF,MAAQ,WACrB6nF,aAAarhD,UAAY,iBAEzBohD,gBAAkBt8D,SAASE,cAAc,kBAEvCo8D,gBAAgBxyD,KAAO,aACvBwyD,gBAAgBtnF,IAAM,SACtBsnF,gBAAgBpkF,IAAM,YAGtB2gC,SACFyjD,gBAAgB5nF,MAAQ,WACxB4nF,gBAAgBphD,UAAY,iBAE5B4hD,WAAWv1D,YAAY30B,KAAK2pF,mBAC5BQ,cAAcx1D,YAAY30B,KAAK0pF,qBAEhC/qC,EAAK3+C,UACJ2pF,aAAaU,SAAW,WAAa1rC,EAAG2rC,YAAYtqF,KAAK8B,aACzD6nF,aAAaY,QAAW,WAAa5rC,EAAG2rC,YAAYtqF,KAAK8B,aACzD4nF,gBAAgBW,SAAW,WAAa1rC,EAAG6rC,eAAexqF,KAAK8B,aAC/D4nF,gBAAgBa,QAAW,WAAa5rC,EAAG6rC,eAAexqF,KAAK8B,aAE/D2oF,gBAAkBr9D,SAASE,cAAc,YACzCm9D,gBAAgBniD,UAAY,gCAC5BmiD,gBAAgB/+B,UAAY,mBAE5Bg/B,aAAet9D,SAASE,cAAc,YACtCo9D,aAAapiD,UAAY,6BACzBoiD,aAAah/B,UAAY,gBAEzBo+B,YAAc18D,SAASE,cAAc,YACrCw8D,YAAYxhD,UAAY,qBACxBwhD,YAAYp+B,UAAY,WAExBk+B,gBAAkBx8D,SAASE,cAAc,YACzCs8D,gBAAgBthD,UAAY,yBAC5BshD,gBAAgBl+B,UAAY,eAE5Bi/B,aAAev9D,SAASE,cAAc,YACtCq9D,aAAariD,UAAY,6BACzBqiD,aAAaj/B,UAAY,cACzBi/B,aAAaC,QAAU70C,UAAK8xC,cAAW7nF,MAAM,QAE7C6qF,YAAcz9D,SAASE,cAAc,YACrCu9D,YAAYviD,UAAY,4BACxBuiD,YAAYn/B,UAAY,aACxBm/B,YAAYD,QAAU70C,UAAK+0C,eAAY9qF,WAEvC+qF,WAAa39D,SAASE,cAAc,YACpCy9D,WAAWziD,UAAY,2BACvByiD,WAAWr/B,UAAY,YACvBq/B,WAAWH,QAAU70C,UAAKi1C,cAAWhrF,WAErCirF,WAAa79D,SAASE,cAAc,YACpC29D,WAAW3iD,UAAY,2BACvB2iD,WAAWv/B,UAAY,iBACvBu/B,WAAWL,QAAU70C,UAAKm1C,kBAAelrF,WAEzC2oE,MAAMh0C,YAAY30B,KAAK+pF,qBACvBphB,MAAMh0C,YAAY30B,KAAKoqF,eACvBzhB,MAAMh0C,YAAY30B,KAAKyqF,sBACvB9hB,MAAMh0C,YAAY30B,KAAKmqF,oBACvBxhB,MAAMh0C,YAAY30B,KAAK0qF,mBACvB/hB,MAAMh0C,YAAY30B,KAAKkqF,iBACvBvhB,MAAMh0C,YAAY30B,KAAK8pF,kBACvBnhB,MAAMh0C,YAAY30B,KAAK4pF,sBAEvBjhB,MAAMh0C,YAAY30B,KAAK2qF,mBACvBhiB,MAAMh0C,YAAY30B,KAAK6qF,kBACvBliB,MAAMh0C,YAAY30B,KAAK+qF,iBACvBpiB,MAAMh0C,YAAY30B,KAAKirF,kEASvBn7B,KAAO,QACPq7B,MAAQ,QACRtnC,OAAS,IAAIG,GAAOhkD,KAAK4oF,wBACzB/kC,OAAO95C,IAAI,SAAS/F,IAAI,CAACkiD,QAAQ,IAEtCsM,GAAmBxyD,KAAK6jD,QAAQ,SAACrD,GAAWmV,EAAKy1B,cAAc5qC,WAC1DqD,OAAOvN,GAAG,OAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,YAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,WAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,WACtDqD,OAAOvN,GAAG,UAAa,SAACkK,GAAWmV,EAAKy1B,cAAc5qC,uDASpC,IAAnBxgD,KAAKkmF,UAAqB,KACxByC,EAAM3oF,KAAK4oF,kBAAkBC,WAAW,WACnBppF,IAArBO,KAAK8oF,mBACF7C,YAAcj6D,OAAO+8D,kBAAoB,IAAMJ,EAAIK,8BACxDL,EAAIM,2BACJN,EAAIO,0BACJP,EAAIQ,yBACJR,EAAIS,wBAA0B,IAEhCT,EAAIU,aAAarpF,KAAKimF,WAAY,EAAG,EAAGjmF,KAAKimF,WAAY,EAAG,OASxD97E,EAAGZ,EAAG8hF,EAAKC,EANXl2E,EAAIpV,KAAK4oF,kBAAkBjoC,YAC3Bn0C,EAAIxM,KAAK4oF,kBAAkB78B,aAC/B48B,EAAIW,UAAU,EAAG,EAAGl0E,EAAG5I,QAKlB25E,kBAAoB,CAACh8E,EAAO,GAAJiL,EAAS7L,EAAO,GAAJiD,QACpC45E,EAAI,IAAOhxE,MAIZm2E,EAHApD,EAAgB,EAAI3mF,KAAK4mF,GAAM,IAE/BoD,EAAO,EAAIxrF,KAAKomF,MAEfiF,EAAM,EAAGA,EAAM,IAAKA,QAClBC,EAAM,EAAGA,EAAMtrF,KAAKomF,EAAGkF,IAC1BnhF,EAAInK,KAAKmmF,kBAAkBh8E,EAAImhF,EAAM9pF,KAAK8mF,IAAIH,EAAekD,GAC7D9hF,EAAIvJ,KAAKmmF,kBAAkB58E,EAAI+hF,EAAM9pF,KAAK+mF,IAAIJ,EAAekD,GAC7DE,EAAMjzC,GAAKowC,SAAS2C,GAPb,EAAI,KAOqBC,EAAME,EAAM,GAC5C7C,EAAIa,UAAY,OAAS+B,EAAInF,EAAI,IAAMmF,EAAIjF,EAAI,IAAMiF,EAAInuF,EAAI,IAC7DurF,EAAI8C,SAASthF,EAAI,GAAKZ,EAAI,GAAK,EAAG,GAGtCo/E,EAAI+C,YAAc,gBAClB/C,EAAIc,OAAOzpF,KAAKmmF,kBAAkBh8E,EAAGnK,KAAKmmF,kBAAkB58E,EAAGvJ,KAAKomF,GACpEuC,EAAIgD,cAECpF,UAAYoC,EAAIiD,aAAa,EAAE,EAAEx2E,EAAE5I,QAErC05E,WAAY,wCAUL1lC,OACR1X,EAAO9oC,KAAK+pF,eAAennC,wBAC3BlO,EAAO8L,EAAMxB,OAAO70C,EAAI2+B,EAAK4L,KAC7BoO,EAAMtC,EAAMxB,OAAOz1C,EAAIu/B,EAAKga,IAE5B+oC,EAAU,GAAM7rF,KAAK+pF,eAAeh+B,aACpC+/B,EAAU,GAAM9rF,KAAK+pF,eAAeppC,YAEpCx2C,EAAIuqC,EAAOo3C,EACXviF,EAAIu5C,EAAM+oC,EAEVE,EAAQvqF,KAAKwqF,MAAM7hF,EAAEZ,GACrB8+E,EAAS,IAAO7mF,KAAKY,IAAIZ,KAAKyqF,KAAK9hF,EAAIA,EAAIZ,EAAIA,GAAIuiF,GAEnDzwB,EAAS75D,KAAK+mF,IAAIwD,GAAS1D,EAASwD,EACpCK,EAAU1qF,KAAK8mF,IAAIyD,GAAS1D,EAASyD,OAEpCtD,oBAAoBh0D,MAAMsuB,IAAMuY,EAAS,GAAMr7D,KAAKwoF,oBAAoBz8B,aAAe,UACvFy8B,oBAAoBh0D,MAAMkgB,KAAOw3C,EAAU,GAAMlsF,KAAKwoF,oBAAoB7nC,YAAc,SAGzFn0C,EAAIu/E,GAAS,EAAIvqF,KAAK4mF,IAC1B57E,EAAIA,EAAI,EAAIA,EAAI,EAAIA,MAChB3D,EAAIw/E,EAASroF,KAAKomF,EAClB6B,EAAM3vC,GAAK4vC,SAASloF,KAAKqmF,MAAMD,EAAGpmF,KAAKqmF,MAAMC,EAAGtmF,KAAKqmF,MAAMjpF,GAC/D6qF,EAAIz7E,EAAIA,EACRy7E,EAAIp/E,EAAIA,MACJm+E,EAAO1uC,GAAKowC,SAAST,EAAIz7E,EAAGy7E,EAAIp/E,EAAGo/E,EAAIQ,GAC3CzB,EAAI,EAAQhnF,KAAKqmF,MAAMlpF,OAClBkpF,MAAQW,OAGR4C,gBAAgBp1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKwmF,aAAaJ,EAAI,IAAMpmF,KAAKwmF,aAAaF,EAAI,IAAMtmF,KAAKwmF,aAAappF,EAAI,IAAM4C,KAAKwmF,aAAarpF,EAAI,SAC5J2sF,YAAYt1D,MAAMq1D,gBAAkB,QAAU7pF,KAAKqmF,MAAMD,EAAI,IAAMpmF,KAAKqmF,MAAMC,EAAI,IAAMtmF,KAAKqmF,MAAMjpF,EAAI,IAAM4C,KAAKqmF,MAAMlpF,EAAI,ylqBCxiB/HgvF,yBAOQC,EAAcC,EAAkBtP,OAAkBkJ,yDAAa,kBACpEr1D,OAASw7D,OACTE,eAAiB,QACjBrgC,UAAYogC,OACZE,eAAgB,OAEhBp9D,QAAU,QACVq9D,aAAc,OACdC,aAAe,OACfvvC,eAAiB,CACpBw/B,SAAS,EACTzkD,QAAQ,EACRg0B,eAAWxsD,EACXitF,YAAY,GAEdp0C,GAAKh7C,OAAO0C,KAAKmvB,QAASnvB,KAAKk9C,qBAE1B6/B,iBAAmBA,OACnB4P,cAAgB,QAChBC,YAAc,QACdC,SAAW,QACXC,WAAa,OACbC,aAAe,QACfC,YAAc,IAAIhH,GAAYC,QAC9Bl6C,aAAUtsC,gDAUN0vB,WACO1vB,IAAZ0vB,EAAuB,MAEpB49D,aAAe,QACfE,mBAEDvQ,GAAU,KACS,iBAAZvtD,OACJA,QAAQ8I,OAAS9I,OAEnB,GAAIA,aAAmBlzB,WACrBkzB,QAAQ8I,OAAS9I,EAAQhsB,YAE3B,GAAuB,WAAnBsiC,GAAOtW,GAAsB,IACrB,MAAXA,QACI,IAAInC,UAAU,+BAEIvtB,IAAtB0vB,EAAQ88B,iBACL98B,QAAQ88B,UAAY98B,EAAQ88B,gBAEZxsD,IAAnBq3C,GAAA3nB,UACGA,QAAQ8I,UAAS9I,SAEG1vB,IAAvB0vB,EAAQu9D,kBACLv9D,QAAQu9D,WAAav9D,EAAQu9D,iBAEZjtF,IAApB0vB,EAAQutD,UACVA,EAAUvtD,EAAQutD,aAGM,kBAAZvtD,QACTA,QAAQ8I,QAAS,EACtBykD,EAAUvtD,GAEgB,mBAAZA,SACTA,QAAQ8I,OAAS9I,EACtButD,GAAU,IAEgB,IAAxB5lC,QAAK3nB,WACPutD,GAAU,QAGPvtD,QAAQutD,QAAUA,OAEpBwQ,kDAOUP,QACVA,cAAgBA,GACQ,IAAzB3sF,KAAKmvB,QAAQutD,eACVwQ,cAC0BztF,IAA3BO,KAAKmvB,QAAQ88B,iBACVA,UAAYjsD,KAAKmvB,QAAQ88B,gBAE3BvD,kDASFwkC,cACAZ,eAAiB,OAElBr0D,KAASj4B,KAAKmvB,SACdg+D,EAAU,EACV3pB,GAAO,MACN,IAAIlN,KAAUt2D,KAAK+8E,iBAClB/8E,KAAK+8E,iBAAiB1/E,eAAei5D,UAClCi2B,eAAgB,EACrB/oB,GAAO,EACe,mBAAXvrC,EAETurC,GADAA,EAAOvrC,EAAOq+B,EAAO,MACNt2D,KAAKotF,cAAcptF,KAAK+8E,iBAAiBzmB,GAAS,CAACA,IAAS,IAEzD,IAAXr+B,IAA+C,IAA5B27B,GAAA37B,QAAAA,EAAeq+B,KACzCkN,GAAO,IAGI,IAATA,SACG+oB,eAAgB,EAGjBY,EAAU,QACPE,UAAU,SAGZC,YAAYh3B,QAGZ82B,cAAcptF,KAAK+8E,iBAAiBzmB,GAAS,CAACA,KAErD62B,UAGCI,mBACAC,6CAUAzhD,QAAU3e,SAASE,cAAc,YACjCye,QAAQzD,UAAY,iCACpB2jB,UAAUt3B,YAAY30B,KAAK+rC,aAC3B,IAAIjvC,EAAI,EAAGA,EAAIkD,KAAK4sF,YAAY5vF,OAAQF,SACtCivC,QAAQpX,YAAY30B,KAAK4sF,YAAY9vF,SAGvC2wF,0DASA,IAAI3wF,EAAI,EAAGA,EAAIkD,KAAK4sF,YAAY5vF,OAAQF,SACtCivC,QAAQvE,YAAYxnC,KAAK4sF,YAAY9vF,SAGvB2C,IAAjBO,KAAK+rC,eACFkgB,UAAUzkB,YAAYxnC,KAAK+rC,cAC3BA,aAAUtsC,QAEZmtF,YAAc,QAEdK,iDAUGh9D,WACJlS,EAAO/d,KAAK2sF,cACP7vF,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IAAK,SACd2C,IAAlBse,EAAKkS,EAAKnzB,IAGT,CACHihB,OAAOte,QAHPse,EAAOA,EAAKkS,EAAKnzB,WAOdihB,oCAWCkS,OACmB,IAAvBjwB,KAAKusF,cAAwB,KAC3Bx0C,EAAO3qB,SAASE,cAAc,OAClCyqB,EAAKzP,UAAY,iDAAmDrY,EAAKjzB,kCAH1D4vF,mCAAAA,8BAIfA,QAAAA,GAAoB,SAAChlD,GACnBmQ,EAAKpjB,YAAYiT,WAEdglD,YAAY3vF,KAAK86C,GACf/3C,KAAK4sF,YAAY5vF,cAEnB,sCASGyG,OACNiqF,EAAMtgE,SAASE,cAAc,OACjCogE,EAAIplD,UAAY,sCAChBolD,EAAIhiC,UAAYjoD,OACX4pF,UAAU,GAAGK,sCAYTjqF,EAAMwsB,OAAM09D,0DACjBD,EAAMtgE,SAASE,cAAc,cACjCogE,EAAIplD,UAAY,kDAAoDrY,EAAKjzB,OAEvE0wF,EAAIhiC,WADc,IAAhBiiC,EACc,SAAWlqF,EAAO,YAGlBA,EAAO,IAElBiqF,wCAWK9wF,EAAKkF,EAAOmuB,OACpB2iD,EAASxlD,SAASE,cAAc,UACpCslD,EAAOtqC,UAAY,0CACfslD,EAAgB,OACNnuF,IAAVqC,IAC0B,IAAxB8xD,GAAAh3D,QAAAA,EAAYkF,KACd8rF,EAAgBh6B,GAAAh3D,QAAAA,EAAYkF,QAI3B,IAAIhF,EAAI,EAAGA,EAAIF,EAAII,OAAQF,IAAK,KAC/Bw5D,EAASlpC,SAASE,cAAc,UACpCgpC,EAAOx0D,MAAQlF,EAAIE,GACfA,IAAM8wF,IACRt3B,EAAO0N,SAAW,YAEpB1N,EAAO5K,UAAY9uD,EAAIE,GACvB81E,EAAOj+C,YAAY2hC,OAGjB3X,EAAK3+C,KACT4yE,EAAOyX,SAAW,WAAa1rC,EAAGvI,QAAQp2C,KAAK8B,MAAOmuB,QAElDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOqqC,sCAWnBh2E,EAAKkF,EAAOmuB,OACjB69D,EAAelxF,EAAI,GACnBwF,EAAMxF,EAAI,GACV0I,EAAM1I,EAAI,GACVmnC,EAAOnnC,EAAI,GACX88C,EAAQtsB,SAASE,cAAc,SACnCosB,EAAMpR,UAAY,yCAEhBoR,EAAMxiB,KAAO,QACbwiB,EAAMt3C,IAAMA,EACZs3C,EAAMp0C,IAAMA,QAGP2gC,IACPyT,EAAM3V,KAAOA,MAGTgqD,EAAc,GACdC,EAAa,UAEHvuF,IAAVqC,EAAqB,CAEnBA,EAAQ,GADC,IACIA,EAAiBM,GAChCs3C,EAAMt3C,IAAMZ,KAAKC,KAFN,IAEWK,GACtBksF,EAAat0C,EAAMt3C,IACnB2rF,EAAc,mBAEPjsF,EANI,IAMaM,IACxBs3C,EAAMt3C,IAAMZ,KAAKC,KAAKK,EAPX,KAQXksF,EAAat0C,EAAMt3C,IACnB2rF,EAAc,mBATH,IAWTjsF,EAAiBwD,GAAe,IAARA,IAC1Bo0C,EAAMp0C,IAAM9D,KAAKC,KAZN,IAYWK,GACtBksF,EAAat0C,EAAMp0C,IACnByoF,EAAc,mBAEhBr0C,EAAM53C,MAAQA,OAGd43C,EAAM53C,MAAQgsF,MAGZ9xF,EAAQoxB,SAASE,cAAc,SACnCtxB,EAAMssC,UAAY,0CAClBtsC,EAAM8F,MAAQu1C,OAAOqC,EAAM53C,WAEvB68C,EAAK3+C,KACT05C,EAAM2wC,SAAW,WAAaruF,EAAM8F,MAAQ9B,KAAK8B,MAAO68C,EAAGvI,QAAQiB,OAAOr3C,KAAK8B,OAAQmuB,IACvFypB,EAAM6wC,QAAW,WAAavuF,EAAM8F,MAAQ9B,KAAK8B,WAE7CymC,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,GAC7CuyC,EAAYxiE,KAAKqtF,UAAUp9D,EAAMsY,EAAOmR,EAAO19C,GAG/B,KAAhB+xF,GAAsB/tF,KAAK+sF,aAAavqB,KAAewrB,SACpDjB,aAAavqB,GAAawrB,OAC1BC,YAAYF,EAAavrB,yDASA,IAA5BxiE,KAAKmvB,QAAQu9D,WAAqB,KAChCwB,EAAiB9gE,SAASE,cAAc,OAC5C4gE,EAAe5lD,UAAY,sCAC3B4lD,EAAexiC,UAAY,mBAC3BwiC,EAAetD,QAAc,WAAOltC,EAAKywC,iBACzCD,EAAeE,YAAc,WAAOF,EAAe5lD,UAAY,6CAC/D4lD,EAAeG,WAAc,WAAOH,EAAe5lD,UAAY,4CAE1DgmD,iBAAmBlhE,SAASE,cAAc,YAC1CghE,iBAAiBhmD,UAAY,qDAE7BskD,YAAY3vF,KAAK+C,KAAKsuF,uBACtB1B,YAAY3vF,KAAKixF,wCAWdj3E,EAAQ2N,kBACO,IAArB5kB,KAAKwsF,cAA+C,IAAvBxsF,KAAKusF,eAA0BvsF,KAAKysF,aAAezsF,KAAK8sF,WAAY,KAC/FY,EAAMtgE,SAASE,cAAc,OACjCogE,EAAIt6D,GAAK,0BACTs6D,EAAIplD,UAAY,0BAChBolD,EAAIhiC,UAAYz0C,EAChBy2E,EAAI9C,QAAU,WAAOj1B,EAAKs3B,qBACrBR,cAAgB,OAChBI,SAAW,CAACn4D,KAAKg5D,EAAK9oE,MAAMA,gDAURnlB,IAAvBO,KAAK6sF,SAASn4D,YACXm4D,SAASn4D,KAAK6S,WAAWC,YAAYxnC,KAAK6sF,SAASn4D,MACxD0qB,aAAap/C,KAAK6sF,SAAS0B,aAC3BnvC,aAAap/C,KAAK6sF,SAAS2B,oBACtB3B,SAAW,oEAUSptF,IAAvBO,KAAK6sF,SAASn4D,KAAoB,KAEhCoU,EADuB9oC,KAAK4sF,YAAY5sF,KAAK6sF,SAASjoE,OAC1Bg+B,6BAC3BiqC,SAASn4D,KAAKF,MAAMkgB,KAAO5L,EAAK4L,KAAO,UACvCm4C,SAASn4D,KAAKF,MAAMsuB,IAAMha,EAAKga,IAAM,GAAK,KAC/C11B,SAASwrB,KAAKjkB,YAAY30B,KAAK6sF,SAASn4D,WACnCm4D,SAAS0B,YAAcpvC,IAAW,WACrC4hB,EAAK8rB,SAASn4D,KAAKF,MAAMi6D,QAAU,IACnC,WACG5B,SAAS2B,cAAgBrvC,IAAW,WACvC4hB,EAAKksB,iBACL,6CAWQa,EAAchsF,EAAOmuB,OAC7By+D,EAAWthE,SAASE,cAAc,SACtCohE,EAASx3D,KAAO,WAChBw3D,EAASpmD,UAAY,wCACrBomD,EAASC,QAAUb,OACLruF,IAAVqC,IACF4sF,EAASC,QAAU7sF,EACfA,IAAUgsF,IACgB,WAAxBroD,GAAOqoD,GACLhsF,IAAUgsF,EAAapR,cACpB4P,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,SAIxCwqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,UAK7C68C,EAAK3+C,KACT0uF,EAASrE,SAAW,WAAY1rC,EAAGvI,QAAQp2C,KAAK2uF,QAAS1+D,QAErDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOmmD,0CAUfZ,EAAchsF,EAAOmuB,OAC9By+D,EAAWthE,SAASE,cAAc,SACtCohE,EAASx3D,KAAO,OAChBw3D,EAASpmD,UAAY,oCACrBomD,EAAS5sF,MAAQA,EACbA,IAAUgsF,QACPxB,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,QAGzC68C,EAAK3+C,KACT0uF,EAASrE,SAAW,WAAY1rC,EAAGvI,QAAQp2C,KAAK8B,MAAOmuB,QAEnDsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAMsY,EAAOmmD,2CAWd9xF,EAAKkF,EAAOmuB,cACtB2+D,EAAehyF,EAAI,GACnB8wF,EAAMtgE,SAASE,cAAc,OAGnB,UAFdxrB,OAAkBrC,IAAVqC,EAAsB8sF,EAAe9sF,IAG3C4rF,EAAIplD,UAAY,0CAChBolD,EAAIl5D,MAAMq1D,gBAAkB/nF,GAG5B4rF,EAAIplD,UAAY,+CAGlBxmC,OAAkBrC,IAAVqC,EAAsB8sF,EAAe9sF,EAC7C4rF,EAAI9C,QAAU,WACZlpB,EAAKmtB,iBAAiB/sF,EAAM4rF,EAAIz9D,QAG9BsY,EAAQvoC,KAAK6tF,WAAW59D,EAAKA,EAAKjzB,OAAO,GAAIizB,QAC5Co9D,UAAUp9D,EAAKsY,EAAOmlD,4CAWZ5rF,EAAO4rF,EAAKz9D,cAE3By9D,EAAI9C,QAAU,kBAEToC,YAAY8B,SAASpB,QACrBV,YAAYxpB,YAEZwpB,YAAYjF,SAASjmF,QACrBkrF,YAAY+B,mBAAkB,SAAC1I,OAC9B2I,EAAc,QAAU3I,EAAMD,EAAI,IAAMC,EAAMC,EAAI,IAAMD,EAAMjpF,EAAI,IAAMipF,EAAMlpF,EAAI,IACtFuwF,EAAIl5D,MAAMq1D,gBAAkBmF,EAC5Bva,EAAKr+B,QAAQ44C,EAAY/+D,WAItB+8D,YAAYiC,kBAAiB,WAChCvB,EAAI9C,QAAU,WACZnW,EAAKoa,iBAAiB/sF,EAAM4rF,EAAIz9D,6CAcxB5uB,OAAK4uB,yDAAO,GAAIi/D,0DACxB1rB,GAAO,EACPvrC,KAASj4B,KAAKmvB,SACdggE,GAAe,MACd,IAAIC,KAAU/tF,KACbA,EAAIhE,eAAe+xF,GAAS,CAC9B5rB,GAAO,MACHzrB,EAAO12C,EAAI+tF,GACXC,EAAU/2C,GAAKqjC,mBAAmB1rD,EAAMm/D,MACtB,mBAAXn3D,IAII,KAHburC,EAAOvrC,EAAOm3D,EAAOn/D,OAIb8nB,aAAgB97C,QAA0B,iBAAT87C,GAAqC,kBAATA,GAAsBA,aAAgB77C,cAClGqwF,eAAgB,EACrB/oB,EAAOxjE,KAAKotF,cAAcr1C,EAAMs3C,GAAS,QACpC9C,eAA8B,IAAd2C,IAKd,IAAT1rB,EAAgB,CAClB2rB,GAAe,MACXrtF,EAAQ9B,KAAKsvF,UAAUD,MAEvBt3C,aAAgB97C,WACbszF,aAAax3C,EAAMj2C,EAAOutF,QAE5B,GAAoB,iBAATt3C,OACTy3C,eAAez3C,EAAMj2C,EAAOutF,QAE9B,GAAoB,kBAATt3C,OACT03C,cAAc13C,EAAMj2C,EAAOutF,QAE7B,GAAIt3C,aAAgB77C,OAAQ,KAE3BwzF,GAAO,MACsB,IAA7B97B,GAAA3jC,QAAAA,EAAa,YACXjwB,KAAK2sF,cAAcgD,QAAQC,SAAWR,IACxCM,GAAO,IAIE,IAATA,UAEmBjwF,IAAjBs4C,EAAK2kC,QAAuB,KAC1BmT,EAAcv3C,GAAKqjC,mBAAmB0T,EAAS,WAC/CS,EAAe9vF,KAAKsvF,UAAUO,OACb,IAAjBC,EAAuB,KACrBvnD,EAAQvoC,KAAK6tF,WAAWuB,EAAQC,GAAS,QACxChC,UAAUgC,EAAS9mD,GACxB4mD,EAAenvF,KAAKotF,cAAcr1C,EAAMs3C,IAAYF,YAG/CM,cAAc13C,EAAM+3C,EAAcT,OAGtC,KACC9mD,EAAQvoC,KAAK6tF,WAAWuB,EAAQC,GAAS,QACxChC,UAAUgC,EAAS9mD,GACxB4mD,EAAenvF,KAAKotF,cAAcr1C,EAAMs3C,IAAYF,QAKxDxsF,QAAQwpB,MAAM,0BAA2B4rB,EAAMq3C,EAAQC,WAKxDF,uCAWIvyF,EAAKkF,EAAOmuB,GACD,iBAAXrzB,EAAI,IAA8B,UAAXA,EAAI,SAC/BmzF,gBAAgBnzF,EAAKkF,EAAOmuB,GAC7BrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,KAEzC,iBAAXlF,EAAI,SACbozF,cAAcpzF,EAAKkF,EAAOmuB,GAC3BrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMA,KAEzC,iBAAXlF,EAAI,UACbqzF,WAAWrzF,EAAKkF,EAAOmuB,GACxBrzB,EAAI,KAAOkF,QAAawqF,eAAervF,KAAK,CAACgzB,KAAKA,EAAMnuB,MAAMu1C,OAAOv1C,sCAYrEA,EAAOmuB,OACTd,EAAUnvB,KAAKkwF,kBAAkBpuF,EAAMmuB,GAEvCjwB,KAAK4wB,OAAOgoB,MAAQ54C,KAAK4wB,OAAOgoB,KAAKiF,SAAW79C,KAAK4wB,OAAOgoB,KAAKiF,QAAQ4C,WACtE7vB,OAAOgoB,KAAKiF,QAAQ4C,KAAK,eAAgBtxB,QAE3Cq9D,aAAc,OACd57D,OAAO4tB,WAAWrvB,6CAYPrtB,EAAOmuB,OAAMkgE,yDAAa,GACtCluC,EAAUkuC,EAIdruF,EAAkB,WADlBA,EAAkB,SAAVA,GAA4BA,IACAA,MAE/B,IAAIhF,EAAI,EAAGA,EAAImzB,EAAKjzB,OAAQF,IACf,WAAZmzB,EAAKnzB,UACkB2C,IAArBwiD,EAAQhyB,EAAKnzB,MACfmlD,EAAQhyB,EAAKnzB,IAAM,IAEjBA,IAAMmzB,EAAKjzB,OAAS,EACtBilD,EAAUA,EAAQhyB,EAAKnzB,IAGvBmlD,EAAQhyB,EAAKnzB,IAAMgF,UAIlBquF,8CAOHhhE,EAAUnvB,KAAKowF,kBACd9B,iBAAiB5iC,UAAY,sBAAwB3L,GAAe5wB,EAAS,KAAM,GAAK,sDAQzFA,EAAU,GACLryB,EAAI,EAAGA,EAAIkD,KAAKssF,eAAetvF,OAAQF,SACzCozF,kBAAkBlwF,KAAKssF,eAAexvF,GAAGgF,MAAO9B,KAAKssF,eAAexvF,GAAGmzB,KAAMd,UAE7EA,WC9sBUkhE,0BAQPpkC,EAAWzV,EAAOm2B,EAAQx9C,mEAE/B8wB,SAAW,IAAIvjD,OACf4zF,WAAY,IAEXC,gBAAgBF,SACd,IAAIG,YAAY,yDAIlBttD,GAAcypC,IAAWA,aAAkB/0B,IAAW+0B,aAAkB2H,KAAa3H,aAAkBzwE,OAAQ,KAC7Gu0F,EAAgBthE,EACtBA,EAAUw9C,EACVA,EAAS8jB,EAKPthE,GAAWA,EAAQwtD,gBACrBh6E,QAAQH,KAAK,6HAGTm8C,aACDzB,eAAiB,CACpB2a,YAAY,EACZL,oBAAqB,IACrBnP,YAAa,CACXC,KAAM,SACNvQ,KAAM,UAER3tB,OAAAA,MAEG+E,QAAUmpB,GAAKuN,WAAW,GAAInI,EAAKR,kBAGnCwL,QAAQuD,IACR98B,GAAYA,QAAiC,IAAfA,EAAQguB,IAAqB,KAE1DuzC,IADCtyC,IAAI+C,KAAK3sB,MAAMyqB,WAAa,iBAE7B0xC,EAAUjzC,EAAKU,IAAI+C,MACfuvC,GAAoBC,GAC1BD,EAAmB1kE,OAAO+8C,iBAAiB4nB,EAAS,MAAMryE,UAC1DqyE,EAAUA,EAAQC,gBAEfzhE,QAAQguB,IAAOuzC,GAAuD,OAAlCA,EAAiBpsF,qBAErD6qB,QAAQguB,IAAMhuB,EAAQguB,IAGzBhuB,IACEA,EAAQquB,gBAAoBruB,QAAQquB,YAAcruB,EAAQquB,aAC1DruB,EAAQ0tD,0BAA8B1tD,QAAQ0tD,sBAAwB1tD,EAAQ0tD,uBAC9E1tD,EAAQ6vC,cAAkB7vC,QAAQ6vC,UAAY7vC,EAAQ6vC,WACtD7vC,EAAQ2tD,0BAA8B3tD,QAAQ2tD,sBAAwB3tD,EAAQ2tD,4BAI9E+T,EAAwBzjE,SAASE,cAAc,UACjDowB,EAAKvuB,QAAQ2tD,sBAAuB,OAChC9e,EAAmBjoB,OAAK5mB,QAAQ2tD,qCAChCprB,EAAgBsM,EAAiBtgB,EAAKU,IAAIsT,eAC3CA,aAAyBx1D,UAAaw1D,aAAyBuM,SAClED,EAAiB6yB,GAEbn/B,aAAyBuM,SAC3B4yB,EAAsBnlC,UAAY,GAClCmlC,EAAsBl8D,YAAY+8B,IAEVjyD,MAAjBiyD,IACPm/B,EAAsBnlC,UAAYgG,YAInCtT,IAAIsT,cAAc/8B,YAAYk8D,KAG9Bt8B,WAAa,KAEb3b,KAAO,CACVwF,IAAKV,EAAKU,IACV5E,SAAUkE,EAAKrI,MACfwI,QAAS,CACPvH,GAAIP,OAAKO,kBACTC,IAAKR,OAAKQ,mBACVkK,KAAM1K,OAAK0K,qBAEbrH,YAAa,GACbd,KAAM,CACJsY,2BACSjS,EAAG8J,SAAS1kB,KAAK4X,OAE1BkV,0BACSlS,EAAG8J,SAAS1kB,KAAKA,MAG1ByX,SAAUzF,KAAA4I,EAAGmyC,kBAAenyC,GAC5BoyC,eAAgBh7C,KAAA4I,EAAGqyC,wBAAqBryC,GACxC1C,OAAQlG,KAAA4I,EAAGsyC,gBAAatyC,GACxBuyC,aAAen7C,KAAA4I,EAAGwyC,sBAAmBxyC,OAKpCjF,MAAQ,IAAImD,GAAMa,EAAK9E,KAAM8E,EAAKvuB,WAClColC,WAAWt3D,KAAKygD,EAAKhE,SACrBd,KAAKc,MAAQgE,EAAKhE,QAGlB+O,SAAW,IAAIV,GAASrK,EAAK9E,KAAM8E,EAAKvuB,WACxCmlC,UAAY,OACZC,WAAWt3D,KAAKygD,EAAK+K,YAGrBiP,YAAc,IAAIgB,GAAYhb,EAAK9E,KAAM8E,EAAKvuB,WAC9ColC,WAAWt3D,KAAKygD,EAAKga,eAGrBvF,QAAU,IAAIwc,GAAQjxB,EAAK9E,KAAM8E,EAAKvuB,WACtColC,WAAWt3D,KAAKygD,EAAKyU,WAErBsT,UAAY,OACZsK,WAAa,OAEb3xB,IAAI+C,KAAKypC,QAAU,SAAApqC,GACtB7B,EAAG8B,KAAK,QAAS9B,EAAGgV,mBAAmBnT,OAEpCpC,IAAI+C,KAAKirB,WAAa,SAAA5rB,GACzB7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,OAE1CpC,IAAI+C,KAAKiwC,cAAgB,SAAA5wC,GAC5B7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,OAE1CpC,IAAI+C,KAAKitC,YAAc,SAAA5tC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,KAE1Cx0B,OAAOqlE,gBACHjzC,IAAI+C,KAAKmwC,cAAgB,SAAA9wC,GAC5B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKowC,cAAgB,SAAA/wC,GAC5B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKqwC,YAAc,SAAAhxC,GAC1B7B,EAAG8B,KAAK,UAAW9B,EAAGgV,mBAAmBnT,SAGtCpC,IAAI+C,KAAKswC,YAAc,SAAAjxC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKuwC,YAAc,SAAAlxC,GAC1B7B,EAAG8B,KAAK,YAAa9B,EAAGgV,mBAAmBnT,OAExCpC,IAAI+C,KAAKwwC,UAAY,SAAAnxC,GACxB7B,EAAG8B,KAAK,UAAW9B,EAAGgV,mBAAmBnT,QAKxCoxC,gBAAiB,IACjBt7C,GAAG,WAAW,cACG,MAAhBqI,EAAG8mB,eACF9mB,EAAGizC,iBAAmBjzC,EAAGxvB,QAAQquB,eACpCmB,EAAGizC,gBAAiB,EACInyF,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,IAAkB,IACxChhC,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,QAC1CiZ,EAAQiF,EAAGkzC,mBAGXrxD,EAA4B/gC,MAApBk/C,EAAGxvB,QAAQqR,MAAqBme,EAAGxvB,QAAQqR,MAAQkZ,EAAMt3C,IACjEq+B,EAA4BhhC,MAApBk/C,EAAGxvB,QAAQsR,IAAqBke,EAAGxvB,QAAQsR,IAAQiZ,EAAMp0C,IACvEq5C,EAAGyX,UAAU51B,EAAOC,EAAK,CAACse,WAAW,SAErCJ,EAAGmzC,IAAI,CAAC/yC,WAAW,IAIlBJ,EAAGmT,kBAAoBnT,EAAGqT,yBAA4BrT,EAAGxvB,QAAQqR,OAAUme,EAAGxvB,QAAQsR,OACtFke,EAAGxvB,QAAQquB,cACdmB,EAAGmT,iBAAkB,EACrBnT,EAAGwT,QAAQL,iBAAkB,EAC7BnT,EAAGP,IAAI+C,KAAK3sB,MAAMyqB,WAAa,UAC/BN,EAAGP,IAAIsT,cAAcnqB,WAAWC,YAAYmX,EAAGP,IAAIsT,eAC/C/S,EAAGxvB,QAAQ0tD,2BACF,kBACFl+B,EAAGxvB,QAAQ0tD,0BACjB,UAKJvmC,GAAG,mBAAmB,WACzBqI,EAAGoG,aAID51B,KACGqvB,WAAWrvB,KAGbypB,KAAKiF,QAAQvH,GAAG,OAAO,SAACtzC,KACtB+uF,OAAO/uF,KACPqtD,YAIHsc,KACGvX,UAAUuX,GAIbn2B,KACG2e,SAAS3e,KAIXub,oFASE,IAAIo6B,GAAansF,KAAMA,KAAKo+C,IAAI6N,UAAW8wB,0CAW7C5qB,SAAWnyD,KAAKmyD,QAAQ+f,UAAU,CAACC,cAAc,SACjDpgB,6CAOI5iC,OAIU,IAFFwrD,GAAUqX,SAAS7iE,EAASqrD,KAG3C73E,QAAQq3C,IAAI,2DAA4D0gC,IAG1Ej/B,GAAKt/C,UAAUqiD,WAAWniD,KAAK2D,KAAMmvB,GAEjC,SAAUA,GACRA,EAAQ+H,OAASl3B,KAAKmvB,QAAQ+H,KAAM,MACjC/H,QAAQ+H,KAAO/H,EAAQ+H,SAGtBuuC,EAAYzlE,KAAKylE,aACnBA,EAAW,KACPwL,EAAYjxE,KAAKm2E,oBAClBhhB,SAAS,WACTA,SAASsQ,EAAUjuB,YACnBwhC,aAAa/H,sCAUjBz6B,OAIHy7C,OAHC3B,WAAY,EAQf2B,EAJGz7C,EAIUe,GADNf,aAAiBoB,IAAWpB,aAAiB89B,GACrB99B,EAIA,IAAIoB,GAAQpB,IAP9B,KAWXx2C,KAAKylE,gBAEFA,UAAUptB,eAEZotB,UAAYwsB,OACZ9/B,SAAWnyD,KAAKmyD,QAAQgD,SAAuB,MAAd88B,EAAqBA,EAAWz6C,MAAQ,wCAOtEm1B,OAEJslB,KACCtlB,EAGA,KACG10C,EAAS,SAAAosC,UAA2B,IAAlBA,EAAM/I,SAE5B22B,EADEtlB,aAAkB/0B,IAAW+0B,aAAkB2H,GACpC,IAAIA,GAAS3H,EAAO,CAAC10C,OAAAA,IAIrB,IAAI2f,GAAQd,GAAA61B,QAAAA,EAAc10C,SATzCg6D,EAAa,KAwBXjyF,KAAK+vE,sBAAsBuE,SACxBvE,WAAWrS,QAAQ,WAErBqS,WAAakiB,OACb9/B,QAAQiD,UAAU68B,mCAOjB1+E,GACFA,GAAQA,EAAKo5D,aACVvX,UAAU7hD,EAAKo5D,QAGlBp5D,GAAQA,EAAKijC,YACV2e,SAAS5hD,EAAKijC,4CAqBV+7B,EAAKpjD,QACXgjC,SAAWnyD,KAAKmyD,QAAQ6mB,aAAazG,GAEtCpjD,GAAWA,EAAQshC,YAChBA,MAAM8hB,EAAKpjD,iDASXnvB,KAAKmyD,SAAWnyD,KAAKmyD,QAAQgkB,gBAAkB,iCAkBlD/iD,EAAIjE,MACHnvB,KAAKylE,WAAmBhmE,MAAN2zB,OAEjBm/C,EAAMrvC,GAAc9P,GAAMA,EAAK,CAACA,GAGhCqyC,EAAYzlE,KAAKylE,UAAU17D,IAAIwoE,GAGjC/xC,EAAQ,KACRC,EAAM,WACVglC,QAAAA,GAAkB,SAAAzR,OACVnrD,EAAImrD,EAASxzB,MAAMjjC,UACnB8V,EAAI,QAAS2gD,EAAWA,EAASvzB,IAAIljC,UAAYy2D,EAASxzB,MAAMjjC,WAExD,OAAVijC,GAAkB33B,EAAI23B,KACxBA,EAAQ33B,IAGE,OAAR43B,GAAgBptB,EAAIotB,KACtBA,EAAMptB,MAKI,OAAVmtB,GAA0B,OAARC,EAAc,KAC5Bke,EAAK3+C,KAEL+3C,EAAO/3C,KAAKmyD,QAAQ3b,MAAM+7B,EAAI,IAChC2f,GAAmC,EAAxBlyF,KAAKw4D,gBAChB25B,EAAwB,KAmCtBC,EAA2B,eACzBC,EAAsBC,GAAsB3zC,EAAI5G,GAElDs6C,EAAoBE,cAAgBF,EAAoBG,SAAWL,EAAsBK,UAC3F7zC,EAAGuU,eAAem/B,EAAoBI,cACtC9zC,EAAGoT,YAeDzP,GAAOnzB,QAA4B1vB,IAAjB0vB,EAAQmzB,MAAqBnzB,EAAQmzB,KACvDowC,GAAUlyD,EAAQC,GAAO,EACzBoe,EAAWyD,EAAuB,KAAf7hB,EAAMD,GAAeh/B,KAAK8D,IAAItF,KAAK05C,MAAMjZ,IAAMzgC,KAAK05C,MAAMlZ,MAAuB,KAAfC,EAAMD,IAE3Fue,GAAY5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA0B5vB,EAAQ4vB,UAElEA,IAEHozC,EAAwB,CAAEI,cAAc,EAAOE,cAAe,EAAGD,SAAU,SAGxE94C,MAAMgF,SAASg0C,EAAS7zC,EAAW,EAAG6zC,EAAS7zC,EAAW,EAAG,CAAEE,UAAAA,IApBtC,WAE5BqzC,OAGWA,EAA0B,QAhDR,SAAChyC,EAAMuyC,EAAUx1D,OACxC41B,EAAiBu/B,GAAsB3zC,EAAI5G,OAE1B,IAAnBgb,IAIAo/B,IACFA,EAAwBp/B,GAGvBo/B,EAAsBK,SAAWz/B,EAAey/B,SAAYL,EAAsBI,eAG7EJ,EAAsBK,SAAWz/B,EAAey/B,SAAWz/B,EAAew/B,eAEhFJ,EAAwBp/B,EACxBm/B,GAAiC,EAAtBvzC,EAAG6Z,qBAGVn4D,EAAO6xF,EACP9xF,EAAK+xF,EAAsBM,aAC3B7gC,EAAYz0B,EAAO/8B,EAAMC,GAAQD,EAAKC,GAAQ+/C,EAEpDzB,EAAGuU,eAAetB,GAEd+gC,GACFh0C,EAAGoT,4CAmDP5iC,EAASnpB,OAEP0zC,EADEqF,GAAa5vB,QAAiC1vB,IAAtB0vB,EAAQ4vB,WAA2B5vB,EAAQ4vB,UAG3C,IAA1B/+C,KAAKylE,UAAUzoE,aAAgDyC,IAAhCO,KAAKylE,UAAU17D,MAAM,GAAG02B,KAEzDiZ,EAAQ15C,KAAK81D,oBACR/S,OAAOrJ,EAAMt3C,IAAI7E,UAAW,CAACwhD,UAAAA,GAAY/4C,KAI9C0zC,EAAQ15C,KAAK6xF,oBACRn4C,MAAMgF,SAAShF,EAAMt3C,IAAKs3C,EAAMp0C,IAAK,CAAEy5C,UAAAA,GAAa/4C,sDAYrD0zC,EAAQ15C,KAAK81D,eACf1zD,EAAoB,OAAds3C,EAAMt3C,IAAes3C,EAAMt3C,IAAI7E,UAAY,KACjD+H,EAAoB,OAAdo0C,EAAMp0C,IAAeo0C,EAAMp0C,IAAI/H,UAAY,KACjDq1F,EAAU,KACVC,EAAU,QAEH,MAAPzwF,GAAsB,MAAPkD,EAAa,KAC1Bu5C,EAAYv5C,EAAMlD,EAClBy8C,GAAY,IACdA,EAAW,QAEPi0C,EAASj0C,EAAW7+C,KAAKq1C,MAAM2J,OAAOpW,MAEtC41B,EAAc,GAChBC,EAAoB,QAGxBnmB,SAAAA,GAAat4C,KAAKmyD,QAAQ3b,OAAO,SAACuB,EAAM90C,MAClC80C,EAAKksB,aAAc,CAErBzF,EAAYv7D,GAAO80C,EAAKsY,QADJ,GAEpBoO,EAAoBD,EAAYv7D,GAAKjG,WAItByhE,EAAoB,qBAG5B3hE,MACPw7C,SAAAA,GAAakmB,GAAa,SAAAI,GACxBA,EAAI9hE,SAFCA,EAAI,EAAGA,EAAI2hE,EAAmB3hE,MAA9BA,SAQXw7C,SAAAA,GAAat4C,KAAKmyD,QAAQ3b,OAAO,SAAAuB,OAG3Bg7C,EACAC,EAHExyD,EAAQyyD,GAASl7C,GACjBtX,EAAMyyD,GAAOn7C,GAIf4d,EAAKxmC,QAAQguB,KACf41C,EAAavyD,GAASuX,EAAKo7C,gBAAmB,IAAML,EACpDE,EAAUvyD,GAASsX,EAAKq7C,eAAiB,IAAMN,IAE/CC,EAAavyD,GAASuX,EAAKq7C,eAAkB,IAAMN,EACnDE,EAAUvyD,GAASsX,EAAKo7C,gBAAkB,IAAML,GAG9CC,EAAY3wF,IACdA,EAAM2wF,EACNH,EAAU76C,GAERi7C,EAAU1tF,IACZA,EAAM0tF,EACNH,EAAU96C,MAIV66C,GAAWC,EAAS,KAChBQ,EAAMT,EAAQQ,eAAiB,GAC/BE,EAAMT,EAAQM,gBAAkB,GAChC7xC,EAAQthD,KAAKq1C,MAAM2J,OAAOpW,MAAQyqD,EAAMC,EAE1ChyC,EAAQ,IACNthD,KAAKmvB,QAAQguB,KACf/6C,EAAM6wF,GAASL,GAAWU,EAAMz0C,EAAWyC,EAC3Ch8C,EAAM4tF,GAAOL,GAAaQ,EAAMx0C,EAAWyC,IAE3Cl/C,EAAM6wF,GAASL,GAAWS,EAAMx0C,EAAWyC,EAC3Ch8C,EAAM4tF,GAAOL,GAAaS,EAAMz0C,EAAWyC,WAM5C,CACLl/C,IAAY,MAAPA,EAAc,IAAI1F,KAAK0F,GAAO,KACnCkD,IAAY,MAAPA,EAAc,IAAI5I,KAAK4I,GAAO,mDASjClD,EAAM,KACNkD,EAAM,KAENtF,KAAKylE,qBACFA,mBAAkB,SAAA1tB,OACfvX,EAAQ8X,GAAKpB,QAAQa,EAAKvX,MAAO,QAAQjjC,UACzCkjC,EAAQ6X,GAAKpB,QAAoBz3C,MAAZs4C,EAAKtX,IAAmBsX,EAAKtX,IAAMsX,EAAKvX,MAAO,QAAQjjC,WACtE,OAAR6E,GAAgBo+B,EAAQp+B,KAC1BA,EAAMo+B,IAEI,OAARl7B,GAAgBm7B,EAAMn7B,KACxBA,EAAMm7B,YAKL,CACLr+B,IAAY,MAAPA,EAAc,IAAI1F,KAAK0F,GAAO,KACnCkD,IAAY,MAAPA,EAAc,IAAI5I,KAAK4I,GAAO,iDAUpBk7C,OACX2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDC,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChDO,EAAsB3iD,KAAKo+C,IAAI3E,gBAAgBmJ,wBAC/Cz4C,EAAInK,KAAKmvB,QAAQguB,IAAMwF,EAAoBhO,MAAQwN,EAAUA,EAAUQ,EAAoBjO,KAC3FnrC,EAAI64C,EAAUO,EAAoBG,IAElC/K,EAAQ/3C,KAAKmyD,QAAQqjB,eAAeh1B,GACpC6jB,EAAQrkE,KAAKmyD,QAAQqkB,gBAAgBh2B,GACrCmP,EAAaH,GAAW+jC,qBAAqB/yC,GAE7CsQ,EAAO9wD,KAAKmyD,QAAQhjC,QAAQ2hC,MAAQ,KACpCnV,EAAQ37C,KAAK44C,KAAKN,KAAKsY,WACvB7sB,EAAO/jC,KAAK44C,KAAKN,KAAKuY,UACtBnwC,EAAO1gB,KAAKixF,QAAQ9mF,GACpB4mD,EAAcD,EAAOA,EAAKpwC,EAAMi7B,EAAO5X,GAAQrjB,EAE/CknB,EAAU0Q,GAAKk7C,UAAUhzC,GAC3BizC,EAAO,YACC,MAAR17C,EAAkE07C,EAAO,OACtD,MAAd9jC,EAA6D8jC,EAAO,cACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKyoD,SAASrK,IAAI4J,aAC1ChoD,KAAKs0D,WAAahc,GAAKo7C,UAAU9rD,EAAS5nC,KAAKs0D,UAAUlW,IAAI4J,YADAyrC,EAAO,OAEpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmyD,QAAQ/T,IAAIujB,UAAoB8xB,EAAO,cACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAK03D,YAAY7H,KAAoB4jC,EAAO,eACpEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKo+C,IAAIY,UAA4By0C,EAAO,cAEtE,CACLjzC,MAAAA,EACAzI,KAAMA,EAAOA,EAAK3kB,GAAK,KACvBwsC,YAAW7nB,KAASA,EAAK6nB,UACzBppB,MAAOuB,EAAOA,EAAKvB,OAAS,GAAI,KAChC6tB,MAAOA,EAAQA,EAAMlI,QAAU,KAC/BxM,WAAYA,EAAaA,EAAWxgC,QAAQiE,GAAK,KACjDqgE,KAAAA,EACAE,MAAOnzC,EAAM4D,SAAW5D,EAAM4D,SAASuvC,MAAQnzC,EAAMmzC,MACrDC,MAAOpzC,EAAM4D,SAAW5D,EAAM4D,SAASwvC,MAAQpzC,EAAMozC,MACrDzpF,EAAAA,EACAZ,EAAAA,EACAmX,KAAAA,EACAqwC,YAAAA,+CAQE/wD,KAAK05C,MAAMqD,aACRrD,MAAMkF,eAEqBn/C,MAA5BO,KAAKmvB,QAAQquB,kBACVgB,WAAWx+C,KAAKmvB,cAElBuqB,MAAM6E,kDASb9C,GAAKt/C,UAAU41D,QAAQ11D,KAAK2D,qCAQvBgD,OACGw9B,EAA0Bx9B,EAA1Bw9B,MAAOC,EAAmBz9B,EAAnBy9B,IAAKse,EAAc/7C,EAAd+7C,UACfte,OAKEiZ,MAAMgF,SAASle,EAAOC,EAAK,CAC9Bse,UAAWA,SALRgE,OAAOviB,EAAMjjC,UAAW,CAC3BwhD,UAAAA,WA7tB8BtD,IA4uBtC,SAASw3C,GAASl7C,UACTO,GAAKpB,QAAQa,EAAKxkC,KAAKitB,MAAO,QAAQjjC,UAQ/C,SAAS21F,GAAOn7C,OACRtX,EAAuBhhC,MAAjBs4C,EAAKxkC,KAAKktB,IAAmBsX,EAAKxkC,KAAKktB,IAAMsX,EAAKxkC,KAAKitB,aAC5D8X,GAAKpB,QAAQzW,EAAK,QAAQljC,UAQnC,SAAS+0F,GAAsBuB,EAAU97C,OAClCA,EAAKnnB,cAED,MAGHkjE,EAAgBD,EAAS1kE,QAAQguB,IAAM02C,EAASx+C,MAAM8b,eAAetoB,OAASgrD,EAASx+C,MAAM6b,cAAcroB,OAC3G8tB,EAAgBk9B,EAASx+C,MAAM2J,OAAOnW,OAEtCw7B,EAAQtsB,EAAKnnB,OACfpU,EAAS6nD,EAAMvhB,IACfyvC,GAAe,EACblqC,EAAcwrC,EAASprC,SAASt5B,QAAQk5B,YAAYC,KAEpDkqC,EAAU,iBACG,UAAfnqC,EACOgc,EAAMx7B,OAASkP,EAAK+K,IAAM/K,EAAKlP,OAG/BkP,EAAK+K,KAIVixC,GAAkD,EAA5BF,EAASr7B,gBAC/Bw7B,EAAex3E,EAASg2E,IACxB3pD,EAASkP,EAAKlP,cAEhBmrD,EAAeD,EACbv3E,EAASs3E,GAAiBt3E,EAASg2E,IAAY3pD,IACjDrsB,GAAUg2E,IAAYqB,EAAS1hC,QAAQhjC,QAAQmrC,OAAOviB,KAAK8iB,UAGtDm5B,EAAenrD,EAASkrD,EAAsBD,EACrDt3E,GAAUg2E,IAAY3pD,EAASirD,EAAgBD,EAAS1hC,QAAQhjC,QAAQmrC,OAAOviB,KAAK8iB,SAGpF03B,GAAe,EAKV,CAAEA,aAAAA,EAAcE,aAFvBj2E,EAAShb,KAAKY,IAAIoa,EAAQm6C,EAAgBm9B,GAEGtB,QAASwB,OC7zBlDC,yBAcFzzD,EACAC,EACAyzD,EACAC,EACAt9B,EACA1N,OACAirC,0DACAC,6EACKC,WAAa,CAAC,EAAG,EAAG,EAAG,SACvBC,WAAa,CAAC,IAAM,GAAK,EAAG,QAC5BC,YAAc,UAEd39B,gBAAkBA,OAClB1N,gBAAkBA,OAClBzD,OAASllB,OACT0a,KAAOza,OAEPkb,MAAQ,OACR84C,cAAgB,OAChBC,gBAAkB,OAClBC,sBAEAP,UAAYA,OACZF,eAAiBA,OACjBC,aAAeA,OAEfE,mBAAqBA,EAEtBH,GAAkBC,EAAc,KAC5Bx1C,EAAK3+C,KACL+lD,EAAe,SAAAjkD,OACb8yF,EAAU9yF,EAASA,GAAS68C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,sBACpE3yF,GAAS68C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,eAAwB91C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,cAA7C,GAC3DG,EAAWj2C,EAAG+1C,gBAAkB/1C,EAAG41C,WAAW51C,EAAG81C,cAGjDG,GAGPV,SACGxuC,QAAiC,EAAvB1lD,KAAK00F,gBAAsB10F,KAAKu0F,WAAWv0F,KAAKy0F,mBAC1D/uC,OAASK,EAAa/lD,KAAK0lD,SAG9ByuC,SACGj5C,MAAQl7C,KAAK00F,gBAAkB10F,KAAKu0F,WAAWv0F,KAAKy0F,mBACpDv5C,KAAO6K,EAAa/lD,KAAKk7C,YAE3By5C,mEAQKxrC,QACPA,gBAAkBA,oCAOf0N,QACHA,gBAAkBA,+CAOjBnd,EAAQ15C,KAAKk7C,KAAOl7C,KAAK0lD,YAC1B/J,MAAQ37C,KAAK62D,gBAAkBnd,MAC9Bm7C,EAAmB70F,KAAKmpD,gBAAkBnpD,KAAK27C,MAC/Cm5C,EAAoBp7C,EAAQ,EAC5Bl4C,KAAK+a,MAAM/a,KAAKw4C,IAAIN,GAASl4C,KAAKuzF,MAClC,OAEDN,cAAgB,OAChBC,gBAAkBlzF,KAAK6D,IAAI,GAAIyvF,OAEhCt0D,EAAQ,EACRs0D,EAAmB,IACrBt0D,EAAQs0D,WAGNE,GAAgB,EACXp+E,EAAI4pB,EAAOh/B,KAAKc,IAAIsU,IAAMpV,KAAKc,IAAIwyF,GAAmBl+E,IAAK,MAC7D89E,gBAAkBlzF,KAAK6D,IAAI,GAAIuR,OAC/B,IAAI7C,EAAI,EAAGA,EAAI/T,KAAKu0F,WAAWv3F,OAAQ+W,IAAK,IAC9B/T,KAAK00F,gBAAkB10F,KAAKu0F,WAAWxgF,IACxC8gF,EAAkB,CAChCG,GAAgB,OACXP,aAAe1gF,aAIF,IAAlBihF,0CAWClzF,UACCA,GAAS9B,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,gBAAmB,2CAQzEz0F,KAAK00F,gBAAkB10F,KAAKu0F,WAAWv0F,KAAKy0F,0DAQ7CQ,EAAYj1F,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,qBACvDz0F,KAAKk1F,aAAal1F,KAAK0lD,QAAWuvC,EAAaj1F,KAAK0lD,OAASuvC,GAAcA,uCAQxEh6C,OACNk6C,EAAcl6C,EAAQm6C,YAAY,SACC,mBAA5Bp1F,KAAKq0F,qBACdc,EAAcn1F,KAAKq0F,mBAAmBp5C,IAGb,iBAAhBk6C,YACCA,GAEoB,iBAAhBA,EACPA,EAGAl6C,EAAQm6C,YAAY,8CAUvBntC,EAAQ,GACRlkB,EAAO/jC,KAAK6wD,UACZwkC,GAAgBtxD,EAAQ/jC,KAAK0lD,OAAS3hB,GAASA,EAC5CjnC,EAAKkD,KAAK0lD,OAAS2vC,EAAer1F,KAAKk7C,KAAKp+C,EAAI,KAASA,GAAKinC,EACjEjnC,GAAKkD,KAAK0lD,QACZuC,EAAMhrD,KAAK,CAACq4F,MAAOt1F,KAAKu1F,SAASz4F,GAAIyM,EAAGvJ,KAAKk1F,aAAap4F,GAAIyD,IAAKP,KAAKw1F,YAAY14F,YAGjFmrD,sCAOGltC,OACJ06E,EAAaz1F,KAAKy0F,aAClBiB,EAAW11F,KAAK0lD,OAChBiwC,EAAS31F,KAAKk7C,KAEdyD,EAAK3+C,KACL41F,EAAoB,WACxBj3C,EAAG+1C,iBAAmB,GAElBmB,EAAoB,WACxBl3C,EAAG+1C,iBAAmB,GAGnB35E,EAAM05E,cAAgB,GAAKz0F,KAAKy0F,cAAgB,GAAO15E,EAAM05E,aAAe,GAAKz0F,KAAKy0F,aAAe,IAE/F15E,EAAM05E,aAAez0F,KAAKy0F,mBAE9BA,aAAe,EACF,GAAdgB,GAGFG,IAFAA,WAOGnB,aAAe,EACF,GAAdgB,GAGFI,IAFAA,cAQEC,EAAY/6E,EAAMm6E,aAAa,GAC/Ba,EAAYh7E,EAAM81C,UAAY91C,EAAM4gC,MAEtCxe,GAAO,EACP8b,EAAQ,GAEJ9b,GAAQ8b,IAAS,GAAG,MAGrB0C,MAAQo6C,GAAa/1F,KAAKu0F,WAAWv0F,KAAKy0F,cAAgBz0F,KAAK00F,qBAC9DsB,EAAWh2F,KAAK62D,gBAAkB72D,KAAK27C,WAGxC+J,OAASgwC,OACTx6C,KAAOl7C,KAAK0lD,OAASswC,MAEpBC,EAAiBj2F,KAAKk7C,KAAOl7C,KAAK27C,MAClCs5C,EAAYj1F,KAAK00F,gBAAkB10F,KAAKs0F,WAAWt0F,KAAKy0F,cACxDyB,EAAcl2F,KAAKm2F,gBAAkBp7E,EAAMo7E,mBAE7Cn2F,KAAKo0F,UAAW,KACZgC,EAAaN,EAAYG,OAC1B/6C,MAASk7C,EAAap2F,KAAK27C,WAC3B+J,OAAS1lD,KAAKk7C,KAAO86C,OAErBh2F,KAAKk0F,qBAIHxuC,QAAUwwC,EAAcl2F,KAAK27C,WAC7BT,KAAOl7C,KAAK0lD,OAASswC,SAJrBtwC,QAAUuvC,EAAaiB,EAAcl2F,KAAK27C,WAC1CT,KAAOl7C,KAAK0lD,OAASswC,OAMzBh2F,KAAKm0F,cAAgBn0F,KAAKk7C,KAAOy6C,EAAO,KAE3CE,IACA14D,GAAO,WAGJn9B,KAAKk0F,gBAAkBl0F,KAAK0lD,OAASgwC,EAAS,KAAS,MACtD11F,KAAKo0F,WAAasB,GAAY,GAE3B,CAELG,IACA14D,GAAO,WAJPx6B,QAAQH,KAAK,uDAQbxC,KAAKk0F,gBAAkBl0F,KAAKm0F,cAAgB6B,EAAYL,EAAOD,GACjEE,IACAz4D,GAAO,GAGTA,GAAO,yCASEr7B,UACJ9B,KAAK62D,iBAAoB/0D,EAAQ9B,KAAK0lD,QAAU1lD,KAAK27C,4CAQhD06C,UACHr2F,KAAK62D,gBAAkBw/B,GAAUr2F,KAAK27C,MAAS37C,KAAK0lD,4+LCjS3D4wC,0BAUQ19C,EAAMzpB,EAASonE,EAAKC,kDAEzBpjE,GAAKs3C,OACL9xB,KAAOA,IAEPsE,eAAiB,CACpBmL,YAAa,OACbE,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfmwC,OAAO,EACPC,iBAAkB,EAClBC,iBAAkB,EAClBC,aAAc,GACdC,aAAc,EACdC,UAAW,GACXluD,MAAO,OACP0yB,SAAS,EACTy7B,YAAY,EACZriD,KAAM,CACJgF,MAAO,CAACt3C,SAAK3C,EAAW6F,SAAK7F,GAC7BhC,gBAAOqE,mBACKg/C,GAAWh/C,EAAMszF,YAAY,MAEzC3lC,MAAO,CAAChE,UAAMhsD,EAAW+0B,WAAO/0B,IAElCk1C,MAAO,CACL+E,MAAO,CAACt3C,SAAK3C,EAAW6F,SAAK7F,GAC7BhC,gBAAOqE,mBACKg/C,GAAWh/C,EAAMszF,YAAY,MAEzC3lC,MAAO,CAAChE,UAAMhsD,EAAW+0B,WAAO/0B,OAI/B+2F,iBAAmBA,IACnBQ,aAAeT,IACflhD,MAAQ,KACR4hD,YAAc,CACjBhvC,MAAO,GACPivC,OAAQ,GACRznC,MAAO,MAGJrR,IAAM,KACNzC,WAAQl8C,IACRi6C,MAAQ,CAAClZ,MAAO,EAAGC,IAAK,KAExBtR,QAAUmpB,GAAKh7C,OAAO,GAAIogD,EAAKR,kBAC/Bi6C,iBAAmB,IAEnB34C,WAAWrvB,KACXyZ,MAAQyO,OAAO,UAAIqG,EAAKvuB,QAAQyZ,OAASriC,QAAQ,KAAM,OACvDqlE,SAAWluB,EAAK9U,QAChBC,OAAS6U,EAAKs5C,aAAap0C,wBAAwB/Z,SACnD0R,QAAS,IAET68C,WAAa,KACbC,cAAgB,IAChBC,eAAiB,IAEjB9rB,WAAa,IACb+rB,QAAS,IACTC,WAAa,OACbC,YAAc,KACdC,cAAe,IAEf/qB,OAAS,KACTgrB,eAAiB,IAGjBjvC,UACajpD,MAAdi+C,EAAK/B,SACFi8C,kBAEFC,UAAY,CAACtB,IAAK74C,EAAK64C,IAAKkB,YAAa/5C,EAAK+5C,YAAatoE,QAASuuB,EAAKvuB,QAASw9C,OAAQjvB,EAAKivB,YAE9FhuB,iBACD/F,KAAKiF,QAAQvH,GAAG,gBAAgB,WACnCqI,EAAGP,IAAI05C,cAActjE,MAAMsuB,cAASnE,EAAG/F,KAAKY,SAASoY,yEAShDrpB,EAAOwvD,GACT/3F,KAAK2sE,OAAOtvE,eAAekrC,UACzBokC,OAAOpkC,GAASwvD,QAElBJ,gBAAkB,sCAQbpvD,EAAOwvD,GACZ/3F,KAAK2sE,OAAOtvE,eAAekrC,UACzBovD,gBAAkB,QAEpBhrB,OAAOpkC,GAASwvD,sCAOXxvD,GACNvoC,KAAK2sE,OAAOtvE,eAAekrC,YACtBvoC,KAAK2sE,OAAOpkC,QACdovD,gBAAkB,sCAQhBxoE,MACLA,EAAS,KACPkhC,GAAS,EACTrwD,KAAKmvB,QAAQk5B,aAAel5B,EAAQk5B,kBAAuC5oD,IAAxB0vB,EAAQk5B,cAC7DgI,GAAS,GAkBX/X,GAAKqQ,oBAhBU,CACb,cACA,kBACA,kBACA,QACA,mBACA,mBACA,eACA,eACA,YACA,QACA,UACA,OACA,QACA,cAE+B3oD,KAAKmvB,QAASA,QAE1Cy8C,SAAWv0B,OAAO,UAAIr3C,KAAKmvB,QAAQyZ,OAASriC,QAAQ,KAAM,MAChD,IAAX8pD,GAAmBrwD,KAAKo+C,IAAIuqB,aACzBzY,YACAsT,gDASJplB,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKmvB,QAAQyZ,WACrCwV,IAAIuqB,MAAMn0C,MAAMqU,OAAS7oC,KAAK6oC,YAE9BuV,IAAI05C,cAAgB1qE,SAASE,cAAc,YAC3C8wB,IAAI05C,cAActjE,MAAMoU,MAAQ,YAChCwV,IAAI05C,cAActjE,MAAMqU,OAAS7oC,KAAK6oC,YACtCuV,IAAI05C,cAActjE,MAAMkL,SAAW,gBACnC0e,IAAI05C,cAActjE,MAAMyqB,WAAa,eACrCb,IAAI05C,cAActjE,MAAMC,QAAU,aAGlC8hE,IAAMnpE,SAASya,gBAAgB,6BAA8B,YAC7D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMsuB,IAAM,WAChByzC,IAAI/hE,MAAMqU,OAAS,YACnB0tD,IAAI/hE,MAAMoU,MAAQ,YAClB2tD,IAAI/hE,MAAMC,QAAU,aACpB2pB,IAAIuqB,MAAMh0C,YAAY30B,KAAKu2F,qDAS5BpsF,EAFJ6tF,GAAwBh4F,KAAKy3F,iBAGvBX,EAAY92F,KAAKmvB,QAAQ2nE,UAG3BvtF,EAAI0uF,KAGN9tF,EAD+B,SAA7BnK,KAAKmvB,QAAQk5B,YAHE,EAObroD,KAAK4oC,MAAQkuD,EAPA,MAUboB,EAAal4C,GAAYhgD,KAAK2sE,WACpCurB,QAAAA,GAAgB,SAAC/6F,EAAGC,UAAMD,EAAIC,GAAK,EAAI,+CAEjB86F,+BAAY,KAAvB/7B,WAC4B,IAAjCn8D,KAAK2sE,OAAOxQ,GAASb,cAAmE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,KAAwE,IAA9Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,UACnIwQ,OAAOxQ,GAASg8B,UAAUrB,EAhBhB,GAgBuC92F,KAAK63F,UAAW1tF,EAAGZ,GACzEA,GAAK6uF,uFAITJ,GAAwBh4F,KAAKy3F,kBACxBC,cAAe,2CAOM,IAAtB13F,KAAK03F,eACPM,GAAwBh4F,KAAKy3F,aAC7BO,GAAwBh4F,KAAKy3F,kBACxBC,cAAe,uCAQjBn9C,QAAS,EACTv6C,KAAKo+C,IAAIuqB,MAAMphC,aACe,SAA7BvnC,KAAKmvB,QAAQk5B,iBACVzP,KAAKwF,IAAI1J,KAAK/f,YAAY30B,KAAKo+C,IAAIuqB,YAGnC/vB,KAAKwF,IAAIzJ,MAAMhgB,YAAY30B,KAAKo+C,IAAIuqB,QAIxC3oE,KAAKo+C,IAAI05C,cAAcvwD,iBACrBqR,KAAKwF,IAAI6S,qBAAqBt8B,YAAY30B,KAAKo+C,IAAI05C,oBAErD15C,IAAI05C,cAActjE,MAAMC,QAAU,4CAOlC8lB,QAAS,EACVv6C,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,YAG5CvqB,IAAI05C,cAActjE,MAAMC,QAAU,wCAQhC+L,EAAOC,QACTiZ,MAAMlZ,MAAQA,OACdkZ,MAAMjZ,IAAMA,uCAQboY,GAAU,EACVw/C,EAAe,MAKd,IAAMl8B,UAFN/d,IAAI05C,cAActjE,MAAMsuB,cAAS9iD,KAAK44C,KAAKY,SAASoY,gBAEnC5xD,KAAK2sE,OACrB3sE,KAAK2sE,OAAOtvE,eAAe8+D,MACQ,IAAjCn8D,KAAK2sE,OAAOxQ,GAASb,cAAmE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,KAAwE,IAA9Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,IACxIk8B,QAIsB,IAAxBr4F,KAAK23F,gBAAyC,IAAjBU,OAC1BnoC,WAEF,MACEsT,YACA36B,OAASwO,OAAOr3C,KAAKg3F,aAAaxiE,MAAMqU,OAAOtiC,QAAQ,KAAM,UAG7D63C,IAAI05C,cAActjE,MAAMqU,iBAAY7oC,KAAK6oC,kBACzCD,OAAiC,IAAzB5oC,KAAKmvB,QAAQmsC,QAAmBjkB,OAAO,UAAIr3C,KAAKmvB,QAAQyZ,OAASriC,QAAQ,KAAM,KAAO,MAE7F8uC,EAAQr1C,KAAKq1C,MACbszB,EAAQ3oE,KAAKo+C,IAAIuqB,MAGvBA,EAAMrgC,UAAY,qBAGbygB,yBAECV,EAAcroD,KAAKmvB,QAAQk5B,YAC3BE,EAAkBvoD,KAAKmvB,QAAQo5B,gBAC/BtC,EAAkBjmD,KAAKmvB,QAAQ82B,gBAE/BqyC,EAAkCt4F,KAAK44C,KAAKwF,IAAI6S,qBAAqB7H,YAG3E/T,EAAM2T,iBAAmBT,EAAkBlT,EAAM4T,gBAAkB,EACnE5T,EAAM6T,iBAAmBjD,EAAkB5Q,EAAM8T,gBAAkB,EAEnE9T,EAAMiU,eAAiBgvC,EAAkCt4F,KAAKwrE,WAAaxrE,KAAK4oC,MAAQ,EAAI5oC,KAAKmvB,QAAQwnE,iBACzGthD,EAAMgU,gBAAkB,EACxBhU,EAAMmU,eAAiB8uC,EAAkCt4F,KAAKwrE,WAAaxrE,KAAK4oC,MAAQ,EAAI5oC,KAAKmvB,QAAQunE,iBACzGrhD,EAAMkU,gBAAkB,EAGJ,SAAhBlB,GACFsgB,EAAMn0C,MAAMsuB,IAAM,IAClB6lB,EAAMn0C,MAAMkgB,KAAO,IACnBi0B,EAAMn0C,MAAMq0B,OAAS,GACrB8f,EAAMn0C,MAAMoU,gBAAW5oC,KAAK4oC,YAC5B+/B,EAAMn0C,MAAMqU,iBAAY7oC,KAAK6oC,kBACxBwM,MAAMzM,MAAQ5oC,KAAK44C,KAAKY,SAAS9E,KAAK9L,WACtCyM,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAAS9E,KAAK7L,SAG5C8/B,EAAMn0C,MAAMsuB,IAAM,GAClB6lB,EAAMn0C,MAAMq0B,OAAS,IACrB8f,EAAMn0C,MAAMkgB,KAAO,IACnBi0B,EAAMn0C,MAAMoU,gBAAW5oC,KAAK4oC,YAC5B+/B,EAAMn0C,MAAMqU,iBAAY7oC,KAAK6oC,kBACxBwM,MAAMzM,MAAQ5oC,KAAK44C,KAAKY,SAAS7E,MAAM/L,WACvCyM,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAAS7E,MAAM9L,QAG/CgQ,EAAU74C,KAAK43F,gBACf/+C,EAAU74C,KAAK8pD,cAAgBjR,GAEJ,IAAvB74C,KAAKmvB,QAAQsnE,WACV8B,yBAGAC,qBAGFC,aAAapwC,UAEbxP,qDAUHA,GAAU,EACdm/C,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,YACnC7uC,EAAcroD,KAAKmvB,QAAL,YACdupE,EAAiDj5F,MAAnCO,KAAKmvB,QAAQk5B,GAAa3O,MAAqB15C,KAAKmvB,QAAQk5B,GAAa3O,MAAQ,GAGjGy6C,GAAe,EACI10F,MAAnBi5F,EAAYpzF,WACTo0C,MAAMjZ,IAAMi4D,EAAYpzF,IAC7B6uF,GAAe,OAEbD,GAAiB,EACEz0F,MAAnBi5F,EAAYt2F,WACTs3C,MAAMlZ,MAAQk4D,EAAYt2F,IAC/B8xF,GAAiB,QAGdv4C,MAAQ,IAAIs4C,GACfj0F,KAAK05C,MAAMlZ,MACXxgC,KAAK05C,MAAMjZ,IACXyzD,EACAC,EACAn0F,KAAKo+C,IAAIuqB,MAAMjS,aACf12D,KAAKq1C,MAAM8T,gBACXnpD,KAAKmvB,QAAQ4nE,WACb/2F,KAAKmvB,QAAQk5B,GAAa5qD,SAGR,IAAhBuC,KAAKu3F,QAAuC93F,MAAnBO,KAAKw3F,iBAC3B77C,MAAMg9C,YAAY34F,KAAKw3F,WAAW77C,YAClCyC,IAAI05C,cAActjE,MAAMC,QAAU,aAElC2pB,IAAI05C,cAActjE,MAAMC,QAAU,aAIpCmkE,aAAe,MAEd3wC,EAAQjoD,KAAK27C,MAAMk9C,cACzB5wC,QAAAA,GACE,SAAAuC,OACQjhD,EAAIihD,EAAKjhD,EACT8gD,EAAUG,EAAK8qC,MACjB3/B,EAAKxmC,QAAL,kBAA+C,IAAZk7B,GACrCsL,EAAKmjC,aAAavvF,EAAI,EAAGihD,EAAKjqD,IAAK8nD,EAAa,uBAAwBsN,EAAKtgB,MAAM4T,iBAEjFoB,GACE9gD,GAAK,GACPosD,EAAKmjC,aAAavvF,EAAI,EAAGihD,EAAKjqD,IAAK8nD,EAAa,uBAAwBsN,EAAKtgB,MAAM8T,kBAGnE,IAAhBwM,EAAK4hC,SACHltC,EACFsL,EAAKojC,YAAYxvF,EAAG8+C,EAAa,oCAAqCsN,EAAKxmC,QAAQunE,iBAAkB/gC,EAAKtgB,MAAMmU,gBAGhHmM,EAAKojC,YAAYxvF,EAAG8+C,EAAa,oCAAqCsN,EAAKxmC,QAAQwnE,iBAAkBhhC,EAAKtgB,MAAMiU,wBAMpH0vC,EAAa,OACuBv5F,IAApCO,KAAKmvB,QAAQk5B,GAAaoH,YAAgEhwD,IAAzCO,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,OACnFutC,EAAah5F,KAAKq1C,MAAM4jD,qBAEpBz8E,GAAgC,IAAvBxc,KAAKmvB,QAAQsnE,MAAiBj1F,KAAK8D,IAAItF,KAAKmvB,QAAQ2nE,UAAWkC,GAAch5F,KAAKmvB,QAAQynE,aAAe,GAAKoC,EAAah5F,KAAKmvB,QAAQynE,aAAe,UAGlK52F,KAAK44F,aAAgB54F,KAAK4oC,MAAQpsB,IAAoC,IAAzBxc,KAAKmvB,QAAQmsC,cACvD1yB,MAAQ5oC,KAAK44F,aAAep8E,OAC5B2S,QAAQyZ,gBAAW5oC,KAAK4oC,YAC7BovD,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,aACpC7mC,SACLxX,GAAU,GAGH74C,KAAK44F,aAAgB54F,KAAK4oC,MAAQpsB,IAAoC,IAAzBxc,KAAKmvB,QAAQmsC,SAAoBt7D,KAAK4oC,MAAQ5oC,KAAK4rE,eAClGhjC,MAAQpnC,KAAK8D,IAAItF,KAAK4rE,SAAU5rE,KAAK44F,aAAep8E,QACpD2S,QAAQyZ,gBAAW5oC,KAAK4oC,YAC7BovD,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,aACpC7mC,SACLxX,GAAU,IAGVm/C,GAAwBh4F,KAAKi3F,YAAYhvC,OACzC+vC,GAAwBh4F,KAAKi3F,YAAYC,QACzCr+C,GAAU,GAGLA,uCAQI/2C,UACJ9B,KAAK27C,MAAMu5C,aAAapzF,yCAQnBqI,UACLnK,KAAK27C,MAAMu9C,cAAc/uF,wCAarBZ,EAAGkiD,EAAMpD,EAAa/f,EAAW6wD,OAEtC5wD,EAAQyvD,GAAsB,MAAOh4F,KAAKi3F,YAAYC,OAAQl3F,KAAKo+C,IAAIuqB,OAC7EpgC,EAAMD,UAAYA,EAClBC,EAAMmjB,UAAYD,EACE,SAAhBpD,GACF9f,EAAM/T,MAAMkgB,gBAAW10C,KAAKmvB,QAAQynE,mBACpCruD,EAAM/T,MAAM4kE,UAAY,UAGxB7wD,EAAM/T,MAAMmgB,iBAAY30C,KAAKmvB,QAAQynE,mBACrCruD,EAAM/T,MAAM4kE,UAAY,QAG1B7wD,EAAM/T,MAAMsuB,cAASv5C,EAAI,GAAM4vF,EAAkBn5F,KAAKmvB,QAAQ0nE,mBAE9DprC,GAAQ,OAEF4tC,EAAe73F,KAAK8D,IAAItF,KAAKq1C,MAAMmW,eAAgBxrD,KAAKq1C,MAAM2U,gBAChEhqD,KAAK44F,aAAentC,EAAKzuD,OAASq8F,SAC/BT,aAAentC,EAAKzuD,OAASq8F,uCAY1B9vF,EAAG8+C,EAAa/f,EAAW9rB,EAAQosB,OACzB,IAAhB5oC,KAAKu3F,OAAiB,KAClB/sC,EAAOwtC,GAAsB,MAAOh4F,KAAKi3F,YAAYhvC,MAAOjoD,KAAKo+C,IAAI05C,eAC3EttC,EAAKliB,UAAYA,EACjBkiB,EAAKkB,UAAY,GAEG,SAAhBrD,EACFmC,EAAKh2B,MAAMkgB,eAAU10C,KAAK4oC,MAAQpsB,QAGlCguC,EAAKh2B,MAAMmgB,gBAAW30C,KAAK4oC,MAAQpsB,QAGrCguC,EAAKh2B,MAAMoU,gBAAWA,QACtB4hB,EAAKh2B,MAAMsuB,cAASv5C,8CASX8+C,MACX2vC,GAAwBh4F,KAAKi3F,YAAYxnC,YAGDhwD,IAApCO,KAAKmvB,QAAQk5B,GAAaoH,YAAgEhwD,IAAzCO,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,KAAoB,KACjGgE,EAAQuoC,GAAsB,MAAOh4F,KAAKi3F,YAAYxnC,MAAOzvD,KAAKo+C,IAAIuqB,OAC5ElZ,EAAMnnB,6CAAwC+f,GAC9CoH,EAAM/D,UAAY1rD,KAAKmvB,QAAQk5B,GAAaoH,MAAMhE,UAGJhsD,IAA1CO,KAAKmvB,QAAQk5B,GAAaoH,MAAMj7B,OAClC8jB,GAAK+lB,WAAW5O,EAAOzvD,KAAKmvB,QAAQk5B,GAAaoH,MAAMj7B,OAGrC,SAAhB6zB,EACFoH,EAAMj7B,MAAMkgB,eAAU10C,KAAKq1C,MAAM4jD,sBAGjCxpC,EAAMj7B,MAAMmgB,gBAAW30C,KAAKq1C,MAAM4jD,sBAGpCxpC,EAAMj7B,MAAMoU,gBAAW5oC,KAAK6oC,aAI9BmvD,GAAwBh4F,KAAKi3F,YAAYxnC,yDAUnC,oBAAqBzvD,KAAKq1C,OAAQ,KAChCikD,EAAYlsE,SAAS06B,eAAe,KACpCgE,EAAmB1+B,SAASE,cAAc,OAChDw+B,EAAiBxjB,UAAY,mCAC7BwjB,EAAiBn3B,YAAY2kE,QACxBl7C,IAAIuqB,MAAMh0C,YAAYm3B,QAEtBzW,MAAM4T,gBAAkB6C,EAAiBC,kBACzC1W,MAAM2U,eAAiB8B,EAAiBnL,iBAExCvC,IAAIuqB,MAAMnhC,YAAYskB,QAGvB,oBAAqB9rD,KAAKq1C,OAAQ,KAChCkkD,EAAYnsE,SAAS06B,eAAe,KACpCkE,EAAmB5+B,SAASE,cAAc,OAChD0+B,EAAiB1jB,UAAY,mCAC7B0jB,EAAiBr3B,YAAY4kE,QACxBn7C,IAAIuqB,MAAMh0C,YAAYq3B,QAEtB3W,MAAM8T,gBAAkB6C,EAAiBD,kBACzC1W,MAAMmW,eAAiBQ,EAAiBrL,iBAExCvC,IAAIuqB,MAAMnhC,YAAYwkB,QAGvB,oBAAqBhsD,KAAKq1C,OAAQ,KAChCmkD,EAAYpsE,SAAS06B,eAAe,KACpC2xC,EAAmBrsE,SAASE,cAAc,OAChDmsE,EAAiBnxD,UAAY,mCAC7BmxD,EAAiB9kE,YAAY6kE,QACxBp7C,IAAIuqB,MAAMh0C,YAAY8kE,QAEtBpkD,MAAM4jD,gBAAkBQ,EAAiB1tC,kBACzC1W,MAAMqkD,eAAiBD,EAAiB94C,iBAExCvC,IAAIuqB,MAAMnhC,YAAYiyD,WA5mBV9gD,ICCvB,SAASghD,GAAOx9B,EAAShtC,IAiDzB,SAASyqE,GAAiBv1B,EAAOw1B,SAExB,CACLrlE,OAFFqlE,OAA4C,IAAnBA,EAAkC,GAAKA,GAExCrlE,OAAS6vC,EAAMl1C,QAAQqlC,WAAWhgC,MACxD6T,OAAQwxD,EAAexxD,QAAUg8B,EAAMl1C,QAAQqlC,WAAWnsB,OAC1D1I,KAAMk6D,EAAel6D,MAAQ0kC,EAAMl1C,QAAQqlC,WAAW70B,KACtD2I,UAAWuxD,EAAevxD,WAAa+7B,EAAM/7B,WCvDjD,SAASwxD,GAAS39B,EAAShtC,ICD3B,SAAS4qE,GAAK59B,EAAShtC,ICOvB,SAAS6qE,GAAW31B,EAAOlI,EAAShtC,EAAS8qE,QACtC7mE,GAAK+oC,OAELhtC,QAAUmpB,GAAK4hD,sBADP,CAAC,WAAY,QAAS,OAAQ,mBAAoB,WAAY,aAAc,SAAU,gBAAiB,SAAS,sBAAuB,qBAClG/qE,QAC7CgrE,uBAAwC16F,IAApB4kE,EAAM/7B,eAC1B2xD,yBAA2BA,OAC3BG,aAAe,OACfjkD,OAAOkuB,GACkB,GAA1BrkE,KAAKm6F,yBACFF,yBAAyB,IAAM,QAEjCx0B,UAAY,QACZnK,aAA4B77D,IAAlB4kE,EAAM/I,SAA+B+I,EAAM/I,QCb5D,SAAS++B,GAAOzhD,EAAMzpB,EAASmrE,EAAM9D,QAC9B59C,KAAOA,OACPsE,eAAiB,CACpBw/B,SAAS,EACT+Z,OAAO,EACP8D,SAAU,GACVC,YAAa,EACb9lD,KAAM,CACJ4mB,SAAS,EACT57B,SAAU,YAEZiV,MAAO,CACL2mB,SAAS,EACT57B,SAAU,mBAIT46D,KAAOA,OACPnrE,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,qBAC/Bs5C,iBAAmBA,OAEnBiB,YAAc,QACdr5C,IAAM,QACNuuB,OAAS,QACTgrB,eAAiB,OACjBjvC,eACAmvC,UAAY,CAACtB,IAAKv2F,KAAKu2F,IAAKkB,YAAaz3F,KAAKy3F,YAAatoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ3sE,KAAK2sE,aAE/FnuB,WAAWrvB,GJtBlBwqE,GAAOjK,KAAO,SAAU5W,EAASzU,EAAOwzB,EAAWr7E,GACjDA,EAASA,GAAU,UACfxW,EAoDN,SAAqB6xF,EAAWxzB,OAC1Br+D,OAAWvG,EAEXo4F,EAAU1oE,SAAW0oE,EAAU1oE,QAAQqlC,YAAcqjC,EAAU1oE,QAAQqlC,WAAWC,UAA4D,mBAAzCojC,EAAU1oE,QAAQqlC,WAAWC,WACpIzuD,EAAW6xF,EAAU1oE,QAAQqlC,WAAWC,UAItC4P,EAAMA,MAAMl1C,SAAWk1C,EAAMA,MAAMl1C,QAAQqlC,YAAc6P,EAAMA,MAAMl1C,QAAQqlC,WAAWC,UAA8D,mBAA3C4P,EAAMA,MAAMl1C,QAAQqlC,WAAWC,WAC5IzuD,EAAWq+D,EAAMA,MAAMl1C,QAAQqlC,WAAWC,iBAErCzuD,EA/DQy0F,CAAY5C,EAAWxzB,GAE7BvnE,EAAI,EAAGA,EAAIg8E,EAAQ97E,OAAQF,OAC7BkJ,EAIA,KACC6zF,EAAiB7zF,EAAS8yE,EAAQh8E,GAAIunE,IACnB,IAAnBw1B,GAAqD,WAA1Bp0D,GAAOo0D,IACpC7B,GAAkBlf,EAAQh8E,GAAG49F,SAAWl+E,EAAQs8D,EAAQh8E,GAAG69F,SAAUf,GAAiBv1B,EAAOw1B,GAAiBhC,EAAUJ,YAAaI,EAAUtB,IAAKzd,EAAQh8E,GAAGyrC,YALjKyvD,GAAkBlf,EAAQh8E,GAAG49F,SAAWl+E,EAAQs8D,EAAQh8E,GAAG69F,SAAUf,GAAiBv1B,GAAQwzB,EAAUJ,YAAaI,EAAUtB,IAAKzd,EAAQh8E,GAAGyrC,QAWrJoxD,GAAOiB,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAC1DgD,EAA0B,GAAbzC,EAEb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,eAGtC4vD,GAAkB7tF,EAAI,GAAM2sF,EAAWvtF,EAAGqwF,GAAiBv1B,GAAQwzB,EAAUJ,YAAaI,EAAUtB,MCrCtGuD,GAASc,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAC5DgD,EAA0B,GAAbzC,EACb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,mBAElC2yD,EAAWv5F,KAAK+a,MAAM,GAAMu6E,GAE5Bn7C,EADgB0oB,EAAMl1C,QAAQ6rE,SAASpyD,MACfmyD,EACxBE,EAAaz5F,KAAK+a,MAAM,GAAM67E,GAC9B8C,EAAa15F,KAAK+a,MAAM,IAAO67E,GAE/B57E,EAAShb,KAAK+a,OAAOu6E,EAAa,EAAIiE,GAAa,MAEvD/C,GAAgB7tF,EAAI,GAAM4wF,EAAWv+E,EAAQjT,EAAIsxF,EAAaI,EAAa,EAAGF,EAAUE,EAAY52B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,OAC9KwjE,GAAgB7tF,EAAI,IAAM4wF,EAAWv+E,EAAS,EAAGjT,EAAIsxF,EAAaK,EAAa,EAAGH,EAAUG,EAAY72B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,OAE1I,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,QAAiB,KACxCx0C,EAAgB,CAClB1T,MAAO6vC,EAAMl1C,QAAQqlC,WAAWhgC,MAChC6T,OAAQg8B,EAAMl1C,QAAQqlC,WAAWnsB,OACjC1I,KAAO0kC,EAAMl1C,QAAQqlC,WAAW70B,KAAOgc,EACvCrT,UAAW+7B,EAAM/7B,WAEnB0vD,GAAkB7tF,EAAI,GAAM4wF,EAAWv+E,EAAQjT,EAAIsxF,EAAaI,EAAa,EAAG/yD,EAAe2vD,EAAUJ,YAAaI,EAAUtB,KAChIyB,GAAkB7tF,EAAI,IAAM4wF,EAAWv+E,EAAS,EAAGjT,EAAIsxF,EAAaK,EAAa,EAAGhzD,EAAe2vD,EAAUJ,YAAaI,EAAUtB,OAWxIuD,GAASpK,KAAO,SAAU1e,EAAUmqB,EAAoBtD,OAGlDuD,EACAn4F,EAAKo4F,EACLh3B,EACAvnE,EAAGiX,EALHunF,EAAe,GACfC,EAAgB,GAKhBC,EAAY,MAGX1+F,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEH,SAD5BunE,EAAQwzB,EAAUlrB,OAAOqE,EAASl0E,KACxBqyB,QAAQqF,QACM,IAAlB6vC,EAAM/I,eAA0E77D,IAArDo4F,EAAU1oE,QAAQw9C,OAAO1tB,WAAW+xB,EAASl0E,MAA0E,IAArD+6F,EAAU1oE,QAAQw9C,OAAO1tB,WAAW+xB,EAASl0E,SACvIiX,EAAI,EAAGA,EAAIonF,EAAmBnqB,EAASl0E,IAAIE,OAAQ+W,IACtDunF,EAAar+F,KAAK,CAChBy9F,SAAUS,EAAmBnqB,EAASl0E,IAAIiX,GAAG2mF,SAC7Ce,WAAYN,EAAmBnqB,EAASl0E,IAAIiX,GAAG0nF,WAC/Cd,SAAUQ,EAAmBnqB,EAASl0E,IAAIiX,GAAG4mF,SAC7CxwF,EAAGgxF,EAAmBnqB,EAASl0E,IAAIiX,GAAG5J,EACtCs2B,IAAK06D,EAAmBnqB,EAASl0E,IAAIiX,GAAG0sB,IACxCl3B,EAAG4xF,EAAmBnqB,EAASl0E,IAAIiX,GAAGxK,EACtC4yD,QAAS6U,EAASl0E,GAClByrC,MAAO4yD,EAAmBnqB,EAASl0E,IAAIiX,GAAGw0B,QAE5CizD,GAAa,KAMH,IAAdA,SAKJF,QAAAA,GAAkB,SAAUn+F,EAAGC,UACzBD,EAAEu9F,WAAat9F,EAAEs9F,SACZv9F,EAAEg/D,QAAU/+D,EAAE++D,SAAW,EAAI,EAG7Bh/D,EAAEu9F,SAAWt9F,EAAEs9F,YAK1BZ,GAAS4B,sBAAsBH,EAAeD,GAGzCx+F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,IAAK,KAEpC8uE,EAA8CnsE,OADlD4kE,EAAQwzB,EAAUlrB,OAAO2uB,EAAax+F,GAAGq/D,UACpBhtC,QAAQ6rE,SAASpvB,SAAwBvH,EAAMl1C,QAAQ6rE,SAASpvB,SAAW,GAAMvH,EAAMl1C,QAAQ6rE,SAASpyD,MAGzH+yD,EAAe,UACQl8F,IAAvB87F,EAFJt4F,EAAMq4F,EAAax+F,GAAG49F,UAGhB59F,EAAI,EAAIw+F,EAAat+F,SACvBo+F,EAAe55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWz3F,IAEzDo4F,EAAWvB,GAAS8B,iBAAiBR,EAAc/2B,EAAOuH,OAEvD,KACCzmB,EAAUroD,GAAKy+F,EAAct4F,GAAK44F,OAASN,EAAct4F,GAAK64F,UAC9D32C,EAAUm2C,EAAat+F,SACzBo+F,EAAe55F,KAAKc,IAAIg5F,EAAan2C,GAASu1C,SAAWz3F,IAE3Do4F,EAAWvB,GAAS8B,iBAAiBR,EAAc/2B,EAAOuH,GAC1D2vB,EAAct4F,GAAK64F,UAAY,GAEH,IAAxBz3B,EAAMl1C,QAAQ9rB,QAAwD,IAAtCghE,EAAMl1C,QAAQ4sE,oBAC5CT,EAAax+F,GAAG69F,SAAWt2B,EAAM+1B,cACnCuB,EAAeJ,EAAct4F,GAAK+4F,oBAClCT,EAAct4F,GAAK+4F,qBAAuB33B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,WAG/EgB,EAAeJ,EAAct4F,GAAKg5F,oBAClCV,EAAct4F,GAAKg5F,qBAAuB53B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,WAGpC,IAAtCt2B,EAAMl1C,QAAQ6rE,SAASkB,aAC9Bb,EAASzyD,MAAQyyD,EAASzyD,MAAQ2yD,EAAct4F,GAAK44F,OACrDR,EAAS7+E,QAAW++E,EAAct4F,GAAK64F,SAAYT,EAASzyD,MAAS,GAAMyyD,EAASzyD,OAAS2yD,EAAct4F,GAAK44F,OAAS,QAIzHM,EAAYd,EAASzyD,MACrBpI,EAAQ86D,EAAax+F,GAAG49F,YAGMj7F,MAA9B67F,EAAax+F,GAAG2+F,WAElBj7D,GAAsB,IADtB27D,EAAYb,EAAax+F,GAAG2+F,WAAaH,EAAax+F,GAAG49F,UAIzDl6D,GAAS66D,EAAS7+E,OAGpBw7E,GAAgBx3D,EAAO86D,EAAax+F,GAAG69F,SAAWgB,EAAcQ,EAAW93B,EAAM+1B,aAAekB,EAAax+F,GAAG69F,SAAUt2B,EAAM/7B,UAAY,WAAYuvD,EAAUJ,YAAaI,EAAUtB,IAAKlyB,EAAM7vC,QAG3J,IAArC6vC,EAAMl1C,QAAQqlC,WAAWkoB,QAAkB,KACzC0f,EAAY,CACd1B,SAAUY,EAAax+F,GAAG49F,SAC1BC,SAAUW,EAAax+F,GAAG69F,SAAWgB,EACrCxxF,EAAGmxF,EAAax+F,GAAGqN,EACnBZ,EAAG+xF,EAAax+F,GAAGyM,EACnB4yD,QAASm/B,EAAax+F,GAAGq/D,QACzB5zB,MAAO+yD,EAAax+F,GAAGyrC,OAEzBoxD,GAAOjK,KAAK,CAAC0M,GAAY/3B,EAAOwzB,EAAWwD,EAAS7+E,WAa1Ds9E,GAAS4B,sBAAwB,SAAUH,EAAeD,WAEpDF,EACKt+F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,IACnCA,EAAI,EAAIw+F,EAAat+F,SACvBo+F,EAAe55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWY,EAAax+F,GAAG49F,WAErE59F,EAAI,IACNs+F,EAAe55F,KAAKY,IAAIg5F,EAAc55F,KAAKc,IAAIg5F,EAAax+F,EAAI,GAAG49F,SAAWY,EAAax+F,GAAG49F,YAE3E,IAAjBU,SAC8C37F,IAA5C87F,EAAcD,EAAax+F,GAAG49F,YAChCa,EAAcD,EAAax+F,GAAG49F,UAAY,CACxCmB,OAAQ,EACRC,SAAU,EACVG,oBAAqB,EACrBD,oBAAqB,IAGzBT,EAAcD,EAAax+F,GAAG49F,UAAUmB,QAAU,IAexD/B,GAAS8B,iBAAmB,SAAUR,EAAc/2B,EAAOuH,OACrDhjC,EAAOpsB,SACP4+E,EAAe/2B,EAAMl1C,QAAQ6rE,SAASpyD,OAASwyD,EAAe,GAChExyD,EAAQwyD,EAAexvB,EAAWA,EAAWwvB,EAE7C5+E,EAAS,EAC4B,SAAjC6nD,EAAMl1C,QAAQ6rE,SAASv0B,MACzBjqD,GAAU,GAAM4+E,EAEwB,UAAjC/2B,EAAMl1C,QAAQ6rE,SAASv0B,QAC9BjqD,GAAU,GAAM4+E,KAKlBxyD,EAAQy7B,EAAMl1C,QAAQ6rE,SAASpyD,MAC/BpsB,EAAS,EAC4B,SAAjC6nD,EAAMl1C,QAAQ6rE,SAASv0B,MACzBjqD,GAAU,GAAM6nD,EAAMl1C,QAAQ6rE,SAASpyD,MAEC,UAAjCy7B,EAAMl1C,QAAQ6rE,SAASv0B,QAC9BjqD,GAAU,GAAM6nD,EAAMl1C,QAAQ6rE,SAASpyD,QAIpC,CAACA,MAAOA,EAAOpsB,OAAQA,IAGhCs9E,GAASuC,iBAAmB,SAAUf,EAAcgB,EAAatrB,EAAUurB,EAAYl0C,MACjFizC,EAAat+F,OAAS,EAAG,IAE3Bs+F,QAAAA,GAAkB,SAAUn+F,EAAGC,UACzBD,EAAEu9F,WAAat9F,EAAEs9F,SACZv9F,EAAEg/D,QAAU/+D,EAAE++D,SAAW,EAAI,EAG7Bh/D,EAAEu9F,SAAWt9F,EAAEs9F,gBAGtBa,EAAgB,GAEpBzB,GAAS4B,sBAAsBH,EAAeD,GAC9CgB,EAAYC,GAAczC,GAAS0C,kBAAkBjB,EAAeD,GACpEgB,EAAYC,GAAYE,iBAAmBp0C,EAC3C2oB,EAAS/zE,KAAKs/F,KAIlBzC,GAAS0C,kBAAoB,SAAUjB,EAAeD,WAChDr4F,EACAy5F,EAAOpB,EAAa,GAAGX,SACvBgC,EAAOrB,EAAa,GAAGX,SAClB79F,EAAI,EAAGA,EAAIw+F,EAAat+F,OAAQF,SAEZ2C,IAAvB87F,EADJt4F,EAAMq4F,EAAax+F,GAAG49F,WAEpBgC,EAAOA,EAAOpB,EAAax+F,GAAG69F,SAAWW,EAAax+F,GAAG69F,SAAW+B,EACpEC,EAAOA,EAAOrB,EAAax+F,GAAG69F,SAAWW,EAAax+F,GAAG69F,SAAWgC,GAGhErB,EAAax+F,GAAG69F,SAAW,EAC7BY,EAAct4F,GAAK+4F,qBAAuBV,EAAax+F,GAAG69F,SAG1DY,EAAct4F,GAAKg5F,qBAAuBX,EAAax+F,GAAG69F,aAI3D,IAAIiC,KAAQrB,EACXA,EAAcl+F,eAAeu/F,KAE/BF,GADAA,EAAOA,EAAOnB,EAAcqB,GAAMZ,oBAAsBT,EAAcqB,GAAMZ,oBAAsBU,GACpFnB,EAAcqB,GAAMX,oBAAsBV,EAAcqB,GAAMX,oBAAsBS,EAElGC,GADAA,EAAOA,EAAOpB,EAAcqB,GAAMZ,oBAAsBT,EAAcqB,GAAMZ,oBAAsBW,GACpFpB,EAAcqB,GAAMX,oBAAsBV,EAAcqB,GAAMX,oBAAsBU,SAI/F,CAACv6F,IAAKs6F,EAAMp3F,IAAKq3F,IChR1B5C,GAAK8C,SAAW,SAAU/jB,EAASzU,MAChB,MAAXyU,GACIA,EAAQ97E,OAAS,EAAG,QAIuB,GAAvCqnE,EAAMl1C,QAAQ2tE,cAAcpgB,QACxBqd,GAAKgD,YAAYjkB,EAASzU,GAG1B01B,GAAKiD,QAAQlkB,KAOjCihB,GAAKa,SAAW,SAAUv2B,EAAOl6D,EAAGZ,EAAGutF,EAAWsB,EAAYP,OAEtD5nE,EAAMgtE,EADNpC,EAA0B,GAAbzC,EAGb0C,EAAU9C,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,MAC7EuE,EAAQ1yD,eAAe,KAAM,IAAKj+B,GAClC2wF,EAAQ1yD,eAAe,KAAM,IAAK7+B,EAAIsxF,GACtCC,EAAQ1yD,eAAe,KAAM,QAAS0uD,GACtCgE,EAAQ1yD,eAAe,KAAM,SAAU,EAAIyyD,GAC3CC,EAAQ1yD,eAAe,KAAM,QAAS,gBAEtCnY,EAAO+nE,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,MACjEnuD,eAAe,KAAM,QAASi8B,EAAM/7B,gBACrB7oC,IAAhB4kE,EAAM7vC,OACNvE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM7vC,OAG7CvE,EAAKmY,eAAe,KAAM,IAAK,IAAMj+B,EAAI,IAAMZ,EAAI,MAAQY,EAAI2sF,GAAa,IAAMvtF,GAC9C,GAAhC86D,EAAMl1C,QAAQ+tE,OAAOxgB,UACrBugB,EAAWjF,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAClC,OAApClyB,EAAMl1C,QAAQ+tE,OAAO70C,YACrB40C,EAAS70D,eAAe,KAAM,IAAK,IAAMj+B,EAAI,MAAQZ,EAAIsxF,GACvD,IAAM1wF,EAAI,IAAMZ,EAAI,MAAQY,EAAI2sF,GAAa,IAAMvtF,EAAI,MAAQY,EAAI2sF,GAAa,KAAOvtF,EAAIsxF,IAG7FoC,EAAS70D,eAAe,KAAM,IAAK,IAAMj+B,EAAI,IAAMZ,EAAI,KAC/CY,EAAI,KAAOZ,EAAIsxF,GAAc,MAC5B1wF,EAAI2sF,GAAa,KAAOvtF,EAAIsxF,GACnC,KAAO1wF,EAAI2sF,GAAa,IAAMvtF,GAEpC0zF,EAAS70D,eAAe,KAAM,QAASi8B,EAAM/7B,UAAY,uBACtB7oC,IAA/B4kE,EAAMl1C,QAAQ+tE,OAAO1oE,OAAsD,KAA/B6vC,EAAMl1C,QAAQ+tE,OAAO1oE,OACjEyoE,EAAS70D,eAAe,KAAM,QAASi8B,EAAMl1C,QAAQ+tE,OAAO1oE,QAI5B,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,UAOzBsb,GAAkB7tF,EAAI,GAAM2sF,EAAWvtF,EANnB,CAChBirB,MAAO6vC,EAAMl1C,QAAQqlC,WAAWhgC,MAChC6T,OAAQg8B,EAAMl1C,QAAQqlC,WAAWnsB,OACjC1I,KAAM0kC,EAAMl1C,QAAQqlC,WAAW70B,KAC/B2I,UAAW+7B,EAAM/7B,WAEoCuvD,EAAUJ,YAAaI,EAAUtB,MAIlGwD,GAAKoD,YAAc,SAAUC,EAAW/4B,EAAOg5B,EAAcxF,MAErB,GAAhCxzB,EAAMl1C,QAAQ+tE,OAAOxgB,QAAiB,KAOlC4gB,EANAC,EAAYlmD,OAAOwgD,EAAUtB,IAAI/hE,MAAMqU,OAAOtiC,QAAQ,KAAK,KAC3D02F,EAAWjF,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC1Er/D,EAAO,IACgC,GAAvCmtC,EAAMl1C,QAAQ2tE,cAAcpgB,UAC5BxlD,EAAO,SAGPsmE,EAAO,EAEPA,EADoC,OAApCn5B,EAAMl1C,QAAQ+tE,OAAO70C,YACd,EAEkC,UAApCgc,EAAMl1C,QAAQ+tE,OAAO70C,YACnBk1C,EAGA/7F,KAAKY,IAAIZ,KAAK8D,IAAI,EAAG++D,EAAM+1B,cAAemD,GAGjDD,EADoC,SAApCj5B,EAAMl1C,QAAQ+tE,OAAO70C,aAA2C,MAAhBg1C,GAAwC59F,MAAhB49F,EAChE,IAAMD,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAC7Cp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,GAClC,KAAMmmE,EAAaA,EAAargG,OAAO,GAAG,GAAI,IAAMqgG,EAAaA,EAAargG,OAAO,GAAG,GAAK,IAC7FgD,KAAKy9F,cAAcJ,EAAanmE,GAAK,GACrCmmE,EAAa,GAAG,GAAI,IAAIA,EAAa,GAAG,GAAK,KAG7C,IAAMD,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAC7Cp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,GAClC,KAAOsmE,EAAO,KAAMJ,EAAU,GAAG,GAAK,KAGlDH,EAAS70D,eAAe,KAAM,QAASi8B,EAAM/7B,UAAY,kBACtB7oC,IAA/B4kE,EAAMl1C,QAAQ+tE,OAAO1oE,OACrByoE,EAAS70D,eAAe,KAAM,QAASi8B,EAAMl1C,QAAQ+tE,OAAO1oE,OAEhEyoE,EAAS70D,eAAe,KAAM,IAAKk1D,KAW3CvD,GAAKrK,KAAO,SAAU0N,EAAW/4B,EAAOwzB,MACnB,MAAbuF,GAAkC39F,MAAb29F,EAAwB,KACzCntE,EAAO+nE,GAAsB,OAAQH,EAAUJ,YAAaI,EAAUtB,KAC1EtmE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM/7B,gBACrB7oC,IAAhB4kE,EAAM7vC,OACNvE,EAAKmY,eAAe,KAAM,QAASi8B,EAAM7vC,WAGzC0C,EAAO,IACgC,GAAvCmtC,EAAMl1C,QAAQ2tE,cAAcpgB,UAC5BxlD,EAAO,KAGXjH,EAAKmY,eAAe,KAAM,IAAK,IAAMg1D,EAAU,GAAG,GAAI,IAAIA,EAAU,GAAG,GAAK,IAAMp9F,KAAKy9F,cAAcL,EAAUlmE,GAAK,MAI5H6iE,GAAK0D,cAAgB,SAASL,EAAUlmE,EAAKwmE,MACrCN,EAAUpgG,OAAS,QAEZ,OAGPF,EADAyP,EAAI2qB,KAEJwmE,MACK5gG,EAAIsgG,EAAUpgG,OAAO,EAAGF,EAAI,EAAGA,IAChCyP,GAAK6wF,EAAUtgG,GAAG,GAAK,IAAMsgG,EAAUtgG,GAAG,GAAK,aAI9CA,EAAI,EAAGA,EAAIsgG,EAAUpgG,OAAQF,IAC9ByP,GAAK6wF,EAAUtgG,GAAG,GAAK,IAAMsgG,EAAUtgG,GAAG,GAAK,WAGhDyP,GAUXwtF,GAAK4D,mBAAqB,SAAUpqF,OAE5BqqF,EAAIn1F,EAAIC,EAAIC,EAAIk1F,EAAKC,EACrBvxF,EAAI,GACRA,EAAEtP,KAAM,CAAEuE,KAAK+a,MAAMhJ,EAAK,GAAGmnF,UAAYl5F,KAAK+a,MAAMhJ,EAAK,GAAGonF,oBAExD39F,EAASuW,EAAKvW,OACTF,EAAI,EAAGA,EAAIE,EAAS,EAAGF,IAE5B8gG,EAAW,GAAL9gG,EAAUyW,EAAK,GAAKA,EAAKzW,EAAI,GACnC2L,EAAK8K,EAAKzW,GACV4L,EAAK6K,EAAKzW,EAAI,GACd6L,EAAM7L,EAAI,EAAIE,EAAUuW,EAAKzW,EAAI,GAAK4L,EAUtCm1F,EAAM,CACFnD,WAAakD,EAAGlD,SAAW,EAAIjyF,EAAGiyF,SAAWhyF,EAAGgyF,WAlBpC,EAAI,GAmBhBC,WAAaiD,EAAGjD,SAAW,EAAIlyF,EAAGkyF,SAAWjyF,EAAGiyF,WAnBpC,EAAI,IAqBpBmD,EAAM,CACFpD,UAAajyF,EAAGiyF,SAAW,EAAIhyF,EAAGgyF,SAAW/xF,EAAG+xF,WAtBpC,EAAI,GAuBhBC,UAAalyF,EAAGkyF,SAAW,EAAIjyF,EAAGiyF,SAAWhyF,EAAGgyF,WAvBpC,EAAI,IA2BpBpuF,EAAEtP,KAAM,CAAE4gG,EAAInD,SAAWmD,EAAIlD,WAC7BpuF,EAAEtP,KAAM,CAAE6gG,EAAIpD,SAAWoD,EAAInD,WAC7BpuF,EAAEtP,KAAM,CAAEyL,EAAGgyF,SAAYhyF,EAAGiyF,kBAGzBpuF,GAcXwtF,GAAKgD,YAAc,SAAUxpF,EAAM8wD,OAC3BojB,EAAQpjB,EAAMl1C,QAAQ2tE,cAAcrV,SAC3B,GAATA,QAAwBhoF,IAAVgoF,SACPznF,KAAK29F,mBAAmBpqF,OAG3BqqF,EAAIn1F,EAAIC,EAAIC,EAAIk1F,EAAKC,EAAKC,EAAIC,EAAIC,EAAIh7D,EAAGoC,EAAG64D,EAAGzxF,EAC/C0xF,EAAQC,EAAQC,EAASC,EAASC,EAASC,EAC3CjyF,EAAI,GACRA,EAAEtP,KAAM,CAAEuE,KAAK+a,MAAMhJ,EAAK,GAAGmnF,UAAYl5F,KAAK+a,MAAMhJ,EAAK,GAAGonF,oBACxD39F,EAASuW,EAAKvW,OACTF,EAAI,EAAGA,EAAIE,EAAS,EAAGF,IAE5B8gG,EAAW,GAAL9gG,EAAUyW,EAAK,GAAKA,EAAKzW,EAAI,GACnC2L,EAAK8K,EAAKzW,GACV4L,EAAK6K,EAAKzW,EAAI,GACd6L,EAAM7L,EAAI,EAAIE,EAAUuW,EAAKzW,EAAI,GAAK4L,EAEtCq1F,EAAKv8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIu4F,EAAGlD,SAAWjyF,EAAGiyF,SAAU,GAAKl5F,KAAK6D,IAAIu4F,EAAGjD,SAAWlyF,EAAGkyF,SAAU,IAC5FqD,EAAKx8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIoD,EAAGiyF,SAAWhyF,EAAGgyF,SAAU,GAAKl5F,KAAK6D,IAAIoD,EAAGkyF,SAAWjyF,EAAGiyF,SAAU,IAC5FsD,EAAKz8F,KAAKyqF,KAAKzqF,KAAK6D,IAAIqD,EAAGgyF,SAAW/xF,EAAG+xF,SAAU,GAAKl5F,KAAK6D,IAAIqD,EAAGiyF,SAAWhyF,EAAGgyF,SAAU,IAY5FwD,EAAS38F,KAAK6D,IAAI44F,EAAIxW,GACtB4W,EAAU78F,KAAK6D,IAAI44F,EAAI,EAAIxW,GAC3B2W,EAAS58F,KAAK6D,IAAI24F,EAAIvW,GACtB6W,EAAU98F,KAAK6D,IAAI24F,EAAI,EAAIvW,GAC3B+W,EAASh9F,KAAK6D,IAAI04F,EAAItW,GAGtBxkD,EAAI,GAFJs7D,EAAU/8F,KAAK6D,IAAI04F,EAAI,EAAItW,IAET,EAAI+W,EAASJ,EAASE,EACxCj5D,EAAI,EAAIg5D,EAAU,EAAIF,EAASC,EAASE,GACxCJ,EAAI,EAAIM,GAAUA,EAASJ,IACnB,IACJF,EAAI,EAAIA,IAEZzxF,EAAI,EAAI0xF,GAAUA,EAASC,IACnB,IACJ3xF,EAAI,EAAIA,GAGZoxF,EAAM,CACFnD,WAAa4D,EAAUV,EAAGlD,SAAWz3D,EAAIx6B,EAAGiyF,SAAW6D,EAAU71F,EAAGgyF,UAAYwD,EAChFvD,WAAa2D,EAAUV,EAAGjD,SAAW13D,EAAIx6B,EAAGkyF,SAAW4D,EAAU71F,EAAGiyF,UAAYuD,GAGpFJ,EAAM,CACFpD,UAAa2D,EAAU51F,EAAGiyF,SAAWr1D,EAAI38B,EAAGgyF,SAAW4D,EAAU31F,EAAG+xF,UAAYjuF,EAChFkuF,UAAa0D,EAAU51F,EAAGkyF,SAAWt1D,EAAI38B,EAAGiyF,SAAW2D,EAAU31F,EAAGgyF,UAAYluF,GAGhE,GAAhBoxF,EAAInD,UAAiC,GAAhBmD,EAAIlD,WACzBkD,EAAMp1F,GAEU,GAAhBq1F,EAAIpD,UAAiC,GAAhBoD,EAAInD,WACzBmD,EAAMp1F,GAEV6D,EAAEtP,KAAM,CAAE4gG,EAAInD,SAAWmD,EAAIlD,WAC7BpuF,EAAEtP,KAAM,CAAE6gG,EAAIpD,SAAWoD,EAAInD,WAC7BpuF,EAAEtP,KAAM,CAAEyL,EAAGgyF,SAAYhyF,EAAGiyF,kBAGzBpuF,GAUfwtF,GAAKiD,QAAU,SAAUzpF,WAEjBhH,EAAI,GACCzP,EAAI,EAAGA,EAAIyW,EAAKvW,OAAQF,IAC7ByP,EAAEtP,KAAK,CAAEsW,EAAKzW,GAAG49F,SAAWnnF,EAAKzW,GAAG69F,kBAEjCpuF,GC/QXytF,GAAW79F,UAAUg5D,SAAW,SAAU3e,GAC3B,MAATA,QACGivB,UAAYjvB,EACQ,GAArB6pB,QAAKlxC,UACPmpB,GAAKmmD,WAAWz+F,KAAKylE,WAAU,SAAUtoE,EAAGC,UACnCD,EAAEgN,EAAI/M,EAAE+M,EAAI,GAAK,WAKvBs7D,UAAY,IAIrBu0B,GAAW79F,UAAUuiG,SAAW,kBACvB1+F,KAAKylE,WAOdu0B,GAAW79F,UAAUwiG,gBAAkB,SAAU7tF,QAC1CspF,aAAetpF,GAOtBkpF,GAAW79F,UAAUqiD,WAAa,SAAUrvB,WAC1B1vB,IAAZ0vB,EAAuB,CAEzBmpB,GAAKqQ,oBADQ,CAAC,WAAY,QAAS,OAAQ,mBAAoB,WAAY,SAAS,sBAAuB,qBAC1E3oD,KAAKmvB,QAASA,GAGd,mBAAtBA,EAAQqlC,aACjBrlC,EAAQqlC,WAAa,CACnBC,SAAUtlC,EAAQqlC,aAItBlc,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,iBACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,cACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UAErCA,EAAQ2tE,eAC0B,UAAhCr3D,GAAOtW,EAAQ2tE,gBACb3tE,EAAQ2tE,cAAc+B,kBACqB,WAAzC1vE,EAAQ2tE,cAAc+B,qBACnB1vE,QAAQ2tE,cAAcrV,MAAQ,EAEa,WAAzCt4D,EAAQ2tE,cAAc+B,qBACxB1vE,QAAQ2tE,cAAcrV,MAAQ,QAG9Bt4D,QAAQ2tE,cAAc+B,gBAAkB,mBACxC1vE,QAAQ2tE,cAAcrV,MAAQ,OAa/CuS,GAAW79F,UAAUg6C,OAAS,SAAUkuB,QACjCA,MAAQA,OACRxwC,QAAUwwC,EAAMxwC,SAAW,aAC3ByU,UAAY+7B,EAAM/7B,WAAatoC,KAAKsoC,WAAa,kBAAoBtoC,KAAKi6F,yBAAyB,GAAK,QACxG3+B,aAA4B77D,IAAlB4kE,EAAM/I,SAA+B+I,EAAM/I,aACrD9mC,MAAQ6vC,EAAM7vC,WACdgqB,WAAW6lB,EAAMl1C,UAaxB6qE,GAAW79F,UAAUg8F,UAAY,SAAUrB,EAAWsB,EAAYP,EAAW1tF,EAAGZ,GAC7D9J,MAAbo4F,GAAuC,MAAbA,IAE5BA,EAAY,CAACtB,IADHnpE,SAASya,gBAAgB,6BAA8B,OAC1C4vD,YAAY,GAAItoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ,CAAC3sE,eAEhEP,MAAL0K,GAAuB,MAALA,IACpBA,EAAI,GAEG1K,MAAL8J,GAAuB,MAALA,IACpBA,EAAI,GAAM6uF,GAEJp4F,KAAKmvB,QAAQqF,WACd,OACHsqE,GAAMlE,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,aAE/C,aACA,QACH8B,GAAOiB,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,aAEhD,MACHkH,GAAKnE,SAAS56F,KAAMmK,EAAGZ,EAAGutF,EAAWsB,EAAYP,SAG9C,CAACmH,KAAMnH,EAAUtB,IAAKhuD,MAAOvoC,KAAK6zB,QAASw0B,YAAaroD,KAAKmvB,QAAQstE,mBAG9EzC,GAAW79F,UAAU8iG,UAAY,SAAUxuB,WACrCisB,EAAOjsB,EAAU,GAAGlnE,EACpBozF,EAAOlsB,EAAU,GAAGlnE,EACfwK,EAAI,EAAGA,EAAI08D,EAAUzzE,OAAQ+W,IACpC2oF,EAAOA,EAAOjsB,EAAU18D,GAAGxK,EAAIknE,EAAU18D,GAAGxK,EAAImzF,EAChDC,EAAOA,EAAOlsB,EAAU18D,GAAGxK,EAAIknE,EAAU18D,GAAGxK,EAAIozF,QAE3C,CAACv6F,IAAKs6F,EAAMp3F,IAAKq3F,EAAMF,iBAAkBz8F,KAAKmvB,QAAQstE,mBC/G/DpC,GAAOl+F,UAAY,IAAIw8C,GAEvB0hD,GAAOl+F,UAAUixC,MAAQ,gBAClBu/B,OAAS,QACTgrB,eAAiB,GAGxB0C,GAAOl+F,UAAU+iG,SAAW,SAAS32D,EAAOwvD,GAGI,GAA1CA,EAAa5oE,QAAQgwE,oBAClBn/F,KAAK2sE,OAAOtvE,eAAekrC,UACzBokC,OAAOpkC,GAASwvD,QAElBJ,gBAAkB,IAI3B0C,GAAOl+F,UAAUkqE,YAAc,SAAS99B,EAAOwvD,QACxCprB,OAAOpkC,GAASwvD,GAGvBsC,GAAOl+F,UAAUijG,YAAc,SAAS72D,GAClCvoC,KAAK2sE,OAAOtvE,eAAekrC,YACtBvoC,KAAK2sE,OAAOpkC,QACdovD,gBAAkB,IAI3B0C,GAAOl+F,UAAUusD,QAAU,gBACpBtK,IAAIuqB,MAAQv7C,SAASE,cAAc,YACnC8wB,IAAIuqB,MAAMrgC,UAAY,kBACtB8V,IAAIuqB,MAAMn0C,MAAMkL,SAAW,gBAC3B0e,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,YACtB1E,IAAIuqB,MAAMn0C,MAAMC,QAAU,aAE1B2pB,IAAIihD,SAAWjyE,SAASE,cAAc,YACtC8wB,IAAIihD,SAAS/2D,UAAY,uBACzB8V,IAAIihD,SAAS7qE,MAAMkL,SAAW,gBAC9B0e,IAAIihD,SAAS7qE,MAAMsuB,IAAM,WAEzByzC,IAAMnpE,SAASya,gBAAgB,6BAA6B,YAC5D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMsuB,IAAM,WAChByzC,IAAI/hE,MAAMoU,MAAQ5oC,KAAKmvB,QAAQorE,SAAW,EAAI,UAC9ChE,IAAI/hE,MAAMqU,OAAS,YAEnBuV,IAAIuqB,MAAMh0C,YAAY30B,KAAKu2F,UAC3Bn4C,IAAIuqB,MAAMh0C,YAAY30B,KAAKo+C,IAAIihD,WAMtChF,GAAOl+F,UAAU+zD,KAAO,WAElBlwD,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,QAOnD0xB,GAAOl+F,UAAUqnE,KAAO,WAEjBxjE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,QAI9C0xB,GAAOl+F,UAAUqiD,WAAa,SAASrvB,GAErCmpB,GAAKqQ,oBADQ,CAAC,UAAU,cAAc,QAAQ,OAAO,SACpB3oD,KAAKmvB,QAASA,IAGjDkrE,GAAOl+F,UAAUk0D,OAAS,eACpBgoC,EAAe,EACfH,EAAal4C,GAAYhgD,KAAK2sE,WAClCurB,QAAAA,GAAgB,SAAU/6F,EAAEC,UAClBD,EAAIC,GAAK,EAAI,SAGlB,IAAIN,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IAAK,KACtCq/D,EAAU+7B,EAAWp7F,GACW,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,IACvIk8B,OAImC,GAAnCr4F,KAAKmvB,QAAQnvB,KAAKs6F,MAAMh/B,SAA2C,GAAvBt7D,KAAK23F,gBAA+C,GAAxB33F,KAAKmvB,QAAQutD,SAAoC,GAAhB2b,OACtGnoC,WAEF,SACEsT,OACmC,YAApCxjE,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,UAA8D,eAApC1/B,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,eACvE0e,IAAIuqB,MAAMn0C,MAAMkgB,KAAO,WACvB0J,IAAIuqB,MAAMn0C,MAAM4kE,UAAY,YAC5Bh7C,IAAIihD,SAAS7qE,MAAM4kE,UAAY,YAC/Bh7C,IAAIihD,SAAS7qE,MAAMkgB,KAAQ10C,KAAKmvB,QAAQorE,SAAW,GAAM,UACzDn8C,IAAIihD,SAAS7qE,MAAMmgB,MAAQ,QAC3B4hD,IAAI/hE,MAAMkgB,KAAO,WACjB6hD,IAAI/hE,MAAMmgB,MAAQ,UAGlByJ,IAAIuqB,MAAMn0C,MAAMmgB,MAAQ,WACxByJ,IAAIuqB,MAAMn0C,MAAM4kE,UAAY,aAC5Bh7C,IAAIihD,SAAS7qE,MAAM4kE,UAAY,aAC/Bh7C,IAAIihD,SAAS7qE,MAAMmgB,MAAS30C,KAAKmvB,QAAQorE,SAAW,GAAM,UAC1Dn8C,IAAIihD,SAAS7qE,MAAMkgB,KAAO,QAC1B6hD,IAAI/hE,MAAMmgB,MAAQ,WAClB4hD,IAAI/hE,MAAMkgB,KAAO,IAGgB,YAApC10C,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,UAA8D,aAApC1/B,KAAKmvB,QAAQnvB,KAAKs6F,MAAM56D,cACvE0e,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,EAAIzL,OAAOr3C,KAAK44C,KAAKwF,IAAIY,OAAOxqB,MAAMsuB,IAAIv8C,QAAQ,KAAK,KAAO,UACpF63C,IAAIuqB,MAAMn0C,MAAMq0B,OAAS,OAE3B,KACCy2C,EAAmBt/F,KAAK44C,KAAKY,SAASwF,OAAOnW,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,YACxFuV,IAAIuqB,MAAMn0C,MAAMq0B,OAAS,EAAIy2C,EAAmBjoD,OAAOr3C,KAAK44C,KAAKwF,IAAIY,OAAOxqB,MAAMsuB,IAAIv8C,QAAQ,KAAK,KAAO,UAC1G63C,IAAIuqB,MAAMn0C,MAAMsuB,IAAM,GAGH,GAAtB9iD,KAAKmvB,QAAQsnE,YACVr4C,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKo+C,IAAIihD,SAASj2C,YAAc,GAAK,UAC7DhL,IAAIihD,SAAS7qE,MAAMmgB,MAAQ,QAC3ByJ,IAAIihD,SAAS7qE,MAAMkgB,KAAO,QAC1B6hD,IAAI/hE,MAAMoU,MAAQ,aAGlBwV,IAAIuqB,MAAMn0C,MAAMoU,MAAQ5oC,KAAKmvB,QAAQorE,SAAW,GAAKv6F,KAAKo+C,IAAIihD,SAASj2C,YAAc,GAAK,UAC1Fm2C,uBAGH1rE,EAAU,OACT/2B,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IACjCq/D,EAAU+7B,EAAWp7F,GACe,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,KACvItoC,GAAW7zB,KAAK2sE,OAAOxQ,GAAStoC,QAAU,eAGzCuqB,IAAIihD,SAAS3zC,UAAY73B,OACzBuqB,IAAIihD,SAAS7qE,MAAM2zC,WAAe,IAAOnoE,KAAKmvB,QAAQorE,SAAYv6F,KAAKmvB,QAAQqrE,YAAe,OAIvGH,GAAOl+F,UAAUojG,gBAAkB,cAC7Bv/F,KAAKo+C,IAAIuqB,MAAMphC,WAAY,KACzB2wD,EAAal4C,GAAYhgD,KAAK2sE,WAClCurB,QAAAA,GAAgB,SAAU/6F,EAAEC,UAClBD,EAAIC,GAAK,EAAI,KAIvB46F,GAAsBh4F,KAAKy3F,iBAEvBztB,EAAUh+C,OAAO+8C,iBAAiB/oE,KAAKo+C,IAAIuqB,OAAO62B,WAClDvH,EAAa5gD,OAAO2yB,EAAQzjE,QAAQ,KAAK,KACzC4D,EAAI8tF,EACJnB,EAAY92F,KAAKmvB,QAAQorE,SACzBnC,EAAa,IAAOp4F,KAAKmvB,QAAQorE,SACjChxF,EAAI0uF,EAAa,GAAMG,EAAa,OAEnC7B,IAAI/hE,MAAMoU,MAAQkuD,EAAY,EAAImB,EAAa,SAE/C,IAAIn7F,EAAI,EAAGA,EAAIo7F,EAAWl7F,OAAQF,IAAK,KACtCq/D,EAAU+7B,EAAWp7F,GACW,GAAhCkD,KAAK2sE,OAAOxQ,GAASb,cAAkE77D,IAA9CO,KAAKw2F,iBAAiBv3C,WAAWkd,IAAuE,GAA7Cn8D,KAAKw2F,iBAAiBv3C,WAAWkd,UAClIwQ,OAAOxQ,GAASg8B,UAAUrB,EAAWsB,EAAYp4F,KAAK63F,UAAW1tF,EAAGZ,GACzEA,GAAK6uF,EAAap4F,KAAKmvB,QAAQqrE,gBChMvC,SAASiF,GAAU7mD,EAAMzpB,QAClBiE,GAAKs3C,UACL9xB,KAAOA,OAEPsE,eAAiB,CACpBu/C,iBAAkB,OAClBiD,aAAc,UACdzzF,MAAM,EACN0zF,UAAU,EACVt8F,OAAO,EACPu8F,YAAa,QACb1C,OAAQ,CACNxgB,SAAS,EACTr0B,YAAa,UAEf7zB,MAAO,OACPwmE,SAAU,CACRpyD,MAAO,GACPszD,YAAY,EACZz1B,MAAO,UAETq2B,cAAe,CACbpgB,SAAS,EACTmiB,gBAAiB,cACjBpX,MAAO,IAETjzB,WAAY,CACVkoB,SAAS,EACT/8C,KAAM,EACNnL,MAAO,UAETqrE,SAAU,GACVC,OAAQ,GACRnzB,OAAQ,CACN1tB,WAAY,UAKX9vB,QAAUmpB,GAAKh7C,OAAO,GAAI0C,KAAKk9C,qBAC/BkB,IAAM,QACN/I,MAAQ,QACRwO,OAAS,UACT8oB,OAAS,QACTozB,oBAAqB,OACrBC,iBAAkB,OAClBC,yBAA0B,OAC1BC,kBAAmB,MAEpBvhD,EAAK3+C,UACJylE,UAAY,UACZsK,WAAa,UAGbE,cAAgB,KACZ,SAAUzvB,EAAOD,EAAQ2vB,GAC9BvxB,EAAGwxB,OAAO5vB,EAAO/J,eAET,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAG0xB,UAAU9vB,EAAO/J,eAEZ,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAG2xB,UAAU/vB,EAAO/J,cAKnB+5B,eAAiB,KACb,SAAU/vB,EAAOD,EAAQ2vB,GAC9BvxB,EAAG6xB,aAAajwB,EAAO/J,eAEf,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAGmyB,gBAAgBvwB,EAAO/J,eAElB,SAAUgK,EAAOD,EAAQ2vB,GACjCvxB,EAAGoyB,gBAAgBxwB,EAAO/J,cAIzBA,MAAQ,QACRy6B,UAAY,QACZkvB,UAAYngG,KAAK44C,KAAKc,MAAMlZ,WAC5B8kC,YAAc,QAEdmyB,YAAc,QACdj5C,WAAWrvB,QACX8qE,yBAA2B,CAAC,QAC5BrhD,KAAKiF,QAAQvH,GAAG,gBAAgB,WACnCqI,EAAGwhD,UAAYxhD,EAAG/F,KAAKc,MAAMlZ,MAC7Bme,EAAG43C,IAAI/hE,MAAMkgB,KAAO4D,GAAKge,OAAOC,QAAQ5X,EAAGtJ,MAAMzM,OAEjD+V,EAAGuhD,kBAAmB,EAEtBvhD,EAAG0R,OAAOh0D,KAAKsiD,WAIZ+J,eACAmvC,UAAY,CAACtB,IAAKv2F,KAAKu2F,IAAKkB,YAAaz3F,KAAKy3F,YAAatoE,QAASnvB,KAAKmvB,QAASw9C,OAAQ3sE,KAAK2sE,QAGtG8yB,GAAUtjG,UAAY,IAAIw8C,GAK1B8mD,GAAUtjG,UAAUusD,QAAU,eACxBigB,EAAQv7C,SAASE,cAAc,OACnCq7C,EAAMrgC,UAAY,sBACb8V,IAAIuqB,MAAQA,OAGZ4tB,IAAMnpE,SAASya,gBAAgB,6BAA8B,YAC7D0uD,IAAI/hE,MAAMkL,SAAW,gBACrB62D,IAAI/hE,MAAMqU,QAAU,GAAK7oC,KAAKmvB,QAAQywE,aAAar5F,QAAQ,KAAM,IAAM,UACvEgwF,IAAI/hE,MAAMC,QAAU,QACzBk0C,EAAMh0C,YAAY30B,KAAKu2F,UAGlBpnE,QAAQ0wE,SAASx3C,YAAc,YAC/B+3C,UAAY,IAAI9J,GAASt2F,KAAK44C,KAAM54C,KAAKmvB,QAAQ0wE,SAAU7/F,KAAKu2F,IAAKv2F,KAAKmvB,QAAQw9C,aAElFx9C,QAAQ0wE,SAASx3C,YAAc,aAC/Bg4C,WAAa,IAAI/J,GAASt2F,KAAK44C,KAAM54C,KAAKmvB,QAAQ0wE,SAAU7/F,KAAKu2F,IAAKv2F,KAAKmvB,QAAQw9C,eACjF3sE,KAAKmvB,QAAQ0wE,SAASx3C,iBAGxBi4C,WAAa,IAAIjG,GAAOr6F,KAAK44C,KAAM54C,KAAKmvB,QAAQ2wE,OAAQ,OAAQ9/F,KAAKmvB,QAAQw9C,aAC7E4zB,YAAc,IAAIlG,GAAOr6F,KAAK44C,KAAM54C,KAAKmvB,QAAQ2wE,OAAQ,QAAS9/F,KAAKmvB,QAAQw9C,aAE/EnJ,QAOPi8B,GAAUtjG,UAAUqiD,WAAa,SAAUrvB,MACrCA,EAAS,MAEiB1vB,IAAxB0vB,EAAQywE,kBAAgDngG,IAAnB0vB,EAAQ0Z,aAC1Cm3D,iBAAkB,OAClBC,yBAA0B,QAEsBxgG,IAA9CO,KAAK44C,KAAKY,SAASC,gBAAgB5Q,aAAgDppC,IAAxB0vB,EAAQywE,aACtEz0C,IAAUh8B,EAAQywE,YAAc,IAAIr5F,QAAQ,KAAM,KAAOvG,KAAK44C,KAAKY,SAASC,gBAAgB5Q,cACzFm3D,iBAAkB,GAG3B1nD,GAAKqQ,oBAVQ,CAAC,WAAY,eAAgB,QAAS,SAAU,cAAe,mBAAoB,QAAS,WAAY,WAAY,OAAQ,UAUxG3oD,KAAKmvB,QAASA,GAC/CmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,iBACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,cACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UACzCmpB,GAAKsmD,aAAa5+F,KAAKmvB,QAASA,EAAS,UAErCA,EAAQ2tE,eAC0B,UAAhCr3D,GAAOtW,EAAQ2tE,gBACb3tE,EAAQ2tE,cAAc+B,kBACqB,WAAzC1vE,EAAQ2tE,cAAc+B,qBACnB1vE,QAAQ2tE,cAAcrV,MAAQ,EAEa,WAAzCt4D,EAAQ2tE,cAAc+B,qBACxB1vE,QAAQ2tE,cAAcrV,MAAQ,QAG9Bt4D,QAAQ2tE,cAAc+B,gBAAkB,mBACxC1vE,QAAQ2tE,cAAcrV,MAAQ,KAMvCznF,KAAKogG,gBACkB3gG,IAArB0vB,EAAQ0wE,gBACLO,UAAU5hD,WAAWx+C,KAAKmvB,QAAQ0wE,eAClCQ,WAAW7hD,WAAWx+C,KAAKmvB,QAAQ0wE,WAIxC7/F,KAAKsgG,iBACgB7gG,IAAnB0vB,EAAQ2wE,cACLQ,WAAW9hD,WAAWx+C,KAAKmvB,QAAQ2wE,aACnCS,YAAY/hD,WAAWx+C,KAAKmvB,QAAQ2wE,SAIzC9/F,KAAK2sE,OAAOtvE,eApMJ,uBAqMLsvE,OAAL,cAAuBnuB,WAAWrvB,GAKlCnvB,KAAKo+C,IAAIuqB,aACNu3B,kBAAiB,OACjBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,MAO7CotC,GAAUtjG,UAAU+zD,KAAO,WAErBlwD,KAAKo+C,IAAIuqB,MAAMphC,iBACZ6W,IAAIuqB,MAAMphC,WAAWC,YAAYxnC,KAAKo+C,IAAIuqB,QAQnD82B,GAAUtjG,UAAUqnE,KAAO,WAEpBxjE,KAAKo+C,IAAIuqB,MAAMphC,iBACbqR,KAAKwF,IAAIY,OAAOrqB,YAAY30B,KAAKo+C,IAAIuqB,QAS9C82B,GAAUtjG,UAAUg5D,SAAW,SAAU3e,OAErC+7B,EADE5zB,EAAK3+C,KAEPq0E,EAAer0E,KAAKylE,aAGjBjvB,EAGA,CAAA,KAAIA,aAAiBoB,IAAWpB,aAAiB89B,UAI9C,IAAItnD,UAAU,wDAHfy4C,UAAYluB,GAAkBf,aAH9BivB,UAAY,QASf4O,OAEF/7B,SAAAA,GAAat4C,KAAKiwE,eAAe,SAAUjqE,EAAUw6C,GACnD6zB,EAAa99B,IAAIiK,EAAOx6C,MAI1BquE,EAAah8B,UAGbk6B,EAAM8B,EAAaj8B,cACdk4B,UAAUiC,IAGbvyE,KAAKylE,UAAW,KAEdryC,EAAKpzB,KAAKozB,MACdklB,SAAAA,GAAat4C,KAAKiwE,eAAe,SAAUjqE,EAAUw6C,GACnD7B,EAAG8mB,UAAUnvB,GAAGkK,EAAOx6C,EAAUotB,MAInCm/C,EAAMvyE,KAAKylE,UAAUrtB,cAChB+3B,OAAOoC,KAShBktB,GAAUtjG,UAAUi5D,UAAY,SAAUuX,OAEpC4F,EADA5zB,EAAK3+C,QAILA,KAAK+vE,WAAY,IACnBz3B,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAAUvqE,EAAUw6C,GACpD7B,EAAGoxB,WAAWx5B,IAAIiK,EAAOx6C,MAI3BusE,EAAMvyE,KAAK+vE,WAAW33B,cACjB23B,WAAa,SACb,IAAIjzE,EAAI,EAAGA,EAAIy1E,EAAIv1E,OAAQF,SACzB0jG,aAAajuB,EAAIz1E,OAKrB6vE,EAGA,CAAA,KAAIA,aAAkB/0B,IAAW+0B,aAAkB2H,UAIhD,IAAItnD,UAAU,wDAHf+iD,WAAapD,YAHboD,WAAa,QAShB/vE,KAAK+vE,WAAY,KAEf38C,EAAKpzB,KAAKozB,MACdklB,SAAAA,GAAat4C,KAAKuwE,gBAAgB,SAAUvqE,EAAUw6C,GACpD7B,EAAGoxB,WAAWz5B,GAAGkK,EAAOx6C,EAAUotB,MAIpCm/C,EAAMvyE,KAAK+vE,WAAW33B,cACjBo4B,aAAa+B,KAItBktB,GAAUtjG,UAAUk0E,UAAY,SAAUkC,QACnCkuB,oBAAoBluB,IAE3BktB,GAAUtjG,UAAUg0E,OAAS,SAAUoC,QAChClC,UAAUkC,IAEjBktB,GAAUtjG,UAAUm0E,UAAY,SAAUiC,QACnClC,UAAUkC,IAEjBktB,GAAUtjG,UAAU20E,gBAAkB,SAAUE,QACzCyvB,oBAAoB,KAAMzvB,IAEjCyuB,GAAUtjG,UAAUq0E,aAAe,SAAUQ,QACtCF,gBAAgBE,IAQvByuB,GAAUtjG,UAAU40E,gBAAkB,SAAUC,OACzC,IAAIl0E,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,SAC9B0jG,aAAaxvB,EAASl0E,SAExBojG,kBAAmB,OACnBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,KAQ3CotC,GAAUtjG,UAAUqkG,aAAe,SAAUrkC,GACvCn8D,KAAK2sE,OAAOtvE,eAAe8+D,KACwB,SAAjDn8D,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWjB,YAAYjjC,QACvBokC,YAAYnB,YAAYjjC,QACxBokC,YAAYlwC,gBAGZ+vC,UAAUhB,YAAYjjC,QACtBmkC,WAAWlB,YAAYjjC,QACvBmkC,WAAWjwC,iBAEXrwD,KAAK2sE,OAAOxQ,KAWvBsjC,GAAUtjG,UAAUukG,aAAe,SAAUr8B,EAAOlI,GAC7Cn8D,KAAK2sE,OAAOtvE,eAAe8+D,SAYzBwQ,OAAOxQ,GAAShmB,OAAOkuB,GACyB,SAAjDrkE,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWh6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAC5CokC,YAAYl6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAE7CikC,UAAUhB,YAAYjjC,QACtBmkC,WAAWlB,YAAYjjC,UAGvBikC,UAAU/5B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAC3CmkC,WAAWj6B,YAAYlK,EAASn8D,KAAK2sE,OAAOxQ,SAE5CkkC,WAAWjB,YAAYjjC,QACvBokC,YAAYnB,YAAYjjC,WAxB1BwQ,OAAOxQ,GAAW,IAAI69B,GAAW31B,EAAOlI,EAASn8D,KAAKmvB,QAASnvB,KAAKi6F,0BACpB,SAAjDj6F,KAAK2sE,OAAOxQ,GAAShtC,QAAQstE,uBAC1B4D,WAAWnB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,SACzCokC,YAAYrB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,WAG1CikC,UAAUlB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,SACxCmkC,WAAWpB,SAAS/iC,EAASn8D,KAAK2sE,OAAOxQ,WAoB7CmkC,WAAWjwC,cACXkwC,YAAYlwC,UAWnBovC,GAAUtjG,UAAUskG,oBAAsB,SAAUluB,EAAKvB,MACjC,MAAlBhxE,KAAKylE,UAAmB,KACtBk7B,EAAgB,GAChBnqD,EAAQx2C,KAAKylE,UAAU17D,MACvB8tC,EAAU73C,KAAKylE,UAAUhuB,OACzBmpD,EAAQ,GACRruB,MACFA,QAAAA,GAAQ,SAAUn/C,GAChBwtE,EAAMxtE,GAAMA,aAKZytE,EAAc,GACT/jG,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,IAAK,KACjCi7C,EAAOvB,EAAM15C,GACbq/D,EAAUpkB,EAAKssB,MACflI,MAAAA,IACFA,EAxbQ,iBA0bV0kC,EAAYxjG,eAAe8+D,GAAW0kC,EAAY1kC,KAAa0kC,EAAY1kC,GAAW,MAIpF2kC,EAAmB,OAClB9vB,GAAYuB,MACVpW,KAAWn8D,KAAK2sE,UACf3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KAEnC4kC,GADJ18B,EAAQrkE,KAAK2sE,OAAOxQ,IACOuiC,WAE3BiC,EAAcxkC,GAAWrlB,GAAAiqD,QAAAA,GAAsB,SAAUhpD,UACvD+oD,EAAiB/oD,EAAKF,IAAYE,EAAKF,GAC/BE,EAAKF,KAAa+oD,EAAM7oD,EAAKF,WAEnCmpD,EAAYH,EAAY1kC,GAC5B0kC,EAAY1kC,IAAYwkC,EAAcxkC,GAASn/D,OAC3C2jG,EAAcxkC,GAASn/D,OAASgkG,IAClCL,EAAcxkC,GAAS6kC,EAAY,GAAK,QAO3ClkG,EAAI,EAAGA,EAAI05C,EAAMx5C,OAAQF,OAGxBq/D,OADJA,GADApkB,EAAOvB,EAAM15C,IACEunE,SAEblI,EAvdQ,iBAydL6U,IAAYuB,GAAQx6B,EAAKF,KAAa+oD,EAAM7oD,EAAKF,MAAcipD,EAAiBzjG,eAAe06C,EAAKF,KAGpG8oD,EAActjG,eAAe8+D,KAChCwkC,EAAcxkC,GAAW,IAAIlgE,MAAM4kG,EAAY1kC,SAG7C8kC,EAAW3oD,GAAK4oD,aAAanpD,GACjCkpD,EAAS92F,EAAImuC,GAAKpB,QAAQa,EAAK5tC,EAAG,QAClC82F,EAASxgE,IAAM6X,GAAKpB,QAAQa,EAAKtX,IAAK,QACtCwgE,EAASE,SAAWppD,EAAKxuC,EACzB03F,EAAS13F,EAAI8tC,OAAOU,EAAKxuC,GACzB03F,EAASppD,GAAWE,EAAKF,OAErBjzB,EAAO+7E,EAAcxkC,GAASn/D,OAAS6jG,EAAY1kC,KACvDwkC,EAAcxkC,GAASv3C,GAASq8E,MAI7B9kC,KAAWn8D,KAAK2sE,OACf3sE,KAAK2sE,OAAOtvE,eAAe8+D,KACxBwkC,EAActjG,eAAe8+D,KAChCwkC,EAAcxkC,GAAW,IAAIlgE,MAAM,SAMpCkgE,KAAWwkC,KACVA,EAActjG,eAAe8+D,MACM,GAAjCwkC,EAAcxkC,GAASn/D,OACrBgD,KAAK2sE,OAAOtvE,eAAe8+D,SACxBqkC,aAAarkC,OAEf,KACDkI,OAAQ5kE,EACWA,MAAnBO,KAAK+vE,aACP1L,EAAQrkE,KAAK+vE,WAAWhmE,IAAIoyD,IAEjB18D,MAAT4kE,IACFA,EAAQ,CAACjxC,GAAI+oC,EAAStoC,QAAS7zB,KAAKmvB,QAAQuwE,aAAevjC,SAExDukC,aAAar8B,EAAOlI,QACpBwQ,OAAOxQ,GAAShH,SAASwrC,EAAcxkC,SAI7C+jC,kBAAmB,OACnBtnD,KAAKiF,QAAQ4C,KAAK,UAAU,CAAC4R,OAAO,MAQ7CotC,GAAUtjG,UAAUk0D,OAAS,eACvBxX,GAAU,OAGTxD,MAAMzM,MAAQ5oC,KAAKo+C,IAAIuqB,MAAMvf,iBAC7B/T,MAAMxM,OAAS7oC,KAAK44C,KAAKY,SAASC,gBAAgB5Q,OACnD7oC,KAAK44C,KAAKY,SAASmY,OAAO7O,IAC1B9iD,KAAK44C,KAAKY,SAASmY,OAAO9I,OAG9BhQ,EAAU74C,KAAK8pD,cAAgBjR,QAG3Bo6B,EAAkBjzE,KAAK44C,KAAKc,MAAMjZ,IAAMzgC,KAAK44C,KAAKc,MAAMlZ,MACxD0yC,EAAUD,GAAmBjzE,KAAKmzE,0BACjCA,oBAAsBF,EAKZ,GAAXp6B,UACG09C,IAAI/hE,MAAMoU,MAAQ0P,GAAKge,OAAOC,OAAO,EAAIv2D,KAAKq1C,MAAMzM,YACpD2tD,IAAI/hE,MAAMkgB,KAAO4D,GAAKge,OAAOC,QAAQv2D,KAAKq1C,MAAMzM,QAGL,GAA5CgrB,KAAC5zD,KAAKmvB,QAAQ0Z,OAAS,WAAY,MAA8C,GAAhC7oC,KAAKigG,+BACnDD,iBAAkB,OAKC,GAAxBhgG,KAAKggG,iBACHhgG,KAAKmvB,QAAQywE,aAAe5/F,KAAKq1C,MAAMxM,OAAS,YAC7C1Z,QAAQywE,YAAc5/F,KAAKq1C,MAAMxM,OAAS,UAC1C0tD,IAAI/hE,MAAMqU,OAAS7oC,KAAKq1C,MAAMxM,OAAS,WAEzCm3D,iBAAkB,QAGlBzJ,IAAI/hE,MAAMqU,QAAU,GAAK7oC,KAAKmvB,QAAQywE,aAAar5F,QAAQ,KAAM,IAAM,KAI/D,GAAXsyC,GAA6B,GAAVq6B,GAA6C,GAA3BlzE,KAAK+/F,oBAAuD,GAAzB//F,KAAKkgG,iBAC/ErnD,EAAU74C,KAAKohG,gBAAkBvoD,OAC5BqnD,kBAAmB,UAIF,GAAlBlgG,KAAKmgG,UAAgB,KACnB3jF,EAASxc,KAAK44C,KAAKc,MAAMlZ,MAAQxgC,KAAKmgG,UACtCzmD,EAAQ15C,KAAK44C,KAAKc,MAAMjZ,IAAMzgC,KAAK44C,KAAKc,MAAMlZ,SAC1B,GAApBxgC,KAAKq1C,MAAMzM,MAAY,KAErBJ,EAAUhsB,GADSxc,KAAKq1C,MAAMzM,MAAQ8Q,QAErC68C,IAAI/hE,MAAMkgB,MAAS10C,KAAKq1C,MAAMzM,MAAQJ,EAAW,kBAIvD83D,WAAWjwC,cACXkwC,YAAYlwC,SACVxX,GAIT4mD,GAAUtjG,UAAUklG,mBAAqB,eAEnCC,EAAY,OACX,IAAInlC,KAAWn8D,KAAK2sE,UACnB3sE,KAAK2sE,OAAOtvE,eAAe8+D,GAAU,KACnCkI,EAAQrkE,KAAK2sE,OAAOxQ,GACH,GAAjBkI,EAAM/I,cAAgE77D,IAA5CO,KAAKmvB,QAAQw9C,OAAO1tB,WAAWkd,IAAqE,GAA3Cn8D,KAAKmvB,QAAQw9C,OAAO1tB,WAAWkd,IACpHmlC,EAAUrkG,KAAK,CAACm2B,GAAG+oC,EAAQolC,OAAOl9B,EAAMl1C,QAAQoyE,SAItDjpD,GAAKmmD,WAAW6C,GAAU,SAASnkG,EAAEC,OAC/BokG,EAAKrkG,EAAEokG,OACPE,EAAKrkG,EAAEmkG,mBACA9hG,IAAP+hG,IAAkBA,EAAG,QACd/hG,IAAPgiG,IAAkBA,EAAG,GAClBD,GAAIC,EAAI,EAAID,EAAGC,GAAM,EAAG,aAE7BzwB,EAAW,IAAI/0E,MAAMqlG,EAAUtkG,QAC1BF,EAAE,EAAGA,EAAGwkG,EAAUtkG,OAAQF,IACjCk0E,EAASl0E,GAAKwkG,EAAUxkG,GAAGs2B,UAEtB49C,GASTyuB,GAAUtjG,UAAUilG,aAAe,cAEjCpJ,GAAwBh4F,KAAKy3F,aACL,GAApBz3F,KAAKq1C,MAAMzM,OAAgC,MAAlB5oC,KAAKylE,UAAmB,KAC/CpB,EAAOvnE,EACPw/F,EAAc,GAGdoF,EAAU1hG,KAAK44C,KAAKN,KAAK44C,cAAclxF,KAAK44C,KAAKY,SAAS2H,KAAKvY,OAC/D+4D,EAAU3hG,KAAK44C,KAAKN,KAAK44C,aAAa,EAAIlxF,KAAK44C,KAAKY,SAAS2H,KAAKvY,OAGlEooC,EAAWhxE,KAAKqhG,wBAChBrwB,EAASh0E,OAAS,EAAG,KACnB+yE,EAAa,YAGZ6xB,iBAAiB5wB,EAAUjB,EAAY2xB,EAASC,QAGhDE,eAAe7wB,EAAUjB,GAGzBjzE,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,SAC1BglG,qBAAqB/xB,EAAWiB,EAASl0E,aAI3CilG,YAAY/wB,EAAUjB,EAAYusB,GAOnB,GAJLt8F,KAAKgiG,aAAahxB,EAAUsrB,UAKzCtE,GAAwBh4F,KAAKy3F,kBACxBsI,oBAAqB,GACnB,OAEJA,oBAAqB,MAGtBkC,OAAQxiG,MACP3C,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC/BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACF,IAAvBkD,KAAKmvB,QAAQ9rB,OAAyC,SAAvBrD,KAAKmvB,QAAQqF,QACL/0B,MAArC4kE,EAAMl1C,QAAQ4sE,qBAAqC13B,EAAMl1C,QAAQ4sE,sBACtDt8F,MAATwiG,SACGC,OAAOnyB,EAAW1L,EAAMjxC,IAAK28C,EAAWkyB,EAAM7uE,KACf,GAAhCixC,EAAMl1C,QAAQ+tE,OAAOxgB,SAAwD,UAArCrY,EAAMl1C,QAAQ+tE,OAAO70C,cACvB,OAApCgc,EAAMl1C,QAAQ+tE,OAAO70C,aAA6D,UAArC45C,EAAM9yE,QAAQ+tE,OAAO70C,aACpE45C,EAAM9yE,QAAQ+tE,OAAO70C,YAAY,QACjC45C,EAAM9yE,QAAQ+tE,OAAO/gC,QAAQkI,EAAMjxC,KAEnCixC,EAAMl1C,QAAQ+tE,OAAO70C,YAAY,QACjCgc,EAAMl1C,QAAQ+tE,OAAO/gC,QAAQ8lC,EAAM7uE,MAIzC6uE,EAAQ59B,SAGP89B,qBAAqBpyB,EAAWiB,EAASl0E,IAAKunE,OAIjD+9B,EAAQ,OACPtlG,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEH,UAD5BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACnBqyB,QAAQqF,OAAoD,GAAhC6vC,EAAMl1C,QAAQ+tE,OAAOxgB,QAAiB,KACtE5D,EAAU/I,EAAWiB,EAASl0E,OACnB,MAAXg8E,GAAqC,GAAlBA,EAAQ97E,mBAG1BolG,EAAM/kG,eAAe2zE,EAASl0E,MACjCslG,EAAMpxB,EAASl0E,IAAMgiG,GAAMjC,SAAS/jB,EAASzU,IAEN,UAArCA,EAAMl1C,QAAQ+tE,OAAO70C,YAAyB,KAC5Cg6C,EAAah+B,EAAMl1C,QAAQ+tE,OAAO/gC,YACA,IAAlCvI,GAAAod,QAAAA,EAAiBqxB,GAAoB,CACvC1/F,QAAQq3C,IAAIqqB,EAAMjxC,GAAK,wCAA0CivE,YAG9DD,EAAM/kG,eAAeglG,KACxBD,EAAMC,GAAcvD,GAAMjC,SAAS9sB,EAAWsyB,GAAariG,KAAK2sE,OAAO01B,KAEzEvD,GAAM3B,YAAYiF,EAAMpxB,EAASl0E,IAAKunE,EAAO+9B,EAAMC,GAAariG,KAAK63F,gBAGrEiH,GAAM3B,YAAYiF,EAAMpxB,EAASl0E,IAAKunE,OAAO5kE,EAAWO,KAAK63F,eAMnEkH,GAAKrP,KAAK1e,EAAUjB,EAAY/vE,KAAK63F,WAChC/6F,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAC/BunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,IACzBizE,EAAWiB,EAASl0E,IAAIE,OAAS,SAC3BqnE,EAAMl1C,QAAQqF,WACf,OACE4tE,EAAM/kG,eAAe2zE,EAASl0E,MACjCslG,EAAMpxB,EAASl0E,IAAMgiG,GAAMjC,SAAS9sB,EAAWiB,EAASl0E,IAAKunE,IAE/Dy6B,GAAMpP,KAAK0S,EAAMpxB,EAASl0E,IAAKunE,EAAOrkE,KAAK63F,eAExC,YAEA,SACwB,SAAvBxzB,EAAMl1C,QAAQqF,OAA2C,UAAvB6vC,EAAMl1C,QAAQqF,OAAyD,GAApC6vC,EAAMl1C,QAAQqlC,WAAWkoB,SAChGid,GAAOjK,KAAK3f,EAAWiB,EAASl0E,IAAKunE,EAAOrkE,KAAK63F,oBAgB/DG,GAAwBh4F,KAAKy3F,cACtB,GAGTgI,GAAUtjG,UAAU+lG,OAAS,SAAU3uF,EAAM+uF,OACvC19E,EAAO29E,EAAIC,EAAIC,EAAcC,EACjC99E,EAAQ,MAEH,IAAI7Q,EAAI,EAAGA,EAAIR,EAAKvW,OAAQ+W,IAAK,CACpC0uF,OAAehjG,EACfijG,OAAejjG,MAEV,IAAIkb,EAAIiK,EAAOjK,EAAI2nF,EAAQtlG,OAAQ2d,IAAK,IAEvC2nF,EAAQ3nF,GAAGxQ,IAAMoJ,EAAKQ,GAAG5J,EAAG,CAC9Bs4F,EAAeH,EAAQ3nF,GACvB+nF,EAAeJ,EAAQ3nF,GACvBiK,EAAQjK,QAGL,GAAI2nF,EAAQ3nF,GAAGxQ,EAAIoJ,EAAKQ,GAAG5J,EAAG,CACjCu4F,EAAeJ,EAAQ3nF,GAErB8nF,EADO,GAAL9nF,EACa+nF,EAGAJ,EAAQ3nF,EAAI,GAE7BiK,EAAQjK,cAKSlb,IAAjBijG,IACFD,EAAeH,EAAQA,EAAQtlG,OAAS,GACxC0lG,EAAeJ,EAAQA,EAAQtlG,OAAS,IAG1CulG,EAAKG,EAAav4F,EAAIs4F,EAAat4F,EACnCq4F,EAAKE,EAAan5F,EAAIk5F,EAAal5F,EAEjCgK,EAAKQ,GAAGxK,EADA,GAANg5F,EACUhvF,EAAKQ,GAAGotF,SAAWuB,EAAan5F,EAGhCgK,EAAKQ,GAAGotF,SAAYqB,EAAKD,GAAOhvF,EAAKQ,GAAG5J,EAAIs4F,EAAat4F,GAAKs4F,EAAal5F,IAmB7Fk2F,GAAUtjG,UAAUylG,iBAAmB,SAAU5wB,EAAUjB,EAAY2xB,EAASC,OAC1Et9B,EAAOvnE,EAAGiX,EAAGgkC,KACbi5B,EAASh0E,OAAS,MACfF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KAEhC2oE,GADJpB,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KACP4hG,cAEI,GAAtBr+B,GAAAgE,EAAMl1C,SAAsB,KAC1BwzE,EAAiB,SAAUxlG,EAAGC,UACzBD,EAAEmC,WAAalC,EAAEkC,UAAY,EAAInC,EAAIC,GAAK,EAAI,GAEnDoiC,EAAQh+B,KAAK8D,IAAI,EAAGgzC,GAAKsqD,kBAAkBn9B,EAAWi8B,EAAS,IAAK,SAAUiB,IAC9E91D,EAAOrrC,KAAKY,IAAIqjE,EAAUzoE,OAAQs7C,GAAKsqD,kBAAkBn9B,EAAWk8B,EAAS,IAAK,QAASgB,GAAkB,GAC7G91D,GAAQ,IACVA,EAAO44B,EAAUzoE,YAEf6lG,EAAgB,IAAI5mG,MAAM4wC,EAAKrN,OAC9BzrB,EAAIyrB,EAAOzrB,EAAI84B,EAAM94B,IACxBgkC,EAAOssB,EAAMoB,UAAU1xD,GACvB8uF,EAAc9uF,EAAEyrB,GAASuY,EAE3Bg4B,EAAWiB,EAASl0E,IAAM+lG,OAI1B9yB,EAAWiB,EAASl0E,IAAMunE,EAAMoB,YAaxCg6B,GAAUtjG,UAAU0lG,eAAiB,SAAU7wB,EAAUjB,MAEnDiB,EAASh0E,OAAS,MACf,IAAIF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,OAEL,GADtBkD,KAAK2sE,OAAOqE,EAASl0E,IACnBqyB,QAAQwwE,SAAkB,KAC9BkD,EAAgB9yB,EAAWiB,EAASl0E,OACpC+lG,EAAc7lG,OAAS,EAAG,KACxB8lG,EACAC,EAAiBF,EAAc7lG,OAM/BgmG,EAAiBD,GADL/iG,KAAK44C,KAAKN,KAAKy4C,eAAe8R,EAAcA,EAAc7lG,OAAS,GAAGmN,GAAKnK,KAAK44C,KAAKN,KAAKy4C,eAAe8R,EAAc,GAAG14F,IAE1I24F,EAAYthG,KAAKY,IAAIZ,KAAKC,KAAK,GAAMshG,GAAiBvhG,KAAK8D,IAAI,EAAG9D,KAAK+a,MAAMymF,aAEzEC,EAAc,IAAIhnG,MAAM8mG,GACnBhvF,EAAI,EAAGA,EAAIgvF,EAAgBhvF,GAAK+uF,EAAW,CAElDG,EADUzhG,KAAK+a,MAAMxI,EAAE+uF,IACND,EAAc9uF,GAEjCg8D,EAAWiB,EAASl0E,IAAMwlE,GAAA2gC,QAAAA,EAAmB,EAAEzhG,KAAK+a,MAAMwmF,EAAeD,OAenFrD,GAAUtjG,UAAU4lG,YAAc,SAAU/wB,EAAUjB,EAAYusB,OAC5D7rB,EAAWpM,EAAOvnE,EAGlBqyB,EAFA+zE,EAAmB,GACnBC,EAAoB,MAEpBnyB,EAASh0E,OAAS,EAAG,KAClBF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC/B2zE,EAAYV,EAAWiB,EAASl0E,IAChCqyB,EAAUnvB,KAAK2sE,OAAOqE,EAASl0E,IAAIqyB,QAC/BshD,EAAUzzE,OAAS,IACrBqnE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,KAEP,IAAlBqyB,EAAQ9rB,OAAoC,QAAlB8rB,EAAQqF,MACH,SAA7BrF,EAAQstE,iBACVyG,EAAmBryB,GAAAqyB,QAAAA,EAAwBzyB,GAG3C0yB,EAAoBtyB,GAAAsyB,QAAAA,EAAyB1yB,GAI/C6rB,EAAYtrB,EAASl0E,IAAMunE,EAAM46B,UAAUxuB,EAAWO,EAASl0E,KAMrEiiG,GAAK1C,iBAAiB6G,EAAkB5G,EAAatrB,EAAU,iBAAkB,QACjF+tB,GAAK1C,iBAAiB8G,EAAmB7G,EAAatrB,EAAU,kBAAmB,WAYvFyuB,GAAUtjG,UAAU6lG,aAAe,SAAUhxB,EAAUsrB,OAIe8G,EAAQC,EAHxExqD,GAAU,EACVyqD,GAAgB,EAChBC,GAAiB,EACjBC,EAAU,IAAKC,EAAW,IAAKC,GAAW,IAAKC,GAAY,OAE3D3yB,EAASh0E,OAAS,EAAG,KAElB,IAAIF,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAAK,KACpCunE,EAAQrkE,KAAK2sE,OAAOqE,EAASl0E,IAC7BunE,GAA2C,SAAlCA,EAAMl1C,QAAQstE,kBACzB6G,GAAgB,EAChBE,EAAU,IACVE,GAAW,KAEJr/B,GAASA,EAAMl1C,QAAQstE,mBAC9B8G,GAAiB,EACjBE,EAAW,IACXE,GAAY,SAKX7mG,EAAI,EAAGA,EAAIk0E,EAASh0E,OAAQF,IAC3Bw/F,EAAYj/F,eAAe2zE,EAASl0E,MACE,IAApCw/F,EAAYtrB,EAASl0E,IAAI8mG,SAC3BR,EAAS9G,EAAYtrB,EAASl0E,IAAIsF,IAClCihG,EAAS/G,EAAYtrB,EAASl0E,IAAIwI,IAEe,SAA7Cg3F,EAAYtrB,EAASl0E,IAAI2/F,kBAC3B6G,GAAgB,EAChBE,EAAUA,EAAUJ,EAASA,EAASI,EACtCE,EAAUA,EAAUL,EAASA,EAASK,IAGtCH,GAAiB,EACjBE,EAAWA,EAAWL,EAASA,EAASK,EACxCE,EAAWA,EAAWN,EAASA,EAASM,IAM3B,GAAjBL,QACGlD,UAAU1hD,SAAS8kD,EAASE,GAEb,GAAlBH,QACGlD,WAAW3hD,SAAS+kD,EAAUE,GAGvC9qD,EAAU74C,KAAK6jG,qBAAqBP,EAAetjG,KAAKogG,YAAcvnD,EACtEA,EAAU74C,KAAK6jG,qBAAqBN,EAAgBvjG,KAAKqgG,aAAexnD,EAElD,GAAlB0qD,GAA2C,GAAjBD,QACvBlD,UAAU0D,WAAY,OACtBzD,WAAWyD,WAAY,SAGvB1D,UAAU0D,WAAY,OACtBzD,WAAWyD,WAAY,QAEzBzD,WAAW9I,QAAU+L,OACrBjD,WAAW7I,WAAax3F,KAAKogG,UAEJ,GAA1BpgG,KAAKqgG,WAAW9I,aAEX6I,UAAU50B,WADK,GAAlB+3B,EAC0BvjG,KAAKqgG,WAAWz3D,MAGhB,EAG9BiQ,EAAU74C,KAAKogG,UAAU/vC,UAAYxX,EACrCA,EAAU74C,KAAKqgG,WAAWhwC,UAAYxX,GAGtCA,EAAU74C,KAAKqgG,WAAWhwC,UAAYxX,MAIpCkrD,EAAa,CAAC,iBAAkB,kBAAmB,kBAAmB,wBACrEjnG,EAAI,EAAGA,EAAIinG,EAAW/mG,OAAQF,KACO,GAApC82D,GAAAod,QAAAA,EAAiB+yB,EAAWjnG,QAC9Bk0E,QAAAA,EAAgBpd,GAAAod,QAAAA,EAAiB+yB,EAAWjnG,IAAK,UAI9C+7C,GAYT4mD,GAAUtjG,UAAU0nG,qBAAuB,SAAUG,EAAU17C,OACzDjI,GAAU,SACE,GAAZ2jD,EACE17C,EAAKlK,IAAIuqB,MAAMphC,YAA6B,GAAf+gB,EAAK/N,SACpC+N,EAAK4H,OACL7P,GAAU,GAIPiI,EAAKlK,IAAIuqB,MAAMphC,YAA6B,GAAf+gB,EAAK/N,SACrC+N,EAAKkb,OACLnjB,GAAU,GAGPA,GAYTo/C,GAAUtjG,UAAU2lG,qBAAuB,SAAUmC,WAC/CzoD,EAAWx7C,KAAK44C,KAAKN,KAAKkD,SACrB1+C,EAAI,EAAGA,EAAImnG,EAAWjnG,OAAQF,IACrCmnG,EAAWnnG,GAAG49F,SAAWl/C,EAASyoD,EAAWnnG,GAAGqN,GAAKnK,KAAKq1C,MAAMzM,MAChEq7D,EAAWnnG,GAAG69F,SAAWsJ,EAAWnnG,GAAGyM,EACd9J,MAArBwkG,EAAWnnG,GAAG2jC,IAChBwjE,EAAWnnG,GAAG2+F,WAAajgD,EAASyoD,EAAWnnG,GAAG2jC,KAAOzgC,KAAKq1C,MAAMzM,MAGpEq7D,EAAWnnG,GAAG2+F,gBAAah8F,GAejCggG,GAAUtjG,UAAUgmG,qBAAuB,SAAU8B,EAAY5/B,OAC3D/b,EAAOtoD,KAAKogG,UACZ7C,EAAYlmD,OAAOr3C,KAAKu2F,IAAI/hE,MAAMqU,OAAOtiC,QAAQ,KAAM,KACrB,SAAlC89D,EAAMl1C,QAAQstE,mBAChBn0C,EAAOtoD,KAAKqgG,gBAET,IAAIvjG,EAAI,EAAGA,EAAImnG,EAAWjnG,OAAQF,IACrCmnG,EAAWnnG,GAAG69F,SAAWn5F,KAAK+a,MAAM+rC,EAAK4sC,aAAa+O,EAAWnnG,GAAGyM,IAEtE86D,EAAMs6B,gBAAgBn9F,KAAKY,IAAIm7F,EAAWj1C,EAAK4sC,aAAa,MCvkC9D,IACIzY,GAAO,UAUPjC,GAAa,CACfxlB,UAAW,CACT0nB,QAAS,SAAYD,IACrBxkD,OAAQ,SAAYwkD,YAAiB,YACrCxwB,UAAW,CAAC7N,IATN,OAUN+8B,SAAU,CAACrsD,OAXF,iBAWoB2tD,YAAiB,aAIhD7jB,iBAAkB,CAAC3hD,OApBR,mBAoB6B,aACxCwlF,iBAAkB,CAACxlF,OAAO,CAAC,OAAO,UAClCyoF,aAAc,CAACzoF,OAtBJ,UAuBXhL,KAAM,SAAYwwE,IAClBkjB,SAAU,SAAYljB,IACtBp5E,MAAM,SAAYo5E,IAClBmjB,YAAa,CAAC3oF,OA1BH,SA0BW1V,OAxBX,UAyBX27F,OAAQ,CACNxgB,QAAS,SAAYD,IACrBp0B,YAAa,CAACpxC,OAAO,CAAC,SAAS,MAAM,OAAO,UAC5CklD,QAAS,CAACrtC,OAzBD,UA0BTqsD,SAAU,SAAYsB,GAAK3tD,OA1BlB,WA4BX0F,MAAO,CAACvd,OAAO,CAAC,OAAO,MAAM,WAC7B+jF,SAAU,CACRpyD,MAAO,CAACrnC,OAjCC,UAkCTqqE,SAAU,CAACrqE,OAlCF,UAmCT26F,WAAY,SAAYzf,IACxBhW,MAAO,CAACxvD,OAAO,CAAC,OAAO,SAAS,UAChCkkE,SAAU,CAACrsD,OAlCF,WAoCXguE,cAAe,CACbpgB,QAAS,SAAYD,IACrBoiB,gBAAiB,CAAC5nF,OAAO,CAAC,cAAe,UAAU,YACnDwwE,MAAO,CAAClmF,OA1CC,UA2CT45E,SAAU,CAACrsD,OAxCF,iBAwCoB2tD,KAE/BjoB,WAAY,CACVkoB,QAAS,SAAYD,IACrBhoB,SAAU,UAAc,YACxB90B,KAAM,CAACp+B,OAhDE,UAiDTizB,MAAO,CAACvd,OAAO,CAAC,SAAS,WACzBkkE,SAAU,CAACrsD,OA/CF,iBA+CoB2tD,YAAiB,aAEhDojB,SAAU,CACRt3C,gBAAiB,SAAYk0B,IAC7Bx2B,gBAAiB,SAAYw2B,IAC7Bn2B,cAAe,SAAYm2B,IAC3Bga,MAAO,SAAYha,IACnB7zC,MAAO,CAAC3xB,OA3DC,SA2DO1V,OAzDP,UA0DT+5D,QAAS,SAAYmhB,IACrBsa,WAAY,SAAYta,IACxB/nC,KAAK,CACHgF,MAAO,CAACt3C,IAAI,CAACb,OA7DN,mBA6D0B,aAAa+D,IAAI,CAAC/D,OA7D5C,mBA6DgE,aAAa45E,SAAU,CAACrsD,OA1DxF,WA2DPrxB,OAAQ,UAAa,YACrBgyD,MAAO,CAAChE,KAAK,CAACx0C,OAjEP,SAiEc1V,OA/Dd,mBA+DkC,aAAaizB,MAAM,CAACvd,OAjEtD,mBAiE0E,aAAakkE,SAAU,CAACrsD,OA5DlG,WA6DPqsD,SAAU,CAACrsD,OA7DJ,WA+DT6lB,MAAM,CACJ+E,MAAO,CAACt3C,IAAI,CAACb,OAnEN,mBAmE0B,aAAa+D,IAAI,CAAC/D,OAnE5C,mBAmEgE,aAAa45E,SAAU,CAACrsD,OAhExF,WAiEPrxB,OAAQ,UAAa,YACrBgyD,MAAO,CAAChE,KAAK,CAACx0C,OAvEP,SAuEc1V,OArEd,mBAqEkC,aAAaizB,MAAM,CAACvd,OAvEtD,mBAuE0E,aAAakkE,SAAU,CAACrsD,OAlElG,WAmEPqsD,SAAU,CAACrsD,OAnEJ,WAqETqsD,SAAU,CAACrsD,OArEF,WAuEXgxE,OAAQ,CACNpjB,QAAS,SAAYD,IACrBga,MAAO,SAAYha,IACnB/nC,KAAM,CACJ4mB,QAAS,SAAYmhB,IACrB/8C,SAAU,CAACzoB,OAAO,CAAC,YAAY,eAAe,WAAW,gBACzDkkE,SAAU,CAACrsD,OA7EJ,WA+ET6lB,MAAO,CACL2mB,QAAS,SAAYmhB,IACrB/8C,SAAU,CAACzoB,OAAO,CAAC,YAAY,eAAe,WAAW,gBACzDkkE,SAAU,CAACrsD,OAlFJ,WAoFTqsD,SAAU,CAACrsD,OApFF,iBAoFoB2tD,KAE/B9P,OAAQ,CACN1tB,WAAY,CAAC29B,IApFP,OAqFNzB,SAAU,CAACrsD,OAxFF,WA2FX+oC,WAAY,SAAY4kB,IACxBE,eAAgB,CAACp7E,OA/FN,UAgGXmzD,WAAY,SAAY+nB,IACxBh8C,IAAK,CAACl/B,OAjGK,SAiGG0I,KA/FL,OA+FWgN,OAnGT,SAmGiBmT,OA5FjB,UA6FX3sB,OAAQ,CACNkpD,YAAa,CACXrsC,YAAa,CAACrD,OAtGP,mBAsG2B,aAClCoD,OAAQ,CAACpD,OAvGF,mBAuGsB,aAC7BmD,OAAQ,CAACnD,OAxGF,mBAwGsB,aAC7BqC,KAAM,CAACrC,OAzGA,mBAyGoB,aAC3BzJ,QAAS,CAACyJ,OA1GH,mBA0GuB,aAC9B7H,IAAK,CAAC6H,OA3GC,mBA2GmB,aAC1B1J,KAAM,CAAC0J,OA5GA,mBA4GoB,aAC3BjN,MAAO,CAACiN,OA7GD,mBA6GqB,aAC5BuE,QAAS,CAACvE,OA9GH,mBA8GuB,aAC9B5N,KAAM,CAAC4N,OA/GA,mBA+GoB,aAC3BkkE,SAAU,CAACrsD,OA3GJ,WA6GT83B,YAAa,CACXtsC,YAAa,CAACrD,OAnHP,mBAmH2B,aAClCoD,OAAQ,CAACpD,OApHF,mBAoHsB,aAC7BmD,OAAQ,CAACnD,OArHF,mBAqHsB,aAC7BqC,KAAM,CAACrC,OAtHA,mBAsHoB,aAC3BzJ,QAAS,CAACyJ,OAvHH,mBAuHuB,aAC9B7H,IAAK,CAAC6H,OAxHC,mBAwHmB,aAC1B1J,KAAM,CAAC0J,OAzHA,mBAyHoB,aAC3BjN,MAAO,CAACiN,OA1HD,mBA0HqB,aAC5BuE,QAAS,CAACvE,OA3HH,mBA2HuB,aAC9B5N,KAAM,CAAC4N,OA5HA,mBA4HoB,aAC3BkkE,SAAU,CAACrsD,OAxHJ,WA0HTqsD,SAAU,CAACrsD,OA1HF,WA4HX1E,OAAQ,UAAa,YACrBye,OAAQ,CAAC5xB,OAlIE,SAkIM1V,OAhIN,UAiIX63C,YAAa,CACX5Y,MAAO,CAACv2B,KAhID,OAgIO1I,OAlIL,SAkIa0V,OApIb,SAoIqBmT,OA7HrB,UA8HTqW,IAAK,CAACx2B,KAjIC,OAiIK1I,OAnIH,SAmIW0V,OArIX,SAqImBmT,OA9HnB,UA+HT4uB,OAAQ,CAAC/hC,OAtIA,UAuITkkE,SAAU,CAACrsD,OAlIF,SAkIUzoB,MApIX,UAsIV3I,OAAO,CAACuZ,OAzIG,UA0IXrE,QAAQ,CACNmoE,QAAS,CAAC6B,IAnIJ,OAoINzB,SAAU,CAACrsD,OAvIF,WAyIXxpB,IAAK,CAAC2E,KA1IG,OA0IG1I,OA5ID,SA4IS0V,OA9IT,SA8IiBmT,OAvIjB,UAwIXisC,UAAW,CAAC90D,OA7ID,SA6IS0V,OA/IT,UAgJXuxC,cAAe,CAACjnD,OA9IL,UA+IXa,IAAK,CAAC6H,KA7IG,OA6IG1I,OA/ID,SA+IS0V,OAjJT,SAiJiBmT,OA1IjB,UA2IXosC,UAAW,CAACj1D,OAhJD,SAgJS0V,OAlJT,UAmJXmmC,SAAU,SAAYq/B,IACtBxN,YAAa,SAAYwN,IACzBp0B,YAAa,CAACpxC,OArJH,UAsJX0hD,gBAAiB,SAAY8jB,IAC7Bx2B,gBAAiB,SAAYw2B,IAC7Bl0B,gBAAiB,SAAYk0B,IAC7Bn2B,cAAe,SAAYm2B,IAC3Bj8C,MAAO,CAACv2B,KAtJC,OAsJK1I,OAxJH,SAwJW0V,OA1JX,SA0JmBmT,OAnJnB,UAoJXq+B,SAAU,CACR9M,MAAO,CAAC1kC,OA5JC,mBA4JmB,aAC5B8sB,KAAM,CAACxiC,OA3JE,mBA2JkB,aAC3B45E,SAAU,CAACrsD,OAzJF,WA2JX8Z,MAAO,CAAC3xB,OAhKG,SAgKK1V,OA9JL,UA+JX87C,SAAU,SAAYo/B,IACtB36B,QAAS,CAAC7qC,OAAQ,CAAC,UAAW,SAAU,UAAW,KACnDsmC,QAAS,CAACh8C,OAjKC,UAkKX+7C,QAAS,CAAC/7C,OAlKC,UAmKXggG,OAAQ,CAAChgG,OAnKE,UAoKX45E,SAAU,CAACrsD,OAjKA,WAoKTiuD,GAAmB,CACrB7wD,OAAQ,CACN0sC,iBAAkB,CAAC,OAAQ,OAAQ,QAAS,UAAW,OAAQ,UAAW,MAAO,OAAQ,OAAQ,SAAU,UAE3G3sD,MAAM,EACN0zF,UAAU,EACVt8F,OAAM,EACN65F,OAAQ,CACNxgB,SAAS,EACTr0B,YAAa,CAAC,OAAO,MAAM,SAAS,UAEtC7zB,MAAO,CAAC,OAAO,MAAM,UACrBwmE,SAAU,CACRpyD,MAAO,CAAC,GAAG,EAAE,IAAI,GACjBgjC,SAAU,CAAC,GAAG,EAAE,IAAI,GACpBswB,YAAY,EACZz1B,MAAO,CAAC,OAAO,SAAS,UAE1Bq2B,cAAe,CACbpgB,SAAS,EACTmiB,gBAAiB,CAAC,cAAc,UAAU,YAE5CrqC,WAAY,CACVkoB,SAAS,EACT/8C,KAAM,CAAC,EAAE,EAAE,GAAG,GACdnL,MAAO,CAAC,SAAU,WAEpBqrE,SAAU,CACRt3C,iBAAiB,EACjBtC,iBAAiB,EACjBK,eAAe,EACfmwC,OAAO,EACP7tD,MAAO,CAAC,GAAG,EAAE,IAAI,GACjB0yB,SAAS,EACTy7B,YAAY,EACZriD,KAAK,CAGH+a,MAAO,CAAChE,KAAK,GAAGj3B,MAAM,KAExBmgB,MAAM,CAGJ8a,MAAO,CAAChE,KAAK,GAAGj3B,MAAM,MAG1BsrE,OAAQ,CACNpjB,SAAS,EACT+Z,OAAO,EACP/hD,KAAM,CACJ4mB,SAAS,EACT57B,SAAU,CAAC,YAAY,eAAe,WAAW,gBAEnDiV,MAAO,CACL2mB,SAAS,EACT57B,SAAU,CAAC,YAAY,eAAe,WAAW,iBAIrDm4B,YAAY,EACZnD,YAAY,EACZj0B,IAAK,GACLhjC,OAAQ,CACNkpD,YAAa,CACXrsC,YAAY,MACZD,OAAY,IACZD,OAAY,QACZd,KAAY,QACZ9L,QAAY,QACZ4B,IAAY,IACZ7B,KAAY,IACZvD,MAAY,MACZwR,QAAY,OACZnS,KAAY,QAEdu9C,YAAa,CACXtsC,YAAY,WACZD,OAAY,eACZD,OAAY,aACZd,KAAY,aACZ9L,QAAY,YACZ4B,IAAY,YACZ7B,KAAY,YACZvD,MAAY,OACZwR,QAAY,OACZnS,KAAY,KAIhBw/B,OAAQ,GACRnrC,OAAQ,GACR4H,IAAK,GACL+wD,UAAW,GACX7N,cAAe,CAAC,EAAG,EAAG,GAAI,GAC1BpmD,IAAK,GACLo0D,UAAW,GACXpZ,UAAS,EACTiL,YAAa,CAAC,OAAQ,SAAU,OAChCsQ,iBAAiB,EACjB1S,iBAAiB,EACjBsC,iBAAiB,EACjBjC,eAAe,EACf9lB,MAAO,GACPoI,MAAO,OACPyU,UAAU,EACVyE,QAAS,CAAC,UAAW,SAAU,UAAW,IAC1CvE,QAAS,CAAC,SAAiB,GAAI,SAAiB,GAChDD,QAAS,CAAC,GAAI,GAAI,SAAiB,GACnCikD,OAAQ,ICjQZ,SAAS2C,GAASj4C,EAAWzV,EAAOm2B,EAAQx9C,0BAEpC+T,GAAcypC,IAAWA,aAAkB/0B,IAAW+0B,aAAkB2H,KAAa3H,aAAkBzwE,OAAQ,KAC/Gu0F,EAAgBthE,EACpBA,EAAUw9C,EACVA,EAAS8jB,EAKPthE,GAAWA,EAAQwtD,gBACrBh6E,QAAQH,KAAK,4HAGXm8C,EAAK3+C,UACJk9C,eAAiB,CACpB1c,MAAO,KACPC,IAAO,KAEPo3B,YAAY,EAEZxP,YAAa,CACXC,KAAM,SACNvQ,KAAM,UAGR3tB,OAAQA,GAERwe,MAAO,KACPC,OAAQ,KACRwtB,UAAW,KACXG,UAAW,WAERrnC,QAAUmpB,GAAKuN,WAAW,GAAI7lD,KAAKk9C,qBAGnCwL,QAAQuD,QAGRsI,WAAa,QAEb3b,KAAO,CACVwF,IAAKp+C,KAAKo+C,IACV5E,SAAUx5C,KAAKq1C,MACfwI,QAAS,CACPvH,GAAIP,UAAKO,WAAQt2C,MACjBu2C,IAAKR,UAAKQ,YAASv2C,MACnBygD,KAAM1K,UAAK0K,aAAUzgD,OAEvBo5C,YAAa,GACbd,KAAM,CACJkD,SAAUzF,KAAA4I,EAAGmyC,kBAAenyC,GAC5BoyC,eAAgBh7C,KAAA4I,EAAGqyC,wBAAqBryC,GACxC1C,OAAQlG,KAAA4I,EAAGsyC,gBAAatyC,GACxBuyC,aAAen7C,KAAA4I,EAAGwyC,sBAAmBxyC,UAKpCjF,MAAQ,IAAImD,GAAM78C,KAAK44C,WACvB2b,WAAWt3D,KAAK+C,KAAK05C,YACrBd,KAAKc,MAAQ15C,KAAK05C,WAGlB+O,SAAW,IAAIV,GAAS/nD,KAAK44C,WAC7B2b,WAAWt3D,KAAK+C,KAAKyoD,eAIrBiP,YAAc,IAAIgB,GAAY14D,KAAK44C,WACnC2b,WAAWt3D,KAAK+C,KAAK03D,kBAGrBysC,UAAY,IAAI1E,GAAUz/F,KAAK44C,WAE/B2b,WAAWt3D,KAAK+C,KAAKmkG,gBAErB1+B,UAAY,UACZsK,WAAa,UAGbz5B,GAAG,OAAO,SAAUkK,GACvB7B,EAAG8B,KAAK,QAAS9B,EAAGgV,mBAAmBnT,YAEpClK,GAAG,aAAa,SAAUkK,GAC7B7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,YAE1CpC,IAAI+C,KAAKiwC,cAAgB,SAAU5wC,GACtC7B,EAAG8B,KAAK,cAAe9B,EAAGgV,mBAAmBnT,UAI1CoxC,gBAAiB,OACjBt7C,GAAG,WAAW,cACG,MAAhBqI,EAAG8mB,eACF9mB,EAAGizC,iBAAmBjzC,EAAGxvB,QAAQquB,eACpCmB,EAAGizC,gBAAiB,EACInyF,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,IAAkB,IACxChhC,MAApBk/C,EAAGxvB,QAAQqR,OAAwC/gC,MAAlBk/C,EAAGxvB,QAAQsR,QAC1CiZ,EAAQiF,EAAGkzC,mBAGbrxD,EAA4B/gC,MAApBk/C,EAAGxvB,QAAQqR,MAAqBme,EAAGxvB,QAAQqR,MAAQkZ,EAAMt3C,IACjEq+B,EAA4BhhC,MAApBk/C,EAAGxvB,QAAQsR,IAAqBke,EAAGxvB,QAAQsR,IAAQiZ,EAAMp0C,IACrEq5C,EAAGyX,UAAU51B,EAAOC,EAAK,CAACse,WAAW,SAErCJ,EAAGmzC,IAAI,CAAC/yC,WAAW,IAIlBJ,EAAGmT,kBAAoBnT,EAAGqT,yBAA4BrT,EAAGxvB,QAAQqR,OAAUme,EAAGxvB,QAAQsR,OACtFke,EAAGxvB,QAAQquB,cACdmB,EAAGmT,iBAAkB,EACrBnT,EAAGP,IAAI+C,KAAK3sB,MAAMyqB,WAAa,UAC/BN,EAAGP,IAAIsT,cAAcnqB,WAAWC,YAAYmX,EAAGP,IAAIsT,eAC/C/S,EAAGxvB,QAAQ0tD,2BACF,kBACFl+B,EAAGxvB,QAAQ0tD,0BACjB,QAML1tD,QACGqvB,WAAWrvB,GAIdw9C,QACGvX,UAAUuX,GAIbn2B,QACG2e,SAAS3e,QAIXub,UAIPmyC,GAAQ/nG,UAAY,IAAIs/C,GAExByoD,GAAQ/nG,UAAUqiD,WAAa,SAAUrvB,IAGpB,IADFwrD,GAAUqX,SAAS7iE,EAASqrD,KAE3C73E,QAAQq3C,IAAI,2DAA4D0gC,IAG1Ej/B,GAAKt/C,UAAUqiD,WAAWniD,KAAK2D,KAAMmvB,IAOvC+0E,GAAQ/nG,UAAUg5D,SAAW,SAAS3e,OAIhCy7C,EAHAmS,EAAiC,MAAlBpkG,KAAKylE,aAQtBwsB,EAJGz7C,EAIUe,GADNf,aAAiBoB,IAAWpB,aAAiB89B,GACrB99B,EAIA,IAAIoB,GAAQpB,IAP9B,KAWXx2C,KAAKylE,gBAEFA,UAAUptB,eAEZotB,UAAYwsB,OACZkS,WAAankG,KAAKmkG,UAAUhvC,SAAuB,MAAd88B,EAAqBA,EAAWz6C,MAAQ,MAE9E4sD,KACwB3kG,MAAtBO,KAAKmvB,QAAQqR,OAA0C/gC,MAApBO,KAAKmvB,QAAQsR,IAAkB,KAChED,EAA8B/gC,MAAtBO,KAAKmvB,QAAQqR,MAAqBxgC,KAAKmvB,QAAQqR,MAAQ,KAC/DC,EAA4BhhC,MAApBO,KAAKmvB,QAAQsR,IAAqBzgC,KAAKmvB,QAAQsR,IAAM,UAC5D21B,UAAU51B,EAAOC,EAAK,CAACse,WAAW,cAGlC+yC,IAAI,CAAC/yC,WAAW,KAS3BmlD,GAAQ/nG,UAAUi5D,UAAY,SAASuX,OAEjCslB,EAKFA,EAJGtlB,EAGIA,aAAkB/0B,IAAW+0B,aAAkB2H,GACzC3H,EAIA,IAAI/0B,GAAQ+0B,GAPZ,UAUVoD,WAAakiB,OACbkS,UAAU/uC,UAAU68B,IAU3BiS,GAAQ/nG,UAAUg8F,UAAY,SAASh8B,EAASvzB,EAAOC,eACtCppC,IAAXmpC,IAAuBA,EAAS,SACrBnpC,IAAXopC,IAAuBA,EAAS,SACGppC,IAAnCO,KAAKmkG,UAAUx3B,OAAOxQ,GACjBn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASg8B,UAAUvvD,EAAMC,GAG/C,sBAAyBszB,EAAU,KAS9C+nC,GAAQ/nG,UAAUkoG,eAAiB,SAASloC,eACH18D,IAAnCO,KAAKmkG,UAAUx3B,OAAOxQ,KAChBn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASb,eAAkE77D,IAAtDO,KAAKmkG,UAAUh1E,QAAQw9C,OAAO1tB,WAAWkd,IAA+E,GAArDn8D,KAAKmkG,UAAUh1E,QAAQw9C,OAAO1tB,WAAWkd,MAcnK+nC,GAAQ/nG,UAAU25D,aAAe,eAC3B1zD,EAAM,KACNkD,EAAM,SAGL,IAAI62D,KAAWn8D,KAAKmkG,UAAUx3B,UAC7B3sE,KAAKmkG,UAAUx3B,OAAOtvE,eAAe8+D,IACO,GAA1Cn8D,KAAKmkG,UAAUx3B,OAAOxQ,GAASb,YAC5B,IAAIx+D,EAAI,EAAGA,EAAIkD,KAAKmkG,UAAUx3B,OAAOxQ,GAASsJ,UAAUzoE,OAAQF,IAAK,KACpEi7C,EAAO/3C,KAAKmkG,UAAUx3B,OAAOxQ,GAASsJ,UAAU3oE,GAChDgF,EAAQw2C,GAAKpB,QAAQa,EAAK5tC,EAAG,QAAQ5M,UACzC6E,EAAa,MAAPA,GAAsBA,EAAMN,EAAdA,EAA8BM,EAClDkD,EAAa,MAAPA,GAAsBA,EAAMxD,EAAdA,EAA8BwD,QAMnD,CACLlD,IAAa,MAAPA,EAAe,IAAI1F,KAAK0F,GAAO,KACrCkD,IAAa,MAAPA,EAAe,IAAI5I,KAAK4I,GAAO,OAWzC4+F,GAAQ/nG,UAAUw3D,mBAAqB,SAAUnT,OAC3C2B,EAAU3B,EAAMxB,OAASwB,EAAMxB,OAAO70C,EAAIq2C,EAAM2B,QAChDC,EAAU5B,EAAMxB,OAASwB,EAAMxB,OAAOz1C,EAAIi3C,EAAM4B,QAChDj4C,EAAIg4C,EAAU7J,GAAKgsD,gBAAgBtkG,KAAKo+C,IAAI3E,iBAC5ClwC,EAAI64C,EAAU9J,GAAKisD,eAAevkG,KAAKo+C,IAAI3E,iBAC3C/4B,EAAO1gB,KAAKixF,QAAQ9mF,GAEpBwlD,EAAaH,GAAW+jC,qBAAqB/yC,GAE7C5Y,EAAU0Q,GAAKk7C,UAAUhzC,GACzBizC,EAAO,KACPn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKyoD,SAASrK,IAAI4J,aACrChoD,KAAKs0D,WAAahc,GAAKo7C,UAAU9rD,EAAS5nC,KAAKs0D,UAAUlW,IAAI4J,YADGyrC,EAAO,OAEvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU/D,UAAUhiD,IAAIuqB,QACrDrwB,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU9D,WAAWjiD,IAAIuqB,OADU8qB,EAAO,YAEvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU7D,WAAWliD,IAAIuqB,QACtDrwB,GAAKo7C,UAAU9rD,EAAS5nC,KAAKmkG,UAAU5D,YAAYniD,IAAIuqB,OADS8qB,EAAO,SAEzD,MAAd9jC,EAAoC8jC,EAAO,cAC3Cn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAK03D,YAAY7H,KAAuB4jC,EAAO,eACvEn7C,GAAKo7C,UAAU9rD,EAAS5nC,KAAKo+C,IAAIY,UAA+By0C,EAAO,kBAE5E3xF,EAAQ,GACRs+F,EAAYpgG,KAAKmkG,UAAU/D,UAC3BC,EAAargG,KAAKmkG,UAAU9D,kBAC3BD,EAAU7lD,QAAUv6C,KAAKylE,UAAUzoE,OAAS,GAC/C8E,EAAM7E,KAAKmjG,EAAUlH,cAAc3vF,KAEhC82F,EAAW9lD,QAAUv6C,KAAKylE,UAAUzoE,OAAS,GAChD8E,EAAM7E,KAAKojG,EAAWnH,cAAc3vF,IAG/B,CACLi3C,MAAOA,EACPmP,WAAYA,EAAaA,EAAWxgC,QAAQiE,GAAK,KACjDqgE,KAAMA,EACNE,MAAOnzC,EAAM4D,SAAW5D,EAAM4D,SAASuvC,MAAQnzC,EAAMmzC,MACrDC,MAAOpzC,EAAM4D,SAAW5D,EAAM4D,SAASwvC,MAAQpzC,EAAMozC,MACrDzpF,EAAGA,EACHZ,EAAGA,EACHmX,KAAMA,EACN5e,MAAOA,IASXoiG,GAAQ/nG,UAAU44D,oBAAsB,kBAC/B,IAAIo3B,GAAansF,KAAMA,KAAKo+C,IAAI6N,UAAW8wB,KCtTpD,IAAMynB,GAAkBxM,QACjBt6F,OAAO8mG,QAER3Q,GAAW,CACfp4C,KAAAA,GACA6E,SAAAA,GACAzD,MAAAA,GACAx5C,MAAAA,GACAmiD,SAAAA,GAEA+O,WAAY,CACV/d,MAAO,CACLutB,KAAAA,GACA6F,eAAAA,GACArD,QAAAA,GACA6B,UAAAA,GACAM,UAAAA,IAGF5E,gBAAAA,GACAnrB,UAAAA,GACA+f,YAAAA,GACAlJ,WAAAA,GACA8mC,SAAAA,GACArC,UAAAA,GACA+F,WAAAA,GACA99B,MAAAA,GACAyS,QAAAA,GACA0rB,OAAAA,GACAoF,UAAAA,GACA13C,SAAAA"}
\ No newline at end of file
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 76934210aa79e..7ce00b7d19a56 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -28,9 +28,7 @@ dagre-d3.min.js
 graphlib-dot.min.js
 sorttable.js
 vis-timeline-graph2d.min.js
-vis-timeline-graph2d.min.js.map
 vis-timeline-graph2d.min.css
-vis-timeline-graph2d.min.css.map
 dataTables.bootstrap.css
 dataTables.bootstrap.min.js
 dataTables.rowsGroup.js
diff --git a/licenses-binary/LICENSE-vis-timeline.txt b/licenses-binary/LICENSE-vis-timeline.txt
index d343783c16791..324402de4dc09 100644
--- a/licenses-binary/LICENSE-vis-timeline.txt
+++ b/licenses-binary/LICENSE-vis-timeline.txt
@@ -1,23 +1,22 @@
-vis-timeline and vis-graph2d
-https://visjs.github.io/vis-timeline/
+vis.js
+https://github.com/almende/vis
 
-Create a fully customizable, interactive timeline with items and ranges.
+A dynamic, browser-based visualization library.
 
-@version 7.3.4
-@date    2020-03-18T17:03:58.105Z
-
-@copyright (c) 2011-2017 Almende B.V, http://almende.com
-@copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+@version 4.20.1-SNAPSHOT
+@date    2017-10-12
 
 @license
-vis.js is dual licensed under both
+Copyright (C) 2011-2017 Almende B.V, http://almende.com
+
+Vis.js is dual licensed under both
 
-  1. The Apache 2.0 License
-     http://www.apache.org/licenses/LICENSE-2.0
+* The Apache 2.0 License
+  http://www.apache.org/licenses/LICENSE-2.0
 
-  and
+and
 
-  2. The MIT License
-     http://opensource.org/licenses/MIT
+* The MIT License
+  http://opensource.org/licenses/MIT
 
-vis.js may be distributed under either license.
+Vis.js may be distributed under either license.
diff --git a/licenses/LICENSE-vis-timeline.txt b/licenses/LICENSE-vis-timeline.txt
index d343783c16791..324402de4dc09 100644
--- a/licenses/LICENSE-vis-timeline.txt
+++ b/licenses/LICENSE-vis-timeline.txt
@@ -1,23 +1,22 @@
-vis-timeline and vis-graph2d
-https://visjs.github.io/vis-timeline/
+vis.js
+https://github.com/almende/vis
 
-Create a fully customizable, interactive timeline with items and ranges.
+A dynamic, browser-based visualization library.
 
-@version 7.3.4
-@date    2020-03-18T17:03:58.105Z
-
-@copyright (c) 2011-2017 Almende B.V, http://almende.com
-@copyright (c) 2017-2019 visjs contributors, https://github.com/visjs
+@version 4.20.1-SNAPSHOT
+@date    2017-10-12
 
 @license
-vis.js is dual licensed under both
+Copyright (C) 2011-2017 Almende B.V, http://almende.com
+
+Vis.js is dual licensed under both
 
-  1. The Apache 2.0 License
-     http://www.apache.org/licenses/LICENSE-2.0
+* The Apache 2.0 License
+  http://www.apache.org/licenses/LICENSE-2.0
 
-  and
+and
 
-  2. The MIT License
-     http://opensource.org/licenses/MIT
+* The MIT License
+  http://opensource.org/licenses/MIT
 
-vis.js may be distributed under either license.
+Vis.js may be distributed under either license.

From 43fee8b913dfb8c0742c488c1259bb3c30f19eaf Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sat, 13 Jun 2020 19:04:33 -0700
Subject: [PATCH 0825/1280] [SPARK-31644][BUILD][FOLLOWUP] Make Spark's guava
 version configurable from the command line for sbt

This PR proposes to support guava version configurable from command line for sbt.

sbt is usually faster than Maven so it's useful for developers.

No.

I confirmed the guava version is changed with the following commands.
```
 $ build/sbt "inspect tree clean"  | grep guava
[info]       +-spark/*:dependencyOverrides = Set(com.google.guava:guava:14.0.1, xerces:xercesImpl:2.12.0, jline:jline:2.14.6, org.apache.avro:avro:1.8.2)
```
```
$ build/sbt -Dguava.version=25.0-jre "inspect tree clean"  | grep guava
[info]       +-spark/*:dependencyOverrides = Set(com.google.guava:guava:25.0-jre, xerces:xercesImpl:2.12.0, jline:jline:2.14.6, org.apache.avro:avro:1.8.2)
```

Closes #28822 from sarutak/guava-version-for-sbt.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 610acb2fe4e2d8f76c4a863e3872d94299b88c4c)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 project/SparkBuild.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index c9521ea268579..50d1527599b71 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -620,8 +620,9 @@ object KubernetesIntegrationTests {
  * Overrides to work around sbt's dependency resolution being different from Maven's.
  */
 object DependencyOverrides {
+  lazy val guavaVersion = sys.props.get("guava.version").getOrElse("14.0.1")
   lazy val settings = Seq(
-    dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
+    dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
     dependencyOverrides += "commons-io" % "commons-io" % "2.4",
     dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
     dependencyOverrides += "jline" % "jline" % "2.14.6",

From e533d34aeda6376235b566337047f75f0a83c746 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sun, 14 Jun 2020 14:17:16 +0900
Subject: [PATCH 0826/1280] [SPARK-31632][CORE][WEBUI][FOLLOWUP] Enrich the
 exception message when application summary is unavailable

### What changes were proposed in this pull request?
<!--
Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
  1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
  2. If you fix some SQL features, you can provide some references of other DBMSes.
  3. If there is design documentation, please add the link.
  4. If there is a discussion in the mailing list, please add the link.
-->
This PR enriches the exception message when application summary is not available.
#28444 covers the case when application information is not available but the case application summary is not available is not covered.

### Why are the changes needed?
<!--
Please clarify why the changes are needed. For instance,
  1. If you propose a new API, clarify the use case for a new API.
  2. If you fix a bug, you can clarify why it is a bug.
-->
To complement #28444 .

### Does this PR introduce _any_ user-facing change?
<!--
Note that it means *any* user-facing change including all aspects such as the documentation fix.
If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
If possible, please also clarify if this is a user-facing change compared to the released Spark versions or within the unreleased branches such as master.
If no, write 'No'.
-->
Yes.
Before this change, we can get the following error message when we access to `/jobs` if application summary is not available.
<img width="707" alt="no-such-element-exception-error-message" src="https://user-images.githubusercontent.com/4736016/84562182-6aadf200-ad8d-11ea-8980-d63edde6fad6.png">

After this change, we can get the following error message. It's like #28444 does.
<img width="1349" alt="enriched-errorm-message" src="https://user-images.githubusercontent.com/4736016/84562189-85806680-ad8d-11ea-8346-4da2ec11df2b.png">

### How was this patch tested?
<!--
If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
If tests were not added, please describe why they were not added and/or why it was difficult to add.
-->

I checked with the following procedure.
1. Set breakpoint in the line of `kvstore.write(appSummary)` in `AppStatusListener#onStartApplicatin`. Only the thread reaching this line should be suspended.
2. Start spark-shell and wait few seconds.
3. Access to `/jobs`

Closes #28820 from sarutak/fix-no-such-element.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c2e5012a0a76734acf94b8716ee293ba2f58ccb4)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/status/AppStatusStore.scala    | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index d7f5ac8d3dbc9..a4fa256b18adb 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -519,7 +519,13 @@ private[spark] class AppStatusStore(
   }
 
   def appSummary(): AppSummary = {
-    store.read(classOf[AppSummary], classOf[AppSummary].getName())
+    try {
+      store.read(classOf[AppSummary], classOf[AppSummary].getName())
+    } catch {
+      case _: NoSuchElementException =>
+        throw new NoSuchElementException("Failed to get the application summary. " +
+          "If you are starting up Spark, please wait a while until it's ready.")
+    }
   }
 
   def close(): Unit = {

From e2cde45a597189c64dbde7f0d6e2c25001578e22 Mon Sep 17 00:00:00 2001
From: TJX2014 <xiaoxingstack@gmail.com>
Date: Sat, 13 Jun 2020 22:21:35 -0700
Subject: [PATCH 0827/1280] [SPARK-31968][SQL] Duplicate partition columns
 check when writing data
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
A unit test is added
Partition duplicate check added in `org.apache.spark.sql.execution.datasources.PartitioningUtils#validatePartitionColumn`

### Why are the changes needed?
When people write data with duplicate partition column, it will cause a `org.apache.spark.sql.AnalysisException: Found duplicate column ...` in loading data from the  writted.

### Does this PR introduce _any_ user-facing change?
Yes.
It will prevent people from using duplicate partition columns to write data.
1. Before the PR:
It will look ok at `df.write.partitionBy("b", "b").csv("file:///tmp/output")`,
but get an exception when read：
`spark.read.csv("file:///tmp/output").show()`
org.apache.spark.sql.AnalysisException: Found duplicate column(s) in the partition schema: `b`;
2. After the PR：
`df.write.partitionBy("b", "b").csv("file:///tmp/output")` will trigger the exception：
org.apache.spark.sql.AnalysisException: Found duplicate column(s) b, b: `b`;

### How was this patch tested?
Unit test.

Closes #28814 from TJX2014/master-SPARK-31968.

Authored-by: TJX2014 <xiaoxingstack@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a4ea599b1b9b8ebaae0100b54e6ac1d7576c6d8c)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/execution/datasources/PartitioningUtils.scala  |  3 +++
 .../spark/sql/sources/PartitionedWriteSuite.scala      | 10 +++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
index f7e225b0cdc96..5846d46e146ed 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
@@ -545,6 +545,9 @@ object PartitioningUtils {
       partitionColumns: Seq[String],
       caseSensitive: Boolean): Unit = {
 
+    SchemaUtils.checkColumnNameDuplication(
+      partitionColumns, partitionColumns.mkString(", "), caseSensitive)
+
     partitionColumnsSchema(schema, partitionColumns, caseSensitive).foreach {
       field => field.dataType match {
         case _: AtomicType => // OK
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
index 1c4e2a967b0a2..6df1c5db14c26 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/PartitionedWriteSuite.scala
@@ -24,7 +24,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext
 
 import org.apache.spark.TestUtils
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol
@@ -156,4 +156,12 @@ class PartitionedWriteSuite extends QueryTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-31968: duplicate partition columns check") {
+    withTempPath { f =>
+      val e = intercept[AnalysisException](
+        Seq((3, 2)).toDF("a", "b").write.partitionBy("b", "b").csv(f.getAbsolutePath))
+      assert(e.getMessage.contains("Found duplicate column(s) b, b: `b`;"))
+    }
+  }
 }

From 30637a81fb50492e3c54759bb3fd7aac1cd0e326 Mon Sep 17 00:00:00 2001
From: uncleGen <hustyugm@gmail.com>
Date: Sun, 14 Jun 2020 14:49:01 +0900
Subject: [PATCH 0828/1280] [SPARK-31593][SS] Remove unnecessary streaming
 query progress update

### What changes were proposed in this pull request?

Structured Streaming progress reporter will always report an `empty` progress when there is no new data. As design, we should provide progress updates every 10s (default) when there is no new data.

Before PR:

![20200428175008](https://user-images.githubusercontent.com/7402327/80474832-88a8ca00-897a-11ea-820b-d4be6127d2fe.jpg)
![20200428175037](https://user-images.githubusercontent.com/7402327/80474844-8ba3ba80-897a-11ea-873c-b7137bd4a804.jpg)
![20200428175102](https://user-images.githubusercontent.com/7402327/80474848-8e061480-897a-11ea-806e-28c6bbf1fe03.jpg)

After PR:

![image](https://user-images.githubusercontent.com/7402327/80475099-f35a0580-897a-11ea-8fb3-53f343df2c3f.png)

### Why are the changes needed?

Fixes a bug around incorrect progress report

### Does this PR introduce any user-facing change?

Fixes a bug around incorrect progress report

### How was this patch tested?

existing ut and manual test

Closes #28391 from uncleGen/SPARK-31593.

Authored-by: uncleGen <hustyugm@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1e40bccf447dccad9d31bccc75d21b8fca77ba52)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../streaming/ProgressReporter.scala          |  2 +-
 .../StreamingDeduplicationSuite.scala         |  7 ++-
 .../StreamingQueryListenerSuite.scala         | 56 +++++++++++++++++--
 3 files changed, 58 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index 0dff1c2fe5768..ea1f2ce3943b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -201,7 +201,7 @@ trait ProgressReporter extends Logging {
 
     if (hasExecuted) {
       // Reset noDataEventTimestamp if we processed any data
-      lastNoExecutionProgressEventTime = Long.MinValue
+      lastNoExecutionProgressEventTime = triggerClock.getTimeMillis()
       updateProgress(newProgress)
     } else {
       val now = triggerClock.getTimeMillis()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
index f63778aef5a7f..51ddc7b49fcda 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingDeduplicationSuite.scala
@@ -281,7 +281,12 @@ class StreamingDeduplicationSuite extends StateStoreMetricsTest {
           if (flag) assertNumStateRows(total = 1, updated = 1)
           else assertNumStateRows(total = 7, updated = 1)
         },
-        AssertOnQuery(q => q.lastProgress.sink.numOutputRows == 0L)
+        AssertOnQuery { q =>
+          eventually(timeout(streamingTimeout)) {
+            q.lastProgress.sink.numOutputRows == 0L
+            true
+          }
+        }
       )
     }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index e585b8a885c9c..6e08b88f538df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -389,7 +389,7 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     // Structured Streaming in Spark 2.0.0. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.0.txt", 1)
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.0.txt", 1)
   }
 
   testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_1") {
@@ -397,14 +397,14 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     // Structured Streaming in Spark 2.0.1. Because we renamed the classes,
     // SparkListenerApplicationEnd is the only valid event and it's the last event. We use it
     // to verify that we can skip broken jsons generated by Structured Streaming.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.1.txt", 1)
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.1.txt", 1)
   }
 
   testQuietly("ReplayListenerBus should ignore broken event jsons generated in 2_0_2") {
     // query-event-logs-version-2.0.2.txt has all types of events generated by
     // Structured Streaming in Spark 2.0.2. SPARK-18516 refactored Structured Streaming query events
     // in 2.1.0. This test is to verify we are able to load events generated by Spark 2.0.2.
-    testReplayListenerBusWithBorkenEventJsons("query-event-logs-version-2.0.2.txt", 5)
+    testReplayListenerBusWithBrokenEventJsons("query-event-logs-version-2.0.2.txt", 5)
   }
 
   test("listener propagates observable metrics") {
@@ -433,9 +433,13 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
 
     try {
+      val noDataProgressIntervalKey = SQLConf.STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL.key
       spark.streams.addListener(listener)
       testStream(df, OutputMode.Append)(
-        StartStream(Trigger.ProcessingTime(100), triggerClock = clock),
+        StartStream(
+          Trigger.ProcessingTime(100),
+          triggerClock = clock,
+          Map(noDataProgressIntervalKey -> "100")),
         // Batch 1
         AddData(inputData, 1, 2),
         AdvanceManualClock(100),
@@ -464,7 +468,49 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     }
   }
 
-  private def testReplayListenerBusWithBorkenEventJsons(
+  test("SPARK-31593: remove unnecessary streaming query progress update") {
+    withSQLConf(SQLConf.STREAMING_NO_DATA_PROGRESS_EVENT_INTERVAL.key -> "100") {
+      @volatile var numProgressEvent = 0
+      val listener = new StreamingQueryListener {
+        override def onQueryStarted(event: QueryStartedEvent): Unit = {}
+        override def onQueryProgress(event: QueryProgressEvent): Unit = {
+          numProgressEvent += 1
+        }
+        override def onQueryTerminated(event: QueryTerminatedEvent): Unit = {}
+      }
+      spark.streams.addListener(listener)
+
+      def checkProgressEvent(count: Int): StreamAction = {
+        AssertOnQuery { _ =>
+          eventually(Timeout(streamingTimeout)) {
+            assert(numProgressEvent == count)
+          }
+          true
+        }
+      }
+
+      try {
+        val input = new MemoryStream[Int](0, sqlContext)
+        val clock = new StreamManualClock()
+        val result = input.toDF().select("value")
+        testStream(result)(
+          StartStream(trigger = Trigger.ProcessingTime(10), triggerClock = clock),
+          AddData(input, 10),
+          checkProgressEvent(1),
+          AdvanceManualClock(10),
+          checkProgressEvent(2),
+          AdvanceManualClock(90),
+          checkProgressEvent(2),
+          AdvanceManualClock(10),
+          checkProgressEvent(3)
+        )
+      } finally {
+        spark.streams.removeListener(listener)
+      }
+    }
+  }
+
+  private def testReplayListenerBusWithBrokenEventJsons(
       fileName: String,
       expectedEventSize: Int): Unit = {
     val input = getClass.getResourceAsStream(s"/structured-streaming/$fileName")

From 9ca5934cb6bf8257d91a33fcf6f2738822fae34a Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Mon, 15 Jun 2020 10:39:55 +0900
Subject: [PATCH 0829/1280] [SPARK-31983][WEBUI][3.0] Fix sorting for duration
 column in structured streaming tab

### What changes were proposed in this pull request?
Sorting result for duration column in tables of structured streaming tab is wrong sometimes.
<img width="1677" alt="Screenshot 2020-06-13 at 1 58 53 PM" src="https://user-images.githubusercontent.com/15366835/84572178-10755700-adb6-11ea-9131-338e8ba7fb24.png">

We are sorting on string, which results in this behaviour.
`sorttable_numeric` and `sorttable_customkey` is used to fix this.

Refer [this](https://github.com/apache/spark/pull/28752#issuecomment-643451586) and [this](https://github.com/apache/spark/pull/28752#issuecomment-643569254)

After changes :
<img width="1677" alt="Screenshot 2020-06-13 at 8 05 32 PM" src="https://user-images.githubusercontent.com/15366835/84572299-a8734080-adb6-11ea-9aa3-b4bc594de4cf.png">

### Why are the changes needed?
Sorting results are wrong for duration column in tables of structured streaming tab.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Screenshots attached.

Closes #28823 from iRakson/testsort.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 .../sql/streaming/ui/StreamingQueryPage.scala | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
index 733676546eab3..43b93a3654f4c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/ui/StreamingQueryPage.scala
@@ -57,12 +57,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
     val name = UIUtils.getQueryName(query)
     val status = UIUtils.getQueryStatus(query)
     val duration = if (queryActive) {
-      SparkUIUtils.formatDurationVerbose(System.currentTimeMillis() - query.startTimestamp)
+      System.currentTimeMillis() - query.startTimestamp
     } else {
       withNoProgress(query, {
         val endTimeMs = query.lastProgress.timestamp
-        SparkUIUtils.formatDurationVerbose(parseProgressTimestamp(endTimeMs) - query.startTimestamp)
-      }, "-")
+        parseProgressTimestamp(endTimeMs) - query.startTimestamp
+      }, 0)
     }
 
     <tr>
@@ -71,7 +71,9 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
       <td> {query.id} </td>
       <td> <a href={statisticsLink}> {query.runId} </a> </td>
       <td> {SparkUIUtils.formatDate(query.startTimestamp)} </td>
-      <td> {duration} </td>
+      <td sorttable_customkey={duration.toString}>
+        {SparkUIUtils.formatDurationVerbose(duration)}
+      </td>
       <td> {withNoProgress(query, {
         (query.recentProgress.map(p => withNumberInvalid(p.inputRowsPerSecond)).sum /
           query.recentProgress.length).formatted("%.2f") }, "NaN")}
@@ -93,8 +95,13 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
         "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
         "Avg Process /sec", "Lastest Batch")
 
+      val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric",
+        "sorttable_numeric", "")
+      // header classes size must be equal to header row size
+      assert(headerRow.size == headerCss.size)
+
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = true),
-        activeQueries, true, Some("activeQueries-table"), Seq(null), false))
+        activeQueries, true, Some("activeQueries-table"), headerCss, false))
     } else {
       None
     }
@@ -104,8 +111,12 @@ private[ui] class StreamingQueryPage(parent: StreamingQueryTab)
         "Name", "Status", "Id", "Run ID", "Start Time", "Duration", "Avg Input /sec",
         "Avg Process /sec", "Lastest Batch", "Error")
 
+      val headerCss = Seq("", "", "", "", "", "sorttable_numeric", "sorttable_numeric",
+        "sorttable_numeric", "", "")
+      assert(headerRow.size == headerCss.size)
+
       Some(SparkUIUtils.listingTable(headerRow, generateDataRow(request, queryActive = false),
-        inactiveQueries, true, Some("completedQueries-table"), Seq(null), false))
+        inactiveQueries, true, Some("completedQueries-table"), headerCss, false))
     } else {
       None
     }

From cd71fbfbb6c8ecc868de756e6b853235417b65ae Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Mon, 15 Jun 2020 06:10:24 +0000
Subject: [PATCH 0830/1280] [SPARK-31926][SQL][TEST-HIVE1.2][TEST-MAVEN] Fix
 concurrency issue for ThriftCLIService to getPortNumber

### What changes were proposed in this pull request?

This PR brings 02f32cfae4082696a6c013fb6beb150040be3228 back which reverted by 4a25200cd74561152e42fb92b02e860f2d5c7d4e because of maven test failure

diffs newly made:
1. add a missing log4j file to test resources
2. Call `SessionState.detachSession()` to clean the thread local one in `afterAll`.
3. Not use dedicated JVMs for sbt test runner too

### Why are the changes needed?

fix the maven test

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

add new tests

Closes #28797 from yaooqinn/SPARK-31926-NEW.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a0187cd6b59a6b6bb2cadc6711bb663d4d35a844)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 project/SparkBuild.scala                      |  1 -
 .../src/test/resources/log4j.properties       | 65 +++++++++++++++++++
 .../thriftserver/SharedThriftServer.scala     | 50 ++++++++++----
 .../ThriftServerQueryTestSuite.scala          |  3 +
 .../ThriftServerWithSparkContextSuite.scala   | 11 +++-
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 +++-
 .../service/cli/thrift/ThriftCLIService.java  |  3 +
 .../cli/thrift/ThriftHttpCLIService.java      | 21 ++++--
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 +++-
 .../service/cli/thrift/ThriftCLIService.java  |  3 +
 .../cli/thrift/ThriftHttpCLIService.java      | 21 ++++--
 11 files changed, 170 insertions(+), 30 deletions(-)
 create mode 100644 sql/hive-thriftserver/src/test/resources/log4j.properties

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 50d1527599b71..6da46bc6e4fd5 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -479,7 +479,6 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
-    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
     "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
diff --git a/sql/hive-thriftserver/src/test/resources/log4j.properties b/sql/hive-thriftserver/src/test/resources/log4j.properties
new file mode 100644
index 0000000000000..21975ba818142
--- /dev/null
+++ b/sql/hive-thriftserver/src/test/resources/log4j.properties
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file hive-thriftserver/target/unit-tests.log
+log4j.rootLogger=DEBUG, CA, FA
+
+#Console Appender
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
+log4j.appender.CA.Threshold = WARN
+
+
+#File Appender
+log4j.appender.FA=org.apache.log4j.FileAppender
+log4j.appender.FA.append=false
+log4j.appender.FA.file=target/unit-tests.log
+log4j.appender.FA.layout=org.apache.log4j.PatternLayout
+log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+# Set the logger level of File Appender to WARN
+log4j.appender.FA.Threshold = DEBUG
+
+# Some packages are noisy for no good reason.
+log4j.additivity.org.apache.hadoop.hive.serde2.lazy.LazyStruct=false
+log4j.logger.org.apache.hadoop.hive.serde2.lazy.LazyStruct=OFF
+
+log4j.additivity.org.apache.hadoop.hive.metastore.RetryingHMSHandler=false
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=OFF
+
+log4j.additivity.hive.log=false
+log4j.logger.hive.log=OFF
+
+log4j.additivity.parquet.hadoop.ParquetRecordReader=false
+log4j.logger.parquet.hadoop.ParquetRecordReader=OFF
+
+log4j.additivity.org.apache.parquet.hadoop.ParquetRecordReader=false
+log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+
+log4j.additivity.org.apache.parquet.hadoop.ParquetOutputCommitter=false
+log4j.logger.org.apache.parquet.hadoop.ParquetOutputCommitter=OFF
+
+log4j.additivity.hive.ql.metadata.Hive=false
+log4j.logger.hive.ql.metadata.Hive=OFF
+
+log4j.additivity.org.apache.hadoop.hive.ql.io.RCFile=false
+log4j.logger.org.apache.hadoop.hive.ql.io.RCFile=ERROR
+
+# Parquet related logging
+log4j.logger.org.apache.parquet.CorruptStatistics=ERROR
+log4j.logger.parquet.CorruptStatistics=ERROR
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index e002bc0117c8b..1c33abff0780c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -24,6 +24,7 @@ import scala.concurrent.duration._
 import scala.util.Try
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
+import org.apache.hadoop.hive.ql.session.SessionState
 import org.apache.hive.service.cli.thrift.ThriftCLIService
 
 import org.apache.spark.sql.test.SharedSparkSession
@@ -33,6 +34,8 @@ trait SharedThriftServer extends SharedSparkSession {
   private var hiveServer2: HiveThriftServer2 = _
   private var serverPort: Int = 0
 
+  def mode: ServerMode.Value
+
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Retries up to 3 times with different port numbers if the server fails to start
@@ -50,14 +53,21 @@ trait SharedThriftServer extends SharedSparkSession {
       hiveServer2.stop()
     } finally {
       super.afterAll()
+      SessionState.detachSession()
     }
   }
 
+  protected def jdbcUri: String = if (mode == ServerMode.http) {
+    s"jdbc:hive2://localhost:$serverPort/default;transportMode=http;httpPath=cliservice"
+  } else {
+    s"jdbc:hive2://localhost:$serverPort/"
+  }
+
   protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
     val user = System.getProperty("user.name")
     require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
-      fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
+      fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
     val statements = connections.map(_.createStatement())
 
     try {
@@ -69,23 +79,35 @@ trait SharedThriftServer extends SharedSparkSession {
   }
 
   private def startThriftServer(attempt: Int): Unit = {
-    logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
+    logInfo(s"Trying to start HiveThriftServer2: mode=$mode, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any free port to use.
+    // Set the HIVE_SERVER2_THRIFT_PORT and HIVE_SERVER2_THRIFT_HTTP_PORT to 0, so it could
+    // randomly pick any free port to use.
     // It's much more robust than set a random port generated by ourselves ahead
     sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
-    hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
-    hiveServer2.getServices.asScala.foreach {
-      case t: ThriftCLIService if t.getPortNumber != 0 =>
-        serverPort = t.getPortNumber
-        logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
-      case _ =>
-    }
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, "0")
+    sqlContext.setConf(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, mode.toString)
+
+    try {
+      hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+      hiveServer2.getServices.asScala.foreach {
+        case t: ThriftCLIService =>
+          serverPort = t.getPortNumber
+          logInfo(s"Started HiveThriftServer2: mode=$mode, port=$serverPort, attempt=$attempt")
+        case _ =>
+      }
 
-    // Wait for thrift server to be ready to serve the query, via executing simple query
-    // till the query succeeds. See SPARK-30345 for more details.
-    eventually(timeout(30.seconds), interval(1.seconds)) {
-      withJdbcStatement { _.execute("SELECT 1") }
+      // Wait for thrift server to be ready to serve the query, via executing simple query
+      // till the query succeeds. See SPARK-30345 for more details.
+      eventually(timeout(30.seconds), interval(1.seconds)) {
+        withJdbcStatement { _.execute("SELECT 1") }
+      }
+    } catch {
+      case e: Exception =>
+        logError("Error start hive server with Context ", e)
+        if (hiveServer2 != null) {
+          hiveServer2.stop()
+        }
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 15cc3109da3f7..553f10a275bce 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -54,6 +54,9 @@ import org.apache.spark.sql.types._
  */
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
+
+  override def mode: ServerMode.Value = ServerMode.binary
+
   override protected def testFile(fileName: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
     // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index 3e1fce78ae71c..d6420dee41adb 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-class ThriftServerWithSparkContextSuite extends SharedThriftServer {
+trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
 
   test("SPARK-29911: Uncache cached tables when session closed") {
     val cacheManager = spark.sharedState.cacheManager
@@ -42,3 +42,12 @@ class ThriftServerWithSparkContextSuite extends SharedThriftServer {
     }
   }
 }
+
+
+class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite {
+  override def mode: ServerMode.Value = ServerMode.binary
+}
+
+class ThriftServerWithSparkContextInHttpSuite extends ThriftServerWithSparkContextSuite {
+  override def mode: ServerMode.Value = ServerMode.http
+}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index e1ee503b81209..00bdf7e19126e 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -45,7 +46,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -100,6 +101,14 @@ public void run() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + serverSocket.getServerSocket().getLocalPort() + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
       server.serve();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index 8fce9d9383438..783e5795aca76 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -175,6 +175,7 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
+      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -633,6 +634,8 @@ public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
     return resp;
   }
 
+  protected abstract void initializeServer();
+
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 1099a00b67eb7..bd64c777c1d76 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.cli.thrift.TCLIService.Iface;
@@ -53,13 +54,8 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -150,6 +146,19 @@ public void run() {
           + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
+  @Override
+  public void run() {
+    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index a7de9c0f3d0d2..ce79e3c8228a6 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -29,6 +29,7 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -46,7 +47,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -101,6 +102,14 @@ public void run() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  @Override
+  public void run() {
+    try {
       server.serve();
     } catch (Throwable t) {
       LOG.error(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index d41c3b493bb47..e46799a1c427d 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -176,6 +176,7 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
+      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -670,6 +671,8 @@ public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
     return resp;
   }
 
+  protected abstract void initializeServer();
+
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 73d5f84476af0..ab9ed5b1f371e 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,6 +28,7 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
+import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.rpc.thrift.TCLIService;
@@ -54,13 +55,8 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
   @Override
-  public void run() {
+  protected void initializeServer() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -151,6 +147,19 @@ public void run() {
           + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
+    } catch (Exception t) {
+      throw new ServiceException("Error initializing " + getName(), t);
+    }
+  }
+
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
+  @Override
+  public void run() {
+    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.error(

From 3209a6a928d399395fda4429fb9d40de6fc7789e Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 15 Jun 2020 07:25:56 +0000
Subject: [PATCH 0831/1280] [SPARK-31992][SQL] Benchmark the EXCEPTION rebase
 mode

### What changes were proposed in this pull request?
- Modify `DateTimeRebaseBenchmark` to benchmark the default date-time rebasing mode - `EXCEPTION` for saving/loading dates/timestamps from/to parquet files. The mode is benchmarked for modern timestamps after 1900-01-01 00:00:00Z and dates after 1582-10-15.
- Regenerate benchmark results in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_252 and OpenJDK 64-Bit Server VM 11.0.7+10 |

### Why are the changes needed?
The `EXCEPTION` rebasing mode is the default mode of the SQL configs `spark.sql.legacy.parquet.datetimeRebaseModeInRead` and `spark.sql.legacy.parquet.datetimeRebaseModeInWrite`. The changes are needed to improve benchmark coverage for default settings.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the benchmark and check results manually.

Closes #28829 from MaxGekk/benchmark-exception-mode.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9d95f1b010293c63eff14ed28773c00c35207dee)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../DateTimeRebaseBenchmark-jdk11-results.txt | 156 ++++++++++--------
 .../DateTimeRebaseBenchmark-results.txt       | 156 ++++++++++--------
 .../benchmark/DateTimeRebaseBenchmark.scala   |  28 ++--
 3 files changed, 183 insertions(+), 157 deletions(-)

diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
index 5d107c1e50fd8..05896a4d69b47 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt
@@ -6,97 +6,109 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  20802          20802           0          4.8         208.0       1.0X
-before 1582, noop                                 10728          10728           0          9.3         107.3       1.9X
-after 1582, rebase off                            32924          32924           0          3.0         329.2       0.6X
-after 1582, rebase on                             32627          32627           0          3.1         326.3       0.6X
-before 1582, rebase off                           21576          21576           0          4.6         215.8       1.0X
-before 1582, rebase on                            23115          23115           0          4.3         231.2       0.9X
+after 1582, noop                                  20023          20023           0          5.0         200.2       1.0X
+before 1582, noop                                 10729          10729           0          9.3         107.3       1.9X
+after 1582, rebase EXCEPTION                      31834          31834           0          3.1         318.3       0.6X
+after 1582, rebase LEGACY                         31997          31997           0          3.1         320.0       0.6X
+after 1582, rebase CORRECTED                      31712          31712           0          3.2         317.1       0.6X
+before 1582, rebase LEGACY                        23663          23663           0          4.2         236.6       0.8X
+before 1582, rebase CORRECTED                     22749          22749           0          4.4         227.5       0.9X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   12880          12984         178          7.8         128.8       1.0X
-after 1582, vec off, rebase on                    13118          13255         174          7.6         131.2       1.0X
-after 1582, vec on, rebase off                     3645           3698          76         27.4          36.4       3.5X
-after 1582, vec on, rebase on                      3709           3727          15         27.0          37.1       3.5X
-before 1582, vec off, rebase off                  13014          13051          36          7.7         130.1       1.0X
-before 1582, vec off, rebase on                   14195          14242          48          7.0         142.0       0.9X
-before 1582, vec on, rebase off                    3680           3773          92         27.2          36.8       3.5X
-before 1582, vec on, rebase on                     4310           4381          87         23.2          43.1       3.0X
+after 1582, vec off, rebase EXCEPTION             12984          13262         257          7.7         129.8       1.0X
+after 1582, vec off, rebase LEGACY                13278          13330          50          7.5         132.8       1.0X
+after 1582, vec off, rebase CORRECTED             13202          13255          50          7.6         132.0       1.0X
+after 1582, vec on, rebase EXCEPTION               3823           3853          40         26.2          38.2       3.4X
+after 1582, vec on, rebase LEGACY                  3846           3876          27         26.0          38.5       3.4X
+after 1582, vec on, rebase CORRECTED               3775           3838          62         26.5          37.7       3.4X
+before 1582, vec off, rebase LEGACY               13671          13692          26          7.3         136.7       0.9X
+before 1582, vec off, rebase CORRECTED            13387          13476         106          7.5         133.9       1.0X
+before 1582, vec on, rebase LEGACY                 4477           4484           7         22.3          44.8       2.9X
+before 1582, vec on, rebase CORRECTED              3729           3773          50         26.8          37.3       3.5X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   3026           3026           0         33.1          30.3       1.0X
-before 1900, noop                                  2995           2995           0         33.4          30.0       1.0X
-after 1900, rebase off                            24294          24294           0          4.1         242.9       0.1X
-after 1900, rebase on                             24480          24480           0          4.1         244.8       0.1X
-before 1900, rebase off                           31120          31120           0          3.2         311.2       0.1X
-before 1900, rebase on                            31201          31201           0          3.2         312.0       0.1X
+after 1900, noop                                   3020           3020           0         33.1          30.2       1.0X
+before 1900, noop                                  3013           3013           0         33.2          30.1       1.0X
+after 1900, rebase EXCEPTION                      28796          28796           0          3.5         288.0       0.1X
+after 1900, rebase LEGACY                         28869          28869           0          3.5         288.7       0.1X
+after 1900, rebase CORRECTED                      28522          28522           0          3.5         285.2       0.1X
+before 1900, rebase LEGACY                        30594          30594           0          3.3         305.9       0.1X
+before 1900, rebase CORRECTED                     30743          30743           0          3.3         307.4       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   18283          18309          39          5.5         182.8       1.0X
-after 1900, vec off, rebase on                    18235          18269          53          5.5         182.4       1.0X
-after 1900, vec on, rebase off                     9563           9589          27         10.5          95.6       1.9X
-after 1900, vec on, rebase on                      9463           9554          81         10.6          94.6       1.9X
-before 1900, vec off, rebase off                  21377          21469         118          4.7         213.8       0.9X
-before 1900, vec off, rebase on                   21265          21422         156          4.7         212.7       0.9X
-before 1900, vec on, rebase off                   12481          12524          46          8.0         124.8       1.5X
-before 1900, vec on, rebase on                    12360          12482         105          8.1         123.6       1.5X
+after 1900, vec off, rebase EXCEPTION             19325          19468         135          5.2         193.3       1.0X
+after 1900, vec off, rebase LEGACY                19568          19602          30          5.1         195.7       1.0X
+after 1900, vec off, rebase CORRECTED             19532          19538           6          5.1         195.3       1.0X
+after 1900, vec on, rebase EXCEPTION               9884           9990          94         10.1          98.8       2.0X
+after 1900, vec on, rebase LEGACY                  9933           9985          49         10.1          99.3       1.9X
+after 1900, vec on, rebase CORRECTED               9967          10043          76         10.0          99.7       1.9X
+before 1900, vec off, rebase LEGACY               24162          24198          37          4.1         241.6       0.8X
+before 1900, vec off, rebase CORRECTED            24034          24056          20          4.2         240.3       0.8X
+before 1900, vec on, rebase LEGACY                12548          12625          72          8.0         125.5       1.5X
+before 1900, vec on, rebase CORRECTED             12580          12660         115          7.9         125.8       1.5X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2984           2984           0         33.5          29.8       1.0X
-before 1900, noop                                  3003           3003           0         33.3          30.0       1.0X
-after 1900, rebase off                            15814          15814           0          6.3         158.1       0.2X
-after 1900, rebase on                             16250          16250           0          6.2         162.5       0.2X
-before 1900, rebase off                           16026          16026           0          6.2         160.3       0.2X
-before 1900, rebase on                            19735          19735           0          5.1         197.3       0.2X
+after 1900, noop                                   3159           3159           0         31.7          31.6       1.0X
+before 1900, noop                                  3038           3038           0         32.9          30.4       1.0X
+after 1900, rebase EXCEPTION                      16885          16885           0          5.9         168.8       0.2X
+after 1900, rebase LEGACY                         17171          17171           0          5.8         171.7       0.2X
+after 1900, rebase CORRECTED                      17353          17353           0          5.8         173.5       0.2X
+before 1900, rebase LEGACY                        20579          20579           0          4.9         205.8       0.2X
+before 1900, rebase CORRECTED                     17544          17544           0          5.7         175.4       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15292          15351          57          6.5         152.9       1.0X
-after 1900, vec off, rebase on                    15753          15886         173          6.3         157.5       1.0X
-after 1900, vec on, rebase off                     4879           4923          52         20.5          48.8       3.1X
-after 1900, vec on, rebase on                      5018           5038          18         19.9          50.2       3.0X
-before 1900, vec off, rebase off                  15257          15311          53          6.6         152.6       1.0X
-before 1900, vec off, rebase on                   18459          18537          90          5.4         184.6       0.8X
-before 1900, vec on, rebase off                    4929           4946          15         20.3          49.3       3.1X
-before 1900, vec on, rebase on                     8254           8339          93         12.1          82.5       1.9X
+after 1900, vec off, rebase EXCEPTION             16304          16345          58          6.1         163.0       1.0X
+after 1900, vec off, rebase LEGACY                16503          16585          75          6.1         165.0       1.0X
+after 1900, vec off, rebase CORRECTED             16413          16463          44          6.1         164.1       1.0X
+after 1900, vec on, rebase EXCEPTION               5017           5034          29         19.9          50.2       3.2X
+after 1900, vec on, rebase LEGACY                  5060           5094          30         19.8          50.6       3.2X
+after 1900, vec on, rebase CORRECTED               4969           4971           1         20.1          49.7       3.3X
+before 1900, vec off, rebase LEGACY               19767          20001         203          5.1         197.7       0.8X
+before 1900, vec off, rebase CORRECTED            16421          16465          38          6.1         164.2       1.0X
+before 1900, vec on, rebase LEGACY                 8535           8608          64         11.7          85.4       1.9X
+before 1900, vec on, rebase CORRECTED              5044           5077          32         19.8          50.4       3.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2987           2987           0         33.5          29.9       1.0X
-before 1900, noop                                  3002           3002           0         33.3          30.0       1.0X
-after 1900, rebase off                            15215          15215           0          6.6         152.1       0.2X
-after 1900, rebase on                             15577          15577           0          6.4         155.8       0.2X
-before 1900, rebase off                           15505          15505           0          6.4         155.1       0.2X
-before 1900, rebase on                            19143          19143           0          5.2         191.4       0.2X
+after 1900, noop                                   2995           2995           0         33.4          29.9       1.0X
+before 1900, noop                                  2981           2981           0         33.5          29.8       1.0X
+after 1900, rebase EXCEPTION                      16196          16196           0          6.2         162.0       0.2X
+after 1900, rebase LEGACY                         16550          16550           0          6.0         165.5       0.2X
+after 1900, rebase CORRECTED                      16908          16908           0          5.9         169.1       0.2X
+before 1900, rebase LEGACY                        20087          20087           0          5.0         200.9       0.1X
+before 1900, rebase CORRECTED                     17171          17171           0          5.8         171.7       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15330          15436         113          6.5         153.3       1.0X
-after 1900, vec off, rebase on                    15515          15549          30          6.4         155.1       1.0X
-after 1900, vec on, rebase off                     6056           6074          19         16.5          60.6       2.5X
-after 1900, vec on, rebase on                      6376           6390          14         15.7          63.8       2.4X
-before 1900, vec off, rebase off                  15490          15523          36          6.5         154.9       1.0X
-before 1900, vec off, rebase on                   18613          18685         118          5.4         186.1       0.8X
-before 1900, vec on, rebase off                    6065           6109          41         16.5          60.6       2.5X
-before 1900, vec on, rebase on                     9052           9082          32         11.0          90.5       1.7X
+after 1900, vec off, rebase EXCEPTION             16688          16787          88          6.0         166.9       1.0X
+after 1900, vec off, rebase LEGACY                17383          17462          73          5.8         173.8       1.0X
+after 1900, vec off, rebase CORRECTED             17317          17329          11          5.8         173.2       1.0X
+after 1900, vec on, rebase EXCEPTION               6342           6348           6         15.8          63.4       2.6X
+after 1900, vec on, rebase LEGACY                  6500           6521          18         15.4          65.0       2.6X
+after 1900, vec on, rebase CORRECTED               6164           6172          11         16.2          61.6       2.7X
+before 1900, vec off, rebase LEGACY               20575          20665          81          4.9         205.7       0.8X
+before 1900, vec off, rebase CORRECTED            17239          17290          61          5.8         172.4       1.0X
+before 1900, vec on, rebase LEGACY                 9310           9373          60         10.7          93.1       1.8X
+before 1900, vec on, rebase CORRECTED              6091           6105          16         16.4          60.9       2.7X
 
 
 ================================================================================================
@@ -107,36 +119,36 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  20653          20653           0          4.8         206.5       1.0X
-before 1582, noop                                 10707          10707           0          9.3         107.1       1.9X
-after 1582                                        28288          28288           0          3.5         282.9       0.7X
-before 1582                                       19196          19196           0          5.2         192.0       1.1X
+after 1582, noop                                  19583          19583           0          5.1         195.8       1.0X
+before 1582, noop                                 10711          10711           0          9.3         107.1       1.8X
+after 1582                                        27864          27864           0          3.6         278.6       0.7X
+before 1582                                       19648          19648           0          5.1         196.5       1.0X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10596          10621          37          9.4         106.0       1.0X
-after 1582, vec on                                 3886           3938          61         25.7          38.9       2.7X
-before 1582, vec off                              10955          10984          26          9.1         109.6       1.0X
-before 1582, vec on                                4236           4258          24         23.6          42.4       2.5X
+after 1582, vec off                               10383          10560         192          9.6         103.8       1.0X
+after 1582, vec on                                 3844           3864          33         26.0          38.4       2.7X
+before 1582, vec off                              10867          10916          48          9.2         108.7       1.0X
+before 1582, vec on                                4158           4170          12         24.0          41.6       2.5X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2988           2988           0         33.5          29.9       1.0X
-before 1900, noop                                  3007           3007           0         33.3          30.1       1.0X
-after 1900                                        18082          18082           0          5.5         180.8       0.2X
-before 1900                                       22669          22669           0          4.4         226.7       0.1X
+after 1900, noop                                   2989           2989           0         33.5          29.9       1.0X
+before 1900, noop                                  3000           3000           0         33.3          30.0       1.0X
+after 1900                                        19426          19426           0          5.1         194.3       0.2X
+before 1900                                       23282          23282           0          4.3         232.8       0.1X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               12029          12035           9          8.3         120.3       1.0X
-after 1900, vec on                                 5194           5197           3         19.3          51.9       2.3X
-before 1900, vec off                              14853          14875          23          6.7         148.5       0.8X
-before 1900, vec on                                7797           7836          60         12.8          78.0       1.5X
+after 1900, vec off                               12089          12102          15          8.3         120.9       1.0X
+after 1900, vec on                                 5210           5325         100         19.2          52.1       2.3X
+before 1900, vec off                              15320          15373          46          6.5         153.2       0.8X
+before 1900, vec on                                7937           7970          48         12.6          79.4       1.5X
 
 
diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
index f2af3cc000f04..3e94d6c6fcfa7 100644
--- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
+++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt
@@ -6,97 +6,109 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to parquet:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23567          23567           0          4.2         235.7       1.0X
-before 1582, noop                                 10570          10570           0          9.5         105.7       2.2X
-after 1582, rebase off                            35335          35335           0          2.8         353.3       0.7X
-after 1582, rebase on                             35645          35645           0          2.8         356.5       0.7X
-before 1582, rebase off                           21824          21824           0          4.6         218.2       1.1X
-before 1582, rebase on                            22532          22532           0          4.4         225.3       1.0X
+after 1582, noop                                  23300          23300           0          4.3         233.0       1.0X
+before 1582, noop                                 10585          10585           0          9.4         105.9       2.2X
+after 1582, rebase EXCEPTION                      35215          35215           0          2.8         352.1       0.7X
+after 1582, rebase LEGACY                         34927          34927           0          2.9         349.3       0.7X
+after 1582, rebase CORRECTED                      35479          35479           0          2.8         354.8       0.7X
+before 1582, rebase LEGACY                        22767          22767           0          4.4         227.7       1.0X
+before 1582, rebase CORRECTED                     22527          22527           0          4.4         225.3       1.0X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from parquet:                   Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off, rebase off                   13194          13266          81          7.6         131.9       1.0X
-after 1582, vec off, rebase on                    13402          13466          89          7.5         134.0       1.0X
-after 1582, vec on, rebase off                     3627           3657          29         27.6          36.3       3.6X
-after 1582, vec on, rebase on                      3818           3839          26         26.2          38.2       3.5X
-before 1582, vec off, rebase off                  13075          13146         115          7.6         130.7       1.0X
-before 1582, vec off, rebase on                   13794          13804          13          7.2         137.9       1.0X
-before 1582, vec on, rebase off                    3655           3675          21         27.4          36.6       3.6X
-before 1582, vec on, rebase on                     4579           4634          72         21.8          45.8       2.9X
+after 1582, vec off, rebase EXCEPTION             13480          13577          94          7.4         134.8       1.0X
+after 1582, vec off, rebase LEGACY                13466          13586         118          7.4         134.7       1.0X
+after 1582, vec off, rebase CORRECTED             13526          13558          41          7.4         135.3       1.0X
+after 1582, vec on, rebase EXCEPTION               3759           3778          28         26.6          37.6       3.6X
+after 1582, vec on, rebase LEGACY                  3957           4004          57         25.3          39.6       3.4X
+after 1582, vec on, rebase CORRECTED               3739           3755          25         26.7          37.4       3.6X
+before 1582, vec off, rebase LEGACY               13986          14038          67          7.1         139.9       1.0X
+before 1582, vec off, rebase CORRECTED            13453          13491          49          7.4         134.5       1.0X
+before 1582, vec on, rebase LEGACY                 4716           4724          10         21.2          47.2       2.9X
+before 1582, vec on, rebase CORRECTED              3701           3750          50         27.0          37.0       3.6X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_INT96 to parquet:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2671           2671           0         37.4          26.7       1.0X
-before 1900, noop                                  2685           2685           0         37.2          26.8       1.0X
-after 1900, rebase off                            23899          23899           0          4.2         239.0       0.1X
-after 1900, rebase on                             24030          24030           0          4.2         240.3       0.1X
-before 1900, rebase off                           30178          30178           0          3.3         301.8       0.1X
-before 1900, rebase on                            30127          30127           0          3.3         301.3       0.1X
+after 1900, noop                                   2790           2790           0         35.8          27.9       1.0X
+before 1900, noop                                  2812           2812           0         35.6          28.1       1.0X
+after 1900, rebase EXCEPTION                      24789          24789           0          4.0         247.9       0.1X
+after 1900, rebase LEGACY                         24539          24539           0          4.1         245.4       0.1X
+after 1900, rebase CORRECTED                      24543          24543           0          4.1         245.4       0.1X
+before 1900, rebase LEGACY                        30496          30496           0          3.3         305.0       0.1X
+before 1900, rebase CORRECTED                     30428          30428           0          3.3         304.3       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_INT96 from parquet:        Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   16613          16685          75          6.0         166.1       1.0X
-after 1900, vec off, rebase on                    16487          16541          47          6.1         164.9       1.0X
-after 1900, vec on, rebase off                     8840           8870          49         11.3          88.4       1.9X
-after 1900, vec on, rebase on                      8795           8813          20         11.4          87.9       1.9X
-before 1900, vec off, rebase off                  20400          20441          62          4.9         204.0       0.8X
-before 1900, vec off, rebase on                   20430          20481          60          4.9         204.3       0.8X
-before 1900, vec on, rebase off                   12211          12290          73          8.2         122.1       1.4X
-before 1900, vec on, rebase on                    12231          12321          95          8.2         122.3       1.4X
+after 1900, vec off, rebase EXCEPTION             17106          17192          75          5.8         171.1       1.0X
+after 1900, vec off, rebase LEGACY                17273          17337          55          5.8         172.7       1.0X
+after 1900, vec off, rebase CORRECTED             17073          17215         128          5.9         170.7       1.0X
+after 1900, vec on, rebase EXCEPTION               8903           8976         117         11.2          89.0       1.9X
+after 1900, vec on, rebase LEGACY                  8793           8876          84         11.4          87.9       1.9X
+after 1900, vec on, rebase CORRECTED               8820           8878          53         11.3          88.2       1.9X
+before 1900, vec off, rebase LEGACY               20997          21069          82          4.8         210.0       0.8X
+before 1900, vec off, rebase CORRECTED            20874          20946          90          4.8         208.7       0.8X
+before 1900, vec on, rebase LEGACY                12024          12090          58          8.3         120.2       1.4X
+before 1900, vec on, rebase CORRECTED             12020          12069          64          8.3         120.2       1.4X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MICROS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2836           2836           0         35.3          28.4       1.0X
-before 1900, noop                                  2812           2812           0         35.6          28.1       1.0X
-after 1900, rebase off                            15976          15976           0          6.3         159.8       0.2X
-after 1900, rebase on                             16197          16197           0          6.2         162.0       0.2X
-before 1900, rebase off                           16140          16140           0          6.2         161.4       0.2X
-before 1900, rebase on                            20410          20410           0          4.9         204.1       0.1X
+after 1900, noop                                   2939           2939           0         34.0          29.4       1.0X
+before 1900, noop                                  2917           2917           0         34.3          29.2       1.0X
+after 1900, rebase EXCEPTION                      15954          15954           0          6.3         159.5       0.2X
+after 1900, rebase LEGACY                         16402          16402           0          6.1         164.0       0.2X
+after 1900, rebase CORRECTED                      16541          16541           0          6.0         165.4       0.2X
+before 1900, rebase LEGACY                        20500          20500           0          4.9         205.0       0.1X
+before 1900, rebase CORRECTED                     16764          16764           0          6.0         167.6       0.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MICROS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15297          15324          40          6.5         153.0       1.0X
-after 1900, vec off, rebase on                    15771          15832          59          6.3         157.7       1.0X
-after 1900, vec on, rebase off                     4922           4949          32         20.3          49.2       3.1X
-after 1900, vec on, rebase on                      5392           5411          17         18.5          53.9       2.8X
-before 1900, vec off, rebase off                  15227          15385         141          6.6         152.3       1.0X
-before 1900, vec off, rebase on                   19611          19658          41          5.1         196.1       0.8X
-before 1900, vec on, rebase off                    4965           5013          54         20.1          49.6       3.1X
-before 1900, vec on, rebase on                     9847           9873          43         10.2          98.5       1.6X
+after 1900, vec off, rebase EXCEPTION             15607          15655          81          6.4         156.1       1.0X
+after 1900, vec off, rebase LEGACY                15616          15676          54          6.4         156.2       1.0X
+after 1900, vec off, rebase CORRECTED             15634          15732         108          6.4         156.3       1.0X
+after 1900, vec on, rebase EXCEPTION               5041           5057          16         19.8          50.4       3.1X
+after 1900, vec on, rebase LEGACY                  5516           5539          29         18.1          55.2       2.8X
+after 1900, vec on, rebase CORRECTED               5087           5104          28         19.7          50.9       3.1X
+before 1900, vec off, rebase LEGACY               19262          19338          79          5.2         192.6       0.8X
+before 1900, vec off, rebase CORRECTED            15718          15755          53          6.4         157.2       1.0X
+before 1900, vec on, rebase LEGACY                10147          10240         114          9.9         101.5       1.5X
+before 1900, vec on, rebase CORRECTED              5062           5080          21         19.8          50.6       3.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP_MILLIS to parquet:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2818           2818           0         35.5          28.2       1.0X
-before 1900, noop                                  2805           2805           0         35.6          28.1       1.0X
-after 1900, rebase off                            15182          15182           0          6.6         151.8       0.2X
-after 1900, rebase on                             15614          15614           0          6.4         156.1       0.2X
-before 1900, rebase off                           15404          15404           0          6.5         154.0       0.2X
-before 1900, rebase on                            19747          19747           0          5.1         197.5       0.1X
+after 1900, noop                                   2915           2915           0         34.3          29.2       1.0X
+before 1900, noop                                  2894           2894           0         34.6          28.9       1.0X
+after 1900, rebase EXCEPTION                      15545          15545           0          6.4         155.4       0.2X
+after 1900, rebase LEGACY                         15840          15840           0          6.3         158.4       0.2X
+after 1900, rebase CORRECTED                      16324          16324           0          6.1         163.2       0.2X
+before 1900, rebase LEGACY                        20359          20359           0          4.9         203.6       0.1X
+before 1900, rebase CORRECTED                     16292          16292           0          6.1         162.9       0.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP_MILLIS from parquet:       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off, rebase off                   15622          15649          24          6.4         156.2       1.0X
-after 1900, vec off, rebase on                    15572          15677         119          6.4         155.7       1.0X
-after 1900, vec on, rebase off                     6345           6358          15         15.8          63.5       2.5X
-after 1900, vec on, rebase on                      6780           6834          92         14.8          67.8       2.3X
-before 1900, vec off, rebase off                  15540          15584          38          6.4         155.4       1.0X
-before 1900, vec off, rebase on                   19590          19653          55          5.1         195.9       0.8X
-before 1900, vec on, rebase off                    6374           6381          10         15.7          63.7       2.5X
-before 1900, vec on, rebase on                    10530          10544          25          9.5         105.3       1.5X
+after 1900, vec off, rebase EXCEPTION             15857          16015         223          6.3         158.6       1.0X
+after 1900, vec off, rebase LEGACY                16174          16231          63          6.2         161.7       1.0X
+after 1900, vec off, rebase CORRECTED             16353          16400          67          6.1         163.5       1.0X
+after 1900, vec on, rebase EXCEPTION               6449           6459           9         15.5          64.5       2.5X
+after 1900, vec on, rebase LEGACY                  7028           7035           6         14.2          70.3       2.3X
+after 1900, vec on, rebase CORRECTED               6585           6623          37         15.2          65.8       2.4X
+before 1900, vec off, rebase LEGACY               19929          20027          95          5.0         199.3       0.8X
+before 1900, vec off, rebase CORRECTED            16401          16451          49          6.1         164.0       1.0X
+before 1900, vec on, rebase LEGACY                10517          10563          40          9.5         105.2       1.5X
+before 1900, vec on, rebase CORRECTED              6659           6675          26         15.0          66.6       2.4X
 
 
 ================================================================================================
@@ -107,36 +119,36 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save DATE to ORC:                         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, noop                                  23825          23825           0          4.2         238.2       1.0X
-before 1582, noop                                 10501          10501           0          9.5         105.0       2.3X
-after 1582                                        32134          32134           0          3.1         321.3       0.7X
-before 1582                                       19947          19947           0          5.0         199.5       1.2X
+after 1582, noop                                  22782          22782           0          4.4         227.8       1.0X
+before 1582, noop                                 10555          10555           0          9.5         105.6       2.2X
+after 1582                                        31497          31497           0          3.2         315.0       0.7X
+before 1582                                       19803          19803           0          5.0         198.0       1.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load DATE from ORC:                       Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1582, vec off                               10034          10056          22         10.0         100.3       1.0X
-after 1582, vec on                                 3664           3698          30         27.3          36.6       2.7X
-before 1582, vec off                              10472          10502          30          9.5         104.7       1.0X
-before 1582, vec on                                4052           4098          42         24.7          40.5       2.5X
+after 1582, vec off                               10180          10214          44          9.8         101.8       1.0X
+after 1582, vec on                                 3785           3804          24         26.4          37.8       2.7X
+before 1582, vec off                              10537          10582          39          9.5         105.4       1.0X
+before 1582, vec on                                4117           4146          25         24.3          41.2       2.5X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Save TIMESTAMP to ORC:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, noop                                   2812           2812           0         35.6          28.1       1.0X
-before 1900, noop                                  2801           2801           0         35.7          28.0       1.0X
-after 1900                                        18290          18290           0          5.5         182.9       0.2X
-before 1900                                       22344          22344           0          4.5         223.4       0.1X
+after 1900, noop                                   2853           2853           0         35.1          28.5       1.0X
+before 1900, noop                                  2999           2999           0         33.3          30.0       1.0X
+after 1900                                        16757          16757           0          6.0         167.6       0.2X
+before 1900                                       21542          21542           0          4.6         215.4       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Load TIMESTAMP from ORC:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-after 1900, vec off                               11257          11279          32          8.9         112.6       1.0X
-after 1900, vec on                                 5296           5310          15         18.9          53.0       2.1X
-before 1900, vec off                              14700          14758          72          6.8         147.0       0.8X
-before 1900, vec on                                8576           8665         150         11.7          85.8       1.3X
+after 1900, vec off                               12212          12254          39          8.2         122.1       1.0X
+after 1900, vec on                                 5369           5390          35         18.6          53.7       2.3X
+before 1900, vec off                              15661          15705          73          6.4         156.6       0.8X
+before 1900, vec on                                8720           8744          29         11.5          87.2       1.4X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
index d6167f98b5a51..7caaa5376db7f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala
@@ -25,8 +25,7 @@ import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA}
 import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
-import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
+import org.apache.spark.sql.internal.SQLConf.{LegacyBehaviorPolicy, ParquetOutputTimestampType}
 
 object DateTime extends Enumeration {
   type DateTime = Value
@@ -118,12 +117,12 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
   private def caseName(
       modernDates: Boolean,
       dateTime: DateTime,
-      rebase: Option[Boolean] = None,
+      mode: Option[LegacyBehaviorPolicy.Value] = None,
       vec: Option[Boolean] = None): String = {
     val period = if (modernDates) "after" else "before"
     val year = if (dateTime == DATE) 1582 else 1900
     val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("")
-    val rebaseFlag = rebase.map(flagToStr).map(flag => s", rebase $flag").getOrElse("")
+    val rebaseFlag = mode.map(_.toString).map(m => s", rebase $m").getOrElse("")
     s"$period $year$vecFlag$rebaseFlag"
   }
 
@@ -131,10 +130,10 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
       basePath: File,
       dateTime: DateTime,
       modernDates: Boolean,
-      rebase: Option[Boolean] = None): String = {
+      mode: Option[LegacyBehaviorPolicy.Value] = None): String = {
     val period = if (modernDates) "after" else "before"
     val year = if (dateTime == DATE) 1582 else 1900
-    val rebaseFlag = rebase.map(flagToStr).map(flag => s"_$flag").getOrElse("")
+    val rebaseFlag = mode.map(_.toString).map(m => s"_$m").getOrElse("")
     basePath.getAbsolutePath + s"/${dateTime}_${period}_$year$rebaseFlag"
   }
 
@@ -160,9 +159,10 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 output = output)
               benchmarkInputs(benchmark, rowsNum, dateTime)
               Seq(true, false).foreach { modernDates =>
-                Seq(false, true).foreach { rebase =>
-                  benchmark.addCase(caseName(modernDates, dateTime, Some(rebase)), 1) { _ =>
-                    val mode = if (rebase) LEGACY else CORRECTED
+                LegacyBehaviorPolicy.values
+                  .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION)
+                  .foreach { mode =>
+                  benchmark.addCase(caseName(modernDates, dateTime, Some(mode)), 1) { _ =>
                     withSQLConf(
                       SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime),
                       SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) {
@@ -170,7 +170,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                         .write
                         .mode("overwrite")
                         .format("parquet")
-                        .save(getPath(path, dateTime, modernDates, Some(rebase)))
+                        .save(getPath(path, dateTime, modernDates, Some(mode)))
                     }
                   }
                 }
@@ -181,13 +181,15 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark {
                 s"Load $dateTime from parquet", rowsNum, output = output)
               Seq(true, false).foreach { modernDates =>
                 Seq(false, true).foreach { vec =>
-                  Seq(false, true).foreach { rebase =>
-                    val name = caseName(modernDates, dateTime, Some(rebase), Some(vec))
+                  LegacyBehaviorPolicy.values
+                    .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION)
+                    .foreach { mode =>
+                    val name = caseName(modernDates, dateTime, Some(mode), Some(vec))
                     benchmark2.addCase(name, 3) { _ =>
                       withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString) {
                         spark.read
                           .format("parquet")
-                          .load(getPath(path, dateTime, modernDates, Some(rebase)))
+                          .load(getPath(path, dateTime, modernDates, Some(mode)))
                           .noop()
                       }
                     }

From 764da2fbc24b63fcb17d231487880c0d58364144 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Mon, 15 Jun 2020 07:48:48 -0700
Subject: [PATCH 0832/1280] [SPARK-31990][SS] Use toSet.toSeq in
 Dataset.dropDuplicates

### What changes were proposed in this pull request?

This PR partially revert SPARK-31292 in order to provide a hot-fix for a bug in `Dataset.dropDuplicates`; we must preserve the input order of `colNames` for `groupCols` because the Streaming's state store depends on the `groupCols` order.

### Why are the changes needed?

Bug fix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests in `DataFrameSuite`.

Closes #28830 from maropu/SPARK-31990.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7f7b4dd5199e7c185aedf51fccc400c7072bed05)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 9d4312b21fd0a..2ddedf3600c5f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -2541,7 +2541,9 @@ class Dataset[T] private[sql](
   def dropDuplicates(colNames: Seq[String]): Dataset[T] = withTypedPlan {
     val resolver = sparkSession.sessionState.analyzer.resolver
     val allColumns = queryExecution.analyzed.output
-    val groupCols = colNames.distinct.flatMap { (colName: String) =>
+    // SPARK-31990: We must keep `toSet.toSeq` here because of the backward compatibility issue
+    // (the Streaming's state store depends on the `groupCols` order).
+    val groupCols = colNames.toSet.toSeq.flatMap { (colName: String) =>
       // It is possibly there are more than one columns with the same name,
       // so we call filter instead of find.
       val cols = allColumns.filter(col => resolver(col.name, colName))

From b70c68ae458d929cbf28a084cecf8252b4a3849f Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sat, 13 Jun 2020 07:12:27 +0900
Subject: [PATCH 0833/1280] [SPARK-31950][SQL][TESTS] Extract SQL keywords from
 the SqlBase.g4 file

### What changes were proposed in this pull request?

This PR intends to extract SQL reserved/non-reserved keywords from the ANTLR grammar file (`SqlBase.g4`) directly.

This approach is based on the cloud-fan suggestion: https://github.com/apache/spark/pull/28779#issuecomment-642033217

### Why are the changes needed?

It is hard to maintain a full set of the keywords in `TableIdentifierParserSuite`, so it would be nice if we could extract them from the `SqlBase.g4` file directly.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #28802 from maropu/SPARK-31950-2.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/catalyst/parser/SqlBase.g4      |   4 +
 .../parser/TableIdentifierParserSuite.scala   | 432 +++++-------------
 2 files changed, 110 insertions(+), 326 deletions(-)

diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 208a503f41980..14a66876eb0d4 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -989,6 +989,7 @@ alterColumnAction
 // You can find the full keywords list by searching "Start of the keywords list" in this file.
 // The non-reserved keywords are listed below. Keywords not in this list are reserved keywords.
 ansiNonReserved
+//--ANSI-NON-RESERVED-START
     : ADD
     | AFTER
     | ALTER
@@ -1165,6 +1166,7 @@ ansiNonReserved
     | VIEW
     | VIEWS
     | WINDOW
+//--ANSI-NON-RESERVED-END
     ;
 
 // When `SQL_standard_keyword_behavior=false`, there are 2 kinds of keywords in Spark SQL.
@@ -1442,6 +1444,7 @@ nonReserved
 //============================
 // Start of the keywords list
 //============================
+//--SPARK-KEYWORD-LIST-START
 ADD: 'ADD';
 AFTER: 'AFTER';
 ALL: 'ALL';
@@ -1694,6 +1697,7 @@ WHERE: 'WHERE';
 WINDOW: 'WINDOW';
 WITH: 'WITH';
 YEAR: 'YEAR';
+//--SPARK-KEYWORD-LIST-END
 //============================
 // End of the keywords list
 //============================
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index bd617bf7e3df6..04969e34fb841 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -16,9 +16,14 @@
  */
 package org.apache.spark.sql.catalyst.parser
 
+import java.util.Locale
+
+import scala.collection.mutable
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.fileToString
 import org.apache.spark.sql.internal.SQLConf
 
 class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
@@ -285,334 +290,109 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "where",
     "with")
 
-  // All the keywords in `docs/sql-keywords.md` are listed below:
-  val allCandidateKeywords = Set(
-    "add",
-    "after",
-    "all",
-    "alter",
-    "analyze",
-    "and",
-    "anti",
-    "any",
-    "archive",
-    "array",
-    "as",
-    "asc",
-    "at",
-    "authorization",
-    "between",
-    "both",
-    "bucket",
-    "buckets",
-    "by",
-    "cache",
-    "cascade",
-    "case",
-    "cast",
-    "change",
-    "check",
-    "clear",
-    "cluster",
-    "clustered",
-    "codegen",
-    "collate",
-    "collection",
-    "column",
-    "columns",
-    "comment",
-    "commit",
-    "compact",
-    "compactions",
-    "compute",
-    "concatenate",
-    "constraint",
-    "cost",
-    "create",
-    "cross",
-    "cube",
-    "current",
-    "current_date",
-    "current_time",
-    "current_timestamp",
-    "current_user",
-    "data",
-    "database",
-    "databases",
-    "day",
-    "dbproperties",
-    "defined",
-    "delete",
-    "delimited",
-    "desc",
-    "describe",
-    "dfs",
-    "directories",
-    "directory",
-    "distinct",
-    "distribute",
-    "div",
-    "drop",
-    "else",
-    "end",
-    "escape",
-    "escaped",
-    "except",
-    "exchange",
-    "exists",
-    "explain",
-    "export",
-    "extended",
-    "external",
-    "extract",
-    "false",
-    "fetch",
-    "fields",
-    "fileformat",
-    "first",
-    "following",
-    "for",
-    "foreign",
-    "format",
-    "formatted",
-    "from",
-    "full",
-    "function",
-    "functions",
-    "global",
-    "grant",
-    "group",
-    "grouping",
-    "having",
-    "hour",
-    "if",
-    "ignore",
-    "import",
-    "in",
-    "index",
-    "indexes",
-    "inner",
-    "inpath",
-    "inputformat",
-    "insert",
-    "intersect",
-    "interval",
-    "into",
-    "is",
-    "items",
-    "join",
-    "keys",
-    "last",
-    "lateral",
-    "lazy",
-    "leading",
-    "left",
-    "like",
-    "limit",
-    "lines",
-    "list",
-    "load",
-    "local",
-    "location",
-    "lock",
-    "locks",
-    "logical",
-    "macro",
-    "map",
-    "minus",
-    "minute",
-    "month",
-    "msck",
-    "namespaces",
-    "natural",
-    "no",
-    "not",
-    "null",
-    "nulls",
-    "of",
-    "on",
-    "only",
-    "option",
-    "options",
-    "or",
-    "order",
-    "out",
-    "outer",
-    "outputformat",
-    "over",
-    "overlaps",
-    "overlay",
-    "overwrite",
-    "partition",
-    "partitioned",
-    "partitions",
-    "percent",
-    "pivot",
-    "placing",
-    "position",
-    "preceding",
-    "primary",
-    "principals",
-    "purge",
-    "query",
-    "range",
-    "recordreader",
-    "recordwriter",
-    "recover",
-    "reduce",
-    "references",
-    "refresh",
-    "rename",
-    "repair",
-    "replace",
-    "reset",
-    "restrict",
-    "revoke",
-    "right",
-    "rlike",
-    "role",
-    "roles",
-    "rollback",
-    "rollup",
-    "row",
-    "rows",
-    "schema",
-    "second",
-    "select",
-    "semi",
-    "separated",
-    "serde",
-    "serdeproperties",
-    "session_user",
-    "set",
-    "sets",
-    "show",
-    "skewed",
-    "some",
-    "sort",
-    "sorted",
-    "start",
-    "statistics",
-    "stored",
-    "stratify",
-    "struct",
-    "substr",
-    "substring",
-    "table",
-    "tables",
-    "tablesample",
-    "tblproperties",
-    "temporary",
-    "terminated",
-    "then",
-    "to",
-    "touch",
-    "trailing",
-    "transaction",
-    "transactions",
-    "transform",
-    "true",
-    "truncate",
-    "type",
-    "unarchive",
-    "unbounded",
-    "uncache",
-    "union",
-    "unique",
-    "unknown",
-    "unlock",
-    "unset",
-    "use",
-    "user",
-    "using",
-    "values",
-    "view",
-    "views",
-    "when",
-    "where",
-    "window",
-    "with",
-    "year")
+  private val sqlSyntaxDefs = {
+    val sqlBasePath = {
+      val sparkHome = {
+        assert(sys.props.contains("spark.test.home") ||
+          sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
+        sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+      }
+      java.nio.file.Paths.get(sparkHome, "sql", "catalyst", "src", "main", "antlr4", "org",
+        "apache", "spark", "sql", "catalyst", "parser", "SqlBase.g4").toFile
+    }
+    fileToString(sqlBasePath).split("\n")
+  }
 
-  val reservedKeywordsInAnsiMode = Set(
-    "all",
-    "and",
-    "anti",
-    "any",
-    "as",
-    "authorization",
-    "both",
-    "case",
-    "cast",
-    "check",
-    "collate",
-    "column",
-    "constraint",
-    "create",
-    "cross",
-    "current_date",
-    "current_time",
-    "current_timestamp",
-    "current_user",
-    "day",
-    "distinct",
-    "else",
-    "end",
-    "escape",
-    "except",
-    "false",
-    "fetch",
-    "for",
-    "foreign",
-    "from",
-    "full",
-    "grant",
-    "group",
-    "having",
-    "hour",
-    "in",
-    "inner",
-    "intersect",
-    "into",
-    "join",
-    "is",
-    "leading",
-    "left",
-    "minute",
-    "month",
-    "natural",
-    "not",
-    "null",
-    "on",
-    "only",
-    "or",
-    "order",
-    "outer",
-    "overlaps",
-    "primary",
-    "references",
-    "right",
-    "select",
-    "semi",
-    "session_user",
-    "minus",
-    "second",
-    "some",
-    "table",
-    "then",
-    "to",
-    "trailing",
-    "union",
-    "unique",
-    "unknown",
-    "user",
-    "using",
-    "when",
-    "where",
-    "with",
-    "year")
+  private def parseAntlrGrammars[T](startTag: String, endTag: String)
+      (f: PartialFunction[String, Seq[T]]): Set[T] = {
+    val keywords = new mutable.ArrayBuffer[T]
+    val default = (_: String) => Nil
+    var startTagFound = false
+    var parseFinished = false
+    val lineIter = sqlSyntaxDefs.toIterator
+    while (!parseFinished && lineIter.hasNext) {
+      val line = lineIter.next()
+      if (line.trim.startsWith(startTag)) {
+        startTagFound = true
+      } else if (line.trim.startsWith(endTag)) {
+        parseFinished = true
+      } else if (startTagFound) {
+        f.applyOrElse(line, default).foreach { symbol =>
+          keywords += symbol
+        }
+      }
+    }
+    assert(keywords.nonEmpty && startTagFound && parseFinished, "cannot extract keywords from " +
+      s"the `SqlBase.g4` file, so please check if the start/end tags (`$startTag` and `$endTag`) " +
+      "are placed correctly in the file.")
+    keywords.toSet
+  }
 
-  val nonReservedKeywordsInAnsiMode = allCandidateKeywords -- reservedKeywordsInAnsiMode
+  // If a symbol does not have the same string with its literal (e.g., `SETMINUS: 'MINUS';`),
+  // we need to map a symbol to actual literal strings.
+  val symbolsToExpandIntoDifferentLiterals = {
+    val kwDef = """([A-Z_]+):(.+);""".r
+    val keywords = parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      case kwDef(symbol, literalDef) =>
+        val splitDefs = literalDef.split("""\|""")
+        val hasMultipleLiterals = splitDefs.length > 1
+        // The case where a symbol has multiple literal definitions,
+        // e.g., `DATABASES: 'DATABASES' | 'SCHEMAS';`.
+        if (hasMultipleLiterals) {
+          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq
+          (symbol, literals) :: Nil
+        } else {
+          val literal = literalDef.replaceAll("'", "").trim
+          // The case where a symbol string and its literal string are different,
+          // e.g., `SETMINUS: 'MINUS';`.
+          if (symbol != literal) {
+            (symbol, literal :: Nil) :: Nil
+          } else {
+            Nil
+          }
+        }
+    }
+    keywords.toMap
+  }
+
+  // All the SQL keywords defined in `SqlBase.g4`
+  val allCandidateKeywords = {
+    val kwDef = """([A-Z_]+):.+;""".r
+    val keywords = parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      // Parses a pattern, e.g., `AFTER: 'AFTER';`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+    keywords
+  }
+
+  val nonReservedKeywordsInAnsiMode = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--ANSI-NON-RESERVED-START", "//--ANSI-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
+
+  test("check # of reserved keywords") {
+    val numReservedKeywords = 78
+    assert(reservedKeywordsInAnsiMode.size == numReservedKeywords,
+      s"The expected number of reserved keywords is $numReservedKeywords, but " +
+        s"${reservedKeywordsInAnsiMode.size} found.")
+  }
 
   test("table identifier") {
     // Regular names.

From ed69190ce0762f3b741b8d175ef8d02da45f3183 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 16 Jun 2020 00:27:45 +0900
Subject: [PATCH 0834/1280] [SPARK-26905][SQL] Follow the SQL:2016 reserved
 keywords

### What changes were proposed in this pull request?

This PR intends to move keywords `ANTI`, `SEMI`, and `MINUS` from reserved to non-reserved.

### Why are the changes needed?

To comply with the ANSI/SQL standard.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #28807 from maropu/SPARK-26905-2.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-ansi-compliance.md               |   6 +-
 .../spark/sql/catalyst/parser/SqlBase.g4      |   3 +
 .../ansi-sql-2016-reserved-keywords.txt       | 401 ++++++++++++++++++
 .../parser/TableIdentifierParserSuite.scala   |  24 +-
 4 files changed, 429 insertions(+), 5 deletions(-)
 create mode 100644 sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index eab194c71ec79..e5ca7e9d10d59 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -135,7 +135,7 @@ Below is a list of all the keywords in Spark SQL.
 |ALTER|non-reserved|non-reserved|reserved|
 |ANALYZE|non-reserved|non-reserved|non-reserved|
 |AND|reserved|non-reserved|reserved|
-|ANTI|reserved|strict-non-reserved|non-reserved|
+|ANTI|non-reserved|strict-non-reserved|non-reserved|
 |ANY|reserved|non-reserved|reserved|
 |ARCHIVE|non-reserved|non-reserved|non-reserved|
 |ARRAY|non-reserved|non-reserved|reserved|
@@ -264,7 +264,7 @@ Below is a list of all the keywords in Spark SQL.
 |MAP|non-reserved|non-reserved|non-reserved|
 |MATCHED|non-reserved|non-reserved|non-reserved|
 |MERGE|non-reserved|non-reserved|non-reserved|
-|MINUS|reserved|strict-non-reserved|non-reserved|
+|MINUS|not-reserved|strict-non-reserved|non-reserved|
 |MINUTE|reserved|non-reserved|reserved|
 |MONTH|reserved|non-reserved|reserved|
 |MSCK|non-reserved|non-reserved|non-reserved|
@@ -325,7 +325,7 @@ Below is a list of all the keywords in Spark SQL.
 |SCHEMA|non-reserved|non-reserved|non-reserved|
 |SECOND|reserved|non-reserved|reserved|
 |SELECT|reserved|non-reserved|reserved|
-|SEMI|reserved|strict-non-reserved|non-reserved|
+|SEMI|non-reserved|strict-non-reserved|non-reserved|
 |SEPARATED|non-reserved|non-reserved|non-reserved|
 |SERDE|non-reserved|non-reserved|non-reserved|
 |SERDEPROPERTIES|non-reserved|non-reserved|non-reserved|
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 14a66876eb0d4..5821a743d4168 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -994,6 +994,7 @@ ansiNonReserved
     | AFTER
     | ALTER
     | ANALYZE
+    | ANTI
     | ARCHIVE
     | ARRAY
     | ASC
@@ -1126,10 +1127,12 @@ ansiNonReserved
     | ROW
     | ROWS
     | SCHEMA
+    | SEMI
     | SEPARATED
     | SERDE
     | SERDEPROPERTIES
     | SET
+    | SETMINUS
     | SETS
     | SHOW
     | SKEWED
diff --git a/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt b/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt
new file mode 100644
index 0000000000000..921491a4a4761
--- /dev/null
+++ b/sql/catalyst/src/test/resources/ansi-sql-2016-reserved-keywords.txt
@@ -0,0 +1,401 @@
+-- This file comes from: https://github.com/postgres/postgres/tree/master/doc/src/sgml/keywords
+ABS
+ACOS
+ALL
+ALLOCATE
+ALTER
+AND
+ANY
+ARE
+ARRAY
+ARRAY_AGG
+ARRAY_MAX_CARDINALITY
+AS
+ASENSITIVE
+ASIN
+ASYMMETRIC
+AT
+ATAN
+ATOMIC
+AUTHORIZATION
+AVG
+BEGIN
+BEGIN_FRAME
+BEGIN_PARTITION
+BETWEEN
+BIGINT
+BINARY
+BLOB
+BOOLEAN
+BOTH
+BY
+CALL
+CALLED
+CARDINALITY
+CASCADED
+CASE
+CAST
+CEIL
+CEILING
+CHAR
+CHAR_LENGTH
+CHARACTER
+CHARACTER_LENGTH
+CHECK
+CLASSIFIER
+CLOB
+CLOSE
+COALESCE
+COLLATE
+COLLECT
+COLUMN
+COMMIT
+CONDITION
+CONNECT
+CONSTRAINT
+CONTAINS
+CONVERT
+COPY
+CORR
+CORRESPONDING
+COS
+COSH
+COUNT
+COVAR_POP
+COVAR_SAMP
+CREATE
+CROSS
+CUBE
+CUME_DIST
+CURRENT
+CURRENT_CATALOG
+CURRENT_DATE
+CURRENT_DEFAULT_TRANSFORM_GROUP
+CURRENT_PATH
+CURRENT_ROLE
+CURRENT_ROW
+CURRENT_SCHEMA
+CURRENT_TIME
+CURRENT_TIMESTAMP
+CURRENT_TRANSFORM_GROUP_FOR_TYPE
+CURRENT_USER
+CURSOR
+CYCLE
+DATE
+DAY
+DEALLOCATE
+DEC
+DECIMAL
+DECFLOAT
+DECLARE
+DEFAULT
+DEFINE
+DELETE
+DENSE_RANK
+DEREF
+DESCRIBE
+DETERMINISTIC
+DISCONNECT
+DISTINCT
+DOUBLE
+DROP
+DYNAMIC
+EACH
+ELEMENT
+ELSE
+EMPTY
+END
+END_FRAME
+END_PARTITION
+END-EXEC
+EQUALS
+ESCAPE
+EVERY
+EXCEPT
+EXEC
+EXECUTE
+EXISTS
+EXP
+EXTERNAL
+EXTRACT
+FALSE
+FETCH
+FILTER
+FIRST_VALUE
+FLOAT
+FLOOR
+FOR
+FOREIGN
+FRAME_ROW
+FREE
+FROM
+FULL
+FUNCTION
+FUSION
+GET
+GLOBAL
+GRANT
+GROUP
+GROUPING
+GROUPS
+HAVING
+HOLD
+HOUR
+IDENTITY
+IN
+INDICATOR
+INITIAL
+INNER
+INOUT
+INSENSITIVE
+INSERT
+INT
+INTEGER
+INTERSECT
+INTERSECTION
+INTERVAL
+INTO
+IS
+JOIN
+JSON_ARRAY
+JSON_ARRAYAGG
+JSON_EXISTS
+JSON_OBJECT
+JSON_OBJECTAGG
+JSON_QUERY
+JSON_TABLE
+JSON_TABLE_PRIMITIVE
+JSON_VALUE
+LAG
+LANGUAGE
+LARGE
+LAST_VALUE
+LATERAL
+LEAD
+LEADING
+LEFT
+LIKE
+LIKE_REGEX
+LISTAGG
+LN
+LOCAL
+LOCALTIME
+LOCALTIMESTAMP
+LOG
+LOG10
+LOWER
+MATCH
+MATCH_NUMBER
+MATCH_RECOGNIZE
+MATCHES
+MAX
+MEASURES
+MEMBER
+MERGE
+METHOD
+MIN
+MINUTE
+MOD
+MODIFIES
+MODULE
+MONTH
+MULTISET
+NATIONAL
+NATURAL
+NCHAR
+NCLOB
+NEW
+NO
+NONE
+NORMALIZE
+NOT
+NTH_VALUE
+NTILE
+NULL
+NULLIF
+NUMERIC
+OCTET_LENGTH
+OCCURRENCES_REGEX
+OF
+OFFSET
+OLD
+OMIT
+ON
+ONE
+ONLY
+OPEN
+OR
+ORDER
+OUT
+OUTER
+OVER
+OVERLAPS
+OVERLAY
+PARAMETER
+PARTITION
+PATTERN
+PER
+PERCENT
+PERCENT_RANK
+PERCENTILE_CONT
+PERCENTILE_DISC
+PERIOD
+PERMUTE
+PORTION
+POSITION
+POSITION_REGEX
+POWER
+PRECEDES
+PRECISION
+PREPARE
+PRIMARY
+PROCEDURE
+PTF
+RANGE
+RANK
+READS
+REAL
+RECURSIVE
+REF
+REFERENCES
+REFERENCING
+REGR_AVGX
+REGR_AVGY
+REGR_COUNT
+REGR_INTERCEPT
+REGR_R2
+REGR_SLOPE
+REGR_SXX
+REGR_SXY
+REGR_SYY
+RELEASE
+RESULT
+RETURN
+RETURNS
+REVOKE
+RIGHT
+ROLLBACK
+ROLLUP
+ROW
+ROW_NUMBER
+ROWS
+RUNNING
+SAVEPOINT
+SCOPE
+SCROLL
+SEARCH
+SECOND
+SEEK
+SELECT
+SENSITIVE
+SESSION_USER
+SET
+SHOW
+SIMILAR
+SIN
+SINH
+SKIP
+SMALLINT
+SOME
+SPECIFIC
+SPECIFICTYPE
+SQL
+SQLEXCEPTION
+SQLSTATE
+SQLWARNING
+SQRT
+START
+STATIC
+STDDEV_POP
+STDDEV_SAMP
+SUBMULTISET
+SUBSET
+SUBSTRING
+SUBSTRING_REGEX
+SUCCEEDS
+SUM
+SYMMETRIC
+SYSTEM
+SYSTEM_TIME
+SYSTEM_USER
+TABLE
+TABLESAMPLE
+TAN
+TANH
+THEN
+TIME
+TIMESTAMP
+TIMEZONE_HOUR
+TIMEZONE_MINUTE
+TO
+TRAILING
+TRANSLATE
+TRANSLATE_REGEX
+TRANSLATION
+TREAT
+TRIGGER
+TRIM
+TRIM_ARRAY
+TRUE
+TRUNCATE
+UESCAPE
+UNION
+UNIQUE
+UNKNOWN
+UNMATCHED
+UNNEST
+UPDATE
+UPPER
+USER
+USING
+VALUE
+VALUES
+VALUE_OF
+VAR_POP
+VAR_SAMP
+VARBINARY
+VARCHAR
+VARYING
+VERSIONING
+WHEN
+WHENEVER
+WHERE
+WIDTH_BUCKET
+WINDOW
+WITH
+WITHIN
+WITHOUT
+YEAR
+DATALINK
+DLNEWCOPY
+DLPREVIOUSCOPY
+DLURLCOMPLETE
+DLURLCOMPLETEWRITE
+DLURLCOMPLETEONLY
+DLURLPATH
+DLURLPATHWRITE
+DLURLPATHONLY
+DLURLSCHEME
+DLURLSERVER
+DLVALUE
+IMPORT
+XML
+XMLAGG
+XMLATTRIBUTES
+XMLBINARY
+XMLCAST
+XMLCOMMENT
+XMLCONCAT
+XMLDOCUMENT
+XMLELEMENT
+XMLEXISTS
+XMLFOREST
+XMLITERATE
+XMLNAMESPACES
+XMLPARSE
+XMLPI
+XMLQUERY
+XMLSERIALIZE
+XMLTABLE
+XMLTEXT
+XMLVALIDATE
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 04969e34fb841..04c427da3ec87 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -16,8 +16,11 @@
  */
 package org.apache.spark.sql.catalyst.parser
 
+import java.io.File
+import java.nio.file.Files
 import java.util.Locale
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.spark.SparkFunSuite
@@ -340,7 +343,12 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
         // The case where a symbol has multiple literal definitions,
         // e.g., `DATABASES: 'DATABASES' | 'SCHEMAS';`.
         if (hasMultipleLiterals) {
-          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq
+          // Filters out inappropriate entries, e.g., `!` in `NOT: 'NOT' | '!';`
+          val litDef = """([A-Z_]+)""".r
+          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq.flatMap {
+            case litDef(lit) => Some(lit)
+            case _ => None
+          }
           (symbol, literals) :: Nil
         } else {
           val literal = literalDef.replaceAll("'", "").trim
@@ -388,12 +396,24 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
   val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
 
   test("check # of reserved keywords") {
-    val numReservedKeywords = 78
+    val numReservedKeywords = 74
     assert(reservedKeywordsInAnsiMode.size == numReservedKeywords,
       s"The expected number of reserved keywords is $numReservedKeywords, but " +
         s"${reservedKeywordsInAnsiMode.size} found.")
   }
 
+  test("reserved keywords in Spark are also reserved in SQL 2016") {
+    withTempDir { dir =>
+      val tmpFile = new File(dir, "tmp")
+      val is = Thread.currentThread().getContextClassLoader
+        .getResourceAsStream("ansi-sql-2016-reserved-keywords.txt")
+      Files.copy(is, tmpFile.toPath)
+      val reservedKeywordsInSql2016 = Files.readAllLines(tmpFile.toPath)
+        .asScala.filterNot(_.startsWith("--")).map(_.trim).toSet
+      assert((reservedKeywordsInAnsiMode  -- reservedKeywordsInSql2016).isEmpty)
+    }
+  }
+
   test("table identifier") {
     // Regular names.
     assert(TableIdentifier("q") === parseTableIdentifier("q"))

From 50be21f8a432feeb48313abd638de0d5e3d4b7e2 Mon Sep 17 00:00:00 2001
From: Prashant Sharma <prashsh1@in.ibm.com>
Date: Mon, 15 Jun 2020 11:26:03 -0700
Subject: [PATCH 0835/1280] [SPARK-31994][K8S] Docker image should use `https`
 urls for only deb.debian.org mirrors

### What changes were proposed in this pull request?
At the moment, we switch to `https` urls for all the debian mirrors, but turns out some of the mirrors do not support. In this patch, we turn on https mode only for `deb.debian.org` mirror (as it supports SSL).

### Why are the changes needed?
It appears, that security.debian.org does not support https.
```
curl https://security.debian.org
curl: (35) LibreSSL SSL_connect: SSL_ERROR_SYSCALL in connection to security.debian.org:443
```

While building the image, it fails in the following way.
```
MacBook-Pro:spark prashantsharma$ bin/docker-image-tool.sh -r scrapcodes -t v3.1.0-1 build
Sending build context to Docker daemon  222.1MB
Step 1/18 : ARG java_image_tag=8-jre-slim
Step 2/18 : FROM openjdk:${java_image_tag}
 ---> 381b20190cf7
Step 3/18 : ARG spark_uid=185
 ---> Using cache
 ---> 65c06f86753c
Step 4/18 : RUN set -ex &&     sed -i 's/http:/https:/g' /etc/apt/sources.list &&     apt-get update &&     ln -s /lib /lib64 &&     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd &&     rm -rf /var/cache/apt/*
 ---> Running in a3461dadd6eb
+ sed -i s/http:/https:/g /etc/apt/sources.list
+ apt-get update
Ign:1 https://security.debian.org/debian-security buster/updates InRelease
Err:2 https://security.debian.org/debian-security buster/updates Release
  Could not handshake: The TLS connection was non-properly terminated. [IP: 151.101.0.204 443]
Get:3 https://deb.debian.org/debian buster InRelease [121 kB]
Get:4 https://deb.debian.org/debian buster-updates InRelease [51.9 kB]
Get:5 https://deb.debian.org/debian buster/main amd64 Packages [7905 kB]
Get:6 https://deb.debian.org/debian buster-updates/main amd64 Packages [7868 B]
Reading package lists...
E: The repository 'https://security.debian.org/debian-security buster/updates Release' does not have a Release file.
The command '/bin/sh -c set -ex &&     sed -i 's/http:/https:/g' /etc/apt/sources.list &&     apt-get update &&     ln -s /lib /lib64 &&     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps &&     mkdir -p /opt/spark &&     mkdir -p /opt/spark/examples &&     mkdir -p /opt/spark/work-dir &&     touch /opt/spark/RELEASE &&     rm /bin/sh &&     ln -sv /bin/bash /bin/sh &&     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su &&     chgrp root /etc/passwd && chmod ug+rw /etc/passwd &&     rm -rf /var/cache/apt/*' returned a non-zero code: 100
Failed to build Spark JVM Docker image, please refer to Docker build output for details.
```

So, if we limit the `https` support to only deb.debian.org, does the trick.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manually, by building an image and testing it by running spark shell against it locally using kubernetes.

Closes #28834 from ScrapCodes/spark-31994/debian_mirror_fix.

Authored-by: Prashant Sharma <prashsh1@in.ibm.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a7d0d353cd745391cf285039a4630b37754a56a0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../kubernetes/docker/src/main/dockerfiles/spark/Dockerfile     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index c1df215b43004..cde9358cb8718 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -28,7 +28,7 @@ ARG spark_uid=185
 # docker build -t spark:latest -f kubernetes/dockerfiles/spark/Dockerfile .
 
 RUN set -ex && \
-    sed -i 's/http:/https:/g' /etc/apt/sources.list && \
+    sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \
     apt-get update && \
     ln -s /lib /lib64 && \
     apt install -y bash tini libc6 libpam-modules krb5-user libnss3 && \

From f5133152578297def5a1066887103338f4f48c11 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 15 Jun 2020 19:09:24 -0700
Subject: [PATCH 0836/1280] Revert
 "[SPARK-31926][SQL][TEST-HIVE1.2][TEST-MAVEN] Fix concurrency issue for
 ThriftCLIService to getPortNumber"

This reverts commit cd71fbfbb6c8ecc868de756e6b853235417b65ae.
---
 project/SparkBuild.scala                      |  1 +
 .../src/test/resources/log4j.properties       | 65 -------------------
 .../thriftserver/SharedThriftServer.scala     | 50 ++++----------
 .../ThriftServerQueryTestSuite.scala          |  3 -
 .../ThriftServerWithSparkContextSuite.scala   | 11 +---
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 +---
 .../service/cli/thrift/ThriftCLIService.java  |  3 -
 .../cli/thrift/ThriftHttpCLIService.java      | 21 ++----
 .../cli/thrift/ThriftBinaryCLIService.java    | 11 +---
 .../service/cli/thrift/ThriftCLIService.java  |  3 -
 .../cli/thrift/ThriftHttpCLIService.java      | 21 ++----
 11 files changed, 30 insertions(+), 170 deletions(-)
 delete mode 100644 sql/hive-thriftserver/src/test/resources/log4j.properties

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6da46bc6e4fd5..50d1527599b71 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -479,6 +479,7 @@ object SparkParallelTestGrouping {
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2ListenerSuite",
+    "org.apache.spark.sql.hive.thriftserver.ThriftServerWithSparkContextSuite",
     "org.apache.spark.sql.kafka010.KafkaDelegationTokenSuite"
   )
 
diff --git a/sql/hive-thriftserver/src/test/resources/log4j.properties b/sql/hive-thriftserver/src/test/resources/log4j.properties
deleted file mode 100644
index 21975ba818142..0000000000000
--- a/sql/hive-thriftserver/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,65 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file hive-thriftserver/target/unit-tests.log
-log4j.rootLogger=DEBUG, CA, FA
-
-#Console Appender
-log4j.appender.CA=org.apache.log4j.ConsoleAppender
-log4j.appender.CA.layout=org.apache.log4j.PatternLayout
-log4j.appender.CA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %p %c: %m%n
-log4j.appender.CA.Threshold = WARN
-
-
-#File Appender
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.append=false
-log4j.appender.FA.file=target/unit-tests.log
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
-
-# Set the logger level of File Appender to WARN
-log4j.appender.FA.Threshold = DEBUG
-
-# Some packages are noisy for no good reason.
-log4j.additivity.org.apache.hadoop.hive.serde2.lazy.LazyStruct=false
-log4j.logger.org.apache.hadoop.hive.serde2.lazy.LazyStruct=OFF
-
-log4j.additivity.org.apache.hadoop.hive.metastore.RetryingHMSHandler=false
-log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=OFF
-
-log4j.additivity.hive.log=false
-log4j.logger.hive.log=OFF
-
-log4j.additivity.parquet.hadoop.ParquetRecordReader=false
-log4j.logger.parquet.hadoop.ParquetRecordReader=OFF
-
-log4j.additivity.org.apache.parquet.hadoop.ParquetRecordReader=false
-log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
-
-log4j.additivity.org.apache.parquet.hadoop.ParquetOutputCommitter=false
-log4j.logger.org.apache.parquet.hadoop.ParquetOutputCommitter=OFF
-
-log4j.additivity.hive.ql.metadata.Hive=false
-log4j.logger.hive.ql.metadata.Hive=OFF
-
-log4j.additivity.org.apache.hadoop.hive.ql.io.RCFile=false
-log4j.logger.org.apache.hadoop.hive.ql.io.RCFile=ERROR
-
-# Parquet related logging
-log4j.logger.org.apache.parquet.CorruptStatistics=ERROR
-log4j.logger.parquet.CorruptStatistics=ERROR
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
index 1c33abff0780c..e002bc0117c8b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SharedThriftServer.scala
@@ -24,7 +24,6 @@ import scala.concurrent.duration._
 import scala.util.Try
 
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars
-import org.apache.hadoop.hive.ql.session.SessionState
 import org.apache.hive.service.cli.thrift.ThriftCLIService
 
 import org.apache.spark.sql.test.SharedSparkSession
@@ -34,8 +33,6 @@ trait SharedThriftServer extends SharedSparkSession {
   private var hiveServer2: HiveThriftServer2 = _
   private var serverPort: Int = 0
 
-  def mode: ServerMode.Value
-
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Retries up to 3 times with different port numbers if the server fails to start
@@ -53,21 +50,14 @@ trait SharedThriftServer extends SharedSparkSession {
       hiveServer2.stop()
     } finally {
       super.afterAll()
-      SessionState.detachSession()
     }
   }
 
-  protected def jdbcUri: String = if (mode == ServerMode.http) {
-    s"jdbc:hive2://localhost:$serverPort/default;transportMode=http;httpPath=cliservice"
-  } else {
-    s"jdbc:hive2://localhost:$serverPort/"
-  }
-
   protected def withJdbcStatement(fs: (Statement => Unit)*): Unit = {
     val user = System.getProperty("user.name")
     require(serverPort != 0, "Failed to bind an actual port for HiveThriftServer2")
     val connections =
-      fs.map { _ => DriverManager.getConnection(jdbcUri, user, "") }
+      fs.map { _ => DriverManager.getConnection(s"jdbc:hive2://localhost:$serverPort", user, "") }
     val statements = connections.map(_.createStatement())
 
     try {
@@ -79,35 +69,23 @@ trait SharedThriftServer extends SharedSparkSession {
   }
 
   private def startThriftServer(attempt: Int): Unit = {
-    logInfo(s"Trying to start HiveThriftServer2: mode=$mode, attempt=$attempt")
+    logInfo(s"Trying to start HiveThriftServer2:, attempt=$attempt")
     val sqlContext = spark.newSession().sqlContext
-    // Set the HIVE_SERVER2_THRIFT_PORT and HIVE_SERVER2_THRIFT_HTTP_PORT to 0, so it could
-    // randomly pick any free port to use.
+    // Set the HIVE_SERVER2_THRIFT_PORT to 0, so it could randomly pick any free port to use.
     // It's much more robust than set a random port generated by ourselves ahead
     sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, "0")
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, "0")
-    sqlContext.setConf(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, mode.toString)
-
-    try {
-      hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
-      hiveServer2.getServices.asScala.foreach {
-        case t: ThriftCLIService =>
-          serverPort = t.getPortNumber
-          logInfo(s"Started HiveThriftServer2: mode=$mode, port=$serverPort, attempt=$attempt")
-        case _ =>
-      }
+    hiveServer2 = HiveThriftServer2.startWithContext(sqlContext)
+    hiveServer2.getServices.asScala.foreach {
+      case t: ThriftCLIService if t.getPortNumber != 0 =>
+        serverPort = t.getPortNumber
+        logInfo(s"Started HiveThriftServer2: port=$serverPort, attempt=$attempt")
+      case _ =>
+    }
 
-      // Wait for thrift server to be ready to serve the query, via executing simple query
-      // till the query succeeds. See SPARK-30345 for more details.
-      eventually(timeout(30.seconds), interval(1.seconds)) {
-        withJdbcStatement { _.execute("SELECT 1") }
-      }
-    } catch {
-      case e: Exception =>
-        logError("Error start hive server with Context ", e)
-        if (hiveServer2 != null) {
-          hiveServer2.stop()
-        }
+    // Wait for thrift server to be ready to serve the query, via executing simple query
+    // till the query succeeds. See SPARK-30345 for more details.
+    eventually(timeout(30.seconds), interval(1.seconds)) {
+      withJdbcStatement { _.execute("SELECT 1") }
     }
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 553f10a275bce..15cc3109da3f7 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -54,9 +54,6 @@ import org.apache.spark.sql.types._
  */
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
-
-  override def mode: ServerMode.Value = ServerMode.binary
-
   override protected def testFile(fileName: String): String = {
     val url = Thread.currentThread().getContextClassLoader.getResource(fileName)
     // Copy to avoid URISyntaxException during accessing the resources in `sql/core`
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index d6420dee41adb..3e1fce78ae71c 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
-trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
+class ThriftServerWithSparkContextSuite extends SharedThriftServer {
 
   test("SPARK-29911: Uncache cached tables when session closed") {
     val cacheManager = spark.sharedState.cacheManager
@@ -42,12 +42,3 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer {
     }
   }
 }
-
-
-class ThriftServerWithSparkContextInBinarySuite extends ThriftServerWithSparkContextSuite {
-  override def mode: ServerMode.Value = ServerMode.binary
-}
-
-class ThriftServerWithSparkContextInHttpSuite extends ThriftServerWithSparkContextSuite {
-  override def mode: ServerMode.Value = ServerMode.http
-}
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index 00bdf7e19126e..e1ee503b81209 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -46,7 +45,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -101,14 +100,6 @@ protected void initializeServer() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + serverSocket.getServerSocket().getLocalPort() + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  @Override
-  public void run() {
-    try {
       server.serve();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index 783e5795aca76..8fce9d9383438 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -175,7 +175,6 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
-      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -634,8 +633,6 @@ public TFetchResultsResp FetchResults(TFetchResultsReq req) throws TException {
     return resp;
   }
 
-  protected abstract void initializeServer();
-
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index bd64c777c1d76..1099a00b67eb7 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.cli.thrift.TCLIService.Iface;
@@ -54,8 +53,13 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -146,19 +150,6 @@ protected void initializeServer() {
           + " mode on port " + connector.getLocalPort()+ " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
-  @Override
-  public void run() {
-    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.fatal(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index ce79e3c8228a6..a7de9c0f3d0d2 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -29,7 +29,6 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.server.ThreadFactoryWithGarbageCleanup;
@@ -47,7 +46,7 @@ public ThriftBinaryCLIService(CLIService cliService) {
   }
 
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       String threadPoolName = "HiveServer2-Handler-Pool";
@@ -102,14 +101,6 @@ protected void initializeServer() {
       String msg = "Starting " + ThriftBinaryCLIService.class.getSimpleName() + " on port "
           + portNum + " with " + minWorkerThreads + "..." + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  @Override
-  public void run() {
-    try {
       server.serve();
     } catch (Throwable t) {
       LOG.error(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
index e46799a1c427d..d41c3b493bb47 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java
@@ -176,7 +176,6 @@ public synchronized void init(HiveConf hiveConf) {
   public synchronized void start() {
     super.start();
     if (!isStarted && !isEmbedded) {
-      initializeServer();
       new Thread(this).start();
       isStarted = true;
     }
@@ -671,8 +670,6 @@ public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req)
     return resp;
   }
 
-  protected abstract void initializeServer();
-
   @Override
   public abstract void run();
 
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index ab9ed5b1f371e..73d5f84476af0 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -28,7 +28,6 @@
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
-import org.apache.hive.service.ServiceException;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.CLIService;
 import org.apache.hive.service.rpc.thrift.TCLIService;
@@ -55,8 +54,13 @@ public ThriftHttpCLIService(CLIService cliService) {
     super(cliService, ThriftHttpCLIService.class.getSimpleName());
   }
 
+  /**
+   * Configure Jetty to serve http requests. Example of a client connection URL:
+   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
+   * e.g. http://gateway:port/hive2/servlets/thrifths2/
+   */
   @Override
-  protected void initializeServer() {
+  public void run() {
     try {
       // Server thread pool
       // Start with minWorkerThreads, expand till maxWorkerThreads and reject subsequent requests
@@ -147,19 +151,6 @@ protected void initializeServer() {
           + " mode on port " + portNum + " path=" + httpPath + " with " + minWorkerThreads + "..."
           + maxWorkerThreads + " worker threads";
       LOG.info(msg);
-    } catch (Exception t) {
-      throw new ServiceException("Error initializing " + getName(), t);
-    }
-  }
-
-  /**
-   * Configure Jetty to serve http requests. Example of a client connection URL:
-   * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ,
-   * e.g. http://gateway:port/hive2/servlets/thrifths2/
-   */
-  @Override
-  public void run() {
-    try {
       httpServer.join();
     } catch (Throwable t) {
       LOG.error(

From c69bbf4652f8ded5108826f38ae2316f31eb10e7 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 16 Jun 2020 04:56:30 +0000
Subject: [PATCH 0837/1280] [SPARK-31959][SQL][3.0] Fix Gregorian-Julian micros
 rebasing while switching standard time zone offset

### What changes were proposed in this pull request?
Fix the bug in microseconds rebasing during transitions from one standard time zone offset to another one. In the PR, I propose to change the implementation of `rebaseGregorianToJulianMicros` which performs rebasing via local timestamps. In the case of overlapping:
1. Check that the original instant belongs to earlier or later instant of overlapped local timestamp.
2. If it is an earlier instant, take zone and DST offsets from the previous day otherwise
3. Set time zone offsets to Julian timestamp from the next day.

Note: The fix assumes that transitions cannot happen more often than once per 2 days.

Adopt the test "SPARK-31959: JST -> HKT at Asia/Hong_Kong in 1945" to outdated tzdb. Old JDK can have outdated time zone database in which Asia/Hong_Kong doesn't have timestamp overlapping in 1945 at all.

### Why are the changes needed?
1. Current implementation handles timestamps overlapping only during daylight saving time but overlapping can happen also during transition from one standard time zone to another one. For example in the case of `Asia/Hong_Kong`, the time zone switched from `Japan Standard Time` (UTC+9) to `Hong Kong Time` (UTC+8) on _Sunday, 18 November, 1945 01:59:59 AM_. The changes allow to handle the special case as well.
2. To fix the test failures on old JDK w/ outdated tzdb like on Jenkins machine `research-jenkins-worker-09`.

### Does this PR introduce _any_ user-facing change?
It might affect micros rebasing in before common era when not-optimised version of `rebaseGregorianToJulianMicros()` is used directly.

### How was this patch tested?
1. By existing tests in `DateTimeUtilsSuite`, `RebaseDateTimeSuite`, `DateFunctionsSuite`, `DateExpressionsSuite` and `TimestampFormatterSuite`.
2. Added new test to `RebaseDateTimeSuite`
3. Regenerated `gregorian-julian-rebase-micros.json` with the step of 30 minutes, and got the same JSON file. The JSON file isn't affected because previously it was generated with the step of 1 week. And the spike in diffs/switch points during 1 hour of timestamp overlapping wasn't detected.

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit c259844)
Signed-off-by: Dongjoon Hyun <dongjoonapache.org>
(cherry picked from commit eae1747)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #28809 from MaxGekk/HongKong-tz-1945-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    | 26 ++++++++++---
 .../catalyst/util/RebaseDateTimeSuite.scala   | 38 +++++++++++++++++++
 2 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index e29fa4b37844d..92d76c88e577b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -326,20 +326,34 @@ object RebaseDateTime {
    */
   private[sql] def rebaseGregorianToJulianMicros(zoneId: ZoneId, micros: Long): Long = {
     val instant = microsToInstant(micros)
-    var ldt = instant.atZone(zoneId).toLocalDateTime
+    val zonedDateTime = instant.atZone(zoneId)
+    var ldt = zonedDateTime.toLocalDateTime
     if (ldt.isAfter(julianEndTs) && ldt.isBefore(gregorianStartTs)) {
       ldt = LocalDateTime.of(gregorianStartDate, ldt.toLocalTime)
     }
     val cal = new Calendar.Builder()
-      // `gregory` is a hybrid calendar that supports both
-      // the Julian and Gregorian calendar systems
+      // `gregory` is a hybrid calendar that supports both the Julian and Gregorian calendar systems
       .setCalendarType("gregory")
       .setDate(ldt.getYear, ldt.getMonthValue - 1, ldt.getDayOfMonth)
       .setTimeOfDay(ldt.getHour, ldt.getMinute, ldt.getSecond)
-      // Local time-line can overlaps, such as at an autumn daylight savings cutover.
-      // This setting selects the original local timestamp mapped to the given `micros`.
-      .set(Calendar.DST_OFFSET, zoneId.getRules.getDaylightSavings(instant).toMillis.toInt)
       .build()
+    // A local timestamp can have 2 instants in the cases of switching from:
+    //  1. Summer to winter time.
+    //  2. One standard time zone to another one. For example, Asia/Hong_Kong switched from JST
+    //     to HKT on 18 November, 1945 01:59:59 AM.
+    // Below we check that the original `instant` is earlier or later instant. If it is an earlier
+    // instant, we take the standard and DST offsets of the previous day otherwise of the next one.
+    val trans = zoneId.getRules.getTransition(ldt)
+    if (trans != null && trans.isOverlap) {
+      val cloned = cal.clone().asInstanceOf[Calendar]
+      // Does the current offset belong to the offset before the transition.
+      // If so, we will take zone offsets from the previous day otherwise from the next day.
+      // This assumes that transitions cannot happen often than once per 2 days.
+      val shift = if (trans.getOffsetBefore == zonedDateTime.getOffset) -1 else 1
+      cloned.add(Calendar.DAY_OF_MONTH, shift)
+      cal.set(Calendar.ZONE_OFFSET, cloned.get(Calendar.ZONE_OFFSET))
+      cal.set(Calendar.DST_OFFSET, cloned.get(Calendar.DST_OFFSET))
+    }
     fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index 0111fa0fd8bbd..6ecdb0514db5b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -409,4 +409,42 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
       }
     }
   }
+
+  test("SPARK-31959: JST -> HKT at Asia/Hong_Kong in 1945") {
+    // The 'Asia/Hong_Kong' time zone switched from 'Japan Standard Time' (JST = UTC+9)
+    // to 'Hong Kong Time' (HKT = UTC+8). After Sunday, 18 November, 1945 01:59:59 AM,
+    // clocks were moved backward to become Sunday, 18 November, 1945 01:00:00 AM.
+    // In this way, the overlap happened w/o Daylight Saving Time.
+    val hkZid = getZoneId("Asia/Hong_Kong")
+    withDefaultTimeZone(hkZid) {
+      var expected = "1945-11-18 01:30:00.0"
+      var ldt = LocalDateTime.of(1945, 11, 18, 1, 30, 0)
+      var earlierMicros = instantToMicros(ldt.atZone(hkZid).withEarlierOffsetAtOverlap().toInstant)
+      var laterMicros = instantToMicros(ldt.atZone(hkZid).withLaterOffsetAtOverlap().toInstant)
+      var overlapInterval = MICROS_PER_HOUR
+      if (earlierMicros + overlapInterval != laterMicros) {
+        // Old JDK might have an outdated time zone database.
+        // See https://bugs.openjdk.java.net/browse/JDK-8228469: "Hong Kong ... Its 1945 transition
+        // from JST to HKT was on 11-18 at 02:00, not 09-15 at 00:00"
+        expected = "1945-09-14 23:30:00.0"
+        ldt = LocalDateTime.of(1945, 9, 14, 23, 30, 0)
+        earlierMicros = instantToMicros(ldt.atZone(hkZid).withEarlierOffsetAtOverlap().toInstant)
+        laterMicros = instantToMicros(ldt.atZone(hkZid).withLaterOffsetAtOverlap().toInstant)
+        // If time zone db doesn't have overlapping at all, set the overlap interval to zero.
+        overlapInterval = laterMicros - earlierMicros
+      }
+      val rebasedEarlierMicros = rebaseGregorianToJulianMicros(hkZid, earlierMicros)
+      val rebasedLaterMicros = rebaseGregorianToJulianMicros(hkZid, laterMicros)
+      def toTsStr(micros: Long): String = toJavaTimestamp(micros).toString
+      assert(toTsStr(rebasedEarlierMicros) === expected)
+      assert(toTsStr(rebasedLaterMicros) === expected)
+      assert(rebasedEarlierMicros + overlapInterval === rebasedLaterMicros)
+      // Check optimized rebasing
+      assert(rebaseGregorianToJulianMicros(earlierMicros) === rebasedEarlierMicros)
+      assert(rebaseGregorianToJulianMicros(laterMicros) === rebasedLaterMicros)
+      // Check reverse rebasing
+      assert(rebaseJulianToGregorianMicros(rebasedEarlierMicros) === earlierMicros)
+      assert(rebaseJulianToGregorianMicros(rebasedLaterMicros) === laterMicros)
+    }
+  }
 }

From 6367d129e7e17d4baf7240fdc2b3f52f3c097b8f Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 16 Jun 2020 06:00:05 +0000
Subject: [PATCH 0838/1280] [SPARK-31986][SQL] Fix Julian-Gregorian micros
 rebasing of overlapping local timestamps

### What changes were proposed in this pull request?
It fixes microseconds rebasing from the hybrid calendar (Julian + Gregorian) to Proleptic Gregorian calendar in the function `RebaseDateTime`.`rebaseJulianToGregorianMicros(zoneId: ZoneId, micros: Long): Long` in the case of local timestamp overlapping.

In the case of overlapping, we look ahead of 1 day to determinate which instant we should take - earlier or later zoned timestamp. If our current standard zone and DST offsets are equal to zone offset of the next date, we choose the later timestamp otherwise the earlier one. For example, the local timestamp **1945-11-18 01:30:00.0** can be mapped to two instants (microseconds since 1970-01-01 00:00:00Z): -761211000000000 or -761207400000000. If the first one is passed to `rebaseJulianToGregorianMicros()`, we take the earlier instant in Proleptic Gregorian calendar while rebasing **1945-11-18T01:30+09:00[Asia/Hong_Kong]** otherwise the later one **1945-11-18T01:30+08:00[Asia/Hong_Kong]**.

Note: The fix assumes that only one transition of standard or DST offsets can occur during a day.

### Why are the changes needed?
Current implementation of `rebaseJulianToGregorianMicros()` handles timestamps overlapping only during daylight saving time but overlapping can happen also during transition from one standard time zone to another one. For example in the case of `Asia/Hong_Kong`, the time zone switched from `Japan Standard Time` (UTC+9) to `Hong Kong Time` (UTC+8) on _Sunday, 18 November, 1945 01:59:59 AM_. The changes allow to handle the special case as well.

### Does this PR introduce _any_ user-facing change?
There is no behaviour change for timestamps of CE after 0001-01-01. The PR might affects timestamps of BCE for which the modified `rebaseJulianToGregorianMicros()` is called directly.

### How was this patch tested?

1. By existing tests in `DateTimeUtilsSuite`, `RebaseDateTimeSuite`, `DateFunctionsSuite`, `DateExpressionsSuite` and `TimestampFormatterSuite`.

2. Added new checks to `RebaseDateTimeSuite`.`SPARK-31959: JST -> HKT at Asia/Hong_Kong in 1945`:
```scala
      assert(rebaseJulianToGregorianMicros(hkZid, rebasedEarlierMicros) === earlierMicros)
      assert(rebaseJulianToGregorianMicros(hkZid, rebasedLaterMicros) === laterMicros)
```

3. Regenerated `julian-gregorian-rebase-micros.json` with the step of 30 minutes, and got the same JSON file. The JSON file isn't affected because previously it was generated with the step of 1 week. And the spike in diffs/switch points during 1 hour of timestamp overlapping wasn't detected.

Closes #28816 from MaxGekk/fix-overlap-julian-2-grep.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit e9145d41f3eae53dcee7d298ee1ae9d065645019)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/util/RebaseDateTime.scala    | 57 ++++++++++---------
 .../catalyst/util/RebaseDateTimeSuite.scala   |  3 +
 2 files changed, 34 insertions(+), 26 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
index 92d76c88e577b..f93878df38cb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.util
 import java.time.{LocalDate, LocalDateTime, LocalTime, ZoneId}
 import java.time.temporal.ChronoField
 import java.util.{Calendar, TimeZone}
+import java.util.Calendar.{DAY_OF_MONTH, DST_OFFSET, ERA, HOUR_OF_DAY, MINUTE, MONTH, SECOND, YEAR, ZONE_OFFSET}
 
 import scala.collection.mutable.AnyRefMap
 
@@ -102,15 +103,15 @@ object RebaseDateTime {
       .setInstant(Math.multiplyExact(days, MILLIS_PER_DAY))
       .build()
     val localDate = LocalDate.of(
-      utcCal.get(Calendar.YEAR),
-      utcCal.get(Calendar.MONTH) + 1,
+      utcCal.get(YEAR),
+      utcCal.get(MONTH) + 1,
       // The number of days will be added later to handle non-existing
       // Julian dates in Proleptic Gregorian calendar.
       // For example, 1000-02-29 exists in Julian calendar because 1000
       // is a leap year but it is not a leap year in Gregorian calendar.
       1)
-      .`with`(ChronoField.ERA, utcCal.get(Calendar.ERA))
-      .plusDays(utcCal.get(Calendar.DAY_OF_MONTH) - 1)
+      .`with`(ChronoField.ERA, utcCal.get(ERA))
+      .plusDays(utcCal.get(DAY_OF_MONTH) - 1)
     Math.toIntExact(localDate.toEpochDay)
   }
 
@@ -350,9 +351,9 @@ object RebaseDateTime {
       // If so, we will take zone offsets from the previous day otherwise from the next day.
       // This assumes that transitions cannot happen often than once per 2 days.
       val shift = if (trans.getOffsetBefore == zonedDateTime.getOffset) -1 else 1
-      cloned.add(Calendar.DAY_OF_MONTH, shift)
-      cal.set(Calendar.ZONE_OFFSET, cloned.get(Calendar.ZONE_OFFSET))
-      cal.set(Calendar.DST_OFFSET, cloned.get(Calendar.DST_OFFSET))
+      cloned.add(DAY_OF_MONTH, shift)
+      cal.set(ZONE_OFFSET, cloned.get(ZONE_OFFSET))
+      cal.set(DST_OFFSET, cloned.get(DST_OFFSET))
     }
     fromMillis(cal.getTimeInMillis) + ldt.get(ChronoField.MICRO_OF_SECOND)
   }
@@ -413,32 +414,36 @@ object RebaseDateTime {
       .setInstant(toMillis(micros))
       .build()
     val localDateTime = LocalDateTime.of(
-      cal.get(Calendar.YEAR),
-      cal.get(Calendar.MONTH) + 1,
+      cal.get(YEAR),
+      cal.get(MONTH) + 1,
       // The number of days will be added later to handle non-existing
       // Julian dates in Proleptic Gregorian calendar.
       // For example, 1000-02-29 exists in Julian calendar because 1000
       // is a leap year but it is not a leap year in Gregorian calendar.
       1,
-      cal.get(Calendar.HOUR_OF_DAY),
-      cal.get(Calendar.MINUTE),
-      cal.get(Calendar.SECOND),
+      cal.get(HOUR_OF_DAY),
+      cal.get(MINUTE),
+      cal.get(SECOND),
       (Math.floorMod(micros, MICROS_PER_SECOND) * NANOS_PER_MICROS).toInt)
-      .`with`(ChronoField.ERA, cal.get(Calendar.ERA))
-      .plusDays(cal.get(Calendar.DAY_OF_MONTH) - 1)
+      .`with`(ChronoField.ERA, cal.get(ERA))
+      .plusDays(cal.get(DAY_OF_MONTH) - 1)
     val zonedDateTime = localDateTime.atZone(zoneId)
-    // Assuming the daylight saving switchover time is 2:00, the local clock will go back to
-    // 2:00 after hitting 2:59. This means the local time between [2:00, 3:00) appears twice, and
-    // can map to two different physical times (seconds from the UTC epoch).
-    // Java 8 time API resolves the ambiguity by picking the earlier physical time. This is the same
-    // as Java 7 time API, except for 2:00 where Java 7 picks the later physical time.
-    // Here we detect the "2:00" case and pick the latter physical time, to be compatible with the
-    // Java 7 date-time.
-    val adjustedZdt = if (cal.get(Calendar.DST_OFFSET) == 0) {
-      zonedDateTime.withLaterOffsetAtOverlap()
-    } else {
-      zonedDateTime
-    }
+    // In the case of local timestamp overlapping, we need to choose the correct time instant
+    // which is related to the original local timestamp. We look ahead of 1 day, and if the next
+    // date has the same standard zone and DST offsets, the current local timestamp belongs to
+    // the period after the transition. In that case, we take the later zoned date time otherwise
+    // earlier one. Here, we assume that transitions happen not often than once per day.
+    val trans = zoneId.getRules.getTransition(localDateTime)
+    val adjustedZdt = if (trans != null && trans.isOverlap) {
+      val dstOffset = cal.get(DST_OFFSET)
+      val zoneOffset = cal.get(ZONE_OFFSET)
+      cal.add(DAY_OF_MONTH, 1)
+      if (zoneOffset == cal.get(ZONE_OFFSET) && dstOffset == cal.get(DST_OFFSET)) {
+        zonedDateTime.withLaterOffsetAtOverlap()
+      } else {
+        zonedDateTime.withEarlierOffsetAtOverlap()
+      }
+    } else zonedDateTime
     instantToMicros(adjustedZdt.toInstant)
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
index 6ecdb0514db5b..971bad1cd957e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala
@@ -445,6 +445,9 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper {
       // Check reverse rebasing
       assert(rebaseJulianToGregorianMicros(rebasedEarlierMicros) === earlierMicros)
       assert(rebaseJulianToGregorianMicros(rebasedLaterMicros) === laterMicros)
+      // Check reverse not-optimized rebasing
+      assert(rebaseJulianToGregorianMicros(hkZid, rebasedEarlierMicros) === earlierMicros)
+      assert(rebaseJulianToGregorianMicros(hkZid, rebasedLaterMicros) === laterMicros)
     }
   }
 }

From e1f9d9252920da1f37deace401bf762e68230d7b Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Tue, 16 Jun 2020 19:20:44 +0900
Subject: [PATCH 0839/1280] [SPARK-31997][SQL][TESTS] Drop test_udtf table when
 SingleSessionSuite test completed
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
`SingleSessionSuite` not do `DROP TABLE IF EXISTS test_udtf` when test completed, then if we do mvn test `HiveThriftBinaryServerSuite`, the test case named `SPARK-11595 ADD JAR with input path having URL scheme` will FAILED because it want to re-create an exists table test_udtf.

### Why are the changes needed?
test suite shouldn't rely on their execution order

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
Manual test，mvn test SingleSessionSuite and HiveThriftBinaryServerSuite in order

Closes #28838 from LuciferYang/drop-test-table.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit d24d27f1bc39e915df23d65f8fda0d83e716b308)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 4e6d4e104021f..396e7f9d397db 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -871,7 +871,7 @@ class SingleSessionSuite extends HiveThriftJdbcTest {
     s"--conf ${HIVE_THRIFT_SERVER_SINGLESESSION.key}=true" :: Nil
 
   test("share the temporary functions across JDBC connections") {
-    withMultipleConnectionJdbcStatement()(
+    withMultipleConnectionJdbcStatement("test_udtf")(
       { statement =>
         val jarPath = "../hive/src/test/resources/TestUDTF.jar"
         val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath"

From 83d928b9805001e6deba3f1599b8b18dc6c857dd Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 17 Jun 2020 12:10:12 +0900
Subject: [PATCH 0840/1280] [SPARK-32011][PYTHON][CORE] Remove warnings about
 pin-thread modes and guide to use collectWithJobGroup

### What changes were proposed in this pull request?

This PR proposes to remove the warning about multi-thread in local properties, and change the guide to use `collectWithJobGroup` for multi-threads for now because:
- It is too noisy to users who don't use multiple threads - the number of this single thread case is arguably more prevailing.
- There was a critical issue found about pin-thread mode SPARK-32010, which will be fixed in Spark 3.1.
- To smoothly migrate, `RDD.collectWithJobGroup` was added, which will be deprecated in Spark 3.1 with SPARK-32010 fixed.

I will target to deprecate `RDD.collectWithJobGroup`, and make this pin-thread mode stable in Spark 3.1. In the future releases, I plan to make this mode as a default mode, and remove `RDD.collectWithJobGroup` away.

### Why are the changes needed?

To avoid guiding users a feature with a critical issue, and provide a proper workaround for now.

### Does this PR introduce _any_ user-facing change?

Yes, warning message and documentation.

### How was this patch tested?

Manually tested:

Before:

```
>>> spark.sparkContext.setLocalProperty("a", "b")
/.../spark/python/pyspark/util.py:141: UserWarning: Currently, 'setLocalProperty' (set to local
properties) with multiple threads does not properly work.
Internally threads on PVM and JVM are not synced, and JVM thread can be reused for multiple
threads on PVM, which fails to isolate local properties for each thread on PVM.
To work around this, you can set PYSPARK_PIN_THREAD to true (see SPARK-22340). However,
note that it cannot inherit the local properties from the parent thread although it isolates each
thread on PVM and JVM with its own local properties.
To work around this, you should manually copy and set the local properties from the parent thread
 to the child thread when you create another thread.
```

After:
```
>>> spark.sparkContext.setLocalProperty("a", "b")
```

Closes #28845 from HyukjinKwon/SPARK-32011.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit feeca63198466640ac461a2a34922493fa6162a8)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/context.py | 37 ++++++-------------------------------
 python/pyspark/util.py    | 27 ---------------------------
 2 files changed, 6 insertions(+), 58 deletions(-)

diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 81b6caab7f450..5bb991e7993f6 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -41,7 +41,6 @@
 from pyspark.traceback_utils import CallSite, first_spark_call
 from pyspark.status import StatusTracker
 from pyspark.profiler import ProfilerCollector, BasicProfiler
-from pyspark.util import _warn_pin_thread
 
 if sys.version > '3':
     xrange = range
@@ -1026,17 +1025,9 @@ def setJobGroup(self, groupId, description, interruptOnCancel=False):
         .. note:: Currently, setting a group ID (set to local properties) with multiple threads
             does not properly work. Internally threads on PVM and JVM are not synced, and JVM
             thread can be reused for multiple threads on PVM, which fails to isolate local
-            properties for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+            properties for each thread on PVM. To work around this, You can use
+            :meth:`RDD.collectWithJobGroup` for now.
         """
-        _warn_pin_thread("setJobGroup")
         self._jsc.setJobGroup(groupId, description, interruptOnCancel)
 
     def setLocalProperty(self, key, value):
@@ -1047,17 +1038,9 @@ def setLocalProperty(self, key, value):
         .. note:: Currently, setting a local property with multiple threads does not properly work.
             Internally threads on PVM and JVM are not synced, and JVM thread
             can be reused for multiple threads on PVM, which fails to isolate local properties
-            for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+            for each thread on PVM. To work around this, You can use
+            :meth:`RDD.collectWithJobGroup`.
         """
-        _warn_pin_thread("setLocalProperty")
         self._jsc.setLocalProperty(key, value)
 
     def getLocalProperty(self, key):
@@ -1074,17 +1057,9 @@ def setJobDescription(self, value):
         .. note:: Currently, setting a job description (set to local properties) with multiple
             threads does not properly work. Internally threads on PVM and JVM are not synced,
             and JVM thread can be reused for multiple threads on PVM, which fails to isolate
-            local properties for each thread on PVM.
-
-            To work around this, you can set `PYSPARK_PIN_THREAD` to
-            `'true'` (see SPARK-22340). However, note that it cannot inherit the local properties
-            from the parent thread although it isolates each thread on PVM and JVM with its own
-            local properties.
-
-            To work around this, you should manually copy and set the local
-            properties from the parent thread to the child thread when you create another thread.
+            local properties for each thread on PVM. To work around this, You can use
+            :meth:`RDD.collectWithJobGroup` for now.
         """
-        _warn_pin_thread("setJobDescription")
         self._jsc.setJobDescription(value)
 
     def sparkUser(self):
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index 93137560de25e..6e53e57f156e6 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -114,33 +114,6 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def _warn_pin_thread(name):
-    if os.environ.get("PYSPARK_PIN_THREAD", "false").lower() == "true":
-        msg = (
-            "PYSPARK_PIN_THREAD feature is enabled. "
-            "However, note that it cannot inherit the local properties from the parent thread "
-            "although it isolates each thread on PVM and JVM with its own local properties. "
-            "\n"
-            "To work around this, you should manually copy and set the local properties from "
-            "the parent thread to the child thread when you create another thread.")
-    else:
-        msg = (
-            "Currently, '%s' (set to local properties) with multiple threads does "
-            "not properly work. "
-            "\n"
-            "Internally threads on PVM and JVM are not synced, and JVM thread can be reused "
-            "for multiple threads on PVM, which fails to isolate local properties for each "
-            "thread on PVM. "
-            "\n"
-            "To work around this, you can set PYSPARK_PIN_THREAD to true (see SPARK-22340). "
-            "However, note that it cannot inherit the local properties from the parent thread "
-            "although it isolates each thread on PVM and JVM with its own local properties. "
-            "\n"
-            "To work around this, you should manually copy and set the local properties from "
-            "the parent thread to the child thread when you create another thread." % name)
-    warnings.warn(msg, UserWarning)
-
-
 def _print_missing_jar(lib_name, pkg_name, jar_name, spark_version):
     print("""
 ________________________________________________________________________________________________

From 185b02b3c9adf161e478b80d2eb63b3e27523dda Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 17 Jun 2020 13:28:47 +0000
Subject: [PATCH 0841/1280] [SPARK-32000][CORE][TESTS] Fix the flaky test for
 partially launched task in barrier-mode

### What changes were proposed in this pull request?

This PR changes the test to get an active executorId and set it as preferred location instead of setting a fixed preferred location.

### Why are the changes needed?

The test is flaky. After checking the [log](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/124086/artifact/core/), I find the root cause is:

Two test cases from different test suites got submitted at the same time because of concurrent execution. In this particular case, the two test cases (from DistributedSuite and BarrierTaskContextSuite) both launch under local-cluster mode. The two applications are submitted at the SAME time so they have the same applications(app-20200615210132-0000). Thus, when the cluster of BarrierTaskContextSuite is launching executors, it failed to create the directory for the executor 0, because the path (/home/jenkins/workspace/work/app-app-20200615210132-0000/0) has been used by the cluster of DistributedSuite. Therefore, it has to launch executor 1 and 2 instead, that lead to non of the tasks can get preferred locality thus they got scheduled together and lead to the test failure.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

The test can not be reproduced locally. We can only know it's been fixed when it's no longer flaky on Jenkins.

Closes #28849 from Ngone51/fix-spark-32000.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 4badef38a52849b4af0b211523de6b09f73397f1)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/scheduler/BarrierTaskContextSuite.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
index 764b4b7587718..9b214afabdd37 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/BarrierTaskContextSuite.scala
@@ -275,11 +275,12 @@ class BarrierTaskContextSuite extends SparkFunSuite with LocalSparkContext with
 
   test("SPARK-31485: barrier stage should fail if only partial tasks are launched") {
     initLocalClusterSparkContext(2)
+    val id = sc.getExecutorIds().head
     val rdd0 = sc.parallelize(Seq(0, 1, 2, 3), 2)
     val dep = new OneToOneDependency[Int](rdd0)
-    // set up a barrier stage with 2 tasks and both tasks prefer executor 0 (only 1 core) for
+    // set up a barrier stage with 2 tasks and both tasks prefer the same executor (only 1 core) for
     // scheduling. So, one of tasks won't be scheduled in one round of resource offer.
-    val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq("executor_h_0"), Seq("executor_h_0")))
+    val rdd = new MyRDD(sc, 2, List(dep), Seq(Seq(s"executor_h_$id"), Seq(s"executor_h_$id")))
     val errorMsg = intercept[SparkException] {
       rdd.barrier().mapPartitions { iter =>
         BarrierTaskContext.get().barrier()

From 70f3ce69ee6ae01393e853ccea16a259804b5d85 Mon Sep 17 00:00:00 2001
From: Dilip Biswal <dkbiswal@gmail.com>
Date: Thu, 18 Jun 2020 22:45:55 +0900
Subject: [PATCH 0842/1280] [SPARK-32020][SQL] Better error message when
 SPARK_HOME or spark.test.home is not set

### What changes were proposed in this pull request?
Better error message when SPARK_HOME or spark,test.home is not set.

### Why are the changes needed?
Currently the error message is not easily consumable as it prints  (see below) the real error after printing the current environment which is rather long.

**Old output**
`
 time.name" -> "Java(TM) SE Runtime Environment", "sun.boot.library.path" -> "/Library/Java/JavaVirtualMachines/jdk1.8.0_221.jdk/Contents/Home/jre/lib",
 "java.vm.version" -> "25.221-b11",
 . . .
 . . .
 . . .
) did not contain key "SPARK_HOME" spark.test.home or SPARK_HOME is not set.
	at org.scalatest.Assertions.newAssertionFailedExceptio
`

**New output**
An exception or error caused a run to abort: spark.test.home or SPARK_HOME is not set.
org.scalatest.exceptions.TestFailedException: spark.test.home or SPARK_HOME is not set
### Does this PR introduce any user-facing change?
`
No.

### How was this patch tested?
Ran the tests in intellej  manually to see the new error.

Closes #28825 from dilipbiswal/minor-spark-31950-followup.

Authored-by: Dilip Biswal <dkbiswal@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e4f503614625f81aa86455ad5f6bec3b61d525f1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../catalyst/parser/TableIdentifierParserSuite.scala |  6 ------
 .../apache/spark/sql/catalyst/plans/SQLHelper.scala  |  9 +++++++++
 .../apache/spark/sql/ExpressionsSchemaSuite.scala    |  6 ------
 .../apache/spark/sql/IntegratedUDFTestUtils.scala    |  9 +--------
 .../org/apache/spark/sql/SQLQueryTestSuite.scala     | 12 +++---------
 5 files changed, 13 insertions(+), 29 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index 04c427da3ec87..a721e17aef02d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -18,7 +18,6 @@ package org.apache.spark.sql.catalyst.parser
 
 import java.io.File
 import java.nio.file.Files
-import java.util.Locale
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
@@ -295,11 +294,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
 
   private val sqlSyntaxDefs = {
     val sqlBasePath = {
-      val sparkHome = {
-        assert(sys.props.contains("spark.test.home") ||
-          sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-        sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-      }
       java.nio.file.Paths.get(sparkHome, "sql", "catalyst", "src", "main", "antlr4", "org",
         "apache", "spark", "sql", "catalyst", "parser", "SqlBase.g4").toFile
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
index d213743946e76..5deab79b9f67d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/SQLHelper.scala
@@ -21,6 +21,8 @@ import java.time.ZoneId
 
 import scala.util.control.NonFatal
 
+import org.scalatest.Assertions.fail
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.getZoneId
@@ -83,4 +85,11 @@ trait SQLHelper {
       }
     }
   }
+
+  protected lazy val sparkHome: String = {
+    if (!(sys.props.contains("spark.test.home") || sys.env.contains("SPARK_HOME"))) {
+      fail("spark.test.home or SPARK_HOME is not set.")
+    }
+    sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
index d69ecd7d5660f..81c09d169efb2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala
@@ -72,12 +72,6 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
     // We use a path based on Spark home for 2 reasons:
     //   1. Maven can't get correct resource directory when resources in other jars.
     //   2. We test subclasses in the hive-thriftserver module.
-    val sparkHome = {
-      assert(sys.props.contains("spark.test.home") ||
-        sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-      sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-    }
-
     java.nio.file.Paths.get(sparkHome,
       "sql", "core", "src", "test", "resources", "sql-functions").toFile
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
index 4a4504a075060..6391d5664d5c6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
@@ -75,14 +75,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
   import scala.sys.process._
 
   private lazy val pythonPath = sys.env.getOrElse("PYTHONPATH", "")
-  private lazy val sparkHome = if (sys.props.contains(Tests.IS_TESTING.key)) {
-    assert(sys.props.contains("spark.test.home") ||
-      sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-    sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-  } else {
-    assert(sys.env.contains("SPARK_HOME"), "SPARK_HOME is not set.")
-    sys.env("SPARK_HOME")
-  }
+
   // Note that we will directly refer pyspark's source, not the zip from a regular build.
   // It is possible the test is being ran without the build.
   private lazy val sourcePath = Paths.get(sparkHome, "python").toAbsolutePath
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 005cac754d975..f43b838c79f24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -19,13 +19,13 @@ package org.apache.spark.sql
 
 import java.io.File
 import java.util.Locale
-import java.util.regex.Pattern
 
-import scala.collection.mutable.{ArrayBuffer, HashMap}
+import scala.collection.mutable.ArrayBuffer
 import scala.util.control.NonFatal
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
 import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile}
@@ -122,7 +122,7 @@ import org.apache.spark.tags.ExtendedSQLTest
  * different types of UDFs. See 'udf/udf-inner-join.sql' as an example.
  */
 @ExtendedSQLTest
-class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
+class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper {
 
   import IntegratedUDFTestUtils._
 
@@ -132,12 +132,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
     // We use a path based on Spark home for 2 reasons:
     //   1. Maven can't get correct resource directory when resources in other jars.
     //   2. We test subclasses in the hive-thriftserver module.
-    val sparkHome = {
-      assert(sys.props.contains("spark.test.home") ||
-        sys.env.contains("SPARK_HOME"), "spark.test.home or SPARK_HOME is not set.")
-      sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
-    }
-
     java.nio.file.Paths.get(sparkHome,
       "sql", "core", "src", "test", "resources", "sql-tests").toFile
   }

From cfeb7faf215b006463c5e676ebf7a3317f7730f1 Mon Sep 17 00:00:00 2001
From: James Yu <yuj@users.noreply.github.com>
Date: Thu, 18 Jun 2020 14:36:20 -0700
Subject: [PATCH 0843/1280] [MINOR][DOCS] Update running-on-kubernetes.md

### What changes were proposed in this pull request?

Fix executor container name typo. `executor` should be `spark-kubernetes-executor`.

### Why are the changes needed?

The Executor pod container name the users actually get from their Kubernetes clusters is different from that described in the documentation.

For example, below is what a user get from an executor pod.
```
Containers:
  spark-kubernetes-executor:
    Container ID:  docker://aaaabbbbccccddddeeeeffff
    Image:         <imagename>
    Image ID:      docker-pullable://0000.dkr.ecr.us-east-0.amazonaws.com/spark
    Port:          7079/TCP
    Host Port:     0/TCP
    Args:
      executor
    State:          Running
      Started:      Thu, 28 May 2020 05:54:04 -0700
    Ready:          True
    Restart Count:  0
    Limits:
      memory:  16Gi
```

### Does this PR introduce _any_ user-facing change?

Document change.

### How was this patch tested?

N/A

Closes #28862 from yuj/patch-1.

Authored-by: James Yu <yuj@users.noreply.github.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ac98a9a07fe3e76c16bd5bb03aca553e80629e01)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/running-on-kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 3abb891ee0d0b..e6a000de1ab08 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1333,7 +1333,7 @@ The following affect the driver and executor containers. All other containers in
   <td>See description</td>
   <td>
     The container name will be assigned by spark ("spark-kubernetes-driver" for the driver container, and
-    "executor" for each executor container) if not defined by the pod template. If the container is defined by the
+    "spark-kubernetes-executor" for each executor container) if not defined by the pod template. If the container is defined by the
     template, the template's name will be used.
   </td>
 </tr>

From 73af23cabee6aac471fbdc9f4a35d118517fad65 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Fri, 19 Jun 2020 12:17:40 +0900
Subject: [PATCH 0844/1280] [MINOR][DOCS] Emphasize the Streaming tab is for
 DStream API

### What changes were proposed in this pull request?
Emphasize the Streaming tab is for DStream API.

### Why are the changes needed?
Some users reported that it's a little confusing of the streaming tab and structured streaming tab.

### Does this PR introduce _any_ user-facing change?
Document change.

### How was this patch tested?
N/A

Closes #28854 from xuanyuanking/minor-doc.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8750363c8ddaa649d52e6d9001b9871d379fc4e1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/web-ui.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/web-ui.md b/docs/web-ui.md
index 6653b00fa77c1..134a8c860d404 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -434,10 +434,10 @@ The tracked operations are listed as follows.
 As an early-release version, the statistics page is still under development and will be improved in
 future releases.
 
-## Streaming Tab
-The web UI includes a Streaming tab if the application uses Spark streaming. This tab displays
-scheduling delay and processing time for each micro-batch in the data stream, which can be useful
-for troubleshooting the streaming application.
+## Streaming (DStreams) Tab
+The web UI includes a Streaming tab if the application uses Spark Streaming with DStream API.
+This tab displays scheduling delay and processing time for each micro-batch in the data stream,
+which can be useful for troubleshooting the streaming application.
 
 ## JDBC/ODBC Server Tab
 We can see this tab when Spark is running as a [distributed SQL engine](sql-distributed-sql-engine.html). It shows information about sessions and submitted SQL operations.

From 5f76cab1625f4365fe703514bdb09b2970f5a74a Mon Sep 17 00:00:00 2001
From: TJX2014 <xiaoxingstack@gmail.com>
Date: Fri, 19 Jun 2020 19:24:34 -0700
Subject: [PATCH 0845/1280] [SPARK-31980][SQL] Function sequence() fails if
 start and end of range are equal dates

### What changes were proposed in this pull request?
1. Add judge equal as bigger condition in `org.apache.spark.sql.catalyst.expressions.Sequence.TemporalSequenceImpl#eval`
2. Unit test for interval `day`, `month`, `year`

### Why are the changes needed?
Bug exists when sequence input get same equal start and end dates, which will occur `while loop` forever

### Does this PR introduce _any_ user-facing change?
Yes,
Before this PR, people will get a `java.lang.ArrayIndexOutOfBoundsException`, when eval as below:
`sql("select sequence(cast('2011-03-01' as date), cast('2011-03-01' as date), interval 1 year)").show(false)
`

### How was this patch tested?
Unit test.

Closes #28819 from TJX2014/master-SPARK-31980.

Authored-by: TJX2014 <xiaoxingstack@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 177a380bcf1f56982760442e8418d28415bef8b0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/collectionOperations.scala     |  4 ++--
 .../CollectionExpressionsSuite.scala           | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 4fd68dcfe5156..7a15bcce836f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -2624,7 +2624,7 @@ object Sequence {
         val maxEstimatedArrayLength =
           getSequenceLength(startMicros, stopMicros, intervalStepInMicros)
 
-        val stepSign = if (stopMicros > startMicros) +1 else -1
+        val stepSign = if (stopMicros >= startMicros) +1 else -1
         val exclusiveItem = stopMicros + stepSign
         val arr = new Array[T](maxEstimatedArrayLength)
         var t = startMicros
@@ -2686,7 +2686,7 @@ object Sequence {
          |
          |  $sequenceLengthCode
          |
-         |  final int $stepSign = $stopMicros > $startMicros ? +1 : -1;
+         |  final int $stepSign = $stopMicros >= $startMicros ? +1 : -1;
          |  final long $exclusiveItem = $stopMicros + $stepSign;
          |
          |  $arr = new $elemType[$arrLength];
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 91773824b845d..c8c97d3e39abe 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -1836,4 +1836,22 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     checkEvaluation(ArrayIntersect(empty, oneNull), Seq.empty)
     checkEvaluation(ArrayIntersect(oneNull, empty), Seq.empty)
   }
+
+  test("SPARK-31980: Start and end equal in month range") {
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 day"))),
+      Seq(Date.valueOf("2018-01-01")))
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 month"))),
+      Seq(Date.valueOf("2018-01-01")))
+    checkEvaluation(new Sequence(
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(Date.valueOf("2018-01-01")),
+      Literal(stringToInterval("interval 1 year"))),
+      Seq(Date.valueOf("2018-01-01")))
+  }
 }

From 2f3618cdf9cfa813d7a44ab9a7ab24035cb1fd33 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sat, 20 Jun 2020 11:12:03 -0700
Subject: [PATCH 0846/1280] [SPARK-32021][SQL][3.0] Increase precision of
 seconds and fractions of `make_interval`

### What changes were proposed in this pull request?
Change precision of seconds and its fraction from 8 to 18 to be able to construct intervals of max allowed microseconds value (long).

### Why are the changes needed?
To improve UX of Spark SQL.

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
- Add tests to IntervalExpressionsSuite
- Add an example to the `MakeInterval` expression
- Add tests to `interval.sql`

Authored-by: Max Gekk max.gekkgmail.com
Signed-off-by: Dongjoon Hyun dongjoonapache.org
(cherry picked from commit 66ba35666a49023cd8da30cf84aff5c6fb1d8799)

Closes #28878 from MaxGekk/make_interval-sec-precision-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/intervalExpressions.scala     |  8 ++++---
 .../IntervalExpressionsSuite.scala            |  9 +++++++-
 .../sql-functions/sql-expression-schema.md    |  4 ++--
 .../resources/sql-tests/inputs/interval.sql   |  2 ++
 .../sql-tests/results/ansi/interval.sql.out   | 21 +++++++++++++++++--
 .../sql-tests/results/interval.sql.out        | 20 ++++++++++++++++--
 6 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
index 1a569a7b89fe1..8cfea1e6f6cb5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala
@@ -149,6 +149,8 @@ case class DivideInterval(
        100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds
       > SELECT _FUNC_(100, null, 3);
        NULL
+      > SELECT _FUNC_(0, 1, 0, 1, 0, 0, 100.000001);
+       1 months 1 days 1 minutes 40.000001 seconds
   """,
   since = "3.0.0")
 // scalastyle:on line.size.limit
@@ -169,7 +171,7 @@ case class MakeInterval(
       days: Expression,
       hours: Expression,
       mins: Expression) = {
-    this(years, months, weeks, days, hours, mins, Literal(Decimal(0, 8, 6)))
+    this(years, months, weeks, days, hours, mins, Literal(Decimal(0, Decimal.MAX_LONG_DIGITS, 6)))
   }
   def this(
       years: Expression,
@@ -191,7 +193,7 @@ case class MakeInterval(
   // Accept `secs` as DecimalType to avoid loosing precision of microseconds while converting
   // them to the fractional part of `secs`.
   override def inputTypes: Seq[AbstractDataType] = Seq(IntegerType, IntegerType, IntegerType,
-    IntegerType, IntegerType, IntegerType, DecimalType(8, 6))
+    IntegerType, IntegerType, IntegerType, DecimalType(Decimal.MAX_LONG_DIGITS, 6))
   override def dataType: DataType = CalendarIntervalType
   override def nullable: Boolean = true
 
@@ -211,7 +213,7 @@ case class MakeInterval(
         day.asInstanceOf[Int],
         hour.asInstanceOf[Int],
         min.asInstanceOf[Int],
-        sec.map(_.asInstanceOf[Decimal]).getOrElse(Decimal(0, 8, 6)))
+        sec.map(_.asInstanceOf[Decimal]).getOrElse(Decimal(0, Decimal.MAX_LONG_DIGITS, 6)))
     } catch {
       case _: ArithmeticException => null
     }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
index 8c972a9917043..6b7be4f1609a5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/IntervalExpressionsSuite.scala
@@ -190,7 +190,8 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         micros: Int = 0): Unit = {
       val secFrac = DateTimeTestUtils.secFrac(seconds, millis, micros)
       val intervalExpr = MakeInterval(Literal(years), Literal(months), Literal(weeks),
-        Literal(days), Literal(hours), Literal(minutes), Literal(Decimal(secFrac, 8, 6)))
+        Literal(days), Literal(hours), Literal(minutes),
+        Literal(Decimal(secFrac, Decimal.MAX_LONG_DIGITS, 6)))
       val totalMonths = years * MONTHS_PER_YEAR + months
       val totalDays = weeks * DAYS_PER_WEEK + days
       val totalMicros = secFrac + minutes * MICROS_PER_MINUTE + hours * MICROS_PER_HOUR
@@ -206,5 +207,11 @@ class IntervalExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     check(years = 10000, micros = -1)
     check(-9999, -11, 0, -31, -23, -59, -59, -999, -999)
     check(years = -10000, micros = 1)
+    check(
+      hours = Int.MaxValue,
+      minutes = Int.MaxValue,
+      seconds = Int.MaxValue,
+      millis = Int.MaxValue,
+      micros = Int.MaxValue)
   }
 }
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 8040d0dd637a1..f8b1190fdf5d0 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -163,7 +163,7 @@
 | org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lcase(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.Lower | lower | SELECT lower('SparkSql') | struct<lower(SparkSql):string> |
 | org.apache.spark.sql.catalyst.expressions.MakeDate | make_date | SELECT make_date(2013, 7, 15) | struct<make_date(2013, 7, 15):date> |
-| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(8,6))):interval> |
+| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(18,6))):interval> |
 | org.apache.spark.sql.catalyst.expressions.MakeTimestamp | make_timestamp | SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887) | struct<make_timestamp(2014, 12, 28, 6, 30, CAST(45.887 AS DECIMAL(8,6))):timestamp> |
 | org.apache.spark.sql.catalyst.expressions.MapConcat | map_concat | SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')) | struct<map_concat(map(1, a, 2, b), map(3, c)):map<int,string>> |
 | org.apache.spark.sql.catalyst.expressions.MapEntries | map_entries | SELECT map_entries(map(1, 'a', 2, 'b')) | struct<map_entries(map(1, a, 2, b)):array<struct<key:int,value:string>>> |
@@ -333,4 +333,4 @@
 | org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') | struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>, a/b/text()):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathLong | xpath_long | SELECT xpath_long('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_long(<a><b>1</b><b>2</b></a>, sum(a/b)):bigint> |
 | org.apache.spark.sql.catalyst.expressions.xml.XPathShort | xpath_short | SELECT xpath_short('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_short(<a><b>1</b><b>2</b></a>, sum(a/b)):smallint> |
-| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
+| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
\ No newline at end of file
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index 600df2c213519..a7e1afef66604 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -29,6 +29,8 @@ select make_interval(1, 2, 3, 4);
 select make_interval(1, 2, 3, 4, 5);
 select make_interval(1, 2, 3, 4, 5, 6);
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009);
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456);
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789);
 
 -- cast string to intervals
 select cast('1 second' as interval);
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 6a2b301c6fb18..e83444d95dd1c 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 90
+-- Number of queries: 92
 
 
 -- !query
@@ -178,11 +178,28 @@ struct<make_interval(1, 2, 3, 4, 5, 6, 0.000000):interval>
 -- !query
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009)
 -- !query schema
-struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(8,6))):interval>
+struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(18,6))):interval>
 -- !query output
 1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds
 
 
+-- !query
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+-- !query schema
+struct<make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456):interval>
+-- !query output
+1 years 2 months 25 days 34293552 hours 30 minutes 12.123456 seconds
+
+
+-- !query
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
+-- !query schema
+struct<>
+-- !query output
+java.lang.ArithmeticException
+Decimal(expanded,1234567890123456789,20,0}) cannot be represented as Decimal(18, 6).
+
+
 -- !query
 select cast('1 second' as interval)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 94562cae951a0..4cdc66950f4a7 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 90
+-- Number of queries: 92
 
 
 -- !query
@@ -173,11 +173,27 @@ struct<make_interval(1, 2, 3, 4, 5, 6, 0.000000):interval>
 -- !query
 select make_interval(1, 2, 3, 4, 5, 6, 7.008009)
 -- !query schema
-struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(8,6))):interval>
+struct<make_interval(1, 2, 3, 4, 5, 6, CAST(7.008009 AS DECIMAL(18,6))):interval>
 -- !query output
 1 years 2 months 25 days 5 hours 6 minutes 7.008009 seconds
 
 
+-- !query
+select make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456)
+-- !query schema
+struct<make_interval(1, 2, 3, 4, 0, 0, 123456789012.123456):interval>
+-- !query output
+1 years 2 months 25 days 34293552 hours 30 minutes 12.123456 seconds
+
+
+-- !query
+select make_interval(0, 0, 0, 0, 0, 0, 1234567890123456789)
+-- !query schema
+struct<make_interval(0, 0, 0, 0, 0, 0, CAST(1234567890123456789 AS DECIMAL(18,6))):interval>
+-- !query output
+NULL
+
+
 -- !query
 select cast('1 second' as interval)
 -- !query schema

From c11078b93af09b61e57e4a653d3a33494725cc1d Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Sun, 21 Jun 2020 16:28:00 -0700
Subject: [PATCH 0847/1280] [SPARK-32034][SQL] Port HIVE-14817: Shutdown the
 SessionManager timeoutChecker thread properly upon shutdown

### What changes were proposed in this pull request?

This PR port https://issues.apache.org/jira/browse/HIVE-14817 for spark thrift server.

### Why are the changes needed?

When stopping the HiveServer2, the non-daemon thread stops the server from terminating

```sql
"HiveServer2-Background-Pool: Thread-79" #79 prio=5 os_prio=31 tid=0x00007fde26138800 nid=0x13713 waiting on condition [0x0000700010c32000]
   java.lang.Thread.State: TIMED_WAITING (sleeping)
	at java.lang.Thread.sleep(Native Method)
	at org.apache.hive.service.cli.session.SessionManager$1.sleepInterval(SessionManager.java:178)
	at org.apache.hive.service.cli.session.SessionManager$1.run(SessionManager.java:156)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
```

Here is an example to reproduce:
https://github.com/yaooqinn/kyuubi/blob/master/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/spark/SparkSQLEngineApp.scala

Also, it causes issues as HIVE-14817 described which

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?

Passing Jenkins

Closes #28870 from yaooqinn/SPARK-32034.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 9f8e15bb2e2189812ee34e3e64baede0d799ba76)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../service/cli/session/SessionManager.java   | 32 ++++++++++++++-----
 .../service/cli/session/SessionManager.java   | 32 ++++++++++++++-----
 2 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
index 859f9c8b449e5..ad6fb3ba37a0e 100644
--- a/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ b/sql/hive-thriftserver/v1.2/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -148,14 +148,20 @@ public synchronized void start() {
     }
   }
 
+  private final Object timeoutCheckerLock = new Object();
+
   private void startTimeoutChecker() {
     final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
-    Runnable timeoutChecker = new Runnable() {
+    final Runnable timeoutChecker = new Runnable() {
       @Override
       public void run() {
-        for (sleepInterval(interval); !shutdown; sleepInterval(interval)) {
+        sleepFor(interval);
+        while (!shutdown) {
           long current = System.currentTimeMillis();
           for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
+            if (shutdown) {
+              break;
+            }
             if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
                 && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
               SessionHandle handle = session.getSessionHandle();
@@ -170,24 +176,34 @@ public void run() {
               session.closeExpiredOperations();
             }
           }
+          sleepFor(interval);
         }
       }
 
-      private void sleepInterval(long interval) {
-        try {
-          Thread.sleep(interval);
-        } catch (InterruptedException e) {
-          // ignore
+      private void sleepFor(long interval) {
+        synchronized (timeoutCheckerLock) {
+          try {
+            timeoutCheckerLock.wait(interval);
+          } catch (InterruptedException e) {
+            // Ignore, and break.
+          }
         }
       }
     };
     backgroundOperationPool.execute(timeoutChecker);
   }
 
+  private void shutdownTimeoutChecker() {
+    shutdown = true;
+    synchronized (timeoutCheckerLock) {
+      timeoutCheckerLock.notify();
+    }
+  }
+
   @Override
   public synchronized void stop() {
     super.stop();
-    shutdown = true;
+    shutdownTimeoutChecker();
     if (backgroundOperationPool != null) {
       backgroundOperationPool.shutdown();
       long timeout = hiveConf.getTimeVar(
diff --git a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
index 49221b13bb892..5a381d170b4f9 100644
--- a/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
+++ b/sql/hive-thriftserver/v2.3/src/main/java/org/apache/hive/service/cli/session/SessionManager.java
@@ -148,14 +148,20 @@ public synchronized void start() {
     }
   }
 
+  private final Object timeoutCheckerLock = new Object();
+
   private void startTimeoutChecker() {
     final long interval = Math.max(checkInterval, 3000L);  // minimum 3 seconds
-    Runnable timeoutChecker = new Runnable() {
+    final Runnable timeoutChecker = new Runnable() {
       @Override
       public void run() {
-        for (sleepInterval(interval); !shutdown; sleepInterval(interval)) {
+        sleepFor(interval);
+        while (!shutdown) {
           long current = System.currentTimeMillis();
           for (HiveSession session : new ArrayList<HiveSession>(handleToSession.values())) {
+            if (shutdown) {
+              break;
+            }
             if (sessionTimeout > 0 && session.getLastAccessTime() + sessionTimeout <= current
                 && (!checkOperation || session.getNoOperationTime() > sessionTimeout)) {
               SessionHandle handle = session.getSessionHandle();
@@ -170,24 +176,34 @@ public void run() {
               session.closeExpiredOperations();
             }
           }
+          sleepFor(interval);
         }
       }
 
-      private void sleepInterval(long interval) {
-        try {
-          Thread.sleep(interval);
-        } catch (InterruptedException e) {
-          // ignore
+      private void sleepFor(long interval) {
+        synchronized (timeoutCheckerLock) {
+          try {
+            timeoutCheckerLock.wait(interval);
+          } catch (InterruptedException e) {
+            // Ignore, and break.
+          }
         }
       }
     };
     backgroundOperationPool.execute(timeoutChecker);
   }
 
+  private void shutdownTimeoutChecker() {
+    shutdown = true;
+    synchronized (timeoutCheckerLock) {
+      timeoutCheckerLock.notify();
+    }
+  }
+
   @Override
   public synchronized void stop() {
     super.stop();
-    shutdown = true;
+    shutdownTimeoutChecker();
     if (backgroundOperationPool != null) {
       backgroundOperationPool.shutdown();
       long timeout = hiveConf.getTimeVar(

From 92f3877b0f4451100ba2aa6cac6b66900f870951 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Mon, 22 Jun 2020 04:58:22 -0700
Subject: [PATCH 0848/1280] [SPARK-32038][SQL] NormalizeFloatingNumbers should
 also work on distinct aggregate

### What changes were proposed in this pull request?

This patch applies `NormalizeFloatingNumbers` to distinct aggregate to fix a regression of distinct aggregate on NaNs.

### Why are the changes needed?

We added `NormalizeFloatingNumbers` optimization rule in 3.0.0 to normalize special floating numbers (NaN and -0.0). But it is missing in distinct aggregate so causes a regression. We need to apply this rule on distinct aggregate to fix it.

### Does this PR introduce _any_ user-facing change?

Yes, fixing a regression of distinct aggregate on NaNs.

### How was this patch tested?

Added unit test.

Closes #28876 from viirya/SPARK-32038.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 2e4557f45ce65ad0cf501c1734f2d4a50b00af54)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/SparkStrategies.scala  | 18 ++++++++++++++++++
 .../sql/execution/aggregate/AggUtils.scala     | 16 ++++------------
 .../spark/sql/DataFrameAggregateSuite.scala    | 16 ++++++++++++++++
 3 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 12a1a1e7fc16e..10cd7b6222c50 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -539,10 +539,28 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               resultExpressions,
               planLater(child))
           } else {
+            // functionsWithDistinct is guaranteed to be non-empty. Even though it may contain
+            // more than one DISTINCT aggregate function, all of those functions will have the
+            // same column expressions. For example, it would be valid for functionsWithDistinct
+            // to be [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but
+            // [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is disallowed because those two distinct
+            // aggregates have different column expressions.
+            val distinctExpressions = functionsWithDistinct.head.aggregateFunction.children
+            val normalizedNamedDistinctExpressions = distinctExpressions.map { e =>
+              // Ideally this should be done in `NormalizeFloatingNumbers`, but we do it here
+              // because `distinctExpressions` is not extracted during logical phase.
+              NormalizeFloatingNumbers.normalize(e) match {
+                case ne: NamedExpression => ne
+                case other => Alias(other, other.toString)()
+              }
+            }
+
             AggUtils.planAggregateWithOneDistinct(
               normalizedGroupingExpressions,
               functionsWithDistinct,
               functionsWithoutDistinct,
+              distinctExpressions,
+              normalizedNamedDistinctExpressions,
               resultExpressions,
               planLater(child))
           }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
index 56a287d4d0279..761ac20e84744 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
@@ -135,20 +135,12 @@ object AggUtils {
       groupingExpressions: Seq[NamedExpression],
       functionsWithDistinct: Seq[AggregateExpression],
       functionsWithoutDistinct: Seq[AggregateExpression],
+      distinctExpressions: Seq[Expression],
+      normalizedNamedDistinctExpressions: Seq[NamedExpression],
       resultExpressions: Seq[NamedExpression],
       child: SparkPlan): Seq[SparkPlan] = {
 
-    // functionsWithDistinct is guaranteed to be non-empty. Even though it may contain more than one
-    // DISTINCT aggregate function, all of those functions will have the same column expressions.
-    // For example, it would be valid for functionsWithDistinct to be
-    // [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is
-    // disallowed because those two distinct aggregates have different column expressions.
-    val distinctExpressions = functionsWithDistinct.head.aggregateFunction.children
-    val namedDistinctExpressions = distinctExpressions.map {
-      case ne: NamedExpression => ne
-      case other => Alias(other, other.toString)()
-    }
-    val distinctAttributes = namedDistinctExpressions.map(_.toAttribute)
+    val distinctAttributes = normalizedNamedDistinctExpressions.map(_.toAttribute)
     val groupingAttributes = groupingExpressions.map(_.toAttribute)
 
     // 1. Create an Aggregate Operator for partial aggregations.
@@ -159,7 +151,7 @@ object AggUtils {
       // DISTINCT column. For example, for AVG(DISTINCT value) GROUP BY key, the grouping
       // expressions will be [key, value].
       createAggregate(
-        groupingExpressions = groupingExpressions ++ namedDistinctExpressions,
+        groupingExpressions = groupingExpressions ++ normalizedNamedDistinctExpressions,
         aggregateExpressions = aggregateExpressions,
         aggregateAttributes = aggregateAttributes,
         resultExpressions = groupingAttributes ++ distinctAttributes ++
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 2293d4ae61aff..f7438f3ffec04 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -1012,4 +1012,20 @@ class DataFrameAggregateSuite extends QueryTest
       }
     }
   }
+
+  test("SPARK-32038: NormalizeFloatingNumbers should work on distinct aggregate") {
+    withTempView("view") {
+      val nan1 = java.lang.Float.intBitsToFloat(0x7f800001)
+      val nan2 = java.lang.Float.intBitsToFloat(0x7fffffff)
+
+      Seq(("mithunr", Float.NaN),
+        ("mithunr", nan1),
+        ("mithunr", nan2),
+        ("abellina", 1.0f),
+        ("abellina", 2.0f)).toDF("uid", "score").createOrReplaceTempView("view")
+
+      val df = spark.sql("select uid, count(distinct score) from view group by 1 order by 1 asc")
+      checkAnswer(df, Row("abellina", 2) :: Row("mithunr", 1) :: Nil)
+    }
+  }
 }

From 664443c29c71bd6424132cb1ec636df420833ed9 Mon Sep 17 00:00:00 2001
From: Dilip Biswal <dkbiswal@gmail.com>
Date: Mon, 22 Jun 2020 21:38:19 +0900
Subject: [PATCH 0849/1280] [MINOR][SQL] Add `IS [NOT] NULL` examples to
 ArrayFilter/ArrayExists

### What changes were proposed in this pull request?
A minor PR that adds a couple of usage examples for ArrayFilter and ArrayExists that shows how to deal with NULL data.

### Why are the changes needed?
Enhances the examples that shows how to filter out null values from an array and also to test if null value exists in an array.

### Does this PR introduce any user-facing change?
No.

### How was this patch tested?
Tested manually.

Closes #28890 from dilipbiswal/array_func_description.

Authored-by: Dilip Biswal <dkbiswal@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 6293c38cff66c503370e970b7e8e0af9f993a3c2)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/expressions/higherOrderFunctions.scala     | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index e91bd0c7015ee..e5cf8c0a023d9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -454,6 +454,8 @@ case class MapFilter(
        [1,3]
       > SELECT _FUNC_(array(0, 2, 3), (x, i) -> x > i);
        [2,3]
+      > SELECT _FUNC_(array(0, null, 2, 3, null), x -> x IS NOT NULL);
+       [0,2,3]
   """,
   since = "2.4.0",
   note = """
@@ -518,6 +520,10 @@ case class ArrayFilter(
        false
       > SELECT _FUNC_(array(1, null, 3), x -> x % 2 == 0);
        NULL
+      > SELECT _FUNC_(array(0, null, 2, 3, null), x -> x IS NULL);
+       true
+      > SELECT _FUNC_(array(1, 2, 3), x -> x IS NULL);
+       false
   """,
   since = "2.4.0")
 case class ArrayExists(

From f18c0f61f8cd3757fc28078a3d92ce69babf04b3 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 24 Jun 2020 11:03:05 +0900
Subject: [PATCH 0850/1280] [SPARK-31918][R] Ignore S4 generic methods under
 SparkR namespace in closure cleaning to support R 4.0.0+
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

This PR proposes to ignore S4 generic methods under SparkR namespace in closure cleaning to support R 4.0.0+.

Currently, when you run the codes that runs R native codes, it fails as below with R 4.0.0:

```r
df <- createDataFrame(lapply(seq(100), function (e) list(value=e)))
count(dapply(df, function(x) as.data.frame(x[x$value < 50,]), schema(df)))
```

```
org.apache.spark.SparkException: R unexpectedly exited.
R worker produced errors: Error in lapply(part, FUN) : attempt to bind a variable to R_UnboundValue
```

The root cause seems to be related to when an S4 generic method is manually included into the closure's environment via `SparkR:::cleanClosure`. For example, when an RRDD is created via `createDataFrame` with calling `lapply` to convert, `lapply` itself:

https://github.com/apache/spark/blob/f53d8c63e80172295e2fbc805c0c391bdececcaa/R/pkg/R/RDD.R#L484

is added into the environment of the cleaned closure - because this is not an exposed namespace; however, this is broken in R 4.0.0+ for an unknown reason with an error message such as "attempt to bind a variable to R_UnboundValue".

Actually, we don't need to add the `lapply` into the environment of the closure because it is not supposed to be called in worker side. In fact, there is no private generic methods supposed to be called in worker side in SparkR at all from my understanding.

Therefore, this PR takes a simpler path to work around just by explicitly excluding the S4 generic methods under SparkR namespace to support R 4.0.0. in SparkR.

### Why are the changes needed?

To support R 4.0.0+ with SparkR, and unblock the releases on CRAN. CRAN requires the tests pass with the latest R.

### Does this PR introduce _any_ user-facing change?

Yes, it will support R 4.0.0 to end-users.

### How was this patch tested?

Manually tested. Both CRAN and tests with R 4.0.1:

```
══ testthat results  ═══════════════════════════════════════════════════════════
[ OK: 13 | SKIPPED: 0 | WARNINGS: 0 | FAILED: 0 ]
✔ |  OK F W S | Context
✔ |  11       | binary functions [2.5 s]
✔ |   4       | functions on binary files [2.1 s]
✔ |   2       | broadcast variables [0.5 s]
✔ |   5       | functions in client.R
✔ |  46       | test functions in sparkR.R [6.3 s]
✔ |   2       | include R packages [0.3 s]
✔ |   2       | JVM API [0.2 s]
✔ |  75       | MLlib classification algorithms, except for tree-based algorithms [86.3 s]
✔ |  70       | MLlib clustering algorithms [44.5 s]
✔ |   6       | MLlib frequent pattern mining [3.0 s]
✔ |   8       | MLlib recommendation algorithms [9.6 s]
✔ | 136       | MLlib regression algorithms, except for tree-based algorithms [76.0 s]
✔ |   8       | MLlib statistics algorithms [0.6 s]
✔ |  94       | MLlib tree-based algorithms [85.2 s]
✔ |  29       | parallelize() and collect() [0.5 s]
✔ | 428       | basic RDD functions [25.3 s]
✔ |  39       | SerDe functionality [2.2 s]
✔ |  20       | partitionBy, groupByKey, reduceByKey etc. [3.9 s]
✔ |   4       | functions in sparkR.R
✔ |  16       | SparkSQL Arrow optimization [19.2 s]
✔ |   6       | test show SparkDataFrame when eager execution is enabled. [1.1 s]
✔ | 1175       | SparkSQL functions [134.8 s]
✔ |  42       | Structured Streaming [478.2 s]
✔ |  16       | tests RDD function take() [1.1 s]
✔ |  14       | the textFile() function [2.9 s]
✔ |  46       | functions in utils.R [0.7 s]
✔ |   0     1 | Windows-specific tests
────────────────────────────────────────────────────────────────────────────────
test_Windows.R:22: skip: sparkJars tag in SparkContext
Reason: This test is only for Windows, skipped
────────────────────────────────────────────────────────────────────────────────

══ Results ═════════════════════════════════════════════════════════════════════
Duration: 987.3 s

OK:       2304
Failed:   0
Warnings: 0
Skipped:  1
...
Status: OK
+ popd
Tests passed.
```

Note that I tested to build SparkR in R 4.0.0, and run the tests with R 3.6.3. It all passed. See also [the comment in the JIRA](https://issues.apache.org/jira/browse/SPARK-31918?focusedCommentId=17142837&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-17142837).

Closes #28907 from HyukjinKwon/SPARK-31918.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 11d2b07b74c73ce6d59ac4f7446f1eb8bc6bbb4b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/utils.R                                |  5 ++++-
 R/pkg/tests/fulltests/test_context.R           |  4 +++-
 .../fulltests/test_mllib_classification.R      | 18 +++++++++---------
 R/pkg/tests/fulltests/test_mllib_clustering.R  |  2 +-
 R/pkg/tests/fulltests/test_mllib_regression.R  |  2 +-
 5 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index 65db9c21d9dbb..cef2fa9b47440 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -529,7 +529,10 @@ processClosure <- function(node, oldEnv, defVars, checkedFuncs, newEnv) {
         # Namespaces other than "SparkR" will not be searched.
         if (!isNamespace(func.env) ||
             (getNamespaceName(func.env) == "SparkR" &&
-               !(nodeChar %in% getNamespaceExports("SparkR")))) {
+               !(nodeChar %in% getNamespaceExports("SparkR")) &&
+                  # Note that generic S4 methods should not be set to the environment of
+                  # cleaned closure. It does not work with R 4.0.0+. See also SPARK-31918.
+                  nodeChar != "" && !methods::isGeneric(nodeChar, func.env))) {
           # Only include SparkR internals.
 
           # Set parameter 'inherits' to FALSE since we do not need to search in
diff --git a/R/pkg/tests/fulltests/test_context.R b/R/pkg/tests/fulltests/test_context.R
index 6be04b321e985..f86872d727a1d 100644
--- a/R/pkg/tests/fulltests/test_context.R
+++ b/R/pkg/tests/fulltests/test_context.R
@@ -26,7 +26,9 @@ test_that("Check masked functions", {
                      "colnames", "colnames<-", "intersect", "rank", "rbind", "sample", "subset",
                      "summary", "transform", "drop", "window", "as.data.frame", "union", "not")
   version <- packageVersion("base")
-  if (as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3) {
+  is33Above <- as.numeric(version$major) >= 3 && as.numeric(version$minor) >= 3
+  is40Above <- as.numeric(version$major) >= 4
+  if (is33Above || is40Above) {
     namesOfMasked <- c("endsWith", "startsWith", namesOfMasked)
   }
   masked <- conflicts(detail = TRUE)$`package:SparkR`
diff --git a/R/pkg/tests/fulltests/test_mllib_classification.R b/R/pkg/tests/fulltests/test_mllib_classification.R
index 2da3a022f8941..2a09c23bc3617 100644
--- a/R/pkg/tests/fulltests/test_mllib_classification.R
+++ b/R/pkg/tests/fulltests/test_mllib_classification.R
@@ -34,7 +34,7 @@ test_that("spark.svmLinear", {
   summary <- summary(model)
 
   # test summary coefficients return matrix type
-  expect_true(class(summary$coefficients) == "matrix")
+  expect_true(any(class(summary$coefficients) == "matrix"))
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   coefs <- summary$coefficients[, "Estimate"]
@@ -130,7 +130,7 @@ test_that("spark.logit", {
   summary <- summary(model)
 
   # test summary coefficients return matrix type
-  expect_true(class(summary$coefficients) == "matrix")
+  expect_true(any(class(summary$coefficients) == "matrix"))
   expect_true(class(summary$coefficients[, 1]) == "numeric")
 
   versicolorCoefsR <- c(1.52, 0.03, -0.53, 0.04, 0.00)
@@ -242,8 +242,8 @@ test_that("spark.logit", {
   # Test binomial logistic regression against two classes with upperBoundsOnCoefficients
   # and upperBoundsOnIntercepts
   u <- matrix(c(1.0, 0.0, 1.0, 0.0), nrow = 1, ncol = 4)
-  model <- spark.logit(training, Species ~ ., upperBoundsOnCoefficients = u,
-                       upperBoundsOnIntercepts = 1.0)
+  model <- suppressWarnings(spark.logit(training, Species ~ ., upperBoundsOnCoefficients = u,
+                                        upperBoundsOnIntercepts = 1.0))
   summary <- summary(model)
   coefsR <- c(-11.13331, 1.00000, 0.00000, 1.00000, 0.00000)
   coefs <- summary$coefficients[, "Estimate"]
@@ -255,8 +255,8 @@ test_that("spark.logit", {
   # Test binomial logistic regression against two classes with lowerBoundsOnCoefficients
   # and lowerBoundsOnIntercepts
   l <- matrix(c(0.0, -1.0, 0.0, -1.0), nrow = 1, ncol = 4)
-  model <- spark.logit(training, Species ~ ., lowerBoundsOnCoefficients = l,
-                       lowerBoundsOnIntercepts = 0.0)
+  model <- suppressWarnings(spark.logit(training, Species ~ ., lowerBoundsOnCoefficients = l,
+                                        lowerBoundsOnIntercepts = 0.0))
   summary <- summary(model)
   coefsR <- c(0, 0, -1, 0, 1.902192)
   coefs <- summary$coefficients[, "Estimate"]
@@ -268,9 +268,9 @@ test_that("spark.logit", {
   # Test multinomial logistic regression with lowerBoundsOnCoefficients
   # and lowerBoundsOnIntercepts
   l <- matrix(c(0.0, -1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -1.0), nrow = 2, ncol = 4)
-  model <- spark.logit(training, Species ~ ., family = "multinomial",
-                       lowerBoundsOnCoefficients = l,
-                       lowerBoundsOnIntercepts = as.array(c(0.0, 0.0)))
+  model <- suppressWarnings(spark.logit(training, Species ~ ., family = "multinomial",
+                                        lowerBoundsOnCoefficients = l,
+                                        lowerBoundsOnIntercepts = as.array(c(0.0, 0.0))))
   summary <- summary(model)
   versicolorCoefsR <- c(42.639465, 7.258104, 14.330814, 16.298243, 11.716429)
   virginicaCoefsR <- c(0.0002970796, 4.79274, 7.65047, 25.72793, 30.0021)
diff --git a/R/pkg/tests/fulltests/test_mllib_clustering.R b/R/pkg/tests/fulltests/test_mllib_clustering.R
index 028ad574b8134..f180aeea28150 100644
--- a/R/pkg/tests/fulltests/test_mllib_clustering.R
+++ b/R/pkg/tests/fulltests/test_mllib_clustering.R
@@ -171,7 +171,7 @@ test_that("spark.kmeans", {
   expect_equal(sort(collect(distinct(select(cluster, "prediction")))$prediction), c(0, 1))
 
   # test summary coefficients return matrix type
-  expect_true(class(summary.model$coefficients) == "matrix")
+  expect_true(any(class(summary.model$coefficients) == "matrix"))
   expect_true(class(summary.model$coefficients[1, ]) == "numeric")
 
   # Test model save/load
diff --git a/R/pkg/tests/fulltests/test_mllib_regression.R b/R/pkg/tests/fulltests/test_mllib_regression.R
index b40c4cb9a9694..929cd27211fdc 100644
--- a/R/pkg/tests/fulltests/test_mllib_regression.R
+++ b/R/pkg/tests/fulltests/test_mllib_regression.R
@@ -116,7 +116,7 @@ test_that("spark.glm summary", {
   rStats <- summary(glm(Sepal.Width ~ Sepal.Length + Species, data = dataset))
 
   # test summary coefficients return matrix type
-  expect_true(class(stats$coefficients) == "matrix")
+  expect_true(any(class(stats$coefficients) == "matrix"))
   expect_true(class(stats$coefficients[, 1]) == "numeric")
 
   coefs <- stats$coefficients

From da8133f8f1df7d2ddfa995974d9a7db06ff4cd5a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 24 Jun 2020 11:05:27 +0900
Subject: [PATCH 0851/1280] [SPARK-32073][R] Drop R < 3.5 support

### What changes were proposed in this pull request?

Spark 3.0 accidentally dropped R < 3.5. It is built by R 3.6.3 which not support R < 3.5:

```
Error in readRDS(pfile) : cannot read workspace version 3 written by R 3.6.3; need R 3.5.0 or newer version.
```

In fact, with SPARK-31918, we will have to drop R < 3.5 entirely to support R 4.0.0. This is inevitable to release on CRAN because they require to make the tests pass with the latest R.

### Why are the changes needed?

To show the supported versions correctly, and support R 4.0.0 to unblock the releases.

### Does this PR introduce _any_ user-facing change?

In fact, no because Spark 3.0.0 already does not work with R < 3.5.
Compared to Spark 2.4, yes. R < 3.5 would not work.

### How was this patch tested?

Jenkins should test it out.

Closes #28908 from HyukjinKwon/SPARK-32073.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b62e2536db9def0d11605ceac8990f72a515e9a0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/WINDOWS.md                 | 4 ++--
 R/pkg/DESCRIPTION            | 2 +-
 R/pkg/inst/profile/general.R | 4 ----
 R/pkg/inst/profile/shell.R   | 4 ----
 docs/index.md                | 3 +--
 5 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/R/WINDOWS.md b/R/WINDOWS.md
index dbc27178bdb8c..9fe4a22bf22b2 100644
--- a/R/WINDOWS.md
+++ b/R/WINDOWS.md
@@ -22,8 +22,8 @@ To build SparkR on Windows, the following steps are required
 
 1. Make sure `bash` is available and in `PATH` if you already have a built-in `bash` on Windows. If you do not have, install [Cygwin](https://www.cygwin.com/).
 
-2. Install R (>= 3.1) and [Rtools](https://cloud.r-project.org/bin/windows/Rtools/). Make sure to
-include Rtools and R in `PATH`. Note that support for R prior to version 3.4 is deprecated as of Spark 3.0.0.
+2. Install R (>= 3.5) and [Rtools](https://cloud.r-project.org/bin/windows/Rtools/). Make sure to
+include Rtools and R in `PATH`.
 
 3. Install JDK that SparkR supports (see `R/pkg/DESCRIPTION`), and set `JAVA_HOME` in the system environment variables.
 
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 21f3eaa5c15f6..86514f237d1e6 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -15,7 +15,7 @@ URL: https://www.apache.org/ https://spark.apache.org/
 BugReports: https://spark.apache.org/contributing.html
 SystemRequirements: Java (>= 8, < 12)
 Depends:
-    R (>= 3.1),
+    R (>= 3.5),
     methods
 Suggests:
     knitr,
diff --git a/R/pkg/inst/profile/general.R b/R/pkg/inst/profile/general.R
index 3efb460846fc2..8c75c19ca7ac3 100644
--- a/R/pkg/inst/profile/general.R
+++ b/R/pkg/inst/profile/general.R
@@ -16,10 +16,6 @@
 #
 
 .First <- function() {
-  if (utils::compareVersion(paste0(R.version$major, ".", R.version$minor), "3.4.0") == -1) {
-    warning("Support for R prior to version 3.4 is deprecated since Spark 3.0.0")
-  }
-
   packageDir <- Sys.getenv("SPARKR_PACKAGE_DIR")
   dirs <- strsplit(packageDir, ",")[[1]]
   .libPaths(c(dirs, .libPaths()))
diff --git a/R/pkg/inst/profile/shell.R b/R/pkg/inst/profile/shell.R
index e4e0d032997de..f6c20e1a5ebc3 100644
--- a/R/pkg/inst/profile/shell.R
+++ b/R/pkg/inst/profile/shell.R
@@ -16,10 +16,6 @@
 #
 
 .First <- function() {
-  if (utils::compareVersion(paste0(R.version$major, ".", R.version$minor), "3.4.0") == -1) {
-    warning("Support for R prior to version 3.4 is deprecated since Spark 3.0.0")
-  }
-
   home <- Sys.getenv("SPARK_HOME")
   .libPaths(c(file.path(home, "R", "lib"), .libPaths()))
   Sys.setenv(NOAWT = 1)
diff --git a/docs/index.md b/docs/index.md
index 38f12dd4db77b..c0771ca170af5 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -44,10 +44,9 @@ source, visit [Building Spark](building-spark.html).
 
 Spark runs on both Windows and UNIX-like systems (e.g. Linux, Mac OS), and it should run on any platform that runs a supported version of Java. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation.
 
-Spark runs on Java 8/11, Scala 2.12, Python 2.7+/3.4+ and R 3.1+.
+Spark runs on Java 8/11, Scala 2.12, Python 2.7+/3.4+ and R 3.5+.
 Java 8 prior to version 8u92 support is deprecated as of Spark 3.0.0.
 Python 2 and Python 3 prior to version 3.6 support is deprecated as of Spark 3.0.0.
-R prior to version 3.4 support is deprecated as of Spark 3.0.0.
 For the Scala API, Spark {{site.SPARK_VERSION}}
 uses Scala {{site.SCALA_BINARY_VERSION}}. You will need to use a compatible Scala version
 ({{site.SCALA_BINARY_VERSION}}.x).

From 25b7d8b6e6e3a11f589badfa4bbbd904b1195307 Mon Sep 17 00:00:00 2001
From: Zhen Li <zhli@microsoft.com>
Date: Tue, 23 Jun 2020 21:43:02 -0500
Subject: [PATCH 0852/1280] [SPARK-32028][WEBUI] fix app id link for multi
 attempts app in history summary page

### What changes were proposed in this pull request?

Fix app id link for multi attempts application in history summary page
If attempt id is available (yarn), app id link url will contain correct attempt id, like `/history/application_1561589317410_0002/1/jobs/`.
If attempt id is not available (standalone), app id link url will not contain fake attempt id, like `/history/app-20190404053606-0000/jobs/`.

### Why are the changes needed?

This PR is for fixing [32028](https://issues.apache.org/jira/browse/SPARK-32028). App id link use application attempt count as attempt id. this would cause link url wrong for below cases:
1. there are multi attempts, all links point to last attempt
![multi_same](https://user-images.githubusercontent.com/10524738/85098505-c45c5500-b1af-11ea-8912-fa5fd72ce064.JPG)

2. if there is one attempt, but attempt id is not 1 (before attempt maybe crash or fail to gerenerate event file). link url points to worng attempt (1) here.
![wrong_attemptJPG](https://user-images.githubusercontent.com/10524738/85098513-c9b99f80-b1af-11ea-8cbc-fd7f745c1080.JPG)

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Tested this manually.

Closes #28867 from zhli1142015/fix-appid-link-in-history-page.

Authored-by: Zhen Li <zhli@microsoft.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit eedc6cc37df9b32995f41bd0e1779101ba1df1b8)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../org/apache/spark/ui/static/historypage-template.html    | 6 +++---
 .../resources/org/apache/spark/ui/static/historypage.js     | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
index 33eb7bfa671bc..7e9927d40c191 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html
@@ -77,12 +77,12 @@
   <tbody>
   {{#applications}}
     <tr>
+      {{#attempts}}
       <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{version}}</td>
-      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td>
+      <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}{{#hasAttemptId}}/{{attemptId}}{{/hasAttemptId}}/jobs/">{{id}}</a></span></td>
       <td {{#hasMultipleAttempts}}style="background-color:#fff"{{/hasMultipleAttempts}}>{{name}}</td>
-      {{#attempts}}
       {{#hasMultipleAttempts}}
-      <td><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td>
+      <td><a href="{{uiroot}}/history/{{id}}{{#hasAttemptId}}/{{attemptId}}{{/hasAttemptId}}/jobs/">{{attemptId}}</a></td>
       {{/hasMultipleAttempts}}
       <td>{{startTime}}</td>
       {{#showCompletedColumns}}
diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index 4df5f07f077d7..3a4c815029f6d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -130,7 +130,7 @@ $(document).ready(function() {
         if (app["attempts"].length > 1) {
             hasMultipleAttempts = true;
         }
-        var num = app["attempts"].length;
+
         for (var j in app["attempts"]) {
           var attempt = app["attempts"][j];
           attempt["startTime"] = formatTimeMillis(attempt["startTimeEpoch"]);
@@ -140,7 +140,8 @@ $(document).ready(function() {
             (attempt.hasOwnProperty("attemptId") ? attempt["attemptId"] + "/" : "") + "logs";
           attempt["durationMillisec"] = attempt["duration"];
           attempt["duration"] = formatDuration(attempt["duration"]);
-          var app_clone = {"id" : id, "name" : name, "version": version, "num" : num, "attempts" : [attempt]};
+          var hasAttemptId = attempt.hasOwnProperty("attemptId");
+          var app_clone = {"id" : id, "name" : name, "version": version, "hasAttemptId" : hasAttemptId, "attempts" : [attempt]};
           array.push(app_clone);
         }
       }

From a36140c3c300beaf50d19381ac72e2524f888e53 Mon Sep 17 00:00:00 2001
From: sidedoorleftroad <sidedoorleftroad@163.com>
Date: Wed, 24 Jun 2020 13:39:55 +0900
Subject: [PATCH 0853/1280] [SPARK-32075][DOCS] Fix a few issues in parameters
 table

### What changes were proposed in this pull request?

Fix a few issues in parameters table in structured-streaming-kafka-integration doc.

### Why are the changes needed?

Make the title of the table consistent with the data.

### Does this PR introduce _any_ user-facing change?

Yes.

Before:
![image](https://user-images.githubusercontent.com/67275816/85414316-8475e300-b59e-11ea-84ec-fa78ecc980b3.png)
After:
![image](https://user-images.githubusercontent.com/67275816/85414562-d61e6d80-b59e-11ea-9fe6-247e0ad4d9ee.png)

Before:
![image](https://user-images.githubusercontent.com/67275816/85414467-b8510880-b59e-11ea-92a0-7205542fe28b.png)
After:
![image](https://user-images.githubusercontent.com/67275816/85414589-de76a880-b59e-11ea-91f2-5073eaf3444b.png)

Before:
![image](https://user-images.githubusercontent.com/67275816/85414502-c69f2480-b59e-11ea-837f-1201f10a56b6.png)
After:
![image](https://user-images.githubusercontent.com/67275816/85414615-e9313d80-b59e-11ea-9b1a-fc11da0b6bc5.png)

### How was this patch tested?

Manually build and check.

Closes #28910 from sidedoorleftroad/SPARK-32075.

Authored-by: sidedoorleftroad <sidedoorleftroad@163.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 986fa01747db4b52bb8ca1165e759ca2d46d26ff)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/structured-streaming-kafka-integration.md | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md
index 016faa735acd6..8dc2a738645ce 100644
--- a/docs/structured-streaming-kafka-integration.md
+++ b/docs/structured-streaming-kafka-integration.md
@@ -528,28 +528,28 @@ The following properties are available to configure the consumer pool:
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.cache.capacity</td>
-  <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
   <td>64</td>
+  <td>The maximum number of consumers cached. Please note that it's a soft limit.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.timeout</td>
-  <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>The minimum amount of time a consumer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for consumer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.cache.jmx.enable</td>
+  <td>false</td>
   <td>Enable or disable JMX for pools created with this configuration instance. Statistics of the pool are available via JMX instance.
   The prefix of JMX name is set to "kafka010-cached-simple-kafka-consumer-pool".
   </td>
-  <td>false</td>
   <td>3.0.0</td>
 </tr>
 </table>
@@ -578,14 +578,14 @@ The following properties are available to configure the fetched data pool:
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.timeout</td>
-  <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>5m (5 minutes)</td>
+  <td>The minimum amount of time a fetched data may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>3.0.0</td>
 </tr>
 <tr>
   <td>spark.kafka.consumer.fetchedData.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for fetched data pool. When non-positive, no idle evictor thread will be run.</td>
   <td>3.0.0</td>
 </tr>
 </table>
@@ -825,14 +825,14 @@ The following properties are available to configure the producer pool:
 <tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr>
 <tr>
   <td>spark.kafka.producer.cache.timeout</td>
-  <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>10m (10 minutes)</td>
+  <td>The minimum amount of time a producer may sit idle in the pool before it is eligible for eviction by the evictor.</td>
   <td>2.2.1</td>
 </tr>
 <tr>
   <td>spark.kafka.producer.cache.evictorThreadRunInterval</td>
-  <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>1m (1 minute)</td>
+  <td>The interval of time between runs of the idle evictor thread for producer pool. When non-positive, no idle evictor thread will be run.</td>
   <td>3.0.0</td>
 </tr>
 </table>

From abcec13e509d7b6e2f2f249e19f01214a7d286a9 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Wed, 24 Jun 2020 22:13:54 +0900
Subject: [PATCH 0854/1280] [SPARK-32080][SPARK-31998][SQL] Simplify
 ArrowColumnVector ListArray accessor

### What changes were proposed in this pull request?

This change simplifies the ArrowColumnVector ListArray accessor to use provided Arrow APIs available in v0.15.0 to calculate element indices.

### Why are the changes needed?

This simplifies the code by avoiding manual calculations on the Arrow offset buffer and makes use of more stable APIs.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Existing tests

Closes #28915 from BryanCutler/arrow-simplify-ArrowColumnVector-ListArray-SPARK-32080.

Authored-by: Bryan Cutler <cutlerb@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit df04107934241965199bd5454c62e1016bb3bdd9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/vectorized/ArrowColumnVector.java | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
index d2220dc2668f9..72fccd451ca12 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ArrowColumnVector.java
@@ -17,7 +17,6 @@
 
 package org.apache.spark.sql.vectorized;
 
-import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.vector.*;
 import org.apache.arrow.vector.complex.*;
 import org.apache.arrow.vector.holders.NullableVarCharHolder;
@@ -458,10 +457,8 @@ final boolean isNullAt(int rowId) {
 
     @Override
     final ColumnarArray getArray(int rowId) {
-      ArrowBuf offsets = accessor.getOffsetBuffer();
-      int index = rowId * ListVector.OFFSET_WIDTH;
-      int start = offsets.getInt(index);
-      int end = offsets.getInt(index + ListVector.OFFSET_WIDTH);
+      int start = accessor.getElementStartIndex(rowId);
+      int end = accessor.getElementEndIndex(rowId);
       return new ColumnarArray(arrayData, start, end - start);
     }
   }

From 77bd69ec71fe894a9c6873ba341b20fcd2b89818 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 24 Jun 2020 14:56:55 +0000
Subject: [PATCH 0855/1280] [SPARK-32089][R][BUILD] Upgrade R version to 4.0.2
 in the release DockerFiile

### What changes were proposed in this pull request?

This PR proposes to upgrade R version to 4.0.2 in the release docker image. As of SPARK-31918, we should make a release with R 4.0.0+ which works with R 3.5+ too.

### Why are the changes needed?

To unblock releases on CRAN.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Manually tested via scripts under `dev/create-release`, manually attaching to the container and checking the R version.

Closes #28922 from HyukjinKwon/SPARK-32089.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 71b6d462fbeebf5e7e9a95896f0dca8297d0b8dd)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 dev/create-release/spark-rm/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 540dc90f42817..44d602415b262 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -21,7 +21,7 @@
 # * Java 8
 # * Ivy
 # * Python (2.7.15/3.6.7)
-# * R-base/R-base-dev (3.6.1)
+# * R-base/R-base-dev (4.0.2)
 # * Ruby 2.3 build utilities
 
 FROM ubuntu:18.04
@@ -43,7 +43,7 @@ ARG GEM_PKGS="jekyll:4.0.0 jekyll-redirect-from:0.16.0 rouge:3.15.0"
 # This is all in a single "RUN" command so that if anything changes, "apt update" is run to fetch
 # the most current package versions (instead of potentially using old versions cached by docker).
 RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
-  echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' >> /etc/apt/sources.list && \
+  echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list && \
   gpg --keyserver keyserver.ubuntu.com --recv-key E298A3A825C0D65DFD57CBB651716619E084DAB9 && \
   gpg -a --export E084DAB9 | apt-key add - && \
   apt-get clean && \

From 13231ff3640bddd5d8994cf690ccadaf38012224 Mon Sep 17 00:00:00 2001
From: gatorsmile <gatorsmile@gmail.com>
Date: Wed, 24 Jun 2020 11:00:20 -0700
Subject: [PATCH 0856/1280] [SPARK-32078][DOC] Add a redirect to sql-ref from
 sql-reference
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
This PR is to add a redirect to sql-ref.html.

### Why are the changes needed?
Before Spark 3.0 release, we are using sql-reference.md, which was replaced by sql-ref.md instead. A number of Google searches I’ve done today have turned up https://spark.apache.org/docs/latest/sql-reference.html, which does not exist any more. Thus, we should add a redirect to sql-ref.html.

### Does this PR introduce _any_ user-facing change?

https://spark.apache.org/docs/latest/sql-reference.html will be redirected to https://spark.apache.org/docs/latest/sql-ref.html

### How was this patch tested?

Build it in my local environment. It works well. The sql-reference.html file was generated. The contents are like:

```
<!DOCTYPE html>
<html lang="en-US">
  <meta charset="utf-8">
  <title>Redirecting&hellip;</title>
  <link rel="canonical" href="http://localhost:4000/sql-ref.html">
  <script>location="http://localhost:4000/sql-ref.html"</script>
  <meta http-equiv="refresh" content="0; url=http://localhost:4000/sql-ref.html">
  <meta name="robots" content="noindex">
  <h1>Redirecting&hellip;</h1>
  <a href="http://localhost:4000/sql-ref.html">Click here if you are not redirected.</a>
</html>
```

Closes #28914 from gatorsmile/addRedirectSQLRef.

Authored-by: gatorsmile <gatorsmile@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit d06604f60a8a2ba0877616370a20aa18be15f8c4)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/_config.yml | 3 +++
 docs/sql-ref.md  | 1 +
 2 files changed, 4 insertions(+)

diff --git a/docs/_config.yml b/docs/_config.yml
index db2f6721e0cc4..5f2970fb645bc 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -8,6 +8,9 @@ gems:
 kramdown:
   entity_output: numeric
 
+plugins:
+  - jekyll-redirect-from
+
 include:
   - _static
   - _modules
diff --git a/docs/sql-ref.md b/docs/sql-ref.md
index f88026b7abf02..8d0c6734c4bd6 100644
--- a/docs/sql-ref.md
+++ b/docs/sql-ref.md
@@ -1,6 +1,7 @@
 ---
 layout: global
 title: SQL Reference
+redirect_from: /sql-reference
 displayTitle: SQL Reference
 license: |
   Licensed to the Apache Software Foundation (ASF) under one or more

From 2d6232aac476e8afe323fd56d461fadaf2550bb1 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 25 Jun 2020 11:04:47 -0700
Subject: [PATCH 0857/1280] [SPARK-32098][PYTHON] Use iloc for positional
 slicing instead of direct slicing in createDataFrame with Arrow

When you use floats are index of pandas, it creates a Spark DataFrame with a wrong results as below when Arrow is enabled:

```bash
./bin/pyspark --conf spark.sql.execution.arrow.pyspark.enabled=true
```

```python
>>> import pandas as pd
>>> spark.createDataFrame(pd.DataFrame({'a': [1,2,3]}, index=[2., 3., 4.])).show()
+---+
|  a|
+---+
|  1|
|  1|
|  2|
+---+
```

This is because direct slicing uses the value as index when the index contains floats:

```python
>>> pd.DataFrame({'a': [1,2,3]}, index=[2., 3., 4.])[2:]
     a
2.0  1
3.0  2
4.0  3
>>> pd.DataFrame({'a': [1,2,3]}, index=[2., 3., 4.]).iloc[2:]
     a
4.0  3
>>> pd.DataFrame({'a': [1,2,3]}, index=[2, 3, 4])[2:]
   a
4  3
```

This PR proposes to explicitly use `iloc` to positionally slide when we create a DataFrame from a pandas DataFrame with Arrow enabled.

FWIW, I was trying to investigate why direct slicing refers the index value or the positional index sometimes but I stopped investigating further after reading this https://pandas.pydata.org/pandas-docs/stable/getting_started/10min.html#selection

> While standard Python / Numpy expressions for selecting and setting are intuitive and come in handy for interactive work, for production code, we recommend the optimized pandas data access methods, `.at`, `.iat`, `.loc` and `.iloc`.

To create the correct Spark DataFrame from a pandas DataFrame without a data loss.

Yes, it is a bug fix.

```bash
./bin/pyspark --conf spark.sql.execution.arrow.pyspark.enabled=true
```
```python
import pandas as pd
spark.createDataFrame(pd.DataFrame({'a': [1,2,3]}, index=[2., 3., 4.])).show()
```

Before:

```
+---+
|  a|
+---+
|  1|
|  1|
|  2|
+---+
```

After:

```
+---+
|  a|
+---+
|  1|
|  2|
|  3|
+---+
```

Manually tested and unittest were added.

Closes #28928 from HyukjinKwon/SPARK-32098.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Bryan Cutler <cutlerb@gmail.com>
(cherry picked from commit 1af19a7b6836f87a3b34189a8a13b6d21d3a37d8)
Signed-off-by: Bryan Cutler <cutlerb@gmail.com>
---
 python/pyspark/sql/pandas/conversion.py | 2 +-
 python/pyspark/sql/tests/test_arrow.py  | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/python/pyspark/sql/pandas/conversion.py b/python/pyspark/sql/pandas/conversion.py
index 251625ae412cf..e6d8e9f24a557 100644
--- a/python/pyspark/sql/pandas/conversion.py
+++ b/python/pyspark/sql/pandas/conversion.py
@@ -413,7 +413,7 @@ def _create_from_pandas_with_arrow(self, pdf, schema, timezone):
 
         # Slice the DataFrame to be batched
         step = -(-len(pdf) // self.sparkContext.defaultParallelism)  # round int up
-        pdf_slices = (pdf[start:start + step] for start in xrange(0, len(pdf), step))
+        pdf_slices = (pdf.iloc[start:start + step] for start in xrange(0, len(pdf), step))
 
         # Create list of Arrow (columns, type) for serializer dump_stream
         arrow_data = [[(c, t) for (_, c), t in zip(pdf_slice.iteritems(), arrow_types)]
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 004c79f290213..1386f8d9948c3 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -415,6 +415,12 @@ def run_test(num_records, num_parts, max_records, use_delay=False):
         for case in cases:
             run_test(*case)
 
+    def test_createDataFrame_with_float_index(self):
+        # SPARK-32098: float index should not produce duplicated or truncated Spark DataFrame
+        self.assertEqual(
+            self.spark.createDataFrame(
+                pd.DataFrame({'a': [1, 2, 3]}, index=[2., 3., 4.])).distinct().count(), 3)
+
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,

From ec06550a89ce671079b266d4275cab8a87e88504 Mon Sep 17 00:00:00 2001
From: Guy Khazma <guykhag@gmail.com>
Date: Fri, 26 Jun 2020 19:12:42 -0700
Subject: [PATCH 0858/1280] [SPARK-32099][DOCS] Remove broken link in cloud
 integration documentation

### What changes were proposed in this pull request?

The 3rd link in `IBM Cloud Object Storage connector for Apache Spark` is broken. The PR removes this link.

### Why are the changes needed?

broken link

### Does this PR introduce _any_ user-facing change?

yes, the broken link is removed from the doc.

### How was this patch tested?

doc generation passes successfully as before

Closes #28927 from guykhazma/spark32099.

Authored-by: Guy Khazma <guykhag@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 44aecaa9124fb2158f009771022c64ede4b582dc)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/cloud-integration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/cloud-integration.md b/docs/cloud-integration.md
index b2a3e77f1ee9d..01f437f38ef17 100644
--- a/docs/cloud-integration.md
+++ b/docs/cloud-integration.md
@@ -257,5 +257,5 @@ Here is the documentation on the standard connectors both from Apache and the cl
 * [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon
 * [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/hadoop/google-cloud-storage-connector). From Google
 * [The Azure Blob Filesystem driver (ABFS)](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-abfs-driver)
-* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage), [how-to-use-connector](https://developer.ibm.com/code/2018/08/16/installing-running-stocator-apache-spark-ibm-cloud-object-storage). From IBM
+* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage). From IBM
 

From 65be3b41bbc4f078050c6c3bf0bfbf4f93d26480 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sun, 28 Jun 2020 12:22:44 -0700
Subject: [PATCH 0859/1280] [SPARK-32115][SQL] Fix SUBSTRING to handle integer
 overflows

### What changes were proposed in this pull request?
Bug fix for overflow case in `UTF8String.substringSQL`.

### Why are the changes needed?
SQL query `SELECT SUBSTRING("abc", -1207959552, -1207959552)` incorrectly returns` "abc"` against expected output of `""`. For query `SUBSTRING("abc", -100, -100)`, we'll get the right output of `""`.

### Does this PR introduce _any_ user-facing change?
Yes, bug fix for the overflow case.

### How was this patch tested?
New UT.

Closes #28937 from xuanyuanking/SPARK-32115.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6484c14c57434dd6961cf9e9e73bbe8aa04cda15)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/unsafe/types/UTF8String.java     | 11 ++++++++++-
 .../apache/spark/unsafe/types/UTF8StringSuite.java    |  4 ++++
 .../catalyst/expressions/StringExpressionsSuite.scala |  4 ++++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 186597fa64780..7205293aa48c5 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -341,8 +341,17 @@ public UTF8String substringSQL(int pos, int length) {
     // to the -ith element before the end of the sequence. If a start index i is 0, it
     // refers to the first element.
     int len = numChars();
+    // `len + pos` does not overflow as `len >= 0`.
     int start = (pos > 0) ? pos -1 : ((pos < 0) ? len + pos : 0);
-    int end = (length == Integer.MAX_VALUE) ? len : start + length;
+
+    int end;
+    if ((long) start + length > Integer.MAX_VALUE) {
+      end = Integer.MAX_VALUE;
+    } else if ((long) start + length < Integer.MIN_VALUE) {
+      end = Integer.MIN_VALUE;
+    } else {
+      end = start + length;
+    }
     return substring(start, end);
   }
 
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
index 8f933877f82e6..70e276f7e5a8b 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
@@ -390,6 +390,10 @@ public void substringSQL() {
     assertEquals(fromString("example"), e.substringSQL(0, Integer.MAX_VALUE));
     assertEquals(fromString("example"), e.substringSQL(1, Integer.MAX_VALUE));
     assertEquals(fromString("xample"), e.substringSQL(2, Integer.MAX_VALUE));
+    assertEquals(EMPTY_UTF8, e.substringSQL(-100, -100));
+    assertEquals(EMPTY_UTF8, e.substringSQL(-1207959552, -1207959552));
+    assertEquals(fromString("pl"), e.substringSQL(-3, 2));
+    assertEquals(EMPTY_UTF8, e.substringSQL(Integer.MIN_VALUE, 6));
   }
 
   @Test
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index f18364d844ce1..967ccc42c632d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -236,6 +236,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       "xample",
       row)
 
+    // Substring with from negative position with negative length
+    checkEvaluation(Substring(s, Literal.create(-1207959552, IntegerType),
+      Literal.create(-1207959552, IntegerType)), "", row)
+
     val s_notNull = 'a.string.notNull.at(0)
 
     assert(Substring(s, Literal.create(0, IntegerType), Literal.create(2, IntegerType)).nullable)

From fb8574a468be62cccd8b3598134d96bb4f217abb Mon Sep 17 00:00:00 2001
From: Warren Zhu <zhonzh@microsoft.com>
Date: Sun, 28 Jun 2020 21:06:45 -0700
Subject: [PATCH 0860/1280] [SPARK-32124][CORE][SHS] Fix taskEndReasonFromJson
 to handle event logs from old Spark versions

### What changes were proposed in this pull request?
Fix bug of exception when parse event log of fetch failed task end reason without `Map Index`

### Why are the changes needed?
When Spark history server read event log produced by older version of spark 2.4 (which don't have `Map Index` field), parsing of TaskEndReason will fail. This will cause TaskEnd event being ignored.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
JsonProtocolSuite.test("FetchFailed Map Index backwards compatibility")

Closes #28941 from warrenzhu25/shs-task.

Authored-by: Warren Zhu <zhonzh@microsoft.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 197ac3b13239d50a1f34a5860940e353ca6b99d5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/util/JsonProtocol.scala    |  5 ++++-
 .../org/apache/spark/util/JsonProtocolSuite.scala     | 11 +++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index d53ca0f5ec6aa..74ff5c7dd539e 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -993,7 +993,10 @@ private[spark] object JsonProtocol {
         val blockManagerAddress = blockManagerIdFromJson(json \ "Block Manager Address")
         val shuffleId = (json \ "Shuffle ID").extract[Int]
         val mapId = (json \ "Map ID").extract[Long]
-        val mapIndex = (json \ "Map Index").extract[Int]
+        val mapIndex = (json \ "Map Index") match {
+          case JNothing => 0
+          case x => x.extract[Int]
+        }
         val reduceId = (json \ "Reduce ID").extract[Int]
         val message = jsonOption(json \ "Message").map(_.extract[String])
         new FetchFailed(blockManagerAddress, shuffleId, mapId, mapIndex, reduceId,
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 5f1c75319faf8..98aaa9e8737f3 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -305,6 +305,17 @@ class JsonProtocolSuite extends SparkFunSuite {
     assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
   }
 
+  test("SPARK-32124: FetchFailed Map Index backwards compatibility") {
+    // FetchFailed in Spark 2.4.0 does not have "Map Index" property.
+    val fetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L, 18, 19,
+      "ignored")
+    val oldEvent = JsonProtocol.taskEndReasonToJson(fetchFailed)
+      .removeField({ _._1 == "Map Index" })
+    val expectedFetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L,
+      0, 19, "ignored")
+    assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
+  }
+
   test("ShuffleReadMetrics: Local bytes read backwards compatibility") {
     // Metrics about local shuffle bytes read were added in 1.3.1.
     val metrics = makeTaskMetrics(1L, 2L, 3L, 4L, 5, 6,

From 51b516a62488f1740c3e4ef337db6703cbc36382 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Sun, 28 Jun 2020 21:35:59 -0700
Subject: [PATCH 0861/1280] [SPARK-32126][SS] Scope Session.active in
 IncrementalExecution

### What changes were proposed in this pull request?

The `optimizedPlan` in IncrementalExecution should also be scoped in `withActive`.

### Why are the changes needed?

Follow-up of SPARK-30798 for the Streaming side.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Existing UT.

Closes #28936 from xuanyuanking/SPARK-30798-follow.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit f944603872284c03c557474bb9e816f20094a630)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/sql/execution/QueryExecution.scala   | 2 +-
 .../spark/sql/execution/streaming/IncrementalExecution.scala    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 1df812d1aa809..50c9c31029d90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -129,7 +129,7 @@ class QueryExecution(
       Option(InsertAdaptiveSparkPlan(AdaptiveExecutionContext(sparkSession, this))))
   }
 
-  private def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
+  protected def executePhase[T](phase: String)(block: => T): T = sparkSession.withActive {
     tracker.measurePhase(phase)(block)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
index 09ae7692ec518..7773ac71c4954 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
@@ -76,7 +76,7 @@ class IncrementalExecution(
    * with the desired literal
    */
   override
-  lazy val optimizedPlan: LogicalPlan = tracker.measurePhase(QueryPlanningTracker.OPTIMIZATION) {
+  lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     sparkSession.sessionState.optimizer.executeAndTrack(withCachedData,
       tracker) transformAllExpressions {
       case ts @ CurrentBatchTimestamp(timestamp, _, _) =>

From a00470d583d1f53a7f5f440be2b0a5fefd35fcec Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Sun, 28 Jun 2020 21:55:19 -0700
Subject: [PATCH 0862/1280] [SPARK-32038][SQL][FOLLOWUP] Make the alias name
 pretty after float/double normalization

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/28876/files

This PR proposes to use the name of the original expression, as the alias name of the normalization expression.

### Why are the changes needed?

make the query plan looks pretty when EXPLAIN.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

manually explain the query

Closes #28919 from cloud-fan/follow.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 835ef425d03f30984e885448fe785905ed1ee9a7)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/sql/execution/SparkStrategies.scala | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 10cd7b6222c50..a983bc850057a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -527,6 +527,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         val normalizedGroupingExpressions = groupingExpressions.map { e =>
           NormalizeFloatingNumbers.normalize(e) match {
             case n: NamedExpression => n
+            // Keep the name of the original expression.
             case other => Alias(other, e.name)(exprId = e.exprId)
           }
         }
@@ -551,7 +552,13 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               // because `distinctExpressions` is not extracted during logical phase.
               NormalizeFloatingNumbers.normalize(e) match {
                 case ne: NamedExpression => ne
-                case other => Alias(other, other.toString)()
+                case other =>
+                  // Keep the name of the original expression.
+                  val name = e match {
+                    case ne: NamedExpression => ne.name
+                    case _ => e.toString
+                  }
+                  Alias(other, name)()
               }
             }
 

From 503e56aec53c73d7673a79d2e150ac288e44ce7e Mon Sep 17 00:00:00 2001
From: xy_xin <xianyin.xxy@alibaba-inc.com>
Date: Mon, 29 Jun 2020 13:16:38 +0000
Subject: [PATCH 0863/1280] [SPARK-32127][SQL] Check rules for MERGE INTO
 should use MergeAction.conditition other than MergeAction.children

### What changes were proposed in this pull request?

This pr fix a bug of check rules for MERGE INTO.

### Why are the changes needed?

SPARK-30924 adds some check rules for MERGE INTO one of which ensures the first MATCHED clause must have a condition. However, it uses `MergeAction.children` in the checking which is not accurate for the case, and it lets the below case pass the check:

```
MERGE INTO testcat1.ns1.ns2.tbl AS target
xxx
WHEN MATCHED THEN UPDATE SET target.col2 = source.col2
WHEN MATCHED THEN DELETE
xxx
```

We should use `MergeAction.condition` instead.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

An existed ut is modified.

Closes #28943 from xianyinxin/SPARK-32127.

Authored-by: xy_xin <xianyin.xxy@alibaba-inc.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/catalyst/parser/AstBuilder.scala  |  2 +-
 .../spark/sql/catalyst/plans/logical/v2Commands.scala  | 10 +++++-----
 .../spark/sql/catalyst/parser/DDLParserSuite.scala     |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index f6877d2ed62f3..5f1bc329ccd06 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -468,7 +468,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
       throw new ParseException("There must be at least one WHEN clause in a MERGE statement", ctx)
     }
     // children being empty means that the condition is not set
-    if (matchedActions.length == 2 && matchedActions.head.children.isEmpty) {
+    if (matchedActions.length == 2 && matchedActions.head.condition.isEmpty) {
       throw new ParseException("When there are 2 MATCHED clauses in a MERGE statement, " +
         "the first MATCHED clause must have a condition", ctx)
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 579157a6f2f2e..b4120d9f64cc5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -346,25 +346,25 @@ case class MergeIntoTable(
   override def children: Seq[LogicalPlan] = Seq(targetTable, sourceTable)
 }
 
-sealed abstract class MergeAction(
-    condition: Option[Expression]) extends Expression with Unevaluable {
+sealed abstract class MergeAction extends Expression with Unevaluable {
+  def condition: Option[Expression]
   override def foldable: Boolean = false
   override def nullable: Boolean = false
   override def dataType: DataType = throw new UnresolvedException(this, "nullable")
   override def children: Seq[Expression] = condition.toSeq
 }
 
-case class DeleteAction(condition: Option[Expression]) extends MergeAction(condition)
+case class DeleteAction(condition: Option[Expression]) extends MergeAction
 
 case class UpdateAction(
     condition: Option[Expression],
-    assignments: Seq[Assignment]) extends MergeAction(condition) {
+    assignments: Seq[Assignment]) extends MergeAction {
   override def children: Seq[Expression] = condition.toSeq ++ assignments
 }
 
 case class InsertAction(
     condition: Option[Expression],
-    assignments: Seq[Assignment]) extends MergeAction(condition) {
+    assignments: Seq[Assignment]) extends MergeAction {
   override def children: Seq[Expression] = condition.toSeq ++ assignments
 }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 20b98550b2c3e..55c3880f54ef5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -1132,8 +1132,8 @@ class DDLParserSuite extends AnalysisTest {
           |MERGE INTO testcat1.ns1.ns2.tbl AS target
           |USING testcat2.ns1.ns2.tbl AS source
           |ON target.col1 = source.col1
-          |WHEN MATCHED THEN DELETE
           |WHEN MATCHED THEN UPDATE SET target.col2 = source.col2
+          |WHEN MATCHED THEN DELETE
           |WHEN NOT MATCHED AND (target.col2='insert')
           |THEN INSERT (target.col1, target.col2) values (source.col1, source.col2)
         """.stripMargin)

From 22e34336da50220073d83768903726e619489942 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 30 Jun 2020 08:09:18 +0000
Subject: [PATCH 0864/1280] [SPARK-29999][SS][FOLLOWUP] Fix test to check the
 actual metadata log directory

### What changes were proposed in this pull request?

This patch fixes the missed spot - the test initializes FileStreamSinkLog with its "output" directory instead of "metadata" directory, hence the verification against sink log was no-op.

### Why are the changes needed?

Without the fix, the verification against sink log was no-op.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Checked with debugger in test, and verified `allFiles()` returns non-zero entries. (It returned zero entry, as there's no metadata.)

Closes #28930 from HeartSaVioR/SPARK-29999-FOLLOWUP-fix-test.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 5472170a2b35864c617bdb846ff7123533765a16)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../execution/streaming/FileStreamSink.scala  | 19 +++++++++++--------
 .../sql/streaming/FileStreamSinkSuite.scala   | 10 ++++++----
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
index b679f163fc561..86a31942f3f94 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/FileStreamSink.scala
@@ -45,8 +45,7 @@ object FileStreamSink extends Logging {
         val hdfsPath = new Path(singlePath)
         val fs = hdfsPath.getFileSystem(hadoopConf)
         if (fs.isDirectory(hdfsPath)) {
-          val metadataPath = new Path(hdfsPath, metadataDir)
-          checkEscapedMetadataPath(fs, metadataPath, sqlConf)
+          val metadataPath = getMetadataLogPath(fs, hdfsPath, sqlConf)
           fs.exists(metadataPath)
         } else {
           false
@@ -55,6 +54,12 @@ object FileStreamSink extends Logging {
     }
   }
 
+  def getMetadataLogPath(fs: FileSystem, path: Path, sqlConf: SQLConf): Path = {
+    val metadataDir = new Path(path, FileStreamSink.metadataDir)
+    FileStreamSink.checkEscapedMetadataPath(fs, metadataDir, sqlConf)
+    metadataDir
+  }
+
   def checkEscapedMetadataPath(fs: FileSystem, metadataPath: Path, sqlConf: SQLConf): Unit = {
     if (sqlConf.getConf(SQLConf.STREAMING_CHECKPOINT_ESCAPED_PATH_CHECK_ENABLED)
         && StreamExecution.containsSpecialCharsInPath(metadataPath)) {
@@ -125,14 +130,12 @@ class FileStreamSink(
     partitionColumnNames: Seq[String],
     options: Map[String, String]) extends Sink with Logging {
 
+  import FileStreamSink._
+
   private val hadoopConf = sparkSession.sessionState.newHadoopConf()
   private val basePath = new Path(path)
-  private val logPath = {
-    val metadataDir = new Path(basePath, FileStreamSink.metadataDir)
-    val fs = metadataDir.getFileSystem(hadoopConf)
-    FileStreamSink.checkEscapedMetadataPath(fs, metadataDir, sparkSession.sessionState.conf)
-    metadataDir
-  }
+  private val logPath = getMetadataLogPath(basePath.getFileSystem(hadoopConf), basePath,
+    sparkSession.sessionState.conf)
   private val fileLog =
     new FileStreamSinkLog(FileStreamSinkLog.VERSION, sparkSession, logPath.toString)
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
index 877965100f018..aa2664c1e5393 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/FileStreamSinkSuite.scala
@@ -555,10 +555,12 @@ abstract class FileStreamSinkSuite extends StreamTest {
             }
           }
 
-          val fs = new Path(outputDir.getCanonicalPath).getFileSystem(
-            spark.sessionState.newHadoopConf())
-          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark,
-            outputDir.getCanonicalPath)
+          val outputDirPath = new Path(outputDir.getCanonicalPath)
+          val hadoopConf = spark.sessionState.newHadoopConf()
+          val fs = outputDirPath.getFileSystem(hadoopConf)
+          val logPath = FileStreamSink.getMetadataLogPath(fs, outputDirPath, conf)
+
+          val sinkLog = new FileStreamSinkLog(FileStreamSinkLog.VERSION, spark, logPath.toString)
 
           val allFiles = sinkLog.allFiles()
           // only files from non-empty partition should be logged

From 7d1b6b148bd6dc84b214e471707e20569258a3d7 Mon Sep 17 00:00:00 2001
From: TJX2014 <xiaoxingstack@gmail.com>
Date: Tue, 30 Jun 2020 08:56:59 -0500
Subject: [PATCH 0865/1280] [SPARK-32068][WEBUI] Correct task lauchtime show
 issue due to timezone in stage tab
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
`formatDate` in utils.js `org/apache/spark/ui/static/utils.js` is partly refactored.

### Why are the changes needed?
In branch-2.4，task launch time is returned as html string from driver,
while in branch-3.x，this is returned in JSON Object as`Date`type  from `org.apache.spark.status.api.v1.TaskData`
Due to：
LaunchTime from jersey server in spark driver is correct, which will be converted to date string like `2020-06-28T02:57:42.605GMT` in json object, then the formatDate in utils.js treat it as date.split(".")[0].replace("T", " ").
So `2020-06-28T02:57:42.605GMT` will be converted to `2020-06-28 02:57:42`, but correct is `2020-06-28 10:57:42` in GMT+8 timezone.
![选区_071](https://user-images.githubusercontent.com/7149304/85937186-b6d36780-b933-11ea-8382-80a3891f1c2a.png)
![选区_070](https://user-images.githubusercontent.com/7149304/85937190-bcc94880-b933-11ea-8860-2083c97ea269.png)

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manual test.

Closes #28918 from TJX2014/master-SPARK-32068-ui-task-lauch-time-tz.

Authored-by: TJX2014 <xiaoxingstack@gmail.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit 165c948e3297fd5ebcee771fcbf7a0c20fccfaae)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../resources/org/apache/spark/ui/static/utils.js     | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js
index 6fc34a9e1f7ea..2e46111bf1ba0 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/utils.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js
@@ -56,13 +56,17 @@ function formatTimeMillis(timeMillis) {
     return "-";
   } else {
     var dt = new Date(timeMillis);
+    return formatDateString(dt);
+  }
+}
+
+function formatDateString(dt) {
     return dt.getFullYear() + "-" +
       padZeroes(dt.getMonth() + 1) + "-" +
       padZeroes(dt.getDate()) + " " +
       padZeroes(dt.getHours()) + ":" +
       padZeroes(dt.getMinutes()) + ":" +
       padZeroes(dt.getSeconds());
-  }
 }
 
 function getTimeZone() {
@@ -161,7 +165,10 @@ function setDataTableDefaults() {
 
 function formatDate(date) {
   if (date <= 0) return "-";
-  else return date.split(".")[0].replace("T", " ");
+  else {
+     var dt = new Date(date.replace("GMT", "Z"))
+     return formatDateString(dt);
+  }
 }
 
 function createRESTEndPointForExecutorsPage(appId) {

From cde50326ca8b357406abe2596ef8a724bb10ad0c Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Tue, 30 Jun 2020 09:28:54 -0700
Subject: [PATCH 0866/1280] [MINOR][DOCS] Fix a typo for a configuration
 property of resources allocation

### What changes were proposed in this pull request?

This PR fixes a typo for a configuration property in the `spark-standalone.md`.
`spark.driver.resourcesfile` should be `spark.driver.resourcesFile`.
I look for similar typo but this is the only typo.

### Why are the changes needed?

The property name is wrong.

### Does this PR introduce _any_ user-facing change?

Yes. The property name is corrected.

### How was this patch tested?

I confirmed the spell of the property name is the correct from the property name defined in o.a.s.internal.config.package.scala.

Closes #28958 from sarutak/fix-resource-typo.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 5176707ac3a451158e5705bfb9a070de2d6c9cab)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/spark-standalone.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index 1e6f8c586d546..566f081bcb4ff 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -359,7 +359,7 @@ Spark Standalone has 2 parts, the first is configuring the resources for the Wor
 
 The user must configure the Workers to have a set of resources available so that it can assign them out to Executors. The <code>spark.worker.resource.{resourceName}.amount</code> is used to control the amount of each resource the worker has allocated. The user must also specify either <code>spark.worker.resourcesFile</code> or <code>spark.worker.resource.{resourceName}.discoveryScript</code> to specify how the Worker discovers the resources its assigned. See the descriptions above for each of those to see which method works best for your setup.
 
-The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesfile</code> or <code>spark.driver.resource.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers, please make sure the resources file or discovery script only returns resources that do not conflict with other Drivers running on the same node.
+The second part is running an application on Spark Standalone. The only special case from the standard Spark resource configs is when you are running the Driver in client mode. For a Driver in client mode, the user can specify the resources it uses via <code>spark.driver.resourcesFile</code> or <code>spark.driver.resource.{resourceName}.discoveryScript</code>. If the Driver is running on the same host as other Drivers, please make sure the resources file or discovery script only returns resources that do not conflict with other Drivers running on the same node.
 
 Note, the user does not need to specify a discovery script when submitting an application as the Worker will start each Executor with the resources it allocates to it.
 

From edc3b37087f084fbabec8f53c45d99dc3370a129 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 1 Jul 2020 14:15:02 +0900
Subject: [PATCH 0867/1280] [SPARK-32142][SQL][TESTS] Keep the original tests
 and codes to avoid potential conflicts in dev

### What changes were proposed in this pull request?

This PR proposes to partially reverts back in the tests and some codes at https://github.com/apache/spark/pull/27728 without touching any behaivours.

Most of changes in tests are back before #27728 by combining `withNestedDataFrame` and `withParquetDataFrame`.

Basically, it addresses the comments https://github.com/apache/spark/pull/27728#discussion_r397655390, and my own comment in another PR at https://github.com/apache/spark/pull/28761#discussion_r446761037

### Why are the changes needed?

For maintenance purpose and to avoid a potential conflicts during backports. And also in case when other codes are matched with this.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Manually tested.

Closes #28955 from HyukjinKwon/SPARK-25556-followup.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8194d9ef788278ec23902da851f2a3c95f5f71bf)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../parquet/ParquetFilterSuite.scala          | 789 +++++++++---------
 .../datasources/parquet/ParquetIOSuite.scala  |  20 +-
 .../datasources/parquet/ParquetTest.scala     |  14 +-
 3 files changed, 411 insertions(+), 412 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index d20a07f420e87..8b922aaed4e68 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -22,6 +22,9 @@ import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 import java.time.{LocalDate, LocalDateTime, ZoneId}
 
+import scala.reflect.ClassTag
+import scala.reflect.runtime.universe.TypeTag
+
 import org.apache.parquet.filter2.predicate.{FilterApi, FilterPredicate, Operators}
 import org.apache.parquet.filter2.predicate.FilterApi._
 import org.apache.parquet.filter2.predicate.Operators.{Column => _, _}
@@ -106,10 +109,18 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   /**
-   * Takes single level `inputDF` dataframe to generate multi-level nested
-   * dataframes as new test data.
+   * Takes a sequence of products `data` to generate multi-level nested
+   * dataframes as new test data. It tests both non-nested and nested dataframes
+   * which are written and read back with Parquet datasource.
+   *
+   * This is different from [[ParquetTest.withParquetDataFrame]] which does not
+   * test nested cases.
    */
-  private def withNestedDataFrame(inputDF: DataFrame)
+  private def withNestedParquetDataFrame[T <: Product: ClassTag: TypeTag](data: Seq[T])
+      (runTest: (DataFrame, String, Any => Any) => Unit): Unit =
+    withNestedParquetDataFrame(spark.createDataFrame(data))(runTest)
+
+  private def withNestedParquetDataFrame(inputDF: DataFrame)
       (runTest: (DataFrame, String, Any => Any) => Unit): Unit = {
     assert(inputDF.schema.fields.length == 1)
     assert(!inputDF.schema.fields.head.dataType.isInstanceOf[StructType])
@@ -138,8 +149,11 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         "`a.b`.`c.d`", // one level nesting with column names containing `dots`
         (x: Any) => Row(x)
       )
-    ).foreach { case (df, colName, resultFun) =>
-      runTest(df, colName, resultFun)
+    ).foreach { case (newDF, colName, resultFun) =>
+      withTempPath { file =>
+        newDF.write.format(dataSourceName).save(file.getCanonicalPath)
+        readParquetFile(file.getCanonicalPath) { df => runTest(df, colName, resultFun) }
+      }
     }
   }
 
@@ -155,7 +169,9 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
     import testImplicits._
     val df = data.map(i => Tuple1(Timestamp.valueOf(i))).toDF()
-    withNestedDataFrame(df) { case (inputDF, colName, fun) =>
+    withNestedParquetDataFrame(df) { case (parquetDF, colName, fun) =>
+      implicit val df: DataFrame = parquetDF
+
       def resultFun(tsStr: String): Any = {
         val parsed = if (java8Api) {
           LocalDateTime.parse(tsStr.replace(" ", "T"))
@@ -166,36 +182,35 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         }
         fun(parsed)
       }
-      withParquetDataFrame(inputDF) { implicit df =>
-        val tsAttr = df(colName).expr
-        assert(df(colName).expr.dataType === TimestampType)
-
-        checkFilterPredicate(tsAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
-          data.map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tsAttr === ts1.ts, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr <=> ts1.ts, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr =!= ts1.ts, classOf[NotEq[_]],
-          Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tsAttr < ts2.ts, classOf[Lt[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr > ts1.ts, classOf[Gt[_]],
-          Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
-        checkFilterPredicate(tsAttr <= ts1.ts, classOf[LtEq[_]], resultFun(ts1))
-        checkFilterPredicate(tsAttr >= ts4.ts, classOf[GtEq[_]], resultFun(ts4))
-
-        checkFilterPredicate(Literal(ts1.ts) === tsAttr, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts1.ts) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts2.ts) > tsAttr, classOf[Lt[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts3.ts) < tsAttr, classOf[Gt[_]], resultFun(ts4))
-        checkFilterPredicate(Literal(ts1.ts) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
-        checkFilterPredicate(Literal(ts4.ts) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
-
-        checkFilterPredicate(!(tsAttr < ts4.ts), classOf[GtEq[_]], resultFun(ts4))
-        checkFilterPredicate(tsAttr < ts2.ts || tsAttr > ts3.ts, classOf[Operators.Or],
-          Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
-      }
+
+      val tsAttr = df(colName).expr
+      assert(df(colName).expr.dataType === TimestampType)
+
+      checkFilterPredicate(tsAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(tsAttr.isNotNull, classOf[NotEq[_]],
+        data.map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tsAttr === ts1.ts, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr <=> ts1.ts, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr =!= ts1.ts, classOf[NotEq[_]],
+        Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tsAttr < ts2.ts, classOf[Lt[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr > ts1.ts, classOf[Gt[_]],
+        Seq(ts2, ts3, ts4).map(i => Row.apply(resultFun(i))))
+      checkFilterPredicate(tsAttr <= ts1.ts, classOf[LtEq[_]], resultFun(ts1))
+      checkFilterPredicate(tsAttr >= ts4.ts, classOf[GtEq[_]], resultFun(ts4))
+
+      checkFilterPredicate(Literal(ts1.ts) === tsAttr, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts1.ts) <=> tsAttr, classOf[Eq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts2.ts) > tsAttr, classOf[Lt[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts3.ts) < tsAttr, classOf[Gt[_]], resultFun(ts4))
+      checkFilterPredicate(Literal(ts1.ts) >= tsAttr, classOf[LtEq[_]], resultFun(ts1))
+      checkFilterPredicate(Literal(ts4.ts) <= tsAttr, classOf[GtEq[_]], resultFun(ts4))
+
+      checkFilterPredicate(!(tsAttr < ts4.ts), classOf[GtEq[_]], resultFun(ts4))
+      checkFilterPredicate(tsAttr < ts2.ts || tsAttr > ts3.ts, classOf[Operators.Or],
+        Seq(Row(resultFun(ts1)), Row(resultFun(ts4))))
     }
   }
 
@@ -226,272 +241,264 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("filter pushdown - boolean") {
     val data = (true :: false :: Nil).map(b => Tuple1.apply(Option(b)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val booleanAttr = df(colName).expr
-        assert(df(colName).expr.dataType === BooleanType)
-
-        checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
-          Seq(Row(resultFun(true)), Row(resultFun(false))))
-
-        checkFilterPredicate(booleanAttr === true, classOf[Eq[_]], resultFun(true))
-        checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]], resultFun(true))
-        checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]], resultFun(false))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val booleanAttr = df(colName).expr
+      assert(df(colName).expr.dataType === BooleanType)
+
+      checkFilterPredicate(booleanAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(booleanAttr.isNotNull, classOf[NotEq[_]],
+        Seq(Row(resultFun(true)), Row(resultFun(false))))
+
+      checkFilterPredicate(booleanAttr === true, classOf[Eq[_]], resultFun(true))
+      checkFilterPredicate(booleanAttr <=> true, classOf[Eq[_]], resultFun(true))
+      checkFilterPredicate(booleanAttr =!= true, classOf[NotEq[_]], resultFun(false))
     }
   }
 
   test("filter pushdown - tinyint") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toByte)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val tinyIntAttr = df(colName).expr
-        assert(df(colName).expr.dataType === ByteType)
-
-        checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1.toByte) === tinyIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
-          classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val tinyIntAttr = df(colName).expr
+      assert(df(colName).expr.dataType === ByteType)
+
+      checkFilterPredicate(tinyIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(tinyIntAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tinyIntAttr === 1.toByte, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr <=> 1.toByte, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr =!= 1.toByte, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(tinyIntAttr < 2.toByte, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr > 3.toByte, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(tinyIntAttr <= 1.toByte, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(tinyIntAttr >= 4.toByte, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1.toByte) === tinyIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1.toByte) <=> tinyIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2.toByte) > tinyIntAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3.toByte) < tinyIntAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1.toByte) >= tinyIntAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4.toByte) <= tinyIntAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(tinyIntAttr < 4.toByte), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(tinyIntAttr < 2.toByte || tinyIntAttr > 3.toByte,
+        classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - smallint") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toShort)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val smallIntAttr = df(colName).expr
-        assert(df(colName).expr.dataType === ShortType)
-
-        checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1.toShort) === smallIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2.toShort) > smallIntAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3.toShort) < smallIntAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1.toShort) >= smallIntAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4.toShort) <= smallIntAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr > 3.toShort,
-          classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val smallIntAttr = df(colName).expr
+      assert(df(colName).expr.dataType === ShortType)
+
+      checkFilterPredicate(smallIntAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(smallIntAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(smallIntAttr === 1.toShort, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr <=> 1.toShort, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr =!= 1.toShort, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(smallIntAttr < 2.toShort, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr > 3.toShort, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(smallIntAttr <= 1.toShort, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(smallIntAttr >= 4.toShort, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1.toShort) === smallIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1.toShort) <=> smallIntAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2.toShort) > smallIntAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3.toShort) < smallIntAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1.toShort) >= smallIntAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4.toShort) <= smallIntAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(smallIntAttr < 4.toShort), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(smallIntAttr < 2.toShort || smallIntAttr > 3.toShort,
+        classOf[Operators.Or], Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - integer") {
     val data = (1 to 4).map(i => Tuple1(Option(i)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val intAttr = df(colName).expr
-        assert(df(colName).expr.dataType === IntegerType)
-
-        checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val intAttr = df(colName).expr
+      assert(df(colName).expr.dataType === IntegerType)
+
+      checkFilterPredicate(intAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(intAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(intAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(intAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(intAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(intAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(intAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(intAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(intAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === intAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> intAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > intAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < intAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= intAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= intAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(intAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(intAttr < 2 || intAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - long") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toLong)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val longAttr = df(colName).expr
-        assert(df(colName).expr.dataType === LongType)
-
-        checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(longAttr < 2 || longAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val longAttr = df(colName).expr
+      assert(df(colName).expr.dataType === LongType)
+
+      checkFilterPredicate(longAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(longAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(longAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(longAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(longAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(longAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(longAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(longAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(longAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === longAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> longAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > longAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < longAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= longAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= longAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(longAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(longAttr < 2 || longAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - float") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toFloat)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val floatAttr = df(colName).expr
-        assert(df(colName).expr.dataType === FloatType)
-
-        checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(floatAttr < 2 || floatAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val floatAttr = df(colName).expr
+      assert(df(colName).expr.dataType === FloatType)
+
+      checkFilterPredicate(floatAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(floatAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(floatAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(floatAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(floatAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(floatAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(floatAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === floatAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> floatAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > floatAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < floatAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= floatAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= floatAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(floatAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(floatAttr < 2 || floatAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - double") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toDouble)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val doubleAttr = df(colName).expr
-        assert(df(colName).expr.dataType === DoubleType)
-
-        checkFilterPredicate(doubleAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(doubleAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(doubleAttr === 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr <=> 1, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr =!= 1, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i))))
-
-        checkFilterPredicate(doubleAttr < 2, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr > 3, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(doubleAttr <= 1, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(doubleAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(Literal(1) === doubleAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(1) <=> doubleAttr, classOf[Eq[_]], resultFun(1))
-        checkFilterPredicate(Literal(2) > doubleAttr, classOf[Lt[_]], resultFun(1))
-        checkFilterPredicate(Literal(3) < doubleAttr, classOf[Gt[_]], resultFun(4))
-        checkFilterPredicate(Literal(1) >= doubleAttr, classOf[LtEq[_]], resultFun(1))
-        checkFilterPredicate(Literal(4) <= doubleAttr, classOf[GtEq[_]], resultFun(4))
-
-        checkFilterPredicate(!(doubleAttr < 4), classOf[GtEq[_]], resultFun(4))
-        checkFilterPredicate(doubleAttr < 2 || doubleAttr > 3, classOf[Operators.Or],
-          Seq(Row(resultFun(1)), Row(resultFun(4))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val doubleAttr = df(colName).expr
+      assert(df(colName).expr.dataType === DoubleType)
+
+      checkFilterPredicate(doubleAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(doubleAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(doubleAttr === 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr <=> 1, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr =!= 1, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i))))
+
+      checkFilterPredicate(doubleAttr < 2, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr > 3, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(doubleAttr <= 1, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(doubleAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(Literal(1) === doubleAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(1) <=> doubleAttr, classOf[Eq[_]], resultFun(1))
+      checkFilterPredicate(Literal(2) > doubleAttr, classOf[Lt[_]], resultFun(1))
+      checkFilterPredicate(Literal(3) < doubleAttr, classOf[Gt[_]], resultFun(4))
+      checkFilterPredicate(Literal(1) >= doubleAttr, classOf[LtEq[_]], resultFun(1))
+      checkFilterPredicate(Literal(4) <= doubleAttr, classOf[GtEq[_]], resultFun(4))
+
+      checkFilterPredicate(!(doubleAttr < 4), classOf[GtEq[_]], resultFun(4))
+      checkFilterPredicate(doubleAttr < 2 || doubleAttr > 3, classOf[Operators.Or],
+        Seq(Row(resultFun(1)), Row(resultFun(4))))
     }
   }
 
   test("filter pushdown - string") {
     val data = (1 to 4).map(i => Tuple1(Option(i.toString)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val stringAttr = df(colName).expr
-        assert(df(colName).expr.dataType === StringType)
-
-        checkFilterPredicate(stringAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(stringAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i.toString))))
-
-        checkFilterPredicate(stringAttr === "1", classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr <=> "1", classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr =!= "1", classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i.toString))))
-
-        checkFilterPredicate(stringAttr < "2", classOf[Lt[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr > "3", classOf[Gt[_]], resultFun("4"))
-        checkFilterPredicate(stringAttr <= "1", classOf[LtEq[_]], resultFun("1"))
-        checkFilterPredicate(stringAttr >= "4", classOf[GtEq[_]], resultFun("4"))
-
-        checkFilterPredicate(Literal("1") === stringAttr, classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(Literal("1") <=> stringAttr, classOf[Eq[_]], resultFun("1"))
-        checkFilterPredicate(Literal("2") > stringAttr, classOf[Lt[_]], resultFun("1"))
-        checkFilterPredicate(Literal("3") < stringAttr, classOf[Gt[_]], resultFun("4"))
-        checkFilterPredicate(Literal("1") >= stringAttr, classOf[LtEq[_]], resultFun("1"))
-        checkFilterPredicate(Literal("4") <= stringAttr, classOf[GtEq[_]], resultFun("4"))
-
-        checkFilterPredicate(!(stringAttr < "4"), classOf[GtEq[_]], resultFun("4"))
-        checkFilterPredicate(stringAttr < "2" || stringAttr > "3", classOf[Operators.Or],
-          Seq(Row(resultFun("1")), Row(resultFun("4"))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val stringAttr = df(colName).expr
+      assert(df(colName).expr.dataType === StringType)
+
+      checkFilterPredicate(stringAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(stringAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+      checkFilterPredicate(stringAttr === "1", classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr <=> "1", classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr =!= "1", classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i.toString))))
+
+      checkFilterPredicate(stringAttr < "2", classOf[Lt[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr > "3", classOf[Gt[_]], resultFun("4"))
+      checkFilterPredicate(stringAttr <= "1", classOf[LtEq[_]], resultFun("1"))
+      checkFilterPredicate(stringAttr >= "4", classOf[GtEq[_]], resultFun("4"))
+
+      checkFilterPredicate(Literal("1") === stringAttr, classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("1") <=> stringAttr, classOf[Eq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("2") > stringAttr, classOf[Lt[_]], resultFun("1"))
+      checkFilterPredicate(Literal("3") < stringAttr, classOf[Gt[_]], resultFun("4"))
+      checkFilterPredicate(Literal("1") >= stringAttr, classOf[LtEq[_]], resultFun("1"))
+      checkFilterPredicate(Literal("4") <= stringAttr, classOf[GtEq[_]], resultFun("4"))
+
+      checkFilterPredicate(!(stringAttr < "4"), classOf[GtEq[_]], resultFun("4"))
+      checkFilterPredicate(stringAttr < "2" || stringAttr > "3", classOf[Operators.Or],
+        Seq(Row(resultFun("1")), Row(resultFun("4"))))
     }
   }
 
@@ -501,38 +508,37 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
 
     val data = (1 to 4).map(i => Tuple1(Option(i.b)))
-    import testImplicits._
-    withNestedDataFrame(data.toDF()) { case (inputDF, colName, resultFun) =>
-      withParquetDataFrame(inputDF) { implicit df =>
-        val binaryAttr: Expression = df(colName).expr
-        assert(df(colName).expr.dataType === BinaryType)
-
-        checkFilterPredicate(binaryAttr === 1.b, classOf[Eq[_]], resultFun(1.b))
-        checkFilterPredicate(binaryAttr <=> 1.b, classOf[Eq[_]], resultFun(1.b))
-
-        checkFilterPredicate(binaryAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-        checkFilterPredicate(binaryAttr.isNotNull, classOf[NotEq[_]],
-          (1 to 4).map(i => Row.apply(resultFun(i.b))))
-
-        checkFilterPredicate(binaryAttr =!= 1.b, classOf[NotEq[_]],
-          (2 to 4).map(i => Row.apply(resultFun(i.b))))
-
-        checkFilterPredicate(binaryAttr < 2.b, classOf[Lt[_]], resultFun(1.b))
-        checkFilterPredicate(binaryAttr > 3.b, classOf[Gt[_]], resultFun(4.b))
-        checkFilterPredicate(binaryAttr <= 1.b, classOf[LtEq[_]], resultFun(1.b))
-        checkFilterPredicate(binaryAttr >= 4.b, classOf[GtEq[_]], resultFun(4.b))
-
-        checkFilterPredicate(Literal(1.b) === binaryAttr, classOf[Eq[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(1.b) <=> binaryAttr, classOf[Eq[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(2.b) > binaryAttr, classOf[Lt[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(3.b) < binaryAttr, classOf[Gt[_]], resultFun(4.b))
-        checkFilterPredicate(Literal(1.b) >= binaryAttr, classOf[LtEq[_]], resultFun(1.b))
-        checkFilterPredicate(Literal(4.b) <= binaryAttr, classOf[GtEq[_]], resultFun(4.b))
-
-        checkFilterPredicate(!(binaryAttr < 4.b), classOf[GtEq[_]], resultFun(4.b))
-        checkFilterPredicate(binaryAttr < 2.b || binaryAttr > 3.b, classOf[Operators.Or],
-          Seq(Row(resultFun(1.b)), Row(resultFun(4.b))))
-      }
+    withNestedParquetDataFrame(data) { case (inputDF, colName, resultFun) =>
+      implicit val df: DataFrame = inputDF
+
+      val binaryAttr: Expression = df(colName).expr
+      assert(df(colName).expr.dataType === BinaryType)
+
+      checkFilterPredicate(binaryAttr === 1.b, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr <=> 1.b, classOf[Eq[_]], resultFun(1.b))
+
+      checkFilterPredicate(binaryAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+      checkFilterPredicate(binaryAttr.isNotNull, classOf[NotEq[_]],
+        (1 to 4).map(i => Row.apply(resultFun(i.b))))
+
+      checkFilterPredicate(binaryAttr =!= 1.b, classOf[NotEq[_]],
+        (2 to 4).map(i => Row.apply(resultFun(i.b))))
+
+      checkFilterPredicate(binaryAttr < 2.b, classOf[Lt[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr > 3.b, classOf[Gt[_]], resultFun(4.b))
+      checkFilterPredicate(binaryAttr <= 1.b, classOf[LtEq[_]], resultFun(1.b))
+      checkFilterPredicate(binaryAttr >= 4.b, classOf[GtEq[_]], resultFun(4.b))
+
+      checkFilterPredicate(Literal(1.b) === binaryAttr, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(1.b) <=> binaryAttr, classOf[Eq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(2.b) > binaryAttr, classOf[Lt[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(3.b) < binaryAttr, classOf[Gt[_]], resultFun(4.b))
+      checkFilterPredicate(Literal(1.b) >= binaryAttr, classOf[LtEq[_]], resultFun(1.b))
+      checkFilterPredicate(Literal(4.b) <= binaryAttr, classOf[GtEq[_]], resultFun(4.b))
+
+      checkFilterPredicate(!(binaryAttr < 4.b), classOf[GtEq[_]], resultFun(4.b))
+      checkFilterPredicate(binaryAttr < 2.b || binaryAttr > 3.b, classOf[Operators.Or],
+        Seq(Row(resultFun(1.b)), Row(resultFun(4.b))))
     }
   }
 
@@ -546,56 +552,57 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
     Seq(false, true).foreach { java8Api =>
       withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
-        val df = data.map(i => Tuple1(Date.valueOf(i))).toDF()
-        withNestedDataFrame(df) { case (inputDF, colName, fun) =>
+        val dates = data.map(i => Tuple1(Date.valueOf(i))).toDF()
+        withNestedParquetDataFrame(dates) { case (inputDF, colName, fun) =>
+          implicit val df: DataFrame = inputDF
+
           def resultFun(dateStr: String): Any = {
             val parsed = if (java8Api) LocalDate.parse(dateStr) else Date.valueOf(dateStr)
             fun(parsed)
           }
-          withParquetDataFrame(inputDF) { implicit df =>
-            val dateAttr: Expression = df(colName).expr
-            assert(df(colName).expr.dataType === DateType)
-
-            checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-            checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
-              data.map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
-              Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
-              resultFun("2018-03-21"))
-            checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
-              resultFun("2018-03-21"))
-
-            checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
-              resultFun("2018-03-21"))
-            checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
-              resultFun("2018-03-18"))
-            checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
-              resultFun("2018-03-21"))
-
-            checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
-              resultFun("2018-03-21"))
-            checkFilterPredicate(
-              dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
-              classOf[Operators.Or],
-              Seq(Row(resultFun("2018-03-18")), Row(resultFun("2018-03-21"))))
-          }
+
+          val dateAttr: Expression = df(colName).expr
+          assert(df(colName).expr.dataType === DateType)
+
+          checkFilterPredicate(dateAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+          checkFilterPredicate(dateAttr.isNotNull, classOf[NotEq[_]],
+            data.map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(dateAttr === "2018-03-18".date, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr <=> "2018-03-18".date, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr =!= "2018-03-18".date, classOf[NotEq[_]],
+            Seq("2018-03-19", "2018-03-20", "2018-03-21").map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(dateAttr < "2018-03-19".date, classOf[Lt[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr > "2018-03-20".date, classOf[Gt[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(dateAttr <= "2018-03-18".date, classOf[LtEq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(dateAttr >= "2018-03-21".date, classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+
+          checkFilterPredicate(Literal("2018-03-18".date) === dateAttr, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-18".date) <=> dateAttr, classOf[Eq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-19".date) > dateAttr, classOf[Lt[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-20".date) < dateAttr, classOf[Gt[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(Literal("2018-03-18".date) >= dateAttr, classOf[LtEq[_]],
+            resultFun("2018-03-18"))
+          checkFilterPredicate(Literal("2018-03-21".date) <= dateAttr, classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+
+          checkFilterPredicate(!(dateAttr < "2018-03-21".date), classOf[GtEq[_]],
+            resultFun("2018-03-21"))
+          checkFilterPredicate(
+            dateAttr < "2018-03-19".date || dateAttr > "2018-03-20".date,
+            classOf[Operators.Or],
+            Seq(Row(resultFun("2018-03-18")), Row(resultFun("2018-03-21"))))
         }
       }
     }
@@ -603,7 +610,9 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
 
   test("filter pushdown - timestamp") {
     Seq(true, false).foreach { java8Api =>
-      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString) {
+      withSQLConf(
+        SQLConf.DATETIME_JAVA8API_ENABLED.key -> java8Api.toString,
+        SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
         // spark.sql.parquet.outputTimestampType = TIMESTAMP_MILLIS
         val millisData = Seq(
           "1000-06-14 08:28:53.123",
@@ -630,11 +639,14 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         withSQLConf(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key ->
           ParquetOutputTimestampType.INT96.toString) {
           import testImplicits._
-          withParquetDataFrame(
-            millisData.map(i => Tuple1(Timestamp.valueOf(i))).toDF()) { implicit df =>
-            val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
-            assertResult(None) {
-              createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+          withTempPath { file =>
+            millisData.map(i => Tuple1(Timestamp.valueOf(i))).toDF
+              .write.format(dataSourceName).save(file.getCanonicalPath)
+            readParquetFile(file.getCanonicalPath) { df =>
+              val schema = new SparkToParquetSchemaConverter(conf).convert(df.schema)
+              assertResult(None) {
+                createParquetFilters(schema).createFilter(sources.IsNull("_1"))
+              }
             }
           }
         }
@@ -653,36 +665,36 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
         val rdd =
           spark.sparkContext.parallelize((1 to 4).map(i => Row(new java.math.BigDecimal(i))))
         val dataFrame = spark.createDataFrame(rdd, StructType.fromDDL(s"a decimal($precision, 2)"))
-        withNestedDataFrame(dataFrame) { case (inputDF, colName, resultFun) =>
-          withParquetDataFrame(inputDF) { implicit df =>
-            val decimalAttr: Expression = df(colName).expr
-            assert(df(colName).expr.dataType === DecimalType(precision, 2))
-
-            checkFilterPredicate(decimalAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
-            checkFilterPredicate(decimalAttr.isNotNull, classOf[NotEq[_]],
-              (1 to 4).map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(decimalAttr === 1, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr <=> 1, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr =!= 1, classOf[NotEq[_]],
-              (2 to 4).map(i => Row.apply(resultFun(i))))
-
-            checkFilterPredicate(decimalAttr < 2, classOf[Lt[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr > 3, classOf[Gt[_]], resultFun(4))
-            checkFilterPredicate(decimalAttr <= 1, classOf[LtEq[_]], resultFun(1))
-            checkFilterPredicate(decimalAttr >= 4, classOf[GtEq[_]], resultFun(4))
-
-            checkFilterPredicate(Literal(1) === decimalAttr, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(Literal(1) <=> decimalAttr, classOf[Eq[_]], resultFun(1))
-            checkFilterPredicate(Literal(2) > decimalAttr, classOf[Lt[_]], resultFun(1))
-            checkFilterPredicate(Literal(3) < decimalAttr, classOf[Gt[_]], resultFun(4))
-            checkFilterPredicate(Literal(1) >= decimalAttr, classOf[LtEq[_]], resultFun(1))
-            checkFilterPredicate(Literal(4) <= decimalAttr, classOf[GtEq[_]], resultFun(4))
-
-            checkFilterPredicate(!(decimalAttr < 4), classOf[GtEq[_]], resultFun(4))
-            checkFilterPredicate(decimalAttr < 2 || decimalAttr > 3, classOf[Operators.Or],
-              Seq(Row(resultFun(1)), Row(resultFun(4))))
-          }
+        withNestedParquetDataFrame(dataFrame) { case (inputDF, colName, resultFun) =>
+          implicit val df: DataFrame = inputDF
+
+          val decimalAttr: Expression = df(colName).expr
+          assert(df(colName).expr.dataType === DecimalType(precision, 2))
+
+          checkFilterPredicate(decimalAttr.isNull, classOf[Eq[_]], Seq.empty[Row])
+          checkFilterPredicate(decimalAttr.isNotNull, classOf[NotEq[_]],
+            (1 to 4).map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(decimalAttr === 1, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr <=> 1, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr =!= 1, classOf[NotEq[_]],
+            (2 to 4).map(i => Row.apply(resultFun(i))))
+
+          checkFilterPredicate(decimalAttr < 2, classOf[Lt[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr > 3, classOf[Gt[_]], resultFun(4))
+          checkFilterPredicate(decimalAttr <= 1, classOf[LtEq[_]], resultFun(1))
+          checkFilterPredicate(decimalAttr >= 4, classOf[GtEq[_]], resultFun(4))
+
+          checkFilterPredicate(Literal(1) === decimalAttr, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(Literal(1) <=> decimalAttr, classOf[Eq[_]], resultFun(1))
+          checkFilterPredicate(Literal(2) > decimalAttr, classOf[Lt[_]], resultFun(1))
+          checkFilterPredicate(Literal(3) < decimalAttr, classOf[Gt[_]], resultFun(4))
+          checkFilterPredicate(Literal(1) >= decimalAttr, classOf[LtEq[_]], resultFun(1))
+          checkFilterPredicate(Literal(4) <= decimalAttr, classOf[GtEq[_]], resultFun(4))
+
+          checkFilterPredicate(!(decimalAttr < 4), classOf[GtEq[_]], resultFun(4))
+          checkFilterPredicate(decimalAttr < 2 || decimalAttr > 3, classOf[Operators.Or],
+            Seq(Row(resultFun(1)), Row(resultFun(4))))
         }
       }
     }
@@ -1195,8 +1207,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("SPARK-16371 Do not push down filters when inner name and outer name are the same") {
-    import testImplicits._
-    withParquetDataFrame((1 to 4).map(i => Tuple1(Tuple1(i))).toDF()) { implicit df =>
+    withParquetDataFrame((1 to 4).map(i => Tuple1(Tuple1(i)))) { implicit df =>
       // Here the schema becomes as below:
       //
       // root
@@ -1336,10 +1347,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 
   test("filter pushdown - StringStartsWith") {
-    withParquetDataFrame {
-      import testImplicits._
-      (1 to 4).map(i => Tuple1(i + "str" + i)).toDF()
-    } { implicit df =>
+    withParquetDataFrame((1 to 4).map(i => Tuple1(i + "str" + i))) { implicit df =>
       checkFilterPredicate(
         '_1.startsWith("").asInstanceOf[Predicate],
         classOf[UserDefinedByInstance[_, _]],
@@ -1385,10 +1393,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
     }
 
     // SPARK-28371: make sure filter is null-safe.
-    withParquetDataFrame {
-      import testImplicits._
-      Seq(Tuple1[String](null)).toDF()
-    } { implicit df =>
+    withParquetDataFrame(Seq(Tuple1[String](null))) { implicit df =>
       checkFilterPredicate(
         '_1.startsWith("blah").asInstanceOf[Predicate],
         classOf[UserDefinedByInstance[_, _]],
@@ -1607,7 +1612,7 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
       expected: Seq[Row]): Unit = {
     val output = predicate.collect { case a: Attribute => a }.distinct
 
-    Seq(("parquet", true), ("", false)).map { case (pushdownDsList, nestedPredicatePushdown) =>
+    Seq(("parquet", true), ("", false)).foreach { case (pushdownDsList, nestedPredicatePushdown) =>
       withSQLConf(
         SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> "true",
         SQLConf.PARQUET_FILTER_PUSHDOWN_DATE_ENABLED.key -> "true",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 79c32976f02ec..2dc8a062bb73d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -85,7 +85,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
    * Writes `data` to a Parquet file, reads it back and check file contents.
    */
   protected def checkParquetFile[T <: Product : ClassTag: TypeTag](data: Seq[T]): Unit = {
-    withParquetDataFrame(data.toDF())(r => checkAnswer(r, data.map(Row.fromTuple)))
+    withParquetDataFrame(data)(r => checkAnswer(r, data.map(Row.fromTuple)))
   }
 
   test("basic data types (without binary)") {
@@ -97,7 +97,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   test("raw binary") {
     val data = (1 to 4).map(i => Tuple1(Array.fill(3)(i.toByte)))
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       assertResult(data.map(_._1.mkString(",")).sorted) {
         df.collect().map(_.getAs[Array[Byte]](0).mkString(",")).sorted
       }
@@ -200,7 +200,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   testStandardAndLegacyModes("struct") {
     val data = (1 to 4).map(i => Tuple1((i, s"val_$i")))
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(struct) =>
         Row(Row(struct.productIterator.toSeq: _*))
@@ -217,7 +217,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(array) =>
         Row(array.map(struct => Row(struct.productIterator.toSeq: _*)))
@@ -236,7 +236,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(array) =>
         Row(array.map { case Tuple1(Tuple1(str)) => Row(Row(str))})
@@ -246,7 +246,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
 
   testStandardAndLegacyModes("nested struct with array of array as field") {
     val data = (1 to 4).map(i => Tuple1((i, Seq(Seq(s"val_$i")))))
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(struct) =>
         Row(Row(struct.productIterator.toSeq: _*))
@@ -263,7 +263,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(m) =>
         Row(m.map { case (k, v) => Row(k.productIterator.toSeq: _*) -> v })
@@ -280,7 +280,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
         )
       )
     }
-    withParquetDataFrame(data.toDF()) { df =>
+    withParquetDataFrame(data) { df =>
       // Structs are converted to `Row`s
       checkAnswer(df, data.map { case Tuple1(m) =>
         Row(m.mapValues(struct => Row(struct.productIterator.toSeq: _*)))
@@ -296,7 +296,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       null.asInstanceOf[java.lang.Float],
       null.asInstanceOf[java.lang.Double])
 
-    withParquetDataFrame((allNulls :: Nil).toDF()) { df =>
+    withParquetDataFrame(allNulls :: Nil) { df =>
       val rows = df.collect()
       assert(rows.length === 1)
       assert(rows.head === Row(Seq.fill(5)(null): _*))
@@ -309,7 +309,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       None.asInstanceOf[Option[Long]],
       None.asInstanceOf[Option[String]])
 
-    withParquetDataFrame((allNones :: Nil).toDF()) { df =>
+    withParquetDataFrame(allNones :: Nil) { df =>
       val rows = df.collect()
       assert(rows.length === 1)
       assert(rows.head === Row(Seq.fill(3)(null): _*))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
index 105f025adc0ad..db8ee724c01c1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTest.scala
@@ -63,18 +63,12 @@ private[sql] trait ParquetTest extends FileBasedDataSourceTest {
       (f: String => Unit): Unit = withDataSourceFile(data)(f)
 
   /**
-   * Writes `df` dataframe to a Parquet file and reads it back as a [[DataFrame]],
+   * Writes `data` to a Parquet file and reads it back as a [[DataFrame]],
    * which is then passed to `f`. The Parquet file will be deleted after `f` returns.
    */
-  protected def withParquetDataFrame(df: DataFrame, testVectorized: Boolean = true)
-      (f: DataFrame => Unit): Unit = {
-    withTempPath { file =>
-      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> "CORRECTED") {
-        df.write.format(dataSourceName).save(file.getCanonicalPath)
-      }
-      readFile(file.getCanonicalPath, testVectorized)(f)
-    }
-  }
+  protected def withParquetDataFrame[T <: Product: ClassTag: TypeTag]
+      (data: Seq[T], testVectorized: Boolean = true)
+      (f: DataFrame => Unit): Unit = withDataSourceDataFrame(data, testVectorized)(f)
 
   /**
    * Writes `data` to a Parquet file, reads it back as a [[DataFrame]] and registers it as a

From f34425371b99463b24588b944dc7cca7dc15e584 Mon Sep 17 00:00:00 2001
From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Date: Tue, 30 Jun 2020 23:33:50 -0700
Subject: [PATCH 0868/1280] [SPARK-32131][SQL] Fix AnalysisException messages
 at UNION/EXCEPT/MINUS operations

### What changes were proposed in this pull request?
fix error exception messages during exceptions on Union and set operations

### Why are the changes needed?
Union and set operations can only be performed on tables with the compatible column types,while when we have more than two column, the exception messages will have wrong column index.

Steps to reproduce:

```
drop table if exists test1;
drop table if exists test2;
drop table if exists test3;
create table if not exists test1(id int, age int, name timestamp);
create table if not exists test2(id int, age timestamp, name timestamp);
create table if not exists test3(id int, age int, name int);
insert into test1 select 1,2,'2020-01-01 01:01:01';
insert into test2 select 1,'2020-01-01 01:01:01','2020-01-01 01:01:01';
insert into test3 select 1,3,4;
```

Query1:
```sql
select * from test1 except select * from test2;
```
Result1:
```
Error: org.apache.spark.sql.AnalysisException: Except can only be performed on tables with the compatible column types. timestamp <> int at the second column of the second table;; 'Except false :- Project [id#620, age#621, name#622] : +- SubqueryAlias `default`.`test1` : +- HiveTableRelation `default`.`test1`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, [id#620, age#621, name#622] +- Project [id#623, age#624, name#625] +- SubqueryAlias `default`.`test2` +- HiveTableRelation `default`.`test2`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, [id#623, age#624, name#625] (state=,code=0)
```

Query2:

```sql
select * from test1 except select * from test3;
```

Result2:

```
Error: org.apache.spark.sql.AnalysisException: Except can only be performed on tables with the compatible column types
 int <> timestamp at the 2th column of the second table;
```

the above query1 has the right exception message
the above query2 have the wrong errors information, it may need to change to the following

```
Error: org.apache.spark.sql.AnalysisException: Except can only be performed on tables with the compatible column types.
int <> timestamp at the  third column of the second table
```

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
unit test

Closes #28951 from GuoPhilipse/32131-correct-error-messages.

Lead-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Co-authored-by: GuoPhilipse <guofei_ok@126.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 02f3b80d3a277e0c19a66c28d935fa41da7b3307)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/catalyst/analysis/CheckAnalysis.scala |  3 +-
 .../sql/catalyst/analysis/AnalysisSuite.scala | 53 +++++++++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 066dc6db0227d..e5b8ae7b22292 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -341,7 +341,8 @@ trait CheckAnalysis extends PredicateHelper {
             def ordinalNumber(i: Int): String = i match {
               case 0 => "first"
               case 1 => "second"
-              case i => s"${i}th"
+              case 2 => "third"
+              case i => s"${i + 1}th"
             }
             val ref = dataTypes(operator.children.head)
             operator.children.tail.zipWithIndex.foreach { case (child, ti) =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 02472e153b09e..8db2036496bfd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -826,4 +826,57 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       }
     }
   }
+
+  test("SPARK-32131: Fix wrong column index when we have more than two columns" +
+    " during union and set operations" ) {
+    val firstTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", FloatType)())
+
+    val secondTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", TimestampType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", FloatType)())
+
+    val thirdTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", TimestampType)(),
+      AttributeReference("d", FloatType)())
+
+    val fourthTable = LocalRelation(
+      AttributeReference("a", StringType)(),
+      AttributeReference("b", DoubleType)(),
+      AttributeReference("c", IntegerType)(),
+      AttributeReference("d", TimestampType)())
+
+    val r1 = Union(firstTable, secondTable)
+    val r2 = Union(firstTable, thirdTable)
+    val r3 = Union(firstTable, fourthTable)
+    val r4 = Except(firstTable, secondTable, isAll = false)
+    val r5 = Intersect(firstTable, secondTable, isAll = false)
+
+    assertAnalysisError(r1,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+
+    assertAnalysisError(r2,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> int at the third column of the second table"))
+
+    assertAnalysisError(r3,
+      Seq("Union can only be performed on tables with the compatible column types. " +
+        "timestamp <> float at the 4th column of the second table"))
+
+    assertAnalysisError(r4,
+      Seq("Except can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+
+    assertAnalysisError(r5,
+      Seq("Intersect can only be performed on tables with the compatible column types. " +
+        "timestamp <> double at the second column of the second table"))
+  }
 }

From 49855e78a3ae430cd67c50d6dd81f6688fb3de15 Mon Sep 17 00:00:00 2001
From: Micah Kornfield <micahk@google.com>
Date: Wed, 1 Jul 2020 07:13:10 +0000
Subject: [PATCH 0869/1280] [SPARK-32095][SQL] Update documentation to reflect
 usage of updated statistics

### What changes were proposed in this pull request?
Update documentation to reflect changes in https://github.com/apache/spark/commit/faf220aad9051c224a630e678c54098861f6b482

I've changed the documentation to reflect updated statistics may be used to improve query plan.

### Why are the changes needed?
I believe the documentation is stale and misleading.

### Does this PR introduce _any_ user-facing change?
Yes, this is a javadoc documentation fix.

### How was this patch tested?
Doc fix.

Closes #28925 from emkornfield/spark-32095.

Authored-by: Micah Kornfield <micahk@google.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a4ba34434c55c7f0eafa32d8c7ea6e052813f293)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/connector/read/SupportsReportStatistics.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
index b839fd5a4a726..1e0c9ca7c7e4b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/read/SupportsReportStatistics.java
@@ -23,9 +23,9 @@
  * A mix in interface for {@link Scan}. Data sources can implement this interface to
  * report statistics to Spark.
  *
- * As of Spark 2.4, statistics are reported to the optimizer before any operator is pushed to the
- * data source. Implementations that return more accurate statistics based on pushed operators will
- * not improve query performance until the planner can push operators before getting stats.
+ * As of Spark 3.0, statistics are reported to the optimizer after operators are pushed to the
+ * data source. Implementations may return more accurate statistics based on pushed operators
+ * which may improve query performance by providing better information to the optimizer.
  *
  * @since 3.0.0
  */

From fef337935179248e88aa4be84a4214fea1c3a11a Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 2 Jul 2020 06:09:54 +0800
Subject: [PATCH 0870/1280] [SPARK-31935][SQL][FOLLOWUP] Hadoop file system
 config should be effective in data source options

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/28760 to fix the remaining issues:
1. should consider data source options when refreshing cache by path at the end of `InsertIntoHadoopFsRelationCommand`
2. should consider data source options when inferring schema for file source
3. should consider data source options when getting the qualified path in file source v2.

### Why are the changes needed?

We didn't catch these issues in https://github.com/apache/spark/pull/28760, because the test case is to check error when initializing the file system. If we initialize the file system multiple times during a simple read/write action, the test case actually only test the first time.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

rewrite the test to make sure the entire data source read/write action can succeed.

Closes #28948 from cloud-fan/fix.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 6edb20df834f7f9b85c1559ef78a3d0d2272e4df)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../spark/sql/v2/avro/AvroDataSourceV2.scala  |  4 +--
 .../spark/sql/execution/CacheManager.scala    | 15 ++++++---
 .../InsertIntoHadoopFsRelationCommand.scala   |  2 +-
 .../datasources/SchemaMergeUtils.scala        |  4 ++-
 .../execution/datasources/orc/OrcUtils.scala  |  2 +-
 .../parquet/ParquetFileFormat.scala           |  3 +-
 .../datasources/parquet/ParquetUtils.scala    |  2 +-
 .../datasources/v2/FileDataSourceV2.scala     | 13 +++++---
 .../datasources/v2/csv/CSVDataSourceV2.scala  |  4 +--
 .../v2/json/JsonDataSourceV2.scala            |  4 +--
 .../datasources/v2/orc/OrcDataSourceV2.scala  |  4 +--
 .../v2/parquet/ParquetDataSourceV2.scala      |  4 +--
 .../v2/text/TextDataSourceV2.scala            |  4 +--
 .../spark/sql/FileBasedDataSourceSuite.scala  | 31 +++++++++++--------
 .../datasources/orc/OrcSourceSuite.scala      |  4 +--
 .../spark/sql/hive/orc/OrcFileFormat.scala    |  4 +--
 16 files changed, 59 insertions(+), 45 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
index c6f52d676422c..969dee0a39696 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
@@ -31,13 +31,13 @@ class AvroDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     AvroTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     AvroTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index 52cec8b202885..7d86c48015406 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -248,12 +248,17 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
    * `HadoopFsRelation` node(s) as part of its logical plan.
    */
   def recacheByPath(spark: SparkSession, resourcePath: String): Unit = {
-    val (fs, qualifiedPath) = {
-      val path = new Path(resourcePath)
-      val fs = path.getFileSystem(spark.sessionState.newHadoopConf())
-      (fs, fs.makeQualified(path))
-    }
+    val path = new Path(resourcePath)
+    val fs = path.getFileSystem(spark.sessionState.newHadoopConf())
+    recacheByPath(spark, path, fs)
+  }
 
+  /**
+   * Tries to re-cache all the cache entries that contain `resourcePath` in one or more
+   * `HadoopFsRelation` node(s) as part of its logical plan.
+   */
+  def recacheByPath(spark: SparkSession, resourcePath: Path, fs: FileSystem): Unit = {
+    val qualifiedPath = fs.makeQualified(resourcePath)
     recacheByCondition(spark, _.plan.find(lookupAndRefresh(_, fs, qualifiedPath)).isDefined)
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
index f11972115e09f..fe733f4238e1a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelationCommand.scala
@@ -192,7 +192,7 @@ case class InsertIntoHadoopFsRelationCommand(
       // refresh cached files in FileIndex
       fileIndex.foreach(_.refresh())
       // refresh data cache if table is cached
-      sparkSession.catalog.refreshByPath(outputPath.toString)
+      sparkSession.sharedState.cacheManager.recacheByPath(sparkSession, outputPath, fs)
 
       if (catalogTable.nonEmpty) {
         CommandUtils.updateTableStats(sparkSession, catalogTable.get)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
index 99882b0f7c7b0..28097c35401c9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/SchemaMergeUtils.scala
@@ -32,10 +32,12 @@ object SchemaMergeUtils extends Logging {
    */
   def mergeSchemasInParallel(
       sparkSession: SparkSession,
+      parameters: Map[String, String],
       files: Seq[FileStatus],
       schemaReader: (Seq[FileStatus], Configuration, Boolean) => Seq[StructType])
       : Option[StructType] = {
-    val serializedConf = new SerializableConfiguration(sparkSession.sessionState.newHadoopConf())
+    val serializedConf = new SerializableConfiguration(
+      sparkSession.sessionState.newHadoopConfWithOptions(parameters))
 
     // !! HACK ALERT !!
     // Here is a hack for Parquet, but it can be used by Orc as well.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index eea9b2a8f9613..d274bcd0edd2c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -109,7 +109,7 @@ object OrcUtils extends Logging {
     val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf)
     if (orcOptions.mergeSchema) {
       SchemaMergeUtils.mergeSchemasInParallel(
-        sparkSession, files, OrcUtils.readOrcSchemasInParallel)
+        sparkSession, options, files, OrcUtils.readOrcSchemasInParallel)
     } else {
       OrcUtils.readSchema(sparkSession, files)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
index 71874104fcf4f..68f49f9442579 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFileFormat.scala
@@ -475,6 +475,7 @@ object ParquetFileFormat extends Logging {
    *     S3 nodes).
    */
   def mergeSchemasInParallel(
+      parameters: Map[String, String],
       filesToTouch: Seq[FileStatus],
       sparkSession: SparkSession): Option[StructType] = {
     val assumeBinaryIsString = sparkSession.sessionState.conf.isParquetBinaryAsString
@@ -490,7 +491,7 @@ object ParquetFileFormat extends Logging {
         .map(ParquetFileFormat.readSchemaFromFooter(_, converter))
     }
 
-    SchemaMergeUtils.mergeSchemasInParallel(sparkSession, filesToTouch, reader)
+    SchemaMergeUtils.mergeSchemasInParallel(sparkSession, parameters, filesToTouch, reader)
   }
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
index 7e7dba92f37b5..b91d75c55c513 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala
@@ -104,7 +104,7 @@ object ParquetUtils {
           .orElse(filesByType.data.headOption)
           .toSeq
       }
-    ParquetFileFormat.mergeSchemasInParallel(filesToTouch, sparkSession)
+    ParquetFileFormat.mergeSchemasInParallel(parameters, filesToTouch, sparkSession)
   }
 
   case class FileTypes(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
index 30a964d7e643f..bbe8835049fa4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
@@ -18,7 +18,10 @@ package org.apache.spark.sql.execution.datasources.v2
 
 import java.util
 
+import scala.collection.JavaConverters._
+
 import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 
 import org.apache.spark.sql.SparkSession
@@ -53,14 +56,16 @@ trait FileDataSourceV2 extends TableProvider with DataSourceRegister {
     paths ++ Option(map.get("path")).toSeq
   }
 
-  protected def getTableName(paths: Seq[String]): String = {
-    val name = shortName() + " " + paths.map(qualifiedPathName).mkString(",")
+  protected def getTableName(map: CaseInsensitiveStringMap, paths: Seq[String]): String = {
+    val hadoopConf = sparkSession.sessionState.newHadoopConfWithOptions(
+      map.asCaseSensitiveMap().asScala.toMap)
+    val name = shortName() + " " + paths.map(qualifiedPathName(_, hadoopConf)).mkString(",")
     Utils.redact(sparkSession.sessionState.conf.stringRedactionPattern, name)
   }
 
-  private def qualifiedPathName(path: String): String = {
+  private def qualifiedPathName(path: String, hadoopConf: Configuration): String = {
     val hdfsPath = new Path(path)
-    val fs = hdfsPath.getFileSystem(sparkSession.sessionState.newHadoopConf())
+    val fs = hdfsPath.getFileSystem(hadoopConf)
     hdfsPath.makeQualified(fs.getUri, fs.getWorkingDirectory).toString
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
index 1f99d4282f6da..69d001b4a615c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
@@ -31,13 +31,13 @@ class CSVDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     CSVTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     CSVTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
index 7a0949e586cd8..9c4e3b8c78026 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
@@ -31,13 +31,13 @@ class JsonDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     JsonTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     JsonTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
index 8665af33b976a..fa2febdc5a984 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
@@ -31,13 +31,13 @@ class OrcDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     OrcTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     OrcTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
index 8cb6186c12ff3..7e7ca964de28f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
@@ -31,13 +31,13 @@ class ParquetDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     ParquetTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     ParquetTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
index 049c717effa26..43bcb61f25962 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
@@ -31,13 +31,13 @@ class TextDataSourceV2 extends FileDataSourceV2 {
 
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     TextTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
-    val tableName = getTableName(paths)
+    val tableName = getTableName(options, paths)
     TextTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index d8157d3c779b9..231a8f2aa7ddd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -18,12 +18,14 @@
 package org.apache.spark.sql
 
 import java.io.{File, FileNotFoundException}
+import java.net.URI
 import java.nio.file.{Files, StandardOpenOption}
 import java.util.Locale
 
 import scala.collection.mutable
 
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{LocalFileSystem, Path}
 
 import org.apache.spark.SparkException
 import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
@@ -845,19 +847,15 @@ class FileBasedDataSourceSuite extends QueryTest
 
   test("SPARK-31935: Hadoop file system config should be effective in data source options") {
     Seq("parquet", "").foreach { format =>
-      withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> format) {
+      withSQLConf(
+        SQLConf.USE_V1_SOURCE_LIST.key -> format,
+        "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+        "fs.file.impl.disable.cache" -> "true") {
         withTempDir { dir =>
-          val path = dir.getCanonicalPath
-          val defaultFs = "nonexistFS://nonexistFS"
-          val expectMessage = "No FileSystem for scheme nonexistFS"
-          val message1 = intercept[java.io.IOException] {
-            spark.range(10).write.option("fs.defaultFS", defaultFs).parquet(path)
-          }.getMessage
-          assert(message1.filterNot(Set(':', '"').contains) == expectMessage)
-          val message2 = intercept[java.io.IOException] {
-            spark.read.option("fs.defaultFS", defaultFs).parquet(path)
-          }.getMessage
-          assert(message2.filterNot(Set(':', '"').contains) == expectMessage)
+          val path = "file:" + dir.getCanonicalPath.stripPrefix("file:")
+          spark.range(10).write.option("ds_option", "value").mode("overwrite").parquet(path)
+          checkAnswer(
+            spark.read.option("ds_option", "value").parquet(path), spark.range(10).toDF())
         }
       }
     }
@@ -932,3 +930,10 @@ object TestingUDT {
     override def userClass: Class[NullData] = classOf[NullData]
   }
 }
+
+class FakeFileSystemRequiringDSOption extends LocalFileSystem {
+  override def initialize(name: URI, conf: Configuration): Unit = {
+    super.initialize(name, conf)
+    require(conf.get("ds_option", "") == "value")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index 73873684f6aaf..b70fd7476ed98 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -213,9 +213,7 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
           Seq(fs.listStatus(path1), fs.listStatus(path2), fs.listStatus(path3)).flatten
 
         val schema = SchemaMergeUtils.mergeSchemasInParallel(
-          spark,
-          fileStatuses,
-          schemaReader)
+          spark, Map.empty, fileStatuses, schemaReader)
 
         assert(schema.isDefined)
         assert(schema.get == StructType(Seq(
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
index 7f2eb14956dc1..356b92b4652b3 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
@@ -70,9 +70,7 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
     val orcOptions = new OrcOptions(options, sparkSession.sessionState.conf)
     if (orcOptions.mergeSchema) {
       SchemaMergeUtils.mergeSchemasInParallel(
-        sparkSession,
-        files,
-        OrcFileOperator.readOrcSchemasInParallel)
+        sparkSession, options, files, OrcFileOperator.readOrcSchemasInParallel)
     } else {
       val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
       OrcFileOperator.readSchema(

From bdbfe6ec15667835000d7e95b0c437b6ab4b251e Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 1 Jul 2020 15:45:39 -0700
Subject: [PATCH 0871/1280] [SPARK-32130][SQL] Disable the JSON option
 `inferTimestamp` by default

Set the JSON option `inferTimestamp` to `false` if an user don't pass it as datasource option.

To prevent perf regression while inferring schemas from JSON with potential timestamps fields.

Yes

- Modified existing tests in `JsonSuite` and `JsonInferSchemaSuite`.
- Regenerated results of `JsonBenchmark` in the environment:

| Item | Description |
| ---- | ----|
| Region | us-west-2 (Oregon) |
| Instance | r3.xlarge |
| AMI | ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190722.1 (ami-06f2f779464715dc5) |
| Java | OpenJDK 64-Bit Server VM 1.8.0_252 and OpenJDK 64-Bit Server VM 11.0.7+10 |

Closes #28966 from MaxGekk/json-inferTimestamps-disable-by-default.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit bcf23307f4fd70590ea10e5e9edb6e9de1f76125)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-migration-guide.md                   |  4 +
 .../spark/sql/catalyst/json/JSONOptions.scala |  2 +-
 .../catalyst/json/JsonInferSchemaSuite.scala  | 56 +++++++-----
 .../JsonBenchmark-jdk11-results.txt           | 86 +++++++++----------
 sql/core/benchmarks/JsonBenchmark-results.txt | 86 +++++++++----------
 .../datasources/json/JsonSuite.scala          |  6 +-
 6 files changed, 129 insertions(+), 111 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index b7b01d07b9b26..fb3fe09a5d439 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -22,6 +22,10 @@ license: |
 * Table of contents
 {:toc}
 
+## Upgrading from Spark SQL 3.0 to 3.0.1
+
+- In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference.
+
 ## Upgrading from Spark SQL 2.4 to 3.0
 
 ### Dataset/DataFrame APIs
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
index f9222f5af54da..70a673bb42457 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JSONOptions.scala
@@ -133,7 +133,7 @@ private[sql] class JSONOptions(
    * Enables inferring of TimestampType from strings matched to the timestamp pattern
    * defined by the timestampFormat option.
    */
-  val inferTimestamp: Boolean = parameters.get("inferTimestamp").map(_.toBoolean).getOrElse(true)
+  val inferTimestamp: Boolean = parameters.get("inferTimestamp").map(_.toBoolean).getOrElse(false)
 
   /** Build a Jackson [[JsonFactory]] using JSON options. */
   def buildJsonFactory(): JsonFactory = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
index bce917c80f93c..8290b38e33934 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala
@@ -35,22 +35,29 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
     assert(inferSchema.inferField(parser) === expectedType)
   }
 
-  def checkTimestampType(pattern: String, json: String): Unit = {
-    checkType(Map("timestampFormat" -> pattern), json, TimestampType)
+  def checkTimestampType(pattern: String, json: String, inferTimestamp: Boolean): Unit = {
+    checkType(
+      Map("timestampFormat" -> pattern, "inferTimestamp" -> inferTimestamp.toString),
+      json,
+      if (inferTimestamp) TimestampType else StringType)
   }
 
   test("inferring timestamp type") {
-    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
-        checkTimestampType("yyyy", """{"a": "2018"}""")
-        checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
-        checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
-        checkTimestampType(
-          "yyyy-MM-dd'T'HH:mm:ss.SSS",
-          """{"a": "2018-12-02T21:04:00.123"}""")
-        checkTimestampType(
-          "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
-          """{"a": "2018-12-02T21:04:00.123567+01:00"}""")
+    Seq(true, false).foreach { inferTimestamp =>
+      Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+          checkTimestampType("yyyy", """{"a": "2018"}""", inferTimestamp)
+          checkTimestampType("yyyy=MM", """{"a": "2018=12"}""", inferTimestamp)
+          checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""", inferTimestamp)
+          checkTimestampType(
+            "yyyy-MM-dd'T'HH:mm:ss.SSS",
+            """{"a": "2018-12-02T21:04:00.123"}""",
+            inferTimestamp)
+          checkTimestampType(
+            "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
+            """{"a": "2018-12-02T21:04:00.123567+01:00"}""",
+            inferTimestamp)
+        }
       }
     }
   }
@@ -71,16 +78,19 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("skip decimal type inferring") {
-    Seq("legacy", "corrected").foreach { legacyParserPolicy =>
-      withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
-        checkType(
-          options = Map(
-            "prefersDecimal" -> "false",
-            "timestampFormat" -> "yyyyMMdd.HHmmssSSS"
-          ),
-          json = """{"a": "20181202.210400123"}""",
-          dt = TimestampType
-        )
+    Seq(true, false).foreach { inferTimestamp =>
+      Seq("legacy", "corrected").foreach { legacyParserPolicy =>
+        withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) {
+          checkType(
+            options = Map(
+              "prefersDecimal" -> "false",
+              "timestampFormat" -> "yyyyMMdd.HHmmssSSS",
+              "inferTimestamp" -> inferTimestamp.toString
+            ),
+            json = """{"a": "20181202.210400123"}""",
+            dt = if (inferTimestamp) TimestampType else StringType
+          )
+        }
       }
     }
   }
diff --git a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
index d0cd591da4c94..ff370847a2e3e 100644
--- a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
@@ -7,106 +7,106 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       68879          68993         116          1.5         688.8       1.0X
-UTF-8 is set                                     115270         115602         455          0.9        1152.7       0.6X
+No encoding                                       69219          69342         116          1.4         692.2       1.0X
+UTF-8 is set                                     143950         143986          55          0.7        1439.5       0.5X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       47452          47538         113          2.1         474.5       1.0X
-UTF-8 is set                                      77330          77354          30          1.3         773.3       0.6X
+No encoding                                       57828          57913         136          1.7         578.3       1.0X
+UTF-8 is set                                      83649          83711          60          1.2         836.5       0.7X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       60470          60900         534          0.2        6047.0       1.0X
-UTF-8 is set                                     104733         104931         189          0.1       10473.3       0.6X
+No encoding                                       64560          65193        1023          0.2        6456.0       1.0X
+UTF-8 is set                                     102925         103174         216          0.1       10292.5       0.6X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      130302         131072         976          0.0      260604.6       1.0X
-UTF-8 is set                                     150860         151284         377          0.0      301720.1       0.9X
+No encoding                                      131002         132316        1160          0.0      262003.1       1.0X
+UTF-8 is set                                     152128         152371         332          0.0      304256.5       0.9X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18619          18684          99          0.5        1861.9       1.0X
-Select 1 column                                   24227          24270          38          0.4        2422.7       0.8X
+Select 10 columns                                 19376          19514         160          0.5        1937.6       1.0X
+Select 1 column                                   24089          24156          58          0.4        2408.9       0.8X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      7947           7971          21          1.3         794.7       1.0X
-Short column with UTF-8                           12700          12753          58          0.8        1270.0       0.6X
-Wide column without encoding                      92632          92955         463          0.1        9263.2       0.1X
-Wide column with UTF-8                           147013         147170         188          0.1       14701.3       0.1X
+Short column without encoding                      8131           8219         103          1.2         813.1       1.0X
+Short column with UTF-8                           13464          13508          44          0.7        1346.4       0.6X
+Wide column without encoding                     108012         108598         914          0.1       10801.2       0.1X
+Wide column with UTF-8                           150988         151369         412          0.1       15098.8       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           713            734          19         14.0          71.3       1.0X
-from_json                                         22019          22429         456          0.5        2201.9       0.0X
-json_tuple                                        27987          28047          74          0.4        2798.7       0.0X
-get_json_object                                   21468          21870         350          0.5        2146.8       0.0X
+Text read                                           753            765          18         13.3          75.3       1.0X
+from_json                                         23182          23446         230          0.4        2318.2       0.0X
+json_tuple                                        31129          31304         181          0.3        3112.9       0.0X
+get_json_object                                   22821          23073         225          0.4        2282.1       0.0X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          2887           2910          24         17.3          57.7       1.0X
-schema inferring                                  31793          31843          43          1.6         635.9       0.1X
-parsing                                           36791          37104         294          1.4         735.8       0.1X
+Text read                                          3078           3101          26         16.2          61.6       1.0X
+schema inferring                                  30225          30434         333          1.7         604.5       0.1X
+parsing                                           32237          32308          63          1.6         644.7       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                         10570          10611          45          4.7         211.4       1.0X
-Schema inferring                                  48729          48763          41          1.0         974.6       0.2X
-Parsing without charset                           35490          35648         141          1.4         709.8       0.3X
-Parsing with UTF-8                                63853          63994         163          0.8        1277.1       0.2X
+Text read                                         10835          10900          86          4.6         216.7       1.0X
+Schema inferring                                  37720          37805         110          1.3         754.4       0.3X
+Parsing without charset                           35464          35538         100          1.4         709.3       0.3X
+Parsing with UTF-8                                67311          67738         381          0.7        1346.2       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     2187           2190           5          4.6         218.7       1.0X
-to_json(timestamp)                                16262          16503         323          0.6        1626.2       0.1X
-write timestamps to files                         11679          11692          12          0.9        1167.9       0.2X
-Create a dataset of dates                          2297           2310          12          4.4         229.7       1.0X
-to_json(date)                                     10904          10956          46          0.9        1090.4       0.2X
-write dates to files                               6610           6645          35          1.5         661.0       0.3X
+Create a dataset of timestamps                     2208           2222          14          4.5         220.8       1.0X
+to_json(timestamp)                                14299          14570         285          0.7        1429.9       0.2X
+write timestamps to files                         12955          12969          13          0.8        1295.5       0.2X
+Create a dataset of dates                          2297           2323          30          4.4         229.7       1.0X
+to_json(date)                                      8509           8561          74          1.2         850.9       0.3X
+write dates to files                               6786           6827          45          1.5         678.6       0.3X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2524           2530           9          4.0         252.4       1.0X
-read timestamps from files                        41002          41052          59          0.2        4100.2       0.1X
-infer timestamps from files                       84621          84939         526          0.1        8462.1       0.0X
-read date text from files                          2292           2302           9          4.4         229.2       1.1X
-read date from files                              16954          16976          21          0.6        1695.4       0.1X
-timestamp strings                                  3067           3077          13          3.3         306.7       0.8X
-parse timestamps from Dataset[String]             48690          48971         243          0.2        4869.0       0.1X
-infer timestamps from Dataset[String]             97463          97786         338          0.1        9746.3       0.0X
-date strings                                       3952           3956           3          2.5         395.2       0.6X
-parse dates from Dataset[String]                  24210          24241          30          0.4        2421.0       0.1X
-from_json(timestamp)                              71710          72242         629          0.1        7171.0       0.0X
-from_json(date)                                   42465          42481          13          0.2        4246.5       0.1X
+read timestamp text from files                     2598           2613          18          3.8         259.8       1.0X
+read timestamps from files                        42007          42028          19          0.2        4200.7       0.1X
+infer timestamps from files                       18102          18120          28          0.6        1810.2       0.1X
+read date text from files                          2355           2360           5          4.2         235.5       1.1X
+read date from files                              17420          17458          33          0.6        1742.0       0.1X
+timestamp strings                                  3099           3101           3          3.2         309.9       0.8X
+parse timestamps from Dataset[String]             48188          48215          25          0.2        4818.8       0.1X
+infer timestamps from Dataset[String]             22929          22988         102          0.4        2292.9       0.1X
+date strings                                       4090           4103          11          2.4         409.0       0.6X
+parse dates from Dataset[String]                  24952          25068         139          0.4        2495.2       0.1X
+from_json(timestamp)                              66038          66352         413          0.2        6603.8       0.0X
+from_json(date)                                   43755          43782          27          0.2        4375.5       0.1X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt
index 46d2410fb47c3..0e4ce9003f70b 100644
--- a/sql/core/benchmarks/JsonBenchmark-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-results.txt
@@ -7,106 +7,106 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       63981          64044          56          1.6         639.8       1.0X
-UTF-8 is set                                     112672         113350         962          0.9        1126.7       0.6X
+No encoding                                       64950          65182         306          1.5         649.5       1.0X
+UTF-8 is set                                     129566         129796         229          0.8        1295.7       0.5X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       51256          51449         180          2.0         512.6       1.0X
-UTF-8 is set                                      83694          83859         148          1.2         836.9       0.6X
+No encoding                                       50896          51277         372          2.0         509.0       1.0X
+UTF-8 is set                                      89712          89763          49          1.1         897.1       0.6X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       58440          59097         569          0.2        5844.0       1.0X
-UTF-8 is set                                     102746         102883         198          0.1       10274.6       0.6X
+No encoding                                       59415          59785         372          0.2        5941.5       1.0X
+UTF-8 is set                                     103059         103165         156          0.1       10305.9       0.6X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      128982         129304         356          0.0      257965.0       1.0X
-UTF-8 is set                                     147247         147415         231          0.0      294494.1       0.9X
+No encoding                                      132951         133122         288          0.0      265901.9       1.0X
+UTF-8 is set                                     149318         149441         107          0.0      298635.3       0.9X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18837          19048         331          0.5        1883.7       1.0X
-Select 1 column                                   24707          24723          14          0.4        2470.7       0.8X
+Select 10 columns                                 18491          18552          85          0.5        1849.1       1.0X
+Select 1 column                                   25908          25946          65          0.4        2590.8       0.7X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      8218           8234          17          1.2         821.8       1.0X
-Short column with UTF-8                           12374          12438         107          0.8        1237.4       0.7X
-Wide column without encoding                     136918         137298         345          0.1       13691.8       0.1X
-Wide column with UTF-8                           176961         177142         257          0.1       17696.1       0.0X
+Short column without encoding                      9264           9307          49          1.1         926.4       1.0X
+Short column with UTF-8                           14707          14727          17          0.7        1470.7       0.6X
+Wide column without encoding                     141138         141347         276          0.1       14113.8       0.1X
+Wide column with UTF-8                           179601         180035         664          0.1       17960.1       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          1268           1278          12          7.9         126.8       1.0X
-from_json                                         23348          23479         176          0.4        2334.8       0.1X
-json_tuple                                        29606          30221        1024          0.3        2960.6       0.0X
-get_json_object                                   21898          22148         226          0.5        2189.8       0.1X
+Text read                                          1173           1184           9          8.5         117.3       1.0X
+from_json                                         23432          23738         338          0.4        2343.2       0.1X
+json_tuple                                        32573          32851         358          0.3        3257.3       0.0X
+get_json_object                                   22442          22489          47          0.4        2244.2       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5887           5944          49          8.5         117.7       1.0X
-schema inferring                                  46696          47054         312          1.1         933.9       0.1X
-parsing                                           32336          32450         129          1.5         646.7       0.2X
+Text read                                          5656           5680          31          8.8         113.1       1.0X
+schema inferring                                  33283          33337          64          1.5         665.7       0.2X
+parsing                                           41771          41929         178          1.2         835.4       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          9756           9769          11          5.1         195.1       1.0X
-Schema inferring                                  51318          51433         108          1.0        1026.4       0.2X
-Parsing without charset                           43609          43743         118          1.1         872.2       0.2X
-Parsing with UTF-8                                60775          60844         106          0.8        1215.5       0.2X
+Text read                                          9626           9668          39          5.2         192.5       1.0X
+Schema inferring                                  39489          39579          91          1.3         789.8       0.2X
+Parsing without charset                           38096          38232         125          1.3         761.9       0.3X
+Parsing with UTF-8                                64565          64725         165          0.8        1291.3       0.1X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     1998           2015          17          5.0         199.8       1.0X
-to_json(timestamp)                                18156          18317         263          0.6        1815.6       0.1X
-write timestamps to files                         12912          12917           5          0.8        1291.2       0.2X
-Create a dataset of dates                          2209           2270          53          4.5         220.9       0.9X
-to_json(date)                                      9433           9489          90          1.1         943.3       0.2X
-write dates to files                               6915           6923           8          1.4         691.5       0.3X
+Create a dataset of timestamps                     1898           1912          13          5.3         189.8       1.0X
+to_json(timestamp)                                20011          20092         119          0.5        2001.1       0.1X
+write timestamps to files                         13388          13427          35          0.7        1338.8       0.1X
+Create a dataset of dates                          2351           2368          18          4.3         235.1       0.8X
+to_json(date)                                     11884          11913          40          0.8        1188.4       0.2X
+write dates to files                               7317           7326           9          1.4         731.7       0.3X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2395           2412          17          4.2         239.5       1.0X
-read timestamps from files                        47269          47334          89          0.2        4726.9       0.1X
-infer timestamps from files                       91806          91851          67          0.1        9180.6       0.0X
-read date text from files                          2118           2133          13          4.7         211.8       1.1X
-read date from files                              17267          17340         115          0.6        1726.7       0.1X
-timestamp strings                                  3906           3935          26          2.6         390.6       0.6X
-parse timestamps from Dataset[String]             52244          52534         279          0.2        5224.4       0.0X
-infer timestamps from Dataset[String]            100488         100714         198          0.1       10048.8       0.0X
-date strings                                       4572           4584          12          2.2         457.2       0.5X
-parse dates from Dataset[String]                  26749          26768          17          0.4        2674.9       0.1X
-from_json(timestamp)                              71414          71867         556          0.1        7141.4       0.0X
-from_json(date)                                   45322          45549         250          0.2        4532.2       0.1X
+read timestamp text from files                     2316           2324          13          4.3         231.6       1.0X
+read timestamps from files                        43712          43900         165          0.2        4371.2       0.1X
+infer timestamps from files                       19302          19328          38          0.5        1930.2       0.1X
+read date text from files                          2090           2099          11          4.8         209.0       1.1X
+read date from files                              18914          18940          44          0.5        1891.4       0.1X
+timestamp strings                                  3785           3793          11          2.6         378.5       0.6X
+parse timestamps from Dataset[String]             51177          51353         160          0.2        5117.7       0.0X
+infer timestamps from Dataset[String]             27907          28119         186          0.4        2790.7       0.1X
+date strings                                       4446           4452           6          2.2         444.6       0.5X
+parse dates from Dataset[String]                  28124          28172          55          0.4        2812.4       0.1X
+from_json(timestamp)                              71432          71827         354          0.1        7143.2       0.0X
+from_json(date)                                   46497          46651         163          0.2        4649.7       0.0X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 19ec5862e6583..d95d4e08f64a3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2610,7 +2610,9 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
   }
 
   test("inferring timestamp type") {
-    def schemaOf(jsons: String*): StructType = spark.read.json(jsons.toDS).schema
+    def schemaOf(jsons: String*): StructType = {
+      spark.read.option("inferTimestamp", true).json(jsons.toDS).schema
+    }
 
     assert(schemaOf(
       """{"a":"2018-12-17T10:11:12.123-01:00"}""",
@@ -2633,6 +2635,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val timestampsWithFormatPath = s"${dir.getCanonicalPath}/timestampsWithFormat.json"
       val timestampsWithFormat = spark.read
         .option("timestampFormat", "dd/MM/yyyy HH:mm")
+        .option("inferTimestamp", true)
         .json(datesRecords)
       assert(timestampsWithFormat.schema === customSchema)
 
@@ -2645,6 +2648,7 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       val readBack = spark.read
         .option("timestampFormat", "yyyy-MM-dd HH:mm:ss")
         .option(DateTimeUtils.TIMEZONE_OPTION, "UTC")
+        .option("inferTimestamp", true)
         .json(timestampsWithFormatPath)
 
       assert(readBack.schema === customSchema)

From 7f11c8f05478391534f871f7c70f13391b5c69ba Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Thu, 2 Jul 2020 13:56:43 +0900
Subject: [PATCH 0872/1280] [SPARK-32136][SQL] NormalizeFloatingNumbers should
 work on null struct

### What changes were proposed in this pull request?

This patch fixes wrong groupBy result if the grouping key is a null-value struct.

### Why are the changes needed?

`NormalizeFloatingNumbers` reconstructs a struct if input expression is StructType. If the input struct is null, it will reconstruct a struct with null-value fields, instead of null.

### Does this PR introduce _any_ user-facing change?

Yes, fixing incorrect groupBy result.

### How was this patch tested?

Unit test.

Closes #28962 from viirya/SPARK-32136.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3f7780d30d712e6d3894bacb5e80113c7a4bcc09)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../optimizer/NormalizeFloatingNumbers.scala         |  5 +++--
 .../apache/spark/sql/DataFrameAggregateSuite.scala   | 12 ++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index 43738204c6704..8d5dbc7dc90eb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
-import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CreateArray, CreateMap, CreateNamedStruct, CreateStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, KnownFloatingPointNormalized, LambdaFunction, NamedLambdaVariable, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CreateArray, CreateMap, CreateNamedStruct, CreateStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, If, IsNull, KnownFloatingPointNormalized, LambdaFunction, Literal, NamedLambdaVariable, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
 import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery, Window}
@@ -123,7 +123,8 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
       val fields = expr.dataType.asInstanceOf[StructType].fields.indices.map { i =>
         normalize(GetStructField(expr, i))
       }
-      CreateStruct(fields)
+      val struct = CreateStruct(fields)
+      KnownFloatingPointNormalized(If(IsNull(expr), Literal(null, struct.dataType), struct))
 
     case _ if expr.dataType.isInstanceOf[ArrayType] =>
       val ArrayType(et, containsNull) = expr.dataType
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index f7438f3ffec04..09f30bb5e2c77 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -1028,4 +1028,16 @@ class DataFrameAggregateSuite extends QueryTest
       checkAnswer(df, Row("abellina", 2) :: Row("mithunr", 1) :: Nil)
     }
   }
+
+  test("SPARK-32136: NormalizeFloatingNumbers should work on null struct") {
+    val df = Seq(
+      A(None),
+      A(Some(B(None))),
+      A(Some(B(Some(1.0))))).toDF
+    val groupBy = df.groupBy("b").agg(count("*"))
+    checkAnswer(groupBy, Row(null, 1) :: Row(Row(null), 1) :: Row(Row(1.0), 1) :: Nil)
+  }
 }
+
+case class B(c: Option[Double])
+case class A(b: Option[B])

From b6bb158d3c1191aac6f4d27d84d7c791c5063b24 Mon Sep 17 00:00:00 2001
From: pancheng <379377944@qq.com>
Date: Thu, 2 Jul 2020 19:21:11 +0900
Subject: [PATCH 0873/1280] [SPARK-32121][SHUFFLE] Support Windows OS in
 ExecutorDiskUtils

### What changes were proposed in this pull request?
Correct file seprate use in `ExecutorDiskUtils.createNormalizedInternedPathname` on Windows

### Why are the changes needed?
`ExternalShuffleBlockResolverSuite` failed on Windows, see detail at:
https://issues.apache.org/jira/browse/SPARK-32121

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
The existed test suite.

Closes #28940 from pan3793/SPARK-32121.

Lead-authored-by: pancheng <379377944@qq.com>
Co-authored-by: chengpan <cheng.pan@idiaoyan.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 7fda184f0fc39613fb68e912c189c54b93c638e6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../network/shuffle/ExecutorDiskUtils.java    | 18 ++++++++++++---
 .../ExternalShuffleBlockResolverSuite.java    | 22 +++++++++++++------
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
index 13f6046dd856b..6549cac011feb 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExecutorDiskUtils.java
@@ -23,11 +23,19 @@
 
 import com.google.common.annotations.VisibleForTesting;
 
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.spark.network.util.JavaUtils;
 
 public class ExecutorDiskUtils {
 
-  private static final Pattern MULTIPLE_SEPARATORS = Pattern.compile(File.separator + "{2,}");
+  private static final Pattern MULTIPLE_SEPARATORS;
+  static {
+    if (SystemUtils.IS_OS_WINDOWS) {
+      MULTIPLE_SEPARATORS = Pattern.compile("[/\\\\]+");
+    } else {
+      MULTIPLE_SEPARATORS = Pattern.compile("/{2,}");
+    }
+  }
 
   /**
    * Hashes a filename into the corresponding local directory, in a manner consistent with
@@ -50,14 +58,18 @@ public static File getFile(String[] localDirs, int subDirsPerLocalDir, String fi
    * the internal code in java.io.File would normalize it later, creating a new "foo/bar"
    * String copy. Unfortunately, we cannot just reuse the normalization code that java.io.File
    * uses, since it is in the package-private class java.io.FileSystem.
+   *
+   * On Windows, separator "\" is used instead of "/".
+   *
+   * "\\" is a legal character in path name on Unix-like OS, but illegal on Windows.
    */
   @VisibleForTesting
   static String createNormalizedInternedPathname(String dir1, String dir2, String fname) {
     String pathname = dir1 + File.separator + dir2 + File.separator + fname;
     Matcher m = MULTIPLE_SEPARATORS.matcher(pathname);
-    pathname = m.replaceAll("/");
+    pathname = m.replaceAll(Matcher.quoteReplacement(File.separator));
     // A single trailing slash needs to be taken care of separately
-    if (pathname.length() > 1 && pathname.endsWith("/")) {
+    if (pathname.length() > 1 && pathname.charAt(pathname.length() - 1) == File.separatorChar) {
       pathname = pathname.substring(0, pathname.length() - 1);
     }
     return pathname.intern();
diff --git a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
index 09b31430b1eb9..6515b6ca035f7 100644
--- a/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
+++ b/common/network-shuffle/src/test/java/org/apache/spark/network/shuffle/ExternalShuffleBlockResolverSuite.java
@@ -25,6 +25,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.io.CharStreams;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
 import org.apache.spark.network.util.MapConfigProvider;
 import org.apache.spark.network.util.TransportConf;
@@ -146,12 +147,19 @@ public void jsonSerializationOfExecutorRegistration() throws IOException {
 
   @Test
   public void testNormalizeAndInternPathname() {
-    assertPathsMatch("/foo", "bar", "baz", "/foo/bar/baz");
-    assertPathsMatch("//foo/", "bar/", "//baz", "/foo/bar/baz");
-    assertPathsMatch("foo", "bar", "baz///", "foo/bar/baz");
-    assertPathsMatch("/foo/", "/bar//", "/baz", "/foo/bar/baz");
-    assertPathsMatch("/", "", "", "/");
-    assertPathsMatch("/", "/", "/", "/");
+    String sep = File.separator;
+    String expectedPathname = sep + "foo" + sep + "bar" + sep + "baz";
+    assertPathsMatch("/foo", "bar", "baz", expectedPathname);
+    assertPathsMatch("//foo/", "bar/", "//baz", expectedPathname);
+    assertPathsMatch("/foo/", "/bar//", "/baz", expectedPathname);
+    assertPathsMatch("foo", "bar", "baz///", "foo" + sep + "bar" + sep + "baz");
+    assertPathsMatch("/", "", "", sep);
+    assertPathsMatch("/", "/", "/", sep);
+    if (SystemUtils.IS_OS_WINDOWS) {
+      assertPathsMatch("/foo\\/", "bar", "baz", expectedPathname);
+    } else {
+      assertPathsMatch("/foo\\/", "bar", "baz", sep + "foo\\" + sep + "bar" + sep + "baz");
+    }
   }
 
   private void assertPathsMatch(String p1, String p2, String p3, String expectedPathname) {
@@ -160,6 +168,6 @@ private void assertPathsMatch(String p1, String p2, String p3, String expectedPa
     assertEquals(expectedPathname, normPathname);
     File file = new File(normPathname);
     String returnedPath = file.getPath();
-    assertTrue(normPathname == returnedPath);
+    assertEquals(normPathname, returnedPath);
   }
 }

From 5361f76d84de4986aa65a687bece94f4220edd94 Mon Sep 17 00:00:00 2001
From: animenon <animenon@mail.com>
Date: Thu, 2 Jul 2020 21:02:00 +0900
Subject: [PATCH 0874/1280] [MINOR][DOCS] Pyspark getActiveSession docstring

### What changes were proposed in this pull request?

Minor fix so that the documentation of `getActiveSession` is fixed.
The sample code snippet doesn't come up formatted rightly, added spacing for this to be fixed.
Also added return to docs.

### Why are the changes needed?

The sample code is getting mixed up as description in the docs.

[Current Doc Link](http://spark.apache.org/docs/latest/api/python/pyspark.sql.html?highlight=getactivesession#pyspark.sql.SparkSession.getActiveSession)

![image](https://user-images.githubusercontent.com/6907950/86331522-d7b6f800-bc66-11ea-998c-42085f5e5b04.png)

### Does this PR introduce _any_ user-facing change?

Yes, documentation of getActiveSession is fixed.
And added description about return.

### How was this patch tested?

Adding a spacing between description and code seems to fix the issue.

Closes #28978 from animenon/docs_minor.

Authored-by: animenon <animenon@mail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 45fe6b62a73540ff010317fc7518b007206707d6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/session.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index 233f4927389d2..e9486a3d3238a 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -265,7 +265,10 @@ def newSession(self):
     @since(3.0)
     def getActiveSession(cls):
         """
-        Returns the active SparkSession for the current thread, returned by the builder.
+        Returns the active SparkSession for the current thread, returned by the builder
+
+        :return: :class:`SparkSession` if an active session exists for the current thread
+
         >>> s = SparkSession.getActiveSession()
         >>> l = [('Alice', 1)]
         >>> rdd = s.sparkContext.parallelize(l)

From 334b1e8c88d6f10d5e4cde8fa400e34fb04faa39 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 2 Jul 2020 21:04:26 +0900
Subject: [PATCH 0875/1280] [MINOR][TEST][SQL] Make in-limit.sql more robust

### What changes were proposed in this pull request?

For queries like `t1d in (SELECT t2d FROM  t2 ORDER  BY t2c LIMIT 2)`, the result can be non-deterministic as the result of the subquery may output different results (it's not sorted by `t2d` and it has shuffle).

This PR makes the test more robust by sorting the output column.

### Why are the changes needed?

avoid flaky test

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

N/A

Closes #28976 from cloud-fan/small.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f83415629b18d628f72a32285f0afc24f29eaa1e)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql-tests/inputs/subquery/in-subquery/in-limit.sql        | 4 ++--
 .../sql-tests/results/subquery/in-subquery/in-limit.sql.out   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
index 481b5e8cc7700..0a16f118f0455 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-limit.sql
@@ -72,7 +72,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d IN (SELECT t2d
                FROM   t2
-               ORDER  BY t2c
+               ORDER  BY t2c, t2d
                LIMIT 2)
 GROUP  BY t1b
 ORDER  BY t1b DESC NULLS FIRST
@@ -93,7 +93,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d NOT IN (SELECT t2d
                    FROM   t2
-                   ORDER  BY t2b DESC nulls first
+                   ORDER  BY t2b DESC nulls first, t2d
                    LIMIT 1)
 GROUP  BY t1b
 ORDER BY t1b NULLS last
diff --git a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
index 1c335445114c7..e24538b9138ba 100644
--- a/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/subquery/in-subquery/in-limit.sql.out
@@ -103,7 +103,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d IN (SELECT t2d
                FROM   t2
-               ORDER  BY t2c
+               ORDER  BY t2c, t2d
                LIMIT 2)
 GROUP  BY t1b
 ORDER  BY t1b DESC NULLS FIRST
@@ -136,7 +136,7 @@ SELECT Count(DISTINCT( t1a )),
 FROM   t1
 WHERE  t1d NOT IN (SELECT t2d
                    FROM   t2
-                   ORDER  BY t2b DESC nulls first
+                   ORDER  BY t2b DESC nulls first, t2d
                    LIMIT 1)
 GROUP  BY t1b
 ORDER BY t1b NULLS last

From 31e1ea165f3cbb503b05452b448010e81474dcad Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 2 Jul 2020 13:26:57 -0700
Subject: [PATCH 0876/1280] [SPARK-32130][SQL][FOLLOWUP] Enable timestamps
 inference in JsonBenchmark

### What changes were proposed in this pull request?
Set the JSON option `inferTimestamp` to `true` for the cases that measure perf of timestamp inference.

### Why are the changes needed?
The PR https://github.com/apache/spark/pull/28966 disabled timestamp inference by default. As a consequence, some benchmarks don't measure perf of timestamp inference from JSON fields. This PR explicitly enable such inference.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By re-generating results of `JsonBenchmark`.

Closes #28981 from MaxGekk/json-inferTimestamps-disable-by-default-followup.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 42f01e314b4874236544cc8b94bef766269385ee)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../JsonBenchmark-jdk11-results.txt           | 86 +++++++++----------
 sql/core/benchmarks/JsonBenchmark-results.txt | 86 +++++++++----------
 .../datasources/json/JsonBenchmark.scala      |  4 +-
 3 files changed, 88 insertions(+), 88 deletions(-)

diff --git a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
index ff370847a2e3e..2d506f03d9f7e 100644
--- a/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-jdk11-results.txt
@@ -7,106 +7,106 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       69219          69342         116          1.4         692.2       1.0X
-UTF-8 is set                                     143950         143986          55          0.7        1439.5       0.5X
+No encoding                                       73307          73400         141          1.4         733.1       1.0X
+UTF-8 is set                                     143834         143925         152          0.7        1438.3       0.5X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       57828          57913         136          1.7         578.3       1.0X
-UTF-8 is set                                      83649          83711          60          1.2         836.5       0.7X
+No encoding                                       50894          51065         292          2.0         508.9       1.0X
+UTF-8 is set                                      98462          99455        1173          1.0         984.6       0.5X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       64560          65193        1023          0.2        6456.0       1.0X
-UTF-8 is set                                     102925         103174         216          0.1       10292.5       0.6X
+No encoding                                       64011          64969        1001          0.2        6401.1       1.0X
+UTF-8 is set                                     102757         102984         311          0.1       10275.7       0.6X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      131002         132316        1160          0.0      262003.1       1.0X
-UTF-8 is set                                     152128         152371         332          0.0      304256.5       0.9X
+No encoding                                      132559         133561        1010          0.0      265117.3       1.0X
+UTF-8 is set                                     151458         152129         611          0.0      302915.4       0.9X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 19376          19514         160          0.5        1937.6       1.0X
-Select 1 column                                   24089          24156          58          0.4        2408.9       0.8X
+Select 10 columns                                 21148          21202          87          0.5        2114.8       1.0X
+Select 1 column                                   24701          24724          21          0.4        2470.1       0.9X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      8131           8219         103          1.2         813.1       1.0X
-Short column with UTF-8                           13464          13508          44          0.7        1346.4       0.6X
-Wide column without encoding                     108012         108598         914          0.1       10801.2       0.1X
-Wide column with UTF-8                           150988         151369         412          0.1       15098.8       0.1X
+Short column without encoding                      6945           6998          59          1.4         694.5       1.0X
+Short column with UTF-8                           11510          11569          51          0.9        1151.0       0.6X
+Wide column without encoding                      95004          95795         790          0.1        9500.4       0.1X
+Wide column with UTF-8                           149223         149409         276          0.1       14922.3       0.0X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                           753            765          18         13.3          75.3       1.0X
-from_json                                         23182          23446         230          0.4        2318.2       0.0X
-json_tuple                                        31129          31304         181          0.3        3112.9       0.0X
-get_json_object                                   22821          23073         225          0.4        2282.1       0.0X
+Text read                                           649            652           3         15.4          64.9       1.0X
+from_json                                         22284          22393          99          0.4        2228.4       0.0X
+json_tuple                                        32310          32824         484          0.3        3231.0       0.0X
+get_json_object                                   22111          22751         568          0.5        2211.1       0.0X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          3078           3101          26         16.2          61.6       1.0X
-schema inferring                                  30225          30434         333          1.7         604.5       0.1X
-parsing                                           32237          32308          63          1.6         644.7       0.1X
+Text read                                          2894           2903           8         17.3          57.9       1.0X
+schema inferring                                  26724          26785          62          1.9         534.5       0.1X
+parsing                                           37502          37632         131          1.3         750.0       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                         10835          10900          86          4.6         216.7       1.0X
-Schema inferring                                  37720          37805         110          1.3         754.4       0.3X
-Parsing without charset                           35464          35538         100          1.4         709.3       0.3X
-Parsing with UTF-8                                67311          67738         381          0.7        1346.2       0.2X
+Text read                                         10994          11010          16          4.5         219.9       1.0X
+Schema inferring                                  45654          45677          37          1.1         913.1       0.2X
+Parsing without charset                           34476          34559          73          1.5         689.5       0.3X
+Parsing with UTF-8                                56987          57002          13          0.9        1139.7       0.2X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     2208           2222          14          4.5         220.8       1.0X
-to_json(timestamp)                                14299          14570         285          0.7        1429.9       0.2X
-write timestamps to files                         12955          12969          13          0.8        1295.5       0.2X
-Create a dataset of dates                          2297           2323          30          4.4         229.7       1.0X
-to_json(date)                                      8509           8561          74          1.2         850.9       0.3X
-write dates to files                               6786           6827          45          1.5         678.6       0.3X
+Create a dataset of timestamps                     2150           2188          35          4.7         215.0       1.0X
+to_json(timestamp)                                17874          18080         294          0.6        1787.4       0.1X
+write timestamps to files                         12518          12538          34          0.8        1251.8       0.2X
+Create a dataset of dates                          2298           2310          18          4.4         229.8       0.9X
+to_json(date)                                     11673          11703          27          0.9        1167.3       0.2X
+write dates to files                               7121           7135          12          1.4         712.1       0.3X
 
 OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2598           2613          18          3.8         259.8       1.0X
-read timestamps from files                        42007          42028          19          0.2        4200.7       0.1X
-infer timestamps from files                       18102          18120          28          0.6        1810.2       0.1X
-read date text from files                          2355           2360           5          4.2         235.5       1.1X
-read date from files                              17420          17458          33          0.6        1742.0       0.1X
-timestamp strings                                  3099           3101           3          3.2         309.9       0.8X
-parse timestamps from Dataset[String]             48188          48215          25          0.2        4818.8       0.1X
-infer timestamps from Dataset[String]             22929          22988         102          0.4        2292.9       0.1X
-date strings                                       4090           4103          11          2.4         409.0       0.6X
-parse dates from Dataset[String]                  24952          25068         139          0.4        2495.2       0.1X
-from_json(timestamp)                              66038          66352         413          0.2        6603.8       0.0X
-from_json(date)                                   43755          43782          27          0.2        4375.5       0.1X
+read timestamp text from files                     2616           2641          34          3.8         261.6       1.0X
+read timestamps from files                        37481          37517          58          0.3        3748.1       0.1X
+infer timestamps from files                       84774          84964         201          0.1        8477.4       0.0X
+read date text from files                          2362           2365           3          4.2         236.2       1.1X
+read date from files                              16583          16612          29          0.6        1658.3       0.2X
+timestamp strings                                  3927           3963          40          2.5         392.7       0.7X
+parse timestamps from Dataset[String]             52827          53004         243          0.2        5282.7       0.0X
+infer timestamps from Dataset[String]            101108         101644         769          0.1       10110.8       0.0X
+date strings                                       4886           4906          26          2.0         488.6       0.5X
+parse dates from Dataset[String]                  27623          27694          62          0.4        2762.3       0.1X
+from_json(timestamp)                              71764          71887         124          0.1        7176.4       0.0X
+from_json(date)                                   46200          46314          99          0.2        4620.0       0.1X
 
 
diff --git a/sql/core/benchmarks/JsonBenchmark-results.txt b/sql/core/benchmarks/JsonBenchmark-results.txt
index 0e4ce9003f70b..c22118f91b3fc 100644
--- a/sql/core/benchmarks/JsonBenchmark-results.txt
+++ b/sql/core/benchmarks/JsonBenchmark-results.txt
@@ -7,106 +7,106 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON schema inferring:                    Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       64950          65182         306          1.5         649.5       1.0X
-UTF-8 is set                                     129566         129796         229          0.8        1295.7       0.5X
+No encoding                                       63839          64000         263          1.6         638.4       1.0X
+UTF-8 is set                                     124633         124945         429          0.8        1246.3       0.5X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a short column:                     Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       50896          51277         372          2.0         509.0       1.0X
-UTF-8 is set                                      89712          89763          49          1.1         897.1       0.6X
+No encoding                                       51720          51901         157          1.9         517.2       1.0X
+UTF-8 is set                                      91161          91190          25          1.1         911.6       0.6X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 count a wide column:                      Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                       59415          59785         372          0.2        5941.5       1.0X
-UTF-8 is set                                     103059         103165         156          0.1       10305.9       0.6X
+No encoding                                       58486          59038         714          0.2        5848.6       1.0X
+UTF-8 is set                                     103045         103350         358          0.1       10304.5       0.6X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 select wide row:                          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-No encoding                                      132951         133122         288          0.0      265901.9       1.0X
-UTF-8 is set                                     149318         149441         107          0.0      298635.3       0.9X
+No encoding                                      134909         135024         105          0.0      269818.6       1.0X
+UTF-8 is set                                     154418         154593         155          0.0      308836.7       0.9X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Select a subset of 10 columns:            Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Select 10 columns                                 18491          18552          85          0.5        1849.1       1.0X
-Select 1 column                                   25908          25946          65          0.4        2590.8       0.7X
+Select 10 columns                                 19538          19620          70          0.5        1953.8       1.0X
+Select 1 column                                   26142          26159          15          0.4        2614.2       0.7X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 creation of JSON parser per line:         Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Short column without encoding                      9264           9307          49          1.1         926.4       1.0X
-Short column with UTF-8                           14707          14727          17          0.7        1470.7       0.6X
-Wide column without encoding                     141138         141347         276          0.1       14113.8       0.1X
-Wide column with UTF-8                           179601         180035         664          0.1       17960.1       0.1X
+Short column without encoding                      8103           8162          53          1.2         810.3       1.0X
+Short column with UTF-8                           13104          13150          58          0.8        1310.4       0.6X
+Wide column without encoding                     135280         135593         375          0.1       13528.0       0.1X
+Wide column with UTF-8                           175189         175483         278          0.1       17518.9       0.0X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 JSON functions:                           Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          1173           1184           9          8.5         117.3       1.0X
-from_json                                         23432          23738         338          0.4        2343.2       0.1X
-json_tuple                                        32573          32851         358          0.3        3257.3       0.0X
-get_json_object                                   22442          22489          47          0.4        2244.2       0.1X
+Text read                                          1225           1234           8          8.2         122.5       1.0X
+from_json                                         22482          22552          95          0.4        2248.2       0.1X
+json_tuple                                        30203          30338         146          0.3        3020.3       0.0X
+get_json_object                                   22219          22245          26          0.5        2221.9       0.1X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Dataset of json strings:                  Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          5656           5680          31          8.8         113.1       1.0X
-schema inferring                                  33283          33337          64          1.5         665.7       0.2X
-parsing                                           41771          41929         178          1.2         835.4       0.1X
+Text read                                          5897           5904          10          8.5         117.9       1.0X
+schema inferring                                  30282          30340          50          1.7         605.6       0.2X
+parsing                                           33304          33577         289          1.5         666.1       0.2X
 
 Preparing data for benchmarking ...
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Json files in the per-line mode:          Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Text read                                          9626           9668          39          5.2         192.5       1.0X
-Schema inferring                                  39489          39579          91          1.3         789.8       0.2X
-Parsing without charset                           38096          38232         125          1.3         761.9       0.3X
-Parsing with UTF-8                                64565          64725         165          0.8        1291.3       0.1X
+Text read                                          9710           9757          80          5.1         194.2       1.0X
+Schema inferring                                  35929          35939           9          1.4         718.6       0.3X
+Parsing without charset                           39175          39227          87          1.3         783.5       0.2X
+Parsing with UTF-8                                59188          59294         109          0.8        1183.8       0.2X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Write dates and timestamps:               Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-Create a dataset of timestamps                     1898           1912          13          5.3         189.8       1.0X
-to_json(timestamp)                                20011          20092         119          0.5        2001.1       0.1X
-write timestamps to files                         13388          13427          35          0.7        1338.8       0.1X
-Create a dataset of dates                          2351           2368          18          4.3         235.1       0.8X
-to_json(date)                                     11884          11913          40          0.8        1188.4       0.2X
-write dates to files                               7317           7326           9          1.4         731.7       0.3X
+Create a dataset of timestamps                     1967           1977           9          5.1         196.7       1.0X
+to_json(timestamp)                                17086          17304         371          0.6        1708.6       0.1X
+write timestamps to files                         12691          12716          28          0.8        1269.1       0.2X
+Create a dataset of dates                          2192           2217          39          4.6         219.2       0.9X
+to_json(date)                                     10541          10656         137          0.9        1054.1       0.2X
+write dates to files                               7259           7311          46          1.4         725.9       0.3X
 
 OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws
 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz
 Read dates and timestamps:                Best Time(ms)   Avg Time(ms)   Stdev(ms)    Rate(M/s)   Per Row(ns)   Relative
 ------------------------------------------------------------------------------------------------------------------------
-read timestamp text from files                     2316           2324          13          4.3         231.6       1.0X
-read timestamps from files                        43712          43900         165          0.2        4371.2       0.1X
-infer timestamps from files                       19302          19328          38          0.5        1930.2       0.1X
-read date text from files                          2090           2099          11          4.8         209.0       1.1X
-read date from files                              18914          18940          44          0.5        1891.4       0.1X
-timestamp strings                                  3785           3793          11          2.6         378.5       0.6X
-parse timestamps from Dataset[String]             51177          51353         160          0.2        5117.7       0.0X
-infer timestamps from Dataset[String]             27907          28119         186          0.4        2790.7       0.1X
-date strings                                       4446           4452           6          2.2         444.6       0.5X
-parse dates from Dataset[String]                  28124          28172          55          0.4        2812.4       0.1X
-from_json(timestamp)                              71432          71827         354          0.1        7143.2       0.0X
-from_json(date)                                   46497          46651         163          0.2        4649.7       0.0X
+read timestamp text from files                     2318           2326          13          4.3         231.8       1.0X
+read timestamps from files                        43345          43627         258          0.2        4334.5       0.1X
+infer timestamps from files                       89570          89621          59          0.1        8957.0       0.0X
+read date text from files                          2099           2107           9          4.8         209.9       1.1X
+read date from files                              18000          18065          98          0.6        1800.0       0.1X
+timestamp strings                                  3937           3956          32          2.5         393.7       0.6X
+parse timestamps from Dataset[String]             56001          56429         539          0.2        5600.1       0.0X
+infer timestamps from Dataset[String]            109410         109963         559          0.1       10941.0       0.0X
+date strings                                       4530           4540           9          2.2         453.0       0.5X
+parse dates from Dataset[String]                  29723          29767          72          0.3        2972.3       0.1X
+from_json(timestamp)                              74106          74619         728          0.1        7410.6       0.0X
+from_json(date)                                   46599          46632          32          0.2        4659.9       0.0X
 
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
index 56930880ed5da..0dbd6b5754afb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmark.scala
@@ -430,7 +430,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
       }
 
       readBench.addCase("infer timestamps from files", numIters) { _ =>
-        spark.read.json(timestampDir).noop()
+        spark.read.option("inferTimestamp", true).json(timestampDir).noop()
       }
 
       val dateSchema = new StructType().add("date", DateType)
@@ -460,7 +460,7 @@ object JsonBenchmark extends SqlBasedBenchmark {
       }
 
       readBench.addCase("infer timestamps from Dataset[String]", numIters) { _ =>
-        spark.read.json(timestampStr).noop()
+        spark.read.option("inferTimestamp", true).json(timestampStr).noop()
       }
 
       def dateStr: Dataset[String] = {

From fc2660c302b0c83a9a8a5bec3cc7ae28f8fecdd6 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 4 Jul 2020 19:01:07 -0700
Subject: [PATCH 0877/1280] [SPARK-32171][SQL][DOCS] Change file locations for
 use db and refresh table

### What changes were proposed in this pull request?

docs/sql-ref-syntax-qry-select-usedb.md -> docs/sql-ref-syntax-ddl-usedb.md
docs/sql-ref-syntax-aux-refresh-table.md -> docs/sql-ref-syntax-aux-cache-refresh-table.md

### Why are the changes needed?
usedb belongs to DDL. Its location should be consistent with other DDL commands file locations
similar reason for refresh table

### Does this PR introduce _any_ user-facing change?
before change, when clicking USE DATABASE, the side bar menu shows select commands
<img width="1200" alt="Screen Shot 2020-07-04 at 9 05 35 AM" src="https://user-images.githubusercontent.com/13592258/86516696-b45f8a80-bdd7-11ea-8dba-3a5cca22aad3.png">

after change, when clicking USE DATABASE, the side bar menu shows DDL commands
<img width="1120" alt="Screen Shot 2020-07-04 at 9 06 06 AM" src="https://user-images.githubusercontent.com/13592258/86516703-bf1a1f80-bdd7-11ea-8a90-ae7eaaafd44c.png">

before change, when clicking refresh table, the side bar menu shows Auxiliary statements
<img width="1200" alt="Screen Shot 2020-07-04 at 9 30 40 AM" src="https://user-images.githubusercontent.com/13592258/86516877-3d2af600-bdd9-11ea-9568-0a6f156f57da.png">

after change, when clicking refresh table, the side bar menu shows Cache statements
<img width="1199" alt="Screen Shot 2020-07-04 at 9 35 21 AM" src="https://user-images.githubusercontent.com/13592258/86516937-b4f92080-bdd9-11ea-8ad1-5f5a7f58d76b.png">

### How was this patch tested?
Manually build and check

Closes #28995 from huaxingao/docs_fix.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
(cherry picked from commit 492d5d174a435c624bd87af9ee3621f4f1c8d1c5)
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
---
 docs/_data/menu-sql.yaml                                      | 4 ++--
 docs/sql-ref-syntax-aux-cache-cache-table.md                  | 2 +-
 docs/sql-ref-syntax-aux-cache-clear-cache.md                  | 2 +-
 ...esh-table.md => sql-ref-syntax-aux-cache-refresh-table.md} | 0
 docs/sql-ref-syntax-aux-cache-refresh.md                      | 2 +-
 docs/sql-ref-syntax-aux-cache-uncache-table.md                | 2 +-
 docs/sql-ref-syntax-aux-cache.md                              | 2 +-
 ...syntax-qry-select-usedb.md => sql-ref-syntax-ddl-usedb.md} | 0
 docs/sql-ref-syntax-ddl.md                                    | 2 +-
 docs/sql-ref-syntax.md                                        | 4 ++--
 10 files changed, 10 insertions(+), 10 deletions(-)
 rename docs/{sql-ref-syntax-aux-refresh-table.md => sql-ref-syntax-aux-cache-refresh-table.md} (100%)
 rename docs/{sql-ref-syntax-qry-select-usedb.md => sql-ref-syntax-ddl-usedb.md} (100%)

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 219e6809a96f0..eea657e684495 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -139,7 +139,7 @@
             - text: REPAIR TABLE
               url: sql-ref-syntax-ddl-repair-table.html
             - text: USE DATABASE
-              url: sql-ref-syntax-qry-select-usedb.html
+              url: sql-ref-syntax-ddl-usedb.html
         - text: Data Manipulation Statements
           url: sql-ref-syntax-dml.html
           subitems:
@@ -207,7 +207,7 @@
                 - text: CLEAR CACHE
                   url: sql-ref-syntax-aux-cache-clear-cache.html
                 - text: REFRESH TABLE
-                  url: sql-ref-syntax-aux-refresh-table.html
+                  url: sql-ref-syntax-aux-cache-refresh-table.html
                 - text: REFRESH
                   url: sql-ref-syntax-aux-cache-refresh.html
             - text: DESCRIBE
diff --git a/docs/sql-ref-syntax-aux-cache-cache-table.md b/docs/sql-ref-syntax-aux-cache-cache-table.md
index 193e209d792b3..fdef3d657dfa3 100644
--- a/docs/sql-ref-syntax-aux-cache-cache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-cache-table.md
@@ -78,5 +78,5 @@ CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testDat
 
 * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
 * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
 * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache-clear-cache.md b/docs/sql-ref-syntax-aux-cache-clear-cache.md
index ee33e6a98296d..a27cd83c146a3 100644
--- a/docs/sql-ref-syntax-aux-cache-clear-cache.md
+++ b/docs/sql-ref-syntax-aux-cache-clear-cache.md
@@ -39,5 +39,5 @@ CLEAR CACHE;
 
 * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
 * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-refresh-table.md b/docs/sql-ref-syntax-aux-cache-refresh-table.md
similarity index 100%
rename from docs/sql-ref-syntax-aux-refresh-table.md
rename to docs/sql-ref-syntax-aux-cache-refresh-table.md
diff --git a/docs/sql-ref-syntax-aux-cache-refresh.md b/docs/sql-ref-syntax-aux-cache-refresh.md
index 82bc12da5d1ac..b10e6fb47aaf7 100644
--- a/docs/sql-ref-syntax-aux-cache-refresh.md
+++ b/docs/sql-ref-syntax-aux-cache-refresh.md
@@ -53,4 +53,4 @@ REFRESH "hdfs://path/to/table";
 * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
 * [UNCACHE TABLE](sql-ref-syntax-aux-cache-uncache-table.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
diff --git a/docs/sql-ref-syntax-aux-cache-uncache-table.md b/docs/sql-ref-syntax-aux-cache-uncache-table.md
index c5a8fbbe08281..96a691e4c3931 100644
--- a/docs/sql-ref-syntax-aux-cache-uncache-table.md
+++ b/docs/sql-ref-syntax-aux-cache-uncache-table.md
@@ -48,5 +48,5 @@ UNCACHE TABLE t1;
 
 * [CACHE TABLE](sql-ref-syntax-aux-cache-cache-table.html)
 * [CLEAR CACHE](sql-ref-syntax-aux-cache-clear-cache.html)
-* [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+* [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
 * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
diff --git a/docs/sql-ref-syntax-aux-cache.md b/docs/sql-ref-syntax-aux-cache.md
index 418b8cc3403b5..0ccb1c61a0da5 100644
--- a/docs/sql-ref-syntax-aux-cache.md
+++ b/docs/sql-ref-syntax-aux-cache.md
@@ -22,5 +22,5 @@ license: |
  * [CACHE TABLE statement](sql-ref-syntax-aux-cache-cache-table.html)
  * [UNCACHE TABLE statement](sql-ref-syntax-aux-cache-uncache-table.html)
  * [CLEAR CACHE statement](sql-ref-syntax-aux-cache-clear-cache.html)
- * [REFRESH TABLE statement](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH TABLE statement](sql-ref-syntax-aux-cache-refresh-table.html)
  * [REFRESH statement](sql-ref-syntax-aux-cache-refresh.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-usedb.md b/docs/sql-ref-syntax-ddl-usedb.md
similarity index 100%
rename from docs/sql-ref-syntax-qry-select-usedb.md
rename to docs/sql-ref-syntax-ddl-usedb.md
diff --git a/docs/sql-ref-syntax-ddl.md b/docs/sql-ref-syntax-ddl.md
index 82fbf0498a20f..cb3e04c0ec910 100644
--- a/docs/sql-ref-syntax-ddl.md
+++ b/docs/sql-ref-syntax-ddl.md
@@ -34,4 +34,4 @@ Data Definition Statements are used to create or modify the structure of databas
  * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
  * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
  * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
- * [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+ * [USE DATABASE](sql-ref-syntax-ddl-usedb.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index d78a01fd655a2..4bf1858428d98 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -36,7 +36,7 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [DROP VIEW](sql-ref-syntax-ddl-drop-view.html)
  * [REPAIR TABLE](sql-ref-syntax-ddl-repair-table.html)
  * [TRUNCATE TABLE](sql-ref-syntax-ddl-truncate-table.html)
- * [USE DATABASE](sql-ref-syntax-qry-select-usedb.html)
+ * [USE DATABASE](sql-ref-syntax-ddl-usedb.html)
 
 ### DML Statements
 
@@ -82,7 +82,7 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
  * [LIST FILE](sql-ref-syntax-aux-resource-mgmt-list-file.html)
  * [LIST JAR](sql-ref-syntax-aux-resource-mgmt-list-jar.html)
  * [REFRESH](sql-ref-syntax-aux-cache-refresh.html)
- * [REFRESH TABLE](sql-ref-syntax-aux-refresh-table.html)
+ * [REFRESH TABLE](sql-ref-syntax-aux-cache-refresh-table.html)
  * [RESET](sql-ref-syntax-aux-conf-mgmt-reset.html)
  * [SET](sql-ref-syntax-aux-conf-mgmt-set.html)
  * [SHOW COLUMNS](sql-ref-syntax-aux-show-columns.html)

From 6a4200c057b4d4827099715969603cd92876336a Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Mon, 6 Jul 2020 21:39:41 +0900
Subject: [PATCH 0878/1280] [SPARK-32162][PYTHON][TESTS] Improve error message
 of Pandas grouped map test with window

### What changes were proposed in this pull request?

Improve the error message in test GroupedMapInPandasTests.test_grouped_over_window_with_key to show the incorrect values.

### Why are the changes needed?

This test failure has come up often in Arrow testing because it tests a struct  with timestamp values through a Pandas UDF. The current error message is not helpful as it doesn't show the incorrect values, only that it failed. This change will instead raise an assertion error with the incorrect values on a failure.

Before:

```
======================================================================
FAIL: test_grouped_over_window_with_key (pyspark.sql.tests.test_pandas_grouped_map.GroupedMapInPandasTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/spark/python/pyspark/sql/tests/test_pandas_grouped_map.py", line 588, in test_grouped_over_window_with_key
    self.assertTrue(all([r[0] for r in result]))
AssertionError: False is not true
```

After:
```
======================================================================
ERROR: test_grouped_over_window_with_key (pyspark.sql.tests.test_pandas_grouped_map.GroupedMapInPandasTests)
----------------------------------------------------------------------
...
AssertionError: {'start': datetime.datetime(2018, 3, 20, 0, 0), 'end': datetime.datetime(2018, 3, 25, 0, 0)}, != {'start': datetime.datetime(2020, 3, 20, 0, 0), 'end': datetime.datetime(2020, 3, 25, 0, 0)}
```

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Improved existing test

Closes #28987 from BryanCutler/pandas-grouped-map-test-output-SPARK-32162.

Authored-by: Bryan Cutler <cutlerb@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/tests/test_pandas_grouped_map.py      | 57 ++++++++++++-------
 1 file changed, 35 insertions(+), 22 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index 76119432662ba..cc6167e619285 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -545,13 +545,13 @@ def f(pdf):
 
     def test_grouped_over_window_with_key(self):
 
-        data = [(0, 1, "2018-03-10T00:00:00+00:00", False),
-                (1, 2, "2018-03-11T00:00:00+00:00", False),
-                (2, 2, "2018-03-12T00:00:00+00:00", False),
-                (3, 3, "2018-03-15T00:00:00+00:00", False),
-                (4, 3, "2018-03-16T00:00:00+00:00", False),
-                (5, 3, "2018-03-17T00:00:00+00:00", False),
-                (6, 3, "2018-03-21T00:00:00+00:00", False)]
+        data = [(0, 1, "2018-03-10T00:00:00+00:00", [0]),
+                (1, 2, "2018-03-11T00:00:00+00:00", [0]),
+                (2, 2, "2018-03-12T00:00:00+00:00", [0]),
+                (3, 3, "2018-03-15T00:00:00+00:00", [0]),
+                (4, 3, "2018-03-16T00:00:00+00:00", [0]),
+                (5, 3, "2018-03-17T00:00:00+00:00", [0]),
+                (6, 3, "2018-03-21T00:00:00+00:00", [0])]
 
         expected_window = [
             {'start': datetime.datetime(2018, 3, 10, 0, 0),
@@ -562,30 +562,43 @@ def test_grouped_over_window_with_key(self):
              'end': datetime.datetime(2018, 3, 25, 0, 0)},
         ]
 
-        expected = {0: (1, expected_window[0]),
-                    1: (2, expected_window[0]),
-                    2: (2, expected_window[0]),
-                    3: (3, expected_window[1]),
-                    4: (3, expected_window[1]),
-                    5: (3, expected_window[1]),
-                    6: (3, expected_window[2])}
+        expected_key = {0: (1, expected_window[0]),
+                        1: (2, expected_window[0]),
+                        2: (2, expected_window[0]),
+                        3: (3, expected_window[1]),
+                        4: (3, expected_window[1]),
+                        5: (3, expected_window[1]),
+                        6: (3, expected_window[2])}
+
+        # id -> array of group with len of num records in window
+        expected = {0: [1],
+                    1: [2, 2],
+                    2: [2, 2],
+                    3: [3, 3, 3],
+                    4: [3, 3, 3],
+                    5: [3, 3, 3],
+                    6: [3]}
 
         df = self.spark.createDataFrame(data, ['id', 'group', 'ts', 'result'])
         df = df.select(col('id'), col('group'), col('ts').cast('timestamp'), col('result'))
 
-        @pandas_udf(df.schema, PandasUDFType.GROUPED_MAP)
         def f(key, pdf):
             group = key[0]
             window_range = key[1]
-            # Result will be True if group and window range equal to expected
-            is_expected = pdf.id.apply(lambda id: (expected[id][0] == group and
-                                                   expected[id][1] == window_range))
-            return pdf.assign(result=is_expected)
 
-        result = df.groupby('group', window('ts', '5 days')).apply(f).select('result').collect()
+            # Make sure the key with group and window values are correct
+            for _, i in pdf.id.iteritems():
+                assert expected_key[i][0] == group, "{} != {}".format(expected_key[i][0], group)
+                assert expected_key[i][1] == window_range, \
+                    "{} != {}".format(expected_key[i][1], window_range)
 
-        # Check that all group and window_range values from udf matched expected
-        self.assertTrue(all([r[0] for r in result]))
+            return pdf.assign(result=[[group] * len(pdf)] * len(pdf))
+
+        result = df.groupby('group', window('ts', '5 days')).applyInPandas(f, df.schema)\
+            .select('id', 'result').collect()
+
+        for r in result:
+            self.assertListEqual(expected[r[0]], r[1])
 
     def test_case_insensitive_grouping_column(self):
         # SPARK-31915: case-insensitive grouping column should work.

From a9ca4abb092bb5d293fc00842e98839fd2d632cf Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 6 Jul 2020 20:07:33 -0700
Subject: [PATCH 0879/1280] [SPARK-32167][SQL] Fix GetArrayStructFields to
 respect inner field's nullability together

### What changes were proposed in this pull request?

Fix nullability of `GetArrayStructFields`. It should consider both the original array's `containsNull` and the inner field's nullability.

### Why are the changes needed?

Fix a correctness issue.

### Does this PR introduce _any_ user-facing change?

Yes. See the added test.

### How was this patch tested?

a new UT and end-to-end test

Closes #28992 from cloud-fan/bug.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 5d296ed39e3dd79ddb10c68657e773adba40a5e0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/complexTypeExtractors.scala   |  2 +-
 .../expressions/ComplexTypeSuite.scala        | 26 +++++++++++++++++++
 .../expressions/SelectedFieldSuite.scala      |  8 +++---
 .../apache/spark/sql/ComplexTypesSuite.scala  | 11 ++++++++
 4 files changed, 42 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 9c600c9d39cf7..89ff4facd25a9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -57,7 +57,7 @@ object ExtractValue {
         val fieldName = v.toString
         val ordinal = findField(fields, fieldName, resolver)
         GetArrayStructFields(child, fields(ordinal).copy(name = fieldName),
-          ordinal, fields.length, containsNull)
+          ordinal, fields.length, containsNull || fields(ordinal).nullable)
 
       case (_: ArrayType, _) => GetArrayItem(child, extraction)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 3df7d02fb6604..dbe43709d1d35 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.catalyst.expressions
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedExtractValue}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
@@ -159,6 +160,31 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(getArrayStructFields(nullArrayStruct, "a"), null)
   }
 
+  test("SPARK-32167: nullability of GetArrayStructFields") {
+    val resolver = SQLConf.get.resolver
+
+    val array1 = ArrayType(
+      new StructType().add("a", "int", nullable = true),
+      containsNull = false)
+    val data1 = Literal.create(Seq(Row(null)), array1)
+    val get1 = ExtractValue(data1, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(get1.containsNull)
+
+    val array2 = ArrayType(
+      new StructType().add("a", "int", nullable = false),
+      containsNull = true)
+    val data2 = Literal.create(Seq(null), array2)
+    val get2 = ExtractValue(data2, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(get2.containsNull)
+
+    val array3 = ArrayType(
+      new StructType().add("a", "int", nullable = false),
+      containsNull = false)
+    val data3 = Literal.create(Seq(Row(1)), array3)
+    val get3 = ExtractValue(data3, Literal("a"), resolver).asInstanceOf[GetArrayStructFields]
+    assert(!get3.containsNull)
+  }
+
   test("CreateArray") {
     val intSeq = Seq(5, 10, 15, 20, 25)
     val longSeq = intSeq.map(_.toLong)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
index 3c826e812b5cc..76d6890cc8f6f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SelectedFieldSuite.scala
@@ -254,13 +254,13 @@ class SelectedFieldSuite extends AnalysisTest {
     StructField("col3", ArrayType(StructType(
       StructField("field1", StructType(
         StructField("subfield1", IntegerType, nullable = false) :: Nil))
-        :: Nil), containsNull = false), nullable = false)
+        :: Nil), containsNull = true), nullable = false)
   }
 
   testSelect(arrayWithStructAndMap, "col3.field2['foo'] as foo") {
     StructField("col3", ArrayType(StructType(
       StructField("field2", MapType(StringType, IntegerType, valueContainsNull = false))
-        :: Nil), containsNull = false), nullable = false)
+        :: Nil), containsNull = true), nullable = false)
   }
 
   //  |-- col1: string (nullable = false)
@@ -471,7 +471,7 @@ class SelectedFieldSuite extends AnalysisTest {
   testSelect(mapWithArrayOfStructKey, "map_keys(col2)[0].field1 as foo") {
     StructField("col2", MapType(
       ArrayType(StructType(
-        StructField("field1", StringType) :: Nil), containsNull = false),
+        StructField("field1", StringType) :: Nil), containsNull = true),
       ArrayType(StructType(
         StructField("field3", StructType(
           StructField("subfield3", IntegerType) ::
@@ -482,7 +482,7 @@ class SelectedFieldSuite extends AnalysisTest {
     StructField("col2", MapType(
       ArrayType(StructType(
         StructField("field2", StructType(
-          StructField("subfield1", IntegerType) :: Nil)) :: Nil), containsNull = false),
+          StructField("subfield1", IntegerType) :: Nil)) :: Nil), containsNull = true),
       ArrayType(StructType(
         StructField("field3", StructType(
           StructField("subfield3", IntegerType) ::
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
index 6b503334f9f23..bdcf7230e3211 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
@@ -17,11 +17,15 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.{ArrayType, StructType}
 
 class ComplexTypesSuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
 
   override def beforeAll(): Unit = {
     super.beforeAll()
@@ -106,4 +110,11 @@ class ComplexTypesSuite extends QueryTest with SharedSparkSession {
     checkAnswer(df1, Row(10, 12) :: Row(11, 13) :: Nil)
     checkNamedStruct(df.queryExecution.optimizedPlan, expectedCount = 0)
   }
+
+  test("SPARK-32167: get field from an array of struct") {
+    val innerStruct = new StructType().add("i", "int", nullable = true)
+    val schema = new StructType().add("arr", ArrayType(innerStruct, containsNull = false))
+    val df = spark.createDataFrame(List(Row(Seq(Row(1), Row(null)))).asJava, schema)
+    checkAnswer(df.select($"arr".getField("i")), Row(Seq(1, null)))
+  }
 }

From b125dedfa954c5209c38662e253a7ad5e197a47d Mon Sep 17 00:00:00 2001
From: Ali Smesseim <ali.smesseim@databricks.com>
Date: Wed, 8 Jul 2020 09:28:16 +0900
Subject: [PATCH 0880/1280] [SPARK-32057][SQL][TEST-HIVE1.2][TEST-HADOOP2.7]
 ExecuteStatement: cancel and close should not transiently ERROR

### What changes were proposed in this pull request?
#28671 introduced a change where the order in which CANCELED state for SparkExecuteStatementOperation is set was changed. Before setting the state to CANCELED, `cleanup()` was called which kills the jobs, causing an exception to be thrown inside `execute()`. This causes the state to transiently become ERROR before being set to CANCELED. This PR fixes the order.

### Why are the changes needed?
Bug: wrong operation state is set.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Unit test in SparkExecuteStatementOperationSuite.scala.

Closes #28912 from alismess-db/execute-statement-operation-cleanup-order.

Authored-by: Ali Smesseim <ali.smesseim@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8b0a54e6ffadd6326fd254f67ebeb1673eb8f33b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 project/SparkBuild.scala                      |  1 +
 .../SparkExecuteStatementOperation.scala      |  2 +-
 .../hive/thriftserver/SparkOperation.scala    |  2 +-
 .../SparkExecuteStatementOperationSuite.scala | 81 ++++++++++++++++++-
 4 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 50d1527599b71..f2db38922c630 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -475,6 +475,7 @@ object SparkParallelTestGrouping {
     "org.apache.spark.ml.classification.LogisticRegressionSuite",
     "org.apache.spark.ml.classification.LinearSVCSuite",
     "org.apache.spark.sql.SQLQueryTestSuite",
+    "org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperationSuite",
     "org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite",
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
     "org.apache.spark.sql.hive.thriftserver.ui.ThriftServerPageSuite",
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
index b193c73563ae0..33f7f21276611 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperation.scala
@@ -336,8 +336,8 @@ private[hive] class SparkExecuteStatementOperation(
     synchronized {
       if (!getStatus.getState.isTerminal) {
         logInfo(s"Cancel query with $statementId")
-        cleanup()
         setState(OperationState.CANCELED)
+        cleanup()
         HiveThriftServer2.eventManager.onStatementCanceled(statementId)
       }
     }
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
index 3da568cfa256e..59516dc26bae6 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkOperation.scala
@@ -46,8 +46,8 @@ private[hive] trait SparkOperation extends Operation with Logging {
   }
 
   abstract override def close(): Unit = {
-    cleanup()
     super.close()
+    cleanup()
     logInfo(s"Close statement with $statementId")
     HiveThriftServer2.eventManager.onOperationClosed(statementId)
   }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
index 13df3fabc4919..4c2f29e0bf394 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/SparkExecuteStatementOperationSuite.scala
@@ -17,10 +17,25 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import java.util
+import java.util.concurrent.Semaphore
+
+import scala.concurrent.duration._
+
+import org.apache.hadoop.hive.conf.HiveConf
+import org.apache.hive.service.cli.OperationState
+import org.apache.hive.service.cli.session.{HiveSession, HiveSessionImpl}
+import org.mockito.Mockito.{doReturn, mock, spy, when, RETURNS_DEEP_STUBS}
+import org.mockito.invocation.InvocationOnMock
+
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.apache.spark.sql.hive.thriftserver.ui.HiveThriftServer2EventManager
+import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{IntegerType, NullType, StringType, StructField, StructType}
 
-class SparkExecuteStatementOperationSuite extends SparkFunSuite {
+class SparkExecuteStatementOperationSuite extends SparkFunSuite with SharedSparkSession {
+
   test("SPARK-17112 `select null` via JDBC triggers IllegalArgumentException in ThriftServer") {
     val field1 = StructField("NULL", NullType)
     val field2 = StructField("(IF(true, NULL, NULL))", NullType)
@@ -42,4 +57,68 @@ class SparkExecuteStatementOperationSuite extends SparkFunSuite {
     assert(columns.get(1).getType().getName == "INT")
     assert(columns.get(1).getComment() == "")
   }
+
+  Seq(
+    (OperationState.CANCELED, (_: SparkExecuteStatementOperation).cancel()),
+    (OperationState.CLOSED, (_: SparkExecuteStatementOperation).close())
+  ).foreach { case (finalState, transition) =>
+    test("SPARK-32057 SparkExecuteStatementOperation should not transiently become ERROR " +
+      s"before being set to $finalState") {
+      val hiveSession = new HiveSessionImpl(ThriftserverShimUtils.testedProtocolVersions.head,
+      "username", "password", new HiveConf, "ip address")
+      hiveSession.open(new util.HashMap)
+
+      HiveThriftServer2.eventManager = mock(classOf[HiveThriftServer2EventManager])
+
+      val spySqlContext = spy(sqlContext)
+
+      // When cancel() is called on the operation, cleanup causes an exception to be thrown inside
+      // of execute(). This should not cause the state to become ERROR. The exception here will be
+      // triggered in our custom cleanup().
+      val signal = new Semaphore(0)
+      val dataFrame = mock(classOf[DataFrame], RETURNS_DEEP_STUBS)
+      when(dataFrame.collect()).thenAnswer((_: InvocationOnMock) => {
+        signal.acquire()
+        throw new RuntimeException("Operation was cancelled by test cleanup.")
+      })
+      val statement = "stmt"
+      doReturn(dataFrame, Nil: _*).when(spySqlContext).sql(statement)
+
+      val executeStatementOperation = new MySparkExecuteStatementOperation(spySqlContext,
+        hiveSession, statement, signal, finalState)
+
+      val run = new Thread() {
+        override def run(): Unit = executeStatementOperation.runInternal()
+      }
+      assert(executeStatementOperation.getStatus.getState === OperationState.INITIALIZED)
+      run.start()
+      eventually(timeout(5.seconds)) {
+        assert(executeStatementOperation.getStatus.getState === OperationState.RUNNING)
+      }
+      transition(executeStatementOperation)
+      run.join()
+      assert(executeStatementOperation.getStatus.getState === finalState)
+    }
+  }
+
+  private class MySparkExecuteStatementOperation(
+      sqlContext: SQLContext,
+      hiveSession: HiveSession,
+      statement: String,
+      signal: Semaphore,
+      finalState: OperationState)
+    extends SparkExecuteStatementOperation(sqlContext, hiveSession, statement,
+      new util.HashMap, false) {
+
+    override def cleanup(): Unit = {
+      super.cleanup()
+      signal.release()
+      // At this point, operation should already be in finalState (set by either close() or
+      // cancel()). We want to check if it stays in finalState after the exception thrown by
+      // releasing the semaphore propagates. We hence need to sleep for a short while.
+      Thread.sleep(1000)
+      // State should not be ERROR
+      assert(getStatus.getState === finalState)
+    }
+  }
 }

From 9237fb2ca90d16208634d70cdff1a0ea9ddcce26 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <xyliyuanjian@gmail.com>
Date: Wed, 8 Jul 2020 09:36:06 +0900
Subject: [PATCH 0881/1280] [SPARK-32124][CORE][FOLLOW-UP] Use the invalid
 value Int.MinValue to fill the map index when the event logs from the old
 Spark version

### What changes were proposed in this pull request?
Use the invalid value Int.MinValue to fill the map index when the event logs from the old Spark version.

### Why are the changes needed?
Follow up PR for #28941.

### Does this PR introduce _any_ user-facing change?
When we use the Spark version 3.0 history server reading the event log written by the old Spark version, we use the invalid value -2 to fill the map index.

### How was this patch tested?
Existing UT.

Closes #28965 from xuanyuanking/follow-up.

Authored-by: Yuanjian Li <xyliyuanjian@gmail.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit 365961155a655f19c9184b16ccd493838c848213)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 core/src/main/scala/org/apache/spark/TaskEndReason.scala  | 3 ++-
 .../main/scala/org/apache/spark/util/JsonProtocol.scala   | 8 ++++++--
 .../scala/org/apache/spark/util/JsonProtocolSuite.scala   | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/TaskEndReason.scala b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
index b13028f868072..6606d317e7b86 100644
--- a/core/src/main/scala/org/apache/spark/TaskEndReason.scala
+++ b/core/src/main/scala/org/apache/spark/TaskEndReason.scala
@@ -90,7 +90,8 @@ case class FetchFailed(
   extends TaskFailedReason {
   override def toErrorString: String = {
     val bmAddressString = if (bmAddress == null) "null" else bmAddress.toString
-    s"FetchFailed($bmAddressString, shuffleId=$shuffleId, mapIndex=$mapIndex, " +
+    val mapIndexString = if (mapIndex == Int.MinValue) "Unknown" else mapIndex.toString
+    s"FetchFailed($bmAddressString, shuffleId=$shuffleId, mapIndex=$mapIndexString, " +
       s"mapId=$mapId, reduceId=$reduceId, message=\n$message\n)"
   }
 
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 74ff5c7dd539e..78fbd0cadc5c3 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -993,8 +993,12 @@ private[spark] object JsonProtocol {
         val blockManagerAddress = blockManagerIdFromJson(json \ "Block Manager Address")
         val shuffleId = (json \ "Shuffle ID").extract[Int]
         val mapId = (json \ "Map ID").extract[Long]
-        val mapIndex = (json \ "Map Index") match {
-          case JNothing => 0
+        val mapIndex = json \ "Map Index" match {
+          case JNothing =>
+            // Note, we use the invalid value Int.MinValue here to fill the map index for backward
+            // compatibility. Otherwise, the fetch failed event will be dropped when the history
+            // server loads the event log written by the Spark version before 3.0.
+            Int.MinValue
           case x => x.extract[Int]
         }
         val reduceId = (json \ "Reduce ID").extract[Int]
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 98aaa9e8737f3..b77cd810a0e61 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -312,7 +312,7 @@ class JsonProtocolSuite extends SparkFunSuite {
     val oldEvent = JsonProtocol.taskEndReasonToJson(fetchFailed)
       .removeField({ _._1 == "Map Index" })
     val expectedFetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 16L,
-      0, 19, "ignored")
+      Int.MinValue, 19, "ignored")
     assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
   }
 

From b49756f9dc28b1246fa15694c6348589a47f724a Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Tue, 7 Jul 2020 19:18:32 -0700
Subject: [PATCH 0882/1280] [SPARK-32163][SQL][3.0] Nested pruning should work
 even with cosmetic variations

### What changes were proposed in this pull request?

This patch proposes to deal with cosmetic variations when processing nested column extractors in `NestedColumnAliasing`. Currently if cosmetic variations are in the nested column extractors, the query is not optimized.

This backports #28988 to branch-3.0.

### Why are the changes needed?

If the expressions extracting nested fields have cosmetic variations like qualifier difference, currently nested column pruning cannot work well.

For example, two attributes which are semantically the same, are referred in a query, but the nested column extractors of them are treated differently when we deal with nested column pruning.

### Does this PR introduce _any_ user-facing change?

Yes, fixing a bug in nested column pruning.

### How was this patch tested?

Unit test.

Closes #29027 from viirya/SPARK-32163-3.0.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/optimizer/NestedColumnAliasing.scala    | 11 ++++++++---
 .../execution/datasources/SchemaPruningSuite.scala   | 12 ++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index 9814bc247fc87..becd7d1dd1738 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -118,10 +118,12 @@ object NestedColumnAliasing {
         case _ => false
       }
 
+    // Note that when we group by extractors with their references, we should remove
+    // cosmetic variations.
     val exclusiveAttrSet = AttributeSet(exclusiveAttrs ++ otherRootReferences)
     val aliasSub = nestedFieldReferences.asInstanceOf[Seq[ExtractValue]]
       .filter(!_.references.subsetOf(exclusiveAttrSet))
-      .groupBy(_.references.head)
+      .groupBy(_.references.head.canonicalized.asInstanceOf[Attribute])
       .flatMap { case (attr, nestedFields: Seq[ExtractValue]) =>
         // Each expression can contain multiple nested fields.
         // Note that we keep the original names to deliver to parquet in a case-sensitive way.
@@ -132,9 +134,12 @@ object NestedColumnAliasing {
 
         // If all nested fields of `attr` are used, we don't need to introduce new aliases.
         // By default, ColumnPruning rule uses `attr` already.
+        // Note that we need to remove cosmetic variations first, so we only count a
+        // nested field once.
         if (nestedFieldToAlias.nonEmpty &&
-            nestedFieldToAlias
-              .map { case (nestedField, _) => totalFieldNum(nestedField.dataType) }
+            nestedFields.map(_.canonicalized)
+              .distinct
+              .map { nestedField => totalFieldNum(nestedField.dataType) }
               .sum < totalFieldNum(attr.dataType)) {
           Some(attr.exprId -> nestedFieldToAlias)
         } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
index a3d4905e82cee..37ec6e29213e7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -301,6 +301,18 @@ abstract class SchemaPruningSuite
     checkAnswer(query, Row("Y.", 1) :: Row("X.", 1) :: Row(null, 2) :: Row(null, 2) :: Nil)
   }
 
+  testSchemaPruning("SPARK-32163: nested pruning should work even with cosmetic variations") {
+    withTempView("contact_alias") {
+      sql("select * from contacts")
+        .select(explode(col("friends.first")), col("friends"))
+        .createOrReplaceTempView("contact_alias")
+
+      val query = sql("select friends.middle, col from contact_alias")
+      checkScan(query, "struct<friends:array<struct<first:string,middle:string>>>")
+      checkAnswer(query, Row(Array("Z."), "Susan") :: Nil)
+    }
+  }
+
   protected def testSchemaPruning(testName: String)(testThunk: => Unit): Unit = {
     test(s"Spark vectorized reader - without partition data column - $testName") {
       withSQLConf(vectorizedReaderEnabledKey -> "true") {

From 37dc51a5dddc2958c18831e9c0809c3b495aa719 Mon Sep 17 00:00:00 2001
From: ulysses <youxiduo@weidian.com>
Date: Wed, 8 Jul 2020 11:30:47 +0900
Subject: [PATCH 0883/1280] [SPARK-30703][SQL][FOLLOWUP] Update SqlBase.g4
 invalid comment

### What changes were proposed in this pull request?

Modify the comment of `SqlBase.g4`.

### Why are the changes needed?

`docs/sql-keywords.md` has already moved to `docs/sql-ref-ansi-compliance.md#sql-keywords`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

No need.

Closes #29033 from ulysses-you/SPARK-30703-FOLLOWUP.

Authored-by: ulysses <youxiduo@weidian.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 65286aec4b3c4e93d8beac6dd1b097ce97d53fd8)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 5821a743d4168..75dae8f3faf6d 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -1441,8 +1441,7 @@ nonReserved
     ;
 
 // NOTE: If you add a new token in the list below, you should update the list of keywords
-// in `docs/sql-keywords.md`. If the token is a non-reserved keyword,
-// please update `ansiNonReserved` and `nonReserved` as well.
+// and reserved tag in `docs/sql-ref-ansi-compliance.md#sql-keywords`.
 
 //============================
 // Start of the keywords list

From 0939730c555c1b931b22a614fcc46ab344f0676f Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 8 Jul 2020 17:46:25 +0900
Subject: [PATCH 0884/1280] [SPARK-32214][SQL] The type conversion function
 generated in makeFromJava for "other"  type uses a wrong variable

### What changes were proposed in this pull request?

This PR fixes an inconsistency in `EvaluatePython.makeFromJava`, which creates a type conversion function for some Java/Scala types.

`other` is a type but it should actually pass `obj`:

```scala
case other => (obj: Any) => nullSafeConvert(other)(PartialFunction.empty)
```

This does not change the output because it always returns `null` for unsupported datatypes.

### Why are the changes needed?

To make the codes coherent, and consistent.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

No behaviour change.

Closes #29029 from sarutak/fix-makeFromJava.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 371b35d2e0ab08ebd853147c6673de3adfad0553)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/execution/python/EvaluatePython.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
index 520afad287648..7fe3263630820 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvaluatePython.scala
@@ -198,7 +198,7 @@ object EvaluatePython {
 
     case udt: UserDefinedType[_] => makeFromJava(udt.sqlType)
 
-    case other => (obj: Any) => nullSafeConvert(other)(PartialFunction.empty)
+    case other => (obj: Any) => nullSafeConvert(obj)(PartialFunction.empty)
   }
 
   private def nullSafeConvert(input: Any)(f: PartialFunction[Any, Any]): Any = {

From ac2c6cd3cfae369f6d1af6abea567263d34a29b2 Mon Sep 17 00:00:00 2001
From: Zhen Li <zhli@microsoft.com>
Date: Wed, 8 Jul 2020 21:58:45 +0900
Subject: [PATCH 0885/1280] [SPARK-32024][WEBUI] Update
 ApplicationStoreInfo.size during HistoryServerDiskManager initializing

### What changes were proposed in this pull request?

Update ApplicationStoreInfo.size to real size during HistoryServerDiskManager initializing.

### Why are the changes needed?

This PR is for fixing bug [32024](https://issues.apache.org/jira/browse/SPARK-32024). We found after history server restart, below error would randomly happen: "java.lang.IllegalStateException: Disk usage tracker went negative (now = -***, delta = -***)" from `HistoryServerDiskManager`.
![Capture](https://user-images.githubusercontent.com/10524738/85034468-fda4ae80-b136-11ea-9011-f0c3e6508002.JPG)

**Cause**: Reading data from level db would trigger table file compaction, which may also trigger size of level db directory changes.  This size change may not be recorded in LevelDB (`ApplicationStoreInfo` in `listing`). When service restarts, `currentUsage` is calculated from real directory size, but `ApplicationStoreInfo` are loaded from leveldb, then `currentUsage` may be less then sum of `ApplicationStoreInfo.size`. In `makeRoom()` function, `ApplicationStoreInfo.size` is used to update usage. Then `currentUsage` becomes negative after several round of `release()` and `lease()` (`makeRoom()`).
**Reproduce**: we can reproduce this issue in dev environment by reducing config value of "spark.history.retainedApplications" and "spark.history.store.maxDiskUsage" to some small values. Here are steps: 1. start history server, load some applications and access some pages (maybe "stages" page to trigger leveldb compaction). 2. restart HS, and refresh pages.
I also added an UT to simulate this case in `HistoryServerDiskManagerSuite`.
**Benefit**: this change would help improve history server reliability.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Add unit test and manually tested it.

Closes #28859 from zhli1142015/update-ApplicationStoreInfo.size-during-disk-manager-initialize.

Authored-by: Zhen Li <zhli@microsoft.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit 8e7fc04637bbb8d2fdc2c758746e0eaf496c4d92)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 .../history/HistoryServerDiskManager.scala    | 21 +++++++--
 .../HistoryServerDiskManagerSuite.scala       | 46 +++++++++++++++++++
 2 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index 0a1f33395ad62..4ddd6d9a780be 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -75,14 +75,29 @@ private class HistoryServerDiskManager(
 
     // Go through the recorded store directories and remove any that may have been removed by
     // external code.
-    val orphans = listing.view(classOf[ApplicationStoreInfo]).asScala.filter { info =>
-      !new File(info.path).exists()
-    }.toSeq
+    val (existences, orphans) = listing
+      .view(classOf[ApplicationStoreInfo])
+      .asScala
+      .toSeq
+      .partition { info =>
+        new File(info.path).exists()
+      }
 
     orphans.foreach { info =>
       listing.delete(info.getClass(), info.path)
     }
 
+    // Reading level db would trigger table file compaction, then it may cause size of level db
+    // directory changed. When service restarts, "currentUsage" is calculated from real directory
+    // size. Update "ApplicationStoreInfo.size" to ensure "currentUsage" equals
+    // sum of "ApplicationStoreInfo.size".
+    existences.foreach { info =>
+      val fileSize = sizeOf(new File(info.path))
+      if (fileSize != info.size) {
+        listing.write(info.copy(size = fileSize))
+      }
+    }
+
     logInfo("Initialized disk manager: " +
       s"current usage = ${Utils.bytesToString(currentUsage.get())}, " +
       s"max usage = ${Utils.bytesToString(maxUsage)}")
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
index f78469e132490..b17880a733615 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
@@ -158,4 +158,50 @@ class HistoryServerDiskManagerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(manager.approximateSize(50L, true) > 50L)
   }
 
+  test("SPARK-32024: update ApplicationStoreInfo.size during initializing") {
+    val manager = mockManager()
+    val leaseA = manager.lease(2)
+    doReturn(3L).when(manager).sizeOf(meq(leaseA.tmpPath))
+    val dstA = leaseA.commit("app1", None)
+    assert(manager.free() === 0)
+    assert(manager.committed() === 3)
+    // Listing store tracks dstA now.
+    assert(store.read(classOf[ApplicationStoreInfo], dstA.getAbsolutePath).size === 3)
+
+    // Simulate: service restarts, new disk manager (manager1) is initialized.
+    val manager1 = mockManager()
+    // Simulate: event KVstore compaction before restart, directory size reduces.
+    doReturn(2L).when(manager1).sizeOf(meq(dstA))
+    doReturn(2L).when(manager1).sizeOf(meq(new File(testDir, "apps")))
+    manager1.initialize()
+    // "ApplicationStoreInfo.size" is updated for dstA.
+    assert(store.read(classOf[ApplicationStoreInfo], dstA.getAbsolutePath).size === 2)
+    assert(manager1.free() === 1)
+    // If "ApplicationStoreInfo.size" is not correctly updated, "IllegalStateException"
+    // would be thrown.
+    val leaseB = manager1.lease(2)
+    assert(manager1.free() === 1)
+    doReturn(2L).when(manager1).sizeOf(meq(leaseB.tmpPath))
+    val dstB = leaseB.commit("app2", None)
+    assert(manager1.committed() === 2)
+    // Listing store tracks dstB only, dstA is evicted by "makeRoom()".
+    assert(store.read(classOf[ApplicationStoreInfo], dstB.getAbsolutePath).size === 2)
+
+    val manager2 = mockManager()
+    // Simulate: cache entities are written after replaying, directory size increases.
+    doReturn(3L).when(manager2).sizeOf(meq(dstB))
+    doReturn(3L).when(manager2).sizeOf(meq(new File(testDir, "apps")))
+    manager2.initialize()
+    // "ApplicationStoreInfo.size" is updated for dstB.
+    assert(store.read(classOf[ApplicationStoreInfo], dstB.getAbsolutePath).size === 3)
+    assert(manager2.free() === 0)
+    val leaseC = manager2.lease(2)
+    doReturn(2L).when(manager2).sizeOf(meq(leaseC.tmpPath))
+    val dstC = leaseC.commit("app3", None)
+    assert(manager2.free() === 1)
+    assert(manager2.committed() === 2)
+    // Listing store tracks dstC only, dstB is evicted by "makeRoom()".
+    assert(store.read(classOf[ApplicationStoreInfo], dstC.getAbsolutePath).size === 2)
+  }
+
 }

From 30e3fcbb081b6b07a12316c9dd63c66f77b8fec7 Mon Sep 17 00:00:00 2001
From: Ryan Blue <blue@apache.org>
Date: Wed, 8 Jul 2020 16:06:40 -0700
Subject: [PATCH 0886/1280] [SPARK-32168][SQL] Fix hidden partitioning
 correctness bug in SQL overwrite

### What changes were proposed in this pull request?

When converting an `INSERT OVERWRITE` query to a v2 overwrite plan, Spark attempts to detect when a dynamic overwrite and a static overwrite will produce the same result so it can use the static overwrite. Spark incorrectly detects when dynamic and static overwrites are equivalent when there are hidden partitions, such as `days(ts)`.

This updates the analyzer rule `ResolveInsertInto` to always use a dynamic overwrite when the mode is dynamic, and static when the mode is static. This avoids the problem by not trying to determine whether the two plans are equivalent and always using the one that corresponds to the partition overwrite mode.

### Why are the changes needed?

This is a correctness bug. If a table has hidden partitions, all of the values for those partitions are dropped instead of dynamically overwriting changed partitions.

This only affects SQL commands (not `DataFrameWriter`) writing to tables that have hidden partitions. It is also only a problem when the partition overwrite mode is dynamic.

### Does this PR introduce _any_ user-facing change?

Yes, it fixes the correctness bug detailed above.

### How was this patch tested?

* This updates the in-memory table to support a hidden partition transform, `days`, and adds a test case to `DataSourceV2SQLSuite` in which the table uses this hidden partition function. This test fails without the fix to `ResolveInsertInto`.
* This updates the test case `InsertInto: overwrite - multiple static partitions - dynamic mode` in `InsertIntoTests`. The result of the SQL command is unchanged, but the SQL command will now use a dynamic overwrite so the test now uses `dynamicOverwriteTest`.

Closes #28993 from rdblue/fix-insert-overwrite-v2-conversion.

Authored-by: Ryan Blue <blue@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 3bb1ac597a6603e8224cb99349419d950ad7318e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  4 +-
 .../spark/sql/connector/InMemoryTable.scala   | 66 ++++++++++++++++---
 .../datasources/v2/BatchScanExec.scala        |  2 +-
 .../spark/sql/DataFrameWriterV2Suite.scala    |  7 --
 .../sql/connector/DataSourceV2SQLSuite.scala  | 38 ++++++++++-
 .../spark/sql/connector/InsertIntoTests.scala | 27 ++++----
 6 files changed, 107 insertions(+), 37 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 6fb103e46a77b..243b555118707 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1041,12 +1041,10 @@ class Analyzer(
 
         val staticPartitions = i.partitionSpec.filter(_._2.isDefined).mapValues(_.get)
         val query = addStaticPartitionColumns(r, i.query, staticPartitions)
-        val dynamicPartitionOverwrite = partCols.size > staticPartitions.size &&
-          conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC
 
         if (!i.overwrite) {
           AppendData.byPosition(r, query)
-        } else if (dynamicPartitionOverwrite) {
+        } else if (conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC) {
           OverwritePartitionsDynamic.byPosition(r, query)
         } else {
           OverwriteByExpression.byPosition(r, query, staticDeleteExpression(r, staticPartitions))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
index 3d7026e180cd1..616fc72320caf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.connector
 
+import java.time.{Instant, ZoneId}
+import java.time.temporal.ChronoUnit
 import java.util
 
 import scala.collection.JavaConverters._
@@ -25,12 +27,13 @@ import scala.collection.mutable
 import org.scalatest.Assertions._
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.connector.catalog._
-import org.apache.spark.sql.connector.expressions.{IdentityTransform, NamedReference, Transform}
+import org.apache.spark.sql.connector.expressions.{BucketTransform, DaysTransform, HoursTransform, IdentityTransform, MonthsTransform, Transform, YearsTransform}
 import org.apache.spark.sql.connector.read._
 import org.apache.spark.sql.connector.write._
 import org.apache.spark.sql.sources.{And, EqualTo, Filter, IsNotNull}
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.types.{DataType, DateType, StructType, TimestampType}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
 /**
@@ -46,10 +49,15 @@ class InMemoryTable(
   private val allowUnsupportedTransforms =
     properties.getOrDefault("allow-unsupported-transforms", "false").toBoolean
 
-  partitioning.foreach { t =>
-    if (!t.isInstanceOf[IdentityTransform] && !allowUnsupportedTransforms) {
-      throw new IllegalArgumentException(s"Transform $t must be IdentityTransform")
-    }
+  partitioning.foreach {
+    case _: IdentityTransform =>
+    case _: YearsTransform =>
+    case _: MonthsTransform =>
+    case _: DaysTransform =>
+    case _: HoursTransform =>
+    case _: BucketTransform =>
+    case t if !allowUnsupportedTransforms =>
+      throw new IllegalArgumentException(s"Transform $t is not a supported transform")
   }
 
   // The key `Seq[Any]` is the partition values.
@@ -66,8 +74,14 @@ class InMemoryTable(
     }
   }
 
+  private val UTC = ZoneId.of("UTC")
+  private val EPOCH_LOCAL_DATE = Instant.EPOCH.atZone(UTC).toLocalDate
+
   private def getKey(row: InternalRow): Seq[Any] = {
-    def extractor(fieldNames: Array[String], schema: StructType, row: InternalRow): Any = {
+    def extractor(
+        fieldNames: Array[String],
+        schema: StructType,
+        row: InternalRow): (Any, DataType) = {
       val index = schema.fieldIndex(fieldNames(0))
       val value = row.toSeq(schema).apply(index)
       if (fieldNames.length > 1) {
@@ -78,10 +92,44 @@ class InMemoryTable(
             throw new IllegalArgumentException(s"Unsupported type, ${dataType.simpleString}")
         }
       } else {
-        value
+        (value, schema(index).dataType)
       }
     }
-    partCols.map(fieldNames => extractor(fieldNames, schema, row))
+
+    partitioning.map {
+      case IdentityTransform(ref) =>
+        extractor(ref.fieldNames, schema, row)._1
+      case YearsTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (days: Int, DateType) =>
+            ChronoUnit.YEARS.between(EPOCH_LOCAL_DATE, DateTimeUtils.daysToLocalDate(days))
+          case (micros: Long, TimestampType) =>
+            val localDate = DateTimeUtils.microsToInstant(micros).atZone(UTC).toLocalDate
+            ChronoUnit.YEARS.between(EPOCH_LOCAL_DATE, localDate)
+        }
+      case MonthsTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (days: Int, DateType) =>
+            ChronoUnit.MONTHS.between(EPOCH_LOCAL_DATE, DateTimeUtils.daysToLocalDate(days))
+          case (micros: Long, TimestampType) =>
+            val localDate = DateTimeUtils.microsToInstant(micros).atZone(UTC).toLocalDate
+            ChronoUnit.MONTHS.between(EPOCH_LOCAL_DATE, localDate)
+        }
+      case DaysTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (days, DateType) =>
+            days
+          case (micros: Long, TimestampType) =>
+            ChronoUnit.DAYS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
+        }
+      case HoursTransform(ref) =>
+        extractor(ref.fieldNames, schema, row) match {
+          case (micros: Long, TimestampType) =>
+            ChronoUnit.HOURS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
+        }
+      case BucketTransform(numBuckets, ref) =>
+        (extractor(ref.fieldNames, schema, row).hashCode() & Integer.MAX_VALUE) % numBuckets
+    }
   }
 
   def withData(data: Array[BufferedRows]): InMemoryTable = dataMap.synchronized {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
index e4e7887017a1d..c199df676ced3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/BatchScanExec.scala
@@ -40,7 +40,7 @@ case class BatchScanExec(
 
   override def hashCode(): Int = batch.hashCode()
 
-  override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions()
+  @transient override lazy val partitions: Seq[InputPartition] = batch.planInputPartitions()
 
   override lazy val readerFactory: PartitionReaderFactory = batch.createReaderFactory()
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
index ac2ebd8bd748b..508eefafd0754 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
@@ -336,7 +336,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(years($"ts"))
         .create()
 
@@ -350,7 +349,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(months($"ts"))
         .create()
 
@@ -364,7 +362,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(days($"ts"))
         .create()
 
@@ -378,7 +375,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     spark.table("source")
         .withColumn("ts", lit("2019-06-01 10:00:00.000000").cast("timestamp"))
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(hours($"ts"))
         .create()
 
@@ -391,7 +387,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
   test("Create: partitioned by bucket(4, id)") {
     spark.table("source")
         .writeTo("testcat.table_name")
-        .tableProperty("allow-unsupported-transforms", "true")
         .partitionedBy(bucket(4, $"id"))
         .create()
 
@@ -596,7 +591,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
         lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
         lit("America/Los_Angeles") as "timezone"))
       .writeTo("testcat.table_name")
-      .tableProperty("allow-unsupported-transforms", "true")
       .partitionedBy(
         years($"ts.created"), months($"ts.created"), days($"ts.created"), hours($"ts.created"),
         years($"ts.modified"), months($"ts.modified"), days($"ts.modified"), hours($"ts.modified")
@@ -624,7 +618,6 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
         lit("2019-09-02 07:00:00.000000").cast("timestamp") as "modified",
         lit("America/Los_Angeles") as "timezone"))
       .writeTo("testcat.table_name")
-      .tableProperty("allow-unsupported-transforms", "true")
       .partitionedBy(bucket(4, $"ts.timezone"))
       .create()
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 8462ce5a6c44f..ef5558a83a306 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.connector
 
+import java.sql.Timestamp
+import java.time.LocalDate
+
 import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkException
@@ -27,7 +30,7 @@ import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
 import org.apache.spark.sql.connector.catalog.CatalogV2Util.withDefaultOwnership
 import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
-import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
+import org.apache.spark.sql.internal.SQLConf.{PARTITION_OVERWRITE_MODE, PartitionOverwriteMode, V2_SESSION_CATALOG_IMPLEMENTATION}
 import org.apache.spark.sql.internal.connector.SimpleTableProvider
 import org.apache.spark.sql.sources.SimpleScanSource
 import org.apache.spark.sql.types.{BooleanType, LongType, StringType, StructField, StructType}
@@ -1630,7 +1633,6 @@ class DataSourceV2SQLSuite
         """
           |CREATE TABLE testcat.t (id int, `a.b` string) USING foo
           |CLUSTERED BY (`a.b`) INTO 4 BUCKETS
-          |OPTIONS ('allow-unsupported-transforms'=true)
         """.stripMargin)
 
       val testCatalog = catalog("testcat").asTableCatalog.asInstanceOf[InMemoryTableCatalog]
@@ -2476,6 +2478,38 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("SPARK-32168: INSERT OVERWRITE - hidden days partition - dynamic mode") {
+    def testTimestamp(daysOffset: Int): Timestamp = {
+      Timestamp.valueOf(LocalDate.of(2020, 1, 1 + daysOffset).atStartOfDay())
+    }
+
+    withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTable(t1) {
+        val df = spark.createDataFrame(Seq(
+          (testTimestamp(1), "a"),
+          (testTimestamp(2), "b"),
+          (testTimestamp(3), "c"))).toDF("ts", "data")
+        df.createOrReplaceTempView("source_view")
+
+        sql(s"CREATE TABLE $t1 (ts timestamp, data string) " +
+            s"USING $v2Format PARTITIONED BY (days(ts))")
+        sql(s"INSERT INTO $t1 VALUES " +
+            s"(CAST(date_add('2020-01-01', 2) AS timestamp), 'dummy'), " +
+            s"(CAST(date_add('2020-01-01', 4) AS timestamp), 'keep')")
+        sql(s"INSERT OVERWRITE TABLE $t1 SELECT ts, data FROM source_view")
+
+        val expected = spark.createDataFrame(Seq(
+          (testTimestamp(1), "a"),
+          (testTimestamp(2), "b"),
+          (testTimestamp(3), "c"),
+          (testTimestamp(4), "keep"))).toDF("ts", "data")
+
+        verifyTable(t1, expected)
+      }
+    }
+  }
+
   private def testV1Command(sqlCommand: String, sqlParams: String): Unit = {
     val e = intercept[AnalysisException] {
       sql(s"$sqlCommand $sqlParams")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
index b88ad5218fcd2..2cc7a1f994645 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/InsertIntoTests.scala
@@ -446,21 +446,18 @@ trait InsertIntoSQLOnlyTests
       }
     }
 
-    test("InsertInto: overwrite - multiple static partitions - dynamic mode") {
-      // Since all partitions are provided statically, this should be supported by everyone
-      withSQLConf(PARTITION_OVERWRITE_MODE.key -> PartitionOverwriteMode.DYNAMIC.toString) {
-        val t1 = s"${catalogAndNamespace}tbl"
-        withTableAndData(t1) { view =>
-          sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " +
-            s"USING $v2Format PARTITIONED BY (id, p)")
-          sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)")
-          sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view")
-          verifyTable(t1, Seq(
-            (2, "a", 2),
-            (2, "b", 2),
-            (2, "c", 2),
-            (4, "keep", 2)).toDF("id", "data", "p"))
-        }
+    dynamicOverwriteTest("InsertInto: overwrite - multiple static partitions - dynamic mode") {
+      val t1 = s"${catalogAndNamespace}tbl"
+      withTableAndData(t1) { view =>
+        sql(s"CREATE TABLE $t1 (id bigint, data string, p int) " +
+          s"USING $v2Format PARTITIONED BY (id, p)")
+        sql(s"INSERT INTO $t1 VALUES (2L, 'dummy', 2), (4L, 'keep', 2)")
+        sql(s"INSERT OVERWRITE TABLE $t1 PARTITION (id = 2, p = 2) SELECT data FROM $view")
+        verifyTable(t1, Seq(
+          (2, "a", 2),
+          (2, "b", 2),
+          (2, "c", 2),
+          (4, "keep", 2)).toDF("id", "data", "p"))
       }
     }
 

From f9d53a6de8b328ec5ee4fae965787ee932a0391d Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Thu, 9 Jul 2020 15:26:38 +0900
Subject: [PATCH 0887/1280] [SPARK-32024][WEBUI][FOLLOWUP] Quick fix on test
 failure on missing when statements

### What changes were proposed in this pull request?

This patch fixes the test failure due to the missing when statements for destination path. Note that it didn't fail on master branch, because 245aee9 got rid of size call in destination path, but still good to not depend on 245aee9.

### Why are the changes needed?

The build against branch-3.0 / branch-2.4 starts to fail after merging SPARK-32024 (#28859) and this patch will fix it.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Ran modified UT against master / branch-3.0 / branch-2.4.

Closes #29046 from HeartSaVioR/QUICKFIX-SPARK-32024.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 161cf2a12698bfebba94e0d406e0b110e4429b6b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/deploy/history/HistoryServerDiskManager.scala     | 2 +-
 .../deploy/history/HistoryServerDiskManagerSuite.scala      | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
index 4ddd6d9a780be..fe18b03a7d603 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServerDiskManager.scala
@@ -248,7 +248,7 @@ private class HistoryServerDiskManager(
     }
   }
 
-  private def appStorePath(appId: String, attemptId: Option[String]): File = {
+  private[history] def appStorePath(appId: String, attemptId: Option[String]): File = {
     val fileName = appId + attemptId.map("_" + _).getOrElse("") + ".ldb"
     new File(appStoreDir, fileName)
   }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
index b17880a733615..9004e86323691 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
@@ -162,6 +162,8 @@ class HistoryServerDiskManagerSuite extends SparkFunSuite with BeforeAndAfter {
     val manager = mockManager()
     val leaseA = manager.lease(2)
     doReturn(3L).when(manager).sizeOf(meq(leaseA.tmpPath))
+    val dstPathA = manager.appStorePath("app1", None)
+    doReturn(3L).when(manager).sizeOf(meq(dstPathA))
     val dstA = leaseA.commit("app1", None)
     assert(manager.free() === 0)
     assert(manager.committed() === 3)
@@ -182,6 +184,8 @@ class HistoryServerDiskManagerSuite extends SparkFunSuite with BeforeAndAfter {
     val leaseB = manager1.lease(2)
     assert(manager1.free() === 1)
     doReturn(2L).when(manager1).sizeOf(meq(leaseB.tmpPath))
+    val dstPathB = manager.appStorePath("app2", None)
+    doReturn(2L).when(manager1).sizeOf(meq(dstPathB))
     val dstB = leaseB.commit("app2", None)
     assert(manager1.committed() === 2)
     // Listing store tracks dstB only, dstA is evicted by "makeRoom()".
@@ -197,6 +201,8 @@ class HistoryServerDiskManagerSuite extends SparkFunSuite with BeforeAndAfter {
     assert(manager2.free() === 0)
     val leaseC = manager2.lease(2)
     doReturn(2L).when(manager2).sizeOf(meq(leaseC.tmpPath))
+    val dstPathC = manager.appStorePath("app3", None)
+    doReturn(2L).when(manager2).sizeOf(meq(dstPathC))
     val dstC = leaseC.commit("app3", None)
     assert(manager2.free() === 1)
     assert(manager2.committed() === 2)

From 86efa456d8e11a2b7e10bce70d4ead20c75acbe1 Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Thu, 9 Jul 2020 15:51:56 +0900
Subject: [PATCH 0888/1280] [SPARK-32160][CORE][PYSPARK] Disallow to create
 SparkContext in executors

### What changes were proposed in this pull request?

This PR proposes to disallow to create `SparkContext` in executors, e.g., in UDFs.

### Why are the changes needed?

Currently executors can create SparkContext, but shouldn't be able to create it.

```scala
sc.range(0, 1).foreach { _ =>
  new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
}
```

### Does this PR introduce _any_ user-facing change?

Yes, users won't be able to create `SparkContext` in executors.

### How was this patch tested?

Addes tests.

Closes #28986 from ueshin/issues/SPARK-32160/disallow_spark_context_in_executors.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit cfecc2030d8b4774c1f4754fe81f57fbc61c9c75)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/SparkContext.scala | 16 +++++++++
 .../org/apache/spark/SparkContextSuite.scala  | 12 +++++++
 python/pyspark/context.py                     | 14 ++++++++
 python/pyspark/tests/test_context.py          |  8 +++++
 .../org/apache/spark/sql/SparkSession.scala   |  2 +-
 ...xternalAppendOnlyUnsafeRowArraySuite.scala | 35 +++++++++----------
 6 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index bcbb7e42f8d66..2761f0da5f772 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -82,6 +82,9 @@ class SparkContext(config: SparkConf) extends Logging {
   // The call site where this SparkContext was constructed.
   private val creationSite: CallSite = Utils.getCallSite()
 
+  // In order to prevent SparkContext from being created in executors.
+  SparkContext.assertOnDriver()
+
   // In order to prevent multiple SparkContexts from being active at the same time, mark this
   // context as having started construction.
   // NOTE: this must be placed at the beginning of the SparkContext constructor.
@@ -2539,6 +2542,19 @@ object SparkContext extends Logging {
     }
   }
 
+  /**
+   * Called to ensure that SparkContext is created or accessed only on the Driver.
+   *
+   * Throws an exception if a SparkContext is about to be created in executors.
+   */
+  private def assertOnDriver(): Unit = {
+    if (TaskContext.get != null) {
+      // we're accessing it during task execution, fail.
+      throw new IllegalStateException(
+        "SparkContext should only be created and accessed on the driver.")
+    }
+  }
+
   /**
    * This function may be used to get or instantiate a SparkContext and register it as a
    * singleton object. Because we can only have one active SparkContext per JVM,
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index 9f8fa89985ae5..2b1e110a39466 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -950,6 +950,18 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       }
     }
   }
+
+  test("SPARK-32160: Disallow to create SparkContext in executors") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+
+    val error = intercept[SparkException] {
+      sc.range(0, 1).foreach { _ =>
+        new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+      }
+    }.getMessage()
+
+    assert(error.contains("SparkContext should only be created and accessed on the driver."))
+  }
 }
 
 object SparkContextSuite {
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 5bb991e7993f6..ecd171a0b8b41 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -38,6 +38,7 @@
 from pyspark.storagelevel import StorageLevel
 from pyspark.resource import ResourceInformation
 from pyspark.rdd import RDD, _load_from_socket, ignore_unicode_prefix
+from pyspark.taskcontext import TaskContext
 from pyspark.traceback_utils import CallSite, first_spark_call
 from pyspark.status import StatusTracker
 from pyspark.profiler import ProfilerCollector, BasicProfiler
@@ -118,6 +119,9 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
             ...
         ValueError:...
         """
+        # In order to prevent SparkContext from being created in executors.
+        SparkContext._assert_on_driver()
+
         self._callsite = first_spark_call() or CallSite(None, None, None)
         if gateway is not None and gateway.gateway_parameters.auth_token is None:
             raise ValueError(
@@ -1145,6 +1149,16 @@ def resources(self):
             resources[name] = ResourceInformation(name, addrs)
         return resources
 
+    @staticmethod
+    def _assert_on_driver():
+        """
+        Called to ensure that SparkContext is created only on the Driver.
+
+        Throws an exception if a SparkContext is about to be created in executors.
+        """
+        if TaskContext.get() is not None:
+            raise Exception("SparkContext should only be created and accessed on the driver.")
+
 
 def _test():
     import atexit
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index c7f435a582210..303635ddab12c 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -267,6 +267,14 @@ def test_resources(self):
             resources = sc.resources
             self.assertEqual(len(resources), 0)
 
+    def test_disallow_to_create_spark_context_in_executors(self):
+        # SPARK-32160: SparkContext should not be created in executors.
+        with SparkContext("local-cluster[3, 1, 1024]") as sc:
+            with self.assertRaises(Exception) as context:
+                sc.range(2).foreach(lambda _: SparkContext())
+            self.assertIn("SparkContext should only be created and accessed on the driver.",
+                          str(context.exception))
+
 
 class ContextTestsWithResources(unittest.TestCase):
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 60a60377d8a3f..e5d8710bfb69f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -1086,7 +1086,7 @@ object SparkSession extends Logging {
   }
 
   private def assertOnDriver(): Unit = {
-    if (Utils.isTesting && TaskContext.get != null) {
+    if (TaskContext.get != null) {
       // we're accessing it during task execution, fail.
       throw new IllegalStateException(
         "SparkSession should only be created and accessed on the driver.")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
index b29de9c4adbaa..98aba3ba25f17 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArraySuite.scala
@@ -27,32 +27,29 @@ import org.apache.spark.sql.catalyst.expressions.UnsafeRow
 
 class ExternalAppendOnlyUnsafeRowArraySuite extends SparkFunSuite with LocalSparkContext {
   private val random = new java.util.Random()
-  private var taskContext: TaskContext = _
-
-  override def afterAll(): Unit = try {
-    TaskContext.unset()
-  } finally {
-    super.afterAll()
-  }
 
   private def withExternalArray(inMemoryThreshold: Int, spillThreshold: Int)
                                (f: ExternalAppendOnlyUnsafeRowArray => Unit): Unit = {
     sc = new SparkContext("local", "test", new SparkConf(false))
 
-    taskContext = MemoryTestingUtils.fakeTaskContext(SparkEnv.get)
+    val taskContext = MemoryTestingUtils.fakeTaskContext(SparkEnv.get)
     TaskContext.setTaskContext(taskContext)
 
-    val array = new ExternalAppendOnlyUnsafeRowArray(
-      taskContext.taskMemoryManager(),
-      SparkEnv.get.blockManager,
-      SparkEnv.get.serializerManager,
-      taskContext,
-      1024,
-      SparkEnv.get.memoryManager.pageSizeBytes,
-      inMemoryThreshold,
-      spillThreshold)
-    try f(array) finally {
-      array.clear()
+    try {
+      val array = new ExternalAppendOnlyUnsafeRowArray(
+        taskContext.taskMemoryManager(),
+        SparkEnv.get.blockManager,
+        SparkEnv.get.serializerManager,
+        taskContext,
+        1024,
+        SparkEnv.get.memoryManager.pageSizeBytes,
+        inMemoryThreshold,
+        spillThreshold)
+      try f(array) finally {
+        array.clear()
+      }
+    } finally {
+      TaskContext.unset()
     }
   }
 

From 29e098bc08477f00b5f83dfc5e73181668bb5f20 Mon Sep 17 00:00:00 2001
From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Date: Thu, 9 Jul 2020 16:14:33 +0900
Subject: [PATCH 0889/1280] [SPARK-32193][SQL][DOCS] Update regexp usage in SQL
 docs

### What changes were proposed in this pull request?
update REGEXP usage and examples in sql-ref-syntx-qry-select-like.cmd

### Why are the changes needed?
make the usage of REGEXP known to more users

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

No tests

Closes #29009 from GuoPhilipse/update-migrate-guide.

Lead-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Co-authored-by: GuoPhilipse <guofei_ok@126.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 09cc6c51eaa489733551e0507d129b06d683207c)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-syntax-qry-select-like.md | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md
index feb5eb7b3c80d..6211faa8d529e 100644
--- a/docs/sql-ref-syntax-qry-select-like.md
+++ b/docs/sql-ref-syntax-qry-select-like.md
@@ -26,7 +26,7 @@ A LIKE predicate is used to search for a specific pattern.
 ### Syntax
 
 ```sql
-[ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | RLIKE regex_pattern }
+[ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | [ RLIKE | REGEXP ] regex_pattern }
 ```
 
 ### Parameters
@@ -44,7 +44,7 @@ A LIKE predicate is used to search for a specific pattern.
 
 * **regex_pattern**
 
-    Specifies a regular expression search pattern to be searched by the `RLIKE` clause.
+    Specifies a regular expression search pattern to be searched by the `RLIKE` or `REGEXP` clause.
 
 ### Examples
 
@@ -90,6 +90,14 @@ SELECT * FROM person WHERE name RLIKE 'M+';
 |200|Mary|null|
 +---+----+----+
 
+SELECT * FROM person WHERE name REGEXP 'M+';
++---+----+----+
+| id|name| age|
++---+----+----+
+|300|Mike|  80|
+|200|Mary|null|
++---+----+----+
+
 SELECT * FROM person WHERE name LIKE '%\_%';
 +---+------+---+
 | id|  name|age|

From 83f0423ed91388518345cc4e66a552195bcf6f7a Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Thu, 9 Jul 2020 07:37:06 +0000
Subject: [PATCH 0890/1280] [SPARK-32148][SS] Fix stream-stream join issue on
 missing to copy reused unsafe row

### What changes were proposed in this pull request?

This patch fixes the odd join result being occurred from stream-stream join for state store format V2.

There're some spots on V2 path which leverage UnsafeProjection. As the result row is reused, the row should be copied to avoid changing value during reading (or make sure the caller doesn't affect by such behavior) but `SymmetricHashJoinStateManager.removeByValueCondition` violates the case.

This patch makes `KeyWithIndexToValueRowConverterV2.convertValue` copy the row by itself so that callers don't need to take care about it. This patch doesn't change the behavior of `KeyWithIndexToValueRowConverterV2.convertToValueRow` to avoid double-copying, as the caller is expected to store the row which the implementation of state store will call `copy()`.

This patch adds such behavior into each method doc in `KeyWithIndexToValueRowConverter`, so that further contributors can read through and make sure the change / new addition doesn't break the contract.

### Why are the changes needed?

Stream-stream join with state store format V2 (newly added in Spark 3.0.0) has a serious correctness bug which brings indeterministic result.

### Does this PR introduce _any_ user-facing change?

Yes, some of Spark 3.0.0 users using stream-stream join from the new checkpoint (as the bug exists to only v2 format path) may encounter wrong join result. This patch will fix it.

### How was this patch tested?

Reported case is converted to the new UT, and confirmed UT passed. All UTs in StreamingInnerJoinSuite and StreamingOuterJoinSuite passed as well

Closes #28975 from HeartSaVioR/SPARK-32148.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 526cb2d1ba2b4c07e10d7011367fdef24a40a927)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../StreamingSymmetricHashJoinExec.scala      |  4 ++
 .../state/SymmetricHashJoinStateManager.scala | 17 ++++++-
 .../sql/streaming/StreamingJoinSuite.scala    | 44 +++++++++++++++++++
 3 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
index 198e17db419a7..57e62dcf513ea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/StreamingSymmetricHashJoinExec.scala
@@ -295,6 +295,10 @@ case class StreamingSymmetricHashJoinExec(
             postJoinFilter(joinedRow.withLeft(leftKeyValue.value).withRight(rightValue))
           }
         }
+
+        // NOTE: we need to make sure `outerOutputIter` is evaluated "after" exhausting all of
+        // elements in `innerOutputIter`, because evaluation of `innerOutputIter` may update
+        // the match flag which the logic for outer join is relying on.
         val removedRowIter = leftSideJoiner.removeOldState()
         val outerOutputIter = removedRowIter.filterNot { kv =>
           stateFormatVersion match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
index 1a0a43c083879..1a5b50dcc7901 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/SymmetricHashJoinStateManager.scala
@@ -451,10 +451,25 @@ class SymmetricHashJoinStateManager(
   }
 
   private trait KeyWithIndexToValueRowConverter {
+    /** Defines the schema of the value row (the value side of K-V in state store). */
     def valueAttributes: Seq[Attribute]
 
+    /**
+     * Convert the value row to (actual value, match) pair.
+     *
+     * NOTE: implementations should ensure the result row is NOT reused during execution, so
+     * that caller can safely read the value in any time.
+     */
     def convertValue(value: UnsafeRow): ValueAndMatchPair
 
+    /**
+     * Build the value row from (actual value, match) pair. This is expected to be called just
+     * before storing to the state store.
+     *
+     * NOTE: depending on the implementation, the result row "may" be reused during execution
+     * (to avoid initialization of object), so the caller should ensure that the logic doesn't
+     * affect by such behavior. Call copy() against the result row if needed.
+     */
     def convertToValueRow(value: UnsafeRow, matched: Boolean): UnsafeRow
   }
 
@@ -493,7 +508,7 @@ class SymmetricHashJoinStateManager(
 
     override def convertValue(value: UnsafeRow): ValueAndMatchPair = {
       if (value != null) {
-        ValueAndMatchPair(valueRowGenerator(value),
+        ValueAndMatchPair(valueRowGenerator(value).copy(),
           value.getBoolean(indexOrdinalInValueWithMatchedRow))
       } else {
         null
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
index 3f218c9cb7fd9..7837b20cd6830 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala
@@ -18,6 +18,7 @@
 package org.apache.spark.sql.streaming
 
 import java.io.File
+import java.sql.Timestamp
 import java.util.{Locale, UUID}
 
 import scala.util.Random
@@ -991,4 +992,47 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
       )
     }
   }
+
+  test("SPARK-32148 stream-stream join regression on Spark 3.0.0") {
+    val input1 = MemoryStream[(Timestamp, String, String)]
+    val df1 = input1.toDF
+      .selectExpr("_1 as eventTime", "_2 as id", "_3 as comment")
+      .withWatermark(s"eventTime", "2 minutes")
+
+    val input2 = MemoryStream[(Timestamp, String, String)]
+    val df2 = input2.toDF
+      .selectExpr("_1 as eventTime", "_2 as id", "_3 as name")
+      .withWatermark(s"eventTime", "4 minutes")
+
+    val joined = df1.as("left")
+      .join(df2.as("right"),
+        expr("""
+               |left.id = right.id AND left.eventTime BETWEEN
+               |  right.eventTime - INTERVAL 30 seconds AND
+               |  right.eventTime + INTERVAL 30 seconds
+             """.stripMargin),
+        joinType = "leftOuter")
+
+    val inputDataForInput1 = Seq(
+      (Timestamp.valueOf("2020-01-01 00:00:00"), "abc", "has no join partner"),
+      (Timestamp.valueOf("2020-01-02 00:00:00"), "abc", "joined with A"),
+      (Timestamp.valueOf("2020-01-02 01:00:00"), "abc", "joined with B"))
+
+    val inputDataForInput2 = Seq(
+      (Timestamp.valueOf("2020-01-02 00:00:10"), "abc", "A"),
+      (Timestamp.valueOf("2020-01-02 00:59:59"), "abc", "B"),
+      (Timestamp.valueOf("2020-01-02 02:00:00"), "abc", "C"))
+
+    val expectedOutput = Seq(
+      (Timestamp.valueOf("2020-01-01 00:00:00"), "abc", "has no join partner", null, null, null),
+      (Timestamp.valueOf("2020-01-02 00:00:00"), "abc", "joined with A",
+        Timestamp.valueOf("2020-01-02 00:00:10"), "abc", "A"),
+      (Timestamp.valueOf("2020-01-02 01:00:00"), "abc", "joined with B",
+        Timestamp.valueOf("2020-01-02 00:59:59"), "abc", "B"))
+
+    testStream(joined)(
+      MultiAddData((input1, inputDataForInput1), (input2, inputDataForInput2)),
+      CheckNewAnswer(expectedOutput.head, expectedOutput.tail: _*)
+    )
+  }
 }

From 30e49c23249213a70266583566db93e113a9e23e Mon Sep 17 00:00:00 2001
From: Erik Erlandson <eerlands@redhat.com>
Date: Thu, 9 Jul 2020 08:42:20 +0000
Subject: [PATCH 0891/1280] [SPARK-32159][SQL] Fix integration between
 Aggregator[Array[_], _, _] and UnresolvedMapObjects

Context: The fix for SPARK-27296 introduced by #25024 allows `Aggregator` objects to appear in queries. This works fine for aggregators with atomic input types, e.g. `Aggregator[Double, _, _]`.

However it can cause a null pointer exception if the input type is `Array[_]`.  This was historically considered an ignorable case for serialization of `UnresolvedMapObjects`, but the new ScalaAggregator class causes these expressions to be serialized over to executors because the resolve-and-bind is being deferred.

### What changes were proposed in this pull request?
A new rule `ResolveEncodersInScalaAgg` that performs the resolution of the expressions contained in the encoders so that properly resolved expressions are serialized over to executors.

### Why are the changes needed?
Applying an aggregator of the form `Aggregator[Array[_], _, _]` using `functions.udaf()` currently causes a null pointer error in Catalyst.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
A unit test has been added that does aggregation with array types for input, buffer, and output. I have done additional testing with my own custom aggregators in the spark REPL.

Closes #28983 from erikerlandson/fix-spark-32159.

Authored-by: Erik Erlandson <eerlands@redhat.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1cb5bfc47a2b4fff824433f8cecabfbac7e050b6)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/objects/objects.scala         |  7 +++
 .../spark/sql/execution/aggregate/udaf.scala  | 27 +++++++++--
 .../sql/expressions/UserDefinedFunction.scala |  3 +-
 .../internal/BaseSessionStateBuilder.scala    |  2 +
 .../sql/hive/HiveSessionStateBuilder.scala    |  2 +
 .../sql/hive/execution/UDAQuerySuite.scala    | 47 +++++++++++++------
 6 files changed, 68 insertions(+), 20 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index d5de95c65e49e..ab2f66b1a53e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -678,6 +678,13 @@ object MapObjects {
       elementType: DataType,
       elementNullable: Boolean = true,
       customCollectionCls: Option[Class[_]] = None): MapObjects = {
+    // UnresolvedMapObjects does not serialize its 'function' field.
+    // If an array expression or array Encoder is not correctly resolved before
+    // serialization, this exception condition may occur.
+    require(function != null,
+      "MapObjects applied with a null function. " +
+      "Likely cause is failure to resolve an array expression or encoder. " +
+      "(See UnresolvedMapObjects)")
     val loopVar = LambdaVariable("MapObject", elementType, elementNullable)
     MapObjects(loopVar, function(loopVar), inputData, customCollectionCls)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
index 544b90a736071..44bc9c2e3a9d0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/udaf.scala
@@ -27,6 +27,8 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Complete}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate}
 import org.apache.spark.sql.catalyst.expressions.codegen.{GenerateMutableProjection, GenerateSafeProjection}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, UserDefinedAggregateFunction}
 import org.apache.spark.sql.types._
 
@@ -458,7 +460,8 @@ case class ScalaUDAF(
 case class ScalaAggregator[IN, BUF, OUT](
     children: Seq[Expression],
     agg: Aggregator[IN, BUF, OUT],
-    inputEncoderNR: ExpressionEncoder[IN],
+    inputEncoder: ExpressionEncoder[IN],
+    bufferEncoder: ExpressionEncoder[BUF],
     nullable: Boolean = true,
     isDeterministic: Boolean = true,
     mutableAggBufferOffset: Int = 0,
@@ -469,9 +472,8 @@ case class ScalaAggregator[IN, BUF, OUT](
   with ImplicitCastInputTypes
   with Logging {
 
-  private[this] lazy val inputDeserializer = inputEncoderNR.resolveAndBind().createDeserializer()
-  private[this] lazy val bufferEncoder =
-    agg.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]].resolveAndBind()
+  // input and buffer encoders are resolved by ResolveEncodersInScalaAgg
+  private[this] lazy val inputDeserializer = inputEncoder.createDeserializer()
   private[this] lazy val bufferSerializer = bufferEncoder.createSerializer()
   private[this] lazy val bufferDeserializer = bufferEncoder.createDeserializer()
   private[this] lazy val outputEncoder = agg.outputEncoder.asInstanceOf[ExpressionEncoder[OUT]]
@@ -479,7 +481,7 @@ case class ScalaAggregator[IN, BUF, OUT](
 
   def dataType: DataType = outputEncoder.objSerializer.dataType
 
-  def inputTypes: Seq[DataType] = inputEncoderNR.schema.map(_.dataType)
+  def inputTypes: Seq[DataType] = inputEncoder.schema.map(_.dataType)
 
   override lazy val deterministic: Boolean = isDeterministic
 
@@ -517,3 +519,18 @@ case class ScalaAggregator[IN, BUF, OUT](
 
   override def nodeName: String = agg.getClass.getSimpleName
 }
+
+/**
+ * An extension rule to resolve encoder expressions from a [[ScalaAggregator]]
+ */
+object ResolveEncodersInScalaAgg extends Rule[LogicalPlan] {
+  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    case p if !p.resolved => p
+    case p => p.transformExpressionsUp {
+      case agg: ScalaAggregator[_, _, _] =>
+        agg.copy(
+          inputEncoder = agg.inputEncoder.resolveAndBind(),
+          bufferEncoder = agg.bufferEncoder.resolveAndBind())
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
index 2ef6e3d291cef..6a20a46756f85 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/UserDefinedFunction.scala
@@ -150,7 +150,8 @@ private[sql] case class UserDefinedAggregator[IN, BUF, OUT](
   // This is also used by udf.register(...) when it detects a UserDefinedAggregator
   def scalaAggregator(exprs: Seq[Expression]): ScalaAggregator[IN, BUF, OUT] = {
     val iEncoder = inputEncoder.asInstanceOf[ExpressionEncoder[IN]]
-    ScalaAggregator(exprs, aggregator, iEncoder, nullable, deterministic)
+    val bEncoder = aggregator.bufferEncoder.asInstanceOf[ExpressionEncoder[BUF]]
+    ScalaAggregator(exprs, aggregator, iEncoder, bEncoder, nullable, deterministic)
   }
 
   override def withName(name: String): UserDefinedAggregator[IN, BUF, OUT] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 3bbdbb002cca8..4ae12f8716752 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlanner, SparkSqlParser}
+import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
 import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
@@ -175,6 +176,7 @@ abstract class BaseSessionStateBuilder(
       new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
+        ResolveEncodersInScalaAgg +:
         new ResolveSessionCatalog(
           catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
         customResolutionRules
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
index 64726755237a6..e25610757a69b 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionStateBuilder.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{SparkOptimizer, SparkPlanner}
+import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
 import org.apache.spark.sql.execution.command.CommandCheck
 import org.apache.spark.sql.execution.datasources._
@@ -76,6 +77,7 @@ class HiveSessionStateBuilder(session: SparkSession, parentState: Option[Session
         new FindDataSourceTable(session) +:
         new ResolveSQLOnFile(session) +:
         new FallBackFileSourceV2(session) +:
+        ResolveEncodersInScalaAgg +:
         new ResolveSessionCatalog(
           catalogManager, conf, catalog.isTempView, catalog.isTempFunction) +:
         customResolutionRules
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
index e6856a58b0ea9..1f1a5568b0201 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/UDAQuerySuite.scala
@@ -119,6 +119,27 @@ object CountSerDeAgg extends Aggregator[Int, CountSerDeSQL, CountSerDeSQL] {
   def outputEncoder: Encoder[CountSerDeSQL] = ExpressionEncoder[CountSerDeSQL]()
 }
 
+object ArrayDataAgg extends Aggregator[Array[Double], Array[Double], Array[Double]] {
+  def zero: Array[Double] = Array(0.0, 0.0, 0.0)
+  def reduce(s: Array[Double], array: Array[Double]): Array[Double] = {
+    require(s.length == array.length)
+    for ( j <- 0 until s.length ) {
+      s(j) += array(j)
+    }
+    s
+  }
+  def merge(s1: Array[Double], s2: Array[Double]): Array[Double] = {
+    require(s1.length == s2.length)
+    for ( j <- 0 until s1.length ) {
+      s1(j) += s2(j)
+    }
+    s1
+  }
+  def finish(s: Array[Double]): Array[Double] = s
+  def bufferEncoder: Encoder[Array[Double]] = ExpressionEncoder[Array[Double]]
+  def outputEncoder: Encoder[Array[Double]] = ExpressionEncoder[Array[Double]]
+}
+
 abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
   import testImplicits._
 
@@ -156,20 +177,11 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
       (3, null, null)).toDF("key", "value1", "value2")
     data2.write.saveAsTable("agg2")
 
-    val data3 = Seq[(Seq[Integer], Integer, Integer)](
-      (Seq[Integer](1, 1), 10, -10),
-      (Seq[Integer](null), -60, 60),
-      (Seq[Integer](1, 1), 30, -30),
-      (Seq[Integer](1), 30, 30),
-      (Seq[Integer](2), 1, 1),
-      (null, -10, 10),
-      (Seq[Integer](2, 3), -1, null),
-      (Seq[Integer](2, 3), 1, 1),
-      (Seq[Integer](2, 3, 4), null, 1),
-      (Seq[Integer](null), 100, -10),
-      (Seq[Integer](3), null, 3),
-      (null, null, null),
-      (Seq[Integer](3), null, null)).toDF("key", "value1", "value2")
+    val data3 = Seq[(Seq[Double], Int)](
+      (Seq(1.0, 2.0, 3.0), 0),
+      (Seq(4.0, 5.0, 6.0), 0),
+      (Seq(7.0, 8.0, 9.0), 0)
+    ).toDF("data", "dummy")
     data3.write.saveAsTable("agg3")
 
     val data4 = Seq[Boolean](true, false, true).toDF("boolvalues")
@@ -184,6 +196,7 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
     spark.udf.register("mydoublesum", udaf(MyDoubleSumAgg))
     spark.udf.register("mydoubleavg", udaf(MyDoubleAvgAgg))
     spark.udf.register("longProductSum", udaf(LongProductSumAgg))
+    spark.udf.register("arraysum", udaf(ArrayDataAgg))
   }
 
   override def afterAll(): Unit = {
@@ -354,6 +367,12 @@ abstract class UDAQuerySuite extends QueryTest with SQLTestUtils with TestHiveSi
         Row(3, 0, null, 1, 3, 0, 0, 0, null, 1, 3, 0, 2, 2) :: Nil)
   }
 
+  test("SPARK-32159: array encoders should be resolved in analyzer") {
+    checkAnswer(
+      spark.sql("SELECT arraysum(data) FROM agg3"),
+      Row(Seq(12.0, 15.0, 18.0)) :: Nil)
+  }
+
   test("verify aggregator ser/de behavior") {
     val data = sparkContext.parallelize((1 to 100).toSeq, 3).toDF("value1")
     val agg = udaf(CountSerDeAgg)

From bad92b98e1febf1aed17434b96e037f94d4a7738 Mon Sep 17 00:00:00 2001
From: moovlin <richjoerger@gmail.com>
Date: Thu, 9 Jul 2020 10:35:21 -0700
Subject: [PATCH 0892/1280] [SPARK-32035][DOCS][EXAMPLES] Fixed typos involving
 AWS Access, Secret, & Sessions tokens

### What changes were proposed in this pull request?
I resolved some of the inconsistencies of AWS env variables. They're fixed in the documentation as well as in the examples. I grep-ed through the repo to try & find any more instances but nothing popped up.

### Why are the changes needed?

As previously mentioned, there is a JIRA request, SPARK-32035, which encapsulates all the issues. But, in summary, the naming of items was inconsistent.

### Does this PR introduce _any_ user-facing change?

Correct names:
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
AWS_SESSION_TOKEN
These are the same that AWS uses in their libraries.

However, looking through the Spark documentation and comments, I see that these are not denoted correctly across the board:

docs/cloud-integration.md
106:1. `spark-submit` reads the `AWS_ACCESS_KEY`, `AWS_SECRET_KEY` <-- both different
107:and `AWS_SESSION_TOKEN` environment variables and sets the associated authentication options

docs/streaming-kinesis-integration.md
232:- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY` with your AWS credentials. <-- secret key different

external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
34: $ export AWS_ACCESS_KEY_ID=<your-access-key>
35: $ export AWS_SECRET_KEY=<your-secret-key> <-- different
48: Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY <-- secret key different

core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
438: val keyId = System.getenv("AWS_ACCESS_KEY_ID")
439: val accessKey = System.getenv("AWS_SECRET_ACCESS_KEY")
448: val sessionToken = System.getenv("AWS_SESSION_TOKEN")

external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
53: * $ export AWS_ACCESS_KEY_ID=<your-access-key>
54: * $ export AWS_SECRET_KEY=<your-secret-key> <-- different
65: * Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY <-- secret key different

external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
59: * $ export AWS_ACCESS_KEY_ID=[your-access-key]
60: * $ export AWS_SECRET_KEY=<your-secret-key> <-- different
71: * Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY <-- secret key different

These were all fixed to match names listed under the "correct names" heading.

### How was this patch tested?

I built the documentation using jekyll and verified that the changes were present & accurate.

Closes #29058 from Moovlin/SPARK-32035.

Authored-by: moovlin <richjoerger@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 9331a5c44baa79998625829e9be624e8564c91ea)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/cloud-integration.md                                     | 2 +-
 docs/streaming-kinesis-integration.md                         | 2 +-
 .../spark/examples/streaming/JavaKinesisWordCountASL.java     | 4 ++--
 .../main/python/examples/streaming/kinesis_wordcount_asl.py   | 4 ++--
 .../apache/spark/examples/streaming/KinesisWordCountASL.scala | 4 ++--
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/cloud-integration.md b/docs/cloud-integration.md
index 01f437f38ef17..749d026528017 100644
--- a/docs/cloud-integration.md
+++ b/docs/cloud-integration.md
@@ -103,7 +103,7 @@ for talking to cloud infrastructures, in which case this module may not be neede
 Spark jobs must authenticate with the object stores to access data within them.
 
 1. When Spark is running in a cloud infrastructure, the credentials are usually automatically set up.
-1. `spark-submit` reads the `AWS_ACCESS_KEY`, `AWS_SECRET_KEY`
+1. `spark-submit` reads the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`
 and `AWS_SESSION_TOKEN` environment variables and sets the associated authentication options
 for the `s3n` and `s3a` connectors to Amazon S3.
 1. In a Hadoop cluster, settings may be set in the `core-site.xml` file.
diff --git a/docs/streaming-kinesis-integration.md b/docs/streaming-kinesis-integration.md
index db813c46949c2..c7959d4201151 100644
--- a/docs/streaming-kinesis-integration.md
+++ b/docs/streaming-kinesis-integration.md
@@ -229,7 +229,7 @@ To run the example,
 
 - Set up Kinesis stream (see earlier section) within AWS. Note the name of the Kinesis stream and the endpoint URL corresponding to the region where the stream was created.
 
-- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY` with your AWS credentials.
+- Set up the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` with your AWS credentials.
 
 - In the Spark root directory, run the example as
 
diff --git a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
index 31ca2fe5c95ff..d704aeb507518 100644
--- a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
+++ b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
@@ -57,7 +57,7 @@
  * Example:
  *      # export AWS keys if necessary
  *      $ export AWS_ACCESS_KEY_ID=[your-access-key]
- *      $ export AWS_SECRET_KEY=<your-secret-key>
+ *      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
  *
  *      # run the example
  *      $ SPARK_HOME/bin/run-example   streaming.JavaKinesisWordCountASL myAppName  mySparkStream \
@@ -68,7 +68,7 @@
  *
  * This code uses the DefaultAWSCredentialsProviderChain to find credentials
  * in the following order:
- *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+ *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
  *    Java System Properties - aws.accessKeyId and aws.secretKey
  *    Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
  *    Instance profile credentials - delivered through the Amazon EC2 metadata service
diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 777a33270c415..5370b793897bb 100644
--- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -32,7 +32,7 @@
   Example:
       # export AWS keys if necessary
       $ export AWS_ACCESS_KEY_ID=<your-access-key>
-      $ export AWS_SECRET_KEY=<your-secret-key>
+      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
 
       # run the example
       $ bin/spark-submit --jars \
@@ -45,7 +45,7 @@
 
   This code uses the DefaultAWSCredentialsProviderChain to find credentials
   in the following order:
-      Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+      Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
       Java System Properties - aws.accessKeyId and aws.secretKey
       Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
       Instance profile credentials - delivered through the Amazon EC2 metadata service
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index 32f4a6759474f..bbb6008c2dddf 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -51,7 +51,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  * Example:
  *      # export AWS keys if necessary
  *      $ export AWS_ACCESS_KEY_ID=<your-access-key>
- *      $ export AWS_SECRET_KEY=<your-secret-key>
+ *      $ export AWS_SECRET_ACCESS_KEY=<your-secret-key>
  *
  *      # run the example
  *      $ SPARK_HOME/bin/run-example  streaming.KinesisWordCountASL myAppName  mySparkStream \
@@ -62,7 +62,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  *
  * This code uses the DefaultAWSCredentialsProviderChain to find credentials
  * in the following order:
- *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_KEY
+ *    Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
  *    Java System Properties - aws.accessKeyId and aws.secretKey
  *    Credential profiles file - default location (~/.aws/credentials) shared by all AWS SDKs
  *    Instance profile credentials - delivered through the Amazon EC2 metadata service

From 9dbe1964334e11bf3715fc5c1b0edac80f78d20b Mon Sep 17 00:00:00 2001
From: Shixiong Zhu <zsxwing@gmail.com>
Date: Fri, 10 Jul 2020 21:14:29 +0900
Subject: [PATCH 0893/1280] [SPARK-32256][SQL][TEST-HADOOP2.7] Force to
 initialize Hadoop VersionInfo in HiveExternalCatalog

### What changes were proposed in this pull request?

Force to initialize Hadoop VersionInfo in HiveExternalCatalog to make sure Hive can get the Hadoop version when using the isolated classloader.

### Why are the changes needed?

This is a regression in Spark 3.0.0 because we switched the default Hive execution version from 1.2.1 to 2.3.7.

Spark allows the user to set `spark.sql.hive.metastore.jars` to specify jars to access Hive Metastore. These jars are loaded by the isolated classloader. Because we also share Hadoop classes with the isolated classloader, the user doesn't need to add Hadoop jars to `spark.sql.hive.metastore.jars`, which means when we are using the isolated classloader, hadoop-common jar is not available in this case. If Hadoop VersionInfo is not initialized before we switch to the isolated classloader, and we try to initialize it using the isolated classloader (the current thread context classloader), it will fail and report `Unknown` which causes Hive to throw the following exception:

```
java.lang.RuntimeException: Illegal Hadoop Version: Unknown (expected A.B.* format)
	at org.apache.hadoop.hive.shims.ShimLoader.getMajorVersion(ShimLoader.java:147)
	at org.apache.hadoop.hive.shims.ShimLoader.loadShims(ShimLoader.java:122)
	at org.apache.hadoop.hive.shims.ShimLoader.getHadoopShims(ShimLoader.java:88)
	at org.apache.hadoop.hive.metastore.ObjectStore.getDataSourceProps(ObjectStore.java:377)
	at org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:268)
	at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:76)
	at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:136)
	at org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:58)
	at org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:67)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStore(HiveMetaStore.java:517)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:482)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:544)
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:370)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78)
	at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:84)
	at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762)
	at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:219)
	at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:67)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1548)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132)
	at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)
	at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3080)
	at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3108)
	at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3349)
	at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:217)
	at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:204)
	at org.apache.hadoop.hive.ql.metadata.Hive.<init>(Hive.java:331)
	at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:292)
	at org.apache.hadoop.hive.ql.metadata.Hive.getInternal(Hive.java:262)
	at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:247)
	at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:543)
	at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:511)
	at org.apache.spark.sql.hive.client.HiveClientImpl.newState(HiveClientImpl.scala:175)
	at org.apache.spark.sql.hive.client.HiveClientImpl.<init>(HiveClientImpl.scala:128)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:301)
	at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:431)
	at org.apache.spark.sql.hive.HiveUtils$.newClientForMetadata(HiveUtils.scala:324)
	at org.apache.spark.sql.hive.HiveExternalCatalog.client$lzycompute(HiveExternalCatalog.scala:72)
	at org.apache.spark.sql.hive.HiveExternalCatalog.client(HiveExternalCatalog.scala:71)
	at org.apache.spark.sql.hive.client.HadoopVersionInfoSuite.$anonfun$new$1(HadoopVersionInfoSuite.scala:63)
	at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
	at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
```

Technically, This is indeed an issue of Hadoop VersionInfo which has been fixed: https://issues.apache.org/jira/browse/HADOOP-14067. But since we are still supporting old Hadoop versions, we should fix it.

Why this issue starts to happen in Spark 3.0.0?

In Spark 2.4.x, we use Hive 1.2.1 by default. It will trigger `VersionInfo` initialization in the static codes of `Hive` class. This will happen when we load `HiveClientImpl` class because `HiveClientImpl.clent` method refers to `Hive` class. At this moment, the thread context classloader is not using the isolcated classloader, so it can access hadoop-common jar on the classpath and initialize it correctly.

In Spark 3.0.0, we use Hive 2.3.7. The static codes of `Hive` class are not accessing `VersionInfo` because of the change in https://issues.apache.org/jira/browse/HIVE-11657. Instead, accessing `VersionInfo` happens when creating a `Hive` object (See the above stack trace). This happens here https://github.com/apache/spark/blob/v3.0.0/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala#L260. But we switch to the isolated classloader before calling `HiveClientImpl.client` (See https://github.com/apache/spark/blob/v3.0.0/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala#L283). This is exactly what I mentioned above: `If Hadoop VersionInfo is not initialized before we switch to the isolated classloader, and we try to initialize it using the isolated classloader (the current thread context classloader), it will fail`

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

The new regression test added in this PR.

Note that the new UT doesn't fail with the default profiles (-Phadoop-3.2) because it's already fixed at Hadoop 3.1. Please use the following to verify this.
```
build/sbt -Phadoop-2.7 -Phive "hive/testOnly *.HadoopVersionInfoSuite"
```

Closes #29059 from zsxwing/SPARK-32256.

Authored-by: Shixiong Zhu <zsxwing@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c8779d9dfc5bf7dbaa19a043d8312c3909499da6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 project/SparkBuild.scala                      |  1 +
 .../spark/sql/hive/HiveExternalCatalog.scala  |  4 ++
 .../hive/client/HadoopVersionInfoSuite.scala  | 72 +++++++++++++++++++
 .../sql/hive/client/HiveClientBuilder.scala   |  2 +-
 4 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index f2db38922c630..a0e138c5bcb65 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -475,6 +475,7 @@ object SparkParallelTestGrouping {
     "org.apache.spark.ml.classification.LogisticRegressionSuite",
     "org.apache.spark.ml.classification.LinearSVCSuite",
     "org.apache.spark.sql.SQLQueryTestSuite",
+    "org.apache.spark.sql.hive.client.HadoopVersionInfoSuite",
     "org.apache.spark.sql.hive.thriftserver.SparkExecuteStatementOperationSuite",
     "org.apache.spark.sql.hive.thriftserver.ThriftServerQueryTestSuite",
     "org.apache.spark.sql.hive.thriftserver.SparkSQLEnvSuite",
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 2faf42028f3a2..571c25e356c08 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -61,6 +61,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
   import HiveExternalCatalog._
   import CatalogTableType._
 
+  // SPARK-32256: Make sure `VersionInfo` is initialized before touching the isolated classloader.
+  // This is to ensure Hive can get the Hadoop version when using the isolated classloader.
+  org.apache.hadoop.util.VersionInfo.getVersion()
+
   /**
    * A Hive client used to interact with the metastore.
    */
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala
new file mode 100644
index 0000000000000..65492abf38cc0
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.client
+
+import java.io.File
+import java.net.URLClassLoader
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
+import org.apache.spark.util.Utils
+
+/**
+ * This test suite requires a clean JVM because it's testing the initialization of static codes in
+ * `org.apache.hadoop.util.VersionInfo`.
+ */
+class HadoopVersionInfoSuite extends SparkFunSuite {
+  override protected val enableAutoThreadAudit = false
+
+  test("SPARK-32256: Hadoop VersionInfo should be preloaded") {
+    val ivyPath =
+      Utils.createTempDir(namePrefix = s"${classOf[HadoopVersionInfoSuite].getSimpleName}-ivy")
+    try {
+      val hadoopConf = new Configuration()
+      hadoopConf.set("test", "success")
+      hadoopConf.set("datanucleus.schema.autoCreateAll", "true")
+      hadoopConf.set("hive.metastore.schema.verification", "false")
+
+      // Download jars for Hive 2.0
+      val client = IsolatedClientLoader.forVersion(
+        hiveMetastoreVersion = "2.0",
+        hadoopVersion = "2.7.4",
+        sparkConf = new SparkConf(),
+        hadoopConf = hadoopConf,
+        config = HiveClientBuilder.buildConf(Map.empty),
+        ivyPath = Some(ivyPath.getCanonicalPath),
+        sharesHadoopClasses = true)
+      val jars = client.classLoader.getParent.asInstanceOf[URLClassLoader].getURLs
+        .map(u => new File(u.toURI))
+        // Drop all Hadoop jars to use the existing Hadoop jars on the classpath
+        .filter(!_.getName.startsWith("org.apache.hadoop_hadoop-"))
+
+      val sparkConf = new SparkConf()
+      sparkConf.set(HiveUtils.HIVE_METASTORE_VERSION, "2.0")
+      sparkConf.set(
+        HiveUtils.HIVE_METASTORE_JARS,
+        jars.map(_.getCanonicalPath).mkString(File.pathSeparator))
+      HiveClientBuilder.buildConf(Map.empty).foreach { case (k, v) =>
+        hadoopConf.set(k, v)
+      }
+      new HiveExternalCatalog(sparkConf, hadoopConf).client.getState
+    } finally {
+      Utils.deleteRecursively(ivyPath)
+    }
+  }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
index ab73f668c6ca6..2ad3afcb214b3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientBuilder.scala
@@ -33,7 +33,7 @@ private[client] object HiveClientBuilder {
       Some(new File(sys.props("java.io.tmpdir"), "hive-ivy-cache").getAbsolutePath))
   }
 
-  private def buildConf(extraConf: Map[String, String]) = {
+  private[client] def buildConf(extraConf: Map[String, String]): Map[String, String] = {
     lazy val warehousePath = Utils.createTempDir()
     lazy val metastorePath = Utils.createTempDir()
     metastorePath.delete()

From 3b0aee3f9557ae1666b63665b08d899fe7682852 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 10 Jul 2020 15:10:28 -0700
Subject: [PATCH 0894/1280] [SPARK-32251][SQL][DOCS][TESTS] Fix SQL keyword
 document

### What changes were proposed in this pull request?

This PR improves the test to make sure all the SQL keywords are documented correctly. It fixes several issues:
1. some keywords are not documented
2. some keywords are not ANSI SQL keywords but documented as reserved/non-reserved.

### Why are the changes needed?

To make sure the implementation matches the doc.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

new test

Closes #29055 from cloud-fan/keyword.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 84db660ebef4f9c543ab2709103c4542b407a829)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/sql-ref-ansi-compliance.md               |  13 +-
 .../spark/sql/catalyst/parser/SqlBase.g4      |   2 +-
 .../spark/sql/catalyst/SQLKeywordSuite.scala  | 170 ++++++++++++++++++
 .../parser/TableIdentifierParserSuite.scala   | 127 +------------
 4 files changed, 182 insertions(+), 130 deletions(-)
 create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index e5ca7e9d10d59..6488ad9cd34c9 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -127,7 +127,7 @@ By default `spark.sql.ansi.enabled` is false.
 
 Below is a list of all the keywords in Spark SQL.
 
-|Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>Default Mode|SQL-2011|
+|Keyword|Spark SQL<br/>ANSI Mode|Spark SQL<br/>Default Mode|SQL-2016|
 |-------|----------------------|-------------------------|--------|
 |ADD|non-reserved|non-reserved|non-reserved|
 |AFTER|non-reserved|non-reserved|non-reserved|
@@ -149,7 +149,7 @@ Below is a list of all the keywords in Spark SQL.
 |BUCKETS|non-reserved|non-reserved|non-reserved|
 |BY|non-reserved|non-reserved|reserved|
 |CACHE|non-reserved|non-reserved|non-reserved|
-|CASCADE|non-reserved|non-reserved|reserved|
+|CASCADE|non-reserved|non-reserved|non-reserved|
 |CASE|reserved|non-reserved|reserved|
 |CAST|reserved|non-reserved|reserved|
 |CHANGE|non-reserved|non-reserved|non-reserved|
@@ -193,7 +193,7 @@ Below is a list of all the keywords in Spark SQL.
 |DIRECTORY|non-reserved|non-reserved|non-reserved|
 |DISTINCT|reserved|non-reserved|reserved|
 |DISTRIBUTE|non-reserved|non-reserved|non-reserved|
-|DIV|non-reserved|non-reserved|non-reserved|
+|DIV|non-reserved|non-reserved|not a keyword|
 |DROP|non-reserved|non-reserved|reserved|
 |ELSE|reserved|non-reserved|reserved|
 |END|reserved|non-reserved|reserved|
@@ -228,7 +228,7 @@ Below is a list of all the keywords in Spark SQL.
 |GROUPING|non-reserved|non-reserved|reserved|
 |HAVING|reserved|non-reserved|reserved|
 |HOUR|reserved|non-reserved|reserved|
-|IF|non-reserved|non-reserved|reserved|
+|IF|non-reserved|non-reserved|not a keyword|
 |IGNORE|non-reserved|non-reserved|non-reserved|
 |IMPORT|non-reserved|non-reserved|non-reserved|
 |IN|reserved|non-reserved|reserved|
@@ -302,12 +302,14 @@ Below is a list of all the keywords in Spark SQL.
 |PROPERTIES|non-reserved|non-reserved|non-reserved|
 |PURGE|non-reserved|non-reserved|non-reserved|
 |QUERY|non-reserved|non-reserved|non-reserved|
+|RANGE|non-reserved|non-reserved|reserved|
 |RECORDREADER|non-reserved|non-reserved|non-reserved|
 |RECORDWRITER|non-reserved|non-reserved|non-reserved|
 |RECOVER|non-reserved|non-reserved|non-reserved|
 |REDUCE|non-reserved|non-reserved|non-reserved|
 |REFERENCES|reserved|non-reserved|reserved|
 |REFRESH|non-reserved|non-reserved|non-reserved|
+|REGEXP|non-reserved|non-reserved|not a keyword|
 |RENAME|non-reserved|non-reserved|non-reserved|
 |REPAIR|non-reserved|non-reserved|non-reserved|
 |REPLACE|non-reserved|non-reserved|non-reserved|
@@ -323,6 +325,7 @@ Below is a list of all the keywords in Spark SQL.
 |ROW|non-reserved|non-reserved|reserved|
 |ROWS|non-reserved|non-reserved|reserved|
 |SCHEMA|non-reserved|non-reserved|non-reserved|
+|SCHEMAS|non-reserved|non-reserved|not a keyword|
 |SECOND|reserved|non-reserved|reserved|
 |SELECT|reserved|non-reserved|reserved|
 |SEMI|non-reserved|strict-non-reserved|non-reserved|
@@ -348,6 +351,7 @@ Below is a list of all the keywords in Spark SQL.
 |TABLES|non-reserved|non-reserved|non-reserved|
 |TABLESAMPLE|non-reserved|non-reserved|reserved|
 |TBLPROPERTIES|non-reserved|non-reserved|non-reserved|
+|TEMP|non-reserved|non-reserved|not a keyword|
 |TEMPORARY|non-reserved|non-reserved|non-reserved|
 |TERMINATED|non-reserved|non-reserved|non-reserved|
 |THEN|reserved|non-reserved|reserved|
@@ -360,6 +364,7 @@ Below is a list of all the keywords in Spark SQL.
 |TRIM|non-reserved|non-reserved|non-reserved|
 |TRUE|non-reserved|non-reserved|reserved|
 |TRUNCATE|non-reserved|non-reserved|reserved|
+|TYPE|non-reserved|non-reserved|non-reserved|
 |UNARCHIVE|non-reserved|non-reserved|non-reserved|
 |UNBOUNDED|non-reserved|non-reserved|non-reserved|
 |UNCACHE|non-reserved|non-reserved|non-reserved|
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index 75dae8f3faf6d..e02bcda52f618 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -1511,6 +1511,7 @@ DIRECTORIES: 'DIRECTORIES';
 DIRECTORY: 'DIRECTORY';
 DISTINCT: 'DISTINCT';
 DISTRIBUTE: 'DISTRIBUTE';
+DIV: 'DIV';
 DROP: 'DROP';
 ELSE: 'ELSE';
 END: 'END';
@@ -1718,7 +1719,6 @@ MINUS: '-';
 ASTERISK: '*';
 SLASH: '/';
 PERCENT: '%';
-DIV: 'DIV';
 TILDE: '~';
 AMPERSAND: '&';
 PIPE: '|';
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
new file mode 100644
index 0000000000000..3d41d02b23df5
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst
+
+import java.io.File
+import java.nio.file.Files
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.plans.SQLHelper
+import org.apache.spark.sql.catalyst.util.fileToString
+
+trait SQLKeywordUtils extends SQLHelper {
+
+  val sqlSyntaxDefs = {
+    val sqlBasePath = {
+      java.nio.file.Paths.get(sparkHome, "sql", "catalyst", "src", "main", "antlr4", "org",
+        "apache", "spark", "sql", "catalyst", "parser", "SqlBase.g4").toFile
+    }
+    fileToString(sqlBasePath).split("\n")
+  }
+
+  // each element is an array of 4 string: the keyword name, reserve or not in Spark ANSI mode,
+  // Spark non-ANSI mode, and the SQL standard.
+  val keywordsInDoc: Array[Array[String]] = {
+    val docPath = {
+      java.nio.file.Paths.get(sparkHome, "docs", "sql-ref-ansi-compliance.md").toFile
+    }
+    fileToString(docPath).split("\n")
+      .dropWhile(!_.startsWith("|Keyword|")).drop(2).takeWhile(_.startsWith("|"))
+      .map(_.stripPrefix("|").split("\\|").map(_.trim))
+  }
+
+  private def parseAntlrGrammars[T](startTag: String, endTag: String)
+      (f: PartialFunction[String, Seq[T]]): Set[T] = {
+    val keywords = new mutable.ArrayBuffer[T]
+    val default = (_: String) => Nil
+    var startTagFound = false
+    var parseFinished = false
+    val lineIter = sqlSyntaxDefs.toIterator
+    while (!parseFinished && lineIter.hasNext) {
+      val line = lineIter.next()
+      if (line.trim.startsWith(startTag)) {
+        startTagFound = true
+      } else if (line.trim.startsWith(endTag)) {
+        parseFinished = true
+      } else if (startTagFound) {
+        f.applyOrElse(line, default).foreach { symbol =>
+          keywords += symbol
+        }
+      }
+    }
+    assert(keywords.nonEmpty && startTagFound && parseFinished, "cannot extract keywords from " +
+      s"the `SqlBase.g4` file, so please check if the start/end tags (`$startTag` and `$endTag`) " +
+      "are placed correctly in the file.")
+    keywords.toSet
+  }
+
+  // If a symbol does not have the same string with its literal (e.g., `SETMINUS: 'MINUS';`),
+  // we need to map a symbol to actual literal strings.
+  val symbolsToExpandIntoDifferentLiterals = {
+    val kwDef = """([A-Z_]+):(.+);""".r
+    val keywords = parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      case kwDef(symbol, literalDef) =>
+        val splitDefs = literalDef.split("""\|""")
+        val hasMultipleLiterals = splitDefs.length > 1
+        // The case where a symbol has multiple literal definitions,
+        // e.g., `DATABASES: 'DATABASES' | 'SCHEMAS';`.
+        if (hasMultipleLiterals) {
+          // Filters out inappropriate entries, e.g., `!` in `NOT: 'NOT' | '!';`
+          val litDef = """([A-Z_]+)""".r
+          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq.flatMap {
+            case litDef(lit) => Some(lit)
+            case _ => None
+          }
+          (symbol, literals) :: Nil
+        } else {
+          val literal = literalDef.replaceAll("'", "").trim
+          // The case where a symbol string and its literal string are different,
+          // e.g., `SETMINUS: 'MINUS';`.
+          if (symbol != literal) {
+            (symbol, literal :: Nil) :: Nil
+          } else {
+            Nil
+          }
+        }
+    }
+    keywords.toMap
+  }
+
+  // All the SQL keywords defined in `SqlBase.g4`
+  val allCandidateKeywords: Set[String] = {
+    val kwDef = """([A-Z_]+):.+;""".r
+    parseAntlrGrammars(
+        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
+      // Parses a pattern, e.g., `AFTER: 'AFTER';`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val nonReservedKeywordsInAnsiMode: Set[String] = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--ANSI-NON-RESERVED-START", "//--ANSI-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
+
+  val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
+}
+
+class SQLKeywordSuite extends SparkFunSuite with SQLKeywordUtils {
+  test("all keywords are documented") {
+    val documentedKeywords = keywordsInDoc.map(_.head).toSet
+    if (allCandidateKeywords != documentedKeywords) {
+      val undocumented = (allCandidateKeywords -- documentedKeywords).toSeq.sorted
+      fail("Some keywords are not documented: " + undocumented.mkString(", "))
+    }
+  }
+
+  test("Spark keywords are documented correctly") {
+    val reservedKeywordsInDoc = keywordsInDoc.filter(_.apply(1) == "reserved").map(_.head).toSet
+    if (reservedKeywordsInAnsiMode != reservedKeywordsInDoc) {
+      val misImplemented = (reservedKeywordsInDoc -- reservedKeywordsInAnsiMode).toSeq.sorted
+      fail("Some keywords are documented as reserved but actually not: " +
+        misImplemented.mkString(", "))
+    }
+  }
+
+  test("SQL 2016 keywords are documented correctly") {
+    withTempDir { dir =>
+      val tmpFile = new File(dir, "tmp")
+      val is = Thread.currentThread().getContextClassLoader
+        .getResourceAsStream("ansi-sql-2016-reserved-keywords.txt")
+      Files.copy(is, tmpFile.toPath)
+      val reservedKeywordsInSql2016 = Files.readAllLines(tmpFile.toPath)
+        .asScala.filterNot(_.startsWith("--")).map(_.trim).toSet
+      val documented = keywordsInDoc.filter(_.last == "reserved").map(_.head).toSet
+      assert((documented -- reservedKeywordsInSql2016).isEmpty)
+    }
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
index a721e17aef02d..f037ce7b9e793 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala
@@ -16,19 +16,11 @@
  */
 package org.apache.spark.sql.catalyst.parser
 
-import java.io.File
-import java.nio.file.Files
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.sql.catalyst.util.fileToString
+import org.apache.spark.sql.catalyst.{SQLKeywordUtils, TableIdentifier}
 import org.apache.spark.sql.internal.SQLConf
 
-class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
+class TableIdentifierParserSuite extends SparkFunSuite with SQLKeywordUtils {
   import CatalystSqlParser._
 
   // Add "$elem$", "$value$" & "$key$"
@@ -292,121 +284,6 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper {
     "where",
     "with")
 
-  private val sqlSyntaxDefs = {
-    val sqlBasePath = {
-      java.nio.file.Paths.get(sparkHome, "sql", "catalyst", "src", "main", "antlr4", "org",
-        "apache", "spark", "sql", "catalyst", "parser", "SqlBase.g4").toFile
-    }
-    fileToString(sqlBasePath).split("\n")
-  }
-
-  private def parseAntlrGrammars[T](startTag: String, endTag: String)
-      (f: PartialFunction[String, Seq[T]]): Set[T] = {
-    val keywords = new mutable.ArrayBuffer[T]
-    val default = (_: String) => Nil
-    var startTagFound = false
-    var parseFinished = false
-    val lineIter = sqlSyntaxDefs.toIterator
-    while (!parseFinished && lineIter.hasNext) {
-      val line = lineIter.next()
-      if (line.trim.startsWith(startTag)) {
-        startTagFound = true
-      } else if (line.trim.startsWith(endTag)) {
-        parseFinished = true
-      } else if (startTagFound) {
-        f.applyOrElse(line, default).foreach { symbol =>
-          keywords += symbol
-        }
-      }
-    }
-    assert(keywords.nonEmpty && startTagFound && parseFinished, "cannot extract keywords from " +
-      s"the `SqlBase.g4` file, so please check if the start/end tags (`$startTag` and `$endTag`) " +
-      "are placed correctly in the file.")
-    keywords.toSet
-  }
-
-  // If a symbol does not have the same string with its literal (e.g., `SETMINUS: 'MINUS';`),
-  // we need to map a symbol to actual literal strings.
-  val symbolsToExpandIntoDifferentLiterals = {
-    val kwDef = """([A-Z_]+):(.+);""".r
-    val keywords = parseAntlrGrammars(
-        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
-      case kwDef(symbol, literalDef) =>
-        val splitDefs = literalDef.split("""\|""")
-        val hasMultipleLiterals = splitDefs.length > 1
-        // The case where a symbol has multiple literal definitions,
-        // e.g., `DATABASES: 'DATABASES' | 'SCHEMAS';`.
-        if (hasMultipleLiterals) {
-          // Filters out inappropriate entries, e.g., `!` in `NOT: 'NOT' | '!';`
-          val litDef = """([A-Z_]+)""".r
-          val literals = splitDefs.map(_.replaceAll("'", "").trim).toSeq.flatMap {
-            case litDef(lit) => Some(lit)
-            case _ => None
-          }
-          (symbol, literals) :: Nil
-        } else {
-          val literal = literalDef.replaceAll("'", "").trim
-          // The case where a symbol string and its literal string are different,
-          // e.g., `SETMINUS: 'MINUS';`.
-          if (symbol != literal) {
-            (symbol, literal :: Nil) :: Nil
-          } else {
-            Nil
-          }
-        }
-    }
-    keywords.toMap
-  }
-
-  // All the SQL keywords defined in `SqlBase.g4`
-  val allCandidateKeywords = {
-    val kwDef = """([A-Z_]+):.+;""".r
-    val keywords = parseAntlrGrammars(
-        "//--SPARK-KEYWORD-LIST-START", "//--SPARK-KEYWORD-LIST-END") {
-      // Parses a pattern, e.g., `AFTER: 'AFTER';`
-      case kwDef(symbol) =>
-        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
-          symbolsToExpandIntoDifferentLiterals(symbol)
-        } else {
-          symbol :: Nil
-        }
-    }
-    keywords
-  }
-
-  val nonReservedKeywordsInAnsiMode = {
-    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
-    parseAntlrGrammars("//--ANSI-NON-RESERVED-START", "//--ANSI-NON-RESERVED-END") {
-      // Parses a pattern, e.g., `    | AFTER`
-      case kwDef(symbol) =>
-        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
-          symbolsToExpandIntoDifferentLiterals(symbol)
-        } else {
-          symbol :: Nil
-        }
-    }
-  }
-
-  val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
-
-  test("check # of reserved keywords") {
-    val numReservedKeywords = 74
-    assert(reservedKeywordsInAnsiMode.size == numReservedKeywords,
-      s"The expected number of reserved keywords is $numReservedKeywords, but " +
-        s"${reservedKeywordsInAnsiMode.size} found.")
-  }
-
-  test("reserved keywords in Spark are also reserved in SQL 2016") {
-    withTempDir { dir =>
-      val tmpFile = new File(dir, "tmp")
-      val is = Thread.currentThread().getContextClassLoader
-        .getResourceAsStream("ansi-sql-2016-reserved-keywords.txt")
-      Files.copy(is, tmpFile.toPath)
-      val reservedKeywordsInSql2016 = Files.readAllLines(tmpFile.toPath)
-        .asScala.filterNot(_.startsWith("--")).map(_.trim).toSet
-      assert((reservedKeywordsInAnsiMode  -- reservedKeywordsInSql2016).isEmpty)
-    }
-  }
 
   test("table identifier") {
     // Regular names.

From 10568ad13a437edd4a2aca36358ff717b15e1104 Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Sat, 11 Jul 2020 06:24:27 -0700
Subject: [PATCH 0895/1280] [SPARK-32220][SQL][3.0] SHUFFLE_REPLICATE_NL Hint
 should not change Non-Cartesian Product join result

### What changes were proposed in this pull request?
In current Join Hint strategies, if we use SHUFFLE_REPLICATE_NL hint, it will directly convert join to Cartesian Product Join and loss join condition making result not correct.

For Example:
```
spark-sql> select * from test4 order by a asc;
1 2
Time taken: 1.063 seconds, Fetched 4 row(s)20/07/08 14:11:25 INFO SparkSQLCLIDriver: Time taken: 1.063 seconds, Fetched 4 row(s)
spark-sql>select * from test5 order by a asc
1 2
2 2
Time taken: 1.18 seconds, Fetched 24 row(s)20/07/08 14:13:59 INFO SparkSQLCLIDriver: Time taken: 1.18 seconds, Fetched 24 row(s)spar
spark-sql>select /*+ shuffle_replicate_nl(test4) */ * from test4 join test5 where test4.a = test5.a order by test4.a asc ;
1 2 1 2
1 2 2 2
Time taken: 0.351 seconds, Fetched 2 row(s)
20/07/08 14:18:16 INFO SparkSQLCLIDriver: Time taken: 0.351 seconds, Fetched 2 row(s)
```

### Why are the changes needed?
Fix wrong data result

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
Added UT

Closes #29070 from AngersZhuuuu/SPARK-32220-branch-3.0.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/SparkStrategies.scala |  4 +--
 .../org/apache/spark/sql/JoinHintSuite.scala  | 27 +++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index a983bc850057a..e041c54c61db7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -244,7 +244,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       //   4. Pick cartesian product if join type is inner like.
       //   5. Pick broadcast nested loop join as the final solution. It may OOM but we don't have
       //      other choice.
-      case ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, condition, left, right, hint) =>
+      case p @ ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, condition, left, right, hint) =>
         def createBroadcastHashJoin(buildLeft: Boolean, buildRight: Boolean) = {
           val wantToBuildLeft = canBuildLeft(joinType) && buildLeft
           val wantToBuildRight = canBuildRight(joinType) && buildRight
@@ -286,7 +286,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         def createCartesianProduct() = {
           if (joinType.isInstanceOf[InnerLike]) {
-            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), condition)))
+            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), p.condition)))
           } else {
             None
           }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
index f68c416941266..71f7a708ad681 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinHintSuite.scala
@@ -570,4 +570,31 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
       assert(joinHints == expectedHints)
     }
   }
+
+  test("SPARK-32220: Non Cartesian Product Join Result Correct with SHUFFLE_REPLICATE_NL hint") {
+    withTempView("t1", "t2") {
+      Seq((1, "4"), (2, "2")).toDF("key", "value").createTempView("t1")
+      Seq((1, "1"), (2, "12.3"), (2, "123")).toDF("key", "value").createTempView("t2")
+      val df1 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key = t2.key")
+      val df2 = sql("SELECT * from t1 join t2 ON t1.key = t2.key")
+      assert(df1.collect().size == df2.collect().size)
+
+      val df3 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2")
+      val df4 = sql("SELECT * from t1 join t2")
+      assert(df3.collect().size == df4.collect().size)
+
+      val df5 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key < t2.key")
+      val df6 = sql("SELECT * from t1 join t2 ON t1.key < t2.key")
+      assert(df5.collect().size == df6.collect().size)
+
+      val df7 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t1.key < 2")
+      val df8 = sql("SELECT * from t1 join t2 ON t1.key < 2")
+      assert(df7.collect().size == df8.collect().size)
+
+
+      val df9 = sql("SELECT /*+ shuffle_replicate_nl(t1) */ * from t1 join t2 ON t2.key < 2")
+      val df10 = sql("SELECT * from t1 join t2 ON t2.key < 2")
+      assert(df9.collect().size == df10.collect().size)
+    }
+  }
 }

From a0e5025fb8c88e53414a24bbe983c3848d71d173 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Sat, 11 Jul 2020 06:27:56 -0700
Subject: [PATCH 0896/1280] [SPARK-32238][SQL] Use Utils.getSimpleName to avoid
 hitting Malformed class name in ScalaUDF

This PR proposes to use `Utils.getSimpleName(function)` instead of `function.getClass.getSimpleName` to get the class name.

For some functions(see the demo below),  using `function.getClass.getSimpleName` can hit "Malformed class name" error.

Yes.  For the demo,

```scala
 object MalformedClassObject extends Serializable {
    class MalformedNonPrimitiveFunction extends (String => Int) with Serializable {
      override def apply(v1: String): Int = v1.toInt / 0
    }
  }
 OuterScopes.addOuterScope(MalformedClassObject)
 val f = new MalformedClassObject.MalformedNonPrimitiveFunction()
 Seq("20").toDF("col").select(udf(f).apply(Column("col"))).collect()
```

Before this PR, user can only see the error about "Malformed class name":

```scala
An exception or error caused a run to abort: Malformed class name
java.lang.InternalError: Malformed class name
	at java.lang.Class.getSimpleName(Class.java:1330)
	at org.apache.spark.sql.catalyst.expressions.ScalaUDF.udfErrorMessage$lzycompute(ScalaUDF.scala:1157)
	at org.apache.spark.sql.catalyst.expressions.ScalaUDF.udfErrorMessage(ScalaUDF.scala:1155)
	at org.apache.spark.sql.catalyst.expressions.ScalaUDF.doGenCode(ScalaUDF.scala:1077)
	at org.apache.spark.sql.catalyst.expressions.Expression.$anonfun$genCode$3(Expression.scala:147)
	at scala.Option.getOrElse(Option.scala:189)
	at org.apache.spark.sql.catalyst.expressions.Expression.genCode(Expression.scala:142)
	at org.apache.spark.sql.catalyst.expressions.Alias.genCode(namedExpressions.scala:160)
	at org.apache.spark.sql.execution.ProjectExec.$anonfun$doConsume$1(basicPhysicalOperators.scala:69)
        ...
```

After this PR, user can see the real root cause of the udf failure:

```scala
org.apache.spark.SparkException: Failed to execute user defined function(UDFSuite$MalformedClassObject$MalformedNonPrimitiveFunction: (string) => int)
	at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
	at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
	at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:753)
	at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:340)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)
	at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)
	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:127)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:464)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:467)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ArithmeticException: / by zero
	at org.apache.spark.sql.UDFSuite$MalformedClassObject$MalformedNonPrimitiveFunction.apply(UDFSuite.scala:677)
	at org.apache.spark.sql.UDFSuite$MalformedClassObject$MalformedNonPrimitiveFunction.apply(UDFSuite.scala:676)
	... 17 more

```

Added a test.

Closes #29050 from Ngone51/fix-malformed-udf.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 0c9196e5493628d343ef67bb9e83d0c95ff3943a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/catalyst/expressions/ScalaUDF.scala   |  5 ++--
 .../scala/org/apache/spark/sql/UDFSuite.scala | 28 +++++++++++++++++++
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index e80f03ea84756..58a9f68bff240 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -24,7 +24,8 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.catalyst.expressions.codegen._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType}
+import org.apache.spark.sql.types.{AbstractDataType, AnyDataType, DataType, UserDefinedType}
+import org.apache.spark.util.Utils
 
 /**
  * User-defined function.
@@ -1115,7 +1116,7 @@ case class ScalaUDF(
   private[this] val resultConverter = CatalystTypeConverters.createToCatalystConverter(dataType)
 
   lazy val udfErrorMessage = {
-    val funcCls = function.getClass.getSimpleName
+    val funcCls = Utils.getSimpleName(function.getClass)
     val inputTypes = children.map(_.dataType.catalogString).mkString(", ")
     val outputType = dataType.catalogString
     s"Failed to execute user defined function($funcCls: ($inputTypes) => $outputType)"
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index e2747d7db9f3f..91e9f1d9afb80 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -19,7 +19,9 @@ package org.apache.spark.sql
 
 import java.math.BigDecimal
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.api.java._
+import org.apache.spark.sql.catalyst.encoders.OuterScopes
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.{QueryExecution, SimpleMode}
 import org.apache.spark.sql.execution.columnar.InMemoryRelation
@@ -581,4 +583,30 @@ class UDFSuite extends QueryTest with SharedSparkSession {
       .toDF("col1", "col2")
     checkAnswer(df.select(myUdf(Column("col1"), Column("col2"))), Row(2020) :: Nil)
   }
+
+  object MalformedClassObject extends Serializable {
+    class MalformedNonPrimitiveFunction extends (String => Int) with Serializable {
+      override def apply(v1: String): Int = v1.toInt / 0
+    }
+
+    class MalformedPrimitiveFunction extends (Int => Int) with Serializable {
+      override def apply(v1: Int): Int = v1 / 0
+    }
+  }
+
+  test("SPARK-32238: Use Utils.getSimpleName to avoid hitting Malformed class name") {
+    OuterScopes.addOuterScope(MalformedClassObject)
+    val f1 = new MalformedClassObject.MalformedNonPrimitiveFunction()
+    val f2 = new MalformedClassObject.MalformedPrimitiveFunction()
+
+    val e1 = intercept[SparkException] {
+      Seq("20").toDF("col").select(udf(f1).apply(Column("col"))).collect()
+    }
+    assert(e1.getMessage.contains("UDFSuite$MalformedClassObject$MalformedNonPrimitiveFunction"))
+
+    val e2 = intercept[SparkException] {
+      Seq(20).toDF("col").select(udf(f2).apply(Column("col"))).collect()
+    }
+    assert(e2.getMessage.contains("UDFSuite$MalformedClassObject$MalformedPrimitiveFunction"))
+  }
 }

From 8a6580cedf509b5f62175e4ed7b2e0882dd89976 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Sat, 11 Jul 2020 10:37:26 -0500
Subject: [PATCH 0897/1280] [SPARK-32232][ML][PYSPARK] Make sure ML has the
 same default solver values between Scala and Python

# What changes were proposed in this pull request?
current problems:
```
        mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
        model = mlp.fit(df)
        path = tempfile.mkdtemp()
        model_path = path + "/mlp"
        model.save(model_path)
        model2 = MultilayerPerceptronClassificationModel.load(model_path)
        self.assertEqual(model2.getSolver(), "l-bfgs")    # this fails because model2.getSolver() returns 'auto'
        model2.transform(df)
        # this fails with Exception pyspark.sql.utils.IllegalArgumentException: MultilayerPerceptronClassifier_dec859ed24ec parameter solver given invalid value auto.
```
FMClassifier/Regression and GeneralizedLinearRegression have the same problems.

Here are the root cause of the problems:
1. In HasSolver, both Scala and Python default solver to 'auto'

2. On Scala side, mlp overrides the default of solver to 'l-bfgs', FMClassifier/Regression overrides the default of solver to 'adamW', and glr overrides the default of solver to 'irls'

3. On Scala side, mlp overrides the default of solver in MultilayerPerceptronClassificationParams, so both MultilayerPerceptronClassification and MultilayerPerceptronClassificationModel have 'l-bfgs' as default

4. On Python side, mlp overrides the default of solver in MultilayerPerceptronClassification, so it has default as 'l-bfgs', but MultilayerPerceptronClassificationModel doesn't override the default so it gets the default from HasSolver which is 'auto'. In theory, we don't care about the solver value or any other params values for MultilayerPerceptronClassificationModel, because we have the fitted model already. That's why on Python side, we never set default values for any of the XXXModel.

5. when calling getSolver on the loaded mlp model, it calls this line of code underneath:
```
    def _transfer_params_from_java(self):
        """
        Transforms the embedded params from the companion Java object.
        """
        ......
                # SPARK-14931: Only check set params back to avoid default params mismatch.
                if self._java_obj.isSet(java_param):
                    value = _java2py(sc, self._java_obj.getOrDefault(java_param))
                    self._set(**{param.name: value})
        ......
```
that's why model2.getSolver() returns 'auto'. The code doesn't get the default Scala value (in this case 'l-bfgs') to set to Python param, so it takes the default value (in this case 'auto') on Python side.

6. when calling model2.transform(df), it calls this underneath:
```
    def _transfer_params_to_java(self):
        """
        Transforms the embedded params to the companion Java object.
        """
        ......
            if self.hasDefault(param):
                pair = self._make_java_param_pair(param, self._defaultParamMap[param])
                pair_defaults.append(pair)
        ......

```
Again, it gets the Python default solver which is 'auto', and this caused the Exception

7. Currently, on Scala side, for some of the algorithms, we set default values in the XXXParam, so both estimator and transformer get the default value. However, for some of the algorithms, we only set default in estimators, and the XXXModel doesn't get the default value. On Python side, we never set defaults for the XXXModel. This causes the default value inconsistency.

8. My proposed solution: set default params in XXXParam for both Scala and Python, so both the estimator and transformer have the same default value for both Scala and Python. I currently only changed solver in this PR. If everyone is OK with the fix, I will change all the other params as well.

I hope my explanation makes sense to your folks :)

### Why are the changes needed?
Fix bug

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
existing and new tests

Closes #29060 from huaxingao/solver_parity.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 99b4b062555329d5da968ad5dbd9e2b22a193a55)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../spark/ml/regression/FMRegressor.scala     | 14 ++--
 .../GeneralizedLinearRegression.scala         |  9 +--
 python/pyspark/ml/classification.py           |  8 +--
 python/pyspark/ml/regression.py               | 16 +++--
 python/pyspark/ml/tests/test_persistence.py   | 65 ++++++++++++++++++-
 5 files changed, 84 insertions(+), 28 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
index b017a1a039ce0..ce9881750c558 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/FMRegressor.scala
@@ -113,6 +113,10 @@ private[ml] trait FactorizationMachinesParams extends PredictorParams
     "The solver algorithm for optimization. Supported options: " +
       s"${supportedSolvers.mkString(", ")}. (Default adamW)",
     ParamValidators.inArray[String](supportedSolvers))
+
+  setDefault(factorSize -> 8, fitIntercept -> true, fitLinear -> true, regParam -> 0.0,
+    miniBatchFraction -> 1.0, initStd -> 0.01, maxIter -> 100, stepSize -> 1.0, tol -> 1E-6,
+    solver -> AdamW)
 }
 
 private[ml] trait FactorizationMachines extends FactorizationMachinesParams {
@@ -309,7 +313,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFactorSize(value: Int): this.type = set(factorSize, value)
-  setDefault(factorSize -> 8)
 
   /**
    * Set whether to fit intercept term.
@@ -319,7 +322,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set whether to fit linear term.
@@ -329,7 +331,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitLinear(value: Boolean): this.type = set(fitLinear, value)
-  setDefault(fitLinear -> true)
 
   /**
    * Set the L2 regularization parameter.
@@ -339,7 +340,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the mini-batch fraction parameter.
@@ -349,7 +349,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMiniBatchFraction(value: Double): this.type = set(miniBatchFraction, value)
-  setDefault(miniBatchFraction -> 1.0)
 
   /**
    * Set the standard deviation of initial coefficients.
@@ -359,7 +358,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setInitStd(value: Double): this.type = set(initStd, value)
-  setDefault(initStd -> 0.01)
 
   /**
    * Set the maximum number of iterations.
@@ -369,7 +367,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the initial step size for the first step (like learning rate).
@@ -379,7 +376,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setStepSize(value: Double): this.type = set(stepSize, value)
-  setDefault(stepSize -> 1.0)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -389,7 +385,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Set the solver algorithm used for optimization.
@@ -400,7 +395,6 @@ class FMRegressor @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> AdamW)
 
   /**
    * Set the random seed for weight initialization.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
index fa41a98749f32..8fda44c756f58 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
@@ -180,6 +180,9 @@ private[regression] trait GeneralizedLinearRegressionBase extends PredictorParam
       s"${supportedSolvers.mkString(", ")}. (Default irls)",
     ParamValidators.inArray[String](supportedSolvers))
 
+  setDefault(family -> Gaussian.name, variancePower -> 0.0, maxIter -> 25, tol -> 1E-6,
+    regParam -> 0.0, solver -> IRLS)
+
   @Since("2.0.0")
   override def validateAndTransformSchema(
       schema: StructType,
@@ -256,7 +259,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setFamily(value: String): this.type = set(family, value)
-  setDefault(family -> Gaussian.name)
 
   /**
    * Sets the value of param [[variancePower]].
@@ -267,7 +269,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.2.0")
   def setVariancePower(value: Double): this.type = set(variancePower, value)
-  setDefault(variancePower -> 0.0)
 
   /**
    * Sets the value of param [[linkPower]].
@@ -304,7 +305,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 25)
 
   /**
    * Sets the convergence tolerance of iterations.
@@ -315,7 +315,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Sets the regularization parameter for L2 regularization.
@@ -331,7 +330,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Sets the value of param [[weightCol]].
@@ -363,7 +361,6 @@ class GeneralizedLinearRegression @Since("2.0.0") (@Since("2.0.0") override val
    */
   @Since("2.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> IRLS)
 
   /**
    * Sets the link prediction (linear predictor) column name.
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 424e16a7abf47..369761e8f6fa7 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -2172,6 +2172,10 @@ class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, H
     initialWeights = Param(Params._dummy(), "initialWeights", "The initial weights of the model.",
                            typeConverter=TypeConverters.toVector)
 
+    def __init__(self):
+        super(_MultilayerPerceptronParams, self).__init__()
+        self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
+
     @since("1.6.0")
     def getLayers(self):
         """
@@ -2275,7 +2279,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(MultilayerPerceptronClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.MultilayerPerceptronClassifier", self.uid)
-        self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2871,9 +2874,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(FMClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.FMClassifier", self.uid)
-        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
-                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
-                         tol=1e-6, solver="adamW")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index f367bb8faa825..9c3c1e6ca01ae 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -1847,6 +1847,11 @@ class _GeneralizedLinearRegressionParams(_JavaPredictorParams, HasFitIntercept,
                       "or empty, we treat all instance offsets as 0.0",
                       typeConverter=TypeConverters.toString)
 
+    def __init__(self):
+        super(_GeneralizedLinearRegressionParams, self).__init__()
+        self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
+                         variancePower=0.0, aggregationDepth=2)
+
     @since("2.0.0")
     def getFamily(self):
         """
@@ -1979,8 +1984,6 @@ def __init__(self, labelCol="label", featuresCol="features", predictionCol="pred
         super(GeneralizedLinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.GeneralizedLinearRegression", self.uid)
-        self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
-                         variancePower=0.0, aggregationDepth=2)
         kwargs = self._input_kwargs
 
         self.setParams(**kwargs)
@@ -2354,6 +2357,12 @@ class _FactorizationMachinesParams(_JavaPredictorParams, HasMaxIter, HasStepSize
     solver = Param(Params._dummy(), "solver", "The solver algorithm for optimization. Supported " +
                    "options: gd, adamW. (Default adamW)", typeConverter=TypeConverters.toString)
 
+    def __init__(self):
+        super(_FactorizationMachinesParams, self).__init__()
+        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
+                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
+                         tol=1e-6, solver="adamW")
+
     @since("3.0.0")
     def getFactorSize(self):
         """
@@ -2445,9 +2454,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(FMRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.FMRegressor", self.uid)
-        self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
-                         miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
-                         tol=1e-6, solver="adamW")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/tests/test_persistence.py b/python/pyspark/ml/tests/test_persistence.py
index d4edcc26e17ac..2f6d451851b4b 100644
--- a/python/pyspark/ml/tests/test_persistence.py
+++ b/python/pyspark/ml/tests/test_persistence.py
@@ -21,19 +21,78 @@
 import unittest
 
 from pyspark.ml import Transformer
-from pyspark.ml.classification import DecisionTreeClassifier, LogisticRegression, OneVsRest, \
-    OneVsRestModel
+from pyspark.ml.classification import DecisionTreeClassifier, FMClassifier, \
+    FMClassificationModel, LogisticRegression, MultilayerPerceptronClassifier, \
+    MultilayerPerceptronClassificationModel, OneVsRest, OneVsRestModel
 from pyspark.ml.clustering import KMeans
 from pyspark.ml.feature import Binarizer, HashingTF, PCA
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.param import Params
 from pyspark.ml.pipeline import Pipeline, PipelineModel
-from pyspark.ml.regression import DecisionTreeRegressor, LinearRegression
+from pyspark.ml.regression import DecisionTreeRegressor, GeneralizedLinearRegression, \
+    GeneralizedLinearRegressionModel, \
+    LinearRegression
 from pyspark.ml.util import DefaultParamsReadable, DefaultParamsWriter
 from pyspark.ml.wrapper import JavaParams
 from pyspark.testing.mlutils import MockUnaryTransformer, SparkSessionTestCase
 
 
+class TestDefaultSolver(SparkSessionTestCase):
+
+    def test_multilayer_load(self):
+        df = self.spark.createDataFrame([(0.0, Vectors.dense([0.0, 0.0])),
+                                         (1.0, Vectors.dense([0.0, 1.0])),
+                                         (1.0, Vectors.dense([1.0, 0.0])),
+                                         (0.0, Vectors.dense([1.0, 1.0]))],
+                                        ["label",  "features"])
+
+        mlp = MultilayerPerceptronClassifier(layers=[2, 2, 2], seed=123)
+        model = mlp.fit(df)
+        self.assertEqual(model.getSolver(), "l-bfgs")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/mlp"
+        model.save(model_path)
+        model2 = MultilayerPerceptronClassificationModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "l-bfgs")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(4), transformed2.take(4))
+
+    def test_fm_load(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(1.0)),
+                                         (0.0, Vectors.sparse(1, [], []))],
+                                        ["label",  "features"])
+        fm = FMClassifier(factorSize=2, maxIter=50, stepSize=2.0)
+        model = fm.fit(df)
+        self.assertEqual(model.getSolver(), "adamW")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/fm"
+        model.save(model_path)
+        model2 = FMClassificationModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "adamW")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(2), transformed2.take(2))
+
+    def test_glr_load(self):
+        df = self.spark.createDataFrame([(1.0, Vectors.dense(0.0, 0.0)),
+                                         (1.0, Vectors.dense(1.0, 2.0)),
+                                         (2.0, Vectors.dense(0.0, 0.0)),
+                                         (2.0, Vectors.dense(1.0, 1.0))],
+                                        ["label",  "features"])
+        glr = GeneralizedLinearRegression(family="gaussian", link="identity", linkPredictionCol="p")
+        model = glr.fit(df)
+        self.assertEqual(model.getSolver(), "irls")
+        transformed1 = model.transform(df)
+        path = tempfile.mkdtemp()
+        model_path = path + "/glr"
+        model.save(model_path)
+        model2 = GeneralizedLinearRegressionModel.load(model_path)
+        self.assertEqual(model2.getSolver(), "irls")
+        transformed2 = model2.transform(df)
+        self.assertEqual(transformed1.take(4), transformed2.take(4))
+
+
 class PersistenceTest(SparkSessionTestCase):
 
     def test_linear_regression(self):

From befb7eb76ee642b601724ec057a0b751095197e8 Mon Sep 17 00:00:00 2001
From: Chuliang Xiao <ChuliangX@gmail.com>
Date: Sun, 12 Jul 2020 09:01:41 -0700
Subject: [PATCH 0898/1280] [MINOR][DOCS] Fix typo in PySpark example in
 ml-datasource.md

### What changes were proposed in this pull request?

This PR changes `true` to `True` in the python code.

### Why are the changes needed?

The previous example is a syntax error.

### Does this PR introduce _any_ user-facing change?

Yes, but this is doc-only typo fix.

### How was this patch tested?

Manually run the example.

Closes #29073 from ChuliangXiao/patch-1.

Authored-by: Chuliang Xiao <ChuliangX@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c56c84af473547c9e9cab7ef6422f2b550084b59)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/ml-datasource.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ml-datasource.md b/docs/ml-datasource.md
index 0f2f5f482ec50..8e9c947b75f38 100644
--- a/docs/ml-datasource.md
+++ b/docs/ml-datasource.md
@@ -86,7 +86,7 @@ Will output:
 In PySpark we provide Spark SQL data source API for loading image data as a DataFrame.
 
 {% highlight python %}
->>> df = spark.read.format("image").option("dropInvalid", true).load("data/mllib/images/origin/kittens")
+>>> df = spark.read.format("image").option("dropInvalid", True).load("data/mllib/images/origin/kittens")
 >>> df.select("image.origin", "image.width", "image.height").show(truncate=False)
 +-----------------------------------------------------------------------+-----+------+
 |origin                                                                 |width|height|

From 4e33454add5528e1214ffa2a1f09500ede11f1ab Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Tue, 14 Jul 2020 00:56:47 -0700
Subject: [PATCH 0899/1280] [SPARK-32220][SQL][3.0][FOLLOW-UP]
 SHUFFLE_REPLICATE_NL Hint should not change Non-Cartesian Product join result

### What changes were proposed in this pull request?
follow comment https://github.com/apache/spark/pull/29035#discussion_r453468999
Explain for pr

### Why are the changes needed?
add comment

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Not need

Closes #29093 from AngersZhuuuu/SPARK-32220-3.0-FOLLOWUP.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/SparkStrategies.scala      | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index e041c54c61db7..28d1ccbf4db36 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -244,7 +244,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       //   4. Pick cartesian product if join type is inner like.
       //   5. Pick broadcast nested loop join as the final solution. It may OOM but we don't have
       //      other choice.
-      case p @ ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, condition, left, right, hint) =>
+      case j @ ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, nonEquiCond, left, right, hint) =>
         def createBroadcastHashJoin(buildLeft: Boolean, buildRight: Boolean) = {
           val wantToBuildLeft = canBuildLeft(joinType) && buildLeft
           val wantToBuildRight = canBuildRight(joinType) && buildRight
@@ -254,7 +254,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               rightKeys,
               joinType,
               buildSide,
-              condition,
+              nonEquiCond,
               planLater(left),
               planLater(right)))
           }
@@ -269,7 +269,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               rightKeys,
               joinType,
               buildSide,
-              condition,
+              nonEquiCond,
               planLater(left),
               planLater(right)))
           }
@@ -278,7 +278,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         def createSortMergeJoin() = {
           if (RowOrdering.isOrderable(leftKeys)) {
             Some(Seq(joins.SortMergeJoinExec(
-              leftKeys, rightKeys, joinType, condition, planLater(left), planLater(right))))
+              leftKeys, rightKeys, joinType, nonEquiCond, planLater(left), planLater(right))))
           } else {
             None
           }
@@ -286,7 +286,9 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         def createCartesianProduct() = {
           if (joinType.isInstanceOf[InnerLike]) {
-            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), p.condition)))
+            // `CartesianProductExec` can't implicitly evaluate equal join condition, here we should
+            // pass the original condition which includes both equal and non-equal conditions.
+            Some(Seq(joins.CartesianProductExec(planLater(left), planLater(right), j.condition)))
           } else {
             None
           }
@@ -311,7 +313,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
               // This join could be very slow or OOM
               val buildSide = getSmallerSide(left, right)
               Seq(joins.BroadcastNestedLoopJoinExec(
-                planLater(left), planLater(right), buildSide, joinType, condition))
+                planLater(left), planLater(right), buildSide, joinType, nonEquiCond))
             }
         }
 

From 785ec2ee6c2473f54b7ca6c01f446cc8bdf883fa Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 14 Jul 2020 12:19:01 -0700
Subject: [PATCH 0900/1280] [SPARK-32307][SQL] ScalaUDF's canonicalized
 expression should exclude inputEncoders

### What changes were proposed in this pull request?

Override `canonicalized` to empty the `inputEncoders` for the canonicalized `ScalaUDF`.

### Why are the changes needed?

The following fails on `branch-3.0` currently, not on Apache Spark 3.0.0 release.

```scala
spark.udf.register("key", udf((m: Map[String, String]) => m.keys.head.toInt))
Seq(Map("1" -> "one", "2" -> "two")).toDF("a").createOrReplaceTempView("t")
checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)

[info]   org.apache.spark.sql.AnalysisException: expression 't.`a`' is neither present in the group by, nor is it an aggregate function. Add to group by or wrap in first() (or first_value) if you don't care which value you get.;;
[info] Aggregate [UDF(a#6)], [UDF(a#6) AS k#8]
[info] +- SubqueryAlias t
[info]    +- Project [value#3 AS a#6]
[info]       +- LocalRelation [value#3]
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis(CheckAnalysis.scala:49)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.failAnalysis$(CheckAnalysis.scala:48)
[info]   at org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:130)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkValidAggregateExpression$1(CheckAnalysis.scala:257)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$10(CheckAnalysis.scala:259)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$10$adapted(CheckAnalysis.scala:259)
[info]   at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
[info]   at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
[info]   at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkValidAggregateExpression$1(CheckAnalysis.scala:259)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$10(CheckAnalysis.scala:259)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.$anonfun$checkAnalysis$10$adapted(CheckAnalysis.scala:259)
[info]   at scala.collection.immutable.List.foreach(List.scala:392)
[info]   at org.apache.spark.sql.catalyst.analysis.CheckAnalysis.checkValidAggregateExpression$1(CheckAnalysis.scala:259)
...
```

We use the rule`ResolveEncodersInUDF` to resolve `inputEncoders` and the original`ScalaUDF` instance will be updated to a new `ScalaUDF` instance with the resolved encoders at the end. Note, during encoder resolving, types like `map`, `array` will be resolved to new expression(e.g. `MapObjects`, `CatalystToExternalMap`).

However, `ExpressionEncoder` can't be canonicalized. Thus, the canonicalized `ScalaUDF`s become different even if their original  `ScalaUDF`s are the same. Finally, it fails the `checkValidAggregateExpression` when this `ScalaUDF` is used as a group expression.

### Does this PR introduce _any_ user-facing change?

Yes, users will not hit the exception after this fix.

### How was this patch tested?

Added tests.

Closes #29106 from Ngone51/spark-32307.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a47b69a88a271e423271709ee491e2de57c5581b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/expressions/ScalaUDF.scala    |  6 ++++++
 .../test/scala/org/apache/spark/sql/UDFSuite.scala   | 12 ++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 58a9f68bff240..2706e4d5581b6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -57,6 +57,12 @@ case class ScalaUDF(
 
   override def toString: String = s"${udfName.getOrElse("UDF")}(${children.mkString(", ")})"
 
+  override lazy val canonicalized: Expression = {
+    // SPARK-32307: `ExpressionEncoder` can't be canonicalized, and technically we don't
+    // need it to identify a `ScalaUDF`.
+    Canonicalize.execute(copy(children = children.map(_.canonicalized), inputEncoders = Nil))
+  }
+
   /**
    * The analyzer should be aware of Scala primitive types so as to make the
    * UDF return null if there is any null input value of these types. On the
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index 91e9f1d9afb80..2ab14d5973b1e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -609,4 +609,16 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     }
     assert(e2.getMessage.contains("UDFSuite$MalformedClassObject$MalformedPrimitiveFunction"))
   }
+
+  test("SPARK-32307: Aggression that use map type input UDF as group expression") {
+    spark.udf.register("key", udf((m: Map[String, String]) => m.keys.head.toInt))
+    Seq(Map("1" -> "one", "2" -> "two")).toDF("a").createOrReplaceTempView("t")
+    checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)
+  }
+
+  test("SPARK-32307: Aggression that use array type input UDF as group expression") {
+    spark.udf.register("key", udf((m: Array[Int]) => m.head))
+    Seq(Array(1)).toDF("a").createOrReplaceTempView("t")
+    checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)
+  }
 }

From b896b173eae6a2f323498debb32b0979441d0126 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 15 Jul 2020 08:44:48 +0900
Subject: [PATCH 0901/1280] [SPARK-32301][PYTHON][TESTS] Add a test case for
 toPandas to work with empty partitioned Spark DataFrame

### What changes were proposed in this pull request?

This PR proposes to port the test case from https://github.com/apache/spark/pull/29098 to branch-3.0 and master.  In the master and branch-3.0, this was fixed together at https://github.com/apache/spark/commit/ecaa495b1fe532c36e952ccac42f4715809476af but no partition case is not being tested.

### Why are the changes needed?

To improve test coverage.

### Does this PR introduce _any_ user-facing change?

No, test-only.

### How was this patch tested?

Unit test was forward-ported.

Closes #29099 from HyukjinKwon/SPARK-32300-1.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 676d92ecceb3d46baa524c725b9f9a14450f1e9d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/tests/test_arrow.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 1386f8d9948c3..7a41d24d6eecb 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -421,6 +421,13 @@ def test_createDataFrame_with_float_index(self):
             self.spark.createDataFrame(
                 pd.DataFrame({'a': [1, 2, 3]}, index=[2., 3., 4.])).distinct().count(), 3)
 
+    def test_no_partition_toPandas(self):
+        # SPARK-32301: toPandas should work from a Spark DataFrame with no partitions
+        # Forward-ported from SPARK-32300.
+        pdf = self.spark.sparkContext.emptyRDD().toDF("col1 int").toPandas()
+        self.assertEqual(len(pdf), 0)
+        self.assertEqual(list(pdf.columns), ["col1"])
+
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,

From 245b83aeab2db9aac7ed41c16d8b229120fa3723 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 15 Jul 2020 08:46:20 +0900
Subject: [PATCH 0902/1280] [MINOR][R] Match collectAsArrowToR with
 non-streaming collectAsArrowToPython

### What changes were proposed in this pull request?

This PR proposes to port forward #29098 to `collectAsArrowToR`. `collectAsArrowToR` follows `collectAsArrowToPython` in branch-2.4 due to the limitation of ARROW-4512. SparkR vectorization currently cannot use streaming format.

### Why are the changes needed?

For simplicity and consistency.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

The same code is being tested in `collectAsArrowToPython` of branch-2.4.

Closes #29100 from HyukjinKwon/minor-parts.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 03b5707b516187aaa8012049fce8b1cd0ac0fddd)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 2ddedf3600c5f..3b6fd2fcc177c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -3492,7 +3492,7 @@ class Dataset[T] private[sql](
         val numPartitions = arrowBatchRdd.partitions.length
 
         // Store collection results for worst case of 1 to N-1 partitions
-        val results = new Array[Array[Array[Byte]]](numPartitions - 1)
+        val results = new Array[Array[Array[Byte]]](Math.max(0, numPartitions - 1))
         var lastIndex = -1  // index of last partition written
 
         // Handler to eagerly write partitions to Python in order

From 74c910afb2101ac1335176a0824b508e9fd9e43f Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 15 Jul 2020 07:43:56 -0700
Subject: [PATCH 0903/1280] [SPARK-32318][SQL][TESTS] Add a test case to
 EliminateSortsSuite for ORDER BY in DISTRIBUTE BY

### What changes were proposed in this pull request?

This PR aims to add a test case to EliminateSortsSuite to protect a valid use case which is using ORDER BY in DISTRIBUTE BY statement.

### Why are the changes needed?

```scala
scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2, x))).toDF("a", "b").repartition(2).createOrReplaceTempView("t")

scala> sql("select * from (select * from t order by b) distribute by a").write.orc("/tmp/master")

$ ls -al /tmp/master/
total 56
drwxr-xr-x  10 dongjoon  wheel  320 Jul 14 22:12 ./
drwxrwxrwt  15 root      wheel  480 Jul 14 22:12 ../
-rw-r--r--   1 dongjoon  wheel    8 Jul 14 22:12 ._SUCCESS.crc
-rw-r--r--   1 dongjoon  wheel   12 Jul 14 22:12 .part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r--   1 dongjoon  wheel   16 Jul 14 22:12 .part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r--   1 dongjoon  wheel   16 Jul 14 22:12 .part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r--   1 dongjoon  wheel    0 Jul 14 22:12 _SUCCESS
-rw-r--r--   1 dongjoon  wheel  119 Jul 14 22:12 part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
-rw-r--r--   1 dongjoon  wheel  932 Jul 14 22:12 part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
-rw-r--r--   1 dongjoon  wheel  939 Jul 14 22:12 part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
```

The following was found during SPARK-32276. If Spark optimizer removes the inner `ORDER BY`, the file size increases.
```scala
scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2, x))).toDF("a", "b").repartition(2).createOrReplaceTempView("t")

scala> sql("select * from (select * from t order by b) distribute by a").write.orc("/tmp/SPARK-32276")

$ ls -al /tmp/SPARK-32276/
total 632
drwxr-xr-x  10 dongjoon  wheel     320 Jul 14 22:08 ./
drwxrwxrwt  14 root      wheel     448 Jul 14 22:08 ../
-rw-r--r--   1 dongjoon  wheel       8 Jul 14 22:08 ._SUCCESS.crc
-rw-r--r--   1 dongjoon  wheel      12 Jul 14 22:08 .part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r--   1 dongjoon  wheel    1188 Jul 14 22:08 .part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r--   1 dongjoon  wheel    1188 Jul 14 22:08 .part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r--   1 dongjoon  wheel       0 Jul 14 22:08 _SUCCESS
-rw-r--r--   1 dongjoon  wheel     119 Jul 14 22:08 part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
-rw-r--r--   1 dongjoon  wheel  150735 Jul 14 22:08 part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
-rw-r--r--   1 dongjoon  wheel  150741 Jul 14 22:08 part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
```

### Does this PR introduce _any_ user-facing change?

No. This only improves the test coverage.

### How was this patch tested?

Pass the GitHub Action or Jenkins.

Closes #29118 from dongjoon-hyun/SPARK-32318.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 8950dcbb1cafccc2ba8bbf030ab7ac86cfe203a4)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/catalyst/optimizer/EliminateSortsSuite.scala     | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
index d7eb048ba8705..e2b599a7c090c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
@@ -284,6 +284,15 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-32318: should not remove orderBy in distribute statement") {
+    val projectPlan = testRelation.select('a, 'b)
+    val orderByPlan = projectPlan.orderBy('b.desc)
+    val distributedPlan = orderByPlan.distribute('a)(1)
+    val optimized = Optimize.execute(distributedPlan.analyze)
+    val correctAnswer = distributedPlan.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("should not remove orderBy in left join clause if there is an outer limit") {
     val projectPlan = testRelation.select('a, 'b)
     val orderByPlan = projectPlan.orderBy('a.asc, 'b.desc)

From 4ef535fffbc1cacbacb035b2b1ac1dffcc0352b4 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 15 Jul 2020 17:43:23 -0700
Subject: [PATCH 0904/1280] Revert "[SPARK-32307][SQL] ScalaUDF's canonicalized
 expression should exclude inputEncoders"

This reverts commit 785ec2ee6c2473f54b7ca6c01f446cc8bdf883fa.
---
 .../spark/sql/catalyst/expressions/ScalaUDF.scala    |  6 ------
 .../test/scala/org/apache/spark/sql/UDFSuite.scala   | 12 ------------
 2 files changed, 18 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 2706e4d5581b6..58a9f68bff240 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -57,12 +57,6 @@ case class ScalaUDF(
 
   override def toString: String = s"${udfName.getOrElse("UDF")}(${children.mkString(", ")})"
 
-  override lazy val canonicalized: Expression = {
-    // SPARK-32307: `ExpressionEncoder` can't be canonicalized, and technically we don't
-    // need it to identify a `ScalaUDF`.
-    Canonicalize.execute(copy(children = children.map(_.canonicalized), inputEncoders = Nil))
-  }
-
   /**
    * The analyzer should be aware of Scala primitive types so as to make the
    * UDF return null if there is any null input value of these types. On the
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
index 2ab14d5973b1e..91e9f1d9afb80 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UDFSuite.scala
@@ -609,16 +609,4 @@ class UDFSuite extends QueryTest with SharedSparkSession {
     }
     assert(e2.getMessage.contains("UDFSuite$MalformedClassObject$MalformedPrimitiveFunction"))
   }
-
-  test("SPARK-32307: Aggression that use map type input UDF as group expression") {
-    spark.udf.register("key", udf((m: Map[String, String]) => m.keys.head.toInt))
-    Seq(Map("1" -> "one", "2" -> "two")).toDF("a").createOrReplaceTempView("t")
-    checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)
-  }
-
-  test("SPARK-32307: Aggression that use array type input UDF as group expression") {
-    spark.udf.register("key", udf((m: Array[Int]) => m.head))
-    Seq(Array(1)).toDF("a").createOrReplaceTempView("t")
-    checkAnswer(sql("SELECT key(a) AS k FROM t GROUP BY key(a)"), Row(1) :: Nil)
-  }
 }

From b745041f698120be21ab889706880e976a599fdb Mon Sep 17 00:00:00 2001
From: SaurabhChawla <saurabhc@qubole.com>
Date: Thu, 16 Jul 2020 13:11:47 +0000
Subject: [PATCH 0905/1280] [SPARK-32234][SQL] Spark sql commands are failing
 on selecting the orc tables

### What changes were proposed in this pull request?
Spark sql commands are failing on selecting the orc tables
Steps to reproduce
Example 1 -
Prerequisite -  This is the location(/Users/test/tpcds_scale5data/date_dim) for orc data which is generated by the hive.
```
val table = """CREATE TABLE `date_dim` (
  `d_date_sk` INT,
  `d_date_id` STRING,
  `d_date` TIMESTAMP,
  `d_month_seq` INT,
  `d_week_seq` INT,
  `d_quarter_seq` INT,
  `d_year` INT,
  `d_dow` INT,
  `d_moy` INT,
  `d_dom` INT,
  `d_qoy` INT,
  `d_fy_year` INT,
  `d_fy_quarter_seq` INT,
  `d_fy_week_seq` INT,
  `d_day_name` STRING,
  `d_quarter_name` STRING,
  `d_holiday` STRING,
  `d_weekend` STRING,
  `d_following_holiday` STRING,
  `d_first_dom` INT,
  `d_last_dom` INT,
  `d_same_day_ly` INT,
  `d_same_day_lq` INT,
  `d_current_day` STRING,
  `d_current_week` STRING,
  `d_current_month` STRING,
  `d_current_quarter` STRING,
  `d_current_year` STRING)
USING orc
LOCATION '/Users/test/tpcds_scale5data/date_dim'"""

spark.sql(table).collect

val u = """select date_dim.d_date_id from date_dim limit 5"""

spark.sql(u).collect
```
Example 2

```
  val table = """CREATE TABLE `test_orc_data` (
  `_col1` INT,
  `_col2` STRING,
  `_col3` INT)
  USING orc"""

spark.sql(table).collect

spark.sql("insert into test_orc_data values(13, '155', 2020)").collect

val df = """select _col2 from test_orc_data limit 5"""
spark.sql(df).collect

```

Its Failing with below error
```
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 2.0 failed 1 times, most recent failure: Lost task 0.0 in stage 2.0 (TID 2, 192.168.0.103, executor driver): java.lang.ArrayIndexOutOfBoundsException: 1
    at org.apache.spark.sql.execution.datasources.orc.OrcColumnarBatchReader.initBatch(OrcColumnarBatchReader.java:156)
    at org.apache.spark.sql.execution.datasources.orc.OrcFileFormat.$anonfun$buildReaderWithPartitionValues$7(OrcFileFormat.scala:258)
    at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.org$apache$spark$sql$execution$datasources$FileScanRDD$$anon$$readCurrentFile(FileScanRDD.scala:141)
    at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:203)
    at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:116)
    at org.apache.spark.sql.execution.FileSourceScanExec$$anon$1.hasNext(DataSourceScanExec.scala:620)
    at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.columnartorow_nextBatch_0$(Unknown Source)
    at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
    at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
    at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:729)
    at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:343)
    at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:895)
    at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:895)
    at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:372)
    at org.apache.spark.rdd.RDD.iterator(RDD.scala:336)
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
    at org.apache.spark.scheduler.Task.run(Task.scala:133)
    at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:445)
    at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1489)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:448)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)`
```

The reason behind this initBatch is not getting the schema that is needed to find out the column value in OrcFileFormat.scala
```
batchReader.initBatch(
 TypeDescription.fromString(resultSchemaString)
```

### Why are the changes needed?
Spark sql queries for orc tables are failing

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Unit test is added for this .Also Tested through spark shell and spark submit the failing queries

Closes #29045 from SaurabhChawla100/SPARK-32234.

Lead-authored-by: SaurabhChawla <saurabhc@qubole.com>
Co-authored-by: SaurabhChawla <s.saurabhtim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 6be8b935a4f7ce0dea2d7aaaf747c2e8e1a9f47a)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/orc/OrcFileFormat.scala       | 10 ++---
 .../execution/datasources/orc/OrcUtils.scala  | 41 +++++++++++++++----
 .../v2/orc/OrcPartitionReaderFactory.scala    | 21 +++++-----
 .../sql/hive/orc/HiveOrcQuerySuite.scala      | 28 +++++++++++++
 4 files changed, 78 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
index fd791ce7c5e19..4dff1ec7ebfb9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
@@ -164,8 +164,6 @@ class OrcFileFormat
     val enableVectorizedReader = supportBatch(sparkSession, resultSchema)
     val capacity = sqlConf.orcVectorizedReaderBatchSize
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(hadoopConf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(hadoopConf, sqlConf.caseSensitiveAnalysis)
 
     val broadcastedConf =
@@ -179,16 +177,18 @@ class OrcFileFormat
 
       val fs = filePath.getFileSystem(conf)
       val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-      val requestedColIdsOrEmptyFile =
+      val resultedColPruneInfo =
         Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
           OrcUtils.requestedColumnIds(
             isCaseSensitive, dataSchema, requiredSchema, reader, conf)
         }
 
-      if (requestedColIdsOrEmptyFile.isEmpty) {
+      if (resultedColPruneInfo.isEmpty) {
         Iterator.empty
       } else {
-        val requestedColIds = requestedColIdsOrEmptyFile.get
+        val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
+        val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+          dataSchema, resultSchema, partitionSchema, conf)
         assert(requestedColIds.length == requiredSchema.length,
           "[BUG] requested column IDs do not match required schema")
         val taskConf = new Configuration(conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index d274bcd0edd2c..e10253989788b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -24,7 +24,7 @@ import scala.collection.JavaConverters._
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
-import org.apache.orc.{OrcFile, Reader, TypeDescription, Writer}
+import org.apache.orc.{OrcConf, OrcFile, Reader, TypeDescription, Writer}
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
 import org.apache.spark.deploy.SparkHadoopUtil
@@ -116,15 +116,17 @@ object OrcUtils extends Logging {
   }
 
   /**
-   * Returns the requested column ids from the given ORC file. Column id can be -1, which means the
-   * requested column doesn't exist in the ORC file. Returns None if the given ORC file is empty.
+   * @return Returns the combination of requested column ids from the given ORC file and
+   *         boolean flag to find if the pruneCols is allowed or not. Requested Column id can be
+   *         -1, which means the requested column doesn't exist in the ORC file. Returns None
+   *         if the given ORC file is empty.
    */
   def requestedColumnIds(
       isCaseSensitive: Boolean,
       dataSchema: StructType,
       requiredSchema: StructType,
       reader: Reader,
-      conf: Configuration): Option[Array[Int]] = {
+      conf: Configuration): Option[(Array[Int], Boolean)] = {
     val orcFieldNames = reader.getSchema.getFieldNames.asScala
     if (orcFieldNames.isEmpty) {
       // SPARK-8501: Some old empty ORC files always have an empty schema stored in their footer.
@@ -136,6 +138,10 @@ object OrcUtils extends Logging {
         assert(orcFieldNames.length <= dataSchema.length, "The given data schema " +
           s"${dataSchema.catalogString} has less fields than the actual ORC physical schema, " +
           "no idea which columns were dropped, fail to read.")
+        // for ORC file written by Hive, no field names
+        // in the physical schema, there is a need to send the
+        // entire dataSchema instead of required schema.
+        // So pruneCols is not done in this case
         Some(requiredSchema.fieldNames.map { name =>
           val index = dataSchema.fieldIndex(name)
           if (index < orcFieldNames.length) {
@@ -143,7 +149,7 @@ object OrcUtils extends Logging {
           } else {
             -1
           }
-        })
+        }, false)
       } else {
         if (isCaseSensitive) {
           Some(requiredSchema.fieldNames.zipWithIndex.map { case (name, idx) =>
@@ -152,7 +158,7 @@ object OrcUtils extends Logging {
             } else {
               -1
             }
-          })
+          }, true)
         } else {
           // Do case-insensitive resolution only if in case-insensitive mode
           val caseInsensitiveOrcFieldMap = orcFieldNames.groupBy(_.toLowerCase(Locale.ROOT))
@@ -170,7 +176,7 @@ object OrcUtils extends Logging {
                   idx
                 }
               }.getOrElse(-1)
-          })
+          }, true)
         }
       }
     }
@@ -199,4 +205,25 @@ object OrcUtils extends Logging {
       s"map<${orcTypeDescriptionString(m.keyType)},${orcTypeDescriptionString(m.valueType)}>"
     case _ => dt.catalogString
   }
+
+  /**
+   * @return Returns the result schema string based on the canPruneCols flag.
+   *         resultSchemaString will be created using resultsSchema in case of
+   *         canPruneCols is true and for canPruneCols as false value
+   *         resultSchemaString will be created using the actual dataSchema.
+   */
+  def orcResultSchemaString(
+      canPruneCols: Boolean,
+      dataSchema: StructType,
+      resultSchema: StructType,
+      partitionSchema: StructType,
+      conf: Configuration): String = {
+    val resultSchemaString = if (canPruneCols) {
+      OrcUtils.orcTypeDescriptionString(resultSchema)
+    } else {
+      OrcUtils.orcTypeDescriptionString(StructType(dataSchema.fields ++ partitionSchema.fields))
+    }
+    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
+    resultSchemaString
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
index 03d58fdcb7f67..7f25f7bd135f0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
@@ -66,24 +66,24 @@ case class OrcPartitionReaderFactory(
   override def buildReader(file: PartitionedFile): PartitionReader[InternalRow] = {
     val conf = broadcastedConf.value.value
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(conf, isCaseSensitive)
 
     val filePath = new Path(new URI(file.filePath))
 
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-    val requestedColIdsOrEmptyFile =
+    val resultedColPruneInfo =
       Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
         OrcUtils.requestedColumnIds(
           isCaseSensitive, dataSchema, readDataSchema, reader, conf)
       }
 
-    if (requestedColIdsOrEmptyFile.isEmpty) {
+    if (resultedColPruneInfo.isEmpty) {
       new EmptyPartitionReader[InternalRow]
     } else {
-      val requestedColIds = requestedColIdsOrEmptyFile.get
+      val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
+      val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+        dataSchema, resultSchema, partitionSchema, conf)
       assert(requestedColIds.length == readDataSchema.length,
         "[BUG] requested column IDs do not match required schema")
 
@@ -112,24 +112,25 @@ case class OrcPartitionReaderFactory(
   override def buildColumnarReader(file: PartitionedFile): PartitionReader[ColumnarBatch] = {
     val conf = broadcastedConf.value.value
 
-    val resultSchemaString = OrcUtils.orcTypeDescriptionString(resultSchema)
-    OrcConf.MAPRED_INPUT_SCHEMA.setString(conf, resultSchemaString)
     OrcConf.IS_SCHEMA_EVOLUTION_CASE_SENSITIVE.setBoolean(conf, isCaseSensitive)
 
     val filePath = new Path(new URI(file.filePath))
 
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
-    val requestedColIdsOrEmptyFile =
+    val resultedColPruneInfo =
       Utils.tryWithResource(OrcFile.createReader(filePath, readerOptions)) { reader =>
         OrcUtils.requestedColumnIds(
           isCaseSensitive, dataSchema, readDataSchema, reader, conf)
       }
 
-    if (requestedColIdsOrEmptyFile.isEmpty) {
+    if (resultedColPruneInfo.isEmpty) {
       new EmptyPartitionReader
     } else {
-      val requestedColIds = requestedColIdsOrEmptyFile.get ++ Array.fill(partitionSchema.length)(-1)
+      val (requestedDataColIds, canPruneCols) = resultedColPruneInfo.get
+      val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
+        dataSchema, resultSchema, partitionSchema, conf)
+      val requestedColIds = requestedDataColIds ++ Array.fill(partitionSchema.length)(-1)
       assert(requestedColIds.length == resultSchema.length,
         "[BUG] requested column IDs do not match required schema")
       val taskConf = new Configuration(conf)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
index 990d9425fb7fc..12ee5bea7c2f9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcQuerySuite.scala
@@ -288,4 +288,32 @@ class HiveOrcQuerySuite extends OrcQueryTest with TestHiveSingleton {
       }
     }
   }
+
+  test("SPARK-32234 read ORC table with column names all starting with '_col'") {
+    Seq("native", "hive").foreach { orcImpl =>
+      Seq("false", "true").foreach { vectorized =>
+        withSQLConf(
+          SQLConf.ORC_IMPLEMENTATION.key -> orcImpl,
+          SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> vectorized) {
+          withTable("test_hive_orc_impl") {
+            spark.sql(
+              s"""
+                 | CREATE TABLE test_hive_orc_impl
+                 | (_col1 INT, _col2 STRING, _col3 INT)
+                 | STORED AS ORC
+               """.stripMargin)
+            spark.sql(
+              s"""
+                 | INSERT INTO
+                 | test_hive_orc_impl
+                 | VALUES(9, '12', 2020)
+               """.stripMargin)
+
+            val df = spark.sql("SELECT _col2 FROM test_hive_orc_impl")
+            checkAnswer(df, Row("12"))
+          }
+        }
+      }
+    }
+  }
 }

From 9b6bea5d5ab4bd3a69ca6eaf7811eb54f9562ee6 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 16 Jul 2020 11:09:27 -0700
Subject: [PATCH 0906/1280] [SPARK-32018][SQL][3.0] UnsafeRow.setDecimal should
 set null with overflowed value

partially backport https://github.com/apache/spark/pull/29026

Closes #29125 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/expressions/UnsafeRow.java      |  2 +-
 .../scala/org/apache/spark/sql/UnsafeRowSuite.scala    | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 034894bd86085..4dc5ce1de047b 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -288,7 +288,7 @@ public void setDecimal(int ordinal, Decimal value, int precision) {
       Platform.putLong(baseObject, baseOffset + cursor, 0L);
       Platform.putLong(baseObject, baseOffset + cursor + 8, 0L);
 
-      if (value == null) {
+      if (value == null || !value.changePrecision(precision, value.scale())) {
         setNullAt(ordinal);
         // keep the offset for future update
         Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32);
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
index a5f904c621e6e..9daa69ce9f155 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
@@ -178,4 +178,14 @@ class UnsafeRowSuite extends SparkFunSuite {
     // Makes sure hashCode on unsafe array won't crash
     unsafeRow.getArray(0).hashCode()
   }
+
+  test("SPARK-32018: setDecimal with overflowed value") {
+    val d1 = new Decimal().set(BigDecimal("10000000000000000000")).toPrecision(38, 18)
+    val row = InternalRow.apply(d1)
+    val unsafeRow = UnsafeProjection.create(Array[DataType](DecimalType(38, 18))).apply(row)
+    assert(unsafeRow.getDecimal(0, 38, 18) === d1)
+    val d2 = (d1 * Decimal(10)).toPrecision(39, 18)
+    unsafeRow.setDecimal(0, d2, 38)
+    assert(unsafeRow.getDecimal(0, 38, 18) === null)
+  }
 }

From 76496e70f155127519f28e2257de457bd4ee91b7 Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Fri, 17 Jul 2020 12:05:45 -0700
Subject: [PATCH 0907/1280] [SPARK-32353][TEST] Update docker/spark-test and
 clean up unused stuff

### What changes were proposed in this pull request?
This PR aims to update the docker/spark-test and clean up unused stuff.

### Why are the changes needed?
Since Spark 3.0.0, Java 11 is supported. We had better use the latest Java and OS.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?
Manually do the following as described in https://github.com/apache/spark/blob/master/external/docker/spark-test/README.md .

```
docker run -v $SPARK_HOME:/opt/spark spark-test-master
docker run -v $SPARK_HOME:/opt/spark spark-test-worker spark://<master_ip>:7077
```

Closes #29150 from williamhyun/docker.

Authored-by: William Hyun <williamhyun3@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7dc1d8917dd01b5d5808460a5eb6e846795ab4bd)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 external/docker/spark-test/base/Dockerfile    | 15 +++------------
 external/docker/spark-test/master/default_cmd |  3 ---
 external/docker/spark-test/worker/default_cmd |  3 ---
 3 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/external/docker/spark-test/base/Dockerfile b/external/docker/spark-test/base/Dockerfile
index 5bec5d3f16548..d4a30c4681cba 100644
--- a/external/docker/spark-test/base/Dockerfile
+++ b/external/docker/spark-test/base/Dockerfile
@@ -15,23 +15,14 @@
 # limitations under the License.
 #
 
-FROM ubuntu:xenial
+FROM ubuntu:20.04
 
 # Upgrade package index
-# install a few other useful packages plus Open Jdk 8
+# install a few other useful packages plus Open Java 11
 # Remove unneeded /var/lib/apt/lists/* after install to reduce the
 # docker image size (by ~30MB)
 RUN apt-get update && \
-    apt-get install -y less openjdk-8-jre-headless iproute2 vim-tiny sudo openssh-server && \
+    apt-get install -y less openjdk-11-jre-headless iproute2 vim-tiny sudo openssh-server && \
     rm -rf /var/lib/apt/lists/*
 
-ENV SCALA_VERSION 2.12.10
-ENV CDH_VERSION cdh4
-ENV SCALA_HOME /opt/scala-$SCALA_VERSION
 ENV SPARK_HOME /opt/spark
-ENV PATH $SPARK_HOME:$SCALA_HOME/bin:$PATH
-
-# Install Scala
-ADD https://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.tgz /
-RUN (cd / && gunzip < scala-$SCALA_VERSION.tgz)|(cd /opt && tar -xvf -)
-RUN rm /scala-$SCALA_VERSION.tgz
diff --git a/external/docker/spark-test/master/default_cmd b/external/docker/spark-test/master/default_cmd
index 5a7da3446f6d2..96a36cd0bb682 100755
--- a/external/docker/spark-test/master/default_cmd
+++ b/external/docker/spark-test/master/default_cmd
@@ -22,7 +22,4 @@ echo "CONTAINER_IP=$IP"
 export SPARK_LOCAL_IP=$IP
 export SPARK_PUBLIC_DNS=$IP
 
-# Avoid the default Docker behavior of mapping our IP address to an unreachable host name
-umount /etc/hosts
-
 /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master -i $IP
diff --git a/external/docker/spark-test/worker/default_cmd b/external/docker/spark-test/worker/default_cmd
index 31b06cb0eb047..2401f5565aa0b 100755
--- a/external/docker/spark-test/worker/default_cmd
+++ b/external/docker/spark-test/worker/default_cmd
@@ -22,7 +22,4 @@ echo "CONTAINER_IP=$IP"
 export SPARK_LOCAL_IP=$IP
 export SPARK_PUBLIC_DNS=$IP
 
-# Avoid the default Docker behavior of mapping our IP address to an unreachable host name
-umount /etc/hosts
-
 /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker $1

From d669dea8214e546f05f26492cfb1d1b16d0e16fe Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sun, 19 Jul 2020 11:11:42 +0900
Subject: [PATCH 0908/1280] [SPARK-32344][SQL] Unevaluable expr is set to
 FIRST/LAST ignoreNullsExpr in distinct aggregates

### What changes were proposed in this pull request?

This PR intends to fix a bug of distinct FIRST/LAST aggregates in v2.4.6/v3.0.0/master;
```
scala> sql("SELECT FIRST(DISTINCT v) FROM VALUES 1, 2, 3 t(v)").show()
...
Caused by: java.lang.UnsupportedOperationException: Cannot evaluate expression: false#37
  at org.apache.spark.sql.catalyst.expressions.Unevaluable$class.eval(Expression.scala:258)
  at org.apache.spark.sql.catalyst.expressions.AttributeReference.eval(namedExpressions.scala:226)
  at org.apache.spark.sql.catalyst.expressions.aggregate.First.ignoreNulls(First.scala:68)
  at org.apache.spark.sql.catalyst.expressions.aggregate.First.updateExpressions$lzycompute(First.scala:82)
  at org.apache.spark.sql.catalyst.expressions.aggregate.First.updateExpressions(First.scala:81)
  at org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$15.apply(HashAggregateExec.scala:268)
```
A root cause of this bug is that the `Aggregation` strategy replaces a foldable boolean `ignoreNullsExpr` expr with a `Unevaluable` expr (`AttributeReference`) for distinct FIRST/LAST aggregate functions. But, this operation cannot be allowed because the `Analyzer` has checked that it must be foldabe;
https://github.com/apache/spark/blob/ffdbbae1d465fe2c710d020de62ca1a6b0b924d9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala#L74-L76
So, this PR proposes to change a vriable for `IGNORE NULLS`  from `Expression` to `Boolean` to avoid the case.

### Why are the changes needed?

Bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added a test in `DataFrameAggregateSuite`.

Closes #29143 from maropu/SPARK-32344.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c7a68a920df433f682b3716f761ff0a2208e18f7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  4 +-
 .../expressions/aggregate/First.scala         | 28 +++++++++-----
 .../catalyst/expressions/aggregate/Last.scala | 19 +++++-----
 .../optimizer/RewriteDistinctAggregates.scala |  2 +-
 .../sql/catalyst/parser/AstBuilder.scala      |  4 +-
 ...stSuite.scala => FirstLastTestSuite.scala} | 18 +++++++--
 .../parser/ExpressionParserSuite.scala        |  8 ++--
 .../org/apache/spark/sql/functions.scala      |  4 +-
 .../sql-functions/sql-expression-schema.md    |  8 ++--
 .../results/postgreSQL/window_part1.sql.out   | 12 +++---
 .../results/postgreSQL/window_part2.sql.out   | 38 +++++++++----------
 .../spark/sql/DataFrameAggregateSuite.scala   |  7 ++++
 12 files changed, 89 insertions(+), 63 deletions(-)
 rename sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/{LastTestSuite.scala => FirstLastTestSuite.scala} (84%)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 243b555118707..68fe5805fcc43 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -780,9 +780,9 @@ class Analyzer(
                 // AggregateFunction's with the exception of First and Last in their default mode
                 // (which we handle) and possibly some Hive UDAF's.
                 case First(expr, _) =>
-                  First(ifExpr(expr), Literal(true))
+                  First(ifExpr(expr), true)
                 case Last(expr, _) =>
-                  Last(ifExpr(expr), Literal(true))
+                  Last(ifExpr(expr), true)
                 case a: AggregateFunction =>
                   a.withNewChildren(a.children.map(ifExpr))
               }.transform {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
index df806edbfda05..65fd43c924d08 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala
@@ -17,8 +17,9 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.types._
@@ -49,12 +50,16 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "2.0.0")
-case class First(child: Expression, ignoreNullsExpr: Expression)
+case class First(child: Expression, ignoreNulls: Boolean)
   extends DeclarativeAggregate with ExpectsInputTypes {
 
-  def this(child: Expression) = this(child, Literal.create(false, BooleanType))
+  def this(child: Expression) = this(child, false)
 
-  override def children: Seq[Expression] = child :: ignoreNullsExpr :: Nil
+  def this(child: Expression, ignoreNullsExpr: Expression) = {
+    this(child, FirstLast.validateIgnoreNullExpr(ignoreNullsExpr, "first"))
+  }
+
+  override def children: Seq[Expression] = child :: Nil
 
   override def nullable: Boolean = true
 
@@ -71,16 +76,11 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
     val defaultCheck = super.checkInputDataTypes()
     if (defaultCheck.isFailure) {
       defaultCheck
-    } else if (!ignoreNullsExpr.foldable) {
-      TypeCheckFailure(
-        s"The second argument of First must be a boolean literal, but got: ${ignoreNullsExpr.sql}")
     } else {
       TypeCheckSuccess
     }
   }
 
-  private def ignoreNulls: Boolean = ignoreNullsExpr.eval().asInstanceOf[Boolean]
-
   private lazy val first = AttributeReference("first", child.dataType)()
 
   private lazy val valueSet = AttributeReference("valueSet", BooleanType)()
@@ -120,3 +120,11 @@ case class First(child: Expression, ignoreNullsExpr: Expression)
 
   override def toString: String = s"$prettyName($child)${if (ignoreNulls) " ignore nulls"}"
 }
+
+object FirstLast {
+  def validateIgnoreNullExpr(exp: Expression, funcName: String): Boolean = exp match {
+    case Literal(b: Boolean, BooleanType) => b
+    case _ => throw new AnalysisException(
+      s"The second argument in $funcName should be a boolean literal.")
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
index e55bced192f34..8d17a48a69f6f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala
@@ -17,8 +17,8 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
-import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckSuccess
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.types._
@@ -49,12 +49,16 @@ import org.apache.spark.sql.types._
   """,
   group = "agg_funcs",
   since = "2.0.0")
-case class Last(child: Expression, ignoreNullsExpr: Expression)
+case class Last(child: Expression, ignoreNulls: Boolean)
   extends DeclarativeAggregate with ExpectsInputTypes {
 
-  def this(child: Expression) = this(child, Literal.create(false, BooleanType))
+  def this(child: Expression) = this(child, false)
 
-  override def children: Seq[Expression] = child :: ignoreNullsExpr :: Nil
+  def this(child: Expression, ignoreNullsExpr: Expression) = {
+    this(child, FirstLast.validateIgnoreNullExpr(ignoreNullsExpr, "last"))
+  }
+
+  override def children: Seq[Expression] = child :: Nil
 
   override def nullable: Boolean = true
 
@@ -71,16 +75,11 @@ case class Last(child: Expression, ignoreNullsExpr: Expression)
     val defaultCheck = super.checkInputDataTypes()
     if (defaultCheck.isFailure) {
       defaultCheck
-    } else if (!ignoreNullsExpr.foldable) {
-      TypeCheckFailure(
-        s"The second argument of Last must be a boolean literal, but got: ${ignoreNullsExpr.sql}")
     } else {
       TypeCheckSuccess
     }
   }
 
-  private def ignoreNulls: Boolean = ignoreNullsExpr.eval().asInstanceOf[Boolean]
-
   private lazy val last = AttributeReference("last", child.dataType)()
 
   private lazy val valueSet = AttributeReference("valueSet", BooleanType)()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
index e5571069a7c41..15aa02ff677de 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDistinctAggregates.scala
@@ -257,7 +257,7 @@ object RewriteDistinctAggregates extends Rule[LogicalPlan] {
 
         // Select the result of the first aggregate in the last aggregate.
         val result = AggregateExpression(
-          aggregate.First(evalWithinGroup(regularGroupId, operator.toAttribute), Literal(true)),
+          aggregate.First(evalWithinGroup(regularGroupId, operator.toAttribute), true),
           mode = Complete,
           isDistinct = false)
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 5f1bc329ccd06..b29fe211454f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -1527,7 +1527,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitFirst(ctx: FirstContext): Expression = withOrigin(ctx) {
     val ignoreNullsExpr = ctx.IGNORE != null
-    First(expression(ctx.expression), Literal(ignoreNullsExpr)).toAggregateExpression()
+    First(expression(ctx.expression), ignoreNullsExpr).toAggregateExpression()
   }
 
   /**
@@ -1535,7 +1535,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitLast(ctx: LastContext): Expression = withOrigin(ctx) {
     val ignoreNullsExpr = ctx.IGNORE != null
-    Last(expression(ctx.expression), Literal(ignoreNullsExpr)).toAggregateExpression()
+    Last(expression(ctx.expression), ignoreNullsExpr).toAggregateExpression()
   }
 
   /**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
similarity index 84%
rename from sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala
rename to sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
index ba36bc074e154..bb6672e1046da 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/LastTestSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/FirstLastTestSuite.scala
@@ -17,14 +17,15 @@
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Literal}
 import org.apache.spark.sql.types.IntegerType
 
-class LastTestSuite extends SparkFunSuite {
+class FirstLastTestSuite extends SparkFunSuite {
   val input = AttributeReference("input", IntegerType, nullable = true)()
-  val evaluator = DeclarativeAggregateEvaluator(Last(input, Literal(false)), Seq(input))
-  val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, Literal(true)), Seq(input))
+  val evaluator = DeclarativeAggregateEvaluator(Last(input, false), Seq(input))
+  val evaluatorIgnoreNulls = DeclarativeAggregateEvaluator(Last(input, true), Seq(input))
 
   test("empty buffer") {
     assert(evaluator.initialize() === InternalRow(null, false))
@@ -106,4 +107,15 @@ class LastTestSuite extends SparkFunSuite {
     val m1 = evaluatorIgnoreNulls.merge(p1, p2)
     assert(evaluatorIgnoreNulls.eval(m1) === InternalRow(1))
   }
+
+  test("SPARK-32344: correct error handling for a type mismatch") {
+    val msg1 = intercept[AnalysisException] {
+      new First(input, Literal(1, IntegerType))
+    }.getMessage
+    assert(msg1.contains("The second argument in first should be a boolean literal"))
+    val msg2 = intercept[AnalysisException] {
+      new Last(input, Literal(1, IntegerType))
+    }.getMessage
+    assert(msg2.contains("The second argument in last should be a boolean literal"))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index c74fab1b27c65..c3430a4a0ecae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -770,10 +770,10 @@ class ExpressionParserSuite extends AnalysisTest {
   }
 
   test("SPARK-19526 Support ignore nulls keywords for first and last") {
-    assertEqual("first(a ignore nulls)", First('a, Literal(true)).toAggregateExpression())
-    assertEqual("first(a)", First('a, Literal(false)).toAggregateExpression())
-    assertEqual("last(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression())
-    assertEqual("last(a)", Last('a, Literal(false)).toAggregateExpression())
+    assertEqual("first(a ignore nulls)", First('a, true).toAggregateExpression())
+    assertEqual("first(a)", First('a, false).toAggregateExpression())
+    assertEqual("last(a ignore nulls)", Last('a, true).toAggregateExpression())
+    assertEqual("last(a)", Last('a, false).toAggregateExpression())
   }
 
   test("timestamp literals") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index 57842ed06e0b3..e5b0c1e1a3a16 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -461,7 +461,7 @@ object functions {
    * @since 2.0.0
    */
   def first(e: Column, ignoreNulls: Boolean): Column = withAggregateFunction {
-    new First(e.expr, Literal(ignoreNulls))
+    First(e.expr, ignoreNulls)
   }
 
   /**
@@ -586,7 +586,7 @@ object functions {
    * @since 2.0.0
    */
   def last(e: Column, ignoreNulls: Boolean): Column = withAggregateFunction {
-    new Last(e.expr, Literal(ignoreNulls))
+    new Last(e.expr, ignoreNulls)
   }
 
   /**
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index f8b1190fdf5d0..070a6f3851545 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -305,12 +305,12 @@
 | org.apache.spark.sql.catalyst.expressions.aggregate.CountMinSketchAgg | count_min_sketch | N/A | N/A |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CovPopulation | covar_pop | SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_pop(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.CovSample | covar_samp | SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2) | struct<covar_samp(CAST(c1 AS DOUBLE), CAST(c2 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.First | first_value | SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first_value(col, false):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.First | first | SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first_value | SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first_value(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.First | first | SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<first(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus | approx_count_distinct | SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1) | struct<approx_count_distinct(col1):bigint> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Kurtosis | kurtosis | SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<kurtosis(CAST(col AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col, false):int> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col, false):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col):int> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Max | max | SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col) | struct<max(col):int> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<max_by(x, y):string> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Min | min | SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col) | struct<min(col):int> |
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
index 2b1de87a6be5e..76567b689445a 100755
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part1.sql.out
@@ -270,7 +270,7 @@ struct<lead((ten * 2), 1, -1) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIR
 -- !query
 SELECT first(ten) OVER (PARTITION BY four ORDER BY ten), ten, four FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<first(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
+struct<first(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
 -- !query output
 0	0	0
 0	0	0
@@ -287,7 +287,7 @@ struct<first(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RA
 -- !query
 SELECT last(four) OVER (ORDER BY ten), ten, four FROM tenk1 WHERE unique2 < 10
 -- !query schema
-struct<last(four, false) OVER (ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
+struct<last(four) OVER (ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int,ten:int,four:int>
 -- !query output
 0	4	0
 1	1	1
@@ -306,7 +306,7 @@ SELECT last(ten) OVER (PARTITION BY four), ten, four FROM
 (SELECT * FROM tenk1 WHERE unique2 < 10 ORDER BY four, ten)s
 ORDER BY four, ten
 -- !query schema
-struct<last(ten, false) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int,ten:int,four:int>
+struct<last(ten) OVER (PARTITION BY four ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int,ten:int,four:int>
 -- !query output
 4	0	0
 4	0	0
@@ -476,7 +476,7 @@ sum(ten) over (partition by four order by ten),
 last(ten) over (partition by four order by ten)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(CAST(ten AS BIGINT)) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
+struct<four:int,ten:int,sum(CAST(ten AS BIGINT)) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
 -- !query output
 0	0	0	0
 0	2	2	2
@@ -506,7 +506,7 @@ sum(ten) over (partition by four order by ten range between unbounded preceding
 last(ten) over (partition by four order by ten range between unbounded preceding and current row)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW):int>
 -- !query output
 0	0	0	0
 0	2	2	2
@@ -536,7 +536,7 @@ sum(ten) over (partition by four order by ten range between unbounded preceding
 last(ten) over (partition by four order by ten range between unbounded preceding and unbounded following)
 FROM (select distinct ten, four from tenk1) ss
 -- !query schema
-struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,last(ten, false) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int>
+struct<four:int,ten:int,sum(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):bigint,last(ten) OVER (PARTITION BY four ORDER BY ten ASC NULLS FIRST RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):int>
 -- !query output
 0	0	20	8
 0	2	20	8
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
index f41659a196ae1..ccddf9db172a6 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out
@@ -101,7 +101,7 @@ from
 window w as
   (order by ss.id asc nulls first range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	1	3
 2	2	1	4
@@ -123,7 +123,7 @@ from
 window w as
   (order by ss.id asc nulls last range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id ASC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	1	3
 2	2	1	4
@@ -145,7 +145,7 @@ from
 window w as
   (order by ss.id desc nulls first range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id DESC NULLS FIRST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	3	1
 2	2	4	1
@@ -167,7 +167,7 @@ from
 window w as
   (order by ss.id desc nulls last range between 2 preceding and 2 following)
 -- !query schema
-struct<id:bigint,y:bigint,first(y, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,y:bigint,first(y) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint,last(y) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CAST((- 2) AS BIGINT) FOLLOWING AND CAST(2 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 1	1	3	1
 2	2	4	1
@@ -182,7 +182,7 @@ NULL	43	42	43
 select x.id, last(x.id) over (order by x.id range between current row and 2147450884 following)
 from range(32764, 32767) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2147450884 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(2147450884 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 32764	32766
 32765	32766
@@ -193,7 +193,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id desc range between current row and 2147450885 following)
 from range(-32766, -32765) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(2147450885 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(2147450885 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 -32766	-32766
 
@@ -202,7 +202,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id range between current row and 4 following)
 from range(2147483644, 2147483647) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(4 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN CURRENT ROW AND CAST(4 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 2147483644	2147483646
 2147483645	2147483646
@@ -213,7 +213,7 @@ struct<id:bigint,last(id, false) OVER (ORDER BY id ASC NULLS FIRST RANGE BETWEEN
 select x.id, last(x.id) over (order by x.id desc range between current row and 5 following)
 from range(-2147483646, -2147483645) x
 -- !query schema
-struct<id:bigint,last(id, false) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(5 AS BIGINT) FOLLOWING):bigint>
+struct<id:bigint,last(id) OVER (ORDER BY id DESC NULLS LAST RANGE BETWEEN CURRENT ROW AND CAST(5 AS BIGINT) FOLLOWING):bigint>
 -- !query output
 -2147483646	-2147483646
 
@@ -272,7 +272,7 @@ from numerics
 window w as (order by f_float4 range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -289,7 +289,7 @@ from numerics
 window w as (order by f_float4 range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -306,7 +306,7 @@ from numerics
 window w as (order by f_float4 range between
              'inf' preceding and 'inf' following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	1	7
@@ -323,7 +323,7 @@ from numerics
 window w as (order by f_float4 range between
              1.1 preceding and 'NaN' following)
 -- !query schema
-struct<id:int,f_float4:float,first(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float4:float,first(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float4 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	2	7
@@ -340,7 +340,7 @@ from numerics
 window w as (order by f_float8 range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -357,7 +357,7 @@ from numerics
 window w as (order by f_float8 range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1) AS FLOAT) FOLLOWING AND CAST(1.1 AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	1
 2	-1.0	2	3
@@ -374,7 +374,7 @@ from numerics
 window w as (order by f_float8 range between
              'inf' preceding and 'inf' following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- CAST(inf AS DOUBLE)) AS FLOAT) FOLLOWING AND CAST(inf AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	1	7
@@ -391,7 +391,7 @@ from numerics
 window w as (order by f_float8 range between
              1.1 preceding and 'NaN' following)
 -- !query schema
-struct<id:int,f_float8:float,first(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
+struct<id:int,f_float8:float,first(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int,last(id) OVER (ORDER BY f_float8 ASC NULLS FIRST RANGE BETWEEN CAST((- 1.1) AS FLOAT) FOLLOWING AND CAST(NaN AS FLOAT) FOLLOWING):int>
 -- !query output
 1	-3.0	1	7
 2	-1.0	2	7
@@ -408,7 +408,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND 1 FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
@@ -425,7 +425,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
@@ -442,7 +442,7 @@ from numerics
 window w as (order by f_numeric range between
              1 preceding and 1.1 following)
 -- !query schema
-struct<id:int,f_numeric:int,first(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id, false) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
+struct<id:int,f_numeric:int,first(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int,last(id) OVER (ORDER BY f_numeric ASC NULLS FIRST RANGE BETWEEN 1 PRECEDING AND CAST(1.1 AS INT) FOLLOWING):int>
 -- !query output
 1	-3	1	1
 2	-1	2	3
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 09f30bb5e2c77..54327b38c100b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -1037,6 +1037,13 @@ class DataFrameAggregateSuite extends QueryTest
     val groupBy = df.groupBy("b").agg(count("*"))
     checkAnswer(groupBy, Row(null, 1) :: Row(Row(null), 1) :: Row(Row(1.0), 1) :: Nil)
   }
+
+  test("SPARK-32344: Unevaluable's set to FIRST/LAST ignoreNullsExpr in distinct aggregates") {
+    val queryTemplate = (agg: String) =>
+      s"SELECT $agg(DISTINCT v) FROM (SELECT v FROM VALUES 1, 2, 3 t(v) ORDER BY v)"
+    checkAnswer(sql(queryTemplate("FIRST")), Row(1))
+    checkAnswer(sql(queryTemplate("LAST")), Row(3))
+  }
 }
 
 case class B(c: Option[Double])

From 8adb244bc26340ad38f79b96c74eb20bf6a74b9c Mon Sep 17 00:00:00 2001
From: Igor Dvorzhak <idv@google.com>
Date: Sun, 19 Jul 2020 12:19:36 -0700
Subject: [PATCH 0909/1280] [MINOR][DOCS] Fix links to Cloud Storage connectors
 docs

Closes #29155 from medb/patch-1.

Authored-by: Igor Dvorzhak <idv@google.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 32a0451376ab775fdd4ac364388e46179d9ee550)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/cloud-integration.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/cloud-integration.md b/docs/cloud-integration.md
index 749d026528017..1eaa8ab4eeee0 100644
--- a/docs/cloud-integration.md
+++ b/docs/cloud-integration.md
@@ -163,7 +163,7 @@ different stores and connectors when renaming directories:
 | Amazon S3     | s3a       | Unsafe                  | O(data) |
 | Azure Storage | wasb      | Safe                    | O(files) |
 | Azure Datalake Gen 2 | abfs | Safe                  | O(1) |
-| Google GCS    | gs        | Safe                    | O(1) |
+| Google Cloud Storage | gs        | Safe                    | O(1) |
 
 As storing temporary files can run up charges; delete
 directories called `"_temporary"` on a regular basis.
@@ -253,9 +253,9 @@ Here is the documentation on the standard connectors both from Apache and the cl
 * [Azure Blob Storage and Azure Datalake Gen 2](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
 * [Azure Data Lake Gen 1](https://hadoop.apache.org/docs/current/hadoop-azure-datalake/index.html).
 * [Hadoop-AWS module (Hadoop 3.x)](https://hadoop.apache.org/docs/current3/hadoop-aws/tools/hadoop-aws/index.html).
-* [Amazon S3 via S3A and S3N (Hadoop 2.x)](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html).
-* [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon
-* [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/hadoop/google-cloud-storage-connector). From Google
+* [Amazon S3 via S3A and S3N (Hadoop 2.x)](https://hadoop.apache.org/docs/current2/hadoop-aws/tools/hadoop-aws/index.html).
+* [Amazon EMR File System (EMRFS)](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-fs.html). From Amazon.
+* [Google Cloud Storage Connector for Spark and Hadoop](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage). From Google.
 * [The Azure Blob Filesystem driver (ABFS)](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-abfs-driver)
-* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage). From IBM
+* IBM Cloud Object Storage connector for Apache Spark: [Stocator](https://github.com/CODAIT/stocator), [IBM Object Storage](https://www.ibm.com/cloud/object-storage). From IBM.
 

From af95171898f83f4b1b249dcc9b74ee2148231dc9 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Mon, 20 Jul 2020 20:49:22 +0900
Subject: [PATCH 0910/1280] [SPARK-32366][DOC] Fix doc link of datetime pattern
 in 3.0 migration guide

### What changes were proposed in this pull request?

In http://spark.apache.org/docs/latest/sql-migration-guide.html#query-engine, there is a invalid reference for datetime reference "sql-ref-datetime-pattern.md". We should fix the link as http://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html.

![image](https://user-images.githubusercontent.com/1097932/87916920-fff57380-ca28-11ea-9028-99b9f9ebdfa4.png)

Also, it is nice to add url for [DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html)
### Why are the changes needed?

Fix migration guide doc

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Build the doc in local env and check it:
![image](https://user-images.githubusercontent.com/1097932/87919723-13a2d900-ca2d-11ea-9923-a29b4cefaf3c.png)

Closes #29162 from gengliangwang/fixDoc.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c2afe1c0b97fe2ae1fd03326968afdadb6806a97)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index fb3fe09a5d439..ce5655acda782 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -140,7 +140,8 @@ license: |
 
   - In Spark 3.0, Proleptic Gregorian calendar is used in parsing, formatting, and converting dates and timestamps as well as in extracting sub-components like years, days and so on. Spark 3.0 uses Java 8 API classes from the `java.time` packages that are based on [ISO chronology](https://docs.oracle.com/javase/8/docs/api/java/time/chrono/IsoChronology.html). In Spark version 2.4 and below, those operations are performed using the hybrid calendar ([Julian + Gregorian](https://docs.oracle.com/javase/7/docs/api/java/util/GregorianCalendar.html). The changes impact on the results for dates before October 15, 1582 (Gregorian) and affect on the following Spark 3.0 API:
 
-    * Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. In Spark 3.0, we define our own pattern strings in `sql-ref-datetime-pattern.md`, which is implemented via `java.time.format.DateTimeFormatter` under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and below, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in [simpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
+    * Parsing/formatting of timestamp/date strings. This effects on CSV/JSON datasources and on the `unix_timestamp`, `date_format`, `to_unix_timestamp`, `from_unixtime`, `to_date`, `to_timestamp` functions when patterns specified by users is used for parsing and formatting. In Spark 3.0, we define our own pattern strings in [Datetime Patterns for Formatting and Parsing](sql-ref-datetime-pattern.html),
+     which is implemented via [DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html) under the hood. New implementation performs strict checking of its input. For example, the `2015-07-22 10:00:00` timestamp cannot be parse if pattern is `yyyy-MM-dd` because the parser does not consume whole input. Another example is the `31/01/2015 00:00` input cannot be parsed by the `dd/MM/yyyy hh:mm` pattern because `hh` supposes hours in the range `1-12`. In Spark version 2.4 and below, `java.text.SimpleDateFormat` is used for timestamp/date string conversions, and the supported patterns are described in [SimpleDateFormat](https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). The old behavior can be restored by setting `spark.sql.legacy.timeParserPolicy` to `LEGACY`.
 
     * The `weekofyear`, `weekday`, `dayofweek`, `date_trunc`, `from_utc_timestamp`, `to_utc_timestamp`, and `unix_timestamp` functions use java.time API for calculation week number of year, day number of week as well for conversion from/to TimestampType values in UTC time zone.
 

From 2d9438601896bfeddd601b08efe0e2497f2417ca Mon Sep 17 00:00:00 2001
From: maruilei <maruilei@jd.com>
Date: Mon, 20 Jul 2020 13:48:57 -0700
Subject: [PATCH 0911/1280] [SPARK-32367][K8S][TESTS] Correct the spelling of
 parameter in KubernetesTestComponents

### What changes were proposed in this pull request?

Correct the spelling of parameter 'spark.executor.instances' in KubernetesTestComponents

### Why are the changes needed?

Parameter spelling error

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Test is not needed.

Closes #29164 from merrily01/SPARK-32367.

Authored-by: maruilei <maruilei@jd.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit ffdca8285ef7c7bd0da2622a81d9c21ada035794)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../deploy/k8s/integrationtest/KubernetesTestComponents.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index 4cfda8ab9ba77..af980f0494369 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -69,7 +69,7 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl
       .set("spark.master", s"k8s://${kubernetesClient.getMasterUrl}")
       .set("spark.kubernetes.namespace", namespace)
       .set("spark.executor.cores", "1")
-      .set("spark.executors.instances", "1")
+      .set("spark.executor.instances", "1")
       .set("spark.app.name", "spark-test-app")
       .set(IS_TESTING.key, "false")
       .set(UI_ENABLED.key, "true")

From 9e7b130ece1200504fde34d8ef40cb6a19509a01 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 20 Jul 2020 13:56:00 -0700
Subject: [PATCH 0912/1280] [SPARK-32368][SQL] pathGlobFilter,
 recursiveFileLookup and basePath should respect case insensitivity

### What changes were proposed in this pull request?

This PR proposes to make the datasource options at `PartitioningAwareFileIndex` respect case insensitivity consistently:
- `pathGlobFilter`
- `recursiveFileLookup `
- `basePath`

### Why are the changes needed?

To support consistent case insensitivity in datasource options.

### Does this PR introduce _any_ user-facing change?

Yes, now users can also use case insensitive options such as `PathglobFilter`.

### How was this patch tested?

Unittest were added. It reuses existing tests and adds extra clues to make it easier to track when the test is broken.

Closes #29165 from HyukjinKwon/SPARK-32368.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 133c5edc807ca87825f61dd9a5d36018620033ee)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../datasources/PartitioningAwareFileIndex.scala | 11 +++++++----
 .../spark/sql/FileBasedDataSourceSuite.scala     | 16 +++++++++-------
 .../execution/datasources/FileIndexSuite.scala   | 16 +++++++++-------
 3 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
index 2e09c729529a6..5341e22f5e670 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
@@ -56,14 +56,17 @@ abstract class PartitioningAwareFileIndex(
 
   protected def leafDirToChildrenFiles: Map[Path, Array[FileStatus]]
 
-  protected lazy val pathGlobFilter = parameters.get("pathGlobFilter").map(new GlobFilter(_))
+  private val caseInsensitiveMap = CaseInsensitiveMap(parameters)
+
+  protected lazy val pathGlobFilter: Option[GlobFilter] =
+    caseInsensitiveMap.get("pathGlobFilter").map(new GlobFilter(_))
 
   protected def matchGlobPattern(file: FileStatus): Boolean = {
     pathGlobFilter.forall(_.accept(file.getPath))
   }
 
-  protected lazy val recursiveFileLookup = {
-    parameters.getOrElse("recursiveFileLookup", "false").toBoolean
+  protected lazy val recursiveFileLookup: Boolean = {
+    caseInsensitiveMap.getOrElse("recursiveFileLookup", "false").toBoolean
   }
 
   override def listFiles(
@@ -215,7 +218,7 @@ abstract class PartitioningAwareFileIndex(
    * and the returned DataFrame will have the column of `something`.
    */
   private def basePaths: Set[Path] = {
-    parameters.get(BASE_PATH_PARAM).map(new Path(_)) match {
+    caseInsensitiveMap.get(BASE_PATH_PARAM).map(new Path(_)) match {
       case Some(userDefinedBasePath) =>
         val fs = userDefinedBasePath.getFileSystem(hadoopConf)
         if (!fs.isDirectory(userDefinedBasePath)) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index 231a8f2aa7ddd..e9bff64d72fc3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -633,13 +633,15 @@ class FileBasedDataSourceSuite extends QueryTest
 
       assert(fileList.toSet === expectedFileList.toSet)
 
-      val fileList2 = spark.read.format("binaryFile")
-        .option("recursiveFileLookup", true)
-        .option("pathGlobFilter", "*.bin")
-        .load(dataPath)
-        .select("path").collect().map(_.getString(0))
-
-      assert(fileList2.toSet === expectedFileList.filter(_.endsWith(".bin")).toSet)
+      withClue("SPARK-32368: 'recursiveFileLookup' and 'pathGlobFilter' can be case insensitive") {
+        val fileList2 = spark.read.format("binaryFile")
+          .option("RecuRsivefileLookup", true)
+          .option("PaThglobFilter", "*.bin")
+          .load(dataPath)
+          .select("path").collect().map(_.getString(0))
+
+        assert(fileList2.toSet === expectedFileList.filter(_.endsWith(".bin")).toSet)
+      }
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
index 553773e2555cf..c412d953d7037 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileIndexSuite.scala
@@ -362,13 +362,15 @@ class FileIndexSuite extends SharedSparkSession {
       val wrongBasePath = new File(dir, "unknown")
       // basePath must be a directory
       wrongBasePath.mkdir()
-      val parameters = Map("basePath" -> wrongBasePath.getCanonicalPath)
-      val fileIndex = new InMemoryFileIndex(spark, Seq(path), parameters, None)
-      val msg = intercept[IllegalArgumentException] {
-        // trigger inferPartitioning()
-        fileIndex.partitionSpec()
-      }.getMessage
-      assert(msg === s"Wrong basePath ${wrongBasePath.getCanonicalPath} for the root path: $path")
+      withClue("SPARK-32368: 'basePath' can be case insensitive") {
+        val parameters = Map("bAsepAtH" -> wrongBasePath.getCanonicalPath)
+        val fileIndex = new InMemoryFileIndex(spark, Seq(path), parameters, None)
+        val msg = intercept[IllegalArgumentException] {
+          // trigger inferPartitioning()
+          fileIndex.partitionSpec()
+        }.getMessage
+        assert(msg === s"Wrong basePath ${wrongBasePath.getCanonicalPath} for the root path: $path")
+      }
     }
   }
 

From 6d8b95c7a2e665c91f9097c94bd89b965b717e72 Mon Sep 17 00:00:00 2001
From: gengjiaan <gengjiaan@360.cn>
Date: Mon, 20 Jul 2020 20:34:51 -0700
Subject: [PATCH 0913/1280] [SPARK-32365][SQL] Add a boundary condition for
 negative index in regexp_extract

### What changes were proposed in this pull request?
The current implement of regexp_extract will throws a unprocessed exception show below:
SELECT regexp_extract('1a 2b 14m', 'd+' -1)

```
java.lang.IndexOutOfBoundsException: No group -1
java.util.regex.Matcher.group(Matcher.java:538)
org.apache.spark.sql.catalyst.expressions.RegExpExtract.nullSafeEval(regexpExpressions.scala:455)
org.apache.spark.sql.catalyst.expressions.TernaryExpression.eval(Expression.scala:704)
org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1$$anonfun$applyOrElse$1.applyOrElse(expressions.scala:52)
org.apache.spark.sql.catalyst.optimizer.ConstantFolding$$anonfun$apply$1$$anonfun$applyOrElse$1.applyOrElse(expressions.scala:45)
```

### Why are the changes needed?
Fix a bug `java.lang.IndexOutOfBoundsException: No group -1`

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
new UT

Closes #29161 from beliefer/regexp_extract-group-not-allow-less-than-zero.

Authored-by: gengjiaan <gengjiaan@360.cn>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 02114f96d64ec5be23fc61be6f6b32df7ad48a6c)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/regexpExpressions.scala       | 12 ++++++++++-
 .../expressions/RegexpExpressionsSuite.scala  |  6 ++++++
 .../sql-tests/inputs/regexp-functions.sql     |  2 ++
 .../results/regexp-functions.sql.out          | 20 ++++++++++++++++++-
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
index fc71943495bc7..933c77ac4f681 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala
@@ -411,7 +411,9 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio
 
 object RegExpExtract {
   def checkGroupIndex(groupCount: Int, groupIndex: Int): Unit = {
-    if (groupCount < groupIndex) {
+    if (groupIndex < 0) {
+      throw new IllegalArgumentException("The specified group index cannot be less than zero")
+    } else if (groupCount < groupIndex) {
       throw new IllegalArgumentException(
         s"Regex group count is $groupCount, but the specified group index is $groupIndex")
     }
@@ -425,6 +427,14 @@ object RegExpExtract {
  */
 @ExpressionDescription(
   usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches `regexp`.",
+  arguments = """
+    Arguments:
+      * str - a string expression.
+      * regexp - a string representing a regular expression.
+          The regex string should be a Java regular expression.
+      * idx - an integer expression that representing the group index. The group index should be
+          non-negative. If `idx` is not specified, the default group index value is 1.
+  """,
   examples = """
     Examples:
       > SELECT _FUNC_('100-200', '(\\d+)-(\\d+)', 1);
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
index ad9492a8d3ab1..cab81f85fda06 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala
@@ -303,6 +303,8 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     val row8 = create_row("100-200", "(\\d+)-(\\d+)", 3)
     val row9 = create_row("100-200", "(\\d+).*", 2)
     val row10 = create_row("100-200", "\\d+", 1)
+    val row11 = create_row("100-200", "(\\d+)-(\\d+)", -1)
+    val row12 = create_row("100-200", "\\d+", -1)
 
     checkExceptionInExpression[IllegalArgumentException](
       expr, row8, "Regex group count is 2, but the specified group index is 3")
@@ -310,6 +312,10 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       expr, row9, "Regex group count is 1, but the specified group index is 2")
     checkExceptionInExpression[IllegalArgumentException](
       expr, row10, "Regex group count is 0, but the specified group index is 1")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row11, "The specified group index cannot be less than zero")
+    checkExceptionInExpression[IllegalArgumentException](
+      expr, row12, "The specified group index cannot be less than zero")
 
     // Test escaping of arguments
     GenerateUnsafeProjection.generate(
diff --git a/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
index c0827a3cba39b..8a531be30d896 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/regexp-functions.sql
@@ -3,7 +3,9 @@ SELECT regexp_extract('1a 2b 14m', '\\d+');
 SELECT regexp_extract('1a 2b 14m', '\\d+', 0);
 SELECT regexp_extract('1a 2b 14m', '\\d+', 1);
 SELECT regexp_extract('1a 2b 14m', '\\d+', 2);
+SELECT regexp_extract('1a 2b 14m', '\\d+', -1);
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)');
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 0);
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 1);
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2);
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', -1);
diff --git a/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
index c92c1ddca774f..64aa6053d8d70 100644
--- a/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/regexp-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 8
+-- Number of queries: 10
 
 
 -- !query
@@ -37,6 +37,15 @@ java.lang.IllegalArgumentException
 Regex group count is 0, but the specified group index is 2
 
 
+-- !query
+SELECT regexp_extract('1a 2b 14m', '\\d+', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
+
+
 -- !query
 SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)')
 -- !query schema
@@ -67,3 +76,12 @@ SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', 2)
 struct<regexp_extract(1a 2b 14m, (\d+)([a-z]+), 2):string>
 -- !query output
 a
+
+
+-- !query
+SELECT regexp_extract('1a 2b 14m', '(\\d+)([a-z]+)', -1)
+-- !query schema
+struct<>
+-- !query output
+java.lang.IllegalArgumentException
+The specified group index cannot be less than zero
\ No newline at end of file

From ba15146aab84c852a5d7c9c586307a58e7b61a8d Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Tue, 21 Jul 2020 12:47:45 +0900
Subject: [PATCH 0914/1280] [SPARK-32362][SQL][TEST] AdaptiveQueryExecSuite
 misses verifying AE results

### What changes were proposed in this pull request?
Verify results for `AdaptiveQueryExecSuite`

### Why are the changes needed?
`AdaptiveQueryExecSuite` misses verifying AE results
```scala
QueryTest.sameRows(result.toSeq, df.collect().toSeq)
```
Even the results are different, no fail.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Exists unit tests.

Closes #29158 from LantaoJin/SPARK-32362.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8a1c24bb0364313f20382e2d14d5670b111a5742)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index 6477ddc11ea72..e18adbda96c0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -68,7 +68,7 @@ class AdaptiveQueryExecSuite
     val result = dfAdaptive.collect()
     withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
       val df = sql(query)
-      QueryTest.sameRows(result.toSeq, df.collect().toSeq)
+      checkAnswer(df, result)
     }
     val planAfter = dfAdaptive.queryExecution.executedPlan
     assert(planAfter.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))

From aaee1f816ebfb79115d3151576ee7c1c9a456609 Mon Sep 17 00:00:00 2001
From: Brandon <brandonJY@users.noreply.github.com>
Date: Tue, 21 Jul 2020 13:42:19 +0900
Subject: [PATCH 0915/1280] [MINOR][DOCS] add link for Debugging your
 Application in running-on-yarn.html#launching-spark-on-yarn

### What changes were proposed in this pull request?
 add link for Debugging your Application in `running-on-yarn.html#launching-spark-on-yar`

### Why are the changes needed?
Currrently on running-on-yarn.html page launching-spark-on-yarn section, it mentions to refer for Debugging your Application. It is better to add a direct link for it to save reader time to find the section
  ![image](https://user-images.githubusercontent.com/20021316/87867542-80cc5500-c9c0-11ea-8560-5ddcb5a308bc.png)

### Does this PR introduce _any_ user-facing change?
Yes.
Docs changes.
1. add link for Debugging your Application in `running-on-yarn.html#launching-spark-on-yarn` section
Updated behavior:
![image](https://user-images.githubusercontent.com/20021316/87867534-6eeab200-c9c0-11ea-94ee-d3fa58157156.png)
2. update Spark Properties link to anchor link only

### How was this patch tested?
manual test has been performed to test the updated

Closes #29154 from brandonJY/patch-1.

Authored-by: Brandon <brandonJY@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1267d80db6abaa130384b8e7b514c39aec3a8c77)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/running-on-yarn.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 166fb874dd9c1..046c545eb1b4c 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -60,7 +60,7 @@ For example:
         examples/jars/spark-examples*.jar \
         10
 
-The above starts a YARN client program which starts the default Application Master. Then SparkPi will be run as a child thread of Application Master. The client will periodically poll the Application Master for status updates and display them in the console. The client will exit once your application has finished running.  Refer to the "Debugging your Application" section below for how to see driver and executor logs.
+The above starts a YARN client program which starts the default Application Master. Then SparkPi will be run as a child thread of Application Master. The client will periodically poll the Application Master for status updates and display them in the console. The client will exit once your application has finished running.  Refer to the [Debugging your Application](#debugging-your-application) section below for how to see driver and executor logs.
 
 To launch a Spark application in `client` mode, do the same, but replace `cluster` with `client`. The following shows how you can run `spark-shell` in `client` mode:
 
@@ -84,7 +84,7 @@ Running Spark on YARN requires a binary distribution of Spark which is built wit
 Binary distributions can be downloaded from the [downloads page](https://spark.apache.org/downloads.html) of the project website.
 To build Spark yourself, refer to [Building Spark](building-spark.html).
 
-To make Spark runtime jars accessible from YARN side, you can specify `spark.yarn.archive` or `spark.yarn.jars`. For details please refer to [Spark Properties](running-on-yarn.html#spark-properties). If neither `spark.yarn.archive` nor `spark.yarn.jars` is specified, Spark will create a zip file with all jars under `$SPARK_HOME/jars` and upload it to the distributed cache.
+To make Spark runtime jars accessible from YARN side, you can specify `spark.yarn.archive` or `spark.yarn.jars`. For details please refer to [Spark Properties](#spark-properties). If neither `spark.yarn.archive` nor `spark.yarn.jars` is specified, Spark will create a zip file with all jars under `$SPARK_HOME/jars` and upload it to the distributed cache.
 
 # Configuration
 

From 972914485f60cd97e28c7dbdb3df41937257527e Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 20 Jul 2020 22:20:16 -0700
Subject: [PATCH 0916/1280] [SPARK-32377][SQL] CaseInsensitiveMap should be
 deterministic for addition

### What changes were proposed in this pull request?

This PR aims to fix `CaseInsensitiveMap` to be deterministic for addition.

### Why are the changes needed?

```scala
import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
var m = CaseInsensitiveMap(Map.empty[String, String])
Seq(("paTh", "1"), ("PATH", "2"), ("Path", "3"), ("patH", "4"), ("path", "5")).foreach { kv =>
  m = (m + kv).asInstanceOf[CaseInsensitiveMap[String]]
  println(m.get("path"))
}
```

**BEFORE**
```
Some(1)
Some(2)
Some(3)
Some(4)
Some(1)
```

**AFTER**
```
Some(1)
Some(2)
Some(3)
Some(4)
Some(5)
```

### Does this PR introduce _any_ user-facing change?

Yes, but this is a bug fix on non-deterministic behavior.

### How was this patch tested?

Pass the newly added test case.

Closes #29172 from dongjoon-hyun/SPARK-32377.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 8c7d6f9733751503f80d5a1b2463904dfefd6843)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/util/CaseInsensitiveMap.scala      | 2 +-
 .../spark/sql/catalyst/util/CaseInsensitiveMap.scala      | 2 +-
 .../spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala | 8 ++++++++
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 699b42bd6d60d..aff294e5d6305 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -40,7 +40,7 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
   override def +[B1 >: T](kv: (String, B1)): Map[String, B1] = {
-    new CaseInsensitiveMap(originalMap + kv)
+    new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(kv._1)) + kv)
   }
 
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 1f8cc6c0f83a2..a74636a80f1ce 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -40,7 +40,7 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
   override def updated[B1 >: T](key: String, value: B1): Map[String, B1] = {
-    new CaseInsensitiveMap[B1](originalMap + (key -> value))
+    new CaseInsensitiveMap[B1](originalMap.filter(!_._1.equalsIgnoreCase(key)) + (key -> value))
   }
 
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
index a8bb1d0afdb87..db8b7403cb60e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/CaseInsensitiveMapSuite.scala
@@ -44,4 +44,12 @@ class CaseInsensitiveMapSuite extends SparkFunSuite {
     assert(m == Map("a" -> "b", "foo" -> "bar", "x" -> "y"))
     shouldBeSerializable(m)
   }
+
+  test("SPARK-32377: CaseInsensitiveMap should be deterministic for addition") {
+    var m = CaseInsensitiveMap(Map.empty[String, String])
+    Seq(("paTh", "1"), ("PATH", "2"), ("Path", "3"), ("patH", "4"), ("path", "5")).foreach { kv =>
+      m = (m + kv).asInstanceOf[CaseInsensitiveMap[String]]
+      assert(m.get("path").contains(kv._2))
+    }
+  }
 }

From ad9e7a2bb0593c7386ef01d202cb56654ea8fa13 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 22 Jul 2020 07:58:45 -0700
Subject: [PATCH 0917/1280] [SPARK-32364][SQL] Use CaseInsensitiveMap for
 DataFrameReader/Writer options

### What changes were proposed in this pull request?

When a user have multiple options like `path`, `paTH`, and `PATH` for the same key `path`, `option/options` is non-deterministic because `extraOptions` is `HashMap`. This PR aims to use `CaseInsensitiveMap` instead of `HashMap` to fix this bug fundamentally.

### Why are the changes needed?

Like the following, DataFrame's `option/options` have been non-deterministic in terms of case-insensitivity because it stores the options at `extraOptions` which is using `HashMap` class.

```scala
spark.read
  .option("paTh", "1")
  .option("PATH", "2")
  .option("Path", "3")
  .option("patH", "4")
  .load("5")
...
org.apache.spark.sql.AnalysisException:
Path does not exist: file:/.../1;
```

### Does this PR introduce _any_ user-facing change?

Yes. However, this is a bug fix for the indeterministic cases.

### How was this patch tested?

Pass the Jenkins or GitHub Action with newly added test cases.

Closes #29160 from dongjoon-hyun/SPARK-32364.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit cd16a10475c110dbf5739a37e8f5f103b5541234)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/util/CaseInsensitiveMap.scala    |  6 ++++-
 .../catalyst/util/CaseInsensitiveMap.scala    |  6 ++++-
 .../apache/spark/sql/DataFrameReader.scala    | 10 ++++-----
 .../apache/spark/sql/DataFrameWriter.scala    |  5 +++--
 .../org/apache/spark/sql/jdbc/JDBCSuite.scala |  3 ++-
 .../sql/test/DataFrameReaderWriterSuite.scala | 22 +++++++++++++++++++
 6 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index aff294e5d6305..c013888d080f2 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -39,10 +39,14 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def contains(k: String): Boolean =
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
-  override def +[B1 >: T](kv: (String, B1)): Map[String, B1] = {
+  override def +[B1 >: T](kv: (String, B1)): CaseInsensitiveMap[B1] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(kv._1)) + kv)
   }
 
+  def ++(xs: TraversableOnce[(String, T)]): CaseInsensitiveMap[T] = {
+    xs.foldLeft(this)(_ + _)
+  }
+
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
 
   override def -(key: String): Map[String, T] = {
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index a74636a80f1ce..66a2f2549b460 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -39,10 +39,14 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def contains(k: String): Boolean =
     keyLowerCasedMap.contains(k.toLowerCase(Locale.ROOT))
 
-  override def updated[B1 >: T](key: String, value: B1): Map[String, B1] = {
+  override def updated[B1 >: T](key: String, value: B1): CaseInsensitiveMap[B1] = {
     new CaseInsensitiveMap[B1](originalMap.filter(!_._1.equalsIgnoreCase(key)) + (key -> value))
   }
 
+  def ++(xs: IterableOnce[(String, T)]): CaseInsensitiveMap[T] = {
+    xs.iterator.foldLeft(this) { (m, kv) => m.updated(kv._1, kv._2) }
+  }
+
   override def iterator: Iterator[(String, T)] = keyLowerCasedMap.iterator
 
   override def removed(key: String): Map[String, T] = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index a7b3d08ec4df7..a291e5825815d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -31,7 +31,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.csv.{CSVHeaderChecker, CSVOptions, UnivocityParser}
 import org.apache.spark.sql.catalyst.expressions.ExprUtils
 import org.apache.spark.sql.catalyst.json.{CreateJacksonParser, JacksonParser, JSONOptions}
-import org.apache.spark.sql.catalyst.util.FailureSafeParser
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, FailureSafeParser}
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsCatalogOptions, SupportsRead}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -238,7 +238,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
         Some("paths" -> objectMapper.writeValueAsString(paths.toArray))
       }
 
-      val finalOptions = sessionOptions ++ extraOptions.toMap ++ pathsOption
+      val finalOptions = sessionOptions ++ extraOptions.originalMap ++ pathsOption
       val dsOptions = new CaseInsensitiveStringMap(finalOptions.asJava)
       val (table, catalog, ident) = provider match {
         case _: SupportsCatalogOptions if userSpecifiedSchema.nonEmpty =>
@@ -276,7 +276,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
         paths = paths,
         userSpecifiedSchema = userSpecifiedSchema,
         className = source,
-        options = extraOptions.toMap).resolveRelation())
+        options = extraOptions.originalMap).resolveRelation())
   }
 
   /**
@@ -290,7 +290,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
     // properties should override settings in extraOptions.
     this.extraOptions ++= properties.asScala
     // explicit url and dbtable should override all
-    this.extraOptions += (JDBCOptions.JDBC_URL -> url, JDBCOptions.JDBC_TABLE_NAME -> table)
+    this.extraOptions ++= Seq(JDBCOptions.JDBC_URL -> url, JDBCOptions.JDBC_TABLE_NAME -> table)
     format("jdbc").load()
   }
 
@@ -879,6 +879,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
 
   private var userSpecifiedSchema: Option[StructType] = None
 
-  private val extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 9aef382470bcc..952f896d1a20f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, NoSuchT
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.Literal
 import org.apache.spark.sql.catalyst.plans.logical.{AppendData, CreateTableAsSelect, CreateTableAsSelectStatement, InsertIntoStatement, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, ReplaceTableAsSelectStatement}
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Implicits, CatalogV2Util, Identifier, SupportsCatalogOptions, Table, TableCatalog, TableProvider, V1Table}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
@@ -768,7 +769,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
     // connectionProperties should override settings in extraOptions.
     this.extraOptions ++= connectionProperties.asScala
     // explicit url and dbtable should override all
-    this.extraOptions += ("url" -> url, "dbtable" -> table)
+    this.extraOptions ++= Seq("url" -> url, "dbtable" -> table)
     format("jdbc").save()
   }
 
@@ -960,7 +961,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
 
   private var mode: SaveMode = SaveMode.ErrorIfExists
 
-  private val extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
   private var partitioningColumns: Option[Seq[String]] = None
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 7c723f083642d..40237bb3cfd09 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1302,7 +1302,8 @@ class JDBCSuite extends QueryTest
     testJdbcOptions(new JDBCOptions(parameters))
     testJdbcOptions(new JDBCOptions(CaseInsensitiveMap(parameters)))
     // test add/remove key-value from the case-insensitive map
-    var modifiedParameters = CaseInsensitiveMap(Map.empty) ++ parameters
+    var modifiedParameters =
+      (CaseInsensitiveMap(Map.empty) ++ parameters).asInstanceOf[Map[String, String]]
     testJdbcOptions(new JDBCOptions(modifiedParameters))
     modifiedParameters -= "dbtable"
     assert(modifiedParameters.get("dbTAblE").isEmpty)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index fe0a8439acc2d..c7ca0125c8a0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -224,6 +224,28 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
     assert(LastOptions.parameters("opt3") == "3")
   }
 
+  test("SPARK-32364: path argument of load function should override all existing options") {
+    spark.read
+      .format("org.apache.spark.sql.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .load("5")
+    assert(LastOptions.parameters("path") == "5")
+  }
+
+  test("SPARK-32364: path argument of save function should override all existing options") {
+    Seq(1).toDF.write
+      .format("org.apache.spark.sql.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .save("5")
+    assert(LastOptions.parameters("path") == "5")
+  }
+
   test("pass partitionBy as options") {
     Seq(1).toDF.write
       .format("org.apache.spark.sql.test")

From f6f6026965d59e1f79dec1740e1b2056adea94e8 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 23 Jul 2020 06:28:08 -0700
Subject: [PATCH 0918/1280] [SPARK-32364][SQL][FOLLOWUP] Add toMap to return
 originalMap and documentation

### What changes were proposed in this pull request?

This is a follow-up of https://github.com/apache/spark/pull/29160. We already removed the indeterministicity. This PR aims the following for the existing code base.
1. Add an explicit document to `DataFrameReader/DataFrameWriter`.

2. Add `toMap` to `CaseInsensitiveMap` in order to return `originalMap: Map[String, T]` because it's more consistent with the existing `case-sensitive key names` behavior for the existing code pattern like `AppendData.byName(..., extraOptions.toMap)`. Previously, it was `HashMap.toMap`.

3. During (2), we need to change the following to keep the original logic using `CaseInsensitiveMap.++`.
```scala
- val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
+ val params = extraOptions ++ connectionProperties.asScala
```

4. Additionally, use `.toMap` in the following because `dsOptions.asCaseSensitiveMap()` is used later.
```scala
- val options = sessionOptions ++ extraOptions
+ val options = sessionOptions.filterKeys(!extraOptions.contains(_)) ++ extraOptions.toMap
  val dsOptions = new CaseInsensitiveStringMap(options.asJava)
```

### Why are the changes needed?

`extraOptions.toMap` is used in several places (e.g. `DataFrameReader`) to hand over `Map[String, T]`. In this case, `CaseInsensitiveMap[T] private (val originalMap: Map[String, T])` had better return `originalMap`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins or GitHub Action with the existing tests and newly add test case at `JDBCSuite`.

Closes #29191 from dongjoon-hyun/SPARK-32364-3.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit aed8dbab1d6725eb17f743c300451fcbdbfa3e97)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../catalyst/util/CaseInsensitiveMap.scala    |  2 ++
 .../catalyst/util/CaseInsensitiveMap.scala    |  2 ++
 .../apache/spark/sql/DataFrameReader.scala    | 20 +++++++++++++++++-
 .../apache/spark/sql/DataFrameWriter.scala    | 20 +++++++++++++++++-
 .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 21 +++++++++++++++++++
 5 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index c013888d080f2..14b8f620017f6 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -52,6 +52,8 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def -(key: String): Map[String, T] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(key)))
   }
+
+  def toMap: Map[String, T] = originalMap
 }
 
 object CaseInsensitiveMap {
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
index 66a2f2549b460..1db4b7a4f227b 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/util/CaseInsensitiveMap.scala
@@ -52,6 +52,8 @@ class CaseInsensitiveMap[T] private (val originalMap: Map[String, T]) extends Ma
   override def removed(key: String): Map[String, T] = {
     new CaseInsensitiveMap(originalMap.filter(!_._1.equalsIgnoreCase(key)))
   }
+
+  def toMap: Map[String, T] = originalMap
 }
 
 object CaseInsensitiveMap {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index a291e5825815d..5b78690b3267c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -94,6 +94,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -121,6 +124,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Boolean): DataFrameReader = option(key, value.toString)
@@ -128,6 +134,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Long): DataFrameReader = option(key, value.toString)
@@ -135,6 +144,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds an input option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Double): DataFrameReader = option(key, value.toString)
@@ -142,6 +154,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * (Scala-specific) Adds input options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -169,6 +184,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
   /**
    * Adds input options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -361,7 +379,7 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
       connectionProperties: Properties): DataFrame = {
     assertNoSpecifiedSchema("jdbc")
     // connectionProperties should override settings in extraOptions.
-    val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
+    val params = extraOptions ++ connectionProperties.asScala
     val options = new JDBCOptions(url, table, params)
     val parts: Array[Partition] = predicates.zipWithIndex.map { case (part, i) =>
       JDBCPartition(part, i) : Partition
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 952f896d1a20f..f463166a9f268 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -105,6 +105,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -132,6 +135,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Boolean): DataFrameWriter[T] = option(key, value.toString)
@@ -139,6 +145,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Long): DataFrameWriter[T] = option(key, value.toString)
@@ -146,6 +155,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds an output option for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * @since 2.0.0
    */
   def option(key: String, value: Double): DataFrameWriter[T] = option(key, value.toString)
@@ -153,6 +165,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * (Scala-specific) Adds output options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -180,6 +195,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
   /**
    * Adds output options for the underlying data source.
    *
+   * All options are maintained in a case-insensitive way in terms of key names.
+   * If a new option has the same key case-insensitively, it will override the existing option.
+   *
    * You can set the following option(s):
    * <ul>
    * <li>`timeZone` (default session local timezone): sets the string that indicates a time zone ID
@@ -288,7 +306,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
       val provider = maybeV2Provider.get
       val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
         provider, df.sparkSession.sessionState.conf)
-      val options = sessionOptions ++ extraOptions
+      val options = sessionOptions.filterKeys(!extraOptions.contains(_)) ++ extraOptions.toMap
       val dsOptions = new CaseInsensitiveStringMap(options.asJava)
 
       def getTable: Table = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 40237bb3cfd09..b554754895391 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -21,6 +21,8 @@ import java.math.BigDecimal
 import java.sql.{Date, DriverManager, SQLException, Timestamp}
 import java.util.{Calendar, GregorianCalendar, Properties}
 
+import scala.collection.JavaConverters._
+
 import org.h2.jdbc.JdbcSQLException
 import org.scalatest.{BeforeAndAfter, PrivateMethodTester}
 
@@ -1716,4 +1718,23 @@ class JDBCSuite extends QueryTest
     jdbcDF = sqlContext.jdbc(urlWithUserAndPass, "TEST.PEOPLE", parts)
     checkAnswer(jdbcDF, Row("mary", 2) :: Nil)
   }
+
+  test("SPARK-32364: JDBCOption constructor") {
+    val extraOptions = CaseInsensitiveMap[String](Map("UrL" -> "url1", "dBTable" -> "table1"))
+    val connectionProperties = new Properties()
+    connectionProperties.put("url", "url2")
+    connectionProperties.put("dbtable", "table2")
+
+    // connection property should override the options in extraOptions
+    val params = extraOptions ++ connectionProperties.asScala
+    assert(params.size == 2)
+    assert(params.get("uRl").contains("url2"))
+    assert(params.get("DbtaBle").contains("table2"))
+
+    // JDBCOptions constructor parameter should overwrite the existing conf
+    val options = new JDBCOptions(url, "table3", params)
+    assert(options.asProperties.size == 2)
+    assert(options.asProperties.get("url") == url)
+    assert(options.asProperties.get("dbtable") == "table3")
+  }
 }

From 01c88bedc87c07194abfab8d9d7e03271ffd9757 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 23 Jul 2020 14:02:38 +0000
Subject: [PATCH 0919/1280] [SPARK-32251][SQL][TESTS][FOLLOWUP] improve SQL
 keyword test

Improve the `SQLKeywordSuite` so that:
1. it checks keywords under default mode as well
2. it checks if there are typos in the doc (found one and fixed in this PR)

better test coverage

no

N/A

Closes #29200 from cloud-fan/test.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit aa54dcf193a2149182da779191cf12f087305726)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-ansi-compliance.md               |  2 +-
 .../spark/sql/catalyst/parser/SqlBase.g4      |  2 +
 .../spark/sql/catalyst/SQLKeywordSuite.scala  | 46 ++++++++++++++++---
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 6488ad9cd34c9..193616120ec99 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -264,7 +264,7 @@ Below is a list of all the keywords in Spark SQL.
 |MAP|non-reserved|non-reserved|non-reserved|
 |MATCHED|non-reserved|non-reserved|non-reserved|
 |MERGE|non-reserved|non-reserved|non-reserved|
-|MINUS|not-reserved|strict-non-reserved|non-reserved|
+|MINUS|non-reserved|strict-non-reserved|non-reserved|
 |MINUTE|reserved|non-reserved|reserved|
 |MONTH|reserved|non-reserved|reserved|
 |MSCK|non-reserved|non-reserved|non-reserved|
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index e02bcda52f618..df6ff9fe5af11 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -1200,6 +1200,7 @@ strictNonReserved
     ;
 
 nonReserved
+//--DEFAULT-NON-RESERVED-START
     : ADD
     | AFTER
     | ALL
@@ -1438,6 +1439,7 @@ nonReserved
     | WINDOW
     | WITH
     | YEAR
+//--DEFAULT-NON-RESERVED-END
     ;
 
 // NOTE: If you add a new token in the list below, you should update the list of keywords
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
index 3d41d02b23df5..082b01173e7b8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala
@@ -38,7 +38,7 @@ trait SQLKeywordUtils extends SQLHelper {
   }
 
   // each element is an array of 4 string: the keyword name, reserve or not in Spark ANSI mode,
-  // Spark non-ANSI mode, and the SQL standard.
+  // Spark default mode, and the SQL standard.
   val keywordsInDoc: Array[Array[String]] = {
     val docPath = {
       java.nio.file.Paths.get(sparkHome, "docs", "sql-ref-ansi-compliance.md").toFile
@@ -135,6 +135,19 @@ trait SQLKeywordUtils extends SQLHelper {
   }
 
   val reservedKeywordsInAnsiMode = allCandidateKeywords -- nonReservedKeywordsInAnsiMode
+
+  val nonReservedKeywordsInDefaultMode: Set[String] = {
+    val kwDef = """\s*[\|:]\s*([A-Z_]+)\s*""".r
+    parseAntlrGrammars("//--DEFAULT-NON-RESERVED-START", "//--DEFAULT-NON-RESERVED-END") {
+      // Parses a pattern, e.g., `    | AFTER`
+      case kwDef(symbol) =>
+        if (symbolsToExpandIntoDifferentLiterals.contains(symbol)) {
+          symbolsToExpandIntoDifferentLiterals(symbol)
+        } else {
+          symbol :: Nil
+        }
+    }
+  }
 }
 
 class SQLKeywordSuite extends SparkFunSuite with SQLKeywordUtils {
@@ -146,11 +159,32 @@ class SQLKeywordSuite extends SparkFunSuite with SQLKeywordUtils {
     }
   }
 
-  test("Spark keywords are documented correctly") {
-    val reservedKeywordsInDoc = keywordsInDoc.filter(_.apply(1) == "reserved").map(_.head).toSet
-    if (reservedKeywordsInAnsiMode != reservedKeywordsInDoc) {
-      val misImplemented = (reservedKeywordsInDoc -- reservedKeywordsInAnsiMode).toSeq.sorted
-      fail("Some keywords are documented as reserved but actually not: " +
+  test("Spark keywords are documented correctly under ANSI mode") {
+    // keywords under ANSI mode should either be reserved or non-reserved.
+    keywordsInDoc.map(_.apply(1)).foreach { desc =>
+      assert(desc == "reserved" || desc == "non-reserved")
+    }
+
+    val nonReservedInDoc = keywordsInDoc.filter(_.apply(1) == "non-reserved").map(_.head).toSet
+    if (nonReservedKeywordsInAnsiMode != nonReservedInDoc) {
+      val misImplemented = ((nonReservedInDoc -- nonReservedKeywordsInAnsiMode) ++
+        (nonReservedKeywordsInAnsiMode -- nonReservedInDoc)).toSeq.sorted
+      fail("Some keywords are documented and implemented inconsistently: " +
+        misImplemented.mkString(", "))
+    }
+  }
+
+  test("Spark keywords are documented correctly under default mode") {
+    // keywords under default mode should either be strict-non-reserved or non-reserved.
+    keywordsInDoc.map(_.apply(2)).foreach { desc =>
+      assert(desc == "strict-non-reserved" || desc == "non-reserved")
+    }
+
+    val nonReservedInDoc = keywordsInDoc.filter(_.apply(2) == "non-reserved").map(_.head).toSet
+    if (nonReservedKeywordsInDefaultMode != nonReservedInDoc) {
+      val misImplemented = ((nonReservedInDoc -- nonReservedKeywordsInDefaultMode) ++
+        (nonReservedKeywordsInDefaultMode -- nonReservedInDoc)).toSeq.sorted
+      fail("Some keywords are documented and implemented inconsistently: " +
         misImplemented.mkString(", "))
     }
   }

From ebac47b96cb38d7cd5d73a3f238017a5f8d77d1a Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 23 Jul 2020 14:24:47 +0000
Subject: [PATCH 0920/1280] [SPARK-32280][SPARK-32372][SQL]
 ResolveReferences.dedupRight should only rewrite attributes for ancestor
 nodes of the conflict plan
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR refactors `ResolveReferences.dedupRight` to make sure it only rewrite attributes for ancestor nodes of the conflict plan.

This is a bug fix.

```scala
sql("SELECT name, avg(age) as avg_age FROM person GROUP BY name")
  .createOrReplaceTempView("person_a")
sql("SELECT p1.name, p2.avg_age FROM person p1 JOIN person_a p2 ON p1.name = p2.name")
  .createOrReplaceTempView("person_b")
sql("SELECT * FROM person_a UNION SELECT * FROM person_b")
  .createOrReplaceTempView("person_c")
sql("SELECT p1.name, p2.avg_age FROM person_c p1 JOIN person_c p2 ON p1.name = p2.name").show()
```
When executing the above query, we'll hit the error:

```scala
[info]   Failed to analyze query: org.apache.spark.sql.AnalysisException: Resolved attribute(s) avg_age#231 missing from name#223,avg_age#218,id#232,age#234,name#233 in operator !Project [name#233, avg_age#231]. Attribute(s) with the same name appear in the operation: avg_age. Please check if the right attribute(s) are used.;;
...
```

The plan below is the problematic plan which is the right plan of a `Join` operator. And, it has conflict plans comparing to the left plan. In this problematic plan, the first `Aggregate` operator (the one under the first child of `Union`) becomes a conflict plan compares to the left one and has a rewrite attribute pair as  `avg_age#218` -> `avg_age#231`. With the current `dedupRight` logic, we'll first replace this `Aggregate` with a new one, and then rewrites the attribute `avg_age#218` from bottom to up. As you can see, projects with the attribute `avg_age#218` of the second child of the `Union` can also be replaced with `avg_age#231`(That means we also rewrite attributes for non-ancestor plans for the conflict plan). Ideally, the attribute `avg_age#218` in the second `Aggregate` operator (the one under the second child of `Union`) should also be replaced. But it didn't because it's an `Alias` while we only rewrite `Attribute` yet. Therefore, the project above the second `Aggregate` becomes unresolved.

```scala
:

:
+- SubqueryAlias p2
   +- SubqueryAlias person_c
      +- Distinct
         +- Union
            :- Project [name#233, avg_age#231]
            :  +- SubqueryAlias person_a
            :     +- Aggregate [name#233], [name#233, avg(cast(age#234 as bigint)) AS avg_age#231]
            :        +- SubqueryAlias person
            :           +- SerializeFromObject [knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).id AS id#232, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).name, true, false) AS name#233, knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).age AS age#234]
            :              +- ExternalRDD [obj#165]
            +- Project [name#233 AS name#227, avg_age#231 AS avg_age#228]
               +- Project [name#233, avg_age#231]
                  +- SubqueryAlias person_b
                     +- !Project [name#233, avg_age#231]
                        +- Join Inner, (name#233 = name#223)
                           :- SubqueryAlias p1
                           :  +- SubqueryAlias person
                           :     +- SerializeFromObject [knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).id AS id#232, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).name, true, false) AS name#233, knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).age AS age#234]
                           :        +- ExternalRDD [obj#165]
                           +- SubqueryAlias p2
                              +- SubqueryAlias person_a
                                 +- Aggregate [name#223], [name#223, avg(cast(age#224 as bigint)) AS avg_age#218]
                                    +- SubqueryAlias person
                                       +- SerializeFromObject [knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).id AS id#222, staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).name, true, false) AS name#223, knownnotnull(assertnotnull(input[0, org.apache.spark.sql.test.SQLTestData$Person, true])).age AS age#224]
                                          +- ExternalRDD [obj#165]
```

Yes, users would no longer hit the error after this fix.

Added test.

Closes #29166 from Ngone51/impr-dedup.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a8e3de36e7d543f1c7923886628ac3178f45f512)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 63 ++++++++++++++++---
 .../org/apache/spark/sql/SQLQuerySuite.scala  | 22 +++++++
 2 files changed, 75 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 68fe5805fcc43..bd5a7975299f4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1181,11 +1181,24 @@ class Analyzer(
             if findAliases(projectList).intersect(conflictingAttributes).nonEmpty =>
           Seq((oldVersion, oldVersion.copy(projectList = newAliases(projectList))))
 
+        // We don't need to search child plan recursively if the projectList of a Project
+        // is only composed of Alias and doesn't contain any conflicting attributes.
+        // Because, even if the child plan has some conflicting attributes, the attributes
+        // will be aliased to non-conflicting attributes by the Project at the end.
+        case _ @ Project(projectList, _)
+          if findAliases(projectList).size == projectList.size =>
+          Nil
+
         case oldVersion @ Aggregate(_, aggregateExpressions, _)
             if findAliases(aggregateExpressions).intersect(conflictingAttributes).nonEmpty =>
           Seq((oldVersion, oldVersion.copy(
             aggregateExpressions = newAliases(aggregateExpressions))))
 
+        // We don't search the child plan recursively for the same reason as the above Project.
+        case _ @ Aggregate(_, aggregateExpressions, _)
+          if findAliases(aggregateExpressions).size == aggregateExpressions.size =>
+          Nil
+
         case oldVersion @ FlatMapGroupsInPandas(_, _, output, _)
             if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
           Seq((oldVersion, oldVersion.copy(output = output.map(_.newInstance()))))
@@ -1226,20 +1239,50 @@ class Analyzer(
       if (conflictPlans.isEmpty) {
         right
       } else {
-        val attributeRewrites = AttributeMap(conflictPlans.flatMap {
-          case (oldRelation, newRelation) => oldRelation.output.zip(newRelation.output)})
-        val conflictPlanMap = conflictPlans.toMap
-        // transformDown so that we can replace all the old Relations in one turn due to
-        // the reason that `conflictPlans` are also collected in pre-order.
-        right transformDown {
-          case r => conflictPlanMap.getOrElse(r, r)
-        } transformUp {
-          case other => other transformExpressions {
+        rewritePlan(right, conflictPlans.toMap)._1
+      }
+    }
+
+    private def rewritePlan(plan: LogicalPlan, conflictPlanMap: Map[LogicalPlan, LogicalPlan])
+      : (LogicalPlan, Seq[(Attribute, Attribute)]) = {
+      if (conflictPlanMap.contains(plan)) {
+        // If the plan is the one that conflict the with left one, we'd
+        // just replace it with the new plan and collect the rewrite
+        // attributes for the parent node.
+        val newRelation = conflictPlanMap(plan)
+        newRelation -> plan.output.zip(newRelation.output)
+      } else {
+        val attrMapping = new mutable.ArrayBuffer[(Attribute, Attribute)]()
+        val newPlan = plan.mapChildren { child =>
+          // If not, we'd rewrite child plan recursively until we find the
+          // conflict node or reach the leaf node.
+          val (newChild, childAttrMapping) = rewritePlan(child, conflictPlanMap)
+          attrMapping ++= childAttrMapping.filter { case (oldAttr, _) =>
+            // `attrMapping` is not only used to replace the attributes of the current `plan`,
+            // but also to be propagated to the parent plans of the current `plan`. Therefore,
+            // the `oldAttr` must be part of either `plan.references` (so that it can be used to
+            // replace attributes of the current `plan`) or `plan.outputSet` (so that it can be
+            // used by those parent plans).
+            (plan.outputSet ++ plan.references).contains(oldAttr)
+          }
+          newChild
+        }
+
+        if (attrMapping.isEmpty) {
+          newPlan -> attrMapping
+        } else {
+          assert(!attrMapping.groupBy(_._1.exprId)
+            .exists(_._2.map(_._2.exprId).distinct.length > 1),
+            "Found duplicate rewrite attributes")
+          val attributeRewrites = AttributeMap(attrMapping)
+          // Using attrMapping from the children plans to rewrite their parent node.
+          // Note that we shouldn't rewrite a node using attrMapping from its sibling nodes.
+          newPlan.transformExpressions {
             case a: Attribute =>
               dedupAttr(a, attributeRewrites)
             case s: SubqueryExpression =>
               s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attributeRewrites))
-          }
+          } -> attrMapping
         }
       }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 093f2dbd1e426..6fab47d15e442 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3467,6 +3467,28 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
             |""".stripMargin), Row(1))
     }
   }
+
+  test("SPARK-32372: ResolveReferences.dedupRight should only rewrite attributes for ancestor " +
+    "plans of the conflict plan") {
+    sql("SELECT name, avg(age) as avg_age FROM person GROUP BY name")
+      .createOrReplaceTempView("person_a")
+    sql("SELECT p1.name, p2.avg_age FROM person p1 JOIN person_a p2 ON p1.name = p2.name")
+      .createOrReplaceTempView("person_b")
+    sql("SELECT * FROM person_a UNION SELECT * FROM person_b")
+      .createOrReplaceTempView("person_c")
+    checkAnswer(
+      sql("SELECT p1.name, p2.avg_age FROM person_c p1 JOIN person_c p2 ON p1.name = p2.name"),
+      Row("jim", 20.0) :: Row("mike", 30.0) :: Nil)
+  }
+
+  test("SPARK-32280: Avoid duplicate rewrite attributes when there're multiple JOINs") {
+    sql("SELECT 1 AS id").createOrReplaceTempView("A")
+    sql("SELECT id, 'foo' AS kind FROM A").createOrReplaceTempView("B")
+    sql("SELECT l.id as id FROM B AS l LEFT SEMI JOIN B AS r ON l.kind = r.kind")
+      .createOrReplaceTempView("C")
+    checkAnswer(sql("SELECT 0 FROM ( SELECT * FROM B JOIN C USING (id)) " +
+      "JOIN ( SELECT * FROM B JOIN C USING (id)) USING (id)"), Row(0))
+  }
 }
 
 case class Foo(bar: Option[String])

From be1b28209d8b6ac1f46e8899b85f7cb691a97051 Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Fri, 24 Jul 2020 03:48:16 +0000
Subject: [PATCH 0921/1280] [SPARK-32237][SQL][3.0] Resolve hint in CTE

### What changes were proposed in this pull request?
The backport of #29062

This PR is to move `Substitution` rule before `Hints` rule in `Analyzer` to avoid hint in CTE not working.

### Why are the changes needed?
Below SQL in Spark3.0 will throw AnalysisException, but it works in Spark2.x
```sql
WITH cte AS (SELECT /*+ REPARTITION(3) */ T.id, T.data FROM $t1 T)
SELECT cte.id, cte.data FROM cte
```
```
Failed to analyze query: org.apache.spark.sql.AnalysisException: cannot resolve '`cte.id`' given input columns: [cte.data, cte.id]; line 3 pos 7;
'Project ['cte.id, 'cte.data]
+- SubqueryAlias cte
   +- Project [id#21L, data#22]
      +- SubqueryAlias T
         +- SubqueryAlias testcat.ns1.ns2.tbl
            +- RelationV2[id#21L, data#22] testcat.ns1.ns2.tbl

'Project ['cte.id, 'cte.data]
+- SubqueryAlias cte
   +- Project [id#21L, data#22]
      +- SubqueryAlias T
         +- SubqueryAlias testcat.ns1.ns2.tbl
            +- RelationV2[id#21L, data#22] testcat.ns1.ns2.tbl
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Add a unit test

Closes #29201 from LantaoJin/SPARK-32237_branch-3.0.

Lead-authored-by: LantaoJin <jinlantao@gmail.com>
Co-authored-by: Alan Jin <jinlantao@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 10 ++++----
 .../sql/catalyst/analysis/AnalysisSuite.scala | 25 ++++++++++++++++++-
 .../org/apache/spark/sql/SQLQuerySuite.scala  | 12 +++++++++
 3 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index bd5a7975299f4..89454c252af53 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -200,16 +200,16 @@ class Analyzer(
   val postHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil
 
   lazy val batches: Seq[Batch] = Seq(
-    Batch("Hints", fixedPoint,
-      new ResolveHints.ResolveJoinStrategyHints(conf),
-      new ResolveHints.ResolveCoalesceHints(conf)),
-    Batch("Simple Sanity Check", Once,
-      LookupFunctions),
     Batch("Substitution", fixedPoint,
       CTESubstitution,
       WindowsSubstitution,
       EliminateUnions,
       new SubstituteUnresolvedOrdinals(conf)),
+    Batch("Hints", fixedPoint,
+      new ResolveHints.ResolveJoinStrategyHints(conf),
+      new ResolveHints.ResolveCoalesceHints(conf)),
+    Batch("Simple Sanity Check", Once,
+      LookupFunctions),
     Batch("Resolution", fixedPoint,
       ResolveTableValuedFunctions ::
       ResolveNamespace(catalogManager) ::
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 8db2036496bfd..453a4e6aa51cf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -25,7 +25,7 @@ import org.apache.log4j.Level
 import org.scalatest.Matchers
 
 import org.apache.spark.api.python.PythonEvalType
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
@@ -879,4 +879,27 @@ class AnalysisSuite extends AnalysisTest with Matchers {
       Seq("Intersect can only be performed on tables with the compatible column types. " +
         "timestamp <> double at the second column of the second table"))
   }
+
+  test("SPARK-32237: Hint in CTE") {
+    val plan = With(
+      Project(
+        Seq(UnresolvedAttribute("cte.a")),
+        UnresolvedRelation(TableIdentifier("cte"))
+      ),
+      Seq(
+        (
+          "cte",
+          SubqueryAlias(
+            AliasIdentifier("cte"),
+            UnresolvedHint(
+              "REPARTITION",
+              Seq(Literal(3)),
+              Project(testRelation.output, testRelation)
+            )
+          )
+        )
+      )
+    )
+    assertAnalysisSuccess(plan)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 6fab47d15e442..f5dba4c1af265 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3468,6 +3468,18 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     }
   }
 
+  test("SPARK-32237: Hint in CTE") {
+    withTable("t") {
+      sql("CREATE TABLE t USING PARQUET AS SELECT 1 AS id")
+      checkAnswer(
+        sql("""
+              |WITH cte AS (SELECT /*+ REPARTITION(3) */ * FROM t)
+              |SELECT * FROM cte
+            """.stripMargin),
+        Row(1) :: Nil)
+    }
+  }
+
   test("SPARK-32372: ResolveReferences.dedupRight should only rewrite attributes for ancestor " +
     "plans of the conflict plan") {
     sql("SELECT name, avg(age) as avg_age FROM person GROUP BY name")

From f50432fd3d835f8b5de3283edce4c146cfafe827 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 24 Jul 2020 07:18:15 -0700
Subject: [PATCH 0922/1280] [SPARK-32363][PYTHON][BUILD][3.0] Fix flakiness in
 pip package testing in Jenkins

### What changes were proposed in this pull request?

This PR backports https://github.com/apache/spark/pull/29117 to branch-3.0 as the flakiness was found in branch-3.0 too: https://github.com/apache/spark/pull/29201#issuecomment-663114741 and https://github.com/apache/spark/pull/29201#issuecomment-663114741

This PR proposes:

- ~~Don't use `--user` in pip packaging test~~
- ~~Pull `source` out of the subshell, and place it first.~~
- Exclude user sitepackages in Python path during pip installation test

to address the flakiness of the pip packaging test in Jenkins.

~~(I think) #29116 caused this flakiness given my observation in the Jenkins log. I had to work around by specifying `--user` but it turned out that it does not properly work in old Conda on Jenkins for some reasons. Therefore, reverting this change back.~~

(I think) the installation at user site-packages affects other environments created by Conda in the old Conda version that Jenkins has. Seems it fails to isolate the environments for some reasons. So, it excludes user sitepackages in the Python path during the test.

~~In addition, #29116 also added some fallback logics of `conda (de)activate` and `source (de)activate` because Conda prefers to use `conda (de)activate` now per the official documentation and `source (de)activate` doesn't work for some reasons in certain environments (see also https://github.com/conda/conda/issues/7980). The problem was that `source` loads things to the current shell so does not affect the current shell. Therefore, this PR pulls `source` out of the subshell.~~

Disclaimer: I made the analysis purely based on Jenkins machine's log in this PR. It may have a different reason I missed during my observation.

### Why are the changes needed?

To make the build and tests pass in Jenkins.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Jenkins tests should test it out.

Closes #29215 from HyukjinKwon/SPARK-32363-3.0.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/run-pip-tests | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 470f21e69d46a..81e33a6f961bd 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -68,6 +68,10 @@ PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall "
 PIP_COMMANDS=("pip install $PIP_OPTIONS $PYSPARK_DIST"
 	      "pip install $PIP_OPTIONS -e python/")
 
+# Jenkins has PySpark installed under user sitepackages shared for some reasons.
+# In this test, explicitly exclude user sitepackages to prevent side effects
+export PYTHONNOUSERSITE=1
+
 for python in "${PYTHON_EXECS[@]}"; do
   for install_command in "${PIP_COMMANDS[@]}"; do
     echo "Testing pip installation with python $python"

From 8a52bda65c8eaaae52e1205d97d16c293ee3d363 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Fri, 24 Jul 2020 09:55:20 -0700
Subject: [PATCH 0923/1280] [SPARK-32310][ML][PYSPARK][3.0] ML params default
 value parity

### What changes were proposed in this pull request?
backporting the changes to 3.0
set params default values in trait Params for feature and tuning in both Scala and Python.

### Why are the changes needed?
Make ML has the same default param values between estimator and its corresponding transformer, and also between Scala and Python.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Existing and modified tests

Closes #29159 from huaxingao/set_default_3.0.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
---
 .../ml/classification/FMClassifier.scala      |  10 --
 .../spark/ml/classification/LinearSVC.scala   |  11 +-
 .../classification/LogisticRegression.scala   |  13 +--
 .../spark/ml/classification/NaiveBayes.scala  |   4 +-
 .../spark/ml/clustering/BisectingKMeans.scala |   7 +-
 .../spark/ml/clustering/GaussianMixture.scala |   7 +-
 .../apache/spark/ml/clustering/KMeans.scala   |  11 +-
 .../org/apache/spark/ml/clustering/LDA.scala  |  11 +-
 .../clustering/PowerIterationClustering.scala |   7 +-
 .../BinaryClassificationEvaluator.scala       |   4 +-
 .../MulticlassClassificationEvaluator.scala   |   8 +-
 .../MultilabelClassificationEvaluator.scala   |   6 +-
 .../ml/evaluation/RankingEvaluator.scala      |   6 +-
 .../ml/evaluation/RegressionEvaluator.scala   |   4 +-
 .../spark/ml/feature/ChiSqSelector.scala      |   9 +-
 .../org/apache/spark/ml/feature/Imputer.scala |   4 +-
 .../spark/ml/feature/MinMaxScaler.scala       |   4 +-
 .../spark/ml/feature/OneHotEncoder.scala      |   5 +-
 .../ml/feature/QuantileDiscretizer.scala      |   4 +-
 .../apache/spark/ml/feature/RFormula.scala    |   6 +-
 .../spark/ml/feature/RobustScaler.scala       |   8 +-
 .../spark/ml/feature/StringIndexer.scala      |   6 +-
 .../spark/ml/feature/VectorIndexer.scala      |   6 +-
 .../spark/ml/feature/VectorSlicer.scala       |   6 +-
 .../apache/spark/ml/feature/Word2Vec.scala    |   9 +-
 .../org/apache/spark/ml/fpm/FPGrowth.scala    |   5 +-
 .../ml/regression/AFTSurvivalRegression.scala |  10 +-
 .../ml/regression/LinearRegression.scala      |  14 +--
 .../org/apache/spark/ml/tree/treeParams.scala |  16 +--
 .../spark/ml/util/DefaultReadWriteTest.scala  |   3 +
 python/pyspark/ml/classification.py           |  86 ++++++++++----
 python/pyspark/ml/clustering.py               |  43 +++++--
 python/pyspark/ml/feature.py                  | 110 +++++++++++++++---
 python/pyspark/ml/fpm.py                      |  12 +-
 python/pyspark/ml/recommendation.py           |  20 +++-
 python/pyspark/ml/regression.py               |  88 ++++++++++----
 python/pyspark/ml/tests/test_param.py         |   7 +-
 python/pyspark/ml/tuning.py                   |  16 ++-
 38 files changed, 368 insertions(+), 238 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index a4d242753a789..1511cb61b8b30 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -86,7 +86,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFactorSize(value: Int): this.type = set(factorSize, value)
-  setDefault(factorSize -> 8)
 
   /**
    * Set whether to fit intercept term.
@@ -96,7 +95,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set whether to fit linear term.
@@ -106,7 +104,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setFitLinear(value: Boolean): this.type = set(fitLinear, value)
-  setDefault(fitLinear -> true)
 
   /**
    * Set the L2 regularization parameter.
@@ -116,7 +113,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the mini-batch fraction parameter.
@@ -126,7 +122,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMiniBatchFraction(value: Double): this.type = set(miniBatchFraction, value)
-  setDefault(miniBatchFraction -> 1.0)
 
   /**
    * Set the standard deviation of initial coefficients.
@@ -136,7 +131,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setInitStd(value: Double): this.type = set(initStd, value)
-  setDefault(initStd -> 0.01)
 
   /**
    * Set the maximum number of iterations.
@@ -146,7 +140,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the initial step size for the first step (like learning rate).
@@ -156,7 +149,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setStepSize(value: Double): this.type = set(stepSize, value)
-  setDefault(stepSize -> 1.0)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -166,7 +158,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Set the solver algorithm used for optimization.
@@ -177,7 +168,6 @@ class FMClassifier @Since("3.0.0") (
    */
   @Since("3.0.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> AdamW)
 
   /**
    * Set the random seed for weight initialization.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index 905789090d625..88621b34beb13 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -54,6 +54,9 @@ private[classification] trait LinearSVCParams extends ClassifierParams with HasR
    */
   final override val threshold: DoubleParam = new DoubleParam(this, "threshold",
     "threshold in binary classification prediction applied to rawPrediction")
+
+  setDefault(regParam -> 0.0, maxIter -> 100, fitIntercept -> true, tol -> 1E-6,
+    standardization -> true, threshold -> 0.0, aggregationDepth -> 2)
 }
 
 /**
@@ -81,7 +84,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -91,7 +93,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Whether to fit an intercept term.
@@ -101,7 +102,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -112,7 +112,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -122,7 +121,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   /**
    * Set the value of param [[weightCol]].
@@ -141,7 +139,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setThreshold(value: Double): this.type = set(threshold, value)
-  setDefault(threshold -> 0.0)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -153,7 +150,6 @@ class LinearSVC @Since("2.2.0") (
    */
   @Since("2.2.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   @Since("2.2.0")
   override def copy(extra: ParamMap): LinearSVC = defaultCopy(extra)
@@ -300,7 +296,6 @@ class LinearSVCModel private[classification] (
 
   @Since("2.2.0")
   def setThreshold(value: Double): this.type = set(threshold, value)
-  setDefault(threshold, 0.0)
 
   private val margin: Vector => Double = (features) => {
     BLAS.dot(features, coefficients) + intercept
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 50c14d086957f..a61eb329977cd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -248,6 +248,10 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
       isSet(lowerBoundsOnIntercepts) || isSet(upperBoundsOnIntercepts)
   }
 
+  setDefault(regParam -> 0.0, elasticNetParam -> 0.0, maxIter -> 100, tol -> 1E-6,
+    fitIntercept -> true, family -> "auto", standardization -> true, threshold -> 0.5,
+    aggregationDepth -> 2)
+
   override protected def validateAndTransformSchema(
       schema: StructType,
       fitting: Boolean,
@@ -290,7 +294,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.2.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set the ElasticNet mixing parameter.
@@ -306,7 +309,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setElasticNetParam(value: Double): this.type = set(elasticNetParam, value)
-  setDefault(elasticNetParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -316,7 +318,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.2.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -327,7 +328,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to fit an intercept term.
@@ -337,7 +337,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.4.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Sets the value of param [[family]].
@@ -347,7 +346,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("2.1.0")
   def setFamily(value: String): this.type = set(family, value)
-  setDefault(family -> "auto")
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -361,11 +359,9 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("1.5.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   @Since("1.5.0")
   override def setThreshold(value: Double): this.type = super.setThreshold(value)
-  setDefault(threshold -> 0.5)
 
   @Since("1.5.0")
   override def getThreshold: Double = super.getThreshold
@@ -396,7 +392,6 @@ class LogisticRegression @Since("1.2.0") (
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Set the lower bounds on coefficients if fitting under bound constrained optimization.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
index 94681ae9ef796..409a39c5b8ca9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala
@@ -63,6 +63,8 @@ private[classification] trait NaiveBayesParams extends PredictorParams with HasW
 
   /** @group getParam */
   final def getModelType: String = $(modelType)
+
+  setDefault(smoothing -> 1.0, modelType -> NaiveBayes.Multinomial)
 }
 
 // scalastyle:off line.size.limit
@@ -106,7 +108,6 @@ class NaiveBayes @Since("1.5.0") (
    */
   @Since("1.5.0")
   def setSmoothing(value: Double): this.type = set(smoothing, value)
-  setDefault(smoothing -> 1.0)
 
   /**
    * Set the model type using a string (case-sensitive).
@@ -116,7 +117,6 @@ class NaiveBayes @Since("1.5.0") (
    */
   @Since("1.5.0")
   def setModelType(value: String): this.type = set(modelType, value)
-  setDefault(modelType -> Multinomial)
 
   /**
    * Sets the value of param [[weightCol]].
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index 6c7112b80569f..b649b1dd95231 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -71,6 +71,8 @@ private[clustering] trait BisectingKMeansParams extends Params with HasMaxIter
   @Since("2.0.0")
   def getMinDivisibleClusterSize: Double = $(minDivisibleClusterSize)
 
+  setDefault(k -> 4, maxIter -> 20, minDivisibleClusterSize -> 1.0)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -225,11 +227,6 @@ class BisectingKMeans @Since("2.0.0") (
     @Since("2.0.0") override val uid: String)
   extends Estimator[BisectingKMeansModel] with BisectingKMeansParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 4,
-    maxIter -> 20,
-    minDivisibleClusterSize -> 1.0)
-
   @Since("2.0.0")
   override def copy(extra: ParamMap): BisectingKMeans = defaultCopy(extra)
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index d779e602545cf..26a866bbecaeb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -59,6 +59,8 @@ private[clustering] trait GaussianMixtureParams extends Params with HasMaxIter w
   @Since("2.0.0")
   def getK: Int = $(k)
 
+  setDefault(k -> 2, maxIter -> 100, tol -> 0.01)
+
   /**
    * Validates and transforms the input schema.
    *
@@ -323,11 +325,6 @@ class GaussianMixture @Since("2.0.0") (
     @Since("2.0.0") override val uid: String)
   extends Estimator[GaussianMixtureModel] with GaussianMixtureParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 100,
-    tol -> 0.01)
-
   @Since("2.0.0")
   override def copy(extra: ParamMap): GaussianMixture = defaultCopy(extra)
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index a42c920e24987..5370318bc6adb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -86,6 +86,9 @@ private[clustering] trait KMeansParams extends Params with HasMaxIter with HasFe
   @Since("1.5.0")
   def getInitSteps: Int = $(initSteps)
 
+  setDefault(k -> 2, maxIter -> 20, initMode -> MLlibKMeans.K_MEANS_PARALLEL, initSteps -> 2,
+    tol -> 1e-4, distanceMeasure -> DistanceMeasure.EUCLIDEAN)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
@@ -270,14 +273,6 @@ class KMeans @Since("1.5.0") (
     @Since("1.5.0") override val uid: String)
   extends Estimator[KMeansModel] with KMeansParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 20,
-    initMode -> MLlibKMeans.K_MEANS_PARALLEL,
-    initSteps -> 2,
-    tol -> 1e-4,
-    distanceMeasure -> DistanceMeasure.EUCLIDEAN)
-
   @Since("1.5.0")
   override def copy(extra: ParamMap): KMeans = defaultCopy(extra)
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index 4e86b3b247ace..c1b76fb40b2f6 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -199,8 +199,6 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
     " with estimates of the topic mixture distribution for each document (often called \"theta\"" +
     " in the literature).  Returns a vector of zeros for an empty document.")
 
-  setDefault(topicDistributionCol -> "topicDistribution")
-
   /** @group getParam */
   @Since("1.6.0")
   def getTopicDistributionCol: String = $(topicDistributionCol)
@@ -315,6 +313,11 @@ private[clustering] trait LDAParams extends Params with HasFeaturesCol with HasM
   @Since("2.0.0")
   def getKeepLastCheckpoint: Boolean = $(keepLastCheckpoint)
 
+  setDefault(maxIter -> 20, k -> 10, optimizer -> "online", checkpointInterval -> 10,
+    learningOffset -> 1024, learningDecay -> 0.51, subsamplingRate -> 0.05,
+    optimizeDocConcentration -> true, keepLastCheckpoint -> true,
+    topicDistributionCol -> "topicDistribution")
+
   /**
    * Validates and transforms the input schema.
    *
@@ -863,10 +866,6 @@ class LDA @Since("1.6.0") (
   @Since("1.6.0")
   def this() = this(Identifiable.randomUID("lda"))
 
-  setDefault(maxIter -> 20, k -> 10, optimizer -> "online", checkpointInterval -> 10,
-    learningOffset -> 1024, learningDecay -> 0.51, subsamplingRate -> 0.05,
-    optimizeDocConcentration -> true, keepLastCheckpoint -> true)
-
   /**
    * The features for LDA should be a `Vector` representing the word counts in a document.
    * The vector should be of length vocabSize, with counts for each term (word).
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
index 812a426a062c1..1466b32bef530 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
@@ -91,7 +91,7 @@ private[clustering] trait PowerIterationClusteringParams extends Params with Has
   @Since("2.4.0")
   def getDstCol: String = $(dstCol)
 
-  setDefault(srcCol -> "src", dstCol -> "dst")
+  setDefault(srcCol -> "src", dstCol -> "dst", k -> 2, maxIter -> 20, initMode -> "random")
 }
 
 /**
@@ -111,11 +111,6 @@ class PowerIterationClustering private[clustering] (
     @Since("2.4.0") override val uid: String)
   extends PowerIterationClusteringParams with DefaultParamsWritable {
 
-  setDefault(
-    k -> 2,
-    maxIter -> 20,
-    initMode -> "random")
-
   @Since("2.4.0")
   def this() = this(Identifiable.randomUID("PowerIterationClustering"))
 
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
index 7733225c0c4d5..cbec87d7dd715 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala
@@ -80,8 +80,6 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
   @Since("3.0.0")
   def setNumBins(value: Int): this.type = set(numBins, value)
 
-  setDefault(numBins -> 1000)
-
   /** @group setParam */
   @Since("1.5.0")
   def setRawPredictionCol(value: String): this.type = set(rawPredictionCol, value)
@@ -94,7 +92,7 @@ class BinaryClassificationEvaluator @Since("1.4.0") (@Since("1.4.0") override va
   @Since("3.0.0")
   def setWeightCol(value: String): this.type = set(weightCol, value)
 
-  setDefault(metricName -> "areaUnderROC")
+  setDefault(metricName -> "areaUnderROC", numBins -> 1000)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
index 1d6540e970383..3bf115e72b431 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MulticlassClassificationEvaluator.scala
@@ -63,8 +63,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("1.5.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "f1")
-
   /** @group setParam */
   @Since("1.5.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -104,8 +102,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setMetricLabel(value: Double): this.type = set(metricLabel, value)
 
-  setDefault(metricLabel -> 0.0)
-
   /**
    * The beta value, which controls precision vs recall weighting,
    * used in `"weightedFMeasure"`, `"fMeasureByLabel"`.
@@ -127,8 +123,6 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setBeta(value: Double): this.type = set(beta, value)
 
-  setDefault(beta -> 1.0)
-
   /**
    * param for eps. log-loss is undefined for p=0 or p=1, so probabilities are clipped to
    * max(eps, min(1 - eps, p)). Must be in range (0, 0.5). The default value is 1e-15.
@@ -149,7 +143,7 @@ class MulticlassClassificationEvaluator @Since("1.5.0") (@Since("1.5.0") overrid
   @Since("3.0.0")
   def setEps(value: Double): this.type = set(eps, value)
 
-  setDefault(eps -> 1e-15)
+  setDefault(metricName -> "f1", eps -> 1e-15, metricLabel -> 0.0, beta -> 1.0)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
index a8db5452bd56c..632e67c59696c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala
@@ -65,8 +65,6 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
   @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "f1Measure")
-
   /**
    * param for the class whose metric will be computed in `"precisionByLabel"`, `"recallByLabel"`,
    * `"f1MeasureByLabel"`.
@@ -86,8 +84,6 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
   /** @group setParam */
   def setMetricLabel(value: Double): this.type = set(metricLabel, value)
 
-  setDefault(metricLabel -> 0.0)
-
   /** @group setParam */
   @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -96,6 +92,8 @@ class MultilabelClassificationEvaluator @Since("3.0.0") (@Since("3.0.0") overrid
   @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
+  setDefault(metricLabel -> 0.0, metricName -> "f1Measure")
+
   @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
     val schema = dataset.schema
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
index c5dea6c177e21..3a2ddf4d97ee3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RankingEvaluator.scala
@@ -61,8 +61,6 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
   @Since("3.0.0")
   def setMetricName(value: String): this.type = set(metricName, value)
 
-  setDefault(metricName -> "meanAveragePrecision")
-
   /**
    * param for ranking position value used in `"meanAveragePrecisionAtK"`, `"precisionAtK"`,
    * `"ndcgAtK"`, `"recallAtK"`. Must be &gt; 0. The default value is 10.
@@ -83,8 +81,6 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
   @Since("3.0.0")
   def setK(value: Int): this.type = set(k, value)
 
-  setDefault(k -> 10)
-
   /** @group setParam */
   @Since("3.0.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -93,6 +89,8 @@ class RankingEvaluator @Since("3.0.0") (@Since("3.0.0") override val uid: String
   @Since("3.0.0")
   def setLabelCol(value: String): this.type = set(labelCol, value)
 
+  setDefault(k -> 10, metricName -> "meanAveragePrecision")
+
   @Since("3.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
     val schema = dataset.schema
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
index 18a8dda0c76ef..de256600982e5 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/RegressionEvaluator.scala
@@ -79,8 +79,6 @@ final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val ui
   @Since("3.0.0")
   def setThroughOrigin(value: Boolean): this.type = set(throughOrigin, value)
 
-  setDefault(throughOrigin -> false)
-
   /** @group setParam */
   @Since("1.4.0")
   def setPredictionCol(value: String): this.type = set(predictionCol, value)
@@ -93,7 +91,7 @@ final class RegressionEvaluator @Since("1.4.0") (@Since("1.4.0") override val ui
   @Since("3.0.0")
   def setWeightCol(value: String): this.type = set(weightCol, value)
 
-  setDefault(metricName -> "rmse")
+  setDefault(metricName -> "rmse", throughOrigin -> false)
 
   @Since("2.0.0")
   override def evaluate(dataset: Dataset[_]): Double = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
index 76f4f944f11d5..c582d92ec538b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/ChiSqSelector.scala
@@ -54,7 +54,6 @@ private[feature] trait ChiSqSelectorParams extends Params
     "Number of features that selector will select, ordered by ascending p-value. If the" +
       " number of features is < numTopFeatures, then this will select all features.",
     ParamValidators.gtEq(1))
-  setDefault(numTopFeatures -> 50)
 
   /** @group getParam */
   @Since("1.6.0")
@@ -70,7 +69,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   final val percentile = new DoubleParam(this, "percentile",
     "Percentile of features that selector will select, ordered by ascending p-value.",
     ParamValidators.inRange(0, 1))
-  setDefault(percentile -> 0.1)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -85,7 +83,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   @Since("2.1.0")
   final val fpr = new DoubleParam(this, "fpr", "The highest p-value for features to be kept.",
     ParamValidators.inRange(0, 1))
-  setDefault(fpr -> 0.05)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -100,7 +97,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   @Since("2.2.0")
   final val fdr = new DoubleParam(this, "fdr",
     "The upper bound of the expected false discovery rate.", ParamValidators.inRange(0, 1))
-  setDefault(fdr -> 0.05)
 
   /** @group getParam */
   def getFdr: Double = $(fdr)
@@ -114,7 +110,6 @@ private[feature] trait ChiSqSelectorParams extends Params
   @Since("2.2.0")
   final val fwe = new DoubleParam(this, "fwe",
     "The upper bound of the expected family-wise error rate.", ParamValidators.inRange(0, 1))
-  setDefault(fwe -> 0.05)
 
   /** @group getParam */
   def getFwe: Double = $(fwe)
@@ -129,11 +124,13 @@ private[feature] trait ChiSqSelectorParams extends Params
     "The selector type of the ChisqSelector. " +
       "Supported options: " + OldChiSqSelector.supportedSelectorTypes.mkString(", "),
     ParamValidators.inArray[String](OldChiSqSelector.supportedSelectorTypes))
-  setDefault(selectorType -> OldChiSqSelector.NumTopFeatures)
 
   /** @group getParam */
   @Since("2.1.0")
   def getSelectorType: String = $(selectorType)
+
+  setDefault(numTopFeatures -> 50, percentile -> 0.1, fpr -> 0.05, fdr -> 0.05, fwe -> 0.05,
+    selectorType -> OldChiSqSelector.NumTopFeatures)
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
index 64f1722f5fcb8..ad1010da5c104 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Imputer.scala
@@ -64,6 +64,8 @@ private[feature] trait ImputerParams extends Params with HasInputCol with HasInp
   /** @group getParam */
   def getMissingValue: Double = $(missingValue)
 
+  setDefault(strategy -> Imputer.mean, missingValue -> Double.NaN)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     if (isSet(inputCol)) {
@@ -144,8 +146,6 @@ class Imputer @Since("2.2.0") (@Since("2.2.0") override val uid: String)
   @Since("3.0.0")
   def setRelativeError(value: Double): this.type = set(relativeError, value)
 
-  setDefault(strategy -> Imputer.mean, missingValue -> Double.NaN)
-
   override def fit(dataset: Dataset[_]): ImputerModel = {
     transformSchema(dataset.schema, logging = true)
     val spark = dataset.sparkSession
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
index c84892c974b90..56798a8e61a61 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
@@ -58,6 +58,8 @@ private[feature] trait MinMaxScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getMax: Double = $(max)
 
+  setDefault(min -> 0.0, max -> 1.0)
+
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType): StructType = {
     require($(min) < $(max), s"The specified min(${$(min)}) is larger or equal to max(${$(max)})")
@@ -93,8 +95,6 @@ class MinMaxScaler @Since("1.5.0") (@Since("1.5.0") override val uid: String)
   @Since("1.5.0")
   def this() = this(Identifiable.randomUID("minMaxScal"))
 
-  setDefault(min -> 0.0, max -> 1.0)
-
   /** @group setParam */
   @Since("1.5.0")
   def setInputCol(value: String): this.type = set(inputCol, value)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
index 0ef092f6be463..5a500fefb57ec 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoder.scala
@@ -53,8 +53,6 @@ private[ml] trait OneHotEncoderBase extends Params with HasHandleInvalid
     "during fitting, invalid data will result in an error.",
     ParamValidators.inArray(OneHotEncoder.supportedHandleInvalids))
 
-  setDefault(handleInvalid, OneHotEncoder.ERROR_INVALID)
-
   /**
    * Whether to drop the last category in the encoded vector (default: true)
    * @group param
@@ -62,12 +60,13 @@ private[ml] trait OneHotEncoderBase extends Params with HasHandleInvalid
   @Since("2.3.0")
   final val dropLast: BooleanParam =
     new BooleanParam(this, "dropLast", "whether to drop the last category")
-  setDefault(dropLast -> true)
 
   /** @group getParam */
   @Since("2.3.0")
   def getDropLast: Boolean = $(dropLast)
 
+  setDefault(handleInvalid -> OneHotEncoder.ERROR_INVALID, dropLast -> true)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     if (isSet(inputCol)) {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
index 4eedfc4dc0efa..b93c9b1fcd204 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/QuantileDiscretizer.scala
@@ -46,7 +46,6 @@ private[feature] trait QuantileDiscretizerBase extends Params
   val numBuckets = new IntParam(this, "numBuckets", "Number of buckets (quantiles, or " +
     "categories) into which data points are grouped. Must be >= 2.",
     ParamValidators.gtEq(2))
-  setDefault(numBuckets -> 2)
 
   /** @group getParam */
   def getNumBuckets: Int = getOrDefault(numBuckets)
@@ -82,7 +81,8 @@ private[feature] trait QuantileDiscretizerBase extends Params
     "how to handle invalid entries. Options are skip (filter out rows with invalid values), " +
     "error (throw an error), or keep (keep invalid values in a special additional bucket).",
     ParamValidators.inArray(Bucketizer.supportedHandleInvalids))
-  setDefault(handleInvalid, Bucketizer.ERROR_INVALID)
+
+  setDefault(handleInvalid -> Bucketizer.ERROR_INVALID, numBuckets -> 2)
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index 7ccfafa4ac813..b8da020017f12 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -60,7 +60,6 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
   @Since("2.1.0")
   val forceIndexLabel: BooleanParam = new BooleanParam(this, "forceIndexLabel",
     "Force to index label whether it is numeric or string")
-  setDefault(forceIndexLabel -> false)
 
   /** @group getParam */
   @Since("2.1.0")
@@ -80,7 +79,6 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
     "type. Options are 'skip' (filter out rows with invalid data), error (throw an error), " +
     "or 'keep' (put invalid data in a special additional bucket, at index numLabels).",
     ParamValidators.inArray(StringIndexer.supportedHandleInvalids))
-  setDefault(handleInvalid, StringIndexer.ERROR_INVALID)
 
   /**
    * Param for how to order categories of a string FEATURE column used by `StringIndexer`.
@@ -113,12 +111,14 @@ private[feature] trait RFormulaBase extends HasFeaturesCol with HasLabelCol with
     "The default value is 'frequencyDesc'. When the ordering is set to 'alphabetDesc', " +
     "RFormula drops the same category as R when encoding strings.",
     ParamValidators.inArray(StringIndexer.supportedStringOrderType))
-  setDefault(stringIndexerOrderType, StringIndexer.frequencyDesc)
 
   /** @group getParam */
   @Since("2.3.0")
   def getStringIndexerOrderType: String = $(stringIndexerOrderType)
 
+  setDefault(forceIndexLabel -> false, handleInvalid -> StringIndexer.ERROR_INVALID,
+    stringIndexerOrderType -> StringIndexer.frequencyDesc)
+
   protected def hasLabelCol(schema: StructType): Boolean = {
     schema.map(_.name).contains($(labelCol))
   }
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
index bd9be779fedbd..12aba69528891 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
@@ -50,8 +50,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getLower: Double = $(lower)
 
-  setDefault(lower -> 0.25)
-
   /**
    * Upper quantile to calculate quantile range, shared by all features
    * Default: 0.75
@@ -64,8 +62,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getUpper: Double = $(upper)
 
-  setDefault(upper -> 0.75)
-
   /**
    * Whether to center the data with median before scaling.
    * It will build a dense output, so take care when applying to sparse input.
@@ -78,8 +74,6 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getWithCentering: Boolean = $(withCentering)
 
-  setDefault(withCentering -> false)
-
   /**
    * Whether to scale the data to quantile range.
    * Default: true
@@ -91,7 +85,7 @@ private[feature] trait RobustScalerParams extends Params with HasInputCol with H
   /** @group getParam */
   def getWithScaling: Boolean = $(withScaling)
 
-  setDefault(withScaling -> true)
+  setDefault(withScaling -> true, lower -> 0.25, upper -> 0.75, withCentering -> false)
 
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType): StructType = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index be32f44287b6a..77e3d902e0456 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -56,8 +56,6 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
     "or 'keep' (put invalid data in a special additional bucket, at index numLabels).",
     ParamValidators.inArray(StringIndexer.supportedHandleInvalids))
 
-  setDefault(handleInvalid, StringIndexer.ERROR_INVALID)
-
   /**
    * Param for how to order labels of string column. The first label after ordering is assigned
    * an index of 0.
@@ -84,6 +82,9 @@ private[feature] trait StringIndexerBase extends Params with HasHandleInvalid wi
   @Since("2.3.0")
   def getStringOrderType: String = $(stringOrderType)
 
+  setDefault(handleInvalid -> StringIndexer.ERROR_INVALID,
+    stringOrderType -> StringIndexer.frequencyDesc)
+
   /** Returns the input and output column names corresponding in pair. */
   private[feature] def getInOutCols(): (Array[String], Array[String]) = {
     ParamValidators.checkSingleVsMultiColumnParams(this, Seq(outputCol), Seq(outputCols))
@@ -155,7 +156,6 @@ class StringIndexer @Since("1.4.0") (
   /** @group setParam */
   @Since("2.3.0")
   def setStringOrderType(value: String): this.type = set(stringOrderType, value)
-  setDefault(stringOrderType, StringIndexer.frequencyDesc)
 
   /** @group setParam */
   @Since("1.4.0")
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
index 866074fb1453e..b7cf4392cd177 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala
@@ -60,8 +60,6 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
     "number of categories of the feature).",
     ParamValidators.inArray(VectorIndexer.supportedHandleInvalids))
 
-  setDefault(handleInvalid, VectorIndexer.ERROR_INVALID)
-
   /**
    * Threshold for the number of values a categorical feature can take.
    * If a feature is found to have {@literal >} maxCategories values, then it is declared
@@ -75,10 +73,10 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
       " If a feature is found to have > maxCategories values, then it is declared continuous.",
     ParamValidators.gtEq(2))
 
-  setDefault(maxCategories -> 20)
-
   /** @group getParam */
   def getMaxCategories: Int = $(maxCategories)
+
+  setDefault(maxCategories -> 20, handleInvalid -> VectorIndexer.ERROR_INVALID)
 }
 
 /**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
index be91844ba39e6..3590b9118f3b8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorSlicer.scala
@@ -57,8 +57,6 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
     "An array of indices to select features from a vector column." +
       " There can be no overlap with names.", VectorSlicer.validIndices)
 
-  setDefault(indices -> Array.emptyIntArray)
-
   /** @group getParam */
   @Since("1.5.0")
   def getIndices: Array[Int] = $(indices)
@@ -79,8 +77,6 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
     "An array of feature names to select features from a vector column." +
       " There can be no overlap with indices.", VectorSlicer.validNames)
 
-  setDefault(names -> Array.empty[String])
-
   /** @group getParam */
   @Since("1.5.0")
   def getNames: Array[String] = $(names)
@@ -97,6 +93,8 @@ final class VectorSlicer @Since("1.5.0") (@Since("1.5.0") override val uid: Stri
   @Since("1.5.0")
   def setOutputCol(value: String): this.type = set(outputCol, value)
 
+  setDefault(indices -> Array.emptyIntArray, names -> Array.empty[String])
+
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     // Validity checks
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
index bbfcbfbe038ef..287fce4243743 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala
@@ -47,7 +47,6 @@ private[feature] trait Word2VecBase extends Params
   final val vectorSize = new IntParam(
     this, "vectorSize", "the dimension of codes after transforming from words (> 0)",
     ParamValidators.gt(0))
-  setDefault(vectorSize -> 100)
 
   /** @group getParam */
   def getVectorSize: Int = $(vectorSize)
@@ -60,7 +59,6 @@ private[feature] trait Word2VecBase extends Params
   final val windowSize = new IntParam(
     this, "windowSize", "the window size (context words from [-window, window]) (> 0)",
     ParamValidators.gt(0))
-  setDefault(windowSize -> 5)
 
   /** @group expertGetParam */
   def getWindowSize: Int = $(windowSize)
@@ -73,7 +71,6 @@ private[feature] trait Word2VecBase extends Params
   final val numPartitions = new IntParam(
     this, "numPartitions", "number of partitions for sentences of words (> 0)",
     ParamValidators.gt(0))
-  setDefault(numPartitions -> 1)
 
   /** @group getParam */
   def getNumPartitions: Int = $(numPartitions)
@@ -86,7 +83,6 @@ private[feature] trait Word2VecBase extends Params
    */
   final val minCount = new IntParam(this, "minCount", "the minimum number of times a token must " +
     "appear to be included in the word2vec model's vocabulary (>= 0)", ParamValidators.gtEq(0))
-  setDefault(minCount -> 5)
 
   /** @group getParam */
   def getMinCount: Int = $(minCount)
@@ -101,13 +97,12 @@ private[feature] trait Word2VecBase extends Params
   final val maxSentenceLength = new IntParam(this, "maxSentenceLength", "Maximum length " +
     "(in words) of each sentence in the input data. Any sentence longer than this threshold will " +
     "be divided into chunks up to the size (> 0)", ParamValidators.gt(0))
-  setDefault(maxSentenceLength -> 1000)
 
   /** @group getParam */
   def getMaxSentenceLength: Int = $(maxSentenceLength)
 
-  setDefault(stepSize -> 0.025)
-  setDefault(maxIter -> 1)
+  setDefault(vectorSize -> 100, windowSize -> 5, numPartitions -> 1, minCount -> 5,
+    maxSentenceLength -> 1000, stepSize -> 0.025, maxIter -> 1)
 
   /**
    * Validate and transform the input schema.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index e50d4255b1f37..de9d57e400551 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -50,7 +50,6 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
    */
   @Since("2.2.0")
   val itemsCol: Param[String] = new Param[String](this, "itemsCol", "items column name")
-  setDefault(itemsCol -> "items")
 
   /** @group getParam */
   @Since("2.2.0")
@@ -66,7 +65,6 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
   val minSupport: DoubleParam = new DoubleParam(this, "minSupport",
     "the minimal support level of a frequent pattern",
     ParamValidators.inRange(0.0, 1.0))
-  setDefault(minSupport -> 0.3)
 
   /** @group getParam */
   @Since("2.2.0")
@@ -95,12 +93,13 @@ private[fpm] trait FPGrowthParams extends Params with HasPredictionCol {
   val minConfidence: DoubleParam = new DoubleParam(this, "minConfidence",
     "minimal confidence for generating Association Rule",
     ParamValidators.inRange(0.0, 1.0))
-  setDefault(minConfidence -> 0.8)
 
   /** @group getParam */
   @Since("2.2.0")
   def getMinConfidence: Double = $(minConfidence)
 
+  setDefault(minSupport -> 0.3, itemsCol -> "items", minConfidence -> 0.8)
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 2da65a0bcf790..86989c54bde3c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -59,7 +59,6 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   /** @group getParam */
   @Since("1.6.0")
   def getCensorCol: String = $(censorCol)
-  setDefault(censorCol -> "censor")
 
   /**
    * Param for quantile probabilities array.
@@ -75,7 +74,6 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
   /** @group getParam */
   @Since("1.6.0")
   def getQuantileProbabilities: Array[Double] = $(quantileProbabilities)
-  setDefault(quantileProbabilities -> Array(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99))
 
   /**
    * Param for quantiles column name.
@@ -94,6 +92,10 @@ private[regression] trait AFTSurvivalRegressionParams extends PredictorParams
     isDefined(quantilesCol) && $(quantilesCol).nonEmpty
   }
 
+  setDefault(censorCol -> "censor",
+    quantileProbabilities -> Array(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99),
+    fitIntercept -> true, maxIter -> 100, tol -> 1E-6, aggregationDepth -> 2)
+
   /**
    * Validates and transforms the input schema with the provided param map.
    * @param schema input schema
@@ -150,7 +152,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Set the maximum number of iterations.
@@ -159,7 +160,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -169,7 +169,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("1.6.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -180,7 +179,6 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Extract [[featuresCol]], [[labelCol]] and [[censorCol]] from input dataset,
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index 64e5e191ffd17..354c583ecc9ae 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -104,6 +104,10 @@ private[regression] trait LinearRegressionParams extends PredictorParams
   @Since("2.3.0")
   def getEpsilon: Double = $(epsilon)
 
+  setDefault(regParam -> 0.0, fitIntercept -> true, standardization -> true,
+    elasticNetParam -> 0.0, maxIter -> 100, tol -> 1E-6, solver -> Auto,
+    aggregationDepth -> 2, loss -> SquaredError, epsilon -> 1.35)
+
   override protected def validateAndTransformSchema(
       schema: StructType,
       fitting: Boolean,
@@ -190,7 +194,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.3.0")
   def setRegParam(value: Double): this.type = set(regParam, value)
-  setDefault(regParam -> 0.0)
 
   /**
    * Set if we should fit the intercept.
@@ -200,7 +203,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.5.0")
   def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
-  setDefault(fitIntercept -> true)
 
   /**
    * Whether to standardize the training features before fitting the model.
@@ -216,7 +218,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.5.0")
   def setStandardization(value: Boolean): this.type = set(standardization, value)
-  setDefault(standardization -> true)
 
   /**
    * Set the ElasticNet mixing parameter.
@@ -232,7 +233,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.4.0")
   def setElasticNetParam(value: Double): this.type = set(elasticNetParam, value)
-  setDefault(elasticNetParam -> 0.0)
 
   /**
    * Set the maximum number of iterations.
@@ -242,7 +242,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.3.0")
   def setMaxIter(value: Int): this.type = set(maxIter, value)
-  setDefault(maxIter -> 100)
 
   /**
    * Set the convergence tolerance of iterations.
@@ -253,7 +252,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.4.0")
   def setTol(value: Double): this.type = set(tol, value)
-  setDefault(tol -> 1E-6)
 
   /**
    * Whether to over-/under-sample training instances according to the given weights in weightCol.
@@ -282,7 +280,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("1.6.0")
   def setSolver(value: String): this.type = set(solver, value)
-  setDefault(solver -> Auto)
 
   /**
    * Suggested depth for treeAggregate (greater than or equal to 2).
@@ -294,7 +291,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.1.0")
   def setAggregationDepth(value: Int): this.type = set(aggregationDepth, value)
-  setDefault(aggregationDepth -> 2)
 
   /**
    * Sets the value of param [[loss]].
@@ -304,7 +300,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.3.0")
   def setLoss(value: String): this.type = set(loss, value)
-  setDefault(loss -> SquaredError)
 
   /**
    * Sets the value of param [[epsilon]].
@@ -314,7 +309,6 @@ class LinearRegression @Since("1.3.0") (@Since("1.3.0") override val uid: String
    */
   @Since("2.3.0")
   def setEpsilon(value: Double): this.type = set(epsilon, value)
-  setDefault(epsilon -> 1.35)
 
   override protected def train(dataset: Dataset[_]): LinearRegressionModel = instrumented { instr =>
     // Extract the number of features before deciding optimization solver.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
index 7e2c287f146fb..19ea8ae4775d8 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
@@ -333,8 +333,6 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
     "Fraction of the training data used for learning each decision tree, in range (0, 1].",
     ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
-  setDefault(subsamplingRate -> 1.0)
-
   /** @group getParam */
   final def getSubsamplingRate: Double = $(subsamplingRate)
 
@@ -386,10 +384,10 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
       || Try(value.toInt).filter(_ > 0).isSuccess
       || Try(value.toDouble).filter(_ > 0).filter(_ <= 1.0).isSuccess)
 
-  setDefault(featureSubsetStrategy -> "auto")
-
   /** @group getParam */
   final def getFeatureSubsetStrategy: String = $(featureSubsetStrategy).toLowerCase(Locale.ROOT)
+
+  setDefault(subsamplingRate -> 1.0, featureSubsetStrategy -> "auto")
 }
 
 /**
@@ -448,8 +446,6 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
     new IntParam(this, "numTrees", "Number of trees to train (at least 1)",
     ParamValidators.gtEq(1))
 
-  setDefault(numTrees -> 20)
-
   /** @group getParam */
   final def getNumTrees: Int = $(numTrees)
 
@@ -461,11 +457,11 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
   final val bootstrap: BooleanParam = new BooleanParam(this, "bootstrap",
     "Whether bootstrap samples are used when building trees.")
 
-  setDefault(bootstrap -> true)
-
   /** @group getParam */
   @Since("3.0.0")
   final def getBootstrap: Boolean = $(bootstrap)
+
+  setDefault(numTrees -> 20, bootstrap -> true)
 }
 
 private[ml] trait RandomForestClassifierParams
@@ -518,9 +514,7 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter with HasS
     "(a.k.a. learning rate) in interval (0, 1] for shrinking the contribution of each estimator.",
     ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
-  setDefault(maxIter -> 20, stepSize -> 0.1, validationTol -> 0.01)
-
-  setDefault(featureSubsetStrategy -> "all")
+  setDefault(maxIter -> 20, stepSize -> 0.1, validationTol -> 0.01, featureSubsetStrategy -> "all")
 
   /** (private[ml]) Create a BoostingStrategy instance to use with the old API. */
   private[ml] def getOldBoostingStrategy(
diff --git a/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala b/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
index 4d9e664850c12..dd0139b94f098 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/util/DefaultReadWriteTest.scala
@@ -63,6 +63,9 @@ trait DefaultReadWriteTest extends TempDirectory { self: Suite =>
           (instance.getOrDefault(p), newInstance.getOrDefault(p)) match {
             case (Array(values), Array(newValues)) =>
               assert(values === newValues, s"Values do not match on param ${p.name}.")
+            case (value: Double, newValue: Double) =>
+              assert(value.isNaN && newValue.isNaN || value == newValue,
+                s"Values do not match on param ${p.name}.")
             case (value, newValue) =>
               assert(value === newValue, s"Values do not match on param ${p.name}.")
           }
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 369761e8f6fa7..c8e15ca1568cc 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -178,6 +178,11 @@ class _LinearSVCParams(_JavaClassifierParams, HasRegParam, HasMaxIter, HasFitInt
                       " all predictions 0.0 and -Inf will make all predictions 1.0.",
                       typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_LinearSVCParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
+                         standardization=True, threshold=0.0, aggregationDepth=2)
+
 
 @inherit_doc
 class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable):
@@ -244,6 +249,8 @@ class LinearSVC(JavaClassifier, _LinearSVCParams, JavaMLWritable, JavaMLReadable
     True
     >>> model.intercept == model2.intercept
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 2.2.0
     """
@@ -262,8 +269,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(LinearSVC, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LinearSVC", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, fitIntercept=True,
-                         standardization=True, threshold=0.0, aggregationDepth=2)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -427,6 +432,10 @@ class _LogisticRegressionParams(_JavaProbabilisticClassifierParams, HasRegParam,
                                     "classes for multinomial regression.",
                                     typeConverter=TypeConverters.toVector)
 
+    def __init__(self, *args):
+        super(_LogisticRegressionParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto")
+
     @since("1.4.0")
     def setThreshold(self, value):
         """
@@ -616,6 +625,8 @@ class LogisticRegression(JavaProbabilisticClassifier, _LogisticRegressionParams,
     True
     >>> model2
     LogisticRegressionModel: uid=..., numClasses=2, numFeatures=2
+    >>> blorModel.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.3.0
     """
@@ -642,7 +653,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(LogisticRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.LogisticRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1E-6, threshold=0.5, family="auto")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
         self._checkThresholdConsistency()
@@ -1130,7 +1140,12 @@ class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
     """
     Params for :py:class:`DecisionTreeClassifier` and :py:class:`DecisionTreeClassificationModel`.
     """
-    pass
+
+    def __init__(self, *args):
+        super(_DecisionTreeClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
 
 
 @inherit_doc
@@ -1197,7 +1212,8 @@ class DecisionTreeClassifier(JavaProbabilisticClassifier, _DecisionTreeClassifie
     >>> model2 = DecisionTreeClassificationModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
-
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> df3 = spark.createDataFrame([
     ...     (1.0, 0.2, Vectors.dense(1.0)),
     ...     (1.0, 0.8, Vectors.dense(1.0)),
@@ -1229,9 +1245,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(DecisionTreeClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.DecisionTreeClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1365,7 +1378,14 @@ class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
     """
     Params for :py:class:`RandomForestClassifier` and :py:class:`RandomForestClassificationModel`.
     """
-    pass
+
+    def __init__(self, *args):
+        super(_RandomForestClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
+                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
+                         bootstrap=True)
 
 
 @inherit_doc
@@ -1437,6 +1457,8 @@ class RandomForestClassifier(JavaProbabilisticClassifier, _RandomForestClassifie
     >>> model2 = RandomForestClassificationModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -1459,11 +1481,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(RandomForestClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.RandomForestClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
-                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1633,6 +1650,14 @@ class _GBTClassifierParams(_GBTParams, _HasVarianceImpurity):
                      "Supported options: " + ", ".join(supportedLossTypes),
                      typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GBTClassifierParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         lossType="logistic", maxIter=20, stepSize=0.1, subsamplingRate=1.0,
+                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
+                         leafCol="", minWeightFractionPerNode=0.0)
+
     @since("1.4.0")
     def getLossType(self):
         """
@@ -1724,6 +1749,8 @@ class GBTClassifier(JavaProbabilisticClassifier, _GBTClassifierParams,
     True
     >>> model.treeWeights == model2.treeWeights
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.trees
     [DecisionTreeRegressionModel...depth=..., DecisionTreeRegressionModel...]
     >>> validation = spark.createDataFrame([(0.0, Vectors.dense(-1.0),)],
@@ -1760,11 +1787,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(GBTClassifier, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.GBTClassifier", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         lossType="logistic", maxIter=20, stepSize=0.1, subsamplingRate=1.0,
-                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
-                         leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1962,6 +1984,10 @@ class _NaiveBayesParams(_JavaPredictorParams, HasWeightCol):
                       "and gaussian.",
                       typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_NaiveBayesParams, self).__init__(*args)
+        self._setDefault(smoothing=1.0, modelType="multinomial")
+
     @since("1.5.0")
     def getSmoothing(self):
         """
@@ -2045,6 +2071,8 @@ class NaiveBayes(JavaProbabilisticClassifier, _NaiveBayesParams, HasThresholds,
     True
     >>> model.theta == model2.theta
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> nb = nb.setThresholds([0.01, 10.00])
     >>> model3 = nb.fit(df)
     >>> result = model3.transform(test0).head()
@@ -2080,7 +2108,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(NaiveBayes, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.classification.NaiveBayes", self.uid)
-        self._setDefault(smoothing=1.0, modelType="multinomial")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2172,8 +2199,8 @@ class _MultilayerPerceptronParams(_JavaProbabilisticClassifierParams, HasSeed, H
     initialWeights = Param(Params._dummy(), "initialWeights", "The initial weights of the model.",
                            typeConverter=TypeConverters.toVector)
 
-    def __init__(self):
-        super(_MultilayerPerceptronParams, self).__init__()
+    def __init__(self, *args):
+        super(_MultilayerPerceptronParams, self).__init__(*args)
         self._setDefault(maxIter=100, tol=1E-6, blockSize=128, stepSize=0.03, solver="l-bfgs")
 
     @since("1.6.0")
@@ -2255,6 +2282,8 @@ class MultilayerPerceptronClassifier(JavaProbabilisticClassifier, _MultilayerPer
     True
     >>> model.weights == model2.weights
     True
+    >>> model.transform(testDF).take(1) == model2.transform(testDF).take(1)
+    True
     >>> mlp2 = mlp2.setInitialWeights(list(range(0, 12)))
     >>> model3 = mlp2.fit(df)
     >>> model3.weights != model2.weights
@@ -2429,6 +2458,8 @@ class OneVsRest(Estimator, _OneVsRestParams, HasParallelism, JavaMLReadable, Jav
     >>> model2 = OneVsRestModel.load(model_path)
     >>> model2.transform(test0).head().newPrediction
     0.0
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.transform(test2).columns
     ['features', 'rawPrediction', 'newPrediction']
 
@@ -2854,6 +2885,17 @@ class FMClassifier(JavaProbabilisticClassifier, _FactorizationMachinesParams, Ja
     DenseVector([14.8232])
     >>> model.factors
     DenseMatrix(1, 2, [0.0163, -0.0051], 1)
+    >>> model_path = temp_path + "/fm_model"
+    >>> model.save(model_path)
+    >>> model2 = FMClassificationModel.load(model_path)
+    >>> model2.intercept
+    -7.316665276826291
+    >>> model2.linear
+    DenseVector([14.8232])
+    >>> model2.factors
+    DenseMatrix(1, 2, [0.0163, -0.0051], 1)
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 3.0.0
     """
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index e1c2732af31b9..2060bbc625a02 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -108,6 +108,10 @@ class _GaussianMixtureParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionC
     k = Param(Params._dummy(), "k", "Number of independent Gaussians in the mixture model. " +
               "Must be > 1.", typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_GaussianMixtureParams, self).__init__(*args)
+        self._setDefault(k=2, tol=0.01, maxIter=100, aggregationDepth=2)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -321,6 +325,8 @@ class GaussianMixture(JavaEstimator, _GaussianMixtureParams, JavaMLWritable, Jav
     Row(mean=DenseVector([0.825, 0.8675]))
     >>> model2.gaussiansDF.select("cov").head()
     Row(cov=DenseMatrix(2, 2, [0.0056, -0.0051, -0.0051, 0.0046], False))
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
     >>> gm2.setWeightCol("weight")
     GaussianMixture...
 
@@ -339,7 +345,6 @@ def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
         super(GaussianMixture, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.GaussianMixture",
                                             self.uid)
-        self._setDefault(k=2, tol=0.01, maxIter=100, aggregationDepth=2)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -493,6 +498,11 @@ class _KMeansParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionCol, HasTo
     initSteps = Param(Params._dummy(), "initSteps", "The number of steps for k-means|| " +
                       "initialization mode. Must be > 0.", typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_KMeansParams, self).__init__(*args)
+        self._setDefault(k=2, initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
+                         distanceMeasure="euclidean")
+
     @since("1.5.0")
     def getK(self):
         """
@@ -622,6 +632,8 @@ class KMeans(JavaEstimator, _KMeansParams, JavaMLWritable, JavaMLReadable):
     array([ True,  True], dtype=bool)
     >>> model.clusterCenters()[1] == model2.clusterCenters()[1]
     array([ True,  True], dtype=bool)
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -637,8 +649,6 @@ def __init__(self, featuresCol="features", predictionCol="prediction", k=2,
         """
         super(KMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid)
-        self._setDefault(k=2, initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
-                         distanceMeasure="euclidean")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -747,6 +757,10 @@ class _BisectingKMeansParams(HasMaxIter, HasFeaturesCol, HasSeed, HasPredictionC
                                     "proportion of points (if < 1.0) of a divisible cluster.",
                                     typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_BisectingKMeansParams, self).__init__(*args)
+        self._setDefault(maxIter=20, k=4, minDivisibleClusterSize=1.0)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -897,6 +911,8 @@ class BisectingKMeans(JavaEstimator, _BisectingKMeansParams, JavaMLWritable, Jav
     array([ True,  True], dtype=bool)
     >>> model.clusterCenters()[1] == model2.clusterCenters()[1]
     array([ True,  True], dtype=bool)
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -913,7 +929,6 @@ def __init__(self, featuresCol="features", predictionCol="prediction", maxIter=2
         super(BisectingKMeans, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.BisectingKMeans",
                                             self.uid)
-        self._setDefault(maxIter=20, k=4, minDivisibleClusterSize=1.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1056,6 +1071,13 @@ class _LDAParams(HasMaxIter, HasFeaturesCol, HasSeed, HasCheckpointInterval):
                                " partition is lost, so set this bit with care.",
                                TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_LDAParams, self).__init__(*args)
+        self._setDefault(maxIter=20, checkpointInterval=10,
+                         k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
+                         subsamplingRate=0.05, optimizeDocConcentration=True,
+                         topicDistributionCol="topicDistribution", keepLastCheckpoint=True)
+
     @since("2.0.0")
     def getK(self):
         """
@@ -1366,6 +1388,8 @@ class LDA(JavaEstimator, _LDAParams, JavaMLReadable, JavaMLWritable):
     >>> local_model_path = temp_path + "/lda_local_model"
     >>> localModel.save(local_model_path)
     >>> sameLocalModel = LocalLDAModel.load(local_model_path)
+    >>> model.transform(df).take(1) == sameLocalModel.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -1385,10 +1409,6 @@ def __init__(self, featuresCol="features", maxIter=20, seed=None, checkpointInte
         """
         super(LDA, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.clustering.LDA", self.uid)
-        self._setDefault(maxIter=20, checkpointInterval=10,
-                         k=10, optimizer="online", learningOffset=1024.0, learningDecay=0.51,
-                         subsamplingRate=0.05, optimizeDocConcentration=True,
-                         topicDistributionCol="topicDistribution", keepLastCheckpoint=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1581,6 +1601,10 @@ class _PowerIterationClusteringParams(HasMaxIter, HasWeightCol):
                    "Name of the input column for destination vertex IDs.",
                    typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_PowerIterationClusteringParams, self).__init__(*args)
+        self._setDefault(k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst")
+
     @since("2.4.0")
     def getK(self):
         """
@@ -1654,6 +1678,8 @@ class PowerIterationClustering(_PowerIterationClusteringParams, JavaParams, Java
     2
     >>> pic2.getMaxIter()
     40
+    >>> pic2.assignClusters(df).take(6) == assignments.take(6)
+    True
 
     .. versionadded:: 2.4.0
     """
@@ -1668,7 +1694,6 @@ def __init__(self, k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst
         super(PowerIterationClustering, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.clustering.PowerIterationClustering", self.uid)
-        self._setDefault(k=2, maxIter=20, initMode="random", srcCol="src", dstCol="dst")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 6df2f74bcfc9d..4f42bb337fe48 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -94,6 +94,8 @@ class Binarizer(JavaTransformer, HasThreshold, HasThresholds, HasInputCol, HasOu
     >>> loadedBinarizer = Binarizer.load(binarizerPath)
     >>> loadedBinarizer.getThreshold() == binarizer.getThreshold()
     True
+    >>> loadedBinarizer.transform(df).take(1) == binarizer.transform(df).take(1)
+    True
     >>> df2 = spark.createDataFrame([(0.5, 0.3)], ["values1", "values2"])
     >>> binarizer2 = Binarizer(thresholds=[0.0, 1.0])
     >>> binarizer2.setInputCols(["values1", "values2"]).setOutputCols(["output1", "output2"])
@@ -197,6 +199,10 @@ class _LSHParams(HasInputCol, HasOutputCol):
                           "and decreasing it improves the running performance.",
                           typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_LSHParams, self).__init__(*args)
+        self._setDefault(numHashTables=1)
+
     def getNumHashTables(self):
         """
         Gets the value of numHashTables or its default value.
@@ -392,7 +398,6 @@ def __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1,
         super(BucketedRandomProjectionLSH, self).__init__()
         self._java_obj = \
             self._new_java_obj("org.apache.spark.ml.feature.BucketedRandomProjectionLSH", self.uid)
-        self._setDefault(numHashTables=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -479,6 +484,8 @@ class Bucketizer(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols, HasOu
     >>> loadedBucketizer = Bucketizer.load(bucketizerPath)
     >>> loadedBucketizer.getSplits() == bucketizer.getSplits()
     True
+    >>> loadedBucketizer.transform(df).take(1) == bucketizer.transform(df).take(1)
+    True
     >>> bucketed = bucketizer.setHandleInvalid("skip").transform(df).collect()
     >>> len(bucketed)
     4
@@ -735,6 +742,8 @@ class CountVectorizer(JavaEstimator, _CountVectorizerParams, JavaMLReadable, Jav
     >>> loadedModel = CountVectorizerModel.load(modelPath)
     >>> loadedModel.vocabulary == model.vocabulary
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     >>> fromVocabModel = CountVectorizerModel.from_vocabulary(["a", "b", "c"],
     ...     inputCol="raw", outputCol="vectors")
     >>> fromVocabModel.transform(df).show(truncate=False)
@@ -922,6 +931,8 @@ class DCT(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWrit
     >>> dctPath = temp_path + "/dct"
     >>> dct.save(dctPath)
     >>> loadedDtc = DCT.load(dctPath)
+    >>> loadedDtc.transform(df1).take(1) == dct.transform(df1).take(1)
+    True
     >>> loadedDtc.getInverse()
     False
 
@@ -1005,6 +1016,8 @@ class ElementwiseProduct(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReada
     >>> loadedEp = ElementwiseProduct.load(elementwiseProductPath)
     >>> loadedEp.getScalingVec() == ep.getScalingVec()
     True
+    >>> loadedEp.transform(df).take(1) == ep.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -1203,6 +1216,8 @@ class HashingTF(JavaTransformer, HasInputCol, HasOutputCol, HasNumFeatures, Java
     >>> loadedHashingTF = HashingTF.load(hashingTFPath)
     >>> loadedHashingTF.getNumFeatures() == hashingTF.getNumFeatures()
     True
+    >>> loadedHashingTF.transform(df).take(1) == hashingTF.transform(df).take(1)
+    True
     >>> hashingTF.indexOf("b")
     5
 
@@ -1287,6 +1302,10 @@ class _IDFParams(HasInputCol, HasOutputCol):
                        "minimum number of documents in which a term should appear for filtering",
                        typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_IDFParams, self).__init__(*args)
+        self._setDefault(minDocFreq=0)
+
     @since("1.4.0")
     def getMinDocFreq(self):
         """
@@ -1347,7 +1366,6 @@ def __init__(self, minDocFreq=0, inputCol=None, outputCol=None):
         """
         super(IDF, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.IDF", self.uid)
-        self._setDefault(minDocFreq=0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1447,6 +1465,10 @@ class _ImputerParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCols, Has
                          "The placeholder for the missing values. All occurrences of missingValue "
                          "will be imputed.", typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_ImputerParams, self).__init__(*args)
+        self._setDefault(strategy="mean", missingValue=float("nan"), relativeError=0.001)
+
     @since("2.2.0")
     def getStrategy(self):
         """
@@ -1584,7 +1606,6 @@ def __init__(self, strategy="mean", missingValue=float("nan"), inputCols=None,
         """
         super(Imputer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Imputer", self.uid)
-        self._setDefault(strategy="mean", missingValue=float("nan"), relativeError=0.001)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1813,6 +1834,8 @@ class MaxAbsScaler(JavaEstimator, _MaxAbsScalerParams, JavaMLReadable, JavaMLWri
     >>> loadedModel = MaxAbsScalerModel.load(modelPath)
     >>> loadedModel.maxAbs == model.maxAbs
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -1953,7 +1976,6 @@ def __init__(self, inputCol=None, outputCol=None, seed=None, numHashTables=1):
         """
         super(MinHashLSH, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinHashLSH", self.uid)
-        self._setDefault(numHashTables=1)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2004,6 +2026,10 @@ class _MinMaxScalerParams(HasInputCol, HasOutputCol):
     max = Param(Params._dummy(), "max", "Upper bound of the output feature range",
                 typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_MinMaxScalerParams, self).__init__(*args)
+        self._setDefault(min=0.0, max=1.0)
+
     @since("1.6.0")
     def getMin(self):
         """
@@ -2067,6 +2093,8 @@ class MinMaxScaler(JavaEstimator, _MinMaxScalerParams, JavaMLReadable, JavaMLWri
     True
     >>> loadedModel.originalMax == model.originalMax
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -2078,7 +2106,6 @@ def __init__(self, min=0.0, max=1.0, inputCol=None, outputCol=None):
         """
         super(MinMaxScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.MinMaxScaler", self.uid)
-        self._setDefault(min=0.0, max=1.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2212,6 +2239,8 @@ class NGram(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWr
     >>> loadedNGram = NGram.load(ngramPath)
     >>> loadedNGram.getN() == ngram.getN()
     True
+    >>> loadedNGram.transform(df).take(1) == ngram.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -2292,6 +2321,8 @@ class Normalizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, Jav
     >>> loadedNormalizer = Normalizer.load(normalizerPath)
     >>> loadedNormalizer.getP() == normalizer.getP()
     True
+    >>> loadedNormalizer.transform(df).take(1) == normalizer.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -2365,6 +2396,10 @@ class _OneHotEncoderParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCol
     dropLast = Param(Params._dummy(), "dropLast", "whether to drop the last category",
                      typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_OneHotEncoderParams, self).__init__(*args)
+        self._setDefault(handleInvalid="error", dropLast=True)
+
     @since("2.3.0")
     def getDropLast(self):
         """
@@ -2425,6 +2460,8 @@ class OneHotEncoder(JavaEstimator, _OneHotEncoderParams, JavaMLReadable, JavaMLW
     >>> loadedModel = OneHotEncoderModel.load(modelPath)
     >>> loadedModel.categorySizes == model.categorySizes
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 2.3.0
     """
@@ -2439,7 +2476,6 @@ def __init__(self, inputCols=None, outputCols=None, handleInvalid="error", dropL
         super(OneHotEncoder, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.feature.OneHotEncoder", self.uid)
-        self._setDefault(handleInvalid="error", dropLast=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -2586,6 +2622,8 @@ class PolynomialExpansion(JavaTransformer, HasInputCol, HasOutputCol, JavaMLRead
     >>> loadedPx = PolynomialExpansion.load(polyExpansionPath)
     >>> loadedPx.getDegree() == px.getDegree()
     True
+    >>> loadedPx.transform(df).take(1) == px.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -2882,6 +2920,11 @@ class _RobustScalerParams(HasInputCol, HasOutputCol, HasRelativeError):
     withScaling = Param(Params._dummy(), "withScaling", "Whether to scale the data to "
                         "quantile range", typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_RobustScalerParams, self).__init__(*args)
+        self._setDefault(lower=0.25, upper=0.75, withCentering=False, withScaling=True,
+                         relativeError=0.001)
+
     @since("3.0.0")
     def getLower(self):
         """
@@ -2957,6 +3000,8 @@ class RobustScaler(JavaEstimator, _RobustScalerParams, JavaMLReadable, JavaMLWri
     True
     >>> loadedModel.range == model.range
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 3.0.0
     """
@@ -2970,8 +3015,6 @@ def __init__(self, lower=0.25, upper=0.75, withCentering=False, withScaling=True
         """
         super(RobustScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RobustScaler", self.uid)
-        self._setDefault(lower=0.25, upper=0.75, withCentering=False, withScaling=True,
-                         relativeError=0.001)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -3117,6 +3160,8 @@ class RegexTokenizer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable,
     True
     >>> loadedReTokenizer.getGaps() == reTokenizer.getGaps()
     True
+    >>> loadedReTokenizer.transform(df).take(1) == reTokenizer.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -3241,6 +3286,8 @@ class SQLTransformer(JavaTransformer, JavaMLReadable, JavaMLWritable):
     >>> loadedSqlTrans = SQLTransformer.load(sqlTransformerPath)
     >>> loadedSqlTrans.getStatement() == sqlTrans.getStatement()
     True
+    >>> loadedSqlTrans.transform(df).take(1) == sqlTrans.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -3295,6 +3342,10 @@ class _StandardScalerParams(HasInputCol, HasOutputCol):
     withStd = Param(Params._dummy(), "withStd", "Scale to unit standard deviation",
                     typeConverter=TypeConverters.toBoolean)
 
+    def __init__(self, *args):
+        super(_StandardScalerParams, self).__init__(*args)
+        self._setDefault(withMean=False, withStd=True)
+
     @since("1.4.0")
     def getWithMean(self):
         """
@@ -3352,6 +3403,8 @@ class StandardScaler(JavaEstimator, _StandardScalerParams, JavaMLReadable, JavaM
     True
     >>> loadedModel.mean == model.mean
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -3363,7 +3416,6 @@ def __init__(self, withMean=False, withStd=True, inputCol=None, outputCol=None):
         """
         super(StandardScaler, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.StandardScaler", self.uid)
-        self._setDefault(withMean=False, withStd=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -3512,6 +3564,8 @@ class StringIndexer(JavaEstimator, _StringIndexerParams, JavaMLReadable, JavaMLW
     >>> loadedModel = StringIndexerModel.load(modelPath)
     >>> loadedModel.labels == model.labels
     True
+    >>> loadedModel.transform(stringIndDf).take(1) == model.transform(stringIndDf).take(1)
+    True
     >>> indexToStringPath = temp_path + "/index-to-string"
     >>> inverter.save(indexToStringPath)
     >>> loadedInverter = IndexToString.load(indexToStringPath)
@@ -3802,6 +3856,8 @@ class StopWordsRemover(JavaTransformer, HasInputCol, HasOutputCol, HasInputCols,
     True
     >>> loadedRemover.getCaseSensitive() == remover.getCaseSensitive()
     True
+    >>> loadedRemover.transform(df).take(1) == remover.transform(df).take(1)
+    True
     >>> df2 = spark.createDataFrame([(["a", "b", "c"], ["a", "b"])], ["text1", "text2"])
     >>> remover2 = StopWordsRemover(stopWords=["b"])
     >>> remover2.setInputCols(["text1", "text2"]).setOutputCols(["words1", "words2"])
@@ -4114,6 +4170,10 @@ class _VectorIndexerParams(HasInputCol, HasOutputCol, HasHandleInvalid):
                           "of categories of the feature).",
                           typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_VectorIndexerParams, self).__init__(*args)
+        self._setDefault(maxCategories=20, handleInvalid="error")
+
     @since("1.4.0")
     def getMaxCategories(self):
         """
@@ -4194,6 +4254,8 @@ class VectorIndexer(JavaEstimator, _VectorIndexerParams, JavaMLReadable, JavaMLW
     True
     >>> loadedModel.categoryMaps == model.categoryMaps
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
     >>> dfWithInvalid = spark.createDataFrame([(Vectors.dense([3.0, 1.0]),)], ["a"])
     >>> indexer.getHandleInvalid()
     'error'
@@ -4214,7 +4276,6 @@ def __init__(self, maxCategories=20, inputCol=None, outputCol=None, handleInvali
         """
         super(VectorIndexer, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.VectorIndexer", self.uid)
-        self._setDefault(maxCategories=20, handleInvalid="error")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -4337,6 +4398,8 @@ class VectorSlicer(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, J
     True
     >>> loadedVs.getNames() == vs.getNames()
     True
+    >>> loadedVs.transform(df).take(1) == vs.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -4436,6 +4499,11 @@ class _Word2VecParams(HasStepSize, HasMaxIter, HasSeed, HasInputCol, HasOutputCo
                               "be divided into chunks up to the size.",
                               typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_Word2VecParams, self).__init__(*args)
+        self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
+                         windowSize=5, maxSentenceLength=1000)
+
     @since("1.4.0")
     def getVectorSize(self):
         """
@@ -4530,6 +4598,8 @@ class Word2Vec(JavaEstimator, _Word2VecParams, JavaMLReadable, JavaMLWritable):
     True
     >>> loadedModel.getVectors().first().vector == model.getVectors().first().vector
     True
+    >>> loadedModel.transform(doc).take(1) == model.transform(doc).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -4543,8 +4613,6 @@ def __init__(self, vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025,
         """
         super(Word2Vec, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.Word2Vec", self.uid)
-        self._setDefault(vectorSize=100, minCount=5, numPartitions=1, stepSize=0.025, maxIter=1,
-                         windowSize=5, maxSentenceLength=1000)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -4736,6 +4804,8 @@ class PCA(JavaEstimator, _PCAParams, JavaMLReadable, JavaMLWritable):
     True
     >>> loadedModel.explainedVariance == model.explainedVariance
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 1.5.0
     """
@@ -4852,6 +4922,11 @@ class _RFormulaParams(HasFeaturesCol, HasLabelCol, HasHandleInvalid):
                           "additional bucket, at index numLabels).",
                           typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_RFormulaParams, self).__init__(*args)
+        self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
+                         handleInvalid="error")
+
     @since("1.5.0")
     def getFormula(self):
         """
@@ -4954,8 +5029,6 @@ def __init__(self, formula=None, featuresCol="features", labelCol="label",
         """
         super(RFormula, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid)
-        self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc",
-                         handleInvalid="error")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -5064,6 +5137,11 @@ class _ChiSqSelectorParams(HasFeaturesCol, HasOutputCol, HasLabelCol):
     fwe = Param(Params._dummy(), "fwe", "The upper bound of the expected family-wise error rate.",
                 typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_ChiSqSelectorParams, self).__init__(*args)
+        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
+                         fpr=0.05, fdr=0.05, fwe=0.05)
+
     @since("2.1.0")
     def getSelectorType(self):
         """
@@ -5160,6 +5238,8 @@ class ChiSqSelector(JavaEstimator, _ChiSqSelectorParams, JavaMLReadable, JavaMLW
     >>> loadedModel = ChiSqSelectorModel.load(modelPath)
     >>> loadedModel.selectedFeatures == model.selectedFeatures
     True
+    >>> loadedModel.transform(df).take(1) == model.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -5175,8 +5255,6 @@ def __init__(self, numTopFeatures=50, featuresCol="features", outputCol=None,
         """
         super(ChiSqSelector, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.ChiSqSelector", self.uid)
-        self._setDefault(numTopFeatures=50, selectorType="numTopFeatures", percentile=0.1,
-                         fpr=0.05, fdr=0.05, fwe=0.05)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py
index 7d933daf9e032..0e302d53d97e7 100644
--- a/python/pyspark/ml/fpm.py
+++ b/python/pyspark/ml/fpm.py
@@ -56,6 +56,11 @@ class _FPGrowthParams(HasPredictionCol):
         "but will affect the association rules generation.",
         typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_FPGrowthParams, self).__init__(*args)
+        self._setDefault(minSupport=0.3, minConfidence=0.8,
+                         itemsCol="items", predictionCol="prediction")
+
     def getItemsCol(self):
         """
         Gets the value of itemsCol or its default value.
@@ -194,6 +199,11 @@ class FPGrowth(JavaEstimator, _FPGrowthParams, JavaMLWritable, JavaMLReadable):
     >>> new_data = spark.createDataFrame([(["t", "s"], )], ["items"])
     >>> sorted(fpm.transform(new_data).first().newPrediction)
     [u'x', u'y', u'z']
+    >>> model_path = temp_path + "/fpm_model"
+    >>> fpm.save(model_path)
+    >>> model2 = FPGrowthModel.load(model_path)
+    >>> fpm.transform(data).take(1) == model2.transform(data).take(1)
+    True
 
     .. versionadded:: 2.2.0
     """
@@ -206,8 +216,6 @@ def __init__(self, minSupport=0.3, minConfidence=0.8, itemsCol="items",
         """
         super(FPGrowth, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.fpm.FPGrowth", self.uid)
-        self._setDefault(minSupport=0.3, minConfidence=0.8,
-                         itemsCol="items", predictionCol="prediction")
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index 99d80aa867bda..62b856046234a 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -46,6 +46,10 @@ class _ALSModelParams(HasPredictionCol, HasBlockSize):
                               "Supported values: 'nan', 'drop'.",
                               typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_ALSModelParams, self).__init__(*args)
+        self._setDefault(blockSize=4096)
+
     @since("1.4.0")
     def getUserCol(self):
         """
@@ -99,6 +103,14 @@ class _ALSParams(_ALSModelParams, HasMaxIter, HasRegParam, HasCheckpointInterval
                               "StorageLevel for ALS model factors.",
                               typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_ALSParams, self).__init__(*args)
+        self._setDefault(rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
+                         implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
+                         ratingCol="rating", nonnegative=False, checkpointInterval=10,
+                         intermediateStorageLevel="MEMORY_AND_DISK",
+                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan")
+
     @since("1.4.0")
     def getRank(self):
         """
@@ -275,6 +287,8 @@ class ALS(JavaEstimator, _ALSParams, JavaMLWritable, JavaMLReadable):
     True
     >>> sorted(model.itemFactors.collect()) == sorted(model2.itemFactors.collect())
     True
+    >>> model.transform(test).take(1) == model2.transform(test).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -294,12 +308,6 @@ def __init__(self, rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemB
         """
         super(ALS, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.recommendation.ALS", self.uid)
-        self._setDefault(rank=10, maxIter=10, regParam=0.1, numUserBlocks=10, numItemBlocks=10,
-                         implicitPrefs=False, alpha=1.0, userCol="user", itemCol="item",
-                         ratingCol="rating", nonnegative=False, checkpointInterval=10,
-                         intermediateStorageLevel="MEMORY_AND_DISK",
-                         finalStorageLevel="MEMORY_AND_DISK", coldStartStrategy="nan",
-                         blockSize=4096)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index 9c3c1e6ca01ae..550efb27c1e97 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -79,6 +79,10 @@ class _LinearRegressionParams(_JavaPredictorParams, HasRegParam, HasElasticNetPa
                     "robustness. Must be > 1.0. Only valid when loss is huber",
                     typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_LinearRegressionParams, self).__init__(*args)
+        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35)
+
     @since("2.3.0")
     def getEpsilon(self):
         """
@@ -158,6 +162,8 @@ class LinearRegression(JavaRegressor, _LinearRegressionParams, JavaMLWritable, J
     True
     >>> model.intercept == model2.intercept
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.numFeatures
     1
     >>> model.write().format("pmml").save(model_path + "_2")
@@ -179,7 +185,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(LinearRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.LinearRegression", self.uid)
-        self._setDefault(maxIter=100, regParam=0.0, tol=1e-6, loss="squaredError", epsilon=1.35)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -584,6 +589,10 @@ class _IsotonicRegressionParams(HasFeaturesCol, HasLabelCol, HasPredictionCol, H
         "The index of the feature if featuresCol is a vector column, no effect otherwise.",
         typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_IsotonicRegressionParams, self).__init__(*args)
+        self._setDefault(isotonic=True, featureIndex=0)
+
     def getIsotonic(self):
         """
         Gets the value of isotonic or its default value.
@@ -633,6 +642,8 @@ class IsotonicRegression(JavaEstimator, _IsotonicRegressionParams, HasWeightCol,
     True
     >>> model.predictions == model2.predictions
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -646,7 +657,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(IsotonicRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.IsotonicRegression", self.uid)
-        self._setDefault(isotonic=True, featureIndex=0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -773,7 +783,11 @@ class _DecisionTreeRegressorParams(_DecisionTreeParams, _TreeRegressorParams, Ha
     .. versionadded:: 3.0.0
     """
 
-    pass
+    def __init__(self, *args):
+        super(_DecisionTreeRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="variance", leafCol="", minWeightFractionPerNode=0.0)
 
 
 @inherit_doc
@@ -829,6 +843,8 @@ class DecisionTreeRegressor(JavaRegressor, _DecisionTreeRegressorParams, JavaMLW
     True
     >>> model.depth == model2.depth
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.transform(test1).head().variance
     0.0
 
@@ -860,9 +876,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(DecisionTreeRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.DecisionTreeRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="variance", leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1017,7 +1030,14 @@ class _RandomForestRegressorParams(_RandomForestParams, _TreeRegressorParams):
 
     .. versionadded:: 3.0.0
     """
-    pass
+
+    def __init__(self, *args):
+        super(_RandomForestRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
+                         impurity="variance", subsamplingRate=1.0, numTrees=20,
+                         featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
+                         bootstrap=True)
 
 
 @inherit_doc
@@ -1078,6 +1098,8 @@ class RandomForestRegressor(JavaRegressor, _RandomForestRegressorParams, JavaMLW
     >>> model2 = RandomForestRegressionModel.load(model_path)
     >>> model.featureImportances == model2.featureImportances
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 1.4.0
     """
@@ -1100,11 +1122,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(RandomForestRegressor, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.RandomForestRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="variance", subsamplingRate=1.0, numTrees=20,
-                         featureSubsetStrategy="auto", leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1275,6 +1292,14 @@ class _GBTRegressorParams(_GBTParams, _TreeRegressorParams):
                      "Supported options: " + ", ".join(supportedLossTypes),
                      typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_GBTRegressorParams, self).__init__(*args)
+        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
+                         maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
+                         checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1,
+                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
+                         leafCol="", minWeightFractionPerNode=0.0)
+
     @since("1.4.0")
     def getLossType(self):
         """
@@ -1338,6 +1363,8 @@ class GBTRegressor(JavaRegressor, _GBTRegressorParams, JavaMLWritable, JavaMLRea
     True
     >>> model.treeWeights == model2.treeWeights
     True
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
     >>> model.trees
     [DecisionTreeRegressionModel...depth=..., DecisionTreeRegressionModel...]
     >>> validation = spark.createDataFrame([(0.0, Vectors.dense(-1.0))],
@@ -1372,11 +1399,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         """
         super(GBTRegressor, self).__init__()
         self._java_obj = self._new_java_obj("org.apache.spark.ml.regression.GBTRegressor", self.uid)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0,
-                         maxMemoryInMB=256, cacheNodeIds=False, subsamplingRate=1.0,
-                         checkpointInterval=10, lossType="squared", maxIter=20, stepSize=0.1,
-                         impurity="variance", featureSubsetStrategy="all", validationTol=0.01,
-                         leafCol="", minWeightFractionPerNode=0.0)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1595,6 +1617,12 @@ class _AFTSurvivalRegressionParams(_JavaPredictorParams, HasMaxIter, HasTol, Has
         "corresponding quantileProbabilities if it is set.",
         typeConverter=TypeConverters.toString)
 
+    def __init__(self, *args):
+        super(_AFTSurvivalRegressionParams, self).__init__(*args)
+        self._setDefault(censorCol="censor",
+                         quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99],
+                         maxIter=100, tol=1E-6)
+
     @since("1.6.0")
     def getCensorCol(self):
         """
@@ -1667,6 +1695,8 @@ class AFTSurvivalRegression(JavaRegressor, _AFTSurvivalRegressionParams,
     True
     >>> model.scale == model2.scale
     True
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 1.6.0
     """
@@ -1685,9 +1715,6 @@ def __init__(self, featuresCol="features", labelCol="label", predictionCol="pred
         super(AFTSurvivalRegression, self).__init__()
         self._java_obj = self._new_java_obj(
             "org.apache.spark.ml.regression.AFTSurvivalRegression", self.uid)
-        self._setDefault(censorCol="censor",
-                         quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99],
-                         maxIter=100, tol=1E-6)
         kwargs = self._input_kwargs
         self.setParams(**kwargs)
 
@@ -1847,8 +1874,8 @@ class _GeneralizedLinearRegressionParams(_JavaPredictorParams, HasFitIntercept,
                       "or empty, we treat all instance offsets as 0.0",
                       typeConverter=TypeConverters.toString)
 
-    def __init__(self):
-        super(_GeneralizedLinearRegressionParams, self).__init__()
+    def __init__(self, *args):
+        super(_GeneralizedLinearRegressionParams, self).__init__(*args)
         self._setDefault(family="gaussian", maxIter=25, tol=1e-6, regParam=0.0, solver="irls",
                          variancePower=0.0, aggregationDepth=2)
 
@@ -1966,6 +1993,8 @@ class GeneralizedLinearRegression(JavaRegressor, _GeneralizedLinearRegressionPar
     True
     >>> model.coefficients[0] == model2.coefficients[0]
     True
+    >>> model.transform(df).take(1) == model2.transform(df).take(1)
+    True
 
     .. versionadded:: 2.0.0
     """
@@ -2357,8 +2386,8 @@ class _FactorizationMachinesParams(_JavaPredictorParams, HasMaxIter, HasStepSize
     solver = Param(Params._dummy(), "solver", "The solver algorithm for optimization. Supported " +
                    "options: gd, adamW. (Default adamW)", typeConverter=TypeConverters.toString)
 
-    def __init__(self):
-        super(_FactorizationMachinesParams, self).__init__()
+    def __init__(self, *args):
+        super(_FactorizationMachinesParams, self).__init__(*args)
         self._setDefault(factorSize=8, fitIntercept=True, fitLinear=True, regParam=0.0,
                          miniBatchFraction=1.0, initStd=0.01, maxIter=100, stepSize=1.0,
                          tol=1e-6, solver="adamW")
@@ -2436,6 +2465,17 @@ class FMRegressor(JavaRegressor, _FactorizationMachinesParams, JavaMLWritable, J
     DenseVector([0.9978])
     >>> model.factors
     DenseMatrix(1, 2, [0.0173, 0.0021], 1)
+    >>> model_path = temp_path + "/fm_model"
+    >>> model.save(model_path)
+    >>> model2 = FMRegressionModel.load(model_path)
+    >>> model2.intercept
+    -0.0032501766849261557
+    >>> model2.linear
+    DenseVector([0.9978])
+    >>> model2.factors
+    DenseMatrix(1, 2, [0.0173, 0.0021], 1)
+    >>> model.transform(test0).take(1) == model2.transform(test0).take(1)
+    True
 
     .. versionadded:: 3.0.0
     """
diff --git a/python/pyspark/ml/tests/test_param.py b/python/pyspark/ml/tests/test_param.py
index 777b4930ce8c9..2b119ea866236 100644
--- a/python/pyspark/ml/tests/test_param.py
+++ b/python/pyspark/ml/tests/test_param.py
@@ -366,15 +366,14 @@ def test_java_params(self):
                 if not name.endswith('Model') and not name.endswith('Params') \
                         and issubclass(cls, JavaParams) and not inspect.isabstract(cls) \
                         and not name.startswith('Java') and name != '_LSH':
-                    # NOTE: disable check_params_exist until there is parity with Scala API
-                    check_params(self, cls(), check_params_exist=False)
+                    check_params(self, cls(), check_params_exist=True)
 
         # Additional classes that need explicit construction
         from pyspark.ml.feature import CountVectorizerModel, StringIndexerModel
         check_params(self, CountVectorizerModel.from_vocabulary(['a'], 'input'),
-                     check_params_exist=False)
+                     check_params_exist=True)
         check_params(self, StringIndexerModel.from_labels(['a', 'b'], 'input'),
-                     check_params_exist=False)
+                     check_params_exist=True)
 
 
 if __name__ == "__main__":
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index cb4542c5d025f..e564ff7a10448 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -200,6 +200,10 @@ class _CrossValidatorParams(_ValidatorParams):
     numFolds = Param(Params._dummy(), "numFolds", "number of folds for cross validation",
                      typeConverter=TypeConverters.toInt)
 
+    def __init__(self, *args):
+        super(_CrossValidatorParams, self).__init__(*args)
+        self._setDefault(numFolds=3)
+
     @since("1.4.0")
     def getNumFolds(self):
         """
@@ -249,6 +253,8 @@ class CrossValidator(Estimator, _CrossValidatorParams, HasParallelism, HasCollec
     [0.5, ...
     >>> evaluator.evaluate(cvModel.transform(dataset))
     0.8333...
+    >>> evaluator.evaluate(cvModelRead.transform(dataset))
+    0.8333...
 
     .. versionadded:: 1.4.0
     """
@@ -261,7 +267,7 @@ def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, numF
                  seed=None, parallelism=1, collectSubModels=False)
         """
         super(CrossValidator, self).__init__()
-        self._setDefault(numFolds=3, parallelism=1)
+        self._setDefault(parallelism=1)
         kwargs = self._input_kwargs
         self._set(**kwargs)
 
@@ -545,6 +551,10 @@ class _TrainValidationSplitParams(_ValidatorParams):
     trainRatio = Param(Params._dummy(), "trainRatio", "Param for ratio between train and\
      validation data. Must be between 0 and 1.", typeConverter=TypeConverters.toFloat)
 
+    def __init__(self, *args):
+        super(_TrainValidationSplitParams, self).__init__(*args)
+        self._setDefault(trainRatio=0.75)
+
     @since("2.0.0")
     def getTrainRatio(self):
         """
@@ -590,6 +600,8 @@ class TrainValidationSplit(Estimator, _TrainValidationSplitParams, HasParallelis
     [0.5, ...
     >>> evaluator.evaluate(tvsModel.transform(dataset))
     0.833...
+    >>> evaluator.evaluate(tvsModelRead.transform(dataset))
+    0.833...
 
     .. versionadded:: 2.0.0
     """
@@ -602,7 +614,7 @@ def __init__(self, estimator=None, estimatorParamMaps=None, evaluator=None, trai
                  parallelism=1, collectSubModels=False, seed=None)
         """
         super(TrainValidationSplit, self).__init__()
-        self._setDefault(trainRatio=0.75, parallelism=1)
+        self._setDefault(parallelism=1)
         kwargs = self._input_kwargs
         self._set(**kwargs)
 

From 7004c989048b08891fb5f62ce2fcf0c89ce1496a Mon Sep 17 00:00:00 2001
From: Andy Grove <andygrove@nvidia.com>
Date: Fri, 24 Jul 2020 11:03:57 -0700
Subject: [PATCH 0924/1280] [SPARK-32430][SQL] Extend SparkSessionExtensions to
 inject rules into AQE query stage preparation

### What changes were proposed in this pull request?

Provide a generic mechanism for plugins to inject rules into the AQE "query prep" stage that happens before query stage creation.

This goes along with https://issues.apache.org/jira/browse/SPARK-32332 where the current AQE implementation doesn't allow for users to properly extend it for columnar processing.

### Why are the changes needed?

The issue here is that we create new query stages but we do not have access to the parent plan of the new query stage so certain things can not be determined because you have to know what the parent did.  With this change it would allow you to add TAGs to be able to figure out what is going on.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

A new unit test is included in the PR.

Closes #29224 from andygrove/insert-aqe-rule.

Authored-by: Andy Grove <andygrove@nvidia.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 64a01c0a559396fccd615dc00576a80bc8cc5648)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/SparkSessionExtensions.scala    | 20 +++++++-
 .../adaptive/AdaptiveSparkPlanExec.scala      |  2 +-
 .../internal/BaseSessionStateBuilder.scala    |  9 +++-
 .../spark/sql/internal/SessionState.scala     |  4 +-
 .../sql/SparkSessionExtensionSuite.scala      | 49 +++++++++++++++++++
 5 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
index 1c2bf9e7c2a57..bd870fb8dcae6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSessionExtensions.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.execution.ColumnarRule
+import org.apache.spark.sql.execution.{ColumnarRule, SparkPlan}
 
 /**
  * :: Experimental ::
@@ -44,6 +44,7 @@ import org.apache.spark.sql.execution.ColumnarRule
  * <li>Customized Parser.</li>
  * <li>(External) Catalog listeners.</li>
  * <li>Columnar Rules.</li>
+ * <li>Adaptive Query Stage Preparation Rules.</li>
  * </ul>
  *
  * The extensions can be used by calling `withExtensions` on the [[SparkSession.Builder]], for
@@ -96,8 +97,10 @@ class SparkSessionExtensions {
   type ParserBuilder = (SparkSession, ParserInterface) => ParserInterface
   type FunctionDescription = (FunctionIdentifier, ExpressionInfo, FunctionBuilder)
   type ColumnarRuleBuilder = SparkSession => ColumnarRule
+  type QueryStagePrepRuleBuilder = SparkSession => Rule[SparkPlan]
 
   private[this] val columnarRuleBuilders = mutable.Buffer.empty[ColumnarRuleBuilder]
+  private[this] val queryStagePrepRuleBuilders = mutable.Buffer.empty[QueryStagePrepRuleBuilder]
 
   /**
    * Build the override rules for columnar execution.
@@ -106,6 +109,13 @@ class SparkSessionExtensions {
     columnarRuleBuilders.map(_.apply(session))
   }
 
+  /**
+   * Build the override rules for the query stage preparation phase of adaptive query execution.
+   */
+  private[sql] def buildQueryStagePrepRules(session: SparkSession): Seq[Rule[SparkPlan]] = {
+    queryStagePrepRuleBuilders.map(_.apply(session)).toSeq
+  }
+
   /**
    * Inject a rule that can override the columnar execution of an executor.
    */
@@ -113,6 +123,14 @@ class SparkSessionExtensions {
     columnarRuleBuilders += builder
   }
 
+  /**
+   * Inject a rule that can override the the query stage preparation phase of adaptive query
+   * execution.
+   */
+  def injectQueryStagePrepRule(builder: QueryStagePrepRuleBuilder): Unit = {
+    queryStagePrepRuleBuilders += builder
+  }
+
   private[this] val resolutionRuleBuilders = mutable.Buffer.empty[RuleBuilder]
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index f6a3333b90cb0..5714c33d3774e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -90,7 +90,7 @@ case class AdaptiveSparkPlanExec(
   // Exchange nodes) after running these rules.
   private def queryStagePreparationRules: Seq[Rule[SparkPlan]] = Seq(
     ensureRequirements
-  )
+  ) ++ context.session.sessionState.queryStagePrepRules
 
   // A list of physical optimizer rules to be applied to a new stage before its execution. These
   // optimizations should be stage-independent.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
index 4ae12f8716752..83a7a557305e9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
-import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlanner, SparkSqlParser}
+import org.apache.spark.sql.execution.{ColumnarRule, QueryExecution, SparkOptimizer, SparkPlan, SparkPlanner, SparkSqlParser}
 import org.apache.spark.sql.execution.aggregate.ResolveEncodersInScalaAgg
 import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin
 import org.apache.spark.sql.execution.command.CommandCheck
@@ -286,6 +286,10 @@ abstract class BaseSessionStateBuilder(
     extensions.buildColumnarRules(session)
   }
 
+  protected def queryStagePrepRules: Seq[Rule[SparkPlan]] = {
+    extensions.buildQueryStagePrepRules(session)
+  }
+
   /**
    * Create a query execution object.
    */
@@ -337,7 +341,8 @@ abstract class BaseSessionStateBuilder(
       () => resourceLoader,
       createQueryExecution,
       createClone,
-      columnarRules)
+      columnarRules,
+      queryStagePrepRules)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
index abd1250628539..cd425b04ef311 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SessionState.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.optimizer.Optimizer
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.streaming.StreamingQueryManager
@@ -73,7 +74,8 @@ private[sql] class SessionState(
     resourceLoaderBuilder: () => SessionResourceLoader,
     createQueryExecution: LogicalPlan => QueryExecution,
     createClone: (SparkSession, SessionState) => SessionState,
-    val columnarRules: Seq[ColumnarRule]) {
+    val columnarRules: Seq[ColumnarRule],
+    val queryStagePrepRules: Seq[Rule[SparkPlan]]) {
 
   // The following fields are lazy to avoid creating the Hive client when creating SessionState.
   lazy val catalog: SessionCatalog = catalogBuilder()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
index d9c90c7dbd085..44e784de5164f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
@@ -26,7 +26,9 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, UnresolvedHint}
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
+import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
 import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.COLUMN_BATCH_SIZE
@@ -145,6 +147,28 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
     }
   }
 
+  test("inject adaptive query prep rule") {
+    val extensions = create { extensions =>
+      // inject rule that will run during AQE query stage preparation and will add custom tags
+      // to the plan
+      extensions.injectQueryStagePrepRule(session => MyQueryStagePrepRule())
+      // inject rule that will run during AQE query stage optimization and will verify that the
+      // custom tags were written in the preparation phase
+      extensions.injectColumnar(session =>
+        MyColumarRule(MyNewQueryStageRule(), MyNewQueryStageRule()))
+    }
+    withSession(extensions) { session =>
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, true)
+      assert(session.sessionState.queryStagePrepRules.contains(MyQueryStagePrepRule()))
+      assert(session.sessionState.columnarRules.contains(
+        MyColumarRule(MyNewQueryStageRule(), MyNewQueryStageRule())))
+      import session.sqlContext.implicits._
+      val data = Seq((100L), (200L), (300L)).toDF("vals").repartition(1)
+      val df = data.selectExpr("vals + 1")
+      df.collect()
+    }
+  }
+
   test("inject columnar") {
     val extensions = create { extensions =>
       extensions.injectColumnar(session =>
@@ -731,6 +755,31 @@ class MyExtensions extends (SparkSessionExtensions => Unit) {
   }
 }
 
+object QueryPrepRuleHelper {
+  val myPrepTag: TreeNodeTag[String] = TreeNodeTag[String]("myPrepTag")
+  val myPrepTagValue: String = "myPrepTagValue"
+}
+
+// this rule will run during AQE query preparation and will write custom tags to each node
+case class MyQueryStagePrepRule() extends Rule[SparkPlan] {
+  override def apply(plan: SparkPlan): SparkPlan = plan.transformDown {
+    case plan =>
+      plan.setTagValue(QueryPrepRuleHelper.myPrepTag, QueryPrepRuleHelper.myPrepTagValue)
+      plan
+  }
+}
+
+// this rule will run during AQE query stage optimization and will verify custom tags were
+// already written during query preparation phase
+case class MyNewQueryStageRule() extends Rule[SparkPlan] {
+  override def apply(plan: SparkPlan): SparkPlan = plan.transformDown {
+    case plan if !plan.isInstanceOf[AdaptiveSparkPlanExec] =>
+      assert(plan.getTagValue(QueryPrepRuleHelper.myPrepTag).get ==
+          QueryPrepRuleHelper.myPrepTagValue)
+      plan
+  }
+}
+
 case class MyRule2(spark: SparkSession) extends Rule[LogicalPlan] {
   override def apply(plan: LogicalPlan): LogicalPlan = plan
 }

From ab14d05624f6cc88b02e29dd5ae8d302ffcfd09d Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Fri, 24 Jul 2020 11:12:28 -0700
Subject: [PATCH 0925/1280] [SPARK-32287][TESTS] Flaky Test:
 ExecutorAllocationManagerSuite.add executors default profile

I wasn't able to reproduce the failure but the best I can tell is that the allocation manager timer triggers and call doRequest. The timeout is 10s so try to increase that to 30seconds.

test failure

no

unit test

Closes #29225 from tgravescs/SPARK-32287.

Authored-by: Thomas Graves <tgraves@nvidia.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit e6ef27be52dcd14dc94384c2ada85861be44d843)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/ExecutorAllocationManagerSuite.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
index 8d958494d52be..0b19146713966 100644
--- a/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala
@@ -1142,7 +1142,7 @@ class ExecutorAllocationManagerSuite extends SparkFunSuite {
       .set(config.DYN_ALLOCATION_TESTING, true)
       // SPARK-22864: effectively disable the allocation schedule by setting the period to a
       // really long value.
-      .set(TEST_SCHEDULE_INTERVAL, 10000L)
+      .set(TEST_SCHEDULE_INTERVAL, 30000L)
   }
 
   private def createManager(

From d71be73355033931a1b999f0ca4a05b95303187d Mon Sep 17 00:00:00 2001
From: Itsuki Toyota <titsuki@cpan.org>
Date: Sun, 26 Jul 2020 09:12:43 -0500
Subject: [PATCH 0926/1280] =?UTF-8?q?[SPARK-32428][EXAMPLES]=20Make=20Bina?=
 =?UTF-8?q?ryClassificationMetricsExample=20cons=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…istently print the metrics on driver's stdout

### What changes were proposed in this pull request?

Call collect on RDD before calling foreach so that it sends the result to the driver node and print it on this node's stdout.

### Why are the changes needed?

Some RDDs in this example (e.g., precision, recall) call println without calling collect.
If the job is under local mode, it sends the data to the driver node and prints the metrics on the driver's stdout.
However if the job is under cluster mode, the job prints the metrics on the executor's stdout.
It seems inconsistent compared to the other metrics nothing to do with RDD (e.g., auPRC, auROC) since these metrics always output the result on the driver's stdout.
All of the metrics should output its result on the driver's stdout.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

This is example code. It doesn't have any tests.

Closes #29222 from titsuki/SPARK-32428.

Authored-by: Itsuki Toyota <titsuki@cpan.org>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 86ead044e3789b3291a38ec2142cbb343d1290c1)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../mllib/BinaryClassificationMetricsExample.scala        | 8 ++++----
 .../spark/examples/mllib/ChiSqSelectorExample.scala       | 2 +-
 .../spark/examples/mllib/ElementwiseProductExample.scala  | 4 ++--
 .../apache/spark/examples/mllib/NormalizerExample.scala   | 4 ++--
 .../spark/examples/mllib/StandardScalerExample.scala      | 4 ++--
 .../org/apache/spark/examples/mllib/TFIDFExample.scala    | 4 ++--
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
index c6312d71cc912..a606cc4218778 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala
@@ -59,13 +59,13 @@ object BinaryClassificationMetricsExample {
 
     // Precision by threshold
     val precision = metrics.precisionByThreshold
-    precision.foreach { case (t, p) =>
+    precision.collect.foreach { case (t, p) =>
       println(s"Threshold: $t, Precision: $p")
     }
 
     // Recall by threshold
     val recall = metrics.recallByThreshold
-    recall.foreach { case (t, r) =>
+    recall.collect.foreach { case (t, r) =>
       println(s"Threshold: $t, Recall: $r")
     }
 
@@ -74,13 +74,13 @@ object BinaryClassificationMetricsExample {
 
     // F-measure
     val f1Score = metrics.fMeasureByThreshold
-    f1Score.foreach { case (t, f) =>
+    f1Score.collect.foreach { case (t, f) =>
       println(s"Threshold: $t, F-score: $f, Beta = 1")
     }
 
     val beta = 0.5
     val fScore = metrics.fMeasureByThreshold(beta)
-    f1Score.foreach { case (t, f) =>
+    fScore.collect.foreach { case (t, f) =>
       println(s"Threshold: $t, F-score: $f, Beta = 0.5")
     }
 
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
index 5e400b7d715b4..6ed59a3366d93 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala
@@ -53,7 +53,7 @@ object ChiSqSelectorExample {
     // $example off$
 
     println("filtered data: ")
-    filteredData.foreach(x => println(x))
+    filteredData.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
index 1855058bb4f3c..d6ec678b422f9 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala
@@ -45,10 +45,10 @@ object ElementwiseProductExample {
     // $example off$
 
     println("transformedData: ")
-    transformedData.foreach(x => println(x))
+    transformedData.collect.foreach(x => println(x))
 
     println("transformedData2: ")
-    transformedData2.foreach(x => println(x))
+    transformedData2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
index b3a9604c2be3e..b1cad7bf8125f 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala
@@ -46,10 +46,10 @@ object NormalizerExample {
     // $example off$
 
     println("data1: ")
-    data1.foreach(x => println(x))
+    data1.collect.foreach(x => println(x))
 
     println("data2: ")
-    data2.foreach(x => println(x))
+    data2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
index 769fc17b3dc65..66a608ced0f83 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala
@@ -49,10 +49,10 @@ object StandardScalerExample {
     // $example off$
 
     println("data1: ")
-    data1.foreach(x => println(x))
+    data1.collect.foreach(x => println(x))
 
     println("data2: ")
-    data2.foreach(x => println(x))
+    data2.collect.foreach(x => println(x))
 
     sc.stop()
   }
diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
index a5bdcd8f2ed32..14b2a2015c241 100644
--- a/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala
@@ -55,10 +55,10 @@ object TFIDFExample {
     // $example off$
 
     println("tfidf: ")
-    tfidf.foreach(x => println(x))
+    tfidf.collect.foreach(x => println(x))
 
     println("tfidfIgnore: ")
-    tfidfIgnore.foreach(x => println(x))
+    tfidfIgnore.collect.foreach(x => println(x))
 
     sc.stop()
   }

From 4b8761e2bb58b24823daebf43ea7750dc283bc07 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sun, 26 Jul 2020 19:25:41 -0700
Subject: [PATCH 0927/1280] [SPARK-32448][K8S][TESTS] Use single version for
 exec-maven-plugin/scalatest-maven-plugin

### What changes were proposed in this pull request?

Two different versions are used for the same artifacts, `exec-maven-plugin` and `scalatest-maven-plugin`. This PR aims to use the same versions for `exec-maven-plugin` and `scalatest-maven-plugin`. In addition, this PR removes `scala-maven-plugin.version` from `K8s` integration suite because it's unused.

### Why are the changes needed?

This will prevent the mistake which upgrades only one place and forgets the others.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins K8S IT.

Closes #29248 from dongjoon-hyun/SPARK-32448.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 13c64c298016eb3882ed20a6f6c60f1ea3988b3b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 pom.xml                                                | 6 ++++--
 resource-managers/kubernetes/integration-tests/pom.xml | 3 ---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index d8d559550a5be..6c67ce41b413a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,7 @@
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <maven.version>3.6.3</maven.version>
+    <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
     <sbt.project.name>spark</sbt.project.name>
     <slf4j.version>1.7.30</slf4j.version>
     <log4j.version>1.2.17</log4j.version>
@@ -162,6 +163,7 @@
     <commons.collections.version>3.2.2</commons.collections.version>
     <scala.version>2.12.10</scala.version>
     <scala.binary.version>2.12</scala.binary.version>
+    <scalatest-maven-plugin.version>2.0.0</scalatest-maven-plugin.version>
     <scalafmt.parameters>--test</scalafmt.parameters>
     <!-- for now, not running scalafmt as part of default verify pipeline -->
     <scalafmt.skip>true</scalafmt.skip>
@@ -2479,7 +2481,7 @@
         <plugin>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest-maven-plugin</artifactId>
-          <version>2.0.0</version>
+          <version>${scalatest-maven-plugin.version}</version>
           <!-- Note config is repeated in surefire config -->
           <configuration>
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
@@ -2615,7 +2617,7 @@
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>1.6.0</version>
+          <version>${exec-maven-plugin.version}</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 6c258a7ce9042..1d58b97805241 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -27,11 +27,8 @@
   <artifactId>spark-kubernetes-integration-tests_2.12</artifactId>
   <properties>
     <download-maven-plugin.version>1.3.0</download-maven-plugin.version>
-    <exec-maven-plugin.version>1.4.0</exec-maven-plugin.version>
     <extraScalaTestArgs></extraScalaTestArgs>
     <kubernetes-client.version>4.9.2</kubernetes-client.version>
-    <scala-maven-plugin.version>3.2.2</scala-maven-plugin.version>
-    <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>
 
     <!-- Integration Test Configuration Properties -->

From 6ed93c3e86c60323328b44cab45faa9ae3050dab Mon Sep 17 00:00:00 2001
From: GuoPhilipse <guofei_ok@126.com>
Date: Tue, 28 Jul 2020 09:41:53 +0900
Subject: [PATCH 0928/1280] [SPARK-31753][SQL][DOCS] Add missing keywords in
 the SQL docs

### What changes were proposed in this pull request?
update sql-ref docs, the following key words will be added in this PR.

CASE/ELSE
WHEN/THEN
MAP KEYS TERMINATED BY
NULL DEFINED AS
LINES TERMINATED BY
ESCAPED BY
COLLECTION ITEMS TERMINATED BY
PIVOT
LATERAL VIEW OUTER?
ROW FORMAT SERDE
ROW FORMAT DELIMITED
FIELDS TERMINATED BY
IGNORE NULLS
FIRST
LAST

### Why are the changes needed?
let more users know the sql key words usage

### Does this PR introduce _any_ user-facing change?
![image](https://user-images.githubusercontent.com/46367746/88148830-c6dc1f80-cc31-11ea-81ea-13bc9dc34550.png)
![image](https://user-images.githubusercontent.com/46367746/88148968-fb4fdb80-cc31-11ea-8649-e8297cf5813e.png)
![image](https://user-images.githubusercontent.com/46367746/88149000-073b9d80-cc32-11ea-9aa4-f914ecd72663.png)
![image](https://user-images.githubusercontent.com/46367746/88149021-0f93d880-cc32-11ea-86ed-7db8672b5aac.png)

### How was this patch tested?
No

Closes #29056 from GuoPhilipse/add-missing-keywords.

Lead-authored-by: GuoPhilipse <guofei_ok@126.com>
Co-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 8de43338be879f0cfeebca328dbbcfd1e5bd70da)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml                      |   6 +
 ...-ref-syntax-ddl-create-table-hiveformat.md |  94 ++++++++++++-
 docs/sql-ref-syntax-qry-select-case.md        | 109 +++++++++++++++
 docs/sql-ref-syntax-qry-select-clusterby.md   |   3 +
 ...sql-ref-syntax-qry-select-distribute-by.md |   3 +
 docs/sql-ref-syntax-qry-select-groupby.md     |  27 ++++
 docs/sql-ref-syntax-qry-select-having.md      |   3 +
 .../sql-ref-syntax-qry-select-lateral-view.md | 125 ++++++++++++++++++
 docs/sql-ref-syntax-qry-select-limit.md       |   3 +
 docs/sql-ref-syntax-qry-select-orderby.md     |   3 +
 docs/sql-ref-syntax-qry-select-pivot.md       | 101 ++++++++++++++
 docs/sql-ref-syntax-qry-select-sortby.md      |   3 +
 docs/sql-ref-syntax-qry-select-where.md       |   3 +
 docs/sql-ref-syntax-qry-select.md             |  56 +++++---
 docs/sql-ref-syntax-qry.md                    |   3 +
 docs/sql-ref-syntax.md                        |   3 +
 16 files changed, 520 insertions(+), 25 deletions(-)
 create mode 100644 docs/sql-ref-syntax-qry-select-case.md
 create mode 100644 docs/sql-ref-syntax-qry-select-lateral-view.md
 create mode 100644 docs/sql-ref-syntax-qry-select-pivot.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index eea657e684495..22fae0cb3ff55 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -187,6 +187,12 @@
                   url: sql-ref-syntax-qry-select-tvf.html
                 - text: Window Function
                   url: sql-ref-syntax-qry-select-window.html
+                - text: CASE Clause
+                  url: sql-ref-syntax-qry-select-case.html
+                - text: LATERAL VIEW Clause
+                  url: sql-ref-syntax-qry-select-lateral-view.html
+                - text: PIVOT Clause
+                  url: sql-ref-syntax-qry-select-pivot.html
             - text: EXPLAIN
               url: sql-ref-syntax-qry-explain.html
         - text: Auxiliary Statements
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 38f8856a24e3d..7bf847df98150 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -36,6 +36,14 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
     [ LOCATION path ]
     [ TBLPROPERTIES ( key1=val1, key2=val2, ... ) ]
     [ AS select_statement ]
+
+row_format:    
+    : SERDE serde_class [ WITH SERDEPROPERTIES (k1=v1, k2=v2, ... ) ]
+    | DELIMITED [ FIELDS TERMINATED BY fields_termiated_char [ ESCAPED BY escaped_char ] ] 
+        [ COLLECTION ITEMS TERMINATED BY collection_items_termiated_char ] 
+        [ MAP KEYS TERMINATED BY map_key_termiated_char ]
+        [ LINES TERMINATED BY row_termiated_char ]
+        [ NULL DEFINED AS null_char ]
 ```
 
 Note that, the clauses between the columns definition clause and the AS SELECT clause can come in
@@ -51,15 +59,55 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
 
 * **EXTERNAL**
 
-    Table is defined using the path provided as LOCATION, does not use default location for this table.
+    Table is defined using the path provided as `LOCATION`, does not use default location for this table.
 
 * **PARTITIONED BY**
 
     Partitions are created on the table, based on the columns specified.
+    
+* **row_format**    
+
+    Use the `SERDE` clause to specify a custom SerDe for one table. Otherwise, use the `DELIMITED` clause to use the native SerDe and specify the delimiter, escape character, null character and so on.
+    
+* **SERDE**
+
+    Specifies a custom SerDe for one table.
+    
+* **serde_class**
+
+    Specifies a fully-qualified class name of a custom SerDe.
+
+* **SERDEPROPERTIES**
+
+    A list of key-value pairs that is used to tag the SerDe definition.
+    
+* **DELIMITED**
+
+    The `DELIMITED` clause can be used to specify the native SerDe and state the delimiter, escape character, null character and so on.
+    
+* **FIELDS TERMINATED BY**
 
-* **ROW FORMAT**
+    Used to define a column separator.
+    
+* **COLLECTION ITEMS TERMINATED BY**
 
-    SERDE is used to specify a custom SerDe or the DELIMITED clause in order to use the native SerDe.
+    Used to define a collection item separator.
+   
+* **MAP KEYS TERMINATED BY**
+
+    Used to define a map key separator.
+    
+* **LINES TERMINATED BY**
+
+    Used to define a row separator.
+    
+* **NULL DEFINED AS**
+
+    Used to define the specific value for NULL.
+    
+* **ESCAPED BY**
+
+    Used for escape mechanism.
 
 * **STORED AS**
 
@@ -114,9 +162,47 @@ CREATE TABLE student (id INT, name STRING)
     PARTITIONED BY (age INT);
 
 --Use Row Format and file format
-CREATE TABLE student (id INT,name STRING)
+CREATE TABLE student (id INT, name STRING)
     ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
     STORED AS TEXTFILE;
+
+--Use complex datatype
+CREATE EXTERNAL TABLE family(
+        name STRING,
+        friends ARRAY<STRING>,
+        children MAP<STRING, INT>,
+        address STRUCT<street: STRING, city: STRING>
+    )
+    ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' ESCAPED BY '\\'
+    COLLECTION ITEMS TERMINATED BY '_'
+    MAP KEYS TERMINATED BY ':'
+    LINES TERMINATED BY '\n'
+    NULL DEFINED AS 'foonull'
+    STORED AS TEXTFILE
+    LOCATION '/tmp/family/';
+
+--Use predefined custom SerDe
+CREATE TABLE avroExample
+    ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+    STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+        OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+    TBLPROPERTIES ('avro.schema.literal'='{ "namespace": "org.apache.hive",
+        "name": "first_schema",
+        "type": "record",
+        "fields": [
+                { "name":"string1", "type":"string" },
+                { "name":"string2", "type":"string" }
+            ] }');
+
+--Use personalized custom SerDe(we may need to `ADD JAR xxx.jar` first to ensure we can find the serde_class,
+--or you may run into `CLASSNOTFOUND` exception)
+ADD JAR /tmp/hive_serde_example.jar;
+
+CREATE EXTERNAL TABLE family (id INT, name STRING)
+    ROW FORMAT SERDE 'com.ly.spark.serde.SerDeExample'
+    STORED AS INPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleInputFormat'
+        OUTPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleOutputFormat'
+    LOCATION '/tmp/family/';
 ```
 
 ### Related Statements
diff --git a/docs/sql-ref-syntax-qry-select-case.md b/docs/sql-ref-syntax-qry-select-case.md
new file mode 100644
index 0000000000000..6136b161d234b
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-case.md
@@ -0,0 +1,109 @@
+---
+layout: global
+title: CASE Clause
+displayTitle: CASE Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+`CASE` clause uses a rule to return a specific result based on the specified condition, similar to if/else statements in other programming languages.
+
+### Syntax
+
+```sql
+CASE [ expression ] { WHEN boolean_expression THEN then_expression } [ ... ]
+    [ ELSE else_expression ]
+END
+```
+
+### Parameters
+    
+* **boolean_expression**
+
+    Specifies any expression that evaluates to a result type `boolean`. Two or
+    more expressions may be combined together using the logical
+    operators ( `AND`, `OR` ).
+
+* **then_expression**
+
+    Specifies the then expression based on the `boolean_expression` condition; `then_expression` and `else_expression` should all be same type or coercible to a common type.
+    
+* **else_expression**
+
+    Specifies the default expression; `then_expression` and `else_expression` should all be same type or coercible to a common type.
+    
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'John', 30),
+    (200, 'Mary', NULL),
+    (300, 'Mike', 80),
+    (400, 'Dan', 50);
+
+SELECT id, CASE WHEN id > 200 THEN 'bigger' ELSE 'small' END FROM person;
++------+--------------------------------------------------+
+|  id  | CASE WHEN (id > 200) THEN bigger ELSE small END  |
++------+--------------------------------------------------+
+| 100  | small                                            |
+| 200  | small                                            |
+| 300  | bigger                                           |
+| 400  | bigger                                           |
++------+--------------------------------------------------+
+
+SELECT id, CASE id WHEN 100 then 'bigger' WHEN  id > 300 THEN '300' ELSE 'small' END FROM person;
++------+-----------------------------------------------------------------------------------------------+
+|  id  | CASE WHEN (id = 100) THEN bigger WHEN (id = CAST((id > 300) AS INT)) THEN 300 ELSE small END  |
++------+-----------------------------------------------------------------------------------------------+
+| 100  | bigger                                                                                        |
+| 200  | small                                                                                         |
+| 300  | small                                                                                         |
+| 400  | small                                                                                         |
++------+-----------------------------------------------------------------------------------------------+
+
+SELECT * FROM person
+    WHERE 
+        CASE 1 = 1 
+            WHEN 100 THEN 'big' 
+            WHEN 200 THEN 'bigger'
+            WHEN 300 THEN 'biggest' 
+            ELSE 'small'
+        END = 'small';
++------+-------+-------+
+|  id  | name  |  age  |
++------+-------+-------+
+| 100  | John  | 30    |
+| 200  | Mary  | NULL  |
+| 300  | Mike  | 80    |
+| 400  | Dan   | 50    |
++------+-------+-------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-clusterby.md b/docs/sql-ref-syntax-qry-select-clusterby.md
index e3bd2ed926ecc..9bcfac5b3b5c6 100644
--- a/docs/sql-ref-syntax-qry-select-clusterby.md
+++ b/docs/sql-ref-syntax-qry-select-clusterby.md
@@ -99,3 +99,6 @@ SELECT age, name FROM person CLUSTER BY age;
 * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-distribute-by.md b/docs/sql-ref-syntax-qry-select-distribute-by.md
index 1fdfb91dad286..fbf662d3359f4 100644
--- a/docs/sql-ref-syntax-qry-select-distribute-by.md
+++ b/docs/sql-ref-syntax-qry-select-distribute-by.md
@@ -94,3 +94,6 @@ SELECT age, name FROM person DISTRIBUTE BY age;
 * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index bd9377ef78df6..6137c0d80f313 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -260,6 +260,30 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
 | San Jose| HondaAccord|  8|
 | San Jose|  HondaCivic|  5|
 +---------+------------+---+
+
+--Prepare data for ignore nulls example
+CREATE TABLE person (id INT, name STRING, age INT);
+INSERT INTO person VALUES
+    (100, 'Mary', NULL),
+    (200, 'John', 30),
+    (300, 'Mike', 80),
+    (400, 'Dan', 50);
+
+--Select the first row in cloumn age
+SELECT FIRST(age) FROM person;
++--------------------+
+| first(age, false)  |
++--------------------+
+| NULL               |
++--------------------+
+
+--Get the first row in cloumn `age` ignore nulls,last row in column `id` and sum of cloumn `id`.
+SELECT FIRST(age IGNORE NULLS), LAST(id), SUM(id) FROM person;
++-------------------+------------------+----------+
+| first(age, true)  | last(id, false)  | sum(id)  |
++-------------------+------------------+----------+
+| 30                | 400              | 1000     |
++-------------------+------------------+----------+
 ```
 
 ### Related Statements
@@ -272,3 +296,6 @@ SELECT city, car_model, sum(quantity) AS sum FROM dealer
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-having.md b/docs/sql-ref-syntax-qry-select-having.md
index 935782c551e1f..59a8c680945e6 100644
--- a/docs/sql-ref-syntax-qry-select-having.md
+++ b/docs/sql-ref-syntax-qry-select-having.md
@@ -125,3 +125,6 @@ SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-lateral-view.md b/docs/sql-ref-syntax-qry-select-lateral-view.md
new file mode 100644
index 0000000000000..f742c8fa57043
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-lateral-view.md
@@ -0,0 +1,125 @@
+---
+layout: global
+title: LATERAL VIEW Clause
+displayTitle: LATERAL VIEW Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `LATERAL VIEW` clause is used in conjunction with generator functions such as `EXPLODE`, which will generate a virtual table containing one or more rows. `LATERAL VIEW` will apply the rows to each original output row.
+
+### Syntax
+
+```sql
+LATERAL VIEW [ OUTER ] generator_function ( expression [ , ... ] ) [ table_alias ] AS column_alias [ , ... ]
+```
+
+### Parameters
+
+* **OUTER**
+
+    If `OUTER` specified, returns null if an input array/map is empty or null. 
+    
+* **generator_function**
+
+    Specifies a generator function (EXPLODE, INLINE, etc.).
+    
+* **table_alias**
+
+    The alias for `generator_function`, which is optional.
+     
+* **column_alias**
+
+    Lists the column aliases of `generator_function`, which may be used in output rows. We may have multiple aliases if `generator_function` have multiple output columns.
+         
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING);
+INSERT INTO person VALUES
+    (100, 'John', 30, 1, 'Street 1'),
+    (200, 'Mary', NULL, 1, 'Street 2'),
+    (300, 'Mike', 80, 3, 'Street 3'),
+    (400, 'Dan', 50, 4, 'Street 4');
+
+SELECT * FROM person
+    LATERAL VIEW EXPLODE(ARRAY(30, 60)) tabelName AS c_age
+    LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age;
++------+-------+-------+--------+-----------+--------+--------+
+|  id  | name  |  age  | class  |  address  | c_age  | d_age  |
++------+-------+-------+--------+-----------+--------+--------+
+| 100  | John  | 30    | 1      | Street 1  | 30     | 40     |
+| 100  | John  | 30    | 1      | Street 1  | 30     | 80     |
+| 100  | John  | 30    | 1      | Street 1  | 60     | 40     |
+| 100  | John  | 30    | 1      | Street 1  | 60     | 80     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 30     | 40     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 30     | 80     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 60     | 40     |
+| 200  | Mary  | NULL  | 1      | Street 2  | 60     | 80     |
+| 300  | Mike  | 80    | 3      | Street 3  | 30     | 40     |
+| 300  | Mike  | 80    | 3      | Street 3  | 30     | 80     |
+| 300  | Mike  | 80    | 3      | Street 3  | 60     | 40     |
+| 300  | Mike  | 80    | 3      | Street 3  | 60     | 80     |
+| 400  | Dan   | 50    | 4      | Street 4  | 30     | 40     |
+| 400  | Dan   | 50    | 4      | Street 4  | 30     | 80     |
+| 400  | Dan   | 50    | 4      | Street 4  | 60     | 40     |
+| 400  | Dan   | 50    | 4      | Street 4  | 60     | 80     |
++------+-------+-------+--------+-----------+--------+--------+
+
+SELECT c_age, COUNT(1) FROM person
+    LATERAL VIEW EXPLODE(ARRAY(30, 60)) AS c_age
+    LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age 
+GROUP BY c_age;
++--------+-----------+
+| c_age  | count(1)  |
++--------+-----------+
+| 60     | 8         |
+| 30     | 8         |
++--------+-----------+
+
+SELECT * FROM person
+    LATERAL VIEW EXPLODE(ARRAY()) tabelName AS c_age;
++-----+-------+------+--------+----------+--------+
+| id  | name  | age  | class  | address  | c_age  |
++-----+-------+------+--------+----------+--------+
++-----+-------+------+--------+----------+--------+
+
+SELECT * FROM person
+    LATERAL VIEW OUTER EXPLODE(ARRAY()) tabelName AS c_age;
++------+-------+-------+--------+-----------+--------+
+|  id  | name  |  age  | class  |  address  | c_age  |
++------+-------+-------+--------+-----------+--------+
+| 100  | John  | 30    | 1      | Street 1  | NULL   |
+| 200  | Mary  | NULL  | 1      | Street 2  | NULL   |
+| 300  | Mike  | 80    | 3      | Street 3  | NULL   |
+| 400  | Dan   | 50    | 4      | Street 4  | NULL   |
++------+-------+-------+--------+-----------+--------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
diff --git a/docs/sql-ref-syntax-qry-select-limit.md b/docs/sql-ref-syntax-qry-select-limit.md
index 03c4df3cbc442..bd64ba890901b 100644
--- a/docs/sql-ref-syntax-qry-select-limit.md
+++ b/docs/sql-ref-syntax-qry-select-limit.md
@@ -104,3 +104,6 @@ org.apache.spark.sql.AnalysisException: The limit expression must evaluate to a
 * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-orderby.md b/docs/sql-ref-syntax-qry-select-orderby.md
index 85bbe514cdc95..13f0ae40cb828 100644
--- a/docs/sql-ref-syntax-qry-select-orderby.md
+++ b/docs/sql-ref-syntax-qry-select-orderby.md
@@ -143,3 +143,6 @@ SELECT * FROM person ORDER BY name ASC, age DESC;
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-pivot.md b/docs/sql-ref-syntax-qry-select-pivot.md
new file mode 100644
index 0000000000000..649c2518b28eb
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-pivot.md
@@ -0,0 +1,101 @@
+---
+layout: global
+title: PIVOT Clause
+displayTitle: PIVOT Clause
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `PIVOT` clause is used for data perspective. We can get the aggregated values based on specific column values, which will be turned to multiple columns used in `SELECT` clause. The `PIVOT` clause can be specified after the table name or subquery.
+
+### Syntax
+
+```sql
+PIVOT ( { aggregate_expression [ AS aggregate_expression_alias ] } [ , ... ]
+    FOR column_list IN ( expression_list ) )
+```
+
+### Parameters
+    
+* **aggregate_expression**
+
+    Specifies an aggregate expression (SUM(a), COUNT(DISTINCT b), etc.).
+    
+* **aggregate_expression_alias**
+
+    Specifies an alias for the aggregate expression.
+     
+* **column_list**
+
+    Contains columns in the `FROM` clause, which specifies the columns we want to replace with new columns. We can use brackets to surround the columns, such as `(c1, c2)`.
+      
+* **expression_list**
+
+    Specifies new columns, which are used to match values in `column_list` as the aggregating condition. We can also add aliases for them.
+    
+### Examples
+
+```sql
+CREATE TABLE person (id INT, name STRING, age INT, class INT, address STRING);
+INSERT INTO person VALUES
+    (100, 'John', 30, 1, 'Street 1'),
+    (200, 'Mary', NULL, 1, 'Street 2'),
+    (300, 'Mike', 80, 3, 'Street 3'),
+    (400, 'Dan', 50, 4, 'Street 4');
+
+SELECT * FROM person
+    PIVOT (
+        SUM(age) AS a, AVG(class) AS c
+        FOR name IN ('John' AS john, 'Mike' AS mike)
+    );
++------+-----------+---------+---------+---------+---------+
+|  id  |  address  | john_a  | john_c  | mike_a  | mike_c  |
++------+-----------+---------+---------+---------+---------+
+| 200  | Street 2  | NULL    | NULL    | NULL    | NULL    |
+| 100  | Street 1  | 30      | 1.0     | NULL    | NULL    |
+| 300  | Street 3  | NULL    | NULL    | 80      | 3.0     |
+| 400  | Street 4  | NULL    | NULL    | NULL    | NULL    |
++------+-----------+---------+---------+---------+---------+
+
+SELECT * FROM person
+    PIVOT (
+        SUM(age) AS a, AVG(class) AS c
+        FOR (name, age) IN (('John', 30) AS c1, ('Mike', 40) AS c2)
+    );
++------+-----------+-------+-------+-------+-------+
+|  id  |  address  | c1_a  | c1_c  | c2_a  | c2_c  |
++------+-----------+-------+-------+-------+-------+
+| 200  | Street 2  | NULL  | NULL  | NULL  | NULL  |
+| 100  | Street 1  | 30    | 1.0   | NULL  | NULL  |
+| 300  | Street 3  | NULL  | NULL  | NULL  | NULL  |
+| 400  | Street 4  | NULL  | NULL  | NULL  | NULL  |
++------+-----------+-------+-------+-------+-------+
+```
+
+### Related Statements
+
+* [SELECT Main](sql-ref-syntax-qry-select.html)
+* [WHERE Clause](sql-ref-syntax-qry-select-where.html)
+* [GROUP BY Clause](sql-ref-syntax-qry-select-groupby.html)
+* [HAVING Clause](sql-ref-syntax-qry-select-having.html)
+* [ORDER BY Clause](sql-ref-syntax-qry-select-orderby.html)
+* [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
+* [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
+* [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-sortby.md b/docs/sql-ref-syntax-qry-select-sortby.md
index 554bdb569d005..09e559adcd6ab 100644
--- a/docs/sql-ref-syntax-qry-select-sortby.md
+++ b/docs/sql-ref-syntax-qry-select-sortby.md
@@ -176,3 +176,6 @@ SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry-select-where.md b/docs/sql-ref-syntax-qry-select-where.md
index ca3f5ec7866c6..9ff7993d40c58 100644
--- a/docs/sql-ref-syntax-qry-select-where.md
+++ b/docs/sql-ref-syntax-qry-select-where.md
@@ -125,3 +125,6 @@ SELECT * FROM person AS parent
 * [CLUSTER BY Clause](sql-ref-syntax-qry-select-clusterby.html)
 * [DISTRIBUTE BY Clause](sql-ref-syntax-qry-select-distribute-by.html)
 * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
\ No newline at end of file
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 987e6479ab20a..453737a2f36b8 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -31,10 +31,10 @@ of a query along with examples.
 ```sql
 [ WITH with_query [ , ... ] ]
 select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_statement, ... ]
-    [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-    [ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ...] } ]
-    [ CLUSTER BY { expression [ , ...] } ]
-    [ DISTRIBUTE BY { expression [, ...] } ]
+    [ ORDER BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ] } ]
+    [ SORT BY { expression [ ASC | DESC ] [ NULLS { FIRST | LAST } ] [ , ... ] } ]
+    [ CLUSTER BY { expression [ , ... ] } ]
+    [ DISTRIBUTE BY { expression [, ... ] } ]
     [ WINDOW { named_window [ , WINDOW named_window, ... ] } ]
     [ LIMIT { ALL | expression } ]
 ```
@@ -42,9 +42,11 @@ select_statement [ { UNION | INTERSECT | EXCEPT } [ ALL | DISTINCT ] select_stat
 While `select_statement` is defined as
 ```sql
 SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
-    FROM { from_item [ , ...] }
+    FROM { from_item [ , ... ] }
+    [ PIVOT clause ]
+    [ LATERAL VIEW clause ] [ ... ] 
     [ WHERE boolean_expression ]
-    [ GROUP BY expression [ , ...] ]
+    [ GROUP BY expression [ , ... ] ]
     [ HAVING boolean_expression ]
 ```
 
@@ -75,7 +77,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 
     **Syntax:** `expression [AS] [alias]`
 
- * **from_item**
+* **from_item**
 
      Specifies a source of input for the query. It can be one of the following:
      * Table relation
@@ -83,62 +85,71 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
      * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
      * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
      * Subquery
+     
+* **PIVOT**
 
+     The `PIVOT` clause is used for data perspective; We can get the aggregated values based on specific column value.
 
- * **WHERE**
+* **LATERAL VIEW**
+     
+     The `LATERAL VIEW` clause is used in conjunction with generator functions such as `EXPLODE`, which will generate a virtual table containing one or more rows. `LATERAL VIEW` will apply the rows to each original output row.
+ 
+* **WHERE**
 
      Filters the result of the FROM clause based on the supplied predicates.
 
- * **GROUP BY**
+* **GROUP BY**
 
      Specifies the expressions that are used to group the rows. This is used in conjunction with aggregate functions
      (MIN, MAX, COUNT, SUM, AVG, etc.) to group rows based on the grouping expressions and aggregate values in each group.
      When a FILTER clause is attached to an aggregate function, only the matching rows are passed to that function.
 
- * **HAVING**
+* **HAVING**
 
      Specifies the predicates by which the rows produced by GROUP BY are filtered. The HAVING clause is used to
      filter rows after the grouping is performed. If HAVING is specified without GROUP BY, it indicates a GROUP BY
      without grouping expressions (global aggregate).
 
- * **ORDER BY**
+* **ORDER BY**
 
      Specifies an ordering of the rows of the complete result set of the query. The output rows are ordered
      across the partitions. This parameter is mutually exclusive with `SORT BY`,
      `CLUSTER BY` and `DISTRIBUTE BY` and can not be specified together.
 
- * **SORT BY**
+* **SORT BY**
 
      Specifies an ordering by which the rows are ordered within each partition. This parameter is mutually
      exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
 
- * **CLUSTER BY**
+* **CLUSTER BY**
 
      Specifies a set of expressions that is used to repartition and sort the rows. Using this clause has
      the same effect of using `DISTRIBUTE BY` and `SORT BY` together.
 
- * **DISTRIBUTE BY**
+* **DISTRIBUTE BY**
 
      Specifies a set of expressions by which the result rows are repartitioned. This parameter is mutually
      exclusive with `ORDER BY` and `CLUSTER BY` and can not be specified together.
 
- * **LIMIT**
+* **LIMIT**
 
      Specifies the maximum number of rows that can be returned by a statement or subquery. This clause
      is mostly used in the conjunction with `ORDER BY` to produce a deterministic result.
 
- * **boolean_expression**
+* **boolean_expression**
 
-     Specifies an expression with a return type of boolean.
+     Specifies any expression that evaluates to a result type `boolean`. Two or
+     more expressions may be combined together using the logical
+     operators ( `AND`, `OR` ).
 
- * **expression**
+* **expression**
 
      Specifies a combination of one or more values, operators, and SQL functions that evaluates to a value.
 
- * **named_window**
+* **named_window**
 
-      Specifies aliases for one or more source window specifications. The source window specifications can
-      be referenced in the widow definitions in the query.
+     Specifies aliases for one or more source window specifications. The source window specifications can
+     be referenced in the widow definitions in the query.
 
 ### Related Statements
 
@@ -159,3 +170,6 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
 * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
 * [Window Function](sql-ref-syntax-qry-select-window.html)
+* [CASE Clause](sql-ref-syntax-qry-select-case.html)
+* [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index 167c394d0fe49..d55ea43d15036 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -45,4 +45,7 @@ ability to generate logical and physical plan for a given query using
   * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
   * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
   * [Window Function](sql-ref-syntax-qry-select-window.html)
+  * [CASE Clause](sql-ref-syntax-qry-select-case.html)
+  * [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+  * [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
 * [EXPLAIN Statement](sql-ref-syntax-qry-explain.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 4bf1858428d98..4318cd69e7370 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -66,6 +66,9 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
    * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
    * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
    * [Window Function](sql-ref-syntax-qry-select-window.html)
+   * [CASE Clause](sql-ref-syntax-qry-select-case.html)
+   * [PIVOT Clause](sql-ref-syntax-qry-select-pivot.html)
+   * [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)
  * [EXPLAIN](sql-ref-syntax-qry-explain.html)
 
 ### Auxiliary Statements

From b35b3ebdb9b39a30d8e261bbf0a80482c149c2a8 Mon Sep 17 00:00:00 2001
From: Shantanu <12621235+hauntsaninja@users.noreply.github.com>
Date: Tue, 28 Jul 2020 11:22:18 +0900
Subject: [PATCH 0929/1280] [MINOR][PYTHON] Fix spacing in error message

### What changes were proposed in this pull request?
Fixes spacing in an error message

### Why are the changes needed?
Makes error messages easier to read

### Does this PR introduce _any_ user-facing change?
Yes, it changes the error message

### How was this patch tested?
This patch doesn't affect any logic, so existing tests should cover it

Closes #29264 from hauntsaninja/patch-1.

Authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 77f2ca6cced1c723d1c2e6082a1534f6436c6d2a)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 5f4a8a2d2db1f..814f796c0bf1c 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -471,7 +471,7 @@ def main(infile, outfile):
         version = utf8_deserializer.loads(infile)
         if version != "%d.%d" % sys.version_info[:2]:
             raise Exception(("Python in worker has different version %s than that in " +
-                             "driver %s, PySpark cannot run with different minor versions." +
+                             "driver %s, PySpark cannot run with different minor versions. " +
                              "Please check environment variables PYSPARK_PYTHON and " +
                              "PYSPARK_DRIVER_PYTHON are correctly set.") %
                             ("%d.%d" % sys.version_info[:2], version))

From f349a78bbbc8791f010a6cf9f789c14f677eef43 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 28 Jul 2020 10:03:39 +0000
Subject: [PATCH 0930/1280] [SPARK-32424][SQL][3.0] Fix silent data change for
 timestamp parsing if f overflow happens
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR backports https://github.com/apache/spark/commit/d315ebf3a739a05a68d0f0ab319920765bf65b0f to branch-3.0
### What changes were proposed in this pull request?

When using `Seconds.toMicros` API to convert epoch seconds to microseconds,

```scala
 /**
     * Equivalent to
     * {link #convert(long, TimeUnit) MICROSECONDS.convert(duration, this)}.
     * param duration the duration
     * return the converted duration,
     * or {code Long.MIN_VALUE} if conversion would negatively
     * overflow, or {code Long.MAX_VALUE} if it would positively overflow.
     */
```
This PR change it to `Math.multiplyExact(epochSeconds, MICROS_PER_SECOND)`

### Why are the changes needed?

fix silent data change between 3.x and 2.x
```
 ~/Downloads/spark/spark-3.1.0-SNAPSHOT-bin-20200722   bin/spark-sql -S -e "select to_timestamp('300000', 'y');"
+294247-01-10 12:00:54.775807
```
```
 kentyaohulk  ~/Downloads/spark/spark-2.4.5-bin-hadoop2.7  bin/spark-sql -S  -e "select to_timestamp('300000', 'y');"
284550-10-19 15:58:1010.448384
```

### Does this PR introduce _any_ user-facing change?

Yes, we will raise `ArithmeticException` instead of giving the wrong answer if overflow.

### How was this patch tested?

add unit test

Closes #29267 from yaooqinn/SPARK-32424-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-ref-datetime-pattern.md              |  2 +-
 .../util/DateTimeFormatterHelper.scala        |  4 +-
 .../catalyst/util/TimestampFormatter.scala    |  3 +-
 .../util/TimestampFormatterSuite.scala        | 10 ++
 .../inputs/datetime-formatting-invalid.sql    |  4 +-
 .../sql-tests/inputs/datetime-formatting.sql  |  2 +-
 .../inputs/datetime-parsing-invalid.sql       |  9 ++
 .../sql-tests/inputs/datetime-parsing.sql     | 28 ++++++
 .../resources/sql-tests/inputs/datetime.sql   |  4 -
 .../sql-tests/results/ansi/datetime.sql.out   | 28 +-----
 .../datetime-formatting-invalid.sql.out       |  4 +-
 .../datetime-formatting-legacy.sql.out        | 18 ++--
 .../results/datetime-formatting.sql.out       | 18 ++--
 .../sql-tests/results/datetime-legacy.sql.out | 26 +----
 .../results/datetime-parsing-invalid.sql.out  | 54 +++++++++-
 .../results/datetime-parsing-legacy.sql.out   | 98 ++++++++++++++++++-
 .../results/datetime-parsing.sql.out          | 98 ++++++++++++++++++-
 .../sql-tests/results/datetime.sql.out        | 28 +-----
 18 files changed, 325 insertions(+), 113 deletions(-)

diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index 3c0bc754f940b..d0299e5a99dc2 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -70,7 +70,7 @@ The count of pattern letters determines the format.
   For formatting, the fraction length would be padded to the number of contiguous 'S' with zeros.
   Spark supports datetime of micro-of-second precision, which has up to 6 significant digits, but can parse nano-of-second with exceeded part truncated.
 
-- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present. 11 or more letters will fail.
+- Year: The count of letters determines the minimum field width below which padding is used. If the count of letters is two, then a reduced two digit form is used. For printing, this outputs the rightmost two digits. For parsing, this will parse using the base value of 2000, resulting in a year within the range 2000 to 2099 inclusive. If the count of letters is less than four (but not two), then the sign is only output for negative years. Otherwise, the sign is output if the pad width is exceeded when 'G' is not present. 7 or more letters will fail.
 
 - Month: It follows the rule of Number/Text. The text form is depend on letters - 'M' denotes the 'standard' form, and 'L' is for 'stand-alone' form. These two forms are different only in some certain languages. For example, in Russian, 'Июль' is the stand-alone form of July, and 'Июля' is the standard form. Here are examples for all supported pattern letters:
   - `'M'` or `'L'`: Month number in a year starting from 1. There is no difference between 'M' and 'L'. Month from 1 to 9 are printed without padding.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
index 5de06af6af12a..f02b2d08c0935 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
@@ -296,7 +296,9 @@ private object DateTimeFormatterHelper {
       // unchecked `ArrayIndexOutOfBoundsException` by the `NumberPrinterParser` for formatting. It
       // makes the call side difficult to handle exceptions and easily leads to silent data change
       // because of the exceptions being suppressed.
-      Seq("y").map(_ * 11)
+      // SPARK-32424: The max year that we can actually handle is 6 digits, otherwise, it will
+      // overflow
+      Seq("y").map(_ * 7)
   }.toSet
 
   /**
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
index 87d8a3e682b8e..093ebb27e0dc5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala
@@ -24,7 +24,6 @@ import java.time.format.{DateTimeFormatter, DateTimeParseException}
 import java.time.temporal.ChronoField.MICRO_OF_SECOND
 import java.time.temporal.TemporalQueries
 import java.util.{Calendar, GregorianCalendar, Locale, TimeZone}
-import java.util.concurrent.TimeUnit.SECONDS
 
 import org.apache.commons.lang3.time.FastDateFormat
 
@@ -83,7 +82,7 @@ class Iso8601TimestampFormatter(
         val epochSeconds = zonedDateTime.toEpochSecond
         val microsOfSecond = zonedDateTime.get(MICRO_OF_SECOND)
 
-        Math.addExact(SECONDS.toMicros(epochSeconds), microsOfSecond)
+        Math.addExact(Math.multiplyExact(epochSeconds, MICROS_PER_SECOND), microsOfSecond)
       } catch checkParsedDiff(s, legacyFormatter.parse)
     }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
index e7d935001a76f..46b216accbadb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala
@@ -432,4 +432,14 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite {
       assert(formatter.format(date(1970, 4, 10)) == "100")
     }
   }
+
+  test("SPARK-32424: avoid silent data change when timestamp overflows") {
+    val formatter = TimestampFormatter("y", UTC, isParsing = true)
+    assert(formatter.parse("294247") === date(294247))
+    assert(formatter.parse("-290307") === date(-290307))
+    val e1 = intercept[ArithmeticException](formatter.parse("294248"))
+    assert(e1.getMessage === "long overflow")
+    val e2 = intercept[ArithmeticException](formatter.parse("-290308"))
+    assert(e2.getMessage === "long overflow")
+  }
 }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
index 9072aa107f252..11bba00e91abf 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting-invalid.sql
@@ -3,8 +3,8 @@
 -- separating this from datetime-formatting.sql, because the text form
 -- for patterns with 5 letters in SimpleDateFormat varies from different JDKs
 select date_format('2018-11-17 13:33:33.333', 'GGGGG');
--- pattern letter count can not be greater than 10
-select date_format('2018-11-17 13:33:33.333', 'yyyyyyyyyyy');
+-- pattern letter count can not be greater than 6
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyy');
 -- q/L in JDK 8 will fail when the count is more than 2
 select date_format('2018-11-17 13:33:33.333', 'qqqqq');
 select date_format('2018-11-17 13:33:33.333', 'QQQQQ');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
index 3b23a7785f6cd..2d70326f4f3c0 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-formatting.sql
@@ -11,7 +11,7 @@ create temporary view v as select col from values
 
 select col, date_format(col, 'G GG GGG GGGG') from v;
 
-select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy') from v;
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v;
 
 select col, date_format(col, 'q qq') from v;
 
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
index 0b313e5a0b9f4..a1c02eaa3b0a0 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing-invalid.sql
@@ -1,5 +1,14 @@
 --- TESTS FOR DATETIME PARSING FUNCTIONS WITH INVALID VALUES ---
 
+-- parsing invalid value with pattern 'y'
+select to_timestamp('294248', 'y'); -- out of year value range [0, 294247]
+select to_timestamp('1', 'yy'); -- the number of digits must be 2 for 'yy'.
+select to_timestamp('-12', 'yy'); -- out of year value range [0, 99] for reduced two digit form
+select to_timestamp('123', 'yy'); -- the number of digits must be 2 for 'yy'.
+select to_timestamp('1', 'yyy'); -- the number of digits must be in [3, 6] for 'yyy'
+
+select to_timestamp('1234567', 'yyyyyyy'); -- the length of 'y' pattern must be less than 7
+
 -- parsing invalid values with pattern 'D'
 select to_timestamp('366', 'D');
 select to_timestamp('9', 'DD');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
index 74866d9c6ffa1..e058bd675c375 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime-parsing.sql
@@ -1,5 +1,33 @@
 --- TESTS FOR DATETIME PARSING FUNCTIONS ---
 
+-- parsing with pattern 'y'.
+-- the range of valid year is [-290307, 294247],
+-- but particularly, some thrift client use java.sql.Timestamp to parse timestamp, which allows
+-- only positive year values less or equal than 9999. So the cases bellow only use [1, 9999] to pass
+-- ThriftServerQueryTestSuite
+select to_timestamp('1', 'y');
+select to_timestamp('009999', 'y');
+
+-- reduced two digit form is used, the range of valid year is 20-[01, 99]
+select to_timestamp('00', 'yy');
+select to_timestamp('99', 'yy');
+
+-- the range of valid year is [-290307, 294247], the number of digits must be in [3, 6] for 'yyy'
+select to_timestamp('001', 'yyy');
+select to_timestamp('009999', 'yyy');
+
+-- the range of valid year is [-9999, 9999], the number of digits must be 4 for 'yyyy'.
+select to_timestamp('0001', 'yyyy');
+select to_timestamp('9999', 'yyyy');
+
+-- the range of valid year is [-99999, 99999], the number of digits must be 5 for 'yyyyy'.
+select to_timestamp('00001', 'yyyyy');
+select to_timestamp('09999', 'yyyyy');
+
+-- the range of valid year is [-290307, 294247], the number of digits must be 6 for 'yyyyyy'.
+select to_timestamp('000001', 'yyyyyy');
+select to_timestamp('009999', 'yyyyyy');
+
 -- parsing with pattern 'D'
 select to_timestamp('9', 'D');
 select to_timestamp('300', 'D');
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index e4a2e5d8c8f3f..db54b5508ec71 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -146,7 +146,3 @@ select from_json('{"time":"26/October/2015"}', 'time Timestamp', map('timestampF
 select from_json('{"date":"26/October/2015"}', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
 select from_csv('26/October/2015', 'time Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'));
 select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy'));
-
-select from_unixtime(1, 'yyyyyyyyyyy-MM-dd');
-select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss');
-select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd');
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index 1cb1aaa6584db..36674040f1302 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 109
+-- Number of queries: 106
 
 
 -- !query
@@ -913,29 +913,3 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
-
--- !query
-select from_unixtime(1, 'yyyyyyyyyyy-MM-dd')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
-
--- !query
-select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss')
--- !query schema
-struct<date_format(TIMESTAMP '2018-11-17 13:33:33', yyyyyyyyyy-MM-dd HH:mm:ss):string>
--- !query output
-0000002018-11-17 13:33:33
-
-
--- !query
-select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
index 248157efacde0..18d1a10068794 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-invalid.sql.out
@@ -12,12 +12,12 @@ You may get a different result due to the upgrading of Spark 3.0: Fail to recogn
 
 
 -- !query
-select date_format('2018-11-17 13:33:33.333', 'yyyyyyyyyyy')
+select date_format('2018-11-17 13:33:33.333', 'yyyyyyy')
 -- !query schema
 struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
index b7bc448a952a7..b37922b20807d 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting-legacy.sql.out
@@ -32,17 +32,17 @@ struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
 
 
 -- !query
-select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy') from v
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v
 -- !query schema
-struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy):string>
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy):string>
 -- !query output
-1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582 0001582 00001582 000001582 0000001582
-1969-12-31 15:00:00	1969 69 1969 1969 01969 001969 0001969 00001969 000001969 0000001969
-1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970 0001970 00001970 000001970 0000001970
-1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996 0001996 00001996 000001996 0000001996
-2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018 0002018 00002018 000002018 0000002018
-2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019 0002019 00002019 000002019 0000002019
-2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100 0002100 00002100 000002100 0000002100
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
index f724658d354df..5bed88e168f1e 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-formatting.sql.out
@@ -32,17 +32,17 @@ struct<col:timestamp,date_format(col, G GG GGG GGGG):string>
 
 
 -- !query
-select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy') from v
+select col, date_format(col, 'y yy yyy yyyy yyyyy yyyyyy') from v
 -- !query schema
-struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy yyyyyyy yyyyyyyy yyyyyyyyy yyyyyyyyyy):string>
+struct<col:timestamp,date_format(col, y yy yyy yyyy yyyyy yyyyyy):string>
 -- !query output
-1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582 0001582 00001582 000001582 0000001582
-1969-12-31 15:00:00	1969 69 1969 1969 01969 001969 0001969 00001969 000001969 0000001969
-1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970 0001970 00001970 000001970 0000001970
-1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996 0001996 00001996 000001996 0000001996
-2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018 0002018 00002018 000002018 0000002018
-2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019 0002019 00002019 000002019 0000002019
-2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100 0002100 00002100 000002100 0000002100
+1582-05-31 19:40:35.123	1582 82 1582 1582 01582 001582
+1969-12-31 15:00:00	1969 69 1969 1969 01969 001969
+1970-12-31 04:59:59.999	1970 70 1970 1970 01970 001970
+1996-03-31 07:03:33.123	1996 96 1996 1996 01996 001996
+2018-11-17 05:33:33.123	2018 18 2018 2018 02018 002018
+2019-12-31 09:33:33.123	2019 19 2019 2019 02019 002019
+2100-01-01 01:33:33.123	2100 00 2100 2100 02100 002100
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
index b244f497aeacc..1342c71feecfa 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 109
+-- Number of queries: 106
 
 
 -- !query
@@ -870,27 +870,3 @@ select from_csv('26/October/2015', 'date Date', map('dateFormat', 'dd/MMMMM/yyyy
 struct<from_csv(26/October/2015):struct<date:date>>
 -- !query output
 {"date":2015-10-26}
-
-
--- !query
-select from_unixtime(1, 'yyyyyyyyyyy-MM-dd')
--- !query schema
-struct<from_unixtime(CAST(1 AS BIGINT), yyyyyyyyyyy-MM-dd):string>
--- !query output
-00000001969-12-31
-
-
--- !query
-select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss')
--- !query schema
-struct<date_format(TIMESTAMP '2018-11-17 13:33:33', yyyyyyyyyy-MM-dd HH:mm:ss):string>
--- !query output
-0000002018-11-17 13:33:33
-
-
--- !query
-select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
--- !query schema
-struct<date_format(CAST(DATE '2018-11-17' AS TIMESTAMP), yyyyyyyyyyy-MM-dd):string>
--- !query output
-00000002018-11-17
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
index 22f21c7b2073a..76008d9c97dc6 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-invalid.sql.out
@@ -1,5 +1,57 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 19
+
+
+-- !query
+select to_timestamp('294248', 'y')
+-- !query schema
+struct<to_timestamp('294248', 'y'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('1', 'yy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('-12', 'yy')
+-- !query schema
+struct<to_timestamp('-12', 'yy'):timestamp>
+-- !query output
+NULL
+
+
+-- !query
+select to_timestamp('123', 'yy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '123' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('1', 'yyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to parse '1' in the new parser. You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0, or set to CORRECTED and treat it as an invalid datetime string.
+
+
+-- !query
+select to_timestamp('1234567', 'yyyyyyy')
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkUpgradeException
+You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
index 58443f1de38ca..258afe43b99ee 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing-legacy.sql.out
@@ -1,5 +1,101 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 25
+
+
+-- !query
+select to_timestamp('1', 'y')
+-- !query schema
+struct<to_timestamp('1', 'y'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'y')
+-- !query schema
+struct<to_timestamp('009999', 'y'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00', 'yy')
+-- !query schema
+struct<to_timestamp('00', 'yy'):timestamp>
+-- !query output
+2000-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'yy')
+-- !query schema
+struct<to_timestamp('99', 'yy'):timestamp>
+-- !query output
+1999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('001', 'yyy')
+-- !query schema
+struct<to_timestamp('001', 'yyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('0001', 'yyyy')
+-- !query schema
+struct<to_timestamp('0001', 'yyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9999', 'yyyy')
+-- !query schema
+struct<to_timestamp('9999', 'yyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00001', 'yyyyy')
+-- !query schema
+struct<to_timestamp('00001', 'yyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('09999', 'yyyyy')
+-- !query schema
+struct<to_timestamp('09999', 'yyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('000001', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('000001', 'yyyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
index 58443f1de38ca..f030141d288ec 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-parsing.sql.out
@@ -1,5 +1,101 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 13
+-- Number of queries: 25
+
+
+-- !query
+select to_timestamp('1', 'y')
+-- !query schema
+struct<to_timestamp('1', 'y'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'y')
+-- !query schema
+struct<to_timestamp('009999', 'y'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00', 'yy')
+-- !query schema
+struct<to_timestamp('00', 'yy'):timestamp>
+-- !query output
+2000-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('99', 'yy')
+-- !query schema
+struct<to_timestamp('99', 'yy'):timestamp>
+-- !query output
+2099-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('001', 'yyy')
+-- !query schema
+struct<to_timestamp('001', 'yyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('0001', 'yyyy')
+-- !query schema
+struct<to_timestamp('0001', 'yyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('9999', 'yyyy')
+-- !query schema
+struct<to_timestamp('9999', 'yyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('00001', 'yyyyy')
+-- !query schema
+struct<to_timestamp('00001', 'yyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('09999', 'yyyyy')
+-- !query schema
+struct<to_timestamp('09999', 'yyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('000001', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('000001', 'yyyyyy'):timestamp>
+-- !query output
+0001-01-01 00:00:00
+
+
+-- !query
+select to_timestamp('009999', 'yyyyyy')
+-- !query schema
+struct<to_timestamp('009999', 'yyyyyy'):timestamp>
+-- !query output
+9999-01-01 00:00:00
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index 3255761545fdd..d3657e874343b 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 109
+-- Number of queries: 106
 
 
 -- !query
@@ -885,29 +885,3 @@ struct<>
 -- !query output
 org.apache.spark.SparkUpgradeException
 You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
-
--- !query
-select from_unixtime(1, 'yyyyyyyyyyy-MM-dd')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html
-
-
--- !query
-select date_format(timestamp '2018-11-17 13:33:33', 'yyyyyyyyyy-MM-dd HH:mm:ss')
--- !query schema
-struct<date_format(TIMESTAMP '2018-11-17 13:33:33', yyyyyyyyyy-MM-dd HH:mm:ss):string>
--- !query output
-0000002018-11-17 13:33:33
-
-
--- !query
-select date_format(date '2018-11-17', 'yyyyyyyyyyy-MM-dd')
--- !query schema
-struct<>
--- !query output
-org.apache.spark.SparkUpgradeException
-You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'yyyyyyyyyyy-MM-dd' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html

From 8cfb7183865c5358a547ec892f10d4f1350300ff Mon Sep 17 00:00:00 2001
From: Xiaochang Wu <xiaochang.wu@intel.com>
Date: Tue, 28 Jul 2020 08:36:11 -0700
Subject: [PATCH 0931/1280] [SPARK-32339][ML][DOC] Improve MLlib BLAS native
 acceleration docs

### What changes were proposed in this pull request?
Rewrite a clearer and complete BLAS native acceleration enabling guide.

### Why are the changes needed?
The document of enabling BLAS native acceleration in ML guide (https://spark.apache.org/docs/latest/ml-guide.html#dependencies) is incomplete and unclear to the user.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
N/A

Closes #29139 from xwu99/blas-doc.

Lead-authored-by: Xiaochang Wu <xiaochang.wu@intel.com>
Co-authored-by: Wu, Xiaochang <xiaochang.wu@intel.com>
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
(cherry picked from commit 44c868b73a7cb293ec81927c28991677bf33ea90)
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
---
 docs/ml-guide.md        |  22 +++------
 docs/ml-linalg-guide.md | 103 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+), 16 deletions(-)
 create mode 100644 docs/ml-linalg-guide.md

diff --git a/docs/ml-guide.md b/docs/ml-guide.md
index ddce98b32f941..1b4a3e4eee557 100644
--- a/docs/ml-guide.md
+++ b/docs/ml-guide.md
@@ -62,23 +62,13 @@ The primary Machine Learning API for Spark is now the [DataFrame](sql-programmin
 
 # Dependencies
 
-MLlib uses the linear algebra package [Breeze](http://www.scalanlp.org/), which depends on
-[netlib-java](https://github.com/fommil/netlib-java) for optimised numerical processing.
-If native libraries[^1] are not available at runtime, you will see a warning message and a pure JVM
-implementation will be used instead.
+MLlib uses linear algebra packages [Breeze](http://www.scalanlp.org/) and [netlib-java](https://github.com/fommil/netlib-java) for optimised numerical processing[^1]. Those packages may call native acceleration libraries such as [Intel MKL](https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html) or [OpenBLAS](http://www.openblas.net) if they are available as system libraries or in runtime library paths. 
 
-Due to licensing issues with runtime proprietary binaries, we do not include `netlib-java`'s native
-proxies by default.
-To configure `netlib-java` / Breeze to use system optimised binaries, include
-`com.github.fommil.netlib:all:1.1.2` (or build Spark with `-Pnetlib-lgpl`) as a dependency of your
-project and read the [netlib-java](https://github.com/fommil/netlib-java) documentation for your
-platform's additional installation instructions.
-
-The most popular native BLAS such as [Intel MKL](https://software.intel.com/en-us/mkl), [OpenBLAS](http://www.openblas.net), can use multiple threads in a single operation, which can conflict with Spark's execution model.
-
-Configuring these BLAS implementations to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of cores each Spark task is configured to use, which is 1 by default and typically left at 1.
-
-Please refer to resources like the following to understand how to configure the number of threads these BLAS implementations use: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) or [Intel oneMKL](https://software.intel.com/en-us/onemkl-linux-developer-guide-improving-performance-with-threading) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded). Note that if nativeBLAS is not properly configured in system, java implementation(f2jBLAS) will be used as fallback option.
+Due to differing OSS licenses, `netlib-java`'s native proxies can't be distributed with Spark. See [MLlib Linear Algebra Acceleration Guide](ml-linalg-guide.html) for how to enable accelerated linear algebra processing. If accelerated native libraries are not enabled, you will see a warning message like below and a pure JVM implementation will be used instead:
+```
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeSystemBLAS
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeRefBLAS
+```
 
 To use MLlib in Python, you will need [NumPy](http://www.numpy.org) version 1.4 or newer.
 
diff --git a/docs/ml-linalg-guide.md b/docs/ml-linalg-guide.md
new file mode 100644
index 0000000000000..739091363473f
--- /dev/null
+++ b/docs/ml-linalg-guide.md
@@ -0,0 +1,103 @@
+---
+layout: global
+title: MLlib Linear Algebra Acceleration Guide
+displayTitle: MLlib Linear Algebra Acceleration Guide
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+     http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+## Introduction
+
+This guide provides necessary information to enable accelerated linear algebra processing for Spark MLlib.
+
+Spark MLlib defines Vector and Matrix as basic data types for machine learning algorithms. On top of them, [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) operations are implemented and supported by [netlib-java](https://github.com/fommil/netlib-Java) (the algorithms may call [Breeze](https://github.com/scalanlp/breeze) and it will in turn call `netlib-java`). `netlib-java` can use optimized native linear algebra libraries (refered to as "native libraries" or "BLAS libraries" hereafter) for faster numerical processing. [Intel MKL](https://software.intel.com/content/www/us/en/develop/tools/math-kernel-library.html) and [OpenBLAS](http://www.openblas.net) are two popular ones.
+
+However due to license differences, the official released Spark binaries by default don't contain native libraries support for `netlib-java`.
+
+The following sections describe how to enable `netlib-java` with native libraries support for Spark MLlib and how to install native libraries and configure them properly.
+
+## Enable `netlib-java` with native library proxies 
+
+`netlib-java` depends on `libgfortran`. It requires GFORTRAN 1.4 or above. This can be obtained by installing `libgfortran` package. After installation, the following command can be used to verify if it is installed properly.
+```
+strings /path/to/libgfortran.so.3.0.0 | grep GFORTRAN_1.4
+```
+
+To build Spark with `netlib-java` native library proxies, you need to add `-Pnetlib-lgpl` to Maven build command line. For example:
+```
+$SPARK_SOURCE_HOME/build/mvn -Pnetlib-lgpl -DskipTests -Pyarn -Phadoop-2.7 clean package
+```
+
+If you only want to enable it in your project, include `com.github.fommil.netlib:all:1.1.2` as a dependency of your project.
+
+## Install native linear algebra libraries
+
+Intel MKL and OpenBLAS are two popular native linear algebra libraries. You can choose one of them based on your preference. We provide basic instructions as below. You can refer to [netlib-java documentation](https://github.com/fommil/netlib-java) for more advanced installation instructions.
+
+### Intel MKL
+
+- Download and install Intel MKL. The installation should be done on all nodes of the cluster. We assume the installation location is $MKLROOT (e.g. /opt/intel/mkl).
+- Create soft links to `libmkl_rt.so` with specific names in system library search paths. For instance, make sure `/usr/local/lib` is in system library search paths and run the following commands:
+```
+$ ln -sf $MKLROOT/lib/intel64/libmkl_rt.so /usr/local/lib/libblas.so.3
+$ ln -sf $MKLROOT/lib/intel64/libmkl_rt.so /usr/local/lib/liblapack.so.3
+```
+
+### OpenBLAS
+
+The installation should be done on all nodes of the cluster. Generic version of OpenBLAS are available with most distributions. You can install it with a distribution package manager like `apt` or `yum`.
+
+For Debian / Ubuntu:
+```
+sudo apt-get install libopenblas-base
+sudo update-alternatives --config libblas.so.3
+```
+For CentOS / RHEL:
+```
+sudo yum install openblas
+```
+
+## Check if native libraries are enabled for MLlib
+
+To verify native libraries are properly loaded, start `spark-shell` and run the following code:
+```
+scala> import com.github.fommil.netlib.BLAS;
+scala> System.out.println(BLAS.getInstance().getClass().getName());
+```
+
+If they are correctly loaded, it should print `com.github.fommil.netlib.NativeSystemBLAS`. Otherwise the warnings should be printed:
+```
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeSystemBLAS
+WARN BLAS: Failed to load implementation from:com.github.fommil.netlib.NativeRefBLAS
+```
+
+If native libraries are not properly configured in the system, the Java implementation (f2jBLAS) will be used as fallback option.
+
+## Spark Configuration
+
+The default behavior of multi-threading in either Intel MKL or OpenBLAS may not be optimal with Spark's execution model [^1].
+
+Therefore configuring these native libraries to use a single thread for operations may actually improve performance (see [SPARK-21305](https://issues.apache.org/jira/browse/SPARK-21305)). It is usually optimal to match this to the number of `spark.task.cpus`, which is `1` by default and typically left at `1`.
+
+You can use the options in `config/spark-env.sh` to set thread number for Intel MKL or OpenBLAS:
+* For Intel MKL:
+```
+MKL_NUM_THREADS=1
+```
+* For OpenBLAS:
+```
+OPENBLAS_NUM_THREADS=1
+```
+
+[^1]: Please refer to the following resources to understand how to configure the number of threads for these BLAS implementations: [Intel MKL](https://software.intel.com/en-us/articles/recommended-settings-for-calling-intel-mkl-routines-from-multi-threaded-applications) or [Intel oneMKL](https://software.intel.com/en-us/onemkl-linux-developer-guide-improving-performance-with-threading) and [OpenBLAS](https://github.com/xianyi/OpenBLAS/wiki/faq#multi-threaded).

From 9f18d54ab26d64a23438bd242e77ba27311495ec Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Wed, 29 Jul 2020 03:58:03 +0000
Subject: [PATCH 0932/1280] [SPARK-32283][CORE] Kryo should support multiple
 user registrators

### What changes were proposed in this pull request?
`spark.kryo.registrator` in 3.0 has a regression problem. From [SPARK-12080](https://issues.apache.org/jira/browse/SPARK-12080), it supports multiple user registrators by
```scala
private val userRegistrators = conf.get("spark.kryo.registrator", "")
    .split(',').map(_.trim)
    .filter(!_.isEmpty)
```
But it donsn't work in 3.0. Fix it by `toSequence` in `Kryo.scala`

### Why are the changes needed?
In previous Spark version (2.x), it supported multiple user registrators by
```scala
private val userRegistrators = conf.get("spark.kryo.registrator", "")
    .split(',').map(_.trim)
    .filter(!_.isEmpty)
```
But it doesn't work in 3.0. It's should be a regression.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Existed unit tests.

Closes #29123 from LantaoJin/SPARK-32283.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 26e6574d58429add645db820a83b70ef9dcd49fe)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/internal/config/Kryo.scala  |  3 ++-
 .../test/scala/org/apache/spark/SparkConfSuite.scala   |  2 +-
 .../org/apache/spark/serializer/KryoBenchmark.scala    |  2 +-
 .../spark/serializer/KryoSerializerBenchmark.scala     |  2 +-
 .../serializer/KryoSerializerDistributedSuite.scala    |  2 +-
 .../apache/spark/serializer/KryoSerializerSuite.scala  | 10 +++++-----
 .../spark/serializer/SerializerPropertiesSuite.scala   |  2 +-
 .../spark/sql/DatasetSerializerRegistratorSuite.scala  |  2 +-
 8 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
index 646d8556538c2..90c59b079461c 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala
@@ -29,7 +29,8 @@ private[spark] object Kryo {
   val KRYO_USER_REGISTRATORS = ConfigBuilder("spark.kryo.registrator")
     .version("0.5.0")
     .stringConf
-    .createOptional
+    .toSequence
+    .createWithDefault(Nil)
 
   val KRYO_CLASSES_TO_REGISTER = ConfigBuilder("spark.kryo.classesToRegister")
     .version("1.2.0")
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 3bc2061c4f2ad..72e7ee0214187 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -221,7 +221,7 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
     conf.registerKryoClasses(Array(classOf[Class1]))
     assert(conf.get(KRYO_CLASSES_TO_REGISTER).toSet === Seq(classOf[Class1].getName).toSet)
 
-    conf.set(KRYO_USER_REGISTRATORS, classOf[CustomRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[CustomRegistrator].getName))
 
     // Kryo doesn't expose a way to discover registered classes, but at least make sure this doesn't
     // blow up.
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
index fd228cded783a..525e682dd5d42 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoBenchmark.scala
@@ -125,7 +125,7 @@ object KryoBenchmark extends BenchmarkBase {
   def createSerializer(useUnsafe: Boolean): SerializerInstance = {
     val conf = new SparkConf()
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-    conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
     conf.set(KRYO_USE_UNSAFE, useUnsafe)
 
     new KryoSerializer(conf).newInstance()
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
index 953b651c72a83..dde0c98704d00 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerBenchmark.scala
@@ -76,7 +76,7 @@ object KryoSerializerBenchmark extends BenchmarkBase {
     conf.set(EXECUTOR_EXTRA_JAVA_OPTIONS,
       "-XX:+UseParallelGC -XX:-UseDynamicNumberOfGCThreads")
     conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-    conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+    conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
     conf.set(KRYO_USE_POOL, usePool)
 
     if (sc != null) {
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
index d4fafab4a5d64..397fdce8ae6e3 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerDistributedSuite.scala
@@ -29,7 +29,7 @@ class KryoSerializerDistributedSuite extends SparkFunSuite with LocalSparkContex
   test("kryo objects are serialised consistently in different processes") {
     val conf = new SparkConf(false)
       .set(config.SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-      .set(config.Kryo.KRYO_USER_REGISTRATORS, classOf[AppJarRegistrator].getName)
+      .set(config.Kryo.KRYO_USER_REGISTRATORS, Seq(classOf[AppJarRegistrator].getName))
       .set(config.TASK_MAX_FAILURES, 1)
       .set(config.BLACKLIST_ENABLED, false)
 
diff --git a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index 4c47a67ee9ffc..229ef69973775 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -42,7 +42,7 @@ import org.apache.spark.util.ThreadUtils
 
 class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
   conf.set(SERIALIZER, "org.apache.spark.serializer.KryoSerializer")
-  conf.set(KRYO_USER_REGISTRATORS, classOf[MyRegistrator].getName)
+  conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[MyRegistrator].getName))
   conf.set(KRYO_USE_UNSAFE, false)
 
   test("SPARK-7392 configuration limits") {
@@ -313,7 +313,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
     import org.apache.spark.SparkException
 
     val conf = new SparkConf(false)
-    conf.set(KRYO_USER_REGISTRATORS, "this.class.does.not.exist")
+    conf.set(KRYO_USER_REGISTRATORS, Seq("this.class.does.not.exist"))
 
     val thrown = intercept[SparkException](new KryoSerializer(conf).newInstance().serialize(1))
     assert(thrown.getMessage.contains("Failed to register classes with Kryo"))
@@ -412,7 +412,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
     val ser = new KryoSerializer(new SparkConf).newInstance().asInstanceOf[KryoSerializerInstance]
     assert(ser.getAutoReset)
     val conf = new SparkConf().set(KRYO_USER_REGISTRATORS,
-      classOf[RegistratorWithoutAutoReset].getName)
+      Seq(classOf[RegistratorWithoutAutoReset].getName))
     val ser2 = new KryoSerializer(conf).newInstance().asInstanceOf[KryoSerializerInstance]
     assert(!ser2.getAutoReset)
   }
@@ -443,7 +443,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
       .set(KRYO_REFERENCE_TRACKING, referenceTracking)
       .set(KRYO_USE_POOL, usePool)
     if (!autoReset) {
-      conf.set(KRYO_USER_REGISTRATORS, classOf[RegistratorWithoutAutoReset].getName)
+      conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[RegistratorWithoutAutoReset].getName))
     }
     val ser = new KryoSerializer(conf)
     val serInstance = ser.newInstance().asInstanceOf[KryoSerializerInstance]
@@ -530,7 +530,7 @@ class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
 
 class KryoSerializerAutoResetDisabledSuite extends SparkFunSuite with SharedSparkContext {
   conf.set(SERIALIZER, classOf[KryoSerializer].getName)
-  conf.set(KRYO_USER_REGISTRATORS, classOf[RegistratorWithoutAutoReset].getName)
+  conf.set(KRYO_USER_REGISTRATORS, Seq(classOf[RegistratorWithoutAutoReset].getName))
   conf.set(KRYO_REFERENCE_TRACKING, true)
   conf.set(SHUFFLE_MANAGER, "sort")
   conf.set(SHUFFLE_SORT_BYPASS_MERGE_THRESHOLD, 200)
diff --git a/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala b/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
index dad080c5fc161..9747f5780dd1e 100644
--- a/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/SerializerPropertiesSuite.scala
@@ -52,7 +52,7 @@ class SerializerPropertiesSuite extends SparkFunSuite {
 
   test("KryoSerializer does not support relocation when auto-reset is disabled") {
     val conf = new SparkConf().set(KRYO_USER_REGISTRATORS,
-      classOf[RegistratorWithoutAutoReset].getName)
+      Seq(classOf[RegistratorWithoutAutoReset].getName))
     val ser = new KryoSerializer(conf)
     assert(!ser.newInstance().asInstanceOf[KryoSerializerInstance].getAutoReset())
     testSupportsRelocationOfSerializedObjects(ser, generateRandomItem)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
index 43de2663b1236..b20d050f2fc4a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSerializerRegistratorSuite.scala
@@ -34,7 +34,7 @@ class DatasetSerializerRegistratorSuite extends QueryTest with SharedSparkSessio
 
   override protected def sparkConf: SparkConf = {
     // Make sure we use the KryoRegistrator
-    super.sparkConf.set(KRYO_USER_REGISTRATORS, TestRegistrator().getClass.getCanonicalName)
+    super.sparkConf.set(KRYO_USER_REGISTRATORS, Seq(TestRegistrator().getClass.getCanonicalName))
   }
 
   test("Kryo registrator") {

From e5b5b7e507ab974bbca3abb0bbf56bf67696d53e Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 29 Jul 2020 08:44:56 -0500
Subject: [PATCH 0933/1280] [SPARK-32175][CORE] Fix the order between
 initialization for ExecutorPlugin and starting heartbeat thread

### What changes were proposed in this pull request?

This PR changes the order between initialization for ExecutorPlugin and starting heartbeat thread in Executor.

### Why are the changes needed?

In the current master, heartbeat thread in a executor starts after plugin initialization so if the initialization takes long time, heartbeat is not sent to driver and the executor will be removed from cluster.

### Does this PR introduce _any_ user-facing change?

Yes. Plugins for executors will be allowed to take long time for initialization.

### How was this patch tested?

New testcase.

Closes #29002 from sarutak/fix-heartbeat-issue.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit 9be088357eff4328248b29a3a49a816756745345)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../scala/org/apache/spark/TestUtils.scala    | 15 +++-
 .../org/apache/spark/executor/Executor.scala  | 12 ++--
 .../apache/spark/executor/ExecutorSuite.scala | 72 ++++++++++++++++++-
 3 files changed, 89 insertions(+), 10 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index d459627930f4c..1e00769260a5f 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -179,11 +179,20 @@ private[spark] object TestUtils {
       destDir: File,
       toStringValue: String = "",
       baseClass: String = null,
-      classpathUrls: Seq[URL] = Seq.empty): File = {
+      classpathUrls: Seq[URL] = Seq.empty,
+      implementsClasses: Seq[String] = Seq.empty,
+      extraCodeBody: String = ""): File = {
     val extendsText = Option(baseClass).map { c => s" extends ${c}" }.getOrElse("")
+    val implementsText =
+      "implements " + (implementsClasses :+ "java.io.Serializable").mkString(", ")
     val sourceFile = new JavaSourceFromString(className,
-      "public class " + className + extendsText + " implements java.io.Serializable {" +
-      "  @Override public String toString() { return \"" + toStringValue + "\"; }}")
+      s"""
+         |public class $className $extendsText $implementsText {
+         |  @Override public String toString() { return "$toStringValue"; }
+         |
+         |  $extraCodeBody
+         |}
+        """.stripMargin)
     createCompiledClass(className, destDir, sourceFile, classpathUrls)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 8aeb16fe5d8c8..e9f1d9c250ad2 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -153,11 +153,6 @@ private[spark] class Executor(
   // for fetching remote cached RDD blocks, so need to make sure it uses the right classloader too.
   env.serializerManager.setDefaultClassLoader(replClassLoader)
 
-  // Plugins need to load using a class loader that includes the executor's user classpath
-  private val plugins: Option[PluginContainer] = Utils.withContextClassLoader(replClassLoader) {
-    PluginContainer(env, resources.asJava)
-  }
-
   // Max size of direct result. If task result is bigger than this, we use the block manager
   // to send the result back.
   private val maxDirectResultSize = Math.min(
@@ -218,6 +213,13 @@ private[spark] class Executor(
 
   heartbeater.start()
 
+  // Plugins need to load using a class loader that includes the executor's user classpath.
+  // Plugins also needs to be initialized after the heartbeater started
+  // to avoid blocking to send heartbeat (see SPARK-32175).
+  private val plugins: Option[PluginContainer] = Utils.withContextClassLoader(replClassLoader) {
+    PluginContainer(env, resources.asJava)
+  }
+
   metricsPoller.start()
 
   private[executor] def numRunningTasks: Int = runningTasks.size()
diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index 31049d104e63d..b198448a299cf 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.executor
 
-import java.io.{Externalizable, ObjectInput, ObjectOutput}
+import java.io.{Externalizable, File, ObjectInput, ObjectOutput}
 import java.lang.Thread.UncaughtExceptionHandler
 import java.nio.ByteBuffer
 import java.util.Properties
@@ -41,6 +41,7 @@ import org.scalatestplus.mockito.MockitoSugar
 import org.apache.spark._
 import org.apache.spark.TaskState.TaskState
 import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.deploy.{SimpleApplicationTest, SparkSubmitSuite}
 import org.apache.spark.internal.config._
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.memory.TestMemoryManager
@@ -52,7 +53,7 @@ import org.apache.spark.scheduler.{DirectTaskResult, FakeTask, ResultTask, Task,
 import org.apache.spark.serializer.{JavaSerializer, SerializerInstance, SerializerManager}
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.storage.{BlockManager, BlockManagerId}
-import org.apache.spark.util.{LongAccumulator, UninterruptibleThread}
+import org.apache.spark.util.{LongAccumulator, UninterruptibleThread, Utils}
 
 class ExecutorSuite extends SparkFunSuite
     with LocalSparkContext with MockitoSugar with Eventually with PrivateMethodTester {
@@ -402,6 +403,73 @@ class ExecutorSuite extends SparkFunSuite
     assert(taskMetrics.getMetricValue("JVMHeapMemory") > 0)
   }
 
+  test("SPARK-32175: Plugin initialization should start after heartbeater started") {
+    withTempDir { tempDir =>
+      val sparkPluginCodeBody =
+        """
+          |@Override
+          |public org.apache.spark.api.plugin.ExecutorPlugin executorPlugin() {
+          |  return new TestExecutorPlugin();
+          |}
+          |
+          |@Override
+          |public org.apache.spark.api.plugin.DriverPlugin driverPlugin() { return null; }
+        """.stripMargin
+      val executorPluginBody =
+        """
+          |@Override
+          |public void init(
+          |    org.apache.spark.api.plugin.PluginContext ctx,
+          |    java.util.Map<String, String> extraConf) {
+          |  try {
+          |    Thread.sleep(8 * 1000);
+          |  } catch (InterruptedException e) {
+          |    throw new RuntimeException(e);
+          |  }
+          |}
+        """.stripMargin
+
+      val compiledExecutorPlugin = TestUtils.createCompiledClass(
+        "TestExecutorPlugin",
+        tempDir,
+        "",
+        null,
+        Seq.empty,
+        Seq("org.apache.spark.api.plugin.ExecutorPlugin"),
+        executorPluginBody)
+
+      val thisClassPath =
+        sys.props("java.class.path").split(File.pathSeparator).map(p => new File(p).toURI.toURL)
+      val compiledSparkPlugin = TestUtils.createCompiledClass(
+        "TestSparkPlugin",
+        tempDir,
+        "",
+        null,
+        Seq(tempDir.toURI.toURL) ++ thisClassPath,
+        Seq("org.apache.spark.api.plugin.SparkPlugin"),
+        sparkPluginCodeBody)
+
+      val jarUrl = TestUtils.createJar(
+        Seq(compiledSparkPlugin, compiledExecutorPlugin),
+        new File(tempDir, "testPlugin.jar"))
+
+      val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
+      val args = Seq(
+        "--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
+        "--name", "testApp",
+        "--master", "local-cluster[1,1,1024]",
+        "--conf", "spark.plugins=TestSparkPlugin",
+        "--conf", "spark.storage.blockManagerSlaveTimeoutMs=" + 5 * 1000,
+        "--conf", "spark.network.timeoutInterval=" + 1000,
+        "--conf", "spark.executor.heartbeatInterval=" + 1000,
+        "--conf", "spark.executor.extraClassPath=" + jarUrl.toString,
+        "--conf", "spark.driver.extraClassPath=" + jarUrl.toString,
+        "--conf", "spark.ui.enabled=false",
+        unusedJar.toString)
+      SparkSubmitSuite.runSparkSubmit(args, timeout = 30.seconds)
+    }
+  }
+
   private def createMockEnv(conf: SparkConf, serializer: JavaSerializer): SparkEnv = {
     val mockEnv = mock[SparkEnv]
     val mockRpcEnv = mock[RpcEnv]

From d00e1040010ac82df7e41e88314a6162b016f807 Mon Sep 17 00:00:00 2001
From: Holden Karau <hkarau@apple.com>
Date: Wed, 29 Jul 2020 21:39:14 +0000
Subject: [PATCH 0934/1280] [SPARK-32397][BUILD] Allow specifying of time for
 build to keep time consistent between modules

### What changes were proposed in this pull request?

Upgrade codehaus maven build helper to allow people to specify a time during the build to avoid snapshot artifacts with different version strings.

### Why are the changes needed?

During builds of snapshots the maven may assign different versions to different artifacts based on the time each individual sub-module starts building.

The timestamp is used as part of the version string when run `maven deploy` on a snapshot build. This results in different sub-modules having different version strings.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Manual build while specifying the current time, ensured the time is consistent in the sub components.

Open question: Ideally I'd like to backport this as well since it's sort of a bug fix and while it does change a dependency version it's not one that is propagated. I'd like to hear folks thoughts about this.

Closes #29274 from holdenk/SPARK-32397-snapshot-artifact-timestamp-differences.

Authored-by: Holden Karau <hkarau@apple.com>
Signed-off-by: DB Tsai <d_tsai@apple.com>
(cherry picked from commit 50911df08eb7a27494dc83bcec3d09701c2babfe)
Signed-off-by: DB Tsai <d_tsai@apple.com>
---
 pom.xml | 40 +++++++++++++++++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index 6c67ce41b413a..86cdd1d7237ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -251,6 +251,8 @@
     <spark.test.webdriver.chrome.driver></spark.test.webdriver.chrome.driver>
 
     <CodeCacheSize>1g</CodeCacheSize>
+    <!-- Needed for consistent times -->
+    <maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
   </properties>
   <repositories>
     <repository>
@@ -2344,11 +2346,39 @@
             </execution>
           </executions>
         </plugin>
-        <plugin>
-          <groupId>org.codehaus.mojo</groupId>
-          <artifactId>build-helper-maven-plugin</artifactId>
-          <version>3.0.0</version>
-        </plugin>
+	<plugin>
+	  <groupId>org.codehaus.mojo</groupId>
+	  <artifactId>build-helper-maven-plugin</artifactId>
+	  <version>3.2.0</version>
+	  <executions>
+	    <execution>
+	      <id>module-timestamp-property</id>
+	      <phase>validate</phase>
+	      <goals>
+		<goal>timestamp-property</goal>
+	      </goals>
+	      <configuration>
+		<name>module.build.timestamp</name>
+		<pattern>${maven.build.timestamp.format}</pattern>
+		<timeSource>current</timeSource>
+		<timeZone>America/Los_Angeles</timeZone>
+	      </configuration>
+	    </execution>
+	    <execution>
+	      <id>local-timestamp-property</id>
+	      <phase>validate</phase>
+	      <goals>
+		<goal>timestamp-property</goal>
+	      </goals>
+	      <configuration>
+		<name>local.build.timestamp</name>
+		<pattern>${maven.build.timestamp.format}</pattern>
+		<timeSource>build</timeSource>
+		<timeZone>America/Los_Angeles</timeZone>
+	      </configuration>
+	    </execution>
+	  </executions>
+	</plugin>
         <plugin>
           <groupId>net.alchim31.maven</groupId>
           <artifactId>scala-maven-plugin</artifactId>

From 235552aba3a7be42360502f8c164a4f23fe9469f Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 30 Jul 2020 15:16:02 +0900
Subject: [PATCH 0935/1280] [SPARK-32478][R][SQL] Error message to show the
 schema mismatch in gapply with Arrow vectorization

### What changes were proposed in this pull request?

This PR proposes to:

1. Fix the error message when the output schema is misbatched with R DataFrame from the given function. For example,

    ```R
    df <- createDataFrame(list(list(a=1L, b="2")))
    count(gapply(df, "a", function(key, group) { group }, structType("a int, b int")))
    ```

    **Before:**

    ```
    Error in handleErrors(returnStatus, conn) :
      ...
      java.lang.UnsupportedOperationException
	    ...
    ```

    **After:**

    ```
    Error in handleErrors(returnStatus, conn) :
     ...
     java.lang.AssertionError: assertion failed: Invalid schema from gapply: expected IntegerType, IntegerType, got IntegerType, StringType
        ...
    ```

2. Update documentation about the schema matching for `gapply` and `dapply`.

### Why are the changes needed?

To show which schema is not matched, and let users know what's going on.

### Does this PR introduce _any_ user-facing change?

Yes, error message is updated as above, and documentation is updated.

### How was this patch tested?

Manually tested and unitttests were added.

Closes #29283 from HyukjinKwon/r-vectorized-error.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/tests/fulltests/test_sparkSQL_arrow.R    | 18 ++++++++++++++++++
 docs/sparkr.md                                 | 16 ++++++++--------
 .../apache/spark/sql/execution/objects.scala   |  9 ++++++++-
 3 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
index 97972753a78fa..16d93763ff038 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
@@ -312,4 +312,22 @@ test_that("Arrow optimization - unsupported types", {
   })
 })
 
+test_that("SPARK-32478: gapply() Arrow optimization - error message for schema mismatch", {
+  skip_if_not_installed("arrow")
+  df <- createDataFrame(list(list(a = 1L, b = "a")))
+
+  conf <- callJMethod(sparkSession, "conf")
+  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
+
+  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
+  tryCatch({
+    expect_error(
+    count(gapply(df, "a", function(key, group) { group }, structType("a int, b int"))),
+    "expected IntegerType, IntegerType, got IntegerType, StringType")
+  },
+  finally = {
+    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
+  })
+})
+
 sparkR.session.stop()
diff --git a/docs/sparkr.md b/docs/sparkr.md
index 9cdc39296f3fe..15578fdd41902 100644
--- a/docs/sparkr.md
+++ b/docs/sparkr.md
@@ -678,12 +678,12 @@ The current supported minimum version is 0.15.1; however, this might change betw
 Arrow optimization is available when converting a Spark DataFrame to an R DataFrame using the call `collect(spark_df)`,
 when creating a Spark DataFrame from an R DataFrame with `createDataFrame(r_df)`, when applying an R native function to each partition
 via `dapply(...)` and when applying an R native function to grouped data via `gapply(...)`.
-To use Arrow when executing these calls, users need to first set the Spark configuration ‘spark.sql.execution.arrow.sparkr.enabled’
-to ‘true’. This is disabled by default.
+To use Arrow when executing these, users need to set the Spark configuration ‘spark.sql.execution.arrow.sparkr.enabled’
+to ‘true’ first. This is disabled by default.
 
-In addition, optimizations enabled by ‘spark.sql.execution.arrow.sparkr.enabled’ could fallback automatically to non-Arrow optimization
-implementation if an error occurs before the actual computation within Spark during converting a Spark DataFrame to/from an R
-DataFrame.
+Whether the optimization is enabled or not, SparkR produces the same results. In addition, the conversion
+between Spark DataFrame and R DataFrame falls back automatically to non-Arrow optimization implementation
+when the optimization fails for any reasons before the actual computation.
 
 <div data-lang="r" markdown="1">
 {% highlight r %}
@@ -710,9 +710,9 @@ collect(gapply(spark_df,
 {% endhighlight %}
 </div>
 
-Using the above optimizations with Arrow will produce the same results as when Arrow is not enabled. Note that even with Arrow,
-`collect(spark_df)` results in the collection of all records in the DataFrame to the driver program and should be done on a
-small subset of the data.
+Note that even with Arrow, `collect(spark_df)` results in the collection of all records in the DataFrame to
+the driver program and should be done on a small subset of the data. In addition, the specified output schema
+in `gapply(...)` and `dapply(...)` should be matched to the R DataFrame's returned by the given function.
 
 ## Supported SQL Types
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
index 4b2d4195ee906..c08db132c946f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/objects.scala
@@ -567,7 +567,14 @@ case class FlatMapGroupsInRWithArrowExec(
       // binary in a batch due to the limitation of R API. See also ARROW-4512.
       val columnarBatchIter = runner.compute(groupedByRKey, -1)
       val outputProject = UnsafeProjection.create(output, output)
-      columnarBatchIter.flatMap(_.rowIterator().asScala).map(outputProject)
+      val outputTypes = StructType.fromAttributes(output).map(_.dataType)
+
+      columnarBatchIter.flatMap { batch =>
+        val actualDataTypes = (0 until batch.numCols()).map(i => batch.column(i).dataType())
+        assert(outputTypes == actualDataTypes, "Invalid schema from gapply(): " +
+          s"expected ${outputTypes.mkString(", ")}, got ${actualDataTypes.mkString(", ")}")
+        batch.rowIterator().asScala
+      }.map(outputProject)
     }
   }
 }

From b40df01105e0edd394eebed553b5a41069180a8f Mon Sep 17 00:00:00 2001
From: Warren Zhu <zhonzh@microsoft.com>
Date: Thu, 30 Jul 2020 21:44:49 +0900
Subject: [PATCH 0936/1280] [SPARK-32227] Fix regression bug in
 load-spark-env.cmd with Spark 3.0.0

Fix regression bug in load-spark-env.cmd with Spark 3.0.0

cmd doesn't support set env twice. So set `SPARK_ENV_CMD=%SPARK_CONF_DIR%\%SPARK_ENV_CMD%` doesn't take effect, which caused regression.

No

Manually tested.
1. Create a spark-env.cmd under conf folder. Inside this, `echo spark-env.cmd`
2. Run old load-spark-env.cmd, nothing printed in the output
2. Run fixed load-spark-env.cmd, `spark-env.cmd` showed in the output.

Closes #29044 from warrenzhu25/32227.

Lead-authored-by: Warren Zhu <zhonzh@microsoft.com>
Co-authored-by: Warren Zhu <warren.zhu25@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 743772095273b464f845efefb3eb59284b06b9be)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 bin/load-spark-env.cmd | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/load-spark-env.cmd b/bin/load-spark-env.cmd
index 5f98cc34b6bab..7e2cf43af3c7d 100644
--- a/bin/load-spark-env.cmd
+++ b/bin/load-spark-env.cmd
@@ -29,10 +29,7 @@ if [%SPARK_ENV_LOADED%] == [] (
     set SPARK_CONF_DIR=%~dp0..\conf
   )
 
-  set SPARK_ENV_CMD=%SPARK_CONF_DIR%\%SPARK_ENV_CMD%
-  if exist %SPARK_ENV_CMD% (
-    call %SPARK_ENV_CMD%
-  )
+  call :LoadSparkEnv
 )
 
 rem Setting SPARK_SCALA_VERSION if not already set.
@@ -60,3 +57,8 @@ rem     set SPARK_SCALA_VERSION=%SCALA_VERSION_2%
 rem   )
 rem )
 exit /b 0
+
+:LoadSparkEnv
+if exist "%SPARK_CONF_DIR%\spark-env.cmd" (
+  call "%SPARK_CONF_DIR%\spark-env.cmd"
+)

From 2a3809058f668195ae4f377a293b228d62d970bb Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 31 Jul 2020 10:37:05 +0900
Subject: [PATCH 0937/1280] [SPARK-32175][SPARK-32175][FOLLOWUP] Remove flaky
 test added in

### What changes were proposed in this pull request?

This PR removes a test added in SPARK-32175(#29002).

### Why are the changes needed?

That test is flaky. It can be mitigated by increasing the timeout but it would rather be simpler to remove the test.
See also the [discussion](https://github.com/apache/spark/pull/29002#issuecomment-666746857).

### Does this PR introduce _any_ user-facing change?

No.

Closes #29314 from sarutak/remove-flaky-test.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
(cherry picked from commit 9d7b1d935f7a2b770d8b2f264cfe4a4db2ad64b6)
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 .../apache/spark/executor/ExecutorSuite.scala | 67 -------------------
 1 file changed, 67 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index b198448a299cf..8e58beff74290 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -403,73 +403,6 @@ class ExecutorSuite extends SparkFunSuite
     assert(taskMetrics.getMetricValue("JVMHeapMemory") > 0)
   }
 
-  test("SPARK-32175: Plugin initialization should start after heartbeater started") {
-    withTempDir { tempDir =>
-      val sparkPluginCodeBody =
-        """
-          |@Override
-          |public org.apache.spark.api.plugin.ExecutorPlugin executorPlugin() {
-          |  return new TestExecutorPlugin();
-          |}
-          |
-          |@Override
-          |public org.apache.spark.api.plugin.DriverPlugin driverPlugin() { return null; }
-        """.stripMargin
-      val executorPluginBody =
-        """
-          |@Override
-          |public void init(
-          |    org.apache.spark.api.plugin.PluginContext ctx,
-          |    java.util.Map<String, String> extraConf) {
-          |  try {
-          |    Thread.sleep(8 * 1000);
-          |  } catch (InterruptedException e) {
-          |    throw new RuntimeException(e);
-          |  }
-          |}
-        """.stripMargin
-
-      val compiledExecutorPlugin = TestUtils.createCompiledClass(
-        "TestExecutorPlugin",
-        tempDir,
-        "",
-        null,
-        Seq.empty,
-        Seq("org.apache.spark.api.plugin.ExecutorPlugin"),
-        executorPluginBody)
-
-      val thisClassPath =
-        sys.props("java.class.path").split(File.pathSeparator).map(p => new File(p).toURI.toURL)
-      val compiledSparkPlugin = TestUtils.createCompiledClass(
-        "TestSparkPlugin",
-        tempDir,
-        "",
-        null,
-        Seq(tempDir.toURI.toURL) ++ thisClassPath,
-        Seq("org.apache.spark.api.plugin.SparkPlugin"),
-        sparkPluginCodeBody)
-
-      val jarUrl = TestUtils.createJar(
-        Seq(compiledSparkPlugin, compiledExecutorPlugin),
-        new File(tempDir, "testPlugin.jar"))
-
-      val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
-      val args = Seq(
-        "--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
-        "--name", "testApp",
-        "--master", "local-cluster[1,1,1024]",
-        "--conf", "spark.plugins=TestSparkPlugin",
-        "--conf", "spark.storage.blockManagerSlaveTimeoutMs=" + 5 * 1000,
-        "--conf", "spark.network.timeoutInterval=" + 1000,
-        "--conf", "spark.executor.heartbeatInterval=" + 1000,
-        "--conf", "spark.executor.extraClassPath=" + jarUrl.toString,
-        "--conf", "spark.driver.extraClassPath=" + jarUrl.toString,
-        "--conf", "spark.ui.enabled=false",
-        unusedJar.toString)
-      SparkSubmitSuite.runSparkSubmit(args, timeout = 30.seconds)
-    }
-  }
-
   private def createMockEnv(conf: SparkConf, serializer: JavaSerializer): SparkEnv = {
     val mockEnv = mock[SparkEnv]
     val mockRpcEnv = mock[RpcEnv]

From 7c91b15c22fe875a44e08781caf3422bfca81b19 Mon Sep 17 00:00:00 2001
From: Andy Grove <andygrove@nvidia.com>
Date: Fri, 31 Jul 2020 11:14:33 -0500
Subject: [PATCH 0938/1280] [SPARK-32332][SQL][3.0] Support columnar exchanges

### What changes were proposed in this pull request?
Backports SPARK-32332 to 3.0 branch.

### Why are the changes needed?
Plugins cannot replace exchanges with columnar versions when AQE is enabled without this patch.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Tests included.

Closes #29310 from andygrove/backport-SPARK-32332.

Authored-by: Andy Grove <andygrove@nvidia.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../adaptive/AdaptiveSparkPlanExec.scala      |  30 ++++-
 .../adaptive/CustomShuffleReaderExec.scala    |  37 ++++--
 .../adaptive/OptimizeLocalShuffleReader.scala |   5 +-
 .../adaptive/OptimizeSkewedJoin.scala         |  17 ++-
 .../execution/adaptive/QueryStageExec.scala   |  24 +++-
 .../execution/adaptive/simpleCosting.scala    |   6 +-
 .../exchange/BroadcastExchangeExec.scala      |  42 +++++-
 .../exchange/ShuffleExchangeExec.scala        |  55 +++++++-
 .../streaming/IncrementalExecution.scala      |   4 +-
 .../sql/SparkSessionExtensionSuite.scala      | 120 +++++++++++++++---
 10 files changed, 272 insertions(+), 68 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 5714c33d3774e..8b59b1236782e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -100,7 +100,12 @@ case class AdaptiveSparkPlanExec(
     // The following two rules need to make use of 'CustomShuffleReaderExec.partitionSpecs'
     // added by `CoalesceShufflePartitions`. So they must be executed after it.
     OptimizeSkewedJoin(conf),
-    OptimizeLocalShuffleReader(conf),
+    OptimizeLocalShuffleReader(conf)
+  )
+
+  // A list of physical optimizer rules to be applied right after a new stage is created. The input
+  // plan to these rules has exchange as its root node.
+  @transient private val postStageCreationRules = Seq(
     ApplyColumnarRulesAndInsertTransitions(conf, context.session.sessionState.columnarRules),
     CollapseCodegenStages(conf)
   )
@@ -227,7 +232,8 @@ case class AdaptiveSparkPlanExec(
       }
 
       // Run the final plan when there's no more unfinished stages.
-      currentPhysicalPlan = applyPhysicalRules(result.newPlan, queryStageOptimizerRules)
+      currentPhysicalPlan = applyPhysicalRules(
+        result.newPlan, queryStageOptimizerRules ++ postStageCreationRules)
       isFinalPlan = true
       executionId.foreach(onUpdatePlan(_, Seq(currentPhysicalPlan)))
       currentPhysicalPlan
@@ -375,10 +381,22 @@ case class AdaptiveSparkPlanExec(
   private def newQueryStage(e: Exchange): QueryStageExec = {
     val optimizedPlan = applyPhysicalRules(e.child, queryStageOptimizerRules)
     val queryStage = e match {
-      case s: ShuffleExchangeExec =>
-        ShuffleQueryStageExec(currentStageId, s.copy(child = optimizedPlan))
-      case b: BroadcastExchangeExec =>
-        BroadcastQueryStageExec(currentStageId, b.copy(child = optimizedPlan))
+      case s: ShuffleExchangeLike =>
+        val newShuffle = applyPhysicalRules(
+          s.withNewChildren(Seq(optimizedPlan)), postStageCreationRules)
+        if (!newShuffle.isInstanceOf[ShuffleExchangeLike]) {
+          throw new IllegalStateException(
+            "Custom columnar rules cannot transform shuffle node to something else.")
+        }
+        ShuffleQueryStageExec(currentStageId, newShuffle)
+      case b: BroadcastExchangeLike =>
+        val newBroadcast = applyPhysicalRules(
+          b.withNewChildren(Seq(optimizedPlan)), postStageCreationRules)
+        if (!newBroadcast.isInstanceOf[BroadcastExchangeLike]) {
+          throw new IllegalStateException(
+            "Custom columnar rules cannot transform broadcast node to something else.")
+        }
+        BroadcastQueryStageExec(currentStageId, newBroadcast)
     }
     currentStageId += 1
     setLogicalLinkForNewQueryStage(queryStage, e)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
index ba3f72592908e..8fd572088b620 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/CustomShuffleReaderExec.scala
@@ -22,7 +22,8 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec}
+import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike}
+import org.apache.spark.sql.vectorized.ColumnarBatch
 
 
 /**
@@ -38,6 +39,8 @@ case class CustomShuffleReaderExec private(
     partitionSpecs: Seq[ShufflePartitionSpec],
     description: String) extends UnaryExecNode {
 
+  override def supportsColumnar: Boolean = child.supportsColumnar
+
   override def output: Seq[Attribute] = child.output
   override lazy val outputPartitioning: Partitioning = {
     // If it is a local shuffle reader with one mapper per task, then the output partitioning is
@@ -47,9 +50,9 @@ case class CustomShuffleReaderExec private(
         partitionSpecs.map(_.asInstanceOf[PartialMapperPartitionSpec].mapIndex).toSet.size ==
           partitionSpecs.length) {
       child match {
-        case ShuffleQueryStageExec(_, s: ShuffleExchangeExec) =>
+        case ShuffleQueryStageExec(_, s: ShuffleExchangeLike) =>
           s.child.outputPartitioning
-        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeExec)) =>
+        case ShuffleQueryStageExec(_, r @ ReusedExchangeExec(_, s: ShuffleExchangeLike)) =>
           s.child.outputPartitioning match {
             case e: Expression => r.updateAttr(e).asInstanceOf[Partitioning]
             case other => other
@@ -64,18 +67,24 @@ case class CustomShuffleReaderExec private(
 
   override def stringArgs: Iterator[Any] = Iterator(description)
 
-  private var cachedShuffleRDD: RDD[InternalRow] = null
+  private def shuffleStage = child match {
+    case stage: ShuffleQueryStageExec => Some(stage)
+    case _ => None
+  }
 
-  override protected def doExecute(): RDD[InternalRow] = {
-    if (cachedShuffleRDD == null) {
-      cachedShuffleRDD = child match {
-        case stage: ShuffleQueryStageExec =>
-          new ShuffledRowRDD(
-            stage.shuffle.shuffleDependency, stage.shuffle.readMetrics, partitionSpecs.toArray)
-        case _ =>
-          throw new IllegalStateException("operating on canonicalization plan")
-      }
+  private lazy val shuffleRDD: RDD[_] = {
+    shuffleStage.map { stage =>
+      stage.shuffle.getShuffleRDD(partitionSpecs.toArray)
+    }.getOrElse {
+      throw new IllegalStateException("operating on canonicalized plan")
     }
-    cachedShuffleRDD
+  }
+
+  override protected def doExecute(): RDD[InternalRow] = {
+    shuffleRDD.asInstanceOf[RDD[InternalRow]]
+  }
+
+  override protected def doExecuteColumnar(): RDD[ColumnarBatch] = {
+    shuffleRDD.asInstanceOf[RDD[ColumnarBatch]]
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index fb6b40c710929..6684376b12539 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -78,10 +78,9 @@ case class OptimizeLocalShuffleReader(conf: SQLConf) extends Rule[SparkPlan] {
   private def getPartitionSpecs(
       shuffleStage: ShuffleQueryStageExec,
       advisoryParallelism: Option[Int]): Seq[ShufflePartitionSpec] = {
-    val shuffleDep = shuffleStage.shuffle.shuffleDependency
-    val numReducers = shuffleDep.partitioner.numPartitions
+    val numMappers = shuffleStage.shuffle.numMappers
+    val numReducers = shuffleStage.shuffle.numPartitions
     val expectedParallelism = advisoryParallelism.getOrElse(numReducers)
-    val numMappers = shuffleDep.rdd.getNumPartitions
     val splitPoints = if (numMappers == 0) {
       Seq.empty
     } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
index 91ae0b9ed2ae2..b3b3eb2151f5e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeSkewedJoin.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 
 import org.apache.commons.io.FileUtils
 
-import org.apache.spark.{MapOutputTrackerMaster, SparkEnv}
+import org.apache.spark.{MapOutputStatistics, MapOutputTrackerMaster, SparkEnv}
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution._
@@ -197,7 +197,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
         val leftParts = if (isLeftSkew && !isLeftCoalesced) {
           val reducerId = leftPartSpec.startReducerIndex
           val skewSpecs = createSkewPartitionSpecs(
-            left.shuffleStage.shuffle.shuffleDependency.shuffleId, reducerId, leftTargetSize)
+            left.mapStats.shuffleId, reducerId, leftTargetSize)
           if (skewSpecs.isDefined) {
             logDebug(s"Left side partition $partitionIndex is skewed, split it into " +
               s"${skewSpecs.get.length} parts.")
@@ -212,7 +212,7 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
         val rightParts = if (isRightSkew && !isRightCoalesced) {
           val reducerId = rightPartSpec.startReducerIndex
           val skewSpecs = createSkewPartitionSpecs(
-            right.shuffleStage.shuffle.shuffleDependency.shuffleId, reducerId, rightTargetSize)
+            right.mapStats.shuffleId, reducerId, rightTargetSize)
           if (skewSpecs.isDefined) {
             logDebug(s"Right side partition $partitionIndex is skewed, split it into " +
               s"${skewSpecs.get.length} parts.")
@@ -287,15 +287,17 @@ case class OptimizeSkewedJoin(conf: SQLConf) extends Rule[SparkPlan] {
 private object ShuffleStage {
   def unapply(plan: SparkPlan): Option[ShuffleStageInfo] = plan match {
     case s: ShuffleQueryStageExec if s.mapStats.isDefined =>
-      val sizes = s.mapStats.get.bytesByPartitionId
+      val mapStats = s.mapStats.get
+      val sizes = mapStats.bytesByPartitionId
       val partitions = sizes.zipWithIndex.map {
         case (size, i) => CoalescedPartitionSpec(i, i + 1) -> size
       }
-      Some(ShuffleStageInfo(s, partitions))
+      Some(ShuffleStageInfo(s, mapStats, partitions))
 
     case CustomShuffleReaderExec(s: ShuffleQueryStageExec, partitionSpecs, _)
       if s.mapStats.isDefined && partitionSpecs.nonEmpty =>
-      val sizes = s.mapStats.get.bytesByPartitionId
+      val mapStats = s.mapStats.get
+      val sizes = mapStats.bytesByPartitionId
       val partitions = partitionSpecs.map {
         case spec @ CoalescedPartitionSpec(start, end) =>
           var sum = 0L
@@ -308,7 +310,7 @@ private object ShuffleStage {
         case other => throw new IllegalArgumentException(
           s"Expect CoalescedPartitionSpec but got $other")
       }
-      Some(ShuffleStageInfo(s, partitions))
+      Some(ShuffleStageInfo(s, mapStats, partitions))
 
     case _ => None
   }
@@ -316,6 +318,7 @@ private object ShuffleStage {
 
 private case class ShuffleStageInfo(
     shuffleStage: ShuffleQueryStageExec,
+    mapStats: MapOutputStatistics,
     partitionsWithSizes: Seq[(CoalescedPartitionSpec, Long)])
 
 private class SkewDesc {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
index 9a9a8b1c477bc..74fe1eaab6f64 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/QueryStageExec.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.exchange._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -80,6 +81,11 @@ abstract class QueryStageExec extends LeafExecNode {
 
   def newReuseInstance(newStageId: Int, newOutput: Seq[Attribute]): QueryStageExec
 
+  /**
+   * Returns the runtime statistics after stage materialization.
+   */
+  def getRuntimeStatistics: Statistics
+
   /**
    * Compute the statistics of the query stage if executed, otherwise None.
    */
@@ -107,6 +113,8 @@ abstract class QueryStageExec extends LeafExecNode {
 
   protected override def doPrepare(): Unit = plan.prepare()
   protected override def doExecute(): RDD[InternalRow] = plan.execute()
+  override def supportsColumnar: Boolean = plan.supportsColumnar
+  protected override def doExecuteColumnar(): RDD[ColumnarBatch] = plan.executeColumnar()
   override def doExecuteBroadcast[T](): Broadcast[T] = plan.executeBroadcast()
   override def doCanonicalize(): SparkPlan = plan.canonicalized
 
@@ -135,15 +143,15 @@ abstract class QueryStageExec extends LeafExecNode {
 }
 
 /**
- * A shuffle query stage whose child is a [[ShuffleExchangeExec]] or [[ReusedExchangeExec]].
+ * A shuffle query stage whose child is a [[ShuffleExchangeLike]] or [[ReusedExchangeExec]].
  */
 case class ShuffleQueryStageExec(
     override val id: Int,
     override val plan: SparkPlan) extends QueryStageExec {
 
   @transient val shuffle = plan match {
-    case s: ShuffleExchangeExec => s
-    case ReusedExchangeExec(_, s: ShuffleExchangeExec) => s
+    case s: ShuffleExchangeLike => s
+    case ReusedExchangeExec(_, s: ShuffleExchangeLike) => s
     case _ =>
       throw new IllegalStateException("wrong plan for shuffle stage:\n " + plan.treeString)
   }
@@ -176,18 +184,20 @@ case class ShuffleQueryStageExec(
     val stats = resultOption.get.asInstanceOf[MapOutputStatistics]
     Option(stats)
   }
+
+  override def getRuntimeStatistics: Statistics = shuffle.runtimeStatistics
 }
 
 /**
- * A broadcast query stage whose child is a [[BroadcastExchangeExec]] or [[ReusedExchangeExec]].
+ * A broadcast query stage whose child is a [[BroadcastExchangeLike]] or [[ReusedExchangeExec]].
  */
 case class BroadcastQueryStageExec(
     override val id: Int,
     override val plan: SparkPlan) extends QueryStageExec {
 
   @transient val broadcast = plan match {
-    case b: BroadcastExchangeExec => b
-    case ReusedExchangeExec(_, b: BroadcastExchangeExec) => b
+    case b: BroadcastExchangeLike => b
+    case ReusedExchangeExec(_, b: BroadcastExchangeLike) => b
     case _ =>
       throw new IllegalStateException("wrong plan for broadcast stage:\n " + plan.treeString)
   }
@@ -224,6 +234,8 @@ case class BroadcastQueryStageExec(
       broadcast.relationFuture.cancel(true)
     }
   }
+
+  override def getRuntimeStatistics: Statistics = broadcast.runtimeStatistics
 }
 
 object BroadcastQueryStageExec {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
index 67cd720bb5b33..cdc57dbc7dcc2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/simpleCosting.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.execution.adaptive
 
 import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
 
 /**
  * A simple implementation of [[Cost]], which takes a number of [[Long]] as the cost value.
@@ -35,13 +35,13 @@ case class SimpleCost(value: Long) extends Cost {
 
 /**
  * A simple implementation of [[CostEvaluator]], which counts the number of
- * [[ShuffleExchangeExec]] nodes in the plan.
+ * [[ShuffleExchangeLike]] nodes in the plan.
  */
 object SimpleCostEvaluator extends CostEvaluator {
 
   override def evaluateCost(plan: SparkPlan): Cost = {
     val cost = plan.collect {
-      case s: ShuffleExchangeExec => s
+      case s: ShuffleExchangeLike => s
     }.size
     SimpleCost(cost)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
index d35bbe9b8adc0..bcdaf61c9c997 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala
@@ -29,6 +29,7 @@ import org.apache.spark.launcher.SparkLauncher
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical.{BroadcastMode, BroadcastPartitioning, Partitioning}
 import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
 import org.apache.spark.sql.execution.joins.HashedRelation
@@ -37,16 +38,43 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
 import org.apache.spark.unsafe.map.BytesToBytesMap
 import org.apache.spark.util.{SparkFatalException, ThreadUtils}
 
+/**
+ * Common trait for all broadcast exchange implementations to facilitate pattern matching.
+ */
+trait BroadcastExchangeLike extends Exchange {
+
+  /**
+   * The broadcast job group ID
+   */
+  def runId: UUID = UUID.randomUUID
+
+  /**
+   * The asynchronous job that prepares the broadcast relation.
+   */
+  def relationFuture: Future[broadcast.Broadcast[Any]]
+
+  /**
+   * For registering callbacks on `relationFuture`.
+   * Note that calling this method may not start the execution of broadcast job.
+   */
+  def completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]]
+
+  /**
+   * Returns the runtime statistics after broadcast materialization.
+   */
+  def runtimeStatistics: Statistics
+}
+
 /**
  * A [[BroadcastExchangeExec]] collects, transforms and finally broadcasts the result of
  * a transformed SparkPlan.
  */
 case class BroadcastExchangeExec(
     mode: BroadcastMode,
-    child: SparkPlan) extends Exchange {
+    child: SparkPlan) extends BroadcastExchangeLike {
   import BroadcastExchangeExec._
 
-  private[sql] val runId: UUID = UUID.randomUUID
+  override val runId: UUID = UUID.randomUUID
 
   override lazy val metrics = Map(
     "dataSize" -> SQLMetrics.createSizeMetric(sparkContext, "data size"),
@@ -60,6 +88,11 @@ case class BroadcastExchangeExec(
     BroadcastExchangeExec(mode.canonicalized, child.canonicalized)
   }
 
+  override def runtimeStatistics: Statistics = {
+    val dataSize = metrics("dataSize").value
+    Statistics(dataSize)
+  }
+
   @transient
   private lazy val promise = Promise[broadcast.Broadcast[Any]]()
 
@@ -68,13 +101,14 @@ case class BroadcastExchangeExec(
    * Note that calling this field will not start the execution of broadcast job.
    */
   @transient
-  lazy val completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]] = promise.future
+  override lazy val completionFuture: scala.concurrent.Future[broadcast.Broadcast[Any]] =
+    promise.future
 
   @transient
   private val timeout: Long = SQLConf.get.broadcastTimeout
 
   @transient
-  private[sql] lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
+  override lazy val relationFuture: Future[broadcast.Broadcast[Any]] = {
     SQLExecution.withThreadLocalCaptured[broadcast.Broadcast[Any]](
       sqlContext.sparkSession, BroadcastExchangeExec.executionContext) {
           try {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index b06742e8470c7..b7da78cb0eefb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.errors._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Divide, Literal, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
+import org.apache.spark.sql.catalyst.plans.logical.Statistics
 import org.apache.spark.sql.catalyst.plans.physical._
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics, SQLShuffleReadMetricsReporter, SQLShuffleWriteMetricsReporter}
@@ -40,13 +41,49 @@ import org.apache.spark.sql.types.StructType
 import org.apache.spark.util.MutablePair
 import org.apache.spark.util.collection.unsafe.sort.{PrefixComparators, RecordComparator}
 
+/**
+ * Common trait for all shuffle exchange implementations to facilitate pattern matching.
+ */
+trait ShuffleExchangeLike extends Exchange {
+
+  /**
+   * Returns the number of mappers of this shuffle.
+   */
+  def numMappers: Int
+
+  /**
+   * Returns the shuffle partition number.
+   */
+  def numPartitions: Int
+
+  /**
+   * Returns whether the shuffle partition number can be changed.
+   */
+  def canChangeNumPartitions: Boolean
+
+  /**
+   * The asynchronous job that materializes the shuffle.
+   */
+  def mapOutputStatisticsFuture: Future[MapOutputStatistics]
+
+  /**
+   * Returns the shuffle RDD with specified partition specs.
+   */
+  def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[_]
+
+  /**
+   * Returns the runtime statistics after shuffle materialization.
+   */
+  def runtimeStatistics: Statistics
+}
+
 /**
  * Performs a shuffle that will result in the desired partitioning.
  */
 case class ShuffleExchangeExec(
     override val outputPartitioning: Partitioning,
     child: SparkPlan,
-    canChangeNumPartitions: Boolean = true) extends Exchange {
+    canChangeNumPartitions: Boolean = true) extends ShuffleExchangeLike {
 
   private lazy val writeMetrics =
     SQLShuffleWriteMetricsReporter.createShuffleWriteMetrics(sparkContext)
@@ -64,7 +101,7 @@ case class ShuffleExchangeExec(
   @transient lazy val inputRDD: RDD[InternalRow] = child.execute()
 
   // 'mapOutputStatisticsFuture' is only needed when enable AQE.
-  @transient lazy val mapOutputStatisticsFuture: Future[MapOutputStatistics] = {
+  @transient override lazy val mapOutputStatisticsFuture: Future[MapOutputStatistics] = {
     if (inputRDD.getNumPartitions == 0) {
       Future.successful(null)
     } else {
@@ -72,6 +109,20 @@ case class ShuffleExchangeExec(
     }
   }
 
+  override def numMappers: Int = shuffleDependency.rdd.getNumPartitions
+
+  override def numPartitions: Int = shuffleDependency.partitioner.numPartitions
+
+  override def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[InternalRow] = {
+    new ShuffledRowRDD(shuffleDependency, readMetrics, partitionSpecs)
+  }
+
+  override def runtimeStatistics: Statistics = {
+    val dataSize = metrics("dataSize").value
+    val rowCount = metrics(SQLShuffleWriteMetricsReporter.SHUFFLE_RECORDS_WRITTEN).value
+    Statistics(dataSize, Some(rowCount))
+  }
+
   /**
    * A [[ShuffleDependency]] that will partition rows of its child based on
    * the partitioning scheme defined in `newPartitioning`. Those partitions of
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
index 7773ac71c4954..bfa60cf7dfd78 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/IncrementalExecution.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.plans.physical.{AllTuples, ClusteredDistribution, HashPartitioning, SinglePartition}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.{LeafExecNode, LocalLimitExec, QueryExecution, SparkPlan, SparkPlanner, UnaryExecNode}
-import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
+import org.apache.spark.sql.execution.exchange.{ShuffleExchangeExec, ShuffleExchangeLike}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.streaming.OutputMode
 import org.apache.spark.util.Utils
@@ -118,7 +118,7 @@ class IncrementalExecution(
           case s: StatefulOperator =>
             statefulOpFound = true
 
-          case e: ShuffleExchangeExec =>
+          case e: ShuffleExchangeLike =>
             // Don't search recursively any further as any child stateful operator as we
             // are only looking for stateful subplans that this plan has narrow dependencies on.
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
index 44e784de5164f..e5e8bc6917799 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionExtensionSuite.scala
@@ -16,19 +16,24 @@
  */
 package org.apache.spark.sql
 
-import java.util.Locale
+import java.util.{Locale, UUID}
 
-import org.apache.spark.{SparkFunSuite, TaskContext}
+import scala.concurrent.Future
+
+import org.apache.spark.{MapOutputStatistics, SparkFunSuite, TaskContext}
+import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParserInterface}
-import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, UnresolvedHint}
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Statistics, UnresolvedHint}
+import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.execution._
-import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, BroadcastExchangeLike, ShuffleExchangeExec, ShuffleExchangeLike}
 import org.apache.spark.sql.execution.vectorized.OnHeapColumnVector
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.COLUMN_BATCH_SIZE
@@ -169,33 +174,61 @@ class SparkSessionExtensionSuite extends SparkFunSuite {
     }
   }
 
-  test("inject columnar") {
+  test("inject columnar AQE on") {
+    testInjectColumnar(true)
+  }
+
+  test("inject columnar AQE off") {
+    testInjectColumnar(false)
+  }
+
+  private def testInjectColumnar(enableAQE: Boolean): Unit = {
+    def collectPlanSteps(plan: SparkPlan): Seq[Int] = plan match {
+      case a: AdaptiveSparkPlanExec =>
+        assert(a.toString.startsWith("AdaptiveSparkPlan isFinalPlan=true"))
+        collectPlanSteps(a.executedPlan)
+      case _ => plan.collect {
+        case _: ReplacedRowToColumnarExec => 1
+        case _: ColumnarProjectExec => 10
+        case _: ColumnarToRowExec => 100
+        case s: QueryStageExec => collectPlanSteps(s.plan).sum
+        case _: MyShuffleExchangeExec => 1000
+        case _: MyBroadcastExchangeExec => 10000
+      }
+    }
+
     val extensions = create { extensions =>
       extensions.injectColumnar(session =>
         MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule()))
     }
     withSession(extensions) { session =>
-      // The ApplyColumnarRulesAndInsertTransitions rule is not applied when enable AQE
-      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, false)
+      session.sessionState.conf.setConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED, enableAQE)
       assert(session.sessionState.columnarRules.contains(
         MyColumarRule(PreRuleReplaceAddWithBrokenVersion(), MyPostRule())))
       import session.sqlContext.implicits._
-      // repartitioning avoids having the add operation pushed up into the LocalTableScan
-      val data = Seq((100L), (200L), (300L)).toDF("vals").repartition(1)
-      val df = data.selectExpr("vals + 1")
-      // Verify that both pre and post processing of the plan worked.
-      val found = df.queryExecution.executedPlan.collect {
-        case rep: ReplacedRowToColumnarExec => 1
-        case proj: ColumnarProjectExec => 10
-        case c2r: ColumnarToRowExec => 100
-      }.sum
-      assert(found == 111)
+      // perform a join to inject a broadcast exchange
+      val left = Seq((1, 50L), (2, 100L), (3, 150L)).toDF("l1", "l2")
+      val right = Seq((1, 50L), (2, 100L), (3, 150L)).toDF("r1", "r2")
+      val data = left.join(right, $"l1" === $"r1")
+        // repartitioning avoids having the add operation pushed up into the LocalTableScan
+        .repartition(1)
+      val df = data.selectExpr("l2 + r2")
+      // execute the plan so that the final adaptive plan is available when AQE is on
+      df.collect()
+      val found = collectPlanSteps(df.queryExecution.executedPlan).sum
+      // 1 MyBroadcastExchangeExec
+      // 1 MyShuffleExchangeExec
+      // 1 ColumnarToRowExec
+      // 2 ColumnarProjectExec
+      // 1 ReplacedRowToColumnarExec
+      // so 11121 is expected.
+      assert(found == 11121)
 
       // Verify that we get back the expected, wrong, result
       val result = df.collect()
-      assert(result(0).getLong(0) == 102L) // Check that broken columnar Add was used.
-      assert(result(1).getLong(0) == 202L)
-      assert(result(2).getLong(0) == 302L)
+      assert(result(0).getLong(0) == 101L) // Check that broken columnar Add was used.
+      assert(result(1).getLong(0) == 201L)
+      assert(result(2).getLong(0) == 301L)
     }
   }
 
@@ -695,6 +728,16 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   def replaceWithColumnarPlan(plan: SparkPlan): SparkPlan =
     try {
       plan match {
+        case e: ShuffleExchangeExec =>
+          // note that this is not actually columnar but demonstrates that exchanges can
+          // be replaced.
+          val replaced = e.withNewChildren(e.children.map(replaceWithColumnarPlan))
+          MyShuffleExchangeExec(replaced.asInstanceOf[ShuffleExchangeExec])
+        case e: BroadcastExchangeExec =>
+          // note that this is not actually columnar but demonstrates that exchanges can
+          // be replaced.
+          val replaced = e.withNewChildren(e.children.map(replaceWithColumnarPlan))
+          MyBroadcastExchangeExec(replaced.asInstanceOf[BroadcastExchangeExec])
         case plan: ProjectExec =>
           new ColumnarProjectExec(plan.projectList.map((exp) =>
             replaceWithColumnarExpression(exp).asInstanceOf[NamedExpression]),
@@ -713,6 +756,41 @@ case class PreRuleReplaceAddWithBrokenVersion() extends Rule[SparkPlan] {
   override def apply(plan: SparkPlan): SparkPlan = replaceWithColumnarPlan(plan)
 }
 
+/**
+ * Custom Exchange used in tests to demonstrate that shuffles can be replaced regardless of
+ * whether AQE is enabled.
+ */
+case class MyShuffleExchangeExec(delegate: ShuffleExchangeExec) extends ShuffleExchangeLike {
+  override def numMappers: Int = delegate.numMappers
+  override def numPartitions: Int = delegate.numPartitions
+  override def canChangeNumPartitions: Boolean = delegate.canChangeNumPartitions
+  override def mapOutputStatisticsFuture: Future[MapOutputStatistics] =
+    delegate.mapOutputStatisticsFuture
+  override def getShuffleRDD(partitionSpecs: Array[ShufflePartitionSpec]): RDD[_] =
+    delegate.getShuffleRDD(partitionSpecs)
+  override def runtimeStatistics: Statistics = delegate.runtimeStatistics
+  override def child: SparkPlan = delegate.child
+  override protected def doExecute(): RDD[InternalRow] = delegate.execute()
+  override def outputPartitioning: Partitioning = delegate.outputPartitioning
+}
+
+/**
+ * Custom Exchange used in tests to demonstrate that broadcasts can be replaced regardless of
+ * whether AQE is enabled.
+ */
+case class MyBroadcastExchangeExec(delegate: BroadcastExchangeExec) extends BroadcastExchangeLike {
+  override def runId: UUID = delegate.runId
+  override def relationFuture: java.util.concurrent.Future[Broadcast[Any]] =
+    delegate.relationFuture
+  override def completionFuture: Future[Broadcast[Any]] = delegate.completionFuture
+  override def runtimeStatistics: Statistics = delegate.runtimeStatistics
+  override def child: SparkPlan = delegate.child
+  override protected def doPrepare(): Unit = delegate.prepare()
+  override protected def doExecute(): RDD[InternalRow] = delegate.execute()
+  override def doExecuteBroadcast[T](): Broadcast[T] = delegate.executeBroadcast()
+  override def outputPartitioning: Partitioning = delegate.outputPartitioning
+}
+
 class ReplacedRowToColumnarExec(override val child: SparkPlan)
   extends RowToColumnarExec(child) {
 

From ea4b288c5c616f09426d4b583ed5dcff14ccaa57 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Sat, 1 Aug 2020 13:09:26 +0800
Subject: [PATCH 0939/1280] [SPARK-32467][UI] Avoid encoding URL twice on https
 redirect

### What changes were proposed in this pull request?

When https is enabled for Spark UI, an HTTP request will be redirected as an encoded HTTPS URL: https://github.com/apache/spark/pull/10238/files#diff-f79a5ead735b3d0b34b6b94486918e1cR312

When we create the redirect url, we will call getRequestURI and getQueryString. Both two methods may return an encoded string. However, we pass them directly to the following URI constructor
```
URI(String scheme, String authority, String path, String query, String fragment)
```
As this URI constructor assumes both path and query parameters are decoded strings, it will encode them again. This makes the redirect URL encoded twice.

This problem is on stage page with HTTPS enabled. The URL of "/taskTable" contains query parameter `order%5B0%5D%5Bcolumn%5D`. After encoded it becomes  `order%255B0%255D%255Bcolumn%255D` and it will be decoded as `order%5B0%5D%5Bcolumn%5D` instead of `order[0][dir]`.  When the parameter `order[0][dir]` is missing, there will be an excetpion from:
https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala#L176
and the stage page fail to load.

To fix the problem, we can try decoding the query parameters before encoding it. This is to make sure we encode the URL

### Why are the changes needed?

Fix a UI issue when HTTPS is enabled

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

A new Unit test + manually test on a cluster

Closes #29271 from gengliangwang/urlEncode.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 71aea02e9ffb0c6f7c72c91054c2a4653e22e801)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../org/apache/spark/ui/JettyUtils.scala      | 29 +++++++++++++++----
 .../scala/org/apache/spark/ui/UISuite.scala   | 21 ++++++++++++++
 2 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 94c99d48e773c..a4ba5654b542e 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.ui
 
-import java.net.{URI, URL}
+import java.net.{URI, URL, URLDecoder}
 import java.util.EnumSet
 import javax.servlet.DispatcherType
 import javax.servlet.http._
@@ -376,8 +376,7 @@ private[spark] object JettyUtils extends Logging {
         if (baseRequest.isSecure) {
           return
         }
-        val httpsURI = createRedirectURI(scheme, baseRequest.getServerName, securePort,
-          baseRequest.getRequestURI, baseRequest.getQueryString)
+        val httpsURI = createRedirectURI(scheme, securePort, baseRequest)
         response.setContentLength(0)
         response.sendRedirect(response.encodeRedirectURL(httpsURI))
         baseRequest.setHandled(true)
@@ -439,16 +438,34 @@ private[spark] object JettyUtils extends Logging {
     handler.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType]))
   }
 
+  private def decodeURL(url: String, encoding: String): String = {
+    if (url == null) {
+      null
+    } else {
+      URLDecoder.decode(url, encoding)
+    }
+  }
+
   // Create a new URI from the arguments, handling IPv6 host encoding and default ports.
-  private def createRedirectURI(
-      scheme: String, server: String, port: Int, path: String, query: String) = {
+  private def createRedirectURI(scheme: String, port: Int, request: Request): String = {
+    val server = request.getServerName
     val redirectServer = if (server.contains(":") && !server.startsWith("[")) {
       s"[${server}]"
     } else {
       server
     }
     val authority = s"$redirectServer:$port"
-    new URI(scheme, authority, path, query, null).toString
+    val queryEncoding = if (request.getQueryEncoding != null) {
+      request.getQueryEncoding
+    } else {
+      // By default decoding the URI as "UTF-8" should be enough for SparkUI
+      "UTF-8"
+    }
+    // The request URL can be raw or encoded here. To avoid the request URL being
+    // encoded twice, let's decode it here.
+    val requestURI = decodeURL(request.getRequestURI, queryEncoding)
+    val queryString = decodeURL(request.getQueryString, queryEncoding)
+    new URI(scheme, authority, requestURI, queryString, null).toString
   }
 
   def toVirtualHosts(connectors: String*): Array[String] = connectors.map("@" + _).toArray
diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
index 2ad4a634cd9a7..56026eaa0072b 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
@@ -262,6 +262,27 @@ class UISuite extends SparkFunSuite {
     }
   }
 
+  test("SPARK-32467: Avoid encoding URL twice on https redirect") {
+    val (conf, securityMgr, sslOptions) = sslEnabledConf()
+    val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)
+    try {
+      val serverAddr = s"http://localhost:${serverInfo.boundPort}"
+
+      val (_, ctx) = newContext("/ctx1")
+      serverInfo.addHandler(ctx, securityMgr)
+
+      TestUtils.withHttpConnection(new URL(s"$serverAddr/ctx%281%29?a%5B0%5D=b")) { conn =>
+        assert(conn.getResponseCode() === HttpServletResponse.SC_FOUND)
+        val location = Option(conn.getHeaderFields().get("Location"))
+          .map(_.get(0)).orNull
+        val expectedLocation = s"https://localhost:${serverInfo.securePort.get}/ctx(1)?a[0]=b"
+        assert(location == expectedLocation)
+      }
+    } finally {
+      stopServer(serverInfo)
+    }
+  }
+
   test("http -> https redirect applies to all URIs") {
     val (conf, securityMgr, sslOptions) = sslEnabledConf()
     val serverInfo = JettyUtils.startJettyServer("0.0.0.0", 0, sslOptions, conf)

From d8d3e870a03f7695eb8fdbaaafa13d7df756801a Mon Sep 17 00:00:00 2001
From: Prakhar Jain <prakharjain09@gmail.com>
Date: Mon, 3 Aug 2020 03:26:03 +0000
Subject: [PATCH 0940/1280] [SPARK-32509][SQL] Ignore unused DPP True Filter in
 Canonicalization

### What changes were proposed in this pull request?
This PR fixes issues relate to Canonicalization of FileSourceScanExec when it contains unused DPP Filter.

### Why are the changes needed?

As part of PlanDynamicPruningFilter rule, the unused DPP Filter are simply replaced by `DynamicPruningExpression(TrueLiteral)` so that they can be avoided. But these unnecessary`DynamicPruningExpression(TrueLiteral)` partition filter inside the FileSourceScanExec affects the canonicalization of the node and so in many cases, this can prevent ReuseExchange from happening.

This PR fixes this issue by ignoring the unused DPP filter in the `def doCanonicalize` method.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT.

Closes #29318 from prakharjain09/SPARK-32509_df_reuse.

Authored-by: Prakhar Jain <prakharjain09@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 7a09e71198a094250f04e0f82f0c7c9860169540)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/DataSourceScanExec.scala    | 10 +++++++-
 .../sql/DynamicPartitionPruningSuite.scala    | 24 +++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 90a3f9788ca42..7f534d5957ce4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -589,12 +589,20 @@ case class FileSourceScanExec(
     new FileScanRDD(fsRelation.sparkSession, readFile, partitions)
   }
 
+  // Filters unused DynamicPruningExpression expressions - one which has been replaced
+  // with DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning
+  private def filterUnusedDynamicPruningExpressions(
+      predicates: Seq[Expression]): Seq[Expression] = {
+    predicates.filterNot(_ == DynamicPruningExpression(Literal.TrueLiteral))
+  }
+
   override def doCanonicalize(): FileSourceScanExec = {
     FileSourceScanExec(
       relation,
       output.map(QueryPlan.normalizeExpressions(_, output)),
       requiredSchema,
-      QueryPlan.normalizePredicates(partitionFilters, output),
+      QueryPlan.normalizePredicates(
+        filterUnusedDynamicPruningExpressions(partitionFilters), output),
       optionalBucketSet,
       QueryPlan.normalizePredicates(dataFilters, output),
       None)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index cdf9ea4b31ee7..1fbc4a6912adb 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -1240,6 +1240,30 @@ abstract class DynamicPartitionPruningSuiteBase
     }
   }
 
+  test("SPARK-32509: Unused Dynamic Pruning filter shouldn't affect " +
+    "canonicalization and exchange reuse") {
+    withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
+      withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
+        val df = sql(
+          """ WITH view1 as (
+            |   SELECT f.store_id FROM fact_stats f WHERE f.units_sold = 70
+            | )
+            |
+            | SELECT * FROM view1 v1 join view1 v2 WHERE v1.store_id = v2.store_id
+          """.stripMargin)
+
+        checkPartitionPruningPredicate(df, false, false)
+        val reuseExchangeNodes = df.queryExecution.executedPlan.collect {
+          case se: ReusedExchangeExec => se
+        }
+        assert(reuseExchangeNodes.size == 1, "Expected plan to contain 1 ReusedExchangeExec " +
+          s"nodes. Found ${reuseExchangeNodes.size}")
+
+        checkAnswer(df, Row(15, 15) :: Nil)
+      }
+    }
+  }
+
   test("Plan broadcast pruning only when the broadcast can be reused") {
     Given("dynamic pruning filter on the build side")
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {

From 64b3b56698dcb8446dd2801ce943938177d3fbd5 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 3 Aug 2020 12:56:23 +0000
Subject: [PATCH 0941/1280] [SPARK-32083][SQL][3.0] AQE should not coalesce
 partitions for SinglePartition

This is a partial backport of https://github.com/apache/spark/pull/29307

Most of the changes are not needed because https://github.com/apache/spark/pull/28226 is in master only.

This PR only backports the safeguard in `ShuffleExchangeExec.canChangeNumPartitions`

Closes #29321 from cloud-fan/aqe.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/execution/SparkStrategies.scala   | 4 ++--
 .../execution/adaptive/OptimizeLocalShuffleReader.scala    | 6 ++++--
 .../spark/sql/execution/exchange/ShuffleExchangeExec.scala | 7 ++++++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 28d1ccbf4db36..f836debcbafda 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -752,7 +752,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
       case logical.Repartition(numPartitions, shuffle, child) =>
         if (shuffle) {
           ShuffleExchangeExec(RoundRobinPartitioning(numPartitions),
-            planLater(child), canChangeNumPartitions = false) :: Nil
+            planLater(child), noUserSpecifiedNumPartition = false) :: Nil
         } else {
           execution.CoalesceExec(numPartitions, planLater(child)) :: Nil
         }
@@ -786,7 +786,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
         execution.RangeExec(r) :: Nil
       case r: logical.RepartitionByExpression =>
         exchange.ShuffleExchangeExec(
-          r.partitioning, planLater(r.child), canChangeNumPartitions = false) :: Nil
+          r.partitioning, planLater(r.child), noUserSpecifiedNumPartition = false) :: Nil
       case ExternalRDD(outputObjAttr, rdd) => ExternalRDDScanExec(outputObjAttr, rdd) :: Nil
       case r: LogicalRDD =>
         RDDScanExec(r.output, r.rdd, "ExistingRDD", r.outputPartitioning, r.outputOrdering) :: Nil
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
index 6684376b12539..31d1f34b64a65 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/OptimizeLocalShuffleReader.scala
@@ -141,8 +141,10 @@ object OptimizeLocalShuffleReader {
   }
 
   def canUseLocalShuffleReader(plan: SparkPlan): Boolean = plan match {
-    case s: ShuffleQueryStageExec => s.shuffle.canChangeNumPartitions
-    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, _, _) => s.shuffle.canChangeNumPartitions
+    case s: ShuffleQueryStageExec =>
+      s.shuffle.canChangeNumPartitions && s.mapStats.isDefined
+    case CustomShuffleReaderExec(s: ShuffleQueryStageExec, _, _) =>
+      s.shuffle.canChangeNumPartitions && s.mapStats.isDefined
     case _ => false
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
index b7da78cb0eefb..24c736951fdc4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/ShuffleExchangeExec.scala
@@ -83,7 +83,12 @@ trait ShuffleExchangeLike extends Exchange {
 case class ShuffleExchangeExec(
     override val outputPartitioning: Partitioning,
     child: SparkPlan,
-    canChangeNumPartitions: Boolean = true) extends ShuffleExchangeLike {
+    noUserSpecifiedNumPartition: Boolean = true) extends ShuffleExchangeLike {
+
+  // If users specify the num partitions via APIs like `repartition`, we shouldn't change it.
+  // For `SinglePartition`, it requires exactly one partition and we can't change it either.
+  def canChangeNumPartitions: Boolean =
+    noUserSpecifiedNumPartition && outputPartitioning != SinglePartition
 
   private lazy val writeMetrics =
     SQLShuffleWriteMetricsReporter.createShuffleWriteMetrics(sparkContext)

From c148a9810a1912dc65406e194570f1bb93d96ca4 Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Mon, 3 Aug 2020 22:57:24 +0900
Subject: [PATCH 0942/1280] [SPARK-32160][CORE][PYSPARK][3.0] Add a config to
 switch allow/disallow to create SparkContext in executors

### What changes were proposed in this pull request?

This is a backport of #29278, but with allowing to create `SparkContext` in executors by default.

This PR adds a config to switch allow/disallow to create `SparkContext` in executors.

- `spark.driver.allowSparkContextInExecutors`

### Why are the changes needed?

Some users or libraries actually create `SparkContext` in executors.
We shouldn't break their workloads.

### Does this PR introduce _any_ user-facing change?

Yes, users will be able to disallow to create `SparkContext` in executors with the config disabled.

### How was this patch tested?

More tests are added.

Closes #29294 from ueshin/issues/SPARK-32160/3.0/add_configs.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/SparkContext.scala |  6 +++--
 .../spark/internal/config/package.scala       |  6 +++++
 .../org/apache/spark/SparkContextSuite.scala  | 13 ++++++++--
 python/pyspark/context.py                     |  6 +++--
 python/pyspark/tests/test_context.py          | 20 ++++++++++++--
 .../org/apache/spark/sql/SparkSession.scala   | 12 ++++++---
 .../spark/sql/SparkSessionBuilderSuite.scala  | 26 ++++++++++++++++++-
 7 files changed, 76 insertions(+), 13 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 2761f0da5f772..c448eee5fca23 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -82,8 +82,10 @@ class SparkContext(config: SparkConf) extends Logging {
   // The call site where this SparkContext was constructed.
   private val creationSite: CallSite = Utils.getCallSite()
 
-  // In order to prevent SparkContext from being created in executors.
-  SparkContext.assertOnDriver()
+  if (!config.get(ALLOW_SPARK_CONTEXT_IN_EXECUTORS)) {
+    // In order to prevent SparkContext from being created in executors.
+    SparkContext.assertOnDriver()
+  }
 
   // In order to prevent multiple SparkContexts from being active at the same time, mark this
   // context as having started construction.
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index bfdd06021757a..38eb90c57ef68 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1814,4 +1814,10 @@ package object config {
     .bytesConf(ByteUnit.BYTE)
     .createOptional
 
+  private[spark] val ALLOW_SPARK_CONTEXT_IN_EXECUTORS =
+    ConfigBuilder("spark.driver.allowSparkContextInExecutors")
+      .doc("If set to true, SparkContext can be created in executors.")
+      .version("3.0.1")
+      .booleanConf
+      .createWithDefault(true)
 }
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index 2b1e110a39466..5533f42859bd1 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -951,17 +951,26 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     }
   }
 
-  test("SPARK-32160: Disallow to create SparkContext in executors") {
+  test("SPARK-32160: Disallow to create SparkContext in executors if the config is set") {
     sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
 
     val error = intercept[SparkException] {
       sc.range(0, 1).foreach { _ =>
-        new SparkContext(new SparkConf().setAppName("test").setMaster("local"))
+        new SparkContext(new SparkConf().setAppName("test").setMaster("local")
+          .set(ALLOW_SPARK_CONTEXT_IN_EXECUTORS, false))
       }
     }.getMessage()
 
     assert(error.contains("SparkContext should only be created and accessed on the driver."))
   }
+
+  test("SPARK-32160: Allow to create SparkContext in executors") {
+    sc = new SparkContext(new SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+
+    sc.range(0, 1).foreach { _ =>
+      new SparkContext(new SparkConf().setAppName("test").setMaster("local")).stop()
+    }
+  }
 }
 
 object SparkContextSuite {
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index ecd171a0b8b41..3c447a10b7058 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -119,8 +119,10 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
             ...
         ValueError:...
         """
-        # In order to prevent SparkContext from being created in executors.
-        SparkContext._assert_on_driver()
+        if (conf is not None and
+                conf.get("spark.driver.allowSparkContextInExecutors", "true").lower() != "true"):
+            # In order to prevent SparkContext from being created in executors.
+            SparkContext._assert_on_driver()
 
         self._callsite = first_spark_call() or CallSite(None, None, None)
         if gateway is not None and gateway.gateway_parameters.auth_token is None:
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index 303635ddab12c..9468b2511ce03 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -268,13 +268,29 @@ def test_resources(self):
             self.assertEqual(len(resources), 0)
 
     def test_disallow_to_create_spark_context_in_executors(self):
-        # SPARK-32160: SparkContext should not be created in executors.
+        # SPARK-32160: SparkContext should not created in executors if the config is set.
+
+        def create_spark_context():
+            conf = SparkConf().set("spark.driver.allowSparkContextInExecutors", "false")
+            with SparkContext(conf=conf):
+                pass
+
         with SparkContext("local-cluster[3, 1, 1024]") as sc:
             with self.assertRaises(Exception) as context:
-                sc.range(2).foreach(lambda _: SparkContext())
+                sc.range(2).foreach(lambda _: create_spark_context())
             self.assertIn("SparkContext should only be created and accessed on the driver.",
                           str(context.exception))
 
+    def test_allow_to_create_spark_context_in_executors(self):
+        # SPARK-32160: SparkContext can be created in executors.
+
+        def create_spark_context():
+            with SparkContext():
+                pass
+
+        with SparkContext("local-cluster[3, 1, 1024]") as sc:
+            sc.range(2).foreach(lambda _: create_spark_context())
+
 
 class ContextTestsWithResources(unittest.TestCase):
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index e5d8710bfb69f..6c809c8592522 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -29,6 +29,7 @@ import org.apache.spark.{SPARK_VERSION, SparkConf, SparkContext, TaskContext}
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Stable, Unstable}
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config.ALLOW_SPARK_CONTEXT_IN_EXECUTORS
 import org.apache.spark.rdd.RDD
 import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
 import org.apache.spark.sql.catalog.Catalog
@@ -900,7 +901,13 @@ object SparkSession extends Logging {
      * @since 2.0.0
      */
     def getOrCreate(): SparkSession = synchronized {
-      assertOnDriver()
+      val sparkConf = new SparkConf()
+      options.foreach { case (k, v) => sparkConf.set(k, v) }
+
+      if (!sparkConf.get(ALLOW_SPARK_CONTEXT_IN_EXECUTORS)) {
+        assertOnDriver()
+      }
+
       // Get the session from current thread's active session.
       var session = activeThreadSession.get()
       if ((session ne null) && !session.sparkContext.isStopped) {
@@ -919,9 +926,6 @@ object SparkSession extends Logging {
 
         // No active nor global default session. Create a new one.
         val sparkContext = userSuppliedContext.getOrElse {
-          val sparkConf = new SparkConf()
-          options.foreach { case (k, v) => sparkConf.set(k, v) }
-
           // set a random app name if not given.
           if (!sparkConf.contains("spark.app.name")) {
             sparkConf.setAppName(java.util.UUID.randomUUID().toString)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 0a522fdbdeed8..6983cda5a35cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.sql
 
 import org.scalatest.BeforeAndAfterEach
 
-import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkContext, SparkException, SparkFunSuite}
+import org.apache.spark.internal.config.ALLOW_SPARK_CONTEXT_IN_EXECUTORS
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf._
@@ -240,4 +241,27 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     assert(session.conf.get(GLOBAL_TEMP_DATABASE) === "globaltempdb-spark-31532-2")
     assert(session.conf.get(WAREHOUSE_PATH) === "SPARK-31532-db-2")
   }
+
+  test("SPARK-32160: Disallow to create SparkSession in executors if the config is set") {
+    val session = SparkSession.builder().master("local-cluster[3, 1, 1024]").getOrCreate()
+
+    val error = intercept[SparkException] {
+      session.range(1).foreach { v =>
+        SparkSession.builder.master("local")
+          .config(ALLOW_SPARK_CONTEXT_IN_EXECUTORS.key, false).getOrCreate()
+        ()
+      }
+    }.getMessage()
+
+    assert(error.contains("SparkSession should only be created and accessed on the driver."))
+  }
+
+  test("SPARK-32160: Allow to create SparkSession in executors") {
+    val session = SparkSession.builder().master("local-cluster[3, 1, 1024]").getOrCreate()
+
+    session.range(1).foreach { v =>
+      SparkSession.builder.master("local").getOrCreate().stop()
+      ()
+    }
+  }
 }

From 6d7ae4a557b89d87a639c58bcd94c96763b8d5a7 Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Tue, 4 Aug 2020 12:43:01 +0900
Subject: [PATCH 0943/1280] [SPARK-32160][CORE][PYSPARK][3.0][FOLLOWUP] Change
 the config name to switch allow/disallow SparkContext in executors

### What changes were proposed in this pull request?

This is a follow-up of #29294.
This PR changes the config name to switch allow/disallow `SparkContext` in executors as per the comment https://github.com/apache/spark/pull/29278#pullrequestreview-460256338.

### Why are the changes needed?

The config name `spark.executor.allowSparkContext` is more reasonable.

### Does this PR introduce _any_ user-facing change?

Yes, the config name is changed.

### How was this patch tested?

Updated tests.

Closes #29341 from ueshin/issues/SPARK-32160/3.0/change_config_name.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/SparkContext.scala       | 2 +-
 .../main/scala/org/apache/spark/internal/config/package.scala | 4 ++--
 core/src/test/scala/org/apache/spark/SparkContextSuite.scala  | 2 +-
 python/pyspark/context.py                                     | 2 +-
 python/pyspark/tests/test_context.py                          | 2 +-
 .../src/main/scala/org/apache/spark/sql/SparkSession.scala    | 4 ++--
 .../scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala | 4 ++--
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index c448eee5fca23..65c08cff2ba11 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -82,7 +82,7 @@ class SparkContext(config: SparkConf) extends Logging {
   // The call site where this SparkContext was constructed.
   private val creationSite: CallSite = Utils.getCallSite()
 
-  if (!config.get(ALLOW_SPARK_CONTEXT_IN_EXECUTORS)) {
+  if (!config.get(EXECUTOR_ALLOW_SPARK_CONTEXT)) {
     // In order to prevent SparkContext from being created in executors.
     SparkContext.assertOnDriver()
   }
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 38eb90c57ef68..0ff6b73d20dff 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1814,8 +1814,8 @@ package object config {
     .bytesConf(ByteUnit.BYTE)
     .createOptional
 
-  private[spark] val ALLOW_SPARK_CONTEXT_IN_EXECUTORS =
-    ConfigBuilder("spark.driver.allowSparkContextInExecutors")
+  private[spark] val EXECUTOR_ALLOW_SPARK_CONTEXT =
+    ConfigBuilder("spark.executor.allowSparkContext")
       .doc("If set to true, SparkContext can be created in executors.")
       .version("3.0.1")
       .booleanConf
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index 5533f42859bd1..ce437a5e42b22 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -957,7 +957,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
     val error = intercept[SparkException] {
       sc.range(0, 1).foreach { _ =>
         new SparkContext(new SparkConf().setAppName("test").setMaster("local")
-          .set(ALLOW_SPARK_CONTEXT_IN_EXECUTORS, false))
+          .set(EXECUTOR_ALLOW_SPARK_CONTEXT, false))
       }
     }.getMessage()
 
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 3c447a10b7058..72e8e84cba757 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -120,7 +120,7 @@ def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None,
         ValueError:...
         """
         if (conf is not None and
-                conf.get("spark.driver.allowSparkContextInExecutors", "true").lower() != "true"):
+                conf.get("spark.executor.allowSparkContext", "true").lower() != "true"):
             # In order to prevent SparkContext from being created in executors.
             SparkContext._assert_on_driver()
 
diff --git a/python/pyspark/tests/test_context.py b/python/pyspark/tests/test_context.py
index 9468b2511ce03..29b49fd4417e1 100644
--- a/python/pyspark/tests/test_context.py
+++ b/python/pyspark/tests/test_context.py
@@ -271,7 +271,7 @@ def test_disallow_to_create_spark_context_in_executors(self):
         # SPARK-32160: SparkContext should not created in executors if the config is set.
 
         def create_spark_context():
-            conf = SparkConf().set("spark.driver.allowSparkContextInExecutors", "false")
+            conf = SparkConf().set("spark.executor.allowSparkContext", "false")
             with SparkContext(conf=conf):
                 pass
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 6c809c8592522..fc941270e1b64 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -29,7 +29,7 @@ import org.apache.spark.{SPARK_VERSION, SparkConf, SparkContext, TaskContext}
 import org.apache.spark.annotation.{DeveloperApi, Experimental, Stable, Unstable}
 import org.apache.spark.api.java.JavaRDD
 import org.apache.spark.internal.Logging
-import org.apache.spark.internal.config.ALLOW_SPARK_CONTEXT_IN_EXECUTORS
+import org.apache.spark.internal.config.EXECUTOR_ALLOW_SPARK_CONTEXT
 import org.apache.spark.rdd.RDD
 import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
 import org.apache.spark.sql.catalog.Catalog
@@ -904,7 +904,7 @@ object SparkSession extends Logging {
       val sparkConf = new SparkConf()
       options.foreach { case (k, v) => sparkConf.set(k, v) }
 
-      if (!sparkConf.get(ALLOW_SPARK_CONTEXT_IN_EXECUTORS)) {
+      if (!sparkConf.get(EXECUTOR_ALLOW_SPARK_CONTEXT)) {
         assertOnDriver()
       }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index 6983cda5a35cc..159d2c02188be 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
 import org.scalatest.BeforeAndAfterEach
 
 import org.apache.spark.{SparkConf, SparkContext, SparkException, SparkFunSuite}
-import org.apache.spark.internal.config.ALLOW_SPARK_CONTEXT_IN_EXECUTORS
+import org.apache.spark.internal.config.EXECUTOR_ALLOW_SPARK_CONTEXT
 import org.apache.spark.internal.config.UI.UI_ENABLED
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf._
@@ -248,7 +248,7 @@ class SparkSessionBuilderSuite extends SparkFunSuite with BeforeAndAfterEach {
     val error = intercept[SparkException] {
       session.range(1).foreach { v =>
         SparkSession.builder.master("local")
-          .config(ALLOW_SPARK_CONTEXT_IN_EXECUTORS.key, false).getOrCreate()
+          .config(EXECUTOR_ALLOW_SPARK_CONTEXT.key, false).getOrCreate()
         ()
       }
     }.getMessage()

From fd445cb46ce940fbc68f85d2db14367d9f58739a Mon Sep 17 00:00:00 2001
From: Wing Yew Poon <wypoon@cloudera.com>
Date: Tue, 4 Aug 2020 11:33:56 -0500
Subject: [PATCH 0944/1280] [SPARK-32003][CORE][3.0] When external shuffle
 service is used, unregister outputs for executor on fetch failure after
 executor is lost

### What changes were proposed in this pull request?

If an executor is lost, the `DAGScheduler` handles the executor loss by removing the executor but does not unregister its outputs if the external shuffle service is used. However, if the node on which the executor runs is lost, the shuffle service may not be able to serve the shuffle files.
In such a case, when fetches from the executor's outputs fail in the same stage, the `DAGScheduler` again removes the executor and by right, should unregister its outputs. It doesn't because the epoch used to track the executor failure has not increased.

We track the epoch for failed executors that result in lost file output separately, so we can unregister the outputs in this scenario. The idea to track a second epoch is due to Attila Zsolt Piros.

### Why are the changes needed?

Without the changes, the loss of a node could require two stage attempts to recover instead of one.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

New unit test. This test fails without the change and passes with it.

Closes #29193 from wypoon/SPARK-32003-3.0.

Authored-by: Wing Yew Poon <wypoon@cloudera.com>
Signed-off-by: Imran Rashid <irashid@cloudera.com>
---
 .../apache/spark/scheduler/DAGScheduler.scala | 100 +++++++++++------
 .../spark/scheduler/DAGSchedulerSuite.scala   | 101 +++++++++++++++---
 2 files changed, 151 insertions(+), 50 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index ec6eff3eed462..51445bf451bec 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -168,13 +168,34 @@ private[spark] class DAGScheduler(
    */
   private val cacheLocs = new HashMap[Int, IndexedSeq[Seq[TaskLocation]]]
 
-  // For tracking failed nodes, we use the MapOutputTracker's epoch number, which is sent with
-  // every task. When we detect a node failing, we note the current epoch number and failed
-  // executor, increment it for new tasks, and use this to ignore stray ShuffleMapTask results.
-  //
-  // TODO: Garbage collect information about failure epochs when we know there are no more
-  //       stray messages to detect.
-  private val failedEpoch = new HashMap[String, Long]
+  /**
+   * Tracks the latest epoch of a fully processed error related to the given executor. (We use
+   * the MapOutputTracker's epoch number, which is sent with every task.)
+   *
+   * When an executor fails, it can affect the results of many tasks, and we have to deal with
+   * all of them consistently. We don't simply ignore all future results from that executor,
+   * as the failures may have been transient; but we also don't want to "overreact" to follow-
+   * on errors we receive. Furthermore, we might receive notification of a task success, after
+   * we find out the executor has actually failed; we'll assume those successes are, in fact,
+   * simply delayed notifications and the results have been lost, if the tasks started in the
+   * same or an earlier epoch. In particular, we use this to control when we tell the
+   * BlockManagerMaster that the BlockManager has been lost.
+   */
+  private val executorFailureEpoch = new HashMap[String, Long]
+
+  /**
+   * Tracks the latest epoch of a fully processed error where shuffle files have been lost from
+   * the given executor.
+   *
+   * This is closely related to executorFailureEpoch. They only differ for the executor when
+   * there is an external shuffle service serving shuffle files and we haven't been notified that
+   * the entire worker has been lost. In that case, when an executor is lost, we do not update
+   * the shuffleFileLostEpoch; we wait for a fetch failure. This way, if only the executor
+   * fails, we do not unregister the shuffle data as it can still be served; but if there is
+   * a failure in the shuffle service (resulting in fetch failure), we unregister the shuffle
+   * data only once, even if we get many fetch failures.
+   */
+  private val shuffleFileLostEpoch = new HashMap[String, Long]
 
   private [scheduler] val outputCommitCoordinator = env.outputCommitCoordinator
 
@@ -1472,7 +1493,8 @@ private[spark] class DAGScheduler(
             val status = event.result.asInstanceOf[MapStatus]
             val execId = status.location.executorId
             logDebug("ShuffleMapTask finished on " + execId)
-            if (failedEpoch.contains(execId) && smt.epoch <= failedEpoch(execId)) {
+            if (executorFailureEpoch.contains(execId) &&
+                smt.epoch <= executorFailureEpoch(execId)) {
               logInfo(s"Ignoring possibly bogus $smt completion from executor $execId")
             } else {
               // The epoch of the task is acceptable (i.e., the task was launched after the most
@@ -1818,12 +1840,8 @@ private[spark] class DAGScheduler(
    * modify the scheduler's internal state. Use executorLost() to post a loss event from outside.
    *
    * We will also assume that we've lost all shuffle blocks associated with the executor if the
-   * executor serves its own blocks (i.e., we're not using external shuffle), the entire slave
-   * is lost (likely including the shuffle service), or a FetchFailed occurred, in which case we
-   * presume all shuffle data related to this executor to be lost.
-   *
-   * Optionally the epoch during which the failure was caught can be passed to avoid allowing
-   * stray fetch failures from possibly retriggering the detection of a node as lost.
+   * executor serves its own blocks (i.e., we're not using an external shuffle service), or the
+   * entire Standalone worker is lost.
    */
   private[scheduler] def handleExecutorLost(
       execId: String,
@@ -1839,29 +1857,44 @@ private[spark] class DAGScheduler(
       maybeEpoch = None)
   }
 
+  /**
+   * Handles removing an executor from the BlockManagerMaster as well as unregistering shuffle
+   * outputs for the executor or optionally its host.
+   *
+   * @param execId executor to be removed
+   * @param fileLost If true, indicates that we assume we've lost all shuffle blocks associated
+   *   with the executor; this happens if the executor serves its own blocks (i.e., we're not
+   *   using an external shuffle service), the entire Standalone worker is lost, or a FetchFailed
+   *   occurred (in which case we presume all shuffle data related to this executor to be lost).
+   * @param hostToUnregisterOutputs (optional) executor host if we're unregistering all the
+   *   outputs on the host
+   * @param maybeEpoch (optional) the epoch during which the failure was caught (this prevents
+   *   reprocessing for follow-on fetch failures)
+   */
   private def removeExecutorAndUnregisterOutputs(
       execId: String,
       fileLost: Boolean,
       hostToUnregisterOutputs: Option[String],
       maybeEpoch: Option[Long] = None): Unit = {
     val currentEpoch = maybeEpoch.getOrElse(mapOutputTracker.getEpoch)
-    if (!failedEpoch.contains(execId) || failedEpoch(execId) < currentEpoch) {
-      failedEpoch(execId) = currentEpoch
-      logInfo("Executor lost: %s (epoch %d)".format(execId, currentEpoch))
+    logDebug(s"Considering removal of executor $execId; " +
+      s"fileLost: $fileLost, currentEpoch: $currentEpoch")
+    if (!executorFailureEpoch.contains(execId) || executorFailureEpoch(execId) < currentEpoch) {
+      executorFailureEpoch(execId) = currentEpoch
+      logInfo(s"Executor lost: $execId (epoch $currentEpoch)")
       blockManagerMaster.removeExecutor(execId)
-      if (fileLost) {
-        hostToUnregisterOutputs match {
-          case Some(host) =>
-            logInfo("Shuffle files lost for host: %s (epoch %d)".format(host, currentEpoch))
-            mapOutputTracker.removeOutputsOnHost(host)
-          case None =>
-            logInfo("Shuffle files lost for executor: %s (epoch %d)".format(execId, currentEpoch))
-            mapOutputTracker.removeOutputsOnExecutor(execId)
-        }
-        clearCacheLocs()
-
-      } else {
-        logDebug("Additional executor lost message for %s (epoch %d)".format(execId, currentEpoch))
+      clearCacheLocs()
+    }
+    if (fileLost &&
+        (!shuffleFileLostEpoch.contains(execId) || shuffleFileLostEpoch(execId) < currentEpoch)) {
+      shuffleFileLostEpoch(execId) = currentEpoch
+      hostToUnregisterOutputs match {
+        case Some(host) =>
+          logInfo(s"Shuffle files lost for host: $host (epoch $currentEpoch)")
+          mapOutputTracker.removeOutputsOnHost(host)
+        case None =>
+          logInfo(s"Shuffle files lost for executor: $execId (epoch $currentEpoch)")
+          mapOutputTracker.removeOutputsOnExecutor(execId)
       }
     }
   }
@@ -1887,11 +1920,12 @@ private[spark] class DAGScheduler(
   }
 
   private[scheduler] def handleExecutorAdded(execId: String, host: String): Unit = {
-    // remove from failedEpoch(execId) ?
-    if (failedEpoch.contains(execId)) {
+    // remove from executorFailureEpoch(execId) ?
+    if (executorFailureEpoch.contains(execId)) {
       logInfo("Host added was in lost list earlier: " + host)
-      failedEpoch -= execId
+      executorFailureEpoch -= execId
     }
+    shuffleFileLostEpoch -= execId
   }
 
   private[scheduler] def handleStageCancellation(stageId: Int, reason: Option[String]): Unit = {
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 02bc216d2192d..d92bd5bcba5ad 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -25,6 +25,9 @@ import scala.annotation.meta.param
 import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet, Map}
 import scala.util.control.NonFatal
 
+import org.mockito.Mockito.spy
+import org.mockito.Mockito.times
+import org.mockito.Mockito.verify
 import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
 import org.scalatest.exceptions.TestFailedException
 import org.scalatest.time.SpanSugar._
@@ -232,6 +235,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
 
   var sparkListener: EventInfoRecordingListener = null
 
+  var blockManagerMaster: BlockManagerMaster = null
   var mapOutputTracker: MapOutputTrackerMaster = null
   var broadcastManager: BroadcastManager = null
   var securityMgr: SecurityManager = null
@@ -245,17 +249,18 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
    */
   val cacheLocations = new HashMap[(Int, Int), Seq[BlockManagerId]]
   // stub out BlockManagerMaster.getLocations to use our cacheLocations
-  val blockManagerMaster = new BlockManagerMaster(null, null, conf, true) {
-      override def getLocations(blockIds: Array[BlockId]): IndexedSeq[Seq[BlockManagerId]] = {
-        blockIds.map {
-          _.asRDDId.map(id => (id.rddId -> id.splitIndex)).flatMap(key => cacheLocations.get(key)).
-            getOrElse(Seq())
-        }.toIndexedSeq
-      }
-      override def removeExecutor(execId: String): Unit = {
-        // don't need to propagate to the driver, which we don't have
-      }
+  class MyBlockManagerMaster(conf: SparkConf) extends BlockManagerMaster(null, null, conf, true) {
+    override def getLocations(blockIds: Array[BlockId]): IndexedSeq[Seq[BlockManagerId]] = {
+      blockIds.map {
+        _.asRDDId.map { id => (id.rddId -> id.splitIndex)
+        }.flatMap { key => cacheLocations.get(key)
+        }.getOrElse(Seq())
+      }.toIndexedSeq
     }
+    override def removeExecutor(execId: String): Unit = {
+      // don't need to propagate to the driver, which we don't have
+    }
+  }
 
   /** The list of results that DAGScheduler has collected. */
   val results = new HashMap[Int, Any]()
@@ -273,6 +278,16 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     override def jobFailed(exception: Exception): Unit = { failure = exception }
   }
 
+  class MyMapOutputTrackerMaster(
+      conf: SparkConf,
+      broadcastManager: BroadcastManager)
+    extends MapOutputTrackerMaster(conf, broadcastManager, true) {
+
+    override def sendTracker(message: Any): Unit = {
+      // no-op, just so we can stop this to avoid leaking threads
+    }
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     init(new SparkConf())
@@ -290,11 +305,8 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     results.clear()
     securityMgr = new SecurityManager(conf)
     broadcastManager = new BroadcastManager(true, conf, securityMgr)
-    mapOutputTracker = new MapOutputTrackerMaster(conf, broadcastManager, true) {
-      override def sendTracker(message: Any): Unit = {
-        // no-op, just so we can stop this to avoid leaking threads
-      }
-    }
+    mapOutputTracker = spy(new MyMapOutputTrackerMaster(conf, broadcastManager))
+    blockManagerMaster = spy(new MyBlockManagerMaster(conf))
     scheduler = new DAGScheduler(
       sc,
       taskScheduler,
@@ -537,6 +549,59 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     assert(mapStatus2(2).location.host === "hostB")
   }
 
+  test("SPARK-32003: All shuffle files for executor should be cleaned up on fetch failure") {
+    // reset the test context with the right shuffle service config
+    afterEach()
+    val conf = new SparkConf()
+    conf.set(config.SHUFFLE_SERVICE_ENABLED.key, "true")
+    init(conf)
+
+    val shuffleMapRdd = new MyRDD(sc, 3, Nil)
+    val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(3))
+    val shuffleId = shuffleDep.shuffleId
+    val reduceRdd = new MyRDD(sc, 3, List(shuffleDep), tracker = mapOutputTracker)
+
+    submit(reduceRdd, Array(0, 1, 2))
+    // Map stage completes successfully,
+    // two tasks are run on an executor on hostA and one on an executor on hostB
+    complete(taskSets(0), Seq(
+      (Success, makeMapStatus("hostA", 3)),
+      (Success, makeMapStatus("hostA", 3)),
+      (Success, makeMapStatus("hostB", 3))))
+    // Now the executor on hostA is lost
+    runEvent(ExecutorLost("exec-hostA", ExecutorExited(-100, false, "Container marked as failed")))
+    // Executor is removed but shuffle files are not unregistered
+    verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
+    verify(mapOutputTracker, times(0)).removeOutputsOnExecutor("exec-hostA")
+
+    // The MapOutputTracker has all the shuffle files
+    val mapStatuses = mapOutputTracker.shuffleStatuses(shuffleId).mapStatuses
+    assert(mapStatuses.count(_ != null) === 3)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostA") === 2)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostB") === 1)
+
+    // Now a fetch failure from the lost executor occurs
+    complete(taskSets(1), Seq(
+      (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)
+    ))
+    // blockManagerMaster.removeExecutor is not called again
+    // but shuffle files are unregistered
+    verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
+    verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("exec-hostA")
+
+    // Shuffle files for exec-hostA should be lost
+    assert(mapStatuses.count(_ != null) === 1)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostA") === 0)
+    assert(mapStatuses.count(s => s != null && s.location.executorId == "exec-hostB") === 1)
+
+    // Additional fetch failure from the executor does not result in further call to
+    // mapOutputTracker.removeOutputsOnExecutor
+    complete(taskSets(1), Seq(
+      (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 1, 0, "ignored"), null)
+    ))
+    verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("exec-hostA")
+  }
+
   test("zero split job") {
     var numResults = 0
     var failureReason: Option[Exception] = None
@@ -762,8 +827,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
     complete(taskSets(1), Seq(
       (Success, 42),
       (FetchFailed(makeBlockManagerId("hostA"), shuffleId, 0L, 0, 0, "ignored"), null)))
-    // this will get called
-    // blockManagerMaster.removeExecutor("exec-hostA")
+    verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
     // ask the scheduler to try it again
     scheduler.resubmitFailedStages()
     // have the 2nd attempt pass
@@ -806,11 +870,14 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi
         (Success, makeMapStatus("hostA", 1)),
         (Success, makeMapStatus("hostB", 1))))
       runEvent(ExecutorLost("exec-hostA", event))
+      verify(blockManagerMaster, times(1)).removeExecutor("exec-hostA")
       if (expectFileLoss) {
+        verify(mapOutputTracker, times(1)).removeOutputsOnExecutor("exec-hostA")
         intercept[MetadataFetchFailedException] {
           mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0)
         }
       } else {
+        verify(mapOutputTracker, times(0)).removeOutputsOnExecutor("exec-hostA")
         assert(mapOutputTracker.getMapSizesByExecutorId(shuffleId, 0).map(_._1).toSet ===
           HashSet(makeBlockManagerId("hostA"), makeBlockManagerId("hostB")))
       }

From ab5034fd181406037dd3273dc8b3ef3af8e0c63a Mon Sep 17 00:00:00 2001
From: Yan Xiaole <xiaole.yan@gmail.com>
Date: Wed, 5 Aug 2020 10:57:11 -0700
Subject: [PATCH 0945/1280] [SPARK-32529][CORE] Fix Historyserver log scan
 aborted by application status change

# What changes were proposed in this pull request?
This PR adds a `FileNotFoundException` try catch block while adding a new entry to history server application listing to skip the non-existing path.

### Why are the changes needed?
If there are a large number (>100k) of applications log dir, listing the log dir will take a few seconds. After getting the path list some applications might have finished already, and the filename will change from `foo.inprogress` to `foo`.

It leads to a problem when adding an entry to the listing, querying file status like `fileSizeForLastIndex` will throw out a `FileNotFoundException` exception if the application was finished. And the exception will abort current loop, in a busy cluster, it will make history server couldn't list and load any application log.

```
20/08/03 15:17:23 ERROR FsHistoryProvider: Exception in checking for event log updates
 java.io.FileNotFoundException: File does not exist: hdfs://xx/logs/spark/application_11111111111111.lz4.inprogress
 at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1527)
 at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1520)
 at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
 at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1520)
 at org.apache.spark.deploy.history.SingleFileEventLogFileReader.status$lzycompute(EventLogFileReaders.scala:170)
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
1. setup another script keeps changing the filename of applications under history log dir
2. launch the history server
3. check whether the `File does not exist` error log was gone.

Closes #29350 from yanxiaole/SPARK-32529.

Authored-by: Yan Xiaole <xiaole.yan@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c1d17df826541580162c9db8ebfbc408ec0c9922)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/deploy/history/FsHistoryProvider.scala  | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 99d3eceb1121a..c26215295e914 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -519,10 +519,17 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
               // If the file is currently not being tracked by the SHS, add an entry for it and try
               // to parse it. This will allow the cleaner code to detect the file as stale later on
               // if it was not possible to parse it.
-              listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime, LogType.EventLogs,
-                None, None, reader.fileSizeForLastIndex, reader.lastIndex, None,
-                reader.completed))
-              reader.fileSizeForLastIndex > 0
+              try {
+                listing.write(LogInfo(reader.rootPath.toString(), newLastScanTime,
+                  LogType.EventLogs, None, None, reader.fileSizeForLastIndex, reader.lastIndex,
+                  None, reader.completed))
+                reader.fileSizeForLastIndex > 0
+              } catch {
+                case _: FileNotFoundException => false
+              }
+
+            case _: FileNotFoundException =>
+              false
           }
         }
         .sortWith { case (entry1, entry2) =>

From d3eea05f5ffe626bd1390723795e133f912cf8f9 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Thu, 6 Aug 2020 13:30:42 +0000
Subject: [PATCH 0946/1280] [SPARK-32546][SQL][3.0] Get table names directly
 from Hive tables

### What changes were proposed in this pull request?
Get table names directly from a sequence of Hive tables in `HiveClientImpl.listTablesByType()` by skipping conversions Hive tables to Catalog tables.

### Why are the changes needed?
A Hive metastore can be shared across many clients. A client can create tables using a SerDe which is not available on other clients, for instance `ROW FORMAT SERDE "com.ibm.spss.hive.serde2.xml.XmlSerDe"`. In the current implementation, other clients get the following exception while getting views:
```
java.lang.RuntimeException: MetaException(message:java.lang.ClassNotFoundException Class com.ibm.spss.hive.serde2.xml.XmlSerDe not found)
```
when `com.ibm.spss.hive.serde2.xml.XmlSerDe` is not available.

### Does this PR introduce _any_ user-facing change?
Yes. For example, `SHOW VIEWS` returns a list of views instead of throwing an exception.

### How was this patch tested?
- By existing test suites like:
```
$ build/sbt -Phive-2.3 "test:testOnly org.apache.spark.sql.hive.client.VersionsSuite"
```
- And manually:

1. Build Spark with Hive 1.2: `./build/sbt package -Phive-1.2 -Phive -Dhadoop.version=2.8.5`

2. Run spark-shell with a custom Hive SerDe, for instance download [json-serde-1.3.8-jar-with-dependencies.jar](https://github.com/cdamak/Twitter-Hive/blob/master/json-serde-1.3.8-jar-with-dependencies.jar) from https://github.com/cdamak/Twitter-Hive:
```
$ ./bin/spark-shell --jars ../Downloads/json-serde-1.3.8-jar-with-dependencies.jar
```

3. Create a Hive table using this SerDe:
```scala
scala> :paste
// Entering paste mode (ctrl-D to finish)

sql(s"""
  |CREATE TABLE json_table2(page_id INT NOT NULL)
  |ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
  |""".stripMargin)

// Exiting paste mode, now interpreting.
res0: org.apache.spark.sql.DataFrame = []

scala> sql("SHOW TABLES").show
+--------+-----------+-----------+
|database|  tableName|isTemporary|
+--------+-----------+-----------+
| default|json_table2|      false|
+--------+-----------+-----------+

scala> sql("SHOW VIEWS").show
+---------+--------+-----------+
|namespace|viewName|isTemporary|
+---------+--------+-----------+
+---------+--------+-----------+
```

4. Quit from the current `spark-shell` and run it without jars:
```
$ ./bin/spark-shell
```

5. Show views. Without the fix, it throws the exception:
```scala
scala> sql("SHOW VIEWS").show
20/08/06 10:53:36 ERROR log: error in initSerDe: java.lang.ClassNotFoundException Class org.openx.data.jsonserde.JsonSerDe not found
java.lang.ClassNotFoundException: Class org.openx.data.jsonserde.JsonSerDe not found
	at org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:2273)
	at org.apache.hadoop.hive.metastore.MetaStoreUtils.getDeserializer(MetaStoreUtils.java:385)
	at org.apache.hadoop.hive.ql.metadata.Table.getDeserializerFromMetaStore(Table.java:276)
	at org.apache.hadoop.hive.ql.metadata.Table.getDeserializer(Table.java:258)
	at org.apache.hadoop.hive.ql.metadata.Table.getCols(Table.java:605)
```

After the fix:
```scala
scala> sql("SHOW VIEWS").show
+---------+--------+-----------+
|namespace|viewName|isTemporary|
+---------+--------+-----------+
+---------+--------+-----------+
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit dc96f2f8d6e08c4bc30bc11d6b29109d2aeb604b)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #29377 from MaxGekk/fix-listTablesByType-for-views-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 6ad5e9d3c9080..d131a6bef06e7 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -759,15 +759,17 @@ private[hive] class HiveClientImpl(
       dbName: String,
       pattern: String,
       tableType: CatalogTableType): Seq[String] = withHiveState {
+    val hiveTableType = toHiveTableType(tableType)
     try {
       // Try with Hive API getTablesByType first, it's supported from Hive 2.3+.
-      shim.getTablesByType(client, dbName, pattern, toHiveTableType(tableType))
+      shim.getTablesByType(client, dbName, pattern, hiveTableType)
     } catch {
       case _: UnsupportedOperationException =>
         // Fallback to filter logic if getTablesByType not supported.
         val tableNames = client.getTablesByPattern(dbName, pattern).asScala
-        val tables = getTablesByName(dbName, tableNames).filter(_.tableType == tableType)
-        tables.map(_.identifier.table)
+        getRawTablesByName(dbName, tableNames)
+          .filter(_.getTableType == hiveTableType)
+          .map(_.getTableName)
     }
   }
 

From 30c3a502667bfa1feaf2230b4fc4cc2d36d9b85a Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Thu, 6 Aug 2020 13:54:15 -0700
Subject: [PATCH 0947/1280] [SPARK-32506][TESTS] Flaky test:
 StreamingLinearRegressionWithTests

### What changes were proposed in this pull request?
The test creates 10 batches of data  to train the model and expects to see error on test data improves as model is trained. If the difference between the 2nd error and the 10th error is smaller than 2, the assertion fails:
```
FAIL: test_train_prediction (pyspark.mllib.tests.test_streaming_algorithms.StreamingLinearRegressionWithTests)
Test that error on test data improves as model is trained.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/runner/work/spark/spark/python/pyspark/mllib/tests/test_streaming_algorithms.py", line 466, in test_train_prediction
    eventually(condition, timeout=180.0)
  File "/home/runner/work/spark/spark/python/pyspark/testing/utils.py", line 81, in eventually
    lastValue = condition()
  File "/home/runner/work/spark/spark/python/pyspark/mllib/tests/test_streaming_algorithms.py", line 461, in condition
    self.assertGreater(errors[1] - errors[-1], 2)
AssertionError: 1.672640157855923 not greater than 2
```
I saw this quite a few time on Jenkins but was not able to reproduce this on my local. These are the ten errors I got:
```
4.517395047937127
4.894265404350079
3.0392090466559876
1.8786361640757654
0.8973106042078115
0.3715780507684368
0.20815690742907672
0.17333033743125845
0.15686783249863873
0.12584413600569616
```
I am thinking of having 15 batches of data instead of 10, so the model can be trained for a longer time. Hopefully the 15th error - 2nd error will always be larger than 2 on Jenkins. These are the 15 errors I got on my local:
```
4.517395047937127
4.894265404350079
3.0392090466559876
1.8786361640757658
0.8973106042078115
0.3715780507684368
0.20815690742907672
0.17333033743125845
0.15686783249863873
0.12584413600569616
0.11883853835108477
0.09400261862100823
0.08887491447353497
0.05984929624986607
0.07583948141520978
```

### Why are the changes needed?
Fix flaky test

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manually tested

Closes #29380 from huaxingao/flaky_test.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
(cherry picked from commit 75c2c53e931187912a92e0b52dae0f772fa970e3)
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
---
 python/pyspark/mllib/tests/test_streaming_algorithms.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 2f35e07cdf71c..5818a7c088841 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -434,9 +434,9 @@ def test_train_prediction(self):
         slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
         slr.setInitialWeights([0.0])
 
-        # Create ten batches with 100 sample points in each.
+        # Create fifteen batches with 100 sample points in each.
         batches = []
-        for i in range(10):
+        for i in range(15):
             batch = LinearDataGenerator.generateLinearInput(
                 0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
             batches.append(self.sc.parallelize(batch))

From c7af0be530d8c2a4935acf9a593034fb85d5572f Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 7 Aug 2020 11:29:18 +0900
Subject: [PATCH 0948/1280] [SPARK-32538][CORE][TEST] Use local time zone for
 the timestamp logged in unit-tests.log

### What changes were proposed in this pull request?

This PR lets the logger log timestamp based on local time zone during test.
`SparkFunSuite` fixes the default time zone to America/Los_Angeles so the timestamp logged in unit-tests.log is also based on the fixed time zone.

### Why are the changes needed?

It's confusable for developers whose time zone is not America/Los_Angeles.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Run existing tests and confirmed uint-tests.log.
If your local time zone is America/Los_Angeles, you can test by setting the environment variable `TZ` like as follows.
```
$ TZ=Asia/Tokyo build/sbt "testOnly org.apache.spark.executor.ExecutorSuite"
$ tail core/target/unit-tests.log
```

Closes #29356 from sarutak/fix-unit-test-log-timezone.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4e267f3eb9ca0df18647c859b75b61b1af800120)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 core/src/test/scala/org/apache/spark/SparkFunSuite.scala | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
index ec641f8294b29..d40207423b7ae 100644
--- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
@@ -64,6 +64,12 @@ abstract class SparkFunSuite
   with Logging {
 // scalastyle:on
 
+  // Initialize the logger forcibly to let the logger log timestamp
+  // based on the local time zone depending on environments.
+  // The default time zone will be set to America/Los_Angeles later
+  // so this initialization is necessary here.
+  log
+
   // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
   TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
   // Add Locale setting

From 4d8642f8d90870f15097eb8124af66d2b8f6f024 Mon Sep 17 00:00:00 2001
From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Date: Fri, 7 Aug 2020 14:29:32 +0900
Subject: [PATCH 0949/1280] [SPARK-32560][SQL] Improve exception message at
 InsertIntoHiveTable.processInsert

### What changes were proposed in this pull request?
improve exception message

### Why are the changes needed?
the before message lack of single quotes, we may improve it to keep consisent.
![image](https://user-images.githubusercontent.com/46367746/89595808-15bbc300-d888-11ea-9914-b05ea7b66461.png)

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
No ,it is only improving the message.

Closes #29376 from GuoPhilipse/improve-exception-message.

Lead-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Co-authored-by: GuoPhilipse <guofei_ok@126.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit aa4d3c19fead4ec2f89b4957b4ccc7482e121e4d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/hive/execution/InsertIntoHiveTable.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index a0bdbec4694bf..05acca46135c3 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -243,7 +243,7 @@ case class InsertIntoHiveTable(
             if (fs.exists(partitionPath)) {
               if (!fs.delete(partitionPath, true)) {
                 throw new RuntimeException(
-                  "Cannot remove partition directory '" + partitionPath.toString)
+                  s"Cannot remove partition directory '$partitionPath'")
               }
             }
           }
@@ -295,7 +295,7 @@ case class InsertIntoHiveTable(
               if (fs.exists(path)) {
                 if (!fs.delete(path, true)) {
                   throw new RuntimeException(
-                    "Cannot remove partition directory '" + path.toString)
+                    s"Cannot remove partition directory '$path'")
                 }
                 // Don't let Hive do overwrite operation since it is slower.
                 doHiveOverwrite = false

From 17ce60561ff1e32b84199bfc8da234f69119b368 Mon Sep 17 00:00:00 2001
From: Prashant Sharma <prashant@apache.org>
Date: Fri, 7 Aug 2020 03:31:45 -0500
Subject: [PATCH 0950/1280] [SPARK-32556][INFRA] Fix release script to have
 urlencoded passwords where required

### What changes were proposed in this pull request?
1. URL encode the `ASF_PASSWORD` of the release manager.
2. Update the image to install `qpdf` and `jq` dep
3. Increase the JVM HEAM memory for maven build.

### Why are the changes needed?
Release script takes hours to run, and if a single failure happens about somewhere midway, then either one has to get down to manually doing stuff or re run the entire script. (This is my understanding) So, I have made the fixes of a few failures, discovered so far.

1. If the release manager password contains a char, that is not allowed in URL, then it fails the build at the clone spark step.
`git clone "https://$ASF_USERNAME:$ASF_PASSWORD$ASF_SPARK_REPO" -b $GIT_BRANCH`

          ^^^ Fails with bad URL

`ASF_USERNAME` may not be URL encoded, but we need to encode `ASF_PASSWORD`.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the release for branch-2.4, using both type of passwords, i.e. passwords with special chars and without it.

Closes #29373 from ScrapCodes/release-script-fix2.

Lead-authored-by: Prashant Sharma <prashant@apache.org>
Co-authored-by: Prashant Sharma <prashsh1@in.ibm.com>
Signed-off-by: Prashant Sharma <prashant@apache.org>
(cherry picked from commit 6c3d0a44056cc5ca1d304b5a8a03d2f02974e58b)
Signed-off-by: Prashant Sharma <prashant@apache.org>
---
 dev/create-release/release-tag.sh      | 6 +++++-
 dev/create-release/release-util.sh     | 2 +-
 dev/create-release/spark-rm/Dockerfile | 2 ++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/dev/create-release/release-tag.sh b/dev/create-release/release-tag.sh
index 39856a9955955..a9a518f9e10d7 100755
--- a/dev/create-release/release-tag.sh
+++ b/dev/create-release/release-tag.sh
@@ -64,8 +64,12 @@ init_maven_sbt
 
 ASF_SPARK_REPO="gitbox.apache.org/repos/asf/spark.git"
 
+function uriencode { jq -nSRr --arg v "$1" '$v|@uri'; }
+
+declare -r ENCODED_ASF_PASSWORD=$(uriencode "$ASF_PASSWORD")
+
 rm -rf spark
-git clone "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH
+git clone "https://$ASF_USERNAME:$ENCODED_ASF_PASSWORD@$ASF_SPARK_REPO" -b $GIT_BRANCH
 cd spark
 
 git config user.name "$GIT_NAME"
diff --git a/dev/create-release/release-util.sh b/dev/create-release/release-util.sh
index 8ee94a67f34fd..55e1d09da5898 100755
--- a/dev/create-release/release-util.sh
+++ b/dev/create-release/release-util.sh
@@ -229,7 +229,7 @@ function init_maven_sbt {
   if [[ $JAVA_VERSION < "1.8." ]]; then
     # Needed for maven central when using Java 7.
     SBT_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
-    MVN_EXTRA_OPTS="-Dhttps.protocols=TLSv1.1,TLSv1.2"
+    MVN_EXTRA_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dhttps.protocols=TLSv1.1,TLSv1.2"
     MVN="$MVN $MVN_EXTRA_OPTS"
   fi
   export MVN MVN_EXTRA_OPTS SBT_OPTS
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 44d602415b262..ff6af6ff57d35 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -63,6 +63,8 @@ RUN apt-get clean && apt-get update && $APT_INSTALL gnupg ca-certificates && \
   $APT_INSTALL nodejs && \
   # Install needed python packages. Use pip for installing packages (for consistency).
   $APT_INSTALL libpython3-dev python3-pip python3-setuptools && \
+  # qpdf is required for CRAN checks to pass.
+  $APT_INSTALL qpdf jq && \
   # Change default python version to python3.
   update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1 && \
   update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2 && \

From cfe62fcef7e57404780cab83130b25052c1769c7 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sat, 8 Aug 2020 08:53:54 -0700
Subject: [PATCH 0951/1280] [SPARK-32564][SQL][TEST][3.0] Inject data
 statistics to simulate plan generation on actual TPCDS data

### What changes were proposed in this pull request?

`TPCDSQuerySuite` currently computes plans with empty TPCDS tables, then checks if plans can be generated correctly. But, the generated plans can be different from actual ones because the input tables are empty (e.g., the plans always use broadcast-hash joins, but actual ones use sort-merge joins for larger tables). To mitigate the issue, this PR defines data statistics constants extracted from generated TPCDS data in `TPCDSTableStats`, then injects the statistics via `spark.sessionState.catalog.alterTableStats` when defining TPCDS tables in `TPCDSQuerySuite`.

Please see a link below about how to extract the table statistics:
 - https://gist.github.com/maropu/f553d32c323ee803d39e2f7fa0b5a8c3

For example, the generated plans of TPCDS `q2` are different with/without this fix:
```
==== w/ this fix: q2 ====
== Physical Plan ==
* Sort (43)
+- Exchange (42)
   +- * Project (41)
      +- * SortMergeJoin Inner (40)
         :- * Sort (28)
         :  +- Exchange (27)
         :     +- * Project (26)
         :        +- * BroadcastHashJoin Inner BuildRight (25)
         :           :- * HashAggregate (19)
         :           :  +- Exchange (18)
         :           :     +- * HashAggregate (17)
         :           :        +- * Project (16)
         :           :           +- * BroadcastHashJoin Inner BuildRight (15)
         :           :              :- Union (9)
         :           :              :  :- * Project (4)
         :           :              :  :  +- * Filter (3)
         :           :              :  :     +- * ColumnarToRow (2)
         :           :              :  :        +- Scan parquet default.web_sales (1)
         :           :              :  +- * Project (8)
         :           :              :     +- * Filter (7)
         :           :              :        +- * ColumnarToRow (6)
         :           :              :           +- Scan parquet default.catalog_sales (5)
         :           :              +- BroadcastExchange (14)
         :           :                 +- * Project (13)
         :           :                    +- * Filter (12)
         :           :                       +- * ColumnarToRow (11)
         :           :                          +- Scan parquet default.date_dim (10)
         :           +- BroadcastExchange (24)
         :              +- * Project (23)
         :                 +- * Filter (22)
         :                    +- * ColumnarToRow (21)
         :                       +- Scan parquet default.date_dim (20)
         +- * Sort (39)
            +- Exchange (38)
               +- * Project (37)
                  +- * BroadcastHashJoin Inner BuildRight (36)
                     :- * HashAggregate (30)
                     :  +- ReusedExchange (29)
                     +- BroadcastExchange (35)
                        +- * Project (34)
                           +- * Filter (33)
                              +- * ColumnarToRow (32)
                                 +- Scan parquet default.date_dim (31)

==== w/o this fix: q2 ====
== Physical Plan ==
* Sort (40)
+- Exchange (39)
   +- * Project (38)
      +- * BroadcastHashJoin Inner BuildRight (37)
         :- * Project (26)
         :  +- * BroadcastHashJoin Inner BuildRight (25)
         :     :- * HashAggregate (19)
         :     :  +- Exchange (18)
         :     :     +- * HashAggregate (17)
         :     :        +- * Project (16)
         :     :           +- * BroadcastHashJoin Inner BuildRight (15)
         :     :              :- Union (9)
         :     :              :  :- * Project (4)
         :     :              :  :  +- * Filter (3)
         :     :              :  :     +- * ColumnarToRow (2)
         :     :              :  :        +- Scan parquet default.web_sales (1)
         :     :              :  +- * Project (8)
         :     :              :     +- * Filter (7)
         :     :              :        +- * ColumnarToRow (6)
         :     :              :           +- Scan parquet default.catalog_sales (5)
         :     :              +- BroadcastExchange (14)
         :     :                 +- * Project (13)
         :     :                    +- * Filter (12)
         :     :                       +- * ColumnarToRow (11)
         :     :                          +- Scan parquet default.date_dim (10)
         :     +- BroadcastExchange (24)
         :        +- * Project (23)
         :           +- * Filter (22)
         :              +- * ColumnarToRow (21)
         :                 +- Scan parquet default.date_dim (20)
         +- BroadcastExchange (36)
            +- * Project (35)
               +- * BroadcastHashJoin Inner BuildRight (34)
                  :- * HashAggregate (28)
                  :  +- ReusedExchange (27)
                  +- BroadcastExchange (33)
                     +- * Project (32)
                        +- * Filter (31)
                           +- * ColumnarToRow (30)
                              +- Scan parquet default.date_dim (29)
```

This comes from the cloud-fan comment: https://github.com/apache/spark/pull/29270#issuecomment-666098964

This is the backport of #29384.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #29390 from maropu/SPARK-32564-BRANCH3.0.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/sql/TPCDSQuerySuite.scala    |  10 +-
 .../org/apache/spark/sql/TPCDSSchema.scala    |   6 +
 .../apache/spark/sql/TPCDSTableStats.scala    | 503 ++++++++++++++++++
 3 files changed, 517 insertions(+), 2 deletions(-)
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index aacb625d7921f..89f09eec8e515 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -46,11 +46,17 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     "q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
     "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
 
+  val sqlConfgs = Seq(
+    SQLConf.CBO_ENABLED.key -> "true",
+    SQLConf.PLAN_STATS_ENABLED.key -> "true",
+    SQLConf.JOIN_REORDER_ENABLED.key -> "true"
+  )
+
   tpcdsQueries.foreach { name =>
     val queryString = resourceToString(s"tpcds/$name.sql",
       classLoader = Thread.currentThread().getContextClassLoader)
     test(name) {
-      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+      withSQLConf(sqlConfgs: _*) {
         // check the plans can be properly generated
         val plan = sql(queryString).queryExecution.executedPlan
         checkGeneratedCode(plan)
@@ -69,7 +75,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     val queryString = resourceToString(s"tpcds-v2.7.0/$name.sql",
       classLoader = Thread.currentThread().getContextClassLoader)
     test(s"$name-v2.7") {
-      withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "true") {
+      withSQLConf(sqlConfgs: _*) {
         // check the plans can be properly generated
         val plan = sql(queryString).queryExecution.executedPlan
         checkGeneratedCode(plan)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
index 43974ad22f2ef..012b7d10d8f78 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.catalyst.TableIdentifier
+
 trait TPCDSSchema {
 
   private val tableColumns = Map(
@@ -255,5 +257,9 @@ trait TPCDSSchema {
          |USING $format
          |${options.mkString("\n")}
        """.stripMargin)
+
+    // To simulate plan generation on actual TPCDS data, injects data stats here
+    spark.sessionState.catalog.alterTableStats(
+      TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala
new file mode 100644
index 0000000000000..f39b4b8b56c2e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSTableStats.scala
@@ -0,0 +1,503 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.catalog.{CatalogColumnStat, CatalogStatistics}
+
+object TPCDSTableStats {
+
+  // These data statistics are extracted from generated TPCDS data with SF=100
+
+  // scalastyle:off line.size.limit
+  val sf100TableStats = Map(
+    "customer" -> CatalogStatistics(500000000L, Some(2000000L), Map(
+      "c_birth_country" -> CatalogColumnStat(Some(196L), None, None, Some(69626), Some(9), Some(20), None, CatalogColumnStat.VERSION),
+      "c_current_cdemo_sk" -> CatalogColumnStat(Some(1184426L), Some("1"), Some("1920798"), Some(69943), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_customer_id" -> CatalogColumnStat(Some(2000000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "c_birth_day" -> CatalogColumnStat(Some(32L), Some("1"), Some("31"), Some(70166), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_last_name" -> CatalogColumnStat(Some(5250L), None, None, Some(70098), Some(7), Some(13), None, CatalogColumnStat.VERSION),
+      "c_login" -> CatalogColumnStat(Some(0L), None, None, Some(2000000), Some(20), Some(20), None, CatalogColumnStat.VERSION),
+      "c_first_shipto_date_sk" -> CatalogColumnStat(Some(3755L), Some("2449028"), Some("2452678"), Some(70080), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_first_name" -> CatalogColumnStat(Some(5146L), None, None, Some(69769), Some(6), Some(11), None, CatalogColumnStat.VERSION),
+      "c_birth_month" -> CatalogColumnStat(Some(12L), Some("1"), Some("12"), Some(69896), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_email_address" -> CatalogColumnStat(Some(1929800L), None, None, Some(70200), Some(28), Some(47), None, CatalogColumnStat.VERSION),
+      "c_current_addr_sk" -> CatalogColumnStat(Some(824389L), Some("1"), Some("1000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_salutation" -> CatalogColumnStat(Some(6L), None, None, Some(69840), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_current_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(69657), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_birth_year" -> CatalogColumnStat(Some(67L), Some("1924"), Some("1992"), Some(69986), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "c_last_review_date" -> CatalogColumnStat(Some(357L), None, None, Some(70102), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "c_preferred_cust_flag" -> CatalogColumnStat(Some(2L), None, None, Some(69778), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "c_first_sales_date_sk" -> CatalogColumnStat(Some(3758L), Some("2448998"), Some("2452648"), Some(69950), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "store_sales" -> CatalogStatistics(42623559552L, Some(287997024L), Map(
+      "ss_ext_sales_price" -> CatalogColumnStat(Some(712602L), Some("0.00"), Some("19878.00"), Some(12955462), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_coupon_amt" -> CatalogColumnStat(Some(925847L), Some("0.00"), Some("19225.00"), Some(12958053), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_store_sk" -> CatalogColumnStat(Some(199L), Some("1"), Some("400"), Some(12950651), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(12953654), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_ext_discount_amt" -> CatalogColumnStat(Some(925847L), Some("0.00"), Some("19225.00"), Some(12958053), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(12957139), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_ext_tax" -> CatalogColumnStat(Some(135620L), Some("0.00"), Some("1762.38"), Some(12957705), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_sold_time_sk" -> CatalogColumnStat(Some(47827L), Some("28800"), Some("75599"), Some(12953300), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(12954088), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_net_paid_inc_tax" -> CatalogColumnStat(Some(1404501L), Some("0.00"), Some("21344.38"), Some(12958941), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(12958327), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(12955252), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_sold_date_sk" -> CatalogColumnStat(Some(1781L), Some("2450816"), Some("2452642"), Some(12955025), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_list_price" -> CatalogColumnStat(Some(19079L), Some("1.00"), Some("200.00"), Some(12952108), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ext_list_price" -> CatalogColumnStat(Some(752016L), Some("1.00"), Some("20000.00"), Some(12955040), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_net_profit" -> CatalogColumnStat(Some(1388760L), Some("-10000.00"), Some("9889.00"), Some(12955156), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ticket_number" -> CatalogColumnStat(Some(24596280L), Some("1"), Some("24000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_net_paid" -> CatalogColumnStat(Some(1058635L), Some("0.00"), Some("19878.00"), Some(12954554), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(12960060), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ss_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(12952082), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(12956686), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ss_sales_price" -> CatalogColumnStat(Some(19261L), Some("0.00"), Some("200.00"), Some(12958587), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "web_sales" -> CatalogStatistics(14688252348L, Some(72001237L), Map(
+      "ws_sold_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(17931), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(17814), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_bill_cdemo_sk" -> CatalogColumnStat(Some(1707373L), Some("1"), Some("1920800"), Some(17833), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_ship" -> CatalogColumnStat(Some(1629659L), Some("0.00"), Some("43468.92"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_sales_price" -> CatalogColumnStat(Some(29143L), Some("0.00"), Some("300.00"), Some(18005), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(17823), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ship_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(17833), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_order_number" -> CatalogColumnStat(Some(6073146L), Some("1"), Some("6000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_sold_date_sk" -> CatalogColumnStat(Some(1781L), Some("2450816"), Some("2452642"), Some(17922), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_web_site_sk" -> CatalogColumnStat(Some(25L), Some("1"), Some("24"), Some(18030), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_bill_customer_sk" -> CatalogColumnStat(Some(1800267L), Some("1"), Some("2000000"), Some(17882), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_coupon_amt" -> CatalogColumnStat(Some(863374L), Some("0.00"), Some("27591.16"), Some(18027), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(17931), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_tax" -> CatalogColumnStat(Some(1695042L), Some("0.00"), Some("32492.90"), Some(18102), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(18014), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_bill_addr_sk" -> CatalogColumnStat(Some(940457L), Some("1"), Some("1000000"), Some(17801), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid" -> CatalogColumnStat(Some(1247989L), Some("0.00"), Some("29810.00"), Some(17968), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ext_discount_amt" -> CatalogColumnStat(Some(934889L), Some("0.00"), Some("29982.00"), Some(17890), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_tax" -> CatalogColumnStat(Some(168876L), Some("0.00"), Some("2682.90"), Some(17800), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(17850), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_customer_sk" -> CatalogColumnStat(Some(1811873L), Some("1"), Some("2000000"), Some(17886), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_paid_inc_ship_tax" -> CatalogColumnStat(Some(2186741L), Some("0.00"), Some("44479.52"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_web_page_sk" -> CatalogColumnStat(Some(2108L), Some("1"), Some("2040"), Some(17920), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_sales_price" -> CatalogColumnStat(Some(908147L), Some("0.00"), Some("29810.00"), Some(17843), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ext_ship_cost" -> CatalogColumnStat(Some(482711L), Some("0.00"), Some("14927.00"), Some(17923), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_bill_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(18011), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ship_date_sk" -> CatalogColumnStat(Some(1898L), Some("2450817"), Some("2452762"), Some(17883), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_ext_list_price" -> CatalogColumnStat(Some(1071558L), Some("1.02"), Some("29997.00"), Some(18001), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_ship_cdemo_sk" -> CatalogColumnStat(Some(1755797L), Some("1"), Some("1920800"), Some(17903), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(17812), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_list_price" -> CatalogColumnStat(Some(29447L), Some("1.00"), Some("300.00"), Some(17824), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ws_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(18116), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ws_net_profit" -> CatalogColumnStat(Some(1482074L), Some("-9997.00"), Some("19840.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "date_dim" -> CatalogStatistics(19284936L, Some(73049L), Map(
+      "d_dom" -> CatalogColumnStat(Some(32L), Some("1"), Some("31"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_quarter" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_fy_week_seq" -> CatalogColumnStat(Some(10010L), Some("1"), Some("10436"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_holiday" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_current_day" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_week_seq" -> CatalogColumnStat(Some(10010L), Some("1"), Some("10436"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_month" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_moy" -> CatalogColumnStat(Some(12L), Some("1"), Some("12"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_date" -> CatalogColumnStat(Some(73049L), None, None, Some(0), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "d_same_day_lq" -> CatalogColumnStat(Some(73049L), Some("2414930"), Some("2487978"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_weekend" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_dow" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_year" -> CatalogColumnStat(Some(202L), Some("1900"), Some("2100"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_week" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_quarter_name" -> CatalogColumnStat(Some(774L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "d_month_seq" -> CatalogColumnStat(Some(2431L), Some("0"), Some("2400"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_fy_year" -> CatalogColumnStat(Some(202L), Some("1900"), Some("2100"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_following_holiday" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_same_day_ly" -> CatalogColumnStat(Some(73049L), Some("2414657"), Some("2487705"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_day_name" -> CatalogColumnStat(Some(7L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "d_qoy" -> CatalogColumnStat(Some(4L), Some("1"), Some("4"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_date_sk" -> CatalogColumnStat(Some(73049L), Some("2415022"), Some("2488070"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_fy_quarter_seq" -> CatalogColumnStat(Some(787L), Some("1"), Some("801"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_current_year" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "d_date_id" -> CatalogColumnStat(Some(73049L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "d_quarter_seq" -> CatalogColumnStat(Some(787L), Some("1"), Some("801"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_last_dom" -> CatalogColumnStat(Some(2386L), Some("2415020"), Some("2488372"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "d_first_dom" -> CatalogColumnStat(Some(2329L), Some("2415021"), Some("2488070"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "household_demographics" -> CatalogStatistics(316800L, Some(7200L), Map(
+      "hd_buy_potential" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(8), Some(10), None, CatalogColumnStat.VERSION),
+      "hd_income_band_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_demo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_vehicle_count" -> CatalogColumnStat(Some(6L), Some("-1"), Some("4"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "hd_dep_count" -> CatalogColumnStat(Some(10L), Some("0"), Some("9"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "call_center" -> CatalogStatistics(16020L, Some(30L), Map(
+      "cc_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1998-01-01"), Some("2002-01-01"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_mkt_desc" -> CatalogColumnStat(Some(21L), None, None, Some(0), Some(59), Some(93), None, CatalogColumnStat.VERSION),
+      "cc_hours" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_street_name" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(8), Some(16), None, CatalogColumnStat.VERSION),
+      "cc_manager" -> CatalogColumnStat(Some(22L), None, None, Some(0), Some(13), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_city" -> CatalogColumnStat(Some(12L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "cc_class" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "cc_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "cc_open_date_sk" -> CatalogColumnStat(Some(15L), Some("2450794"), Some("2451146"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_market_manager" -> CatalogColumnStat(Some(25L), None, None, Some(0), Some(13), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_street_number" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "cc_call_center_sk" -> CatalogColumnStat(Some(30L), Some("1"), Some("30"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_name" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(14), Some(19), None, CatalogColumnStat.VERSION),
+      "cc_suite_number" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "cc_mkt_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_company_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_division" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_closed_date_sk" -> CatalogColumnStat(Some(0L), None, None, Some(30), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_state" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "cc_street_type" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "cc_call_center_id" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "cc_sq_ft" -> CatalogColumnStat(Some(22L), Some("1670015"), Some("31896816"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_mkt_class" -> CatalogColumnStat(Some(26L), None, None, Some(0), Some(35), Some(50), None, CatalogColumnStat.VERSION),
+      "cc_tax_percentage" -> CatalogColumnStat(Some(10L), Some("0.00"), Some("0.12"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cc_division_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "cc_rec_end_date" -> CatalogColumnStat(Some(3L), Some("2000-01-01"), Some("2001-12-31"), Some(15), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_employees" -> CatalogColumnStat(Some(22L), Some("2935"), Some("69020"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cc_county" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(15), Some(17), None, CatalogColumnStat.VERSION),
+      "cc_company" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_page" -> CatalogStatistics(3774000L, Some(20400L), Map(
+      "cp_catalog_page_id" -> CatalogColumnStat(Some(20400L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "cp_department" -> CatalogColumnStat(Some(1L), None, None, Some(205), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "cp_description" -> CatalogColumnStat(Some(19304L), None, None, Some(190), Some(75), Some(99), None, CatalogColumnStat.VERSION),
+      "cp_start_date_sk" -> CatalogColumnStat(Some(88L), Some("2450815"), Some("2453005"), Some(196), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_end_date_sk" -> CatalogColumnStat(Some(101L), Some("2450844"), Some("2453186"), Some(206), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_catalog_page_number" -> CatalogColumnStat(Some(186L), Some("1"), Some("188"), Some(208), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_type" -> CatalogColumnStat(Some(3L), None, None, Some(197), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "cp_catalog_page_sk" -> CatalogColumnStat(Some(18915L), Some("1"), Some("20400"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cp_catalog_number" -> CatalogColumnStat(Some(110L), Some("1"), Some("109"), Some(197), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_returns" -> CatalogStatistics(2189464848L, Some(14404374L), Map(
+      "cr_return_amount" -> CatalogColumnStat(Some(563788L), Some("0.00"), Some("28778.31"), Some(288408), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(288369), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_refunded_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(288053), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_catalog_page_sk" -> CatalogColumnStat(Some(11224L), Some("1"), Some("17108"), Some(288041), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_reason_sk" -> CatalogColumnStat(Some(55L), Some("1"), Some("55"), Some(287890), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returning_cdemo_sk" -> CatalogColumnStat(Some(1839372L), Some("1"), Some("1920800"), Some(288128), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(288038), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_cash" -> CatalogColumnStat(Some(592855L), Some("0.00"), Some("24544.84"), Some(287638), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_cdemo_sk" -> CatalogColumnStat(Some(1842661L), Some("1"), Some("1920800"), Some(287556), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_order_number" -> CatalogColumnStat(Some(8816832L), Some("2"), Some("16000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(287768), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_refunded_customer_sk" -> CatalogColumnStat(Some(1894309L), Some("1"), Some("2000000"), Some(287207), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_amt_inc_tax" -> CatalogColumnStat(Some(855338L), Some("0.00"), Some("29353.87"), Some(288246), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(287581), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returned_date_sk" -> CatalogColumnStat(Some(2027L), Some("2450821"), Some("2452921"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_ship_cost" -> CatalogColumnStat(Some(347391L), Some("0.00"), Some("14130.96"), Some(287952), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_returning_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(288203), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(287844), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_store_credit" -> CatalogColumnStat(Some(415989L), Some("0.00"), Some("22167.49"), Some(288118), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_return_tax" -> CatalogColumnStat(Some(101557L), Some("0.00"), Some("2390.75"), Some(288599), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_refunded_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(287752), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_call_center_sk" -> CatalogColumnStat(Some(31L), Some("1"), Some("30"), Some(288179), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_returned_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cr_net_loss" -> CatalogColumnStat(Some(550908L), Some("0.50"), Some("15781.83"), Some(287954), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_reversed_charge" -> CatalogColumnStat(Some(410432L), Some("0.00"), Some("23801.24"), Some(288476), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cr_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(288581), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "catalog_sales" -> CatalogStatistics(29375401260L, Some(143997065L), Map(
+      "cs_ship_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(720450), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_bill_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(718886), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_ship_tax" -> CatalogColumnStat(Some(2570239L), Some("0.00"), Some("45460.80"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ext_sales_price" -> CatalogColumnStat(Some(977146L), Some("0.00"), Some("29808.00"), Some(719228), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_sold_date_sk" -> CatalogColumnStat(Some(1772L), Some("2450815"), Some("2452654"), Some(719581), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_tax" -> CatalogColumnStat(Some(189612L), Some("0.00"), Some("2619.36"), Some(719627), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_sold_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(720917), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_ship" -> CatalogColumnStat(Some(1870941L), Some("0.00"), Some("43725.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(719624), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_list_price" -> CatalogColumnStat(Some(1101138L), Some("1.00"), Some("29997.00"), Some(719642), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_quantity" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(720147), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_discount_amt" -> CatalogColumnStat(Some(965247L), Some("0.00"), Some("29765.00"), Some(719820), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_net_profit" -> CatalogColumnStat(Some(1640663L), Some("-10000.00"), Some("19840.00"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_bill_hdemo_sk" -> CatalogColumnStat(Some(7083L), Some("1"), Some("7200"), Some(719849), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_ship_cost" -> CatalogColumnStat(Some(508914L), Some("0.00"), Some("14896.00"), Some(719848), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ship_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(720292), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_wholesale_cost" -> CatalogColumnStat(Some(9503L), Some("1.00"), Some("100.00"), Some(721114), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_date_sk" -> CatalogColumnStat(Some(1887L), Some("2450817"), Some("2452744"), Some(719625), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_ext_wholesale_cost" -> CatalogColumnStat(Some(382964L), Some("1.00"), Some("10000.00"), Some(719924), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_bill_cdemo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(720208), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_sales_price" -> CatalogColumnStat(Some(29282L), Some("0.00"), Some("300.00"), Some(719781), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_net_paid" -> CatalogColumnStat(Some(1330351L), Some("0.00"), Some("29760.00"), Some(719706), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_promo_sk" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(720194), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_call_center_sk" -> CatalogColumnStat(Some(31L), Some("1"), Some("30"), Some(719767), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_catalog_page_sk" -> CatalogColumnStat(Some(11224L), Some("1"), Some("17108"), Some(719180), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_bill_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(719473), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_list_price" -> CatalogColumnStat(Some(29447L), Some("1.00"), Some("300.00"), Some(720328), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_customer_sk" -> CatalogColumnStat(Some(1903054L), Some("1"), Some("2000000"), Some(720582), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_coupon_amt" -> CatalogColumnStat(Some(982009L), Some("0.00"), Some("28422.94"), Some(719631), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_addr_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(718680), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_order_number" -> CatalogColumnStat(Some(15603123L), Some("1"), Some("16000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cs_net_paid_inc_tax" -> CatalogColumnStat(Some(1807594L), Some("0.00"), Some("31745.52"), Some(719354), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "cs_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(720146), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "customer_address" -> CatalogStatistics(245000000L, Some(1000000L), Map(
+      "ca_country" -> CatalogColumnStat(Some(1L), None, None, Some(30097), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "ca_address_id" -> CatalogColumnStat(Some(1000000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "ca_address_sk" -> CatalogColumnStat(Some(943039L), Some("1"), Some("1000000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ca_county" -> CatalogColumnStat(Some(1957L), None, None, Some(30028), Some(14), Some(28), None, CatalogColumnStat.VERSION),
+      "ca_gmt_offset" -> CatalogColumnStat(Some(6L), Some("-10.00"), Some("-5.00"), Some(30131), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "ca_street_type" -> CatalogColumnStat(Some(20L), None, None, Some(30124), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "ca_street_name" -> CatalogColumnStat(Some(8292L), None, None, Some(30178), Some(9), Some(21), None, CatalogColumnStat.VERSION),
+      "ca_city" -> CatalogColumnStat(Some(977L), None, None, Some(30183), Some(9), Some(20), None, CatalogColumnStat.VERSION),
+      "ca_location_type" -> CatalogColumnStat(Some(3L), None, None, Some(30172), Some(9), Some(13), None, CatalogColumnStat.VERSION),
+      "ca_suite_number" -> CatalogColumnStat(Some(76L), None, None, Some(30047), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "ca_zip" -> CatalogColumnStat(Some(8311L), None, None, Some(30370), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "ca_state" -> CatalogColumnStat(Some(54L), None, None, Some(30124), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "ca_street_number" -> CatalogColumnStat(Some(1034L), None, None, Some(30226), Some(3), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "customer_demographics" -> CatalogStatistics(182476000L, Some(1920800L), Map(
+      "cd_dep_employed_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_dep_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_education_status" -> CatalogColumnStat(Some(7L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "cd_marital_status" -> CatalogColumnStat(Some(5L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "cd_gender" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "cd_purchase_estimate" -> CatalogColumnStat(Some(20L), Some("500"), Some("10000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_demo_sk" -> CatalogColumnStat(Some(1847065L), Some("1"), Some("1920800"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "cd_credit_rating" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "cd_dep_college_count" -> CatalogColumnStat(Some(7L), Some("0"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "income_band" -> CatalogStatistics(400L, Some(20L), Map(
+      "ib_income_band_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ib_lower_bound" -> CatalogColumnStat(Some(19L), Some("0"), Some("190001"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "ib_upper_bound" -> CatalogColumnStat(Some(20L), Some("10000"), Some("200000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "inventory" -> CatalogStatistics(9583920000L, Some(399330000L), Map(
+      "inv_date_sk" -> CatalogColumnStat(Some(267L), Some("2450815"), Some("2452635"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_item_sk" -> CatalogColumnStat(Some(206807L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "inv_quantity_on_hand" -> CatalogColumnStat(Some(1026L), Some("0"), Some("1000"), Some(19969395), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "item" -> CatalogStatistics(94248000L, Some(204000L), Map(
+      "i_container" -> CatalogColumnStat(Some(1L), None, None, Some(510), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "i_formulation" -> CatalogColumnStat(Some(158873L), None, None, Some(530), Some(20), Some(20), None, CatalogColumnStat.VERSION),
+      "i_brand_id" -> CatalogColumnStat(Some(862L), Some("1001001"), Some("10016017"), Some(516), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_manager_id" -> CatalogColumnStat(Some(99L), Some("1"), Some("100"), Some(506), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_rec_end_date" -> CatalogColumnStat(Some(3L), None, None, Some(102000), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "i_class" -> CatalogColumnStat(Some(100L), None, None, Some(499), Some(8), Some(15), None, CatalogColumnStat.VERSION),
+      "i_wholesale_cost" -> CatalogColumnStat(Some(6297L), Some("0.02"), Some("88.91"), Some(489), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "i_item_id" -> CatalogColumnStat(Some(104042L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "i_manufact_id" -> CatalogColumnStat(Some(1026L), Some("1"), Some("1000"), Some(498), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_item_desc" -> CatalogColumnStat(Some(158754L), None, None, Some(508), Some(101), Some(200), None, CatalogColumnStat.VERSION),
+      "i_rec_start_date" -> CatalogColumnStat(Some(4L), None, None, Some(522), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "i_color" -> CatalogColumnStat(Some(88L), None, None, Some(524), Some(6), Some(10), None, CatalogColumnStat.VERSION),
+      "i_product_name" -> CatalogColumnStat(Some(203486L), None, None, Some(514), Some(23), Some(30), None, CatalogColumnStat.VERSION),
+      "i_category" -> CatalogColumnStat(Some(10L), None, None, Some(482), Some(6), Some(11), None, CatalogColumnStat.VERSION),
+      "i_class_id" -> CatalogColumnStat(Some(16L), Some("1"), Some("16"), Some(491), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_item_sk" -> CatalogColumnStat(Some(204000L), Some("1"), Some("204000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_manufact" -> CatalogColumnStat(Some(997L), None, None, Some(544), Some(12), Some(15), None, CatalogColumnStat.VERSION),
+      "i_brand" -> CatalogColumnStat(Some(671L), None, None, Some(510), Some(17), Some(22), None, CatalogColumnStat.VERSION),
+      "i_current_price" -> CatalogColumnStat(Some(8736L), Some("0.09"), Some("99.99"), Some(518), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "i_category_id" -> CatalogColumnStat(Some(10L), Some("1"), Some("10"), Some(515), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "i_size" -> CatalogColumnStat(Some(7L), None, None, Some(515), Some(5), Some(11), None, CatalogColumnStat.VERSION),
+      "i_units" -> CatalogColumnStat(Some(21L), None, None, Some(503), Some(5), Some(7), None, CatalogColumnStat.VERSION)
+    )),
+    "promotion" -> CatalogStatistics(268000L, Some(1000L), Map(
+      "p_promo_name" -> CatalogColumnStat(Some(10L), None, None, Some(18), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "p_end_date_sk" -> CatalogColumnStat(Some(564L), Some("2450116"), Some("2450967"), Some(12), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_radio" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_demo" -> CatalogColumnStat(Some(1L), None, None, Some(16), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_item_sk" -> CatalogColumnStat(Some(986L), Some("280"), Some("203966"), Some(14), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_dmail" -> CatalogColumnStat(Some(2L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_discount_active" -> CatalogColumnStat(Some(1L), None, None, Some(19), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_promo_id" -> CatalogColumnStat(Some(1000L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "p_channel_tv" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_cost" -> CatalogColumnStat(Some(1L), Some("1000.00"), Some("1000.00"), Some(14), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "p_channel_email" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_event" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_catalog" -> CatalogColumnStat(Some(1L), None, None, Some(14), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_promo_sk" -> CatalogColumnStat(Some(1000L), Some("1"), Some("1000"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_purpose" -> CatalogColumnStat(Some(1L), None, None, Some(13), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "p_start_date_sk" -> CatalogColumnStat(Some(577L), Some("2450100"), Some("2450915"), Some(17), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "p_channel_press" -> CatalogColumnStat(Some(1L), None, None, Some(15), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "p_channel_details" -> CatalogColumnStat(Some(990L), None, None, Some(10), Some(40), Some(60), None, CatalogColumnStat.VERSION),
+      "p_response_target" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(19), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "reason" -> CatalogStatistics(3630L, Some(55L), Map(
+      "r_reason_sk" -> CatalogColumnStat(Some(55L), Some("1"), Some("55"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "r_reason_id" -> CatalogColumnStat(Some(55L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "r_reason_desc" -> CatalogColumnStat(Some(55L), None, None, Some(0), Some(14), Some(43), None, CatalogColumnStat.VERSION)
+    )),
+    "ship_mode" -> CatalogStatistics(2420L, Some(20L), Map(
+      "sm_carrier" -> CatalogColumnStat(Some(19L), None, None, Some(0), Some(7), Some(14), None, CatalogColumnStat.VERSION),
+      "sm_ship_mode_sk" -> CatalogColumnStat(Some(20L), Some("1"), Some("20"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "sm_code" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(5), Some(7), None, CatalogColumnStat.VERSION),
+      "sm_type" -> CatalogColumnStat(Some(5L), None, None, Some(0), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "sm_contract" -> CatalogColumnStat(Some(18L), None, None, Some(0), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "sm_ship_mode_id" -> CatalogColumnStat(Some(20L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION)
+    )),
+    "store" -> CatalogStatistics(207432L, Some(402L), Map(
+      "s_rec_end_date" -> CatalogColumnStat(Some(3L), None, None, Some(201), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "s_state" -> CatalogColumnStat(Some(9L), None, None, Some(2), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "s_street_type" -> CatalogColumnStat(Some(20L), None, None, Some(6), Some(5), Some(9), None, CatalogColumnStat.VERSION),
+      "s_floor_space" -> CatalogColumnStat(Some(300L), Some("5004767"), Some("9997773"), Some(3), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_division_name" -> CatalogColumnStat(Some(1L), None, None, Some(5), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_market_desc" -> CatalogColumnStat(Some(318L), None, None, Some(5), Some(59), Some(100), None, CatalogColumnStat.VERSION),
+      "s_street_name" -> CatalogColumnStat(Some(255L), None, None, Some(6), Some(9), Some(16), None, CatalogColumnStat.VERSION),
+      "s_county" -> CatalogColumnStat(Some(9L), None, None, Some(4), Some(15), Some(17), None, CatalogColumnStat.VERSION),
+      "s_rec_start_date" -> CatalogColumnStat(Some(4L), None, None, Some(4), Some(10), Some(10), None, CatalogColumnStat.VERSION),
+      "s_company_id" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(4), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_market_manager" -> CatalogColumnStat(Some(275L), None, None, Some(7), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "s_number_employees" -> CatalogColumnStat(Some(94L), Some("200"), Some("300"), Some(5), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_manager" -> CatalogColumnStat(Some(300L), None, None, Some(5), Some(13), Some(20), None, CatalogColumnStat.VERSION),
+      "s_country" -> CatalogColumnStat(Some(1L), None, None, Some(4), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "s_store_name" -> CatalogColumnStat(Some(10L), None, None, Some(2), Some(4), Some(5), None, CatalogColumnStat.VERSION),
+      "s_division_id" -> CatalogColumnStat(Some(1L), Some("1"), Some("1"), Some(2), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_street_number" -> CatalogColumnStat(Some(274L), None, None, Some(5), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "s_company_name" -> CatalogColumnStat(Some(1L), None, None, Some(3), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(4), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "s_store_sk" -> CatalogColumnStat(Some(399L), Some("1"), Some("402"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_city" -> CatalogColumnStat(Some(18L), None, None, Some(5), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "s_zip" -> CatalogColumnStat(Some(99L), None, None, Some(6), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "s_market_id" -> CatalogColumnStat(Some(10L), Some("1"), Some("10"), Some(6), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_hours" -> CatalogColumnStat(Some(3L), None, None, Some(4), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "s_suite_number" -> CatalogColumnStat(Some(75L), None, None, Some(2), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "s_closed_date_sk" -> CatalogColumnStat(Some(70L), Some("2450823"), Some("2451313"), Some(296), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "s_store_id" -> CatalogColumnStat(Some(210L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "s_geography_class" -> CatalogColumnStat(Some(1L), None, None, Some(3), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "s_tax_precentage" -> CatalogColumnStat(Some(12L), Some("0.00"), Some("0.11"), Some(5), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "store_returns" -> CatalogStatistics(4837573440L, Some(28795080L), Map(
+      "sr_item_sk" -> CatalogColumnStat(Some(197284L), Some("1"), Some("204000"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_reversed_charge" -> CatalogColumnStat(Some(423824L), Some("0.00"), Some("16099.52"), Some(1009035), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_amt_inc_tax" -> CatalogColumnStat(Some(807709L), Some("0.00"), Some("20002.89"), Some(1006919), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_store_sk" -> CatalogColumnStat(Some(199L), Some("1"), Some("400"), Some(1007164), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_quantity" -> CatalogColumnStat(Some(103L), Some("1"), Some("100"), Some(1007948), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(1008291), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_refunded_cash" -> CatalogColumnStat(Some(559685L), Some("0.00"), Some("17556.95"), Some(1008003), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_time_sk" -> CatalogColumnStat(Some(31932L), Some("28799"), Some("61199"), Some(1009330), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(1008253), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_amt" -> CatalogColumnStat(Some(492928L), Some("0.00"), Some("18973.20"), Some(1007419), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_returned_date_sk" -> CatalogColumnStat(Some(2010L), Some("2450820"), Some("2452822"), Some(1007464), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_tax" -> CatalogColumnStat(Some(86618L), Some("0.00"), Some("1611.71"), Some(1008618), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_return_ship_cost" -> CatalogColumnStat(Some(290195L), Some("0.00"), Some("9578.25"), Some(1007846), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_reason_sk" -> CatalogColumnStat(Some(57L), Some("1"), Some("55"), Some(1008299), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_net_loss" -> CatalogColumnStat(Some(487073L), Some("0.50"), Some("10447.72"), Some(1007153), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_cdemo_sk" -> CatalogColumnStat(Some(1807132L), Some("1"), Some("1920800"), Some(1006835), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(1008547), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_store_credit" -> CatalogColumnStat(Some(410133L), Some("0.00"), Some("15642.11"), Some(1007102), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_customer_sk" -> CatalogColumnStat(Some(1860981L), Some("1"), Some("2000000"), Some(1008429), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "sr_ticket_number" -> CatalogColumnStat(Some(15853105L), Some("1"), Some("23999996"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "time_dim" -> CatalogStatistics(10886400L, Some(86400L), Map(
+      "t_sub_shift" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "t_time" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_second" -> CatalogColumnStat(Some(60L), Some("0"), Some("59"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_hour" -> CatalogColumnStat(Some(25L), Some("0"), Some("23"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_am_pm" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "t_minute" -> CatalogColumnStat(Some(60L), Some("0"), Some("59"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_shift" -> CatalogColumnStat(Some(3L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "t_time_id" -> CatalogColumnStat(Some(80197L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "t_time_sk" -> CatalogColumnStat(Some(83505L), Some("0"), Some("86399"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "t_meal_time" -> CatalogColumnStat(Some(3L), None, None, Some(50400), Some(7), Some(9), None, CatalogColumnStat.VERSION)
+    )),
+    "warehouse" -> CatalogStatistics(3855L, Some(15L), Map(
+      "w_state" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "w_street_name" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(10), Some(14), None, CatalogColumnStat.VERSION),
+      "w_warehouse_name" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(17), Some(20), None, CatalogColumnStat.VERSION),
+      "w_county" -> CatalogColumnStat(Some(8L), None, None, Some(0), Some(14), Some(16), None, CatalogColumnStat.VERSION),
+      "w_street_number" -> CatalogColumnStat(Some(14L), None, None, Some(1), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "w_warehouse_sq_ft" -> CatalogColumnStat(Some(14L), Some("73065"), Some("977787"), Some(1), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "w_city" -> CatalogColumnStat(Some(11L), None, None, Some(0), Some(8), Some(13), None, CatalogColumnStat.VERSION),
+      "w_warehouse_sk" -> CatalogColumnStat(Some(15L), Some("1"), Some("15"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "w_suite_number" -> CatalogColumnStat(Some(13L), None, None, Some(1), Some(8), Some(9), None, CatalogColumnStat.VERSION),
+      "w_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "w_gmt_offset" -> CatalogColumnStat(Some(2L), Some("-6.00"), Some("-5.00"), Some(1), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "w_street_type" -> CatalogColumnStat(Some(10L), None, None, Some(1), Some(5), Some(7), None, CatalogColumnStat.VERSION),
+      "w_warehouse_id" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "w_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION)
+    )),
+    "web_page" -> CatalogStatistics(281520L, Some(2040L), Map(
+      "wp_type" -> CatalogColumnStat(Some(7L), None, None, Some(19), Some(7), Some(9), None, CatalogColumnStat.VERSION),
+      "wp_autogen_flag" -> CatalogColumnStat(Some(2L), None, None, Some(25), Some(1), Some(1), None, CatalogColumnStat.VERSION),
+      "wp_url" -> CatalogColumnStat(Some(1L), None, None, Some(25), Some(18), Some(18), None, CatalogColumnStat.VERSION),
+      "wp_image_count" -> CatalogColumnStat(Some(7L), Some("1"), Some("7"), Some(20), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_max_ad_count" -> CatalogColumnStat(Some(5L), Some("0"), Some("4"), Some(21), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_customer_sk" -> CatalogColumnStat(Some(486L), Some("711"), Some("1996257"), Some(1471), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_rec_end_date" -> CatalogColumnStat(Some(3L), Some("1999-09-03"), Some("2001-09-02"), Some(1020), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_creation_date_sk" -> CatalogColumnStat(Some(121L), Some("2450672"), Some("2450815"), Some(20), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_link_count" -> CatalogColumnStat(Some(25L), Some("2"), Some("25"), Some(16), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_web_page_id" -> CatalogColumnStat(Some(1004L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "wp_web_page_sk" -> CatalogColumnStat(Some(2040L), Some("1"), Some("2040"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1997-09-03"), Some("2001-09-03"), Some(21), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_access_date_sk" -> CatalogColumnStat(Some(100L), Some("2452548"), Some("2452648"), Some(19), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "wp_char_count" -> CatalogColumnStat(Some(1493L), Some("303"), Some("8523"), Some(25), Some(4), Some(4), None, CatalogColumnStat.VERSION)
+    )),
+    "web_returns" -> CatalogStatistics(1439534000L, Some(7197670L), Map(
+      "wr_returning_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(323850), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_reversed_charge" -> CatalogColumnStat(Some(346909L), Some("0.00"), Some("22972.36"), Some(323810), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returned_date_sk" -> CatalogColumnStat(Some(2189L), Some("2450820"), Some("2453002"), Some(324185), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_order_number" -> CatalogColumnStat(Some(4098425L), Some("1"), Some("5999999"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_fee" -> CatalogColumnStat(Some(9571L), Some("0.50"), Some("100.00"), Some(324065), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_account_credit" -> CatalogColumnStat(Some(334119L), Some("0.00"), Some("23028.27"), Some(324422), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_customer_sk" -> CatalogColumnStat(Some(1808850L), Some("1"), Some("2000000"), Some(324191), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_customer_sk" -> CatalogColumnStat(Some(1813094L), Some("1"), Some("2000000"), Some(324024), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_tax" -> CatalogColumnStat(Some(88469L), Some("0.00"), Some("2551.16"), Some(323621), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_web_page_sk" -> CatalogColumnStat(Some(1994L), Some("1"), Some("2040"), Some(324900), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_item_sk" -> CatalogColumnStat(Some(197284L), Some("1"), Some("204000"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_reason_sk" -> CatalogColumnStat(Some(57L), Some("1"), Some("55"), Some(323666), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_amt_inc_tax" -> CatalogColumnStat(Some(683544L), Some("0.00"), Some("29493.38"), Some(323171), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_net_loss" -> CatalogColumnStat(Some(494707L), Some("0.50"), Some("15068.96"), Some(324438), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_cdemo_sk" -> CatalogColumnStat(Some(1755065L), Some("1"), Some("1920800"), Some(323863), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returned_time_sk" -> CatalogColumnStat(Some(84232L), Some("0"), Some("86399"), Some(323677), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_amt" -> CatalogColumnStat(Some(453015L), Some("0.00"), Some("28346.31"), Some(323473), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_cdemo_sk" -> CatalogColumnStat(Some(1755065L), Some("1"), Some("1920800"), Some(323899), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_returning_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(323999), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_cash" -> CatalogColumnStat(Some(484316L), Some("0.00"), Some("26466.56"), Some(324693), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_hdemo_sk" -> CatalogColumnStat(Some(6609L), Some("1"), Some("7200"), Some(324230), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_quantity" -> CatalogColumnStat(Some(103L), Some("1"), Some("100"), Some(323764), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_return_ship_cost" -> CatalogColumnStat(Some(302038L), Some("0.00"), Some("13602.60"), Some(323341), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "wr_refunded_addr_sk" -> CatalogColumnStat(Some(925738L), Some("1"), Some("1000000"), Some(324482), Some(8), Some(8), None, CatalogColumnStat.VERSION)
+    )),
+    "web_site" -> CatalogStatistics(11760L, Some(24L), Map(
+      "web_rec_end_date" -> CatalogColumnStat(Some(3L), Some("1999-08-16"), Some("2001-08-15"), Some(12), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_market_manager" -> CatalogColumnStat(Some(21L), None, None, Some(0), Some(13), Some(15), None, CatalogColumnStat.VERSION),
+      "web_country" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(13), Some(13), None, CatalogColumnStat.VERSION),
+      "web_street_name" -> CatalogColumnStat(Some(24L), None, None, Some(0), Some(10), Some(14), None, CatalogColumnStat.VERSION),
+      "web_city" -> CatalogColumnStat(Some(11L), None, None, Some(0), Some(10), Some(15), None, CatalogColumnStat.VERSION),
+      "web_mkt_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_close_date_sk" -> CatalogColumnStat(Some(8L), Some("2443328"), Some("2447131"), Some(4), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_street_number" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(3), Some(3), None, CatalogColumnStat.VERSION),
+      "web_gmt_offset" -> CatalogColumnStat(Some(2L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "web_rec_start_date" -> CatalogColumnStat(Some(4L), Some("1997-08-16"), Some("2001-08-16"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_mkt_class" -> CatalogColumnStat(Some(17L), None, None, Some(0), Some(32), Some(47), None, CatalogColumnStat.VERSION),
+      "web_county" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(14), Some(17), None, CatalogColumnStat.VERSION),
+      "web_class" -> CatalogColumnStat(Some(1L), None, None, Some(0), Some(7), Some(7), None, CatalogColumnStat.VERSION),
+      "web_site_sk" -> CatalogColumnStat(Some(24L), Some("1"), Some("24"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_manager" -> CatalogColumnStat(Some(19L), None, None, Some(0), Some(13), Some(16), None, CatalogColumnStat.VERSION),
+      "web_suite_number" -> CatalogColumnStat(Some(20L), None, None, Some(0), Some(9), Some(9), None, CatalogColumnStat.VERSION),
+      "web_site_id" -> CatalogColumnStat(Some(12L), None, None, Some(0), Some(16), Some(16), None, CatalogColumnStat.VERSION),
+      "web_company_name" -> CatalogColumnStat(Some(6L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION),
+      "web_state" -> CatalogColumnStat(Some(9L), None, None, Some(0), Some(2), Some(2), None, CatalogColumnStat.VERSION),
+      "web_mkt_desc" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(66), Some(92), None, CatalogColumnStat.VERSION),
+      "web_name" -> CatalogColumnStat(Some(4L), None, None, Some(0), Some(6), Some(6), None, CatalogColumnStat.VERSION),
+      "web_company_id" -> CatalogColumnStat(Some(6L), Some("1"), Some("6"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_street_type" -> CatalogColumnStat(Some(15L), None, None, Some(0), Some(4), Some(9), None, CatalogColumnStat.VERSION),
+      "web_open_date_sk" -> CatalogColumnStat(Some(12L), Some("2450628"), Some("2450807"), Some(0), Some(4), Some(4), None, CatalogColumnStat.VERSION),
+      "web_tax_percentage" -> CatalogColumnStat(Some(8L), Some("0.00"), Some("0.12"), Some(0), Some(8), Some(8), None, CatalogColumnStat.VERSION),
+      "web_zip" -> CatalogColumnStat(Some(14L), None, None, Some(0), Some(5), Some(5), None, CatalogColumnStat.VERSION)
+    ))
+  )
+  // scalastyle:on line.size.limit
+}

From 7caecae59bc30abbe0bf6128ca9d5622035b0b44 Mon Sep 17 00:00:00 2001
From: JoeyValentine <rlaalsdn0506@naver.com>
Date: Sat, 8 Aug 2020 12:36:07 -0700
Subject: [PATCH 0952/1280] [MINOR][DOCS] Fix typos at
 ExecutorAllocationManager.scala

### What changes were proposed in this pull request?

This PR fixes some typos in <code>core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala</code> file.

### Why are the changes needed?

<code>spark.dynamicAllocation.sustainedSchedulerBacklogTimeout</code> (N) is used only after the <code>spark.dynamicAllocation.schedulerBacklogTimeout</code> (M) is exceeded.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

No test needed.

Closes #29351 from JoeyValentine/master.

Authored-by: JoeyValentine <rlaalsdn0506@naver.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit dc3fac81848f557e3dac3f35686af325a18d0291)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/ExecutorAllocationManager.scala    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 74103dc3e25b8..a6d2849f4fbbe 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -45,8 +45,8 @@ import org.apache.spark.util.{Clock, SystemClock, ThreadUtils, Utils}
  * executors that could run all current running and pending tasks at once.
  *
  * Increasing the target number of executors happens in response to backlogged tasks waiting to be
- * scheduled. If the scheduler queue is not drained in N seconds, then new executors are added. If
- * the queue persists for another M seconds, then more executors are added and so on. The number
+ * scheduled. If the scheduler queue is not drained in M seconds, then new executors are added. If
+ * the queue persists for another N seconds, then more executors are added and so on. The number
  * added in each round increases exponentially from the previous round until an upper bound has been
  * reached. The upper bound is based both on a configured property and on the current number of
  * running and pending tasks, as described above.

From 1f1fc8ba0873974c0013e82cdf420b733f772564 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sat, 8 Aug 2020 16:33:25 -0700
Subject: [PATCH 0953/1280] [SPARK-32564][SQL][TEST][FOLLOWUP] Re-enable
 TPCDSQuerySuite with empty tables

### What changes were proposed in this pull request?

This is the follow-up PR of #29384 to address the cloud-fan comment: https://github.com/apache/spark/pull/29384#issuecomment-670595111
This PR re-enables `TPCDSQuerySuite` with empty tables for better test coverages.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Existing tests.

Closes #29391 from maropu/SPARK-32564-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 1df855bef2b2dbe330cafb0d10e0b4af813a311a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/sql/TPCDSQuerySuite.scala    | 27 +++++++++++++++----
 .../org/apache/spark/sql/TPCDSSchema.scala    |  6 -----
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index 89f09eec8e515..e72c31f4cf6e4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.util.resourceToString
 import org.apache.spark.sql.internal.SQLConf
 
@@ -46,11 +47,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     "q81", "q82", "q83", "q84", "q85", "q86", "q87", "q88", "q89", "q90",
     "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
 
-  val sqlConfgs = Seq(
-    SQLConf.CBO_ENABLED.key -> "true",
-    SQLConf.PLAN_STATS_ENABLED.key -> "true",
-    SQLConf.JOIN_REORDER_ENABLED.key -> "true"
-  )
+  val sqlConfgs: Seq[(String, String)] = Nil
 
   tpcdsQueries.foreach { name =>
     val queryString = resourceToString(s"tpcds/$name.sql",
@@ -104,3 +101,23 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
     }
   }
 }
+
+class TPCDSQueryWithStatsSuite extends TPCDSQuerySuite {
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    for (tableName <- tableNames) {
+      // To simulate plan generation on actual TPCDS data, injects data stats here
+      spark.sessionState.catalog.alterTableStats(
+        TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
+    }
+  }
+
+  // Sets configurations for enabling the optimization rules that
+  // exploit data statistics.
+  override val sqlConfgs = Seq(
+    SQLConf.CBO_ENABLED.key -> "true",
+    SQLConf.PLAN_STATS_ENABLED.key -> "true",
+    SQLConf.JOIN_REORDER_ENABLED.key -> "true"
+  )
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
index 012b7d10d8f78..43974ad22f2ef 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql
 
-import org.apache.spark.sql.catalyst.TableIdentifier
-
 trait TPCDSSchema {
 
   private val tableColumns = Map(
@@ -257,9 +255,5 @@ trait TPCDSSchema {
          |USING $format
          |${options.mkString("\n")}
        """.stripMargin)
-
-    // To simulate plan generation on actual TPCDS data, injects data stats here
-    spark.sessionState.catalog.alterTableStats(
-      TableIdentifier(tableName), Some(TPCDSTableStats.sf100TableStats(tableName)))
   }
 }

From 9391705958a8aa41360063e1a70968daf1a1a975 Mon Sep 17 00:00:00 2001
From: "wangguangxin.cn" <wangguangxin.cn@gmail.com>
Date: Sun, 9 Aug 2020 12:12:14 -0700
Subject: [PATCH 0954/1280] [SPARK-32559][SQL][3.0] Fix the trim logic in
 UTF8String.toInt/toLong did't handle non-ASCII characters correctly

### What changes were proposed in this pull request?

This is a backport of https://github.com/apache/spark/pull/29375
The trim logic in Cast expression introduced in https://github.com/apache/spark/pull/26622 trim non-ASCII characters unexpectly.

Before this patch
![image](https://user-images.githubusercontent.com/1312321/89513154-caad9b80-d806-11ea-9ebe-17c9e7d1b5b3.png)

After this patch
![image](https://user-images.githubusercontent.com/1312321/89513196-d731f400-d806-11ea-959c-6a7dc29dcd49.png)

### Why are the changes needed?
The behavior described above doesn't make sense, and also doesn't consistent with the behavior when cast a string to double/float, as well as doesn't consistent with the behavior of Hive

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
Added more UT

Closes #29393 from WangGuangxin/cast-bugfix-branch-3.0.

Authored-by: wangguangxin.cn <wangguangxin.cn@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/unsafe/types/UTF8String.java | 12 +++---
 .../test/resources/sql-tests/inputs/cast.sql  |  5 +++
 .../resources/sql-tests/inputs/datetime.sql   |  2 +
 .../resources/sql-tests/inputs/interval.sql   |  1 +
 .../sql-tests/results/ansi/datetime.sql.out   | 30 ++++++++++++-
 .../sql-tests/results/ansi/interval.sql.out   | 16 ++++++-
 .../resources/sql-tests/results/cast.sql.out  | 42 ++++++++++++++++++-
 .../sql-tests/results/datetime-legacy.sql.out | 30 ++++++++++++-
 .../sql-tests/results/datetime.sql.out        | 30 ++++++++++++-
 .../sql-tests/results/interval.sql.out        | 16 ++++++-
 10 files changed, 172 insertions(+), 12 deletions(-)

diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 7205293aa48c5..43bd7976c5d33 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -575,14 +575,14 @@ public UTF8String trim() {
   public UTF8String trimAll() {
     int s = 0;
     // skip all of the whitespaces (<=0x20) in the left side
-    while (s < this.numBytes && getByte(s) <= ' ') s++;
+    while (s < this.numBytes && Character.isWhitespace(getByte(s))) s++;
     if (s == this.numBytes) {
       // Everything trimmed
       return EMPTY_UTF8;
     }
     // skip all of the whitespaces (<=0x20) in the right side
     int e = this.numBytes - 1;
-    while (e > s && getByte(e) <= ' ') e--;
+    while (e > s && Character.isWhitespace(getByte(e))) e--;
     if (s == 0 && e == numBytes - 1) {
       // Nothing trimmed
       return this;
@@ -1119,11 +1119,11 @@ public boolean toLong(LongWrapper toLongResult) {
 
   private boolean toLong(LongWrapper toLongResult, boolean allowDecimal) {
     int offset = 0;
-    while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
+    while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
     if (offset == this.numBytes) return false;
 
     int end = this.numBytes - 1;
-    while (end > offset && getByte(end) <= ' ') end--;
+    while (end > offset && Character.isWhitespace(getByte(end))) end--;
 
     byte b = getByte(offset);
     final boolean negative = b == '-';
@@ -1216,11 +1216,11 @@ public boolean toInt(IntWrapper intWrapper) {
 
   private boolean toInt(IntWrapper intWrapper, boolean allowDecimal) {
     int offset = 0;
-    while (offset < this.numBytes && getByte(offset) <= ' ') offset++;
+    while (offset < this.numBytes && Character.isWhitespace(getByte(offset))) offset++;
     if (offset == this.numBytes) return false;
 
     int end = this.numBytes - 1;
-    while (end > offset && getByte(end) <= ' ') end--;
+    while (end > offset && Character.isWhitespace(getByte(end))) end--;
 
     byte b = getByte(offset);
     final boolean negative = b == '-';
diff --git a/sql/core/src/test/resources/sql-tests/inputs/cast.sql b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
index 972ebdd01f61e..81c741a5ca8e9 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/cast.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/cast.sql
@@ -70,6 +70,11 @@ select cast(' 1' as bigint);
 select cast(' 1' as float);
 select cast(' 1 ' as DOUBLE);
 select cast('1.0 ' as DEC);
+select cast('1中文' as tinyint);
+select cast('1中文' as smallint);
+select cast('1中文' as INT);
+select cast('中文1' as bigint);
+select cast('1中文' as bigint);
 
 -- trim string before cast to boolean
 select cast('\t\t true \n\r ' as boolean);
diff --git a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
index db54b5508ec71..ae5831c61913c 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/datetime.sql
@@ -33,6 +33,8 @@ select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01');
 
 select date '2019-01-01\t';
 select timestamp '2019-01-01\t';
+select date '2020-01-01中文';
+select timestamp '2019-01-01中文';
 
 -- time add/sub
 select timestamp'2011-11-11 11:11:11' + interval '2' day;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index a7e1afef66604..ebc39f5bc44a6 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -160,6 +160,7 @@ select interval '2-2\t' year to month;
 select interval '-\t2-2\t' year to month;
 select interval '\n0 12:34:46.789\t' day to second;
 select interval '\n-\t10\t 12:34:46.789\t' day to second;
+select interval '中文 interval 1 day';
 
 -- interval overflow if (ansi) exception else NULL
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b);
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index 36674040f1302..ec007aa8e721d 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 106
+-- Number of queries: 108
 
 
 -- !query
@@ -141,6 +141,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
 2019-01-01 00:00:00
 
 
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
 -- !query
 select timestamp'2011-11-11 11:11:11' + interval '2' day
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index e83444d95dd1c..6a6a1df7eeb0e 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 92
+-- Number of queries: 93
 
 
 -- !query
@@ -875,6 +875,20 @@ select interval '\n-\t10\t 12:34:46.789\t' day to second
 ----------------^^^
 
 
+-- !query
+select interval '中文 interval 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)
+
+== SQL ==
+select interval '中文 interval 1 day'
+-------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/cast.sql.out b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
index 35b4c0e79720b..d4872ca03199b 100644
--- a/sql/core/src/test/resources/sql-tests/results/cast.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cast.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 46
+-- Number of queries: 51
 
 
 -- !query
@@ -353,6 +353,46 @@ struct<CAST(1.0  AS DECIMAL(10,0)):decimal(10,0)>
 1
 
 
+-- !query
+select cast('1中文' as tinyint)
+-- !query schema
+struct<CAST(1中文 AS TINYINT):tinyint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as smallint)
+-- !query schema
+struct<CAST(1中文 AS SMALLINT):smallint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as INT)
+-- !query schema
+struct<CAST(1中文 AS INT):int>
+-- !query output
+NULL
+
+
+-- !query
+select cast('中文1' as bigint)
+-- !query schema
+struct<CAST(中文1 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
+-- !query
+select cast('1中文' as bigint)
+-- !query schema
+struct<CAST(1中文 AS BIGINT):bigint>
+-- !query output
+NULL
+
+
 -- !query
 select cast('\t\t true \n\r ' as boolean)
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
index 1342c71feecfa..fbbdb5f3feb48 100644
--- a/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 106
+-- Number of queries: 108
 
 
 -- !query
@@ -115,6 +115,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
 2019-01-01 00:00:00
 
 
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
 -- !query
 select timestamp'2011-11-11 11:11:11' + interval '2' day
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
index d3657e874343b..d7e960a9a3db3 100755
--- a/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/datetime.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 106
+-- Number of queries: 108
 
 
 -- !query
@@ -115,6 +115,34 @@ struct<TIMESTAMP '2019-01-01 00:00:00':timestamp>
 2019-01-01 00:00:00
 
 
+-- !query
+select date '2020-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the DATE value: 2020-01-01中文(line 1, pos 7)
+
+== SQL ==
+select date '2020-01-01中文'
+-------^^^
+
+
+-- !query
+select timestamp '2019-01-01中文'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the TIMESTAMP value: 2019-01-01中文(line 1, pos 7)
+
+== SQL ==
+select timestamp '2019-01-01中文'
+-------^^^
+
+
 -- !query
 select timestamp'2011-11-11 11:11:11' + interval '2' day
 -- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 4cdc66950f4a7..63ed9b567c058 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 92
+-- Number of queries: 93
 
 
 -- !query
@@ -852,6 +852,20 @@ select interval '\n-\t10\t 12:34:46.789\t' day to second
 ----------------^^^
 
 
+-- !query
+select interval '中文 interval 1 day'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 中文 interval 1 day(line 1, pos 7)
+
+== SQL ==
+select interval '中文 interval 1 day'
+-------^^^
+
+
 -- !query
 select -(a) from values (interval '-2147483648 months', interval '2147483647 months') t(a, b)
 -- !query schema

From 0f4989cb6f52f2266768a075b98b4dea7f25e89c Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sun, 9 Aug 2020 18:36:35 -0700
Subject: [PATCH 0955/1280] [SPARK-32393][SQL][TEST] Add tests for all the
 character types in PostgresIntegrationSuite

### What changes were proposed in this pull request?

This PR intends to add tests to check if all the character types in PostgreSQL supported.

The document for character types in PostgreSQL: https://www.postgresql.org/docs/current/datatype-character.html

Closes #29192.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Add tests.

Closes #29394 from maropu/pr29192.

Lead-authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Co-authored-by: kujon <jakub.korzeniowski@vortexa.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b2c45f7dcfe62e76f74726c97385440fead70646)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/jdbc/PostgresIntegrationSuite.scala   | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 599f00def0750..19144918561ff 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -77,8 +77,13 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
       "'172.16.0.42']::inet[], ARRAY['192.168.0.0/24', '10.1.0.0/16']::cidr[], " +
       """ARRAY['{"a": "foo", "b": "bar"}', '{"a": 1, "b": 2}']::json[], """ +
       """ARRAY['{"a": 1, "b": 2, "c": 3}']::jsonb[])"""
-    )
-      .executeUpdate()
+    ).executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE char_types (" +
+      "c0 char(4), c1 character(4), c2 character varying(4), c3 varchar(4), c4 bpchar)"
+    ).executeUpdate()
+    conn.prepareStatement("INSERT INTO char_types VALUES " +
+      "('abcd', 'efgh', 'ijkl', 'mnop', 'q')").executeUpdate()
   }
 
   test("Type mapping for various types") {
@@ -219,4 +224,16 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(rows(0).getShort(0) === 1)
     assert(rows(0).getShort(1) === 2)
   }
+
+  test("character type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "char_types", new Properties)
+    val row = df.collect()
+    assert(row.length == 1)
+    assert(row(0).length === 5)
+    assert(row(0).getString(0) === "abcd")
+    assert(row(0).getString(1) === "efgh")
+    assert(row(0).getString(2) === "ijkl")
+    assert(row(0).getString(3) === "mnop")
+    assert(row(0).getString(4) === "q")
+  }
 }

From e4c6ebf3620f4cec857b02e206e4e0e4a941ce79 Mon Sep 17 00:00:00 2001
From: kujon <jakub.korzeniowski@vortexa.com>
Date: Sun, 9 Aug 2020 19:03:37 -0700
Subject: [PATCH 0956/1280] [SPARK-32576][SQL] Support PostgreSQL `bpchar` type
 and array of char type

### What changes were proposed in this pull request?
This PR fixes the support for char(n)[], character(n)[] data types. Prior to this change, a user would get `Unsupported type ARRAY` exception when attempting to interact with the table with such types.

The description is a bit more detailed in the [JIRA](https://issues.apache.org/jira/browse/SPARK-32393) itself, but the crux of the issue is that postgres driver names char and character types as `bpchar`. The relevant driver code can be found [here](https://github.com/pgjdbc/pgjdbc/blob/master/pgjdbc/src/main/java/org/postgresql/jdbc/TypeInfoCache.java#L85-L87). `char` is very likely to be still needed, as it seems that pg makes a distinction between `char(1)` and `char(n > 1)` as per [this code](https://github.com/pgjdbc/pgjdbc/blob/b7fd9f3cef734b4c219e2f6bc6c19acf68b2990b/pgjdbc/src/main/java/org/postgresql/core/Oid.java#L64).

### Why are the changes needed?
For completeness of the pg dialect support.

### Does this PR introduce _any_ user-facing change?
Yes, successful reads of tables with bpchar array instead of errors after this fix.

### How was this patch tested?
Unit tests

Closes #29192 from kujon/fix_postgres_bpchar_array_support.

Authored-by: kujon <jakub.korzeniowski@vortexa.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 0ae94ad32fc19abb9845528b10f79915a03224f2)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala  | 2 +-
 .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala    | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
index c8d8a3392128e..a1ce25a0464c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
@@ -57,7 +57,7 @@ private object PostgresDialect extends JdbcDialect {
     case "int8" | "oid" => Some(LongType)
     case "float4" => Some(FloatType)
     case "money" | "float8" => Some(DoubleType)
-    case "text" | "varchar" | "char" | "cidr" | "inet" | "json" | "jsonb" | "uuid" =>
+    case "text" | "varchar" | "char" | "bpchar" | "cidr" | "inet" | "json" | "jsonb" | "uuid" =>
       Some(StringType)
     case "bytea" => Some(BinaryType)
     case "timestamp" | "timestamptz" | "time" | "timetz" => Some(TimestampType)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index b554754895391..a06879377c031 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -851,6 +851,8 @@ class JDBCSuite extends QueryTest
     assert(Postgres.getCatalystType(java.sql.Types.OTHER, "jsonb", 1, null) === Some(StringType))
     assert(Postgres.getCatalystType(java.sql.Types.ARRAY, "_numeric", 0, md) ==
       Some(ArrayType(DecimalType.SYSTEM_DEFAULT)))
+    assert(Postgres.getCatalystType(java.sql.Types.ARRAY, "_bpchar", 64, md) ==
+      Some(ArrayType(StringType)))
     assert(Postgres.getJDBCType(FloatType).map(_.databaseTypeDefinition).get == "FLOAT4")
     assert(Postgres.getJDBCType(DoubleType).map(_.databaseTypeDefinition).get == "FLOAT8")
     assert(Postgres.getJDBCType(ByteType).map(_.databaseTypeDefinition).get == "SMALLINT")

From 8ff615d429bdc6d2132902c83babb82c7f1deacd Mon Sep 17 00:00:00 2001
From: Yuanjian Li <yuanjian.li@databricks.com>
Date: Mon, 10 Aug 2020 14:01:31 +0900
Subject: [PATCH 0957/1280] [SPARK-32456][SS] Check the Distinct by assuming it
 as Aggregate for Structured Streaming

### What changes were proposed in this pull request?
Check the Distinct nodes by assuming it as Aggregate in `UnsupportOperationChecker` for streaming.

### Why are the changes needed?
We want to fix 2 things here:

1. Give better error message for Distinct related operations in append mode that doesn't have a watermark

We use the union streams as the example, distinct in SQL has the same issue. Since the union clause in SQL has the requirement of deduplication, the parser will generate `Distinct(Union)` and the optimizer rule `ReplaceDistinctWithAggregate` will change it to `Aggregate(Union)`. This logic is of both batch and streaming queries. However, in the streaming, the aggregation will be wrapped by state store operations so we need extra checking logic in `UnsupportOperationChecker`.

Before this change, the SS union queries in Append mode will get the following confusing error when the watermark is lacking.
```
java.util.NoSuchElementException: None.get
	at scala.None$.get(Option.scala:529)
	at scala.None$.get(Option.scala:527)
	at org.apache.spark.sql.execution.streaming.StateStoreSaveExec.$anonfun$doExecute$9(statefulOperators.scala:346)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at org.apache.spark.util.Utils$.timeTakenMs(Utils.scala:561)
	at org.apache.spark.sql.execution.streaming.StateStoreWriter.timeTakenMs(statefulOperators.scala:112)
...
```

2. Make `Distinct` in complete mode runnable.

Before this fix, the distinct in complete mode will throw the exception:
```
Complete output mode not supported when there are no streaming aggregations on streaming DataFrames/Datasets;
```

### Does this PR introduce _any_ user-facing change?
Yes, return a better error message.

### How was this patch tested?
New UT added.

Closes #29256 from xuanyuanking/SPARK-32456.

Authored-by: Yuanjian Li <yuanjian.li@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b03761e3303e979999d4faa5cf4d1719a82e06cb)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../UnsupportedOperationChecker.scala         |  8 ++-
 .../sql/streaming/StreamingQuerySuite.scala   | 51 ++++++++++++++++++-
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
index 423f89fefa093..676e28f7d4a80 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala
@@ -91,7 +91,13 @@ object UnsupportedOperationChecker extends Logging {
 
     /** Collect all the streaming aggregates in a sub plan */
     def collectStreamingAggregates(subplan: LogicalPlan): Seq[Aggregate] = {
-      subplan.collect { case a: Aggregate if a.isStreaming => a }
+      subplan.collect {
+        case a: Aggregate if a.isStreaming => a
+        // Since the Distinct node will be replaced to Aggregate in the optimizer rule
+        // [[ReplaceDistinctWithAggregate]], here we also need to check all Distinct node by
+        // assuming it as Aggregate.
+        case d @ Distinct(c: LogicalPlan) if d.isStreaming => Aggregate(c.output, c.output, c)
+      }
     }
 
     val mapGroupsWithStates = plan.collect {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
index 77f5c856ff0f4..434b6a20ace63 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
@@ -33,8 +33,9 @@ import org.scalatestplus.mockito.MockitoSugar
 
 import org.apache.spark.{SparkException, TestUtils}
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{Column, DataFrame, Dataset, Row}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Dataset, Row}
 import org.apache.spark.sql.catalyst.expressions.{Literal, Rand, Randn, Shuffle, Uuid}
+import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Complete
 import org.apache.spark.sql.connector.read.InputPartition
 import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2}
 import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
@@ -1106,6 +1107,54 @@ class StreamingQuerySuite extends StreamTest with BeforeAndAfter with Logging wi
     }
   }
 
+  test("SPARK-32456: SQL union in streaming query of append mode without watermark") {
+    val inputData1 = MemoryStream[Int]
+    val inputData2 = MemoryStream[Int]
+    withTempView("s1", "s2") {
+      inputData1.toDF().createOrReplaceTempView("s1")
+      inputData2.toDF().createOrReplaceTempView("s2")
+      val unioned = spark.sql(
+        "select s1.value from s1 union select s2.value from s2")
+      checkExceptionMessage(unioned)
+    }
+  }
+
+  test("SPARK-32456: distinct in streaming query of append mode without watermark") {
+    val inputData = MemoryStream[Int]
+    withTempView("deduptest") {
+      inputData.toDF().toDF("value").createOrReplaceTempView("deduptest")
+      val distinct = spark.sql("select distinct value from deduptest")
+      checkExceptionMessage(distinct)
+    }
+  }
+
+  test("SPARK-32456: distinct in streaming query of complete mode") {
+    val inputData = MemoryStream[Int]
+    withTempView("deduptest") {
+      inputData.toDF().toDF("value").createOrReplaceTempView("deduptest")
+      val distinct = spark.sql("select distinct value from deduptest")
+
+      testStream(distinct, Complete)(
+        AddData(inputData, 1, 2, 3, 3, 4),
+        CheckAnswer(Row(1), Row(2), Row(3), Row(4))
+      )
+    }
+  }
+
+  private def checkExceptionMessage(df: DataFrame): Unit = {
+    withTempDir { outputDir =>
+      withTempDir { checkpointDir =>
+        val exception = intercept[AnalysisException](
+          df.writeStream
+            .option("checkpointLocation", checkpointDir.getCanonicalPath)
+            .start(outputDir.getCanonicalPath))
+        assert(exception.getMessage.contains(
+          "Append output mode not supported when there are streaming aggregations on streaming " +
+            "DataFrames/DataSets without watermark"))
+      }
+    }
+  }
+
   /** Create a streaming DF that only execute one batch in which it returns the given static DF */
   private def createSingleTriggerStreamingDF(triggerDF: DataFrame): DataFrame = {
     require(!triggerDF.isStreaming)

From eaae91b589b12273c429da1b49578802434157de Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Mon, 10 Aug 2020 18:43:41 +0900
Subject: [PATCH 0958/1280] [MINOR] add test_createDataFrame_empty_partition in
 pyspark arrow tests

### What changes were proposed in this pull request?
add test_createDataFrame_empty_partition in pyspark arrow tests

### Why are the changes needed?
test edge cases.

### Does this PR introduce _any_ user-facing change?
no.

### How was this patch tested?
N/A

Closes #29398 from WeichenXu123/add_one_pyspark_arrow_test.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit fc62d720769e3267132f31ee847f2783923b3195)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/tests/test_arrow.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 7a41d24d6eecb..42f064a05ed91 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -428,6 +428,12 @@ def test_no_partition_toPandas(self):
         self.assertEqual(len(pdf), 0)
         self.assertEqual(list(pdf.columns), ["col1"])
 
+    def test_createDataFrame_empty_partition(self):
+        pdf = pd.DataFrame({"c1": [1], "c2": ["string"]})
+        df = self.spark.createDataFrame(pdf)
+        self.assertEqual([Row(c1=1, c2='string')], df.collect())
+        self.assertGreater(self.spark.sparkContext.defaultParallelism, len(pdf))
+
 
 @unittest.skipIf(
     not have_pandas or not have_pyarrow,

From 843ff0367e45034bfc1e174a939f336bcc8d2391 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Mon, 10 Aug 2020 19:05:50 +0900
Subject: [PATCH 0959/1280] [SPARK-32576][SQL][TEST][FOLLOWUP] Add tests for
 all the character array types in PostgresIntegrationSuite

### What changes were proposed in this pull request?

This is a follow-up PR of #29192 that adds integration tests for character arrays in `PostgresIntegrationSuite`.

### Why are the changes needed?

For better test coverage.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Add tests.

Closes #29397 from maropu/SPARK-32576-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 7990ea14090c13e1fd1e42bc519b54144bd3aa76)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/jdbc/PostgresIntegrationSuite.scala   | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
index 19144918561ff..2b676be68dfa4 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/PostgresIntegrationSuite.scala
@@ -84,6 +84,13 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     ).executeUpdate()
     conn.prepareStatement("INSERT INTO char_types VALUES " +
       "('abcd', 'efgh', 'ijkl', 'mnop', 'q')").executeUpdate()
+
+    conn.prepareStatement("CREATE TABLE char_array_types (" +
+      "c0 char(4)[], c1 character(4)[], c2 character varying(4)[], c3 varchar(4)[], c4 bpchar[])"
+    ).executeUpdate()
+    conn.prepareStatement("INSERT INTO char_array_types VALUES " +
+      """('{"a", "bcd"}', '{"ef", "gh"}', '{"i", "j", "kl"}', '{"mnop"}', '{"q", "r"}')"""
+    ).executeUpdate()
   }
 
   test("Type mapping for various types") {
@@ -236,4 +243,16 @@ class PostgresIntegrationSuite extends DockerJDBCIntegrationSuite {
     assert(row(0).getString(3) === "mnop")
     assert(row(0).getString(4) === "q")
   }
+
+  test("SPARK-32576: character array type tests") {
+    val df = sqlContext.read.jdbc(jdbcUrl, "char_array_types", new Properties)
+    val row = df.collect()
+    assert(row.length == 1)
+    assert(row(0).length === 5)
+    assert(row(0).getSeq[String](0) === Seq("a   ", "bcd "))
+    assert(row(0).getSeq[String](1) === Seq("ef  ", "gh  "))
+    assert(row(0).getSeq[String](2) === Seq("i", "j", "kl"))
+    assert(row(0).getSeq[String](3) === Seq("mnop"))
+    assert(row(0).getSeq[String](4) === Seq("q", "r"))
+  }
 }

From 6749ad836eb76f807c7962891b169d264971247a Mon Sep 17 00:00:00 2001
From: Luca Canali <luca.canali@cern.ch>
Date: Mon, 10 Aug 2020 09:32:01 -0700
Subject: [PATCH 0960/1280] [SPARK-32409][DOC] Document dependency between
 spark.metrics.staticSources.enabled and JVMSource registration

### What changes were proposed in this pull request?
Document the dependency between the config `spark.metrics.staticSources.enabled` and JVMSource registration.

### Why are the changes needed?

This PT just documents the dependency between config `spark.metrics.staticSources.enabled` and JVM source registration.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?
Manually tested.

Closes #29203 from LucaCanali/bugJVMMetricsRegistration.

Authored-by: Luca Canali <luca.canali@cern.ch>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 99f50c6286f4d86c02a15a0efd3046888ac45c75)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/monitoring.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index 4da0f8e9d71e9..1808167cf1bd8 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -1042,7 +1042,7 @@ This is the component with the largest amount of instrumented metrics
   - memory.remainingOnHeapMem_MB
 
 - namespace=HiveExternalCatalog
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - fileCacheHits.count
   - filesDiscovered.count
@@ -1051,7 +1051,7 @@ This is the component with the largest amount of instrumented metrics
   - partitionsFetched.count
 
 - namespace=CodeGenerator
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - compilationTime (histogram)
   - generatedClassSize (histogram)
@@ -1214,7 +1214,7 @@ when running in local mode.
   - shuffle-server.usedHeapMemory
 
 - namespace=HiveExternalCatalog
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - fileCacheHits.count
   - filesDiscovered.count
@@ -1223,7 +1223,7 @@ when running in local mode.
   - partitionsFetched.count
 
 - namespace=CodeGenerator
-  - **note:**: these metrics are conditional to a configuration parameter:
+  - **note:** these metrics are conditional to a configuration parameter:
     `spark.metrics.staticSources.enabled` (default is true) 
   - compilationTime (histogram)
   - generatedClassSize (histogram)
@@ -1240,6 +1240,8 @@ when running in local mode.
 Notes: 
   - Activate this source by setting the relevant `metrics.properties` file entry or the 
   configuration parameter:`spark.metrics.conf.*.source.jvm.class=org.apache.spark.metrics.source.JvmSource`  
+  - These metrics are conditional to a configuration parameter:
+    `spark.metrics.staticSources.enabled` (default is true)
   - This source is available for driver and executor instances and is also available for other instances.  
   - This source provides information on JVM metrics using the 
   [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/3.1.0/manual/jvm/)

From 93eb5674348c00b89f0cf43123ec18ff6b90a27d Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 10 Aug 2020 16:24:08 -0700
Subject: [PATCH 0961/1280] [SPARK-32528][SQL][TEST][3.0] The analyze method
 should make sure the plan is analyzed

### What changes were proposed in this pull request?

backport https://github.com/apache/spark/pull/29349 to 3.0.
This PR updates the `analyze` method to make sure the plan can be resolved. It also fixes some miswritten optimizer tests.

### Why are the changes needed?

It's error-prone if the `analyze` method can return an unresolved plan.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

test only

Closes #29400 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/catalyst/dsl/package.scala      |  7 ++-
 .../sql/catalyst/analysis/AnalysisSuite.scala |  8 +++
 .../BinaryComparisonSimplificationSuite.scala |  4 +-
 .../optimizer/FilterPushdownSuite.scala       | 57 +++++++------------
 .../catalyst/optimizer/JoinReorderSuite.scala |  4 +-
 .../PullupCorrelatedPredicatesSuite.scala     |  3 +-
 ...ReplaceNullWithFalseInPredicateSuite.scala |  5 +-
 .../optimizer/SimplifyCastsSuite.scala        |  9 +--
 .../optimizer/SimplifyConditionalSuite.scala  | 23 ++++----
 .../optimizer/StarJoinReorderSuite.scala      | 19 ++++---
 10 files changed, 68 insertions(+), 71 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
index 26f5bee72092c..141bcfaa739e0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
@@ -418,8 +418,11 @@ package object dsl {
       def distribute(exprs: Expression*)(n: Int): LogicalPlan =
         RepartitionByExpression(exprs, logicalPlan, numPartitions = n)
 
-      def analyze: LogicalPlan =
-        EliminateSubqueryAliases(analysis.SimpleAnalyzer.execute(logicalPlan))
+      def analyze: LogicalPlan = {
+        val analyzed = analysis.SimpleAnalyzer.execute(logicalPlan)
+        analysis.SimpleAnalyzer.checkAnalysis(analyzed)
+        EliminateSubqueryAliases(analyzed)
+      }
 
       def hint(name: String, parameters: Any*): LogicalPlan =
         UnresolvedHint(name, parameters, logicalPlan)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 453a4e6aa51cf..6ec341db42d3a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -25,6 +25,7 @@ import org.apache.log4j.Level
 import org.scalatest.Matchers
 
 import org.apache.spark.api.python.PythonEvalType
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.{AliasIdentifier, TableIdentifier}
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, InMemoryCatalog, SessionCatalog}
 import org.apache.spark.sql.catalyst.dsl.expressions._
@@ -46,6 +47,13 @@ import org.apache.spark.sql.types._
 class AnalysisSuite extends AnalysisTest with Matchers {
   import org.apache.spark.sql.catalyst.analysis.TestRelations._
 
+  test("fail for unresolved plan") {
+    intercept[AnalysisException] {
+      // `testRelation` does not have column `b`.
+      testRelation.select('b).analyze
+    }
+  }
+
   test("union project *") {
     val plan = (1 to 120)
       .map(_ => testRelation)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
index 9c71cc8e0d291..c02691848c8f0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BinaryComparisonSimplificationSuite.scala
@@ -119,9 +119,7 @@ class BinaryComparisonSimplificationSuite extends PlanTest with PredicateHelper
 
     // GetStructField with different names are semantically equal; thus, `EqualTo(fieldA1, fieldA2)`
     // will be optimized to `TrueLiteral` by `SimplifyBinaryComparison`.
-    val originalQuery = nonNullableRelation
-        .where(EqualTo(fieldA1, fieldA2))
-        .analyze
+    val originalQuery = nonNullableRelation.where(EqualTo(fieldA1, fieldA2))
 
     val optimized = Optimize.execute(originalQuery)
     val correctAnswer = nonNullableRelation.analyze
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
index 70e29dca46e9e..50de3a436bc33 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{BooleanType, IntegerType}
+import org.apache.spark.sql.types.IntegerType
 import org.apache.spark.unsafe.types.CalendarInterval
 
 class FilterPushdownSuite extends PlanTest {
@@ -664,14 +664,14 @@ class FilterPushdownSuite extends PlanTest {
     val generator = Explode('c_arr)
     val originalQuery = {
       testRelationWithArrayType
-        .generate(generator, alias = Some("arr"))
+        .generate(generator, alias = Some("arr"), outputNames = Seq("c"))
         .where(('b >= 5) && ('c > 6))
     }
     val optimized = Optimize.execute(originalQuery.analyze)
     val referenceResult = {
       testRelationWithArrayType
         .where('b >= 5)
-        .generate(generator, alias = Some("arr"))
+        .generate(generator, alias = Some("arr"), outputNames = Seq("c"))
         .where('c > 6).analyze
     }
 
@@ -1135,49 +1135,32 @@ class FilterPushdownSuite extends PlanTest {
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer)
   }
 
-  test("join condition pushdown: deterministic and non-deterministic") {
-    val x = testRelation.subquery('x)
-    val y = testRelation.subquery('y)
-
-    // Verify that all conditions except the watermark touching condition are pushed down
-    // by the optimizer and others are not.
-    val originalQuery = x.join(y, condition = Some("x.a".attr === 5 && "y.a".attr === 5 &&
-      "x.a".attr === Rand(10) && "y.b".attr === 5))
-    val correctAnswer =
-      x.where("x.a".attr === 5).join(y.where("y.a".attr === 5 && "y.b".attr === 5),
-        condition = Some("x.a".attr === Rand(10)))
-
-    // CheckAnalysis will ensure nondeterministic expressions not appear in join condition.
-    // TODO support nondeterministic expressions in join condition.
-    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
-      checkAnalysis = false)
-  }
-
   test("watermark pushdown: no pushdown on watermark attribute #1") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, 'b.timestamp, attrC)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('b, interval, testRelation)
-      .where('a === 5 && 'b === 10 && 'c === 5)
+    val originalQuery = EventTimeWatermark('b, interval, relation)
+      .where('a === 5 && 'b === new java.sql.Timestamp(0) && 'c === 5)
     val correctAnswer = EventTimeWatermark(
-      'b, interval, testRelation.where('a === 5 && 'c === 5))
-      .where('b === 10)
+      'b, interval, relation.where('a === 5 && 'c === 5))
+      .where('b === new java.sql.Timestamp(0))
 
-    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
-      checkAnalysis = false)
+    comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze)
   }
 
   test("watermark pushdown: no pushdown for nondeterministic filter") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, attrB, 'c.timestamp)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('c, interval, testRelation)
-      .where('a === 5 && 'b === Rand(10) && 'c === 5)
+    val originalQuery = EventTimeWatermark('c, interval, relation)
+      .where('a === 5 && 'b === Rand(10) && 'c === new java.sql.Timestamp(0))
     val correctAnswer = EventTimeWatermark(
-      'c, interval, testRelation.where('a === 5))
-      .where('b === Rand(10) && 'c === 5)
+      'c, interval, relation.where('a === 5))
+      .where('b === Rand(10) && 'c === new java.sql.Timestamp(0))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
@@ -1185,13 +1168,14 @@ class FilterPushdownSuite extends PlanTest {
 
   test("watermark pushdown: full pushdown") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation(attrA, attrB, 'c.timestamp)
 
     // Verify that all conditions except the watermark touching condition are pushed down
     // by the optimizer and others are not.
-    val originalQuery = EventTimeWatermark('c, interval, testRelation)
+    val originalQuery = EventTimeWatermark('c, interval, relation)
       .where('a === 5 && 'b === 10)
     val correctAnswer = EventTimeWatermark(
-      'c, interval, testRelation.where('a === 5 && 'b === 10))
+      'c, interval, relation.where('a === 5 && 'b === 10))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
@@ -1199,11 +1183,12 @@ class FilterPushdownSuite extends PlanTest {
 
   test("watermark pushdown: no pushdown on watermark attribute #2") {
     val interval = new CalendarInterval(2, 2, 2000L)
+    val relation = LocalRelation('a.timestamp, attrB, attrC)
 
-    val originalQuery = EventTimeWatermark('a, interval, testRelation)
-      .where('a === 5 && 'b === 10)
+    val originalQuery = EventTimeWatermark('a, interval, relation)
+      .where('a === new java.sql.Timestamp(0) && 'b === 10)
     val correctAnswer = EventTimeWatermark(
-      'a, interval, testRelation.where('b === 10)).where('a === 5)
+      'a, interval, relation.where('b === 10)).where('a === new java.sql.Timestamp(0))
 
     comparePlans(Optimize.execute(originalQuery.analyze), correctAnswer.analyze,
       checkAnalysis = false)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
index f7755000b3f39..36db2e2dd0ae2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
@@ -307,9 +307,9 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     val originalPlan2 =
       t1.join(t2, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t2.k-1-5")))
         .hint("broadcast")
-        .join(t4, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
+        .join(t4, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
         .hint("broadcast")
-        .join(t3, Inner, Some(nameToAttr("t1.k-1-2") === nameToAttr("t4.k-1-2")))
+        .join(t3, Inner, Some(nameToAttr("t4.v-1-10") === nameToAttr("t3.v-1-100")))
 
     assertEqualPlans(originalPlan2, originalPlan2)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
index 2d86d5a97e769..191edc81034a5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesSuite.scala
@@ -91,9 +91,8 @@ class PullupCorrelatedPredicatesSuite extends PlanTest {
         .select(max('d))
     val scalarSubquery =
       testRelation
-        .where(ScalarSubquery(subPlan))
+        .where(ScalarSubquery(subPlan) === 1)
         .select('a).analyze
-    assert(scalarSubquery.resolved)
 
     val optimized = Optimize.execute(scalarSubquery)
     val doubleOptimized = Optimize.execute(optimized)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
index c7f42f08406e7..eb52c5b74772c 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ReplaceNullWithFalseInPredicateSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.optimizer
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
@@ -50,10 +51,10 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
   }
 
   test("Not expected type - replaceNullWithFalse") {
-    val e = intercept[IllegalArgumentException] {
+    val e = intercept[AnalysisException] {
       testFilter(originalCond = Literal(null, IntegerType), expectedCond = FalseLiteral)
     }.getMessage
-    assert(e.contains("but got the type `int` in `CAST(NULL AS INT)"))
+    assert(e.contains("'CAST(NULL AS INT)' of type int is not a boolean"))
   }
 
   test("replace null in branches of If") {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
index 7b3f5b084b015..0ccf8aea660b2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyCastsSuite.scala
@@ -42,7 +42,8 @@ class SimplifyCastsSuite extends PlanTest {
 
   test("nullable element to non-nullable element array cast") {
     val input = LocalRelation('a.array(ArrayType(IntegerType, true)))
-    val plan = input.select('a.cast(ArrayType(IntegerType, false)).as("casted")).analyze
+    val attr = input.output.head
+    val plan = input.select(attr.cast(ArrayType(IntegerType, false)).as("casted"))
     val optimized = Optimize.execute(plan)
     // Though cast from `ArrayType(IntegerType, true)` to `ArrayType(IntegerType, false)` is not
     // allowed, here we just ensure that `SimplifyCasts` rule respect the plan.
@@ -60,8 +61,9 @@ class SimplifyCastsSuite extends PlanTest {
 
   test("nullable value map to non-nullable value map cast") {
     val input = LocalRelation('m.map(MapType(StringType, StringType, true)))
-    val plan = input.select('m.cast(MapType(StringType, StringType, false))
-      .as("casted")).analyze
+    val attr = input.output.head
+    val plan = input.select(attr.cast(MapType(StringType, StringType, false))
+      .as("casted"))
     val optimized = Optimize.execute(plan)
     // Though cast from `MapType(StringType, StringType, true)` to
     // `MapType(StringType, StringType, false)` is not allowed, here we just ensure that
@@ -69,4 +71,3 @@ class SimplifyCastsSuite extends PlanTest {
     comparePlans(optimized, plan, checkAnalysis = false)
   }
 }
-
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
index 8ad7c12020b82..7b4ef54c627e2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/SimplifyConditionalSuite.scala
@@ -18,13 +18,14 @@
 package org.apache.spark.sql.catalyst.optimizer
 
 import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules._
-import org.apache.spark.sql.types.{IntegerType, NullType}
+import org.apache.spark.sql.types.{BooleanType, IntegerType}
 
 
 class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
@@ -34,20 +35,18 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
       BooleanSimplification, ConstantFolding, SimplifyConditionals) :: Nil
   }
 
+  private val relation = LocalRelation('a.int, 'b.int, 'c.boolean)
+
   protected def assertEquivalent(e1: Expression, e2: Expression): Unit = {
-    val correctAnswer = Project(Alias(e2, "out")() :: Nil, OneRowRelation()).analyze
-    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, OneRowRelation()).analyze)
+    val correctAnswer = Project(Alias(e2, "out")() :: Nil, relation).analyze
+    val actual = Optimize.execute(Project(Alias(e1, "out")() :: Nil, relation).analyze)
     comparePlans(actual, correctAnswer)
   }
 
   private val trueBranch = (TrueLiteral, Literal(5))
   private val normalBranch = (NonFoldableLiteral(true), Literal(10))
   private val unreachableBranch = (FalseLiteral, Literal(20))
-  private val nullBranch = (Literal.create(null, NullType), Literal(30))
-
-  val isNotNullCond = IsNotNull(UnresolvedAttribute(Seq("a")))
-  val isNullCond = IsNull(UnresolvedAttribute("b"))
-  val notCond = Not(UnresolvedAttribute("c"))
+  private val nullBranch = (Literal.create(null, BooleanType), Literal(30))
 
   test("simplify if") {
     assertEquivalent(
@@ -59,7 +58,7 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
       Literal(20))
 
     assertEquivalent(
-      If(Literal.create(null, NullType), Literal(10), Literal(20)),
+      If(Literal.create(null, BooleanType), Literal(10), Literal(20)),
       Literal(20))
   }
 
@@ -127,9 +126,9 @@ class SimplifyConditionalSuite extends PlanTest with PredicateHelper {
   test("simplify CaseWhen if all the outputs are semantic equivalence") {
     // When the conditions in `CaseWhen` are all deterministic, `CaseWhen` can be removed.
     assertEquivalent(
-      CaseWhen((isNotNullCond, Subtract(Literal(3), Literal(2))) ::
-        (isNullCond, Literal(1)) ::
-        (notCond, Add(Literal(6), Literal(-5))) ::
+      CaseWhen(('a.isNotNull, Subtract(Literal(3), Literal(2))) ::
+        ('b.isNull, Literal(1)) ::
+        (!'c, Add(Literal(6), Literal(-5))) ::
         Nil,
         Add(Literal(2), Literal(-1))),
       Literal(1)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
index 10e970d534c49..17dc958841c48 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinReorderSuite.scala
@@ -226,7 +226,7 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     assertEqualPlans(query, expected)
   }
 
-  test("Test 3:  Star join on a subset of dimensions since join column is not unique") {
+  test("Test 3: Star join on a subset of dimensions since join column is not unique") {
     // Star join:
     //   (=)  (=)
     // d1 - f1 - d2
@@ -254,9 +254,9 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     val expected =
       f1.join(d1.where(nameToAttr("d1_c2") === 2), Inner,
           Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
-        .join(d3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
-        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
-        .join(s3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("s3_c2")))
+        .join(d3, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
+        .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_c4")))
+        .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
         .select(outputsOf(d1, f1, d2, s3, d3): _*)
 
     assertEqualPlans(query, expected)
@@ -316,20 +316,23 @@ class StarJoinReorderSuite extends PlanTest with StatsEstimationTestBase {
     // Positional join reordering: d3_ns, f1, d1, d2, s3
     // Star join reordering: empty
 
+    val d3_pk1 = d3_ns.output.find(_.name == "d3_pk1").get
+    val d3_fk1 = d3_ns.output.find(_.name == "d3_fk1").get
+
     val query =
       d3_ns.join(f1).join(d1).join(d2).join(s3)
         .where((nameToAttr("f1_fk2") === nameToAttr("d2_pk1")) &&
           (nameToAttr("d2_c2") === 2) &&
           (nameToAttr("f1_fk1") === nameToAttr("d1_pk1")) &&
-          (nameToAttr("f1_fk3") === nameToAttr("d3_pk1")) &&
-          (nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+          (nameToAttr("f1_fk3") === d3_pk1) &&
+          (d3_fk1 === nameToAttr("s3_pk1")))
 
     val equivQuery =
-      d3_ns.join(f1, Inner, Some(nameToAttr("f1_fk3") === nameToAttr("d3_pk1")))
+      d3_ns.join(f1, Inner, Some(nameToAttr("f1_fk3") === d3_pk1))
         .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk1")))
         .join(d2.where(nameToAttr("d2_c2") === 2), Inner,
           Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk1")))
-        .join(s3, Inner, Some(nameToAttr("d3_fk1") === nameToAttr("s3_pk1")))
+        .join(s3, Inner, Some(d3_fk1 === nameToAttr("s3_pk1")))
 
     assertEqualPlans(query, equivQuery)
   }

From bfe94890061138515ec7b6bcb27165e61e150ea7 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 5 Aug 2020 10:35:03 -0700
Subject: [PATCH 0962/1280] [SPARK-32543][R] Remove arrow::as_tibble usage in
 SparkR

SparkR increased the minimal version of Arrow R version to 1.0.0 at SPARK-32452, and Arrow R 0.14 dropped `as_tibble`. We can remove the usage in SparkR.

To remove codes unused anymore.

No.

GitHub Actions will test them out.

Closes #29361 from HyukjinKwon/SPARK-32543.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 R/pkg/R/DataFrame.R   |  7 +------
 R/pkg/R/deserialize.R | 13 +------------
 2 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 15b3ce2935427..f9a07f65325fb 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1234,12 +1234,7 @@ setMethod("collect",
                 output <- tryCatch({
                   doServerAuth(conn, authSecret)
                   arrowTable <- arrow::read_arrow(readRaw(conn))
-                  # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-                  if (exists("as_tibble", envir = asNamespace("arrow"))) {
-                    as.data.frame(arrow::as_tibble(arrowTable), stringsAsFactors = stringsAsFactors)
-                  } else {
-                    as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
-                  }
+                  as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
                 }, finally = {
                   close(conn)
                 })
diff --git a/R/pkg/R/deserialize.R b/R/pkg/R/deserialize.R
index 3e7c456bd548d..5d22340fb62a0 100644
--- a/R/pkg/R/deserialize.R
+++ b/R/pkg/R/deserialize.R
@@ -233,24 +233,13 @@ readMultipleObjectsWithKeys <- function(inputCon) {
 
 readDeserializeInArrow <- function(inputCon) {
   if (requireNamespace("arrow", quietly = TRUE)) {
-    # Arrow drops `as_tibble` since 0.14.0, see ARROW-5190.
-    useAsTibble <- exists("as_tibble", envir = asNamespace("arrow"))
-
-
     # Currently, there looks no way to read batch by batch by socket connection in R side,
     # See ARROW-4512. Therefore, it reads the whole Arrow streaming-formatted binary at once
     # for now.
     dataLen <- readInt(inputCon)
     arrowData <- readBin(inputCon, raw(), as.integer(dataLen), endian = "big")
     batches <- arrow::RecordBatchStreamReader$create(arrowData)$batches()
-
-    if (useAsTibble) {
-      as_tibble <- get("as_tibble", envir = asNamespace("arrow"))
-      # Read all groupped batches. Tibble -> data.frame is cheap.
-      lapply(batches, function(batch) as.data.frame(as_tibble(batch)))
-    } else {
-      lapply(batches, function(batch) as.data.frame(batch))
-    }
+    lapply(batches, function(batch) as.data.frame(batch))
   } else {
     stop("'arrow' package should be installed.")
   }

From 292bfc3b92d0d59b3f05781a3a3436286f317bce Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Wed, 12 Aug 2020 13:16:57 +0900
Subject: [PATCH 0963/1280] [SPARK-32586][SQL] Fix NumberFormatException error
 message when ansi is enabled

### What changes were proposed in this pull request?

This pr fixes the error message of `NumberFormatException` when casting invalid input to FractionalType and enabling **ansi**:
```
spark-sql> set spark.sql.ansi.enabled=true;
spark.sql.ansi.enabled	true
spark-sql> create table SPARK_32586 using parquet as select 's' s;
spark-sql> select * from SPARK_32586 where s > 1.13D;
java.lang.NumberFormatException: invalid input syntax for type numeric: columnartorow_value_0
```

After this pr:
```
spark-sql> select * from SPARK_32586 where s > 1.13D;
java.lang.NumberFormatException: invalid input syntax for type numeric: s
```

### Why are the changes needed?

Improve error message.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test.

Closes #29405 from wangyum/SPARK-32586.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 5d130f03607d2448e2f01814de7d330c512901b7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/catalyst/expressions/Cast.scala   |  6 +++---
 .../sql/catalyst/expressions/CastSuite.scala    | 17 +++++++++--------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index ef70915a5c969..c8985d4a12129 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -1153,7 +1153,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
       case StringType =>
         (c, evPrim, evNull) =>
           val handleException = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"""throw new NumberFormatException("invalid input syntax for type numeric: " + $c);"""
           } else {
             s"$evNull =true;"
           }
@@ -1513,7 +1513,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val floatStr = ctx.freshVariable("floatStr", StringType)
         (c, evPrim, evNull) =>
           val handleNull = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"""throw new NumberFormatException("invalid input syntax for type numeric: " + $c);"""
           } else {
             s"$evNull = true;"
           }
@@ -1549,7 +1549,7 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
         val doubleStr = ctx.freshVariable("doubleStr", StringType)
         (c, evPrim, evNull) =>
           val handleNull = if (ansiEnabled) {
-            s"""throw new NumberFormatException("invalid input syntax for type numeric: $c");"""
+            s"""throw new NumberFormatException("invalid input syntax for type numeric: " + $c);"""
           } else {
             s"$evNull = true;"
           }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index f5637200583ef..08aa2aa68771e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -1332,24 +1332,25 @@ class AnsiCastSuite extends CastSuiteBase {
       val array = Literal.create(Seq("123", "true", "f", null),
         ArrayType(StringType, containsNull = true))
       checkExceptionInExpression[NumberFormatException](
-        cast(array, ArrayType(dataType, containsNull = true)), "invalid input")
+        cast(array, ArrayType(dataType, containsNull = true)),
+        "invalid input syntax for type numeric: true")
       checkExceptionInExpression[NumberFormatException](
-        cast("string", dataType), "invalid input")
+        cast("string", dataType), "invalid input syntax for type numeric: string")
       checkExceptionInExpression[NumberFormatException](
-        cast("123-string", dataType), "invalid input")
+        cast("123-string", dataType), "invalid input syntax for type numeric: 123-string")
       checkExceptionInExpression[NumberFormatException](
-        cast("2020-07-19", dataType), "invalid input")
+        cast("2020-07-19", dataType), "invalid input syntax for type numeric: 2020-07-19")
       checkExceptionInExpression[NumberFormatException](
-        cast("1.23", dataType), "invalid input")
+        cast("1.23", dataType), "invalid input syntax for type numeric: 1.23")
     }
 
     Seq(DoubleType, FloatType, DecimalType.USER_DEFAULT).foreach { dataType =>
       checkExceptionInExpression[NumberFormatException](
-        cast("string", dataType), "invalid input")
+        cast("string", dataType), "invalid input syntax for type numeric: string")
       checkExceptionInExpression[NumberFormatException](
-        cast("123.000.00", dataType), "invalid input")
+        cast("123.000.00", dataType), "invalid input syntax for type numeric: 123.000.00")
       checkExceptionInExpression[NumberFormatException](
-        cast("abc.com", dataType), "invalid input")
+        cast("abc.com", dataType), "invalid input syntax for type numeric: abc.com")
     }
   }
 

From e7d45f846ab50247ac8ad01a86e0dedb0fefe8b4 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 12 Aug 2020 13:32:16 +0900
Subject: [PATCH 0964/1280] [SPARK-32594][SQL] Fix serialization of dates
 inserted to Hive tables

### What changes were proposed in this pull request?
Fix `DaysWritable` by overriding parent's method `def get(doesTimeMatter: Boolean): Date` from `DateWritable` instead of `Date get()` because the former one uses the first one. The bug occurs because `HiveOutputWriter.write()` call `def get(doesTimeMatter: Boolean): Date` transitively with default implementation from the parent class  `DateWritable` which doesn't respect date rebases and uses not initialized `daysSinceEpoch` (0 which `1970-01-01`).

### Why are the changes needed?
The changes fix the bug:
```sql
spark-sql> CREATE TABLE table1 (d date);
spark-sql> INSERT INTO table1 VALUES (date '2020-08-11');
spark-sql> SELECT * FROM table1;
1970-01-01
```
The expected result of the last SQL statement must be **2020-08-11** but got **1970-01-01**.

### Does this PR introduce _any_ user-facing change?
Yes. After the fix, `INSERT` work correctly:
```sql
spark-sql> SELECT * FROM table1;
2020-08-11
```

### How was this patch tested?
Add new test to `HiveSerDeReadWriteSuite`

Closes #29409 from MaxGekk/insert-date-into-hive-table.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0477d234672d6b02f906428dcf2536f26fb4fd04)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/datasources/DaysWritable.scala    | 4 +++-
 .../sql/hive/execution/HiveSerDeReadWriteSuite.scala      | 8 ++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
index 486f678e49d7d..56c176e024aca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
@@ -54,7 +54,9 @@ class DaysWritable(
   }
 
   override def getDays: Int = julianDays
-  override def get(): Date = new Date(DateWritable.daysToMillis(julianDays))
+  override def get(doesTimeMatter: Boolean): Date = {
+    new Date(DateWritable.daysToMillis(julianDays, doesTimeMatter))
+  }
 
   override def set(d: Int): Unit = {
     gregorianDays = d
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
index f8ba7bf2c1a62..10a0cdb760362 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
@@ -184,4 +184,12 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS
       checkComplexTypes(fileFormat)
     }
   }
+
+  test("SPARK-32594: insert dates to a Hive table") {
+    withTable("table1") {
+      sql("CREATE TABLE table1 (d date)")
+      sql("INSERT INTO table1 VALUES (date '2020-08-11')")
+      checkAnswer(spark.table("table1"), Row(Date.valueOf("2020-08-11")))
+    }
+  }
 }

From 9a3811dbf5f1234c1587337a3d74823d1d163b53 Mon Sep 17 00:00:00 2001
From: Tin Hang To <tinto@us.ibm.com>
Date: Wed, 12 Aug 2020 06:39:10 +0000
Subject: [PATCH 0965/1280] [SPARK-31703][SQL] Parquet RLE float/double are
 read incorrectly on big endian platforms

### What changes were proposed in this pull request?
This PR fixes the issue introduced during SPARK-26985.

SPARK-26985 changes the `putDoubles()` and `putFloats()` methods to respect the platform's endian-ness.  However, that causes the RLE paths in VectorizedRleValuesReader.java to read the RLE entries in parquet as BIG_ENDIAN on big endian platforms (i.e., as is), even though parquet data is always in little endian format.

The comments in `WriteableColumnVector.java` say those methods are used for "ieee formatted doubles in platform native endian" (or floats), but since the data in parquet is always in little endian format, use of those methods appears to be inappropriate.

To demonstrate the problem with spark-shell:

```scala
import org.apache.spark._
import org.apache.spark.sql._
import org.apache.spark.sql.types._

var data = Seq(
  (1.0, 0.1),
  (2.0, 0.2),
  (0.3, 3.0),
  (4.0, 4.0),
  (5.0, 5.0))

var df = spark.createDataFrame(data).write.mode(SaveMode.Overwrite).parquet("/tmp/data.parquet2")
var df2 = spark.read.parquet("/tmp/data.parquet2")
df2.show()
```

result:

```scala
+--------------------+--------------------+
|                  _1|                  _2|
+--------------------+--------------------+
|           3.16E-322|-1.54234871366845...|
|         2.0553E-320|         2.0553E-320|
|          2.561E-320|          2.561E-320|
|4.66726145843124E-62|         1.0435E-320|
|        3.03865E-319|-1.54234871366757...|
+--------------------+--------------------+
```

Also tests in ParquetIOSuite that involve float/double data would fail, e.g.,

- basic data types (without binary)
- read raw Parquet file

/examples/src/main/python/mllib/isotonic_regression_example.py would fail as well.

Purposed code change is to add `putDoublesLittleEndian()` and `putFloatsLittleEndian()` methods for parquet to invoke, just like the existing `putIntsLittleEndian()` and `putLongsLittleEndian()`.  On little endian platforms they would call `putDoubles()` and `putFloats()`, on big endian they would read the entries as little endian like pre-SPARK-26985.

No new unit-test is introduced as the existing ones are actually sufficient.

### Why are the changes needed?
RLE float/double data in parquet files will not be read back correctly on big endian platforms.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
All unit tests (mvn test) were ran and OK.

Closes #29383 from tinhto-000/SPARK-31703.

Authored-by: Tin Hang To <tinto@us.ibm.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a418548dad57775fbb10b4ea690610bad1a8bfb0)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../parquet/VectorizedPlainValuesReader.java  |   4 +-
 .../vectorized/OffHeapColumnVector.java       |  26 ++++
 .../vectorized/OnHeapColumnVector.java        |  24 ++++
 .../vectorized/WritableColumnVector.java      |  12 ++
 .../vectorized/ColumnarBatchSuite.scala       | 116 +++++++++++++++---
 5 files changed, 166 insertions(+), 16 deletions(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
index eddbf39178e9a..994779b618829 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java
@@ -169,7 +169,7 @@ public final void readFloats(int total, WritableColumnVector c, int rowId) {
 
     if (buffer.hasArray()) {
       int offset = buffer.arrayOffset() + buffer.position();
-      c.putFloats(rowId, total, buffer.array(), offset);
+      c.putFloatsLittleEndian(rowId, total, buffer.array(), offset);
     } else {
       for (int i = 0; i < total; i += 1) {
         c.putFloat(rowId + i, buffer.getFloat());
@@ -184,7 +184,7 @@ public final void readDoubles(int total, WritableColumnVector c, int rowId) {
 
     if (buffer.hasArray()) {
       int offset = buffer.arrayOffset() + buffer.position();
-      c.putDoubles(rowId, total, buffer.array(), offset);
+      c.putDoublesLittleEndian(rowId, total, buffer.array(), offset);
     } else {
       for (int i = 0; i < total; i += 1) {
         c.putDouble(rowId + i, buffer.getDouble());
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 3b919c7d471f4..982a26c0ef775 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -425,6 +425,19 @@ public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putFloats(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      long offset = data + 4L * rowId;
+      for (int i = 0; i < count; ++i, offset += 4) {
+        Platform.putFloat(null, offset, bb.getFloat(srcIndex + (4 * i)));
+      }
+    }
+  }
+
   @Override
   public float getFloat(int rowId) {
     if (dictionary == null) {
@@ -480,6 +493,19 @@ public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putDoubles(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      long offset = data + 8L * rowId;
+      for (int i = 0; i < count; ++i, offset += 8) {
+        Platform.putDouble(null, offset, bb.getDouble(srcIndex + (8 * i)));
+      }
+    }
+  }
+
   @Override
   public double getDouble(int rowId) {
     if (dictionary == null) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index 1bf3126664177..625b78be4de94 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -403,6 +403,18 @@ public void putFloats(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putFloats(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      for (int i = 0; i < count; ++i) {
+        floatData[i + rowId] = bb.getFloat(srcIndex + (4 * i));
+      }
+    }
+  }
+
   @Override
   public float getFloat(int rowId) {
     if (dictionary == null) {
@@ -452,6 +464,18 @@ public void putDoubles(int rowId, int count, byte[] src, int srcIndex) {
     }
   }
 
+  @Override
+  public void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex) {
+    if (!bigEndianPlatform) {
+      putDoubles(rowId, count, src, srcIndex);
+    } else {
+      ByteBuffer bb = ByteBuffer.wrap(src).order(ByteOrder.LITTLE_ENDIAN);
+      for (int i = 0; i < count; ++i) {
+        doubleData[i + rowId] = bb.getDouble(srcIndex + (8 * i));
+      }
+    }
+  }
+
   @Override
   public double getDouble(int rowId) {
     if (dictionary == null) {
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
index f04b0707ff7b5..8c0f1e1257503 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
@@ -312,6 +312,12 @@ public WritableColumnVector reserveDictionaryIds(int capacity) {
    */
   public abstract void putFloats(int rowId, int count, byte[] src, int srcIndex);
 
+  /**
+   * Sets values from [src[srcIndex], src[srcIndex + count * 4]) to [rowId, rowId + count)
+   * The data in src must be ieee formatted floats in little endian.
+   */
+  public abstract void putFloatsLittleEndian(int rowId, int count, byte[] src, int srcIndex);
+
   /**
    * Sets `value` to the value at rowId.
    */
@@ -333,6 +339,12 @@ public WritableColumnVector reserveDictionaryIds(int capacity) {
    */
   public abstract void putDoubles(int rowId, int count, byte[] src, int srcIndex);
 
+  /**
+   * Sets values from [src[srcIndex], src[srcIndex + count * 8]) to [rowId, rowId + count)
+   * The data in src must be ieee formatted doubles in little endian.
+   */
+  public abstract void putDoublesLittleEndian(int rowId, int count, byte[] src, int srcIndex);
+
   /**
    * Puts a byte array that already exists in this column.
    */
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
index 37d028d6a713f..a369b2d6900f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnarBatchSuite.scala
@@ -447,13 +447,6 @@ class ColumnarBatchSuite extends SparkFunSuite {
       Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET, 2.234f)
       Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET + 4, 1.123f)
 
-      if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-        // Ensure array contains Little Endian floats
-        val bb = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN)
-        Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET, bb.getFloat(0))
-        Platform.putFloat(buffer, Platform.BYTE_ARRAY_OFFSET + 4, bb.getFloat(4))
-      }
-
       column.putFloats(idx, 1, buffer, 4)
       column.putFloats(idx + 1, 1, buffer, 0)
       reference += 1.123f
@@ -491,6 +484,57 @@ class ColumnarBatchSuite extends SparkFunSuite {
       }
   }
 
+  testVector("[SPARK-31703] Float API - Little Endian", 1024, FloatType) {
+    column =>
+      val seed = System.currentTimeMillis()
+      val random = new Random(seed)
+      val reference = mutable.ArrayBuffer.empty[Float]
+
+      var idx = 0
+
+      val littleEndian = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN)
+      littleEndian.putFloat(0, 1.357f)
+      littleEndian.putFloat(4, 2.468f)
+      val arr = new Array[Byte](littleEndian.remaining)
+      littleEndian.get(arr)
+
+      column.putFloatsLittleEndian(idx, 1, arr, 4)
+      column.putFloatsLittleEndian(idx + 1, 1, arr, 0)
+      reference += 2.468f
+      reference += 1.357f
+      idx += 2
+
+      column.putFloatsLittleEndian(idx, 2, arr, 0)
+      reference += 1.357f
+      reference += 2.468f
+      idx += 2
+
+      while (idx < column.capacity) {
+        val single = random.nextBoolean()
+        if (single) {
+          val v = random.nextFloat()
+          column.putFloat(idx, v)
+          reference += v
+          idx += 1
+        } else {
+          val n = math.min(random.nextInt(column.capacity / 20), column.capacity - idx)
+          val v = random.nextFloat()
+          column.putFloats(idx, n, v)
+          var i = 0
+          while (i < n) {
+            reference += v
+            i += 1
+          }
+          idx += n
+        }
+      }
+
+      reference.zipWithIndex.foreach { v =>
+        assert(v._1 == column.getFloat(v._2),
+          "Seed = " + seed + " VectorType=" + column.getClass.getSimpleName)
+      }
+  }
+
   testVector("Double APIs", 1024, DoubleType) {
     column =>
       val seed = System.currentTimeMillis()
@@ -531,13 +575,6 @@ class ColumnarBatchSuite extends SparkFunSuite {
       Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET, 2.234)
       Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET + 8, 1.123)
 
-      if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
-        // Ensure array contains Little Endian doubles
-        val bb = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN)
-        Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET, bb.getDouble(0))
-        Platform.putDouble(buffer, Platform.BYTE_ARRAY_OFFSET + 8, bb.getDouble(8))
-      }
-
       column.putDoubles(idx, 1, buffer, 8)
       column.putDoubles(idx + 1, 1, buffer, 0)
       reference += 1.123
@@ -575,6 +612,57 @@ class ColumnarBatchSuite extends SparkFunSuite {
       }
   }
 
+  testVector("[SPARK-31703] Double API - Little Endian", 1024, DoubleType) {
+    column =>
+      val seed = System.currentTimeMillis()
+      val random = new Random(seed)
+      val reference = mutable.ArrayBuffer.empty[Double]
+
+      var idx = 0
+
+      val littleEndian = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN)
+      littleEndian.putDouble(0, 1.357)
+      littleEndian.putDouble(8, 2.468)
+      val arr = new Array[Byte](littleEndian.remaining)
+      littleEndian.get(arr)
+
+      column.putDoublesLittleEndian(idx, 1, arr, 8)
+      column.putDoublesLittleEndian(idx + 1, 1, arr, 0)
+      reference += 2.468
+      reference += 1.357
+      idx += 2
+
+      column.putDoublesLittleEndian(idx, 2, arr, 0)
+      reference += 1.357
+      reference += 2.468
+      idx += 2
+
+      while (idx < column.capacity) {
+        val single = random.nextBoolean()
+        if (single) {
+          val v = random.nextDouble()
+          column.putDouble(idx, v)
+          reference += v
+          idx += 1
+        } else {
+          val n = math.min(random.nextInt(column.capacity / 20), column.capacity - idx)
+          val v = random.nextDouble()
+          column.putDoubles(idx, n, v)
+          var i = 0
+          while (i < n) {
+            reference += v
+            i += 1
+          }
+          idx += n
+        }
+      }
+
+      reference.zipWithIndex.foreach { v =>
+        assert(v._1 == column.getDouble(v._2),
+          "Seed = " + seed + " VectorType=" + column.getClass.getSimpleName)
+      }
+  }
+
   testVector("String APIs", 7, StringType) {
     column =>
       val reference = mutable.ArrayBuffer.empty[String]

From ecc2997f5fab946c2640387c1caf13b01c42b362 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 12 Aug 2020 20:59:59 +0900
Subject: [PATCH 0966/1280] [SPARK-32599][SQL][TESTS] Check the TEXTFILE file
 format in `HiveSerDeReadWriteSuite`

### What changes were proposed in this pull request?
- Test TEXTFILE together with the PARQUET and ORC file formats in `HiveSerDeReadWriteSuite`
- Remove the "SPARK-32594: insert dates to a Hive table" added by #29409

### Why are the changes needed?
- To improve test coverage, and test other row SerDe - `org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe`.
- The removed test is not needed anymore because the bug reported in SPARK-32594 is triggered by the TEXTFILE file format too.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the modified test suite `HiveSerDeReadWriteSuite`.

Closes #29417 from MaxGekk/textfile-HiveSerDeReadWriteSuite.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f664aaaab13997bf61381aecfd4703f7e32e8fa1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/hive/execution/HiveSerDeReadWriteSuite.scala   | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
index 10a0cdb760362..ac9ae8c9229db 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSerDeReadWriteSuite.scala
@@ -154,7 +154,7 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS
     }
   }
 
-  Seq("PARQUET", "ORC").foreach { fileFormat =>
+  Seq("PARQUET", "ORC", "TEXTFILE").foreach { fileFormat =>
     test(s"Read/Write Hive $fileFormat serde table") {
       // Numeric Types
       checkNumericTypes(fileFormat, "TINYINT", 2)
@@ -184,12 +184,4 @@ class HiveSerDeReadWriteSuite extends QueryTest with SQLTestUtils with TestHiveS
       checkComplexTypes(fileFormat)
     }
   }
-
-  test("SPARK-32594: insert dates to a Hive table") {
-    withTable("table1") {
-      sql("CREATE TABLE table1 (d date)")
-      sql("INSERT INTO table1 VALUES (date '2020-08-11')")
-      checkAnswer(spark.table("table1"), Row(Date.valueOf("2020-08-11")))
-    }
-  }
 }

From 742c4df2277dd5c11b6935271be6aaaf254d0049 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 12 Aug 2020 21:05:50 +0900
Subject: [PATCH 0967/1280] [SPARK-32250][SPARK-27510][CORE][TEST] Fix flaky
 MasterSuite.test(...) in Github Actions

### What changes were proposed in this pull request?

Set more dispatcher threads for the flaky test.

### Why are the changes needed?

When running test on Github Actions machine, the available processors in JVM  is only 2, while on Jenkins it's 32. For this specific test, 2 available processors, which also decides the number of threads in Dispatcher, are not enough to consume the messages. In the worst situation, `MockExecutorLaunchFailWorker` would occupy these 2 threads for handling messages `LaunchDriver`, `LaunchExecutor` at the same time but leave no thread for the driver to handle the message `RegisteredApplication`. At the end, it results in a deadlock situation and causes the test failure.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

We can check whether the test is still flaky in Github Actions after this fix.

Closes #29408 from Ngone51/spark-32250.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c6ea98323fd23393541efadd814a611a25fa78b2)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/deploy/master/MasterSuite.scala  | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 0cf573c2490b3..994eb6f14e411 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -689,7 +689,16 @@ class MasterSuite extends SparkFunSuite
     val master = makeAliveMaster()
     var worker: MockExecutorLaunchFailWorker = null
     try {
-      worker = new MockExecutorLaunchFailWorker(master)
+      val conf = new SparkConf()
+      // SPARK-32250: When running test on Github Action machine, the available processors in JVM
+      // is only 2, while on Jenkins it's 32. For this specific test, 2 available processors, which
+      // also decides number of threads in Dispatcher, is not enough to consume the messages. In
+      // the worst situation, MockExecutorLaunchFailWorker would occupy these 2 threads for
+      // handling messages LaunchDriver, LaunchExecutor at the same time but leave no thread for
+      // the driver to handle the message RegisteredApplication. At the end, it results in the dead
+      // lock situation. Therefore, we need to set more threads to avoid the dead lock.
+      conf.set(Network.RPC_NETTY_DISPATCHER_NUM_THREADS, 6)
+      worker = new MockExecutorLaunchFailWorker(master, conf)
       worker.rpcEnv.setupEndpoint("worker", worker)
       val workerRegMsg = RegisterWorker(
         worker.id,

From 21c2fa4369d64a2c8b697526b57a5274b9bde9b5 Mon Sep 17 00:00:00 2001
From: Chen Zhang <izchen@126.com>
Date: Thu, 13 Aug 2020 11:54:43 +0900
Subject: [PATCH 0968/1280] [MINOR] Update URL of the parquet project in code
 comment

### What changes were proposed in this pull request?
Update URL of the parquet project in code comment.

### Why are the changes needed?
The original url is not available.

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
No test needed.

Closes #29416 from izchen/Update-Parquet-URL.

Authored-by: Chen Zhang <izchen@126.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 08d86ebc057165865c58b4fa3d4dd67039946e26)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../datasources/parquet/VectorizedRleValuesReader.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
index 24347a4e3a0c5..03595068f0d31 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedRleValuesReader.java
@@ -46,7 +46,7 @@ public final class VectorizedRleValuesReader extends ValuesReader
   // Current decoding mode. The encoded data contains groups of either run length encoded data
   // (RLE) or bit packed data. Each group contains a header that indicates which group it is and
   // the number of values in the group.
-  // More details here: https://github.com/Parquet/parquet-format/blob/master/Encodings.md
+  // More details here: https://github.com/apache/parquet-format/blob/master/Encodings.md
   private enum MODE {
     RLE,
     PACKED

From 89765f556f26252aed1add71a9da84209ff03493 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Thu, 13 Aug 2020 03:52:12 +0000
Subject: [PATCH 0969/1280] [SPARK-32018][SQL][FOLLOWUP][3.0] Throw exception
 on decimal value overflow of sum aggregation

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/29125
In branch 3.0:
1. for hash aggregation, before https://github.com/apache/spark/pull/29125 there will be a runtime exception on decimal overflow of sum aggregation; after https://github.com/apache/spark/pull/29125, there could be a wrong result.
2. for sort aggregation, with/without https://github.com/apache/spark/pull/29125, there could be a wrong result on decimal overflow.

While in master branch(the future 3.1 release), the problem doesn't exist since in https://github.com/apache/spark/pull/27627 there is a flag for marking whether overflow happens in aggregation buffer. However, the aggregation buffer is written in steaming checkpoints. Thus, we can't change to aggregation buffer to resolve the issue.

As there is no easy solution for returning null/throwing exception regarding `spark.sql.ansi.enabled` on overflow in branch 3.0, we have to make a choice here: always throw exception on decimal value overflow of sum aggregation.
### Why are the changes needed?

Avoid returning wrong result in decimal value sum aggregation.

### Does this PR introduce _any_ user-facing change?

Yes, there is always exception on decimal value overflow of sum aggregation, instead of a possible wrong result.

### How was this patch tested?

Unit test case

Closes #29404 from gengliangwang/fixSum.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalyst/expressions/aggregate/Sum.scala  | 19 ++++++++--
 .../spark/sql/DataFrameAggregateSuite.scala   | 37 +++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
index d2daaac72fc85..d442549f20e80 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
@@ -71,23 +71,36 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
   )
 
   override lazy val updateExpressions: Seq[Expression] = {
+    val sumWithChild = resultType match {
+      case d: DecimalType =>
+        CheckOverflow(coalesce(sum, zero) + child.cast(sumDataType), d, nullOnOverflow = false)
+      case _ =>
+        coalesce(sum, zero) + child.cast(sumDataType)
+    }
+
     if (child.nullable) {
       Seq(
         /* sum = */
-        coalesce(coalesce(sum, zero) + child.cast(sumDataType), sum)
+        coalesce(sumWithChild, sum)
       )
     } else {
       Seq(
         /* sum = */
-        coalesce(sum, zero) + child.cast(sumDataType)
+        sumWithChild
       )
     }
   }
 
   override lazy val mergeExpressions: Seq[Expression] = {
+    val sumWithRight = resultType match {
+      case d: DecimalType =>
+        CheckOverflow(coalesce(sum.left, zero) + sum.right, d, nullOnOverflow = false)
+
+      case _ => coalesce(sum.left, zero) + sum.right
+    }
     Seq(
       /* sum = */
-      coalesce(coalesce(sum.left, zero) + sum.right, sum.left)
+      coalesce(sumWithRight, sum.left)
     )
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 54327b38c100b..8c0358e205b07 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -21,6 +21,7 @@ import scala.util.Random
 
 import org.scalatest.Matchers.the
 
+import org.apache.spark.SparkException
 import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
@@ -1044,6 +1045,42 @@ class DataFrameAggregateSuite extends QueryTest
     checkAnswer(sql(queryTemplate("FIRST")), Row(1))
     checkAnswer(sql(queryTemplate("LAST")), Row(3))
   }
+
+  private def exceptionOnDecimalOverflow(df: DataFrame): Unit = {
+    val msg = intercept[SparkException] {
+      df.collect()
+    }.getCause.getMessage
+    assert(msg.contains("cannot be represented as Decimal(38, 18)"))
+  }
+
+  test("SPARK-32018: Throw exception on decimal overflow at partial aggregate phase") {
+    val decimalString = "1" + "0" * 19
+    val union = spark.range(0, 1, 1, 1).union(spark.range(0, 11, 1, 1))
+    val hashAgg = union
+      .select(expr(s"cast('$decimalString' as decimal (38, 18)) as d"), lit("1").as("key"))
+      .groupBy("key")
+      .agg(sum($"d").alias("sumD"))
+      .select($"sumD")
+    exceptionOnDecimalOverflow(hashAgg)
+
+    val sortAgg = union
+      .select(expr(s"cast('$decimalString' as decimal (38, 18)) as d"), lit("a").as("str"),
+      lit("1").as("key")).groupBy("key")
+      .agg(sum($"d").alias("sumD"), min($"str").alias("minStr")).select($"sumD", $"minStr")
+    exceptionOnDecimalOverflow(sortAgg)
+  }
+
+  test("SPARK-32018: Throw exception on decimal overflow at merge aggregation phase") {
+    val decimalString = "5" + "0" * 19
+    val union = spark.range(0, 1, 1, 1).union(spark.range(0, 1, 1, 1))
+      .union(spark.range(0, 1, 1, 1))
+    val agg = union
+      .select(expr(s"cast('$decimalString' as decimal (38, 18)) as d"), lit("1").as("key"))
+      .groupBy("key")
+      .agg(sum($"d").alias("sumD"))
+      .select($"sumD")
+    exceptionOnDecimalOverflow(agg)
+  }
 }
 
 case class B(c: Option[Double])

From 81d7747f972bf996616394a5313594301ffabf6e Mon Sep 17 00:00:00 2001
From: alexander-daskalov <alexander.daskalov@adevinta.com>
Date: Fri, 14 Aug 2020 22:10:41 +0900
Subject: [PATCH 0970/1280] [MINOR][SQL] Fixed approx_count_distinct rsd param
 description

### What changes were proposed in this pull request?

In the docs concerning the approx_count_distinct I have changed the description of the rsd parameter from **_maximum estimation error allowed_** to _**maximum relative standard deviation allowed**_

### Why are the changes needed?

Maximum estimation error allowed can be misleading. You can set the target relative standard deviation, which affects the estimation error, but on given runs the estimation error can still be above the rsd parameter.

### Does this PR introduce _any_ user-facing change?

This PR should make it easier for users reading the docs to understand that the rsd parameter in approx_count_distinct doesn't cap the estimation error, but just sets the target deviation instead,

### How was this patch tested?

No tests, as no code changes were made.

Closes #29424 from Comonut/fix-approx_count_distinct-rsd-param-description.

Authored-by: alexander-daskalov <alexander.daskalov@adevinta.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 10edeafc69250afef8c71ed7b3c77992f67aa4ff)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 R/pkg/R/functions.R                                           | 2 +-
 python/pyspark/sql/functions.py                               | 4 ++--
 .../aggregate/ApproxCountDistinctForIntervals.scala           | 3 ++-
 .../catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala  | 4 ++--
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala    | 4 ++--
 sql/core/src/main/scala/org/apache/spark/sql/functions.scala  | 4 ++--
 6 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index 173dbc4b39bad..edec2f4d2e56e 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -2199,7 +2199,7 @@ setMethod("pmod", signature(y = "Column"),
             column(jc)
           })
 
-#' @param rsd maximum estimation error allowed (default = 0.05).
+#' @param rsd maximum relative standard deviation allowed (default = 0.05).
 #'
 #' @rdname column_aggregate_functions
 #' @aliases approx_count_distinct,Column-method
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 476aab433e1ec..0bf6bc65384bd 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -335,8 +335,8 @@ def approx_count_distinct(col, rsd=None):
     """Aggregate function: returns a new :class:`Column` for approximate distinct count of
     column `col`.
 
-    :param rsd: maximum estimation error allowed (default = 0.05). For rsd < 0.01, it is more
-        efficient to use :func:`countDistinct`
+    :param rsd: maximum relative standard deviation allowed (default = 0.05).
+        For rsd < 0.01, it is more efficient to use :func:`countDistinct`
 
     >>> df.agg(approx_count_distinct(df.age).alias('distinct_ages')).collect()
     [Row(distinct_ages=2)]
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
index f96a087972f1f..103f55e58febd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxCountDistinctForIntervals.scala
@@ -39,7 +39,8 @@ import org.apache.spark.unsafe.Platform
  *                            and its elements should be sorted into ascending order.
  *                            Duplicate endpoints are allowed, e.g. (1, 5, 5, 10), and ndv for
  *                            interval (5, 5] would be 1.
- * @param relativeSD The maximum estimation error allowed in the HyperLogLogPlusPlus algorithm.
+ * @param relativeSD The maximum relative standard deviation allowed
+ *                   in the HyperLogLogPlusPlus algorithm.
  */
 case class ApproxCountDistinctForIntervals(
     child: Expression,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
index aed36902b1567..b3cc9a31fec2f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala
@@ -41,13 +41,13 @@ import org.apache.spark.sql.types._
  * https://docs.google.com/document/d/1gyjfMHy43U9OWBXxfaeG-3MjGzejW1dlpyMwEYAAWEI/view?fullscreen#
  *
  * @param child to estimate the cardinality of.
- * @param relativeSD the maximum estimation error allowed.
+ * @param relativeSD the maximum relative standard deviation allowed.
  */
 // scalastyle:on
 @ExpressionDescription(
   usage = """
     _FUNC_(expr[, relativeSD]) - Returns the estimated cardinality by HyperLogLog++.
-      `relativeSD` defines the maximum estimation error allowed.""",
+      `relativeSD` defines the maximum relative standard deviation allowed.""",
   examples = """
     Examples:
       > SELECT _FUNC_(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index fd57095031648..cdf72039b2a0e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1578,8 +1578,8 @@ object SQLConf {
   val NDV_MAX_ERROR =
     buildConf("spark.sql.statistics.ndv.maxError")
       .internal()
-      .doc("The maximum estimation error allowed in HyperLogLog++ algorithm when generating " +
-        "column level statistics.")
+      .doc("The maximum relative standard deviation allowed in HyperLogLog++ algorithm " +
+        "when generating column level statistics.")
       .version("2.1.1")
       .doubleConf
       .createWithDefault(0.05)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index e5b0c1e1a3a16..cf889f5616671 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -262,7 +262,7 @@ object functions {
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
-   * @param rsd maximum estimation error allowed (default = 0.05)
+   * @param rsd maximum relative standard deviation allowed (default = 0.05)
    *
    * @group agg_funcs
    * @since 2.1.0
@@ -274,7 +274,7 @@ object functions {
   /**
    * Aggregate function: returns the approximate number of distinct items in a group.
    *
-   * @param rsd maximum estimation error allowed (default = 0.05)
+   * @param rsd maximum relative standard deviation allowed (default = 0.05)
    *
    * @group agg_funcs
    * @since 2.1.0

From 05144a5c10cd37ebdbb55fde37d677def49af11f Mon Sep 17 00:00:00 2001
From: Ruifeng Zheng <ruifengz@apache.org>
Date: Sat, 15 Aug 2020 01:37:47 +0000
Subject: [PATCH 0971/1280] Preparing Spark release v3.0.1-rc1

---
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 2 +-
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 38 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/assembly/pom.xml b/assembly/pom.xml
index 8bef9d8bc56cb..c47425a14cf69 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index fc1441d80b7bf..0948c6b1616e4 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index de2a6fb18d69b..e7b16f1aa0b9f 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index 6c0c016fdf94f..c32a74fb30e97 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index b8df19152a606..56ca7258804b2 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 8119709a6aeff..6b60cb36a6b30 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 96ac525880213..3eb1b77f7d756 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index f385f07f951ce..bfdbc7e526e37 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 0c9befc921af6..45970b7077bbb 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index 5f2970fb645bc..65be4c4a9c482 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -17,7 +17,7 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.1-SNAPSHOT
+SPARK_VERSION: 3.0.1
 SPARK_VERSION_SHORT: 3.0.1
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
diff --git a/examples/pom.xml b/examples/pom.xml
index 1b59c629a8c73..19c4d405faac5 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 87183e72f5de3..b8cd328b2cb83 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index b57e3c0e25ae8..0f8d076318bbd 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 9426db5e11354..750797fd7af3b 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index d77364d0210fe..8d2e8804d8471 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index 70a3406701315..b329026fd6e49 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index a338b6511298f..6a0d72a903952 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 599ae172671e5..3ef13b76954ba 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 9fa18d9f82833..3166ee63c9ed3 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index acfc2c5a36b1a..02403f11ea82b 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index f0666536bb706..3f1dd80931a8e 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 48f559e4822f3..cc436182c6908 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 0759ce5c114b3..1e9102758d5f8 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index ed34a370a8274..cd1dbce4a5d3c 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index b8ae344e63e65..c88ee850cecb7 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 86cdd1d7237ab..0bf1eb70c0565 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.1-SNAPSHOT</version>
+  <version>3.0.1</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index 906298ff9964a..b19dd9e6c7314 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.1.dev0"
+__version__ = "3.0.1"
diff --git a/repl/pom.xml b/repl/pom.xml
index b826b87f53efc..fa48a4577e0b5 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index 168f7f5be8b0f..c0e469dd0cfc4 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 1d58b97805241..e859d964be7b1 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index e0c997f869aba..a8b7ae4c30891 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index 95299d2066ea3..ed51acfe16b79 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 6667c7bd089dc..d84e6e33d0934 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index fbfcdf88fa26e..5f540dcdd1007 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index f60820dd45f34..2c84451230dbe 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index e8c1598b3669c..cd186284dbc76 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 1e99b6c21522b..0a88f49782d6a 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 5430a3e600365..a38469847b6e5 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 38ab936287fdf3a61208c079aa5aae796417dd9b Mon Sep 17 00:00:00 2001
From: Ruifeng Zheng <ruifengz@apache.org>
Date: Sat, 15 Aug 2020 01:37:54 +0000
Subject: [PATCH 0972/1280] Preparing development version 3.0.2-SNAPSHOT

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 86514f237d1e6..a17dea9093771 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.1
+Version: 3.0.2
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index c47425a14cf69..e265bb948d9d4 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index 0948c6b1616e4..f433764648ccd 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index e7b16f1aa0b9f..4f31606fddf94 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index c32a74fb30e97..a3ac6611a3127 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 56ca7258804b2..5c5dfbabe862e 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 6b60cb36a6b30..8041d7a093da9 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 3eb1b77f7d756..e3a550288d71a 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index bfdbc7e526e37..09965b573706c 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 45970b7077bbb..19b5b30051602 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index 65be4c4a9c482..abb3b49fff54d 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -17,8 +17,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.1
-SPARK_VERSION_SHORT: 3.0.1
+SPARK_VERSION: 3.0.2-SNAPSHOT
+SPARK_VERSION_SHORT: 3.0.2
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index 19c4d405faac5..2082ece41569a 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index b8cd328b2cb83..923b3bd6981dc 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 0f8d076318bbd..4f16290891c3e 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 750797fd7af3b..7852552ed5a54 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 8d2e8804d8471..24f2d9acc1ee6 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index b329026fd6e49..c5d41b05baf3b 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 6a0d72a903952..c53a5392ff917 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 3ef13b76954ba..4f8cc83219dbc 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 3166ee63c9ed3..2dd1e786d3634 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index 02403f11ea82b..f1dd31eee35a2 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 3f1dd80931a8e..d760d5c32099c 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index cc436182c6908..860f648ff9e57 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 1e9102758d5f8..48ee744806ed2 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index cd1dbce4a5d3c..fd103d80268e3 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index c88ee850cecb7..135c4ca54a491 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 0bf1eb70c0565..e9ae204341c56 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.1</version>
+  <version>3.0.2-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index b19dd9e6c7314..e6410c11cf86d 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.1"
+__version__ = "3.0.2.dev0"
diff --git a/repl/pom.xml b/repl/pom.xml
index fa48a4577e0b5..92911ee094b4d 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index c0e469dd0cfc4..ef6c38a64ec8c 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index e859d964be7b1..693e720df1aa3 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index a8b7ae4c30891..ec93eefa1d8f7 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index ed51acfe16b79..f0031cf2466b5 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index d84e6e33d0934..e4d5ee614cb04 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 5f540dcdd1007..1a1673fdfd212 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 2c84451230dbe..f376afbd19599 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index cd186284dbc76..1b4c6dc110e97 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 0a88f49782d6a..251570496d805 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index a38469847b6e5..41662b1957703 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 6a889248265b63b2df0249df1c552082f5ed721e Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Sat, 15 Aug 2020 12:31:32 -0700
Subject: [PATCH 0973/1280] [SPARK-32625][SQL] Log error message when falling
 back to interpreter mode

### What changes were proposed in this pull request?

This pr log the error message when falling back to interpreter mode.

### Why are the changes needed?

Not all error messages are in `CodeGenerator`, such as:
```
21:48:44.612 WARN org.apache.spark.sql.catalyst.expressions.Predicate: Expr codegen error and falling back to interpreter mode
java.lang.IllegalArgumentException: Can not interpolate org.apache.spark.sql.types.Decimal into code block.
	at org.apache.spark.sql.catalyst.expressions.codegen.Block$BlockHelper$.$anonfun$code$1(javaCode.scala:240)
	at org.apache.spark.sql.catalyst.expressions.codegen.Block$BlockHelper$.$anonfun$code$1$adapted(javaCode.scala:236)
	at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
	at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manual test.

Closes #29440 from wangyum/SPARK-32625.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit c280c7f529e2766dd7dd45270bde340c28b9d74b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/CodeGeneratorWithInterpretedFallback.scala   | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
index 07fa813a98922..3b7219477bd69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CodeGeneratorWithInterpretedFallback.scala
@@ -51,9 +51,8 @@ abstract class CodeGeneratorWithInterpretedFallback[IN, OUT] extends Logging {
         try {
           createCodeGeneratedObject(in)
         } catch {
-          case NonFatal(_) =>
-            // We should have already seen the error message in `CodeGenerator`
-            logWarning("Expr codegen error and falling back to interpreter mode")
+          case NonFatal(e) =>
+            logWarning("Expr codegen error and falling back to interpreter mode", e)
             createInterpretedObject(in)
         }
     }

From c4807ced3913a4d524892dc7bab502250687a43c Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sun, 16 Aug 2020 12:07:37 -0500
Subject: [PATCH 0974/1280] [SPARK-32610][DOCS] Fix the link to
 metrics.dropwizard.io in monitoring.md to refer the proper version

### What changes were proposed in this pull request?

This PR fixes the link to metrics.dropwizard.io in monitoring.md to refer the proper version of the library.

### Why are the changes needed?

There are links to metrics.dropwizard.io in monitoring.md but the link targets refer the version 3.1.0, while we use 4.1.1.
Now that users can create their own metrics using the dropwizard library, it's better to fix the links to refer the proper version.

### Does this PR introduce _any_ user-facing change?

Yes. The modified links refer the version 4.1.1.

### How was this patch tested?

Build the docs and visit all the modified links.

Closes #29426 from sarutak/fix-dropwizard-url.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 9a79bbc8b6e426e7b29a9f4867beb396014d8046)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/monitoring.md | 8 ++++----
 pom.xml            | 4 ++++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index 1808167cf1bd8..4608a4e61a41c 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -718,7 +718,7 @@ The JSON end point is exposed at: `/applications/[app-id]/executors`, and the Pr
 The Prometheus endpoint is experimental and conditional to a configuration parameter: `spark.ui.prometheus.enabled=true` (the default is `false`).
 In addition, aggregated per-stage peak values of the executor memory metrics are written to the event log if
 `spark.eventLog.logStageExecutorMetrics` is true.  
-Executor memory metrics are also exposed via the Spark metrics system based on the Dropwizard metrics library.
+Executor memory metrics are also exposed via the Spark metrics system based on the [Dropwizard metrics library](http://metrics.dropwizard.io/4.1.1).
 A list of the available metrics, with a short description:
 
 <table class="table">
@@ -922,7 +922,7 @@ keep the paths consistent in both modes.
 # Metrics
 
 Spark has a configurable metrics system based on the
-[Dropwizard Metrics Library](http://metrics.dropwizard.io/).
+[Dropwizard Metrics Library](http://metrics.dropwizard.io/4.1.1).
 This allows users to report Spark metrics to a variety of sinks including HTTP, JMX, and CSV
 files. The metrics are generated by sources embedded in the Spark code base. They
 provide instrumentation for specific activities and Spark components.
@@ -1016,7 +1016,7 @@ activates the JVM source:
 ## List of available metrics providers 
 
 Metrics used by Spark are of multiple types: gauge, counter, histogram, meter and timer, 
-see [Dropwizard library documentation for details](https://metrics.dropwizard.io/3.1.0/getting-started/).
+see [Dropwizard library documentation for details](https://metrics.dropwizard.io/4.1.1/getting-started.html).
 The following list of components and metrics reports the name and some details about the available metrics,
 grouped per component instance and source namespace.
 The most common time of metrics used in Spark instrumentation are gauges and counters. 
@@ -1244,7 +1244,7 @@ Notes:
     `spark.metrics.staticSources.enabled` (default is true)
   - This source is available for driver and executor instances and is also available for other instances.  
   - This source provides information on JVM metrics using the 
-  [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/3.1.0/manual/jvm/)
+  [Dropwizard/Codahale Metric Sets for JVM instrumentation](https://metrics.dropwizard.io/4.1.1/manual/jvm.html)
    and in particular the metric sets BufferPoolMetricSet, GarbageCollectorMetricSet and MemoryUsageGaugeSet. 
 
 ### Component instance = applicationMaster
diff --git a/pom.xml b/pom.xml
index e9ae204341c56..1bf5de07024c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,6 +145,10 @@
     <chill.version>0.9.5</chill.version>
     <ivy.version>2.4.0</ivy.version>
     <oro.version>2.0.8</oro.version>
+    <!--
+    If you changes codahale.metrics.version, you also need to change
+    the link to metrics.dropwizard.io in docs/monitoring.md.
+    -->
     <codahale.metrics.version>4.1.1</codahale.metrics.version>
     <avro.version>1.8.2</avro.version>
     <avro.mapred.classifier>hadoop2</avro.mapred.classifier>

From ee1237483eca1c2f2d9630fce91e2394049ce3a4 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Mon, 17 Aug 2020 13:46:41 +0000
Subject: [PATCH 0975/1280] [3.0][SQL] Revert SPARK-32018

### What changes were proposed in this pull request?

Revert SPARK-32018 related changes in branch 3.0: https://github.com/apache/spark/pull/29125 and  https://github.com/apache/spark/pull/29404

### Why are the changes needed?

https://github.com/apache/spark/pull/29404 is made to fix correctness regression introduced by https://github.com/apache/spark/pull/29125. However, the behavior of decimal overflow is strange in non-ansi mode:
1. from 3.0.0 to 3.0.1: decimal overflow will throw exceptions instead of returning null on decimal overflow
2. from 3.0.1 to 3.1.0: decimal overflow will return null instead of throwing exceptions.

So, this PR proposes to revert both https://github.com/apache/spark/pull/29404 and https://github.com/apache/spark/pull/29125. So that Spark will return null on decimal overflow in Spark 3.0.0 and Spark 3.0.1.

### Does this PR introduce _any_ user-facing change?

Yes, Spark will return null on decimal overflow in Spark 3.0.1.

### How was this patch tested?

Unit tests

Closes #29450 from gengliangwang/revertDecimalOverflow.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/expressions/UnsafeRow.java   |  2 +-
 .../catalyst/expressions/aggregate/Sum.scala  | 19 ++--------
 .../spark/sql/DataFrameAggregateSuite.scala   | 37 -------------------
 .../org/apache/spark/sql/UnsafeRowSuite.scala | 10 -----
 4 files changed, 4 insertions(+), 64 deletions(-)

diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
index 4dc5ce1de047b..034894bd86085 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeRow.java
@@ -288,7 +288,7 @@ public void setDecimal(int ordinal, Decimal value, int precision) {
       Platform.putLong(baseObject, baseOffset + cursor, 0L);
       Platform.putLong(baseObject, baseOffset + cursor + 8, 0L);
 
-      if (value == null || !value.changePrecision(precision, value.scale())) {
+      if (value == null) {
         setNullAt(ordinal);
         // keep the offset for future update
         Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32);
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
index d442549f20e80..d2daaac72fc85 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala
@@ -71,36 +71,23 @@ case class Sum(child: Expression) extends DeclarativeAggregate with ImplicitCast
   )
 
   override lazy val updateExpressions: Seq[Expression] = {
-    val sumWithChild = resultType match {
-      case d: DecimalType =>
-        CheckOverflow(coalesce(sum, zero) + child.cast(sumDataType), d, nullOnOverflow = false)
-      case _ =>
-        coalesce(sum, zero) + child.cast(sumDataType)
-    }
-
     if (child.nullable) {
       Seq(
         /* sum = */
-        coalesce(sumWithChild, sum)
+        coalesce(coalesce(sum, zero) + child.cast(sumDataType), sum)
       )
     } else {
       Seq(
         /* sum = */
-        sumWithChild
+        coalesce(sum, zero) + child.cast(sumDataType)
       )
     }
   }
 
   override lazy val mergeExpressions: Seq[Expression] = {
-    val sumWithRight = resultType match {
-      case d: DecimalType =>
-        CheckOverflow(coalesce(sum.left, zero) + sum.right, d, nullOnOverflow = false)
-
-      case _ => coalesce(sum.left, zero) + sum.right
-    }
     Seq(
       /* sum = */
-      coalesce(sumWithRight, sum.left)
+      coalesce(coalesce(sum.left, zero) + sum.right, sum.left)
     )
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 8c0358e205b07..54327b38c100b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -21,7 +21,6 @@ import scala.util.Random
 
 import org.scalatest.Matchers.the
 
-import org.apache.spark.SparkException
 import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
@@ -1045,42 +1044,6 @@ class DataFrameAggregateSuite extends QueryTest
     checkAnswer(sql(queryTemplate("FIRST")), Row(1))
     checkAnswer(sql(queryTemplate("LAST")), Row(3))
   }
-
-  private def exceptionOnDecimalOverflow(df: DataFrame): Unit = {
-    val msg = intercept[SparkException] {
-      df.collect()
-    }.getCause.getMessage
-    assert(msg.contains("cannot be represented as Decimal(38, 18)"))
-  }
-
-  test("SPARK-32018: Throw exception on decimal overflow at partial aggregate phase") {
-    val decimalString = "1" + "0" * 19
-    val union = spark.range(0, 1, 1, 1).union(spark.range(0, 11, 1, 1))
-    val hashAgg = union
-      .select(expr(s"cast('$decimalString' as decimal (38, 18)) as d"), lit("1").as("key"))
-      .groupBy("key")
-      .agg(sum($"d").alias("sumD"))
-      .select($"sumD")
-    exceptionOnDecimalOverflow(hashAgg)
-
-    val sortAgg = union
-      .select(expr(s"cast('$decimalString' as decimal (38, 18)) as d"), lit("a").as("str"),
-      lit("1").as("key")).groupBy("key")
-      .agg(sum($"d").alias("sumD"), min($"str").alias("minStr")).select($"sumD", $"minStr")
-    exceptionOnDecimalOverflow(sortAgg)
-  }
-
-  test("SPARK-32018: Throw exception on decimal overflow at merge aggregation phase") {
-    val decimalString = "5" + "0" * 19
-    val union = spark.range(0, 1, 1, 1).union(spark.range(0, 1, 1, 1))
-      .union(spark.range(0, 1, 1, 1))
-    val agg = union
-      .select(expr(s"cast('$decimalString' as decimal (38, 18)) as d"), lit("1").as("key"))
-      .groupBy("key")
-      .agg(sum($"d").alias("sumD"))
-      .select($"sumD")
-    exceptionOnDecimalOverflow(agg)
-  }
 }
 
 case class B(c: Option[Double])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
index 9daa69ce9f155..a5f904c621e6e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala
@@ -178,14 +178,4 @@ class UnsafeRowSuite extends SparkFunSuite {
     // Makes sure hashCode on unsafe array won't crash
     unsafeRow.getArray(0).hashCode()
   }
-
-  test("SPARK-32018: setDecimal with overflowed value") {
-    val d1 = new Decimal().set(BigDecimal("10000000000000000000")).toPrecision(38, 18)
-    val row = InternalRow.apply(d1)
-    val unsafeRow = UnsafeProjection.create(Array[DataType](DecimalType(38, 18))).apply(row)
-    assert(unsafeRow.getDecimal(0, 38, 18) === d1)
-    val d2 = (d1 * Decimal(10)).toPrecision(39, 18)
-    unsafeRow.setDecimal(0, d2, 38)
-    assert(unsafeRow.getDecimal(0, 38, 18) === null)
-  }
 }

From 6cdc32fb33cf717dbc6ad6e674f8c942535683ba Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Mon, 17 Aug 2020 13:19:49 -0700
Subject: [PATCH 0976/1280] [SPARK-32622][SQL][TEST] Add case-sensitivity test
 for ORC predicate pushdown

### What changes were proposed in this pull request?

During working on SPARK-25557, we found that ORC predicate pushdown doesn't have case-sensitivity test. This PR proposes to add case-sensitivity test for ORC predicate pushdown.

### Why are the changes needed?

Increasing test coverage for ORC predicate pushdown.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Pass Jenkins tests.

Closes #29427 from viirya/SPARK-25557-followup3.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b33066f42bd474f5f80b14221f97d09a76e0b398)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../datasources/orc/OrcFilterSuite.scala      | 97 ++++++++++++++++++-
 .../datasources/orc/OrcFilterSuite.scala      | 97 ++++++++++++++++++-
 2 files changed, 190 insertions(+), 4 deletions(-)

diff --git a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index 88b4b243b543a..beb72322326fe 100644
--- a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -25,8 +25,8 @@ import scala.collection.JavaConverters._
 
 import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
 
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, Column, DataFrame}
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
@@ -469,5 +469,98 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
       ).get.toString
     }
   }
+
+  test("SPARK-32622: case sensitivity in predicate pushdown") {
+    withTempPath { dir =>
+      val count = 10
+      val tableName = "spark_32622"
+      val tableDir1 = dir.getAbsoluteFile + "/table1"
+
+      // Physical ORC files have both `A` and `a` fields.
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A", "id as a")
+          .write.mode("overwrite").orc(tableDir1)
+      }
+
+      // Metastore table has both `A` and `a` fields too.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG, a LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a, A from $tableName"), (0 until count).map(c => Row(c, c - 1)))
+
+          val actual1 = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual1.count() == 1)
+
+          val actual2 = stripSparkFilter(sql(s"select A from $tableName where a < 0"))
+          assert(actual2.count() == 0)
+        }
+
+        // Exception thrown for ambiguous case.
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          val e = intercept[AnalysisException] {
+            sql(s"select a from $tableName where a < 0").collect()
+          }
+          assert(e.getMessage.contains(
+            "Reference 'a' is ambiguous"))
+        }
+      }
+
+      // Metastore table has only `A` field.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          val e = intercept[SparkException] {
+            sql(s"select A from $tableName where A < 0").collect()
+          }
+          assert(e.getCause.isInstanceOf[RuntimeException] && e.getCause.getMessage.contains(
+            """Found duplicate field(s) "A": [A, a] in case-insensitive mode"""))
+        }
+      }
+
+      // Physical ORC files have only `A` field.
+      val tableDir2 = dir.getAbsoluteFile + "/table2"
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A")
+          .write.mode("overwrite").orc(tableDir2)
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (a LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
+          // TODO: ORC predicate pushdown should work under case-insensitive analysis.
+          // assert(actual.count() == 1)
+        }
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select A from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+    }
+  }
 }
 
diff --git a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index 2263179515a5f..a3e450c5cd9f5 100644
--- a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -25,8 +25,8 @@ import scala.collection.JavaConverters._
 
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
 
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.{AnalysisException, Column, DataFrame}
+import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.planning.PhysicalOperation
@@ -470,5 +470,98 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
       ).get.toString
     }
   }
+
+  test("SPARK-32622: case sensitivity in predicate pushdown") {
+    withTempPath { dir =>
+      val count = 10
+      val tableName = "spark_32622"
+      val tableDir1 = dir.getAbsoluteFile + "/table1"
+
+      // Physical ORC files have both `A` and `a` fields.
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A", "id as a")
+          .write.mode("overwrite").orc(tableDir1)
+      }
+
+      // Metastore table has both `A` and `a` fields too.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG, a LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a, A from $tableName"), (0 until count).map(c => Row(c, c - 1)))
+
+          val actual1 = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual1.count() == 1)
+
+          val actual2 = stripSparkFilter(sql(s"select A from $tableName where a < 0"))
+          assert(actual2.count() == 0)
+        }
+
+        // Exception thrown for ambiguous case.
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          val e = intercept[AnalysisException] {
+            sql(s"select a from $tableName where a < 0").collect()
+          }
+          assert(e.getMessage.contains(
+            "Reference 'a' is ambiguous"))
+        }
+      }
+
+      // Metastore table has only `A` field.
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir1'
+             """.stripMargin)
+
+          val e = intercept[SparkException] {
+            sql(s"select A from $tableName where A < 0").collect()
+          }
+          assert(e.getCause.isInstanceOf[RuntimeException] && e.getCause.getMessage.contains(
+            """Found duplicate field(s) "A": [A, a] in case-insensitive mode"""))
+        }
+      }
+
+      // Physical ORC files have only `A` field.
+      val tableDir2 = dir.getAbsoluteFile + "/table2"
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+        spark.range(count).repartition(count).selectExpr("id - 1 as A")
+          .write.mode("overwrite").orc(tableDir2)
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (a LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
+          // TODO: ORC predicate pushdown should work under case-insensitive analysis.
+          // assert(actual.count() == 1)
+        }
+      }
+
+      withTable(tableName) {
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
+          sql(
+            s"""
+               |CREATE TABLE $tableName (A LONG) USING ORC LOCATION '$tableDir2'
+             """.stripMargin)
+
+          checkAnswer(sql(s"select A from $tableName"), (0 until count).map(c => Row(c - 1)))
+
+          val actual = stripSparkFilter(sql(s"select A from $tableName where A < 0"))
+          assert(actual.count() == 1)
+        }
+      }
+    }
+  }
 }
 

From a36514e1fbf6920f0d18a7d8fb690822843eb2a2 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Tue, 18 Aug 2020 06:50:05 +0000
Subject: [PATCH 0977/1280] [3.0][SPARK-32518][CORE]
 CoarseGrainedSchedulerBackend.maxNumConcurrentTasks should consider all kinds
 of resources

### What changes were proposed in this pull request?

1.  Make `CoarseGrainedSchedulerBackend.maxNumConcurrentTasks()` considers all kinds of resources when calculating the max concurrent tasks

2. Refactor `calculateAvailableSlots()` to make it be able to be used for both `CoarseGrainedSchedulerBackend` and `TaskSchedulerImpl`

### Why are the changes needed?

Currently, `CoarseGrainedSchedulerBackend.maxNumConcurrentTasks()` only considers the CPU for the max concurrent tasks. This can cause the application to hang when a barrier stage requires extra custom resources but the cluster doesn't have enough corresponding resources. Because, without the checking for other custom resources in `maxNumConcurrentTasks`, the barrier stage can be submitted to the `TaskSchedulerImpl`. But the `TaskSchedulerImpl` won't launch tasks for the barrier stage due to the insufficient task slots calculated by `TaskSchedulerImpl.calculateAvailableSlots` (which does check all kinds of resources).

If the barrier stage doesn't launch all the tasks in one true, the application will fail and suggest user to disable delay scheduling. However, this actually a misleading suggestion since the real root cause is not enough resources.

### Does this PR introduce _any_ user-facing change?

Yes. In case of a barrier stage requires more custom resources than the cluster has, previously, the application will fail with misleading suggestion of disabling delay scheduling. After this PR, the application will fail with the error message saying not enough resources.

### How was this patch tested?

Added a unit test.

Closes #29395 from Ngone51/backport-spark-32518.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../scala/org/apache/spark/SparkContext.scala | 10 +++---
 .../apache/spark/internal/config/Tests.scala  | 15 ++++++++
 .../BarrierJobAllocationFailed.scala          |  4 +--
 .../apache/spark/scheduler/DAGScheduler.scala | 10 +++---
 .../scheduler/ExecutorResourceInfo.scala      |  1 +
 .../spark/scheduler/SchedulerBackend.scala    |  3 +-
 .../spark/scheduler/TaskSchedulerImpl.scala   | 36 ++++++++++++++++++-
 .../CoarseGrainedSchedulerBackend.scala       | 22 ++++++++++--
 .../spark/BarrierStageOnSubmittedSuite.scala  | 36 +++++++++++++++++++
 .../org/apache/spark/SparkContextSuite.scala  |  2 ++
 10 files changed, 124 insertions(+), 15 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 65c08cff2ba11..66fe1d7f2ba13 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -1597,7 +1597,8 @@ class SparkContext(config: SparkConf) extends Logging {
   }
 
   /**
-   * Get the max number of tasks that can be concurrent launched currently.
+   * Get the max number of tasks that can be concurrent launched based on the resources
+   * could be used, even if some of them are being used at the moment.
    * Note that please don't cache the value returned by this method, because the number can change
    * due to add/remove executors.
    *
@@ -2776,8 +2777,9 @@ object SparkContext extends Logging {
       }
       // some cluster managers don't set the EXECUTOR_CORES config by default (standalone
       // and mesos coarse grained), so we can't rely on that config for those.
-      val shouldCheckExecCores = executorCores.isDefined || sc.conf.contains(EXECUTOR_CORES) ||
+      var shouldCheckExecCores = executorCores.isDefined || sc.conf.contains(EXECUTOR_CORES) ||
         (master.equalsIgnoreCase("yarn") || master.startsWith("k8s"))
+      shouldCheckExecCores &= !sc.conf.get(SKIP_VALIDATE_CORES_TESTING)
 
       // Number of cores per executor must meet at least one task requirement.
       if (shouldCheckExecCores && execCores < taskCores) {
@@ -2833,7 +2835,7 @@ object SparkContext extends Logging {
           limitingResourceName = taskReq.resourceName
         }
       }
-      if(!shouldCheckExecCores && Utils.isDynamicAllocationEnabled(sc.conf)) {
+      if(!shouldCheckExecCores) {
         // if we can't rely on the executor cores config throw a warning for user
         logWarning("Please ensure that the number of slots available on your " +
           "executors is limited by the number of cores to task cpus and not another " +
@@ -2857,7 +2859,7 @@ object SparkContext extends Logging {
             s"result in wasted resources due to resource ${limitingResourceName} limiting the " +
             s"number of runnable tasks per executor to: ${numSlots}. Please adjust " +
             s"your configuration."
-          if (Utils.isTesting) {
+          if (sc.conf.get(RESOURCES_WARNING_TESTING)) {
             throw new SparkException(message)
           } else {
             logWarning(message)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
index 232264d6c8e49..e328ed026bfb0 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala
@@ -61,4 +61,19 @@ private[spark] object Tests {
     .version("3.0.0")
     .intConf
     .createWithDefault(2)
+
+  val RESOURCES_WARNING_TESTING = ConfigBuilder("spark.resources.warnings.testing")
+    .version("3.0.1")
+    .booleanConf
+    .createWithDefault(false)
+
+  // This configuration is used for unit tests to allow skipping the task cpus to cores validation
+  // to allow emulating standalone mode behavior while running in local mode. Standalone mode
+  // by default doesn't specify a number of executor cores, it just uses all the ones available
+  // on the host.
+  val SKIP_VALIDATE_CORES_TESTING =
+  ConfigBuilder("spark.testing.skipValidateCores")
+    .version("3.0.1")
+    .booleanConf
+    .createWithDefault(false)
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
index 2274e6898adf6..043c6b90384b4 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BarrierJobAllocationFailed.scala
@@ -60,6 +60,6 @@ private[spark] object BarrierJobAllocationFailed {
   val ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER =
     "[SPARK-24819]: Barrier execution mode does not allow run a barrier stage that requires " +
       "more slots than the total number of slots in the cluster currently. Please init a new " +
-      "cluster with more CPU cores or repartition the input RDD(s) to reduce the number of " +
-      "slots required to run this barrier stage."
+      "cluster with more resources(e.g. CPU, GPU) or repartition the input RDD(s) to reduce " +
+      "the number of slots required to run this barrier stage."
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index 51445bf451bec..b483b52662270 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -453,10 +453,12 @@ private[spark] class DAGScheduler(
    * submission.
    */
   private def checkBarrierStageWithNumSlots(rdd: RDD[_]): Unit = {
-    val numPartitions = rdd.getNumPartitions
-    val maxNumConcurrentTasks = sc.maxNumConcurrentTasks
-    if (rdd.isBarrier() && numPartitions > maxNumConcurrentTasks) {
-      throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks)
+    if (rdd.isBarrier()) {
+      val numPartitions = rdd.getNumPartitions
+      val maxNumConcurrentTasks = sc.maxNumConcurrentTasks
+      if (numPartitions > maxNumConcurrentTasks) {
+        throw new BarrierJobSlotsNumberCheckFailed(numPartitions, maxNumConcurrentTasks)
+      }
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
index fd04db8c09d76..508c6cebd9fe3 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ExecutorResourceInfo.scala
@@ -36,4 +36,5 @@ private[spark] class ExecutorResourceInfo(
   override protected def resourceName = this.name
   override protected def resourceAddresses = this.addresses
   override protected def slotsPerAddress: Int = numParts
+  def totalAddressAmount: Int = resourceAddresses.length * slotsPerAddress
 }
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
index 9159d2a0158d5..7b76af2f489f9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala
@@ -77,7 +77,8 @@ private[spark] trait SchedulerBackend {
   def getDriverAttributes: Option[Map[String, String]] = None
 
   /**
-   * Get the max number of tasks that can be concurrent launched currently.
+   * Get the max number of tasks that can be concurrent launched based on the resources
+   * could be used, even if some of them are being used at the moment.
    * Note that please don't cache the value returned by this method, because the number can change
    * due to add/remove executors.
    *
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 97125f6238b22..46641e5bf5580 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -446,7 +446,17 @@ private[spark] class TaskSchedulerImpl(
     // of locality levels so that it gets a chance to launch local tasks on all of them.
     // NOTE: the preferredLocality order: PROCESS_LOCAL, NODE_LOCAL, NO_PREF, RACK_LOCAL, ANY
     for (taskSet <- sortedTaskSets) {
-      val availableSlots = availableCpus.map(c => c / CPUS_PER_TASK).sum
+      // we only need to calculate available slots if using barrier scheduling, otherwise the
+      // value is -1
+      val availableSlots = if (taskSet.isBarrier) {
+        val availableResourcesAmount = availableResources.map { resourceMap =>
+          // note that the addresses here have been expanded according to the numParts
+          resourceMap.map { case (name, addresses) => (name, addresses.length) }
+        }
+        calculateAvailableSlots(this, availableCpus, availableResourcesAmount)
+      } else {
+        -1
+      }
       // Skip the barrier taskSet if the available slots are less than the number of pending tasks.
       if (taskSet.isBarrier && availableSlots < taskSet.numTasks) {
         // Skip the launch process.
@@ -933,6 +943,30 @@ private[spark] object TaskSchedulerImpl {
 
   val SCHEDULER_MODE_PROPERTY = SCHEDULER_MODE.key
 
+  /**
+   * Calculate the max available task slots given the `availableCpus` and `availableResources`
+   * from a collection of executors.
+   *
+   * @param scheduler the TaskSchedulerImpl instance
+   * @param availableCpus an Array of the amount of available cpus from the executors.
+   * @param availableResources an Array of the resources map from the executors. In the resource
+   *                           map, it maps from the resource name to its amount.
+   * @return the number of max task slots
+   */
+  def calculateAvailableSlots(
+      scheduler: TaskSchedulerImpl,
+      availableCpus: Array[Int],
+      availableResources: Array[Map[String, Int]]): Int = {
+    val cpusPerTask = scheduler.CPUS_PER_TASK
+    val resourcesReqsPerTask = scheduler.resourcesReqsPerTask
+    availableCpus.zip(availableResources).map { case (cpu, resources) =>
+      val cpuNum = cpu / cpusPerTask
+      resourcesReqsPerTask.map { req =>
+        resources.get(req.resourceName).map(_ / req.amount).getOrElse(0)
+      }.reduceOption(Math.min).map(_.min(cpuNum)).getOrElse(cpuNum)
+    }.sum
+  }
+
   /**
    * Used to balance containers across hosts.
    *
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 9d8fb8f83856d..8b55e2c7dbee5 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -563,10 +563,26 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
       !executorsPendingLossReason.contains(id)
   }
 
+  /**
+   * Get the max number of tasks that can be concurrent launched based on the resources
+   * could be used, even if some of them are being used at the moment.
+   * Note that please don't cache the value returned by this method, because the number can change
+   * due to add/remove executors.
+   *
+   * @return The max number of tasks that can be concurrent launched currently.
+   */
   override def maxNumConcurrentTasks(): Int = synchronized {
-    executorDataMap.values.map { executor =>
-      executor.totalCores / scheduler.CPUS_PER_TASK
-    }.sum
+    val (cpus, resources) = {
+      executorDataMap
+        .filter { case (id, _) => isExecutorActive(id) }
+        .values.toArray.map { executor =>
+        (
+          executor.totalCores,
+          executor.resourcesInfo.map { case (name, rInfo) => (name, rInfo.totalAddressAmount) }
+        )
+      }.unzip
+    }
+    TaskSchedulerImpl.calculateAvailableSlots(scheduler, cpus, resources)
   }
 
   // this function is for testing only
diff --git a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
index 435b927068e60..7052d1a2028bb 100644
--- a/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
+++ b/core/src/test/scala/org/apache/spark/BarrierStageOnSubmittedSuite.scala
@@ -19,9 +19,12 @@ package org.apache.spark
 
 import scala.concurrent.duration._
 
+import org.apache.spark.TestUtils.createTempScriptWithExpectedOutput
 import org.apache.spark.internal.config._
 import org.apache.spark.rdd.{PartitionPruningRDD, RDD}
+import org.apache.spark.resource.TestResourceIDs.{EXECUTOR_GPU_ID, TASK_GPU_ID, WORKER_GPU_ID}
 import org.apache.spark.scheduler.BarrierJobAllocationFailed._
+import org.apache.spark.scheduler.BarrierJobSlotsNumberCheckFailed
 import org.apache.spark.util.ThreadUtils
 
 /**
@@ -259,4 +262,37 @@ class BarrierStageOnSubmittedSuite extends SparkFunSuite with LocalSparkContext
     testSubmitJob(sc, rdd,
       message = ERROR_MESSAGE_BARRIER_REQUIRE_MORE_SLOTS_THAN_CURRENT_TOTAL_NUMBER)
   }
+
+  test("SPARK-32518: CoarseGrainedSchedulerBackend.maxNumConcurrentTasks should " +
+    "consider all kinds of resources for the barrier stage") {
+    withTempDir { dir =>
+      val discoveryScript = createTempScriptWithExpectedOutput(
+        dir, "gpuDiscoveryScript", """{"name": "gpu","addresses":["0"]}""")
+
+      val conf = new SparkConf()
+        .setMaster("local-cluster[1, 2, 1024]")
+        .setAppName("test-cluster")
+        .set(WORKER_GPU_ID.amountConf, "1")
+        .set(WORKER_GPU_ID.discoveryScriptConf, discoveryScript)
+        .set(EXECUTOR_GPU_ID.amountConf, "1")
+        .set(TASK_GPU_ID.amountConf, "1")
+        // disable barrier stage retry to fail the application as soon as possible
+        .set(BARRIER_MAX_CONCURRENT_TASKS_CHECK_MAX_FAILURES, 1)
+        // disable the check to simulate the behavior of Standalone in order to
+        // reproduce the issue.
+        .set(Tests.SKIP_VALIDATE_CORES_TESTING, true)
+      sc = new SparkContext(conf)
+      // setup an executor which will have 2 CPUs and 1 GPU
+      TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+
+      val exception = intercept[BarrierJobSlotsNumberCheckFailed] {
+        sc.parallelize(Range(1, 10), 2)
+          .barrier()
+          .mapPartitions { iter => iter }
+          .collect()
+      }
+      assert(exception.getMessage.contains("[SPARK-24819]: Barrier execution " +
+        "mode does not allow run a barrier stage that requires more slots"))
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index ce437a5e42b22..dc1c0451c628d 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -36,6 +36,7 @@ import org.scalatest.concurrent.Eventually
 
 import org.apache.spark.TestUtils._
 import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.Tests.RESOURCES_WARNING_TESTING
 import org.apache.spark.internal.config.UI._
 import org.apache.spark.resource.ResourceAllocation
 import org.apache.spark.resource.ResourceUtils._
@@ -890,6 +891,7 @@ class SparkContextSuite extends SparkFunSuite with LocalSparkContext with Eventu
       .setAppName("test-cluster")
     conf.set(TASK_GPU_ID.amountConf, "2")
     conf.set(EXECUTOR_GPU_ID.amountConf, "4")
+    conf.set(RESOURCES_WARNING_TESTING, true)
 
     var error = intercept[SparkException] {
       sc = new SparkContext(conf)

From 753d414386b3d0fdca831792d671dafe116c83d6 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 19 Aug 2020 13:50:29 +0900
Subject: [PATCH 0978/1280] [SPARK-32652][SQL] ObjectSerializerPruning fails
 for RowEncoder

### What changes were proposed in this pull request?

Update `ObjectSerializerPruning.alignNullTypeInIf`, to consider the isNull check generated in `RowEncoder`, which is `Invoke(inputObject, "isNullAt", BooleanType, Literal(index) :: Nil)`.

### Why are the changes needed?

Query fails if we don't fix this bug, due to type mismatch in `If`.

### Does this PR introduce _any_ user-facing change?

Yes, the failed query can run after this fix.

### How was this patch tested?

new tests

Closes #29467 from cloud-fan/bug.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f33b64a6567f93e5515521b8b1e7761e16f667d0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/optimizer/objects.scala      | 10 +++++-
 .../ObjectSerializerPruningSuite.scala        | 33 ++++++++++++++++++-
 .../spark/sql/DatasetOptimizationSuite.scala  | 13 ++++++--
 3 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
index 0fa29e87e5f39..9643f5827b910 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/objects.scala
@@ -162,10 +162,18 @@ object ObjectSerializerPruning extends Rule[LogicalPlan] {
    * Note: we should do `transformUp` explicitly to change data types.
    */
   private def alignNullTypeInIf(expr: Expression) = expr.transformUp {
-    case i @ If(_: IsNull, Literal(null, dt), ser) if !dt.sameType(ser.dataType) =>
+    case i @ If(IsNullCondition(), Literal(null, dt), ser) if !dt.sameType(ser.dataType) =>
       i.copy(trueValue = Literal(null, ser.dataType))
   }
 
+  object IsNullCondition {
+    def unapply(expr: Expression): Boolean = expr match {
+      case _: IsNull => true
+      case i: Invoke if i.functionName == "isNullAt" => true
+      case _ => false
+    }
+  }
+
   /**
    * This method prunes given serializer expression by given pruned data type. For example,
    * given a serializer creating struct(a int, b int) and pruned data type struct(a int),
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
index 0dd4d6a245f5c..6d7c4c3c7e9d2 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ObjectSerializerPruningSuite.scala
@@ -22,8 +22,9 @@ import scala.reflect.runtime.universe.TypeTag
 
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
 import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.objects.Invoke
 import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -107,4 +108,34 @@ class ObjectSerializerPruningSuite extends PlanTest {
       comparePlans(optimized, expected)
     }
   }
+
+  test("SPARK-32652: Prune nested serializers: RowEncoder") {
+    withSQLConf(SQLConf.SERIALIZER_NESTED_SCHEMA_PRUNING_ENABLED.key -> "true") {
+      val testRelation = LocalRelation('i.struct(StructType.fromDDL("a int, b string")), 'j.int)
+      val rowEncoder = RowEncoder(new StructType()
+        .add("i", new StructType().add("a", "int").add("b", "string"))
+        .add("j", "int"))
+      val serializerObject = CatalystSerde.serialize(
+        CatalystSerde.deserialize(testRelation)(rowEncoder))(rowEncoder)
+      val query = serializerObject.select($"i.a")
+      val optimized = Optimize.execute(query.analyze)
+
+      val prunedSerializer = serializerObject.serializer.head.transformDown {
+        case CreateNamedStruct(children) => CreateNamedStruct(children.take(2))
+      }.transformUp {
+        // Aligns null literal in `If` expression to make it resolvable.
+        case i @ If(invoke: Invoke, Literal(null, dt), ser) if invoke.functionName == "isNullAt" &&
+            !dt.sameType(ser.dataType) =>
+          i.copy(trueValue = Literal(null, ser.dataType))
+      }.asInstanceOf[NamedExpression]
+
+      // `name` in `GetStructField` affects `comparePlans`. Maybe we can ignore
+      // `name` in `GetStructField.equals`?
+      val expected = serializerObject.copy(serializer = Seq(prunedSerializer))
+        .select($"i.a").analyze.transformAllExpressions {
+        case g: GetStructField => g.copy(name = None)
+      }
+      comparePlans(optimized, expected)
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
index 0ac99905f35f4..5b8c80b471bb4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
@@ -18,12 +18,13 @@
 package org.apache.spark.sql
 
 import org.apache.spark.metrics.source.CodegenMetrics
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
 import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression}
-import org.apache.spark.sql.catalyst.expressions.objects.ExternalMapToCatalyst
 import org.apache.spark.sql.catalyst.plans.logical.SerializeFromObject
-import org.apache.spark.sql.functions.expr
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.StructType
 
 class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -195,4 +196,12 @@ class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
       checkCodegenCache(() => Seq(Seq(Map("abc" -> 1))).toDS())
     }
   }
+
+  test("SPARK-32652: Pruned nested serializers: RowEncoder") {
+    val df = Seq(("a", 1), ("b", 2), ("c", 3)).toDF("i", "j")
+    val encoder = RowEncoder(new StructType().add("s", df.schema))
+    val query = df.map(row => Row(row))(encoder).select("s.i")
+    testSerializer(query, Seq(Seq("i")))
+    checkAnswer(query, Seq(Row("a"), Row("b"), Row("c")))
+  }
 }

From b3a971a5b88ed1811c693bcc05fd7f2b4ade7773 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 18 Aug 2020 19:35:15 +0900
Subject: [PATCH 0979/1280] [SPARK-32647][INFRA] Report SparkR test results
 with JUnit reporter

This PR proposes to generate JUnit XML test report in SparkR tests that can be leveraged in both Jenkins and GitHub Actions.

**GitHub Actions**

![Screen Shot 2020-08-18 at 12 42 46 PM](https://user-images.githubusercontent.com/6477701/90467934-55b85b00-e150-11ea-863c-c8415e764ddb.png)

**Jenkins**

![Screen Shot 2020-08-18 at 2 03 42 PM](https://user-images.githubusercontent.com/6477701/90472509-a5505400-e15b-11ea-9165-777ec9b96eaa.png)

NOTE that while I am here, I am switching back the console reporter from "progress" to "summary". Currently non-ascii codes are broken in Jenkins console and switching it to "summary" can work around it.
"summary" is the default format used in testthat 1.x.

To check the test failures more easily.

No, dev-only

It is tested in GitHub Actions at https://github.com/HyukjinKwon/spark/pull/23/checks?check_run_id=996586446
In case of Jenkins, https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/127525/testReport/

Closes #29456 from HyukjinKwon/sparkr-junit.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/tests/run-all.R | 9 ++++++---
 appveyor.yml          | 4 ++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index bf02ecdad66ff..20e61b2dda3ac 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -61,15 +61,18 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
     set.seed(42)
 
     # TODO (SPARK-30663) To be removed once testthat 1.x is removed from all builds
-    if (grepl("^1\\..*", packageVersion("testthat"))) {
+    if (packageVersion("testthat")$major <= 1) {
       # testthat 1.x
       test_runner <- testthat:::run_tests
       reporter <- "summary"
-
     } else {
       # testthat >= 2.0.0
       test_runner <- testthat:::test_package_dir
-      reporter <- testthat::default_reporter()
+      dir.create("target/test-reports", showWarnings = FALSE)
+      reporter <- MultiReporter$new(list(
+        SummaryReporter$new(),
+        JunitReporter$new(file = "target/test-reports/test-results.xml")
+      ))
     }
 
     test_runner("SparkR",
diff --git a/appveyor.yml b/appveyor.yml
index a4da5f9040ded..e0d2396bcbed4 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -41,8 +41,8 @@ cache:
 install:
   # Install maven and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
-  # Required package for R unit tests
-  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow'), repos='https://cloud.r-project.org/')"
+  # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat.
+  - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')"
   - cmd: R -e "packageVersion('knitr'); packageVersion('rmarkdown'); packageVersion('testthat'); packageVersion('e1071'); packageVersion('survival'); packageVersion('arrow')"
 
 build_script:

From 6dc7457620130a7aa03b14303d1475a64348e399 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Wed, 19 Aug 2020 05:20:26 -0700
Subject: [PATCH 0980/1280] [SPARK-32624][SQL] Use getCanonicalName to fix
 byte[] compile issue

### What changes were proposed in this pull request?
```scala
scala> Array[Byte](1, 2).getClass.getName
res13: String = [B

scala> Array[Byte](1, 2).getClass.getCanonicalName
res14: String = byte[]
```

This pr replace `getClass.getName` with `getClass.getCanonicalName` in `CodegenContext.addReferenceObj` to fix `byte[]` compile issue:
```
...
/* 030 */       value_1 = org.apache.spark.sql.catalyst.util.TypeUtils.compareBinary(value_2, (([B) references[0] /* min */)) >= 0 && org.apache.spark.sql.catalyst.util.TypeUtils.compareBinary(value_2, (([B) references[1] /* max */)) <= 0;
/* 031 */     }
/* 032 */     return !isNull_1 && value_1;
/* 033 */   }
/* 034 */
/* 035 */
/* 036 */ }

20:49:54.886 WARN org.apache.spark.sql.catalyst.expressions.Predicate: Expr codegen error and falling back to interpreter mode
java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 30, Column 81: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 30, Column 81: Unexpected token "[" in primary
...
```

### Why are the changes needed?

Fix compile issue when compiling generated code.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test.

Closes #29439 from wangyum/SPARK-32624.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Yuming Wang <wgyumg@gmail.com>
(cherry picked from commit 409fea30cc40ce24a17325ec63d2f847ce49f5a6)
Signed-off-by: Yuming Wang <wgyumg@gmail.com>
---
 .../sql/catalyst/expressions/codegen/CodeGenerator.scala   | 2 +-
 .../sql/catalyst/expressions/CodeGenerationSuite.scala     | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 58c95c94ba198..c8aa83eb18068 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -133,7 +133,7 @@ class CodegenContext extends Logging {
   def addReferenceObj(objName: String, obj: Any, className: String = null): String = {
     val idx = references.length
     references += obj
-    val clsName = Option(className).getOrElse(obj.getClass.getName)
+    val clsName = Option(className).getOrElse(obj.getClass.getCanonicalName)
     s"(($clsName) references[$idx] /* $objName */)"
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index db52229f860b4..09cc01d804d37 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -536,6 +536,13 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     GenerateUnsafeProjection.generate(exprs, true)
     GenerateMutableProjection.generate(exprs, true)
   }
+
+  test("SPARK-32624: Use getCanonicalName to fix byte[] compile issue") {
+    val ctx = new CodegenContext
+    val bytes = new Array[Byte](3)
+    val byteObj = ctx.addReferenceObj("bytes", bytes)
+    assert(byteObj == "((byte[]) references[0] /* bytes */)")
+  }
 }
 
 case class HugeCodeIntExpression(value: Int) extends Expression {

From 56ec5ddcac8233011c17fc7d120a284707f0f712 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 19 Aug 2020 07:20:26 -0700
Subject: [PATCH 0981/1280] [SPARK-32249][INFRA][3.0] Run Github Actions builds
 in branch-3.0

### What changes were proposed in this pull request?

This PR proposes to backport the following JIRAs:

- SPARK-32245
- SPARK-32292
- SPARK-32252
- SPARK-32316
- SPARK-32408
- SPARK-32303
- SPARK-32363
- SPARK-32419
- SPARK-32422
- SPARK-32491
- SPARK-32493
- SPARK-32496
- SPARK-32497
- SPARK-32357
- SPARK-32606
- SPARK-32605
- SPARK-32248
- SPARK-32645
- Minor renaming d0dfe49#diff-02d9c370a663741451423342d5869b21

in order to enable GitHub Actions in branch-3.0.

### Why are the changes needed?

To be able to run the tests in branch-3.0. Jenkins jobs are unstable.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Build in this PR will test.

Closes #29460 from HyukjinKwon/SPARK-32249.

Lead-authored-by: HyukjinKwon <gurwls223@apache.org>
Co-authored-by: Hyukjin Kwon <gurwls223@apache.org>
Co-authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .github/workflows/build_and_test.yml          | 283 ++++++++++++++++++
 .github/workflows/master.yml                  | 156 ----------
 .github/workflows/test_report.yml             |  24 ++
 .../spark/tags/GitHubActionsUnstableTest.java |  30 ++
 .../org/apache/spark/tags/SlowHiveTest.java   |  30 ++
 dev/run-pip-tests                             |  10 +-
 dev/run-tests.py                              | 185 +++++++++---
 dev/sparktestsupport/modules.py               |  93 ++++--
 project/SparkBuild.scala                      |   9 +
 python/pyspark/sql/tests/test_arrow.py        |   3 +
 python/pyspark/sql/tests/test_types.py        |   3 +-
 .../pyspark/streaming/tests/test_dstream.py   |  15 +-
 python/run-tests.py                           |   2 +-
 .../spark/sql/IntegratedUDFTestUtils.scala    |   2 +-
 .../ThriftServerQueryTestSuite.scala          |   2 +
 .../ThriftServerWithSparkContextSuite.scala   |   3 +
 .../execution/HiveCompatibilitySuite.scala    |   2 +
 .../HiveExternalCatalogVersionsSuite.scala    |   3 +-
 .../spark/sql/hive/HiveSparkSubmitSuite.scala |   3 +-
 .../spark/sql/hive/client/VersionsSuite.scala |   3 +-
 .../execution/AggregationQuerySuite.scala     |   2 +
 .../sql/hive/execution/HiveDDLSuite.scala     |   3 +
 .../sql/hive/execution/HiveQuerySuite.scala   |   2 +
 .../hive/execution/Hive_2_1_DDLSuite.scala    |   3 +-
 .../sql/hive/execution/SQLQuerySuite.scala    |   3 +
 25 files changed, 639 insertions(+), 235 deletions(-)
 create mode 100644 .github/workflows/build_and_test.yml
 delete mode 100644 .github/workflows/master.yml
 create mode 100644 .github/workflows/test_report.yml
 create mode 100644 common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java
 create mode 100644 common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
new file mode 100644
index 0000000000000..581ad8a66c2b5
--- /dev/null
+++ b/.github/workflows/build_and_test.yml
@@ -0,0 +1,283 @@
+name: Build and test
+
+on:
+  push:
+    branches:
+    - branch-3.0
+  pull_request:
+    branches:
+    - branch-3.0
+
+jobs:
+  # Build: build Spark and run the tests for specified modules.
+  build:
+    name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 1.8
+        hadoop:
+          - hadoop2.7
+        hive:
+          - hive2.3
+        # TODO(SPARK-32246): We don't test 'streaming-kinesis-asl' for now.
+        # Kinesis tests depends on external Amazon kinesis service.
+        # Note that the modules below are from sparktestsupport/modules.py.
+        modules:
+          - >-
+            core, unsafe, kvstore, avro,
+            network-common, network-shuffle, repl, launcher,
+            examples, sketch, graphx
+          - >-
+            catalyst, hive-thriftserver
+          - >-
+            streaming, sql-kafka-0-10, streaming-kafka-0-10,
+            mllib-local, mllib,
+            yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl
+          - >-
+            pyspark-sql, pyspark-mllib
+          - >-
+            pyspark-core, pyspark-streaming, pyspark-ml
+          - >-
+            sparkr
+        # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
+        included-tags: [""]
+        # Some tests are disabled in GitHun Actions. Ideally, we should remove this tag
+        # and run all tests.
+        excluded-tags: ["org.apache.spark.tags.GitHubActionsUnstableTest"]
+        comment: [""]
+        include:
+          # Hive tests
+          - modules: hive
+            java: 1.8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            included-tags: org.apache.spark.tags.SlowHiveTest
+            comment: "- slow tests"
+          - modules: hive
+            java: 1.8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            excluded-tags: org.apache.spark.tags.SlowHiveTest,org.apache.spark.tags.GitHubActionsUnstableTest
+            comment: "- other tests"
+          # SQL tests
+          - modules: sql
+            java: 1.8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            included-tags: org.apache.spark.tags.ExtendedSQLTest
+            comment: "- slow tests"
+          - modules: sql
+            java: 1.8
+            hadoop: hadoop2.7
+            hive: hive2.3
+            excluded-tags: org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.GitHubActionsUnstableTest
+            comment: "- other tests"
+    env:
+      MODULES_TO_TEST: ${{ matrix.modules }}
+      EXCLUDED_TAGS: ${{ matrix.excluded-tags }}
+      INCLUDED_TAGS: ${{ matrix.included-tags }}
+      HADOOP_PROFILE: ${{ matrix.hadoop }}
+      HIVE_PROFILE: ${{ matrix.hive }}
+      # GitHub Actions' default miniconda to use in pip packaging test.
+      CONDA_PREFIX: /usr/share/miniconda
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v1
+      with:
+        path: build
+        key: build-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          build-
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/plugins.sbt') }}
+        restore-keys: |
+          ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-
+    - name: Install JDK ${{ matrix.java }}
+      uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.java }}
+    # PySpark
+    - name: Install PyPy3
+      # Note that order of Python installations here matters because default python3 is
+      # overridden by pypy3.
+      uses: actions/setup-python@v2
+      if: contains(matrix.modules, 'pyspark')
+      with:
+        python-version: pypy3
+        architecture: x64
+    - name: Install Python 2.7
+      uses: actions/setup-python@v2
+      if: contains(matrix.modules, 'pyspark')
+      with:
+        python-version: 2.7
+        architecture: x64
+    - name: Install Python 3.8
+      uses: actions/setup-python@v2
+      # We should install one Python that is higher then 3+ for SQL and Yarn because:
+      # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
+      # - Yarn has a Python specific test too, for example, YarnClusterSuite.
+      if: contains(matrix.modules, 'yarn') || contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      with:
+        python-version: 3.8
+        architecture: x64
+    - name: Install Python packages (Python 2.7 and PyPy3)
+      if: contains(matrix.modules, 'pyspark')
+      # PyArrow is not supported in PyPy yet, see ARROW-2651.
+      # TODO(SPARK-32247): scipy installation with PyPy fails for an unknown reason.
+      run: |
+        python2.7 -m pip install numpy pyarrow pandas scipy xmlrunner
+        python2.7 -m pip list
+        # PyPy does not have xmlrunner
+        pypy3 -m pip install numpy pandas
+        pypy3 -m pip list
+    - name: Install Python packages (Python 3.8)
+      if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      run: |
+        python3.8 -m pip install numpy pyarrow pandas scipy xmlrunner
+        python3.8 -m pip list
+    # SparkR
+    - name: Install R 4.0
+      if: contains(matrix.modules, 'sparkr')
+      run: |
+        sudo sh -c "echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list"
+        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
+        sudo apt-get update
+        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
+    - name: Install R packages
+      if: contains(matrix.modules, 'sparkr')
+      run: |
+        # qpdf is required to reduce the size of PDFs to make CRAN check pass. See SPARK-32497.
+        sudo apt-get install -y libcurl4-openssl-dev qpdf
+        sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')"
+        # Show installed packages in R.
+        sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]'
+    # Run the tests.
+    - name: Run tests
+      run: |
+        # Hive tests become flaky when running in parallel as it's too intensive.
+        if [[ "$MODULES_TO_TEST" == "hive" ]]; then export SERIAL_SBT_TESTS=1; fi
+        mkdir -p ~/.m2
+        ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
+        rm -rf ~/.m2/repository/org/apache/spark
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
+        path: "**/target/unit-tests.log"
+
+  # Static analysis, and documentation build
+  lint:
+    name: Linters, licenses, dependencies and documentation generation
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          docs-maven-
+    - name: Install JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Install Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+        architecture: x64
+    - name: Install Python linter dependencies
+      run: |
+        pip3 install flake8 sphinx numpy
+    - name: Install R 4.0
+      run: |
+        sudo sh -c "echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list"
+        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
+        sudo apt-get update
+        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
+    - name: Install R linter dependencies and SparkR
+      run: |
+        sudo apt-get install -y libcurl4-openssl-dev
+        sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
+        sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
+        ./R/install-dev.sh
+    - name: Install Ruby 2.7 for documentation generation
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Install dependencies for documentation generation
+      run: |
+        sudo apt-get install -y libcurl4-openssl-dev pandoc
+        pip install sphinx mkdocs numpy
+        gem install jekyll jekyll-redirect-from rouge
+        sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
+    - name: Scala linter
+      run: ./dev/lint-scala
+    - name: Java linter
+      run: ./dev/lint-java
+    - name: Python linter
+      run: ./dev/lint-python
+    - name: R linter
+      run: ./dev/lint-r
+    - name: License test
+      run: ./dev/check-license
+    - name: Dependencies test
+      run: ./dev/test-dependencies.sh
+    - name: Run documentation build
+      run: |
+        cd docs
+        jekyll build
+
+  java11:
+    name: Java 11 build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: java11-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          java11-maven-
+    - name: Install Java 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Build with Maven
+      run: |
+        export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
+        export MAVEN_CLI_OPTS="--no-transfer-progress"
+        mkdir -p ~/.m2
+        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install
+        rm -rf ~/.m2/repository/org/apache/spark
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
deleted file mode 100644
index 36f9e33f9bbb4..0000000000000
--- a/.github/workflows/master.yml
+++ /dev/null
@@ -1,156 +0,0 @@
-name: master
-
-on:
-  push:
-    branches:
-    - branch-3.0
-  pull_request:
-    branches:
-    - branch-3.0
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java: [ '1.8', '11' ]
-        hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
-        hive: [ 'hive-1.2', 'hive-2.3' ]
-        exclude:
-        - java: '11'
-          hive: 'hive-1.2'
-        - hadoop: 'hadoop-3.2'
-          hive: 'hive-1.2'
-    name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
-
-    steps:
-    - uses: actions/checkout@master
-    # We split caches because GitHub Action Cache has a 400MB-size limit.
-    - uses: actions/cache@v1
-      with:
-        path: build
-        key: build-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          build-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/com
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/org
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/net
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository/io
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
-    - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v1
-      with:
-        java-version: ${{ matrix.java }}
-    - name: Build with Maven
-      run: |
-        export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
-        export MAVEN_CLI_OPTS="--no-transfer-progress"
-        mkdir -p ~/.m2
-        ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
-        rm -rf ~/.m2/repository/org/apache/spark
-
-
-  lint:
-    runs-on: ubuntu-latest
-    name: Linters (Java/Scala/Python), licenses, dependencies
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '11'
-    - uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-        architecture: 'x64'
-    - name: Scala
-      run: ./dev/lint-scala
-    - name: Java
-      run: ./dev/lint-java
-    - name: Python
-      run: |
-        pip install flake8 sphinx numpy
-        ./dev/lint-python
-    - name: License
-      run: ./dev/check-license
-    - name: Dependencies
-      run: ./dev/test-dependencies.sh
-
-  lintr:
-    runs-on: ubuntu-latest
-    name: Linter (R)
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '11'
-    - uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: '3.6.2'
-    - name: Install lib
-      run: |
-        sudo apt-get install -y libcurl4-openssl-dev
-    - name: install R packages
-      run: |
-        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
-        sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
-    - name: package and install SparkR
-      run: ./R/install-dev.sh
-    - name: lint-r
-      run: ./dev/lint-r
-
-  docs:
-    runs-on: ubuntu-latest
-    name: Generate documents
-    steps:
-    - uses: actions/checkout@master
-    - uses: actions/cache@v1
-      with:
-        path: ~/.m2/repository
-        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          docs-maven-repo-
-    - uses: actions/setup-java@v1
-      with:
-        java-version: '1.8'
-    - uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-        architecture: 'x64'
-    - uses: actions/setup-ruby@v1
-      with:
-        ruby-version: '2.7'
-    - uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: '3.6.2'
-    - name: Install lib and pandoc
-      run: |
-        sudo apt-get install -y libcurl4-openssl-dev pandoc
-    - name: Install packages
-      run: |
-        pip install sphinx mkdocs numpy
-        gem install jekyll jekyll-redirect-from rouge
-        sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
-    - name: Run jekyll build
-      run: |
-        cd docs
-        jekyll build
diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml
new file mode 100644
index 0000000000000..93cdb86687261
--- /dev/null
+++ b/.github/workflows/test_report.yml
@@ -0,0 +1,24 @@
+name: Report test results
+on:
+  workflow_run:
+    workflows: ["Build and test"]
+    types:
+      - completed
+
+jobs:
+  test_report:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download test results to report
+      uses: dawidd6/action-download-artifact@v2
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        workflow: ${{ github.event.workflow_run.workflow_id }}
+        commit: ${{ github.event.workflow_run.head_commit.id }}
+    - name: Publish test report
+      uses: scacap/action-surefire-report@v1
+      with:
+        check_name: Report test results
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        report_paths: "**/target/test-reports/*.xml"
+        commit: ${{ github.event.workflow_run.head_commit.id }}
diff --git a/common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java b/common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java
new file mode 100644
index 0000000000000..a602656bb22f8
--- /dev/null
+++ b/common/tags/src/test/java/org/apache/spark/tags/GitHubActionsUnstableTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface GitHubActionsUnstableTest { }
diff --git a/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java b/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java
new file mode 100644
index 0000000000000..a7e6f352667d7
--- /dev/null
+++ b/common/tags/src/test/java/org/apache/spark/tags/SlowHiveTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface SlowHiveTest { }
diff --git a/dev/run-pip-tests b/dev/run-pip-tests
index 81e33a6f961bd..b322d3f61b444 100755
--- a/dev/run-pip-tests
+++ b/dev/run-pip-tests
@@ -63,7 +63,7 @@ fi
 PYSPARK_VERSION=$(python3 -c "exec(open('python/pyspark/version.py').read());print(__version__)")
 PYSPARK_DIST="$FWDIR/python/dist/pyspark-$PYSPARK_VERSION.tar.gz"
 # The pip install options we use for all the pip commands
-PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall "
+PIP_OPTIONS="--upgrade --no-cache-dir --force-reinstall"
 # Test both regular user and edit/dev install modes.
 PIP_COMMANDS=("pip install $PIP_OPTIONS $PYSPARK_DIST"
 	      "pip install $PIP_OPTIONS -e python/")
@@ -80,8 +80,12 @@ for python in "${PYTHON_EXECS[@]}"; do
     VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
     rm -rf "$VIRTUALENV_PATH"
     if [ -n "$USE_CONDA" ]; then
+      if [ -f "$CONDA_PREFIX/etc/profile.d/conda.sh" ]; then
+        # See also https://github.com/conda/conda/issues/7980
+        source "$CONDA_PREFIX/etc/profile.d/conda.sh"
+      fi
       conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
-      source activate "$VIRTUALENV_PATH"
+      source activate "$VIRTUALENV_PATH" || conda activate "$VIRTUALENV_PATH"
     else
       mkdir -p "$VIRTUALENV_PATH"
       virtualenv --python=$python "$VIRTUALENV_PATH"
@@ -124,7 +128,7 @@ for python in "${PYTHON_EXECS[@]}"; do
 
     # conda / virtualenv environments need to be deactivated differently
     if [ -n "$USE_CONDA" ]; then
-      source deactivate
+      source deactivate || conda deactivate
     else
       deactivate
     fi
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 5255a77ec2081..976854dad8214 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -49,13 +49,12 @@ def determine_modules_for_files(filenames):
     ['pyspark-core', 'sql']
     >>> [x.name for x in determine_modules_for_files(["file_not_matched_by_any_subproject"])]
     ['root']
-    >>> [x.name for x in determine_modules_for_files( \
-            [".github/workflows/master.yml", "appveyor.yml"])]
+    >>> [x.name for x in determine_modules_for_files(["appveyor.yml"])]
     []
     """
     changed_modules = set()
     for filename in filenames:
-        if filename in (".github/workflows/master.yml", "appveyor.yml"):
+        if filename in ("appveyor.yml",):
             continue
         matched_at_least_one_module = False
         for module in modules.all_modules:
@@ -101,28 +100,52 @@ def setup_test_environ(environ):
         os.environ[k] = v
 
 
-def determine_modules_to_test(changed_modules):
+def determine_modules_to_test(changed_modules, deduplicated=True):
     """
     Given a set of modules that have changed, compute the transitive closure of those modules'
     dependent modules in order to determine the set of modules that should be tested.
 
     Returns a topologically-sorted list of modules (ties are broken by sorting on module names).
+    If ``deduplicated`` is disabled, the modules are returned without tacking the deduplication
+    by dependencies into account.
 
     >>> [x.name for x in determine_modules_to_test([modules.root])]
     ['root']
     >>> [x.name for x in determine_modules_to_test([modules.build])]
     ['root']
+    >>> [x.name for x in determine_modules_to_test([modules.core])]
+    ['root']
+    >>> [x.name for x in determine_modules_to_test([modules.launcher])]
+    ['root']
     >>> [x.name for x in determine_modules_to_test([modules.graphx])]
     ['graphx', 'examples']
-    >>> x = [x.name for x in determine_modules_to_test([modules.sql])]
-    >>> x # doctest: +NORMALIZE_WHITESPACE
+    >>> [x.name for x in determine_modules_to_test([modules.sql])]
+    ... # doctest: +NORMALIZE_WHITESPACE
     ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver',
      'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml']
+    >>> sorted([x.name for x in determine_modules_to_test(
+    ...     [modules.sparkr, modules.sql], deduplicated=False)])
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['avro', 'examples', 'hive', 'hive-thriftserver', 'mllib', 'pyspark-ml',
+     'pyspark-mllib', 'pyspark-sql', 'repl', 'sparkr', 'sql', 'sql-kafka-0-10']
+    >>> sorted([x.name for x in determine_modules_to_test(
+    ...     [modules.sql, modules.core], deduplicated=False)])
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['avro', 'catalyst', 'core', 'examples', 'graphx', 'hive', 'hive-thriftserver',
+     'mllib', 'mllib-local', 'pyspark-core', 'pyspark-ml', 'pyspark-mllib',
+     'pyspark-sql', 'pyspark-streaming', 'repl', 'root',
+     'sparkr', 'sql', 'sql-kafka-0-10', 'streaming', 'streaming-kafka-0-10',
+     'streaming-kinesis-asl']
     """
     modules_to_test = set()
     for module in changed_modules:
-        modules_to_test = modules_to_test.union(determine_modules_to_test(module.dependent_modules))
+        modules_to_test = modules_to_test.union(
+            determine_modules_to_test(module.dependent_modules, deduplicated))
     modules_to_test = modules_to_test.union(set(changed_modules))
+
+    if not deduplicated:
+        return modules_to_test
+
     # If we need to run all of the tests, then we should short-circuit and return 'root'
     if modules.root in modules_to_test:
         return [modules.root]
@@ -415,7 +438,7 @@ def run_scala_tests_sbt(test_modules, test_profiles):
     exec_sbt(profiles_and_goals)
 
 
-def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags):
+def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags):
     """Function to properly execute all tests passed in as a set from the
     `determine_test_suites` function"""
     set_title_and_block("Running Spark unit tests", "BLOCK_SPARK_UNIT_TESTS")
@@ -425,6 +448,8 @@ def run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags):
     test_profiles = extra_profiles + \
         list(set(itertools.chain.from_iterable(m.build_profile_flags for m in test_modules)))
 
+    if included_tags:
+        test_profiles += ['-Dtest.include.tags=' + ",".join(included_tags)]
     if excluded_tags:
         test_profiles += ['-Dtest.exclude.tags=' + ",".join(excluded_tags)]
 
@@ -532,6 +557,24 @@ def parse_opts():
         "-p", "--parallelism", type=int, default=8,
         help="The number of suites to test in parallel (default %(default)d)"
     )
+    parser.add_argument(
+        "-m", "--modules", type=str,
+        default=None,
+        help="A comma-separated list of modules to test "
+             "(default: %s)" % ",".join(sorted([m.name for m in modules.all_modules]))
+    )
+    parser.add_argument(
+        "-e", "--excluded-tags", type=str,
+        default=None,
+        help="A comma-separated list of tags to exclude in the tests, "
+             "e.g., org.apache.spark.tags.ExtendedHiveTest "
+    )
+    parser.add_argument(
+        "-i", "--included-tags", type=str,
+        default=None,
+        help="A comma-separated list of tags to include in the tests, "
+             "e.g., org.apache.spark.tags.ExtendedHiveTest "
+    )
 
     args, unknown = parser.parse_known_args()
     if unknown:
@@ -564,11 +607,20 @@ def main():
               " install one and retry.")
         sys.exit(2)
 
-    # install SparkR
-    if which("R"):
-        run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
-    else:
-        print("Cannot install SparkR as R was not found in PATH")
+    # Install SparkR
+    should_only_test_modules = opts.modules is not None
+    test_modules = []
+    if should_only_test_modules:
+        str_test_modules = [m.strip() for m in opts.modules.split(",")]
+        test_modules = [m for m in modules.all_modules if m.name in str_test_modules]
+
+    if not should_only_test_modules or modules.sparkr in test_modules:
+        # If tests modules are specified, we will not run R linter.
+        # SparkR needs the manual SparkR installation.
+        if which("R"):
+            run_cmd([os.path.join(SPARK_HOME, "R", "install-dev.sh")])
+        else:
+            print("Cannot install SparkR as R was not found in PATH")
 
     if os.environ.get("AMPLAB_JENKINS"):
         # if we're on the Amplab Jenkins build servers setup variables
@@ -582,27 +634,69 @@ def main():
         # /home/jenkins/anaconda2/envs/py36/bin
         os.environ["PATH"] = "/home/anaconda/envs/py36/bin:" + os.environ.get("PATH")
     else:
-        # else we're running locally and can use local settings
+        # else we're running locally or Github Actions.
         build_tool = "sbt"
         hadoop_version = os.environ.get("HADOOP_PROFILE", "hadoop2.7")
         hive_version = os.environ.get("HIVE_PROFILE", "hive2.3")
-        test_env = "local"
+        if "GITHUB_ACTIONS" in os.environ:
+            test_env = "github_actions"
+        else:
+            test_env = "local"
 
     print("[info] Using build tool", build_tool, "with Hadoop profile", hadoop_version,
           "and Hive profile", hive_version, "under environment", test_env)
     extra_profiles = get_hadoop_profiles(hadoop_version) + get_hive_profiles(hive_version)
 
-    changed_modules = None
-    changed_files = None
-    if test_env == "amplab_jenkins" and os.environ.get("AMP_JENKINS_PRB"):
+    changed_modules = []
+    changed_files = []
+    included_tags = []
+    excluded_tags = []
+    if should_only_test_modules:
+        # If we're running the tests in Github Actions, attempt to detect and test
+        # only the affected modules.
+        if test_env == "github_actions":
+            if os.environ["GITHUB_BASE_REF"] != "":
+                # Pull requests
+                changed_files = identify_changed_files_from_git_commits(
+                    os.environ["GITHUB_SHA"], target_branch=os.environ["GITHUB_BASE_REF"])
+            else:
+                # Build for each commit.
+                changed_files = identify_changed_files_from_git_commits(
+                    os.environ["GITHUB_SHA"], target_ref=os.environ["GITHUB_PREV_SHA"])
+
+            modules_to_test = determine_modules_to_test(
+                determine_modules_for_files(changed_files), deduplicated=False)
+
+            if modules.root not in modules_to_test:
+                # If root module is not found, only test the intersected modules.
+                # If root module is found, just run the modules as specified initially.
+                test_modules = list(set(modules_to_test).intersection(test_modules))
+
+        changed_modules = test_modules
+        if len(changed_modules) == 0:
+            print("[info] There are no modules to test, exiting without testing.")
+            return
+
+    # If we're running the tests in AMPLab Jenkins, calculate the diff from the targeted branch, and
+    # detect modules to test.
+    elif test_env == "amplab_jenkins" and os.environ.get("AMP_JENKINS_PRB"):
         target_branch = os.environ["ghprbTargetBranch"]
         changed_files = identify_changed_files_from_git_commits("HEAD", target_branch=target_branch)
         changed_modules = determine_modules_for_files(changed_files)
+        test_modules = determine_modules_to_test(changed_modules)
         excluded_tags = determine_tags_to_exclude(changed_modules)
 
+    # If there is no changed module found, tests all.
     if not changed_modules:
         changed_modules = [modules.root]
-        excluded_tags = []
+    if not test_modules:
+        test_modules = determine_modules_to_test(changed_modules)
+
+    if opts.excluded_tags:
+        excluded_tags.extend([t.strip() for t in opts.excluded_tags.split(",")])
+    if opts.included_tags:
+        included_tags.extend([t.strip() for t in opts.included_tags.split(",")])
+
     print("[info] Found the following changed modules:",
           ", ".join(x.name for x in changed_modules))
 
@@ -615,33 +709,32 @@ def main():
         test_environ.update(m.environ)
     setup_test_environ(test_environ)
 
-    test_modules = determine_modules_to_test(changed_modules)
-
-    # license checks
-    run_apache_rat_checks()
-
-    # style checks
-    if not changed_files or any(f.endswith(".scala")
-                                or f.endswith("scalastyle-config.xml")
-                                for f in changed_files):
-        run_scala_style_checks(extra_profiles)
     should_run_java_style_checks = False
-    if not changed_files or any(f.endswith(".java")
-                                or f.endswith("checkstyle.xml")
-                                or f.endswith("checkstyle-suppressions.xml")
-                                for f in changed_files):
-        # Run SBT Checkstyle after the build to prevent a side-effect to the build.
-        should_run_java_style_checks = True
-    if not changed_files or any(f.endswith("lint-python")
-                                or f.endswith("tox.ini")
-                                or f.endswith(".py")
-                                for f in changed_files):
-        run_python_style_checks()
-    if not changed_files or any(f.endswith(".R")
-                                or f.endswith("lint-r")
-                                or f.endswith(".lintr")
-                                for f in changed_files):
-        run_sparkr_style_checks()
+    if not should_only_test_modules:
+        # license checks
+        run_apache_rat_checks()
+
+        # style checks
+        if not changed_files or any(f.endswith(".scala")
+                                    or f.endswith("scalastyle-config.xml")
+                                    for f in changed_files):
+            run_scala_style_checks(extra_profiles)
+        if not changed_files or any(f.endswith(".java")
+                                    or f.endswith("checkstyle.xml")
+                                    or f.endswith("checkstyle-suppressions.xml")
+                                    for f in changed_files):
+            # Run SBT Checkstyle after the build to prevent a side-effect to the build.
+            should_run_java_style_checks = True
+        if not changed_files or any(f.endswith("lint-python")
+                                    or f.endswith("tox.ini")
+                                    or f.endswith(".py")
+                                    for f in changed_files):
+            run_python_style_checks()
+        if not changed_files or any(f.endswith(".R")
+                                    or f.endswith("lint-r")
+                                    or f.endswith(".lintr")
+                                    for f in changed_files):
+            run_sparkr_style_checks()
 
     # determine if docs were changed and if we're inside the amplab environment
     # note - the below commented out until *all* Jenkins workers can get `jekyll` installed
@@ -663,7 +756,7 @@ def main():
         build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks)
 
     # run the test suites
-    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags)
+    run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags, included_tags)
 
     modules_with_python_tests = [m for m in test_modules if m.python_test_goals]
     if modules_with_python_tests:
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 391e4bbe1b1f0..75bdec0f40f0a 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -100,9 +100,75 @@ def __hash__(self):
     ]
 )
 
+kvstore = Module(
+    name="kvstore",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/kvstore/",
+    ],
+    sbt_test_goals=[
+        "kvstore/test",
+    ],
+)
+
+network_common = Module(
+    name="network-common",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/network-common/",
+    ],
+    sbt_test_goals=[
+        "network-common/test",
+    ],
+)
+
+network_shuffle = Module(
+    name="network-shuffle",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/network-shuffle/",
+    ],
+    sbt_test_goals=[
+        "network-shuffle/test",
+    ],
+)
+
+unsafe = Module(
+    name="unsafe",
+    dependencies=[tags],
+    source_file_regexes=[
+        "common/unsafe",
+    ],
+    sbt_test_goals=[
+        "unsafe/test",
+    ],
+)
+
+launcher = Module(
+    name="launcher",
+    dependencies=[tags],
+    source_file_regexes=[
+        "launcher/",
+    ],
+    sbt_test_goals=[
+        "launcher/test",
+    ],
+)
+
+core = Module(
+    name="core",
+    dependencies=[kvstore, network_common, network_shuffle, unsafe, launcher],
+    source_file_regexes=[
+        "core/",
+    ],
+    sbt_test_goals=[
+        "core/test",
+    ],
+)
+
 catalyst = Module(
     name="catalyst",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "sql/catalyst/",
     ],
@@ -111,7 +177,6 @@ def __hash__(self):
     ],
 )
 
-
 sql = Module(
     name="sql",
     dependencies=[catalyst],
@@ -123,7 +188,6 @@ def __hash__(self):
     ],
 )
 
-
 hive = Module(
     name="hive",
     dependencies=[sql],
@@ -142,7 +206,6 @@ def __hash__(self):
     ]
 )
 
-
 repl = Module(
     name="repl",
     dependencies=[hive],
@@ -154,7 +217,6 @@ def __hash__(self):
     ],
 )
 
-
 hive_thriftserver = Module(
     name="hive-thriftserver",
     dependencies=[hive],
@@ -192,7 +254,6 @@ def __hash__(self):
     ]
 )
 
-
 sketch = Module(
     name="sketch",
     dependencies=[tags],
@@ -204,10 +265,9 @@ def __hash__(self):
     ]
 )
 
-
 graphx = Module(
     name="graphx",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "graphx/",
     ],
@@ -216,10 +276,9 @@ def __hash__(self):
     ]
 )
 
-
 streaming = Module(
     name="streaming",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "streaming",
     ],
@@ -235,7 +294,7 @@ def __hash__(self):
 # fail other PRs.
 streaming_kinesis_asl = Module(
     name="streaming-kinesis-asl",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "external/kinesis-asl/",
         "external/kinesis-asl-assembly/",
@@ -254,21 +313,23 @@ def __hash__(self):
 
 streaming_kafka_0_10 = Module(
     name="streaming-kafka-0-10",
-    dependencies=[streaming],
+    dependencies=[streaming, core],
     source_file_regexes=[
         # The ending "/" is necessary otherwise it will include "sql-kafka" codes
         "external/kafka-0-10/",
         "external/kafka-0-10-assembly",
+        "external/kafka-0-10-token-provider",
     ],
     sbt_test_goals=[
         "streaming-kafka-0-10/test",
+        "token-provider-kafka-0-10/test"
     ]
 )
 
 
 mllib_local = Module(
     name="mllib-local",
-    dependencies=[tags],
+    dependencies=[tags, core],
     source_file_regexes=[
         "mllib-local",
     ],
@@ -302,10 +363,9 @@ def __hash__(self):
     ]
 )
 
-
 pyspark_core = Module(
     name="pyspark-core",
-    dependencies=[],
+    dependencies=[core],
     source_file_regexes=[
         "python/(?!pyspark/(ml|mllib|sql|streaming))"
     ],
@@ -339,7 +399,6 @@ def __hash__(self):
     ]
 )
 
-
 pyspark_sql = Module(
     name="pyspark-sql",
     dependencies=[pyspark_core, hive, avro],
@@ -578,7 +637,7 @@ def __hash__(self):
 # No other modules should directly depend on this module.
 root = Module(
     name="root",
-    dependencies=[build],  # Changes to build should trigger all tests.
+    dependencies=[build, core],  # Changes to build should trigger all tests.
     source_file_regexes=[],
     # In order to run all of the tests, enable every test profile:
     build_profile_flags=list(set(
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index a0e138c5bcb65..7a9634ac0fcbb 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1017,6 +1017,15 @@ object TestSettings {
       sys.props.get("test.exclude.tags").map { tags =>
         Seq("--exclude-categories=" + tags)
       }.getOrElse(Nil): _*),
+    // Include tags defined in a system property
+    testOptions in Test += Tests.Argument(TestFrameworks.ScalaTest,
+      sys.props.get("test.include.tags").map { tags =>
+        tags.split(",").flatMap { tag => Seq("-n", tag) }.toSeq
+      }.getOrElse(Nil): _*),
+    testOptions in Test += Tests.Argument(TestFrameworks.JUnit,
+      sys.props.get("test.include.tags").map { tags =>
+        Seq("--include-categories=" + tags)
+      }.getOrElse(Nil): _*),
     // Show full stack trace and duration in test cases.
     testOptions in Test += Tests.Argument("-oDF"),
     testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 42f064a05ed91..15c5cf1ad9b05 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -21,6 +21,9 @@
 import time
 import unittest
 import warnings
+import sys
+if sys.version >= '3':
+    basestring = unicode = str
 
 from pyspark import SparkContext, SparkConf
 from pyspark.sql import Row, SparkSession
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 81402f52af3b3..016cafd669019 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -725,7 +725,8 @@ def assertCollectSuccess(typecode, value):
         if sys.version_info[0] < 3:
             all_types = set(['c', 'b', 'B', 'u', 'h', 'H', 'i', 'I', 'l', 'L', 'f', 'd'])
         else:
-            all_types = set(array.typecodes)
+            # PyPy seems not having array.typecodes.
+            all_types = set(['b', 'B', 'u', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd'])
         unsupported_types = all_types - set(supported_types)
         # test unsupported types
         for t in unsupported_types:
diff --git a/python/pyspark/streaming/tests/test_dstream.py b/python/pyspark/streaming/tests/test_dstream.py
index 7ecdf6b0b12db..89edb23070c69 100644
--- a/python/pyspark/streaming/tests/test_dstream.py
+++ b/python/pyspark/streaming/tests/test_dstream.py
@@ -30,8 +30,9 @@
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class BasicOperationTests(PySparkStreamingTestCase):
 
     def test_map(self):
@@ -394,8 +395,9 @@ def failed_func(i):
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class WindowFunctionTests(PySparkStreamingTestCase):
 
     timeout = 15
@@ -474,8 +476,9 @@ def func(dstream):
 
 
 @unittest.skipIf(
-    "pypy" in platform.python_implementation().lower() and "COVERAGE_PROCESS_START" in os.environ,
-    "PyPy implementation causes to hang DStream tests forever when Coverage report is used.")
+    "pypy" in platform.python_implementation().lower(),
+    "The tests fail in PyPy3 implementation for an unknown reason. "
+    "With PyPy, it causes to hang DStream tests forever when Coverage report is used.")
 class CheckpointTests(unittest.TestCase):
 
     setupCalled = False
diff --git a/python/run-tests.py b/python/run-tests.py
index b677a5134ec93..a404c5364e9bb 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -161,7 +161,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
 
 
 def get_default_python_executables():
-    python_execs = [x for x in ["python3.6", "python2.7", "pypy"] if which(x)]
+    python_execs = [x for x in ["python3.8", "python2.7", "pypy3", "pypy"] if which(x)]
 
     if "python3.6" not in python_execs:
         p = which("python3")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
index 6391d5664d5c6..80346b350c142 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/IntegratedUDFTestUtils.scala
@@ -197,7 +197,7 @@ object IntegratedUDFTestUtils extends SQLHelper {
 
   lazy val pythonExec: String = {
     val pythonExec = sys.env.getOrElse(
-      "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3.6"))
+      "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3"))
     if (TestUtils.testCommandAvailable(pythonExec)) {
       pythonExec
     } else {
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
index 15cc3109da3f7..8b16674f05a84 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala
@@ -33,6 +33,7 @@ import org.apache.spark.sql.catalyst.util.fileToString
 import org.apache.spark.sql.execution.HiveResult
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.GitHubActionsUnstableTest
 
 /**
  * Re-run all the tests in SQLQueryTestSuite via Thrift Server.
@@ -52,6 +53,7 @@ import org.apache.spark.sql.types._
  *   2. Support DESC command.
  *   3. Support SHOW command.
  */
+@GitHubActionsUnstableTest
 class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServer {
 
   override protected def testFile(fileName: String): String = {
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
index 3e1fce78ae71c..73547d752024b 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.hive.thriftserver
 
+import org.apache.spark.tags.GitHubActionsUnstableTest
+
+@GitHubActionsUnstableTest
 class ThriftServerWithSparkContextSuite extends SharedThriftServer {
 
   test("SPARK-29911: Uncache cached tables when session closed") {
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index db1f6fbd97d90..4112512e56fee 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -26,10 +26,12 @@ import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHive
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
+import org.apache.spark.tags.SlowHiveTest
 
 /**
  * Runs the test cases that are included in the hive distribution.
  */
+@SlowHiveTest
 class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
   // TODO: bundle in jar files... get from classpath
   private lazy val hiveQueryDir = TestHive.getHiveFile(
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 8be3d26bfc93a..aa96fa035c4f0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.CatalogTableType
 import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.test.SQLTestUtils
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.Utils
 
 /**
@@ -46,6 +46,7 @@ import org.apache.spark.util.Utils
  * expected version under this local directory, e.g. `/tmp/spark-test/spark-2.0.3`, we will skip the
  * downloading for this spark version.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
   private val isTestAtLeastJava9 = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
index 8b97489e2d818..3a7e92ee1c00b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala
@@ -38,12 +38,13 @@ import org.apache.spark.sql.hive.test.{HiveTestJars, TestHiveContext}
 import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
 import org.apache.spark.sql.internal.StaticSQLConf.WAREHOUSE_PATH
 import org.apache.spark.sql.types.{DecimalType, StructType}
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.{ResetSystemProperties, Utils}
 
 /**
  * This suite tests spark-submit with applications using HiveContext.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class HiveSparkSubmitSuite
   extends SparkSubmitTestUtils
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index 8642a5ff16812..5ddaaf3c9a77c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -41,7 +41,7 @@ import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveVersion
 import org.apache.spark.sql.types.IntegerType
 import org.apache.spark.sql.types.StructType
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, GitHubActionsUnstableTest}
 import org.apache.spark.util.{MutableURLClassLoader, Utils}
 
 /**
@@ -52,6 +52,7 @@ import org.apache.spark.util.{MutableURLClassLoader, Utils}
  */
 // TODO: Refactor this to `HiveClientSuite` and make it a subclass of `HiveVersionSuite`
 @ExtendedHiveTest
+@GitHubActionsUnstableTest
 class VersionsSuite extends SparkFunSuite with Logging {
 
   override protected val enableAutoThreadAudit = false
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
index fac981267f4d7..1dd2ad3837cc3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/AggregationQuerySuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.unsafe.UnsafeAlignedOffset
 
 
@@ -1054,6 +1055,7 @@ abstract class AggregationQuerySuite extends QueryTest with SQLTestUtils with Te
 class HashAggregationQuerySuite extends AggregationQuerySuite
 
 
+@SlowHiveTest
 class HashAggregationQueryWithControlledFallbackSuite extends AggregationQuerySuite {
 
   override protected def checkAnswer(actual: => DataFrame, expectedAnswer: Seq[Row]): Unit = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index e8cf4ad5d9f28..d0aa618fc2731 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -44,9 +44,11 @@ import org.apache.spark.sql.internal.SQLConf.ORC_IMPLEMENTATION
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
 // TODO(gatorsmile): combine HiveCatalogedDDLSuite and HiveDDLSuite
+@SlowHiveTest
 class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeAndAfterEach {
   override def afterEach(): Unit = {
     try {
@@ -404,6 +406,7 @@ class HiveCatalogedDDLSuite extends DDLSuite with TestHiveSingleton with BeforeA
   }
 }
 
+@SlowHiveTest
 class HiveDDLSuite
   extends QueryTest with SQLTestUtils with TestHiveSingleton with BeforeAndAfterEach {
   import testImplicits._
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index b10a8cb8bf2bf..e798a352f72bf 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.hive.test.{HiveTestJars, TestHive}
 import org.apache.spark.sql.hive.test.TestHive._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
+import org.apache.spark.tags.SlowHiveTest
 
 case class TestData(a: Int, b: String)
 
@@ -46,6 +47,7 @@ case class TestData(a: Int, b: String)
  * A set of test cases expressed in Hive QL that are not covered by the tests
  * included in the hive distribution.
  */
+@SlowHiveTest
 class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAndAfter {
   import org.apache.spark.sql.hive.test.TestHive.implicits._
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
index b20ef035594da..6f37e39a532d6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
@@ -27,13 +27,14 @@ import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils}
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.StaticSQLConf._
 import org.apache.spark.sql.types._
-import org.apache.spark.tags.ExtendedHiveTest
+import org.apache.spark.tags.{ExtendedHiveTest, SlowHiveTest}
 import org.apache.spark.util.Utils
 
 /**
  * A separate set of DDL tests that uses Hive 2.1 libraries, which behave a little differently
  * from the built-in ones.
  */
+@SlowHiveTest
 @ExtendedHiveTest
 class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton with BeforeAndAfterEach
   with BeforeAndAfterAll {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index d12eae0e410b1..a46db32b9adee 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -43,6 +43,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.SlowHiveTest
 import org.apache.spark.util.Utils
 
 case class Nested1(f1: Nested2)
@@ -2559,6 +2560,8 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
   }
 }
 
+@SlowHiveTest
 class SQLQuerySuite extends SQLQuerySuiteBase with DisableAdaptiveExecutionSuite
+@SlowHiveTest
 class SQLQuerySuiteAE extends SQLQuerySuiteBase with EnableAdaptiveExecutionSuite
 

From 2b41bc928199df6d546ef055e233b9c58b019b38 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 19 Aug 2020 07:22:52 -0700
Subject: [PATCH 0982/1280] [SPARK-32451][R][3.0] Support Apache Arrow 1.0.0

### What changes were proposed in this pull request?

This PR ports back https://github.com/apache/spark/pull/29252 to support Arrow 1.0.0.

Currently, SparkR with Arrow tests fails with the latest Arrow version in branch-3.0, see https://github.com/apache/spark/pull/29460/checks?check_run_id=996972267

### Why are the changes needed?

To support higher Arrow R version with SparkR.

### Does this PR introduce _any_ user-facing change?

Yes, users will be able to use SparkR with Arrow 1.0.0+.

### How was this patch tested?

Manually tested, GitHub Actions will test it.

Closes #29462 from HyukjinKwon/SPARK-32451-3.0.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 R/pkg/R/DataFrame.R | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index f9a07f65325fb..3a60d1c970440 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1231,9 +1231,14 @@ setMethod("collect",
                 authSecret <- portAuth[[2]]
                 conn <- socketConnection(
                   port = port, blocking = TRUE, open = "wb", timeout = connectionTimeout)
+                version <- packageVersion("arrow")
                 output <- tryCatch({
                   doServerAuth(conn, authSecret)
-                  arrowTable <- arrow::read_arrow(readRaw(conn))
+                  if (version$minor >= 17 || version$major >= 1) {
+                    arrowTable <- arrow::read_ipc_stream(readRaw(conn))
+                  } else {
+                    arrowTable <- arrow::read_arrow(readRaw(conn))
+                  }
                   as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
                 }, finally = {
                   close(conn)

From c4a12f22f8656b59bc9a341f6ed008167983d128 Mon Sep 17 00:00:00 2001
From: Burak Yavuz <brkyvz@gmail.com>
Date: Wed, 19 Aug 2020 16:25:35 +0000
Subject: [PATCH 0983/1280] [SPARK-28863][SQL] Introduce AlreadyPlanned to
 prevent reanalysis of V1FallbackWriters

### What changes were proposed in this pull request?

This PR introduces a LogicalNode AlreadyPlanned, and related physical plan and preparation rule.

With the DataSourceV2 write operations, we have a way to fallback to the V1 writer APIs using InsertableRelation. The gross part is that we're in physical land, but the InsertableRelation takes a logical plan, so we have to pass the logical plans to these physical nodes, and then potentially go through re-planning. This re-planning can cause issues for an already optimized plan.

A useful primitive could be specifying that a plan is ready for execution through a logical node AlreadyPlanned. This would wrap a physical plan, and then we can go straight to execution.

### Why are the changes needed?

To avoid having a physical plan that is disconnected from the physical plan that is being executed in V1WriteFallback execution. When a physical plan node executes a logical plan, the inner query is not connected to the running physical plan. The physical plan that actually runs is not visible through the Spark UI and its metrics are not exposed. In some cases, the EXPLAIN plan doesn't show it.

### Does this PR introduce _any_ user-facing change?

Nope

### How was this patch tested?

V1FallbackWriterSuite tests that writes still work

Closes #29469 from brkyvz/alreadyAnalyzed2.

Lead-authored-by: Burak Yavuz <brkyvz@gmail.com>
Co-authored-by: Burak Yavuz <burak@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 278d0dd25bc1479ecda42d6f722106e4763edfae)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/AlreadyPlanned.scala  | 61 ++++++++++++++
 .../spark/sql/execution/SparkStrategies.scala |  1 +
 .../datasources/v2/DataSourceV2Strategy.scala | 10 +--
 .../datasources/v2/V1FallbackWriters.scala    | 22 ++---
 .../v2/WriteToDataSourceV2Exec.scala          |  4 -
 .../sql/connector/V1WriteFallbackSuite.scala  | 42 ++++++++++
 .../sql/execution/AlreadyPlannedSuite.scala   | 80 +++++++++++++++++++
 7 files changed, 196 insertions(+), 24 deletions(-)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala
new file mode 100644
index 0000000000000..9dd956a9c2e08
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
+import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute}
+import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
+
+/**
+ * A special node that allows skipping query planning all the way to physical execution. This node
+ * can be used when a query was planned to the physical level, but we had to go back to logical plan
+ * land for some reason (e.g. V1 DataSource write execution). This will allow the metrics, and the
+ * query plan to properly appear as part of the query execution.
+ */
+case class AlreadyPlanned(physicalPlan: SparkPlan) extends LogicalPlan with MultiInstanceRelation {
+  override def children: Seq[LogicalPlan] = Nil
+  override lazy val resolved: Boolean = true
+  override val output: Seq[Attribute] = physicalPlan.output
+  override def newInstance(): LogicalPlan = {
+    val newAttrs = output.map(a => a.exprId -> a.newInstance())
+    val attrMap = newAttrs.toMap
+    val projections = physicalPlan.output.map { o =>
+      Alias(o, o.name)(attrMap(o.exprId).exprId, o.qualifier, Option(o.metadata))
+    }
+    AlreadyPlanned(ProjectExec(projections, physicalPlan))
+  }
+}
+
+/** Query execution that skips re-analysis and planning. */
+class AlreadyPlannedExecution(
+    session: SparkSession,
+    plan: AlreadyPlanned) extends QueryExecution(session, plan) {
+  override lazy val analyzed: LogicalPlan = plan
+  override lazy val optimizedPlan: LogicalPlan = plan
+  override lazy val sparkPlan: SparkPlan = plan.physicalPlan
+  override lazy val executedPlan: SparkPlan = plan.physicalPlan
+}
+
+object AlreadyPlanned {
+  def dataFrame(sparkSession: SparkSession, query: SparkPlan): DataFrame = {
+    val qe = new AlreadyPlannedExecution(sparkSession, AlreadyPlanned(query))
+    new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index f836debcbafda..ec6e36535af1b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -678,6 +678,7 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
   object BasicOperators extends Strategy {
     def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
+      case planned: AlreadyPlanned => planned.physicalPlan :: Nil
       case d: DataWritingCommand => DataWritingCommandExec(d, planLater(d.query)) :: Nil
       case r: RunnableCommand => ExecutedCommandExec(r) :: Nil
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index cca80c0cb6d57..f2ff0060654f3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -122,10 +122,10 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
       val writeOptions = new CaseInsensitiveStringMap(options.asJava)
       catalog match {
         case staging: StagingTableCatalog =>
-          AtomicCreateTableAsSelectExec(staging, ident, parts, query, planLater(query),
+          AtomicCreateTableAsSelectExec(staging, ident, parts, planLater(query),
             propsWithOwner, writeOptions, ifNotExists) :: Nil
         case _ =>
-          CreateTableAsSelectExec(catalog, ident, parts, query, planLater(query),
+          CreateTableAsSelectExec(catalog, ident, parts, planLater(query),
             propsWithOwner, writeOptions, ifNotExists) :: Nil
       }
 
@@ -152,7 +152,6 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             staging,
             ident,
             parts,
-            query,
             planLater(query),
             propsWithOwner,
             writeOptions,
@@ -162,7 +161,6 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             catalog,
             ident,
             parts,
-            query,
             planLater(query),
             propsWithOwner,
             writeOptions,
@@ -172,7 +170,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     case AppendData(r: DataSourceV2Relation, query, writeOptions, _) =>
       r.table.asWritable match {
         case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
-          AppendDataExecV1(v1, writeOptions.asOptions, query) :: Nil
+          AppendDataExecV1(v1, writeOptions.asOptions, planLater(query)) :: Nil
         case v2 =>
           AppendDataExec(v2, writeOptions.asOptions, planLater(query)) :: Nil
       }
@@ -186,7 +184,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
       }.toArray
       r.table.asWritable match {
         case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
-          OverwriteByExpressionExecV1(v1, filters, writeOptions.asOptions, query) :: Nil
+          OverwriteByExpressionExecV1(v1, filters, writeOptions.asOptions, planLater(query)) :: Nil
         case v2 =>
           OverwriteByExpressionExec(v2, filters, writeOptions.asOptions, planLater(query)) :: Nil
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
index 7502a8750aee4..28c4e28004446 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
@@ -20,14 +20,11 @@ package org.apache.spark.sql.execution.datasources.v2
 import java.util.UUID
 
 import org.apache.spark.SparkException
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.connector.catalog.SupportsWrite
 import org.apache.spark.sql.connector.write.{LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
-import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.{AlreadyPlanned, SparkPlan}
 import org.apache.spark.sql.sources.{AlwaysTrue, Filter, InsertableRelation}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -39,7 +36,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 case class AppendDataExecV1(
     table: SupportsWrite,
     writeOptions: CaseInsensitiveStringMap,
-    plan: LogicalPlan) extends V1FallbackWriters {
+    query: SparkPlan) extends V1FallbackWriters {
 
   override protected def run(): Seq[InternalRow] = {
     writeWithV1(newWriteBuilder().buildForV1Write())
@@ -61,7 +58,7 @@ case class OverwriteByExpressionExecV1(
     table: SupportsWrite,
     deleteWhere: Array[Filter],
     writeOptions: CaseInsensitiveStringMap,
-    plan: LogicalPlan) extends V1FallbackWriters {
+    query: SparkPlan) extends V1FallbackWriters {
 
   private def isTruncate(filters: Array[Filter]): Boolean = {
     filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
@@ -84,7 +81,7 @@ case class OverwriteByExpressionExecV1(
 /** Some helper interfaces that use V2 write semantics through the V1 writer interface. */
 sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
   override def output: Seq[Attribute] = Nil
-  override final def children: Seq[SparkPlan] = Nil
+  override final def children: Seq[SparkPlan] = Seq(query)
 
   def table: SupportsWrite
   def writeOptions: CaseInsensitiveStringMap
@@ -100,11 +97,9 @@ sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
   protected def newWriteBuilder(): V1WriteBuilder = {
     val info = LogicalWriteInfoImpl(
       queryId = UUID.randomUUID().toString,
-      schema = plan.schema,
+      schema = query.schema,
       options = writeOptions)
-    val writeBuilder = table.newWriteBuilder(info)
-
-    writeBuilder.asV1Builder
+    table.newWriteBuilder(info).asV1Builder
   }
 }
 
@@ -112,11 +107,10 @@ sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
  * A trait that allows Tables that use V1 Writer interfaces to append data.
  */
 trait SupportsV1Write extends SparkPlan {
-  // TODO: We should be able to work on SparkPlans at this point.
-  def plan: LogicalPlan
+  def query: SparkPlan
 
   protected def writeWithV1(relation: InsertableRelation): Seq[InternalRow] = {
-    relation.insert(Dataset.ofRows(sqlContext.sparkSession, plan), overwrite = false)
+    relation.insert(AlreadyPlanned.dataFrame(sqlContext.sparkSession, query), overwrite = false)
     Nil
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index 616e18ee85a6b..bf961329a5d14 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -62,7 +62,6 @@ case class CreateTableAsSelectExec(
     catalog: TableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
-    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
@@ -119,7 +118,6 @@ case class AtomicCreateTableAsSelectExec(
     catalog: StagingTableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
-    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
@@ -153,7 +151,6 @@ case class ReplaceTableAsSelectExec(
     catalog: TableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
-    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
@@ -219,7 +216,6 @@ case class AtomicReplaceTableAsSelectExec(
     catalog: StagingTableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
-    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
index 10ed2048dbf61..4b52a4cbf4116 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/V1WriteFallbackSuite.scala
@@ -25,10 +25,14 @@ import scala.collection.mutable
 import org.scalatest.BeforeAndAfter
 
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row, SaveMode, SparkSession, SQLContext}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, Table, TableCapability}
 import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform, Transform}
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
+import org.apache.spark.sql.internal.SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION
 import org.apache.spark.sql.internal.connector.SimpleTableProvider
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.test.SharedSparkSession
@@ -124,6 +128,23 @@ class V1WriteFallbackSuite extends QueryTest with SharedSparkSession with Before
     }
     assert(e3.getMessage.contains("schema"))
   }
+
+  test("fallback writes should only analyze plan once") {
+    SparkSession.clearActiveSession()
+    SparkSession.clearDefaultSession()
+    try {
+      val session = SparkSession.builder()
+        .master("local[1]")
+        .withExtensions(_.injectPostHocResolutionRule(_ => OnlyOnceRule))
+        .config(V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[V1FallbackTableCatalog].getName)
+        .getOrCreate()
+      val df = session.createDataFrame(Seq((1, "x"), (2, "y"), (3, "z")))
+      df.write.mode("append").option("name", "t1").format(v2Format).saveAsTable("test")
+    } finally {
+      SparkSession.setActiveSession(spark)
+      SparkSession.setDefaultSession(spark)
+    }
+  }
 }
 
 class V1WriteFallbackSessionCatalogSuite
@@ -318,3 +339,24 @@ class InMemoryTableWithV1Fallback(
     }
   }
 }
+
+/** A rule that fails if a query plan is analyzed twice. */
+object OnlyOnceRule extends Rule[LogicalPlan] {
+  private val tag = TreeNodeTag[String]("test")
+  private val counts = new mutable.HashMap[LogicalPlan, Int]()
+
+  override def apply(plan: LogicalPlan): LogicalPlan = {
+    if (plan.getTagValue(tag).isEmpty) {
+      plan.setTagValue(tag, "abc")
+      plan
+    } else {
+      val cnt = counts.getOrElseUpdate(plan, 0) + 1
+      // This rule will be run as injectPostHocResolutionRule, and is supposed to be run only twice.
+      // Once during planning and once during checkBatchIdempotence
+      assert(cnt <= 1, "This rule shouldn't have been called again")
+      counts.put(plan, cnt)
+      plan
+    }
+
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala
new file mode 100644
index 0000000000000..9152dcf7a3697
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AlreadyPlannedSuite extends SparkPlanTest with SharedSparkSession {
+
+  import testImplicits._
+
+  test("simple execution") {
+    val df = spark.range(10)
+    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+
+    checkAnswer(planned, identity, df.toDF().collect())
+  }
+
+  test("planning on top works - projection") {
+    val df = spark.range(10)
+    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+
+    checkAnswer(
+      planned.withColumn("data", 'id + 1),
+      identity,
+      df.withColumn("data", 'id + 1).collect())
+  }
+
+  test("planning on top works - filter") {
+    val df = spark.range(10)
+    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+
+    checkAnswer(planned.where('id < 5), identity, df.where('id < 5).toDF().collect())
+  }
+
+  test("planning on top works - joins") {
+    val df = spark.range(10)
+    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+
+    val plannedLeft = planned.alias("l")
+    val dfLeft = df.alias("l")
+    val plannedRight = planned.alias("r")
+    val dfRight = df.alias("r")
+
+    checkAnswer(
+      plannedLeft.where('id < 3).join(plannedRight, Seq("id")),
+      identity,
+      dfLeft.where('id < 3).join(dfRight, Seq("id")).collect())
+
+    checkAnswer(
+      plannedLeft.where('id < 3).join(plannedRight, plannedLeft("id") === plannedRight("id")),
+      identity,
+      dfLeft.where('id < 3).join(dfRight, dfLeft("id") === dfRight("id")).collect())
+
+    checkAnswer(
+      plannedLeft.join(plannedRight, Seq("id")).where('id < 3),
+      identity,
+      dfLeft.join(dfRight, Seq("id")).where('id < 3).collect())
+
+    checkAnswer(
+      plannedLeft.join(plannedRight, plannedLeft("id") === plannedRight("id")).where($"l.id" < 3),
+      identity,
+      dfLeft.join(dfRight, dfLeft("id") === dfRight("id")).where($"l.id" < 3).collect())
+  }
+}

From b87ec5dcecaf4ad0397832ecedc249183671d367 Mon Sep 17 00:00:00 2001
From: Xingbo Jiang <xingbo.jiang@databricks.com>
Date: Thu, 20 Aug 2020 07:08:30 +0000
Subject: [PATCH 0984/1280] [SPARK-32658][CORE] Fix `PartitionWriterStream`
 partition length overflow

### What changes were proposed in this pull request?

The `count` in `PartitionWriterStream` should be a long value, instead of int. The issue is introduced by apache/sparkabef84a . When the overflow happens, the shuffle index file would record wrong index of a reduceId, thus lead to `FetchFailedException: Stream is corrupted` error.

Besides the fix, I also added some debug logs, so in the future it's easier to debug similar issues.

### Why are the changes needed?

This is a regression and bug fix.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

A Spark user reported this issue when migrating their workload to 3.0. One of the jobs fail deterministically on Spark 3.0 without the patch, and the job succeed after applied the fix.

Closes #29474 from jiangxb1987/fixPartitionWriteStream.

Authored-by: Xingbo Jiang <xingbo.jiang@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f793977e9ac2ef597fca4a95356affbfbf864f88)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/shuffle/sort/UnsafeShuffleWriter.java   | 15 +++++++++++++++
 .../sort/io/LocalDiskShuffleMapOutputWriter.java  |  6 ++++--
 .../spark/shuffle/IndexShuffleBlockResolver.scala |  1 +
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
index d09282e61a9c7..3eee1e478616d 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/UnsafeShuffleWriter.java
@@ -18,6 +18,7 @@
 package org.apache.spark.shuffle.sort;
 
 import java.nio.channels.Channels;
+import java.util.Arrays;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import java.io.*;
@@ -274,6 +275,8 @@ private long[] mergeSpills(SpillInfo[] spills) throws IOException {
         // Here, we don't need to perform any metrics updates because the bytes written to this
         // output file would have already been counted as shuffle bytes written.
         partitionLengths = spills[0].partitionLengths;
+        logger.debug("Merge shuffle spills for mapId {} with length {}", mapId,
+            partitionLengths.length);
         maybeSingleFileWriter.get().transferMapSpillFile(spills[0].file, partitionLengths);
       } else {
         partitionLengths = mergeSpillsUsingStandardWriter(spills);
@@ -360,6 +363,7 @@ private void mergeSpillsWithFileStream(
       SpillInfo[] spills,
       ShuffleMapOutputWriter mapWriter,
       @Nullable CompressionCodec compressionCodec) throws IOException {
+    logger.debug("Merge shuffle spills with FileStream for mapId {}", mapId);
     final int numPartitions = partitioner.numPartitions();
     final InputStream[] spillInputStreams = new InputStream[spills.length];
 
@@ -369,6 +373,11 @@ private void mergeSpillsWithFileStream(
         spillInputStreams[i] = new NioBufferedFileInputStream(
           spills[i].file,
           inputBufferSizeInBytes);
+        // Only convert the partitionLengths when debug level is enabled.
+        if (logger.isDebugEnabled()) {
+          logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i,
+              Arrays.toString(spills[i].partitionLengths));
+        }
       }
       for (int partition = 0; partition < numPartitions; partition++) {
         boolean copyThrewException = true;
@@ -431,6 +440,7 @@ private void mergeSpillsWithFileStream(
   private void mergeSpillsWithTransferTo(
       SpillInfo[] spills,
       ShuffleMapOutputWriter mapWriter) throws IOException {
+    logger.debug("Merge shuffle spills with TransferTo for mapId {}", mapId);
     final int numPartitions = partitioner.numPartitions();
     final FileChannel[] spillInputChannels = new FileChannel[spills.length];
     final long[] spillInputChannelPositions = new long[spills.length];
@@ -439,6 +449,11 @@ private void mergeSpillsWithTransferTo(
     try {
       for (int i = 0; i < spills.length; i++) {
         spillInputChannels[i] = new FileInputStream(spills[i].file).getChannel();
+        // Only convert the partitionLengths when debug level is enabled.
+        if (logger.isDebugEnabled()) {
+          logger.debug("Partition lengths for mapId {} in Spill {}: {}", mapId, i,
+              Arrays.toString(spills[i].partitionLengths));
+        }
       }
       for (int partition = 0; partition < numPartitions; partition++) {
         boolean copyThrewException = true;
diff --git a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
index a6529fd76188a..1c3eb34f9be37 100644
--- a/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
+++ b/core/src/main/java/org/apache/spark/shuffle/sort/io/LocalDiskShuffleMapOutputWriter.java
@@ -112,6 +112,8 @@ public long[] commitAllPartitions() throws IOException {
     }
     cleanUp();
     File resolvedTmp = outputTempFile != null && outputTempFile.isFile() ? outputTempFile : null;
+    log.debug("Writing shuffle index file for mapId {} with length {}", mapId,
+        partitionLengths.length);
     blockResolver.writeIndexFileAndCommit(shuffleId, mapId, partitionLengths, resolvedTmp);
     return partitionLengths;
   }
@@ -210,14 +212,14 @@ public long getNumBytesWritten() {
 
   private class PartitionWriterStream extends OutputStream {
     private final int partitionId;
-    private int count = 0;
+    private long count = 0;
     private boolean isClosed = false;
 
     PartitionWriterStream(int partitionId) {
       this.partitionId = partitionId;
     }
 
-    public int getCount() {
+    public long getCount() {
       return count;
     }
 
diff --git a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
index af2c82e771970..b7caf4f0aeabf 100644
--- a/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
+++ b/core/src/main/scala/org/apache/spark/shuffle/IndexShuffleBlockResolver.scala
@@ -209,6 +209,7 @@ private[spark] class IndexShuffleBlockResolver(
         }
       }
     } finally {
+      logDebug(s"Shuffle index for mapId $mapId: ${lengths.mkString("[", ",", "]")}")
       if (indexTmp.exists() && !indexTmp.delete()) {
         logError(s"Failed to delete temporary index file at ${indexTmp.getAbsolutePath}")
       }

From 87d7ab6c6e96db2bd019d743d9459d6f00240829 Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Thu, 20 Aug 2020 13:43:15 +0000
Subject: [PATCH 0985/1280] [SPARK-32608][SQL][3.0] Script Transform ROW FORMAT
 DELIMIT value should format value

### What changes were proposed in this pull request?
For SQL
```
SELECT TRANSFORM(a, b, c)
  ROW FORMAT DELIMITED
  FIELDS TERMINATED BY ','
  LINES TERMINATED BY '\n'
  NULL DEFINED AS 'null'
  USING 'cat' AS (a, b, c)
  ROW FORMAT DELIMITED
  FIELDS TERMINATED BY ','
  LINES TERMINATED BY '\n'
  NULL DEFINED AS 'NULL'
FROM testData
```
The correct

TOK_TABLEROWFORMATFIELD should be `, `nut actually ` ','`

TOK_TABLEROWFORMATLINES should be `\n`  but actually` '\n'`

### Why are the changes needed?
Fix string value format

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT

Closes #29487 from AngersZhuuuu/SPARK-32608-3.0.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/parser/ParserUtils.scala     |  5 ++
 .../spark/sql/execution/SparkSqlParser.scala  |  7 --
 .../sql/execution/SparkSqlParserSuite.scala   | 45 ++++++++++-
 .../execution/ScriptTransformationSuite.scala | 77 +++++++++++++++++--
 4 files changed, 118 insertions(+), 16 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
index a3779698a5ac3..f2dab941cb8b2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParserUtils.scala
@@ -83,6 +83,11 @@ object ParserUtils {
     node.getText.slice(1, node.getText.size - 1)
   }
 
+  /** Collect the entries if any. */
+  def entry(key: String, value: Token): Seq[(String, String)] = {
+    Option(value).toSeq.map(x => key -> string(x))
+  }
+
   /** Get the origin (line and position) of the token. */
   def position(token: Token): Origin = {
     val opt = Option(token)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index aa139cb6b0c3b..44069f3b20b4a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -587,10 +587,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
    */
   override def visitRowFormatDelimited(
       ctx: RowFormatDelimitedContext): CatalogStorageFormat = withOrigin(ctx) {
-    // Collect the entries if any.
-    def entry(key: String, value: Token): Seq[(String, String)] = {
-      Option(value).toSeq.map(x => key -> string(x))
-    }
     // TODO we need proper support for the NULL format.
     val entries =
       entry("field.delim", ctx.fieldsTerminatedBy) ++
@@ -689,9 +685,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
         // expects a seq of pairs in which the old parsers' token names are used as keys.
         // Transforming the result of visitRowFormatDelimited would be quite a bit messier than
         // retrieving the key value pairs ourselves.
-        def entry(key: String, value: Token): Seq[(String, String)] = {
-          Option(value).map(t => key -> t.getText).toSeq
-        }
         val entries = entry("TOK_TABLEROWFORMATFIELD", c.fieldsTerminatedBy) ++
           entry("TOK_TABLEROWFORMATCOLLITEMS", c.collectionItemsTerminatedBy) ++
           entry("TOK_TABLEROWFORMATMAPKEYS", c.keysTerminatedBy) ++
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
index 06574a9f8fd2c..6a58c8f3903e5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
@@ -21,8 +21,9 @@ import org.apache.spark.sql.SaveMode
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedAlias, UnresolvedAttribute, UnresolvedRelation, UnresolvedStar}
 import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStorageFormat, CatalogTable, CatalogTableType}
-import org.apache.spark.sql.catalyst.expressions.{Ascending, Concat, SortOrder}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, RepartitionByExpression, Sort}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Concat, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.execution.command._
 import org.apache.spark.sql.execution.datasources.{CreateTable, RefreshResource}
 import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
@@ -251,4 +252,44 @@ class SparkSqlParserSuite extends AnalysisTest {
     assertEqual("ADD FILE /path with space/abc.txt", AddFileCommand("/path with space/abc.txt"))
     assertEqual("ADD JAR /path with space/abc.jar", AddJarCommand("/path with space/abc.jar"))
   }
+
+  test("SPARK-32608: script transform with row format delimit") {
+    assertEqual(
+      """
+        |SELECT TRANSFORM(a, b, c)
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY ','
+        |  COLLECTION ITEMS TERMINATED BY '#'
+        |  MAP KEYS TERMINATED BY '@'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'null'
+        |  USING 'cat' AS (a, b, c)
+        |  ROW FORMAT DELIMITED
+        |  FIELDS TERMINATED BY ','
+        |  COLLECTION ITEMS TERMINATED BY '#'
+        |  MAP KEYS TERMINATED BY '@'
+        |  LINES TERMINATED BY '\n'
+        |  NULL DEFINED AS 'NULL'
+        |FROM testData
+      """.stripMargin,
+      ScriptTransformation(
+        Seq('a, 'b, 'c),
+        "cat",
+        Seq(AttributeReference("a", StringType)(),
+          AttributeReference("b", StringType)(),
+          AttributeReference("c", StringType)()),
+        UnresolvedRelation(TableIdentifier("testData")),
+        ScriptInputOutputSchema(
+          Seq(("TOK_TABLEROWFORMATFIELD", ","),
+            ("TOK_TABLEROWFORMATCOLLITEMS", "#"),
+            ("TOK_TABLEROWFORMATMAPKEYS", "@"),
+            ("TOK_TABLEROWFORMATLINES", "\n"),
+            ("TOK_TABLEROWFORMATNULL", "null")),
+          Seq(("TOK_TABLEROWFORMATFIELD", ","),
+            ("TOK_TABLEROWFORMATCOLLITEMS", "#"),
+            ("TOK_TABLEROWFORMATMAPKEYS", "@"),
+            ("TOK_TABLEROWFORMATLINES", "\n"),
+            ("TOK_TABLEROWFORMATNULL", "NULL")), None, None,
+          List.empty, List.empty, None, None, false)))
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
index b97eb869a9e54..15a932f6d660c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.{SparkPlan, SparkPlanTest, UnaryExecNode}
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.test.SQLTestUtils
@@ -61,6 +62,14 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
 
   private val uncaughtExceptionHandler = new TestUncaughtExceptionHandler
 
+  // In Hive 1.2, the string representation of a decimal omits trailing zeroes.
+  // But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
+  val decimalToString: Column => Column = if (HiveUtils.isHive23) {
+    c => c.cast("string")
+  } else {
+    c => c.cast("decimal(1, 0)").cast("string")
+  }
+
   protected override def beforeAll(): Unit = {
     super.beforeAll()
     defaultUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler
@@ -212,13 +221,6 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
           |FROM v
         """.stripMargin)
 
-      // In Hive 1.2, the string representation of a decimal omits trailing zeroes.
-      // But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
-      val decimalToString: Column => Column = if (HiveUtils.isHive23) {
-        c => c.cast("string")
-      } else {
-        c => c.cast("decimal(1, 0)").cast("string")
-      }
       checkAnswer(query, identity, df.select(
         'a.cast("string"),
         'b.cast("string"),
@@ -263,6 +265,67 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
     assert(e.getMessage.contains("Subprocess exited with status"))
     assert(uncaughtExceptionHandler.exception.isEmpty)
   }
+
+
+  test("SPARK-32608: Script Transform ROW FORMAT DELIMIT value should format value") {
+    withTempView("v") {
+      val df = Seq(
+        (1, "1", 1.0, BigDecimal(1.0), new Timestamp(1)),
+        (2, "2", 2.0, BigDecimal(2.0), new Timestamp(2)),
+        (3, "3", 3.0, BigDecimal(3.0), new Timestamp(3))
+      ).toDF("a", "b", "c", "d", "e") // Note column d's data type is Decimal(38, 18)
+      df.createTempView("v")
+
+      // input/output with same delimit
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, cast(e as string))
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  COLLECTION ITEMS TERMINATED BY '#'
+             |  MAP KEYS TERMINATED BY '@'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (a, b, c, d, e)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  COLLECTION ITEMS TERMINATED BY '#'
+             |  MAP KEYS TERMINATED BY '@'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          'a.cast("string"),
+          'b.cast("string"),
+          'c.cast("string"),
+          decimalToString('d),
+          'e.cast("string")).collect())
+
+      // input/output with different delimit and show result
+      checkAnswer(
+        sql(
+          s"""
+             |SELECT TRANSFORM(a, b, c, d, cast(e as string))
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY ','
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'null'
+             |  USING 'cat' AS (value)
+             |  ROW FORMAT DELIMITED
+             |  FIELDS TERMINATED BY '&'
+             |  LINES TERMINATED BY '\n'
+             |  NULL DEFINED AS 'NULL'
+             |FROM v
+        """.stripMargin), identity, df.select(
+          concat_ws(",",
+            'a.cast("string"),
+            'b.cast("string"),
+            'c.cast("string"),
+            decimalToString('d),
+            'e.cast("string"))).collect())
+    }
+  }
 }
 
 private case class ExceptionInjectingOperator(child: SparkPlan) extends UnaryExecNode {

From 29a10a456438be1ebd71e947e82fb13b1dec34be Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 20 Aug 2020 15:23:25 +0000
Subject: [PATCH 0986/1280] [SPARK-28863][SQL][FOLLOWUP] Do not reuse the
 physical plan

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/29469

Instead of passing the physical plan to the fallbacked v1 source directly and skipping analysis, optimization, planning altogether, this PR proposes to pass the optimized plan.

### Why are the changes needed?

It's a bit risky to pass the physical plan directly. When the fallbacked v1 source applies more operations to the input DataFrame, it will re-apply the post-planning physical rules like `CollapseCodegenStages`, `InsertAdaptiveSparkPlan`, etc., which is very tricky.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

existing test suite with some new tests

Closes #29489 from cloud-fan/follow.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit d378dc5f6db6fe37426728bea714f44b94a94861)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/execution/AlreadyOptimized.scala      | 37 +++++++++++
 .../spark/sql/execution/AlreadyPlanned.scala  | 61 -------------------
 .../spark/sql/execution/SparkStrategies.scala |  1 -
 .../datasources/v2/DataSourceV2Strategy.scala | 10 +--
 .../datasources/v2/V1FallbackWriters.scala    | 20 +++---
 .../v2/WriteToDataSourceV2Exec.scala          |  4 ++
 ...uite.scala => AlreadyOptimizedSuite.scala} | 31 ++++++----
 7 files changed, 77 insertions(+), 87 deletions(-)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala
 rename sql/core/src/test/scala/org/apache/spark/sql/execution/{AlreadyPlannedSuite.scala => AlreadyOptimizedSuite.scala} (68%)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala
new file mode 100644
index 0000000000000..e40b1141b43eb
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
+import org.apache.spark.sql.catalyst.encoders.RowEncoder
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+/** Query execution that skips re-analysis and optimize. */
+class AlreadyOptimizedExecution(
+    session: SparkSession,
+    plan: LogicalPlan) extends QueryExecution(session, plan) {
+  override lazy val analyzed: LogicalPlan = plan
+  override lazy val optimizedPlan: LogicalPlan = plan
+}
+
+object AlreadyOptimized {
+  def dataFrame(sparkSession: SparkSession, optimized: LogicalPlan): DataFrame = {
+    val qe = new AlreadyOptimizedExecution(sparkSession, optimized)
+    new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala
deleted file mode 100644
index 9dd956a9c2e08..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyPlanned.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution
-
-import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute}
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
-
-/**
- * A special node that allows skipping query planning all the way to physical execution. This node
- * can be used when a query was planned to the physical level, but we had to go back to logical plan
- * land for some reason (e.g. V1 DataSource write execution). This will allow the metrics, and the
- * query plan to properly appear as part of the query execution.
- */
-case class AlreadyPlanned(physicalPlan: SparkPlan) extends LogicalPlan with MultiInstanceRelation {
-  override def children: Seq[LogicalPlan] = Nil
-  override lazy val resolved: Boolean = true
-  override val output: Seq[Attribute] = physicalPlan.output
-  override def newInstance(): LogicalPlan = {
-    val newAttrs = output.map(a => a.exprId -> a.newInstance())
-    val attrMap = newAttrs.toMap
-    val projections = physicalPlan.output.map { o =>
-      Alias(o, o.name)(attrMap(o.exprId).exprId, o.qualifier, Option(o.metadata))
-    }
-    AlreadyPlanned(ProjectExec(projections, physicalPlan))
-  }
-}
-
-/** Query execution that skips re-analysis and planning. */
-class AlreadyPlannedExecution(
-    session: SparkSession,
-    plan: AlreadyPlanned) extends QueryExecution(session, plan) {
-  override lazy val analyzed: LogicalPlan = plan
-  override lazy val optimizedPlan: LogicalPlan = plan
-  override lazy val sparkPlan: SparkPlan = plan.physicalPlan
-  override lazy val executedPlan: SparkPlan = plan.physicalPlan
-}
-
-object AlreadyPlanned {
-  def dataFrame(sparkSession: SparkSession, query: SparkPlan): DataFrame = {
-    val qe = new AlreadyPlannedExecution(sparkSession, AlreadyPlanned(query))
-    new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index ec6e36535af1b..f836debcbafda 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -678,7 +678,6 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
   object BasicOperators extends Strategy {
     def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
-      case planned: AlreadyPlanned => planned.physicalPlan :: Nil
       case d: DataWritingCommand => DataWritingCommandExec(d, planLater(d.query)) :: Nil
       case r: RunnableCommand => ExecutedCommandExec(r) :: Nil
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index f2ff0060654f3..cca80c0cb6d57 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -122,10 +122,10 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
       val writeOptions = new CaseInsensitiveStringMap(options.asJava)
       catalog match {
         case staging: StagingTableCatalog =>
-          AtomicCreateTableAsSelectExec(staging, ident, parts, planLater(query),
+          AtomicCreateTableAsSelectExec(staging, ident, parts, query, planLater(query),
             propsWithOwner, writeOptions, ifNotExists) :: Nil
         case _ =>
-          CreateTableAsSelectExec(catalog, ident, parts, planLater(query),
+          CreateTableAsSelectExec(catalog, ident, parts, query, planLater(query),
             propsWithOwner, writeOptions, ifNotExists) :: Nil
       }
 
@@ -152,6 +152,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             staging,
             ident,
             parts,
+            query,
             planLater(query),
             propsWithOwner,
             writeOptions,
@@ -161,6 +162,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
             catalog,
             ident,
             parts,
+            query,
             planLater(query),
             propsWithOwner,
             writeOptions,
@@ -170,7 +172,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
     case AppendData(r: DataSourceV2Relation, query, writeOptions, _) =>
       r.table.asWritable match {
         case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
-          AppendDataExecV1(v1, writeOptions.asOptions, planLater(query)) :: Nil
+          AppendDataExecV1(v1, writeOptions.asOptions, query) :: Nil
         case v2 =>
           AppendDataExec(v2, writeOptions.asOptions, planLater(query)) :: Nil
       }
@@ -184,7 +186,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
       }.toArray
       r.table.asWritable match {
         case v1 if v1.supports(TableCapability.V1_BATCH_WRITE) =>
-          OverwriteByExpressionExecV1(v1, filters, writeOptions.asOptions, planLater(query)) :: Nil
+          OverwriteByExpressionExecV1(v1, filters, writeOptions.asOptions, query) :: Nil
         case v2 =>
           OverwriteByExpressionExec(v2, filters, writeOptions.asOptions, planLater(query)) :: Nil
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
index 28c4e28004446..560da39314b36 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
@@ -22,9 +22,10 @@ import java.util.UUID
 import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.connector.catalog.SupportsWrite
 import org.apache.spark.sql.connector.write.{LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
-import org.apache.spark.sql.execution.{AlreadyPlanned, SparkPlan}
+import org.apache.spark.sql.execution.{AlreadyOptimized, SparkPlan}
 import org.apache.spark.sql.sources.{AlwaysTrue, Filter, InsertableRelation}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -36,7 +37,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 case class AppendDataExecV1(
     table: SupportsWrite,
     writeOptions: CaseInsensitiveStringMap,
-    query: SparkPlan) extends V1FallbackWriters {
+    plan: LogicalPlan) extends V1FallbackWriters {
 
   override protected def run(): Seq[InternalRow] = {
     writeWithV1(newWriteBuilder().buildForV1Write())
@@ -58,7 +59,7 @@ case class OverwriteByExpressionExecV1(
     table: SupportsWrite,
     deleteWhere: Array[Filter],
     writeOptions: CaseInsensitiveStringMap,
-    query: SparkPlan) extends V1FallbackWriters {
+    plan: LogicalPlan) extends V1FallbackWriters {
 
   private def isTruncate(filters: Array[Filter]): Boolean = {
     filters.length == 1 && filters(0).isInstanceOf[AlwaysTrue]
@@ -81,7 +82,7 @@ case class OverwriteByExpressionExecV1(
 /** Some helper interfaces that use V2 write semantics through the V1 writer interface. */
 sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
   override def output: Seq[Attribute] = Nil
-  override final def children: Seq[SparkPlan] = Seq(query)
+  override final def children: Seq[SparkPlan] = Nil
 
   def table: SupportsWrite
   def writeOptions: CaseInsensitiveStringMap
@@ -97,9 +98,11 @@ sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
   protected def newWriteBuilder(): V1WriteBuilder = {
     val info = LogicalWriteInfoImpl(
       queryId = UUID.randomUUID().toString,
-      schema = query.schema,
+      schema = plan.schema,
       options = writeOptions)
-    table.newWriteBuilder(info).asV1Builder
+    val writeBuilder = table.newWriteBuilder(info)
+
+    writeBuilder.asV1Builder
   }
 }
 
@@ -107,10 +110,11 @@ sealed trait V1FallbackWriters extends V2CommandExec with SupportsV1Write {
  * A trait that allows Tables that use V1 Writer interfaces to append data.
  */
 trait SupportsV1Write extends SparkPlan {
-  def query: SparkPlan
+  def plan: LogicalPlan
 
   protected def writeWithV1(relation: InsertableRelation): Seq[InternalRow] = {
-    relation.insert(AlreadyPlanned.dataFrame(sqlContext.sparkSession, query), overwrite = false)
+    // The `plan` is already optimized, we should not analyze and optimize it again.
+    relation.insert(AlreadyOptimized.dataFrame(sqlContext.sparkSession, plan), overwrite = false)
     Nil
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
index bf961329a5d14..616e18ee85a6b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/WriteToDataSourceV2Exec.scala
@@ -62,6 +62,7 @@ case class CreateTableAsSelectExec(
     catalog: TableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
+    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
@@ -118,6 +119,7 @@ case class AtomicCreateTableAsSelectExec(
     catalog: StagingTableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
+    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
@@ -151,6 +153,7 @@ case class ReplaceTableAsSelectExec(
     catalog: TableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
+    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
@@ -216,6 +219,7 @@ case class AtomicReplaceTableAsSelectExec(
     catalog: StagingTableCatalog,
     ident: Identifier,
     partitioning: Seq[Transform],
+    plan: LogicalPlan,
     query: SparkPlan,
     properties: Map[String, String],
     writeOptions: CaseInsensitiveStringMap,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala
similarity index 68%
rename from sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala
rename to sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala
index 9152dcf7a3697..c266aa92f01cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyPlannedSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala
@@ -17,40 +17,47 @@
 
 package org.apache.spark.sql.execution
 
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.execution.adaptive.EnableAdaptiveExecutionSuite
 import org.apache.spark.sql.test.SharedSparkSession
 
-class AlreadyPlannedSuite extends SparkPlanTest with SharedSparkSession {
+class AlreadyOptimizedSuite extends QueryTest with SharedSparkSession {
 
   import testImplicits._
 
   test("simple execution") {
     val df = spark.range(10)
-    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
 
-    checkAnswer(planned, identity, df.toDF().collect())
+    checkAnswer(planned, df.toDF().collect())
   }
 
   test("planning on top works - projection") {
     val df = spark.range(10)
-    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
 
     checkAnswer(
       planned.withColumn("data", 'id + 1),
-      identity,
       df.withColumn("data", 'id + 1).collect())
   }
 
   test("planning on top works - filter") {
     val df = spark.range(10)
-    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
 
-    checkAnswer(planned.where('id < 5), identity, df.where('id < 5).toDF().collect())
+    checkAnswer(planned.where('id < 5), df.where('id < 5).toDF().collect())
+  }
+
+  test("planning on top works - aggregate") {
+    val df = spark.range(10)
+    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
+
+    checkAnswer(planned.groupBy('id).count(), df.groupBy('id).count().collect())
   }
 
   test("planning on top works - joins") {
     val df = spark.range(10)
-    val planned = AlreadyPlanned.dataFrame(spark, df.queryExecution.sparkPlan)
+    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
 
     val plannedLeft = planned.alias("l")
     val dfLeft = df.alias("l")
@@ -59,22 +66,20 @@ class AlreadyPlannedSuite extends SparkPlanTest with SharedSparkSession {
 
     checkAnswer(
       plannedLeft.where('id < 3).join(plannedRight, Seq("id")),
-      identity,
       dfLeft.where('id < 3).join(dfRight, Seq("id")).collect())
 
     checkAnswer(
       plannedLeft.where('id < 3).join(plannedRight, plannedLeft("id") === plannedRight("id")),
-      identity,
       dfLeft.where('id < 3).join(dfRight, dfLeft("id") === dfRight("id")).collect())
 
     checkAnswer(
       plannedLeft.join(plannedRight, Seq("id")).where('id < 3),
-      identity,
       dfLeft.join(dfRight, Seq("id")).where('id < 3).collect())
 
     checkAnswer(
       plannedLeft.join(plannedRight, plannedLeft("id") === plannedRight("id")).where($"l.id" < 3),
-      identity,
       dfLeft.join(dfRight, dfLeft("id") === dfRight("id")).where($"l.id" < 3).collect())
   }
 }
+
+class AlreadyOptimizedAQESuite extends AlreadyOptimizedSuite with EnableAdaptiveExecutionSuite

From 8755e3f42f2abf8cbbf2be02dafefdab3d3a1766 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Thu, 20 Aug 2020 17:18:29 +0000
Subject: [PATCH 0987/1280] [SPARK-32621][SQL][3.0] path' option can cause
 issues while inferring schema in CSV/JSON datasources

### What changes were proposed in this pull request?

This PR is backporting #29437 to branch-3.0.

When CSV/JSON datasources infer schema (e.g, `def inferSchema(files: Seq[FileStatus])`, they use the `files` along with the original options. `files` in `inferSchema` could have been deduced from the "path" option if the option was present, so this can cause issues (e.g., reading more data, listing the path again) since the "path" option is **added** to the `files`.

### Why are the changes needed?

The current behavior can cause the following issue:
```scala
class TestFileFilter extends PathFilter {
  override def accept(path: Path): Boolean = path.getParent.getName != "p=2"
}

val path = "/tmp"
val df = spark.range(2)
df.write.json(path + "/p=1")
df.write.json(path + "/p=2")

val extraOptions = Map(
  "mapred.input.pathFilter.class" -> classOf[TestFileFilter].getName,
  "mapreduce.input.pathFilter.class" -> classOf[TestFileFilter].getName
)

// This works fine.
assert(spark.read.options(extraOptions).json(path).count == 2)

// The following with "path" option fails with the following:
// assertion failed: Conflicting directory structures detected. Suspicious paths
//	file:/tmp
//	file:/tmp/p=1
assert(spark.read.options(extraOptions).format("json").option("path", path).load.count() === 2)
```

### Does this PR introduce _any_ user-facing change?

Yes, the above failure doesn't happen and you get the consistent experience when you use `spark.read.csv(path)` or `spark.read.format("csv").option("path", path).load`.

### How was this patch tested?

Updated existing tests.

Closes #29478 from imback82/backport-29437.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/v2/avro/AvroDataSourceV2.scala  |  6 +++--
 .../execution/datasources/DataSource.scala    |  8 +++++--
 .../datasources/csv/CSVDataSource.scala       |  2 +-
 .../datasources/json/JsonDataSource.scala     |  2 +-
 .../datasources/v2/FileDataSourceV2.scala     |  7 ++++++
 .../datasources/v2/csv/CSVDataSourceV2.scala  |  6 +++--
 .../v2/json/JsonDataSourceV2.scala            |  6 +++--
 .../datasources/v2/orc/OrcDataSourceV2.scala  |  6 +++--
 .../v2/parquet/ParquetDataSourceV2.scala      |  7 ++++--
 .../v2/text/TextDataSourceV2.scala            |  6 +++--
 .../execution/datasources/csv/CSVSuite.scala  | 16 +++++++++++--
 .../datasources/json/JsonSuite.scala          | 24 ++++++++++++++++---
 12 files changed, 75 insertions(+), 21 deletions(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
index 969dee0a39696..2d21c493d5711 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/v2/avro/AvroDataSourceV2.scala
@@ -32,12 +32,14 @@ class AvroDataSourceV2 extends FileDataSourceV2 {
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    AvroTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    AvroTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    AvroTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    AvroTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
index 588a9b4eb4d09..e97139ffbd071 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
@@ -190,9 +190,11 @@ case class DataSource(
     val dataSchema = userSpecifiedSchema.map { schema =>
       StructType(schema.filterNot(f => partitionSchema.exists(p => equality(p.name, f.name))))
     }.orElse {
+      // Remove "path" option so that it is not added to the paths returned by
+      // `tempFileIndex.allFiles()`.
       format.inferSchema(
         sparkSession,
-        caseInsensitiveOptions,
+        caseInsensitiveOptions - "path",
         tempFileIndex.allFiles())
     }.getOrElse {
       throw new AnalysisException(
@@ -366,9 +368,11 @@ case class DataSource(
         val fileCatalog = new MetadataLogFileIndex(sparkSession, basePath,
           caseInsensitiveOptions, userSpecifiedSchema)
         val dataSchema = userSpecifiedSchema.orElse {
+          // Remove "path" option so that it is not added to the paths returned by
+          // `fileCatalog.allFiles()`.
           format.inferSchema(
             sparkSession,
-            caseInsensitiveOptions,
+            caseInsensitiveOptions - "path",
             fileCatalog.allFiles())
         }.getOrElse {
           throw new AnalysisException(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
index 375cec597166c..e8005a8d1f8c6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
@@ -156,7 +156,7 @@ object TextInputCSVDataSource extends CSVDataSource {
           sparkSession,
           paths = paths,
           className = classOf[TextFileFormat].getName,
-          options = options.parameters
+          options = options.parameters.originalMap
         ).resolveRelation(checkFilesExist = false))
         .select("value").as[String](Encoders.STRING)
     } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
index 7ec2267e3461f..1d2e63c16ab97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
@@ -120,7 +120,7 @@ object TextInputJsonDataSource extends JsonDataSource {
         sparkSession,
         paths = inputPaths.map(_.getPath.toString),
         className = classOf[TextFileFormat].getName,
-        options = parsedOptions.parameters
+        options = parsedOptions.parameters.originalMap
       ).resolveRelation(checkFilesExist = false))
       .select("value").as(Encoders.STRING)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
index bbe8835049fa4..fa871fe866fcd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala
@@ -56,6 +56,13 @@ trait FileDataSourceV2 extends TableProvider with DataSourceRegister {
     paths ++ Option(map.get("path")).toSeq
   }
 
+  protected def getOptionsWithoutPaths(map: CaseInsensitiveStringMap): CaseInsensitiveStringMap = {
+    val withoutPath = map.asCaseSensitiveMap().asScala.filterKeys { k =>
+      !k.equalsIgnoreCase("path") && !k.equalsIgnoreCase("paths")
+    }
+    new CaseInsensitiveStringMap(withoutPath.asJava)
+  }
+
   protected def getTableName(map: CaseInsensitiveStringMap, paths: Seq[String]): String = {
     val hadoopConf = sparkSession.sessionState.newHadoopConfWithOptions(
       map.asCaseSensitiveMap().asScala.toMap)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
index 69d001b4a615c..c577cbf8d756a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/csv/CSVDataSourceV2.scala
@@ -32,12 +32,14 @@ class CSVDataSourceV2 extends FileDataSourceV2 {
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    CSVTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    CSVTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    CSVTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    CSVTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
index 9c4e3b8c78026..cd0eba0ca9c51 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/json/JsonDataSourceV2.scala
@@ -32,13 +32,15 @@ class JsonDataSourceV2 extends FileDataSourceV2 {
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    JsonTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    JsonTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    JsonTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    JsonTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
index fa2febdc5a984..6303723e4b805 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcDataSourceV2.scala
@@ -32,13 +32,15 @@ class OrcDataSourceV2 extends FileDataSourceV2 {
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    OrcTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    OrcTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    OrcTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    OrcTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
index 7e7ca964de28f..4590660bc7b7e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/parquet/ParquetDataSourceV2.scala
@@ -32,13 +32,16 @@ class ParquetDataSourceV2 extends FileDataSourceV2 {
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    ParquetTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    ParquetTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    ParquetTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    ParquetTable(
+      tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
index 43bcb61f25962..f375a128d8177 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/text/TextDataSourceV2.scala
@@ -32,13 +32,15 @@ class TextDataSourceV2 extends FileDataSourceV2 {
   override def getTable(options: CaseInsensitiveStringMap): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    TextTable(tableName, sparkSession, options, paths, None, fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    TextTable(tableName, sparkSession, optionsWithoutPaths, paths, None, fallbackFileFormat)
   }
 
   override def getTable(options: CaseInsensitiveStringMap, schema: StructType): Table = {
     val paths = getPaths(options)
     val tableName = getTableName(options, paths)
-    TextTable(tableName, sparkSession, options, paths, Some(schema), fallbackFileFormat)
+    val optionsWithoutPaths = getOptionsWithoutPaths(options)
+    TextTable(tableName, sparkSession, optionsWithoutPaths, paths, Some(schema), fallbackFileFormat)
   }
 }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 366cf11871fa0..d54146a0fb065 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -1450,10 +1450,22 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       val ds = sampledTestData.coalesce(1)
       ds.write.text(path.getAbsolutePath)
 
-      val readback = spark.read
+      val readback1 = spark.read
         .option("inferSchema", true).option("samplingRatio", 0.1)
         .csv(path.getCanonicalPath)
-      assert(readback.schema == new StructType().add("_c0", IntegerType))
+      assert(readback1.schema == new StructType().add("_c0", IntegerType))
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option gets added again to the paths that have already been listed.
+        // This results in reading more data than necessary and causes different schema to be
+        // inferred when sampling ratio is involved.
+        val readback2 = spark.read
+          .option("inferSchema", true).option("samplingRatio", 0.1)
+          .option("path", path.getCanonicalPath)
+          .format("csv")
+          .load
+        assert(readback2.schema == new StructType().add("_c0", IntegerType))
+      }
     })
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index d95d4e08f64a3..38b5e775dffbf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -1556,6 +1556,15 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
         "mapreduce.input.pathFilter.class" -> classOf[TestFileFilter].getName
       )
       assert(spark.read.options(extraOptions).json(path).count() === 2)
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option is used again on top of the paths that have already been
+        // listed. When a partition is removed by TestFileFilter, this will cause a conflict while
+        // inferring partitions because the original path in the "path" option will list the
+        // partition directory that has been removed.
+        assert(
+          spark.read.options(extraOptions).format("json").option("path", path).load.count() === 2)
+      }
     }
   }
 
@@ -2141,9 +2150,18 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
     )(withTempPath { path =>
       val ds = sampledTestData.coalesce(1)
       ds.write.text(path.getAbsolutePath)
-      val readback = spark.read.option("samplingRatio", 0.1).json(path.getCanonicalPath)
-
-      assert(readback.schema == new StructType().add("f1", LongType))
+      val readback1 = spark.read.option("samplingRatio", 0.1).json(path.getCanonicalPath)
+      assert(readback1.schema == new StructType().add("f1", LongType))
+
+      withClue("SPARK-32621: 'path' option can cause issues while inferring schema") {
+        // During infer, "path" option gets added again to the paths that have already been listed.
+        // This results in reading more data than necessary and causes different schema to be
+        // inferred when sampling ratio is involved.
+        val readback2 = spark.read
+          .option("samplingRatio", 0.1).option("path", path.getCanonicalPath)
+          .format("json").load
+        assert(readback2.schema == new StructType().add("f1", LongType))
+      }
     })
   }
 

From 2932926a0ed8456ad73e1d34bee8ccc03293c635 Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Fri, 21 Aug 2020 13:12:43 +0800
Subject: [PATCH 0988/1280] [SPARK-32660][SQL][DOC] Show Avro related API in
 documentation

### What changes were proposed in this pull request?

Currently, the Avro related APIs are missing in the documentation https://spark.apache.org/docs/latest/api/scala/org/apache/spark/index.html . This PR is to:
1. Mark internal Avro related classes as private
2. Show Avro related API in Spark official API documentation

### Why are the changes needed?

Better documentation.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Build doc and preview:
![image](https://user-images.githubusercontent.com/1097932/90623042-d156ee00-e1ca-11ea-9edd-2c45b3001fd8.png)

![image](https://user-images.githubusercontent.com/1097932/90623047-d451de80-e1ca-11ea-94ba-02921b64d6f1.png)

![image](https://user-images.githubusercontent.com/1097932/90623058-d6b43880-e1ca-11ea-849a-b9ea9efe6527.png)

Closes #29476 from gengliangwang/avroAPIDoc.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit de141a32714fd2dbc4be2d540adabf328bbce2c4)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../sql/avro/SparkAvroKeyOutputFormat.java    |  4 ++--
 .../spark/sql/avro/AvroDataToCatalyst.scala   |  2 +-
 .../spark/sql/avro/AvroDeserializer.scala     |  2 +-
 .../apache/spark/sql/avro/AvroOptions.scala   |  4 ++--
 .../spark/sql/avro/AvroSerializer.scala       |  2 +-
 .../org/apache/spark/sql/avro/AvroUtils.scala |  2 +-
 .../spark/sql/avro/CatalystDataToAvro.scala   |  2 +-
 .../spark/sql/avro/SchemaConverters.scala     | 21 ++++++++++++++++---
 project/SparkBuild.scala                      |  5 +++--
 9 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
index 55696a6ac2e0f..a4555844b5117 100644
--- a/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
+++ b/external/avro/src/main/java/org/apache/spark/sql/avro/SparkAvroKeyOutputFormat.java
@@ -35,8 +35,8 @@
 
 // A variant of `AvroKeyOutputFormat`, which is used to inject the custom `RecordWriterFactory` so
 // that we can set avro file metadata.
-public class SparkAvroKeyOutputFormat extends AvroKeyOutputFormat<GenericRecord> {
-  public SparkAvroKeyOutputFormat(Map<String, String> metadata) {
+class SparkAvroKeyOutputFormat extends AvroKeyOutputFormat<GenericRecord> {
+  SparkAvroKeyOutputFormat(Map<String, String> metadata) {
     super(new SparkRecordWriterFactory(metadata));
   }
 
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
index 79c72057c5823..0fdf5c6af13b9 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGe
 import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMode}
 import org.apache.spark.sql.types._
 
-case class AvroDataToCatalyst(
+private[avro] case class AvroDataToCatalyst(
     child: Expression,
     jsonFormatSchema: String,
     options: Map[String, String])
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
index 4fc804067d54f..7580957a0de36 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDeserializer.scala
@@ -42,7 +42,7 @@ import org.apache.spark.unsafe.types.UTF8String
 /**
  * A deserializer to deserialize data in avro format to data in catalyst format.
  */
-class AvroDeserializer(
+private[sql] class AvroDeserializer(
     rootAvroType: Schema,
     rootCatalystType: DataType,
     datetimeRebaseMode: LegacyBehaviorPolicy.Value) {
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
index f3ea78583fbc0..8972b0553c522 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.internal.SQLConf
 /**
  * Options for Avro Reader and Writer stored in case insensitive manner.
  */
-class AvroOptions(
+private[sql] class AvroOptions(
     @transient val parameters: CaseInsensitiveMap[String],
     @transient val conf: Configuration) extends Logging with Serializable {
 
@@ -95,7 +95,7 @@ class AvroOptions(
     parameters.get("mode").map(ParseMode.fromString).getOrElse(FailFastMode)
 }
 
-object AvroOptions {
+private[sql] object AvroOptions {
   def apply(parameters: Map[String, String]): AvroOptions = {
     val hadoopConf = SparkSession
       .getActiveSession
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
index d6cfbc5ffe758..2c0bffa509ec8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroSerializer.scala
@@ -43,7 +43,7 @@ import org.apache.spark.sql.types._
 /**
  * A serializer to serialize data in catalyst format to data in avro format.
  */
-class AvroSerializer(
+private[sql] class AvroSerializer(
     rootCatalystType: DataType,
     rootAvroType: Schema,
     nullable: Boolean,
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
index 70dcd58a600fc..9ff89f60c05f8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
@@ -37,7 +37,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
-object AvroUtils extends Logging {
+private[sql] object AvroUtils extends Logging {
   def inferSchema(
       spark: SparkSession,
       options: Map[String, String],
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
index 7732c83aaf7c4..53910b752fdd6 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.types.{BinaryType, DataType}
 
-case class CatalystDataToAvro(
+private[avro] case class CatalystDataToAvro(
     child: Expression,
     jsonFormatSchema: Option[String]) extends UnaryExpression {
 
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
index 3947d327dfac6..0e01fbf6fe2e8 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/SchemaConverters.scala
@@ -24,6 +24,7 @@ import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 import org.apache.avro.LogicalTypes.{Date, Decimal, TimestampMicros, TimestampMillis}
 import org.apache.avro.Schema.Type._
 
+import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.sql.catalyst.util.RandomUUIDGenerator
 import org.apache.spark.sql.types._
 import org.apache.spark.sql.types.Decimal.{maxPrecisionForBytes, minBytesForPrecision}
@@ -32,21 +33,29 @@ import org.apache.spark.sql.types.Decimal.{maxPrecisionForBytes, minBytesForPrec
  * This object contains method that are used to convert sparkSQL schemas to avro schemas and vice
  * versa.
  */
+@DeveloperApi
 object SchemaConverters {
   private lazy val uuidGenerator = RandomUUIDGenerator(new Random().nextLong())
 
   private lazy val nullSchema = Schema.create(Schema.Type.NULL)
 
+  /**
+   * Internal wrapper for SQL data type and nullability.
+   *
+   * @since 2.4.0
+   */
   case class SchemaType(dataType: DataType, nullable: Boolean)
 
   /**
-   * This function takes an avro schema and returns a sql schema.
+   * Converts an Avro schema to a corresponding Spark SQL schema.
+   *
+   * @since 2.4.0
    */
   def toSqlType(avroSchema: Schema): SchemaType = {
     toSqlTypeHelper(avroSchema, Set.empty)
   }
 
-  def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
+  private def toSqlTypeHelper(avroSchema: Schema, existingRecordNames: Set[String]): SchemaType = {
     avroSchema.getType match {
       case INT => avroSchema.getLogicalType match {
         case _: Date => SchemaType(DateType, nullable = false)
@@ -133,6 +142,11 @@ object SchemaConverters {
     }
   }
 
+  /**
+   * Converts a Spark SQL schema to a corresponding Avro schema.
+   *
+   * @since 2.4.0
+   */
   def toAvroType(
       catalystType: DataType,
       nullable: Boolean = false,
@@ -192,4 +206,5 @@ object SchemaConverters {
   }
 }
 
-class IncompatibleSchemaException(msg: String, ex: Throwable = null) extends Exception(msg, ex)
+private[avro] class IncompatibleSchemaException(
+  msg: String, ex: Throwable = null) extends Exception(msg, ex)
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 7a9634ac0fcbb..a774a1100fd58 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -847,6 +847,7 @@ object Unidoc {
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive/test")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalog/v2/utils")))
       .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hive")))
+      .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/v2/avro")))
   }
 
   private def ignoreClasspaths(classpaths: Seq[Classpath]): Seq[Classpath] = {
@@ -862,10 +863,10 @@ object Unidoc {
 
     unidocProjectFilter in(ScalaUnidoc, unidoc) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
-        yarn, tags, streamingKafka010, sqlKafka010, avro),
+        yarn, tags, streamingKafka010, sqlKafka010),
     unidocProjectFilter in(JavaUnidoc, unidoc) :=
       inAnyProject -- inProjects(OldDeps.project, repl, examples, tools, kubernetes,
-        yarn, tags, streamingKafka010, sqlKafka010, avro),
+        yarn, tags, streamingKafka010, sqlKafka010),
 
     unidocAllClasspaths in (ScalaUnidoc, unidoc) := {
       ignoreClasspaths((unidocAllClasspaths in (ScalaUnidoc, unidoc)).value)

From f73e6caa2392a2aa456ec5951fbf15cb514af3ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cattilapiros=E2=80=9D?=
 <piros.attila.zsolt@gmail.com>
Date: Fri, 21 Aug 2020 01:02:33 -0500
Subject: [PATCH 0989/1280] [SPARK-32663][CORE] Avoid individual closing of
 pooled TransportClients (which must be closed through the pool)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

Removing the individual `close` method calls on the pooled `TransportClient` instances.
The pooled clients should be only closed via `TransportClientFactory#close()`.

### Why are the changes needed?

Reusing a closed `TransportClient` leads to the exception `java.nio.channels.ClosedChannelException`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

This is a trivial case which is not tested by specific test.

Closes #29492 from attilapiros/SPARK-32663.

Authored-by: “attilapiros” <piros.attila.zsolt@gmail.com>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
(cherry picked from commit 79b4dea1b08adc9d4b545a1af29ac50fa603936a)
Signed-off-by: Mridul Muralidharan <mridulatgmail.com>
---
 .../spark/network/shuffle/ExternalBlockStoreClient.java     | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
index d6185f089d3c0..98247f24a036b 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalBlockStoreClient.java
@@ -175,8 +175,6 @@ public void onSuccess(ByteBuffer response) {
           logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) +
             " via external shuffle service from executor: " + execId, t);
           numRemovedBlocksFuture.complete(0);
-        } finally {
-          client.close();
         }
       }
 
@@ -185,7 +183,6 @@ public void onFailure(Throwable e) {
         logger.warn("Error trying to remove RDD blocks " + Arrays.toString(blockIds) +
           " via external shuffle service from executor: " + execId, e);
         numRemovedBlocksFuture.complete(0);
-        client.close();
       }
     });
     return numRemovedBlocksFuture;
@@ -212,8 +209,6 @@ public void onSuccess(ByteBuffer response) {
               Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service",
               t.getCause());
             hostLocalDirsCompletable.completeExceptionally(t);
-          } finally {
-            client.close();
           }
         }
 
@@ -223,7 +218,6 @@ public void onFailure(Throwable t) {
             Arrays.toString(getLocalDirsMessage.execIds) + " via external shuffle service",
             t.getCause());
           hostLocalDirsCompletable.completeExceptionally(t);
-          client.close();
         }
       });
     } catch (IOException | InterruptedException e) {

From a5f4230f619673da882da01b5d6acd9ba20a79e3 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apache.org>
Date: Fri, 21 Aug 2020 16:48:54 +0900
Subject: [PATCH 0990/1280] [SPARK-32674][DOC] Add suggestion for parallel
 directory listing in tuning doc

### What changes were proposed in this pull request?

This adds some tuning guide for increasing parallelism of directory listing.

### Why are the changes needed?

Sometimes when job input has large number of directories, the listing can become a bottleneck. There are a few parameters to tune this. This adds some info to Spark tuning guide to make the knowledge better shared.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

N/A

Closes #29498 from sunchao/SPARK-32674.

Authored-by: Chao Sun <sunchao@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit bf221debd02b11003b092201d0326302196e4ba5)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-performance-tuning.md | 22 ++++++++++++++++++++++
 docs/tuning.md                 | 11 +++++++++++
 2 files changed, 33 insertions(+)

diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 5e6f049a51e95..5d8c3b698c703 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -114,6 +114,28 @@ that these options will be deprecated in future release as more optimizations ar
     </td>
     <td>1.1.0</td>
   </tr>
+  <tr>
+    <td><code>spark.sql.sources.parallelPartitionDiscovery.threshold</code></td>
+    <td>32</td>
+    <td>
+      Configures the threshold to enable parallel listing for job input paths. If the number of
+      input paths is larger than this threshold, Spark will list the files by using Spark distributed job.
+      Otherwise, it will fallback to sequential listing. This configuration is only effective when
+      using file-based data sources such as Parquet, ORC and JSON.
+    </td>
+    <td>1.5.0</td>
+  </tr>
+  <tr>
+    <td><code>spark.sql.sources.parallelPartitionDiscovery.parallelism</code></td>
+    <td>10000</td>
+    <td>
+      Configures the maximum listing parallelism for job input paths. In case the number of input
+      paths is larger than this value, it will be throttled down to use this value. Same as above,
+      this configuration is only effective when using file-based data sources such as Parquet, ORC
+      and JSON.
+    </td>
+    <td>2.1.1</td>
+  </tr>
 </table>
 
 ## Join Strategy Hints for SQL Queries
diff --git a/docs/tuning.md b/docs/tuning.md
index 8e29e5d2e9e74..18d4a6205f4ff 100644
--- a/docs/tuning.md
+++ b/docs/tuning.md
@@ -264,6 +264,17 @@ parent RDD's number of partitions. You can pass the level of parallelism as a se
 or set the config property `spark.default.parallelism` to change the default.
 In general, we recommend 2-3 tasks per CPU core in your cluster.
 
+## Parallel Listing on Input Paths
+
+Sometimes you may also need to increase directory listing parallelism when job input has large number of directories,
+otherwise the process could take a very long time, especially when against object store like S3.
+If your job works on RDD with Hadoop input formats (e.g., via `SparkContext.sequenceFile`), the parallelism is
+controlled via [`spark.hadoop.mapreduce.input.fileinputformat.list-status.num-threads`](https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml) (currently default is 1).
+
+For Spark SQL with file-based data sources, you can tune `spark.sql.sources.parallelPartitionDiscovery.threshold` and
+`spark.sql.sources.parallelPartitionDiscovery.parallelism` to improve listing parallelism. Please
+refer to [Spark SQL performance tuning guide](sql-performance-tuning.html) for more details.
+
 ## Memory Usage of Reduce Tasks
 
 Sometimes, you will get an OutOfMemoryError not because your RDDs don't fit in memory, but because the

From 9ccc79036f0c562b4b3d0e38c8fdfbb70f6e3319 Mon Sep 17 00:00:00 2001
From: Brandon Jiang <brandonJY@users.noreply.github.com>
Date: Sat, 22 Aug 2020 10:08:39 +0900
Subject: [PATCH 0991/1280] [MINOR][DOCS] backport PR#29443 to fix typo in
 doc,log messages and comments

### What changes were proposed in this pull request?
backport PR #29443 to fix typo for docs, log messages and comments

### Why are the changes needed?
typo fix to increase readability

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
manual test has been performed to test the updated

Closes #29512 from brandonJY/branch-3.0.

Authored-by: Brandon Jiang <brandonJY@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/java/org/apache/spark/network/util/TransportConf.java  | 2 +-
 .../src/main/java/org/apache/spark/api/plugin/DriverPlugin.java | 2 +-
 .../apache/spark/resource/ResourceDiscoveryScriptPlugin.scala   | 2 +-
 docs/job-scheduling.md                                          | 2 +-
 docs/sql-ref-syntax-qry-select-groupby.md                       | 2 +-
 docs/sql-ref-syntax-qry-select-hints.md                         | 2 +-
 docs/sql-ref.md                                                 | 2 +-
 .../src/main/java/org/apache/spark/launcher/LauncherServer.java | 2 +-
 .../org/apache/spark/sql/connector/catalog/TableCatalog.java    | 2 +-
 .../org/apache/spark/sql/catalyst/QueryPlanningTracker.scala    | 2 +-
 .../sql/execution/datasources/v2/ShowTablePropertiesExec.scala  | 2 +-
 11 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index 6c37f9a382376..646e4278811f4 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -290,7 +290,7 @@ public boolean sharedByteBufAllocators() {
   }
 
   /**
-  * If enabled then off-heap byte buffers will be prefered for the shared ByteBuf allocators.
+  * If enabled then off-heap byte buffers will be preferred for the shared ByteBuf allocators.
   */
   public boolean preferDirectBufsForSharedByteBufAllocators() {
     return conf.getBoolean("spark.network.io.preferDirectBufs", true);
diff --git a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
index 0c0d0df8ae682..1d676ff781c70 100644
--- a/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
+++ b/core/src/main/java/org/apache/spark/api/plugin/DriverPlugin.java
@@ -41,7 +41,7 @@ public interface DriverPlugin {
    * initialization.
    * <p>
    * It's recommended that plugins be careful about what operations are performed in this call,
-   * preferrably performing expensive operations in a separate thread, or postponing them until
+   * preferably performing expensive operations in a separate thread, or postponing them until
    * the application has fully started.
    *
    * @param sc The SparkContext loading the plugin.
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
index 11a9bb86d3034..d861e91771673 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceDiscoveryScriptPlugin.scala
@@ -29,7 +29,7 @@ import org.apache.spark.util.Utils.executeAndGetOutput
 /**
  * The default plugin that is loaded into a Spark application to control how custom
  * resources are discovered. This executes the discovery script specified by the user
- * and gets the json output back and contructs ResourceInformation objects from that.
+ * and gets the json output back and constructs ResourceInformation objects from that.
  * If the user specifies custom plugins, this is the last one to be executed and
  * throws if the resource isn't discovered.
  *
diff --git a/docs/job-scheduling.md b/docs/job-scheduling.md
index eaacfa49c657c..5c19c77f37a81 100644
--- a/docs/job-scheduling.md
+++ b/docs/job-scheduling.md
@@ -95,7 +95,7 @@ varies across cluster managers:
 In standalone mode, simply start your workers with `spark.shuffle.service.enabled` set to `true`.
 
 In Mesos coarse-grained mode, run `$SPARK_HOME/sbin/start-mesos-shuffle-service.sh` on all
-slave nodes with `spark.shuffle.service.enabled` set to `true`. For instance, you may do so
+worker nodes with `spark.shuffle.service.enabled` set to `true`. For instance, you may do so
 through Marathon.
 
 In YARN mode, follow the instructions [here](running-on-yarn.html#configuring-the-external-shuffle-service).
diff --git a/docs/sql-ref-syntax-qry-select-groupby.md b/docs/sql-ref-syntax-qry-select-groupby.md
index 6137c0d80f313..934e5f70d4b08 100644
--- a/docs/sql-ref-syntax-qry-select-groupby.md
+++ b/docs/sql-ref-syntax-qry-select-groupby.md
@@ -58,7 +58,7 @@ aggregate_name ( [ DISTINCT ] expression [ , ... ] ) [ FILTER ( WHERE boolean_ex
 
 * **grouping_expression**
 
-    Specifies the critieria based on which the rows are grouped together. The grouping of rows is performed based on
+    Specifies the criteria based on which the rows are grouped together. The grouping of rows is performed based on
     result values of the grouping expressions. A grouping expression may be a column alias, a column position
     or an expression.
 
diff --git a/docs/sql-ref-syntax-qry-select-hints.md b/docs/sql-ref-syntax-qry-select-hints.md
index 247ce48e79445..5f1cb4c5bed0b 100644
--- a/docs/sql-ref-syntax-qry-select-hints.md
+++ b/docs/sql-ref-syntax-qry-select-hints.md
@@ -31,7 +31,7 @@ Hints give users a way to suggest how Spark SQL to use specific approaches to ge
 
 ### Partitioning Hints
 
-Partitioning hints allow users to suggest a partitioning stragety that Spark should follow. `COALESCE`, `REPARTITION`,
+Partitioning hints allow users to suggest a partitioning strategy that Spark should follow. `COALESCE`, `REPARTITION`,
 and `REPARTITION_BY_RANGE` hints are supported and are equivalent to `coalesce`, `repartition`, and
 `repartitionByRange` [Dataset APIs](api/scala/org/apache/spark/sql/Dataset.html), respectively. These hints give users
 a way to tune performance and control the number of output files in Spark SQL. When multiple partitioning hints are
diff --git a/docs/sql-ref.md b/docs/sql-ref.md
index 8d0c6734c4bd6..6a87166f7133d 100644
--- a/docs/sql-ref.md
+++ b/docs/sql-ref.md
@@ -32,7 +32,7 @@ Spark SQL is Apache Spark's module for working with structured data. This guide
    * [Integration with Hive UDFs/UDAFs/UDTFs](sql-ref-functions-udf-hive.html)
  * [Identifiers](sql-ref-identifier.html)
  * [Literals](sql-ref-literals.html)
- * [Null Semanitics](sql-ref-null-semantics.html)
+ * [Null Semantics](sql-ref-null-semantics.html)
  * [SQL Syntax](sql-ref-syntax.html)
    * [DDL Statements](sql-ref-syntax-ddl.html)
    * [DML Statements](sql-ref-syntax-dml.html)
diff --git a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
index 3ff77878f68a8..d5a277ba581a0 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/LauncherServer.java
@@ -364,7 +364,7 @@ public void close() throws IOException {
      *
      * This method allows a short period for the above to happen (same amount of time as the
      * connection timeout, which is configurable). This should be fine for well-behaved
-     * applications, where they close the connection arond the same time the app handle detects the
+     * applications, where they close the connection around the same time the app handle detects the
      * app has finished.
      *
      * In case the connection is not closed within the grace period, this method forcefully closes
diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
index 1809b9cdb52e5..b818515adf9c0 100644
--- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
+++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java
@@ -176,7 +176,7 @@ Table alterTable(
    * @param newIdent the new table identifier of the table
    * @throws NoSuchTableException If the table to rename doesn't exist or is a view
    * @throws TableAlreadyExistsException If the new table name already exists or is a view
-   * @throws UnsupportedOperationException If the namespaces of old and new identiers do not
+   * @throws UnsupportedOperationException If the namespaces of old and new identifiers do not
    *                                       match (optional)
    */
   void renameTable(Identifier oldIdent, Identifier newIdent)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
index cd75407c7ee7a..35551d8ba77dc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/QueryPlanningTracker.scala
@@ -28,7 +28,7 @@ import org.apache.spark.util.BoundedPriorityQueue
  * There are two separate concepts we track:
  *
  * 1. Phases: These are broad scope phases in query planning, as listed below, i.e. analysis,
- * optimizationm and physical planning (just planning).
+ * optimization and physical planning (just planning).
  *
  * 2. Rules: These are the individual Catalyst rules that we track. In addition to time, we also
  * track the number of invocations and effective invocations.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
index fef63cb8253ca..95715fd1af56e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowTablePropertiesExec.scala
@@ -36,7 +36,7 @@ case class ShowTablePropertiesExec(
     import scala.collection.JavaConverters._
     val toRow = RowEncoder(schema).resolveAndBind().createSerializer()
 
-    // The reservered properties are accessible through DESCRIBE
+    // The reserved properties are accessible through DESCRIBE
     val properties = catalogTable.properties.asScala
       .filter { case (k, v) => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(k) }
     propertyKey match {

From 87f1d51bcbd758387150a30e2cbc8540a2c6b227 Mon Sep 17 00:00:00 2001
From: "Robert (Bobby) Evans" <bobby@apache.org>
Date: Sat, 22 Aug 2020 11:07:14 +0900
Subject: [PATCH 0992/1280] [SPARK-32672][SQL] Fix data corruption in boolean
 bit set compression

## What changes were proposed in this pull request?

This fixed SPARK-32672 a data corruption.  Essentially the BooleanBitSet CompressionScheme would miss nulls at the end of a CompressedBatch.  The values would then default to false.

### Why are the changes needed?
It fixes data corruption

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
I manually tested it against the original issue that was producing errors for me.  I also added in a unit test.

Closes #29506 from revans2/SPARK-32672.

Authored-by: Robert (Bobby) Evans <bobby@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 12f4331b9eb563cb0cfbf6a241d1d085ca4f7676)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../compression/compressionSchemes.scala      |  6 ++---
 .../compression/BooleanBitSetSuite.scala      | 26 +++++++++++++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
index 00a1d54b41709..3cc59af9b7ce3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/compression/compressionSchemes.scala
@@ -318,7 +318,8 @@ private[columnar] case object RunLengthEncoding extends CompressionScheme {
       var valueCountLocal = 0
       var currentValueLocal: Long = 0
 
-      while (valueCountLocal < runLocal || (pos < capacity)) {
+      while (pos < capacity) {
+        assert(valueCountLocal <= runLocal)
         if (pos != nextNullIndex) {
           if (valueCountLocal == runLocal) {
             currentValueLocal = getFunction(buffer)
@@ -616,7 +617,6 @@ private[columnar] case object BooleanBitSet extends CompressionScheme {
     override def hasNext: Boolean = visited < count
 
     override def decompress(columnVector: WritableColumnVector, capacity: Int): Unit = {
-      val countLocal = count
       var currentWordLocal: Long = 0
       var visitedLocal: Int = 0
       val nullsBuffer = buffer.duplicate().order(ByteOrder.nativeOrder())
@@ -626,7 +626,7 @@ private[columnar] case object BooleanBitSet extends CompressionScheme {
       var pos = 0
       var seenNulls = 0
 
-      while (visitedLocal < countLocal) {
+      while (pos < capacity) {
         if (pos != nextNullIndex) {
           val bit = visitedLocal % BITS_PER_LONG
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
index 192db0e910d03..111a620df8c24 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/BooleanBitSetSuite.scala
@@ -156,4 +156,30 @@ class BooleanBitSetSuite extends SparkFunSuite {
   test(s"$BooleanBitSet: multiple words and 1 more bit for decompression()") {
     skeletonForDecompress(BITS_PER_LONG * 2 + 1)
   }
+
+  test(s"$BooleanBitSet: Only nulls for decompression()") {
+    val builder = TestCompressibleColumnBuilder(new NoopColumnStats, BOOLEAN, BooleanBitSet)
+    val numRows = 10
+
+    val rows = Seq.fill[InternalRow](numRows)({
+      val row = new GenericInternalRow(1)
+      row.setNullAt(0)
+      row
+    })
+    rows.foreach(builder.appendFrom(_, 0))
+    val buffer = builder.build()
+
+    // Rewinds, skips column header and 4 more bytes for compression scheme ID
+    val headerSize = CompressionScheme.columnHeaderSize(buffer)
+    buffer.position(headerSize)
+    assertResult(BooleanBitSet.typeId, "Wrong compression scheme ID")(buffer.getInt())
+
+    val decoder = BooleanBitSet.decoder(buffer, BOOLEAN)
+    val columnVector = new OnHeapColumnVector(numRows, BooleanType)
+    decoder.decompress(columnVector, numRows)
+
+    (0 until numRows).foreach { rowNum =>
+      assert(columnVector.isNullAt(rowNum))
+    }
+  }
 }

From a6df16b36210da32359c77205920eaee98d3e232 Mon Sep 17 00:00:00 2001
From: Yuanjian Li <yuanjian.li@databricks.com>
Date: Sat, 22 Aug 2020 21:32:23 +0900
Subject: [PATCH 0993/1280] [SPARK-31792][SS][DOC][FOLLOW-UP] Rephrase the
 description for some operations

### What changes were proposed in this pull request?
Rephrase the description for some operations to make it clearer.

### Why are the changes needed?
Add more detail in the document.

### Does this PR introduce _any_ user-facing change?
No, document only.

### How was this patch tested?
Document only.

Closes #29269 from xuanyuanking/SPARK-31792-follow.

Authored-by: Yuanjian Li <yuanjian.li@databricks.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit 8b26c69ce7f9077775a3c7bbabb1c47ee6a51a23)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 docs/web-ui.md                                         | 10 +++++-----
 .../sql/execution/streaming/MicroBatchExecution.scala  |  3 +--
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/docs/web-ui.md b/docs/web-ui.md
index 134a8c860d404..fe2604364a12b 100644
--- a/docs/web-ui.md
+++ b/docs/web-ui.md
@@ -425,11 +425,11 @@ queries. Currently, it contains the following metrics.
 * **Batch Duration.** The process duration of each batch. 
 * **Operation Duration.** The amount of time taken to perform various operations in milliseconds.
 The tracked operations are listed as follows.
-    * addBatch: Adds result data of the current batch to the sink.
-    * getBatch: Gets a new batch of data to process.
-    * latestOffset: Gets the latest offsets for sources. 
-    * queryPlanning: Generates the execution plan.
-    * walCommit: Writes the offsets to the metadata log.
+    * addBatch: Time taken to read the micro-batch's input data from the sources, process it, and write the batch's output to the sink. This should take the bulk of the micro-batch's time.
+    * getBatch: Time taken to prepare the logical query to read the input of the current micro-batch from the sources.
+    * latestOffset & getOffset: Time taken to query the maximum available offset for this source.
+    * queryPlanning: Time taken to generates the execution plan.
+    * walCommit: Time taken to write the offsets to the metadata log.
     
 As an early-release version, the statistics page is still under development and will be improved in
 future releases.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index e022bfb6835d2..e0731db1f3c18 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -566,8 +566,7 @@ class MicroBatchExecution(
     val nextBatch =
       new Dataset(lastExecution, RowEncoder(lastExecution.analyzed.schema))
 
-    val batchSinkProgress: Option[StreamWriterCommitProgress] =
-      reportTimeTaken("addBatch") {
+    val batchSinkProgress: Option[StreamWriterCommitProgress] = reportTimeTaken("addBatch") {
       SQLExecution.withNewExecutionId(lastExecution) {
         sink match {
           case s: Sink => s.addBatch(currentBatchId, nextBatch)

From 85c9e8c54c30b69c39075e97cd3cac295be09303 Mon Sep 17 00:00:00 2001
From: Louiszr <zxhst14@gmail.com>
Date: Sat, 22 Aug 2020 09:27:31 -0500
Subject: [PATCH 0994/1280] [SPARK-32092][ML][PYSPARK] Fix parameters not being
 copied in CrossValidatorModel.copy(), read() and write()

### What changes were proposed in this pull request?

Changed the definitions of `CrossValidatorModel.copy()/_to_java()/_from_java()` so that exposed parameters (i.e. parameters with `get()` methods) are copied in these methods.

### Why are the changes needed?

Parameters are copied in the respective Scala interface for `CrossValidatorModel.copy()`.
It fits the semantics to persist parameters when calling `CrossValidatorModel.save()` and `CrossValidatorModel.load()` so that the user gets the same model by saving and loading it after. Not copying across `numFolds` also causes bugs like Array index out of bound and losing sub-models because this parameters will always default to 3 (as described in the JIRA ticket).

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Tests for `CrossValidatorModel.copy()` and `save()`/`load()` are updated so that they check parameters before and after function calls.

Closes #29445 from Louiszr/master.

Authored-by: Louiszr <zxhst14@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit d9eb06ea37cab185f1e49c641313be9707270252)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 python/pyspark/ml/tests/test_tuning.py | 131 ++++++++++++++++++++++---
 python/pyspark/ml/tuning.py            |  67 ++++++++++---
 2 files changed, 172 insertions(+), 26 deletions(-)

diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index 6bcc3f93e1b3b..b250740f8c4f1 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -89,15 +89,50 @@ def test_copy(self):
         grid = (ParamGridBuilder()
                 .addGrid(iee.inducedError, [100.0, 0.0, 10000.0])
                 .build())
-        cv = CrossValidator(estimator=iee, estimatorParamMaps=grid, evaluator=evaluator)
+        cv = CrossValidator(
+            estimator=iee,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            numFolds=2
+        )
         cvCopied = cv.copy()
-        self.assertEqual(cv.getEstimator().uid, cvCopied.getEstimator().uid)
+        for param in [
+            lambda x: x.getEstimator().uid,
+            # SPARK-32092: CrossValidator.copy() needs to copy all existing params
+            lambda x: x.getNumFolds(),
+            lambda x: x.getFoldCol(),
+            lambda x: x.getCollectSubModels(),
+            lambda x: x.getParallelism(),
+            lambda x: x.getSeed()
+        ]:
+            self.assertEqual(param(cv), param(cvCopied))
 
         cvModel = cv.fit(dataset)
         cvModelCopied = cvModel.copy()
         for index in range(len(cvModel.avgMetrics)):
             self.assertTrue(abs(cvModel.avgMetrics[index] - cvModelCopied.avgMetrics[index])
                             < 0.0001)
+        # SPARK-32092: CrossValidatorModel.copy() needs to copy all existing params
+        for param in [
+            lambda x: x.getNumFolds(),
+            lambda x: x.getFoldCol(),
+            lambda x: x.getSeed()
+        ]:
+            self.assertEqual(param(cvModel), param(cvModelCopied))
+
+        cvModel.avgMetrics[0] = 'foo'
+        self.assertNotEqual(
+            cvModelCopied.avgMetrics[0],
+            'foo',
+            "Changing the original avgMetrics should not affect the copied model"
+        )
+        cvModel.subModels[0] = 'foo'
+        self.assertNotEqual(
+            cvModelCopied.subModels[0],
+            'foo',
+            "Changing the original subModels should not affect the copied model"
+        )
 
     def test_fit_minimize_metric(self):
         dataset = self.spark.createDataFrame([
@@ -166,16 +201,39 @@ def test_save_load_trained_model(self):
         lr = LogisticRegression()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
         evaluator = BinaryClassificationEvaluator()
-        cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
+        cv = CrossValidator(
+            estimator=lr,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            numFolds=4,
+            seed=42
+        )
         cvModel = cv.fit(dataset)
         lrModel = cvModel.bestModel
 
-        cvModelPath = temp_path + "/cvModel"
-        lrModel.save(cvModelPath)
-        loadedLrModel = LogisticRegressionModel.load(cvModelPath)
+        lrModelPath = temp_path + "/lrModel"
+        lrModel.save(lrModelPath)
+        loadedLrModel = LogisticRegressionModel.load(lrModelPath)
         self.assertEqual(loadedLrModel.uid, lrModel.uid)
         self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
 
+        # SPARK-32092: Saving and then loading CrossValidatorModel should not change the params
+        cvModelPath = temp_path + "/cvModel"
+        cvModel.save(cvModelPath)
+        loadedCvModel = CrossValidatorModel.load(cvModelPath)
+        for param in [
+            lambda x: x.getNumFolds(),
+            lambda x: x.getFoldCol(),
+            lambda x: x.getSeed(),
+            lambda x: len(x.subModels)
+        ]:
+            self.assertEqual(param(cvModel), param(loadedCvModel))
+
+        self.assertTrue(all(
+            loadedCvModel.isSet(param) for param in loadedCvModel.params
+        ))
+
     def test_save_load_simple_estimator(self):
         temp_path = tempfile.mkdtemp()
         dataset = self.spark.createDataFrame(
@@ -451,16 +509,35 @@ def test_save_load_trained_model(self):
         lr = LogisticRegression()
         grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
         evaluator = BinaryClassificationEvaluator()
-        tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
+        tvs = TrainValidationSplit(
+            estimator=lr,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True,
+            seed=42
+        )
         tvsModel = tvs.fit(dataset)
         lrModel = tvsModel.bestModel
 
-        tvsModelPath = temp_path + "/tvsModel"
-        lrModel.save(tvsModelPath)
-        loadedLrModel = LogisticRegressionModel.load(tvsModelPath)
+        lrModelPath = temp_path + "/lrModel"
+        lrModel.save(lrModelPath)
+        loadedLrModel = LogisticRegressionModel.load(lrModelPath)
         self.assertEqual(loadedLrModel.uid, lrModel.uid)
         self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
 
+        tvsModelPath = temp_path + "/tvsModel"
+        tvsModel.save(tvsModelPath)
+        loadedTvsModel = TrainValidationSplitModel.load(tvsModelPath)
+        for param in [
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvsModel), param(loadedTvsModel))
+
+        self.assertTrue(all(
+            loadedTvsModel.isSet(param) for param in loadedTvsModel.params
+        ))
+
     def test_save_load_simple_estimator(self):
         # This tests saving and loading the trained model only.
         # Save/load for TrainValidationSplit will be added later: SPARK-13786
@@ -662,11 +739,30 @@ def test_copy(self):
         grid = ParamGridBuilder() \
             .addGrid(iee.inducedError, [100.0, 0.0, 10000.0]) \
             .build()
-        tvs = TrainValidationSplit(estimator=iee, estimatorParamMaps=grid, evaluator=evaluator)
+        tvs = TrainValidationSplit(
+            estimator=iee,
+            estimatorParamMaps=grid,
+            evaluator=evaluator,
+            collectSubModels=True
+        )
         tvsModel = tvs.fit(dataset)
         tvsCopied = tvs.copy()
         tvsModelCopied = tvsModel.copy()
 
+        for param in [
+            lambda x: x.getCollectSubModels(),
+            lambda x: x.getParallelism(),
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvs), param(tvsCopied))
+
+        for param in [
+            lambda x: x.getSeed(),
+            lambda x: x.getTrainRatio(),
+        ]:
+            self.assertEqual(param(tvsModel), param(tvsModelCopied))
+
         self.assertEqual(tvs.getEstimator().uid, tvsCopied.getEstimator().uid,
                          "Copied TrainValidationSplit has the same uid of Estimator")
 
@@ -678,6 +774,19 @@ def test_copy(self):
             self.assertEqual(tvsModel.validationMetrics[index],
                              tvsModelCopied.validationMetrics[index])
 
+        tvsModel.validationMetrics[0] = 'foo'
+        self.assertNotEqual(
+            tvsModelCopied.validationMetrics[0],
+            'foo',
+            "Changing the original validationMetrics should not affect the copied model"
+        )
+        tvsModel.subModels[0] = 'foo'
+        self.assertNotEqual(
+            tvsModelCopied.subModels[0],
+            'foo',
+            "Changing the original subModels should not affect the copied model"
+        )
+
 
 if __name__ == "__main__":
     from pyspark.ml.tests.test_tuning import *
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index e564ff7a10448..91f34ef24d021 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -479,9 +479,9 @@ def copy(self, extra=None):
         if extra is None:
             extra = dict()
         bestModel = self.bestModel.copy(extra)
-        avgMetrics = self.avgMetrics
-        subModels = self.subModels
-        return CrossValidatorModel(bestModel, avgMetrics, subModels)
+        avgMetrics = list(self.avgMetrics)
+        subModels = [model.copy() for model in self.subModels]
+        return self._copyValues(CrossValidatorModel(bestModel, avgMetrics, subModels), extra=extra)
 
     @since("2.3.0")
     def write(self):
@@ -505,8 +505,17 @@ def _from_java(cls, java_stage):
         avgMetrics = _java2py(sc, java_stage.avgMetrics())
         estimator, epms, evaluator = super(CrossValidatorModel, cls)._from_java_impl(java_stage)
 
-        py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)._set(estimator=estimator)
-        py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator)
+        py_stage = cls(bestModel=bestModel, avgMetrics=avgMetrics)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "numFolds": java_stage.getNumFolds(),
+            "foldCol": java_stage.getFoldCol(),
+            "seed": java_stage.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            py_stage = py_stage._set(**{param_name: param_val})
 
         if java_stage.hasSubModels():
             py_stage.subModels = [[JavaParams._from_java(sub_model)
@@ -530,9 +539,18 @@ def _to_java(self):
                                              _py2java(sc, self.avgMetrics))
         estimator, epms, evaluator = super(CrossValidatorModel, self)._to_java_impl()
 
-        _java_obj.set("evaluator", evaluator)
-        _java_obj.set("estimator", estimator)
-        _java_obj.set("estimatorParamMaps", epms)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "numFolds": self.getNumFolds(),
+            "foldCol": self.getFoldCol(),
+            "seed": self.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            java_param = _java_obj.getParam(param_name)
+            pair = java_param.w(param_val)
+            _java_obj.set(pair)
 
         if self.subModels is not None:
             java_sub_models = [[sub_model._to_java() for sub_model in fold_sub_models]
@@ -818,8 +836,11 @@ def copy(self, extra=None):
             extra = dict()
         bestModel = self.bestModel.copy(extra)
         validationMetrics = list(self.validationMetrics)
-        subModels = self.subModels
-        return TrainValidationSplitModel(bestModel, validationMetrics, subModels)
+        subModels = [model.copy() for model in self.subModels]
+        return self._copyValues(
+            TrainValidationSplitModel(bestModel, validationMetrics, subModels),
+            extra=extra
+        )
 
     @since("2.3.0")
     def write(self):
@@ -847,8 +868,16 @@ def _from_java(cls, java_stage):
                                            cls)._from_java_impl(java_stage)
         # Create a new instance of this stage.
         py_stage = cls(bestModel=bestModel,
-                       validationMetrics=validationMetrics)._set(estimator=estimator)
-        py_stage = py_stage._set(estimatorParamMaps=epms)._set(evaluator=evaluator)
+                       validationMetrics=validationMetrics)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "trainRatio": java_stage.getTrainRatio(),
+            "seed": java_stage.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            py_stage = py_stage._set(**{param_name: param_val})
 
         if java_stage.hasSubModels():
             py_stage.subModels = [JavaParams._from_java(sub_model)
@@ -871,9 +900,17 @@ def _to_java(self):
             _py2java(sc, self.validationMetrics))
         estimator, epms, evaluator = super(TrainValidationSplitModel, self)._to_java_impl()
 
-        _java_obj.set("evaluator", evaluator)
-        _java_obj.set("estimator", estimator)
-        _java_obj.set("estimatorParamMaps", epms)
+        params = {
+            "evaluator": evaluator,
+            "estimator": estimator,
+            "estimatorParamMaps": epms,
+            "trainRatio": self.getTrainRatio(),
+            "seed": self.getSeed(),
+        }
+        for param_name, param_val in params.items():
+            java_param = _java_obj.getParam(param_name)
+            pair = java_param.w(param_val)
+            _java_obj.set(pair)
 
         if self.subModels is not None:
             java_sub_models = [sub_model._to_java() for sub_model in self.subModels]

From f5d5422a4f87f69514d95f80f5f3db8246d61256 Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Sun, 23 Aug 2020 08:20:05 -0700
Subject: [PATCH 0995/1280] 
 [SPARK-32608][SQL][3.0][FOLLOW-UP][TEST-HADOOP2.7][TEST-HIVE1.2] Script
 Transform ROW FORMAT DELIMIT value should format value

### What changes were proposed in this pull request?
As mentioned in https://github.com/apache/spark/pull/29428#issuecomment-678735163 by viirya ,
fix bug in UT, since in script transformation no-serde mode, output of decimal is same in both hive-1.2/hive-2.3

### Why are the changes needed?
FIX UT

### Does this PR introduce _any_ user-facing change?
NO

### How was this patch tested?
EXISTED UT

Closes #29521 from AngersZhuuuu/SPARK-32608-3.0-FOLLOW-UP.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 .../spark/sql/hive/execution/ScriptTransformationSuite.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
index 15a932f6d660c..0d1fe20130c0e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ScriptTransformationSuite.scala
@@ -299,7 +299,7 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
           'a.cast("string"),
           'b.cast("string"),
           'c.cast("string"),
-          decimalToString('d),
+          'd.cast("string"),
           'e.cast("string")).collect())
 
       // input/output with different delimit and show result
@@ -322,7 +322,7 @@ class ScriptTransformationSuite extends SparkPlanTest with SQLTestUtils with Tes
             'a.cast("string"),
             'b.cast("string"),
             'c.cast("string"),
-            decimalToString('d),
+            'd.cast("string"),
             'e.cast("string"))).collect())
     }
   }

From f088c28a53571afe5146100fd2e76c2b5ec92862 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 23 Aug 2020 12:43:30 -0700
Subject: [PATCH 0996/1280] 
 [SPARK-32594][SQL][FOLLOWUP][TEST-HADOOP2.7][TEST-HIVE1.2] Override `get()`
 and use Julian days in `DaysWritable`

### What changes were proposed in this pull request?
Override `def get: Date` in `DaysWritable` use the `daysToMillis(int d)` from the parent class `DateWritable` instead of `long daysToMillis(int d, boolean doesTimeMatter)`.

### Why are the changes needed?
It fixes failures of `HiveSerDeReadWriteSuite` with the profile `hive-1.2`. In that case, the parent class `DateWritable` has different implementation before the commit to Hive https://github.com/apache/hive/commit/da3ed68eda10533f3c50aae19731ac6d059cda87. In particular, `get()` calls `new Date(daysToMillis(daysSinceEpoch))` instead of overrided `def get(doesTimeMatter: Boolean): Date` in the child class. The `get()` method returns wrong result `1970-01-01` because it uses not updated `daysSinceEpoch`.

### Does this PR introduce _any_ user-facing change?
Yes.

### How was this patch tested?
By running the test suite `HiveSerDeReadWriteSuite`:
```
$ build/sbt -Phive-1.2 -Phadoop-2.7 "test:testOnly org.apache.spark.sql.hive.execution.HiveSerDeReadWriteSuite"
```
and
```
$ build/sbt -Phive-2.3 -Phadoop-2.7 "test:testOnly org.apache.spark.sql.hive.execution.HiveSerDeReadWriteSuite"
```

Closes #29523 from MaxGekk/insert-date-into-hive-table-1.2.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
(cherry picked from commit 1c798f973fa8307cc1f15eec067886e8e9aecb59)
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 .../apache/spark/sql/execution/datasources/DaysWritable.scala  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
index 56c176e024aca..a04c2fcbbac12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DaysWritable.scala
@@ -54,6 +54,9 @@ class DaysWritable(
   }
 
   override def getDays: Int = julianDays
+  override def get: Date = {
+    new Date(DateWritable.daysToMillis(julianDays))
+  }
   override def get(doesTimeMatter: Boolean): Date = {
     new Date(DateWritable.daysToMillis(julianDays, doesTimeMatter))
   }

From 898211b54e2c9e212f19d8bad6b7e91b66e5659a Mon Sep 17 00:00:00 2001
From: mingjial <mingjial@google.com>
Date: Sun, 23 Aug 2020 17:40:59 -0700
Subject: [PATCH 0997/1280] [SPARK-32609][TEST] Add Tests for Incorrect
 exchange reuse with DataSourceV2

### What changes were proposed in this pull request?
Copy  to master branch the unit test added for branch-2.4(https://github.com/apache/spark/pull/29430).

### Why are the changes needed?
The unit test will pass at master branch, indicating that issue reported in https://issues.apache.org/jira/browse/SPARK-32609 is already fixed at master branch. But adding this unit test for future possible failure catch.

### Does this PR introduce _any_ user-facing change?
no.

### How was this patch tested?
sbt test run

Closes #29435 from mingjialiu/master.

Authored-by: mingjial <mingjial@google.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b9585cde31fe99aecca42146c71c552218cba591)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/connector/DataSourceV2Suite.scala     | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
index 2d8761f872da7..a9c521eb46499 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
@@ -394,6 +394,25 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
       checkAnswer(df, (0 until 3).map(i => Row(i)))
     }
   }
+
+  test("SPARK-32609: DataSourceV2 with different pushedfilters should be different") {
+    def getScanExec(query: DataFrame): BatchScanExec = {
+      query.queryExecution.executedPlan.collect {
+        case d: BatchScanExec => d
+      }.head
+    }
+
+    Seq(classOf[AdvancedDataSourceV2], classOf[JavaAdvancedDataSourceV2]).foreach { cls =>
+      withClue(cls.getName) {
+        val df = spark.read.format(cls.getName).load()
+        val q1 = df.select('i).filter('i > 6)
+        val q2 = df.select('i).filter('i > 5)
+        val scan1 = getScanExec(q1)
+        val scan2 = getScanExec(q2)
+        assert(!scan1.equals(scan2))
+      }
+    }
+  }
 }
 
 

From da60de563a92bb85902681fb0569b43bbc489559 Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 24 Aug 2020 09:43:41 +0900
Subject: [PATCH 0998/1280] [SPARK-32552][SQL][DOCS] Complete the documentation
 for Table-valued Function

# What changes were proposed in this pull request?
There are two types of TVF. We only documented one type. Adding the doc for the 2nd type.

### Why are the changes needed?
complete Table-valued Function doc

### Does this PR introduce _any_ user-facing change?
<img width="1099" alt="Screen Shot 2020-08-06 at 5 30 25 PM" src="https://user-images.githubusercontent.com/13592258/89595926-c5eae680-d80a-11ea-918b-0c3646f9930e.png">

<img width="1100" alt="Screen Shot 2020-08-06 at 5 30 49 PM" src="https://user-images.githubusercontent.com/13592258/89595929-c84d4080-d80a-11ea-9803-30eb502ccd05.png">

<img width="1101" alt="Screen Shot 2020-08-06 at 5 31 19 PM" src="https://user-images.githubusercontent.com/13592258/89595931-ca170400-d80a-11ea-8812-2f009746edac.png">

<img width="1100" alt="Screen Shot 2020-08-06 at 5 31 40 PM" src="https://user-images.githubusercontent.com/13592258/89595934-cb483100-d80a-11ea-9e18-9357aa9f2c5c.png">

### How was this patch tested?
Manually build and check

Closes #29355 from huaxingao/tvf.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit db74fd0d3320f120540133094a9975963941b98c)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-syntax-qry-select-tvf.md | 99 ++++++++++++++++++++++-----
 1 file changed, 80 insertions(+), 19 deletions(-)

diff --git a/docs/sql-ref-syntax-qry-select-tvf.md b/docs/sql-ref-syntax-qry-select-tvf.md
index cc8d7c34645fb..b04e2f5c3fbd7 100644
--- a/docs/sql-ref-syntax-qry-select-tvf.md
+++ b/docs/sql-ref-syntax-qry-select-tvf.md
@@ -21,28 +21,14 @@ license: |
 
 ### Description
 
-A table-valued function (TVF) is a function that returns a relation or a set of rows.
-
-### Syntax
-
-```sql
-function_name ( expression [ , ... ] ) [ table_alias ]
-```
-
-### Parameters
-
-* **expression**
-
-    Specifies a combination of one or more values, operators and SQL functions that results in a value.
-
-* **table_alias**
-
-    Specifies a temporary name with an optional column name list.
-
-    **Syntax:** `[ AS ] table_name [ ( column_name [ , ... ] ) ]`
+A table-valued function (TVF) is a function that returns a relation or a set of rows. There are two types of TVFs in Spark SQL:
+1. a TVF that can be specified in a FROM clause, e.g. range;
+2. a TVF that can be specified in SELECT/LATERAL VIEW clauses, e.g. explode.
 
 ### Supported Table-valued Functions
 
+#### TVFs that can be specified in a FROM clause:
+
 |Function|Argument Type(s)|Description|
 |--------|----------------|-----------|
 |**range** ( *end* )|Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from 0 to *end* (exclusive) with step value 1.|
@@ -50,6 +36,20 @@ function_name ( expression [ , ... ] ) [ table_alias ]
 |**range** ( *start, end, step* )|Long, Long, Long|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value.|
 |**range** ( *start, end, step, numPartitions* )|Long, Long, Long, Int|Creates a table with a single *LongType* column named *id*, <br/> containing rows in a range from *start* to *end* (exclusive) with *step* value, with partition number *numPartitions* specified.|
 
+#### TVFs that can be specified in SELECT/LATERAL VIEW clauses:
+
+|Function|Argument Type(s)|Description|
+|--------|----------------|-----------|
+|**explode** ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows, or the elements of map *expr* into multiple rows and columns. Unless specified otherwise, uses the default column name col for elements of the array or key and value for the elements of the map.|
+|**explode_outer** <br> ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows, or the elements of map *expr* into multiple rows and columns. Unless specified otherwise, uses the default column name col for elements of the array or key and value for the elements of the map.|
+|**inline** ( *expr* )|Expression|Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.|
+|**inline_outer** <br> ( *expr* )|Expression|Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.|
+|**posexplode** <br> ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows with positions, or the elements of map *expr* into multiple rows and columns with positions. Unless specified otherwise, uses the column name pos for position, col for elements of the array or key and value for elements of the map.|
+|**posexplode_outer** ( *expr* )|Array/Map|Separates the elements of array *expr* into multiple rows with positions, or the elements of map *expr* into multiple rows and columns with positions. Unless specified otherwise, uses the column name pos for position, col for elements of the array or key and value for elements of the map.|
+|**stack** ( *n, expr1, ..., exprk* )|Seq[Expression]|Separates *expr1, ..., exprk* into n rows. Uses column names col0, col1, etc. by default unless specified otherwise.|
+|**json_tuple** <br> ( *jsonStr, p1, p2, ..., pn* )|Seq[Expression]|Returns a tuple like the function *get_json_object*, but it takes multiple names. All the input parameters and output column types are string.|
+|**parse_url** <br> ( *url, partToExtract[, key]* )|Seq[Expression]|Extracts a part from a URL.|
+
 ### Examples
 
 ```sql
@@ -98,8 +98,69 @@ SELECT * FROM range(5, 8) AS test;
 |  6|
 |  7|
 +---+
+
+SELECT explode(array(10, 20));
++---+
+|col|
++---+
+| 10|
+| 20|
++---+
+
+SELECT inline(array(struct(1, 'a'), struct(2, 'b')));
++----+----+
+|col1|col2|
++----+----+
+|   1|   a|
+|   2|   b|
++----+----+
+
+SELECT posexplode(array(10,20));
++---+---+
+|pos|col|
++---+---+
+|  0| 10|
+|  1| 20|
++---+---+
+
+SELECT stack(2, 1, 2, 3);
++----+----+
+|col0|col1|
++----+----+
+|   1|   2|
+|   3|null|
++----+----+
+
+SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');
++---+---+
+| c0| c1|
++---+---+
+|  1|  2|
++---+---+
+
+SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');
++-----------------------------------------------------+
+|parse_url(http://spark.apache.org/path?query=1, HOST)|
++-----------------------------------------------------+
+|                                     spark.apache.org|
++-----------------------------------------------------+
+
+-- Use explode in a LATERAL VIEW clause
+CREATE TABLE test (c1 INT);
+INSERT INTO test VALUES (1);
+INSERT INTO test VALUES (2);
+SELECT * FROM test LATERAL VIEW explode (ARRAY(3,4)) AS c2;
++--+--+
+|c1|c2|
++--+--+
+| 1| 3|
+| 1| 4|
+| 2| 3|
+| 2| 4|
++--+--+
 ```
 
 ### Related Statements
 
 * [SELECT](sql-ref-syntax-qry-select.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)

From 8aa644e9a991cd7f965aec082adcc3a3d19d452f Mon Sep 17 00:00:00 2001
From: Louiszr <zxhst14@gmail.com>
Date: Sun, 23 Aug 2020 21:10:52 -0700
Subject: [PATCH 0999/1280] [SPARK-32092][ML][PYSPARK][3.0] Removed foldCol
 related code

### What changes were proposed in this pull request?

- Removed `foldCol` related code introduced in #29445 which is causing issues in the base branch.
- Fixed `CrossValidatorModel.copy()` so that it correctly calls `.copy()` on the models instead of lists of models.

### Why are the changes needed?

- `foldCol` is from 3.1 hence causing tests to fail.
- `CrossValidatorModel.copy()` is supposed to shallow copy models not lists of models.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

- Existing tests created in #29445 ran and passed.
- Updated `test_copy` to make sure `copy()` is called on models instead of lists of models.

Closes #29524 from Louiszr/remove-foldcol-3.0.

Authored-by: Louiszr <zxhst14@gmail.com>
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
---
 python/pyspark/ml/tests/test_tuning.py | 11 ++++-------
 python/pyspark/ml/tuning.py            |  7 ++++---
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index b250740f8c4f1..b1acaf695245d 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -101,7 +101,6 @@ def test_copy(self):
             lambda x: x.getEstimator().uid,
             # SPARK-32092: CrossValidator.copy() needs to copy all existing params
             lambda x: x.getNumFolds(),
-            lambda x: x.getFoldCol(),
             lambda x: x.getCollectSubModels(),
             lambda x: x.getParallelism(),
             lambda x: x.getSeed()
@@ -116,7 +115,6 @@ def test_copy(self):
         # SPARK-32092: CrossValidatorModel.copy() needs to copy all existing params
         for param in [
             lambda x: x.getNumFolds(),
-            lambda x: x.getFoldCol(),
             lambda x: x.getSeed()
         ]:
             self.assertEqual(param(cvModel), param(cvModelCopied))
@@ -127,9 +125,9 @@ def test_copy(self):
             'foo',
             "Changing the original avgMetrics should not affect the copied model"
         )
-        cvModel.subModels[0] = 'foo'
+        cvModel.subModels[0][0].getInducedError = lambda: 'foo'
         self.assertNotEqual(
-            cvModelCopied.subModels[0],
+            cvModelCopied.subModels[0][0].getInducedError(),
             'foo',
             "Changing the original subModels should not affect the copied model"
         )
@@ -224,7 +222,6 @@ def test_save_load_trained_model(self):
         loadedCvModel = CrossValidatorModel.load(cvModelPath)
         for param in [
             lambda x: x.getNumFolds(),
-            lambda x: x.getFoldCol(),
             lambda x: x.getSeed(),
             lambda x: len(x.subModels)
         ]:
@@ -780,9 +777,9 @@ def test_copy(self):
             'foo',
             "Changing the original validationMetrics should not affect the copied model"
         )
-        tvsModel.subModels[0] = 'foo'
+        tvsModel.subModels[0].getInducedError = lambda: 'foo'
         self.assertNotEqual(
-            tvsModelCopied.subModels[0],
+            tvsModelCopied.subModels[0].getInducedError(),
             'foo',
             "Changing the original subModels should not affect the copied model"
         )
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index 91f34ef24d021..6283c8bfd3df9 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -480,7 +480,10 @@ def copy(self, extra=None):
             extra = dict()
         bestModel = self.bestModel.copy(extra)
         avgMetrics = list(self.avgMetrics)
-        subModels = [model.copy() for model in self.subModels]
+        subModels = [
+            [sub_model.copy() for sub_model in fold_sub_models]
+            for fold_sub_models in self.subModels
+        ]
         return self._copyValues(CrossValidatorModel(bestModel, avgMetrics, subModels), extra=extra)
 
     @since("2.3.0")
@@ -511,7 +514,6 @@ def _from_java(cls, java_stage):
             "estimator": estimator,
             "estimatorParamMaps": epms,
             "numFolds": java_stage.getNumFolds(),
-            "foldCol": java_stage.getFoldCol(),
             "seed": java_stage.getSeed(),
         }
         for param_name, param_val in params.items():
@@ -544,7 +546,6 @@ def _to_java(self):
             "estimator": estimator,
             "estimatorParamMaps": epms,
             "numFolds": self.getNumFolds(),
-            "foldCol": self.getFoldCol(),
             "seed": self.getSeed(),
         }
         for param_name, param_val in params.items():

From 4a67f1ea750baaf907769a0d31b413dbe1b062b3 Mon Sep 17 00:00:00 2001
From: Michael Munday <mike.munday@ibm.com>
Date: Mon, 24 Aug 2020 10:19:30 -0500
Subject: [PATCH 1000/1280] [SPARK-32588][CORE][TEST] Fix SizeEstimator
 initialization in tests

In order to produce consistent results from SizeEstimator the tests
override some system properties that are used during SizeEstimator
initialization. However there were several places where either the
compressed references property wasn't set or the system properties
were set but the SizeEstimator not re-initialized.

This caused failures when running the tests with a large heap build
of OpenJ9 because it does not use compressed references unlike most
environments.

### What changes were proposed in this pull request?
Initialize SizeEstimator class explicitly in the tests where required to avoid relying on a particular environment.

### Why are the changes needed?
Test failures can be seen when compressed references are disabled (e.g. using an OpenJ9 large heap build or Hotspot with a large heap).

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Tests run on machine running OpenJ9 large heap build.

Closes #29407 from mundaym/fix-sizeestimator.

Authored-by: Michael Munday <mike.munday@ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit bc23bb78823f4fa02385b7b2a0270cd1b98bce34)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../spark/storage/BlockManagerSuite.scala     | 26 +++++++++--
 .../spark/storage/MemoryStoreSuite.scala      | 29 +++++++++++--
 .../spark/util/SizeEstimatorSuite.scala       | 43 ++++++++++++-------
 3 files changed, 75 insertions(+), 23 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
index 8d06768a2b284..9cf531383ce31 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala
@@ -130,10 +130,28 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
     blockManager
   }
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
-    System.setProperty("os.arch", "amd64")
+    reinitializeSizeEstimator("amd64", "true")
     conf = new SparkConf(false)
     init(conf)
 
@@ -152,12 +170,12 @@ class BlockManagerSuite extends SparkFunSuite with Matchers with BeforeAndAfterE
         liveListenerBus, None, blockManagerInfo)),
       rpcEnv.setupEndpoint("blockmanagerHeartbeat",
       new BlockManagerMasterHeartbeatEndpoint(rpcEnv, true, blockManagerInfo)), conf, true))
-
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
   }
 
   override def afterEach(): Unit = {
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
+
     try {
       conf = null
       allStores.foreach(_.stop())
diff --git a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
index ccd7e4b62ad9e..c46ab2d199f0b 100644
--- a/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/MemoryStoreSuite.scala
@@ -26,6 +26,7 @@ import org.scalatest._
 
 import org.apache.spark._
 import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.Tests.TEST_USE_COMPRESSED_OOPS_KEY
 import org.apache.spark.memory.{MemoryMode, UnifiedMemoryManager}
 import org.apache.spark.serializer.{KryoSerializer, SerializerManager}
 import org.apache.spark.storage.memory.{BlockEvictionHandler, MemoryStore, PartiallySerializedBlock, PartiallyUnrolledIterator}
@@ -51,12 +52,34 @@ class MemoryStoreSuite
   implicit def StringToBlockId(value: String): BlockId = new TestBlockId(value)
   def rdd(rddId: Int, splitId: Int): RDDBlockId = RDDBlockId(rddId, splitId)
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
     super.beforeEach()
     // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
-    System.setProperty("os.arch", "amd64")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
+    reinitializeSizeEstimator("amd64", "true")
+  }
+
+  override def afterEach(): Unit = {
+    super.afterEach()
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
   }
 
   def makeMemoryStore(maxMem: Long): (MemoryStore, BlockInfoManager) = {
diff --git a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
index d4f2053e0b2f4..6183ba9faa6b4 100644
--- a/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/SizeEstimatorSuite.scala
@@ -73,15 +73,35 @@ class SizeEstimatorSuite
   with PrivateMethodTester
   with ResetSystemProperties {
 
+  // Save modified system properties so that we can restore them after tests.
+  val originalArch = System.getProperty("os.arch")
+  val originalCompressedOops = System.getProperty(TEST_USE_COMPRESSED_OOPS_KEY)
+
+  def reinitializeSizeEstimator(arch: String, useCompressedOops: String): Unit = {
+    def set(k: String, v: String): Unit = {
+      if (v == null) {
+        System.clearProperty(k)
+      } else {
+        System.setProperty(k, v)
+      }
+    }
+    set("os.arch", arch)
+    set(TEST_USE_COMPRESSED_OOPS_KEY, useCompressedOops)
+    val initialize = PrivateMethod[Unit](Symbol("initialize"))
+    SizeEstimator invokePrivate initialize()
+  }
+
   override def beforeEach(): Unit = {
-    // Set the arch to 64-bit and compressedOops to true to get a deterministic test-case
     super.beforeEach()
-    System.setProperty("os.arch", "amd64")
-    System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "true")
+    // Set the arch to 64-bit and compressedOops to true so that SizeEstimator
+    // provides identical results accross all systems in these tests.
+    reinitializeSizeEstimator("amd64", "true")
   }
 
   override def afterEach(): Unit = {
     super.afterEach()
+    // Restore system properties and SizeEstimator to their original states.
+    reinitializeSizeEstimator(originalArch, originalCompressedOops)
   }
 
   test("simple classes") {
@@ -178,11 +198,7 @@ class SizeEstimatorSuite
   }
 
   test("32-bit arch") {
-    System.setProperty("os.arch", "x86")
-
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
-
+    reinitializeSizeEstimator("x86", "true")
     assertResult(40)(SizeEstimator.estimate(DummyString("")))
     assertResult(48)(SizeEstimator.estimate(DummyString("a")))
     assertResult(48)(SizeEstimator.estimate(DummyString("ab")))
@@ -192,11 +208,7 @@ class SizeEstimatorSuite
   // NOTE: The String class definition varies across JDK versions (1.6 vs. 1.7) and vendors
   // (Sun vs IBM). Use a DummyString class to make tests deterministic.
   test("64-bit arch with no compressed oops") {
-    System.setProperty("os.arch", "amd64")
-    System.setProperty(TEST_USE_COMPRESSED_OOPS_KEY, "false")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
-
+    reinitializeSizeEstimator("amd64", "false")
     assertResult(56)(SizeEstimator.estimate(DummyString("")))
     assertResult(64)(SizeEstimator.estimate(DummyString("a")))
     assertResult(64)(SizeEstimator.estimate(DummyString("ab")))
@@ -214,14 +226,13 @@ class SizeEstimatorSuite
   }
 
   test("class field blocks rounding on 64-bit VM without useCompressedOops") {
+    reinitializeSizeEstimator("amd64", "false")
     assertResult(24)(SizeEstimator.estimate(new DummyClass5))
     assertResult(32)(SizeEstimator.estimate(new DummyClass6))
   }
 
   test("check 64-bit detection for s390x arch") {
-    System.setProperty("os.arch", "s390x")
-    val initialize = PrivateMethod[Unit](Symbol("initialize"))
-    SizeEstimator invokePrivate initialize()
+    reinitializeSizeEstimator("s390x", "true")
     // Class should be 32 bytes on s390x if recognised as 64 bit platform
     assertResult(32)(SizeEstimator.estimate(new DummyClass7))
   }

From 007acba6e3b0e45e334bed5942692dd88c61b3ea Mon Sep 17 00:00:00 2001
From: Huaxin Gao <huaxing@us.ibm.com>
Date: Mon, 24 Aug 2020 08:47:01 -0700
Subject: [PATCH 1001/1280] [SPARK-32676][3.0][ML] Fix double caching in
 KMeans/BiKMeans

### What changes were proposed in this pull request?
backporting https://github.com/apache/spark/pull/29501

### Why are the changes needed?
avoid double caching

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
Existing tests

Closes #29528 from huaxingao/kmeans_3.0.

Authored-by: Huaxin Gao <huaxing@us.ibm.com>
Signed-off-by: Huaxin Gao <huaxing@us.ibm.com>
---
 .../spark/ml/clustering/BisectingKMeans.scala | 33 +++++--------
 .../apache/spark/ml/clustering/KMeans.scala   | 33 +++++--------
 .../mllib/clustering/BisectingKMeans.scala    | 47 +++++++++----------
 .../spark/mllib/clustering/KMeans.scala       | 33 ++++++-------
 4 files changed, 60 insertions(+), 86 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index b649b1dd95231..b3f2d223628ea 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -28,9 +28,8 @@ import org.apache.spark.ml.util._
 import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.clustering.{BisectingKMeans => MLlibBisectingKMeans,
   BisectingKMeansModel => MLlibBisectingKMeansModel}
-import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors}
+import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
 import org.apache.spark.mllib.linalg.VectorImplicits._
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, IntegerType, StructType}
@@ -275,21 +274,6 @@ class BisectingKMeans @Since("2.0.0") (
   override def fit(dataset: Dataset[_]): BisectingKMeansModel = instrumented { instr =>
     transformSchema(dataset.schema, logging = true)
 
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
-    } else {
-      lit(1.0)
-    }
-
-    val instances: RDD[(OldVector, Double)] = dataset
-      .select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w).rdd.map {
-      case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight)
-    }
-    if (handlePersistence) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, k, maxIter, seed,
@@ -301,11 +285,18 @@ class BisectingKMeans @Since("2.0.0") (
       .setMinDivisibleClusterSize($(minDivisibleClusterSize))
       .setSeed($(seed))
       .setDistanceMeasure($(distanceMeasure))
-    val parentModel = bkm.runWithWeight(instances, Some(instr))
-    val model = copyValues(new BisectingKMeansModel(uid, parentModel).setParent(this))
-    if (handlePersistence) {
-      instances.unpersist()
+
+    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
+      col($(weightCol)).cast(DoubleType)
+    } else {
+      lit(1.0)
     }
+    val instances = dataset.select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w)
+      .rdd.map { case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight) }
+
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    val parentModel = bkm.runWithWeight(instances, handlePersistence, Some(instr))
+    val model = copyValues(new BisectingKMeansModel(uid, parentModel).setParent(this))
 
     val summary = new BisectingKMeansSummary(
       model.transform(dataset),
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index 5370318bc6adb..e182f3d1f5733 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -31,7 +31,6 @@ import org.apache.spark.ml.util.Instrumentation.instrumented
 import org.apache.spark.mllib.clustering.{DistanceMeasure, KMeans => MLlibKMeans, KMeansModel => MLlibKMeansModel}
 import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors}
 import org.apache.spark.mllib.linalg.VectorImplicits._
-import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{DoubleType, IntegerType, StructType}
@@ -329,22 +328,6 @@ class KMeans @Since("1.5.0") (
   override def fit(dataset: Dataset[_]): KMeansModel = instrumented { instr =>
     transformSchema(dataset.schema, logging = true)
 
-    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
-    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
-      col($(weightCol)).cast(DoubleType)
-    } else {
-      lit(1.0)
-    }
-
-    val instances: RDD[(OldVector, Double)] = dataset
-      .select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w).rdd.map {
-      case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight)
-    }
-
-    if (handlePersistence) {
-      instances.persist(StorageLevel.MEMORY_AND_DISK)
-    }
-
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, featuresCol, predictionCol, k, initMode, initSteps, distanceMeasure,
@@ -357,8 +340,19 @@ class KMeans @Since("1.5.0") (
       .setSeed($(seed))
       .setEpsilon($(tol))
       .setDistanceMeasure($(distanceMeasure))
-    val parentModel = algo.runWithWeight(instances, Option(instr))
+
+    val w = if (isDefined(weightCol) && $(weightCol).nonEmpty) {
+      col($(weightCol)).cast(DoubleType)
+    } else {
+      lit(1.0)
+    }
+    val instances = dataset.select(DatasetUtils.columnToVector(dataset, getFeaturesCol), w)
+      .rdd.map { case Row(point: Vector, weight: Double) => (OldVectors.fromML(point), weight) }
+
+    val handlePersistence = dataset.storageLevel == StorageLevel.NONE
+    val parentModel = algo.runWithWeight(instances, handlePersistence, Some(instr))
     val model = copyValues(new KMeansModel(uid, parentModel).setParent(this))
+
     val summary = new KMeansSummary(
       model.transform(dataset),
       $(predictionCol),
@@ -369,9 +363,6 @@ class KMeans @Since("1.5.0") (
 
     model.setSummary(Some(summary))
     instr.logNamedValue("clusterSizes", summary.clusterSizes)
-    if (handlePersistence) {
-      instances.unpersist()
-    }
     model
   }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
index 7c12697be95c8..7255f12846525 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeans.scala
@@ -153,30 +153,25 @@ class BisectingKMeans private (
     this
   }
 
-  private[spark] def run(
-      input: RDD[Vector],
-      instr: Option[Instrumentation]): BisectingKMeansModel = {
-    val instances: RDD[(Vector, Double)] = input.map {
-      case (point) => (point, 1.0)
-    }
-    runWithWeight(instances, None)
-  }
-
   private[spark] def runWithWeight(
-      input: RDD[(Vector, Double)],
+      instances: RDD[(Vector, Double)],
+      handlePersistence: Boolean,
       instr: Option[Instrumentation]): BisectingKMeansModel = {
-    val d = input.map(_._1.size).first
+    val d = instances.map(_._1.size).first
     logInfo(s"Feature dimension: $d.")
 
-    val dMeasure: DistanceMeasure = DistanceMeasure.decodeFromString(this.distanceMeasure)
-    // Compute and cache vector norms for fast distance computation.
-    val norms = input.map(d => Vectors.norm(d._1, 2.0))
-    val vectors = input.zip(norms).map {
-      case ((x, weight), norm) => new VectorWithNorm(x, norm, weight)
-    }
-    if (input.getStorageLevel == StorageLevel.NONE) {
+    val dMeasure = DistanceMeasure.decodeFromString(this.distanceMeasure)
+    val norms = instances.map(d => Vectors.norm(d._1, 2.0))
+    val vectors = instances.zip(norms)
+      .map { case ((x, weight), norm) => new VectorWithNorm(x, norm, weight) }
+
+    if (handlePersistence) {
       vectors.persist(StorageLevel.MEMORY_AND_DISK)
+    } else {
+      // Compute and cache vector norms for fast distance computation.
+      norms.persist(StorageLevel.MEMORY_AND_DISK)
     }
+
     var assignments = vectors.map(v => (ROOT_INDEX, v))
     var activeClusters = summarize(d, assignments, dMeasure)
     instr.foreach(_.logNumExamples(activeClusters.values.map(_.size).sum))
@@ -244,13 +239,11 @@ class BisectingKMeans private (
       }
       level += 1
     }
-    if (preIndices != null) {
-      preIndices.unpersist()
-    }
-    if (indices != null) {
-      indices.unpersist()
-    }
-    vectors.unpersist()
+
+    if (preIndices != null) { preIndices.unpersist() }
+    if (indices != null) { indices.unpersist() }
+    if (handlePersistence) { vectors.unpersist() } else { norms.unpersist() }
+
     val clusters = activeClusters ++ inactiveClusters
     val root = buildTree(clusters, dMeasure)
     val totalCost = root.leafNodes.map(_.cost).sum
@@ -264,7 +257,9 @@ class BisectingKMeans private (
    */
   @Since("1.6.0")
   def run(input: RDD[Vector]): BisectingKMeansModel = {
-    run(input, None)
+    val instances = input.map(point => (point, 1.0))
+    val handlePersistence = input.getStorageLevel == StorageLevel.NONE
+    runWithWeight(instances, handlePersistence, None)
   }
 
   /**
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
index a3cf7f96478af..b03151247d8e7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
@@ -209,30 +209,27 @@ class KMeans private (
    */
   @Since("0.8.0")
   def run(data: RDD[Vector]): KMeansModel = {
-    val instances: RDD[(Vector, Double)] = data.map {
-      case (point) => (point, 1.0)
-    }
-    runWithWeight(instances, None)
+    val instances = data.map(point => (point, 1.0))
+    val handlePersistence = data.getStorageLevel == StorageLevel.NONE
+    runWithWeight(instances, handlePersistence, None)
   }
 
   private[spark] def runWithWeight(
-      data: RDD[(Vector, Double)],
+      instances: RDD[(Vector, Double)],
+      handlePersistence: Boolean,
       instr: Option[Instrumentation]): KMeansModel = {
+    val norms = instances.map { case (v, _) => Vectors.norm(v, 2.0) }
+    val vectors = instances.zip(norms)
+      .map { case ((v, w), norm) => new VectorWithNorm(v, norm, w) }
 
-    // Compute squared norms and cache them.
-    val norms = data.map { case (v, _) =>
-      Vectors.norm(v, 2.0)
-    }
-
-    val zippedData = data.zip(norms).map { case ((v, w), norm) =>
-      new VectorWithNorm(v, norm, w)
-    }
-
-    if (data.getStorageLevel == StorageLevel.NONE) {
-      zippedData.persist(StorageLevel.MEMORY_AND_DISK)
+    if (handlePersistence) {
+      vectors.persist(StorageLevel.MEMORY_AND_DISK)
+    } else {
+      // Compute squared norms and cache them.
+      norms.persist(StorageLevel.MEMORY_AND_DISK)
     }
-    val model = runAlgorithmWithWeight(zippedData, instr)
-    zippedData.unpersist()
+    val model = runAlgorithmWithWeight(vectors, instr)
+    if (handlePersistence) { vectors.unpersist() } else { norms.unpersist() }
 
     model
   }

From 82aef3e22db3855ae95d0af2c2a4a668ca7676d2 Mon Sep 17 00:00:00 2001
From: Yesheng Ma <kimi.ysma@gmail.com>
Date: Tue, 25 Aug 2020 11:20:01 +0900
Subject: [PATCH 1002/1280] [MINOR][SQL] Add missing documentation for LongType
 mapping

### What changes were proposed in this pull request?

Added Java docs for Long data types in the Row class.

### Why are the changes needed?

The Long datatype is somehow missing in Row.scala's `apply` and `get` methods.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Existing UTs.

Closes #29534 from yeshengm/docs-fix.

Authored-by: Yesheng Ma <kimi.ysma@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 3eee915b474c58cff9ea108f67073ed9c0c86224)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
index 5b17f1d65f1bf..3928a8c95eee4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Row.scala
@@ -160,6 +160,7 @@ trait Row extends Serializable {
    *   ByteType -> java.lang.Byte
    *   ShortType -> java.lang.Short
    *   IntegerType -> java.lang.Integer
+   *   LongType -> java.lang.Long
    *   FloatType -> java.lang.Float
    *   DoubleType -> java.lang.Double
    *   StringType -> String
@@ -188,6 +189,7 @@ trait Row extends Serializable {
    *   ByteType -> java.lang.Byte
    *   ShortType -> java.lang.Short
    *   IntegerType -> java.lang.Integer
+   *   LongType -> java.lang.Long
    *   FloatType -> java.lang.Float
    *   DoubleType -> java.lang.Double
    *   StringType -> String

From 6c88d7c1259ea9fe89f5c8190c683bba506d528e Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Tue, 25 Aug 2020 04:42:39 +0000
Subject: [PATCH 1003/1280] 
 [SPARK-32646][SQL][3.0][TEST-HADOOP2.7][TEST-HIVE1.2] ORC predicate pushdown
 should work with case-insensitive analysis

### What changes were proposed in this pull request?

This PR proposes to fix ORC predicate pushdown under case-insensitive analysis case. The field names in pushed down predicates don't need to match in exact letter case with physical field names in ORC files, if we enable case-insensitive analysis.

### Why are the changes needed?

Currently ORC predicate pushdown doesn't work with case-insensitive analysis. A predicate "a < 0" cannot pushdown to ORC file with field name "A" under case-insensitive analysis.

But Parquet predicate pushdown works with this case. We should make ORC predicate pushdown work with case-insensitive analysis too.

### Does this PR introduce _any_ user-facing change?

Yes, after this PR, under case-insensitive analysis, ORC predicate pushdown will work.

### How was this patch tested?

Unit tests.

Closes #29513 from viirya/fix-orc-pushdown-3.0.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../datasources/orc/OrcFileFormat.scala       | 16 +++--
 .../datasources/orc/OrcFiltersBase.scala      | 35 ++++++++-
 .../execution/datasources/orc/OrcUtils.scala  | 14 ++++
 .../v2/orc/OrcPartitionReaderFactory.scala    | 22 +++++-
 .../datasources/v2/orc/OrcScan.scala          |  2 +-
 .../datasources/v2/orc/OrcScanBuilder.scala   |  9 +--
 .../datasources/orc/OrcFilters.scala          | 72 ++++++++++---------
 .../datasources/orc/OrcFilterSuite.scala      | 49 ++++++++++++-
 .../datasources/orc/OrcFilters.scala          | 70 +++++++++---------
 .../datasources/orc/OrcFilterSuite.scala      | 49 ++++++++++++-
 10 files changed, 253 insertions(+), 85 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
index 4dff1ec7ebfb9..69badb4f7d595 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
@@ -153,11 +153,6 @@ class OrcFileFormat
       filters: Seq[Filter],
       options: Map[String, String],
       hadoopConf: Configuration): (PartitionedFile) => Iterator[InternalRow] = {
-    if (sparkSession.sessionState.conf.orcFilterPushDown) {
-      OrcFilters.createFilter(dataSchema, filters).foreach { f =>
-        OrcInputFormat.setSearchArgument(hadoopConf, f, dataSchema.fieldNames)
-      }
-    }
 
     val resultSchema = StructType(requiredSchema.fields ++ partitionSchema.fields)
     val sqlConf = sparkSession.sessionState.conf
@@ -169,6 +164,8 @@ class OrcFileFormat
     val broadcastedConf =
       sparkSession.sparkContext.broadcast(new SerializableConfiguration(hadoopConf))
     val isCaseSensitive = sparkSession.sessionState.conf.caseSensitiveAnalysis
+    val orcFilterPushDown = sparkSession.sessionState.conf.orcFilterPushDown
+    val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
 
     (file: PartitionedFile) => {
       val conf = broadcastedConf.value.value
@@ -186,6 +183,15 @@ class OrcFileFormat
       if (resultedColPruneInfo.isEmpty) {
         Iterator.empty
       } else {
+        // ORC predicate pushdown
+        if (orcFilterPushDown) {
+          OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).map { fileSchema =>
+            OrcFilters.createFilter(fileSchema, filters).foreach { f =>
+              OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
+            }
+          }
+        }
+
         val (requestedColIds, canPruneCols) = resultedColPruneInfo.get
         val resultSchemaString = OrcUtils.orcResultSchemaString(canPruneCols,
           dataSchema, resultSchema, partitionSchema, conf)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
index e673309188756..4554899ec2827 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFiltersBase.scala
@@ -17,14 +17,45 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.util.Locale
+
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
+import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.sources.{And, Filter}
-import org.apache.spark.sql.types.{AtomicType, BinaryType, DataType}
+import org.apache.spark.sql.types.{AtomicType, BinaryType, DataType, StructType}
 
 /**
  * Methods that can be shared when upgrading the built-in Hive.
  */
 trait OrcFiltersBase {
 
+  case class OrcPrimitiveField(fieldName: String, fieldType: DataType)
+
+  protected[sql] def getDataTypeMap(
+      schema: StructType,
+      caseSensitive: Boolean): Map[String, OrcPrimitiveField] = {
+    val fields = schema.flatMap { f =>
+      if (isSearchableType(f.dataType)) {
+        Some(quoteIfNeeded(f.name) -> OrcPrimitiveField(quoteIfNeeded(f.name), f.dataType))
+      } else {
+        None
+      }
+    }
+
+    if (caseSensitive) {
+      fields.toMap
+    } else {
+      // Don't consider ambiguity here, i.e. more than one field are matched in case insensitive
+      // mode, just skip pushdown for these fields, they will trigger Exception when reading,
+      // See: SPARK-25175.
+      val dedupPrimitiveFields = fields
+        .groupBy(_._1.toLowerCase(Locale.ROOT))
+        .filter(_._2.size == 1)
+        .mapValues(_.head._2)
+      CaseInsensitiveMap(dedupPrimitiveFields)
+    }
+  }
+
   private[sql] def buildTree(filters: Seq[Filter]): Option[Filter] = {
     filters match {
       case Seq() => None
@@ -40,7 +71,7 @@ trait OrcFiltersBase {
    * Return true if this is a searchable type in ORC.
    * Both CharType and VarcharType are cleaned at AstBuilder.
    */
-  protected[sql] def isSearchableType(dataType: DataType) = dataType match {
+  private def isSearchableType(dataType: DataType) = dataType match {
     case BinaryType => false
     case _: AtomicType => true
     case _ => false
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index e10253989788b..be36432d6325f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -92,6 +92,20 @@ object OrcUtils extends Logging {
     }
   }
 
+  def readCatalystSchema(
+      file: Path,
+      conf: Configuration,
+      ignoreCorruptFiles: Boolean): Option[StructType] = {
+    readSchema(file, conf, ignoreCorruptFiles) match {
+      case Some(schema) =>
+        Some(CatalystSqlParser.parseDataType(schema.toString).asInstanceOf[StructType])
+
+      case None =>
+        // Field names is empty or `FileFormatException` was thrown but ignoreCorruptFiles is true.
+        None
+    }
+  }
+
   /**
    * Reads ORC file schemas in multi-threaded manner, using native version of ORC.
    * This is visible for testing.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
index 7f25f7bd135f0..1f38128e98fa5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
@@ -31,9 +31,10 @@ import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader}
 import org.apache.spark.sql.execution.datasources.PartitionedFile
-import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, OrcDeserializer, OrcUtils}
+import org.apache.spark.sql.execution.datasources.orc.{OrcColumnarBatchReader, OrcDeserializer, OrcFilters, OrcUtils}
 import org.apache.spark.sql.execution.datasources.v2._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.{AtomicType, StructType}
 import org.apache.spark.sql.vectorized.ColumnarBatch
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -52,10 +53,13 @@ case class OrcPartitionReaderFactory(
     broadcastedConf: Broadcast[SerializableConfiguration],
     dataSchema: StructType,
     readDataSchema: StructType,
-    partitionSchema: StructType) extends FilePartitionReaderFactory {
+    partitionSchema: StructType,
+    filters: Array[Filter]) extends FilePartitionReaderFactory {
   private val resultSchema = StructType(readDataSchema.fields ++ partitionSchema.fields)
   private val isCaseSensitive = sqlConf.caseSensitiveAnalysis
   private val capacity = sqlConf.orcVectorizedReaderBatchSize
+  private val orcFilterPushDown = sqlConf.orcFilterPushDown
+  private val ignoreCorruptFiles = sqlConf.ignoreCorruptFiles
 
   override def supportColumnarReads(partition: InputPartition): Boolean = {
     sqlConf.orcVectorizedReaderEnabled && sqlConf.wholeStageEnabled &&
@@ -63,6 +67,16 @@ case class OrcPartitionReaderFactory(
       resultSchema.forall(_.dataType.isInstanceOf[AtomicType])
   }
 
+  private def pushDownPredicates(filePath: Path, conf: Configuration): Unit = {
+    if (orcFilterPushDown) {
+      OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).map { fileSchema =>
+        OrcFilters.createFilter(fileSchema, filters).foreach { f =>
+          OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
+        }
+      }
+    }
+  }
+
   override def buildReader(file: PartitionedFile): PartitionReader[InternalRow] = {
     val conf = broadcastedConf.value.value
 
@@ -70,6 +84,8 @@ case class OrcPartitionReaderFactory(
 
     val filePath = new Path(new URI(file.filePath))
 
+    pushDownPredicates(filePath, conf)
+
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
     val resultedColPruneInfo =
@@ -116,6 +132,8 @@ case class OrcPartitionReaderFactory(
 
     val filePath = new Path(new URI(file.filePath))
 
+    pushDownPredicates(filePath, conf)
+
     val fs = filePath.getFileSystem(conf)
     val readerOptions = OrcFile.readerOptions(conf).filesystem(fs)
     val resultedColPruneInfo =
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
index 62894fa7a2538..35e3b1a88e087 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScan.scala
@@ -48,7 +48,7 @@ case class OrcScan(
     // The partition values are already truncated in `FileScan.partitions`.
     // We should use `readPartitionSchema` as the partition schema here.
     OrcPartitionReaderFactory(sparkSession.sessionState.conf, broadcastedConf,
-      dataSchema, readDataSchema, readPartitionSchema)
+      dataSchema, readDataSchema, readPartitionSchema, pushedFilters)
   }
 
   override def equals(obj: Any): Boolean = obj match {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
index 9f40f5faa2e99..6a9cb250ca3d9 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcScanBuilder.scala
@@ -22,11 +22,11 @@ import scala.collection.JavaConverters._
 import org.apache.orc.mapreduce.OrcInputFormat
 
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
 import org.apache.spark.sql.connector.read.{Scan, SupportsPushDownFilters}
 import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex
 import org.apache.spark.sql.execution.datasources.orc.OrcFilters
 import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -55,12 +55,7 @@ case class OrcScanBuilder(
 
   override def pushFilters(filters: Array[Filter]): Array[Filter] = {
     if (sparkSession.sessionState.conf.orcFilterPushDown) {
-      OrcFilters.createFilter(schema, filters).foreach { f =>
-        // The pushed filters will be set in `hadoopConf`. After that, we can simply use the
-        // changed `hadoopConf` in executors.
-        OrcInputFormat.setSearchArgument(hadoopConf, f, schema.fieldNames)
-      }
-      val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
+      val dataTypeMap = OrcFilters.getDataTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
       // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
       val newFilters = filters.filter(!_.containsNestedColumn)
       _pushedFilters = OrcFilters.convertibleFilters(schema, dataTypeMap, newFilters).toArray
diff --git a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index b68563956c82c..a068347634cda 100644
--- a/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v1.2/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -27,7 +27,7 @@ import org.apache.orc.storage.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 
@@ -68,7 +68,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * Create ORC filter as a SearchArgument instance.
    */
   def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
+    val dataTypeMap = getDataTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
     // Combines all convertible filters using `And` to produce a single conjunction
     // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
     val newFilters = filters.filter(!_.containsNestedColumn)
@@ -83,7 +83,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
 
   def convertibleFilters(
       schema: StructType,
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       filters: Seq[Filter]): Seq[Filter] = {
     import org.apache.spark.sql.sources._
 
@@ -141,7 +141,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
   /**
    * Get PredicateLeafType which is corresponding to the given DataType.
    */
-  private def getPredicateLeafType(dataType: DataType) = dataType match {
+  private[sql] def getPredicateLeafType(dataType: DataType) = dataType match {
     case BooleanType => PredicateLeaf.Type.BOOLEAN
     case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
     case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
@@ -181,7 +181,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Builder = {
     import org.apache.spark.sql.sources._
@@ -217,11 +217,11 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Option[Builder] = {
     def getType(attribute: String): PredicateLeaf.Type =
-      getPredicateLeafType(dataTypeMap(attribute))
+      getPredicateLeafType(dataTypeMap(attribute).fieldType)
 
     import org.apache.spark.sql.sources._
 
@@ -231,39 +231,47 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
     // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().equals(name, getType(name), castedValue).end())
+      case EqualTo(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .equals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case EqualNullSafe(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().nullSafeEquals(name, getType(name), castedValue).end())
+      case EqualNullSafe(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .nullSafeEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThan(name, getType(name), castedValue).end())
+      case LessThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThanEquals(name, getType(name), castedValue).end())
+      case LessThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThanEquals(name, getType(name), castedValue).end())
+      case GreaterThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThan(name, getType(name), castedValue).end())
+      case GreaterThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case IsNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startAnd().isNull(name, getType(name)).end())
+      case IsNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startAnd()
+          .isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case IsNotNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startNot().isNull(name, getType(name)).end())
+      case IsNotNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startNot()
+          .isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case In(name, values) if isSearchableType(dataTypeMap(name)) =>
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name)))
-        Some(builder.startAnd().in(name, getType(name),
+      case In(name, values) if dataTypeMap.contains(name) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name).fieldType))
+        Some(builder.startAnd().in(dataTypeMap(name).fieldName, getType(name),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
 
       case _ => None
diff --git a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index beb72322326fe..a3c2343a73b97 100644
--- a/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v1.2/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -24,6 +24,7 @@ import java.sql.{Date, Timestamp}
 import scala.collection.JavaConverters._
 
 import org.apache.orc.storage.ql.io.sarg.{PredicateLeaf, SearchArgument}
+import org.apache.orc.storage.ql.io.sarg.SearchArgumentFactory.newBuilder
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
@@ -542,8 +543,7 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
           checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
 
           val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
-          // TODO: ORC predicate pushdown should work under case-insensitive analysis.
-          // assert(actual.count() == 1)
+          assert(actual.count() == 1)
         }
       }
 
@@ -562,5 +562,50 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-32646: Case-insensitive field resolution for pushdown when reading ORC") {
+    import org.apache.spark.sql.sources._
+
+    def getOrcFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        caseSensitive: String): Option[SearchArgument] = {
+      var orcFilter: Option[SearchArgument] = None
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
+        orcFilter =
+          OrcFilters.createFilter(schema, filters)
+      }
+      orcFilter
+    }
+
+    def testFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        expected: SearchArgument): Unit = {
+      val caseSensitiveFilters = getOrcFilter(schema, filters, "true")
+      val caseInsensitiveFilters = getOrcFilter(schema, filters, "false")
+
+      assert(caseSensitiveFilters.isEmpty)
+      assert(caseInsensitiveFilters.isDefined)
+
+      assert(caseInsensitiveFilters.get.getLeaves().size() > 0)
+      assert(caseInsensitiveFilters.get.getLeaves().size() == expected.getLeaves().size())
+      (0 until expected.getLeaves().size()).foreach { index =>
+        assert(caseInsensitiveFilters.get.getLeaves().get(index) == expected.getLeaves().get(index))
+      }
+    }
+
+    val schema = StructType(Seq(StructField("cint", IntegerType)))
+    testFilter(schema, Seq(GreaterThan("CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema, Seq(
+      And(GreaterThan("CINT", 1), EqualTo("Cint", 2))),
+      newBuilder.startAnd()
+        .startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`()
+        .equals("cint", OrcFilters.getPredicateLeafType(IntegerType), 2L)
+        .`end`().build())
+  }
 }
 
diff --git a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index 4b642080d25ad..9f1927eff3993 100644
--- a/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/v2.3/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable
 
 import org.apache.spark.SparkException
 import org.apache.spark.sql.catalyst.util.DateTimeUtils.{instantToMicros, localDateToDays, toJavaDate, toJavaTimestamp}
-import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.quoteIfNeeded
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types._
 
@@ -68,7 +68,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * Create ORC filter as a SearchArgument instance.
    */
   def createFilter(schema: StructType, filters: Seq[Filter]): Option[SearchArgument] = {
-    val dataTypeMap = schema.map(f => quoteIfNeeded(f.name) -> f.dataType).toMap
+    val dataTypeMap = getDataTypeMap(schema, SQLConf.get.caseSensitiveAnalysis)
     // Combines all convertible filters using `And` to produce a single conjunction
     // TODO (SPARK-25557): ORC doesn't support nested predicate pushdown, so they are removed.
     val newFilters = filters.filter(!_.containsNestedColumn)
@@ -83,7 +83,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
 
   def convertibleFilters(
       schema: StructType,
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       filters: Seq[Filter]): Seq[Filter] = {
     import org.apache.spark.sql.sources._
 
@@ -141,7 +141,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
   /**
    * Get PredicateLeafType which is corresponding to the given DataType.
    */
-  private def getPredicateLeafType(dataType: DataType) = dataType match {
+  private[sql] def getPredicateLeafType(dataType: DataType) = dataType match {
     case BooleanType => PredicateLeaf.Type.BOOLEAN
     case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
     case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
@@ -181,7 +181,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Builder = {
     import org.apache.spark.sql.sources._
@@ -217,11 +217,11 @@ private[sql] object OrcFilters extends OrcFiltersBase {
    * @return the builder so far.
    */
   private def buildLeafSearchArgument(
-      dataTypeMap: Map[String, DataType],
+      dataTypeMap: Map[String, OrcPrimitiveField],
       expression: Filter,
       builder: Builder): Option[Builder] = {
     def getType(attribute: String): PredicateLeaf.Type =
-      getPredicateLeafType(dataTypeMap(attribute))
+      getPredicateLeafType(dataTypeMap(attribute).fieldType)
 
     import org.apache.spark.sql.sources._
 
@@ -231,39 +231,45 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
     // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().equals(name, getType(name), castedValue).end())
+      case EqualTo(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .equals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case EqualNullSafe(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().nullSafeEquals(name, getType(name), castedValue).end())
+      case EqualNullSafe(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .nullSafeEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThan(name, getType(name), castedValue).end())
+      case LessThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case LessThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startAnd().lessThanEquals(name, getType(name), castedValue).end())
+      case LessThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startAnd()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThan(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThanEquals(name, getType(name), castedValue).end())
+      case GreaterThan(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThanEquals(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case GreaterThanOrEqual(name, value) if isSearchableType(dataTypeMap(name)) =>
-        val castedValue = castLiteralValue(value, dataTypeMap(name))
-        Some(builder.startNot().lessThan(name, getType(name), castedValue).end())
+      case GreaterThanOrEqual(name, value) if dataTypeMap.contains(name) =>
+        val castedValue = castLiteralValue(value, dataTypeMap(name).fieldType)
+        Some(builder.startNot()
+          .lessThan(dataTypeMap(name).fieldName, getType(name), castedValue).end())
 
-      case IsNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startAnd().isNull(name, getType(name)).end())
+      case IsNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startAnd().isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case IsNotNull(name) if isSearchableType(dataTypeMap(name)) =>
-        Some(builder.startNot().isNull(name, getType(name)).end())
+      case IsNotNull(name) if dataTypeMap.contains(name) =>
+        Some(builder.startNot().isNull(dataTypeMap(name).fieldName, getType(name)).end())
 
-      case In(name, values) if isSearchableType(dataTypeMap(name)) =>
-        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name)))
-        Some(builder.startAnd().in(name, getType(name),
+      case In(name, values) if dataTypeMap.contains(name) =>
+        val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(name).fieldType))
+        Some(builder.startAnd().in(dataTypeMap(name).fieldName, getType(name),
           castedValues.map(_.asInstanceOf[AnyRef]): _*).end())
 
       case _ => None
diff --git a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
index a3e450c5cd9f5..cb69413277f63 100644
--- a/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
+++ b/sql/core/v2.3/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilterSuite.scala
@@ -24,6 +24,7 @@ import java.sql.{Date, Timestamp}
 import scala.collection.JavaConverters._
 
 import org.apache.hadoop.hive.ql.io.sarg.{PredicateLeaf, SearchArgument}
+import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory.newBuilder
 
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
@@ -543,8 +544,7 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
           checkAnswer(sql(s"select a from $tableName"), (0 until count).map(c => Row(c - 1)))
 
           val actual = stripSparkFilter(sql(s"select a from $tableName where a < 0"))
-          // TODO: ORC predicate pushdown should work under case-insensitive analysis.
-          // assert(actual.count() == 1)
+          assert(actual.count() == 1)
         }
       }
 
@@ -563,5 +563,50 @@ class OrcFilterSuite extends OrcTest with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-32646: Case-insensitive field resolution for pushdown when reading ORC") {
+    import org.apache.spark.sql.sources._
+
+    def getOrcFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        caseSensitive: String): Option[SearchArgument] = {
+      var orcFilter: Option[SearchArgument] = None
+      withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive) {
+        orcFilter =
+          OrcFilters.createFilter(schema, filters)
+      }
+      orcFilter
+    }
+
+    def testFilter(
+        schema: StructType,
+        filters: Seq[Filter],
+        expected: SearchArgument): Unit = {
+      val caseSensitiveFilters = getOrcFilter(schema, filters, "true")
+      val caseInsensitiveFilters = getOrcFilter(schema, filters, "false")
+
+      assert(caseSensitiveFilters.isEmpty)
+      assert(caseInsensitiveFilters.isDefined)
+
+      assert(caseInsensitiveFilters.get.getLeaves().size() > 0)
+      assert(caseInsensitiveFilters.get.getLeaves().size() == expected.getLeaves().size())
+      (0 until expected.getLeaves().size()).foreach { index =>
+        assert(caseInsensitiveFilters.get.getLeaves().get(index) == expected.getLeaves().get(index))
+      }
+    }
+
+    val schema = StructType(Seq(StructField("cint", IntegerType)))
+    testFilter(schema, Seq(GreaterThan("CINT", 1)),
+      newBuilder.startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`().build())
+    testFilter(schema, Seq(
+      And(GreaterThan("CINT", 1), EqualTo("Cint", 2))),
+      newBuilder.startAnd()
+        .startNot()
+        .lessThanEquals("cint", OrcFilters.getPredicateLeafType(IntegerType), 1L).`end`()
+        .equals("cint", OrcFilters.getPredicateLeafType(IntegerType), 2L)
+        .`end`().build())
+  }
 }
 

From 21ac7e2b696a6fc6958fdb84ae2b960892c7349e Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 25 Aug 2020 13:17:03 +0000
Subject: [PATCH 1004/1280] [SPARK-32683][DOCS][SQL] Fix doc error and add
 migration guide for datetime pattern F

### What changes were proposed in this pull request?

This PR fixes the doc error and add a migration guide for datetime pattern.

### Why are the changes needed?
This is a bug of the doc that we inherited from JDK https://bugs.openjdk.java.net/browse/JDK-8169482

The SimpleDateFormatter(**F Day of week in month**) we used in 2.x and the DatetimeFormatter(**F week-of-month**) we use now both have the opposite meanings to what they declared in the java docs. And unfortunately, this also leads to silent data change in Spark too.

The `week-of-month` is actually the pattern `W` in DatetimeFormatter, which is banned to use in Spark 3.x.

If we want to keep pattern `F`, we need to accept the behavior change with proper migration guide and fix the doc in Spark

### Does this PR introduce _any_ user-facing change?

Yes, doc changed

### How was this patch tested?

passing ci doc generating job

Closes #29538 from yaooqinn/SPARK-32683.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1f3bb5175749816be1f0bc793ed5239abf986000)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md      | 2 ++
 docs/sql-ref-datetime-pattern.md | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index ce5655acda782..532ac70fb91be 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -173,6 +173,8 @@ license: |
 
   - Since Spark 3.0, when using `EXTRACT` expression to extract the second field from date/timestamp values, the result will be a `DecimalType(8, 6)` value with 2 digits for second part, and 6 digits for the fractional part with microsecond precision. e.g. `extract(second from to_timestamp('2019-09-20 10:10:10.1'))` results `10.100000`.  In Spark version 2.4 and earlier, it returns an `IntegerType` value and the result for the former example is `10`.
 
+  - In Spark 3.0, datetime pattern letter `F` is **aligned day of week in month** that represents the concept of the count of days within the period of a week where the weeks are aligned to the start of the month. In Spark version 2.4 and earlier, it is **week of month** that represents the concept of the count of weeks within the month where weeks start on a fixed day-of-week, e.g. `2020-07-30` is 30 days (4 weeks and 2 days) after the first day of the month, so `date_format(date '2020-07-30', 'F')` returns 2 in Spark 3.0, but as a week count in Spark 2.x, it returns 5 because it locates in the 5th week of July 2020, where week one is 2020-07-01 to 07-04.
+
 ### Data Sources
 
   - In Spark version 2.4 and below, when reading a Hive SerDe table with Spark native data sources(parquet/orc), Spark infers the actual file schema and update the table schema in metastore. In Spark 3.0, Spark doesn't infer the schema anymore. This should not cause any problems to end users, but if it does, set `spark.sql.hive.caseSensitiveInferenceMode` to `INFER_AND_SAVE`.
diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md
index d0299e5a99dc2..4b02cdad36109 100644
--- a/docs/sql-ref-datetime-pattern.md
+++ b/docs/sql-ref-datetime-pattern.md
@@ -37,7 +37,7 @@ Spark uses pattern letters in the following table for date and timestamp parsing
 |**d**|day-of-month|number(3)|28|
 |**Q/q**|quarter-of-year|number/text|3; 03; Q3; 3rd quarter|
 |**E**|day-of-week|text|Tue; Tuesday|
-|**F**|week-of-month|number(1)|3|
+|**F**|aligned day of week in month|number(1)|3|
 |**a**|am-pm-of-day|am-pm|PM|
 |**h**|clock-hour-of-am-pm (1-12)|number(2)|12|
 |**K**|hour-of-am-pm (0-11)|number(2)|0|

From 68ff809060c312fee1fbe8d46657e6c7aec62dba Mon Sep 17 00:00:00 2001
From: Sean Owen <srowen@gmail.com>
Date: Wed, 26 Aug 2020 00:25:58 +0900
Subject: [PATCH 1005/1280] [SPARK-32614][SQL] Don't apply comment processing
 if 'comment' unset for CSV

Spark's CSV source can optionally ignore lines starting with a comment char. Some code paths check to see if it's set before applying comment logic (i.e. not set to default of `\0`), but many do not, including the one that passes the option to Univocity. This means that rows beginning with a null char were being treated as comments even when 'disabled'.

To avoid dropping rows that start with a null char when this is not requested or intended. See JIRA for an example.

Nothing beyond the effect of the bug fix.

Existing tests plus new test case.

Closes #29516 from srowen/SPARK-32614.

Authored-by: Sean Owen <srowen@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a9d4e60a90d4d6765642e6bf7810da117af6437b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2       |  2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3       |  2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3       |  2 +-
 pom.xml                                       |  2 +-
 .../spark/sql/catalyst/csv/CSVExprUtils.scala | 11 +++++++---
 .../spark/sql/catalyst/csv/CSVOptions.scala   | 10 +++++++--
 .../execution/datasources/csv/CSVSuite.scala  | 22 ++++++++++++++-----
 7 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 82d5a06f906d8..e32ea645638f0 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -200,7 +200,7 @@ stream/2.9.6//stream-2.9.6.jar
 stringtemplate/3.2.1//stringtemplate-3.2.1.jar
 super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xercesImpl/2.12.0//xercesImpl-2.12.0.jar
 xml-apis/1.4.01//xml-apis-1.4.01.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 17c787e42fdba..168d6196fe1fe 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -213,7 +213,7 @@ stream/2.9.6//stream-2.9.6.jar
 super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 transaction-api/1.1//transaction-api-1.1.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 velocity/1.5//velocity-1.5.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
 xercesImpl/2.12.0//xercesImpl-2.12.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index b5a10b5dba378..d730b4a0e1cb0 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -229,7 +229,7 @@ super-csv/2.2.0//super-csv-2.2.0.jar
 threeten-extra/1.5.0//threeten-extra-1.5.0.jar
 token-provider/1.0.1//token-provider-1.0.1.jar
 transaction-api/1.1//transaction-api-1.1.jar
-univocity-parsers/2.8.3//univocity-parsers-2.8.3.jar
+univocity-parsers/2.9.0//univocity-parsers-2.9.0.jar
 velocity/1.5//velocity-1.5.jar
 woodstox-core/5.0.3//woodstox-core-5.0.3.jar
 xbean-asm7-shaded/4.15//xbean-asm7-shaded-4.15.jar
diff --git a/pom.xml b/pom.xml
index 1bf5de07024c2..32a13082bac91 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2287,7 +2287,7 @@
       <dependency>
         <groupId>com.univocity</groupId>
         <artifactId>univocity-parsers</artifactId>
-        <version>2.8.3</version>
+        <version>2.9.0</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hive</groupId>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
index 3e83c1dcb4758..efe4188377142 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala
@@ -25,8 +25,13 @@ object CSVExprUtils {
    * This is currently being used in CSV reading path and CSV schema inference.
    */
   def filterCommentAndEmpty(iter: Iterator[String], options: CSVOptions): Iterator[String] = {
-    iter.filter { line =>
-      line.trim.nonEmpty && !line.startsWith(options.comment.toString)
+    if (options.isCommentSet) {
+      val commentPrefix = options.comment.toString
+      iter.filter { line =>
+        line.trim.nonEmpty && !line.startsWith(commentPrefix)
+      }
+    } else {
+      iter.filter(_.trim.nonEmpty)
     }
   }
 
@@ -34,7 +39,7 @@ object CSVExprUtils {
     if (options.isCommentSet) {
       val commentPrefix = options.comment.toString
       iter.dropWhile { line =>
-        line.trim.isEmpty || line.trim.startsWith(commentPrefix)
+        line.trim.isEmpty || line.startsWith(commentPrefix)
       }
     } else {
       iter.dropWhile(_.trim.isEmpty)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
index 9d09cab4fd482..f2191fcf35f1a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -220,7 +220,9 @@ class CSVOptions(
     format.setQuote(quote)
     format.setQuoteEscape(escape)
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
-    format.setComment(comment)
+    if (isCommentSet) {
+      format.setComment(comment)
+    }
     lineSeparatorInWrite.foreach(format.setLineSeparator)
 
     writerSettings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceFlagInWrite)
@@ -242,7 +244,11 @@ class CSVOptions(
     format.setQuoteEscape(escape)
     lineSeparator.foreach(format.setLineSeparator)
     charToEscapeQuoteEscaping.foreach(format.setCharToEscapeQuoteEscaping)
-    format.setComment(comment)
+    if (isCommentSet) {
+      format.setComment(comment)
+    } else {
+      settings.setCommentProcessingEnabled(false)
+    }
 
     settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceInRead)
     settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhiteSpaceInRead)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index d54146a0fb065..9ba2cab3fe31c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -1902,25 +1902,26 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
 
   test("SPARK-25387: bad input should not cause NPE") {
     val schema = StructType(StructField("a", IntegerType) :: Nil)
-    val input = spark.createDataset(Seq("\u0000\u0000\u0001234"))
+    val input = spark.createDataset(Seq("\u0001\u0000\u0001234"))
 
     checkAnswer(spark.read.schema(schema).csv(input), Row(null))
     checkAnswer(spark.read.option("multiLine", true).schema(schema).csv(input), Row(null))
-    assert(spark.read.csv(input).collect().toSet == Set(Row()))
+    assert(spark.read.schema(schema).csv(input).collect().toSet == Set(Row(null)))
   }
 
   test("SPARK-31261: bad csv input with `columnNameCorruptRecord` should not cause NPE") {
     val schema = StructType(
       StructField("a", IntegerType) :: StructField("_corrupt_record", StringType) :: Nil)
-    val input = spark.createDataset(Seq("\u0000\u0000\u0001234"))
+    val input = spark.createDataset(Seq("\u0001\u0000\u0001234"))
 
     checkAnswer(
       spark.read
         .option("columnNameOfCorruptRecord", "_corrupt_record")
         .schema(schema)
         .csv(input),
-      Row(null, null))
-    assert(spark.read.csv(input).collect().toSet == Set(Row()))
+      Row(null, "\u0001\u0000\u0001234"))
+    assert(spark.read.schema(schema).csv(input).collect().toSet ==
+      Set(Row(null, "\u0001\u0000\u0001234")))
   }
 
   test("field names of inferred schema shouldn't compare to the first row") {
@@ -2353,6 +2354,17 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       checkAnswer(csv, Row(null))
     }
   }
+
+  test("SPARK-32614: don't treat rows starting with null char as comment") {
+    withTempPath { path =>
+      Seq("\u0000foo", "bar", "baz").toDS.write.text(path.getCanonicalPath)
+      val df = spark.read.format("csv")
+        .option("header", "false")
+        .option("inferSchema", "true")
+        .load(path.getCanonicalPath)
+      assert(df.count() == 3)
+    }
+  }
 }
 
 class CSVv1Suite extends CSVSuite {

From 9e937c5602db8248badfdc2015403e85d3beb684 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Tue, 25 Aug 2020 18:46:10 -0700
Subject: [PATCH 1006/1280] [SPARK-32620][SQL] Reset the numPartitions metric
 when DPP is enabled

### What changes were proposed in this pull request?

This pr reset the `numPartitions` metric when DPP is enabled. Otherwise, it is always a [static value](https://github.com/apache/spark/blob/18cac6a9f0bf4a6d449393f1ee84004623b3c893/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala#L215).

### Why are the changes needed?

Fix metric issue.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test and manual test

For [this test case](https://github.com/apache/spark/blob/18cac6a9f0bf4a6d449393f1ee84004623b3c893/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala#L252-L280).

Before this pr:
![image](https://user-images.githubusercontent.com/5399861/90301798-9310b480-ded4-11ea-9294-49bcaba46f83.png)

After this pr:
![image](https://user-images.githubusercontent.com/5399861/90301709-0fef5e80-ded4-11ea-942d-4d45d1dd15bc.png)

Closes #29436 from wangyum/SPARK-32620.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Yuming Wang <wgyumg@gmail.com>
(cherry picked from commit 1354cf0842fe9cac2af84eeaa2fac9409db7b128)
Signed-off-by: Yuming Wang <wgyumg@gmail.com>
---
 .../apache/spark/sql/execution/DataSourceScanExec.scala    | 1 +
 .../apache/spark/sql/DynamicPartitionPruningSuite.scala    | 7 ++++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 7f534d5957ce4..03b214cfd339f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -239,6 +239,7 @@ case class FileSourceScanExec(
       setFilesNumAndSizeMetric(ret, false)
       val timeTakenMs = (System.nanoTime() - startTime) / 1000 / 1000
       driverMetrics("pruningTime") = timeTakenMs
+      driverMetrics("numPartitions") = ret.length
       ret
     } else {
       selectedPartitions
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 1fbc4a6912adb..45f4309d3da3b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -319,6 +319,8 @@ abstract class DynamicPartitionPruningSuiteBase
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
       withTable("fact", "dim") {
+        val numPartitions = 10
+
         spark.range(10)
           .map { x => Tuple3(x, x + 1, 0) }
           .toDF("did", "d1", "d2")
@@ -328,7 +330,7 @@ abstract class DynamicPartitionPruningSuiteBase
           .saveAsTable("dim")
 
         spark.range(100)
-          .map { x => Tuple2(x, x % 10) }
+          .map { x => Tuple2(x, x % numPartitions) }
           .toDF("f1", "fid")
           .write.partitionBy("fid")
           .format(tableFormat)
@@ -355,6 +357,7 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(!scan1.metrics.contains("staticFilesSize"))
         val allFilesNum = scan1.metrics("numFiles").value
         val allFilesSize = scan1.metrics("filesSize").value
+        assert(scan1.metrics("numPartitions").value === numPartitions)
 
         // No dynamic partition pruning, so no static metrics
         // Only files from fid = 5 partition are scanned
@@ -367,6 +370,7 @@ abstract class DynamicPartitionPruningSuiteBase
         val partFilesSize = scan2.metrics("filesSize").value
         assert(0 < partFilesNum && partFilesNum < allFilesNum)
         assert(0 < partFilesSize && partFilesSize < allFilesSize)
+        assert(scan2.metrics("numPartitions").value === 1)
 
         // Dynamic partition pruning is used
         // Static metrics are as-if reading the whole fact table
@@ -378,6 +382,7 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(scan3.metrics("staticFilesSize").value == allFilesSize)
         assert(scan3.metrics("numFiles").value == partFilesNum)
         assert(scan3.metrics("filesSize").value == partFilesSize)
+        assert(scan3.metrics("numPartitions").value === 1)
       }
     }
   }

From 24552b68d404aace6523c2b681b114b27d82a90e Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 26 Aug 2020 12:25:59 +0900
Subject: [PATCH 1007/1280] [SPARK-32695][INFRA] Explicitly cache and hash
 'build' directly in GitHub Actions

### What changes were proposed in this pull request?

This PR proposes to explicitly cache and hash the files/directories under 'build' for SBT and Zinc at GitHub Actions. Otherwise, it can end up with overwriting `build` directory. See also https://github.com/apache/spark/pull/29286#issuecomment-679368436

Previously, other files like `build/mvn` and `build/sbt` are also cached and overwritten. So, when you have some changes there, they are ignored.

### Why are the changes needed?

To make GitHub Actions build stable.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

The builds in this PR test it out.

Closes #29536 from HyukjinKwon/SPARK-32695.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b07e7429a6af27418da271ac7c374f325e843a25)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/build_and_test.yml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 581ad8a66c2b5..b79fa161f12ba 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -92,10 +92,14 @@ jobs:
         fetch-depth: 0
     # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
     - name: Cache Scala, SBT, Maven and Zinc
-      uses: actions/cache@v1
+      uses: actions/cache@v2
       with:
-        path: build
-        key: build-${{ hashFiles('**/pom.xml') }}
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
         restore-keys: |
           build-
     - name: Cache Maven local repository
@@ -109,7 +113,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.ivy2/cache
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/plugins.sbt') }}
+        key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-
     - name: Install JDK ${{ matrix.java }}

From 9e8fb4874233ba9b933d2a99e73e683c2fbde594 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Wed, 26 Aug 2020 06:57:43 +0000
Subject: [PATCH 1008/1280] [SPARK-32659][SQL] Fix the data issue when pruning
 DPP on non-atomic type

### What changes were proposed in this pull request?

Use `InSet` expression to fix data issue when pruning DPP on non-atomic type. for example:
   ```scala
    spark.range(1000)
    .select(col("id"), col("id").as("k"))
    .write
    .partitionBy("k")
    .format("parquet")
    .mode("overwrite")
    .saveAsTable("df1");

   spark.range(100)
   .select(col("id"), col("id").as("k"))
   .write
   .partitionBy("k")
   .format("parquet")
   .mode("overwrite")
   .saveAsTable("df2")

   spark.sql("set spark.sql.optimizer.dynamicPartitionPruning.fallbackFilterRatio=2")
   spark.sql("set spark.sql.optimizer.dynamicPartitionPruning.reuseBroadcastOnly=false")
   spark.sql("SELECT df1.id, df2.k FROM df1 JOIN df2 ON struct(df1.k) = struct(df2.k) AND df2.id < 2").show
   ```
   It should return two records, but it returns empty.

### Why are the changes needed?

Fix data issue

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Add new unit test.

Closes #29475 from wangyum/SPARK-32659.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a8b568800e64f6a163da28e5e53441f84355df14)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/execution/subquery.scala | 21 +++-----
 .../sql/DynamicPartitionPruningSuite.scala    | 51 ++++++++++++++++++-
 2 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
index c2270c57eb941..9d15c76faa1b5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
@@ -114,9 +114,10 @@ case class InSubqueryExec(
     child: Expression,
     plan: BaseSubqueryExec,
     exprId: ExprId,
-    private var resultBroadcast: Broadcast[Array[Any]] = null) extends ExecSubqueryExpression {
+    private var resultBroadcast: Broadcast[Set[Any]] = null) extends ExecSubqueryExpression {
 
-  @transient private var result: Array[Any] = _
+  @transient private var result: Set[Any] = _
+  @transient private lazy val inSet = InSet(child, result)
 
   override def dataType: DataType = BooleanType
   override def children: Seq[Expression] = child :: Nil
@@ -131,14 +132,11 @@ case class InSubqueryExec(
 
   def updateResult(): Unit = {
     val rows = plan.executeCollect()
-    result = child.dataType match {
-      case _: StructType => rows.toArray
-      case _ => rows.map(_.get(0, child.dataType))
-    }
+    result = rows.map(_.get(0, child.dataType)).toSet
     resultBroadcast = plan.sqlContext.sparkContext.broadcast(result)
   }
 
-  def values(): Option[Array[Any]] = Option(resultBroadcast).map(_.value)
+  def values(): Option[Set[Any]] = Option(resultBroadcast).map(_.value)
 
   private def prepareResult(): Unit = {
     require(resultBroadcast != null, s"$this has not finished")
@@ -149,17 +147,12 @@ case class InSubqueryExec(
 
   override def eval(input: InternalRow): Any = {
     prepareResult()
-    val v = child.eval(input)
-    if (v == null) {
-      null
-    } else {
-      result.contains(v)
-    }
+    inSet.eval(input)
   }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
     prepareResult()
-    InSet(child, result.toSet).doGenCode(ctx, ev)
+    inSet.doGenCode(ctx, ev)
   }
 
   override lazy val canonicalized: InSubqueryExec = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 45f4309d3da3b..bf91bdb79cf05 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql
 
 import org.scalatest.GivenWhenThen
 
-import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
+import org.apache.spark.sql.catalyst.expressions.{CodegenObjectFactoryMode, DynamicPruningExpression, Expression}
 import org.apache.spark.sql.catalyst.plans.ExistenceJoin
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
@@ -1309,6 +1309,55 @@ abstract class DynamicPartitionPruningSuiteBase
       )
     }
   }
+
+  test("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") {
+    withSQLConf(
+      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "2", // Make sure insert DPP
+      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false") {
+      withTable("df1", "df2") {
+        spark.range(1000)
+          .select(col("id"), col("id").as("k"))
+          .write
+          .partitionBy("k")
+          .format(tableFormat)
+          .mode("overwrite")
+          .saveAsTable("df1")
+
+        spark.range(100)
+          .select(col("id"), col("id").as("k"))
+          .write
+          .partitionBy("k")
+          .format(tableFormat)
+          .mode("overwrite")
+          .saveAsTable("df2")
+
+        Seq(CodegenObjectFactoryMode.NO_CODEGEN,
+          CodegenObjectFactoryMode.CODEGEN_ONLY).foreach { mode =>
+          Seq(true, false).foreach { pruning =>
+            withSQLConf(
+              SQLConf.CODEGEN_FACTORY_MODE.key -> mode.toString,
+              SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> s"$pruning") {
+              val df = sql(
+                """
+                  |SELECT df1.id, df2.k
+                  |FROM df1
+                  |  JOIN df2
+                  |  ON struct(df1.k) = struct(df2.k)
+                  |    AND df2.id < 2
+                  |""".stripMargin)
+              if (pruning) {
+                checkPartitionPruningPredicate(df, true, false)
+              } else {
+                checkPartitionPruningPredicate(df, false, false)
+              }
+
+              checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
+            }
+          }
+        }
+      }
+    }
+  }
 }
 
 class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase {

From 60f485671a07a93ae8a8506ed2c0999cfe6ded7b Mon Sep 17 00:00:00 2001
From: waleedfateem <waleed.fateem@gmail.com>
Date: Thu, 27 Aug 2020 09:05:50 -0500
Subject: [PATCH 1009/1280] [SPARK-32701][CORE][DOCS]
 mapreduce.fileoutputcommitter.algorithm.version default value

The current documentation states that the default value of spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version is 1 which is not entirely true since this configuration isn't set anywhere in Spark but rather inherited from the Hadoop FileOutputCommitter class.

### What changes were proposed in this pull request?

I'm submitting this change, to clarify that the default value will entirely depend on the Hadoop version of the runtime environment.

### Why are the changes needed?

An application would end up using algorithm version 1 on certain environments but without any changes the same exact application will use version 2 on environments running Hadoop 3.0 and later. This can have pretty bad consequences in certain scenarios, for example, two tasks can partially overwrite their output if speculation is enabled. Also, please refer to the following JIRA:
https://issues.apache.org/jira/browse/MAPREDUCE-7282

### Does this PR introduce _any_ user-facing change?

Yes. Configuration page content was modified where previously we explicitly highlighted that the default version for the FileOutputCommitter algorithm was v1, this now has changed to "Dependent on environment" with additional information in the description column to elaborate.

### How was this patch tested?

Checked changes locally in browser

Closes #29541 from waleedfateem/SPARK-32701.

Authored-by: waleedfateem <waleed.fateem@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 8749b2b6fae5ee0ce7b48aae6d859ed71e98491d)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 docs/configuration.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index 2701fdb030ce8..95ff282bfa60d 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1761,11 +1761,16 @@ Apart from these, the following properties are also available, and may be useful
 </tr>
 <tr>
   <td><code>spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version</code></td>
-  <td>1</td>
+  <td>Dependent on environment</td>
   <td>
     The file output committer algorithm version, valid algorithm version number: 1 or 2.
     Version 2 may have better performance, but version 1 may handle failures better in certain situations,
     as per <a href="https://issues.apache.org/jira/browse/MAPREDUCE-4815">MAPREDUCE-4815</a>.
+    The default value depends on the Hadoop version used in an environment:
+    1 for Hadoop versions lower than 3.0
+    2 for Hadoop versions 3.0 and higher
+    It's important to note that this can change back to 1 again in the future once <a href="https://issues.apache.org/jira/browse/MAPREDUCE-7282">MAPREDUCE-7282</a>
+    is fixed and merged.
   </td>
   <td>2.2.0</td>
 </tr>

From 2b147c4cd50da32fe2b4167f97c8142102a0510d Mon Sep 17 00:00:00 2001
From: zhengruifeng <ruifengz@apache.org>
Date: Fri, 28 Aug 2020 03:25:14 +0000
Subject: [PATCH 1010/1280] Preparing Spark release v3.0.1-rc3

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index a17dea9093771..86514f237d1e6 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.2
+Version: 3.0.1
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index e265bb948d9d4..c47425a14cf69 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index f433764648ccd..0948c6b1616e4 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index 4f31606fddf94..e7b16f1aa0b9f 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index a3ac6611a3127..c32a74fb30e97 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 5c5dfbabe862e..56ca7258804b2 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 8041d7a093da9..6b60cb36a6b30 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index e3a550288d71a..3eb1b77f7d756 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index 09965b573706c..bfdbc7e526e37 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 19b5b30051602..45970b7077bbb 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index abb3b49fff54d..65be4c4a9c482 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -17,8 +17,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.2-SNAPSHOT
-SPARK_VERSION_SHORT: 3.0.2
+SPARK_VERSION: 3.0.1
+SPARK_VERSION_SHORT: 3.0.1
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index 2082ece41569a..19c4d405faac5 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index 923b3bd6981dc..b8cd328b2cb83 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 4f16290891c3e..0f8d076318bbd 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 7852552ed5a54..750797fd7af3b 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 24f2d9acc1ee6..8d2e8804d8471 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index c5d41b05baf3b..b329026fd6e49 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index c53a5392ff917..6a0d72a903952 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 4f8cc83219dbc..3ef13b76954ba 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 2dd1e786d3634..3166ee63c9ed3 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index f1dd31eee35a2..02403f11ea82b 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index d760d5c32099c..3f1dd80931a8e 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 860f648ff9e57..cc436182c6908 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 48ee744806ed2..1e9102758d5f8 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index fd103d80268e3..cd1dbce4a5d3c 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 135c4ca54a491..c88ee850cecb7 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 32a13082bac91..7c60f8f99bd87 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.2-SNAPSHOT</version>
+  <version>3.0.1</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index e6410c11cf86d..b19dd9e6c7314 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.2.dev0"
+__version__ = "3.0.1"
diff --git a/repl/pom.xml b/repl/pom.xml
index 92911ee094b4d..fa48a4577e0b5 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index ef6c38a64ec8c..c0e469dd0cfc4 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index 693e720df1aa3..e859d964be7b1 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index ec93eefa1d8f7..a8b7ae4c30891 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index f0031cf2466b5..ed51acfe16b79 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index e4d5ee614cb04..d84e6e33d0934 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 1a1673fdfd212..5f540dcdd1007 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index f376afbd19599..2c84451230dbe 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 1b4c6dc110e97..cd186284dbc76 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 251570496d805..0a88f49782d6a 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index 41662b1957703..a38469847b6e5 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.2-SNAPSHOT</version>
+    <version>3.0.1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From ed7be8ca4835fc30b54b92c6862ccd549917dfce Mon Sep 17 00:00:00 2001
From: zhengruifeng <ruifengz@apache.org>
Date: Fri, 28 Aug 2020 03:25:21 +0000
Subject: [PATCH 1011/1280] Preparing development version 3.0.2-SNAPSHOT

---
 R/pkg/DESCRIPTION                                      | 2 +-
 assembly/pom.xml                                       | 2 +-
 common/kvstore/pom.xml                                 | 2 +-
 common/network-common/pom.xml                          | 2 +-
 common/network-shuffle/pom.xml                         | 2 +-
 common/network-yarn/pom.xml                            | 2 +-
 common/sketch/pom.xml                                  | 2 +-
 common/tags/pom.xml                                    | 2 +-
 common/unsafe/pom.xml                                  | 2 +-
 core/pom.xml                                           | 2 +-
 docs/_config.yml                                       | 4 ++--
 examples/pom.xml                                       | 2 +-
 external/avro/pom.xml                                  | 2 +-
 external/docker-integration-tests/pom.xml              | 2 +-
 external/kafka-0-10-assembly/pom.xml                   | 2 +-
 external/kafka-0-10-sql/pom.xml                        | 2 +-
 external/kafka-0-10-token-provider/pom.xml             | 2 +-
 external/kafka-0-10/pom.xml                            | 2 +-
 external/kinesis-asl-assembly/pom.xml                  | 2 +-
 external/kinesis-asl/pom.xml                           | 2 +-
 external/spark-ganglia-lgpl/pom.xml                    | 2 +-
 graphx/pom.xml                                         | 2 +-
 hadoop-cloud/pom.xml                                   | 2 +-
 launcher/pom.xml                                       | 2 +-
 mllib-local/pom.xml                                    | 2 +-
 mllib/pom.xml                                          | 2 +-
 pom.xml                                                | 2 +-
 python/pyspark/version.py                              | 2 +-
 repl/pom.xml                                           | 2 +-
 resource-managers/kubernetes/core/pom.xml              | 2 +-
 resource-managers/kubernetes/integration-tests/pom.xml | 2 +-
 resource-managers/mesos/pom.xml                        | 2 +-
 resource-managers/yarn/pom.xml                         | 2 +-
 sql/catalyst/pom.xml                                   | 2 +-
 sql/core/pom.xml                                       | 2 +-
 sql/hive-thriftserver/pom.xml                          | 2 +-
 sql/hive/pom.xml                                       | 2 +-
 streaming/pom.xml                                      | 2 +-
 tools/pom.xml                                          | 2 +-
 39 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 86514f237d1e6..a17dea9093771 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: SparkR
 Type: Package
-Version: 3.0.1
+Version: 3.0.2
 Title: R Front End for 'Apache Spark'
 Description: Provides an R Front end for 'Apache Spark' <https://spark.apache.org>.
 Authors@R: c(person("Shivaram", "Venkataraman", role = c("aut", "cre"),
diff --git a/assembly/pom.xml b/assembly/pom.xml
index c47425a14cf69..e265bb948d9d4 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml
index 0948c6b1616e4..f433764648ccd 100644
--- a/common/kvstore/pom.xml
+++ b/common/kvstore/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
index e7b16f1aa0b9f..4f31606fddf94 100644
--- a/common/network-common/pom.xml
+++ b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
index c32a74fb30e97..a3ac6611a3127 100644
--- a/common/network-shuffle/pom.xml
+++ b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
index 56ca7258804b2..5c5dfbabe862e 100644
--- a/common/network-yarn/pom.xml
+++ b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 6b60cb36a6b30..8041d7a093da9 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/tags/pom.xml b/common/tags/pom.xml
index 3eb1b77f7d756..e3a550288d71a 100644
--- a/common/tags/pom.xml
+++ b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
index bfdbc7e526e37..09965b573706c 100644
--- a/common/unsafe/pom.xml
+++ b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/core/pom.xml b/core/pom.xml
index 45970b7077bbb..19b5b30051602 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/docs/_config.yml b/docs/_config.yml
index 65be4c4a9c482..abb3b49fff54d 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -17,8 +17,8 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-SPARK_VERSION: 3.0.1
-SPARK_VERSION_SHORT: 3.0.1
+SPARK_VERSION: 3.0.2-SNAPSHOT
+SPARK_VERSION_SHORT: 3.0.2
 SCALA_BINARY_VERSION: "2.12"
 SCALA_VERSION: "2.12.10"
 MESOS_VERSION: 1.0.0
diff --git a/examples/pom.xml b/examples/pom.xml
index 19c4d405faac5..2082ece41569a 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/external/avro/pom.xml b/external/avro/pom.xml
index b8cd328b2cb83..923b3bd6981dc 100644
--- a/external/avro/pom.xml
+++ b/external/avro/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
index 0f8d076318bbd..4f16290891c3e 100644
--- a/external/docker-integration-tests/pom.xml
+++ b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
index 750797fd7af3b..7852552ed5a54 100644
--- a/external/kafka-0-10-assembly/pom.xml
+++ b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
index 8d2e8804d8471..24f2d9acc1ee6 100644
--- a/external/kafka-0-10-sql/pom.xml
+++ b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml
index b329026fd6e49..c5d41b05baf3b 100644
--- a/external/kafka-0-10-token-provider/pom.xml
+++ b/external/kafka-0-10-token-provider/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
index 6a0d72a903952..c53a5392ff917 100644
--- a/external/kafka-0-10/pom.xml
+++ b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
index 3ef13b76954ba..4f8cc83219dbc 100644
--- a/external/kinesis-asl-assembly/pom.xml
+++ b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
index 3166ee63c9ed3..2dd1e786d3634 100644
--- a/external/kinesis-asl/pom.xml
+++ b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
index 02403f11ea82b..f1dd31eee35a2 100644
--- a/external/spark-ganglia-lgpl/pom.xml
+++ b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 3f1dd80931a8e..d760d5c32099c 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index cc436182c6908..860f648ff9e57 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 1e9102758d5f8..48ee744806ed2 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
index cd1dbce4a5d3c..fd103d80268e3 100644
--- a/mllib-local/pom.xml
+++ b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/mllib/pom.xml b/mllib/pom.xml
index c88ee850cecb7..135c4ca54a491 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/pom.xml b/pom.xml
index 7c60f8f99bd87..32a13082bac91 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.12</artifactId>
-  <version>3.0.1</version>
+  <version>3.0.2-SNAPSHOT</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>
diff --git a/python/pyspark/version.py b/python/pyspark/version.py
index b19dd9e6c7314..e6410c11cf86d 100644
--- a/python/pyspark/version.py
+++ b/python/pyspark/version.py
@@ -16,4 +16,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.0.1"
+__version__ = "3.0.2.dev0"
diff --git a/repl/pom.xml b/repl/pom.xml
index fa48a4577e0b5..92911ee094b4d 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
index c0e469dd0cfc4..ef6c38a64ec8c 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml
index e859d964be7b1..693e720df1aa3 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
index a8b7ae4c30891..ec93eefa1d8f7 100644
--- a/resource-managers/mesos/pom.xml
+++ b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
index ed51acfe16b79..f0031cf2466b5 100644
--- a/resource-managers/yarn/pom.xml
+++ b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index d84e6e33d0934..e4d5ee614cb04 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 5f540dcdd1007..1a1673fdfd212 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 2c84451230dbe..f376afbd19599 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index cd186284dbc76..1b4c6dc110e97 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 0a88f49782d6a..251570496d805 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 
diff --git a/tools/pom.xml b/tools/pom.xml
index a38469847b6e5..41662b1957703 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.1</version>
+    <version>3.0.2-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

From 87327a28fbceade366faf0877fcc8e728d822742 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Fri, 28 Aug 2020 15:14:57 +0900
Subject: [PATCH 1012/1280] [SPARK-28612][SQL][FOLLOWUP] Correct method doc of
 DataFrameWriterV2.replace()

### What changes were proposed in this pull request?

This patch corrects the method doc of DataFrameWriterV2.replace() which explanation of exception is described oppositely.

### Why are the changes needed?

The method doc is incorrect.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Only doc change.

Closes #29568 from HeartSaVioR/SPARK-28612-FOLLOWUP-fix-doc-nit.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 73bfed3633be4322ae2a56df1f382d4749cc87f2)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
index 15ff7fa0530cc..87f35410172d6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriterV2.scala
@@ -324,7 +324,7 @@ trait CreateTableWriter[T] extends WriteConfigMethods[CreateTableWriter[T]] {
    * [[org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException]].
    *
    * @throws org.apache.spark.sql.catalyst.analysis.CannotReplaceMissingTableException
-   *         If the table already exists
+   *         If the table does not exist
    */
   @throws(classOf[CannotReplaceMissingTableException])
   def replace(): Unit

From 0dff99b172e0d85c1855fe7a027931bdaf9b3aa7 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Sun, 30 Aug 2020 06:50:44 +0900
Subject: [PATCH 1013/1280] [SPARK-32693][SQL][3.0] Compare two dataframes with
 same schema except nullable property

### What changes were proposed in this pull request?

This PR changes key data types check in `HashJoin` to use `sameType`. This backports #29555 to branch-3.0.

### Why are the changes needed?

Looks at the resolving condition of `SetOperation`, it requires only each left data types should be `sameType` as the right ones. Logically the `EqualTo` expression in equi-join, also requires only left data type `sameType` as right data type. Then `HashJoin` requires left keys data type exactly the same as right keys data type, looks not reasonable.

It makes inconsistent results when doing `except` between two dataframes.

If two dataframes don't have nested fields, even their field nullable property different, `HashJoin` passes the key type check because it checks field individually so field nullable property is ignored.

If two dataframes have nested fields like struct, `HashJoin` fails the key type check because now it compare two struct types and nullable property now affects.

### Does this PR introduce _any_ user-facing change?

Yes. Making consistent `except` operation between dataframes.

### How was this patch tested?

Unit test.

Closes #29575 from viirya/SPARK-32693-3.0.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/execution/joins/HashJoin.scala  |  7 +++-
 .../apache/spark/sql/DataFrameJoinSuite.scala | 39 +++++++++++++++++++
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 305741e2984a4..931a75a83d4ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -80,8 +80,11 @@ trait HashJoin {
   }
 
   protected lazy val (buildKeys, streamedKeys) = {
-    require(leftKeys.map(_.dataType) == rightKeys.map(_.dataType),
-      "Join keys from two sides should have same types")
+    require(leftKeys.length == rightKeys.length &&
+      leftKeys.map(_.dataType)
+        .zip(rightKeys.map(_.dataType))
+        .forall(types => types._1.sameType(types._2)),
+      "Join keys from two sides should have same length and types")
     val lkeys = bindReferences(HashJoin.rewriteKeyExpr(leftKeys), left.output)
     val rkeys = bindReferences(HashJoin.rewriteKeyExpr(rightKeys), right.output)
     buildSide match {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 6b772e53ac184..a49f95f1ed134 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.{Inner, InnerLike, LeftOuter, RightOuter}
 import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Filter, HintInfo, Join, JoinHint, LogicalPlan, Project}
@@ -29,6 +31,7 @@ import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
 
 class DataFrameJoinSuite extends QueryTest
   with SharedSparkSession
@@ -418,4 +421,40 @@ class DataFrameJoinSuite extends QueryTest
       }
     }
   }
+
+  test("SPARK-32693: Compare two dataframes with same schema except nullable property") {
+    val schema1 = StructType(
+      StructField("a", IntegerType, false) ::
+        StructField("b", IntegerType, false) ::
+        StructField("c", IntegerType, false) :: Nil)
+    val rowSeq1: List[Row] = List(Row(10, 1, 1), Row(10, 50, 2))
+    val df1 = spark.createDataFrame(rowSeq1.asJava, schema1)
+
+    val schema2 = StructType(
+      StructField("a", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("c", IntegerType) :: Nil)
+    val rowSeq2: List[Row] = List(Row(10, 1, 1))
+    val df2 = spark.createDataFrame(rowSeq2.asJava, schema2)
+
+    checkAnswer(df1.except(df2), Row(10, 50, 2))
+
+    val schema3 = StructType(
+      StructField("a", IntegerType, false) ::
+        StructField("b", IntegerType, false) ::
+        StructField("c", IntegerType, false) ::
+        StructField("d", schema1, false) :: Nil)
+    val rowSeq3: List[Row] = List(Row(10, 1, 1, Row(10, 1, 1)), Row(10, 50, 2, Row(10, 50, 2)))
+    val df3 = spark.createDataFrame(rowSeq3.asJava, schema3)
+
+    val schema4 = StructType(
+      StructField("a", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("b", IntegerType) ::
+        StructField("d", schema2) :: Nil)
+    val rowSeq4: List[Row] = List(Row(10, 1, 1, Row(10, 1, 1)))
+    val df4 = spark.createDataFrame(rowSeq4.asJava, schema4)
+
+    checkAnswer(df3.except(df4), Row(10, 50, 2, Row(10, 50, 2)))
+  }
 }

From 70c477e3fa6427c6ec9314b10ce4b0e7b9eed3ce Mon Sep 17 00:00:00 2001
From: herman <herman@databricks.com>
Date: Tue, 21 Apr 2020 18:17:19 -0700
Subject: [PATCH 1014/1280] [SPARK-31511][SQL] Make BytesToBytesMap iterators
 thread-safe

### What changes were proposed in this pull request?
This PR increases the thread safety of the `BytesToBytesMap`:
- It makes the `iterator()` and `destructiveIterator()` methods used their own `Location` object. This used to be shared, and this was causing issues when the map was being iterated over in two threads by two different iterators.
- Removes the `safeIterator()` function. This is not needed anymore.
- Improves the documentation of a couple of methods w.r.t. thread-safety.

### Why are the changes needed?
It is unexpected an iterator shares the object it is returning with all other iterators. This is a violation of the iterator contract, and it causes issues with iterators over a map that are consumed in different threads.

### Does this PR introduce any user-facing change?
No

### How was this patch tested?
Existing tests.

Closes #28286 from hvanhovell/SPARK-31511.

Authored-by: herman <herman@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit cf6038499d3d5686ab3377f550c2fe507d022fd3)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/unsafe/map/BytesToBytesMap.java     | 23 ++++++++-----------
 .../sql/execution/joins/HashedRelation.scala  |  2 +-
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 64c240cea8d7a..6e028886f2318 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -406,17 +406,10 @@ private void handleFailedDelete() {
    *
    * For efficiency, all calls to `next()` will return the same {@link Location} object.
    *
-   * If any other lookups or operations are performed on this map while iterating over it, including
-   * `lookup()`, the behavior of the returned iterator is undefined.
+   * The returned iterator is thread-safe. However if the map is modified while iterating over it,
+   * the behavior of the returned iterator is undefined.
    */
   public MapIterator iterator() {
-    return new MapIterator(numValues, loc, false);
-  }
-
-  /**
-   * Returns a thread safe iterator that iterates of the entries of this map.
-   */
-  public MapIterator safeIterator() {
     return new MapIterator(numValues, new Location(), false);
   }
 
@@ -427,19 +420,20 @@ public MapIterator safeIterator() {
    *
    * For efficiency, all calls to `next()` will return the same {@link Location} object.
    *
-   * If any other lookups or operations are performed on this map while iterating over it, including
-   * `lookup()`, the behavior of the returned iterator is undefined.
+   * The returned iterator is thread-safe. However if the map is modified while iterating over it,
+   * the behavior of the returned iterator is undefined.
    */
   public MapIterator destructiveIterator() {
     updatePeakMemoryUsed();
-    return new MapIterator(numValues, loc, true);
+    return new MapIterator(numValues, new Location(), true);
   }
 
   /**
    * Looks up a key, and return a {@link Location} handle that can be used to test existence
    * and read/write values.
    *
-   * This function always return the same {@link Location} instance to avoid object allocation.
+   * This function always returns the same {@link Location} instance to avoid object allocation.
+   * This function is not thread-safe.
    */
   public Location lookup(Object keyBase, long keyOffset, int keyLength) {
     safeLookup(keyBase, keyOffset, keyLength, loc,
@@ -451,7 +445,8 @@ public Location lookup(Object keyBase, long keyOffset, int keyLength) {
    * Looks up a key, and return a {@link Location} handle that can be used to test existence
    * and read/write values.
    *
-   * This function always return the same {@link Location} instance to avoid object allocation.
+   * This function always returns the same {@link Location} instance to avoid object allocation.
+   * This function is not thread-safe.
    */
   public Location lookup(Object keyBase, long keyOffset, int keyLength, int hash) {
     safeLookup(keyBase, keyOffset, keyLength, loc, hash);
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
index 4001338662d53..13180d6b20902 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashedRelation.scala
@@ -177,7 +177,7 @@ private[joins] class UnsafeHashedRelation(
   }
 
   override def keys(): Iterator[InternalRow] = {
-    val iter = binaryMap.safeIterator()
+    val iter = binaryMap.iterator()
 
     new Iterator[InternalRow] {
       val unsafeRow = new UnsafeRow(numKeys)

From 985c593944fcf625bb020fe94189e27aeaab99bd Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 31 Aug 2020 17:39:12 +0900
Subject: [PATCH 1015/1280] [SPARK-32747][R][TESTS] Deduplicate configuration
 set/unset in test_sparkSQL_arrow.R

### What changes were proposed in this pull request?

This PR proposes to deduplicate configuration set/unset in `test_sparkSQL_arrow.R`.
Setting `spark.sql.execution.arrow.sparkr.enabled` can be globally done instead of doing it in each test case.

### Why are the changes needed?

To duduplicate the codes.

### Does this PR introduce _any_ user-facing change?

No, dev-only

### How was this patch tested?

Manually ran the tests.

Closes #29592 from HyukjinKwon/SPARK-32747.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 2491cf1ae1085832f21545ae563a789c34bdd098)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/tests/fulltests/test_sparkSQL_arrow.R | 201 ++++++--------------
 1 file changed, 60 insertions(+), 141 deletions(-)

diff --git a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
index 16d93763ff038..06743488fdf11 100644
--- a/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
+++ b/R/pkg/tests/fulltests/test_sparkSQL_arrow.R
@@ -19,7 +19,10 @@ library(testthat)
 
 context("SparkSQL Arrow optimization")
 
-sparkSession <- sparkR.session(master = sparkRTestMaster, enableHiveSupport = FALSE)
+sparkSession <- sparkR.session(
+  master = sparkRTestMaster,
+  enableHiveSupport = FALSE,
+  sparkConfig = list(spark.sql.execution.arrow.sparkr.enabled = "true"))
 
 test_that("createDataFrame/collect Arrow optimization", {
   skip_if_not_installed("arrow")
@@ -35,29 +38,13 @@ test_that("createDataFrame/collect Arrow optimization", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(mtcars)), expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(mtcars)), expected)
 })
 
 test_that("createDataFrame/collect Arrow optimization - many partitions (partition order test)", {
   skip_if_not_installed("arrow")
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)),
-                 collect(createDataFrame(mtcars, numPartitions = 1)))
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(mtcars, numPartitions = 32)),
+               collect(createDataFrame(mtcars, numPartitions = 1)))
 })
 
 test_that("createDataFrame/collect Arrow optimization - type specification", {
@@ -81,13 +68,7 @@ test_that("createDataFrame/collect Arrow optimization - type specification", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_equal(collect(createDataFrame(rdf)), expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_equal(collect(createDataFrame(rdf)), expected)
 })
 
 test_that("dapply() Arrow optimization", {
@@ -98,36 +79,30 @@ test_that("dapply() Arrow optimization", {
   arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
 
   callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false")
-  tryCatch({
-    ret <- dapply(df,
-    function(rdf) {
-      stopifnot(is.data.frame(rdf))
-      rdf
-    },
-    schema(df))
-    expected <- collect(ret)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
   tryCatch({
     ret <- dapply(df,
                   function(rdf) {
                     stopifnot(is.data.frame(rdf))
-                    # mtcars' hp is more then 50.
-                    stopifnot(all(rdf$hp > 50))
                     rdf
                   },
                   schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-    expect_equal(count(ret), nrow(mtcars))
+    expected <- collect(ret)
   },
   finally = {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
+
+  ret <- dapply(df,
+                function(rdf) {
+                  stopifnot(is.data.frame(rdf))
+                  # mtcars' hp is more then 50.
+                  stopifnot(all(rdf$hp > 50))
+                  rdf
+                },
+                schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
+  expect_equal(count(ret), nrow(mtcars))
 })
 
 test_that("dapply() Arrow optimization - type specification", {
@@ -154,15 +129,9 @@ test_that("dapply() Arrow optimization - type specification", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- dapply(df, function(rdf) { rdf }, schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- dapply(df, function(rdf) { rdf }, schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
 })
 
 test_that("dapply() Arrow optimization - type specification (date and timestamp)", {
@@ -170,18 +139,8 @@ test_that("dapply() Arrow optimization - type specification (date and timestamp)
   rdf <- data.frame(list(list(a = as.Date("1990-02-24"),
                               b = as.POSIXct("1990-02-24 12:34:56"))))
   df <- createDataFrame(rdf)
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- dapply(df, function(rdf) { rdf }, schema(df))
-    expect_equal(collect(ret), rdf)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- dapply(df, function(rdf) { rdf }, schema(df))
+  expect_equal(collect(ret), rdf)
 })
 
 test_that("gapply() Arrow optimization", {
@@ -209,28 +168,22 @@ test_that("gapply() Arrow optimization", {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                 "gear",
-                 function(key, grouped) {
-                   if (length(key) > 0) {
-                     stopifnot(is.numeric(key[[1]]))
-                   }
-                   stopifnot(is.data.frame(grouped))
-                   stopifnot(length(colnames(grouped)) == 11)
-                   # mtcars' hp is more then 50.
-                   stopifnot(all(grouped$hp > 50))
-                   grouped
-                 },
-                 schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-    expect_equal(count(ret), nrow(mtcars))
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+               "gear",
+               function(key, grouped) {
+                 if (length(key) > 0) {
+                   stopifnot(is.numeric(key[[1]]))
+                 }
+                 stopifnot(is.data.frame(grouped))
+                 stopifnot(length(colnames(grouped)) == 11)
+                 # mtcars' hp is more then 50.
+                 stopifnot(all(grouped$hp > 50))
+                 grouped
+               },
+               schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
+  expect_equal(count(ret), nrow(mtcars))
 })
 
 test_that("gapply() Arrow optimization - type specification", {
@@ -250,26 +203,19 @@ test_that("gapply() Arrow optimization - type specification", {
   callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "false")
   tryCatch({
     ret <- gapply(df,
-                 "a",
-                 function(key, grouped) { grouped }, schema(df))
+                  "a",
+                  function(key, grouped) { grouped }, schema(df))
     expected <- collect(ret)
   },
   finally = {
     callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
   })
 
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                 "a",
-                 function(key, grouped) { grouped }, schema(df))
-    actual <- collect(ret)
-    expect_equal(actual, expected)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+                "a",
+                function(key, grouped) { grouped }, schema(df))
+  actual <- collect(ret)
+  expect_equal(actual, expected)
 })
 
 test_that("gapply() Arrow optimization - type specification (date and timestamp)", {
@@ -277,57 +223,30 @@ test_that("gapply() Arrow optimization - type specification (date and timestamp)
   rdf <- data.frame(list(list(a = as.Date("1990-02-24"),
                               b = as.POSIXct("1990-02-24 12:34:56"))))
   df <- createDataFrame(rdf)
-
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    ret <- gapply(df,
-                  "a",
-                  function(key, grouped) { grouped }, schema(df))
-    expect_equal(collect(ret), rdf)
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  ret <- gapply(df,
+                "a",
+                function(key, grouped) { grouped }, schema(df))
+  expect_equal(collect(ret), rdf)
 })
 
 test_that("Arrow optimization - unsupported types", {
   skip_if_not_installed("arrow")
 
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type")
-    expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type")
-    expect_error(checkSchemaInArrow(structType("a ARRAY<INT>")), "not support array type")
-    expect_error(checkSchemaInArrow(structType("a MAP<INT, INT>")), "not support map type")
-    expect_error(checkSchemaInArrow(structType("a STRUCT<a: INT>")),
-                 "not support nested struct type")
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
+  expect_error(checkSchemaInArrow(structType("a FLOAT")), "not support float type")
+  expect_error(checkSchemaInArrow(structType("a BINARY")), "not support binary type")
+  expect_error(checkSchemaInArrow(structType("a ARRAY<INT>")), "not support array type")
+  expect_error(checkSchemaInArrow(structType("a MAP<INT, INT>")), "not support map type")
+  expect_error(checkSchemaInArrow(structType("a STRUCT<a: INT>")),
+               "not support nested struct type")
 })
 
 test_that("SPARK-32478: gapply() Arrow optimization - error message for schema mismatch", {
   skip_if_not_installed("arrow")
   df <- createDataFrame(list(list(a = 1L, b = "a")))
 
-  conf <- callJMethod(sparkSession, "conf")
-  arrowEnabled <- sparkR.conf("spark.sql.execution.arrow.sparkr.enabled")[[1]]
-
-  callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", "true")
-  tryCatch({
-    expect_error(
+  expect_error(
     count(gapply(df, "a", function(key, group) { group }, structType("a int, b int"))),
     "expected IntegerType, IntegerType, got IntegerType, StringType")
-  },
-  finally = {
-    callJMethod(conf, "set", "spark.sql.execution.arrow.sparkr.enabled", arrowEnabled)
-  })
 })
 
 sparkR.session.stop()

From 3a516ae68c773668edf966474e61a208a51a2f96 Mon Sep 17 00:00:00 2001
From: Lu WANG <lu.wang@databricks.com>
Date: Tue, 1 Sep 2020 10:07:34 +0900
Subject: [PATCH 1016/1280] [MINOR][R] Fix a R style in try and finally at
 DataFrame.R

Fix the R style issue which is not catched by the R style checker. Got error:
```
R/DataFrame.R:1244:17: style: Closing curly-braces should always be on their own line, unless it's followed by an else.
}, finally = {
 ^
lintr checks failed.
```

Closes #29574 from lu-wang-dl/fix-r-style.

Lead-authored-by: Lu WANG <lu.wang@databricks.com>
Co-authored-by: Lu Wang <38018689+lu-wang-dl@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 701e5934140e477426cf2a68f2c5746be67418d6)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/R/DataFrame.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index 3a60d1c970440..4195ec8f07b8b 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -1240,7 +1240,8 @@ setMethod("collect",
                     arrowTable <- arrow::read_arrow(readRaw(conn))
                   }
                   as.data.frame(arrowTable, stringsAsFactors = stringsAsFactors)
-                }, finally = {
+                },
+                finally = {
                   close(conn)
                 })
                 return(output)

From 98a1247f379170afbd9a64a3a51d6235ff3df02e Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Tue, 1 Sep 2020 05:51:04 +0000
Subject: [PATCH 1017/1280] [SPARK-32659][SQL][FOLLOWUP] Improve test for
 pruning DPP on non-atomic type

### What changes were proposed in this pull request?

Improve test for pruning DPP on non-atomic type:
- Avoid creating new partition tables. This may take 30 seconds..
- Add test `array` type.

### Why are the changes needed?

Improve test.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

N/A

Closes #29595 from wangyum/SPARK-32659-test.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit a701bc79e3bb936e8fb4a4fbe11e9e0bd0ccd8ac)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/DynamicPartitionPruningSuite.scala    | 68 +++++++------------
 1 file changed, 26 insertions(+), 42 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index bf91bdb79cf05..47c5c9eacdfd2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -19,7 +19,8 @@ package org.apache.spark.sql
 
 import org.scalatest.GivenWhenThen
 
-import org.apache.spark.sql.catalyst.expressions.{CodegenObjectFactoryMode, DynamicPruningExpression, Expression}
+import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode._
 import org.apache.spark.sql.catalyst.plans.ExistenceJoin
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper}
@@ -1311,48 +1312,31 @@ abstract class DynamicPartitionPruningSuiteBase
   }
 
   test("SPARK-32659: Fix the data issue when pruning DPP on non-atomic type") {
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_FALLBACK_FILTER_RATIO.key -> "2", // Make sure insert DPP
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false") {
-      withTable("df1", "df2") {
-        spark.range(1000)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df1")
-
-        spark.range(100)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df2")
-
-        Seq(CodegenObjectFactoryMode.NO_CODEGEN,
-          CodegenObjectFactoryMode.CODEGEN_ONLY).foreach { mode =>
-          Seq(true, false).foreach { pruning =>
-            withSQLConf(
-              SQLConf.CODEGEN_FACTORY_MODE.key -> mode.toString,
-              SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> s"$pruning") {
-              val df = sql(
-                """
-                  |SELECT df1.id, df2.k
-                  |FROM df1
-                  |  JOIN df2
-                  |  ON struct(df1.k) = struct(df2.k)
-                  |    AND df2.id < 2
-                  |""".stripMargin)
-              if (pruning) {
-                checkPartitionPruningPredicate(df, true, false)
-              } else {
-                checkPartitionPruningPredicate(df, false, false)
-              }
-
-              checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
+    Seq(NO_CODEGEN, CODEGEN_ONLY).foreach { mode =>
+      Seq(true, false).foreach { pruning =>
+        withSQLConf(
+          SQLConf.CODEGEN_FACTORY_MODE.key -> mode.toString,
+          SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> s"$pruning") {
+          Seq("struct", "array").foreach { dataType =>
+            val df = sql(
+              s"""
+                 |SELECT f.date_id, f.product_id, f.units_sold, f.store_id FROM fact_stats f
+                 |JOIN dim_stats s
+                 |ON $dataType(f.store_id) = $dataType(s.store_id) WHERE s.country = 'DE'
+              """.stripMargin)
+
+            if (pruning) {
+              checkPartitionPruningPredicate(df, false, true)
+            } else {
+              checkPartitionPruningPredicate(df, false, false)
             }
+
+            checkAnswer(df,
+              Row(1030, 2, 10, 3) ::
+              Row(1040, 2, 50, 3) ::
+              Row(1050, 2, 50, 3) ::
+              Row(1060, 2, 50, 3) :: Nil
+            )
           }
         }
       }

From d1c6a0021b9fe4c29a743e3504bcb0e59dfbbe45 Mon Sep 17 00:00:00 2001
From: Kris Mok <kris.mok@databricks.com>
Date: Tue, 1 Sep 2020 15:15:11 +0900
Subject: [PATCH 1018/1280] [SPARK-32624][SQL][FOLLOWUP] Fix regression in
 CodegenContext.addReferenceObj on nested Scala types

### What changes were proposed in this pull request?

Use `CodeGenerator.typeName()` instead of `Class.getCanonicalName()` in `CodegenContext.addReferenceObj()` for getting the runtime class name for an object.

### Why are the changes needed?

https://github.com/apache/spark/pull/29439 fixed a bug in `CodegenContext.addReferenceObj()` for `Array[Byte]` (i.e. Spark SQL's `BinaryType`) objects, but unfortunately it introduced a regression for some nested Scala types.

For example, for `implicitly[Ordering[UTF8String]]`, after that PR `CodegenContext.addReferenceObj()` would return `((null) references[0] /* ... */)`. The actual type for `implicitly[Ordering[UTF8String]]` is `scala.math.LowPriorityOrderingImplicits$$anon$3` in Scala 2.12.10, and `Class.getCanonicalName()` returns `null` for that class.

On the other hand, `Class.getName()` is safe to use for all non-array types, and Janino will happily accept the type name returned from `Class.getName()` for nested types. `CodeGenerator.typeName()` happens to do the right thing by correctly handling arrays and otherwise use `Class.getName()`. So it's a better alternative than `Class.getCanonicalName()`.

Side note: rule of thumb for using Java reflection in Spark: it may be tempting to use `Class.getCanonicalName()`, but for functions that may need to handle Scala types, please avoid it due to potential issues with nested Scala types.
Instead, use `Class.getName()` or utility functions in `org.apache.spark.util.Utils` (e.g. `Utils.getSimpleName()` or `Utils.getFormattedClassName()` etc).

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added new unit test case for the regression case in `CodeGenerationSuite`.

Closes #29602 from rednaxelafx/spark-32624-followup.

Authored-by: Kris Mok <kris.mok@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 6e5bc39e17d4cf02806761170de6ddeb634aa343)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../expressions/codegen/CodeGenerator.scala   |  2 +-
 .../expressions/CodeGenerationSuite.scala     | 21 ++++++++++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index c8aa83eb18068..aa6d6e67ece04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -133,7 +133,7 @@ class CodegenContext extends Logging {
   def addReferenceObj(objName: String, obj: Any, className: String = null): String = {
     val idx = references.length
     references += obj
-    val clsName = Option(className).getOrElse(obj.getClass.getCanonicalName)
+    val clsName = Option(className).getOrElse(CodeGenerator.typeName(obj.getClass))
     s"(($clsName) references[$idx] /* $objName */)"
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
index 09cc01d804d37..f1de63adc3d9a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.expressions
 
 import java.sql.Timestamp
 
+import scala.math.Ordering
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.metrics.source.CodegenMetrics
 import org.apache.spark.sql.Row
@@ -537,11 +539,24 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
     GenerateMutableProjection.generate(exprs, true)
   }
 
-  test("SPARK-32624: Use getCanonicalName to fix byte[] compile issue") {
+  test("SPARK-32624: Use CodeGenerator.typeName() to fix byte[] compile issue") {
     val ctx = new CodegenContext
     val bytes = new Array[Byte](3)
-    val byteObj = ctx.addReferenceObj("bytes", bytes)
-    assert(byteObj == "((byte[]) references[0] /* bytes */)")
+    val refTerm = ctx.addReferenceObj("bytes", bytes)
+    assert(refTerm == "((byte[]) references[0] /* bytes */)")
+  }
+
+  test("SPARK-32624: CodegenContext.addReferenceObj should work for nested Scala class") {
+    // emulate TypeUtils.getInterpretedOrdering(StringType)
+    val ctx = new CodegenContext
+    val comparator = implicitly[Ordering[UTF8String]]
+    val refTerm = ctx.addReferenceObj("comparator", comparator)
+
+    // Expecting result:
+    //   "((scala.math.LowPriorityOrderingImplicits$$anon$3) references[0] /* comparator */)"
+    // Using lenient assertions to be resilient to annonymous class numbering changes
+    assert(!refTerm.contains("null"))
+    assert(refTerm.contains("scala.math.LowPriorityOrderingImplicits$$anon$"))
   }
 }
 

From 88bc08e262261852bb7adc709818513edd5850ba Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 2 Sep 2020 09:43:32 +0900
Subject: [PATCH 1019/1280] [SPARK-32774][BUILD] Don't track docs/.jekyll-cache

### What changes were proposed in this pull request?

This PR changes .gitignore not to track docs/.jekyll-cache.

### Why are the changes needed?

When I build docs, docs/.jekyll-cache can be created and it should not be tracked.
```
$ git status
On branch master
Your branch is up to date with 'origin/master'.

Untracked files:
  (use "git add <file>..." to include in what will be committed)
	docs/.jekyll-cache/
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Applied the change and confirmed the result of `git status`
```
$ git status
On branch untrack-jekyll-cache
nothing to commit, working tree clean
```

Closes #29622 from sarutak/untrack-jekyll-cache.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 198fdee39be95..1f4b323c926ac 100644
--- a/.gitignore
+++ b/.gitignore
@@ -79,6 +79,7 @@ target/
 unit-tests.log
 work/
 docs/.jekyll-metadata
+docs/.jekyll-cache
 
 # For Hive
 TempStatsStore/

From 83ade58869c0f7755129333d960303a137ee9d22 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 2 Sep 2020 10:03:07 +0900
Subject: [PATCH 1020/1280] [SPARK-32771][DOCS] The example of
 expressions.Aggregator in Javadoc / Scaladoc is wrong

### What changes were proposed in this pull request?

This PR modifies an example for `expressions.Aggregator` in Javadoc and Scaladoc.
The definition of `bufferEncoder` and `outputEncoder` are added.

### Why are the changes needed?

To correct the example.
The current example is wrong and doesn't work because `bufferEncoder` and `outputEncoder` are not defined.

### Does this PR introduce _any_ user-facing change?

Yes.
Before this change, the scaladoc and javadoc are like as follows.
![wrong-example-java](https://user-images.githubusercontent.com/4736016/91897528-5ebf3580-ecd5-11ea-8d7b-e846b776ebbb.png)
![wrong-example](https://user-images.githubusercontent.com/4736016/91897509-58c95480-ecd5-11ea-81a3-98774083b689.png)

After this change, the docs are like as follows.
![fixed-example-java](https://user-images.githubusercontent.com/4736016/91897592-78607d00-ecd5-11ea-9e55-03fd9c9c6b54.png)
![fixed-example](https://user-images.githubusercontent.com/4736016/91897609-7c8c9a80-ecd5-11ea-837e-9dbcada6cd53.png)

### How was this patch tested?

Build with `build/sbt unidoc` and confirmed the generated javadoc/scaladoc and got the screenshots above.

Closes #29617 from sarutak/fix-aggregator-doc.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 812d0918a88ca5677c0bdbd8cffd765d53de50ca)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/sql/expressions/Aggregator.scala     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
index 878dc0d83f45a..62d04cf7f7cac 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala
@@ -35,6 +35,8 @@ import org.apache.spark.sql.execution.aggregate.TypedAggregateExpression
  *     def reduce(b: Int, a: Data): Int = b + a.i
  *     def merge(b1: Int, b2: Int): Int = b1 + b2
  *     def finish(r: Int): Int = r
+ *     def bufferEncoder: Encoder[Int] = Encoders.scalaInt
+ *     def outputEncoder: Encoder[Int] = Encoders.scalaInt
  *   }.toColumn()
  *
  *   val ds: Dataset[Data] = ...

From 850672870faed6677d166c0029335600a15f4153 Mon Sep 17 00:00:00 2001
From: liwensun <liwen.sun@databricks.com>
Date: Wed, 2 Sep 2020 18:05:06 +0900
Subject: [PATCH 1021/1280] [SPARK-32776][SS] Limit in streaming should not be
 optimized away by PropagateEmptyRelation

PropagateEmptyRelation will not be applied to LIMIT operators in streaming queries.

Right now, the limit operator in a streaming query may get optimized away when the relation is empty. This can be problematic for stateful streaming, as this empty batch will not write any state store files, and the next batch will fail when trying to read these state store files and throw a file not found error.

We should not let PropagateEmptyRelation optimize away the Limit operator for streaming queries.

This PR is intended as a small and safe fix for PropagateEmptyRelation. A fundamental fix that can prevent this from happening again in the future and in other optimizer rules is more desirable, but that's a much larger task.

No

unit tests.

Closes #29623 from liwensun/spark-32776.

Authored-by: liwensun <liwen.sun@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f0851e95c6a2c05716a32c961bf2842e18fe1bc7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../optimizer/PropagateEmptyRelation.scala    |  4 +-
 .../PropagateEmptyRelationSuite.scala         |  6 +++
 .../sql/streaming/StreamingQuerySuite.scala   | 38 +++++++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
index b19e13870aa65..7535f9c9c32d7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelation.scala
@@ -85,8 +85,8 @@ object PropagateEmptyRelation extends Rule[LogicalPlan] with PredicateHelper wit
       case _: Filter => empty(p)
       case _: Sample => empty(p)
       case _: Sort => empty(p)
-      case _: GlobalLimit => empty(p)
-      case _: LocalLimit => empty(p)
+      case _: GlobalLimit if !p.isStreaming => empty(p)
+      case _: LocalLimit if !p.isStreaming => empty(p)
       case _: Repartition => empty(p)
       case _: RepartitionByExpression => empty(p)
       // An aggregate with non-empty group expression will return one output row per group when the
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
index 9c7d4c7d8d233..535be12af5161 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/PropagateEmptyRelationSuite.scala
@@ -221,4 +221,10 @@ class PropagateEmptyRelationSuite extends PlanTest {
     val optimized = Optimize.execute(query.analyze)
     assert(optimized.resolved)
   }
+
+  test("should not optimize away limit if streaming") {
+    val query = LocalRelation(Nil, Nil, isStreaming = true).limit(1).analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
index 434b6a20ace63..1f408d55fd811 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQuerySuite.scala
@@ -34,7 +34,9 @@ import org.scalatestplus.mockito.MockitoSugar
 import org.apache.spark.{SparkException, TestUtils}
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Dataset, Row}
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Literal, Rand, Randn, Shuffle, Uuid}
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes.Complete
 import org.apache.spark.sql.connector.read.InputPartition
 import org.apache.spark.sql.connector.read.streaming.{Offset => OffsetV2}
@@ -1141,6 +1143,42 @@ class StreamingQuerySuite extends StreamTest with BeforeAndAfter with Logging wi
     }
   }
 
+  testQuietly("limit on empty batch should not cause state store error") {
+    // The source only produces two batches, the first batch is empty and the second batch has data.
+    val source = new Source {
+      var batchId = 0
+      override def stop(): Unit = {}
+      override def getOffset: Option[Offset] = {
+        Some(LongOffset(batchId + 1))
+      }
+      override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
+        if (batchId == 0) {
+          batchId += 1
+          Dataset.ofRows(spark, LocalRelation(schema.toAttributes, Nil, isStreaming = true))
+        } else {
+          Dataset.ofRows(spark,
+            LocalRelation(schema.toAttributes, InternalRow(10) :: Nil, isStreaming = true))
+        }
+      }
+      override def schema: StructType = MockSourceProvider.fakeSchema
+    }
+
+    MockSourceProvider.withMockSources(source) {
+      val df = spark.readStream
+        .format("org.apache.spark.sql.streaming.util.MockSourceProvider")
+        .load()
+        .limit(1)
+
+      testStream(df)(
+        StartStream(),
+        AssertOnQuery { q =>
+          q.processAllAvailable()
+          true
+        },
+        CheckAnswer(10))
+    }
+  }
+
   private def checkExceptionMessage(df: DataFrame): Unit = {
     withTempDir { outputDir =>
       withTempDir { checkpointDir =>

From 5288fb41b71c6b7d19c115457e22283df100273d Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 3 Sep 2020 23:49:17 +0800
Subject: [PATCH 1022/1280] [SPARK-32788][SQL] non-partitioned table scan
 should not have partition filter

### What changes were proposed in this pull request?

This PR fixes a bug `FileSourceStrategy`, which generates partition filters even if the table is not partitioned. This can confuse `FileSourceScanExec`, which mistakenly think the table is partitioned and tries to update the `numPartitions` metrics, and cause a failure. We should not generate partition filters for non-partitioned table.

### Why are the changes needed?

The bug was exposed by https://github.com/apache/spark/pull/29436.

### Does this PR introduce _any_ user-facing change?

Yes, fix a bug.

### How was this patch tested?

new test

Closes #29637 from cloud-fan/refactor.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Yuming Wang <yumwang@ebay.com>
(cherry picked from commit 76330e029540fa7ff4311c682135efd8bf0804e5)
Signed-off-by: Yuming Wang <yumwang@ebay.com>
---
 .../spark/sql/execution/DataSourceScanExec.scala       |  7 +++----
 .../sql/execution/datasources/FileSourceStrategy.scala | 10 +++++++---
 .../scala/org/apache/spark/sql/SQLQuerySuite.scala     |  7 +++++++
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 03b214cfd339f..2147abd151fa2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -209,9 +209,6 @@ case class FileSourceScanExec(
     val ret =
       relation.location.listFiles(
         partitionFilters.filterNot(isDynamicPruningFilter), dataFilters)
-    if (relation.partitionSchemaOption.isDefined) {
-      driverMetrics("numPartitions") = ret.length
-    }
     setFilesNumAndSizeMetric(ret, true)
     val timeTakenMs = NANOSECONDS.toMillis(
       (System.nanoTime() - startTime) + optimizerMetadataTimeNs)
@@ -239,7 +236,6 @@ case class FileSourceScanExec(
       setFilesNumAndSizeMetric(ret, false)
       val timeTakenMs = (System.nanoTime() - startTime) / 1000 / 1000
       driverMetrics("pruningTime") = timeTakenMs
-      driverMetrics("numPartitions") = ret.length
       ret
     } else {
       selectedPartitions
@@ -433,6 +429,9 @@ case class FileSourceScanExec(
       driverMetrics("staticFilesNum") = filesNum
       driverMetrics("staticFilesSize") = filesSize
     }
+    if (relation.partitionSchemaOption.isDefined) {
+      driverMetrics("numPartitions") = partitions.length
+    }
   }
 
   override lazy val metrics = Map(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index 477937d66ad9b..b1e03fedbc1e2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -154,11 +154,15 @@ object FileSourceStrategy extends Strategy with Logging {
         l.resolve(
           fsRelation.partitionSchema, fsRelation.sparkSession.sessionState.analyzer.resolver)
       val partitionSet = AttributeSet(partitionColumns)
-      val partitionKeyFilters =
-        ExpressionSet(normalizedFilters
+      val partitionKeyFilters = if (partitionColumns.isEmpty) {
+        ExpressionSet(Nil)
+      } else {
+        val predicates = ExpressionSet(normalizedFilters
           .filter(_.references.subsetOf(partitionSet)))
+        logInfo(s"Pruning directories with: ${predicates.mkString(",")}")
+        predicates
+      }
 
-      logInfo(s"Pruning directories with: ${partitionKeyFilters.mkString(",")}")
 
       // subquery expressions are filtered out because they can't be used to prune buckets or pushed
       // down as data filters, yet they would be executed
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index f5dba4c1af265..979ae882726d9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3501,6 +3501,13 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
     checkAnswer(sql("SELECT 0 FROM ( SELECT * FROM B JOIN C USING (id)) " +
       "JOIN ( SELECT * FROM B JOIN C USING (id)) USING (id)"), Row(0))
   }
+
+  test("SPARK-32788: non-partitioned table scan should not have partition filter") {
+    withTable("t") {
+      spark.range(1).write.saveAsTable("t")
+      checkAnswer(sql("SELECT id FROM t WHERE (SELECT true)"), Row(0L))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From 0b5266b425c8a2b6539b845df878fd992a17a8c2 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Fri, 4 Sep 2020 08:01:04 +0800
Subject: [PATCH 1023/1280] [SPARK-32767][SQL][3.0] Bucket join should work if
 spark.sql.shuffle.partitions larger than bucket number

This backports #29612 to branch-3.0. Original PR description:

### What changes were proposed in this pull request?

Bucket join should work if `spark.sql.shuffle.partitions` larger than bucket number, such as:
```scala
spark.range(1000).write.bucketBy(432, "id").saveAsTable("t1")
spark.range(1000).write.bucketBy(34, "id").saveAsTable("t2")
sql("set spark.sql.shuffle.partitions=600")
sql("set spark.sql.autoBroadcastJoinThreshold=-1")
sql("select * from t1 join t2 on t1.id = t2.id").explain()
```

Before this pr:
```
== Physical Plan ==
*(5) SortMergeJoin [id#26L], [id#27L], Inner
:- *(2) Sort [id#26L ASC NULLS FIRST], false, 0
:  +- Exchange hashpartitioning(id#26L, 600), true
:     +- *(1) Filter isnotnull(id#26L)
:        +- *(1) ColumnarToRow
:           +- FileScan parquet default.t1[id#26L] Batched: true, DataFilters: [isnotnull(id#26L)], Format: Parquet, PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint>, SelectedBucketsCount: 432 out of 432
+- *(4) Sort [id#27L ASC NULLS FIRST], false, 0
   +- Exchange hashpartitioning(id#27L, 600), true
      +- *(3) Filter isnotnull(id#27L)
         +- *(3) ColumnarToRow
            +- FileScan parquet default.t2[id#27L] Batched: true, DataFilters: [isnotnull(id#27L)], Format: Parquet, PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint>, SelectedBucketsCount: 34 out of 34
```

After this pr:
```
== Physical Plan ==
*(4) SortMergeJoin [id#26L], [id#27L], Inner
:- *(1) Sort [id#26L ASC NULLS FIRST], false, 0
:  +- *(1) Filter isnotnull(id#26L)
:     +- *(1) ColumnarToRow
:        +- FileScan parquet default.t1[id#26L] Batched: true, DataFilters: [isnotnull(id#26L)], Format: Parquet, PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint>, SelectedBucketsCount: 432 out of 432
+- *(3) Sort [id#27L ASC NULLS FIRST], false, 0
   +- Exchange hashpartitioning(id#27L, 432), true
      +- *(2) Filter isnotnull(id#27L)
         +- *(2) ColumnarToRow
            +- FileScan parquet default.t2[id#27L] Batched: true, DataFilters: [isnotnull(id#27L)], Format: Parquet, PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint>, SelectedBucketsCount: 34 out of 34
```

### Why are the changes needed?

Spark 2.4 support this.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test.

Closes #29624 from wangyum/SPARK-32767-3.0.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Yuming Wang <yumwang@ebay.com>
---
 .../exchange/EnsureRequirements.scala         | 15 ++++++----
 .../spark/sql/sources/BucketedReadSuite.scala | 28 +++++++++++++++++++
 2 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
index 3242ac21ab324..c242320635a59 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala
@@ -85,11 +85,16 @@ case class EnsureRequirements(conf: SQLConf) extends Rule[SparkPlan] {
         childrenIndexes.map(children).filterNot(_.isInstanceOf[ShuffleExchangeExec])
           .map(_.outputPartitioning.numPartitions)
       val expectedChildrenNumPartitions = if (nonShuffleChildrenNumPartitions.nonEmpty) {
-        // Here we pick the max number of partitions among these non-shuffle children as the
-        // expected number of shuffle partitions. However, if it's smaller than
-        // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
-        // expected number of shuffle partitions.
-        math.max(nonShuffleChildrenNumPartitions.max, conf.defaultNumShufflePartitions)
+        if (nonShuffleChildrenNumPartitions.length == childrenIndexes.length) {
+          // Here we pick the max number of partitions among these non-shuffle children.
+          nonShuffleChildrenNumPartitions.max
+        } else {
+          // Here we pick the max number of partitions among these non-shuffle children as the
+          // expected number of shuffle partitions. However, if it's smaller than
+          // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the
+          // expected number of shuffle partitions.
+          math.max(nonShuffleChildrenNumPartitions.max, conf.defaultNumShufflePartitions)
+        }
       } else {
         childrenNumPartitions.max
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index 14ba008e89961..558bfd7f4675b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -843,4 +843,32 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-32767 Bucket join should work if SHUFFLE_PARTITIONS larger than bucket number") {
+    withSQLConf(
+      SQLConf.SHUFFLE_PARTITIONS.key -> "9",
+      SQLConf.COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> "10")  {
+
+      val testSpec1 = BucketedTableTestSpec(
+        Some(BucketSpec(8, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = false,
+        expectedSort = false)
+      val testSpec2 = BucketedTableTestSpec(
+        Some(BucketSpec(6, Seq("i", "j"), Seq("i", "j"))),
+        numPartitions = 1,
+        expectedShuffle = true,
+        expectedSort = true)
+      Seq(false, true).foreach { enableAdaptive =>
+        withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> s"$enableAdaptive") {
+          Seq((testSpec1, testSpec2), (testSpec2, testSpec1)).foreach { specs =>
+            testBucketing(
+              bucketedTableTestSpecLeft = specs._1,
+              bucketedTableTestSpecRight = specs._2,
+              joinCondition = joinCondition(Seq("i", "j")))
+          }
+        }
+      }
+    }
+  }
 }

From 7320db21bda01c162623839350de908042f4ebb8 Mon Sep 17 00:00:00 2001
From: Zhenhua Wang <wzh_zju@163.com>
Date: Fri, 4 Sep 2020 09:33:20 +0900
Subject: [PATCH 1024/1280] [SPARK-32786][SQL][TEST] Improve performance for
 some slow DPP tests

### What changes were proposed in this pull request?

The whole `DynamicPartitionPruningSuite` takes about 2 min on my laptop (either AE on or off). The slowest tests are `test("simple inner join triggers DPP with mock-up tables")` and `test("cleanup any DPP filter that isn't pushed down due to expression id clashes")`, which totally take about 1 min.

We can reuse existing test tables or use smaller tables to reduce the cost. After that, the two tests takes only about 1 sec in total, leading to 2x speedup for the suite.

### Why are the changes needed?

To speedup DPP test suites.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Modified two existing tests.

Closes #29636 from wzhfy/improve_dpp_test.

Authored-by: Zhenhua Wang <wzh_zju@163.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/DynamicPartitionPruningSuite.scala    | 30 +++++--------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 47c5c9eacdfd2..8486d1d4d370f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -254,29 +254,15 @@ abstract class DynamicPartitionPruningSuiteBase
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
-      withTable("df1", "df2") {
-        spark.range(1000)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df1")
-
-        spark.range(100)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df2")
-
-        val df = sql("SELECT df1.id, df2.k FROM df1 JOIN df2 ON df1.k = df2.k AND df2.id < 2")
+      val df = sql(
+        """
+          |SELECT f.date_id, f.store_id FROM fact_sk f
+          |JOIN dim_store s ON f.store_id = s.store_id AND s.country = 'NL'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, true, false)
+      checkPartitionPruningPredicate(df, true, false)
 
-        checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
-      }
+      checkAnswer(df, Row(1000, 1) :: Row(1010, 2) :: Row(1020, 2) :: Nil)
     }
   }
 
@@ -1157,7 +1143,7 @@ abstract class DynamicPartitionPruningSuiteBase
   test("cleanup any DPP filter that isn't pushed down due to expression id clashes") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
-        spark.range(1000).select($"id".as("A"), $"id".as("AA"))
+        spark.range(20).select($"id".as("A"), $"id".as("AA"))
           .write.partitionBy("A").format(tableFormat).mode("overwrite").saveAsTable("fact")
         spark.range(10).select($"id".as("B"), $"id".as("BB"))
           .write.format(tableFormat).mode("overwrite").saveAsTable("dim")

From 0c7616b9b5c5f2d69b0410b988bf5e3955e34a6c Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Sat, 5 Sep 2020 23:49:17 +0800
Subject: [PATCH 1025/1280] [SPARK-32791][SQL] Non-partitioned table metric
 should not have dynamic partition pruning time

### What changes were proposed in this pull request?

This pr make non-partitioned table metric should not have dynamic partition pruning time.

### Why are the changes needed?

It is useless for non-partitioned table.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manual test

Before this pr:
![image](https://user-images.githubusercontent.com/5399861/92141803-87fed380-ee45-11ea-9784-09625b246fea.png)
After this pr:
![image](https://user-images.githubusercontent.com/5399861/92141774-7c131180-ee45-11ea-8a9e-6775c592f496.png)

Closes #29641 from wangyum/SPARK-32791.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Yuming Wang <yumwang@ebay.com>
(cherry picked from commit 0b3bb45b893005d73afb15e4904ac6440c87811c)
Signed-off-by: Yuming Wang <yumwang@ebay.com>
---
 .../spark/sql/execution/DataSourceScanExec.scala      | 11 ++++++-----
 .../spark/sql/DynamicPartitionPruningSuite.scala      |  3 +++
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 2147abd151fa2..447e0a689feb5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -438,9 +438,7 @@ case class FileSourceScanExec(
     "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"),
     "numFiles" -> SQLMetrics.createMetric(sparkContext, "number of files read"),
     "metadataTime" -> SQLMetrics.createTimingMetric(sparkContext, "metadata time"),
-    "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read"),
-    "pruningTime" ->
-      SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time")
+    "filesSize" -> SQLMetrics.createSizeMetric(sparkContext, "size of files read")
   ) ++ {
     // Tracking scan time has overhead, we can't afford to do it for each row, and can only do
     // it for each batch.
@@ -451,9 +449,12 @@ case class FileSourceScanExec(
     }
   } ++ {
     if (relation.partitionSchemaOption.isDefined) {
-      Some("numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"))
+      Map(
+        "numPartitions" -> SQLMetrics.createMetric(sparkContext, "number of partitions read"),
+        "pruningTime" ->
+          SQLMetrics.createTimingMetric(sparkContext, "dynamic partition pruning time"))
     } else {
-      None
+      Map.empty[String, SQLMetric]
     }
   } ++ staticMetrics
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 8486d1d4d370f..94163315b34f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -345,6 +345,7 @@ abstract class DynamicPartitionPruningSuiteBase
         val allFilesNum = scan1.metrics("numFiles").value
         val allFilesSize = scan1.metrics("filesSize").value
         assert(scan1.metrics("numPartitions").value === numPartitions)
+        assert(scan1.metrics("pruningTime").value === -1)
 
         // No dynamic partition pruning, so no static metrics
         // Only files from fid = 5 partition are scanned
@@ -358,6 +359,7 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(0 < partFilesNum && partFilesNum < allFilesNum)
         assert(0 < partFilesSize && partFilesSize < allFilesSize)
         assert(scan2.metrics("numPartitions").value === 1)
+        assert(scan2.metrics("pruningTime").value === -1)
 
         // Dynamic partition pruning is used
         // Static metrics are as-if reading the whole fact table
@@ -370,6 +372,7 @@ abstract class DynamicPartitionPruningSuiteBase
         assert(scan3.metrics("numFiles").value == partFilesNum)
         assert(scan3.metrics("filesSize").value == partFilesSize)
         assert(scan3.metrics("numPartitions").value === 1)
+        assert(scan3.metrics("pruningTime").value !== -1)
       }
     }
   }

From c2c7c9ef78441682a585abb1dede9b668802a224 Mon Sep 17 00:00:00 2001
From: "sandeep.katta" <sandeep.katta2007@gmail.com>
Date: Mon, 7 Sep 2020 15:10:33 +0900
Subject: [PATCH 1026/1280] [SPARK-32779][SQL] Avoid using synchronized API of
 SessionCatalog in withClient flow, this leads to DeadLock

### What changes were proposed in this pull request?

No need of using database name in `loadPartition` API of `Shim_v3_0` to get the hive table, in hive there is a overloaded method which gives hive table using table name. By using this API dependency with `SessionCatalog` can be removed in Shim layer

### Why are the changes needed?
To avoid deadlock when communicating with Hive metastore 3.1.x
```
Found one Java-level deadlock:
=============================
"worker3":
  waiting to lock monitor 0x00007faf0be602b8 (object 0x00000007858f85f0, a org.apache.spark.sql.hive.HiveSessionCatalog),
  which is held by "worker0"
"worker0":
  waiting to lock monitor 0x00007faf0be5fc88 (object 0x0000000785c15c80, a org.apache.spark.sql.hive.HiveExternalCatalog),
  which is held by "worker3"

Java stack information for the threads listed above:
===================================================
"worker3":
  at org.apache.spark.sql.catalyst.catalog.SessionCatalog.getCurrentDatabase(SessionCatalog.scala:256)
  - waiting to lock <0x00000007858f85f0> (a org.apache.spark.sql.hive.HiveSessionCatalog)
  at org.apache.spark.sql.hive.client.Shim_v3_0.loadPartition(HiveShim.scala:1332)
  at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$loadPartition$1(HiveClientImpl.scala:870)
  at org.apache.spark.sql.hive.client.HiveClientImpl$$Lambda$4459/1387095575.apply$mcV$sp(Unknown Source)
  at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
  at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:294)
  at org.apache.spark.sql.hive.client.HiveClientImpl$$Lambda$2227/313239499.apply(Unknown Source)
  at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:227)
  at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:226)
  - locked <0x0000000785ef9d78> (a org.apache.spark.sql.hive.client.IsolatedClientLoader)
  at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:276)
  at org.apache.spark.sql.hive.client.HiveClientImpl.loadPartition(HiveClientImpl.scala:860)
  at org.apache.spark.sql.hive.HiveExternalCatalog.$anonfun$loadPartition$1(HiveExternalCatalog.scala:911)
  at org.apache.spark.sql.hive.HiveExternalCatalog$$Lambda$4457/2037578495.apply$mcV$sp(Unknown Source)
  at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
  at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:99)
  - locked <0x0000000785c15c80> (a org.apache.spark.sql.hive.HiveExternalCatalog)
  at org.apache.spark.sql.hive.HiveExternalCatalog.loadPartition(HiveExternalCatalog.scala:890)
  at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.loadPartition(ExternalCatalogWithListener.scala:179)
  at org.apache.spark.sql.catalyst.catalog.SessionCatalog.loadPartition(SessionCatalog.scala:512)
  at org.apache.spark.sql.execution.command.LoadDataCommand.run(tables.scala:383)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
  - locked <0x00000007b1690ff8> (a org.apache.spark.sql.execution.command.ExecutedCommandExec)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
  at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229)
  at org.apache.spark.sql.Dataset$$Lambda$2084/428667685.apply(Unknown Source)
  at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3616)
  at org.apache.spark.sql.Dataset$$Lambda$2085/559530590.apply(Unknown Source)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
  at org.apache.spark.sql.execution.SQLExecution$$$Lambda$2093/139449177.apply(Unknown Source)
  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
  at org.apache.spark.sql.execution.SQLExecution$$$Lambda$2086/1088974677.apply(Unknown Source)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3614)
  at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229)
  at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
  at org.apache.spark.sql.Dataset$$$Lambda$1959/1977822284.apply(Unknown Source)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
  at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
  at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:606)
  at org.apache.spark.sql.SparkSession$$Lambda$1899/424830920.apply(Unknown Source)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:601)
  at $line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anon$1.run(<console>:45)
  at java.lang.Thread.run(Thread.java:748)
"worker0":
  at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:99)
  - waiting to lock <0x0000000785c15c80
  > (a org.apache.spark.sql.hive.HiveExternalCatalog)
  at org.apache.spark.sql.hive.HiveExternalCatalog.tableExists(HiveExternalCatalog.scala:851)
  at org.apache.spark.sql.catalyst.catalog.ExternalCatalogWithListener.tableExists(ExternalCatalogWithListener.scala:146)
  at org.apache.spark.sql.catalyst.catalog.SessionCatalog.tableExists(SessionCatalog.scala:432)
  - locked <0x00000007858f85f0> (a org.apache.spark.sql.hive.HiveSessionCatalog)
  at org.apache.spark.sql.catalyst.catalog.SessionCatalog.requireTableExists(SessionCatalog.scala:185)
  at org.apache.spark.sql.catalyst.catalog.SessionCatalog.loadPartition(SessionCatalog.scala:509)
  at org.apache.spark.sql.execution.command.LoadDataCommand.run(tables.scala:383)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
  - locked <0x00000007b529af58> (a org.apache.spark.sql.execution.command.ExecutedCommandExec)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
  at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229)
  at org.apache.spark.sql.Dataset$$Lambda$2084/428667685.apply(Unknown Source)
  at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3616)
  at org.apache.spark.sql.Dataset$$Lambda$2085/559530590.apply(Unknown Source)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
  at org.apache.spark.sql.execution.SQLExecution$$$Lambda$2093/139449177.apply(Unknown Source)
  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
  at org.apache.spark.sql.execution.SQLExecution$$$Lambda$2086/1088974677.apply(Unknown Source)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3614)
  at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229)
  at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
  at org.apache.spark.sql.Dataset$$$Lambda$1959/1977822284.apply(Unknown Source)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
  at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
  at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:606)
  at org.apache.spark.sql.SparkSession$$Lambda$1899/424830920.apply(Unknown Source)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:601)
  at $line14.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anon$1.run(<console>:45)
  at java.lang.Thread.run(Thread.java:748)

Found 1 deadlock.
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Tested using below script by executing in spark-shell and I found no dead lock

launch spark-shell using ./bin/spark-shell --conf "spark.sql.hive.metastore.jars=maven" --conf spark.sql.hive.metastore.version=3.1 --conf spark.hadoop.datanucleus.schema.autoCreateAll=true

**code**
```
def testHiveDeadLock = {
      import scala.collection.mutable.ArrayBuffer
      import scala.util.Random
      println("test hive DeadLock")
      spark.sql("drop database if exists testDeadLock cascade")
      spark.sql("create database testDeadLock")
      spark.sql("use testDeadLock")
      val tableCount = 100
      val tableNamePrefix = "testdeadlock"
      for (i <- 0 until tableCount) {
        val tableName = s"$tableNamePrefix${i + 1}"
        spark.sql(s"drop table if exists $tableName")
        spark.sql(s"create table $tableName (a bigint) partitioned by (b bigint) stored as orc")
      }

      val threads = new ArrayBuffer[Thread]
      for (i <- 0 until tableCount) {
        threads.append(new Thread( new Runnable {
          override def run: Unit = {
            val tableName = s"$tableNamePrefix${i + 1}"
            val rand = Random
            val df = spark.range(0, 20000).toDF("a")
            val location = s"/tmp/${rand.nextLong.abs}"
            df.write.mode("overwrite").orc(location)
            spark.sql(
              s"""
        LOAD DATA LOCAL INPATH '$location' INTO TABLE $tableName partition (b=$i)""")
          }
        }, s"worker$i"))
        threads(i).start()
      }

      for (i <- 0 until tableCount) {
        println(s"Joining with thread $i")
        threads(i).join()
      }
      for (i <- 0 until tableCount) {
        val tableName = s"$tableNamePrefix${i + 1}"
        spark.sql(s"select count(*) from $tableName").show(false)
      }
      println("All done")
    }

    for(i <- 0 until 100) {
      testHiveDeadLock
      println(s"completed {$i}th iteration")
    }
  }
```

Closes #29649 from sandeep-katta/metastore3.1DeadLock.

Authored-by: sandeep.katta <sandeep.katta2007@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b0322bf05ab582ecd915be374b6e3915742049d7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/sql/hive/client/HiveShim.scala | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 8df43b785759e..8d0a0f63bc615 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -1329,8 +1329,7 @@ private[client] class Shim_v3_0 extends Shim_v2_3 {
       isSrcLocal: Boolean): Unit = {
     val session = SparkSession.getActiveSession
     assert(session.nonEmpty)
-    val database = session.get.sessionState.catalog.getCurrentDatabase
-    val table = hive.getTable(database, tableName)
+    val table = hive.getTable(tableName)
     val loadFileType = if (replace) {
       clazzLoadFileType.getEnumConstants.find(_.toString.equalsIgnoreCase("REPLACE_ALL"))
     } else {

From a22e1c5ac8dc561be84027012b23e2789d2971ad Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 8 Sep 2020 09:45:56 +0900
Subject: [PATCH 1027/1280] [SPARK-32810][SQL][3.0] CSV/JSON data sources
 should avoid globbing paths when inferring schema

### What changes were proposed in this pull request?
In the PR, I propose to fix an issue with the CSV and JSON data sources in Spark SQL when both of the following are true:
* no user specified schema
* some file paths contain escaped glob metacharacters, such as `[``]`, `{``}`, `*` etc.

### Why are the changes needed?
To fix the issue when the follow two queries try to read from paths `[abc].csv` and `[abc].json`:
```scala
spark.read.csv("""/tmp/\[abc\].csv""").show
spark.read.json("""/tmp/\[abc\].json""").show
```
but would end up hitting an exception:
```
org.apache.spark.sql.AnalysisException: Path does not exist: file:/tmp/[abc].csv;
  at org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$checkAndGlobPathIfNecessary$1(DataSource.scala:722)
  at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:244)
  at scala.collection.immutable.List.foreach(List.scala:392)
```

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
Added new test cases in `DataFrameReaderWriterSuite`.

Closes #29662 from MaxGekk/globbing-paths-when-inferring-schema-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../execution/datasources/DataSource.scala    | 32 +++++++++++++++----
 .../datasources/csv/CSVDataSource.scala       |  2 +-
 .../datasources/json/JsonDataSource.scala     |  2 +-
 .../execution/datasources/v2/FileTable.scala  | 10 +++++-
 .../sql/test/DataFrameReaderWriterSuite.scala | 23 +++++++++++++
 5 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
index e97139ffbd071..54faadf73b25f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala
@@ -67,8 +67,9 @@ import org.apache.spark.util.Utils
  * metadata.  For example, when reading a partitioned table from a file system, partition columns
  * will be inferred from the directory layout even if they are not specified.
  *
- * @param paths A list of file system paths that hold data.  These will be globbed before and
- *              qualified. This option only works when reading from a [[FileFormat]].
+ * @param paths A list of file system paths that hold data. These will be globbed before if
+ *              the "__globPaths__" option is true, and will be qualified. This option only works
+ *              when reading from a [[FileFormat]].
  * @param userSpecifiedSchema An optional specification of the schema of the data. When present
  *                            we skip attempting to infer the schema.
  * @param partitionColumns A list of column names that the relation is partitioned by. This list is
@@ -116,6 +117,15 @@ case class DataSource(
   private val caseInsensitiveOptions = CaseInsensitiveMap(options)
   private val equality = sparkSession.sessionState.conf.resolver
 
+  /**
+   * Whether or not paths should be globbed before being used to access files.
+   */
+  def globPaths: Boolean = {
+    options.get(DataSource.GLOB_PATHS_KEY)
+      .map(_ == "true")
+      .getOrElse(true)
+  }
+
   bucketSpec.map { bucket =>
     SchemaUtils.checkColumnNameDuplication(
       bucket.bucketColumnNames, "in the bucket definition", equality)
@@ -234,7 +244,7 @@ case class DataSource(
         // For glob pattern, we do not check it because the glob pattern might only make sense
         // once the streaming job starts and some upstream source starts dropping data.
         val hdfsPath = new Path(path)
-        if (!SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
+        if (!globPaths || !SparkHadoopUtil.get.isGlobPath(hdfsPath)) {
           val fs = hdfsPath.getFileSystem(newHadoopConfiguration())
           if (!fs.exists(hdfsPath)) {
             throw new AnalysisException(s"Path does not exist: $path")
@@ -577,7 +587,7 @@ case class DataSource(
       checkFilesExist: Boolean): Seq[Path] = {
     val allPaths = caseInsensitiveOptions.get("path") ++ paths
     DataSource.checkAndGlobPathIfNecessary(allPaths.toSeq, newHadoopConfiguration(),
-      checkEmptyGlobPath, checkFilesExist)
+      checkEmptyGlobPath, checkFilesExist, enableGlobbing = globPaths)
   }
 }
 
@@ -740,6 +750,11 @@ object DataSource extends Logging {
     }
   }
 
+  /**
+   * The key in the "options" map for deciding whether or not to glob paths before use.
+   */
+  val GLOB_PATHS_KEY = "__globPaths__"
+
   /**
    * Checks and returns files in all the paths.
    */
@@ -747,12 +762,17 @@ object DataSource extends Logging {
       paths: Seq[String],
       hadoopConf: Configuration,
       checkEmptyGlobPath: Boolean,
-      checkFilesExist: Boolean): Seq[Path] = {
+      checkFilesExist: Boolean,
+      enableGlobbing: Boolean): Seq[Path] = {
     val allGlobPath = paths.flatMap { path =>
       val hdfsPath = new Path(path)
       val fs = hdfsPath.getFileSystem(hadoopConf)
       val qualified = hdfsPath.makeQualified(fs.getUri, fs.getWorkingDirectory)
-      val globPath = SparkHadoopUtil.get.globPathIfNecessary(fs, qualified)
+      val globPath = if (enableGlobbing) {
+        SparkHadoopUtil.get.globPathIfNecessary(fs, qualified)
+      } else {
+        qualified :: Nil
+      }
 
       if (checkEmptyGlobPath && globPath.isEmpty) {
         throw new AnalysisException(s"Path does not exist: $qualified")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
index e8005a8d1f8c6..d0283f39707c3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVDataSource.scala
@@ -156,7 +156,7 @@ object TextInputCSVDataSource extends CSVDataSource {
           sparkSession,
           paths = paths,
           className = classOf[TextFileFormat].getName,
-          options = options.parameters.originalMap
+          options = options.parameters.originalMap ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
         ).resolveRelation(checkFilesExist = false))
         .select("value").as[String](Encoders.STRING)
     } else {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
index 1d2e63c16ab97..920f83e6bea97 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonDataSource.scala
@@ -120,7 +120,7 @@ object TextInputJsonDataSource extends JsonDataSource {
         sparkSession,
         paths = inputPaths.map(_.getPath.toString),
         className = classOf[TextFileFormat].getName,
-        options = parsedOptions.parameters.originalMap
+        options = parsedOptions.parameters.originalMap ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value").as(Encoders.STRING)
   }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
index 59dc3ae56bf25..1658a250462b3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileTable.scala
@@ -53,7 +53,7 @@ abstract class FileTable(
     } else {
       // This is a non-streaming file based datasource.
       val rootPathsSpecified = DataSource.checkAndGlobPathIfNecessary(paths, hadoopConf,
-        checkEmptyGlobPath = true, checkFilesExist = true)
+        checkEmptyGlobPath = true, checkFilesExist = true, enableGlobbing = globPaths)
       val fileStatusCache = FileStatusCache.getOrCreate(sparkSession)
       new InMemoryFileIndex(
         sparkSession, rootPathsSpecified, caseSensitiveMap, userSpecifiedSchema, fileStatusCache)
@@ -139,6 +139,14 @@ abstract class FileTable(
    * 2. Catalog support is required, which is still under development for data source V2.
    */
   def fallbackFileFormat: Class[_ <: FileFormat]
+
+  /**
+   * Whether or not paths should be globbed before being used to access files.
+   */
+  private def globPaths: Boolean = {
+    val entry = options.get(DataSource.GLOB_PATHS_KEY)
+    Option(entry).map(_ == "true").getOrElse(true)
+  }
 }
 
 object FileTable {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index c7ca0125c8a0f..f48f445a30bad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -1105,4 +1105,27 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
       }
     }
   }
+
+  test("SPARK-32810: CSV and JSON data sources should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    def escape(str: String): String = {
+      """(\[|\]|\{|\})""".r.replaceAllIn(str, """\\$1""")
+    }
+
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+
+      // test CSV writer / reader without specifying schema
+      val csvTableName = "[abc]"
+      spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
+      val csvDf = spark.read.csv(s"$basePath/${escape(csvTableName)}")
+      assert(csvDf.collect sameElements Array(Row("0"), Row("1"), Row("2")))
+
+      // test JSON writer / reader without specifying schema
+      val jsonTableName = "{def}"
+      spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
+      val jsonDf = spark.read.json(s"$basePath/${escape(jsonTableName)}")
+      assert(jsonDf.collect sameElements Array(Row(0), Row(1), Row(2)))
+    }
+  }
 }

From 6b47abd760902eafa44bc37228774a58864e0a4d Mon Sep 17 00:00:00 2001
From: itholic <haejoon309@naver.com>
Date: Tue, 8 Sep 2020 12:22:13 +0900
Subject: [PATCH 1028/1280] [SPARK-32812][PYTHON][TESTS] Avoid initiating a
 process during the main process for run-tests.py

### What changes were proposed in this pull request?

In certain environments, seems it fails to run `run-tests.py` script as below:

```
Traceback (most recent call last):
 File "<string>", line 1, in <module>
...

raise RuntimeError('''
RuntimeError:
 An attempt has been made to start a new process before the
 current process has finished its bootstrapping phase.

This probably means that you are not using fork to start your
 child processes and you have forgotten to use the proper idiom
 in the main module:

if __name__ == '__main__':
 freeze_support()
 ...

The "freeze_support()" line can be omitted if the program
 is not going to be frozen to produce an executable.
Traceback (most recent call last):
...
 raise EOFError
EOFError

```

The reason is that `Manager.dict()` launches another process when the main process is initiated.

It works in most environments for an unknown reason but it should be good to avoid such pattern as guided from Python itself.

### Why are the changes needed?

To prevent the test failure for Python.

### Does this PR introduce _any_ user-facing change?

No, it fixes a test script.

### How was this patch tested?

Manually ran the script after fixing.

```
Running PySpark tests. Output is in /.../python/unit-tests.log
Will test against the following Python executables: ['/.../python3', 'python3.8']
Will test the following Python tests: ['pyspark.sql.dataframe']
/.../python3 python_implementation is CPython
/.../python3 version is: Python 3.8.5
python3.8 python_implementation is CPython
python3.8 version is: Python 3.8.5
Starting test(/.../python3): pyspark.sql.dataframe
Starting test(python3.8): pyspark.sql.dataframe
Finished test(/.../python3): pyspark.sql.dataframe (33s)
Finished test(python3.8): pyspark.sql.dataframe (34s)
Tests passed in 34 seconds
```

Closes #29666 from itholic/SPARK-32812.

Authored-by: itholic <haejoon309@naver.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c8c082ce380b2357623511c6625503fb3f1d65bf)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/run-tests.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/run-tests.py b/python/run-tests.py
index a404c5364e9bb..9fb0983d5f1fb 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -51,7 +51,7 @@ def print_red(text):
     print('\033[31m' + text + '\033[0m')
 
 
-SKIPPED_TESTS = Manager().dict()
+SKIPPED_TESTS = None
 LOG_FILE = os.path.join(SPARK_HOME, "python/unit-tests.log")
 FAILURE_REPORTING_LOCK = Lock()
 LOGGER = logging.getLogger()
@@ -142,6 +142,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
             skipped_counts = len(skipped_tests)
             if skipped_counts > 0:
                 key = (pyspark_python, test_name)
+                assert SKIPPED_TESTS is not None
                 SKIPPED_TESTS[key] = skipped_tests
             per_test_output.close()
         except:
@@ -322,4 +323,5 @@ def process_queue(task_queue):
 
 
 if __name__ == "__main__":
+    SKIPPED_TESTS = Manager().dict()
     main()

From f42b56c17c4da9a5952dc9d87df4ad8bea369a4d Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 7 Sep 2020 20:43:43 -0700
Subject: [PATCH 1029/1280] [SPARK-32764][SQL] -0.0 should be equal to 0.0

This is a Spark 3.0 regression introduced by https://github.com/apache/spark/pull/26761. We missed a corner case that `java.lang.Double.compare` treats 0.0 and -0.0 as different, which breaks SQL semantic.

This PR adds back the `OrderingUtil`, to provide custom compare methods that take care of 0.0 vs -0.0

Fix a correctness bug.

Yes, now `SELECT  0.0 > -0.0` returns false correctly as Spark 2.x.

new tests

Closes #29647 from cloud-fan/float.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 4144b6da5215e9264056c2f6720fd819e18e23a5)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../expressions/codegen/CodeGenerator.scala   | 10 ++-
 .../sql/catalyst/util/SQLOrderingUtil.scala   | 41 ++++++++++
 .../apache/spark/sql/types/DoubleType.scala   |  3 +-
 .../apache/spark/sql/types/FloatType.scala    |  3 +-
 .../org/apache/spark/sql/types/numerics.scala |  5 +-
 .../catalyst/expressions/PredicateSuite.scala | 16 ++++
 .../catalyst/util/SQLOrderingUtilSuite.scala  | 75 +++++++++++++++++++
 .../org/apache/spark/sql/DataFrameSuite.scala |  5 ++
 8 files changed, 151 insertions(+), 7 deletions(-)
 create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
 create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index aa6d6e67ece04..510696ea845f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -38,7 +38,7 @@ import org.apache.spark.metrics.source.CodegenMetrics
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.Block._
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, MapData}
+import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData, MapData, SQLOrderingUtil}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
@@ -624,8 +624,12 @@ class CodegenContext extends Logging {
   def genComp(dataType: DataType, c1: String, c2: String): String = dataType match {
     // java boolean doesn't support > or < operator
     case BooleanType => s"($c1 == $c2 ? 0 : ($c1 ? 1 : -1))"
-    case DoubleType => s"java.lang.Double.compare($c1, $c2)"
-    case FloatType => s"java.lang.Float.compare($c1, $c2)"
+    case DoubleType =>
+      val clsName = SQLOrderingUtil.getClass.getName.stripSuffix("$")
+      s"$clsName.compareDoubles($c1, $c2)"
+    case FloatType =>
+      val clsName = SQLOrderingUtil.getClass.getName.stripSuffix("$")
+      s"$clsName.compareFloats($c1, $c2)"
     // use c1 - c2 may overflow
     case dt: DataType if isPrimitiveType(dt) => s"($c1 > $c2 ? 1 : $c1 < $c2 ? -1 : 0)"
     case BinaryType => s"org.apache.spark.sql.catalyst.util.TypeUtils.compareBinary($c1, $c2)"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
new file mode 100644
index 0000000000000..3b7f748c2817b
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtil.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+object SQLOrderingUtil {
+
+  /**
+   * A special version of double comparison that follows SQL semantic:
+   *  1. NaN == NaN
+   *  2. NaN is greater than any non-NaN double
+   *  3. -0.0 == 0.0
+   */
+  def compareDoubles(x: Double, y: Double): Int = {
+    if (x == y) 0 else java.lang.Double.compare(x, y)
+  }
+
+  /**
+   * A special version of float comparison that follows SQL semantic:
+   *  1. NaN == NaN
+   *  2. NaN is greater than any non-NaN float
+   *  3. -0.0 == 0.0
+   */
+  def compareFloats(x: Float, y: Float): Int = {
+    if (x == y) 0 else java.lang.Float.compare(x, y)
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
index 01268a9ff1667..ea4f39d4b19d2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DoubleType.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.typeTag
 import scala.util.Try
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 
 /**
  * The data type representing `Double` values. Please use the singleton `DataTypes.DoubleType`.
@@ -38,7 +39,7 @@ class DoubleType private() extends FractionalType {
   private[sql] val numeric = implicitly[Numeric[Double]]
   private[sql] val fractional = implicitly[Fractional[Double]]
   private[sql] val ordering =
-    (x: Double, y: Double) => java.lang.Double.compare(x, y)
+    (x: Double, y: Double) => SQLOrderingUtil.compareDoubles(x, y)
   private[sql] val asIntegral = DoubleType.DoubleAsIfIntegral
 
   override private[sql] def exactNumeric = DoubleExactNumeric
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
index 1491f5904baef..f00046facf693 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/FloatType.scala
@@ -22,6 +22,7 @@ import scala.reflect.runtime.universe.typeTag
 import scala.util.Try
 
 import org.apache.spark.annotation.Stable
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 
 /**
  * The data type representing `Float` values. Please use the singleton `DataTypes.FloatType`.
@@ -38,7 +39,7 @@ class FloatType private() extends FractionalType {
   private[sql] val numeric = implicitly[Numeric[Float]]
   private[sql] val fractional = implicitly[Fractional[Float]]
   private[sql] val ordering =
-    (x: Float, y: Float) => java.lang.Float.compare(x, y)
+    (x: Float, y: Float) => SQLOrderingUtil.compareFloats(x, y)
   private[sql] val asIntegral = FloatType.FloatAsIfIntegral
 
   override private[sql] def exactNumeric = FloatExactNumeric
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
index 3956629cf6a57..7026ff7de2e0d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/numerics.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.types
 import scala.math.Numeric._
 import scala.math.Ordering
 
+import org.apache.spark.sql.catalyst.util.SQLOrderingUtil
 import org.apache.spark.sql.types.Decimal.DecimalIsConflicted
 
 private[sql] object ByteExactNumeric extends ByteIsIntegral with Ordering.ByteOrdering {
@@ -148,7 +149,7 @@ private[sql] object FloatExactNumeric extends FloatIsFractional {
     }
   }
 
-  override def compare(x: Float, y: Float): Int = java.lang.Float.compare(x, y)
+  override def compare(x: Float, y: Float): Int = SQLOrderingUtil.compareFloats(x, y)
 }
 
 private[sql] object DoubleExactNumeric extends DoubleIsFractional {
@@ -176,7 +177,7 @@ private[sql] object DoubleExactNumeric extends DoubleIsFractional {
     }
   }
 
-  override def compare(x: Double, y: Double): Int = java.lang.Double.compare(x, y)
+  override def compare(x: Double, y: Double): Int = SQLOrderingUtil.compareDoubles(x, y)
 }
 
 private[sql] object DecimalExactNumeric extends DecimalIsConflicted {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index 1ad0a8ed758f4..a36baec1a0b99 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -538,4 +538,20 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
     val inSet = InSet(BoundReference(0, IntegerType, true), Set.empty)
     checkEvaluation(inSet, false, row)
   }
+
+  test("SPARK-32764: compare special double/float values") {
+    checkEvaluation(EqualTo(Literal(Double.NaN), Literal(Double.NaN)), true)
+    checkEvaluation(EqualTo(Literal(Double.NaN), Literal(Double.PositiveInfinity)), false)
+    checkEvaluation(EqualTo(Literal(0.0D), Literal(-0.0D)), true)
+    checkEvaluation(GreaterThan(Literal(Double.NaN), Literal(Double.PositiveInfinity)), true)
+    checkEvaluation(GreaterThan(Literal(Double.NaN), Literal(Double.NaN)), false)
+    checkEvaluation(GreaterThan(Literal(0.0D), Literal(-0.0D)), false)
+
+    checkEvaluation(EqualTo(Literal(Float.NaN), Literal(Float.NaN)), true)
+    checkEvaluation(EqualTo(Literal(Float.NaN), Literal(Float.PositiveInfinity)), false)
+    checkEvaluation(EqualTo(Literal(0.0F), Literal(-0.0F)), true)
+    checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.PositiveInfinity)), true)
+    checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.NaN)), false)
+    checkEvaluation(GreaterThan(Literal(0.0F), Literal(-0.0F)), false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala
new file mode 100644
index 0000000000000..6fe774e8afcb6
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/SQLOrderingUtilSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import java.lang.{Double => JDouble, Float => JFloat}
+
+import org.apache.spark.SparkFunSuite
+
+class SQLOrderingUtilSuite extends SparkFunSuite {
+
+  test("compareDoublesSQL") {
+    def shouldMatchDefaultOrder(a: Double, b: Double): Unit = {
+      assert(SQLOrderingUtil.compareDoubles(a, b) === JDouble.compare(a, b))
+      assert(SQLOrderingUtil.compareDoubles(b, a) === JDouble.compare(b, a))
+    }
+    shouldMatchDefaultOrder(0d, 0d)
+    shouldMatchDefaultOrder(0d, 1d)
+    shouldMatchDefaultOrder(-1d, 1d)
+    shouldMatchDefaultOrder(Double.MinValue, Double.MaxValue)
+
+    val specialNaN = JDouble.longBitsToDouble(0x7ff1234512345678L)
+    assert(JDouble.isNaN(specialNaN))
+    assert(JDouble.doubleToRawLongBits(Double.NaN) != JDouble.doubleToRawLongBits(specialNaN))
+
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.NaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, specialNaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(specialNaN, Double.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NaN, Double.NegativeInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.PositiveInfinity, Double.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(Double.NegativeInfinity, Double.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(0.0d, -0.0d) === 0)
+    assert(SQLOrderingUtil.compareDoubles(-0.0d, 0.0d) === 0)
+  }
+
+  test("compareFloatsSQL") {
+    def shouldMatchDefaultOrder(a: Float, b: Float): Unit = {
+      assert(SQLOrderingUtil.compareFloats(a, b) === JFloat.compare(a, b))
+      assert(SQLOrderingUtil.compareFloats(b, a) === JFloat.compare(b, a))
+    }
+    shouldMatchDefaultOrder(0f, 0f)
+    shouldMatchDefaultOrder(0f, 1f)
+    shouldMatchDefaultOrder(-1f, 1f)
+    shouldMatchDefaultOrder(Float.MinValue, Float.MaxValue)
+
+    val specialNaN = JFloat.intBitsToFloat(-6966608)
+    assert(JFloat.isNaN(specialNaN))
+    assert(JFloat.floatToRawIntBits(Float.NaN) != JFloat.floatToRawIntBits(specialNaN))
+
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.NaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, specialNaN) === 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(specialNaN, Float.PositiveInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NaN, Float.NegativeInfinity) > 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.PositiveInfinity, Float.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(Float.NegativeInfinity, Float.NaN) < 0)
+    assert(SQLOrderingUtil.compareDoubles(0.0f, -0.0f) === 0)
+    assert(SQLOrderingUtil.compareDoubles(-0.0f, 0.0f) === 0)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 954a4bd9331ed..f8ce3f49218de 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2450,6 +2450,11 @@ class DataFrameSuite extends QueryTest
       assert(df.schema === new StructType().add(StructField("d", DecimalType(38, 0))))
     }
   }
+
+  test("SPARK-32764: -0.0 and 0.0 should be equal") {
+    val df = Seq(0.0 -> -0.0).toDF("pos", "neg")
+    checkAnswer(df.select($"pos" > $"neg"), Row(false))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)

From 3b32ddfd844ffb53e1dc1bed5a90b4d5750a4e45 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 8 Sep 2020 06:30:11 +0000
Subject: [PATCH 1030/1280] [SPARK-32785][SQL][3.0] Interval with dangling
 parts should not result null

THIS PR brings https://github.com/apache/spark/pull/29635 to branch-3.0 and targets v3.0.2

### What changes were proposed in this pull request?

bugfix for incomplete interval values, e.g. interval '1', interval '1 day 2', currently these cases will result null, but actually we should fail them with IllegalArgumentsException

### Why are the changes needed?

correctness

### Does this PR introduce _any_ user-facing change?

yes, incomplete intervals will throw exception now

#### before
```
bin/spark-sql -S -e "select interval '1', interval '+', interval '1 day -'"

NULL NULL NULL
```
#### after

```
-- !query
select interval '1'
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.catalyst.parser.ParseException

Cannot parse the INTERVAL value: 1(line 1, pos 7)

== SQL ==
select interval '1'
```

### How was this patch tested?

unit tests added

Closes #29658 from yaooqinn/SPARK-32785-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 docs/sql-migration-guide.md                   |   4 +
 .../sql/catalyst/util/IntervalUtils.scala     |   5 +-
 .../catalyst/util/IntervalUtilsSuite.scala    |  13 +++
 .../resources/sql-tests/inputs/interval.sql   |   8 ++
 .../sql-tests/results/ansi/interval.sql.out   | 100 +++++++++++++++++-
 .../sql-tests/results/interval.sql.out        | 100 +++++++++++++++++-
 6 files changed, 227 insertions(+), 3 deletions(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 532ac70fb91be..e64c03755725c 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -22,6 +22,10 @@ license: |
 * Table of contents
 {:toc}
 
+## Upgrading from Spark SQL 3.0.1 to 3.0.2
+
+  - In Spark 3.0.2, incomplete interval literals, e.g. `INTERVAL '1'`, `INTERVAL '1 DAY 2'` will fail with IllegalArgumentException. In Spark 3.0.1 and earlier, they result `NULL`s.
+
 ## Upgrading from Spark SQL 3.0 to 3.0.1
 
 - In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 833942d7b0104..af5061b005cb2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -735,7 +735,10 @@ object IntervalUtils {
     val result = state match {
       case UNIT_SUFFIX | UNIT_END | TRIM_BEFORE_SIGN =>
         new CalendarInterval(months, days, microseconds)
-      case _ => null
+      case TRIM_BEFORE_VALUE => throwIAE(s"expect a number after '$currentWord' but hit EOL")
+      case VALUE | VALUE_FRACTIONAL_PART =>
+        throwIAE(s"expect a unit name after '$currentWord' but hit EOL")
+      case _ => throwIAE(s"unknown error when parsing '$currentWord'")
     }
 
     result
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 3d9372c9b43b7..1858378edc894 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -77,6 +77,19 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
     }
   }
 
+  test("string to interval: interval with dangling parts should not results null") {
+    checkFromInvalidString("+", "expect a number after '+' but hit EOL")
+    checkFromInvalidString("-", "expect a number after '-' but hit EOL")
+    checkFromInvalidString("+ 2", "expect a unit name after '2' but hit EOL")
+    checkFromInvalidString("- 1", "expect a unit name after '1' but hit EOL")
+    checkFromInvalidString("1", "expect a unit name after '1' but hit EOL")
+    checkFromInvalidString("1.2", "expect a unit name after '1.2' but hit EOL")
+    checkFromInvalidString("1 day 2", "expect a unit name after '2' but hit EOL")
+    checkFromInvalidString("1 day 2.2", "expect a unit name after '2.2' but hit EOL")
+    checkFromInvalidString("1 day -", "expect a number after '-' but hit EOL")
+    checkFromInvalidString("-.", "expect a unit name after '-.' but hit EOL")
+  }
+
   test("string to interval: multiple units") {
     Seq(
       "-1 MONTH 1 day -1 microseconds" -> new CalendarInterval(-1, 1, -1),
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index ebc39f5bc44a6..9ddefa5a34b87 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -180,3 +180,11 @@ SELECT
   to_json(from_json('{"a":"1 days"}', 'a interval')),
   to_json(map('a', interval 25 month 100 day 130 minute)),
   from_json(to_json(map('a', interval 25 month 100 day 130 minute)), 'a interval');
+
+select interval '+';
+select interval '+.';
+select interval '1';
+select interval '1.2';
+select interval '- 2';
+select interval '1 day -';
+select interval '1 day 1';
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 6a6a1df7eeb0e..eb84f3e87f053 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 93
+-- Number of queries: 100
 
 
 -- !query
@@ -956,3 +956,101 @@ SELECT
 struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
 -- !query output
 {"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
+
+
+-- !query
+select interval '+'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +(line 1, pos 7)
+
+== SQL ==
+select interval '+'
+-------^^^
+
+
+-- !query
+select interval '+.'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +.(line 1, pos 7)
+
+== SQL ==
+select interval '+.'
+-------^^^
+
+
+-- !query
+select interval '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1(line 1, pos 7)
+
+== SQL ==
+select interval '1'
+-------^^^
+
+
+-- !query
+select interval '1.2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1.2(line 1, pos 7)
+
+== SQL ==
+select interval '1.2'
+-------^^^
+
+
+-- !query
+select interval '- 2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: - 2(line 1, pos 7)
+
+== SQL ==
+select interval '- 2'
+-------^^^
+
+
+-- !query
+select interval '1 day -'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day -(line 1, pos 7)
+
+== SQL ==
+select interval '1 day -'
+-------^^^
+
+
+-- !query
+select interval '1 day 1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
+
+== SQL ==
+select interval '1 day 1'
+-------^^^
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index 63ed9b567c058..c5dd1e001bbe0 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 93
+-- Number of queries: 100
 
 
 -- !query
@@ -928,3 +928,101 @@ SELECT
 struct<from_json({"a":"1 days"}):struct<a:interval>,to_json(from_json({"a":"1 days"})):string,to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes')):string,from_json(to_json(map(a, INTERVAL '2 years 1 months 100 days 2 hours 10 minutes'))):struct<a:interval>>
 -- !query output
 {"a":1 days}	{"a":"1 days"}	{"a":"2 years 1 months 100 days 2 hours 10 minutes"}	{"a":2 years 1 months 100 days 2 hours 10 minutes}
+
+
+-- !query
+select interval '+'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +(line 1, pos 7)
+
+== SQL ==
+select interval '+'
+-------^^^
+
+
+-- !query
+select interval '+.'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: +.(line 1, pos 7)
+
+== SQL ==
+select interval '+.'
+-------^^^
+
+
+-- !query
+select interval '1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1(line 1, pos 7)
+
+== SQL ==
+select interval '1'
+-------^^^
+
+
+-- !query
+select interval '1.2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1.2(line 1, pos 7)
+
+== SQL ==
+select interval '1.2'
+-------^^^
+
+
+-- !query
+select interval '- 2'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: - 2(line 1, pos 7)
+
+== SQL ==
+select interval '- 2'
+-------^^^
+
+
+-- !query
+select interval '1 day -'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day -(line 1, pos 7)
+
+== SQL ==
+select interval '1 day -'
+-------^^^
+
+
+-- !query
+select interval '1 day 1'
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
+
+== SQL ==
+select interval '1 day 1'
+-------^^^

From 4656ee57217004d1d679d4a45a4ce8278ded0188 Mon Sep 17 00:00:00 2001
From: sychen <sychen@ctrip.com>
Date: Tue, 8 Sep 2020 11:54:04 +0000
Subject: [PATCH 1031/1280] [SPARK-31511][FOLLOW-UP][TEST][SQL] Make
 BytesToBytesMap iterators thread-safe

### What changes were proposed in this pull request?
Before SPARK-31511 is fixed, `BytesToBytesMap` iterator() is not thread-safe and may cause data inaccuracy.
We need to add a unit test.

### Why are the changes needed?
Increase test coverage to ensure that iterator() is thread-safe.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
add ut

Closes #29669 from cxzl25/SPARK-31511-test.

Authored-by: sychen <sychen@ctrip.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit bd3dc2f54d871d152331612c53f586181f4e87fc)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../execution/joins/HashedRelationSuite.scala | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
index 3526aa254c280..5e7985e28a8e5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/HashedRelationSuite.scala
@@ -358,6 +358,45 @@ class HashedRelationSuite extends SharedSparkSession {
     assert(java.util.Arrays.equals(os.toByteArray, os2.toByteArray))
   }
 
+  test("SPARK-31511: Make BytesToBytesMap iterators thread-safe") {
+    val ser = sparkContext.env.serializer.newInstance()
+    val key = Seq(BoundReference(0, LongType, false))
+
+    val unsafeProj = UnsafeProjection.create(
+      Seq(BoundReference(0, LongType, false), BoundReference(1, IntegerType, true)))
+    val rows = (0 until 10000).map(i => unsafeProj(InternalRow(Int.int2long(i), i + 1)).copy())
+    val unsafeHashed = UnsafeHashedRelation(rows.iterator, key, 1, mm)
+
+    val os = new ByteArrayOutputStream()
+    val thread1 = new Thread {
+      override def run(): Unit = {
+        val out = new ObjectOutputStream(os)
+        unsafeHashed.asInstanceOf[UnsafeHashedRelation].writeExternal(out)
+        out.flush()
+      }
+    }
+
+    val thread2 = new Thread {
+      override def run(): Unit = {
+        val threadOut = new ObjectOutputStream(new ByteArrayOutputStream())
+        unsafeHashed.asInstanceOf[UnsafeHashedRelation].writeExternal(threadOut)
+        threadOut.flush()
+      }
+    }
+
+    thread1.start()
+    thread2.start()
+    thread1.join()
+    thread2.join()
+
+    val unsafeHashed2 = ser.deserialize[UnsafeHashedRelation](ser.serialize(unsafeHashed))
+    val os2 = new ByteArrayOutputStream()
+    val out2 = new ObjectOutputStream(os2)
+    unsafeHashed2.writeExternal(out2)
+    out2.flush()
+    assert(java.util.Arrays.equals(os.toByteArray, os2.toByteArray))
+  }
+
   // This test require 4G heap to run, should run it manually
   ignore("build HashedRelation that is larger than 1G") {
     val unsafeProj = UnsafeProjection.create(

From 9b39e4b7aefcedf09764b3528a32bdf0b77e331b Mon Sep 17 00:00:00 2001
From: manuzhang <owenzhang1990@gmail.com>
Date: Tue, 8 Sep 2020 13:36:05 +0000
Subject: [PATCH 1032/1280] [SPARK-32753][SQL][3.0] Only copy tags to node with
 no tags

This PR backports https://github.com/apache/spark/pull/29593 to branch-3.0

### What changes were proposed in this pull request?
Only copy tags to node with no tags when transforming plans.

### Why are the changes needed?
cloud-fan [made a good point](https://github.com/apache/spark/pull/29593#discussion_r482013121) that it doesn't make sense to append tags to existing nodes when nodes are removed. That will cause such bugs as duplicate rows when deduplicating and repartitioning by the same column with AQE.

```
spark.range(10).union(spark.range(10)).createOrReplaceTempView("v1")
val df = spark.sql("select id from v1 group by id distribute by id")
println(df.collect().toArray.mkString(","))
println(df.queryExecution.executedPlan)

// With AQE
[4],[0],[3],[2],[1],[7],[6],[8],[5],[9],[4],[0],[3],[2],[1],[7],[6],[8],[5],[9]
AdaptiveSparkPlan(isFinalPlan=true)
+- CustomShuffleReader local
   +- ShuffleQueryStage 0
      +- Exchange hashpartitioning(id#183L, 10), true
         +- *(3) HashAggregate(keys=[id#183L], functions=[], output=[id#183L])
            +- Union
               :- *(1) Range (0, 10, step=1, splits=2)
               +- *(2) Range (0, 10, step=1, splits=2)

// Without AQE
[4],[7],[0],[6],[8],[3],[2],[5],[1],[9]
*(4) HashAggregate(keys=[id#206L], functions=[], output=[id#206L])
+- Exchange hashpartitioning(id#206L, 10), true
   +- *(3) HashAggregate(keys=[id#206L], functions=[], output=[id#206L])
      +- Union
         :- *(1) Range (0, 10, step=1, splits=2)
         +- *(2) Range (0, 10, step=1, splits=2)
```

It's too expensive to detect node removal so we make a compromise only to copy tags to node with no tags.

### Does this PR introduce any user-facing change?
Yes. Fix a bug.

### How was this patch tested?
Add test.

Closes #29665 from manuzhang/spark-32753-3.0.

Authored-by: manuzhang <owenzhang1990@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/trees/TreeNode.scala      |  7 ++++++-
 .../adaptive/AdaptiveQueryExecSuite.scala        | 16 +++++++++++++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index c4a106702a515..4c7474299587b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -91,7 +91,12 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   private val tags: mutable.Map[TreeNodeTag[_], Any] = mutable.Map.empty
 
   protected def copyTagsFrom(other: BaseType): Unit = {
-    tags ++= other.tags
+    // SPARK-32753: it only makes sense to copy tags to a new node
+    // but it's too expensive to detect other cases likes node removal
+    // so we make a compromise here to copy tags to node with no tags
+    if (tags.isEmpty) {
+      tags ++= other.tags
+    }
   }
 
   def setTagValue[T](tag: TreeNodeTag[T], value: T): Unit = {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
index e18adbda96c0f..6d97a6bb47d0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
 import org.apache.spark.sql.execution.{PartialReducerPartitionSpec, ReusedSubqueryExec, ShuffledRowRDD, SparkPlan}
 import org.apache.spark.sql.execution.adaptive.OptimizeLocalShuffleReader.LOCAL_SHUFFLE_READER_DESCRIPTION
 import org.apache.spark.sql.execution.command.DataWritingCommandExec
-import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec}
+import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange, ReusedExchangeExec, ShuffleExchangeExec}
 import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BuildRight, SortMergeJoinExec}
 import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate
 import org.apache.spark.sql.functions._
@@ -889,4 +889,18 @@ class AdaptiveQueryExecSuite
       }
     }
   }
+
+  test("SPARK-32753: Only copy tags to node with no tags") {
+    withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") {
+      withTempView("v1") {
+        spark.range(10).union(spark.range(10)).createOrReplaceTempView("v1")
+
+        val (_, adaptivePlan) = runAdaptiveAndVerifyResult(
+          "SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id")
+        assert(collect(adaptivePlan) {
+          case s: ShuffleExchangeExec => s
+        }.length == 1)
+      }
+    }
+  }
 }

From 8c0b9cbf68693db22314637a75f28e5aa954aff8 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 8 Sep 2020 14:16:13 +0000
Subject: [PATCH 1033/1280] [SPARK-32815][ML][3.0] Fix LibSVM data source
 loading error on file paths with glob metacharacters

### What changes were proposed in this pull request?
In the PR, I propose to fix an issue with LibSVM datasource when both of the following are true:
* no user specified schema
* some file paths contain escaped glob metacharacters, such as `[``]`, `{``}`, `*` etc.

The fix is a backport of https://github.com/apache/spark/pull/29670, and it is based on another bug fix for CSV/JSON datasources https://github.com/apache/spark/pull/29659.

### Why are the changes needed?
To fix the issue when the follow two queries try to read from paths `[abc]`:
```scala
spark.read.format("libsvm").load("""/tmp/\[abc\].csv""").show
```
but would end up hitting an exception:
```
Path does not exist: file:/private/var/folders/p3/dfs6mf655d7fnjrsjvldh0tc0000gn/T/spark-6ef0ae5e-ff9f-4c4f-9ff4-0db3ee1f6a82/[abc]/part-00000-26406ab9-4e56-45fd-a25a-491c18a05e76-c000.libsvm;
org.apache.spark.sql.AnalysisException: Path does not exist: file:/private/var/folders/p3/dfs6mf655d7fnjrsjvldh0tc0000gn/T/spark-6ef0ae5e-ff9f-4c4f-9ff4-0db3ee1f6a82/[abc]/part-00000-26406ab9-4e56-45fd-a25a-491c18a05e76-c000.libsvm;
	at org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$checkAndGlobPathIfNecessary$3(DataSource.scala:770)
	at org.apache.spark.util.ThreadUtils$.$anonfun$parmap$2(ThreadUtils.scala:373)
	at scala.concurrent.Future$.$anonfun$apply$1(Future.scala:659)
	at scala.util.Success.$anonfun$map$1(Try.scala:255)
	at scala.util.Success.map(Try.scala:213)
```

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
Added UT to `LibSVMRelationSuite`.

Closes #29675 from MaxGekk/globbing-paths-when-inferring-schema-ml-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../ml/source/libsvm/LibSVMRelation.scala     |  2 +-
 .../org/apache/spark/mllib/util/MLUtils.scala |  3 ++-
 .../source/libsvm/LibSVMRelationSuite.scala   | 20 +++++++++++++++++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
index da8f3a24ff27e..11be1d85fbead 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
@@ -100,7 +100,7 @@ private[libsvm] class LibSVMFileFormat
         "though the input. If you know the number in advance, please specify it via " +
         "'numFeatures' option to avoid the extra scan.")
 
-      val paths = files.map(_.getPath.toUri.toString)
+      val paths = files.map(_.getPath.toString)
       val parsed = MLUtils.parseLibSVMFile(sparkSession, paths)
       MLUtils.computeNumFeatures(parsed)
     }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index 9198334ba02a1..24113003c9675 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -110,7 +110,8 @@ object MLUtils extends Logging {
       DataSource.apply(
         sparkSession,
         paths = paths,
-        className = classOf[TextFileFormat].getName
+        className = classOf[TextFileFormat].getName,
+        options = Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value")
 
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
index 263ad26657545..0999892364e2c 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
@@ -191,4 +191,24 @@ class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
       spark.sql("DROP TABLE IF EXISTS libsvmTable")
     }
   }
+
+  test("SPARK-32815: Test LibSVM data source on file paths with glob metacharacters") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test libsvm writer / reader without specifying schema
+      val svmFileName = "[abc]"
+      val escapedSvmFileName = "\\[abc\\]"
+      val rawData = new java.util.ArrayList[Row]()
+      rawData.add(Row(1.0, Vectors.sparse(2, Seq((0, 2.0), (1, 3.0)))))
+      val struct = new StructType()
+        .add("labelFoo", DoubleType, false)
+        .add("featuresBar", VectorType, false)
+      val df = spark.createDataFrame(rawData, struct)
+      df.write.format("libsvm").save(s"$basePath/$svmFileName")
+      val df2 = spark.read.format("libsvm").load(s"$basePath/$escapedSvmFileName")
+      val row1 = df2.first()
+      val v = row1.getAs[SparseVector](1)
+      assert(v == Vectors.sparse(2, Seq((0, 2.0), (1, 3.0))))
+    }
+  }
 }

From 3f20f1413ccb1f685b96de96e5e705db6a70e058 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 9 Sep 2020 08:21:59 +0900
Subject: [PATCH 1034/1280] [SPARK-32638][SQL][3.0] Corrects references when
 adding aliases in WidenSetOperationTypes

### What changes were proposed in this pull request?

This PR intends to fix a bug where references can be missing when adding aliases to widen data types in `WidenSetOperationTypes`. For example,
```
CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
SELECT t.v FROM (
  SELECT v FROM t3
  UNION ALL
  SELECT v + v AS v FROM t3
) t;

org.apache.spark.sql.AnalysisException: Resolved attribute(s) v#1 missing from v#3 in operator !Project [v#1]. Attribute(s) with the same name appear in the operation: v. Please check if the right attribute(s) are used.;;
!Project [v#1]  <------ the reference got missing
+- SubqueryAlias t
   +- Union
      :- Project [cast(v#1 as decimal(11,0)) AS v#3]
      :  +- Project [v#1]
      :     +- SubqueryAlias t3
      :        +- SubqueryAlias tbl
      :           +- LocalRelation [v#1]
      +- Project [v#2]
         +- Project [CheckOverflow((promote_precision(cast(v#1 as decimal(11,0))) + promote_precision(cast(v#1 as decimal(11,0)))), DecimalType(11,0), true) AS v#2]
            +- SubqueryAlias t3
               +- SubqueryAlias tbl
                  +- LocalRelation [v#1]
```
In the case, `WidenSetOperationTypes` added the alias `cast(v#1 as decimal(11,0)) AS v#3`, then the reference in the top `Project` got missing. This PR correct the reference (`exprId` and widen `dataType`) after adding aliases in the rule.

This backport for 3.0 comes from #29485 and #29643

### Why are the changes needed?

bugfixes

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Added unit tests

Closes #29680 from maropu/SPARK-32638-BRANCH3.0.

Lead-authored-by: Wenchen Fan <wenchen@databricks.com>
Co-authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 109 ++----------------
 .../sql/catalyst/analysis/TypeCoercion.scala  |  59 ++++++----
 .../spark/sql/catalyst/plans/QueryPlan.scala  |  85 ++++++++++++++
 .../plans/logical/AnalysisHelper.scala        |  15 ++-
 .../catalyst/analysis/TypeCoercionSuite.scala |  18 ++-
 .../resources/sql-tests/inputs/except.sql     |  19 +++
 .../sql-tests/inputs/intersect-all.sql        |  15 +++
 .../test/resources/sql-tests/inputs/union.sql |  14 +++
 .../sql-tests/results/except.sql.out          |  58 +++++++++-
 .../sql-tests/results/intersect-all.sql.out   |  42 ++++++-
 .../resources/sql-tests/results/union.sql.out |  43 ++++++-
 11 files changed, 348 insertions(+), 129 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 89454c252af53..729d316811349 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1239,108 +1239,13 @@ class Analyzer(
       if (conflictPlans.isEmpty) {
         right
       } else {
-        rewritePlan(right, conflictPlans.toMap)._1
-      }
-    }
-
-    private def rewritePlan(plan: LogicalPlan, conflictPlanMap: Map[LogicalPlan, LogicalPlan])
-      : (LogicalPlan, Seq[(Attribute, Attribute)]) = {
-      if (conflictPlanMap.contains(plan)) {
-        // If the plan is the one that conflict the with left one, we'd
-        // just replace it with the new plan and collect the rewrite
-        // attributes for the parent node.
-        val newRelation = conflictPlanMap(plan)
-        newRelation -> plan.output.zip(newRelation.output)
-      } else {
-        val attrMapping = new mutable.ArrayBuffer[(Attribute, Attribute)]()
-        val newPlan = plan.mapChildren { child =>
-          // If not, we'd rewrite child plan recursively until we find the
-          // conflict node or reach the leaf node.
-          val (newChild, childAttrMapping) = rewritePlan(child, conflictPlanMap)
-          attrMapping ++= childAttrMapping.filter { case (oldAttr, _) =>
-            // `attrMapping` is not only used to replace the attributes of the current `plan`,
-            // but also to be propagated to the parent plans of the current `plan`. Therefore,
-            // the `oldAttr` must be part of either `plan.references` (so that it can be used to
-            // replace attributes of the current `plan`) or `plan.outputSet` (so that it can be
-            // used by those parent plans).
-            (plan.outputSet ++ plan.references).contains(oldAttr)
-          }
-          newChild
-        }
-
-        if (attrMapping.isEmpty) {
-          newPlan -> attrMapping
-        } else {
-          assert(!attrMapping.groupBy(_._1.exprId)
-            .exists(_._2.map(_._2.exprId).distinct.length > 1),
-            "Found duplicate rewrite attributes")
-          val attributeRewrites = AttributeMap(attrMapping)
-          // Using attrMapping from the children plans to rewrite their parent node.
-          // Note that we shouldn't rewrite a node using attrMapping from its sibling nodes.
-          newPlan.transformExpressions {
-            case a: Attribute =>
-              dedupAttr(a, attributeRewrites)
-            case s: SubqueryExpression =>
-              s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attributeRewrites))
-          } -> attrMapping
-        }
-      }
-    }
-
-    private def dedupAttr(attr: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
-      val exprId = attrMap.getOrElse(attr, attr).exprId
-      attr.withExprId(exprId)
-    }
-
-    /**
-     * The outer plan may have been de-duplicated and the function below updates the
-     * outer references to refer to the de-duplicated attributes.
-     *
-     * For example (SQL):
-     * {{{
-     *   SELECT * FROM t1
-     *   INTERSECT
-     *   SELECT * FROM t1
-     *   WHERE EXISTS (SELECT 1
-     *                 FROM t2
-     *                 WHERE t1.c1 = t2.c1)
-     * }}}
-     * Plan before resolveReference rule.
-     *    'Intersect
-     *    :- Project [c1#245, c2#246]
-     *    :  +- SubqueryAlias t1
-     *    :     +- Relation[c1#245,c2#246] parquet
-     *    +- 'Project [*]
-     *       +- Filter exists#257 [c1#245]
-     *       :  +- Project [1 AS 1#258]
-     *       :     +- Filter (outer(c1#245) = c1#251)
-     *       :        +- SubqueryAlias t2
-     *       :           +- Relation[c1#251,c2#252] parquet
-     *       +- SubqueryAlias t1
-     *          +- Relation[c1#245,c2#246] parquet
-     * Plan after the resolveReference rule.
-     *    Intersect
-     *    :- Project [c1#245, c2#246]
-     *    :  +- SubqueryAlias t1
-     *    :     +- Relation[c1#245,c2#246] parquet
-     *    +- Project [c1#259, c2#260]
-     *       +- Filter exists#257 [c1#259]
-     *       :  +- Project [1 AS 1#258]
-     *       :     +- Filter (outer(c1#259) = c1#251) => Updated
-     *       :        +- SubqueryAlias t2
-     *       :           +- Relation[c1#251,c2#252] parquet
-     *       +- SubqueryAlias t1
-     *          +- Relation[c1#259,c2#260] parquet  => Outer plan's attributes are de-duplicated.
-     */
-    private def dedupOuterReferencesInSubquery(
-        plan: LogicalPlan,
-        attrMap: AttributeMap[Attribute]): LogicalPlan = {
-      plan transformDown { case currentFragment =>
-        currentFragment transformExpressions {
-          case OuterReference(a: Attribute) =>
-            OuterReference(dedupAttr(a, attrMap))
-          case s: SubqueryExpression =>
-            s.withNewPlan(dedupOuterReferencesInSubquery(s.plan, attrMap))
+        val planMapping = conflictPlans.toMap
+        right.transformUpWithNewOutput {
+          case oldPlan =>
+            val newPlanOpt = planMapping.get(oldPlan)
+            newPlanOpt.map { newPlan =>
+              newPlan -> oldPlan.output.zip(newPlan.output)
+            }.getOrElse(oldPlan -> Nil)
         }
       }
     }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
index a6f8e12849ccb..df8835dd915ad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
@@ -326,25 +326,42 @@ object TypeCoercion {
    *
    * This rule is only applied to Union/Except/Intersect
    */
-  object WidenSetOperationTypes extends Rule[LogicalPlan] {
-
-    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperatorsUp {
-      case s @ Except(left, right, isAll) if s.childrenResolved &&
-        left.output.length == right.output.length && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
-        assert(newChildren.length == 2)
-        Except(newChildren.head, newChildren.last, isAll)
-
-      case s @ Intersect(left, right, isAll) if s.childrenResolved &&
-        left.output.length == right.output.length && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
-        assert(newChildren.length == 2)
-        Intersect(newChildren.head, newChildren.last, isAll)
-
-      case s: Union if s.childrenResolved &&
+  object WidenSetOperationTypes extends TypeCoercionRule {
+
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = {
+      plan resolveOperatorsUpWithNewOutput {
+        case s @ Except(left, right, isAll) if s.childrenResolved &&
+          left.output.length == right.output.length && !s.resolved =>
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            assert(newChildren.length == 2)
+            val attrMapping = left.output.zip(newChildren.head.output)
+            Except(newChildren.head, newChildren.last, isAll) -> attrMapping
+          }
+
+        case s @ Intersect(left, right, isAll) if s.childrenResolved &&
+          left.output.length == right.output.length && !s.resolved =>
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(left :: right :: Nil)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            assert(newChildren.length == 2)
+            val attrMapping = left.output.zip(newChildren.head.output)
+            Intersect(newChildren.head, newChildren.last, isAll) -> attrMapping
+          }
+
+        case s: Union if s.childrenResolved &&
           s.children.forall(_.output.length == s.children.head.output.length) && !s.resolved =>
-        val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
-        s.makeCopy(Array(newChildren))
+          val newChildren: Seq[LogicalPlan] = buildNewChildrenWithWiderTypes(s.children)
+          if (newChildren.isEmpty) {
+            s -> Nil
+          } else {
+            val attrMapping = s.children.head.output.zip(newChildren.head.output)
+            s.copy(children = newChildren) -> attrMapping
+          }
+      }
     }
 
     /** Build new children with the widest types for each attribute among all the children */
@@ -360,8 +377,7 @@ object TypeCoercion {
         // Add an extra Project if the targetTypes are different from the original types.
         children.map(widenTypes(_, targetTypes))
       } else {
-        // Unable to find a target type to widen, then just return the original set.
-        children
+        Nil
       }
     }
 
@@ -387,7 +403,8 @@ object TypeCoercion {
     /** Given a plan, add an extra project on top to widen some columns' data types. */
     private def widenTypes(plan: LogicalPlan, targetTypes: Seq[DataType]): LogicalPlan = {
       val casted = plan.output.zip(targetTypes).map {
-        case (e, dt) if e.dataType != dt => Alias(Cast(e, dt), e.name)()
+        case (e, dt) if e.dataType != dt =>
+          Alias(Cast(e, dt, Some(SQLConf.get.sessionLocalTimeZone)), e.name)()
         case (e, _) => e
       }
       Project(casted, plan)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index 7133fb231d672..fed5df69580ee 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -17,6 +17,8 @@
 
 package org.apache.spark.sql.catalyst.plans
 
+import scala.collection.mutable
+
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode, TreeNodeTag}
@@ -168,6 +170,89 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
     }.toSeq
   }
 
+  /**
+   * A variant of `transformUp`, which takes care of the case that the rule replaces a plan node
+   * with a new one that has different output expr IDs, by updating the attribute references in
+   * the parent nodes accordingly.
+   *
+   * @param rule the function to transform plan nodes, and return new nodes with attributes mapping
+   *             from old attributes to new attributes. The attribute mapping will be used to
+   *             rewrite attribute references in the parent nodes.
+   * @param skipCond a boolean condition to indicate if we can skip transforming a plan node to save
+   *                 time.
+   */
+  def transformUpWithNewOutput(
+      rule: PartialFunction[PlanType, (PlanType, Seq[(Attribute, Attribute)])],
+      skipCond: PlanType => Boolean = _ => false): PlanType = {
+    def rewrite(plan: PlanType): (PlanType, Seq[(Attribute, Attribute)]) = {
+      if (skipCond(plan)) {
+        plan -> Nil
+      } else {
+        val attrMapping = new mutable.ArrayBuffer[(Attribute, Attribute)]()
+        var newPlan = plan.mapChildren { child =>
+          val (newChild, childAttrMapping) = rewrite(child)
+          attrMapping ++= childAttrMapping
+          newChild
+        }
+
+        val attrMappingForCurrentPlan = attrMapping.filter {
+          // The `attrMappingForCurrentPlan` is used to replace the attributes of the
+          // current `plan`, so the `oldAttr` must be part of `plan.references`.
+          case (oldAttr, _) => plan.references.contains(oldAttr)
+        }
+
+        val (planAfterRule, newAttrMapping) = CurrentOrigin.withOrigin(origin) {
+          rule.applyOrElse(newPlan, (plan: PlanType) => plan -> Nil)
+        }
+        newPlan = planAfterRule
+
+        if (attrMappingForCurrentPlan.nonEmpty) {
+          assert(!attrMappingForCurrentPlan.groupBy(_._1.exprId)
+            .exists(_._2.map(_._2.exprId).distinct.length > 1),
+            "Found duplicate rewrite attributes")
+
+          val attributeRewrites = AttributeMap(attrMappingForCurrentPlan)
+          // Using attrMapping from the children plans to rewrite their parent node.
+          // Note that we shouldn't rewrite a node using attrMapping from its sibling nodes.
+          newPlan = newPlan.transformExpressions {
+            case a: AttributeReference =>
+              updateAttr(a, attributeRewrites)
+            case pe: PlanExpression[PlanType] =>
+              pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attributeRewrites))
+          }
+        }
+
+        attrMapping ++= newAttrMapping.filter {
+          case (a1, a2) => a1.exprId != a2.exprId
+        }
+        newPlan -> attrMapping
+      }
+    }
+    rewrite(this)._1
+  }
+
+  private def updateAttr(attr: Attribute, attrMap: AttributeMap[Attribute]): Attribute = {
+    val exprId = attrMap.getOrElse(attr, attr).exprId
+    attr.withExprId(exprId)
+  }
+
+  /**
+   * The outer plan may have old references and the function below updates the
+   * outer references to refer to the new attributes.
+   */
+  private def updateOuterReferencesInSubquery(
+      plan: PlanType,
+      attrMap: AttributeMap[Attribute]): PlanType = {
+    plan.transformDown { case currentFragment =>
+      currentFragment.transformExpressions {
+        case OuterReference(a: AttributeReference) =>
+          OuterReference(updateAttr(a, attrMap))
+        case pe: PlanExpression[PlanType] =>
+          pe.withNewPlan(updateOuterReferencesInSubquery(pe.plan, attrMap))
+      }
+    }
+  }
+
   lazy val schema: StructType = StructType.fromAttributes(output)
 
   /** Returns the output schema in the tree format. */
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
index 9404a809b453c..30447db1acc04 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.catalyst.plans.logical
 
 import org.apache.spark.sql.catalyst.analysis.CheckAnalysis
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.plans.QueryPlan
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, TreeNode}
 import org.apache.spark.util.Utils
@@ -120,6 +120,19 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
     }
   }
 
+  /**
+   * A variant of `transformUpWithNewOutput`, which skips touching already analyzed plan.
+   */
+  def resolveOperatorsUpWithNewOutput(
+      rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])])
+  : LogicalPlan = {
+    if (!analyzed) {
+      transformUpWithNewOutput(rule, skipCond = _.analyzed)
+    } else {
+      self
+    }
+  }
+
   /**
    * Recursively transforms the expressions of a tree, skipping nodes that have already
    * been analyzed.
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
index 1ea1ddb8bbd08..7b80de908fa08 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
@@ -21,13 +21,12 @@ import java.sql.Timestamp
 
 import org.apache.spark.sql.catalyst.analysis.TypeCoercion._
 import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.plans.PlanTest
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.CalendarInterval
 
 class TypeCoercionSuite extends AnalysisTest {
   import TypeCoercionSuite._
@@ -1417,6 +1416,21 @@ class TypeCoercionSuite extends AnalysisTest {
     }
   }
 
+  test("SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes") {
+    val t1 = LocalRelation(AttributeReference("v", DecimalType(10, 0))())
+    val t2 = LocalRelation(AttributeReference("v", DecimalType(11, 0))())
+    val p1 = t1.select(t1.output.head).as("p1")
+    val p2 = t2.select(t2.output.head).as("p2")
+    val union = p1.union(p2)
+    val wp1 = widenSetOperationTypes(union.select(p1.output.head, $"p2.v"))
+    assert(wp1.isInstanceOf[Project])
+    // The attribute `p1.output.head` should be replaced in the root `Project`.
+    assert(wp1.expressions.forall(_.find(_ == p1.output.head).isEmpty))
+    val wp2 = widenSetOperationTypes(Aggregate(Nil, sum(p1.output.head).as("v") :: Nil, union))
+    assert(wp2.isInstanceOf[Aggregate])
+    assert(wp2.missingInput.isEmpty)
+  }
+
   /**
    * There are rules that need to not fire before child expressions get resolved.
    * We use this test to make sure those rules do not fire early.
diff --git a/sql/core/src/test/resources/sql-tests/inputs/except.sql b/sql/core/src/test/resources/sql-tests/inputs/except.sql
index 1d579e65f3473..ffdf1f4f3d24d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/except.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/except.sql
@@ -55,3 +55,22 @@ FROM   t1
 WHERE  t1.v >= (SELECT   min(t2.v)
                 FROM     t2
                 WHERE    t2.k = t1.k);
+
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
+SELECT t.v FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t;
+
+-- Clean-up
+DROP VIEW IF EXISTS t1;
+DROP VIEW IF EXISTS t2;
+DROP VIEW IF EXISTS t3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
index b0b2244048caa..077caa5dd44a0 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
@@ -155,6 +155,21 @@ SELECT * FROM tab2;
 -- Restore the property
 SET spark.sql.legacy.setopsPrecedence.enabled = false;
 
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW tab3 AS VALUES (decimal(1)), (decimal(2)) tbl3(v);
+SELECT t.v FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t;
+
 -- Clean-up 
 DROP VIEW IF EXISTS tab1;
 DROP VIEW IF EXISTS tab2;
+DROP VIEW IF EXISTS tab3;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/union.sql b/sql/core/src/test/resources/sql-tests/inputs/union.sql
index 6da1b9b49b226..8a5b6c50fc1e3 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/union.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/union.sql
@@ -45,10 +45,24 @@ SELECT array(1, 2), 'str'
 UNION ALL
 SELECT array(1, 2, 3, NULL), 1;
 
+-- SPARK-32638: corrects references when adding aliases in WidenSetOperationTypes
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v);
+SELECT t.v FROM (
+  SELECT v FROM t3
+  UNION ALL
+  SELECT v + v AS v FROM t3
+) t;
+
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  UNION
+  SELECT v + v AS v FROM t3
+) t;
 
 -- Clean-up
 DROP VIEW IF EXISTS t1;
 DROP VIEW IF EXISTS t2;
+DROP VIEW IF EXISTS t3;
 DROP VIEW IF EXISTS p1;
 DROP VIEW IF EXISTS p2;
 DROP VIEW IF EXISTS p3;
diff --git a/sql/core/src/test/resources/sql-tests/results/except.sql.out b/sql/core/src/test/resources/sql-tests/results/except.sql.out
index 62d695219d01d..061b122eac7cf 100644
--- a/sql/core/src/test/resources/sql-tests/results/except.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/except.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 9
+-- Number of queries: 15
 
 
 -- !query
@@ -103,3 +103,59 @@ WHERE  t1.v >= (SELECT   min(t2.v)
 struct<k:string>
 -- !query output
 two
+
+
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+1
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  EXCEPT
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+1
+
+
+-- !query
+DROP VIEW IF EXISTS t1
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t2
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+DROP VIEW IF EXISTS t3
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
index 4762082dc3be2..b99f63393cc4d 100644
--- a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 22
+-- Number of queries: 26
 
 
 -- !query
@@ -291,6 +291,38 @@ struct<key:string,value:string>
 spark.sql.legacy.setopsPrecedence.enabled	false
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW tab3 AS VALUES (decimal(1)), (decimal(2)) tbl3(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+2
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM tab3
+  INTERSECT
+  SELECT v + v AS v FROM tab3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+2
+
+
 -- !query
 DROP VIEW IF EXISTS tab1
 -- !query schema
@@ -305,3 +337,11 @@ DROP VIEW IF EXISTS tab2
 struct<>
 -- !query output
 
+
+
+-- !query
+DROP VIEW IF EXISTS tab3
+-- !query schema
+struct<>
+-- !query output
+
diff --git a/sql/core/src/test/resources/sql-tests/results/union.sql.out b/sql/core/src/test/resources/sql-tests/results/union.sql.out
index 44002406836a4..ce3c761bc5d2d 100644
--- a/sql/core/src/test/resources/sql-tests/results/union.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/union.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 16
+-- Number of queries: 20
 
 
 -- !query
@@ -126,6 +126,39 @@ struct<array(1, 2):array<int>,str:string>
 [1,2]	str
 
 
+-- !query
+CREATE OR REPLACE TEMPORARY VIEW t3 AS VALUES (decimal(1)) tbl(v)
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+SELECT t.v FROM (
+  SELECT v FROM t3
+  UNION ALL
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<v:decimal(11,0)>
+-- !query output
+1
+2
+
+
+-- !query
+SELECT SUM(t.v) FROM (
+  SELECT v FROM t3
+  UNION
+  SELECT v + v AS v FROM t3
+) t
+-- !query schema
+struct<sum(v):decimal(21,0)>
+-- !query output
+3
+
+
 -- !query
 DROP VIEW IF EXISTS t1
 -- !query schema
@@ -142,6 +175,14 @@ struct<>
 
 
 
+-- !query
+DROP VIEW IF EXISTS t3
+-- !query schema
+struct<>
+-- !query output
+
+
+
 -- !query
 DROP VIEW IF EXISTS p1
 -- !query schema

From e86d90b21d4b3d6658b3cb6dd30daafb32b0c1bd Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Wed, 9 Sep 2020 10:28:40 +0900
Subject: [PATCH 1035/1280] [SPARK-32824][CORE] Improve the error message when
 the user forgets the .amount in a resource config

### What changes were proposed in this pull request?

If the user forgets to specify .amount on a resource config like spark.executor.resource.gpu, the error message thrown is very confusing:

```
ERROR SparkContext: Error initializing SparkContext.java.lang.StringIndexOutOfBoundsException: String index out of range:
-1 at java.lang.String.substring(String.java:1967) at
```

This makes it so we have a readable error thrown

### Why are the changes needed?

confusing error for users
### Does this PR introduce _any_ user-facing change?

just error message

### How was this patch tested?

Tested manually on standalone cluster

Closes #29685 from tgravescs/SPARK-32824.

Authored-by: Thomas Graves <tgraves@nvidia.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e8634d8f6f8548852a284a32c1b7da24bedd8ff7)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/resource/ResourceUtils.scala    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 994b36386330c..16fe897827c33 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -148,7 +148,12 @@ private[spark] object ResourceUtils extends Logging {
 
   def listResourceIds(sparkConf: SparkConf, componentName: String): Seq[ResourceID] = {
     sparkConf.getAllWithPrefix(s"$componentName.$RESOURCE_PREFIX.").map { case (key, _) =>
-      key.substring(0, key.indexOf('.'))
+      val index = key.indexOf('.')
+      if (index < 0) {
+        throw new SparkException(s"You must specify an amount config for resource: $key " +
+          s"config: $componentName.$RESOURCE_PREFIX.$key")
+      }
+      key.substring(0, index)
     }.distinct.map(name => new ResourceID(componentName, name))
   }
 

From 86b9dd9b414cab9489f00ef7acec5c3264fa312d Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Wed, 9 Sep 2020 10:33:21 +0900
Subject: [PATCH 1036/1280] [SPARK-32823][WEB UI] Fix the master ui resources
 reporting

### What changes were proposed in this pull request?

Fixes the master UI for properly summing the resources total across multiple workers.
field:
Resources in use: 0 / 8 gpu

The bug here is that it was creating MutableResourceInfo and then reducing using the + operator.  the + operator in MutableResourceInfo simple adds the address from one to the addresses of the other.  But its using a HashSet so if the addresses are the same then you lose the correct amount.  ie worker1 has gpu addresses 0,1,2,3 and worker2 has addresses 0,1,2,3 then you only see 4 total GPUs when there are 8.

In this case we don't really need to create the MutableResourceInfo at all because we just want the sums for used and total so just remove the use of it.  The other uses of it are per Worker so those should be ok.

### Why are the changes needed?

fix UI

### Does this PR introduce _any_ user-facing change?

UI

### How was this patch tested?

tested manually on standalone cluster with multiple workers and multiple GPUs and multiple fpgas

Closes #29683 from tgravescs/SPARK-32823.

Lead-authored-by: Thomas Graves <tgraves@nvidia.com>
Co-authored-by: Thomas Graves <tgraves@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 514bf563a7fa1f470b2ab088c0838317500a9aab)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/deploy/StandaloneResourceUtils.scala  | 10 +++++-----
 .../org/apache/spark/deploy/master/ui/MasterPage.scala | 10 ++++------
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
index e08709ec36ab0..c7c31a85b0636 100644
--- a/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/StandaloneResourceUtils.scala
@@ -149,11 +149,11 @@ private[spark] object StandaloneResourceUtils extends Logging {
 
   // used for UI
   def formatResourcesUsed(
-      resourcesTotal: Map[String, ResourceInformation],
-      resourcesUsed: Map[String, ResourceInformation]): String = {
-    resourcesTotal.map { case (rName, rInfo) =>
-      val used = resourcesUsed(rName).addresses.length
-      val total = rInfo.addresses.length
+      resourcesTotal: Map[String, Int],
+      resourcesUsed: Map[String, Int]): String = {
+    resourcesTotal.map { case (rName, totalSize) =>
+      val used = resourcesUsed(rName)
+      val total = totalSize
       s"$used / $total $rName"
     }.mkString(", ")
   }
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index f64b449851d86..fcbeba9eef1a9 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -76,19 +76,17 @@ private[ui] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
 
   private def formatMasterResourcesInUse(aliveWorkers: Array[WorkerInfo]): String = {
     val totalInfo = aliveWorkers.map(_.resourcesInfo)
-      .map(resources => toMutable(resources))
       .flatMap(_.toIterator)
       .groupBy(_._1) // group by resource name
       .map { case (rName, rInfoArr) =>
-        rName -> rInfoArr.map(_._2).reduce(_ + _)
-      }.map { case (k, v) => (k, v.toResourceInformation) }
+      rName -> rInfoArr.map(_._2.addresses.size).sum
+    }
     val usedInfo = aliveWorkers.map(_.resourcesInfoUsed)
-      .map (resources => toMutable(resources))
       .flatMap(_.toIterator)
       .groupBy(_._1) // group by resource name
       .map { case (rName, rInfoArr) =>
-      rName -> rInfoArr.map(_._2).reduce(_ + _)
-    }.map { case (k, v) => (k, v.toResourceInformation) }
+      rName -> rInfoArr.map(_._2.addresses.size).sum
+    }
     formatResourcesUsed(totalInfo, usedInfo)
   }
 

From 4c0f9d8b44f63a3d1faaeece8b1d6b47c3bfe75f Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Wed, 9 Sep 2020 12:23:05 +0900
Subject: [PATCH 1037/1280] [SPARK-32813][SQL] Get default config of
 ParquetSource vectorized reader if no active SparkSession

### What changes were proposed in this pull request?

If no active SparkSession is available, let `FileSourceScanExec.needsUnsafeRowConversion` look at default SQL config of ParquetSource vectorized reader instead of failing the query execution.

### Why are the changes needed?

Fix a bug that if no active SparkSession is available, file-based data source scan for Parquet Source will throw exception.

### Does this PR introduce _any_ user-facing change?

Yes, this change fixes the bug.

### How was this patch tested?

Unit test.

Closes #29667 from viirya/SPARK-32813.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit de0dc52a842bf4374c1ae4f9546dd95b3f35c4f1)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/execution/DataSourceScanExec.scala    |  2 +-
 .../sql/execution/SQLExecutionSuite.scala     | 40 ++++++++++++++++++-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 447e0a689feb5..0fcb0dd1f658e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -175,7 +175,7 @@ case class FileSourceScanExec(
 
   private lazy val needsUnsafeRowConversion: Boolean = {
     if (relation.fileFormat.isInstanceOf[ParquetSource]) {
-      SparkSession.getActiveSession.get.sessionState.conf.parquetVectorizedReaderEnabled
+      sqlContext.conf.parquetVectorizedReaderEnabled
     } else {
       false
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
index 8bf7fe62cd49b..81e692076b432 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLExecutionSuite.scala
@@ -17,11 +17,17 @@
 
 package org.apache.spark.sql.execution
 
+import java.util.concurrent.Executors
+
 import scala.collection.parallel.immutable.ParRange
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
 
 import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite}
 import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{Row, SparkSession}
+import org.apache.spark.sql.types._
+import org.apache.spark.util.ThreadUtils
 
 class SQLExecutionSuite extends SparkFunSuite {
 
@@ -119,6 +125,38 @@ class SQLExecutionSuite extends SparkFunSuite {
 
     spark.stop()
   }
+
+  test("SPARK-32813: Table scan should work in different thread") {
+    val executor1 = Executors.newSingleThreadExecutor()
+    val executor2 = Executors.newSingleThreadExecutor()
+    var session: SparkSession = null
+    SparkSession.cleanupAnyExistingSession()
+
+    withTempDir { tempDir =>
+      try {
+        val tablePath = tempDir.toString + "/table"
+        val df = ThreadUtils.awaitResult(Future {
+          session = SparkSession.builder().appName("test").master("local[*]").getOrCreate()
+
+          session.createDataFrame(
+            session.sparkContext.parallelize(Row(Array(1, 2, 3)) :: Nil),
+            StructType(Seq(
+              StructField("a", ArrayType(IntegerType, containsNull = false), nullable = false))))
+            .write.parquet(tablePath)
+
+          session.read.parquet(tablePath)
+        }(ExecutionContext.fromExecutorService(executor1)), 1.minute)
+
+        ThreadUtils.awaitResult(Future {
+          assert(df.rdd.collect()(0) === Row(Seq(1, 2, 3)))
+        }(ExecutionContext.fromExecutorService(executor2)), 1.minute)
+      } finally {
+        executor1.shutdown()
+        executor2.shutdown()
+        session.stop()
+      }
+    }
+  }
 }
 
 object SQLExecutionSuite {

From 837843bea40eec842c782e3c719f8d81024d8a06 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 9 Sep 2020 21:16:16 +0900
Subject: [PATCH 1038/1280] [SPARK-32810][SQL][TESTS][FOLLOWUP][3.0] Check path
 globbing in JSON/CSV datasources v1 and v2

### What changes were proposed in this pull request?
In the PR, I propose to move the test `SPARK-32810: CSV and JSON data sources should be able to read files with escaped glob metacharacter in the paths` from `DataFrameReaderWriterSuite` to `CSVSuite` and to `JsonSuite`. This will allow to run the same test in `CSVv1Suite`/`CSVv2Suite` and in `JsonV1Suite`/`JsonV2Suite`.

### Why are the changes needed?
To improve test coverage by checking JSON/CSV datasources v1 and v2.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running affected test suites:
```
$ build/sbt "sql/test:testOnly org.apache.spark.sql.execution.datasources.csv.*"
$ build/sbt "sql/test:testOnly org.apache.spark.sql.execution.datasources.json.*"
```

Closes #29690 from MaxGekk/globbing-paths-when-inferring-schema-dsv2-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../execution/datasources/csv/CSVSuite.scala  | 13 +++++++++++
 .../datasources/json/JsonSuite.scala          | 13 +++++++++++
 .../sql/test/DataFrameReaderWriterSuite.scala | 23 -------------------
 3 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 9ba2cab3fe31c..4e93ea3e8e42f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2365,6 +2365,19 @@ abstract class CSVSuite extends QueryTest with SharedSparkSession with TestCsvDa
       assert(df.count() == 3)
     }
   }
+
+  test("SPARK-32810: CSV data source should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test CSV writer / reader without specifying schema
+      val csvTableName = "[abc]"
+      spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
+      val readback = spark.read
+        .csv(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(csvTableName, """\\$1""")}")
+      assert(readback.collect sameElements Array(Row("0"), Row("1"), Row("2")))
+    }
+  }
 }
 
 class CSVv1Suite extends CSVSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 38b5e775dffbf..8eb5432f8b61a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2706,6 +2706,19 @@ abstract class JsonSuite extends QueryTest with SharedSparkSession with TestJson
       checkAnswer(json, Row(null))
     }
   }
+
+  test("SPARK-32810: JSON data source should be able to read files with " +
+    "escaped glob metacharacter in the paths") {
+    withTempDir { dir =>
+      val basePath = dir.getCanonicalPath
+      // test JSON writer / reader without specifying schema
+      val jsonTableName = "{def}"
+      spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
+      val readback = spark.read
+        .json(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(jsonTableName, """\\$1""")}")
+      assert(readback.collect sameElements Array(Row(0), Row(1), Row(2)))
+    }
+  }
 }
 
 class JsonV1Suite extends JsonSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index f48f445a30bad..c7ca0125c8a0f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -1105,27 +1105,4 @@ class DataFrameReaderWriterSuite extends QueryTest with SharedSparkSession with
       }
     }
   }
-
-  test("SPARK-32810: CSV and JSON data sources should be able to read files with " +
-    "escaped glob metacharacter in the paths") {
-    def escape(str: String): String = {
-      """(\[|\]|\{|\})""".r.replaceAllIn(str, """\\$1""")
-    }
-
-    withTempDir { dir =>
-      val basePath = dir.getCanonicalPath
-
-      // test CSV writer / reader without specifying schema
-      val csvTableName = "[abc]"
-      spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
-      val csvDf = spark.read.csv(s"$basePath/${escape(csvTableName)}")
-      assert(csvDf.collect sameElements Array(Row("0"), Row("1"), Row("2")))
-
-      // test JSON writer / reader without specifying schema
-      val jsonTableName = "{def}"
-      spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
-      val jsonDf = spark.read.json(s"$basePath/${escape(jsonTableName)}")
-      assert(jsonDf.collect sameElements Array(Row(0), Row(1), Row(2)))
-    }
-  }
 }

From e632e7c87498de6b1f1ecd31a568ac0f4505a6a5 Mon Sep 17 00:00:00 2001
From: Tathagata Das <tathagata.das1565@gmail.com>
Date: Wed, 9 Sep 2020 18:31:50 -0400
Subject: [PATCH 1039/1280] [SPARK-32794][SS] Fixed rare corner case error in
 micro-batch engine with some stateful queries + no-data-batches + V1 sources

### What changes were proposed in this pull request?

Make MicroBatchExecution explicitly call `getBatch` when the start and end offsets are the same.

### Why are the changes needed?

Structured Streaming micro-batch engine has the contract with V1 data sources that, after a restart, it will call `source.getBatch()` on the last batch attempted before the restart. However, a very rare combination of sequences violates this contract. It occurs only when
- The streaming query has specific types of stateful operations with watermarks (e.g., aggregation in append, mapGroupsWithState with timeouts).
    - These queries can execute a batch even without new data when the previous updates the watermark and the stateful ops are such that the new watermark can cause new output/cleanup. Such batches are called no-data-batches.
- The last batch before termination was an incomplete no-data-batch. Upon restart, the micro-batch engine fails to call `source.getBatch` when attempting to re-execute the incomplete no-data-batch.

This occurs because no-data-batches has the same and end offsets, and when a batch is executed, if the start and end offset is same then calling `source.getBatch` is skipped as it is assumed the generated plan will be empty. This only affects V1 data sources which rely on this invariant to detect in the source whether the query is being started from scratch or restarted.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

New unit test with a mock v1 source that fails without the fix.

Closes #29696 from tdas/SPARK-32794-3.0.

Authored-by: Tathagata Das <tathagata.das1565@gmail.com>
Signed-off-by: Tathagata Das <tathagata.das1565@gmail.com>
---
 .../streaming/MicroBatchExecution.scala       |  11 ++
 .../streaming/MicroBatchExecutionSuite.scala  | 123 ++++++++++++++++++
 .../spark/sql/streaming/StreamTest.scala      |   8 ++
 3 files changed, 142 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
index e0731db1f3c18..f85cfbcb33d83 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecution.scala
@@ -318,6 +318,17 @@ class MicroBatchExecution(
               committedOffsets ++= availableOffsets
               watermarkTracker.setWatermark(
                 math.max(watermarkTracker.currentWatermark, commitMetadata.nextBatchWatermarkMs))
+            } else if (latestCommittedBatchId == latestBatchId - 1) {
+              availableOffsets.foreach {
+                case (source: Source, end: Offset) =>
+                  val start = committedOffsets.get(source).map(_.asInstanceOf[Offset])
+                  if (start.map(_ == end).getOrElse(true)) {
+                    source.getBatch(start, end)
+                  }
+                case nonV1Tuple =>
+                  // The V2 API does not have the same edge case requiring getBatch to be called
+                  // here, so we do nothing here.
+              }
             } else if (latestCommittedBatchId < latestBatchId - 1) {
               logWarning(s"Batch completion log latest batch id is " +
                 s"${latestCommittedBatchId}, which is not trailing " +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
index c228740df07c8..60b8c9627844b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/MicroBatchExecutionSuite.scala
@@ -19,8 +19,13 @@ package org.apache.spark.sql.execution.streaming
 
 import org.scalatest.BeforeAndAfter
 
+import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.plans.logical.Range
+import org.apache.spark.sql.connector.read.streaming
+import org.apache.spark.sql.connector.read.streaming.SparkDataStream
 import org.apache.spark.sql.functions.{count, window}
 import org.apache.spark.sql.streaming.StreamTest
+import org.apache.spark.sql.types.{LongType, StructType}
 
 class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
 
@@ -68,4 +73,122 @@ class MicroBatchExecutionSuite extends StreamTest with BeforeAndAfter {
       CheckNewAnswer((25, 1), (30, 1))   // This should not throw the error reported in SPARK-24156
     )
   }
+
+  test("no-data-batch re-executed after restart should call V1 source.getBatch()") {
+    val testSource = ReExecutedBatchTestSource(spark)
+    val df = testSource.toDF()
+      .withColumn("eventTime", $"value".cast("timestamp"))
+      .withWatermark("eventTime", "10 seconds")
+      .groupBy(window($"eventTime", "5 seconds") as 'window)
+      .agg(count("*") as 'count)
+      .select($"window".getField("start").cast("long").as[Long])
+
+    /** Reset this test source so that it appears to be a new source requiring initialization */
+    def resetSource(): StreamAction = Execute("reset source") { _ =>
+      testSource.reset()  // Make it look like a new source that needs to be re-initialized
+      require(testSource.currentOffset === 0)
+      require(testSource.getBatchCallCount === 0)
+    }
+
+    /** Add data to this test source by incrementing its available offset */
+    def addData(numNewRows: Int): StreamAction = new AddData {
+      override def addData(query: Option[StreamExecution]): (SparkDataStream, streaming.Offset) = {
+        testSource.incrementAvailableOffset(numNewRows)
+        (testSource, testSource.getOffset.get)
+      }
+    }
+
+    testStream(df)(
+      addData(numNewRows = 10),   // generate values 1...10, sets watermark to 0
+      CheckAnswer(),
+      addData(numNewRows = 10),   // generate values 11...20, sets watermark to 10
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5),          // start time of windows closed and outputted
+      Execute("verify source internal state before stop") { q =>
+        // Last batch should be a no-data batch
+        require(q.lastProgress.numInputRows === 0)
+        // Source should have expected internal state
+        require(testSource.currentOffset === 20)
+        // getBatch should be called only for 2 batches with data, not for no-data-batches
+        assert(testSource.getBatchCallCount === 2)
+      },
+      StopStream,
+
+      /* Verify that if the last no-data-batch was incomplete, getBatch() is called only once */
+      Execute("mark last batch as incomplete") { q =>
+        // Delete the last committed batch from the commit log to signify that the last batch
+        // (a no-data batch) did not complete and has to be re-executed on restart.
+        val commit = q.commitLog.getLatest().map(_._1).getOrElse(-1L)
+        q.commitLog.purgeAfter(commit - 1)
+      },
+      resetSource(),
+      StartStream(),
+      ProcessAllAvailable(),  // allow initialization and re-execution
+      Execute("verify source.getBatch() called after re-executed no-data-batch") { q =>
+        // After restart, getBatch() should be called once even for no-data batch
+        assert(testSource.getBatchCallCount === 1)
+        assert(testSource.currentOffset === 20)
+      },
+      addData(numNewRows = 10),   // generate values 21...30, sets watermark to 20
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5, 10, 15),
+      StopStream,
+
+      /* Verify that if the last no-data-batch was complete, getBatch() is still called only once */
+      Execute("verify last batch was complete") { q =>
+        // Verify that the commit log records the last batch as completed
+        require(q.commitLog.getLatest().map(_._1).get === q.offsetLog.getLatest().map(_._1).get)
+      },
+      resetSource(),
+      StartStream(),
+      ProcessAllAvailable(),      // allow initialization to completed
+      Execute("verify source.getBatch() called even if no-data-batch was not re-executed") { q =>
+        // After restart, getBatch() should be called even for no-data batch, but only once
+        assert(testSource.getBatchCallCount === 1)
+        assert(testSource.currentOffset === 30)
+      },
+      addData(numNewRows = 10),   // generate values 31...40, sets watermark to 30
+      ProcessAllAvailable(),      // let no-data-batch be executed
+      CheckAnswer(0, 5, 10, 15, 20, 25)
+    )
+  }
+
+
+  case class ReExecutedBatchTestSource(spark: SparkSession) extends Source {
+    @volatile var currentOffset = 0L
+    @volatile var getBatchCallCount = 0
+
+    override def getOffset: Option[Offset] = {
+      if (currentOffset <= 0) None else Some(LongOffset(currentOffset))
+    }
+
+    override def getBatch(start: Option[Offset], end: Offset): DataFrame = {
+      getBatchCallCount = getBatchCallCount + 1
+      if (currentOffset == 0) currentOffset = getOffsetValue(end)
+      val plan = Range(
+        start.map(getOffsetValue).getOrElse(0L) + 1L, getOffsetValue(end) + 1L, 1, None,
+        isStreaming = true)
+      Dataset.ofRows(spark, plan)
+    }
+
+    def incrementAvailableOffset(numNewRows: Int): Unit = {
+      currentOffset = currentOffset + numNewRows
+    }
+
+    def reset(): Unit = {
+      currentOffset = 0L
+      getBatchCallCount = 0
+    }
+    def toDF(): DataFrame = Dataset.ofRows(spark, StreamingExecutionRelation(this, spark))
+    override def schema: StructType = new StructType().add("value", LongType)
+    override def stop(): Unit = {}
+    private def getOffsetValue(offset: Offset): Long = {
+      offset match {
+        case s: SerializedOffset => LongOffset(s).offset
+        case l: LongOffset => l.offset
+        case _ => throw new IllegalArgumentException("incorrect offset type: " + offset)
+      }
+    }
+  }
 }
+
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
index 8d5439534b513..93bfd64455408 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamTest.scala
@@ -305,6 +305,14 @@ trait StreamTest extends QueryTest with SharedSparkSession with TimeLimits with
     def apply(func: StreamExecution => Any): AssertOnQuery = apply("Execute")(func)
   }
 
+  /** Call `StreamingQuery.processAllAvailable()` to wait. */
+  object ProcessAllAvailable {
+    def apply(): AssertOnQuery = AssertOnQuery { query =>
+      query.processAllAvailable()
+      true
+    }
+  }
+
   object AwaitEpoch {
     def apply(epoch: Long): AssertOnQuery =
       Execute {

From 5a81f600bdad7ea93e7c3116c08b3786658887b3 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 9 Sep 2020 19:46:55 -0700
Subject: [PATCH 1040/1280] [SPARK-32836][SS][TESTS] Fix
 DataStreamReaderWriterSuite to check writer options correctly

### What changes were proposed in this pull request?

This PR aims to fix the test coverage at `DataStreamReaderWriterSuite`.

### Why are the changes needed?

Currently, the test case checks `DataStreamReader` options instead of `DataStreamWriter` options.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the revised test case.

Closes #29701 from dongjoon-hyun/SPARK-32836.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 06a994517fc3080b14f01183eeb17e56ab52eaa8)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../test/DataStreamReaderWriterSuite.scala         | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index f9fc540c2ab80..b64638751aed0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -169,18 +169,20 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
 
     LastOptions.clear()
 
-    df.writeStream
+    val query = df.writeStream
       .format("org.apache.spark.sql.streaming.test")
-      .option("opt1", "1")
-      .options(Map("opt2" -> "2"))
+      .option("opt1", "5")
+      .options(Map("opt2" -> "4"))
       .options(map)
       .option("checkpointLocation", newMetadataDir)
       .start()
-      .stop()
 
-    assert(LastOptions.parameters("opt1") == "1")
-    assert(LastOptions.parameters("opt2") == "2")
+    assert(LastOptions.parameters("opt1") == "5")
+    assert(LastOptions.parameters("opt2") == "4")
     assert(LastOptions.parameters("opt3") == "3")
+    assert(LastOptions.parameters.contains("checkpointLocation"))
+
+    query.stop()
   }
 
   test("partitioning") {

From 44acb5a03a1c3cebe0935ec6f2b2d59afbb8f7e2 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 9 Sep 2020 23:41:32 -0700
Subject: [PATCH 1041/1280] [SPARK-32832][SS] Use CaseInsensitiveMap for
 DataStreamReader/Writer options

This PR aims to fix indeterministic behavior on DataStreamReader/Writer options like the following.
```scala
scala> spark.readStream.format("parquet").option("paTh", "1").option("PATH", "2").option("Path", "3").option("patH", "4").option("path", "5").load()
org.apache.spark.sql.AnalysisException: Path does not exist: 1;
```

This will make the behavior deterministic.

Yes, but the previous behavior is indeterministic.

Pass the newly test cases.

Closes #29702 from dongjoon-hyun/SPARK-32832.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 2f85f9516cfc33a376871cf27f9fb4ac30ecbed8)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/streaming/DataStreamReader.scala      |  5 +--
 .../sql/streaming/DataStreamWriter.scala      |  5 +--
 .../test/DataStreamReaderWriterSuite.scala    | 31 +++++++++++++++++++
 3 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
index 1d7e4d3059c68..6b30949a4dc80 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala
@@ -24,6 +24,7 @@ import scala.collection.JavaConverters._
 import org.apache.spark.annotation.Evolving
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SparkSession}
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{SupportsRead, TableProvider}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -210,7 +211,7 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
       case provider: TableProvider if !provider.isInstanceOf[FileDataSourceV2] =>
         val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
           source = provider, conf = sparkSession.sessionState.conf)
-        val options = sessionOptions ++ extraOptions
+        val options = sessionOptions ++ extraOptions.toMap
         val dsOptions = new CaseInsensitiveStringMap(options.asJava)
         val table = DataSourceV2Utils.getTableFromProvider(provider, dsOptions, userSpecifiedSchema)
         import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Implicits._
@@ -520,5 +521,5 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo
 
   private var userSpecifiedSchema: Option[StructType] = None
 
-  private var extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
index 1d0ca4d9453a5..07ab4008ca63f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala
@@ -26,6 +26,7 @@ import org.apache.spark.annotation.Evolving
 import org.apache.spark.api.java.function.VoidFunction2
 import org.apache.spark.sql._
 import org.apache.spark.sql.catalyst.streaming.InternalOutputModes
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.connector.catalog.{SupportsWrite, TableProvider}
 import org.apache.spark.sql.connector.catalog.TableCapability._
 import org.apache.spark.sql.execution.command.DDLUtils
@@ -349,7 +350,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
         val provider = cls.getConstructor().newInstance().asInstanceOf[TableProvider]
         val sessionOptions = DataSourceV2Utils.extractSessionConfigs(
           source = provider, conf = df.sparkSession.sessionState.conf)
-        val options = sessionOptions ++ extraOptions
+        val options = sessionOptions ++ extraOptions.toMap
         val dsOptions = new CaseInsensitiveStringMap(options.asJava)
         val table = DataSourceV2Utils.getTableFromProvider(
           provider, dsOptions, userSpecifiedSchema = None)
@@ -472,7 +473,7 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) {
 
   private var trigger: Trigger = Trigger.ProcessingTime(0L)
 
-  private var extraOptions = new scala.collection.mutable.HashMap[String, String]
+  private var extraOptions = CaseInsensitiveMap[String](Map.empty)
 
   private var foreachWriter: ForeachWriter[T] = null
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index b64638751aed0..d90af35347acc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -185,6 +185,37 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
     query.stop()
   }
 
+  test("SPARK-32832: later option should override earlier options for load()") {
+    spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .load()
+    assert(LastOptions.parameters("path") == "5")
+  }
+
+  test("SPARK-32832: later option should override earlier options for start()") {
+    val ds = spark.readStream
+      .format("org.apache.spark.sql.streaming.test")
+      .load()
+    assert(LastOptions.parameters.isEmpty)
+
+    val query = ds.writeStream
+      .format("org.apache.spark.sql.streaming.test")
+      .option("checkpointLocation", newMetadataDir)
+      .option("paTh", "1")
+      .option("PATH", "2")
+      .option("Path", "3")
+      .option("patH", "4")
+      .option("path", "5")
+      .start()
+    assert(LastOptions.parameters("path") == "5")
+    query.stop()
+  }
+
   test("partitioning") {
     val df = spark.readStream
       .format("org.apache.spark.sql.streaming.test")

From 5708045fbf3721e97cad65d9aef9d859fcc2da11 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Thu, 10 Sep 2020 16:06:54 +0900
Subject: [PATCH 1042/1280] [SPARK-32819][SQL][3.0] ignoreNullability parameter
 should be effective recursively

### What changes were proposed in this pull request?

This patch proposes to check `ignoreNullability` parameter recursively in `equalsStructurally` method. This backports #29698 to branch-3.0.

### Why are the changes needed?

`equalsStructurally` is used to check type equality. We can optionally ask to ignore nullability check. But the parameter `ignoreNullability` is not passed recursively down to nested types. So it produces weird error like:

```
data type mismatch: argument 3 requires array<array<string>> type, however ... is of array<array<string>> type.
```

when running the query `select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array( x ) ) )`.

### Does this PR introduce _any_ user-facing change?

Yes, fixed a bug when running user query.

### How was this patch tested?

Unit tests.

Closes #29705 from viirya/SPARK-32819-3.0.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/types/DataType.scala |   8 +-
 .../spark/sql/types/DataTypeSuite.scala       | 110 +++++++++++++++++-
 .../inputs/higher-order-functions.sql         |   3 +
 .../ansi/higher-order-functions.sql.out       |  10 +-
 .../results/higher-order-functions.sql.out    |  10 +-
 5 files changed, 132 insertions(+), 9 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index fe8d7efc9dc12..3f70b7647f195 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -328,19 +328,19 @@ object DataType {
       ignoreNullability: Boolean = false): Boolean = {
     (from, to) match {
       case (left: ArrayType, right: ArrayType) =>
-        equalsStructurally(left.elementType, right.elementType) &&
+        equalsStructurally(left.elementType, right.elementType, ignoreNullability) &&
           (ignoreNullability || left.containsNull == right.containsNull)
 
       case (left: MapType, right: MapType) =>
-        equalsStructurally(left.keyType, right.keyType) &&
-          equalsStructurally(left.valueType, right.valueType) &&
+        equalsStructurally(left.keyType, right.keyType, ignoreNullability) &&
+          equalsStructurally(left.valueType, right.valueType, ignoreNullability) &&
           (ignoreNullability || left.valueContainsNull == right.valueContainsNull)
 
       case (StructType(fromFields), StructType(toFields)) =>
         fromFields.length == toFields.length &&
           fromFields.zip(toFields)
             .forall { case (l, r) =>
-              equalsStructurally(l.dataType, r.dataType) &&
+              equalsStructurally(l.dataType, r.dataType, ignoreNullability) &&
                 (ignoreNullability || l.nullable == r.nullable)
             }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index b71dc91eed4fa..52f80b1a52e72 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -423,10 +423,15 @@ class DataTypeSuite extends SparkFunSuite {
   checkCatalogString(MapType(IntegerType, StringType))
   checkCatalogString(MapType(IntegerType, createStruct(40)))
 
-  def checkEqualsStructurally(from: DataType, to: DataType, expected: Boolean): Unit = {
-    val testName = s"equalsStructurally: (from: $from, to: $to)"
+  def checkEqualsStructurally(
+      from: DataType,
+      to: DataType,
+      expected: Boolean,
+      ignoreNullability: Boolean = false): Unit = {
+    val testName = s"equalsStructurally: (from: $from, to: $to, " +
+      s"ignoreNullability: $ignoreNullability)"
     test(testName) {
-      assert(DataType.equalsStructurally(from, to) === expected)
+      assert(DataType.equalsStructurally(from, to, ignoreNullability) === expected)
     }
   }
 
@@ -453,6 +458,105 @@ class DataTypeSuite extends SparkFunSuite {
     new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType, false)),
     new StructType().add("f2", IntegerType).add("g", new StructType().add("f1", StringType)),
     false)
+  checkEqualsStructurally(
+    new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType, false)),
+    new StructType().add("f2", IntegerType).add("g", new StructType().add("f1", StringType)),
+    true,
+    ignoreNullability = true)
+  checkEqualsStructurally(
+    new StructType().add("f1", IntegerType).add("f", new StructType().add("f2", StringType)),
+    new StructType().add("f2", IntegerType, nullable = false)
+      .add("g", new StructType().add("f1", StringType)),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    ArrayType(
+      ArrayType(IntegerType, false), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    ArrayType(
+      ArrayType(IntegerType, true), false),
+    ArrayType(
+      ArrayType(IntegerType, false), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), false),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, false), true),
+    false,
+    ignoreNullability = false)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), false),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, false), true),
+    true,
+    ignoreNullability = true)
+
+  checkEqualsStructurally(
+    MapType(
+      ArrayType(IntegerType, false), ArrayType(IntegerType, true), true),
+    MapType(
+      ArrayType(IntegerType, true), ArrayType(IntegerType, true), true),
+    true,
+    ignoreNullability = true)
 
   test("SPARK-25031: MapType should produce current formatted string for complex types") {
     val keyType: DataType = StructType(Seq(
diff --git a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
index cfa06aea82b04..73dfa91827b08 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/higher-order-functions.sql
@@ -92,3 +92,6 @@ select transform_values(ys, (k, v) -> k + v) as v from nested;
 -- use non reversed keywords: all is non reversed only if !ansi
 select transform(ys, all -> all * all) as v from values (array(32, 97)) as t(ys);
 select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(ys);
+
+-- SPARK-32819: Aggregate on nested string arrays
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)));
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
index 7bef1bad4507e..6d26fae73b11c 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -282,3 +282,11 @@ no viable alternative at input 'all'(line 1, pos 22)
 == SQL ==
 select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(ys)
 ----------------------^^^
+
+
+-- !query
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
+-- !query schema
+struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+-- !query output
+[[""]]
diff --git a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
index d35d0d5d944bb..7b31b5690998c 100644
--- a/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 29
+-- Number of queries: 30
 
 
 -- !query
@@ -270,3 +270,11 @@ select transform(ys, (all, i) -> all + i) as v from values (array(32, 97)) as t(
 struct<v:array<int>>
 -- !query output
 [32,98]
+
+
+-- !query
+select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
+-- !query schema
+struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+-- !query output
+[[""]]

From 4fdd818fd9cf017b4f149c28fe944c628addf4f5 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 11 Sep 2020 09:16:12 +0900
Subject: [PATCH 1043/1280] [SPARK-32840][SQL][3.0] Invalid interval value can
 happen to be just adhesive with the unit

THIS PR backports https://github.com/apache/spark/pull/29708 to 3.0

### What changes were proposed in this pull request?
In this PR, we add a checker for STRING form interval value ahead for parsing multiple units intervals and fail directly if the interval value contains alphabets to prevent correctness issues like `interval '1 day 2' day`=`3 days`.

### Why are the changes needed?

fix correctness issue

### Does this PR introduce _any_ user-facing change?

yes, in spark 3.0.0 `interval '1 day 2' day`=`3 days` but now we fail with ParseException
### How was this patch tested?

add a test.

Closes #29716 from yaooqinn/SPARK-32840-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/parser/AstBuilder.scala      | 11 +++++-
 .../resources/sql-tests/inputs/interval.sql   |  4 ++
 .../sql-tests/results/ansi/interval.sql.out   | 38 ++++++++++++++++++-
 .../sql-tests/results/interval.sql.out        | 38 ++++++++++++++++++-
 4 files changed, 88 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index b29fe211454f6..307a72eee0376 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2107,7 +2107,16 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
         val kvs = units.indices.map { i =>
           val u = units(i).getText
           val v = if (values(i).STRING() != null) {
-            string(values(i).STRING())
+            val value = string(values(i).STRING())
+            // SPARK-32840: For invalid cases, e.g. INTERVAL '1 day 2' hour,
+            // INTERVAL 'interval 1' day, we need to check ahead before they are concatenated with
+            // units and become valid ones, e.g. '1 day 2 hour'.
+            // Ideally, we only ensure the value parts don't contain any units here.
+            if (value.exists(Character.isLetter)) {
+              throw new ParseException("Can only use numbers in the interval value part for" +
+                s" multiple unit value pairs interval form, but got invalid value: $value", ctx)
+            }
+            value
           } else {
             values(i).getText
           }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index 9ddefa5a34b87..9ad968ecda936 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -188,3 +188,7 @@ select interval '1.2';
 select interval '- 2';
 select interval '1 day -';
 select interval '1 day 1';
+
+select interval '1 day 2' day;
+select interval 'interval 1' day;
+select interval '-\t 1' day;
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index eb84f3e87f053..5a66db960ccbd 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 100
+-- Number of queries: 103
 
 
 -- !query
@@ -1054,3 +1054,39 @@ Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
 == SQL ==
 select interval '1 day 1'
 -------^^^
+
+
+-- !query
+select interval '1 day 2' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16)
+
+== SQL ==
+select interval '1 day 2' day
+----------------^^^
+
+
+-- !query
+select interval 'interval 1' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16)
+
+== SQL ==
+select interval 'interval 1' day
+----------------^^^
+
+
+-- !query
+select interval '-\t 1' day
+-- !query schema
+struct<INTERVAL '-1 days':interval>
+-- !query output
+-1 days
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index c5dd1e001bbe0..baf7f16908a5b 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 100
+-- Number of queries: 103
 
 
 -- !query
@@ -1026,3 +1026,39 @@ Cannot parse the INTERVAL value: 1 day 1(line 1, pos 7)
 == SQL ==
 select interval '1 day 1'
 -------^^^
+
+
+-- !query
+select interval '1 day 2' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: 1 day 2(line 1, pos 16)
+
+== SQL ==
+select interval '1 day 2' day
+----------------^^^
+
+
+-- !query
+select interval 'interval 1' day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+Can only use numbers in the interval value part for multiple unit value pairs interval form, but got invalid value: interval 1(line 1, pos 16)
+
+== SQL ==
+select interval 'interval 1' day
+----------------^^^
+
+
+-- !query
+select interval '-\t 1' day
+-- !query schema
+struct<INTERVAL '-1 days':interval>
+-- !query output
+-1 days

From cf14897d355efbf4acb3497ef1b74cd3a9c35d59 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Fri, 11 Sep 2020 09:22:56 +0900
Subject: [PATCH 1044/1280] [SPARK-32677][SQL][DOCS][MINOR] Improve code
 comment in CreateFunctionCommand

### What changes were proposed in this pull request?

We made a mistake in https://github.com/apache/spark/pull/29502, as there is no code comment to explain why we can't load the UDF class when creating functions. This PR improves the code comment.

### Why are the changes needed?

To avoid making the same mistake.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

N/A

Closes #29713 from cloud-fan/comment.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 328d81a2d1131742bcfba5117896c093db39e721)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../org/apache/spark/sql/execution/command/functions.scala    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
index 6fdc7f4a58195..d55d6967a0291 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
@@ -88,7 +88,9 @@ case class CreateFunctionCommand(
       } else {
         // For a permanent, we will store the metadata into underlying external catalog.
         // This function will be loaded into the FunctionRegistry when a query uses it.
-        // We do not load it into FunctionRegistry right now.
+        // We do not load it into FunctionRegistry right now, to avoid loading the resource and
+        // UDF class immediately, as the Spark application to create the function may not have
+        // access to the resource and/or UDF class.
         catalog.createFunction(func, ignoreIfExists)
       }
     }

From ec45d10d26621a0541d937bf6850e153b6cd426b Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 11 Sep 2020 11:48:34 -0700
Subject: [PATCH 1045/1280] [SPARK-32845][SS][TESTS] Add sinkParameter to check
 sink options robustly in DataStreamReaderWriterSuite

This PR aims to add `sinkParameter`  to check sink options robustly and independently in DataStreamReaderWriterSuite

`LastOptions.parameters` is designed to catch three cases: `sourceSchema`, `createSource`, `createSink`. However, `StreamQuery.stop` invokes `queryExecutionThread.join`, `runStream`, `createSource` immediately and reset the stored options by `createSink`.

To catch `createSink` options, currently, the test suite is trying a workaround pattern. However, we observed a flakiness in this pattern sometimes. If we split `createSink` option separately, we don't need this workaround and can eliminate this flakiness.

```scala
val query = df.writeStream.
   ...
   .start()
assert(LastOptions.paramters(..))
query.stop()
```

No. This is a test-only change.

Pass the newly updated test case.

Closes #29730 from dongjoon-hyun/SPARK-32845.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b4be6a6d12bf62f02cffe0bcc97ef32d27827d57)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../test/DataStreamReaderWriterSuite.scala    | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
index d90af35347acc..8bf7e276e12c2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamReaderWriterSuite.scala
@@ -43,11 +43,13 @@ object LastOptions {
   var mockStreamSourceProvider = mock(classOf[StreamSourceProvider])
   var mockStreamSinkProvider = mock(classOf[StreamSinkProvider])
   var parameters: Map[String, String] = null
+  var sinkParameters: Map[String, String] = null
   var schema: Option[StructType] = null
   var partitionColumns: Seq[String] = Nil
 
   def clear(): Unit = {
     parameters = null
+    sinkParameters = null
     schema = null
     partitionColumns = null
     reset(mockStreamSourceProvider)
@@ -101,7 +103,7 @@ class DefaultSource extends StreamSourceProvider with StreamSinkProvider {
       parameters: Map[String, String],
       partitionColumns: Seq[String],
       outputMode: OutputMode): Sink = {
-    LastOptions.parameters = parameters
+    LastOptions.sinkParameters = parameters
     LastOptions.partitionColumns = partitionColumns
     LastOptions.mockStreamSinkProvider.createSink(spark, parameters, partitionColumns, outputMode)
     (_: Long, _: DataFrame) => {}
@@ -169,20 +171,19 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
 
     LastOptions.clear()
 
-    val query = df.writeStream
+    df.writeStream
       .format("org.apache.spark.sql.streaming.test")
       .option("opt1", "5")
       .options(Map("opt2" -> "4"))
       .options(map)
       .option("checkpointLocation", newMetadataDir)
       .start()
+      .stop()
 
-    assert(LastOptions.parameters("opt1") == "5")
-    assert(LastOptions.parameters("opt2") == "4")
-    assert(LastOptions.parameters("opt3") == "3")
-    assert(LastOptions.parameters.contains("checkpointLocation"))
-
-    query.stop()
+    assert(LastOptions.sinkParameters("opt1") == "5")
+    assert(LastOptions.sinkParameters("opt2") == "4")
+    assert(LastOptions.sinkParameters("opt3") == "3")
+    assert(LastOptions.sinkParameters.contains("checkpointLocation"))
   }
 
   test("SPARK-32832: later option should override earlier options for load()") {
@@ -203,7 +204,7 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
       .load()
     assert(LastOptions.parameters.isEmpty)
 
-    val query = ds.writeStream
+    ds.writeStream
       .format("org.apache.spark.sql.streaming.test")
       .option("checkpointLocation", newMetadataDir)
       .option("paTh", "1")
@@ -212,8 +213,8 @@ class DataStreamReaderWriterSuite extends StreamTest with BeforeAndAfter {
       .option("patH", "4")
       .option("path", "5")
       .start()
-    assert(LastOptions.parameters("path") == "5")
-    query.stop()
+      .stop()
+    assert(LastOptions.sinkParameters("path") == "5")
   }
 
   test("partitioning") {

From 2e04689932b96321012ad9a5acaaf3ac306bba37 Mon Sep 17 00:00:00 2001
From: "sandeep.katta" <sandeep.katta2007@gmail.com>
Date: Sat, 12 Sep 2020 13:22:54 -0700
Subject: [PATCH 1046/1280] [SPARK-32779][SQL][FOLLOW-UP] Delete Unused code

### What changes were proposed in this pull request?
Follow-up PR as per the review comments in [29649](https://github.com/apache/spark/pull/29649/files/8d45542e915bea1b321f42988b407091065a2539#r487140171)

### Why are the changes needed?
Delete the un used code

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Existing UT

Closes #29736 from sandeep-katta/deadlockfollowup.

Authored-by: sandeep.katta <sandeep.katta2007@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 2009f953406aa5b4fdcdcd35f4c7c143f34d53e3)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../main/scala/org/apache/spark/sql/hive/client/HiveShim.scala  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 8d0a0f63bc615..c7cbf32bff7a0 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -1327,8 +1327,6 @@ private[client] class Shim_v3_0 extends Shim_v2_3 {
       inheritTableSpecs: Boolean,
       isSkewedStoreAsSubdir: Boolean,
       isSrcLocal: Boolean): Unit = {
-    val session = SparkSession.getActiveSession
-    assert(session.nonEmpty)
     val table = hive.getTable(tableName)
     val loadFileType = if (replace) {
       clazzLoadFileType.getEnumConstants.find(_.toString.equalsIgnoreCase("REPLACE_ALL"))

From d4d2f5c128cf5de98ca18daf30538ad132d99331 Mon Sep 17 00:00:00 2001
From: "bowen.li" <bowenli86@gmail.com>
Date: Sat, 12 Sep 2020 21:45:55 -0700
Subject: [PATCH 1047/1280] [SPARK-32865][DOC] python section in quickstart
 page doesn't display SPARK_VERSION correctly

### What changes were proposed in this pull request?

In https://github.com/apache/spark/blame/master/docs/quick-start.md#L402,it should be `{{site.SPARK_VERSION}}` rather than `{site.SPARK_VERSION}`

### Why are the changes needed?

SPARK_VERSION isn't displayed correctly, as shown below

![image](https://user-images.githubusercontent.com/1892692/93006726-d03c8680-f514-11ea-85e3-1d7cfb682ef2.png)

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

tested locally, as shown below

![image](https://user-images.githubusercontent.com/1892692/93006712-a6835f80-f514-11ea-8d78-6831c9d65265.png)

Closes #29738 from bowenli86/doc.

Authored-by: bowen.li <bowenli86@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 0549c20c6fccc7ff412818d7352b141590f88b1b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 docs/quick-start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/quick-start.md b/docs/quick-start.md
index e7a16a3461653..557fc187fb81d 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -399,7 +399,7 @@ If you are building a packaged PySpark application or library you can add it to
 
 {% highlight python %}
     install_requires=[
-        'pyspark=={site.SPARK_VERSION}'
+        'pyspark=={{site.SPARK_VERSION}}'
     ]
 {% endhighlight %}
 

From 828603d3c0dbe2ad88a8f55c075b7163868cd6d9 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 14 Sep 2020 13:54:21 -0700
Subject: [PATCH 1048/1280] [SPARK-32876][SQL] Change default fallback versions
 to 3.0.1 and 2.4.7 in HiveExternalCatalogVersionsSuite

### What changes were proposed in this pull request?

The Jenkins job fails to get the versions. This was fixed by adding temporary fallbacks at https://github.com/apache/spark/pull/28536.
This still doesn't work without the temporary fallbacks. See https://github.com/apache/spark/pull/29694

This PR adds new fallbacks since 2.3 is EOL and Spark 3.0.1 and 2.4.7 are released.

### Why are the changes needed?

To test correctly in Jenkins.

### Does this PR introduce _any_ user-facing change?

No, dev-only

### How was this patch tested?

Jenkins and GitHub Actions builds should test.

Closes #29748 from HyukjinKwon/SPARK-32876.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 0696f0467270969f40e9baa829533bdb55f4002a)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/hive/HiveExternalCatalogVersionsSuite.scala       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index aa96fa035c4f0..cbfdb7fac88d8 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -243,7 +243,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
         .filter(_ < org.apache.spark.SPARK_VERSION)
     } catch {
       // do not throw exception during object initialization.
-      case NonFatal(_) => Seq("2.3.4", "2.4.5") // A temporary fallback to use a specific version
+      case NonFatal(_) => Seq("3.0.1", "2.4.7") // A temporary fallback to use a specific version
     }
   }
 

From 990d49a28d6b7d1bd011aa7f1666217bcc920a77 Mon Sep 17 00:00:00 2001
From: Ankur Dave <ankurdave@gmail.com>
Date: Mon, 14 Sep 2020 13:58:15 -0700
Subject: [PATCH 1049/1280] [SPARK-32872][CORE] Prevent BytesToBytesMap at
 MAX_CAPACITY from exceeding growth threshold

### What changes were proposed in this pull request?

When BytesToBytesMap is at `MAX_CAPACITY` and reaches its growth threshold, `numKeys >= growthThreshold` is true but `longArray.size() / 2 < MAX_CAPACITY` is false. This correctly prevents the map from growing, but `canGrowArray` incorrectly remains true. Therefore the map keeps accepting new keys and exceeds its growth threshold. If we attempt to spill the map in this state, the UnsafeKVExternalSorter will not be able to reuse the long array for sorting. By this point the task has typically consumed all available memory, so the allocation of the new pointer array is likely to fail.

This PR fixes the issue by setting `canGrowArray` to false in this case. This prevents the map from accepting new elements when it cannot grow to accommodate them.

### Why are the changes needed?

Without this change, hash aggregations will fail when the number of groups per task is greater than `MAX_CAPACITY / 2 = 2^28` (approximately 268 million), and when the grouping aggregation is the only memory-consuming operator in its stage.

For example, the final aggregation in `SELECT COUNT(DISTINCT id) FROM tbl` fails when `tbl` contains 1 billion distinct values and when `spark.sql.shuffle.partitions=1`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Reproducing this issue requires building a very large BytesToBytesMap. Because this is infeasible to do in a unit test, this PR was tested manually by adding the following test to AbstractBytesToBytesMapSuite. Before this PR, the test fails in 8.5 minutes. With this PR, the test passes in 1.5 minutes.

```java
public abstract class AbstractBytesToBytesMapSuite {
  // ...
  Test
  public void respectGrowthThresholdAtMaxCapacity() {
    TestMemoryManager memoryManager2 =
        new TestMemoryManager(
            new SparkConf()
            .set(package$.MODULE$.MEMORY_OFFHEAP_ENABLED(), true)
            .set(package$.MODULE$.MEMORY_OFFHEAP_SIZE(), 25600 * 1024 * 1024L)
            .set(package$.MODULE$.SHUFFLE_SPILL_COMPRESS(), false)
            .set(package$.MODULE$.SHUFFLE_COMPRESS(), false));
    TaskMemoryManager taskMemoryManager2 = new TaskMemoryManager(memoryManager2, 0);
    final long pageSizeBytes = 8000000 + 8; // 8 bytes for end-of-page marker
    final BytesToBytesMap map = new BytesToBytesMap(taskMemoryManager2, 1024, pageSizeBytes);

    try {
      // Insert keys into the map until it stops accepting new keys.
      for (long i = 0; i < BytesToBytesMap.MAX_CAPACITY; i++) {
        if (i % (1024 * 1024) == 0) System.out.println("Inserting element " + i);
        final long[] value = new long[]{i};
        BytesToBytesMap.Location loc = map.lookup(value, Platform.LONG_ARRAY_OFFSET, 8);
        Assert.assertFalse(loc.isDefined());
        boolean success =
            loc.append(value, Platform.LONG_ARRAY_OFFSET, 8, value, Platform.LONG_ARRAY_OFFSET, 8);
        if (!success) break;
      }

      // The map should grow to its max capacity.
      long capacity = map.getArray().size() / 2;
      Assert.assertTrue(capacity == BytesToBytesMap.MAX_CAPACITY);

      // The map should stop accepting new keys once it has reached its growth
      // threshold, which is half the max capacity.
      Assert.assertTrue(map.numKeys() == BytesToBytesMap.MAX_CAPACITY / 2);

      map.free();
    } finally {
      map.free();
    }
  }
}
```

Closes #29744 from ankurdave/SPARK-32872.

Authored-by: Ankur Dave <ankurdave@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 72550c3be7120fcf2844d6914e883f1bec30d93f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/unsafe/map/BytesToBytesMap.java     | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 6e028886f2318..4036856e41851 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -738,12 +738,21 @@ public boolean append(Object kbase, long koff, int klen, Object vbase, long voff
         longArray.set(pos * 2 + 1, keyHashcode);
         isDefined = true;
 
-        // We use two array entries per key, so the array size is twice the capacity.
-        // We should compare the current capacity of the array, instead of its size.
-        if (numKeys >= growthThreshold && longArray.size() / 2 < MAX_CAPACITY) {
-          try {
-            growAndRehash();
-          } catch (SparkOutOfMemoryError oom) {
+        // If the map has reached its growth threshold, try to grow it.
+        if (numKeys >= growthThreshold) {
+          // We use two array entries per key, so the array size is twice the capacity.
+          // We should compare the current capacity of the array, instead of its size.
+          if (longArray.size() / 2 < MAX_CAPACITY) {
+            try {
+              growAndRehash();
+            } catch (SparkOutOfMemoryError oom) {
+              canGrowArray = false;
+            }
+          } else {
+            // The map is already at MAX_CAPACITY and cannot grow. Instead, we prevent it from
+            // accepting any more new elements to make sure we don't exceed the load factor. If we
+            // need to spill later, this allows UnsafeKVExternalSorter to reuse the array for
+            // sorting.
             canGrowArray = false;
           }
         }

From fe6ff1577d18fc919db926894b32500e17e07ecb Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Mon, 14 Sep 2020 18:24:52 -0700
Subject: [PATCH 1050/1280] [SPARK-32715][CORE] Fix memory leak when failed to
 store pieces of broadcast

### What changes were proposed in this pull request?
In TorrentBroadcast.scala
```scala
L133: if (!blockManager.putSingle(broadcastId, value, MEMORY_AND_DISK, tellMaster = false))
L137: TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
L147: if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true))
```
After the original value is saved successfully(TorrentBroadcast.scala: L133), but the following `blockifyObject()`(L137) or store piece(L147) steps are failed. There is no opportunity to release broadcast from memory.

This patch is to remove all pieces of the broadcast when failed to blockify or failed to store some pieces of a broadcast.

### Why are the changes needed?
We use Spark thrift-server as a long-running service. A bad query submitted a heavy BroadcastNestLoopJoin operation and made driver full GC. We killed the bad query but we found the driver's memory usage was still high and full GCs were still frequent. By investigating with GC dump and log, we found the broadcast may memory leak.

> 2020-08-19T18:54:02.824-0700: [Full GC (Allocation Failure)
2020-08-19T18:54:02.824-0700: [Class Histogram (before full gc):
116G->112G(170G), 184.9121920 secs]
[Eden: 32.0M(7616.0M)->0.0B(8704.0M) Survivors: 1088.0M->0.0B Heap: 116.4G(170.0G)->112.9G(170.0G)], [Metaspace: 177285K->177270K(182272K)]
1: 676531691 72035438432 [B
2: 676502528 32472121344 org.apache.spark.sql.catalyst.expressions.UnsafeRow
3: 99551 12018117568 [Ljava.lang.Object;
4: 26570 4349629040 [I
5: 6 3264536688 [Lorg.apache.spark.sql.catalyst.InternalRow;
6: 1708819 256299456 [C
7: 2338 179615208 [J
8: 1703669 54517408 java.lang.String
9: 103860 34896960 org.apache.spark.status.TaskDataWrapper
10: 177396 25545024 java.net.URI
...

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Manually test. This UT is hard to write and the patch is straightforward.

Closes #29558 from LantaoJin/SPARK-32715.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 7a9b066c66d29e946b4f384292021123beb6fe57)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/broadcast/TorrentBroadcast.scala    | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
index 77fbbc08c2103..1024d9b5060bc 100644
--- a/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
+++ b/core/src/main/scala/org/apache/spark/broadcast/TorrentBroadcast.scala
@@ -133,22 +133,30 @@ private[spark] class TorrentBroadcast[T: ClassTag](obj: T, id: Long)
     if (!blockManager.putSingle(broadcastId, value, MEMORY_AND_DISK, tellMaster = false)) {
       throw new SparkException(s"Failed to store $broadcastId in BlockManager")
     }
-    val blocks =
-      TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
-    if (checksumEnabled) {
-      checksums = new Array[Int](blocks.length)
-    }
-    blocks.zipWithIndex.foreach { case (block, i) =>
+    try {
+      val blocks =
+        TorrentBroadcast.blockifyObject(value, blockSize, SparkEnv.get.serializer, compressionCodec)
       if (checksumEnabled) {
-        checksums(i) = calcChecksum(block)
+        checksums = new Array[Int](blocks.length)
       }
-      val pieceId = BroadcastBlockId(id, "piece" + i)
-      val bytes = new ChunkedByteBuffer(block.duplicate())
-      if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) {
-        throw new SparkException(s"Failed to store $pieceId of $broadcastId in local BlockManager")
+      blocks.zipWithIndex.foreach { case (block, i) =>
+        if (checksumEnabled) {
+          checksums(i) = calcChecksum(block)
+        }
+        val pieceId = BroadcastBlockId(id, "piece" + i)
+        val bytes = new ChunkedByteBuffer(block.duplicate())
+        if (!blockManager.putBytes(pieceId, bytes, MEMORY_AND_DISK_SER, tellMaster = true)) {
+          throw new SparkException(s"Failed to store $pieceId of $broadcastId " +
+            s"in local BlockManager")
+        }
       }
+      blocks.length
+    } catch {
+      case t: Throwable =>
+        logError(s"Store broadcast $broadcastId fail, remove all pieces of the broadcast")
+        blockManager.removeBroadcast(id, tellMaster = true)
+        throw t
     }
-    blocks.length
   }
 
   /** Fetch torrent blocks from the driver and/or other executors. */

From cb6a0d08cc020d9a2c19173c9023a9f5e565dd6c Mon Sep 17 00:00:00 2001
From: Tanel Kiis <tanel.kiis@gmail.com>
Date: Wed, 16 Sep 2020 12:13:15 +0900
Subject: [PATCH 1051/1280] [SPARK-32688][SQL][TEST] Add special values to
 LiteralGenerator for float and double

### What changes were proposed in this pull request?

The `LiteralGenerator` for float and double datatypes was supposed to yield special values (NaN, +-inf) among others, but the `Gen.chooseNum` method does not yield values that are outside the defined range. The `Gen.chooseNum` for a wide range of floats and doubles does not yield values in the "everyday" range as stated in https://github.com/typelevel/scalacheck/issues/113 .

There is an similar class `RandomDataGenerator` that is used in some other tests. Added `-0.0` and `-0.0f` as special values to there too.

These changes revealed an inconsistency with the equality check between `-0.0` and `0.0`.

### Why are the changes needed?

The `LiteralGenerator` is mostly used in the `checkConsistencyBetweenInterpretedAndCodegen` method in `MathExpressionsSuite`. This change would have caught the bug fixed in #29495 .

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Locally reverted #29495 and verified that the existing test cases caught the bug.

Closes #29515 from tanelk/SPARK-32688.

Authored-by: Tanel Kiis <tanel.kiis@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 6051755bfe23a0e4564bf19476ec34cd7fd6008d)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/RandomDataGenerator.scala       |  4 ++--
 .../expressions/LiteralGenerator.scala        | 19 +++++++++++++++----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
index 6a5bdc4f6fc3d..3e2dc3f3af5f1 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/RandomDataGenerator.scala
@@ -260,10 +260,10 @@ object RandomDataGenerator {
           new MathContext(precision)).bigDecimal)
       case DoubleType => randomNumeric[Double](
         rand, r => longBitsToDouble(r.nextLong()), Seq(Double.MinValue, Double.MinPositiveValue,
-          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0))
+          Double.MaxValue, Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN, 0.0, -0.0))
       case FloatType => randomNumeric[Float](
         rand, r => intBitsToFloat(r.nextInt()), Seq(Float.MinValue, Float.MinPositiveValue,
-          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f))
+          Float.MaxValue, Float.PositiveInfinity, Float.NegativeInfinity, Float.NaN, 0.0f, -0.0f))
       case ByteType => randomNumeric[Byte](
         rand, _.nextInt().toByte, Seq(Byte.MinValue, Byte.MaxValue, 0.toByte))
       case IntegerType => randomNumeric[Int](
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
index d92eb01b69bf0..c8e3b0e157319 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralGenerator.scala
@@ -68,16 +68,27 @@ object LiteralGenerator {
   lazy val longLiteralGen: Gen[Literal] =
     for { l <- Arbitrary.arbLong.arbitrary } yield Literal.create(l, LongType)
 
+  // The floatLiteralGen and doubleLiteralGen will 50% of the time yield arbitrary values
+  // and 50% of the time will yield some special values that are more likely to reveal
+  // corner cases. This behavior is similar to the integral value generators.
   lazy val floatLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Float.MinValue / 2, Float.MaxValue / 2,
-        Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Float.NaN, Float.PositiveInfinity, Float.NegativeInfinity, Float.MinPositiveValue,
+          Float.MaxValue, -Float.MaxValue, 0.0f, -0.0f, 1.0f, -1.0f),
+        Arbitrary.arbFloat.arbitrary
+      )
     } yield Literal.create(f, FloatType)
 
   lazy val doubleLiteralGen: Gen[Literal] =
     for {
-      f <- Gen.chooseNum(Double.MinValue / 2, Double.MaxValue / 2,
-        Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity)
+      f <- Gen.oneOf(
+        Gen.oneOf(
+          Double.NaN, Double.PositiveInfinity, Double.NegativeInfinity, Double.MinPositiveValue,
+          Double.MaxValue, -Double.MaxValue, 0.0, -0.0, 1.0, -1.0),
+        Arbitrary.arbDouble.arbitrary
+      )
     } yield Literal.create(f, DoubleType)
 
   // TODO cache the generated data

From 75a225e8cd3add561a0677bd560c3b007f3eea9e Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Wed, 16 Sep 2020 20:16:15 +0900
Subject: [PATCH 1052/1280] [SPARK-32888][DOCS] Add user document about header
 flag and RDD as path for reading CSV

### What changes were proposed in this pull request?

This proposes to enhance user document of the API for loading a Dataset of strings storing CSV rows. If the header option is set to true, the API will remove all lines same with the header.

### Why are the changes needed?

This behavior can confuse users. We should explicitly document it.

### Does this PR introduce _any_ user-facing change?

No. Only doc change.

### How was this patch tested?

Only doc change.

Closes #29765 from viirya/SPARK-32888.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 550c1c9cfb5e6439cdd835388fe90a9ca1ebc695)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/readwriter.py                               | 3 +++
 .../src/main/scala/org/apache/spark/sql/DataFrameReader.scala  | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 336345e383729..0c54abbb7df21 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -421,6 +421,9 @@ def csv(self, path, schema=None, sep=None, encoding=None, quote=None, escape=Non
                         character. By default (None), it is disabled.
         :param header: uses the first line as names of columns. If None is set, it uses the
                        default value, ``false``.
+                       .. note:: if the given path is a RDD of Strings, this header
+                       option will remove all lines same with the header if exists.
+
         :param inferSchema: infers the input schema automatically from data. It requires one extra
                        pass over the data. If None is set, it uses the default value, ``false``.
         :param enforceSchema: If it is set to ``true``, the specified or inferred schema will be
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 5b78690b3267c..e8e31059aec37 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -566,6 +566,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
    * If the enforceSchema is set to `false`, only the CSV header in the first line is checked
    * to conform specified or inferred schema.
    *
+   * @note if `header` option is set to `true` when calling this API, all lines same with
+   * the header will be removed if exists.
+   *
    * @param csvDataset input Dataset with one CSV row per record
    * @since 2.2.0
    */

From aa9563eb6afa97ef5cf35c947218cacbc31d69ff Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 16 Sep 2020 10:13:47 -0700
Subject: [PATCH 1053/1280] [SPARK-32897][PYTHON] Don't show a deprecation
 warning at SparkSession.builder.getOrCreate

### What changes were proposed in this pull request?

In PySpark shell, if you call `SparkSession.builder.getOrCreate` as below:

```python
import warnings
from pyspark.sql import SparkSession, SQLContext
warnings.simplefilter('always', DeprecationWarning)
spark.stop()
SparkSession.builder.getOrCreate()
```

it shows the deprecation warning as below:

```
/.../spark/python/pyspark/sql/context.py:72: DeprecationWarning: Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.
  DeprecationWarning)
```

via https://github.com/apache/spark/blob/d3304268d3046116d39ec3d54a8e319dce188f36/python/pyspark/sql/session.py#L222

We shouldn't print the deprecation warning from it. This is the only place ^.

### Why are the changes needed?

To prevent to inform users that `SparkSession.builder.getOrCreate` is deprecated mistakenly.

### Does this PR introduce _any_ user-facing change?

Yes, it won't show a deprecation warning to end users for calling `SparkSession.builder.getOrCreate`.

### How was this patch tested?

Manually tested as above.

Closes #29768 from HyukjinKwon/SPARK-32897.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Takuya UESHIN <ueshin@databricks.com>
(cherry picked from commit 657e39a3346daf0c67cff3cf90fe68176c479747)
Signed-off-by: Takuya UESHIN <ueshin@databricks.com>
---
 python/pyspark/sql/context.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 0b7a7dabf296e..d9e5080cf0a9d 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -72,9 +72,10 @@ def __init__(self, sparkContext, sparkSession=None, jsqlContext=None):
         >>> df.rdd.map(lambda x: (x.i, x.s, x.d, x.l, x.b, x.time, x.row.a, x.list)).collect()
         [(1, u'string', 1.0, 1, True, datetime.datetime(2014, 8, 1, 14, 1, 5), 1, [1, 2, 3])]
         """
-        warnings.warn(
-            "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
-            DeprecationWarning)
+        if sparkSession is None:
+            warnings.warn(
+                "Deprecated in 3.0.0. Use SparkSession.builder.getOrCreate() instead.",
+                DeprecationWarning)
 
         self._sc = sparkContext
         self._jsc = self._sc._jsc

From 2e94d9af6e0d0c06eaba650dc70f07cc5b2bd791 Mon Sep 17 00:00:00 2001
From: Tom van Bussel <tom.vanbussel@databricks.com>
Date: Thu, 17 Sep 2020 12:35:40 +0200
Subject: [PATCH 1054/1280] [SPARK-32900][CORE] Allow UnsafeExternalSorter to
 spill when there are nulls

### What changes were proposed in this pull request?

This PR changes the way `UnsafeExternalSorter.SpillableIterator` checks whether it has spilled already, by checking whether `inMemSorter` is null. It also allows it to spill other `UnsafeSorterIterator`s than `UnsafeInMemorySorter.SortedIterator`.

### Why are the changes needed?

Before this PR `UnsafeExternalSorter.SpillableIterator` could not spill when there are NULLs in the input and radix sorting is used. Currently, Spark determines whether UnsafeExternalSorter.SpillableIterator has not spilled yet by checking whether `upstream` is an instance of `UnsafeInMemorySorter.SortedIterator`. When radix sorting is used and there are NULLs in the input however, `upstream` will be an instance of `UnsafeExternalSorter.ChainedIterator` instead, and Spark will assume that the `SpillableIterator` iterator has spilled already, and therefore cannot spill again when it's supposed to spill.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

A test was added to `UnsafeExternalSorterSuite` (and therefore also to `UnsafeExternalSorterRadixSortSuite`). I manually confirmed that the test failed in `UnsafeExternalSorterRadixSortSuite` without this patch.

Closes #29772 from tomvanbussel/SPARK-32900.

Authored-by: Tom van Bussel <tom.vanbussel@databricks.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit e5e54a3614ffd2a9150921e84e5b813d5cbf285a)
Signed-off-by: herman <herman@databricks.com>
---
 .../unsafe/sort/UnsafeExternalSorter.java     | 69 +++++++++++--------
 .../unsafe/sort/UnsafeInMemorySorter.java     |  1 +
 .../unsafe/sort/UnsafeSorterIterator.java     |  2 +
 .../unsafe/sort/UnsafeSorterSpillMerger.java  |  5 ++
 .../unsafe/sort/UnsafeSorterSpillReader.java  |  5 ++
 .../sort/UnsafeExternalSorterSuite.java       | 33 +++++++++
 6 files changed, 88 insertions(+), 27 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index 55e4e609c3c7b..71b9a5bc11542 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -501,11 +501,15 @@ private static void spillIterator(UnsafeSorterIterator inMemIterator,
    */
   class SpillableIterator extends UnsafeSorterIterator {
     private UnsafeSorterIterator upstream;
-    private UnsafeSorterIterator nextUpstream = null;
     private MemoryBlock lastPage = null;
     private boolean loaded = false;
     private int numRecords = 0;
 
+    private Object currentBaseObject;
+    private long currentBaseOffset;
+    private int currentRecordLength;
+    private long currentKeyPrefix;
+
     SpillableIterator(UnsafeSorterIterator inMemIterator) {
       this.upstream = inMemIterator;
       this.numRecords = inMemIterator.getNumRecords();
@@ -516,23 +520,26 @@ public int getNumRecords() {
       return numRecords;
     }
 
+    @Override
+    public long getCurrentPageNumber() {
+      throw new UnsupportedOperationException();
+    }
+
     public long spill() throws IOException {
       synchronized (this) {
-        if (!(upstream instanceof UnsafeInMemorySorter.SortedIterator && nextUpstream == null
-          && numRecords > 0)) {
+        if (inMemSorter == null || numRecords <= 0) {
           return 0L;
         }
 
-        UnsafeInMemorySorter.SortedIterator inMemIterator =
-          ((UnsafeInMemorySorter.SortedIterator) upstream).clone();
+        long currentPageNumber = upstream.getCurrentPageNumber();
 
-       ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
+        ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();
         // Iterate over the records that have not been returned and spill them.
         final UnsafeSorterSpillWriter spillWriter =
           new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords);
-        spillIterator(inMemIterator, spillWriter);
+        spillIterator(upstream, spillWriter);
         spillWriters.add(spillWriter);
-        nextUpstream = spillWriter.getReader(serializerManager);
+        upstream = spillWriter.getReader(serializerManager);
 
         long released = 0L;
         synchronized (UnsafeExternalSorter.this) {
@@ -540,8 +547,7 @@ public long spill() throws IOException {
           // is accessing the current record. We free this page in that caller's next loadNext()
           // call.
           for (MemoryBlock page : allocatedPages) {
-            if (!loaded || page.pageNumber !=
-                    ((UnsafeInMemorySorter.SortedIterator)upstream).getCurrentPageNumber()) {
+            if (!loaded || page.pageNumber != currentPageNumber) {
               released += page.size();
               freePage(page);
             } else {
@@ -575,22 +581,26 @@ public void loadNext() throws IOException {
       try {
         synchronized (this) {
           loaded = true;
-          if (nextUpstream != null) {
-            // Just consumed the last record from in memory iterator
-            if(lastPage != null) {
-              // Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
-              // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
-              // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
-              // `SpillableIterator` in sequence, which may happen in
-              // `TaskMemoryManager.acquireExecutionMemory`.
-              pageToFree = lastPage;
-              lastPage = null;
-            }
-            upstream = nextUpstream;
-            nextUpstream = null;
+          // Just consumed the last record from in memory iterator
+          if (lastPage != null) {
+            // Do not free the page here, while we are locking `SpillableIterator`. The `freePage`
+            // method locks the `TaskMemoryManager`, and it's a bad idea to lock 2 objects in
+            // sequence. We may hit dead lock if another thread locks `TaskMemoryManager` and
+            // `SpillableIterator` in sequence, which may happen in
+            // `TaskMemoryManager.acquireExecutionMemory`.
+            pageToFree = lastPage;
+            lastPage = null;
           }
           numRecords--;
           upstream.loadNext();
+
+          // Keep track of the current base object, base offset, record length, and key prefix,
+          // so that the current record can still be read in case a spill is triggered and we
+          // switch to the spill writer's iterator.
+          currentBaseObject = upstream.getBaseObject();
+          currentBaseOffset = upstream.getBaseOffset();
+          currentRecordLength = upstream.getRecordLength();
+          currentKeyPrefix = upstream.getKeyPrefix();
         }
       } finally {
         if (pageToFree != null) {
@@ -601,22 +611,22 @@ public void loadNext() throws IOException {
 
     @Override
     public Object getBaseObject() {
-      return upstream.getBaseObject();
+      return currentBaseObject;
     }
 
     @Override
     public long getBaseOffset() {
-      return upstream.getBaseOffset();
+      return currentBaseOffset;
     }
 
     @Override
     public int getRecordLength() {
-      return upstream.getRecordLength();
+      return currentRecordLength;
     }
 
     @Override
     public long getKeyPrefix() {
-      return upstream.getKeyPrefix();
+      return currentKeyPrefix;
     }
   }
 
@@ -693,6 +703,11 @@ public int getNumRecords() {
       return numRecords;
     }
 
+    @Override
+    public long getCurrentPageNumber() {
+      return current.getCurrentPageNumber();
+    }
+
     @Override
     public boolean hasNext() {
       while (!current.hasNext() && !iterators.isEmpty()) {
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
index 660eb790a550b..ff641a24a7b3e 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
@@ -330,6 +330,7 @@ public void loadNext() {
     @Override
     public long getBaseOffset() { return baseOffset; }
 
+    @Override
     public long getCurrentPageNumber() {
       return currentPageNumber;
     }
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
index 1b3167fcc250c..d9f22311d07c2 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterIterator.java
@@ -34,4 +34,6 @@ public abstract class UnsafeSorterIterator {
   public abstract long getKeyPrefix();
 
   public abstract int getNumRecords();
+
+  public abstract long getCurrentPageNumber();
 }
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
index ab800288dcb43..f8603c5799e9b 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillMerger.java
@@ -70,6 +70,11 @@ public int getNumRecords() {
         return numRecords;
       }
 
+      @Override
+      public long getCurrentPageNumber() {
+        throw new UnsupportedOperationException();
+      }
+
       @Override
       public boolean hasNext() {
         return !priorityQueue.isEmpty() || (spillReader != null && spillReader.hasNext());
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
index a524c4790407d..db79efd008530 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillReader.java
@@ -89,6 +89,11 @@ public int getNumRecords() {
     return numRecords;
   }
 
+  @Override
+  public long getCurrentPageNumber() {
+    throw new UnsupportedOperationException();
+  }
+
   @Override
   public boolean hasNext() {
     return (numRecordsRemaining > 0);
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
index 43977717f6c97..087d090c1c60e 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
@@ -359,6 +359,39 @@ public void forcedSpillingWithReadIterator() throws Exception {
     assertSpillFilesWereCleanedUp();
   }
 
+  @Test
+  public void forcedSpillingNullsWithReadIterator() throws Exception {
+    final UnsafeExternalSorter sorter = newSorter();
+    long[] record = new long[100];
+    final int recordSize = record.length * 8;
+    final int n = (int) pageSizeBytes / recordSize * 3;
+    for (int i = 0; i < n; i++) {
+      boolean isNull = i % 2 == 0;
+      sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, isNull);
+    }
+    assertTrue(sorter.getNumberOfAllocatedPages() >= 2);
+
+    UnsafeExternalSorter.SpillableIterator iter =
+            (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator();
+    final int numRecordsToReadBeforeSpilling = n / 3;
+    for (int i = 0; i < numRecordsToReadBeforeSpilling; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+    }
+
+    assertTrue(iter.spill() > 0);
+    assertEquals(0, iter.spill());
+
+    for (int i = numRecordsToReadBeforeSpilling; i < n; i++) {
+      assertTrue(iter.hasNext());
+      iter.loadNext();
+    }
+    assertFalse(iter.hasNext());
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
+  }
+
   @Test
   public void forcedSpillingWithNotReadIterator() throws Exception {
     final UnsafeExternalSorter sorter = newSorter();

From b3b6f381108b770c38753ebdb55e209361674008 Mon Sep 17 00:00:00 2001
From: Udbhav30 <u.agrawal30@gmail.com>
Date: Thu, 17 Sep 2020 09:25:17 -0700
Subject: [PATCH 1055/1280] [SPARK-32887][DOC] Correct the typo for SHOW TABLE

### What changes were proposed in this pull request?
Correct the typo in Show Table document

### Why are the changes needed?
Current Document of Show Table returns in parse error, so it is misleading to users

### Does this PR introduce _any_ user-facing change?
Yes, the document of show table is corrected now

### How was this patch tested?
NA

Closes #29758 from Udbhav30/showtable.

Authored-by: Udbhav30 <u.agrawal30@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 88e87bc8ebfa5aa1a8cc8928672749517ae0c41f)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-ref-syntax-aux-show-table.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/sql-ref-syntax-aux-show-table.md b/docs/sql-ref-syntax-aux-show-table.md
index 0ce0a3eefa538..3314402ea3e2b 100644
--- a/docs/sql-ref-syntax-aux-show-table.md
+++ b/docs/sql-ref-syntax-aux-show-table.md
@@ -97,7 +97,7 @@ SHOW TABLE EXTENDED LIKE 'employee';
 +--------+---------+-----------+--------------------------------------------------------------+
 
 -- showing the multiple table details with pattern matching
-SHOW TABLE EXTENDED  LIKE `employe*`;
+SHOW TABLE EXTENDED  LIKE 'employe*';
 +--------+---------+-----------+--------------------------------------------------------------+
 |database|tableName|isTemporary|                         information                          |
 +--------+---------+-----------+--------------------------------------------------------------+
@@ -146,7 +146,7 @@ SHOW TABLE EXTENDED  LIKE `employe*`;
 +--------+---------+----------+---------------------------------------------------------------+
   
 -- show partition file system details
-SHOW TABLE EXTENDED  IN default LIKE `employee` PARTITION (`grade=1`);
+SHOW TABLE EXTENDED  IN default LIKE 'employee' PARTITION (grade=1);
 +--------+---------+-----------+--------------------------------------------------------------+
 |database|tableName|isTemporary|                         information                          |
 +--------+---------+-----------+--------------------------------------------------------------+
@@ -169,7 +169,7 @@ SHOW TABLE EXTENDED  IN default LIKE `employee` PARTITION (`grade=1`);
 +--------+---------+-----------+--------------------------------------------------------------+
 
 -- show partition file system details with regex fails as shown below
-SHOW TABLE EXTENDED  IN default LIKE `empl*` PARTITION (`grade=1`);
+SHOW TABLE EXTENDED  IN default LIKE 'empl*' PARTITION (grade=1);
 Error: Error running query: org.apache.spark.sql.catalyst.analysis.NoSuchTableException:
  Table or view 'emplo*' not found in database 'default'; (state=,code=0)
 ```

From 17a5195dead75a4dffd150f3f69fa4983e05561b Mon Sep 17 00:00:00 2001
From: Zhenhua Wang <wzh_zju@163.com>
Date: Thu, 17 Sep 2020 12:22:35 -0500
Subject: [PATCH 1056/1280] [SPARK-32738][CORE][3.0] Should reduce the number
 of active threads if fatal error happens in `Inbox.process`

This is a backport for [pr#29580](https://github.com/apache/spark/pull/29580) to branch 3.0.

### What changes were proposed in this pull request?

Processing for `ThreadSafeRpcEndpoint` is controlled by  `numActiveThreads` in `Inbox`. Now if any fatal error happens during `Inbox.process`, `numActiveThreads` is not reduced. Then other threads can not process messages in that inbox, which causes the endpoint to "hang". For other type of endpoints, we also should keep  `numActiveThreads` correct.

This problem is more serious in previous Spark 2.x versions since the driver, executor and block manager endpoints are all thread safe endpoints.

To fix this, we should reduce the number of active threads if fatal error happens in `Inbox.process`.

### Why are the changes needed?

`numActiveThreads` is not correct when fatal error happens and will cause the described problem.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Add a new test.

Closes #29763 from wzhfy/deal_with_fatal_error_3.0.

Authored-by: Zhenhua Wang <wzh_zju@163.com>
Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com>
---
 .../org/apache/spark/rpc/netty/Inbox.scala    | 20 +++++++++++++++++++
 .../apache/spark/rpc/netty/InboxSuite.scala   | 13 ++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
index 2ed03f7430c32..472401b23fe8e 100644
--- a/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
+++ b/core/src/main/scala/org/apache/spark/rpc/netty/Inbox.scala
@@ -200,6 +200,16 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint)
    * Calls action closure, and calls the endpoint's onError function in the case of exceptions.
    */
   private def safelyCall(endpoint: RpcEndpoint)(action: => Unit): Unit = {
+    def dealWithFatalError(fatal: Throwable): Unit = {
+      inbox.synchronized {
+        assert(numActiveThreads > 0, "The number of active threads should be positive.")
+        // Should reduce the number of active threads before throw the error.
+        numActiveThreads -= 1
+      }
+      logError(s"An error happened while processing message in the inbox for $endpointName", fatal)
+      throw fatal
+    }
+
     try action catch {
       case NonFatal(e) =>
         try endpoint.onError(e) catch {
@@ -209,8 +219,18 @@ private[netty] class Inbox(val endpointName: String, val endpoint: RpcEndpoint)
             } else {
               logError("Ignoring error", ee)
             }
+          case fatal: Throwable =>
+            dealWithFatalError(fatal)
         }
+      case fatal: Throwable =>
+        dealWithFatalError(fatal)
     }
   }
 
+  // exposed only for testing
+  def getNumActiveThreads: Int = {
+    inbox.synchronized {
+      inbox.numActiveThreads
+    }
+  }
 }
diff --git a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
index c74c728b3e3f3..8b1c602cd8e58 100644
--- a/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rpc/netty/InboxSuite.scala
@@ -136,4 +136,17 @@ class InboxSuite extends SparkFunSuite {
 
     endpoint.verifySingleOnNetworkErrorMessage(cause, remoteAddress)
   }
+
+  test("SPARK-32738: should reduce the number of active threads when fatal error happens") {
+    val endpoint = mock(classOf[TestRpcEndpoint])
+    when(endpoint.receive).thenThrow(new OutOfMemoryError())
+
+    val dispatcher = mock(classOf[Dispatcher])
+    val inbox = new Inbox("name", endpoint)
+    inbox.post(OneWayMessage(null, "hi"))
+    intercept[OutOfMemoryError] {
+      inbox.process(dispatcher)
+    }
+    assert(inbox.getNumActiveThreads == 0)
+  }
 }

From ecc2f5d9e227b62f418d65708f516ffe8e690f96 Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Fri, 18 Sep 2020 08:17:23 +0900
Subject: [PATCH 1057/1280] [SPARK-32635][SQL] Fix foldable propagation

### What changes were proposed in this pull request?
This PR rewrites `FoldablePropagation` rule to replace attribute references in a node with foldables coming only from the node's children.

Before this PR in the case of this example (with setting`spark.sql.optimizer.excludedRules=org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation`):
```scala
val a = Seq("1").toDF("col1").withColumn("col2", lit("1"))
val b = Seq("2").toDF("col1").withColumn("col2", lit("2"))
val aub = a.union(b)
val c = aub.filter($"col1" === "2").cache()
val d = Seq("2").toDF( "col4")
val r = d.join(aub, $"col2" === $"col4").select("col4")
val l = c.select("col2")
val df = l.join(r, $"col2" === $"col4", "LeftOuter")
df.show()
```
foldable propagation happens incorrectly:
```
 Join LeftOuter, (col2#6 = col4#34)                                                              Join LeftOuter, (col2#6 = col4#34)
!:- Project [col2#6]                                                                             :- Project [1 AS col2#6]
 :  +- InMemoryRelation [col1#4, col2#6], StorageLevel(disk, memory, deserialized, 1 replicas)   :  +- InMemoryRelation [col1#4, col2#6], StorageLevel(disk, memory, deserialized, 1 replicas)
 :        +- Union                                                                               :        +- Union
 :           :- *(1) Project [value#1 AS col1#4, 1 AS col2#6]                                    :           :- *(1) Project [value#1 AS col1#4, 1 AS col2#6]
 :           :  +- *(1) Filter (isnotnull(value#1) AND (value#1 = 2))                            :           :  +- *(1) Filter (isnotnull(value#1) AND (value#1 = 2))
 :           :     +- *(1) LocalTableScan [value#1]                                              :           :     +- *(1) LocalTableScan [value#1]
 :           +- *(2) Project [value#10 AS col1#13, 2 AS col2#15]                                 :           +- *(2) Project [value#10 AS col1#13, 2 AS col2#15]
 :              +- *(2) Filter (isnotnull(value#10) AND (value#10 = 2))                          :              +- *(2) Filter (isnotnull(value#10) AND (value#10 = 2))
 :                 +- *(2) LocalTableScan [value#10]                                             :                 +- *(2) LocalTableScan [value#10]
 +- Project [col4#34]                                                                            +- Project [col4#34]
    +- Join Inner, (col2#6 = col4#34)                                                               +- Join Inner, (col2#6 = col4#34)
       :- Project [value#31 AS col4#34]                                                                :- Project [value#31 AS col4#34]
       :  +- LocalRelation [value#31]                                                                  :  +- LocalRelation [value#31]
       +- Project [col2#6]                                                                             +- Project [col2#6]
          +- Union false, false                                                                           +- Union false, false
             :- Project [1 AS col2#6]                                                                        :- Project [1 AS col2#6]
             :  +- LocalRelation [value#1]                                                                   :  +- LocalRelation [value#1]
             +- Project [2 AS col2#15]                                                                       +- Project [2 AS col2#15]
                +- LocalRelation [value#10]                                                                     +- LocalRelation [value#10]

```
and so the result is wrong:
```
+----+----+
|col2|col4|
+----+----+
|   1|null|
+----+----+
```

After this PR foldable propagation will not happen incorrectly and the result is correct:
```
+----+----+
|col2|col4|
+----+----+
|   2|   2|
+----+----+
```

### Why are the changes needed?
To fix a correctness issue.

### Does this PR introduce _any_ user-facing change?
Yes, fixes a correctness issue.

### How was this patch tested?
Existing and new UTs.

Closes #29771 from peter-toth/SPARK-32635-fix-foldable-propagation.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4ced58862c707aa916f7a55d15c3887c94c9b210)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../catalyst/expressions/AttributeMap.scala   |   2 +
 .../catalyst/expressions/AttributeMap.scala   |   2 +
 .../sql/catalyst/optimizer/expressions.scala  | 121 +++++++++++-------
 .../org/apache/spark/sql/DataFrameSuite.scala |  12 ++
 4 files changed, 88 insertions(+), 49 deletions(-)

diff --git a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
index 75a8bec018a1f..42b92d4593c77 100644
--- a/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
+++ b/sql/catalyst/src/main/scala-2.12/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
@@ -26,6 +26,8 @@ object AttributeMap {
   def apply[A](kvs: Seq[(Attribute, A)]): AttributeMap[A] = {
     new AttributeMap(kvs.map(kv => (kv._1.exprId, kv)).toMap)
   }
+
+  def empty[A]: AttributeMap[A] = new AttributeMap(Map.empty)
 }
 
 class AttributeMap[A](val baseMap: Map[ExprId, (Attribute, A)])
diff --git a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
index 4caa3d0461875..e6b53e3e6548f 100644
--- a/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
+++ b/sql/catalyst/src/main/scala-2.13/org/apache/spark/sql/catalyst/expressions/AttributeMap.scala
@@ -26,6 +26,8 @@ object AttributeMap {
   def apply[A](kvs: Seq[(Attribute, A)]): AttributeMap[A] = {
     new AttributeMap(kvs.map(kv => (kv._1.exprId, kv)).toMap)
   }
+
+  def empty[A]: AttributeMap[A] = new AttributeMap(Map.empty)
 }
 
 class AttributeMap[A](val baseMap: Map[ExprId, (Attribute, A)])
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index bd400f86ea2c1..7773f5c80e061 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -620,59 +620,82 @@ object NullPropagation extends Rule[LogicalPlan] {
  */
 object FoldablePropagation extends Rule[LogicalPlan] {
   def apply(plan: LogicalPlan): LogicalPlan = {
-    var foldableMap = AttributeMap(plan.flatMap {
-      case Project(projectList, _) => projectList.collect {
-        case a: Alias if a.child.foldable => (a.toAttribute, a)
-      }
-      case _ => Nil
-    })
-    val replaceFoldable: PartialFunction[Expression, Expression] = {
-      case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+    CleanupAliases(propagateFoldables(plan)._1)
+  }
+
+  private def propagateFoldables(plan: LogicalPlan): (LogicalPlan, AttributeMap[Alias]) = {
+    plan match {
+      case p: Project =>
+        val (newChild, foldableMap) = propagateFoldables(p.child)
+        val newProject =
+          replaceFoldable(p.withNewChildren(Seq(newChild)).asInstanceOf[Project], foldableMap)
+        val newFoldableMap = AttributeMap(newProject.projectList.collect {
+          case a: Alias if a.child.foldable => (a.toAttribute, a)
+        })
+        (newProject, newFoldableMap)
+
+      // We can not replace the attributes in `Expand.output`. If there are other non-leaf
+      // operators that have the `output` field, we should put them here too.
+      case e: Expand =>
+        val (newChild, foldableMap) = propagateFoldables(e.child)
+        val expandWithNewChildren = e.withNewChildren(Seq(newChild)).asInstanceOf[Expand]
+        val newExpand = if (foldableMap.isEmpty) {
+          expandWithNewChildren
+        } else {
+          val newProjections = expandWithNewChildren.projections.map(_.map(_.transform {
+            case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+          }))
+          if (newProjections == expandWithNewChildren.projections) {
+            expandWithNewChildren
+          } else {
+            expandWithNewChildren.copy(projections = newProjections)
+          }
+        }
+        (newExpand, foldableMap)
+
+      case u: UnaryNode if canPropagateFoldables(u) =>
+        val (newChild, foldableMap) = propagateFoldables(u.child)
+        val newU = replaceFoldable(u.withNewChildren(Seq(newChild)), foldableMap)
+        (newU, foldableMap)
+
+      // Join derives the output attributes from its child while they are actually not the
+      // same attributes. For example, the output of outer join is not always picked from its
+      // children, but can also be null. We should exclude these miss-derived attributes when
+      // propagating the foldable expressions.
+      // TODO(cloud-fan): It seems more reasonable to use new attributes as the output attributes
+      // of outer join.
+      case j: Join =>
+        val (newChildren, foldableMaps) = j.children.map(propagateFoldables).unzip
+        val foldableMap = AttributeMap(
+          foldableMaps.foldLeft(Iterable.empty[(Attribute, Alias)])(_ ++ _.baseMap.values).toSeq)
+        val newJoin =
+          replaceFoldable(j.withNewChildren(newChildren).asInstanceOf[Join], foldableMap)
+        val missDerivedAttrsSet: AttributeSet = AttributeSet(newJoin.joinType match {
+          case _: InnerLike | LeftExistence(_) => Nil
+          case LeftOuter => newJoin.right.output
+          case RightOuter => newJoin.left.output
+          case FullOuter => newJoin.left.output ++ newJoin.right.output
+        })
+        val newFoldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
+          case (attr, _) => missDerivedAttrsSet.contains(attr)
+        }.toSeq)
+        (newJoin, newFoldableMap)
+
+      // For other plans, they are not safe to apply foldable propagation, and they should not
+      // propagate foldable expressions from children.
+      case o =>
+        val newOther = o.mapChildren(propagateFoldables(_)._1)
+        (newOther, AttributeMap.empty)
     }
+  }
 
+  private def replaceFoldable(plan: LogicalPlan, foldableMap: AttributeMap[Alias]): plan.type = {
     if (foldableMap.isEmpty) {
       plan
     } else {
-      CleanupAliases(plan.transformUp {
-        // We can only propagate foldables for a subset of unary nodes.
-        case u: UnaryNode if foldableMap.nonEmpty && canPropagateFoldables(u) =>
-          u.transformExpressions(replaceFoldable)
-
-        // Join derives the output attributes from its child while they are actually not the
-        // same attributes. For example, the output of outer join is not always picked from its
-        // children, but can also be null. We should exclude these miss-derived attributes when
-        // propagating the foldable expressions.
-        // TODO(cloud-fan): It seems more reasonable to use new attributes as the output attributes
-        // of outer join.
-        case j @ Join(left, right, joinType, _, _) if foldableMap.nonEmpty =>
-          val newJoin = j.transformExpressions(replaceFoldable)
-          val missDerivedAttrsSet: AttributeSet = AttributeSet(joinType match {
-            case _: InnerLike | LeftExistence(_) => Nil
-            case LeftOuter => right.output
-            case RightOuter => left.output
-            case FullOuter => left.output ++ right.output
-          })
-          foldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
-            case (attr, _) => missDerivedAttrsSet.contains(attr)
-          }.toSeq)
-          newJoin
-
-        // We can not replace the attributes in `Expand.output`. If there are other non-leaf
-        // operators that have the `output` field, we should put them here too.
-        case expand: Expand if foldableMap.nonEmpty =>
-          expand.copy(projections = expand.projections.map { projection =>
-            projection.map(_.transform(replaceFoldable))
-          })
-
-        // For other plans, they are not safe to apply foldable propagation, and they should not
-        // propagate foldable expressions from children.
-        case other if foldableMap.nonEmpty =>
-          val childrenOutputSet = AttributeSet(other.children.flatMap(_.output))
-          foldableMap = AttributeMap(foldableMap.baseMap.values.filterNot {
-            case (attr, _) => childrenOutputSet.contains(attr)
-          }.toSeq)
-          other
-      })
+      plan transformExpressions {
+        case a: AttributeReference if foldableMap.contains(a) => foldableMap(a)
+      }
     }
   }
 
@@ -680,7 +703,7 @@ object FoldablePropagation extends Rule[LogicalPlan] {
    * Whitelist of all [[UnaryNode]]s for which allow foldable propagation.
    */
   private def canPropagateFoldables(u: UnaryNode): Boolean = u match {
-    case _: Project => true
+    // Handling `Project` is moved to `propagateFoldables`.
     case _: Filter => true
     case _: SubqueryAlias => true
     case _: Aggregate => true
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index f8ce3f49218de..7869005ea2de9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2455,6 +2455,18 @@ class DataFrameSuite extends QueryTest
     val df = Seq(0.0 -> -0.0).toDF("pos", "neg")
     checkAnswer(df.select($"pos" > $"neg"), Row(false))
   }
+
+  test("SPARK-32635: Replace references with foldables coming only from the node's children") {
+    val a = Seq("1").toDF("col1").withColumn("col2", lit("1"))
+    val b = Seq("2").toDF("col1").withColumn("col2", lit("2"))
+    val aub = a.union(b)
+    val c = aub.filter($"col1" === "2").cache()
+    val d = Seq("2").toDF("col4")
+    val r = d.join(aub, $"col2" === $"col4").select("col4")
+    val l = c.select("col2")
+    val df = l.join(r, $"col2" === $"col4", "LeftOuter")
+    checkAnswer(df, Row("2", "2"))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)

From 5581a92bb8b1d96278796e6c18812c5a2931db11 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 18 Sep 2020 10:47:06 +0900
Subject: [PATCH 1058/1280] [SPARK-32908][SQL] Fix target error calculation in
 `percentile_approx()`
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
1. Change the target error calculation according to the paper [Space-Efficient Online Computation of Quantile Summaries](http://infolab.stanford.edu/~datar/courses/cs361a/papers/quantiles.pdf). It says that the error `e = max(gi, deltai)/2` (see the page 59). Also this has clear explanation [ε-approximate quantiles](http://www.mathcs.emory.edu/~cheung/Courses/584/Syllabus/08-Quantile/Greenwald.html#proofprop1).
2. Added a test to check different accuracies.
3. Added an input CSV file `percentile_approx-input.csv.bz2` to the resource folder `sql/catalyst/src/main/resources` for the test.

### Why are the changes needed?
To fix incorrect percentile calculation, see an example in SPARK-32908.

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
- By running existing tests in `QuantileSummariesSuite` and in `ApproximatePercentileQuerySuite`.
- Added new test `SPARK-32908: maximum target error in percentile_approx` to `ApproximatePercentileQuerySuite`.

Closes #29784 from MaxGekk/fix-percentile_approx-2.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 75dd86400c3c2348a4139586fbbead840512b909)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/util/QuantileSummaries.scala |   2 +-
 .../test-data/percentile_approx-input.csv.bz2 | Bin 0 -> 124614 bytes
 .../sql/ApproximatePercentileQuerySuite.scala |  21 +++++++++++++++++-
 3 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
index 3a0490d07733d..e07aa0ffa42fc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/QuantileSummaries.scala
@@ -254,7 +254,7 @@ class QuantileSummaries(
 
     // Target rank
     val rank = math.ceil(quantile * count).toLong
-    val targetError = relativeError * count
+    val targetError = sampled.map(s => s.delta + s.g).max / 2
     // Minimum rank at current sample
     var minRank = 0L
     var i = 0
diff --git a/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2 b/sql/core/src/test/resources/test-data/percentile_approx-input.csv.bz2
new file mode 100644
index 0000000000000000000000000000000000000000..f85e2896b3a89562bc5c49ca7eb915997297d9a4
GIT binary patch
literal 124614
zcmYhhWl$Rcv#8xtpt!rcyGx+O-3bmw0|A1&^t8CU26qSq3k2sa?(UG#;_lkg-ud1;
z^PQQyfA*PK`?IsNJG07GZnC0EmRu&5+LvX-7^LNs@7|I8&o8GB1L_sXsbr1jfvz?O
zII?rcuSm+Y6CUicMKz`5Wl=4+W?JtlnwLu<w(TC^VqMe-{I=EHTk<X0$}A|@(}<#>
zvs2AzC)g|O`e_ck_Nu{FMN(JQ<73dN8WM_+jDHV7PA=t&AQ3x#CR<Tgg}sNpGg|JQ
zP&3f-mKndbfO{^|)4JkfS1MejcP$T`!)|TukS%U1vEkJan4lvt`U0MY)C)*zZ*P6h
z)`J_XK&&Y%ojy=3)lf80DoL$YuYoxYJdGIJ2hc*GRgM*@j!Se~5#*vc#hDT`=Ak{-
zd}3R><2iQiwKdRd9O!~v&o=k-ugr!lTkGK!2}0d#v>+8+^Ot<NZ8ckxo_dXvZ949h
z*^so7Ei42V)EJ!mnG+`yf{^br9YewrdgYVQ==B*0)QM`&L>rtdRir}?H@-Aob5k!|
z%2(Uzg&}LqwzWXxX4^o>2R+o;92;~U!7nkHH?vdHst{}<DYb6p(gXxT;6|08X=@J%
z-;%pa%K8)#irfan@jbEJcJ$b-Mz?CD7H1$ZttvcqajPXyv*Gp4bq}fKj$jWYw^F^I
z{c=0P5{+syceT~lvuFZW!aIR5x8+t39!uj{2-3+DjVAS*^q>Q#B*EPwNQjY@6uPgI
zViiP(Zre=ESu6A)w3cumY2DMl^c+G?>mTO}6x7fb@gdimAW+y^+$?eqQVY98Idnp&
zKtps?aG+rp+{1Gt*f%w=wNq*+Yn|K$o@<$Qwbp5hgmm&-X;ID9I8zc*Re+x6U7E%b
z){qX1rPgSRgv-?cIuEnlLUFU@Ibb!cqe8vPZHH~j?M0g0ti9DzoMo%^Y>3BZN2-X$
z1XK$}<`sbvo>Iv2CXl*Uisu0g+3Uh<>z<RE#Mc!ZWDSAMSn@V{R&F<alPuqYm|XyE
zB>Fy@9@Wy?gFNV^Am5ONi|e!Bt!Z0nJM$z#?H%i8o^XUxN2ePyx5*lk*#;?MZ8EB+
z!Z$JQsO{2Q+_8k|pU4%a+M<FGmW_lpADs0B%HI92D1Q52diRdv9nZUWcv$b=m6QEn
zdiM_F8%ECmZ|3>0^_}7W`oi%3FY@3S@zwu%d}(Fl!!k}J#o;hE{f@(;T%)3!0VAs*
zf#q^&^S#3}R;|EDZ6iyhsF7IHgmugr!GPM;HH2;u*#Bnxe>5LVo1%s<%SesR5$9b3
zX0CB{B~A{(cy%g01h0}JNeihAB&*TPh$YDq1e4_&SE#X?MgKqH|HY+DzC=o1O}-=^
zPNOzhRyEZ&Hge7gp=Da5iU-82u200N;eci3;%H&=Dd~VJ*|gU`zr%Y+0etrki-LzN
zD;8*^%)tR92G#IsYmyq9CJ92Td&QLbKEtA{-I5@%m>nI@SZ(AFEH*($C1h4k1F1HU
zhoZ)mSBnK^wq5<59b(*IYKm2Bs)mPw!&S%$1R8JdfYtOQl6VZ=)>sL_NL(HsRX9zS
zaa62X2h@cElIww2UZDg8a$>`g)9j#*MK#J+0$x0|NC(vh?Enc}WHng>GQCRcouQGf
zG$?4XQpuD~u%Sv1%*%^3@vPhFgOry`%&1rIV0^A&cbq$e>2JFUCb{WBE0$p9ri`1%
z+wkj`dhn^?&U%kkzzUoV8)2EZ$(adnG%LMqUi(({xw13a8X}3L+sc_%F&6v=rs!Bv
z?)?yG6)TA(Y+77~N+KolzDdm?+l_doMpJf3-d)s}G+}-tG}0y`t<qd#LDJ0_mrEK<
zRfd`pn%p~belY!8TIuq6_4fPcoW~drT#1AjYhjdnzb~9HT#3fYV9w#sj}Vqh=|fuE
ztw+9pvr&Pk0BX}p%y?-T7ec3fYL*$9#k;ZE%O(ZUUF~z~zDh@mVZhlPW=24wCet|0
zT5R{}RizQ*_xGU5*y5^rreI~SRL`WNb6bLQuPMuu2OZd5Z)R`mi_LuHs7#-T%Tc@u
zN4{s+gL6D|%oEHbAS@$=L!egsG;Q^VMP1GFha78jY%JxS#6Wt`>*lxK7y!U_6OdPP
zdF@;!+-1L_>2RY24NR5k?($k_!M!Fs`Mi;J_uS^67s?)yGET}=uh@*LUfJz7`oh!8
z%8FChd8PO<O=G=rkLUw23TLZ8;2FeMUzU_jH#jaD<d0`(W@!`u`Hgm!X|@FLlfx#2
z?n@~A1b?2tB@(K-6U7_zz#FJnS_A_eYpvin>avNCC3T2#t?sZf*UVCL0hFjTTyp}R
zZMwESYr=kQ@d>9arCM|g9A(;}pz&>9^mWowmEUXU*L@3J<a}sc3m$HNa_w72EIDQH
zKK7BXi3yAKxO4Jf4CD>eYc~~6F$*YHHtMt5N8Q17N!beeLmgy2%k23>;)I0RkHDkv
zT?pYxXN(ot5&XY8W|Xf~5aQ?H?LoE~hFw&=Yb*QqPKr^Pctq(_h#WiAJlasllIx5w
zFKKuH^^(Kh5dNk8M%d$r)P58D3vwegWG}|}e8n-YSWzL}nteY)s-RcwiGZ!C#eX+o
zr6)u!&_gCzGXw&-lVyb?f6Huqakm`}JTAl+7W=XHFme5Zw+wWc)DN^-0~fT{zKT`@
z%UO@Se{8!@nFU?-Ddb4CHEc)(nkjuaeMQfG4N>9dAQJq<7hiPllPfeM-Wu8cNoHr<
zh@drTa8qs+?*#7htT(TtW85VVMn%3iOi|=gdA6_KkM)@NPE(=dlvO6N=p4wW7v;UN
zAX8r?JD;%)bhNH8A&@`(+pjWrN^&yK2+Q?YtG%#Hs!1-k!?*@1_6oPuKZTY^MC<Aq
z#*(vU_=21$`ule}Vf|kmLq8XGEI4(yeT0qMMTvkd6~bN1@?PWy=aonpUR<k6)z{qb
zYW~{;7kP_|>sz%PvG*>S5J9?-bDi~`l317B37~pxL4D{&pxy`FPmpfh^BN`6Y^Rt5
zw$FoW$q80mkMsI+M}<AM;83-eptOtK@5QU~6FwDC+D0^7aN^Lp0DRr+ck5K-2~5*2
z3K1x@L|224BdSWR8MAc{aq+(vPIL$;T;~TQ`-B^0ln!h%`JSP-Ai1$8h8}!tPP$J;
zNxpP0oQ+8ehC5fYst&Q0S~5LoE1&#A_a#j9Tvyk}d{Dj@wvwIa$pA<TZ?_EehEXW?
zF+AXl=V5PEtiMojaqs&f3ilbBQq9aV{^+6Z`loR;3HkSBRfVrEB(q^?{?R-B58mph
zk)GL`nH*fmsK4tj^cfVXTXy=pQakvKJ@kBTiC8|2=930Rxk@F*?J;Z%iEWz~)c)4)
zQ6&0k$8a$Z6Yh*F)dVUEf8*}*@Dj88Ey#*jNM=%&Z5@JkHCTL!OzPZ|ai9+#so#%q
zEIzCCV?@0S+~6tC7BH#gNj86Yfzr<?MEX(=pK3A5f5;mwnn)7)PMl6Cj_r(EC>D}9
z9VB|HuxfMb$v&zc9Cr8vKN0s&T!vP16tSCTQ=SEL3{%b)+aou}8LvDQnmCJwd)BQu
z&M?whV{T;EeqZeKR5ipA%I8xs9@$KrM(Rc5*krKMb2@eo_Gl{c-OWdE1<Qsf+x1WQ
z&D>B)kr|3w62gPpAYASZ<`uWdrYydo@Vpd!8%cQ8OyCEK<oOq!ZU1iGBBf%xX}cdP
zI=%!)&!li1RpZ@>NKwc233ewYkZEdicCOp_LVcg=CF87iM?#geo2twCP5+`sOI2MD
z#WD6UvO^*#<Y(xh%f~g5A8=yVE`cpbOt>lQ{cbxk#UI7_QdK>-{8IM!A8cDSDe{?c
zBNO*)gg;079opjSHE9-aIXq#$4om!jlgEt-lP#FFrd0>i80ePj8K2vzCS!@)eJMBu
z2rR|W!1gv&G#!>F*qp5VjZhil5I+(#tD~#yXLlFURO_X`yeO4aDP%(`f4btMk%HV2
z8~M}*fGxpt(9g|13v(qzh2z8N!L-noNHrNC*V&6K7jU{8+1CF*f$)RV@Dxu<JB5$w
zi@Dvb58^v*4T-Z4wEo(c=N1I%U6$p&u7yV~+fUoS+*x>@+0+KSWkKs!-|JFEU4P_|
z0Zc}=YSuC%7RHUBt4xJ0OL~SCHH<xES2JTR2JvhPLRi?d_>6UEPa!V{jPeS<=Q7*=
zLd|$u=k{efJ#PLr9y>svQ$mbcx8`9Dp@S?!&j?@gSTSMv@~r2Ws=li%ZK32wgI6J`
z`%7Q)L80`NenLqs(OZW}4<@G<eha<|ps-M=;cW99XBZdrUmf2wA&ScYSH3qnvLAui
zRHY(kC$`YZ&*%6YVua(*Nm?K$w7UELj7gEu*{G$HA~lKTOI?Bbe&3CPz5?ygZC41(
zv=(D&iZ>g<m}pX=Y!yhJxF@;ZzN9!^0IxzL<E7aoKg}5TK)dho2dCvp`br>q+*=(^
z0JGm~l9$H`@R+S%rfD#CYeatFlJZN%A4$Ns1A~IkU@Y5Of^5aN$i{(ab=^sM+2Hu}
zxYB{2q6&sT_-Y}KwrxTd>u=--TO3*4x~FjwtkTLq?6Lt8*kqixLp23qd@dTp?Gq#)
z1=j*Tg-@peDlfXpt&Fc;V70=m`Z^3OuXBwK4@=9JWH>4Ue@S!wu4qm-z8SFEcx$Gf
zL^jYircr4KT;<h?f5@}Kn4yf&C4G^TkZwzsx>_QVstXShcUt{FkRsz=R}hv7bJOsN
z*nNbdy6RJ&@wk@3CEtukLK!HtaNB>p|M}`o^H*RC5CDtd69U6K-7`&9crO^ln}4Go
zAKn!Xv~-a>G8+x``02{I#W)u)k;K(T)SGS<@ZLEFJ`Qtx_X&MzI3-54XOKR06KHjB
zsd!-t%S+204D%e>qgose`kKf)b<On~r9#DzZD-%F?GS-}KYN5?DZvgI?3WIC;fM7T
z*^J=abYH)pg3%-O(X5OE?@fQfINQ#GJI$N=xn)1jY4bfJ_#6o@9>PY}wrZk-9`Akx
zFgx;(z&*v~(oIIT2Z-BN%g^t;Vs8x31A7-W!+Ta>>Vy+VabB~s8Q<Xp^25~&pSUv|
z#no8cW6DInYu5Y%VSB{ShL?O2AJ<<<6fRn8Yh@Oc#xad>ydKwQ784XU*@&qYumCl2
z#g{a6j}Rm&vyt}{8~9H@QE6B?3b5m81Qm>h7XKU13K>{(P<2=|6Q-Y`h~T{6_|Ekh
z6Of>k*9H;9gT^zRm)0ZidM$FT;uD~;-9u~x>wMOSLxLA#GoJKUAqE452%6kybzG87
z)?&r;imwfZM=3SAsP149fj>syK4DI#D@Y=;dP`f{5Te6uu)_t@$j0LPs}^s<pKcFR
zEQ;aT=EzY}t4?95c!ruE2*(Z1l!|kj6+KVo_H}#`Zn8(c{)0dsuEH^!TmhH->3y2a
zKCAN3Wg-qCrF@!nM!KdDzxFz)1rsU^CGaNOf2GS4DC%$GEPtkbd&13KXd2Z#Mbz)v
zj)ZJDUT)BL=7xN+ReA55NvI89$371Yr$sT0l?eoQP%AOeD)tZ*(~remu0&xm_eQ)d
zX0}Me#cOEvzx1Y88lLO-3kPDcD(8v9LjrRU8OkwG&8cazkRbDFhySis#mV8qOhgaG
z)3bF}x1U_1uCBgMr4z_Y?46bF_F$qv15pPmnw*<DPy3wF6Dfli0ar+^5Bki8iUr1O
zO(6MdP`!yA&zT9Q^x~{17ern+r^m(d0s!-t{csU&5F_2=dzvmP)jQ2a$kZVJl9}kI
zSRfLILv`g>{Z%jMr5Q@H-sO_mqo6fa=GyH0C$l@1WZz@ce)Biw!Rli!W0ndQT`t;c
zsb)g+oQf{Tg**^r3$vIm>_$=00CCN8e2^~As5G!)xjNx0JL?by+=3%je{|K<w~L0H
zhi_^fc!cBTUSWbL7MO7|(cY1m9oy<eAzFMa5iyt?buOEv5iz{Em#icUy{j@8VX;rn
z^a&_Ws;`c03LmRT92guIS|Pxuk5_CzcK$q7O%njSWsP6!ITy+szp?_iE?lXUPh2h8
zhVY@&Z;@$tv*15LLbm-<(K#c_HDSe&L>)h*E2g+I!QG|(5qmL0$<38jkKKSG!WH>Q
zJj$L_7SoqbEG(hEwy(H^Ub-qC%yjS{U9<UOyVrWs|Dx>5+pU1WOz=w{K2$3xbHe5*
zWLypx_mt4z4iCXCf}&<NFj4!ocjoPH?UnsL|6y_8H~o=ZlvxzVnBk5YW<!4tZV?zf
z0_@G%7ARcC6zA7|w))K^>yq@v6)ojS=jyt^Tg20+LC5IdC@!bIiYsbubff6i>LBTz
zz~~Be%*{sM*p*Qem*#=*4oXu?5^ZjLYMA7sDW<vSHMFIiA{ESAa56@jyqqQ~$4>(|
zi-*=c$`DvlQrsW#y5)YUB=qDYr4bVqWUOtiNZXa}MT%|B9k>UnT}Dpmn?|(Se<v5V
zv0h@*P^T+nu(EU=rGq^GgP1Znq*%0e^k2jgO!|7gI&l~aw9Dj&o`l^a^?x|_7Gs-$
zqU`7kHmYk=^D|noKYcP=JO7>iZSrqmIEF8bbu}lA*6<~_Ci0gm(~kkWnTu{JJz}eR
z&mKjZ!LPH;dIIBEY6NqGF@Lv>{A}y*%kM%n9SIzsx`Q?tuAc4}sRR>Zr#dKcy&UN7
z-oTa#6uRbZ=Q6y1;Iuj+>r@*|V?;GJ^{wfgr53?S1abAKR+D|^9u4|#^eXIn=EjfW
z_uV#-ibMt>?lL6oRzsS~EF8l=XI<;HsLRoF*%?Xl&=|C)uyx*k<zmU&VGPcD<n+Dz
zHDpjJThCs7-?LptK`8bcbPu7cY?LXbl9#)_`^^v4Ggo|;9z-A1eaAA1avz4h;h3*A
z`i4e$o#f22`DNs0s<t>>=5~4g_Apb4r&Am>4avKnx3<k9t@-d=u(;F>{OGc~OfzzT
zKYL)m8@Au^Cz`T!q#w5SU^3C6DRaVQ>vp-VQ0hQhqmvRvR4iS7UuqtaYvvsuDYe>O
z$ECpxS*e?+%x+-@^{*EX26phOdUtvIV7lYi*)E4Y5bxijcKpc)D8aV}vLOKmhC426
zF7B{ZEXl&c+PM!hhrV{FghsjUZwUwI@y;7BY2IBp;4grW>tYj%(`q~tx*xArB@)R-
zp*3oK{w07%lCN$!!Lsta|D#rbSXmBhmQIIRPyZ?$drk8fiXlMRkbn{mq*UDJ_m=ko
zdF*BOvJWg*XS6F1Fx%C6C;RIS98>yet0m(X$4~A(2b@ExL`FRZ1Bl}fb;a8vnc>@8
zBxskOq$RW6CmGS55t6!t>CBx?$gSR;nU*GhWHxd$L6F6PF5)eG8+`0G{eT6PyGHi=
zmj;UHZ3(80%9)-cG>G%P)!V&vFYz9|IFgSwhcId4rPT*TtZ0}>q_NN)>tD#0sANlR
zGHCP*n~)(-MV){ewPp3D9uw&ifB)g@mk$^7+cdZ+=hr^oJytF+n0|v49Yxhk3il{)
z1Y)S>c_*Ei7Rcr!ct$72+7`_2E6+1)0bf}c@{gm0-o{}G`V~RpV`it(*b}u2bscI_
zPKqq{wGV3gI*}$lGrNj60?a_QI;_Le|G0~FL5s|IE3o>!!m<^5zWs}1aFWJjWX<tS
z)v5C8!pvt*r1JPE>Gu)@*zTGlA62wcr&1kd=iNOJD|PCng1}gtEQ@b&sxEE*G(5#T
zmyt+IgP4UGBmSX@y(~tc67pv!u?3+ONy});M!yiNldhB*k5RvoEZC--paOLdUK1pm
zX|m~_gS`RbG(Peo@21Qs(*t5j8yjFcmpRNCpmZ+2e+C}Njpeo-+EErHTMklxcyg7*
za9n9#6%TVa!V1+I6-e_Adns9H3at$s;C)V6=sxUWAHrgdAk_#3Q|FO3iVxEwz%_zO
zq^@$kM*X{U$+A1cAs}1~t~@~VMN)HkDoJzb`99(#hNICED)%&7RL>i4Lp5s4dE$!E
zFe4eXGwo=6X|_?xCqADOqE|e1+gB7}7Fn^NRwVp+=e8j{X>w)S3oWZ*1-DYw*-4tr
zGcSzEC6dD7=G2^UYR3>w%rtTE3d0Rnl~;7e(k{%mxe;w<D%|`(9VKb`r2@wX#ld7p
zF2`hyer_o*Je41*M!ix>Q{UByP?bxgUYlCv`P8ew*8W~jbaYJZx2ty6_WuQKo5|i8
zkMhMLT?`J=+S@rE&@=;6Ruai9PG3m75B-(~8V#0@8uo2;u2li%{KM9YKaQD-aOUO`
zQMtf79L;IUaalBzt)$8dcTq8q%9(Y#_uO^fw6Sg~9LL&1KU;HY*0uAgSv7~LCc|D(
zDm+=W^~2WRuJ-~gmYpSKA$5{4o}{V;VY>`>1hWTPb051>Hsl{zIh=9X*S08YEV6oU
zul_ja-wrEx?SF_~Qj1kt)!O=_xk+pFzdz3Yad%;^p56Z23k)upx?__qh;WseCfiQu
z(VU#>b3+|`JDqB8YeR-GF=CD=3xl{?T5voY7UXXj!`XJB%8!HFY4qz-R-u}w55Yd(
z>@HzAXt{@(%#IVCKCwG=vGeQGJUtLK;;C%3_?JQpi6I|O5iB1Is}k|jo9rz`ixGq>
z)b>f)cAZPo)fxEr4MfHVcb}wKHzuuqBvX67-rwNDj!@fo$s3;|&cd1WmzWS31_`S=
z!pXfPj?vlL%UQEea1?TdqY@#7(F3U2j`Mnq@tO(mzLOf657lJjLARK@?EILQVy+v&
z(Nv$C)|rttVw<(-$E_O8&GyaSQ)3X~1K3S1@(IK`+q04ul;|ZsREaz}{-(^E)R>xm
zQW@_rX!@!i_Qx&E4<L)zIFXRv#-}Y!5J&{our>(bcjQ-0Z@jS~xc2hpFfMIr6-VcA
zTlopqgcrfm#D=~Wdsf~{Om)O44mPJ6(tPc?rU?GvrFFn@Y<M52obJfP3S2mpYvRwQ
zAKVf98%FSZ>deUuyCn6sVZ=?I?_*#~;Vey_GH<f)oA|B^H+l|lG%`@0_oS(m+-$PE
zMa)G`(XnX%jqWt<1GmY92FW=kmdLQ_$SC6&RTyqPfg}e@H|g)ypmu+q?`toftFN&~
zGJ>B%=<qi-|IbtENb}<TiDW%yGw<?TM;-Ozwf*B4p4vCN75fhfBpyab5#QN;V-UFk
zW>0%hd6{e81giu3D(8ISO>U-kG21tv#~Ni@S*6p*t#|EAOzx=ahxsyVwtIzrL%xm4
zBUS^g%%YhR5rK)34~^)J*3{jQ+xX1O<?z@uW?|Oo<^Af`!8S6|r<slI7ysjQhW$z8
zFAk@x5S-W|E<lKGL*(ifTa2m3*?7ildzj0_L-&hhj2dZiTp|2`;KXdW2DpG5oc&%f
zlobd1*R@Lp5G<C4EG|%<+i0*o{AD*V6c$qZH}memu{-a6#-#5TGFbGavan^<+|4B1
z_VYRl$VtZJz?9{l6*k{1?g&t!7j>j~H4!UJoEriy08SYx7i>ACJBNyP${!|Ee3Bv2
ziB^rl?>I~{mGW}bgmRdh?ipU%ehc*2<l;XNU5dRuD^Ya?USszkJuB!(J@P|RK+dj~
z2&%nC=Z7lXLD=Rux<2_BM$l(Tr%3IoKFQ|=&<g8RY`Rj4FV>%dbJQB5OCyky@N_<W
zgLY_sram-8%g1*mmbtBtzDg{1l}1%$<t3(Ic53kpp5-;>(#11@LgNh~#aW<gnAs6y
zAlm15e)1T<2Wi*VVaSH2CB1Ci=d^PPqW&3s5f#be2?Zl8L`SSKM<rg{UtCq!;*#Vj
zme8gW)dh1>ZzY!^K7nOhK<{oP{>G$|9(Tj1KuxA>bg+}OuYs-qumQc%j7F&(<<W||
zZa-zMuCzG6fo#<D_ScQH%CxTfdu3}nFM<}?<1n3+bzyN;5Y*-DYaiYp>ZqP54I0~V
zvh`&1EjyT{p{mW&03bgEBoWK2@A(f$$lrctw1W%Tm^v?9>eqrV;XubaU)toSs{5Rl
z^B?JWCr{m3MIO7tp>o=4F~03IesOP%j(fks9;aC$)<H<@Md=i#_{t*V?*BL{^D>M&
zNMpz{7dWIWkXkXXUx?n_LcJ)f!Ips2?*%q&byQuMISOGNOY_gKa+<QWHzpYTt?F|5
zFG#R1?pfM?NfGgm41MOWr4qzS%tybIx#ynrcSC3MT(_dc<&o&c(#;$?b#GFk*AdP8
zV&Q?7558DSQs5F-|KF}ScV>g_xh08iJv%bBzjh=-{$50SJFZ8`sk8bJj;5>$6;C36
z%HfL4oKFDXbDFZgjR&0!jC|~LV2s%0)+fQ;3B}N~Rv|H<XKDR4pl7><=vbTCMMcJ8
z4fPH)-Pp$-wFcC}q*ViS?25Z<3RdX@UE8M)>ezVvQ*o35S_I6DhBO)LsRH;GIVoN#
zikDVxafM)!lKo|tf!*eL&bkduy}MEJ!^!jUFw>R2&$TUt3LJewC#rp2xG>kgV3u_U
za60Xax8#QSr2Y<F0jT1VyD8yy^B-oN@ox&P4=Za|j|NZ<QA72N(ml*Dp&ReiUkuI@
zyyWznekd+)Z@mUgfge<2kiDRCC4A*SraCm@8%EA@ov2rnB>yRY;8<{N<K>^lCvoLF
z+5@sHYVqKUw)@(<_!-;I>>6$DR<Us#J!MOpuiQhJLIvX0Sst}Q3O(84=o#?yQ~Z{@
z#n^$5(ZP3%03_tfIIXL!Q0id*V<|CJG)GGE>iz~W>G6aISZs$<n$sW+$~@Ve49ZUP
zLyrOyG%x@fjuf|3-};-af50Zc7dJc3k?a4a>*~<o(86F~?Ec#vgM&uUP5R<!jT|4p
zUkZ6N_K}nBOlIReFwYI^!Mob};K6KYl0%^q+09-Uxsk7nQlKrR@yIKg9NYewX9KP`
zs)~_UNnt1MzMME(vpDtT^_i8HI~qHT6V5wC(;Eg7$ePn|$Cp?OH+;Qp=9o7gUgkoJ
z$m_M|npc_A7;*U3bth=bI&Kh*T`zFq_Q%?_q{w|&2`cjL@Z!q~4b+%u2^<Wa!y=gw
z@!ORrHD9|!3|t3iEys=cYNf<-fm~*WGyHL40OJPmM;e<x%tv!zL4AJFE`tkOK#8E5
ztb%v$3tJni`_|*`A+yqIkMv`(6JlnKbfr|@&-}ZDbdYAcQ%^9e@#{c83p-2b?T99w
zlIVjF6J7VRIE-G|N3y6EFsoQcGU{f<SJdxR<L*D2oUDmFWKfCvv$Ik#HP8P3GJNgs
z=u2_Ep6D}RRf94}Nntv}C%N>fbSFJk=|_N17se#aM!t2$g6nhSo;(BKvP`=pX^A5p
zgUdD4!+1&DD&b9NhIgUi^<F`IL3C7HHZ7c!XlUtu561`E!`oTllI_y!otQ`x*kodD
zV@`1e0RhyA4Cp3K3M!vOEf!5ARtb2*d(r)o3C$a_dok>iMl_PdoDI6W0A}DMuaIaZ
zY8l4$$}($efA^C6@qBRIWRT5FB2_$a?%-jpO7dt5Qxf`|U(4G#P|R_=87*Z@^yb23
zJ%sF6TYF6K<p}b2jxuhrL8dK8%}^$uAv+z^T;!+ipG3nFScy~wYxk=Y$HjD6#k%~x
zF3)FQM5Ho1j_eL1?=$U}DM#QDU_8vZLqI-R`HxQ1b)IS6<Jw($V2l!fSZAuTi~I7{
z2z<>PD3uYz6Gz&g&OMdmvl1StwoaYE*7GMptEb0$t5-a~QNy#p>m(#O);b_EhU}p<
z1DDPxx!O6qx#8Yh>d%ffqm;ybW=1GE7J<t)QNp&0(~l-(3Tw*VDzW{Y!4^!!Ibw3b
zrq7EhgZ7}mCftQvZhfs`(QRneX)wWlMpigmaXDc<+5uxp!Q$~vxlvr}B%$YPP%tXs
zt-Ia}@mxmd2f%7@`YPMi{QThg)I3{R5RA*XYyiUbCphs6&5ia`VH#9#LGBULM4CIO
zD~-A0U#l(g?b-y$)A&!Sn1Mxf%@+x1Og)B{yLUREMEv+Axd9~@5??y#;Fot>V{bOZ
zLuKw}z|LPOjG+nkiUX0;|Iqp>6X#B$zu2U=TCHfv4M#qv6_W{!;&J*a=f*>;{B!zD
zPG6r_l3nSibk}_{rp;?jg`IU-EaxtJZ_Ih6T=X=OCA&q501o7#-tCLGp}bgeW@^gz
zt3B!uHsRE(LBD!on%+fkLlP-PQ8B+DwzHo8)q&z&{{7)G93zPNMYP=%TY=AtX;Pg*
zwQdaWs$;rF74b-lwCG#R)AC^4$_?HlTOu2MEFLk6*qsA>rnb@5VnlbfHW~6w8$GXD
zFd~x54UQ|5nJc&QHEE(bIOgNc=2uDjDA^Sgp1&RTBqpG5)>J`ZeN~Y-lAzV-D}DQI
znC9%6CL<x(qmuSxJ9;CkVsXqN|EyjfogQ5;bBEQi{f|(kU%zVk!TM;QyrR@wK!+x-
zD<C5+WnbqJe<0&nSpK^X&!benSX*0N_W?_;Hnr~b>%XFccK`Lul{kko$p5%h$xyMD
z_Dsl&Uu%-zv3=6=RP+k)PsUur`x0`u-g*aw_|xArv5RU&{<r>4#mI@oA0N`=EtK5b
zeA>}0(s_tN$6$9=rX|5i`F*x{Q-2)Bv_R^fzcX#2SfaEyva=DjDGSSv4Xh}_a9~zZ
zwUf4Pg4p`j+Og#-W`m9Bv=owS&Ny*fS9k4dsdMaHnnv;+$$LhM#g39e$AEF1eWkp!
z6{j!sPxSJ(!RCT-+FYLqPM;z1?*sj(DrCiAE+vODx2w{DE=gjqB8hB9lxNS>m#p-$
zZJ~Ux?t`CC*V=to0+(qqJ8-##7Ti>3sP$@ZPbA3`&+E=(PjEbf-db%6LHHw`Tjuio
zT}wvCp)^B&*C{VPdpGuu^>mlCuJF=rpTXnpy>Zg46#STpOx^!suOn3Wl92!XXo8);
zi85z@LGi0Z+JZm`7tP2ba^+|MOU10N^-rqR11)(Dd2v~p9~!yZO3EBI*<YqC+)`#@
zn0su>wiK<ShIiC;01WsyMI%CKP`>iHXu|l7mQjvRfu{ya+<hJNTn8#VD=3=3$2mj?
z=d$Pp%mR0La-Qy92BOxCN$hz{o<-CRSehMNgRFN!>*mvb^5Qyzkv6w+9x|Vp@KZcC
zC=`5$#~hXBD{s)3j6lGH#DOwttr!>KCzY_E{oTfu#$4(i+_n9Pn7|e-)8B!+^K#1!
z4ycV&O?g{uClO^5c!Z=G1A#pBA3jfZAqVZwX6$6JNKF<N(`dSmOyM(WEg_aK%^HP6
zA3lnM7PtE_B-T~xzI<t{FH%^G`jSy?n^w5JQQV2Ewmab9tcg}`pWA%BqctmVam`mB
z_7@ahs^bf_3f-x&4{I-f;+PsDrr($q6P@c^)yO&s4=Q+yeXlYq7<^QHHiybPi;~PO
zFriE-a@<X5`b`?p7n)rBB0=$&&d!8Mobk!?+S>OD>YNvNk6ZtR$|2Y|NX4dpd2U{t
z1pepBTR_eppjO6?1C5yp)MgL>_z38NU6}zce4Z*b?k>+$Y$)AiNto?_EGrwt7G}yC
zESbJ|zIA0;)}@o`a#r>fdJdlS96#jj|5G~EsEDKZ%z3zV{Z^`Z&t&4w_$)GFK>gWh
zqcCZ+iGEd3;X=dDS-KnhoML3u?~A|I-`w24UM@3njRP6qajgPtP(g(3X(cENFY&X1
zwXREq7{t4B^BZR|CD98aY4@0Sm^<Kq7((`6@(R?LED})(xuIk{a(_D6!I*&qC3x|%
z75P9tyNhL_qUwcu7+)ludY`XC%r|$wX}NkFkMKrlOnJAL-A5Q3GVv?n#DEyj)Z6Tf
zJQsz;9wR^r$;SKYQhL_2Rd@YjRHFpn-NDZu5J^?N#*$Y!>K$y{-O4$(ii7flSn3=i
zf?H^<2<eaK52DVZ;MW`(Cxso^)8(`AOp@_9aS3YeZnAoJ%J(Df`W<UkXjv_^hG{bU
zC~kt&2On|zP`Sq5ATaXxnL1=FsYH0%@jE}sNDSGm2_^sZeEZV%^V6_+Z!?7+->I~u
zUXU=rLx${MC>*CRu-VI)P8DlMX!^fXZcX_ZMO0K3gRSBa=7^bOgf9ku7;_CBcx)<D
zTxiP287R@@1V(O3&HDR3wm8kRaWFA!NZRmtXlV|`qs`QRzv=<3HiH+M(*&=H8hz6q
zUyr@b5z-R}fL~FR!C~*hNd&H=ncnS(m065wzi_1a8%&1%Zm}O2XTYZ~Er1?UC97JZ
zBg4X;EBt3ss;Nh%&)qx)K*ccSRjXVl)K!Ix$baahl@#<Q>bClC1DqNt`-GsQ-F>aB
zOtKPHR&y4tN_s&R6BDE2nsrbvM6=4iXI_1>-cut_np?{F6_6E;+%5Fd`>GoNfcr5?
zixaa9Gz;CLT}MjRg^QkNsJnW<xZQdPTo=WknWw(AuX5LquJm1Z^*ToPmKOD)ZYMXw
zxw_Vh?|#Xxu*!8tOa^O-I8JrumJs@cnhh!Li)=a1^4nSat149C1N?fgjfF9=_0#(b
zO@VIk_#=N`!pr=kpH7o}WB;*Fnq>4!Ltjh%L0g9z(b!y`S6y6wJ@7>(vaAQJ@5({z
z7mEmF=3N>%gLcoZ=cYF;3p&DAt151cuB^c}k?&#ik@@mX!XEbLjG{-4Y}E$qc4P>S
z1vN5>c_ljQtqq>J%hvvNqbE9=UOiF|#hl=zSI_521Gr`03q$R13u-??P>`Y6fi<W^
znaf$2gq7`^ChMrX4IsV_Zsr~H=#;q2qE$RT)4ddu#0mD*Q(}m=auoFz5b#`)0@WQZ
zKvzu|p^*PrQo>)qsLr}&ynO%R#rN$><7&WyseV}|EWeHIkA~=goik;Nip*)0zVPRy
zlK{@ni|E4Z@PmLjkNdD7R;8<1qOJD4fHDCQt?&0QLZ>Z)rFM5H?(wrU^j`0`WtQzQ
z#D>h_@lSylNQ`5xe(8Xhdll*aQ~z=k{bI)&K{u22w?EWI2Pe^FotI)yqlFl0%ATTv
znHJ;OM@6$ubrm<3GE-38X@^tCS&_`xj`T!sO#==@vFQ0-uKP=n9>p0^kVp^(z}7-I
zvZ>w)E1d0!Idwa~sl|dT@Y~<zVc{*`ir|5(*fM;$JPF_%3!-L>D%O&HG!*vL6?E2~
z+|RtQE&1>Cjz7W+3TStqq`eB0rVcp8n+&5^SQ6IBYYDTeLUX(8O!PcgNzk_E7Kp>y
zsCdUj)}BHJ<;7a4Ta^2U$#FXSWLZJodR~^x5EUJge=YtuMrS=zy?a=KNWeY5cx|bZ
zpzQVez)RYkY}J?MR4ETo?nTcUlc&5fO;>$T^ZdP2LL9VGpVE*!lt{O42j-1{R0{<@
zg_N@GKbJA3SOENS{5I)YJ-^_SJp3lwwbaw3x)zE~{7CP#a}&{2J{mQ9>AbUfVg;vo
z_#D#*!Y7sc3jU+ZHicLodCB7HG;bw;U=$#pF6};U3DC%Id{19!5h{8%&e=p+1%k3<
z?2V*OblfI`M`gDB&6v=h@e6=p4e!d@%lw4oab6<kOtOsguH~O?S68?_wm}xAn@+mL
z<7Fn}5zh%d?)w=#dqU*Zb}Ra`x2UW8U8y8zm9B98-(3^2+P8+qtP%RZaiF`0Z^F5z
z@eN9O)t-6x_r6sj+16S&pX`d20EQ~YIPv|~Z%Gp5h0Y6FE1q;qF#^R4usO|AOy6E#
z*HG39ei3C}#?aMy_Y8SEP7I0stZ3r*bpj5{I^EQ5mSo19jq`6`57gIb#WT1Q;#2qq
z5eaXgpEeREnL(^<_>=Zgt~*JjY$PjZer1W9U6b+zAz`e!6{P|zyO+H@OHWi@pT~qE
z9HijI&I!ENfWn0At>)h@PAcRQYJOb1W`>MRbBJ)DsPCN3)$a|d1)`j=9h@ZA7bAb)
zy31wNFp<w0gFU#7SJ(6s00ify%<qHiwy9)MeuVs1nx0z@l5p;6Xlx7ht#q32Wxc^h
zBT#J3L|uLJwNq_m+_ZY!$OKM{ff`aA2TVwYM!sLk^Y{-ZwCeLHp|0G%<X{_Rm8iO~
zrGKnr>Pt8S_mt0oC9#k4(ot6Y2bwcgF#L6UpvjlKU&`C#b|<wZ_nu0tFXG|n0L(+R
zF|(w1cAwh!AX?yFY=2|3oo|?Br0V@x=X<VM0tlWdMTOunj4<nBJ>-NX;ox|QRk0yd
zWHo;GqvPAed;Zq9wRQrfD>09fsGD?(f*8y-Ijh+PVY5v&yV50`HbTm`pal<FT&vxD
z@vYNA`gO7`kZDwpD0wX}yiU%<J@k0zGCg#q*;}d3Y0K7PwC8|QW{`Ckz_;gF3zrk0
zaSTU|-TvCMAac6mMF?bs1St{PJy}WTx-#o%L@GW?(4B>a9bG<9+T5KCK*eW{n!LLm
z)A=QdiX4V}j|+@wGEV#LcnDCO&Mb2amd#^`Jcu2ByrliO^N9)8L+>E3KOET%!(Y2R
zRC-u}G_9$diO#zRRGut>8Fm)0S}ZLxH!`?ljj!wKLcgS!$d85{nEAX_Iu}uB@&2e;
z@=3`#uKx9Pe>)s9HmRPNYocTewK%Bfp4>K}U;fj0!O7Jq9dX%*;qYM;>uQ?YJD_b8
zyr9P+M!ghCJX>@=rV#6HT`&G{6w|>Ql{yT5HG<v;Cb?~334Z_>dK)@{8AOY}ZaMGM
z7?d)EHjd4NNDhJIYMd~GF6ku^`ac~Y3=?fo`Nn+xIVBLNC$=-U<;K*{9*7Tf%*6<;
zH)m<K=k)OO)i_JGnSG%G6`Kw%GjI_y41Z2*>`SfB^7s%I6(V(k2BUn8SC5a*7<HD9
zXLnb9j+q{3zX{hU-9jZz1ZO5hOSrd~a-A>LG5ZC-_J;-1u?fU9w}#8N!j97O%^I25
zq9n@LJrhqcFCw3VU`(vbzdjI^toiL<k11<wAW?S(vd`l(m22|@GQdu@&OJ3SX5-OS
z)l&SBv8mRGmV(!$n~xbqY&PUu!eqC%Y2S>G_^ImQw!I^rTWZMIK4FdaL5$BB&gFdD
z^0zBJ*5dE=IsSdF-3V1{(+$V<?{1m)XB=X}xHcNz0mD5IevV!La4lzqUCzjC22Xub
zVEKY`d2z-dtNhpcg73jXNziubhNt`#kc}2;oLuOnSra|^Fh8kWI3&kR(bHi~=;123
zJh1YGszx216GKfjHzAXFAz!2T9F$`oHx<nEdYszeX<LuXBPi9u;#Gq#{i-;7K7I<a
z?M_V)_l8e&RSD~_zWLy8C*Oyj=84ZFUjKGYoM+0hy)y=SE!7PJiz6#JLj=3e_Gq|@
z1-x!2lpK&kC3Zo0cN~g_yZpw_$+e6cyUO1E<7$q({XA#-bSn`*BKb6V5qUowoR+hi
zk({;<GkKYcsGb2cgUml#7wk%9q}WFKPrSc=x+cYYg9i2!i(B3L6oNT`*b(hN4YZuu
zcy!PkwnEfe?ftWIW_N@bMTe)4c`1s+D!`o6l>R2l_?_b6wF`GE*G^ZvJ&NsCV{d1A
z8de1-L7=ShSU=AvpKI&)Ey^>}{lh|0&S7sR&N)83&4TMl`iBND0f{(EEVEQTFnXM7
zoCz3PsaYQV)jSv<hfw6+(yV`_L)x>=E1Tc(615MBU%v6+9URTH(?^#II)Zxk5($FF
z{b(bLK7V#P3*W(3T^bqA<dPmCR;%VLJ!o1^^l+3fF3c9WG(6~@ckmyHWw*EilcZa_
zJ9)3PIp4Em>*OeL1gI4ZxGMREGZO0;^W2sQmje4`LQ9`a)LnM2mujN(n0hgQgVlZ#
zd0p6xqwQKY;v!YLk`p`1u*BoGcHN{=v+m79zBN?EM?*SQSY|bTW9q|5W3#!IB*%sR
zx}_7gbj)S6Jh5wDeVD^rP35TCJ)_-IliQ7Gux8};mu|nuzLmg0bg4#UOU-Z}u;*%L
zdK>I2ie*P;O!GAcV=24~P|IEz1%zU^o|82vmeqNE2>Del)>5)L9C6bV9M;4nBMGjb
zduDv(YmdPc*AEHDCO73m5zK8tYM4tbkG_5T7p=%edLWR^A<A;+!PZ3t96hztwwr2Z
zb~SROuo{QA#WK9O*Gp3yYdF?)<lMq#YnjVHV$y@Q&t~3zS{rMzw)pM6SJwKkZnQ>P
zM9DTrklD_V=BXlqaTQC2-ab4AZT{7aVcEJGjY|f9V>7w0m-0~2a09FV)8@@|vcU%L
z?zm%_)ydCd+k{0*H^eswVoTEXZ^u)be;*$13F(ac*x4dLzVDIA-usv|G#j2);!*3`
zMi21O%+rV~%Nv-UojEY^9r}uydV(LSB9cTjIgi47FR3?MA3jf~Tzw<<GoEoQNFS3%
zb>7pBOZN#lKS9Zjy+dN4AI6T7-iZr?Y|CGKZ3#*1k_d@y_xBb7(*&{wypEiYCx3$#
z0gsA?lw1?)(n$W`N{u+hQR3N*APk^<&j<Z8?BNza0(m6(IddH=vx!8R2vk<Yswymb
zxAdgOLdB-yH{S!2io{T(hVo&e5!a_ozdG(ph}Ei+oqRK<^Doi5al{K9&O+<sOP6nH
zy9>R|`NVoFRij$n3amAkXvwMy1J4oz2JGvJ@&u=z2fwAlrUh%=oMO#^=L-BaoJFrF
zh`$PfR3CNVpat?idN2g1K|f)#01%L@^JC0<H`XrEvPl1V_SuxmufU2VsJJ_w2~(K+
z3Vf>I^#??y$L^M9B<K6pD)O0qlUKOkyLpsz`>((px$L~%oAk2p5;+?OzwT!I?sIr)
z6gj2?JJS_%A;@F!aT6DHSYSgQH4>9-!q3;sp1c{Mj~3_;9tF@EmhkL3(`>S2yvTKQ
zuD=i|gA3i65SJncN`~TLcxE1_KfOGosV7s}&5&f);WQ59^t4SbrFIBC&5_96fLF$y
zd3{-9R|eTzVR==mH~nRqi1emCJfEPDb=KmxoOm{KBdZ2mEO~j+x0Y49qeHh!%>$l_
zQ4igmzbKsxdQ0-_=XIZ-NH^Gkvh3qtlNWr&fAxBK;EP-JAHCPwc;*E56^?hy*z3%$
z+!ohuF%A^Ty51m%9V;VYbZ&Tb?AbD9BX`#AK5d0rcJ-@1-4o;UapvxG7%MlW=Z(1?
z1KWNKr6YO+EggpoP0^)k{Tnk&@H%Z+qcm{w)!LU<N+#s>iavP@RsBhP&}fzmfVtEj
zo;4cHkVrH98iHsGAvF(oY>zNV54pI9c`3MTL-<rpp+3IbvVoNRODog()1``Z1KWrj
zruy(4Y_%e%iVybf836(_^9E=d*v<n*(}@5e;EU9~{iFMUBwGfL`@t2#(NA-A4jB#!
z15j|a;ZL<2Lf%fPUpIl3NA@gFMD+nYWR5iInGwB5=ggR3{J=`et@;vqih5iq2NcJu
z!TMzP&)H<hX)-Z+!G3<I3;U=T&ldCdI*sJl`X{)yKucNYstG&lZ~xKU8<Y3)SX<bf
zul=+k6UwsK3t~x-7P#ecMd?=N8(Ki?n{Xp2nD#J9ku7Gk4_J~v;}B&l&No_*{Rw(P
zh^1bhJeypu?+V^!D*CfqeQ7?sA|Ncji1Qj(jn;59bI>s@@@-o-iv^u;R`<^-n$m9v
zMwD~6k?&SD6`kII4}|2xx6aXSX!E`r2V%|kvCh<?=+dCq%;lHVK842lsZ{BHi%pC?
zDxwlvGVA_7ke|ZTFN%vH{}?hd#vGh*emtEdTxp2LYX}D;yn`@QW(vn0Jy?74=z<8W
zqqRks_K8`;?5SG(N;%X+H5@h+xkefg9s0@%yJq+t@yQiExI~l?HF4wyi@$qVo0J*p
z=!?K>@x?&t^}5C0I0dA^oO6hPZBZ$Eb>wosJfVI)P^|LVRYA?NdFk7Iu8OvsLkzL|
z*>1$|R@TWl$J=7L1AqKJ_JmTKe+m!AnfY6H>7t@pb!CF=RWcEex(I`K<2<L|12;0D
zh9w`T_6hrutfshs@i<wBB*`7qUO5Z)pN~lJf#NGpqdnn<&Wq&NQ&pf*Z>;w$Wp)Z~
zzdo~V%Rq3~ilNRrj#dYJKbxK^TsQQ43V1ut31bjC7T$K&K&EyA@e-?J)VS&8iUEVj
za*wh$1L-I7o&>WbI){{CMZ~1@mafeC1G6uUrA5&1PeEzDtHofPGYNfSzS|P<l;euu
zE~9E}Q=};;(0LOhNw%-M$C&qXg)w45=8^iPL8t7sdk6lZFG>Y2^O=A8ukkUG<4j7f
z|H!dLL3eaq(&lOV7<Y_z9_@It1&0$3)PC$*RqEH%R1tYB`#w9+5~9U$<F{to2nX9t
zglhUpNSZD3#|?yp!ZV0^a;{>3M@U{1XojSj;GItul`g$s!$Xuz(chWZ*=_b$KfId$
z{V7})USI?%Eio*0f<ME&^iG(Z+8s$k{zWd@{r9Mh%@*y*3k|#RiMoF#iB%m1tSY-#
zWo+9&ffaT_^nyP5X?e-R6`wU0#b2nvFUD^i7Z`I7u9{|{DfgH&D5<6yf$Z9}4sqE}
zzE2#QczKMhnp@JcXa3EKjpJ_kSI;&(FkA7qEXddlawXOfFdTs3t$(!5I`z2NseXKN
z#SCkDx}!*<E2;dgqy2o0kF`B|7*F-N{$N6V!*{|TMu?=5`hN4p3a()jCPmvMG4ifF
z?8Y03>bN?x*|W7wS7RJ0S!lkt=ANL&e(?`-b0ZRv>hl%Ww#xT$GoaR>IA8{!l2(Vr
zMCcR{XR#T0P5Gkf5W`U&g3kk$T6^@EzXa{$An-9)D8)nXz=8>7c-~&s3|W_#-6wXF
zV7L9v(5@2k^79BGQ12wd@p5~qZn4WUuz(s7`hBaY$Ua%wgnIpuHk){EqNV8CW5R(X
zwGv7BB(xUD@KRvyFN6Ah@8_WXb|WwYoPG;(obqT>79ddZC~IL30v_`z#x2va<(%@)
zZ_iwj5YpurKV0evjprGi^1ltkTLb4KLgeCW{5UBXkH?KVO4ey`D>YAr$w^()`=s+$
zLL@e8G)fI^H$cge3zI+6o$vjeIZ~(yz$ywpjt8C78yru@0HWEN+74x(Lq6OlwfI?J
z;r&hz;!4VSn{B3gcTU0RZsRPuhp}t51^K23gwXljq#+|zkNDi&Q(VT!cD<K0N+B*8
z4yroU0<LnYrx<{*J${tYsl(YK<FU`JSu>x>#E8n#ZsEC`BfcTAhXVTj@p$Z~Qt)8%
z1+X$s?x>}6bgtmKrCdNNn;$mosXQeQ)Nesx4*R_pao)?8Vl8Q(SXuhm9AkJ;Up&N_
z;r_byjz(3uk$ii;Gvs1ifqX^QBG7J~<VsnMu)Y3<H|I;RNV%+aMW%$;$9{{xC$awd
zXRGd=hEL#u=pV3e%QUh$Ke-pW(H*Gh8GOoLg;`$i-OZmwNQpAfTFi1cs-s)*c&;9I
zCU9wqB{$!IjohkJV4n=iOxC)U-I*(5=?kj1L35p|RO}IjG{#TnTLcpM<|5OUEq!YZ
zzL`t=S!2W*YB_2G*_w<b&jzBU(nQhAL)>|KD^ST_90fbN5L84<$Bm5UuHMb3sHc~j
ztCag&&p(=KpGcI;VP99h?4(2{Z3^eNO{9u6X(O@KhgVtvK9u2oro@HRnXyF??F(}G
zNoOR8B5~`+zzObvc`FTxwCHJ)zkh!JIApUWO0=BF%dV?k*e#06z8D_$w6ne8d7l*f
zBzwoAYz0&?Cig2VXZ!DCRG7bgc!m9S07qB7G6YQPrf6xKjgy})Y3G4H%G?fjsX}#J
z@JCuZf8md_uRfpaOcii7A7`cuZI+Xejd2QJZBteVL`Uw*1`c^5v_(@L&f6KlMc545
z|4PDOlY{fn@cgRvWUFNrmcArT)J+Lk8MYSKs_Rb-C{^u{Y>g=6StG@B{{JXC3$8Y_
ztzGX@S}5-B?(Ps;+}+&??hrztUyHjG*A{no0&HAEa1Fto5Q>EscF(xI|6z_f=6dB>
zm`~5vMLm2-&67eA+9WK3mIEZVQ9&nz3zW#-yW!y1p;bm;j}di4szyPY`8`BCxXN;-
zjW0@KQ%^SQ#N?PlRKW>C-Rfm0iB5%k99~;m?R`4qj&*mQqr^(yG$$**S(u|ylXP!a
zFkO<eD16{-rfd5ezmThWZrV<NxApV)Q(s6jpSg%=*D&f#S|jsn@Q<hIb#~6nQS8W|
zDEMdflKdz3>G?{^{)CmEKicJM(UeRT{Z&R%buZT@Et)AQ^a`gYU#6Uf;03xi8IxFq
z&dzgtQ*Ao9CYucUKi3&R$s}NMh?uJn;i<1%4Y<6Rrtq-%qU~Ol9ZCbZL5EZAv}wkh
zT^r5{@WY7|RONcJUnX(;g+6xu*dSfBmerK4uPiC8F+4^BGy+VqrPskeS=M=jhy1YO
zC@i*o8AP-~^T6<`UtdBWO<bccetMZ{xLxiHYxu<%M_d1@1$SM1loKci(b;%fZ4#A+
z=KG!O`116tfbd{5uP54b7n5BwD(k=LRkPu9@TaD?*h-_TY<hM0663n7w^I;)S-^fE
zCE~Lh{vZQIgt*oHa;EdW4Yj(Ql#1H93gq4*f_dHb>(?84uiMUY?<Ujz6vqilU2d`W
zM6PH#wq<BE@?ux6918`0vay}(1ny3?Cp19Cc2Xl|>~R)pJcjogp0MPi=}BkX0#)%o
z4I4zoGh;JXeFSg^6}IXxa<HMO7jd6t;b6~r-t#Y4FH`}PO%V!QHr(VlM<&WNODu5f
zj!?Tqp3gr<&xh+1P4AWA{q#gc764aqH4L|_Rb*j%>^MyXt8RgI(h6<D01#Fo4@p(N
zcVYiV)W1*UFYZL~7_b%AL1^l1ZV~VP;9h^zPXvJRs{T@11{aU7M57I)nWT!^BNoe{
zWf8<nzjXcHb_^yOH)cc2<b7Gci8i#>e=<x*{=IE`r4Wy+eTh%IjI*GqW5q8jH$(k3
zw}Q99V@Zvh+_SmjNqtdW+-5Hlmox!?*qxK>W(7D?(wEbUv?<IRPEi8K2CR@}MajgF
zUK@OpOM&CRvtqQ_$Wqk52x%(LP#k_9GE?Matm_{G?0idkS1Oi=HpNb=X`cvfl(^F@
zu)lQeaZjK*``-kR_Who!5&G+UHZxwVIOFsF<x*r1oHVM*f@LDrNPclWl%x=g7t7Sp
zR&@F}X8F{^NudY*Zf;Jma0m6vCQ?#b^Eg)&TDeIypf`6RGuhhxFTgqk=}oC-F2$+r
z5G|wT=7;LDLH-}ukNr;c7m$pPerB@zzNl3$V&Fe}6=({9^^q)>_tBB!Em0;jmdtQk
zLk$HH`jfu%_*OaNsFC{D-l>$3u-1Akn;Lxo?F=c9fPe8<WhQ^1*?)hK)w9D731pJk
zzLt<BOG1c5Qh$bJW&gV9L_>IHG9`SmGr@G#4cU{Q<Tg|Q;uOi(4|d?v&<6_ze7?Rr
zhR_IawRXKMYlu&X2m)CjYWmdqY31Krc{}WGyzu6ltIIWyb&iC5C$84AQ>IxBo3a-e
z5O&3mt&>x*YYp>$GGHsYSw51@YVcX1pB@gu_M9@GOw#AP0x)<hBD%ks7y(Y^7gbFc
z?XL|z>FdO_m_#l44wdF=ldYkh5e(c$-iZkK)dE}f@WjCg#1+On(wBMs^TYyZ5+&Bk
zNc*UB%ft2hroKVhi^V*gPvr~B3-*dX^Gj!o$n_~A73qvGzCihv$trCr-`5(XD=^<2
zsm|Fl{ZH)CXkMwp!68!ZJ?CRl$Z|slwn|!&?J>I2+6sG_V7EuIsbqZXkNV1($Y}#Z
ze}hCYB~xI+SEtxHo_|HuSj?;VI`Vt$7M52%-^iDp{c+rPr62yTl#Cni+*a#2@}EM-
zFyl!1`xkZk-ol71%IV;(iQx);vByII4PE`#Tm=;}O|h#h-kk0;nBRlPqP;*FMR)N5
z5e=JZ=VqKr6G@xAF$Ev6yAZG`Tc2{^QJz~z&Ou8p!zbqhmlNM_l&9P_KlgizWLTT*
zgpCR$4Lpb*)~GBcPE`!`y<vMwyH}YFEW3ibyGi`|l`6U(o2L01?22yuwr)R&iCK)b
zw(Xpk#+4{8fOO+MG6-k(OUNd52ZlFfsKw~}=K4y4T1v}cF6qoZXzGBMqY^D@k(55c
z@DI@Oa?DvQz)!$T5JJYK7Z3!On<TO|t+f=aOOVW8U@6VdK?0fr&J8u>H@vOZ*~|%B
zikN!8-BrWsf_sqTpLOb#DzcS}>tcq@5K@)Tt$w%QWT3EH8RORw`cqlG{ni%tXNkpn
z6ztXwI;hHXxa0#C7>ZV?@ej%Aki=Jw-qy7Xo`(4a7UukEOYLNHkg&%m7Fuhr@_Z#6
zzbHm<JGe$QOPcj{3Z|y|W!mkx{%t7EH$%8qYb{cvyNMQ}Eck||VXt4YmTyKaa2P^L
z#XTd$?bbg0^!|~Wj;N#c>a74p>6$j#gEE@ky+_M$5_lC2yd65U1*+boC|{Mw{rh88
z)i@2fNdjMBG<+H$>Aku!UPjmKXjx`(;G%R5T_UKej|JbZIa61OZHSI2Tf*nJ32dH&
zvWG9W0P@Z#`6Ecd#GpgumnQGz#Y`)b!syRfU;!XohOgHhn4PF+yL8YFSFVZ;#j<*v
zQ7SYzJGwTv_i6PIOI~X?s(x;ZZ@tIHW&>F=ykzj>Bc5;JqUMN!8PzpyBRKhYm=WD(
z!k*Rnm!XlF9L=9%ZZM$vr-Y7}=ptX=1(gyzKxLWW{fk+=S{c)%hb8HOC+n=AnOfA4
z7*w`VrhdcD!jZpx%*Ks$NhYno_8L#};coZQOF$=Mu4c+TEAGV+dMLTWueqltB(k#T
zW%hc0JXx}iI9=TqiqI^$$>k=tjc;R9LF~CXgx^lUo@>xe>>C>l;&PaL^$Q<nwPs)G
z?YpA5!SGTIZlT-eh1*aaQzI<<P8-(|7`HKCs44zI)2TARPd`E_^~7YsluP$vKA-Ny
z$nVyH<3Bd+y)?hV=`AMapw{4t7sm@Gvxw$ULQ#`xb68|soisOx$nO-Z*|cHbN7tA*
zCR@?Q)VM}1#d4gE{Cd2wv#7SQA*kJa9J?_cbH^j$>1>>PXECtNyU)M3hcoFaLf_VJ
zpJ2D4_;hvP0v~(My;ceW5uxF*eJ%h~*pFxh&XLoTHnSBSvnrA3PJuV}`fiH#;WA}T
zy+S9%R-yAg(rk2RZ4G#s58dhSe>+Wp@l>h6zy0gC)iI*Wd30pu)<@P=<nNv;#}PLo
z-|on!3D31;a;yBs`8T^;0>SBz*HL(qp_9jTAClbo2(&{0;z~EpzN2R2@*fI5n`6SE
zrSZ#m{Mgd)NT0LLDRJ!vCo<;E?~d~_j@kO*%Quj3-++F30%_)l{(x;}rdr#5mV(jk
zjE!#PgO#^)b(4>a@zDnH`1u(eE(W{PZqnQ8!|DO%%`zGUL7pUOiOs>K{H>lnHLDzE
zCX7=aT-sn^-ek5LoGr;ARCZwmTT@~Y6Ao#eoo=!Qvj_P2IL;LuR%-2h^UB#e3#wDt
zPyXtxpd%WrJR=%wD8_*APV>6ALQTN;Tr9}Df?eoC<+qBp6SGFu1Rq(PCfPOJD966d
z@nA%)R=`&M!bZ$Xs@PMX0W^H~RSUZy!5#Q@M)2CRKaMwk$K&V13CLt*I)&Lg_WH;=
z64L2~;xlwT!Zu+27&TEKioIEPS*v+lS$jrFHM;kC@XQPph9@!^uSxBK5hyqttL@P~
zM9)P+$Pz$ut%RFlDw*fq?^dTCq?0}fVQ`WOsj+#^YMc;`CS$w*hQ=sm*eet_F(-mt
zT)Sf<dC6_t%X<C=d0|S!#m5egh@7|8qyI<dS04%Re@nG%iN0>v=*;!9Z_@$!t={Zl
zi@CMK0aM%fL%=3>?T?`-#Aa~HW%t3-!3ig35Je22uyknkOnK*uyf#}0XiS&Q$z{%b
zE10>{;H!(MNiuPSciB#HH^{kRAhW7(e9bvLIk*(lh4RfjWp3NZ!20;pYe^27m=6U<
zqi>#CNXz{U68A|GxYLi`?YN^3OV}ijb{@$1*Sqp%*Ytsgsh1vs)(@2TxRjnLBK2Rt
z=%$Fdp!D4dXl8VsRo4Y&B5rnPtJ3C^fuUwcC>ncy$~F4kve6!*cUSl%rW9}#;nQ|^
zuZnP1@0o*}{eEHG(`goUrO8<SdtU&8puh@g@Z;enjStSL0ro>!+6F49odfJx=GC??
zy{q5%9`9w?nPKz%wItJDY)6bC?`;<ZQsn)Le!TDop3By5_y_G=VdTyeH$l#Z!ogK-
zVdrI6HRoiYG^ybD5cQ;r60U{9#`U7$s2C7`D-|OA&#|`Y3;W@C8zP1L{S6ni5t#HS
z8TgSbSuUt;;}PDbMAccNkdidn3io4X>G}`YBEmYX1a!g~-*gf+_>dGry9S9lwio%%
zV=Ln?^eKC_v3vJ;^~^CUb%h_~w$UW6Dbp2}$7fZv#k%S_&?OT3*q^}mCaal%T<WC)
zNejPp6Xa{~U?L^S8%W||?|9Ce-~BOqmvt;EbtPdw<he@o)uGnPr9WqvV$*c`jIt7^
zp*pY2`PU^DbG+`Ho^efN^sDCMor>@JF+~r!V9?fax=-&2%EjQ_a0vTGy1*OW2(_95
z$dV5<OhN~IG>|g+BD4c9OIPfm-A6ykwN0)w@L<J4W-j`&J~*NvZ7aMBz{|JB+9Nbn
zpFd0Z`7wb1P$Y2n8_|IEC!f#JUapTzKaQ^j&;nEzDWbm}xEf3I-*%JPm&ET)N?{xH
zdN#>fY8_WG<TZb#Phn&7CTp(-pgqqQoi-mNWRtnbZk1YK^|61xGC~ee4kAGRTC+~<
z2`LK#J14b9JuPVw!vqE(Eqk5WV8|`2^RMk<!;31qt?_yB$#1V5pGRwisV+52Dkn$K
z3rQ;dccr5*!pSp5l+i;8c4`|PtEh6z7oSftmz!-Vg~iLMAK1u<veAQM81rF@yx~Xg
zHckFKgoziCRphVVLJA`86;75YWbyZi!S{^NzR%pD44*v2H>flTMjJL^#za;$d*xAw
zl;ZBg7Lz(kif=Vjh3o4BxtDnje&p;Soay8WBYj<J;;q&9PSC%*7p=nYj|9EdpK@NR
zKSsX%PvTM<hh*XZB6<vq0#ssu4rLIQl)>0Dw&+HxTwfBC&!?D}96YzoqgT$iyQb&r
zlyyhbXiUDY2Z;1FrJ!}!L#TI^m^^~Pl9`qEPFYI7^1{Cix>f9}<g22@t(i3B<suW8
z+q)?xTEXnG!%AwNfHoUBl{#ZbFJ($^B$w|epY-lGxN!0c4Nk&#w~_ha_pZEQbhJ9v
zpm#O7xM!@JU8&WY?*NS*Zp$0L0WRU=g9_hiNmoS`mWs<P)SYHiZd_U$>W(TFoT>JM
zbM$4bWGU&`=?LPE`&yciRj6vuWm~y06R&U~L4;U}0@v(xy1QuKkOjN~hXB0D7f%zZ
zd1M>zv1MA8nRY6}u8(Xf>w56!i|N49Z+ZC59O0!>UPdze0l~eyhI3?RBtS-6dFjeR
z&!C6dEi>k{U8o5P^R+c$cPGTQwpd$CyNJ^ClwCO9#lw_pW3CQw44p>1Y>As6?kqx;
z(9c9>Ce6(js~@O`vQ|0?+<@#WSZ9DN1Cw?aV!VR@0|wviGwssOa7Gl2XV<*S<GA~(
zU&sYJOFt@V&)st}ZMOVuLqMJxzs%}Pq>NftASIoIl{}`F6ws>R!|bPIqf>CT)|@re
zuwjwNacy-Ro>~KU5K|&VHA6Sx6(Y4raD4jd=1AGIo7M3Le36%b#j2J3b@8lzJbSwE
zx%y_q17AHfNxEkS?F$D~_T(+LNT~;AxeNWuaJDF)r98%K_qeC>y$=N;I0EnnpLE^E
zMfLY!CSI$~F3c4|(&>5dCool#uu^~foV!mGa8q<#v&b8mlOSv(lLh+$CzuS?DHJ#8
z3^B!0vVQrlyTj>H%Oxo6KU5l5z1JSNa1eRMe?qEEvmoLZMtU0It*9supkHIPH)GUT
z_ffHFm(lZWzMt&$mIqOCJr_=g9;&J82!Gd*=OW#-%XNB8eyMJTD5HtSYUGQf;PL@7
z0+-gfpJ7O@Mv1fgvI*=c#0IdbD(IONSk!bJytF)~gUj?@?oz@PyDUTpS9h;?t_5Rq
z9FXC*tvK!v`GH*-#kErw=MebAh7%}%+0wcT&Uq#7*tW2UUYMWbZGC?~f3hH>dctYS
zB)${R#%D3~by0SA^;JRZ1wCKMHT55Y91JdO(di$${V4G#bZG&_^sh;_$v+ISIU+wg
zBo}gdg+AXslt8vUAiZBBmOwdvuJESj<LeN7=o#D8LUz649u#*Q9=9p&<c%+x4Da-G
z+$;$+scGryx!kXfqx_Xf0h28!cD8d`mZn}cj31164Th4sUrNuc2~z7)q2qHIUmmI^
z5SAAvzRq=`ho>+@4lk=XTIc631$BnOJJ&Q?2*GBU;a8WpxOQoRTQR_KHKCOZzDvwC
z4vOY6gxvh-HKSzm&9Z-;u(ZwAZT51yx3Dk=EZOZr(81mcC`!=P)iqv)F9pff^9>UX
z9p|b6zxLQi(7o^{O!_R)+yf53-dNTXX)^srwj)p1E2!SW7|MuHxvAM}B?mIl89wv_
zC8dC{Peb)(j}gU_*n<I9OE+EAf*K<?2+w-;%CDdtRJ%FlQK+enx6LTegz=kQauRdC
zKejT2_7ik!m`9;kQq%=V?A&jEz!10Ef21uy1P1zm`_HVbWU%7c6>cqb*Pj-J*}-uI
z@$!q~SYZeDFfFBXG<?txcA+tp5t;V7-gP|`;N*5xdOXmo20~hNJnQR+hiw<NdNAni
zhsve^(rvzuh8hHp_x&jm<oL<LmVzkrw3`bPj6kTcJ<y%hWfAo!!(@eGE;>vrujHws
zc^i0Mtx+&=d0;`F-qO?F(|zBDA3;Xo?Dl4R>ZKq4t}V)Ey*FxAY^PB}--Tnx+EgWd
z6}r8RKz{2%z@84w-s{ay!;j59#zAFnQyTWe$mN~1k^iz12W-DZC8sC|%0-#Qn{$cH
zfi^lC^ZYH`gLgHRyW2(qw1*oTegZ3iQrY^+mIg|Np!?40D%xy*zUaw4>4oG$=V^hv
zSxeK4WOH%RPx{;~y*2m%V3El6`Qj!WZAX@rL-ba$+#so%cMi&1@*0%+<Q~FUhBj=e
zKqc8owJ&VPf)%xXRkkbBT230ArF||@Kce`ebeEtfthTS#=|jM2xGy?xksKAZ!JFMn
zrO#xoq(D9Rqs;2P!X=gcJf|YszE=(pIhy0#LtScy>MrCG33cAY?f6R1(;oamcD7rW
z%pli0t68zbxk>k>WxK#h)b3`r?vNmjkRYrUDw2y_9M2qk?<{mCbR*5nlBg%Y@}uu2
zb;H(0@UTgBnl9fkFXC<KK(-hBNU0yLeGK>@Lvz;8w!q<uUu~(4RWz1-><k&MPV~hP
z8?k?Yl)dWvWd-kU%lDy@Qh1J^-D&)~<}L5<Z(Je+kY*?4lX;GFpV=!<)7<3_`8;tN
z2xEW#$O7#G-mXar;GT&`s}ktb7;ayo*dvs&yZ9vlBELB1AKuq6?qKF-*s1{i0slP_
zpgW=r`hiMuKc3AYhM7QFhKrdoR@Fcb2gZ6Dsqsmpp8;f4wS#eGi%ygt6ZTQP-h;Ef
zoL9Z{I6?Y53m)PZ?Ps#l_e~J#Q~1w$6WcvyaWN-+h$CYBro{2eP+Jp0qS6oTie`=4
zxf?_kWVA=7uH`^kb#^0|YNhRHP`t@8Qok#wdJD$V27w5rF-blcJOss8sqX<{;@B9P
zLyqol1rMmyF_*cnN5@^JZeHA`(Iq6w=DpC>8A+K=PU58?%QD#3+UyKHulcgA=7=Nw
zvh;$R3IWA~elI<8krsUumogGxWm9d+kk3^yRXy>8%4&|%(ZRtiyQ>O40sY!B2|4$>
zr4l;ZBHpo!ESH+gB}lnoNozCXe8oVybAGm~X!6S>p#<9e{yRk5OO!E4-N2x|v%4Pt
zYxuIA;kJ%7Ph~XvyE+|cOFD{CQ@b18r4=mElROswTYHK}TFGACphJ@o@Q!~T<Pu4m
zo%VtDVfML7I`r?A_gJ>54AR+T4wJ^Qt-n}dmwVuXK8t(4IuF7PEm?Nx{ACZ{40RkP
z2I`$<jzIdJLBTXf@I`ZNZqGb=!?HY<nEU7KGSR|*U)K%1xsde;bwbah!|$O0W_*@S
z0{~6n+pw@Hnz7YeE59||sX&gU3Zc;rf}zw-yQT>MaYo=wv*!Az6<XUCXbTEh1TSm0
zVwoX&#@nfFT%}aw=s^-;yYww|8~<8`fCvuln{SiUy+)gEV}K2u&L6fq`M09yH0=Wx
zhFP9YsB?Yc3R*PZtzfcfu#l;CNsQZ;yEYuT0FU%YKKo;r-!(Jtn4P3YLW@{(C*<o_
zl+72zzpFv6iQrz%=8<g7v-;x@*EKNAuoofLQ|f+$Gp!W^DdSkaST;vSw*;=p7sCwq
z_-ZOMuUWC_9W&kAi9+gBzLFjz6Ad)Ki@D+7+MnxY!!lFXuq)GNZUtMKj<OA9+SbSt
zEaT8<ja+L_ckx&a_rHGApmyoMF3Kq*P-wo_fDySwaP5S+!M2o@zu|&1k?I|VN)4zW
z-Hc9J`NhZr-lOO(YXn@athx7f)K3`rAE|uzPRAc5{!erBDP>Gw8grvoC|7|)LZz5r
z$VC^NS@NOmUaV@0p!TXdU{k)nd`^6e_?2NGk%&(MQb|7<Gv5!~+X9XeLwK?1(nnMp
z0)<TN?ISK4QoN2jyBE4gIx;Ad_;CgO{ay$GGhGRSY~3}8COjv);-^-KeZdddiL`ka
zAb;BXiQn>>+LvJl7ooh6g%kSh@FQb`>iT*x2DfL;Pf|C4XEEoaY{~y{^41ZJ5(upY
zX+Y_!4uib=?9_w8qr#^C<G(z^j8%<Tg}P{qKoGAJEa`ml{Sne0HGkt>^S9K6+;S#^
zlweL3%W*wNGOug@34k3Rxtv*qn85Yp<C#mwjX$b<82J)jmzIpuHUtXFs0I4XZ~Ygj
z9&h}tlMsKdYa!Ye3=G|fDEOSNXI#p-rI79?3PAWyTMBuOXfO+e!_6dX8sCN}kgEiT
z=@D@kRW9d)^)sY;Gi1Fjn@9qIq_;rXrN^GJ2d|&U?48wEfnIvqVYULKxfLix@pP{o
zx;~bqkM#*YXE%7rNoGP-ac3G$8!O#c(}a0S|KX8PB+nSr!krpdFtm`Sb!!x4LV2hf
z2_Dw@@%Y`jh?P&R7O+?Lxv*+mLzkTs4jYfu8b6H<!S_6Uq@KhE8OE*iF)TF(D8FTo
zZ0+6J9~c%%iDWKsUb!xEF%x$wNEjoljDO6G3mL0&w)%n{PG<a3En_B-G5z1924;ox
z1<vUsBWe@*6JC;Q)cP??na)l2PVBu3-)my*1VFD8L8V(nU;Tpa+9R34ekrb7SJ3-v
z7xvO=B$LTS$S(tGf9h3;cs8OPF$_PtGvRXZi<2m(fa1d(^l*`L^6$6<gbf&N{n<29
z_|w&<c@s)2iw5zb<jN+du_9*;$1CgjoQS9l3#q8T*jv{R|NM62W;!@)=`55@VYFpv
zPJxHJrkW{m7g_n6{ne#%g#sy3@7Skt7`@ZGMB*q}ZK45IRxl4$s5MN+o2ND*kukab
zM3}wylX<$0S@BI&MpYAmRnOF#S3>SQ84}~UbRQfQU@Vf;9}+J6*1tFM#*w_2MOz_L
zH^F(q$15iA@_b+#;TQRGu&#CVxq|kzI{ggO3^Y{x{x;;um{p}U{?Pr`>ag?VV*>79
zpZe}`LFZZ<CY!hW&CJZ)GKK!Uolx-%J2Ow>1-jXO&L{LC8@5U`-a>aZ>i6;{Z{Qn^
z?60Gzo28wNq>_EAW;sp_52xVOg=RFrtq@V7b8oEAE%!8|xJ{JO5kZFH6ImCR=Dncj
z<ZQE~%}%^SFjsohd-HWU=dxSMyqv3<&<6%FC?18#xI<c18W62n%C{oDB5=<pFh1i%
zqpI!S>HP9`;yD^abr+sM5@*tz$|T$sIAB0E1jd?)aKm+?(WmTcwxR!l>4Ujhg{DH-
zLdFkLtj#u#s0?@naVQfdqNSbHvTi>2{S{>iFsF9KAh*2{4bm?c-$Nko#9URud7Mt8
zH)`xFZjjJjHwtihyk!k-Eoo?)sOddUX$~p5tj);TgOo<#0SSa7I`cfj>w1{=0E3-b
zcS+OZmg@xQ4I{IDNBB@feb)BY4AUcoM&PbDps^limiysaLhOq!3tdt#491TFldzzZ
zk>+<NfV%jkO%noD5}>%`n6<f6FJG(Zf_ALt*BXiPLiU#`E>M@1I4C4=6t|k0HcB2#
z{Ty38ktzHk?{+T^+iXe(Q(2AA(N|4Xdb|K-X{fpD=_JFesVH#m?ZB+>-kH>&+o*o`
zwjC3_x!xz4d7SY(Wk}bJ#SlzWJ)us4i_Oj8clI7-b1G1Gqm7>^HS=|Z%!03Z(Ul9q
zc$5O?#z}+silY~izoeMbQdd$#y{OT%3kQ9F;>{Jqna!QTIP7|5V82%~&FlZuY$p6W
z?)!}AI;>B^iu>qs1m%h#e%76>(i>qH9k8NCrbPz#^4ySSBj50#U&lbRfP#8>Swq;j
zPQV<jWAB-{E4sHVp+NI4R-!YVE1-%FJr`cTW5f|xA{9)3(~+RC8OXXQ2r{NVbYo9-
z2XS>aU1;YSs{Y3s91x+qEaCdh9y65-UB1NiE$9_`S@i6GsFsOg|2ER!j^E1qTW!i$
z6rY%DgT{NMyWcTk;}qn;#bdvzNg1+7)4$-7(-DL_6TNl$Uh0_!ZNFluj3u)uz%U%w
z+H)VlsKzGcU{}`HJ#8J`<hKzF$qjHen=W-%97SUDWKRIpuJ_dIcWgrrxjUKY-^&(;
z-#s&ENJh1mh1}ItU84(HWqAWf(#u|iT{A)<lsGjvwR)&#9l>AzjP>TmW6s_txh%`%
z^yH)4X6jtdHwbdU)2nf*>1&1QBHEBst%#5@hM(&++Z+p&hk4Sq5;LDp<O&W{EvLGO
zq-fE@<Y_6o@#<DR@(myP<`3alh;a$~4R6{yn=aXk2g`&3fk5{a9$hfi-r8Qhq|GpO
z*m+0%2l6906TNiJN5HMmTs%ImhA{;T&>%uM=zcVKsOy*S7{viZM63J7<uaR-$^zDY
zToqm0uqC>lf)}s*;bkL?kph`NEy})S${j3LohT-H328|3@VQ~^YqP?8cNKW&_>()&
zTV80>`ABnjaN}Yq6U5)wQ76u*d<hbWVO#{Cxj&58%q)_!!Wlx78fT&(2p0S{e!oH}
z8C!JPeoSUB&^8bn{FpWg%lUliOmwgFkE#=O@MQk>+2!h>>!OR{?p&T7>+<xQkSV@G
zIo~s}ukVhK%H#OkWQU{j>+SYzp@r_Y$qKposl55gzkv3`{tTwEFi<VwrF|%eDc4)z
zMsuPP%l(&B-ZfDd8o7AOxgL)K;w(m={A<f!6NCE9@}W2;A;KT74}Lk0dtwJTbMrjj
z40<>>T65a>y7jhS9iw)fH~CGb*4;@8c@cys+}2`sCVhNQDb63@N#rd6=*ArXo1L|%
ze4BT50_kWzT9Q{I#rwYcqTUd1*|K^oFpS@L0*)Ehe%CnthL#}m<~7kHn35VB*$MNe
zw1WTFx97U250Y#TTyL;vFhumXzrd%zJ!?L+-I+JZQ<Vt3`1u+p<BiXpsu6h^DfwG?
zlTr9fL(Cz@lDnJ1q>Py}rI^9&iUeweIJRPA|KHbcz}=VWT<<c~b$%#rnkEq`4n0OB
zx*~4Wn|)IjuOOag$qser(sz99jzn=J&*#nw8u-0jKG+@$Q!1<oC6F14tlKo$?wvk7
z_eK%>*ZC}Y-U$a^p&RpiC~RK??Dh~Lg(2c&Eq=C4kOI5#3@{O9(;7;;Elm%|Sy(mz
zcAYex(W-aS3~v@C*mn>M`3y|83n`}zIMYYBuG<<FGI=-u==oVvUW@H}oecI0g)deH
z-QS8`*Q$E}Ir(JNKGWEZ4r5e|(TV<&U5P|x#!^2uu7SNvEHggMi5dmO+H<WggVT|A
zQl)JzRNW}t!v2-s_OGlMB4NWvXJ(hLrU2fBeVgKdH3&ky7fa_V<{p~BxjuY|=gnKj
zWwM!u@ds^sKj*(I>s7lh0&-=Iv{mSrn8Hfcl6M~(mC_z^euXZzchlMB_fsqkZ_fe6
z<J>?-*1@c__-ce%KqY(WAzDX`$?xDl@hjgHWUT*pBi!J=eA=&w`LvQ0g<E)_ju%h{
zHX!Ybv!a=ruzMf$%#ec5voFYUkMU(RZbH|vP#1V=OI--_$o-J%g#kIM&0@KcoX>mS
z$<*PTq%vNU^peXC;zRI|>74djZwPL1s8IOnH7DUtgtv$1m(xB3x<uk7v4XJSnpR?v
zTy>z=YO!oKM|RKUrdM^?B@x=+zdcP@f(^ZxfnKFuU89iEZa+8WzBBZ9T-q+5rs$jh
zK8(z(9PgagtWAVh8nu?vaz*7lw$V4q+0Z!!o4uF+?heW|hy=YG4b`}5VPdaw9yk5H
zNL#w-LGSF`E_=|Mu^XxPUeb4Uo2Rv<VQFNub6%*Nvvr#|!<ERW1TwO5z`&4hhJqWO
zx7oHNJjdR+8=_~{bvIixcdqiROPM~6O)xb@3t4O_*q_2N<j>sU3uLtKq}?NddBAE5
z-m7pDt%z1TFF1c9x~E+MZl%&w6R{e0GR$CC8uj-5Rgt>4{pW8Bsa4+etUhzHeLTNP
zCA*5IEbu%+6$irxHyb;@EcsyA$Vjfu%NvCWOhY&He!td6{3t#mEoTWoGsf-(wfMrG
z?x{IBv2_7M%g^-k?nbv>NPYMI$WBo`hJa9!>j|Ka#IzK{GF_x=TU?Remnz~Sigf52
zUqx83g`eE+H}$JHQ%hvt-uoY`7aJ6LdjsGm-^N`UhwiwZdOTv9884ikL@n54j;Dz0
z+Vh*cq*hYa8X?KYNxE^J0inc~UiQ)pTVvt$itv5U1=;s@PKs6VM~w(Tkr+u1r{iW!
zV6zWdca5?#e<~xuR0Ko0hibUp8VMveULGj?di~U!KcIAx$f|}OCQ{Yi^C6#}b+)S`
zh;z!R_%v^)Up1PlH_lzYHNHYQ2+Elc6DJYdcEP2T`AtGBm0UURH-IKuGZ|F8>2uP_
z=u0Q{o)YUc#jI4)hq3Xq8uIo4Yl67D?0~6fKCpeiaK(<OSVr7DCz2QU8}D`?8^()V
z_Jtlr?yb(_RXv2f+jqa#<h#cF*cb@;y{;m!gEy(bZ|)t{#V4`woNM|E7Cz|!!X_T%
zw(%g1?t8K#A^cZTE}h#bR$^F#W_#eqw@o76+(azDZ0Wo<!_ZH|M(P(?-2-yD_M6Gx
zr1L$pk~6OD$W{N~s_UcT)!uY*d8iFdcx5CL!TdEZ`kZreJjN_dNeUuPirHP*E}lxt
zq%y>*{~4!(`h1OXhQ2G&`Par2&wlfAK9@feTF?uRSSZ~x?2pddz(_xa0n*$?MJT%r
zT&f^fe`=(=ZlnP-Cd9a*uH;BLto8;vOZN=L6d8Gl1tTxiJ{yV?%6nbAeFpgF8g*aX
z89A8Sr4cU3kr;q^t4@r+b$3ei(LJ>>jjj4@$wj&!`Rgc)saW91<L3FDrTBKQbcOk9
zY}zEQPz=UuX=;yTRz@@phMIm(_agV4Z|qe#0x|ukY+`qO)wE@xZTdt|*hyEPW#}H`
zA+8~MsynD4uOrp0TDjZZB*axJx3Gr-TH9nDh%gc|Jb<&q0#w$|EFjyO&RK0bhA{ty
zoHNnFeA2Gq(V3S*KPDolau3Zb73^Z7-BT2FAIztrI34$PL)5-wRFPgMrRP@I>Bc53
z!<i;vZyIdGzpv68Bg;AQUppcFp5>xjkD?*aaR~~96>2W_>=9o@o%o$3ee@L9__1Hu
zl#Aied-O{;$)3$e2%)R5kX~NbSt%jo0`5Ia9Wwi@zVSbQq-FAB>o3qxCa{{eM=>h0
zPf6Be<UD6`w6d}!(NAOw{f(PP!REyxj%1v-outL&+49!>Za>ZT-F?2^JafcvW~szJ
zsCJeNJ_3#UusLA~8mWp$j#k5<0hvtL6)#%-a!>WYYtsxGYs_-b8G(Z8+UWuS^bhwL
z;p|ub#Cce`RQ7Rb-~-&TwPl+kLsQlM=Qk#cK68mzWRwC^-d*)bF0^Sy&I}e3ESF)o
zX(DJ^Bi>qYf+a4##pMQ{VIc}s9>+7-X{00g8_w$3214F$m)iS7%50KXkK@i$WhJ50
zy63`9lwTp1XUzUGi~TM{vnI*&hn3q=Au1PpaN&~re8rJZP40et8&6GMzu9jvA6kWb
z-*^8Qe`K+1-ufH06g_9WwJ;b!LBN5;EfR4y1)$tu$;qTY)p-=@1Zw$Qt&rY`Z?a81
zwIa@~0x4}=5IQBXE;l5`2MgbQrtNpZlZ=UJW<oR_DzE8`zcFAtVR8<(w<|?V+mNc)
zG;+=2P5INlL#9bhm)lC|raBGXM5x3*VI8BX7)%5SntJ|dre49|g<nxHK^yOTnG$fO
zKN>5fVy&SRD`zoHcX_jnbeZ0{9LK0l{44NJIz!q`xu!<@ibPt~#WayMjm;<wE&QD_
zAbFN1935udPHOs?`dd)Rg;Nh&fKp^s{)w76Vq9jxMZg#32hhKQh(G-tHG}C~3V8gH
z{in4cj3x15-zr7!TbWKH>G(7eb9sUMiVcx27n^(UZui+O>9&IMa--GGWOCzmz+Nzi
zYh344*)Uz#kgXctG-G=+=7ywqn+@Z?Ett<XiD71Tox?r;z=xJokYlXW^l@6vGdZcJ
z9qf*i5<MbavQ}yHj$V>_7W$dSw*1tmp@BdhM!P-WD9eK*{NF}>52WD{;^Hh2v-}gw
z;t(^+Mo2Wql_9^$(QZtfs2nBE+^JwDoVxq<4q_rwXQ*`tjaojd4R>eIg1CIPGXJ)k
z0my>qWd)t<E{DvyCfZ<ymxl=$_2WE6%nxv{Gw3V$RD?<oqMpX*=x-i=^91Fa(hAq(
zKtr~N#a{D=ZuzRIC@bgg4cP(cF+H0(y|1Bnuc0{c!lu0YW;GCLBb4T!qXv%qqe6XI
zb?&O9vdLT5lc5QOUp8X7lXxlHV>e;L!!<nkQp;B2uoV#^u8uY$+2L~IH)ile8jj<-
z3Ox=~{ARL<8T%ibG_vw<zHARLP=trs9%%Mw|27UiYqb?O($DxHj<@oxxhgN>mlmk8
z$w(hlp+Smbq#g<jr(TtAClt+{u(Pk{4B03H<9AYu&K^mSd@mf*lCOAw^r{;Q911#b
zpDtSSuf~(<lxu(ZXEw*v;$3DaKl<}~{T7Vxt5@xIzkT7!GC2-(C(@eCwua2^hwnmC
z9Ez%x9NHD<avn<9gKz8fR);$qwM&)r7kd{e@O8VnZ+Uug1#dE6qu)WdSl?G?;yJfN
z;HHTk!_sTPnEi&c1%}Qs?WTTTQMMxWQzTi&e%rHUMH!2v4RUeU7?n6CQ&jRPGe2JZ
zz3+oGSZdGDy?ZyU^z&B}x=1DyGWYqD#IEYaUKVn_5rg0nIKK5W(e6{Vp06xVKHVve
z72-`5?+Hz=8H0ZRyZkYea*FH19Ku$o;lbc^&0|?sTa<Q4#RC?3ebDv5z`389n7u#?
zdai9|c+F}}<n@C@=;4+o+J~l|HP*_5|Ky<?jJ{p#=mne{*J&Sn$`nqDez-YTM=KAf
zK&^Y381=%)k&y7TONiA%#3*h$AO~myEk>-_-|S)^c1~?)5w2%va6h{}F1mK^GF+D}
zU)qh1W<l+>gQ};PEMK|J_Igns3s#pUR(?Z~QWNI3QZF(y?vG_1WxG$Q0bjlDxNjz}
z*s&D?uJcJXE&B>F0SB)lH*<|jd&`iOGmem^8iz(Xwt^L9zh%f~dTZOLR1(=$GGGSH
z3g|O2Gph6oTsMe2cU}W`YG4gsn#T;G+=qM_Y*l(@=TW6&_mftXi>}i%vq@NK-|)r6
zR%O>s+hbK_+=M^U3U5k+$=O>%CqnF@h4tj~>tcV-D?VwoQ@fR3C!fKA3<D}%2V8V&
zova&YB7Ztf2ijuYbsT4h#Woz<489AzZ(=P%F*1hBy6L`xsut>g=`O$9^&dmiDywXg
zpXyf7l8P23f#JR%4V%}&8^6u!eb6K}pH~;H#MJpRJHY=^*}#@w<Isi)S~H^xR0uOu
zPj-HO?SHw-KeV8ua7@SXcYE$9ci4lG=Wa4HYDH8qVrybnX-2|_O|D;N;BpI{EwFF%
zTpO#&-$mlD1u!GSb=Nq86s4#;4B$In;XT_eg&IvS!)*;*kP}Q2HB!TJxjIVLZKz^_
zJyfTK)nPW30Bx(#nIN0(<eW!^06vO`qWz>~Pv`P5{+*D?!?8ZDEITTHi(4+yrZq)!
zkl|0!2OIp~VK<BT^1X8_gF$sqgc7s|M~MlEbVH4&r^owbwpMt1%6#=Vm)AF(90K5J
zXxgJqKDj$Hu)9gglr^DcCwU}lbH;PanMq)SABlk<+)5{)B%Nn~)NVbT44<0VbEiLV
z`Sx7wm+xqf=%LP=*|`^ag4`&Vr-lBMO>k~pP}=@%B9C`9%jHFYlUyRp?BEMDJwFw7
zGFo~c;G@qR0tgSqg@n^Pk$u~KKS;MtJ+YsAmZW@V6(UMv5qnAyT-*p6wJ3EAqIs=<
zWwbkNY^mw;NHhprZ4Z1M9$S#3AeWL8pY!6m(b$S@j1PFgbKfxt`|~B2WXdqfc{f0x
zuQhv&nDrA_U#sFX&tXuoB0>?O%FxqCMH+R{d=)V*soKr<;rQohvU}j&Yk2x&!Razr
z=Fa#KQanFr&FjLVRH1xhbtFi6;dR01nRm0!CbsIC^x1Ej<G0>u5i#sbslQ`1CYOHg
zTjSEKE1=>M@zWsPO}ez5Kj@DnKHsLD!a8u?Jg+~8Hv{7C6OXQ>p%^45&g&w?ShQB_
zmI6nKHKO@filx2PsWWV#M>LWUKgg2EjKY5V+m5N7k(3l!Br<!-jqE?8VXdi4Bcd(P
zu6-!BAIZdaw{4A`h`>}<#%2qCU$~j_wfJ_jqhxQOo&#0wo?*5buvMm$74P1@jA^)7
zCMqMamN}itGa8zM3DL2uIZV-+Y?Tvri9oMS7;}+DDqU%J=fvz0GNx!#&{O1al^8_g
z71c0wY*Wl+KaS+cHl*_R9E{t~a%aMI$)p#htA_7y%=~!KFel7mfIqZ10+F$$ES9Lc
zu!(4{+$lUR9=z1uJ#KNlo9bP04VL}cD_c*^4D35Vx*>_J9-kdp192nvpiz#X8BeBV
zicV{e2w&QC*lU#R%oJ&oO5cnnePLq5GB*Vavbd-<9m^|<%by^je-#`Sg}WfRDYd-1
z3S~B1p((c9@Q=WAv=ll^XNP*3Y}e$27tEpqofsmjZ2yX^X`{pzWY@QmPwbfM*3YgF
zPM%cz+K<28W>IZB1rb(+n4~MkYL(nSqHgNM;LEARp8k2&J;749>q!z!T*@N!8{mF-
zOyDM)xf-!if9WtNIla@BAYZpv?lZXJ{MOK)RyYe>A_ojY_ot{}+<!I_+H}xf)yTNI
zOyE295PHPlO(qa|OsF$(12yI~$7rXUbFE1l5!Uo9^U5jj_kwSBsf#^D9;PjeFFU1?
z#ki72);48NwGOef6!Yp|iAs*18h4s?a?>+J!Q4SGHYUaS8fkU6GW508o?py}vLdg*
z&}JTAa(UTFkiJFQB^N2gj+ViwdAcsRNuNBz)$^@9?QU443f4ZILe30oOIW#W>DFZS
zypfk+vh_z9yh$FJb~+I`%eN+BB}LalW^#JAKrwPXb*JA*1Sm`{LGEqMEDyRh|IsK8
z@~?5)VrmO~7rHDnZ7-_9GeBk>b@5%KvgVf@rES=1ZTp~qHq=Af`a6AP2}xKbmDJvU
zW|wGw@xH*31=JzSVf%@`s0fcitNL2~8bG#HGBt0lf=SXf0{OANFuCs#4L$}?6X?wC
z+BD&64P?qF$#nh&a!t^>GkkRTbhdSBldp|4U>T}f`GEi&U`mwRJ-0V(V`adL-o^^W
z>Q11}n2K=qV{%{<J9|O0IY;O$i)r;S=bc;3hZ%Uin$KDbE~8PLzmL`UdfQ$%idQ(l
zON4@-j)$C`G9vwp&YCxjYx(X*GJx}lB1yriKT+;69Xp_&=N}`7Q(U7^`V@)PJDA!f
z-XVZg)L423RGk}Ou&vY=Y*PYGr>}kQM@&jv9cZN^)dJQCoy_iz+wHp~S=c;c<RWVB
z8x@ST(*#LqB^o_FHySMv#`roITB&f1S}{6bOnu0FP`y`mqZ&V1zDx^EFAP^BM8XW$
z;G65NY@CgfT+bW7ZtG|JL{fXYg{^Am5jP$fzYv`w?d|-MhO1Dy27y!Wt>v>|d8(Cp
zmDT=zovBkL4Zdy#<v}VTRw8lvcOF9rc3XO2=+=PA-+CeMSIQ|p^_;t!pG)j~h_ta1
z(RBjyP9I4ncf+5k8b;4fXu<dsPM6PQw{ZtBpk1NxeVlS%<EiQ3?f3dhIF4V-jiyZA
zrqA790EW?x_o^Zn_(OwIR7i(ArIHtgvTXy2a&&=Q!=6BeAOX94v>qbWSmvtZvg9Wu
zAR*1QMa<Op8@1Sk^lY98kK>j%syYkA4TKrz_exai@;qnO<Wcz`+BKG5f}WA4Rs35S
z`<%kyy}fx(E*-Hh8Pq~FM1C+0pJ0YU?5vu)eUm<3@N#qJH~BgvRDo1D?_5(yVfS_@
zCV}VE$^*S>m$02zd6L_ujWwd<0hDrMlU=>x2lWu(#A;Tss1cnb?3_g-gvkZp+j(t6
zB`jUUNyio1Mbp^3?Je-=ZE96ceLinC@VmrY$}=IJk~TWx)<r_RO^eokmyhM+sgI$>
zT?q0nDgqjYT&&+Ug7}Dp7rAR7EwAM$>VV(u6*rZ#K8bGHmK}`&e~8yEVWmq#G{_Yb
z8I8356pK19MiqNn7u?U1CUVWg7h3!>)m!A(&!je3mEv~i$~SrWk~LG^l!lXqMjJlY
zW_@Ih1fH%f$;x+sT`a<K@8*5xzSKlU!_@6!GDVbVsgUbFhZGjIK4L(}?Dx@WQFroJ
z5i}z`FPaf^9_AJu^4sq=G)`BHp75IyiDY;ZkxS8+eG31~p$g^hYaQ1r_P5;jkUWwr
zf|!gXlnz-M_q=|oR}d5$015Hl4aCzgNl}Q5?7eJ#+plmw)N|K_-YK9tEdR5;9P4?J
zto-l_XEuW>9F9Ojpe|x$L9c`zJJxHb{b;XSp<~ZOme1!rpuMG>c@gZ*@Y-zqdjl6<
zJEueQxQ4(I1#U`BGC#ol<woFz=MJ<ML9eyKVbN}(n5RxZU$UsPk{A|mgJOdW1DDNR
z&gnm^YIwwdmjeG;3EnupwZqQv-o9Dz)QTUJbl103I(VQ2GR!Vo5>+2JA2x}4kr*^&
zj>%+KeVG4XP>9W&m%DoBVb2m2L)9me*gKH+v_XKF4Qyd2jG30UJcycQKT{qTW@Q_U
z?ilCkUtktX1=gFnWbCM`p(W{a2ly=d4jU1{YOGuFq-iQ=>JR>X)W)XrUT@XwkT22H
zIE;!AazW?WZBZ%J9Y4Ki@qiEUA?!&{18RJ%)-s{-Y81<CGg<%Q@Ux6vA>=icNsRaX
zJ&mq+wm`t(Wgptz)b(kTM&g!PFZcBrrCt%pp0iJ-$ztE(Yfsbf;_C!=_iv?EP^pEV
z!S##y<HkQa2~rRe0LwlDYvp^gi*q{M-kblkvtOfFRAm|0nvkvJ8eEp2dz<uYW<P7n
z{pO?PJbG9sK-nNSw|2*$ekIOi+(mGs0%B%_s?4Jwpqtz}Vq(b{92d=NDc65bHk59=
zQ(Y)aCNZ7S<vEV-(JVM#w*rlKJ)IHsm&(5hhugdvjFJw&M4aDyYP!4yI{s|th9+E}
z@JJ_9!KV{`Sb-5ym=CpFRj+S2^0#EGT!`P)l)OaSgTK<Ki1<44={#ieSj`)JOCrl9
z+ibN<uUKbh_xv!5AAYc>2&nveJo4<8;K3Q>RhFoAFR|HtMBW={@Tq>}>B+t+VlN?g
zbJG~<Ow&e)%vjBfsL(UEJ>DA54Yw>4dA3~sr&j+^_sevo_VF7aDn;QYWBybYvvow^
z4BN;Wa_?;Rv3hehc@C^#OOlr2!Ua)f7^OPtvG#n#E~_>bX-`x#$}42>Bic)l>lB_j
z2jm>}lEP;B=KM7L`3kf!lkL(xq!t1zzR8nlM#ROL`VrC8%<Lk{U;1s*IKlch!-~9V
z8~NpSfZZLk_y}-J@hl@hgD?*^l`H$J^x>+L0xPmQmb#hEeJu)&GW4`qPl5yl50)Z*
zS{(F>k%BMh(ajd(oji+laLc6?U8`lQvDps;(zZ}6FI6$G%)zIhA3=m;#hY2)5Q;q~
zm#7c8V!7-2Lvgb#x5Di+m*daXcQk*^{o@pn>{-K5i$vO9SuhJfYIy}oly<%Eh7)vW
zWX@66UR#lKdHAOEWab?F(TslVQ@naxp<&hXMVVyY!3(oqp~NB4tx1jzkh^f@tzv9u
zhSyIdPeiRyRB*@GH|?bF1fS`C(X5D}A3M*K79i>|jP#Elo>(~`O^*2d6l@hnUx2Ax
z1Un@=o=Sk?@z)$oyM8zRn)X^;6KiI>?`rbR#9xFsa3&4Mv2zvtjUR_uB6~9q#&M`Q
zrEQYqk|ZlRk<hX<@FB^Wx&~r%;Xo5r2vQZ?2ms<X`|iOMexC<9Ck>ehGfhr^8$ump
zN2HtJ9tow<_CbhVMhu$6I*}}HI~zUbfAKnnqA5LhKXa~qhwKSGFh{b<%?F@x(xHCq
zY{3p$FJQ$pp~}`;g5~VDJbn%Z*|zoV{Knn60>fAbu!{s~mC=<lx%nOY+l|G>z<TRv
z&yBw&BVSIg6h@Gw<EE8=SA5u{AM~gzKD%M4MswP)<B{Rr`*{wTE=jr$G0V>_$}OT*
zo;&AjF>9I?n3kdZX*PAHp(ClS07k-q(=-U)3n_77vF=>NjbpU!7LnoTT5B-Sb;98-
zIK;D~J3ipzrZ<c}jE;0E5#HE2p&goQTCnupslsVIDVg<XjrYN7cT?b-W;_J>_YYRP
z0Vb3mPR<q6oD)NDP$>38K?rNZH7NNtK=Gr=E=R~G(_vV{!=^@_?V*H|$(&@N2}Rw1
z_$Wo#kjX_kZK#r@<UK2cM_0?q=lP)FGX|*?JA>-9MPmhKEXn3W|96M}L}z0RQ*&&k
z?D&degPd;(M4GmHK^LELWx*XX342Lt++|o(DURH#&pM%5OGox%Iv`V`i>lv0(rR~&
z0H!_$0kh<o`Vt`UN74y(B}RY;Rw6{&)}J!@fvoEQa+yT80Pnww$A_-nG1ZsP7{O{h
zoV3#At~6?`j`f1W^JWVTntHknTtV?Q`sY-|I1CHU`}BZHqm8)>x4bAvCHVpkXq&xd
zxZxkagByc7QRmDs>gH)sPq{z`<=x3?M6a}@AV&zD$|)U#=Z_5CG1*Y%R}s^R-=ZL*
z0M;N7xWqa4CbkgCm2g>jv%Pe*#<yrfK2Hk20;%9D79@Rc=BKL1yT6dLC9j!kKp7C*
zNW`WC-O<+MsE+O-SN>g}Oa1Gfm+pAyhBB%s2!!Zdt!TH_@eVQL9*Ao+UoA5-PS&1Z
zqtlz|9nvV=t1_=k^DvaqSRx@<+o(F4PQdm=5pMDo+2I4NStQYOol$vESylThLk0go
zhR%Ym4Xj(Ew=G(vNO70q?u5V;cXuxq+&zKL;O_3lU4ugy+}$Aw6bVi!779$i=kh1c
z-shb6UCS0!=lIY3jCZg5pkH&VeL(9cEcaePcs{U(2tU{plQg4TB|>}>1au{qf4<gV
z$fy=~$1ke%-I#dhzpf7M(ZnT2d9%u}Fvexj$N4#0D<$ELq>n7s2|}D*b7igd4)ne=
z6d7VzY>8CYSYBsQ8_9GzN~giqfTHOttpTDFsXkU}MVrQQlM;r&-OPzE$xSwoBD8vS
zL^fMy@{N8Bs^SjUr7C|uH@W(46MM_==AUS5k1<-1a}VuOO6*sBC?utD1Pm9#r*#Q4
zPD|Sf*%c$@YAcIW!s4F>TS2$wFzU@lda(f3A6tX3-ujhzC5a{Z0OHQ`nc$rqO9O&O
z{wUqDPQQtjQ=f6(+=0aG-~j+>#O$EWZ;rC&D~E5HS)AS1;)}@?!VI{3^SAQg3nXqb
z)lR5h|Eoz|n2i5r2|a7*M?tm)G<kG8nITg;g)-f(IrJ*q&<YmebBjw#Z6DHT-5=~X
zcx{v=EbA)={8Zt1<@sPo{)uCTjp)|}Ynj|&Pr1WN0h|z^|M|i&QAv8;vuHXrgdba*
z@?Wz0eb*=+!JdtH)+}uO*D0MI^oGTId*|_B=CqCuty$+jv^7!ZZc&ah51xKpOO#m;
z|B)q)2R$>r#U>+<3bfmahZ$P>H}0*Rd39>Mw>c*irk9mndR%^g!&b+WaIwEvd{Mhc
zg$v$H;d<ROjG%xl`Ui0WaX$*?zQuf_$u+6Bdo#VtK!lQzEEX}M-pTT;8RxM;MV{mq
z2Q$!O_c%^wE2%2+OJ2D#Xhgphb$7Rb?>A|>HP2%6f38@+<ueD6vvlou({4p0SObHC
z&?N?QTc(a72ZQ(M7L4~`Pd^UH*8ENuI}&-}byIWRXM<Ciekt<}l^C8n?fRh>k}&(P
zegfbUA-Lxq_oKw=du1{!#K*%T973_Mq-J&17jrN;l6Y@vF#?<bFmnXUkTzDhO4maW
z2{NRzlM^C-SPE%)87I>TNvR4aF5#E`(ms>={oFH`9wRbR%AcCf#D!g>NUS`6H0&Pw
zFPM^MC0Rd}9s;SR!W}QN+0<Rx1BHoEoy9Z;rA~|ZvXUi5DWGKOu$edWkzs<!=6Y4i
zYvpOOv1<Qk4@LYRE=Tfwd{G#24S@<)qPWP#>A=DK`?CzqfQ`$MKK3(jzi*fIXf8jx
zHBOFw?TntQnv+JFd?`^7rtit!j<mI3?#F&3xIE$$NK&Qw_lrOR=_=T6O&_Joe13|Y
z1j4Cjx;fM}wvF&0o;nv+SJa+scv*v@Gmci4i1vv|*U7u7Jip;@`9-`5Q@(@t{o4mk
zXuT-D8T>t$!mIS++4PqdE1hCpLF1Fb)y>Pvq92OPyx|{&a<fNh!(>D&G)0K8WKRO^
zZl!z%Eig5ttNB-E^$Q9pvq&8cTjC{uEX_f*haa1UNzuTUS45QL&gq^F?|KimLaFe=
zH@2#Lx1+wOEg59rN4blxl@j|rkXc_`v6)`Ww|!A{l$`%Lu2Uk3HuCCB?Bv(%*7(IP
zqj8Va5Gy@O_haR?7U)~Qq{{VP;+~Wz5+;AV=z^$jSv6NvdCG#<JoHY{&oBJt&I%2(
zw9aRBEFZE`_hsvnEB#I#p$7ImFU9A#hOPj#mZ}CPWgp>$=;Nsn<f605OeH>hDvErX
zINPbO)MBfUqD~okZ*1HL10HZ#UW<f_;Y#q$US}E!O59P6R2k<r%vEVNEp{(FmHh2t
zeIBP?fbTLq6m3!8jM{2GLOO)0CLn&Cp9%VHOZ^GwpW9WN_A9=@MnJt1v-cj>W9+s>
za>t*X^HdJ;Fpm%$DL->2*uRdPoG|gBJoq@c#jdh<u__b=gu&DK?}iO(mj9_Ckp<{w
zi@o+pS<5EX%IVa2(B7V8pL>yN5q_4EzM+`^Jt%%qOGb_yMxWI+p<Is#3kcYPur2)j
zMZjMJi6@G~I~9ObvGoY?n@;EPx!p<HH5QU^{)Zv~svVPND#mhPZxnhF=tC-PQEd1T
z&g3OkrnWbe1dyzZA{qFn$FHGGJ;eo0im(i1TnsI&?V<0{k6}KP?a{kEnuNFqdu)p$
zn$UCa#c#UIVP?BtF$>cD#M<qtW48iQ)to;@>dF_jh%s7daEfB^nW0n*E~T4(e}RBe
z_d1@djkMRHdUYd%BV^jTXR5}`(j=hznz2g16wjAb-f9D7Q&2sV9(VHcaj)qI7vrEk
zwfW=ali$D6p>-}*5^LUK@#eu1Q&|$2A&mdmu}tRxc&~ytB}uisyF8}cJ57K_r+~`E
zj4#EkjJ)M)x{YC9H8E)iHzWp62a!nLsQ9-`ZwbYdz>qw|zMi09>XL+=kM|NPjwJOY
zYrx;PTMb6FjqFs4@bxYrk}48_hYW=d`8BDru<Aiq%}*)s6C1k=EGk!bcRlq289lkV
z9a7SRdPL~<;JsSnmzc`&K<N1VR!PsB`$<}*H$M*l?y-1>aVOrQrAg0cB2r7GEHv+W
zIW<}B+q&+al>JAaJkVlVg=A!ENU%d@M_Y+E&o2P!ZS3s32hdti{GX!J%~aI8hNJUd
zRr!xH@g8*5u`ALbiO$$O6|KP0uI17X{!9b54a?mo$Me=KC3^J36GE{v(pWB_#`d&^
zwP2yo8jq8!TwZX6h&`L}KPJU;5elIc8NZWDIP;p}Kw@jL5^68*8S~~;Gj9i_{y!%?
zQ&PG<SJ5Zhfj5(08XUQ9F&WJU-h9jnNsZ7d8OLNPmAkz1`vhC!MWMz8d!s=9+BC%k
zKse(OZx$`GzN(OIvs+%tktRpk;c-ZN-Tp0u1?s+8mRZCRZ^F*KuUQB--#S(>x*<cF
z6}zA$M&mTE-o}|NMW^xC8GikP=r*4NSj#zNBuK;;tI04sDvQSHJwxK&Ek-@L5+y_A
z<TrOfC_~B0WB&$yai&SwDSvHsDSi6C1U7?UEk~Q~s8yEfOqD=kfU_PdDxAmbOU$gc
zmAs%AX+LFmnAN6mitJvh^r=^Ju4&wfyIgY16K}X~+S2-oxTj)DtT_4_;+dRb*&(Y^
zV;PVRmoD5P{v6i&86dFnz$Y*QxA9tr&GcWIssEaQn0Gr9Adgo@b(1D7`~V?)5c}!+
z-XMch4W@ETSO_MBzVIagX>Ml2sZvmdKTZ&9;to3t$w0;>Wf2!l<9NJej?yf8dK-*?
z5eQT`InrfhUg^8mcYE?C(4}p$s|W3T6L(ijmFodkES{!Oye+(`rft+ct7Ag^y_ZD2
zE+d|n0+K*YEVR(Y4ul^yr+mThVDCN?>R0hGq2hz@2D^3V@D&>DoJcJ#JFq&Bj2rF#
zpYQ_Y(tQ3w=DK3TL?-WiZO+*tAjZp>2mqEmKr+laFi&0e9?}yubEiLtw9GYgCn(Er
z6<+Z-R$fc^OY9_>HWK^JV!Va^d^fo><L)b+>&hqKC3*c6VXkl)n#zI60RLQFwhFji
zYx!ncT^X!$)=s|pBKaT}RkW{^482x&t|D(wazwPBwf*7J|CXZsaffI0>dV=3czp)T
z-RA=;2d83>*bc-_ocrc$zcmqjab4((wKC08OW;~yc-Rq5(KPbn=K|>PFhA^^q>Lpt
z%@b!4aFFbA_+XI$X@F>%8;w0Ves5I%Tv#`MC<yzoCfZthNgLd6Xoa**KM)KzS!}yx
zSLGK~H9kHS<rbhvy-&agD_1$Z@zI*Tu)AU9jy}$54$Rc@%@5Q&&hHvr+N${VMX?Sh
zFM$Jq7%A;}e&W@B$$kMk*?$A|BzQF-@G2`q+fXO)qhYXQw#?2!I5n+G8VjCv_{`bR
z#%+HSCjX(dRU9IB7IEM=)KviXe?;sPje}3Ko{d4=ktGaa!3q^PAo8^ek#p&5P??!w
z5!ht$FO`>{lt;+#3+OD3Z9zEZL&kmS6COcG)IF(g^~TP>I$CaVne`qgCwxzqlnn*b
z&`%n7-T>0+bH@P8Um&X*vBz$Ws6d@6A-+&2^3hk<N0=_zzWX7sRlt8^yrOMufiMN8
zj0YXA=a%@LRY(`REW~Q`-aGI*@m%Iu5d5M#g+t2oLK>UqBGqO!`W)1YSLq&$YG(`z
zTy7ieeQav$yrC#EjvZKGJ00ViB8sQ0JKOeyHin(ZnYW%6wfn-9mmKvQ$O6<N%?D%P
zCGf?#v@fB7S4kI8gW-Une1NKf*-~Z(QQ!GHXFQ8@YA;LsPRADX=?DAL@B8jLICHA}
z!+m0;9Sbz5kog%jMkmgj>hQ5d$m`V0K|0vv4{5MFehr5b!g@0FRY2k{TGCqLiLyO>
zOC>&kP_i!azX8BYuIhy?TT*%0IepltX0vFOey^9xT0f!t%S9>Y++V-4^a(xh^-hR5
ze$*rX#$u(R$D|O>aKd=|Q48ou*s+AgRPRl%Ats^aeAF%}yy3LRi;T&NSg&fdBOs{(
zh@cvM;!piN*W`4vc%cX(beeu>8u9g$L;T#G%OXSYDu;V`3rv_YiKfbh*%sU}q%0f-
zc_af5Qsfk#Jk70+Is1(ZnJCFeyqx0?MP4a3|0j!#c0CxVw)fu_Z9v1}%D{%XB3jm;
zJ<aa@5gg;-_%J^`2Mxeo&Tg-noRi^ux&b=rQ6Ijqwe{8O*;n(GV<I|*5PZGoRn-x)
zyxzpjmg`iTkkTJ1n5W8U=McJ-22s29pvjjYCxeH*#?tOTMLRf3Gc#>=|HMdp2QEvs
zcu<?xjm;(3XR6RtIQaEY?nKR=vO^_j`I4AYV<&z4w-U)9))|2~!E!DgGj+yGv|mop
zx4%p!Kc<#AE7XuylpjesTopGD^&Q7@o6bS8AlIFU1=OJj-F1<X;^6lRhHU21hmqKU
z5*Y}^g?%mq-&oKKUU-Wz6V!pf3vOx-X8Vp=m#Ns>&$1Qu3)oL18T#j?sePshDlpV=
zpx9UzdZhhVQ0^a=;{mqE8ddBq1A)g=?uG!Rc5j=H;Z`@DZc};=BrP5w@z4KM3xCj8
z06R<7h?tjiGP;f($$@%4PLQg}EmbQR&cVE^af=q00){S&p=9$$_X1QIw+g==EHcYL
z+T>W)6ONs3VGyOHF!E{=Z>oc|e!A21O4jK*X=kDOdk4IdN=YyJfTkjl+elZ>Qk8-a
zx-|p)FMhW7@>O9~=?L2Cxx(kxK-IPQPji|R6+Z@itQ5x#MS+}-9C@Suv_65$C0JXc
zzPpd3;xabH3%ug_OZDr<tEgxNQ}YXA34ow)IWr5I(T;G6%A-RCepUGEn@wF#{XhhF
z;;$JcX;G(HQ<uSxA1bGvzl$ZDaSdl}RaaI>0UfR>CmuH;<sYnnAfjp@YJF6?5E)j{
zRIbA_@h@|Qv}=87@bN2h)8b7W?g%@}#7wh6QQ@O%L-+j70bAv42_+;u+-rlBwNc`^
zLIs4o-i1Qi!SCxN>!FjtjQg2bgYOMjMILV{7Vbv=Q=_^vPIT^AP{+!8kAN54T}kfK
zr#i#%Vo0SZqTk+)`YRX}_MQ0_DJ`zcXna5w*0M!F$Q@eXu3S}asy_yH+kNEPm{zgb
z!g<(=#6H>*CKC^}S}oaviC)Dk``r*qZdVPYT=C6A4(5mbJhnM=1niqW1%E+CVi)>f
z)g=skeoe&OzrVAJ1jc@yoOq|7{@C5(R|fep^lREq91)ERziRJymBq(uNn4wsPts8v
z4%a}?6uS#d8B|uu)riG!HL}Blp)p(Eq_%eShP5bvYHf5`z?DQvj_`+ln~Eig#hdSf
zdVOI16ZU&6NAgwIZ;A0@e6x;L*XyQ^%`bJ(!SyKUhsQwjK&!_NNkfX57Dd3B!^dR5
z0T~OAU^yWzBd#Cl-MZ=1UheZ`ff3EylH0?WR?eDsN~l?k>}zS4&cW<BJXqY{x?w0*
zd#oJiY*POn>i}DO&kD*e3Q+3c42<N>6%+WV2(l%kw(w2lD&?|KCPVkj@9BkWGfoVe
z8U0rOr^Zx0LfUIhl<NseYcRj6u{Vq{O%}Fxr{Z}+P3+4KgS|sY)Y?PtTRoz91~mPB
zIO@v8sizWZGGC!<R@d%DC#2Lp{>DYq8ePt%LHt*6Dz~|dbVV#!*y&u5&{AE2?!>cX
ziRBW9s(N^ZajXi}&^N=|2|b0Z4<pBQ>zrP8$k`xYidPj>>?;`nSZ7LZX}Tw~f^+Z;
zHCfmTe-uhVtgu^YK)C6s*8d}Atakj>U%f<#eQrokUtxofT%7J`iD{W#KJK`cT(I-y
zO~1f!Vw;j3ok+MM{Jq?wt?zy9q+km+mOJWWCFIeW;n)<x%NnlhA>xV`MbVx?6{&Hu
zjf&r=6)6>y^)IYt9WW|Isk9Sc?<e1Z@;N>?<`UI|gDENJv=Ju)sWRuN%XJVk{0Fb+
zlMUCY&D?2La+K(-$(F4x#F?c~;b7n(o%HPL8LKyyy1&K<Kkrn62r5S-0?87m+ex0c
zg-{4p*4rH@vkHH5T2K;ELT~nO>R<lc9?7p2*17(<z{{eLytWgH;Awd2+8c#SJ*-SH
zk*b6HWWY6dzIfkLZv(CcT7g>4dlNLgqs(#$+ylpUJn8<)ti|WO$1_5q^XBBo;wLJX
zdmqnvNdqZ=qfUwcWDnaR>Qu!+Asn1_w5fZBXcsg+XPaV`@QmMVWs`Lr+ii&O`bn>m
zysS_PYwu6HlK@dO4R#a1$(L5xkiWFoTiN*Wx_|{})mx~NVpNyxA^<gooX&WsCik7L
zBU=RwK$3KuUle5T7luk)sm?<FS>Gg@9D}7-NVtDbn67#Nw<ODS2(eeZ5J&f_FQ0~~
zL#MzBl}n_hYulsZEY1*@qOmQp`!`0oM;yeWc8VKFg!H11R&INl^a}35O*)a(IThaK
z9!hO0Tt4MR(0Ws)#+alYW`A!VcAwMbn<{?X2yy+Dzb8;Db~MGUmhoudG#d{1D)1N9
zBt4r}xVahWrdV$OhW^1|VR;u`%+#D4nYOtgQBnfbB{nTgEA+<FBB(mm>%1mtj5!sk
zvu4c{b8Qp?H+gVuZaqNc$TPKwt`CQ>jlHkk*e{An9~UgOwLeI!s0GEQ>Q^oXs<*;{
zvR0M_Sjx3jy9Hs=5K!}-ppe1UaA$*#7u$i!im)3(Z#*aR{powHk#{Xq-GaG=zKnrf
z$-1fZS7vt~f})&t-4U<b;5Gaqw9QsG1njoa+*${7oydZ=)22`KNhd_9R&3anv#i=b
zYlx9#)so^3#kk+irNO$pyd>2jq5o=WzCBR4x(K|fLT!qJEH?x=Q)HSmK6-b)alUGp
zi1=U#RGFWwJ?Do1EBxmZlbCNR2|$sMW=Eq|0_xmhlt8x*&_W+~t##Q#P<WcFAt9?>
z+Dg&_zaem>+t-=Mkh+z|)164VWp7}a$d|>AJ^EKfqpoDQPvWXtKpb86-E&<cvtSoj
zsV?)%qj6OTO^=Vq?Qjq#s99~Ubna)zYuI{gy`0c!ve+hIYz$XE!vCO}Ayyv$Nz`cZ
zTKrL)|1E<<x6=@F-+-3x&w`UVxLfm$dGa81w_^0>bUMmP<l7kmfiX2+(dJU0I^*df
zy??^fh%ZL>WH^0yhRy@e^M@<d|L-UZzR$$&5faE8Tm0}99+mQ!&#YNElh~?fz-7JS
z%x)O#s<uX*NERDwK+MHr!nyO5W?*O{A*t`j+-US{nH(k`Y$bMOfi{cQ+_Q<t3i%V;
zJl$W7U$eG*FYSA1&Q_5I@VVjPq#IC7o*x4`%Z{sUn`Y}wpSJO%^bNI=G<gtw7eh(G
z%={J#<W!OO<bFMgpJxjruuQtN#ugNSuI_YTNsmw7(d7K(DU`%jijA48dUZVcS~bBR
zuNC9jRU&E2RX<sE%kfl4r-O0rDmPX?v3|Y654yxkGdJTW<Im@wO54SCyKIRgNi&bP
zB3-k4aVx;Nv0x(*LtyV2iZMTbs0W}o?M*|k*6!U*t|Ysto9NvFYS^p;LdoI}?VXCb
z<W}r3pevOCb*I0b#3-!4<8Lmd>Mv(oaLB6(oBheaQ)h6oCbCe{Hg(o<{dsxTqS~GO
zeIl9KIRm?_B!&NB%nr>cB&q^Z;LHG$Y<2irSKG0@A6QTq4|@<@Aa3C4f7D$9ef*yL
za|t4wa=VAjRqwdR27f5<5k*gY?ZEM@y<9owEQ8mk?Fk!yg2`-P!WC!4Gz>TvB40a1
zc!@f~>}DYaCE^=`q<L$eb{QclaR9xk{{ARA$~a+Pa-cVl2<yhxU5xsME}M|*7kJgJ
zRd~GiTI82|7a)w*@-p=%mNH?p8}m$l-Rh5^5xJ68)Xq>z<7@dbSF>a3A;Pz_RSHFZ
z=UAefa9&?_30BdZrdKvQ=>xZP>%SXUlStz~NQ4LmSj@6n&(m}Xk1HM~UVlm6AbGnJ
zJwB=kmpI^yImyP?)LoyGs;Gh<J?49ewb@Lqv@JvKO2xJ>hGrXd*tWeWr&3$q`lNDr
zGv{qxC7l+92BOAmGnZ0h8?4QutV!>WeL|+cr}$a}4;CyI4O9Q)-cxOLmnpq}-gLrN
z<&rjBu%TL67vWAw58)(bW|4Fl!OId~8=3%T_<QO};%a_r(<T!gx{mgI>M(gsqs9YJ
zYA+=)!Yw2Rcx^@Rq*g_<P1G%So}i!w3=yBkvyHK}swDFAMj?S})EriCj8Q0H6g~+d
zHxkF<Wce(>RtsO_5Yh#Gi?tSNbK3MA#5JLI_Z3sTz^;A6g;PFDuwBlta%T`H$p@v*
zuFlQWBPlFX3+kMP_N~xz#?7W^nFjaxm%vD{&qkPM_ii=w)JNa3#arJ2T8B*@nYExk
zG=+9c55xlU9z&12Lx1q~^1QR8r#O~LQJYm#(%8}vqCda2`Xhxx;(2yMIcJ3MQnj33
zdHsPM>F%FzhzH*Ad*#2ks3vF8#mP8lq~M^ho7gR<zdb`ELqgwi4g`EsU7RQgx}4iA
zjl;9~@pGTm3ob}xx2a=HBml^NxI_lXwl1%v>xKk1i0Y+2pM$Dki2pT0r?f23WzRwF
zm>sxxt{TVcfWkfK)Sb~eQ0Aa5P%Y+d96XWp^CyqBqbNjlBWRSp(cFHg8}HOZ(+o!`
zL|PfOd@(-)|8fvNUmvVBjHEZ+lnNQEiD3L1ySe&a_FEonU~0>iKsE?IUNinN+4?8M
z;tu$&x%n8EL=3K+TMxzx7l92u)&}AU7{V_O2&lGe$b8(`guplWvB44Nmjsmn6?wK!
z>%0#&sx}{F1%ywFJ|Yx}65l{}mW7ifOb(4f7;K}GdT(#yw-S`qE-QLpi5EysVMd9Y
zO}TBH&~QaP_Nm_AT*W)sBW*2=LMW10?(x<yZa7A9Z|bvf@#Imh7;P3y6ocF+!mc6Y
z#Fwl6&V(e{JOhqNb_uDVt5NhjYVSTPi_JMp;XSXT5D_Fbj=d6fg+&t;0dq$oMy<5f
zuCk_DFu+D?j+~9q;e{DLJl|m?xXo}omhf>UcCSY?j~y@(VkDLXs=UspFMg)lglunu
zi9VvhqQH#SF|DUE|I)XKGcN<HKnjdrIeSIUUteBG(Lz$eNhb^x)y4VwYYMrg*U6bT
zgPaPDe+4>$_%QDpMmP30WTEo`?dAUBEQhjuoWOONJ9f|Rmo$OUogtS(aHYB;Z0C4T
zTSHaH)m>Zm!)Op?#QF|w<M`<CsC?O=NTUOJEwV4&r{K`SFxrtBBPO&K6G-c*(zKH^
zg<~9@nA2Qy&JOF{u3Zg6ThvDGp0=cH>c_B%uU}jD-jiFKNyqd8PRPv|Onx56NE8+s
z^5zfPd`j;!cxg@3*)VCr0`t@u$Xk93+QaaLPD^`3v4M;H<tjm=;1pPDT^^GxV}&8n
zqKEvBY{zU5^38;!TH!x)@MM-7(COPq&uA{eQgcE7n&*M3??c_B8XO)Exb}GU7W1Zy
zt$U=mC2UpZNE7>lp*W{_iw^nmSPN#aNx5s%`=TkHf?OzbMWrd!&d3}!rI*-#?%HlI
zsOF@;;q%tOyZPFV(eNx^W9Ap(Zb6~lCMOSM>+C0U(@wijv(eDO*cANjRI{P6qjSiQ
z8oTPyWSgDJApVi;kUPIOsJo(MyaI8uck1lq!yNCsdnGpS-K%v4!|Wo&CYK~F?sTkC
z!$!N_P-36drPn@7G?L?^`HtcC29>t-tO23tzxohZY&~|e@2Blp&%NYT73SaNs?9*_
zqV!A;=^at^{_Vo%$~6-XK_`axEA%B?%$84G`7+7fBLQV&qg<dP2~;h$l&{zMa)4+5
zf+M*qbCl1n@4EEmdh+)W`xm?+2+ToVKTp!x+8J8%CjIewk_t4G2Q@Dj<y;@mrCs-G
zB(U*6AfMB~M_@^pi4lW~H6g?F|Ni)9D>y(@!w2F%r5diJvd2FyrhA;HGsoLXHVYH}
zUcG6zv)&>nAWDi~k%^jD$Vl9Z5~JP<m6l9F95UAUR1mbk!WEF^#YkG1WcaE|1jXex
z$)2U3gJg(sp2TL&GAs?8G_(+2jRuP<gT|A@Jm5Og5_twbO1X-Sfrf;R^NLdS3Ceq{
zJ9^E{ekH&k=sG)TtKy<XFSb<o^L5BKAG<*iI#<xGR%dj)H2=FU3fbVnuq88hXi}fB
zC<svKQwWRozmw%(_O7ci&rSh+U)xN+;dcR?+_yU5q>J1GV&rzD8GB0T0j~MK>q=VK
zF&@Cx**TgYr&u!F$+cN(f)Zu!40$e(`lNxZjlPu*;wO^gU|QY*{^IaCaPYhvwP+_W
z6{>L-&zG{dfKLor5+)iH1Wtp8w@OZ#MI5NC^TEp=cO@Dz$bB7u`P#GH{FTp7g0&#A
za+Ue<dN8mLy)gHq$Fo%9(d5Hn;AxjjN#E>pVXYJP8ObHGc|*vRAf1M*U)`)mHbZps
zRq>GyD`S&Ht117cuHH<9B`s;+Hj#kfPEqWZ;lK;^z%FH6#nn?_${c(|{;6d5^YtyX
zhl%tS!r(qhe0e6bDJ*dyhyZZ=yHED3O56Bs$<B#E-;~d$WqBPZ2&}czsW@a!hxWx?
z)}cEgY3Zu5RrFp9@~UwJ;h%JhO~s((Ikaf^_4#v*p1TQO2rx{0PPN%)ooUwInv)@2
z#X_7bQjih<oPRjfto^6zT>BW|oegg2wW7MP=%Pi-qlCM7BBdTf;fiWJR7ZfU-4k@;
zYhD;7Yo6-iC7EMASw<CJtQ`EkPEAsb`JQ;AG@spif-Em%R+-=C`^1mC3xZX?N|x`I
zM?1i5eJuHHioIyy4A1mnlq@oPPcc^&$+akC9v^Cb$VO!bj+l_q<><I9wa6S0KILZ0
zm7Ag&kn+wgve1;cQIpYTMj#;Yh~_1WFEKUY2fmE(1daLg=+Em0JeqAMwFR9dSGN=o
z`!zbDBao!|yPPn!1DK+zkVQBA;9tk^t4)}aMK(=_Lw<?*m&*Cd_sqX_&`50vOaN@l
zD0Mw8a9?xebTv(p%~A!xjk2j(xmx<{MPDY~jr|DJYTY?M0IP&DTvmFKUMWI2EsnR!
zAX4h%tfY+b{+3F7meL?d+o%Aa@GUxGepX(#*lK!8=Wmu=Vf3airLvXsUzSbw{r`EL
zddEXg7gk{0BQB1$9l|}u4V{7n<Mdl#m&xc$xIa#+Trp4rm11LvtZIix|8!w^sNW_Q
znC-dS3LRSl>VhM!YFP&Zz|XJlA7em#0Hu!Vrp#h}wl^>^%LvKE&K?vEO@jsL_MChf
z@ccD>dFwzK>`b<f6}V*tD(e)<QOGFl`b*2%HIR5u=hj#lX(l8SwBWx<u4gSIpxuxM
z$xHpqyd#xh%`q=YZY7m9(`qy{s~tj&W8yS0VbQkvE5PlwzvZ?=LLkFN-lCdacyBQE
zao)-IN-tA^1tPo>ol9k4%Ds3-&3wHTt8u$HCPRDyv{#q?+bDEsT6_vAPd?z+<c+Q;
ztzTy+3)<z~-jUik85E0bfr&z`cO56pFZo=f>?&#I_wuj<NRMXqvWJ@34AjyGIf$?_
zg&JW$N8`Jybd~zHBMUB}dCop9+8Fv#E)K@}?E3O+L1QUTnN~uXEnEeq6TRi_(!P5(
zMC^U{uWKtbM&{DZl+(O_PkiX9eNlI6O{ova5)`?inevye+W%DSokc*Yk73KzjDLet
z)U?O;M&>7|5W*H2vvIz3T`yzr5Quqgit^SjEV1|!^@A$WOl>$O$iku_tvr7_JIx1q
zEV4a#q~z#fbB3&QSy?UmB&G@l@W)mTw!b$JnMWybU(SC!X)|$6-&f)g?fL8+gwoqR
z;b~8~bRg;?hgqJSXj$X$@^q^$4*)lSUIBj_4qw_F9ia!#zl_BRZ`7$miU@t0bK<cm
zO*ZLSHsv~fouDZ)$l}kaP|4l@U5e7C?NTX7684Dw@DpI|hVjHfK74Tap+Xyqtd*~}
z_v#4_+01$aR?=G*FspY+D=Y-LOGR(e-Cuo1;ZrFvF2`9i2|WQ~1nB&iHnfKQneRq&
zg34^7kgmL4syUxkr&II&btWFRY$@O+!aT#@mX)k?D{NUM{lRbSkzZuny}2g5rPpi6
zk?ZxVL^Q({JZJ7L@L)#@|AiJ08MUsH|9ZR(^y0(BvJQ1E<rSd??j3Eh9xIdvRY`5O
z-yB-Q&iCwp^aIxqc<i+e;|?u<9uCU5WxU39)mD`3ZD)l1w?0zZ?jDx0*<1tETb?l+
zZO$cTs!3NM^AzVi@n0Pn_jWs}oeZF8OqN(i@MIpUfo>QJ#q3l-0rb|FQn2P;YgC@>
zR*o1GorN7$`y5t}nXo$}OH~gOHPlM`u~>Z5dtiXF6J?#%a=m1W_X~OD$n*u(l@KA~
zsH?y!BD~wQV6g`9nMe!q_GGZp*$#*i0dwFh&8|Iig<8N!+kQEh_5Eo(+hA&^9v=x@
zxDYcce4%ICF>ebw84P*qH^U99?UP>P;7)g7qAS*UexWXP$9&PTcRSRC7+Tt^^NUij
zSs!lY)Q^?;pCj^i|A5fY?a@$k5&m1FO{oXsI4|JTl~m{a*FKgZXKx3&F&74S0KosK
zQ$-ecdb2zNC;u#o*IUSmqIHtAbR><@Vo3J0f;=GTmdejPv=7zw$i;U{^hc-6Eevk}
zB395(7G-=CCFDy#8Hh~&7l%SdDjg8#A%)71YH37O3J{B|onG9V_Cb0{9F$XO9yky!
zcR#0szM(StMx7Sk1bEH4DQaMx_GIRw$CL$AJ@%JLy|u{9?`#$9Q_`2wv(VPc{QK-=
z1F_#fZlpq;{Lt>xQ%nU%z}z4O=82~i0P?CQ@Ex!+SDSMd;$wXQw!2kUmz;6T=J$w-
zWh{jmIwqfPeMd4FMu*dL*|GPwqdEI{))GtQ{|vwZ&6`vG;?{#a2n505k|N!zDBX}O
z7`ec0h-X$fRuTxi!Zn^Vz0l%N_d9aA@+Ro`IWmd)*k!LBURZzhsA4)1I1U6kA@tw>
z=C{vKHQ6qqoLGewrT6Vj5J~#afHXrn{->gKQTvWMEkaUKivG^*^Hb2dPqKN3zm3M8
z-I#;i8W<v)xZNHe2)h5<OrU+zLvFfy;;_q1P!?nMI_5)hpy~>jwe8Pz>3fj9QT`|-
zFh@`O5w;Y|Vou!=zo^J>u4JP^^I0+MTh27zdUN#ceBksIde&LK&yTKu36A(fr`FQl
zAE3nA%5!1bulCb^_n`u2tdq*bo!z8WA-O$z%yKzHqb#};gMQrnp;#*+Yq~ppf<acA
zXEe#{N<-OhBhEJNQP-u>u12OiOTE8OtMsM#Z@xTgUVDF8Cl1!6>WSbwL+tC|w8(I;
z0h}6jE##WHu5XtdiIH_b)m<J}-hM|Lo7B2%8dKyadnb{yyo%FC1Z+~2zO#p@<T&$1
zFV7`BO2juUy`Nh>);YQiYNqTawX@qev*#`G#yg^}$*B4eu8NeyQ01SjHwD;t>?U*V
zTV6Ugozo{m18qivu+p9$Qrd~hjdIbahNY~nsX&EM<Yam81yW%@&<)uY_=eHz^}6>2
z>Fm>QzRX=}9C@fj^%-5t?K6(B4s{(+KQrnInfw<#&E1t?VJ|bPmoycMj*c;IQ*o$7
zW;Ptn2YU3?QLE6}VIgwLk;ju%pmrNcY6ats<X8BT``pow;=7`EHw@bV84)s!9bzM<
zsbSBqf9+~cORfchTprnQXu2=zHBxqo3(zH1#|lIvijZ0Q3lqD8LA19qLw%^Dh*t@-
zFwf2u4Jv=G<Z5MNv**IE+%YGujCEF2ouyRz(Q5)IDNU2J)rRzWBcCU^BQ-I;(RR;}
z#IThSr=5@7UI{{VC;VK=lTky0)QL6Fsm3k5{cm$rpr+^mWVoroEU}`W5l}bK&RHAt
zn~AW%nr)+w0P8pD@SkHZULS7>&#7$~!`<$G{6Doxv|Wjw?Tm=1=KO=~D<ef!Xw*Bq
zT&F1+<#-;FH*)$3xpFrS(UYV~ra@bzq@vxO)IOc+l((-&gv_DIewW`yrd8S^+Yh7~
zS<HI8@m6-}2E&w_rTr2p@_KJ+k=`nqNWKbv3}s%)zRtsD;>fn>kptzDSa`D<`lKp}
zZ4@+MKXX_#HO*kYgBgth_`J@Wf+ZxbH9f9Pa=Oo=_8NTi%QWSpal$JiCxqS_E2chA
z-C<QH(7qn|)Xd6<CAkj8WzT;@TSFDP$TyXdUlA%+G`!e6=s%&}^&+9c{zrI|GJaOI
z`OhB`Crj-^@f0=?_b0Yt5d-w`ZniFx*?FA_!}$>ef{fjJSHjdj_S1#-QnpfYtsHiT
zT-_75bLL3mCSHp64m`XKwEVGrasG_2mf^V5CGla1uOuaTn^YVlIjV<2$A_z$IC@I?
zjoW1^|Gg0kAxqc;yfks8x&1Sw(jrt@T;`lATvwWx$I+S6X0MS0&wk2KKm2^IMD7^L
z%jQhPEp^a*YbLzL0MV6c<U62CBK*sENJ}k_e;x-|bJZCCz&oRq!-ZpSGm^4d`(fr=
z0C(EE<$()DZmT<k>0$$Um-#3)=9Doy_LcR_)UxvTno(pU(w|5s8n@V#4+B|s%0({I
zOb(9(2*@8CH;SX;@T0TEPITWd(ZWxcXo#*8b0(m1G$#5$<ASu(XHkxmkvSH6ujHYX
zL)&OK9`i2}K6js&_#03B(-q?t-c}Ni^WHv(!^RasfkHi2NfC3@v<a_npV0%6-FTlQ
z7$tHkjhp6IXMq=MXtr6ZHo&EGQTMfE;veMhDdBJ;AW+TnjvSfZelj`ycD%R%x|4D3
zv4(A?5Q?yAih6HH|7<MG3Zg8h2lUNGl4Z2<C|W<m$<3Xr<@WhdDoNqqJfu~dEKN*b
zzD)cMf7xRRbmV7!84i54QTShIiGT=8&?4ihWQ)RGOJVSh^E@@}c*nvbDxQ7Bv@wN=
zOs4(R{o}{}Y~2PykNFQB!}(!dq!Zf$$WR-uizi6fa>2Gg_4f!}o$YD<pIzZ`h)&f7
zfoXV8fE!XZDv?83!YWIoMG6}FPl7DLB1zvAg)bq=tbd&6g1*#$1I~`<>+i<rgugG;
zuGW?l!g)M#!{_~vt6F}e4hBFQ+bj|^pKEG7vD2g&an**r#>EOGq;jC;B?!1mef?*H
zLG-kPFFc0TyMJ)E;Sobd>^q<koUXO#BRp=;CgB#(f>IZ=yW%Mg$tn7vD=X<ScL{*%
zwv5ezevN+G5&gsrqZu-6Lv3yT*D=2pcZw}`jstuE4IW_!JSC=|U{?_I+T7U4_^Fv~
z&Jdofo?@Pa8%HMi9~^92FdLLq45l5afIvIg0ZigKIdcIqcOUIb@>)4SMgNR_SZkci
zQ;BG|YN3#vAiUl=G<j=1@&yUlpPhGoH7)Dr`qNh#A#*>_WD~`TDoDqi_eBIui90qJ
z1~V$uOM)oFB^~{OZTXrzXfvE>7R%Z<C(W}9d;|kiE5~HfNzUPX3!nD#R8!P-7S@Fh
zj;qU6ro8>9+|nyL&WEZ8<3ns%LOon=ocod8yp-FcvL6_zbU!~rqA*dY#(`9BNuH^Q
zva+;}<F&R8B5P13^gD)e#2x_E8-<am_~-I@o5kUo{nS!)&gH=CBQ9&Hk1yJbynbnw
zW0QFC9R`e}zM^8*Q6zWf(#iB+6+W9iE}9C=(~ZrsHqL!)e2;Cts<^~!R6xfmSX_tW
zDlMcICr;G5y`b22I(s)SD4R9~gX+jxw25a6nmdfLGPGq3{^%`IOO;yT;geGA3$zJ|
zh~KNd_*cd7K1-AUpn&?a7;pDELI!#z(_NZ3mtsJx>kmp=zjfxT9&b+<&FPNK4&y~*
z+;R$XMB+^AljEx988MB^X7(7bnrcsC`?jRO8ftI7$vsLtYVh5U<8wU<K~>=SPyuIU
z+Okt?c~?jKJ}8Y;VvmtV6lwX0V=>q}2>cx}>vSxefZ{_?=K}ap;1IFVQ+`n%#Kftl
zWXQ?4uAtwk1J_N;Tgw~x@`~uEGH%0xD2$^y5MKo~kpox#5;b@r?TXS8jN;G|IIQ5=
zz<*2Etwe#5CplB^C=%@e>LJ>BSrJ`KNdu*)13LrY2&KG1A&eP2D__(}#?OX^oggla
zyrI*zcHY?=)1O`yc)oVDhTE_HACvlnc?+=Kwd(rP8_Fa{A==cU>+F|;R;44GVfk0`
z*+7-jZB;V2H42!pPZMBq1g7b)3SsxG4?17#>sv6K23+EOiuC=eBo(gFr|@m@dy|p8
zXLKyI7`Y_znM(O7;Gt@dAUGGJ_`yomsdv+1f>StOR8UKWE|d9b`@bJl)@P2xeeWj4
z#5d25@`ZzfKpz2lb^KbNV+xKXZHtp8EsNGTy(Qd=?{sCQFB5$i+MRIY4;PXuUxn($
zMI6oFe_U>P4e+f!xS@#|i88f`wNdtUrHZWghcPTp+gmr@*o=guy#Ao&N86(H?Kq7W
zJtYUG`z?9&#l@psZ)8|ZX+G;Y{T$tF)r9zs4L)Mb4#bQNCvI^m=F(d&fm@0<{0T;^
zGI9RL8fKaFq`=(?;rA-a4++cva5bG9-YV}iL0Xill}c<yKbY+I#WNo9?N)!a3Kk0d
zSJTGPj!VF5EC>w+{6PJfJyi!abUBBCaKdPB9qsxfr<Zqh=h2N_eQSP!hwl#>1d<LE
zx`|_RWl58XQVYWMF$NBD6X^4-%+INpV?OhSV#Yt9ebW6zsVq+7Gkc|LR;-_!p)FXG
z>67cF{ojRT53i%G^#e)I8EK4kPQx1sEmL1YTeh~gCNnmE^3CP{2b$uf^+`#(rY*%N
zLU%sJr(6BsQLZ3+CNX+A<M?hnG(~w<lKbN8y;ibhyxgqKE<;0GbFc4<8$g}G<R)~W
z;PSdvl^(~Q@&1`{l1EYHr1mEWE-uuYF!N#xJ|T1dA*@uB^i!CS?#5R66k2O@SqUn=
z@*Wiioj)){h;UT0Pc_aQA!k}1)M~{9*)`0>6$eQUGf@-z?RHi{*_vx?-itq_A`jaV
zwmoSRj#K&XhdO->H@&kdxkeTo3|4hW=H2oCrCBzC@9jbIG|3!XxD>DF#RLiM6og+!
z>Sb;JVmmRF98CSj>PHQ#44QpC^GR^02Zu(VP+-wFr{I2~m)^bblAWGQ#I!cC{z7kM
zCk+6jK1qiY1ZRP8xq0zrg`F#k5=@g?3zH7QNc!8B=%Vn(wtu}Hczr%Z%Wr=aXp0RC
zfvL`n?YP){T<`Ygg@L|sVqq!G2A<p7YTX|_1PwN765^bL!GJ&3EzNqY_3lO?&*C}i
zn3j`Y0>&TCg3f6r`4W^=Zi2jBp2qDqZh*XQL9X>}lPOBq`VB$KHmoqi?Rm_{q=Ajv
zBfGVeh`mQuN&_X<ea{5N=3Zrzz;~)#-9kGrjIAuf#8<z<%#UpAuXC}&qUbLAVpCe0
z9RduGO|Yu=g>_}H$)O!oM|iQ7S5JvWKDx%~2hWsKxto8?uP&L{c_fWWhL!?O`wWzb
z8Sn?s8@^>^mzrEvo_k@YG$yZK@lh&U!nO77fj@2xH_fs@?O%f>IBb&_0PwA3ZQaT^
z>1TvvGOwRr%P8K?2LQE(RX;+39wW%s9cQXtqg+i^8O<>7g!(2FP=UJM=oQn?No_jd
znwG&{>$;uqpkd{0XZqmsE7{I;aD-V$-i>Vd$sehMp>y61tN1-56K>j8!&E&IyAZTD
zpSMyBG%9mt2PZ_=RDC4*W&uM(c3G2}hfa53m7W8~bDX;z7F(_4OYjsAb}Kqq4vOKM
zr+dY}9y=yCR^Lm{{s18Lh4MVl!BsnjHjJHUBfML)d)}sJ5p5hwg=3amP8!7ujyYL@
zBf_4;DI4Zw>~foYhot#3K^BNvumt$=WLscOnxsetbwOR?H|)!>yKKbVva`$Qy|{O#
z*kzplZ?H&brc|Qc4J7&KOQ8Ya5;?#BVOk%0U7luvBWP)r9e!RUNQcrzlCecimsPwN
zwA{^nKZ>g~>#)2sq1}qOv<kHOs-DEv(b^CXQ@gPZmDZty2l<W!3#E~Ss^|-ZqM9ZV
ziT$wVcO{7(P?Pi#2IJj?pxxjNPpGeg#G*jL_l9JLQXGw6a2XPiDWFHeRA9YU!p25L
zG+(NbN-32}7-1Z@<;alP94XCk&{P>WrA-$($VycQ@x`TT%?BykD@oXiUu=X$VblF=
zB!*BptrYHSdGF-cl$ZzZn1@<{6vN{R!WMd=Vg~xe*-}Z;wTm?+(jsgAG8Qc9>v_31
z{qI+?L}(*Crnk?jishDrm*DOXceiK8)~pbbyIGxu#eB23r<NUHjC4ugUAlWI>CP_I
zT$YK~sY@S{avt5kq*tRiwQHji>-9c~q#<qhBFdLi{;IR8n4V*+02AXKY{N)+<tV1(
z#b9r&cJ#=>(EsIj`1XwDdN3uvKA8=#BVK9ZN7jU7_7~Aw7F=InoD<7SCWhGuZyn#Z
zM>`8?<~`Cc^)3$>@yj#CUNX6!#Ou#rltr<cPjxffdr7`qzpYs*PuV!;dB9E@cAin5
zILob#hWwRyJJLq?GGNX56O7msmHA2F?!%JJoaWoDOj-A+&mtE~R-FU!3;A`4Uvw9f
zM$aC~Gd`DYOPDu&!q2yP@J+j~*DX!4P=9pY<uvzBR7KB<I^O<kqI;^ZF;}3^Fiq_>
zJdxsP++B{tCkEl`EU?ZP)O3c=4oN}Gv;ukvuV&EB8=*2AH@PeRyJ22KOS;g}7j@5>
zYe)u(Lss#JagjbpE4#@<FXn2uoE6cju^vg&w3hb%oK|dSbs4JuTzi#)c#oAzA~>DU
zKRTEJhxw%Wi%wl2!!OZge|A~Cw#Lv}bvdpgGQOd~P!EJ+=%m=%rKoLV29cBmp`v8_
zZE<~J!K)}N*#kkM#C%JatSoH5jbd9lJPo@NgPAw2m!?jpHKepha&P>z0pGNZUq1gi
zH~8h+>axEeNicTknqGfz(T4*DD%_%`_Nx?3^)xmfrr*TOeSKw;l5XlJ5bWBDWeEs*
zDnA4pkS4o6H6-(wr#u-B)z{{CT08x>yHAJo!CVFA_%blRc~1kj*rcg1=y8ruOLjIf
z#K?}MB`SYCh{qf#b?tKAGKL=Cnd%L4h$NSJf8J!WTE7DTIM{5Y-`d@-JB_k+)1DXf
zcnxeETbq=%r+JwTAlG}<v7eoCHUY=zqOZT$l0%r?Np||)Z5QA@HQ6&x1Naf4MZpf>
zTtQ7UGA~2~O&>a9odv@$^iR(=*TWLnd2_yfvnyEbSK7ezr=Z8&-KiLx2t^Ej@l=<5
zkYB*_d_=3+6Vb7nE5yXDsN*K-+beFcOwhf+Pj+8S7dTRmv&U*ky}o7FRQ$JQ*kB=$
zW}PG?l}G(BGUrf1!x2qbmle9zr^~-)P&)D{1J9nHxP7wMsHu7WBtWj|%%x;>?t+q$
z&t6HmGC^@$?66YqXdE!)`TngfZgtKH#|ic(<&dyo9tHL`9d)jxrS}O_#g#QluI9B@
z%e#7YPgq=>e;Bi@6|Gu+sP%m~9!X-{G%3XGCwAvDA5-GfyR&JU;`N!XA5CZpqmB7i
z<77rR2O^kXGd^-#k{X*hvj0n;W@|y*W1YBVAxb^Sgg-C!%gmbq`=?8{@lOYis96)z
zKAF(Ds?R~<@!tK~pmfW5*IkZ_t`A8Kq>l@xGVbzP(>^(C1KY4%m*(3(oMLTRxm8`C
z$!Md<3@+G){3!)Inb0?OL8@xS$~-V9C%SaL-n0?>v`@+$3@?F9>HK!m-<Or%qSV5a
z5Bw8<RM5-<3vUDgvY;HIMd2u@-&kyUJ|AYEL_^?ur*;oLx7X~=PZ6uqTp)6Z@7+a&
z)FmSzSCasc$5baaZ;mHd@MYm&+ud;!@+!KUrdUD=HPwV#gK%g8yf{t-_Gpy?shQ09
zN~`S_?{=buj5z@uP@ud@{0=d5JMErUN*Zo_ySH)*PTP}e<(x-rYpvP~Y!&Fg64r5%
zhqT$5li!tHkBy_aq7Vw)?2kKwC>cF5hk&qAqElGBB;2P4bOb(fmd5#+#EWuv`YI3Y
z;8;Xr53<tF*dcpcG5<6=)2U8O?oIb#QgL*6U+;~E$&x?Dr>McN&SG~{bN_3vQ(`mq
zDDM8fxF$@g5Bvod2(hrdp0aD?l5H@qx1UuS4H!Z=2-6wjwn|wON$JUl%7o9x*SO*?
z_33n<{1M0pJS>*>VP1k+zS)A&<eCKenGeh3p=BzJOF0ru3=_tB!uinbpV#TlN@K>!
zvV*z$-AsO&d#82C2U52;S#Fz6%wLHIG^^p#=PxDK$1|iSYE>`^tYnrZ-|SS&Fh$H^
zJKX*Qce`Vtb(jy@S9LEy@j#Doffw-P-R;)wflcIx_HWerD_oak`g{(z0weX9P;u7&
z1h*A&CE@3z$0@fuKcU@TAdn9F*2DkdSR%a>&A0wL#l_OF`CN6X<CiDT3_P7rI>H5Y
zY}XlG^$SzXy*D6TV*P$7_he|^p0D(uiBF1MSvvI6l3Jb>%_bhJf9Y5>Ma76n=t#89
zdFo0fsOBKg-A_wHqkoHvL%C{Y%Rho|$F`>0O9y}UJ|Aqc;`dZ%>=+y6-0p|I3fB9S
z3QjIhE|F23FX>OV%+rr4(n2+3Es7%SK1dSWRW=-U)yj`{d7BP*kvtVViAJqCBoc^|
zQyRCOSjm*;UF`-2+kAYr?;UT)#f_^9jmCqAG>kfr9@2g@+B22{C|y1AcU`d&9@5-b
zH>deFTwB-K7?JFq#{QblV4n+b?Y&ea)>`KE+gK(!;6TY*NYV@dq{rwMvFx1u7>t45
z-2kP7tLM0x_`F;!u>(GPyAQ|L*|VdZTJ@o9m_FUg&46P+-}O?TAU_3?&8II>O|@-a
z%)94)8-ZZ#@ski&6ou6$F-;o6g(BWpdEGf?XoxhSps8?9!riN95tPF5Fy20TZ5~Rv
z!f1USVCPa>XK6feMKG6hx61)zjyk}>u?=mnt+Pf$OJ;jtATsaAQtY0()H<xXW?IPH
z(9K8)9y!Q&P<AZV9=m+}L^woe@RUh$OSV1CAR2sk03_h`OWy1u)=HI$0@#=k?a(=Y
zR}AK03Xp1fj?UYuCYH;F^ZnDV4U7DMFSCA~oglEFZ{Rhbw!Mu}MHb)jR(2~G5IPE`
z)%GApKi@TSeAOkjPBJp-TG+nK3}?tN%gu>4oCz>^Vwq=+UpOub-|(AMu?-D@;$~0J
z^@(!X0Kw`;N1_&{-#a1nC}KTRg}?MU3FIz2PgHxdzNSX@e>eY!2DS-)fJ-@wo#c@T
zyG$A4PDwzRds=!L9M;b_P_n1teCO31qmD>FZ+De^RmkD}J8Z9Zq-p%)$d=c-5(Fw?
z(ZWVkvF0fZBgzn*YaebvzhZZgvuS&i>~xSYr)!3K&|gaTbtjnqKSO8X*97<f?XOCA
zHz?iRF_iA^?$J3$k4Ne5jw#)p1GZbbMt4o=9L-SB-|O*@IOh}RdS8cgfx(kI=%=8H
zD6^Qbffm$yOObUhji$$E+>*L#r~6uHPcUyK0+8HVGz?Z{nex_q4>~9U=);IJ^AgJF
z7yMVaKIKbh(&b6=UF2qLeAL}r{QGv?xo+@iwIurToxbz`swMuyyXUHeLqvx!E8j^W
zFzEvc?4$PUERPqjy-A2WTyR%Cx8$$5Cxn-~P*r{1!6#>hB8#@1YKh0*N97<Fut;!1
z$Q^xG%a%N+0@>i<-W(=uw@v{%pn-RNfAq4}HAjlSedi03rtzhz);q0JU%QF4<nWul
zF_I!WY5gIC$E<H>2t22xeALa0=UpuJ*6Qvb=q;D`MTz>AW8U6EZA=97w5kBai*V`i
zQ>*Tfx)QOH<O$6pm<v~6OPDaz_79uYjF~=P^wyEcYtBKErwV7^5UMvbyCI$IT0#@}
zk5HMX3?5r=&K1k^$^mxcGsfEc4&sK&GS8RhJ&H&3OH6BN9At<0gxdSCM=<ktN~_RM
za_Kr*==hP}@Q~Lwv)eP<b#0xA)B0JV_9|M$VaggzyFhCvmQ+E<ptHE1!FAeFU>7;n
zYf}KN_W7rve2y0>yaB=WmM`)u8SKOz3(`1yfmSAVnjoet<P*vI^8jP7=q1fv%=$y!
z@bbxDn=SrUU;S?m<Zqtxd|hTV=VNIMJ0)wu9#KXZWX}4GyR7BNthV>}R)NX;1<l6k
zVW94jy7vXT<U(pOSmJoyTF;7?GaU`kH0Hm$yK$VF)NuY8RJX$JU`;!v8gT6%W{;zq
zK;1;TLw1RF2B#Ma4QF-FX;pS}GPvyQ?5jvBJZR4Ynt(U>t^S#Pyd-9C=s)W|l6_D4
zh?mQ;g2RtKE*;<`K(Mo_VM#n%v<(_`ILk-MNruS#qzlvJVj7<W?klHMcD5+ESATyh
zAH%JrMmT@VrelE9;Hxva+I{EF`w_T4x02x(f<YcG5jRczrkY8{-_yY-E)dh>5eJHM
z!}}{1l1-RA#r0k4tg0+?I(!K~2+sSB`DeH)Hah6UDl=auD=<8j33A=UP<@`GTP&9b
zb$x9#a%OHC#zsOY<M_ufyOZ<|3&|pj*iwmCXW1SkK%Qs0;+CeSdfgLxNGB#)ZDI(o
zeU5WYS;Il^hnlFeiAibU*4SZ}jELTT+M@XVWmqUKRAN+8CPBLZUpZIAG#*gedazQh
z;yR(HLb?NZbN!!w#zJ^V#i5jfDq4E%N7&zXumkS#@V6;JO5MRw2tub5J#1j-j#V-w
zyWG2xt$t-PZ(BfS5?Vg~JJ{YK<GTN5W32m%ChG3D`l4RK#1KC|DJT+V&RrTGtHT8V
z@Ms{!1*`;Y0Jhw1A<r3^B>eB?oTO1Km#Bje2(s!h^XEj34e{tTx0UyiAu$#0bq-b+
z<9N9=MRZSWf^XsU%WsVqn$vjY3Enc%iF+r4#?PhGZ{5dS?5z@Bj%nASOLi(PxMqer
zm3LxyjaNP|bwO$JD8aS+i(d8yz|s_Z)6z%|Io4BD@%a1?Isc@sA?GpWHU7+9cUKc_
zSb*B`u5dUyEINV8r~^_VT*$v-NuS*x_B|4$5F9IW(0U_|i=F7jBQJCcM%M8YaRyy)
z9OV&PFowHi=2j}H>TiUhSPUZH@nz`j%5)dzzT3<2>f#(+7ghxv-3Yw7kS}(5?F$tL
zrawFoyfiW80-izJun*x*vRFwGq+TrFy+%61{g{hV21<fc#^-9DR-N(jV7d(^;{NYM
zDu!$hreP<43h=*b#J*98daak4*z#xmer&lm)s^TGR}d8}#S1!YDr!<X760;|9KnnL
zy7GZ+)VvgL2W;&P$|z4_d*`VL_9^=rrM_0zQ=S*8F2n+YDhuID&g1cEDDk3*PU%X&
zqeDER6GdBRAcZ|;Prs3OcPm#NT_{}kc4c-1<8ttHYz(3_I;$<*@5e{b7fCWD6HAMl
z@BQnl(w2hX1M0$^m&3)GCkJ$dS=cVA>S7d1h3O<r;ezt(FLx5Z(vlOppPRCRbxh<S
z$8kC$wqNz8*Zf%r5>`-K#ihh8;*i*HqZ;M2*%m$ZnrsZ&86dF_sJx;ZnE|}WWLj?p
zw~t|e939?sHhF!jd?N&m*D!Dod0%C=)ivB+0MODN_q5QmmoKg2hSHG#CQFZnV&n(h
zu=SnN$&Do;8SBOe_C8orKBfy|WHUiwkzceMu42CGr90oU(J$AH!@|G95gXP!n|f=7
z0jdc4J=<N<>FG<8)ZGe$&Ol|GQx4%J{i@j=p~28_c0LrIZ<qx7QSCS8)T@49D1`CK
zMtZv-8_bqDL;dYbOmoUIz|!}h!nL|v?weY1BNNfu<BXP1zy0L?&S;m=p5u5eGp!3_
zO;n$3lGfp-dl}n3){ECwE#1C<_}lWI#V%ThZd$Q?2hZQa<PHV;;SKMd83t=d>I<Z}
zbzij@5HHzHbvmCb!)`pji%$#dNJK`cTPwZAhpGS<pVzzUtbn1i6%9DvQ2sC&$Z51K
zEh`^dT*q`TE@{QuMi(o&LFA?y+<DFj$4wH)yH?$J@7Bofbvv!r$%mSf=9`M>?YM(9
zTU*R4Q7a^qPj-Yu!;Sl`8{Jzeyv+;pJjBsGX~%kpFsz~n+8%!SVN=62E#vcafc)Bc
z2^Dc+o$tU5AovMOUGA!uEnoO8Vy;JIp4pz&BYH4kCK4jv<4Lp^;xAp9<Ab@?DtG<}
zws3wsw~0I3`d|McjQGAhXmWX_kJsMt)^0mb4&!=wj&?y&SxfNgs;wLUx)lK!od!&i
zw3gn__NBjOfaYX@BuACw>|D!afN*fBKS9%{j+)pZLnKyxl#ses2Bx3s`b-9C&vyQj
z0`R2q77t1gaG}eo59RJ1-}gJ8BEMTB)G6s96m~FXPJS23v_N2uH>xKwf<MQ_Fv^s`
zQu5FKNfQ%2RRAnQI|NkFSp-I$vAb6xul6*PXsFs$rgiC(D_nKcK|=Hao3FBSX?3Hk
zunqSfgG3X;oeS|MHf(|DmDT>iiHxk8G)W_;UTM91Escu2pi)7sV!XSPw^4f^?(qDX
ze%|Tv3`Q{lE}?)nA~VMphEV5?BO1wLGIOJ9k+j1FRUW@KkssQTtKkVeC=F@rp<ks+
zJA+iu3W5SMp|z07jZ9wtaVfU0p+7-(5gG-qf}B02;!0KYW7=&KYWQE78bJb6in_17
zr8nHqI<k;RQo+cA%<zbi1x_=Ze4zsi<d0nC$H=||mBCVt)aS_9=L9m<LAbB23q?dA
z#?CLO6zv{Zt!9B0FRW!^+<k}lY4ku?kMAAsm!13G1AiO0j{s)_$8D&e&k115fz=?=
zTn};G-6^T!nz~6(mVtVFu3ecFUA|(vF>3yO*m@b89@$#d6w0_$R^nJDOqoh}&|V@G
z6-BibbLK4{9IW`X`%!}wr(G*{#W?h;P2j}5qwV|2Pg4hQ4r8|x1j_h%zrj@N7?0z=
zHTJBVvJF&eMnTzoIj|n&-SPE()edacz!g=?TJxJkQrkO99jjufWkPC#zoFHItezoc
zV!J(p|3;wKSA3{GH4u_1-#8F+M_<O~*Cx5xxXAOHY!Z9ZZpulaT<Al|SA|p9g;zAM
zgMA8O!8dh+YMnuOkkKrBwVBV{3OBT8A9_`V_HYVoPxoN&{Syyn&U{BVBM4PcLH65m
zc_r$dA`m_@+2Ss0w(FFMY1E~Xt^|J44Y-LMj_ACrp-=3SRqKa%7(Me8GjSQfY<n{6
ztfa}s{>H9|r9krQj%Tv>L{;?l+|t}O@NLMvV@}WA$gHUTw3#?<AM?ZR-lZ99JW8RJ
z<b@SPARCIhv<0P=xVX<o736ne&2hR<dDo(PX1o9OurLRWNOGi(;hu+*&4-4*t}gU`
z3wkA={~diYyHfmDDm*B|__3&PYdM9XUHX%SEJv`1;J-kEU|>Q`Rb@5Bx0X4-9~K_V
zFAqFif#Zk)8gcl40mv$;>-^(jy_~Bjb-y8@;D<E})0cWr)v3*1=nlP~+le!`JtLZ7
z6ePmnv-y94Z?4ohoNYxMM^WvM52&MbUMv>j&a3_xiXZTxosg&Q@L{}4_PT<EUu*c2
zn;|F=!G;j&RDzn<+a4nL1zfcO7I&(a*7c>juyKEOpM>*$q2`hU)$_8~*@HUgjSXld
zo!KC4ZBi0So7T<5CRDD>;*~5rOW`k<E+b9y%=Rdsh(9hB*{l#ZmC)ZnUsjV;z#WS|
zlK8c+u@b=BJA2qP^eUNfDl#Hp>1G{0&UPN}Uz|GzWK|2i?IXHjNx!{SZ&Q&Uk>9TT
zrTD8E^mc-s0CGl4hd-GKU@V&7v+i|Dr|p-|S8(04UyN$jFq6peh`@yJ$bSJe2K0wv
z-$#CK2@+_2WVBF5XoPmpN_?_H^2QwOWLT}_#mPkca^0BT-oF+%aQ;XK62d09Rm4)v
zPqyldUPHwGV;w(%==paRcBVg4#YozYg1(9Bq<4+)@%PG2GD`Lt1_s(OIpxbjZR2qm
zt+T4GL`?1;NXSst<Z>uq3(L<PM#Pv`Fm<91##Lsv-I`f;Qg(6mhZnLU)+FlHP^M19
zN;im;l46%HiONUDpyhygNBda+Z$$43mI~1O5Ta6-!K+FBt)-rDYnodfu6NhtE*_UC
zEmgNbuo`Mse2mj$pM1p5Ig17~xiro4WT{+#KP(*aAN;{7aZjE`a4qX!Y7;|G(8p-0
zd_m41Plx-ye(U&rJlv#K=^7vkcb<^1dt1pK=%6C>KcQX5h{jX8EP(PI^DzLVJ$GIr
z5;TCxjC8UMO03SEw0|$=IP2_@HaMOsuPQJNyJ}r8J)eX*cxQV}vQxDcI*an=KjIFn
z=+q?5Hw&>2jtNvwx-UWXRH=*J`+e=sGS-=EekD%wb55N;v}!nc7bZ{@bn&?6+ASG6
z7JID3Od)}3VW^r6?0%Lvs5M*`hCoI387*g&_{A-gl^WSCcdIxxBHt@u;MiVzB1!zc
z=Qi-_jn+US^0GmLW^$}|yZbECurJ2fsR6nt^oN_B1*i78UF#7Tf`V82MQ8R+Gj^Ga
z>eL{{E6W$mmtArp^IN9FCKhIy;MqYv6Qt}wcOEN2!;+m0xEUMEW0Xfqw4q*r_u`8s
zP#}rYBCON!N~uqRyEsMfx+#hd1Mcgci)d@}{5?<KK?EQekzpPLFLc-5scaL)Zsu77
z@~dr%L2|8rHeJnka$3U(6&dZhQXu2t4P{qQ(B~)p<8vVodqAXy!a~S%*y~vpM<R@C
zqC{zce={9}Kb2q~<Sq2<mt>>TumzpP&+$#HCH&x{?_dyeu4iPy9Q=S}VVgF8BrBQ0
zA<}n{&UXH&A}=yD+!Be(Re!eXX^~DFm8J)X*jYV)xTw8qS<85iEjt(!2Qg`*CsTyk
z8P`O`2hv17JbpBias+~Z6tpyth%&F$=i%Vukfma=dL3?8fGGj{9brM16ofEhZBr(j
z3fp-|%C2ZxnY5@z&u=+@+k-4c)9YF{WlF4LnvRegpLQ+T)Hi}a`Jw<tudiCvY(+W~
zz&c#A*^8?ZaV$AAY3bF0NK>LQMk@&r;U1vE3_m%buD0qqb*iRt(cY3=%?L<*dtM9`
zUzI2|k<NwhK2hup2^{GvsiWfyp|S$Xfr1lv74Eh3`Mj{~Hr_TNLGAVPn8!>p@09Ao
zUYq7g$Ru}d_7F-!@U`=Rv9G7eRZ3FN=R=^%ibMFL2{JX9<=$yee{KL&r%>U6`7c;S
zTT<3c?vB%UphApcV<DyQ@WaTty3g-NwcY%5QZy{iy*%c{DXiN!OGvK&-=N^!wN=tR
zm`G6ho}XQ@`L3VG7BV;tKb&iH2sNoWRq!CXIDbP2NkAuNMn*U|h-pU96@>x{rP&|i
zaz$Atwtf|IhS~L%oUw3r49{tc+)Kh0L;hazj0N%yk&?${%sK}KW{r>lvm1Z9S{2u2
zjMo^Lf2|y<=@`*7rsQ_B>Pi^-bhiGd19Rg`d~*5ri5@Q0B{^@$UgR4CY#x6PkS!z;
zYzT3!TaUoUzOzHjlfB9$^MfmNX>Q`p7TaWAJkQdZ5>*RnJ1#NkHcd~la^wJ=RUV4r
zgw=fdOPM!|RBC*)qRNCsk79w6!O8|xEY81kbAzi{yy_P<W|um*po3Opxr<Nn7aU1s
zDOu_|BabMn7U#;z8$Zq;_-f9~yrtyH6z>dA)}07mN>V)7qdcSlNh|eNp_G4{bn8ff
zZd{-Ir`PS#(5S4A{;kZAq_QPe3p4+K70JWd*-5KFH+Lx=W$)6hz;5=)D-+RR4EsdQ
zwcQrunAYed@fD%r6JKyEg=!Lib0_g?V9DVpkOTWge6?Xa@DD<<3~BW~E>@lOusSlq
z@s3+9*kql07}5tGfO2FiFh{S8Oagsa2O`}m3w26N*f>LvdjI-+iR<#6s?9$$ADXv_
z&6P|i_NbG0!<}iORG<GXh<!HJx!&W-;|PwJ6l)faqtKOV!!1S*LYg5&eqA`%Xh83`
z>s}cRu=Xb0Z8~4HSvv^F<?!>RNJi02N0Nz+w*e5Rgu`nf_@D56wFfbBlFsxM#q!Cw
z7@+Ng^drC46#<EC!Z*iK;pWnB^R0WaUS^xVSsf;5q&+w=2JamXuCspIYQxn~{1LBx
zZxZl0&qH5)GK)D4t-2h^d3E~1uZ!Fa4X(V?<zVLD(HuiK*-r(Vxs2JX$T_f{r-l%w
zUf(CX#FDl&j#r^>=e%VBFHa8^D-q(3Haq_9O+|U&@=O%c_gRs~HyWP+0Q^T~k0~g5
z598!*L!Unu573~x$CH6SPZyj9f|0AZEOV}?P?wsNqIi*X=G~cH1CgoIlmQ1AHur4z
zhw#xA53SElGe;5r@45OS%h>i&DQiRxU~kinJ!JSFq>QQ}Q?HNGUoU#|+zqUz(Lqtc
zkDmUPykb1(rWM_>^Bci*Y-gQUb0wNZXL9EnAQD9|p!PUa?z?)h|7N27r7`=!A^WzQ
zcdKOArs!KWDpgr+e&;zCn#)l3o*w&aeRcWxmWy#laoQ@SLH4a;K3|PB<bgc)iIJQ6
zR9{Ul3vAbNa6<4@x$~!1#MD&9?P$%Qb5ca>S%OF>>$?cWCJlG~T4rPR%-TRJl5`h7
zMx0{=_6H+(VuGErkZTGqr7S8J)z#4nzXy8Sq8?GjP)n!^%zuaj=8J&iVV(tkc7+Jx
zM%(9oB-dWdhAR4C6VRe|zEz9-YE?%zn90tx2W<^JLzoxS!$}6`!l97V^zK)#4I2;4
zhCGd(US<I^?(Dm*uB>p6<!6FY53>-wL!4>80mh0IJ1DUHRAiFst3a}*HYG$6zUq<i
zU+HeD0v)CT<W=8oFmPEL<cXWijF=?#B((WL-fq`~svE?1A{Yz)L8o>$B2+KrpXuzC
z|NS?i!nrx{U`%KJfQm}#N>qQKjzqv)o_($G^YzmeUv*SjtKkjnIDEFw5&46NFjwn8
zRTD-6T-uVsj{QZK{JQI{12DGUD942S9K%F;mk9F|i4CMw_5?vr=}yN?j8(w0SIR<N
z)P7E*p*w4%w}MD*C?8!&YQ|Y?wX~uy?=On`z0daOmZZ)(_KQ;ey>GlydDGnJ*ag+d
z)O8rAQ8<X0d7*ga{%qx4Ci3Q6vmpO)wH2xHfv|OCbSxCo3lFKVY$S{AEkA5XM4r>n
zyvliQYi6mF-4T6yl{%n>n4LEsiQW@A{`AWerr{3nYM@iOm<kjys<4a^6}%LpjS_^?
zQ8F=2dp*Fl13UThB*GN8BMZ0A;V7@^q2V1-6|AlQ1Qos)_4@tS6C0zKA}gCtso+cp
zp=np0gEOP<g|S65uD<IRS-n`4(vUen)zXK#SSJqsUpM_inl?kYAD~{LSJ?yc44mIF
z%9Vm7-BqpdvVU+R%yQMPReo-lj~I>Od}#P;#cyde*(}P&>$yPY0G@qX>;)DmL{C+W
z#))N?*)8(kw8*%VH&VyNkhC(kUZ?A9q+t4>ZA+k<HBZ35U`9L!Xf#TXQAZ{BVnrh^
zxuh*Ic7*H7hXYT)Q*P?M%FesysKvUH{IfNOdL#>#R^iD7f?UfrRWPrP-f=zu5m1Ut
z=+iHi?UYPkkC*)HTQ>m<^hX-*XPd-k#NyOL7{XH~ob1;0l}iGi?7ydt>yR-3q2G;P
zCl4SqieCt2Oi166PQ_wcWEA|}RJ?sat-N}c4)y$~LPK9{c@cVQnSidRy@wG?K4{lF
z?Xt@g4>jDrV8lEn^oEENEnJDFe9P(kL`(5)?;!<@Tq;JG{}EV$ZXp;<3Oo8hkTOS)
zr$)lKE~DfPzMQf5pe1%`|Ey1seZZmK>v{C}-eoU72}%E~jKv3`=b{mvr;73$l0d(H
zH(0r~>>a!fs4<ofbAAnpth}E@6zf|C9@0T{_L(|euU^4VLyU%7MPdjqK9j<gQ=V!2
z{a3r$2a#X5<#ysA){+;WFh;GfeSJS+C;1T3;aQ^EUwQrTe+g0%{$e}NV<(Mmyq#8f
zG1Z{zNg9Fe3O$qIw-87+2~*4%?{RXiHM-ZdnwyY@E=qL3KXv*FPOU13ozXE_2<-mz
z5QHl=dNE|CTbmh1{ao%k-BdPTr42c0bjezlI+503CoFV{(X)?O%u-PEO${d3-akE6
z08A_tUmGyBXCKImO1AFpgCFfnV50F~8Pv&r_R_oIRY8g8)XK`@WjvyD?4^4%)rsK)
zW#0nh)RDyej(N6()N||ihu%XjXw%KPrci98Mv%KIP2>lUa;_!52$nt92)^~ld$%yZ
ziNT`5@Fe34e5vEJzth%*MV>SY8y&D?gg9<`=_RUsiSy@@XU`kDCepDUDPAHwU9#UR
zk^Fx9aRYeO;!ypF|J&CmN^e$KKf3(f9B~L`7f|7&i7%Y~lo`T$P2qVZKNs1-<GXr1
ztJDVm`X0(dIL<%VUAs=gma$ieSlIzJca^sb_OdSjdzLZ)@Kc8NXOiVhHNoBW6>TUY
zyIiA^sb&$8x%eXd<v7K=#SB_#s2=WSa7gW&@J;@nEQ%F@GA={)%j0e{1R>6<s%}z#
zA+J#iU(!wyVCOn(HT_}wZ0$)Xf^jK(%X&A-{^>SDX~KgWc%Ye$m#Bbv?&3A*V?b^J
zZ|o})paWuXI@2sE*U7S&0SDqy_y*SEjDqL4SY^Ngh(q@98x-xapRl6YFnX5<yy;vC
zxDQ1xA4syvF@}e(4dRSb9=(xY|HCJlon7z?jPbFr)oFkaOM3Uu6ql)7vD`hCTY%!B
zI_yV7@y9wckXCu*worxbLzA@SfZ2{Amd4uy1MyC#VQa}>Q?za*gYlEow}mV=IZfd;
zCMIaw!Z>r3SofI|k&e>Nq@1Y?>b==o{6@z^lwISQ4WOs^O9nVNsX^U>?!mycZX`#A
z#N)9j?!gNBOe1k&`$aX~F92d2>J|ggs~<E`=#qPB!iAnpvOfy!7Y;^1jywU|BZ_u4
zDONdjkBX<xLxHQ9IMaJ9u|Maz%nyL?e^KlpA*K|brO#Ek(094DEgCKZAa9zti7G{!
zB0(y)9|IS_*|0B>DiOetzI;E?t3#?>io;38VO#;Pab|R&=I;oi(i3?y^|J8ZGTZVl
zW^!qhcrQ<KxKMMaCp#AoMNWYEWiL|HAFL;|RmEbUz~SyJFbF%~(0w9~%u1b`8pf2*
zueP+y*eos#ZujZg1PTODy;pk<DE>J%6YOje%2HuU*Zk@?@sAXaf*;Dcb9HrH-@j&U
zj!C=sKeQh2FZ%1;Z;M7c^ofKhv%;1>I!7Ob1S_JNm<je~lw|k0g4)~4Vz!z*zkT4%
zI9uz=hkYvRmOPy|&Mr?*g7BljX>-+wg{=R3lkilao?Q<71tq>VkviUEGfH>747n7P
zwu4IC2>_Cv62)uIJGsL{yuIlVb*H@}eYxTt4ONB_qfIHYliRw5T=aR2Mr{8Wc#_gM
z*UU}_nJs9ldA0JcqTNyaJ;I_ODSz5}RutSrcBuQ_<KRh6%&SjyOzL~MX~JK;yO%8e
zUEhwKOl#jk?|RFxB>#8lqj0z&1rGa{+zmK26O$e2V-9HSrE31-NvEvoQ7zk2F{6oc
z&@nHS8fMyNgV~ryc=dV?QwMMkiYvUl>D3ix6)HC99Tg4v3BB4=`S*d8FV&{VPYn0W
zfyPC%6El~LbaOnhh$kKgeM_j%kZ8l%H04GlZ*I4eB_jMlII&%Tar|$><M7U8eD#k`
zRt9%GwFv&&kGOzr^w0Z;s*r|N_ideQl%<-iJLgr@ljQqapk=bXOXhoje2P&xxdz>^
zz5a%yyjAw2(VEg)GJgA?U=(wpFstTSL9=Bkv2#P^m4yvuVbcD5`*zfsglY|F>3175
z_ev-yo7eHWMX?I1Gj<EdNhXw5+l}rn%luRM+Ob(v&Z!=x<3aUv`6}$x>|rOVk6YKh
zumAi){(I;77djKhNS{l3oMnn|`MUgOU%}OgUKJNtsjh5#r^81X2;M$5p`H1h$$GIq
z){lm{t>a?takOoOe+$U=_UG=pbn_ncN8K1=e78%}>}|AQL3Q9yXOlDLquLy{XL|OW
zz+;pE8ayDp=qOKqqi<7pqRelEU0}rU%SEMgv8>BJxG^s0)BL?&CeweqFHNmvIz)A;
z#6?j%(|ZQPR8O(Rx@%Y)rr<?qG79}}+?1!(j8AYAvVS@Gm6_j1^yi`OenYqh2u+by
zHtanxD{OCQr>gyQ9gSSJndabY=H|7<(cB9WU5y>mPZ1r$>5?u}+>N%5FH%-iI1V>o
z(6$()9InnDdzDo1j^-b+WleXm`1ga5TFU4agW`Vt3Gg(+p0{`?FQU32T_lKp`fkI7
z^}3@)I&=PvHqwrm%iqlYJX_#9|8O?PjsZKYW0jum+LN}UgFMHoQ)PYt+l7F;{()Zj
ztAVynjU}&bN&Je)IJ|3}IQsnhhe>m$B8SEW2LowxV)%0iAt?Rjb&zxikqIU~hlYsg
z#d~T55?%eXd$|#}{Hxd%*>B}cGAcfwY8GuOh&dW8y4&B`E}CdFHp$#QStHwwM>i6`
z?5&}p_GOEvmh#4=4nAhs{EleZ;uxKEs)^$d6F?~v=g8sKRiXP|si(!OzSZ+yGy2)r
zK?`&i7B0g$WOGc)er1dLzCD2}XCK1tL>1H?SbD6e&xa8R&axK@eYRg<YiI*JlOX(J
z1oDNWjKrXt0f5nuWlYI@M^eL){xv9$)U3v9M~t)0h1pVh>~M$#_cDPaDyduu-#FY0
z6u~Xm%o4jNO9=WOrg_udJzxdk^#4c}yvgn=P5DxI5k&hMft!Cz*X0zsMa;gGEdaKs
zp3E##Ext9s^gsUgJOu`xd7UK)OuD8&wSgAE$CfH%K`}p*6geoDAi1*p>!xD1ucRXN
zO%gZdL-4MDW_3pLAD#Q9hwFEmVaNYP?h!`N%bG^&sG{Qc?;bu15BLgzAs-Gv+w#nF
zIm&^*<&W{7y>Fun<8dVNs?DUfnvS<bebYKcE8<)U=I{AL6XD_NV#bW_PCs`aqEFdd
z*4X<_3_P|Ffy|7C8~PCrqL->6LPVIrmE%4Om}Df_D_>zS;yTi!iSTo(_$*gShHxh=
z?c1N*RM}}DH0E!)$%q_Ghw^DD>F2?s{l@2FnhTSq(dyscDt-c1Hp!r33g&yvJDg0d
zQNc{Yef6*(WS?t7@Dp=74{r+$tLx8ZT{-|QfVR%~Ue>$n_G>|y9r`RSdUV~~fxljO
zpw&rm0zL-LncRh#PU3M1)hEXKM^<tH)%}`qLW!v-cY2pxJ(~$r+XFN8FD711ZIxCA
zBwpV;nd5yDpRuajpOj5{l@~d*COT>2zzd|~?eIHsa=pE{4f<r$Vz>7N35^17l${3j
zNd$hooZS{Ij*82=w`U?l7<9m&J&>ErKa>EHuu*?HiaYr}Rm(w?`1qjN#)gRRofv0H
zjfjZ`gX{UG4fBY-_CcsaEnh#>+$L@cC8P!LjwmW085Al%)S=;Vo)W<XZ^n<UHaWI~
zJThcw*%XtQXs=-7Hm9ePJC$Su7wD8ryW^zpIw8E5d9(}pz1Qz`yURj}8jzIF7A9*U
z)(h=fHSUbg$ow|sbV1?=_5;L9Gt`ib$6qM)EXZ|p`K|!W>HlNwKdG3o*o(p|{_i6%
zHAiRnLcp{CIBT`fGhFU`!8x@vW~oomBLBM<7DIv;?il7{XoG9mT+>yQ@i@8Ebi5rw
z(mb=2g6Y9JOpo_bpImA(eYC6kVSFjRO>TY?f@0RFU!oxEOipHF%}w7l)!Hsj6z(e*
zN6Yo8F#X#I1dPbWTcbm3S~HSEYhTsY&IhKw5wpkV{Q%24T3f}3d&YV;)?!g+=<tk(
zvpu~$<%C$%Am4|DKB{ROe<^6H^t-Tn8a0cA0|i5VZ1RERCef5GLwgDTDuHqp!8pA=
zaPRcwN-cmhsW??Af@uCIj}2zQew_9~2-@>YawR3kokU-5<C_4MRZinDA%Mal9KpFv
z*^q*aJ)@=Chw3;9Gz`@1B}?}mVKY>7w#JU+QAUjKmB`wBuYgBzCAb>>M0#g#``c-~
z1nkoe9AIhW{gn`A(!RK>*)veF@r%Tj%*!YIXxTLt(4_Kc0M5JA^Px6ZX@Sf4-_wK(
zkY$|mQ|{}6=7$9$8yQvX|DknO=NYBQuz<NRmB=LUhhQrF&rKY6O$Flb**rU9f}r-9
zbFSA@e<4RhI=y)9>6FF7LoyRDy*a364NdCZzcu>?RSo}in`&OC`fE)tIFi}t_|9wJ
za^L)7=%A)~k&UM`kmr#Jhz!6B^|2Y<Sc$9G-CdynFyeOy(*m^?IE@v4;B7gs4mRxd
zu@>&@oO0jPqDj%vzEB!aVZ(?F6byyi6vCwZy7?1*s1}u3i`Bl2S@JyEZH~%32^tw!
zdYO*#4)MH$=UW?(9O{GCOVTj?!Y$kRacsQmVtuPFO#(pFhoS}F^lP&?!QI>GW>>_K
zk#d7|BlV=sc{t@xqJ`NG?Amn52G46IbgR1}vZ#j8Cg~2mmoJtkYZdq&$3}@(GSjJ~
znR}&rib%9lINrj{gak$~re-zes3<9%O$v~@QM@>OHmjV_$K84pbkmIrVV`{WDL437
znDX4F_XmEMd7bnGZPWbCNVr@=IEB2Q$wkb#F_Dlp*sc^6OW{6MMvo|@_AXW($e@C#
zcDo&QWo!ahgJR-BUC0VYmHTY0w??H(Qdh$UEnVoftn4_~rIv+@^xDDjYP#d~o~B^N
zK@aBboJkv-_zxaOU{@nX`ZIyDOuR;=U<icKB09lR7#jsXj!-Xna&FJhC(hKNC76<b
z)B3LTy8MSHeV9?HaGC$iSz%p(s)K$`y<@{6KR|scNbY{YJL-$O%^5(vM^A1G=sSPI
zRO)KgAHCU-covX8lEb@s!g%s#aAwx7ROEgdX8KwaT$z}>JAssEDZ5WVOz6#>x)J7%
zDLA=vDX`}GV9a%~io*CouqCS7A=#$EqNw1N{7eabwKmeXjLKUj&hi^#-*`Z+XiL^<
zB);qt?7|r!o%g$?_HNJdLE^T=jYzrnH#H43fsS6*dJz+tsr$Rx`8Lc;V-MIpJ@c@|
zFO4rgljJ<TT(H<z=U9&(_G$tj#IkLS`)T=(JOg<-uVXsS-vC{?`>m!r%wYQDSo|TW
z+afU!kSZM(^SUj{YOCp%5`j2BbDIl;2*dqwxv8*U`lmblm#hREBywJEH!xinu|dQc
z80gZs=p}Hr95l+F4!$%-(zJ#Ifkj;C&l|E4FP_Eu#sd{yl;-6-I5#DDc@U;>Now@g
z=}D&IURG6HAXf}!Po<b(xyS9Mxi`WLiDVk$n~>+6P*DBC;Jpj_@@|ubt+1>}m%mu9
zIqv+4GP6!XOzdy29-AGXrew%t?OIi{-vq9LHw}57Dc@f)m^^*PAx$xrme;(=L>WIE
z*a#@1A!v}fd;&#lqpY<3enL%l820NA_I9mB!mkM7{AG!6W&df+`K7I9wo3Zg0S4RN
z`bH+CwF-P||4~PFuktf;Cay;a+&0z5IN{$NPl}3%G3`B3^#L}-tjL~<G4zzyG**4f
zCsmS9Pr70jDEf=r^ftTl(0t3Z%-Z)~?ehEk47B8uFVkJ=SN!8orF70&(z|4dxL(1T
zb}~LcxDqP%`o;Zw_4o6}5yChsgoN9(huuHirrwGPxsQMQJDPo>Zko}?3)$v`&ib2O
zP8l$8{S<BE*sI0GJQN5(G=j&bFf9PJ<^VR8SyH30DSJ<T)PV6qsC&s?5#VIWd`2Wr
zi^==QEi%7Dym1TOSG(67*hW__UGu{Psu2xoIJ|qTc#W$4IVr5ck<@P8#>rPK=;f1P
zn=wS{Kuevn&&~bj!l=grTu2gICRo$tqH}#-x^>E(M;J?=TY@zb8o4Z3uGJreabt7b
znv*3PRWMwy%TxH`n7#`*9CF-f%eY}G{C;WqW$SX3<)i(^V!5?OL^m{P6Dfn7yry{%
zYeHC(Y`FmoLoDZ%a)K`{i-|HUAOh=*+68s}^(}|*Dbxa)tae+Wb$R=5lO_x#gf8qh
zND}Rdf@~U-2YPXAniMj#ufFT*zI>rS_~nJ9EFTRWbzIR1WRYR<b9=J`{^c~#?ZXt@
z*>?p3G$Ln|f7a;MEVBxg4#`EXEMu~Mi{Em0FzIeazC%)Rt<A&S?uJDxRcdNf6J0|4
zNeH?&k!SHE->bBzS;Uj{|1Gx;!=3MuyDQK$0qC_%HqbQc@zO{E(*pO1-2iOc<*LgT
z&CM=IyY4;gcVIbfh+N?N1_AYplbSaUrWE5TdC&m=#a5LWz3Fk=D}SfB(lY-&jwRl3
zXp}&j01R&`R$%|9i8i#DdzAP?ChtPTTK_X7shis(&;bn631?u&p%+s_pr(6=^yeZE
zpC!WmT5BI#d>?MS@r1Be`QhVfBVv}Hb2CO=0r@+9Ax}jwx(F)B)K2Aa$t<!gO>LSr
z#Tu{9mFD)><*wzBTyllpYSMF-*-=uq?PdOu%0DjhOBzAWbn}N4Kul2!^NOF@HHUnc
z)~?KbQNgdfbfP3%K2FVs)0~#~TpG@COcNthb}OrD`+f6PqoU5nY+af981Ep3M?%hf
z=bze*r#K*xF(EAzB7?bFK#Na70KYlszkz{YkIFH<$R@cHj)me*!PMNn<bMRj(8O2z
ztMzGI$u!L^e|02R&!((%M)MZJNB3gFv%fzu*8aS|b;^c;F0O4za)aTyhFrENz8vax
z)n<V6ED-JAYiGa$BuZi8di&DF>OVm02Uxig9CVShAP7@VziL5d_v+f0im=g)@$)gL
z+%JS!A`Sy8LLRUN%l}PeOg&}7o#ill9#zFz8DsbmX`@^P1H|Z~NbY{k*@P~WNIbb7
z%Xi>=-Rvinu200>QGR*u&@7S0GTU4-6YFxS$IlNx4IoZXSZ8R5nW!yQdK$1~=?4M8
zk9^F;pt1;%#TTGAf)C3zKUkusGR@GniF#CpV=rySdqkStPj=ic-U6)G%W->|uJo^I
z6YGnT(H>I`ufknpCcEWoJ`?McljGTIglGo4gSZ&EloVFQhm*#75pbxLbBCLSfmJ1~
zgP;8tg2|^0q2q$nsHzm?;CF3OT@E|D-6pp?jK$NCR1^4N)S*2w{@h@xzCB~BB9aj+
z`yVhBUT2#x-M1Abw$Su%>o@XkvU)|j)^Atpw2{uR=W3y9eoJL<3vU!^k{kcq>W8Uh
zQ8-~&!f`QIS{*4%{a23=@Pc%<tMfTZxZbtIKrx{7YKH2*uC8=?#%@4Vgb<^;n|J)f
z>xKD{?Waa!{VZhWU^U`KcZO>>gS<qCZuH=<2RFPW`3eO}f263_Je_>Y{cW(o(Ztha
zHo3f@2;QnvCoZ4BQPPnv@+@fAF|SRiy&x&7ELVEWM|oY6;QsHjPVgC&7CIff>9OtF
ztkBTxDU}B`q7PkfSqG*TlBr%zuga4vllD@xCdJaCxl-!C{B3b>B3#G-MhG+aL3;nn
z)!!3xiG(R)H8=p{&2L1<;)3PdEN978(A{1K>N@&duefqcl&2@Z1@Vf4>g?VmCX=%f
z5GyzrTbHpX&U#v-M_R{)PN)a7UN@v?_YdFgIT!i$4n_P)vH{80SwDApTUpATE$QnQ
z+Jw$Mi=h_9gKnQ9)-Cucu13Q%`U>e?Y79d7k0pBxq|UlAmON>}oExn#2A<0_Oul9j
zp;`*5vb0~;BoV&Z#ELR6mOB_1-d81e&ZVLc5-+9QZM~K31hC~{x9#|(_Eo>TpZV!f
z6{2qP5ouV<#MIcp;zZa>`LnA>dws@y59_JOs+QH~9GUvs0qe`1zWiBI``L=xhl13B
z_8sq*y39`9)>3O$M@zROq4882Y1CLFNB*g|;;g)iO7p|hRkpwWg0(O<#=jAo=PXkd
zWrSWB6yW+-=`Zhx2LVh0@55A`Mx?cDdcR5?Y1%$c926%wGyNudVQhO_)Bo3qDXBmY
zmg!BLFSst9DJ2#&yPJnb%Gzw(m&~O0ziSa9p6k(AD>*~i<1p_&6ZRov1?S}xjuv`a
zuI91S#KDk?$X%||&8>f}Lc@23;eLvSKK>r#Ux_RS7zY;^37fpV#QB>HiTNJ+1oz$S
zgP9*1wrzcv*U()Cqc_)@C-17mPM-G6Z+O-p%cJ;zm`&|5<yHy%66=@l8}82t<8_88
z5FETjewD2@1RKe7T{8ix^J0DQ!yl;tPAPeEYCE4lS;Aw3EHt?N6&jrz%qrI?LnyyD
zW>|M!&k`|tauE<8eWJWl9MZ^{>>8C4i`!(K)O9SjXg#s6wPi)#Hm=3lGM25%sh769
zUbl56uR|YN2m&tRGOd64haBGf=>B%T+VT|(mj-@}Bo?svKAUX@Nu2<i=1mnh6~$Oq
zk#&}i!fG~aL@y3=sQx;ag!u;WYLN;7?^h%T)AqjyapDu!+i-7uI`PRH6z+wt{+aX)
zM_Gj*$!`VK4LQ;Cm{dhiZMhO2YxD_YYYuB0*{)Z7WvuvcEmg)*<PVG)T^U$35ian`
z_2%qB8>Y0@Y3K?uE0n!RDBk!F(#<e?IQlkTUU-*uO9Twe`GR-Dj~m;v3XY@g<y4zy
zn7B3FrAa_@Tie-ik!*?~&0n$cA-1gsg5&W5^IdYa7bGt&%Ni~xDr-adIV}pm#!b(A
zU^i}>(y?o##7luJhNY?Nk1IHL$)lUQKm2$-F`qh1Q_A8Z^erg%5Bk6wH14B#7u`%2
zS(t}`BFnlI#FK%Zrgisti8ob0URSmX9t25S&_2HdtaMel2YByQN$Z8+gV*t5P+C@@
zdkOos4;`m)MnJy137C77lN8>X?k;OGd91Bq&Y9GvrK|t_pmlo$gsoKGI)8q-E9Lcu
z6|!#h6WDV?tZW*tr<!PyG_kUrPDN_&4~~=d@en=iz%eFwA*!UW8h0V`=Rwu;?2PU4
z=GER0F(gV_VYzty4+#Za&bGNp;qLVa^OYz$KQ!^ct`t!dM0g!GZ>VPTk20V`dyr^>
zg1_vVq>}oKJ~y>yq=Wvwea)JsmSqy+M||D|qEV`tB3=yHm+`vu9!KN6hRI#bsoDNQ
zi0r_)pYfG24tn)Ie#U!xa%=Su{IYMQs{*oxDF(8+&95PbZH~Y2A5h#&M*J4kfAR@d
zB!56~E=xt?<OWOu*k(Ap7#uZy+&s&~wjPR8I$yyr$vj(?d|Uf0YYTxGP7n|+fgDX|
zzg5jnT%TuZZ<<V+v-9QlU0X^zR3sC!=b3(2IO`MsvyagH5Ub^+Aw-xVawD?0nq<NU
zhcN<@;rp4gI=t6s0j#jUa#Hr`lsHlP>&{%Br%Kq1E}oR}NoHN0;IBpF8k~HQhqBWA
zQV<D6`{5xETy6Wbh3foMAV(eRMfavMa{_@IChP>gpO4)_E<}e-R&fRmm`AiKyokyr
zecifoAeUIX7_C0>T!@s~bZFsTfh5-To%<2*wCWV~vAz#nnXD@yISv=HpA4EJb?hIO
z5}Bk69zA<NBif{g{qsU)(wvCJ1qe>KFEyC)PH@~N90q<@5zfH&LtPj{uA8Mw)Zd4h
z;ZK(4R_|^cWHKg7BI@<Beld4hq+lPV30`N)SPwv&#yh>GT6;wu)0R`|oP#*5&3}Ij
zYSc@Nxhl@_M8%2p1zZrEm>5|2-eh>TwtP4oYz^IQ+NlqopTFKly3iF<Ok6vH*46WE
zip|4*!6BZ7Rz9VIES#G*D2wP7Stnl$8kjwi!t>^+EKTt$H4-09>v*cs8`>D*Z1siG
z+dtv-`>H~#a3q()1+g28Dtr57MJRJk1f4b8wVo(#Rd=(Ct*63Hc1%0tN~oaL!9~J+
zmbUdC>q4-d)mV%-0rqf#$|<9nGCgJ%p0ns_U1UKN9+_Q|Q}ENDbT!u4%Eui=&zN!=
z9!%?A@NRBC8vG$-g*NPFsJH0vxl5(0PHlD~@2iePsDzI}@dbFtFeoCSCuLM>aZK)`
z<TDGKE4Kgw(TSe!y_?~>jM0m(npC&B&K1W}H{)JC<A4`$A%ZS4lpr1ajzeKSD@kGL
zC8N75(pHHTMf#W2Hz7_Zh#b}vg9gmc9P)Cviv@1%Ld8T~m=ct+9@A=nLXTAiW`l4+
zuhx1<*oB;B0Q}DY>KzDo9V+5Z#Luj(5+48Vlrq}PXvtToRtG6Lw!qJ%?gump4e(#-
zx@`R~*CG+I(5Abx<Cgbp2jzNg?^HveZ!8?>F6!prxF7y>#}d&wmZqTXAd1F(H}C|y
z3}mnJLX$VbZSBfug#8UF2-=|{-oz}s=@_Bm57uOoI#t3sTFrE!n_k|)OF%IZ^(-Ft
zhmw^8JT(;JxKHfme!YxbPQe`9i=4x7(pUA3nUt-2@z^dEtBwlc>9(V${!C4DKWBNm
z&~gdWvG@Nv+OSUzQc?YAM=0Kbm~TNK_YK08vHD~HY4pVnGO_J0ABtYV@gk>vi`pTA
zwaOJiqO+T4J9+Q>F`b7AAiAA5c5v$hx1H=$Ef*N0o%4d5f1uE|ZM%ByGB3ZZ?}<L0
zaQ)BAi9(%k;WM?7WD&@LAAVQd$Vc#+UB=^(A`CuMRn*rWSIw&EVoaoWc@7}u&!6tW
z51jLG@Ll@y0zeJoLR9veLvwgon&d(8)^4gJ@i(wI^?RqYE6$zbtvgNhVABLiu7ruW
zJJadlW+U+CpG9`=dk9Vo2pZ_)F5?-e1^^p2kW8-s!`{VwFNBafB_ucT$0_rh5BlzE
zD&Fv`m*;lQT6-8JtLPPdNP^Z`Pl}qHUuhvE8R`73R;xQrk%z7|qLHir?kbFAP_<6p
zNXBb#fhz=TU|hVLJHiJ96SKuLMQ@}zN;YzRw@hSA8$$_<!C`_nbJ(nHu&TyTes#HU
z=5a^}UdCeph9Y(}(N-gXVzksfRq>E<Pja;q&SutQ=*|~}>FS8pGD06<%&dz8vJ;+#
z`K2q*qYom>$IHDL(b1Xyg0wE*O#R_KV)yO#P`S>cgNksOlh#Y42}n)3KmtT)CAga@
z<&Jn}EUP>K_)7k1<S(pCjs!|Jx+wL^VP?|SvdEd0Bx5vwDFB!FlV##yVwVyyJm_T4
z^04c|aEdP3BFx(;i`{NHqw$^I7N1a^mI^I{A=jQ|O^o#uXH|9f6zVc}M<@<4!FwG#
zv4Jsa$7pPvFmse!#)XR;+N>{xe}2{{?K+MMKUzjdXeX3@f}uJOV#bJLG_-rgT`(fw
zM5n4QPoCM~pp^X0<(YQsXehS;z70vX7w!laP&q9f$FHvFvS53BRsMe=FJ^C!lU9F@
z)B!)0^qdYD_(d3<4j&@{u(kW$T@&8N4CUWfN)%uzIvWB=0(GuQQ*B1juNR`V1HY5D
z!tEh7ENjE4$PdEg@Fu>&J<dSrHIYA^eo*2zwj#UR=(brP?HB4KeFx<_my4h0*K0+Y
zdXQ2Ed5{RKinB}RD|?8CVt$JS2MRN){w;vL4?+l?X87tcCjsB%jT5wOj`)NB>^gUS
ztSh7SPYt`p6y1ZTY;}rvJ0-Rn2)vNVcoOm2oV!XpcG;Z=E4%dv^6-6Rqj^>n9r_09
zaFSvF7Fg+ECKl8syyLAFa<WO%HUFtokUf2DYMI3<tmS<9f7VHg!}Wn1^k0(12QGtg
zI*ps6u0DsPd=EUb&=}6fr~ZO%HaA#q?YNvt_MByoRO9SV5;<3Pys=|ZF0r;ca`oK{
zdvSX}KRdD9_*qBWw-!mPZo3neI(VI>y5|D<0AV)LQbf2CG<QhEBi8`DN`bT$J}Xwf
zA&>Ljfi*S6nUN4+UdHZKbT2+|3ij`KxqnDw=08KZ&LO>4mWs-bzvOu|Zq)h?ZNpKN
zbxYSE<VDA;<i#@DTiRlgV(E9N4$g6Ld*MZB+&Kxkt9hEQu<=9!j}@hpP#N21u0Vp8
z1!I^u`&wx9_C^byyBZYn@N(^{;~orE)r4^A=TM^3GcAp;U4zvd1ux0=-})@8dTvPY
zi2Vz3&iq!6@_Zl^(gp~SwnPNq>)p;of)#}&UnZJ9Ra6f(4TXg}<&{2ksEA`BQU?Q{
z0X#d@$sfW@JyiM@#?=+<M(JQ^@+*XJe)|R5;z+#bkK00f0-mQ!-7ZbmQ3!GQ1whQk
zuQhO`a2J>rTStRhyu?<o^sH%fPAHLWH6OYvoxWx1PnAw|a0%{w=tZjfw%Rjm|KjNF
z8tiHsG~v0xD-@KOtu5c5)>t}vnjIE+<r%1avX<{Y1<`8cq;tDdGp^dw<V4P@eu$>3
zMy&<=&+*IK2c9>&*hGMydti1S_1H7s3#<VKXg0)d+WC>X+W^4>{MQ~fK9!X&{JXTG
zD4yhQ1D%<SOTrDC-lTPNDd|))f0!;dNy>e?Jxq`B#9=oqLHW*Ik`FO6Yk4A<S-?sx
z5!?#C9wR@%llH_h$>H8JpG((G{kC~hj0a`aj3=;`VTXe{@X9Vyt|>(eh+R@_kQrm%
z>QtmeUWIvmCoLOpW?o}j4_v8=lUVvseQoJuLJ{2twlhPuB+t&o%1SGThNfek1hBhT
zgWS=LVE|p7w65Z<5tfY-X7i$Qs&b-MlZWQ2M>K|l(0q(Qj>}`(fswU?Nb(6K;vSbm
ze3HydH9>v8!E9ax67A>GC=a6Zy5vPKJ}O<H`SG;2TjkJm-{C))_K;f>?kZIoWEg!d
zL+1jrHX*2`buRx|6!M0Vr*P&Es;XSv9KQ%*$d34-9ZpOvy=KP~@^Y&^8@{sKy=vaF
zSGa?XW&J>^T-cd?-WF%qd}Ct25MtOGX4I1hosN)F@Vxn2JN`EAG_dU}>sASGd||bs
z0AkHc>RUxuW3q%m^VJl-3Bz$~FvEkT-x_;55FFy$I4hFueemfk<63&H^pX(Jk3PVz
z=i#~eM4nY4<E(!!hQ*-!TK%tpOy68fA_!f-7d7|8K9NjS?-<f6@pmY&E&IVN`76PO
z{bY>^(U?aOPDDSKa?Wtwa+W@IyZcw*n+-jMn%5nd4yPZ2r(%DD-JDDM*Vc!;n{i}=
z$NuUeBz3m(*1Q=}yN@ei3e%Lx-N=>6qHuRT`JN$mEpI3OEpY;{?-V!PljkgN>dc;(
zahI>1snt>Go;Hb#n!Ua1`0X(pw&6jWJwr2yLp@Yk=BOwm8|I(!`$jbVXRY&mn3t-_
z+7J6>|90W<x#^+Mm6aWW2+22MPO|LrBIqp=1F%66^1FLp*Y@#8_S!g8tIbJ^tl%ro
zzmcGw>sdb(`TwWrYy6qs-~U4sbI-O$!qzY|Wt^4do@tYtnOPeun^ErS(4iYQ2gkjs
z*-Rt%QJaPtx#?ESGL&xCggTw$?wmvPjiWxI^Z4!e2kh~9z4v~-uj_d|ujli+w&Q$J
zb4P5`-Mb%sPCn`^-kuByUW(OiipscVUSu)+I?!}~MHDomw~0i%{7>oOZ8w{>;;dWK
z&y1fOjQCOe<!{>Mm!il{r{v)xd=Om^WI%sBGKDbPP`M;tG$kwF7Sd~DXnGA-$S)d<
zOx|JT>mM!!T~VTNKMnBrKbm`Sp{|UgoTi@iy7m;G_-e@TVoKNI^0CNAFTSQ-%yJ$-
z{QY|JRr?o=<?4*)Q<)c!rRnc(J(fz2@HmkiqxI&U)=egVY3aPqzB=24+vWCz22vo$
zAZ@hU{z1r3yMKG-Z}s@uou7>S&N%ptr2l&~a*u19ss7zEe}Cty_1zOVD<9a>{^a*x
zqX&*<*qs0-wnx0WkhvCkJc)2R>Xf&n@99&^p+69<*)<>jes*spejBs2?te33vAOkQ
z#KoDARL=}Mp^JC?+Wc1mn7#F_@R;_gB(5KEDdW!Fg?G<*iuu(z_9IfVH=wELihgnJ
zp`*ji;k;v4mg4`o==A*KVe_jm-kq=(=<r`1k*zJ9zWMGLAZz9JaJ`=UZg%;@8xKDo
zwzW<ynP5s_-S7_r$&UvQxHU&BgX1f(4iig$`$`_Pk?pqI#`=G($}-D0Uz1v+|8DKP
z%6ake!oA;TtJX5tzQ(+rc)WtA8*VlozWgyJ>z~x1Hq)%u^I1{dpT1s?tG*yPmpd0f
zci@?W@|fPULi9|})#&W5_sf|}Lq}}fiLu8H9)BRz9j@0mi?pcPq0K!D|CE=RZ?!_b
zX8ExDO{n()pkd@$^p(902avt~7414B?RpXCzMfh)UY-2O=!;bzOlzj)Pf%LlhiS#e
zk>sMYlsR+U=$AqNt4f!=ObV}f`{+NT?83TtsR4~oS7MCQo@d~S-_87TH+lRTB=~J`
z-0bGFy}uX)+<6(eoEZP%7U4Zjr?-{0KXk#6`S+dMhpby4^i@Rmt-{XrRVUAx7He6Z
zEDMM?l0Hj_oXtU9LO(bue*Nt98yex^qC?G<5QlWf^{s!O+W5g*dz|jSd2k5~?SG8>
z_jSzpx7eY9`K+dIyPtbr{`5Et{;}<6_WwErKKjT@d%IpJel~q_VD7h#yLazLT1}?~
z!?dk3*7i4lOcp+blJ?i`<$kw#8CcC4xb`=;#Lyo+jxU46O>Ar3`{V6hTlpEQ7p;CC
z?|<)teY<>eB4RY|S%U@M@;BteY6<07NB%2~nUfR0M^W}3S^4OD>EPeXkF;;8!yAN3
zaq|(AQsc~Dek0O5JbnVDKjma-Hh{c;`R?%dd)KBt07nN<U&6M4^0~Fo&;R$p!Qxf&
zm+SiCt^GXIH^%$+N%VSs^oH$Ub2+mMPO%^CIwbVkPSl%@uiGcD--_;+mc93R&rd9S
zTO7MNH+kAWaNt_{fj>!|?vDbhS|{V|8~jglsXu)?=(sfxKDUCobgp9?{`sz65@Wo6
z>b(3i;Q(wR;B&y?p!h7?!)xJ8-Lt1YA4|N*{^iK2v->jw7e4)Md8GW-U!`AHRC?g;
zsjN$#!T(iUO0sFGaY7fi5<t!~c!vIK-)fvy>gxnXjP~I-M68WtSq4*@BQEeY#c&mF
z1W~TQxTRC%jI?y#%^zwc8xT<ea$xD6suil496JzQVy~6SUD9-xa#{7d-1U%Z(O!YX
zO|p%{&f;~{-j71LGH4-uL<kcJ_d-410Sv=)(Ab<r8%JHb-B?TsB%4k`mqF0%tx^iw
zHlo7Y%Yu|080So!e<U#*UhNl&ayNiZXpsnB!6fp>@`R9JF?XScYluQN+U?FFkkX1e
zZ$=}*W$?T~l+G?cfu$pcYe-Gw?dhp`j$<pVkajl)sG2Pzy;x&)VR#P3j@r$V%Jzsb
zrGc^0O;R451N6r6%RCVRsPJ@m)$qFE{BAOlL@x6PJl+m>ypD`PxU#Xk0wp0-v~Juz
z0H?VyR!b72mH@=<o|Sn?0hn9?BL+s=cS08zw(&dZXRmpC_&iN}*0-DjRGUY9dr*PD
zJeDtVC)hjUidaRX&{5zq`h2k%V-;H7%4|2CM~F6eCZd~M3!+=<txn1)-q4?Va77EU
zKqimV<;|Q+)Qd)kg^74D7FE-c-!@3IfzA1}=tfflawIJx8l)|A*rP=RQfA8CXZDz|
zY;O0L>3IoR6$H$RhAGt~is>K*s4+F)C{Qt0T?|ptLVau$bEDFBfkSquD-%p;dHyej
z;1J?1fMt?;3kVc$U~&y%V7dY;#@Pz&p2@rf=ZQRrJFKD-MXsyh;-+vngD*(4ts#<-
zdzJt`8e?F;dZ#_YW8+dJ&(`#eoJa-Yk&Wmg3kg&{kCKQ5X+<>F&7?fl<zR0o5seD(
z6^SrxX{kvvV26z<gn35b3Z3?Da#>2GMR)1<Tq5LXPzrzE1_4n01ne{o0jtj<N-qyn
z878CyC?v)1@Dd>U0I3Cnddd(QjFNL$h1gOYZ$}ArL}dqpK`UV*6mBl8uSDn|ej6;Y
zKuAFxa^<4cs!dB_w_*L)At?LpS>v<=y_U9AoPlA@;DE2pS73oq@SlfYIj5u1sTm{U
zviOllE0^Nz02)xRz7yHTI$-#lT{%+;h)yhyfz8`ET}g8KZ-@eX?1)9k0=?{A>(@7`
zj2mawC99&mP3I%(G5RoUDgKT;40@rmW+#`pRZ~ltH=rpGMH#x*{u$GYrrCSiLQ`y9
z2O*hU(Q1=Rk)<hCi*jPoUT{^uVxt?b-S_jB2AJ!|s~~_ZFF;dj_=CqPi-_)CvcTA@
zgCbkO-ga9kz{J!eItI)v4NAUC4fk2)<uo?SF(`PV7R8V@XErKf7*g%jvV1YO8yP+=
zM;pXjt~=a3AheXStA$u9i6Pb{n6UJ1Uv%og#^ao;*k7mNMgEYUWP4BrbBo8K0QV5v
za+)AIT1W8|^`a6iTZW#ufyoFW8(WqQYEKm11}ntYw|X5Nw>64wS3>2ev_fr9M>L+2
zqAy9vnuV>{G?!}3PKY6U1ffJ44JhIlIzBTLeE~kji(PX4?MOhc6&p;bKEfW5uM8w8
zp?Pc&h_nhaf{)lB>|w-u820eYEgQ*?vn4G3rqeB=imN<zJKL%pF|odt3?I?$wR<t2
z?;h8f=#>z$Kas==aA2|b{w2h<GJ*Ao0)mTzC)EB8gQ<^%b9-}Is4a8mOb8I_ga(DH
z!?oo0z?`m`z?5ApI*UbNP@q1IYJPbhP!XL{K;|la4KaB;3L2;$rVpD-05;lL_k!&`
z0^}Zj87ZYctzM^LE4uimz2!cU(c}Ri7Q`sq!734pDQ&cqt=8h9xg{74L)xy*!eWMw
zI>kKBUNo6h*tG0>B?t$XV{BQY!_wRVMh@Dg_A<LhF13loRD>i048knyn<iSGEhTzp
z#96V3*j#F(__(RPwk+UT5^7h|^#}bpw^)2zN};V>BTQ;dxRO%9VDmt5d9fu-Y51z9
zrY?;b$_z7orNw9=LShV@<Pfu`y=m6?fT89gWDCwz*FM&Z<}xV7Xm7P~$RYq-e!YYR
z$S1C7EZAVYHj;FqWEjmr`M-iRk(47A!H9EeG@C3!cs?d?kOmw&B*N_Q;D|Xuu%pl|
znxbC87~w+EqGm#g#)gdyZIpY(*ZA^SCyf{}0xVO*!}plgoqHNW+FQvr)in$w?hIdR
zQBnd5qW^2DWYjw~xqqFk>%Gjc8KoFSHKxd7zV#GM)DmQjU3VrFHjbW7WLqJ&NKX9X
zlFQN!cGOSKcafu-j4Bvk@@2b_4D~Xp_Vx69qk4(i{HFLvyc@x_h~;MJQVLcjjF1`|
z>u9km7^r8*GX5jXf%${;5rZ;@`u7)BSJ_H0BGt%i1(*-2@Upihp@rqK6zCA;0A!EQ
z8ds(vwAUa7XG<7nR*!sA5J!8pwk}Ws(Uy*%tITpnz!gC=U|puPO$orG=K$-x{jJd_
z2PSybMjZgxlvtcM><6gC0mpKhpGJ#U^v;TGg(X7Q(JM>Q*nu8?F6)&6hhj9Vbx(sP
zWlxAz3HC8jH%@W^orgdn69D)_rg~n|(k%jY&owcBn6;uePhb^!5*6@4enW)L)Ksz8
zms(sA6pYm%%G62(iF)4Amj{o`Vy#+;qS`UsXN6wo<u-mE%w-sygJk$Re<ctN_+V6m
zp%jn@S9&RwzN|TKFOX`dD+}TaB}2>!?kVtEyx7_&KyD8oAq9C!kByl}x8#aM7*X*k
zs;E~QVp!m_Cu-aP3Z00yaEd8`SIpW&>|8&z#;NW{8eo@Die*a`JniB}WMGemKnFhf
zg=-AGu_85qs{EwS6f#Rr3dyd-M#2b;rhKhn@^i0~;-=H{X)Gzbqh-W9s*w@cAhhY<
z%>7DJD*@$=(3ZjQLg$Z}gTiIsUuhm$zK8kjWuI=va|w=1L*0d)QOYs7w6Unz%W~Sr
zR!1s-4!x||vZ9x`%gxtRFJaF@KFwZB?CC7hHQZZF0fvM*jg$taajOQE-j^pyj8X)q
zq9?OO5To=36W!ywO>JH;?^!6c(ou`PV{~GZN5iP!e{tJ!Ha5{Em+Jr8M;yxJb(T|1
z77FIcJGtTc9xC{`Wt%)-Zs;CtJMjuw@90Q5e$I}H%1M72F*YjR8Kq=<i2ykg^__WJ
zQ&IWvNs6*!ir24HOSWS)Fo?|C<f&F7(3d4>4I?suT8y6PpPWFBN-nn>hj31eMbcNC
zXTb20U&Mm8poC_&d+gbErqHG_rDl)@AC*u5gf;4nCIXX}FZ^gH0_l;}+{a=A>QoWS
z>aKOd0jm~4fE>gvhK_bGndNWU3yDPH&Y<eU<pkmzoAXajIku@rvh$y9O8G-@vk4Pw
zqQ>rPR6CZ_%hJZ4*pOsT#7-?8r5arnlzDkjmblttzt2Z3Xj+a7D|xmrVYCTUyz?s0
zN=$tiw+Mi9Z_-)R^l6dH#G=r~3JehoA+c-AOYgv7#?J5p|BEd(n~rtAfRr?s*-{yo
z2NjAhwuUmYLUtESaIjM|^#~ZGW)>OWhJ{T?j-N}>E`<;2Xehe~0)swinjXPioHyF$
zJ5?%Q4z~6|*9^04X(4v-+|a9Ic^q=Sps_qa>pupns7BE7{#L~DKKvE8f*6>h&TFUY
z4QtNeF<lAPV>nw*Q|vU%#&SifAg_rK0L731h0Cz?W2YggHV#J1Q>8da0{QSunH|yJ
zQnDiY`ep$*+D!zyMNboGSd9OkCp2A(oeC{BSY4KTCg&uW$#v(weX0ZL+nMs8e$daf
zr`^%iab*7ZUh7S+c@pKdqPm|T-5Jwz9^|0QgrPEo2slUtxxPS9k^f1+H94XA0x&2q
zCxvI;L~S7^tBc*4mC?|ry+8_?KYV!fYQ79;9#ftBn{FH|q0l|^ZH!Di+|c91exHY%
zYWOa1-_K~*(c2REWjSjT^`Cei5{N4U%n?2+&Df!{YK#1lou{#Z?}v9!?M3r%Ad#A?
z=vn6J1h%^T!p7=WmH9Zfz}tI0?>S=vx6f;(1f<ots?IvKs729nt^%B;p;9_Y<!Obs
zB!{!c0rg@5+Ji8xqXtFJ0>=t)fHNk-4LypEZY;e#I*!J|aD^oh9ksI^wx%)*r*09~
z!QPEt3|1HGe$QcCErbcT3{2nRo9107#zKrAyTX-V?otbu2jDfpbbU>A9u6Q*`A!ul
zdcyP@TljL3JtkLaTAo)x&uWqd8jGb;1yY%K#oH7nqI%=LAoJMDs}mFgD`LbRSUBS&
z35tdelF$Y<h~(v<=>2DRTV5!^7-WRd+*cc_4Rq1G<SElSIVW?3XTS;I7AKepvDm#)
z(Z#?nx(N7`kvlnMtUlVx+~gKpKI#eyd+L7s&!d%86pu@*0M#ZGKRml2LjzoM!lRo@
zLp*L<iNC=7wa=j|=^Y4~$M+W*_*`kTZG_gQ#dx^SYd!V!Sh*mp3bzOSrQqU)1x%Tm
zmoh*IAMvZrj6<xzLK$r80gLbgU$e3Q&7B(?O@H_3>KMN^FhxTN_8`c7SC~o*Ny|7r
z!m7)=2S%pP1>1IXtt(C11Ey3P-uavsPZ-tCmD7UqX#;|{1T*9_PpFiyW>UpYwdC=D
zfc*XSm2)99R-xnKlHrQWb3BE-J`FKQP7?xhkap_?<(?AhmaWJQAKRjEAGgYi$KFxm
z3ci+xdk2fRa*Nx{Gb$mf;o1-&y1tcB;aiLrE^8ucYI;jSzKWXd)lE+K;lcT-Dv{Es
z|HW-5O|hsV<T=jjjs>h_{7kapm857i2x~n~qL7tl!-jiH^sUOrZJZVMOEg)ctsx3r
zW+W>3HJvRB3MhR7qXuSFjvrsjEl+;$jD)8Zq&!&ig=Q8opF~A>OWjSSx%0c`F#;U}
zYJieJBJ%{4Af^n@5Ubq4V7Q!ga5eXqhG-rF%%&?%t~JpzRbhVWY*GVuD_3xJD6Jxp
z(YCBUSV?G3t7C4}bliCkSqV1^!G;=Gr8IV*l?$;)FQjtayG1Kj-9ncl8X%WuorsYp
zYi|HW`3gO+hfP?%3SMS}go$c~jklNzr*B7lFoJV~y>r{udWJ51jwCLkG=S5H++*a!
z!ir3KxH1bY_6xs=b;fkgYs<iLKNA&5fxPx(0t;;rr8;{ZPyiL8u|RC<qwtM7B7<x(
z7Pgi9m?o-kHTlOKc@x+dX0Z-V*7)!Mr^D$8OlW?X(&mzrb|f}U-?NTmbNjHo*pr6J
zIN#V<ob7c88L*-*LE0JR<+MPw@)z5Xs!emb)YO-L+Vca^=u-7fPQZ|a%qzvShz*Xa
zCm})ecQ$in0N?9!mJGV7a}iXJkm}-!OsRJfQY8b&4jltGw5u^1s$C16_Kkyf#wOpR
z3nPlf`eMx9L)91%O@%aRu;R$e@-5&*G-^a6VuU#ItBvzI7$(zM!8Bk=>R&|RUx0&I
z61#8&QDZ}!bkND-uC5_K_^(06!9HuhunjZ+8zieM$oe}a1xe~n9)TtM684xXYH@l@
zh$S{JuMEKL98J*{%^5d0WL&OqQu?Ivhq@PP1Hba5AQw4{uTW@jVCJsAf6vQbrl0{L
zU|T^?RwEgcKFYu$zT}W=xX(<@TU>_37>sr?!)Ke$(w${BINetQD8^!}y@A<i=HPrW
zC8U;-%Mu_NWWC;G@#I06ssOfdj+xi_%G8ZzHXx)OPL|b2#;FH%y>=$Z%K>FpVi%z<
z;d7<FqXPK!d(E)>!>ff&(#JnJO?yUyNN$fMj1-OpS2R(7Ta4u@DZ!2+fW0+Y5hi4T
z+?Wd8Rg`Iba(ncX>g&a3Wm@9v16)Jh$A3_Qom08%X89vz1?+kEvyu{f*Ie2h7%Bu<
zwHeXB9TMtAi}xyuoZ6PtV66I9f?J5G{;C^uE_0OPO9FFc;6g_#nA1WFrI)xCnI5eo
zmldbjYiL@uURKNnTQSrpCp3sfB=XyyLlNHII0L9}RCKwo3JbKE4TV~D1to+Nq6j7R
zue)%09JG|jD+u6uMh&$PEvAdlA|*|EW?Yl?U}wW{e%bov+y_1Z!%|2a^2$YP3WW?{
zZ`pgqNf_Svd+d&Ga6T1D#@IRlO2H})<gV=!v=AksSe$jVCU|0t7V7B5I@Zjob+aP5
zzD*Y1cO6>%{QJB$6T_e6uJ5sM^dM~53RmDfj*8F$*2O0W<UGFZyZ|ito;$CtG*PC&
zTt&pZJUNI6artlfbCO#|4`u~yj4L%z?t6Uk;*6%peJ(VFhzf3{B;S_Ynn}b!cHISN
z3oK!dUSWb14b2&^9eE4{7y~TmMH?NbW>h6VaWd^CcTQkx{peA%9g?VcHA9j<$*$Pa
znP_rO(%K7ohipk!Z<H8Ra?P2RA|<qzKG8+DhlQh3q@IosPMBK6`DzBRHx8=Trx~z2
zCpqPd-_&J-Rv?fg{!-tfFLrr>Nq@Ddfs4ZgC@&A2?DqUA(GAb=BIZ2uJ?cx<u_i1t
z0uupq0AuUM%-j+Im`iGu1jZ6{nJB<IpD!oUlqTze1S2+)n*I`$)0jRDK|A8Zy{Nt=
z9%Dgt+B}0m67v-nq)A0#%Tr~rGtuL+la9@8i6uV)XO(_K@fX@PSDK?rij1WwUED5e
z04K4y%bUJ_@rVYi(AHQCQNKnD7u7qP%rf1gx_5)PhMV<Hnvy+Bp^z9NnSsfo`<21A
zz5rB+m6$Do7PfqR?nWoo7WiF3HT~oNRc(sYaEJ2|;~+e31xC=o-uOcn2wi-)($_?x
zP!;O})%&iG8bUvzsg`d0RvTC+AR)C3U@N>iMqiSaD@ee(%^2HZz5s_!Q3R0?MDzy&
zDdH{dEom9Vo0>OzXWtmbj>NJpq3Q@B6dlB(Bn>8#@~orL5H%@m3sY30jL{GYE>EOz
z?rBJ6F;*vu&rGe)5=AlE%a=A+Y~5Ni8=<+1dG|Q+EWuUbI`oLy-iymC8X(MN3=g>b
zT5P3^Q7K@p?gf`YS^CR45IySa^PmgydW(7?VPSyNZ_7~PBI7o6ZbR7a(#Fe0pSoA<
z04Tt!Jw`Y4{OjO5#ru6OR!Scw<gFwZu?oh3e&k;EJSK->dG=jU2xmV%uNg5%A+$%w
zXe&^eHC)yErO{ZMIt6oNYKpZ_BN(95OW4318wWXGyf%5!*|^0fRnBV1w?Iyo&@j71
z&y3L{(J|Hu42ni|4pO``DDx)dsjNiyz|oZaWlb^FjQ=7zl(2$m%VCf!DBj0TvM~Cs
zhMT$m!HtlYiq54~_(++?nR=&-vv28hSBDtBM2yRGukf&ECg8kCNM@lUKzqy6`9os~
z*w|MFibgpe-#|U#Y<wEPtFEC}*oKOXL+0EgMTQvi2c;ZVvikb}hFX-k<vqqR>Wy{)
zEr;l%cn0A&#az%<OTClCPO;HzHcZ`nE>SOtEV6g5RL}&O1<*7utwaGF6`Qx3DO1|M
z<bRQt^ia_rzQ*Wq1Wm#Id$L+YN-4OEwWVH`8yJ$V4i}5J5MMCbQ(@3MM30mT(-k+S
zjB!;(P_TC<m_%wYhJe*DtVf_MBR7C@R)KeP8jsx`>#NKhP0(>C&_vcDrmp!)26Tu0
z`CJ#?rRGl&jLORJQT>kuYV)MwrX#+&u(=@EakJ3z!$d<YTTwH(Dwdx95e(B$9TaK4
zZUMe3B?zk%!edp-$C~A?vp&rgtNAX2lcGxRtnY=0Is(;$7XAfwRZa~eWYCE;$o_==
ztDH--M}=q2*h1iFo1JuJUZ4X3BE_I`u_)A6nU?=;{AuZy7o<vFg87V=HEk7@h-U2!
zjtpeo^XO@Cmf>=)%02LjE~BcuINY`~lov>l1=5!<^AUMZeOYewtuM7+0-$+{Q3bt3
zp*t4vU*C1*3rB1Nck;@1qH6{Izov3Rg_T&USb+(kDps3@qBJ%h&Ly4W)jTE4V@kcK
zIw-QujCqglS&`Ibj7>!rC)&IDfH{XoZ!dxhVuP3>Q2}Agz<EGR%xPwyF+5ZikT(J!
zZ7G^S1#s@!PAK*_KJvvpN+Di-hYWEDrGrCMIR^+#)W40=?54)hL?bvh%|>!kBq|MI
z|1oUjZHld9>ROV>BRx4yzrVg(Bq}o1w{?1fkOky}3LP<d3Zos)f8u0eXeZ3%l-CNG
z0sc^xr@+Ap5N~GK2Flx;;OR_*prrti9=(KC`shM02Z=O5Hj?cD+-G2nH6wRRy}#;R
zceU3Rrv+3;C+=k(twIAl@MQRJ0VXEKXpaCctDwh?9=fz7RzNgSL2;h#8m4XeRGn+K
z3&24Id^uoP*7Bw3Xa~O46syhBfFkenY;HIH=|$9cHfM~03YY?;Ss-`mPBON^k*LVM
zrD^rcabyg@)|}q<92(XAzMGgXpF5VOw^|tY2Lo+n)6Z7eW`0iJ?_-gr3TFn?Am6Io
zur@~nPNH61SKPf^1+P$I?j;m7KhL=Gvqp&sP<oQo=SibfCi)a*czn`JYqmDf6JvTy
zf%{jrNxsIxP6@H9D4}+BaC^oAw!EKWfw1><+KYA_>I!VHW7Gx(d6NOTBKvhO>b*io
zA>0ZH)<}KI6bZz8%{;MMi7-{AO^otosYn1!L3CpiKs#CO0ed9w@HZH1NA8b<00So9
zcZdD8k!J!WXI^2qwN<FF3veWgx!rSgY=4gm#B%+ioICY276=kT-4n$OWx5Q7Ot{f+
z!8xcART&}Wh37n^`={7rxm1R_yqIh&x0%_~Ato+ccsToaUz~8^E-0|<FZGhdbui6^
z#}?HlvU0lU$p(fzkhM>LksPJz)B@sx_Av$FzDo1VV3)D7GY`@Xs3&RqO_$%r&w|W|
zVdeA{;0SFlr4R+?xzULasyz`8#V&dIvP9izMi(WsAkb@*&*T4>rW&tnW54_J%B_K2
ziW0k$06PhwB?OCW!c-VeDz*xnUx-C@0rMKl^V92gqMhN|E*wO?3`9jANLmY5Ptm_$
zGg9lbm_HC}spRJlke%K|M3y&j643CWtg#gKTV@%on7sE(j|^F!AjupybxKu0z#ijz
zv4&qT0><hth4ky2V)ad#_ZS7h{32cd!MQrY82N%Y)Pr)rV;H9C{dI27Y>Ebnu|J@=
z^sn>5o*afn%iCzi9%D-h2kqu%Pm}3KC@nGcP`o(2K0@98z<Hb0W%1EIGtv6X8n)78
zH}(O+9(8%rP?_lFYh(&?3#bT=gpaCh%bC@^MvOsaK+ZvHf>P627L=x9I*A|}bX0Xt
z6r)jGOK|gbQl`O3Eu!MQF&kZJCvl<?Tx7U3Eh_M_`+B9o3l#Rv4x_sAB?eT0_jk5n
z{~~+oBQzl}oMM?eh$#b=HaQ?IY+Rl)%cLOd6kZ)pkf`1i#+tvSenpl$aa%%BljiX-
zrI}MyfR{p`y(O$tb)%FHc%9uTsEDX&izb#bw0H7(oK-^L82O7$V`lzjr;(3%)1|0n
z`(81Uvd2(V=ww!8T4DsUaXheSbK3@{$|oUQ(~@;EsBhxSQ~gC1y=lfDru}xHM}XT@
z|8}zlTf;t9>F%Qn(@nxFOhH=BUn@b~siR1g5YSLv40T?D?$p>@-4UEf5?PQE2+JYF
zjpdYQGF$^<qAA=`Ari!>88Nsw&kEw+J%N2h|DSKh)uO7xJJ9Gs5z!t3AgtKy7Izt=
zfhutAs^Iso*w`T2%lb^5odUgkgT`?AG>y+6W?Op4?niLa)bDXkVI=5NN*b>Lv55gd
z_VlWf_S#EXBCuW}F+G>UBRTXkR@qX(Pjs5*yAYyGe2_+ZPl;wwLP%LW#-^eOny9@D
z<qvHKdgD}(Pm8BbLs{5-eW7(>bx>9I69C7Oj#b52i65kBv&v{d&h1$i1jGbZGAigS
zQSr5xqC6){eu{=#o;E;xIi+ayd^nGN#Y;p&K6%`2I1k_y>lnm)hp%>)<5VCn5D)&5
zWVG8Ge{e>9M2`R=QwRG0RHaAbuwF2{EWF;OdRBk6`12F7LV3dC2j33MX?ZRN&f&#M
z&}wd~B3sf*KjYDFrJ8G|JF{nDE5F8(de_z%2PiN`>RA5>m_N{GXi(=(&<!H6h*QKV
z8GyNIOF~-K*lQH4c!)VQY9=?eAwsMh8+;GWLBfr)e5F#r=~lAs$#XIKj{>s-xKbEx
zw|4+H?3q8_&VWWjgHC(2V2r|`6mR;9{=+n_CTZJ98LT?N*Tk$0z#n$N<kcqN#aXMP
z=Gn0?n9CNDg#D^o#lu&D-J2{z4jVVdAbxJDr-I9L=?8n3)yJMjH{Ku1SN7)>eE)AO
zc`eXu=YJDtBUEjZ?3T?^oPLBklSkX2!OM*F#Y%!ufF$3j1kr%y=QPHA2FAqCjGKr@
zkoph`zKM2*pOAu?x+a!cNbIcX!jck9Za6&u%jVLCt*H|FJYb}JNo@*>p8eHXgn_N6
zs9@Y`?j>Y7!QSKjnv50FktX=5(IKtD<tceK4z`UHLvSuPE?7>~1|?Mf_^zfBZ<-LF
zu9JA}fKTe%1WaB74Aamb>K)yr+)n^KEB~uM@G{SKfeOxHmBzepS$`w6j$7q_DWI>o
z@CT_0b~FInZ)f;aoTPP(9A-)izCyr&0A&YyuNFRW^V+hGW|0!DQtl?aRKApI2ok+N
z`}7y|7`YV0iEpjgyQ)6v0=x%NY1;D<{b9*O`o$J1mQHQVC2ITz-@Nf|@4GbA#g&cl
zzl>sVjgTUzPuTqCw1SKiCGNh)AtZ|0x~5|-Vc7tw+R|ZIW&~33NIJl=yc{I(syr&W
zg&0x8<MWCpRU|CXY?8C6mk?vsE)lnlkZ^8Z80l}{95Da@2#`CMVxt0Po9fN^B0XPW
zTd7zDHi4S1-^W&tRR!T=+=HEEnz4Is;k0fyxy@)^_+%S}&*+&mrz?M9e&KEAhM6rx
z>@{sC1dH11i>g-n{`g`_;JpBz28{+PJ^$uIi*7<=${b_%@jEgWC=U&~M?uptY(ZlT
zOE`H!_%^zw&P%imMv$@t#xG%cd{yEwh8Es-iaRZ7`JE;skggWC=uyy#j0zv1ihn)U
zE=UB*jo>0IuOizQ8pyODL4I)E?rAkr#9<|<5Hq<Tq<)nvpfqji28%^Stob4($S;=Z
zHj`k1<t~8>C5<gc`Szf@ivl~0HM5pUy;^9M{&E@Ys^BW@;q`Y&Xsk<-+5ZNmWmb14
z^zaOm3<9f$L(y1O$ruK<0m5c=o>hy#C0nNHDBKdse3d49%CGYLkNFm7^#IsZ2{N(+
zezk^sxIon;dS(jQAITWKKd5k=Ay+WQ)<U5dR&<$0+ENh+(j){<ntygQS1su-PEa2y
zY)Z!NJSXp{y3ua<CfDzSZ|PsoWj)~1KpHgGT;lfYO+CW+wzY>Cr4@7K#;J4m#mJbu
z-Ss*5;!kG|uT8sspEL@gW-sdD;3JkoREw0Ry6VNq<6r+Q4vBCC0RS?eCSnE?W1WBo
zVQeQ3M&x8iFP0)RhbKQ~T~AAKY<{I|P0;OC)%61b@B1#LU*&%N`R8`>olhGl;~7?q
z{Ez7Bn>^h<aMd(*_EcO`>Imljy~^aa=B-VQFoY=i(D|vTrPK0e6Dz-Ya>ykO*ZQNB
zX#g#`W+DWd{Ged<t`wtF6^yktbFFuZDXj}!6d>=Gr#0JB%V-i*f7J}U>5_Iw^FSoM
z<Rmj+<B~~F*#5MEi?b+U{ShOMtjqu2W$SUqY%0HW)-mwjsR!*?BmQ=|{q6a!@%fPx
zSKz60>o`_QtN#z@hrh3;Jsh2mxG9h5iacRpGtktR@M7R<S@jLqDdpWG<xbWoE<E7x
zOIk6Yf!NjBUHEl~lId;(pJE<?Obp;e?Fow`kK>YSoWW14`X8^I!cDKm#m=U-EiD~C
z51iI|8brV8UnN+3{jw)zFJRn!2^=W-&B=JqCViizXZ_TtS-6p)RxUgtRd!swQMq~T
z!<zoy1HC5-4_8Q+Vmki07F%_Lgf&I<6DA&BO6qO=LAsT@ULK}?>-t|n1magcKO;1@
zi_^!bJvCm98AwN!YX0Qintd)m)N7vpV%!P}?Dh9M=Wo9Cqks$jeaQM=oBc^xeQnww
z65tu<y}RhnTthBrI5aNgND8a<*zeRpP1hrsPWsb4q09Al*gyMD^4$GQOjF0=n3SW#
z=E{E5!AifjV+pZ7w?}Qdq{Thl{)-PX!#K$1)=?~2iQu%CS)K~w5Pp7`TYiIoepP-?
z@9PF|o_semHzOZCg8#`bLo&SQh(pQ>Bj$!KeL^C7hr6M3>sD3z9E9)>sa^jo^z}k%
zYstY&PV!XiV$V)fvz3$Yj;|+Zrv8@XAfi_l7Yx+*i}(eG;UbZ;xZ44Y+iS9J>h#~m
zM{+mj%8QqGSA_sv$ismQ-w)jf8}FV&Mr!i@RX$Gr)57t4Y+X{ipBBdlWzcx)2`zoK
z$au=O;}e7s_GVweM0CP!_!0N)$x{if_w4*D!;D$u{KF#_Yu1vjjIsTR=YLueMDakM
z+C7Rsf23_J#uMX1%tnf!<g6ZFV((Z%K+V{H^nuI^YZrsOaTU-e+6F5H)Y|*R_B!-p
zl_za|Uyq2EEbR#Ux@Rf6853q%s~L<%y?KADbYk*ntTY_BIe5m5MVsH!y&fWdn&t_g
zmvzRYlDF+v-XBQvKQ0X19542!7Uv&4^LGs6(9HN&ZdU%h5TK8p4UA35rq35vnzsgB
zIPe{OVa@knz9gvXVSXSZbBZ*94ENsv-53`%t;gT!<-n`Y^-t?d`sYiX|0<4AiFG0g
zk>~tNtW9p#&j{X)krkng@%*o*lCr()p<hI)FBZ!j+Afa8y*X~`q*5+dC%w_3Zl_>Q
z#Kc2-Uw0SW1$h26Ub2BTFca;Sb*)Rw-ud;wP4-PYYC0BcJ!y@&muMNcX2V{NQZ*vh
zsApClyG@^|ad~(S4w*hy#W-NwIUBStQ(}^lx1+`@zc_l2qY}#lqXz;SXha#=Goniq
zoyv`JDj)iDnJizB%8z9Xs98Ecs`@`RXa!et{!WJjavhiE^7hW>zIQ1{w{q@wKPb4x
zE+&ACh^Bz%-O)dDNnKf&CrV*{1>1bH6_;g3X{jd>eVUq88AT6w;g5i-<7ud`WBlcS
z12uH&qJ3(0<>R;fwmfu7`P%8}#Ao&%&nLMb4w;&`Yt*fN(TRa!nvZ_d9Srr2z;=QB
zz~ozhZE}vKKAQg3QLpA*>*(FUy8i!GrxBN*h91<g8uiBRe6!a6Y{`1s=>Jd=vEVv?
z(gtfA8M?=0Ew-}phdT4dIQmmXX(!q-`;z+vuB=Mho+&Avi=?yMg75-SO_~1_*!v^!
ziXYi=*B-8&ig^{z6j-DBT=LI7eDK~Hs3Ec6`Z%4%;RU5HoaC%BTt|8$aL+oh*NXj4
zKYZWI3y{z9xbE8-Lo=cFI?41&p<{YhAavoT7Jq6%5uQpcK2+C_T!14>m9|srPm&Dq
z{ty3t)&Bkr^E_ZJwg+_S69)A1&Zi%hL}gTxF7AfI?lTUcGK1uK%XqBP(tT_{+s}g#
z!fCdna^I{J4tJaez0YfVe(y)X#QB+#-_B7Cs=77(@8@Bk1PurSoc!ibRu6=lygpEo
zb~jWRV>)fLxMaq5A6Ne;!^Xz(MUQj*^RARzn@z?0d!gwAS3k5Fn2+TqGLh}~aHM;o
z(UX$7W;=Vo)n@*P$c1w^`+iMbPaG|ydJC1P*XuI1*>r;zW_(8;l^C|7>dsCuZuIb;
z9!&ztvRC=_sS`;(9Zt<F)cbEaO|bFRk(RA;ZE@ei*WveN(Xal&bRaIAjp*yzd@Bd_
z2VWYv1`|X?KnqyjK31hps(DAFRlb}nvWDLA@P83SVQsU?x6`wK-z$8DI<Pp9<hND0
zxHtI(4k*0V{k+q7v~lAge#XJ89dzNMc`N1c*#A_i>|AO4y_>5mN2VrtwAD7CY0?}e
zE+e-)ejjeO;rs!M$l-oyvwIIoXC1tEAGl`n1-4@y(r}?O=jG!Bt5$tHSKP@xWn{YY
z_(#^*R+m8wqNqs+k6`#?ep}l6&lTtX$f!WS_=ax4Zr=l^ZY&v)kSl(t2a6BOt^TV^
zz`hth$z3vD>-8pU4WQFXY4Dg(e@uzpjZ2ptL|2!?Z#Q*+k{46a?TeytD!J0cOq4y|
zQPci@=wi=b7jPGQML2u1?0KA1%y>gQ3j?%wTMA77Z4*MqM7*L(TwZ2%e|S7&9wV!}
zlRS}>aC2Xh%l1DHEbt3I&OXw`mHS@{C{+n)dJngN+ulY7yy`@|%*9#{?QbUU6`Y&;
z*s*!j;m|+H4o}!U!2#d@NYm*za;d6gdt&!Oug|^5*;SQxoz&64eZ;P#*#nQ4x5th%
z;5{~zH3yRLIgOI+a|5_$R4I8mMs|%<+HBuD+9)BTSYZlLb@la!^6SRUj$!+FZI`e`
zrZ3Kz9z6N=Qd}|m<7H-<=;6So-u2b)DXMK%sYw|=K61Bo7*a&C%>4Y)!<+jUiHJv5
zy}dh*ar&6r(D8>XbvmipdF4>mgAmhv-`+dR|4q<@ex3ZVP2NJOVD?g|YTe5NWbL>^
z_Jff?w-Gy;K7On8P5#>0+=a6lx~(SVIO)y0u^oAM>A=wQo`#=WF9?69sWwzg{-^N7
zQpv)Rn4a6=twt{6Jb>>~$290LwCcyFBiN24$)fp*eqU%*p^wA;k1v}IsMm&*9+Z}g
zkM~_F=>y+J$V)gU>k=QNzwzNsdaWHlI8LstD!4xq=lEn`?N!a0^MvAiMZ^wkddW(m
z$Bm1Tr<tOo6S{K&CUVr<rjwgbCnHMXM^A3}1Cg^^qfgCR8_MtR)Rb+N<~N)<eDUnZ
ztg}}%Zm8=X+#i=dZg$T*?n68B&gnkM%%Nyg@rT0lAPz#`I(8(Yoc{A!^mcZ4UaHSK
z;T4D8lb_#o*hzrMuVa{r7tR*WdD&MdoDZg;B;Oo=%$99;>wY_E$cVA_CZBx#=0-~S
z*v7t3<wtBjRK4u^9Q$Mdy`mU6uqB#(5nt6?iY$+LB)=J%{Ak?#tlaL{hx>=D?ML$6
zXYZMvz8bzW&Z+Sr$lWRcMtcU%a{6MkzMa42H&zzV{M&~I-tpG&BJ7x2XFJX`9bCY|
z{Y|Z$`^*k{jXyZQYNpsmiFSsr^?lTxd_1u~K+wGs-F8^8<U%$N7@56QD$q(s@;Oyg
zBH^=B?|(Kru(#9c)J0Tr8|oc9`R)?6>$hX+DeD)txzpuOgH9xF<Q;)82<o|rW;pjB
zlVg2l*QTK%qS?Y0W*)M<es$3zty5m=y0b=xJp3U06Z#6^O(%B#bX#X2)uy$2pdt2o
z(|cL=&qu6Jxu>qb_@PUVC;^N9(Fr~n=Q9fkb*i4VpPX8px%<c1l(MvDocC91BxPn6
zlB7FYCO7ku8<3XP5(?|$><Ut%vX|W6)(qWVlv3f0l<KW6F9s0#@b~C=+bNBW-#BNh
zFPuNbOevo7DClXtwpjam?8Vvw?HSeO3@7wdMqTPU<+gF?(E|(DIbDj4h2L?lpw@_V
zqteZL@xi{EU;2v01XB}<ij~4BwW`;cAgoGe{`_|05g_nHa`w2Yy>UUYeS?K&jD%I$
zGw=3a!!*6aoll-k_c5^P`^xz~@^!4FU-bT&Q(0R3{YnQrLzxr{o7U`9M%b>UPwZnA
z%Y7;~_0rP0vXjOmJ2$?eYvbBj7k(CGjJphfbNqY({ydaZd9E|L&wRivLM6)Nm<ev8
zW|S%ABk;}vE?m&yk-R6V_qQbB_v(7YaGxnR3DovA!eR24*NLcucNT`f{#Et7`l1e`
zdlIcoZtt0=Z@G=jzBERxk9WW@vs#FcoyGTIGHr3`A`e>jVHi-jC|w`#Ol|%-VDkp-
z7Scx-?2W6gK67EfE``YY7Y;l`a(SNMFlOX$8as`+080Qim#0GNmS#5`paBsp$;~eX
zrC&`r2cxPIL%aX&rCGGN_sveoPXs&l`da@o?!11kN)a+)R-Jx8v&+*h`E-l>tl&Dm
z)U)S-pr`)o@9q=Q?t}mPFlJQ(JNQY;VMGSlJ!vYsbFk^(()|_3RT6S!2lM*$rNe&U
ztiyk7b+pP1)=osRLnDTaEBsQH!~Rj)zAs2?XbF3*9R0MlwmOqJUKY^P)@D@h6JW-D
zSlArz{?9Re>#X6b_vo~U*}ST^j@N5$z{EA&H|=+feIryV*|Wk9l_mE_<)koA92%H-
zK{YJV`>2OxPlGKNk)_j*JO1tO?9rD^rhl6!h3|as{<iiJ<af0&&U$Dh?u^7j5_5%r
zv4`K@Hy+}CesO=`kbO`(KA$a0&d^T1)A4)=(2`Jbck#bVKb^q`hj7;u_P6fT%ClQb
zl%FqJe(cDW<OyQ?Hh!i`1S`)NI(w2(lx|Y(5AL<L{R5|iyAMB|J`gH&xEFRUE)v!k
zW|O6TCF9bk17|K>v_JE_@$TsalzD7+>8UZq=YO7iJW5$A%sTQ<_{g28n0Wgwe*OK2
zGYe<T&nK#=7bR-@_00$DrH4<M0&~-k=p*rKOX-T}1F45#I#1(VZbhl&vPT+vHe}aM
z%mrjWs2ysqqleF)P(ITc;A(w1h#D332olkZ$pL=sHtN_LOoMYj*>pU+=V+hnOF>$7
z{PD1*xX_0w;VRnMZZ(yEuW`g5Svqwy_+A74%c*@C7YFpQH`<ScnzMKC{6dadp@T>{
z6Z+Hmfm4LF(?5=lWZzf5?L>sQ`JZS_cy0Y?!5(aos{Id@bIC?s>cZ~5dVcM{G*8Sx
z6}$&g>#i%yGWZyG?*i#-dcY6u<EI~IR-V0M=Qwc6r(2>AOSbo?ko}Sw)+byGwhzwM
zoc2{Sxrv^<dB0?73U($b?u@yyZEyI#??%5%wh{UGjlRpTQ+|s_F*{F3G&lA*88!X8
z!El@1;Wul-7VOk+SeDMc!-Mu-k1W?pLm#MV@IduHI(Hj`TPbdC%f1t)|H5H(YTxRI
zTQ_tLF5X#CjwC+NcyY{1P|vXYX5G4Q5dYeyNpUyMx2{IDMi<R0ZT)ZQkFTHR{cR4<
zvoKIukN;lt(KL^<58fY&Jn$uOEHP9iX=FJ}RQ8ln-2-@?y_YBJR>GORZC%cG&mU8Y
zK6OsVFi$^{5L>?8Sv|sW&kxULZ$v(NziZwCV}A2pq`$V=_S&h$fX0V`L&YO~k>5X^
z`Ke<~d!4q3E)&D;VMYh`aoUS9#T-joJlDZ2`$zg!|F4meM}Mt3L^pKinrLPmJ*4%w
zJH8!T{B=1*gu(DXe!AwF!M_l29QoH@j?p&ny>X{qDn7nQu?NI^(F~j(twHm!Ivp<U
z!x1}M`*<Vlxob_A?9Cp)B-HxkGaTeXo)K@!;c#B{*N9ssq44Z;zdv6Y>n}-#ZG_1|
z-^U$4FGp8|occNC{oPyWqX}2vUHfnRNo%=n>J^WH_G9DDP!cEmen~<R-9x_p_i5>L
zvP^4b_OGVjrl(Gw`_McV+cvR*!Fi<V8qP_?!g-pFRut9zNjLIwNR4y_TNtABOREuh
zgKg0c4uUC;UQ^YmEKN|_j6yzJm^+%#`FqsOUzvmQ0~-H2bicXqC}$yRyFb>mG0>CN
z0W)H^>PJmoNSZe4oU9wa@<Cx<0%)vR*I6<>`}^_Hp!0~rC#PiKJAwzcq4*^2)Z)CX
z^iYO*j40mQthH4y`B=rb-^ZR3LlY-Xo?bd)ZBkVR=yPndJ9kFlaJa9%6gk_se|(3l
zW!3DqI8)s4;0h$2hrfq%c1UUM@&5k)?mWh&?tT%1_?PvaxDO|vj~>6iQt7mZSEz*&
z+;ty|zS+0Cb}5>17~9$_>J^q>%lP@T)6sD7Jhl0K`A?@OYx;~U{ylo7d(r}a|Ksbe
zUe(SR`Qh<LH<A(@&utmhwkCAP_E$VC=;HRKE@^MA1v~}!G)N*(`TU5T0q5m4f1YL;
zedtf5%ouxz3YHc-7gARf@JDS1$NCSoEZ*^0`_yy7{@26MThtUsT!{Sa{%KOc53zKG
zlDo=_iArQ`AMFml=krZRzol(1dsQ`pE4j<?#BBN1m-jZR-M~_4l3g;t0`XH`33iTT
zFnOw-!Fk;_?0nH7=t*@ju^Dsf+?Ss|dmdBxn~zDvKOPj#_QQca&DF=YbwXVx7>}b<
z)E%jlw)>8ElU^U4n3Uh&d~&fr<KJhMN_gIWligblNI&|V8q^OjRe5R9QTYMWRjH%!
zu9DryQWsP^Q@$@O>mRg=?Tie}VV_pywwB|}JPyrP#rVuFls8u0U@962p(P8IWv3Z-
z(uUr|uR-ORKh~&e4NadX!u5Wod18L`UAq3lv-rg2vSiIqJyFrU@E_;qFQP-E4v7R3
z&xo5=*DwYX=ajCm$9$(8oL}yPv5>N9of9}AKa{k-^tRJ*0Pm51H^rfe<DPOi{*zL&
z_J83E9|H#fvl6)+G2yT_^wAc)^L;+Vt}PLl_kE~k);q0|VQ=F)m$l#kk`-^2HXu&h
zMtb-V`o`M>{#wzuJu@Lsvft|3;{8YvZSAqYPc64wdXj!JJ@V3}<h0wdEyf|I+K*nj
zVIr&`**Gcx>;JQFygIqz$O>v@3IKorjvve(zaI*?$&ZOIiGKxeAFI}+T}lIgzoK(R
zum3eBl-dR9>#g~HM!L9D7<dmg_UozEH2K-3|Es8re%nGG|H1e8Zc4lS7GUyW_<)h3
zZG6VGXuvVtFGa9f+0!hjj-4&)K{dsjVF3C=p-D0O*bfK^%@3?2@asRAXSn-LiUP;c
z3tGLej0{Djx<v4*WJXZD{eCDCH?{fH#PrxNCFp^P7ZaCjR;b@2w{^$^D7-A=p9`b~
zH0Q6i(+P`XYlh+YW(2iGRzC4TM6LaNV=bi&1kgY6{%-uQZU>QVO=e{a^;iCVwsETe
zKnToj7(w~#;Khf-fCK(_{<?GmUh@64v&P2mM^!ft7|$B;-&mXbDBm?bM&Cae_JrQR
zKc=-WG2Y(m)NT%N%&Pm3-<CGQBq9BU4IJyEdTXaYb#+{g06NfZpF4PmhsGu1M!xfJ
z4&)87LVFHP>`bkmUmZE`^ISY2DVsL?p_1h0pDUT_10VF~aqR#7@eAXCXKd&Ga|2Zl
z{Y6s_%Nf>}rVe86T=^x|28IBnLpqYay_iY=ToU!pkM}O_h2j1y>WtGle@5SU4)LA%
zXDNMs-odJsDLN4u>bm4#8N5CadgaA~!}xOR$-|LR7t+UjcosDMuBOnV0ne)PzP?I*
zSJ!8j0G+@uF*mpAwFP|#M*Uiwmk#g$AbyJablt&wbHkNuar4S*-SN#LQBDwE#|qp4
zbCGLM69V#bfgG~Bm}u1m$hFssRLOw|wnQ5-ky`?Sk#<8=vRjcpf)it<F4wX{6t-B2
zvH69On0%#)p%O1v6ip!X?-!W@5F)D<h!0RazpE~|Nn4j7(smtm<*3gQG?wt12$)Pw
zXycfdk5btarMVcDpb~GVWoIR}lBmn!|I0-sc-2KSc1e6%N=*zo&<eaagY7z)E7WCj
zp_(mxwoQvwlL1sELR1op111f*BrgzW4{5g)v9la3A^2IL>&0x5>(KtQ`&9-;uz|6O
zam+4~I)@<hX@R6^qX>X}D~aZe8c2f$v$tx3b8z`G0ujyut3%oRT-k1Idu^CNb5RQi
zVCRkcMF$4@0DNN9n@~UkOAnuDOVefQdQ~trSo$#QkkLTHG8Tde!6SFuYl!rnd`+#I
zU_us0N1<JU(I#?fR!!yJ1fod8uuLn(We7UzI~l+0_t(KzEy=m-UYj*+k+w~vH$IRF
zV9W4ms}_3%dtT!d9y*$UPec^jz|_1DDhFwxmuxpNUjd`4=it~PH6|P7#m^zbhj2N*
zRLyRhz?*(}8b-4f__Ej-RoC7WMpN@5BB;h<c+NeV6*^aC*tC(D&`1eh^Sf$BktRw*
zK}tR^CruEui<hgm$VT~~t;h)bT_}n-E>~&XLeq;*(L$;Z5;dqn-v9qt_0D0Y@SGHF
zYo=8T$2;1HslQqXBYBDHm?VlZ5-*08%&GJ*8Y~fj6AXhG+V3LL)W(VWR$%RRA3Kn~
zBoqNsx5Ef^=`;<Qnt<RIeR*_`$~J^p#!J+Mh*CV6OF}kM46$y$)C7VW0R*z9+aqf6
zV355!MV%8!2XHi928sEC5JPvm25GmVmWj~<=a#^zP>zWy{Chq>hk_3z*f`-S1e#Uz
zTtF^XWvXTJ@*yUw^8wLpBY++w2k8q9qHD91Ky|iS02`bCRB0%L(FAI&{F*WOZq;K3
zuvP9dTt%W9(^jPW|0sGFN2dS({l91>RL({!W=?a+rb&vNR&ySPP2{jS&h(-;l0zlO
zIn5X*%DG`q!yKcSWlo(m$LgI^uZZ3eK1ko+^Y<q_Pxr^;`M9p@c3D_DYxY`zsd$l!
zVKZmT%v#7Jb?8|?K2(Ncws(YylC!us0P=;dN}&Pq&FG;{F_li`Sqn86vCb4ZN<eu>
zDw5=chBeLLP;ICY{|KPNYD1H4hBc2ZU}>UeEGW}VPE#n!SxiXN8h>iU+*jsdl?GEn
z`>3xm#by8np@plYLyaetm59!YG{CJb%0F#@;U-ckS~AX8tYxUHXg^IejLf527)b=H
z4J{hbf=T0{$cM5gGqpu*9yariE2G_x=wzswX$vW)AR?2vD%_J8Bc={C18QnPItizo
zhb9xL3rAdi&nT#9AHj>115aH#Q_f6qob8|@?qhE$anxEEL6r-!%zW56fO-U1cC)^m
zUo=dBa+B}T>P4zlkRrf~X^j*|JLn70Y#>0u@P$9Z3G=8$$!J+Btt=*#b{qjhH=hRH
zV4rfFB~w~KsFR9J9K_>AfG-_p;iicNokX4#24f#$p_oaLmdH|@Jlv6RU_jWT?ai!;
zQ<RS@qY|oplmfw)3MxQEkYa5#07SQ`cokf8;v*z<k|#k=5=dYKQq5;1m`4|stUpx&
znayUp5I94*3`_0tx5ky-ODY-=6}2F#I7{P&%9X2did3eSatIomIAEy&B!*B>Bn-_4
z59lP)+@utX2Q<NagHR})1|BGrrE05}`AOuq2}_m36Pd?Vq8^4=Gz+oq(T-rv6JRrp
zO2g(Wn*oqdq_T9aFeVWYY2YG?^oKeE@W45<;=vo}^&@o;HKocC(?DD=8z=Nq%|cUg
z%!hE~m-7E_KoKbugW`la=6)qU&5!U$0EP7)U)<i8>1QqOE36=MGELUm5)2|5QB8c)
zxtxb_V?=djD_?DqvNF+Q#siVcB$do0AP^xW5IcetmLyU!c;KUGhSO@A<;;+POIwr8
zCO`q3pE{IQXNiX{SWXHY5EY-;Xm)c6(d1rTE>SdT+vt%I-)M*nm0{7&0h9J20g^-s
zPkOWng|A9B80Xp%Mfor=?Zhz1LK-XqWEL&TkD@PPUgc#bF-4C}is&RYn9-?R`cXG)
zz$LrKEuMg~lGcFe$uXWi5tx`Kac;-(kU_k1h@LvT*}497xruL#12mUtG$>I_<%QVW
zS%8NDBbh|R@Nu@J1qptVCCs$5vp%iDbI>=Ua+8U9Y0?;+H4ezUqa%e{0)YLx42U8f
zLrle-vd}uog&p(q_Zx1O(-HA0qSmS-P!F&88cnJrz)9*=Q#fhB)n=1a^bD#Yk%t%d
zBmzDq1CE8AHHb$zB2XL=I5JdlQaV%t$XrU1)=q~^C=-Oi=HN21Yd#>Na`_QK6{uDX
zm^xxYWgrWwii+?__(Qg2OuMDT=3AC1P037b{v|v~P7{d6{ok)5ni~qZEKR-~nhnHk
z7S+`ewxQIzAvy3%ra2fq0wj#(j&X&h72wdZF{rbaNLiII9;z)OT6Vm7Qrz6vcu+x|
zebmKcz@Y{zbJV8}ElN^hHp`?S(ve7!NSfmRW<dffXq2ets}6-jab|!siU=7e(w)J4
zQ7Zi?8iZ0YN+x2~L{o7fu#`^XOF2yxwKz#eMSvDHLyBNi%FHD)L}_jkazOm@*CUJo
zGz+p+5Mq;HO4irr0BhVi9|z^|WJ@W)sx6%kII2Yl(^b-rBZOYo)@D+j$h#`(JmJt3
zwoWdJR<$5Cd!&j7tjI6nG+5#*;=^W&q;<NQ4_neY&CNp7oQU5NW)Mw%ue6;q!=Hbf
zQgt*yUm)dYhpuAL)vVJjC9UDw(5F?k8WJo}3Gudac|@KB%alkf2P&y9@+VVWGC_bD
z6?iraZkn26%35+@ffSn{?)OMoL15`jVjgFQfT=QY>f(?eRfe3Q7JrYQ?(FVC#hdt8
zFJ^*}bSmXY%ZTh52`*3K^9Koed;uR0N_>SDdRT>ggyH6!y@QF0$)R%1@SsGh(U=)e
z2=J#0$qH2oE1XtIl*YrrPg}vqO`A0^#g7Eohg!;I?ds51S<OuBSk-B^Y?ATI1yFd#
z5e%&y*fPO^rB{~7h2pSDT$nl9T+%v|k)MPCCGvp`vpU-qE$UyDCk4FaaG`sEdA^vB
zhYabCDYr@ypl4j+ru$=_r0;=uYZH0OWt#2uT>pk->AtUk7mX3gl^$_}_+Koby1Edp
znvfl;rnxO5Xe%zHx*mfW3B1&Zkk5yQ2V>-9Ycp>uOHO7FpK&Uh)ylFmC4(65jI<MJ
z?#`L<!39l%aiXA|_ufT1EUxt6PN5Mc9u6POvP;VH_pd(j#^lqYO_S&Po<S(f3F{RS
z^El#^f|_Tkbxh<EuPK_*#Iv~0c0<>cPx$(4QYaPuP1E6P0=GHs#mD+>&g&aqK_)@z
zS-hK|F{uv%FqP-lw#-Ew7A+zvwHq5m+J@++G*~OmoBit6&X12%R|0RJ(R$#~s2i*;
z7z(jjaGY^Z5BM<8EE=G^V3&JG@E42X$AO3t-lJH68nPb`kB%)$e7yfri3F1TZ?3=n
z{k4UA6Rlf&KYh0b18+(DYGGv*ZK)Xd=u@1adXL$98B9aHV{z7H@x+iXSIdh%=KO~1
zHvs|+!=dc?jOMhpWD0+8ZL1r|Wc~TuwS_Z~z@SFuP+Ov%Bjdi|<#C;r;<mwa0iS&r
zZnnkv-@XNIaoMqz{U@e$L+M@;&7>sMdA28U*8Jf*GyeY5b<1>f7yP2>Y^$OmQdmL!
z{zh;jKl4d}-W=mD`5*kj+R(Q^iNjN^|C^rkco!6zc)T23F@daI6!%T2fvhu!;DuWr
zUip^M`dqm}aQBGRXaBEhnoT-l!I*}4OMDh5&S1Ra`@W!SeV$$A<d{9%U)x`c-&@Ol
z^ybsJ<v5ys&UcTQ{ibQHlIfLx0vi5j`uzgQYkEg((>1b5!^`HC+)r^s2<hwtwaD$l
z)r3Gv)ExHaJQ=Rj;N#@=4WZ=wS|P_Pw!gWy!fI2}bg<B3Juk|eJxUYrlp1a#`#oB#
z%vAi4F5@CU$3i|$&(YekcoD5x#-X3NrCfJ4`o~-`ft;b-=iIbZp`)vVa#V!$6@Tmv
z&P7$@Bl?samc$GRJWE_v$L(&m&*TGqk@w7}54w_xl-IZacJ6Nc*QBrI)4=`syc1iI
zAdQv4f`t3UkGIcEJJ|)+PW&&zbyf=sF&us1=>6uq@UBme^a^`8f!3mNjwFe3HT@c%
zn0{B>NpRI_MEkBubTk4g$p}gqA&_s}`S8!SQE%%5FK}l{C!E8~$$FBX%|wT%+{pc|
zOc<H_Uqy1rzEkh>B#2*3@(oBuhMcHE%s+4!-DA<g`m;{raREDp_r9h_xU6AaFT9t+
zx)tQR^gZ%ZlG6OHtouArFVpG}cIHdth{sh39I_*MyDKKb3~jBvr2MeE1lx+k+62e@
zx>C*~o(tA%cJ)WCP|qL>;32y4LPhJ(4oj@%iu=V*x7OFvE77Bu;v70trP|>|6xa|v
ze{XVl4VA-celtHF@iB2_3r>wN1t;wC8MEj9<CkG@7i1Db(zcjdQtpf3vnv}7kyl}=
zcUpN*wnU@cCA%cr>Yet<du=gC(hq#&OIJ2v@88#dm;bFYY=c^~s=YEX;{lwUKP!Jd
z956gQOlg|>&pbpvqCFc?Ognkd@l7&PZT(~WGqbD;>)w3t@(#^ET$fs<y{Bshgj(6g
z_Rr%E5$go(rOBWiXN@ZUw=OT+n7CeSyh<n%az<}mrZL`tmi0co-{ikzXWsX9wJ6+a
zx{#+!*|%zQjxa|mfA~#NVM*GB{OlkP2Y&DhS6zA*H8mdS)6PqKh6ck-TGMl#EXl`0
zWSyKMTNeGA3X184YZ|kp?0Mfr(!~ZNfqMiUc4keq;v5=#)47Vn2+8@Ekb`+wZWb%b
zZFxO|tXoi`Drcz3^2Af%KOQbM@%P|6)WiO}monXo5(}~hYp?rnxEcBgetdrQRrE(%
ze(hd<drAN2pW^N=dZ)r4Wg_CcN<J%SllzERjrZJ-7I64=U(k)5_at(AL!jXS-=nzT
zr~jR(A(s^*<ge}0z*J#mLxLka!2))HbL*A~VQ=kTQjD8<fA%gZAu00oS9eC<xv?(C
zxp@CsEjiUNAJ1y6GrJeXEh-*p1$#@SoYL)gR4Z?+=6kDoogz`*Maa#@2&N~=kZ+$}
z4w<>$w#HxeH-A&(`vcT87a*cpr*k!^^W_5<Y2A~me5+&oH@^O~RCIYP>6O~)U;FRG
z-&RlmV~6(q#l&=8yC&UuyZ7<;wLg19QLeH6J}>;XfyID#{mIj><7+9i=OfvjtfnP6
zy;6o-p{bf)QqBa<>40IPv+5@H`)9<XxQ~}pe=KHesI0M9peWCvSD6@>TOPV2V9FcT
zl!Qz!6&0S=BcwiG&o}3=`>!p1Dy9Z?oIUR;rnoXX?A4m^$sn*is=r~I^YcHv6OFB=
z&8uN=@vg>WnXdGvsUK^7pY@&A@C)v924ZIeILx{PM&KIfWua)Bm8;z1Nw(tCurBF*
z)k2BA72HL>@nIWgE0o&^?fl%SgW@s;%$mQgpK9K>$c5{UG3NH*zbZuQH?eE5%HOr#
zmP>N4F|-RDg&%YxVQ)e10~@fqz$a{`uB(dVd@v3B<xRVeJl~JpGdlS9*42iTdapab
zO=oNrrpYh2HvQuME_q+FHt?VL{0yav%?kWiF1)yqB<hm2#VC&L@Uj!Q4eQ*{iSIqI
zedO)=kF@U7_ld^*lIBmESW>*w;XHgQ%b$|>Wrm`|on&kq7bUk(mR0du9EY*vWRLeX
zhL`^@@81h5Kne89?upD7*?lYj`e}yu8R05Coda;5Nw;k>dIw9R{%1(e?SPkOW`DSO
zJjBB<@aGqw1k#K3JvAg7MhQ)EY)pGRY-Ji$!drJ%y*A6dFR|dDljXUbgv9_iv0dfG
z{#L&&%Hd^NB+}(aeq8QzA9JT^!-!3~eZ23+&i-K%$T(DO_-zIG(^!vG{%il)To#(P
z03%@H4TnvjJb}L2t^WnCa;t)m_o+Ng(B`J|W13{a4^&>hlQ7r{kTm|%%nd$(NH6qh
z)=}=$vViqCLn2P|Ez6`U9~EtI!W%(xEUE^}F0YVOIaRpBdnJ*700-+DJSIRjwF_6a
z8`^Lfo8xO$9og1>Gh)N2kS{5N3z!d{e;G3mhrC*wqw+dC-}n4ePmR?=XN|7N)&Gks
zm%T(l03Fe~9;*^g)s>19V^_uQAi#fHrDbKEKu(I2Yp&n=iGL;%p0kwLvzlcsR<<F7
zD{g*H)p>#e&~>x9ZAQe0?TM_|FSoB#|67+vnfG8em_9H}>)hBGNmwtK^7zB}=XWX9
zDA)PUnr<7zXbzPW+XD2R*sJ{)O}u5fZqAh>{Vz9uv*ODOUCx=MJzZA(Kg8^vvq|0t
zDWeIg{Wb>y=+gbjL8u3aX7$DHLV1Uauf|kR=ilP5Now=9hbjkZlOz>Fe*LJcb9>SJ
zodR4q^Y3$qnJOoxL$l7gZTVpIpB%=Qd?97Y3nbdy%E0*fd0~cb*$so4B3~9fadXmD
z`PcKcdmlJ|rgK(!@Eo#5lt=aeG5<dKonoxk<CFX6N$2c0?BYK6+me6khq>G=Ry_z?
zlitKp+!<7eAIh@U$u8HP0qqw^1f!bZXZQzic}v6`yo$n=@Jh7J4U$}|&Mo+vB7)3%
zm6Mm2hf(gW5aZQbO>2n%dq(B$1vb}6c=_Wfm)d2ebxrMW>iVPhN|J6;cdPCb-yL=R
zUF$_ZCQkTl;{ru~ZhpOWV8GDB)Wt(6Cb{y>i51%4T|V3nZBUHSc-s`Z78Jg#oy8K|
zE1H-yY!f>_+cfSzLdsR*{O!D7{AlZjv6F(s7W6}?9B1ZEBbaV9qQAGdv8(Z;#ZjTD
z;HReMxjEHs%tEFU7FXw|S@i?iwtzxeouZl1_F^(01lXZa=c^sA=jpmg9~Yq<nsFY7
z;X@KE^*8LSNZI;iSXyB#;$qeA#%G}>-=l8b6oUKA&2Q_eSCE{Kp0RhKL`Y1<?4Lii
zQ<Mx_Z?)*s=1H2U@IlqZotJ;OfC{=J!^tobXuk^923c|FG$HNpR_9fuk9}U|iw2`&
z*IG@Ci!ND1&fWP@2=1eH?Optxb!joU=p9MTCLdcYIDYbV=A`82N0*2(9|{>>G!bJL
z7~{Ke&U<pN@dNI@5Ky^-Th(NS&G=<6)jx2I*7b7u>`4M&YWAsJy0&6-p!{$kv_4}@
zI9W<TxBvXm&l=+V*sI$Q2+K2D5sx~=@t8lUWA8BQRodK(TgN(EcGuMHj^6Z1h)>!_
zxnBG&MowSax81j@+Bdg-Bn_^+p0V+%_?N!FHKw-)o_W5_$**`chWc8Y$mQz3u#Fb?
zAA7@j6ZO}TT@sX`>}E6As2<*5{I|`eDXx1c<4i*v*;}y8NJV@OxEr+IbNYvibW9Y3
zA7J?B6#{6uO8z@qZV469+h1!C4T<}e)xPxva^+0x&Ih+@KNz;b_a?V)A)Xv?%6q-_
z{Qvbt8~gawexw==d>u_|a$%mce{9-L_|}DWXx$aHZzTSj9thQnd}Vd(89v)s@cbLD
z5Dz=KQnuqDk@tHm26n!u5^wWf`<B`n;sw6o%lk;gH;=4#P?wvh#f2xivji6qishp8
zr?3Z=_>C+z%Z`K{62H<C*NpapnRhNez+Ahhd{v_wdk+2dOJnPI&ljDW8+b9#QNXMD
zN^crldn=UHh?#F}*T7J|QrpP=P4H?fN#XYpl6wGN-oH8>oN?Vo7xZW9&YdNleehp<
z>on||wg5{8x<lQS{29YO!R#Sf?RUAl2EXgnQA9{)re!Ag7bp8|#7flH#5I)k=xbTm
zSyyd&CV{>SHP9vbZPH_n>9v`DqC2bWiJEEUfL64Rd;D9qk@HN0+U+)Pn;X2e;5E?t
ztcL-1YdPV&)X^X*>-98-N<z70toSs<&|Ee=O;`NjenV+MNO3us#nnI1OCY}9-nhWX
zS=#GITTiy#`PWM`aB4>uWd1uT$?{FoianLDMc4{a?&FW;_Zz6pC1%)8LaYq_>2~sb
z3_;Q&Rf0t^8GSv6YuBTWXXfb5c18Q$LXGhe1S`OU-T|);EJ#im(R@8IeZ}pBD=+-A
zwymP`hUd-Ttscf*2KB6Uy30=fwUBj|Ze4gj4};TuK2u}!<mY79>+A+ne)7K8WTo^Q
zulxC%E_Zhu{}eA$X4w&g|8#H8zL~T8s6Udtr6(LZ-{RRWw*3l(3z$pHqRW~e+TBzQ
zn~pE^$f`!d2DJjZ8M_GZ=JfNW#7K{CpU3|ENu4`N{sj@JrMpdm-x7+3;?@uK8k5(g
zKINq|Ch&lNke86_ef~%9j7!3!IKOa;?xmI-e%8)0j$Be7p7eYDZ|KUcaA$2vtm_q<
zvVK~N&x$9AQr}zCy~qsga#)?$VQQx7`04ok&C9uPwIxvRJ>1QJEt;10LxkYc+?w~P
zghZ6<M0+ntKdat|R<GLm#diu|SR$TobqUV-E-o4uQ!0jzMNRk;^1H{E-#IDl>GcjQ
z-&V<Vdt1>WIeKEvGvH3>=Lr!?EjeoM2O?luvKErc{vUrS_t*Ou=`F%$(L<d#$J=ho
zcvy;rX|Hv=0=l<L^c@vY<?m6*HTsM=b{4-!M<ySpNbn5b^}W3o&}cdA{eAcC;M>=Y
zr$o>Bd8^N>V<ABL?Qi!WOF@m@7p_~mWuj5krmCT9;yZP@DE#u3l$)GaQqA2Xk#$3p
zuoiR0$@mSw!@_Y^EU|0|l=gS?srn}6{%VNk5W3!)su8Q#>{iw#!epktO}rW%ALy&P
zrnwl@{>75!oFamr=qEmXzxju;UXOh0YlWTDx@8j7s>`i)zi#+;{|5Vp|J-~3+!L>3
zi?#yV|1avlH*vWT54muaPl-8C=RxMy98Oo@F~MFQq}1YdL8XQw2C=g)=7{VCW{pdh
z`)!iw;mpmSns?jqw~+N)i{ci|Jn(*MtX-!+)hXyu`|K6N1(M7mOe6l<yT_-h9|v2_
zq2|nOcqRKAY{9F;19<UhC{tr)f@3_{<auvGCH4F0rOuCp8{SXggS%prl}m5B0?JzI
zSnd7(d&p6<d96zp6?MgYYqi)rXP^9In71@Hx_aVGscgBeW$3=gm$&{;Os9=rJs>aQ
z5_kTe?>z_B6WFP%XY!P!rwVb9SqZ7oYYUg&GUm?AoFZ^D+)vl|dn0;sBIJ4F@pGAf
z1~Nu;JRtv9xMsDdYCn=)M(|ee-v0Vero_)5uZGkv0p!BTJ9=vprKP)}@Aj9onmH#F
z3pM{&_!EB>j$CuwQQ!UX;oGkN@fSfQmru>sC)Xin{B5#p5~k%%#41;^`;?YSQ_2rc
z+HxsGW`<4W0@fiO*5oX9)9cRI%CjbkCEw8ifLLcA-Qi>v)8T@vBRKDru9wEe#dR%D
z-}f*cRoxcj<czU07Oh|excI%II_}jwVXei6L)i7!U7c%h`#vwJFDKT5Ff7x!KgFSy
zdKaJ^AQ5q+rGkr}>u#&O7B(ZFt+#dE_?gH2cx1*Df_c%qk3J5G?jV5aVC#mgHvJIr
zeTzSm?>{)N4M>m=qeAcJNc=h$FB0r-pOD+>ZZh^_&aY!XTBRnTRWZWKM0H2%UA+aQ
zL+HL#qtycgb#D`K(yr0%?!>G&)Skd)OON3}@Zpb+Ez=*m=Lgh@9cQ~9R6APAo^B(Z
z|E8z;Ayg`%d&7~Yalvcm0_~D!UswJmo7A<0wj`M4igg#!Ia1_)QPO{@$!O6sAh!%J
z0zhT}zd<1w7fEF;U^7mph4>7Y!3NXi(AW$poGUV?u7XECaz3vwKnVlq^Dw3G({P{W
zE{kl04IhRMIj(Fqcch8wD)F#<HT-myF%4OMS|87}p`XO*q#y{HI6H}CACh`A2rfDZ
zJ`NS#9}Z8IA{f1tLsu2kPoW(U7R|6$SxP%gG;uN)4H{KfS3_&wf9P9qd>LjzGHA<-
zA$m|3EEy}-nU7v!WR9Yx@xq8_vEfHB&N>*#aI;Y!*ba?SuE>+JVOpM7_otQB9nG8q
z@@Astpoi`f6BuMMt9U30*DQx_E}}JK@Y)zI?6?Xy&)NDXK!3Mru|Qdg(xLL1wOqQ0
z&S|Kr9r3hDI2;NylVO$kTIH?WW7_#n%B+Ean7oQJTNdwJE(k$AN|Vje5?yF<lSG$8
zlFsA0UoHYzO<9$5RV*67(ZPlIoo^M~OQ{nRoN9*&Jp))_cIH0r0=J_WZDoMKreid0
zGu%t0vOvF*j9-mth67YN%Buw8B6m?(;<PdwbrQoP$Vp(d;YZDQXsuO6>guscoQ}*d
zVdqE{T*QjSW}d>r*#fr^Y=*NC6;q4D!cXR;yMVE@GBFiEEmj&m8Yn>#lYxs9fuL+H
zyjJGn=Ex)+BS6vo?9qyRTxW_trm@UEQke~q$d2pt%!w&*%WvZ=N@iBM3II9`rfC*I
zi+hO_pdhsuLg~thIz|JEsaa?eh_l|w$2B*?eJmv+1X_3log>%KnL^EJFEL0c-pSnD
zhh0;(8t@T6DFo0_A7w7M&DwP+$N^BgbJmN(hE%5C(Hl0&aW>mHAU26RTi~Y<D#~YC
zJDU^5Ey}<tY~ap@kTQ$2*fxA;O&yg?m{z8VkK(ws?(^f#Mahm5tPBW0U087M<*EIF
z<0|QDM1Vt?m}|oqkf4I{#nbwLMB}1RG_Avuk)edxux_B)vtmaQ*v;^Zf=G4sYRgO}
zL6m=kB>5<<O59ovfG^6*{7R~YG_Lq)YMs_+fs$N2Qri5Iec0=bi1JnNm?a?2m1mhP
zo^(FDorKIQX&3*;r`ekUsNvx7NBGm^ZFK-W+)e$Z!a8L$AmdYY19ft{S*iI&u#8AE
zx70Ti5bny!MM*3fX9Cn?7)xKnfRZKwSfQ+J9+}Eb9D!m0IaZVh8Gg=TIm2TeOnh1a
zB8KT(Z5e^IvzWNKwX;RDkw8pD=a?B@Sn7DE@}5N*M8b$J43NU)NJnkf#KU3WUouFU
zf-OME3urxu_F1(jN^6qT4RBaF6eh9w<&C=q0R?_w0L=vpDeH#Pv)Jr#D)V>^6rREc
z7x|m4BLP{Ib=G%*Aa^gRRyh5Hn1%wVHXT5y$hn1Biq|xY4oVyw#f8WlK#RD<$wjz&
zRjxJ>RH;lZuf+p$CoDwNZdqL&J|~*Qvl%WEMTt}u7blL9v|W0~axFq4R&<JokLz=_
zPpoN3g`C$fbzs9%!A^-Q@BM^t$!F5(q7@c3HP8|gD8(S+AzM=q3eN-i(M4H2B}<63
zEWki=lL8PKs`+$K3bf$uYmlo+d!qh={oOWNi`ltQIi1PUWuey3GLpF{0QQmY`nY$1
zbjqO$_>$5H;vq-8LqETj2jH_@nXTCqnMj)3{i2ET0)Q-#fo#J3Y<n7!m`YAB0Q5|j
z=*SRs)lFepIP*1`g?OvtY-hI?Ua71FDSl<qf)dX0^m|3T6|K$&4>PQ}wE*IWb-LLg
z7;}OH9?bk`eJixonq@HJU^M7?@82SNt{KzM&rjFVHHin#U0xJc<YOr3NHU;@b?mwG
z9Kz{xoSQXgwK)R>q1ANmy?6;|Wx|;t;{ovEnJWe$$k0)nmqtZ{*=#>-t<W-0^c=AV
zEfXkJUeYS*6J7~=XCNvI=YoznOAG3NsWxkq#6*eyqL)~^QiST(Tqi$hoeD5olXE-g
zZ;pDZ-n@n+Q(3jKzAn90MJ2?<WJg`kwl$bo*1gj3VtSs-Sk0{1DRj&_GI#DI?tH~a
z;%*QD`pPpr*9@N049~;#ShSB-yo?(6&3v>70FxEzv@}^|_NnL6U7a?`zKo*xe5aUD
zqnEmDm$2^D<^`qjRGchD=#}*oAA+Pf*pQl0Qay?k4^W4pPK~|9u&OoPZszk)MFWxI
zD3FXx?NwVi5MmYpQr&{v86D9fSt92pE>fJ!c57GG8O<8Tf=X-#8gw$`xr|X^T;+(R
zv;wQbYfQyC%MK8`;1A14P8VEyn@^ort}-6;smR7K*`kyTyi6^|xqG;sVvcxK&BG$8
zuVu-LP~Hgz%!2cb6*BohmL;PPt1yje=_gn3(^u~e^fL}b`&nb;^U>;8gLrY6D;Y0c
z>1O+t@RNMW-v|Ep<B~{&Ip;cdfepOx&mRT}Emn?He2>j583+B1_)+%X&W8sF8eqM&
z3(YlF2YM!Ldlw5@H^W6IOJ~LQYL%*kT!Z(Fic|#scejf#LtXzA{u8%~|87X?e}3;K
zTf<|*?9XeW@}CLEF`k%|hJyR7G{c>Y4FMO;1*o+#563PCL3B=^_9xRS_D7iU(MF5|
zeT--3{s+PR|1x#9lM<*)`8(2HeVi_2J&Aqg_eJL(qO2>bW+`4(N>{!0L+xuMb1`iF
zYN))TA2xiFOTRcWgVgTrzkhVgCS0y}(z4+k;bsZ?6~xNqeo;cz_4D3Ry`wI=8PaRU
z0zcCE7M4SGddVYl<f;e4{kx{jojH9y8#%U3HHPDI`)k5K4(9PQPm^{VyQTvM-ce2c
zT=spt{0Btn(v!nG1wjHguPWH`<c1fAaDhoejuv5A8AtEak&Hr5(sko_OXFWts-EZ)
z(gR;&faO;7oN=nhJ8Q_@_(+WN3M6gYo!e*dgFZLPNqYjRwb^_1doV4qOw!7=IyKm*
z_jYjc8>#T8?7LbyORGYBzUt(p(V}^!p6U&?u1(Z$WyJQX*K6f|2A6l`{>1YfmoAdb
z*$~y$^P5i^GS<|HH;itpn~;!K@}?$y-Coz_VuMk;Q+g2>h(kep^lwv7tl}G5o<X~I
z>~6-E+DFV}&lExG>SZaMhwX2&dYBYX<!@`ot08QvEAzuY^9*8B@oEV3Sy5!sBZ1O5
zPtEPqT>-_IiN&?JwLk4#D6O-hB=`$T$U11`UK8JCr{F@>mEXugTA(0=kQT0E?oK>r
zzx}yp%Rpq-(FSfTfW`mQ=32Q@|M^=?Sy>j0c>~T}@O$o&E=YT;wZ9iLR<I*J3NuWy
zI|S9=OF5i_SGs~GYlBm8UHfC_n=2GN!Z3f{9d2*W&Js^++|l9D%xSIcWHH_yU&=eC
zY>a;#K!o+g5I)au71)1h6w0kuhocK9?u3-D&-}O8vr$%(vXa5UO&_}UcV=#S%+09}
zoNTaqW~(3~St+pLL25m|)cnxm*GfZLb&cPhYrl{%(R=SoXvD}ix5a7^|DaK`y;tuS
zy#Fc@UZK{E%Hgc-1ObFErQGZGon=fKRxbaEn8RRL&v)0!sTI><kNAlE6-!<fE?N(Y
zs~?!jhQ(HIh4f7yUL;a)5q>=S&yVhw!wz29j5BTBr!D6T=k<5jJFV%2i24R*;X$Y|
zZCbH3Bs?E-9<8$0-F=+;XY(M={!rI=>X_bMRFbuts9*bxQZ5KJBW2SAvz>>VsGYM9
z`S4w|ZJ|+4r?hW5>hGxhO6}HVt%VD%J2rsc^{l~nTg&C@#O{lWpT-5-3wGA)iGs1i
z+=dV%0i0(oEwNW9h0f3k2W@<*Wnhd-W<#UqL)hh#K}3kB--44w8{{TUtiOYib9-~N
zZ$8oJXE2N;caf8ELTkv8vtFV4Bt__~R3>F1;Qj*aZ<~w}Jx8liJ2NWV$tgwce)HlE
z=Z^%G`Ql3Kq?Pijc(KFc+U=E^%W;<SznkugiO|eRQynoNI2ui9XNz7WvaC-P9wdFX
zM;URwW^C_nf8wiia;C_;UD?-Y1&{o6{e3Zhvb)=7Fy3#B<=V{sQj4+%OrJHvc5XCe
z5x6?kMfWQRSU9XjEEE;c=1T_U&fPS)D9K+^u8a^fJkk*?ikH%4YtK1hd3PrQr5lLn
z-)Eabxi|C#6KZw}qYPL(tC9eMUj(0|p0`oZ-hDe6JKE+B#cn$MWxonO_ax8rcin-%
zB7I_LCq7qQ#+(9a9$z`G;^ecAYrc)>A&bo3RA950y2xVL8C_>xWpH2aA`;k*F6lTO
z1S2An`|9z}Ycq8$qrVgWs5=aJ@yc}LTW%A-ODou&7T7qEkkr#zJNe^LXqZ)~tDkuj
zT5Iod+6}gJ*Zr?S5jG3)tt`h+eVr8@FjXQ{St}O7P)A5rTqIuU@VitI5#^QYxBfQ4
z*3Reoo3)AM7s-CN@cTbv&n7cdqmPaZPt0&CWW^%tmg_?pAwlJe=R;-O3WjCk_B1;)
z{GNG6Vy@Z8|8tEx9r0DM(9}|5#L1J?aK)j##+CAgt+4G}zds$mfg`LE(w1O`jGwp5
z*!RY+GP=A=n?AMHO37<(YHe;RSq~WKeYW^n;+s|VyKc{TyTR&PBQIZOypRe(4~I>A
z=-r*l){q+BYCV@6a@*!=Y}^C&Sh34$M$1ECsJw=^O={A<Twb=_=byTEY!@5P?NNWk
za)qwQ(;sOJ*gbqwjO4GBVbD?Vg8JJ*m9WnUF&tf*=tPn-y+`r%tJ_PAOuqBr&TSnB
z{1ilSN@0^=>c8{rKr1OS_d=g(((!ovNM@WzmOp3sX-s6?z|dy2Uqga@r`~<JzY_tN
zc6Nof?e6z)AXL<kj-|ruo=SwrBE<k>wAQt>-#2P6-%)3b{ddFi)qmB(Y*A}lfV;js
zZCkbJj@QOaZ<pfB1;5PuA8(uftx4-OFKxxAAAsei=6hrv+E?ajba&Gpz1XK86mR4p
z3lS^rr3e3fTD4tE4;H9AaBvR*JihPUvVh?H!Ca-w{1pPbp@!DGA1qIzolQgVP0g}2
zf8PDAp62-m*P&^`u0(QQ^1K`UOG47OYqq*WPk4B6ROL{(ZvI^ZNkyY^-&<-%LtUU%
z@IpXBL09w-@j<v?QSw+M-PP!$r&~#&H$br7*6<!=>cL55tpCo)pu~aR+PUA)Ra8Y(
zc(d11BHY6~^EG@CH?j{eva{!9kjy%YQeahx{Q&8t=dvhg*w3yrK;>sgy2}F-#}bp@
zZHvcPbdLWUWouR2p~8SRzrQ_qA=xuml9hs9wB25YuX(ixabSF@+{$sn9bbf{*MWaA
zyps6(nSV)SsV^C9=r!H`pxTm6R4yFNCGTw`jaTqlwxl^L4%gfyIu?>veez!`7P=Of
zw1TS)h2}m!o}8t%nQ(~Q%lQu4anN(J2YZ#1wPn4c&v@24(y+KTwZV(q^ou&WN26>m
zQiV+JeD6Urw#?#I{M_OO<%|jiMfn!h3XGyTmxld!u4jr%{WG05|Bk$9bA{bc!o&p3
zb@>Y9pAo-=0=1l3v3dL~=h}3Mb9tE+TxgQ&GTZsR+|xT6<JIjQg%@cXW@{&bB=i)M
zuctg86Nq^620vFYr8t0cLa%yN$<>VBHh{J%F9wq%l~ZSQ%-YljAeA;HK2a8Li5`0*
zrF`4l1oXRv`Qe7^_kRtWZMbU*z;BAj?U(cPzBZkB`qNMm*>JvxO*$HE8er-`48Ewh
zt`b9R4K-Xb_!oOqUA_Nmo^217idVk777-9$JfXk1KTV#m=KTG1xi->3N8V$Re5gA$
z_bA*n<l{N|Nd@_zwssByy8)N_J&P8kr6osX6`ls1Tb{dhsk>wRn8fepqB$NzJ}d4U
z{yRkax582LM!tqXg60vaLw?%R4kV?gbc<iRjIEp-ep?+56RBk99|EsG{oum|<0PB@
z^58pFWnJ36I@^=i)!f7;|7_$VSvR~V^+|d<q^6-y`Tn`#rj~qO*3ut}!^*!gqfZ$5
zoN3&v7(kn8SK0s%H6b_8$22#wK^2|D{kBT{+iyG#6}R=x6Q~Fa$m5e9Pc%v-6~`wo
z^xu9p7uvcKaJ!Q{@8HvClvN%zxKg8!U-61A+{(N<roVpmsllms1(6H~>wWOl3KD;y
z_vPxeW7A!z)QkPa=C-EQuEO)JMsK4TDIXN-16p`}4lq6C1T=u_NUB_-<TrhA|IM1d
z!PeoJ<RXa5)#ZCTW9JBZa=#;A&}B&-(JIG#3l;X#$)bZ_rd~MmllIcXc6TnMME93S
zo*?7%=`j|JK=1#wD#Sh4->}KMjp%Ddym2krhV4&u4z1VAooU+*O0c}?tg`D%PUzac
zf1IJc^8Xx=OoDep@^=1nt;aTTx%_jFPPjKWi9{z1N6N@6)<yqF_451t`OZIND-V6(
zSEphD>rM8uo`2!iV;A9SQHQ$7pdRd;h<fgLSAL5Lp7nT(s&y?MU@ppMM~xrc)e2m>
zbRT;yKVc>Cvd5z9kM&e1Gn(!Nq8PSGeI9|5jaI8h91E`q-|N>W$jUcpom5VB@<&P#
zRd(i8TM^AGy7Z!jwl(X_H&-9`iY*~V{ccO&R#)V@^0N-*v>l&Auh7~Lr_Tj_Bpf?D
z>85e@(_s3QGf&quL?^AZMd=2jI?LrxS473|SHo8wq19qZ?yV~g7s{=8GS7>fGE`=c
zIeIqTC7)lPwo)71ROrCKc{Ov0l>j{aS%>{+jfu@iuC(ej&5DS@6>xI(;fOy+kffB*
zZS=9qKJ=HyNr}m(DJkjF_`D;S^}*Vl+f_j~3f|w+TTxd|?P!EBGjZiT;SK(ump-m*
zEVceVGk!v<1@p|H;dlh<2jb0}nYosZ+Qx1V&}c-4XNcH!NNVOcrFj_AHb}K}XOUei
zk?GJDO-*O&+9W@?8*YV~tjkdHH-1v;*@Snsua=f7UGdoqEusw(_K(^5^XI#-E!u=w
zIScH`%7Tl^A#VJ!f(+L!G5^T)14ex0e(;aW1+if<fw{PftmjKSMQR1fl+&3auhogG
z{9ow{QpBYuz4iSV9T(s(S$zI>%f;i6)*t%3P<LhiMc?yo*EMa><m+P#B=L1%BexJB
z*U?p!_S~P7b))5(<tN6lqyF{y>xSEp*ELgFFo4h*-~n5xS98ix9%DE94}NOC((acv
zrte1)UH_){YrYoeb_+L}?=-NA#Wf?Iq|_K(Ryyl=d*h&UA*yYaV6mt9l);Yyd5`ZL
zM@TFdZvXK;ZJhCan_s~B2zwi4qkCH;76~6Mjl8%iv0;1i$&Ysq8hTD!iUupLRVYCd
zm~_MFQ9}&y1kx271h4T2-63!}4Xt-6niDNHt60z~NxVTI%VRf`rH?p;dn3*bdkrAW
zzP-=5|8B3XQQO<_$9C(=TwXKhP{_@+*JAo_=NUDt>xLi8*ET#Rb2=veUGnH6!Lw)J
z@=|^CH%|@lG0dvv52ts2nheB*I)^kBB3gb<PP9Gd6&19}<d7+8WB)PZI+?pA5MdRE
zbAoQe50Z9)hTH1dguB~CPYS~Fzp&2C82l@Fx&>so%V^xe9BhA-Z_S#Q3C1||(^j7j
zHgpfgzrVFf-qVrcsK!QJ9d<*eu$SVf2mYclTrJnb-+Fzyj%5Q7tDbtZ#Yx>6jan<N
ztlZOYV>r_3IT>e0xZp(#-|{yx_Qb#BF&gBOiEpSg2Cyq3deG^%^Ns9H;OE^>^WaXt
zUV!BpiLrcZOyvVlpXV##S0Xz9O{AO&Q}hM7E-x0P<x<{P+SIh1LYJbS6b3$hF$}88
z_}Mh);!pUd<bPSDdiWfA^GcM!LN2!<T+Do?zogv>b#9xsdX74I@q1c$iEo3s%<kH?
zg?Hai+n#bKuaXv#UqUCG-G8)h+MCvM=+KkDw`wGtY+L+3*yy%T!I$Tti#h*1$_!Q<
zkc{{yrG9fNOq~Q7Ov8@`+rL_|1hSJzI>t2?{GqMu_<$+c>;oj*tQ;-8DKb(2DGhPl
z+D2EX=us08`Od2)J2!bG?XT{31o+QvC?tN3P%cRt_S1n&76E3w<@Q3@NaCQNPPfcL
za{H49)OV$QV%%aLP&EHz?%9+-#?EB1aCLt>7tY^OM*t)76@%1p3viVv=INKOzlrj@
z?$U+Ikkin$m^;yAz~vhMzN;q5P+r(Syr<WDf46xuXQ?DL?5N)Q%>H%@bw9AfeH{Cv
zEUhKyCu=1}46S`dBr!!LtBjb>nR?=fn5A&nr@qB^XPC|uc?Y`~T&WI!(so(0{o({z
z$)a^?3vT3bb31?MQGD+}?WC!ti?gZ}V5r*DUMo24BK%Hg!_=>ewg0A1!galDdqTU;
zAba2*F1^WLpog^Ob0wcV1AA<WDot$?4agS+IzX}8nl_v{>Nk1)>i9~|k4~G^&mS7=
zJ0=gbofKW1PqjGhJ-h-5xdoSr-8fr_(|LferQAB}Fw5oLm%ClpXQ(nmlZ=`so=o$N
zW3f+ejmo%+Ta(WmD|$WIIv35jTJ0Tt-RvND*znUE##D!QqlU)Hod1cHx1F|yep5<L
zOZ3_m&qe10qU~^8u5{FrlYL}=t8ip9JS&vB7uMmyZ1KrF(191~iIlIBKHoj#<lryt
zX(t^~sM*SMAzy1Z<=ES3jR4TdQh&P;(fIWWF{n39Hra99&T3~74oMpsp2cz=HnY^M
zdF2_w+Ah*y9Uv=Py3Dir^Q3L65&4r7qQ$FJoW!i&?MX7Cb4KVY&IKdC@)Xi^QYC^d
zGhrk+t#e>52TTx_s7)8D6VB%(&9_RV7;s@B6{fBFY^~+y??L66oPipooRzs(yYOqt
z5lPY-9P}*+kOjiIX{;h`(OgR@5@gImXcbi)8$_{maU#M*HCwK+%d>zpq9>sMez_G7
z&tii{WMv6JGY_1$!<gftE(fRM{FLlCYytiMZ*Jg~RWu?pz@$!`toCUQ33%#QZ@qlh
zjklbJqfVE*af*{2p%{p&y8o}Kv=BExF%T$4#nX4w&U|P5UUDQ&Yt{Nm=lSoxX<AW>
zvXw-ZrzM!6jrCuq)+(<vMJ?v}nL~z!`E<m40Mxu)o5_q?AqdA3jVNW2m8Z))5Gc3|
z>vs2Y1wE0q;smg>;jD;{=rjO+DTAJ~!jmB*Ln54+CIlbz^Rj48P;}dTn5q7%*~Djr
zy@cy(I4)Cs#aG<SHRJRZ`dwb5AkR4xDu~W+GUO!NAS$wYf)mYvmJ?b{Icv&_S*@Tn
zrOsANm0;cp8|2plF8wCtG8v-xc}CG)ksV^uVurug9)B;}W0<XpUAR@Kt`AdKX2{$O
zc`1lIC4nPv$d>ke(sTCKniL1D-=@|Z<0RKE84X{lp`alvTEn$Di&0zE9?zm@WI07D
z(D1ehf-X;cU_&y<Aj0X^pHYDxPBRvD4VwZx)UTu#V>(_2W<U1N<<JaD#Ky<sRd6Mb
z1Q%rT4FBC2S+8XRosp4h0WH3ngId=tGh^+;=@cbDdQmo>EL4t*WVG9?x6gM@AV<UV
zGRMzm=9@>%vMTKFDz6nK3Unj4PuV-Im*YI^<(15_=flh<+QG}Kem(;&%Xz@B4NG$y
zS=#f6=O<P$h;w&wk&J{%0--$(!?{4+|1K6JwzaF~7x^b+$GeAv3lt}Z%x~U0Yfb`r
zhLB()8Oq_OA*^qr(up_2JYPzE3eX0k5iH>8$SDqGz_jL{W3**`ny2y`H^feHU=<Sq
zLVTk&<z_gj$9~rF_O*LsmW)RGTQ+=*4r<hHU4J>_ls`v@3(2l_k#<>cSwHEKvMlo}
zMI8Z8eBD-OiY(#-mm5QTMQPQ=&9>qh7B7M`gI!!a{Y-rB>YPB#ZzFk;3+2puu?FWd
zcg2S_&!e6w@XlFAB?kb1rMNkpvddz2dz4@mW|&>LILdI;Pyk83@sU>Tqj~=gzKNXi
zih`}v7TC(lhkZR2(5ye#aN~Vic_w*|$gNPDcg}D%!*E2(GcE0kKX#@i)-q)qIg82J
z>MrW$5>8%iH;8PY+(3LxUl8EDhrVhbi1@X!rTDg+Kx>pAtGE)PEt1AV)wflEpV*uU
zon&#_$~|^{N!FMEw1>@tmFzA2&WzuBPu^6zzU+-7nE}&oSeVhIl@kTfdpSM6xa>I)
z^PD65$%udF2p#f>jJ>Pni53(I*iL?aZ1n&us{lyy*sBqg3~!bIKN}N@EWVCtwR%EZ
z4jmd{!$WY(^nTAdy@R5`vtjx}&lP6Oi|l43Zu7GtU)01$WiaXKPamzTX*I~0dpLS+
zYj+@ZPq)giY%%f*UitLc3Vr?{zXS_uv)1IPwE`w2a2@w|a=V~I6*1K4iA3sIj=M`O
z*%cHhTjek?Q%#7KnR8(jzLV9!d^;q?kib9|8701v2CEiKE&?-dGi*pkT1Bx-=2lXd
z!r7kpHW6~J4RtW6cswdO(D^HNxy*-uYZOJ%caf51IxK_ab${h!@oO;K?d#Ch=Myz1
zb_m+$q#*Jz8B4exQ#4uY++E1w{z>IODCcr&9m*$-7A{z5mhXz(x|LsKqNrS3pisDG
z*JeBcAHC6;6eDu0dLX;qM93OSxkz+me1FsP^!M)e;X$j6Z74vXTMQ)ov{>0ujkz-G
zf4hO>9l0#~Nb!7aNs99qGTd3qW3XfEN$&@H)Nvc)wSvc6;%Z$Pds7}FzZ!fClGj@@
zw9k5sy}Yh|l39(esquFf)OuO7#_tS^F+v>t_b$bTwYcjN5Yyp%h5zB>t+p=;nqxO+
zxKzR5N}cg!P50D>W$ooKO5fkX3pSYbX76Xsjqlxd26;Dw>kHha%AMOcrw<O~JX^4B
zy}C%4*tPb|#e0SL%u$jI20!&0^Kr!gKpj#%ZqfNCwcAzW8{>B0?o=72WIsMval)%h
z2vV1=Ba+%waDf9>pZ8dh&9v>jQ33ut-nH-araCa<|CUFx2g2bx(!OG3-9IChpYdrx
z_Ig`mg;a}so_+G*@_!8U4i0)wtA!-YhWgfC6r@ic&P^rcE)d+CDUip%3xDgBl!T}R
zNae#*U+6@|ngq04qA$9piRzUfvx=lo56);CTCLsGI~o5y<r2}~0$s$8j+l~lE#Zvb
zUwG~-i^A{aFU#3zy=DMT-;j2-u&ros=fBnRwkAD2X?OLiT1pjMh{%IHLy4&6=lt5A
z)#R6n1S`pelI|bRsI0w5D>ZD6S4q&?rxa_nO7!U{_u>hu=hd}i-ECW28xr#O=!z-T
zc9y*OC#9FeV-hw7MW?G2L}HXlicXI4V&=c|-|wJG3X)sck)N7E|E#eJKDUd9R+k(w
zhC<NU7D8*LGa+%@DEBd2kGNc2kxA<Fj%el3uK18GF_6((R$S^?$4mBa*y@*J(KoZZ
z=^&>Ej#FAe4)K8zBoJv=<)-v-cZvV}70Pv5XuOGyu{zf|@awUZk8wz)T%FeI!HfjE
z{SBOsZgtSFKDCVFZF`HzCuE=2H*BKA<*fz9MWpMsKxoek{+6eOXS|rrio}KYC(>dX
z`7Bobx2{Y%|J~DlcZy`QS}d3Tq#}ZzcH+U8#e!BzU&>$P;R)Ww-|p5@H0y8XRkSk+
zyJ6dNB_UtN9qwwIgLtxXw)ui^>bLw#nKdidFekD%_-tMmlJ($vXy4}RJm;q`_u$$r
zl1uGeMxiQMj)^k&qZ_514Jc1;agTcusKA?eAEFh!Ct$vypGxxJ+`meudK`|mICPiH
zdFurtDwWfJ7UsGZ^QPVL0>6-N1}=(e#h1spqmuHR$xOuHSzwnc>*7tz_>p}z{5GmG
zi!+JEqONgu^Ez(+P@SoC%O!)-CS5XLQA8C+DJp{2oMDfvXL<mH55G|Ud?)esCOJ=I
zCM=M>21X`rQyn~8o9AejG;6s+lU8tL*SC42A<E7$P=UO~@MdBc_s-x#C|sXjQe9h>
zlf<Ieo2eVmmlkHOD)TUPqFTX5`}=d#sD$mQZ|NR6?~@l4i;H%@yMMel6ViVrsvtdY
zX$#Mb$>2|ucNSJpj<T0J7+t*wrE6^A(X2^wnl41w10}O#6Z&+nJ$PdvT07zb=jYEa
ztkX5m`4XIJYXR>q6k3J&m(ZXTBEOcF2BVAW<AL(2O8HUK@HJHFVeibumkV>K*Ab=G
zw(*umGo1N$D|HAZj?UQTMucC#^c8`C^bx4nRXT^1-_xI#XB4nurFYd9y}Ke!djVFY
z^@7w#!;R7X8yUeje{0M|X_QAB4K?_WWStsO)!YBCu9oa-^vP+<OsR(ERrf8drH4mX
zArE`HgCpj`!D3DJOCq0|gpZ<uH3p9O+&(er{Px>-Tyhmk$izHEtq$ePPV?XGb~}1N
z0+00un0Ph+x(K~NgI%cNw6r#7U7LI0*c$rW?Z=Cxp4NuA)u574RMp1Jgk^8CsgoIt
z^G#_mQ~Tmw5MTju)>c5||Kfq4_7q%i-6>vv;qS=sYQs2rlQZ3Q{Z_z%N&(l4$`3{k
zPMyCJ9+K}uj7XT*8S3g1r^=^=Kkg~|@_@Lgx?j4UI$zyBK-WGK<nrU?;L>DKZv6Lu
z;=>H*L@%$T+ydR~ZGJVmE%2H@+@9VF9^7Vm1-WlbXiV*%q$S^YZ-O34%7|EUy7Yxl
zJg^){diI>ITgOxX8+)WBq^$1x6C@per6<{<PP2YZ^I2&Odr^!x1Q@bk`3GK)iFR@n
zsP2dNTUf7r-`bSaMOX!28|Vvg8BMu$pze$N_IcMbukDvOB5%dbj$IDnUWZR6wh^RE
zY->JqD4wI~@N=!e=|~#O*|eR^9o-+p=WDy=H&oM~`k$HF`F@2{FEyB!(7PhHcUZ-P
zgfcI^cRSqi(X<wA8YX$Ghi--V7ao`dvQjGFZ{(X08#~st=K`uFrPH6aURwsg2$i!q
zF(SBd>fqE)#l`Vs=yV~AsQpEIhudFe?5nMOt#s$Y8#KgQd#qpMDvI98E7_qvJBxl5
z)ut~}#-lYgW9N$!e0QUd;3}mHGW&?HsMe*;N_Z%4d2p){h9HooY*aFhH+N@jeKYx5
zwuG;GOU_r_ur7+^u_vjIuN(9PHc8{|4K1FLUJ2JuBjDSfXZ~3w;Xixc9x%PvG^WV?
z5%wKxespS$(y0EW`rT_6Z<7#>@vp|kqa`-ntuD}H%t7q4KH~rh)o871tO>QPCZ!OU
z!<H*eLRHssAWYMUNn_gcrK_J8@_5@*a*#5kCn^49c&g!>JkzROodwfrV6(7kSoDlK
z)<U!*Tyaae%&ODcw6_oZ&vsI}DRnAQE>~TjWB0(P6|z5~vk>`I{kdN_OWc}Qv50Y3
zyT4o=xBXnbT<Y1?$p4R`^YCZ0?f?FLx7BJ<Vil!U>`m>}BKF=%?7c#9p=$IhHDb>Y
zv-V1iAf(n+I})^{R*cq&wn6WTaiRKqJb%M+zK+*<e7~R1`^}ciBXf~)!8sJ`UxvO&
z#`LF(^aRDWy$0GlO|7?1(Cvsg=ewH4g*u=O`V^5k7wt$nig&qrc(bGM{dmm#S$n{f
zgf~;pP~TTGH*|4FryR<z+&~(>RTPM-#6A$#(5P0j{`6_tMz5yXThj3l(Fm6I_{c8`
z6Md98(`myi;V3!lL+l3L-Rm_9_2h3~yUA@o_K5j7f>pm03Pyc!DEl?pU%c!b8#t)M
z0n!P2s%w0q6Z!qHBJiYH7{dfnEhc(Q3Tte%Z$xRNDmiHT(>-AoVzhL%S`Ai}wo$*O
zj5b=3^HFFJ4pj+hE<<5~`H$44kXVCk-V?QQ)T7hWl5CfP=cbSY!Yyrc#|i!RV-G3}
zYLgV44F!|CDXk*1(f4=;Mc;J;W6!{!{U&4@U?DbMipQA>^E`=O|ITUYW!VJVVznu;
zr@P5q7YV%fun?N8eJ)j3HLK-&M9kUhZnp3TMV1qWrjiuzzo1C&Cm1&4v9F+N8`?-U
zPJ4|bg-0<q*#q&BaK*6mawXC7mDdBwPu;YB13B40-e#wS08(vjAza(!Xg!!2cCWA2
zt3MCQ!LHm@-GO7ek%y2vP*NMRR(Jz{*u6k1O5qc`_36WIt~S-s>6C^mxpvWBp2vAF
zlbP2`@6t}+T(lYaJB##KEwqYhCQd)inda4Fj=p-WX;d9ygJESkDEY>$IJpPSs0d|p
zZWOJ%EXyaWSH(6)k8$VD<<{_<Plx`awaluDS+_{Id=XS#Db4KgkD<`yyL>pyLHXI|
z-;Fn+IKff_MfrbpRd}A|H>?fhJdSyCUs<#(TzOJ*U)+75;+PiLmArJIK@b^;^~kn?
z;bG4CN*{@y#NH*!ZtQUAI}Bfxyycn|GN<9SX!=LQq>0gNeA=9FH##E?_+dw|JeW@!
zMR7TQ@bKjfA?1hTP56hLTl%kYhOw!-SfSQ4%JEJ!wAs5n2FZrOtnh`Q<e!Hty$Acv
zL>s1YF?$9iM#jEs_I5<%Uvz0>Mhqh(jt-X8PYD)53k!&9*zCG_#&th`rBy*)?u+4t
zvyGnEkgVnPs@V8FcwFL|E^^~tm^5cM`nB`PTjo#7Swehhon#K>FVnE6gF1qMz>|vs
z-&@U*B7-J*R4=|DF=weKAD>VX4#$n4mn6W2X#eO)2ZeZ$W8O`(Syco|VXTA6RbE3u
zG7PKxE=L=jlQ&G-^mjcf`Xyc+e4`l{vTZK0CEF_<p7_u7sHmWUT@K5;6R33K-_0|$
z`nTGi8U<qQ8smY5-DfrmgtnJEv<%Wl(_H7pjz(1k=YtV+HS#ssiUi1RXvk@}Gl<Vp
zD}DsGbKY}Te8pB0cjd+4N_d|m#@|&0z)L~4{416Qmn&k`{aL5;uER%`41TAoh~Pn6
zvYDd*NoDITu-kcP&ig-XtC!PP10jc}Pe2W*%jF8Pt8qI>QvBDX*Kvwn@hc+8t_Wk#
z$q}8htE?=A=jc%SzBzU;_vcM^S}>4@^JP_6xNTP-)u9xD?k&E_<xg<`T%&wF;!*Nj
z8|shqY<FBI8qCeWA!*KKz`&?B`yS=yYIz%YNZs-XWDGo+c!sZ)fRV<wy#oxA4eDup
z_Ue~Tc8Qyt(Pn{C)*S0!dp45a5~bO;Uu5mQt?r;$40|Mom-prD#SE00z4k|46qEST
zPTe$yc41wY#OQDEV@S;DW}kX|c&?p$4TZ!`0|?)y54DU;jktR)m5-=h%*juw*~Llc
zgZca9B=obMo%`>z45i3q$B95N(psO!QQ?qaZrJ+8+^a(-LT66{-NC>`AJ<K{NTw^m
z4~!R<^}rKz|FWIJpUpHEb}C4iEj-2zFQQ((x#8i0z-kr2XNmpG6|W?3mq!JNL!z(z
z1id$kI5X|4!PyU-R2&uJ*vmNs&-|1D5f+0bIX*NP56LUoxOGw)!&w~}11KkUi+!`w
z8{FvjLKt7X*iiK+YHik*4v!$^&t~TF=hU=D7YUBvyKnW|h0+LIfQeveFuaUbGHVe8
zb>J36^HgIVXy+MWxe;e$T0|85qH}3zOl#!D2zWo_(;GzQ4#<L_@Za=KLvaZ<GBQUp
z&H3G6cp55Yep#qFz0IbT8nmQsZZR6Re@cdNt6AXlgY;aAPxgLrUG4$im0|i$uD$_&
zK3wH761iDA*1$Pvj;D+e^O6@1;kO1o+*BN!_Ls~zzq**Bq->3*lRzg_^8Y@}v@3iu
zhIbM!9!kG4BZw+rQXE^;?Q21gP1d_ux5@^R>2=%Zj)S}mbo#kyZG<~J$Kn6{)(Kko
zYB+`hrH(D$+@W!>%aGbmG*>eYs_>!pT)(##Fh5$f%y^K+JbpK_P&@ZLlE$1TC}S;Z
z!m}^Fv5*_Qlo%Pbnhe|g5c-AnSvu}M6K=gV83Z5CXyIC8*=>{M*_H^yljaExDA?pp
z$<2YbuX+0<EkRpUJ17Sp0B7}q@4}DSAeTS1D=}yMvNJe(G;B&RF4dE#jod#}IV$XI
z^85~a-UQ_{!p{-U{`49Bn`5fYZuHCK(`H$y)RE)D(39>vbW4v;h4YGcbzM-#!ap(@
zJ;JCIwye$R9|7}rY_yzlRV5+RcyqA9BbhWrm&+wdHB|8!nUcZC-U2K4aeHQW_cP$3
zY9Sj%Z8pDA2AxSE>nTsQ#HTqg?TCpgWQE@K>*gLXqKft_*0*pnAS$?HLCtkgAt`^c
z$Rz+>-O)^orso}_8FZ9dX;Sfc&!k{pfw4=EjpGwcyULUMhA*&A+#8aq5O46@MzCq(
z?#SF0&sGAlqx`k#cXk6fTldh+`|FbN#u!EP=V(*gqRVHlR)+9xo?3X8v+4NK-Y}4w
zbSrZs^_{m<p`D_%gFaefXdWxhGxmr;Cq4(&4rWfAAr3qHsdzuwlE|_@?bPv!ANEyf
zrufV(Ab`#Zr_`r*){t^heV(MA`hZiaS7v9CGSEaQR|K?sL#C}`y=Iua^Hz4xp-b2b
z>=K8Oyrl`ry7_S$GZd2+uZZF8zX^KZ5JKP8!_@FQ<@)QG{=xyjhKLPAqMxe5U*Wu+
z4@e~E8klz8mFoLYHq5(V9=wd~bVhz7LZl1S43!ma^d%$dR2CiwWjJ;k3X6)0VRzaU
z+Vqjo&@c5M*1dEIl1|jIVc+*$2+3K~>Sn4HIrPt-I1Y(gD`J79;5G{FU|{ToGsW((
zxe+1<5sqLI?#vf?PZMZ)mVUr)DA=d?Ae`WcsC6!Uj4(G^<0&tZg`-2i2HKzlNK-(O
zQ7y#Jml59Lo1yx4G7c~iZqA60Dij8~TR);GC9$a>P{r)n?ykpPaanP8%e4KdVdLe_
zVbS}YDc0hRb-s{cTO9G9>EcYg9$bLoGL~0eKY}#Lsg!a4-bbdvi^_|M3(AE17s)#8
z5uKDcd%><V0Tax352cu}9O?nykM1e--t7UohN=eIfb8{Zmu1Ec!|z8rF$H;a<vz~#
zTmKMb6&J2$T8dZETS<ckEj%q*9f5IvT9V56WcZ*SR-xCK8;HU_>8~S5O>w($Q;jyk
znOx)QaC^niYCB8=UBD!72S$SQ3V^}m2cQR7Y^AV1t1%{&^>h-s<I)>N)Om$e>EcWX
zl8i2&t-B{Z?AAYK^W1AhWf1SBE^5f73KS$W*e)y9PxvI8f<xLOaY+X|@q@cG9Y1Ee
ze1{<Y&Cz3Sk&81=ZyPk$VIwo$k4__nWweBEgFFz}iVLnBc*MqfEfw}Sr%92kkD*tE
z6FVEaQjS>%XKxrPpU&9x$W*)y^U>QCQ&BO0o*#wCgXR-ccSC`H3*qhM3J6Iztja*3
z0l)iPS!)LtHNH-igo<eQ8t6}U%Z8Uc;zoa-@}mIE1_0@JeV~)i-Cb*Y8*NFZ-qAB^
zi8aTnlI$MiQa{B|iYo<-bfI{@Y4?n=VK52r7dbnKDkZVI7O1=x#bvO|bu=uAM0>8v
zJ5%q_oXft+e~{eqD$EH@Zx^^Ixx1EA3)NaUH~oSLwI?tV7%6sANPA$CcuQ8J&p5g~
z^DTFbS4yVPxX_u6#8P8<FbIzUx6qMNHyY$y!lzt)N;FHD#$Q+jsXc8mU5`=a@I)Zy
z$i;T+#A3P~1hFFmX9$_Y5m4!5eFD<`iPvu;hrxKDr6*}c`CYfWlHxI;gtZP)J7@a=
zq`E{Tzm#wM3StPKEzQ3V6=)%7CywCcZ~PnmsyJ1@{x#@Lh^sWqcP{2-<#%5hu!s!5
zu8XU2eoo-6#X?mHE4AZA$h?Fj{b`AfG#EM~%0=YpWhJ-!%5-9jUDtw@oGF~yU2k(b
zF**g^_f;7p><ciP6tBjHnP<OlXDR=hGPJ~SMD(<EM9QB7z#>s-2%oleZbA3A@44xz
z{11f)i5`FUR-kqZCQNAs=gXkF6|+M8LDJ}cX;h&>#(gtu$Hgu3cF3NXZNhDu`_uT0
zyJ!5!7eUd7>=z<|sI=W$?|X<yY0`Qf;!jQZ`e~mMuCk!Fgr1~lP~GrnHk=*6S;y`W
zF6>%8z_`kO(Gs|2-q2B?RuokfzBCf+&rOnylAM3cu?@Zu+-TZAetEGbEk$<H8Z`GF
z@l-mLa|!z7m{_`8>D^*>Qra<l>^omCZ4p_1v5!k{sMe-%l`iAi*hD@zj}$!JX?aC*
zO7dQ2quqh9Rd%i?U#-rnE{Mt-nCl-KL2{U$cn^uxHhrJGidbs@>hmV%zxF#NYWcs%
zIY*THmL{f}gAL!el5Mqd88g#^=8;s1jcHh8;!zOKhoG+KujcbYxg^k%w%rsjp7Wbn
zkdjBfuTZPu)85OymTxt)gAv-#^VCimyp@wnWcuHq&c6fcXF;oAuPqSJEig-uR`F}h
z_=q!9YCE@cPSbWi1@P;U(Ua$AMvscw!m=zoD`qSkCeShsm3`+Sqia=)9g3X4N(DK3
zvg`E;-(dhIn6NE;Zk_y~S8~?eBuNsEwYQ_8lTFW@a3Xs<)+_!VGXAB^{;`!S(t$>b
z<^96t)fH^ZyYjzhnlssLv>P0Ju3x*9X82R#xx3UBw*YuWkyIq)?+o3Vgw*$+yLdxI
zyVio|6+OzW<^~c!glBYH*G@)T_CD!i-%U^YxKMNA>9U2o38du2otdu_{cSOS-d!!~
zx@~9P6eor^>{S+;b~Ti?<m{cm&X{Pk8#<}jci+MO^?R|NG>6%W!e#4Ik~?=xb@dJ1
zY8l|Cj%}<izx9=Q$u)zM?1T5@%0FX4>(4qgLFW2*Vq{}OJC!6eFfTWnB0g}{?9g;x
zNBkN3=Xa{8a>nW7NiK<wen*YfER_4KgXs*<XJ}$<zJkWimg$#4kEfrlp26LwYln+b
z3L>WJd!*0)V9P(R12^5L?G3K!7b1VU+Lj8Ej3W+ZvCmn4E{yq_io!nE?mWL(a<v=`
zJ6g*OboKJ%M-38$IuxB>7T~6P`}OK>h|zyx`#;=FmgIiASCb8PhGwX!guLzkiM*mI
z_|lq!OR(bq`G$Q<!1!RrHU^p&LPQlfw8&mV1&CZ$$6i2K{*a}=Txc)wa0RbmSn>9G
z4Zw(%?}fRKGE*d!GpkiSxbVg3<<?L=y@7F*;C|3N^0l11z-+|-x_y+>D?ySkeD{i%
z*GZ8O;&s$KsySm0lQDyS(zMUvd?;=H_*?Y9E4g3+cVDmOD)U#(iF)H(dXmS`u)?}2
zz6AgF9Fd;E>xZK9MrbEQ_Vt@;Kd(-z+D=#m;Yw8P)9ZJ*g8Me~+t2AoDenx=SH3rQ
zM-k5cm~?I1Xa>Bj(9c>Do3#Of30Ah9Ev-a*q$yVwZYdMC?U)b}OH`;9apZHXL7}sS
zK;gEsdg|Yxxwx@|mb~1XvOR-wEuh+R?Sl*AV!XlTJaoeGJ&`!7z4BmVgBqh~&*rR3
z-nhN1K*wSX-!XC}5M0_s&Sug+em&^nc$o*Pd2Ut!yrgM&>AHUUJZ*U2EWXO*CwSDV
zTCKt(jc4zlLB!Ljhb|m}KBZIhO4JLDX0FNiRmSKc98%X^jvP-ZE4j4c*=9e++dPk7
zALF(GC=&RWsOm|hIAM(Jh?u7GYOKQ5=KF|fy@6G|693p)S02dD%_*|4r_Qb3^Sq#?
z>kdiITHxL)_u5f}JJnaBNG1>Y-^DVWXsk7A<MahuK}|HrkcWaD->q$|3R#6VW~O*8
z^vyb!@5E5)KOgKk?w+Q_A-GITB{Qy>UD%iLjz1DjB8;O7EakJY7fMoWn;%chbotJK
zrxn^(8&?JOlNRRoGUs9xa|?FMwcH~xeDbM8pa(N#PkbJQk+`@%iW{pI(FG@V_sJL6
zMtt(xk$NcVkF*CB7t~Q3L&g-$icsJkIW)<}U+YHd*H{P-D$kS27Q*v9>5g{V%sHE`
zGEV-tXhb!S8h|Ce`~jU`_Pxo*D!s_znzsU<eK=q$Na77TQ1xS;jW5}XlXuDr&MOxL
z4TqfUP`bJ@T5<;-0r0@OP@4eBZEnG2Jz{VNAP(V}iIoPQm#|-^vC*Vx^IPp~`^{eU
z<Z!8-9VV|9T`~T$zTENO@tkOrh<{1WhYE(Q@m3;VTGvb^o9w{p|H5W;ZaQ&g^1qtl
z?3~dlR1<;$Z-D*GA}tU;u2y#X^ZpTtVi2Ey>>e?bB?|~o%Bb08lAGN@7-Q^%goMgQ
z&sL);V!KigH9Dx?*rt(oh4ie{HJb;M7sbCfB#ya@9O$QjHa+A%7j~!Pc*Ms)<pg3V
z+c$~38zF&Ev3(ta17TKPd1<r3vH`HOUdXE--+fnDp~!#??jx4phU_FX2b+tzciyy0
ztXm&PY7s7q$FcZhFYgzf<7!2pdaFzOgodhB$CG&T9I(nQDjdW@kHP7MuID`p^OB?Q
zMGO%V?b$M%)gf(F1Fucehs=O2n3d}nXkULe#q3CikSU#RmaoIRWz~uOh2)Ep{XPl2
zku}_Tc^2YE&Jp6(x8ojNgkg!Vh%aXn?4`y>T<n+LorhjXer7kK<qk06c|EEWkKRY)
z)7#iHl_h>ggszOE6Z~6H7U%L8wB2!b>LimpMXumMQfaCS|FiT3bqSLbW-y(@I9m4p
zsbE|%+zu_fPFlXh-!q|*OOo2Vlg%kzYwHGFqpvdk2v;H@;S41wO4&}ky-Q4yr{($X
zjf3$&HKVDniPy}PCvqtAl+J?4yPoQIxsP!!7Mv8J!UD9INQav9I(>fdw=>Q^v55GY
zU$uu?-d8v^Enn{tLz`0YlkmIcj{pIT7H<>M@bxcinI^|_F?_~vqiV-KhRU#g<Ea|#
z<{@#svb}kgQa)c7&zO;^$e0;Nb2CF@k?bHN2onc7+|}E8sF;9V&NeSy%8gFUcMIw9
z{|1HeSLc0pQXx*F%D2d^3=c@fDSj<p75TCW;aK)f*P&u|1LYjEBkoi)S>FxR4ObL@
z9Ol<@=@k3$;K{~XE5FuCO>Qh_*VF5VE{P-HV#lGlh)~<k#ZV>9fNeKQu*y2u;Bi_R
zL&EBXu&E3m?`p<E&W#v&+n&k6A6MSxdqFWfd@!LIm`J;=igO{pP(nDm-o$Yj38Jz3
zXQX&N8&3MZNp;F6XZ^kVjCO%tMN>RAAe)R_n0H9&(i0`S`4x}IxWe&m#UroxMK4>x
zFtX#~vS>e1#h@oW(Q3Ahw$E$jBhyx(z&wmdRob0j3NMXnLcflD7)C^<JQxADXp{}b
zProGc6Q{UzD-<~*HAe=A3@yAM5qowoI0r@o`4)~YsueCTLT7d>74P7kmrTrTt`~wH
zV}N@bm>tc?hwGtw#!q0-_~S2wg}ch?1!B8X4_49@axM!0x>8_tx6SVAo9vQs2#!<b
z2UF0b<Yp5_v&XR!m1#W$y4qbv6G7gqCBrU2M}3_qH+3jKv7(=09{B(LR6kyU)MkpM
z_+=XtHmR3$%=n8!?Hg{$k5{S9OOp<!X2!WSB+_`SOlu{<dQ}UF^Ad`~EJHBgM&{@;
z#)Y#ZrVohaSBR%&_>!;!k@}9B<!tY(f?g6|enU&R+q8@8O>wlnAw)@G$+28PmfcU#
zA-*?5H_ne=tG^d0RX)a;SQU(i2S~77^e`wVM7s#t9Y5o`j$(PO5b)DKZ(Jc;l|k0_
z^4a-1u_skO3l^*ELQ<ACcz#tv%bX_{d0TA`1=tmaq6>KPOE?0~|02?C*x7|);th=E
z-+#PIn`PGR!swWNYII-ur!@R+!Z->CU&@(wa+4BcAni@QbzOO1apunW);o}Pu>*gt
zcbNxFUZF$(Tn!9N3chFiE=g0AiMPu`cW`UdZ7AVA4r<Eie{;tRP1wPuFtuMr3?Ipd
z7Ax1OhAMXu0o@^G8iP%2sP=a^>^ub|+2%Zg>-0C`4A#Py8_8to%bgFa-OyJqifXPj
z7GJIrKV4<e#%E?-)bZkF7p|fv-J!RSZBjfNog+g?swFK07Q<?ywjE$%3P(n(lvX|V
zLxTfir32dB*eGuUxF)2bS}QF0M?$^ys}?EVoIKTHd?r7FIIEl5(QPkD!C!TC^x4iv
zkLlKV98RcL-wx2PQfghA3lUx(@Q@oNDpGdb%tMm%_v&D$T}7EdwnDiuKAVX7h<z$s
z!%`s8G4rhZo?zPQtxJ_@HxE5K<~>X%*P_Y-Ch8_p5X&hx;J3>o+)*|gF%7)N-Xm7N
znrk(!5;^jpsR&KnsO+_8UqZ{c@@RAziHS5nFvy0C#K<FCXpqd!XQ9%d_5|_YZ<^=g
zl*tR+cf5V}_RP}u59pG=P&x(GijxQ-)Tm~%2{s>K9+hIczLyt2QLim|JE27Ht(P27
z$lN0~*}Xu@RCWtRGb|VKHn_AmWxuZ1RZCvw=rDmShw}T>R1VMHv`HHnY?g4}9e4C7
zc|?Zyd$Wb8U>^6`gPhB2>ROIM$55))!e@rSupfS4oc}&;%8<X*0VGtOa5P`ZDJgxn
zjCfKzuUg(eb}CH)r>6%!^2hk|h3wod{cPCbap~0-RED6K>pvpgVtPdHbw}1fbq(yi
z1J8gs5kQ47T|1(Pqo0eNY+n}h0>d~pq@){w#}O#Ma9{!L!taEE-&cZt{~1czM=iB3
zMDr}+LI7lfLa2k8RYQw+o90WAV@J&Pjkf8OXuhf)6G+oFRglpr>e`8x_Ngok%osXf
zcDPU}TXmr%o<$0~gv;9TO6XJk7TyDMC>-a|b`hs>g&rkLdL7NV<OHW()t_=8bMK@r
zW}|b;`uc^27)!Q^c#qO)Ur|aE=hp+EeolP8p>eW6@YTa7skUPCtPA<KOy)K+9j5v!
zXLmRMaTc2E@!yJ6E%m7&vu``{z8vE-9XCH$VvBtApJq#?h>48aoOR1<ODQof4yItl
z<^__mbNOaGfEgCW5q|pv4q8s`ix?KO?eG4aTV0hbHuRB2^a}NqYZr7)TDp2SNA19G
zKGwkDu#_&(<_26uH%B4wwElpM#mQb?hVDol<C9TF%LXWk^UrLD=v$@4h~@lqw1C?0
z7x3dT=65=D?+Q=n#*%N?NIqn2uTXXjxxcl|^g-D~O>R3YA1?XfKcP-6IxsZb`Oh%&
z`0vtR)p}3m=9w3sjzGK(@5%fgi>T@ah&?)(e%C*f>5#FxXC%`@_Iz*S2R;Ac<>cJW
zF~!D(8Qz(X4m^K>`t?(7_xH@!M=T;|8Y+8N1%<bv>mg|&l720<&VL*p)76UzO}sAR
zx#-b8fSqQ7ksi{Muhu)Yo-GD_@HaSrpp9u@y+{~S%z9)uAF>4Ed^v;ef$QoIr9RE_
z_XTtqtCPD>BcTUDQajb0oQl=;HaMQc5+6gef{#tnU&qyurPBT4Yxb>*e$cmFA|bM0
zCU1o-tFTVX))F#dQkabxlL7@tML_Ls)AVc$;+Sb`qpzP!FBDm}LM{!Z%!W_J;ZQ$)
z&kT|pzDF0h?oIwJYi4MIN2+sKM%ew;C)^VAn9%Lqsy#|1qd6|f2Px~tK0@-Of@2yt
zqt>?9w3YFjd40EUrbeP_rJSlaz$n>w)1bS?-4IV$UpWo~D~}0|Ep`${X4C;WCIahh
z=F~Hl<dEgx8!{BR7G3;Afo2ip_f$hBJdfo@ZFxyJ&TD~;vgvP~E}HO_5~N541g|Hh
z-Ze9JDT&9WdG<G#ylH9G1Zv~s1mybd5V^5P95V8jxt}tEWH8wz+yB<vMmQV0j113D
zKk^f<{T>yF=IfPdUs`&36snA_+MQx^B$Y&CzVZ>?N9-a98FQOwkv_)xw)r-zez}E>
zxdk}+J+n8$*|)c?YD)+c{fn6M<yYKso{B*Rtt#f;oPIt=_Y}X(KQ2<C==upGoJthg
z>s^C!F}93Pj!L;yj15gBnuV&PX~OiKLTwm`Hn(+*q`0gJGp2TTPl+AY3SHtb;&Dup
zg4BL$T;!Nr)6|c^GxoA6uG4@yGZ^ncItxX!KnxhJHp~8r!d7f3N%fcL-y5SFC{Yp}
zXbX-EZ7c>2ym%2(Q|beHMckNN*ny!F1XuKHts-J+b&h%J-QM}{Gll-6wc8tEC<qk9
zjGK8T4k)%6fRx7x0SE$ns3)A*->!lFv>FHlqz4(Mo^kq+$!A1VepKdbGdo@w9A#-P
zqj&(}n>_j-mVi;q@pIjjQ#q<N;bzLb=jS3}D&^c@P@wDdpz#owz(??kG4c9UmMJQ@
zu1J1A(2yx&yyGS0?!)@&@zruN*FlLuF4tEs<4(7A^g*9fY^iwdb49!mDUg$1;#Pue
zQ>`=!Bq(W=W#cCuFz>0*&b0z@{3r-h`K8E1Hh|4R<LdDW{h&VSf!&_|mVKy9e*42O
zZzKBxvu--lhR+RoN3;&qUPOIBKEmZJ#`V`_FVrDHGPT>F<QU*;N!EZ(d1)Ku047JJ
zu$rNXID<H(kT|vT@7jGk#Y2jzWm&DT(0X3ME4rC)a%RBP1&|`)N)~aLj}9z#lJK&*
zidtVxAm^*KZ-Ny^<0>fPfz<96PS5B`(~1-{hCV;b`-y7_Ynbe(Q9JdR*+nI4Z`ZAC
zryD{BZ*a2~;mhIVt6z`!%s8U`o7T(&gn5K)7lC@wJM4BcYtTTYW%+nQ6`7IhC+y4$
zo8Wh)MWcNbm@6Dknv)hbugWH5oyC2Ih<*wU;Q5wx$zMCJmLb=LIEC~&Jdc)0eQ<<M
zrqNu#mSdO5%#ypOcSy_k9C7?@$WMW<w~ifh<Uw~{R^osLp#T~U=0gyf(~0!tnW9z;
zvV9b2+`WF$KK&SIg7YtQprc+@_e{`5bG7>(dFP$pc+>7p+fzw%Dlt6f76G;&Wh7Vf
zh58rO!)Y;mdFJLpA0a+cj$i66x2Ot_m3y9ZKp^ZgJ(y$HSb<vMox-#I7wTdVg_=Y>
zjSWr%4Zhac!RpTjJ9b!8UAUUskG`xqz*8uEks%c(qD?VWVJo1)>jZ|#k_3HO8Iu8I
z))UMHo#u;q(`3hvG9V(8G$?2W2fzd_MjE(rtGPAkruJ9QU`64|J?J1+I)NT>FCz;1
za1|G0czJjC)&^@+jAEVLf^LHmhF+(EV_ba6#q7~+zi4Oe6Xg^=Ni+cmgY<w;|IDbB
zotZ>5`Lm&_4%6Tlm3+~$`upy7EgFB9AmJJ0i%CQYk3e-}<j=_by~>kij&~=24-`#N
zt(FE#pL8VRHL4>rt>;m8EwI~W=gUTRPfSRYnV0-G_P3@%KfcqA&m3H^+F}^vkqgML
zoAs&rwQq01`Rx|CC8pcg|9eZf^lt=(a&Nk^o2!YXw#)bU^z_SwnDv1F?C7LGGk(&A
zI1zqM!!<j&{$AacH(D5Z>Slc6&mH0PSeAKlsI+*wgnk=<b!qxeN^T+oNNLio@jVP$
z!`G!=eETa79vJD_(J5cUw2!%zmw7lFwTj)G2g&!7pax->64{_$$Bejc)<r<zBQK<2
zaCb=`f0J|pD6krJq*vrpcs6=k+#Ff3wn|ThuKIfhexBO^WE-Z9xPCqQI%g@u;C6sQ
zCV=(`0WLbV!p_RC_>~IxkJ_dI3(5i?ZcFsX75pzTgrut?$O^-ITykzddaEd&^lvSl
zY`~9w=bD~g-h+30oB);^))gF;VkxRdZBOv@YSrqs#!50+U=&U(h!D$X*#_zu%vHJD
zTdlu%F<NfZ@=m(#ilE1yZ07>=$$cXkQ56=!BG{-O{zWZ4b+7%zsH)XwbuB2bggl|%
zJA-@Y^ox$a|BlrKT3FTx`umCHZGF^r`Hf)`A6`q!9RI<_;^2HdGq33<looIKP*QUE
zE%rfHz~$H7ZO%0OW6vurXUOm6F{9nj!QgVatPJ>TH%I^F^j4otC>Cpbx*8qTG1s3G
zRO2A29h7fGMJ?0ZqnDUEtuM6Z))h1hx;S5o)?Xi@v==gUj*0$#id)@md|0R-rhU~)
z+>7PvzjfOQ5)RFQse2s^6Gn@NiD-8>O<<wz*8D8Q=W2y3&AK4)evA*op*L&NC!KV)
zoXr>998WgNQ&vWd(F`&qvWmCa-2<07yqd7*R2jP#4$hULPQ8V#EnLhAn#yFQ?Fndk
zD4dJrd81dh_%NUMvcbkV4L8Z()6;!IkPn>3C7$W&^At+MWgl@Vnp!Q$@mdX+o}yz#
zVlZ6?87)0h9AO($nT=auU9qFYem1t*{vFqSIhu<Nv-%5VVT#(PrKuc|cL($r^vP}X
z*J>psD(yHh5`TH&S4ZK?-z9ZDR90tzp$lE1X&b51r3>pBp+p?xnde`b_h?JQ!Lm-F
zyy}c*O)Od7HcZBAAxYmp*@r>Ae~SBfUxoYKuk!zkB+5)g>G(xV6dc;|4|YEt@-&G4
z;e9$V(zzZ82A}yEm8UA%4*I^>$5nQ-Y$PrBN%Mf%zY0$Hsa?Ol(sk!UC(d+Lc{a|Z
zSznVUk4NjSBAtVDv0rM);!Q^`ei-|+&+G<QGyPDSj(3?yFKe6OMM~m(PdDJy**`bF
ztDL=y$rK4j#Xr|do=fRh7_Q=C{<yM2l~XFC7A*!jpG_*&MNSD3n*Mv9i}%8CyKJ=!
z%#)Cpa(7KW)h&zX(U1m?eI7>}yt}#y;+r9|ZIsV!1zp!CQV>m*?LS_dQG<u8lU=$3
z_0K`k`lycCE$ZaQx&3ZFj)n`lLUp|FcNqiviUEXcAq|2gA_p9$uJZa^&2UjCLRRnG
zh~Dmv;0v|F`+5~t8wBegUB|8@p<txA9UyI)N5A{4szB{$31%!UFLexU_bp1ffiGEv
z&BOPruLSDg-vM@~sh0NC?t1`#n8;!AMy)nam2buMYca}81t*%;%<V~g<Fi@6a9I&9
zxdaTaAMHGvCyg?ME8D9d?A6GWi*J}^W3_3A-ojwwFW%DOZEOla)u>$7!IWV~QB4sa
z{#Ey|Bj;erYxCFgjps~cEPlOhj>2HuTV2iNt(-=2C+S$KCBC0ilrk_8T0bl(dm_zT
zu8#^QP@{`#?p`o`@C!dIm2%7H4cx21>pDYZpx~9*^P5Dw6!MBTtJxELfeP5MY3V-p
zimKzGJSQy?g6Ih5mhXiojNJLw)IM7~zw=gj1Rz_xct-e#T;~J!<~wy=IESs4tuJ3q
z&!#|gVhRU$)L~p-YOf!hum07$eEgDB)~xes%}H3LJX7ZJV`NTT@dQN4o%mZ!Oj_w+
z3`G23Ua1}<Th;B3EtW!Pjh!th#I9Ff$F6aJNy3+`IT-!AINy{$9WzL%;jq2|_2O6M
z@dv(eEYZX<qBt!^0bOb<^(!jSFHew__&)+=7+CQ1?^c6g!>FW`mju=`YC{7#r}Hy`
z*>*vpqw~lk>{D1LlCb04ysoj_eSRSQ{5c0`KSVtbA|Gg}8(E=_d(o4woij5BYf_vw
z5Bm=5Cx<KnbfbA!TU>Tgt`)%%j=+BVEV{=9zoZj=RhQ4K)b**E#n`G&4WieNI33FO
zqnT9+W997b!e)<hU{-ZKOt=H`^LtAK^X^`S-j7RHQ!|jnwXptMjf!CY&GNVH4K*+*
z3!JjjWnl{B*>aEdS0jj12M74BJvZJpR|BPz9e%f@_Yd_tckw&|GWS(&6so7xE+mK}
zg@)!_c-i27)-ehm1Goq)_qF$)>ot?@kfe@nqt2w*(`wZatNJE^C>f(CX9pZMtv9>3
zgXWeNG{m7xuH=Q~dXV?#rj)az*itvuX!T0$r^KddyWr^<4>_Tq*XybeN9LLn<mp4x
z28y#~_$<XXQmK=x$W(vX&)W21p=w3GJsCo+9kZJ^DCcPxRUQ2?%fscZhE~}&me;Rl
zusXYxDd01kpHD}XLv3XkcE>-P_AI+txo-T|$7b<6C%@uD0pTl+{?#F4%#3v9ien>o
z?QtPI#&%8PbRw8lXyE>d5T9dKKdO3|@O#!`SozGt+~i;Y#$dHMp!P|GKuzM@fz^6T
zwb<d9SJj2=jaxr8hd*lRc%p(vZ%He`KpJC{d)e#N%2kp3oCyI^g@c73hf=4pbEe}{
z9PH^2tV}v{UCnrScE_if-Y$8&BYkY20<05GGs7SSHo&sWZ`Fz>KNsB&&;f|k?vd^T
zi_=rebDoL)ayh}tA|=twNH|^1^C0GG?8&^s$fKzAgtM)UO8t+;-!;S_whMZ90@A(p
zF8?8sw!I&5OS0kocjbP(O1kO=mCnpBRohnnjCA^eX)d8=@xv~^3)g#+7f!kloW})!
zxv5v1QA2XH^zSIBY+U_k2kPQESg_}KgUU_Qe%KIyC`qQ{l$hjTf5e6=o%~uNxrlGK
z%Ynxq$TZo_Kknyh$-413TkI8?_wRzw5jR-3W^d)8k-Twaz!}~f;4YK=vv)eCatxmB
zE<DW}NKxVa`#O5Q#=oZ4IoHF0GY8Kq_suKuWm1*Drz;dE*9>(F%25tV&O!%o9PP=>
zmwytu+4x3iDhWz`d!8U2lpu8si;ku|HOx`i$d|EuBp9En9C14@*DX-AK}q(`g@`@n
z&lpN6<@zWx=s{=uSEuvWF^PHuYn@?8jkMLKm}YNZ!HrZ=<0f@>+0~_F_#CY4I<bGw
zjzehP9h^l~PUAcKdoKq}Q<&%O%uiAqLz|_2=NuB_A)Uni6|UZ6xsc0>SUDBpmzf+X
zm|{cy&w`aKY1@O%q&~R3A(+cSWv(gzjuSJ-c5LV#mHIx-|D{8({OP{Ju!ZzeJz_vS
zJp(6&Oj$mBGyePA-*q#Na&d*)7-XQvH6ro(NQcfBo`d9-n#=8Gn(T#)1TK&-_~#;j
zA!H=+$SIL5HB?+k8N)RBVIvf!^tO#K)*1Pkofkdh=@JDDtXjL%mzB*)hKH;PCxZD`
z>`?a={k@8tpz?8y`u=}GnlEH_t1PCu5{m)a%u=9>zlC-MmSRp*1<%NFeJ`6WiQ!O_
zrSx<wa?D%$xo17tTKo9UmJ5_{S|s&Vg*#78Wv7mx;1uMEeEGeIy}{1cW_o84zNPtX
zpr<FOfb2vN+9GxJa*s8dxWfG{ac}&SIcCkxT6)caOJZnDr`OyEWmH4~Q6KCmOOd71
zGsKfzGI|tJdfi4HMPm03x^NW1!)wp%gic+-m$4%og8ZxO+M|&0ABo}7*6*edYULUl
zefq+V!OjKG^X^zFKMtZ7b~C}(MllV$D-To<q$`#V6?Hscqnd?^L%g~k;zN&|YnQQ6
z1B3@IUbe~+ijP;bH5cUs)|{)n(7M%68+uGQ`?m%>iF>JiFO+MaYSQQD&IUd4vb%z5
zp8gpX&scOft*pS&SoIX6r-ORz6<0<S{_;$3vQx~cBgk9mvpG|#nd6N84ZJ~%{7(%?
zHkYrc>T*(O|M(ZPm7$1CZ}$S>6L{v&CG%#<G@pAOW^W^&K?@YvIf0HX@qf&m1ljiw
ziOVOL<&H$<^_bPtLjru>7u=ogPHZNDlC#1(<z#ei;ej}Dd?F=?kEX=G;?k2L4ZL27
z@qgLp1JKFm_0@j+9GJ~C8dAEEzr0x*l{w$=C~qf>O}c<}unZxpDo`ol>(Y)DT<YyR
z2Tl&}*}M)ke%D%8RBQIB)yo76b<8&`Dd0!qhmQ8+r0|t*fjKI3@vV3A7`~uZS$NFG
zWa15EwEph>PDl>|;Iu^;S}uAu@A<k}=@zymB()Y#DmdS`6#l368!3kME>t3Z@uNUF
zzXpf3i$cL>n;Ud1%JgMj9i_U30))y4Eo1Ap;E)V<wWOJ-q|j-y;1elESjg_1Ig6lI
z!ItJeMi>_yoi{8)wD-<>fuTHb4q6=Azx~O@tT-mHOc;Coi=yk#m<{*c@y>+7VJ)Y`
zH7^7!95U5VfHwXRuP<1Xn<oL}ln(`c5X4h?W8@_7vpo)#Q8WTZ$+m^7GDPn=;E=hz
z5qhRJ%Nt_AC|rE0^CWA6?2lkrK4Hk%Cz%<!9&>9E0H((a)#Ei4w54M^6L;5p`W9uK
zqNM^kZ5crYth^~JZNrEiSA;6vPvMtB#FB%Q!-YL4^38vUk$o|~1T%9bg}8!cRk>2L
zoS466Ntm73m#QVE8IoO<M}CgR-^Em>#8f153=oRXWqqfWxY|L8-HK5%xhn)4Q?N?k
zxw1VJ`PeV|Cm!j2_`~GCGX$(36gFf|0PzuA(r2KNL}i6`Uu2;1gs0-13(pML+6TK)
zwr*1HC_6K`6ie~jk5rZYA+q`u2$2y?<BjnS`C;(&mTBv|=tD?k^cjE+Y7C8l)PssP
z_T@_&s$J_M1!F5ha(mJTFk3)29Be2B;dY3`7_8&?8gE4$i9jP6%VQq+D#n>L_Mw#e
zq7R&11cgPc8v2wG*LMlFaK@B6qev;qhf&K!^@_cT^i)P*Bcpw)nd5FU+jpmv9%8xI
zt3%SK#JrbOCs{*4!M1qYee?T5LdIJS4gDl9h-rzeSs3=YSu&$ByF&~kk@mL4-wllD
z9w^E{L=8=DN6Zdll|oVukBodk=#;wP`GP7z5w57)KE3xL$pAK<Ujdifo12Qi#85-C
z{i}K5BO%M|&i@!8z*G0byJAli6vlmqS7q%BUg#j<c^7bB>wBsUio`)hZu)$!bix3c
zX;lvIlHMpR$(iur<c5`{k1g$`Koe(2Q`RRq8SV`f@`!-h4GiA;0c9ihdW=fcPcUCq
zR25>F!dnEMqkK){O0*mA=bRlQ<3##GlFUQD7A}_p(Hmwm>F{Kco5HVfAh%U{qlr&b
z+l%f3?c(RpQsgNl9#^RS$1KA)ST^Y~`g{^Rd^0Xg;JxA_-_IXy3^Wz!l*PL)WPW(0
zQFQ!2X0)yHuihSL|LPiSy=9v5=k4*dZaoNn31K*G@W*F)F%J<hi;s_3-`V%Yl`HE6
zEv&)sGqci7YiEW8T-`Tvg^e$4Ut}Jcp)U`%t!h5?9>6C(u9_-~V9fFEastnp?gn%4
zhqJsPv>s1AoZloX9|0ZY3(5lTT%DFQ8~o}aeTH>;o;F2a>H;tB;fnSyBrWi$z*QE&
z#lyZ8t~H6+hxEQacE_&WH_9%3=u7fzJ^DW2<*iitod@_IrqyW9Tc;&ErFVNb?IPMc
zYlqh!BU2A2nH9g%z`W?r3#nRn(K>I<6_167R%Arcu2@3!tVC>)0adCst%HplOX_l7
zv*O1fdtl7q?1i0dkf;)BGJ5?e)P^Q|cUp$|o0(&@J<Up_aCazgA{xNcK#8UvbCF`s
zW0X4Yqi}bv_f<J{ZQrNyCPp&g%=L6`K+K$VKq+(+Sz~7!Hj#I|-W4U(vTFCS3LkXP
zEO<Ut>NwrwyYbok^8Z|{it|ooMeY>vOU8pob5q7J-xyzw-kSp-hqX9Lv$zgaeF?Hc
zItT7uOxQCTBbEy$Z(dsJ-+AnSe8X;uH<e}1ay^y-nf~V|mE<;111OQa?EGqxOoKl{
zA8*O{jH`wyqq5vf*9G~^-1L+?a0>3zT$wq16+ye{^+9TX@|;dO$oI5;8zUYrr;RhH
zyUX{}j1tZ3)i2Xu4d&U~_z$n!a4Yi%IyXXXe4pLy<S`wgNAgchOLWEB!|6iZM*B-}
zCo}7I$A0#KY$q!72?}+1zBJjou4228AJ8K<QZ=7!KR!)~(Vx<Ncr&Ehr@^#~3{(0Q
zdxt9LHW$%e67GU6Xp@9lynY2PPsOYsin@+TH;8$z2UK2&Z)HcoNZuwPVCU~UJo_QD
z>uS}NkWca0sFc@`U~}C12JHUv#w9)T${U?PpuA|T&`j;%kF7L~^@0P%XS_Y{`K<~K
zyTCt}ohwv0<4jR@fY-kJN}NJ5nuU-z(DmsLd<5cU3$V3&xF$MD*qg^kE1nHd*QrQR
zXggd>6CKE8Dl(#})Udgpt45*byU&Ow@!ih6PzR_fRVqifUfzxzJ^qFQG0e(ik4l{m
zfEOZJ0&IT{lv@PDy1&~z2J2%4YHuL=jrUtzBhN#GKM=is=id{Y9{nr}m<A+zct2m7
zu>HzL(^fA>LYHKXBZ=E?<$qm21m6iSzK7L!$D?c)N`Fled<OKRt33OTp{#04@Hq9e
z@^TTwEueWA^D;@TNz7V$q<!4R6vr%fP<&N&#9($%KHGWuEVktF-Ql;dt;$!Rli5*E
zeui=O=Q#?(S=IB;HYXVjZAGV((8DWL*Ha9Uiueej$|wzT(j{-GyvhMoA}UwZ^|{?i
zYSSnzONux6ENb!5a`a-VS^YG=K;@F<_p}&~An>-NTwEF5BJa<JEC@L9lCUdGNLrrL
z;YmEbSmctJN>^oGEMc+sGwp4Z)<B%Gn%D>r4+rK*f*ALvVw5P33Fj}Cp2+l{irG!_
z7p7az*2ERkO+MGP^*)qH{B>qC9!HjVg?5+yHU+>yV>Aoaad*k-LV<Vws<lf0_lB-I
zpzTlW^Ic$Yl%JxdzjqPc`}}8QCFp?xE`sR3<ekA&Ois00?o&nwW4M|z)hd=NTU64O
zfJ8t7H!RL?KlFVu+HYP>X&^k#ed&RhrkRZ3^6qD6qF+tioS&BOfi$+(p{X3UUgTXs
zMw8vAy_qz9jU+hYeCbZhaGuB9*#HL};xtYEJXqE{1n0?D18-{OXd%H-H&K*h&WhkC
zebczh>(s<uRn3U0I3+yORSaE8AEdA|S$`ZbI^X#(OaStzYyuFx#WG8aMTI}fXqf@e
zEg(%uACgltd5ZQ)mEO+HVIrz(sRBW<m6w-kiwTJM->Z3tlR3d+tFLwQE=ZrWFQLr_
z?n2dS+d+o;{O<s(+LAIn=fmCpVe?OYI2y8LDUqUL9a&bQ0w%W?5(LD1cQX^OZp&1U
zxGr)vxs7OCi||>!k_BpXv*_U6#s1s+IamddgLS<Utb$M<GEUGUf-wA%3R<JI*GjzD
zS8tpUt(cneDKHJCtaj250Vw?B<jUYgto4vl13HLmx6M)dpCMf@XDjn`l{$B(xH|UB
zQ!-$G2K5WL$?rQ{dVWj>8w7oNEz$}_<?SDVCsS<VNdc!n<HK0a7yhI~e0BbPn$7U2
z@=Af-b`Ki?;IM0M5yVJTY=(w_5s!d@*AEzxd(;-H+-hORF2(se^?pdL{p9c^B%mr_
zGAs7Td0T(WGTlq!yGE+;N!@yd={0vYG@hVP*HlxD`8RdwtOXf?P+IzRkt*hLn5uY~
z!88ACH7efQifbUG*U_XB@ig(}vZAeO0s0ugwws|<I7B2VEEOj_>=ycUog$Rcru>c4
z>N*&iB_((&DL6j7sA=vs>(S%c@&okwxlQWko{t5lwv2qGp7ZA?GtuodZhfivn3qid
zJ7Ei|VJ#~1Wuw!!FD%W^#fX($04(UgUVpD#lb0RiryM)dx1V3@<S9YI)+6fR*?7(^
z6zV|q*#QY7c4T=LZtwpc7rnV*#1`1~3ni{}Z*BE<!5#kI*aDq&Sk%AyEm)Ki%-l7S
zF<26^ie6lo+n28hms;{n83>AM*DQ)iL0<G@`aj?3yBv#rz7!{VT@}uHe<y|LP*NXO
zkLV(`YYE_gt3L&q)I4_?wthGG%n&`&5Ko5E^OWT?o6Nombk5iDCb=fOZqJDhuhB>C
zZBY#UWeGvy$YQ--xg$TsXAY_qNlLTl2jplaL#R>D3zoJYi#?0&ntX@KR7hovTpv5D
z)?~WYM|<ukOh?^rZ<=l82d?T=iyQ+WLQb})X|@}|EK_H5$~@UTwU(}bqVZPTX#L}x
z=_{yUQ8yr{%=&o2p5Pbfr*t;MUp-)<X~mtd)1W{rB1zdA{PK2tfN1#kTt0>HVQ6))
z;hBl~;g|JmpWe4#psYvD;FEb}?dfm@x0}9x#4?IX`Q%Ly1B3%~uKUoBJ7!Cl*6*z{
zER7_lDgy)S+Pm7L1D0LGCk%MhN<U9JiG@rq$YTH#B#}HhnPa0epL_{<!y+wFf$@`z
z-p1-GY|3R+>(Ll%qW`iwCXAu(%)he-t4tw%Ri;O`;ngblBbi=JjwqTgTc6AK|Ig*E
z(+zrpzY1A0X9i-j%ImW&GhPz8*6y@sj{q#G7#~~JoK0}1-Bm&u7U@U>DEd#`bBFgn
z`G|80KQ1JOv8$~%%tTdso^NO^WKFig<AeUL-HA?+-Bv{w1$#kxZseNN%!fzN_O9mA
zN-y_A%$g1G_pf8eoCM!b5$qXNg=mA{8*>K_eV7dQ;zamMf3Sm!hy3lBWkUn&*w1lq
zAUW>b8GoYGmU;U4Laqf|gj=#J$_N8GIFpb9Bh>HX--AVQm`oehlkv%wJpCgN2lCDy
zI4>T=QgFKc-yi?W4hN#l50ryDrxFEYeqyo@3SLE=wgm?CP%q>`ZN`<s%iGiu{g!5R
z$%U<_k)7v1UGKRo;~4pjm+}7|=!74ckAUGe*-cFyEIx82UKh1OlfAmaD-omhA49Gl
zGp_|?UaA@OuepuA5D@>n&UEK4%hRh-8Jz#<v_a1=KdAq#oI#x~l$k4%L3@vi_UC?4
zO9JCHebeT7bYnv1{^}cX4dO~lIlJ4PztZz7z36&pHF>GdC9d$SB*HaW=R%IH0b-y$
z@xx7QbQ*2s*gGD&bAr$2T#{s&eS@>z$vge$8q;!+BP053X$8cR{zg*Bu~RYrdkt0m
zs&}nz#jx(!ytG#O^1vY*duD2HZk(9}&^1mtsU2rBu8ww8Yo2uf?j>AV685wEpcH{9
zBPHa?G;|C+6f1N&=&_9WaIzsPX4}yz(=k!`zN@+?N89h=^Z2qHgMZ%^{vP2<FbUC9
zsaEmXp&_^NE8JJ7RPZI}al&_lQII01Bs_*Vrg;0CSMRrk-r}ico$Cijm>i(~EC(Cr
z=OE$9?t9>WosZcE)3ZsI5lJf7ty>d6=vheC<IUUpwlQM92=k%HlX08~E3EM7+e9Xe
zAN-$UC$1`?Vb9xe`tM)Qa@Ui&?LvA^1pW=qp}1kEwi<%hQ^hoyXC>C(rn=s<k=`HZ
z__(q`qUM(M!fyXESaqWy1^z7)QfB7wfwELwGnui-+>(^B+`u!}@B4DJTs*w5h)wSv
zraVaWX?vpa=d$XgwWA5AtW~8yXi{LbkM>R-T|=3?s68vb_`#5#h_#x#p}nm9X@a*5
z5d<;|3E?LdFMrMYMAaU%i89W)1~u*W%^x~<e7xM2iDt1HdeHaXWq94rF8j<;d-_He
zU~IX8?bF(<uVWP$(x(FbqkrrQ@%1w}R>6K<2o7Dx91c^!Zbm0{YX1Nr>|y|d7&P>L
ziF3G3i5Waz>Cwlhj0>x!B#?|`R-Ey*6<)9j^6#4kQMT{51FkzC4=~}Nm%NuOmZAh{
zgY71&aQ!*GV+7;d;y&*D#Kg5X=j{^*#V5J#gm9s2ghNkF3&x^+N9~LFh^V}Cg4K(w
zpF<zrY8O`802=E`%-CEVn&kyC7ZzgPr7D;)b{A#w&!^|_`0dW8mff|QADfi8OQ!$7
z07fFY-T5?=kK{GS%x|*aa_)?KaYS$R*=aj>I)?3(5wsG}+eo<7xn@1`%oxGXyJjtR
z5fP=OHbtfJz1%s)laCdjOHBE$)9TX^=C)HD+{bCww3bAJAsvOMuI~v3m`Zy&j5lE_
zkg1Iul*GGXQGq@vG=k?&6dDGlRb#43NGg%75RI!?#<EkwSRw{xJ8w6+spjTzv{NhP
zEC~sfLdsjmm4@IXD#mBmYH}pT4qYY*_PiMdkm_yY^y7LGZIc<3lBHbM%*$kZ1R#<c
zhQ_6ZAju-<+)!!}5nPhxB!;ujUUM;Vl?o^Wy2CG$&dyWpA;g$SnRn#NBNl09CR#4H
zvPgFJ<xHH&<f6olu_S!nv1={ZTa%XyS(kLRmvmam-*Rp=qW8D+i2LQqOVJxa!KJF$
zyOmJND@auV3CX#Qg`{ndBtFaLcQLlRA!#EbH<@H=^Cj4?o4mM55oaug;GmKP__O8<
z@NvkDK`UXkY$9lNK@5>3A;!gVgp&yrRfcx(#E=RCvg1<j@(^{)`uTg4b1u<#NJy+Q
zb2F14cFQ`{ID~QuaTTky%)7x>$ps=g+@khZ9%LTv+c1*u)GXr)s8dBx1Pf5UD^+3=
z&K;OT&o4zJXbK9EX7?kR3(3o^EY!YaZzL9^o6F4Gmegx5N1Fo`2<VptV3JqPWF9&=
zm5PSuW?YQBHzp$~k(%8a<2K20F_&`fiHuBK?ne2X!s~_*gptZqxqNk5jOtKz-s{rZ
z1}hB}(sCDI#KcLjD|l{#1QTmsQ8xtWw&?q4R~ae&EyfZ2(jw-lpiMW|od!{`?JqKs
zAp=FSRwhPnVc7Ae>&i$#4Gw9TRI<$G37N4s=3xyWkSU}RX;hC`r*{R+s3D7NS#vqR
zZLN#2WkE(Q!o<EZUEeQTz_gY+mL;SW6K368do-#&?*M$#6(Vm{OUNDwJ7miNI;a^@
z9ct{k_8OSYi)`5uL+39WjhDv9MAYlJmhRJS8+O`I%PFee%*3_HeaKvxjgWOqWr*CX
zL~Ag<W$(FE08yBSE?8aI^LIHl&S{xhA!ExOLltwh#bqjG#}rj=GDh2_24QzmaCv>a
zd6w9swGm{4b6nlOVR!YHS3?KOb?oKEaUrmhHp}hC_Tb402x_yhyt=Q=O#5k=)S4x*
zNdi7=*V4+0l!`Q@Cg1CZwG(b1q%e4r5E#E%e<yZjUBpWpY|BiLK-gEa1;25ev=IB4
z5v5M2sDY_mX=~)KUfq24D?1fyP~YFV&y)8$EW~r%!B`?WLffpbn?VtN@6Jr*iPSM8
zafPK_-SvrY+Z)+yUCC?{<RB<N4;r*L#J|L-zo182CoCzPxH+Z4O<_dDtlqcemv82n
zwQs4uX)%j6zp2;8E}@SuWX-1Uavndlv0b5d)j?yF#QT?q=6>4*MeL91?oE80wo>_z
z=7=Gax2bJnYjlCcChgnGYRiXEkGZ>>$TPPU7hwTWHMI2o$(Gj<c!vksm&Hv($ybwF
zwhl2P%VGLMWVWv%CoZ{cKUo&BsN{~wmFEY%nw?H|P=4e&N}rM^Q2DFx>xqA#TuYqv
z^Su485v$)kJ1NzKAO{FX<IQ2UW9d|p@wxe;)2h`(k~=u&Z{3?|Zam}*{rc`VnPlMW
zEwQ)qe8O+7j}q7O_U>i4mY-$5;fzo-e!wv_bbF1TcT9>B0rerxBjnITwW&yeIc!Fu
z1@m-{Wr88b6^aX7$_fry6vbvLrheNuNq5`k%>8z%d4ysju6A_DaXx6iB%|Jz*MD8V
z5<em9f{{GK#_<d2#u$8g1VvK!i$Q`tc!VdueW;&&oA`{z-K{<rL_b=(raVpZ5UW%O
zw5!k1zthcMWb`dGaXYXv-6ke5>K|p{w4)S_((ADD*4bqmV~ZgiVRv8C%+Wt;vkx}m
zUhJn-ksL*u$5QqmnlZQ64pI+4UF6jBN7>@TjAO>D7a@20%u7vcSk>}X{(;Bc_VZT#
z&tMuNs9@LYAX>c3xaEB)?Z7RrrScK1hPRCEmRhNic>ZYlJI#HEy#qdd`wL_87L=!x
zt)X+z*%e0W+ptt<KDN(*pCm&*n1jST3Dzn3G!Y#a$E%np!4i7g1kv>$j$-$%t^F&h
zLvvA9#&TM0{b8ucOl2RM^E%JmqA^#k6`i~=q90`iQy7SGn4de}aKz@>IM2tItUAD5
zWz4+H><qS6T%*r_a`s5A9Br<%A|lj~8*?LtvP!WLw2vF}c^!2{a&-flE2$9am-_&p
zK#F2RR#)q*{G~gQ6zN0n8eb7-)J2+$>S6tDf${Pn)$Nt@h$wbL=D%ceeYbT-w(Q)8
zlBT_V%hva`!#079b&gs2#&+4#ffkYa(c!f*W8%mp(ng4dmWG~!115aRIM}ixD1C-p
zg<?Kay}!KarV&JS#t6QCqVr;&gjmFnoK2<U<aW%@&edV8EmA%|W!If`I+h`ivzv$?
zLK*r-Jc)@V>c$x4s@I<-X(R@lJoLA){5|W-r4<l)5wDJrGHuo6yly95+IfYBS`~+T
zRKHkiVzwQX6JrB&CB~^k(jAk<^~{z*Z5BT^*YD7e<nny+K4J?(j{5kV{oVGy-<b-p
zlsV}K<Mc!?qE#XZsS!1H$6FdpTCb_gTzrUzi1Ne{4Iqz&pHoU&UuWAEt>))&nK7cs
z&SW~7usYgog4=hSm$D-9s-Jq}9Zzh0kqbX7eh7*3MVcTh+=K337mG|*m(ZneWcep2
zdGmC%r^!d3V=LDboV%_#`&|1?&Sqt{huM5)r{=KKs?S-7zn{=4WClf$EYb9c(mey-
z4fydzp;1p6#ucer<8q5rTCGBgexXt%J|HHWhtK3tRFc*vF9#J;vdM*FPfy;4$*)p<
z(&+rxcW$N6q_MUlpCyPJkU{q#nRiv9dVaXqSmGbE>xrsz%RK64UFS$!jSBKZV<cjz
zl0wNOn#Y8IvPnN!eAgU*F}Bk^Dc)Z$Qnw4ZEq+g*k$y~KTqelE7Q}8<?{9U1)V{^*
z8Z}4EbM()oe5uMs*2LpcGC>q$$U`LbA#OI%)8VGN*wT-_?qEwuigeAI8R@3)*)l2!
z>vl@(mkjzGL-)|f?T1uWtssw?`T4gHZ?<8_>383!UEVgD#P)L+E8F$?t~`wO%32fj
z7Y_+fm=%IPB>O2mRGN!=*{7bnnXuNcn=ZW}^HEw-W$sqEi?1>Z%>8_K^y0{{Ymk-2
zXVPq_tiJu4()kQagC~2LtXEc42hdp6_VWpHu#jZ0h{ChmHRjU8uh$b*#5jlM#L(G9
zE)hPE<4fF!-QjVP8*;7@`X4oYk>B65DPPlA*Ia!*oU<x4LP#>-&F!eOP2A%)(QW1(
zWT)5U=HA=CiEeV|({%insjXUHr4#6P(-C;aWVHHC^ti}_**lgI+mwmqyC>!~ujDWD
z%R1VgV7S$4cd@y}c)Rqvx?N7?TFN4s`!G1oVUaPeAsbFI$sq`jzHBd;%fF$=5lT~u
z2MEfxh_j?*`F!V@9>N(L5GLGX4<A~mWGPhFiDMxq%C}I~{SVnODMUi|F^PPoX6;dC
z+fx%WnTuIZSX*ZtUKhmtn#Y!YPP2PuLfainkIXuo)`e-q%$fBHB=M|RFzb7TsyjHA
ziX25J7qGJ#CN`8DOn`5ch}2xr29RGzWFLuM@*$d%&i;jtY_!+NrJ>i`3_ksRclR#8
zB>iWM^DcIVakDM1Az6QZ$vL-e9Hjg5auJ$^cv@CbWsok`5I2g_F!&_7z3$}8EgPOo
zPccx@2qPIDCZ;n|sy|vYs8wN|sdDEquY9KIVjN@h4oOWtn0*ZICo%{_B%f|(%5{^_
z&cN?GWJj|s?6R;_%#|B@+dyjid|d(ZMDFity97`{5Rc6OT`Qhx`3!X#D8|O-%Pj-l
zJ%2nOu6}sxR|F4Jb)qptr7G`pyDreEAw{$q4EovvpUo9%>K0%yuZsFMOsFlta$0iq
zv+to_JrzX|Zf51G(Gp1%Yh-n(d9NnE*n7}w;*%l!BR_6Zn5xnt<SycSC7+TsT#qGv
zeJn2e+987xY9FNJnIR}i4_(Y>eB2g%pT4Jomx!XBWoFtHQIH9eY$HqQ1Ko$*<%b%j
zYEM4T*DRutUnQFxfgB^+em+v#HAu0wRrL-dCj{$=r2F;66U(_n6I1r}!YI9QJ<`bg
z=I-{~vv>9t$MCOGwRPr6m#v~BcLP$%CPU=AgCtbfj~i<>WfPFj&L+RFx%AS|RV&Gj
zqwagNe6^a&sh9Q#a{S_Jyr_*@R(yuIDr+_e8u`UT=|nC{FSm4Y{E=>B8m>NTiGQ|V
z<itGHZIh@Un|MI$$LmdtL*)eSr)R`c9&900nWTWWyo(?%={WmN5e8!(%4UREs#nEA
zeG(@rBSm$M3U>Ktd-~7Mp7v3>mKLXd11R%e=z%_UnLS6?Y?hIDyeb4p;xVI$c@T@m
zw1yFI&8qju$ktf1FuR!|(QNG2)o+$owV5g>Hhr+w6~`Ir9D1o0MuV^9*YXQm_DoB;
z8*E<4pPX!KslJA_s@21}giE&XkEE;+Q0p4vX8yC2r`hB@F&&qO%*o?7I^p<2*D-xg
zZ7U8i!G~UMT{jB1@<i_4n*DLeWt(JaOsDI)mopDguC!D^U)g@T<Co@abK7!@8K1Me
zVDz$y#dr0dzjo%?%khWY?c~Z;AEzIYO}J_cY><szz+b-Jd(7rF)co5<$&W1h`*Bwo
zqD$Mhy|Vczw0p}16tv5WZt_|zncS_&qAw^`enClG?T5)cN%=d<b<Dh9DO|wHrj4|%
z4Iq&L8#Gv+qN|^TGcL@z7$MF|Evu1AJe`of*>M|z8vWm2k=aio{+#ExiK0Dn$;WQC
zOH}injFo~bN}_y`1wK!*b0HIrZNrJrrhPPPk(ETUR(2kA$plTGHmG&mn~W_}B-?_=
zx@EToFR+QhlB~bimQQ-~7)6OSh(Dnmkwfy_D|c)ytonXU6h+2*MD*`B?yjBEP9#ju
zT$4R3iQJz*CA7X)Ahlnab6ovw&A#@B=k2GK^XoN1h{q)<#Oqm*a#m#GW<9#|&ZPN=
z%wNb<=jd+!IAi38=4)Cy)+-WnRotxX&$BV{>yPG&qD6`<+onzCm`(kEA+ePWL~vzc
z#5`w-&SpSak4JLF6n<%~+XCIlW0{@U9m+49%v16cG_gVKUNyh6W>|&KyOm>`9f5JB
zeYE0?)W$GjjIoQ%J;+}v^RH8>a)m=0#Z9}9xL?%PcHXna&zoN|HHe~%qr|Rz_133*
zjiQ|iGwv*t?_nPoCBX<cMf9q03DqF^@_pG#c1nLkor^~9UZ)&gl*qKaLT{zXIP&6|
zmvz*6cS<ULg{7m<E(g}1Co@pOburJZ`6eRc`JS`XtKWT<e3$xTjC&3@N0{&H`N|Qb
zL1_2rVq!dfWNY*Lov~}JUnCx<&2<ylu;QZ(NFaz;GIOTwPo+T`q8a9~`Qu#KMXzKz
zEeuQ*bJEU8M3u%BugxuPK^~%!c?jzi*rY#0uYPXLO=4BFOothBQ`g(7{kd!2wp;Tt
zpp)zGcGP;1oJZG*iPkG(S~!7U8FGpjZw}f6>3)7C5CR}$Z;v?B`L@-iJK1ssxfV{f
zY2SQ8@9U(pnnoC7Sray7VF*ZP32h`1#F8T!ES5z2USful?z_vhMrA_MMdi}`*rQGg
zx%Y7P^*i>|_FeIqjF}~7O2IINP!&SAb1yO28r<8zyS>=F<f&#!Y`?R9<ysuxaZQ>R
zNn&cHTUQ|!ncrsYq+rRmche0g(pwmcttlfZpEq?2?7PDEZN0J#;#E)uUnW-eX^=RY
z*Ej1n>SUV<*>{cLEwiaIBMZfrElY<GZZhsBQYaIywJzM#Yuk2?+mhRJh1+P%%Y_Qb
zAdLhqB1s1gI&G$zrr(NDo-paH%)z+Q5<r=|F*DYQ<1Q3;Gc30VRFMSufw8em#^iv<
z>ui%%PA6ma{du@cR-}?nNKAnQzoy+qUF>ZtzQH!xOdLW7S1onG#0dK*fFAG_B1^=v
z4Ru}3vhGGkW!@64GB`O*M2(l0<g#}+NwFGflWc*;vf;N2O@uy$>|9-hab-dbQtnK$
z9`2U$-d(<O!E@ZAfC{;@WK(tPh2z6LWHSv`9+*I+gb1XHq*E!BS~R9+dvt4e(B{*!
z?6X{uiYK-OrovHsOLN#4i5DUD^~{Uhm8r$fVUkG+23}LH+fAZ*%r}%djqNToArzN!
zw0S2JQt01a=F8V6PVR~rh%0OxGP%%&qzp0PxQIz4fHiOIy0!6hH!#(WB$5!R+|Qc%
z?e}Mm)Pxm#DRRuX<+EMcq?FddlO|>ta~#WWAF^PI4&*IE<|(Ne+pB9+uV-0$*H32V
zkePX=2GvnT1~^=~N-}0=95Ia8wne=##~E;BMiN{mvALz~#m3yG^70sw!qP!wTBO40
z!*PQ=y_my@(_oS@dHJG7a{~uiU14tKA6vw`MU~`|5=3njU#VftRpi$ga}~ghR}?<+
zG-kXbH@3-rJAJ&-a8sFUByvk(Z9(?SSk%cTwB46^xlz@7voo0B_F{E9U0A$?fJ_<I
zzgF|>G*&57)??q--c26%^XMGqGtcLnDORFXXctnvh1~7Otj2LJ^|U!6>lq~qqTH4H
z_j5VeL932kPeO7LuC!Z5qny1<x^mVx?7ua)2>m87Q7@D9%cTyXqy`8p=cdY}mNCR4
z%pe6h)hzg1!YYX4)~xiaOC1^}Pk!cK?sAXKM_P_GDX*FOYhr$r_h5a<4(0Jv^HKJ;
zb$bq^GSaloMEpm@T|sUdKvDGh2Kg(hCUle>8IX!XF26OdD_bGiC~2<9KXU&jdmEPy
zdh^@*n%#o5ieA&MCX9u)AFkxl&nZvTD!$vsT2k67yw73B-)-?DLaEUsD?6AN)-`GM
z?V2>5WqX%rPeR%{n*RH@Txw2Ve&4yErzAfi%Z|$T?){GmOGe2{8>G8dG*6)om6gxa
zP~tGN?`^LSv*vBhow~%2q|$e1T`DJA`);YG)8^AHihY*l#)_YqRhfGA$L4ReUOj{h
z26j0}5<#~7S~fLg9nc2SAeeA*G1sD6Y#CEvPO{n7Cvx~Tj>s0r_djIax09DgnztT1
z$$PgctW9d;=JS%ZIPWvqQHp&CjGkMRq*u&CbgMr<eNMbXqPO)5lSCGGIr{4IOoRnG
z)OzX>qw-I->&<)UVhSAe3+yX?nfZs(FC-RvRdM+%?)K5<W<`z-7JS^ct!Y-3rTt7t
zGn#rle=8;JZH6}opJw^DKU$ei<XKKn%@MNU*52eQ-c;Lz*^6>W<d&YL=NFmwwOG6b
z^|A32FdsLG5E%&j;>MeVoV$=Ym|vKyBt&iylh;+LZb%;_c##+F_1o=&BSWN@kTbCF
zB|dx1B5^Xw1#7Z91BrVG)#MA~wjZ5W8<sY|m?1w!t8wFK$x-Z;ZEw40+r!yZwelP3
zHfmIML-TKyiSltjT<(`^YaW+O+c?^M^7(!JDqJlNjcMp!zg>KsO@DCp>s@{J*{ae%
zZdr$5Em~W*woTIN?&cE|vTV9OO>EezeLC2AJ)YMs-QT=(n2T9%J~9}|)Zbe>%3zf(
z=3JrqC1!WU!w3W#bm7Fkt$O9LyQ)e!c)7`yq)|or;<m?fW%b+2BqE3JOZruyrf&PQ
zPd2Hap{^w7FB|;|*X9Ug0=!FFc+pXM-LpNWu}aUv59(#F1M0RMF@XrkDWVAoA4J{G
zc~^e8h&*C(samLW+iUqT3COCC=2KZ!AIojF9yi)oZJ(LO`$_g+6Ns^*K7P19pPLX~
z=3gBydf<&D6TM6lKBO(u4iOKAb;m-4BHc9*(meT5!wv+#C=w17eU$TR&!mos=)OAr
z$anOH%<fv{%Db4=D5Yja_T7rBdn{UBLtMzTqBHXgHHQ6n=D!EtX9lfmDJIrgP!i7&
zKGp2yoV{G3FQhxYx;UEG8jMW6ZGJ*X$Z|vWS(v15S#)0^&P?P1=Pc@O%C4bl^NNes
zzV=v%ajfJk60&m3e1-LgBC^l2mRC1%g~)iFFcHNcRj&q8NZ2W>D3XtyP+|fTy1@A9
zht-Zxa#5jEARnJ(4oGzg`}NV*x6_0AV*bj!kCWx(bL8Sc`55=3BcOK!YsmQ@lP|!E
zX}`XBMCFGFM^C=mENjA8;u}lkul64O!w}<*b=;g{59Es)kK|qK`(<Fx%P()qZVy`;
zw9nUt;cb?YYYfcR#2vn2VdLJG8lo>VnGLy%jPUJ}s5Xito;`k}<WcZQCuk^c@_-%X
za{BS#(jx2E=eZWV(ZWk{4ElYs@ZWiP&TH+J&qAYrNUG9<df8}?GEr*M)5#N3MTTx!
zf_gynHdc;ebcGPgw8t%e`NGK{0!hTVE+P=^QTnNiAs>zWh0<PE^c}SR&XJ!stb$B@
z(8))BmBGOaL%RcBv99)6B;ETlQP@w@mjx#NyiJ5&Bx)xavyHW?oce3KQ1_vZa^;Vp
zT3nTG35v65p{aSeIZJBdVEMRZWmY3Aspi!+yvpNlZI~;Kd|{bjq8>^S-OPRK2>r}t
zUI)=a87%?v42YuNS}`$Rv8B}6WusKJj2li|ddfdjGUHuJzn<l59B<~jhvs>{IJr;V
z-^{V~tWaM5r2PBKZHll_REvq+qKWm)r=PcUU1#PPj?k^!{gC<!q!dLvxyKV^dm*?(
zVU`X;SrvmSN}Nw-{g(Z=V0o@TpNnO3;VvFP^K_4)Uz=^{@ikNC_6t+8t)Y9V+{+P{
z3*!>_LH1ZY?&0s*deVYCIdy)>dP<jjfJ$2ZJpRbPIdUiEPn<{euUuPVzn#P0!rlln
zAEYn$CATLzXHybvTR>l={Kkh8D8HIoiAtcVE+@Zjo=tB#QtI!wKT0v<@6tJ(zBVx}
zJtgd>$T1*&$bL?`ncU1e{(f=ut*RQ-UN96zNhg?c5E?^|FLU<fxqn0KktoFaMWtMl
zKO}Zc{POM3KqIJomuq3giENMHWqY<Cxc!kM_lTqlo&i3&b=1a}N!j1C98Ho=z_Cjb
zdy~G}Y=Dxzj(tv`ZipPvcFPDBY9<Uwq`CRTt<9O1tuhk%^v5CPoUr|++I&cdmdWnF
zr>BJwESQ#a7xLef$%IG_tYc5wtgKi_HkBW3#tHSED<c?I3~%M@B*@x6cx$WOV++MZ
zBZef+TCyy5a`+<O%ry*4X!_xNpFB>u>MGJPBUs0i8>sc-a`pRf$0s7kk4v!j{>^bU
zFS=)69vj>)tqmAOjCPlqsCMEzXdjTmFo+`a3?hB8><D7(h;OzTIchIZtTL8yH?+~0
za}<2V`NrLZ*5pPeWby<tlwruUScG`{lUAfyEAnlT{hj7&jmf;S+UFr}!)^2H^Iy3!
zjdAN7b7rQmVUHRY8WcG8U&;G!`D7b-_X$`#aG_45R=vA!L+9qdXRMxK`a|32xmua@
zFSC|@x~TPy2%hY?%28{dW%=17n`FECbvsPt$A4jC@ej}J#N&dBD9^9STaCfgHZN^e
zxtLlZ%1?65-*EGDBizhoV(qHWcKkn{0-q&CIR0!yp4-G8LmK=%*nXcp!;P)nzhtWS
zG_upu%a<ovM;vwQ5cxhG?5$(TELC~g=_?q9w$G=WYmG4588%(fmQT%jxeqq4SkluW
z%bk*<MOWnR%@xtqk`eswb{*!r_QEvAJNm-k&k*nJzQJdF!Ix$s8~S=>y_~q+=2sm=
z{gzndB8q$0bViTgK#4Eww^O$&HQ%P2D38(z)^-K{xo3okzcDS1FJ~?$ewLr4$3B-K
z9FSq|cWd##hU?@XyU(|tgdZj{*K9cBS=Meqy0%wdxRW6EQ5&CKT#?p|bjb>}!AL{g
zht!p#Kzf5d#H^2WP~UfkRcXd5b?w$%81tA}Ahq?ut%bEzog^^~EDk<X0<$T45YHcW
zA=Cod1&Ixs*!!fhr`a!wXinjD@_DrPZvQxy$7k!L*+&;__qSSIWP5xst)6F+0zbU~
zmpsYd<rw*`N0P9AAm7^sA0iPsY<a(sc^gDyT)coX(*-U_KK2*+qY>}cdf}WgFA(eG
z;=1{%yCWN)rAaE>c#F9M?IC5syuT4!Yu0|Hq$^9`CES1{9*!S~_<=bfO=A-BA0}wQ
z<@?$ei?e0>(zQD#v`5Lt)u{>8^Fw}?W)rOu*#$L=-7GDLUG=w+INo~uvL=bV@9Ye9
z732@eUC3*f+pmz1TI_b{X*!Yoo*35BWO$N(!p6W>v`J{76`!Y8^nSEdAe0Zj*8c2%
z`p_6}?GFk?0YpnXbs9L4;=ebadd%O_#q5SMy2SSWx`?LjxBHWhp>NKik3A^keRjxx
z<<}bdt@-8G>yqGh^65`Mkm7S@t2R~1iEbtDE~l=|9cbf6SeGIA<=-dVC(?;!WBMQ=
z2>K?=mmk)~5H+t8XUO~OOws4!f=kGx*ye%zR~SpYbXOpbkdRLhClb!L>ou$HK#>UV
zls=ZciD%5Ht0T5$Ecqs&qK_Nub1%MD*MPI;elpXIw!KSQq4q@<PId{@7B6n_r2!ym
z0k&BsCi$WP`{eBJO*Sju(oHxYV)bTfP-H+uhwAE{gIo0<kGzj^U2AchlB`(8KAYDq
z7Z@zdrx(SLIQawYrrE}#s!^mW`HM6`B>T@6mVJg1WP030+zXGDOu0wnWs$^sWP&ad
zM;f#ka`anivVNG^nRmGkWb=xiPbFiX7;s)?-!@!o<(FP@vL>?XNIPi$d&8CYA*haA
zzOwFPv6x)SXEO5BXMVcqwyvR)1c83c##fkxdwKS2h`EezX}1d{<h7?36t{NT%#GdG
zwiSlUxG~q~HcFp%&bY?9obBc3ZVe-Hvd$L^hBU@j5)cfzQt4E#JErEc<1I>CxKg%H
zvo)N{nZn|gVRv+?i0P?|n|qR)(ppp=5({$+Eje;CFD~2mX=~l<nyANQOSWs3RX`nM
zm7*$;e(TF#I^%zK!-OVz3DRM3&4Y7mys^(tb9Gh7w7toTPIoU3eP$*qDm;D2VBv6g
z49JDt+>542n~+-w5^m;*y6wQpZe$j=*^XgZXhzZxnsim9kD(>Q5&@{g!%=#v7}QO6
zvvzlDb9Xr6YZ2YGrNZ*tE;7Z$x90+iD;ch^c*}(Jt#@Ygxn;RI6NcTU^Om|U!faTO
zOqX`zVlR?fNWjOaS>DUk?{HS)hNAtRxx&%+4C`g4swwL<l1MnssU_bsW?h=j<1I0p
zDAYt*cX5s0KQj-LZXzy}vdAWxNkG8~C(F^8(x#EdgQggCf`gq>QTnN&Hp4b3ilC!y
zxmRrWU}3XWE}BQ=`QJqqc#Arft}{8ix{9!IEUER+G%6M(5MHv9tz<Zb=3Pk}nX+De
z?E_?lhSRTyoNpnjzOdJeZl_wkxR|^ba<R7MYGue}D|v8nn{wJ@gcc5ErN?BprV=80
z5fN>hozC;QR}oqx_GE;CmHC+YzT1~`IVECF$h?M2m&Ds9yJV@dBJ(G*2<j*#n{Y2~
u+{tlSw`IE3@^>Z4-6Z2NwwD`{qDGrG?ZOiyD_{U0#oUoj6eJS6C&R$o4OaR9

literal 0
HcmV?d00001

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
index 2b4abed645910..4991e397eb11c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ApproximatePercentileQuerySuite.scala
@@ -150,7 +150,7 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession
       (1 to 1000).toDF("col").createOrReplaceTempView(table)
       checkAnswer(
         spark.sql(s"SELECT percentile_approx(col, array(0.25 + 0.25D), 200 + 800) FROM $table"),
-        Row(Seq(499))
+        Row(Seq(500))
       )
     }
   }
@@ -296,4 +296,23 @@ class ApproximatePercentileQuerySuite extends QueryTest with SharedSparkSession
     buffer.quantileSummaries
     assert(buffer.isCompressed)
   }
+
+  test("SPARK-32908: maximum target error in percentile_approx") {
+    withTempView(table) {
+      spark.read
+        .schema("col int")
+        .csv(testFile("test-data/percentile_approx-input.csv.bz2"))
+        .repartition(1)
+        .createOrReplaceTempView(table)
+      checkAnswer(
+        spark.sql(
+          s"""SELECT
+             |  percentile_approx(col, 0.77, 1000),
+             |  percentile_approx(col, 0.77, 10000),
+             |  percentile_approx(col, 0.77, 100000),
+             |  percentile_approx(col, 0.77, 1000000)
+             |FROM $table""".stripMargin),
+        Row(18, 17, 17, 17))
+    }
+  }
 }

From 2d55de5db06ff522a8e7fded3760c3f65c8bbf93 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 17 Sep 2020 22:07:47 -0700
Subject: [PATCH 1059/1280] [SPARK-32906][SQL] Struct field names should not
 change after normalizing floats

### What changes were proposed in this pull request?

This PR intends to fix a minor bug when normalizing floats for struct types;
```
scala> import org.apache.spark.sql.execution.aggregate.HashAggregateExec
scala> val df = Seq(Tuple1(Tuple1(-0.0d)), Tuple1(Tuple1(0.0d))).toDF("k")
scala> val agg = df.distinct()
scala> agg.explain()
== Physical Plan ==
*(2) HashAggregate(keys=[k#40], functions=[])
+- Exchange hashpartitioning(k#40, 200), true, [id=#62]
   +- *(1) HashAggregate(keys=[knownfloatingpointnormalized(if (isnull(k#40)) null else named_struct(col1, knownfloatingpointnormalized(normalizenanandzero(k#40._1)))) AS k#40], functions=[])
      +- *(1) LocalTableScan [k#40]

scala> val aggOutput = agg.queryExecution.sparkPlan.collect { case a: HashAggregateExec => a.output.head }
scala> aggOutput.foreach { attr => println(attr.prettyJson) }
### Final Aggregate ###
[ {
  "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference",
  "num-children" : 0,
  "name" : "k",
  "dataType" : {
    "type" : "struct",
    "fields" : [ {
      "name" : "_1",
                ^^^
      "type" : "double",
      "nullable" : false,
      "metadata" : { }
    } ]
  },
  "nullable" : true,
  "metadata" : { },
  "exprId" : {
    "product-class" : "org.apache.spark.sql.catalyst.expressions.ExprId",
    "id" : 40,
    "jvmId" : "a824e83f-933e-4b85-a1ff-577b5a0e2366"
  },
  "qualifier" : [ ]
} ]

### Partial Aggregate ###
[ {
  "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference",
  "num-children" : 0,
  "name" : "k",
  "dataType" : {
    "type" : "struct",
    "fields" : [ {
      "name" : "col1",
                ^^^^
      "type" : "double",
      "nullable" : true,
      "metadata" : { }
    } ]
  },
  "nullable" : true,
  "metadata" : { },
  "exprId" : {
    "product-class" : "org.apache.spark.sql.catalyst.expressions.ExprId",
    "id" : 40,
    "jvmId" : "a824e83f-933e-4b85-a1ff-577b5a0e2366"
  },
  "qualifier" : [ ]
} ]
```

### Why are the changes needed?

bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #29780 from maropu/FixBugInNormalizedFloatingNumbers.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
(cherry picked from commit b49aaa33e13814a448be51a7e65a29cb515b8248)
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 .../sql/catalyst/optimizer/NormalizeFloatingNumbers.scala | 6 +++---
 .../org/apache/spark/sql/DataFrameAggregateSuite.scala    | 8 ++++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index 8d5dbc7dc90eb..f0cf6717ade27 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -120,10 +120,10 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
       KnownFloatingPointNormalized(NormalizeNaNAndZero(expr))
 
     case _ if expr.dataType.isInstanceOf[StructType] =>
-      val fields = expr.dataType.asInstanceOf[StructType].fields.indices.map { i =>
-        normalize(GetStructField(expr, i))
+      val fields = expr.dataType.asInstanceOf[StructType].fieldNames.zipWithIndex.map {
+        case (name, i) => Seq(Literal(name), normalize(GetStructField(expr, i)))
       }
-      val struct = CreateStruct(fields)
+      val struct = CreateNamedStruct(fields.flatten.toSeq)
       KnownFloatingPointNormalized(If(IsNull(expr), Literal(null, struct.dataType), struct))
 
     case _ if expr.dataType.isInstanceOf[ArrayType] =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
index 54327b38c100b..2cb7790028b33 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala
@@ -1044,6 +1044,14 @@ class DataFrameAggregateSuite extends QueryTest
     checkAnswer(sql(queryTemplate("FIRST")), Row(1))
     checkAnswer(sql(queryTemplate("LAST")), Row(3))
   }
+
+  test("SPARK-32906: struct field names should not change after normalizing floats") {
+    val df = Seq(Tuple1(Tuple2(-0.0d, Double.NaN)), Tuple1(Tuple2(0.0d, Double.NaN))).toDF("k")
+    val aggs = df.distinct().queryExecution.sparkPlan.collect { case a: HashAggregateExec => a }
+    assert(aggs.length == 2)
+    assert(aggs.head.output.map(_.dataType.simpleString).head ===
+      aggs.last.output.map(_.dataType.simpleString).head)
+  }
 }
 
 case class B(c: Option[Double])

From ffcd757cf54583dfd589b42e87897fe8a255077f Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 18 Sep 2020 07:41:21 +0000
Subject: [PATCH 1060/1280] [SPARK-32905][CORE][YARN] ApplicationMaster fails
 to receive UpdateDelegationTokens message

### What changes were proposed in this pull request?

With a long-running application in kerberized mode, the AMEndpiont handles `UpdateDelegationTokens` message wrong, which is an OneWayMessage that should be handled in the `receive` function.

```java
20-09-15 18:53:01 INFO yarn.YarnAllocator: Received 22 containers from YARN, launching executors on 0 of them.
20-09-16 12:52:28 ERROR netty.Inbox: Ignoring error
org.apache.spark.SparkException: NettyRpcEndpointRef(spark-client://YarnAM) does not implement 'receive'
	at org.apache.spark.rpc.RpcEndpoint$$anonfun$receive$1.applyOrElse(RpcEndpoint.scala:70)
	at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:115)
	at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:203)
	at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100)
	at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:75)
	at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:41)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
20-09-17 06:52:28 ERROR netty.Inbox: Ignoring error
org.apache.spark.SparkException: NettyRpcEndpointRef(spark-client://YarnAM) does not implement 'receive'
	at org.apache.spark.rpc.RpcEndpoint$$anonfun$receive$1.applyOrElse(RpcEndpoint.scala:70)
	at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:115)
	at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:203)
	at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100)
	at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:75)
	at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:41)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
```

### Why are the changes needed?

bugfix, without a proper token refresher, the long-running apps are going to fail potentially in kerberized cluster

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

Passing jenkins

and verify manually

I am running the sub-module `kyuubi-spark-sql-engine` of https://github.com/yaooqinn/kyuubi

The simplest way to reproduce the bug and verify this fix is to follow these steps

#### 1 build the `kyuubi-spark-sql-engine` module
```
mvn clean package -pl :kyuubi-spark-sql-engine
```
#### 2. config the spark with Kerberos settings towards your secured cluster

#### 3. start it in the background
```
nohup bin/spark-submit --class org.apache.kyuubi.engine.spark.SparkSQLEngine ../kyuubi-spark-sql-engine-1.0.0-SNAPSHOT.jar > kyuubi.log &
```

#### 4. check the AM log and see

"Updating delegation tokens ..." for SUCCESS

"Inbox: Ignoring error ...... does not implement 'receive'" for FAILURE

Closes #29777 from yaooqinn/SPARK-32905.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 9e9d4b6994a29fb139fd50d24b5418a900c7f072)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/deploy/yarn/ApplicationMaster.scala  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 862acd8c03a82..99efa5c497abc 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -775,6 +775,11 @@ private[spark] class ApplicationMaster(
       driver.send(RegisterClusterManager(self))
     }
 
+    override def receive: PartialFunction[Any, Unit] = {
+      case UpdateDelegationTokens(tokens) =>
+        SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
+    }
+
     override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
       case r: RequestExecutors =>
         Option(allocator) match {
@@ -806,9 +811,6 @@ private[spark] class ApplicationMaster(
           case None =>
             logWarning("Container allocator is not ready to find executor loss reasons yet.")
         }
-
-      case UpdateDelegationTokens(tokens) =>
-        SparkHadoopUtil.get.addDelegationTokens(tokens, sparkConf)
     }
 
     override def onDisconnected(remoteAddress: RpcAddress): Unit = {

From 20cd7bba4e6521b5c6d5eb370210d9488e977c62 Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Fri, 18 Sep 2020 18:13:11 +0900
Subject: [PATCH 1061/1280] [SPARK-32930][CORE] Replace deprecated
 isFile/isDirectory methods

### What changes were proposed in this pull request?

This PR aims to replace deprecated `isFile` and `isDirectory` methods.

```diff
- fs.isDirectory(hadoopPath)
+ fs.getFileStatus(hadoopPath).isDirectory
```

```diff
- fs.isFile(new Path(inProgressLog))
+ fs.getFileStatus(new Path(inProgressLog)).isFile
```

### Why are the changes needed?

It shows deprecation warnings.

- https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test%20(Dashboard)/job/spark-master-test-sbt-hadoop-3.2-hive-2.3/1244/consoleFull

```
[warn] /home/jenkins/workspace/spark-master-test-sbt-hadoop-3.2-hive-2.3/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala:815: method isFile in class FileSystem is deprecated: see corresponding Javadoc for more information.
[warn]             if (!fs.isFile(new Path(inProgressLog))) {
```

```
[warn] /home/jenkins/workspace/spark-master-test-sbt-hadoop-3.2-hive-2.3/core/src/main/scala/org/apache/spark/SparkContext.scala:1884: method isDirectory in class FileSystem is deprecated: see corresponding Javadoc for more information.
[warn]           if (fs.isDirectory(hadoopPath)) {
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins.

Closes #29796 from williamhyun/filesystem.

Authored-by: William Hyun <williamhyun3@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 78928879810a2e96dbb6ec4608b548a0072a040f)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/SparkContext.scala   | 2 +-
 .../spark/deploy/history/EventLogFileWritersSuite.scala   | 6 +++---
 .../apache/spark/sql/hive/execution/HiveDDLSuite.scala    | 2 +-
 .../scala/org/apache/spark/streaming/util/HdfsUtils.scala | 8 ++++++--
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 66fe1d7f2ba13..9f9d611b79376 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -1853,7 +1853,7 @@ class SparkContext(config: SparkConf) extends Logging {
           if (!fs.exists(hadoopPath)) {
             throw new FileNotFoundException(s"Jar ${path} not found")
           }
-          if (fs.isDirectory(hadoopPath)) {
+          if (fs.getFileStatus(hadoopPath).isDirectory) {
             throw new IllegalArgumentException(
               s"Directory ${path} is not allowed for addJar")
           }
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
index 060b878fb8ef2..e9b739ce7a4c6 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
@@ -213,7 +213,7 @@ class SingleEventLogFileWriterSuite extends EventLogFileWritersSuite {
       compressionCodecShortName)
 
     val finalLogPath = new Path(logPath)
-    assert(fileSystem.exists(finalLogPath) && fileSystem.isFile(finalLogPath))
+    assert(fileSystem.exists(finalLogPath) && fileSystem.getFileStatus(finalLogPath).isFile)
     assert(expectedLines === readLinesFromEventLogFile(finalLogPath, fileSystem))
   }
 }
@@ -357,10 +357,10 @@ class RollingEventLogFilesWriterSuite extends EventLogFileWritersSuite {
       expectedLines: Seq[String]): Unit = {
     val logDirPath = getAppEventLogDirPath(logBaseDir, appId, appAttemptId)
 
-    assert(fileSystem.exists(logDirPath) && fileSystem.isDirectory(logDirPath))
+    assert(fileSystem.exists(logDirPath) && fileSystem.getFileStatus(logDirPath).isDirectory)
 
     val appStatusFile = getAppStatusFilePath(logDirPath, appId, appAttemptId, inProgress = false)
-    assert(fileSystem.exists(appStatusFile) && fileSystem.isFile(appStatusFile))
+    assert(fileSystem.exists(appStatusFile) && fileSystem.getFileStatus(appStatusFile).isFile)
 
     val eventLogFiles = listEventLogFiles(logDirPath)
     val allLines = mutable.ArrayBuffer[String]()
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index d0aa618fc2731..86385f0f56b5c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -1191,7 +1191,7 @@ class HiveDDLSuite
       expectedDBUri,
       Map.empty))
     // the database directory was created
-    assert(fs.exists(dbPath) && fs.isDirectory(dbPath))
+    assert(fs.exists(dbPath) && fs.getFileStatus(dbPath).isDirectory)
     sql(s"USE $dbName")
 
     val tabName = "tab1"
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
index 146577214de17..006bcad5d68c2 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
@@ -58,7 +58,7 @@ private[streaming] object HdfsUtils {
         // If we are really unlucky, the file may be deleted as we're opening the stream.
         // This can happen as clean up is performed by daemon threads that may be left over from
         // previous runs.
-        if (!dfs.isFile(dfsPath)) null else throw e
+        if (!dfs.getFileStatus(dfsPath).isFile) null else throw e
     }
   }
 
@@ -92,6 +92,10 @@ private[streaming] object HdfsUtils {
   def checkFileExists(path: String, conf: Configuration): Boolean = {
     val hdpPath = new Path(path)
     val fs = getFileSystemForPath(hdpPath, conf)
-    fs.isFile(hdpPath)
+    try {
+      fs.getFileStatus(hdpPath).isFile
+    } catch {
+      case _: FileNotFoundException => false
+    }
   }
 }

From 7746c20ce73887876f19d66d48077608c1cf2575 Mon Sep 17 00:00:00 2001
From: Peter Toth <peter.toth@gmail.com>
Date: Fri, 18 Sep 2020 13:56:19 -0700
Subject: [PATCH 1062/1280] [SPARK-32635][SQL][FOLLOW-UP] Add a new test case
 in catalyst module

### What changes were proposed in this pull request?
This is a follow-up PR to https://github.com/apache/spark/pull/29771 and just adds a new test case.

### Why are the changes needed?
To have better test coverage.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
New UT.

Closes #29802 from peter-toth/SPARK-32635-fix-foldable-propagation-followup.

Authored-by: Peter Toth <peter.toth@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3309a2be071f2d3f6122f3634aea998d6fa53876)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../optimizer/FoldablePropagationSuite.scala         | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
index 0d48ecb31cfa4..59dfd3a7932bd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FoldablePropagationSuite.scala
@@ -180,4 +180,16 @@ class FoldablePropagationSuite extends PlanTest {
       .select((Literal(1) + 3).as('res)).analyze
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-32635: Replace references with foldables coming only from the node's children") {
+    val leftExpression = 'a.int
+    val left = LocalRelation(leftExpression).select('a)
+    val rightExpression = Alias(Literal(2), "a")(leftExpression.exprId)
+    val right = LocalRelation('b.int).select('b, rightExpression).select('b)
+    val join = left.join(right, joinType = LeftOuter, condition = Some('b === 'a))
+
+    val query = join.analyze
+    val optimized = Optimize.execute(query)
+    comparePlans(optimized, query)
+  }
 }

From 03fb144d6a09829e39fa8ae39b7522fa2e00b248 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Fri, 18 Sep 2020 14:02:14 -0700
Subject: [PATCH 1063/1280] [SPARK-32898][CORE] Fix wrong executorRunTime when
 task killed before real start

### What changes were proposed in this pull request?

Only calculate the executorRunTime when taskStartTimeNs > 0. Otherwise, set executorRunTime to 0.

### Why are the changes needed?

bug fix.

It's possible that a task be killed (e.g., by another successful attempt) before it reaches "taskStartTimeNs = System.nanoTime()". In this case, taskStartTimeNs is still 0 since it hasn't been really initialized. And we will get the wrong executorRunTime by calculating System.nanoTime() - taskStartTimeNs.

### Does this PR introduce _any_ user-facing change?

Yes, users will see the correct executorRunTime.

### How was this patch tested?

Pass existing tests.

Closes #29789 from Ngone51/fix-SPARK-32898.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit f1dc479d39a6f05df7155008d8ec26dff42bb06c)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 core/src/main/scala/org/apache/spark/executor/Executor.scala | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index e9f1d9c250ad2..83e4a739f5817 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -367,7 +367,9 @@ private[spark] class Executor(
       // Report executor runtime and JVM gc time
       Option(task).foreach(t => {
         t.metrics.setExecutorRunTime(TimeUnit.NANOSECONDS.toMillis(
-          System.nanoTime() - taskStartTimeNs))
+          // SPARK-32898: it's possible that a task is killed when taskStartTimeNs has the initial
+          // value(=0) still. In this case, the executorRunTime should be considered as 0.
+          if (taskStartTimeNs > 0) System.nanoTime() - taskStartTimeNs else 0))
         t.metrics.setJvmGCTime(computeTotalGcTime() - startGCTime)
       })
 

From 0a4b668b62cf198358897a145f7d844614cd1931 Mon Sep 17 00:00:00 2001
From: Zhen Li <zhli@microsoft.com>
Date: Mon, 21 Sep 2020 09:05:40 -0500
Subject: [PATCH 1064/1280] [SPARK-32886][WEBUI] fix 'undefined' link in event
 timeline view

### What changes were proposed in this pull request?

Fix ".../jobs/undefined" link from "Event Timeline" in jobs page. Job page link in "Event Timeline" view is constructed by fetching job page link defined in job list below. when job count exceeds page size of job table, only links of jobs in job table can be fetched from page. Other jobs' link would be 'undefined', and links of them in "Event Timeline" are broken, they are redirected to some wired URL like ".../jobs/undefined". This PR is fixing this wrong link issue. With this PR, job link in "Event Timeline" view would always redirect to correct job page.

### Why are the changes needed?

Wrong link (".../jobs/undefined") in "Event Timeline" of jobs page. for example, the first job in below page is not in table below, as job count(116) exceeds page size(100). When clicking it's item in "Event Timeline", page is redirected to ".../jobs/undefined", which is wrong. Links in "Event Timeline" should always be correct.
![undefinedlink](https://user-images.githubusercontent.com/10524738/93184779-83fa6d80-f6f1-11ea-8a80-1a304ca9cbb2.JPG)

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Manually tested.

Closes #29757 from zhli1142015/fix-link-event-timeline-view.

Authored-by: Zhen Li <zhli@microsoft.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit d01594e8d186e63a6c3ce361e756565e830d5237)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../apache/spark/ui/static/timeline-view.js   | 53 ++++++++++++-------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
index 5be8cffd1f8db..220b76a0f1b27 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/timeline-view.js
@@ -42,26 +42,31 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime, offset) {
   setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
   setupExecutorEventAction();
 
+  function getIdForJobEntry(baseElem) {
+    var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
+    var jobId = jobIdText.match("\\(Job (\\d+)\\)$")[1];
+    return jobId;
+  }
+
+  function getSelectorForJobEntry(jobId) {
+    return "#job-" + jobId;
+  }
+
   function setupJobEventAction() {
     $(".vis-item.vis-range.job.application-timeline-object").each(function() {
-      var getSelectorForJobEntry = function(baseElem) {
-        var jobIdText = $($(baseElem).find(".application-timeline-content")[0]).text();
-        var jobId = jobIdText.match("\\(Job (\\d+)\\)$")[1];
-       return "#job-" + jobId;
-      };
-
       $(this).click(function() {
-        var jobPagePath = $(getSelectorForJobEntry(this)).find("a.name-link").attr("href");
-          window.location.href = jobPagePath
+        var jobId = getIdForJobEntry(this);
+        var jobPagePath = uiRoot + appBasePath + "/jobs/job/?id=" + jobId;
+        window.location.href = jobPagePath;
       });
 
       $(this).hover(
         function() {
-          $(getSelectorForJobEntry(this)).addClass("corresponding-item-hover");
+          $(getSelectorForJobEntry(getIdForJobEntry(this))).addClass("corresponding-item-hover");
           $($(this).find("div.application-timeline-content")[0]).tooltip("show");
         },
         function() {
-          $(getSelectorForJobEntry(this)).removeClass("corresponding-item-hover");
+          $(getSelectorForJobEntry(getIdForJobEntry(this))).removeClass("corresponding-item-hover");
           $($(this).find("div.application-timeline-content")[0]).tooltip("hide");
         }
       );
@@ -125,26 +130,34 @@ function drawJobTimeline(groupArray, eventObjArray, startTime, offset) {
   setupZoomable("#job-timeline-zoom-lock", jobTimeline);
   setupExecutorEventAction();
 
+  function getStageIdAndAttemptForStageEntry(baseElem) {
+    var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
+    var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)$")[1].split(".");
+    return stageIdAndAttempt;
+  }
+
+  function getSelectorForStageEntry(stageIdAndAttempt) {
+    return "#stage-" + stageIdAndAttempt[0] + "-" + stageIdAndAttempt[1];
+  }
+
   function setupStageEventAction() {
     $(".vis-item.vis-range.stage.job-timeline-object").each(function() {
-      var getSelectorForStageEntry = function(baseElem) {
-        var stageIdText = $($(baseElem).find(".job-timeline-content")[0]).text();
-        var stageIdAndAttempt = stageIdText.match("\\(Stage (\\d+\\.\\d+)\\)$")[1].split(".");
-        return "#stage-" + stageIdAndAttempt[0] + "-" + stageIdAndAttempt[1];
-      };
-
       $(this).click(function() {
-        var stagePagePath = $(getSelectorForStageEntry(this)).find("a.name-link").attr("href")
-        window.location.href = stagePagePath
+        var stageIdAndAttempt = getStageIdAndAttemptForStageEntry(this);
+        var stagePagePath = uiRoot + appBasePath +
+          "/stages/stage/?id=" + stageIdAndAttempt[0] + "&attempt=" + stageIdAndAttempt[1];
+        window.location.href = stagePagePath;
       });
 
       $(this).hover(
         function() {
-          $(getSelectorForStageEntry(this)).addClass("corresponding-item-hover");
+          $(getSelectorForStageEntry(getStageIdAndAttemptForStageEntry(this)))
+            .addClass("corresponding-item-hover");
           $($(this).find("div.job-timeline-content")[0]).tooltip("show");
         },
         function() {
-          $(getSelectorForStageEntry(this)).removeClass("corresponding-item-hover");
+          $(getSelectorForStageEntry(getStageIdAndAttemptForStageEntry(this)))
+            .removeClass("corresponding-item-hover");
           $($(this).find("div.job-timeline-content")[0]).tooltip("hide");
         }
       );

From b27bbbbbe46572feebf549a2314b6c9ea0d39c8a Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Mon, 21 Sep 2020 14:06:54 -0700
Subject: [PATCH 1065/1280] [SPARK-32718][SQL][3.0] Remove unnecessary keywords
 for interval units

Backport https://github.com/apache/spark/pull/29560 to 3.0, as it's kind of a bug fix for the ANSI mode. People can't use `year`,  `month`, etc. functions under ANSI mode.

Closes #29823 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-ref-ansi-compliance.md               |  6 --
 .../spark/sql/catalyst/parser/SqlBase.g4      | 26 +-------
 .../sql/catalyst/parser/AstBuilder.scala      |  2 +-
 .../resources/sql-tests/inputs/interval.sql   |  4 ++
 .../sql-tests/results/ansi/datetime.sql.out   | 10 +--
 .../sql-tests/results/ansi/interval.sql.out   | 64 +++++++++++++------
 .../sql-tests/results/interval.sql.out        | 42 +++++++++++-
 .../apache/spark/sql/TPCDSQuerySuite.scala    |  6 ++
 8 files changed, 100 insertions(+), 60 deletions(-)

diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md
index 193616120ec99..948a36e15f3ce 100644
--- a/docs/sql-ref-ansi-compliance.md
+++ b/docs/sql-ref-ansi-compliance.md
@@ -181,7 +181,6 @@ Below is a list of all the keywords in Spark SQL.
 |DATA|non-reserved|non-reserved|non-reserved|
 |DATABASE|non-reserved|non-reserved|non-reserved|
 |DATABASES|non-reserved|non-reserved|non-reserved|
-|DAY|reserved|non-reserved|reserved|
 |DBPROPERTIES|non-reserved|non-reserved|non-reserved|
 |DEFINED|non-reserved|non-reserved|non-reserved|
 |DELETE|non-reserved|non-reserved|reserved|
@@ -227,7 +226,6 @@ Below is a list of all the keywords in Spark SQL.
 |GROUP|reserved|non-reserved|reserved|
 |GROUPING|non-reserved|non-reserved|reserved|
 |HAVING|reserved|non-reserved|reserved|
-|HOUR|reserved|non-reserved|reserved|
 |IF|non-reserved|non-reserved|not a keyword|
 |IGNORE|non-reserved|non-reserved|non-reserved|
 |IMPORT|non-reserved|non-reserved|non-reserved|
@@ -265,8 +263,6 @@ Below is a list of all the keywords in Spark SQL.
 |MATCHED|non-reserved|non-reserved|non-reserved|
 |MERGE|non-reserved|non-reserved|non-reserved|
 |MINUS|non-reserved|strict-non-reserved|non-reserved|
-|MINUTE|reserved|non-reserved|reserved|
-|MONTH|reserved|non-reserved|reserved|
 |MSCK|non-reserved|non-reserved|non-reserved|
 |NAMESPACE|non-reserved|non-reserved|non-reserved|
 |NAMESPACES|non-reserved|non-reserved|non-reserved|
@@ -326,7 +322,6 @@ Below is a list of all the keywords in Spark SQL.
 |ROWS|non-reserved|non-reserved|reserved|
 |SCHEMA|non-reserved|non-reserved|non-reserved|
 |SCHEMAS|non-reserved|non-reserved|not a keyword|
-|SECOND|reserved|non-reserved|reserved|
 |SELECT|reserved|non-reserved|reserved|
 |SEMI|non-reserved|strict-non-reserved|non-reserved|
 |SEPARATED|non-reserved|non-reserved|non-reserved|
@@ -384,4 +379,3 @@ Below is a list of all the keywords in Spark SQL.
 |WHERE|reserved|non-reserved|reserved|
 |WINDOW|non-reserved|non-reserved|reserved|
 |WITH|reserved|non-reserved|reserved|
-|YEAR|reserved|non-reserved|reserved|
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index df6ff9fe5af11..922ff10b5d5f6 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -818,7 +818,7 @@ errorCapturingMultiUnitsInterval
     ;
 
 multiUnitsInterval
-    : (intervalValue intervalUnit)+
+    : (intervalValue unit+=identifier)+
     ;
 
 errorCapturingUnitToUnitInterval
@@ -826,7 +826,7 @@ errorCapturingUnitToUnitInterval
     ;
 
 unitToUnitInterval
-    : value=intervalValue from=intervalUnit TO to=intervalUnit
+    : value=intervalValue from=identifier TO to=identifier
     ;
 
 intervalValue
@@ -834,16 +834,6 @@ intervalValue
     | STRING
     ;
 
-intervalUnit
-    : DAY
-    | HOUR
-    | MINUTE
-    | MONTH
-    | SECOND
-    | YEAR
-    | identifier
-    ;
-
 colPosition
     : position=FIRST | position=AFTER afterCol=errorCapturingIdentifier
     ;
@@ -1251,7 +1241,6 @@ nonReserved
     | DATA
     | DATABASE
     | DATABASES
-    | DAY
     | DBPROPERTIES
     | DEFINED
     | DELETE
@@ -1295,7 +1284,6 @@ nonReserved
     | GROUP
     | GROUPING
     | HAVING
-    | HOUR
     | IF
     | IGNORE
     | IMPORT
@@ -1328,8 +1316,6 @@ nonReserved
     | MAP
     | MATCHED
     | MERGE
-    | MINUTE
-    | MONTH
     | MSCK
     | NAMESPACE
     | NAMESPACES
@@ -1384,7 +1370,6 @@ nonReserved
     | ROW
     | ROWS
     | SCHEMA
-    | SECOND
     | SELECT
     | SEPARATED
     | SERDE
@@ -1438,7 +1423,6 @@ nonReserved
     | WHERE
     | WINDOW
     | WITH
-    | YEAR
 //--DEFAULT-NON-RESERVED-END
     ;
 
@@ -1501,7 +1485,6 @@ CURRENT_USER: 'CURRENT_USER';
 DATA: 'DATA';
 DATABASE: 'DATABASE';
 DATABASES: 'DATABASES' | 'SCHEMAS';
-DAY: 'DAY';
 DBPROPERTIES: 'DBPROPERTIES';
 DEFINED: 'DEFINED';
 DELETE: 'DELETE';
@@ -1547,7 +1530,6 @@ GRANT: 'GRANT';
 GROUP: 'GROUP';
 GROUPING: 'GROUPING';
 HAVING: 'HAVING';
-HOUR: 'HOUR';
 IF: 'IF';
 IGNORE: 'IGNORE';
 IMPORT: 'IMPORT';
@@ -1584,8 +1566,6 @@ MACRO: 'MACRO';
 MAP: 'MAP';
 MATCHED: 'MATCHED';
 MERGE: 'MERGE';
-MINUTE: 'MINUTE';
-MONTH: 'MONTH';
 MSCK: 'MSCK';
 NAMESPACE: 'NAMESPACE';
 NAMESPACES: 'NAMESPACES';
@@ -1643,7 +1623,6 @@ ROLLUP: 'ROLLUP';
 ROW: 'ROW';
 ROWS: 'ROWS';
 SCHEMA: 'SCHEMA';
-SECOND: 'SECOND';
 SELECT: 'SELECT';
 SEMI: 'SEMI';
 SEPARATED: 'SEPARATED';
@@ -1701,7 +1680,6 @@ WHEN: 'WHEN';
 WHERE: 'WHERE';
 WINDOW: 'WINDOW';
 WITH: 'WITH';
-YEAR: 'YEAR';
 //--SPARK-KEYWORD-LIST-END
 //============================
 // End of the keywords list
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 307a72eee0376..938976eb74ab9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -2100,7 +2100,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   override def visitMultiUnitsInterval(ctx: MultiUnitsIntervalContext): CalendarInterval = {
     withOrigin(ctx) {
-      val units = ctx.intervalUnit().asScala
+      val units = ctx.unit.asScala
       val values = ctx.intervalValue().asScala
       try {
         assert(units.length == values.length)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index 9ad968ecda936..8f6cf0504baf2 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -55,6 +55,7 @@ select interval '15:40:32.99899999' hour to second;
 select interval '40:32.99899999' minute to second;
 select interval '40:32' minute to second;
 select interval 30 day day;
+select interval 30 days days;
 
 -- invalid day-time string intervals
 select interval '20 15:40:32.99899999' day to hour;
@@ -90,6 +91,9 @@ select interval '12:11:10' hour to second '1' year;
 select interval (-30) day;
 select interval (a + 1) day;
 select interval 30 day day day;
+select interval (-30) days;
+select interval (a + 1) days;
+select interval 30 days days days;
 
 -- Interval year-month arithmetic
 
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
index ec007aa8e721d..7e5568dc5cb36 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
@@ -114,15 +114,9 @@ struct<weekday(CAST(2007-02-03 AS DATE)):int,weekday(CAST(2009-07-30 AS DATE)):i
 -- !query
 select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
 -- !query schema
-struct<>
+struct<year(CAST(1500-01-01 AS DATE)):int,month(CAST(1500-01-01 AS DATE)):int,dayofyear(CAST(1500-01-01 AS DATE)):int>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
-
-no viable alternative at input 'year'(line 1, pos 7)
-
-== SQL ==
-select year('1500-01-01'), month('1500-01-01'), dayOfYear('1500-01-01')
--------^^^
+1500	1	1
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
index 5a66db960ccbd..2cee0c8886935 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 103
+-- Number of queries: 107
 
 
 -- !query
@@ -355,15 +355,17 @@ struct<INTERVAL '40 minutes 32 seconds':interval>
 -- !query
 select interval 30 day day
 -- !query schema
-struct<>
+struct<day:interval>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
+30 days
 
-no viable alternative at input 'day'(line 1, pos 23)
 
-== SQL ==
-select interval 30 day day
------------------------^^^
+-- !query
+select interval 30 days days
+-- !query schema
+struct<days:interval>
+-- !query output
+30 days
 
 
 -- !query
@@ -655,41 +657,63 @@ select interval (-30) day
 -- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.catalyst.parser.ParseException
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
-no viable alternative at input 'day'(line 1, pos 22)
 
-== SQL ==
-select interval (-30) day
-----------------------^^^
+-- !query
+select interval (a + 1) day
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
 
 
 -- !query
-select interval (a + 1) day
+select interval 30 day day day
 -- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-no viable alternative at input 'day'(line 1, pos 24)
+extraneous input 'day' expecting {<EOF>, ';'}(line 1, pos 27)
 
 == SQL ==
-select interval (a + 1) day
-------------------------^^^
+select interval 30 day day day
+---------------------------^^^
 
 
 -- !query
-select interval 30 day day day
+select interval (-30) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval (a + 1) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval 30 days days days
 -- !query schema
 struct<>
 -- !query output
 org.apache.spark.sql.catalyst.parser.ParseException
 
-no viable alternative at input 'day'(line 1, pos 23)
+extraneous input 'days' expecting {<EOF>, ';'}(line 1, pos 29)
 
 == SQL ==
-select interval 30 day day day
------------------------^^^
+select interval 30 days days days
+-----------------------------^^^
 
 
 -- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index baf7f16908a5b..3068e8ede70f9 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 103
+-- Number of queries: 107
 
 
 -- !query
@@ -354,6 +354,14 @@ struct<day:interval>
 30 days
 
 
+-- !query
+select interval 30 days days
+-- !query schema
+struct<days:interval>
+-- !query output
+30 days
+
+
 -- !query
 select interval '20 15:40:32.99899999' day to hour
 -- !query schema
@@ -670,6 +678,38 @@ select interval 30 day day day
 ---------------------------^^^
 
 
+-- !query
+select interval (-30) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval (a + 1) days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+Undefined function: 'interval'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
+
+
+-- !query
+select interval 30 days days days
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.parser.ParseException
+
+extraneous input 'days' expecting {<EOF>, ';'}(line 1, pos 29)
+
+== SQL ==
+select interval 30 days days days
+-----------------------------^^^
+
+
 -- !query
 create temporary view interval_arithmetic as
   select CAST(dateval AS date), CAST(tsval AS timestamp) from values
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
index e72c31f4cf6e4..f4c6fb9b75ff8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.util.resourceToString
 import org.apache.spark.sql.internal.SQLConf
@@ -121,3 +122,8 @@ class TPCDSQueryWithStatsSuite extends TPCDSQuerySuite {
     SQLConf.JOIN_REORDER_ENABLED.key -> "true"
   )
 }
+
+class TPCDSQueryANSISuite extends TPCDSQuerySuite {
+  override protected def sparkConf: SparkConf =
+    super.sparkConf.set(SQLConf.ANSI_ENABLED, true)
+}

From 8a481d8c336360bc5dfa518af70590251e5b61bc Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 22 Sep 2020 10:58:33 -0700
Subject: [PATCH 1066/1280] [SPARK-32659][SQL][FOLLOWUP][3.0] Broadcast Array
 instead of Set in InSubqueryExec

### What changes were proposed in this pull request?

This is a followup of https://github.com/apache/spark/pull/29475.

This PR updates the code to broadcast the Array instead of Set, which was the behavior before #29475

### Why are the changes needed?

The size of Set can be much bigger than Array. It's safer to keep the behavior the same as before and build the set at the executor side.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

existing tests

Closes #29840 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../org/apache/spark/sql/execution/subquery.scala    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
index 9d15c76faa1b5..48d6210c83868 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/subquery.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.ArrayBuffer
 import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{expressions, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{AttributeSeq, CreateNamedStruct, Expression, ExprId, InSet, ListQuery, Literal, PlanExpression}
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, ExprId, InSet, ListQuery, Literal, PlanExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
 import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.internal.SQLConf
@@ -114,10 +114,10 @@ case class InSubqueryExec(
     child: Expression,
     plan: BaseSubqueryExec,
     exprId: ExprId,
-    private var resultBroadcast: Broadcast[Set[Any]] = null) extends ExecSubqueryExpression {
+    private var resultBroadcast: Broadcast[Array[Any]] = null) extends ExecSubqueryExpression {
 
-  @transient private var result: Set[Any] = _
-  @transient private lazy val inSet = InSet(child, result)
+  @transient private var result: Array[Any] = _
+  @transient private lazy val inSet = InSet(child, result.toSet)
 
   override def dataType: DataType = BooleanType
   override def children: Seq[Expression] = child :: Nil
@@ -132,11 +132,11 @@ case class InSubqueryExec(
 
   def updateResult(): Unit = {
     val rows = plan.executeCollect()
-    result = rows.map(_.get(0, child.dataType)).toSet
+    result = rows.map(_.get(0, child.dataType))
     resultBroadcast = plan.sqlContext.sparkContext.broadcast(result)
   }
 
-  def values(): Option[Set[Any]] = Option(resultBroadcast).map(_.value)
+  def values(): Option[Array[Any]] = Option(resultBroadcast).map(_.value)
 
   private def prepareResult(): Unit = {
     require(resultBroadcast != null, s"$this has not finished")

From 58124bd4e5ab2cfdfdc0a6b7c553c25678258c20 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 23 Sep 2020 20:14:12 +0900
Subject: [PATCH 1067/1280] [MINOR][SQL][3.0] Improve examples for
 `percentile_approx()`

### What changes were proposed in this pull request?
In the PR, I propose to replace current examples for `percentile_approx()` with **only one** input value by example **with multiple values** in the input column.

### Why are the changes needed?
Current examples are pretty trivial, and don't demonstrate function's behaviour on a sequence of values.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
- by running `ExpressionInfoSuite`
- `./dev/scalastyle`

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: HyukjinKwon <gurwls223apache.org>
(cherry picked from commit b53da23a28fe149cc75d593c5c36f7020a8a2752)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #29848 from MaxGekk/example-percentile_approx-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../expressions/aggregate/ApproximatePercentile.scala     | 8 ++++----
 .../test/resources/sql-functions/sql-expression-schema.md | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index d06eeeef23936..32f21fc6e2d59 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -60,10 +60,10 @@ import org.apache.spark.sql.types._
   """,
   examples = """
     Examples:
-      > SELECT _FUNC_(10.0, array(0.5, 0.4, 0.1), 100);
-       [10.0,10.0,10.0]
-      > SELECT _FUNC_(10.0, 0.5, 100);
-       10.0
+      > SELECT _FUNC_(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);
+       [1,1,0]
+      > SELECT _FUNC_(col, 0.5, 100) FROM VALUES (0), (6), (7), (9), (10) AS tab(col);
+       7
   """,
   group = "agg_funcs",
   since = "2.1.0")
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 070a6f3851545..b84abe5afc1ef 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -285,8 +285,8 @@
 | org.apache.spark.sql.catalyst.expressions.XxHash64 | xxhash64 | SELECT xxhash64('Spark', array(123), 2) | struct<xxhash64(Spark, array(123), 2):bigint> |
 | org.apache.spark.sql.catalyst.expressions.Year | year | SELECT year('2016-07-30') | struct<year(CAST(2016-07-30 AS DATE)):int> |
 | org.apache.spark.sql.catalyst.expressions.ZipWith | zip_with | SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)) | struct<zip_with(array(1, 2, 3), array(a, b, c), lambdafunction(named_struct(y, namedlambdavariable(), x, namedlambdavariable()), namedlambdavariable(), namedlambdavariable())):array<struct<y:string,x:int>>> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | approx_percentile | SELECT approx_percentile(10.0, array(0.5, 0.4, 0.1), 100) | struct<approx_percentile(10.0, array(0.5, 0.4, 0.1), 100):array<decimal(3,1)>> |
-| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | percentile_approx | SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100) | struct<percentile_approx(10.0, array(0.5, 0.4, 0.1), 100):array<decimal(3,1)>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | approx_percentile | SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col) | struct<approx_percentile(col, array(0.5, 0.4, 0.1), 100):array<int>> |
+| org.apache.spark.sql.catalyst.expressions.aggregate.ApproximatePercentile | percentile_approx | SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col) | struct<percentile_approx(col, array(0.5, 0.4, 0.1), 100):array<int>> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Average | avg | SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<avg(col):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.Average | mean | SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<mean(col):double> |
 | org.apache.spark.sql.catalyst.expressions.aggregate.BitAndAgg | bit_and | SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col) | struct<bit_and(col):int> |

From 542dc97525860e67e3ddcd543cecc8654b19715d Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 23 Sep 2020 20:15:52 +0900
Subject: [PATCH 1068/1280] [SPARK-32306][SQL][DOCS][3.0] Clarify the result of
 `percentile_approx()`

### What changes were proposed in this pull request?
More precise description of the result of the `percentile_approx()` function and its synonym `approx_percentile()`. The proposed sentence clarifies that  the function returns **one of elements** (or array of elements) from the input column.

### Why are the changes needed?
To improve Spark docs and avoid misunderstanding of the function behavior.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
`./dev/scalastyle`

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Liang-Chi Hsieh <viiryagmail.com>
(cherry picked from commit 7c14f177eb5b52d491f41b217926cc8ca5f0ce4c)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #29845 from MaxGekk/doc-percentile_approx-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../aggregate/ApproximatePercentile.scala            | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
index 32f21fc6e2d59..3327f4ccf4461 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala
@@ -49,11 +49,13 @@ import org.apache.spark.sql.types._
  */
 @ExpressionDescription(
   usage = """
-    _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric
-      column `col` at the given percentage. The value of percentage must be between 0.0
-      and 1.0. The `accuracy` parameter (default: 10000) is a positive numeric literal which
-      controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields
-      better accuracy, `1.0/accuracy` is the relative error of the approximation.
+    _FUNC_(col, percentage [, accuracy]) - Returns the approximate `percentile` of the numeric
+      column `col` which is the smallest value in the ordered `col` values (sorted from least to
+      greatest) such that no more than `percentage` of `col` values is less than the value
+      or equal to that value. The value of percentage must be between 0.0 and 1.0. The `accuracy`
+      parameter (default: 10000) is a positive numeric literal which controls approximation accuracy
+      at the cost of memory. Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is
+      the relative error of the approximation.
       When `percentage` is an array, each value of the percentage array must be between 0.0 and 1.0.
       In this case, returns the approximate percentile array of column `col` at the given
       percentage array.

From 21b6b6988e9999666b994839cb403e6409956c7f Mon Sep 17 00:00:00 2001
From: Russell Spitzer <russell.spitzer@gmail.com>
Date: Wed, 23 Sep 2020 20:02:20 -0700
Subject: [PATCH 1069/1280] [SPARK-32977][SQL][DOCS] Fix JavaDoc on Default
 Save Mode

### What changes were proposed in this pull request?

The default is always ErrorsOnExist regardless of DataSource version. Fixing the JavaDoc to reflect this.

### Why are the changes needed?

To fix documentation

### Does this PR introduce _any_ user-facing change?

Doc change.

### How was this patch tested?

Manual.

Closes #29853 from RussellSpitzer/SPARK-32977.

Authored-by: Russell Spitzer <russell.spitzer@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit b3f0087e39c8ad69cf1e53145d62eb73df48efd5)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../src/main/scala/org/apache/spark/sql/DataFrameWriter.scala  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index f463166a9f268..d16404f3ef6fa 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -60,8 +60,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
    * <li>`SaveMode.ErrorIfExists`: throw an exception at runtime.</li>
    * </ul>
    * <p>
-   * When writing to data source v1, the default option is `ErrorIfExists`. When writing to data
-   * source v2, the default option is `Append`.
+   * The default option is `ErrorIfExists`.
    *
    * @since 1.4.0
    */

From 4b84e571f06dec953d532e60942b0e96b4100abe Mon Sep 17 00:00:00 2001
From: ulysses <youxiduo@weidian.com>
Date: Thu, 24 Sep 2020 22:16:05 -0700
Subject: [PATCH 1070/1280] [SPARK-32877][SQL][TEST] Add test for Hive UDF
 complex decimal type

### What changes were proposed in this pull request?

Add test to cover Hive UDF whose input contains complex decimal type.
Add comment to explain why we can't make `HiveSimpleUDF` extend `ImplicitTypeCasts`.

### Why are the changes needed?

For better test coverage with Hive which we compatible or not.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Add test.

Closes #29863 from ulysses-you/SPARK-32877-test.

Authored-by: ulysses <youxiduo@weidian.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit f2fc96667481169affbc20cec95b9fc1c19fc7c3)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../org/apache/spark/sql/hive/hiveUDFs.scala  |  5 +++
 .../sql/hive/execution/HiveUDFSuite.scala     | 31 +++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
index 05d608a2016a5..3117781a84bbf 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala
@@ -41,6 +41,11 @@ import org.apache.spark.sql.hive.HiveShim._
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
 
+/**
+ * Here we cannot extends `ImplicitTypeCasts` to compatible with UDF input data type, the reason is:
+ * we use children data type to reflect UDF method first and will get exception if it fails so that
+ * we can never go into `ImplicitTypeCasts`.
+ */
 private[hive] case class HiveSimpleUDF(
     name: String, funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
   extends Expression
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 7bca2af379934..d3c8428b71806 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.execution.command.FunctionsCommand
 import org.apache.spark.sql.functions.max
+import org.apache.spark.sql.hive.HiveUtils
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
@@ -658,6 +659,25 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
     }
   }
 
+  test("SPARK-32877: add test for Hive UDF complex decimal type") {
+    assume(HiveUtils.isHive23)
+    withUserDefinedFunction("testArraySum" -> false) {
+      sql(s"CREATE FUNCTION testArraySum AS '${classOf[ArraySumUDF].getName}'")
+      checkAnswer(
+        sql("SELECT testArraySum(array(1, 1.1, 1.2))"),
+        Seq(Row(3.3)))
+
+      val msg = intercept[AnalysisException] {
+        sql("SELECT testArraySum(1)")
+      }.getMessage
+      assert(msg.contains(s"No handler for UDF/UDAF/UDTF '${classOf[ArraySumUDF].getName}'"))
+
+      val msg2 = intercept[AnalysisException] {
+        sql("SELECT testArraySum(1, 2)")
+      }.getMessage
+      assert(msg2.contains(s"No handler for UDF/UDAF/UDTF '${classOf[ArraySumUDF].getName}'"))
+    }
+  }
 }
 
 class TestPair(x: Int, y: Int) extends Writable with Serializable {
@@ -741,3 +761,14 @@ class StatelessUDF extends UDF {
     result
   }
 }
+
+class ArraySumUDF extends UDF {
+  import scala.collection.JavaConverters._
+  def evaluate(values: java.util.List[java.lang.Double]): java.lang.Double = {
+    var r = 0d
+    for (v <- values.asScala) {
+      r += v
+    }
+    r
+  }
+}

From 4425c3a9c35d4ab59976d8a409d18c933a5f9180 Mon Sep 17 00:00:00 2001
From: Kris Mok <kris.mok@databricks.com>
Date: Sat, 26 Sep 2020 16:03:59 -0700
Subject: [PATCH 1071/1280] [SPARK-32999][SQL] Use Utils.getSimpleName to avoid
 hitting Malformed class name in TreeNode

### What changes were proposed in this pull request?

Use `Utils.getSimpleName` to avoid hitting `Malformed class name` error in `TreeNode`.

### Why are the changes needed?

On older JDK versions (e.g. JDK8u), nested Scala classes may trigger `java.lang.Class.getSimpleName` to throw an `java.lang.InternalError: Malformed class name` error.

Similar to https://github.com/apache/spark/pull/29050, we should use  Spark's `Utils.getSimpleName` utility function in place of `Class.getSimpleName` to avoid hitting the issue.

### Does this PR introduce _any_ user-facing change?

Fixes a bug that throws an error when invoking `TreeNode.nodeName`, otherwise no changes.

### How was this patch tested?

Added new unit test case in `TreeNodeSuite`. Note that the test case assumes the test code can trigger the expected error, otherwise it'll skip the test safely, for compatibility with newer JDKs.

Manually tested on JDK8u and JDK11u and observed expected behavior:
- JDK8u: the test case triggers the "Malformed class name" issue and the fix works;
- JDK11u: the test case does not trigger the "Malformed class name" issue, and the test case is safely skipped.

Closes #29875 from rednaxelafx/spark-32999-getsimplename.

Authored-by: Kris Mok <kris.mok@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 9a155d42a3202fbafc48f8b722bbc27cce522e11)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/trees/TreeNode.scala   |  9 ++++---
 .../sql/catalyst/trees/TreeNodeSuite.scala    | 26 +++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index 4c7474299587b..4dc3627cd6a50 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.util.Utils
 
 /** Used by [[TreeNode.getNodeNumbered]] when traversing the tree for a given number */
 private class MutableInt(var i: Int)
@@ -516,11 +517,13 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
     mapChildren(_.clone(), forceCopy = true)
   }
 
+  private def simpleClassName: String = Utils.getSimpleName(this.getClass)
+
   /**
    * Returns the name of this type of TreeNode.  Defaults to the class name.
    * Note that we remove the "Exec" suffix for physical operators here.
    */
-  def nodeName: String = getClass.getSimpleName.replaceAll("Exec$", "")
+  def nodeName: String = simpleClassName.replaceAll("Exec$", "")
 
   /**
    * The arguments that should be included in the arg string.  Defaults to the `productIterator`.
@@ -739,7 +742,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
   protected def jsonFields: List[JField] = {
     val fieldNames = getConstructorParameterNames(getClass)
     val fieldValues = productIterator.toSeq ++ otherCopyArgs
-    assert(fieldNames.length == fieldValues.length, s"${getClass.getSimpleName} fields: " +
+    assert(fieldNames.length == fieldValues.length, s"$simpleClassName fields: " +
       fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(", "))
 
     fieldNames.zip(fieldValues).map {
@@ -793,7 +796,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
       try {
         val fieldNames = getConstructorParameterNames(p.getClass)
         val fieldValues = p.productIterator.toSeq
-        assert(fieldNames.length == fieldValues.length, s"${getClass.getSimpleName} fields: " +
+        assert(fieldNames.length == fieldValues.length, s"$simpleClassName fields: " +
           fieldNames.mkString(", ") + s", values: " + fieldValues.mkString(", "))
         ("product-class" -> JString(p.getClass.getName)) :: fieldNames.zip(fieldValues).map {
           case (name, value) => name -> parseToJson(value)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index f5259706325eb..8ee64ed20a2a6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -734,4 +734,30 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
     assertDifferentInstance(leaf, leafCloned)
     assert(leaf.child.eq(leafCloned.asInstanceOf[FakeLeafPlan].child))
   }
+
+  object MalformedClassObject extends Serializable {
+    case class MalformedNameExpression(child: Expression) extends TaggingExpression
+  }
+
+  test("SPARK-32999: TreeNode.nodeName should not throw malformed class name error") {
+    val testTriggersExpectedError = try {
+      classOf[MalformedClassObject.MalformedNameExpression].getSimpleName
+      false
+    } catch {
+      case ex: java.lang.InternalError if ex.getMessage.contains("Malformed class name") =>
+        true
+      case ex: Throwable => throw ex
+    }
+    // This test case only applies on older JDK versions (e.g. JDK8u), and doesn't trigger the
+    // issue on newer JDK versions (e.g. JDK11u).
+    assume(testTriggersExpectedError, "the test case didn't trigger malformed class name error")
+
+    val expr = MalformedClassObject.MalformedNameExpression(Literal(1))
+    try {
+      expr.nodeName
+    } catch {
+      case ex: java.lang.InternalError if ex.getMessage.contains("Malformed class name") =>
+        fail("TreeNode.nodeName should not throw malformed class name error")
+    }
+  }
 }

From 424f16e8cbe7e9ba02e808bf084cd7361388e377 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 29 Sep 2020 05:13:01 +0000
Subject: [PATCH 1072/1280] [SPARK-33015][SQL] Compute the current date only
 once

### What changes were proposed in this pull request?
Compute the current date at the specified time zone using timestamp taken at the start of query evaluation.

### Why are the changes needed?
According to the doc for [current_date()](http://spark.apache.org/docs/latest/api/sql/#current_date), the current date should be computed at the start of query evaluation but it can be computed multiple times. As a consequence of that, the function can return different values if the query is executed at the border of two dates.

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
By existing test suites `ComputeCurrentTimeSuite` and `DateExpressionsSuite`.

Closes #29889 from MaxGekk/fix-current_date.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 68cd5677ae0e3891e6bb4938a64ff98810656ba8)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/catalyst/optimizer/finishAnalysis.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 80d85827657fd..09e0118c994da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -79,10 +79,10 @@ object ComputeCurrentTime extends Rule[LogicalPlan] {
     val currentTime = Literal.create(timestamp, timeExpr.dataType)
 
     plan transformAllExpressions {
-      case CurrentDate(Some(timeZoneId)) =>
+      case currentDate @ CurrentDate(Some(timeZoneId)) =>
         currentDates.getOrElseUpdate(timeZoneId, {
           Literal.create(
-            LocalDate.now(DateTimeUtils.getZoneId(timeZoneId)),
+            DateTimeUtils.microsToDays(timestamp, currentDate.zoneId),
             DateType)
         })
       case CurrentTimestamp() | Now() => currentTime

From 118de10b5963427e88820704a02af97b73b26942 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 29 Sep 2020 05:20:12 +0000
Subject: [PATCH 1073/1280] [MINOR][DOCS] Document when `current_date` and
 `current_timestamp` are evaluated

### What changes were proposed in this pull request?
Explicitly document that `current_date` and `current_timestamp` are executed at the start of query evaluation. And all calls of `current_date`/`current_timestamp` within the same query return the same value

### Why are the changes needed?
Users could expect that `current_date` and `current_timestamp` return the current date/timestamp at the moment of query execution but in fact the functions are folded by the optimizer at the start of query evaluation:
https://github.com/apache/spark/blob/0df8dd60733066076967f0525210bbdb5e12415a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala#L71-L91

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
by running `./dev/scalastyle`.

Closes #29892 from MaxGekk/doc-current_date.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 1b60ff5afea0637f74c5f064642225b35b13b069)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 R/pkg/R/functions.R                                  |  6 ++++--
 python/pyspark/sql/functions.py                      |  6 ++++--
 .../catalyst/expressions/datetimeExpressions.scala   | 12 ++++++------
 .../main/scala/org/apache/spark/sql/functions.scala  |  6 ++++--
 4 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R
index edec2f4d2e56e..4cb23d72ef51a 100644
--- a/R/pkg/R/functions.R
+++ b/R/pkg/R/functions.R
@@ -4028,7 +4028,8 @@ setMethod("date_trunc",
           })
 
 #' @details
-#' \code{current_date}: Returns the current date as a date column.
+#' \code{current_date}: Returns the current date at the start of query evaluation as a date column.
+#' All calls of current_date within the same query return the same value.
 #'
 #' @rdname column_datetime_functions
 #' @aliases current_date current_date,missing-method
@@ -4044,7 +4045,8 @@ setMethod("current_date",
           })
 
 #' @details
-#' \code{current_timestamp}: Returns the current timestamp as a timestamp column.
+#' \code{current_timestamp}: Returns the current timestamp at the start of query evaluation as
+#' a timestamp column. All calls of current_timestamp within the same query return the same value.
 #'
 #' @rdname column_datetime_functions
 #' @aliases current_timestamp current_timestamp,missing-method
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 0bf6bc65384bd..041f841817657 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -909,7 +909,8 @@ def ntile(n):
 @since(1.5)
 def current_date():
     """
-    Returns the current date as a :class:`DateType` column.
+    Returns the current date at the start of query evaluation as a :class:`DateType` column.
+    All calls of current_date within the same query return the same value.
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.current_date())
@@ -917,7 +918,8 @@ def current_date():
 
 def current_timestamp():
     """
-    Returns the current timestamp as a :class:`TimestampType` column.
+    Returns the current timestamp at the start of query evaluation as a :class:`TimestampType`
+    column. All calls of current_timestamp within the same query return the same value.
     """
     sc = SparkContext._active_spark_context
     return Column(sc._jvm.functions.current_timestamp())
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 62f38b315b109..b2ff2fa609b99 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -58,13 +58,12 @@ trait TimeZoneAwareExpression extends Expression {
 
 /**
  * Returns the current date at the start of query evaluation.
- * All calls of current_date within the same query return the same value.
- *
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
-    _FUNC_() - Returns the current date at the start of query evaluation.
+    _FUNC_() - Returns the current date at the start of query evaluation. All calls of current_date within the same query return the same value.
 
     _FUNC_ - Returns the current date at the start of query evaluation.
   """,
@@ -80,6 +79,7 @@ trait TimeZoneAwareExpression extends Expression {
   """,
   group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class CurrentDate(timeZoneId: Option[String] = None)
   extends LeafExpression with TimeZoneAwareExpression with CodegenFallback {
 
@@ -107,13 +107,12 @@ abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallbac
 
 /**
  * Returns the current timestamp at the start of query evaluation.
- * All calls of current_timestamp within the same query return the same value.
- *
  * There is no code generation since this expression should get constant folded by the optimizer.
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
-    _FUNC_() - Returns the current timestamp at the start of query evaluation.
+    _FUNC_() - Returns the current timestamp at the start of query evaluation. All calls of current_timestamp within the same query return the same value.
 
     _FUNC_ - Returns the current timestamp at the start of query evaluation.
   """,
@@ -129,6 +128,7 @@ abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallbac
   """,
   group = "datetime_funcs",
   since = "1.5.0")
+// scalastyle:on line.size.limit
 case class CurrentTimestamp() extends CurrentTimestampLike {
   override def prettyName: String = "current_timestamp"
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index cf889f5616671..ff8ee2555276a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -2692,7 +2692,8 @@ object functions {
   }
 
   /**
-   * Returns the current date as a date column.
+   * Returns the current date at the start of query evaluation as a date column.
+   * All calls of current_date within the same query return the same value.
    *
    * @group datetime_funcs
    * @since 1.5.0
@@ -2700,7 +2701,8 @@ object functions {
   def current_date(): Column = withExpr { CurrentDate() }
 
   /**
-   * Returns the current timestamp as a timestamp column.
+   * Returns the current timestamp at the start of query evaluation as a timestamp column.
+   * All calls of current_timestamp within the same query return the same value.
    *
    * @group datetime_funcs
    * @since 1.5.0

From 97d8634450b39c1f4e5308b8a5308650e1e7489a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 28 Sep 2020 21:54:00 -0700
Subject: [PATCH 1074/1280] [SPARK-33021][PYTHON][TESTS] Move functions related
 test cases into test_functions.py

Move functions related test cases from `test_context.py` to `test_functions.py`.

To group the similar test cases.

Nope, test-only.

Jenkins and GitHub Actions should test.

Closes #29898 from HyukjinKwon/SPARK-33021.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 python/pyspark/sql/tests/test_context.py   | 101 --------------------
 python/pyspark/sql/tests/test_functions.py | 102 ++++++++++++++++++++-
 2 files changed, 101 insertions(+), 102 deletions(-)

diff --git a/python/pyspark/sql/tests/test_context.py b/python/pyspark/sql/tests/test_context.py
index 92e54340f8143..3a0c7bb913519 100644
--- a/python/pyspark/sql/tests/test_context.py
+++ b/python/pyspark/sql/tests/test_context.py
@@ -30,7 +30,6 @@
 from pyspark import SparkContext, SQLContext
 from pyspark.sql import Row, SparkSession
 from pyspark.sql.types import *
-from pyspark.sql.window import Window
 from pyspark.testing.utils import ReusedPySparkTestCase
 
 
@@ -112,99 +111,6 @@ def test_save_and_load_table(self):
 
         shutil.rmtree(tmpPath)
 
-    def test_window_functions(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        w = Window.partitionBy("value").orderBy("key")
-        from pyspark.sql import functions as F
-        sel = df.select(df.value, df.key,
-                        F.max("key").over(w.rowsBetween(0, 1)),
-                        F.min("key").over(w.rowsBetween(0, 1)),
-                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
-                        F.row_number().over(w),
-                        F.rank().over(w),
-                        F.dense_rank().over(w),
-                        F.ntile(2).over(w))
-        rs = sorted(sel.collect())
-        expected = [
-            ("1", 1, 1, 1, 1, 1, 1, 1, 1),
-            ("2", 1, 1, 1, 3, 1, 1, 1, 1),
-            ("2", 1, 2, 1, 3, 2, 1, 1, 1),
-            ("2", 2, 2, 2, 3, 3, 3, 2, 2)
-        ]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_window_functions_without_partitionBy(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        w = Window.orderBy("key", df.value)
-        from pyspark.sql import functions as F
-        sel = df.select(df.value, df.key,
-                        F.max("key").over(w.rowsBetween(0, 1)),
-                        F.min("key").over(w.rowsBetween(0, 1)),
-                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
-                        F.row_number().over(w),
-                        F.rank().over(w),
-                        F.dense_rank().over(w),
-                        F.ntile(2).over(w))
-        rs = sorted(sel.collect())
-        expected = [
-            ("1", 1, 1, 1, 4, 1, 1, 1, 1),
-            ("2", 1, 1, 1, 4, 2, 2, 2, 1),
-            ("2", 1, 2, 1, 4, 3, 2, 2, 2),
-            ("2", 2, 2, 2, 4, 4, 4, 3, 2)
-        ]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_window_functions_cumulative_sum(self):
-        df = self.spark.createDataFrame([("one", 1), ("two", 2)], ["key", "value"])
-        from pyspark.sql import functions as F
-
-        # Test cumulative sum
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding, 0)))
-        rs = sorted(sel.collect())
-        expected = [("one", 1), ("two", 3)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-        # Test boundary values less than JVM's Long.MinValue and make sure we don't overflow
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding - 1, 0)))
-        rs = sorted(sel.collect())
-        expected = [("one", 1), ("two", 3)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-        # Test boundary values greater than JVM's Long.MaxValue and make sure we don't overflow
-        frame_end = Window.unboundedFollowing + 1
-        sel = df.select(
-            df.key,
-            F.sum(df.value).over(Window.rowsBetween(Window.currentRow, frame_end)))
-        rs = sorted(sel.collect())
-        expected = [("one", 3), ("two", 2)]
-        for r, ex in zip(rs, expected):
-            self.assertEqual(tuple(r), ex[:len(r)])
-
-    def test_collect_functions(self):
-        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
-        from pyspark.sql import functions
-
-        self.assertEqual(
-            sorted(df.select(functions.collect_set(df.key).alias('r')).collect()[0].r),
-            [1, 2])
-        self.assertEqual(
-            sorted(df.select(functions.collect_list(df.key).alias('r')).collect()[0].r),
-            [1, 1, 1, 2])
-        self.assertEqual(
-            sorted(df.select(functions.collect_set(df.value).alias('r')).collect()[0].r),
-            ["1", "2"])
-        self.assertEqual(
-            sorted(df.select(functions.collect_list(df.value).alias('r')).collect()[0].r),
-            ["1", "2", "2", "2"])
-
     def test_limit_and_take(self):
         df = self.spark.range(1, 1000, numPartitions=10)
 
@@ -223,13 +129,6 @@ def assert_runs_only_one_job_stage_and_task(job_group_name, f):
         # Regression test for SPARK-17514: limit(n).collect() should the perform same as take(n)
         assert_runs_only_one_job_stage_and_task("collect_limit", lambda: df.limit(1).collect())
 
-    def test_datetime_functions(self):
-        from pyspark.sql import functions
-        from datetime import date
-        df = self.spark.range(1).selectExpr("'2017-01-22' as dateCol")
-        parse_result = df.select(functions.to_date(functions.col("dateCol"))).first()
-        self.assertEquals(date(2017, 1, 22), parse_result['to_date(`dateCol`)'])
-
     def test_unbounded_frames(self):
         from pyspark.sql import functions as F
         from pyspark.sql import window
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index fa9ee57ff5f90..fd2ad22fa7b78 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -18,7 +18,7 @@
 import datetime
 import sys
 
-from pyspark.sql import Row
+from pyspark.sql import Row, Window
 from pyspark.sql.functions import udf, input_file_name
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 
@@ -337,6 +337,106 @@ def test_overlay(self):
 
         self.assertListEqual(actual, expected)
 
+    def test_window_functions(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        w = Window.partitionBy("value").orderBy("key")
+        from pyspark.sql import functions as F
+        sel = df.select(df.value, df.key,
+                        F.max("key").over(w.rowsBetween(0, 1)),
+                        F.min("key").over(w.rowsBetween(0, 1)),
+                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
+                        F.row_number().over(w),
+                        F.rank().over(w),
+                        F.dense_rank().over(w),
+                        F.ntile(2).over(w))
+        rs = sorted(sel.collect())
+        expected = [
+            ("1", 1, 1, 1, 1, 1, 1, 1, 1),
+            ("2", 1, 1, 1, 3, 1, 1, 1, 1),
+            ("2", 1, 2, 1, 3, 2, 1, 1, 1),
+            ("2", 2, 2, 2, 3, 3, 3, 2, 2)
+        ]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_window_functions_without_partitionBy(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        w = Window.orderBy("key", df.value)
+        from pyspark.sql import functions as F
+        sel = df.select(df.value, df.key,
+                        F.max("key").over(w.rowsBetween(0, 1)),
+                        F.min("key").over(w.rowsBetween(0, 1)),
+                        F.count("key").over(w.rowsBetween(float('-inf'), float('inf'))),
+                        F.row_number().over(w),
+                        F.rank().over(w),
+                        F.dense_rank().over(w),
+                        F.ntile(2).over(w))
+        rs = sorted(sel.collect())
+        expected = [
+            ("1", 1, 1, 1, 4, 1, 1, 1, 1),
+            ("2", 1, 1, 1, 4, 2, 2, 2, 1),
+            ("2", 1, 2, 1, 4, 3, 2, 2, 2),
+            ("2", 2, 2, 2, 4, 4, 4, 3, 2)
+        ]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_window_functions_cumulative_sum(self):
+        df = self.spark.createDataFrame([("one", 1), ("two", 2)], ["key", "value"])
+        from pyspark.sql import functions as F
+
+        # Test cumulative sum
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding, 0)))
+        rs = sorted(sel.collect())
+        expected = [("one", 1), ("two", 3)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+        # Test boundary values less than JVM's Long.MinValue and make sure we don't overflow
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.unboundedPreceding - 1, 0)))
+        rs = sorted(sel.collect())
+        expected = [("one", 1), ("two", 3)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+        # Test boundary values greater than JVM's Long.MaxValue and make sure we don't overflow
+        frame_end = Window.unboundedFollowing + 1
+        sel = df.select(
+            df.key,
+            F.sum(df.value).over(Window.rowsBetween(Window.currentRow, frame_end)))
+        rs = sorted(sel.collect())
+        expected = [("one", 3), ("two", 2)]
+        for r, ex in zip(rs, expected):
+            self.assertEqual(tuple(r), ex[:len(r)])
+
+    def test_collect_functions(self):
+        df = self.spark.createDataFrame([(1, "1"), (2, "2"), (1, "2"), (1, "2")], ["key", "value"])
+        from pyspark.sql import functions
+
+        self.assertEqual(
+            sorted(df.select(functions.collect_set(df.key).alias('r')).collect()[0].r),
+            [1, 2])
+        self.assertEqual(
+            sorted(df.select(functions.collect_list(df.key).alias('r')).collect()[0].r),
+            [1, 1, 1, 2])
+        self.assertEqual(
+            sorted(df.select(functions.collect_set(df.value).alias('r')).collect()[0].r),
+            ["1", "2"])
+        self.assertEqual(
+            sorted(df.select(functions.collect_list(df.value).alias('r')).collect()[0].r),
+            ["1", "2", "2", "2"])
+
+    def test_datetime_functions(self):
+        from pyspark.sql import functions
+        from datetime import date
+        df = self.spark.range(1).selectExpr("'2017-01-22' as dateCol")
+        parse_result = df.select(functions.to_date(functions.col("dateCol"))).first()
+        self.assertEquals(date(2017, 1, 22), parse_result['to_date(`dateCol`)'])
+
 
 if __name__ == "__main__":
     import unittest

From 2160dc52163f017bc164ad18ca6ebe6868070402 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 29 Sep 2020 19:34:43 +0900
Subject: [PATCH 1075/1280] [SPARK-33015][SQL][FOLLOWUP][3.0] Use
 millisToDays() in the ComputeCurrentTime rule

### What changes were proposed in this pull request?
Use `millisToDays()` instead of `microsToDays()` because the former one is not available in `branch-3.0`.

### Why are the changes needed?
To fix the build failure:
```
[ERROR] [Error] /home/jenkins/workspace/spark-branch-3.0-maven-snapshots/spark/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala:85: value microsToDays is not a member of object org.apache.spark.sql.catalyst.util.DateTimeUtils
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `./build/sbt clean package` and `ComputeCurrentTimeSuite`.

Closes #29901 from MaxGekk/fix-current_date-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/catalyst/optimizer/finishAnalysis.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
index 09e0118c994da..ba7e85285db2f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala
@@ -82,7 +82,7 @@ object ComputeCurrentTime extends Rule[LogicalPlan] {
       case currentDate @ CurrentDate(Some(timeZoneId)) =>
         currentDates.getOrElseUpdate(timeZoneId, {
           Literal.create(
-            DateTimeUtils.microsToDays(timestamp, currentDate.zoneId),
+            DateTimeUtils.millisToDays(DateTimeUtils.toMillis(timestamp), currentDate.zoneId),
             DateType)
         })
       case CurrentTimestamp() | Now() => currentTime

From d3cc564de2e27d6f40d360a35ac86d17b39f1498 Mon Sep 17 00:00:00 2001
From: Tom van Bussel <tom.vanbussel@databricks.com>
Date: Tue, 29 Sep 2020 13:05:33 +0200
Subject: [PATCH 1076/1280] [SPARK-32901][CORE] Do not allocate memory while
 spilling UnsafeExternalSorter

### What changes were proposed in this pull request?

This PR changes `UnsafeExternalSorter` to no longer allocate any memory while spilling. In particular it removes the allocation of a new pointer array in `UnsafeInMemorySorter`. Instead the new pointer array is allocated whenever the next record is inserted into the sorter.

### Why are the changes needed?

Without this change the `UnsafeExternalSorter` could throw an OOM while spilling. The following sequence of events would have triggered an OOM:

1. `UnsafeExternalSorter` runs out of space in its pointer array and attempts to allocate a new large array to replace the old one.
2. `TaskMemoryManager` tries to allocate the memory backing the new large array using `MemoryManager`, but `MemoryManager` is only willing to return most but not all of the memory requested.
3. `TaskMemoryManager` asks `UnsafeExternalSorter` to spill, which causes `UnsafeExternalSorter` to spill the current run to disk, to free its record pages and to reset its `UnsafeInMemorySorter`.
4. `UnsafeInMemorySorter` frees the old pointer array, and tries to allocate a new small pointer array.
5. `TaskMemoryManager` tries to allocate the memory backing the small array using `MemoryManager`, but `MemoryManager` is unwilling to give it any memory, as the `TaskMemoryManager` is still holding on to the memory it got for the new large array.
6. `TaskMemoryManager` again asks `UnsafeExternalSorter` to spill, but this time there is nothing to spill.
7. `UnsafeInMemorySorter` receives less memory than it requested, and causes a `SparkOutOfMemoryError` to be thrown, which causes the current task to fail.

With the changes in the PR the following will happen instead:

1. `UnsafeExternalSorter` runs out of space in its pointer array and attempts to allocate a new large array to replace the old one.
2. `TaskMemoryManager` tries to allocate the memory backing the new large array using `MemoryManager`, but `MemoryManager` is only willing to return most but not all of the memory requested.
3. `TaskMemoryManager` asks `UnsafeExternalSorter` to spill, which causes `UnsafeExternalSorter` to spill the current run to disk, to free its record pages and to reset its `UnsafeInMemorySorter`.
4. `UnsafeInMemorySorter` frees the old pointer array.
5. `TaskMemoryManager` returns control to `UnsafeExternalSorter.growPointerArrayIfNecessary` (either by returning the the new large array or by throwing a `SparkOutOfMemoryError`).
6. `UnsafeExternalSorter` either frees the new large array or it ignores the `SparkOutOfMemoryError` depending on what happened in the previous step.
7. `UnsafeExternalSorter` successfully allocates a new small pointer array and operation continues as normal.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Tests were added in `UnsafeExternalSorterSuite` and `UnsafeInMemorySorterSuite`.

Closes #29785 from tomvanbussel/SPARK-32901.

Authored-by: Tom van Bussel <tom.vanbussel@databricks.com>
Signed-off-by: herman <herman@databricks.com>
(cherry picked from commit f167002522d50eefb261c8ba2d66a23b781a38c4)
Signed-off-by: herman <herman@databricks.com>
---
 .../unsafe/sort/UnsafeExternalSorter.java     | 96 ++++++++++++++-----
 .../unsafe/sort/UnsafeInMemorySorter.java     | 55 +++++------
 .../sort/UnsafeExternalSorterSuite.java       | 46 ++++-----
 .../sort/UnsafeInMemorySorterSuite.java       | 40 ++++----
 .../spark/memory/TestMemoryManager.scala      |  8 ++
 5 files changed, 143 insertions(+), 102 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index 71b9a5bc11542..2096453d8060a 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -203,6 +203,10 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
     }
 
     if (inMemSorter == null || inMemSorter.numRecords() <= 0) {
+      // There could still be some memory allocated when there are no records in the in-memory
+      // sorter. We will not spill it however, to ensure that we can always process at least one
+      // record before spilling. See the comments in `allocateMemoryForRecordIfNecessary` for why
+      // this is necessary.
       return 0L;
     }
 
@@ -224,7 +228,7 @@ public long spill(long size, MemoryConsumer trigger) throws IOException {
     // Note that this is more-or-less going to be a multiple of the page size, so wasted space in
     // pages will currently be counted as memory spilled even though that space isn't actually
     // written to disk. This also counts the space needed to store the sorter's pointer array.
-    inMemSorter.reset();
+    inMemSorter.freeMemory();
     // Reset the in-memory sorter's pointer array only after freeing up the memory pages holding the
     // records. Otherwise, if the task is over allocated memory, then without freeing the memory
     // pages, we might not be able to get memory for the pointer array.
@@ -325,7 +329,7 @@ public void cleanupResources() {
       deleteSpillFiles();
       freeMemory();
       if (inMemSorter != null) {
-        inMemSorter.free();
+        inMemSorter.freeMemory();
         inMemSorter = null;
       }
     }
@@ -339,40 +343,53 @@ public void cleanupResources() {
   private void growPointerArrayIfNecessary() throws IOException {
     assert(inMemSorter != null);
     if (!inMemSorter.hasSpaceForAnotherRecord()) {
+      if (inMemSorter.numRecords() <= 0) {
+        // Spilling was triggered just before this method was called. The pointer array was freed
+        // during the spill, so a new pointer array needs to be allocated here.
+        LongArray array = allocateArray(inMemSorter.getInitialSize());
+        inMemSorter.expandPointerArray(array);
+        return;
+      }
+
       long used = inMemSorter.getMemoryUsage();
-      LongArray array;
+      LongArray array = null;
       try {
         // could trigger spilling
         array = allocateArray(used / 8 * 2);
       } catch (TooLargePageException e) {
         // The pointer array is too big to fix in a single page, spill.
         spill();
-        return;
       } catch (SparkOutOfMemoryError e) {
-        // should have trigger spilling
-        if (!inMemSorter.hasSpaceForAnotherRecord()) {
+        if (inMemSorter.numRecords() > 0) {
           logger.error("Unable to grow the pointer array");
           throw e;
         }
-        return;
+        // The new array could not be allocated, but that is not an issue as it is longer needed,
+        // as all records were spilled.
       }
-      // check if spilling is triggered or not
-      if (inMemSorter.hasSpaceForAnotherRecord()) {
-        freeArray(array);
-      } else {
-        inMemSorter.expandPointerArray(array);
+
+      if (inMemSorter.numRecords() <= 0) {
+        // Spilling was triggered while trying to allocate the new array.
+        if (array != null) {
+          // We succeeded in allocating the new array, but, since all records were spilled, a
+          // smaller array would also suffice.
+          freeArray(array);
+        }
+        // The pointer array was freed during the spill, so a new pointer array needs to be
+        // allocated here.
+        array = allocateArray(inMemSorter.getInitialSize());
       }
+      inMemSorter.expandPointerArray(array);
     }
   }
 
   /**
-   * Allocates more memory in order to insert an additional record. This will request additional
-   * memory from the memory manager and spill if the requested memory can not be obtained.
+   * Allocates an additional page in order to insert an additional record. This will request
+   * additional memory from the memory manager and spill if the requested memory can not be
+   * obtained.
    *
    * @param required the required space in the data page, in bytes, including space for storing
-   *                      the record size. This must be less than or equal to the page size (records
-   *                      that exceed the page size are handled via a different code path which uses
-   *                      special overflow pages).
+   *                 the record size.
    */
   private void acquireNewPageIfNecessary(int required) {
     if (currentPage == null ||
@@ -384,6 +401,37 @@ private void acquireNewPageIfNecessary(int required) {
     }
   }
 
+  /**
+   * Allocates more memory in order to insert an additional record. This will request additional
+   * memory from the memory manager and spill if the requested memory can not be obtained.
+   *
+   * @param required the required space in the data page, in bytes, including space for storing
+   *                 the record size.
+   */
+  private void allocateMemoryForRecordIfNecessary(int required) throws IOException {
+    // Step 1:
+    // Ensure that the pointer array has space for another record. This may cause a spill.
+    growPointerArrayIfNecessary();
+    // Step 2:
+    // Ensure that the last page has space for another record. This may cause a spill.
+    acquireNewPageIfNecessary(required);
+    // Step 3:
+    // The allocation in step 2 could have caused a spill, which would have freed the pointer
+    // array allocated in step 1. Therefore we need to check again whether we have to allocate
+    // a new pointer array.
+    //
+    // If the allocation in this step causes a spill event then it will not cause the page
+    // allocated in the previous step to be freed. The function `spill` only frees memory if at
+    // least one record has been inserted in the in-memory sorter. This will not be the case if
+    // we have spilled in the previous step.
+    //
+    // If we did not spill in the previous step then `growPointerArrayIfNecessary` will be a
+    // no-op that does not allocate any memory, and therefore can't cause a spill event.
+    //
+    // Thus there is no need to call `acquireNewPageIfNecessary` again after this step.
+    growPointerArrayIfNecessary();
+  }
+
   /**
    * Write a record to the sorter.
    */
@@ -398,11 +446,10 @@ public void insertRecord(
       spill();
     }
 
-    growPointerArrayIfNecessary();
-    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final int uaoSize = UnsafeAlignedOffset.getUaoSize();
     // Need 4 or 8 bytes to store the record length.
     final int required = length + uaoSize;
-    acquireNewPageIfNecessary(required);
+    allocateMemoryForRecordIfNecessary(required);
 
     final Object base = currentPage.getBaseObject();
     final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
@@ -425,10 +472,9 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
       Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull)
     throws IOException {
 
-    growPointerArrayIfNecessary();
-    int uaoSize = UnsafeAlignedOffset.getUaoSize();
+    final int uaoSize = UnsafeAlignedOffset.getUaoSize();
     final int required = keyLen + valueLen + (2 * uaoSize);
-    acquireNewPageIfNecessary(required);
+    allocateMemoryForRecordIfNecessary(required);
 
     final Object base = currentPage.getBaseObject();
     final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor);
@@ -561,7 +607,7 @@ public long spill() throws IOException {
         assert(inMemSorter != null);
         released += inMemSorter.getMemoryUsage();
         totalSortTimeNanos += inMemSorter.getSortTimeNanos();
-        inMemSorter.free();
+        inMemSorter.freeMemory();
         inMemSorter = null;
         taskContext.taskMetrics().incMemoryBytesSpilled(released);
         taskContext.taskMetrics().incDiskBytesSpilled(writeMetrics.bytesWritten());
@@ -656,7 +702,7 @@ public UnsafeSorterIterator getIterator(int startIndex) throws IOException {
         }
         i += spillWriter.recordsSpilled();
       }
-      if (inMemSorter != null) {
+      if (inMemSorter != null && inMemSorter.numRecords() > 0) {
         UnsafeSorterIterator iter = inMemSorter.getSortedIterator();
         moveOver(iter, startIndex - i);
         queue.add(iter);
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
index ff641a24a7b3e..33be899b6b438 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
@@ -159,32 +159,26 @@ private int getUsableCapacity() {
     return (int) (array.size() / (radixSortSupport != null ? 2 : 1.5));
   }
 
+  public long getInitialSize() {
+    return initialSize;
+  }
+
   /**
    * Free the memory used by pointer array.
    */
-  public void free() {
+  public void freeMemory() {
     if (consumer != null) {
       if (array != null) {
         consumer.freeArray(array);
       }
-      array = null;
-    }
-  }
 
-  public void reset() {
-    if (consumer != null) {
-      consumer.freeArray(array);
-      // the call to consumer.allocateArray may trigger a spill which in turn access this instance
-      // and eventually re-enter this method and try to free the array again.  by setting the array
-      // to null and its length to 0 we effectively make the spill code-path a no-op.  setting the
-      // array to null also indicates that it has already been de-allocated which prevents a double
-      // de-allocation in free().
+      // Set the array to null instead of allocating a new array. Allocating an array could have
+      // triggered another spill and this method already is called from UnsafeExternalSorter when
+      // spilling. Attempting to allocate while spilling is dangerous, as we could be holding onto
+      // a large partially complete allocation, which may prevent other memory from being allocated.
+      // Instead we will allocate the new array when it is necessary.
       array = null;
       usableCapacity = 0;
-      pos = 0;
-      nullBoundaryPos = 0;
-      array = consumer.allocateArray(initialSize);
-      usableCapacity = getUsableCapacity();
     }
     pos = 0;
     nullBoundaryPos = 0;
@@ -217,18 +211,20 @@ public boolean hasSpaceForAnotherRecord() {
   }
 
   public void expandPointerArray(LongArray newArray) {
-    if (newArray.size() < array.size()) {
-      // checkstyle.off: RegexpSinglelineJava
-      throw new SparkOutOfMemoryError("Not enough memory to grow pointer array");
-      // checkstyle.on: RegexpSinglelineJava
+    if (array != null) {
+      if (newArray.size() < array.size()) {
+        // checkstyle.off: RegexpSinglelineJava
+        throw new SparkOutOfMemoryError("Not enough memory to grow pointer array");
+        // checkstyle.on: RegexpSinglelineJava
+      }
+      Platform.copyMemory(
+        array.getBaseObject(),
+        array.getBaseOffset(),
+        newArray.getBaseObject(),
+        newArray.getBaseOffset(),
+        pos * 8L);
+      consumer.freeArray(array);
     }
-    Platform.copyMemory(
-      array.getBaseObject(),
-      array.getBaseOffset(),
-      newArray.getBaseObject(),
-      newArray.getBaseOffset(),
-      pos * 8L);
-    consumer.freeArray(array);
     array = newArray;
     usableCapacity = getUsableCapacity();
   }
@@ -347,6 +343,11 @@ public long getCurrentPageNumber() {
    * {@code next()} will return the same mutable object.
    */
   public UnsafeSorterIterator getSortedIterator() {
+    if (numRecords() == 0) {
+      // `array` might be null, so make sure that it is not accessed by returning early.
+      return new SortedIterator(0, 0);
+    }
+
     int offset = 0;
     long start = System.nanoTime();
     if (sortComparator != null) {
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
index 087d090c1c60e..9904a0a113fdd 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorterSuite.java
@@ -23,7 +23,6 @@
 import java.util.LinkedList;
 import java.util.UUID;
 
-import org.hamcrest.Matchers;
 import scala.Tuple2$;
 
 import org.junit.After;
@@ -38,7 +37,6 @@
 import org.apache.spark.executor.TaskMetrics;
 import org.apache.spark.internal.config.package$;
 import org.apache.spark.memory.TestMemoryManager;
-import org.apache.spark.memory.SparkOutOfMemoryError;
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.serializer.JavaSerializer;
 import org.apache.spark.serializer.SerializerInstance;
@@ -551,40 +549,28 @@ public void testGetIterator() throws Exception {
   }
 
   @Test
-  public void testOOMDuringSpill() throws Exception {
+  public void testNoOOMDuringSpill() throws Exception {
     final UnsafeExternalSorter sorter = newSorter();
-    // we assume that given default configuration,
-    // the size of the data we insert to the sorter (ints)
-    // and assuming we shouldn't spill before pointers array is exhausted
-    // (memory manager is not configured to throw at this point)
-    // - so this loop runs a reasonable number of iterations (<2000).
-    // test indeed completed within <30ms (on a quad i7 laptop).
-    for (int i = 0; sorter.hasSpaceForAnotherRecord(); ++i) {
+    for (int i = 0; i < 100; i++) {
       insertNumber(sorter, i);
     }
-    // we expect the next insert to attempt growing the pointerssArray first
-    // allocation is expected to fail, then a spill is triggered which
-    // attempts another allocation which also fails and we expect to see this
-    // OOM here.  the original code messed with a released array within the
-    // spill code and ended up with a failed assertion.  we also expect the
-    // location of the OOM to be
-    // org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset
-    memoryManager.markconsequentOOM(2);
-    try {
-      insertNumber(sorter, 1024);
-      fail("expected OutOfMmoryError but it seems operation surprisingly succeeded");
-    }
-    // we expect an SparkOutOfMemoryError here, anything else (i.e the original NPE is a failure)
-    catch (SparkOutOfMemoryError oom){
-      String oomStackTrace = Utils.exceptionString(oom);
-      assertThat("expected SparkOutOfMemoryError in " +
-        "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset",
-        oomStackTrace,
-        Matchers.containsString(
-          "org.apache.spark.util.collection.unsafe.sort.UnsafeInMemorySorter.reset"));
+
+    // Check that spilling still succeeds when the task is starved for memory.
+    memoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.spill();
+    memoryManager.resetConsequentOOM();
+
+    // Ensure that records can be appended after spilling, i.e. check that the sorter will allocate
+    // the new pointer array that it could not allocate while spilling.
+    for (int i = 0; i < 100; ++i) {
+      insertNumber(sorter, i);
     }
+
+    sorter.cleanupResources();
+    assertSpillFilesWereCleanedUp();
   }
 
+
   private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end)
       throws IOException {
     for (int i = start; i < end; i++) {
diff --git a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
index 2b8a0602730e1..9d4909ddce792 100644
--- a/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
+++ b/core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java
@@ -20,6 +20,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import org.apache.spark.unsafe.array.LongArray;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -27,7 +28,6 @@
 import org.apache.spark.SparkConf;
 import org.apache.spark.memory.TestMemoryConsumer;
 import org.apache.spark.memory.TestMemoryManager;
-import org.apache.spark.memory.SparkOutOfMemoryError;
 import org.apache.spark.memory.TaskMemoryManager;
 import org.apache.spark.unsafe.Platform;
 import org.apache.spark.unsafe.memory.MemoryBlock;
@@ -37,7 +37,6 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.isIn;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 
 public class UnsafeInMemorySorterSuite {
@@ -147,7 +146,7 @@ public int compare(
   }
 
   @Test
-  public void freeAfterOOM() {
+  public void testNoOOMDuringReset() {
     final SparkConf sparkConf = new SparkConf();
     sparkConf.set(package$.MODULE$.MEMORY_OFFHEAP_ENABLED(), false);
 
@@ -156,12 +155,7 @@ public void freeAfterOOM() {
     final TaskMemoryManager memoryManager = new TaskMemoryManager(
             testMemoryManager, 0);
     final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager);
-    final MemoryBlock dataPage = memoryManager.allocatePage(2048, consumer);
-    final Object baseObject = dataPage.getBaseObject();
-    // Write the records into the data page:
-    long position = dataPage.getBaseOffset();
 
-    final HashPartitioner hashPartitioner = new HashPartitioner(4);
     // Use integer comparison for comparing prefixes (which are partition ids, in this case)
     final PrefixComparator prefixComparator = PrefixComparators.LONG;
     final RecordComparator recordComparator = new RecordComparator() {
@@ -179,18 +173,24 @@ public int compare(
     UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager,
             recordComparator, prefixComparator, 100, shouldUseRadixSort());
 
-    testMemoryManager.markExecutionAsOutOfMemoryOnce();
-    try {
-      sorter.reset();
-      fail("expected SparkOutOfMemoryError but it seems operation surprisingly succeeded");
-    } catch (SparkOutOfMemoryError oom) {
-      // as expected
-    }
-    // [SPARK-21907] this failed on NPE at
-    // org.apache.spark.memory.MemoryConsumer.freeArray(MemoryConsumer.java:108)
-    sorter.free();
-    // simulate a 'back to back' free.
-    sorter.free();
+    // Ensure that the sorter does not OOM while freeing its memory.
+    testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.freeMemory();
+    testMemoryManager.resetConsequentOOM();
+    Assert.assertFalse(sorter.hasSpaceForAnotherRecord());
+
+    // Get the sorter in an usable state again by allocating a new pointer array.
+    LongArray array = consumer.allocateArray(1000);
+    sorter.expandPointerArray(array);
+
+    // Ensure that it is safe to call freeMemory() multiple times.
+    testMemoryManager.markconsequentOOM(Integer.MAX_VALUE);
+    sorter.freeMemory();
+    sorter.freeMemory();
+    testMemoryManager.resetConsequentOOM();
+    Assert.assertFalse(sorter.hasSpaceForAnotherRecord());
+
+    assertEquals(0L, memoryManager.cleanUpAllAllocatedMemory());
   }
 
 }
diff --git a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
index 60f67699f81be..987f383c9c4fa 100644
--- a/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
+++ b/core/src/test/scala/org/apache/spark/memory/TestMemoryManager.scala
@@ -119,6 +119,14 @@ class TestMemoryManager(conf: SparkConf)
     consequentOOM += n
   }
 
+  /**
+   * Undos the effects of [[markExecutionAsOutOfMemoryOnce]] and [[markconsequentOOM]] and lets
+   * calls to [[acquireExecutionMemory()]] (if there is enough memory available).
+   */
+  def resetConsequentOOM(): Unit = synchronized {
+    consequentOOM = 0
+  }
+
   def limit(avail: Long): Unit = synchronized {
     require(avail >= 0)
     available = avail

From 39bfae25979aecbe8058beb2a4882fde9f141eba Mon Sep 17 00:00:00 2001
From: Akshat Bordia <akshat.bordia31@gmail.com>
Date: Tue, 29 Sep 2020 08:38:43 -0500
Subject: [PATCH 1077/1280] [MINOR][DOCS] Fixing log message for better clarity

Fixing log message for better clarity.

Closes #29870 from akshatb1/master.

Lead-authored-by: Akshat Bordia <akshat.bordia31@gmail.com>
Co-authored-by: Akshat Bordia <akshat.bordia@citrix.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 7766fd13c9e7cb72b97fdfee224d3958fbe882a0)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 core/src/main/scala/org/apache/spark/SparkConf.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 40915e3904f7e..802100e05b728 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -577,7 +577,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
     // If spark.executor.heartbeatInterval bigger than spark.network.timeout,
     // it will almost always cause ExecutorLostFailure. See SPARK-22754.
     require(executorTimeoutThresholdMs > executorHeartbeatIntervalMs, "The value of " +
-      s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be no less than the value of " +
+      s"${networkTimeout}=${executorTimeoutThresholdMs}ms must be greater than the value of " +
       s"${EXECUTOR_HEARTBEAT_INTERVAL.key}=${executorHeartbeatIntervalMs}ms.")
   }
 

From ae8b35a0d24f8c83bbbb597d668875793a8dbca6 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Tue, 29 Sep 2020 16:46:04 +0000
Subject: [PATCH 1078/1280] [SPARK-33018][SQL] Fix estimate statistics issue if
 child has 0 bytes

### What changes were proposed in this pull request?

This pr fix estimate statistics issue if child has 0 bytes.

### Why are the changes needed?
The `sizeInBytes` can be `0` when AQE and CBO are enabled(`spark.sql.adaptive.enabled`=true, `spark.sql.cbo.enabled`=true and `spark.sql.cbo.planStats.enabled`=true). This will generate incorrect BroadcastJoin, resulting in Driver OOM. For example:
![SPARK-33018](https://user-images.githubusercontent.com/5399861/94457606-647e3d00-01e7-11eb-85ee-812ae6efe7bb.jpg)

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manual test.

Closes #29894 from wangyum/SPARK-33018.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 711d8dd28afd9af92b025f9908534e5f1d575042)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../SizeInBytesOnlyStatsPlanVisitor.scala     |  3 ++-
 .../statsEstimation/JoinEstimationSuite.scala | 22 +++++++++++++++++++
 .../StatsEstimationTestBase.scala             |  9 +++++---
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
index da36db7ae1f5f..a586988fd3253 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/SizeInBytesOnlyStatsPlanVisitor.scala
@@ -53,7 +53,8 @@ object SizeInBytesOnlyStatsPlanVisitor extends LogicalPlanVisitor[Statistics] {
    */
   override def default(p: LogicalPlan): Statistics = p match {
     case p: LeafNode => p.computeStats()
-    case _: LogicalPlan => Statistics(sizeInBytes = p.children.map(_.stats.sizeInBytes).product)
+    case _: LogicalPlan =>
+      Statistics(sizeInBytes = p.children.map(_.stats.sizeInBytes).filter(_ > 0L).product)
   }
 
   override def visitAggregate(p: Aggregate): Statistics = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
index 6c5a2b247fc23..cdfc863cc0212 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/JoinEstimationSuite.scala
@@ -551,4 +551,26 @@ class JoinEstimationSuite extends StatsEstimationTestBase {
       attributeStats = AttributeMap(Nil))
     assert(join.stats == expectedStats)
   }
+
+  test("SPARK-33018 Fix estimate statistics issue if child has 0 bytes") {
+    case class MyStatsTestPlan(
+        outputList: Seq[Attribute],
+        sizeInBytes: BigInt) extends LeafNode {
+      override def output: Seq[Attribute] = outputList
+      override def computeStats(): Statistics = Statistics(sizeInBytes = sizeInBytes)
+    }
+
+    val left = MyStatsTestPlan(
+      outputList = Seq("key-1-2", "key-2-4").map(nameToAttr),
+      sizeInBytes = BigInt(100))
+
+    val right = MyStatsTestPlan(
+      outputList = Seq("key-1-2", "key-2-3").map(nameToAttr),
+      sizeInBytes = BigInt(0))
+
+    val join = Join(left, right, LeftOuter,
+      Some(EqualTo(nameToAttr("key-2-4"), nameToAttr("key-2-3"))), JoinHint.NONE)
+
+    assert(join.stats == Statistics(sizeInBytes = 100))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
index 9dceca59f5b87..0a27e31b3c9f6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/StatsEstimationTestBase.scala
@@ -26,17 +26,20 @@ import org.apache.spark.sql.types.{IntegerType, StringType}
 
 trait StatsEstimationTestBase extends SparkFunSuite {
 
-  var originalValue: Boolean = false
+  var originalCBOValue: Boolean = false
+  var originalPlanStatsValue: Boolean = false
 
   override def beforeAll(): Unit = {
     super.beforeAll()
     // Enable stats estimation based on CBO.
-    originalValue = SQLConf.get.getConf(SQLConf.CBO_ENABLED)
+    originalCBOValue = SQLConf.get.getConf(SQLConf.CBO_ENABLED)
+    originalPlanStatsValue = SQLConf.get.getConf(SQLConf.PLAN_STATS_ENABLED)
     SQLConf.get.setConf(SQLConf.CBO_ENABLED, true)
   }
 
   override def afterAll(): Unit = {
-    SQLConf.get.setConf(SQLConf.CBO_ENABLED, originalValue)
+    SQLConf.get.setConf(SQLConf.CBO_ENABLED, originalCBOValue)
+    SQLConf.get.setConf(SQLConf.PLAN_STATS_ENABLED, originalPlanStatsValue)
     super.afterAll()
   }
 

From f3b80f88324e8a1a76d01d13cfc1fc7082238214 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Tue, 29 Sep 2020 12:02:45 -0700
Subject: [PATCH 1079/1280] [SPARK-33019][CORE] Use
 spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version=1 by default

### What changes were proposed in this pull request?

Apache Spark 3.1's default Hadoop profile is `hadoop-3.2`. Instead of having a warning documentation, this PR aims to use a consistent and safer version of Apache Hadoop file output committer algorithm which is `v1`. This will prevent a silent correctness regression during migration from Apache Spark 2.4/3.0 to Apache Spark 3.1.0. Of course, if there is a user-provided configuration, `spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version=2`, that will be used still.

### Why are the changes needed?

Apache Spark provides multiple distributions with Hadoop 2.7 and Hadoop 3.2. `spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version` depends on the Hadoop version. Apache Hadoop 3.0 switches the default algorithm from `v1` to `v2` and now there exists a discussion to remove `v2`. We had better provide a consistent default behavior of `v1` across various Spark distributions.

- [MAPREDUCE-7282](https://issues.apache.org/jira/browse/MAPREDUCE-7282) MR v2 commit algorithm should be deprecated and not the default

### Does this PR introduce _any_ user-facing change?

Yes. This changes the default behavior. Users can override this conf.

### How was this patch tested?

Manual.

**BEFORE (spark-3.0.1-bin-hadoop3.2)**
```scala
scala> sc.version
res0: String = 3.0.1

scala> sc.hadoopConfiguration.get("mapreduce.fileoutputcommitter.algorithm.version")
res1: String = 2
```

**AFTER**
```scala
scala> sc.hadoopConfiguration.get("mapreduce.fileoutputcommitter.algorithm.version")
res0: String = 1
```

Closes #29895 from dongjoon-hyun/SPARK-DEFAUT-COMMITTER.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit cc06266ade5a4eb35089501a3b32736624208d4c)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../org/apache/spark/deploy/SparkHadoopUtil.scala      |  3 +++
 docs/configuration.md                                  | 10 ++--------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index 1180501e8c738..6f799a542bc1e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -462,6 +462,9 @@ private[spark] object SparkHadoopUtil {
     for ((key, value) <- conf.getAll if key.startsWith("spark.hadoop.")) {
       hadoopConf.set(key.substring("spark.hadoop.".length), value)
     }
+    if (conf.getOption("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version").isEmpty) {
+      hadoopConf.set("mapreduce.fileoutputcommitter.algorithm.version", "1")
+    }
   }
 
   private def appendSparkHiveConfigs(conf: SparkConf, hadoopConf: Configuration): Unit = {
diff --git a/docs/configuration.md b/docs/configuration.md
index 95ff282bfa60d..36e4f4591d1f9 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1761,16 +1761,10 @@ Apart from these, the following properties are also available, and may be useful
 </tr>
 <tr>
   <td><code>spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version</code></td>
-  <td>Dependent on environment</td>
+  <td>1</td>
   <td>
     The file output committer algorithm version, valid algorithm version number: 1 or 2.
-    Version 2 may have better performance, but version 1 may handle failures better in certain situations,
-    as per <a href="https://issues.apache.org/jira/browse/MAPREDUCE-4815">MAPREDUCE-4815</a>.
-    The default value depends on the Hadoop version used in an environment:
-    1 for Hadoop versions lower than 3.0
-    2 for Hadoop versions 3.0 and higher
-    It's important to note that this can change back to 1 again in the future once <a href="https://issues.apache.org/jira/browse/MAPREDUCE-7282">MAPREDUCE-7282</a>
-    is fixed and merged.
+    Note that 2 may cause a correctness issue like MAPREDUCE-7282.
   </td>
   <td>2.2.0</td>
 </tr>

From db6ba049c43e2aa1521ed39c9f2b802ad04d111f Mon Sep 17 00:00:00 2001
From: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Date: Thu, 1 Oct 2020 08:15:53 +0900
Subject: [PATCH 1080/1280] [SPARK-31753][SQL][DOCS][FOLLOW-UP] Add missing
 keywords in the SQL docs

### What changes were proposed in this pull request?
update sql-ref docs, the following key words will be added in this PR.

CLUSTERED BY
SORTED BY
INTO num_buckets BUCKETS

### Why are the changes needed?
let more users know the sql key words usage

### Does this PR introduce _any_ user-facing change?
No
![image](https://user-images.githubusercontent.com/46367746/94428281-0a6b8080-01c3-11eb-9ff3-899f8da602ca.png)
![image](https://user-images.githubusercontent.com/46367746/94428285-0d667100-01c3-11eb-8a54-90e7641d917b.png)
![image](https://user-images.githubusercontent.com/46367746/94428288-0f303480-01c3-11eb-9e1d-023538aa6e2d.png)

### How was this patch tested?
generate html test

Closes #29883 from GuoPhilipse/add-sql-missing-keywords.

Lead-authored-by: GuoPhilipse <46367746+GuoPhilipse@users.noreply.github.com>
Co-authored-by: GuoPhilipse <guofei_ok@126.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 3bdbb5546d2517dda6f71613927cc1783c87f319)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 ...-ref-syntax-ddl-create-table-datasource.md |  7 +++-
 ...-ref-syntax-ddl-create-table-hiveformat.md | 32 +++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/docs/sql-ref-syntax-ddl-create-table-datasource.md b/docs/sql-ref-syntax-ddl-create-table-datasource.md
index d334447a91011..ba0516afbbfad 100644
--- a/docs/sql-ref-syntax-ddl-create-table-datasource.md
+++ b/docs/sql-ref-syntax-ddl-create-table-datasource.md
@@ -67,7 +67,12 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
 
 * **SORTED BY**
 
-    Determines the order in which the data is stored in buckets. Default is Ascending order.
+    Specifies an ordering of bucket columns. Optionally, one can use ASC for an ascending order or DESC for a descending order after any column names in the SORTED BY clause.
+    If not specified, ASC is assumed by default.
+   
+* **INTO num_buckets BUCKETS**
+
+    Specifies buckets numbers, which is used in `CLUSTERED BY` clause.
 
 * **LOCATION**
 
diff --git a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
index 7bf847df98150..3a8c8d5b1160a 100644
--- a/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
+++ b/docs/sql-ref-syntax-ddl-create-table-hiveformat.md
@@ -31,6 +31,9 @@ CREATE [ EXTERNAL ] TABLE [ IF NOT EXISTS ] table_identifier
     [ COMMENT table_comment ]
     [ PARTITIONED BY ( col_name2[:] col_type2 [ COMMENT col_comment2 ], ... ) 
         | ( col_name1, col_name2, ... ) ]
+    [ CLUSTERED BY ( col_name1, col_name2, ...) 
+        [ SORTED BY ( col_name1 [ ASC | DESC ], col_name2 [ ASC | DESC ], ... ) ] 
+        INTO num_buckets BUCKETS ]
     [ ROW FORMAT row_format ]
     [ STORED AS file_format ]
     [ LOCATION path ]
@@ -65,6 +68,21 @@ as any order. For example, you can write COMMENT table_comment after TBLPROPERTI
 
     Partitions are created on the table, based on the columns specified.
     
+* **CLUSTERED BY**
+
+    Partitions created on the table will be bucketed into fixed buckets based on the column specified for bucketing.
+
+    **NOTE:** Bucketing is an optimization technique that uses buckets (and bucketing columns) to determine data partitioning and avoid data shuffle.
+
+* **SORTED BY**
+
+    Specifies an ordering of bucket columns. Optionally, one can use ASC for an ascending order or DESC for a descending order after any column names in the SORTED BY clause.
+    If not specified, ASC is assumed by default.
+
+* **INTO num_buckets BUCKETS**
+
+    Specifies buckets numbers, which is used in `CLUSTERED BY` clause.
+    
 * **row_format**    
 
     Use the `SERDE` clause to specify a custom SerDe for one table. Otherwise, use the `DELIMITED` clause to use the native SerDe and specify the delimiter, escape character, null character and so on.
@@ -203,6 +221,20 @@ CREATE EXTERNAL TABLE family (id INT, name STRING)
     STORED AS INPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleInputFormat'
         OUTPUTFORMAT 'com.ly.spark.example.serde.io.SerDeExampleOutputFormat'
     LOCATION '/tmp/family/';
+
+--Use `CLUSTERED BY` clause to create bucket table without `SORTED BY`
+CREATE TABLE clustered_by_test1 (ID INT, AGE STRING)
+    CLUSTERED BY (ID)
+    INTO 4 BUCKETS
+    STORED AS ORC
+
+--Use `CLUSTERED BY` clause to create bucket table with `SORTED BY`
+CREATE TABLE clustered_by_test2 (ID INT, NAME STRING)
+    PARTITIONED BY (YEAR STRING)
+    CLUSTERED BY (ID, NAME)
+    SORTED BY (ID ASC)
+    INTO 3 BUCKETS
+    STORED AS PARQUET
 ```
 
 ### Related Statements

From bc29602e740393aa00c3154986000eaf1be2f965 Mon Sep 17 00:00:00 2001
From: iRakson <raksonrakesh@gmail.com>
Date: Thu, 1 Oct 2020 20:50:16 +0900
Subject: [PATCH 1081/1280] [SQL][DOC][MINOR] Corrects input table names in the
 examples of CREATE FUNCTION doc

### What changes were proposed in this pull request?
Fix Typo

### Why are the changes needed?
To maintain consistency.
Correct table name should be used for SELECT command.

### Does this PR introduce _any_ user-facing change?
Yes. Now CREATE FUNCTION doc will show the correct name of table.

### How was this patch tested?
Manually. Doc changes.

Closes #29920 from iRakson/fixTypo.

Authored-by: iRakson <raksonrakesh@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit d3dbe1a9076c8a76be0590ca071bfbec6114813b)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-syntax-ddl-create-function.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sql-ref-syntax-ddl-create-function.md b/docs/sql-ref-syntax-ddl-create-function.md
index aa6c1fad7b56b..dfa4f4f8123d8 100644
--- a/docs/sql-ref-syntax-ddl-create-function.md
+++ b/docs/sql-ref-syntax-ddl-create-function.md
@@ -112,7 +112,7 @@ SHOW USER FUNCTIONS;
 +------------------+
 
 -- Invoke the function. Every selected value should be incremented by 10.
-SELECT simple_udf(c1) AS function_return_value FROM t1;
+SELECT simple_udf(c1) AS function_return_value FROM test;
 +---------------------+
 |function_return_value|
 +---------------------+
@@ -150,7 +150,7 @@ CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
     USING JAR '/tmp/SimpleUdfR.jar';
 
 -- Invoke the function. Every selected value should be incremented by 20.
-SELECT simple_udf(c1) AS function_return_value FROM t1;
+SELECT simple_udf(c1) AS function_return_value FROM test;
 +---------------------+
 |function_return_value|
 +---------------------+

From 41e1919d24f573d5bef14df72729bdaccd35a82e Mon Sep 17 00:00:00 2001
From: Shruti Gumma <shruti_gumma@apple.com>
Date: Thu, 1 Oct 2020 19:28:06 -0700
Subject: [PATCH 1082/1280] [SPARK-32996][WEB-UI][3.0] Handle empty
 ExecutorMetrics in ExecutorMetricsJsonSerializer

### What changes were proposed in this pull request?
This is a backport PR for branch-3.0. This change was raised to `master` branch in `https://github.com/apache/spark/pull/29872`

When `peakMemoryMetrics` in `ExecutorSummary` is `Option.empty`, then the `ExecutorMetricsJsonSerializer#serialize` method does not execute the `jsonGenerator.writeObject` method. This causes the json to be generated with `peakMemoryMetrics` key added to the serialized string, but no corresponding value.
This causes an error to be thrown when it is the next key `attributes` turn to be added to the json:
`com.fasterxml.jackson.core.JsonGenerationException: Can not write a field name, expecting a value
`

### Why are the changes needed?
At the start of the Spark job, if `peakMemoryMetrics` is `Option.empty`, then it causes
a `com.fasterxml.jackson.core.JsonGenerationException` to be thrown when we navigate to the Executors tab in Spark UI.
Complete stacktrace:

> com.fasterxml.jackson.core.JsonGenerationException: Can not write a field name, expecting a value
> 	at com.fasterxml.jackson.core.JsonGenerator._reportError(JsonGenerator.java:2080)
> 	at com.fasterxml.jackson.core.json.WriterBasedJsonGenerator.writeFieldName(WriterBasedJsonGenerator.java:161)
> 	at com.fasterxml.jackson.databind.ser.BeanPropertyWriter.serializeAsField(BeanPropertyWriter.java:725)
> 	at com.fasterxml.jackson.databind.ser.std.BeanSerializerBase.serializeFields(BeanSerializerBase.java:721)
> 	at com.fasterxml.jackson.databind.ser.BeanSerializer.serialize(BeanSerializer.java:166)
> 	at com.fasterxml.jackson.databind.ser.std.CollectionSerializer.serializeContents(CollectionSerializer.java:145)
> 	at com.fasterxml.jackson.module.scala.ser.IterableSerializer.serializeContents(IterableSerializerModule.scala:26)
> 	at com.fasterxml.jackson.module.scala.ser.IterableSerializer.serializeContents$(IterableSerializerModule.scala:25)
> 	at com.fasterxml.jackson.module.scala.ser.UnresolvedIterableSerializer.serializeContents(IterableSerializerModule.scala:54)
> 	at com.fasterxml.jackson.module.scala.ser.UnresolvedIterableSerializer.serializeContents(IterableSerializerModule.scala:54)
> 	at com.fasterxml.jackson.databind.ser.std.AsArraySerializerBase.serialize(AsArraySerializerBase.java:250)
> 	at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider._serialize(DefaultSerializerProvider.java:480)
> 	at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:319)
> 	at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:4094)
> 	at com.fasterxml.jackson.databind.ObjectMapper.writeValueAsString(ObjectMapper.java:3404)
> 	at org.apache.spark.ui.exec.ExecutorsPage.allExecutorsDataScript$1(ExecutorsTab.scala:64)
> 	at org.apache.spark.ui.exec.ExecutorsPage.render(ExecutorsTab.scala:76)
> 	at org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:89)
> 	at org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:80)
> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
> 	at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
> 	at org.sparkproject.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
> 	at org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
> 	at org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)
> 	at org.sparkproject.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
> 	at org.sparkproject.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
> 	at org.sparkproject.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
> 	at org.sparkproject.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
> 	at org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> 	at org.sparkproject.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> 	at org.sparkproject.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> 	at org.sparkproject.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> 	at org.sparkproject.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> 	at org.sparkproject.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:753)
> 	at org.sparkproject.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
> 	at org.sparkproject.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> 	at org.sparkproject.jetty.server.Server.handle(Server.java:505)
> 	at org.sparkproject.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> 	at org.sparkproject.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> 	at org.sparkproject.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> 	at org.sparkproject.jetty.io.FillInterest.fillable(FillInterest.java:103)
> 	at org.sparkproject.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> 	at org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
> 	at org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
> 	at org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
> 	at org.sparkproject.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
> 	at org.sparkproject.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
> 	at org.sparkproject.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
> 	at org.sparkproject.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
> 	at java.base/java.lang.Thread.run(Thread.java:834)

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Unit test

Closes #29914 from shrutig/SPARK-32996-3.0.

Authored-by: Shruti Gumma <shruti_gumma@apple.com>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 .../org/apache/spark/status/api/v1/api.scala  | 16 ++++--
 .../status/api/v1/ExecutorSummarySuite.scala  | 51 +++++++++++++++++++
 2 files changed, 63 insertions(+), 4 deletions(-)
 create mode 100644 core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala

diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 5ec9b36393764..37db64abb9633 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -127,6 +127,10 @@ private[spark] class ExecutorMetricsJsonDeserializer
       new TypeReference[Option[Map[String, java.lang.Long]]] {})
     metricsMap.map(metrics => new ExecutorMetrics(metrics))
   }
+
+  override def getNullValue(ctxt: DeserializationContext): Option[ExecutorMetrics] = {
+    None
+  }
 }
 /** serializer for peakMemoryMetrics: convert ExecutorMetrics to map with metric name as key */
 private[spark] class ExecutorMetricsJsonSerializer
@@ -135,11 +139,15 @@ private[spark] class ExecutorMetricsJsonSerializer
       metrics: Option[ExecutorMetrics],
       jsonGenerator: JsonGenerator,
       serializerProvider: SerializerProvider): Unit = {
-    metrics.foreach { m: ExecutorMetrics =>
-      val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
-        metric -> m.getMetricValue(metric)
+    if (metrics.isEmpty) {
+      jsonGenerator.writeNull()
+    } else {
+      metrics.foreach { m: ExecutorMetrics =>
+        val metricsMap = ExecutorMetricType.metricToOffset.map { case (metric, _) =>
+          metric -> m.getMetricValue(metric)
+        }
+        jsonGenerator.writeObject(metricsMap)
       }
-      jsonGenerator.writeObject(metricsMap)
     }
   }
 
diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala
new file mode 100644
index 0000000000000..2723af75227e4
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/status/api/v1/ExecutorSummarySuite.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.status.api.v1
+
+import java.util.Date
+
+import com.fasterxml.jackson.core.`type`.TypeReference
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+
+import org.apache.spark.SparkFunSuite
+
+class ExecutorSummarySuite extends SparkFunSuite {
+
+  test("Check ExecutorSummary serialize and deserialize with empty peakMemoryMetrics") {
+    val mapper = new ObjectMapper().registerModule(DefaultScalaModule)
+    val executorSummary = new ExecutorSummary("id", "host:port", true, 1,
+      10, 10, 1, 1, 1,
+      0, 0, 1, 100,
+      1, 100, 100,
+      10, false, 20, new Date(1600984336352L),
+      Option.empty, Option.empty, Map(), Option.empty, Set(), Option.empty, Map(), Map())
+    val expectedJson = "{\"id\":\"id\",\"hostPort\":\"host:port\",\"isActive\":true," +
+      "\"rddBlocks\":1,\"memoryUsed\":10,\"diskUsed\":10,\"totalCores\":1,\"maxTasks\":1," +
+      "\"activeTasks\":1,\"failedTasks\":0,\"completedTasks\":0,\"totalTasks\":1," +
+      "\"totalDuration\":100,\"totalGCTime\":1,\"totalInputBytes\":100," +
+      "\"totalShuffleRead\":100,\"totalShuffleWrite\":10,\"isBlacklisted\":false," +
+      "\"maxMemory\":20,\"addTime\":1600984336352,\"removeTime\":null,\"removeReason\":null," +
+      "\"executorLogs\":{},\"memoryMetrics\":null,\"blacklistedInStages\":[]," +
+      "\"peakMemoryMetrics\":null,\"attributes\":{},\"resources\":{}}"
+    val json = mapper.writeValueAsString(executorSummary)
+    assert(expectedJson.equals(json))
+    val deserializeExecutorSummary = mapper.readValue(json, new TypeReference[ExecutorSummary] {})
+    assert(deserializeExecutorSummary.peakMemoryMetrics == None)
+  }
+
+}

From 31684d671ca48cab771517796660676056ccc304 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 2 Oct 2020 15:12:33 +0900
Subject: [PATCH 1083/1280] [SPARK-33051][INFRA][R] Uses setup-r to install R
 in GitHub Actions build

### What changes were proposed in this pull request?

At SPARK-32493, the R installation was switched to manual installation because setup-r was broken. This seems fixed in the upstream so we should better switch it back.

### Why are the changes needed?

To avoid maintaining the installation steps by ourselve.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

GitHub Actions build in this PR should test it.

Closes #29931 from HyukjinKwon/recover-r-build.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit b205be5ff6926454b0afe76e4c3438cfa0f34832)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/build_and_test.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index b79fa161f12ba..acef583b0a766 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -161,12 +161,10 @@ jobs:
         python3.8 -m pip list
     # SparkR
     - name: Install R 4.0
+      uses: r-lib/actions/setup-r@v1
       if: contains(matrix.modules, 'sparkr')
-      run: |
-        sudo sh -c "echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list"
-        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
-        sudo apt-get update
-        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
+      with:
+        r-version: 4.0
     - name: Install R packages
       if: contains(matrix.modules, 'sparkr')
       run: |
@@ -223,11 +221,9 @@ jobs:
       run: |
         pip3 install flake8 sphinx numpy
     - name: Install R 4.0
-      run: |
-        sudo sh -c "echo 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/' >> /etc/apt/sources.list"
-        curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xE298A3A825C0D65DFD57CBB651716619E084DAB9" | sudo apt-key add
-        sudo apt-get update
-        sudo apt-get install -y r-base r-base-dev libcurl4-openssl-dev
+      uses: r-lib/actions/setup-r@v1
+      with:
+        r-version: 4.0
     - name: Install R linter dependencies and SparkR
       run: |
         sudo apt-get install -y libcurl4-openssl-dev

From c9b62711fdec24160c4bdeff8fc09eedb0b75ee0 Mon Sep 17 00:00:00 2001
From: Sean Owen <srowen@gmail.com>
Date: Sat, 3 Oct 2020 13:12:55 -0500
Subject: [PATCH 1084/1280] [SPARK-33043][ML] Handle
 spark.driver.maxResultSize=0 in RowMatrix heuristic computation

### What changes were proposed in this pull request?

RowMatrix contains a computation based on spark.driver.maxResultSize. However, when this value is set to 0, the computation fails (log of 0). The fix is simply to correctly handle this setting, which means unlimited result size, by using a tree depth of 1 in the RowMatrix method.

### Why are the changes needed?

Simple bug fix to make several Spark ML functions which use RowMatrix run correctly in this case.

### Does this PR introduce _any_ user-facing change?

Not other than the bug fix of course.

### How was this patch tested?

Existing RowMatrix tests plus a new test.

Closes #29925 from srowen/SPARK-33043.

Authored-by: Sean Owen <srowen@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit f86171aea43479f54ac2bbbca8f128baa3fc4a8c)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../mllib/linalg/distributed/RowMatrix.scala      |  6 +++++-
 .../mllib/linalg/distributed/RowMatrixSuite.scala | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index 20e26cee9e0d6..07b9d91c1f59b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -786,11 +786,15 @@ class RowMatrix @Since("1.0.0") (
    * Based on the formulae: (numPartitions)^(1/depth) * objectSize <= DriverMaxResultSize
    * @param aggregatedObjectSizeInBytes the size, in megabytes, of the object being tree aggregated
    */
-  private[spark] def getTreeAggregateIdealDepth(aggregatedObjectSizeInBytes: Long) = {
+  private[spark] def getTreeAggregateIdealDepth(aggregatedObjectSizeInBytes: Long): Int = {
     require(aggregatedObjectSizeInBytes > 0,
       "Cannot compute aggregate depth heuristic based on a zero-size object to aggregate")
 
     val maxDriverResultSizeInBytes = rows.conf.get[Long](MAX_RESULT_SIZE)
+    if (maxDriverResultSizeInBytes <= 0) {
+      // Unlimited result size, so 1 is OK
+      return 1
+    }
 
     require(maxDriverResultSizeInBytes > aggregatedObjectSizeInBytes,
       s"Cannot aggregate object of size $aggregatedObjectSizeInBytes Bytes, "
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
index 0a4b11935580a..adc4eeef91bb1 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/distributed/RowMatrixSuite.scala
@@ -25,6 +25,7 @@ import breeze.linalg.{norm => brzNorm, svd => brzSvd, DenseMatrix => BDM, DenseV
 import breeze.numerics.abs
 
 import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.config.MAX_RESULT_SIZE
 import org.apache.spark.mllib.linalg.{Matrices, Vector, Vectors}
 import org.apache.spark.mllib.random.RandomRDDs
 import org.apache.spark.mllib.util.{LocalClusterSparkContext, MLlibTestSparkContext}
@@ -121,6 +122,20 @@ class RowMatrixSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(objectBiggerThanResultSize.getMessage.contains("it's bigger than maxResultSize"))
   }
 
+  test("SPARK-33043: getTreeAggregateIdealDepth with unlimited driver size") {
+    val originalMaxResultSize = sc.conf.get[Long](MAX_RESULT_SIZE)
+    sc.conf.set(MAX_RESULT_SIZE, 0L)
+    try {
+      val nbPartitions = 100
+      val vectors = sc.emptyRDD[Vector]
+        .repartition(nbPartitions)
+      val rowMat = new RowMatrix(vectors)
+      assert(rowMat.getTreeAggregateIdealDepth(700 * 1024 * 1024) === 1)
+    } finally {
+      sc.conf.set(MAX_RESULT_SIZE, originalMaxResultSize)
+    }
+  }
+
   test("similar columns") {
     val colMags = Vectors.dense(math.sqrt(126), math.sqrt(66), math.sqrt(94))
     val expected = BDM(

From 75003fc99fa2f66622c2accca5e48842acc4c90c Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sun, 4 Oct 2020 16:11:06 -0700
Subject: [PATCH 1085/1280] [SPARK-33065][TESTS] Expand the stack size of a
 thread in a test in LocalityPlacementStrategySuite for Java 11 with sbt

### What changes were proposed in this pull request?

This PR fixes an issue that a test in `LocalityPlacementStrategySuite` fails with Java 11 due to `StackOverflowError`.

```
[info] - handle large number of containers and tasks (SPARK-18750) *** FAILED *** (170 milliseconds)
[info]   StackOverflowError should not be thrown; however, got:
[info]
[info]   java.lang.StackOverflowError
[info]          at java.base/java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1012)
[info]          at java.base/java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1541)
[info]          at java.base/java.lang.ClassLoader.getClassLoadingLock(ClassLoader.java:668)
[info]          at java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:591)
[info]          at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:579)
[info]          at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178)
[info]          at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:522)
```

The solution is to expand the stack size of a thread in the test from 32KB to 256KB.
Currently, the stack size is specified as 32KB but the actual stack size can be greater than 32KB.
According to the code of Hotspot, the minimum stack size is prefer to the specified size.

Java 8: https://hg.openjdk.java.net/jdk8u/jdk8u/hotspot/file/c92ba514724d/src/os/linux/vm/os_linux.cpp#l900
Java 11: https://hg.openjdk.java.net/jdk-updates/jdk11u/file/73edf743a93a/src/hotspot/os/posix/os_posix.cpp#l1555

For Linux on x86_64, the minimum stack size seems to be 224KB and 136KB for Java 8 and Java 11 respectively. So, the actual stack size should be 224KB rather than 32KB for Java 8 on x86_64/Linux.
As the test passes for Java 8 but doesn't for Java 11, 224KB is enough while 136KB is not.
So I think specifing 256KB is reasonable for the new stack size.

### Why are the changes needed?

To pass the test for Java 11.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Following command with Java 11.
```
build/sbt -Pyarn clean package "testOnly org.apache.spark.deploy.yarn.LocalityPlacementStrategySuite"
```

Closes #29943 from sarutak/fix-stack-size.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit fab53212cb110a81696cee8546c35095332f6e09)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/deploy/yarn/LocalityPlacementStrategySuite.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
index bb14e09ad09c2..cf2c3840a6e9b 100644
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -42,7 +42,7 @@ class LocalityPlacementStrategySuite extends SparkFunSuite {
       }
     }
 
-    val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 32 * 1024)
+    val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 256 * 1024)
     thread.start()
     thread.join()
 

From 46a62cab765f4684dc634d92d81ecddcc42cee35 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 6 Oct 2020 09:09:58 +0900
Subject: [PATCH 1086/1280] [SPARK-33069][INFRA] Skip test result report if no
 JUnit XML files are found

### What changes were proposed in this pull request?

This PR proposes to skip test reporting ("Report test results") if there are no JUnit XML files are found.

Currently, we're running and skipping the tests dynamically. For example,
- if there are only changes in SparkR at the underlying commit, it only runs the SparkR tests, and skip the other tests and generate JUnit XML files for SparkR test cases.
- if there are only changes in `docs` at the underlying commit, the build skips all tests except linters and do not generate any JUnit XML files.

When test reporting ("Report test results") job is triggered after the main build ("Build and test
") is finished, and there are no JUnit XML files found, it reports the case as a failure. See https://github.com/apache/spark/runs/1196184007 as an example.

This PR works around it by simply skipping the testing report when there are no JUnit XML files are found.
Please see https://github.com/apache/spark/pull/29906#issuecomment-702525542 for more details.

### Why are the changes needed?

To avoid false alarm for test results.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Manually tested in my fork.

Positive case:

https://github.com/HyukjinKwon/spark/runs/1208624679?check_suite_focus=true
https://github.com/HyukjinKwon/spark/actions/runs/288996327

Negative case:

https://github.com/HyukjinKwon/spark/runs/1208229838?check_suite_focus=true
https://github.com/HyukjinKwon/spark/actions/runs/289000058

Closes #29946 from HyukjinKwon/test-junit-files.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a0aa8f33a9420feb9228b51a3dfad2e7e86d65a5)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/test_report.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml
index 93cdb86687261..060a8795b6a77 100644
--- a/.github/workflows/test_report.yml
+++ b/.github/workflows/test_report.yml
@@ -15,7 +15,16 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         workflow: ${{ github.event.workflow_run.workflow_id }}
         commit: ${{ github.event.workflow_run.head_commit.id }}
+    - name: Check if JUnit report XML files exist
+      run: |
+        if ls **/target/test-reports/*.xml > /dev/null 2>&1; then
+          echo '::set-output name=FILE_EXISTS::true'
+        else
+          echo '::set-output name=FILE_EXISTS::false'
+        fi
+      id: check-junit-file
     - name: Publish test report
+      if: steps.check-junit-file.outputs.FILE_EXISTS == 'true'
       uses: scacap/action-surefire-report@v1
       with:
         check_name: Report test results

From 4f71231af51a3da5d7964a218a878c0cf3037c10 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Tue, 6 Oct 2020 18:11:24 +0900
Subject: [PATCH 1087/1280] [SPARK-33073][PYTHON] Improve error handling on
 Pandas to Arrow conversion failures

### What changes were proposed in this pull request?

This improves error handling when a failure in conversion from Pandas to Arrow occurs. And fixes tests to be compatible with upcoming Arrow 2.0.0 release.

### Why are the changes needed?

Current tests will fail with Arrow 2.0.0 because of a change in error message when the schema is invalid. For these cases, the current error message also includes information on disabling safe conversion config, which is mainly meant for floating point truncation and overflow. The tests have been updated to use a message that is show for past Arrow versions, and upcoming.

If the user enters an invalid schema, the error produced by pyarrow is not consistent and either `TypeError` or `ArrowInvalid`, with the latter being caught, and raised as a `RuntimeError` with the extra info.

The error handling is improved by:

- narrowing the exception type to `TypeError`s, which `ArrowInvalid` is a subclass and what is raised on safe conversion failures.
- The exception is only raised with additional information on disabling "spark.sql.execution.pandas.convertToArrowArraySafely" if it is enabled in the first place.
- The original exception is chained to better show it to the user.

### Does this PR introduce _any_ user-facing change?

Yes, the error re-raised changes from a RuntimeError to a ValueError, which better categorizes this type of error and in-line with the original Arrow error.

### How was this patch tested?

Existing tests, using pyarrow 1.0.1 and 2.0.0-snapshot

Closes #29951 from BryanCutler/arrow-better-handle-pandas-errors-SPARK-33073.

Authored-by: Bryan Cutler <cutlerb@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0812d6c17cc4876bb87a9d1fec35ec8c7b2365f0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/pandas/serializers.py        | 17 ++++++++++-------
 python/pyspark/sql/tests/test_arrow.py          |  9 +++++----
 .../sql/tests/test_pandas_grouped_map.py        | 15 ++++++++-------
 3 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index 4dd15d14b9c53..b164a388d01bd 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -156,13 +156,16 @@ def create_array(s, t):
                 s = _check_series_convert_timestamps_internal(s, self._timezone)
             try:
                 array = pa.Array.from_pandas(s, mask=mask, type=t, safe=self._safecheck)
-            except pa.ArrowException as e:
-                error_msg = "Exception thrown when converting pandas.Series (%s) to Arrow " + \
-                            "Array (%s). It can be caused by overflows or other unsafe " + \
-                            "conversions warned by Arrow. Arrow safe type check can be " + \
-                            "disabled by using SQL config " + \
-                            "`spark.sql.execution.pandas.convertToArrowArraySafely`."
-                raise RuntimeError(error_msg % (s.dtype, t), e)
+            except ValueError as e:
+                if self._safecheck:
+                    error_msg = "Exception thrown when converting pandas.Series (%s) to " + \
+                                "Arrow Array (%s). It can be caused by overflows or other " + \
+                                "unsafe conversions warned by Arrow. Arrow safe type check " + \
+                                "can be disabled by using SQL config " + \
+                                "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+                    raise ValueError(error_msg % (s.dtype, t)) from e
+                else:
+                    raise e
             return array
 
         arrs = []
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 15c5cf1ad9b05..2c6231da3c161 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -266,11 +266,12 @@ def test_createDataFrame_with_schema(self):
     def test_createDataFrame_with_incorrect_schema(self):
         pdf = self.create_pandas_data_frame()
         fields = list(self.schema)
-        fields[0], fields[1] = fields[1], fields[0]  # swap str with int
+        fields[5], fields[6] = fields[6], fields[5]  # swap decimal with date
         wrong_schema = StructType(fields)
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "integer.*required"):
-                self.spark.createDataFrame(pdf, schema=wrong_schema)
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
+                    self.spark.createDataFrame(pdf, schema=wrong_schema)
 
     def test_createDataFrame_with_names(self):
         pdf = self.create_pandas_data_frame()
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index cc6167e619285..8e02b291cddea 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -450,15 +450,16 @@ def int_index(pdf):
         def column_name_typo(pdf):
             return pd.DataFrame({'iid': pdf.id, 'v': pdf.v})
 
-        @pandas_udf('id long, v int', PandasUDFType.GROUPED_MAP)
+        @pandas_udf('id long, v decimal', PandasUDFType.GROUPED_MAP)
         def invalid_positional_types(pdf):
-            return pd.DataFrame([(u'a', 1.2)])
+            return pd.DataFrame([(1, datetime.date(2020, 10, 5))])
 
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
-                grouped_df.apply(column_name_typo).collect()
-            with self.assertRaisesRegexp(Exception, "an integer is required"):
-                grouped_df.apply(invalid_positional_types).collect()
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
+                    grouped_df.apply(column_name_typo).collect()
+                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
+                    grouped_df.apply(invalid_positional_types).collect()
 
     def test_positional_assignment_conf(self):
         with self.sql_conf({

From d51b8d669d71e6d447f0c008088a073f6ed8986c Mon Sep 17 00:00:00 2001
From: Michael Munday <mike.munday@ibm.com>
Date: Tue, 6 Oct 2020 08:31:06 -0500
Subject: [PATCH 1088/1280] [SPARK-27428][CORE][TEST] Increase receive buffer
 size used in StatsdSinkSuite

### What changes were proposed in this pull request?

Increase size of socket receive buffer in these tests.

### Why are the changes needed?

The socket receive buffer size set in this test was too small for
the StatsdSinkSuite tests to run reliably on some systems. For a
test in this suite to run reliably the buffer needs to be large
enough to hold all the data in the packets being sent in a test
along with any additional kernel or protocol overhead. The amount
of kernel overhead per packet can vary from system to system but is
typically far higher than the protocol overhead.

If the receive buffer is too small and fills up then packets are
silently dropped. This leads to the test failing with a timeout.

If the socket defaults to a larger receive buffer (normally true)
then we should keep that size.

As well as increasing the minimum buffer size I've also decoupled
the datagram packet buffer size from the receive buffer size. The
receive buffer should in general be far larger to account for the
fact that multiple packets might be buffered, as well as the
aforementioned overhead. Any truncated data in individual packets
will be picked up by the tests.

### Does this PR introduce _any_ user-facing change?

No, this only affects the tests.

### How was this patch tested?
Existing tests on IBM Z and x86.

Closes #29819 from mundaym/fix-statsd.

Authored-by: Michael Munday <mike.munday@ibm.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit b5e4b8c73e10743eef4d35b6e82053a5a065b2ed)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../spark/metrics/sink/StatsdSinkSuite.scala  | 29 ++++++++++++++-----
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
index 0e21a36071c42..3d4b8c868d6fc 100644
--- a/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
+++ b/core/src/test/scala/org/apache/spark/metrics/sink/StatsdSinkSuite.scala
@@ -35,12 +35,27 @@ class StatsdSinkSuite extends SparkFunSuite {
     STATSD_KEY_UNIT -> "seconds",
     STATSD_KEY_HOST -> "127.0.0.1"
   )
-  private val socketTimeout = 30000 // milliseconds
-  private val socketBufferSize = 8192
+  // The maximum size of a single datagram packet payload. Payloads
+  // larger than this will be truncated.
+  private val maxPayloadSize = 256 // bytes
+
+  // The receive buffer must be large enough to hold all inflight
+  // packets. This includes any kernel and protocol overhead.
+  // This value was determined experimentally and should be
+  // increased if timeouts are seen.
+  private val socketMinRecvBufferSize = 16384 // bytes
+  private val socketTimeout = 30000           // milliseconds
 
   private def withSocketAndSink(testCode: (DatagramSocket, StatsdSink) => Any): Unit = {
     val socket = new DatagramSocket
-    socket.setReceiveBufferSize(socketBufferSize)
+
+    // Leave the receive buffer size untouched unless it is too
+    // small. If the receive buffer is too small packets will be
+    // silently dropped and receive operations will timeout.
+    if (socket.getReceiveBufferSize() < socketMinRecvBufferSize) {
+      socket.setReceiveBufferSize(socketMinRecvBufferSize)
+    }
+
     socket.setSoTimeout(socketTimeout)
     val props = new Properties
     defaultProps.foreach(e => props.put(e._1, e._2))
@@ -61,7 +76,7 @@ class StatsdSinkSuite extends SparkFunSuite {
       sink.registry.register("counter", counter)
       sink.report()
 
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       socket.receive(p)
 
       val result = new String(p.getData, 0, p.getLength, UTF_8)
@@ -77,7 +92,7 @@ class StatsdSinkSuite extends SparkFunSuite {
       sink.registry.register("gauge", gauge)
       sink.report()
 
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       socket.receive(p)
 
       val result = new String(p.getData, 0, p.getLength, UTF_8)
@@ -87,7 +102,7 @@ class StatsdSinkSuite extends SparkFunSuite {
 
   test("metrics StatsD sink with Histogram") {
     withSocketAndSink { (socket, sink) =>
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       val histogram = new Histogram(new UniformReservoir)
       histogram.update(10)
       histogram.update(20)
@@ -121,7 +136,7 @@ class StatsdSinkSuite extends SparkFunSuite {
 
   test("metrics StatsD sink with Timer") {
     withSocketAndSink { (socket, sink) =>
-      val p = new DatagramPacket(new Array[Byte](socketBufferSize), socketBufferSize)
+      val p = new DatagramPacket(new Array[Byte](maxPayloadSize), maxPayloadSize)
       val timer = new Timer()
       timer.update(1, SECONDS)
       timer.update(2, SECONDS)

From 2076abc0d3f1055254d65697c21f26727a502391 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 7 Oct 2020 09:11:42 +0900
Subject: [PATCH 1089/1280] Revert "[SPARK-33073][PYTHON] Improve error
 handling on Pandas to Arrow conversion failures"

This reverts commit 4f71231af51a3da5d7964a218a878c0cf3037c10.
---
 python/pyspark/sql/pandas/serializers.py        | 17 +++++++----------
 python/pyspark/sql/tests/test_arrow.py          |  9 ++++-----
 .../sql/tests/test_pandas_grouped_map.py        | 15 +++++++--------
 3 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index b164a388d01bd..4dd15d14b9c53 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -156,16 +156,13 @@ def create_array(s, t):
                 s = _check_series_convert_timestamps_internal(s, self._timezone)
             try:
                 array = pa.Array.from_pandas(s, mask=mask, type=t, safe=self._safecheck)
-            except ValueError as e:
-                if self._safecheck:
-                    error_msg = "Exception thrown when converting pandas.Series (%s) to " + \
-                                "Arrow Array (%s). It can be caused by overflows or other " + \
-                                "unsafe conversions warned by Arrow. Arrow safe type check " + \
-                                "can be disabled by using SQL config " + \
-                                "`spark.sql.execution.pandas.convertToArrowArraySafely`."
-                    raise ValueError(error_msg % (s.dtype, t)) from e
-                else:
-                    raise e
+            except pa.ArrowException as e:
+                error_msg = "Exception thrown when converting pandas.Series (%s) to Arrow " + \
+                            "Array (%s). It can be caused by overflows or other unsafe " + \
+                            "conversions warned by Arrow. Arrow safe type check can be " + \
+                            "disabled by using SQL config " + \
+                            "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+                raise RuntimeError(error_msg % (s.dtype, t), e)
             return array
 
         arrs = []
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 2c6231da3c161..15c5cf1ad9b05 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -266,12 +266,11 @@ def test_createDataFrame_with_schema(self):
     def test_createDataFrame_with_incorrect_schema(self):
         pdf = self.create_pandas_data_frame()
         fields = list(self.schema)
-        fields[5], fields[6] = fields[6], fields[5]  # swap decimal with date
+        fields[0], fields[1] = fields[1], fields[0]  # swap str with int
         wrong_schema = StructType(fields)
-        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
-            with QuietTest(self.sc):
-                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
-                    self.spark.createDataFrame(pdf, schema=wrong_schema)
+        with QuietTest(self.sc):
+            with self.assertRaisesRegexp(Exception, "integer.*required"):
+                self.spark.createDataFrame(pdf, schema=wrong_schema)
 
     def test_createDataFrame_with_names(self):
         pdf = self.create_pandas_data_frame()
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index 8e02b291cddea..cc6167e619285 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -450,16 +450,15 @@ def int_index(pdf):
         def column_name_typo(pdf):
             return pd.DataFrame({'iid': pdf.id, 'v': pdf.v})
 
-        @pandas_udf('id long, v decimal', PandasUDFType.GROUPED_MAP)
+        @pandas_udf('id long, v int', PandasUDFType.GROUPED_MAP)
         def invalid_positional_types(pdf):
-            return pd.DataFrame([(1, datetime.date(2020, 10, 5))])
+            return pd.DataFrame([(u'a', 1.2)])
 
-        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
-            with QuietTest(self.sc):
-                with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
-                    grouped_df.apply(column_name_typo).collect()
-                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
-                    grouped_df.apply(invalid_positional_types).collect()
+        with QuietTest(self.sc):
+            with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
+                grouped_df.apply(column_name_typo).collect()
+            with self.assertRaisesRegexp(Exception, "an integer is required"):
+                grouped_df.apply(invalid_positional_types).collect()
 
     def test_positional_assignment_conf(self):
         with self.sql_conf({

From 23207fcfb4987d5b06664876b5d9f7c4417954f9 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Tue, 6 Oct 2020 23:11:45 -0700
Subject: [PATCH 1090/1280] [SPARK-33035][SQL][3.0] Updates the obsoleted
 entries of attribute mapping in QueryPlan#transformUpWithNewOutput

### What changes were proposed in this pull request?

This PR intends to fix corner-case bugs in the `QueryPlan#transformUpWithNewOutput` that is used to propagate updated `ExprId`s in a bottom-up way. Let's say we have a rule to simply assign new `ExprId`s in a projection list like this;
```scala
case class TestRule extends Rule[LogicalPlan] {
  override def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithNewOutput {
    case p  Project(projList, _) =>
      val newPlan = p.copy(projectList = projList.map { _.transform {
        // Assigns a new `ExprId` for references
        case a: AttributeReference => Alias(a, a.name)()
      }}.asInstanceOf[Seq[NamedExpression]])

      val attrMapping = p.output.zip(newPlan.output)
      newPlan -> attrMapping
  }
}
```
Then, this rule is applied into a plan below;
```scala
(3) Project [a#5, b#6]
+- (2) Project [a#5, b#6]
   +- (1) Project [a#5, b#6]
      +- LocalRelation <empty>, [a#5, b#6]
```
In the first transformation, the rule assigns new `ExprId`s in `(1) Project` (e.g., a#5 AS a#7, b#6 AS b#8). In the second transformation, the rule corrects the input references of `(2) Project`  first by using attribute mapping given from `(1) Project` (a#5->a#7 and b#6->b#8) and then assigns new `ExprId`s (e.g., a#7 AS a#9, b#8 AS b#10). But, in the third transformation, the rule fails because it tries to correct the references of `(3) Project` by using incorrect attribute mapping (a#7->a#9 and b#8->b#10) even though the correct one is a#5->a#9 and b#6->b#10. To fix this issue, this PR modified the code to update the attribute mapping entries that are obsoleted by generated entries in a given rule.

This is the backport of https://github.com/apache/spark/pull/29911.

### Why are the changes needed?

bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests in `QueryPlanSuite`.

Closes #29953 from maropu/SPARK-33035-FOLLOWUP.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/plans/QueryPlan.scala  | 26 ++++++++++++++-----
 .../sql/catalyst/plans/QueryPlanSuite.scala   | 26 ++++++++++++++++---
 2 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
index fed5df69580ee..840f1334d07c1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -201,11 +201,6 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
           case (oldAttr, _) => plan.references.contains(oldAttr)
         }
 
-        val (planAfterRule, newAttrMapping) = CurrentOrigin.withOrigin(origin) {
-          rule.applyOrElse(newPlan, (plan: PlanType) => plan -> Nil)
-        }
-        newPlan = planAfterRule
-
         if (attrMappingForCurrentPlan.nonEmpty) {
           assert(!attrMappingForCurrentPlan.groupBy(_._1.exprId)
             .exists(_._2.map(_._2.exprId).distinct.length > 1),
@@ -222,10 +217,27 @@ abstract class QueryPlan[PlanType <: QueryPlan[PlanType]] extends TreeNode[PlanT
           }
         }
 
-        attrMapping ++= newAttrMapping.filter {
+        val (planAfterRule, newAttrMapping) = CurrentOrigin.withOrigin(origin) {
+          rule.applyOrElse(newPlan, (plan: PlanType) => plan -> Nil)
+        }
+
+        val newValidAttrMapping = newAttrMapping.filter {
           case (a1, a2) => a1.exprId != a2.exprId
         }
-        newPlan -> attrMapping
+
+        // Updates the `attrMapping` entries that are obsoleted by generated entries in `rule`.
+        // For example, `attrMapping` has a mapping entry 'id#1 -> id#2' and `rule`
+        // generates a new entry 'id#2 -> id#3'. In this case, we need to update
+        // the corresponding old entry from 'id#1 -> id#2' to '#id#1 -> #id#3'.
+        val updatedAttrMap = AttributeMap(newValidAttrMapping)
+        val transferAttrMapping = attrMapping.map {
+          case (a1, a2) => (a1, updatedAttrMap.getOrElse(a2, a2))
+        }
+        val newOtherAttrMapping = {
+          val existingAttrMappingSet = transferAttrMapping.map(_._2).toSet
+          newValidAttrMapping.filterNot { case (_, a) => existingAttrMappingSet.contains(a) }
+        }
+        planAfterRule -> (transferAttrMapping ++ newOtherAttrMapping).toSeq
       }
     }
     rewrite(this)._1
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
index 91ce187f4d270..404c8895c4d11 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/plans/QueryPlanSuite.scala
@@ -20,9 +20,11 @@ package org.apache.spark.sql.catalyst.plans
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
-import org.apache.spark.sql.catalyst.dsl.plans
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ListQuery, Literal, NamedExpression}
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, Project, Union}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, Expression, ListQuery, Literal, NamedExpression}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation, LogicalPlan, Project, Union}
+import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
 import org.apache.spark.sql.types.IntegerType
 
@@ -31,7 +33,7 @@ class QueryPlanSuite extends SparkFunSuite {
   test("origin remains the same after mapExpressions (SPARK-23823)") {
     CurrentOrigin.setPosition(0, 0)
     val column = AttributeReference("column", IntegerType)(NamedExpression.newExprId)
-    val query = plans.DslLogicalPlan(plans.table("table")).select(column)
+    val query = DslLogicalPlan(table("table")).select(column)
     CurrentOrigin.reset()
 
     val mappedQuery = query mapExpressions {
@@ -83,4 +85,20 @@ class QueryPlanSuite extends SparkFunSuite {
     assert(countRelationsInPlan == 2)
     assert(countRelationsInPlanAndSubqueries == 5)
   }
+
+  test("SPARK-33035: consecutive attribute updates in parent plan nodes") {
+    val testRule = new Rule[LogicalPlan] {
+      override def apply(plan: LogicalPlan): LogicalPlan = plan.transformUpWithNewOutput {
+        case p @ Project(projList, _) =>
+          // Assigns new `ExprId`s for output references
+          val newPlan = p.copy(projectList = projList.map { ne => Alias(ne, ne.name)() })
+          val attrMapping = p.output.zip(newPlan.output)
+          newPlan -> attrMapping
+      }
+    }
+
+    val t = LocalRelation('a.int, 'b.int)
+    val plan = t.select($"a", $"b").select($"a", $"b").select($"a", $"b").analyze
+    assert(testRule(plan).resolved)
+  }
 }

From 7981f670d78cf1c769e7a5303c055ee8de7c4eec Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Wed, 7 Oct 2020 17:09:48 +0900
Subject: [PATCH 1091/1280] [SPARK-33073][PYTHON][3.0] Improve error handling
 on Pandas to Arrow conversion failures

### What changes were proposed in this pull request?

This improves error handling when a failure in conversion from Pandas to Arrow occurs. And fixes tests to be compatible with upcoming Arrow 2.0.0 release.

### Why are the changes needed?

Current tests will fail with Arrow 2.0.0 because of a change in error message when the schema is invalid. For these cases, the current error message also includes information on disabling safe conversion config, which is mainly meant for floating point truncation and overflow. The tests have been updated to use a message that is show for past Arrow versions, and upcoming.

If the user enters an invalid schema, the error produced by pyarrow is not consistent and either `TypeError` or `ArrowInvalid`, with the latter being caught, and raised as a `RuntimeError` with the extra info.

The error handling is improved by:

- narrowing the exception type to `TypeError`s, which `ArrowInvalid` is a subclass and what is raised on safe conversion failures.
- The exception is only raised with additional information on disabling "spark.sql.execution.pandas.convertToArrowArraySafely" if it is enabled in the first place.
- The original exception is chained to better show it to the user (only for Spark 3.1+ which requires Python 3)

### Does this PR introduce _any_ user-facing change?

Yes, the error re-raised changes from a RuntimeError to a ValueError, which better categorizes this type of error and in-line with the original Arrow error.

### How was this patch tested?

Existing tests, using pyarrow 1.0.1 and 2.0.0-snapshot, and Python 2 with 0.15.1

Closes #29962 from BryanCutler/arrow-better-handle-pandas-errors-SPARK-33073-branch-3.0.

Authored-by: Bryan Cutler <cutlerb@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/pandas/serializers.py        | 17 ++++++++++-------
 python/pyspark/sql/tests/test_arrow.py          |  9 +++++----
 .../sql/tests/test_pandas_grouped_map.py        | 15 ++++++++-------
 3 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index 4dd15d14b9c53..106684eb9b795 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -156,13 +156,16 @@ def create_array(s, t):
                 s = _check_series_convert_timestamps_internal(s, self._timezone)
             try:
                 array = pa.Array.from_pandas(s, mask=mask, type=t, safe=self._safecheck)
-            except pa.ArrowException as e:
-                error_msg = "Exception thrown when converting pandas.Series (%s) to Arrow " + \
-                            "Array (%s). It can be caused by overflows or other unsafe " + \
-                            "conversions warned by Arrow. Arrow safe type check can be " + \
-                            "disabled by using SQL config " + \
-                            "`spark.sql.execution.pandas.convertToArrowArraySafely`."
-                raise RuntimeError(error_msg % (s.dtype, t), e)
+            except ValueError as e:
+                if self._safecheck:
+                    error_msg = "Exception thrown when converting pandas.Series (%s) to " + \
+                                "Arrow Array (%s). It can be caused by overflows or other " + \
+                                "unsafe conversions warned by Arrow. Arrow safe type check " + \
+                                "can be disabled by using SQL config " + \
+                                "`spark.sql.execution.pandas.convertToArrowArraySafely`."
+                    raise ValueError(error_msg % (s.dtype, t), e)
+                else:
+                    raise e
             return array
 
         arrs = []
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 15c5cf1ad9b05..2c6231da3c161 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -266,11 +266,12 @@ def test_createDataFrame_with_schema(self):
     def test_createDataFrame_with_incorrect_schema(self):
         pdf = self.create_pandas_data_frame()
         fields = list(self.schema)
-        fields[0], fields[1] = fields[1], fields[0]  # swap str with int
+        fields[5], fields[6] = fields[6], fields[5]  # swap decimal with date
         wrong_schema = StructType(fields)
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "integer.*required"):
-                self.spark.createDataFrame(pdf, schema=wrong_schema)
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
+                    self.spark.createDataFrame(pdf, schema=wrong_schema)
 
     def test_createDataFrame_with_names(self):
         pdf = self.create_pandas_data_frame()
diff --git a/python/pyspark/sql/tests/test_pandas_grouped_map.py b/python/pyspark/sql/tests/test_pandas_grouped_map.py
index cc6167e619285..8e02b291cddea 100644
--- a/python/pyspark/sql/tests/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_grouped_map.py
@@ -450,15 +450,16 @@ def int_index(pdf):
         def column_name_typo(pdf):
             return pd.DataFrame({'iid': pdf.id, 'v': pdf.v})
 
-        @pandas_udf('id long, v int', PandasUDFType.GROUPED_MAP)
+        @pandas_udf('id long, v decimal', PandasUDFType.GROUPED_MAP)
         def invalid_positional_types(pdf):
-            return pd.DataFrame([(u'a', 1.2)])
+            return pd.DataFrame([(1, datetime.date(2020, 10, 5))])
 
-        with QuietTest(self.sc):
-            with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
-                grouped_df.apply(column_name_typo).collect()
-            with self.assertRaisesRegexp(Exception, "an integer is required"):
-                grouped_df.apply(invalid_positional_types).collect()
+        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
+            with QuietTest(self.sc):
+                with self.assertRaisesRegexp(Exception, "KeyError: 'id'"):
+                    grouped_df.apply(column_name_typo).collect()
+                with self.assertRaisesRegexp(Exception, "[D|d]ecimal.*got.*date"):
+                    grouped_df.apply(invalid_positional_types).collect()
 
     def test_positional_assignment_conf(self):
         with self.sql_conf({

From 45475afda9d0f328110b553006cb652ae19a31fb Mon Sep 17 00:00:00 2001
From: Stijn De Haes <stijndehaes@gmail.com>
Date: Wed, 7 Oct 2020 09:52:00 -0700
Subject: [PATCH 1092/1280] [SPARK-32067][K8S] Use unique ConfigMap name for
 executor pod template

### What changes were proposed in this pull request?

The pod template configmap always had the same name. This PR makes it unique.

### Why are the changes needed?

If you scheduled 2 spark jobs they will both use the same configmap name this will result in conflicts. This PR fixes that

**BEFORE**
```
$ kubectl get cm --all-namespaces -w | grep podspec
podspec-configmap                              1      65s
```

**AFTER**
```
$ kubectl get cm --all-namespaces -w | grep podspec
aaece65ef82e4a30b7b7800aad600d4f   spark-test-app-aac9f37502b2ca55-driver-podspec-conf-map   1      0s
```

This can be seen when running the integration tests

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Unit tests and the integration tests test if this works

Closes #29934 from stijndehaes/bugfix/SPARK-32067-unique-name-for-template-configmap.

Authored-by: Stijn De Haes <stijndehaes@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3099fd9f9d576c96642c0e66c74797b8882b70bb)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/deploy/k8s/Constants.scala     | 2 +-
 .../deploy/k8s/features/PodTemplateConfigMapStep.scala    | 6 ++++--
 .../k8s/features/PodTemplateConfigMapStepSuite.scala      | 8 +++++---
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index a3c74ff7b2885..e885e1ff2665e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -75,7 +75,7 @@ private[spark] object Constants {
   val EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME = "pod-spec-template.yml"
   val EXECUTOR_POD_SPEC_TEMPLATE_MOUNTPATH = "/opt/spark/pod-template"
   val POD_TEMPLATE_VOLUME = "pod-template-volume"
-  val POD_TEMPLATE_CONFIGMAP = "podspec-configmap"
+  val POD_TEMPLATE_CONFIGMAP = "driver-podspec-conf-map"
   val POD_TEMPLATE_KEY = "podspec-configmap-key"
 
   // Miscellaneous
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
index 7f41ca43589b6..1040419a4a6e9 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStep.scala
@@ -31,6 +31,8 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
 
   private val hasTemplate = conf.contains(KUBERNETES_EXECUTOR_PODTEMPLATE_FILE)
 
+  private val configmapName = s"${conf.resourceNamePrefix}-$POD_TEMPLATE_CONFIGMAP"
+
   def configurePod(pod: SparkPod): SparkPod = {
     if (hasTemplate) {
       val podWithVolume = new PodBuilder(pod.pod)
@@ -38,7 +40,7 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
             .addNewVolume()
               .withName(POD_TEMPLATE_VOLUME)
               .withNewConfigMap()
-                .withName(POD_TEMPLATE_CONFIGMAP)
+                .withName(configmapName)
                 .addNewItem()
                   .withKey(POD_TEMPLATE_KEY)
                   .withPath(EXECUTOR_POD_SPEC_TEMPLATE_FILE_NAME)
@@ -76,7 +78,7 @@ private[spark] class PodTemplateConfigMapStep(conf: KubernetesConf)
       val podTemplateString = Files.toString(new File(podTemplateFile), StandardCharsets.UTF_8)
       Seq(new ConfigMapBuilder()
           .withNewMetadata()
-            .withName(POD_TEMPLATE_CONFIGMAP)
+            .withName(configmapName)
           .endMetadata()
           .addToData(POD_TEMPLATE_KEY, podTemplateString)
         .build())
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
index 051320fa44c5e..1b38fd6a0d2ab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/PodTemplateConfigMapStepSuite.scala
@@ -16,13 +16,14 @@
  */
 package org.apache.spark.deploy.k8s.features
 
-import java.io.{File, PrintWriter}
+import java.io.PrintWriter
 import java.nio.file.Files
 
 import io.fabric8.kubernetes.api.model.ConfigMap
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._
+import org.apache.spark.deploy.k8s.Constants._
 import org.apache.spark.util.Utils
 
 class PodTemplateConfigMapStepSuite extends SparkFunSuite {
@@ -56,8 +57,9 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite {
 
     assert(configuredPod.pod.getSpec.getVolumes.size() === 1)
     val volume = configuredPod.pod.getSpec.getVolumes.get(0)
+    val generatedResourceName = s"${kubernetesConf.resourceNamePrefix}-$POD_TEMPLATE_CONFIGMAP"
     assert(volume.getName === Constants.POD_TEMPLATE_VOLUME)
-    assert(volume.getConfigMap.getName === Constants.POD_TEMPLATE_CONFIGMAP)
+    assert(volume.getConfigMap.getName === generatedResourceName)
     assert(volume.getConfigMap.getItems.size() === 1)
     assert(volume.getConfigMap.getItems.get(0).getKey === Constants.POD_TEMPLATE_KEY)
     assert(volume.getConfigMap.getItems.get(0).getPath ===
@@ -70,7 +72,7 @@ class PodTemplateConfigMapStepSuite extends SparkFunSuite {
 
     val resources = step.getAdditionalKubernetesResources()
     assert(resources.size === 1)
-    assert(resources.head.getMetadata.getName === Constants.POD_TEMPLATE_CONFIGMAP)
+    assert(resources.head.getMetadata.getName === generatedResourceName)
     assert(resources.head.isInstanceOf[ConfigMap])
     val configMap = resources.head.asInstanceOf[ConfigMap]
     assert(configMap.getData.size() === 1)

From a7e43185715549f14decc018f7a58e2119c99aae Mon Sep 17 00:00:00 2001
From: Yuning Zhang <yuning.zhang@databricks.com>
Date: Thu, 8 Oct 2020 12:18:06 +0900
Subject: [PATCH 1093/1280] [SPARK-33089][SQL] make avro format propagate
 Hadoop config from DS options to underlying HDFS file system

### What changes were proposed in this pull request?

In `AvroUtils`'s `inferSchema()`, propagate Hadoop config from DS options to underlying HDFS file system.

### Why are the changes needed?

There is a bug that when running:
```scala
spark.read.format("avro").options(conf).load(path)
```
The underlying file system will not receive the `conf` options.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

unit test added

Closes #29971 from yuningzh-db/avro_options.

Authored-by: Yuning Zhang <yuning.zhang@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit bbc887bf73233b8c65ace05929290c0de4f63de8)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/sql/avro/AvroUtils.scala    |  2 +-
 .../scala/org/apache/spark/sql/avro/AvroSuite.scala    | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
index 9ff89f60c05f8..a9f34bbea4c87 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroUtils.scala
@@ -42,7 +42,7 @@ private[sql] object AvroUtils extends Logging {
       spark: SparkSession,
       options: Map[String, String],
       files: Seq[FileStatus]): Option[StructType] = {
-    val conf = spark.sessionState.newHadoopConf()
+    val conf = spark.sessionState.newHadoopConfWithOptions(options)
     val parsedOptions = new AvroOptions(options, conf)
 
     if (parsedOptions.parameters.contains(ignoreExtensionKey)) {
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index e2ae489446d85..d2f49aef3774f 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -1799,6 +1799,16 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       assert(version === SPARK_VERSION_SHORT)
     }
   }
+
+  test("SPARK-33089: should propagate Hadoop config from DS options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      val conf = Map("ds_option" -> "value")
+      val path = "file:" + testAvro.stripPrefix("file:")
+      spark.read.format("avro").options(conf).load(path)
+    }
+  }
 }
 
 class AvroV1Suite extends AvroSuite {

From 782ab8e244252696c50b4b432d07a56c374b8680 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 8 Oct 2020 16:29:15 +0900
Subject: [PATCH 1094/1280] [SPARK-33091][SQL] Avoid using map instead of
 foreach to avoid potential side effect at callers of
 OrcUtils.readCatalystSchema

### What changes were proposed in this pull request?

This is a kind of a followup of SPARK-32646. New JIRA was filed to control the fixed versions properly.

When you use `map`, it might be lazily evaluated and not executed. To avoid this,  we should better use `foreach`. See also SPARK-16694. Current codes look not causing any bug for now but it should be best to fix to avoid potential issues.

### Why are the changes needed?

To avoid potential issues from `map` being lazy and not executed.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Ran related tests. CI in this PR should verify.

Closes #29974 from HyukjinKwon/SPARK-32646.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 5effa8ea261ba59214afedc2853d1b248b330ca6)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/sql/execution/datasources/orc/OrcFileFormat.scala     | 2 +-
 .../datasources/v2/orc/OrcPartitionReaderFactory.scala          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
index 69badb4f7d595..c540007d5aad1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFileFormat.scala
@@ -185,7 +185,7 @@ class OrcFileFormat
       } else {
         // ORC predicate pushdown
         if (orcFilterPushDown) {
-          OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).map { fileSchema =>
+          OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { fileSchema =>
             OrcFilters.createFilter(fileSchema, filters).foreach { f =>
               OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
             }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
index 1f38128e98fa5..b0ddee0a6b336 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/orc/OrcPartitionReaderFactory.scala
@@ -69,7 +69,7 @@ case class OrcPartitionReaderFactory(
 
   private def pushDownPredicates(filePath: Path, conf: Configuration): Unit = {
     if (orcFilterPushDown) {
-      OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).map { fileSchema =>
+      OrcUtils.readCatalystSchema(filePath, conf, ignoreCorruptFiles).foreach { fileSchema =>
         OrcFilters.createFilter(fileSchema, filters).foreach { f =>
           OrcInputFormat.setSearchArgument(conf, f, fileSchema.fieldNames)
         }

From c1b660efa31fdb79df058c847244791c9bec90ff Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 8 Oct 2020 11:50:53 -0700
Subject: [PATCH 1095/1280] [SPARK-33096][K8S] Use LinkedHashMap instead of Map
 for newlyCreatedExecutors

### What changes were proposed in this pull request?

This PR aims to use `LinkedHashMap` instead of `Map` for `newlyCreatedExecutors`.

### Why are the changes needed?

This makes log messages (INFO/DEBUG) more readable. This is helpful when `spark.kubernetes.allocation.batch.size` is large and especially when K8s dynamic allocation is used.

**BEFORE**
```
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 8 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 2 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 5 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 4 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 7 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 10 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 9 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 3 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 DEBUG ExecutorPodsAllocator: Executor with id 6 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:24:21 INFO ExecutorPodsAllocator: Deleting 9 excess pod requests (5,10,6,9,2,7,3,8,4).
```

**AFTER**
```
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 2 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 3 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 4 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 5 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 6 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 7 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 8 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 9 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 DEBUG ExecutorPodsAllocator: Executor with id 10 was not found in the Kubernetes cluster since it was created 0 milliseconds ago.
20/10/08 10:25:17 INFO ExecutorPodsAllocator: Deleting 9 excess pod requests (2,3,4,5,6,7,8,9,10).
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CI or `build/sbt -Pkubernetes "kubernetes/test"`

Closes #29979 from dongjoon-hyun/SPARK-K8S-LOG.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 4987db8c88b49a0c0d8503b6291455e92e114efa)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
index b394f35b15111..66cba55d8ef3a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
@@ -64,7 +64,7 @@ private[spark] class ExecutorPodsAllocator(
 
   // Executor IDs that have been requested from Kubernetes but have not been detected in any
   // snapshot yet. Mapped to the timestamp when they were created.
-  private val newlyCreatedExecutors = mutable.Map.empty[Long, Long]
+  private val newlyCreatedExecutors = mutable.LinkedHashMap.empty[Long, Long]
 
   private val dynamicAllocationEnabled = Utils.isDynamicAllocationEnabled(conf)
 

From dcffa56f04a1764619a69789fbf7169ffea2bdd4 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 9 Oct 2020 20:49:10 +0900
Subject: [PATCH 1096/1280] [SPARK-33101][ML][3.0] Make LibSVM format propagate
 Hadoop config from DS options to underlying HDFS file system

### What changes were proposed in this pull request?
Propagate LibSVM options to Hadoop configs in the LibSVM datasource.

### Why are the changes needed?
There is a bug that when running:
```scala
spark.read.format("libsvm").options(conf).load(path)
```
The underlying file system will not receive the `conf` options.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes, for example, users should read files from Azure Data Lake successfully:
```scala
def hadoopConf1() = Map[String, String](
  s"fs.adl.oauth2.access.token.provider.type" -> "ClientCredential",
  s"fs.adl.oauth2.client.id" -> dbutils.secrets.get(scope = "...", key = "..."),
  s"fs.adl.oauth2.credential" -> dbutils.secrets.get(scope = "...", key = "..."),
  s"fs.adl.oauth2.refresh.url" -> s"https://login.microsoftonline.com/.../oauth2/token")
val df = spark.read.format("libsvm").options(hadoopConf1).load("adl://....azuredatalakestore.net/foldersp1/...")
```
and not get the following exception because the settings above are not propagated to the filesystem:
```java
java.lang.IllegalArgumentException: No value for fs.adl.oauth2.access.token.provider found in conf file.
	at ....adl.AdlFileSystem.getNonEmptyVal(AdlFileSystem.java:820)
	at ....adl.AdlFileSystem.getCustomAccessTokenProvider(AdlFileSystem.java:220)
	at ....adl.AdlFileSystem.getAccessTokenProvider(AdlFileSystem.java:257)
	at ....adl.AdlFileSystem.initialize(AdlFileSystem.java:164)
	at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2669)
```

### How was this patch tested?
Added UT to `LibSVMRelationSuite`.

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Dongjoon Hyun <dhyunapple.com>
(cherry picked from commit 1234c66fa6b6d2c45edb40237788fa3bfdf96cf3)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #29986 from MaxGekk/ml-option-propagation-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/ml/source/libsvm/LibSVMRelation.scala    |  2 +-
 .../org/apache/spark/mllib/util/MLUtils.scala      |  6 ++++--
 .../ml/source/libsvm/LibSVMRelationSuite.scala     | 14 ++++++++++++--
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
index 11be1d85fbead..df64de4b10075 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/source/libsvm/LibSVMRelation.scala
@@ -101,7 +101,7 @@ private[libsvm] class LibSVMFileFormat
         "'numFeatures' option to avoid the extra scan.")
 
       val paths = files.map(_.getPath.toString)
-      val parsed = MLUtils.parseLibSVMFile(sparkSession, paths)
+      val parsed = MLUtils.parseLibSVMFile(sparkSession, paths, options)
       MLUtils.computeNumFeatures(parsed)
     }
 
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index 24113003c9675..2065bd4dc9dec 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -105,13 +105,15 @@ object MLUtils extends Logging {
   }
 
   private[spark] def parseLibSVMFile(
-      sparkSession: SparkSession, paths: Seq[String]): RDD[(Double, Array[Int], Array[Double])] = {
+      sparkSession: SparkSession,
+      paths: Seq[String],
+      options: Map[String, String]): RDD[(Double, Array[Int], Array[Double])] = {
     val lines = sparkSession.baseRelationToDataFrame(
       DataSource.apply(
         sparkSession,
         paths = paths,
         className = classOf[TextFileFormat].getName,
-        options = Map(DataSource.GLOB_PATHS_KEY -> "false")
+        options = options ++ Map(DataSource.GLOB_PATHS_KEY -> "false")
       ).resolveRelation(checkFilesExist = false))
       .select("value")
 
diff --git a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
index 0999892364e2c..cc0ca308cb668 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/source/libsvm/LibSVMRelationSuite.scala
@@ -27,12 +27,13 @@ import org.apache.spark.ml.attribute.AttributeGroup
 import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector, Vectors}
 import org.apache.spark.ml.linalg.SQLDataTypes.VectorType
 import org.apache.spark.mllib.util.MLlibTestSparkContext
-import org.apache.spark.sql.{Row, SaveMode}
+import org.apache.spark.sql.{FakeFileSystemRequiringDSOption, Row, SaveMode}
+import org.apache.spark.sql.catalyst.plans.SQLHelper
 import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
 import org.apache.spark.util.Utils
 
 
-class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
+class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext with SQLHelper {
   // Path for dataset
   var path: String = _
 
@@ -211,4 +212,13 @@ class LibSVMRelationSuite extends SparkFunSuite with MLlibTestSparkContext {
       assert(v == Vectors.sparse(2, Seq((0, 2.0), (1, 3.0))))
     }
   }
+
+  test("SPARK-33101: should propagate Hadoop config from DS options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      val df = spark.read.option("ds_option", "value").format("libsvm").load(path)
+      assert(df.columns(0) == "label")
+    }
+  }
 }

From 9892b3e452de490f7d2346bacf971ec13b92b219 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 9 Oct 2020 21:14:26 +0900
Subject: [PATCH 1097/1280] [SPARK-33094][SQL][3.0] Make ORC format propagate
 Hadoop config from DS options to underlying HDFS file system

### What changes were proposed in this pull request?
Propagate ORC options to Hadoop configs in Hive `OrcFileFormat` and in the regular ORC datasource.

### Why are the changes needed?
There is a bug that when running:
```scala
spark.read.format("orc").options(conf).load(path)
```
The underlying file system will not receive the conf options.

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
Added UT to `OrcSourceSuite`.

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Dongjoon Hyun <dhyunapple.com>
(cherry picked from commit c5f6af9f17498bb0ec393c16616f2d99e5d3ee3d)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #29985 from MaxGekk/orc-option-propagation-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../execution/datasources/orc/OrcUtils.scala    |  6 +++---
 .../datasources/orc/OrcSourceSuite.scala        | 17 ++++++++++++++++-
 .../spark/sql/hive/orc/OrcFileFormat.scala      |  2 +-
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index be36432d6325f..c29287f1789e6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -81,10 +81,10 @@ object OrcUtils extends Logging {
     }
   }
 
-  def readSchema(sparkSession: SparkSession, files: Seq[FileStatus])
+  def readSchema(sparkSession: SparkSession, files: Seq[FileStatus], options: Map[String, String])
       : Option[StructType] = {
     val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
-    val conf = sparkSession.sessionState.newHadoopConf()
+    val conf = sparkSession.sessionState.newHadoopConfWithOptions(options)
     files.toIterator.map(file => readSchema(file.getPath, conf, ignoreCorruptFiles)).collectFirst {
       case Some(schema) =>
         logDebug(s"Reading schema from file $files, got Hive schema string: $schema")
@@ -125,7 +125,7 @@ object OrcUtils extends Logging {
       SchemaMergeUtils.mergeSchemasInParallel(
         sparkSession, options, files, OrcUtils.readOrcSchemasInParallel)
     } else {
-      OrcUtils.readSchema(sparkSession, files)
+      OrcUtils.readSchema(sparkSession, files, options)
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
index b70fd7476ed98..5fa23a40e677d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcSourceSuite.scala
@@ -32,7 +32,7 @@ import org.apache.orc.impl.RecordReaderImpl
 import org.scalatest.BeforeAndAfterAll
 
 import org.apache.spark.{SPARK_VERSION_SHORT, SparkException}
-import org.apache.spark.sql.{Row, SPARK_VERSION_METADATA_KEY}
+import org.apache.spark.sql.{FakeFileSystemRequiringDSOption, Row, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.execution.datasources.SchemaMergeUtils
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
@@ -538,6 +538,21 @@ abstract class OrcSuite extends OrcTest with BeforeAndAfterAll {
       }
     }
   }
+
+  test("SPARK-33094: should propagate Hadoop config from DS options to underlying file system") {
+    withSQLConf(
+      "fs.file.impl" -> classOf[FakeFileSystemRequiringDSOption].getName,
+      "fs.file.impl.disable.cache" -> "true") {
+      Seq(false, true).foreach { mergeSchema =>
+        withTempPath { dir =>
+          val path = dir.getAbsolutePath
+          val conf = Map("ds_option" -> "value", "mergeSchema" -> mergeSchema.toString)
+          spark.range(1).write.options(conf).orc(path)
+          checkAnswer(spark.read.options(conf).orc(path), Row(0))
+        }
+      }
+    }
+  }
 }
 
 class OrcSourceSuite extends OrcSuite with SharedSparkSession {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
index 356b92b4652b3..d1ee1baadcbce 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/orc/OrcFileFormat.scala
@@ -75,7 +75,7 @@ class OrcFileFormat extends FileFormat with DataSourceRegister with Serializable
       val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
       OrcFileOperator.readSchema(
         files.map(_.getPath.toString),
-        Some(sparkSession.sessionState.newHadoopConf()),
+        Some(sparkSession.sessionState.newHadoopConfWithOptions(options)),
         ignoreCorruptFiles
       )
     }

From 0601fc7bfc85b76be960c0b4ce642fd39675926c Mon Sep 17 00:00:00 2001
From: Pablo <pablo.langa@stratio.com>
Date: Mon, 12 Oct 2020 14:18:34 -0700
Subject: [PATCH 1098/1280] [SPARK-33118][SQL] CREATE TEMPORARY TABLE fails
 with location

We have a problem when you use CREATE TEMPORARY TABLE with LOCATION

```scala
spark.range(3).write.parquet("/tmp/testspark1")

sql("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path '/tmp/testspark1')")
sql("CREATE TEMPORARY TABLE t USING parquet LOCATION '/tmp/testspark1'")
```
```scala
org.apache.spark.sql.AnalysisException: Unable to infer schema for Parquet. It must be specified manually.;
  at org.apache.spark.sql.execution.datasources.DataSource.$anonfun$getOrInferFileFormatSchema$12(DataSource.scala:200)
  at scala.Option.getOrElse(Option.scala:189)
  at org.apache.spark.sql.execution.datasources.DataSource.getOrInferFileFormatSchema(DataSource.scala:200)
  at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:408)
  at org.apache.spark.sql.execution.datasources.CreateTempViewUsing.run(ddl.scala:94)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
  at org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
  at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229)
  at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3618)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3616)
  at org.apache.spark.sql.Dataset.<init>(Dataset.scala:229)
  at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
  at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97)
  at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:607)
  at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:764)
  at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:602)
```
This bug was introduced by SPARK-30507.
sparksqlparser --> visitCreateTable --> visitCreateTableClauses --> cleanTableOptions extract the path from the options but in this case CreateTempViewUsing need the path in the options map.

To fix the problem

No

Unit testing and manual testing

Closes #30014 from planga82/bugfix/SPARK-33118_create_temp_table_location.

Authored-by: Pablo <pablo.langa@stratio.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 819f12ee2fe3cce0c59221c2b02831274c769b23)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../apache/spark/sql/execution/SparkSqlParser.scala |  6 ++++--
 .../spark/sql/execution/SparkSqlParserSuite.scala   | 13 +++++++++++--
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 44069f3b20b4a..3e4bc48553c5a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -196,7 +196,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
         operationNotAllowed("CREATE TEMPORARY TABLE IF NOT EXISTS", ctx)
       }
 
-      val (_, _, _, options, _, _) = visitCreateTableClauses(ctx.createTableClauses())
+      val (_, _, _, options, location, _) = visitCreateTableClauses(ctx.createTableClauses())
       val provider = Option(ctx.tableProvider).map(_.multipartIdentifier.getText).getOrElse(
         throw new ParseException("CREATE TEMPORARY TABLE without a provider is not allowed.", ctx))
       val schema = Option(ctx.colTypeList()).map(createSchema)
@@ -205,7 +205,9 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) {
           "CREATE TEMPORARY VIEW ... USING ... instead")
 
       val table = tableIdentifier(ident, "CREATE TEMPORARY VIEW", ctx)
-      CreateTempViewUsing(table, schema, replace = false, global = false, provider, options)
+      val optionsWithLocation = location.map(l => options + ("path" -> l)).getOrElse(options)
+      CreateTempViewUsing(table, schema, replace = false, global = false, provider,
+        optionsWithLocation)
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
index 6a58c8f3903e5..343d3c1c13469 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala
@@ -25,8 +25,8 @@ import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, Concat, SortOrder}
 import org.apache.spark.sql.catalyst.plans.logical._
 import org.apache.spark.sql.execution.command._
-import org.apache.spark.sql.execution.datasources.{CreateTable, RefreshResource}
-import org.apache.spark.sql.internal.{HiveSerDe, SQLConf}
+import org.apache.spark.sql.execution.datasources.{CreateTable, CreateTempViewUsing, RefreshResource}
+import org.apache.spark.sql.internal.{HiveSerDe, SQLConf, StaticSQLConf}
 import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
 
 /**
@@ -81,6 +81,15 @@ class SparkSqlParserSuite extends AnalysisTest {
     intercept("REFRESH", "Resource paths cannot be empty in REFRESH statements")
   }
 
+  test("SPARK-33118 CREATE TMEPORARY TABLE with LOCATION") {
+    assertEqual("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path '/data/tmp/testspark1')",
+      CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet",
+        Map("path" -> "/data/tmp/testspark1")))
+    assertEqual("CREATE TEMPORARY TABLE t USING parquet LOCATION '/data/tmp/testspark1'",
+      CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet",
+        Map("path" -> "/data/tmp/testspark1")))
+  }
+
   private def createTableUsing(
       table: String,
       database: Option[String] = None,

From 9430ae673010a7c24f09b5d041f9f9304d27c233 Mon Sep 17 00:00:00 2001
From: Denis Pyshev <git@gemelen.net>
Date: Tue, 13 Oct 2020 21:37:26 +0900
Subject: [PATCH 1099/1280] [SPARK-33115][BUILD][DOCS] Fix javadoc errors in
 `kvstore` and `unsafe` modules

### What changes were proposed in this pull request?

Fix Javadoc generation errors in `kvstore` and `unsafe` modules according to error message hints.

### Why are the changes needed?

Fixes `doc` task failures which prevented other tasks successful executions (eg `publishLocal` task depends on `doc` task).

### Does this PR introduce _any_ user-facing change?

No.
Meaning of text in Javadoc is stayed the same.

### How was this patch tested?

Run `build/sbt kvstore/Compile/doc`, `build/sbt unsafe/Compile/doc` and `build/sbt doc` without errors.

Closes #30007 from gemelen/feature/doc-task-fix.

Authored-by: Denis Pyshev <git@gemelen.net>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1b0875b6924b4f29aa3cdecc26f8103fcae3dc55)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../java/org/apache/spark/util/kvstore/InMemoryStore.java    | 5 +++--
 .../main/java/org/apache/spark/unsafe/types/UTF8String.java  | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
index 42e090bc83ed1..431c7e42774e4 100644
--- a/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
+++ b/common/kvstore/src/main/java/org/apache/spark/util/kvstore/InMemoryStore.java
@@ -164,8 +164,9 @@ public void clear() {
   }
 
   /**
-   * An alias class for the type "ConcurrentHashMap<Comparable<Object>, Boolean>", which is used
-   * as a concurrent hashset for storing natural keys and the boolean value doesn't matter.
+   * An alias class for the type "{@literal ConcurrentHashMap<Comparable<Object>, Boolean>}",
+   * which is used as a concurrent hashset for storing natural keys
+   * and the boolean value doesn't matter.
    */
   private static class NaturalKeys extends ConcurrentHashMap<Comparable<Object>, Boolean> {}
 
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 43bd7976c5d33..b8dda22240042 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -563,7 +563,7 @@ public UTF8String trim() {
   }
 
   /**
-   * Trims whitespaces (<= ASCII 32) from both ends of this string.
+   * Trims whitespaces ({@literal <=} ASCII 32) from both ends of this string.
    *
    * Note that, this method is the same as java's {@link String#trim}, and different from
    * {@link UTF8String#trim()} which remove only spaces(= ASCII 32) from both ends.

From 205b65e35c45d3e4f5b3f798a3b48410e94ab2c5 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 14 Oct 2020 12:14:32 +0900
Subject: [PATCH 1100/1280] [SPARK-33134][SQL][3.0] Return partial results only
 for root JSON objects

### What changes were proposed in this pull request?
In the PR, I propose to restrict the partial result feature only by root JSON objects. JSON datasource as well as `from_json()` will return `null` for malformed nested JSON objects.

### Why are the changes needed?
1. To not raise exception to users in the PERMISSIVE mode
2. To fix a regression and to have the same behavior as Spark 2.4.x has
3. Current implementation of partial result is supposed to work only for root (top-level) JSON objects, and not tested for bad nested complex JSON fields.

### Does this PR introduce _any_ user-facing change?
Yes. Before the changes, the code below:
```scala
    val pokerhand_raw = Seq("""[{"cards": [19], "playerId": 123456}]""").toDF("events")
    val event = new StructType().add("playerId", LongType).add("cards", ArrayType(new StructType().add("id", LongType).add("rank", StringType)))
    val pokerhand_events = pokerhand_raw.select(from_json($"events", ArrayType(event)).as("event"))
    pokerhand_events.show
```
throws the exception even in the default **PERMISSIVE** mode:
```java
java.lang.ClassCastException: java.lang.Long cannot be cast to org.apache.spark.sql.catalyst.util.ArrayData
  at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow.getArray(rows.scala:48)
  at org.apache.spark.sql.catalyst.expressions.BaseGenericInternalRow.getArray$(rows.scala:48)
  at org.apache.spark.sql.catalyst.expressions.GenericInternalRow.getArray(rows.scala:195)
```

After the changes:
```
+-----+
|event|
+-----+
| null|
+-----+
```

### How was this patch tested?
Added a test to `JsonFunctionsSuite`.

Closes #30032 from MaxGekk/json-skip-row-wrong-schema-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/catalyst/json/JacksonParser.scala    |  7 ++++---
 .../org/apache/spark/sql/JsonFunctionsSuite.scala  | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index e038f777c7a41..70f7f8f468f3a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -86,7 +86,7 @@ class JacksonParser(
     val elementConverter = makeConverter(st)
     val fieldConverters = st.map(_.dataType).map(makeConverter).toArray
     (parser: JsonParser) => parseJsonToken[Iterable[InternalRow]](parser, st) {
-      case START_OBJECT => Some(convertObject(parser, st, fieldConverters))
+      case START_OBJECT => Some(convertObject(parser, st, fieldConverters, isRoot = true))
         // SPARK-3308: support reading top level JSON arrays and take every element
         // in such an array as a row
         //
@@ -375,7 +375,8 @@ class JacksonParser(
   private def convertObject(
       parser: JsonParser,
       schema: StructType,
-      fieldConverters: Array[ValueConverter]): InternalRow = {
+      fieldConverters: Array[ValueConverter],
+      isRoot: Boolean = false): InternalRow = {
     val row = new GenericInternalRow(schema.length)
     var badRecordException: Option[Throwable] = None
 
@@ -386,7 +387,7 @@ class JacksonParser(
             row.update(index, fieldConverters(index).apply(parser))
           } catch {
             case e: SparkUpgradeException => throw e
-            case NonFatal(e) =>
+            case NonFatal(e) if isRoot =>
               badRecordException = badRecordException.orElse(Some(e))
               parser.skipChildren()
           }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
index a4ed7ccc5508a..5e3931cecf1c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
@@ -712,4 +712,18 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
          | """.stripMargin)
     checkAnswer(toDF("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"), toDF("yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"))
   }
+
+  test("SPARK-33134: return partial results only for root JSON objects") {
+    val st = new StructType()
+      .add("c1", LongType)
+      .add("c2", ArrayType(new StructType().add("c3", LongType).add("c4", StringType)))
+    val df1 = Seq("""{"c2": [19], "c1": 123456}""").toDF("c0")
+    checkAnswer(df1.select(from_json($"c0", st)), Row(Row(123456, null)))
+    val df2 = Seq("""{"data": {"c2": [19], "c1": 123456}}""").toDF("c0")
+    checkAnswer(df2.select(from_json($"c0", new StructType().add("data", st))), Row(Row(null)))
+    val df3 = Seq("""[{"c2": [19], "c1": 123456}]""").toDF("c0")
+    checkAnswer(df3.select(from_json($"c0", ArrayType(st))), Row(null))
+    val df4 = Seq("""{"c2": [19]}""").toDF("c0")
+    checkAnswer(df4.select(from_json($"c0", MapType(StringType, st))), Row(null))
+  }
 }

From 2ebea135a1f4e4cb3187ffc8e77d3f52d3b3a91a Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Wed, 14 Oct 2020 08:30:03 -0700
Subject: [PATCH 1101/1280] [SPARK-33136][SQL] Fix mistakenly swapped parameter
 in V2WriteCommand.outputResolved

### What changes were proposed in this pull request?

This PR proposes to fix a bug on calling `DataType.equalsIgnoreCompatibleNullability` with mistakenly swapped parameters in `V2WriteCommand.outputResolved`. The order of parameters for `DataType.equalsIgnoreCompatibleNullability` are `from` and `to`, which says that the right order of matching variables are `inAttr` and `outAttr`.

### Why are the changes needed?

Spark throws AnalysisException due to unresolved operator in v2 write, while the operator is unresolved due to a bug that parameters to call `DataType.equalsIgnoreCompatibleNullability` in `outputResolved` have been swapped.

### Does this PR introduce _any_ user-facing change?

Yes, end users no longer suffer on unresolved operator in v2 write if they're trying to write dataframe containing non-nullable complex types against table matching complex types as nullable.

### How was this patch tested?

New UT added.

Closes #30033 from HeartSaVioR/SPARK-33136.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 8e5cb1d276686ec428e4e6aa1c3cfd6bb99e4e9a)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../catalyst/plans/logical/v2Commands.scala   |  2 +-
 .../spark/sql/DataFrameWriterV2Suite.scala    | 87 ++++++++++++++++++-
 2 files changed, 84 insertions(+), 5 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index b4120d9f64cc5..d2830a9bd0145 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -45,7 +45,7 @@ trait V2WriteCommand extends Command {
           case (inAttr, outAttr) =>
             // names and types must match, nullability must be compatible
             inAttr.name == outAttr.name &&
-              DataType.equalsIgnoreCompatibleNullability(outAttr.dataType, inAttr.dataType) &&
+              DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outAttr.dataType) &&
               (outAttr.nullable || !inAttr.nullable)
         }
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
index 508eefafd0754..ff5c6242987de 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWriterV2Suite.scala
@@ -23,16 +23,15 @@ import scala.collection.JavaConverters._
 
 import org.scalatest.BeforeAndAfter
 
-import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NoSuchTableException, TableAlreadyExistsException}
-import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic}
+import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, NamedRelation, NoSuchTableException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.plans.logical.{AppendData, LogicalPlan, OverwriteByExpression, OverwritePartitionsDynamic, V2WriteCommand}
 import org.apache.spark.sql.connector.{InMemoryTable, InMemoryTableCatalog}
 import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
 import org.apache.spark.sql.connector.expressions.{BucketTransform, DaysTransform, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, YearsTransform}
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType}
-import org.apache.spark.sql.types.TimestampType
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType, MapType, StringType, StructField, StructType, TimestampType}
 import org.apache.spark.sql.util.QueryExecutionListener
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.Utils
@@ -101,6 +100,86 @@ class DataFrameWriterV2Suite extends QueryTest with SharedSparkSession with Befo
     assert(v2.catalog.exists(_ == catalogPlugin))
   }
 
+  case class FakeV2WriteCommand(table: NamedRelation, query: LogicalPlan) extends V2WriteCommand
+
+  test("SPARK-33136 output resolved on complex types for V2 write commands") {
+    val tableCatalog = catalog("testcat")
+
+    def assertTypeCompatibility(name: String, fromType: DataType, toType: DataType): Unit = {
+      val fromTableName = s"from_table_$name"
+      tableCatalog.createTable(
+        Identifier.of(Array(), fromTableName),
+        StructType(Array(StructField("col", fromType))),
+        Array.empty,
+        new java.util.HashMap[String, String]())
+
+      val toTable = tableCatalog.createTable(
+        Identifier.of(Array(), s"to_table_$name"),
+        StructType(Array(StructField("col", toType))),
+        Array.empty,
+        new java.util.HashMap[String, String]())
+
+      val df = spark.table(s"testcat.$fromTableName")
+
+      val relation = DataSourceV2Relation.create(toTable, Some(tableCatalog), None)
+      val writeCommand = FakeV2WriteCommand(relation, df.queryExecution.analyzed)
+
+      assert(writeCommand.outputResolved, s"Unable to write from type $fromType to type $toType.")
+    }
+
+    // The major difference between `from` and `to` is that `from` is a complex type
+    // with non-nullable, whereas `to` is same data type with flipping nullable.
+
+    // nested struct type
+    val fromStructType = StructType(Array(
+      StructField("s", StringType),
+      StructField("i_nonnull", IntegerType, nullable = false),
+      StructField("st", StructType(Array(
+        StructField("l", LongType),
+        StructField("s_nonnull", StringType, nullable = false))))))
+
+    val toStructType = StructType(Array(
+      StructField("s", StringType),
+      StructField("i_nonnull", IntegerType),
+      StructField("st", StructType(Array(
+        StructField("l", LongType),
+        StructField("s_nonnull", StringType))))))
+
+    assertTypeCompatibility("struct", fromStructType, toStructType)
+
+    // array type
+    assertTypeCompatibility("array", ArrayType(LongType, containsNull = false),
+      ArrayType(LongType, containsNull = true))
+
+    // array type with struct type
+    val fromArrayWithStructType = ArrayType(
+      StructType(Array(StructField("s", StringType, nullable = false))),
+      containsNull = false)
+
+    val toArrayWithStructType = ArrayType(
+      StructType(Array(StructField("s", StringType))),
+      containsNull = true)
+
+    assertTypeCompatibility("array_struct", fromArrayWithStructType, toArrayWithStructType)
+
+    // map type
+    assertTypeCompatibility("map", MapType(IntegerType, StringType, valueContainsNull = false),
+      MapType(IntegerType, StringType, valueContainsNull = true))
+
+    // map type with struct type
+    val fromMapWithStructType = MapType(
+      IntegerType,
+      StructType(Array(StructField("s", StringType, nullable = false))),
+      valueContainsNull = false)
+
+    val toMapWithStructType = MapType(
+      IntegerType,
+      StructType(Array(StructField("s", StringType))),
+      valueContainsNull = true)
+
+    assertTypeCompatibility("map_struct", fromMapWithStructType, toMapWithStructType)
+  }
+
   test("Append: basic append") {
     spark.sql("CREATE TABLE testcat.table_name (id bigint, data string) USING foo")
 

From d9669bdf0ff4ed9951d7077b8dc9ad94507615c5 Mon Sep 17 00:00:00 2001
From: Adam Binford <adam.binford@radiantsolutions.com>
Date: Thu, 15 Oct 2020 11:59:29 +0900
Subject: [PATCH 1102/1280] [SPARK-33146][CORE] Check for non-fatal errors when
 loading new applications in SHS

### What changes were proposed in this pull request?

Adds an additional check for non-fatal errors when attempting to add a new entry to the history server application listing.

### Why are the changes needed?

A bad rolling event log folder (missing appstatus file or no log files) would cause no applications to be loaded by the Spark history server. Figuring out why invalid event log folders are created in the first place will be addressed in separate issues, this just lets the history server skip the invalid folder and successfully load all the valid applications.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

New UT

Closes #30037 from Kimahriman/bug/rolling-log-crashing-history.

Authored-by: Adam Binford <adam.binford@radiantsolutions.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit 9ab0ec4e38e5df0537b38cb0f89e004ad57bec90)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 .../deploy/history/FsHistoryProvider.scala    |  3 ++
 .../history/FsHistoryProviderSuite.scala      | 49 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index c26215295e914..59707083648a7 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -526,6 +526,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
                 reader.fileSizeForLastIndex > 0
               } catch {
                 case _: FileNotFoundException => false
+                case NonFatal(e) =>
+                  logWarning(s"Error while reading new log ${reader.rootPath}", e)
+                  false
               }
 
             case _: FileNotFoundException =>
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index c2f34fc3a95ed..f3beb35f1f011 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -1470,6 +1470,55 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
+  test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+      val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)
+
+      val provider = new FsHistoryProvider(conf)
+
+      val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf)
+      writer.start()
+
+      writeEventsToRollingWriter(writer, Seq(
+        SparkListenerApplicationStart("app", Some("app"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(dir.listFiles().size === 1)
+      assert(provider.getListing.length === 1)
+
+      // Manually delete the appstatus file to make an invalid rolling event log
+      val appStatusPath = RollingEventLogFilesWriter.getAppStatusFilePath(new Path(writer.logPath),
+        "app", None, true)
+      fs.delete(appStatusPath, false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 0)
+
+      // Create a new application
+      val writer2 = new RollingEventLogFilesWriter("app2", None, dir.toURI, conf, hadoopConf)
+      writer2.start()
+      writeEventsToRollingWriter(writer2, Seq(
+        SparkListenerApplicationStart("app2", Some("app2"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+
+      // Both folders exist but only one application found
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 1)
+      assert(dir.listFiles().size === 2)
+
+      // Make sure a new provider sees the valid application
+      provider.stop()
+      val newProvider = new FsHistoryProvider(conf)
+      newProvider.checkForLogs()
+      assert(newProvider.getListing.length === 1)
+    }
+  }
+
   /**
    * Asks the provider to check for logs and calls a function to perform checks on the updated
    * app list. Example:

From 0b7b811c6464ac8a0fe5230dc49aefc2f5507db8 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 14 Oct 2020 20:48:13 -0700
Subject: [PATCH 1103/1280] [SPARK-33153][SQL][TESTS] Ignore Spark 2.4 in
 HiveExternalCatalogVersionsSuite on Python 3.8/3.9

### What changes were proposed in this pull request?

This PR aims to ignore Apache Spark 2.4.x distribution in HiveExternalCatalogVersionsSuite if Python version is 3.8 or 3.9.

### Why are the changes needed?

Currently, `HiveExternalCatalogVersionsSuite` is broken on the latest OS like `Ubuntu 20.04` because its default Python version is 3.8. PySpark 2.4.x doesn't work on Python 3.8 due to SPARK-29536.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually.
```
$ python3 --version
Python 3.8.5

$ build/sbt "hive/testOnly *.HiveExternalCatalogVersionsSuite"
...
[info] All tests passed.
[info] Passed: Total 1, Failed 0, Errors 0, Passed 1
```

Closes #30044 from dongjoon-hyun/SPARK-33153.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit ec34a001ad0ef57a496f29a6523d905128875b17)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../src/main/scala/org/apache/spark/TestUtils.scala | 13 +++++++++++++
 .../sql/hive/HiveExternalCatalogVersionsSuite.scala |  3 ++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala
index 1e00769260a5f..054e7b07d1da6 100644
--- a/core/src/main/scala/org/apache/spark/TestUtils.scala
+++ b/core/src/main/scala/org/apache/spark/TestUtils.scala
@@ -249,6 +249,19 @@ private[spark] object TestUtils {
     attempt.isSuccess && attempt.get == 0
   }
 
+  def isPythonVersionAtLeast38(): Boolean = {
+    val attempt = if (Utils.isWindows) {
+      Try(Process(Seq("cmd.exe", "/C", "python3 --version"))
+        .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9")))
+        .exitValue())
+    } else {
+      Try(Process(Seq("sh", "-c", "python3 --version"))
+        .run(ProcessLogger(s => s.startsWith("Python 3.8") || s.startsWith("Python 3.9")))
+        .exitValue())
+    }
+    attempt.isSuccess && attempt.get == 0
+  }
+
   /**
    * Returns the response code from an HTTP(S) URL.
    */
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index cbfdb7fac88d8..b81b7e8ec0c0f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -234,7 +234,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
   // Tests the latest version of every release line.
   val testingVersions: Seq[String] = {
     import scala.io.Source
-    try {
+    val versions: Seq[String] = try {
       Source.fromURL(s"${releaseMirror}/spark").mkString
         .split("\n")
         .filter(_.contains("""<li><a href="spark-"""))
@@ -245,6 +245,7 @@ object PROCESS_TABLES extends QueryTest with SQLTestUtils {
       // do not throw exception during object initialization.
       case NonFatal(_) => Seq("3.0.1", "2.4.7") // A temporary fallback to use a specific version
     }
+    versions.filter(v => v.startsWith("3") || !TestUtils.isPythonVersionAtLeast38())
   }
 
   protected var spark: SparkSession = _

From e40c147a5d194adbba13f12590959dc68347ec14 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 14 Oct 2020 21:47:46 -0700
Subject: [PATCH 1104/1280] Revert "[SPARK-33146][CORE] Check for non-fatal
 errors when loading new applications in SHS"

This reverts commit d9669bdf0ff4ed9951d7077b8dc9ad94507615c5.
---
 .../deploy/history/FsHistoryProvider.scala    |  3 --
 .../history/FsHistoryProviderSuite.scala      | 49 -------------------
 2 files changed, 52 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 59707083648a7..c26215295e914 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -526,9 +526,6 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
                 reader.fileSizeForLastIndex > 0
               } catch {
                 case _: FileNotFoundException => false
-                case NonFatal(e) =>
-                  logWarning(s"Error while reading new log ${reader.rootPath}", e)
-                  false
               }
 
             case _: FileNotFoundException =>
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index f3beb35f1f011..c2f34fc3a95ed 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -1470,55 +1470,6 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
-  test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") {
-    withTempDir { dir =>
-      val conf = createTestConf(true)
-      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
-      val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
-      val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)
-
-      val provider = new FsHistoryProvider(conf)
-
-      val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf)
-      writer.start()
-
-      writeEventsToRollingWriter(writer, Seq(
-        SparkListenerApplicationStart("app", Some("app"), 0, "user", None),
-        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
-      provider.checkForLogs()
-      provider.cleanLogs()
-      assert(dir.listFiles().size === 1)
-      assert(provider.getListing.length === 1)
-
-      // Manually delete the appstatus file to make an invalid rolling event log
-      val appStatusPath = RollingEventLogFilesWriter.getAppStatusFilePath(new Path(writer.logPath),
-        "app", None, true)
-      fs.delete(appStatusPath, false)
-      provider.checkForLogs()
-      provider.cleanLogs()
-      assert(provider.getListing.length === 0)
-
-      // Create a new application
-      val writer2 = new RollingEventLogFilesWriter("app2", None, dir.toURI, conf, hadoopConf)
-      writer2.start()
-      writeEventsToRollingWriter(writer2, Seq(
-        SparkListenerApplicationStart("app2", Some("app2"), 0, "user", None),
-        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
-
-      // Both folders exist but only one application found
-      provider.checkForLogs()
-      provider.cleanLogs()
-      assert(provider.getListing.length === 1)
-      assert(dir.listFiles().size === 2)
-
-      // Make sure a new provider sees the valid application
-      provider.stop()
-      val newProvider = new FsHistoryProvider(conf)
-      newProvider.checkForLogs()
-      assert(newProvider.getListing.length === 1)
-    }
-  }
-
   /**
    * Asks the provider to check for logs and calls a function to perform checks on the updated
    * app list. Example:

From d0f1120f3fb524a52df71e03c3d28ac82f76c1a3 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 16 Oct 2020 10:28:15 +0900
Subject: [PATCH 1105/1280] [SPARK-33163][SQL][TESTS] Check the metadata key
 'org.apache.spark.legacyDateTime' in Avro/Parquet files

### What changes were proposed in this pull request?
Added a couple tests to `AvroSuite` and to `ParquetIOSuite` to check that the metadata key 'org.apache.spark.legacyDateTime' is written correctly depending on the SQL configs:
- spark.sql.legacy.avro.datetimeRebaseModeInWrite
- spark.sql.legacy.parquet.datetimeRebaseModeInWrite

This is a follow up https://github.com/apache/spark/pull/28137.

### Why are the changes needed?
1. To improve test coverage
2. To make sure that the metadata key is actually saved to Avro/Parquet files

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the added tests:
```
$ build/sbt "testOnly org.apache.spark.sql.execution.datasources.parquet.ParquetIOSuite"
$ build/sbt "avro/test:testOnly org.apache.spark.sql.avro.AvroV1Suite"
$ build/sbt "avro/test:testOnly org.apache.spark.sql.avro.AvroV2Suite"
```

Closes #30061 from MaxGekk/parquet-test-metakey.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 38c05af1d5538fc6ad00cdb57c1a90e90d04e25d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/avro/AvroSuite.scala | 40 ++++++++++++---
 .../datasources/parquet/ParquetIOSuite.scala  | 51 ++++++++++++++-----
 2 files changed, 73 insertions(+), 18 deletions(-)

diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index d2f49aef3774f..5d7d2e486c524 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -1788,15 +1788,19 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  private def checkMetaData(path: java.io.File, key: String, expectedValue: String): Unit = {
+    val avroFiles = path.listFiles()
+      .filter(f => f.isFile && !f.getName.startsWith(".") && !f.getName.startsWith("_"))
+    assert(avroFiles.length === 1)
+    val reader = DataFileReader.openReader(avroFiles(0), new GenericDatumReader[GenericRecord]())
+    val value = reader.asInstanceOf[DataFileReader[_]].getMetaString(key)
+    assert(value === expectedValue)
+  }
+
   test("SPARK-31327: Write Spark version into Avro file metadata") {
     withTempPath { path =>
       spark.range(1).repartition(1).write.format("avro").save(path.getCanonicalPath)
-      val avroFiles = path.listFiles()
-        .filter(f => f.isFile && !f.getName.startsWith(".") && !f.getName.startsWith("_"))
-      assert(avroFiles.length === 1)
-      val reader = DataFileReader.openReader(avroFiles(0), new GenericDatumReader[GenericRecord]())
-      val version = reader.asInstanceOf[DataFileReader[_]].getMetaString(SPARK_VERSION_METADATA_KEY)
-      assert(version === SPARK_VERSION_SHORT)
+      checkMetaData(path, SPARK_VERSION_METADATA_KEY, SPARK_VERSION_SHORT)
     }
   }
 
@@ -1809,6 +1813,30 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
       spark.read.format("avro").options(conf).load(path)
     }
   }
+
+  test("SPARK-33163: write the metadata key 'org.apache.spark.legacyDateTime'") {
+    def saveTs(dir: java.io.File): Unit = {
+      Seq(Timestamp.valueOf("2020-10-15 01:02:03")).toDF()
+        .repartition(1)
+        .write
+        .format("avro")
+        .save(dir.getAbsolutePath)
+    }
+    withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        checkMetaData(dir, SPARK_LEGACY_DATETIME, "")
+      }
+    }
+    Seq(CORRECTED, EXCEPTION).foreach { mode =>
+      withSQLConf(SQLConf.LEGACY_AVRO_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+        withTempPath { dir =>
+          saveTs(dir)
+          checkMetaData(dir, SPARK_LEGACY_DATETIME, null)
+        }
+      }
+    }
+  }
 }
 
 class AvroV1Suite extends AvroSuite {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index 2dc8a062bb73d..ff406f7bc62de 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -859,20 +859,24 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
     }
   }
 
-  test("Write Spark version into Parquet metadata") {
-    withTempPath { dir =>
-      val path = dir.getAbsolutePath
-      spark.range(1).repartition(1).write.parquet(path)
-      val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
-
-      val conf = new Configuration()
-      val hadoopInputFile = HadoopInputFile.fromPath(new Path(file), conf)
-      val parquetReadOptions = HadoopReadOptions.builder(conf).build()
-      val m = ParquetFileReader.open(hadoopInputFile, parquetReadOptions)
-      val metaData = m.getFileMetaData.getKeyValueMetaData
+  private def getMetaData(dir: java.io.File): Map[String, String] = {
+    val file = SpecificParquetRecordReaderBase.listDirectory(dir).get(0)
+    val conf = new Configuration()
+    val hadoopInputFile = HadoopInputFile.fromPath(new Path(file), conf)
+    val parquetReadOptions = HadoopReadOptions.builder(conf).build()
+    val m = ParquetFileReader.open(hadoopInputFile, parquetReadOptions)
+    val metadata = try {
+      m.getFileMetaData.getKeyValueMetaData
+    } finally {
       m.close()
+    }
+    metadata.asScala.toMap
+  }
 
-      assert(metaData.get(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
+  test("Write Spark version into Parquet metadata") {
+    withTempPath { dir =>
+      spark.range(1).repartition(1).write.parquet(dir.getAbsolutePath)
+      assert(getMetaData(dir)(SPARK_VERSION_METADATA_KEY) === SPARK_VERSION_SHORT)
     }
   }
 
@@ -1109,6 +1113,29 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       }
     }
   }
+
+  test("SPARK-33163: write the metadata key 'org.apache.spark.legacyDateTime'") {
+    def saveTs(dir: java.io.File): Unit = {
+      Seq(Timestamp.valueOf("2020-10-15 01:02:03")).toDF()
+        .repartition(1)
+        .write
+        .parquet(dir.getAbsolutePath)
+    }
+    withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> LEGACY.toString) {
+      withTempPath { dir =>
+        saveTs(dir)
+        assert(getMetaData(dir)(SPARK_LEGACY_DATETIME) === "")
+      }
+    }
+    Seq(CORRECTED, EXCEPTION).foreach { mode =>
+      withSQLConf(SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) {
+        withTempPath { dir =>
+          saveTs(dir)
+          assert(getMetaData(dir).get(SPARK_LEGACY_DATETIME).isEmpty)
+        }
+      }
+    }
+  }
 }
 
 class JobCommitFailureParquetOutputCommitter(outputPath: Path, context: TaskAttemptContext)

From 160f45817ff03df6eed368c2c5c12878ac7c7637 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Fri, 16 Oct 2020 11:39:09 +0900
Subject: [PATCH 1106/1280] [SPARK-33165][SQL][TEST] Remove
 dependencies(scalatest,scalactic) from Benchmark

### What changes were proposed in this pull request?

This PR proposes to remove `assert` from `Benchmark` for making it easier to run benchmark codes via `spark-submit`.

### Why are the changes needed?

Since the current `Benchmark` (`master` and `branch-3.0`) has `assert`, we need to pass the proper jars of `scalatest` and `scalactic`;
 - scalatest-core_2.12-3.2.0.jar
 - scalatest-compatible-3.2.0.jar
 - scalactic_2.12-3.0.jar
```
./bin/spark-submit --jars scalatest-core_2.12-3.2.0.jar,scalatest-compatible-3.2.0.jar,scalactic_2.12-3.0.jar,./sql/catalyst/target/spark-catalyst_2.12-3.1.0-SNAPSHOT-tests.jar,./core/target/spark-core_2.12-3.1.0-SNAPSHOT-tests.jar --class org.apache.spark.sql.execution.benchmark.TPCDSQueryBenchmark ./sql/core/target/spark-sql_2.12-3.1.0-SNAPSHOT-tests.jar --data-location /tmp/tpcds-sf1
```

This update can make developers submit benchmark codes without these dependencies;
```
./bin/spark-submit --jars ./sql/catalyst/target/spark-catalyst_2.12-3.1.0-SNAPSHOT-tests.jar,./core/target/spark-core_2.12-3.1.0-SNAPSHOT-tests.jar --class org.apache.spark.sql.execution.benchmark.TPCDSQueryBenchmark ./sql/core/target/spark-sql_2.12-3.1.0-SNAPSHOT-tests.jar --data-location /tmp/tpcds-sf1
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually checked.

Closes #30064 from maropu/RemoveDepInBenchmark.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a5c17de24148ac3ef290e091dcf2978e26afa58c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../test/scala/org/apache/spark/benchmark/Benchmark.scala    | 5 -----
 .../spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala  | 3 ++-
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
index 9629f5ab1a3dd..f81920f9f393d 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
@@ -26,7 +26,6 @@ import scala.util.Try
 
 import org.apache.commons.io.output.TeeOutputStream
 import org.apache.commons.lang3.SystemUtils
-import org.scalatest.Assertions._
 
 import org.apache.spark.util.Utils
 
@@ -161,7 +160,6 @@ private[spark] class Benchmark(
     // scalastyle:off
     println(s"  Stopped after $i iterations, ${NANOSECONDS.toMillis(runTimes.sum)} ms")
     // scalastyle:on
-    assert(runTimes.nonEmpty)
     val best = runTimes.min
     val avg = runTimes.sum / runTimes.size
     val stdev = if (runTimes.size > 1) {
@@ -183,18 +181,15 @@ private[spark] object Benchmark {
     private var timeStart: Long = 0L
 
     def startTiming(): Unit = {
-      assert(timeStart == 0L, "Already started timing.")
       timeStart = System.nanoTime
     }
 
     def stopTiming(): Unit = {
-      assert(timeStart != 0L, "Have not started timing.")
       accumulatedTime += System.nanoTime - timeStart
       timeStart = 0L
     }
 
     def totalTime(): Long = {
-      assert(timeStart == 0L, "Have not stopped timing.")
       accumulatedTime
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
index ad3d79760adf0..7bbf0795eb052 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
@@ -31,7 +31,8 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation
  * To run this:
  * {{{
  *   1. without sbt:
- *        bin/spark-submit --class <this class> <spark sql test jar> --data-location <location>
+ *        bin/spark-submit --jars <spark core test jar>,<spark catalyst test jar>
+ *          --class <this class> <spark sql test jar> --data-location <location>
  *   2. build/sbt "sql/test:runMain <this class> --data-location <TPCDS data location>"
  *   3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt
  *        "sql/test:runMain <this class> --data-location <location>"

From 37d6b3c0fafa98922ed1ecf4f8634d962f5bb9d9 Mon Sep 17 00:00:00 2001
From: Linhong Liu <linhong.liu@databricks.com>
Date: Fri, 16 Oct 2020 03:36:21 +0000
Subject: [PATCH 1107/1280] [SPARK-32761][SQL][3.0] Allow aggregating multiple
 foldable distinct expressions

### What changes were proposed in this pull request?
For queries with multiple foldable distinct columns, since they will be eliminated during
execution, it's not mandatory to let `RewriteDistinctAggregates` handle this case. And
in the current code, `RewriteDistinctAggregates` *dose* miss some "aggregating with
multiple foldable distinct expressions" cases.
For example: `select count(distinct 2), count(distinct 2, 3)` will be missed.

But in the planner, this will trigger an error that "multiple distinct expressions" are not allowed.
As the foldable distinct columns can be eliminated finally, we can allow this in the aggregation
planner check.

### Why are the changes needed?
bug fix

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
added test case

Authored-by: Linhong Liu <linhong.liudatabricks.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit a410658c9bc244e325702dc926075bd835b669ff)

Closes #30052 from linhongliu-db/SPARK-32761-3.0.

Authored-by: Linhong Liu <linhong.liu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../org/apache/spark/sql/execution/SparkStrategies.scala    | 6 ++++--
 .../test/scala/org/apache/spark/sql/DataFrameSuite.scala    | 4 ++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index f836debcbafda..689d1eb62ffa5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -517,7 +517,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
 
         val (functionsWithDistinct, functionsWithoutDistinct) =
           aggregateExpressions.partition(_.isDistinct)
-        if (functionsWithDistinct.map(_.aggregateFunction.children.toSet).distinct.length > 1) {
+        if (functionsWithDistinct.map(
+          _.aggregateFunction.children.filterNot(_.foldable).toSet).distinct.length > 1) {
           // This is a sanity check. We should not reach here when we have multiple distinct
           // column sets. Our `RewriteDistinctAggregates` should take care this case.
           sys.error("You hit a query analyzer bug. Please report your query to " +
@@ -548,7 +549,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
             // to be [COUNT(DISTINCT foo), MAX(DISTINCT foo)], but
             // [COUNT(DISTINCT bar), COUNT(DISTINCT foo)] is disallowed because those two distinct
             // aggregates have different column expressions.
-            val distinctExpressions = functionsWithDistinct.head.aggregateFunction.children
+            val distinctExpressions =
+              functionsWithDistinct.head.aggregateFunction.children.filterNot(_.foldable)
             val normalizedNamedDistinctExpressions = distinctExpressions.map { e =>
               // Ideally this should be done in `NormalizeFloatingNumbers`, but we do it here
               // because `distinctExpressions` is not extracted during logical phase.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 7869005ea2de9..85cbe458aebde 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2467,6 +2467,10 @@ class DataFrameSuite extends QueryTest
     val df = l.join(r, $"col2" === $"col4", "LeftOuter")
     checkAnswer(df, Row("2", "2"))
   }
+
+  test("SPARK-32761: aggregating multiple distinct CONSTANT columns") {
+     checkAnswer(sql("select count(distinct 2), count(distinct 2,3)"), Row(1, 1))
+  }
 }
 
 case class GroupByKey(a: Int, b: Int)

From 698ac6a7efb6063e60f1429299cb640d1b25cc2e Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 16 Oct 2020 13:50:57 +0900
Subject: [PATCH 1108/1280] [SPARK-33165][SQL][TESTS][FOLLOW-UP] Use
 scala.Predef.assert instead

### What changes were proposed in this pull request?

This PR proposes to use `scala.Predef.assert` instead of `org.scalatest.Assertions.assert` removed at https://github.com/apache/spark/pull/30064

### Why are the changes needed?

Just to keep the same behaviour.

### Does this PR introduce _any_ user-facing change?

No, dev-only

### How was this patch tested?

Recover the existing asserts.

Closes #30065 from HyukjinKwon/SPARK-33165.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/test/scala/org/apache/spark/benchmark/Benchmark.scala | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
index f81920f9f393d..022fcbb25b0af 100644
--- a/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
+++ b/core/src/test/scala/org/apache/spark/benchmark/Benchmark.scala
@@ -160,6 +160,7 @@ private[spark] class Benchmark(
     // scalastyle:off
     println(s"  Stopped after $i iterations, ${NANOSECONDS.toMillis(runTimes.sum)} ms")
     // scalastyle:on
+    assert(runTimes.nonEmpty)
     val best = runTimes.min
     val avg = runTimes.sum / runTimes.size
     val stdev = if (runTimes.size > 1) {
@@ -181,15 +182,18 @@ private[spark] object Benchmark {
     private var timeStart: Long = 0L
 
     def startTiming(): Unit = {
+      assert(timeStart == 0L, "Already started timing.")
       timeStart = System.nanoTime
     }
 
     def stopTiming(): Unit = {
+      assert(timeStart != 0L, "Have not started timing.")
       accumulatedTime += System.nanoTime - timeStart
       timeStart = 0L
     }
 
     def totalTime(): Long = {
+      assert(timeStart == 0L, "Have not stopped timing.")
       accumulatedTime
     }
   }

From b66bd79e117f0f38ecc161355d6c4f5ab37ba6c9 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Fri, 16 Oct 2020 12:52:45 -0700
Subject: [PATCH 1109/1280] [SPARK-33171][INFRA] Mark
 ParquetV*FilterSuite/ParquetV*SchemaPruningSuite as ExtendedSQLTest

### What changes were proposed in this pull request?

This PR aims to mark ParquetV1FilterSuite and ParquetV2FilterSuite as `ExtendedSQLTest`.
- ParquetV1FilterSuite/ParquetV2FilterSuite
- ParquetV1SchemaPruningSuite/ParquetV2SchemaPruningSuite

### Why are the changes needed?

Currently, `sql - other tests` is the longest job. This PR will move the above tests to `sql - slow tests` job.

**BEFORE**
- https://github.com/apache/spark/runs/1264150802 (1 hour 37 minutes)

**AFTER**
- https://github.com/apache/spark/pull/30068/checks?check_run_id=1265879896 (1 hour 21 minutes)

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Github Action with the reduced time.

Closes #30068 from dongjoon-hyun/MOVE3.

Lead-authored-by: Dongjoon Hyun <dongjoon@apache.org>
Co-authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/execution/datasources/parquet/ParquetFilterSuite.scala | 3 +++
 .../datasources/parquet/ParquetSchemaPruningSuite.scala        | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
index 8b922aaed4e68..307234dcb86e9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetFilterSuite.scala
@@ -45,6 +45,7 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.ExtendedSQLTest
 import org.apache.spark.util.{AccumulatorContext, AccumulatorV2}
 
 /**
@@ -1598,6 +1599,7 @@ abstract class ParquetFilterSuite extends QueryTest with ParquetTest with Shared
   }
 }
 
+@ExtendedSQLTest
 class ParquetV1FilterSuite extends ParquetFilterSuite {
   override protected def sparkConf: SparkConf =
     super
@@ -1677,6 +1679,7 @@ class ParquetV1FilterSuite extends ParquetFilterSuite {
   }
 }
 
+@ExtendedSQLTest
 class ParquetV2FilterSuite extends ParquetFilterSuite {
   // TODO: enable Parquet V2 write path after file source V2 writers are workable.
   override protected def sparkConf: SparkConf =
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
index c64e95078e916..cab93bd96fff4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.execution.datasources.SchemaPruningSuite
 import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
 import org.apache.spark.sql.execution.datasources.v2.parquet.ParquetScan
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.tags.ExtendedSQLTest
 
 abstract class ParquetSchemaPruningSuite extends SchemaPruningSuite with AdaptiveSparkPlanHelper {
   override protected val dataSourceName: String = "parquet"
@@ -33,6 +34,7 @@ abstract class ParquetSchemaPruningSuite extends SchemaPruningSuite with Adaptiv
 
 }
 
+@ExtendedSQLTest
 class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite {
   override protected def sparkConf: SparkConf =
     super
@@ -40,6 +42,7 @@ class ParquetV1SchemaPruningSuite extends ParquetSchemaPruningSuite {
       .set(SQLConf.USE_V1_SOURCE_LIST, "parquet")
 }
 
+@ExtendedSQLTest
 class ParquetV2SchemaPruningSuite extends ParquetSchemaPruningSuite {
   // TODO: enable Parquet V2 write path after file source V2 writers are workable.
   override protected def sparkConf: SparkConf =

From 1bec8a3e17ffd81126b36037a852cfc648764006 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sat, 25 Jul 2020 10:16:01 -0700
Subject: [PATCH 1110/1280] [SPARK-32436][CORE] Initialize numNonEmptyBlocks in
 HighlyCompressedMapStatus.readExternal

### What changes were proposed in this pull request?

This PR aims to initialize `numNonEmptyBlocks` in `HighlyCompressedMapStatus.readExternal`.

In Scala 2.12, this is initialized to `-1` via the following.
```scala
protected def this() = this(null, -1, null, -1, null, -1)  // For deserialization only
```

### Why are the changes needed?

In Scala 2.13, this causes several UT failures because `HighlyCompressedMapStatus.readExternal` doesn't initialize this field. The following is one example.

- org.apache.spark.scheduler.MapStatusSuite
```
MapStatusSuite:
- compressSize
- decompressSize
*** RUN ABORTED ***
  java.lang.NoSuchFieldError: numNonEmptyBlocks
  at org.apache.spark.scheduler.HighlyCompressedMapStatus.<init>(MapStatus.scala:181)
  at org.apache.spark.scheduler.HighlyCompressedMapStatus$.apply(MapStatus.scala:281)
  at org.apache.spark.scheduler.MapStatus$.apply(MapStatus.scala:73)
  at org.apache.spark.scheduler.MapStatusSuite.$anonfun$new$8(MapStatusSuite.scala:64)
  at scala.runtime.java8.JFunction1$mcVD$sp.apply(JFunction1$mcVD$sp.scala:18)
  at scala.collection.immutable.List.foreach(List.scala:333)
  at org.apache.spark.scheduler.MapStatusSuite.$anonfun$new$7(MapStatusSuite.scala:61)
  at scala.runtime.java8.JFunction1$mcVJ$sp.apply(JFunction1$mcVJ$sp.scala:18)
  at scala.collection.immutable.List.foreach(List.scala:333)
  at org.apache.spark.scheduler.MapStatusSuite.$anonfun$new$6(MapStatusSuite.scala:60)
  ...
```

### Does this PR introduce _any_ user-facing change?

No. This is a private class.

### How was this patch tested?

1. Pass the GitHub Action or Jenkins with the existing tests.
2. Test with Scala-2.13 with `MapStatusSuite`.
```
$ dev/change-scala-version.sh 2.13
$ build/mvn test -pl core --am -Pscala-2.13 -Dtest=none -DwildcardSuites=org.apache.spark.scheduler.MapStatusSuite
...
MapStatusSuite:
- compressSize
- decompressSize
- MapStatus should never report non-empty blocks' sizes as 0
- large tasks should use org.apache.spark.scheduler.HighlyCompressedMapStatus
- HighlyCompressedMapStatus: estimated size should be the average non-empty block size
- SPARK-22540: ensure HighlyCompressedMapStatus calculates correct avgSize
- RoaringBitmap: runOptimize succeeded
- RoaringBitmap: runOptimize failed
- Blocks which are bigger than SHUFFLE_ACCURATE_BLOCK_THRESHOLD should not be underestimated.
- SPARK-21133 HighlyCompressedMapStatus#writeExternal throws NPE
Run completed in 7 seconds, 971 milliseconds.
Total number of tests run: 10
Suites: completed 2, aborted 0
Tests: succeeded 10, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```

Closes #29231 from dongjoon-hyun/SPARK-32436.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit f9f18673dca16c78a77c3386a1629136031bf2ba)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
index 7f8893ff3b9d8..60052402642f9 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/MapStatus.scala
@@ -206,6 +206,7 @@ private[spark] class HighlyCompressedMapStatus private (
 
   override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {
     loc = BlockManagerId(in)
+    numNonEmptyBlocks = -1 // SPARK-32436 Scala 2.13 doesn't initialize this during deserialization
     emptyBlocks = new RoaringBitmap()
     emptyBlocks.readExternal(in)
     avgSize = in.readLong()

From fab10f0dd37ecc8f77d5d8115118ec0a74b59da0 Mon Sep 17 00:00:00 2001
From: ulysses <youxiduo@weidian.com>
Date: Fri, 16 Oct 2020 22:11:13 -0700
Subject: [PATCH 1111/1280] [SPARK-33131][SQL][3.0] Fix grouping sets with
 having clause can not resolve qualified col name

This is [#30029](https://github.com/apache/spark/pull/30029) backport for branch-3.0.

### What changes were proposed in this pull request?

Correct the resolution of having clause.

### Why are the changes needed?

Grouping sets construct new aggregate lost the qualified name of grouping expression. Here is a example:
```
-- Works resolved by `ResolveReferences`
select c1 from values (1) as t1(c1) group by grouping sets(t1.c1) having c1 = 1

-- Works because of the extra expression c1
select c1 as c2 from values (1) as t1(c1) group by grouping sets(t1.c1) having t1.c1 = 1

-- Failed
select c1 from values (1) as t1(c1) group by grouping sets(t1.c1) having t1.c1 = 1
```

It wroks with `Aggregate` without grouping sets through `ResolveReferences`, but Grouping sets not works since the exprId has been changed.

### Does this PR introduce _any_ user-facing change?

Yes, bug fix.

### How was this patch tested?

add test.

Closes #30077 from ulysses-you/SPARK-33131-branch-3.0.

Authored-by: ulysses <youxiduo@weidian.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  2 +-
 .../resources/sql-tests/inputs/having.sql     |  6 ++++
 .../sql-tests/results/having.sql.out          | 32 +++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 729d316811349..bf3d4f0332c53 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -457,7 +457,7 @@ class Analyzer(
      */
     private def constructGroupByAlias(groupByExprs: Seq[Expression]): Seq[Alias] = {
       groupByExprs.map {
-        case e: NamedExpression => Alias(e, e.name)()
+        case e: NamedExpression => Alias(e, e.name)(qualifier = e.qualifier)
         case other => Alias(other, other.toString)()
       }
     }
diff --git a/sql/core/src/test/resources/sql-tests/inputs/having.sql b/sql/core/src/test/resources/sql-tests/inputs/having.sql
index 3b75be19b5677..2799b1a94d085 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/having.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/having.sql
@@ -24,3 +24,9 @@ SELECT SUM(a) AS b, CAST('2020-01-01' AS DATE) AS fake FROM VALUES (1, 10), (2,
 SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY GROUPING SETS ((b), (a, b)) HAVING b > 10;
 SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY CUBE(a, b) HAVING b > 10;
 SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b) HAVING b > 10;
+
+-- SPARK-33131: Grouping sets with having clause can not resolve qualified col name.
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1;
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1;
diff --git a/sql/core/src/test/resources/sql-tests/results/having.sql.out b/sql/core/src/test/resources/sql-tests/results/having.sql.out
index 1b3ac7865159f..6508143e6f9fe 100644
--- a/sql/core/src/test/resources/sql-tests/results/having.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/having.sql.out
@@ -81,3 +81,35 @@ SELECT SUM(a) AS b FROM VALUES (1, 10), (2, 20) AS T(a, b) GROUP BY ROLLUP(a, b)
 struct<b:bigint>
 -- !query output
 2
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY GROUPING SETS(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY CUBE(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY ROLLUP(t.c1) HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1
+
+
+-- !query
+SELECT c1 FROM VALUES (1, 2) as t(c1, c2) GROUP BY t.c1 HAVING t.c1 = 1
+-- !query schema
+struct<c1:int>
+-- !query output
+1

From 56a60ca3f2cdaffd884bde7a486125ca77344f50 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Sat, 17 Oct 2020 21:02:25 -0700
Subject: [PATCH 1112/1280] [SPARK-33170][SQL] Add SQL config to control
 fast-fail behavior in FileFormatWriter

### What changes were proposed in this pull request?

This patch proposes to add a config we can control fast-fail behavior in FileFormatWriter and set it false by default.

### Why are the changes needed?

In SPARK-29649, we catch `FileAlreadyExistsException` in `FileFormatWriter` and fail fast for the task set to prevent task retry.

Due to latest discussion, it is important to be able to keep original behavior that is to retry tasks even `FileAlreadyExistsException` is thrown, because `FileAlreadyExistsException` could be recoverable in some cases.

We are going to add a config we can control this behavior and set it false for fast-fail by default.

### Does this PR introduce _any_ user-facing change?

Yes. By default the task in FileFormatWriter will retry even if `FileAlreadyExistsException` is thrown. This is the behavior before Spark 3.0. User can control fast-fail behavior by enabling it.

### How was this patch tested?

Unit test.

Closes #30073 from viirya/SPARK-33170.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3010e9044e068216d7a7a9ec510453ecbb159f95)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../apache/spark/sql/internal/SQLConf.scala   | 16 ++++++++++
 .../datasources/FileFormatWriter.scala        |  3 +-
 .../spark/sql/sources/InsertSuite.scala       | 32 ++++++++++++-------
 3 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index cdf72039b2a0e..c74888340ac12 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1993,6 +1993,20 @@ object SQLConf {
       .stringConf
       .createWithDefault("")
 
+  val FASTFAIL_ON_FILEFORMAT_OUTPUT =
+    buildConf("spark.sql.execution.fastFailOnFileFormatOutput")
+      .internal()
+      .doc("Whether to fast fail task execution when writing output to FileFormat datasource. " +
+        "If this is enabled, in `FileFormatWriter` we will catch `FileAlreadyExistsException` " +
+        "and fast fail output task without further task retry. Only enabling this if you know " +
+        "the `FileAlreadyExistsException` of the output task is unrecoverable, i.e., further " +
+        "task attempts won't be able to success. If the `FileAlreadyExistsException` might be " +
+        "recoverable, you should keep this as disabled and let Spark to retry output tasks. " +
+        "This is disabled by default.")
+      .version("3.0.2")
+      .booleanConf
+      .createWithDefault(false)
+
   object PartitionOverwriteMode extends Enumeration {
     val STATIC, DYNAMIC = Value
   }
@@ -3090,6 +3104,8 @@ class SQLConf extends Serializable with Logging {
   def disabledV2StreamingMicroBatchReaders: String =
     getConf(DISABLED_V2_STREAMING_MICROBATCH_READERS)
 
+  def fastFailFileFormatOutput: Boolean = getConf(FASTFAIL_ON_FILEFORMAT_OUTPUT)
+
   def concatBinaryAsString: Boolean = getConf(CONCAT_BINARY_AS_STRING)
 
   def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
index 219c778b9164a..abb88ae73cabf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
@@ -39,6 +39,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCo
 import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
 import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, DateTimeUtils}
 import org.apache.spark.sql.execution.{ProjectExec, SortExec, SparkPlan, SQLExecution}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.StringType
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.{SerializableConfiguration, Utils}
@@ -283,7 +284,7 @@ object FileFormatWriter extends Logging {
     } catch {
       case e: FetchFailedException =>
         throw e
-      case f: FileAlreadyExistsException =>
+      case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput =>
         // If any output file to write already exists, it does not make sense to re-run this task.
         // We throw the exception and let Executor throw ExceptionFailure to abort the job.
         throw new TaskOutputFileAlreadyExistException(f)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index e56ecd1da07b2..995d4b0e7230e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -825,21 +825,29 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
   }
 
   test("Stop task set if FileAlreadyExistsException was thrown") {
-    withSQLConf("fs.file.impl" -> classOf[FileExistingTestFileSystem].getName,
-        "fs.file.impl.disable.cache" -> "true") {
-      withTable("t") {
-        sql(
-          """
-            |CREATE TABLE t(i INT, part1 INT) USING PARQUET
-            |PARTITIONED BY (part1)
+    Seq(true, false).foreach { fastFail =>
+      withSQLConf("fs.file.impl" -> classOf[FileExistingTestFileSystem].getName,
+        "fs.file.impl.disable.cache" -> "true",
+        SQLConf.FASTFAIL_ON_FILEFORMAT_OUTPUT.key -> fastFail.toString) {
+        withTable("t") {
+          sql(
+            """
+              |CREATE TABLE t(i INT, part1 INT) USING PARQUET
+              |PARTITIONED BY (part1)
           """.stripMargin)
 
-        val df = Seq((1, 1)).toDF("i", "part1")
-        val err = intercept[SparkException] {
-          df.write.mode("overwrite").format("parquet").insertInto("t")
+          val df = Seq((1, 1)).toDF("i", "part1")
+          val err = intercept[SparkException] {
+            df.write.mode("overwrite").format("parquet").insertInto("t")
+          }
+
+          if (fastFail) {
+            assert(err.getCause.getMessage.contains("can not write to output file: " +
+              "org.apache.hadoop.fs.FileAlreadyExistsException"))
+          } else {
+            assert(err.getCause.getMessage.contains("Task failed while writing rows"))
+          }
         }
-        assert(err.getCause.getMessage.contains("can not write to output file: " +
-          "org.apache.hadoop.fs.FileAlreadyExistsException"))
       }
     }
   }

From 7e65b12e5ac80344cd25d82a45d2447e369c3250 Mon Sep 17 00:00:00 2001
From: Keiji Yoshida <kjmrknsn@gmail.com>
Date: Sun, 18 Oct 2020 16:47:04 +0900
Subject: [PATCH 1113/1280] [MINOR][DOCS][EXAMPLE] Fix the Python
 manual_load_options_csv example

### What changes were proposed in this pull request?
This pull request changes the `sep` parameter's value from `:` to `;` in the example of `examples/src/main/python/sql/datasource.py`. This code snippet is shown on the Spark SQL Guide documentation. The `sep` parameter's value should be `;` since the data in https://github.com/apache/spark/blob/master/examples/src/main/resources/people.csv is separated by `;`.

### Why are the changes needed?
To fix the example code so that it can be executed properly.

### Does this PR introduce _any_ user-facing change?
Yes.
This code snippet is shown on the Spark SQL Guide documentation: https://spark.apache.org/docs/latest/sql-data-sources-load-save-functions.html#manually-specifying-options

### How was this patch tested?
By building the documentation and checking the Spark SQL Guide documentation manually in the local environment.

Closes #30082 from kjmrknsn/fix-example-python-datasource.

Authored-by: Keiji Yoshida <kjmrknsn@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 examples/src/main/python/sql/datasource.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/src/main/python/sql/datasource.py b/examples/src/main/python/sql/datasource.py
index 265f135e1e5f2..9f8fdd74e0242 100644
--- a/examples/src/main/python/sql/datasource.py
+++ b/examples/src/main/python/sql/datasource.py
@@ -96,7 +96,7 @@ def basic_datasource_example(spark):
 
     # $example on:manual_load_options_csv$
     df = spark.read.load("examples/src/main/resources/people.csv",
-                         format="csv", sep=":", inferSchema="true", header="true")
+                         format="csv", sep=";", inferSchema="true", header="true")
     # $example off:manual_load_options_csv$
 
     # $example on:manual_save_options_orc$

From 05fbbb1fcea27b8f22009440d95a76a74de1f044 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sun, 18 Oct 2020 09:21:07 -0700
Subject: [PATCH 1114/1280] [SPARK-33176][K8S] Use 11-jre-slim as default in
 K8s Dockerfile

### What changes were proposed in this pull request?

This PR aims to use `openjdk:11-jre-slim` as default in K8s Dockerfile.

### Why are the changes needed?

Although Apache Spark supports both Java8/Java11, there is a difference.

1. Java8-built distribution can run both Java8/Java11
2. Java11-built distribution can run on Java11, but not Java8.

In short, we had better use Java11 in Dockerfile to embrace both cases without any issues.

### Does this PR introduce _any_ user-facing change?

Yes. This will remove the change of user frustration when they build with JDK11 and build the image without overriding Java base image.

### How was this patch tested?

Pass the K8s IT.

Closes #30083 from dongjoon-hyun/SPARK-33176.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../kubernetes/docker/src/main/dockerfiles/spark/Dockerfile     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
index cde9358cb8718..23d2b035e9e80 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-ARG java_image_tag=8-jre-slim
+ARG java_image_tag=11-jre-slim
 
 FROM openjdk:${java_image_tag}
 

From 0bff1f67564cd2b2d4c0f465b3358929c1a2741e Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Mon, 19 Oct 2020 14:13:37 +0900
Subject: [PATCH 1115/1280] [SPARK-33123][INFRA] Ignore GitHub only changes in
 Amplab Jenkins build

### What changes were proposed in this pull request?

This PR aims to ignore GitHub only changes in Amplab Jenkins build.

### Why are the changes needed?

This will save server resources.

### Does this PR introduce _any_ user-facing change?

No, this is a dev-only change.

### How was this patch tested?

Manually. I used the following doctest during testing and removed it at the clean-up.

E2E tests:

```
cd dev
cat test.py
```

```python
import importlib
runtests = importlib.import_module("run-tests")
print([x.name for x in runtests.determine_modules_for_files([".github/workflows/build_and_test.yml"])])
```

```python
$ GITHUB_ACTIONS=1 python test.py
['root']
$ python test.py
[]
```

Unittests:

```bash
$ GITHUN_ACTIONS=1 python3 -m doctest dev/run-tests.py
$ python3 -m doctest dev/run-tests.py
```

Closes #30020 from williamhyun/SPARK-33123.

Lead-authored-by: William Hyun <williamhyun3@gmail.com>
Co-authored-by: Hyukjin Kwon <gurwls223@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit e6c53c2c1b538d6272df4d1ca294d04f8b49bd6c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/run-tests.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/run-tests.py b/dev/run-tests.py
index 976854dad8214..fde1defe25b69 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -43,7 +43,8 @@ def determine_modules_for_files(filenames):
     """
     Given a list of filenames, return the set of modules that contain those files.
     If a file is not associated with a more specific submodule, then this method will consider that
-    file to belong to the 'root' module. GitHub Action and Appveyor files are ignored.
+    file to belong to the 'root' module. `.github` directory is counted only in GitHub Actions,
+    and `appveyor.yml` is always ignored because this file is dedicated only to AppVeyor builds.
 
     >>> sorted(x.name for x in determine_modules_for_files(["python/pyspark/a.py", "sql/core/foo"]))
     ['pyspark-core', 'sql']
@@ -56,6 +57,8 @@ def determine_modules_for_files(filenames):
     for filename in filenames:
         if filename in ("appveyor.yml",):
             continue
+        if ("GITHUB_ACTIONS" not in os.environ) and filename.startswith(".github"):
+            continue
         matched_at_least_one_module = False
         for module in modules.all_modules:
             if module.contains_file(filename):

From 15ed3121994d4c7e6f1db7e0b5b2cea5565b495a Mon Sep 17 00:00:00 2001
From: Yan Xiaole <xiaole.yan@gmail.com>
Date: Sun, 9 Aug 2020 16:47:31 -0700
Subject: [PATCH 1116/1280] [SPARK-32557][CORE] Logging and swallowing the
 exception per entry in History server

### What changes were proposed in this pull request?
This PR adds a try catch wrapping the History server scan logic to log and swallow the exception per entry.

### Why are the changes needed?
As discussed in #29350 , one entry failure shouldn't affect others.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Manually tested.

Closes #29374 from yanxiaole/SPARK-32557.

Authored-by: Yan Xiaole <xiaole.yan@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/deploy/history/FsHistoryProvider.scala   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index c26215295e914..f5e7c4fa6ac91 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -27,6 +27,7 @@ import java.util.zip.ZipOutputStream
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.io.Source
+import scala.util.control.NonFatal
 import scala.xml.Node
 
 import com.fasterxml.jackson.annotation.JsonIgnore
@@ -528,7 +529,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
                 case _: FileNotFoundException => false
               }
 
-            case _: FileNotFoundException =>
+            case NonFatal(e) =>
+              logWarning(s"Error while filtering log ${reader.rootPath}", e)
               false
           }
         }

From 02f80cf293739f4d2881316897dbdcea74daa0bc Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Thu, 15 Oct 2020 15:28:52 +0900
Subject: [PATCH 1117/1280] Revert "Revert "[SPARK-33146][CORE] Check for
 non-fatal errors when loading new applications in SHS""

This reverts commit e40c147a5d194adbba13f12590959dc68347ec14.
---
 .../deploy/history/FsHistoryProvider.scala    |  3 ++
 .../history/FsHistoryProviderSuite.scala      | 49 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index f5e7c4fa6ac91..7e63d5500b2ab 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -527,6 +527,9 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
                 reader.fileSizeForLastIndex > 0
               } catch {
                 case _: FileNotFoundException => false
+                case NonFatal(e) =>
+                  logWarning(s"Error while reading new log ${reader.rootPath}", e)
+                  false
               }
 
             case NonFatal(e) =>
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index c2f34fc3a95ed..f3beb35f1f011 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -1470,6 +1470,55 @@ class FsHistoryProviderSuite extends SparkFunSuite with Matchers with Logging {
     }
   }
 
+  test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+      val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)
+
+      val provider = new FsHistoryProvider(conf)
+
+      val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf)
+      writer.start()
+
+      writeEventsToRollingWriter(writer, Seq(
+        SparkListenerApplicationStart("app", Some("app"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(dir.listFiles().size === 1)
+      assert(provider.getListing.length === 1)
+
+      // Manually delete the appstatus file to make an invalid rolling event log
+      val appStatusPath = RollingEventLogFilesWriter.getAppStatusFilePath(new Path(writer.logPath),
+        "app", None, true)
+      fs.delete(appStatusPath, false)
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 0)
+
+      // Create a new application
+      val writer2 = new RollingEventLogFilesWriter("app2", None, dir.toURI, conf, hadoopConf)
+      writer2.start()
+      writeEventsToRollingWriter(writer2, Seq(
+        SparkListenerApplicationStart("app2", Some("app2"), 0, "user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+
+      // Both folders exist but only one application found
+      provider.checkForLogs()
+      provider.cleanLogs()
+      assert(provider.getListing.length === 1)
+      assert(dir.listFiles().size === 2)
+
+      // Make sure a new provider sees the valid application
+      provider.stop()
+      val newProvider = new FsHistoryProvider(conf)
+      newProvider.checkForLogs()
+      assert(newProvider.getListing.length === 1)
+    }
+  }
+
   /**
    * Asks the provider to check for logs and calls a function to perform checks on the updated
    * app list. Example:

From b1d5a08a17e3a6a38978a4c12fd2cf9cf58160f1 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 19 Oct 2020 17:14:13 +0900
Subject: [PATCH 1118/1280] Revert "[SPARK-33069][INFRA] Skip test result
 report if no JUnit XML files are found"

This reverts commit 46a62cab765f4684dc634d92d81ecddcc42cee35.
---
 .github/workflows/test_report.yml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/.github/workflows/test_report.yml b/.github/workflows/test_report.yml
index 060a8795b6a77..93cdb86687261 100644
--- a/.github/workflows/test_report.yml
+++ b/.github/workflows/test_report.yml
@@ -15,16 +15,7 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         workflow: ${{ github.event.workflow_run.workflow_id }}
         commit: ${{ github.event.workflow_run.head_commit.id }}
-    - name: Check if JUnit report XML files exist
-      run: |
-        if ls **/target/test-reports/*.xml > /dev/null 2>&1; then
-          echo '::set-output name=FILE_EXISTS::true'
-        else
-          echo '::set-output name=FILE_EXISTS::false'
-        fi
-      id: check-junit-file
     - name: Publish test report
-      if: steps.check-junit-file.outputs.FILE_EXISTS == 'true'
       uses: scacap/action-surefire-report@v1
       with:
         check_name: Report test results

From c3af7c6e66e4385c57ae38839653804bf6e3b1c1 Mon Sep 17 00:00:00 2001
From: liaoaoyuan97 <al3468@columbia.edu>
Date: Tue, 20 Oct 2020 10:23:58 +0900
Subject: [PATCH 1119/1280] [SPARK-33181][SQL][DOCS] Document Load Table
 Directly from File in SQL Select Reference

### What changes were proposed in this pull request?

Add the link to the feature: "Run SQL on files directly" to SQL reference documentation page

### Why are the changes needed?

To make SQL Reference complete

### Does this PR introduce _any_ user-facing change?

yes. Previously, reading in sql from file directly is not included in the documentation: https://spark.apache.org/docs/latest/sql-ref-syntax-qry-select.html, not listed in from_items. The new link is added to the select statement documentation, like the below:

![image](https://user-images.githubusercontent.com/16770242/96517999-c34f3900-121e-11eb-8d56-c4ba0432855e.png)
![image](https://user-images.githubusercontent.com/16770242/96518808-8126f700-1220-11eb-8c98-fb398eee0330.png)

### How was this patch tested?

Manually built and tested

Closes #30095 from liaoaoyuan97/master.

Authored-by: liaoaoyuan97 <al3468@columbia.edu>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit f65a24412b6691ecdb4254e70d6e7abc846edb66)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-ref-syntax-qry-select.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 453737a2f36b8..655766d4c6d22 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -85,6 +85,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
      * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
      * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
      * Subquery
+     * [File](sql-data-sources-load-save-functions.html#run-sql-on-files-directly)
      
 * **PIVOT**
 

From 3b5b5334cc91ef146642ef2bec286bd63224b1f8 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 20 Oct 2020 17:35:09 +0900
Subject: [PATCH 1120/1280] [SPARK-33190][INFRA][TESTS] Set upper bound of
 PyArrow version in GitHub Actions

PyArrow is uploaded into PyPI today (https://pypi.org/project/pyarrow/), and some tests fail with PyArrow 2.0.0+:

```
======================================================================
ERROR [0.774s]: test_grouped_over_window_with_key (pyspark.sql.tests.test_pandas_grouped_map.GroupedMapInPandasTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/__w/spark/spark/python/pyspark/sql/tests/test_pandas_grouped_map.py", line 595, in test_grouped_over_window_with_key
    .select('id', 'result').collect()
  File "/__w/spark/spark/python/pyspark/sql/dataframe.py", line 588, in collect
    sock_info = self._jdf.collectToPython()
  File "/__w/spark/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
    answer, self.gateway_client, self.target_id, self.name)
  File "/__w/spark/spark/python/pyspark/sql/utils.py", line 117, in deco
    raise converted from None
pyspark.sql.utils.PythonException:
  An exception was thrown from the Python worker. Please see the stack trace below.
Traceback (most recent call last):
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/worker.py", line 601, in main
    process()
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/worker.py", line 593, in process
    serializer.dump_stream(out_iter, outfile)
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/pandas/serializers.py", line 255, in dump_stream
    return ArrowStreamSerializer.dump_stream(self, init_stream_yield_batches(), stream)
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/pandas/serializers.py", line 81, in dump_stream
    for batch in iterator:
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/sql/pandas/serializers.py", line 248, in init_stream_yield_batches
    for series in iterator:
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/worker.py", line 426, in mapper
    return f(keys, vals)
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/worker.py", line 170, in <lambda>
    return lambda k, v: [(wrapped(k, v), to_arrow_type(return_type))]
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/worker.py", line 158, in wrapped
    result = f(key, pd.concat(value_series, axis=1))
  File "/__w/spark/spark/python/lib/pyspark.zip/pyspark/util.py", line 68, in wrapper
    return f(*args, **kwargs)
  File "/__w/spark/spark/python/pyspark/sql/tests/test_pandas_grouped_map.py", line 590, in f
    "{} != {}".format(expected_key[i][1], window_range)
AssertionError: {'start': datetime.datetime(2018, 3, 15, 0, 0), 'end': datetime.datetime(2018, 3, 20, 0, 0)} != {'start': datetime.datetime(2018, 3, 15, 0, 0, tzinfo=<StaticTzInfo 'Etc/UTC'>), 'end': datetime.datetime(2018, 3, 20, 0, 0, tzinfo=<StaticTzInfo 'Etc/UTC'>)}
```

https://github.com/apache/spark/runs/1278917457

This PR proposes to set the upper bound of PyArrow in GitHub Actions build. This should be removed when we properly support PyArrow 2.0.0+ (SPARK-33189).

To make build pass.

No, dev-only.

GitHub Actions in this build will test it out.

Closes #30098 from HyukjinKwon/hot-fix-test.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit eb9966b70055a67dd02451c78ec205d913a38a42)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/build_and_test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index acef583b0a766..649ce950bdd38 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -149,7 +149,7 @@ jobs:
       # PyArrow is not supported in PyPy yet, see ARROW-2651.
       # TODO(SPARK-32247): scipy installation with PyPy fails for an unknown reason.
       run: |
-        python2.7 -m pip install numpy pyarrow pandas scipy xmlrunner
+        python2.7 -m pip install numpy 'pyarrow<2.0.0' pandas scipy xmlrunner
         python2.7 -m pip list
         # PyPy does not have xmlrunner
         pypy3 -m pip install numpy pandas
@@ -157,7 +157,7 @@ jobs:
     - name: Install Python packages (Python 3.8)
       if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
       run: |
-        python3.8 -m pip install numpy pyarrow pandas scipy xmlrunner
+        python3.8 -m pip install numpy 'pyarrow<2.0.0' pandas scipy xmlrunner
         python3.8 -m pip list
     # SparkR
     - name: Install R 4.0

From 4373c71632abfa2a69d36909a49673cbaa050865 Mon Sep 17 00:00:00 2001
From: Keiji Yoshida <kjmrknsn@gmail.com>
Date: Wed, 21 Oct 2020 00:36:45 +0900
Subject: [PATCH 1121/1280] [MINOR][DOCS] Fix the description about to_avro and
 from_avro functions

### What changes were proposed in this pull request?
This pull request changes the description about `to_avro` and `from_avro` functions to include Python as a supported language as the functions have been supported in Python since Apache Spark 3.0.0 [[SPARK-26856](https://issues.apache.org/jira/browse/SPARK-26856)].

### Why are the changes needed?
Same as above.

### Does this PR introduce _any_ user-facing change?
Yes. The description changed by this pull request is on https://spark.apache.org/docs/latest/sql-data-sources-avro.html#to_avro-and-from_avro.

### How was this patch tested?
Tested manually by building and checking the document in the local environment.

Closes #30105 from kjmrknsn/fix-docs-sql-data-sources-avro.

Authored-by: Keiji Yoshida <kjmrknsn@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 46ad325e56abd95c0ffdbe64aad78582da8c725d)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-data-sources-avro.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index d926ae7703268..69b165ed28bae 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -88,7 +88,7 @@ Kafka key-value record will be augmented with some metadata, such as the ingesti
 * If the "value" field that contains your data is in Avro, you could use `from_avro()` to extract your data, enrich it, clean it, and then push it downstream to Kafka again or write it out to a file.
 * `to_avro()` can be used to turn structs into Avro records. This method is particularly useful when you would like to re-encode multiple columns into a single one when writing data out to Kafka.
 
-Both functions are currently only available in Scala and Java.
+Both functions are currently only available in Scala, Java, and Python.
 
 <div class="codetabs">
 <div data-lang="scala" markdown="1">

From 5e331553726f838f2f14788c135f3497319b4714 Mon Sep 17 00:00:00 2001
From: Bryan Cutler <cutlerb@gmail.com>
Date: Wed, 21 Oct 2020 09:13:33 +0900
Subject: [PATCH 1122/1280] [SPARK-33189][PYTHON][TESTS] Add env var to tests
 for legacy nested timestamps in pyarrow

Add an environment variable `PYARROW_IGNORE_TIMEZONE` to pyspark tests in run-tests.py to use legacy nested timestamp behavior. This means that when converting arrow to pandas, nested timestamps with timezones will have the timezone localized during conversion.

The default behavior was changed in PyArrow 2.0.0 to propagate timezone information. Using the environment variable enables testing with newer versions of pyarrow until the issue can be fixed in SPARK-32285.

No

Existing tests

Closes #30111 from BryanCutler/arrow-enable-legacy-nested-timestamps-SPARK-33189.

Authored-by: Bryan Cutler <cutlerb@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 47a6568265525002021c1e5cfa4330f5b1a91469)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/build_and_test.yml | 4 ++--
 python/run-tests.py                  | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 649ce950bdd38..d1203c665010a 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -149,7 +149,7 @@ jobs:
       # PyArrow is not supported in PyPy yet, see ARROW-2651.
       # TODO(SPARK-32247): scipy installation with PyPy fails for an unknown reason.
       run: |
-        python2.7 -m pip install numpy 'pyarrow<2.0.0' pandas scipy xmlrunner
+        python2.7 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
         python2.7 -m pip list
         # PyPy does not have xmlrunner
         pypy3 -m pip install numpy pandas
@@ -157,7 +157,7 @@ jobs:
     - name: Install Python packages (Python 3.8)
       if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
       run: |
-        python3.8 -m pip install numpy 'pyarrow<2.0.0' pandas scipy xmlrunner
+        python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
         python3.8 -m pip list
     # SparkR
     - name: Install R 4.0
diff --git a/python/run-tests.py b/python/run-tests.py
index 9fb0983d5f1fb..a647f13934de3 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -76,6 +76,8 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         'PYSPARK_PYTHON': which(pyspark_python),
         'PYSPARK_DRIVER_PYTHON': which(pyspark_python),
         'PYSPARK_ROW_FIELD_SORTING_ENABLED': 'true'
+        # Preserve legacy nested timezone behavior for pyarrow>=2, remove after SPARK-32285
+        'PYARROW_IGNORE_TIMEZONE': '1',
     })
 
     # Create a unique temp directory under 'target/' for each run. The TMPDIR variable is

From a36b3c438607f922d944ee8e773eefbe76aae7fb Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Wed, 21 Oct 2020 17:31:19 +0900
Subject: [PATCH 1123/1280] [SPARK-32785][SQL][DOCS][FOLLOWUP][3.0] Update
 migration guide for incomplete interval literals
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

Address comments  https://github.com/apache/spark/pull/29635#discussion_r507241899 to improve migration guide

### Why are the changes needed?

improve migration guide

### Does this PR introduce _any_ user-facing change?

NO，only doc update

### How was this patch tested?

passing GitHub action

Closes #30117 from yaooqinn/SPARK-32785-F30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-migration-guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index e64c03755725c..85d7073ff08d4 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -24,7 +24,7 @@ license: |
 
 ## Upgrading from Spark SQL 3.0.1 to 3.0.2
 
-  - In Spark 3.0.2, incomplete interval literals, e.g. `INTERVAL '1'`, `INTERVAL '1 DAY 2'` will fail with IllegalArgumentException. In Spark 3.0.1 and earlier, they result `NULL`s.
+  - In Spark 3.0.2, `IllegalArgumentException` is returned for the incomplete interval literals, e.g. `INTERVAL '1'`, `INTERVAL '1 DAY 2'`, which are invalid. In Spark 3.0.1, these literals result in `NULL`s.
 
 ## Upgrading from Spark SQL 3.0 to 3.0.1
 

From e31fe6c97c30237c6c7a28603ed884a13c9e2dd5 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 22 Oct 2020 10:00:18 +0900
Subject: [PATCH 1124/1280] [SPARK-33189][FOLLOWUP][3.0] Fix syntax error in
 python/run-tests.py

### What changes were proposed in this pull request?

This PR aims to fix syntax error.

### Why are the changes needed?

```
========================================================================
Running Python style checks
========================================================================
pycodestyle checks failed.
*** Error compiling './python/run-tests.py'...
  File "./python/run-tests.py", line 80
    'PYARROW_IGNORE_TIMEZONE': '1',
                            ^
SyntaxError: invalid syntax
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the Jenkins.

Closes #30125 from dongjoon-hyun/SPARK-33189-2.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/run-tests.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/run-tests.py b/python/run-tests.py
index a647f13934de3..89df1a3d91c76 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -75,9 +75,9 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         'SPARK_PREPEND_CLASSES': '1',
         'PYSPARK_PYTHON': which(pyspark_python),
         'PYSPARK_DRIVER_PYTHON': which(pyspark_python),
-        'PYSPARK_ROW_FIELD_SORTING_ENABLED': 'true'
+        'PYSPARK_ROW_FIELD_SORTING_ENABLED': 'true',
         # Preserve legacy nested timezone behavior for pyarrow>=2, remove after SPARK-32285
-        'PYARROW_IGNORE_TIMEZONE': '1',
+        'PYARROW_IGNORE_TIMEZONE': '1'
     })
 
     # Create a unique temp directory under 'target/' for each run. The TMPDIR variable is

From 933dc6cb7b3de1d8ccaf73d124d6eb95b947ed19 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 15 Oct 2020 09:08:14 -0700
Subject: [PATCH 1125/1280] [SPARK-32247][INFRA] Install and test scipy with
 PyPy in GitHub Actions

### What changes were proposed in this pull request?

This PR proposes to install `scipy` as well in PyPy. It will test several ML specific test cases in PyPy as well. For example, https://github.com/apache/spark/blob/31a16fbb405a19dc3eb732347e0e1f873b16971d/python/pyspark/mllib/tests/test_linalg.py#L487

It was not installed when GitHub Actions build was added because it failed to install for an unknown reason. Seems like it's fixed in the latest scipy.

### Why are the changes needed?

To improve test coverage.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

GitHub Actions build in this PR will test it out.

Closes #30054 from HyukjinKwon/SPARK-32247.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/build_and_test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index d1203c665010a..6afa41b9e949c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -147,12 +147,11 @@ jobs:
     - name: Install Python packages (Python 2.7 and PyPy3)
       if: contains(matrix.modules, 'pyspark')
       # PyArrow is not supported in PyPy yet, see ARROW-2651.
-      # TODO(SPARK-32247): scipy installation with PyPy fails for an unknown reason.
       run: |
         python2.7 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
         python2.7 -m pip list
         # PyPy does not have xmlrunner
-        pypy3 -m pip install numpy pandas
+        pypy3 -m pip install numpy pandas scipy
         pypy3 -m pip list
     - name: Install Python packages (Python 3.8)
       if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))

From f7c7f4f8b6fe7ed55c1aed5494311ba23844d019 Mon Sep 17 00:00:00 2001
From: Shiqi Sun <s.sun@salesforce.com>
Date: Sat, 24 Oct 2020 09:55:57 -0700
Subject: [PATCH 1126/1280] [SPARK-30821][K8S] Handle executor failure with
 multiple containers

Handle executor failure with multiple containers

Added a spark property spark.kubernetes.executor.checkAllContainers,
with default being false. When it's true, the executor snapshot will
take all containers in the executor into consideration when deciding
whether the executor is in "Running" state, if the pod restart policy is
"Never". Also, added the new spark property to the doc.

### What changes were proposed in this pull request?

Checking of all containers in the executor pod when reporting executor status, if the `spark.kubernetes.executor.checkAllContainers` property is set to true.

### Why are the changes needed?

Currently, a pod remains "running" as long as there is at least one running container. This prevents Spark from noticing when a container has failed in an executor pod with multiple containers. With this change, user can configure the behavior to be different. Namely, if any container in the executor pod has failed, either the executor process or one of its sidecars, the pod is considered to be failed, and it will be rescheduled.

### Does this PR introduce _any_ user-facing change?

Yes, new spark property added.
User is now able to choose whether to turn on this feature using the `spark.kubernetes.executor.checkAllContainers` property.

### How was this patch tested?

Unit test was added and all passed.
I tried to run integration test by following the instruction [here](https://spark.apache.org/developer-tools.html) (section "Testing K8S") and also [here](https://github.com/apache/spark/blob/master/resource-managers/kubernetes/integration-tests/README.md), but I wasn't able to run it smoothly as it fails to talk with minikube cluster. Maybe it's because my minikube version is too new (I'm using v1.13.1)...? Since I've been trying it for two days and still can't make it work, I decided to submit this PR and hopefully the Jenkins test will pass.

Closes #29924 from huskysun/exec-sidecar-failure.

Authored-by: Shiqi Sun <s.sun@salesforce.com>
Signed-off-by: Holden Karau <hkarau@apple.com>
(cherry picked from commit f65952772702f0a8772c93b79f562f35c337f5a5)
Signed-off-by: Holden Karau <hkarau@apple.com>
---
 docs/running-on-kubernetes.md                 |  8 +++
 .../org/apache/spark/deploy/k8s/Config.scala  |  8 +++
 .../cluster/k8s/ExecutorPodsSnapshot.scala    | 16 +++++-
 .../k8s/KubernetesClusterManager.scala        |  3 +
 ...erministicExecutorPodsSnapshotsStore.scala |  2 +
 .../k8s/ExecutorLifecycleTestUtils.scala      | 32 ++++++++++-
 .../k8s/ExecutorPodsSnapshotSuite.scala       | 56 +++++++++++++------
 .../k8s/ExecutorPodsSnapshotsStoreSuite.scala |  1 +
 8 files changed, 108 insertions(+), 18 deletions(-)

diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index e6a000de1ab08..f49f953938391 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1157,6 +1157,14 @@ See the [configuration page](configuration.html) for information on Spark config
   </td>
   <td>3.0.0</td>
 </tr>
+<tr>
+  <td><code>spark.kubernetes.executor.checkAllContainers</code></td>
+  <td>false</td>
+  <td>
+  Specify whether executor pods should be check all containers (including sidecars) or only the executor container when determining the pod status.
+  </td>
+  <td>3.1.0</td>
+</tr>
 <tr>
   <td><code>spark.kubernetes.submission.connectionTimeout</code></td>
   <td>10000</td>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 22f4c7505c09b..b08875bdd3420 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -409,6 +409,14 @@ private[spark] object Config extends Logging {
       .stringConf
       .createOptional
 
+  val KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS =
+    ConfigBuilder("spark.kubernetes.executor.checkAllContainers")
+      .doc("If set to true, all containers in the executor pod will be checked when reporting" +
+        "executor status.")
+      .version("3.1.0")
+      .booleanConf
+      .createWithDefault(false)
+
   val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label."
   val KUBERNETES_DRIVER_ANNOTATION_PREFIX = "spark.kubernetes.driver.annotation."
   val KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX = "spark.kubernetes.driver.service.annotation."
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
index 435a5f1461c92..bd8da1080111f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
@@ -18,6 +18,7 @@ package org.apache.spark.scheduler.cluster.k8s
 
 import java.util.Locale
 
+import io.fabric8.kubernetes.api.model.ContainerStateTerminated
 import io.fabric8.kubernetes.api.model.Pod
 
 import org.apache.spark.deploy.k8s.Constants._
@@ -37,6 +38,7 @@ private[spark] case class ExecutorPodsSnapshot(executorPods: Map[Long, ExecutorP
 }
 
 object ExecutorPodsSnapshot extends Logging {
+  private var shouldCheckAllContainers: Boolean = _
 
   def apply(executorPods: Seq[Pod]): ExecutorPodsSnapshot = {
     ExecutorPodsSnapshot(toStatesByExecutorId(executorPods))
@@ -44,6 +46,10 @@ object ExecutorPodsSnapshot extends Logging {
 
   def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState])
 
+  def setShouldCheckAllContainers(watchAllContainers: Boolean): Unit = {
+    shouldCheckAllContainers = watchAllContainers
+  }
+
   private def toStatesByExecutorId(executorPods: Seq[Pod]): Map[Long, ExecutorPodState] = {
     executorPods.map { pod =>
       (pod.getMetadata.getLabels.get(SPARK_EXECUTOR_ID_LABEL).toLong, toState(pod))
@@ -59,7 +65,15 @@ object ExecutorPodsSnapshot extends Logging {
         case "pending" =>
           PodPending(pod)
         case "running" =>
-          PodRunning(pod)
+          if (shouldCheckAllContainers &&
+            "Never" == pod.getSpec.getRestartPolicy &&
+            pod.getStatus.getContainerStatuses.stream
+              .map[ContainerStateTerminated](cs => cs.getState.getTerminated)
+              .anyMatch(t => t != null && t.getExitCode != 0)) {
+            PodFailed(pod)
+          } else {
+            PodRunning(pod)
+          }
         case "failed" =>
           PodFailed(pod)
         case "succeeded" =>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
index b9d7a7083f41a..f2638129cd786 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
@@ -91,10 +91,13 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
     val schedulerExecutorService = ThreadUtils.newDaemonSingleThreadScheduledExecutor(
       "kubernetes-executor-maintenance")
 
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(
+      sc.conf.get(KUBERNETES_EXECUTOR_CHECK_ALL_CONTAINERS))
     val subscribersExecutor = ThreadUtils
       .newDaemonThreadPoolScheduledExecutor(
         "kubernetes-executor-snapshots-subscribers", 2)
     val snapshotsStore = new ExecutorPodsSnapshotsStoreImpl(subscribersExecutor)
+
     val removedExecutorsCache = CacheBuilder.newBuilder()
       .expireAfterWrite(3, TimeUnit.MINUTES)
       .build[java.lang.Long, java.lang.Long]()
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
index 9ac7e0222054a..e64797213f33f 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
@@ -21,6 +21,8 @@ import scala.collection.mutable
 
 class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore {
 
+  ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
+
   private val snapshotsBuffer = mutable.Buffer.empty[ExecutorPodsSnapshot]
   private val subscribers = mutable.Buffer.empty[Seq[ExecutorPodsSnapshot] => Unit]
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
index 2e883623a4b1c..30312d27b59ed 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala
@@ -71,6 +71,33 @@ object ExecutorLifecycleTestUtils {
       .build()
   }
 
+  /**
+   * [SPARK-30821]
+   * This creates a pod with one container in running state and one container in failed
+   * state (terminated with non-zero exit code). This pod is used for unit-testing the
+   * spark.kubernetes.executor.checkAllContainers Spark Conf.
+   */
+  def runningExecutorWithFailedContainer(executorId: Long): Pod = {
+    new PodBuilder(podWithAttachedContainerForId(executorId))
+      .editOrNewStatus()
+        .withPhase("running")
+        .addNewContainerStatus()
+          .withNewState()
+            .withNewTerminated()
+              .withExitCode(1)
+            .endTerminated()
+          .endState()
+        .endContainerStatus()
+        .addNewContainerStatus()
+          .withNewState()
+            .withNewRunning()
+            .endRunning()
+          .endState()
+        .endContainerStatus()
+      .endStatus()
+      .build()
+  }
+
   def succeededExecutor(executorId: Long): Pod = {
     new PodBuilder(podWithAttachedContainerForId(executorId))
       .editOrNewStatus()
@@ -112,7 +139,10 @@ object ExecutorLifecycleTestUtils {
         .addToLabels(SPARK_APP_ID_LABEL, TEST_SPARK_APP_ID)
         .addToLabels(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)
         .addToLabels(SPARK_EXECUTOR_ID_LABEL, executorId.toString)
-        .endMetadata()
+      .endMetadata()
+      .editOrNewSpec()
+        .withRestartPolicy("Never")
+      .endSpec()
       .build()
     val container = new ContainerBuilder()
       .withName("spark-executor")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
index 70e19c904eddb..96ad6b5ab32a5 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
@@ -16,31 +16,55 @@
  */
 package org.apache.spark.scheduler.cluster.k8s
 
+import io.fabric8.kubernetes.api.model.Pod
+
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils._
 
 class ExecutorPodsSnapshotSuite extends SparkFunSuite {
 
+  def testCase(pod: Pod, state: Pod => ExecutorPodState): (Pod, ExecutorPodState) =
+    (pod, state(pod))
+
+  def doTest(testCases: Seq[(Pod, ExecutorPodState)]): Unit = {
+    val snapshot = ExecutorPodsSnapshot(testCases.map(_._1))
+    for (((_, state), i) <- testCases.zipWithIndex) {
+      assertResult(state.getClass.getName, s"executor ID $i") {
+        snapshot.executorPods(i).getClass.getName
+      }
+    }
+  }
+
   test("States are interpreted correctly from pod metadata.") {
-    val pods = Seq(
-      pendingExecutor(0),
-      runningExecutor(1),
-      succeededExecutor(2),
-      failedExecutorWithoutDeletion(3),
-      deletedExecutor(4),
-      unknownExecutor(5))
-    val snapshot = ExecutorPodsSnapshot(pods)
-    assert(snapshot.executorPods ===
-      Map(
-        0L -> PodPending(pods(0)),
-        1L -> PodRunning(pods(1)),
-        2L -> PodSucceeded(pods(2)),
-        3L -> PodFailed(pods(3)),
-        4L -> PodDeleted(pods(4)),
-        5L -> PodUnknown(pods(5))))
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
+    val testCases = Seq(
+      testCase(pendingExecutor(0), PodPending),
+      testCase(runningExecutor(1), PodRunning),
+      testCase(succeededExecutor(2), PodSucceeded),
+      testCase(failedExecutorWithoutDeletion(3), PodFailed),
+      testCase(deletedExecutor(4), PodDeleted),
+      testCase(unknownExecutor(5), PodUnknown)
+    )
+    doTest(testCases)
+  }
+
+  test("SPARK-30821: States are interpreted correctly from pod metadata"
+    + " when configured to check all containers.") {
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(true)
+    val testCases = Seq(
+      testCase(pendingExecutor(0), PodPending),
+      testCase(runningExecutor(1), PodRunning),
+      testCase(runningExecutorWithFailedContainer(2), PodFailed),
+      testCase(succeededExecutor(3), PodSucceeded),
+      testCase(failedExecutorWithoutDeletion(4), PodFailed),
+      testCase(deletedExecutor(5), PodDeleted),
+      testCase(unknownExecutor(6), PodUnknown)
+    )
+    doTest(testCases)
   }
 
   test("Updates add new pods for non-matching ids and edit existing pods for matching ids") {
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
     val originalPods = Seq(
       pendingExecutor(0),
       runningExecutor(1))
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
index cf54b3c4eb329..614c198bd9caf 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
@@ -35,6 +35,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
   before {
     eventBufferScheduler = new DeterministicScheduler()
     eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler)
+    ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
   }
 
   test("Subscribers get notified of events periodically.") {

From 80716d1a216da12bba0ceeab8c11895d37d5559b Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Sun, 25 Oct 2020 16:15:55 -0700
Subject: [PATCH 1127/1280] [SPARK-33228][SQL] Don't uncache data when
 replacing a view having the same logical plan

### What changes were proposed in this pull request?

SPARK-30494's updated the `CreateViewCommand` code to implicitly drop cache when replacing an existing view. But, this change drops cache even when replacing a view having the same logical plan. A sequence of queries to reproduce this as follows;
```
// Spark v2.4.6+
scala> val df = spark.range(1).selectExpr("id a", "id b")
scala> df.cache()
scala> df.explain()
== Physical Plan ==
*(1) ColumnarToRow
+- InMemoryTableScan [a#2L, b#3L]
      +- InMemoryRelation [a#2L, b#3L], StorageLevel(disk, memory, deserialized, 1 replicas)
            +- *(1) Project [id#0L AS a#2L, id#0L AS b#3L]
               +- *(1) Range (0, 1, step=1, splits=4)

scala> df.createOrReplaceTempView("t")
scala> sql("select * from t").explain()
== Physical Plan ==
*(1) ColumnarToRow
+- InMemoryTableScan [a#2L, b#3L]
      +- InMemoryRelation [a#2L, b#3L], StorageLevel(disk, memory, deserialized, 1 replicas)
            +- *(1) Project [id#0L AS a#2L, id#0L AS b#3L]
               +- *(1) Range (0, 1, step=1, splits=4)

// If one re-runs the same query `df.createOrReplaceTempView("t")`, the cache's swept away
scala> df.createOrReplaceTempView("t")
scala> sql("select * from t").explain()
== Physical Plan ==
*(1) Project [id#0L AS a#2L, id#0L AS b#3L]
+- *(1) Range (0, 1, step=1, splits=4)

// Until v2.4.6
scala> val df = spark.range(1).selectExpr("id a", "id b")
scala> df.cache()
scala> df.createOrReplaceTempView("t")
scala> sql("select * from t").explain()
20/10/23 22:33:42 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException
== Physical Plan ==
*(1) InMemoryTableScan [a#2L, b#3L]
   +- InMemoryRelation [a#2L, b#3L], StorageLevel(disk, memory, deserialized, 1 replicas)
         +- *(1) Project [id#0L AS a#2L, id#0L AS b#3L]
            +- *(1) Range (0, 1, step=1, splits=4)

scala> df.createOrReplaceTempView("t")
scala> sql("select * from t").explain()
== Physical Plan ==
*(1) InMemoryTableScan [a#2L, b#3L]
   +- InMemoryRelation [a#2L, b#3L], StorageLevel(disk, memory, deserialized, 1 replicas)
         +- *(1) Project [id#0L AS a#2L, id#0L AS b#3L]
            +- *(1) Range (0, 1, step=1, splits=4)
```

### Why are the changes needed?

bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #30140 from maropu/FixBugInReplaceView.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 87b498462b82fce02dd50286887092cf7858d2e8)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/execution/command/views.scala   | 10 ++++----
 .../apache/spark/sql/CachedTableSuite.scala   | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
index 23f1d6c983413..0ba76ee94a860 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala
@@ -110,17 +110,19 @@ case class CreateViewCommand(
     verifyTemporaryObjectsNotExists(catalog)
 
     if (viewType == LocalTempView) {
-      if (replace && catalog.getTempView(name.table).isDefined) {
-        logDebug(s"Try to uncache ${name.quotedString} before replacing.")
+      if (replace && catalog.getTempView(name.table).isDefined &&
+          !catalog.getTempView(name.table).get.sameResult(child)) {
+        logInfo(s"Try to uncache ${name.quotedString} before replacing.")
         CommandUtils.uncacheTableOrView(sparkSession, name.quotedString)
       }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
       catalog.createTempView(name.table, aliasedPlan, overrideIfExists = replace)
     } else if (viewType == GlobalTempView) {
-      if (replace && catalog.getGlobalTempView(name.table).isDefined) {
+      if (replace && catalog.getGlobalTempView(name.table).isDefined &&
+          !catalog.getGlobalTempView(name.table).get.sameResult(child)) {
         val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)
         val globalTempView = TableIdentifier(name.table, Option(db))
-        logDebug(s"Try to uncache ${globalTempView.quotedString} before replacing.")
+        logInfo(s"Try to uncache ${globalTempView.quotedString} before replacing.")
         CommandUtils.uncacheTableOrView(sparkSession, globalTempView.quotedString)
       }
       val aliasedPlan = aliasPlan(sparkSession, analyzedPlan)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 20f2a7f947b81..adc725ed9b062 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1184,4 +1184,28 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       assert(spark.sharedState.cacheManager.isEmpty)
     }
   }
+
+  test("SPARK-33228: Don't uncache data when replacing an existing view having the same plan") {
+    withTempView("tempView") {
+      spark.catalog.clearCache()
+      val df = spark.range(1).selectExpr("id a", "id b")
+      df.cache()
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceTempView("tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceTempView("tempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+    }
+
+    withTempView("tempGlobalTempView") {
+      spark.catalog.clearCache()
+      val df = spark.range(1).selectExpr("id a", "id b")
+      df.cache()
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceGlobalTempView("tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+      df.createOrReplaceGlobalTempView("tempGlobalTempView")
+      assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
+    }
+  }
 }

From 590ccb3ac4c32dd67226d2765406d23b949574f6 Mon Sep 17 00:00:00 2001
From: Yuning Zhang <yuning.zhang@databricks.com>
Date: Mon, 26 Oct 2020 16:19:06 +0900
Subject: [PATCH 1128/1280] [SPARK-33197][SQL] Make changes to
 spark.sql.analyzer.maxIterations take effect at runtime

### What changes were proposed in this pull request?

Make changes to `spark.sql.analyzer.maxIterations` take effect at runtime.

### Why are the changes needed?

`spark.sql.analyzer.maxIterations` is not a static conf. However, before this patch, changing `spark.sql.analyzer.maxIterations` at runtime does not take effect.

### Does this PR introduce _any_ user-facing change?

Yes. Before this patch, changing `spark.sql.analyzer.maxIterations` at runtime does not take effect.

### How was this patch tested?

modified unit test

Closes #30108 from yuningzh-db/dynamic-analyzer-max-iterations.

Authored-by: Yuning Zhang <yuning.zhang@databricks.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit a21945ce6c725896d19647891d1f9fa9ef74bd87)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/analysis/Analyzer.scala      | 16 +++-----
 .../sql/catalyst/analysis/AnalysisSuite.scala | 41 +++++++++++++++++++
 2 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index bf3d4f0332c53..2c9681b3e8be9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -129,8 +129,7 @@ object AnalysisContext {
  */
 class Analyzer(
     override val catalogManager: CatalogManager,
-    conf: SQLConf,
-    maxIterations: Int)
+    conf: SQLConf)
   extends RuleExecutor[LogicalPlan] with CheckAnalysis with LookupCatalog {
 
   private val v1SessionCatalog: SessionCatalog = catalogManager.v1SessionCatalog
@@ -141,12 +140,7 @@ class Analyzer(
   def this(catalog: SessionCatalog, conf: SQLConf) = {
     this(
       new CatalogManager(conf, FakeV2SessionCatalog, catalog),
-      conf,
-      conf.analyzerMaxIterations)
-  }
-
-  def this(catalogManager: CatalogManager, conf: SQLConf) = {
-    this(catalogManager, conf, conf.analyzerMaxIterations)
+      conf)
   }
 
   def executeAndCheck(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
@@ -181,9 +175,9 @@ class Analyzer(
    * If the plan cannot be resolved within maxIterations, analyzer will throw exception to inform
    * user to increase the value of SQLConf.ANALYZER_MAX_ITERATIONS.
    */
-  protected val fixedPoint =
+  protected def fixedPoint =
     FixedPoint(
-      maxIterations,
+      conf.analyzerMaxIterations,
       errorOnExceed = true,
       maxIterationsSetting = SQLConf.ANALYZER_MAX_ITERATIONS.key)
 
@@ -199,7 +193,7 @@ class Analyzer(
    */
   val postHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil
 
-  lazy val batches: Seq[Batch] = Seq(
+  override def batches: Seq[Batch] = Seq(
     Batch("Substitution", fixedPoint,
       CTESubstitution,
       WindowsSubstitution,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index 6ec341db42d3a..b4327471354e8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -910,4 +910,45 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     )
     assertAnalysisSuccess(plan)
   }
+
+  test("SPARK-33197: Make sure changes to ANALYZER_MAX_ITERATIONS take effect at runtime") {
+    // RuleExecutor only throw exception or log warning when the rule is supposed to run
+    // more than once.
+    val maxIterations = 2
+    val maxIterationsEnough = 5
+    withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterations.toString) {
+      val conf = SQLConf.get
+      val testAnalyzer = new Analyzer(
+        new SessionCatalog(new InMemoryCatalog, FunctionRegistry.builtin, conf), conf)
+
+      val plan = testRelation2.select(
+        $"a" / Literal(2) as "div1",
+        $"a" / $"b" as "div2",
+        $"a" / $"c" as "div3",
+        $"a" / $"d" as "div4",
+        $"e" / $"e" as "div5")
+
+      val message1 = intercept[TreeNodeException[LogicalPlan]] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message1.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+
+      withSQLConf(SQLConf.ANALYZER_MAX_ITERATIONS.key -> maxIterationsEnough.toString) {
+        try {
+          testAnalyzer.execute(plan)
+        } catch {
+          case ex: TreeNodeException[_]
+            if ex.getMessage.contains(SQLConf.ANALYZER_MAX_ITERATIONS.key) =>
+              fail("analyzer.execute should not reach max iterations.")
+        }
+      }
+
+      val message2 = intercept[TreeNodeException[LogicalPlan]] {
+        testAnalyzer.execute(plan)
+      }.getMessage
+      assert(message2.startsWith(s"Max iterations ($maxIterations) reached for batch Resolution, " +
+        s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
+    }
+  }
 }

From 22392be4315900346ac884ad32bd6332ffc3e64e Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Mon, 26 Oct 2020 12:31:05 -0700
Subject: [PATCH 1129/1280] [SPARK-33230][SQL] Hadoop committers to get unique
 job ID in "spark.sql.sources.writeJobUUID"

### What changes were proposed in this pull request?

This reinstates the old option `spark.sql.sources.write.jobUUID` to set a unique jobId in the jobconf so that hadoop MR committers have a unique ID which is (a) consistent across tasks and workers and (b) not brittle compared to generated-timestamp job IDs. The latter matches that of what JobID requires, but as they are generated per-thread, may not always be unique within a cluster.

### Why are the changes needed?

If a committer (e.g s3a staging committer) uses job-attempt-ID as a unique ID then any two jobs started within the same second have the same ID, so can clash.

### Does this PR introduce _any_ user-facing change?

Good Q. It is "developer-facing" in the context of anyone writing a committer. But it reinstates a property which was in Spark 1.x and "went away"

### How was this patch tested?

Testing: no test here. You'd have to create a new committer which extracted the value in both job and task(s) and verified consistency. That is possible (with a task output whose records contained the UUID), but it would be pretty convoluted and a high maintenance cost.

Because it's trying to address a race condition, it's hard to regenerate the problem downstream and so verify a fix in a test run...I'll just look at the logs to see what temporary dir is being used in the cluster FS and verify it's a UUID

Closes #30141 from steveloughran/SPARK-33230-jobId.

Authored-by: Steve Loughran <stevel@cloudera.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 02fa19f102122f06e4358cf86c5e903fda28b289)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/execution/datasources/FileFormatWriter.scala  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
index abb88ae73cabf..a71aeb47872ce 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala
@@ -133,7 +133,7 @@ object FileFormatWriter extends Logging {
       fileFormat.prepareWrite(sparkSession, job, caseInsensitiveOptions, dataSchema)
 
     val description = new WriteJobDescription(
-      uuid = UUID.randomUUID().toString,
+      uuid = UUID.randomUUID.toString,
       serializableHadoopConf = new SerializableConfiguration(job.getConfiguration),
       outputWriterFactory = outputWriterFactory,
       allColumns = outputSpec.outputColumns,
@@ -164,6 +164,10 @@ object FileFormatWriter extends Logging {
 
     SQLExecution.checkSQLExecutionId(sparkSession)
 
+    // propagate the decription UUID into the jobs, so that committers
+    // get an ID guaranteed to be unique.
+    job.getConfiguration.set("spark.sql.sources.writeJobUUID", description.uuid)
+
     // This call shouldn't be put into the `try` block below because it only initializes and
     // prepares the job, any exception thrown from here shouldn't cause abortJob() to be called.
     committer.setupJob(job)

From c95d925a9add139e87c7b9f5c95571fdb034f724 Mon Sep 17 00:00:00 2001
From: Ankur Dave <ankurdave@gmail.com>
Date: Tue, 27 Oct 2020 13:20:22 -0700
Subject: [PATCH 1130/1280] [SPARK-33260][SQL] Fix incorrect results from
 SortExec when sortOrder is Stream

### What changes were proposed in this pull request?

The following query produces incorrect results. The query has two essential features: (1) it contains a string aggregate, resulting in a `SortExec` node, and (2) it contains a duplicate grouping key, causing `RemoveRepetitionFromGroupExpressions` to produce a sort order stored as a `Stream`.

```sql
SELECT bigint_col_1, bigint_col_9, MAX(CAST(bigint_col_1 AS string))
FROM table_4
GROUP BY bigint_col_1, bigint_col_9, bigint_col_9
```

When the sort order is stored as a `Stream`, the line `ordering.map(_.child.genCode(ctx))` in `GenerateOrdering#createOrderKeys()` produces unpredictable side effects to `ctx`. This is because `genCode(ctx)` modifies `ctx`. When ordering is a `Stream`, the modifications will not happen immediately as intended, but will instead occur lazily when the returned `Stream` is used later.

Similar bugs have occurred at least three times in the past: https://issues.apache.org/jira/browse/SPARK-24500, https://issues.apache.org/jira/browse/SPARK-25767, https://issues.apache.org/jira/browse/SPARK-26680.

The fix is to check if `ordering` is a `Stream` and force the modifications to happen immediately if so.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added a unit test for `SortExec` where `sortOrder` is a `Stream`. The test previously failed and now passes.

Closes #30160 from ankurdave/SPARK-33260.

Authored-by: Ankur Dave <ankurdave@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 3f2a2b5fe6ada37ef86f00737387e6cf2496df74)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../expressions/codegen/GenerateOrdering.scala      |  4 +++-
 .../org/apache/spark/sql/execution/SortSuite.scala  | 13 +++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
index 63bd59e7628b2..5d00519d27c53 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
@@ -71,7 +71,9 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], BaseOrdering] with
     ctx.INPUT_ROW = row
     // to use INPUT_ROW we must make sure currentVars is null
     ctx.currentVars = null
-    ordering.map(_.child.genCode(ctx))
+    // SPARK-33260: To avoid unpredictable modifications to `ctx` when `ordering` is a Stream, we
+    // use `toIndexedSeq` to make the transformation eager.
+    ordering.toIndexedSeq.map(_.child.genCode(ctx))
   }
 
   /**
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
index 7654a9d982059..6a4f3f62641f8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala
@@ -97,6 +97,19 @@ class SortSuite extends SparkPlanTest with SharedSparkSession {
     }
   }
 
+  test("SPARK-33260: sort order is a Stream") {
+    val input = Seq(
+      ("Hello", 4, 2.0),
+      ("Hello", 1, 1.0),
+      ("World", 8, 3.0)
+    )
+    checkAnswer(
+      input.toDF("a", "b", "c"),
+      (child: SparkPlan) => SortExec(Stream('a.asc, 'b.asc, 'c.asc), global = true, child = child),
+      input.sortBy(t => (t._1, t._2, t._3)).map(Row.fromTuple),
+      sortAnswers = false)
+  }
+
   // Test sorting on different data types
   for (
     dataType <- DataTypeTestUtils.atomicTypes ++ Set(NullType);

From e37859acffe0ccc6d2fe07683927c5701f11a265 Mon Sep 17 00:00:00 2001
From: Stuart White <stuart@spotright.com>
Date: Wed, 28 Oct 2020 08:36:14 +0900
Subject: [PATCH 1131/1280] [SPARK-33246][SQL][DOCS] Correct documentation for
 null semantics of "NULL AND False"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

The documentation of the Spark SQL null semantics states that "NULL AND False" yields NULL.  This is incorrect.  "NULL AND False" yields False.

```
Seq[(java.lang.Boolean, java.lang.Boolean)](
  (null, false)
)
  .toDF("left_operand", "right_operand")
  .withColumn("AND", 'left_operand && 'right_operand)
  .show(truncate = false)

+------------+-------------+-----+
|left_operand|right_operand|AND  |
+------------+-------------+-----+
|null        |false        |false|
+------------+-------------+-----+
```

I propose the documentation be updated to reflect that "NULL AND False" yields False.

This contribution is my original work and I license it to the project under the project’s open source license.

### Why are the changes needed?

This change improves the accuracy of the documentation.

### Does this PR introduce _any_ user-facing change?

Yes.  This PR introduces a fix to the documentation.

### How was this patch tested?

Since this is only a documentation change, no tests were added.

Closes #30161 from stwhit/SPARK-33246.

Authored-by: Stuart White <stuart@spotright.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 7d11d972c356140d21909c6a62cdb8d813bd015e)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/sql-ref-null-semantics.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-ref-null-semantics.md b/docs/sql-ref-null-semantics.md
index fb5d2a312d0e1..3c12e7a28b64e 100644
--- a/docs/sql-ref-null-semantics.md
+++ b/docs/sql-ref-null-semantics.md
@@ -125,7 +125,7 @@ The following tables illustrate the behavior of logical operators when one or bo
 |True|NULL|True|NULL|
 |False|NULL|NULL|False|
 |NULL|True|True|NULL|
-|NULL|False|NULL|NULL|
+|NULL|False|NULL|False|
 |NULL|NULL|NULL|NULL|
 
 |operand|NOT|

From 737a85033a543613fbdc6ac5737a0d23f7e9f108 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Sun, 28 Jun 2020 21:49:10 -0700
Subject: [PATCH 1132/1280] [SPARK-32090][SQL] Improve UserDefinedType.equal()
 to make it be symmetrical

### What changes were proposed in this pull request?

This PR fix `UserDefinedType.equal()` by comparing the UDT class instead of checking `acceptsType()`.

### Why are the changes needed?

It's weird that equality comparison between two UDT types can have different result by switching the order:

```scala
// ExampleSubTypeUDT.userClass is a subclass of ExampleBaseTypeUDT.userClass
val udt1 = new ExampleBaseTypeUDT
val udt2 = new ExampleSubTypeUDT
println(udt1 == udt2) // true
println(udt2 == udt1) // false
```

### Does this PR introduce _any_ user-facing change?

Yes.

Before:
```scala
// ExampleSubTypeUDT.userClass is a subclass of ExampleBaseTypeUDT.userClass
val udt1 = new ExampleBaseTypeUDT
val udt2 = new ExampleSubTypeUDT
println(udt1 == udt2) // true
println(udt2 == udt1) // false
```

After:
```scala
// ExampleSubTypeUDT.userClass is a subclass of ExampleBaseTypeUDT.userClass
val udt1 = new ExampleBaseTypeUDT
val udt2 = new ExampleSubTypeUDT
println(udt1 == udt2) // false
println(udt2 == udt1) // false
```

### How was this patch tested?

Added a unit test.

Closes #28923 from Ngone51/fix-udt-equal.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/types/UserDefinedType.scala      |  2 +-
 .../spark/sql/UserDefinedTypeSuite.scala       | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
index 6af16e2dba105..592ce03606d4b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
@@ -90,7 +90,7 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
   override def hashCode(): Int = getClass.hashCode()
 
   override def equals(other: Any): Boolean = other match {
-    case that: UserDefinedType[_] => this.acceptsType(that)
+    case that: UserDefinedType[_] => this.getClass == that.getClass
     case _ => false
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
index ed8ab1cb3a603..7c126f5e9a0b5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/UserDefinedTypeSuite.scala
@@ -134,6 +134,24 @@ class UserDefinedTypeSuite extends QueryTest with SharedSparkSession with Parque
     MyLabeledPoint(1.0, new TestUDT.MyDenseVector(Array(0.1, 1.0))),
     MyLabeledPoint(0.0, new TestUDT.MyDenseVector(Array(0.3, 3.0)))).toDF()
 
+
+  test("SPARK-32090: equal") {
+    val udt1 = new ExampleBaseTypeUDT
+    val udt2 = new ExampleSubTypeUDT
+    val udt3 = new ExampleSubTypeUDT
+    assert(udt1 !== udt2)
+    assert(udt2 !== udt1)
+    assert(udt2 === udt3)
+    assert(udt3 === udt2)
+  }
+
+  test("SPARK-32090: acceptsType") {
+    val udt1 = new ExampleBaseTypeUDT
+    val udt2 = new ExampleSubTypeUDT
+    assert(udt1.acceptsType(udt2))
+    assert(!udt2.acceptsType(udt1))
+  }
+
   test("register user type: MyDenseVector for MyLabeledPoint") {
     val labels: RDD[Double] = pointsRDD.select('label).rdd.map { case Row(v: Double) => v }
     val labelsArrays: Array[Double] = labels.collect()

From ba2a11330038771d085f2ea9b50940ec3b8b97c5 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Wed, 28 Oct 2020 11:21:35 +0900
Subject: [PATCH 1133/1280] [SPARK-33264][SQL][DOCS] Add a dedicated page for
 SQL-on-file in SQL documents

### What changes were proposed in this pull request?

This PR intends to add a dedicated page for SQL-on-file in SQL documents.
This comes from the comment: https://github.com/apache/spark/pull/30095/files#r508965149

### Why are the changes needed?

For better documentations.

### Does this PR introduce _any_ user-facing change?

<img width="544" alt="Screen Shot 2020-10-28 at 9 56 59" src="https://user-images.githubusercontent.com/692303/97378051-c1fbcb80-1904-11eb-86c0-a88c5269d41c.png">

### How was this patch tested?

N/A

Closes #30165 from maropu/DocForFile.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit c2bea045e3628081bca1ba752669a5bc009ebd00)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/_data/menu-sql.yaml               |  2 +
 docs/sql-ref-syntax-qry-select-file.md | 76 ++++++++++++++++++++++++++
 docs/sql-ref-syntax-qry-select.md      |  3 +-
 docs/sql-ref-syntax-qry.md             |  1 +
 docs/sql-ref-syntax.md                 |  1 +
 5 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 docs/sql-ref-syntax-qry-select-file.md

diff --git a/docs/_data/menu-sql.yaml b/docs/_data/menu-sql.yaml
index 22fae0cb3ff55..581fd1de6ad2d 100644
--- a/docs/_data/menu-sql.yaml
+++ b/docs/_data/menu-sql.yaml
@@ -175,6 +175,8 @@
                   url: sql-ref-syntax-qry-select-hints.html
                 - text: Inline Table
                   url: sql-ref-syntax-qry-select-inline-table.html
+                - text: File
+                  url: sql-ref-syntax-qry-select-file.html
                 - text: JOIN
                   url: sql-ref-syntax-qry-select-join.html
                 - text: LIKE Predicate
diff --git a/docs/sql-ref-syntax-qry-select-file.md b/docs/sql-ref-syntax-qry-select-file.md
new file mode 100644
index 0000000000000..c3dc406ee79e6
--- /dev/null
+++ b/docs/sql-ref-syntax-qry-select-file.md
@@ -0,0 +1,76 @@
+---
+layout: global
+title: File
+displayTitle: File
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+ 
+     http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+You can query a file with a specified format directly with SQL.
+
+### Syntax
+
+```sql
+file_format.`file_path`
+```
+
+### Parameters
+
+* **file_format**
+
+    Specifies a file format for a given file path, could be TEXTFILE, ORC, PARQUET, etc.
+
+* **file_path**
+
+    Specifies a file path with a given format.
+
+### Examples
+
+```sql
+-- PARQUET file
+SELECT * FROM parquet.`examples/src/main/resources/users.parquet`;
++------+--------------+----------------+
+|  name|favorite_color|favorite_numbers|
++------+--------------+----------------+
+|Alyssa|          null|  [3, 9, 15, 20]|
+|   Ben|           red|              []|
++------+--------------+----------------+
+
+-- ORC file
+SELECT * FROM orc.`examples/src/main/resources/users.orc`;
++------+--------------+----------------+
+|  name|favorite_color|favorite_numbers|
++------+--------------+----------------+
+|Alyssa|          null|  [3, 9, 15, 20]|
+|   Ben|           red|              []|
++------+--------------+----------------+
+
+-- JSON file
+SELECT * FROM json.`examples/src/main/resources/people.json`;
++----+-------+
+| age|   name|
++----+-------+
+|null|Michael|
+|  30|   Andy|
+|  19| Justin|
++----+-------+
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
diff --git a/docs/sql-ref-syntax-qry-select.md b/docs/sql-ref-syntax-qry-select.md
index 655766d4c6d22..bac7c2bc6a06d 100644
--- a/docs/sql-ref-syntax-qry-select.md
+++ b/docs/sql-ref-syntax-qry-select.md
@@ -85,7 +85,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
      * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
      * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
      * Subquery
-     * [File](sql-data-sources-load-save-functions.html#run-sql-on-files-directly)
+     * [File](sql-ref-syntax-qry-select-file.html)
      
 * **PIVOT**
 
@@ -165,6 +165,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { named_expression [ , ... ] }
 * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
 * [Hints](sql-ref-syntax-qry-select-hints.html)
 * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+* [File](sql-ref-syntax-qry-select-file.html)
 * [JOIN](sql-ref-syntax-qry-select-join.html)
 * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
 * [Set Operators](sql-ref-syntax-qry-select-setops.html)
diff --git a/docs/sql-ref-syntax-qry.md b/docs/sql-ref-syntax-qry.md
index d55ea43d15036..6751b90e12443 100644
--- a/docs/sql-ref-syntax-qry.md
+++ b/docs/sql-ref-syntax-qry.md
@@ -39,6 +39,7 @@ ability to generate logical and physical plan for a given query using
   * [Common Table Expression](sql-ref-syntax-qry-select-cte.html)
   * [Hints](sql-ref-syntax-qry-select-hints.html)
   * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+  * [File](sql-ref-syntax-qry-select-file.html)
   * [JOIN](sql-ref-syntax-qry-select-join.html)
   * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
   * [Set Operators](sql-ref-syntax-qry-select-setops.html)
diff --git a/docs/sql-ref-syntax.md b/docs/sql-ref-syntax.md
index 4318cd69e7370..148ce107c94bd 100644
--- a/docs/sql-ref-syntax.md
+++ b/docs/sql-ref-syntax.md
@@ -56,6 +56,7 @@ Spark SQL is Apache Spark's module for working with structured data. The SQL Syn
    * [HAVING Clause](sql-ref-syntax-qry-select-having.html)
    * [Hints](sql-ref-syntax-qry-select-hints.html)
    * [Inline Table](sql-ref-syntax-qry-select-inline-table.html)
+   * [File](sql-ref-syntax-qry-select-file.html)
    * [JOIN](sql-ref-syntax-qry-select-join.html)
    * [LIKE Predicate](sql-ref-syntax-qry-select-like.html)
    * [LIMIT Clause](sql-ref-syntax-qry-select-limit.html)

From f6c72e6f0f1edeadd41f8e3825e2dcb1996aa433 Mon Sep 17 00:00:00 2001
From: "zky.zhoukeyong" <zky.zhoukeyong@alibaba-inc.com>
Date: Wed, 28 Oct 2020 13:17:28 +0000
Subject: [PATCH 1134/1280] [SPARK-33208][SQL] Update the document of
 SparkSession#sql

Change-Id: I82db1f9e8f667573aa3a03e05152cbed0ea7686b

### What changes were proposed in this pull request?
Update the document of SparkSession#sql, mention that this API eagerly runs DDL/DML commands, but not for SELECT queries.

### Why are the changes needed?
To clarify the behavior of SparkSession#sql.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
No needed.

Closes #30168 from waitinfuture/master.

Authored-by: zky.zhoukeyong <zky.zhoukeyong@alibaba-inc.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit b26ae98407c6c017a4061c0c420f48685ddd6163)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 4 ++--
 .../src/main/scala/org/apache/spark/sql/SparkSession.scala    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 68ce82d5badda..7cf0b6bb70364 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -641,8 +641,8 @@ class SQLContext private[sql](val sparkSession: SparkSession)
   }
 
   /**
-   * Executes a SQL query using Spark, returning the result as a `DataFrame`. The dialect that is
-   * used for SQL parsing can be configured with 'spark.sql.dialect'.
+   * Executes a SQL query using Spark, returning the result as a `DataFrame`.
+   * This API eagerly runs DDL/DML commands, but not for SELECT queries.
    *
    * @group basic
    * @since 1.3.0
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index fc941270e1b64..f89e58c6e971c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -595,7 +595,7 @@ class SparkSession private(
 
   /**
    * Executes a SQL query using Spark, returning the result as a `DataFrame`.
-   * The dialect that is used for SQL parsing can be configured with 'spark.sql.dialect'.
+   * This API eagerly runs DDL/DML commands, but not for SELECT queries.
    *
    * @since 2.0.0
    */

From 3ce335d98cb3ce9a4c5a961d0f021fc4782b0f0c Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Wed, 28 Oct 2020 10:00:29 -0700
Subject: [PATCH 1135/1280] [SPARK-33267][SQL] Fix NPE issue on 'In' filter
 when one of values contains null

### What changes were proposed in this pull request?

This PR proposes to fix the NPE issue on `In` filter when one of values contain null. In real case, you can trigger this issue when you try to push down the filter with `in (..., null)` against V2 source table. `DataSourceStrategy` caches the mapping (filter instance -> expression) in HashMap, which leverages hash code on the key, hence it could trigger the NPE issue.

### Why are the changes needed?

This is an obvious bug as `In` filter doesn't care about null value when calculating hash code.

### Does this PR introduce _any_ user-facing change?

Yes, previously the query with having `null` in "in" condition against data source V2 source table supporting push down filter failed with NPE, whereas after the PR the query will not fail.

### How was this patch tested?

UT added. The new UT fails without the PR and passes with the PR.

Closes #30170 from HeartSaVioR/SPARK-33267.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit a744fea3be12f1a53ab553040b95da730210bc88)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/sql/sources/filters.scala   |  2 +-
 .../apache/spark/sql/connector/DataSourceV2Suite.scala | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
index 7533793253513..2b44a3a861ed9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/sources/filters.scala
@@ -164,7 +164,7 @@ case class In(attribute: String, values: Array[Any]) extends Filter {
     var h = attribute.hashCode
     values.foreach { v =>
       h *= 41
-      h += v.hashCode()
+      h += (if (v != null) v.hashCode() else 0)
     }
     h
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
index a9c521eb46499..c2edcce8ce2ce 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala
@@ -413,6 +413,16 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS
       }
     }
   }
+
+  test("SPARK-33267: push down with condition 'in (..., null)' should not throw NPE") {
+    Seq(classOf[AdvancedDataSourceV2], classOf[JavaAdvancedDataSourceV2]).foreach { cls =>
+      withClue(cls.getName) {
+        val df = spark.read.format(cls.getName).load()
+        // before SPARK-33267 below query just threw NPE
+        df.select('i).where("i in (1, null)").collect()
+      }
+    }
+  }
 }
 
 

From f5dc06ed694978bea2738bd778d3c63d2c516aea Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Wed, 28 Oct 2020 14:21:32 -0700
Subject: [PATCH 1136/1280] [SPARK-32119][CORE][3.0] ExecutorPlugin doesn't
 work with Standalone Cluster and Kubernetes with --jars

### What changes were proposed in this pull request?

This is a backport PR for branch-3.0.

This PR changes Executor to load jars and files added by --jars and --files on Executor initialization.
To avoid downloading those jars/files twice, they are assosiated with `startTime` as their uploaded timestamp.

### Why are the changes needed?

ExecutorPlugin can't work with Standalone Cluster and Kubernetes
when a jar which contains plugins and files used by the plugins are added by --jars and --files option with spark-submit.

This is because jars and files added by --jars and --files are not loaded on Executor initialization.
I confirmed it works with YARN because jars/files are distributed as distributed cache.

### Does this PR introduce _any_ user-facing change?

Yes. jars/files added by --jars and --files are downloaded on each executor on initialization.

### How was this patch tested?

Added a new testcase.

Closes #29621 from sarutak/fix-plugin-issue-3.0.

Lead-authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Co-authored-by: Luca Canali <luca.canali@cern.ch>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/SparkContext.scala | 27 ++++--
 .../org/apache/spark/executor/Executor.scala  | 14 ++++
 .../apache/spark/JavaSparkContextSuite.java   |  7 +-
 .../spark/deploy/SparkSubmitSuite.scala       | 83 ++++++++++++++++++-
 docs/monitoring.md                            |  6 --
 5 files changed, 122 insertions(+), 15 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 9f9d611b79376..3ccbea5136f4a 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -380,6 +380,7 @@ class SparkContext(config: SparkConf) extends Logging {
   try {
     _conf = config.clone()
     _conf.validateSettings()
+    _conf.set("spark.app.startTime", startTime.toString)
 
     if (!_conf.contains("spark.master")) {
       throw new SparkException("A master URL must be set in your configuration")
@@ -487,11 +488,17 @@ class SparkContext(config: SparkConf) extends Logging {
 
     // Add each JAR given through the constructor
     if (jars != null) {
-      jars.foreach(addJar)
+      jars.foreach(jar => addJar(jar, true))
+      if (addedJars.nonEmpty) {
+        _conf.set("spark.app.initial.jar.urls", addedJars.keys.toSeq.mkString(","))
+      }
     }
 
     if (files != null) {
-      files.foreach(addFile)
+      files.foreach(file => addFile(file, false, true))
+      if (addedFiles.nonEmpty) {
+        _conf.set("spark.app.initial.file.urls", addedFiles.keys.toSeq.mkString(","))
+      }
     }
 
     _executorMemory = _conf.getOption(EXECUTOR_MEMORY.key)
@@ -1495,7 +1502,7 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addFile(path: String): Unit = {
-    addFile(path, false)
+    addFile(path, false, false)
   }
 
   /**
@@ -1517,6 +1524,10 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addFile(path: String, recursive: Boolean): Unit = {
+    addFile(path, recursive, false)
+  }
+
+  private def addFile(path: String, recursive: Boolean, addedOnSubmit: Boolean): Unit = {
     val uri = new Path(path).toUri
     val schemeCorrectedURI = uri.getScheme match {
       case null => new File(path).getCanonicalFile.toURI
@@ -1554,7 +1565,7 @@ class SparkContext(config: SparkConf) extends Logging {
           path
         }
     }
-    val timestamp = System.currentTimeMillis
+    val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
     if (addedFiles.putIfAbsent(key, timestamp).isEmpty) {
       logInfo(s"Added file $path at $key with timestamp $timestamp")
       // Fetch the file locally so that closures which are run on the driver can still use the
@@ -1564,7 +1575,7 @@ class SparkContext(config: SparkConf) extends Logging {
       postEnvironmentUpdate()
     } else {
       logWarning(s"The path $path has been added already. Overwriting of added paths " +
-       "is not supported in the current version.")
+        "is not supported in the current version.")
     }
   }
 
@@ -1827,6 +1838,10 @@ class SparkContext(config: SparkConf) extends Logging {
    * @note A path can be added only once. Subsequent additions of the same path are ignored.
    */
   def addJar(path: String): Unit = {
+    addJar(path, false)
+  }
+
+  private def addJar(path: String, addedOnSubmit: Boolean): Unit = {
     def addLocalJarFile(file: File): String = {
       try {
         if (!file.exists()) {
@@ -1891,7 +1906,7 @@ class SparkContext(config: SparkConf) extends Logging {
         }
       }
       if (key != null) {
-        val timestamp = System.currentTimeMillis
+        val timestamp = if (addedOnSubmit) startTime else System.currentTimeMillis
         if (addedJars.putIfAbsent(key, timestamp).isEmpty) {
           logInfo(s"Added JAR $path at $key with timestamp $timestamp")
           postEnvironmentUpdate()
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 83e4a739f5817..baf9e461eb8ff 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -213,6 +213,20 @@ private[spark] class Executor(
 
   heartbeater.start()
 
+  private val appStartTime = conf.getLong("spark.app.startTime", 0)
+
+  // To allow users to distribute plugins and their required files
+  // specified by --jars and --files on application submission, those jars/files should be
+  // downloaded and added to the class loader via updateDependencies.
+  // This should be done before plugin initialization below
+  // because executors search plugins from the class loader and initialize them.
+  private val Seq(initialUserJars, initialUserFiles) = Seq("jar", "file").map { key =>
+    conf.getOption(s"spark.app.initial.$key.urls").map { urls =>
+      Map(urls.split(",").map(url => (url, appStartTime)): _*)
+    }.getOrElse(Map.empty)
+  }
+  updateDependencies(initialUserFiles, initialUserJars)
+
   // Plugins need to load using a class loader that includes the executor's user classpath.
   // Plugins also needs to be initialized after the heartbeater started
   // to avoid blocking to send heartbeat (see SPARK-32175).
diff --git a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
index 0f489fb219010..b188ee16b97d0 100644
--- a/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
+++ b/core/src/test/java/test/org/apache/spark/JavaSparkContextSuite.java
@@ -28,6 +28,7 @@
 
 import org.apache.spark.api.java.*;
 import org.apache.spark.*;
+import org.apache.spark.util.Utils;
 
 /**
  * Java apps can use both Java-friendly JavaSparkContext and Scala SparkContext.
@@ -35,14 +36,16 @@
 public class JavaSparkContextSuite implements Serializable {
 
   @Test
-  public void javaSparkContext() {
+  public void javaSparkContext() throws IOException {
+    File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark");
+    String dummyJarFile = File.createTempFile(tempDir.toString(), "jarFile").toString();
     String[] jars = new String[] {};
     java.util.Map<String, String> environment = new java.util.HashMap<>();
 
     new JavaSparkContext(new SparkConf().setMaster("local").setAppName("name")).stop();
     new JavaSparkContext("local", "name", new SparkConf()).stop();
     new JavaSparkContext("local", "name").stop();
-    new JavaSparkContext("local", "name", "sparkHome", "jarFile").stop();
+    new JavaSparkContext("local", "name", "sparkHome", dummyJarFile).stop();
     new JavaSparkContext("local", "name", "sparkHome", jars).stop();
     new JavaSparkContext("local", "name", "sparkHome", jars, environment).stop();
   }
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 071508713e405..fb2a65e8c07cf 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -566,7 +566,8 @@ class SparkSubmitSuite
       }
     }
 
-    val clArgs2 = Seq("--class", "org.SomeClass", "thejar.jar")
+    val dummyJarFile = TestUtils.createJarWithClasses(Seq.empty)
+    val clArgs2 = Seq("--class", "org.SomeClass", dummyJarFile.toString)
     val appArgs2 = new SparkSubmitArguments(clArgs2)
     val (_, _, conf2, _) = submit.prepareSubmitEnvironment(appArgs2)
     assert(!conf2.contains(UI_SHOW_CONSOLE_PROGRESS))
@@ -1216,6 +1217,86 @@ class SparkSubmitSuite
     testRemoteResources(enableHttpFs = true, blacklistSchemes = Seq("*"))
   }
 
+  test("SPARK-32119: Jars and files should be loaded when Executors launch for plugins") {
+    val tempDir = Utils.createTempDir()
+    val tempFileName = "test.txt"
+    val tempFile = new File(tempDir, tempFileName)
+
+    // scalastyle:off println
+    Utils.tryWithResource {
+      new PrintWriter(tempFile)
+    } { writer =>
+      writer.println("SparkPluginTest")
+    }
+    // scalastyle:on println
+
+    val sparkPluginCodeBody =
+      """
+        |@Override
+        |public org.apache.spark.api.plugin.ExecutorPlugin executorPlugin() {
+        |  return new TestExecutorPlugin();
+        |}
+        |
+        |@Override
+        |public org.apache.spark.api.plugin.DriverPlugin driverPlugin() { return null; }
+      """.stripMargin
+    val executorPluginCodeBody =
+      s"""
+        |@Override
+        |public void init(
+        |    org.apache.spark.api.plugin.PluginContext ctx,
+        |    java.util.Map<String, String> extraConf) {
+        |  String str = null;
+        |  try (java.io.BufferedReader reader =
+        |    new java.io.BufferedReader(new java.io.InputStreamReader(
+        |      new java.io.FileInputStream("$tempFileName")))) {
+        |    str = reader.readLine();
+        |  } catch (java.io.IOException e) {
+        |    throw new RuntimeException(e);
+        |  } finally {
+        |    assert str == "SparkPluginTest";
+        |  }
+        |}
+      """.stripMargin
+
+    val compiledExecutorPlugin = TestUtils.createCompiledClass(
+      "TestExecutorPlugin",
+      tempDir,
+      "",
+      null,
+      Seq.empty,
+      Seq("org.apache.spark.api.plugin.ExecutorPlugin"),
+      executorPluginCodeBody)
+
+    val thisClassPath =
+      sys.props("java.class.path").split(File.pathSeparator).map(p => new File(p).toURI.toURL)
+    val compiledSparkPlugin = TestUtils.createCompiledClass(
+      "TestSparkPlugin",
+      tempDir,
+      "",
+      null,
+      Seq(tempDir.toURI.toURL) ++ thisClassPath,
+      Seq("org.apache.spark.api.plugin.SparkPlugin"),
+      sparkPluginCodeBody)
+
+    val jarUrl = TestUtils.createJar(
+      Seq(compiledSparkPlugin, compiledExecutorPlugin),
+      new File(tempDir, "testplugin.jar"))
+
+    val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
+    val unusedFile = Files.createTempFile(tempDir.toPath, "unused", null)
+    val args = Seq(
+      "--class", SimpleApplicationTest.getClass.getName.stripSuffix("$"),
+      "--name", "testApp",
+      "--master", "local-cluster[1,1,1024]",
+      "--conf", "spark.plugins=TestSparkPlugin",
+      "--conf", "spark.ui.enabled=false",
+      "--jars", jarUrl.toString + "," + unusedJar.toString,
+      "--files", tempFile.toString + "," + unusedFile.toString,
+      unusedJar.toString)
+    runSparkSubmit(args)
+  }
+
   private def testRemoteResources(
       enableHttpFs: Boolean,
       blacklistSchemes: Seq[String] = Nil): Unit = {
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 4608a4e61a41c..8471417184974 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -1326,9 +1326,3 @@ Both take a comma-separated list of class names that implement the
 possible for one list to be placed in the Spark default config file, allowing users to
 easily add other plugins from the command line without overwriting the config file's list. Duplicate
 plugins are ignored.
-
-Distribution of the jar files containing the plugin code is currently not done by Spark. The user
-or admin should make sure that the jar files are available to Spark applications, for example, by
-including the plugin jar with the Spark distribution. The exception to this rule is the YARN
-backend, where the <code>--jars</code> command line option (or equivalent config entry) can be
-used to make the plugin code available to both executors and cluster-mode drivers.

From f03bca87eb1ffbcb1e891c088f76c87a162df711 Mon Sep 17 00:00:00 2001
From: Nathan Wreggit <obbay2@hotmail.com>
Date: Thu, 29 Oct 2020 10:28:50 +0900
Subject: [PATCH 1137/1280] [SQL][MINOR] Update from_unixtime doc

This PR fixes from_unixtime documentation to show that fmt is optional parameter.

Yes, documentation update.
**Before change:**
![image](https://user-images.githubusercontent.com/4176173/97497659-18c6cc80-1928-11eb-93d8-453ef627ac7c.png)

**After change:**
![image](https://user-images.githubusercontent.com/4176173/97496153-c5537f00-1925-11eb-8102-457e85e019d5.png)

Style check using: ./dev/run-tests
Manual check and screenshotting with: ./sql/create-docs.sh
Manual verification of behavior with latest spark-sql binary.

Closes #30176 from Obbay2/from_unixtime_doc.

Authored-by: Nathan Wreggit <obbay2@hotmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/expressions/datetimeExpressions.scala     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index b2ff2fa609b99..9d6705822d551 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -937,17 +937,20 @@ abstract class UnixTime extends ToTimestamp {
  */
 // scalastyle:off line.size.limit
 @ExpressionDescription(
-  usage = "_FUNC_(unix_time, format) - Returns `unix_time` in the specified `format`.",
+  usage = "_FUNC_(unix_time[, format]) - Returns `unix_time` in the specified `format`.",
   arguments = """
     Arguments:
       * unix_time - UNIX Timestamp to be converted to the provided format.
       * format - Date/time format pattern to follow. See <a href="https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html">Datetime Patterns</a>
-                 for valid date and time format patterns.
+                 for valid date and time format patterns. The 'yyyy-MM-dd HH:mm:ss' pattern is used if omitted.
   """,
   examples = """
     Examples:
       > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss');
        1969-12-31 16:00:00
+
+      > SELECT _FUNC_(0);
+       1969-12-31 16:00:00
   """,
   group = "datetime_funcs",
   since = "1.5.0")

From 563a678bc276fd6ead1f7858e0e37373133f4ac2 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 29 Oct 2020 19:10:01 -0700
Subject: [PATCH 1138/1280] [SPARK-33292][SQL] Make Literal ArrayBasedMapData
 string representation disambiguous

### What changes were proposed in this pull request?

This PR aims to wrap `ArrayBasedMapData` literal representation with `map(...)`.

### Why are the changes needed?

Literal ArrayBasedMapData has inconsistent string representation from `LogicalPlan` to `Optimized Logical Plan/Physical Plan`. Also, the representation at `Optimized Logical Plan` and `Physical Plan` is ambiguous like `[1 AS a#0, keys: [key1], values: [value1] AS b#1]`.

**BEFORE**
```scala
scala> spark.version
res0: String = 2.4.7

scala> sql("SELECT 1 a, map('key1', 'value1') b").explain(true)
== Parsed Logical Plan ==
'Project [1 AS a#0, 'map(key1, value1) AS b#1]
+- OneRowRelation

== Analyzed Logical Plan ==
a: int, b: map<string,string>
Project [1 AS a#0, map(key1, value1) AS b#1]
+- OneRowRelation

== Optimized Logical Plan ==
Project [1 AS a#0, keys: [key1], values: [value1] AS b#1]
+- OneRowRelation

== Physical Plan ==
*(1) Project [1 AS a#0, keys: [key1], values: [value1] AS b#1]
+- Scan OneRowRelation[]
```

**AFTER**
```scala
scala> spark.version
res0: String = 3.1.0-SNAPSHOT

scala> sql("SELECT 1 a, map('key1', 'value1') b").explain(true)
== Parsed Logical Plan ==
'Project [1 AS a#4, 'map(key1, value1) AS b#5]
+- OneRowRelation

== Analyzed Logical Plan ==
a: int, b: map<string,string>
Project [1 AS a#4, map(key1, value1) AS b#5]
+- OneRowRelation

== Optimized Logical Plan ==
Project [1 AS a#4, map(keys: [key1], values: [value1]) AS b#5]
+- OneRowRelation

== Physical Plan ==
*(1) Project [1 AS a#4, map(keys: [key1], values: [value1]) AS b#5]
+- *(1) Scan OneRowRelation[]
```

### Does this PR introduce _any_ user-facing change?

Yes. This changes the query plan's string representation in `explain` command and UI. However, this is a bug fix.

### How was this patch tested?

Pass the CI with the newly added test case.

Closes #30190 from dongjoon-hyun/SPARK-33292.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 838791bf0b8290143001fe8f94b1fbbd53a181d2)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../org/apache/spark/sql/catalyst/expressions/literals.scala     | 1 +
 .../spark/sql/catalyst/expressions/LiteralExpressionSuite.scala  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 213a58a3244e2..9e96ab8a9b6ca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -297,6 +297,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
   override def toString: String = value match {
     case null => "null"
     case binary: Array[Byte] => s"0x" + DatatypeConverter.printHexBinary(binary)
+    case d: ArrayBasedMapData => s"map(${d.toString})"
     case other => other.toString
   }
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index 4714635a3370b..bb86135021b91 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -209,6 +209,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
     }
     checkMapLiteral(Map("a" -> 1, "b" -> 2, "c" -> 3))
     checkMapLiteral(Map("1" -> 1.0, "2" -> 2.0, "3" -> 3.0))
+    assert(Literal.create(Map("a" -> 1)).toString === "map(keys: [a], values: [1])")
   }
 
   test("struct") {

From 8f5760345fc71fdcc3c3c3a16e7e426471075429 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 29 Oct 2020 19:41:55 -0700
Subject: [PATCH 1139/1280] [SPARK-33268][SQL][PYTHON][3.0] Fix bugs for
 casting data from/to PythonUserDefinedType

### What changes were proposed in this pull request?

This PR intends to fix bugs for casting data from/to PythonUserDefinedType. A sequence of queries to reproduce this issue is as follows;
```
>>> from pyspark.sql import Row
>>> from pyspark.sql.functions import col
>>> from pyspark.sql.types import *
>>> from pyspark.testing.sqlutils import *
>>>
>>> row = Row(point=ExamplePoint(1.0, 2.0))
>>> df = spark.createDataFrame([row])
>>> df.select(col("point").cast(PythonOnlyUDT()))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/maropu/Repositories/spark/spark-master/python/pyspark/sql/dataframe.py", line 1402, in select
    jdf = self._jdf.select(self._jcols(*cols))
  File "/Users/maropu/Repositories/spark/spark-master/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1305, in __call__
  File "/Users/maropu/Repositories/spark/spark-master/python/pyspark/sql/utils.py", line 111, in deco
    return f(*a, **kw)
  File "/Users/maropu/Repositories/spark/spark-master/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py", line 328, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling o44.select.
: java.lang.NullPointerException
	at org.apache.spark.sql.types.UserDefinedType.acceptsType(UserDefinedType.scala:84)
	at org.apache.spark.sql.catalyst.expressions.Cast$.canCast(Cast.scala:96)
	at org.apache.spark.sql.catalyst.expressions.CastBase.checkInputDataTypes(Cast.scala:267)
	at org.apache.spark.sql.catalyst.expressions.CastBase.resolved$lzycompute(Cast.scala:290)
	at org.apache.spark.sql.catalyst.expressions.CastBase.resolved(Cast.scala:290)
```
A root cause of this issue is that, since `PythonUserDefinedType#userClassis` always null, `isAssignableFrom` in `UserDefinedType#acceptsType` throws a null exception. To fix it, this PR defines  `acceptsType` in `PythonUserDefinedType` and filters out the null case in `UserDefinedType#acceptsType`.

This backport comes from https://github.com/apache/spark/pull/30169.

### Why are the changes needed?

Bug fixes.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests.

Closes #30191 from maropu/SPARK-33268-BRANCH3.0.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 python/pyspark/sql/tests/test_types.py                   | 9 +++++++++
 .../org/apache/spark/sql/types/UserDefinedType.scala     | 9 +++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 016cafd669019..786564a59936e 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -26,6 +26,7 @@
 
 from pyspark.sql import Row
 from pyspark.sql.functions import col, UserDefinedFunction
+from pyspark.sql.utils import AnalysisException
 from pyspark.sql.types import *
 from pyspark.sql.types import _array_signed_int_typecode_ctype_mappings, _array_type_mappings, \
     _array_unsigned_int_typecode_ctype_mappings, _infer_type, _make_type_verifier, _merge_type
@@ -437,6 +438,14 @@ def test_cast_to_string_with_udt(self):
         result = df.select(col('point').cast('string'), col('pypoint').cast('string')).head()
         self.assertEqual(result, Row(point=u'(1.0, 2.0)', pypoint=u'[3.0, 4.0]'))
 
+    def test_cast_to_udt_with_udt(self):
+        from pyspark.sql.functions import col
+        row = Row(point=ExamplePoint(1.0, 2.0), python_only_point=PythonOnlyPoint(1.0, 2.0))
+        df = self.spark.createDataFrame([row])
+        self.assertRaises(AnalysisException, lambda: df.select(col("point").cast(PythonOnlyUDT())))
+        self.assertRaises(AnalysisException,
+                          lambda: df.select(col("python_only_point").cast(ExamplePointUDT())))
+
     def test_struct_type(self):
         struct1 = StructType().add("f1", StringType(), True).add("f2", StringType(), True, None)
         struct2 = StructType([StructField("f1", StringType(), True),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
index 592ce03606d4b..689c30f6b7fa9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/UserDefinedType.scala
@@ -78,8 +78,8 @@ abstract class UserDefinedType[UserType >: Null] extends DataType with Serializa
    */
   override private[spark] def asNullable: UserDefinedType[UserType] = this
 
-  override private[sql] def acceptsType(dataType: DataType) = dataType match {
-    case other: UserDefinedType[_] =>
+  override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
+    case other: UserDefinedType[_] if this.userClass != null && other.userClass != null =>
       this.getClass == other.getClass ||
         this.userClass.isAssignableFrom(other.userClass)
     case _ => false
@@ -131,6 +131,11 @@ private[sql] class PythonUserDefinedType(
       ("sqlType" -> sqlType.jsonValue)
   }
 
+  override private[sql] def acceptsType(dataType: DataType): Boolean = dataType match {
+    case other: PythonUserDefinedType => pyUDT == other.pyUDT
+    case _ => false
+  }
+
   override def equals(other: Any): Boolean = other match {
     case that: PythonUserDefinedType => pyUDT == that.pyUDT
     case _ => false

From 83f259fb96c2d86d827c41efd3510cea024901ff Mon Sep 17 00:00:00 2001
From: allisonwang-db <66282705+allisonwang-db@users.noreply.github.com>
Date: Fri, 30 Oct 2020 08:21:08 +0000
Subject: [PATCH 1140/1280] [SPARK-33183][SQL][3.0] Fix Optimizer rule
 EliminateSorts and add a physical rule to remove redundant sorts

Backport #30093 for branch-3.0. I've updated the configuration version to 2.4.8.

### What changes were proposed in this pull request?
This PR aims to fix a correctness bug in the optimizer rule EliminateSorts. It also adds a new physical rule to remove redundant sorts that cannot be eliminated in the Optimizer rule after the bugfix.

### Why are the changes needed?
A global sort should not be eliminated even if its child is ordered since we don't know if its child ordering is global or local. For example, in the following scenario, the first sort shouldn't be removed because it has a stronger guarantee than the second sort even if the sort orders are the same for both sorts.
```
Sort(orders, global = True, ...)
  Sort(orders, global = False, ...)
```
Since there is no straightforward way to identify whether a node's output ordering is local or global, we should not remove a global sort even if its child is already ordered.

### Does this PR introduce any user-facing change?
Yes

### How was this patch tested?
Unit tests

Closes #30195 from allisonwang-db/SPARK-33183-branch-3.0.

Authored-by: allisonwang-db <66282705+allisonwang-db@users.noreply.github.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/optimizer/Optimizer.scala    |  16 ++-
 .../apache/spark/sql/internal/SQLConf.scala   |   7 ++
 .../optimizer/EliminateSortsSuite.scala       | 102 +++++++++++++++--
 .../spark/sql/execution/QueryExecution.scala  |   1 +
 .../sql/execution/RemoveRedundantSorts.scala  |  46 ++++++++
 .../adaptive/AdaptiveSparkPlanExec.scala      |   2 +
 .../spark/sql/execution/PlannerSuite.scala    |  13 ---
 .../execution/RemoveRedundantSortsSuite.scala | 108 ++++++++++++++++++
 8 files changed, 267 insertions(+), 28 deletions(-)
 create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
 create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index d93c4a5bc459a..4629cbbfa7f51 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -971,7 +971,7 @@ object CombineFilters extends Rule[LogicalPlan] with PredicateHelper {
 /**
  * Removes Sort operation. This can happen:
  * 1) if the sort order is empty or the sort order does not have any reference
- * 2) if the child is already sorted
+ * 2) if the Sort operator is a local sort and the child is already sorted
  * 3) if there is another Sort operator separated by 0...n Project/Filter operators
  * 4) if the Sort operator is within Join separated by 0...n Project/Filter operators only,
  *    and the Join conditions is deterministic
@@ -979,12 +979,18 @@ object CombineFilters extends Rule[LogicalPlan] with PredicateHelper {
  *    and the aggregate function is order irrelevant
  */
 object EliminateSorts extends Rule[LogicalPlan] {
-  def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+  def apply(plan: LogicalPlan): LogicalPlan = plan transform applyLocally
+
+  private val applyLocally: PartialFunction[LogicalPlan, LogicalPlan] = {
     case s @ Sort(orders, _, child) if orders.isEmpty || orders.exists(_.child.foldable) =>
       val newOrders = orders.filterNot(_.child.foldable)
-      if (newOrders.isEmpty) child else s.copy(order = newOrders)
-    case Sort(orders, true, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
-      child
+      if (newOrders.isEmpty) {
+        applyLocally.lift(child).getOrElse(child)
+      } else {
+        s.copy(order = newOrders)
+      }
+    case Sort(orders, false, child) if SortOrder.orderingSatisfies(child.outputOrdering, orders) =>
+      applyLocally.lift(child).getOrElse(child)
     case s @ Sort(_, _, child) => s.copy(child = recursiveRemoveSort(child))
     case j @ Join(originLeft, originRight, _, cond, _) if cond.forall(_.deterministic) =>
       j.copy(left = recursiveRemoveSort(originLeft), right = recursiveRemoveSort(originRight))
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index c74888340ac12..23d1d70bdb22c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -1201,6 +1201,13 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val REMOVE_REDUNDANT_SORTS_ENABLED = buildConf("spark.sql.execution.removeRedundantSorts")
+    .internal()
+    .doc("Whether to remove redundant physical sort node")
+    .version("2.4.8")
+    .booleanConf
+    .createWithDefault(true)
+
   val STATE_STORE_PROVIDER_CLASS =
     buildConf("spark.sql.streaming.stateStore.providerClass")
       .internal()
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
index e2b599a7c090c..e34f141412ca6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/EliminateSortsSuite.scala
@@ -97,12 +97,34 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("remove redundant order by") {
+  test("SPARK-33183: remove consecutive no-op sorts") {
+    val plan = testRelation.orderBy().orderBy().orderBy()
+    val optimized = Optimize.execute(plan.analyze)
+    val correctAnswer = testRelation.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove redundant sort by") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst)
-    val unnecessaryReordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst)
+    val unnecessaryReordered = orderedPlan.limit(2).select('a).sortBy('a.asc, 'b.desc_nullsFirst)
     val optimized = Optimize.execute(unnecessaryReordered.analyze)
     val correctAnswer = orderedPlan.limit(2).select('a).analyze
-    comparePlans(Optimize.execute(optimized), correctAnswer)
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove all redundant local sorts") {
+    val orderedPlan = testRelation.sortBy('a.asc).orderBy('a.asc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.orderBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: should not remove global sort") {
+    val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc_nullsFirst)
+    val reordered = orderedPlan.limit(2).select('a).orderBy('a.asc, 'b.desc_nullsFirst)
+    val optimized = Optimize.execute(reordered.analyze)
+    val correctAnswer = reordered.analyze
+    comparePlans(optimized, correctAnswer)
   }
 
   test("do not remove sort if the order is different") {
@@ -113,22 +135,39 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("filters don't affect order") {
+  test("SPARK-33183: remove top level local sort with filter operators") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
-    val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.where('a > Literal(10)).sortBy('a.asc, 'b.desc)
     val optimized = Optimize.execute(filteredAndReordered.analyze)
     val correctAnswer = orderedPlan.where('a > Literal(10)).analyze
     comparePlans(optimized, correctAnswer)
   }
 
-  test("limits don't affect order") {
+  test("SPARK-33183: keep top level global sort with filter operators") {
+    val projectPlan = testRelation.select('a, 'b)
+    val orderedPlan = projectPlan.orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc)
+    val optimized = Optimize.execute(filteredAndReordered.analyze)
+    val correctAnswer = projectPlan.where('a > Literal(10)).orderBy('a.asc, 'b.desc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: limits should not affect order for local sort") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
-    val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.limit(Literal(10)).sortBy('a.asc, 'b.desc)
     val optimized = Optimize.execute(filteredAndReordered.analyze)
     val correctAnswer = orderedPlan.limit(Literal(10)).analyze
     comparePlans(optimized, correctAnswer)
   }
 
+  test("SPARK-33183: should not remove global sort with limit operators") {
+    val orderedPlan = testRelation.select('a, 'b).orderBy('a.asc, 'b.desc)
+    val filteredAndReordered = orderedPlan.limit(Literal(10)).orderBy('a.asc, 'b.desc)
+    val optimized = Optimize.execute(filteredAndReordered.analyze)
+    val correctAnswer = filteredAndReordered.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("different sorts are not simplified if limit is in between") {
     val orderedPlan = testRelation.select('a, 'b).orderBy('b.desc).limit(Literal(10))
       .orderBy('a.asc)
@@ -137,11 +176,11 @@ class EliminateSortsSuite extends PlanTest {
     comparePlans(optimized, correctAnswer)
   }
 
-  test("range is already sorted") {
+  test("SPARK-33183: should not remove global sort with range operator") {
     val inputPlan = Range(1L, 1000L, 1, 10)
     val orderedPlan = inputPlan.orderBy('id.asc)
     val optimized = Optimize.execute(orderedPlan.analyze)
-    val correctAnswer = inputPlan.analyze
+    val correctAnswer = orderedPlan.analyze
     comparePlans(optimized, correctAnswer)
 
     val reversedPlan = inputPlan.orderBy('id.desc)
@@ -152,10 +191,18 @@ class EliminateSortsSuite extends PlanTest {
     val negativeStepInputPlan = Range(10L, 1L, -1, 10)
     val negativeStepOrderedPlan = negativeStepInputPlan.orderBy('id.desc)
     val negativeStepOptimized = Optimize.execute(negativeStepOrderedPlan.analyze)
-    val negativeStepCorrectAnswer = negativeStepInputPlan.analyze
+    val negativeStepCorrectAnswer = negativeStepOrderedPlan.analyze
     comparePlans(negativeStepOptimized, negativeStepCorrectAnswer)
   }
 
+  test("SPARK-33183: remove local sort with range operator") {
+    val inputPlan = Range(1L, 1000L, 1, 10)
+    val orderedPlan = inputPlan.sortBy('id.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = inputPlan.analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
   test("sort should not be removed when there is a node which doesn't guarantee any order") {
     val orderedPlan = testRelation.select('a, 'b)
     val groupedAndResorted = orderedPlan.groupBy('a)(sum('a)).orderBy('a.asc)
@@ -319,4 +366,39 @@ class EliminateSortsSuite extends PlanTest {
     val correctAnswer = PushDownOptimizer.execute(noOrderByPlan.analyze)
     comparePlans(optimized, correctAnswer)
   }
+
+  test("SPARK-33183: remove consecutive global sorts with the same ordering") {
+    Seq(
+      (testRelation.orderBy('a.asc).orderBy('a.asc), testRelation.orderBy('a.asc)),
+      (testRelation.orderBy('a.asc, 'b.desc).orderBy('a.asc), testRelation.orderBy('a.asc))
+    ).foreach { case (ordered, answer) =>
+      val optimized = Optimize.execute(ordered.analyze)
+      comparePlans(optimized, answer.analyze)
+    }
+  }
+
+  test("SPARK-33183: remove consecutive local sorts with the same ordering") {
+    val orderedPlan = testRelation.sortBy('a.asc).sortBy('a.asc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.sortBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: remove consecutive local sorts with different ordering") {
+    val orderedPlan = testRelation.sortBy('b.asc).sortBy('a.desc).sortBy('a.asc)
+    val optimized = Optimize.execute(orderedPlan.analyze)
+    val correctAnswer = testRelation.sortBy('a.asc).analyze
+    comparePlans(optimized, correctAnswer)
+  }
+
+  test("SPARK-33183: should keep global sort when child is a local sort with the same ordering") {
+    val correctAnswer = testRelation.orderBy('a.asc).analyze
+    Seq(
+      testRelation.sortBy('a.asc).orderBy('a.asc),
+      testRelation.orderBy('a.asc).sortBy('a.asc).orderBy('a.asc)
+    ).foreach { ordered =>
+      val optimized = Optimize.execute(ordered.analyze)
+      comparePlans(optimized, correctAnswer)
+    }
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 50c9c31029d90..574a67fecf936 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -297,6 +297,7 @@ object QueryExecution {
     Seq(
       PlanDynamicPruningFilters(sparkSession),
       PlanSubqueries(sparkSession),
+      RemoveRedundantSorts(sparkSession.sessionState.conf),
       EnsureRequirements(sparkSession.sessionState.conf),
       ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
         sparkSession.sessionState.columnarRules),
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
new file mode 100644
index 0000000000000..71f36c8c1dd5a
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/RemoveRedundantSorts.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.catalyst.expressions.SortOrder
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.internal.SQLConf
+
+/**
+ * Remove redundant SortExec node from the spark plan. A sort node is redundant when
+ * its child satisfies both its sort orders and its required child distribution. Note
+ * this rule differs from the Optimizer rule EliminateSorts in that this rule also checks
+ * if the child satisfies the required distribution so that it is safe to remove not only a
+ * local sort but also a global sort when its child already satisfies required sort orders.
+ */
+case class RemoveRedundantSorts(conf: SQLConf) extends Rule[SparkPlan] {
+  def apply(plan: SparkPlan): SparkPlan = {
+    if (!conf.getConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED)) {
+      plan
+    } else {
+      removeSorts(plan)
+    }
+  }
+
+  private def removeSorts(plan: SparkPlan): SparkPlan = plan transform {
+    case s @ SortExec(orders, _, child, _)
+        if SortOrder.orderingSatisfies(child.outputOrdering, orders) &&
+          child.outputPartitioning.satisfies(s.requiredChildDistribution.head) =>
+      child
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 8b59b1236782e..4e73f06f36f1d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -83,12 +83,14 @@ case class AdaptiveSparkPlanExec(
     )
   }
 
+  @transient private val removeRedundantSorts = RemoveRedundantSorts(conf)
   @transient private val ensureRequirements = EnsureRequirements(conf)
 
   // A list of physical plan rules to be applied before creation of query stages. The physical
   // plan should reach a final status of query stages (i.e., no more addition or removal of
   // Exchange nodes) after running these rules.
   private def queryStagePreparationRules: Seq[Rule[SparkPlan]] = Seq(
+    removeRedundantSorts,
     ensureRequirements
   ) ++ context.session.sessionState.queryStagePrepRules
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
index 9c8e44323ce98..b7be0f1320e61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/PlannerSuite.scala
@@ -234,19 +234,6 @@ class PlannerSuite extends SharedSparkSession with AdaptiveSparkPlanHelper {
     }
   }
 
-  test("SPARK-23375: Cached sorted data doesn't need to be re-sorted") {
-    val query = testData.select('key, 'value).sort('key.desc).cache()
-    assert(query.queryExecution.optimizedPlan.isInstanceOf[InMemoryRelation])
-    val resorted = query.sort('key.desc)
-    assert(resorted.queryExecution.optimizedPlan.collect { case s: Sort => s}.isEmpty)
-    assert(resorted.select('key).collect().map(_.getInt(0)).toSeq ==
-      (1 to 100).reverse)
-    // with a different order, the sort is needed
-    val sortedAsc = query.sort('key)
-    assert(sortedAsc.queryExecution.optimizedPlan.collect { case s: Sort => s}.size == 1)
-    assert(sortedAsc.select('key).collect().map(_.getInt(0)).toSeq == (1 to 100))
-  }
-
   test("PartitioningCollection") {
     withTempView("normal", "small", "tiny") {
       testData.createOrReplaceTempView("normal")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
new file mode 100644
index 0000000000000..1978d22cb87bb
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+
+abstract class RemoveRedundantSortsSuiteBase
+    extends QueryTest
+    with SharedSparkSession
+    with AdaptiveSparkPlanHelper {
+  import testImplicits._
+
+  private def checkNumSorts(df: DataFrame, count: Int): Unit = {
+    val plan = df.queryExecution.executedPlan
+    assert(collectWithSubqueries(plan) { case s: SortExec => s }.length == count)
+  }
+
+  private def checkSorts(query: String, enabledCount: Int, disabledCount: Int): Unit = {
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "true") {
+      val df = sql(query)
+      checkNumSorts(df, enabledCount)
+      val result = df.collect()
+      withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "false") {
+        val df = sql(query)
+        checkNumSorts(df, disabledCount)
+        checkAnswer(df, result)
+      }
+    }
+  }
+
+  test("remove redundant sorts with limit") {
+    withTempView("t") {
+      spark.range(100).select('id as "key").createOrReplaceTempView("t")
+      val query =
+        """
+          |SELECT key FROM
+          | (SELECT key FROM t WHERE key > 10 ORDER BY key DESC LIMIT 10)
+          |ORDER BY key DESC
+          |""".stripMargin
+      checkSorts(query, 0, 1)
+    }
+  }
+
+  test("remove redundant sorts with sort merge join") {
+    withTempView("t1", "t2") {
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t1")
+      spark.range(1000).select('id as "key").createOrReplaceTempView("t2")
+      val query = """
+        |SELECT /*+ MERGE(t1) */ t1.key FROM
+        | (SELECT key FROM t1 WHERE key > 10 ORDER BY key DESC LIMIT 10) t1
+        |JOIN
+        | (SELECT key FROM t2 WHERE key > 50 ORDER BY key DESC LIMIT 100) t2
+        |ON t1.key = t2.key
+        |ORDER BY t1.key
+      """.stripMargin
+
+      val queryAsc = query + " ASC"
+      checkSorts(queryAsc, 2, 3)
+
+      // The top level sort should not be removed since the child output ordering is ASC and
+      // the required ordering is DESC.
+      val queryDesc = query + " DESC"
+      checkSorts(queryDesc, 3, 3)
+    }
+  }
+
+  test("cached sorted data doesn't need to be re-sorted") {
+    withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "true") {
+      val df = spark.range(1000).select('id as "key").sort('key.desc).cache()
+      val resorted = df.sort('key.desc)
+      val sortedAsc = df.sort('key.asc)
+      checkNumSorts(df, 0)
+      checkNumSorts(resorted, 0)
+      checkNumSorts(sortedAsc, 1)
+      val result = resorted.collect()
+      withSQLConf(SQLConf.REMOVE_REDUNDANT_SORTS_ENABLED.key -> "false") {
+        val resorted = df.sort('key.desc)
+        checkNumSorts(resorted, 1)
+        checkAnswer(resorted, result)
+      }
+    }
+  }
+}
+
+class RemoveRedundantSortsSuite extends RemoveRedundantSortsSuiteBase
+  with DisableAdaptiveExecutionSuite
+
+class RemoveRedundantSortsSuiteAE extends RemoveRedundantSortsSuiteBase
+  with EnableAdaptiveExecutionSuite

From fc105310d57f8186680320f3d460638c3a1fbb70 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apple.com>
Date: Sat, 31 Oct 2020 09:49:18 -0700
Subject: [PATCH 1141/1280] [SPARK-33290][SQL] REFRESH TABLE should invalidate
 cache even though the table itself may not be cached

### What changes were proposed in this pull request?

In `CatalogImpl.refreshTable`, this moves the `uncacheQuery` call out of the condition `if (cache.nonEmpty)` so that it will be called whether the table itself is cached or not.

### Why are the changes needed?

In the case like the following:
```sql
CREATE TABLE t ...;
CREATE VIEW t1 AS SELECT * FROM t;
REFRESH TABLE t;
```

If the table `t` is refreshed, the view `t1` which is depending on `t` will not be invalidated. This could lead to incorrect result and is similar to [SPARK-19765](https://issues.apache.org/jira/browse/SPARK-19765).

On the other hand, if we have:

```sql
CREATE TABLE t ...;
CACHE TABLE t;
CREATE VIEW t1 AS SELECT * FROM t;
REFRESH TABLE t;
```

Then the view `t1` will be refreshed. The behavior is somewhat inconsistent.

### Does this PR introduce _any_ user-facing change?

Yes, with the change any cache that are depending on the table refreshed will be invalidated with the change. Previously this only happens if the table itself is cached.

### How was this patch tested?

Added a new UT for the case.

Closes #30187 from sunchao/SPARK-33290.

Authored-by: Chao Sun <sunchao@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 32b78d3795d5c4fd533b0267647977ed4f02ee49)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/internal/CatalogImpl.scala      | 12 ++++--
 .../apache/spark/sql/CachedTableSuite.scala   | 42 +++++++++++++++++++
 2 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
index 7ca9fbb40d9f5..7c76168b0dd9b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala
@@ -464,6 +464,9 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
    * If this table is cached as an InMemoryRelation, drop the original cached version and make the
    * new version cached lazily.
    *
+   * In addition, refreshing a table also invalidate all caches that have reference to the table
+   * in a cascading manner. This is to prevent incorrect result from the otherwise staled caches.
+   *
    * @group cachemgmt
    * @since 2.0.0
    */
@@ -484,14 +487,17 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
     // If this table is cached as an InMemoryRelation, drop the original
     // cached version and make the new version cached lazily.
     val cache = sparkSession.sharedState.cacheManager.lookupCachedData(table)
+
+    // uncache the logical plan.
+    // note this is a no-op for the table itself if it's not cached, but will invalidate all
+    // caches referencing this table.
+    sparkSession.sharedState.cacheManager.uncacheQuery(table, cascade = true)
+
     if (cache.nonEmpty) {
       // save the cache name and cache level for recreation
       val cacheName = cache.get.cachedRepresentation.cacheBuilder.tableName
       val cacheLevel = cache.get.cachedRepresentation.cacheBuilder.storageLevel
 
-      // uncache the logical plan.
-      sparkSession.sharedState.cacheManager.uncacheQuery(table, cascade = true)
-
       // recache with the same name and cache level.
       sparkSession.sharedState.cacheManager.cacheQuery(table, cacheName, cacheLevel)
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index adc725ed9b062..6313370476c93 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1208,4 +1208,46 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       assert(spark.sharedState.cacheManager.lookupCachedData(df).isDefined)
     }
   }
+
+  test("SPARK-33290: REFRESH TABLE should invalidate all caches referencing the table") {
+    withTable("t") {
+      withTempPath { path =>
+        withTempView("tempView1", "tempView2") {
+          Seq((1 -> "a")).toDF("i", "j").write.parquet(path.getCanonicalPath)
+          sql(s"CREATE TABLE t USING parquet LOCATION '${path.toURI}'")
+          sql("CREATE TEMPORARY VIEW tempView1 AS SELECT * FROM t")
+          sql("CACHE TABLE tempView2 AS SELECT i FROM tempView1")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq(Row(1, "a")))
+          checkAnswer(sql("SELECT * FROM tempView2"), Seq(Row(1)))
+
+          Utils.deleteRecursively(path)
+          sql("REFRESH TABLE tempView1")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq.empty)
+          checkAnswer(sql("SELECT * FROM tempView2"), Seq.empty)
+        }
+      }
+    }
+  }
+
+  test("SPARK-33290: querying temporary view after REFRESH TABLE fails with FNFE") {
+    withTable("t") {
+      withTempPath { path =>
+        withTempView("tempView1") {
+          Seq((1 -> "a")).toDF("i", "j").write.parquet(path.getCanonicalPath)
+          sql(s"CREATE TABLE t USING parquet LOCATION '${path.toURI}'")
+          sql("CREATE TEMPORARY VIEW tempView1 AS SELECT * FROM t")
+          checkAnswer(sql("SELECT * FROM tempView1"), Seq(Row(1, "a")))
+
+          Utils.deleteRecursively(path)
+          sql("REFRESH TABLE t")
+          checkAnswer(sql("SELECT * FROM t"), Seq.empty)
+          val exception = intercept[Exception] {
+            checkAnswer(sql("SELECT * FROM tempView1"), Seq.empty)
+          }
+          assert(exception.getMessage.contains("FileNotFoundException"))
+          assert(exception.getMessage.contains("REFRESH TABLE"))
+        }
+      }
+    }
+  }
 }

From 49e9575674b42190629d71eb8114318634625091 Mon Sep 17 00:00:00 2001
From: "wangguangxin.cn" <wangguangxin.cn@gmail.com>
Date: Sat, 31 Oct 2020 15:14:46 -0700
Subject: [PATCH 1142/1280] [SPARK-33306][SQL] Timezone is needed when cast
 date to string

### What changes were proposed in this pull request?
When `spark.sql.legacy.typeCoercion.datetimeToString.enabled` is enabled, spark will cast date to string when compare date with string. In Spark3, timezone is needed when casting date to string as https://github.com/apache/spark/blob/72ad9dcd5d484a8dd64c08889de85ef9de2a6077/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala#L309.

Howerver, the timezone may not be set because `CastBase.needsTimeZone` returns false for this kind of casting.

A simple way to reproduce this is
```
spark-shell --conf spark.sql.legacy.typeCoercion.datetimeToString.enabled=true

```
when we execute the following sql,
```
select a.d1 from
(select to_date(concat('2000-01-0', id)) as d1 from range(1, 2)) a
join
(select concat('2000-01-0', id) as d2 from range(1, 2)) b
on a.d1 = b.d2
```
it will throw
```
java.util.NoSuchElementException: None.get
  at scala.None$.get(Option.scala:529)
  at scala.None$.get(Option.scala:527)
  at org.apache.spark.sql.catalyst.expressions.TimeZoneAwareExpression.zoneId(datetimeExpressions.scala:56)
  at org.apache.spark.sql.catalyst.expressions.TimeZoneAwareExpression.zoneId$(datetimeExpressions.scala:56)
  at org.apache.spark.sql.catalyst.expressions.CastBase.zoneId$lzycompute(Cast.scala:253)
  at org.apache.spark.sql.catalyst.expressions.CastBase.zoneId(Cast.scala:253)
  at org.apache.spark.sql.catalyst.expressions.CastBase.dateFormatter$lzycompute(Cast.scala:287)
  at org.apache.spark.sql.catalyst.expressions.CastBase.dateFormatter(Cast.scala:287)
```

### Why are the changes needed?
As described above, it's a bug here.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Add more UT

Closes #30213 from WangGuangxin/SPARK-33306.

Authored-by: wangguangxin.cn <wangguangxin.cn@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 69c27f49acf2fe6fbc8335bde2aac4afd4188678)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/expressions/Cast.scala     |  1 +
 .../org/apache/spark/sql/SQLQuerySuite.scala      | 15 +++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index c8985d4a12129..388fb231f3ce3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -108,6 +108,7 @@ object Cast {
    */
   def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match {
     case (StringType, TimestampType | DateType) => true
+    case (DateType, StringType) => true
     case (DateType, TimestampType) => true
     case (TimestampType, StringType) => true
     case (TimestampType, DateType) => true
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 979ae882726d9..8f278f9e51b20 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3508,6 +3508,21 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       checkAnswer(sql("SELECT id FROM t WHERE (SELECT true)"), Row(0L))
     }
   }
+
+  test("SPARK-33306: Timezone is needed when cast Date to String") {
+    withTempView("t1", "t2") {
+      spark.sql("select to_date(concat('2000-01-0', id)) as d from range(1, 2)")
+        .createOrReplaceTempView("t1")
+      spark.sql("select concat('2000-01-0', id) as d from range(1, 2)")
+        .createOrReplaceTempView("t2")
+      val result = Date.valueOf("2000-01-01")
+
+      checkAnswer(sql("select t1.d from t1 join t2 on t1.d = t2.d"), Row(result))
+      withSQLConf(SQLConf.LEGACY_CAST_DATETIME_TO_STRING.key -> "true") {
+        checkAnswer(sql("select t1.d from t1 join t2 on t1.d = t2.d"), Row(result))
+      }
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From 92ba08d42d7317db58eb6fefe1e117d0ad84c78a Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Mon, 2 Nov 2020 09:06:58 +0900
Subject: [PATCH 1143/1280] [SPARK-33277][PYSPARK][SQL][3.0] Use
 ContextAwareIterator to stop consuming after the task ends

### What changes were proposed in this pull request?

This is a backport of #30177.

As the Python evaluation consumes the parent iterator in a separate thread, it could consume more data from the parent even after the task ends and the parent is closed. Thus, we should use `ContextAwareIterator` to stop consuming after the task ends.

### Why are the changes needed?

Python/Pandas UDF right after off-heap vectorized reader could cause executor crash.

E.g.,:

```py
spark.range(0, 100000, 1, 1).write.parquet(path)

spark.conf.set("spark.sql.columnVector.offheap.enabled", True)

def f(x):
    return 0

fUdf = udf(f, LongType())

spark.read.parquet(path).select(fUdf('id')).head()
```

This is because, the Python evaluation consumes the parent iterator in a separate thread and it consumes more data from the parent even after the task ends and the parent is closed. If an off-heap column vector exists in the parent iterator, it could cause segmentation fault which crashes the executor.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests, and manually.

Closes #30217 from ueshin/issues/SPARK-33277/3.0/python_pandas_udf.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/sql/tests/test_pandas_map.py   | 22 +++++++++++++++++++
 .../sql/tests/test_pandas_udf_scalar.py       | 19 ++++++++++++++++
 python/pyspark/sql/tests/test_udf.py          | 20 +++++++++++++++++
 .../sql/execution/python/EvalPythonExec.scala | 18 ++++++++++++++-
 .../execution/python/MapInPandasExec.scala    |  7 +++---
 5 files changed, 82 insertions(+), 4 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_map.py b/python/pyspark/sql/tests/test_pandas_map.py
index f1956a2523e48..3f82dfada2812 100644
--- a/python/pyspark/sql/tests/test_pandas_map.py
+++ b/python/pyspark/sql/tests/test_pandas_map.py
@@ -16,12 +16,15 @@
 #
 import os
 import sys
+import shutil
+import tempfile
 import time
 import unittest
 
 if sys.version >= '3':
     unicode = str
 
+from pyspark.sql import Row
 from pyspark.sql.functions import pandas_udf, PandasUDFType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
@@ -117,6 +120,25 @@ def func(iterator):
         expected = df.collect()
         self.assertEquals(actual, expected)
 
+    # SPARK-33277
+    def test_map_in_pandas_with_column_vector(self):
+        path = tempfile.mkdtemp()
+        shutil.rmtree(path)
+
+        try:
+            self.spark.range(0, 200000, 1, 1).write.parquet(path)
+
+            def func(iterator):
+                for pdf in iterator:
+                    yield pd.DataFrame({'id': [0] * len(pdf)})
+
+            for offheap in ["true", "false"]:
+                with self.sql_conf({"spark.sql.columnVector.offheap.enabled": offheap}):
+                    self.assertEquals(
+                        self.spark.read.parquet(path).mapInPandas(func, 'id long').head(), Row(0))
+        finally:
+            shutil.rmtree(path)
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_map import *
diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index 7260e80e2cfca..375bb2676d03b 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -1133,6 +1133,25 @@ def test_datasource_with_udf(self):
         finally:
             shutil.rmtree(path)
 
+    # SPARK-33277
+    def test_pandas_udf_with_column_vector(self):
+        path = tempfile.mkdtemp()
+        shutil.rmtree(path)
+
+        try:
+            self.spark.range(0, 200000, 1, 1).write.parquet(path)
+
+            @pandas_udf(LongType())
+            def udf(x):
+                return pd.Series([0] * len(x))
+
+            for offheap in ["true", "false"]:
+                with self.sql_conf({"spark.sql.columnVector.offheap.enabled": offheap}):
+                    self.assertEquals(
+                        self.spark.read.parquet(path).select(udf('id')).head(), Row(0))
+        finally:
+            shutil.rmtree(path)
+
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_udf_scalar import *
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index ea7ec9f1e7c0c..c6e66825007fd 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -670,6 +670,26 @@ def f(*a):
         r = df.select(fUdf(*df.columns))
         self.assertEqual(r.first()[0], "success")
 
+    # SPARK-33277
+    def test_udf_with_column_vector(self):
+        path = tempfile.mkdtemp()
+        shutil.rmtree(path)
+
+        try:
+            self.spark.range(0, 100000, 1, 1).write.parquet(path)
+
+            def f(x):
+                return 0
+
+            fUdf = udf(f, LongType())
+
+            for offheap in ["true", "false"]:
+                with self.sql_conf({"spark.sql.columnVector.offheap.enabled": offheap}):
+                    self.assertEquals(
+                        self.spark.read.parquet(path).select(fUdf('id')).head(), Row(0))
+        finally:
+            shutil.rmtree(path)
+
 
 class UDFInitializationTests(unittest.TestCase):
     def tearDown(self):
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
index a0f23e925d237..d49df0fc5c263 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
@@ -88,6 +88,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
 
     inputRDD.mapPartitions { iter =>
       val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(iter, context)
 
       // The queue used to buffer input rows so we can drain it to
       // combine input with output from Python.
@@ -119,7 +120,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
       })
 
       // Add rows to queue to join later with the result.
-      val projectedRowIter = iter.map { inputRow =>
+      val projectedRowIter = contextAwareIterator.map { inputRow =>
         queue.add(inputRow.asInstanceOf[UnsafeRow])
         projection(inputRow)
       }
@@ -136,3 +137,18 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
     }
   }
 }
+
+/**
+ * A TaskContext aware iterator.
+ *
+ * As the Python evaluation consumes the parent iterator in a separate thread,
+ * it could consume more data from the parent even after the task ends and the parent is closed.
+ * Thus, we should use ContextAwareIterator to stop consuming after the task ends.
+ */
+class ContextAwareIterator[IN](iter: Iterator[IN], context: TaskContext) extends Iterator[IN] {
+
+  override def hasNext: Boolean =
+    !context.isCompleted() && !context.isInterrupted() && iter.hasNext
+
+  override def next(): IN = iter.next()
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
index 2bb808119c0ae..7fc18f885a2d3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
@@ -61,16 +61,17 @@ case class MapInPandasExec(
       val pythonRunnerConf = ArrowUtils.getPythonRunnerConfMap(conf)
       val outputTypes = child.schema
 
+      val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(inputIter, context)
+
       // Here we wrap it via another row so that Python sides understand it
       // as a DataFrame.
-      val wrappedIter = inputIter.map(InternalRow(_))
+      val wrappedIter = contextAwareIterator.map(InternalRow(_))
 
       // DO NOT use iter.grouped(). See BatchIterator.
       val batchIter =
         if (batchSize > 0) new BatchIterator(wrappedIter, batchSize) else Iterator(wrappedIter)
 
-      val context = TaskContext.get()
-
       val columnarBatchIter = new ArrowPythonRunner(
         chainedFunc,
         PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,

From 131179a799c6e162b804d7e997a564ca21464120 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 2 Nov 2020 11:10:39 +0900
Subject: [PATCH 1144/1280] [SPARK-33313][TESTS][R][3.0][2.4] Add testthat 3.x
 support

### What changes were proposed in this pull request?

This PR proposes to port back https://github.com/apache/spark/pull/30219 but keeps testthat 1.x support in other branches.

This PR modifies `R/pkg/tests/run-all.R` by:

- ~Removing `testthat` 1.x support, as Jenkins has been upgraded to 2.x with SPARK-30637 and this code is no longer relevant.~
- Add `testthat` 3.x support to avoid CI failures.

### Why are the changes needed?

Currently used internal API has been removed in the latest `testthat` release.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Tests executed against `testthat == 2.3.2` and `testthat == 3.0.0`

Closes #30220 from HyukjinKwon/SPARK-33313.

Lead-authored-by: HyukjinKwon <gurwls223@apache.org>
Co-authored-by: zero323 <mszymkiewicz@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 R/pkg/tests/run-all.R | 48 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R
index 20e61b2dda3ac..0a6a6c4ea3d1d 100644
--- a/R/pkg/tests/run-all.R
+++ b/R/pkg/tests/run-all.R
@@ -60,25 +60,49 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) {
     # set random seed for predictable results. mostly for base's sample() in tree and classification
     set.seed(42)
 
-    # TODO (SPARK-30663) To be removed once testthat 1.x is removed from all builds
-    if (packageVersion("testthat")$major <= 1) {
+    test_runner <- if (packageVersion("testthat")$major <= 1) {
       # testthat 1.x
-      test_runner <- testthat:::run_tests
-      reporter <- "summary"
+      function(path, package, reporter, filter) {
+        testthat:::run_tests(
+          test_path = path,
+          package = package,
+          filter = filter,
+          reporter = reporter
+        )
+      }
+    } else if (packageVersion("testthat")$major == 2) {
+      # testthat >= 2.0.0, < 3.0.0
+      function(path, package, reporter, filter) {
+        testthat:::test_package_dir(
+          test_path = path,
+          package = package,
+          filter = filter,
+          reporter = reporter
+        )
+      }
+    } else {
+      # testthat >= 3.0.0
+      testthat::test_dir
+    }
+
+    reporter <- if (packageVersion("testthat")$major <= 1) {
+      "summary"
     } else {
-      # testthat >= 2.0.0
-      test_runner <- testthat:::test_package_dir
       dir.create("target/test-reports", showWarnings = FALSE)
-      reporter <- MultiReporter$new(list(
+      MultiReporter$new(list(
         SummaryReporter$new(),
-        JunitReporter$new(file = "target/test-reports/test-results.xml")
+        JunitReporter$new(
+          file = file.path(getwd(), "target/test-reports/test-results.xml")
+        )
       ))
     }
 
-    test_runner("SparkR",
-                file.path(sparkRDir, "pkg", "tests", "fulltests"),
-                NULL,
-                reporter)
+    test_runner(
+      path = file.path(sparkRDir, "pkg", "tests", "fulltests"),
+      package = "SparkR",
+      reporter = reporter,
+      filter = NULL
+    )
   }
 
   SparkR:::uninstallDownloadedSpark()

From 71ef48e29e644f529b9d1c51ceb00c7cb91e18f0 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Mon, 2 Nov 2020 21:53:57 -0800
Subject: [PATCH 1145/1280] [SPARK-33156][INFRA][3.0] Upgrade GithubAction
 image from 18.04 to 20.04

### What changes were proposed in this pull request?

This PR aims to upgrade `Github Action` runner image from `Ubuntu 18.04 (LTS)` to `Ubuntu 20.04 (LTS)`.

### Why are the changes needed?

`ubuntu-latest` in `GitHub Action` is still `Ubuntu 18.04 (LTS)`.
- https://github.com/actions/virtual-environments#available-environments

This upgrade will prepare AmbLab Jenkins upgrade.
### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the `Github Action` in this PR.

Closes #30231 from dongjoon-hyun/SPARK-33156-3.0.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/build_and_test.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 6afa41b9e949c..e4c1e84cf53d9 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -12,7 +12,8 @@ jobs:
   # Build: build Spark and run the tests for specified modules.
   build:
     name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})"
-    runs-on: ubuntu-latest
+    # Ubuntu 20.04 is the latest LTS. The next LTS is 22.04.
+    runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
@@ -196,7 +197,7 @@ jobs:
   # Static analysis, and documentation build
   lint:
     name: Linters, licenses, dependencies and documentation generation
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
@@ -258,7 +259,7 @@ jobs:
 
   java11:
     name: Java 11 build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2

From d99ff20cb00e204a9781812cb3fe3465b4f3a20e Mon Sep 17 00:00:00 2001
From: Stijn De Haes <stijndehaes@gmail.com>
Date: Mon, 2 Nov 2020 23:42:51 -0800
Subject: [PATCH 1146/1280] [SPARK-24266][K8S][3.0] Restart the watcher when we
 receive a version changed from k8s

### What changes were proposed in this pull request?

This is a straight application of #28423 onto branch-3.0

Restart the watcher when it failed with a HTTP_GONE code from the kubernetes api. Which means a resource version has changed.

For more relevant information see here: https://github.com/fabric8io/kubernetes-client/issues/1075

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

This was tested in #28423 by running spark-submit to a k8s cluster.

Closes #29533 from jkleckner/backport-SPARK-24266-to-branch-3.0.

Authored-by: Stijn De Haes <stijndehaes@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../submit/KubernetesClientApplication.scala  | 52 ++++++++++++-------
 .../k8s/submit/LoggingPodStatusWatcher.scala  | 35 ++++++++++---
 .../spark/deploy/k8s/submit/ClientSuite.scala |  1 +
 3 files changed, 61 insertions(+), 27 deletions(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
index 8e5532d70b5da..b5ad444867864 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala
@@ -21,8 +21,10 @@ import java.util.{Collections, UUID}
 import java.util.Properties
 
 import io.fabric8.kubernetes.api.model._
-import io.fabric8.kubernetes.client.KubernetesClient
+import io.fabric8.kubernetes.client.{KubernetesClient, Watch}
+import io.fabric8.kubernetes.client.Watcher.Action
 import scala.collection.mutable
+import scala.util.control.Breaks._
 import scala.util.control.NonFatal
 
 import org.apache.spark.SparkConf
@@ -122,25 +124,37 @@ private[spark] class Client(
         .endSpec()
       .build()
     val driverPodName = resolvedDriverPod.getMetadata.getName
-    Utils.tryWithResource(
-      kubernetesClient
-        .pods()
-        .withName(driverPodName)
-        .watch(watcher)) { _ =>
-      val createdDriverPod = kubernetesClient.pods().create(resolvedDriverPod)
-      try {
-        val otherKubernetesResources =
-          resolvedDriverSpec.driverKubernetesResources ++ Seq(configMap)
-        addDriverOwnerReference(createdDriverPod, otherKubernetesResources)
-        kubernetesClient.resourceList(otherKubernetesResources: _*).createOrReplace()
-      } catch {
-        case NonFatal(e) =>
-          kubernetesClient.pods().delete(createdDriverPod)
-          throw e
-      }
 
-      val sId = Seq(conf.namespace, driverPodName).mkString(":")
-      watcher.watchOrStop(sId)
+    var watch: Watch = null
+    val createdDriverPod = kubernetesClient.pods().create(resolvedDriverPod)
+    try {
+      val otherKubernetesResources = resolvedDriverSpec.driverKubernetesResources ++ Seq(configMap)
+      addDriverOwnerReference(createdDriverPod, otherKubernetesResources)
+      kubernetesClient.resourceList(otherKubernetesResources: _*).createOrReplace()
+    } catch {
+      case NonFatal(e) =>
+        kubernetesClient.pods().delete(createdDriverPod)
+        throw e
+    }
+    val sId = Seq(conf.namespace, driverPodName).mkString(":")
+    breakable {
+      while (true) {
+        val podWithName = kubernetesClient
+          .pods()
+          .withName(driverPodName)
+        // Reset resource to old before we start the watch, this is important for race conditions
+        watcher.reset()
+        watch = podWithName.watch(watcher)
+
+        // Send the latest pod state we know to the watcher to make sure we didn't miss anything
+        watcher.eventReceived(Action.MODIFIED, podWithName.get())
+
+        // Break the while loop if the pod is completed or we don't want to wait
+        if(watcher.watchOrStop(sId)) {
+          watch.close()
+          break
+        }
+      }
     }
   }
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
index ce3c80c0f85b1..aa27a9ef508ca 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.k8s.submit
 import io.fabric8.kubernetes.api.model.Pod
 import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
 import io.fabric8.kubernetes.client.Watcher.Action
+import java.net.HttpURLConnection.HTTP_GONE
 
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.deploy.k8s.KubernetesDriverConf
@@ -26,7 +27,8 @@ import org.apache.spark.deploy.k8s.KubernetesUtils._
 import org.apache.spark.internal.Logging
 
 private[k8s] trait LoggingPodStatusWatcher extends Watcher[Pod] {
-  def watchOrStop(submissionId: String): Unit
+  def watchOrStop(submissionId: String): Boolean
+  def reset(): Unit
 }
 
 /**
@@ -42,10 +44,16 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
 
   private var podCompleted = false
 
+  private var resourceTooOldReceived = false
+
   private var pod = Option.empty[Pod]
 
   private def phase: String = pod.map(_.getStatus.getPhase).getOrElse("unknown")
 
+  override def reset(): Unit = {
+    resourceTooOldReceived = false
+  }
+
   override def eventReceived(action: Action, pod: Pod): Unit = {
     this.pod = Option(pod)
     action match {
@@ -62,7 +70,12 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
 
   override def onClose(e: KubernetesClientException): Unit = {
     logDebug(s"Stopping watching application $appId with last-observed phase $phase")
-    closeWatch()
+    if(e != null && e.getCode == HTTP_GONE) {
+      resourceTooOldReceived = true
+      logDebug(s"Got HTTP Gone code, resource version changed in k8s api: $e")
+    } else {
+      closeWatch()
+    }
   }
 
   private def logLongStatus(): Unit = {
@@ -78,20 +91,26 @@ private[k8s] class LoggingPodStatusWatcherImpl(conf: KubernetesDriverConf)
     this.notifyAll()
   }
 
-  override def watchOrStop(sId: String): Unit = if (conf.get(WAIT_FOR_APP_COMPLETION)) {
+  override def watchOrStop(sId: String): Boolean = if (conf.get(WAIT_FOR_APP_COMPLETION)) {
     logInfo(s"Waiting for application ${conf.appName} with submission ID $sId to finish...")
     val interval = conf.get(REPORT_INTERVAL)
     synchronized {
-      while (!podCompleted) {
+      while (!podCompleted && !resourceTooOldReceived) {
         wait(interval)
         logInfo(s"Application status for $appId (phase: $phase)")
       }
     }
-    logInfo(
-      pod.map { p => s"Container final statuses:\n\n${containersDescription(p)}" }
-        .getOrElse("No containers were found in the driver pod."))
-    logInfo(s"Application ${conf.appName} with submission ID $sId finished")
+
+    if(podCompleted) {
+      logInfo(
+        pod.map { p => s"Container final statuses:\n\n${containersDescription(p)}" }
+          .getOrElse("No containers were found in the driver pod."))
+      logInfo(s"Application ${conf.appName} with submission ID $sId finished")
+    }
+    podCompleted
   } else {
     logInfo(s"Deployed Spark application ${conf.appName} with submission ID $sId into Kubernetes")
+    // Always act like the application has completed since we don't want to wait for app completion
+    true
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
index 5d49ac0bbaafa..d9ec3feb526ee 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala
@@ -136,6 +136,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter {
     createdResourcesArgumentCaptor = ArgumentCaptor.forClass(classOf[HasMetadata])
     when(podOperations.create(FULL_EXPECTED_POD)).thenReturn(POD_WITH_OWNER_REFERENCE)
     when(namedPods.watch(loggingPodStatusWatcher)).thenReturn(mock[Watch])
+    when(loggingPodStatusWatcher.watchOrStop(kconf.namespace + ":" + POD_NAME)).thenReturn(true)
     doReturn(resourceList)
       .when(kubernetesClient)
       .resourceList(createdResourcesArgumentCaptor.capture())

From 55105a0784459331d5506eee9f37c2e655a2a6a0 Mon Sep 17 00:00:00 2001
From: neko <echohlne@gmail.com>
Date: Tue, 3 Nov 2020 08:49:52 -0600
Subject: [PATCH 1147/1280] [SPARK-33284][WEB-UI] In the Storage UI page,
 clicking any field to sort the table will cause the header content to be lost

### What changes were proposed in this pull request?
In the old version of spark in the storage UI page, the sorting function is normal, but sorting in the new version will cause the header content to be lost, So I try to fix the bug.

### Why are the changes needed?

The header field of the table on the page is similar to the following, **note that each th contains the span attribute**:

```html
<thead>
    <tr>
        ....
        <th width="" class="">
              <span data-toggle="tooltip" title="" data-original-title="StorageLevel displays where the persisted RDD is stored, format of the persisted RDD (serialized or de-serialized) andreplication factor of the persisted RDD">
                Storage Level
              </span>
        </th>
       .....
    </tr>
</thead>
```

Since  [PR#26136](https://github.com/apache/spark/pull/26136), if the `th` in the table itself contains the `span` attribute, the `span` will be deleted directly after clicking the sort, and the original header content will be lost.

There are three problems  in `sorttable.js`:

1. `sortrevind.class = "sorttable_sortrevind"` in  [sorttab.js#107](https://github.com/apache/spark/blob/9d5e48ea95d1c3017a51ff69584f32a18901b2b5/core/src/main/resources/org/apache/spark/ui/static/sorttable.js#L107) and  `sortfwdind.class = "sorttable_sortfwdind"` in  [sorttab.js#125](https://github.com/apache/spark/blob/9d5e48ea95d1c3017a51ff69584f32a18901b2b5/core/src/main/resources/org/apache/spark/ui/static/sorttable.js#L125)
sorttable_xx attribute should be assigned to`className` instead of `class`, as javascript uses `rowlists[j].className.search` rather than `rowlists[j].class.search` to determine whether the component has a sorting flag or not.
2.  `rowlists[j].className.search(/\sorttable_sortrevind\b/)` in  [sorttab.js#120](https://github.com/apache/spark/blob/9d5e48ea95d1c3017a51ff69584f32a18901b2b5/core/src/main/resources/org/apache/spark/ui/static/sorttable.js#L120) was wrong. The original intention is to search whether `className` contains  the word `sorttable_sortrevind` , but the expression is wrong,  it should be `\bsorttable_sortrevind\b` instead of `\sorttable_sortrevind\b`
3. The if check statement in the following code snippet ([sorttab.js#141](https://github.com/apache/spark/blob/9d5e48ea95d1c3017a51ff69584f32a18901b2b5/core/src/main/resources/org/apache/spark/ui/static/sorttable.js#L141)) was wrong. **If the `search` function does not find the target, it will return -1, but Boolean(-1) is actually equals true**. This statement will cause span to be deleted even if it does not contain `sorttable_sortfwdind` or `sorttable_sortrevind`.
```javascript
rowlists = this.parentNode.getElementsByTagName("span");
for (var j=0; j < rowlists.length; j++) {
              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)
                  || rowlists[j].className.search(/\sorttable_sortrevind\b/) ) {
                  rowlists[j].parentNode.removeChild(rowlists[j]);
              }
          }
```

### Does this PR introduce _any_ user-facing change?
NO.

### How was this patch tested?
The manual test result of the ui page is as below:

![fix sorted](https://user-images.githubusercontent.com/52202080/97543194-daeaa680-1a02-11eb-8b11-8109c3e4e9a3.gif)

Closes #30182 from akiyamaneko/ui_storage_sort_error.

Authored-by: neko <echohlne@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 56c623e98c54fdb4d47c9264ae1b282ecb2b7291)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../org/apache/spark/ui/static/sorttable.js        | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/sorttable.js b/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
index ecd580e5c64aa..3f98a0379dc3c 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/sorttable.js
@@ -99,12 +99,12 @@ sorttable = {
                                                     'sorttable_sorted_reverse');
             rowlists = this.parentNode.getElementsByTagName("span");
             for (var j=0; j < rowlists.length; j++) {
-                if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)) {
+                if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/) != -1) {
                     rowlists[j].parentNode.removeChild(rowlists[j]);
                 }
             }
             sortrevind = document.createElement('span');
-            sortrevind.class = "sorttable_sortrevind";
+            sortrevind.className = "sorttable_sortrevind";
             sortrevind.innerHTML = stIsIE ? '&nbsp<font face="webdings">5</font>' : '&nbsp;&#x25BE;';
             this.appendChild(sortrevind);
             return;
@@ -117,12 +117,12 @@ sorttable = {
                                                   'sorttable_sorted');
             rowlists = this.parentNode.getElementsByTagName("span");
             for (var j=0; j < rowlists.length; j++) {
-                if (rowlists[j].className.search(/\sorttable_sortrevind\b/)) {
+                if (rowlists[j].className.search(/\bsorttable_sortrevind\b/) != -1) {
                     rowlists[j].parentNode.removeChild(rowlists[j]);
                 }
             }
             sortfwdind = document.createElement('span');
-            sortfwdind.class = "sorttable_sortfwdind";
+            sortfwdind.className = "sorttable_sortfwdind";
             sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
             this.appendChild(sortfwdind);
             return;
@@ -138,15 +138,15 @@ sorttable = {
           });
           rowlists = this.parentNode.getElementsByTagName("span");
           for (var j=0; j < rowlists.length; j++) {
-              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/)
-                  || rowlists[j].className.search(/\sorttable_sortrevind\b/) ) {
+              if (rowlists[j].className.search(/\bsorttable_sortfwdind\b/) != -1
+                  || rowlists[j].className.search(/\bsorttable_sortrevind\b/) != -1) {
                   rowlists[j].parentNode.removeChild(rowlists[j]);
               }
           }
 
           this.className += ' sorttable_sorted';
           sortfwdind = document.createElement('span');
-          sortfwdind.class = "sorttable_sortfwdind";
+          sortfwdind.className = "sorttable_sortfwdind";
           sortfwdind.innerHTML = stIsIE ? '&nbsp<font face="webdings">6</font>' : '&nbsp;&#x25B4;';
           this.appendChild(sortfwdind);
 

From 5dd36f3195fa2906e96fc2c08b2b836fc81197fe Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Tue, 3 Nov 2020 22:30:34 -0800
Subject: [PATCH 1148/1280] [SPARK-33333][BUILD][3.0] Upgrade Jetty to
 9.4.28.v20200408

### What changes were proposed in this pull request?

This PR aims to upgrade Jetty to 9.4.28.v20200408 in `branch-3.0` for `Apache Spark 3.0.2` like `master` branch `Apache Spark 3.1`.

### Why are the changes needed?

To bring the bug fixes.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CIs.

Closes #30240 from dongjoon-hyun/SPARK-33333.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 32a13082bac91..de3b69db18fbb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@
     <orc.classifier></orc.classifier>
     <hive.parquet.group>com.twitter</hive.parquet.group>
     <hive.parquet.version>1.6.0</hive.parquet.version>
-    <jetty.version>9.4.18.v20190429</jetty.version>
+    <jetty.version>9.4.28.v20200408</jetty.version>
     <javaxservlet.version>3.1.0</javaxservlet.version>
     <chill.version>0.9.5</chill.version>
     <ivy.version>2.4.0</ivy.version>

From e7a62116029149e9c601c6345934ad8bd7ff8e82 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 4 Nov 2020 08:35:10 -0800
Subject: [PATCH 1149/1280] [SPARK-33338][SQL] GROUP BY using literal map
 should not fail

### What changes were proposed in this pull request?

This PR aims to fix `semanticEquals` works correctly on `GetMapValue` expressions having literal maps with `ArrayBasedMapData` and `GenericArrayData`.

### Why are the changes needed?

This is a regression from Apache Spark 1.6.x.
```scala
scala> sc.version
res1: String = 1.6.3

scala> sqlContext.sql("SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k]").show
+---+
|_c0|
+---+
| v1|
+---+
```

Apache Spark 2.x ~ 3.0.1 raise`RuntimeException` for the following queries.
```sql
CREATE TABLE t USING ORC AS SELECT map('k1', 'v1') m, 'k1' k
SELECT map('k1', 'v1')[k] FROM t GROUP BY 1
SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k]
SELECT map('k1', 'v1')[k] a FROM t GROUP BY a
```

**BEFORE**
```scala
Caused by: java.lang.RuntimeException: Couldn't find k#3 in [keys: [k1], values: [v1][k#3]#6]
	at scala.sys.package$.error(package.scala:27)
	at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1$$anonfun$applyOrElse$1.apply(BoundAttribute.scala:85)
	at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1$$anonfun$applyOrElse$1.apply(BoundAttribute.scala:79)
	at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
```

**AFTER**
```sql
spark-sql> SELECT map('k1', 'v1')[k] FROM t GROUP BY 1;
v1
Time taken: 1.278 seconds, Fetched 1 row(s)
spark-sql> SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k];
v1
Time taken: 0.313 seconds, Fetched 1 row(s)
spark-sql> SELECT map('k1', 'v1')[k] a FROM t GROUP BY a;
v1
Time taken: 0.265 seconds, Fetched 1 row(s)
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CIs with the newly added test case.

Closes #30246 from dongjoon-hyun/SPARK-33338.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 42c0b175ce6ee4bf1104b6a8cef6bb6477693781)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/catalyst/expressions/literals.scala |  2 ++
 .../catalyst/expressions/ComplexTypeSuite.scala   | 15 +++++++++++++++
 .../org/apache/spark/sql/SQLQuerySuite.scala      | 12 ++++++++++++
 3 files changed, 29 insertions(+)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 9e96ab8a9b6ca..413d0af61a05c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -316,6 +316,8 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
       (value, o.value) match {
         case (null, null) => true
         case (a: Array[Byte], b: Array[Byte]) => util.Arrays.equals(a, b)
+        case (a: ArrayBasedMapData, b: ArrayBasedMapData) =>
+          a.keyArray == b.keyArray && a.valueArray == b.valueArray
         case (a, b) => a != null && a.equals(b)
       }
     case _ => false
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index dbe43709d1d35..54eb07b51e9f3 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -22,6 +22,7 @@ import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, UnresolvedExtractValue}
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
+import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
@@ -466,4 +467,18 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
     CreateNamedStruct(Seq("a", "x", "b", 2.0)).genCode(ctx)
     assert(ctx.inlinedMutableStates.isEmpty)
   }
+
+  test("SPARK-33338: semanticEquals should handle static GetMapValue correctly") {
+    val keys = new Array[UTF8String](1)
+    val values = new Array[UTF8String](1)
+    keys(0) = UTF8String.fromString("key")
+    values(0) = UTF8String.fromString("value")
+
+    val d1 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values))
+    val d2 = new ArrayBasedMapData(new GenericArrayData(keys), new GenericArrayData(values))
+    val m1 = GetMapValue(Literal.create(d1, MapType(StringType, StringType)), Literal("a"))
+    val m2 = GetMapValue(Literal.create(d2, MapType(StringType, StringType)), Literal("a"))
+
+    assert(m1.semanticEquals(m2))
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 8f278f9e51b20..bcd3e79dcc054 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3523,6 +3523,18 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       }
     }
   }
+
+  test("SPARK-33338: GROUP BY using literal map should not fail") {
+    withTempDir { dir =>
+      sql(s"CREATE TABLE t USING ORC LOCATION '${dir.toURI}' AS SELECT map('k1', 'v1') m, 'k1' k")
+      Seq(
+        "SELECT map('k1', 'v1')[k] FROM t GROUP BY 1",
+        "SELECT map('k1', 'v1')[k] FROM t GROUP BY map('k1', 'v1')[k]",
+        "SELECT map('k1', 'v1')[k] a FROM t GROUP BY a").foreach { statement =>
+        checkAnswer(sql(statement), Row("v1"))
+      }
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From b43572e73b2078cb8c554f023b12c89b0b97bda5 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 4 Nov 2020 20:38:22 -0800
Subject: [PATCH 1150/1280] [SPARK-33162][INFRA][3.0] Use pre-built image at
 GitHub Action PySpark jobs

### What changes were proposed in this pull request?

This is a backport of https://github.com/apache/spark/pull/30059 .

This PR aims to use `pre-built image` at Github Action PySpark jobs. To isolate the changes, `pyspark` jobs are split from the main job. The docker image is built by the following.

| Item                   | URL                |
| --------------- | ------------- |
| Dockerfile         | https://github.com/dongjoon-hyun/ApacheSparkGitHubActionImage/blob/main/Dockerfile |
| Builder               | https://github.com/dongjoon-hyun/ApacheSparkGitHubActionImage/blob/main/.github/workflows/build.yml |
| Image Location | https://hub.docker.com/r/dongjoon/apache-spark-github-action-image |

Please note that.
1. The community still will use `build_and_test.yml` to add new features like as we did until now. The `Dockerfile` will be updated regularly.
2. When Apache Spark gets an official docker repository location, we will use it.
3. Also, it's the best if we keep this docker file and builder script at a new Apache Spark dev branch instead of outside GitHub repository.

### Why are the changes needed?

This will reduce the Python and its package installation time.

**BEFORE (branch-3.0)**
![Screen Shot 2020-11-04 at 2 28 49 PM](https://user-images.githubusercontent.com/9700541/98174664-17f2e500-1eaa-11eb-9222-018eead9c418.png)

**AFTER (branch-3.0)**
![Screen Shot 2020-11-04 at 2 29 43 PM](https://user-images.githubusercontent.com/9700541/98174758-378a0d80-1eaa-11eb-8e6a-929158c2fea3.png)

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the GitHub Action on this PR without `package installation steps`.

Closes #30253 from dongjoon-hyun/GHA-3.0.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/build_and_test.yml | 112 ++++++++++++++++++++-------
 1 file changed, 82 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index e4c1e84cf53d9..7956d9e37b441 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -37,10 +37,6 @@ jobs:
             streaming, sql-kafka-0-10, streaming-kafka-0-10,
             mllib-local, mllib,
             yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl
-          - >-
-            pyspark-sql, pyspark-mllib
-          - >-
-            pyspark-core, pyspark-streaming, pyspark-ml
           - >-
             sparkr
         # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
@@ -121,41 +117,17 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: ${{ matrix.java }}
-    # PySpark
-    - name: Install PyPy3
-      # Note that order of Python installations here matters because default python3 is
-      # overridden by pypy3.
-      uses: actions/setup-python@v2
-      if: contains(matrix.modules, 'pyspark')
-      with:
-        python-version: pypy3
-        architecture: x64
-    - name: Install Python 2.7
-      uses: actions/setup-python@v2
-      if: contains(matrix.modules, 'pyspark')
-      with:
-        python-version: 2.7
-        architecture: x64
     - name: Install Python 3.8
       uses: actions/setup-python@v2
       # We should install one Python that is higher then 3+ for SQL and Yarn because:
       # - SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
       # - Yarn has a Python specific test too, for example, YarnClusterSuite.
-      if: contains(matrix.modules, 'yarn') || contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      if: contains(matrix.modules, 'yarn') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
       with:
         python-version: 3.8
         architecture: x64
-    - name: Install Python packages (Python 2.7 and PyPy3)
-      if: contains(matrix.modules, 'pyspark')
-      # PyArrow is not supported in PyPy yet, see ARROW-2651.
-      run: |
-        python2.7 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
-        python2.7 -m pip list
-        # PyPy does not have xmlrunner
-        pypy3 -m pip install numpy pandas scipy
-        pypy3 -m pip list
     - name: Install Python packages (Python 3.8)
-      if: contains(matrix.modules, 'pyspark') || (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
+      if: (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-'))
       run: |
         python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
         python3.8 -m pip list
@@ -194,6 +166,86 @@ jobs:
         name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }}
         path: "**/target/unit-tests.log"
 
+  pyspark:
+    name: "Build modules: ${{ matrix.modules }}"
+    runs-on: ubuntu-20.04
+    container:
+       image: dongjoon/apache-spark-github-action-image:20201025
+    strategy:
+      fail-fast: false
+      matrix:
+        modules:
+          - >-
+            pyspark-sql, pyspark-mllib
+          - >-
+            pyspark-core, pyspark-streaming, pyspark-ml
+    env:
+      MODULES_TO_TEST: ${{ matrix.modules }}
+      HADOOP_PROFILE: hadoop2.7
+      HIVE_PROFILE: hive2.3
+      # GitHub Actions' default miniconda to use in pip packaging test.
+      CONDA_PREFIX: /usr/share/miniconda
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: pyspark-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          pyspark-maven-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: pyspark-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          pyspark-ivy-
+    - name: Install Python 2.7
+      uses: actions/setup-python@v2
+      with:
+        python-version: 2.7
+        architecture: x64
+    - name: Install Python packages (Python 2.7 )
+      run: |
+        python2.7 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
+        python2.7 -m pip list
+    # Run the tests.
+    - name: Run tests
+      run: |
+        mkdir -p ~/.m2
+        ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST"
+        rm -rf ~/.m2/repository/org/apache/spark
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-${{ matrix.modules }}--1.8-hadoop2.7-hive2.3
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-${{ matrix.modules }}--1.8-hadoop2.7-hive2.3
+        path: "**/target/unit-tests.log"
+
   # Static analysis, and documentation build
   lint:
     name: Linters, licenses, dependencies and documentation generation

From 14eb8b164df5fdb3715b7212ba3f5b2e88ec7c53 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 4 Nov 2020 21:56:21 -0800
Subject: [PATCH 1151/1280] [SPARK-33239][INFRA][3.0] Use pre-built image at
 GitHub Action SparkR job

### What changes were proposed in this pull request?

This is a backport of https://github.com/apache/spark/pull/30066 .

This PR aims to use a pre-built image for Github Action SparkR job.

### Why are the changes needed?

This will reduce the execution time and the flakiness.

**BEFORE (branch-3.0: 21 minutes 7 seconds)**
![Screen Shot 2020-11-04 at 8 53 50 PM](https://user-images.githubusercontent.com/9700541/98199386-e39a1b80-1edf-11eb-8dec-c6819ebb3f0d.png)

**AFTER**
No R and R package installation steps.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the GitHub Action `sparkr` job in this PR.

Closes #30258 from dongjoon-hyun/SPARK-33239-3.0.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/build_and_test.yml | 79 ++++++++++++++++++++++------
 1 file changed, 63 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 7956d9e37b441..9b4f41ac9945b 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -37,8 +37,6 @@ jobs:
             streaming, sql-kafka-0-10, streaming-kafka-0-10,
             mllib-local, mllib,
             yarn, mesos, kubernetes, hadoop-cloud, spark-ganglia-lgpl
-          - >-
-            sparkr
         # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
         included-tags: [""]
         # Some tests are disabled in GitHun Actions. Ideally, we should remove this tag
@@ -131,20 +129,6 @@ jobs:
       run: |
         python3.8 -m pip install numpy 'pyarrow<3.0.0' pandas scipy xmlrunner
         python3.8 -m pip list
-    # SparkR
-    - name: Install R 4.0
-      uses: r-lib/actions/setup-r@v1
-      if: contains(matrix.modules, 'sparkr')
-      with:
-        r-version: 4.0
-    - name: Install R packages
-      if: contains(matrix.modules, 'sparkr')
-      run: |
-        # qpdf is required to reduce the size of PDFs to make CRAN check pass. See SPARK-32497.
-        sudo apt-get install -y libcurl4-openssl-dev qpdf
-        sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')"
-        # Show installed packages in R.
-        sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]'
     # Run the tests.
     - name: Run tests
       run: |
@@ -246,6 +230,69 @@ jobs:
         name: unit-tests-log-${{ matrix.modules }}--1.8-hadoop2.7-hive2.3
         path: "**/target/unit-tests.log"
 
+  sparkr:
+    name: Build modules - sparkr
+    runs-on: ubuntu-20.04
+    container:
+       image: dongjoon/apache-spark-github-action-image:20201025
+    env:
+      HADOOP_PROFILE: hadoop2.7
+      HIVE_PROFILE: hive2.3
+      GITHUB_PREV_SHA: ${{ github.event.before }}
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@v2
+      # In order to fetch changed files
+      with:
+        fetch-depth: 0
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: sparkr-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          sparkr-maven-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: sparkr-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          sparkr-ivy-
+    - name: Run tests
+      run: |
+        mkdir -p ~/.m2
+        # The followings are also used by `r-lib/actions/setup-r` to avoid
+        # R issues at docker environment
+        export TZ=UTC
+        export _R_CHECK_SYSTEM_CLOCK_=FALSE
+        ./dev/run-tests --parallelism 2 --modules sparkr
+        rm -rf ~/.m2/repository/org/apache/spark
+    - name: Upload test results to report
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: test-results-sparkr--1.8-hadoop2.7-hive2.3
+        path: "**/target/test-reports/*.xml"
+    - name: Upload unit tests log files
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+        name: unit-tests-log-sparkr--1.8-hadoop2.7-hive2.3
+        path: "**/target/unit-tests.log"
+
   # Static analysis, and documentation build
   lint:
     name: Linters, licenses, dependencies and documentation generation

From 74d8eacbe9cdc0b25a177543eb48ac54bd065cbb Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 5 Nov 2020 16:15:44 +0900
Subject: [PATCH 1152/1280] Revert "[SPARK-33277][PYSPARK][SQL][3.0] Use
 ContextAwareIterator to stop consuming after the task ends"

This reverts commit 92ba08d42d7317db58eb6fefe1e117d0ad84c78a.
---
 python/pyspark/sql/tests/test_pandas_map.py   | 22 -------------------
 .../sql/tests/test_pandas_udf_scalar.py       | 19 ----------------
 python/pyspark/sql/tests/test_udf.py          | 20 -----------------
 .../sql/execution/python/EvalPythonExec.scala | 18 +--------------
 .../execution/python/MapInPandasExec.scala    |  7 +++---
 5 files changed, 4 insertions(+), 82 deletions(-)

diff --git a/python/pyspark/sql/tests/test_pandas_map.py b/python/pyspark/sql/tests/test_pandas_map.py
index 3f82dfada2812..f1956a2523e48 100644
--- a/python/pyspark/sql/tests/test_pandas_map.py
+++ b/python/pyspark/sql/tests/test_pandas_map.py
@@ -16,15 +16,12 @@
 #
 import os
 import sys
-import shutil
-import tempfile
 import time
 import unittest
 
 if sys.version >= '3':
     unicode = str
 
-from pyspark.sql import Row
 from pyspark.sql.functions import pandas_udf, PandasUDFType
 from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
     pandas_requirement_message, pyarrow_requirement_message
@@ -120,25 +117,6 @@ def func(iterator):
         expected = df.collect()
         self.assertEquals(actual, expected)
 
-    # SPARK-33277
-    def test_map_in_pandas_with_column_vector(self):
-        path = tempfile.mkdtemp()
-        shutil.rmtree(path)
-
-        try:
-            self.spark.range(0, 200000, 1, 1).write.parquet(path)
-
-            def func(iterator):
-                for pdf in iterator:
-                    yield pd.DataFrame({'id': [0] * len(pdf)})
-
-            for offheap in ["true", "false"]:
-                with self.sql_conf({"spark.sql.columnVector.offheap.enabled": offheap}):
-                    self.assertEquals(
-                        self.spark.read.parquet(path).mapInPandas(func, 'id long').head(), Row(0))
-        finally:
-            shutil.rmtree(path)
-
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_map import *
diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index 375bb2676d03b..7260e80e2cfca 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -1133,25 +1133,6 @@ def test_datasource_with_udf(self):
         finally:
             shutil.rmtree(path)
 
-    # SPARK-33277
-    def test_pandas_udf_with_column_vector(self):
-        path = tempfile.mkdtemp()
-        shutil.rmtree(path)
-
-        try:
-            self.spark.range(0, 200000, 1, 1).write.parquet(path)
-
-            @pandas_udf(LongType())
-            def udf(x):
-                return pd.Series([0] * len(x))
-
-            for offheap in ["true", "false"]:
-                with self.sql_conf({"spark.sql.columnVector.offheap.enabled": offheap}):
-                    self.assertEquals(
-                        self.spark.read.parquet(path).select(udf('id')).head(), Row(0))
-        finally:
-            shutil.rmtree(path)
-
 
 if __name__ == "__main__":
     from pyspark.sql.tests.test_pandas_udf_scalar import *
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index c6e66825007fd..ea7ec9f1e7c0c 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -670,26 +670,6 @@ def f(*a):
         r = df.select(fUdf(*df.columns))
         self.assertEqual(r.first()[0], "success")
 
-    # SPARK-33277
-    def test_udf_with_column_vector(self):
-        path = tempfile.mkdtemp()
-        shutil.rmtree(path)
-
-        try:
-            self.spark.range(0, 100000, 1, 1).write.parquet(path)
-
-            def f(x):
-                return 0
-
-            fUdf = udf(f, LongType())
-
-            for offheap in ["true", "false"]:
-                with self.sql_conf({"spark.sql.columnVector.offheap.enabled": offheap}):
-                    self.assertEquals(
-                        self.spark.read.parquet(path).select(fUdf('id')).head(), Row(0))
-        finally:
-            shutil.rmtree(path)
-
 
 class UDFInitializationTests(unittest.TestCase):
     def tearDown(self):
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
index d49df0fc5c263..a0f23e925d237 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
@@ -88,7 +88,6 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
 
     inputRDD.mapPartitions { iter =>
       val context = TaskContext.get()
-      val contextAwareIterator = new ContextAwareIterator(iter, context)
 
       // The queue used to buffer input rows so we can drain it to
       // combine input with output from Python.
@@ -120,7 +119,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
       })
 
       // Add rows to queue to join later with the result.
-      val projectedRowIter = contextAwareIterator.map { inputRow =>
+      val projectedRowIter = iter.map { inputRow =>
         queue.add(inputRow.asInstanceOf[UnsafeRow])
         projection(inputRow)
       }
@@ -137,18 +136,3 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
     }
   }
 }
-
-/**
- * A TaskContext aware iterator.
- *
- * As the Python evaluation consumes the parent iterator in a separate thread,
- * it could consume more data from the parent even after the task ends and the parent is closed.
- * Thus, we should use ContextAwareIterator to stop consuming after the task ends.
- */
-class ContextAwareIterator[IN](iter: Iterator[IN], context: TaskContext) extends Iterator[IN] {
-
-  override def hasNext: Boolean =
-    !context.isCompleted() && !context.isInterrupted() && iter.hasNext
-
-  override def next(): IN = iter.next()
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
index 7fc18f885a2d3..2bb808119c0ae 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
@@ -61,17 +61,16 @@ case class MapInPandasExec(
       val pythonRunnerConf = ArrowUtils.getPythonRunnerConfMap(conf)
       val outputTypes = child.schema
 
-      val context = TaskContext.get()
-      val contextAwareIterator = new ContextAwareIterator(inputIter, context)
-
       // Here we wrap it via another row so that Python sides understand it
       // as a DataFrame.
-      val wrappedIter = contextAwareIterator.map(InternalRow(_))
+      val wrappedIter = inputIter.map(InternalRow(_))
 
       // DO NOT use iter.grouped(). See BatchIterator.
       val batchIter =
         if (batchSize > 0) new BatchIterator(wrappedIter, batchSize) else Iterator(wrappedIter)
 
+      val context = TaskContext.get()
+
       val columnarBatchIter = new ArrowPythonRunner(
         chainedFunc,
         PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,

From c43231c6cd2d38fb9e6c6435ca08b735c9aacb61 Mon Sep 17 00:00:00 2001
From: Sarvesh Dave <sarveshdave1@gmail.com>
Date: Thu, 5 Nov 2020 16:22:31 +0900
Subject: [PATCH 1153/1280] [MINOR][SS][DOCS] Update join type in stream static
 joins code examples

### What changes were proposed in this pull request?
Update join type in stream static joins code examples in structured streaming programming guide.
1) Scala, Java and Python examples have a common issue.
    The join keyword is "right_join", it should be "left_outer".

    _Reasons:_
    a) This code snippet is an example of "left outer join" as the streaming df is on left and static df is on right. Also, right outer    join between stream df(left) and static df(right) is not supported.
    b) The keyword "right_join/left_join" is unsupported and it should be "right_outer/left_outer".

So, all of these code snippets have been updated to "left_outer".

2) R exmaple is correct, but the example is of "right_outer" with static df (left) and streaming df(right).
It is changed to "left_outer" to make it consistent with other three examples of scala, java and python.

### Why are the changes needed?
To fix the mistake in example code of documentation.

### Does this PR introduce _any_ user-facing change?
Yes, it is a user-facing change (but documentation update only).

**Screenshots 1: Scala/Java/python example (similar issue)**
_Before:_
<img width="941" alt="Screenshot 2020-11-05 at 12 16 09 AM" src="https://user-images.githubusercontent.com/62717942/98155351-19e59400-1efc-11eb-8142-e6a25a5e6497.png">

_After:_
<img width="922" alt="Screenshot 2020-11-05 at 12 17 12 AM" src="https://user-images.githubusercontent.com/62717942/98155503-5d400280-1efc-11eb-96e1-5ba0f3c35c82.png">

**Screenshots 2: R example (Make it consistent with above change)**
_Before:_
<img width="896" alt="Screenshot 2020-11-05 at 12 19 57 AM" src="https://user-images.githubusercontent.com/62717942/98155685-ac863300-1efc-11eb-93bc-b7ca4dd34634.png">

_After:_
<img width="919" alt="Screenshot 2020-11-05 at 12 20 51 AM" src="https://user-images.githubusercontent.com/62717942/98155739-c0ca3000-1efc-11eb-8f95-a7538fa784b7.png">

### How was this patch tested?
The change was tested locally.
1) cd docs/
    SKIP_API=1 jekyll build
2) Verify docs/_site/structured-streaming-programming-guide.html file in browser.

Closes #30252 from sarveshdave1/doc-update-stream-static-joins.

Authored-by: Sarvesh Dave <sarveshdave1@gmail.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit e66201b30bc1f3da7284af14b32e5e6200768dbd)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 docs/structured-streaming-programming-guide.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/structured-streaming-programming-guide.md b/docs/structured-streaming-programming-guide.md
index 69d744d09e1e5..31b1ca9d47c0e 100644
--- a/docs/structured-streaming-programming-guide.md
+++ b/docs/structured-streaming-programming-guide.md
@@ -1115,7 +1115,7 @@ val staticDf = spark.read. ...
 val streamingDf = spark.readStream. ...
 
 streamingDf.join(staticDf, "type")          // inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join")  // right outer join with a static DF  
+streamingDf.join(staticDf, "type", "left_outer")  // left outer join with a static DF
 
 {% endhighlight %}
 
@@ -1126,7 +1126,7 @@ streamingDf.join(staticDf, "type", "right_join")  // right outer join with a sta
 Dataset<Row> staticDf = spark.read(). ...;
 Dataset<Row> streamingDf = spark.readStream(). ...;
 streamingDf.join(staticDf, "type");         // inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join");  // right outer join with a static DF
+streamingDf.join(staticDf, "type", "left_outer");  // left outer join with a static DF
 {% endhighlight %}
 
 
@@ -1137,7 +1137,7 @@ streamingDf.join(staticDf, "type", "right_join");  // right outer join with a st
 staticDf = spark.read. ...
 streamingDf = spark.readStream. ...
 streamingDf.join(staticDf, "type")  # inner equi-join with a static DF
-streamingDf.join(staticDf, "type", "right_join")  # right outer join with a static DF
+streamingDf.join(staticDf, "type", "left_outer")  # left outer join with a static DF
 {% endhighlight %}
 
 </div>
@@ -1149,10 +1149,10 @@ staticDf <- read.df(...)
 streamingDf <- read.stream(...)
 joined <- merge(streamingDf, staticDf, sort = FALSE)  # inner equi-join with a static DF
 joined <- join(
+            streamingDf,
             staticDf,
-            streamingDf, 
             streamingDf$value == staticDf$value,
-            "right_outer")  # right outer join with a static DF
+            "left_outer")  # left outer join with a static DF
 {% endhighlight %}
 
 </div>

From 6da60bfcc22dee9498ea3a27d6a9f3eedbf1a8f2 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Thu, 5 Nov 2020 09:23:41 -0800
Subject: [PATCH 1154/1280] [SPARK-33362][SQL] skipSchemaResolution should
 still require query to be resolved

### What changes were proposed in this pull request?

Fix a small bug in `V2WriteCommand.resolved`. It should always require the `table` and `query` to be resolved.

### Why are the changes needed?

To prevent potential bugs that we skip resolve the input query.

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

a new test

Closes #30265 from cloud-fan/ds-minor-2.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 26ea417b1448d679fdc777705ee2f99f4e741ef3)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  3 ++-
 .../catalyst/plans/logical/v2Commands.scala   | 26 ++++++++++---------
 .../analysis/DataSourceV2AnalysisSuite.scala  |  9 +++++++
 3 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 2c9681b3e8be9..2f30ba3f39e44 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1363,7 +1363,8 @@ class Analyzer(
       case a @ Aggregate(groupingExprs, aggExprs, appendColumns: AppendColumns) =>
         a.mapExpressions(resolveExpressionTopDown(_, appendColumns))
 
-      case o: OverwriteByExpression if !o.outputResolved =>
+      case o: OverwriteByExpression
+          if !(o.table.resolved && o.query.resolved && o.outputResolved) =>
         // do not resolve expression attributes until the query attributes are resolved against the
         // table by ResolveOutputRelation. that rule will alias the attributes to the table's names.
         o
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index d2830a9bd0145..35874a9fc2979 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -35,20 +35,20 @@ trait V2WriteCommand extends Command {
 
   override def children: Seq[LogicalPlan] = Seq(query)
 
-  override lazy val resolved: Boolean = outputResolved
+  override lazy val resolved: Boolean = table.resolved && query.resolved && outputResolved
 
   def outputResolved: Boolean = {
+    assert(table.resolved && query.resolved,
+      "`outputResolved` can only be called when `table` and `query` are both resolved.")
     // If the table doesn't require schema match, we don't need to resolve the output columns.
-    table.skipSchemaResolution || {
-      table.resolved && query.resolved && query.output.size == table.output.size &&
-        query.output.zip(table.output).forall {
-          case (inAttr, outAttr) =>
-            // names and types must match, nullability must be compatible
-            inAttr.name == outAttr.name &&
-              DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outAttr.dataType) &&
-              (outAttr.nullable || !inAttr.nullable)
-        }
-    }
+    table.skipSchemaResolution || (query.output.size == table.output.size &&
+      query.output.zip(table.output).forall {
+        case (inAttr, outAttr) =>
+          // names and types must match, nullability must be compatible
+          inAttr.name == outAttr.name &&
+            DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outAttr.dataType) &&
+            (outAttr.nullable || !inAttr.nullable)
+      })
   }
 }
 
@@ -86,7 +86,9 @@ case class OverwriteByExpression(
     query: LogicalPlan,
     writeOptions: Map[String, String],
     isByName: Boolean) extends V2WriteCommand {
-  override lazy val resolved: Boolean = outputResolved && deleteExpr.resolved
+  override lazy val resolved: Boolean = {
+    table.resolved && query.resolved && outputResolved && deleteExpr.resolved
+  }
 }
 
 object OverwriteByExpression {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
index e466d558db1ef..5aa2b988cf66b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
@@ -230,6 +230,15 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
 
   def byPosition(table: NamedRelation, query: LogicalPlan): LogicalPlan
 
+  test("skipSchemaResolution should still require query to be resolved") {
+    val table = TestRelationAcceptAnySchema(StructType(Seq(
+      StructField("a", FloatType),
+      StructField("b", DoubleType))).toAttributes)
+    val query = UnresolvedRelation(Seq("t"))
+    val parsedPlan = byName(table, query)
+    assertNotResolved(parsedPlan)
+  }
+
   test("byName: basic behavior") {
     val query = TestRelation(table.schema.toAttributes)
 

From 3223e3ec5d80543c30fc2736b6def79644babb9e Mon Sep 17 00:00:00 2001
From: Hannah Amundson <amundson.hannah@heb.com>
Date: Sun, 8 Nov 2020 20:29:24 +0900
Subject: [PATCH 1155/1280] [SPARK-32860][DOCS][SQL] Updating documentation
 about map support in Encoders

### What changes were proposed in this pull request?

Javadocs updated for the encoder to include maps as a collection type

### Why are the changes needed?

The javadocs were not updated with fix SPARK-16706

### Does this PR introduce _any_ user-facing change?

Yes, the javadocs are updated

### How was this patch tested?

sbt was run to ensure it meets scalastyle

Closes #30274 from hannahkamundson/SPARK-32860.

Lead-authored-by: Hannah Amundson <amundson.hannah@heb.com>
Co-authored-by: Hannah <48397717+hannahkamundson@users.noreply.github.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1090b1b00a4aa6168fd5b69f227f28309c42b6fd)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
index 5d31b5bbf12af..24045b5a43a64 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoders.scala
@@ -146,7 +146,7 @@ object Encoders {
    *  - String
    *  - java.math.BigDecimal, java.math.BigInteger
    *  - time related: java.sql.Date, java.sql.Timestamp, java.time.LocalDate, java.time.Instant
-   *  - collection types: only array and java.util.List currently, map support is in progress
+   *  - collection types: array, java.util.List, and map
    *  - nested java bean.
    *
    * @since 1.6.0

From 808dd8fe40564e8f4f25c9faf1897a9efbb2cbe1 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Sun, 8 Nov 2020 21:23:46 -0800
Subject: [PATCH 1156/1280] [SPARK-33371][PYTHON][3.0] Update setup.py and
 tests for Python 3.9

### What changes were proposed in this pull request?

This PR is a backport of https://github.com/apache/spark/pull/30277

This PR proposes to fix PySpark to officially support Python 3.9. The main codes already work. We should just note that we support Python 3.9.

Also, this PR fixes some minor fixes into the test codes.
- `Thread.isAlive` is removed in Python 3.9, and `Thread.is_alive` exists in Python 3.6+, see https://docs.python.org/3/whatsnew/3.9.html#removed
- Fixed `TaskContextTestsWithWorkerReuse.test_barrier_with_python_worker_reuse` and `TaskContextTests.test_barrier` to be less flaky. This becomes more flaky in Python 3.9 for some reasons.

NOTE that PyArrow does not support Python 3.9 yet.

### Why are the changes needed?

To officially support Python 3.9.

### Does this PR introduce _any_ user-facing change?

Yes, it officially supports Python 3.9.

### How was this patch tested?

Manually ran the tests:

```
$  ./run-tests --python-executable=python
Running PySpark tests. Output is in /.../spark/python/unit-tests.log
Will test against the following Python executables: ['python']
Will test the following Python modules: ['pyspark-core', 'pyspark-ml', 'pyspark-mllib', 'pyspark-resource', 'pyspark-sql', 'pyspark-streaming']
python python_implementation is CPython
python version is: Python 3.9.0
Starting test(python): pyspark.ml.tests.test_base
Starting test(python): pyspark.ml.tests.test_evaluation
Starting test(python): pyspark.ml.tests.test_algorithms
Starting test(python): pyspark.ml.tests.test_feature
Finished test(python): pyspark.ml.tests.test_base (12s)
Starting test(python): pyspark.ml.tests.test_image
Finished test(python): pyspark.ml.tests.test_evaluation (15s)
Starting test(python): pyspark.ml.tests.test_linalg
Finished test(python): pyspark.ml.tests.test_feature (25s)
Starting test(python): pyspark.ml.tests.test_param
Finished test(python): pyspark.ml.tests.test_image (17s)
Starting test(python): pyspark.ml.tests.test_persistence
Finished test(python): pyspark.ml.tests.test_param (17s)
Starting test(python): pyspark.ml.tests.test_pipeline
Finished test(python): pyspark.ml.tests.test_linalg (30s)
Starting test(python): pyspark.ml.tests.test_stat
Finished test(python): pyspark.ml.tests.test_pipeline (6s)
Starting test(python): pyspark.ml.tests.test_training_summary
Finished test(python): pyspark.ml.tests.test_stat (12s)
Starting test(python): pyspark.ml.tests.test_tuning
Finished test(python): pyspark.ml.tests.test_algorithms (68s)
Starting test(python): pyspark.ml.tests.test_wrapper
Finished test(python): pyspark.ml.tests.test_persistence (51s)
Starting test(python): pyspark.mllib.tests.test_algorithms
Finished test(python): pyspark.ml.tests.test_training_summary (33s)
Starting test(python): pyspark.mllib.tests.test_feature
Finished test(python): pyspark.ml.tests.test_wrapper (19s)
Starting test(python): pyspark.mllib.tests.test_linalg
Finished test(python): pyspark.mllib.tests.test_feature (26s)
Starting test(python): pyspark.mllib.tests.test_stat
Finished test(python): pyspark.mllib.tests.test_stat (22s)
Starting test(python): pyspark.mllib.tests.test_streaming_algorithms
Finished test(python): pyspark.mllib.tests.test_algorithms (53s)
Starting test(python): pyspark.mllib.tests.test_util
Finished test(python): pyspark.mllib.tests.test_linalg (54s)
Starting test(python): pyspark.sql.tests.test_arrow
Finished test(python): pyspark.sql.tests.test_arrow (0s) ... 61 tests were skipped
Starting test(python): pyspark.sql.tests.test_catalog
Finished test(python): pyspark.mllib.tests.test_util (11s)
Starting test(python): pyspark.sql.tests.test_column
Finished test(python): pyspark.sql.tests.test_catalog (16s)
Starting test(python): pyspark.sql.tests.test_conf
Finished test(python): pyspark.sql.tests.test_column (17s)
Starting test(python): pyspark.sql.tests.test_context
Finished test(python): pyspark.sql.tests.test_context (6s) ... 3 tests were skipped
Starting test(python): pyspark.sql.tests.test_dataframe
Finished test(python): pyspark.sql.tests.test_conf (11s)
Starting test(python): pyspark.sql.tests.test_datasources
Finished test(python): pyspark.sql.tests.test_datasources (19s)
Starting test(python): pyspark.sql.tests.test_functions
Finished test(python): pyspark.sql.tests.test_dataframe (35s) ... 3 tests were skipped
Starting test(python): pyspark.sql.tests.test_group
Finished test(python): pyspark.sql.tests.test_functions (32s)
Starting test(python): pyspark.sql.tests.test_pandas_cogrouped_map
Finished test(python): pyspark.sql.tests.test_pandas_cogrouped_map (1s) ... 15 tests were skipped
Starting test(python): pyspark.sql.tests.test_pandas_grouped_map
Finished test(python): pyspark.sql.tests.test_group (19s)
Starting test(python): pyspark.sql.tests.test_pandas_map
Finished test(python): pyspark.sql.tests.test_pandas_grouped_map (0s) ... 21 tests were skipped
Starting test(python): pyspark.sql.tests.test_pandas_udf
Finished test(python): pyspark.sql.tests.test_pandas_map (0s) ... 6 tests were skipped
Starting test(python): pyspark.sql.tests.test_pandas_udf_grouped_agg
Finished test(python): pyspark.sql.tests.test_pandas_udf (0s) ... 6 tests were skipped
Starting test(python): pyspark.sql.tests.test_pandas_udf_scalar
Finished test(python): pyspark.sql.tests.test_pandas_udf_grouped_agg (0s) ... 13 tests were skipped
Starting test(python): pyspark.sql.tests.test_pandas_udf_typehints
Finished test(python): pyspark.sql.tests.test_pandas_udf_scalar (0s) ... 50 tests were skipped
Starting test(python): pyspark.sql.tests.test_pandas_udf_window
Finished test(python): pyspark.sql.tests.test_pandas_udf_typehints (0s) ... 10 tests were skipped
Starting test(python): pyspark.sql.tests.test_readwriter
Finished test(python): pyspark.sql.tests.test_pandas_udf_window (0s) ... 14 tests were skipped
Starting test(python): pyspark.sql.tests.test_serde
Finished test(python): pyspark.sql.tests.test_serde (19s)
Starting test(python): pyspark.sql.tests.test_session
Finished test(python): pyspark.mllib.tests.test_streaming_algorithms (120s)
Starting test(python): pyspark.sql.tests.test_streaming
Finished test(python): pyspark.sql.tests.test_readwriter (25s)
Starting test(python): pyspark.sql.tests.test_types
Finished test(python): pyspark.ml.tests.test_tuning (208s)
Starting test(python): pyspark.sql.tests.test_udf
Finished test(python): pyspark.sql.tests.test_session (31s)
Starting test(python): pyspark.sql.tests.test_utils
Finished test(python): pyspark.sql.tests.test_streaming (35s)
Starting test(python): pyspark.streaming.tests.test_context
Finished test(python): pyspark.sql.tests.test_types (34s)
Starting test(python): pyspark.streaming.tests.test_dstream
Finished test(python): pyspark.sql.tests.test_utils (14s)
Starting test(python): pyspark.streaming.tests.test_kinesis
Finished test(python): pyspark.streaming.tests.test_kinesis (0s) ... 2 tests were skipped
Starting test(python): pyspark.streaming.tests.test_listener
Finished test(python): pyspark.streaming.tests.test_listener (11s)
Starting test(python): pyspark.tests.test_appsubmit
Finished test(python): pyspark.sql.tests.test_udf (39s)
Starting test(python): pyspark.tests.test_broadcast
Finished test(python): pyspark.streaming.tests.test_context (23s)
Starting test(python): pyspark.tests.test_conf
Finished test(python): pyspark.tests.test_conf (15s)
Starting test(python): pyspark.tests.test_context
Finished test(python): pyspark.tests.test_broadcast (33s)
Starting test(python): pyspark.tests.test_daemon
Finished test(python): pyspark.tests.test_daemon (5s)
Starting test(python): pyspark.tests.test_install_spark
Finished test(python): pyspark.tests.test_context (44s)
Starting test(python): pyspark.tests.test_join
Finished test(python): pyspark.tests.test_appsubmit (68s)
Starting test(python): pyspark.tests.test_profiler
Finished test(python): pyspark.tests.test_join (7s)
Starting test(python): pyspark.tests.test_rdd
Finished test(python): pyspark.tests.test_profiler (9s)
Starting test(python): pyspark.tests.test_rddbarrier
Finished test(python): pyspark.tests.test_rddbarrier (7s)
Starting test(python): pyspark.tests.test_readwrite
Finished test(python): pyspark.streaming.tests.test_dstream (107s)
Starting test(python): pyspark.tests.test_serializers
Finished test(python): pyspark.tests.test_serializers (8s)
Starting test(python): pyspark.tests.test_shuffle
Finished test(python): pyspark.tests.test_readwrite (14s)
Starting test(python): pyspark.tests.test_taskcontext
Finished test(python): pyspark.tests.test_install_spark (65s)
Starting test(python): pyspark.tests.test_util
Finished test(python): pyspark.tests.test_shuffle (8s)
Starting test(python): pyspark.tests.test_worker
Finished test(python): pyspark.tests.test_util (5s)
Starting test(python): pyspark.accumulators
Finished test(python): pyspark.accumulators (5s)
Starting test(python): pyspark.broadcast
Finished test(python): pyspark.broadcast (6s)
Starting test(python): pyspark.conf
Finished test(python): pyspark.tests.test_worker (14s)
Starting test(python): pyspark.context
Finished test(python): pyspark.conf (4s)
Starting test(python): pyspark.ml.classification
Finished test(python): pyspark.tests.test_rdd (60s)
Starting test(python): pyspark.ml.clustering
Finished test(python): pyspark.context (21s)
Starting test(python): pyspark.ml.evaluation
Finished test(python): pyspark.tests.test_taskcontext (69s)
Starting test(python): pyspark.ml.feature
Finished test(python): pyspark.ml.evaluation (26s)
Starting test(python): pyspark.ml.fpm
Finished test(python): pyspark.ml.clustering (45s)
Starting test(python): pyspark.ml.functions
Finished test(python): pyspark.ml.fpm (24s)
Starting test(python): pyspark.ml.image
Finished test(python): pyspark.ml.functions (17s)
Starting test(python): pyspark.ml.linalg.__init__
Finished test(python): pyspark.ml.linalg.__init__ (0s)
Starting test(python): pyspark.ml.recommendation
Finished test(python): pyspark.ml.classification (74s)
Starting test(python): pyspark.ml.regression
Finished test(python): pyspark.ml.image (8s)
Starting test(python): pyspark.ml.stat
Finished test(python): pyspark.ml.stat (29s)
Starting test(python): pyspark.ml.tuning
Finished test(python): pyspark.ml.regression (53s)
Starting test(python): pyspark.mllib.classification
Finished test(python): pyspark.ml.tuning (35s)
Starting test(python): pyspark.mllib.clustering
Finished test(python): pyspark.ml.feature (103s)
Starting test(python): pyspark.mllib.evaluation
Finished test(python): pyspark.mllib.classification (33s)
Starting test(python): pyspark.mllib.feature
Finished test(python): pyspark.mllib.evaluation (21s)
Starting test(python): pyspark.mllib.fpm
Finished test(python): pyspark.ml.recommendation (103s)
Starting test(python): pyspark.mllib.linalg.__init__
Finished test(python): pyspark.mllib.linalg.__init__ (1s)
Starting test(python): pyspark.mllib.linalg.distributed
Finished test(python): pyspark.mllib.feature (26s)
Starting test(python): pyspark.mllib.random
Finished test(python): pyspark.mllib.fpm (23s)
Starting test(python): pyspark.mllib.recommendation
Finished test(python): pyspark.mllib.clustering (50s)
Starting test(python): pyspark.mllib.regression
Finished test(python): pyspark.mllib.random (13s)
Starting test(python): pyspark.mllib.stat.KernelDensity
Finished test(python): pyspark.mllib.stat.KernelDensity (1s)
Starting test(python): pyspark.mllib.stat._statistics
Finished test(python): pyspark.mllib.linalg.distributed (42s)
Starting test(python): pyspark.mllib.tree
Finished test(python): pyspark.mllib.stat._statistics (19s)
Starting test(python): pyspark.mllib.util
Finished test(python): pyspark.mllib.regression (33s)
Starting test(python): pyspark.profiler
Finished test(python): pyspark.mllib.recommendation (36s)
Starting test(python): pyspark.rdd
Finished test(python): pyspark.profiler (9s)
Starting test(python): pyspark.resource.tests.test_resources
Finished test(python): pyspark.mllib.tree (19s)
Starting test(python): pyspark.serializers
Finished test(python): pyspark.mllib.util (21s)
Starting test(python): pyspark.shuffle
Finished test(python): pyspark.resource.tests.test_resources (9s)
Starting test(python): pyspark.sql.avro.functions
Finished test(python): pyspark.shuffle (1s)
Starting test(python): pyspark.sql.catalog
Finished test(python): pyspark.rdd (22s)
Starting test(python): pyspark.sql.column
Finished test(python): pyspark.serializers (12s)
Starting test(python): pyspark.sql.conf
Finished test(python): pyspark.sql.conf (6s)
Starting test(python): pyspark.sql.context
Finished test(python): pyspark.sql.catalog (14s)
Starting test(python): pyspark.sql.dataframe
Finished test(python): pyspark.sql.avro.functions (15s)
Starting test(python): pyspark.sql.functions
Finished test(python): pyspark.sql.column (24s)
Starting test(python): pyspark.sql.group
Finished test(python): pyspark.sql.context (20s)
Starting test(python): pyspark.sql.pandas.conversion
Finished test(python): pyspark.sql.pandas.conversion (13s)
Starting test(python): pyspark.sql.pandas.group_ops
Finished test(python): pyspark.sql.group (36s)
Starting test(python): pyspark.sql.pandas.map_ops
Finished test(python): pyspark.sql.pandas.group_ops (21s)
Starting test(python): pyspark.sql.pandas.serializers
Finished test(python): pyspark.sql.pandas.serializers (0s)
Starting test(python): pyspark.sql.pandas.typehints
Finished test(python): pyspark.sql.pandas.typehints (0s)
Starting test(python): pyspark.sql.pandas.types
Finished test(python): pyspark.sql.pandas.types (0s)
Starting test(python): pyspark.sql.pandas.utils
Finished test(python): pyspark.sql.pandas.utils (0s)
Starting test(python): pyspark.sql.readwriter
Finished test(python): pyspark.sql.dataframe (56s)
Starting test(python): pyspark.sql.session
Finished test(python): pyspark.sql.functions (57s)
Starting test(python): pyspark.sql.streaming
Finished test(python): pyspark.sql.pandas.map_ops (12s)
Starting test(python): pyspark.sql.types
Finished test(python): pyspark.sql.types (10s)
Starting test(python): pyspark.sql.udf
Finished test(python): pyspark.sql.streaming (16s)
Starting test(python): pyspark.sql.window
Finished test(python): pyspark.sql.session (19s)
Starting test(python): pyspark.streaming.util
Finished test(python): pyspark.streaming.util (0s)
Starting test(python): pyspark.util
Finished test(python): pyspark.util (0s)
Finished test(python): pyspark.sql.readwriter (24s)
Finished test(python): pyspark.sql.udf (13s)
Finished test(python): pyspark.sql.window (14s)
Tests passed in 780 seconds

```

Closes #30288 from HyukjinKwon/SPARK-33371-3.0.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 python/pyspark/tests/test_taskcontext.py | 8 ++++----
 python/pyspark/tests/test_worker.py      | 2 +-
 python/setup.py                          | 1 +
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/python/pyspark/tests/test_taskcontext.py b/python/pyspark/tests/test_taskcontext.py
index 57c880ad8043d..b2fa9919c65da 100644
--- a/python/pyspark/tests/test_taskcontext.py
+++ b/python/pyspark/tests/test_taskcontext.py
@@ -127,12 +127,12 @@ def f(iterator):
 
         def context_barrier(x):
             tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
+            time.sleep(random.randint(1, 5) * 2)
             tc.barrier()
             return time.time()
 
         times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
-        self.assertTrue(max(times) - min(times) < 1)
+        self.assertTrue(max(times) - min(times) < 2)
 
     def test_all_gather(self):
         """
@@ -235,7 +235,7 @@ def f(iterator):
 
         def context_barrier(x):
             tc = BarrierTaskContext.get()
-            time.sleep(random.randint(1, 10))
+            time.sleep(random.randint(1, 5) * 2)
             tc.barrier()
             return (time.time(), os.getpid())
 
@@ -243,7 +243,7 @@ def context_barrier(x):
         times = list(map(lambda x: x[0], result))
         pids = list(map(lambda x: x[1], result))
         # check both barrier and worker reuse effect
-        self.assertTrue(max(times) - min(times) < 1)
+        self.assertTrue(max(times) - min(times) < 2)
         for pid in pids:
             self.assertTrue(pid in worker_pids)
 
diff --git a/python/pyspark/tests/test_worker.py b/python/pyspark/tests/test_worker.py
index 9d7deb23da604..bfcbc43b7f533 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -137,7 +137,7 @@ def count():
         t.daemon = True
         t.start()
         t.join(5)
-        self.assertTrue(not t.isAlive())
+        self.assertTrue(not t.is_alive())
         self.assertEqual(100000, rdd.count())
 
     def test_with_different_versions_of_python(self):
diff --git a/python/setup.py b/python/setup.py
index a1cde31aa0ada..78b5a691a7ce3 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -226,6 +226,7 @@ def _supports_symlinks():
             'Programming Language :: Python :: 3.6',
             'Programming Language :: Python :: 3.7',
             'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
             'Programming Language :: Python :: Implementation :: CPython',
             'Programming Language :: Python :: Implementation :: PyPy']
     )

From c157fa36e8e1aa18dcf87b6c774970b8122df0dc Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Mon, 9 Nov 2020 08:32:51 +0000
Subject: [PATCH 1157/1280] [SPARK-33372][SQL] Fix InSet bucket pruning

### What changes were proposed in this pull request?

This pr fix `InSet` bucket pruning because of it's values should not be `Literal`:
https://github.com/apache/spark/blob/cbd3fdea62dab73fc4a96702de8fd1f07722da66/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala#L253-L255

### Why are the changes needed?

Fix bug.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test and manual test:

```scala
spark.sql("select id as a, id as b from range(10000)").write.bucketBy(100, "a").saveAsTable("t")
spark.sql("select * from t where a in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)").show
```

Before this PR | After this PR
-- | --
![image](https://user-images.githubusercontent.com/5399861/98380788-fb120980-2083-11eb-8fae-4e21ad873e9b.png) | ![image](https://user-images.githubusercontent.com/5399861/98381095-5ba14680-2084-11eb-82ca-2d780c85305c.png)

Closes #30279 from wangyum/SPARK-33372.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit 69799c514ff9874c57bf94d4de21ea4cd0cbbf8d)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/datasources/FileSourceStrategy.scala | 5 ++---
 .../org/apache/spark/sql/sources/BucketedReadSuite.scala     | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
index b1e03fedbc1e2..4b9e0c6da11dd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileSourceStrategy.scala
@@ -89,9 +89,8 @@ object FileSourceStrategy extends Strategy with Logging {
       case expressions.In(a: Attribute, list)
         if list.forall(_.isInstanceOf[Literal]) && a.name == bucketColumnName =>
         getBucketSetFromIterable(a, list.map(e => e.eval(EmptyRow)))
-      case expressions.InSet(a: Attribute, hset)
-        if hset.forall(_.isInstanceOf[Literal]) && a.name == bucketColumnName =>
-        getBucketSetFromIterable(a, hset.map(e => expressions.Literal(e).eval(EmptyRow)))
+      case expressions.InSet(a: Attribute, hset) if a.name == bucketColumnName =>
+        getBucketSetFromIterable(a, hset)
       case expressions.IsNull(a: Attribute) if a.name == bucketColumnName =>
         getBucketSetFromValue(a, null)
       case expressions.And(left, right) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index 558bfd7f4675b..df8ca33b893cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -188,7 +188,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils {
 
       // Case 4: InSet
       val inSetExpr = expressions.InSet($"j".expr,
-        Set(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3).map(lit(_).expr))
+        Set(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3))
       checkPrunedAnswers(
         bucketSpec,
         bucketValues = Seq(bucketValue, bucketValue + 1, bucketValue + 2, bucketValue + 3),

From a4184959d705150632729d18fc52135cc94c37fe Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Tue, 10 Nov 2020 10:15:55 +0900
Subject: [PATCH 1158/1280] [SPARK-33397][YARN][DOC] Fix generating md to html
 for available-patterns-for-shs-custom-executor-log-url

### What changes were proposed in this pull request?

1. replace `{{}}`  with `&#123;&#123;&#125;&#125;`
2. using `<code></code>` in td-tag

### Why are the changes needed?

to fix this.
![image](https://user-images.githubusercontent.com/8326978/98544155-8c74bc00-22ce-11eb-8889-8dacb726b762.png)

### Does this PR introduce _any_ user-facing change?

yes, you will see the correct online doc with this change

![image](https://user-images.githubusercontent.com/8326978/98545256-2e48d880-22d0-11eb-9dd9-b8cae3df8659.png)

### How was this patch tested?

shown as the above pic via jekyll serve.

Closes #30298 from yaooqinn/SPARK-33397.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 036c11b0d4ee88ce88d7869bbbbba6a589754786)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 docs/running-on-yarn.md | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 046c545eb1b4c..7a03c91cd6320 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -560,48 +560,48 @@ To use a custom metrics.properties for the application master and executors, upd
 <table class="table">
     <tr><th>Pattern</th><th>Meaning</th></tr>
     <tr>
-      <td>{{HTTP_SCHEME}}</td>
-      <td>`http://` or `https://` according to YARN HTTP policy. (Configured via `yarn.http.policy`)</td>
+      <td>&#123;&#123;HTTP_SCHEME&#125;&#125;</td>
+      <td><code>http://</code> or <code>https://</code> according to YARN HTTP policy. (Configured via <code>yarn.http.policy</code>)</td>
     </tr>
     <tr>
-      <td>{{NM_HOST}}</td>
+      <td>&#123;&#123;NM_HOST&#125;&#125;</td>
       <td>The "host" of node where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_PORT}}</td>
+      <td>&#123;&#123;NM_PORT&#125;&#125;</td>
       <td>The "port" of node manager where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_HTTP_PORT}}</td>
+      <td>&#123;&#123;NM_HTTP_PORT&#125;&#125;</td>
       <td>The "port" of node manager's http server where container was run.</td>
     </tr>
     <tr>
-      <td>{{NM_HTTP_ADDRESS}}</td>
+      <td>&#123;&#123;NM_HTTP_ADDRESS&#125;&#125;</td>
       <td>Http URI of the node on which the container is allocated.</td>
     </tr>
     <tr>
-      <td>{{CLUSTER_ID}}</td>
-      <td>The cluster ID of Resource Manager. (Configured via `yarn.resourcemanager.cluster-id`)</td>
+      <td>&#123;&#123;CLUSTER_ID&#125;&#125;</td>
+      <td>The cluster ID of Resource Manager. (Configured via <code>yarn.resourcemanager.cluster-id</code>)</td>
     </tr>
     <tr>
-      <td>{{CONTAINER_ID}}</td>
+      <td>&#123;&#123;CONTAINER_ID&#125;&#125;</td>
       <td>The ID of container.</td>
     </tr>
     <tr>
-      <td>{{USER}}</td>
-      <td>'SPARK_USER' on system environment.</td>
+      <td>&#123;&#123;USER&#125;&#125;</td>
+      <td><code>SPARK_USER</code> on system environment.</td>
     </tr>
     <tr>
-      <td>{{FILE_NAME}}</td>
-      <td>`stdout`, `stderr`.</td>
+      <td>&#123;&#123;FILE_NAME&#125;&#125;</td>
+      <td><code>stdout</code>, <code>stderr</code>.</td>
     </tr>
 </table>
 
 For example, suppose you would like to point log url link to Job History Server directly instead of let NodeManager http server redirects it, you can configure `spark.history.custom.executor.log.url` as below:
 
- `{{HTTP_SCHEME}}<JHS_HOST>:<JHS_PORT>/jobhistory/logs/{{NM_HOST}}:{{NM_PORT}}/{{CONTAINER_ID}}/{{CONTAINER_ID}}/{{USER}}/{{FILE_NAME}}?start=-4096`
+<code>&#123;&#123;HTTP_SCHEME&#125;&#125;&lt;JHS_HOST&gt;:&lt;JHS_PORT&gt;/jobhistory/logs/&#123;&#123;NM_HOST&#125;&#125;:&#123;&#123;NM_PORT&#125;&#125;/&#123;&#123;CONTAINER_ID&#125;&#125;/&#123;&#123;CONTAINER_ID&#125;&#125;/&#123;&#123;USER&#125;&#125;/&#123;&#123;FILE_NAME&#125;&#125;?start=-4096</code>
 
- NOTE: you need to replace `<JHS_POST>` and `<JHS_PORT>` with actual value.
+NOTE: you need to replace `<JHS_POST>` and `<JHS_PORT>` with actual value.
 
 # Resource Allocation and Configuration Overview
 

From 1aa8f4f39911b855fe9c86c9be86ec1eb720e277 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Tue, 10 Nov 2020 11:14:38 +0900
Subject: [PATCH 1159/1280] [SPARK-33405][BUILD][3.0] Upgrade commons-compress
 to 1.20

### What changes were proposed in this pull request?

This PR aims to upgrade `commons-compress` from 1.8 to 1.20.

### Why are the changes needed?

- https://commons.apache.org/proper/commons-compress/security-reports.html

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CIs.

Closes #30305 from dongjoon-hyun/SPARK-33405-3.0.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +-
 pom.xml                                 | 6 ++++++
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index e32ea645638f0..2068d11ffe38e 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -36,7 +36,7 @@ commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 168d6196fe1fe..06fcff8d94f27 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -34,7 +34,7 @@ commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index d730b4a0e1cb0..9c219816ca1a2 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -31,7 +31,7 @@ commons-cli/1.2//commons-cli-1.2.jar
 commons-codec/1.10//commons-codec-1.10.jar
 commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.8.1//commons-compress-1.8.1.jar
+commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
 commons-crypto/1.0.0//commons-crypto-1.0.0.jar
 commons-daemon/1.0.13//commons-daemon-1.0.13.jar
diff --git a/pom.xml b/pom.xml
index de3b69db18fbb..d41d61c1e12d3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -176,6 +176,7 @@
     <snappy.version>1.1.7.5</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <commons-codec.version>1.10</commons-codec.version>
+    <commons-compress.version>1.20</commons-compress.version>
     <commons-io.version>2.4</commons-io.version>
     <!-- org.apache.commons/commons-lang/-->
     <commons-lang2.version>2.6</commons-lang2.version>
@@ -529,6 +530,11 @@
         <artifactId>commons-codec</artifactId>
         <version>${commons-codec.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>${commons-compress.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-math3</artifactId>

From b905d655e7e2edff742e0c89cadf68dfbf32f7bd Mon Sep 17 00:00:00 2001
From: "xuewei.linxuewei" <xuewei.linxuewei@alibaba-inc.com>
Date: Tue, 10 Nov 2020 07:23:47 +0000
Subject: [PATCH 1160/1280] [SPARK-33391][SQL] element_at with CreateArray not
 respect one based index
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

element_at with CreateArray not respect one based index.

repo step:

```
var df = spark.sql("select element_at(array(3, 2, 1), 0)")
df.printSchema()

df = spark.sql("select element_at(array(3, 2, 1), 1)")
df.printSchema()

df = spark.sql("select element_at(array(3, 2, 1), 2)")
df.printSchema()

df = spark.sql("select element_at(array(3, 2, 1), 3)")
df.printSchema()

root
– element_at(array(3, 2, 1), 0): integer (nullable = false)

root
– element_at(array(3, 2, 1), 1): integer (nullable = false)

root
– element_at(array(3, 2, 1), 2): integer (nullable = false)

root
– element_at(array(3, 2, 1), 3): integer (nullable = true)

correct answer should be
0 true which is outOfBounds return default true.
1 false
2 false
3 false

```

For expression eval, it respect the oneBasedIndex, but within checking the nullable, it calculates with zeroBasedIndex using `computeNullabilityFromArray`.

### Why are the changes needed?

Correctness issue.

### Does this PR introduce any user-facing change?

No.

### How was this patch tested?

Added UT and existing UT.

Closes #30296 from leanken/leanken-SPARK-33391.

Authored-by: xuewei.linxuewei <xuewei.linxuewei@alibaba-inc.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit e3a768dd79558b04f6ae71380876bcde2354008c)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../expressions/collectionOperations.scala    | 30 +++++++++++++++
 .../CollectionExpressionsSuite.scala          | 38 +++++++++++++++----
 2 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 7a15bcce836f5..26b3003790e48 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -1954,6 +1954,36 @@ case class ElementAt(left: Expression, right: Expression)
     }
   }
 
+  private def nullability(elements: Seq[Expression], ordinal: Int): Boolean = {
+    if (ordinal == 0) {
+      false
+    } else if (elements.length < math.abs(ordinal)) {
+      true
+    } else {
+      if (ordinal < 0) {
+        elements(elements.length + ordinal).nullable
+      } else {
+        elements(ordinal - 1).nullable
+      }
+    }
+  }
+
+  override def computeNullabilityFromArray(child: Expression, ordinal: Expression): Boolean = {
+    if (ordinal.foldable && !ordinal.nullable) {
+      val intOrdinal = ordinal.eval().asInstanceOf[Number].intValue()
+      child match {
+        case CreateArray(ar, _) =>
+          nullability(ar, intOrdinal)
+        case GetArrayStructFields(CreateArray(elements, _), field, _, _, _) =>
+          nullability(elements, intOrdinal) || field.nullable
+        case _ =>
+          true
+      }
+    } else {
+      true
+    }
+  }
+
   override def nullable: Boolean = left.dataType match {
     case _: ArrayType => computeNullabilityFromArray(left, right)
     case _: MapType => true
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index c8c97d3e39abe..7d278b6e9151b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -1115,11 +1115,18 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     val a = AttributeReference("a", IntegerType, nullable = false)()
     val b = AttributeReference("b", IntegerType, nullable = true)()
     val array = CreateArray(a :: b :: Nil)
-    assert(!ElementAt(array, Literal(0)).nullable)
-    assert(ElementAt(array, Literal(1)).nullable)
-    assert(!ElementAt(array, Subtract(Literal(2), Literal(2))).nullable)
+    assert(!ElementAt(array, Literal(1)).nullable)
+    assert(!ElementAt(array, Literal(-2)).nullable)
+    assert(ElementAt(array, Literal(2)).nullable)
+    assert(ElementAt(array, Literal(-1)).nullable)
+    assert(!ElementAt(array, Subtract(Literal(2), Literal(1))).nullable)
     assert(ElementAt(array, AttributeReference("ordinal", IntegerType)()).nullable)
 
+    // CreateArray case invalid indices
+    assert(!ElementAt(array, Literal(0)).nullable)
+    assert(ElementAt(array, Literal(4)).nullable)
+    assert(ElementAt(array, Literal(-4)).nullable)
+
     // GetArrayStructFields case
     val f1 = StructField("a", IntegerType, nullable = false)
     val f2 = StructField("b", IntegerType, nullable = true)
@@ -1128,19 +1135,34 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
     val inputArray1 = CreateArray(c :: Nil)
     val inputArray1ContainsNull = c.nullable
     val stArray1 = GetArrayStructFields(inputArray1, f1, 0, 2, inputArray1ContainsNull)
-    assert(!ElementAt(stArray1, Literal(0)).nullable)
+    assert(!ElementAt(stArray1, Literal(1)).nullable)
+    assert(!ElementAt(stArray1, Literal(-1)).nullable)
     val stArray2 = GetArrayStructFields(inputArray1, f2, 1, 2, inputArray1ContainsNull)
-    assert(ElementAt(stArray2, Literal(0)).nullable)
+    assert(ElementAt(stArray2, Literal(1)).nullable)
+    assert(ElementAt(stArray2, Literal(-1)).nullable)
 
     val d = AttributeReference("d", structType, nullable = true)()
     val inputArray2 = CreateArray(c :: d :: Nil)
     val inputArray2ContainsNull = c.nullable || d.nullable
     val stArray3 = GetArrayStructFields(inputArray2, f1, 0, 2, inputArray2ContainsNull)
-    assert(!ElementAt(stArray3, Literal(0)).nullable)
-    assert(ElementAt(stArray3, Literal(1)).nullable)
+    assert(!ElementAt(stArray3, Literal(1)).nullable)
+    assert(!ElementAt(stArray3, Literal(-2)).nullable)
+    assert(ElementAt(stArray3, Literal(2)).nullable)
+    assert(ElementAt(stArray3, Literal(-1)).nullable)
     val stArray4 = GetArrayStructFields(inputArray2, f2, 1, 2, inputArray2ContainsNull)
-    assert(ElementAt(stArray4, Literal(0)).nullable)
     assert(ElementAt(stArray4, Literal(1)).nullable)
+    assert(ElementAt(stArray4, Literal(-2)).nullable)
+    assert(ElementAt(stArray4, Literal(2)).nullable)
+    assert(ElementAt(stArray4, Literal(-1)).nullable)
+
+    // GetArrayStructFields case invalid indices
+    assert(!ElementAt(stArray3, Literal(0)).nullable)
+    assert(ElementAt(stArray3, Literal(4)).nullable)
+    assert(ElementAt(stArray3, Literal(-4)).nullable)
+
+    assert(ElementAt(stArray4, Literal(0)).nullable)
+    assert(ElementAt(stArray4, Literal(4)).nullable)
+    assert(ElementAt(stArray4, Literal(-4)).nullable)
   }
 
   test("Concat") {

From 4a1c143f1a042a9a23d00929670eadbdb1afca11 Mon Sep 17 00:00:00 2001
From: lrz <lrz@lrzdeMacBook-Pro.local>
Date: Tue, 10 Nov 2020 19:39:18 +0900
Subject: [PATCH 1161/1280] [SPARK-33339][PYTHON] Pyspark application will hang
 due to non Exception error

### What changes were proposed in this pull request?

When a system.exit exception occurs during the process, the python worker exits abnormally, and then the executor task is still waiting for the worker for reading from socket, causing it to hang.
The system.exit exception may be caused by the user's error code, but spark should at least throw an error to remind the user, not get stuck
we can run a simple test to reproduce this case:

```
from pyspark.sql import SparkSession
def err(line):
  raise SystemExit
spark = SparkSession.builder.appName("test").getOrCreate()
spark.sparkContext.parallelize(range(1,2), 2).map(err).collect()
spark.stop()
```

### Why are the changes needed?

to make sure pyspark application won't hang if there's non-Exception error in python worker

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

added a new test and also manually tested the case above

Closes #30248 from li36909/pyspark.

Lead-authored-by: lrz <lrz@lrzdeMacBook-Pro.local>
Co-authored-by: Hyukjin Kwon <gurwls223@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 27bb40b6297361985e3590687f0332a72b71bc85)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 python/pyspark/tests/test_worker.py | 9 +++++++++
 python/pyspark/worker.py            | 4 ++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/tests/test_worker.py b/python/pyspark/tests/test_worker.py
index bfcbc43b7f533..f51d4b24f11b9 100644
--- a/python/pyspark/tests/test_worker.py
+++ b/python/pyspark/tests/test_worker.py
@@ -98,6 +98,15 @@ def raise_exception(_):
             self.assertRaises(Exception, lambda: rdd.foreach(raise_exception))
         self.assertEqual(100, rdd.map(str).count())
 
+    def test_after_non_exception_error(self):
+        # SPARK-33339: Pyspark application will hang due to non Exception
+        def raise_system_exit(_):
+            raise SystemExit()
+        rdd = self.sc.parallelize(range(100), 1)
+        with QuietTest(self.sc):
+            self.assertRaises(Exception, lambda: rdd.foreach(raise_system_exit))
+        self.assertEqual(100, rdd.map(str).count())
+
     def test_after_jvm_exception(self):
         tempFile = tempfile.NamedTemporaryFile(delete=False)
         tempFile.write(b"Hello World!")
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index 814f796c0bf1c..0bce87d5c9f90 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -608,7 +608,7 @@ def process():
         # reuse.
         TaskContext._setTaskContext(None)
         BarrierTaskContext._setTaskContext(None)
-    except Exception:
+    except BaseException:
         try:
             exc_info = traceback.format_exc()
             if isinstance(exc_info, bytes):
@@ -622,7 +622,7 @@ def process():
         except IOError:
             # JVM close the socket
             pass
-        except Exception:
+        except BaseException:
             # Write the error to stderr if it happened while serializing
             print("PySpark worker failed with exception:", file=sys.stderr)
             print(traceback.format_exc(), file=sys.stderr)

From 577dbb96835f13f4cd92ea4caab9e6dece00be50 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Wed, 11 Nov 2020 15:24:05 +0900
Subject: [PATCH 1162/1280] [SPARK-33417][SQL][TEST] Correct the behaviour of
 query filters in TPCDSQueryBenchmark

### What changes were proposed in this pull request?

This PR intends to fix the behaviour of query filters in `TPCDSQueryBenchmark`. We can use an option `--query-filter` for selecting TPCDS queries to run, e.g., `--query-filter q6,q8,q13`. But, the current master has a weird behaviour about the option. For example, if we pass `--query-filter q6` so as to run the TPCDS q6 only, `TPCDSQueryBenchmark` runs `q6` and `q6-v2.7` because the `filterQueries` method does not respect the name suffix. So, there is no way now to run the TPCDS q6 only.

### Why are the changes needed?

Bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually checked.

Closes #30324 from maropu/FilterBugInTPCDSQueryBenchmark.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4b367976a877adb981f65d546e1522fdf30d0731)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../benchmark/TPCDSQueryBenchmark.scala       | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
index 7bbf0795eb052..43bc7c12937ec 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/TPCDSQueryBenchmark.scala
@@ -98,11 +98,16 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
     }
   }
 
-  def filterQueries(
+  private def filterQueries(
       origQueries: Seq[String],
-      args: TPCDSQueryBenchmarkArguments): Seq[String] = {
-    if (args.queryFilter.nonEmpty) {
-      origQueries.filter(args.queryFilter.contains)
+      queryFilter: Set[String],
+      nameSuffix: String = ""): Seq[String] = {
+    if (queryFilter.nonEmpty) {
+      if (nameSuffix.nonEmpty) {
+        origQueries.filter { name => queryFilter.contains(s"$name$nameSuffix") }
+      } else {
+        origQueries.filter(queryFilter.contains)
+      }
     } else {
       origQueries
     }
@@ -125,6 +130,7 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q98", "q99")
 
     // This list only includes TPC-DS v2.7 queries that are different from v1.4 ones
+    val nameSuffixForQueriesV2_7 = "-v2.7"
     val tpcdsQueriesV2_7 = Seq(
       "q5a", "q6", "q10a", "q11", "q12", "q14", "q14a", "q18a",
       "q20", "q22", "q22a", "q24", "q27a", "q34", "q35", "q35a", "q36a", "q47", "q49",
@@ -132,8 +138,9 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
       "q80a", "q86a", "q98")
 
     // If `--query-filter` defined, filters the queries that this option selects
-    val queriesV1_4ToRun = filterQueries(tpcdsQueries, benchmarkArgs)
-    val queriesV2_7ToRun = filterQueries(tpcdsQueriesV2_7, benchmarkArgs)
+    val queriesV1_4ToRun = filterQueries(tpcdsQueries, benchmarkArgs.queryFilter)
+    val queriesV2_7ToRun = filterQueries(tpcdsQueriesV2_7, benchmarkArgs.queryFilter,
+      nameSuffix = nameSuffixForQueriesV2_7)
 
     if ((queriesV1_4ToRun ++ queriesV2_7ToRun).isEmpty) {
       throw new RuntimeException(
@@ -143,6 +150,6 @@ object TPCDSQueryBenchmark extends SqlBasedBenchmark {
     val tableSizes = setupTables(benchmarkArgs.dataLocation)
     runTpcdsQueries(queryLocation = "tpcds", queries = queriesV1_4ToRun, tableSizes)
     runTpcdsQueries(queryLocation = "tpcds-v2.7.0", queries = queriesV2_7ToRun, tableSizes,
-      nameSuffix = "-v2.7")
+      nameSuffix = nameSuffixForQueriesV2_7)
   }
 }

From 1e2984bc9525d38ddd31ade48f5caf83213fc853 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 11 Nov 2020 13:50:39 +0000
Subject: [PATCH 1163/1280] [SPARK-33412][SQL][3.0] OverwriteByExpression
 should resolve its delete condition based on the table relation not the input
 query

backport https://github.com/apache/spark/pull/30318 to 3.0

Closes #30328 from cloud-fan/backport.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/analysis/Analyzer.scala  |  9 ++++-----
 .../sql/catalyst/plans/logical/v2Commands.scala |  3 ++-
 .../analysis/DataSourceV2AnalysisSuite.scala    | 17 ++++++++++++-----
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 2f30ba3f39e44..7795d709b3440 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1363,11 +1363,10 @@ class Analyzer(
       case a @ Aggregate(groupingExprs, aggExprs, appendColumns: AppendColumns) =>
         a.mapExpressions(resolveExpressionTopDown(_, appendColumns))
 
-      case o: OverwriteByExpression
-          if !(o.table.resolved && o.query.resolved && o.outputResolved) =>
-        // do not resolve expression attributes until the query attributes are resolved against the
-        // table by ResolveOutputRelation. that rule will alias the attributes to the table's names.
-        o
+      case o: OverwriteByExpression if o.table.resolved =>
+        // The delete condition of `OverwriteByExpression` will be passed to the table
+        // implementation and should be resolved based on the table schema.
+        o.copy(deleteExpr = resolveExpressionBottomUp(o.deleteExpr, o.table))
 
       case m @ MergeIntoTable(targetTable, sourceTable, _, _, _)
         if !m.resolved && targetTable.resolved && sourceTable.resolved =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 35874a9fc2979..9077f7ad02b69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.plans.logical
 
 import org.apache.spark.sql.catalyst.analysis.{NamedRelation, UnresolvedException}
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, Unevaluable}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, Expression, Unevaluable}
 import org.apache.spark.sql.catalyst.plans.DescribeTableSchema
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.TableChange.{AddColumn, ColumnChange}
@@ -89,6 +89,7 @@ case class OverwriteByExpression(
   override lazy val resolved: Boolean = {
     table.resolved && query.resolved && outputResolved && deleteExpr.resolved
   }
+  override def inputSet: AttributeSet = AttributeSet(table.output)
 }
 
 object OverwriteByExpression {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
index 5aa2b988cf66b..3810434cffb53 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DataSourceV2AnalysisSuite.scala
@@ -588,9 +588,7 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
         Alias(Cast(a, DoubleType, Some(conf.sessionLocalTimeZone)), "x")(),
         Alias(Cast(b, DoubleType, Some(conf.sessionLocalTimeZone)), "y")()),
         query),
-      LessThanOrEqual(
-        AttributeReference("x", DoubleType, nullable = false)(x.exprId),
-        Literal(15.0d)))
+      LessThanOrEqual(x, Literal(15.0d)))
 
     assertNotResolved(parsedPlan)
     checkAnalysis(parsedPlan, expectedPlan)
@@ -598,7 +596,7 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
   }
 
   protected def testNotResolvedOverwriteByExpression(): Unit = {
-    val xRequiredTable = TestRelation(StructType(Seq(
+    val table = TestRelation(StructType(Seq(
       StructField("x", DoubleType, nullable = false),
       StructField("y", DoubleType))).toAttributes)
 
@@ -607,10 +605,19 @@ abstract class DataSourceV2AnalysisBaseSuite extends AnalysisTest {
       StructField("b", DoubleType))).toAttributes)
 
     // the write is resolved (checked above). this test plan is not because of the expression.
-    val parsedPlan = OverwriteByExpression.byPosition(xRequiredTable, query,
+    val parsedPlan = OverwriteByExpression.byPosition(table, query,
       LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
 
     assertNotResolved(parsedPlan)
     assertAnalysisError(parsedPlan, Seq("cannot resolve", "`a`", "given input columns", "x, y"))
+
+    val tableAcceptAnySchema = TestRelationAcceptAnySchema(StructType(Seq(
+      StructField("x", DoubleType, nullable = false),
+      StructField("y", DoubleType))).toAttributes)
+
+    val parsedPlan2 = OverwriteByExpression.byPosition(tableAcceptAnySchema, query,
+      LessThanOrEqual(UnresolvedAttribute(Seq("a")), Literal(15.0d)))
+    assertNotResolved(parsedPlan2)
+    assertAnalysisError(parsedPlan2, Seq("cannot resolve", "`a`", "given input columns", "x, y"))
   }
 }

From 3edec10891f6d4db0f3fe792b453cf384d7c3f40 Mon Sep 17 00:00:00 2001
From: Steve Loughran <stevel@cloudera.com>
Date: Wed, 11 Nov 2020 14:27:48 -0800
Subject: [PATCH 1164/1280] [SPARK-33402][CORE] Jobs launched in same second
 have duplicate MapReduce JobIDs

### What changes were proposed in this pull request?

1. Applies the SQL changes in SPARK-33230 to SparkHadoopWriter, so that `rdd.saveAsNewAPIHadoopDataset` passes in a unique job UUID in `spark.sql.sources.writeJobUUID`
1. `SparkHadoopWriterUtils.createJobTrackerID` generates a JobID by appending a random long number to the supplied timestamp to ensure the probability of a collision is near-zero.
1. With tests of uniqueness, round trips and negative jobID rejection.

### Why are the changes needed?

Without this, if more than one job is started in the same second *and the committer expects application attempt IDs to be unique* is at risk of clashing with other jobs.

With the fix,

* those committers which use the ID set in `spark.sql.sources.writeJobUUID` as a priority ID will pick that up instead and so be unique.
* committers which use the Hadoop JobID for unique paths and filenames will get the randomly generated jobID.  Assuming all clocks in a cluster in sync, the probability of two jobs launched in the same second has dropped from 1 to 1/(2^63)

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit tests

There's a new test suite SparkHadoopWriterUtilsSuite which creates jobID, verifies they are unique even for the same timestamp and that they can be marshalled to string and parsed back in the hadoop code, which contains some (brittle) assumptions about the format of job IDs.

Functional Integration Tests

1. Hadoop-trunk built with [HADOOP-17318], publishing to local maven repository
1. Spark built with hadoop.version=3.4.0-SNAPSHOT to pick up these JARs.
1. Spark + Object store integration tests at [https://github.com/hortonworks-spark/cloud-integration](https://github.com/hortonworks-spark/cloud-integration) were built against that local spark version
1. And executed against AWS london.

The tests were run with `fs.s3a.committer.require.uuid=true`, so the s3a committers fail fast if they don't get a job ID down. This showed that `rdd.saveAsNewAPIHadoopDataset` wasn't setting the UUID option. It again uses the current Date value for an app attempt -which is not guaranteed to be unique.

With the change applied to spark, the relevant tests work, therefore the committers are getting unique job IDs.

Closes #30319 from steveloughran/BUG/SPARK-33402-jobuuid.

Authored-by: Steve Loughran <stevel@cloudera.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 318a173fcee11902820593fe4ac992a90e6bb00e)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/internal/io/SparkHadoopWriter.scala |   7 +-
 .../internal/io/SparkHadoopWriterUtils.scala  |  25 ++++-
 .../io/SparkHadoopWriterUtilsSuite.scala      | 102 ++++++++++++++++++
 3 files changed, 131 insertions(+), 3 deletions(-)
 create mode 100644 core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala

diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
index a619f10bbf064..a5d2c5c88ae03 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriter.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.internal.io
 
 import java.text.NumberFormat
-import java.util.{Date, Locale}
+import java.util.{Date, Locale, UUID}
 
 import scala.reflect.ClassTag
 
@@ -70,6 +70,11 @@ object SparkHadoopWriter extends Logging {
     // Assert the output format/key/value class is set in JobConf.
     config.assertConf(jobContext, rdd.conf)
 
+    // propagate the description UUID into the jobs, so that committers
+    // get an ID guaranteed to be unique.
+    jobContext.getConfiguration.set("spark.sql.sources.writeJobUUID",
+      UUID.randomUUID.toString)
+
     val committer = config.createCommitter(commitJobId)
     committer.setupJob(jobContext)
 
diff --git a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
index de828a6d6156e..657842c620f30 100644
--- a/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
+++ b/core/src/main/scala/org/apache/spark/internal/io/SparkHadoopWriterUtils.scala
@@ -20,7 +20,7 @@ package org.apache.spark.internal.io
 import java.text.SimpleDateFormat
 import java.util.{Date, Locale}
 
-import scala.util.DynamicVariable
+import scala.util.{DynamicVariable, Random}
 
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.mapred.{JobConf, JobID}
@@ -37,14 +37,35 @@ private[spark]
 object SparkHadoopWriterUtils {
 
   private val RECORDS_BETWEEN_BYTES_WRITTEN_METRIC_UPDATES = 256
+  private val RAND = new Random()
 
+  /**
+   * Create a job ID.
+   *
+   * @param time (current) time
+   * @param id job number
+   * @return a job ID
+   */
   def createJobID(time: Date, id: Int): JobID = {
+    if (id < 0) {
+      throw new IllegalArgumentException("Job number is negative")
+    }
     val jobtrackerID = createJobTrackerID(time)
     new JobID(jobtrackerID, id)
   }
 
+  /**
+   * Generate an ID for a job tracker.
+   * @param time (current) time
+   * @return a string for a job ID
+   */
   def createJobTrackerID(time: Date): String = {
-    new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(time)
+    val base = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(time)
+    var l1 = RAND.nextLong()
+    if (l1 < 0) {
+      l1 = -l1
+    }
+    base + l1
   }
 
   def createPathFromString(path: String, conf: JobConf): Path = {
diff --git a/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala b/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala
new file mode 100644
index 0000000000000..33b58ec9e6665
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/internal/io/SparkHadoopWriterUtilsSuite.scala
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.internal.io
+
+import java.util.Date
+
+import org.apache.hadoop.mapreduce.JobID
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.internal.io.SparkHadoopWriterUtils.createJobID
+
+/**
+ * Unit tests for functions in SparkHadoopWriterUtils.
+ */
+class SparkHadoopWriterUtilsSuite extends SparkFunSuite {
+
+  /**
+   * Core test of JobID generation:
+   * They are created.
+   * The job number is converted to the job ID.
+   * They round trip to string and back
+   * (which implies that the full string matches the regexp
+   * in the JobID class).
+   */
+  test("JobID Generation") {
+    val jobNumber = 1010
+    val j1 = createJobID(new Date(), jobNumber)
+    assert(jobNumber == j1.getId,
+      s"Job number mismatch in $j1")
+
+    val jobStr = j1.toString
+    // the string value begins with job_
+    assert(jobStr.startsWith("job_"),
+      s"wrong prefix of $jobStr")
+    // and the hadoop code can parse it
+    val j2 = roundTrip(j1)
+    assert(j1.getId == j2.getId, "Job ID mismatch")
+    assert(j1.getJtIdentifier == j2.getJtIdentifier, "Job identifier mismatch")
+  }
+
+  /**
+   * This is the problem surfacing in situations where committers expect
+   * Job IDs to be unique: if the timestamp is (exclusively) used
+   * then there will conflict in directories created.
+   */
+  test("JobIDs generated at same time are different") {
+    val now = new Date()
+    val j1 = createJobID(now, 1)
+    val j2 = createJobID(now, 1)
+    assert(j1.toString != j2.toString)
+  }
+
+  /**
+   * There's nothing explicitly in the Hadoop classes to stop
+   * job numbers being negative.
+   * There's some big assumptions in the FileOutputCommitter about attempt IDs
+   * being positive during any recovery operations; for safety the ID
+   * job number is validated.
+   */
+  test("JobIDs with negative job number") {
+    intercept[IllegalArgumentException] {
+      createJobID(new Date(), -1)
+    }
+  }
+
+  /**
+   * If someone ever does reinstate use of timestamps,
+   * make sure that the case of timestamp == 0 is handled.
+   */
+  test("JobIDs on Epoch are different") {
+    val j1 = createJobID(new Date(0), 0)
+    val j2 = createJobID(new Date(0), 0)
+    assert (j1.toString != j2.toString)
+  }
+
+  /**
+   * Do a round trip as a string and back again.
+   * This uses the JobID parser.
+   * @param jobID job ID
+   * @return the returned jobID
+   */
+  private def roundTrip(jobID: JobID): JobID = {
+    val parsedJobId = JobID.forName(jobID.toString)
+    assert(jobID == parsedJobId, "Round trip was inconsistent")
+    parsedJobId
+  }
+}

From 00be83a15807cb5b26b0cd7fe491060e27bf4c81 Mon Sep 17 00:00:00 2001
From: Utkarsh <utkarsh.agarwal@databricks.com>
Date: Wed, 11 Nov 2020 14:45:49 -0800
Subject: [PATCH 1165/1280] [SPARK-33404][SQL][3.0] Fix incorrect results in
 `date_trunc` expression

**Backport #30303 to 3.0**

### What changes were proposed in this pull request?
The following query produces incorrect results:
```
SELECT date_trunc('minute', '1769-10-17 17:10:02')
```
Spark currently incorrectly returns
```
1769-10-17 17:10:02
```
against the expected return value of
```
1769-10-17 17:10:00
```
**Steps to repro**
Run the following commands in spark-shell:
```
spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles")
spark.sql("SELECT date_trunc('minute', '1769-10-17 17:10:02')").show()
```
This happens as `truncTimestamp` in package `org.apache.spark.sql.catalyst.util.DateTimeUtils` incorrectly assumes that time zone offsets can never have the granularity of a second and thus does not account for time zone adjustment when truncating the given timestamp to `minute`.
This assumption is currently used when truncating the timestamps to `microsecond, millisecond, second, or minute`.

This PR fixes this issue and always uses time zone knowledge when truncating timestamps regardless of the truncation unit.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added new tests to `DateTimeUtilsSuite` which previously failed and pass now.

Closes #30339 from utkarsh39/date_trunc_fix_3.0.

Authored-by: Utkarsh <utkarsh.agarwal@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/catalyst/util/DateTimeUtils.scala     |  6 ++--
 .../catalyst/util/DateTimeUtilsSuite.scala    | 34 +++++++++++++------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index 3f8417f2f3651..344a6ae428881 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -771,8 +771,12 @@ object DateTimeUtils {
    * Trunc level should be generated using `parseTruncLevel()`, should be between 0 and 9.
    */
   def truncTimestamp(t: SQLTimestamp, level: Int, zoneId: ZoneId): SQLTimestamp = {
+    // Time zone offsets have a maximum precision of seconds (see `java.time.ZoneOffset`). Hence
+    // truncation to microsecond, millisecond, and second can be done
+    // without using time zone information. This results in a performance improvement.
     level match {
       case TRUNC_TO_MICROSECOND => t
+      case TRUNC_TO_MINUTE => truncToUnit(t, zoneId, ChronoUnit.MINUTES)
       case TRUNC_TO_HOUR => truncToUnit(t, zoneId, ChronoUnit.HOURS)
       case TRUNC_TO_DAY => truncToUnit(t, zoneId, ChronoUnit.DAYS)
       case _ =>
@@ -781,8 +785,6 @@ object DateTimeUtils {
           case TRUNC_TO_MILLISECOND => millis
           case TRUNC_TO_SECOND =>
             millis - Math.floorMod(millis, MILLIS_PER_SECOND)
-          case TRUNC_TO_MINUTE =>
-            millis - Math.floorMod(millis, MILLIS_PER_MINUTE)
           case _ => // Try to truncate date levels
             val dDays = millisToDays(millis, zoneId)
             daysToMillis(truncDate(dDays, level), zoneId)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index d526ae18ab65e..f92d65c937928 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -516,18 +516,32 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
     assert(time == None)
   }
 
-  test("truncTimestamp") {
-    def testTrunc(
-        level: Int,
-        expected: String,
-        inputTS: SQLTimestamp,
-        zoneId: ZoneId = defaultZoneId): Unit = {
-      val truncated =
-        DateTimeUtils.truncTimestamp(inputTS, level, zoneId)
-      val expectedTS = toTimestamp(expected, defaultZoneId)
-      assert(truncated === expectedTS.get)
+  def testTrunc(
+      level: Int,
+      expected: String,
+      inputTS: Long,
+      zoneId: ZoneId = defaultZoneId): Unit = {
+    val truncated = DateTimeUtils.truncTimestamp(inputTS, level, zoneId)
+    val expectedTS = toTimestamp(expected, defaultZoneId)
+    assert(truncated === expectedTS.get)
+  }
+
+  test("SPARK-33404: test truncTimestamp when time zone offset from UTC has a " +
+    "granularity of seconds") {
+    for (zid <- ALL_TIMEZONES) {
+      withDefaultTimeZone(zid) {
+        val inputTS = DateTimeUtils.stringToTimestamp(
+          UTF8String.fromString("1769-10-17T17:10:02.123456"), defaultZoneId)
+        testTrunc(DateTimeUtils.TRUNC_TO_MINUTE, "1769-10-17T17:10:00", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_SECOND, "1769-10-17T17:10:02", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MILLISECOND, "1769-10-17T17:10:02.123", inputTS.get, zid)
+        testTrunc(DateTimeUtils.TRUNC_TO_MICROSECOND, "1769-10-17T17:10:02.123456",
+          inputTS.get, zid)
+      }
     }
+  }
 
+  test("truncTimestamp") {
     val defaultInputTS = DateTimeUtils.stringToTimestamp(
       UTF8String.fromString("2015-03-05T09:32:05.359123"), defaultZoneId)
     val defaultInputTS1 = DateTimeUtils.stringToTimestamp(

From 2eadedc9adbd49fc5c14b532ff9a54be17275824 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 12 Nov 2020 15:37:04 +0900
Subject: [PATCH 1166/1280] [SPARK-33408][K8S][R][3.0] Use R 3.6.3 in K8s R
 image
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

This PR aims to upgrade K8s R image to use R 3.6.3 which is the same version installed in Jenkins Servers.

### Why are the changes needed?

Jenkins Server is using `R 3.6.3`.
```
+ SPARK_HOME=/home/jenkins/workspace/SparkPullRequestBuilder-K8s
+ /usr/bin/R CMD check --as-cran --no-tests SparkR_3.1.0.tar.gz
* using log directory ‘/home/jenkins/workspace/SparkPullRequestBuilder-K8s/R/SparkR.Rcheck’
* using R version 3.6.3 (2020-02-29)
```

OpenJDK docker image is using `R 3.5.2 (2018-12-20)` which is old and currently `spark-3.0.1` fails to run SparkR.
```
$ cd spark-3.0.1-bin-hadoop3.2

$ bin/docker-image-tool.sh -R kubernetes/dockerfiles/spark/bindings/R/Dockerfile -n build

$ bin/spark-submit --master k8s://https://192.168.64.49:8443 --deploy-mode cluster --conf spark.kubernetes.container.image=spark-r:latest local:///opt/spark/examples/src/main/r/dataframe.R

$ k logs dataframe-r-b1c14b75b0c09eeb-driver
...
+ exec /usr/bin/tini -s -- /opt/spark/bin/spark-submit --conf spark.driver.bindAddress=172.17.0.4 --deploy-mode client --properties-file /opt/spark/conf/spark.properties --class org.apache.spark.deploy.RRunner local:///opt/spark/examples/src/main/r/dataframe.R
20/11/10 06:03:58 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
log4j:WARN No appenders could be found for logger (io.netty.util.internal.logging.InternalLoggerFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
Error: package or namespace load failed for ‘SparkR’ in rbind(info, getNamespaceInfo(env, "S3methods")):
 number of columns of matrices must match (see arg 2)
In addition: Warning message:
package ‘SparkR’ was built under R version 4.0.2
Execution halted
```

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Pass K8s IT.

Closes #30310 from dongjoon-hyun/SPARK-33408.

Lead-authored-by: Dongjoon Hyun <dongjoon@apache.org>
Co-authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../src/main/dockerfiles/spark/bindings/R/Dockerfile     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
index 59f375b707ca7..bd645e40677d0 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
@@ -25,7 +25,14 @@ USER 0
 
 RUN mkdir ${SPARK_HOME}/R
 
-RUN apt-get update && apt install -y r-base r-base-dev && rm -rf /var/cache/apt/*
+# Install R 3.6.3 (http://cloud.r-project.org/bin/linux/debian/)
+RUN \
+  echo "deb http://cloud.r-project.org/bin/linux/debian buster-cran35/" >> /etc/apt/sources.list && \
+  apt install -y gnupg && \
+  apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' && \
+  apt-get update && \
+  apt install -y -t buster-cran35 r-base r-base-dev && \
+  rm -rf /var/cache/apt/*
 
 COPY R ${SPARK_HOME}/R
 ENV R_HOME /usr/lib/R

From 5ee76e6e10d863bdc7446e3b1e14d98d7e50fd8a Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 12 Nov 2020 18:53:06 +0900
Subject: [PATCH 1167/1280] [MINOR][DOC] spark.executor.memoryOverhead is not
 cluster-mode only

### What changes were proposed in this pull request?

Remove "in cluster mode" from the description of `spark.executor.memoryOverhead`

### Why are the changes needed?

fix correctness issue in documentaion

### Does this PR introduce _any_ user-facing change?

yes, users may not get confused about the description `spark.executor.memoryOverhead`

### How was this patch tested?

pass GA doc generation

Closes #30311 from yaooqinn/minordoc.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 4335af075a8ad27c4906f03ae5f8cd8f9a754e5a)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../scala/org/apache/spark/internal/config/package.scala   | 4 ++--
 docs/configuration.md                                      | 7 +++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 0ff6b73d20dff..f3474efebab01 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -301,8 +301,8 @@ package object config {
     .createWithDefaultString("1g")
 
   private[spark] val EXECUTOR_MEMORY_OVERHEAD = ConfigBuilder("spark.executor.memoryOverhead")
-    .doc("The amount of non-heap memory to be allocated per executor in cluster mode, " +
-      "in MiB unless otherwise specified.")
+    .doc("The amount of non-heap memory to be allocated per executor, in MiB unless otherwise" +
+      " specified.")
     .version("2.3.0")
     .bytesConf(ByteUnit.MiB)
     .createOptional
diff --git a/docs/configuration.md b/docs/configuration.md
index 36e4f4591d1f9..5a2c6b3e4a0bc 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -274,10 +274,9 @@ of the most common options to set are:
  <td><code>spark.executor.memoryOverhead</code></td>
   <td>executorMemory * 0.10, with minimum of 384 </td>
   <td>
-    Amount of additional memory to be allocated per executor process in cluster mode, in MiB unless
-    otherwise specified. This is memory that accounts for things like VM overheads, interned strings,
-    other native overheads, etc. This tends to grow with the executor size (typically 6-10%).
-    This option is currently supported on YARN and Kubernetes.
+    Amount of additional memory to be allocated per executor process, in MiB unless otherwise specified.
+    This is memory that accounts for things like VM overheads, interned strings, other native overheads, etc.
+    This tends to grow with the executor size (typically 6-10%). This option is currently supported on YARN and Kubernetes.
     <br/>
     <em>Note:</em> Additional memory includes PySpark executor memory 
     (when <code>spark.executor.pyspark.memory</code> is not configured) and memory used by other

From e684720243d9b8c86f04b4b221ff86977c3bd171 Mon Sep 17 00:00:00 2001
From: Chao Sun <sunchao@apple.com>
Date: Thu, 12 Nov 2020 18:02:20 -0800
Subject: [PATCH 1168/1280] [SPARK-33435][SQL][3.0] DSv2: REFRESH TABLE should
 invalidate caches referencing the table

### What changes were proposed in this pull request?

This is a backport for PR #30359.

This changes `RefreshTableExec` in DSv2 to also invalidate caches with references to the target table to be refreshed. The change itself is similar to what's done in #30211. Note that though, since we currently don't support caching a DSv2 table directly, this doesn't add recache logic as in the DSv1 impl. I marked it as a TODO for now.

Note there is some conflicts in the backport: in branch-3.0 `DataSourceV2Strategy` we don't have a `ResolvedTable` when analyzing `RefreshTable` so instead in `RefreshTableExec` this loads the table from the catalog if it exists, and the rest is the same.

### Why are the changes needed?

Currently the behavior in DSv1 and DSv2 is inconsistent w.r.t refreshing table: in DSv1 we invalidate both metadata cache as well as all table caches that are related to the table, but in DSv2 we only do the former. This addresses the issue and make the behavior consistent.

### Does this PR introduce _any_ user-facing change?

Yes, now refreshing a v2 table also invalidate all the related caches.

### How was this patch tested?

Added a new UT.

Closes #30360 from sunchao/SPARK-33435-branch-3.0.

Authored-by: Chao Sun <sunchao@apple.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../datasources/v2/DataSourceV2Strategy.scala      |  2 +-
 .../datasources/v2/RefreshTableExec.scala          | 13 ++++++++++++-
 .../spark/sql/connector/DataSourceV2SQLSuite.scala | 14 ++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index cca80c0cb6d57..877aea178c552 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -130,7 +130,7 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
       }
 
     case RefreshTable(catalog, ident) =>
-      RefreshTableExec(catalog, ident) :: Nil
+      RefreshTableExec(session, catalog, ident) :: Nil
 
     case ReplaceTable(catalog, ident, schema, parts, props, orCreate) =>
       val propsWithOwner = CatalogV2Util.withDefaultOwnership(props)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
index 2a19ff304a9e0..9717b9f954671 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/RefreshTableExec.scala
@@ -17,15 +17,26 @@
 
 package org.apache.spark.sql.execution.datasources.v2
 
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.connector.catalog.{Identifier, TableCatalog}
+import org.apache.spark.sql.connector.catalog.{Identifier, Table, TableCatalog}
 
 case class RefreshTableExec(
+    session: SparkSession,
     catalog: TableCatalog,
     ident: Identifier) extends V2CommandExec {
   override protected def run(): Seq[InternalRow] = {
     catalog.invalidateTable(ident)
+
+    if (catalog.tableExists(ident)) {
+      val table = catalog.loadTable(ident)
+      // invalidate all caches referencing the given table
+      // TODO(SPARK-33437): re-cache the table itself once we support caching a DSv2 table
+      val v2Relation = DataSourceV2Relation.create(table, Some(catalog), Some(ident))
+      session.sharedState.cacheManager.uncacheQuery(session, v2Relation, cascade = true)
+    }
+
     Seq.empty
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index ef5558a83a306..df71a859899b7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -1711,6 +1711,20 @@ class DataSourceV2SQLSuite
     }
   }
 
+  test("SPARK-33435: REFRESH TABLE should invalidate all caches referencing the table") {
+    val tblName = "testcat.ns.t"
+    withTable(tblName) {
+      withTempView("t") {
+        sql(s"CREATE TABLE $tblName (id bigint) USING foo")
+        sql(s"CACHE TABLE t AS SELECT id FROM $tblName")
+
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table("t")).isDefined)
+        sql(s"REFRESH TABLE $tblName")
+        assert(spark.sharedState.cacheManager.lookupCachedData(spark.table("t")).isEmpty)
+      }
+    }
+  }
+
   test("REPLACE TABLE: v1 table") {
     val e = intercept[AnalysisException] {
       sql(s"CREATE OR REPLACE TABLE tbl (a int) USING ${classOf[SimpleScanSource].getName}")

From 921daa8ef817c56918612eec3e8ea4d9967a33ad Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 12 Nov 2020 21:19:51 -0800
Subject: [PATCH 1169/1280] [SPARK-33439][INFRA] Use SERIAL_SBT_TESTS=1 for SQL
 modules

### What changes were proposed in this pull request?

This PR aims to decrease the parallelism of `SQL` module like `Hive` module.

### Why are the changes needed?

GitHub Action `sql - slow tests` become flaky.
- https://github.com/apache/spark/runs/1393670291
- https://github.com/apache/spark/runs/1393088031

### Does this PR introduce _any_ user-facing change?

No. This is dev-only feature.
Although this will increase the running time, but it's better than flakiness.

### How was this patch tested?

Pass the GitHub Action stably.

Closes #30365 from dongjoon-hyun/SPARK-33439.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a70a2b02ce7d18947778d37c8fffb3f1b1b5b154)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .github/workflows/build_and_test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 9b4f41ac9945b..574e1bfd042ee 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -132,8 +132,8 @@ jobs:
     # Run the tests.
     - name: Run tests
       run: |
-        # Hive tests become flaky when running in parallel as it's too intensive.
-        if [[ "$MODULES_TO_TEST" == "hive" ]]; then export SERIAL_SBT_TESTS=1; fi
+        # Hive and SQL tests become flaky when running in parallel as it's too intensive.
+        if [[ "$MODULES_TO_TEST" == "hive" ]] || [[ "$MODULES_TO_TEST" == "sql" ]]; then export SERIAL_SBT_TESTS=1; fi
         mkdir -p ~/.m2
         ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
         rm -rf ~/.m2/repository/org/apache/spark

From 45bdb583624436510d38d8cf719de1020d2058ec Mon Sep 17 00:00:00 2001
From: artiship <meilziner@gmail.com>
Date: Sun, 15 Nov 2020 16:56:35 -0800
Subject: [PATCH 1170/1280] [SPARK-33358][SQL] Return code when command process
 failed

Exit Spark SQL CLI processing loop if one of the commands (sub sql statement) process failed

This is a regression at Apache Spark 3.0.0.

```
$ cat 1.sql
select * from nonexistent_table;
select 2;
```

**Apache Spark 2.4.7**
```
spark-2.4.7-bin-hadoop2.7:$ bin/spark-sql -f 1.sql
20/11/15 16:14:38 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Error in query: Table or view not found: nonexistent_table; line 1 pos 14
```

**Apache Spark 3.0.1**
```
$ bin/spark-sql -f 1.sql
Error in query: Table or view not found: nonexistent_table; line 1 pos 14;
'Project [*]
+- 'UnresolvedRelation [nonexistent_table]

2
Time taken: 2.786 seconds, Fetched 1 row(s)
```

**Apache Hive 1.2.2**
```
apache-hive-1.2.2-bin:$ bin/hive -f 1.sql

Logging initialized using configuration in jar:file:/Users/dongjoon/APACHE/hive-release/apache-hive-1.2.2-bin/lib/hive-common-1.2.2.jar!/hive-log4j.properties
FAILED: SemanticException [Error 10001]: Line 1:14 Table not found 'nonexistent_table'
```

Yes. This is a fix of regression.

Pass the UT.

Closes #30263 from artiship/SPARK-33358.

Authored-by: artiship <meilziner@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 1ae6d64b5fb9c20e41deffaf670a42b0b545477f)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index c7848afd822d5..6abb905eaeeeb 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -499,7 +499,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
             val ignoreErrors = HiveConf.getBoolVar(conf, HiveConf.ConfVars.CLIIGNOREERRORS)
             if (ret != 0 && !ignoreErrors) {
               CommandProcessorFactory.clean(conf.asInstanceOf[HiveConf])
-              ret
+              return ret
             }
           }
         }

From 265363d89e8ffc452e100289f8f1a94cd9d6cc44 Mon Sep 17 00:00:00 2001
From: aof00 <x14562573449@gmail.com>
Date: Mon, 16 Nov 2020 10:32:00 +0900
Subject: [PATCH 1171/1280] [SPARK-33451][DOCS] Change to
 'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes' in
 documentation

### What changes were proposed in this pull request?

In the 'Optimizing Skew Join' section of the following two pages:
1. [https://spark.apache.org/docs/3.0.0/sql-performance-tuning.html](https://spark.apache.org/docs/3.0.0/sql-performance-tuning.html)
2. [https://spark.apache.org/docs/3.0.1/sql-performance-tuning.html](https://spark.apache.org/docs/3.0.1/sql-performance-tuning.html)

The configuration 'spark.sql.adaptive.skewedPartitionThresholdInBytes' should be changed to 'spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes', The former is missing the 'skewJoin'.

### Why are the changes needed?

To document the correct name of configuration

### Does this PR introduce _any_ user-facing change?

Yes, this is a user-facing doc change.

### How was this patch tested?

Jenkins / CI builds in this PR.

Closes #30376 from aof00/doc_change.

Authored-by: aof00 <x14562573449@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 0933f1c6c27c7a087c8dbe485bd08371ce5f5695)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-performance-tuning.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-performance-tuning.md b/docs/sql-performance-tuning.md
index 5d8c3b698c703..49b32e7562e0b 100644
--- a/docs/sql-performance-tuning.md
+++ b/docs/sql-performance-tuning.md
@@ -280,7 +280,7 @@ Data skew can severely downgrade the performance of join queries. This feature d
        <td><code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code></td>
        <td>10</td>
        <td>
-         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewedPartitionThresholdInBytes</code>.
+         A partition is considered as skewed if its size is larger than this factor multiplying the median partition size and also larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code>.
        </td>
        <td>3.0.0</td>
      </tr>

From 26c0404214563bb558662e68ea73357c4f4021ed Mon Sep 17 00:00:00 2001
From: Josh Soref <jsoref@users.noreply.github.com>
Date: Tue, 17 Nov 2020 15:25:42 +0900
Subject: [PATCH 1172/1280] [MINOR][GRAPHX][3.0] Correct typos in the
 sub-modules: graphx, external, and examples

### What changes were proposed in this pull request?

This PR intends to fix typos in the sub-modules: graphx, external, and examples.
Split per holdenk https://github.com/apache/spark/pull/30323#issuecomment-725159710

NOTE: The misspellings have been reported at https://github.com/jsoref/spark/commit/706a726f87a0bbf5e31467fae9015218773db85b#commitcomment-44064356

Backport of #30326

### Why are the changes needed?

Misspelled words make it harder to read / understand content.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

No testing was performed

Closes #30342 from jsoref/branch-3.0-30326.

Authored-by: Josh Soref <jsoref@users.noreply.github.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../spark/examples/streaming/JavaCustomReceiver.java |  2 +-
 .../examples/streaming/JavaNetworkWordCount.java     |  2 +-
 .../streaming/JavaRecoverableNetworkWordCount.java   |  2 +-
 .../examples/streaming/JavaSqlNetworkWordCount.java  |  2 +-
 .../src/main/python/ml/train_validation_split.py     |  2 +-
 examples/src/main/python/sql/arrow.py                |  4 ++--
 .../streaming/recoverable_network_wordcount.py       |  2 +-
 .../main/python/streaming/sql_network_wordcount.py   |  2 +-
 .../spark/examples/streaming/CustomReceiver.scala    |  2 +-
 .../spark/examples/streaming/NetworkWordCount.scala  |  2 +-
 .../streaming/RecoverableNetworkWordCount.scala      |  2 +-
 .../examples/streaming/SqlNetworkWordCount.scala     |  2 +-
 .../streaming/StatefulNetworkWordCount.scala         |  2 +-
 .../spark/sql/jdbc/DockerJDBCIntegrationSuite.scala  |  2 +-
 .../sql/kafka010/KafkaContinuousSourceSuite.scala    |  4 ++--
 .../sql/kafka010/KafkaMicroBatchSourceSuite.scala    | 12 ++++++------
 .../spark/sql/kafka010/KafkaRelationSuite.scala      |  4 ++--
 .../apache/spark/sql/kafka010/KafkaTestUtils.scala   |  4 ++--
 .../spark/streaming/kafka010/KafkaRDDSuite.scala     |  2 +-
 .../examples/streaming/JavaKinesisWordCountASL.java  |  2 +-
 .../examples/streaming/kinesis_wordcount_asl.py      |  2 +-
 .../examples/streaming/KinesisWordCountASL.scala     |  6 +++---
 .../streaming/kinesis/KinesisUtilsPythonHelper.scala |  2 +-
 .../org/apache/spark/graphx/lib/PageRankSuite.scala  |  6 +++---
 24 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
index 47692ec982890..f84a1978de1ad 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java
@@ -67,7 +67,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(1000));
 
     // Create an input stream with the custom receiver on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     JavaReceiverInputDStream<String> lines = ssc.receiverStream(
       new JavaCustomReceiver(args[0], Integer.parseInt(args[1])));
     JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
index b217672def88e..d56134bd99e36 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java
@@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));
 
     // Create a JavaReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
index 45a876decff8b..c3543237fc7a8 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java
@@ -126,7 +126,7 @@ private static JavaStreamingContext createContext(String ip,
     ssc.checkpoint(checkpointDirectory);
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(ip, port);
     JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(SPACE.split(x)).iterator());
     JavaPairDStream<String, Integer> wordCounts = words.mapToPair(s -> new Tuple2<>(s, 1))
diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
index 948d1a2111780..5d30698c93372 100644
--- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java
@@ -59,7 +59,7 @@ public static void main(String[] args) throws Exception {
     JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));
 
     // Create a JavaReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     JavaReceiverInputDStream<String> lines = ssc.socketTextStream(
diff --git a/examples/src/main/python/ml/train_validation_split.py b/examples/src/main/python/ml/train_validation_split.py
index d4f9184bf576e..5e3dc7b3ec2fa 100644
--- a/examples/src/main/python/ml/train_validation_split.py
+++ b/examples/src/main/python/ml/train_validation_split.py
@@ -17,7 +17,7 @@
 
 """
 This example demonstrates applying TrainValidationSplit to split data
-and preform model selection.
+and perform model selection.
 Run with:
 
   bin/spark-submit examples/src/main/python/ml/train_validation_split.py
diff --git a/examples/src/main/python/sql/arrow.py b/examples/src/main/python/sql/arrow.py
index b7d8467172fab..b9fb29f466a40 100644
--- a/examples/src/main/python/sql/arrow.py
+++ b/examples/src/main/python/sql/arrow.py
@@ -68,7 +68,7 @@ def func(s1: pd.Series, s2: pd.Series, s3: pd.DataFrame) -> pd.DataFrame:
         s3['col2'] = s1 + s2.str.len()
         return s3
 
-    # Create a Spark DataFrame that has three columns including a sturct column.
+    # Create a Spark DataFrame that has three columns including a struct column.
     df = spark.createDataFrame(
         [[1, "a string", ("a nested string",)]],
         "long_col long, string_col string, struct_col struct<col1:string>")
@@ -308,7 +308,7 @@ def asof_join(l, r):
     ser_to_frame_pandas_udf_example(spark)
     print("Running pandas_udf example: Series to Series")
     ser_to_ser_pandas_udf_example(spark)
-    print("Running pandas_udf example: Iterator of Series to Iterator of Seires")
+    print("Running pandas_udf example: Iterator of Series to Iterator of Series")
     iter_ser_to_iter_ser_pandas_udf_example(spark)
     print("Running pandas_udf example: Iterator of Multiple Series to Iterator of Series")
     iter_sers_to_iter_ser_pandas_udf_example(spark)
diff --git a/examples/src/main/python/streaming/recoverable_network_wordcount.py b/examples/src/main/python/streaming/recoverable_network_wordcount.py
index a39c4d0b5b8cd..e9f2db97d87e6 100644
--- a/examples/src/main/python/streaming/recoverable_network_wordcount.py
+++ b/examples/src/main/python/streaming/recoverable_network_wordcount.py
@@ -68,7 +68,7 @@ def createContext(host, port, outputPath):
     ssc = StreamingContext(sc, 1)
 
     # Create a socket stream on target ip:port and count the
-    # words in input stream of \n delimited text (eg. generated by 'nc')
+    # words in input stream of \n delimited text (e.g. generated by 'nc')
     lines = ssc.socketTextStream(host, port)
     words = lines.flatMap(lambda line: line.split(" "))
     wordCounts = words.map(lambda x: (x, 1)).reduceByKey(lambda x, y: x + y)
diff --git a/examples/src/main/python/streaming/sql_network_wordcount.py b/examples/src/main/python/streaming/sql_network_wordcount.py
index ab3cfc067994d..22fe26073a690 100644
--- a/examples/src/main/python/streaming/sql_network_wordcount.py
+++ b/examples/src/main/python/streaming/sql_network_wordcount.py
@@ -54,7 +54,7 @@ def getSparkSessionInstance(sparkConf):
     ssc = StreamingContext(sc, 1)
 
     # Create a socket stream on target ip:port and count the
-    # words in input stream of \n delimited text (eg. generated by 'nc')
+    # words in input stream of \n delimited text (e.g. generated by 'nc')
     lines = ssc.socketTextStream(host, int(port))
     words = lines.flatMap(lambda line: line.split(" "))
 
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
index 0f47deaf1021b..626f4b4d3ccdf 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala
@@ -50,7 +50,7 @@ object CustomReceiver {
     val ssc = new StreamingContext(sparkConf, Seconds(1))
 
     // Create an input stream with the custom receiver on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     val lines = ssc.receiverStream(new CustomReceiver(args(0), args(1).toInt))
     val words = lines.flatMap(_.split(" "))
     val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
index 26bb51dde3a1d..7d981dfb949ea 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala
@@ -47,7 +47,7 @@ object NetworkWordCount {
     val ssc = new StreamingContext(sparkConf, Seconds(1))
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
index 243c22e71275c..98c1807b1ed62 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala
@@ -112,7 +112,7 @@ object RecoverableNetworkWordCount {
     ssc.checkpoint(checkpointDirectory)
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     val lines = ssc.socketTextStream(ip, port)
     val words = lines.flatMap(_.split(" "))
     val wordCounts = words.map((_, 1)).reduceByKey(_ + _)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
index 778be7baaeeac..7daa0014e0f1c 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala
@@ -51,7 +51,7 @@ object SqlNetworkWordCount {
     val ssc = new StreamingContext(sparkConf, Seconds(2))
 
     // Create a socket stream on target ip:port and count the
-    // words in input stream of \n delimited text (eg. generated by 'nc')
+    // words in input stream of \n delimited text (e.g. generated by 'nc')
     // Note that no duplication in storage level only for running locally.
     // Replication necessary in distributed scenario for fault tolerance.
     val lines = ssc.socketTextStream(args(0), args(1).toInt, StorageLevel.MEMORY_AND_DISK_SER)
diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
index 46f01edf7deec..8a5fcda9cd990 100644
--- a/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala
@@ -52,7 +52,7 @@ object StatefulNetworkWordCount {
     val initialRDD = ssc.sparkContext.parallelize(List(("hello", 1), ("world", 1)))
 
     // Create a ReceiverInputDStream on target ip:port and count the
-    // words in input stream of \n delimited test (eg. generated by 'nc')
+    // words in input stream of \n delimited test (e.g. generated by 'nc')
     val lines = ssc.socketTextStream(args(0), args(1).toInt)
     val words = lines.flatMap(_.split(" "))
     val wordDstream = words.map(x => (x, 1))
diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
index 519a5cddb3639..996e697bf62ca 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala
@@ -44,7 +44,7 @@ abstract class DatabaseOnDocker {
   val env: Map[String, String]
 
   /**
-   * Wheather or not to use ipc mode for shared memory when starting docker image
+   * Whether or not to use ipc mode for shared memory when starting docker image
    */
   val usesIpc: Boolean
 
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
index 76c25980fc33f..43cbe28fc1fe1 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaContinuousSourceSuite.scala
@@ -33,7 +33,7 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
@@ -99,7 +99,7 @@ class KafkaContinuousSourceSuite extends KafkaSourceSuiteBase with KafkaContinuo
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index bdad214a91343..2a3e135b0e63e 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -567,7 +567,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
     val rows = spark.table("kafkaWatermark").collect()
     assert(rows.length === 1, s"Unexpected results: ${rows.toList}")
     val row = rows(0)
-    // We cannot check the exact window start time as it depands on the time that messages were
+    // We cannot check the exact window start time as it depends on the time that messages were
     // inserted by the producer. So here we just use a low bound to make sure the internal
     // conversion works.
     assert(
@@ -833,7 +833,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 
     val topicPartition = new TopicPartition(topic, 0)
     // The message values are the same as their offsets to make the test easy to follow
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       testStream(mapped)(
         StartStream(Trigger.ProcessingTime(100), clock),
         waitUntilBatchProcessed,
@@ -956,7 +956,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
 
     val topicPartition = new TopicPartition(topic, 0)
     // The message values are the same as their offsets to make the test easy to follow
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       testStream(mapped)(
         StartStream(Trigger.ProcessingTime(100), clock),
         waitUntilBatchProcessed,
@@ -1047,7 +1047,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
       .load()
       .select($"value".as[String])
 
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       producer.beginTransaction()
       (0 to 3).foreach { i =>
         producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
@@ -1063,7 +1063,7 @@ abstract class KafkaMicroBatchSourceSuiteBase extends KafkaSourceSuiteBase {
         // this case, if we forget to reset `FetchedData._nextOffsetInFetchedData` or
         // `FetchedData._offsetAfterPoll` (See SPARK-25495), the next batch will see incorrect
         // values and return wrong results hence fail the test.
-        testUtils.withTranscationalProducer { producer =>
+        testUtils.withTransactionalProducer { producer =>
           producer.beginTransaction()
           (4 to 7).foreach { i =>
             producer.send(new ProducerRecord[String, String](topic, i.toString)).get()
@@ -1776,7 +1776,7 @@ abstract class KafkaSourceSuiteBase extends KafkaSourceTest {
     withTable(table) {
       val topic = newTopic()
       testUtils.createTopic(topic)
-      testUtils.withTranscationalProducer { producer =>
+      testUtils.withTransactionalProducer { producer =>
         val df = spark
           .readStream
           .format("kafka")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
index e5f3a229622e1..6f5dc0bb081ba 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
@@ -503,7 +503,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   test("read Kafka transactional messages: read_committed") {
     val topic = newTopic()
     testUtils.createTopic(topic)
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       val df = spark
         .read
         .format("kafka")
@@ -552,7 +552,7 @@ abstract class KafkaRelationSuiteBase extends QueryTest with SharedSparkSession
   test("read Kafka transactional messages: read_uncommitted") {
     val topic = newTopic()
     testUtils.createTopic(topic)
-    testUtils.withTranscationalProducer { producer =>
+    testUtils.withTransactionalProducer { producer =>
       val df = spark
         .read
         .format("kafka")
diff --git a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
index a0a113c7a9449..942e2ad5e3727 100644
--- a/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
+++ b/external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
@@ -565,7 +565,7 @@ class KafkaTestUtils(
   }
 
   /** Call `f` with a `KafkaProducer` that has initialized transactions. */
-  def withTranscationalProducer(f: KafkaProducer[String, String] => Unit): Unit = {
+  def withTransactionalProducer(f: KafkaProducer[String, String] => Unit): Unit = {
     val props = producerConfiguration
     props.put("transactional.id", UUID.randomUUID().toString)
     val producer = new KafkaProducer[String, String](props)
@@ -614,7 +614,7 @@ class KafkaTestUtils(
     // ensure that logs from all replicas are deleted if delete topic is marked successful
     assert(servers.forall(server => topicAndPartitions.forall(tp =>
       server.getLogManager().getLog(tp).isEmpty)),
-      s"topic $topic still exists in log mananger")
+      s"topic $topic still exists in log manager")
     // ensure that topic is removed from all cleaner offsets
     assert(servers.forall(server => topicAndPartitions.forall { tp =>
       val checkpoints = server.getLogManager().liveLogDirs.map { logDir =>
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
index d6123e16dd238..2053d3655d860 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
@@ -42,7 +42,7 @@ class KafkaRDDSuite extends SparkFunSuite with BeforeAndAfterAll {
   private val sparkConf = new SparkConf().setMaster("local[4]")
     .setAppName(this.getClass.getSimpleName)
     // Set a timeout of 10 seconds that's going to be used to fetch topics/partitions from kafka.
-    // Othewise the poll timeout defaults to 2 minutes and causes test cases to run longer.
+    // Otherwise the poll timeout defaults to 2 minutes and causes test cases to run longer.
     .set("spark.streaming.kafka.consumer.poll.ms", "10000")
 
   private var sc: SparkContext = _
diff --git a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
index d704aeb507518..244873af70de9 100644
--- a/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
+++ b/external/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
@@ -49,7 +49,7 @@
  *
  * Usage: JavaKinesisWordCountASL [app-name] [stream-name] [endpoint-url] [region-name]
  *   [app-name] is the name of the consumer app, used to track the read data in DynamoDB
- *   [stream-name] name of the Kinesis stream (ie. mySparkStream)
+ *   [stream-name] name of the Kinesis stream (i.e. mySparkStream)
  *   [endpoint-url] endpoint of the Kinesis service
  *     (e.g. https://kinesis.us-east-1.amazonaws.com)
  *
diff --git a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 5370b793897bb..66c9ed80f3b33 100644
--- a/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/external/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -23,7 +23,7 @@
 
   Usage: kinesis_wordcount_asl.py <app-name> <stream-name> <endpoint-url> <region-name>
     <app-name> is the name of the consumer app, used to track the read data in DynamoDB
-    <stream-name> name of the Kinesis stream (ie. mySparkStream)
+    <stream-name> name of the Kinesis stream (i.e. mySparkStream)
     <endpoint-url> endpoint of the Kinesis service
       (e.g. https://kinesis.us-east-1.amazonaws.com)
     <region-name> region name of the Kinesis endpoint (e.g. us-east-1)
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index bbb6008c2dddf..d6a9160eed98e 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -43,7 +43,7 @@ import org.apache.spark.streaming.kinesis.KinesisInputDStream
  *
  * Usage: KinesisWordCountASL <app-name> <stream-name> <endpoint-url> <region-name>
  *   <app-name> is the name of the consumer app, used to track the read data in DynamoDB
- *   <stream-name> name of the Kinesis stream (ie. mySparkStream)
+ *   <stream-name> name of the Kinesis stream (i.e. mySparkStream)
  *   <endpoint-url> endpoint of the Kinesis service
  *     (e.g. https://kinesis.us-east-1.amazonaws.com)
  *
@@ -167,9 +167,9 @@ object KinesisWordCountASL extends Logging {
  * Usage: KinesisWordProducerASL <stream-name> <endpoint-url> \
  *   <records-per-sec> <words-per-record>
  *
- *   <stream-name> is the name of the Kinesis stream (ie. mySparkStream)
+ *   <stream-name> is the name of the Kinesis stream (i.e. mySparkStream)
  *   <endpoint-url> is the endpoint of the Kinesis service
- *     (ie. https://kinesis.us-east-1.amazonaws.com)
+ *     (i.e. https://kinesis.us-east-1.amazonaws.com)
  *   <records-per-sec> is the rate of records per second to put onto the stream
  *   <words-per-record> is the number of words per record
  *
diff --git a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
index c89dedd3366d1..0056438c4eefb 100644
--- a/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
+++ b/external/kinesis-asl/src/main/scala/org/apache/spark/streaming/kinesis/KinesisUtilsPythonHelper.scala
@@ -46,7 +46,7 @@ private class KinesisUtilsPythonHelper {
     // scalastyle:on
     if (!(stsAssumeRoleArn != null && stsSessionName != null && stsExternalId != null)
         && !(stsAssumeRoleArn == null && stsSessionName == null && stsExternalId == null)) {
-      throw new IllegalArgumentException("stsAssumeRoleArn, stsSessionName, and stsExtenalId " +
+      throw new IllegalArgumentException("stsAssumeRoleArn, stsSessionName, and stsExternalId " +
         "must all be defined or all be null")
     }
     if (awsAccessKeyId == null && awsSecretKey != null) {
diff --git a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
index a5e2fc5c9a74f..8008a89c6cd5f 100644
--- a/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
+++ b/graphx/src/test/scala/org/apache/spark/graphx/lib/PageRankSuite.scala
@@ -274,8 +274,8 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
     withSpark { sc =>
       // Check that implementation can handle large vertexIds, SPARK-25149
       val vertexIdOffset = Int.MaxValue.toLong + 1
-      val sourceOffest = 4
-      val source = vertexIdOffset + sourceOffest
+      val sourceOffset = 4
+      val source = vertexIdOffset + sourceOffset
       val numIter = 10
       val vertices = vertexIdOffset until vertexIdOffset + numIter
       val chain1 = vertices.zip(vertices.tail)
@@ -285,7 +285,7 @@ class PageRankSuite extends SparkFunSuite with LocalSparkContext {
       val tol = 0.0001
       val errorTol = 1.0e-1
 
-      val a = resetProb / (1 - Math.pow(1 - resetProb, numIter - sourceOffest))
+      val a = resetProb / (1 - Math.pow(1 - resetProb, numIter - sourceOffset))
       // We expect the rank to decay as (1 - resetProb) ^ distance
       val expectedRanks = sc.parallelize(vertices).map { vid =>
         val rank = if (vid < source) {

From c301d9cdd8faa1e288a64cf630375eb3f657cab9 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 18 Nov 2020 20:14:35 -0800
Subject: [PATCH 1173/1280] [SPARK-33464][INFRA][3.0] Add/remove (un)necessary
 cache and restructure GitHub Actions yaml

### What changes were proposed in this pull request?

This PR backports https://github.com/apache/spark/pull/30391. Note that it's a partial backport.

This PR proposes:
- Add `~/.sbt` directory into the build cache, see also https://github.com/sbt/sbt/issues/3681
- ~Move `hadoop-2` below to put up together with `java-11` and `scala-213`, see https://github.com/apache/spark/pull/30391#discussion_r524881430~
- Remove unnecessary `.m2` cache if you run SBT tests only.
- Remove `rm ~/.m2/repository/org/apache/spark`. If you don't `sbt publishLocal` or `mvn install`, we don't need to care about it.
- ~Use Java 8 in Scala 2.13 build. We can switch the Java version to 11 used for release later.~
- Add caches into linters. The linter scripts uses `sbt` in, for example, `./dev/lint-scala`, and uses `mvn` in, for example, `./dev/lint-java`. Also, it requires to `sbt package` in Jekyll build, see: https://github.com/apache/spark/blob/master/docs/_plugins/copy_api_dirs.rb#L160-L161. We need full caches here for SBT, Maven and build tools.
- Use the same syntax of Java version, 1.8 -> 8.

### Why are the changes needed?

- Remove unnecessary stuff
- Cache what we can in the build

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

It will be tested in GitHub Actions build at the current PR

Closes #30416 from HyukjinKwon/SPARK-33464-3.0.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .github/workflows/build_and_test.yml | 84 +++++++++++++---------------
 1 file changed, 40 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 574e1bfd042ee..f7ffbca0b402c 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         java:
-          - 1.8
+          - 8
         hadoop:
           - hadoop2.7
         hive:
@@ -46,26 +46,26 @@ jobs:
         include:
           # Hive tests
           - modules: hive
-            java: 1.8
+            java: 8
             hadoop: hadoop2.7
             hive: hive2.3
             included-tags: org.apache.spark.tags.SlowHiveTest
             comment: "- slow tests"
           - modules: hive
-            java: 1.8
+            java: 8
             hadoop: hadoop2.7
             hive: hive2.3
             excluded-tags: org.apache.spark.tags.SlowHiveTest,org.apache.spark.tags.GitHubActionsUnstableTest
             comment: "- other tests"
           # SQL tests
           - modules: sql
-            java: 1.8
+            java: 8
             hadoop: hadoop2.7
             hive: hive2.3
             included-tags: org.apache.spark.tags.ExtendedSQLTest
             comment: "- slow tests"
           - modules: sql
-            java: 1.8
+            java: 8
             hadoop: hadoop2.7
             hive: hive2.3
             excluded-tags: org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.GitHubActionsUnstableTest
@@ -94,16 +94,10 @@ jobs:
           build/zinc-*
           build/scala-*
           build/*.jar
+          ~/.sbt
         key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
         restore-keys: |
           build-
-    - name: Cache Maven local repository
-      uses: actions/cache@v2
-      with:
-        path: ~/.m2/repository
-        key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          ${{ matrix.java }}-${{ matrix.hadoop }}-maven-
     - name: Cache Ivy local repository
       uses: actions/cache@v2
       with:
@@ -111,7 +105,7 @@ jobs:
         key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
         restore-keys: |
           ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-
-    - name: Install JDK ${{ matrix.java }}
+    - name: Install Java ${{ matrix.java }}
       uses: actions/setup-java@v1
       with:
         java-version: ${{ matrix.java }}
@@ -134,9 +128,7 @@ jobs:
       run: |
         # Hive and SQL tests become flaky when running in parallel as it's too intensive.
         if [[ "$MODULES_TO_TEST" == "hive" ]] || [[ "$MODULES_TO_TEST" == "sql" ]]; then export SERIAL_SBT_TESTS=1; fi
-        mkdir -p ~/.m2
         ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
-        rm -rf ~/.m2/repository/org/apache/spark
     - name: Upload test results to report
       if: always()
       uses: actions/upload-artifact@v2
@@ -185,16 +177,10 @@ jobs:
           build/zinc-*
           build/scala-*
           build/*.jar
+          ~/.sbt
         key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
         restore-keys: |
           build-
-    - name: Cache Maven local repository
-      uses: actions/cache@v2
-      with:
-        path: ~/.m2/repository
-        key: pyspark-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          pyspark-maven-
     - name: Cache Ivy local repository
       uses: actions/cache@v2
       with:
@@ -214,24 +200,22 @@ jobs:
     # Run the tests.
     - name: Run tests
       run: |
-        mkdir -p ~/.m2
         ./dev/run-tests --parallelism 2 --modules "$MODULES_TO_TEST"
-        rm -rf ~/.m2/repository/org/apache/spark
     - name: Upload test results to report
       if: always()
       uses: actions/upload-artifact@v2
       with:
-        name: test-results-${{ matrix.modules }}--1.8-hadoop2.7-hive2.3
+        name: test-results-${{ matrix.modules }}--8-hadoop2.7-hive2.3
         path: "**/target/test-reports/*.xml"
     - name: Upload unit tests log files
       if: failure()
       uses: actions/upload-artifact@v2
       with:
-        name: unit-tests-log-${{ matrix.modules }}--1.8-hadoop2.7-hive2.3
+        name: unit-tests-log-${{ matrix.modules }}--8-hadoop2.7-hive2.3
         path: "**/target/unit-tests.log"
 
   sparkr:
-    name: Build modules - sparkr
+    name: "Build modules: sparkr"
     runs-on: ubuntu-20.04
     container:
        image: dongjoon/apache-spark-github-action-image:20201025
@@ -254,16 +238,10 @@ jobs:
           build/zinc-*
           build/scala-*
           build/*.jar
+          ~/.sbt
         key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
         restore-keys: |
           build-
-    - name: Cache Maven local repository
-      uses: actions/cache@v2
-      with:
-        path: ~/.m2/repository
-        key: sparkr-maven-${{ hashFiles('**/pom.xml') }}
-        restore-keys: |
-          sparkr-maven-
     - name: Cache Ivy local repository
       uses: actions/cache@v2
       with:
@@ -273,24 +251,22 @@ jobs:
           sparkr-ivy-
     - name: Run tests
       run: |
-        mkdir -p ~/.m2
         # The followings are also used by `r-lib/actions/setup-r` to avoid
         # R issues at docker environment
         export TZ=UTC
         export _R_CHECK_SYSTEM_CLOCK_=FALSE
         ./dev/run-tests --parallelism 2 --modules sparkr
-        rm -rf ~/.m2/repository/org/apache/spark
     - name: Upload test results to report
       if: always()
       uses: actions/upload-artifact@v2
       with:
-        name: test-results-sparkr--1.8-hadoop2.7-hive2.3
+        name: test-results-sparkr--8-hadoop2.7-hive2.3
         path: "**/target/test-reports/*.xml"
     - name: Upload unit tests log files
       if: failure()
       uses: actions/upload-artifact@v2
       with:
-        name: unit-tests-log-sparkr--1.8-hadoop2.7-hive2.3
+        name: unit-tests-log-sparkr--8-hadoop2.7-hive2.3
         path: "**/target/unit-tests.log"
 
   # Static analysis, and documentation build
@@ -300,17 +276,37 @@ jobs:
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
+    # Cache local repositories. Note that GitHub Actions cache has a 2G limit.
+    - name: Cache Scala, SBT, Maven and Zinc
+      uses: actions/cache@v2
+      with:
+        path: |
+          build/apache-maven-*
+          build/zinc-*
+          build/scala-*
+          build/*.jar
+          ~/.sbt
+        key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+        restore-keys: |
+          build-
+    - name: Cache Ivy local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.ivy2/cache
+        key: docs-ivy-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+        restore-keys: |
+          docs-ivy-
     - name: Cache Maven local repository
       uses: actions/cache@v2
       with:
         path: ~/.m2/repository
-        key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
+        key: docs-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
           docs-maven-
-    - name: Install JDK 1.8
+    - name: Install Java 8
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 8
     - name: Install Python 3.6
       uses: actions/setup-python@v2
       with:
@@ -356,8 +352,8 @@ jobs:
         cd docs
         jekyll build
 
-  java11:
-    name: Java 11 build
+  java-11:
+    name: Java 11 build with Maven
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout Spark repository
@@ -377,6 +373,6 @@ jobs:
       run: |
         export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
         export MAVEN_CLI_OPTS="--no-transfer-progress"
-        mkdir -p ~/.m2
+        # It uses Maven's 'install' intentionally, see https://github.com/apache/spark/pull/26414.
         ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -Phadoop-cloud -Djava.version=11 install
         rm -rf ~/.m2/repository/org/apache/spark

From 1101938d6189f5d4ecf1ffc23eee841e038722d7 Mon Sep 17 00:00:00 2001
From: Yuming Wang <yumwang@ebay.com>
Date: Thu, 19 Nov 2020 14:01:42 +0800
Subject: [PATCH 1174/1280] [SPARK-27421][SQL] Fix filter for int column and
 value class java.lang.String when pruning partition column

### What changes were proposed in this pull request?

This pr fix filter for int column and value class java.lang.String when pruning partition column.

How to reproduce this issue:
```scala
spark.sql("CREATE table test (name STRING) partitioned by (id int) STORED AS PARQUET")
spark.sql("CREATE VIEW test_view as select cast(id as string) as id, name from test")
spark.sql("SELECT * FROM test_view WHERE id = '0'").explain
```
```
20/11/15 06:19:01 INFO audit: ugi=root ip=unknown-ip-addr cmd=get_partitions_by_filter : db=default tbl=test
20/11/15 06:19:01 INFO MetaStoreDirectSql: Unable to push down SQL filter: Cannot push down filter for int column and value class java.lang.String
20/11/15 06:19:01 ERROR SparkSQLDriver: Failed in [SELECT * FROM test_view WHERE id = '0']
java.lang.RuntimeException: Caught Hive MetaException attempting to get partition metadata by filter from Hive. You can set the Spark configuration setting spark.sql.hive.manageFilesourcePartitions to false to work around this problem, however this will result in degraded performance. Please report a bug: https://issues.apache.org/jira/browse/SPARK
 at org.apache.spark.sql.hive.client.Shim_v0_13.getPartitionsByFilter(HiveShim.scala:828)
 at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$getPartitionsByFilter$1(HiveClientImpl.scala:745)
 at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:294)
 at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:227)
 at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:226)
 at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:276)
 at org.apache.spark.sql.hive.client.HiveClientImpl.getPartitionsByFilter(HiveClientImpl.scala:743)
```

### Why are the changes needed?

Fix bug.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Unit test.

Closes #30380 from wangyum/SPARK-27421.

Authored-by: Yuming Wang <yumwang@ebay.com>
Signed-off-by: Yuming Wang <yumwang@ebay.com>
(cherry picked from commit 014e1fbb3aba81a803c963fc0b7f4a8d1d70e253)
Signed-off-by: Yuming Wang <yumwang@ebay.com>
---
 .../org/apache/spark/sql/hive/client/HiveShim.scala      | 2 +-
 .../sql/hive/client/HivePartitionFilteringSuite.scala    | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index c7cbf32bff7a0..d11bf94a1fd86 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -729,7 +729,7 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
       def unapply(expr: Expression): Option[Attribute] = {
         expr match {
           case attr: Attribute => Some(attr)
-          case Cast(child @ AtomicType(), dt: AtomicType, _)
+          case Cast(child @ IntegralType(), dt: IntegralType, _)
               if Cast.canUpCast(child.dataType.asInstanceOf[AtomicType], dt) => unapply(child)
           case _ => None
         }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
index 2d615f6fdc261..edb1a27df806f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HivePartitionFilteringSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType, StructType}
+import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType, StringType, StructType}
 import org.apache.spark.util.Utils
 
 class HivePartitionFilteringSuite(version: String)
@@ -276,6 +276,13 @@ class HivePartitionFilteringSuite(version: String)
     buildClient(new Configuration(), sharesHadoopClasses = false)
   }
 
+  test("getPartitionsByFilter: chunk in ('ab', 'ba') and ((cast(ds as string)>'20170102')") {
+    val day = (20170101 to 20170103, 0 to 4, Seq("ab", "ba"))
+    testMetastorePartitionFiltering(
+      attr("chunk").in("ab", "ba") && (attr("ds").cast(StringType) > "20170102"),
+      day :: Nil)
+  }
+
   private def testMetastorePartitionFiltering(
       filterExpr: Expression,
       expectedDs: Seq[Int],

From 6b7172bc3b8275df9b88ea8d22b331acedaf75bb Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 19 Nov 2020 19:42:20 +0900
Subject: [PATCH 1175/1280] [SPARK-33483][INFRA][TESTS][3.0] Fix rat exclusion
 patterns and add a LICENSE

### What changes were proposed in this pull request?

This PR fixes the RAT exclusion rule which was originated from SPARK-1144 (Apache Spark 1.0)

### Why are the changes needed?

This prevents the situation like https://github.com/apache/spark/pull/30415.

Currently, it missed `catalog` directory due to `.log` rule.
```
$ dev/check-license
Could not find Apache license headers in the following files:
 !????? /Users/dongjoon/APACHE/spark-merge/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/MetadataColumn.java
 !????? /Users/dongjoon/APACHE/spark-merge/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsMetadataColumns.java
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CI with the new rule.

Closes #30424 from dongjoon-hyun/SPARK-33483-3.0.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/.rat-excludes                             | 28 ++++++++++++-------
 .../resources/data/scripts/test_transform.py  | 18 ++++++++++++
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 7ce00b7d19a56..20a4002def2fb 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -39,11 +39,11 @@ jquery.dataTables.1.10.18.min.js
 jquery.mustache.js
 jsonFormatter.min.css
 jsonFormatter.min.js
-.*avsc
-.*txt
-.*json
-.*data
-.*log
+.*\.avsc
+.*\.txt
+.*\.json
+.*\.data
+.*\.log
 pyspark-coverage-site/
 cloudpickle.py
 heapq3.py
@@ -96,17 +96,17 @@ local-1430917381535_2
 DESCRIPTION
 NAMESPACE
 test_support/*
-.*Rd
+.*\.Rd
 help/*
 html/*
 INDEX
 .lintr
 gen-java.*
-.*avpr
-.*parquet
+.*\.avpr
+.*\.parquet
 spark-deps-.*
-.*csv
-.*tsv
+.*\.csv
+.*\.tsv
 .*\.sql
 .Rbuildignore
 META-INF/*
@@ -119,3 +119,11 @@ vote.tmpl
 SessionManager.java
 SessionHandler.java
 GangliaReporter.java
+_metadata
+_SUCCESS
+part-00000
+.*\.res
+flights_tiny.txt.1
+over1k
+over10k
+exported_table/*
diff --git a/sql/hive/src/test/resources/data/scripts/test_transform.py b/sql/hive/src/test/resources/data/scripts/test_transform.py
index ac6d11d8b919c..dedb370f6c90e 100755
--- a/sql/hive/src/test/resources/data/scripts/test_transform.py
+++ b/sql/hive/src/test/resources/data/scripts/test_transform.py
@@ -1,3 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
 import sys
 
 delim = sys.argv[1]

From d7c2daebeed2ac2b858dbe91025894f0ff964a21 Mon Sep 17 00:00:00 2001
From: liucht <liucht@inspur.com>
Date: Fri, 20 Nov 2020 22:19:35 +0900
Subject: [PATCH 1176/1280] [SPARK-33422][DOC] Fix the correct display of left
 menu item

### What changes were proposed in this pull request?
Limit the height of the menu area on the left to display vertical scroll bar

### Why are the changes needed?

The bottom menu item cannot be displayed when the left menu tree is long

### Does this PR introduce any user-facing change?

Yes, if the menu item shows more, you'll see it by pulling down the vertical scroll bar

before:
![image](https://user-images.githubusercontent.com/28332082/98805115-16995d80-2452-11eb-933a-3b72c14bea78.png)

after:
![image](https://user-images.githubusercontent.com/28332082/98805418-7e4fa880-2452-11eb-9a9b-8d265078297c.png)

### How was this patch tested?
NA

Closes #30335 from liucht-inspur/master.

Authored-by: liucht <liucht@inspur.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit cbc8be24c896ed25be63ef9a111ff015af4fabec)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/css/main.css | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/css/main.css b/docs/css/main.css
index bb34d6e9bec0d..b873992dee8d0 100755
--- a/docs/css/main.css
+++ b/docs/css/main.css
@@ -79,6 +79,7 @@ body .container-wrapper {
   margin-right: auto;
   border-radius: 15px;
   position: relative;
+  min-height: 100vh;
 }
 
 .title {
@@ -159,6 +160,7 @@ a:hover code {
   max-width: 914px;
   line-height: 1.6; /* Inspired by Github's wiki style */
   padding-left: 30px;
+  min-height: 100vh;
 }
 
 .dropdown-menu {
@@ -239,6 +241,7 @@ a.anchorjs-link:hover { text-decoration: none; }
   border-bottom-width: 0px;
   margin-top: 0px;
   width: 210px;
+  height: 80%;
   float: left;
   position: fixed;
   overflow-y: scroll;

From 1e525c1156ee5558bda3937751588a50bc132c38 Mon Sep 17 00:00:00 2001
From: allisonwang-db <66282705+allisonwang-db@users.noreply.github.com>
Date: Fri, 20 Nov 2020 09:47:54 -0800
Subject: [PATCH 1177/1280] [SPARK-33472][SQL][3.0] Adjust RemoveRedundantSorts
 rule order

Backport #30373 for branch-3.0.

### What changes were proposed in this pull request?
This PR switched the order for the rule `RemoveRedundantSorts` and `EnsureRequirements` so that `EnsureRequirements` will be invoked before `RemoveRedundantSorts` to avoid IllegalArgumentException when instantiating PartitioningCollection.

### Why are the changes needed?
`RemoveRedundantSorts` rule uses SparkPlan's `outputPartitioning` to check whether a sort node is redundant. Currently, it is added before `EnsureRequirements`. Since `PartitioningCollection` requires left and right partitioning to have the same number of partitions, which is not necessarily true before applying `EnsureRequirements`, the rule can fail with the following exception:
```
IllegalArgumentException: requirement failed: PartitioningCollection requires all of its partitionings have the same numPartitions.
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Unit test

Closes #30438 from allisonwang-db/spark-33472-3.0.

Authored-by: allisonwang-db <66282705+allisonwang-db@users.noreply.github.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/QueryExecution.scala  |  4 ++-
 .../spark/sql/execution/SparkPlan.scala       |  7 +++++-
 .../adaptive/AdaptiveSparkPlanExec.scala      |  4 +--
 .../execution/RemoveRedundantSortsSuite.scala | 25 +++++++++++++++++++
 4 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 574a67fecf936..7f5a5e3c2333c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -297,8 +297,10 @@ object QueryExecution {
     Seq(
       PlanDynamicPruningFilters(sparkSession),
       PlanSubqueries(sparkSession),
-      RemoveRedundantSorts(sparkSession.sessionState.conf),
       EnsureRequirements(sparkSession.sessionState.conf),
+      // `RemoveRedundantSorts` needs to be added before `EnsureRequirements` to guarantee the same
+      // number of partitions when instantiating PartitioningCollection.
+      RemoveRedundantSorts(sparkSession.sessionState.conf),
       ApplyColumnarRulesAndInsertTransitions(sparkSession.sessionState.conf,
         sparkSession.sessionState.columnarRules),
       CollapseCodegenStages(sparkSession.sessionState.conf),
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
index ead8c00031112..062aa69b3adb3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala
@@ -135,7 +135,12 @@ abstract class SparkPlan extends QueryPlan[SparkPlan] with Logging with Serializ
   def longMetric(name: String): SQLMetric = metrics(name)
 
   // TODO: Move to `DistributedPlan`
-  /** Specifies how data is partitioned across different nodes in the cluster. */
+  /**
+   * Specifies how data is partitioned across different nodes in the cluster.
+   * Note this method may fail if it is invoked before `EnsureRequirements` is applied
+   * since `PartitioningCollection` requires all its partitionings to have
+   * the same number of partitions.
+   */
   def outputPartitioning: Partitioning = UnknownPartitioning(0) // TODO: WRONG WIDTH!
 
   /**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
index 4e73f06f36f1d..187827ca6005e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
@@ -90,8 +90,8 @@ case class AdaptiveSparkPlanExec(
   // plan should reach a final status of query stages (i.e., no more addition or removal of
   // Exchange nodes) after running these rules.
   private def queryStagePreparationRules: Seq[Rule[SparkPlan]] = Seq(
-    removeRedundantSorts,
-    ensureRequirements
+    ensureRequirements,
+    removeRedundantSorts
   ) ++ context.session.sessionState.queryStagePrepRules
 
   // A list of physical optimizer rules to be applied to a new stage before its execution. These
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
index 1978d22cb87bb..9cea81239f37d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/RemoveRedundantSortsSuite.scala
@@ -18,7 +18,9 @@
 package org.apache.spark.sql.execution
 
 import org.apache.spark.sql.{DataFrame, QueryTest}
+import org.apache.spark.sql.catalyst.plans.physical.{RangePartitioning, UnknownPartitioning}
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper, DisableAdaptiveExecutionSuite, EnableAdaptiveExecutionSuite}
+import org.apache.spark.sql.execution.joins.SortMergeJoinExec
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -99,6 +101,29 @@ abstract class RemoveRedundantSortsSuiteBase
       }
     }
   }
+
+  test("SPARK-33472: shuffled join with different left and right side partition numbers") {
+    withTempView("t1", "t2") {
+      spark.range(0, 100, 1, 2).select('id as "key").createOrReplaceTempView("t1")
+      (0 to 100).toDF("key").createOrReplaceTempView("t2")
+
+      val query = """
+        |SELECT /*+ MERGE(t1) */ t1.key
+        |FROM t1 JOIN t2 ON t1.key = t2.key
+        |WHERE t1.key > 10 AND t2.key < 50
+        |ORDER BY t1.key ASC
+      """.stripMargin
+
+      val df = sql(query)
+      val sparkPlan = df.queryExecution.sparkPlan
+      val join = sparkPlan.collect { case j: SortMergeJoinExec => j }.head
+      val leftPartitioning = join.left.outputPartitioning
+      assert(leftPartitioning.isInstanceOf[RangePartitioning])
+      assert(leftPartitioning.numPartitions == 2)
+      assert(join.right.outputPartitioning == UnknownPartitioning(0))
+      checkSorts(query, 3, 3)
+    }
+  }
 }
 
 class RemoveRedundantSortsSuite extends RemoveRedundantSortsSuiteBase

From b70584f2894243b29b0549ca9d55b51074b46781 Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Mon, 23 Nov 2020 10:38:40 +0900
Subject: [PATCH 1178/1280] [MINOR][INFRA] Suppress warning in check-license

### What changes were proposed in this pull request?
This PR aims to suppress the warning `File exists` in check-license

### Why are the changes needed?

**BEFORE**
```
% dev/check-license
Attempting to fetch rat
RAT checks passed.

% dev/check-license
mkdir: target: File exists
RAT checks passed.
```

**AFTER**
```
% dev/check-license
Attempting to fetch rat
RAT checks passed.

% dev/check-license
RAT checks passed.
```

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Manually do dev/check-license twice.

Closes #30460 from williamhyun/checklicense.

Authored-by: William Hyun <williamhyun3@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit a45923852342ce3f9454743a71740b09e6efe859)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/check-license | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/check-license b/dev/check-license
index 0cc17ffe55c67..bd255954d6db4 100755
--- a/dev/check-license
+++ b/dev/check-license
@@ -67,7 +67,7 @@ mkdir -p "$FWDIR"/lib
     exit 1
 }
 
-mkdir target
+mkdir -p target
 $java_cmd -jar "$rat_jar" -E "$FWDIR"/dev/.rat-excludes -d "$FWDIR" > target/rat-results.txt
 
 if [ $? -ne 0 ]; then

From 200417e47ac400a48af61a2ce119da0041b93712 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 23 Nov 2020 19:35:58 -0800
Subject: [PATCH 1179/1280] [SPARK-33524][SQL][TESTS] Change `InMemoryTable`
 not to use Tuple.hashCode for `BucketTransform`

This PR aims to change `InMemoryTable` not to use `Tuple.hashCode` for `BucketTransform`.

SPARK-32168 made `InMemoryTable` to handle `BucketTransform` as a hash of `Tuple` which is dependents on Scala versions.
- https://github.com/apache/spark/blob/master/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala#L159

**Scala 2.12.10**
```scala
$ bin/scala
Welcome to Scala 2.12.10 (OpenJDK 64-Bit Server VM, Java 1.8.0_272).
Type in expressions for evaluation. Or try :help.

scala> (1, 1).hashCode
res0: Int = -2074071657
```

**Scala 2.13.3**
```scala
Welcome to Scala 2.13.3 (OpenJDK 64-Bit Server VM, Java 1.8.0_272).
Type in expressions for evaluation. Or try :help.

scala> (1, 1).hashCode
val res0: Int = -1669302457
```

Yes. This is a correctness issue.

Pass the UT with both Scala 2.12/2.13.

Closes #30477 from dongjoon-hyun/SPARK-33524.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 8380e00419281cd1b1fc5706d23d5231356a3379)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/sql/connector/InMemoryTable.scala  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
index 616fc72320caf..98b6a3bf682ad 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/InMemoryTable.scala
@@ -128,7 +128,9 @@ class InMemoryTable(
             ChronoUnit.HOURS.between(Instant.EPOCH, DateTimeUtils.microsToInstant(micros))
         }
       case BucketTransform(numBuckets, ref) =>
-        (extractor(ref.fieldNames, schema, row).hashCode() & Integer.MAX_VALUE) % numBuckets
+        val (value, dataType) = extractor(ref.fieldNames, schema, row)
+        val valueHashCode = if (value == null) 0 else value.hashCode
+        ((valueHashCode + 31 * dataType.hashCode()) & Integer.MAX_VALUE) % numBuckets
     }
   }
 

From efae8b68e27e1491f1f65015439daf657377bc35 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Tue, 24 Nov 2020 09:50:10 -0800
Subject: [PATCH 1180/1280] [SPARK-33535][INFRA][TESTS] Export LANG to
 en_US.UTF-8 in run-tests-jenkins script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
It seems that Jenkins tests tasks in many pr have test failed. The failed cases include:

-  `org.apache.spark.sql.hive.thriftserver.SparkThriftServerProtocolVersionsSuite.HIVE_CLI_SERVICE_PROTOCOL_V1 get binary type`
- `org.apache.spark.sql.hive.thriftserver.SparkThriftServerProtocolVersionsSuite.HIVE_CLI_SERVICE_PROTOCOL_V2 get binary type`
- `org.apache.spark.sql.hive.thriftserver.SparkThriftServerProtocolVersionsSuite.HIVE_CLI_SERVICE_PROTOCOL_V3 get binary type`
- `org.apache.spark.sql.hive.thriftserver.SparkThriftServerProtocolVersionsSuite.HIVE_CLI_SERVICE_PROTOCOL_V4 get binary type`
- `org.apache.spark.sql.hive.thriftserver.SparkThriftServerProtocolVersionsSuite.HIVE_CLI_SERVICE_PROTOCOL_V5 get binary type`

The error message as follows:

```
Error Messageorg.scalatest.exceptions.TestFailedException: "[?](" did not equal "[�]("Stacktracesbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: "[?](" did not equal "[�]("
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:472)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:471)
	at org.scalatest.Assertions$.newAssertionFailedException(Assertions.scala:1231)
	at org.scalatest.Assertions$AssertionsHelper.macroAssert(Assertions.scala:1295)
	at org.apache.spark.sql.hive.thriftserver.SparkThriftServerProtocolVersionsSuite.$anonfun$new$26(SparkThriftServerProtocolVersionsSuite.scala:302)
```

But they can pass the GitHub Action, maybe it's related to the `LANG` of the Jenkins build machine, this pr add `export LANG="en_US.UTF-8"` in `run-test-jenkins` script.

### Why are the changes needed?
Ensure LANG in Jenkins test process is `en_US.UTF-8` to pass `HIVE_CLI_SERVICE_PROTOCOL_VX` related tests

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Jenkins tests pass

Closes #30487 from LuciferYang/SPARK-33535.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 048a9821c788b6796d52d1e2a0cd174377ebd0f0)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 dev/run-tests-jenkins | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins
index c3adc696a5122..c155d4ea3f076 100755
--- a/dev/run-tests-jenkins
+++ b/dev/run-tests-jenkins
@@ -26,6 +26,7 @@ FWDIR="$( cd "$( dirname "$0" )/.." && pwd )"
 cd "$FWDIR"
 
 export PATH=/home/anaconda/envs/py36/bin:$PATH
+export LANG="en_US.UTF-8"
 
 PYTHON_VERSION_CHECK=$(python3 -c 'import sys; print(sys.version_info < (3, 6, 0))')
 if [[ "$PYTHON_VERSION_CHECK" == "True" ]]; then

From 8eedc41a8868b55c70a4f081c4344bf5b8aff4c0 Mon Sep 17 00:00:00 2001
From: shane knapp <incomplete@gmail.com>
Date: Wed, 25 Nov 2020 15:55:20 -0800
Subject: [PATCH 1181/1280] [SPARK-33565][PYTHON][BUILD][3.0] Remove py38
 spark3

### What changes were proposed in this pull request?
remove python 3.8 from python/run-tests.py and stop build breaks

### Why are the changes needed?
the python tests are running against the bare-bones system install of python3, rather than an anaconda environment.

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
jenkins

see also https://github.com/apache/spark/pull/30506

Closes #30509 from shaneknapp/remove-py38-spark3.

Authored-by: shane knapp <incomplete@gmail.com>
Signed-off-by: shane knapp <incomplete@gmail.com>
---
 python/run-tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/run-tests.py b/python/run-tests.py
index 89df1a3d91c76..9fe9f4c5adedb 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -164,7 +164,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
 
 
 def get_default_python_executables():
-    python_execs = [x for x in ["python3.8", "python2.7", "pypy3", "pypy"] if which(x)]
+    python_execs = [x for x in ["python3.6", "python2.7", "pypy3", "pypy"] if which(x)]
 
     if "python3.6" not in python_execs:
         p = which("python3")

From 7503c4a3ab1854d62c49d02221c26fb220546ede Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 26 Nov 2020 12:31:02 +0900
Subject: [PATCH 1182/1280] [SPARK-33565][INFRA][FOLLOW-UP][3.0] Keep the test
 coverage with Python 3.8 in GitHub Actions

### What changes were proposed in this pull request?

This is a backport PR of https://github.com/apache/spark/pull/30510

This PR proposes to keep the test coverage with Python 3.8 in GitHub Actions. It is not tested for now in Jenkins due to an env issue.

**Before this change in GitHub Actions:**

```
========================================================================
Running PySpark tests
========================================================================
Running PySpark tests. Output is in /__w/spark/spark/python/unit-tests.log
Will test against the following Python executables: ['/usr/bin/python3', 'python2.7', 'pypy3']
...
```

**After this change in GitHub Actions:**

```

========================================================================
Running PySpark tests
========================================================================
Running PySpark tests. Output is in /__w/spark/spark/python/unit-tests.log
Will test against the following Python executables: ['python3.8', 'python2.7', 'pypy3']
```

### Why are the changes needed?

To keep the test coverage with Python 3.8 in GitHub Actions.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

GitHub Actions in this build will test.

Closes #30511 from HyukjinKwon/SPARK-33565-3.0.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/run-tests.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dev/run-tests.py b/dev/run-tests.py
index fde1defe25b69..d58884fc7783c 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -484,6 +484,12 @@ def run_python_tests(test_modules, parallelism, with_coverage=False):
     if test_modules != [modules.root]:
         command.append("--modules=%s" % ','.join(m.name for m in test_modules))
     command.append("--parallelism=%i" % parallelism)
+    if "GITHUB_ACTIONS" in os.environ:
+        # See SPARK-33565. Python 3.8 was temporarily removed as its default Python executables
+        # to test because of Jenkins environment issue. Once Jenkins has Python 3.8 to test,
+        # we should remove this change back and add python3.8 into python/run-tests.py script.
+        command.append("--python-executable=%s" % ','.join(
+            x for x in ["python3.8", "python2.7", "pypy3", "pypy"] if which(x)))
     run_cmd(command)
 
     if with_coverage:

From f67f80b6665176c7fd66300d389bdc6047d273c3 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 29 Nov 2020 12:18:07 -0800
Subject: [PATCH 1183/1280] [SPARK-33585][SQL][DOCS] Fix the comment for
 `SQLContext.tables()` and mention the `database` column

### What changes were proposed in this pull request?
Change the comments for `SQLContext.tables()` to "The returned DataFrame has three columns, database, tableName and isTemporary".

### Why are the changes needed?
Currently, the comment mentions only 2 columns but `tables()` returns 3 columns actually:
```scala
scala> spark.range(10).createOrReplaceTempView("view1")
scala> val tables = spark.sqlContext.tables()
tables: org.apache.spark.sql.DataFrame = [database: string, tableName: string ... 1 more field]

scala> tables.printSchema
root
 |-- database: string (nullable = false)
 |-- tableName: string (nullable = false)
 |-- isTemporary: boolean (nullable = false)

scala> tables.show
+--------+---------+-----------+
|database|tableName|isTemporary|
+--------+---------+-----------+
| default|       t1|      false|
| default|       t2|      false|
| default|      ymd|      false|
|        |    view1|       true|
+--------+---------+-----------+
```

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `./dev/scalastyle`

Closes #30526 from MaxGekk/sqlcontext-tables-doc.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit a088a801ed8c17171545c196a3f26ce415de0cd1)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
index 7cf0b6bb70364..dd237962110ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLContext.scala
@@ -661,7 +661,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   /**
    * Returns a `DataFrame` containing names of existing tables in the current database.
-   * The returned DataFrame has two columns, tableName and isTemporary (a Boolean
+   * The returned DataFrame has three columns, database, tableName and isTemporary (a Boolean
    * indicating if a table is a temporary one or not).
    *
    * @group ddl_ops
@@ -673,7 +673,7 @@ class SQLContext private[sql](val sparkSession: SparkSession)
 
   /**
    * Returns a `DataFrame` containing names of existing tables in the given database.
-   * The returned DataFrame has two columns, tableName and isTemporary (a Boolean
+   * The returned DataFrame has three columns, database, tableName and isTemporary (a Boolean
    * indicating if a table is a temporary one or not).
    *
    * @group ddl_ops

From f6638cfd624ee3b31f68e3d2b539dbae351730a4 Mon Sep 17 00:00:00 2001
From: Pascal Gillet <pascal.gillet@stack-labs.com>
Date: Mon, 30 Nov 2020 19:31:42 +0900
Subject: [PATCH 1184/1280] [SPARK-33579][UI] Fix executor blank page behind
 proxy

### What changes were proposed in this pull request?

Fix some "hardcoded" API urls in Web UI.
More specifically, we avoid the use of `location.origin` when constructing URLs for internal API calls within the JavaScript.
Instead, we use `apiRoot` global variable.

### Why are the changes needed?

On one hand, it allows us to build relative URLs. On the other hand, `apiRoot` reflects the Spark property `spark.ui.proxyBase` which can be set to change the root path of the Web UI.

If `spark.ui.proxyBase` is actually set, original URLs become incorrect, and we end up with an executors blank page.
I encounter this bug when accessing the Web UI behind a proxy (in my case a Kubernetes Ingress).

See the following link for more context:
https://github.com/jupyterhub/jupyter-server-proxy/issues/57#issuecomment-699163115

### Does this PR introduce _any_ user-facing change?

Yes, as all the changes introduced are in the JavaScript for the Web UI.

### How the changes have been tested ?
I modified/debugged the JavaScript as in the commit with the help of the developer tools in Google Chrome, while accessing the Web UI of my Spark app behind my k8s ingress.

Closes #30523 from pgillet/fix-executors-blank-page-behind-proxy.

Authored-by: Pascal Gillet <pascal.gillet@stack-labs.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
(cherry picked from commit 6e5446e61f278e9afac342e8f33905f5630aa7d5)
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 .../main/resources/org/apache/spark/ui/static/stagepage.js  | 2 +-
 core/src/main/resources/org/apache/spark/ui/static/utils.js | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index ee2b7b353d62e..b2964957b8334 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -70,7 +70,7 @@ function stageEndPoint(appId) {
             return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
         }
     }
-    return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId;
+    return uiRoot + "/api/v1/applications/" + appId + "/stages/" + stageId;
 }
 
 function getColumnNameForTaskMetricSummary(columnKey) {
diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js
index 2e46111bf1ba0..d15d0035844de 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/utils.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js
@@ -105,7 +105,7 @@ function getStandAloneAppId(cb) {
   }
   // Looks like Web UI is running in standalone mode
   // Let's get application-id using REST End Point
-  $.getJSON(location.origin + "/api/v1/applications", function(response, status, jqXHR) {
+  $.getJSON(uiRoot + "/api/v1/applications", function(response, status, jqXHR) {
     if (response && response.length > 0) {
       var appId = response[0].id;
       cb(appId);
@@ -152,7 +152,7 @@ function createTemplateURI(appId, templateName) {
     var baseURI = words.slice(0, ind).join('/') + '/static/' + templateName + '-template.html';
     return baseURI;
   }
-  return location.origin + "/static/" + templateName + "-template.html";
+  return uiRoot + "/static/" + templateName + "-template.html";
 }
 
 function setDataTableDefaults() {
@@ -190,5 +190,5 @@ function createRESTEndPointForExecutorsPage(appId) {
             return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors";
         }
     }
-    return location.origin + "/api/v1/applications/" + appId + "/allexecutors";
+    return uiRoot + "/api/v1/applications/" + appId + "/allexecutors";
 }

From 03291c80c5b1aa2b18e53617676f36d40e01188f Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 30 Nov 2020 08:37:13 -0800
Subject: [PATCH 1185/1280] [SPARK-33588][SQL][3.0] Respect the
 `spark.sql.caseSensitive` config while resolving partition spec in v1 `SHOW
 TABLE EXTENDED`

### What changes were proposed in this pull request?
Perform partition spec normalization in `ShowTablesCommand` according to the table schema before getting partitions from the catalog. The normalization via `PartitioningUtils.normalizePartitionSpec()` adjusts the column names in partition specification, w.r.t. the real partition column names and case sensitivity.

### Why are the changes needed?
Even when `spark.sql.caseSensitive` is `false` which is the default value, v1 `SHOW TABLE EXTENDED` is case sensitive:
```sql
spark-sql> CREATE TABLE tbl1 (price int, qty int, year int, month int)
         > USING parquet
         > partitioned by (year, month);
spark-sql> INSERT INTO tbl1 PARTITION(year = 2015, month = 1) SELECT 1, 1;
spark-sql> SHOW TABLE EXTENDED LIKE 'tbl1' PARTITION(YEAR = 2015, Month = 1);
Error in query: Partition spec is invalid. The spec (YEAR, Month) must match the partition spec (year, month) defined in table '`default`.`tbl1`';
```

### Does this PR introduce _any_ user-facing change?
Yes. After the changes, the `SHOW TABLE EXTENDED` command respects the SQL config. And for example above, it returns correct result:
```sql
spark-sql> SHOW TABLE EXTENDED LIKE 'tbl1' PARTITION(YEAR = 2015, Month = 1);
default	tbl1	false	Partition Values: [year=2015, month=1]
Location: file:/Users/maximgekk/spark-warehouse/tbl1/year=2015/month=1
Serde Library: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
Storage Properties: [serialization.format=1, path=file:/Users/maximgekk/spark-warehouse/tbl1]
Partition Parameters: {transient_lastDdlTime=1606595118, totalSize=623, numFiles=1}
Created Time: Sat Nov 28 23:25:18 MSK 2020
Last Access: UNKNOWN
Partition Statistics: 623 bytes
```

### How was this patch tested?
By running the modified test suite via:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *DDLSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Dongjoon Hyun <dongjoonapache.org>
(cherry picked from commit 0054fc937f804660c6501d9d3f6319f3047a68f8)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #30549 from MaxGekk/show-table-case-sensitive-spec-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/command/tables.scala  | 17 +++++++++-----
 .../sql-tests/results/show-tables.sql.out     |  2 +-
 .../sql/execution/command/DDLSuite.scala      | 22 +++++++++++++++++++
 3 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index fc8cc11bb1067..75e0d2c76339f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -884,12 +884,17 @@ case class ShowTablesCommand(
       //
       // Note: tableIdentifierPattern should be non-empty, otherwise a [[ParseException]]
       // should have been thrown by the sql parser.
-      val tableIdent = TableIdentifier(tableIdentifierPattern.get, Some(db))
-      val table = catalog.getTableMetadata(tableIdent).identifier
-      val partition = catalog.getPartition(tableIdent, partitionSpec.get)
-      val database = table.database.getOrElse("")
-      val tableName = table.table
-      val isTemp = catalog.isTemporaryTable(table)
+      val table = catalog.getTableMetadata(TableIdentifier(tableIdentifierPattern.get, Some(db)))
+      val tableIdent = table.identifier
+      val normalizedSpec = PartitioningUtils.normalizePartitionSpec(
+        partitionSpec.get,
+        table.partitionColumnNames,
+        tableIdent.quotedString,
+        sparkSession.sessionState.conf.resolver)
+      val partition = catalog.getPartition(tableIdent, normalizedSpec)
+      val database = tableIdent.database.getOrElse("")
+      val tableName = tableIdent.table
+      val isTemp = catalog.isTemporaryTable(tableIdent)
       val information = partition.simpleString
       Seq(Row(database, tableName, isTemp, s"$information\n"))
     }
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
index a95b02c7f7743..60c5e6d5642b7 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
@@ -224,7 +224,7 @@ SHOW TABLE EXTENDED LIKE 'show_t1' PARTITION(a='Us', d=1)
 struct<>
 -- !query output
 org.apache.spark.sql.AnalysisException
-Partition spec is invalid. The spec (a, d) must match the partition spec (c, d) defined in table '`showdb`.`show_t1`';
+a is not a valid partition column in table `showdb`.`show_t1`.;
 
 
 -- !query
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index e4709e469dca3..d0118ef1caeda 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -32,6 +32,7 @@ import org.apache.spark.sql.catalyst.{QualifiedTableName, TableIdentifier}
 import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchDatabaseException, NoSuchPartitionException, NoSuchTableException, TempTableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
+import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
@@ -3030,6 +3031,27 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-33588: case sensitivity of partition spec") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"""
+        |CREATE TABLE $t (price int, qty int, year int, month int)
+        |USING $dataSource
+        |PARTITIONED BY (year, month)""".stripMargin)
+      sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+      Seq(
+        true -> "PARTITION(year = 2015, month = 1)",
+        false -> "PARTITION(YEAR = 2015, Month = 1)"
+      ).foreach { case (caseSensitive, partitionSpec) =>
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          val df = sql(s"SHOW TABLE EXTENDED LIKE '$t' $partitionSpec")
+          val information = df.select("information").first().getString(0)
+          assert(information.contains("Partition Values: [year=2015, month=1]"))
+        }
+      }
+    }
+  }
 }
 
 object FakeLocalFsFileSystem {

From 242581f4926c994bfc5af388cae31645112b2798 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 1 Dec 2020 06:44:15 +0900
Subject: [PATCH 1186/1280] [SPARK-33440][CORE] Use current timestamp with
 warning log in HadoopFSDelegationTokenProvider when the issue date for token
 is not set up properly

### What changes were proposed in this pull request?

This PR proposes to use current timestamp with warning log when the issue date for token is not set up properly. The next section will explain the rationalization with details.

### Why are the changes needed?

Unfortunately not every implementations respect the `issue date` in `AbstractDelegationTokenIdentifier`, which Spark relies on while calculating. The default value of issue date is 0L, which is far from actual issue date, breaking logic on calculating next renewal date under some circumstance, leading to 0 interval (immediate) on rescheduling token renewal.

In HadoopFSDelegationTokenProvider, Spark calculates token renewal interval as below:

https://github.com/apache/spark/blob/2c64b731ae6a976b0d75a95901db849b4a0e2393/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala#L123-L134

The interval is calculated as `token.renew() - identifier.getIssueDate`, which is providing correct interval assuming both `token.renew()` and `identifier.getIssueDate` produce correct value, but it's going to be weird when `identifier.getIssueDate` provides 0L (default value), like below:

```
20/10/13 06:34:19 INFO security.HadoopFSDelegationTokenProvider: Renewal interval is 1603175657000 for token S3ADelegationToken/IDBroker
20/10/13 06:34:19 INFO security.HadoopFSDelegationTokenProvider: Renewal interval is 86400048 for token HDFS_DELEGATION_TOKEN
```

Hopefully we pick the minimum value as safety guard (so in this case, `86400048` is being picked up), but the safety guard leads unintentional bad impact on this case.

https://github.com/apache/spark/blob/2c64b731ae6a976b0d75a95901db849b4a0e2393/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala#L58-L71

Spark leverages the interval being calculated in above, "minimum" value of intervals, and blindly adds the value to token's issue date to calculates the next renewal date for the token, and picks "minimum" value again. In problematic case, the value would be `86400048` (86400048 + 0) which is quite smaller than current timestamp.

https://github.com/apache/spark/blob/2c64b731ae6a976b0d75a95901db849b4a0e2393/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala#L228-L234

The next renewal date is subtracted with current timestamp again to get the interval, and multiplexed by configured ratio to produce the final schedule interval. In problematic case, this value goes to negative.

https://github.com/apache/spark/blob/2c64b731ae6a976b0d75a95901db849b4a0e2393/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala#L180-L188

There's a safety guard to not allow negative value, but that's simply 0 meaning schedule immediately. This triggers next calculation of next renewal date to calculate the schedule interval, lead to the same behavior, hence updating delegation token immediately and continuously.

As we fetch token just before the calculation happens, the actual issue date is likely slightly before, hence it's not that dangerous to use current timestamp as issue date for the token the issue date has not been set up properly. Still, it's better not to leave the token implementation as it is, so we log warn message to let end users consult with token implementer.

### Does this PR introduce _any_ user-facing change?

Yes. End users won't encounter the tight loop of schedule of token renewal after the PR. In end users' perspective of reflection, there's nothing end users need to change.

### How was this patch tested?

Manually tested with problematic environment.

Closes #30366 from HeartSaVioR/SPARK-33440.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit f5d2165c95fe83f24be9841807613950c1d5d6d0)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 .../HadoopDelegationTokenManager.scala        |  4 ++-
 .../HadoopFSDelegationTokenProvider.scala     | 27 ++++++++++++++++---
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
index 3168c763df4df..6ce195b6c7a34 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopDelegationTokenManager.scala
@@ -178,7 +178,7 @@ private[spark] class HadoopDelegationTokenManager(
 
   private def scheduleRenewal(delay: Long): Unit = {
     val _delay = math.max(0, delay)
-    logInfo(s"Scheduling renewal in ${UIUtils.formatDuration(delay)}.")
+    logInfo(s"Scheduling renewal in ${UIUtils.formatDuration(_delay)}.")
 
     val renewalTask = new Runnable() {
       override def run(): Unit = {
@@ -230,6 +230,8 @@ private[spark] class HadoopDelegationTokenManager(
         val now = System.currentTimeMillis
         val ratio = sparkConf.get(CREDENTIALS_RENEWAL_INTERVAL_RATIO)
         val delay = (ratio * (nextRenewal - now)).toLong
+        logInfo(s"Calculated delay on renewal is $delay, based on next renewal $nextRenewal " +
+          s"and the ratio $ratio, and current time $now")
         scheduleRenewal(delay)
         creds
       }
diff --git a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
index 4e91e72361488..cd9516b849565 100644
--- a/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/security/HadoopFSDelegationTokenProvider.scala
@@ -63,7 +63,8 @@ private[deploy] class HadoopFSDelegationTokenProvider
             val identifier = token
               .decodeIdentifier()
               .asInstanceOf[AbstractDelegationTokenIdentifier]
-            identifier.getIssueDate + interval
+            val tokenKind = token.getKind.toString
+            getIssueDate(tokenKind, identifier) + interval
           }
         if (nextRenewalDates.isEmpty) None else Some(nextRenewalDates.min)
       }
@@ -126,13 +127,33 @@ private[deploy] class HadoopFSDelegationTokenProvider
       Try {
         val newExpiration = token.renew(hadoopConf)
         val identifier = token.decodeIdentifier().asInstanceOf[AbstractDelegationTokenIdentifier]
-        val interval = newExpiration - identifier.getIssueDate
-        logInfo(s"Renewal interval is $interval for token ${token.getKind.toString}")
+        val tokenKind = token.getKind.toString
+        val interval = newExpiration - getIssueDate(tokenKind, identifier)
+        logInfo(s"Renewal interval is $interval for token $tokenKind")
         interval
       }.toOption
     }
     if (renewIntervals.isEmpty) None else Some(renewIntervals.min)
   }
+
+  private def getIssueDate(kind: String, identifier: AbstractDelegationTokenIdentifier): Long = {
+    val now = System.currentTimeMillis()
+    val issueDate = identifier.getIssueDate
+    if (issueDate > now) {
+      logWarning(s"Token $kind has set up issue date later than current time. (provided: " +
+        s"$issueDate / current timestamp: $now) Please make sure clocks are in sync between " +
+        "machines. If the issue is not a clock mismatch, consult token implementor to check " +
+        "whether issue date is valid.")
+      issueDate
+    } else if (issueDate > 0L) {
+      issueDate
+    } else {
+      logWarning(s"Token $kind has not set up issue date properly. (provided: $issueDate) " +
+        s"Using current timestamp ($now) as issue date instead. Consult token implementor to fix " +
+        "the behavior.")
+      now
+    }
+  }
 }
 
 private[deploy] object HadoopFSDelegationTokenProvider {

From 6abfeb6884a3cdfe4c6e621219e6cf5a35d6467e Mon Sep 17 00:00:00 2001
From: Gengliang Wang <gengliang.wang@databricks.com>
Date: Wed, 2 Dec 2020 01:36:41 +0800
Subject: [PATCH 1187/1280] [SPARK-33611][UI] Avoid encoding twice on the query
 parameter of rewritten proxy URL

### What changes were proposed in this pull request?

When running Spark behind a reverse proxy(e.g. Nginx, Apache HTTP server), the request URL can be encoded twice if we pass the query string directly to the constructor of `java.net.URI`:
```
> val uri = "http://localhost:8081/test"
> val query = "order%5B0%5D%5Bcolumn%5D=0"  // query string of URL from the reverse proxy
> val rewrittenURI = URI.create(uri.toString())

> new URI(rewrittenURI.getScheme(),
      rewrittenURI.getAuthority(),
      rewrittenURI.getPath(),
      query,
      rewrittenURI.getFragment()).toString
result: http://localhost:8081/test?order%255B0%255D%255Bcolumn%255D=0
```

In Spark's stage page, the URL of "/taskTable" contains query parameter order[0][dir]. After encoding twice, the query parameter becomes `order%255B0%255D%255Bdir%255D` and it will be decoded as `order%5B0%5D%5Bdir%5D` instead of `order[0][dir]`. As a result, there will be NullPointerException from https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/status/api/v1/StagesResource.scala#L176
Other than that, the other parameter may not work as expected after encoded twice.

This PR is to fix the bug by calling the method `URI.create(String URL)` directly. This convenience method can avoid encoding twice on the query parameter.
```
> val uri = "http://localhost:8081/test"
> val query = "order%5B0%5D%5Bcolumn%5D=0"
> URI.create(s"$uri?$query").toString
result: http://localhost:8081/test?order%5B0%5D%5Bcolumn%5D=0

> URI.create(s"$uri?$query").getQuery
result: order[0][column]=0
```

### Why are the changes needed?

Fix a potential bug when Spark's reverse proxy is enabled.
The bug itself is similar to https://github.com/apache/spark/pull/29271.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Add a new unit test.
Also, Manual UI testing for master, worker and app UI with an nginx proxy

Spark config:
```
spark.ui.port 8080
spark.ui.reverseProxy=true
spark.ui.reverseProxyUrl=/path/to/spark/
```
nginx config:
```
server {
    listen 9000;
    set $SPARK_MASTER http://127.0.0.1:8080;
    # split spark UI path into prefix and local path within master UI
    location ~ ^(/path/to/spark/) {
        # strip prefix when forwarding request
        rewrite /path/to/spark(/.*) $1  break;
        #rewrite /path/to/spark/ "/" ;
        # forward to spark master UI
        proxy_pass $SPARK_MASTER;
        proxy_intercept_errors on;
        error_page 301 302 307 = handle_redirects;
    }
    location handle_redirects {
        set $saved_redirect_location '$upstream_http_location';
        proxy_pass $saved_redirect_location;
    }
}
```

Closes #30552 from gengliangwang/decodeProxyRedirect.

Authored-by: Gengliang Wang <gengliang.wang@databricks.com>
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
(cherry picked from commit 5d0045eedf4b138c031accac2b1fa1e8d6f3f7c6)
Signed-off-by: Gengliang Wang <gengliang.wang@databricks.com>
---
 .../scala/org/apache/spark/ui/JettyUtils.scala   | 16 ++++++----------
 .../test/scala/org/apache/spark/ui/UISuite.scala |  9 +++++++++
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index a4ba5654b542e..3820a880b2109 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -400,17 +400,13 @@ private[spark] object JettyUtils extends Logging {
       uri.append(rest)
     }
 
-    val rewrittenURI = URI.create(uri.toString())
-    if (query != null) {
-      return new URI(
-          rewrittenURI.getScheme(),
-          rewrittenURI.getAuthority(),
-          rewrittenURI.getPath(),
-          query,
-          rewrittenURI.getFragment()
-        ).normalize()
+    val queryString = if (query == null) {
+      ""
+    } else {
+      s"?$query"
     }
-    rewrittenURI.normalize()
+    // SPARK-33611: use method `URI.create` to avoid percent-encoding twice on the query string.
+    URI.create(uri.toString() + queryString).normalize()
   }
 
   def createProxyLocationHeader(
diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
index 56026eaa0072b..c7e1dfe71d563 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala
@@ -216,6 +216,15 @@ class UISuite extends SparkFunSuite {
     assert(rewrittenURI === null)
   }
 
+  test("SPARK-33611: Avoid encoding twice on the query parameter of proxy rewrittenURI") {
+    val prefix = "/worker-id"
+    val target = "http://localhost:8081"
+    val path = "/worker-id/json"
+    val rewrittenURI =
+      JettyUtils.createProxyURI(prefix, target, path, "order%5B0%5D%5Bcolumn%5D=0")
+    assert(rewrittenURI.toString === "http://localhost:8081/json?order%5B0%5D%5Bcolumn%5D=0")
+  }
+
   test("verify rewriting location header for reverse proxy") {
     val clientRequest = mock(classOf[HttpServletRequest])
     var headerValue = "http://localhost:4040/jobs"

From e59179b7326112f526e4c000e21146df283d861c Mon Sep 17 00:00:00 2001
From: neko <echohlne@gmail.com>
Date: Wed, 2 Dec 2020 09:24:19 -0600
Subject: [PATCH 1188/1280] [SPARK-33504][CORE] The application log in the
 Spark history server contains sensitive attributes should be redacted

### What changes were proposed in this pull request?
To make sure the sensitive attributes to be redacted in the history server log.

### Why are the changes needed?
We found the secure attributes like password  in SparkListenerJobStart and SparkListenerStageSubmitted events would not been redated, resulting in sensitive attributes can be viewd directly.
The screenshot can be viewed in the attachment of JIRA spark-33504
### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
muntual test works well, I have also added unit testcase.

Closes #30446 from akiyamaneko/eventlog_unredact.

Authored-by: neko <echohlne@gmail.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
(cherry picked from commit 28dad1ba770e5b7f7cf542da1ae3f05975a969c6)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../scheduler/EventLoggingListener.scala      | 24 ++++++-
 .../scheduler/EventLoggingListenerSuite.scala | 64 ++++++++++++++++++-
 2 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 24e2a5e4d4a62..5673c02fb52b4 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -18,7 +18,9 @@
 package org.apache.spark.scheduler
 
 import java.net.URI
+import java.util.Properties
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
@@ -103,7 +105,7 @@ private[spark] class EventLoggingListener(
 
   // Events that do not trigger a flush
   override def onStageSubmitted(event: SparkListenerStageSubmitted): Unit = {
-    logEvent(event)
+    logEvent(event.copy(properties = redactProperties(event.properties)))
     if (shouldLogStageExecutorMetrics) {
       // record the peak metrics for the new stage
       liveStageExecutorMetrics.put((event.stageInfo.stageId, event.stageInfo.attemptNumber()),
@@ -156,7 +158,9 @@ private[spark] class EventLoggingListener(
     logEvent(event, flushLogger = true)
   }
 
-  override def onJobStart(event: SparkListenerJobStart): Unit = logEvent(event, flushLogger = true)
+  override def onJobStart(event: SparkListenerJobStart): Unit = {
+    logEvent(event.copy(properties = redactProperties(event.properties)), flushLogger = true)
+  }
 
   override def onJobEnd(event: SparkListenerJobEnd): Unit = logEvent(event, flushLogger = true)
 
@@ -246,6 +250,22 @@ private[spark] class EventLoggingListener(
     logWriter.stop()
   }
 
+  private def redactProperties(properties: Properties): Properties = {
+    if (properties == null) {
+      return properties
+    }
+    val redactedProperties = new Properties
+    // properties may contain some custom local properties such as stage/job description
+    // only properties in sparkConf need to be redacted.
+    val (globalProperties, localProperties) = properties.asScala.toSeq.partition {
+      case (key, _) => sparkConf.contains(key)
+    }
+    (Utils.redact(sparkConf, globalProperties) ++ localProperties).foreach {
+      case (key, value) => redactedProperties.setProperty(key, value)
+    }
+    redactedProperties
+  }
+
   private[spark] def redactEvent(
       event: SparkListenerEnvironmentUpdate): SparkListenerEnvironmentUpdate = {
     // environmentDetails maps a string descriptor to a set of properties
diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index 046564d65b767..e0e64069c856f 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.scheduler
 
 import java.io.{File, InputStream}
-import java.util.Arrays
+import java.util.{Arrays, Properties}
 
 import scala.collection.immutable.Map
 import scala.collection.mutable
@@ -96,6 +96,68 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
     assert(redactedProps(key) == "*********(redacted)")
   }
 
+  test("Spark-33504 sensitive attributes redaction in properties") {
+    val (secretKey, secretPassword) = ("spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
+      "secret_password")
+    val (customKey, customValue) = ("parse_token", "secret_password")
+
+    val conf = getLoggingConf(testDirPath, None).set(secretKey, secretPassword)
+
+    val properties = new Properties()
+    properties.setProperty(secretKey, secretPassword)
+    properties.setProperty(customKey, customValue)
+
+    val logName = "properties-reaction-test"
+    val eventLogger = new EventLoggingListener(logName, None, testDirPath.toUri(), conf)
+    val listenerBus = new LiveListenerBus(conf)
+
+    val stageId = 1
+    val jobId = 1
+    val stageInfo = new StageInfo(stageId, 0, stageId.toString, 0,
+      Seq.empty, Seq.empty, "details",
+      resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
+
+    val events = Array(SparkListenerStageSubmitted(stageInfo, properties),
+      SparkListenerJobStart(jobId, 0, Seq(stageInfo), properties))
+
+    eventLogger.start()
+    listenerBus.start(Mockito.mock(classOf[SparkContext]), Mockito.mock(classOf[MetricsSystem]))
+    listenerBus.addToEventLogQueue(eventLogger)
+    events.foreach(event => listenerBus.post(event))
+    listenerBus.stop()
+    eventLogger.stop()
+
+    val logData = EventLogFileReader.openEventLog(new Path(eventLogger.logWriter.logPath),
+      fileSystem)
+    try {
+      val lines = readLines(logData)
+      val logStart = SparkListenerLogStart(SPARK_VERSION)
+      assert(lines.size === 3)
+      assert(lines(0).contains("SparkListenerLogStart"))
+      assert(lines(1).contains("SparkListenerStageSubmitted"))
+      assert(lines(2).contains("SparkListenerJobStart"))
+
+      lines.foreach{
+        line => JsonProtocol.sparkEventFromJson(parse(line)) match {
+          case logStartEvent: SparkListenerLogStart =>
+            assert(logStartEvent == logStart)
+
+          case stageSubmittedEvent: SparkListenerStageSubmitted =>
+            assert(stageSubmittedEvent.properties.getProperty(secretKey) == "*********(redacted)")
+            assert(stageSubmittedEvent.properties.getProperty(customKey) ==  customValue)
+
+          case jobStartEvent : SparkListenerJobStart =>
+            assert(jobStartEvent.properties.getProperty(secretKey) == "*********(redacted)")
+            assert(jobStartEvent.properties.getProperty(customKey) ==  customValue)
+
+          case _ => assert(false)
+        }
+      }
+    } finally {
+      logData.close()
+    }
+  }
+
   test("Executor metrics update") {
     testStageExecutorMetricsEventLogging()
   }

From 3fb9f6f670328d31bf24fcb6b805715f5828ce06 Mon Sep 17 00:00:00 2001
From: Thomas Graves <tgraves@nvidia.com>
Date: Wed, 2 Dec 2020 14:38:19 -0600
Subject: [PATCH 1189/1280] Revert "[SPARK-33504][CORE] The application log in
 the Spark history server contains sensitive attributes should be redacted"

### What changes were proposed in this pull request?

Revert SPARK-33504 on branch-3.0 compilation error. Original PR https://github.com/apache/spark/pull/30446

This reverts commit e59179b7326112f526e4c000e21146df283d861c.

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #30576 from tgravescs/revert33504.

Authored-by: Thomas Graves <tgraves@nvidia.com>
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
 .../scheduler/EventLoggingListener.scala      | 24 +------
 .../scheduler/EventLoggingListenerSuite.scala | 64 +------------------
 2 files changed, 3 insertions(+), 85 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index 5673c02fb52b4..24e2a5e4d4a62 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -18,9 +18,7 @@
 package org.apache.spark.scheduler
 
 import java.net.URI
-import java.util.Properties
 
-import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.apache.hadoop.conf.Configuration
@@ -105,7 +103,7 @@ private[spark] class EventLoggingListener(
 
   // Events that do not trigger a flush
   override def onStageSubmitted(event: SparkListenerStageSubmitted): Unit = {
-    logEvent(event.copy(properties = redactProperties(event.properties)))
+    logEvent(event)
     if (shouldLogStageExecutorMetrics) {
       // record the peak metrics for the new stage
       liveStageExecutorMetrics.put((event.stageInfo.stageId, event.stageInfo.attemptNumber()),
@@ -158,9 +156,7 @@ private[spark] class EventLoggingListener(
     logEvent(event, flushLogger = true)
   }
 
-  override def onJobStart(event: SparkListenerJobStart): Unit = {
-    logEvent(event.copy(properties = redactProperties(event.properties)), flushLogger = true)
-  }
+  override def onJobStart(event: SparkListenerJobStart): Unit = logEvent(event, flushLogger = true)
 
   override def onJobEnd(event: SparkListenerJobEnd): Unit = logEvent(event, flushLogger = true)
 
@@ -250,22 +246,6 @@ private[spark] class EventLoggingListener(
     logWriter.stop()
   }
 
-  private def redactProperties(properties: Properties): Properties = {
-    if (properties == null) {
-      return properties
-    }
-    val redactedProperties = new Properties
-    // properties may contain some custom local properties such as stage/job description
-    // only properties in sparkConf need to be redacted.
-    val (globalProperties, localProperties) = properties.asScala.toSeq.partition {
-      case (key, _) => sparkConf.contains(key)
-    }
-    (Utils.redact(sparkConf, globalProperties) ++ localProperties).foreach {
-      case (key, value) => redactedProperties.setProperty(key, value)
-    }
-    redactedProperties
-  }
-
   private[spark] def redactEvent(
       event: SparkListenerEnvironmentUpdate): SparkListenerEnvironmentUpdate = {
     // environmentDetails maps a string descriptor to a set of properties
diff --git a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index e0e64069c856f..046564d65b767 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.scheduler
 
 import java.io.{File, InputStream}
-import java.util.{Arrays, Properties}
+import java.util.Arrays
 
 import scala.collection.immutable.Map
 import scala.collection.mutable
@@ -96,68 +96,6 @@ class EventLoggingListenerSuite extends SparkFunSuite with LocalSparkContext wit
     assert(redactedProps(key) == "*********(redacted)")
   }
 
-  test("Spark-33504 sensitive attributes redaction in properties") {
-    val (secretKey, secretPassword) = ("spark.executorEnv.HADOOP_CREDSTORE_PASSWORD",
-      "secret_password")
-    val (customKey, customValue) = ("parse_token", "secret_password")
-
-    val conf = getLoggingConf(testDirPath, None).set(secretKey, secretPassword)
-
-    val properties = new Properties()
-    properties.setProperty(secretKey, secretPassword)
-    properties.setProperty(customKey, customValue)
-
-    val logName = "properties-reaction-test"
-    val eventLogger = new EventLoggingListener(logName, None, testDirPath.toUri(), conf)
-    val listenerBus = new LiveListenerBus(conf)
-
-    val stageId = 1
-    val jobId = 1
-    val stageInfo = new StageInfo(stageId, 0, stageId.toString, 0,
-      Seq.empty, Seq.empty, "details",
-      resourceProfileId = ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
-
-    val events = Array(SparkListenerStageSubmitted(stageInfo, properties),
-      SparkListenerJobStart(jobId, 0, Seq(stageInfo), properties))
-
-    eventLogger.start()
-    listenerBus.start(Mockito.mock(classOf[SparkContext]), Mockito.mock(classOf[MetricsSystem]))
-    listenerBus.addToEventLogQueue(eventLogger)
-    events.foreach(event => listenerBus.post(event))
-    listenerBus.stop()
-    eventLogger.stop()
-
-    val logData = EventLogFileReader.openEventLog(new Path(eventLogger.logWriter.logPath),
-      fileSystem)
-    try {
-      val lines = readLines(logData)
-      val logStart = SparkListenerLogStart(SPARK_VERSION)
-      assert(lines.size === 3)
-      assert(lines(0).contains("SparkListenerLogStart"))
-      assert(lines(1).contains("SparkListenerStageSubmitted"))
-      assert(lines(2).contains("SparkListenerJobStart"))
-
-      lines.foreach{
-        line => JsonProtocol.sparkEventFromJson(parse(line)) match {
-          case logStartEvent: SparkListenerLogStart =>
-            assert(logStartEvent == logStart)
-
-          case stageSubmittedEvent: SparkListenerStageSubmitted =>
-            assert(stageSubmittedEvent.properties.getProperty(secretKey) == "*********(redacted)")
-            assert(stageSubmittedEvent.properties.getProperty(customKey) ==  customValue)
-
-          case jobStartEvent : SparkListenerJobStart =>
-            assert(jobStartEvent.properties.getProperty(secretKey) == "*********(redacted)")
-            assert(jobStartEvent.properties.getProperty(customKey) ==  customValue)
-
-          case _ => assert(false)
-        }
-      }
-    } finally {
-      logData.close()
-    }
-  }
-
   test("Executor metrics update") {
     testStageExecutorMetricsEventLogging()
   }

From 6f4587a10d553969a2f753d30d86ae3b1077e1f6 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Wed, 2 Dec 2020 12:58:41 -0800
Subject: [PATCH 1190/1280] [SPARK-33631][DOCS][TEST] Clean up
 spark.core.connection.ack.wait.timeout from configuration.md

SPARK-9767  remove `ConnectionManager` and related files, the configuration `spark.core.connection.ack.wait.timeout` previously used by `ConnectionManager` is no longer used by other Spark code, but it still exists in the `configuration.md`.

So this pr cleans up the useless configuration item spark.core.connection.ack.wait.timeout` from `configuration.md`.

Clean up useless configuration from `configuration.md`.

No

Pass the Jenkins or GitHub Action

Closes #30569 from LuciferYang/SPARK-33631.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 92bfbcb2e372e8fecfe65bc582c779d9df4036bb)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/storage/BlockManagerReplicationSuite.scala  |  2 --
 docs/configuration.md                                 | 11 -----------
 2 files changed, 13 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
index 59ace850d0bd2..6238d862d62df 100644
--- a/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/BlockManagerReplicationSuite.scala
@@ -92,8 +92,6 @@ trait BlockManagerReplicationBehavior extends SparkFunSuite
     conf.set(MEMORY_STORAGE_FRACTION, 0.999)
     conf.set(STORAGE_UNROLL_MEMORY_THRESHOLD, 512L)
 
-    // to make a replication attempt to inactive store fail fast
-    conf.set("spark.core.connection.ack.wait.timeout", "1s")
     // to make cached peers refresh frequently
     conf.set(STORAGE_CACHED_PEERS_TTL, 10)
 
diff --git a/docs/configuration.md b/docs/configuration.md
index 5a2c6b3e4a0bc..ebfee12122e43 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1887,7 +1887,6 @@ Apart from these, the following properties are also available, and may be useful
   <td>120s</td>
   <td>
     Default timeout for all network interactions. This config will be used in place of
-    <code>spark.core.connection.ack.wait.timeout</code>,
     <code>spark.storage.blockManagerSlaveTimeoutMs</code>,
     <code>spark.shuffle.io.connectionTimeout</code>, <code>spark.rpc.askTimeout</code> or
     <code>spark.rpc.lookupTimeout</code> if they are not configured.
@@ -1950,16 +1949,6 @@ Apart from these, the following properties are also available, and may be useful
   </td>
   <td>1.4.0</td>
 </tr>
-<tr>
-  <td><code>spark.core.connection.ack.wait.timeout</code></td>
-  <td><code>spark.network.timeout</code></td>
-  <td>
-    How long for the connection to wait for ack to occur before timing
-    out and giving up. To avoid unwilling timeout caused by long pause like GC,
-    you can set larger value.
-  </td>
-  <td>1.1.1</td>
-</tr>
 <tr>
   <td><code>spark.network.maxRemoteBlockSizeFetchToMem</code></td>
   <td>200m</td>

From 13ca88cf388703ee2e2d72dc4ef2d482b829a7df Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Wed, 2 Dec 2020 21:28:52 -0800
Subject: [PATCH 1191/1280] [SPARK-33636][PYTHON][ML][3.0] Add labelsArray to
 PySpark StringIndexer

### What changes were proposed in this pull request?

This is a followup to add missing `labelsArray` to PySpark `StringIndexer`.

### Why are the changes needed?

`labelsArray` is for multi-column case for `StringIndexer`. We should provide this accessor at PySpark side too.

### Does this PR introduce _any_ user-facing change?

Yes, `labelsArray` was missing in PySpark `StringIndexer` in Spark 3.0.

### How was this patch tested?

Unit test.

Closes #30580 from viirya/SPARK-33636.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Liang-Chi Hsieh <viirya@gmail.com>
---
 python/pyspark/ml/feature.py            | 9 +++++++++
 python/pyspark/ml/tests/test_feature.py | 1 +
 2 files changed, 10 insertions(+)

diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 4f42bb337fe48..ded263e07a26f 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -3765,6 +3765,15 @@ def labels(self):
         """
         return self._call_java("labels")
 
+    @property
+    @since("3.0.2")
+    def labelsArray(self):
+        """
+        Array of ordered list of labels, corresponding to indices to be assigned
+        for each input column.
+        """
+        return self._call_java("labelsArray")
+
 
 @inherit_doc
 class IndexToString(JavaTransformer, HasInputCol, HasOutputCol, JavaMLReadable, JavaMLWritable):
diff --git a/python/pyspark/ml/tests/test_feature.py b/python/pyspark/ml/tests/test_feature.py
index 4c6bfa696b110..cdd5ad50f89b1 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -236,6 +236,7 @@ def test_string_indexer_from_labels(self):
         model = StringIndexerModel.from_labels(["a", "b", "c"], inputCol="label",
                                                outputCol="indexed", handleInvalid="keep")
         self.assertEqual(model.labels, ["a", "b", "c"])
+        self.assertEqual(model.labelsArray, [("a", "b", "c")])
 
         df1 = self.spark.createDataFrame([
             (0, "a"),

From c4318a1184c2cb8e6a3010bed408be41e569a822 Mon Sep 17 00:00:00 2001
From: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Date: Fri, 4 Dec 2020 01:37:44 +0900
Subject: [PATCH 1192/1280] [SPARK-33629][PYTHON] Make spark.buffer.size
 configuration visible on driver side

`spark.buffer.size` not applied in driver from pyspark. In this PR I've fixed this issue.

Apply the mentioned config on driver side.

No.

Existing unit tests + manually.

Added the following code temporarily:
```
def local_connect_and_auth(port, auth_secret):
...
            sock.connect(sa)
            print("SPARK_BUFFER_SIZE: %d" % int(os.environ.get("SPARK_BUFFER_SIZE", 65536))) <- This is the addition
            sockfile = sock.makefile("rwb", int(os.environ.get("SPARK_BUFFER_SIZE", 65536)))
...
```

Test:
```

echo "spark.buffer.size 10000" >> conf/spark-defaults.conf

$ ./bin/pyspark
Python 3.8.5 (default, Jul 21 2020, 10:48:26)
[Clang 11.0.3 (clang-1103.0.32.62)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
20/12/03 13:38:13 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
20/12/03 13:38:14 WARN SparkEnv: I/O encryption enabled without RPC encryption: keys will be visible on the wire.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /__ / .__/\_,_/_/ /_/\_\   version 3.1.0-SNAPSHOT
      /_/

Using Python version 3.8.5 (default, Jul 21 2020 10:48:26)
Spark context Web UI available at http://192.168.0.189:4040
Spark context available as 'sc' (master = local[*], app id = local-1606999094506).
SparkSession available as 'spark'.
>>> sc.setLogLevel("TRACE")
>>> sc.parallelize([0, 2, 3, 4, 6], 5).glom().collect()
...
SPARK_BUFFER_SIZE: 10000
...
[[0], [2], [3], [4], [6]]
>>>
```

Closes #30592 from gaborgsomogyi/SPARK-33629.

Authored-by: Gabor Somogyi <gabor.g.somogyi@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit bd711863fdcdde21a7d64de8a9b6b7a8bf7c19ec)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../main/scala/org/apache/spark/api/python/PythonUtils.scala  | 4 ++++
 python/pyspark/context.py                                     | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
index 490b48719b6be..78eb6907c3898 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala
@@ -85,4 +85,8 @@ private[spark] object PythonUtils {
   def getBroadcastThreshold(sc: JavaSparkContext): Long = {
     sc.conf.get(org.apache.spark.internal.config.BROADCAST_FOR_UDF_COMPRESSION_THRESHOLD)
   }
+
+  def getSparkBufferSize(sc: JavaSparkContext): Int = {
+    sc.conf.get(org.apache.spark.internal.config.BUFFER_SIZE)
+  }
 }
diff --git a/python/pyspark/context.py b/python/pyspark/context.py
index 72e8e84cba757..25c2a5bfc0725 100644
--- a/python/pyspark/context.py
+++ b/python/pyspark/context.py
@@ -211,6 +211,8 @@ def _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize,
         # data via a socket.
         # scala's mangled names w/ $ in them require special treatment.
         self._encryption_enabled = self._jvm.PythonUtils.isEncryptionEnabled(self._jsc)
+        os.environ["SPARK_BUFFER_SIZE"] = \
+            str(self._jvm.PythonUtils.getSparkBufferSize(self._jsc))
 
         self.pythonExec = os.environ.get("PYSPARK_PYTHON", 'python')
         self.pythonVer = "%d.%d" % sys.version_info[:2]

From 6121c8f76df6cc4f0085af07a4db34aa0f055ee5 Mon Sep 17 00:00:00 2001
From: german <germanschiavon@gmail.com>
Date: Sat, 5 Dec 2020 06:51:54 +0900
Subject: [PATCH 1193/1280] [SPARK-33660][DOCS][SS] Fix Kafka Headers
 Documentation

### What changes were proposed in this pull request?

Update kafka headers documentation, type is not longer a map but an array

[jira](https://issues.apache.org/jira/browse/SPARK-33660)

### Why are the changes needed?
To help users

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?

It is only documentation

Closes #30605 from Gschiavon/SPARK-33660-fix-kafka-headers-documentation.

Authored-by: german <germanschiavon@gmail.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
(cherry picked from commit d671e053e9806d6b4e43a39f5018aa9718790160)
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 docs/structured-streaming-kafka-integration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md
index 8dc2a738645ce..600cf58a9fd8e 100644
--- a/docs/structured-streaming-kafka-integration.md
+++ b/docs/structured-streaming-kafka-integration.md
@@ -61,7 +61,7 @@ val df = spark
   .option("includeHeaders", "true")
   .load()
 df.selectExpr("CAST(key AS STRING)", "CAST(value AS STRING)", "headers")
-  .as[(String, String, Map)]
+  .as[(String, String, Array[(String, Array[Byte])])]
 
 // Subscribe to multiple topics
 val df = spark

From 874357174d8a2c9ef207a4dcd0ca819e658b2881 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 4 Dec 2020 15:10:51 -0800
Subject: [PATCH 1194/1280] [SPARK-33571][SQL][DOCS][3.0] Add a ref to INT96
 config from the doc for
 `spark.sql.legacy.parquet.datetimeRebaseModeInWrite/Read`

### What changes were proposed in this pull request?
For the SQL configs `spark.sql.legacy.parquet.datetimeRebaseModeInWrite` and `spark.sql.legacy.parquet.datetimeRebaseModeInRead`, improve their descriptions by:
1. Explicitly document on which parquet types, those configs influence on
2. Refer to corresponding configs for `INT96`

### Why are the changes needed?
To avoid user confusions like reposted in SPARK-33571, and make the config description more precise.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running `./dev/scalastyle`.

Closes #30604 from MaxGekk/clarify-rebase-docs-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../scala/org/apache/spark/sql/internal/SQLConf.scala     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 23d1d70bdb22c..0cf980cfc0477 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -2537,7 +2537,9 @@ object SQLConf {
         "to the legacy hybrid (Julian + Gregorian) calendar when writing Parquet files. " +
         "When CORRECTED, Spark will not do rebase and write the dates/timestamps as it is. " +
         "When EXCEPTION, which is the default, Spark will fail the writing if it sees " +
-        "ancient dates/timestamps that are ambiguous between the two calendars.")
+        "ancient dates/timestamps that are ambiguous between the two calendars. " +
+        "This config influences on writes of the following parquet logical types: DATE, " +
+        "TIMESTAMP_MILLIS, TIMESTAMP_MICROS. Spark always performs rebasing for the INT96 type.")
       .version("3.0.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))
@@ -2552,7 +2554,9 @@ object SQLConf {
         "When CORRECTED, Spark will not do rebase and read the dates/timestamps as it is. " +
         "When EXCEPTION, which is the default, Spark will fail the reading if it sees " +
         "ancient dates/timestamps that are ambiguous between the two calendars. This config is " +
-        "only effective if the writer info (like Spark, Hive) of the Parquet files is unknown.")
+        "only effective if the writer info (like Spark, Hive) of the Parquet files is unknown. " +
+        "This config influences on reads of the following parquet logical types: DATE, " +
+        "TIMESTAMP_MILLIS, TIMESTAMP_MICROS. Spark always performs rebasing for the INT96 type.")
       .version("3.0.0")
       .stringConf
       .transform(_.toUpperCase(Locale.ROOT))

From 66b1bdb24119c345db4d0fb0a5d80f7beadff582 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Sun, 6 Dec 2020 12:03:14 +0900
Subject: [PATCH 1195/1280] [MINOR] Fix string interpolation in
 CommandUtils.scala and KafkaDataConsumer.scala

### What changes were proposed in this pull request?

This PR proposes to fix a string interpolation in `CommandUtils.scala` and `KafkaDataConsumer.scala`.

### Why are the changes needed?

To fix a string interpolation bug.

### Does this PR introduce _any_ user-facing change?

Yes, the string will be correctly constructed.

### How was this patch tested?

Existing tests since they were used in exception/log messages.

Closes #30609 from imback82/fix_cache_str_interporlation.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 154f6044033d1a3b4c19c64b206b168bf919cb3b)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala  | 2 +-
 .../org/apache/spark/sql/execution/command/CommandUtils.scala   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
index 5f23029d9fed3..df51c4f77f079 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/consumer/KafkaDataConsumer.scala
@@ -275,7 +275,7 @@ private[kafka010] class KafkaDataConsumer(
     val fetchedData = getOrRetrieveFetchedData(offset)
 
     logDebug(s"Get $groupId $topicPartition nextOffset ${fetchedData.nextOffsetInFetchedData} " +
-      "requested $offset")
+      s"requested $offset")
 
     // The following loop is basically for `failOnDataLoss = false`. When `failOnDataLoss` is
     // `false`, first, we will try to fetch the record at `offset`. If no such record exists, then
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
index 7e456a6af6fc9..34e0ac954010e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CommandUtils.scala
@@ -390,7 +390,7 @@ object CommandUtils extends Logging {
     try {
       sparkSession.catalog.uncacheTable(name)
     } catch {
-      case NonFatal(e) => logWarning("Exception when attempting to uncache $name", e)
+      case NonFatal(e) => logWarning(s"Exception when attempting to uncache $name", e)
     }
   }
 }

From a11a07a81fbbe160804c7220f32298327d427152 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 6 Dec 2020 14:21:46 -0800
Subject: [PATCH 1196/1280] [SPARK-33667][SQL][3.0] Respect the
 `spark.sql.caseSensitive` config while resolving partition spec in v1 `SHOW
 PARTITIONS`

### What changes were proposed in this pull request?
Preprocess the partition spec passed to the V1 SHOW PARTITIONS implementation `ShowPartitionsCommand`, and normalize the passed spec according to the partition columns w.r.t the case sensitivity flag  **spark.sql.caseSensitive**.

### Why are the changes needed?
V1 SHOW PARTITIONS is case sensitive in fact, and doesn't respect the SQL config **spark.sql.caseSensitive** which is false by default, for instance:
```sql
spark-sql> CREATE TABLE tbl1 (price int, qty int, year int, month int)
         > USING parquet
         > PARTITIONED BY (year, month);
spark-sql> INSERT INTO tbl1 PARTITION(year = 2015, month = 1) SELECT 1, 1;
spark-sql> SHOW PARTITIONS tbl1 PARTITION(YEAR = 2015, Month = 1);
Error in query: Non-partitioning column(s) [YEAR, Month] are specified for SHOW PARTITIONS;
```
The `SHOW PARTITIONS` command must show the partition `year = 2015, month = 1` specified by `YEAR = 2015, Month = 1`.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes, the command above works as expected:
```sql
spark-sql> SHOW PARTITIONS tbl1 PARTITION(YEAR = 2015, Month = 1);
year=2015/month=1
```

### How was this patch tested?
By running the affected test suites:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly org.apache.spark.sql.hive.execution.HiveCatalogedDDLSuite"
```

Closes #30626 from MaxGekk/show-partitions-case-sensitivity-test-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/command/tables.scala  | 18 +++++++--------
 .../sql/execution/command/DDLSuite.scala      | 23 ++++++++++++++++++-
 .../sql/hive/execution/HiveCommandSuite.scala |  2 +-
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index 75e0d2c76339f..d8efc8b21b7f0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -1012,20 +1012,18 @@ case class ShowPartitionsCommand(
     DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW PARTITIONS")
 
     /**
-     * Validate the partitioning spec by making sure all the referenced columns are
+     * Normalizes the partition spec w.r.t the partition columns and case sensitivity settings,
+     * and validates the spec by making sure all the referenced columns are
      * defined as partitioning columns in table definition. An AnalysisException exception is
      * thrown if the partitioning spec is invalid.
      */
-    if (spec.isDefined) {
-      val badColumns = spec.get.keySet.filterNot(table.partitionColumnNames.contains)
-      if (badColumns.nonEmpty) {
-        val badCols = badColumns.mkString("[", ", ", "]")
-        throw new AnalysisException(
-          s"Non-partitioning column(s) $badCols are specified for SHOW PARTITIONS")
-      }
-    }
+    val normalizedSpec = spec.map(partitionSpec => PartitioningUtils.normalizePartitionSpec(
+      partitionSpec,
+      table.partitionColumnNames,
+      table.identifier.quotedString,
+      sparkSession.sessionState.conf.resolver))
 
-    val partNames = catalog.listPartitionNames(tableName, spec)
+    val partNames = catalog.listPartitionNames(tableName, normalizedSpec)
     partNames.map(Row(_))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index d0118ef1caeda..64a706d1e0ec0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -3032,7 +3032,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     }
   }
 
-  test("SPARK-33588: case sensitivity of partition spec") {
+  test("SPARK-33588: case sensitivity of partition spec in SHOW TABLE") {
     val t = "part_table"
     withTable(t) {
       sql(s"""
@@ -3052,6 +3052,27 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-33667: case sensitivity of partition spec in SHOW PARTITIONS") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"""
+        |CREATE TABLE $t (price int, qty int, year int, month int)
+        |USING $dataSource
+        |PARTITIONED BY (year, month)""".stripMargin)
+      sql(s"INSERT INTO $t PARTITION(year = 2015, month = 1) SELECT 1, 1")
+      Seq(
+        true -> "PARTITION(year = 2015, month = 1)",
+        false -> "PARTITION(YEAR = 2015, Month = 1)"
+      ).foreach { case (caseSensitive, partitionSpec) =>
+        withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
+          checkAnswer(
+            sql(s"SHOW PARTITIONS $t $partitionSpec"),
+            Row("year=2015/month=1"))
+        }
+      }
+    }
+  }
 }
 
 object FakeLocalFsFileSystem {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
index dcec8bf5c0cc6..2b7cfe57fdae3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
@@ -475,7 +475,7 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       val message2 = intercept[AnalysisException] {
         sql("SHOW PARTITIONS parquet_tab4 PARTITION(abcd=2015, xyz=1)")
       }.getMessage
-      assert(message2.contains("Non-partitioning column(s) [abcd, xyz] are specified"))
+      assert(message2.contains("abcd is not a valid partition column"))
 
       val message3 = intercept[AnalysisException] {
         sql("SHOW PARTITIONS parquet_view1")

From 8029d66260caa6cb4e43b450523e6551488ea617 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 7 Dec 2020 10:07:50 +0900
Subject: [PATCH 1197/1280] [SPARK-33675][INFRA][3.0] Add GitHub Action job to
 publish snapshot

### What changes were proposed in this pull request?

This PR aims to add `GitHub Action` job to publish snapshot from `branch-3.0`.

https://repository.apache.org/content/groups/snapshots/org/apache/spark/spark-core_2.12/3.0.2-SNAPSHOT/

### Why are the changes needed?

This will remove our maintenance burden for `branch-3.0` and will stop automatically when we don't have any commit on `branch-3.0`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

N/A

Closes #30630 from dongjoon-hyun/SPARK-33675-3.0.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/publish_snapshot.yml | 31 ++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 .github/workflows/publish_snapshot.yml

diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
new file mode 100644
index 0000000000000..90132f90336cd
--- /dev/null
+++ b/.github/workflows/publish_snapshot.yml
@@ -0,0 +1,31 @@
+name: Publish Snapshot
+
+on:
+  push:
+    branches:
+    - branch-3.0
+
+jobs:
+  publish-snapshot:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Spark repository
+      uses: actions/checkout@master
+    - name: Cache Maven local repository
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: snapshot-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          snapshot-maven-
+    - name: Install Java 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Publish snapshot
+      env:
+        ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+        ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+        GPG_KEY: "not_used"
+        GPG_PASSPHRASE: "not_used"
+      run: ./dev/create-release/release-build.sh publish-snapshot

From 313a460a0ef5f528dce4da51ed60664a721238ca Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 7 Dec 2020 10:13:22 +0900
Subject: [PATCH 1198/1280] [SPARK-33681][K8S][TESTS][3.0] Increase K8s IT
 timeout to 3 minutes

### What changes were proposed in this pull request?

This PR aims to increase the timeout of K8s integration test of `branch-3.0/2.4` from 2 minutes to 3 minutes which is consistent with `master/branch-3.1`.

### Why are the changes needed?

This will reduce the chance of this kind of failures.
- https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder-K8s/36905/console
```
...
  20/12/07 00:11:23 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources
  20/12/07 00:11:38 WARN TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources
  " did not contain "PySpark Worker Memory Check is: True" The application did not complete.. (KubernetesSuite.scala:249)
```

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the K8s IT Jenkins job.

Closes #30632 from dongjoon-hyun/SPARK-33681.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/deploy/k8s/integrationtest/KubernetesSuite.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index dbb84e3e77dbd..e6fb4e1bdffd4 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -425,6 +425,6 @@ private[spark] object KubernetesSuite {
   val SPARK_DFS_READ_WRITE_TEST = "org.apache.spark.examples.DFSReadWriteTest"
   val SPARK_REMOTE_MAIN_CLASS: String = "org.apache.spark.examples.SparkRemoteFileTest"
   val SPARK_DRIVER_MAIN_CLASS: String = "org.apache.spark.examples.DriverSubmissionTest"
-  val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes))
+  val TIMEOUT = PatienceConfiguration.Timeout(Span(3, Minutes))
   val INTERVAL = PatienceConfiguration.Interval(Span(2, Seconds))
 }

From 1bb37dc2a658b9eeca62d631a081b2f3d70fe291 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Sun, 6 Dec 2020 18:38:44 -0800
Subject: [PATCH 1199/1280] [SPARK-33675][INFRA][3.0][FOLLOWUP] Set GIT_REF to
 branch-3.0

---
 .github/workflows/publish_snapshot.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
index 90132f90336cd..0d9ee17f4c278 100644
--- a/.github/workflows/publish_snapshot.yml
+++ b/.github/workflows/publish_snapshot.yml
@@ -28,4 +28,5 @@ jobs:
         ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
         GPG_KEY: "not_used"
         GPG_PASSPHRASE: "not_used"
+        GIT_REF: "branch-3.0"
       run: ./dev/create-release/release-build.sh publish-snapshot

From 8acbe5b822b7b0ad49079aa223ad52afe70b5afa Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Mon, 7 Dec 2020 11:42:18 +0800
Subject: [PATCH 1200/1280] [SPARK-33592][ML][PYTHON][3.0] Backport Fix:
 Pyspark ML Validator params in estimatorParamMaps may be lost after saving
 and reloading

Fix: Pyspark ML Validator params in estimatorParamMaps may be lost after saving and reloading

When saving validator estimatorParamMaps, will check all nested stages in tuned estimator to get correct param parent.

Two typical cases to manually test:
~~~python
tokenizer = Tokenizer(inputCol="text", outputCol="words")
hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
lr = LogisticRegression()
pipeline = Pipeline(stages=[tokenizer, hashingTF, lr])

paramGrid = ParamGridBuilder() \
    .addGrid(hashingTF.numFeatures, [10, 100]) \
    .addGrid(lr.maxIter, [100, 200]) \
    .build()
tvs = TrainValidationSplit(estimator=pipeline,
                           estimatorParamMaps=paramGrid,
                           evaluator=MulticlassClassificationEvaluator())

tvs.save(tvsPath)
loadedTvs = TrainValidationSplit.load(tvsPath)

~~~

~~~python
lr = LogisticRegression()
ova = OneVsRest(classifier=lr)
grid = ParamGridBuilder().addGrid(lr.maxIter, [100, 200]).build()
evaluator = MulticlassClassificationEvaluator()
tvs = TrainValidationSplit(estimator=ova, estimatorParamMaps=grid, evaluator=evaluator)

tvs.save(tvsPath)
loadedTvs = TrainValidationSplit.load(tvsPath)

~~~

Bug fix.

No

Unit test.

Closes #30539 from WeichenXu123/fix_tuning_param_maps_io.

Authored-by: Weichen Xu <weichen.xudatabricks.com>
Signed-off-by: Ruifeng Zheng <ruifengzfoxmail.com>
(cherry picked from commit 80161238fe9393aabd5fcd56752ff1e43f6989b1)
Signed-off-by: Weichen Xu <weichen.xudatabricks.com>

### What changes were proposed in this pull request?

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #30590 from WeichenXu123/SPARK-33592-bp-3.0.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: Weichen Xu <weichen.xu@databricks.com>
---
 dev/sparktestsupport/modules.py        |  1 +
 python/pyspark/ml/classification.py    | 48 +-------------
 python/pyspark/ml/param/__init__.py    |  6 ++
 python/pyspark/ml/pipeline.py          | 53 +--------------
 python/pyspark/ml/tests/test_tuning.py | 47 +++++++++++--
 python/pyspark/ml/tests/test_util.py   | 84 +++++++++++++++++++++++
 python/pyspark/ml/tuning.py            | 92 ++++++++++++++++++++++++--
 python/pyspark/ml/util.py              | 38 +++++++++++
 8 files changed, 261 insertions(+), 108 deletions(-)
 create mode 100644 python/pyspark/ml/tests/test_util.py

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 75bdec0f40f0a..8705d523208fe 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -548,6 +548,7 @@ def __hash__(self):
         "pyspark.ml.tests.test_stat",
         "pyspark.ml.tests.test_training_summary",
         "pyspark.ml.tests.test_tuning",
+        "pyspark.ml.tests.test_util",
         "pyspark.ml.tests.test_wrapper",
     ],
     blacklisted_python_implementations=[
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index c8e15ca1568cc..1392bc763dd94 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -27,9 +27,9 @@
     _HasVarianceImpurity, _TreeClassifierParams, _TreeEnsembleParams
 from pyspark.ml.regression import _FactorizationMachinesParams, DecisionTreeRegressionModel
 from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, \
+from pyspark.ml.wrapper import JavaParams, \
     JavaPredictor, _JavaPredictorParams, JavaPredictionModel, JavaWrapper
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
+from pyspark.ml.common import inherit_doc
 from pyspark.ml.linalg import Vectors
 from pyspark.sql import DataFrame
 from pyspark.sql.functions import udf, when
@@ -2635,50 +2635,6 @@ def _to_java(self):
         _java_obj.setRawPredictionCol(self.getRawPredictionCol())
         return _java_obj
 
-    def _make_java_param_pair(self, param, value):
-        """
-        Makes a Java param pair.
-        """
-        sc = SparkContext._active_spark_context
-        param = self._resolveParam(param)
-        _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.classification.OneVsRest",
-                                             self.uid)
-        java_param = _java_obj.getParam(param.name)
-        if isinstance(value, JavaParams):
-            # used in the case of an estimator having another estimator as a parameter
-            # the reason why this is not in _py2java in common.py is that importing
-            # Estimator and Model in common.py results in a circular import with inherit_doc
-            java_value = value._to_java()
-        else:
-            java_value = _py2java(sc, value)
-        return java_param.w(java_value)
-
-    def _transfer_param_map_to_java(self, pyParamMap):
-        """
-        Transforms a Python ParamMap into a Java ParamMap.
-        """
-        paramMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
-        for param in self.params:
-            if param in pyParamMap:
-                pair = self._make_java_param_pair(param, pyParamMap[param])
-                paramMap.put([pair])
-        return paramMap
-
-    def _transfer_param_map_from_java(self, javaParamMap):
-        """
-        Transforms a Java ParamMap into a Python ParamMap.
-        """
-        sc = SparkContext._active_spark_context
-        paramMap = dict()
-        for pair in javaParamMap.toList():
-            param = pair.param()
-            if self.hasParam(str(param.name())):
-                if param.name() == "classifier":
-                    paramMap[self.getParam(param.name())] = JavaParams._from_java(pair.value())
-                else:
-                    paramMap[self.getParam(param.name())] = _java2py(sc, pair.value())
-        return paramMap
-
 
 class OneVsRestModel(Model, _OneVsRestParams, JavaMLReadable, JavaMLWritable):
     """
diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py
index 1be8755c7b982..f83875736375b 100644
--- a/python/pyspark/ml/param/__init__.py
+++ b/python/pyspark/ml/param/__init__.py
@@ -426,6 +426,12 @@ def _resolveParam(self, param):
         else:
             raise ValueError("Cannot resolve %r as a param." % param)
 
+    def _testOwnParam(self, param_parent, param_name):
+        """
+        Test the ownership. Return True or False
+        """
+        return self.uid == param_parent and self.hasParam(param_name)
+
     @staticmethod
     def _dummy():
         """
diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py
index 53d07ec9660d9..09e0748ffbb3b 100644
--- a/python/pyspark/ml/pipeline.py
+++ b/python/pyspark/ml/pipeline.py
@@ -25,8 +25,8 @@
 from pyspark.ml.base import Estimator, Model, Transformer
 from pyspark.ml.param import Param, Params
 from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaParams, JavaWrapper
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
+from pyspark.ml.wrapper import JavaParams
+from pyspark.ml.common import inherit_doc
 
 
 @inherit_doc
@@ -174,55 +174,6 @@ def _to_java(self):
 
         return _java_obj
 
-    def _make_java_param_pair(self, param, value):
-        """
-        Makes a Java param pair.
-        """
-        sc = SparkContext._active_spark_context
-        param = self._resolveParam(param)
-        java_param = sc._jvm.org.apache.spark.ml.param.Param(param.parent, param.name, param.doc)
-        if isinstance(value, Params) and hasattr(value, "_to_java"):
-            # Convert JavaEstimator/JavaTransformer object or Estimator/Transformer object which
-            # implements `_to_java` method (such as OneVsRest, Pipeline object) to java object.
-            # used in the case of an estimator having another estimator as a parameter
-            # the reason why this is not in _py2java in common.py is that importing
-            # Estimator and Model in common.py results in a circular import with inherit_doc
-            java_value = value._to_java()
-        else:
-            java_value = _py2java(sc, value)
-        return java_param.w(java_value)
-
-    def _transfer_param_map_to_java(self, pyParamMap):
-        """
-        Transforms a Python ParamMap into a Java ParamMap.
-        """
-        paramMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
-        for param in self.params:
-            if param in pyParamMap:
-                pair = self._make_java_param_pair(param, pyParamMap[param])
-                paramMap.put([pair])
-        return paramMap
-
-    def _transfer_param_map_from_java(self, javaParamMap):
-        """
-        Transforms a Java ParamMap into a Python ParamMap.
-        """
-        sc = SparkContext._active_spark_context
-        paramMap = dict()
-        for pair in javaParamMap.toList():
-            param = pair.param()
-            if self.hasParam(str(param.name())):
-                java_obj = pair.value()
-                if sc._jvm.Class.forName("org.apache.spark.ml.PipelineStage").isInstance(java_obj):
-                    # Note: JavaParams._from_java support both JavaEstimator/JavaTransformer class
-                    # and Estimator/Transformer class which implements `_from_java` static method
-                    # (such as OneVsRest, Pipeline class).
-                    py_obj = JavaParams._from_java(java_obj)
-                else:
-                    py_obj = _java2py(sc, java_obj)
-                paramMap[self.getParam(param.name())] = py_obj
-        return paramMap
-
 
 @inherit_doc
 class PipelineWriter(MLWriter):
diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index b1acaf695245d..b266ced7deb07 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -73,7 +73,21 @@ def test_addGrid(self):
                     .build())
 
 
-class CrossValidatorTests(SparkSessionTestCase):
+class ValidatorTestUtilsMixin:
+    def assert_param_maps_equal(self, paramMaps1, paramMaps2):
+        self.assertEqual(len(paramMaps1), len(paramMaps2))
+        for paramMap1, paramMap2 in zip(paramMaps1, paramMaps2):
+            self.assertEqual(set(paramMap1.keys()), set(paramMap2.keys()))
+            for param in paramMap1.keys():
+                v1 = paramMap1[param]
+                v2 = paramMap2[param]
+                if isinstance(v1, Params):
+                    self.assertEqual(v1.uid, v2.uid)
+                else:
+                    self.assertEqual(v1, v2)
+
+
+class CrossValidatorTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
 
     def test_copy(self):
         dataset = self.spark.createDataFrame([
@@ -253,7 +267,7 @@ def test_save_load_simple_estimator(self):
         loadedCV = CrossValidator.load(cvPath)
         self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
         self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
-        self.assertEqual(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
 
         # test save/load of CrossValidatorModel
         cvModelPath = temp_path + "/cvModel"
@@ -348,6 +362,7 @@ def test_save_load_nested_estimator(self):
         cvPath = temp_path + "/cv"
         cv.save(cvPath)
         loadedCV = CrossValidator.load(cvPath)
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
         self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
 
@@ -364,6 +379,7 @@ def test_save_load_nested_estimator(self):
         cvModelPath = temp_path + "/cvModel"
         cvModel.save(cvModelPath)
         loadedModel = CrossValidatorModel.load(cvModelPath)
+        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
 
     def test_save_load_pipeline_estimator(self):
@@ -398,6 +414,11 @@ def test_save_load_pipeline_estimator(self):
                                   estimatorParamMaps=paramGrid,
                                   evaluator=MulticlassClassificationEvaluator(),
                                   numFolds=2)  # use 3+ folds in practice
+        cvPath = temp_path + "/cv"
+        crossval.save(cvPath)
+        loadedCV = CrossValidator.load(cvPath)
+        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedCV.getEstimator().uid, crossval.getEstimator().uid)
 
         # Run cross-validation, and choose the best set of parameters.
         cvModel = crossval.fit(training)
@@ -418,6 +439,11 @@ def test_save_load_pipeline_estimator(self):
                                    estimatorParamMaps=paramGrid,
                                    evaluator=MulticlassClassificationEvaluator(),
                                    numFolds=2)  # use 3+ folds in practice
+        cv2Path = temp_path + "/cv2"
+        crossval2.save(cv2Path)
+        loadedCV2 = CrossValidator.load(cv2Path)
+        self.assert_param_maps_equal(loadedCV2.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedCV2.getEstimator().uid, crossval2.getEstimator().uid)
 
         # Run cross-validation, and choose the best set of parameters.
         cvModel2 = crossval2.fit(training)
@@ -436,7 +462,7 @@ def test_save_load_pipeline_estimator(self):
             self.assertEqual(loadedStage.uid, originalStage.uid)
 
 
-class TrainValidationSplitTests(SparkSessionTestCase):
+class TrainValidationSplitTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
 
     def test_fit_minimize_metric(self):
         dataset = self.spark.createDataFrame([
@@ -557,7 +583,8 @@ def test_save_load_simple_estimator(self):
         loadedTvs = TrainValidationSplit.load(tvsPath)
         self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
         self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
-        self.assertEqual(loadedTvs.getEstimatorParamMaps(), tvs.getEstimatorParamMaps())
+        self.assert_param_maps_equal(
+            loadedTvs.getEstimatorParamMaps(), tvs.getEstimatorParamMaps())
 
         tvsModelPath = temp_path + "/tvsModel"
         tvsModel.save(tvsModelPath)
@@ -638,6 +665,7 @@ def test_save_load_nested_estimator(self):
         tvsPath = temp_path + "/tvs"
         tvs.save(tvsPath)
         loadedTvs = TrainValidationSplit.load(tvsPath)
+        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
         self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
 
@@ -653,6 +681,7 @@ def test_save_load_nested_estimator(self):
         tvsModelPath = temp_path + "/tvsModel"
         tvsModel.save(tvsModelPath)
         loadedModel = TrainValidationSplitModel.load(tvsModelPath)
+        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
         self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
 
     def test_save_load_pipeline_estimator(self):
@@ -686,6 +715,11 @@ def test_save_load_pipeline_estimator(self):
         tvs = TrainValidationSplit(estimator=pipeline,
                                    estimatorParamMaps=paramGrid,
                                    evaluator=MulticlassClassificationEvaluator())
+        tvsPath = temp_path + "/tvs"
+        tvs.save(tvsPath)
+        loadedTvs = TrainValidationSplit.load(tvsPath)
+        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
 
         # Run train validation split, and choose the best set of parameters.
         tvsModel = tvs.fit(training)
@@ -705,6 +739,11 @@ def test_save_load_pipeline_estimator(self):
         tvs2 = TrainValidationSplit(estimator=nested_pipeline,
                                     estimatorParamMaps=paramGrid,
                                     evaluator=MulticlassClassificationEvaluator())
+        tvs2Path = temp_path + "/tvs2"
+        tvs2.save(tvs2Path)
+        loadedTvs2 = TrainValidationSplit.load(tvs2Path)
+        self.assert_param_maps_equal(loadedTvs2.getEstimatorParamMaps(), paramGrid)
+        self.assertEqual(loadedTvs2.getEstimator().uid, tvs2.getEstimator().uid)
 
         # Run train validation split, and choose the best set of parameters.
         tvsModel2 = tvs2.fit(training)
diff --git a/python/pyspark/ml/tests/test_util.py b/python/pyspark/ml/tests/test_util.py
new file mode 100644
index 0000000000000..498a649e480a8
--- /dev/null
+++ b/python/pyspark/ml/tests/test_util.py
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+from pyspark.ml import Pipeline
+from pyspark.ml.classification import LogisticRegression, OneVsRest
+from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.linalg import Vectors
+from pyspark.ml.util import MetaAlgorithmReadWrite
+from pyspark.testing.mlutils import SparkSessionTestCase
+
+
+class MetaAlgorithmReadWriteTests(SparkSessionTestCase):
+
+    def test_getAllNestedStages(self):
+        def _check_uid_set_equal(stages, expected_stages):
+            uids = set(map(lambda x: x.uid, stages))
+            expected_uids = set(map(lambda x: x.uid, expected_stages))
+            self.assertEqual(uids, expected_uids)
+
+        df1 = self.spark.createDataFrame([
+            (Vectors.dense([1., 2.]), 1.0),
+            (Vectors.dense([-1., -2.]), 0.0),
+        ], ['features', 'label'])
+        df2 = self.spark.createDataFrame([
+            (1., 2., 1.0),
+            (1., 2., 0.0),
+        ], ['a', 'b', 'label'])
+        vs = VectorAssembler(inputCols=['a', 'b'], outputCol='features')
+        lr = LogisticRegression()
+        pipeline = Pipeline(stages=[vs, lr])
+        pipelineModel = pipeline.fit(df2)
+        ova = OneVsRest(classifier=lr)
+        ovaModel = ova.fit(df1)
+
+        ova_pipeline = Pipeline(stages=[vs, ova])
+        nested_pipeline = Pipeline(stages=[ova_pipeline])
+
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(pipeline),
+            [pipeline, vs, lr]
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(pipelineModel),
+            [pipelineModel] + pipelineModel.stages
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(ova),
+            [ova, lr]
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(ovaModel),
+            [ovaModel, lr] + ovaModel.models
+        )
+        _check_uid_set_equal(
+            MetaAlgorithmReadWrite.getAllNestedStages(nested_pipeline),
+            [nested_pipeline, ova_pipeline, vs, ova, lr]
+        )
+
+
+if __name__ == "__main__":
+    from pyspark.ml.tests.test_util import *  # noqa: F401
+
+    try:
+        import xmlrunner  # type: ignore[import]
+        testRunner = xmlrunner.XMLTestRunner(output='target/test-reports', verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index 6283c8bfd3df9..e16ea5753f1b3 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -26,8 +26,9 @@
 from pyspark.ml.param import Params, Param, TypeConverters
 from pyspark.ml.param.shared import HasCollectSubModels, HasParallelism, HasSeed
 from pyspark.ml.util import *
-from pyspark.ml.wrapper import JavaParams
-from pyspark.sql.functions import rand
+from pyspark.ml.wrapper import JavaParams, JavaEstimator, JavaWrapper
+from pyspark.sql.functions import col, lit, rand, UserDefinedFunction
+from pyspark.sql.types import BooleanType
 
 __all__ = ['ParamGridBuilder', 'CrossValidator', 'CrossValidatorModel', 'TrainValidationSplit',
            'TrainValidationSplitModel']
@@ -50,6 +51,10 @@ def _parallelFitTasks(est, train, eva, validation, epm, collectSubModel):
 
     def singleTask():
         index, model = next(modelIter)
+        # TODO: duplicate evaluator to take extra params from input
+        #  Note: Supporting tuning params in evaluator need update method
+        #  `MetaAlgorithmReadWrite.getAllNestedStages`, make it return
+        #  all nested stages and evaluators
         metric = eva.evaluate(model.transform(validation, epm[index]))
         return index, metric, model if collectSubModel else None
 
@@ -169,8 +174,16 @@ def _from_java_impl(cls, java_stage):
         # Load information from java_stage to the instance.
         estimator = JavaParams._from_java(java_stage.getEstimator())
         evaluator = JavaParams._from_java(java_stage.getEvaluator())
-        epms = [estimator._transfer_param_map_from_java(epm)
-                for epm in java_stage.getEstimatorParamMaps()]
+        if isinstance(estimator, JavaEstimator):
+            epms = [estimator._transfer_param_map_from_java(epm)
+                    for epm in java_stage.getEstimatorParamMaps()]
+        elif MetaAlgorithmReadWrite.isMetaEstimator(estimator):
+            # Meta estimator such as Pipeline, OneVsRest
+            epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_from_java(
+                estimator, java_stage.getEstimatorParamMaps())
+        else:
+            raise ValueError('Unsupported estimator used in tuning: ' + str(estimator))
+
         return estimator, epms, evaluator
 
     def _to_java_impl(self):
@@ -181,15 +194,80 @@ def _to_java_impl(self):
         gateway = SparkContext._gateway
         cls = SparkContext._jvm.org.apache.spark.ml.param.ParamMap
 
-        java_epms = gateway.new_array(cls, len(self.getEstimatorParamMaps()))
-        for idx, epm in enumerate(self.getEstimatorParamMaps()):
-            java_epms[idx] = self.getEstimator()._transfer_param_map_to_java(epm)
+        estimator = self.getEstimator()
+        if isinstance(estimator, JavaEstimator):
+            java_epms = gateway.new_array(cls, len(self.getEstimatorParamMaps()))
+            for idx, epm in enumerate(self.getEstimatorParamMaps()):
+                java_epms[idx] = self.getEstimator()._transfer_param_map_to_java(epm)
+        elif MetaAlgorithmReadWrite.isMetaEstimator(estimator):
+            # Meta estimator such as Pipeline, OneVsRest
+            java_epms = _ValidatorSharedReadWrite.meta_estimator_transfer_param_maps_to_java(
+                estimator, self.getEstimatorParamMaps())
+        else:
+            raise ValueError('Unsupported estimator used in tuning: ' + str(estimator))
 
         java_estimator = self.getEstimator()._to_java()
         java_evaluator = self.getEvaluator()._to_java()
         return java_estimator, java_epms, java_evaluator
 
 
+class _ValidatorSharedReadWrite:
+    @staticmethod
+    def meta_estimator_transfer_param_maps_to_java(pyEstimator, pyParamMaps):
+        pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator)
+        stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages))
+        sc = SparkContext._active_spark_context
+
+        paramMapCls = SparkContext._jvm.org.apache.spark.ml.param.ParamMap
+        javaParamMaps = SparkContext._gateway.new_array(paramMapCls, len(pyParamMaps))
+
+        for idx, pyParamMap in enumerate(pyParamMaps):
+            javaParamMap = JavaWrapper._new_java_obj("org.apache.spark.ml.param.ParamMap")
+            for pyParam, pyValue in pyParamMap.items():
+                javaParam = None
+                for pyStage, javaStage in stagePairs:
+                    if pyStage._testOwnParam(pyParam.parent, pyParam.name):
+                        javaParam = javaStage.getParam(pyParam.name)
+                        break
+                if javaParam is None:
+                    raise ValueError('Resolve param in estimatorParamMaps failed: ' + str(pyParam))
+                if isinstance(pyValue, Params) and hasattr(pyValue, "_to_java"):
+                    javaValue = pyValue._to_java()
+                else:
+                    javaValue = _py2java(sc, pyValue)
+                pair = javaParam.w(javaValue)
+                javaParamMap.put([pair])
+            javaParamMaps[idx] = javaParamMap
+        return javaParamMaps
+
+    @staticmethod
+    def meta_estimator_transfer_param_maps_from_java(pyEstimator, javaParamMaps):
+        pyStages = MetaAlgorithmReadWrite.getAllNestedStages(pyEstimator)
+        stagePairs = list(map(lambda stage: (stage, stage._to_java()), pyStages))
+        sc = SparkContext._active_spark_context
+        pyParamMaps = []
+        for javaParamMap in javaParamMaps:
+            pyParamMap = dict()
+            for javaPair in javaParamMap.toList():
+                javaParam = javaPair.param()
+                pyParam = None
+                for pyStage, javaStage in stagePairs:
+                    if pyStage._testOwnParam(javaParam.parent(), javaParam.name()):
+                        pyParam = pyStage.getParam(javaParam.name())
+                if pyParam is None:
+                    raise ValueError('Resolve param in estimatorParamMaps failed: ' +
+                                     javaParam.parent() + '.' + javaParam.name())
+                javaValue = javaPair.value()
+                if sc._jvm.Class.forName("org.apache.spark.ml.util.DefaultParamsWritable") \
+                        .isInstance(javaValue):
+                    pyValue = JavaParams._from_java(javaValue)
+                else:
+                    pyValue = _java2py(sc, javaValue)
+                pyParamMap[pyParam] = pyValue
+            pyParamMaps.append(pyParamMap)
+        return pyParamMaps
+
+
 class _CrossValidatorParams(_ValidatorParams):
     """
     Params for :py:class:`CrossValidator` and :py:class:`CrossValidatorModel`.
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index aac2b38d3f57d..0f9d8e72ab2b7 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -584,3 +584,41 @@ def summary(self):
         no summary exists.
         """
         return (self._call_java("summary"))
+
+
+class MetaAlgorithmReadWrite:
+
+    @staticmethod
+    def isMetaEstimator(pyInstance):
+        from pyspark.ml import Estimator, Pipeline
+        from pyspark.ml.tuning import _ValidatorParams
+        from pyspark.ml.classification import OneVsRest
+        return isinstance(pyInstance, Pipeline) or isinstance(pyInstance, OneVsRest) or \
+            (isinstance(pyInstance, Estimator) and isinstance(pyInstance, _ValidatorParams))
+
+    @staticmethod
+    def getAllNestedStages(pyInstance):
+        from pyspark.ml import Pipeline, PipelineModel
+        from pyspark.ml.tuning import _ValidatorParams
+        from pyspark.ml.classification import OneVsRest, OneVsRestModel
+
+        # TODO: We need to handle `RFormulaModel.pipelineModel` here after Pyspark RFormulaModel
+        #  support pipelineModel property.
+        if isinstance(pyInstance, Pipeline):
+            pySubStages = pyInstance.getStages()
+        elif isinstance(pyInstance, PipelineModel):
+            pySubStages = pyInstance.stages
+        elif isinstance(pyInstance, _ValidatorParams):
+            raise ValueError('PySpark does not support nested validator.')
+        elif isinstance(pyInstance, OneVsRest):
+            pySubStages = [pyInstance.getClassifier()]
+        elif isinstance(pyInstance, OneVsRestModel):
+            pySubStages = [pyInstance.getClassifier()] + pyInstance.models
+        else:
+            pySubStages = []
+
+        nestedStages = []
+        for pySubStage in pySubStages:
+            nestedStages.extend(MetaAlgorithmReadWrite.getAllNestedStages(pySubStage))
+
+        return [pyInstance] + nestedStages

From 955565849593b521b801f4166f8bbac9e411b0e1 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 7 Dec 2020 19:19:46 +0900
Subject: [PATCH 1201/1280] [SPARK-33670][SQL][3.0] Verify the partition
 provider is Hive in v1 SHOW TABLE EXTENDED

### What changes were proposed in this pull request?
Invoke the check `DDLUtils.verifyPartitionProviderIsHive()` from V1 implementation of `SHOW TABLE EXTENDED` when partition specs are specified.

This PR is some kind of follow up https://github.com/apache/spark/pull/16373 and https://github.com/apache/spark/pull/15515.

### Why are the changes needed?
To output an user friendly error with recommendation like
**"
... partition metadata is not stored in the Hive metastore. To import this information into the metastore, run `msck repair table tableName`
"**
instead of silently output an empty result.

### Does this PR introduce _any_ user-facing change?
Yes.

### How was this patch tested?
By running the affected test suites, in particular:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *HiveCatalogedDDLSuite"
$ build/sbt -Phive-2.3 -Phive-thriftserver "hive/test:testOnly *PartitionProviderCompatibilitySuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: HyukjinKwon <gurwls223apache.org>
(cherry picked from commit 29096a8869c95221dc75ce7fd3d098680bef4f55)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #30640 from MaxGekk/show-table-extended-verifyPartitionProviderIsHive-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/command/tables.scala       |  3 +++
 .../spark/sql/execution/command/DDLSuite.scala     | 10 ++++++++++
 .../hive/PartitionProviderCompatibilitySuite.scala | 14 ++++++++++----
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index d8efc8b21b7f0..d550683485779 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -885,6 +885,9 @@ case class ShowTablesCommand(
       // Note: tableIdentifierPattern should be non-empty, otherwise a [[ParseException]]
       // should have been thrown by the sql parser.
       val table = catalog.getTableMetadata(TableIdentifier(tableIdentifierPattern.get, Some(db)))
+
+      DDLUtils.verifyPartitionProviderIsHive(sparkSession, table, "SHOW TABLE EXTENDED")
+
       val tableIdent = table.identifier
       val normalizedSpec = PartitioningUtils.normalizePartitionSpec(
         partitionSpec.get,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 64a706d1e0ec0..cf464d1c37d4b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -3073,6 +3073,16 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-33670: show partitions from a datasource table") {
+    import testImplicits._
+    val t = "part_datasrc"
+    withTable(t) {
+      val df = (1 to 3).map(i => (i, s"val_$i", i * 2)).toDF("a", "b", "c")
+      df.write.partitionBy("a").format("parquet").mode(SaveMode.Overwrite).saveAsTable(t)
+      assert(sql(s"SHOW TABLE EXTENDED LIKE '$t' PARTITION(a = 1)").count() === 1)
+    }
+  }
 }
 
 object FakeLocalFsFileSystem {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
index 80afc9d8f44bc..e1b0637963b75 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionProviderCompatibilitySuite.scala
@@ -53,7 +53,8 @@ class PartitionProviderCompatibilitySuite
       s"ALTER TABLE $tableName PARTITION (partCol=1) SET LOCATION '/foo'",
       s"ALTER TABLE $tableName DROP PARTITION (partCol=1)",
       s"DESCRIBE $tableName PARTITION (partCol=1)",
-      s"SHOW PARTITIONS $tableName")
+      s"SHOW PARTITIONS $tableName",
+      s"SHOW TABLE EXTENDED LIKE '$tableName' PARTITION (partCol=1)")
 
     withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "true") {
       for (cmd <- unsupportedCommands) {
@@ -124,10 +125,15 @@ class PartitionProviderCompatibilitySuite
         }
         // disabled
         withSQLConf(SQLConf.HIVE_MANAGE_FILESOURCE_PARTITIONS.key -> "false") {
-          val e = intercept[AnalysisException] {
-            spark.sql(s"show partitions test")
+          Seq(
+            "SHOW PARTITIONS test",
+            "SHOW TABLE EXTENDED LIKE 'test' PARTITION (partCol=1)"
+          ).foreach { showPartitions =>
+            val e = intercept[AnalysisException] {
+              spark.sql(showPartitions)
+            }
+            assert(e.getMessage.contains("filesource partition management is disabled"))
           }
-          assert(e.getMessage.contains("filesource partition management is disabled"))
           spark.sql("refresh table test")
           assert(spark.sql("select * from test").count() == 5)
         }

From 46a0ec551b25b909e474aa7685aeb90e22d5de1a Mon Sep 17 00:00:00 2001
From: Linhong Liu <linhong.liu@databricks.com>
Date: Tue, 8 Dec 2020 03:44:53 +0000
Subject: [PATCH 1202/1280] [SPARK-32680][SQL][3.0] Don't Preprocess V2 CTAS
 with Unresolved Query

### What changes were proposed in this pull request?
The analyzer rule `PreprocessTableCreation` will preprocess table creation related logical plan. But for
CTAS, if the sub-query can't be resolved, preprocess it will cause "Invalid call to toAttribute on unresolved
object" (instead of a user-friendly error msg: "table or view not found").
This PR fixes this wrongly preprocess for CTAS using V2 catalog.

### Why are the changes needed?
bug fix

### Does this PR introduce _any_ user-facing change?
The error message for CTAS with a non-exists table changed from:
`UnresolvedException: Invalid call to toAttribute on unresolved object, tree: xxx` to
`AnalysisException: Table or view not found: xxx`

### How was this patch tested?
added test

Closes #30649 from linhongliu-db/fix-ctas-3.0.

Authored-by: Linhong Liu <linhong.liu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/execution/datasources/rules.scala   | 2 +-
 .../test/scala/org/apache/spark/sql/DataFrameSuite.scala | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 95343e2872def..888678d6ff36d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -240,7 +240,7 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
         c.copy(tableDesc = normalizedTable.copy(schema = reorderedSchema))
       }
 
-    case create: V2CreateTablePlan =>
+    case create: V2CreateTablePlan if create.childrenResolved =>
       val schema = create.tableSchema
       val partitioning = create.partitioning
       val identifier = create.tableName
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 85cbe458aebde..7c410e862e10c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.expressions.Uuid
 import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
 import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, OneRowRelation}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.connector.FakeV2Provider
 import org.apache.spark.sql.execution.{FilterExec, QueryExecution, WholeStageCodegenExec}
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.HashAggregateExec
@@ -2361,6 +2362,14 @@ class DataFrameSuite extends QueryTest
     assert(e.getMessage.contains("Table or view not found:"))
   }
 
+  test("SPARK-32680: Don't analyze CTAS with unresolved query") {
+    val v2Source = classOf[FakeV2Provider].getName
+    val e = intercept[AnalysisException] {
+      sql(s"CREATE TABLE t USING $v2Source AS SELECT * from nonexist")
+    }
+    assert(e.getMessage.contains("Table or view not found:"))
+  }
+
   test("CalendarInterval reflection support") {
     val df = Seq((1, new CalendarInterval(1, 2, 3))).toDF("a", "b")
     checkAnswer(df.selectExpr("b"), Row(new CalendarInterval(1, 2, 3)))

From ea7c2a15a02d8c8cf3e3f5a1260da76829d59596 Mon Sep 17 00:00:00 2001
From: luluorta <luluorta@gmail.com>
Date: Tue, 8 Dec 2020 20:45:25 +0900
Subject: [PATCH 1203/1280] [SPARK-33677][SQL] Skip LikeSimplification rule if
 pattern contains any escapeChar

### What changes were proposed in this pull request?
`LikeSimplification` rule does not work correctly for many cases that have patterns containing escape characters, for example:

`SELECT s LIKE 'm%aca' ESCAPE '%' FROM t`
`SELECT s LIKE 'maacaa' ESCAPE 'a' FROM t`

For simpilicy, this PR makes this rule just be skipped if `pattern` contains any `escapeChar`.

### Why are the changes needed?
Result corrupt.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added Unit test.

Closes #30625 from luluorta/SPARK-33677.

Authored-by: luluorta <luluorta@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
(cherry picked from commit 99613cd5815b2de12274027dee0c0a6c0c57bd95)
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/optimizer/expressions.scala  | 18 ++++---
 .../optimizer/LikeSimplificationSuite.scala   | 48 +++++++++++++++++++
 .../org/apache/spark/sql/SQLQuerySuite.scala  | 14 ++++++
 3 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index 7773f5c80e061..7cbeb47c30be9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -525,27 +525,33 @@ object LikeSimplification extends Rule[LogicalPlan] {
   private val equalTo = "([^_%]*)".r
 
   def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
-    case Like(input, Literal(pattern, StringType), escapeChar) =>
+    case l @ Like(input, Literal(pattern, StringType), escapeChar) =>
       if (pattern == null) {
         // If pattern is null, return null value directly, since "col like null" == null.
         Literal(null, BooleanType)
       } else {
-        val escapeStr = String.valueOf(escapeChar)
         pattern.toString match {
-          case startsWith(prefix) if !prefix.endsWith(escapeStr) =>
+          // There are three different situations when pattern containing escapeChar:
+          // 1. pattern contains invalid escape sequence, e.g. 'm\aca'
+          // 2. pattern contains escaped wildcard character, e.g. 'ma\%ca'
+          // 3. pattern contains escaped escape character, e.g. 'ma\\ca'
+          // Although there are patterns can be optimized if we handle the escape first, we just
+          // skip this rule if pattern contains any escapeChar for simplicity.
+          case p if p.contains(escapeChar) => l
+          case startsWith(prefix) =>
             StartsWith(input, Literal(prefix))
           case endsWith(postfix) =>
             EndsWith(input, Literal(postfix))
           // 'a%a' pattern is basically same with 'a%' && '%a'.
           // However, the additional `Length` condition is required to prevent 'a' match 'a%a'.
-          case startsAndEndsWith(prefix, postfix) if !prefix.endsWith(escapeStr) =>
+          case startsAndEndsWith(prefix, postfix) =>
             And(GreaterThanOrEqual(Length(input), Literal(prefix.length + postfix.length)),
               And(StartsWith(input, Literal(prefix)), EndsWith(input, Literal(postfix))))
-          case contains(infix) if !infix.endsWith(escapeStr) =>
+          case contains(infix) =>
             Contains(input, Literal(infix))
           case equalTo(str) =>
             EqualTo(input, Literal(str))
-          case _ => Like(input, Literal.create(pattern, StringType), escapeChar)
+          case _ => l
         }
       }
   }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
index 436f62e4225c8..1812dce0da426 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LikeSimplificationSuite.scala
@@ -116,4 +116,52 @@ class LikeSimplificationSuite extends PlanTest {
     val optimized2 = Optimize.execute(originalQuery2.analyze)
     comparePlans(optimized2, originalQuery2.analyze)
   }
+
+  test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
+    val originalQuery1 =
+      testRelation
+        .where(('a like "abc%") || ('a like "\\abc%"))
+    val optimized1 = Optimize.execute(originalQuery1.analyze)
+    val correctAnswer1 = testRelation
+      .where(StartsWith('a, "abc") || ('a like "\\abc%"))
+      .analyze
+    comparePlans(optimized1, correctAnswer1)
+
+    val originalQuery2 =
+      testRelation
+        .where(('a like "%xyz") || ('a like "%xyz\\"))
+    val optimized2 = Optimize.execute(originalQuery2.analyze)
+    val correctAnswer2 = testRelation
+      .where(EndsWith('a, "xyz") || ('a like "%xyz\\"))
+      .analyze
+    comparePlans(optimized2, correctAnswer2)
+
+    val originalQuery3 =
+      testRelation
+        .where(('a like ("@bc%def", '@')) || ('a like "abc%def"))
+    val optimized3 = Optimize.execute(originalQuery3.analyze)
+    val correctAnswer3 = testRelation
+      .where(('a like ("@bc%def", '@')) ||
+        (Length('a) >= 6 && (StartsWith('a, "abc") && EndsWith('a, "def"))))
+      .analyze
+    comparePlans(optimized3, correctAnswer3)
+
+    val originalQuery4 =
+      testRelation
+        .where(('a like "%mn%") || ('a like ("%mn%", '%')))
+    val optimized4 = Optimize.execute(originalQuery4.analyze)
+    val correctAnswer4 = testRelation
+      .where(Contains('a, "mn") || ('a like ("%mn%", '%')))
+      .analyze
+    comparePlans(optimized4, correctAnswer4)
+
+    val originalQuery5 =
+      testRelation
+        .where(('a like "abc") || ('a like ("abbc", 'b')))
+    val optimized5 = Optimize.execute(originalQuery5.analyze)
+    val correctAnswer5 = testRelation
+      .where(('a === "abc") || ('a like ("abbc", 'b')))
+      .analyze
+    comparePlans(optimized5, correctAnswer5)
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index bcd3e79dcc054..1e2bb4810a6bd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3535,6 +3535,20 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       }
     }
   }
+
+  test("SPARK-33677: LikeSimplification should be skipped if pattern contains any escapeChar") {
+    withTempView("df") {
+      Seq("m@ca").toDF("s").createOrReplaceTempView("df")
+
+      val e = intercept[AnalysisException] {
+        sql("SELECT s LIKE 'm%@ca' ESCAPE '%' FROM df").collect()
+      }
+      assert(e.message.contains("the pattern 'm%@ca' is invalid, " +
+        "the escape character is not allowed to precede '@'"))
+
+      checkAnswer(sql("SELECT s LIKE 'm@@ca' ESCAPE '@' FROM df"), Row(true))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])

From eae6a3e9dc75912e2fbe80d86f05cce629de8022 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 8 Dec 2020 11:41:35 -0800
Subject: [PATCH 1204/1280] [SPARK-32110][SQL] normalize special floating
 numbers in HyperLogLog++

### What changes were proposed in this pull request?

Currently, Spark treats 0.0 and -0.0 semantically equal, while it still retains the difference between them so that users can see -0.0 when displaying the data set.

The comparison expressions in Spark take care of the special floating numbers and implement the correct semantic. However, Spark doesn't always use these comparison expressions to compare values, and we need to normalize the special floating numbers before comparing them in these places:
1. GROUP BY
2. join keys
3. window partition keys

This PR fixes one more place that compares values without using comparison expressions: HyperLogLog++

### Why are the changes needed?

Fix the query result

### Does this PR introduce _any_ user-facing change?

Yes, the result of HyperLogLog++ becomes correct now.

### How was this patch tested?

a new test case, and a few more test cases that pass before this PR to improve test coverage.

Closes #30673 from cloud-fan/bug.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 6fd234503cf1e85715ccd3bda42f29dae1daa71b)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../optimizer/NormalizeFloatingNumbers.scala  | 45 +++++-----
 .../util/HyperLogLogPlusPlusHelper.scala      |  8 +-
 .../catalyst/expressions/PredicateSuite.scala | 90 +++++++++++++++++++
 .../aggregate/HyperLogLogPlusPlusSuite.scala  | 24 ++++-
 4 files changed, 144 insertions(+), 23 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index f0cf6717ade27..592652212b22e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -134,6 +134,28 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
 
     case _ => throw new IllegalStateException(s"fail to normalize $expr")
   }
+
+  val FLOAT_NORMALIZER: Any => Any = (input: Any) => {
+    val f = input.asInstanceOf[Float]
+    if (f.isNaN) {
+      Float.NaN
+    } else if (f == -0.0f) {
+      0.0f
+    } else {
+      f
+    }
+  }
+
+  val DOUBLE_NORMALIZER: Any => Any = (input: Any) => {
+    val d = input.asInstanceOf[Double]
+    if (d.isNaN) {
+      Double.NaN
+    } else if (d == -0.0d) {
+      0.0d
+    } else {
+      d
+    }
+  }
 }
 
 case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with ExpectsInputTypes {
@@ -143,27 +165,8 @@ case class NormalizeNaNAndZero(child: Expression) extends UnaryExpression with E
   override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(FloatType, DoubleType))
 
   private lazy val normalizer: Any => Any = child.dataType match {
-    case FloatType => (input: Any) => {
-      val f = input.asInstanceOf[Float]
-      if (f.isNaN) {
-        Float.NaN
-      } else if (f == -0.0f) {
-        0.0f
-      } else {
-        f
-      }
-    }
-
-    case DoubleType => (input: Any) => {
-      val d = input.asInstanceOf[Double]
-      if (d.isNaN) {
-        Double.NaN
-      } else if (d == -0.0d) {
-        0.0d
-      } else {
-        d
-      }
-    }
+    case FloatType => NormalizeFloatingNumbers.FLOAT_NORMALIZER
+    case DoubleType => NormalizeFloatingNumbers.DOUBLE_NORMALIZER
   }
 
   override def nullSafeEval(input: Any): Any = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
index ea619c6a7666c..6471a746f2edf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/HyperLogLogPlusPlusHelper.scala
@@ -22,6 +22,7 @@ import java.util
 
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.XxHash64Function
+import org.apache.spark.sql.catalyst.optimizer.NormalizeFloatingNumbers.{DOUBLE_NORMALIZER, FLOAT_NORMALIZER}
 import org.apache.spark.sql.types._
 
 // A helper class for HyperLogLogPlusPlus.
@@ -88,7 +89,12 @@ class HyperLogLogPlusPlusHelper(relativeSD: Double) extends Serializable {
    *
    * Variable names in the HLL++ paper match variable names in the code.
    */
-  def update(buffer: InternalRow, bufferOffset: Int, value: Any, dataType: DataType): Unit = {
+  def update(buffer: InternalRow, bufferOffset: Int, _value: Any, dataType: DataType): Unit = {
+    val value = dataType match {
+      case FloatType => FLOAT_NORMALIZER.apply(_value)
+      case DoubleType => DOUBLE_NORMALIZER.apply(_value)
+      case _ => _value
+    }
     // Create the hashed value 'x'.
     val x = XxHash64Function.hash(value, dataType, 42L)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index a36baec1a0b99..6f75623dc59ae 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -554,4 +554,94 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(GreaterThan(Literal(Float.NaN), Literal(Float.NaN)), false)
     checkEvaluation(GreaterThan(Literal(0.0F), Literal(-0.0F)), false)
   }
+
+  test("SPARK-32110: compare special double/float values in array") {
+    def createUnsafeDoubleArray(d: Double): Literal = {
+      Literal(UnsafeArrayData.fromPrimitiveArray(Array(d)), ArrayType(DoubleType))
+    }
+    def createSafeDoubleArray(d: Double): Literal = {
+      Literal(new GenericArrayData(Array(d)), ArrayType(DoubleType))
+    }
+    def createUnsafeFloatArray(d: Double): Literal = {
+      Literal(UnsafeArrayData.fromPrimitiveArray(Array(d.toFloat)), ArrayType(FloatType))
+    }
+    def createSafeFloatArray(d: Double): Literal = {
+      Literal(new GenericArrayData(Array(d.toFloat)), ArrayType(FloatType))
+    }
+    def checkExpr(
+        exprBuilder: (Expression, Expression) => Expression,
+        left: Double,
+        right: Double,
+        expected: Any): Unit = {
+      // test double
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleArray(left), createUnsafeDoubleArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleArray(left), createSafeDoubleArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeDoubleArray(left), createSafeDoubleArray(right)), expected)
+      // test float
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatArray(left), createUnsafeFloatArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatArray(left), createSafeFloatArray(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeFloatArray(left), createSafeFloatArray(right)), expected)
+    }
+
+    checkExpr(EqualTo, Double.NaN, Double.NaN, true)
+    checkExpr(EqualTo, Double.NaN, Double.PositiveInfinity, false)
+    checkExpr(EqualTo, 0.0, -0.0, true)
+    checkExpr(GreaterThan, Double.NaN, Double.PositiveInfinity, true)
+    checkExpr(GreaterThan, Double.NaN, Double.NaN, false)
+    checkExpr(GreaterThan, 0.0, -0.0, false)
+  }
+
+  test("SPARK-32110: compare special double/float values in struct") {
+    def createUnsafeDoubleRow(d: Double): Literal = {
+      val dt = new StructType().add("d", "double")
+      val converter = UnsafeProjection.create(dt)
+      val unsafeRow = converter.apply(InternalRow(d))
+      Literal(unsafeRow, dt)
+    }
+    def createSafeDoubleRow(d: Double): Literal = {
+      Literal(InternalRow(d), new StructType().add("d", "double"))
+    }
+    def createUnsafeFloatRow(d: Double): Literal = {
+      val dt = new StructType().add("f", "float")
+      val converter = UnsafeProjection.create(dt)
+      val unsafeRow = converter.apply(InternalRow(d.toFloat))
+      Literal(unsafeRow, dt)
+    }
+    def createSafeFloatRow(d: Double): Literal = {
+      Literal(InternalRow(d.toFloat), new StructType().add("f", "float"))
+    }
+    def checkExpr(
+        exprBuilder: (Expression, Expression) => Expression,
+        left: Double,
+        right: Double,
+        expected: Any): Unit = {
+      // test double
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleRow(left), createUnsafeDoubleRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeDoubleRow(left), createSafeDoubleRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeDoubleRow(left), createSafeDoubleRow(right)), expected)
+      // test float
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatRow(left), createUnsafeFloatRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createUnsafeFloatRow(left), createSafeFloatRow(right)), expected)
+      checkEvaluation(
+        exprBuilder(createSafeFloatRow(left), createSafeFloatRow(right)), expected)
+    }
+
+    checkExpr(EqualTo, Double.NaN, Double.NaN, true)
+    checkExpr(EqualTo, Double.NaN, Double.PositiveInfinity, false)
+    checkExpr(EqualTo, 0.0, -0.0, true)
+    checkExpr(GreaterThan, Double.NaN, Double.PositiveInfinity, true)
+    checkExpr(GreaterThan, Double.NaN, Double.NaN, false)
+    checkExpr(GreaterThan, 0.0, -0.0, false)
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
index 98fd04c9cca91..1afccea5aef15 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlusSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.expressions.aggregate
 
+import java.lang.{Double => JDouble}
 import java.util.Random
 
 import scala.collection.mutable
@@ -24,7 +25,7 @@ import scala.collection.mutable
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{BoundReference, SpecificInternalRow}
-import org.apache.spark.sql.types.{DataType, IntegerType}
+import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType}
 
 class HyperLogLogPlusPlusSuite extends SparkFunSuite {
 
@@ -153,4 +154,25 @@ class HyperLogLogPlusPlusSuite extends SparkFunSuite {
     // Check if the buffers are equal.
     assert(buffer2 == buffer1a, "Buffers should be equal")
   }
+
+  test("SPARK-32110: add 0.0 and -0.0") {
+    val (hll, input, buffer) = createEstimator(0.05, DoubleType)
+    input.setDouble(0, 0.0)
+    hll.update(buffer, input)
+    input.setDouble(0, -0.0)
+    hll.update(buffer, input)
+    evaluateEstimate(hll, buffer, 1);
+  }
+
+  test("SPARK-32110: add NaN") {
+    val (hll, input, buffer) = createEstimator(0.05, DoubleType)
+    input.setDouble(0, Double.NaN)
+    hll.update(buffer, input)
+    val specialNaN = JDouble.longBitsToDouble(0x7ff1234512345678L)
+    assert(JDouble.isNaN(specialNaN))
+    assert(JDouble.doubleToRawLongBits(Double.NaN) != JDouble.doubleToRawLongBits(specialNaN))
+    input.setDouble(0, specialNaN)
+    hll.update(buffer, input)
+    evaluateEstimate(hll, buffer, 1);
+  }
 }

From a4c5e544242ecb11d2480a36b70855b4a05cb842 Mon Sep 17 00:00:00 2001
From: Holden Karau <hkarau@apple.com>
Date: Thu, 10 Dec 2020 11:35:55 +0900
Subject: [PATCH 1205/1280] [SPARK-33727][K8S] Fall back from gnupg.net to
 openpgp.org

### What changes were proposed in this pull request?

While building R docker image if we can't fetch the key from gnupg.net fall back to openpgp.org

### Why are the changes needed?

gnupg.net key servers are flaky and sometimes fail to resolve or return keys.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Tried to add key on my desktop, it failed, then tried to add key with openpgp.org and it succeed.

Closes #30696 from holdenk/SPARK-33727-gnupg-server-is-flaky.

Authored-by: Holden Karau <hkarau@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 991b7977b5006e1e0d02b7d67a3e0fc50f5a9f66)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../docker/src/main/dockerfiles/spark/bindings/R/Dockerfile     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
index bd645e40677d0..f63f2d0d58e22 100644
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile
@@ -29,7 +29,7 @@ RUN mkdir ${SPARK_HOME}/R
 RUN \
   echo "deb http://cloud.r-project.org/bin/linux/debian buster-cran35/" >> /etc/apt/sources.list && \
   apt install -y gnupg && \
-  apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' && \
+  (apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' || apt-key adv --keyserver keys.openpgp.org --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF') && \
   apt-get update && \
   apt install -y -t buster-cran35 r-base r-base-dev && \
   rm -rf /var/cache/apt/*

From 2921c4ebfc0400b88a7c19a1f865d6444ac79089 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Thu, 10 Dec 2020 12:21:49 +0900
Subject: [PATCH 1206/1280] [SPARK-33725][BUILD][3.0] Upgrade snappy-java to
 1.1.8.2

### What changes were proposed in this pull request?

This upgrades snappy-java to 1.1.8.2.

### Why are the changes needed?

Minor version upgrade that includes:

- [Fixed](https://github.com/xerial/snappy-java/pull/265) an initialization issue when using a recent Mac OS X version
- Support Apple Silicon (M1, Mac-aarch64)
- Fixed the pure-java Snappy fallback logic when no native library for your platform is found.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Unit test.

Closes #30698 from viirya/upgrade-snappy-3.0.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +-
 pom.xml                                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index 2068d11ffe38e..cc23a79a745d8 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -188,7 +188,7 @@ shims/0.7.45//shims-0.7.45.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 snappy/0.2//snappy-0.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 06fcff8d94f27..80bee1491c8fe 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -202,7 +202,7 @@ shims/0.7.45//shims-0.7.45.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index 9c219816ca1a2..da18e10e5a037 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -217,7 +217,7 @@ shims/0.7.45//shims-0.7.45.jar
 slf4j-api/1.7.30//slf4j-api-1.7.30.jar
 slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
 snakeyaml/1.24//snakeyaml-1.24.jar
-snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
+snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
 spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
 spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
 spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
diff --git a/pom.xml b/pom.xml
index d41d61c1e12d3..60b57fb2d039e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -173,7 +173,7 @@
     <scalafmt.skip>true</scalafmt.skip>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
     <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-    <snappy.version>1.1.7.5</snappy.version>
+    <snappy.version>1.1.8.2</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <commons-codec.version>1.10</commons-codec.version>
     <commons-compress.version>1.20</commons-compress.version>

From 83af03643fa6858c539f78f27cf9231b3d98d4da Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Thu, 10 Dec 2020 01:19:50 -0800
Subject: [PATCH 1207/1280] [SPARK-33732][K8S][TESTS][3.0] Kubernetes
 integration tests doesn't work with Minikube 1.9+

### What changes were proposed in this pull request?

This is a backport of #30700 .

This PR changes `Minikube.scala` for Kubernetes integration tests to work with Minikube 1.9+.
`Minikube.scala` assumes that `apiserver.key` and `apiserver.crt` are in `~/.minikube/`.
But as of Minikube 1.9, they are in `~/.minikube/profiles/<profile>`.

### Why are the changes needed?

Currently, Kubernetes integration tests doesn't work with Minikube 1.9+.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

I confirmed the following test passes.
```
$ build/sbt -Pkubernetes -Pkubernetes-integration-tests package 'kubernetes-integration-tests/testOnly -- -z "SparkPi with no"'
```

Closes #30702 from sarutak/minikube-1.9-branch-3.0.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../backend/minikube/Minikube.scala           | 25 +++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
index a7d35b67d1b92..7753b0977aa57 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala
@@ -16,7 +16,7 @@
  */
 package org.apache.spark.deploy.k8s.integrationtest.backend.minikube
 
-import java.nio.file.Paths
+import java.nio.file.{Files, Paths}
 
 import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient}
 
@@ -68,15 +68,23 @@ private[spark] object Minikube extends Logging {
   def getKubernetesClient: DefaultKubernetesClient = {
     val kubernetesMaster = s"https://${getMinikubeIp}:8443"
     val userHome = System.getProperty("user.home")
+    val minikubeBasePath = Paths.get(userHome, MINIKUBE_PATH).toString
+    val profileDir = if (Files.exists(Paths.get(minikubeBasePath, "apiserver.crt"))) {
+      // For Minikube <1.9
+      ""
+    } else {
+      // For Minikube >=1.9
+      Paths.get("profiles", executeMinikube("profile")(0)).toString
+    }
+    val apiServerCertPath = Paths.get(minikubeBasePath, profileDir, "apiserver.crt")
+    val apiServerKeyPath = Paths.get(minikubeBasePath, profileDir, "apiserver.key")
     val kubernetesConf = new ConfigBuilder()
       .withApiVersion("v1")
       .withMasterUrl(kubernetesMaster)
       .withCaCertFile(
         Paths.get(userHome, MINIKUBE_PATH, "ca.crt").toFile.getAbsolutePath)
-      .withClientCertFile(
-        Paths.get(userHome, MINIKUBE_PATH, "apiserver.crt").toFile.getAbsolutePath)
-      .withClientKeyFile(
-        Paths.get(userHome, MINIKUBE_PATH, "apiserver.key").toFile.getAbsolutePath)
+      .withClientCertFile(apiServerCertPath.toFile.getAbsolutePath)
+      .withClientKeyFile(apiServerKeyPath.toFile.getAbsolutePath)
       .build()
     new DefaultKubernetesClient(kubernetesConf)
   }
@@ -120,8 +128,11 @@ private[spark] object Minikube extends Logging {
 
   private def executeMinikube(action: String, args: String*): Seq[String] = {
     ProcessUtils.executeProcess(
-      Array("bash", "-c", s"minikube $action ${args.mkString(" ")}"),
-      MINIKUBE_STARTUP_TIMEOUT_SECONDS)
+      Array("bash", "-c", s"MINIKUBE_IN_STYLE=true minikube $action ${args.mkString(" ")}"),
+      MINIKUBE_STARTUP_TIMEOUT_SECONDS).filter{x =>
+      !x.contains("There is a newer version of minikube") &&
+      !x.contains("https://github.com/kubernetes")
+    }
   }
 
   def minikubeServiceAction(args: String*): String = {

From 728bdb7b0da29f393f528eff960156ca973cbf38 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Fri, 11 Dec 2020 14:15:56 +0900
Subject: [PATCH 1208/1280] [SPARK-33749][BUILD][PYTHON] Exclude target
 directory in pycodestyle and flake8

Once you build and ran K8S tests, Python lint fails as below:

```bash
$ ./dev/lint-python
```

Before this PR:

```
starting python compilation test...
python compilation succeeded.

downloading pycodestyle from https://raw.githubusercontent.com/PyCQA/pycodestyle/2.6.0/pycodestyle.py...
starting pycodestyle test...
pycodestyle checks failed:
./resource-managers/kubernetes/integration-tests/target/spark-dist-unpacked/python/pyspark/cloudpickle/cloudpickle.py:15:101: E501 line too long (105 > 100 characters)
./resource-managers/kubernetes/integration-tests/target/spark-dist-unpacked/python/docs/source/conf.py:60:101: E501 line too long (124 > 100 characters)
...
```

After this PR:

```
starting python compilation test...
python compilation succeeded.

downloading pycodestyle from https://raw.githubusercontent.com/PyCQA/pycodestyle/2.6.0/pycodestyle.py...
starting pycodestyle test...
pycodestyle checks passed.

starting flake8 test...
flake8 checks passed.

starting mypy test...
mypy checks passed.

starting sphinx-build tests...
sphinx-build checks passed.
```

This PR excludes target directory to avoid such cases in the future.

To make it easier to run linters

No, dev-only.

Manually tested va running `./dev/lint-python`.

Closes #30718 from HyukjinKwon/SPARK-33749.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit cd7a30641f25f99452b7eb46ee2b3c5d59b2c542)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 dev/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/tox.ini b/dev/tox.ini
index 3ee13c09b5de7..49b673bd63db6 100644
--- a/dev/tox.ini
+++ b/dev/tox.ini
@@ -16,4 +16,4 @@
 [pycodestyle]
 ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504
 max-line-length=100
-exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*
+exclude=*/target/*,cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/*,.git/*

From 9439e11e65ed808b2dd990a27cacd44b2a6f0009 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Fri, 11 Dec 2020 01:49:13 -0800
Subject: [PATCH 1209/1280] [SPARK-33740][SQL][3.0] hadoop configs in
 hive-site.xml can overrides pre-existing hadoop ones

Backport  #30709 to 3.0

### What changes were proposed in this pull request?

 org.apache.hadoop.conf.Configuration#setIfUnset will ignore those with defaults too

### Why are the changes needed?

    fix a regression

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

new tests

Closes #30720 from yaooqinn/SPARK-33740-30.

Authored-by: Kent Yao <yaooqinn@hotmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../org/apache/spark/sql/internal/SharedState.scala  | 12 +++++++++---
 sql/core/src/test/resources/hive-site.xml            |  5 +++++
 .../apache/spark/sql/internal/SharedStateSuite.scala | 11 +++++++++++
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
index ce4385d88f1e9..1922a58ce16af 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SharedState.scala
@@ -228,14 +228,20 @@ object SharedState extends Logging {
   def loadHiveConfFile(
       sparkConf: SparkConf,
       hadoopConf: Configuration): Unit = {
+    def containsInSparkConf(key: String): Boolean = {
+      sparkConf.contains(key) || sparkConf.contains("spark.hadoop." + key) ||
+        (key.startsWith("hive") && sparkConf.contains("spark." + key))
+    }
+
     val hiveWarehouseKey = "hive.metastore.warehouse.dir"
-    val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+    val configFile = Utils.getContextOrSparkClassLoader.getResourceAsStream("hive-site.xml")
     if (configFile != null) {
       logInfo(s"loading hive config file: $configFile")
       val hadoopConfTemp = new Configuration()
+      hadoopConfTemp.clear()
       hadoopConfTemp.addResource(configFile)
-      hadoopConfTemp.asScala.foreach { entry =>
-        hadoopConf.setIfUnset(entry.getKey, entry.getValue)
+      for (entry <- hadoopConfTemp.asScala if !containsInSparkConf(entry.getKey)) {
+        hadoopConf.set(entry.getKey, entry.getValue)
       }
     }
     // hive.metastore.warehouse.dir only stay in hadoopConf
diff --git a/sql/core/src/test/resources/hive-site.xml b/sql/core/src/test/resources/hive-site.xml
index 17297b3e22a7e..4bf6189b73ca9 100644
--- a/sql/core/src/test/resources/hive-site.xml
+++ b/sql/core/src/test/resources/hive-site.xml
@@ -23,4 +23,9 @@
       <value>true</value>
       <description>Internal marker for test.</description>
   </property>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/tmp/hive_one</value>
+    <description>default is /tmp/hadoop-${user.name} and will be overridden </description>
+  </property>
 </configuration>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
index 81bf15342423c..4d33fc1855ab2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SharedStateSuite.scala
@@ -52,4 +52,15 @@ class SharedStateSuite extends SharedSparkSession {
     assert(conf.isInstanceOf[Configuration])
     assert(conf.asInstanceOf[Configuration].get("fs.defaultFS") == "file:///")
   }
+
+  test("SPARK-33740: hadoop configs in hive-site.xml can overrides pre-existing hadoop ones") {
+    val conf = new SparkConf()
+    val hadoopConf = new Configuration()
+    SharedState.loadHiveConfFile(conf, hadoopConf)
+    assert(hadoopConf.get("hadoop.tmp.dir") === "/tmp/hive_one")
+    hadoopConf.clear()
+    SharedState.loadHiveConfFile(
+      conf.set("spark.hadoop.hadoop.tmp.dir", "noop"), hadoopConf)
+    assert(hadoopConf.get("hadoop.tmp.dir") === null)
+  }
 }

From 253416536edbc1ddce6e6e632c50f3abefb208e5 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sat, 12 Dec 2020 00:54:40 +0900
Subject: [PATCH 1210/1280] [SPARK-33757][INFRA][R] Fix the R dependencies
 build error on GitHub Actions and AppVeyor

### What changes were proposed in this pull request?

This PR fixes the R dependencies build error on GitHub Actions and AppVeyor.
The reason seems that `usethis` package is updated 2020/12/10.
https://cran.r-project.org/web/packages/usethis/index.html

### Why are the changes needed?

To keep the build clean.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Should be done by GitHub Actions.

Closes #30737 from sarutak/fix-r-dependencies-build-error.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit fb2e3af4b5d92398d57e61b766466cc7efd9d7cb)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/build_and_test.yml | 3 +++
 appveyor.yml                         | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index f7ffbca0b402c..6020fe491f043 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -322,6 +322,9 @@ jobs:
     - name: Install R linter dependencies and SparkR
       run: |
         sudo apt-get install -y libcurl4-openssl-dev
+        # dependencies for usethis 1.6.3.
+        sudo Rscript -e "install.packages(c('clipr', 'cli', 'crayon', 'desc', 'fs', 'gh', 'glue', 'purrr', 'rematch2', 'rlang', 'rprojroot', 'whisker', 'withr', 'yaml', 'git2r', 'rstudioapi'), repos='https://cloud.r-project.org/')"
+        sudo Rscript -e "install.packages('https://cran.r-project.org/src/contrib/Archive/usethis/usethis_1.6.3.tar.gz', repos=NULL, type='source')"
         sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
         sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
         ./R/install-dev.sh
diff --git a/appveyor.yml b/appveyor.yml
index e0d2396bcbed4..1941b7722791f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -41,6 +41,9 @@ cache:
 install:
   # Install maven and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
+  # usethis and its dependencies
+  - cmd: Rscript -e "install.packages(c('clipr', 'cli', 'crayon', 'desc', 'fs', 'gh', 'glue', 'purrr', 'rematch2', 'rlang', 'rprojroot', 'whisker', 'withr', 'yaml', 'git2r', 'rstudioapi'), repos='https://cloud.r-project.org/')"
+  - cmd: Rscript -e "install.packages('https://cran.r-project.org/src/contrib/Archive/usethis/usethis_1.6.3.tar.gz', repos=NULL, type='source')"
   # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat.
   - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')"
   - cmd: R -e "packageVersion('knitr'); packageVersion('rmarkdown'); packageVersion('testthat'); packageVersion('e1071'); packageVersion('survival'); packageVersion('arrow')"

From fe38821cad20a3b41825ee415083247d94d28c32 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 11 Dec 2020 09:58:54 -0800
Subject: [PATCH 1211/1280] [SPARK-33742][SQL][3.0] Throw
 PartitionsAlreadyExistException from HiveExternalCatalog.createPartitions()

### What changes were proposed in this pull request?
Throw `PartitionsAlreadyExistException` from `createPartitions()` in Hive external catalog when a partition exists. Currently, `HiveExternalCatalog.createPartitions()` throws `AlreadyExistsException` wrapped by `AnalysisException`.

In the PR, I propose to catch `AlreadyExistsException` in `HiveClientImpl` and replace it by `PartitionsAlreadyExistException`.

### Why are the changes needed?
The behaviour of Hive external catalog deviates from V1/V2 in-memory catalogs that throw `PartitionsAlreadyExistException`. To improve user experience with Spark SQL, it would be better to throw the same exception.

### Does this PR introduce _any_ user-facing change?
Yes

### How was this patch tested?
By running existing test suites:
```
$ build/sbt -Phive -Phive-thriftserver "hive/test:testOnly org.apache.spark.sql.hive.client.VersionsSuite"
$ build/sbt -Phive -Phive-thriftserver "hive/test:testOnly org.apache.spark.sql.hive.execution.HiveDDLSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Dongjoon Hyun <dongjoonapache.org>
(cherry picked from commit fab2995972761503563fa2aa547c67047c51bd33)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #30730 from MaxGekk/hive-partition-exception-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../sql/hive/client/HiveClientImpl.scala      | 15 ++++++++++--
 .../spark/sql/hive/client/VersionsSuite.scala | 23 ++++++++++++++++++-
 .../sql/hive/execution/HiveDDLSuite.scala     | 19 ++++++++++++++-
 3 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index d131a6bef06e7..93ef5d447b07a 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive.client
 
 import java.io.{File, PrintStream}
 import java.lang.{Iterable => JIterable}
+import java.lang.reflect.InvocationTargetException
 import java.nio.charset.StandardCharsets.UTF_8
 import java.util.{Locale, Map => JMap}
 import java.util.concurrent.TimeUnit._
@@ -48,7 +49,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.Expression
@@ -605,7 +606,17 @@ private[hive] class HiveClientImpl(
       table: String,
       parts: Seq[CatalogTablePartition],
       ignoreIfExists: Boolean): Unit = withHiveState {
-    shim.createPartitions(client, db, table, parts, ignoreIfExists)
+    def replaceExistException(e: Throwable): Unit = e match {
+      case _: HiveException if e.getCause.isInstanceOf[AlreadyExistsException] =>
+        throw new PartitionsAlreadyExistException(db, table, parts.map(_.spec))
+      case _ => throw e
+    }
+    try {
+      shim.createPartitions(client, db, table, parts, ignoreIfExists)
+    } catch {
+      case e: InvocationTargetException => replaceExistException(e.getCause)
+      case e: Throwable => replaceExistException(e)
+    }
   }
 
   override def dropPartitions(
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index 5ddaaf3c9a77c..41b90db3b2e7e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPermanentFunctionException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPermanentFunctionException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Literal}
 import org.apache.spark.sql.catalyst.util.quietly
@@ -593,6 +593,27 @@ class VersionsSuite extends SparkFunSuite with Logging {
       assert(client.getPartitionOption("default", "src_part", spec).isEmpty)
     }
 
+    test(s"$version: createPartitions if already exists") {
+      val partitions = Seq(CatalogTablePartition(
+        Map("key1" -> "101", "key2" -> "102"),
+        storageFormat))
+      try {
+        client.createPartitions("default", "src_part", partitions, ignoreIfExists = false)
+        val errMsg = intercept[PartitionsAlreadyExistException] {
+          client.createPartitions("default", "src_part", partitions, ignoreIfExists = false)
+        }.getMessage
+        assert(errMsg.contains("partitions already exists"))
+      } finally {
+        client.dropPartitions(
+          "default",
+          "src_part",
+          partitions.map(_.spec),
+          ignoreIfNotExists = true,
+          purge = false,
+          retainData = false)
+      }
+    }
+
     ///////////////////////////////////////////////////////////////////////////
     // Function related API
     ///////////////////////////////////////////////////////////////////////////
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 86385f0f56b5c..718b6f276c62e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterEach
 import org.apache.spark.SparkException
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionsAlreadyExistException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.connector.catalog.CatalogManager
@@ -2733,4 +2733,21 @@ class HiveDDLSuite
       assert(sql("SELECT * FROM t2 WHERE c = 'A'").collect().isEmpty)
     }
   }
+
+  test("SPARK-33742: partition already exists") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (data string) PARTITIONED BY (id bigint)")
+      sql(s"ALTER TABLE t ADD PARTITION (id=2) LOCATION 'loc1'")
+
+      val errMsg = intercept[PartitionsAlreadyExistException] {
+        sql(s"ALTER TABLE t ADD PARTITION (id=1) LOCATION 'loc'" +
+          " PARTITION (id=2) LOCATION 'loc1'")
+      }.getMessage
+      assert(errMsg.contains("The following partitions already exists"))
+
+      sql(s"ALTER TABLE t ADD IF NOT EXISTS PARTITION (id=1) LOCATION 'loc'" +
+        " PARTITION (id=2) LOCATION 'loc1'")
+      checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1"), Row("id=2")))
+    }
+  }
 }

From 14e77abbf869753151cc9b3cf54ad0f58fc5e482 Mon Sep 17 00:00:00 2001
From: linzebing <linzebing1995@gmail.com>
Date: Sun, 13 Dec 2020 22:00:05 +0900
Subject: [PATCH 1212/1280] [MINOR][UI] Correct JobPage's skipped/pending
 tableHeaderId

### What changes were proposed in this pull request?

Current Spark Web UI job page's header link of pending/skipped stages is inconsistent with their statuses. See the picture below:
![image](https://user-images.githubusercontent.com/9404831/101998894-1e843180-3c8c-11eb-8d94-10df9edb68e7.png)

### Why are the changes needed?

The code determining the `pendingOrSkippedTableId` has the wrong logic. As explained in the code:
> If the job is completed, then any pending stages are displayed as "skipped" [code pointer](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala#L266)

This PR fixes the logic for `pendingOrSkippedTableId` which aligns with the stage statuses.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Verified that header link is consistent with stage status with the fix.

Closes #30749 from linzebing/ui_bug.

Authored-by: linzebing <linzebing1995@gmail.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
(cherry picked from commit 0277fddaef17b615354c735a2c89cdced5f1d8f6)
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index 54582df919a43..c80b6d13970a9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -280,9 +280,9 @@ private[ui] class JobPage(parent: JobsTab, store: AppStatusStore) extends WebUIP
 
     val pendingOrSkippedTableId =
       if (isComplete) {
-        "pending"
-      } else {
         "skipped"
+      } else {
+        "pending"
       }
 
     val activeStagesTable =

From 7cd1aab11953b676ac8d4f6084f4a77396bca89d Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Sun, 13 Dec 2020 17:27:39 -0800
Subject: [PATCH 1213/1280] [SPARK-33757][INFRA][R][FOLLOWUP] Provide more
 simple solution

### What changes were proposed in this pull request?

This PR proposes a better solution for the R build failure on GitHub Actions.
The issue is solved in #30737 but I noticed the following two things.

* We can use the latest `usethis` if we install additional libraries on the GitHub Actions environment.
* For tests on AppVeyor, `usethis` is not necessary, so I partially revert the previous change.

### Why are the changes needed?

For more simple solution.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Confirmed on GitHub Actions and AppVeyor on my account.

Closes #30753 from sarutak/followup-SPARK-33757.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit b135db3b1a5c0b2170e98b97f6160bcf55903799)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .github/workflows/build_and_test.yml | 5 +----
 appveyor.yml                         | 3 ---
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 6020fe491f043..d80b132882a1a 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -321,10 +321,7 @@ jobs:
         r-version: 4.0
     - name: Install R linter dependencies and SparkR
       run: |
-        sudo apt-get install -y libcurl4-openssl-dev
-        # dependencies for usethis 1.6.3.
-        sudo Rscript -e "install.packages(c('clipr', 'cli', 'crayon', 'desc', 'fs', 'gh', 'glue', 'purrr', 'rematch2', 'rlang', 'rprojroot', 'whisker', 'withr', 'yaml', 'git2r', 'rstudioapi'), repos='https://cloud.r-project.org/')"
-        sudo Rscript -e "install.packages('https://cran.r-project.org/src/contrib/Archive/usethis/usethis_1.6.3.tar.gz', repos=NULL, type='source')"
+        sudo apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev
         sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
         sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
         ./R/install-dev.sh
diff --git a/appveyor.yml b/appveyor.yml
index 1941b7722791f..e0d2396bcbed4 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -41,9 +41,6 @@ cache:
 install:
   # Install maven and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
-  # usethis and its dependencies
-  - cmd: Rscript -e "install.packages(c('clipr', 'cli', 'crayon', 'desc', 'fs', 'gh', 'glue', 'purrr', 'rematch2', 'rlang', 'rprojroot', 'whisker', 'withr', 'yaml', 'git2r', 'rstudioapi'), repos='https://cloud.r-project.org/')"
-  - cmd: Rscript -e "install.packages('https://cran.r-project.org/src/contrib/Archive/usethis/usethis_1.6.3.tar.gz', repos=NULL, type='source')"
   # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat.
   - cmd: R -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')"
   - cmd: R -e "packageVersion('knitr'); packageVersion('rmarkdown'); packageVersion('testthat'); packageVersion('e1071'); packageVersion('survival'); packageVersion('arrow')"

From d652b47ce2693eedf3d465e50a252f6b80fe8ba7 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 14 Dec 2020 18:13:42 +0900
Subject: [PATCH 1214/1280] [SPARK-33770][SQL][TESTS][3.1][3.0] Fix the `ALTER
 TABLE .. DROP PARTITION` tests that delete files out of partition path

### What changes were proposed in this pull request?
Modify the tests that add partitions with `LOCATION`, and where the number of nested folders in `LOCATION` doesn't match to the number of partitioned columns. In that case, `ALTER TABLE .. DROP PARTITION` tries to access (delete) folder out of the "base" path in `LOCATION`.

The problem belongs to Hive's MetaStore method `drop_partition_common`:
https://github.com/apache/hive/blob/8696c82d07d303b6dbb69b4d443ab6f2b241b251/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java#L4876
which tries to delete empty partition sub-folders recursively starting from the most deeper partition sub-folder up to the base folder. In the case when the number of sub-folder is not equal to the number of partitioned columns `part_vals.size()`, the method will try to list and delete folders out of the base path.

### Why are the changes needed?
To fix test failures like https://github.com/apache/spark/pull/30643#issuecomment-743774733:
```
org.apache.spark.sql.hive.execution.command.AlterTableAddPartitionSuite.ALTER TABLE .. ADD PARTITION Hive V1: SPARK-33521: universal type conversions of partition values
sbt.ForkMain$ForkError: org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: File file:/home/jenkins/workspace/SparkPullRequestBuilder/target/tmp/spark-832cb19c-65fd-41f3-ae0b-937d76c07897 does not exist;
	at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:112)
	at org.apache.spark.sql.hive.HiveExternalCatalog.dropPartitions(HiveExternalCatalog.scala:1014)
...
Caused by: sbt.ForkMain$ForkError: org.apache.hadoop.hive.metastore.api.MetaException: File file:/home/jenkins/workspace/SparkPullRequestBuilder/target/tmp/spark-832cb19c-65fd-41f3-ae0b-937d76c07897 does not exist
	at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.drop_partition_with_environment_context(HiveMetaStore.java:3381)
	at sun.reflect.GeneratedMethodAccessor304.invoke(Unknown Source)
```

The issue can be reproduced by the following steps:
1. Create a base folder, for example: `/Users/maximgekk/tmp/part-location`
2. Create a sub-folder in the base folder and drop permissions for it:
```
$ mkdir /Users/maximgekk/tmp/part-location/aaa
$ chmod a-rwx chmod a-rwx /Users/maximgekk/tmp/part-location/aaa
$ ls -al /Users/maximgekk/tmp/part-location
total 0
drwxr-xr-x   3 maximgekk  staff    96 Dec 13 18:42 .
drwxr-xr-x  33 maximgekk  staff  1056 Dec 13 18:32 ..
d---------   2 maximgekk  staff    64 Dec 13 18:42 aaa
```
3. Create a table with a partition folder in the base folder:
```sql
spark-sql> create table tbl (id int) partitioned by (part0 int, part1 int);
spark-sql> alter table tbl add partition (part0=1,part1=2) location '/Users/maximgekk/tmp/part-location/tbl';
```
4. Try to drop this partition:
```
spark-sql> alter table tbl drop partition (part0=1,part1=2);
20/12/13 18:46:07 ERROR HiveClientImpl:
======================
Attempt to drop the partition specs in table 'tbl' database 'default':
Map(part0 -> 1, part1 -> 2)
In this attempt, the following partitions have been dropped successfully:

The remaining partitions have not been dropped:
[1, 2]
======================

Error in query: org.apache.hadoop.hive.ql.metadata.HiveException: Error accessing file:/Users/maximgekk/tmp/part-location/aaa;
org.apache.spark.sql.AnalysisException: org.apache.hadoop.hive.ql.metadata.HiveException: Error accessing file:/Users/maximgekk/tmp/part-location/aaa;
```
The command fails because it tries to access to the sub-folder `aaa` that is out of the partition path `/Users/maximgekk/tmp/part-location/tbl`.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
By running the affected tests from local IDEA which does not have access to folders out of partition paths.

Lead-authored-by: Max Gekk <max.gekkgmail.com>
Co-authored-by: Maxim Gekk <max.gekkgmail.com>
Signed-off-by: HyukjinKwon <gurwls223apache.org>
(cherry picked from commit 9160d59ae379910ca3bbd04ee25d336afff28abd)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #30756 from MaxGekk/fix-drop-partition-location-3.1.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/catalog/ExternalCatalogSuite.scala  |  9 +++++++--
 .../org/apache/spark/sql/hive/StatisticsSuite.scala  | 12 ++++++++----
 .../spark/sql/hive/execution/HiveDDLSuite.scala      |  4 ++--
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
index 55712d0da518d..d310538e302de 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala
@@ -408,8 +408,8 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac
       partitionColumnNames = Seq("partCol1", "partCol2"))
     catalog.createTable(table, ignoreIfExists = false)
 
-    val newLocationPart1 = newUriForDatabase()
-    val newLocationPart2 = newUriForDatabase()
+    val newLocationPart1 = newUriForPartition(Seq("p1=1", "p2=2"))
+    val newLocationPart2 = newUriForPartition(Seq("p1=3", "p2=4"))
 
     val partition1 =
       CatalogTablePartition(Map("partCol1" -> "1", "partCol2" -> "2"),
@@ -991,6 +991,11 @@ abstract class CatalogTestUtils {
 
   def newUriForDatabase(): URI = new URI(Utils.createTempDir().toURI.toString.stripSuffix("/"))
 
+  def newUriForPartition(parts: Seq[String]): URI = {
+    val path = parts.foldLeft(Utils.createTempDir())(new java.io.File(_, _))
+    new URI(path.toURI.toString.stripSuffix("/"))
+  }
+
   def newDb(name: String): CatalogDatabase = {
     CatalogDatabase(name, name + " description", newUriForDatabase(), Map.empty)
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index c1eab63ec073f..69c31a9c42ae5 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -993,12 +993,16 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
           assert(fetched1.get.colStats.size == 2)
 
           withTempPaths(numPaths = 2) { case Seq(dir1, dir2) =>
-            val file1 = new File(dir1 + "/data")
+            val partDir1 = new File(new File(dir1, "ds=2008-04-09"), "hr=11")
+            val file1 = new File(partDir1, "data")
+            file1.getParentFile.mkdirs()
             Utils.tryWithResource(new PrintWriter(file1)) { writer =>
               writer.write("1,a")
             }
 
-            val file2 = new File(dir2 + "/data")
+            val partDir2 = new File(new File(dir2, "ds=2008-04-09"), "hr=12")
+            val file2 = new File(partDir2, "data")
+            file2.getParentFile.mkdirs()
             Utils.tryWithResource(new PrintWriter(file2)) { writer =>
               writer.write("1,a")
             }
@@ -1007,8 +1011,8 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
             sql(
               s"""
                  |ALTER TABLE $table ADD
-                 |PARTITION (ds='2008-04-09', hr='11') LOCATION '${dir1.toURI.toString}'
-                 |PARTITION (ds='2008-04-09', hr='12') LOCATION '${dir2.toURI.toString}'
+                 |PARTITION (ds='2008-04-09', hr='11') LOCATION '${partDir1.toURI.toString}'
+                 |PARTITION (ds='2008-04-09', hr='12') LOCATION '${partDir1.toURI.toString}'
             """.stripMargin)
             if (autoUpdate) {
               val fetched2 = checkTableStats(table, hasSizeInBytes = true, expectedRowCounts = None)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 718b6f276c62e..4dd14887a0be4 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -605,8 +605,8 @@ class HiveDDLSuite
     val tab = "tab_with_partitions"
     withTempDir { tmpDir =>
       val basePath = new File(tmpDir.getCanonicalPath)
-      val part1Path = new File(basePath + "/part1")
-      val part2Path = new File(basePath + "/part2")
+      val part1Path = new File(new File(basePath, "part10"), "part11")
+      val part2Path = new File(new File(basePath, "part20"), "part21")
       val dirSet = part1Path :: part2Path :: Nil
 
       // Before data insertion, all the directory are empty

From f2c80796d8a2443005f2285c2638dfad2570ca44 Mon Sep 17 00:00:00 2001
From: Terry Kim <yuminkim@gmail.com>
Date: Wed, 16 Dec 2020 13:50:42 +0000
Subject: [PATCH 1215/1280] [SPARK-33786][SQL][3.0] The storage level for a
 cache should be respected when a table name is altered

### What changes were proposed in this pull request?

This is a back port of #30774.

This PR proposes to retain the cache's storage level when a table name is altered by `ALTER TABLE ... RENAME TO ...`.

### Why are the changes needed?

Currently, when a table name is altered, the table's cache is refreshed (if exists), but the storage level is not retained. For example:
```scala
        def getStorageLevel(tableName: String): StorageLevel = {
          val table = spark.table(tableName)
          val cachedData = spark.sharedState.cacheManager.lookupCachedData(table).get
          cachedData.cachedRepresentation.cacheBuilder.storageLevel
        }

        Seq(1 -> "a").toDF("i", "j").write.parquet(path.getCanonicalPath)
        sql(s"CREATE TABLE old USING parquet LOCATION '${path.toURI}'")
        sql("CACHE TABLE old OPTIONS('storageLevel' 'MEMORY_ONLY')")
        val oldStorageLevel = getStorageLevel("old")

        sql("ALTER TABLE old RENAME TO new")
        val newStorageLevel = getStorageLevel("new")
```
`oldStorageLevel` will be `StorageLevel(memory, deserialized, 1 replicas)` whereas `newStorageLevel` will be `StorageLevel(disk, memory, deserialized, 1 replicas)`, which is the default storage level.

### Does this PR introduce _any_ user-facing change?

Yes, now the storage level for the cache will be retained.

### How was this patch tested?

Added a unit test.

Closes #30793 from imback82/alter_table_rename_cache_fix_3.0.

Authored-by: Terry Kim <yuminkim@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/command/tables.scala  | 14 ++++++-------
 .../apache/spark/sql/CachedTableSuite.scala   | 20 +++++++++++++++++++
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index d550683485779..42f4a7245031c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -21,7 +21,6 @@ import java.net.{URI, URISyntaxException}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.ArrayBuffer
-import scala.util.Try
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.{FileContext, FsConstants, Path}
@@ -192,18 +191,19 @@ case class AlterTableRenameCommand(
     } else {
       val table = catalog.getTableMetadata(oldName)
       DDLUtils.verifyAlterTableType(catalog, table, isView)
-      // If an exception is thrown here we can just assume the table is uncached;
-      // this can happen with Hive tables when the underlying catalog is in-memory.
-      val wasCached = Try(sparkSession.catalog.isCached(oldName.unquotedString)).getOrElse(false)
-      if (wasCached) {
+      // If `optStorageLevel` is defined, the old table was cached.
+      val optCachedData = sparkSession.sharedState.cacheManager.lookupCachedData(
+        sparkSession.table(oldName.unquotedString))
+      val optStorageLevel = optCachedData.map(_.cachedRepresentation.cacheBuilder.storageLevel)
+      if (optStorageLevel.isDefined) {
         CommandUtils.uncacheTableOrView(sparkSession, oldName.unquotedString)
       }
       // Invalidate the table last, otherwise uncaching the table would load the logical plan
       // back into the hive metastore cache
       catalog.refreshTable(oldName)
       catalog.renameTable(oldName, newName)
-      if (wasCached) {
-        sparkSession.catalog.cacheTable(newName.unquotedString)
+      optStorageLevel.foreach { storageLevel =>
+        sparkSession.catalog.cacheTable(newName.unquotedString, storageLevel)
       }
     }
     Seq.empty[Row]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 6313370476c93..e14c72eee9646 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1250,4 +1250,24 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       }
     }
   }
+
+  test("SPARK-33786: Cache's storage level should be respected when a table name is altered.") {
+    withTable("old", "new") {
+      withTempPath { path =>
+        def getStorageLevel(tableName: String): StorageLevel = {
+          val table = spark.table(tableName)
+          val cachedData = spark.sharedState.cacheManager.lookupCachedData(table).get
+          cachedData.cachedRepresentation.cacheBuilder.storageLevel
+        }
+        Seq(1 -> "a").toDF("i", "j").write.parquet(path.getCanonicalPath)
+        sql(s"CREATE TABLE old USING parquet LOCATION '${path.toURI}'")
+        sql("CACHE TABLE old OPTIONS('storageLevel' 'MEMORY_ONLY')")
+        val oldStorageLevel = getStorageLevel("old")
+
+        sql("ALTER TABLE old RENAME TO new")
+        val newStorageLevel = getStorageLevel("new")
+        assert(oldStorageLevel === newStorageLevel)
+      }
+    }
+  }
 }

From a77b70daf418ec75130d2ab7847aeb77aa602ecd Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 16 Dec 2020 23:30:47 +0900
Subject: [PATCH 1216/1280] [SPARK-33788][SQL][3.1][3.0][2.4] Throw
 NoSuchPartitionsException from HiveExternalCatalog.dropPartitions()

### What changes were proposed in this pull request?
Throw `NoSuchPartitionsException` from `ALTER TABLE .. DROP TABLE` for not existing partitions of a table in V1 Hive external catalog.

### Why are the changes needed?
The behaviour of Hive external catalog deviates from V1/V2 in-memory catalogs that throw `NoSuchPartitionsException`. To improve user experience with Spark SQL, it would be better to throw the same exception.

### Does this PR introduce _any_ user-facing change?
Yes, the command throws `NoSuchPartitionsException` instead of the general exception `AnalysisException`.

### How was this patch tested?
By running new UT via:
```
$ build/sbt -Phive -Phive-thriftserver "test:testOnly *HiveDDLSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: HyukjinKwon <gurwls223apache.org>
(cherry picked from commit 3dfdcf4f92ef5e739f15c22c93d673bb2233e617)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #30802 from MaxGekk/hive-drop-partition-exception-3.1.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/hive/client/HiveClientImpl.scala |  6 ++----
 .../sql/hive/execution/HiveDDLSuite.scala      | 18 +++++++++++++++++-
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 93ef5d447b07a..f3118368b9829 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -49,7 +49,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.metrics.source.HiveCatalogMetrics
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, PartitionsAlreadyExistException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchPartitionException, NoSuchPartitionsException, PartitionsAlreadyExistException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
 import org.apache.spark.sql.catalyst.expressions.Expression
@@ -637,9 +637,7 @@ private[hive] class HiveClientImpl(
         // (b='1', c='1') and (b='1', c='2'), a partial spec of (b='1') will match both.
         val parts = client.getPartitions(hiveTable, s.asJava).asScala
         if (parts.isEmpty && !ignoreIfNotExists) {
-          throw new AnalysisException(
-            s"No partition is dropped. One partition spec '$s' does not exist in table '$table' " +
-            s"database '$db'")
+          throw new NoSuchPartitionsException(db, table, Seq(s))
         }
         parts.map(_.getValues)
       }.distinct
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 4dd14887a0be4..765119dec04b3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterEach
 import org.apache.spark.SparkException
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, PartitionsAlreadyExistException, TableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionException, NoSuchPartitionsException, PartitionsAlreadyExistException, TableAlreadyExistsException}
 import org.apache.spark.sql.catalyst.catalog._
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.connector.catalog.CatalogManager
@@ -2750,4 +2750,20 @@ class HiveDDLSuite
       checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1"), Row("id=2")))
     }
   }
+
+  test("SPARK-33788: partition not exists") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (data string) PARTITIONED BY (id bigint)")
+      sql(s"ALTER TABLE t ADD PARTITION (id=1)")
+
+      val errMsg = intercept[NoSuchPartitionsException] {
+        sql(s"ALTER TABLE t DROP PARTITION (id=1), PARTITION (id=2)")
+      }.getMessage
+      assert(errMsg.contains("partitions not found in table"))
+
+      checkAnswer(sql("SHOW PARTITIONS t"), Seq(Row("id=1")))
+      sql(s"ALTER TABLE t DROP IF EXISTS PARTITION (id=1), PARTITION (id=2)")
+      checkAnswer(sql("SHOW PARTITIONS t"), Seq.empty)
+    }
+  }
 }

From da272f7ffb370b10d151da124586c52ca343e056 Mon Sep 17 00:00:00 2001
From: Sander Goos <sander.goos@databricks.com>
Date: Wed, 16 Dec 2020 23:33:09 +0900
Subject: [PATCH 1217/1280] [SPARK-33793][TESTS][3.0] Introduce withExecutor to
 ensure proper cleanup in tests

Backport of: https://github.com/apache/spark/pull/30783

### What changes were proposed in this pull request?
This PR introduces a helper method `withExecutor` that handles the creation of an Executor object and ensures that it is always stopped in a finally block. The tests in ExecutorSuite have been refactored to use this method.

### Why are the changes needed?
Recently an issue was discovered that leaked Executors (which are not explicitly stopped after a test) can cause other tests to fail due to the JVM being killed after 10 min. It is therefore crucial that tests always stop the Executor. By introducing this helper method, a simple pattern is established that can be easily adopted in new tests, which reduces the risk of regressions.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Run the ExecutorSuite locally.

Closes #30801 from sander-goos/SPARK-33793-close-executors-3.0.

Authored-by: Sander Goos <sander.goos@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/executor/ExecutorSuite.scala | 99 ++++++++++---------
 1 file changed, 54 insertions(+), 45 deletions(-)

diff --git a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
index 8e58beff74290..6b3df6d0c9970 100644
--- a/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
+++ b/core/src/test/scala/org/apache/spark/executor/ExecutorSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.executor
 
 import java.io.{Externalizable, File, ObjectInput, ObjectOutput}
 import java.lang.Thread.UncaughtExceptionHandler
+import java.net.URL
 import java.nio.ByteBuffer
 import java.util.Properties
 import java.util.concurrent.{ConcurrentHashMap, CountDownLatch, TimeUnit}
@@ -53,7 +54,7 @@ import org.apache.spark.scheduler.{DirectTaskResult, FakeTask, ResultTask, Task,
 import org.apache.spark.serializer.{JavaSerializer, SerializerInstance, SerializerManager}
 import org.apache.spark.shuffle.FetchFailedException
 import org.apache.spark.storage.{BlockManager, BlockManagerId}
-import org.apache.spark.util.{LongAccumulator, UninterruptibleThread, Utils}
+import org.apache.spark.util.{LongAccumulator, SparkUncaughtExceptionHandler, UninterruptibleThread, Utils}
 
 class ExecutorSuite extends SparkFunSuite
     with LocalSparkContext with MockitoSugar with Eventually with PrivateMethodTester {
@@ -64,6 +65,33 @@ class ExecutorSuite extends SparkFunSuite
     super.afterEach()
   }
 
+  /**
+   * Creates an Executor with the provided arguments, is then passed to `f`
+   * and will be stopped after `f` returns.
+   */
+  def withExecutor(
+      executorId: String,
+      executorHostname: String,
+      env: SparkEnv,
+      userClassPath: Seq[URL] = Nil,
+      isLocal: Boolean = true,
+      uncaughtExceptionHandler: UncaughtExceptionHandler
+        = new SparkUncaughtExceptionHandler,
+      resources: immutable.Map[String, ResourceInformation]
+        = immutable.Map.empty[String, ResourceInformation])(f: Executor => Unit): Unit = {
+    var executor: Executor = null
+    try {
+      executor = new Executor(executorId, executorHostname, env, userClassPath, isLocal,
+        uncaughtExceptionHandler, resources)
+
+      f(executor)
+    } finally {
+      if (executor != null) {
+        executor.stop()
+      }
+    }
+  }
+
   test("SPARK-15963: Catch `TaskKilledException` correctly in Executor.TaskRunner") {
     // mock some objects to make Executor.launchTask() happy
     val conf = new SparkConf
@@ -116,10 +144,8 @@ class ExecutorSuite extends SparkFunSuite
         }
       })
 
-    var executor: Executor = null
-    try {
-      executor = new Executor("id", "localhost", env, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
+    withExecutor("id", "localhost", env) { executor =>
+
       // the task will be launched in a dedicated worker thread
       executor.launchTask(mockExecutorBackend, taskDescription)
 
@@ -139,11 +165,6 @@ class ExecutorSuite extends SparkFunSuite
       assert(executorSuiteHelper.testFailedReason.toErrorString === "TaskKilled (test)")
       assert(executorSuiteHelper.taskState === TaskState.KILLED)
     }
-    finally {
-      if (executor != null) {
-        executor.stop()
-      }
-    }
   }
 
   test("SPARK-19276: Handle FetchFailedExceptions that are hidden by user exceptions") {
@@ -255,25 +276,24 @@ class ExecutorSuite extends SparkFunSuite
     confs.foreach { case (k, v) => conf.set(k, v) }
     val serializer = new JavaSerializer(conf)
     val env = createMockEnv(conf, serializer)
-    val executor =
-      new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
-    val executorClass = classOf[Executor]
-
-    // Save all heartbeats sent into an ArrayBuffer for verification
-    val heartbeats = ArrayBuffer[Heartbeat]()
-    val mockReceiver = mock[RpcEndpointRef]
-    when(mockReceiver.askSync(any[Heartbeat], any[RpcTimeout])(any))
-      .thenAnswer((invocation: InvocationOnMock) => {
-        val args = invocation.getArguments()
-        heartbeats += args(0).asInstanceOf[Heartbeat]
-        HeartbeatResponse(false)
-      })
-    val receiverRef = executorClass.getDeclaredField("heartbeatReceiverRef")
-    receiverRef.setAccessible(true)
-    receiverRef.set(executor, mockReceiver)
+    withExecutor("id", "localhost", SparkEnv.get) { executor =>
+      val executorClass = classOf[Executor]
 
-    f(executor, heartbeats)
+      // Save all heartbeats sent into an ArrayBuffer for verification
+      val heartbeats = ArrayBuffer[Heartbeat]()
+      val mockReceiver = mock[RpcEndpointRef]
+      when(mockReceiver.askSync(any[Heartbeat], any[RpcTimeout])(any))
+        .thenAnswer((invocation: InvocationOnMock) => {
+          val args = invocation.getArguments()
+          heartbeats += args(0).asInstanceOf[Heartbeat]
+          HeartbeatResponse(false)
+        })
+      val receiverRef = executorClass.getDeclaredField("heartbeatReceiverRef")
+      receiverRef.setAccessible(true)
+      receiverRef.set(executor, mockReceiver)
+
+      f(executor, heartbeats)
+    }
   }
 
   private def heartbeatZeroAccumulatorUpdateTest(dropZeroMetrics: Boolean): Unit = {
@@ -354,10 +374,7 @@ class ExecutorSuite extends SparkFunSuite
     val taskDescription = createResultTaskDescription(serializer, taskBinary, rdd, 0)
 
     val mockBackend = mock[ExecutorBackend]
-    var executor: Executor = null
-    try {
-      executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        resources = immutable.Map.empty[String, ResourceInformation])
+    withExecutor("id", "localhost", SparkEnv.get) { executor =>
       executor.launchTask(mockBackend, taskDescription)
 
       // Ensure that the executor's metricsPoller is polled so that values are recorded for
@@ -368,10 +385,6 @@ class ExecutorSuite extends SparkFunSuite
       eventually(timeout(5.seconds), interval(10.milliseconds)) {
         assert(executor.numRunningTasks === 0)
       }
-    } finally {
-      if (executor != null) {
-        executor.stop()
-      }
     }
 
     // Verify that peak values for task metrics get sent in the TaskResult
@@ -466,12 +479,11 @@ class ExecutorSuite extends SparkFunSuite
       poll: Boolean = false): (TaskFailedReason, UncaughtExceptionHandler) = {
     val mockBackend = mock[ExecutorBackend]
     val mockUncaughtExceptionHandler = mock[UncaughtExceptionHandler]
-    var executor: Executor = null
     val timedOut = new AtomicBoolean(false)
-    try {
-      executor = new Executor("id", "localhost", SparkEnv.get, userClassPath = Nil, isLocal = true,
-        uncaughtExceptionHandler = mockUncaughtExceptionHandler,
-        resources = immutable.Map.empty[String, ResourceInformation])
+
+    withExecutor("id", "localhost", SparkEnv.get,
+        uncaughtExceptionHandler = mockUncaughtExceptionHandler) { executor =>
+
       // the task will be launched in a dedicated worker thread
       executor.launchTask(mockBackend, taskDescription)
       if (killTask) {
@@ -504,11 +516,8 @@ class ExecutorSuite extends SparkFunSuite
         assert(executor.numRunningTasks === 0)
       }
       assert(!timedOut.get(), "timed out waiting to be ready to kill tasks")
-    } finally {
-      if (executor != null) {
-        executor.stop()
-      }
     }
+
     val orderedMock = inOrder(mockBackend)
     val statusCaptor = ArgumentCaptor.forClass(classOf[ByteBuffer])
     orderedMock.verify(mockBackend)

From b86ea0f895599e621f7ed1aa4213378f9b5cf621 Mon Sep 17 00:00:00 2001
From: ulysses-you <ulyssesyou18@gmail.com>
Date: Thu, 17 Dec 2020 08:44:42 +0000
Subject: [PATCH 1218/1280] [SPARK-33733][SQL][3.0] PullOutNondeterministic
 should check and collect deterministic field

backport [#30703](https://github.com/apache/spark/pull/30703) for branch-3.0.

### What changes were proposed in this pull request?

The deterministic field is wider than `NonDerterministic`, we should keep same range between pull out and check analysis.

### Why are the changes needed?

For example
```
select * from values(1), (4) as t(c1) order by java_method('java.lang.Math', 'abs', c1)
```

We will get exception since `java_method` deterministic field is false but not a `NonDeterministic`
```
Exception in thread "main" org.apache.spark.sql.AnalysisException: nondeterministic expressions are only allowed in
Project, Filter, Aggregate or Window, found:
 java_method('java.lang.Math', 'abs', t.`c1`) ASC NULLS FIRST
in operator Sort [java_method(java.lang.Math, abs, c1#1) ASC NULLS FIRST], true
               ;;
```

### Does this PR introduce _any_ user-facing change?

Yes.

### How was this patch tested?

Add test.

Closes #30771 from ulysses-you/SPARK-33733-branch-3.0.

Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  5 ++++-
 .../expressions/CallMethodViaReflection.scala |  6 ++---
 .../sql/catalyst/analysis/AnalysisSuite.scala | 22 +++++++++++++++++++
 3 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 7795d709b3440..1307fc5acd5f4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -2711,7 +2711,10 @@ class Analyzer(
 
     private def getNondeterToAttr(exprs: Seq[Expression]): Map[Expression, NamedExpression] = {
       exprs.filterNot(_.deterministic).flatMap { expr =>
-        val leafNondeterministic = expr.collect { case n: Nondeterministic => n }
+        val leafNondeterministic = expr.collect {
+          case n: Nondeterministic => n
+          case udf: UserDefinedExpression if !udf.deterministic => udf
+        }
         leafNondeterministic.distinct.map { e =>
           val ne = e match {
             case n: NamedExpression => n
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
index e6a4c8f1d3749..6d711af192ad3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala
@@ -53,7 +53,7 @@ import org.apache.spark.util.Utils
        a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
   """)
 case class CallMethodViaReflection(children: Seq[Expression])
-  extends Expression with CodegenFallback {
+  extends Nondeterministic with CodegenFallback {
 
   override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("reflect")
 
@@ -76,11 +76,11 @@ case class CallMethodViaReflection(children: Seq[Expression])
     }
   }
 
-  override lazy val deterministic: Boolean = false
   override def nullable: Boolean = true
   override val dataType: DataType = StringType
+  override protected def initializeInternal(partitionIndex: Int): Unit = {}
 
-  override def eval(input: InternalRow): Any = {
+  override protected def evalInternal(input: InternalRow): Any = {
     var i = 0
     while (i < argExprs.length) {
       buffer(i) = argExprs(i).eval(input).asInstanceOf[Object]
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index b4327471354e8..63c277918a429 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -951,4 +951,26 @@ class AnalysisSuite extends AnalysisTest with Matchers {
         s"please set '${SQLConf.ANALYZER_MAX_ITERATIONS.key}' to a larger value."))
     }
   }
+
+  test("SPARK-33733: PullOutNondeterministic should check and collect deterministic field") {
+    val reflect =
+      CallMethodViaReflection(Seq("java.lang.Math", "abs", testRelation.output.head))
+    val udf = ScalaUDF(
+      (s: String) => s,
+      StringType,
+      Literal.create(null, StringType) :: Nil,
+      Option(ExpressionEncoder[String]().resolveAndBind()) :: Nil,
+      udfDeterministic = false)
+
+    Seq(reflect, udf).foreach { e: Expression =>
+      val plan = Sort(Seq(e.asc), false, testRelation)
+      val projected = Alias(e, "_nondeterministic")()
+      val expect =
+        Project(testRelation.output,
+          Sort(Seq(projected.toAttribute.asc), false,
+            Project(testRelation.output :+ projected,
+              testRelation)))
+      checkAnalysis(plan, expect)
+    }
+  }
 }

From cd683b330796d3d9eb91278d6cd23f907da2913c Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 17 Dec 2020 19:08:04 +0900
Subject: [PATCH 1219/1280] [SPARK-33819][CORE][3.0]
 SingleFileEventLogFileReader/RollingEventLogFilesFileReader should be
 `package private`

### What changes were proposed in this pull request?

This PR aims to convert `EventLogFileReader`'s derived classes into `package private`.
- SingleFileEventLogFileReader
- RollingEventLogFilesFileReader

`EventLogFileReader` itself is used in `scheduler` module during tests.

### Why are the changes needed?

This classes were designed to be internal. This PR hides it explicitly to reduce the maintenance burden.

### Does this PR introduce _any_ user-facing change?

Yes, but these were exposed accidentally.

### How was this patch tested?

Pass CIs.

Closes #30820 from dongjoon-hyun/SPARK-33819-3.0.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/deploy/history/EventLogFileReaders.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 9f63a6441a838..30ff2c3315750 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -164,7 +164,7 @@ object EventLogFileReader {
  * FileNotFoundException could occur if the log file is renamed before getting the
  * status of log file.
  */
-class SingleFileEventLogFileReader(
+private[history] class SingleFileEventLogFileReader(
     fs: FileSystem,
     path: Path) extends EventLogFileReader(fs, path) {
   private lazy val status = fileSystem.getFileStatus(rootPath)
@@ -203,7 +203,7 @@ class SingleFileEventLogFileReader(
  * This reader lists the files only once; if caller would like to play with updated list,
  * it needs to create another reader instance.
  */
-class RollingEventLogFilesFileReader(
+private[history] class RollingEventLogFilesFileReader(
     fs: FileSystem,
     path: Path) extends EventLogFileReader(fs, path) {
   import RollingEventLogFilesWriter._

From 99eb02745e85a11dbbb1fa5de1167f07cd499797 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Thu, 17 Dec 2020 08:52:09 -0600
Subject: [PATCH 1220/1280] [SPARK-33774][UI][CORE] Back to Master" returns 500
 error in Standalone cluster

### What changes were proposed in this pull request?

Initiate the `masterWebUiUrl` with the `webUi. webUrl` instead of the `masterPublicAddress`.

### Why are the changes needed?

Since [SPARK-21642](https://issues.apache.org/jira/browse/SPARK-21642), `WebUI` has changed from `localHostName` to `localCanonicalHostName` as the hostname to set up the web UI. However, the `masterPublicAddress` is from `RpcEnv`'s host address, which still uses `localHostName`. As a result, it returns the wrong Master web URL to the Worker.

### Does this PR introduce _any_ user-facing change?

Yes, when users click "Back to Master" in the Worker page:

Before this PR:

<img width="3258" alt="WeChat4acbfd163f51c76a5f9bc388c7479785" src="https://user-images.githubusercontent.com/16397174/102057951-b9664280-3e29-11eb-8749-5ee293902bdf.png">

After this PR:

![image](https://user-images.githubusercontent.com/16397174/102058016-d438b700-3e29-11eb-8641-a23a6b2f542e.png)

(Return to the Master page successfully.)

### How was this patch tested?

Tested manually.

Closes #30759 from Ngone51/fix-back-to-master.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 34e4d87023535c086a0aa43fe194f794b41e09b7)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 core/src/main/scala/org/apache/spark/deploy/master/Master.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 3818a872b9f88..ab6cb304b59eb 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -143,7 +143,7 @@ private[deploy] class Master(
     logInfo(s"Running Spark version ${org.apache.spark.SPARK_VERSION}")
     webUi = new MasterWebUI(this, webUiPort)
     webUi.bind()
-    masterWebUiUrl = s"${webUi.scheme}$masterPublicAddress:${webUi.boundPort}"
+    masterWebUiUrl = webUi.webUrl
     if (reverseProxy) {
       masterWebUiUrl = conf.get(UI_REVERSE_PROXY_URL).orElse(Some(masterWebUiUrl)).get
       webUi.addProxy()

From 3ef6827b25e289924ba03d3521ccce7f2adb4d92 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 17 Dec 2020 16:16:05 -0800
Subject: [PATCH 1221/1280] [SPARK-33822][SQL] Use the `CastSupport.cast`
 method in HashJoin

### What changes were proposed in this pull request?

This PR intends to fix the bug that throws a unsupported exception when running [the TPCDS q5](https://github.com/apache/spark/blob/master/sql/core/src/test/resources/tpcds/q5.sql) with AQE enabled ([this option is enabled by default now via SPARK-33679](https://github.com/apache/spark/commit/031c5ef280e0cba8c4718a6457a44b6cccb17f46)):
```
java.lang.UnsupportedOperationException: BroadcastExchange does not support the execute() code path.
  at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecute(BroadcastExchangeExec.scala:189)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
  at org.apache.spark.sql.execution.exchange.ReusedExchangeExec.doExecute(Exchange.scala:60)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
  at org.apache.spark.sql.execution.adaptive.QueryStageExec.doExecute(QueryStageExec.scala:115)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
  at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:321)
  at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:397)
  at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:118)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)
  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
  ...
```

I've checked the AQE code and I found `EnsureRequirements` wrongly puts `BroadcastExchange` on a top of `BroadcastQueryStage` in the `reOptimize` phase as follows:
```
+- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#2183]
  +- BroadcastQueryStage 2
    +- ReusedExchange [d_date_sk#1086], BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#1963]
```
A root cause is that a `Cast` class in a required child's distribution does not have a `timeZoneId` field (`timeZoneId=None`), and a `Cast` class in `child.outputPartitioning` has it. So, this difference can make the distribution requirement check fail in `EnsureRequirements`:
https://github.com/apache/spark/blob/1e85707738a830d33598ca267a6740b3f06b1861/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala#L47-L50

The `Cast` class that does not have a `timeZoneId` field is generated in the `HashJoin` object. To fix this issue, this PR proposes to use the `CastSupport.cast` method there.

### Why are the changes needed?

Bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually checked that q5 passed.

Closes #30818 from maropu/BugfixInAQE.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 51ef4430dcbc934d43315ee6bdc851c9be84a1f2)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/joins/HashJoin.scala  | 13 ++++----
 .../execution/joins/BroadcastJoinSuite.scala  | 33 +++++++++++--------
 2 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 931a75a83d4ef..0447d7fd13d0a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -17,7 +17,8 @@
 
 package org.apache.spark.sql.execution.joins
 
-import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
+import org.apache.spark.sql.catalyst.analysis.CastSupport
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.plans._
@@ -240,7 +241,7 @@ trait HashJoin {
   }
 }
 
-object HashJoin {
+object HashJoin extends CastSupport with SQLConfHelper {
   /**
    * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long.
    *
@@ -255,14 +256,14 @@ object HashJoin {
     }
 
     var keyExpr: Expression = if (keys.head.dataType != LongType) {
-      Cast(keys.head, LongType)
+      cast(keys.head, LongType)
     } else {
       keys.head
     }
     keys.tail.foreach { e =>
       val bits = e.dataType.defaultSize * 8
       keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)),
-        BitwiseAnd(Cast(e, LongType), Literal((1L << bits) - 1)))
+        BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1)))
     }
     keyExpr :: Nil
   }
@@ -275,13 +276,13 @@ object HashJoin {
     // jump over keys that have a higher index value than the required key
     if (keys.size == 1) {
       assert(index == 0)
-      Cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
+      cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
     } else {
       val shiftedBits =
         keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum
       val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1
       // build the schema for unpacking the required key
-      Cast(BitwiseAnd(
+      cast(BitwiseAnd(
         ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)),
         Literal(mask)), keys(index).dataType)
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 335ef257920c4..ef0a596f21104 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -239,33 +239,40 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
     assert(HashJoin.rewriteKeyExpr(l :: l :: Nil) === l :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(l :: i :: Nil) === l :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(i :: Nil) === Cast(i, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(i :: Nil) ===
+      Cast(i, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: l :: Nil) === i :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(i, LongType), Literal(32)),
-        BitwiseAnd(Cast(i, LongType), Literal((1L << 32) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal(32)),
+        BitwiseAnd(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 32) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: i :: Nil) === i :: i :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(s :: Nil) === Cast(s, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(s :: Nil) ===
+      Cast(s, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: l :: Nil) === s :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
-        BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+        BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
         BitwiseOr(ShiftLeft(
-          BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-            BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+            BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)),
+              Literal((1L << 16) - 1))),
           Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: s :: Nil) ===
       s :: s :: s :: s :: s :: Nil)
 

From 5ce0b9fc607a26c6f3b2fa0eec7a5c1f9010ff1f Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Thu, 17 Dec 2020 16:37:40 -0800
Subject: [PATCH 1222/1280] Revert "[SPARK-33822][SQL] Use the
 `CastSupport.cast` method in HashJoin"

This reverts commit 3ef6827b25e289924ba03d3521ccce7f2adb4d92.
---
 .../spark/sql/execution/joins/HashJoin.scala  | 13 ++++----
 .../execution/joins/BroadcastJoinSuite.scala  | 33 ++++++++-----------
 2 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 0447d7fd13d0a..931a75a83d4ef 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -17,8 +17,7 @@
 
 package org.apache.spark.sql.execution.joins
 
-import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
-import org.apache.spark.sql.catalyst.analysis.CastSupport
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.plans._
@@ -241,7 +240,7 @@ trait HashJoin {
   }
 }
 
-object HashJoin extends CastSupport with SQLConfHelper {
+object HashJoin {
   /**
    * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long.
    *
@@ -256,14 +255,14 @@ object HashJoin extends CastSupport with SQLConfHelper {
     }
 
     var keyExpr: Expression = if (keys.head.dataType != LongType) {
-      cast(keys.head, LongType)
+      Cast(keys.head, LongType)
     } else {
       keys.head
     }
     keys.tail.foreach { e =>
       val bits = e.dataType.defaultSize * 8
       keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)),
-        BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1)))
+        BitwiseAnd(Cast(e, LongType), Literal((1L << bits) - 1)))
     }
     keyExpr :: Nil
   }
@@ -276,13 +275,13 @@ object HashJoin extends CastSupport with SQLConfHelper {
     // jump over keys that have a higher index value than the required key
     if (keys.size == 1) {
       assert(index == 0)
-      cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
+      Cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
     } else {
       val shiftedBits =
         keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum
       val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1
       // build the schema for unpacking the required key
-      cast(BitwiseAnd(
+      Cast(BitwiseAnd(
         ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)),
         Literal(mask)), keys(index).dataType)
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index ef0a596f21104..335ef257920c4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -239,40 +239,33 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
     assert(HashJoin.rewriteKeyExpr(l :: l :: Nil) === l :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(l :: i :: Nil) === l :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(i :: Nil) ===
-      Cast(i, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(i :: Nil) === Cast(i, LongType) :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: l :: Nil) === i :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal(32)),
-        BitwiseAnd(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 32) - 1))) ::
-        Nil)
+      BitwiseOr(ShiftLeft(Cast(i, LongType), Literal(32)),
+        BitwiseAnd(Cast(i, LongType), Literal((1L << 32) - 1))) :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: i :: Nil) === i :: i :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(s :: Nil) ===
-      Cast(s, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(s :: Nil) === Cast(s, LongType) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: l :: Nil) === s :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
-        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
-        Nil)
+      BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
+        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
-        BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
-          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
+        BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
+          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
-        Nil)
+        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
         BitwiseOr(ShiftLeft(
-          BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
-            BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)),
-              Literal((1L << 16) - 1))),
+          BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
+            BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
           Literal(16)),
-          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
+          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
-        Nil)
+        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: s :: Nil) ===
       s :: s :: s :: s :: s :: Nil)
 

From 1615b0ec28a316b04c31e2ae4e54495c105f7cf2 Mon Sep 17 00:00:00 2001
From: Takeshi Yamamuro <yamamuro@apache.org>
Date: Thu, 17 Dec 2020 18:43:21 -0800
Subject: [PATCH 1223/1280] [SPARK-33822][SQL][3.0] Use the `CastSupport.cast`
 method in HashJoin

### What changes were proposed in this pull request?

This PR intends to fix the bug that throws a unsupported exception when running [the TPCDS q5](https://github.com/apache/spark/blob/master/sql/core/src/test/resources/tpcds/q5.sql) with AQE enabled ([this option is enabled by default now via SPARK-33679](https://github.com/apache/spark/commit/031c5ef280e0cba8c4718a6457a44b6cccb17f46)):
```
java.lang.UnsupportedOperationException: BroadcastExchange does not support the execute() code path.
  at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecute(BroadcastExchangeExec.scala:189)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
  at org.apache.spark.sql.execution.exchange.ReusedExchangeExec.doExecute(Exchange.scala:60)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
  at org.apache.spark.sql.execution.adaptive.QueryStageExec.doExecute(QueryStageExec.scala:115)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:180)
  at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:218)
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:215)
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:176)
  at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:321)
  at org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(SparkPlan.scala:397)
  at org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(BroadcastExchangeExec.scala:118)
  at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:185)
  at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
  ...
```

I've checked the AQE code and I found `EnsureRequirements` wrongly puts `BroadcastExchange` on a top of `BroadcastQueryStage` in the `reOptimize` phase as follows:
```
+- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#2183]
  +- BroadcastQueryStage 2
    +- ReusedExchange [d_date_sk#1086], BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#1963]
```
A root cause is that a `Cast` class in a required child's distribution does not have a `timeZoneId` field (`timeZoneId=None`), and a `Cast` class in `child.outputPartitioning` has it. So, this difference can make the distribution requirement check fail in `EnsureRequirements`:
https://github.com/apache/spark/blob/1e85707738a830d33598ca267a6740b3f06b1861/sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/EnsureRequirements.scala#L47-L50

The `Cast` class that does not have a `timeZoneId` field is generated in the `HashJoin` object. To fix this issue, this PR proposes to use the `CastSupport.cast` method there.

This is a backport PR for #30818.

### Why are the changes needed?

Bugfix.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually checked that q5 passed with AQE enabled.

Closes #30830 from maropu/SPARK-33822-BRANCH3.0.

Authored-by: Takeshi Yamamuro <yamamuro@apache.org>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../spark/sql/execution/joins/HashJoin.scala  | 15 ++++++---
 .../execution/joins/BroadcastJoinSuite.scala  | 33 +++++++++++--------
 2 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
index 931a75a83d4ef..0275741844bca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/HashJoin.scala
@@ -18,12 +18,14 @@
 package org.apache.spark.sql.execution.joins
 
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.CastSupport
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReferences
 import org.apache.spark.sql.catalyst.plans._
 import org.apache.spark.sql.catalyst.plans.physical.Partitioning
 import org.apache.spark.sql.execution.{ExplainUtils, RowIterator, SparkPlan}
 import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{IntegralType, LongType}
 
 trait HashJoin {
@@ -240,7 +242,10 @@ trait HashJoin {
   }
 }
 
-object HashJoin {
+object HashJoin extends CastSupport {
+
+  override def conf: SQLConf = SQLConf.get
+
   /**
    * Try to rewrite the key as LongType so we can use getLong(), if they key can fit with a long.
    *
@@ -255,14 +260,14 @@ object HashJoin {
     }
 
     var keyExpr: Expression = if (keys.head.dataType != LongType) {
-      Cast(keys.head, LongType)
+      cast(keys.head, LongType)
     } else {
       keys.head
     }
     keys.tail.foreach { e =>
       val bits = e.dataType.defaultSize * 8
       keyExpr = BitwiseOr(ShiftLeft(keyExpr, Literal(bits)),
-        BitwiseAnd(Cast(e, LongType), Literal((1L << bits) - 1)))
+        BitwiseAnd(cast(e, LongType), Literal((1L << bits) - 1)))
     }
     keyExpr :: Nil
   }
@@ -275,13 +280,13 @@ object HashJoin {
     // jump over keys that have a higher index value than the required key
     if (keys.size == 1) {
       assert(index == 0)
-      Cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
+      cast(BoundReference(0, LongType, nullable = false), keys(index).dataType)
     } else {
       val shiftedBits =
         keys.slice(index + 1, keys.size).map(_.dataType.defaultSize * 8).sum
       val mask = (1L << (keys(index).dataType.defaultSize * 8)) - 1
       // build the schema for unpacking the required key
-      Cast(BitwiseAnd(
+      cast(BitwiseAnd(
         ShiftRightUnsigned(BoundReference(0, LongType, nullable = false), Literal(shiftedBits)),
         Literal(mask)), keys(index).dataType)
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
index 335ef257920c4..ef0a596f21104 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
@@ -239,33 +239,40 @@ abstract class BroadcastJoinSuiteBase extends QueryTest with SQLTestUtils
     assert(HashJoin.rewriteKeyExpr(l :: l :: Nil) === l :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(l :: i :: Nil) === l :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(i :: Nil) === Cast(i, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(i :: Nil) ===
+      Cast(i, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: l :: Nil) === i :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(i, LongType), Literal(32)),
-        BitwiseAnd(Cast(i, LongType), Literal((1L << 32) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal(32)),
+        BitwiseAnd(Cast(i, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 32) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(i :: i :: i :: Nil) === i :: i :: i :: Nil)
 
-    assert(HashJoin.rewriteKeyExpr(s :: Nil) === Cast(s, LongType) :: Nil)
+    assert(HashJoin.rewriteKeyExpr(s :: Nil) ===
+      Cast(s, LongType, Some(conf.sessionLocalTimeZone)) :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: l :: Nil) === s :: l :: Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: Nil) ===
-      BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+      BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
-        BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+        BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: Nil) ===
       BitwiseOr(ShiftLeft(
         BitwiseOr(ShiftLeft(
-          BitwiseOr(ShiftLeft(Cast(s, LongType), Literal(16)),
-            BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseOr(ShiftLeft(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal(16)),
+            BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)),
+              Literal((1L << 16) - 1))),
           Literal(16)),
-          BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))),
+          BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))),
         Literal(16)),
-        BitwiseAnd(Cast(s, LongType), Literal((1L << 16) - 1))) :: Nil)
+        BitwiseAnd(Cast(s, LongType, Some(conf.sessionLocalTimeZone)), Literal((1L << 16) - 1))) ::
+        Nil)
     assert(HashJoin.rewriteKeyExpr(s :: s :: s :: s :: s :: Nil) ===
       s :: s :: s :: s :: s :: Nil)
 

From faf4a0e0e451071f330981bd5aeb5a47fa7d102b Mon Sep 17 00:00:00 2001
From: Sean Owen <srowen@gmail.com>
Date: Thu, 17 Dec 2020 19:09:57 -0800
Subject: [PATCH 1224/1280] [SPARK-33831][UI] Update to jetty 9.4.34

Update Jetty to 9.4.34

Picks up fixes and improvements, including a possible CVE fix.

https://github.com/eclipse/jetty.project/releases/tag/jetty-9.4.33.v20201020
https://github.com/eclipse/jetty.project/releases/tag/jetty-9.4.34.v20201102

No.

Existing tests.

Closes #30828 from srowen/SPARK-33831.

Authored-by: Sean Owen <srowen@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
(cherry picked from commit 131a23d88a56280d47584aed93bc8fb617550717)
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 60b57fb2d039e..58eb8895a95aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@
     <orc.classifier></orc.classifier>
     <hive.parquet.group>com.twitter</hive.parquet.group>
     <hive.parquet.version>1.6.0</hive.parquet.version>
-    <jetty.version>9.4.28.v20200408</jetty.version>
+    <jetty.version>9.4.34.v20201102</jetty.version>
     <javaxservlet.version>3.1.0</javaxservlet.version>
     <chill.version>0.9.5</chill.version>
     <ivy.version>2.4.0</ivy.version>

From f67c3c26a0ed326774bb6bbe71c0f745260d1323 Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Fri, 18 Dec 2020 09:55:01 -0800
Subject: [PATCH 1225/1280] [SPARK-33593][SQL][3.0] Vector reader got incorrect
 data with binary partition value

### What changes were proposed in this pull request?

Currently when enable parquet vectorized reader, use binary type as partition col will return incorrect value as below UT
```scala
test("Parquet vector reader incorrect with binary partition value") {
  Seq(false, true).foreach(tag => {
    withSQLConf("spark.sql.parquet.enableVectorizedReader" -> tag.toString) {
      withTable("t1") {
        sql(
          """CREATE TABLE t1(name STRING, id BINARY, part BINARY)
            | USING PARQUET PARTITIONED BY (part)""".stripMargin)
        sql(s"INSERT INTO t1 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
        if (tag) {
          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t1"),
            Row("a", "Spark SQL", ""))
        } else {
          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t1"),
            Row("a", "Spark SQL", "Spark SQL"))
        }
      }
    }
  })
}
```

### Why are the changes needed?
Fix data incorrect issue

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Added UT

Closes #30839 from AngersZhuuuu/SPARK-33593-3.0.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../vectorized/ColumnVectorUtils.java         |  5 ++
 .../org/apache/spark/sql/SQLQuerySuite.scala  | 26 +++++++
 .../orc/OrcColumnarBatchReaderSuite.scala     | 77 ++++++++++++++++++-
 .../datasources/parquet/ParquetIOSuite.scala  |  9 ++-
 4 files changed, 114 insertions(+), 3 deletions(-)

diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
index bce6aa28c42a1..25aabcd086289 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java
@@ -54,6 +54,8 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
     } else {
       if (t == DataTypes.BooleanType) {
         col.putBooleans(0, capacity, row.getBoolean(fieldIdx));
+      } else if (t == DataTypes.BinaryType) {
+        col.putByteArray(0, row.getBinary(fieldIdx));
       } else if (t == DataTypes.ByteType) {
         col.putBytes(0, capacity, row.getByte(fieldIdx));
       } else if (t == DataTypes.ShortType) {
@@ -94,6 +96,9 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
         col.putInts(0, capacity, row.getInt(fieldIdx));
       } else if (t instanceof TimestampType) {
         col.putLongs(0, capacity, row.getLong(fieldIdx));
+      } else {
+        throw new RuntimeException(String.format("DataType %s is not supported" +
+            " in column vectorized reader.", t.sql()));
       }
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 1e2bb4810a6bd..fdad69d1f1bbd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3549,6 +3549,32 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       checkAnswer(sql("SELECT s LIKE 'm@@ca' ESCAPE '@' FROM df"), Row(true))
     }
   }
+
+  test("SPARK-33593: Vector reader got incorrect data with binary partition value") {
+    Seq("false", "true").foreach(value => {
+      withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> value) {
+        withTable("t1") {
+          sql(
+            """CREATE TABLE t1(name STRING, id BINARY, part BINARY)
+              |USING PARQUET PARTITIONED BY (part)""".stripMargin)
+          sql("INSERT INTO t1 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
+          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t1"),
+            Row("a", "Spark SQL", "Spark SQL"))
+        }
+      }
+
+      withSQLConf(SQLConf.ORC_VECTORIZED_READER_ENABLED.key -> value) {
+        withTable("t2") {
+          sql(
+            """CREATE TABLE t2(name STRING, id BINARY, part BINARY)
+              |USING ORC PARTITIONED BY (part)""".stripMargin)
+          sql("INSERT INTO t2 PARTITION(part = 'Spark SQL') VALUES('a', X'537061726B2053514C')")
+          checkAnswer(sql("SELECT name, cast(id as string), cast(part as string) FROM t2"),
+            Row("a", "Spark SQL", "Spark SQL"))
+        }
+      }
+    })
+  }
 }
 
 case class Foo(bar: Option[String])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
index 719bf91e1786b..bfcef46339908 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcColumnarBatchReaderSuite.scala
@@ -17,16 +17,29 @@
 
 package org.apache.spark.sql.execution.datasources.orc
 
+import java.io.File
+
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapreduce.{JobID, TaskAttemptID, TaskID, TaskType}
+import org.apache.hadoop.mapreduce.lib.input.FileSplit
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.orc.TypeDescription
 
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase
 import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector, WritableColumnVector}
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{StructField, StructType}
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.unsafe.types.UTF8String.fromString
 
 class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession {
+
+  import testImplicits._
+
   private val dataSchema = StructType.fromDDL("col1 int, col2 int")
   private val partitionSchema = StructType.fromDDL("p1 string, p2 string")
   private val partitionValues = InternalRow(fromString("partValue1"), fromString("partValue2"))
@@ -77,4 +90,66 @@ class OrcColumnarBatchReaderSuite extends QueryTest with SharedSparkSession {
       assert(p1.getUTF8String(0) === partitionValues.getUTF8String(0))
     }
   }
+
+  test("SPARK-33593: partition column types") {
+    withTempPath { dir =>
+      Seq(1).toDF().repartition(1).write.orc(dir.getCanonicalPath)
+
+      val dataTypes =
+        Seq(StringType, BooleanType, ByteType, BinaryType, ShortType, IntegerType, LongType,
+          FloatType, DoubleType, DecimalType(25, 5), DateType, TimestampType)
+
+      val constantValues =
+        Seq(
+          UTF8String.fromString("a string"),
+          true,
+          1.toByte,
+          "Spark SQL".getBytes,
+          2.toShort,
+          3,
+          Long.MaxValue,
+          0.25.toFloat,
+          0.75D,
+          Decimal("1234.23456"),
+          DateTimeUtils.fromJavaDate(java.sql.Date.valueOf("2015-01-01")),
+          DateTimeUtils.fromJavaTimestamp(java.sql.Timestamp.valueOf("2015-01-01 23:50:59.123")))
+
+      dataTypes.zip(constantValues).foreach { case (dt, v) =>
+        val schema = StructType(StructField("col1", IntegerType) :: StructField("pcol", dt) :: Nil)
+        val partitionValues = new GenericInternalRow(Array(v))
+        val file = new File(SpecificParquetRecordReaderBase.listDirectory(dir).get(0))
+        val fileSplit = new FileSplit(new Path(file.getCanonicalPath), 0L, file.length, Array.empty)
+        val taskConf = sqlContext.sessionState.newHadoopConf()
+        val orcFileSchema = TypeDescription.fromString(schema.simpleString)
+        val vectorizedReader = new OrcColumnarBatchReader(4096)
+        val attemptId = new TaskAttemptID(new TaskID(new JobID(), TaskType.MAP, 0), 0)
+        val taskAttemptContext = new TaskAttemptContextImpl(taskConf, attemptId)
+
+        try {
+          vectorizedReader.initialize(fileSplit, taskAttemptContext)
+          vectorizedReader.initBatch(
+            orcFileSchema,
+            schema.toArray,
+            Array(0, -1),
+            Array(-1, 0),
+            partitionValues)
+          vectorizedReader.nextKeyValue()
+          val row = vectorizedReader.getCurrentValue.getRow(0)
+
+          // Use `GenericMutableRow` by explicitly copying rather than `ColumnarBatch`
+          // in order to use get(...) method which is not implemented in `ColumnarBatch`.
+          val actual = row.copy().get(1, dt)
+          val expected = v
+          if (dt.isInstanceOf[BinaryType]) {
+            assert(actual.asInstanceOf[Array[Byte]]
+              sameElements expected.asInstanceOf[Array[Byte]])
+          } else {
+            assert(actual == expected)
+          }
+        } finally {
+          vectorizedReader.close()
+        }
+      }
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
index ff406f7bc62de..0984711dfcdfd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
@@ -791,7 +791,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
       Seq(1).toDF().repartition(1).write.parquet(dir.getCanonicalPath)
 
       val dataTypes =
-        Seq(StringType, BooleanType, ByteType, ShortType, IntegerType, LongType,
+        Seq(StringType, BooleanType, ByteType, BinaryType, ShortType, IntegerType, LongType,
           FloatType, DoubleType, DecimalType(25, 5), DateType, TimestampType)
 
       val constantValues =
@@ -799,6 +799,7 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           UTF8String.fromString("a string"),
           true,
           1.toByte,
+          "Spark SQL".getBytes,
           2.toShort,
           3,
           Long.MaxValue,
@@ -826,7 +827,11 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
           // in order to use get(...) method which is not implemented in `ColumnarBatch`.
           val actual = row.copy().get(1, dt)
           val expected = v
-          assert(actual == expected)
+          if (dt.isInstanceOf[BinaryType]) {
+            assert(actual.asInstanceOf[Array[Byte]] sameElements expected.asInstanceOf[Array[Byte]])
+          } else {
+            assert(actual == expected)
+          }
         } finally {
           vectorizedReader.close()
         }

From 7881622899082f73b99d9e92f6b08979005ba4df Mon Sep 17 00:00:00 2001
From: Vlad Glinsky <vladhlinsky@gmail.com>
Date: Fri, 18 Dec 2020 15:19:09 -0800
Subject: [PATCH 1226/1280] [SPARK-33841][CORE][3.0] Fix issue with jobs
 disappearing intermittently from the SHS under high load

### What changes were proposed in this pull request?

Mark SHS event log entries that were `processing` at the beginning of the `checkForLogs` run as not stale and check for this mark before deleting an event log. This fixes the issue when a particular job was displayed in the SHS and disappeared after some time, but then, in several minutes showed up again.

### Why are the changes needed?

The issue is caused by [SPARK-29043](https://issues.apache.org/jira/browse/SPARK-29043), which is designated to improve the concurrent performance of the History Server. The [change](https://github.com/apache/spark/pull/25797/files#) breaks the ["app deletion" logic](https://github.com/apache/spark/pull/25797/files#diff-128a6af0d78f4a6180774faedb335d6168dfc4defff58f5aa3021fc1bd767bc0R563) because of missing proper synchronization for `processing` event log entries. Since SHS now [filters out](https://github.com/apache/spark/pull/25797/files#diff-128a6af0d78f4a6180774faedb335d6168dfc4defff58f5aa3021fc1bd767bc0R462) all `processing` event log entries, such entries do not have a chance to be [updated with the new `lastProcessed`](https://github.com/apache/spark/pull/25797/files#diff-128a6af0d78f4a6180774faedb335d6168dfc4defff58f5aa3021fc1bd767bc0R472) time and thus any entity that completes processing right after [filtering](https://github.com/apache/spark/pull/25797/files#diff-128a6af0d78f4a6180774faedb335d6168dfc4defff58f5aa3021fc1bd767bc0R462) and before [the check for stale entities](https://github.com/apache/spark/pull/25797/files#diff-128a6af0d78f4a6180774faedb335d6168dfc4defff58f5aa3021fc1bd767bc0R560) will be identified as stale and will be deleted from the UI until the next `checkForLogs` run. This is because [updated `lastProcessed` time is used as criteria](https://github.com/apache/spark/pull/25797/files#diff-128a6af0d78f4a6180774faedb335d6168dfc4defff58f5aa3021fc1bd767bc0R557), and event log entries that missed to be updated with a new time, will match that criteria.

The issue can be reproduced by generating a big number of event logs and uploading them to the SHS event log directory on S3. Essentially, around 800(82.6 MB) copies of an event log file were created using [shs-monitor](https://github.com/vladhlinsky/shs-monitor) script. Strange behavior of SHS counting the total number of applications was noticed - at first, the number was increasing as expected, but with the next page refresh, the total number of applications decreased. No errors were logged by SHS.

241 entities are displayed at `20:50:42`:
![1-241-entities-at-20-50](https://user-images.githubusercontent.com/61428392/102611539-c2138d00-4137-11eb-9bbd-d77b22041f3b.png)
203 entities are displayed at `20:52:17`:
![2-203-entities-at-20-52](https://user-images.githubusercontent.com/61428392/102611561-cdff4f00-4137-11eb-91ed-7405fe58a695.png)
The number of loaded applications over time:
![4-loaded-applications](https://user-images.githubusercontent.com/61428392/102611586-d8b9e400-4137-11eb-8747-4007fc5469de.png)

### Does this PR introduce _any_ user-facing change?

Yes, SHS users won't face the behavior when the number of displayed applications decreases periodically.

### How was this patch tested?

Tested using [shs-monitor](https://github.com/vladhlinsky/shs-monitor) script:
* Build SHS with the proposed change
* Download Hadoop AWS and AWS Java SDK
* Prepare S3 bucket and user for programmatic access, grant required roles to the user. Get access key and secret key
* Configure SHS to read event logs from S3
* Start [monitor](https://github.com/vladhlinsky/shs-monitor/blob/main/monitor.sh) script to query SHS API
* Run 8 [producers](https://github.com/vladhlinsky/shs-monitor/blob/main/producer.sh) for ~10 mins, create 805(83.1 MB) event log copies
* Wait for SHS to load all the applications
* Verify that the number of loaded applications increases continuously over time
![5-loaded-applications-fixed](https://user-images.githubusercontent.com/61428392/102617363-bf1d9a00-4141-11eb-9bae-f982d02fd30f.png)

For more details, please refer to the [shs-monitor](https://github.com/vladhlinsky/shs-monitor) repository.

Closes #30842 from vladhlinsky/SPARK-33841-branch-3.0.

Authored-by: Vlad Glinsky <vladhlinsky@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
---
 .../deploy/history/FsHistoryProvider.scala    | 26 ++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 7e63d5500b2ab..b31333fc48218 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -460,9 +460,21 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       val newLastScanTime = clock.getTimeMillis()
       logDebug(s"Scanning $logDir with lastScanTime==$lastScanTime")
 
+      // Mark entries that are processing as not stale. Such entries do not have a chance to be
+      // updated with the new 'lastProcessed' time and thus any entity that completes processing
+      // right after this check and before the check for stale entities will be identified as stale
+      // and will be deleted from the UI until the next 'checkForLogs' run.
+      val notStale = mutable.HashSet[String]()
       val updated = Option(fs.listStatus(new Path(logDir))).map(_.toSeq).getOrElse(Nil)
         .filter { entry => !isBlacklisted(entry.getPath) }
-        .filter { entry => !isProcessing(entry.getPath) }
+        .filter { entry =>
+          if (isProcessing(entry.getPath)) {
+            notStale.add(entry.getPath.toString())
+            false
+          } else {
+            true
+          }
+        }
         .flatMap { entry => EventLogFileReader(fs, entry) }
         .filter { reader =>
           try {
@@ -562,12 +574,14 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
         .last(newLastScanTime - 1)
         .asScala
         .toList
-      stale.filterNot(isProcessing).foreach { log =>
-        log.appId.foreach { appId =>
-          cleanAppData(appId, log.attemptId, log.logPath)
-          listing.delete(classOf[LogInfo], log.logPath)
+      stale.filterNot(isProcessing)
+        .filterNot(info => notStale.contains(info.logPath))
+        .foreach { log =>
+          log.appId.foreach { appId =>
+            cleanAppData(appId, log.attemptId, log.logPath)
+            listing.delete(classOf[LogInfo], log.logPath)
+          }
         }
-      }
 
       lastScanTime.set(newLastScanTime)
     } catch {

From faf8dd575eb1a40b5bde91191baa4ae741269531 Mon Sep 17 00:00:00 2001
From: Xianjin YE <advancedxy@gmail.com>
Date: Sun, 20 Dec 2020 08:51:17 -0600
Subject: [PATCH 1227/1280] [SPARK-33756][SQL] Make BytesToBytesMap's
 MapIterator idempotent

### What changes were proposed in this pull request?
Make MapIterator of BytesToBytesMap `hasNext` method idempotent

### Why are the changes needed?
The `hasNext` maybe called multiple times, if not guarded, second call of hasNext method after reaching the end of iterator will throw NoSuchElement exception.

### Does this PR introduce _any_ user-facing change?
NO.

### How was this patch tested?
Update a unit test to cover this case.

Closes #30728 from advancedxy/SPARK-33756.

Authored-by: Xianjin YE <advancedxy@gmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 13391683e7a863671d3d719dc81e20ec2a870725)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../org/apache/spark/unsafe/map/BytesToBytesMap.java   | 10 ++++++----
 .../spark/unsafe/map/AbstractBytesToBytesMapSuite.java |  2 ++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 4036856e41851..888cc8c3a6cd6 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -393,10 +393,12 @@ public void remove() {
     }
 
     private void handleFailedDelete() {
-      // remove the spill file from disk
-      File file = spillWriters.removeFirst().getFile();
-      if (file != null && file.exists() && !file.delete()) {
-        logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
+      if (spillWriters.size() > 0) {
+        // remove the spill file from disk
+        File file = spillWriters.removeFirst().getFile();
+        if (file != null && file.exists() && !file.delete()) {
+          logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
+        }
       }
     }
   }
diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
index 6e995a3929a75..ab177cefa1cca 100644
--- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
+++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
@@ -561,6 +561,8 @@ public void spillInIterator() throws IOException {
         iter2.next();
       }
       assertFalse(iter2.hasNext());
+      // calls hasNext twice deliberately, make sure it's idempotent
+      assertFalse(iter2.hasNext());
     } finally {
       map.free();
       for (File spillFile : spillFilesCreated) {

From 78dbb4a672ddac3d77fa40a4c7fa8c70abf0f26d Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Mon, 21 Dec 2020 03:29:00 -0800
Subject: [PATCH 1228/1280] [SPARK-33853][SQL] EXPLAIN CODEGEN and
 BenchmarkQueryTest don't show subquery code

### What changes were proposed in this pull request?

This PR fixes an issue that `EXPLAIN CODEGEN` and `BenchmarkQueryTest` don't show the corresponding code for subqueries.

The following example is about `EXPLAIN CODEGEN`.
```
spark.conf.set("spark.sql.adaptive.enabled", "false")
val df = spark.range(1, 100)
df.createTempView("df")
spark.sql("SELECT (SELECT min(id) AS v FROM df)").explain("CODEGEN")

scala> spark.sql("SELECT (SELECT min(id) AS v FROM df)").explain("CODEGEN")
Found 1 WholeStageCodegen subtrees.
== Subtree 1 / 1 (maxMethodCodeSize:55; maxConstantPoolSize:97(0.15% used); numInnerClasses:0) ==
*(1) Project [Subquery scalar-subquery#3, [id=#24] AS scalarsubquery()#5L]
:  +- Subquery scalar-subquery#3, [id=#24]
:     +- *(2) HashAggregate(keys=[], functions=[min(id#0L)], output=[v#2L])
:        +- Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#20]
:           +- *(1) HashAggregate(keys=[], functions=[partial_min(id#0L)], output=[min#8L])
:              +- *(1) Range (1, 100, step=1, splits=12)
+- *(1) Scan OneRowRelation[]

Generated code:
/* 001 */ public Object generate(Object[] references) {
/* 002 */   return new GeneratedIteratorForCodegenStage1(references);
/* 003 */ }
/* 004 */
/* 005 */ // codegenStageId=1
/* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator {
/* 007 */   private Object[] references;
/* 008 */   private scala.collection.Iterator[] inputs;
/* 009 */   private scala.collection.Iterator rdd_input_0;
/* 010 */   private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] project_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[1];
/* 011 */
/* 012 */   public GeneratedIteratorForCodegenStage1(Object[] references) {
/* 013 */     this.references = references;
/* 014 */   }
/* 015 */
/* 016 */   public void init(int index, scala.collection.Iterator[] inputs) {
/* 017 */     partitionIndex = index;
/* 018 */     this.inputs = inputs;
/* 019 */     rdd_input_0 = inputs[0];
/* 020 */     project_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0);
/* 021 */
/* 022 */   }
/* 023 */
/* 024 */   private void project_doConsume_0() throws java.io.IOException {
/* 025 */     // common sub-expressions
/* 026 */
/* 027 */     project_mutableStateArray_0[0].reset();
/* 028 */
/* 029 */     if (false) {
/* 030 */       project_mutableStateArray_0[0].setNullAt(0);
/* 031 */     } else {
/* 032 */       project_mutableStateArray_0[0].write(0, 1L);
/* 033 */     }
/* 034 */     append((project_mutableStateArray_0[0].getRow()));
/* 035 */
/* 036 */   }
/* 037 */
/* 038 */   protected void processNext() throws java.io.IOException {
/* 039 */     while ( rdd_input_0.hasNext()) {
/* 040 */       InternalRow rdd_row_0 = (InternalRow) rdd_input_0.next();
/* 041 */       ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1);
/* 042 */       project_doConsume_0();
/* 043 */       if (shouldStop()) return;
/* 044 */     }
/* 045 */   }
/* 046 */
/* 047 */ }
```

After this change, the corresponding code for subqueries are shown.
```
Found 3 WholeStageCodegen subtrees.
== Subtree 1 / 3 (maxMethodCodeSize:282; maxConstantPoolSize:206(0.31% used); numInnerClasses:0) ==
*(1) HashAggregate(keys=[], functions=[partial_min(id#0L)], output=[min#8L])
+- *(1) Range (1, 100, step=1, splits=12)

Generated code:
/* 001 */ public Object generate(Object[] references) {
/* 002 */   return new GeneratedIteratorForCodegenStage1(references);
/* 003 */ }
/* 004 */
/* 005 */ // codegenStageId=1
/* 006 */ final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator {
/* 007 */   private Object[] references;
/* 008 */   private scala.collection.Iterator[] inputs;
/* 009 */   private boolean agg_initAgg_0;
/* 010 */   private boolean agg_bufIsNull_0;
/* 011 */   private long agg_bufValue_0;
/* 012 */   private boolean range_initRange_0;
/* 013 */   private long range_nextIndex_0;
/* 014 */   private TaskContext range_taskContext_0;
/* 015 */   private InputMetrics range_inputMetrics_0;
/* 016 */   private long range_batchEnd_0;
/* 017 */   private long range_numElementsTodo_0;
/* 018 */   private boolean agg_agg_isNull_2_0;
/* 019 */   private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] range_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[3];
/* 020 */
/* 021 */   public GeneratedIteratorForCodegenStage1(Object[] references) {
/* 022 */     this.references = references;
/* 023 */   }
/* 024 */
/* 025 */   public void init(int index, scala.collection.Iterator[] inputs) {
/* 026 */     partitionIndex = index;
/* 027 */     this.inputs = inputs;
/* 028 */
/* 029 */     range_taskContext_0 = TaskContext.get();
/* 030 */     range_inputMetrics_0 = range_taskContext_0.taskMetrics().inputMetrics();
/* 031 */     range_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0);
/* 032 */     range_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0);
/* 033 */     range_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 0);
/* 034 */
/* 035 */   }
/* 036 */
/* 037 */   private void agg_doAggregateWithoutKey_0() throws java.io.IOException {
/* 038 */     // initialize aggregation buffer
/* 039 */     agg_bufIsNull_0 = true;
/* 040 */     agg_bufValue_0 = -1L;
/* 041 */
/* 042 */     // initialize Range
/* 043 */     if (!range_initRange_0) {
/* 044 */       range_initRange_0 = true;
/* 045 */       initRange(partitionIndex);
/* 046 */     }
/* 047 */
/* 048 */     while (true) {
/* 049 */       if (range_nextIndex_0 == range_batchEnd_0) {
/* 050 */         long range_nextBatchTodo_0;
/* 051 */         if (range_numElementsTodo_0 > 1000L) {
/* 052 */           range_nextBatchTodo_0 = 1000L;
/* 053 */           range_numElementsTodo_0 -= 1000L;
/* 054 */         } else {
/* 055 */           range_nextBatchTodo_0 = range_numElementsTodo_0;
/* 056 */           range_numElementsTodo_0 = 0;
/* 057 */           if (range_nextBatchTodo_0 == 0) break;
/* 058 */         }
/* 059 */         range_batchEnd_0 += range_nextBatchTodo_0 * 1L;
/* 060 */       }
/* 061 */
/* 062 */       int range_localEnd_0 = (int)((range_batchEnd_0 - range_nextIndex_0) / 1L);
/* 063 */       for (int range_localIdx_0 = 0; range_localIdx_0 < range_localEnd_0; range_localIdx_0++) {
/* 064 */         long range_value_0 = ((long)range_localIdx_0 * 1L) + range_nextIndex_0;
/* 065 */
/* 066 */         agg_doConsume_0(range_value_0);
/* 067 */
/* 068 */         // shouldStop check is eliminated
/* 069 */       }
/* 070 */       range_nextIndex_0 = range_batchEnd_0;
/* 071 */       ((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(range_localEnd_0);
/* 072 */       range_inputMetrics_0.incRecordsRead(range_localEnd_0);
/* 073 */       range_taskContext_0.killTaskIfInterrupted();
/* 074 */     }
/* 075 */
/* 076 */   }
/* 077 */
/* 078 */   private void initRange(int idx) {
/* 079 */     java.math.BigInteger index = java.math.BigInteger.valueOf(idx);
/* 080 */     java.math.BigInteger numSlice = java.math.BigInteger.valueOf(12L);
/* 081 */     java.math.BigInteger numElement = java.math.BigInteger.valueOf(99L);
/* 082 */     java.math.BigInteger step = java.math.BigInteger.valueOf(1L);
/* 083 */     java.math.BigInteger start = java.math.BigInteger.valueOf(1L);
/* 084 */     long partitionEnd;
/* 085 */
/* 086 */     java.math.BigInteger st = index.multiply(numElement).divide(numSlice).multiply(step).add(start);
/* 087 */     if (st.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
/* 088 */       range_nextIndex_0 = Long.MAX_VALUE;
/* 089 */     } else if (st.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) {
/* 090 */       range_nextIndex_0 = Long.MIN_VALUE;
/* 091 */     } else {
/* 092 */       range_nextIndex_0 = st.longValue();
/* 093 */     }
/* 094 */     range_batchEnd_0 = range_nextIndex_0;
/* 095 */
/* 096 */     java.math.BigInteger end = index.add(java.math.BigInteger.ONE).multiply(numElement).divide(numSlice)
/* 097 */     .multiply(step).add(start);
/* 098 */     if (end.compareTo(java.math.BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
/* 099 */       partitionEnd = Long.MAX_VALUE;
/* 100 */     } else if (end.compareTo(java.math.BigInteger.valueOf(Long.MIN_VALUE)) < 0) {
/* 101 */       partitionEnd = Long.MIN_VALUE;
/* 102 */     } else {
/* 103 */       partitionEnd = end.longValue();
/* 104 */     }
/* 105 */
/* 106 */     java.math.BigInteger startToEnd = java.math.BigInteger.valueOf(partitionEnd).subtract(
/* 107 */       java.math.BigInteger.valueOf(range_nextIndex_0));
/* 108 */     range_numElementsTodo_0  = startToEnd.divide(step).longValue();
/* 109 */     if (range_numElementsTodo_0 < 0) {
/* 110 */       range_numElementsTodo_0 = 0;
/* 111 */     } else if (startToEnd.remainder(step).compareTo(java.math.BigInteger.valueOf(0L)) != 0) {
/* 112 */       range_numElementsTodo_0++;
/* 113 */     }
/* 114 */   }
/* 115 */
/* 116 */   private void agg_doConsume_0(long agg_expr_0_0) throws java.io.IOException {
/* 117 */     // do aggregate
/* 118 */     // common sub-expressions
/* 119 */
/* 120 */     // evaluate aggregate functions and update aggregation buffers
/* 121 */
/* 122 */     agg_agg_isNull_2_0 = true;
/* 123 */     long agg_value_2 = -1L;
/* 124 */
/* 125 */     if (!agg_bufIsNull_0 && (agg_agg_isNull_2_0 ||
/* 126 */         agg_value_2 > agg_bufValue_0)) {
/* 127 */       agg_agg_isNull_2_0 = false;
/* 128 */       agg_value_2 = agg_bufValue_0;
/* 129 */     }
/* 130 */
/* 131 */     if (!false && (agg_agg_isNull_2_0 ||
/* 132 */         agg_value_2 > agg_expr_0_0)) {
/* 133 */       agg_agg_isNull_2_0 = false;
/* 134 */       agg_value_2 = agg_expr_0_0;
/* 135 */     }
/* 136 */
/* 137 */     agg_bufIsNull_0 = agg_agg_isNull_2_0;
/* 138 */     agg_bufValue_0 = agg_value_2;
/* 139 */
/* 140 */   }
/* 141 */
/* 142 */   protected void processNext() throws java.io.IOException {
/* 143 */     while (!agg_initAgg_0) {
/* 144 */       agg_initAgg_0 = true;
/* 145 */       long agg_beforeAgg_0 = System.nanoTime();
/* 146 */       agg_doAggregateWithoutKey_0();
/* 147 */       ((org.apache.spark.sql.execution.metric.SQLMetric) references[2] /* aggTime */).add((System.nanoTime() - agg_beforeAgg_0) / 1000000);
/* 148 */
/* 149 */       // output the result
/* 150 */
/* 151 */       ((org.apache.spark.sql.execution.metric.SQLMetric) references[1] /* numOutputRows */).add(1);
/* 152 */       range_mutableStateArray_0[2].reset();
/* 153 */
/* 154 */       range_mutableStateArray_0[2].zeroOutNullBytes();
/* 155 */
/* 156 */       if (agg_bufIsNull_0) {
/* 157 */         range_mutableStateArray_0[2].setNullAt(0);
/* 158 */       } else {
/* 159 */         range_mutableStateArray_0[2].write(0, agg_bufValue_0);
/* 160 */       }
/* 161 */       append((range_mutableStateArray_0[2].getRow()));
/* 162 */     }
/* 163 */   }
/* 164 */
/* 165 */ }
```

### Why are the changes needed?

For better debuggability.

### Does this PR introduce _any_ user-facing change?

Yes. After this change, users can see subquery code by `EXPLAIN CODEGEN`.

### How was this patch tested?

New test.

Closes #30859 from sarutak/explain-codegen-subqueries.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit f4e1069bb835e3e132f7758e5842af79f26cd162)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/execution/debug/package.scala      | 15 ++++++++++-----
 .../apache/spark/sql/BenchmarkQueryTest.scala    | 14 ++++++++++----
 .../org/apache/spark/sql/ExplainSuite.scala      | 16 ++++++++++++++++
 3 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
index 6c40104e52a5f..3cbebca14f7dc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/debug/package.scala
@@ -107,12 +107,17 @@ package object debug {
    */
   def codegenStringSeq(plan: SparkPlan): Seq[(String, String, ByteCodeStats)] = {
     val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
-    plan transform {
-      case s: WholeStageCodegenExec =>
-        codegenSubtrees += s
-        s
-      case s => s
+
+    def findSubtrees(plan: SparkPlan): Unit = {
+      plan foreach {
+        case s: WholeStageCodegenExec =>
+          codegenSubtrees += s
+        case s =>
+          s.subqueries.foreach(findSubtrees)
+      }
     }
+
+    findSubtrees(plan)
     codegenSubtrees.toSeq.sortBy(_.codegenStageId).map { subtree =>
       val (_, source) = subtree.doCodeGen()
       val codeStats = try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
index 07afd4195c3d4..174e7345d56b6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
@@ -50,11 +50,17 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
 
   protected def checkGeneratedCode(plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
     val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
-    plan foreach {
-      case s: WholeStageCodegenExec =>
-        codegenSubtrees += s
-      case _ =>
+
+    def findSubtrees(plan: SparkPlan): Unit = {
+      plan foreach {
+        case s: WholeStageCodegenExec =>
+          codegenSubtrees += s
+        case s =>
+          s.subqueries.foreach(findSubtrees)
+      }
     }
+
+    findSubtrees(plan)
     codegenSubtrees.toSeq.foreach { subtree =>
       val code = subtree.doCodeGen()._2
       val (_, ByteCodeStats(maxMethodCodeSize, _, _)) = try {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
index d41d624f1762d..158d9392c0f5c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ExplainSuite.scala
@@ -228,6 +228,22 @@ class ExplainSuite extends ExplainSuiteHelper with DisableAdaptiveExecutionSuite
     }
   }
 
+  test("SPARK-33853: explain codegen - check presence of subquery") {
+    withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+      withTempView("df") {
+        val df1 = spark.range(1, 100)
+        df1.createTempView("df")
+
+        val sqlText = "EXPLAIN CODEGEN SELECT (SELECT min(id) FROM df)"
+        val expectedText = "Found 3 WholeStageCodegen subtrees."
+
+        withNormalizedExplain(sqlText) { normalizedOutput =>
+          assert(normalizedOutput.contains(expectedText))
+        }
+      }
+    }
+  }
+
   test("explain formatted - check presence of subquery in case of DPP") {
     withTable("df1", "df2") {
       withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",

From c9fe712a6bb4234b37cd0d87cfd2fe8a983a0552 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Mon, 21 Dec 2020 11:11:25 -0800
Subject: [PATCH 1229/1280] [SPARK-33869][PYTHON][SQL][TESTS] Have a separate
 metastore directory for each PySpark test job

### What changes were proposed in this pull request?

This PR proposes to have its own metastore directory to avoid potential conflict in catalog operations.

### Why are the changes needed?

To make PySpark tests less flaky.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

Manually tested by trying some sleeps in https://github.com/apache/spark/pull/30873.

Closes #30875 from HyukjinKwon/SPARK-33869.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 38bbccab7560f2cfd00f9f85ca800434efe950b4)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 python/run-tests.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/python/run-tests.py b/python/run-tests.py
index 9fe9f4c5adedb..f269478b2a2b8 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -87,12 +87,17 @@ def run_individual_python_test(target_dir, test_name, pyspark_python):
         tmp_dir = os.path.join(target_dir, str(uuid.uuid4()))
     os.mkdir(tmp_dir)
     env["TMPDIR"] = tmp_dir
+    metastore_dir = os.path.join(tmp_dir, str(uuid.uuid4()))
+    while os.path.isdir(metastore_dir):
+        metastore_dir = os.path.join(metastore_dir, str(uuid.uuid4()))
+    os.mkdir(metastore_dir)
 
     # Also override the JVM's temp directory by setting driver and executor options.
     java_options = "-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true".format(tmp_dir)
     spark_args = [
         "--conf", "spark.driver.extraJavaOptions='{0}'".format(java_options),
         "--conf", "spark.executor.extraJavaOptions='{0}'".format(java_options),
+        "--conf", "spark.sql.warehouse.dir='{0}'".format(metastore_dir),
         "pyspark-shell"
     ]
     env["PYSPARK_SUBMIT_ARGS"] = " ".join(spark_args)

From 0820beb60018210ee718b96acb37feb8d8445251 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Tue, 22 Dec 2020 10:30:34 +0900
Subject: [PATCH 1230/1280] [SPARK-28863][SQL][FOLLOWUP][3.0] Make sure
 optimized plan will not be re-analyzed

backport https://github.com/apache/spark/pull/30777 to 3.0

----------

### What changes were proposed in this pull request?

It's a known issue that re-analyzing an optimized plan can lead to various issues. We made several attempts to avoid it from happening, but the current solution `AlreadyOptimized` is still not 100% safe, as people can inject catalyst rules to call analyzer directly.

This PR proposes a simpler and safer idea: we set the `analyzed` flag to true after optimization, and analyzer will skip processing plans whose `analyzed` flag is true.

### Why are the changes needed?

make the code simpler and safer

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

existing tests.

Closes #30872 from cloud-fan/ds.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../sql/catalyst/analysis/Analyzer.scala      |  1 +
 .../plans/logical/AnalysisHelper.scala        |  7 +-
 .../sql/execution/AlreadyOptimized.scala      | 37 --------
 .../spark/sql/execution/QueryExecution.scala  |  7 +-
 .../datasources/v2/V1FallbackWriters.scala    |  6 +-
 .../sql/execution/AlreadyOptimizedSuite.scala | 85 -------------------
 6 files changed, 16 insertions(+), 127 deletions(-)
 delete mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala
 delete mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 1307fc5acd5f4..fbe60412b2f39 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -144,6 +144,7 @@ class Analyzer(
   }
 
   def executeAndCheck(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
+    if (plan.analyzed) return plan
     AnalysisHelper.markInAnalyzer {
       val analyzed = executeAndTrack(plan, tracker)
       try {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
index 30447db1acc04..9b6fee64be8f2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala
@@ -46,7 +46,7 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
    * Recursively marks all nodes in this plan tree as analyzed.
    * This should only be called by [[CheckAnalysis]].
    */
-  private[catalyst] def setAnalyzed(): Unit = {
+  private[sql] def setAnalyzed(): Unit = {
     if (!_analyzed) {
       _analyzed = true
       children.foreach(_.setAnalyzed())
@@ -180,6 +180,11 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
     super.transformAllExpressions(rule)
   }
 
+  override def clone(): LogicalPlan = {
+    val cloned = super.clone()
+    if (analyzed) cloned.setAnalyzed()
+    cloned
+  }
 }
 
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala
deleted file mode 100644
index e40b1141b43eb..0000000000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/AlreadyOptimized.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution
-
-import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.sql.catalyst.encoders.RowEncoder
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-
-/** Query execution that skips re-analysis and optimize. */
-class AlreadyOptimizedExecution(
-    session: SparkSession,
-    plan: LogicalPlan) extends QueryExecution(session, plan) {
-  override lazy val analyzed: LogicalPlan = plan
-  override lazy val optimizedPlan: LogicalPlan = plan
-}
-
-object AlreadyOptimized {
-  def dataFrame(sparkSession: SparkSession, optimized: LogicalPlan): DataFrame = {
-    val qe = new AlreadyOptimizedExecution(sparkSession, optimized)
-    new Dataset[Row](qe, RowEncoder(qe.analyzed.schema))
-  }
-}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 7f5a5e3c2333c..ed36c78a77460 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -79,7 +79,12 @@ class QueryExecution(
   lazy val optimizedPlan: LogicalPlan = executePhase(QueryPlanningTracker.OPTIMIZATION) {
     // clone the plan to avoid sharing the plan instance between different stages like analyzing,
     // optimizing and planning.
-    sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
+    val plan = sparkSession.sessionState.optimizer.executeAndTrack(withCachedData.clone(), tracker)
+    // We do not want optimized plans to be re-analyzed as literals that have been constant folded
+    // and such can cause issues during analysis. While `clone` should maintain the `analyzed` state
+    // of the LogicalPlan, we set the plan as analyzed here as well out of paranoia.
+    plan.setAnalyzed()
+    plan
   }
 
   private def assertOptimized(): Unit = optimizedPlan
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
index 560da39314b36..95082bc466b90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V1FallbackWriters.scala
@@ -20,12 +20,13 @@ package org.apache.spark.sql.execution.datasources.v2
 import java.util.UUID
 
 import org.apache.spark.SparkException
+import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.Attribute
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.connector.catalog.SupportsWrite
 import org.apache.spark.sql.connector.write.{LogicalWriteInfoImpl, SupportsOverwrite, SupportsTruncate, V1WriteBuilder, WriteBuilder}
-import org.apache.spark.sql.execution.{AlreadyOptimized, SparkPlan}
+import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.sources.{AlwaysTrue, Filter, InsertableRelation}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
@@ -113,8 +114,7 @@ trait SupportsV1Write extends SparkPlan {
   def plan: LogicalPlan
 
   protected def writeWithV1(relation: InsertableRelation): Seq[InternalRow] = {
-    // The `plan` is already optimized, we should not analyze and optimize it again.
-    relation.insert(AlreadyOptimized.dataFrame(sqlContext.sparkSession, plan), overwrite = false)
+    relation.insert(Dataset.ofRows(sqlContext.sparkSession, plan), overwrite = false)
     Nil
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala
deleted file mode 100644
index c266aa92f01cc..0000000000000
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/AlreadyOptimizedSuite.scala
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.execution
-
-import org.apache.spark.sql.QueryTest
-import org.apache.spark.sql.execution.adaptive.EnableAdaptiveExecutionSuite
-import org.apache.spark.sql.test.SharedSparkSession
-
-class AlreadyOptimizedSuite extends QueryTest with SharedSparkSession {
-
-  import testImplicits._
-
-  test("simple execution") {
-    val df = spark.range(10)
-    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
-
-    checkAnswer(planned, df.toDF().collect())
-  }
-
-  test("planning on top works - projection") {
-    val df = spark.range(10)
-    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
-
-    checkAnswer(
-      planned.withColumn("data", 'id + 1),
-      df.withColumn("data", 'id + 1).collect())
-  }
-
-  test("planning on top works - filter") {
-    val df = spark.range(10)
-    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
-
-    checkAnswer(planned.where('id < 5), df.where('id < 5).toDF().collect())
-  }
-
-  test("planning on top works - aggregate") {
-    val df = spark.range(10)
-    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
-
-    checkAnswer(planned.groupBy('id).count(), df.groupBy('id).count().collect())
-  }
-
-  test("planning on top works - joins") {
-    val df = spark.range(10)
-    val planned = AlreadyOptimized.dataFrame(spark, df.queryExecution.optimizedPlan)
-
-    val plannedLeft = planned.alias("l")
-    val dfLeft = df.alias("l")
-    val plannedRight = planned.alias("r")
-    val dfRight = df.alias("r")
-
-    checkAnswer(
-      plannedLeft.where('id < 3).join(plannedRight, Seq("id")),
-      dfLeft.where('id < 3).join(dfRight, Seq("id")).collect())
-
-    checkAnswer(
-      plannedLeft.where('id < 3).join(plannedRight, plannedLeft("id") === plannedRight("id")),
-      dfLeft.where('id < 3).join(dfRight, dfLeft("id") === dfRight("id")).collect())
-
-    checkAnswer(
-      plannedLeft.join(plannedRight, Seq("id")).where('id < 3),
-      dfLeft.join(dfRight, Seq("id")).where('id < 3).collect())
-
-    checkAnswer(
-      plannedLeft.join(plannedRight, plannedLeft("id") === plannedRight("id")).where($"l.id" < 3),
-      dfLeft.join(dfRight, dfLeft("id") === dfRight("id")).where($"l.id" < 3).collect())
-  }
-}
-
-class AlreadyOptimizedAQESuite extends AlreadyOptimizedSuite with EnableAdaptiveExecutionSuite

From 7af54fd76ade27b1cabe90bce1b959c2224f31ad Mon Sep 17 00:00:00 2001
From: ulysses-you <ulyssesyou18@gmail.com>
Date: Tue, 22 Dec 2020 15:10:46 +0900
Subject: [PATCH 1231/1280] [SPARK-33860][SQL] Make
 CatalystTypeConverters.convertToCatalyst match special Array value

### What changes were proposed in this pull request?

Add some case to match Array whose element type is primitive.

### Why are the changes needed?

We will get exception when use `Literal.create(Array(1, 2, 3), ArrayType(IntegerType))` .
```
Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: Literal must have a corresponding value to array<int>, but class int[] found.
	at scala.Predef$.require(Predef.scala:281)
	at org.apache.spark.sql.catalyst.expressions.Literal$.validateLiteralValue(literals.scala:215)
	at org.apache.spark.sql.catalyst.expressions.Literal.<init>(literals.scala:292)
	at org.apache.spark.sql.catalyst.expressions.Literal$.create(literals.scala:140)
```
And same problem with other array whose element is primitive.

### Does this PR introduce _any_ user-facing change?

Yes.

### How was this patch tested?

Add test.

Closes #30868 from ulysses-you/SPARK-33860.

Authored-by: ulysses-you <ulyssesyou18@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1dd63dccd893162f8ef969e42273a794ad73e49c)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/catalyst/CatalystTypeConverters.scala   |  4 +++-
 .../catalyst/expressions/LiteralExpressionSuite.scala | 11 +++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
index 34d2f45e715e9..2b467243d12e3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
@@ -456,7 +456,9 @@ object CatalystTypeConverters {
     case d: JavaBigDecimal => new DecimalConverter(DecimalType(d.precision, d.scale)).toCatalyst(d)
     case seq: Seq[Any] => new GenericArrayData(seq.map(convertToCatalyst).toArray)
     case r: Row => InternalRow(r.toSeq.map(convertToCatalyst): _*)
-    case arr: Array[Any] => new GenericArrayData(arr.map(convertToCatalyst))
+    case arr: Array[Byte] => arr
+    case arr: Array[Char] => StringConverter.toCatalyst(arr)
+    case arr: Array[_] => new GenericArrayData(arr.map(convertToCatalyst))
     case map: Map[_, _] =>
       ArrayBasedMapData(
         map,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index bb86135021b91..794e8c0656d30 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -317,4 +317,15 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
       assert(literalStr === expected)
     }
   }
+
+  test("SPARK-33860: Make CatalystTypeConverters.convertToCatalyst match special Array value") {
+    assert(Literal(Array(1, 2, 3)) == Literal.create(Array(1, 2, 3), ArrayType(IntegerType)))
+    assert(Literal(Array(1L, 2L, 3L)) == Literal.create(Array(1L, 2L, 3L), ArrayType(LongType)))
+    assert(Literal(Array(1D, 2D, 3D)) == Literal.create(Array(1D, 2D, 3D), ArrayType(DoubleType)))
+    assert(Literal("123") == Literal.create(Array('1', '2', '3'), StringType))
+    assert(Literal(Array(1.toByte, 2.toByte, 3.toByte)) ==
+      Literal.create(Array(1.toByte, 2.toByte, 3.toByte), BinaryType))
+    assert(Literal(Array("1", "2", "3")) ==
+      Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
+  }
 }

From 73f562668adbc0d08d44ed8d1b25df0c5cb6c55a Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Wed, 23 Dec 2020 00:22:42 +0900
Subject: [PATCH 1232/1280] [BUILD][MINOR] Do not publish snapshots from forks

### What changes were proposed in this pull request?
The GitHub workflow `Publish Snapshot` publishes master and 3.1 branch via Nexus. For this, the workflow uses `secrets.NEXUS_USER` and `secrets.NEXUS_PW` secrets. These are not available in forks where this workflow fails every day:

- https://github.com/G-Research/spark/actions/runs/431626797
- https://github.com/G-Research/spark/actions/runs/433153049
- https://github.com/G-Research/spark/actions/runs/434680048
- https://github.com/G-Research/spark/actions/runs/436958780

### Why are the changes needed?
Avoid attempting to publish snapshots from forked repositories.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Code review only.

Closes #30884 from EnricoMi/branch-do-not-publish-snapshots-from-forks.

Authored-by: Enrico Minack <github@enrico.minack.dev>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 1d450250eb1db7e4f40451f369db830a8f01ec15)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .github/workflows/publish_snapshot.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml
index 0d9ee17f4c278..5d4fdf4c97ded 100644
--- a/.github/workflows/publish_snapshot.yml
+++ b/.github/workflows/publish_snapshot.yml
@@ -7,6 +7,7 @@ on:
 
 jobs:
   publish-snapshot:
+    if: github.repository == 'apache/spark'
     runs-on: ubuntu-latest
     steps:
     - name: Checkout Spark repository

From 4299a4819dd537448f2b721d14fc268250e86b12 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Wed, 23 Dec 2020 10:46:25 +0900
Subject: [PATCH 1233/1280] Revert "[SPARK-33860][SQL] Make
 CatalystTypeConverters.convertToCatalyst match special Array value"

This reverts commit 7af54fd76ade27b1cabe90bce1b959c2224f31ad.
---
 .../spark/sql/catalyst/CatalystTypeConverters.scala   |  4 +---
 .../catalyst/expressions/LiteralExpressionSuite.scala | 11 -----------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
index 2b467243d12e3..34d2f45e715e9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
@@ -456,9 +456,7 @@ object CatalystTypeConverters {
     case d: JavaBigDecimal => new DecimalConverter(DecimalType(d.precision, d.scale)).toCatalyst(d)
     case seq: Seq[Any] => new GenericArrayData(seq.map(convertToCatalyst).toArray)
     case r: Row => InternalRow(r.toSeq.map(convertToCatalyst): _*)
-    case arr: Array[Byte] => arr
-    case arr: Array[Char] => StringConverter.toCatalyst(arr)
-    case arr: Array[_] => new GenericArrayData(arr.map(convertToCatalyst))
+    case arr: Array[Any] => new GenericArrayData(arr.map(convertToCatalyst))
     case map: Map[_, _] =>
       ArrayBasedMapData(
         map,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index 794e8c0656d30..bb86135021b91 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -317,15 +317,4 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
       assert(literalStr === expected)
     }
   }
-
-  test("SPARK-33860: Make CatalystTypeConverters.convertToCatalyst match special Array value") {
-    assert(Literal(Array(1, 2, 3)) == Literal.create(Array(1, 2, 3), ArrayType(IntegerType)))
-    assert(Literal(Array(1L, 2L, 3L)) == Literal.create(Array(1L, 2L, 3L), ArrayType(LongType)))
-    assert(Literal(Array(1D, 2D, 3D)) == Literal.create(Array(1D, 2D, 3D), ArrayType(DoubleType)))
-    assert(Literal("123") == Literal.create(Array('1', '2', '3'), StringType))
-    assert(Literal(Array(1.toByte, 2.toByte, 3.toByte)) ==
-      Literal.create(Array(1.toByte, 2.toByte, 3.toByte), BinaryType))
-    assert(Literal(Array("1", "2", "3")) ==
-      Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
-  }
 }

From 8c4e166f7e993bec36c5983cc96d57bbdc3df3d6 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Wed, 23 Dec 2020 23:43:21 +0900
Subject: [PATCH 1234/1280] [SPARK-33891][DOCS][CORE] Update dynamic allocation
 related documents

### What changes were proposed in this pull request?

This PR aims to update the followings.
- Remove the outdated requirement for `spark.shuffle.service.enabled` in `configuration.md`
- Dynamic allocation section in `job-scheduling.md`

### Why are the changes needed?

To make the document up-to-date.

### Does this PR introduce _any_ user-facing change?

No, it's a documentation update.

### How was this patch tested?

Manual.

**BEFORE**
![Screen Shot 2020-12-23 at 2 22 04 AM](https://user-images.githubusercontent.com/9700541/102986441-ae647f80-44c5-11eb-97a3-87c2d368952a.png)
![Screen Shot 2020-12-23 at 2 22 34 AM](https://user-images.githubusercontent.com/9700541/102986473-bcb29b80-44c5-11eb-8eae-6802001c6dfa.png)

**AFTER**
![Screen Shot 2020-12-23 at 2 25 36 AM](https://user-images.githubusercontent.com/9700541/102986767-2df24e80-44c6-11eb-8540-e74856a4c313.png)
![Screen Shot 2020-12-23 at 2 21 13 AM](https://user-images.githubusercontent.com/9700541/102986366-8e34c080-44c5-11eb-8054-1efd07c9458c.png)

Closes #30906 from dongjoon-hyun/SPARK-33891.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 47d1aa4e93f668774fd0b16c780d3b1f6200bcd8)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/configuration.md  |  3 +--
 docs/job-scheduling.md | 17 +++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/configuration.md b/docs/configuration.md
index ebfee12122e43..a9daabe366fd8 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -917,8 +917,7 @@ Apart from these, the following properties are also available, and may be useful
   <td>false</td>
   <td>
     Enables the external shuffle service. This service preserves the shuffle files written by
-    executors so the executors can be safely removed. This must be enabled if
-    <code>spark.dynamicAllocation.enabled</code> is "true". The external shuffle service
+    executors so the executors can be safely removed. The external shuffle service
     must be set up in order to enable it. See
     <a href="job-scheduling.html#configuration-and-setup">dynamic allocation
     configuration and setup documentation</a> for more information.
diff --git a/docs/job-scheduling.md b/docs/job-scheduling.md
index 5c19c77f37a81..2164b2eb2555c 100644
--- a/docs/job-scheduling.md
+++ b/docs/job-scheduling.md
@@ -79,18 +79,19 @@ are no longer used and request them again later when there is demand. This featu
 useful if multiple applications share resources in your Spark cluster.
 
 This feature is disabled by default and available on all coarse-grained cluster managers, i.e.
-[standalone mode](spark-standalone.html), [YARN mode](running-on-yarn.html), and
-[Mesos coarse-grained mode](running-on-mesos.html#mesos-run-modes).
+[standalone mode](spark-standalone.html), [YARN mode](running-on-yarn.html),
+[Mesos coarse-grained mode](running-on-mesos.html#mesos-run-modes) and [K8s mode](running-on-kubernetes.html).
+
 
 ### Configuration and Setup
 
-There are two requirements for using this feature. First, your application must set
-`spark.dynamicAllocation.enabled` to `true`. Second, you must set up an *external shuffle service*
-on each worker node in the same cluster and set `spark.shuffle.service.enabled` to true in your
-application. The purpose of the external shuffle service is to allow executors to be removed
+There are two ways for using this feature.
+First, your application must set both `spark.dynamicAllocation.enabled` and `spark.dynamicAllocation.shuffleTracking.enabled` to `true`.
+Second, your application must set both `spark.dynamicAllocation.enabled` and `spark.shuffle.service.enabled` to `true`
+after you set up an *external shuffle service* on each worker node in the same cluster.
+The purpose of the shuffle tracking or the external shuffle service is to allow executors to be removed
 without deleting shuffle files written by them (more detail described
-[below](job-scheduling.html#graceful-decommission-of-executors)). The way to set up this service
-varies across cluster managers:
+[below](job-scheduling.html#graceful-decommission-of-executors)). While it is simple to enable shuffle tracking, the way to set up the external shuffle service varies across cluster managers:
 
 In standalone mode, simply start your workers with `spark.shuffle.service.enabled` set to `true`.
 

From 83adba71febcd850564e1342ebf4562ffc059c13 Mon Sep 17 00:00:00 2001
From: Takuya UESHIN <ueshin@databricks.com>
Date: Wed, 23 Dec 2020 14:48:01 -0800
Subject: [PATCH 1235/1280] [SPARK-33277][PYSPARK][SQL] Use
 ContextAwareIterator to stop consuming after the task ends

### What changes were proposed in this pull request?

This is a retry of #30177.

This is not a complete fix, but it would take long time to complete (#30242).
As discussed offline, at least using `ContextAwareIterator` should be helpful enough for many cases.

As the Python evaluation consumes the parent iterator in a separate thread, it could consume more data from the parent even after the task ends and the parent is closed. Thus, we should use `ContextAwareIterator` to stop consuming after the task ends.

### Why are the changes needed?

Python/Pandas UDF right after off-heap vectorized reader could cause executor crash.

E.g.,:

```py
spark.range(0, 100000, 1, 1).write.parquet(path)

spark.conf.set("spark.sql.columnVector.offheap.enabled", True)

def f(x):
    return 0

fUdf = udf(f, LongType())

spark.read.parquet(path).select(fUdf('id')).head()
```

This is because, the Python evaluation consumes the parent iterator in a separate thread and it consumes more data from the parent even after the task ends and the parent is closed. If an off-heap column vector exists in the parent iterator, it could cause segmentation fault which crashes the executor.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Added tests, and manually.

Closes #30899 from ueshin/issues/SPARK-33277/context_aware_iterator.

Authored-by: Takuya UESHIN <ueshin@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 5c9b421c3711ba373b4d5cbbd83a8ece91291ed0)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../apache/spark/ContextAwareIterator.scala   | 40 +++++++++++++++++++
 .../sql/execution/python/EvalPythonExec.scala |  5 ++-
 .../execution/python/MapInPandasExec.scala    |  9 +++--
 3 files changed, 48 insertions(+), 6 deletions(-)
 create mode 100644 core/src/main/scala/org/apache/spark/ContextAwareIterator.scala

diff --git a/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala b/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala
new file mode 100644
index 0000000000000..c4d0dd8aceab0
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/ContextAwareIterator.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark
+
+import org.apache.spark.annotation.DeveloperApi
+
+/**
+ * :: DeveloperApi ::
+ * A TaskContext aware iterator.
+ *
+ * As the Python evaluation consumes the parent iterator in a separate thread,
+ * it could consume more data from the parent even after the task ends and the parent is closed.
+ * If an off-heap access exists in the parent iterator, it could cause segmentation fault
+ * which crashes the executor.
+ * Thus, we should use [[ContextAwareIterator]] to stop consuming after the task ends.
+ */
+@DeveloperApi
+class ContextAwareIterator[+T](val context: TaskContext, val delegate: Iterator[T])
+  extends Iterator[T] {
+
+  override def hasNext: Boolean =
+    !context.isCompleted() && !context.isInterrupted() && delegate.hasNext
+
+  override def next(): T = delegate.next()
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
index a0f23e925d237..57ccdb194b1f5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/EvalPythonExec.scala
@@ -21,7 +21,7 @@ import java.io.File
 
 import scala.collection.mutable.ArrayBuffer
 
-import org.apache.spark.{SparkEnv, TaskContext}
+import org.apache.spark.{ContextAwareIterator, SparkEnv, TaskContext}
 import org.apache.spark.api.python.ChainedPythonFunctions
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -88,6 +88,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
 
     inputRDD.mapPartitions { iter =>
       val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(context, iter)
 
       // The queue used to buffer input rows so we can drain it to
       // combine input with output from Python.
@@ -119,7 +120,7 @@ abstract class EvalPythonExec(udfs: Seq[PythonUDF], resultAttrs: Seq[Attribute],
       })
 
       // Add rows to queue to join later with the result.
-      val projectedRowIter = iter.map { inputRow =>
+      val projectedRowIter = contextAwareIterator.map { inputRow =>
         queue.add(inputRow.asInstanceOf[UnsafeRow])
         projection(inputRow)
       }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
index 2bb808119c0ae..71f51f1abc6f5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/MapInPandasExec.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.python
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.TaskContext
+import org.apache.spark.{ContextAwareIterator, TaskContext}
 import org.apache.spark.api.python.{ChainedPythonFunctions, PythonEvalType}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
@@ -61,16 +61,17 @@ case class MapInPandasExec(
       val pythonRunnerConf = ArrowUtils.getPythonRunnerConfMap(conf)
       val outputTypes = child.schema
 
+      val context = TaskContext.get()
+      val contextAwareIterator = new ContextAwareIterator(context, inputIter)
+
       // Here we wrap it via another row so that Python sides understand it
       // as a DataFrame.
-      val wrappedIter = inputIter.map(InternalRow(_))
+      val wrappedIter = contextAwareIterator.map(InternalRow(_))
 
       // DO NOT use iter.grouped(). See BatchIterator.
       val batchIter =
         if (batchSize > 0) new BatchIterator(wrappedIter, batchSize) else Iterator(wrappedIter)
 
-      val context = TaskContext.get()
-
       val columnarBatchIter = new ArrowPythonRunner(
         chainedFunc,
         PythonEvalType.SQL_MAP_PANDAS_ITER_UDF,

From 1445129fd5c65cf390b158d4b501e5973bd7a6f3 Mon Sep 17 00:00:00 2001
From: sychen <sychen@ctrip.com>
Date: Fri, 25 Dec 2020 00:54:26 +0900
Subject: [PATCH 1236/1280] [SPARK-33900][WEBUI] Show shuffle read size /
 records correctly when only remotebytesread is available

### What changes were proposed in this pull request?
Shuffle Read Size / Records can also be displayed in remoteBytesRead>0 localBytesRead=0.

current:
![image](https://user-images.githubusercontent.com/3898450/103079421-c4ca2280-460e-11eb-9e2f-49d35b5d324d.png)
fix:
![image](https://user-images.githubusercontent.com/3898450/103079439-cc89c700-460e-11eb-9a41-6b2882980d11.png)

### Why are the changes needed?
At present, the page only displays the data of Shuffle Read Size / Records when localBytesRead>0.
When there is only remote reading, metrics cannot be seen on the stage page.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
manual test

Closes #30916 from cxzl25/SPARK-33900.

Authored-by: sychen <sychen@ctrip.com>
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
(cherry picked from commit 700f5ab65c1c84522302ce92d176adf229c34daa)
Signed-off-by: Kousuke Saruta <sarutak@oss.nttdata.com>
---
 .../src/main/resources/org/apache/spark/ui/static/stagepage.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
index b2964957b8334..67d6d741e5537 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js
@@ -821,7 +821,8 @@ $(document).ready(function () {
                         },
                         {
                             data : function (row, type) {
-                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
+                                if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics &&
+                                    (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0 || row.taskMetrics.shuffleReadMetrics.remoteBytesRead > 0)) {
                                     var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
                                     if (type === 'display') {
                                         return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;

From 65dd1d0c8a6bf603d6884d25d474d4e7c203283c Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 27 Dec 2020 17:59:09 +0900
Subject: [PATCH 1237/1280] [SPARK-33911][SQL][DOCS][3.0] Update the SQL
 migration guide about changes in `HiveClientImpl`

### What changes were proposed in this pull request?
Update the SQL migration guide about the changes made by:
- https://github.com/apache/spark/pull/30778
- https://github.com/apache/spark/pull/30711

### Why are the changes needed?
To inform users about the recent changes in the upcoming releases.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
N/A

Closes #30932 from MaxGekk/sql-migr-guide-hiveclientimpl-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-migration-guide.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md
index 85d7073ff08d4..fe2f62978cb62 100644
--- a/docs/sql-migration-guide.md
+++ b/docs/sql-migration-guide.md
@@ -26,6 +26,10 @@ license: |
 
   - In Spark 3.0.2, `IllegalArgumentException` is returned for the incomplete interval literals, e.g. `INTERVAL '1'`, `INTERVAL '1 DAY 2'`, which are invalid. In Spark 3.0.1, these literals result in `NULL`s.
 
+  - In Spark 3.0.2, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
+
 ## Upgrading from Spark SQL 3.0 to 3.0.1
 
 - In Spark 3.0, JSON datasource and JSON function `schema_of_json` infer TimestampType from string values if they match to the pattern defined by the JSON option `timestampFormat`. Since version 3.0.1, the timestamp type inference is disabled by default. Set the JSON option `inferTimestamp` to `true` to enable such type inference.
@@ -219,6 +223,12 @@ license: |
 
     * The decimal string representation can be different between Hive 1.2 and Hive 2.3 when using `TRANSFORM` operator in SQL for script transformation, which depends on hive's behavior. In Hive 1.2, the string representation omits trailing zeroes. But in Hive 2.3, it is always padded to 18 digits with trailing zeroes if necessary.
 
+## Upgrading from Spark SQL 2.4.7 to 2.4.8
+
+  - In Spark 2.4.8, `AnalysisException` is replaced by its sub-classes that are thrown for tables from Hive external catalog in the following situations:
+    * `ALTER TABLE .. ADD PARTITION` throws `PartitionsAlreadyExistException` if new partition exists already
+    * `ALTER TABLE .. DROP PARTITION` throws `NoSuchPartitionsException` for not existing partitions
+    
 ## Upgrading from Spark SQL 2.4.5 to 2.4.6
 
   - In Spark 2.4.6, the `RESET` command does not reset the static SQL configuration values to the default. It only clears the runtime SQL configuration values.

From 91a2260890683a7d0dbc745fab087bf265e68243 Mon Sep 17 00:00:00 2001
From: Liang-Chi Hsieh <viirya@gmail.com>
Date: Wed, 30 Dec 2020 16:15:41 +0900
Subject: [PATCH 1238/1280] [MINOR][SS] Call fetchEarliestOffsets when it is
 necessary

### What changes were proposed in this pull request?

This minor patch changes two variables where calling `fetchEarliestOffsets` to `lazy` because these values are not always necessary.

### Why are the changes needed?

To avoid unnecessary Kafka RPC calls.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Unit test.

Closes #30969 from viirya/ss-minor3.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 4a669f583089fc704cdc46cff8f1680470a068ee)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala   | 2 +-
 .../main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
index 6599e7e0fe707..e60054dba9b01 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchStream.scala
@@ -167,7 +167,7 @@ private[kafka010] class KafkaMicroBatchStream(
       limit: Long,
       from: PartitionOffsetMap,
       until: PartitionOffsetMap): PartitionOffsetMap = {
-    val fromNew = kafkaOffsetReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
+    lazy val fromNew = kafkaOffsetReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
     val sizes = until.flatMap {
       case (tp, end) =>
         // If begin isn't defined, something's wrong, but let alert logic in getBatch handle it
diff --git a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
index 57879c7ca31cf..a1397de9c7a04 100644
--- a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
+++ b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaSource.scala
@@ -159,7 +159,7 @@ private[kafka010] class KafkaSource(
       limit: Long,
       from: Map[TopicPartition, Long],
       until: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {
-    val fromNew = kafkaReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
+    lazy val fromNew = kafkaReader.fetchEarliestOffsets(until.keySet.diff(from.keySet).toSeq)
     val sizes = until.flatMap {
       case (tp, end) =>
         // If begin isn't defined, something's wrong, but let alert logic in getBatch handle it

From b156c1f9073cdb27e1ca8d7752aa0793d160ad0b Mon Sep 17 00:00:00 2001
From: "Pradyumn Agrawal (pradyumn.ag)" <pradyumn.ag@media.net>
Date: Wed, 30 Dec 2020 17:25:46 -0800
Subject: [PATCH 1239/1280] [SPARK-33942][DOCS] Remove `hiveClientCalls.count`
 in `CodeGenerator` metrics docs

### What changes were proposed in this pull request?
Removed the **hiveClientCalls.count** in CodeGenerator metrics in Component instance = Executor

### Why are the changes needed?
Wrong information regarding metrics was being displayed on Monitoring Documentation. I had added referred documentation for adding metrics logging in Graphite. This metric was not being reported. I had to check if the issue was at my application end or spark code or documentation. Documentation had the wrong info.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Manual, checked it on my forked repository feature branch [SPARK-33942](https://github.com/coderbond007/spark/blob/SPARK-33942/docs/monitoring.md)

Closes #30976 from coderbond007/SPARK-33942.

Authored-by: Pradyumn Agrawal (pradyumn.ag) <pradyumn.ag@media.net>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 13e8c2840969a17d5ba113686501abd3c23e3c23)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/monitoring.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index 8471417184974..ac6b693c0fc82 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -1228,7 +1228,6 @@ when running in local mode.
   - compilationTime (histogram)
   - generatedClassSize (histogram)
   - generatedMethodSize (histogram)
-  - hiveClientCalls.count
   - sourceCodeSize (histogram)
 
 - namespace=plugin.\<Plugin Class Name>

From 39867a8faff28c68b7f3754c8cd015af457e1145 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Fri, 1 Jan 2021 02:00:45 -0800
Subject: [PATCH 1240/1280] [SPARK-33931][INFRA][3.0] Recover GitHub Action
 `build_and_test` job

### What changes were proposed in this pull request?

This is a backport of https://github.com/apache/spark/pull/30959 .
This PR aims to recover GitHub Action `build_and_test` job.

### Why are the changes needed?

Currently, `build_and_test` job fails to start because of  the following in master/branch-3.1 at least.
```
r-lib/actions/setup-rv1 is not allowed to be used in apache/spark.
Actions in this workflow must be: created by GitHub, verified in the GitHub Marketplace,
within a repository owned by apache or match the following:
adoptopenjdk/*, apache/*, gradle/wrapper-validation-action.
```
- https://github.com/apache/spark/actions/runs/449826457

![Screen Shot 2020-12-28 at 10 06 11 PM](https://user-images.githubusercontent.com/9700541/103262174-f1f13a80-4958-11eb-8ceb-631527155775.png)

### Does this PR introduce _any_ user-facing change?

No. This is a test infra.

### How was this patch tested?

To check GitHub Action `build_and_test` job on this PR.

Closes #30986 from dongjoon-hyun/SPARK-33931-3.0.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .github/workflows/build_and_test.yml | 32 +++++++++++-----------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index d80b132882a1a..d1c404e6fe35a 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -273,6 +273,8 @@ jobs:
   lint:
     name: Linters, licenses, dependencies and documentation generation
     runs-on: ubuntu-20.04
+    container:
+      image: dongjoon/apache-spark-github-action-image:20201025
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v2
@@ -303,10 +305,6 @@ jobs:
         key: docs-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
           docs-maven-
-    - name: Install Java 8
-      uses: actions/setup-java@v1
-      with:
-        java-version: 8
     - name: Install Python 3.6
       uses: actions/setup-python@v2
       with:
@@ -314,27 +312,21 @@ jobs:
         architecture: x64
     - name: Install Python linter dependencies
       run: |
-        pip3 install flake8 sphinx numpy
-    - name: Install R 4.0
-      uses: r-lib/actions/setup-r@v1
-      with:
-        r-version: 4.0
+        python3.6 -m pip install install flake8 sphinx numpy
     - name: Install R linter dependencies and SparkR
       run: |
-        sudo apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev
-        sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
-        sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
+        apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev
+        Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
+        Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
         ./R/install-dev.sh
-    - name: Install Ruby 2.7 for documentation generation
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.7
     - name: Install dependencies for documentation generation
       run: |
-        sudo apt-get install -y libcurl4-openssl-dev pandoc
-        pip install sphinx mkdocs numpy
+        apt-get install -y libcurl4-openssl-dev pandoc
+        python3.6 -m pip install sphinx mkdocs numpy
+        apt-get update -y
+        apt-get install -y ruby ruby-dev
         gem install jekyll jekyll-redirect-from rouge
-        sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
+        Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
     - name: Scala linter
       run: ./dev/lint-scala
     - name: Java linter
@@ -350,6 +342,8 @@ jobs:
     - name: Run documentation build
       run: |
         cd docs
+        export LC_ALL=C.UTF-8
+        export LANG=C.UTF-8
         jekyll build
 
   java-11:

From dda431ad974802a8f6413c9aac267373634da0fe Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Sun, 3 Jan 2021 11:23:46 +0900
Subject: [PATCH 1241/1280] [SPARK-33963][SQL] Canonicalize `HiveTableRelation`
 w/o table stats

### What changes were proposed in this pull request?
Skip table stats in canonicalizing of `HiveTableRelation`.

### Why are the changes needed?
The changes fix a regression comparing to Spark 3.0, see SPARK-33963.

### Does this PR introduce _any_ user-facing change?
Yes. After changes Spark behaves as in the version 3.0.1.

### How was this patch tested?
By running new UT:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *CachedTableSuite"
```

Closes #30995 from MaxGekk/fix-caching-hive-table.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit fc7d0165d29e04a8e78577c853a701bdd8a2af4a)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/sql/catalyst/catalog/interface.scala  |  3 ++-
 .../org/apache/spark/sql/hive/CachedTableSuite.scala   | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 4e63ee7428d72..7d9b57a3f171f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -676,7 +676,8 @@ case class HiveTableRelation(
     },
     partitionCols = partitionCols.zipWithIndex.map {
       case (attr, index) => attr.withExprId(ExprId(index + dataCols.length))
-    }
+    },
+    tableStats = None
   )
 
   override def computeStats(): Statistics = {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index fc793534641df..6c0ab2f84498d 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -429,4 +429,14 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       }
     }
   }
+
+  test("SPARK-33963: do not use table stats while looking in table cache") {
+    val t = "table_on_test"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (col int)")
+      assert(!spark.catalog.isCached(t))
+      sql(s"CACHE TABLE $t")
+      assert(spark.catalog.isCached(t))
+    }
+  }
 }

From 9f1bf4e47c4978be7d55cfadb7da6b7863942bc8 Mon Sep 17 00:00:00 2001
From: Ruifeng Zheng <ruifengz@foxmail.com>
Date: Sun, 3 Jan 2021 11:52:46 -0600
Subject: [PATCH 1242/1280] [SPARK-33398] Fix loading tree models prior to
 Spark 3.0

### What changes were proposed in this pull request?
In https://github.com/apache/spark/pull/21632/files#diff-0fdae8a6782091746ed20ea43f77b639f9c6a5f072dd2f600fcf9a7b37db4f47, a new field `rawCount` was added into `NodeData`, which cause that a tree model trained in 2.4 can not be loaded in 3.0/3.1/master;
field `rawCount` is only used in training, and not used in `transform`/`predict`/`featureImportance`. So I just set it to -1L.

### Why are the changes needed?
to support load old tree model in 3.0/3.1/master

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
added testsuites

Closes #30889 from zhengruifeng/fix_tree_load.

Authored-by: Ruifeng Zheng <ruifengz@foxmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit 6b7527e381591bcd51be205853aea3e349893139)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../org/apache/spark/ml/tree/treeModels.scala |  48 ++++++++++++------
 .../ml-models/dtc-2.4.7/data/._SUCCESS.crc    | Bin 0 -> 8 bytes
 ...-894c-ca4eac67c690-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 .../ml-models/dtc-2.4.7/data/_SUCCESS         |   0
 ...406c-894c-ca4eac67c690-c000.snappy.parquet | Bin 0 -> 3242 bytes
 .../dtc-2.4.7/metadata/._SUCCESS.crc          | Bin 0 -> 8 bytes
 .../dtc-2.4.7/metadata/.part-00000.crc        | Bin 0 -> 16 bytes
 .../ml-models/dtc-2.4.7/metadata/_SUCCESS     |   0
 .../ml-models/dtc-2.4.7/metadata/part-00000   |   1 +
 .../ml-models/dtr-2.4.7/data/._SUCCESS.crc    | Bin 0 -> 8 bytes
 ...-84af-d861adcb9ca8-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 .../ml-models/dtr-2.4.7/data/_SUCCESS         |   0
 ...4b3d-84af-d861adcb9ca8-c000.snappy.parquet | Bin 0 -> 3264 bytes
 .../dtr-2.4.7/metadata/._SUCCESS.crc          | Bin 0 -> 8 bytes
 .../dtr-2.4.7/metadata/.part-00000.crc        | Bin 0 -> 12 bytes
 .../ml-models/dtr-2.4.7/metadata/_SUCCESS     |   0
 .../ml-models/dtr-2.4.7/metadata/part-00000   |   1 +
 .../ml-models/gbtc-2.4.7/data/._SUCCESS.crc   | Bin 0 -> 8 bytes
 ...-91c0-6da8cc01fb43-c000.snappy.parquet.crc | Bin 0 -> 44 bytes
 .../ml-models/gbtc-2.4.7/data/_SUCCESS        |   0
 ...41c7-91c0-6da8cc01fb43-c000.snappy.parquet | Bin 0 -> 4542 bytes
 .../gbtc-2.4.7/metadata/._SUCCESS.crc         | Bin 0 -> 8 bytes
 .../gbtc-2.4.7/metadata/.part-00000.crc       | Bin 0 -> 16 bytes
 .../ml-models/gbtc-2.4.7/metadata/_SUCCESS    |   0
 .../ml-models/gbtc-2.4.7/metadata/part-00000  |   1 +
 .../gbtc-2.4.7/treesMetadata/._SUCCESS.crc    | Bin 0 -> 8 bytes
 ...-813c-ddc394101e21-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 .../gbtc-2.4.7/treesMetadata/_SUCCESS         |   0
 ...4a90-813c-ddc394101e21-c000.snappy.parquet | Bin 0 -> 3075 bytes
 .../ml-models/gbtr-2.4.7/data/._SUCCESS.crc   | Bin 0 -> 8 bytes
 ...-9aab-639288bfae6d-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
 .../ml-models/gbtr-2.4.7/data/_SUCCESS        |   0
 ...4511-9aab-639288bfae6d-c000.snappy.parquet | Bin 0 -> 3740 bytes
 .../gbtr-2.4.7/metadata/._SUCCESS.crc         | Bin 0 -> 8 bytes
 .../gbtr-2.4.7/metadata/.part-00000.crc       | Bin 0 -> 16 bytes
 .../ml-models/gbtr-2.4.7/metadata/_SUCCESS    |   0
 .../ml-models/gbtr-2.4.7/metadata/part-00000  |   1 +
 .../gbtr-2.4.7/treesMetadata/._SUCCESS.crc    | Bin 0 -> 8 bytes
 ...-ad9c-4be239c2215a-c000.snappy.parquet.crc | Bin 0 -> 32 bytes
 .../gbtr-2.4.7/treesMetadata/_SUCCESS         |   0
 ...4fd8-ad9c-4be239c2215a-c000.snappy.parquet | Bin 0 -> 3038 bytes
 .../ml-models/rfc-2.4.7/data/._SUCCESS.crc    | Bin 0 -> 8 bytes
 ...-b112-25b4b11c9009-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
 .../ml-models/rfc-2.4.7/data/_SUCCESS         |   0
 ...4485-b112-25b4b11c9009-c000.snappy.parquet | Bin 0 -> 3836 bytes
 .../rfc-2.4.7/metadata/._SUCCESS.crc          | Bin 0 -> 8 bytes
 .../rfc-2.4.7/metadata/.part-00000.crc        | Bin 0 -> 16 bytes
 .../ml-models/rfc-2.4.7/metadata/_SUCCESS     |   0
 .../ml-models/rfc-2.4.7/metadata/part-00000   |   1 +
 .../rfc-2.4.7/treesMetadata/._SUCCESS.crc     | Bin 0 -> 8 bytes
 ...-a823-70c7afdcbdc5-c000.snappy.parquet.crc | Bin 0 -> 36 bytes
 .../rfc-2.4.7/treesMetadata/_SUCCESS          |   0
 ...4c4e-a823-70c7afdcbdc5-c000.snappy.parquet | Bin 0 -> 3391 bytes
 .../ml-models/rfr-2.4.7/data/._SUCCESS.crc    | Bin 0 -> 8 bytes
 ...-b681-981caaeca996-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
 .../ml-models/rfr-2.4.7/data/_SUCCESS         |   0
 ...40fc-b681-981caaeca996-c000.snappy.parquet | Bin 0 -> 3797 bytes
 .../rfr-2.4.7/metadata/._SUCCESS.crc          | Bin 0 -> 8 bytes
 .../rfr-2.4.7/metadata/.part-00000.crc        | Bin 0 -> 16 bytes
 .../ml-models/rfr-2.4.7/metadata/_SUCCESS     |   0
 .../ml-models/rfr-2.4.7/metadata/part-00000   |   1 +
 .../rfr-2.4.7/treesMetadata/._SUCCESS.crc     | Bin 0 -> 8 bytes
 ...-9b86-d95edaabcde8-c000.snappy.parquet.crc | Bin 0 -> 32 bytes
 .../rfr-2.4.7/treesMetadata/_SUCCESS          |   0
 ...447a-9b86-d95edaabcde8-c000.snappy.parquet | Bin 0 -> 3055 bytes
 .../DecisionTreeClassifierSuite.scala         |  12 +++++
 .../classification/GBTClassifierSuite.scala   |  14 +++++
 .../MultilayerPerceptronClassifierSuite.scala |   2 +-
 .../RandomForestClassifierSuite.scala         |  16 +++++-
 .../spark/ml/feature/HashingTFSuite.scala     |   2 +-
 .../spark/ml/feature/StringIndexerSuite.scala |   2 +-
 .../DecisionTreeRegressorSuite.scala          |  16 +++++-
 .../ml/regression/GBTRegressorSuite.scala     |  12 +++++
 .../RandomForestRegressorSuite.scala          |  12 +++++
 74 files changed, 122 insertions(+), 20 deletions(-)
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS
 create mode 100644 mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet

diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 162641f605264..67b9166a0f44d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -31,8 +31,10 @@ import org.apache.spark.ml.util.{DefaultParamsReader, DefaultParamsWriter}
 import org.apache.spark.ml.util.DefaultParamsReader.Metadata
 import org.apache.spark.mllib.tree.impurity.ImpurityCalculator
 import org.apache.spark.mllib.tree.model.{DecisionTreeModel => OldDecisionTreeModel}
-import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.{Dataset, SparkSession}
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.functions.{col, lit, struct}
+import org.apache.spark.sql.types.StructType
+import org.apache.spark.util.VersionUtils
 import org.apache.spark.util.collection.OpenHashMap
 
 /**
@@ -401,8 +403,13 @@ private[ml] object DecisionTreeModelReadWrite {
     }
 
     val dataPath = new Path(path, "data").toString
-    val data = sparkSession.read.parquet(dataPath).as[NodeData]
-    buildTreeFromNodes(data.collect(), impurityType)
+    var df = sparkSession.read.parquet(dataPath)
+    val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+    if (major.toInt < 3) {
+      df = df.withColumn("rawCount", lit(-1L))
+    }
+
+    buildTreeFromNodes(df.as[NodeData].collect(), impurityType)
   }
 
   /**
@@ -497,25 +504,36 @@ private[ml] object EnsembleModelReadWrite {
     }
 
     val treesMetadataPath = new Path(path, "treesMetadata").toString
-    val treesMetadataRDD: RDD[(Int, (Metadata, Double))] = sql.read.parquet(treesMetadataPath)
-      .select("treeID", "metadata", "weights").as[(Int, String, Double)].rdd.map {
-      case (treeID: Int, json: String, weights: Double) =>
+    val treesMetadataRDD = sql.read.parquet(treesMetadataPath)
+      .select("treeID", "metadata", "weights")
+      .as[(Int, String, Double)].rdd
+      .map { case (treeID: Int, json: String, weights: Double) =>
         treeID -> ((DefaultParamsReader.parseMetadata(json, treeClassName), weights))
-    }
+      }
 
     val treesMetadataWeights = treesMetadataRDD.sortByKey().values.collect()
     val treesMetadata = treesMetadataWeights.map(_._1)
     val treesWeights = treesMetadataWeights.map(_._2)
 
     val dataPath = new Path(path, "data").toString
-    val nodeData: Dataset[EnsembleNodeData] =
-      sql.read.parquet(dataPath).as[EnsembleNodeData]
-    val rootNodesRDD: RDD[(Int, Node)] =
-      nodeData.rdd.map(d => (d.treeID, d.nodeData)).groupByKey().map {
-        case (treeID: Int, nodeData: Iterable[NodeData]) =>
-          treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
+    var df = sql.read.parquet(dataPath)
+    val (major, minor) = VersionUtils.majorMinorVersion(metadata.sparkVersion)
+    if (major.toInt < 3) {
+      val newNodeDataCol = df.schema("nodeData").dataType match {
+        case StructType(fields) =>
+          val cols = fields.map(f => col(s"nodeData.${f.name}")) :+ lit(-1L).as("rawCount")
+          struct(cols: _*)
+      }
+      df = df.withColumn("nodeData", newNodeDataCol)
+    }
+
+    val rootNodesRDD = df.as[EnsembleNodeData].rdd
+      .map(d => (d.treeID, d.nodeData))
+      .groupByKey()
+      .map { case (treeID: Int, nodeData: Iterable[NodeData]) =>
+        treeID -> DecisionTreeModelReadWrite.buildTreeFromNodes(nodeData.toArray, impurityType)
       }
-    val rootNodes: Array[Node] = rootNodesRDD.sortByKey().values.collect()
+    val rootNodes = rootNodesRDD.sortByKey().values.collect()
     (metadata, treesMetadata.zip(rootNodes), treesWeights)
   }
 
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/.part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3ac562a8b2b843f0df27c8239a8aa4e4e7102065
GIT binary patch
literal 36
scmYc;N@ieSU}9jt&;0(&Q}%+r><dq4D=gCAdHdGOKu*_BIk%eY0R2-C8UO$Q

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet b/mllib/src/test/resources/ml-models/dtc-2.4.7/data/part-00000-bd7ae42f-c890-406c-894c-ca4eac67c690-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..09c38d602c10529ce5d11d0012f25259b3dd8ae9
GIT binary patch
literal 3242
zcmcInO=u%k9Dg&JPN%b}^pH2qK%`~dhOERP-F~JNTUnG<qE_9TuFItJ(hN>Mc4nfM
zQYa$gLG~a$_OOSth!h0zuy}C6lLrrrcwQF|9y}G#ivHi5?<Q%xsG(`{|G)QszyH_!
zO}qKRn{yc9GOkpx3|}NH6;(k%2&wcJQcyW%fum5TZVui&ymRoT`>=p#@Z~yufHk!S
z9(bLs2cNKvu!v>KAOs9juyV4lhTj;|5HIY@NnB2ZgH^mx2N)nw-}&v2>7Tl{ep!F_
z*S~(Szq^+vl5jpv6z$9s4SIn{$n&`)9+)=qOLZD6LS;ymllEWS;Kn7iU!Ra9$ZBi~
z+~KRf0l7VkjS4QnK@p3^kZ8ZYk)>n@GG6fDO<KXRFL_$b0-qa`K7REXR?c*`_iDxG
z7p5Pb%J$>|>sKb*XlF!y@&Fnr4SIns&Q`_6u&((^)Yre^HGnSAAXR~`(XmLZQD}5B
zXnw=d=xTzNSfkLshTyCOt$@{_&mtRHx|*OR795Z-vr(O9-9>m6uhN5bB;#&XTT!>}
zzOkW*<zNysUI2PpWk4v$iRddXUquLZgC8N-$woV0$1p&&xM9vyxR8#LdXp#<3m~Lo
z5GBfCUi)+Edfw}q?t8bqtrlsUcnV>D7OA{#AuPiex#tqg-tz5kCvZr1yVrMZe+Mn{
z>SScR<(s|-A&F>FPzA^Ke9Q;?$B9GQq~iycux;7^L@6d5()6#jYzLUAExY!1D|7(#
znAdY`9{|~6zDZ2K?~<207Wt4eYwWb?lkKi+6Yq8RM$08$tLs?jL_w)0JN@>x_$*Rn
zt*9NBq#_h#Rze!h9`lN<e8efAa3yK&458AB&|ruW3R}9v6|t<%3X+yC8$<e*Q@-b5
z!&-b{&Q~O@0<9Kk9jWXwhKGF25ch%M|HvsnbESto;15a;y_$nwGz9-h2=GO1CEHro
z(x{cdSWxcp%BOtkNQg4ZC=O+$;|fw1%{#S*FdmC>Mr<}x*&K_2Jt!$7gk}i#e$Y|c
zk7U>w!2zcna;1IRSz6cRrX<#|U=JtQh_D7H)0#p&oQP+I-Z`pXDZLpI;(eJSE((&U
zm4k`TOge+U%PC)QrTa2KCvrof6Khzok1w&xB2l{@AdkK>r@JdF-*E6#dVDGBZbqR-
z=vMkF988|hD|_cyVxMu!0arShgv1hsqKPHeki?YVClwj|s@iq84YOy0a2ZU)jQ5G_
zfl#SqSB(W@8L3wWc6hfP1LW>@pZJE`{lI7w-?U8MR6ifz)%~3w(N}fPcl%quUeTL2
zaV$??eS25$m~FabTi_12*&Prc;)0{ocO0|fz=o#jc%-67sQT*e-d-g>ok(edVs-oU
z<i$ZE#r+Q;m}tj1asH<3nmgbQh5JTOl{nC?ZpQ}^_g;gn!;JUR;Y0OCmw?KD%&5nh
z7p-9Y$Vf1ryba0C{WmD=@e~8Z#XDno(vPtCp#J3Xk9_jQpm8++XDc($(3twd`P@8j
zXm0Ky^_B3hKG$|<!pl7~XUtWNxtVgK4}Ut$m@74_xzeoF=C3xENONgve%`7sRhMdH
gvAR`jEZ3H+4brgYu4u<agwFUuPfZ}S3IC=118407;Q#;t

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000000000000000000000000000000..22b1eb8089ce39e1d8cd08b5e9ea7f33adda65ab
GIT binary patch
literal 16
XcmYc;N@ieSU}E^OEy8bC-p?8UDO?6D

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..ef92265f89fff
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/dtc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.DecisionTreeClassificationModel","timestamp":1608687929358,"sparkVersion":"2.4.7","uid":"dtc_bc7ad285bb73","paramMap":{},"defaultParamMap":{"impurity":"gini","maxDepth":5,"labelCol":"label","maxMemoryInMB":256,"featuresCol":"features","predictionCol":"prediction","minInfoGain":0.0,"seed":159147643,"rawPredictionCol":"rawPrediction","minInstancesPerNode":1,"cacheNodeIds":false,"probabilityCol":"probability","maxBins":32,"checkpointInterval":10},"numFeatures":692,"numClasses":2}
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/.part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..f6465e27d902b970d0c02f55a6643c503c23b6cd
GIT binary patch
literal 36
scmYc;N@ieSU}6xJ`mkO@eOl+-&f@EauS{ZgO;pQ{o*O%5t=s1W0OT|dod5s;

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet b/mllib/src/test/resources/ml-models/dtr-2.4.7/data/part-00000-39b027f0-a437-4b3d-84af-d861adcb9ca8-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..2904f8433b78f0af84b859c8c0411e40577ae440
GIT binary patch
literal 3264
zcmcIn&2QUe7=P_Ju45%t4}Fa-LR&>^l%Zgy>u0Ks6pc*<MNnXu398AdoyNlXnC$>n
zsf0LisyIv&TsTZ(2M!eoG;w4*O+8E?5E47UW$GVL^$ddNef^mxX*&T`Qu%q__x=5z
zulMJ6?WH$oFv0~~EMgHJ1T17_jzb8^<QI}qA!dPNyh_{*d|CJ=;7j&l4zJ*st8g5w
zsShuSzkS)c^rUk2nRWN)E1y?jN9^aYLW$CZk@FY@Z~3qy`%jd4fEV_0X*O|?fRiOW
zSH(PGl;1fxn*61G<EM)D<gb16htERBex_h&FH^tQXS&M*QwOhA@hD+hLMX{_*H@Aw
z+(*@vz@e)dfjg|EWOOAnb}mS=x&pk<W3`AU;53i<Y`}JXC3uQw8217OUL$o)e%~Y|
zo?sKn)qCImQGfPW%6H4z7w0B_JeKm|{r;4LodNaXeQ2xD?*%qeT$-|svwm^Y#h}}N
z!%rVNM}m|&vPOm_vWB5S73tgzMkA{cT4W7F`woKBVbMn$$Fet8JRM}R8lgoN91zdZ
z0iLA!BD{>3y&DkgR;dwm>&_c15?}D<vBwKQ&&d=BB^Vxk!=y_H!EW#)1UsQ;0!~*k
zOc5!rALSSxk4H(oQIwGd5JJ|A5+yLJ{5iIob~>8<?hR+7VKg;7hA=ygWY*LX7U4nc
z*oJOyxMsWMIRvxW>Ds2dh2~i~6Buu~n(IJFJeucZ&N3Yrv)=v@!!nvi%k?a7Q!@dG
zP^2uQ?p|(~7BCT8w9U<i?*Qne)3Hn!0O?}7ZfI`THePAz#s`F1p{F(1*lgRT;k<5N
zYuJX<Xj}RzQ4p$3tJ}OBo<)+VWR<<VkcEOw3rHc^V^$KSj~VF`CNEq(LB5nMj4#9p
zh0TA;WU;7BbAl2t8%6q^kq#Kxa520vr|XhZgjRE;jzkP8!x0-Ygngj+j~MA!CV#{N
ze!uRJtJ&{G1MnYm9zL%u^|qF_G-xF-PDo#~(k(XskP9=4D-K~K<BCw+JBaZyA7(^n
zBbLo$9<ci*Wr)xW$Y#&$DCtKuY?R<OBYnZ-_ef_+U89>4S%ZSzPtg&f4UVQYhPa;!
zXNBB3qFyn*84%)uND${afmaIN#P>`(g}%c`UorUu5uhWvA<&UEDA?UJtt{e|RS$Xi
zl{wyBQM$*#C%>Bxy4#~rLv$;C6&_@c=hb`XXkxpJbeqXP$UtI|LXpHGYd~Vs-l!yk
zUzXeUrmA%`5H6K!nEIY!J0MhY@`5_2E+YAS-wx-drGnhu>>93Wx8GNrhO6nCtI4-U
zwsY=Q$H*<`9M|q{xVd7kZW@;E<d)yw&b72ASu%BS``gSGh>u}|qt&%6t!BZ7x@I{>
zF&CibmbZ6yis9)<N+T4#-6bc_4iYHte*i&6JH!d|*KAwc0=F;R*SxBPfo`;0E{M4E
z8f5Kfyb})}syDm@ME*m@T!?wr3Wl$Y2;=G75Z&B=gF;_VAwXEX6NV@L3JVYBo<9De
zZ@v&TjOPDrWzRb_q&{;#>!%IP`VNv`_3!EnO?%3}+*32^Oi7)YD%860r^A%CRMzWD
y^>TUkLT$mQFD%T?>ZOI!LfM!vZIo+^<;7CXsOdB3mAx!NCw$Oo3ZXUl-{l{OEe~}7

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000000000000000000000000000000..bbad108766e144fcfa28032a8cab16b15aa8f5b3
GIT binary patch
literal 12
TcmYc;N@ieSU}7lORyYL!5a$BN

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..2895223cffde6
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/dtr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.DecisionTreeRegressionModel","timestamp":1608687932847,"sparkVersion":"2.4.7","uid":"dtr_c16a90fcdaf8","paramMap":{},"defaultParamMap":{"labelCol":"label","checkpointInterval":10,"minInfoGain":0.0,"maxMemoryInMB":256,"minInstancesPerNode":1,"maxBins":32,"seed":926680331,"cacheNodeIds":false,"maxDepth":5,"predictionCol":"prediction","featuresCol":"features","impurity":"variance"},"numFeatures":692}
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/.part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..13fc4ed696effdbf3b5bba54547271c8fca20e04
GIT binary patch
literal 44
zcmV+{0Mq|ta$^7h00IE8y8<UAFdxJ=g4po#7#Fn1_DTd{N#RHX{=1i5*IIT`sS*Tb
CK@p+=

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtc-2.4.7/data/part-00000-dacbde64-c861-41c7-91c0-6da8cc01fb43-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..5682038fcc7641206146fbdf497eae505c430a28
GIT binary patch
literal 4542
zcmcIoe{2)i9e-!vIp+`;L%8EiUFuzfWs2+C`C*dG+FhEYOV_jo1~gsUB~E<C)*L4}
z+fc}mGDs<iw2N3r0ks{g)@g0i3_6Wch_I0uWh(8aPD0Zdkww#tl`cf<#`uGE`@XY%
z=fn<8_v7TReeZkkeLvq{@AEEM)3DCNkc^pG(h5H$LbMgZMSw7fDP?YRGER6vGqjZ>
zv6h252(ieKHv$D^CiH150aJGq%yUF5!((Gj(~WpGUZ(@mgW%;9hykPoga$E!l!7b(
zDFZQq+y=4`#0&zSxR}a}CkDk6tkpcxMxb8WhM}AURbt!SAewhBiLF_;x8@%<1ilUH
zn6o+Gb`#>KYmYB~`s!(6$aUh%D|@Gsh$7JRB03=a>DSksXV{~HFY&!=H*J@aZ+b>n
zF@xjD5fhC@L5_eN1xb0qeZgcSg*S|xDEvG#O*oqUE$|x$3)i&3<?fP@$=rRO7AED#
z;c4-_d-w;#-x_(R#De2U6HE%?u;Lu|iXZ>=&JTOquL)0H{^z;ysbc~%5d)K}H(xmO
z%z5E{YDe&+(dUINXYT0x`X7Hhw(KVj@HsTLfg;GKea3sqVIQ^7yy4Zc7xohb`O`5Y
zM@}b=bj6XeLkE>>41#bl&b;K)IlP%c6lx)eC9RoakR@ima{cBVD^I+4&q*A?fPon&
zTbT|V!2t9Hxbv7um5|5@X}#Y>lg%lLBL|v|W;9?j(~vTXJBM1ut#5dD?fb<t@#1S+
zu2qbA#e?Xzi8l?^snkQ^WhaOVvE%5Yb*BcZQ{{i2yuR+)yF{wCX`oSTo_cZey9X*$
z9UCu&j-C99v7-sm;(aSlrx$*7J-IC{|7_#$J~o9~kT|MTZYufks)OlsmP^72Wv0iI
zt&|zPZ*qwDiJx5@zVhJRp48)xjh_#E+h%GL8^qwYDq)q^ozkCrB`RHAZsKUOn4((f
z&yyn6LjFZCn#pnaqTd&`QE6Mfn5K4}`TpjWrdqK{T=1g@zxwRrDzWTj`_LV39@cVV
zBbA~JVn2NUT?iVVGTD&Z)LXG!6cwLLA{68h!MsQ~nL)hY42;2=tHfhs_HLj#r8QQ*
z7Haa474Nn3W8D`i{j{LRn{trQ8_GpQ?-Z8|`nh-ePz&!^aJu>fee;J+8}|444<1|l
z!(S!FOLpz5T^<O(9D04!di9r6c)Q|S9wC^wpc`3S%Qo@VI_uY}@0`-Qks_uWrjz&9
z@BPhC;5|XkHsoj2_J;P~zTo@Ao|k^V=cV1g4a$}9VVekb)Mb8;_zoytxW`x6hB=^I
z@YBj(D%66dS#8KW_<eWt*!32swUt?n9j_S5vDCld%4aIPJ#&my;01|!iX-tF%!%4)
z7OKv(p11R|LqTn0p2()lIRelMFJcRr`*2cmeK0p-j>O!E`3-YnA$o@Sa_HI!CeJJ8
zw;<>MOvV;Ifcu?r{|?C8AWy)!9i$Ky&(vbpMzAQ0s#7+61CxsxG3N;80JN+VtmSpA
zJCy5KbVFddEEUesg3-Vlg+(%XnegtLINN>K+XNyA`tVwjD4~%%WTZ*)5hK0GjM63>
z4aO)3iCCkJ6dyIx4zx@JGr*Mr5sFSTWvz@37o9A_frCqM{^c>8Qcf*EJSQ^(6SP(F
zMTeY>&T?~cOIfVfFMp>!))kVv{a};Ptw5Y1E=$q_O{hl4(b1lO)Z~x*8FS{DE*yY^
z6(*JT%2FWQ6%Y4BGYUG~-J6iZ@vV5BDL43L+#ipDy9QK4a%3bNi!-{+{1Pc5bxYBB
zW<+lChrtkrH%6pjyfGAx01_UT%Hd6+tO86^vEE2H4hEHnWkJawPsq{((SY<6Mz$!c
z{c&kij~td_kI3sovJ?yTL;_#n1%@(36WxvKTEsdnR?C>3wnDO2khCS28AiuC+2;s$
zKVheRw|KFz;Fk{)2aI<s&b<9)0voF#X@kX~CKr;bm>kuy$8|6TY-%qBcw=fKRK(^5
zy_G4QZ8NVvitP(}c35w}p!cbSlXqZiY@2_$7sHw~6v0wUuOJ=Pvx9p3qydl$PgRUm
zs4a%{i4h~MAPtrlHBmF$N@=HD8v*nn!5*jC110uPjB1qmr-v40zP1?ljzvYHR9aMt
za9YP+*V%V00+d3+#wdl_VkkGti}Zrl{R-3s!G22EZ<IqzXzUCSd81;cC*bX>pw03&
zl4ARB1bdUPPcUk`VXDBk^O(3uyQP<^wrkbOWOn6@-I(WpdMRLNOp7tJQ8E9|Ta@A>
zgQaa2#N7LPb~`xPvpROa&VJsK>xLQ3JO^I`Fy5}pX1#p-oRQ9C3MJGbf_<K_U%nke
z)i@W2s*Q?-%3fK{Izi8IJ@O`(zt;~`>{3|SwONv5K(8EC>8f_EM%>DR8L_^I3kbX~
zA;n#C&lXp=6!!;!t-06fem=gnSK{mWSX@qY#d#+m3`>zfjIV#ZpO5;x@uLjS!=vof
za1>}$l0gwoL?Zsq2uuq4BQeRz=PdK}{oA)YRSUV}LA5a-*l<9d&VPAG?cC9<1bPzK
z+4=jOJD&arCX^FSwWG%Gmu3G}P-joM^_gO*j&}7#<8b1|9)?6`le|6OeJI-8S;uFk
zipZ-r=UX)OeAR4x`7&sS@_%DRIm=Z8YVvN;v-#(}y6`_}qd8v_RHLdl|Ibp+c;Qf?
z=N-?JU(_`7n1Hx%WI5!XZrPS49h=AHal1S=M`r?F#BBcBx<IftSXWn7+3A&nUT;-Z
p!0mN=>!ccYS6%1oy4CJZsWaeNX&Hn6z})(S`r$8xHSnLCe*+~Bn1}!X

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000000000000000000000000000000..a810dd9107fe7a9e566a46e6250dde025f3242d1
GIT binary patch
literal 16
XcmYc;N@ieSU}9K*@!2Z#Q*ti>DzFAq

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..675fea29ba9e2
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/gbtc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.GBTClassificationModel","timestamp":1608687932103,"sparkVersion":"2.4.7","uid":"gbtc_81db008b4f25","paramMap":{"maxIter":2},"defaultParamMap":{"seed":-1287390502,"maxMemoryInMB":256,"stepSize":0.1,"validationTol":0.01,"maxBins":32,"checkpointInterval":10,"predictionCol":"prediction","lossType":"logistic","rawPredictionCol":"rawPrediction","featuresCol":"features","cacheNodeIds":false,"maxIter":20,"featureSubsetStrategy":"all","impurity":"gini","minInstancesPerNode":1,"minInfoGain":0.0,"maxDepth":5,"subsamplingRate":1.0,"labelCol":"label","probabilityCol":"probability"},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/.part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..101c2071193880d091fe121057591aa15c9e1ba4
GIT binary patch
literal 36
scmYc;N@ieSU}9JsB^xi8eS~r2^3y!Yc`qA2)kv%{(JuB5_hRG(0NL9NVgLXD

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtc-2.4.7/treesMetadata/part-00000-81137d9f-31e3-4a90-813c-ddc394101e21-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..e232de340042335ad16e22327618ad8714d80f2b
GIT binary patch
literal 3075
zcmeHJ&5zqe6dz}k;4Mp4q}sJbKw;F4b|qu%xN+iDq{4P32<ld=6>tDm)x@4;P5o7S
zoOZR!2?=rHzyXOpaPDEn9mIcv)WhBo9C|=pkPy#KvJOfL+nzYAWhI`O_kM5Yea&z0
z-F;m^!WH4FB9!5S7x<b~<`6<s8Gew6Hs(e!!Pm0sOaP>RdPVq}pD0#A(L@`MSB|jl
zktD%eIE;rXiHLnb)g&VEebx8Wm=0r_ByJd}JJfcwj}K!?_s^zxLx*};!OZn(!iXQ?
zE#1^QW~Xf#P2JEHoMU}V<4hN<G}Rkw8x&I4fsw=Fd$wa}9n!Yire?qbm?r)%0idI?
zf*m>_smJz;NgT!0acu^0w?YpviZN&>lmf19n5JpmfITR9KnY9X5EJ3q2t<81=mmrD
zJ>tT?G*yEc@@U%)z!yeS!LA>rvC9s@!idB!32X{V;2ryZ6uJTH1&qcc0z1$_)z18O
z2b|G!Kz~5Igywj=)DPpsUa-53x0)?8pWC4kJHT75=@!6kfSv5oIMW9>1$(4Vy~&32
zPdpw=|A$Z+8f{yI*zLB}I!~yc%UcVFezt!gPz#QoMRp<1iz#uAI4@0(B5*~{E#%;M
z48``O`t()_{#RBWL+!nIja9CFd5#L>Zu3xJT=)0O+(m_DReTnx)z7aN<;A_TF;`uc
zCzY3ZD6GFuICwWtl=<ZT>V!k~_iuBRU)4VlQu=Rl(lqCqpK$umtJ|+$-oIKx{ujS+
zYgwvnzfrP&MA1j-YN!>t0qD?m>%#T9X3axZ!1MH`#2<f~JwM!proChqgsyaRSrDFy
z3MKjW`)heiy_Z0B`9!E9p|p;qDuce<+d;zGdP(Aa%7{Z4S%<!V307ZvNZsK9OORNW
zYjU^2*8pjqNAgThD2Zk9GcjoJ-G8}H<Qkj@?7_AgqnfZLw|G>Q3v)TjH?N7m35`+r
z;(EEbUY?H$%va#W^|GAe!1Xd;g{NXoxHPHcS&ipKc+834ag9^4D=ld)&uBPvxZ?5a
zVj0PbBrUMQ(S*uLm7K{TX>&m$dElwQ;vS@wsd4yFoi!urlsm%gFanN)6Beg7gCGxF
z>SdJa(<2-ZKU<nKGFY050@tEL8pA}8dO#995SkIJX`9Qq;#fIDm>DcEz?>Tl{{aCT
z$SB%!glW4h0Gu$*4FA_2?v9c4!Sw8J`Eh;fth%o1nyS|~`l;(Vb<(k%L1$nAMedt)
l094g*G*dGz+R|*R4^&v|)4ro`%3YxElNbG2L1+)?|KH>&tQi0R

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/.part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..c35b81f9cb25f9937724e672820a28927b2d5819
GIT binary patch
literal 40
wcmYc;N@ieSU}A848!YST&7zunY2MT`+>_*IG-_^M(ACJ<-{3LthfZ!Q01OBZNdN!<

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtr-2.4.7/data/part-00000-3b5433ff-d346-4511-9aab-639288bfae6d-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..ba26a44f46f11305e83139315761785711bd35ed
GIT binary patch
literal 3740
zcmcInO>7%Q6rNpgoQ>nOQPo-2V6~Q&SgGV%&VL-GO;FP+MHU3;feNa!&W@86{_E}<
zh(e^CIdDKINI6BJ2UPU{2t^#K_KJD{!38czsLFw3gnFt#@ZPR>b{%ihlpiaxciz1B
zeeZAH*o!Z|HcTbjPfr)&Ln1^@5_k#lPYI3ESeC|NL#A?4l~AgJ4j~E|3amg7k%dSK
zEN18oTA(Rfpt5h_uRwfeOhmg`cys8wP7J}5gASj-IY;M<Kn#-92Nz;Leb%_}X!go;
zH|!rSf9e>aq5ImO1;W!*b$xZ1Wp77qJvlyyNI@?);CdoP_IiOgPsfVjKJcn<?friC
z$NJ6hXSGLverw#{<h-4Ho@P5=r?idlb`1C$bg@WJAm0=rU4nFduB%(xDE7z_xzZ;~
z`?EbNx!l)3;6_A9aSqr==tPzdfOV8clOE^wIqxYLZBPmriMN4?bnjLp81+CT(!KEY
zm%o%>IPWAq82s%*^5WRp?+ygN|DZE?Pt&8{e*g_4cS?cKPqq`hN|UW*$Fta8=+j^D
z>fn%YN0o37o6lbzjft^F4#mV>AJbnQjrkQgs{5D<eLm2567Kq#{t66whtQSCotPMb
z;PJ3r^bQLzVt&Od;`Xa^iag}DY?~A?&ZwAx2$6*7!rm?>ju6myK0z1yLZ<N|6|t+N
zcq^f&s1kG(rM{#73P5^NEk{X&iitl?EyS&cX1;UNT3TUM4M3?lNYtopGIn*IjEkyV
ztLtoDvo+f9?umv@XbhBM4U_4{l5Ny$u0S@bjizbXt7zBfEnc@Z+X8>1WL#3EiecGQ
zboIMgg;iP2c30A}W&jY<_f%NfzO-UgfC<+z(^y_<2>`mwYE%px0G-WvnQ3;@WLIlC
zdmot-j<sg9<+^Dw>lO3bipi{%dPRSV7f97tYgR9DTcX4h$;7>koP=ZzN^-)>3>B5Q
z@}Z!7EM(-FBgSVU3k8!fV4Qnc+02%JVB<20CWd%&A*l}Jrl4#JpaPg~&tl#2RQM~B
zNCFtdj$mJiNg7LxOL7n_P#6dFH$mAKKy`*6;#u!0I8F8gQWE0$wiLp}Ywj@pEGWMU
znQaM}I_Dz}j!tP9(?%4T2Bm0XfoG__pPWAO`Uj{7lwDET5;Gf79%OJhA~j}`kf1b-
zdJ9V3kx-{K0|9P9%|m7&vV=^=kW@bDaeAP=Y<|}}RNI2`xsZ9%1FhhXL~I3r6&Aow
zZwPeY00ppjdU>1Sj6iI+&2SZl{V>k4r+8u9)7W+j9gZDAxg%sA#(~2hmB`_*!Z;rH
zIkkt=@x}p)!_W1>Zip+pg7S@!dEDo9L)(-+WcCKn)!mbaM%q3hoKQb<PVL>35GtlF
zbo6$1f$~Ye5{DG1YTaDUX$=iVaL$>=xp$dq!B|zhMsj1hNur+X(6DY*axl1WHJP0=
z>+j{N%+_?6(&`;~Ep4whSb92b*=BRePG{3)gH?1ZJ^kidx~5fe%boAA)lxNTFfN%1
zf?Bgu(MlCiDr*&sWz!yVdU|brJ<AciYk^}Nht|-!divQZ{rkp8(d$ig_V|8#*V2Cg
z;oNK-hsUp(rnU;=*6nu9Ee3aVsa~_;rn6p#M7NT>9&{gyc6iqDj^vDKj(Oao@q5+B
zc=j^*m+`+raqe&qz>{}G&j#;#Zuk${=+LhR9F%+Wf0nZCiw37Yc04zp)-=aZAnHr4
z8FIdArdmTMHJlsH=Y~^5r6zm<OKDRDy*yPe6h=o%6RbQjF*>T}C-M^oHlAN9lqL(4
Z`4TJX!{-wB;P>Uxf5eIrvIxIR{{~z=dsP4c

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000000000000000000000000000000..7dc6e149db71e18a5578bb119d6cccf8ff106d34
GIT binary patch
literal 16
XcmYc;N@ieSU}AWubJ20OmhChEB{c;9

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..a9a712e626510
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/gbtr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.GBTRegressionModel","timestamp":1608687942434,"sparkVersion":"2.4.7","uid":"gbtr_0a74cb2536ff","paramMap":{"maxIter":2},"defaultParamMap":{"impurity":"variance","maxMemoryInMB":256,"maxDepth":5,"subsamplingRate":1.0,"validationTol":0.01,"labelCol":"label","maxIter":20,"checkpointInterval":10,"minInfoGain":0.0,"predictionCol":"prediction","stepSize":0.1,"cacheNodeIds":false,"lossType":"squared","seed":-131597770,"featureSubsetStrategy":"all","featuresCol":"features","minInstancesPerNode":1,"maxBins":32},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/.part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..b681b9f615bf5fd53072363c0a42bb2759ce32e3
GIT binary patch
literal 32
ocmYc;N@ieSU}D(PYL`-J`#$%RNMP*72{FP6dkmCyJ~ayf0I@y`tN;K2

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet b/mllib/src/test/resources/ml-models/gbtr-2.4.7/treesMetadata/part-00000-6b9124f5-87fe-4fd8-ad9c-4be239c2215a-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..9a7e77acf546461ce1f1efb2814d5164828a9af4
GIT binary patch
literal 3038
zcmeHJ&5zqe6ragvQ@2{s7GY!4AVr3aQpvUKd^pJ}P-s^`mIkF2>IGG0Vvn<?wv*c9
zl+_XhH~s>&-2(?sJ?uY##0??wKOk}7!gY^GNEL7Lz1=KZA%QroWhLV`Z{Fwp-h1tL
z?wBGH*Tt7LF%1hr6tZ#}BZTBM{2&Px7QTQWWI35L0QV0sh(BK#AI!i(3l*NM?5Va-
z;#l2Odr?;>ePZuWJ?@j}fgbpJM7t4<W3LzLTh#XW;-iStJ2P^t=TKkOROSUVW+dpV
zo2F&dtXj2RE|x7*Q%ADy(uki0Ek(VoSK&b7Iq>DM=zf({Xw9fnr(zP=0M#UD5fHRD
z)KrJMB=K2$PE-T(*%s}y9d)y!sXpmY|7OnzS)(Ngu(=B#f?Z7q+;WK@Qy}h8`$50w
zg{&De8Vv}XFb$0hYtf(=?KZ>K4Ry0vv4G}<%@CXkZ5p>}#6<x!ctR=IY?ds`DwV7C
zlBT+puml1$HemY2ortL8*$iABZ_U496LjA2La?n=)Ko9%Cy~c?!GQsZJg$_pa1-3#
zd&Gm78M-kX%KwX0)eN)bRH$37mbjPmtSnE}-0dY{I2q1kfq6Crr|0F^g;*ZXf8&}=
z)BQ~HheH6{ubHEpDfnMmc>>t`=NMak<?H7_7``O{z;NRT?&*I4F#O~GbK(oPK0gay
z_=7LcL>K<*@{9PV+zeC~uPEzjRm1F)Q-OpN{QBD`Q1tW=!+Y(puyB#_?_V5Z<&SLv
zzuU&?fV}zYTWRb*=F598-Mx*G|Fn?WE~Lz(i@y}6NmqwMD5yE{jfFZaK*HfUx-JU`
zkNNZcb*RTnsKB(|81Vx0lvPYAx87eHRm=M^l&gS<86>7Qkep#qa+_O7T-!*=LO>aD
z2qPO%(l5d8g-6uu?ywk1X(g*P@<J9QZ3sx2o)c42TKZH9^FrfqT7{AYe?T9Yl^<lq
zHKih;j51d@lJd<J={qq$Xq<PBQ58SyYM*zGd6QoN+<E6%J~lw-SVZBzloc<Hkvzrt
z2!sb%`WEN+rG~tuw473K7UW6?uSsd7XtI1_;2VsmG<kJ2q#^n036A)ouS3^=m{6ui
zy+``A7|FlkJ(cbDc>{`Bl-LZC?0VGaBiP4#YDfaUH7;bZHDQHujCN@RgfQ`;NBh7u
z8Q7+h`4~9VW)f!PnG8_ph23K!zyKKeW~|30Ov>fl2&b2D#b?=YZ;0gWiSO5fC^rdK
z&eTmqH*<wf;`vUF)as5~bL;g|v13seCdHCtScX-n6~nG~s`aYTp&iG(sx)9SAHV3&
KRfO6w+5ZJm3ZA|I

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/.part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..5bb3a22617543ca15ec103fe962cebdd5230711f
GIT binary patch
literal 40
wcmYc;N@ieSU}De|FucBw?a{4JH!Z!gJ4teXR<7A1-pG>lahiSnrXFu!02kN~uK)l5

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfc-2.4.7/data/part-00000-e41a7b98-91f8-4485-b112-25b4b11c9009-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..d9ec35a4de580d57b00c6375223f941923fe74b2
GIT binary patch
literal 3836
zcmcInU5Fc16uvV%$<BtPO(l1jXwVS4A%!&Tc7L-hY@|iEHri$FgP>I>J2$(7lVm$H
zv9@Jl1s{A+L0EjTf<hm(O7X#mJ}j({eGn;F-<IzB=0m|nL_~yo&P?Vfo7tcGGvt!o
zbI<+0^LNjk%so9nOa+>xsVaO4goqge4+n!1D$;n7reHy&Vn!BFDuWIo5-9{`AQ0kQ
z<c~hkoTSrLdIXU&I%?t`9!4Jw+!}!qg#qXpnsv~lHabdubbkioBclkB;`$=*Cf^@}
z%>vER;~k5W)36qJc4h!OMdbjt=q&Hd$emxd2O(0>iye2kxX5)5c;;xeN@M6xA^Z3T
zb4PyvWow2cxK~!5`s4@w&#jr!!TTRdzuld=_3TeqQ@6H|yW20()$Hc(l>)c#=j>>H
z!1i;MqZg`>A26T-(d5L{=PO)4n0_uH9^*(gktF$qNOB4B_*qbZXZ4v17;%b@7ij?;
zh|*}LLk(NIjd1jXQjnxy$2|4lycxql6#Dx5#T#+q+!wz_Fz$17t~VYLPTkq~^2Wpa
zpMC!0%>0v=Kl=LPjeR2G#?duqX9Ma<?3Dt2NJQ++ix`mPG*;I`c+gU;FmPO^nLy?R
zl*$6?V9ok75XXy7ruj(fRok}$Ep~iMRdES-VI5-4VoZ8+D>zl^qWd#Y4zWCl|2>$F
z``(7{i!cf>dht=qx89%m_<UcaQ!9kl=sdbkf;q38_i~=BNb!PG^j%T_8=kg9LHgl$
zWOKIp1R3B&P#PE@NvN!o6_7(^l`BvI&=jha$M_3z^Z@vveFS5UQr~KS1`uM#CgMqo
z^69@Gn~j^xit)y4=E4$dD1b@%K_W*jgR!$`$QUn+&6di}D3(H#&Kj?&gvLNAwrns}
zTd=fN(-DYTW4Uc;))m~Ba3?QXie-YiQ8FgTg07hs<sJQgrn3fXTFy*ZR5SoW`T?EQ
zt<y`I4oo<Y8QS7fM*z?fb6M9c0JIn5b*5NtgPm=v>@8$Y+q)HuEw&7endglQO9nHS
zTDp3G6-bq6wi~Cty+lf-GwJobn1Nso3S!!g4CSSi^d2Xzad~lipT3+fLcRoa7>_PS
zK7XA<urWc5rVCzhA*eRw9ZtH-feK*y{TI)TKO-TS5BLm??Zyn`CxHEcpY~4u!QD87
zYh4K8@=|Xzt@6?;pI;Y%skh5wr|Ff3d9@xzra>W^p7jFNT~>BExn%^@1?dhieG1f3
z*U7_l+z=->A>%y(wJX$h@EtzC38@chntQa;5SHy&NGbTmZ&Q86N#Ah!?HE*q-#d48
z(h%sicnEah9NE~nIq6$2zZUnZ4LgNjZ8!_d%e{n$J?SOJIh0+8F~D&z0UUn!L=Jxz
z#<6zP&OM2yU)n{n_wn9c4Jm1plfDB7kGj>+)phsi#pw*3RcI}ER=1OTdfMG6Y*(Fr
zu$=@~F?68~xHnbO&&Q+`gg};C#$riXR-hA??513LlNly-S2=R3G+LS<@`)Y|^O9bI
z{(Y&<tdh~XTxu{&QK4DOH^pl?>&i0AP326>XfIf~Vy><+T{Ux4FJH?wl?E<3Z6207
zs#+7eCNn_LZ0ovG(?O}O=q4-XT;$x;wUw2khv4209>!s4HPutkJvya-?f58atBuAU
z-fs7H`VS!N3(mvw{8tP^xdP(OC3nF|hG*zPt7*Z7XTAV|?gV)yXg(xu@2KNd=`rR!
z%)=(lyIy^aM^A%)C;uB1`zrSUyx{Fqv%%}$+xQR4XwSC<57e{f|4e1q`-Vq-Xn!sq
zR5XWBAo6pa9&)l_WIIhKJ6swbDh+1~wKlwmWtGXYTA!?!%Oj_1<E%bDJ~E;XjSr2N
h+1SuRxi(Rr7^<<FI(#C%4*y^G|B$OOLgwIq@81~LkIDc5

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000000000000000000000000000000..58bda6df9cca76cbe0c755e25286b3d2cded154b
GIT binary patch
literal 16
XcmYc;N@ieSU}9*_y6ih`{)WW>CKUzH

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..07748b070ee84
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/rfc-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.classification.RandomForestClassificationModel","timestamp":1608687930713,"sparkVersion":"2.4.7","uid":"rfc_db1adb353f1e","paramMap":{"numTrees":2},"defaultParamMap":{"impurity":"gini","predictionCol":"prediction","numTrees":20,"maxDepth":5,"featureSubsetStrategy":"auto","subsamplingRate":1.0,"featuresCol":"features","checkpointInterval":10,"rawPredictionCol":"rawPrediction","cacheNodeIds":false,"labelCol":"label","seed":207336481,"probabilityCol":"probability","maxBins":32,"minInstancesPerNode":1,"minInfoGain":0.0,"maxMemoryInMB":256},"numFeatures":692,"numClasses":2,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/.part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..729c5bb30d7fe171323f7f76f91dc0f90530d162
GIT binary patch
literal 36
scmYc;N@ieSU}7*02|4&HvPGAZZL(bE?HEzD3k&Z*`aEN*!Pl-b0Nk+-?*IS*

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfc-2.4.7/treesMetadata/part-00000-21082d24-b666-4c4e-a823-70c7afdcbdc5-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..610882198c79067d9362d7d976664b3389ff8033
GIT binary patch
literal 3391
zcmeHK&1)M+6rWwMlvU!UPLmb8L?!l88wZp{T3IW}2+}wXt%4myFr|=&Fxs7!C)p2W
zcO}Od-P@jeZo%{vOi!hU-un;q++)e9he8a#m%dqjN2;A%N)9%{ntAi)y*Kmb{eDmN
zI}h^`k`|<ESxJC_D2W+a;1EKx0RK=Lt&Yt=AZD1DDS-ALA4<Q7y^TrOXrR^Ca|f#F
z;4oCzRX=EIxP{GKqJ=FSJkwl9i`S@4O&n3*(>94oLm2D?gxqD@hD+PNMI1G&M${!?
zgx!|9o;Px3qg<*Ka-}sRtHw6JAOYJC-gIqEE5U{~wP0pN<`ZHTv1ylxUaXj~03l#^
z8-t;PZdSF3joVIC9|=_#zt|?OAMDq>?G1HZFM<;Y-h9^bsTb9}hy)$%KpeR&)7_vR
zoU)*U<$ze!WCyzI!}{19+`81OdA9!<rVwRLV|$qd&g7XStdrmYoD__LFLqUIR<ppc
zjh&D%+kOK#s6(Uuf$^aLtW&qu4qy$uHmOHJ5)uLza4T<=jPhDV&y}*OgB!%@?T&|_
zWfL5=0}}S6gBc4Kzt|+LXjfe?X4L@i)!#VVgh=gn<^L<=5YCwy*X@Scu<{LiO@G^r
zkFq#@m8T?sbVt+qosofwTkGBVga|HvMv`Vd$aj1^r##F-(x;xlsZXxkh0_1~!jC+B
z&CR_AAULDoxy+ZtqModG{W|*kj523Q=XOM>a_-aPUuRbN`=9eAVR4vp<!WY->(uqc
zC8Q>H%Wr_r2maP9a9HYamp3PofhAXol!eu?f=)$OE-#fn8KFih^YIlC3iY28?&S%G
zjBii4)iGo|JmEHWxNE(q{_rnuAufn}H!9V<l>ch!CxLqF2|n@B%Omb-z&-e$zi;yQ
zim4wGm%LY;;Nr)>>EH0FJNi|`oB)Ram26481nkkH3WSbt162r8Oce0N7)FHG1AoW2
zM<ZaG8B$Q<1vDq|%GT%eas59DAyyYlNhI;hNKQsT9kop)%`fw^=#mIqIKs<7PZweJ
z!XBZ`-6%w9LCGl96)^*rmPMot_DOtNNIy+`D`NE&+EX%c0`LdXtaLKcyiycVQW?36
zt^9r?eI&1Rs^=`&d+j4uWVCY5f@96iBo4ky-*e7_aa;Btu;4f*9Kz(HvMh+o5OHEi
zIO5Vjb1R2Rbp&&x(SHD*!un|euqFmg207_Jm(l`Kva&qYQFnUnubhaxUnJk0(g>eB
z8g#zT+a%Hge@`1Ar2L9IP^0}8w4Q1h1#L58BtnV9de27(s)t>+)I(QT>Z?K{OqwJB
zf!B7RWpO~M->D8Pv;Ml99a<QQhc>_+^_mkVAOPrfhpLkn`WQY6!t&e9@mVoE?jreq
z|LnKiV6lH$i+L@d)AEa}jW%_xMO>~}cG<2}3VOpJwqX<sR?f&76;jNZl}4#j$~6cy
Ul{b|t{PgL4(056M>hRO)U#^e?8vp<R

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/.part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..52cf21f6701ca76c5bd96f3e5caf318e14601856
GIT binary patch
literal 40
wcmYc;N@ieSU}9idr)<=mvrtcvY1^%p=O3+9i2G5rC}784V?mjv+$$IY0Tlrb{r~^~

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfr-2.4.7/data/part-00000-4a69607d-6edb-40fc-b681-981caaeca996-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..75a3f0459864c1eb15eda041d202cd6469b1c172
GIT binary patch
literal 3797
zcmcInU5Fc16ux&do1INJWm<ZNi3E(H4JpK!ZQXRYtnNy;Wh<dwY9B;dbarR5J2*+U
zGZSkqh4rZ-ic)+KpB51*B5P@(4^k}nP>N#dL*Fc=P#=mf2A_%wdd^Jd&SW#&Zv7b&
zl5_64-*^7*xykGc$3~dQWVWLQA0i=AmcT>d%?L}fR0-BZCNU|iil|k=h7g4mBN8YQ
z5>4C%!y+rySPFr5)X)&O2)GV!+zFrxd%_1zdZ4HcK|=}DTZ*_<risM6ho%91mQ8oT
zi@7`BKY#X?cyE$WDz5xGB#ESxmUa>{^uu!>T)2I2vX4jtD0n)PYWJHLMT{1VBE;1r
zKs?1JYLEkPE|+_9^^dRTZr|{UKOB;ZVE?WB&L@}hH;{A(QzOwyfpsrwzZO7JJN;;d
z#OG_k`4MHW*VuL>oq;@3Y2}qEIw+BqqZsULS`s^x11h8BU|J<}5c{zy0D<Sw6u2?Q
zCQ7UVFiDoo2IO;7!MAAaQ7vrGT3Ca@%Z;DVok~I+l%LwRZDjhOesab=xh@$eE}dPJ
zzxpwfjLy}KlM$>0UUaTPA4&aM;9Zvtm1S!X3V1eL3R>zfc={49qSLB~YuKZq)QkBU
z#F1k=yG4~JFl1~|+=qMmanT~g4JA;o^EHZ#y}t$yBknKo*^i4BA#NyzxPsRZgw@y#
zb{3vWUec9OUeX7XN~-9$bB`8)3S$_P>>a?w&($UwpyGqcfx#je!U8<cOhG~<He6!@
zCRrRthk**_LZq9$w1&>2jV}R$l=V0SA~PZL=k}RlXIZn~c+FW@G8!6SGGP}{ldf$T
zho{M?pi0e_ZcJ;g#$<m@Fm=LGV3b<64c%OD%~sP_NM>WXZJX{X+?Ni7mt4(tz}+Mn
z6;;tP9hV8d{h(nP4WsG$lDMdu0EF~ImQi;PESVNCp`5bK#igzSpa+~~%X9(ITO6(%
zn%lOG!%f|I3z;+CZp}3oTefL9NA2TFw&5(bEd3E)AXU2AZXDoyi6UpRnJWb;3(49g
zN|_)tOi*OyJ*u3e1!>|T12<EGdWje?@jjr&pHc)H6{TdR$de06^&l@&<r@knfa$j`
z799^rD3KV<TSAtl)?-HM<6-|!mEUOLmN3DO^`l2@4A(U=hKrZpW4ce3`?PRP1g8E8
zild}o8yD4D5}9_1$;=E-RBu~(?G&^TP*;>Yf^tzPtR;Dn(W!~lm{3BZ+BoXFQ1QNm
zdL0@G@Gg`-W-1~}%&d$_<@HodD}>GOQLR$tJS|*LK}X=h4Nr9_#RbsW5(6DMM;`Wh
zs$8Ij&KBNnI4QzzLn#h>HO;ZdcwziQ*>e~p9II*I2!|$egi;*G&8=STNiy^DI*N^t
z_xf&-m1|V_juvii4Z5Lc;BGL6Gg&yRFj^qlg>&+Tk@gM>FVy#`@-c-y5GtlFELAAQ
zgsOb8O_4#bsx5o5tSxIWiOb$lF28Bm4op{dU{Cqk@;;*O?z3=CSY?>sC)$Qvwp%C5
z4a3!R7}n~S(&@Z=YT3x|&pWQ&UU2iJeBCrG-O2BN^>n_eHE_uv^RU!4HJdOs4I32A
zwq<Ga78unv%P~s%06D+^^qDgyju6}q9Ah)Irq0dtk53t1J0XhRYNNB8_dB?q{sRc_
zg5x+me$BSEQ=sl%a>xB*a7PzfO&2aa=LjUao8&W5_n~MTXC1Ff&Y0(zn=Kl@UPFw>
zFGF}I{~HwVD(3(^c@OE?=ylIG{)0B!@K*u{<=*_CrR@2=!KpVL&)lP$W)lWPeX%=3
zo^IH=?$F7Nlt(J%kz8@U4Zp*3+Ss^WAFGd#@7*(BHR{#s-o1LITB(j3qm_m6`F-R2
aD)Yv?KC(M=1^$>m{6*eQ5i$#ZX#WP9+>Les

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/.part-00000.crc
new file mode 100644
index 0000000000000000000000000000000000000000..1a72b8e0298442276506431fafc8c925497ded55
GIT binary patch
literal 16
XcmYc;N@ieSU}6aL`H*sfwdOPcBd!HV

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000 b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
new file mode 100644
index 0000000000000..cccbb8f9f3b05
--- /dev/null
+++ b/mllib/src/test/resources/ml-models/rfr-2.4.7/metadata/part-00000
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.regression.RandomForestRegressionModel","timestamp":1608687933536,"sparkVersion":"2.4.7","uid":"rfr_d946d96b7ff0","paramMap":{"numTrees":2},"defaultParamMap":{"numTrees":20,"featureSubsetStrategy":"auto","maxDepth":5,"minInstancesPerNode":1,"labelCol":"label","cacheNodeIds":false,"checkpointInterval":10,"featuresCol":"features","maxMemoryInMB":256,"predictionCol":"prediction","minInfoGain":0.0,"subsamplingRate":1.0,"impurity":"variance","seed":235498149,"maxBins":32},"numFeatures":692,"numTrees":2}
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/._SUCCESS.crc
new file mode 100644
index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c
GIT binary patch
literal 8
PcmYc;N@ieSU}69O2$TUk

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/.part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet.crc
new file mode 100644
index 0000000000000000000000000000000000000000..8081f8842670e94dabc89246bcee00c2bd49a151
GIT binary patch
literal 32
ocmYc;N@ieSU}88t<>t$D>o-D<hnc3>EAC8n2q~DiVZs+v0Kj|=7XSbN

literal 0
HcmV?d00001

diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/_SUCCESS
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet b/mllib/src/test/resources/ml-models/rfr-2.4.7/treesMetadata/part-00000-dfe4db51-d349-447a-9b86-d95edaabcde8-c000.snappy.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..093c3468db30122ef0c7b69316fa611f342d48a9
GIT binary patch
literal 3055
zcmeHJ&1>UE6dx&CuM)CL3YAu3T}X%8WI+{@9m}yX4b6sj9h?%vQfNyFV`=PYS07$|
zcsB$cdMTw83WfHt=N@wKp~q5s=&exbztF?}1^Ej~pJY4Ahj-Hy%3flOJ@aPf{pP(l
z@Aux$`@6b;geBokMc`n-3w%-H7=)0-!4DGA%BdL$_+lzfH^A;+ZwY_g93L#eK?AKE
z%^hOXAz_G{*bn+D84z=ys^NeHk5$)E1KJO07}~z4Zc@`u2M+^EcTdFIzC|6Z;K+7q
z7!h}Xo4TQ`8Ef@MwOTXk3eI$WOoQ|+pj6aVwGIbj+k%-D1$zcvU8N0b8P!z-766*K
z+XNIHCJMG_kHk*2lM69i(0n}bZ7*tj5e<gK0ZdK7E_t>|2hl!m)&P}K(63hw1JuG@
zvgHli14FA71@|b4VzBJK4~I`?fbNhkb;i<c2qMtCO<g}YXnWfmxLK(gAhW%;2ljd<
z4R>hp5qJU66l|vMOJ{8hkb1-k>11=y|B%?=08NFf4gzY~W&|D?L*;Y8v)w@)*wF!8
zY)Aq-#lT4-{okPit1b<dWl~xpb<L{R{#B@EnQ|@?nJ3RU=S&=D#3Ogs4Cl>=`&B~q
z(s%KSM|bXVBnc$jUxnlQEd0;S9YOB>_Z+)%=c`LHEDame;UU48*75f<6YRA|jrr}J
zJZSThyz~H``T0oQBAr<Y_4Yb*yK_35rXiaD^o`p;gPyBI=I;^n<%mJ`uSd+vDcbpN
z#5{U|mNNDzeY>^CsDCoLzOl}1{my*QWPTYjoNK?=WItJ$|3Y5ku(JCN%OwdgFc5J^
zU<KgbG0`G)ybeUm34BVk@R#y44-Wx8$LskBczT8uHnAL<6Ii+PU@=R^y%5ZEiBLcS
zTS8JH0`hHdB4M$_O1w)WVv&fHfZP{g_2yG*_xGa^iJV-NTV=iono2y9r{@G#<itm!
zSLR!<C>nAREC4*XO?g-p7Udd`3UUsYY5C!<_@huBwytZ)bq&eO{dEn2az7=g3%$iv
zm<pZF!slX9SddE`FQ4RimV`%)_!CopF1B)F$wx1t7<o)gY?|U+mgM5mTOx;KMUpOb
zfWvXyCf&%oHYDA<up@lpsL-)JiD{$;{!?|zNzx1E5Jv|CXp3+d1+f_cRrhS_q<zt6
zhu9--x-{l9Sek5wrjGV$00J*|p!PekY1&0i*e>l06Xit16foBTaJJV!qXI6F_F=O%
zOjt8lVEJ(heAyj7Pmr`VvHe{)Se`g*Syy#U)t6VgvF%vPWUXQK)_RRbwbC_c58BbH
jr5T#hpf$~GbnA_})}>ubzbCh#RUdy5TR><BTK#_j#xab|

literal 0
HcmV?d00001

diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
index d1ade85cea049..13efdf13d9e54 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/DecisionTreeClassifierSuite.scala
@@ -446,6 +446,18 @@ class DecisionTreeClassifierSuite extends MLTest with DefaultReadWriteTest {
 
     testDefaultReadWrite(model)
   }
+
+  test("SPARK-33398: Load DecisionTreeClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/dtc-2.4.7")
+    val model = DecisionTreeClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.numNodes === 5)
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private[ml] object DecisionTreeClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
index a2208edcb8394..37e695ef88b14 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/GBTClassifierSuite.scala
@@ -545,6 +545,20 @@ class GBTClassifierSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(gbt, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load GBTClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/gbtc-2.4.7")
+    val model = GBTClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.getNumTrees === 2)
+    assert(model.totalNumNodes === 22)
+    assert(model.trees.map(_.numNodes) === Array(5, 17))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object GBTClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
index 902af71e42f86..7cf4e65baf557 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifierSuite.scala
@@ -240,6 +240,6 @@ class MultilayerPerceptronClassifierSuite extends MLTest with DefaultReadWriteTe
 
     val metadata = spark.read.json(s"$mlpPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
index e30e93ad4628c..28b245f7f05de 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/RandomForestClassifierSuite.scala
@@ -25,10 +25,10 @@ import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.tree._
 import org.apache.spark.ml.tree.impl.TreeTests
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
+import org.apache.spark.ml.util.TestingUtils._
 import org.apache.spark.mllib.regression.{LabeledPoint => OldLabeledPoint}
 import org.apache.spark.mllib.tree.{EnsembleTestHelper, RandomForest => OldRandomForest}
 import org.apache.spark.mllib.tree.configuration.{Algo => OldAlgo}
-import org.apache.spark.mllib.util.TestingUtils._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.{DataFrame, Row}
 
@@ -319,6 +319,20 @@ class RandomForestClassifierSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(rf, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load RandomForestClassificationModel prior to Spark 3.0") {
+    val path = testFile("ml-models/rfc-2.4.7")
+    val model = RandomForestClassificationModel.load(path)
+    assert(model.numClasses === 2)
+    assert(model.numFeatures === 692)
+    assert(model.getNumTrees === 2)
+    assert(model.totalNumNodes === 10)
+    assert(model.trees.map(_.numNodes) === Array(3, 7))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object RandomForestClassifierSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
index 8fd192fa56500..861bf1e0b1292 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/HashingTFSuite.scala
@@ -99,7 +99,7 @@ class HashingTFSuite extends MLTest with DefaultReadWriteTest {
 
     val metadata = spark.read.json(s"$hashingTFPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
 
     intercept[IllegalArgumentException] {
       loadedHashingTF.save(hashingTFPath)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
index 948140897d8cc..c8247b9c8f3bf 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StringIndexerSuite.scala
@@ -483,6 +483,6 @@ class StringIndexerSuite extends MLTest with DefaultReadWriteTest {
 
     val metadata = spark.read.json(s"$modelPath/metadata")
     val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
-    assert(sparkVersionStr == "2.4.4")
+    assert(sparkVersionStr === "2.4.4")
   }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
index 49ebcb385640e..9cb0345400bc4 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/DecisionTreeRegressorSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ml.regression
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.feature.LabeledPoint
-import org.apache.spark.ml.linalg.Vector
+import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.tree.impl.TreeTests
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.ml.util.TestingUtils._
@@ -236,6 +236,20 @@ class DecisionTreeRegressorSuite extends MLTest with DefaultReadWriteTest {
       TreeTests.allParamSettings ++ Map("maxDepth" -> 0),
       TreeTests.allParamSettings ++ Map("maxDepth" -> 0), checkModelData)
   }
+
+  test("SPARK-33398: Load DecisionTreeRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/dtr-2.4.7")
+    val model = DecisionTreeRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.numNodes === 5)
+    assert(model.featureImportances ~==
+      Vectors.sparse(692, Array(100, 434),
+        Array(0.03987240829346093, 0.960127591706539)) absTol 1e-4)
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private[ml] object DecisionTreeRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
index 04b0d4b8470f3..7d84df6326397 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GBTRegressorSuite.scala
@@ -370,6 +370,18 @@ class GBTRegressorSuite extends MLTest with DefaultReadWriteTest {
     testEstimatorAndModelReadWrite(gbt, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load GBTRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/gbtr-2.4.7")
+    val model = GBTRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.totalNumNodes === 6)
+    assert(model.trees.map(_.numNodes) === Array(5, 1))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object GBTRegressorSuite extends SparkFunSuite {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
index 31dc6d379e76c..4df8366ec2443 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/RandomForestRegressorSuite.scala
@@ -221,6 +221,18 @@ class RandomForestRegressorSuite extends MLTest with DefaultReadWriteTest{
     testEstimatorAndModelReadWrite(rf, continuousData, allParamSettings,
       allParamSettings, checkModelData)
   }
+
+  test("SPARK-33398: Load RandomForestRegressionModel prior to Spark 3.0") {
+    val path = testFile("ml-models/rfr-2.4.7")
+    val model = RandomForestRegressionModel.load(path)
+    assert(model.numFeatures === 692)
+    assert(model.totalNumNodes === 8)
+    assert(model.trees.map(_.numNodes) === Array(5, 3))
+
+    val metadata = spark.read.json(s"$path/metadata")
+    val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+    assert(sparkVersionStr === "2.4.7")
+  }
 }
 
 private object RandomForestRegressorSuite extends SparkFunSuite {

From e882c9058fa010f133cd794c114c67cfd4541186 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 4 Jan 2021 13:33:41 -0800
Subject: [PATCH 1243/1280] [SPARK-33950][SQL][3.1][3.0] Refresh cache in v1
 `ALTER TABLE .. DROP PARTITION`

### What changes were proposed in this pull request?
Invoke `refreshTable()` from `AlterTableDropPartitionCommand.run()` after partitions dropping. In particular, this re-creates the cache associated with the modified table.

### Why are the changes needed?
This fixes the issues portrayed by the example:
```sql
spark-sql> CREATE TABLE tbl1 (col0 int, part0 int) USING parquet PARTITIONED BY (part0);
spark-sql> INSERT INTO tbl1 PARTITION (part0=0) SELECT 0;
spark-sql> INSERT INTO tbl1 PARTITION (part0=1) SELECT 1;
spark-sql> CACHE TABLE tbl1;
spark-sql> SELECT * FROM tbl1;
0	0
1	1
spark-sql> ALTER TABLE tbl1 DROP PARTITION (part0=0);
spark-sql> SELECT * FROM tbl1;
0	0
1	1
```
The last query must not return `0	0` since it was deleted by previous command.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes for the example above:
```sql
...
spark-sql> ALTER TABLE tbl1 DROP PARTITION (part0=0);
spark-sql> SELECT * FROM tbl1;
1	1
```

### How was this patch tested?
By running the affected test suites:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *CachedTableSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit 67195d0d977caa5a458e8a609c434205f9b54d1b)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #31006 from MaxGekk/drop-partition-refresh-cache-3.1.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit eef0e4c0389c815e8df25fd2c23a73fd85e20029)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../spark/sql/execution/command/ddl.scala       |  1 +
 .../org/apache/spark/sql/CachedTableSuite.scala | 15 +++++++++++++++
 .../spark/sql/hive/CachedTableSuite.scala       | 17 ++++++++++++++++-
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index f41c4eca203af..748bb1b35384a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -580,6 +580,7 @@ case class AlterTableDropPartitionCommand(
       table.identifier, normalizedSpecs, ignoreIfNotExists = ifExists, purge = purge,
       retainData = retainData)
 
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     CommandUtils.updateTableStats(sparkSession, table)
 
     Seq.empty[Row]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index e14c72eee9646..0e8122ef88edf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1270,4 +1270,19 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       }
     }
   }
+
+  test("SPARK-33950: refresh cache after partition dropping") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index 6c0ab2f84498d..ed3ccb4afd255 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.hive
 
 import java.io.File
 
-import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, SaveMode}
+import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation}
@@ -439,4 +439,19 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       assert(spark.catalog.isCached(t))
     }
   }
+
+  test("SPARK-33950: refresh cache after partition dropping") {
+    withTable("t") {
+      sql(s"CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
+    }
+  }
 }

From 36e845b4d20678d62a9d155fce136dc4e7a29021 Mon Sep 17 00:00:00 2001
From: LantaoJin <jinlantao@gmail.com>
Date: Mon, 4 Jan 2021 21:37:26 -0800
Subject: [PATCH 1244/1280] [SPARK-34000][CORE] Fix
 stageAttemptToNumSpeculativeTasks java.util.NoSuchElementException

### What changes were proposed in this pull request?
From below log, Stage 600 could be removed from `stageAttemptToNumSpeculativeTasks` by `onStageCompleted()`, but the speculative task 306.1 in stage 600 threw `NoSuchElementException` when it entered into `onTaskEnd()`.
```
21/01/04 03:00:32,259 WARN [task-result-getter-2] scheduler.TaskSetManager:69 : Lost task 306.1 in stage 600.0 (TID 283610, hdc49-mcc10-01-0510-4108-039-tess0097.stratus.rno.ebay.com, executor 27): TaskKilled (another attempt succeeded)
21/01/04 03:00:32,259 INFO [task-result-getter-2] scheduler.TaskSetManager:57 : Task 306.1 in stage 600.0 (TID 283610) failed, but the task will not be re-executed (either because the task failed with a shuffle data fetch failure, so the
previous stage needs to be re-run, or because a different copy of the task has already succeeded).
21/01/04 03:00:32,259 INFO [task-result-getter-2] cluster.YarnClusterScheduler:57 : Removed TaskSet 600.0, whose tasks have all completed, from pool default
21/01/04 03:00:32,259 INFO [HiveServer2-Handler-Pool: Thread-5853] thriftserver.SparkExecuteStatementOperation:190 : Returning result set with 50 rows from offsets [5378600, 5378650) with 1fe245f8-a7f9-4ec0-bcb5-8cf324cbbb47
21/01/04 03:00:32,260 ERROR [spark-listener-group-executorManagement] scheduler.AsyncEventQueue:94 : Listener ExecutorAllocationListener threw an exception
java.util.NoSuchElementException: key not found: Stage 600 (Attempt 0)
        at scala.collection.MapLike.default(MapLike.scala:235)
        at scala.collection.MapLike.default$(MapLike.scala:234)
        at scala.collection.AbstractMap.default(Map.scala:63)
        at scala.collection.mutable.HashMap.apply(HashMap.scala:69)
        at org.apache.spark.ExecutorAllocationManager$ExecutorAllocationListener.onTaskEnd(ExecutorAllocationManager.scala:621)
        at org.apache.spark.scheduler.SparkListenerBus.doPostEvent(SparkListenerBus.scala:45)
        at org.apache.spark.scheduler.SparkListenerBus.doPostEvent$(SparkListenerBus.scala:28)
        at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:38)
        at org.apache.spark.scheduler.AsyncEventQueue.doPostEvent(AsyncEventQueue.scala:38)
        at org.apache.spark.util.ListenerBus.postToAll(ListenerBus.scala:115)
        at org.apache.spark.util.ListenerBus.postToAll$(ListenerBus.scala:99)
        at org.apache.spark.scheduler.AsyncEventQueue.super$postToAll(AsyncEventQueue.scala:116)
        at org.apache.spark.scheduler.AsyncEventQueue.$anonfun$dispatch$1(AsyncEventQueue.scala:116)
        at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
        at org.apache.spark.scheduler.AsyncEventQueue.org$apache$spark$scheduler$AsyncEventQueue$$dispatch(AsyncEventQueue.scala:102)
        at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.$anonfun$run$1(AsyncEventQueue.scala:97)
        at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1320)
        at org.apache.spark.scheduler.AsyncEventQueue$$anon$2.run(AsyncEventQueue.scala:97)
```

### Why are the changes needed?
To avoid throwing the java.util.NoSuchElementException

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
This is a protective patch and it's not easy to reproduce in UT due to the event order is not fixed in a async queue.

Closes #31025 from LantaoJin/SPARK-34000.

Authored-by: LantaoJin <jinlantao@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit a7d3fcd354289c1d0f5c80887b4f33beb3ad96a2)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/ExecutorAllocationManager.scala  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index a6d2849f4fbbe..fffa7fd498b96 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -618,7 +618,11 @@ private[spark] class ExecutorAllocationManager(
 
         if (taskEnd.taskInfo.speculative) {
           stageAttemptToSpeculativeTaskIndices.get(stageAttempt).foreach {_.remove{taskIndex}}
-          stageAttemptToNumSpeculativeTasks(stageAttempt) -= 1
+          // If the previous task attempt succeeded first and it was the last task in a stage,
+          // the stage may have been removed before handing this speculative TaskEnd event.
+          if (stageAttemptToNumSpeculativeTasks.contains(stageAttempt)) {
+            stageAttemptToNumSpeculativeTasks(stageAttempt) -= 1
+          }
         }
 
         taskEnd.reason match {

From 7a2f4da27207de64759bc02e3d3fce1e7b5be10a Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 5 Jan 2021 19:48:10 +0900
Subject: [PATCH 1245/1280] [SPARK-34010][SQL][DODCS] Use python3 instead of
 python in SQL documentation build

### What changes were proposed in this pull request?

This PR proposes to use python3 instead of python in SQL documentation build.
After SPARK-29672, we use `sql/create-docs.sh` everywhere in Spark dev. We should fix it in `sql/create-docs.sh` too.
This blocks release because the release container does not have `python` but only `python3`.

### Why are the changes needed?

To unblock the release.

### Does this PR introduce _any_ user-facing change?

No, dev-only.

### How was this patch tested?

I manually ran the script

Closes #31041 from HyukjinKwon/SPARK-34010.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 8d09f9649510bf5d812c82b04f7711b9252a7db0)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 sql/create-docs.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/create-docs.sh b/sql/create-docs.sh
index 6614c714e90c7..8721df874ee73 100755
--- a/sql/create-docs.sh
+++ b/sql/create-docs.sh
@@ -27,14 +27,14 @@ set -e
 FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
 SPARK_HOME="$(cd "`dirname "${BASH_SOURCE[0]}"`"/..; pwd)"
 
-if ! hash python 2>/dev/null; then
-  echo "Missing python in your path, skipping SQL documentation generation."
+if ! hash python3 2>/dev/null; then
+  echo "Missing python3 in your path, skipping SQL documentation generation."
   exit 0
 fi
 
 if ! hash mkdocs 2>/dev/null; then
   echo "Missing mkdocs in your path, trying to install mkdocs for SQL documentation generation."
-  pip install mkdocs
+  pip3 install mkdocs
 fi
 
 pushd "$FWDIR" > /dev/null

From 1179b8b46c15ccbde932206f65cde7ff51842a98 Mon Sep 17 00:00:00 2001
From: Tanel Kiis <tanel.kiis@reach-u.com>
Date: Tue, 5 Jan 2021 11:23:42 -0800
Subject: [PATCH 1246/1280] [SPARK-33935][SQL][3.0] Fix CBO cost function

### What changes were proposed in this pull request?

Changed the cost function in CBO to match documentation.

### Why are the changes needed?

The parameter `spark.sql.cbo.joinReorder.card.weight` is documented as:
```
The weight of cardinality (number of rows) for plan cost comparison in join reorder: rows * weight + size * (1 - weight).
```
The implementation in `JoinReorderDP.betterThan` does not match this documentaiton:
```
def betterThan(other: JoinPlan, conf: SQLConf): Boolean = {
      if (other.planCost.card == 0 || other.planCost.size == 0) {
        false
      } else {
        val relativeRows = BigDecimal(this.planCost.card) / BigDecimal(other.planCost.card)
        val relativeSize = BigDecimal(this.planCost.size) / BigDecimal(other.planCost.size)
        relativeRows * conf.joinReorderCardWeight +
          relativeSize * (1 - conf.joinReorderCardWeight) < 1
      }
    }
```

This different implementation has an unfortunate consequence:
given two plans A and B, both A betterThan B and B betterThan A might give the same results. This happes when one has many rows with small sizes and other has few rows with large sizes.

A example values, that have this fenomen with the default weight value (0.7):
A.card = 500, B.card = 300
A.size = 30, B.size = 80
Both A betterThan B and B betterThan A would have score above 1 and would return false.

This happens with several of the TPCDS queries.

The new implementation does not have this behavior.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

New and existing UTs

Closes #31042 from tanelk/SPARK-33935_cbo_cost_function_3.0.

Authored-by: Tanel Kiis <tanel.kiis@reach-u.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../catalyst/optimizer/CostBasedJoinReorder.scala | 13 +++++--------
 .../sql/catalyst/optimizer/JoinReorderSuite.scala | 15 +++++++++++++++
 .../optimizer/StarJoinCostBasedReorderSuite.scala |  8 ++++----
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
index f92d8f5b8e534..93c608dc71a24 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/CostBasedJoinReorder.scala
@@ -344,14 +344,11 @@ object JoinReorderDP extends PredicateHelper with Logging {
     }
 
     def betterThan(other: JoinPlan, conf: SQLConf): Boolean = {
-      if (other.planCost.card == 0 || other.planCost.size == 0) {
-        false
-      } else {
-        val relativeRows = BigDecimal(this.planCost.card) / BigDecimal(other.planCost.card)
-        val relativeSize = BigDecimal(this.planCost.size) / BigDecimal(other.planCost.size)
-        relativeRows * conf.joinReorderCardWeight +
-          relativeSize * (1 - conf.joinReorderCardWeight) < 1
-      }
+      val thisCost = BigDecimal(this.planCost.card) * conf.joinReorderCardWeight +
+        BigDecimal(this.planCost.size) * (1 - conf.joinReorderCardWeight)
+      val otherCost = BigDecimal(other.planCost.card) * conf.joinReorderCardWeight +
+        BigDecimal(other.planCost.size) * (1 - conf.joinReorderCardWeight)
+      thisCost < otherCost
     }
   }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
index 36db2e2dd0ae2..75fe3dd4062a7 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.optimizer
 import org.apache.spark.sql.catalyst.dsl.expressions._
 import org.apache.spark.sql.catalyst.dsl.plans._
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap}
+import org.apache.spark.sql.catalyst.optimizer.JoinReorderDP.JoinPlan
 import org.apache.spark.sql.catalyst.plans.{Cross, Inner, PlanTest}
 import org.apache.spark.sql.catalyst.plans.logical.{ColumnStat, LocalRelation, LogicalPlan}
 import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -385,4 +386,18 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
   private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
     plans.map(_.output).reduce(_ ++ _)
   }
+
+  test("SPARK-33935: betterThan should be consistent") {
+    val plan1 = JoinPlan(null, null, null, Cost(300, 80))
+    val plan2 = JoinPlan(null, null, null, Cost(500, 30))
+
+    // cost1 = 300*0.7 + 80*0.3 = 234
+    // cost2 = 500*0.7 + 30*0.3 = 359
+
+    assert(!plan1.betterThan(plan1, conf))
+    assert(!plan2.betterThan(plan2, conf))
+
+    assert(plan1.betterThan(plan2, conf))
+    assert(!plan2.betterThan(plan1, conf))
+  }
 }
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
index 5b8e59ae7cb31..b7cf383732923 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/StarJoinCostBasedReorderSuite.scala
@@ -293,12 +293,12 @@ class StarJoinCostBasedReorderSuite extends PlanTest with StatsEstimationTestBas
           (nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
 
     val expected =
-      f1.join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
-        .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk")))
-        .join(t3.join(t4, Inner, Some(nameToAttr("t3_c1") === nameToAttr("t4_c1"))), Inner,
-          Some(nameToAttr("t3_c1") === nameToAttr("t4_c1")))
+      t3.join(t4, Inner, Some(nameToAttr("t3_c1") === nameToAttr("t4_c1")))
         .join(t1.join(t2, Inner, Some(nameToAttr("t1_c1") === nameToAttr("t2_c1"))), Inner,
           Some(nameToAttr("t1_c2") === nameToAttr("t4_c2")))
+        .join(f1
+          .join(d2, Inner, Some(nameToAttr("f1_fk2") === nameToAttr("d2_pk")))
+          .join(d1, Inner, Some(nameToAttr("f1_fk1") === nameToAttr("d1_pk"))))
         .select(outputsOf(d1, t1, t2, t3, t4, f1, d2): _*)
 
     assertEqualPlans(query, expected)

From 9ba6db94a2fef23053f208b0b740b8b0a1cc7d88 Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Tue, 5 Jan 2021 19:45:09 +0000
Subject: [PATCH 1247/1280] [SPARK-33844][SQL][3.0] InsertIntoHiveDir command
 should check col name too
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

In hive-1.2.1, hive serde just split `serdeConstants.LIST_COLUMNS` and `serdeConstants.LIST_COLUMN_TYPES` use comma.

When we use spark 2.4 with UT
```
  test("insert overwrite directory with comma col name") {
    withTempDir { dir =>
      val path = dir.toURI.getPath

      val v1 =
        s"""
           | INSERT OVERWRITE DIRECTORY '${path}'
           | STORED AS TEXTFILE
           | SELECT 1 as a, 'c' as b, if(1 = 1, "true", "false")
         """.stripMargin

      sql(v1).explain(true)

      sql(v1).show()
    }
  }
```
failed with as below since column name contains `,` then column names and column types size not equal.
```
19:56:05.618 ERROR org.apache.spark.sql.execution.datasources.FileFormatWriter:  [ angerszhu ] Aborting job dd774f18-93fa-431f-9468-3534c7d8acda.
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost, executor driver): org.apache.hadoop.hive.serde2.SerDeException: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe: columns has 5 elements while columns.types has 3 elements!
	at org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters.extractColumnInfo(LazySerDeParameters.java:145)
	at org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters.<init>(LazySerDeParameters.java:85)
	at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.initialize(LazySimpleSerDe.java:125)
	at org.apache.spark.sql.hive.execution.HiveOutputWriter.<init>(HiveFileFormat.scala:119)
	at org.apache.spark.sql.hive.execution.HiveFileFormat$$anon$1.newInstance(HiveFileFormat.scala:103)
	at org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.newOutputWriter(FileFormatDataWriter.scala:120)
	at org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.<init>(FileFormatDataWriter.scala:108)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:287)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:219)
	at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:218)
	at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:121)
	at org.apache.spark.executor.Executor$TaskRunner$$anonfun$12.apply(Executor.scala:461)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:467)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

```

After hive-2.3 we will set COLUMN_NAME_DELIMITER to special char when col name cntains ','：
https://github.com/apache/hive/blob/6f4c35c9e904d226451c465effdc5bfd31d395a0/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java#L1180-L1188
https://github.com/apache/hive/blob/6f4c35c9e904d226451c465effdc5bfd31d395a0/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java#L1044-L1075

And in script transform, we parse column name  to avoid this problem
https://github.com/apache/spark/blob/554600c2af0dbc8979955807658fafef5dc66c08/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationExec.scala#L257-L261

So I think in `InsertIntoHiveDirComman`, we should do same thing too. And I have verified this method can make spark-2.4 work well.

### Why are the changes needed?
More save use serde

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?

Closes #31038 from AngersZhuuuu/SPARK-33844-3.0.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../hive/execution/InsertIntoHiveDirCommand.scala  |  9 +++++++--
 .../spark/sql/hive/execution/HiveDDLSuite.scala    | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
index b66c302a7d7ea..7ef637ed553ad 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveDirCommand.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.execution.command.DDLUtils
 import org.apache.spark.sql.hive.client.HiveClientImpl
 import org.apache.spark.sql.util.SchemaUtils
 
@@ -63,12 +64,16 @@ case class InsertIntoHiveDirCommand(
       s"when inserting into ${storage.locationUri.get}",
       sparkSession.sessionState.conf.caseSensitiveAnalysis)
 
-    val hiveTable = HiveClientImpl.toHiveTable(CatalogTable(
+    val table = CatalogTable(
       identifier = TableIdentifier(storage.locationUri.get.toString, Some("default")),
+      provider = Some(DDLUtils.HIVE_PROVIDER),
       tableType = org.apache.spark.sql.catalyst.catalog.CatalogTableType.VIEW,
       storage = storage,
       schema = outputColumns.toStructType
-    ))
+    )
+    DDLUtils.checkDataColNames(table)
+
+    val hiveTable = HiveClientImpl.toHiveTable(table)
     hiveTable.getMetadata.put(serdeConstants.SERIALIZATION_LIB,
       storage.serde.getOrElse(classOf[LazySimpleSerDe].getName))
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 765119dec04b3..c8726c7ae4bc9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -2766,4 +2766,18 @@ class HiveDDLSuite
       checkAnswer(sql("SHOW PARTITIONS t"), Seq.empty)
     }
   }
+
+  test("SPARK-33844: Insert overwrite directory should check schema too") {
+    withView("v") {
+      spark.range(1).createTempView("v")
+      withTempPath { path =>
+        val e = intercept[AnalysisException] {
+          spark.sql(s"INSERT OVERWRITE LOCAL DIRECTORY '${path.getCanonicalPath}' " +
+            s"STORED AS PARQUET SELECT ID, if(1=1, 1, 0), abs(id), '^-' FROM v")
+        }.getMessage
+        assert(e.contains("Attribute name \"(IF((1 = 1), 1, 0))\" contains" +
+          " invalid character(s) among \" ,;{}()\\n\\t=\". Please use alias to rename it."))
+      }
+    }
+  }
 }

From 98cb0cda4101111130a6c9bc244632acd02826a3 Mon Sep 17 00:00:00 2001
From: "Jungtaek Lim (HeartSaVioR)" <kabhwan.opensource@gmail.com>
Date: Tue, 5 Jan 2021 21:59:49 -0800
Subject: [PATCH 1248/1280] [SPARK-33635][SS] Adjust the order of check in
 KafkaTokenUtil.needTokenUpdate to remedy perf regression

### What changes were proposed in this pull request?

This PR proposes to adjust the order of check in KafkaTokenUtil.needTokenUpdate, so that short-circuit applies on the non-delegation token cases (insecure + secured without delegation token) and remedies the performance regression heavily.

### Why are the changes needed?

There's a serious performance regression between Spark 2.4 vs Spark 3.0 on read path against Kafka data source.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Manually ran a reproducer (https://github.com/codegorillauk/spark-kafka-read with modification to just count instead of writing to Kafka topic) with measuring the time.

> the branch applying the change with adding measurement

https://github.com/HeartSaVioR/spark/commits/debug-SPARK-33635-v3.0.1

> the branch only adding measurement

https://github.com/HeartSaVioR/spark/commits/debug-original-ver-SPARK-33635-v3.0.1

> the result (before the fix)

count: 10280000
Took 41.634007047 secs

21/01/06 13:16:07 INFO KafkaDataConsumer: debug ver. 17-original
21/01/06 13:16:07 INFO KafkaDataConsumer: Total time taken to retrieve: 82118 ms

> the result (after the fix)

count: 10280000
Took 7.964058475 secs

21/01/06 13:08:22 INFO KafkaDataConsumer: debug ver. 17
21/01/06 13:08:22 INFO KafkaDataConsumer: Total time taken to retrieve: 987 ms

Closes #31056 from HeartSaVioR/SPARK-33635.

Authored-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit fa9309001a47a2b87f7a735f964537886ed9bd4c)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
index 49109d363ba8f..370e727bde05e 100644
--- a/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
+++ b/external/kafka-0-10-token-provider/src/main/scala/org/apache/spark/kafka010/KafkaTokenUtil.scala
@@ -295,8 +295,8 @@ private[spark] object KafkaTokenUtil extends Logging {
       sparkConf: SparkConf,
       params: ju.Map[String, Object],
       clusterConfig: Option[KafkaTokenClusterConf]): Boolean = {
-    if (HadoopDelegationTokenManager.isServiceEnabled(sparkConf, "kafka") &&
-        clusterConfig.isDefined && params.containsKey(SaslConfigs.SASL_JAAS_CONFIG)) {
+    if (clusterConfig.isDefined && params.containsKey(SaslConfigs.SASL_JAAS_CONFIG) &&
+        HadoopDelegationTokenManager.isServiceEnabled(sparkConf, "kafka")) {
       logDebug("Delegation token used by connector, checking if uses the latest token.")
       val connectorJaasParams = params.get(SaslConfigs.SASL_JAAS_CONFIG).asInstanceOf[String]
       getTokenJaasParams(clusterConfig.get) != connectorJaasParams

From 403bca4b4a03dbb0e4dfbd8ecd48b82124c66dc4 Mon Sep 17 00:00:00 2001
From: Baohe Zhang <baohe.zhang@verizonmedia.com>
Date: Wed, 6 Jan 2021 00:51:57 -0800
Subject: [PATCH 1249/1280] [SPARK-33029][CORE][WEBUI][3.0] Fix the UI executor
 page incorrectly marking the driver as blacklisted

This is a backport of #30954

### What changes were proposed in this pull request?
Filter out the driver entity when updating the exclusion status of live executors(including the driver), so the driver won't be marked as blacklisted in the UI even if the node that hosts the driver has been marked as blacklisted.

### Why are the changes needed?
Before this change, if we run spark with the standalone mode and with spark.blacklist.enabled=true. The driver will be marked as blacklisted when the host that hosts that driver has been marked as blacklisted. While it's incorrect because the exclude list feature will exclude executors only and the driver is still active.
![image](https://user-images.githubusercontent.com/26694233/103732959-3494c180-4fae-11eb-9da0-2c906309ea83.png)
After the fix, the driver won't be marked as blacklisted.
![image](https://user-images.githubusercontent.com/26694233/103732974-3fe7ed00-4fae-11eb-90d1-7ee44d4ed7c9.png)

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Manual test. Reopen the UI and see the driver is no longer marked as blacklisted.

Closes #31057 from baohe-zhang/SPARK-33029-3.0.

Authored-by: Baohe Zhang <baohe.zhang@verizonmedia.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../scala/org/apache/spark/status/AppStatusListener.scala | 8 +++++---
 .../executor_memory_usage_expectation.json                | 2 +-
 .../executor_node_blacklisting_expectation.json           | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index c3f22f32993a8..dfa4f92f8b87e 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -281,10 +281,12 @@ private[spark] class AppStatusListener(
 
     // Implicitly blacklist every available executor for the stage associated with this node
     Option(liveStages.get((event.stageId, event.stageAttemptId))).foreach { stage =>
-      val executorIds = liveExecutors.values.filter(_.host == event.hostId).map(_.executorId).toSeq
+      val executorIds = liveExecutors.values.filter(exec => exec.host == event.hostId
+        && exec.executorId != SparkContext.DRIVER_IDENTIFIER).map(_.executorId).toSeq
       setStageBlackListStatus(stage, now, executorIds: _*)
     }
-    liveExecutors.values.filter(_.hostname == event.hostId).foreach { exec =>
+    liveExecutors.values.filter(exec => exec.hostname == event.hostId
+      && exec.executorId != SparkContext.DRIVER_IDENTIFIER).foreach { exec =>
       addBlackListedStageTo(exec, event.stageId, now)
     }
   }
@@ -333,7 +335,7 @@ private[spark] class AppStatusListener(
 
     // Implicitly (un)blacklist every executor associated with the node.
     liveExecutors.values.foreach { exec =>
-      if (exec.hostname == host) {
+      if (exec.hostname == host && exec.executorId != SparkContext.DRIVER_IDENTIFIER) {
         exec.isBlacklisted = blacklisted
         liveUpdate(exec, now)
       }
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index 7c3f77d8c10cf..388e89ceff054 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -16,7 +16,7 @@
   "totalInputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
+  "isBlacklisted" : false,
   "maxMemory" : 908381388,
   "addTime" : "2016-11-16T22:33:31.477GMT",
   "executorLogs" : { },
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
index 0986e85f16b3e..d030d47b2e8cc 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -16,7 +16,7 @@
   "totalInputBytes" : 0,
   "totalShuffleRead" : 0,
   "totalShuffleWrite" : 0,
-  "isBlacklisted" : true,
+  "isBlacklisted" : false,
   "maxMemory" : 908381388,
   "addTime" : "2016-11-16T22:33:31.477GMT",
   "executorLogs" : { },

From aaa3dcc2c9effde3dd3b4bbe04f7c06e299294cb Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Wed, 6 Jan 2021 20:57:03 +0900
Subject: [PATCH 1250/1280] [SPARK-34012][SQL][3.0] Keep behavior consistent
 when conf `spark.sqllegacy.parser.havingWithoutGroupByAsWhere` is true with
 migration guide

### What changes were proposed in this pull request?
In https://github.com/apache/spark/pull/22696 we support HAVING without GROUP BY means global aggregate
But since we treat having as Filter before, in this way will cause a lot of analyze error, after https://github.com/apache/spark/pull/28294 we use `UnresolvedHaving` to instead `Filter` to solve such problem, but break origin logical about treat `SELECT 1 FROM range(10) HAVING true` as `SELECT 1 FROM range(10) WHERE true`   .
This PR fix this issue and add UT.

NOTE: This backport comes from https://github.com/apache/spark/pull/31039

### Why are the changes needed?
Keep consistent behavior of migration guide.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
added UT

Closes #31049 from AngersZhuuuu/SPARK-34012-3.0.

Authored-by: angerszhu <angers.zhu@gmail.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../sql/catalyst/parser/AstBuilder.scala      |  6 +-
 .../resources/sql-tests/inputs/group-by.sql   | 10 +++
 .../sql-tests/results/group-by.sql.out        | 63 ++++++++++++++++++-
 3 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 938976eb74ab9..2fcee5a9e86e4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -723,7 +723,11 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
     val withProject = if (aggregationClause == null && havingClause != null) {
       if (conf.getConf(SQLConf.LEGACY_HAVING_WITHOUT_GROUP_BY_AS_WHERE)) {
         // If the legacy conf is set, treat HAVING without GROUP BY as WHERE.
-        withHavingClause(havingClause, createProject())
+        val predicate = expression(havingClause.booleanExpression) match {
+          case p: Predicate => p
+          case e => Cast(e, BooleanType)
+        }
+        Filter(predicate, createProject())
       } else {
         // According to SQL standard, HAVING without GROUP BY means global aggregate.
         withHavingClause(havingClause, Aggregate(Nil, namedExpressions, withFilter))
diff --git a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
index fedf03d774e42..3f5f5568b0720 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/group-by.sql
@@ -86,6 +86,16 @@ SELECT 1 FROM range(10) HAVING MAX(id) > 0;
 
 SELECT id FROM range(10) HAVING id > 0;
 
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true;
+
+SELECT 1 FROM range(10) HAVING true;
+
+SELECT 1 FROM range(10) HAVING MAX(id) > 0;
+
+SELECT id FROM range(10) HAVING id > 0;
+
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false;
+
 -- Test data
 CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES
   (1, true), (1, false),
diff --git a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
index 50eb2a9f22f69..e5b705895340e 100644
--- a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out
@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 56
+-- Number of queries: 61
 
 
 -- !query
@@ -277,6 +277,67 @@ org.apache.spark.sql.AnalysisException
 grouping expressions sequence is empty, and '`id`' is not an aggregate function. Wrap '()' in windowing function(s) or wrap '`id`' in first() (or first_value) if you don't care which value you get.;
 
 
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=true
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	true
+
+
+-- !query
+SELECT 1 FROM range(10) HAVING true
+-- !query schema
+struct<1:int>
+-- !query output
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+
+
+-- !query
+SELECT 1 FROM range(10) HAVING MAX(id) > 0
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+
+Aggregate/Window/Generate expressions are not valid in where clause of the query.
+Expression in where clause: [(max(`id`) > CAST(0 AS BIGINT))]
+Invalid expressions: [max(`id`)];
+
+
+-- !query
+SELECT id FROM range(10) HAVING id > 0
+-- !query schema
+struct<id:bigint>
+-- !query output
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+
+-- !query
+SET spark.sql.legacy.parser.havingWithoutGroupByAsWhere=false
+-- !query schema
+struct<key:string,value:string>
+-- !query output
+spark.sql.legacy.parser.havingWithoutGroupByAsWhere	false
+
+
 -- !query
 CREATE OR REPLACE TEMPORARY VIEW test_agg AS SELECT * FROM VALUES
   (1, true), (1, false),

From c9c3d6faaa3b5f6656caaeb14a152dd77404cbc0 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 6 Jan 2021 05:00:57 -0800
Subject: [PATCH 1251/1280] [SPARK-34011][SQL][3.1][3.0] Refresh cache in
 `ALTER TABLE .. RENAME TO PARTITION`

### What changes were proposed in this pull request?
1. Invoke `refreshTable()` from `AlterTableRenamePartitionCommand.run()` after partitions renaming. In particular, this re-creates the cache associated with the modified table.
2. Refresh the cache associated with tables from v2 table catalogs in the `ALTER TABLE .. RENAME TO PARTITION` command.

### Why are the changes needed?
This fixes the issues portrayed by the example:
```sql
spark-sql> CREATE TABLE tbl1 (col0 int, part0 int) USING parquet PARTITIONED BY (part0);
spark-sql> INSERT INTO tbl1 PARTITION (part0=0) SELECT 0;
spark-sql> INSERT INTO tbl1 PARTITION (part0=1) SELECT 1;
spark-sql> CACHE TABLE tbl1;
spark-sql> SELECT * FROM tbl1;
0	0
1	1
spark-sql> ALTER TABLE tbl1 PARTITION (part0=0) RENAME TO PARTITION (part=2);
spark-sql> SELECT * FROM tbl1;
0	0
1	1
```
The last query must not return `0	2` since `0  0` was renamed by previous command.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes for the example above:
```sql
...
spark-sql> ALTER TABLE tbl1 PARTITION (part=0) RENAME TO PARTITION (part=2);
spark-sql> SELECT * FROM tbl1;
0	2
1	1
```

### How was this patch tested?
By running the affected test suite:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *CachedTableSuite"
```

Closes #31060 from MaxGekk/rename-partition-refresh-cache-3.1.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit f18d68a3d59fc82a3611ce92cccfd9b52df29360)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../apache/spark/sql/execution/command/ddl.scala  |  1 +
 .../org/apache/spark/sql/CachedTableSuite.scala   | 15 +++++++++++++++
 .../apache/spark/sql/hive/CachedTableSuite.scala  | 15 +++++++++++++++
 3 files changed, 31 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 748bb1b35384a..11e695ad1d1cf 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -535,6 +535,7 @@ case class AlterTableRenamePartitionCommand(
 
     catalog.renamePartitions(
       tableName, Seq(normalizedOldPartition), Seq(normalizedNewPartition))
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     Seq.empty[Row]
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index 0e8122ef88edf..b535734bf2310 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1285,4 +1285,19 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
     }
   }
+
+  test("SPARK-34011: refresh cache after partition renaming") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 2), Row(1, 1)))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index ed3ccb4afd255..dc909fd3d3ef2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -454,4 +454,19 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
     }
   }
+
+  test("SPARK-34011: refresh cache after partition renaming") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      sql("ALTER TABLE t PARTITION (part=0) RENAME TO PARTITION (part=2)")
+      assert(spark.catalog.isCached("t"))
+      QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 2), Row(1, 1)))
+    }
+  }
 }

From e7d53449f198bd8c5ee97d58f285994e31ea2d1a Mon Sep 17 00:00:00 2001
From: fwang12 <fwang12@ebay.com>
Date: Fri, 8 Jan 2021 10:44:12 +0900
Subject: [PATCH 1252/1280] [SPARK-33100][SQL][3.0] Ignore a semicolon inside a
 bracketed comment in spark-sql

### What changes were proposed in this pull request?
Now the spark-sql does not support parse the sql statements with bracketed comments.
For the sql statements:
```
/* SELECT 'test'; */
SELECT 'test';
```
Would be split to two statements:
The first one: `/* SELECT 'test'`
The second one: `*/ SELECT 'test'`

Then it would throw an exception because the first one is illegal.
In this PR, we ignore the content in bracketed comments while splitting the sql statements.
Besides, we ignore the comment without any content.

NOTE: This backport comes from https://github.com/apache/spark/pull/29982

### Why are the changes needed?
Spark-sql might split the statements inside bracketed comments and it is not correct.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Added UT.

Closes #31033 from turboFei/SPARK-33100.

Authored-by: fwang12 <fwang12@ebay.com>
Signed-off-by: Takeshi Yamamuro <yamamuro@apache.org>
---
 .../hive/thriftserver/SparkSQLCLIDriver.scala | 50 ++++++++++++++++---
 .../sql/hive/thriftserver/CliSuite.scala      | 23 +++++++++
 2 files changed, 65 insertions(+), 8 deletions(-)

diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 6abb905eaeeeb..581aa68b6ba3f 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -518,15 +518,32 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
   // Note: [SPARK-31595] if there is a `'` in a double quoted string, or a `"` in a single quoted
   // string, the origin implementation from Hive will not drop the trailing semicolon as expected,
   // hence we refined this function a little bit.
+  // Note: [SPARK-33100] Ignore a semicolon inside a bracketed comment in spark-sql.
   private def splitSemiColon(line: String): JList[String] = {
     var insideSingleQuote = false
     var insideDoubleQuote = false
-    var insideComment = false
+    var insideSimpleComment = false
+    var bracketedCommentLevel = 0
     var escape = false
     var beginIndex = 0
+    var leavingBracketedComment = false
+    var isStatement = false
     val ret = new JArrayList[String]
 
+    def insideBracketedComment: Boolean = bracketedCommentLevel > 0
+    def insideComment: Boolean = insideSimpleComment || insideBracketedComment
+    def statementInProgress(index: Int): Boolean = isStatement || (!insideComment &&
+      index > beginIndex && !s"${line.charAt(index)}".trim.isEmpty)
+
     for (index <- 0 until line.length) {
+      // Checks if we need to decrement a bracketed comment level; the last character '/' of
+      // bracketed comments is still inside the comment, so `insideBracketedComment` must keep true
+      // in the previous loop and we decrement the level here if needed.
+      if (leavingBracketedComment) {
+        bracketedCommentLevel -= 1
+        leavingBracketedComment = false
+      }
+
       if (line.charAt(index) == '\'' && !insideComment) {
         // take a look to see if it is escaped
         // See the comment above about SPARK-31595
@@ -549,21 +566,34 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
           // Sample query: select "quoted value --"
           //                                    ^^ avoids starting a comment if it's inside quotes.
         } else if (hasNext && line.charAt(index + 1) == '-') {
-          // ignore quotes and ;
-          insideComment = true
+          // ignore quotes and ; in simple comment
+          insideSimpleComment = true
         }
       } else if (line.charAt(index) == ';') {
         if (insideSingleQuote || insideDoubleQuote || insideComment) {
           // do not split
         } else {
-          // split, do not include ; itself
-          ret.add(line.substring(beginIndex, index))
+          if (isStatement) {
+            // split, do not include ; itself
+            ret.add(line.substring(beginIndex, index))
+          }
           beginIndex = index + 1
+          isStatement = false
         }
       } else if (line.charAt(index) == '\n') {
-        // with a new line the inline comment should end.
+        // with a new line the inline simple comment should end.
         if (!escape) {
-          insideComment = false
+          insideSimpleComment = false
+        }
+      } else if (line.charAt(index) == '/' && !insideSimpleComment) {
+        val hasNext = index + 1 < line.length
+        if (insideSingleQuote || insideDoubleQuote) {
+          // Ignores '/' in any case of quotes
+        } else if (insideBracketedComment && line.charAt(index - 1) == '*' ) {
+          // Decrements `bracketedCommentLevel` at the beginning of the next loop
+          leavingBracketedComment = true
+        } else if (hasNext && !insideBracketedComment && line.charAt(index + 1) == '*') {
+          bracketedCommentLevel += 1
         }
       }
       // set the escape
@@ -572,8 +602,12 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
       } else if (line.charAt(index) == '\\') {
         escape = true
       }
+
+      isStatement = statementInProgress(index)
+    }
+    if (isStatement) {
+      ret.add(line.substring(beginIndex))
     }
-    ret.add(line.substring(beginIndex))
     ret
   }
 }
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
index ea1a371151c36..4a075d301b601 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala
@@ -551,4 +551,27 @@ class CliSuite extends SparkFunSuite with BeforeAndAfterAll with Logging {
       errorResponses = Seq("AnalysisException"))(
       ("", "Error in query: The second argument of 'date_sub' function needs to be an integer."))
   }
+
+  test("SPARK-33100: Ignore a semicolon inside a bracketed comment in spark-sql") {
+    runCliWithin(4.minute)(
+      "/* SELECT 'test';*/ SELECT 'test';" -> "test",
+      ";;/* SELECT 'test';*/ SELECT 'test';" -> "test",
+      "/* SELECT 'test';*/;; SELECT 'test';" -> "test",
+      "SELECT 'test'; -- SELECT 'test';" -> "test",
+      "SELECT 'test'; /* SELECT 'test';*/;" -> "test",
+      "/*$meta chars{^\\;}*/ SELECT 'test';" -> "test",
+      "/*\nmulti-line\n*/ SELECT 'test';" -> "test",
+      "/*/* multi-level bracketed*/ SELECT 'test';" -> "test"
+    )
+  }
+
+  test("SPARK-33100: test sql statements with hint in bracketed comment") {
+    runCliWithin(2.minute)(
+      "CREATE TEMPORARY VIEW t1 AS SELECT * FROM VALUES(1, 2) AS t1(k, v);" -> "",
+      "CREATE TEMPORARY VIEW t2 AS SELECT * FROM VALUES(2, 1) AS t2(k, v);" -> "",
+      "EXPLAIN SELECT /*+ MERGEJOIN(t1) */ t1.* FROM t1 JOIN t2 ON t1.k = t2.v;" -> "SortMergeJoin",
+      "EXPLAIN SELECT /* + MERGEJOIN(t1) */ t1.* FROM t1 JOIN t2 ON t1.k = t2.v;"
+        -> "BroadcastHashJoin"
+    )
+  }
 }

From 471a0896198f878a1adc974e2c72da081001d964 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 11 Jan 2021 09:36:16 +0900
Subject: [PATCH 1253/1280] [SPARK-34055][SQL][3.0] Refresh cache in `ALTER
 TABLE .. ADD PARTITION`

### What changes were proposed in this pull request?
Invoke `refreshTable()` from `CatalogImpl` which refreshes the cache in v1 `ALTER TABLE .. ADD PARTITION`.

### Why are the changes needed?
This fixes the issues portrayed by the example:
```sql
spark-sql> create table tbl (col int, part int) using parquet partitioned by (part);
spark-sql> insert into tbl partition (part=0) select 0;
spark-sql> cache table tbl;
spark-sql> select * from tbl;
0	0
spark-sql> show table extended like 'tbl' partition(part=0);
default	tbl	false	Partition Values: [part=0]
Location: file:/Users/maximgekk/proj/add-partition-refresh-cache-2/spark-warehouse/tbl/part=0
...
```
Create new partition by copying the existing one:
```
$ cp -r /Users/maximgekk/proj/add-partition-refresh-cache-2/spark-warehouse/tbl/part=0 /Users/maximgekk/proj/add-partition-refresh-cache-2/spark-warehouse/tbl/part=1
```
```sql
spark-sql> alter table tbl add partition (part=1) location '/Users/maximgekk/proj/add-partition-refresh-cache-2/spark-warehouse/tbl/part=1';
spark-sql> select * from tbl;
0	0
```

The last query must return `0	1` since it has been added by `ALTER TABLE .. ADD PARTITION`.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes for the example above:
```sql
...
spark-sql> alter table tbl add partition (part=1) location '/Users/maximgekk/proj/add-partition-refresh-cache-2/spark-warehouse/tbl/part=1';
spark-sql> select * from tbl;
0	0
0	1
```

### How was this patch tested?
By running the affected test suite:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *CachedTableSuite"
```

Closes #31116 from MaxGekk/add-partition-refresh-cache-2-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../spark/sql/execution/command/ddl.scala     |  1 +
 .../apache/spark/sql/CachedTableSuite.scala   | 31 +++++++++++++++++++
 .../spark/sql/hive/CachedTableSuite.scala     | 29 +++++++++++++++++
 3 files changed, 61 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 11e695ad1d1cf..1380e1e1203cd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -484,6 +484,7 @@ case class AlterTableAddPartitionCommand(
       catalog.createPartitions(table.identifier, batch, ignoreIfExists = ifNotExists)
     }
 
+    sparkSession.catalog.refreshTable(table.identifier.quotedString)
     if (table.stats.nonEmpty) {
       if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
         val addedSize = CommandUtils.calculateTotalLocationSize(sparkSession, table.identifier,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index b535734bf2310..de8d595ff52f2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -17,9 +17,13 @@
 
 package org.apache.spark.sql
 
+import java.io.File
+
 import scala.collection.mutable.HashSet
 import scala.concurrent.duration._
 
+import org.apache.commons.io.FileUtils
+
 import org.apache.spark.CleanerListener
 import org.apache.spark.executor.DataReadMethod._
 import org.apache.spark.executor.DataReadMethod.DataReadMethod
@@ -1300,4 +1304,31 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 2), Row(1, 1)))
     }
   }
+
+  test("SPARK-34055: refresh cache in partition adding") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0)))
+
+      // Create new partition (part = 1) in the filesystem
+      val information = sql("SHOW TABLE EXTENDED LIKE 't' PARTITION (part = 0)")
+        .select("information")
+        .first().getString(0)
+      val part0Loc = information
+        .split("\\r?\\n")
+        .filter(_.startsWith("Location:"))
+        .head
+        .replace("Location: file:", "")
+      val part1Loc = part0Loc.replace("part=0", "part=1")
+      FileUtils.copyDirectory(new File(part0Loc), new File(part1Loc))
+
+      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index dc909fd3d3ef2..26918c5609456 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.hive
 
 import java.io.File
 
+import org.apache.commons.io.FileUtils
+
 import org.apache.spark.sql.{AnalysisException, Dataset, QueryTest, Row, SaveMode}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
@@ -469,4 +471,31 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       QueryTest.checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 2), Row(1, 1)))
     }
   }
+
+  test("SPARK-34055: refresh cache in partition adding") {
+    withTable("t") {
+      sql("CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0)))
+
+      // Create new partition (part = 1) in the filesystem
+      val information = sql("SHOW TABLE EXTENDED LIKE 't' PARTITION (part = 0)")
+        .select("information")
+        .first().getString(0)
+      val part0Loc = information
+        .split("\\r?\\n")
+        .filter(_.startsWith("Location:"))
+        .head
+        .replace("Location: file:", "")
+      val part1Loc = part0Loc.replace("part=0", "part=1")
+      FileUtils.copyDirectory(new File(part0Loc), new File(part1Loc))
+
+      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
+    }
+  }
 }

From 16cab5c305584fea5e806ff9ca4217f17deeac78 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 11 Jan 2021 04:57:59 +0000
Subject: [PATCH 1254/1280] [SPARK-33591][SQL][3.0] Recognize `null` in
 partition spec values

### What changes were proposed in this pull request?
1. Recognize `null` while parsing partition specs, and put `null` instead of `"null"` as partition values.
2. For V1 catalog: replace `null` by `__HIVE_DEFAULT_PARTITION__`.

### Why are the changes needed?
Currently, `null` in partition specs is recognized as the `"null"` string which could lead to incorrect results, for example:
```sql
spark-sql> CREATE TABLE tbl5 (col1 INT, p1 STRING) USING PARQUET PARTITIONED BY (p1);
spark-sql> INSERT INTO TABLE tbl5 PARTITION (p1 = null) SELECT 0;
spark-sql> SELECT isnull(p1) FROM tbl5;
false
```
Even we inserted a row to the partition with the `null` value, **the resulted table doesn't contain `null`**.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes, the example above works as expected:
```sql
spark-sql> SELECT isnull(p1) FROM tbl5;
true
```

### How was this patch tested?
By running the affected test suites:
```
$ build/sbt -Phive -Phive-thriftserver "test:testOnly *SQLQuerySuite"
$ build/sbt -Phive -Phive-thriftserver "test:testOnly *CatalogedDDLSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit 157b72ac9fa0057d5fd6d7ed52a6c4b22ebd1dfc)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #31095 from MaxGekk/partition-spec-value-null-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../catalog/ExternalCatalogUtils.scala        | 10 ++++++
 .../catalyst/catalog/InMemoryCatalog.scala    |  7 +++-
 .../sql/catalyst/catalog/SessionCatalog.scala |  2 +-
 .../sql/catalyst/parser/AstBuilder.scala      |  1 +
 .../org/apache/spark/sql/SQLQuerySuite.scala  |  9 +++++
 .../sql/execution/command/DDLSuite.scala      |  8 +++++
 .../spark/sql/hive/HiveExternalCatalog.scala  | 33 ++++++++++---------
 .../hive/execution/InsertIntoHiveTable.scala  |  2 ++
 8 files changed, 55 insertions(+), 17 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
index ae3b75dc3334b..9cb71bb51294c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogUtils.scala
@@ -159,6 +159,10 @@ object ExternalCatalogUtils {
     }
   }
 
+  private def isNullPartitionValue(value: String): Boolean = {
+    value == null || value == DEFAULT_PARTITION_NAME
+  }
+
   /**
    * Returns true if `spec1` is a partial partition spec w.r.t. `spec2`, e.g. PARTITION (a=1) is a
    * partial partition spec w.r.t. PARTITION (a=1,b=2).
@@ -167,9 +171,15 @@ object ExternalCatalogUtils {
       spec1: TablePartitionSpec,
       spec2: TablePartitionSpec): Boolean = {
     spec1.forall {
+      case (partitionColumn, value) if isNullPartitionValue(value) =>
+        isNullPartitionValue(spec2(partitionColumn))
       case (partitionColumn, value) => spec2(partitionColumn) == value
     }
   }
+
+  def convertNullPartitionValues(spec: TablePartitionSpec): TablePartitionSpec = {
+    spec.mapValues(v => if (v == null) DEFAULT_PARTITION_NAME else v).toMap
+  }
 }
 
 object CatalogUtils {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
index 31644a5ae4e35..c05950300279f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala
@@ -541,7 +541,12 @@ class InMemoryCatalog(
 
     listPartitions(db, table, partialSpec).map { partition =>
       partitionColumnNames.map { name =>
-        escapePathName(name) + "=" + escapePathName(partition.spec(name))
+        val partValue = if (partition.spec(name) == null) {
+          DEFAULT_PARTITION_NAME
+        } else {
+          escapePathName(partition.spec(name))
+        }
+        escapePathName(name) + "=" + partValue
       }.mkString("/")
     }.sorted
   }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index b79857cdccd22..389e410234154 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -1104,7 +1104,7 @@ class SessionCatalog(
    */
   private def requireNonEmptyValueInPartitionSpec(specs: Seq[TablePartitionSpec]): Unit = {
     specs.foreach { s =>
-      if (s.values.exists(_.isEmpty)) {
+      if (s.values.exists(v => v != null && v.isEmpty)) {
         val spec = s.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]")
         throw new AnalysisException(
           s"Partition spec is invalid. The spec ($spec) contains an empty partition column value")
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 2fcee5a9e86e4..91519c021481c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -520,6 +520,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
    */
   protected def visitStringConstant(ctx: ConstantContext): String = withOrigin(ctx) {
     ctx match {
+      case _: NullLiteralContext => null
       case s: StringLiteralContext => createString(s)
       case o => o.getText
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index fdad69d1f1bbd..8039c9b6f04b5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -3575,6 +3575,15 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
       }
     })
   }
+
+  test("SPARK-33591: null as a partition value") {
+    val t = "part_table"
+    withTable(t) {
+      sql(s"CREATE TABLE $t (col1 INT, p1 STRING) USING PARQUET PARTITIONED BY (p1)")
+      sql(s"INSERT INTO TABLE $t PARTITION (p1 = null) SELECT 0")
+      checkAnswer(sql(s"SELECT * FROM $t"), Row(0, null))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index cf464d1c37d4b..5986cdc78d6b4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -1731,6 +1731,14 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
     // use int literal as partition value for int type partition column
     sql("ALTER TABLE tab1 DROP PARTITION (a=9, b=9)")
     assert(catalog.listPartitions(tableIdent).isEmpty)
+
+    // null partition values
+    createTablePartition(catalog, Map("a" -> null, "b" -> null), tableIdent)
+    val nullPartValue = if (isUsingHiveMetastore) "__HIVE_DEFAULT_PARTITION__" else null
+    assert(catalog.listPartitions(tableIdent).map(_.spec).toSet ==
+      Set(Map("a" -> nullPartValue, "b" -> nullPartValue)))
+    sql("ALTER TABLE tab1 DROP PARTITION (a = null, b = null)")
+    assert(catalog.listPartitions(tableIdent).isEmpty)
   }
 
   protected def testRenamePartitions(isDatasourceTable: Boolean): Unit = {
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 571c25e356c08..bd44b0b8f139c 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -951,9 +951,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
   // Hive metastore is not case preserving and the partition columns are always lower cased. We need
   // to lower case the column names in partition specification before calling partition related Hive
   // APIs, to match this behaviour.
-  private def lowerCasePartitionSpec(spec: TablePartitionSpec): TablePartitionSpec = {
+  private def toMetaStorePartitionSpec(spec: TablePartitionSpec): TablePartitionSpec = {
     // scalastyle:off caselocale
-    spec.map { case (k, v) => k.toLowerCase -> v }
+    val lowNames = spec.map { case (k, v) => k.toLowerCase -> v }
+    ExternalCatalogUtils.convertNullPartitionValues(lowNames)
     // scalastyle:on caselocale
   }
 
@@ -1002,8 +1003,9 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       }
       p.copy(storage = p.storage.copy(locationUri = Some(partitionPath.toUri)))
     }
-    val lowerCasedParts = partsWithLocation.map(p => p.copy(spec = lowerCasePartitionSpec(p.spec)))
-    client.createPartitions(db, table, lowerCasedParts, ignoreIfExists)
+    val metaStoreParts = partsWithLocation
+      .map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
+    client.createPartitions(db, table, metaStoreParts, ignoreIfExists)
   }
 
   override def dropPartitions(
@@ -1015,7 +1017,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       retainData: Boolean): Unit = withClient {
     requireTableExists(db, table)
     client.dropPartitions(
-      db, table, parts.map(lowerCasePartitionSpec), ignoreIfNotExists, purge, retainData)
+      db, table, parts.map(toMetaStorePartitionSpec), ignoreIfNotExists, purge, retainData)
   }
 
   override def renamePartitions(
@@ -1024,7 +1026,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       specs: Seq[TablePartitionSpec],
       newSpecs: Seq[TablePartitionSpec]): Unit = withClient {
     client.renamePartitions(
-      db, table, specs.map(lowerCasePartitionSpec), newSpecs.map(lowerCasePartitionSpec))
+      db, table, specs.map(toMetaStorePartitionSpec), newSpecs.map(toMetaStorePartitionSpec))
 
     val tableMeta = getTable(db, table)
     val partitionColumnNames = tableMeta.partitionColumnNames
@@ -1040,7 +1042,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       val fs = tablePath.getFileSystem(hadoopConf)
       val newParts = newSpecs.map { spec =>
         val rightPath = renamePartitionDirectory(fs, tablePath, partitionColumnNames, spec)
-        val partition = client.getPartition(db, table, lowerCasePartitionSpec(spec))
+        val partition = client.getPartition(db, table, toMetaStorePartitionSpec(spec))
         partition.copy(storage = partition.storage.copy(locationUri = Some(rightPath.toUri)))
       }
       alterPartitions(db, table, newParts)
@@ -1150,12 +1152,12 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       newParts: Seq[CatalogTablePartition]): Unit = withClient {
-    val lowerCasedParts = newParts.map(p => p.copy(spec = lowerCasePartitionSpec(p.spec)))
+    val metaStoreParts = newParts.map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
 
     val rawTable = getRawTable(db, table)
 
     // convert partition statistics to properties so that we can persist them through hive api
-    val withStatsProps = lowerCasedParts.map { p =>
+    val withStatsProps = metaStoreParts.map { p =>
       if (p.stats.isDefined) {
         val statsProperties = statsToProperties(p.stats.get)
         p.copy(parameters = p.parameters ++ statsProperties)
@@ -1171,7 +1173,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       spec: TablePartitionSpec): CatalogTablePartition = withClient {
-    val part = client.getPartition(db, table, lowerCasePartitionSpec(spec))
+    val part = client.getPartition(db, table, toMetaStorePartitionSpec(spec))
     restorePartitionMetadata(part, getTable(db, table))
   }
 
@@ -1209,7 +1211,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       db: String,
       table: String,
       spec: TablePartitionSpec): Option[CatalogTablePartition] = withClient {
-    client.getPartitionOption(db, table, lowerCasePartitionSpec(spec)).map { part =>
+    client.getPartitionOption(db, table, toMetaStorePartitionSpec(spec)).map { part =>
       restorePartitionMetadata(part, getTable(db, table))
     }
   }
@@ -1224,7 +1226,7 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
     val catalogTable = getTable(db, table)
     val partColNameMap = buildLowerCasePartColNameMap(catalogTable).mapValues(escapePathName)
     val clientPartitionNames =
-      client.getPartitionNames(catalogTable, partialSpec.map(lowerCasePartitionSpec))
+      client.getPartitionNames(catalogTable, partialSpec.map(toMetaStorePartitionSpec))
     clientPartitionNames.map { partitionPath =>
       val partSpec = PartitioningUtils.parsePathFragmentAsSeq(partitionPath)
       partSpec.map { case (partName, partValue) =>
@@ -1243,11 +1245,12 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       table: String,
       partialSpec: Option[TablePartitionSpec] = None): Seq[CatalogTablePartition] = withClient {
     val partColNameMap = buildLowerCasePartColNameMap(getTable(db, table))
-    val res = client.getPartitions(db, table, partialSpec.map(lowerCasePartitionSpec)).map { part =>
-      part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
+    val metaStoreSpec = partialSpec.map(toMetaStorePartitionSpec)
+    val res = client.getPartitions(db, table, metaStoreSpec)
+      .map { part => part.copy(spec = restorePartitionSpec(part.spec, partColNameMap))
     }
 
-    partialSpec match {
+    metaStoreSpec match {
       // This might be a bug of Hive: When the partition value inside the partial partition spec
       // contains dot, and we ask Hive to list partitions w.r.t. the partial partition spec, Hive
       // treats dot as matching any single character and may return more partitions than we
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 05acca46135c3..39d5b711ab87e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -131,6 +131,7 @@ case class InsertIntoHiveTable(
     val numDynamicPartitions = partition.values.count(_.isEmpty)
     val numStaticPartitions = partition.values.count(_.nonEmpty)
     val partitionSpec = partition.map {
+      case (key, Some(null)) => key -> ExternalCatalogUtils.DEFAULT_PARTITION_NAME
       case (key, Some(value)) => key -> value
       case (key, None) => key -> ""
     }
@@ -227,6 +228,7 @@ case class InsertIntoHiveTable(
             val caseInsensitiveDpMap = CaseInsensitiveMap(dpMap)
 
             val updatedPartitionSpec = partition.map {
+              case (key, Some(null)) => key -> ExternalCatalogUtils.DEFAULT_PARTITION_NAME
               case (key, Some(value)) => key -> value
               case (key, None) if caseInsensitiveDpMap.contains(key) =>
                 key -> caseInsensitiveDpMap(key)

From 4cbc17792a2a31bd0c04cc5840294663646f81c5 Mon Sep 17 00:00:00 2001
From: Weichen Xu <weichen.xu@databricks.com>
Date: Mon, 11 Jan 2021 18:14:52 +0800
Subject: [PATCH 1255/1280] [MINOR][3.1][3.0] Improve flaky NaiveBayes test

### What changes were proposed in this pull request?

Current test may sometimes fail under different BLAS library. Due to some absTol check. Error like
```
Expected 0.7 and 0.6485507246376814 to be within 0.05 using absolute tolerance...

```

* Change absTol to relTol: The `absTol 0.05` in some cases (such as compare 0.1 and 0.05) is a big difference
* Remove the `exp` when comparing params. The `exp` will amplify the relative error.

### Why are the changes needed?
Flaky test

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
N/A

Closes #31004 from WeichenXu123/improve_bayes_tests.

Authored-by: Weichen Xu <weichen.xudatabricks.com>
Signed-off-by: Ruifeng Zheng <ruifengzfoxmail.com>

### What changes were proposed in this pull request?

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #31123 from WeichenXu123/bp-3.1-nb-test.

Authored-by: Weichen Xu <weichen.xu@databricks.com>
Signed-off-by: Ruifeng Zheng <ruifengz@foxmail.com>
(cherry picked from commit d33f0d44ec7c58a86c11c40a8f5933b333175ceb)
Signed-off-by: Ruifeng Zheng <ruifengz@foxmail.com>
---
 .../apache/spark/mllib/classification/NaiveBayesSuite.scala   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
index 47dac3ec29a5c..c20674718b101 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
@@ -107,9 +107,9 @@ class NaiveBayesSuite extends SparkFunSuite with MLlibTestSparkContext {
     val modelIndex = piData.indices.zip(model.labels.map(_.toInt))
     try {
       for (i <- modelIndex) {
-        assert(math.exp(piData(i._2)) ~== math.exp(model.pi(i._1)) absTol 0.05)
+        assert(piData(i._2) ~== model.pi(i._1) relTol 0.15)
         for (j <- thetaData(i._2).indices) {
-          assert(math.exp(thetaData(i._2)(j)) ~== math.exp(model.theta(i._1)(j)) absTol 0.05)
+          assert(thetaData(i._2)(j) ~== model.theta(i._1)(j) relTol 0.15)
         }
       }
     } catch {

From ecfa01570aa1276ec4c770a99054f36cc92e73ec Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Mon, 11 Jan 2021 14:36:13 +0000
Subject: [PATCH 1256/1280] [SPARK-34060][SQL][3.0] Fix Hive table caching
 while updating stats by `ALTER TABLE .. DROP PARTITION`

### What changes were proposed in this pull request?
Fix canonicalisation of `HiveTableRelation` by normalisation of `CatalogTable`, and exclude table stats and temporary fields from the canonicalized plan.

### Why are the changes needed?
This fixes the issue demonstrated by the example below:
```scala
scala> spark.conf.set("spark.sql.statistics.size.autoUpdate.enabled", true)
scala> sql(s"CREATE TABLE tbl (id int, part int) USING hive PARTITIONED BY (part)")
scala> sql("INSERT INTO tbl PARTITION (part=0) SELECT 0")
scala> sql("INSERT INTO tbl PARTITION (part=1) SELECT 1")
scala> sql("CACHE TABLE tbl")
scala> sql("SELECT * FROM tbl").show(false)
+---+----+
|id |part|
+---+----+
|0  |0   |
|1  |1   |
+---+----+

scala> spark.catalog.isCached("tbl")
scala> sql("ALTER TABLE tbl DROP PARTITION (part=0)")
scala> spark.catalog.isCached("tbl")
res19: Boolean = false
```
`ALTER TABLE .. DROP PARTITION` must keep the table in the cache.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes, the drop partition command keeps the table in the cache while updating table stats:
```scala
scala> sql("ALTER TABLE tbl DROP PARTITION (part=0)")
scala> spark.catalog.isCached("tbl")
res19: Boolean = true
```

### How was this patch tested?
By running new UT:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *ShowCreateTableSuite"
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *CachedTableSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit d97e99157e8d3b7434610fd78af90911c33662c9)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #31126 from MaxGekk/fix-caching-hive-table-2-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../sql/catalyst/catalog/interface.scala      | 29 ++++++++++++++++---
 .../spark/sql/ShowCreateTableSuite.scala      | 25 +---------------
 .../spark/sql/hive/CachedTableSuite.scala     | 29 +++++++++++++++++++
 3 files changed, 55 insertions(+), 28 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 7d9b57a3f171f..802e5780afb86 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -413,6 +413,30 @@ object CatalogTable {
   val VIEW_QUERY_OUTPUT_PREFIX = VIEW_PREFIX + "query.out."
   val VIEW_QUERY_OUTPUT_NUM_COLUMNS = VIEW_QUERY_OUTPUT_PREFIX + "numCols"
   val VIEW_QUERY_OUTPUT_COLUMN_NAME_PREFIX = VIEW_QUERY_OUTPUT_PREFIX + "col."
+
+  def normalize(table: CatalogTable): CatalogTable = {
+    val nondeterministicProps = Set(
+      "CreateTime",
+      "transient_lastDdlTime",
+      "grantTime",
+      "lastUpdateTime",
+      "last_modified_by",
+      "last_modified_time",
+      "Owner:",
+      // The following are hive specific schema parameters which we do not need to match exactly.
+      "totalNumberFiles",
+      "maxFileSize",
+      "minFileSize"
+    )
+
+    table.copy(
+      createTime = 0L,
+      lastAccessTime = 0L,
+      properties = table.properties.filterKeys(!nondeterministicProps.contains(_)).toMap,
+      stats = None,
+      ignoredProperties = Map.empty
+    )
+  }
 }
 
 /**
@@ -667,10 +691,7 @@ case class HiveTableRelation(
   def isPartitioned: Boolean = partitionCols.nonEmpty
 
   override def doCanonicalize(): HiveTableRelation = copy(
-    tableMeta = tableMeta.copy(
-      storage = CatalogStorageFormat.empty,
-      createTime = -1
-    ),
+    tableMeta = CatalogTable.normalize(tableMeta),
     dataCols = dataCols.zipWithIndex.map {
       case (attr, index) => attr.withExprId(ExprId(index))
     },
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
index 4e85f739b95a2..6253626856bcd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala
@@ -220,29 +220,6 @@ abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils {
   }
 
   protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = {
-    def normalize(table: CatalogTable): CatalogTable = {
-      val nondeterministicProps = Set(
-        "CreateTime",
-        "transient_lastDdlTime",
-        "grantTime",
-        "lastUpdateTime",
-        "last_modified_by",
-        "last_modified_time",
-        "Owner:",
-        // The following are hive specific schema parameters which we do not need to match exactly.
-        "totalNumberFiles",
-        "maxFileSize",
-        "minFileSize"
-      )
-
-      table.copy(
-        createTime = 0L,
-        lastAccessTime = 0L,
-        properties = table.properties.filterKeys(!nondeterministicProps.contains(_)),
-        stats = None,
-        ignoredProperties = Map.empty
-      )
-    }
-    assert(normalize(actual) == normalize(expected))
+    assert(CatalogTable.normalize(actual) == CatalogTable.normalize(expected))
   }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index 26918c5609456..e3141f504ea90 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
 import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation}
 import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
 import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SQLTestUtils
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.storage.RDDBlockId
@@ -498,4 +499,32 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
       checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
     }
   }
+
+  test("SPARK-34060: update stats of cached table") {
+    withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+      def checkTableSize(expected: String): Unit = {
+        val stats =
+          sql("DESCRIBE TABLE EXTENDED t")
+            .select("data_type")
+            .where("col_name = 'Statistics'")
+            .first()
+            .getString(0)
+        assert(stats.contains(expected))
+      }
+
+      sql("CREATE TABLE t (id int, part int) USING hive PARTITIONED BY (part)")
+      sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+      sql("INSERT INTO t PARTITION (part=1) SELECT 1")
+      assert(!spark.catalog.isCached("t"))
+      sql("CACHE TABLE t")
+      assert(spark.catalog.isCached("t"))
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(1, 1)))
+      checkTableSize("4 bytes")
+
+      sql("ALTER TABLE t DROP PARTITION (part=0)")
+      assert(spark.catalog.isCached("t"))
+      checkTableSize("2 bytes")
+      checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 1)))
+    }
+  }
 }

From 27c03b658ba99a8f939ae28301f6bfb39b282395 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Tue, 12 Jan 2021 13:02:21 +0900
Subject: [PATCH 1257/1280] [SPARK-34059][SQL][CORE][3.0] Use for/foreach
 rather than map to make sure execute it eagerly

### What changes were proposed in this pull request?

This is a backport of https://github.com/apache/spark/pull/31110. I ran intelliJ inspection again in this branch.

This PR is basically a followup of https://github.com/apache/spark/pull/14332.
Calling `map` alone might leave it not executed due to lazy evaluation, e.g.)

```
scala> val foo = Seq(1,2,3)
foo: Seq[Int] = List(1, 2, 3)

scala> foo.map(println)
1
2
3
res0: Seq[Unit] = List((), (), ())

scala> foo.view.map(println)
res1: scala.collection.SeqView[Unit,Seq[_]] = SeqViewM(...)

scala> foo.view.foreach(println)
1
2
3
```

We should better use `foreach` to make sure it's executed where the output is unused or `Unit`.

### Why are the changes needed?

To prevent the potential issues by not executing `map`.

### Does this PR introduce _any_ user-facing change?

No, the current codes look not causing any problem for now.

### How was this patch tested?

I found these item by running IntelliJ inspection, double checked one by one, and fixed them. These should be all instances across the codebase ideally.

Closes #31138 from HyukjinKwon/SPARK-34059-3.0.

Authored-by: HyukjinKwon <gurwls223@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/api/python/PythonRunner.scala    | 2 +-
 .../org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala    | 2 +-
 .../scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala | 2 +-
 .../sql/catalyst/analysis/UnsupportedOperationsSuite.scala  | 6 +++---
 .../scala/org/apache/spark/sql/execution/CacheManager.scala | 2 +-
 .../datasources/v2/SetCatalogAndNamespaceExec.scala         | 4 ++--
 .../columnar/compression/PassThroughEncodingSuite.scala     | 4 ++--
 7 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
index 39c4abea933a7..820cb798050f0 100644
--- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
+++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala
@@ -230,7 +230,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT](
             /* backlog */ 1,
             InetAddress.getByName("localhost")))
           // A call to accept() for ServerSocket shall block infinitely.
-          serverSocket.map(_.setSoTimeout(0))
+          serverSocket.foreach(_.setSoTimeout(0))
           new Thread("accept-connections") {
             setDaemon(true)
 
diff --git a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
index 82569653c1f23..12ebddf72b03d 100644
--- a/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
+++ b/external/avro/src/test/scala/org/apache/spark/sql/avro/AvroLogicalTypeSuite.scala
@@ -246,7 +246,7 @@ abstract class AvroLogicalTypeSuite extends QueryTest with SharedSparkSession {
     dataFileWriter.create(schema, new File(avroFile))
     val logicalType = LogicalTypes.decimal(precision, scale)
 
-    decimalInputData.map { x =>
+    decimalInputData.foreach { x =>
       val avroRec = new GenericData.Record(schema)
       val decimal = new java.math.BigDecimal(x).setScale(scale)
       val bytes =
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
index 7b2f6a2535eda..402d80f6dd2fb 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosSchedulerBackendUtil.scala
@@ -137,7 +137,7 @@ private[mesos] object MesosSchedulerBackendUtil extends Logging {
     val containerInfo = ContainerInfo.newBuilder()
       .setType(containerType)
 
-    conf.get(EXECUTOR_DOCKER_IMAGE).map { image =>
+    conf.get(EXECUTOR_DOCKER_IMAGE).foreach { image =>
       val forcePullImage = conf
         .get(EXECUTOR_DOCKER_FORCE_PULL_IMAGE).contains(true)
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
index 3ec6fdeedd4b8..c3bb7bfc80fcf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationsSuite.scala
@@ -694,7 +694,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       OutputMode.Append(),
       expectFailure = expectFailure)
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType2 =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType2 =>
       testGlobalWatermarkLimit(
         s"streaming-stream $joinType2 after stream-stream $joinType join in Append mode",
         streamRelation.join(
@@ -740,7 +740,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
         isMapGroupsWithState = false, null, streamRelation).groupBy("*")(count("*")),
       OutputMode.Append())
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType =>
       assertFailOnGlobalWatermarkLimit(
         s"stream-stream $joinType after FlatMapGroupsWithState in Append mode",
         streamRelation.join(
@@ -773,7 +773,7 @@ class UnsupportedOperationsSuite extends SparkFunSuite {
       Deduplicate(Seq(attribute), streamRelation).groupBy("a")(count("*")),
       OutputMode.Append())
 
-    Seq(Inner, LeftOuter, RightOuter).map { joinType =>
+    Seq(Inner, LeftOuter, RightOuter).foreach { joinType =>
       assertPassOnGlobalWatermarkLimit(
         s"$joinType join after deduplicate in Append mode",
         streamRelation.join(Deduplicate(Seq(attribute), streamRelation), joinType = joinType,
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index 7d86c48015406..cee8585e387a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -189,7 +189,7 @@ class CacheManager extends Logging with AdaptiveSparkPlanHelper {
       // Remove the cache entry before creating a new ones.
       cachedData = cachedData.filterNot(cd => needToRecache.exists(_ eq cd))
     }
-    needToRecache.map { cd =>
+    needToRecache.foreach { cd =>
       cd.cachedRepresentation.cacheBuilder.clearCache()
       // Turn off AQE so that the outputPartitioning of the underlying plan can be leveraged.
       val sessionWithAqeOff = getOrCloneSessionWithAqeOff(spark)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
index 9e6f00e0923ea..b13cea266707b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetCatalogAndNamespaceExec.scala
@@ -32,8 +32,8 @@ case class SetCatalogAndNamespaceExec(
   override protected def run(): Seq[InternalRow] = {
     // The catalog is updated first because CatalogManager resets the current namespace
     // when the current catalog is set.
-    catalogName.map(catalogManager.setCurrentCatalog)
-    namespace.map(ns => catalogManager.setCurrentNamespace(ns.toArray))
+    catalogName.foreach(catalogManager.setCurrentCatalog)
+    namespace.foreach(ns => catalogManager.setCurrentNamespace(ns.toArray))
 
     Seq.empty
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
index f946a6779ec95..c6fe64d1058ab 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/compression/PassThroughEncodingSuite.scala
@@ -46,7 +46,7 @@ class PassThroughSuite extends SparkFunSuite {
 
       val builder = TestCompressibleColumnBuilder(columnStats, columnType, PassThrough)
 
-      input.map { value =>
+      input.foreach { value =>
         val row = new GenericInternalRow(1)
         columnType.setField(row, 0, value)
         builder.appendFrom(row, 0)
@@ -98,7 +98,7 @@ class PassThroughSuite extends SparkFunSuite {
       val row = new GenericInternalRow(1)
       val nullRow = new GenericInternalRow(1)
       nullRow.setNullAt(0)
-      input.map { value =>
+      input.foreach { value =>
         if (value == nullValue) {
           builder.appendFrom(nullRow, 0)
         } else {

From a30d20ff9021dd09f2e72ed6a932f40ef98f4c22 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Mon, 11 Jan 2021 21:27:43 -0800
Subject: [PATCH 1258/1280] [SPARK-31952][SQL][3.0] Fix incorrect memory spill
 metric when doing Aggregate

### What changes were proposed in this pull request?

This PR takes over https://github.com/apache/spark/pull/28780.

1. Counted the spilled memory size when creating the `UnsafeExternalSorter` with the existing `InMemorySorter`

2. Accumulate the `totalSpillBytes` when merging two `UnsafeExternalSorter`

### Why are the changes needed?

As mentioned in https://github.com/apache/spark/pull/28780:

> It happends when hash aggregate downgrades to sort based aggregate.
`UnsafeExternalSorter.createWithExistingInMemorySorter` calls spill on an `InMemorySorter` immediately, but the memory pointed by `InMemorySorter` is acquired by outside `BytesToBytesMap`, instead the allocatedPages in `UnsafeExternalSorter`. So the memory spill bytes metric is always 0, but disk bytes spill metric is right.

Besides, this PR also fixes the `UnsafeExternalSorter.merge` by accumulating the `totalSpillBytes` of two sorters. Thus, we can report the correct spilled size in `HashAggregateExec.finishAggregate`.

Issues can be reproduced by the following step by checking the SQL metrics in UI:

```
bin/spark-shell --driver-memory 512m --executor-memory 512m --executor-cores 1 --conf "spark.default.parallelism=1"
scala> sql("select id, count(1) from range(10000000) group by id").write.csv("/tmp/result.json")
```

Before:

<img width="200" alt="WeChatfe5146180d91015e03b9a27852e9a443" src="https://user-images.githubusercontent.com/16397174/103625414-e6fc6280-4f75-11eb-8b93-c55095bdb5b8.png">

After:

<img width="200" alt="WeChat42ab0e73c5fbc3b14c12ab85d232071d" src="https://user-images.githubusercontent.com/16397174/103625420-e8c62600-4f75-11eb-8e1f-6f5e8ab561b9.png">

### Does this PR introduce _any_ user-facing change?

Yes, users can see the correct spill metrics after this PR.

### How was this patch tested?

Tested manually and added UTs.

Closes #31140 from Ngone51/cp-spark-31952.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../unsafe/sort/UnsafeExternalSorter.java     |  6 +-
 .../sql/execution/UnsafeKVExternalSorter.java |  3 +-
 .../UnsafeKVExternalSorterSuite.scala         | 95 +++++++++++++++----
 3 files changed, 86 insertions(+), 18 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
index 2096453d8060a..e4e369baf9dfa 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeExternalSorter.java
@@ -104,11 +104,14 @@ public static UnsafeExternalSorter createWithExistingInMemorySorter(
       int initialSize,
       long pageSizeBytes,
       int numElementsForSpillThreshold,
-      UnsafeInMemorySorter inMemorySorter) throws IOException {
+      UnsafeInMemorySorter inMemorySorter,
+      long existingMemoryConsumption) throws IOException {
     UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager,
       serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize,
         pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */);
     sorter.spill(Long.MAX_VALUE, sorter);
+    taskContext.taskMetrics().incMemoryBytesSpilled(existingMemoryConsumption);
+    sorter.totalSpillBytes += existingMemoryConsumption;
     // The external sorter will be used to insert records, in-memory sorter is not needed.
     sorter.inMemSorter = null;
     return sorter;
@@ -496,6 +499,7 @@ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen,
    */
   public void merge(UnsafeExternalSorter other) throws IOException {
     other.spill();
+    totalSpillBytes += other.totalSpillBytes;
     spillWriters.addAll(other.spillWriters);
     // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`.
     other.spillWriters.clear();
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
index 7a9f61a2cc33d..42ceebccdd1ee 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeKVExternalSorter.java
@@ -165,7 +165,8 @@ public UnsafeKVExternalSorter(
         (int) (long) SparkEnv.get().conf().get(package$.MODULE$.SHUFFLE_SORT_INIT_BUFFER_SIZE()),
         pageSizeBytes,
         numElementsForSpillThreshold,
-        inMemSorter);
+        inMemSorter,
+        map.getTotalMemoryConsumption());
 
       // reset the map, so we can re-use it to insert new records. the inMemSorter will not used
       // anymore, so the underline array could be used by map again.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
index 8aa003a3dfeb0..f630cd8322c61 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/UnsafeKVExternalSorterSuite.scala
@@ -210,23 +210,8 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession
   test("SPARK-23376: Create UnsafeKVExternalSorter with BytesToByteMap having duplicated keys") {
     val memoryManager = new TestMemoryManager(new SparkConf())
     val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
-    val map = new BytesToBytesMap(taskMemoryManager, 64, taskMemoryManager.pageSizeBytes())
-
-    // Key/value are a unsafe rows with a single int column
+    val map = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
     val schema = new StructType().add("i", IntegerType)
-    val key = new UnsafeRow(1)
-    key.pointTo(new Array[Byte](32), 32)
-    key.setInt(0, 1)
-    val value = new UnsafeRow(1)
-    value.pointTo(new Array[Byte](32), 32)
-    value.setInt(0, 2)
-
-    for (_ <- 1 to 65) {
-      val loc = map.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
-      loc.append(
-        key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
-        value.getBaseObject, value.getBaseOffset, value.getSizeInBytes)
-    }
 
     // Make sure we can successfully create a UnsafeKVExternalSorter with a `BytesToBytesMap`
     // which has duplicated keys and the number of entries exceeds its capacity.
@@ -245,4 +230,82 @@ class UnsafeKVExternalSorterSuite extends SparkFunSuite with SharedSparkSession
       TaskContext.unset()
     }
   }
+
+  test("SPARK-31952: create UnsafeKVExternalSorter with existing map should count spilled memory " +
+    "size correctly") {
+    val memoryManager = new TestMemoryManager(new SparkConf())
+    val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
+    val map = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val schema = new StructType().add("i", IntegerType)
+
+    try {
+      val context = new TaskContextImpl(0, 0, 0, 0, 0, taskMemoryManager, new Properties(), null)
+      TaskContext.setTaskContext(context)
+      val expectedSpillSize = map.getTotalMemoryConsumption
+      val sorter = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map)
+      assert(sorter.getSpillSize === expectedSpillSize)
+    } finally {
+      TaskContext.unset()
+    }
+  }
+
+  test("SPARK-31952: UnsafeKVExternalSorter.merge should accumulate totalSpillBytes") {
+    val memoryManager = new TestMemoryManager(new SparkConf())
+    val taskMemoryManager = new TaskMemoryManager(memoryManager, 0)
+    val map1 = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val map2 = createBytesToBytesMapWithDuplicateKeys(taskMemoryManager)
+    val schema = new StructType().add("i", IntegerType)
+
+    try {
+      val context = new TaskContextImpl(0, 0, 0, 0, 0, taskMemoryManager, new Properties(), null)
+      TaskContext.setTaskContext(context)
+      val expectedSpillSize = map1.getTotalMemoryConsumption + map2.getTotalMemoryConsumption
+      val sorter1 = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map1)
+      val sorter2 = new UnsafeKVExternalSorter(
+        schema,
+        schema,
+        sparkContext.env.blockManager,
+        sparkContext.env.serializerManager,
+        taskMemoryManager.pageSizeBytes(),
+        Int.MaxValue,
+        map2)
+      sorter1.merge(sorter2)
+      assert(sorter1.getSpillSize === expectedSpillSize)
+    } finally {
+      TaskContext.unset()
+    }
+  }
+
+  private def createBytesToBytesMapWithDuplicateKeys(taskMemoryManager: TaskMemoryManager)
+    : BytesToBytesMap = {
+    val map = new BytesToBytesMap(taskMemoryManager, 64, taskMemoryManager.pageSizeBytes())
+    // Key/value are a unsafe rows with a single int column
+    val key = new UnsafeRow(1)
+    key.pointTo(new Array[Byte](32), 32)
+    key.setInt(0, 1)
+    val value = new UnsafeRow(1)
+    value.pointTo(new Array[Byte](32), 32)
+    value.setInt(0, 2)
+    for (_ <- 1 to 65) {
+      val loc = map.lookup(key.getBaseObject, key.getBaseOffset, key.getSizeInBytes)
+      loc.append(
+        key.getBaseObject, key.getBaseOffset, key.getSizeInBytes,
+        value.getBaseObject, value.getBaseOffset, value.getSizeInBytes)
+    }
+    map
+  }
 }

From 7cfc45b4e21c0d97e6f593b847099e837cb254bf Mon Sep 17 00:00:00 2001
From: huangtianhua <huangtianhua223@gmail.com>
Date: Tue, 12 Jan 2021 13:17:37 -0800
Subject: [PATCH 1259/1280] [SPARK-32691][3.0] Bump commons-crypto to v1.1.0

The package commons-crypto-1.0.0 doesn't support
aarch64 platform, change to use v1.1.0.

### What changes were proposed in this pull request?
Update the package commons-crypto to v1.1.0 to support aarch64 platform
- https://issues.apache.org/jira/browse/CRYPTO-139

NOTE: This backport comes from #30275

### Why are the changes needed?
The package commons-crypto-1.0.0 available in the Maven repository
doesn't support aarch64 platform. It costs long time in
CryptoRandomFactory.getCryptoRandom(properties).nextBytes(iv) when NettyBlockRpcSever
receive block data from client,  if the time more than the default value 120s, IOException raised and client
will retry replicate the block data to other executors. But in fact the replication is complete,
it makes the replication number incorrect.
This makes DistributedSuite tests pass.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Pass the CIs.

Closes #31078 from huangtianhua/origin/branch-3.0.

Authored-by: huangtianhua <huangtianhua223@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 dev/deps/spark-deps-hadoop-2.7-hive-1.2 | 2 +-
 dev/deps/spark-deps-hadoop-2.7-hive-2.3 | 2 +-
 dev/deps/spark-deps-hadoop-3.2-hive-2.3 | 2 +-
 pom.xml                                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
index cc23a79a745d8..8c6cc15c150b3 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
@@ -38,7 +38,7 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-digester/1.8//commons-digester-1.8.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
index 80bee1491c8fe..3e82df2bcdfa1 100644
--- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
@@ -36,7 +36,7 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration/1.6//commons-configuration-1.6.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-digester/1.8//commons-digester-1.8.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
index da18e10e5a037..8de3433f2d9fb 100644
--- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
@@ -33,7 +33,7 @@ commons-collections/3.2.2//commons-collections-3.2.2.jar
 commons-compiler/3.0.16//commons-compiler-3.0.16.jar
 commons-compress/1.20//commons-compress-1.20.jar
 commons-configuration2/2.1.1//commons-configuration2-2.1.1.jar
-commons-crypto/1.0.0//commons-crypto-1.0.0.jar
+commons-crypto/1.1.0//commons-crypto-1.1.0.jar
 commons-daemon/1.0.13//commons-daemon-1.0.13.jar
 commons-dbcp/1.4//commons-dbcp-1.4.jar
 commons-httpclient/3.1//commons-httpclient-3.1.jar
diff --git a/pom.xml b/pom.xml
index 58eb8895a95aa..2ec79c63e1021 100644
--- a/pom.xml
+++ b/pom.xml
@@ -201,7 +201,7 @@
     -->
     <paranamer.version>2.8</paranamer.version>
     <maven-antrun.version>1.8</maven-antrun.version>
-    <commons-crypto.version>1.0.0</commons-crypto.version>
+    <commons-crypto.version>1.1.0</commons-crypto.version>
     <!--
     If you are changing Arrow version specification, please check ./python/pyspark/sql/utils.py,
     and ./python/setup.py too.

From 0c4fdeaa38a7cebd13cab08962192f3c0c124419 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Wed, 13 Jan 2021 05:22:55 +0000
Subject: [PATCH 1260/1280] [SPARK-34084][SQL][3.0] Fix auto updating of table
 stats in `ALTER TABLE .. ADD PARTITION`

### What changes were proposed in this pull request?
Fix an issue in `ALTER TABLE .. ADD PARTITION` which happens when:
- A table doesn't have stats
- `spark.sql.statistics.size.autoUpdate.enabled` is `true`

In that case, `ALTER TABLE .. ADD PARTITION` does not update table stats automatically.

### Why are the changes needed?
The changes fix the issue demonstrated by the example:
```sql
spark-sql> create table tbl (col0 int, part int) partitioned by (part);
spark-sql> insert into tbl partition (part = 0) select 0;
spark-sql> set spark.sql.statistics.size.autoUpdate.enabled=true;
spark-sql> alter table tbl add partition (part = 1);
```
the `add partition` command should update table stats but it does not. There are no stats in the output of:
```
spark-sql> describe table extended tbl;
```

### Does this PR introduce _any_ user-facing change?
Yes. After the changes, `ALTER TABLE .. ADD PARTITION` updates stats even when a table doesn't have stats before the command:
```sql
spark-sql> alter table tbl add partition (part = 1);
spark-sql> describe table extended tbl;
col0	int	NULL
part	int	NULL
# Partition Information
# col_name	data_type	comment
part	int	NULL

# Detailed Table Information
...
Statistics	2 bytes
```

### How was this patch tested?
By running new UT and existing test suites:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *.StatisticsSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit 6c047958f9fcf4cac848695915deea289c65ddc1)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #31158 from MaxGekk/fix-stats-in-add-partition-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/execution/command/ddl.scala     | 20 +++++++++----------
 .../spark/sql/hive/StatisticsSuite.scala      | 16 +++++++++++++++
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 1380e1e1203cd..8fa21395e0420 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -485,17 +485,17 @@ case class AlterTableAddPartitionCommand(
     }
 
     sparkSession.catalog.refreshTable(table.identifier.quotedString)
-    if (table.stats.nonEmpty) {
-      if (sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
-        val addedSize = CommandUtils.calculateTotalLocationSize(sparkSession, table.identifier,
-          parts.map(_.storage.locationUri))
-        if (addedSize > 0) {
-          val newStats = CatalogStatistics(sizeInBytes = table.stats.get.sizeInBytes + addedSize)
-          catalog.alterTableStats(table.identifier, Some(newStats))
-        }
-      } else {
-        catalog.alterTableStats(table.identifier, None)
+    if (table.stats.nonEmpty && sparkSession.sessionState.conf.autoSizeUpdateEnabled) {
+      // Updating table stats only if new partition is not empty
+      val addedSize = CommandUtils.calculateTotalLocationSize(sparkSession, table.identifier,
+        parts.map(_.storage.locationUri))
+      if (addedSize > 0) {
+        val newStats = CatalogStatistics(sizeInBytes = table.stats.get.sizeInBytes + addedSize)
+        catalog.alterTableStats(table.identifier, Some(newStats))
       }
+    } else {
+      // Re-calculating of table size including all partitions
+      CommandUtils.updateTableStats(sparkSession, table)
     }
     Seq.empty[Row]
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index 69c31a9c42ae5..046ab01640444 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -1562,4 +1562,20 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
       }
     }
   }
+
+  test("SPARK-34084: auto update table stats") {
+    Seq("parquet", "hive").foreach { format =>
+      withTable("t") {
+        withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "false") {
+          sql(s"CREATE TABLE t (col0 int, part int) USING $format PARTITIONED BY (part)")
+          sql("INSERT INTO t PARTITION (part=0) SELECT 0")
+          assert(getCatalogTable("t").stats.isEmpty)
+        }
+        withSQLConf(SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> "true") {
+          sql("ALTER TABLE t ADD PARTITION (part=1)")
+          assert(getTableStats("t").sizeInBytes > 0)
+        }
+      }
+    }
+  }
 }

From dbc18d672b052c6281c5eb2c284a1faf3c0ef198 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dhyun@apple.com>
Date: Thu, 14 Jan 2021 09:29:31 +0900
Subject: [PATCH 1261/1280] [SPARK-34103][INFRA] Fix MiMaExcludes by moving
 SPARK-23429 from 2.4 to 3.0

### What changes were proposed in this pull request?

This PR aims to fix `MiMaExcludes` rule by moving SPARK-23429 from 2.4 to 3.0.

### Why are the changes needed?

SPARK-23429 was added at Apache Spark 3.0.0.
This should land on `master` and `branch-3.1` and `branch-3.0`.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the MiMa rule.

Closes #31174 from dongjoon-hyun/SPARK-34103.

Authored-by: Dongjoon Hyun <dhyun@apple.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 9e93fdb146b7af09922e7c11ce714bcc7e658115)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 project/MimaExcludes.scala | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index d1ed48a11e67a..b79962446d84b 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -36,6 +36,12 @@ object MimaExcludes {
 
   // Exclude rules for 3.0.x
   lazy val v30excludes = v24excludes ++ Seq(
+    // [SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.apply"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.copy"),
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.this"),
+    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate$"),
+
     // [SPARK-29306] Add support for Stage level scheduling for executors
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productElement"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages#RetrieveSparkAppConfig.productArity"),
@@ -460,12 +466,6 @@ object MimaExcludes {
 
   // Exclude rules for 2.4.x
   lazy val v24excludes = v23excludes ++ Seq(
-    // [SPARK-23429][CORE] Add executor memory metrics to heartbeat and expose in executors REST API
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.apply"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.copy"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate.this"),
-    ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.scheduler.SparkListenerExecutorMetricsUpdate$"),
-
     // [SPARK-25248] add package private methods to TaskContext
     ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.markTaskFailed"),
     ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.TaskContext.markInterrupted"),

From fcd10a66dc0a4e1f5ef1da7b0da7fcd5711674e6 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Thu, 14 Jan 2021 11:00:02 +0900
Subject: [PATCH 1262/1280] [SPARK-33557][CORE][MESOS][3.0] Ensure the
 relationship between STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT and
 NETWORK_TIMEOUT
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?
As described in SPARK-33557, `HeartbeatReceiver` and `MesosCoarseGrainedSchedulerBackend` will always use `Network.NETWORK_TIMEOUT.defaultValueString` as value of `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT` when we configure `NETWORK_TIMEOUT` without configure `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT`, this is different from the relationship described in `configuration.md`.

To fix this problem，the main change of this pr as follow:

- Remove the explicitly default value of `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT`

- Use actual value of `NETWORK_TIMEOUT` as `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT` when `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT` not configured in `HeartbeatReceiver` and `MesosCoarseGrainedSchedulerBackend`

### Why are the changes needed?
To ensure the relationship between `NETWORK_TIMEOUT` and  `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT` as we described in `configuration.md`

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?

- Pass the Jenkins or GitHub Action

- Manual test configure `NETWORK_TIMEOUT` and `STORAGE_BLOCKMANAGER_HEARTBEAT_TIMEOUT` locally

Closes #31175 from dongjoon-hyun/SPARK-33557.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala | 4 +++-
 .../scala/org/apache/spark/internal/config/package.scala     | 2 +-
 .../org/apache/spark/repl/ExecutorClassLoaderSuite.scala     | 1 +
 .../cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala   | 5 ++++-
 4 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
index 2ac72e66d6f32..be630722308fa 100644
--- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
+++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala
@@ -80,7 +80,9 @@ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock)
   // executor ID -> timestamp of when the last heartbeat from this executor was received
   private val executorLastSeen = new HashMap[String, Long]
 
-  private val executorTimeoutMs = sc.conf.get(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT)
+  private val executorTimeoutMs = sc.conf.get(
+    config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT
+  ).getOrElse(Utils.timeStringAsMs(s"${sc.conf.get(Network.NETWORK_TIMEOUT)}s"))
 
   private val checkTimeoutIntervalMs = sc.conf.get(Network.NETWORK_TIMEOUT_INTERVAL)
 
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index f3474efebab01..0afbb5278b1da 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -434,7 +434,7 @@ package object config {
     ConfigBuilder("spark.storage.blockManagerSlaveTimeoutMs")
       .version("0.7.0")
       .timeConf(TimeUnit.MILLISECONDS)
-      .createWithDefaultString(Network.NETWORK_TIMEOUT.defaultValueString)
+      .createOptional
 
   private[spark] val STORAGE_CLEANUP_FILES_AFTER_EXECUTOR_EXIT =
     ConfigBuilder("spark.storage.cleanupFilesAfterExecutorExit")
diff --git a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
index 5428fa4ee9df7..818b9900d0836 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ExecutorClassLoaderSuite.scala
@@ -233,6 +233,7 @@ class ExecutorClassLoaderSuite
       .setMaster("local")
       .setAppName("executor-class-loader-test")
       .set("spark.network.timeout", "11s")
+      .set("spark.network.timeoutInterval", "11s")
       .set("spark.repl.class.outputDir", tempDir1.getAbsolutePath)
     val sc = new SparkContext(conf)
     try {
diff --git a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
index e916125ffdb67..dff0b0622cd87 100644
--- a/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
+++ b/resource-managers/mesos/src/main/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -34,6 +34,7 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkContext, SparkExceptio
 import org.apache.spark.deploy.mesos.config._
 import org.apache.spark.deploy.security.HadoopDelegationTokenManager
 import org.apache.spark.internal.config
+import org.apache.spark.internal.config.Network
 import org.apache.spark.internal.config.Tests.IS_TESTING
 import org.apache.spark.launcher.{LauncherBackend, SparkAppHandle}
 import org.apache.spark.network.netty.SparkTransportConf
@@ -643,7 +644,9 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
           .registerDriverWithShuffleService(
             slave.hostname,
             externalShufflePort,
-            sc.conf.get(config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT),
+            sc.conf.get(
+              config.STORAGE_BLOCKMANAGER_SLAVE_TIMEOUT
+            ).getOrElse(Utils.timeStringAsMs(s"${sc.conf.get(Network.NETWORK_TIMEOUT)}s")),
             sc.conf.get(config.EXECUTOR_HEARTBEAT_INTERVAL))
         slave.shuffleRegistered = true
       }

From dc1816d1c4c273526ebcf477a56cd45afabaefa8 Mon Sep 17 00:00:00 2001
From: yangjie01 <yangjie01@baidu.com>
Date: Fri, 15 Jan 2021 20:19:37 +0900
Subject: [PATCH 1263/1280] [SPARK-34118][CORE][SQL][3.0] Replaces filter and
 check for emptiness with exists or forall

### What changes were proposed in this pull request?
This pr use `exists` or `forall` to simplify `filter + emptiness check`, it's semantically consistent, but looks simpler. The rule as follow:

- `seq.filter(p).size == 0)` -> `!seq.exists(p)`
- `seq.filter(p).length > 0` -> `seq.exists(p)`
- `seq.filterNot(p).isEmpty` -> `seq.forall(p)`
- `seq.filterNot(p).nonEmpty` -> `!seq.forall(p)`

### Why are the changes needed?
Code Simpilefications.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Pass the Jenkins or GitHub Action

Closes #31190 from LuciferYang/SPARK-34118-30.

Authored-by: yangjie01 <yangjie01@baidu.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/api/r/RUtils.scala       | 4 ++--
 .../test/scala/org/apache/spark/util/FileAppenderSuite.scala  | 2 +-
 .../org/apache/spark/sql/execution/DataSourceScanExec.scala   | 4 ++--
 .../apache/spark/sql/connector/DataSourceV2UtilsSuite.scala   | 4 ++--
 .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala  | 2 +-
 .../spark/sql/streaming/StreamingQueryListenerSuite.scala     | 2 +-
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
index 311fade127839..784a57e7b98a8 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
@@ -43,9 +43,9 @@ private[spark] object RUtils {
    * Check if SparkR is installed before running tests that use SparkR.
    */
   def isSparkRInstalled: Boolean = {
-    localSparkRPackagePath.filter { pkgDir =>
+    localSparkRPackagePath.exists { pkgDir =>
       new File(Seq(pkgDir, "SparkR").mkString(File.separator)).exists
-    }.isDefined
+    }
   }
 
   /**
diff --git a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
index 21e69550785a4..12d97573ff6ee 100644
--- a/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/FileAppenderSuite.scala
@@ -337,7 +337,7 @@ class FileAppenderSuite extends SparkFunSuite with BeforeAndAfter with Logging {
     assert(generatedFiles.size > 1)
     if (isCompressed) {
       assert(
-        generatedFiles.filter(_.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)).size > 0)
+        generatedFiles.exists(_.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)))
     }
     val allText = generatedFiles.map { file =>
       if (file.getName.endsWith(RollingFileAppender.GZIP_LOG_SUFFIX)) {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 0fcb0dd1f658e..e2bf132a2e18f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -409,7 +409,7 @@ case class FileSourceScanExec(
   }
 
   /** SQL metrics generated only for scans using dynamic partition pruning. */
-  private lazy val staticMetrics = if (partitionFilters.filter(isDynamicPruningFilter).nonEmpty) {
+  private lazy val staticMetrics = if (partitionFilters.exists(isDynamicPruningFilter)) {
     Map("staticFilesNum" -> SQLMetrics.createMetric(sparkContext, "static number of files read"),
       "staticFilesSize" -> SQLMetrics.createSizeMetric(sparkContext, "static size of files read"))
   } else {
@@ -422,7 +422,7 @@ case class FileSourceScanExec(
       static: Boolean): Unit = {
     val filesNum = partitions.map(_.files.size.toLong).sum
     val filesSize = partitions.map(_.files.map(_.getLen).sum).sum
-    if (!static || partitionFilters.filter(isDynamicPruningFilter).isEmpty) {
+    if (!static || !partitionFilters.exists(isDynamicPruningFilter)) {
       driverMetrics("numFiles") = filesNum
       driverMetrics("filesSize") = filesSize
     } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
index 01fcced5b12a8..a58bab276a41b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2UtilsSuite.scala
@@ -37,8 +37,8 @@ class DataSourceV2UtilsSuite extends SparkFunSuite {
     val source = new DataSourceV2WithSessionConfig
     val confs = DataSourceV2Utils.extractSessionConfigs(source, conf)
     assert(confs.size == 2)
-    assert(confs.keySet.filter(_.startsWith("spark.datasource")).size == 0)
-    assert(confs.keySet.filter(_.startsWith("not.exist.prefix")).size == 0)
+    assert(!confs.keySet.exists(_.startsWith("spark.datasource")))
+    assert(!confs.keySet.exists(_.startsWith("not.exist.prefix")))
     assert(confs.keySet.contains("foo.bar"))
     assert(confs.keySet.contains("whateverConfigName"))
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index a06879377c031..1f584a569c3f3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -696,7 +696,7 @@ class JDBCSuite extends QueryTest
   test("Remap types via JdbcDialects") {
     JdbcDialects.registerDialect(testH2Dialect)
     val df = spark.read.jdbc(urlWithUserAndPass, "TEST.PEOPLE", new Properties())
-    assert(df.schema.filter(_.dataType != org.apache.spark.sql.types.StringType).isEmpty)
+    assert(!df.schema.exists(_.dataType != org.apache.spark.sql.types.StringType))
     val rows = df.collect()
     assert(rows(0).get(0).isInstanceOf[String])
     assert(rows(0).get(1).isInstanceOf[String])
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
index 6e08b88f538df..a9aec6dbbdcd4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingQueryListenerSuite.scala
@@ -49,7 +49,7 @@ class StreamingQueryListenerSuite extends StreamTest with BeforeAndAfter {
     assert(spark.streams.active.isEmpty)
     // Skip check default `StreamingQueryStatusListener` which is for streaming UI.
     assert(spark.streams.listListeners()
-      .filterNot(_.isInstanceOf[StreamingQueryStatusListener]).isEmpty)
+      .forall(_.isInstanceOf[StreamingQueryStatusListener]))
     // Make sure we don't leak any events to the next test
     spark.sparkContext.listenerBus.waitUntilEmpty()
   }

From d81f48203078409687f291b9e43f4ad864b6921a Mon Sep 17 00:00:00 2001
From: sychen <sychen@ctrip.com>
Date: Fri, 15 Jan 2021 20:51:00 +0900
Subject: [PATCH 1264/1280] [SPARK-33790][CORE][3.0] Reduce the rpc call of
 getFileStatus in SingleFileEventLogFileReader

### What changes were proposed in this pull request?
`FsHistoryProvider#checkForLogs` already has `FileStatus` when constructing `SingleFileEventLogFileReader`, and there is no need to get the `FileStatus` again when `SingleFileEventLogFileReader#fileSizeForLastIndex`.

### Why are the changes needed?
This can reduce a lot of rpc calls and improve the speed of the history server.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
exist ut

Closes #31187 from HeartSaVioR/SPARK-33790-branch-3.0.

Lead-authored-by: sychen <sychen@ctrip.com>
Co-authored-by: Jungtaek Lim <kabhwan.opensource@gmail.com>
Signed-off-by: Jungtaek Lim (HeartSaVioR) <kabhwan.opensource@gmail.com>
---
 .../apache/spark/deploy/history/EventLogFileReaders.scala  | 7 ++++---
 project/MimaExcludes.scala                                 | 5 ++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 30ff2c3315750..6fe3a7bcafa06 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -116,7 +116,7 @@ object EventLogFileReader {
 
   def apply(fs: FileSystem, status: FileStatus): Option[EventLogFileReader] = {
     if (isSingleEventLog(status)) {
-      Some(new SingleFileEventLogFileReader(fs, status.getPath))
+      Some(new SingleFileEventLogFileReader(fs, status.getPath, Option(status)))
     } else if (isRollingEventLogs(status)) {
       Some(new RollingEventLogFilesFileReader(fs, status.getPath))
     } else {
@@ -166,8 +166,9 @@ object EventLogFileReader {
  */
 private[history] class SingleFileEventLogFileReader(
     fs: FileSystem,
-    path: Path) extends EventLogFileReader(fs, path) {
-  private lazy val status = fileSystem.getFileStatus(rootPath)
+    path: Path,
+    maybeStatus: Option[FileStatus] = None) extends EventLogFileReader(fs, path) {
+  private lazy val status = maybeStatus.getOrElse(fileSystem.getFileStatus(rootPath))
 
   override def lastIndex: Option[Long] = None
 
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index b79962446d84b..b4e36dc76384d 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -461,7 +461,10 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryListener#QueryStartedEvent.this"),
 
     // [SPARK-30667][CORE] Add allGather method to BarrierTaskContext
-    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.RequestToSync")
+    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.RequestToSync"),
+
+    // [SPARK-33790][CORE] Reduce the rpc call of getFileStatus in SingleFileEventLogFileReader
+    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.history.SingleFileEventLogFileReader.this")
   )
 
   // Exclude rules for 2.4.x

From 70fa108494a13e939d0aafc0491e30488dbe1caa Mon Sep 17 00:00:00 2001
From: KevinSmile <kevinwang013@hotmail.com>
Date: Fri, 15 Jan 2021 09:01:26 -0600
Subject: [PATCH 1265/1280] [SPARK-32598][SCHEDULER] Fix missing driver logs
 under UI App-Executors tab in standalone cluster mode

### What changes were proposed in this pull request?
Fix  [SPARK-32598] (missing driver logs under UI-ApplicationDetails-Executors tab in standalone cluster mode) .

The direct bug is: the original author forgot to implement `getDriverLogUrls` in `StandaloneSchedulerBackend`

https://github.com/apache/spark/blob/1de272f98d0ff22d0dd151797f22b8faf310963a/core/src/main/scala/org/apache/spark/scheduler/SchedulerBackend.scala#L70-L75

So we set DriverLogUrls as env in `DriverRunner`, and retrieve it at `StandaloneSchedulerBackend`.

### Why are the changes needed?
Fix bug  [SPARK-32598].

### Does this PR introduce _any_ user-facing change?
Yes. User will see driver logs (standalone cluster mode) under UI-ApplicationDetails-Executors tab now.

Before:
![image](https://user-images.githubusercontent.com/17903517/93901055-b5de8600-fd28-11ea-879a-d97e6f70cc6e.png)

After:
![image](https://user-images.githubusercontent.com/17903517/93901080-baa33a00-fd28-11ea-8895-3787c5efbf88.png)

### How was this patch tested?
Re-check the real case in [SPARK-32598] and found this user-facing bug fixed.

Closes #29644 from KevinSmile/kw-dev-master.

Authored-by: KevinSmile <kevinwang013@hotmail.com>
Signed-off-by: Sean Owen <srowen@gmail.com>
(cherry picked from commit c75c29dcaa9458a9ce0dd7a4d5fafbffb4b7f6a6)
Signed-off-by: Sean Owen <srowen@gmail.com>
---
 .../org/apache/spark/deploy/worker/DriverRunner.scala | 11 +++++++++++
 .../scala/org/apache/spark/deploy/worker/Worker.scala |  1 +
 .../cluster/StandaloneSchedulerBackend.scala          |  8 ++++++++
 .../org/apache/spark/deploy/DeployTestUtils.scala     |  1 +
 .../apache/spark/deploy/worker/DriverRunnerTest.scala |  3 ++-
 5 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
index 53ec7b3a88f35..2caee1846595e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala
@@ -33,9 +33,11 @@ import org.apache.spark.deploy.master.DriverState
 import org.apache.spark.deploy.master.DriverState.DriverState
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.{DRIVER_RESOURCES_FILE, SPARK_DRIVER_PREFIX}
+import org.apache.spark.internal.config.UI.UI_REVERSE_PROXY
 import org.apache.spark.internal.config.Worker.WORKER_DRIVER_TERMINATE_TIMEOUT
 import org.apache.spark.resource.ResourceInformation
 import org.apache.spark.rpc.RpcEndpointRef
+import org.apache.spark.ui.UIUtils
 import org.apache.spark.util.{Clock, ShutdownHookManager, SystemClock, Utils}
 
 /**
@@ -50,6 +52,7 @@ private[deploy] class DriverRunner(
     val driverDesc: DriverDescription,
     val worker: RpcEndpointRef,
     val workerUrl: String,
+    val workerWebUiUrl: String,
     val securityManager: SecurityManager,
     val resources: Map[String, ResourceInformation] = Map.empty)
   extends Logging {
@@ -190,6 +193,14 @@ private[deploy] class DriverRunner(
     val builder = CommandUtils.buildProcessBuilder(driverDesc.command.copy(javaOpts = javaOpts),
       securityManager, driverDesc.mem, sparkHome.getAbsolutePath, substituteVariables)
 
+    // add WebUI driver log url to environment
+    val reverseProxy = conf.get(UI_REVERSE_PROXY)
+    val workerUrlRef = UIUtils.makeHref(reverseProxy, driverId, workerWebUiUrl)
+    builder.environment.put("SPARK_DRIVER_LOG_URL_STDOUT",
+      s"$workerUrlRef/logPage?driverId=$driverId&logType=stdout")
+    builder.environment.put("SPARK_DRIVER_LOG_URL_STDERR",
+      s"$workerUrlRef/logPage?driverId=$driverId&logType=stderr")
+
     runDriver(builder, driverDir, driverDesc.supervise)
   }
 
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index 73cac880d464a..08fdd0ab61aa3 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -628,6 +628,7 @@ private[deploy] class Worker(
         driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),
         self,
         workerUri,
+        workerWebUiUrl,
         securityMgr,
         resources_)
       drivers(driverId) = driver
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
index a9b607d8cc38c..1b8b8cb4cf852 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/StandaloneSchedulerBackend.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.scheduler.cluster
 
+import java.util.Locale
 import java.util.concurrent.Semaphore
 import java.util.concurrent.atomic.AtomicBoolean
 
@@ -216,6 +217,13 @@ private[spark] class StandaloneSchedulerBackend(
     }
   }
 
+  override def getDriverLogUrls: Option[Map[String, String]] = {
+    val prefix = "SPARK_DRIVER_LOG_URL_"
+    val driverLogUrls = sys.env.filterKeys(_.startsWith(prefix))
+      .map(e => (e._1.substring(prefix.length).toLowerCase(Locale.ROOT), e._2)).toMap
+    if (driverLogUrls.nonEmpty) Some(driverLogUrls) else None
+  }
+
   private def waitForRegistration() = {
     registrationBarrier.acquire()
   }
diff --git a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
index 31f065ec55749..b182b11a0e85e 100644
--- a/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/DeployTestUtils.scala
@@ -102,6 +102,7 @@ private[deploy] object DeployTestUtils {
       createDriverDesc(),
       null,
       "spark://worker",
+      "http://publicAddress:80",
       new SecurityManager(conf))
   }
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
index c3b580e7ccac4..e429ddfd570de 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/DriverRunnerTest.scala
@@ -40,7 +40,8 @@ class DriverRunnerTest extends SparkFunSuite {
     val worker = mock(classOf[RpcEndpointRef])
     doNothing().when(worker).send(any())
     spy(new DriverRunner(conf, "driverId", new File("workDir"), new File("sparkHome"),
-      driverDescription, worker, "spark://1.2.3.4/worker/", new SecurityManager(conf)))
+      driverDescription, worker, "spark://1.2.3.4/worker/", "http://publicAddress:80",
+      new SecurityManager(conf)))
   }
 
   private def createProcessBuilderAndProcess(): (ProcessBuilderLike, Process) = {

From f7591e55c6a4e230c825b683cc9ba526563e2a6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9Cattilapiros=E2=80=9D?=
 <piros.attila.zsolt@gmail.com>
Date: Fri, 15 Jan 2021 13:09:26 -0800
Subject: [PATCH 1266/1280] [SPARK-33711][K8S][3.0] Avoid race condition
 between POD lifecycle manager and scheduler backend
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### What changes were proposed in this pull request?

Missing POD detection is extended by timestamp (and time limit) based check to avoid wrongfully detection of missing POD detection.

The two new timestamps:
- `fullSnapshotTs` is introduced for the `ExecutorPodsSnapshot` which only updated by the pod polling snapshot source
- `registrationTs` is introduced for the `ExecutorData` and it is initialized at the executor registration at the scheduler backend

Moreover a new config `spark.kubernetes.executor.missingPodDetectDelta` is used to specify the accepted delta between the two.

### Why are the changes needed?

Watching a POD (`ExecutorPodsWatchSnapshotSource`) only inform about single POD changes. This could wrongfully lead to detecting of missing PODs (PODs known by scheduler backend but missing from POD snapshots) by the executor POD lifecycle manager.

A key indicator of this error is seeing this log message:

> "The executor with ID [some_id] was not found in the cluster but we didn't get a reason why. Marking the executor as failed. The executor may have been deleted but the driver missed the deletion event."

So one of the problem is running the missing POD detection check even when a single POD is changed without having a full consistent snapshot about all the PODs (see `ExecutorPodsPollingSnapshotSource`).
The other problem could be the race between the executor POD lifecycle manager and the scheduler backend: so even in case of a having a full snapshot the registration at the scheduler backend could precede the snapshot polling (and processing of those polled snapshots).

### Does this PR introduce any user-facing change?

Yes. When the POD is missing then the reason message explaining the executor's exit is extended with both timestamps (the polling time and the executor registration time) and even the new config is mentioned.

### How was this patch tested?

The existing unit tests are extended.

(cherry picked from commit 6bd7a6200f8beaab1c68b2469df05870ea788d49)

Closes #31195 from attilapiros/SPARK-33711-branch-3.0.

Authored-by: “attilapiros” <piros.attila.zsolt@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../CoarseGrainedSchedulerBackend.scala       |  6 ++-
 .../scheduler/cluster/ExecutorData.scala      |  4 +-
 .../org/apache/spark/deploy/k8s/Config.scala  | 11 +++++
 .../cluster/k8s/ExecutorPodsAllocator.scala   |  2 +-
 .../k8s/ExecutorPodsLifecycleManager.scala    | 43 ++++++++++---------
 .../cluster/k8s/ExecutorPodsSnapshot.scala    | 12 +++---
 .../k8s/ExecutorPodsSnapshotsStoreImpl.scala  | 14 ++++--
 ...erministicExecutorPodsSnapshotsStore.scala |  8 +++-
 .../ExecutorPodsLifecycleManagerSuite.scala   | 22 +++++++---
 .../k8s/ExecutorPodsSnapshotSuite.scala       |  4 +-
 .../k8s/ExecutorPodsSnapshotsStoreSuite.scala | 33 ++++++++------
 11 files changed, 103 insertions(+), 56 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
index 8b55e2c7dbee5..2a06b1a98e91a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala
@@ -236,7 +236,7 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
           }
           val data = new ExecutorData(executorRef, executorAddress, hostname,
             0, cores, logUrlHandler.applyPattern(logUrls, attributes), attributes,
-            resourcesInfo, resourceProfileId)
+            resourcesInfo, resourceProfileId, registrationTs = System.currentTimeMillis())
           // This must be synchronized because variables mutated
           // in this block are read when requesting executors
           CoarseGrainedSchedulerBackend.this.synchronized {
@@ -557,6 +557,10 @@ class CoarseGrainedSchedulerBackend(scheduler: TaskSchedulerImpl, val rpcEnv: Rp
     executorDataMap.keySet.toSeq
   }
 
+  def getExecutorsWithRegistrationTs(): Map[String, Long] = synchronized {
+    executorDataMap.mapValues(v => v.registrationTs).toMap
+  }
+
   override def isExecutorActive(id: String): Boolean = synchronized {
     executorDataMap.contains(id) &&
       !executorsPendingToRemove.contains(id) &&
diff --git a/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala b/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
index 062146174f6a8..86b44e835368c 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/cluster/ExecutorData.scala
@@ -30,6 +30,7 @@ import org.apache.spark.scheduler.ExecutorResourceInfo
  * @param totalCores The total number of cores available to the executor
  * @param resourcesInfo The information of the currently available resources on the executor
  * @param resourceProfileId The id of the ResourceProfile being used by this executor
+ * @param registrationTs The registration timestamp of this executor
  */
 private[cluster] class ExecutorData(
     val executorEndpoint: RpcEndpointRef,
@@ -40,6 +41,7 @@ private[cluster] class ExecutorData(
     override val logUrlMap: Map[String, String],
     override val attributes: Map[String, String],
     override val resourcesInfo: Map[String, ExecutorResourceInfo],
-    override val resourceProfileId: Int
+    override val resourceProfileId: Int,
+    val registrationTs: Long
 ) extends ExecutorInfo(executorHost, totalCores, logUrlMap, attributes,
   resourcesInfo, resourceProfileId)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index b08875bdd3420..8a415058cc690 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -417,6 +417,17 @@ private[spark] object Config extends Logging {
       .booleanConf
       .createWithDefault(false)
 
+  val KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA =
+    ConfigBuilder("spark.kubernetes.executor.missingPodDetectDelta")
+      .doc("When a registered executor's POD is missing from the Kubernetes API server's polled " +
+        "list of PODs then this delta time is taken as the accepted time difference between the " +
+        "registration time and the time of the polling. After this time the POD is considered " +
+        "missing from the cluster and the executor will be removed.")
+      .version("3.1.1")
+      .timeConf(TimeUnit.MILLISECONDS)
+      .checkValue(delay => delay > 0, "delay must be a positive time value")
+      .createWithDefaultString("30s")
+
   val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label."
   val KUBERNETES_DRIVER_ANNOTATION_PREFIX = "spark.kubernetes.driver.annotation."
   val KUBERNETES_DRIVER_SERVICE_ANNOTATION_PREFIX = "spark.kubernetes.driver.service.annotation."
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
index 66cba55d8ef3a..3869ef90bfe5a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocator.scala
@@ -70,7 +70,7 @@ private[spark] class ExecutorPodsAllocator(
 
   private val hasPendingPods = new AtomicBoolean()
 
-  private var lastSnapshot = ExecutorPodsSnapshot(Nil)
+  private var lastSnapshot = ExecutorPodsSnapshot()
 
   // Executors that have been deleted by this allocator but not yet detected as deleted in
   // a snapshot from the API server. This is used to deny registration from these executors
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
index d6b75824d05d8..b1a6694b9fa3a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala
@@ -44,6 +44,9 @@ private[spark] class ExecutorPodsLifecycleManager(
   private val eventProcessingInterval = conf.get(KUBERNETES_EXECUTOR_EVENT_PROCESSING_INTERVAL)
 
   private lazy val shouldDeleteExecutors = conf.get(KUBERNETES_DELETE_EXECUTORS)
+  private lazy val missingPodDetectDelta = conf.get(KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA)
+
+  private var lastFullSnapshotTs: Long = 0
 
   def start(schedulerBackend: KubernetesClusterSchedulerBackend): Unit = {
     snapshotsStore.addSubscriber(eventProcessingInterval) {
@@ -83,33 +86,34 @@ private[spark] class ExecutorPodsLifecycleManager(
 
     // Reconcile the case where Spark claims to know about an executor but the corresponding pod
     // is missing from the cluster. This would occur if we miss a deletion event and the pod
-    // transitions immediately from running io absent. We only need to check against the latest
-    // snapshot for this, and we don't do this for executors in the deleted executors cache or
-    // that we just removed in this round.
-    if (snapshots.nonEmpty) {
-      val latestSnapshot = snapshots.last
-      (schedulerBackend.getExecutorIds().map(_.toLong).toSet
-        -- latestSnapshot.executorPods.keySet
-        -- execIdsRemovedInThisRound).foreach { missingExecutorId =>
-        if (removedExecutorsCache.getIfPresent(missingExecutorId) == null) {
-          val exitReasonMessage = s"The executor with ID $missingExecutorId was not found in the" +
-            s" cluster but we didn't get a reason why. Marking the executor as failed. The" +
-            s" executor may have been deleted but the driver missed the deletion event."
+    // transitions immediately from running to absent. We only need to check against the latest
+    // fresh full snapshot (coming from ExecutorPodsPollingSnapshotSource) for this, and we don't
+    // do this for executors in the deleted executors cache or that we just removed in this round.
+    if (snapshots.nonEmpty && lastFullSnapshotTs != snapshots.last.fullSnapshotTs) {
+      lastFullSnapshotTs = snapshots.last.fullSnapshotTs
+      val lostExecutorsWithRegistrationTs =
+        schedulerBackend.getExecutorsWithRegistrationTs().map(t => (t._1.toLong, t._2)) --
+        snapshots.last.executorPods.keySet -- execIdsRemovedInThisRound
+
+      lostExecutorsWithRegistrationTs.foreach { case (lostExecId, lostExecRegistrationTs) =>
+        if (removedExecutorsCache.getIfPresent(lostExecId) == null &&
+            lastFullSnapshotTs - lostExecRegistrationTs > missingPodDetectDelta) {
+          val exitReasonMessage = s"The executor with ID $lostExecId (registered at " +
+            s"$lostExecRegistrationTs ms) was not found in the cluster at the polling time " +
+            s"($lastFullSnapshotTs ms) which is after the accepted detect delta time " +
+            s"($missingPodDetectDelta ms) configured by " +
+            s"`${KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA.key}`. " +
+            "The executor may have been deleted but the driver missed the deletion event. " +
+            "Marking this executor as failed."
           logDebug(exitReasonMessage)
           val exitReason = ExecutorExited(
             UNKNOWN_EXIT_CODE,
             exitCausedByApp = false,
             exitReasonMessage)
-          schedulerBackend.doRemoveExecutor(missingExecutorId.toString, exitReason)
-          execIdsRemovedInThisRound += missingExecutorId
+          schedulerBackend.doRemoveExecutor(lostExecId.toString, exitReason)
         }
       }
     }
-
-    if (execIdsRemovedInThisRound.nonEmpty) {
-      logDebug(s"Removed executors with ids ${execIdsRemovedInThisRound.mkString(",")}" +
-        s" from Spark that were either found to be deleted or non-existent in the cluster.")
-    }
   }
 
   private def onFinalNonDeletedState(
@@ -186,4 +190,3 @@ private[spark] class ExecutorPodsLifecycleManager(
 private object ExecutorPodsLifecycleManager {
   val UNKNOWN_EXIT_CODE = -1
 }
-
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
index bd8da1080111f..16064dc66544a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshot.scala
@@ -27,24 +27,26 @@ import org.apache.spark.internal.Logging
 /**
  * An immutable view of the current executor pods that are running in the cluster.
  */
-private[spark] case class ExecutorPodsSnapshot(executorPods: Map[Long, ExecutorPodState]) {
+private[spark] case class ExecutorPodsSnapshot(
+    executorPods: Map[Long, ExecutorPodState],
+    fullSnapshotTs: Long) {
 
   import ExecutorPodsSnapshot._
 
   def withUpdate(updatedPod: Pod): ExecutorPodsSnapshot = {
     val newExecutorPods = executorPods ++ toStatesByExecutorId(Seq(updatedPod))
-    new ExecutorPodsSnapshot(newExecutorPods)
+    new ExecutorPodsSnapshot(newExecutorPods, fullSnapshotTs)
   }
 }
 
 object ExecutorPodsSnapshot extends Logging {
   private var shouldCheckAllContainers: Boolean = _
 
-  def apply(executorPods: Seq[Pod]): ExecutorPodsSnapshot = {
-    ExecutorPodsSnapshot(toStatesByExecutorId(executorPods))
+  def apply(executorPods: Seq[Pod], fullSnapshotTs: Long): ExecutorPodsSnapshot = {
+    ExecutorPodsSnapshot(toStatesByExecutorId(executorPods), fullSnapshotTs)
   }
 
-  def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState])
+  def apply(): ExecutorPodsSnapshot = ExecutorPodsSnapshot(Map.empty[Long, ExecutorPodState], 0)
 
   def setShouldCheckAllContainers(watchAllContainers: Boolean): Unit = {
     shouldCheckAllContainers = watchAllContainers
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
index 8aa20bfbc82db..502892314d667 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreImpl.scala
@@ -23,7 +23,11 @@ import javax.annotation.concurrent.GuardedBy
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-import org.apache.spark.util.{ThreadUtils, Utils}
+import org.apache.spark.internal.Logging
+import org.apache.spark.util.Clock
+import org.apache.spark.util.SystemClock
+import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.Utils
 
 /**
  * Controls the propagation of the Spark application's executor pods state to subscribers that
@@ -47,8 +51,10 @@ import org.apache.spark.util.{ThreadUtils, Utils}
  * time-windowed chunks. Each subscriber can choose to receive their snapshot chunks at different
  * time intervals.
  */
-private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: ScheduledExecutorService)
-  extends ExecutorPodsSnapshotsStore {
+private[spark] class ExecutorPodsSnapshotsStoreImpl(
+    subscribersExecutor: ScheduledExecutorService,
+    clock: Clock = new SystemClock)
+  extends ExecutorPodsSnapshotsStore with Logging {
 
   private val SNAPSHOT_LOCK = new Object()
 
@@ -93,7 +99,7 @@ private[spark] class ExecutorPodsSnapshotsStoreImpl(subscribersExecutor: Schedul
   }
 
   override def replaceSnapshot(newSnapshot: Seq[Pod]): Unit = SNAPSHOT_LOCK.synchronized {
-    currentSnapshot = ExecutorPodsSnapshot(newSnapshot)
+    currentSnapshot = ExecutorPodsSnapshot(newSnapshot, clock.getTimeMillis())
     addCurrentSnapshotToSubscribers()
   }
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
index e64797213f33f..2a727cf19981a 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/DeterministicExecutorPodsSnapshotsStore.scala
@@ -19,10 +19,14 @@ package org.apache.spark.scheduler.cluster.k8s
 import io.fabric8.kubernetes.api.model.Pod
 import scala.collection.mutable
 
+import org.apache.spark.util.ManualClock
+
 class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore {
 
   ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
 
+  val clock = new ManualClock()
+
   private val snapshotsBuffer = mutable.Buffer.empty[ExecutorPodsSnapshot]
   private val subscribers = mutable.Buffer.empty[Seq[ExecutorPodsSnapshot] => Unit]
 
@@ -47,7 +51,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
   }
 
   override def replaceSnapshot(newSnapshot: Seq[Pod]): Unit = {
-    currentSnapshot = ExecutorPodsSnapshot(newSnapshot)
+    currentSnapshot = ExecutorPodsSnapshot(newSnapshot, clock.getTimeMillis())
     snapshotsBuffer += currentSnapshot
   }
 
@@ -56,7 +60,7 @@ class DeterministicExecutorPodsSnapshotsStore extends ExecutorPodsSnapshotsStore
       case (_, PodDeleted(_)) => false
       case _ => true
     }
-    currentSnapshot = ExecutorPodsSnapshot(nonDeleted)
+    currentSnapshot = ExecutorPodsSnapshot(nonDeleted, clock.getTimeMillis())
     snapshotsBuffer += currentSnapshot
   }
 }
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
index 9920f4d3ea737..7021cb78da0fa 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala
@@ -56,7 +56,7 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
     val removedExecutorsCache = CacheBuilder.newBuilder().build[java.lang.Long, java.lang.Long]
     snapshotsStore = new DeterministicExecutorPodsSnapshotsStore()
     namedExecutorPods = mutable.Map.empty[String, PodResource[Pod, DoneablePod]]
-    when(schedulerBackend.getExecutorIds()).thenReturn(Seq.empty[String])
+    when(schedulerBackend.getExecutorsWithRegistrationTs()).thenReturn(Map.empty[String, Long])
     when(kubernetesClient.pods()).thenReturn(podOperations)
     when(podOperations.withName(any(classOf[String]))).thenAnswer(namedPodsAnswer())
     eventHandlerUnderTest = new ExecutorPodsLifecycleManager(
@@ -90,13 +90,23 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte
 
   test("When the scheduler backend lists executor ids that aren't present in the cluster," +
     " remove those executors from Spark.") {
-    when(schedulerBackend.getExecutorIds()).thenReturn(Seq("1"))
-    val msg = s"The executor with ID 1 was not found in the cluster but we didn't" +
-      s" get a reason why. Marking the executor as failed. The executor may have been" +
-      s" deleted but the driver missed the deletion event."
-    val expectedLossReason = ExecutorExited(-1, exitCausedByApp = false, msg)
+      when(schedulerBackend.getExecutorsWithRegistrationTs()).thenReturn(Map("1" -> 7L))
+    val missingPodDelta =
+      eventHandlerUnderTest.conf.get(Config.KUBERNETES_EXECUTOR_MISSING_POD_DETECT_DELTA)
+    snapshotsStore.clock.advance(missingPodDelta + 7)
     snapshotsStore.replaceSnapshot(Seq.empty[Pod])
     snapshotsStore.notifySubscribers()
+    verify(schedulerBackend, never()).doRemoveExecutor(any(), any())
+
+    // 1 more millisecond and the accepted delta is over so the missing POD will be detected
+    snapshotsStore.clock.advance(1)
+    snapshotsStore.replaceSnapshot(Seq.empty[Pod])
+    snapshotsStore.notifySubscribers()
+    val msg = "The executor with ID 1 (registered at 7 ms) was not found in the cluster at " +
+      "the polling time (30008 ms) which is after the accepted detect delta time (30000 ms) " +
+      "configured by `spark.kubernetes.executor.missingPodDetectDelta`. The executor may have " +
+      "been deleted but the driver missed the deletion event. Marking this executor as failed."
+    val expectedLossReason = ExecutorExited(-1, exitCausedByApp = false, msg)
     verify(schedulerBackend).doRemoveExecutor("1", expectedLossReason)
   }
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
index 96ad6b5ab32a5..7d25450f2edf4 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotSuite.scala
@@ -27,7 +27,7 @@ class ExecutorPodsSnapshotSuite extends SparkFunSuite {
     (pod, state(pod))
 
   def doTest(testCases: Seq[(Pod, ExecutorPodState)]): Unit = {
-    val snapshot = ExecutorPodsSnapshot(testCases.map(_._1))
+    val snapshot = ExecutorPodsSnapshot(testCases.map(_._1), 0)
     for (((_, state), i) <- testCases.zipWithIndex) {
       assertResult(state.getClass.getName, s"executor ID $i") {
         snapshot.executorPods(i).getClass.getName
@@ -68,7 +68,7 @@ class ExecutorPodsSnapshotSuite extends SparkFunSuite {
     val originalPods = Seq(
       pendingExecutor(0),
       runningExecutor(1))
-    val originalSnapshot = ExecutorPodsSnapshot(originalPods)
+    val originalSnapshot = ExecutorPodsSnapshot(originalPods, 0)
     val snapshotWithUpdatedPod = originalSnapshot.withUpdate(succeededExecutor(1))
     assert(snapshotWithUpdatedPod.executorPods ===
       Map(
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
index 614c198bd9caf..b4240bbacf106 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsSnapshotsStoreSuite.scala
@@ -26,15 +26,18 @@ import scala.collection.mutable
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.deploy.k8s.Constants._
+import org.apache.spark.util.ManualClock
 
 class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter {
 
   private var eventBufferScheduler: DeterministicScheduler = _
   private var eventQueueUnderTest: ExecutorPodsSnapshotsStoreImpl = _
+  private var clock: ManualClock = _
 
   before {
     eventBufferScheduler = new DeterministicScheduler()
-    eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler)
+    clock = new ManualClock()
+    eventQueueUnderTest = new ExecutorPodsSnapshotsStoreImpl(eventBufferScheduler, clock)
     ExecutorPodsSnapshot.setShouldCheckAllContainers(false)
   }
 
@@ -52,6 +55,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     assert(receivedSnapshots1 === Seq(ExecutorPodsSnapshot()))
     assert(receivedSnapshots2 === Seq(ExecutorPodsSnapshot()))
 
+    clock.advance(100)
     pushPodWithIndex(1)
     // Force time to move forward so that the buffer is emitted, scheduling the
     // processing task on the subscription executor...
@@ -60,7 +64,7 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
 
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     assert(receivedSnapshots2 === Seq(ExecutorPodsSnapshot()))
 
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
@@ -68,29 +72,29 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     // Don't repeat snapshots
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     assert(receivedSnapshots2 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
     pushPodWithIndex(2)
     pushPodWithIndex(3)
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
 
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)), 0)))
     assert(receivedSnapshots2 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
 
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots1 === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(1), podWithIndex(2), podWithIndex(3)), 0)))
     assert(receivedSnapshots1 === receivedSnapshots2)
   }
 
@@ -113,13 +117,14 @@ class ExecutorPodsSnapshotsStoreSuite extends SparkFunSuite with BeforeAndAfter
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0)))
+    clock.advance(100)
     eventQueueUnderTest.replaceSnapshot(Seq(podWithIndex(2)))
     eventBufferScheduler.tick(1000, TimeUnit.MILLISECONDS)
     assert(receivedSnapshots === Seq(
       ExecutorPodsSnapshot(),
-      ExecutorPodsSnapshot(Seq(podWithIndex(1))),
-      ExecutorPodsSnapshot(Seq(podWithIndex(2)))))
+      ExecutorPodsSnapshot(Seq(podWithIndex(1)), 0),
+      ExecutorPodsSnapshot(Seq(podWithIndex(2)), 100)))
   }
 
   private def pushPodWithIndex(index: Int): Unit =

From 1ab0f02b3e755c6ec057d6561ffb6ddd7e53c7cd Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Fri, 15 Jan 2021 17:02:29 -0800
Subject: [PATCH 1267/1280] [SPARK-34060][SQL][FOLLOWUP] Preserve
 serializability of canonicalized CatalogTable

### What changes were proposed in this pull request?
Replace `toMap` by `map(identity).toMap` while getting canonicalized representation of `CatalogTable`. `CatalogTable` became not serializable after https://github.com/apache/spark/pull/31112 due to usage of `filterKeys`. The workaround was taken from https://github.com/scala/bug/issues/7005.

### Why are the changes needed?
This prevents the errors like:
```
[info]   org.apache.spark.SparkException: Job aborted due to stage failure: Task not serializable: java.io.NotSerializableException: scala.collection.immutable.MapLike$$anon$1
[info]   Cause: java.io.NotSerializableException: scala.collection.immutable.MapLike$$anon$1
```

### Does this PR introduce _any_ user-facing change?
Should not.

### How was this patch tested?
By running the test suite affected by https://github.com/apache/spark/pull/31112:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *AlterTableDropPartitionSuite"
```

Closes #31197 from MaxGekk/fix-caching-hive-table-2-followup.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit c3d81fbe79014f693cf93c02e31af401727761d7)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../org/apache/spark/sql/catalyst/catalog/interface.scala    | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
index 802e5780afb86..6d658326369b5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala
@@ -432,7 +432,10 @@ object CatalogTable {
     table.copy(
       createTime = 0L,
       lastAccessTime = 0L,
-      properties = table.properties.filterKeys(!nondeterministicProps.contains(_)).toMap,
+      properties = table.properties
+        .filterKeys(!nondeterministicProps.contains(_))
+        .map(identity)
+        .toMap,
       stats = None,
       ignoredProperties = Map.empty
     )

From 403a4ac5762637cbed1fa570447a51d45ea03f69 Mon Sep 17 00:00:00 2001
From: William Hyun <williamhyun3@gmail.com>
Date: Sat, 16 Jan 2021 11:24:32 -0800
Subject: [PATCH 1268/1280] [MINOR][DOCS] Update Parquet website link

### What changes were proposed in this pull request?
This PR aims to update the Parquet website link from http://parquet.io to https://parquet.apache.orc

### Why are the changes needed?
The old website goes to the incubator site.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
N/A

Closes #31208 from williamhyun/minor-parquet.

Authored-by: William Hyun <williamhyun3@gmail.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit 1cf09b77eb1bdd98cdfa37703f9318b821831c13)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 docs/sql-data-sources-parquet.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-data-sources-parquet.md b/docs/sql-data-sources-parquet.md
index 7875b10b0f697..d34f8a698facb 100644
--- a/docs/sql-data-sources-parquet.md
+++ b/docs/sql-data-sources-parquet.md
@@ -22,7 +22,7 @@ license: |
 * Table of contents
 {:toc}
 
-[Parquet](http://parquet.io) is a columnar format that is supported by many other data processing systems.
+[Parquet](https://parquet.apache.org) is a columnar format that is supported by many other data processing systems.
 Spark SQL provides support for both reading and writing Parquet files that automatically preserves the schema
 of the original data. When reading Parquet files, all columns are automatically converted to be nullable for
 compatibility reasons.

From d8ce2249d0fb62dce161f5caa4a9143295949f93 Mon Sep 17 00:00:00 2001
From: Mitsuru Kariya <Mitsuru.Kariya@oss.nttdata.com>
Date: Mon, 18 Jan 2021 13:18:03 +0900
Subject: [PATCH 1269/1280] [MINOR][DOCS] Fix typos in sql-ref-datatypes.md

### What changes were proposed in this pull request?
Fixing typos in the docs sql-ref-datatypes.md.

### Why are the changes needed?
To display '<element_type>' correctly.

### Does this PR introduce _any_ user-facing change?
No.

### How was this patch tested?
Manually run jekyll.

before this fix
![image](https://user-images.githubusercontent.com/2217224/104865408-3df33600-597f-11eb-857b-c6223ff9159a.png)

after this fix
![image](https://user-images.githubusercontent.com/2217224/104865458-62e7a900-597f-11eb-8a21-6d838eecaaf2.png)

Closes #31221 from kariya-mitsuru/fix-typo.

Authored-by: Mitsuru Kariya <Mitsuru.Kariya@oss.nttdata.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 536a7258a829299a13035eb3550e6ce6f7632677)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/sql-ref-datatypes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/sql-ref-datatypes.md b/docs/sql-ref-datatypes.md
index f27f1a0ca967f..144594bd5a222 100644
--- a/docs/sql-ref-datatypes.md
+++ b/docs/sql-ref-datatypes.md
@@ -191,7 +191,7 @@ The following table shows the type names as well as aliases used in Spark SQL pa
 |**BinaryType**|BINARY|
 |**DecimalType**|DECIMAL, DEC, NUMERIC|
 |**CalendarIntervalType**|INTERVAL|
-|**ArrayType**|ARRAY<element_type>|
+|**ArrayType**|ARRAY\<element_type>|
 |**StructType**|STRUCT<field1_name: field1_type, field2_name: field2_type, ...>|
 |**MapType**|MAP<key_type, value_type>|
 

From 70c0bc9a3358fc36b967cc28ae9232fe7437ab6a Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Mon, 18 Jan 2021 14:31:17 +0900
Subject: [PATCH 1270/1280] [SPARK-33819][CORE][FOLLOWUP][3.0] Restore the
 constructor of SingleFileEventLogFileReader to remove Mima exclusion

### What changes were proposed in this pull request?

This PR proposes to remove Mima exclusion via restoring the old constructor of SingleFileEventLogFileReader. This partially adopts the remaining parts of #30814 which was excluded while porting back.

### Why are the changes needed?

To remove unnecessary Mima exclusion.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass CIs.

Closes #31225 from HeartSaVioR/SPARK-33819-followup-branch-3.0.

Authored-by: Dongjoon Hyun <dongjoon@apache.org>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../apache/spark/deploy/history/EventLogFileReaders.scala    | 4 +++-
 project/MimaExcludes.scala                                   | 5 +----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
index 6fe3a7bcafa06..b4771c80a175f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileReaders.scala
@@ -167,9 +167,11 @@ object EventLogFileReader {
 private[history] class SingleFileEventLogFileReader(
     fs: FileSystem,
     path: Path,
-    maybeStatus: Option[FileStatus] = None) extends EventLogFileReader(fs, path) {
+    maybeStatus: Option[FileStatus]) extends EventLogFileReader(fs, path) {
   private lazy val status = maybeStatus.getOrElse(fileSystem.getFileStatus(rootPath))
 
+  def this(fs: FileSystem, path: Path) = this(fs, path, None)
+
   override def lastIndex: Option[Long] = None
 
   override def fileSizeForLastIndex: Long = status.getLen
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index b4e36dc76384d..b79962446d84b 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -461,10 +461,7 @@ object MimaExcludes {
     ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryListener#QueryStartedEvent.this"),
 
     // [SPARK-30667][CORE] Add allGather method to BarrierTaskContext
-    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.RequestToSync"),
-
-    // [SPARK-33790][CORE] Reduce the rpc call of getFileStatus in SingleFileEventLogFileReader
-    ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.history.SingleFileEventLogFileReader.this")
+    ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.RequestToSync")
   )
 
   // Exclude rules for 2.4.x

From f705b659c2de7ad94f4b5a45d10223daf7788d70 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 19 Jan 2021 03:31:25 +0000
Subject: [PATCH 1271/1280] [SPARK-34027][SQL][3.0] Refresh cache in `ALTER
 TABLE .. RECOVER PARTITIONS`

### What changes were proposed in this pull request?
Invoke `refreshTable()` from `CatalogImpl` which refreshes the cache in v1 `ALTER TABLE .. RECOVER PARTITIONS`.

### Why are the changes needed?
This fixes the issues portrayed by the example:
```sql
spark-sql> create table tbl (col int, part int) using parquet partitioned by (part);
spark-sql> insert into tbl partition (part=0) select 0;
spark-sql> cache table tbl;
spark-sql> select * from tbl;
0	0
spark-sql> show table extended like 'tbl' partition(part=0);
default	tbl	false	Partition Values: [part=0]
Location: file:/Users/maximgekk/proj/recover-partitions-refresh-cache/spark-warehouse/tbl/part=0
...
```
Create new partition by copying the existing one:
```
$ cp -r /Users/maximgekk/proj/recover-partitions-refresh-cache/spark-warehouse/tbl/part=0 /Users/maximgekk/proj/recover-partitions-refresh-cache/spark-warehouse/tbl/part=1
```
```sql
spark-sql> alter table tbl recover partitions;
spark-sql> select * from tbl;
0	0
```

The last query must return `0	1` since it has been recovered by `ALTER TABLE .. RECOVER PARTITIONS`.

### Does this PR introduce _any_ user-facing change?
Yes. After the changes for the example above:
```sql
...
spark-sql> alter table tbl recover partitions;
spark-sql> select * from tbl;
0	0
0	1
```

### How was this patch tested?
By running the affected test suite:
```
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *CachedTableSuite"
$ build/sbt -Phive -Phive-thriftserver "test:testOnly *HiveSchemaInferenceSuite"
```

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: Wenchen Fan <wenchendatabricks.com>
(cherry picked from commit dee596e3efe54651aa1e7c467b4f987f662e60b0)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #31236 from MaxGekk/recover-partitions-refresh-cache-3.0.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../apache/spark/sql/execution/command/ddl.scala |  2 +-
 .../org/apache/spark/sql/CachedTableSuite.scala  | 16 ++++++++++++++--
 .../sql/hive/HiveSchemaInferenceSuite.scala      | 11 ++++++-----
 .../hive/PartitionedTablePerfStatsSuite.scala    |  4 ++--
 4 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 8fa21395e0420..588f52d2e69b1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -675,7 +675,7 @@ case class AlterTableRecoverPartitionsCommand(
     // This is always the case for Hive format tables, but is not true for Datasource tables created
     // before Spark 2.1 unless they are converted via `msck repair table`.
     spark.sessionState.catalog.alterTable(table.copy(tracksPartitionsInCatalog = true))
-    catalog.refreshTable(tableName)
+    spark.catalog.refreshTable(tableIdentWithDB)
     logInfo(s"Recovered all partitions ($total).")
     Seq.empty[Row]
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index de8d595ff52f2..dc16418d50199 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1305,7 +1305,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
     }
   }
 
-  test("SPARK-34055: refresh cache in partition adding") {
+  private def testCacheRefreshing(cmd: String => DataFrame): Unit = {
     withTable("t") {
       sql("CREATE TABLE t (id int, part int) USING parquet PARTITIONED BY (part)")
       sql("INSERT INTO t PARTITION (part=0) SELECT 0")
@@ -1326,9 +1326,21 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
       val part1Loc = part0Loc.replace("part=0", "part=1")
       FileUtils.copyDirectory(new File(part0Loc), new File(part1Loc))
 
-      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$part1Loc'")
+      cmd(part1Loc)
       assert(spark.catalog.isCached("t"))
       checkAnswer(sql("SELECT * FROM t"), Seq(Row(0, 0), Row(0, 1)))
     }
   }
+
+  test("SPARK-34055: refresh cache in partition adding") {
+    testCacheRefreshing { location =>
+      sql(s"ALTER TABLE t ADD PARTITION (part=1) LOCATION '$location'")
+    }
+  }
+
+  test("SPARK-34027: refresh cache in partitions recovering") {
+    testCacheRefreshing { _ =>
+      sql("ALTER TABLE t RECOVER PARTITIONS")
+    }
+  }
 }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
index 590ef949ffbd7..6496d3154feea 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSchemaInferenceSuite.scala
@@ -118,11 +118,6 @@ class HiveSchemaInferenceSuite
         properties = Map.empty),
       true)
 
-    // Add partition records (if specified)
-    if (!partitionCols.isEmpty) {
-      spark.catalog.recoverPartitions(TEST_TABLE_NAME)
-    }
-
     // Check that the table returned by HiveExternalCatalog has schemaPreservesCase set to false
     // and that the raw table returned by the Hive client doesn't have any Spark SQL properties
     // set (table needs to be obtained from client since HiveExternalCatalog filters these
@@ -130,6 +125,12 @@ class HiveSchemaInferenceSuite
     assert(!externalCatalog.getTable(DATABASE, TEST_TABLE_NAME).schemaPreservesCase)
     val rawTable = client.getTable(DATABASE, TEST_TABLE_NAME)
     assert(rawTable.properties.filterKeys(_.startsWith(DATASOURCE_SCHEMA_PREFIX)) == Map.empty)
+
+    // Add partition records (if specified)
+    if (!partitionCols.isEmpty) {
+      spark.catalog.recoverPartitions(TEST_TABLE_NAME)
+    }
+
     schema
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
index 3af163af0968c..f9ae1e00cfa36 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/PartitionedTablePerfStatsSuite.scala
@@ -405,8 +405,8 @@ class PartitionedTablePerfStatsSuite
           })
           executorPool.shutdown()
           executorPool.awaitTermination(30, TimeUnit.SECONDS)
-          assert(HiveCatalogMetrics.METRIC_FILES_DISCOVERED.getCount() == 50)
-          assert(HiveCatalogMetrics.METRIC_PARALLEL_LISTING_JOB_COUNT.getCount() == 1)
+          assert(HiveCatalogMetrics.METRIC_FILES_DISCOVERED.getCount() == 100)
+          assert(HiveCatalogMetrics.METRIC_PARALLEL_LISTING_JOB_COUNT.getCount() == 2)
         }
       }
     }

From 67b9f6cdfe03e4c24ff6678cc16ebf8c293b0751 Mon Sep 17 00:00:00 2001
From: Max Gekk <max.gekk@gmail.com>
Date: Tue, 19 Jan 2021 18:11:06 +0900
Subject: [PATCH 1272/1280] [SPARK-34153][SQL][3.1][3.0] Remove unused
 `getRawTable()` from `HiveExternalCatalog.alterPartitions()`

Remove unused call of `getRawTable()` from `HiveExternalCatalog.alterPartitions()`.

It reduces the number of calls to Hive External catalog.

No

By existing test suites.

Authored-by: Max Gekk <max.gekkgmail.com>
Signed-off-by: HyukjinKwon <gurwls223apache.org>
(cherry picked from commit bea10a6274df939e518a931dba4e6e0ae1fa6d01)
Signed-off-by: Max Gekk <max.gekkgmail.com>

Closes #31241 from MaxGekk/remove-getRawTable-from-alterPartitions-3.1.

Authored-by: Max Gekk <max.gekk@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit 246ff315282302705dc3bdc5471df20c7bb6c7c9)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 .../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala  | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index bd44b0b8f139c..2d4e45aecdbb1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -1153,9 +1153,6 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
       table: String,
       newParts: Seq[CatalogTablePartition]): Unit = withClient {
     val metaStoreParts = newParts.map(p => p.copy(spec = toMetaStorePartitionSpec(p.spec)))
-
-    val rawTable = getRawTable(db, table)
-
     // convert partition statistics to properties so that we can persist them through hive api
     val withStatsProps = metaStoreParts.map { p =>
       if (p.stats.isDefined) {

From 5a93bcbcc27d7d2f1e816e466478f67d4e3df105 Mon Sep 17 00:00:00 2001
From: Liang Zhang <liang.zhang@databricks.com>
Date: Wed, 20 Jan 2021 08:26:52 +0800
Subject: [PATCH 1273/1280] [MINOR][ML] Increase the timeout for
 StreamingLinearRegressionSuite to 60s

### What changes were proposed in this pull request?

Increase the timeout for StreamingLinearRegressionSuite to 60s to deflake the test.

### Why are the changes needed?

Reduce merge conflict.

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #31248 from liangz1/increase-timeout.

Authored-by: Liang Zhang <liang.zhang@databricks.com>
Signed-off-by: Weichen Xu <weichen.xu@databricks.com>
(cherry picked from commit f7ff7ff0a5a251d7983d933dbe8c20882356e5bf)
Signed-off-by: Weichen Xu <weichen.xu@databricks.com>
---
 .../spark/mllib/regression/StreamingLinearRegressionSuite.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
index 8e2d7d10f2ce2..b8342f84be44b 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/regression/StreamingLinearRegressionSuite.scala
@@ -31,7 +31,7 @@ class StreamingLinearRegressionSuite
   with TestSuiteBase {
 
   // use longer wait time to ensure job completion
-  override def maxWaitTimeMillis: Int = 20000
+  override def maxWaitTimeMillis: Int = 60000
 
   // Assert that two values are equal within tolerance epsilon
   def assertEqual(v1: Double, v2: Double, epsilon: Double): Unit = {

From b5b1da961e65227c1fbc59544390f31def1d15f1 Mon Sep 17 00:00:00 2001
From: Norbert Schultz <norbert.schultz@reactivecore.de>
Date: Wed, 20 Jan 2021 09:39:13 +0900
Subject: [PATCH 1274/1280] [SPARK-34115][CORE] Check SPARK_TESTING as lazy val
 to avoid slowdown

### What changes were proposed in this pull request?
Check SPARK_TESTING as lazy val to avoid slow down when there are many environment variables

### Why are the changes needed?
If there are many environment variables, sys.env slows is very slow. As Utils.isTesting is called very often during Dataframe-Optimization, this can slow down evaluation very much.

An example for triggering the problem can be found in the bug ticket https://issues.apache.org/jira/browse/SPARK-34115

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
With the example provided in the ticket.

Closes #31244 from nob13/bug/34115.

Lead-authored-by: Norbert Schultz <norbert.schultz@reactivecore.de>
Co-authored-by: Norbert Schultz <noschultz@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit c3d8352ca1a59ce5cc37840919c0e799f5150efa)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 core/src/main/scala/org/apache/spark/util/Utils.scala | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index c7db2127a6f04..867cd1983be8f 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1879,7 +1879,9 @@ private[spark] object Utils extends Logging {
    * Indicates whether Spark is currently running unit tests.
    */
   def isTesting: Boolean = {
-    sys.env.contains("SPARK_TESTING") || sys.props.contains(IS_TESTING.key)
+    // Scala's `sys.env` creates a ton of garbage by constructing Scala immutable maps, so
+    // we directly use the Java APIs instead.
+    System.getenv("SPARK_TESTING") != null || System.getProperty(IS_TESTING.key) != null
   }
 
   /**

From 89443ab1118b0e07acd639609094961f783b01e1 Mon Sep 17 00:00:00 2001
From: "yi.wu" <yi.wu@databricks.com>
Date: Wed, 20 Jan 2021 13:36:14 +0000
Subject: [PATCH 1275/1280] [SPARK-34178][SQL] Copy tags for the new node
 created by MultiInstanceRelation.newInstance

### What changes were proposed in this pull request?

Call `copyTagsFrom` for the new node created by `MultiInstanceRelation.newInstance()`.

### Why are the changes needed?

```scala
val df = spark.range(2)
df.join(df, df("id") <=> df("id")).show()
```

For this query, it's supposed to be non-ambiguous join by the rule `DetectAmbiguousSelfJoin` because of the same attribute reference in the condition:

https://github.com/apache/spark/blob/537a49fc0966b0b289b67ac9c6ea20093165b0da/sql/core/src/main/scala/org/apache/spark/sql/execution/analysis/DetectAmbiguousSelfJoin.scala#L125

However, `DetectAmbiguousSelfJoin` can not apply this prediction due to the right side plan doesn't contain the dataset_id TreeNodeTag, which is missing after `MultiInstanceRelation.newInstance`. That's why we should preserve the tags info for the copied node.

Fortunately, the query is still considered as non-ambiguous join because `DetectAmbiguousSelfJoin` only checks the left side plan and the reference is the same as the left side plan. However, this's not the expected behavior but only a coincidence.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Updated a unit test

Closes #31260 from Ngone51/fix-missing-tags.

Authored-by: yi.wu <yi.wu@databricks.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
(cherry picked from commit f4989772229e2ba35f1d005727b7d4d9f1369895)
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../spark/sql/catalyst/analysis/Analyzer.scala       |  6 +++++-
 .../apache/spark/sql/catalyst/trees/TreeNode.scala   |  2 +-
 .../org/apache/spark/sql/DataFrameJoinSuite.scala    | 12 +++++++++++-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index fbe60412b2f39..2dbabfcffebca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1012,7 +1012,10 @@ class Analyzer(
           }
           val key = catalog.name +: ident.namespace :+ ident.name
           AnalysisContext.get.relationCache.get(key).map(_.transform {
-            case multi: MultiInstanceRelation => multi.newInstance()
+            case multi: MultiInstanceRelation =>
+              val newRelation = multi.newInstance()
+              newRelation.copyTagsFrom(multi)
+              newRelation
           }).orElse {
             loaded.foreach(AnalysisContext.get.relationCache.update(key, _))
             loaded
@@ -1164,6 +1167,7 @@ class Analyzer(
         case oldVersion: MultiInstanceRelation
             if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
           val newVersion = oldVersion.newInstance()
+          newVersion.copyTagsFrom(oldVersion)
           Seq((oldVersion, newVersion))
 
         case oldVersion: SerializeFromObject
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index 4dc3627cd6a50..68bbd1b7dac5e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -91,7 +91,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    */
   private val tags: mutable.Map[TreeNodeTag[_], Any] = mutable.Map.empty
 
-  protected def copyTagsFrom(other: BaseType): Unit = {
+  def copyTagsFrom(other: BaseType): Unit = {
     // SPARK-32753: it only makes sense to copy tags to a new node
     // but it's too expensive to detect other cases likes node removal
     // so we make a compromise here to copy tags to node with no tags
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index a49f95f1ed134..0cf81b4867be6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Filter, HintInfo,
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
+import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin.LogicalPlanWithDatasetId
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
@@ -264,7 +265,16 @@ class DataFrameJoinSuite extends QueryTest
     withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
       val df = spark.range(2)
       // this throws an exception before the fix
-      df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
+      val plan = df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
+
+      plan match {
+        // SPARK-34178: we can't match the plan before the fix due to
+        // the right side plan doesn't contains dataset id.
+        case Join(
+          LogicalPlanWithDatasetId(_, leftId),
+          LogicalPlanWithDatasetId(_, rightId), _, _, _) =>
+          assert(leftId === rightId)
+      }
     }
   }
 

From 4690063f5c5d38a9f73ac76daf0d200df1a0d887 Mon Sep 17 00:00:00 2001
From: Andy Zhang <yue.zhang@databricks.com>
Date: Wed, 20 Jan 2021 16:38:01 -0800
Subject: [PATCH 1276/1280] [MINOR][TESTS] Increase tolerance to 0.2 for
 NaiveBayesSuite

### What changes were proposed in this pull request?
This test fails flakily. I found it failing in 1 out of 80 runs.
```
  Expected -0.35667494393873245 and -0.41914521201224453 to be within 0.15 using relative tolerance.
```
Increasing relative tolerance to 0.2 should improve flakiness.
```
0.2 * 0.35667494393873245 = 0.071 > 0.062 = |-0.35667494393873245 - (-0.41914521201224453)|
```

### Why are the changes needed?

### Does this PR introduce _any_ user-facing change?

### How was this patch tested?

Closes #31266 from Loquats/NaiveBayesSuite-reltol.

Authored-by: Andy Zhang <yue.zhang@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit c8c70d50026a8ed0b202f456b02df5adc905c4f7)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
(cherry picked from commit dad201e05d100021ab504a775684e5a31d3b04c7)
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 .../apache/spark/mllib/classification/NaiveBayesSuite.scala   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
index c20674718b101..4de10842c52d1 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
@@ -107,9 +107,9 @@ class NaiveBayesSuite extends SparkFunSuite with MLlibTestSparkContext {
     val modelIndex = piData.indices.zip(model.labels.map(_.toInt))
     try {
       for (i <- modelIndex) {
-        assert(piData(i._2) ~== model.pi(i._1) relTol 0.15)
+        assert(piData(i._2) ~== model.pi(i._1) relTol 0.2)
         for (j <- thetaData(i._2).indices) {
-          assert(thetaData(i._2)(j) ~== model.theta(i._1)(j) relTol 0.15)
+          assert(thetaData(i._2)(j) ~== model.theta(i._1)(j) relTol 0.2)
         }
       }
     } catch {

From 4e80f8cf49b18dcd4132f74253b01521a0cf65f3 Mon Sep 17 00:00:00 2001
From: HyukjinKwon <gurwls223@apache.org>
Date: Thu, 21 Jan 2021 09:55:52 +0900
Subject: [PATCH 1277/1280] Revert "[SPARK-34178][SQL] Copy tags for the new
 node created by MultiInstanceRelation.newInstance"

This reverts commit 89443ab1118b0e07acd639609094961f783b01e1.
---
 .../spark/sql/catalyst/analysis/Analyzer.scala       |  6 +-----
 .../apache/spark/sql/catalyst/trees/TreeNode.scala   |  2 +-
 .../org/apache/spark/sql/DataFrameJoinSuite.scala    | 12 +-----------
 3 files changed, 3 insertions(+), 17 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 2dbabfcffebca..fbe60412b2f39 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1012,10 +1012,7 @@ class Analyzer(
           }
           val key = catalog.name +: ident.namespace :+ ident.name
           AnalysisContext.get.relationCache.get(key).map(_.transform {
-            case multi: MultiInstanceRelation =>
-              val newRelation = multi.newInstance()
-              newRelation.copyTagsFrom(multi)
-              newRelation
+            case multi: MultiInstanceRelation => multi.newInstance()
           }).orElse {
             loaded.foreach(AnalysisContext.get.relationCache.update(key, _))
             loaded
@@ -1167,7 +1164,6 @@ class Analyzer(
         case oldVersion: MultiInstanceRelation
             if oldVersion.outputSet.intersect(conflictingAttributes).nonEmpty =>
           val newVersion = oldVersion.newInstance()
-          newVersion.copyTagsFrom(oldVersion)
           Seq((oldVersion, newVersion))
 
         case oldVersion: SerializeFromObject
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index 68bbd1b7dac5e..4dc3627cd6a50 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -91,7 +91,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product {
    */
   private val tags: mutable.Map[TreeNodeTag[_], Any] = mutable.Map.empty
 
-  def copyTagsFrom(other: BaseType): Unit = {
+  protected def copyTagsFrom(other: BaseType): Unit = {
     // SPARK-32753: it only makes sense to copy tags to a new node
     // but it's too expensive to detect other cases likes node removal
     // so we make a compromise here to copy tags to node with no tags
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 0cf81b4867be6..a49f95f1ed134 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.plans.logical.{BROADCAST, Filter, HintInfo,
 import org.apache.spark.sql.connector.catalog.CatalogManager
 import org.apache.spark.sql.execution.FileSourceScanExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
-import org.apache.spark.sql.execution.analysis.DetectAmbiguousSelfJoin.LogicalPlanWithDatasetId
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.execution.exchange.BroadcastExchangeExec
 import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
@@ -265,16 +264,7 @@ class DataFrameJoinSuite extends QueryTest
     withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
       val df = spark.range(2)
       // this throws an exception before the fix
-      val plan = df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
-
-      plan match {
-        // SPARK-34178: we can't match the plan before the fix due to
-        // the right side plan doesn't contains dataset id.
-        case Join(
-          LogicalPlanWithDatasetId(_, leftId),
-          LogicalPlanWithDatasetId(_, rightId), _, _, _) =>
-          assert(leftId === rightId)
-      }
+      df.join(df, df("id") <=> df("id")).queryExecution.optimizedPlan
     }
   }
 

From 785998b4e01e0bf97b480c2c86b8000b559ae551 Mon Sep 17 00:00:00 2001
From: Angerszhuuuu <angers.zhu@gmail.com>
Date: Thu, 21 Jan 2021 11:36:09 +0900
Subject: [PATCH 1278/1280] [SPARK-34181][DOC] Update Prerequisites for build
 doc of ruby 3.0 issue

### What changes were proposed in this pull request?
When ruby version is 3.0, jekyll server will failed with
```
yi.zhu$ SKIP_API=1 jekyll serve --watch
Configuration file: /Users/yi.zhu/Documents/project/Angerszhuuuu/spark/docs/_config.yml
            Source: /Users/yi.zhu/Documents/project/Angerszhuuuu/spark/docs
       Destination: /Users/yi.zhu/Documents/project/Angerszhuuuu/spark/docs/_site
 Incremental build: disabled. Enable with --incremental
      Generating...
                    done in 5.085 seconds.
 Auto-regeneration: enabled for '/Users/yi.zhu/Documents/project/Angerszhuuuu/spark/docs'
                    ------------------------------------------------
      Jekyll 4.2.0   Please append `--trace` to the `serve` command
                     for any additional information or backtrace.
                    ------------------------------------------------
<internal:/usr/local/Cellar/ruby/3.0.0_1/lib/ruby/3.0.0/rubygems/core_ext/kernel_require.rb>:85:in `require': cannot load such file -- webrick (LoadError)
	from <internal:/usr/local/Cellar/ruby/3.0.0_1/lib/ruby/3.0.0/rubygems/core_ext/kernel_require.rb>:85:in `require'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/commands/serve/servlet.rb:3:in `<top (required)>'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/commands/serve.rb:179:in `require_relative'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/commands/serve.rb:179:in `setup'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/commands/serve.rb:100:in `process'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/command.rb:91:in `block in process_with_graceful_fail'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/command.rb:91:in `each'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/command.rb:91:in `process_with_graceful_fail'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/lib/jekyll/commands/serve.rb:86:in `block (2 levels) in init_with_program'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/mercenary-0.4.0/lib/mercenary/command.rb:221:in `block in execute'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/mercenary-0.4.0/lib/mercenary/command.rb:221:in `each'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/mercenary-0.4.0/lib/mercenary/command.rb:221:in `execute'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/mercenary-0.4.0/lib/mercenary/program.rb:44:in `go'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/mercenary-0.4.0/lib/mercenary.rb:21:in `program'
	from /Users/yi.zhu/.gem/ruby/3.0.0/gems/jekyll-4.2.0/exe/jekyll:15:in `<top (required)>'
	from /usr/local/bin/jekyll:23:in `load'
	from /usr/local/bin/jekyll:23:in `<main>'
```

This issue is solved in https://github.com/jekyll/jekyll/issues/8523

### Why are the changes needed?
Fix build issue

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Not need

Closes #31263 from AngersZhuuuu/SPARK-34181.

Lead-authored-by: Angerszhuuuu <angers.zhu@gmail.com>
Co-authored-by: AngersZhuuuu <angers.zhu@gmail.com>
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
(cherry picked from commit faa4f0c2bd6aae82c79067cb255d5708aa632078)
Signed-off-by: HyukjinKwon <gurwls223@apache.org>
---
 docs/README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/README.md b/docs/README.md
index 22039871cf63d..a1c569ed8712c 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -39,6 +39,12 @@ installed. Also install the following libraries:
 $ sudo gem install jekyll jekyll-redirect-from rouge
 ```
 
+If your ruby version is 3.0 or higher, you should also install `webrick`.
+
+```sh
+$ sudo gem install jekyll jekyll-redirect-from webrick
+```
+
 Note: If you are on a system with both Ruby 1.9 and Ruby 2.0 you may need to replace gem with gem2.0.
 
 ### R Documentation

From c59a423f73e7e63a9f66468c6972f63cd2890fe3 Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 22 Jan 2021 04:28:22 +0000
Subject: [PATCH 1279/1280] [SPARK-33813][SQL] Fix the issue that JDBC source
 can't treat MS SQL Server's spatial types

### What changes were proposed in this pull request?

This PR fixes the issue that reading tables which contain spatial datatypes from MS SQL Server fails.
MS SQL server supports two non-standard spatial JDBC types, `geometry` and  `geography` but Spark SQL can't treat them

```
java.sql.SQLException: Unrecognized SQL type -157
 at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.getCatalystType(JdbcUtils.scala:251)
 at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$getSchema$1(JdbcUtils.scala:321)
 at scala.Option.getOrElse(Option.scala:189)
 at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.getSchema(JdbcUtils.scala:321)
 at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:63)
 at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation$.getSchema(JDBCRelation.scala:226)
 at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:35)
 at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:364)
 at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:366)
 at org.apache.spark.sql.DataFrameReader.$anonfun$load$2(DataFrameReader.scala:355)
 at scala.Option.getOrElse(Option.scala:189)
 at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:355)
 at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:240)
 at org.apache.spark.sql.DataFrameReader.jdbc(DataFrameReader.scala:381)
```

Considering the [data type mapping](https://docs.microsoft.com/ja-jp/sql/connect/jdbc/using-basic-data-types?view=sql-server-ver15) says, I think those spatial types can be mapped to Catalyst's `BinaryType`.

### Why are the changes needed?

To provide better support.

### Does this PR introduce _any_ user-facing change?

Yes. MS SQL Server users can use `geometry` and `geography` types in datasource tables.

### How was this patch tested?

New test case added to `MsSqlServerIntegrationSuite`.

Closes #31283 from sarutak/mssql-spatial-types.

Authored-by: Kousuke Saruta <sarutak@oss.nttdata.com>
Signed-off-by: Wenchen Fan <wenchen@databricks.com>
---
 .../jdbc/MsSqlServerIntegrationSuite.scala    | 121 ++++++++++++++++++
 .../spark/sql/jdbc/MsSqlServerDialect.scala   |   8 ++
 2 files changed, 129 insertions(+)

diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
index 5738307095933..7076e18c404f5 100644
--- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
+++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/MsSqlServerIntegrationSuite.scala
@@ -100,6 +100,37 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
         |'the', 'lazy',
         |'dog')
       """.stripMargin).executeUpdate()
+    conn.prepareStatement(
+      """
+        |CREATE TABLE spatials (
+        |point geometry,
+        |line geometry,
+        |circle geometry,
+        |curve geography,
+        |polygon geometry,
+        |curve_polygon geography,
+        |multi_point geometry,
+        |multi_line geometry,
+        |multi_polygon geometry,
+        |geometry_collection geometry)
+      """.stripMargin).executeUpdate()
+    conn.prepareStatement(
+      """
+        |INSERT INTO spatials VALUES (
+        |'POINT(3 4 7 2.5)',
+        |'LINESTRING(1 0, 0 1, -1 0)',
+        |'CIRCULARSTRING(
+        |  -122.358 47.653, -122.348 47.649, -122.348 47.658, -122.358 47.658, -122.358 47.653)',
+        |'COMPOUNDCURVE(
+        |  CIRCULARSTRING(-122.358 47.653, -122.348 47.649,
+        |    -122.348 47.658, -122.358 47.658, -122.358 47.653))',
+        |'POLYGON((-20 -20, -20 20, 20 20, 20 -20, -20 -20), (10 0, 0 10, 0 -10, 10 0))',
+        |'CURVEPOLYGON((-122.3 47, 122.3 47, 125.7 49, 121 38, -122.3 47))',
+        |'MULTIPOINT((2 3), (7 8 9.5))',
+        |'MULTILINESTRING((0 2, 1 1), (1 0, 1 1))',
+        |'MULTIPOLYGON(((2 2, 2 -2, -2 -2, -2 2, 2 2)),((1 1, 3 1, 3 3, 1 3, 1 1)))',
+        |'GEOMETRYCOLLECTION(LINESTRING(1 1, 3 5),POLYGON((-1 -1, -1 -5, -5 -5, -5 -1, -1 -1)))')
+      """.stripMargin).executeUpdate()
   }
 
   test("Basic test") {
@@ -225,4 +256,94 @@ class MsSqlServerIntegrationSuite extends DockerJDBCIntegrationSuite {
     df2.write.jdbc(jdbcUrl, "datescopy", new Properties)
     df3.write.jdbc(jdbcUrl, "stringscopy", new Properties)
   }
+
+  test("SPARK-33813: MsSqlServerDialect should support spatial types") {
+    val df = spark.read.jdbc(jdbcUrl, "spatials", new Properties)
+    val rows = df.collect()
+    assert(rows.length == 1)
+    val row = rows(0)
+    val types = row.toSeq.map(x => x.getClass.toString)
+    assert(types.length == 10)
+    assert(types(0) == "class [B")
+    assert(row.getAs[Array[Byte]](0) ===
+      Array(0, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0,
+        16, 64, 0, 0, 0, 0, 0, 0, 28, 64, 0, 0, 0, 0, 0, 0, 4, 64))
+    assert(types(1) == "class [B")
+    assert(row.getAs[Array[Byte]](1) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        -16, 63, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
+        0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 2))
+    assert(types(2) == "class [B")
+    assert(row.getAs[Array[Byte]](2) ===
+      Array[Byte](0, 0, 0, 0, 2, 4, 5, 0, 0, 0, -12, -3, -44, 120, -23, -106,
+        94, -64, -35, 36, 6, -127, -107, -45, 71, 64, -125, -64, -54, -95, 69,
+        -106, 94, -64, 80, -115, -105, 110, 18, -45, 71, 64, -125, -64, -54,
+        -95, 69, -106, 94, -64, 78, 98, 16, 88, 57, -44, 71, 64, -12, -3, -44,
+        120, -23, -106, 94, -64, 78, 98, 16, 88, 57, -44, 71, 64, -12, -3, -44,
+        120, -23, -106, 94, -64, -35, 36, 6, -127, -107, -45, 71, 64, 1, 0, 0,
+        0, 2, 0, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 8))
+    assert(types(3) == "class [B")
+    assert(row.getAs[Array[Byte]](3) ===
+      Array[Byte](-26, 16, 0, 0, 2, 4, 5, 0, 0, 0, -35, 36, 6, -127, -107, -45,
+        71, 64, -12, -3, -44, 120, -23, -106, 94, -64, 80, -115, -105, 110, 18,
+        -45, 71, 64, -125, -64, -54, -95, 69, -106, 94, -64, 78, 98, 16, 88, 57,
+        -44, 71, 64, -125, -64, -54, -95, 69, -106, 94, -64, 78, 98, 16, 88, 57,
+        -44, 71, 64, -12, -3, -44, 120, -23, -106, 94, -64, -35, 36, 6, -127, -107,
+        -45, 71, 64, -12, -3, -44, 120, -23, -106, 94, -64, 1, 0, 0, 0, 3, 0, 0,
+        0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 9, 2, 0, 0, 0, 3, 1))
+    assert(types(5) == "class [B")
+    assert(row.getAs[Array[Byte]](4) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0,
+        0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, 64,
+        0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 52, 64, 0, 0, 0, 0, 0, 0, 52,
+        64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0, 0, 52, -64, 0, 0, 0, 0, 0,
+        0, 52, -64, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 36, -64, 0, 0, 0, 0, 0, 0, 36, 64, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0,
+        0, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 3))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](5) ===
+      Array[Byte](-26, 16, 0, 0, 2, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, -128, 71, 64, 51,
+        51, 51, 51, 51, -109, 94, -64, 0, 0, 0, 0, 0, -128, 71, 64, 51, 51, 51, 51,
+        51, -109, 94, 64, 0, 0, 0, 0, 0, -128, 72, 64, -51, -52, -52, -52, -52, 108,
+        95, 64, 0, 0, 0, 0, 0, 0, 67, 64, 0, 0, 0, 0, 0, 64, 94, 64, 0, 0, 0, 0, 0,
+        -128, 71, 64, 51, 51, 51, 51, 51, -109, 94, -64, 1, 0, 0, 0, 1, 0, 0, 0, 0,
+        1, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 10))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](6) ===
+      Array[Byte](0, 0, 0, 0, 1, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0,
+        0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 28, 64, 0, 0, 0, 0, 0, 0, 32, 64, 0, 0, 0, 0,
+        0, 0, -8, -1, 0, 0, 0, 0, 0, 0, 35, 64, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0,
+        0, 0, 3, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+        0, 0, 0, 0, 1, 0, 0, 0, 1))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](7) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 64, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0,
+        0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0,
+        0, 0, 0, 0, -16, 63, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 3, 0, 0, 0,
+        -1, -1, -1, -1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 2))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](8) ===
+      Array[Byte](0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0,
+        0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0, 0,
+        0, 0, 0, 0, -64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0, 0, 0, 0, 0, 0, -64, 0, 0,
+        0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0,
+        0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0,
+        0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 8, 64, 0,
+        0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, -16, 63,
+        0, 0, 0, 0, 0, 0, -16, 63, 2, 0, 0, 0, 2, 0, 0, 0, 0, 2, 5, 0, 0, 0, 3, 0,
+        0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 3))
+    assert(types(6) === "class [B")
+    assert(row.getAs[Array[Byte]](9) ===
+      Array[Byte](0, 0, 0, 0, 1, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0,
+        0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 8, 64, 0, 0, 0, 0, 0, 0, 20, 64, 0, 0,
+        0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0, -16, -65,
+        0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, 20,
+        -64, 0, 0, 0, 0, 0, 0, 20, -64, 0, 0, 0, 0, 0, 0, -16, -65, 0, 0, 0, 0, 0, 0,
+        -16, -65, 0, 0, 0, 0, 0, 0, -16, -65, 2, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 0, 0,
+        0, 3, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0,
+        0, 0, 0, 1, 0, 0, 0, 3))
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
index 72284b5996201..8ce57f7806b2c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
@@ -25,6 +25,13 @@ import org.apache.spark.sql.types._
 
 private object MsSqlServerDialect extends JdbcDialect {
 
+  // Special JDBC types in Microsoft SQL Server.
+  // https://github.com/microsoft/mssql-jdbc/blob/v8.2.2/src/main/java/microsoft/sql/Types.java
+  private object SpecificTypes {
+    val GEOMETRY = -157
+    val GEOGRAPHY = -158
+  }
+
   override def canHandle(url: String): Boolean =
     url.toLowerCase(Locale.ROOT).startsWith("jdbc:sqlserver")
 
@@ -40,6 +47,7 @@ private object MsSqlServerDialect extends JdbcDialect {
         sqlType match {
           case java.sql.Types.SMALLINT => Some(ShortType)
           case java.sql.Types.REAL => Some(FloatType)
+          case SpecificTypes.GEOMETRY | SpecificTypes.GEOGRAPHY => Some(BinaryType)
           case _ => None
         }
       }

From c36fe778fc9a04a0389f45f395a85db4a68d583c Mon Sep 17 00:00:00 2001
From: Kousuke Saruta <sarutak@oss.nttdata.com>
Date: Fri, 22 Jan 2021 14:22:10 +0900
Subject: [PATCH 1280/1280] Modify the URL to comply with the version of
 mssql-jdbc driver.

---
 .../scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
index 8ce57f7806b2c..72cdc68d56116 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.types._
 private object MsSqlServerDialect extends JdbcDialect {
 
   // Special JDBC types in Microsoft SQL Server.
-  // https://github.com/microsoft/mssql-jdbc/blob/v8.2.2/src/main/java/microsoft/sql/Types.java
+  // https://github.com/microsoft/mssql-jdbc/blob/v7.2.1/src/main/java/microsoft/sql/Types.java
   private object SpecificTypes {
     val GEOMETRY = -157
     val GEOGRAPHY = -158